From 9b88cdcf8fe9cdcfb77f527b0200886b83b82bb1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 6 Feb 2025 21:50:12 +0000 Subject: [PATCH 001/719] Updated results summary [skip ci] --- README.md | 89 +-- dbversion | 2 +- summary.csv | 59 -- summary.json | 1771 +------------------------------------------------- 4 files changed, 17 insertions(+), 1904 deletions(-) diff --git a/README.md b/README.md index 0c630a279..b0204f58a 100644 --- a/README.md +++ b/README.md @@ -1,82 +1,23 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-06 21:21:30,151 submission_checker1.py:3263 INFO] Results=59, NoResults=0, Power Results=0 -[2025-02-06 21:21:30,151 submission_checker1.py:3270 INFO] --- -[2025-02-06 21:21:30,151 submission_checker1.py:3271 INFO] Closed Results=2, Closed Power Results=0 +[2025-02-06 21:50:07,867 submission_checker1.py:3263 INFO] Results=0, NoResults=0, Power Results=0 +[2025-02-06 21:50:07,867 submission_checker1.py:3270 INFO] --- +[2025-02-06 21:50:07,867 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-06 21:21:30,151 submission_checker1.py:3276 INFO] Open Results=57, Open Power Results=0 +[2025-02-06 21:50:07,867 submission_checker1.py:3276 INFO] Open Results=0, Open Power Results=0 -[2025-02-06 21:21:30,151 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-06 21:50:07,867 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-06 21:21:30,151 submission_checker1.py:3286 INFO] --- -[2025-02-06 21:21:30,151 submission_checker1.py:3288 INFO] Systems=45, Power Systems=0 -[2025-02-06 21:21:30,151 submission_checker1.py:3292 INFO] Closed Systems=1, Closed Power Systems=0 -[2025-02-06 21:21:30,151 submission_checker1.py:3297 INFO] Open Systems=45, Open Power Systems=0 -[2025-02-06 21:21:30,151 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-06 21:21:30,151 submission_checker1.py:3307 INFO] --- -[2025-02-06 21:21:30,152 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-06 21:21:31,321 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-06 21:50:07,867 submission_checker1.py:3286 INFO] --- +[2025-02-06 21:50:07,867 submission_checker1.py:3288 INFO] Systems=0, Power Systems=0 +[2025-02-06 21:50:07,867 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-06 21:50:07,867 submission_checker1.py:3297 INFO] Open Systems=0, Open Power Systems=0 +[2025-02-06 21:50:07,867 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-06 21:50:07,867 submission_checker1.py:3307 INFO] --- +[2025-02-06 21:50:07,867 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-06 21:50:08,679 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:----------------------|:-----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|--------------:|:-----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|:----------------------------------|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | Community | available | open | edge | default | default-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73704 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0102599 | ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.406358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.596 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.264988 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.327316 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412266 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3387 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.440048 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.336027 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.277839 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.329647 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.14067 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.357886 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.349311 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1296 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352003 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74836 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.8155 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 42.4133 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-default_config | gptj-99 | gptj-99 | Offline | 50.3446 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.4. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.412747 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.0188 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0425 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0356089 | TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch-v2.5.1-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387999 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.08903 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.323181 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 20.9527 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.350957 | nan | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | (darwin-23.6.0) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.43471 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 75016.1 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 32 | MLCommons | available | open | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80229.7 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.4. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | -| 33 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 42.3041 | GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 34 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352784 | nan | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 35 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.3279 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 36 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config | retinanet | retinanet | Offline | 0.409818 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.3. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 37 | MLCommons | available | open | edge | gh_windows | gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 17.8036 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v2.4.0. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 38 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch-v2.5.0-cu124 | gptj-99 | gptj-99 | Offline | 47.0771 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 39 | MLCommons | available | open | edge | gh_windows-latest | gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config | retinanet | retinanet | Offline | 0.313782 | nan | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline | pytorch v2.5.1 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 40 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409259 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 41 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0454 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 42 | MLCommons | available | open | datacenter | 3b07702db56d | 3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.374837 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 43 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43494 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 44 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.009 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 45 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config | retinanet | retinanet | Offline | 0.412839 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline | pytorch v2.5.0 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 46 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | retinanet | retinanet | Offline | 0.433265 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 47 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config | resnet50 | resnet | Offline | 23.227 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline | onnxruntime v1.19.2 | Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35) | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 48 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431711 | nan | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 49 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0164 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 50 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | gptj-99 | gptj-99 | Offline | 48.3215 | GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 51 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.5.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352694 | nan | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline | pytorch v2.5.1 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 52 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.5168 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Windows-2022Server-10.0.20348-SP0 | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 53 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31453 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 54 | MLCommons | available | open | edge | gh_windows-latest_x86 | gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config | resnet50 | resnet | Offline | 21.7501 | acc: 76.000 | 1 | undefined | 1 | 1 | nan | 0 | open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline | tf v2.17.0 | Windows-2022Server-10.0.20348-SP0 | Automated by MLCommons CM v3.2.6. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 55 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | gptj-99 | gptj-99 | Offline | 46.0817 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 56 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.4.1-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.346565 | CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline | pytorch v2.4.1 | Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35) | Automated by MLCommons CM v2.3.9. | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 57 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Server | 73015.3 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Queries/s | int8 | -| 58 | MLCommons | available | closed | datacenter | gh_ubuntu_x86 | gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config | resnet50 | resnet | Offline | 80458.4 | acc: 76.078 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline | TensorRT | Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31) | Automated by MLCommons CM v3.2.5. | 1 | 0 | v5.0 | 0 | False | Samples/s | int8 | \ No newline at end of file +| Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|----------------|----------------|------------|--------------|--------------|------------|---------|---------------|------------|----------|------------|-------------------|-----------------------------|----------------------------|-----------------------------|--------------------------|-------------------------|------------|-------------|--------------------|---------|--------------|----------|-----------|------------|-------------|---------|---------------------| \ No newline at end of file diff --git a/dbversion b/dbversion index 01d65e21e..f0751103c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -628 +629 diff --git a/summary.csv b/summary.csv index 9a6e83b29..b85f9e25d 100644 --- a/summary.csv +++ b/summary.csv @@ -1,60 +1 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -Community,available,open,"edge","default",default-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73704,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0102599,ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.406358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.596,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.264988,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.327316,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412266,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.3387,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.440048,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.336027,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.277839,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.329647,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.14067,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.357886,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.349311,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.1296,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352003,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74836,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.8155,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.0-cu124,gptj-99,gptj-99,Offline,42.4133,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-default_config,gptj-99,gptj-99,Offline,50.3446,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.4. ",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.412747,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.0188,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0425,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0356089,TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch-v2.5.1-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387999,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.08903,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.323181,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,20.9527,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.350957,,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1"," (darwin-23.6.0)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.43471,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,75016.1,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Queries/s,"int8" -MLCommons,available,open,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80229.7,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.4. ",1,0,v5.0,0,False,Samples/s,"int8" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,gptj-99,gptj-99,Offline,42.3041,GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352784,,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.3279,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config,retinanet,retinanet,Offline,0.409818,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.3. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_windows",gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,17.8036,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v2.4.0. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch-v2.5.0-cu124,gptj-99,gptj-99,Offline,47.0771,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest",gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config,retinanet,retinanet,Offline,0.313782,,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline,"pytorch v2.5.1","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409259,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0454,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","3b07702db56d",3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base,stable-diffusion-xl,stable-diffusion-xl,Offline,0.374837,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43494,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.009,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config,retinanet,retinanet,Offline,0.412839,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline,"pytorch v2.5.0","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,retinanet,retinanet,Offline,0.433265,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config,resnet50,resnet,Offline,23.227,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline,"onnxruntime v1.19.2","Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431711,,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0164,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,gptj-99,gptj-99,Offline,48.3215,GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.5.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352694,,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline,"pytorch v2.5.1","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.5168,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Windows-2022Server-10.0.20348-SP0","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31453,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_windows-latest_x86",gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config,resnet50,resnet,Offline,21.7501,acc: 76.000,1,"undefined",1,1,"N/A","0",open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline,"tf v2.17.0","Windows-2022Server-10.0.20348-SP0","Automated by MLCommons CM v3.2.6. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,gptj-99,gptj-99,Offline,46.0817,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.4.1-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.346565,CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline,"pytorch v2.4.1","Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)","Automated by MLCommons CM v2.3.9. ",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,closed,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Server,73015.3,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.5. ",1,0,v5.0,0,False,Queries/s,"int8" -MLCommons,available,closed,"datacenter","gh_ubuntu_x86",gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config,resnet50,resnet,Offline,80458.4,acc: 76.078,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline,"TensorRT","Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)","Automated by MLCommons CM v3.2.5. ",1,0,v5.0,0,False,Samples/s,"int8" diff --git a/summary.json b/summary.json index 4f24c2064..0e0dcd235 100644 --- a/summary.json +++ b/summary.json @@ -1,1772 +1,3 @@ { - "0":{ - "Organization":"Community", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"default", - "Platform":"default-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":1.73704, - "Accuracy":"acc: 80.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/Community\/results\/default-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "1":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", - "Scenario":"Offline", - "Result":0.0102599, - "Accuracy":"ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "2":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", - "Scenario":"Offline", - "Result":0.406358, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "3":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest_x86", - "Platform":"gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":17.596, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v3.2.4. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "4":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.264988, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "5":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest", - "Platform":"gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.327316, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config\/retinanet\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v3.2.3. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "6":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.412266, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "7":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":17.3387, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "8":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.440048, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config\/retinanet\/offline", - "framework":"pytorch v2.5.1", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "9":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest", - "Platform":"gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.336027, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "10":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.277839, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.5.0", - "operating_system":" (darwin-23.6.0)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "11":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest", - "Platform":"gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.329647, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.5.0", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "12":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", - "Scenario":"Offline", - "Result":8.14067, - "Accuracy":"F1: 80.0", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "13":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.5.1-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", - "Scenario":"Offline", - "Result":0.357886, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.5.1-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "14":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest", - "Platform":"gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.349311, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "15":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":49.1296, - "Accuracy":"GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "16":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", - "Scenario":"Offline", - "Result":0.352003, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "17":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":1.74836, - "Accuracy":"acc: 80.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "18":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows", - "Platform":"gh_windows-reference-cpu-tf_v2.17.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":21.8155, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-reference-cpu-tf_v2.17.0-default_config\/resnet50\/offline", - "framework":"tf v2.17.0", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v2.4.0. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "19":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.5.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":42.4133, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.5.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.5.0", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "20":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.4.1-default_config", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":50.3446, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.4.1-default_config\/gptj-99\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v2.3.4. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "21":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.412747, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config\/retinanet\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "22":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":21.0188, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config\/resnet50\/offline", - "framework":"tf v2.17.0", - "operating_system":"Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "23":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":21.0425, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "24":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch-v2.5.1-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", - "Scenario":"Offline", - "Result":0.0356089, - "Accuracy":"TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch-v2.5.1-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "25":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch-v2.5.1-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", - "Scenario":"Offline", - "Result":0.387999, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch-v2.5.1-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "26":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":8.08903, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "27":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.323181, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":" (darwin-23.6.0)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "28":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":20.9527, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":" (darwin-23.6.0)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "29":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.350957, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config\/retinanet\/offline", - "framework":"pytorch v2.4.1", - "operating_system":" (darwin-23.6.0)", - "notes":"Automated by MLCommons CM v3.2.3. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "30":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.43471, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "31":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu_x86", - "Platform":"gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Server", - "Result":75016.1, - "Accuracy":"acc: 76.078", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config\/resnet50\/server", - "framework":"TensorRT", - "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "notes":"Automated by MLCommons CM v3.2.4. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Queries\/s", - "weight_data_types":"int8" - }, - "32":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu_x86", - "Platform":"gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":80229.7, - "Accuracy":"acc: 76.078", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config\/resnet50\/offline", - "framework":"TensorRT", - "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "notes":"Automated by MLCommons CM v3.2.4. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"int8" - }, - "33":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch-v2.5.1-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":42.3041, - "Accuracy":"GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch-v2.5.1-cu124\/gptj-99\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "34":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch-v2.5.1-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", - "Scenario":"Offline", - "Result":0.352784, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch-v2.5.1-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "35":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest_x86", - "Platform":"gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":17.3279, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "36":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.409818, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config\/retinanet\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "notes":"Automated by MLCommons CM v3.2.3. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "37":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows", - "Platform":"gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":17.8036, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v2.4.0. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "38":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch-v2.5.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":47.0771, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch-v2.5.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.5.0", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "39":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest", - "Platform":"gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.313782, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config\/retinanet\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "40":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.409259, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "41":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", - "Scenario":"Offline", - "Result":10.0454, - "Accuracy":"acc: 74.800", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "42":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"3b07702db56d", - "Platform":"3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", - "Scenario":"Offline", - "Result":0.374837, - "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v2.3.9. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "43":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.43494, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "44":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":23.009, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "45":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.412839, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.5.0", - "operating_system":"Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "46":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.433265, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":"Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "47":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":23.227, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.19.2", - "operating_system":"Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "48":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.431711, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "49":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":23.0164, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "50":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.5.1-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":48.3215, - "Accuracy":"GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":16, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.5.1-cu124\/gptj-99\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "51":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.5.1-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", - "Scenario":"Offline", - "Result":0.352694, - "Accuracy":null, - "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":16, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.5.1-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.5.1", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "52":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest_x86", - "Platform":"gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":21.5168, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "53":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", - "Scenario":"Offline", - "Result":6.31453, - "Accuracy":"F1: 80.0", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "54":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_windows-latest_x86", - "Platform":"gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":21.7501, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":1, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config\/resnet50\/offline", - "framework":"tf v2.17.0", - "operating_system":"Windows-2022Server-10.0.20348-SP0", - "notes":"Automated by MLCommons CM v3.2.6. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "55":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.4.1-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":46.0817, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.4.1-cu124\/gptj-99\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v2.3.9. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "56":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.4.1-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", - "Scenario":"Offline", - "Result":0.346565, - "Accuracy":"CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.4.1-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.4.1", - "operating_system":"Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "notes":"Automated by MLCommons CM v2.3.9. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "57":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"closed", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu_x86", - "Platform":"gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Server", - "Result":73015.3, - "Accuracy":"acc: 76.078", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"closed\/MLCommons\/results\/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config\/resnet50\/server", - "framework":"TensorRT", - "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "notes":"Automated by MLCommons CM v3.2.5. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Queries\/s", - "weight_data_types":"int8" - }, - "58":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"closed", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu_x86", - "Platform":"gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":80458.4, - "Accuracy":"acc: 76.078", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"closed\/MLCommons\/results\/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config\/resnet50\/offline", - "framework":"TensorRT", - "operating_system":"Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "notes":"Automated by MLCommons CM v3.2.5. ", - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"int8" - } + } \ No newline at end of file From 966504930803b5ba1b22b34715e3d31716559a51 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 7 Feb 2025 05:43:41 +0000 Subject: [PATCH 002/719] Updated results summary [skip ci] --- README.md | 31 +- dbversion | 2 +- summary.csv | 1 + summary.json | 31 +- summary.xlsx | Bin 14569 -> 6365 bytes summary_results.json | 2090 +----------------------------------------- 6 files changed, 49 insertions(+), 2106 deletions(-) diff --git a/README.md b/README.md index b0204f58a..306cc8b50 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,24 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-06 21:50:07,867 submission_checker1.py:3263 INFO] Results=0, NoResults=0, Power Results=0 -[2025-02-06 21:50:07,867 submission_checker1.py:3270 INFO] --- -[2025-02-06 21:50:07,867 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-07 05:43:36,988 submission_checker1.py:3263 INFO] Results=1, NoResults=0, Power Results=0 +[2025-02-07 05:43:36,988 submission_checker1.py:3270 INFO] --- +[2025-02-07 05:43:36,988 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-06 21:50:07,867 submission_checker1.py:3276 INFO] Open Results=0, Open Power Results=0 +[2025-02-07 05:43:36,988 submission_checker1.py:3276 INFO] Open Results=1, Open Power Results=0 -[2025-02-06 21:50:07,867 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-07 05:43:36,988 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-06 21:50:07,867 submission_checker1.py:3286 INFO] --- -[2025-02-06 21:50:07,867 submission_checker1.py:3288 INFO] Systems=0, Power Systems=0 -[2025-02-06 21:50:07,867 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-06 21:50:07,867 submission_checker1.py:3297 INFO] Open Systems=0, Open Power Systems=0 -[2025-02-06 21:50:07,867 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-06 21:50:07,867 submission_checker1.py:3307 INFO] --- -[2025-02-06 21:50:07,867 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-06 21:50:08,679 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-07 05:43:36,988 submission_checker1.py:3286 INFO] --- +[2025-02-07 05:43:36,988 submission_checker1.py:3288 INFO] Systems=1, Power Systems=0 +[2025-02-07 05:43:36,988 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-07 05:43:36,988 submission_checker1.py:3297 INFO] Open Systems=1, Open Power Systems=0 +[2025-02-07 05:43:36,988 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-07 05:43:36,988 submission_checker1.py:3307 INFO] --- +[2025-02-07 05:43:36,988 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-07 05:43:37,891 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|----------------|----------------|------------|--------------|--------------|------------|---------|---------------|------------|----------|------------|-------------------|-----------------------------|----------------------------|-----------------------------|--------------------------|-------------------------|------------|-------------|--------------------|---------|--------------|----------|-----------|------------|-------------|---------|---------------------| \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:-------------|:------------------------------------------------------|:--------------|:--------------|:-----------|---------:|:----------------------------------------------------------------------------|------------------:|:----------------------------|---------------------------:|----------------------------:|-------------------------:|------------------------:|:---------------------------------------------------------------------------------------------------|:---------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:---------|:--------------------| +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index f0751103c..1217bfdb2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -629 +630 diff --git a/summary.csv b/summary.csv index b85f9e25d..fded8e476 100644 --- a/summary.csv +++ b/summary.csv @@ -1 +1,2 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 0e0dcd235..b15840692 100644 --- a/summary.json +++ b/summary.json @@ -1,3 +1,32 @@ { - + "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", + "Scenario":"Offline", + "Result":0.40413, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + } } \ No newline at end of file diff --git a/summary.xlsx b/summary.xlsx index 8e08ce4ea2dcbf3268e8641c843414eaf8d98a4a..dfa626c91c2b90eb05bf16b79f9b3efe2cf5b091 100644 GIT binary patch delta 3560 zcmZ`+2T)Vn7EOo(n15At{$GAWs5Iyx#2RUL4 zRE)tOP|`6Fh?DxpR@>Xl*Tu`%-XZ|+Lc~b>dwP_nb$Pv&Wz(e`#I=t}$@!XIhK1=0 zb#WJMK&}R9A>m^`n79rbNoyvyrjNoemS9)LiEW9hyr%{3pBBz#Zsa<6EOBN{g?F(a z&p$_X4)7DzHk**-QB8BAl|29MSu+*}miOASip!9kFp7PB`@AE&sQ+^d5rw#Ig^Mm0 zf%*Bu-hmCaMG+EfY37R}wnQ+&3Vu;9-@)JBD%E!f2Uv++{4m2MYBDE%{DR;o4i4)PmvikyBF#rrV`{iMHfH|W#1f$+^r z0M%5fOta7Aw#EK->EpX<_ddx*<4HfKw9kaY0riNb{iG8Cr5q0w4R0}D-Jfj=@+P*g zoTK9l0sm?Ni3P=9SpcR1v*_HgUHeWg{VN&}h#y4qM8Sd5)Mnf}SvH;3186zL<|cPVrmE&Os~{|1zpO}TXm!NBM}GFTzG$?++2n(nJ84)rOGX@9-&~IPQ%&N% zqPwP<4PJn$bmtF)e7EBRmg!FJu&1S^8cVG)M;$B|`}uA$e{v}6=@103|txW8dG8mUbSS?5#)F@)LosyUwsF4U%Wdn zwx&)uhslQDT1U)O3?Xs2$0CNdHD7Z2vg?p8+M6z~ns!z`m~xdH3o-LC?RNc6o2GF$ zCe3n@?_jU%>tq^7p7q@dEeqKvwk8{&_Xjr&+E=CMJ{*Hs;La0E7h_kSos3RJ7y|<) zQ-`s&{S(Z?a(C5M zTrvkEP9m@+hz%`O4A_2-KO&x43}BDKy?8q7>ZxPUk$v4a2C6#vis1qwJKvoG%rhqxncnhj*~-9aea@oVDo;DKt)mA~?}+)IZb`Lprj&*krz;oYIkF!c z#ij?J?H5-B6w619*~X*gi1I?04iXsR)#EogWC?Wff%}tDUl1GYf7QJGPzuN{t!o?Q zCh^w6&U;0fCJ9+XgLRg!@Sv-RZEm=yzsA$MJ6NHAlt9LN;@R1ECG$ zy{ZGA_15g#8}wG1D;_<;InG&cuIpJWYWi<0E_;M*E>Ai61O)@c@hcajT!`H_CQ9mc z(n72!tH`M_Px!IbvzL!^FrtwPk_{H!#j~9Q`c>oc4R3}y9+d*EjfmHWpO-%+%4j;UxSCB0}fw>2z2ZW0(?I%n^CNgX~3n|U%TnGF|P z?{MeHO6F_cpqN}JzwDL?*0cHP);)x8o6WF{<4T@_@c4U6reG)^6RQfx*h7M@XgNDq zGGrD zE%(oIKlpi-@p*hJyyv?EX1pZOIVqG+g_a(S_T_*y+xHve?)_YWztL+N~ zQL>a5tp70b9#ZnD)|>p|akB1$z?mKfE276W^vhd&>880ID_fC2(8EW{ZCu8y%z6?g z83uio$kSa?35J z0yYf(y22bNn2V0tVz21eh^Th@Hn9melpbV!s202Y+!1IEB``;DIzC+Dz23^I;tokL zx_&d#u9r~^;d;7ZzwL#cu3=b&Xse%i_D{p1ZwZp#H|5>-#Zmml#ylQI)o~`qll1YI zSHJR&!li@ar)3jo$X{V{`S*R+@5}3JS-C%7>Q<+D%AYs2NF@Mo?7>~ z+y8DW?K}p^d>NiG5;1Cqz*V;$shuQXxT9@h!!tRzZ42FNtLC$EVJ)loDBodEYxf&n z9Cwq|8v{=@LbWV4CxX(wL*hw-3*x5S1q@n%#;A4{va`9;Yg&+rr7xY6nfZjP zPMiA)!CCnmMgX7vM z|;^QQ1FA*qzTY@_gLXI!;MAtgE~DEP$E`w<#Pfi z@2~geKB+}rDrYkq2AsOK72Ux+k*W@~VGP{76A*JWM;|+4JLaYOa{*Ae^LLFf7Qs2| z#NG+erJYRX2*Q-bnI+`-%WQIA!zSVER&dVdSBHchV!cg$BkQw#vsL33u)^KRG_*r= zZT{Q)x~f;(Zds8zUx>vNU3?Q-$~hB83^oi7Uj`KSaxYanNU}x;YK_D+tu4hTDeH*Ehei@`P6!Op$+L47EbiZ z6d29j3da?cVTxxNt<5)|c9Kdyx6=LjLMIIR4yCWLn%AoI&NS&Qr*mPMjW6j@y`)#* zp?#m!XtR*oi5|?5{I1b(z;z((r*Vc*PriQnwe?MGpSyz40`jHaUIskpV(wu5KoFx~ zjrGXDBtuo8`**rXk;)UbI_D|%^L>O1;p3|hM{VNlHIzmWF037z4gtBCIQG3nrS!Q6 zc7>YQk2qW+z9-9>(pPd`>v)Zh0$<1W&~$UT-mdkk@VYwu#kWK~Rs!&?oaT7uOzTN* zC#z2MvthlXz-t0yELuHg7r)mr%503Bm{yauBQic4@83S5VIS#9WJL{{A5hY9C1n~cGf7iP?9|@~rhc~xH6QzLU+Nherc1qq p{WFoy-UJsCyCN6-zcc$gBs468i`1;d&jJNOL7~)SdfdNE{{!i6EVuvw delta 11802 zcmZ{~1y~#1);63FT!XtdxKpe^@uJ1uy|}wIDNtO47A@{B#ickDDHLtdQi{7%;7_0P zzTdgd`M)o@vL`dwo_p5Zd)drdE4;hzP<3SxG7$g(Ku7!}0CVSp5Bfj=AP)rqAVeH7 zl5utMvT^Y;)Ao0>@igJ`b9S0Y8F%^21AcY=5IyppjaR+Af(xe1=+g4PY2q00aJ8p$!LU+ta-*agejSCi!JK28kZptF7Rxaj+AErj=e=b8H*^J zvlcIh;@l;uM}ElPvd91QoeqfIu$50dUzLq<9G4binKM;=_exPOjKfEef}?qOpT#_g zJ-_Avwd&6F=O}VD9eI=_H?$y{ttOia<#&>NaLbyF@@qXJL+OhQoqEPlkwm1_J3EYb z{7!twHp-;yUN?8Kdq>Zru=rs6=QC;OwEpCN4ECAY77|2*<#D~1 z$j&kiSPYn8%Rj{bBNy9gFqwB}n;7>L^&;R2GCV>1t5*|Q-IZvw*$?|vVx!|)1kLqm zp^vl?2BZUG6194bTqC%j1tnJ?(LuYCCmi~Dz+uP1nDy8CEwkb348Juo+gf%(=#CFE zPKtJ)B;MXfhn3(WCx?zIeu>Pvm0X^QxU`FZ)9$2PW zc@cw&3Rxt<%Mv=^A0~G&RAUI&*PQ0Lw`DDNs#{1AP7qwx=DNr%#BRQ57lZCl*yNV5 zlMj)r8(T-`Xn=i~K=rg@tMJ2$3eMBP)V!C5-J0}0Cc^5WBfQ!X6b5}u4E|ybkdly2 z+~hTKro0e?Rq{74P|k&!v4;)*puHGNesL?J=E(=fGx@V;8=0J2VK!m7wgcPFk@?Zj zJGl55nnc>6{|D)R6chYEn%UU5|2`K40Bk0Yg6N=WMxM*UM8j*>T6X;Y3U4sG0k{^u zIcX4;Ov=$WpBmL->k^z*oX1nGrMrU}DOpDLCc8QDhTw017xXygJ%4Cr^=LxnVqv#k z-0vWhBCpR#+r@sm3|OJGfdp>J_PBry&XgTjkH#a3Ls8IcQrl<>*#!U*-$ABfRE=AWL2b}7v zv@L(zT{IOk;1rIgr}ySETl|hQ&7V9rs*nYhKzTCx3*HIhY!Pts3=62K;JjI3rJ~g6ipp+E(-v*U8$N>B7&C+8XJ?+pkRq zguVsWA^s(h45(j^hF~A4DVakxp&>OOnfob{f$6s{QrL&woRRqtsVKZnJuP(MS7ttu zzYcqLGx1_Q+a5^IYe_soZKC`_xcjS?E_8T^iyF~DUnzXff|BTQUw=yQEmc7wt_Um9 zesfttTmPBW!<$pvU)z`X#9DD2xs~m#E`@GO^IK&IrV00;$IHEr_$`16)W)YE8zV9& z;{J9XgqE+A<2K`DP`jr&KRzM6lj=yJHKSbFeqF)F{gb=4%UQ=*uIja2w8T0YB)_fK zR*Sh_`(5ditxUx%FxB|{I_nFG^BdFqD4N#QGS{>L8OsA)`Ja#LmQT-3Wq*GZB<g6wIW&dN=5@iE`S>E-s!e z#G$eNfiO@tW1xz?n~&@5?!&CV>I`jApFYzTuH`B*sX>dAGJ!8Chlk&Dg9hQApUn1K z*0;l6T&Ff;RJHuk(u%1zGSS&|BDsPYe;fz2upc*FGQ*~u+~zDNDS zX~-Ybq=ZDi9`;<8!$;p&gvF+%2Z022wvR`KESet}M-w*tQx}MTNwd)(`jZctu=#!Q z)wK{*&E+=WF|+nxy7)YxB5nFeTtHV_%!ZkCK0_>^;fLGlq#;Z~^6qzi!E^z1y~Zz1 zcCFHAY+1^tg3QtAm2S(tkCy#tcm?UyPXTL3%JRMD*pn}#*`~}Yx$m0}-f`z_jFilkA&qvTWo%#xQbI96hUh7Bw#`FPCk{NUe`1h!eApl{N!$C?cGa}y`@XusYg z68+G!^V(>{6y<~EdknroOf2g=<-upVb|148tF4*!a=u*3%HtlaaH-Zy14G-@6nYnh zpFuvBnPFhi-DD>Mh`7)($lj-iA!FyWOCu>s&_ULLri29QEA(k)AKQh(pvc<3VT=@! zztR$6`bUxRz0dFD6!-%=$uIn#pIp<9T&E<$tnc&-LYV#J6nY8&C@DzF|JE-Ejpq_1 z*rYrq$a19nUP*Bdw7`Wd2yDwZx%Ae*QUItwV?d{CW9)=UuXYXYUUNNl-HXte#??rq zVKL^`rff^K=uc$25GZP7K%e5ybkyHATbE1VSmqVq8pf+UA8KSZn<#*hjx&e{&tx^E z6vw7w@s#V^Dhl;KE{ofUnXS4hW{sAxR<6Dl>XhqFspu5g9@nwD&15X^$Ph1|(ABbc zThK9z`_TX4#^6GK8=0O|!W)v8G1f!vkmk9WZAACg)4 z_(U0lE*PIquvWwvyv_%|xyO9^;5*RfIw0V&d6C_EDLV2^#Zn@JR&s^cLj=|?@0l(# z-sGHBGFedG$fId~^%6QrK+QE+hpZv5K<6EXtSLW$RNm-N*1sLuPI3MYr92dIEnore za%|*B$y1m50c7_z11P=s@W?J;k7*v)yP<@#p&3myYiM$4@!h9*Esn2!>U&lgpW_oH z5703N;v-QkZ5PZVCh&a(wn3DZXq4ftpfKtta(t=2(Rep3W~k`>)S$~<$0GH!QW+-E zK;MR0mGR|s!Wf3x#neNX=!v1blARY1;}r%DL&A*Y1q+UKLNwavPjl)Q__*M=-ngjW zvy1>5oIe+d ziNht05mzn=PVK=Tu!4Axhf@NH2-_V zv+brp1C0THq%67v+{iR>r<)X(wmbpbg zBi5k3r)0NOJo51i?{qwi%EzO>YHv+Q@Pr-t4y098_z=2SDjZKn%W~v2iet%ewJMrU zu{}9Edc)pSH8KA(AY80zIj($0|it_#p2`ot=wcZu6L0;2TP3WR_TDw6os)1I z|Kga>9HR}8jVjB-QNODVcyr`qUI2q6dIkSbqutueB776r%@=6Q2%w4Qvk`sur~ z8bSR+dq!&0+GD%XiL7*h-SOl7{#L`)i#+b3#$8hrfr0?eJX~97;ps(C?c>J7eom38 z5eM4r*L4rUiR-W$7c0#;()GAACk9U#@~}zbmUP59 z-+BmUsiBT<)}EhlZ#v}S$?dBqkC((3=z8*zsqG69n|_gT_%b;;)c39~y z`5%uuj*^~MGBeIS=zT|_`)T7`fW!G8r`ku&uAM)`Spl9<-Nc!d&1QD%9u}nZ8^;38>Fjd2GK;YUgyDh3YiB+*c^9T&q%!$#GKG znkKlW-#X`QPn7MGcT;en-FKWXr7s7){JiN<7hv-Zt2(Bi&$#)q&B1h{+W)&-(1JSq ze7#6O-t~mb`FX{O^QNqiKR2cLNO(eV_#585dqsP3Rvgbr!QXDN80>GNqFN$5nUnmU zpS_bc%t}fiyjg+K0YqoZZ0ZO;suqMO};l@JG*k1SZmT;Fb}b7Ykbr4m_-|RFZOqB z=hW?Gj(-fPMl|j&-E=(%)4HeWuy*!B*3&h34! z#J^lZ{bF*>Io_1=E8+1aD3*ou(f+_(PR=wb($Dijm4QHvj~q~Y2*kSw`Q;w1v>>Yq zT3d%i-JoqFZaqm)>cXG-{^GmDA+>*sx68fq*b#dkZf(tcQ8i1JnOgeba9XL1?jRhA zP84dNP8UNlzu2;w$VkL>;!5PRyZPBJ z<@T5ni?)tdiqVmz*8@FZc}ViuIP+OSRN+H>D*lJJn)D^VNlQaF_FSZQz5Gli11_Ck zWZ`l9i0{j3*xLG4H{tX)Mi(Qm-_o;g*nfw{l^uS5{>!iA#$6HH9fr?XzH5$p0>O`B zj#3cgMe-}55vvzn-`!nhTw%;DM^igkC0I!~Vz@4qKLD}7Nm+=|it}7VquIy?+0qC? zEDhx}gkD=yvs6Sjwz`n=L|+dBM_D*k%cu#Es0`%JM)(L6;6R4s=h%vxfQ$>`odzfp zd@K%$Kv@&e+Bx+jbTmAbv}U@}EOX>35X^3%42Yt2W%lv?>R`E|N=1u9+|%`_xp8 zuUr-DS=Rj#P2G8BTgpu=4o~@niKie(&gnl0{(B1hD4;e&$}}!Pj{^qn48Zg<0Ittz z(LF;+)bseE#rbX!oNI1CFWi@T$b2>G>&b4QJ^f)mWjappz%T;yV=LB)W)ky3J%sf6 z7?~nhJtoW;4)jn&9T~)iI|gqNTmnWeiFO*0BwgA7+4iHn@3T5MkbQj>(ZAZf2bgrM z?1=wDb!BwKV3>DV)oXy-zy+hcSvWBn-xo1P*)&9@4iJ0?MlzDqrCG!DztGj!C+Ke% zvP4ytN~dc}TE9f$(*(G3tk}@Fa9D5;B!A#P%XPJPF^c|8So6M8Um2Y1PCiv+(d1!* zDT+<^-c*r+m=3NXCn5<6^MhlrlhXl{us3%bVT;jVwLElLHaci-E)qE_%rAsxApE^r zMT4H%gILt|aHIa9VTgEoTHMFW;5FU7_TzoG5TZSnsFV-{kM<}@;Ll|1DD~LA3SVeo zUu{v7ko>&&Jwx?@=Jm#a8kc%WWo42+YKE>8ZADW~F&$ldB9!%|fU(qrvGUqo2qBp^T11~v&OR=CY(&;;yW7`&5O z=1H#=a(P!-@}vgXMwt6ugNfUfM8qSe;a@e1l(1F;phcAW2UT8}2^wry@0mh3oa#Ma z=!69#gWcR$5Ql`8U{*Pb50yH@???lFQ5nbXckMtV6NgG6@}h<t%%V{IMF-G8<;dE~n5UIep&ciQ-pcD(8<kgsFbW7i>Io$jpZb%9G4OVS8f z^S`h}=2|`ohW*kIdeLq<_q!CwU?>5&5(c@0&~ZLw%h-*Mu+Ysld>FyQe@P%M`0FItr2h9gde+mo;%aMu zDg7Zz{Y+scs|AdijYqo_#jVC&5L`pIT>=uOkJ>810n%@vh5?lG{`9larHS9lEB9~z zqXhhK4A3PE;3k8FiHQ!eGGUIkDg0lF0z=Ap}-fqbBSirkiM@?T5` zguxo4+DUSNj2m!5V9I%D18j8Z;`j>6@_!YJ57W7!XKD16Nu}1&EA2W37g^)}6 zzGuN;x0GOXmDMx3!^B|zy=P~vi--uoZbM5TjS8}+O#`Bih9yen9U?>w6G|ZL7+*0p?fcG6hREY z3~rcwz@ueBHi)1WtTW9wO7H^hr>3IrH#fmJf#)5LwW%DuvVj6NdZlwbV?*(^u^9~& zg)?Y;QP4AdMZJjkmb-h2U^w~uULAI#qR#`6FSr7i9bkR;ni^Ds+hl!E_fBqD`lK#2 z>_k&gc9oZSTr{EY&g5IW5(&B23!@g_=y8L?2Gcn2X{A4Kg=JBMlz|8-=K+BTDRmoW z@uv5Zef5DK=YAiNx%vKc%l0@+Un0Uzph%=%G+OnO=bqC)bn}MM^M=U|mnEhKaD|s# z^WoBXSc957^z>oRC3h^yVaZN=Ah)|98ePEp#jY5t7meNMq6MfPxBahKyYgY7Pi7I{ zi>Za3`!|>{#$b!Mrd8+8Ie#44z|BYlsaU4SYVKL&>npv4gxR4gyyO7c!EvC$0F}Ji zeoR27c=0f72E<7&>J-Uzuq4yIp-AwDTC2Y2+*A06Zp^Tgq1e3sf~k1PAs}gyBdN(x zpCye14}(4uNP;&Gh_rrA3;4=>T89KEnA+f7Kao_HLk0dyr8ZPIMMPfY>S67wOR?~6 z32#$&12b^y=l!S|*%#SJ$xz`sNQg3IkcI@_vF)XRUXKk9)JLs@bld>a#A^{t%Momt z0IWtcuJ4p zaYy!P^DVZ-2i6E~VjC7jj+jxu?Ug>`{7z4Ao@fIJ<#zq1!t6oEIB##!ou+B5kB(O*1;LuaTL41U_{hvp z>#%vq%%P|Hd4mZ0L&al;Lk9n$-ve#x8A-^#7C}fybyH&JcZW}tm-vV!iPoHZOHyJ~ zaZB|*?xnx>Gfl4ND-LMIvZNY+or3?$96fyzfA5(E`A5XqHl?nF@DOJ2j{e#O$~eL< z+CKd&7)C6(S zio#}KG5yNf;FrO}WX-qMBK`eDe-3xWloWscGDaI3gnI{B)1(2Rf=jw$y(LrI7Q~4v zd6QOBzY+c`o`F_b`V-A=I9*7G6}yjbF4E?CuA*+kXF9OZAhnfY(s)H9uKO?nNLJB3 z;{8_b>%&BF2pQc`9X4#lXA=kx-o~wj(cOKc#@a&|uQib$$LJfR1V)4+hUNG<4Q|Ji zPC)NoH^2u`G@z_k8(*CZsn*JxaBYVPYGoB|Bis>-{33najKL! z3`9w)sO?5~SkB5JSR2KTuo2MDYv(}*HI}rg0EofpW=I`TPn>co=kK53h(4<*se+=u zerZnolSD8g6FRBGHXps(0CEIl;#I=*@2F_7aPgr3$j5@w6;MRVfc`nYT7MBk{^W)2 zkaM&5K}oe$ihcU3To$^B6ScB6KBF@=3H;Ne2qX+0by0)^gx=unry2E8ZNY*VER|Pl z)wl4MJR~9S5!09TV!UwdAAvTjcs~NG7uT1NFXe*;ir_k^EAKf2#VS@JO?nunT=b!0 zqo7*Z>8~@J7WrvSehTRRICXydsD3u0SAbgaxCk9d(_oqel!U)4j`?E;m=xh9BO>#b zJH2}G(4Q@5s`II_IDD#JMQE2p+(Be3M0nG0Xkg{5<_Q9hC`bYQ#F{*cd0atkDGV^R zXnRwK2`jCY7D65TM}xj$=|j=Q?ciXrGM1v{&uEp8@=*`WZ>1Pxuz&)qi{tVF%kEo# z<^=3$Wdqb-XOJxOTbf`B=u$X!Fnv@>8$u-YU8y;V}!%Giy9}*$VlI&)`%fO@NdUJk{sd#jIH1|d{pP%Kuv z&nJc{gB=rEylVUe81ee$lmbLOIFf5>^ z*1MZ4@e^^X(2+PO6Gn!CX90wfHUOtH5zY$k9MB3Wu3vta_8Q`F7j zaUy59L{VKpskxm$X`_n73te4m#1(bA_TuDB48F?cHp58^eOOTxmzWa0{q?IrFq`C&;(EJ(XjNdKJPG*&1perVisIdpuX@z%P|IKK3E^B{EI#I(iK>v865 z#p21q%^hDYXOGFRz{@}TI;ZVbBu2Gy-ML+e*GVys(9_CWKS5gVo|V|Qy4oGxR0nEr z3e6Uh+jHZeJaeAz#!u4cW6Z*<*B*6dd*-ej24{{72KVL*s~&qxntK8Tgr^cv&(BZ2 zG&;UaMZcGe^XU?#%`^M);Qyk-@QJl%L3|=@d}-^*mGss;Wl9gqj(fD^)Y5rBzLyyE zVdBY@7CQa#1p3eVR@KJCuEXEX2_ygj@89(;OIKINzpn%1H2&$D=)7jl^$UT$K%npOP#f&(nPA5HZC(! zU7N=%bX2N>T17uEC*Hb1PvauhJ6Fwm=$H=-39|7te6jE)q?d&F+n0E65T==uv%@wx z)5M$-6#E^scaATzp9jlZr_B& zKkwQ7B#0z(|LQD}szus;)2KZ z-)=SDTR|))tg5fHUFE`xg{+J=?7uiCO}{0-=`v8GU^5Q--`0Wu=^Xgr*!f!rQF#<# zP2NMNfDRfOX#Erhzg;uSAM5@ib^~|C%qWolShlf@+_G4jV#VGkD&CMr_4)BGwcgXb zO@@H#DyqO}e!;A=W6tOI&9GSfbgVW1>*VBkrb=pA0tuYY)*dtdL9L0|xCLL7k+_uz z5xa~{=RYq#St*fkCNNzToqt)pP|zrQdTp672Hj187(Z8ST(tVL=2S3kFV_hZC;Ysr z&{yHWT2IPEy;bV}Syd$M&__Kt>}jjWNa-eXa+eoJEn#U_g(7iB-=cI_iJs*>C=X&v zQsRAUMZjlBdWA9ZOgH^a7?W%wuENlYM*3>putr8=`%OZ&HGO_GO!#Ee{##M2h-+QO zIaE&X_YP$eGv8*sjIBSo;rKFjgAn4`KK)JTRDB=a(426qAl&+NJWrZs3JQb}ZeIN9aDEmM)`? zPIJ51Nyc!0xTw-;VG3Nbt?j79J)24U15^y}93QVMNZ6Ny^}~Cq9mQmOu9t~n&yOV( zfaHN<(cy11WBa3eYrMQ=(~RczV9~z{y$40J!X!7v3S1wKccO|EPp=2l>NYafLWG_= zMS4VJ(zh5UfK8rV(gQsb7mW}i^Vz=VRE&0icoBC z*Xh^k^-g1D-7jnDv-633uB-UqE@KZLmRivQ1vE(@@%_OwWX(UrNn3i&8q)>Kg^NYDGO=s>@bg8b}Bc^I5 zu|>gLgVGWi$T`$Gy1c7C+)S|w(CEkWC(k@Eu)fM!b6_9c#?^*2Oh9(gX%oZun#9d0 z&9~SI_ci_CJYjjhY5F4w>z2YGpJ^q#aP`5=#QJ+$JcrC)3AxnCoRJ7xssXe^%F*T9 z+4+xFPm_*wZ-etgS&@6KaEV1(rp@bP`+`o!yF8t%Yl{09CHXrbZLSpZu4l{Ayp3EdD!E;pN%8K*Fs zqqkQsX#Mx5P+{SYww@zmwMuBgJ>~o^YMM?M6Ti>fQ*`!YE>&qAEO)U?wS}V3;aEn_ zI`gjsCv;_!>wk%`8{xdIHOCTKj&9RjTSiSH(eoh0UeW*(C&(JL0-k^VJm9!9BM-5{ zX?EtbfeJ-d1`>g;%M$5QsSNQh&?u&9?IfWefe-Ic|Fr`!#ur0L(z^j({K>SS|46pM!cFyQx&`jz~c& zuv&~00bn;%?*h>c++2DrzNmHfEUYJM}|!)a3)3{f~p2LB}0~-=}EEViu}e zoS{9(7S6TrUw1%o@p<||C|^!ukcLDO)pYNVTmuhoSxul(-|K%knN4N(uIf#}5?P^@ zBP^_YrO&$gk(__Nh1xkbEwe0e0*`dNePsqJG9J*17qJ)F1}B;i-OS(gGsl5Kg6SK> zU1d8$y6z19h#FTL~w zJxfv#}8ecnt|mHI>VO+5uDJn`ajXkUoz-ky`SbGtGb?}oxr8M zI95C8M*%1`m)UbIK_Ai~Bd=NXwYm|)z|NJLy0=>LHB~Up4tY) z5g{?<#ql;yB92iSQb|79 z_jfJUbCUU7oGTdi@JK^ENOSyC_t`U5Bm?56q@$eYzNo4Dg-(TLb*gqc`TX=ONQ3;O z4rck+ZwMtmj`&vyJ2!`5Uan8Kce8(B&1gAsXa?@1w6efb*fYwf`vSGh4;FzCZ$-B< zx}&DlZxNNy6)Tw~p%u_1x`72ouE!oL4XoZE(oW8!g>E9jSsq|ZW|;V~v)>-R*|-pz zdi18R%viYir0ZSQd~$1Rf-1*Q%|4uqwwfLqE%Axl^=DbVdVdAk$M&r%bCR-!7OvA@ zj2tz^ldk>IzDPG#v=@}F=X6@sCoIS;C9TZAo)#`Kk&V$``%#OW6*tW}q{jRD& zLG+@|0tol9RV}c}Z?KMZN<7$S7LDBZdZAJ}JRA|htf}c6F;5!!ddsqGV4{-<1(mQG zt3yf5wP&I`I)p3)g%+f{83{SMU~X2e?;pS|AHjXc_Law1nRS$Z5oy9xbW4?53oCoP zDY?D|n$s6yw`z#ZL27gRYV#ehep90hKN0iruiU4FZ*sH03)2tl$KM*R_~!Vc3Y%#@ zxiP(p79dSP3pE|J*RmM&VTD(kl(mFQOj;dC+C7PcLzWY$!$2j9KHU5$! zzMSrwSyQz`MjWK_b zL4J-$8V%Fz+IEEx=n<@`65^2$KnhV%lZWHVGl+-erA2W9O%}M1D=-gnYcIGzLnAw) zozduzW#e}`uxc-UIR6CkYDH!_S?PmX;l#BNNZa({O zdHz-H|97$f-zfUH|5ucMO8*EH*8d>kzn=~M4Wq!297sw}_3soF0Qk$@|9)Wa)}AFF zp_3#}lCmMaVN1RveSx&dkqpBjPF5lV|Bn|jBO^v?<4d+6ButJX1Izv&^FTDqg8vr? sv4y?E<_7?*U9HqTT-`jmE!^DxK9{H~qaYj;0D%4Xi#TIg;P2l52i>oOk^lez diff --git a/summary_results.json b/summary_results.json index 9e5a52842..13f56f14a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -3,258 +3,6 @@ "ID": "5.0-0001", "Submitter": "MLCommons", "Availability": "available", - "Category": "closed", - "Suite": "datacenter", - "System": "gh_ubuntu_x86", - "Platform": "gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.078", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.5. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/closed/MLCommons/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 80458.4, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0001", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "closed", - "Suite": "datacenter", - "System": "gh_ubuntu_x86", - "Platform": "gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Server", - "Accuracy": "acc: 76.078", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.5. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/closed/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/closed/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/closed/MLCommons/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "ColorKey": "availableMLCommons", - "Performance_Result": 73015.3, - "Performance_Units": "Queries/s" - }, - { - "ID": "5.0-0002", - "Submitter": "Community", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "default", - "Platform": "default-reference-cpu-tvm-onnx_v1.19.2-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 80.000", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", - "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/Community/results/default-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/Community/code", - "Unique ID (e.g. for Audit)": "edge/open/Community/default-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", - "ColorKey": "availableCommunity", - "Performance_Result": 1.73704, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0003", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "datacenter", - "System": "3b07702db56d", - "Platform": "3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base", - "UsedModel": "stable-diffusion-xl", - "Model": "stable-diffusion-xl", - "Scenario": "Offline", - "Accuracy": "CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 1, - "Location": "open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl/offline", - "Software": "pytorch v2.4.1", - "operating_system": "Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "Notes": "Automated by MLCommons CM v2.3.9. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/3b07702db56d-reference-gpu-pytorch_v2.4.1-scc24-base/stable-diffusion-xl", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.374837, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0004", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "datacenter", - "System": "gh_action", - "Platform": "gh_action-reference-cpu-pytorch-v2.5.1-default_config", - "UsedModel": "llama2-70b-99", - "Model": "llama2-70b-99", - "Scenario": "Offline", - "Accuracy": "ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch-v2.5.1-default_config/llama2-70b-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.387999, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0005", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "datacenter", - "System": "gh_action", - "Platform": "gh_action-reference-cpu-pytorch-v2.5.1-default_config", - "UsedModel": "mixtral-8x7b", - "Model": "mixtral-8x7b", - "Scenario": "Offline", - "Accuracy": "TOKENS_PER_SAMPLE: 342.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-cpu-pytorch-v2.5.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch-v2.5.1-default_config/mixtral-8x7b", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.0356089, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0006", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "datacenter", - "System": "gh_action", - "Platform": "gh_action-reference-cpu-pytorch_v2.5.1-default_config", - "UsedModel": "llama2-70b-99", - "Model": "llama2-70b-99", - "Scenario": "Offline", - "Accuracy": "ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.5.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.5.1-default_config/llama2-70b-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.357886, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0007", - "Submitter": "MLCommons", - "Availability": "available", "Category": "open", "Suite": "datacenter", "System": "gh_action", @@ -284,1843 +32,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.406358, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0008", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "datacenter", - "System": "gh_action", - "Platform": "gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "UsedModel": "mixtral-8x7b", - "Model": "mixtral-8x7b", - "Scenario": "Offline", - "Accuracy": "ROUGE1: 70.1571 ROUGE2: 41.2698 ROUGEL: 45.0262 TOKENS_PER_SAMPLE: 245.3 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline", - "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.0102599, + "Performance_Result": 0.40413, "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0009", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "datacenter", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "UsedModel": "rgat", - "Model": "rgat", - "Scenario": "Offline", - "Accuracy": "acc: 74.800", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline", - "Software": "pytorch v2.4.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", - "ColorKey": "availableMLCommons", - "Performance_Result": 10.0454, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0010", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "datacenter", - "System": "gh_ubuntu_x86", - "Platform": "gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.078", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/offline", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.4. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 80229.7, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0010", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "datacenter", - "System": "gh_ubuntu_x86", - "Platform": "gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Server", - "Accuracy": "acc: 76.078", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50/server", - "Software": "TensorRT", - "operating_system": "Ubuntu 20.04 (linux-6.2.0-39-generic-glibc2.31)", - "Notes": "Automated by MLCommons CM v3.2.4. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "int8", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu_x86-nvidia-gpu-TensorRT-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 75016.1, - "Performance_Units": "Queries/s" - }, - { - "ID": "5.0-0011", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "default", - "Platform": "default-mlcommons_cpp-cpu-onnxruntime-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", - "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.43471, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0012", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch-v2.5.0-cu124", - "UsedModel": "gptj-99", - "Model": "gptj-99", - "Scenario": "Offline", - "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99/offline", - "Software": "pytorch v2.5.0", - "operating_system": "Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.0-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch-v2.5.0-cu124/gptj-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 47.0771, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0013", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch-v2.5.1-cu124", - "UsedModel": "gptj-99", - "Model": "gptj-99", - "Scenario": "Offline", - "Accuracy": "GEN_LEN: 264", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch-v2.5.1-cu124/gptj-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 42.3041, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0014", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch-v2.5.1-cu124", - "UsedModel": "stable-diffusion-xl", - "Model": "stable-diffusion-xl", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-49-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch-v2.5.1-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch-v2.5.1-cu124/stable-diffusion-xl", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.352784, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0015", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch_v2.4.1-cu124", - "UsedModel": "gptj-99", - "Model": "gptj-99", - "Scenario": "Offline", - "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99/offline", - "Software": "pytorch v2.4.1", - "operating_system": "Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "Notes": "Automated by MLCommons CM v2.3.9. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.4.1-cu124/gptj-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 46.0817, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0016", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch_v2.4.1-cu124", - "UsedModel": "stable-diffusion-xl", - "Model": "stable-diffusion-xl", - "Scenario": "Offline", - "Accuracy": "CLIP_SCORE: 15.18544016778469 FID_SCORE: 235.69504308101006", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl/offline", - "Software": "pytorch v2.4.1", - "operating_system": "Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "Notes": "Automated by MLCommons CM v2.3.9. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.4.1-cu124/stable-diffusion-xl", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.346565, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0017", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch_v2.4.1-default_config", - "UsedModel": "gptj-99", - "Model": "gptj-99", - "Scenario": "Offline", - "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 1, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99/offline", - "Software": "pytorch v2.4.1", - "operating_system": "Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "Notes": "Automated by MLCommons CM v2.3.4. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.4.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.4.1-default_config/gptj-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 50.3446, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0018", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch_v2.5.0-cu124", - "UsedModel": "gptj-99", - "Model": "gptj-99", - "Scenario": "Offline", - "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99/offline", - "Software": "pytorch v2.5.0", - "operating_system": "Ubuntu 22.04 (linux-6.2.0-39-generic-glibc2.35)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.0-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.5.0-cu124/gptj-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 42.4133, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0019", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch_v2.5.1-cu124", - "UsedModel": "gptj-99", - "Model": "gptj-99", - "Scenario": "Offline", - "Accuracy": "GEN_LEN: 264", - "Nodes": 1, - "Processor": "AMD Ryzen 9 7950X 16-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 16, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 1, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.5.1-cu124/gptj-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 48.3215, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0020", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch_v2.5.1-cu124", - "UsedModel": "stable-diffusion-xl", - "Model": "stable-diffusion-xl", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD Ryzen 9 7950X 16-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 16, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 1, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.5.1-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.5.1-cu124/stable-diffusion-xl", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.352694, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0021", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "UsedModel": "gptj-99", - "Model": "gptj-99", - "Scenario": "Offline", - "Accuracy": "GEN_LEN: 264", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline", - "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 49.1296, - "Performance_Units": "Tokens/s" - }, - { - "ID": "5.0-0022", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_action", - "Platform": "gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "UsedModel": "stable-diffusion-xl", - "Model": "stable-diffusion-xl", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node": 1, - "host_processor_core_count": 24, - "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, - "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline", - "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.352003, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0023", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", - "Software": "onnxruntime", - "operating_system": " (darwin-23.6.0)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 8.08903, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0024", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "UsedModel": "bert-99", - "Model": "bert-99", - "Scenario": "Offline", - "Accuracy": "F1: 80.0", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline", - "Software": "deepsparse v1.8.0", - "operating_system": " (darwin-23.6.0)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 8.14067, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0025", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline", - "Software": "onnxruntime v1.19.2", - "operating_system": " (darwin-23.6.0)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 20.9527, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0026", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline", - "Software": "onnxruntime v1.19.2", - "operating_system": " (darwin-23.6.0)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.323181, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0027", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline", - "Software": "onnxruntime v1.20.1", - "operating_system": " (darwin-23.6.0)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 17.3387, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0028", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline", - "Software": "onnxruntime v1.20.1", - "operating_system": " (darwin-23.6.0)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.412266, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0029", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline", - "Software": "pytorch v2.4.1", - "operating_system": " (darwin-23.6.0)", - "Notes": "Automated by MLCommons CM v3.2.3. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.350957, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0030", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline", - "Software": "pytorch v2.5.0", - "operating_system": " (darwin-23.6.0)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.277839, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0031", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline", - "Software": "pytorch v2.5.1", - "operating_system": " (darwin-23.6.0)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.440048, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0032", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_macos-latest_x86", - "Platform": "gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 3, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline", - "Software": "pytorch v2.6.0", - "operating_system": " (darwin-23.6.0)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.264988, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0033", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest", - "Platform": "gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 80.000", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", - "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 1.74836, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0034", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", - "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 23.0164, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0035", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", - "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.431711, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0036", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "UsedModel": "bert-99", - "Model": "bert-99", - "Scenario": "Offline", - "Accuracy": "F1: 80.0", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline", - "Software": "deepsparse v1.8.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", - "ColorKey": "availableMLCommons", - "Performance_Result": 6.31453, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0037", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline", - "Software": "onnxruntime v1.19.2", - "operating_system": "Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 23.227, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0038", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline", - "Software": "onnxruntime v1.19.2", - "operating_system": "Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.433265, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0039", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline", - "Software": "onnxruntime v1.20.1", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 23.009, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0040", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline", - "Software": "onnxruntime v1.20.1", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.43494, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0041", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline", - "Software": "pytorch v2.4.1", - "operating_system": "Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "Notes": "Automated by MLCommons CM v3.2.3. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.409818, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0042", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline", - "Software": "pytorch v2.5.0", - "operating_system": "Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.0-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.412839, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0043", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.5.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.412747, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0044", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline", - "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.409259, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0045", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline", - "Software": "tf v2.17.0", - "operating_system": "Ubuntu 22.04 (linux-6.5.0-1025-azure-glibc2.35)", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 21.0188, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0046", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_ubuntu-latest_x86", - "Platform": "gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "AMD EPYC 7763 64-Core Processor", - "host_processors_per_node": 1, - "host_processor_core_count": 2, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline", - "Software": "tf v2.18.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 21.0425, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0047", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest", - "Platform": "gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet/offline", - "Software": "onnxruntime v1.19.2", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest-reference-cpu-onnxruntime_v1.19.2-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.349311, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0048", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest", - "Platform": "gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline", - "Software": "onnxruntime v1.20.1", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.336027, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0049", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest", - "Platform": "gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet/offline", - "Software": "pytorch v2.4.1", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "Automated by MLCommons CM v3.2.3. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest-reference-cpu-pytorch_v2.4.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.327316, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0050", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest", - "Platform": "gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet/offline", - "Software": "pytorch v2.5.0", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest-reference-cpu-pytorch_v2.5.0-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.329647, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0051", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest", - "Platform": "gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config", - "UsedModel": "retinanet", - "Model": "retinanet", - "Scenario": "Offline", - "Accuracy": "", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet/offline", - "Software": "pytorch v2.5.1", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest-reference-cpu-pytorch_v2.5.1-default_config/retinanet", - "ColorKey": "availableMLCommons", - "Performance_Result": 0.313782, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0052", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest_x86", - "Platform": "gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline", - "Software": "onnxruntime v1.19.2", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "Automated by MLCommons CM v3.2.4. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 17.596, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0053", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest_x86", - "Platform": "gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline", - "Software": "onnxruntime v1.20.1", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 17.3279, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0054", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest_x86", - "Platform": "gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50/offline", - "Software": "tf v2.17.0", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "Automated by MLCommons CM v3.2.6. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest_x86-reference-cpu-tf_v2.17.0-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 21.7501, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0055", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows-latest_x86", - "Platform": "gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline", - "Software": "tf v2.18.0", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 21.5168, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0056", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows", - "Platform": "gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50/offline", - "Software": "onnxruntime v1.19.2", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "Automated by MLCommons CM v2.4.0. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-reference-cpu-onnxruntime_v1.19.2-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 17.8036, - "Performance_Units": "Samples/s" - }, - { - "ID": "5.0-0057", - "Submitter": "MLCommons", - "Availability": "available", - "Category": "open", - "Suite": "edge", - "System": "gh_windows", - "Platform": "gh_windows-reference-cpu-tf_v2.17.0-default_config", - "UsedModel": "resnet50", - "Model": "resnet", - "Scenario": "Offline", - "Accuracy": "acc: 76.000", - "Nodes": 1, - "Processor": "undefined", - "host_processors_per_node": 1, - "host_processor_core_count": 1, - "Accelerator": "", - "a#": "", - "Location": "open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50/offline", - "Software": "tf v2.17.0", - "operating_system": "Windows-2022Server-10.0.20348-SP0", - "Notes": "Automated by MLCommons CM v2.4.0. ", - "compliance": 1, - "errors": 0, - "version": "v5.0", - "inferred": 0, - "has_power": false, - "weight_data_types": "fp32", - "p#": 1, - "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_windows-reference-cpu-tf_v2.17.0-default_config", - "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", - "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_windows-reference-cpu-tf_v2.17.0-default_config/resnet50", - "ColorKey": "availableMLCommons", - "Performance_Result": 21.8155, - "Performance_Units": "Samples/s" } ] \ No newline at end of file From 60d16b0d872d4f07d579ccba508cb5172ebec6bb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 7 Feb 2025 10:52:22 +0000 Subject: [PATCH 003/719] Updated results summary [skip ci] --- README.md | 33 +++++++++++++++++---------------- dbversion | 2 +- summary.csv | 1 + summary.json | 30 ++++++++++++++++++++++++++++++ summary.xlsx | Bin 6365 -> 7490 bytes summary_results.json | 36 ++++++++++++++++++++++++++++++++++++ 6 files changed, 85 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 306cc8b50..7e951c167 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,25 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-07 05:43:36,988 submission_checker1.py:3263 INFO] Results=1, NoResults=0, Power Results=0 -[2025-02-07 05:43:36,988 submission_checker1.py:3270 INFO] --- -[2025-02-07 05:43:36,988 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-07 10:52:17,312 submission_checker1.py:3263 INFO] Results=2, NoResults=0, Power Results=0 +[2025-02-07 10:52:17,312 submission_checker1.py:3270 INFO] --- +[2025-02-07 10:52:17,312 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-07 05:43:36,988 submission_checker1.py:3276 INFO] Open Results=1, Open Power Results=0 +[2025-02-07 10:52:17,312 submission_checker1.py:3276 INFO] Open Results=2, Open Power Results=0 -[2025-02-07 05:43:36,988 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-07 10:52:17,312 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-07 05:43:36,988 submission_checker1.py:3286 INFO] --- -[2025-02-07 05:43:36,988 submission_checker1.py:3288 INFO] Systems=1, Power Systems=0 -[2025-02-07 05:43:36,988 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-07 05:43:36,988 submission_checker1.py:3297 INFO] Open Systems=1, Open Power Systems=0 -[2025-02-07 05:43:36,988 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-07 05:43:36,988 submission_checker1.py:3307 INFO] --- -[2025-02-07 05:43:36,988 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-07 05:43:37,891 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-07 10:52:17,312 submission_checker1.py:3286 INFO] --- +[2025-02-07 10:52:17,312 submission_checker1.py:3288 INFO] Systems=2, Power Systems=0 +[2025-02-07 10:52:17,312 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-07 10:52:17,312 submission_checker1.py:3297 INFO] Open Systems=2, Open Power Systems=0 +[2025-02-07 10:52:17,312 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-07 10:52:17,312 submission_checker1.py:3307 INFO] --- +[2025-02-07 10:52:17,312 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-07 10:52:18,276 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:-------------|:------------------------------------------------------|:--------------|:--------------|:-----------|---------:|:----------------------------------------------------------------------------|------------------:|:----------------------------|---------------------------:|----------------------------:|-------------------------:|------------------------:|:---------------------------------------------------------------------------------------------------|:---------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:---------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:------------------------------------------------------------------|:--------------|:--------------|:-----------|---------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|-------------------------:|------------------------:|:----------------------------------------------------------------------------------------------------------|:---------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 1217bfdb2..9246847bf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -630 +631 diff --git a/summary.csv b/summary.csv index fded8e476..329b857cd 100644 --- a/summary.csv +++ b/summary.csv @@ -1,2 +1,3 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b15840692..289970c24 100644 --- a/summary.json +++ b/summary.json @@ -28,5 +28,35 @@ "has_power":false, "Units":"Tokens\/s", "weight_data_types":"fp32" + }, + "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":22.9844, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" } } \ No newline at end of file diff --git a/summary.xlsx b/summary.xlsx index dfa626c91c2b90eb05bf16b79f9b3efe2cf5b091..e9a08b606c7e6e36274e6eb0df153d79e6701e12 100644 GIT binary patch delta 3707 zcmZ`+2{hE*`=2q{Mlo3jBTL96YuU17OO`Px$=J7P>|=>6@wH5wov|w!*$JbPwX!u5 zQpvuAh-`xx$@0^C{_nr@zUTM5_uPBWx#u}|d7kHUKc71;*Lof>HG&-B1c5+IhwTCg z)7XSJ2ZKO7^dJz&;f%Eo-unjD`-Z)Fun*SXRyxSbvm^DLcdIPyB5g0GbM&-gbxB#w z>-n3fP|l_ghG@MfMw41g(tuQ)<)@Rqi=b!ue~J4q+3{yvnoB zeV;}(8$SZpt+!L}*}}vQ&+Ml#loa z_k}*&^lBarY33(2o%Bh)GSi~h#wan{dU>nPaMU)5i5n9*$;@xj;=(i_VNfcajy}ly z%1>wV@Ckz?2lYI2=5|`2`96c^j@`$%x{rl(I`!WJHR#i%=4@R-Q&IKF)paA3eY&33 zx@CXqZ5a(fz=z6VRtRta20Kb{NpIsIl4V`u6L z$BoJT;oX5b5Vei+vyrx8#5P*dl>SDb1>YetGP_&)igEboqfBS2iAr_K@gI^MKM=ON z4B?wh6LYZTM#Y^YPb0uWVCb{Q@1&`Re%FH#%jkrF=9C`yHdz>9H4I9oS=D|@zbx|Y zGF^S`0))xKD{CmF&l}j?;$k)yU@Bt#M9kz;-5_^|Y$*JghMWwTKuXjgjx>zq^gf49 zY>kdyg0+wEj)&fb>WL1SD=qRYDHIFT*a{`15XkE8DFeeYDb{Rx!M?&rZzx*+DdaS* zK}KLa|IEg&)|*mBbDmr$9B2DOq~;PO0$AHJ{`7f!X^{pNu)m-5NaIAOh@bpt@*CCl1Xnwi zqVFciq8UMWu-UDcK=o%CRf*<-P$W!Jf-p|9|6^p&$ zFZ0(R|5tLR8~NwR!YHeIoP5TtZ&6<0+R_>F!n5FWZYPY@!cZnBcnKdkVQIQcw|-T= z4)h@12w7~gWPde&;C_4wJ?R~TP?syMlfVtFhQi76+{`l9&hE}`U;%tw^2cW=S7}4V zw)d#17az(OtU3ixJEKnen3nPOKC2WblqfsXUkx)Ov#%6rbDOuBSKccVeQe-t{#BIP zl*E~uQMW0Zd@!w3fJ_{&zqc5b)({|qK`Rid7R zdudOf?>NVGqtmr_WdY)%%N_OT6xPsT-8CVz*2`AkLZk<-#Z}24!|?MGEx(r4=SS(B zhQvV|vZla#9=Y?~%6hye?AO=h%xom3V(`mSz+#@lX4{h|t3N{*KfS#0qt(IX1jWC% zR3Q8ONHO$SSI)>y5aV zHfqi2Bu49vv>XPSS4ASz#!Xn;yH=thU)Zod`H ziZOd-m?SMD!SalZVBo!(%~yVz*3dyf?x3#m#V_%RA(j`c;6=}|VMUs44k^qMaWBj= zobpib+N7ZL+>1e)62+KgZi>8SgWIeTjE&+{v0*8R0%XfFQ9MnkEP_=pcQ&%wbhGA5 zSL(&*L42F)`C`t-VDX~fl*ZAgPnunz-*ZIDJ(ZFuHudJIQMG(`>GQ-Z&(&2oEVW!}T2p=(3}NG8KQ^7CIZ&z&u)FNK9D$u)PrJQWuX z{wH3)`O(eT948BtLP<`TA z1rL7uX!<(z{>#{t#ibz&%-1ur#(6Ckoh9G$z$JlaU2f;tkl?HC(&i68B`$wjN@rip zoJP6RtEfj7E>2%paP{Br-_cB}@sK@#i$4}&SxKdkQC^2JMNnm$ z7UB-3Po6Z7adg{%Oe!}_rRKeRNk7{w!XWBj*X*T>b0e1#2;;*rM_XE#mCY{+WO8@I zB|sWMeIU-X33>%BT3rbdR@ZM!W$5Vm+9-@TJz(^rbV9r^Ww8ddK0?ysw>vfu_g8EIvGh ziy6!S<)7pY?KQ&W@KiJntLx7YEj>!mP*0;;C`VNb5|wmx+x@(7pr!}-czY3wdxwClXsz8%q{z-R3WUYtHF%5GD)D` zDDg*W_r%NIV%OU3vs%Hqws{qT=mNSQ%kg23z9G6AET~Sh`gx>rh7~ zdP@Gw12Li{ham8`RC(MeH7sm=ycXj1wlq@vid6{5$2Z5HhbQ%)5C3%N@97| zUSObE{aCr;Px^0#AGo^CMzQyeF_q>(oADM45W`yzR@DQpC8lgAq!t@fth?f)ui?(r z3ALO$UZ6f(tKA4Y(=oImATjrJzRzrouh{Zlp+_}<-OW*mV9{P7N4-A6a#S#)RN($R zf%BH`h~OYAs73Bso+e}B#u*9^9#p*R(w1kVl6JKDw!tC>F&-*&$$2mUqr_h!2P9k= zl)<-^mG**gh`52h-S z%n1jsw8V_c!kTH4gq8bEq*QhnDWh5}o76GU#?V~D8Ow?@%x(jsdpljMYv;X4)f&yg z8$oJ4akI;0{|ZW~7ggaE#^M2Ku*DK6`8l({IDHnE^_t(O#kt2M^XHQtp%+M9l#ATc z=`a30*7%bX@_y@->b5L00wqSFeV#2Z{b*=n+BR*`oc4b{!52s)nT(ILE0985H{t}^Z9yw=>i3=>TCl(UR3NkbNid&|N z;N=W9ddLZ;pAoX%o-B%3u}~{o2~sqDxk~;y7w&@a46JxgU6dtv8xtt4x8zh*kPGO9 zypzBbM0W)!w2nIlsoyeqoR*H56YSw;-;-I(jC?Q8C)9@>%N&Y2W|Vd5 zAuJoR{}|x@e@;^N;uUP53cGVR1pa{1lO!U3`@K#jD zb38qh@l)vB-h-$Hj8yr0{5Rhzp@JaQ(9J^$3Z~S{w@)bu}6t3Ox(oY!($y|Bku61&;iGZ5y!ZQhH%*bVcQWBC7mvQv*`Azz!M)VB7jaUxr Mn?pLf{+Cz&1J#6)OaK4? delta 2593 zcmZvec{mhm8^*^>V#YFL#xgUOQTDN=88OKok+sFvSVx$V8N-Q)$xz%(%quPtDy7YMi|ua z7k=1oBV)zZ#qRfLt5V|9NKkw7F>zJo9aWipfkxQQ9}0(8bj9ZjAB5%|n+cIMwjWg# zJC^Rrt$5nA*fWkVmGL1Pm(u~yz^!U_uLGL*0em*N(x`e^DyGZx~R{+_nmuOG9pog#U;>_Dcr#T9|`+fz;9l%S5K;}Vfvd%ziG2t?JN zTtF^}$!{(zpE&6M0x}*0W$0yT&7>C^LMF>QkctiGO9w2JQiGJPogZEW#a7px@;~WY z>$kHZ47QTw&F9H$NgTY(Q$PQz_w`MX{9LqyEnJRwRerFi7IHjANvNz@&s@V9P*OAn&0o;Skp(c$la}#q1tC~O+AUZKs z@~ADxZkj*<0Lq~d>aQL`p^-f)6tY_AMgOGKzr6<`uiuK3QI6kNtPU?G)&E|8X%bvP zUFf)k*!-RxV>C=~SeNWYy=rhRvq|!F^UXP;8%IQdj0p(z8aggzv2RK=^XqxOM~Q8y zo=@(Ckm@Pnq3RcYfSlMh9TNs|~- zT|X6#S)_{>ln`$~3lyL6;MtAtRJFUZuugwOT z`=l>iKGTVE)F0Wf-^_wIG3(g5c2cCg7qZ*DZ|hFP%}JO<(jEBJ?hrk$P|FpgW| z@DJd`eXj{{_y_rV(7cJKgK2&NK0$j$h)8##3~9jHKWr$KE~vn9Z_)nSyQBNftCsQBa6iSw((*$GvJ- zDw6LO&g4w2CD+avonJRpxe4ZsBpP2^;^Fa1)=R|d85jp2N|7mjkWqqei!9M&n{nTN z!vpA}X14{3X~*6G*u8`FL3N>AB<}BTpRa&|qbpyRVovGYhOYqcRX!7=UQ>ZYE6NiZ zg%jNn9jh(-{NbAb;`=3RFwBG)~c&B=$iE@y0fg5R&(ZuIu}c1yo8Or;TGp6 z&Te;ql>pR(m%h4PkS@>hhAk(X!W?f9DPFcT3N=XGi$e3>Ge%C@tG&q(yBv|wvC(2} zZ{aOxNK&yjEjd&2$0#Nod~QyyUCL(jSrMP|VWhB0yh8-?Z+v1*iz`}NEMIU%Q z!dpCyh``Lu9<;dP?p)sETf`DPmW2(0*QV2vJd_?ss$sN^ta*6X#UvcsF`;4xY5UTL zmH%fQ&bmU#a=^2DQ=7~s7JJN{I|?i+J_bWtVWr80>a|M?T>fQ-1&&Yax2LzDcK(;q=6gM{d9JQ zQkZl4ptpe?O*a&tp*PkPe0Kr=%v`=8P=wCGiUa`wLjQ}^gG&83Rr}_EVGaHU`m)Nt z_aZvY8vIeEp61%C8_|nvcc?Ep@d@24_q1l)5*{#%XOZ{OiFIh2T zC5LV!2tV$e8p?7jbhBm?!>t($xx(+*rIp6il3rD76%@fAN`(V&S9&brEK)C)b30E5 zb})1cM33H?E%GGbZ*H&}Ys4MtWj7}n&qch7%AJE5zcUpB!gk$Q%0tc4dXhZ^w$?Yx zfy+KoUv09adkV}eE__}ivdCEJIZUk61t+F+HQ zFxp>hf|cmpWKT3#n0<3vmierYdOV$M2Bl3`0QcXUY&#_C>DYyTIAHc8>;iIF(5T1W z>QgobF6Gd&A>$^Y#x?SKB-PE9aE*KVPVy6CD6RaBlxz zSEJGxd`L53C5SN$LNEf5La1NI-%sgT9>y3Fjrz4e{}jC8Wk{k7fL|aCH`%F=2xAy^5cRA1ehRW+41^>?5CVVzu5tozCc5|ZFJs?p_y7O^ diff --git a/summary_results.json b/summary_results.json index 13f56f14a..e2bd26a08 100644 --- a/summary_results.json +++ b/summary_results.json @@ -34,5 +34,41 @@ "ColorKey": "availableMLCommons", "Performance_Result": 0.40413, "Performance_Units": "Tokens/s" + }, + { + "ID": "5.0-0002", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", + "Software": "onnxruntime", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 22.9844, + "Performance_Units": "Samples/s" } ] \ No newline at end of file From b27a5b6d4481df8d97b1ee92a4085919c6e93339 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 7 Feb 2025 10:58:37 +0000 Subject: [PATCH 004/719] Updated results summary [skip ci] --- README.md | 37 ++++++++------- dbversion | 2 +- summary.csv | 3 ++ summary.json | 90 ++++++++++++++++++++++++++++++++++++ summary.xlsx | Bin 7490 -> 7968 bytes summary_results.json | 108 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 222 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7e951c167..59e8a60fb 100644 --- a/README.md +++ b/README.md @@ -1,25 +1,28 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-07 10:52:17,312 submission_checker1.py:3263 INFO] Results=2, NoResults=0, Power Results=0 -[2025-02-07 10:52:17,312 submission_checker1.py:3270 INFO] --- -[2025-02-07 10:52:17,312 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-07 10:58:32,810 submission_checker1.py:3263 INFO] Results=5, NoResults=0, Power Results=0 +[2025-02-07 10:58:32,810 submission_checker1.py:3270 INFO] --- +[2025-02-07 10:58:32,810 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-07 10:52:17,312 submission_checker1.py:3276 INFO] Open Results=2, Open Power Results=0 +[2025-02-07 10:58:32,810 submission_checker1.py:3276 INFO] Open Results=5, Open Power Results=0 -[2025-02-07 10:52:17,312 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-07 10:58:32,810 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-07 10:52:17,312 submission_checker1.py:3286 INFO] --- -[2025-02-07 10:52:17,312 submission_checker1.py:3288 INFO] Systems=2, Power Systems=0 -[2025-02-07 10:52:17,312 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-07 10:52:17,312 submission_checker1.py:3297 INFO] Open Systems=2, Open Power Systems=0 -[2025-02-07 10:52:17,312 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-07 10:52:17,312 submission_checker1.py:3307 INFO] --- -[2025-02-07 10:52:17,312 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-07 10:52:18,276 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-07 10:58:32,810 submission_checker1.py:3286 INFO] --- +[2025-02-07 10:58:32,810 submission_checker1.py:3288 INFO] Systems=5, Power Systems=0 +[2025-02-07 10:58:32,810 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-07 10:58:32,810 submission_checker1.py:3297 INFO] Open Systems=5, Open Power Systems=0 +[2025-02-07 10:58:32,810 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-07 10:58:32,810 submission_checker1.py:3307 INFO] --- +[2025-02-07 10:58:32,810 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-07 10:58:33,942 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:------------------------------------------------------------------|:--------------|:--------------|:-----------|---------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|-------------------------:|------------------------:|:----------------------------------------------------------------------------------------------------------|:---------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:--------------------------------------------------------------------|:--------------|:--------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|-------------------------:|------------------------:|:-----------------------------------------------------------------------------------------------------------|:------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1626 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 9246847bf..c9a777fb5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -631 +632 diff --git a/summary.csv b/summary.csv index 329b857cd..a9d9fbdb9 100644 --- a/summary.csv +++ b/summary.csv @@ -1,3 +1,6 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1626,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27262,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 289970c24..2a27ca826 100644 --- a/summary.json +++ b/summary.json @@ -30,6 +30,66 @@ "weight_data_types":"fp32" }, "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.431199, + "Accuracy":"mAP: 49.593", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "2":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", + "Scenario":"Offline", + "Result":10.1626, + "Accuracy":"acc: 75.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -58,5 +118,35 @@ "has_power":false, "Units":"Samples\/s", "weight_data_types":"fp32" + }, + "4":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", + "Scenario":"Offline", + "Result":6.27262, + "Accuracy":"F1: 80.0", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" } } \ No newline at end of file diff --git a/summary.xlsx b/summary.xlsx index e9a08b606c7e6e36274e6eb0df153d79e6701e12..11c8a95f434ad1f5dfa032f55063ba14e2758b75 100644 GIT binary patch delta 3384 zcmZ9Pc{J3G*T83tHHIO?7}-XatT7rMM3(H0tWDO55VG&f*Ml&Y$V6lmLWV5avQIts zeQTKP*^-haYi0U*-gDmbzUOz(x#ym9?>+aPd;j>Hd;d`@(5xAToNHk!9iN1NK#fqs zG(-rnu@9J0gGFsukA?1K3~>YCORn$o6OhA9$oPm^vlo2MgSHm4{^Ggs$mC3Dv)+Qs z_qnI)@ze>`){488!Kj~Y!%Focb{Oka+0kHG-Q=Y1dfW6$AHXqWL>7sZ!x7)ardA4a zPJQ0#+{oQ^6cyJMbuED1y_{8xP`*K^LUQNfL4SS|8_@GY|Ylyy< ztu(X#y5kjo!?vF(F33!zHJzI|{Lqvy2U_M~v zT3ju~IjDP@;$L2TAp2xxi@GVzp`1-pNxcQ{EiRka&&rVbGj0+V0fACj5A8+|9H=k6 zsy{PEI|9$`^$!uRcCj}?@#5?^h#F!J5is#7p#g#L&qgH%>6%}h1@r2JY!QqZ*hJod z^T9Vp^aZSefvFU2A(f4{$KP4WyTF+Kz7vN%Rp)K@5gUiMuF&CFoW?#ja&c!&pS~th zbK{dSxmySk(c~^RlFms`S)No}u1(uXP4{EaVnbDflSF!LKoc4XbD)d`#Z}&z{dAEK z5E7j*2Z8C#N`Vx+^1(2j`9zhjs~@81g%er=K)AAB^jZ%6{h$SJ?b#Th@RR|h7@rS< z=`)iZl_BP6l86CdisZUScN(ag^!CaUxv)1}8) z=`5U(oh3dEkDgF);lF$Vtc_{m!1}X+dhAEe!F+WKaY(3H5&bM@2fN%2R4SzBdAGdY{Ot`w<|%w0+Q z2a$i(%#U>N$%$Db1HGf_aiWUD@j)v7Tm^9T_TFcf5jvLc2V|RNtg>I2DMMYvgV6}_ z&y2n!&>51GS0*b6N3=W-!mT4fiUCznkF_()aq?b3l?fOdo;-#AoPRf@8sKyetuXQ` zw?7PoPUuJnb-{czLl=uHtgS>U9&P$I;J3nd!f6Mzo3!@cy@e6-rB>oKtZhS1&So8x zo9VPHWs4raXc{3@cDN7{RI1-!G{BtMdaI$+rJ_n82+f=P)^M`_Wu`?BX%0oI^x?Hi z=p=T(0yrnHU)T@PGrL(ta0=yOa1(kAB=i_Cgv6A?JmrHbpz%D9jM^?K4Ncd>B*pME zx2q~G*N5i|qO)JWeqrhAs7RWjb>1nLGl>6smC|`BwZAjv!Ioo{-KT39V+cJr2n1q2 z(`%pvY|UvHJqQ%Z1p@J%c|yJ9Lwx)_0|-e>Fkm?6j?XfhyLa)(!q!{}>(K{gmZHD@o-`z<_S zP+wuXZ*1c?=9z%DU?Cf(S}oTb5-Z{%3;ES>dbV#58NpVkrP^BkawvmWcsmc+POP0L ztQtS(4#Uji8NuD{6ybuF5I}l4c5}fKhM6ZZf+v}DqlSVydp(0~b`r}eT1@eOCZ1jYWQ>QLL$k(6M`@8Z1gUS*7 zt?>4j(YBTGwZ+Nmx_-+1uwR3+6VAKFD$`tB^FU%HO{3dnXk%!`WUDg&vz{$?WsT$=t+^? zP65?^tok0Ce-`N_8%G+ZjUqXF9)|vr>`b}PyKnky(RfEJ{Ft3(ika@DSShr?bD*jaF6lGN#;E)cc<%otAM(wb73I z8%pXFfah6K`oAk-d68{$T?+jja7k?M!3n*z`*rt!7B@HcB8yGp%=L>_q)At9Mn5?| zmQ>&Sd8067kEL0{JxO=;ZP|uBPl=k=STPFD{hn))F)(+Ptpv?0*kHQbP;`L55j1RQ zZjW#HrSysDGTX|-E^4xtPyN;tzq6Gl)Bk!?;F{<_Ak-PN{zcar zqypTX{asxw0{uN6xCO|CdV6JAnffd%z}mN~Yige!W^i>18$ctjKaTl`RsisRaXO*1{Q91=5< zv&bGhHmb13E?TrSGsbAo#6zS>J7GoJ`}jF{1UU{uMQ%CiIHo7k_$A+?z>mavtRg!} z)>gg@Iq*@eq9uhh1N5X4H^e578^$f-#|Z}UVcaA7s%ON0xa)GUmWv8nMO z2)-ddwQ{wA`sIR*?AFEj#yqN|f#(G;Nme`_wZvARyAbYnf1wPgA@3U}?2cO6l75Z@ z($XqZmuVk6Y0J^lwI(@H+Z-&HX6%wKJ2Oonn46IDbH81Lm zBybzJUvslRxl%RuA6KemDPcOAIyBo5ws!H(z1|WxzP2BhN^r<_3FTd@Gz#oV_Z2s^ z@$G(4+H>ulAD@j^8(aOhKVJCWJFIqm(r|HE8Ht@V1qo!4almyF2xPbB_TY zi5y+DyNm|1dU%x~gSDmvisFZheZWNi+ScvbKT5N`0VZJoth*#()yB4tsK?M>%`R4- z$ndw9X$pmzqy_Y$%SRVgt$FI#?x_i5frKEHHdN0x;9KQGJL)Ba>!2LY%=CYw{;nnX zUyy-Z@`e!#FYp5y7U>nar@_JI4OEic3B%`-anyiPEZ5i^GiesyUj6h7LR?cp*49n= zgF7glci6sLcCF`qg%Yn_#u9;f;<^8wm)^|TxNj%C+&^vyZb&{zwUFtWXrNx94P)Oi zh#c8!JH;|29KHKsrW>exPE%wn{i}Y=5}z8TAg(tod!-Jz8e>7b*I8zV&SHyum^Vd0 zH-|Ww^>l34bs#Dgfk)SD>o2en~&m z-mZuyYQhZJv^e~34%|4|Z*wRr74DW1*E6~}rCM00Dmz$x+E+nS!U!h*&I_eO_&Wlz z>m2`HDRoVJF?9d`FS#zDj`;6`bHPRfyoF*A2Ih0d{m_;(SJ%k})G|Sn_vH3YZCUYju-bdz8h%_QX zg=z92ok$K5l^*9Dv+&mSzVG$C?|Wa@{onWd&-c1N*Y_?q>o|G84+7OTF&7AgfIu$L z+zDZL?h-@@a2g33F^16=onaUyiQ>V&#zMKqG{XY5zVg(EEoW|ty2=qTNCFb2zQY$o zo^o@T^Zu~q$y9m4JZ_Swf1I}AF6{&PEtJsbHzSy5o=?wno`rAo|NJzOCs9t!uR}ZP zmb&3r$9}wBwdr2e6du?EalC9rt(i?-ysMg$gTeuC@RR4%e4gu%_cYXA-lr94<2z|> z0pla54$iA!r*~5i)iKmVv~zAJor|zgmBNy?G`}>ItWbJF`{q*4c3)bh8K2W_NzmQTGk?4;fvzNXnFnO&1m#XM0FB2iRgo64l=zg9B>j5zObu+bDf>T!hq zcKSZXX~vp_ZkRA5`o~1_2h^(g;fEaqHGuBivZXCMQ?})xRZub^6G4R1Fmq0_rw)#1 z#{4i!fg-l(uPwPm#PT{Ublf^=(97Le`39>%jPA>`7WRqU+Z<+Ym6GKz>~n4J_vtup zN|wAZ_uKj)nA6)vHP?z*NSm7heZlQv#UZ$Mnefe>>*Yw?8w z>`y-Rr)#GJ%xQgc#awIg^unQxhr3lcgzJ~EZNw^$->@yS{H&RuO%g@AtAEz%!P z$<~gu7Ll7q^8vAI^ON48F*JZW>SUMfOYOTlUfqhxizQApaC7LTaz2gI=Y=FhPH7ov zv^n-wP4~Px+c3&#dpaahTmw9NjC!;)Hb0!9W8oF*uygWa9oKOfEd5sdjdCT+qU|kO z?R%JH6h}rhyeouB*PQa-D!g3`*O4-1ZZS*cOm@M{rn)!oznQJ(rJw%1?2K`e~CY`;O(m$cr52Ug$1^DU7_?1 zhVZ5fov%w}O+sWMgEg`*Y`n^7s2CBCjd4PgNOe#CgEpAK6*@->J27((ar#N39MCntM z>`Z7_oDw|gr4%e5Gk1-D?`gO{`%3g|yNkq=vE9q!bM_yCt}B@8RW@mm-z~<-a*j#w z({Voj~y?GuTm|wAvsQ~F12P!1OYKv@Anq4mZP=jX;p;aZH+czZvQDSP}U!_DtPWF7q zwTE$@2(>8OOV4vf4AN$NZCmHldEJ?JF)@56wymf~mTVJJNKyDP@+A`ZNg>mBES zR?5R=eQHE$Mxs%Q#9|9~Gs}2Z;!X-Ts$i*x7dmNPeZ}J>><9v(&^-Zb4pZ*5 zoPHnA5X;UKQ#HW^DbVvr+=RW68~yPv!)=Qy@%dvoF>cR7B7{@NpE37{^hIw_L|S-n z&&|dBl&x2%?U4=x7R;+hFB-#+Y!qa;Xz3ur=~HV@!fv`jhXB3Zrk00%wefc%55>T< z0i(l)8y8q?nu`LR<`T>M!^-&sLi_W>p$#LR9orF)a&{-L)gjUnq)xbtR3zwD-NeeY zVwp*<{nCY~HvC%ML{HwChDfJEZ=+ruG-KwxAt)&A@^*?_ykM>? z8))|xLv4T`y^iBq@~OC;ist)cNi6B4&w;FBHVyq3c?3ur0QM9uAH1`2l6>~GGyci2 z&=yC>i$Z#a8@(xKT3ybhDJoZaf-Wx_Qn`;3oE(*{uc0a1(1 zQ&Q*v$HMWa;!gTCGuOum%^%CrCu}E|qPC4>pEoio&s-h*_L^aA*=*c!-?5pAoBGEq znk!+^MtW`CkJTBm;QxUU^d+o5MU)N(=C!Tn@d6+aR}cjHg=%Dg4%N>q%-6@^N*E=G zOx2DI44^rV*iY%gXdi0t*Xb-}*;?A1eJjmwY(e8I{El#N*|S(zf727KRQV zzN0}7B7g3UjNI;lzfXMSc%3kimII1ct}F3A1s%M5aq@iJgZ7dREL2ty5FTh>+vMT> z69Xy+N@9wXAC(}=jUo# z(4kG2AqF&LGXIANn&$x|k9^&y(kOrB)EP1?@UYLF_@MXK3#mg?6{cfE$dYS-(LdxM zJqDw5dcKjCaD&(Z3;7A{El=y2P!A0obN{cOf@7l(?gI-!Q@h#{3taCdV%OT{;1c{VIWtPM zzaPt+jQcVJFEft4_)T|Srr<>p_kv}Ge|Jj(Z*7qn99QR&r%pM*j!;%CPBYQ0YuyAb z9K~Z~h&tAOYe1w?jLVI*VTa~crp^MHH^ar%n2Rjgho(k#PH->K&fa`v?C55E$fJdm zX)S*&z^E^;)|uE1|JsYB6h<5^ht>8xB8c6~Qf#)(T&?UKf82k^uW|FZMO4v+;%bC_ znZW9NI?XFI8e=Ac?FM~vx;LGMYJ9P@x&PnSSpN;?x_1w+3VF8`rj$2Q9 zqBB{mNx5BJyv+1dBf15q)%9*gQR8#{%m86j?v4w+-2cAs50*ik2zr5&@N|!eC?c*> zF?A+M>Kf)fVn__suJ@qWTsU(@izfpFgYNwB=`40XkuTboY%@z#7>m)dCJkLB8p&7d z0U4)ppD&M3ygqmMP8GjOU(Y>d=d{z9w&@sf|GS_Y2M^5GzcZ8BnLeN!(ig``s zPo(wLRwDb6a+eA6%sFY+KL(;KR=S3~rtNkGE9@`WjT&B3QFwuOHMQ-4D)PT;>^ zSY<^M`M=u}1d{zp>VGCK-a9u!@r(diB=@o6A%*`3`fpj9Xf9vTTwwo!TvMeJ0)TjK gq!RL9(X*AL1w3VPo0Uuj9?0fyDyaxvlKbWRC+*Eu*8l(j diff --git a/summary_results.json b/summary_results.json index e2bd26a08..4dccc1f77 100644 --- a/summary_results.json +++ b/summary_results.json @@ -40,6 +40,78 @@ "Submitter": "MLCommons", "Availability": "available", "Category": "open", + "Suite": "datacenter", + "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "UsedModel": "rgat", + "Model": "rgat", + "Scenario": "Offline", + "Accuracy": "acc: 75.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline", + "Software": "pytorch v2.4.0", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", + "ColorKey": "availableMLCommons", + "Performance_Result": 10.1626, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0003", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "default", + "Platform": "default-mlcommons_cpp-cpu-onnxruntime-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 49.593", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", + "Software": "onnxruntime", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.431199, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0004", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", "Suite": "edge", "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", @@ -70,5 +142,41 @@ "ColorKey": "availableMLCommons", "Performance_Result": 22.9844, "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0005", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "UsedModel": "bert-99", + "Model": "bert-99", + "Scenario": "Offline", + "Accuracy": "F1: 80.0", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline", + "Software": "deepsparse v1.8.0", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", + "ColorKey": "availableMLCommons", + "Performance_Result": 6.27262, + "Performance_Units": "Samples/s" } ] \ No newline at end of file From fdb012ff901cdb3e5895145e82e61c55a726755b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 7 Feb 2025 13:58:42 +0000 Subject: [PATCH 005/719] Updated results summary [skip ci] --- README.md | 35 ++++++++++++++++++----------------- dbversion | 2 +- summary.csv | 1 + summary.json | 36 +++++++++++++++++++++++++++++++++--- summary.xlsx | Bin 7968 -> 8107 bytes summary_results.json | 38 +++++++++++++++++++++++++++++++++++++- 6 files changed, 90 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 59e8a60fb..e0aaf9c84 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,29 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-07 10:58:32,810 submission_checker1.py:3263 INFO] Results=5, NoResults=0, Power Results=0 -[2025-02-07 10:58:32,810 submission_checker1.py:3270 INFO] --- -[2025-02-07 10:58:32,810 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-07 13:58:37,296 submission_checker1.py:3263 INFO] Results=6, NoResults=0, Power Results=0 +[2025-02-07 13:58:37,296 submission_checker1.py:3270 INFO] --- +[2025-02-07 13:58:37,296 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-07 10:58:32,810 submission_checker1.py:3276 INFO] Open Results=5, Open Power Results=0 +[2025-02-07 13:58:37,296 submission_checker1.py:3276 INFO] Open Results=6, Open Power Results=0 -[2025-02-07 10:58:32,810 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-07 13:58:37,296 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-07 10:58:32,810 submission_checker1.py:3286 INFO] --- -[2025-02-07 10:58:32,810 submission_checker1.py:3288 INFO] Systems=5, Power Systems=0 -[2025-02-07 10:58:32,810 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-07 10:58:32,810 submission_checker1.py:3297 INFO] Open Systems=5, Open Power Systems=0 -[2025-02-07 10:58:32,810 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-07 10:58:32,810 submission_checker1.py:3307 INFO] --- -[2025-02-07 10:58:32,810 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-07 10:58:33,942 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-07 13:58:37,296 submission_checker1.py:3286 INFO] --- +[2025-02-07 13:58:37,296 submission_checker1.py:3288 INFO] Systems=6, Power Systems=0 +[2025-02-07 13:58:37,296 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-07 13:58:37,296 submission_checker1.py:3297 INFO] Open Systems=6, Open Power Systems=0 +[2025-02-07 13:58:37,296 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-07 13:58:37,296 submission_checker1.py:3307 INFO] --- +[2025-02-07 13:58:37,296 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-07 13:58:38,216 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:--------------------------------------------------------------------|:--------------|:--------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|-------------------------:|------------------------:|:-----------------------------------------------------------------------------------------------------------|:------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1626 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74129 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1626 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index c9a777fb5..636b54bd0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -632 +633 diff --git a/summary.csv b/summary.csv index a9d9fbdb9..141659457 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,6 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74129,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1626,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2a27ca826..96122976f 100644 --- a/summary.json +++ b/summary.json @@ -30,6 +30,36 @@ "weight_data_types":"fp32" }, "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":1.74129, + "Accuracy":"acc: 80.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -59,7 +89,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,7 +119,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -119,7 +149,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 11c8a95f434ad1f5dfa032f55063ba14e2758b75..9618fc1c9808ed448ec4e806ccd41b81e508222b 100644 GIT binary patch delta 2449 zcmV;C32yeFKC3^l)&~l*6*gtx1ONc~6_elxF@FRBqbZ$EFF;$$ZnaM+O&o|Oi8HZ7 z>9=1y0frpAOh;O&^8$(UI~O1OA3N@U-Xt+viLRg_U}*bgH9O!Nok zI2>qStlsF!)r?`k4E;}pU+BR#5` z8W8V?DBgP5>>F$0cuPD&lleh|!K3X%+Uk(D6v3=Vn8`#7Add>fsv~m+V(*b!qJK?w zqD{#G2r-k1Hpugr#QYdh%d(};vL)HuAWNBO!7=?cGCzvc5^XCJ{nC~kbm}3_WujXr zMl#{ni3^#qz})$&K)}FWDiBcBZwdsI>`H;O>d3VM0Tbj#fjD*KR)K)pjui+f?nHrr z>P}_E2GzY&AfUSU3ItU5L4mw6+kYJOY;)x43Tl6@K;V3n5gQ;E3ItT`tBf>3)h=bE z4JX_VQM`z*6cOM|J7Rvbd$xO}-@=BmB0msd0&S*>`hviB3b&xX_doLvhgbjJf1Qo5 zr?;o&)`0T8!7^g2;#v1eVk9yDP58RcuQHs6izp3o&UY_(!7ybDT0AEa6@ME6x>&b$ z(_lHq<(9(`I}2kn9&PP&O!sYSDG%5eILxa|HtA zGLjKVbP5EDPDUir$w(W#!hgSYj)+bX0gCR3`Ar9k4mOMx`GImypv_ca1D1k21pc?!lI^CO}s$KvxzZ{KS!w zHkg?s8Ic62K%f9+L=vC^fdX`o2vA`T1$e~##)Sfe4P!;E!24_hZKjGL0DZcX5eI&t zxPO`e>q}sOamMFgPJgjj1iQNV`x5+!9==0@bN4`kj&65s2cFp&2WMLe&X;(E==4TUwke$X;jD-;Vy_t< zXn2$HVG@1SP_PNB(?OKw5y>O=T16M9lSS+e6B3-3Nfl}IqgBACC;o}kJMo>fPslz) z1H^m2LwWDYDCO~;BCTOR^Oktx<;y6gD8?c0sHyYqLSCIWme-7Avec!LJvX}NDpFlzoP$8gG)zYF;r0xlsMJ#djRI+7faRbi_-5h7s zLJ2FWRI+~CLM6?47vO9ir0hVWGR@?rzH-b6 z8Q40r|QCvf;fy&4n~xlI<8aDW2X&`+~+5$)!ujf*a=LnqvsvtF8)U9G^796*?+42-dqY$ zhyrr~$M(ABN1-7i=T(22V4eS^F`as|8J%r572hgFRi*XWH=W5x^B!u)PA4{Udzz%K zS2s!3kcD;BwO?)e2Z_7tuV(lIv+EFH2n2)o(6y6L86SVkYQr!PMfd#*qrELZNE3wZ zKwGkEpwN_-?1VINM6CysS|@*B#c`Yjyy|jv@0__as9YXo8~uWHLaPd8Ns1y+Omm@{ z3Vm!=@d8E8Q^jek6;$X54lV14k{QCZg*U4WSTDdu0i+yZrb2t~4Z+y4JxJ=3;7~!_ zX)CD@rEPz(q3oMBfb%qc#1cH^)Kh!{#%3g<0m<2jG(WcO0i0oIL4xuQXGw;yUJq8f z$%bCJaZ2Hj1}3c+adaP?7>!-mC0)^5L;ehZtzX~!Rb!z}r(%HWk~6|AP_J!W;@ct| zuBFbe!-;nS_|q}y6K}&J=RV+32tdwpFUn`}{IVrkp*m0VS)Ar^I^SdknJq{$-=0R^ z^boEQ?9q0ssIL z2><{T00000000000001hfoGHY9YF$i5tBV0IRWUCcOE7LgZI$2lc63K0m+lN9zFsc P82z$*?aJYJOYz3Ws-spJo(DV_4JdgO;Cp1iKkwrV<^1JZcc9iOe;KyF(g@wtw}B zwiO2;#7rf+AkW_t^Fu@<%Z@(Fj$-eEELEZd$MpBe{2+8f+kZU$+vcgSE2#aM1_4zYtB4D#cCJA{)xN4o8&vH= zMY;g_wna1_qDxH#_@*5&zurFEe$sDX!$gxGaB>T6?lkoUHuoC0An)nVyyMa3-_x(N z$<^J>akVsHVr{TY_^Nz1Ly}opDtjmhm>H=$pGYtZy zGgc8rfEok}P(>60Y7i(u|9}8B=1_nK%&&bYK-e%*19U-tx6tNJQ-5DzbFXm=MrrE* z-vlVO&b8M{d=bWlc>gHFix=<@6aWAK2mk;8KLF=)0<-Q6hXD%O!8T&J0ssIt z2$O#f8h@2lO>g2b5WVlOu(Fr-kPp5Rf`AGEU9DPL1V!qeBI68Ttz&0xhn9c8reUjX zoK(9f#G4tvnKy4>c=MVubOfoW;3KEy1r7pEL`wO7gv_uuUw`G^SGfe7ssn(|nBNY9(9baC4kDu9 z+Rz4F2Nj(DDqvb7y~r646&(-tcp*|?Ltl@F{uwC2H35z#6;*u0sa92R1*%}WiZX^X z+;*d2>&9_4__*6K%ApEBiJ!ogZ!vtI)ih$JLEmTS3cS6Cj2viEq*Py5GpB+Hc?TzJ z#D8L&QLRB%1Ct}B3~wo;+AcaN4Ota^FcMllFQp)$l(62bu=!FP1;e`>H#$%?Hx zg2H9kD5jIwe%Q?0wRzcPb>HX7iTPpmHUjM7wshPPoH6Hpwi zUL{;vo#pLWvi~#N$31gQ0dkdNsUSJFyuN*r4g0xdRpGxHw=4cIp^~ZI zIQC@OWH~_H*z3jC(n}kLbvC>;#_xY{u5bSL$G-uyb`fC+1c5byvXk2xAAiki!!Qtp z?|llRyDix!Z4kBtZON&DLQ`6L6Vk>JwSJJ)I)3{qPU0kxQ;(~e{dRUywb-dH`32{t z(KX65f)dc&2&vl|y|0()93?(5Em&tX)Mx;Y7R_D7E#=0+t1}i{kl>RD(w=f#qiqP5 zV(j@A6!Teh=qPTCQ!K>NwSU+${>@sz1tIrXL12OfhWEhKPDFG>3O*t2kLyN&;21hk zpo7PGmgBQGfK&cv!>C+1l?;Q0o7R&!xp!Vp#=h^fz8tMFe~v#_FK^?jsnq*ZaX?Ka zIOPslFs`ZaWf2e8F&|d(#2X3Xaftejx8ov*J`hO^Ko4 Date: Fri, 7 Feb 2025 17:37:25 +0000 Subject: [PATCH 006/719] Updated results summary [skip ci] --- README.md | 43 ++++++++++++++++++++++--------------------- dbversion | 2 +- summary.csv | 3 ++- summary.json | 40 +++++++++++++++++++++++++++++++++++----- summary.xlsx | Bin 8107 -> 8357 bytes summary_results.json | 42 +++++++++++++++++++++++++++++++++++++++--- 6 files changed, 99 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index e0aaf9c84..ff7df7c70 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,30 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-07 13:58:37,296 submission_checker1.py:3263 INFO] Results=6, NoResults=0, Power Results=0 -[2025-02-07 13:58:37,296 submission_checker1.py:3270 INFO] --- -[2025-02-07 13:58:37,296 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-07 17:37:20,100 submission_checker1.py:3263 INFO] Results=7, NoResults=0, Power Results=0 +[2025-02-07 17:37:20,100 submission_checker1.py:3270 INFO] --- +[2025-02-07 17:37:20,100 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-07 13:58:37,296 submission_checker1.py:3276 INFO] Open Results=6, Open Power Results=0 +[2025-02-07 17:37:20,100 submission_checker1.py:3276 INFO] Open Results=7, Open Power Results=0 -[2025-02-07 13:58:37,296 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-07 17:37:20,100 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-07 13:58:37,296 submission_checker1.py:3286 INFO] --- -[2025-02-07 13:58:37,296 submission_checker1.py:3288 INFO] Systems=6, Power Systems=0 -[2025-02-07 13:58:37,296 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-07 13:58:37,296 submission_checker1.py:3297 INFO] Open Systems=6, Open Power Systems=0 -[2025-02-07 13:58:37,296 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-07 13:58:37,296 submission_checker1.py:3307 INFO] --- -[2025-02-07 13:58:37,296 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-07 13:58:38,216 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-07 17:37:20,100 submission_checker1.py:3286 INFO] --- +[2025-02-07 17:37:20,100 submission_checker1.py:3288 INFO] Systems=7, Power Systems=0 +[2025-02-07 17:37:20,100 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-07 17:37:20,100 submission_checker1.py:3297 INFO] Open Systems=7, Open Power Systems=0 +[2025-02-07 17:37:20,100 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-07 17:37:20,100 submission_checker1.py:3307 INFO] --- +[2025-02-07 17:37:20,100 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-07 17:37:21,306 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:--------------------------------------------------------------------|:--------------|:--------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|-------------------------:|------------------------:|:-----------------------------------------------------------------------------------------------------------|:------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74129 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1626 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:--------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:-----------------------------------------------------------------------------------------------------------|:------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74129 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1485 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 636b54bd0..321183f60 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -633 +634 diff --git a/summary.csv b/summary.csv index 141659457..8ed2327c5 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74129,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1626,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1485,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27262,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 96122976f..302b84a92 100644 --- a/summary.json +++ b/summary.json @@ -30,6 +30,36 @@ "weight_data_types":"fp32" }, "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", + "Scenario":"Offline", + "Result":0.352519, + "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -59,7 +89,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,7 +119,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -99,7 +129,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1626, + "Result":10.1485, "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -119,7 +149,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,7 +179,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 9618fc1c9808ed448ec4e806ccd41b81e508222b..22b3b254d6e25554baa7a025103df2212846fb77 100644 GIT binary patch delta 2708 zcmV;F3TySNKczvi)&~mY^llAu1pojM8k67$AAgx$Z=*U8hVRd>uuw0y(gp$vhT z+w{9>o6R<7b*>@`CP53B#U^e4{TfKpWIRo~(vh+^37BW>nXz9yK<~?Q9FZl>c*v5z zX6ki~(8ObYn9Tdy^x?eL)(97F;=2(`XkUAwT>CQkr}xCNpL{{7AhLsm_qBx(Y1c5g zw|}6q%XOC0L|zG47P~^8W^;q58Fl^AD2@!PUU!Vx4HK>Ep__fA4;BQWM^BlT$21XD zCmD@gA>HSTFy-2y=ZErI5f8~|(AU12-Qf?jt{H<~IrhIHed1fkqzL7av7g1smEYG& z)KJ*-(v5LOd>XiUBqr?Xl7{nzkZH3E6Mv6Iy!<6`D069PBzB+sT0aBu(!*0%xPxBCo=8@X zz*D!F8MAAdGA&-w`mJa#d^1)JlXU};aTM8G4jMgUS#-Q2zC)7-*f8B{Fy!zXAAc;| z2TK{UzB}YZC7J;Fjv(eP@_-=LE;2=s*YI~@Y2dLmln#Iw6P2(*DDM&v?-J8}#7-Pd zJdUPf4^lNziDnZ7@;>tLJ~G{l?8IWDv3#|a4qCfIoTx;5ml&!hy}8Hk02nsV*~-& z{e~cK+yNYXwK?i?1&KXT5gP-Dwu)^P(HMu>lK`9PaHWD3B*_bTFml`wwwI~ZL4H$Iz;uI_F=l|>1cRe#Am6pQ>( z_gHMik^GHi#UYPUngz2k31}wkCOQ+8iy6-!5t%(z4hwikil_9Sv9Z z8RfE+;Kr(^sBLo+Z;n(-dFSFYF^?zo=G&>z!bS+$TEp^7ht?EQYY_KSXDx!!EL7?bVL=hc= z0ADUzdqjtrgB{>%kNGu{32@WC*)|8d7$KvBJH~AbDiWC=;PJhRH~<;94yefaV&;0q zV?|9qMO0nY=2DO8i(r|zm@O^c(d#uYH?8Ielz50xKA^;DYe$I<+kdjnHVuLj{R1rmq2m+O;B8n0bU%-vyqP<6n$Ov%z+hcy+20FQc4I{)q5alhjxl?T% zxcwmr@O#`otl08Lxyj`<`6w^D*wS3F2a07}?t` z6)ZlX&7Eqas1HHF4m$4qMe3V9x1ImH5NVe7z1^)?8Gaic}0Wq*VW;($nQ6{|aJK0s{u zSkIse7;m}%mnd>~PxL8G<$BCGEq2AWEJgoVlLra$1EPFbIaaCC(XE!X^{`&4(%5?A z9n)-1hct>f@tDk7Ww~jsJ_Ah5?zDPi{)!i`+!H@6s(d%9-tUVrnG^o2`f}iOhd-=( zsr6uzN0f+{lz;X$`kcy%DuUIB{~T1`3`m-VEDOa;HG1*UB9Gib%=}NKQ+X2p%IT*g z@5t#MdCu`?VjYt{k)f|q8C^L_`BEQoYtT#GIlXnWd6;k#(Le@T*JS~pRbL^>W5Lq$ z$kxas6N1I-vjug1niXdajRcH|^+}PE&DWg4Z%|7E1he4`lmQC0Kj&*i0{{Re3X^{h zCx7iBpFlzoWkNtFw0%BEOXOeK%Qusd(x z%qw=-5Ie3lL*XS)b46$1Wot9=Qk9l~!N72grcU3j-=hcM?D+dVvV(p+crb);Jx7bx zml=wpu!q9FJ>vpUw|MK)pt>cmMEhp6t#Iqx`;QBMghJnmyue531Oy}0i+`LrjN&tJ zrGeP#g?@z4-R-Bv%_dnb*2!jiy}DW)cY9_!oTeibX*GK0O+Yd*?jk@DhFbz#5UNFE zdT#5s*%`_6>Nwx+M{|ziqNLC=(at96b>RIDe9(a#{XgST6NhXEO-G#dZ5K4Hkt8k4 zlMy&$k3uVr6kxcZt5|Ir+J6kbGYy_re$aRSqO2V`ese{`)9O&1=p+Z<@na`2`5YxN zFr@M~sZJy0AFKDD*?uRN0hF@DLc)>W(41Kv3O=v;(||GlYCbRL%PIN@m%4j^)^`u6 z@5Kh0U-&me&)q=hZkQK)CWLB2?G82zSWH(V)Q_D(Y*=e{wOl2e**)!A4-7xl!(`+`32{t(GAKof)dc&2&p#>`dH4>N0j)$v|ydl(4akdG;8l_ zZYeho-kh=Ef&`yLkoJ_@25m#I6l2e~pqS61Lq~CKoMIuCZhwO<+L4G(NWn*>{c+s@5FA4X3Uu%|&vJb625`zxHVn#@Q^~NmFljxDqkHG&Xzcqw z>&w9!^XK?$@%lcjno4~*6$jMTf>Z8*1>@QpUl;Lk9rIxkPrQ~Op7v3ncsng}>;sX+ z0Q4C5tbCTwFD1)4Y70`_r=&KwB}E^$dgh zD}S7LR3{lu`IC_3iO2YDzBFi-V=pL;lGw0K(=ie+O0~*EFMp2@5{8kFFNj|xIAxWS z9LFBx>C;t|QEk`{BK}PD2jw^%XkV<}=*iWLVZRLhPlR9S!8H;+c_!q$xVa7nnki62 zVlT@q?sF92&?{m#BQIAtTCNzcT3eX-B!8ymFG?a_N?SvTw;5XEf|P?=iJ*48quy|t@bEz_~vVmMx7ytOf^sAYy19`lC%oV=jCj6gF_ z42;#YO`;ns-@nXtgz`Kmy`z+Ng4P}Oe=1eEMbfwbz#wE_VX zvoofZC1~2q^AEfq?2xWq-s5)xA?7pt|=81XTAyfxI!>9QAB-tBTZ?kg+0)ON(k`YOC3IvKyMkLY6NE^Juzjcm?P7wi$?uhwK2Z|0h zj1~ETa!#PlRAB>_f;$BQig(}nK+!E%i{hC}t7!0TJa^IHyY|XssXVq$O7$()mTtMa z{rmF!h|ax3ezVSj{4BlGvg|k1*Hx>pD@zM(6wYO&4YC-?h@?IR0)N#fBa-?Q2wd{I zN7SbBUeZKetvs6H8SKxOW`e~|hXo38VJuYVuWzxN2RdmupD z*4^&Gma!&4S1mwS79jk@k&!l-nIjpI1gJou0A)lHpaOvcbdLy7VGad&#QesE0)!1? zMXkX5YyxejiXi}fx_^@q2Y#Tqf0_X6OJIO;#^+y7u~-DVy7~JO{D>aDLxOYnK!T2L zcWeiq*%$|BTM5pWc!cAaBA@V@RhIkK>b!4e_2$*t*iT#>uBAaFP6@rZI>}>Ex0RNz^5nviPJmrowHBKK0^bAVL$Vhc;e;DD5WUIA@8WE^X)=j zoi~=(jAZ2%|8jmNjFDt_x58e4b8*+yP)G>d-H4Lxk8Owl0kd!om;nwgEQ?9q0ssIL z2><{T0F!_p7LzOw4S)6_pFlzoP$8gG)zYF;r0xlsMJ#djRII)%} z+DgU1+DTdTTvsGJ$gyx(0aP&?#&ayyE8n`J;DA%H1CTj!1Aos8-5h7sLJ2FWRI+~C zLM6?_DS3&E%!Na?A+hw{S38EZ2D^ zC2*Y>?Qur%I)5XXG&dcDgj`4Om4!ylbIuYFf*I!*pAE$%0RkLLb1^m+FW5%DU~YJK z&!kzzQb1aJ!Ai2^I58TWfK=TZ=Zw`EI@WPK+w(lX0S|OM#S*JnDaYMohTYlSnB_TB z_r2Y%a3a-&*c>*%)qsGV6s27plq1P*wGW(79h$-htAC)a5=J-K*8MPGABvT0H3X$i zuyRbUUb|tlZrjx5TWo|KSD7L3uA{GLB;L2`#JejsgwcU9F{*WP@py@zfzjjhC9;Ed z+<7*FFq@$1{OcG+QP@IZ+a5Cxs5!iqHK@9h57@TJ)*1Ksu>3Ohdnojs$P0XgPC(E@ zt;mVPD1SZ!cM6D|R_I3vJw4n`?-#52^l`Nq&E|L0<8IHC!%5OZkxHXyRsw>8Q40r| zQCvf;fyT-1NoVhcoTpnZ_{epEus|saFiXu=(X;FTh0sq#R+ULVNEG!Pv1qNa~W{P(j>jE2$5qZGW(# z?3*@#^E7?L5mF7D-3*gMA0ZnpEQ?9q0ssIL2><{T00000 z000000001hfoGF{A3+Ru5dZ*qY%g Date: Fri, 7 Feb 2025 19:29:59 +0000 Subject: [PATCH 007/719] Updated results summary [skip ci] --- README.md | 39 ++++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 1 + summary.json | 40 +++++++++++++++++++++++++++++++++++----- summary.xlsx | Bin 8357 -> 8513 bytes summary_results.json | 42 +++++++++++++++++++++++++++++++++++++++--- 6 files changed, 96 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ff7df7c70..16584548f 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,31 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-07 17:37:20,100 submission_checker1.py:3263 INFO] Results=7, NoResults=0, Power Results=0 -[2025-02-07 17:37:20,100 submission_checker1.py:3270 INFO] --- -[2025-02-07 17:37:20,100 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-07 19:29:54,330 submission_checker1.py:3263 INFO] Results=8, NoResults=0, Power Results=0 +[2025-02-07 19:29:54,330 submission_checker1.py:3270 INFO] --- +[2025-02-07 19:29:54,330 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-07 17:37:20,100 submission_checker1.py:3276 INFO] Open Results=7, Open Power Results=0 +[2025-02-07 19:29:54,330 submission_checker1.py:3276 INFO] Open Results=8, Open Power Results=0 -[2025-02-07 17:37:20,100 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-07 19:29:54,330 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-07 17:37:20,100 submission_checker1.py:3286 INFO] --- -[2025-02-07 17:37:20,100 submission_checker1.py:3288 INFO] Systems=7, Power Systems=0 -[2025-02-07 17:37:20,100 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-07 17:37:20,100 submission_checker1.py:3297 INFO] Open Systems=7, Open Power Systems=0 -[2025-02-07 17:37:20,100 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-07 17:37:20,100 submission_checker1.py:3307 INFO] --- -[2025-02-07 17:37:20,100 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-07 17:37:21,306 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-07 19:29:54,330 submission_checker1.py:3286 INFO] --- +[2025-02-07 19:29:54,330 submission_checker1.py:3288 INFO] Systems=7, Power Systems=0 +[2025-02-07 19:29:54,330 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-07 19:29:54,330 submission_checker1.py:3297 INFO] Open Systems=7, Open Power Systems=0 +[2025-02-07 19:29:54,330 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-07 19:29:54,330 submission_checker1.py:3307 INFO] --- +[2025-02-07 19:29:54,330 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-07 19:29:55,365 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:--------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:-----------------------------------------------------------------------------------------------------------|:------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74129 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1485 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74129 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1485 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 321183f60..7d71bf4ad 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -634 +635 diff --git a/summary.csv b/summary.csv index 8ed2327c5..426430485 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,6 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0101,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74129,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 302b84a92..add7d52da 100644 --- a/summary.json +++ b/summary.json @@ -30,6 +30,36 @@ "weight_data_types":"fp32" }, "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", + "Scenario":"Offline", + "Result":49.0101, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + }, + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -59,7 +89,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,7 +119,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -119,7 +149,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,7 +179,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -179,7 +209,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 22b3b254d6e25554baa7a025103df2212846fb77..8a03359a835b88d71e443f8d16ec1d27b98bab03 100644 GIT binary patch delta 2891 zcmV-R3$*m5LBT?>)&~mm7y=~S1pojg9+Th)9)I1LU2mI85Qgv1uRtgltyBYx4gQer zs&SeR$4Oi_Np-F)0*j3bEQQ5M{{30NcJ0{-ait^q=-eb`d1iKImKO%L|M?{fl@(31 zfW-rqXojlL*kxW2F9zzv{duFKDj9cT&k0#f2kI-$)X&4e`p+!=nJp>hif|BT19i!H z(tp$S%w5vR$uySGSe)@$8aZ4XrVBkwQtEi6Q55Q?Vc2@)1hHCq=%t7FV7?!?^o+TA zL}Ok#NonYC5q`D|l1v@;y+E8R@=;Ri57aNDH~K+{st^0++>Zf$&USt)MJi8>{VWcy zyn$*Is4lVRWf;>`@u=_QA)m45OByVeTz{0!Dok7!X60Wc3PdeURf(LJf!b7+=fLC3 z!m+XpJdYNIRuwnTIE#K%r^wntWlHR|EiuT}x~q|vVVhR59ROwNn|fKo}z#F+tVN5Cl~1tBf>3 z)y|Pvz(f}t1i2$#A~V1%t;T%3Ik$O2UBiKiJb?|I+(4UKWC?)H9pV;T{(sZoar?u| zzo%bkldIe74`nlyiEd~S@MZp_xh&G7Q2dQVrz3tPH1+2}?9){AYjn??^Rr+YB97{laOu~7hM|{U0Ysp zc#oG63*0tGG9rl%L7?blL=qi>fD81ZT_ZZg9HeqpV?LJ6Z4lt~t~t0hO_14vbZ**E zk;wXhi0_c7z?Oc(B4sEtE(B}O$QfZ2Nz;TUT7epQyH6k-Po5m_Hl{5vEna4(&9{y#!IMk)sg@v*b_NNifBMc#NMc5x+k z<&_2kIh7F$TtFikk$;qkAW(@iA}JAx1$epW)+iB~0rsIa=Ho8V$u%69$P?H=k2lcf z7O{b1Ll98kY4^Bd%b!HInKweGaIvGgq8a6@sBB2*E25FEX8w6cdW_@_kl(DkBR`_G zTf}@reZ;`!SB`@V8ccGEF`(xeA&jIzjDZqF7)gT|1HV3O8GlTLm;)%oItSzJ$*mBj zkqI)1^hzjM-C|Zyj2HvzINb^V_;Q@TbnJio9q}j$9wNzgU}XK>M(cLo`o2r>kL+KF z{88N^P?HGLhMJTaS(6AOX%hL!FKH5CBu$b!O(G5?P1ZOV6S$3VEsac&N$e)nB(%Ck ztl)?LJA`Q)P=Ax;zoyCk$D~JTBCf|Yqs4==Bf8T+SLHEMJV2HA>%b~i+M3xmclx4A zmHN&uSVYqW9nmn%6qkvrRhHY)YHM&tdXLpE+HYLU$6_xi8h@Q&oJ%_~xJ}$##c#_z$yi4VVE5{T_Z6bprqZ`jZh4 z9e>+y+Da6CKVLEOd1)Vf#rT2<6#^2W5`rM9(kE-Mw~bqS#%E?slfOR$CTZ)j(^i#` zBG#UnwJ&QgGXC&X7U&3!=agJ&wqa=qNKBKQWLMh5{k0xwhzm>-Tu=g6+B0zN!{nba z=K{&AgkNa~A*zvS^7sHH<_4{R$Vf_AiGPLs$}*Ez3~<5^0HQ2R$Fe-L#5vJWOlu+} zt?Ov0Ci#Cg%o@bEwaJ+0ld+iG(gcdJDJEm{1T^3-27*~m+xP>?1#g2};B_I|Xi?x2 zJGyV}^)PG)@6xo$3AEvR`V|N_6^7-!BN15*TF%isFfs$lcA#mUF+mRcieFJs0iV9juxBGGvxc;0C}!HqYO~Lc$?OsW+ZP!T{YU(xb5A;&40okA1ops0hsLr>eCdFtoy%$JNuH+l$@cXp)n0K~70_XHm)#lk;BF5-?7vw-+j6EDbEoaMG#IZcLDLUGQ9U6|>r;{>=iei6<>g;5HIF-L-`?NXO! zf#q~FLT+dbLuHj;9B<$l&M^4F@jS!xJpa4lIgT+5EDxcZ#X4Fo))Mg)1DLIrn`k$? z+e#O-J?W4^XgOhEJEspq*UR}!)RB%FI72@O{m>g)gMnfWXTuDvF2iiR)awde+s&o# zxGDd6o%|0_O9KQH0JC-yTL=W3655ZG)fyjv%TB{E5JmTVMdW?rB&9&DB$WVlMI|JP zK)+}q8`cFh%`U8ZULNOs6m4A4rfV*&t4B!y2*xCxo}G1 zj|L{KgE+bmPK?H;X_98vT0{N}f39EN+ErtrPN!mks)94ZEKsj)Rp84a9ImF$ufvJA z0{G)G=o4@KB3&QwC*HHoZfG;1<4m=e*1MAdDVXg%6`bd zi36a^efnh9Geo_k0k*B p08jz|01*HH00000000000DysC8|tRWgY+k003QsNk0Gp delta 2721 zcmV;S3SRZWLZv~l)&~mY^llAu1pojM8k67$9)Hc5U2mg05Qgv1udq-rw$cUy3FJdb zR@?NuX`9V9XLYV33MN4dn8hY-|NR8vY&iGsUWh0g!i?D5NX#i zxPP~xvCDOq(nMYfSQfiNo@R4{rx|tq(kPA$t6q1E*bNh{>Y5_)?g^+2p3x5-jMZEkaaVT?XX(V=^`&vUIPoXasMaN_j z`aUfZtr0Khg2g|oOXjMB%G7jLwr1VjSa)=@S$8bEn2(o;u565I)Y8LKSGa>-#-2!4 zj=)p5m>IKcnKCV2(fX}uE_^dq4U=^Pk#Q8+TMimMV_9^(BECbD2iP#(YB1#R8-E`x z+y_e;vc5ayL?xO4`HmpwF7kjN)-Ezdkk{~cVrk&9G?Wg27Za7RK`8GM5APDweZ)>2 zO+1dKVh>U^QHf>~1oA%e@IEr#i|oW=qp^Iol@40FL!78YdzTohL}!;cRS6TU(K7@A ztK%F&K;kbD1W5D}K|ofo5Cp8DYkvfBc99zd0l6I^2uSWNf`IJasfY!#dygO>yJG|a z+5LtfZ`=VKe6>01as`P!Q4t#iGgJ`^WbG6|K-SJwqyYvxM|=V9UaS%1Zgz=`09V=` z^Xv7s_119%8%BtKaQQ%+J7fyL!1pTRfR!+Q^*b0`{x?3GjIQo(K9xlYmw#2sJQR!k zQTJGE#F6}sWyK+nQkn&`FbQZT>n1uAl#3b9ALEc0B?zA_ZM|-YjMB2$F^ZbTxE&2w z_8H}}l;Fmyrl@Uto_T3n^U}P=lH@td6CsP5+KQTaED6H7_F40(aKnge$2}uJ_R(*L z2uT)?RJvCFz2VpQ$}h9p`+xOj!!$b`Ty!m5bS-tp!L8~p&gFD7;3n~(sAK>x5iZ}onw+^Vt`C{gJ z#bZTHK1Ebr)#g%<>5E{Qx0o$0-O=kcFE_2`2b6e-P(Gl_Dw7FAl z9Ju`<2=IH{KCIaCN4d%6HTftnyV%lPu?LD}T<(LjrKy`8-TL#5@DRa$Kz@_;{DXG?&VrFYtIy=yS*xB|;|AVHvv z6NItB!W|-vB0`LTDnuAXh8P2NxM46MVjVyw?(5)sb8(|ZrGI6F4B~)DZWXILY(7A2 z_gK%M3K(y>|CcCocTe;wP33ybI4yR?wk$>eSd#|{@dKiKSUFay($TG!we_%GsnXbb z;~mp%PKPv#IPsXwT4lLutv&-x%tg6 zBk#!R9(m63XJQ?bK9QlXQ5ju1O8HVBacj^^-8sE=vw4_s645{gTGwR(pH*KW%45OO z^2pZ6BNKwf>$3%QeVP?#4UGhhiSrI0Kj&*i0{{RelY9>y ze_e0dFcf{?uduS0?IE8)LJ(y_Kqs_*AZVxVDKg0oSaa;mc3AuGX9(@KIO#S`iiC2m z?dx;zIhTw-JQfrkKuE&bh2=Y*g@C0zBW!nJ-QQi>LkmfTS%xWRaA7@xv_4GUjHOga zZ)Nhr+ACF#Tvw)hD6n*R2~09m2) zQZe$af?17(ku@1hG8wDMHP3*KT{RiIN1z5bDKIPuZ{qiiDA@$pASyw{pcuOcz=B6j9plG5ux`f7^SAwS?B{$A5-x?Xy6etL+%*EJPyy3a}j)hUX z8?MYt+)Kz>8>~}S0;dMSIVjEEIA^BMXkr7$vpvuAJMf^1XINn^D_wE-sA2c|YQ~F# z>wfR{2~L%Eh%F!op$!6dT2^-Xf21pt?vq2{g!<47a;$}xq@3ks=e{biSCds+ZGy_C zSXWFXkHfG#Z{N%-cGwU*t~Ep9B~NojXW(UPGw@QCmVm*)aEzu-->u)H2jJ}Z`#rLQ zemr(ZdQC9g#LX0)wv>)ZQ}3x9+{f8U9`z(?o= z1S8aooH&f)GjOGW*y)9SgwWmXr^U@CSuNJdW_rE4S{!$KW;&dvBNS;hdge_)GBEBU zKoN#p0$UKOMPqty>$ce$$@A(s-|a_pj^d)E&@$1^Ch2wH{SJK4fgAll<53fbYzIw8 zob_!NG_8>&Ez6S;IAf1Oe=Cg?V7Q>GSZx{F48AiBo>qR)cmJZS9XWn;MZ?qTP@L!_ z2jB5yCouUOB{49h@;9kYBjg{e_n+B*Czk<~vcy8dk>1dpSse;Kulmz~G5%^kFXziC z`Usc0dw|w=52)|O2AN;@H$%_eK<93l7kegzYC`P}HVarxS0mJqNS#4!SZj8*TqT>? z?OG2EKh(oy7<)lH)E`IsLFjThe~t!v_zZ(VG>oG-9C*E+VXxC{8G3EAr5-zBTj827Avmd`IG%Qore6Icm^DE z-+qGj53{oz$^r`H^llAu1pojM8k2_~RsnF6-X9_ywLj-;L<0Z-B?MG00000gWwC? diff --git a/summary_results.json b/summary_results.json index 1a5b49bc1..a0a510c8e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -115,6 +115,42 @@ "Suite": "edge", "System": "gh_action", "Platform": "gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "UsedModel": "gptj-99", + "Model": "gptj-99", + "Scenario": "Offline", + "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Nodes": 1, + "Processor": "Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node": 1, + "host_processor_core_count": 24, + "Accelerator": "NVIDIA GeForce RTX 4090", + "a#": 2, + "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline", + "Software": "pytorch v2.6.0", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", + "ColorKey": "availableMLCommons", + "Performance_Result": 49.0101, + "Performance_Units": "Tokens/s" + }, + { + "ID": "5.0-0005", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_action", + "Platform": "gh_action-reference-gpu-pytorch_v2.6.0-cu124", "UsedModel": "stable-diffusion-xl", "Model": "stable-diffusion-xl", "Scenario": "Offline", @@ -144,7 +180,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0005", + "ID": "5.0-0006", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -180,7 +216,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0006", + "ID": "5.0-0007", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -216,7 +252,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0007", + "ID": "5.0-0008", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From adcc2f2184cead79a3322548f9f7000e16201ba4 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Feb 2025 22:25:17 +0000 Subject: [PATCH 008/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 8513 -> 8515 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 16584548f..ca93e28ca 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-07 19:29:54,330 submission_checker1.py:3263 INFO] Results=8, NoResults=0, Power Results=0 -[2025-02-07 19:29:54,330 submission_checker1.py:3270 INFO] --- -[2025-02-07 19:29:54,330 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-08 22:25:12,884 submission_checker1.py:3263 INFO] Results=8, NoResults=0, Power Results=0 +[2025-02-08 22:25:12,884 submission_checker1.py:3270 INFO] --- +[2025-02-08 22:25:12,884 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-07 19:29:54,330 submission_checker1.py:3276 INFO] Open Results=8, Open Power Results=0 +[2025-02-08 22:25:12,884 submission_checker1.py:3276 INFO] Open Results=8, Open Power Results=0 -[2025-02-07 19:29:54,330 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-08 22:25:12,884 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-07 19:29:54,330 submission_checker1.py:3286 INFO] --- -[2025-02-07 19:29:54,330 submission_checker1.py:3288 INFO] Systems=7, Power Systems=0 -[2025-02-07 19:29:54,330 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-07 19:29:54,330 submission_checker1.py:3297 INFO] Open Systems=7, Open Power Systems=0 -[2025-02-07 19:29:54,330 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-07 19:29:54,330 submission_checker1.py:3307 INFO] --- -[2025-02-07 19:29:54,330 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-07 19:29:55,365 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-08 22:25:12,884 submission_checker1.py:3286 INFO] --- +[2025-02-08 22:25:12,884 submission_checker1.py:3288 INFO] Systems=7, Power Systems=0 +[2025-02-08 22:25:12,884 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-08 22:25:12,885 submission_checker1.py:3297 INFO] Open Systems=7, Open Power Systems=0 +[2025-02-08 22:25:12,885 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-08 22:25:12,885 submission_checker1.py:3307 INFO] --- +[2025-02-08 22:25:12,885 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-08 22:25:13,928 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,8 +24,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74129 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75985 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1485 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3715 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 7d71bf4ad..a8f78cee4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -635 +636 diff --git a/summary.csv b/summary.csv index 426430485..193bd9289 100644 --- a/summary.csv +++ b/summary.csv @@ -2,8 +2,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0101,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74129,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75985,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1485,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3715,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27262,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index add7d52da..12777da3b 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74129, + "Result":1.75985, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,8 +159,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1485, - "Accuracy":"acc: 75.000", + "Result":10.3715, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8a03359a835b88d71e443f8d16ec1d27b98bab03..bcea8ad64fa3eb8122cbf3bce0d8fa398a2338e0 100644 GIT binary patch delta 2278 zcmVC^)&~WI6hGSDliLS2e>Rrws&SeR$4Oi_Np-F)0*j3bEak&V{{7hn z+qGvWjw>C>N9QIn%QLexv%D~{z0WUkq%3Kchb-xT>U)wtM|;ZpZS8)Lg5aQyss{bBJJvW?k#BS<{C?B!p{UOi`{}Be`a$% zPc!QJl~Ek&reWB6?1qV2d+26|_+UX0di0EWWlWQzc9PM^EqM6(B2062(DOrnPUNFx zH0Y~eNO$;y5LF-as<|IS`kZh5RzxaKjQtb`SAJhL1gcByc@@SaQ+yh@WmHVr^Cb=E zi-MQU5+)vt^6IY=hrE`is>JR~e_w5?%5&%!3*lHQiLppf)A; z%9a>pW9?|9W!R=A*5fszD;vF@wTke}E!;sbW6w%f&5)uaV8T4NZ~7jfGg}ALBP^n zAPA7cC4#hekt+lNOEW?cf3P;!2m)$5Mi5Zk8w3H>y_FFYRQC=+Ky@bw0;>BBL7Mp0 zX5&|zEnlvnYNs+{fiQ+LVuGrjAqc42R~c!7s+}XTfQc^F2y#cfL}q|j+8*=K`rP^n zbqxo`@&q<;aszE{ktF~&cZgeX`A>ew9Sko2o_w8-uWqkDRLxM%f4iZ1SS-pX&1111 zNBnQhJ01QjrCBfwlYnNtU!${v=EW>8pW-kVtxrB%TAHC38KqT|rHfulzZnl#_8HB2 z3zO@sUZ%0}JoD1D;iYARCCN)xCIxS%8Y^n%u_OrR>L=Bwg&Rh>`mv`6$m9CkF;e19 zE9Ie8|K5n}dmR_4e_IT#)g;7maM87K(Y56jhxd3HvA}I}C?k^S5Cn=&MkLW82)IBm z+IvKYn1fWV_Lz@ka~lMBy=@L|O=DzsAf1~wR3x%KAmTeDDsbPNw2!FB`C{gJ;&rCM z-(G50jXBq2`dYBeTg;ZGW@|>nD~Z`UK#9jl=7a|I}~CE3K3Z!Q2aY2 zDsV5IbpAg=JVq)92=TGA^GIx3rbXU(Bs#bf9eJgJKu%@E0vFIwMkFO72vnksNJ>Ov z0bVYgJxWAoe}H}H9`lg{baD*`#_|L<(BloXxkYTC*boHNcj6pZY~>U0Hp@oj3Kv_N z6U}I`jH`xpwj>&HH1p3p(qkldfc&P;mi&m;ww%rz-yDd6%dZ*-7c`jU6k|ZoGej6k zffxfNh%k}{F$R8p*f5w1F$YkF`y7lmCpSWrM#jh_f6^f{NEu=+kl!R|20kSJ|=xiQ+_>W zITa7ee-7_X|6G;FNbvww-me0yRcUKx+uZ7lYE|l6yI?WR=5$D-C|5kjt5#cXM(eG? zDd|4eyJ)|0J|9c`P&EE-RPXr}VKP_pU-fRH-5vfg4dKiSre#Ex;x(mxmA<5Wp+vH( z;=cs-KBAIlAiRSjM@>}; z7%SEXq9mIg-oZb!+YOij1?q{29CovK4^aYt?;5V9W&)gmk_!Wo6WkHl0}~w@%PR4u z1D19!r>Q4-n${=V*Pm2-#bs4MhtXYjXV8PYJ@~K(S7dR){T@7M#l*h@jEDX7~>)CQK*%?P=nF$|SeesR2kV>rX$ z2gmaa&-486hUYlOFt9v?ZWilkwOC6=#8V7lwpwnY-Ry2FUC{QVLk6Mcgn{jxJ_ube z=Pyx5I%?ny{UG#1Z)gn$iaDGOGmuWI8fN3AURUVaZZ3VtP5IC39^;;s&)M@! zC9^^`$;mv;Nt!PjLMWNjV)lI)dDDLZs?(6WiD$qu_w6UB{{geJ9LfR(gcLv8-jjkK zTLIjY-X9_y>WPRPb^`za`U(I56aWAK0000000000fPrQYlYtKwlQJMG0S=R0AUXkn zlb9eT8{;wn`XvDX08jz|01*HH00000000000DysEli46X0kxADAtMH79smFU0AUU% AeEhif|BT19i!H($n?KUDC+OG?vgZf$&USt)MJi8>{VWcyyn$*Is4lVRWf;>`@u=_QA)m45OByVe zT$If!Ok5UbROwNn|fKo}z#F+tVN5Cl~1tBf>3)y|Pvz(f}t1i2$#A~V1%t;T%3Ik$O2 zUBiKiJb?|I+(4UKWC?)H9pV;T{?p%a`@_q>r(b82tJ~`jWiynCe{N_I@MZp_xh&G7 zQ2dQVrz3tPH1+2}?9){AYjnfD81Z zT_ZZg9HeqpV?LJ6Z4lt~t~t0hO_14vbZ**Ek;wXhi0_c7z6k-Po5m_Hl{5vEn za4(&9{y#!IMk)sg@v*b_NNifBMc#NMc5x+k<&_2kIh7F$TtFikk(7uaP>C`kDG`YU zc)94-C=r-C|Zyj2HvzINb^V_;Q@T zbnJio9q}j$9wNzgU}XK>M(cLo`o2r>kL+KF{88N^P?HGLhMJTaS(6AOX%hL!FKH5C zBu$b!O(G5?P1ZOV6S$3VEsac&N$e)nB(%Cktl)?LJA`Q)P?O}prpf)sq(^BYuE#W^ z#e=dVf4b8@SLHEMJV2HA>%b~i+M3xmclx4AmHN&uSVYqW9nmn%6qkvrRhHY)YHM&t zdXLpE+HYLU$6_xi8h@Q&oJ% z_~xJ}$##c#_z$z(4VVE1{T_Z6b+dU7Q38K2)o7`i04Jd2!a(E%cLespM2E(*N_^>n zrJc)Z>Peoa^~v`2C)HkYSryP>beG*3^x$p}KJ392SzK_x2M^jfBpI|9=&}xaS;P$z z#Z`6Q8%oF%lfukPP?n~zw_OZfevqkt5ZAWjnmtqa>vw-+j6EDbEoaMG#IZcLDLQ}J zh8-G?+NYB;hl*lj!DWc#_e1fb#-W*koC1*&!SEYjuFuY6AV#rw3JFQ@1R zuI2OqZSNnEYlTWNKh)n2y-WjLrlBVGQV?ni>NXO!f#q~FLT+dbLuHj;9B<$l&M^4F z@jS!xJpa4lIgT+5EDxcZ#X4Fo))GbW6a$#8mYZldyW2_^v_0vNL1;N)U^}M|Lf6ar zOVp8$8aP8g2>s9-T7!XN4rjv*tS-ZBywvLoUE9s2@3<-dd7bLDtbPSSPe9QI@1A0>v~JYFDE7%_@FC zk@Hk>T5AO*I)X#X>ZV|ZFn?{~)oKIQ3vf{YDMy$o(cXJQFm`MYlDZ@~R1mk?O6o&t zci2$&O?QCvG`+(TJmu6=d;-R1B%&V4*@!ehwr&BOVW>fZ@(yQ7hR=dU41ca)-r7}Tp-!h_fU1Hs!Yoj)ZAew%%OV`Erp~X! ziMImy<1y$HZ~Y=&AMhvyAYI%+`ItRDZ&s+v(|jJMd7R#FvIWT(WPbZ~8hO=!2FiZO zzlj5&%YFR`sz0-}9LfR(@)!an-IIhLTLIgX;2$Cz{T_Z6bprqZ`U(I56aWAK00000 z00000fPrO`9Uwsg43l3VIstx@njj_{n-bcOB>?~cPyzq|5dZ)H0000000000fPr6= T+aNvxvy&PjBL-z200000zYZ?c diff --git a/summary_results.json b/summary_results.json index a0a510c8e..7f333ff22 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1485, + "Performance_Result": 10.3715, "Performance_Units": "Samples/s" }, { @@ -212,7 +212,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74129, + "Performance_Result": 1.75985, "Performance_Units": "Samples/s" }, { From 62ef8833ab60e1687737a4de9f510e3e647506ed Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 01:16:46 +0000 Subject: [PATCH 009/719] Updated results summary [skip ci] --- README.md | 48 +++++++++++++------------- dbversion | 2 +- summary.csv | 4 ++- summary.json | 74 ++++++++++++++++++++++++++++++++++++---- summary.xlsx | Bin 8515 -> 8765 bytes summary_results.json | 78 +++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 171 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index ca93e28ca..5c69f411c 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,33 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-08 22:25:12,884 submission_checker1.py:3263 INFO] Results=8, NoResults=0, Power Results=0 -[2025-02-08 22:25:12,884 submission_checker1.py:3270 INFO] --- -[2025-02-08 22:25:12,884 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 01:16:40,932 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 01:16:40,932 submission_checker1.py:3270 INFO] --- +[2025-02-09 01:16:40,932 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-08 22:25:12,884 submission_checker1.py:3276 INFO] Open Results=8, Open Power Results=0 +[2025-02-09 01:16:40,932 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-08 22:25:12,884 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 01:16:40,932 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-08 22:25:12,884 submission_checker1.py:3286 INFO] --- -[2025-02-08 22:25:12,884 submission_checker1.py:3288 INFO] Systems=7, Power Systems=0 -[2025-02-08 22:25:12,884 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-08 22:25:12,885 submission_checker1.py:3297 INFO] Open Systems=7, Open Power Systems=0 -[2025-02-08 22:25:12,885 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-08 22:25:12,885 submission_checker1.py:3307 INFO] --- -[2025-02-08 22:25:12,885 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-08 22:25:13,928 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 01:16:40,932 submission_checker1.py:3286 INFO] --- +[2025-02-09 01:16:40,932 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 01:16:40,932 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 01:16:40,932 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 01:16:40,932 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 01:16:40,932 submission_checker1.py:3307 INFO] --- +[2025-02-09 01:16:40,932 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 01:16:41,948 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:--------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:-----------------------------------------------------------------------------------------------------------|:------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75985 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3715 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73955 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3715 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432787 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index a8f78cee4..aa1044913 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -636 +637 diff --git a/summary.csv b/summary.csv index 193bd9289..d47db0114 100644 --- a/summary.csv +++ b/summary.csv @@ -1,9 +1,11 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0101,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75985,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73955,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3715,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432787,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27262,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 12777da3b..6fc75ac00 100644 --- a/summary.json +++ b/summary.json @@ -30,6 +30,36 @@ "weight_data_types":"fp32" }, "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.369785, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -59,7 +89,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,7 +119,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -99,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.75985, + "Result":1.73955, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -119,7 +149,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,7 +179,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -179,7 +209,37 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.432787, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -209,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index bcea8ad64fa3eb8122cbf3bce0d8fa398a2338e0..283b7f9f360d13fb24fb5a90f748020a3c209ce4 100644 GIT binary patch delta 3155 zcmV-Z46O6RLcK(=)&~kQlcdI+1^@t2C6nL>AAg!#Z`w!{fZz95SXLieX|X*9-y)z& zNOK{CP@1ILC*+s`tol~lkmTR**d_$dIdN#U64kyWF#gWWoEd-2_|pFT6b90YrNGg5J$WtKl4of-2JQl7^gq71?RaWDu}XavcLpK*+)47@@DR-x3P#CDXrkQH!`jK4KaFp&-M?9PP9zTt} zWyqtvswCxsn~Uye3qQ%^ZhPvBV?{nv%4Z$u=Lv!<+e+hyDLK7Y)%K1)R^4{`jX z_;NYz$XbD_h&?O2F-oN=pSjB*AIFau+<%`haxrX15#q%`R{oVjUyP+LOQHMJk(;vg z=uh)SQL(h}r&C@Gv@ChcERVzQ)giX7pbBN?I+kf{6YVIhrJ1@>%*S)U*D-3#YNxLb;00s*5tA`mdTUkRj1pKT_6wwd^J1*3L?5d%z54&C!4GcMN{*X^Kz4=7JDi6deB?Pr#ur|m`q*oWg45BCti{?ykyykqv$DJ zM!DEbHP+O`i=&x8m;aWhJb!onAd^3})fur|e?3G>V$;e+*UEoy^y^#IFIH_alva~5 z$DyUOXz46GtO^*M<7r-j3DU<1e~DfR*iH-I9LHLYr^|D+=RdD+u^{A z0ns`b$ZHE4Nz4!E@eM{SfQ+mI8aZD~T(7vCX^5+rs;b6Z_?SHBaewM9CM#Vr6|Lbd zncmvN#6zUAhlzLAmWeG#(ZrPwEuu|}Xyb_n3Y}oY0HfX6W+(+NJP6vM51*lq76kP<_E0THyE)1GP3_aA|4`@Jw&{>cb3Gap&RUtCDEZx zbnr|AK~6AYfK{`H5r1SNfj|>6f=tAS2~;|FYD^@;feV*4;e7{qas>+pL<&$g7&9cM z0PaIEVu5N$&S7N>T`$fWfm_z_WSd*sd04w&-jW_7yM4B#&aQu^W7%)qbzmATU|A0I z0thbxvMnH{y7k6L$kRFR@gT?~FBYRxg>Fi#*J#G< z=)QX6snEnHybYlF6GY&@5uZq5pz{! zD@)U-S@q71l%#%~`uTI!dhycFGH|Q&iyEp0zMOoYKkRGX(;s;Px(2Ilk$@xOAm3L$KlJj1$U=B zExw7-oyBqf@}(G(&D&7je*v@H4VVE6H82{CvI77Bgbb5@4;z11-*4hL5PrYE!pgnu zKIA8flOW0p0j#l zsEtQF9goELmL-slOfeprHPC?D7$_-I*2W)bDtH^*0IxF9Mzf6M#L<0gr-xzNc(>nY zDTOwC&;AC=O@(1G>v)Ka1}$dj6Bq{r$#kGem89Zx`{aL=Wvu)P&q|2ZE>DGkvTc~0 zNScvdnx#U$^vnh1ZFD75sJwYyvKTmLivQwwg)xpHgOUjKFi{R~*(rE&FB83*kjPBEe`L4j;C98I_2`W7!GGiio z3UEKKtMZlyqKKU|n#tgmr+uQ=@Orgvc+Ep!Q+_wfK>SD&&HP``Ma)S#v$?^s>c+LeE}&E3Pz+#Vuq8@}b(2%Q0Ehz7n9 zVn4h9O9_O=0NXx7_jh0Ct8KKNZ=&twcD9#PJaMuHjnhOMp{Q-co`{Dd{lm zpcEY%I~3xt4p@$JSxr4j_xq|I`}&hQUP)eL&@t#Py))>+-5%WQ!4+9taK8r+dT=Gi z2Dg8e#6og_cEerTN6)jkL87=Q&V9xxeJZ7p(;Sp0>#Ic|$2v+9pbh%hZ+v(j#E*%@og%pIA69zc8 zQ-2V;Ud&#sj$B}YI&peQ&(5B)o_>T+HNj=$4&Wf z#Q0yc4ijMrf3RHFP9*^V08jz|01*HH0Az1tP;zf@b1!3Wa%C=fZET&A%}&EG5QOhM zMdW*8JB13YB$WX5L?tAOK-?^Q+gijwvJGk99ye*zR_cl4dS<_!UAdZVb({Ty56YSv z(So21jIdIfb&cMa^ZXuVA#x*mYYo(B2La8Rn@Tt)f2@aBZyorkAY=(-0u!!An;0F# zIEW2s9*X2JNnBa4c}%5WW5>leUjr@)nPLr*OCCAi19LYL(E%yNh;%=`?Ez9?XhDOC z0n>uwlQ)9b;c7##oH@0MI|o;-M{#s-gBp!p*A?BQx2F6Qe=c6$`c-pf_NNknno0^L zJn(3JNK@hSA|0;fAuiI1R|@3gF6md^4vP$ZAhQ&J4RMdk$K>gGIY&)N%DbGDIeAzT z%IK8c(y#r?~cPyzq|5dZ)H0000000000fPq^aliVOZ0KqwcjR0E3*HkR$G zaheauNnAHcb*?M|i;W5_<-#4=O!`BGqW?ZyfCo6&o6PLENPa9 zEa|I6GgO5p9`nOw-d7*)&l??8$qP5}-H0W$ufEb;{XF=q_sp`N`GV3y;SQ3#uP%xr z?SJZe?k#BS<{C?B!p{UOi`{}BW^+AHGwS-4Q5@-}Vc2@?hKX8x=w^rbU_lUi^o)6B zOp~H^lF`U5c=-7uOmlV6^Fw}4XiAR7~0PB@O3`f`6CI5+)vt^6IY=hrE`is>JR~Uu~+&bLbZf z;aFLOzE4G=RmCgwg2g}TQ)K0!HYN7TmKbDX?P#QB*rp}c<29ly8@-;jitx-W+(9p6 z&q`L!kf*L#8PYXPUKUS`eiOqYHhncPT@NTc4^h44pxM)x!tsju4o&Xizyln3+trWe3(fY6B{^vCPH`#| zon2xm6V5JiCKCim;VXiGE9@LWz<<(QAPA7cC4#hekt+lNOEW?cur}8S0%|)(5K!D3 z1Oe5(l@SwE_YOfobteb{s{0K=n)uab<5!z4U#_5Pr!r!JForT>f~uV%2&mdu8EJy5 zog=Y;i7wU%a!0&GW`I}P9`n)q-1-T14F|^Z1U7JT18r`RB>*;eh+A;^Pk(;L9Sko2 zo_w8-uWqkDRLxM%yPEJ@_;6m)k3k@W6DkBznWDI3Q5+Z^?A6k-Po5m_Hl{5vEna4(&7{y#!IMk)sg@v*b>NNifBMc#NMI=B)Y zd8L6sPG!Ub7tl~fB!49$2vnksNJ>Ov0bVYgJxWAofPLs5^N|B|at#N@@&q=};|;XA zMQoth5CqhB;v83ONhd7y~7UFp>r_27Z0mFn^c|F$YkF`y7lmCpSWr zM#jh_(kr24b&FX+F=7m;<76xRJHnG`YvYh`?=xYiVSROky{oCZW|W zVg*0^-yuxffPb1K|20kSJ|=xiQ+_>WITa7e4)0F?T$RU2@c>ocuL7%8X=`TN-0F*J zRq9*2U@^_+bV#EpS3Jh6R$FdH>#e~l=|0xGXuokjA4~jDH2!W>@A(yBGFS3n^=_iw z9sV#4;mixBWki+YHKl!(zNCDiM6#;lzXbI@qLOAI%V@&lwVqvUWR{UTh?)PP3aU)P zUnTu;;+@#t6VLwmNijbveTApKMtOGCEaC?i)}WWVb9&=u^DxPkhz2}SL*p;kS-mq^ z{VrHq{jyZ$i4_Hl*GCKL`ZNWPRPb^`za`U;ag z5F3A1+iu!Q6n#HmG4gq7AAH65f(R7?5}^`;AgR(PYp}PCTYJW5W=xa6KLaLd>#@^T zm5?IVo|&~TYcDeX@KhG)2#n{HTxqspX$VM6lbmE%+Qa>|9%zUQOcGpB0$18IaP7n7 zpE2hG$*Y84X$K*yk!kYy043%It$@f#N?Cu2h5X7glUEFI!VdtVEKJ9;JhQ|((NIil zA|$QrXs9One>Kb+#J9D{nCFwRnB39?im@psWAg+w;4TJ&Sx(#d1IYz%gInNrA=+qB z;1WB!Z|(IkYzObsw8#mx;d}ZO2sagm<-8*iSq)mw(K|3Q1Ic!vX`STaef#8$7L$8*s=QzQt1!kf6u_`h4`b!< zj;7)pW=ikYRH#Wja7a1_EGcV-W2L}32wA=I&P=}1#)gKaTb5<_;7%J)u)tDQvg77S z!{+U3M$3}Qd1r?R$3kw1&L9OQ7X*LwxT^K)S#~5oL`TQ)d zHhjym5jq3T2n~EA^!)GwtRxT`1JCvmy1)ClSns0EVjJzIx0}`CblXzta2$`2?;5V9 zW&)gmk_!Wo6WkHl0}~w@%PR4u1D19!r>Q4-n${=V*Pm2-#bs4MhtXYjXV8PYJ@~K( zS7dR){T@7M6yAx}&SGcQ3|n!et4F?9Jsruso#+m37Y zOyRHJ{f#m9aQw8KEo%|S^4zBAXd8BDIBK6x${Z?+{T-^all|dT{*vwI;u3(0SD0}) zl^3Yi*|SKOQ@`>#=p5QOi23ZuI%TOlO~+kv*^)Igyw zExie8lZaYBNNOFweHABh67VU<)y#f7yQp05Rh#^Q^U~-F6|vlL*qDa$BKY2$o{(`3@BGS#;Jx9LMUH(Sk{Ey< Date: Sun, 9 Feb 2025 01:22:57 +0000 Subject: [PATCH 010/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 8765 -> 8761 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5c69f411c..64d696d0b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 01:16:40,932 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 01:16:40,932 submission_checker1.py:3270 INFO] --- -[2025-02-09 01:16:40,932 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 01:22:52,348 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 01:22:52,348 submission_checker1.py:3270 INFO] --- +[2025-02-09 01:22:52,348 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 01:16:40,932 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 01:22:52,348 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 01:16:40,932 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 01:22:52,348 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 01:16:40,932 submission_checker1.py:3286 INFO] --- -[2025-02-09 01:16:40,932 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 01:16:40,932 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 01:16:40,932 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 01:16:40,932 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 01:16:40,932 submission_checker1.py:3307 INFO] --- -[2025-02-09 01:16:40,932 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 01:16:41,948 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 01:22:52,348 submission_checker1.py:3286 INFO] --- +[2025-02-09 01:22:52,348 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 01:22:52,349 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 01:22:52,349 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 01:22:52,349 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 01:22:52,349 submission_checker1.py:3307 INFO] --- +[2025-02-09 01:22:52,349 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 01:22:53,352 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73955 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3715 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2903 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432787 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27262 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index aa1044913..231a7d578 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -637 +638 diff --git a/summary.csv b/summary.csv index d47db0114..97a7f67d1 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73955,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3715,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2903,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432787,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27262,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6fc75ac00..db7f8e887 100644 --- a/summary.json +++ b/summary.json @@ -189,8 +189,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3715, - "Accuracy":"acc: 74.400", + "Result":10.2903, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.27262, + "Result":6.175, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 283b7f9f360d13fb24fb5a90f748020a3c209ce4..9dd56b68b5be76b56d458ae55f6cf5063c87bf8e 100644 GIT binary patch delta 1662 zcmV-^27&p#M7cz;)&~ji;F@)q1^@t0v)>1i1%JYcDscyY567qD;dprm-$&1CPRH2*F>Lr>nJG!sl zH-CA>MT2oP^^03%Zcx1)ll#$J%6?Yw-Izzc@48l0=FP^-fJ^yv!aFj5O2k|h*~-%N zX;!^)BPFRHr+)riwO+jPvkcsB7*9Wzomxiz&n5r(!TVqyeeldre@pr&sUwNpUr>=< z*-G(&>Q=X%xO0B(rgJ~aq=3&vqMBmKQYc!z!&83Fbc9l*oLw2N76aQB@d0uwEM5RpmK`s1D%IC=Z;PS?nur5{ zf6Ox*I}WjPnlb~$ykb((5^SJ~rT6rSeuZZ(#A=(TQbO4@e~gbb&FD7GQmJ2h783F%x>PCD-n=S#3_@_t zfAPD<7{`!7Nu_?6YKJ#`C%@5B%iWSo{SglWlGXw%%BrNX7T^@5ieB?GQ9hcOXE~QsL3K&q zvG#RqTjJJt4>vP+fQW1P4t5bbe*<`cdcGABKfC}71%y_QxIRMncVB1AO|+V=qs{ns zwV0i5J31YW;{ozLD{%CefMB4#r39HW+G5y3DO)tQFXUe>uo~y8nr4#jcGYR@n@{?9 zrFoG-%b>gTPOk&+ci>(JuF2wp`yF`Dfon0=xT_@=l07sV?$SPHp2am1f5k;{?laEV zQ>lcU=AboMS1sBY+UCbzpC57M;=bKcwO{u8fwFgScKWA-N{@QZ9e2A2R7qM)UKQ+R=%$&N1npLA~mJMm%1az9+U z%ej`h-;~OKt%jz(V#>Grf8mvAzaIXWPv>KF1J_DxfY$eqsPBYY?%zb!G`)xiUBsgY z^^$*7=l>nw=#sJaxvfhgAFl?eA6j0hP0p7E)58|_f;S6>ScDM&`+mXL@&bn-bTeB< zi`h~kM4Pvh#e5ZQCUg3|xHrAap&SzF4tZU;+00DIoMi;yJyZR;%-) zePV^47Uy|&1!|}1tE=8>q|J2g)|bBHw)!_>^e?jr6k!MiYa@cKlWiOze`>=p5QOi2 z3ZuI%Ss|qrwgYX+sewXMT5=Q8CK0uMkkmSP`zlW2B;ZqztC{_Fc2PCk={EZXACxsU zDhq-#Fv3b@HZ}TK&hrP9g~*NMtu;`iJp?ps?keG!upZvLb>O3dkR^}_Ot>0tV{{DT zAhw`+D3Ze@ac#ZkF_nIUe;pU!d;^#ga*s7cE_vkm0Ll z23!^;zIY>e9Yz~^<;tm5+&dVxp2fkv3u-WST~~CI-kS24_-pa{-mjV~b2ya%)KpS1 z;eki%n+jhS>2NI%agk2ERv@4DNgsJTEpqGwnWX^i827AvPM%+uBy-eILZ>;QIeA=? zl2OX0bafbc(|-Z#(~y4?&wyj@+fUH^0kf|i$^r@R;F@)q1^@t0lY$^u0hW{6AR+`O z_HUDtAr=B45R*U<7LzU^DggwOSs^+Bc9WDLCIo9Ef~}LjAr=8wlhq+U0jrY~A|nP} I9{>OV0Pk%n8~^|S delta 1712 zcmV;h22c69M7>0?)&~hPlcdI+1^@t2v)>1i1%IMj`qnM|T`TD)p~sAPz}SRVV#Z|9 zO3Yv@2?kk-8EhrNAS;`7RuUB;D{Cs~Zw|g%2}=ethbxvMnH{y7k6L$kRFR@gT?~FBYRxg>Fi#*J#G< z=zqR?snEnHybYlF6GY&@5uZq5pz{! zD@)U-S@q71l%#%~`uTI!dhycFGH|Q&iyEp0zMOoYA}i=OKJ5MPx(2Ilk$@xOAm3L$KlJj1$U=B zExw7-oyBqf@}(G(&D&7je*sWS0|b+?AR4o@4tfCuH82{ClQIx6et!f8?3zSQa*Ol2Q%DtfE5FVn;(2P5-H2+8}#U(16<*C@E9c#vf=ZcpKaRuQJg_vy9}#(S2*Dhhf`zx8G+eg*JT8{sziTg<&!4 zc!-P!EoSHw7zYE%bf8I*%mw6a zbR|=$ym?)+7&vE&|KfLrF^(aFk_h!MQ4Vj}UVI~^lDicX>LVUFB%KA8lvPP$CBPX7 z8NK3XDt)vu&#-jMvg{t*Y2yhIM9NBL+^jWh-Y%ys&zY?E;TVxv$PUp<*h48B1oXJ5 z^x|1&BtAwbf5*VmX##s9g%-t(?$d+$uE5?Ut9G&pDm^1IV5czeWKU!dbMqM%}R$si9qQX8G80<^BFw?qxIi!zV5i8_oxWrc82EbKc~p|@c?00 zpRy8Azj&L}pr$16SY6fHmAK8_!_C|tB5WJJ<=6G!M`ja|dNnT{oG3YM6Gw8wH9^C7}6~a)5TjUD`*_ zv$#Q`f4C^lea0w#Dy5Lq9F!*Ot3?+>*ZerB^CPZo$2EJZ_RD@hQ1%{JYVev$3R;1Ym>7o_A+hZm@s?CJ9V zN%tnOn$NKe`@^NXoa>nTZK?d%YG~UlDt*fze_o05>*0^ZY%xJMa4oe4XmkIFTq{&^ z|0b%o=|w!~A|5rUm;9q9|L^!lmyC7KZB-KaWIaS~Xn3JAIbRmcz%d*zc(Y)zfid>K z?-$20yuiW;-ON|fa=wxXR_5(=xmZWr>D@*y9UIGq6oi%&2DVdw5V~H>UaXE>V1eWL zEkWpq*s}%$rB>%h``8FQCC>Be3gk{xS68#uaF^-YZZ3VtP5E!c_+L;<0|b*WAsVwH z6hjCEuw2(plW80ve@??N5QOhMMdW*8JB13YB$WX5L?tAOK-?^Q+gijwvJGk99ye*z zR_cl4dS<_!UAdZVb({Ty56YSv(So21jIdIfb&cMa^ZXuVA#x*mYYo(B2La8Rn@Tt) ztcO=`9r&mqWC>&f6Rt*^7#+hnhz)2SisUd!Tv@MqOr>9Af5*i)Ujr@)nPLr*OCCAi z19LYL(E%yNh;%=`?Ez9?XhDOC0n>uwlQ)9b;c7##oH@0MI|o;-M{#s-gBp!p*A?BQ zx2F6Qe=c6$`c-pf_NNknno0^LJn(3JQ{nR>9j@geF4Bot3gqK1=~vzkiwu1rvlM_0 zagWN!8 zlcdI+1^@t2lYSsq0h*K6AR-$zFdB@q0{{So3;+NW00000000000001hfh3a?AwdBM zlT{%)0eX{=AtoEJT-Qz|0RR9{0ssIJ00000000000001hfm@T&AwB`GlMo^!24Non G0002goGVBG diff --git a/summary_results.json b/summary_results.json index b8629845b..d6407cc82 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.3715, + "Performance_Result": 10.2903, "Performance_Units": "Samples/s" }, { @@ -320,7 +320,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.27262, + "Performance_Result": 6.175, "Performance_Units": "Samples/s" }, { From 712da63107f75e8076fc7ff51075cf6650a6016f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 01:28:59 +0000 Subject: [PATCH 011/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 8761 -> 8762 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 64d696d0b..36d3fcac1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 01:22:52,348 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 01:22:52,348 submission_checker1.py:3270 INFO] --- -[2025-02-09 01:22:52,348 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 01:28:54,184 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 01:28:54,184 submission_checker1.py:3270 INFO] --- +[2025-02-09 01:28:54,184 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 01:22:52,348 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 01:28:54,185 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 01:22:52,348 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 01:28:54,185 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 01:22:52,348 submission_checker1.py:3286 INFO] --- -[2025-02-09 01:22:52,348 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 01:22:52,349 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 01:22:52,349 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 01:22:52,349 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 01:22:52,349 submission_checker1.py:3307 INFO] --- -[2025-02-09 01:22:52,349 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 01:22:53,352 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 01:28:54,185 submission_checker1.py:3286 INFO] --- +[2025-02-09 01:28:54,185 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 01:28:54,185 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 01:28:54,185 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 01:28:54,185 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 01:28:54,185 submission_checker1.py:3307 INFO] --- +[2025-02-09 01:28:54,185 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 01:28:55,458 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73955 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2903 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1121 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432787 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 231a7d578..427950ea4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -638 +639 diff --git a/summary.csv b/summary.csv index 97a7f67d1..7afb3a3b9 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73955,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2903,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1121,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432787,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index db7f8e887..4b5db379c 100644 --- a/summary.json +++ b/summary.json @@ -189,8 +189,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2903, - "Accuracy":"acc: 75.800", + "Result":10.1121, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 9dd56b68b5be76b56d458ae55f6cf5063c87bf8e..8051c28845df691951ef426fa34906b7a5fb4e89 100644 GIT binary patch delta 778 zcmV+l1NHp5M7l(%jXRxYvPevbf-W2Of0bT8uUBYKetp56yQ`>kSs(R~~{YeS@BHNF_B>;seXerypUFs{!hVmKSQ1^JT&Gu!X(g&4M8o zA;kZ_Uof`3z#$0T%$Ct&wp0kw=IvxLUqzeA-C8Xjm#Bplgboe^7oR=|UC*a4R;(6S zfIUA5{g8N0ucy`O{Aiz8p{K=pUR{COY5MA_w;E|PUAy(A@3^i0jTrq4Py|v zlL*qDa$BKY2$o{(`3@BGS#;(FrN| zh_pYh9RPx3XhDGv9_Lw(FWvx7`N@VsxpFEQdJB`*vpBl#}Y(SY!Sif39EN zhE-Fk52xaQ>QZpZ9k5_rUE=E^9ore6Icm^DE-+qGn53{QsO#=lDz}En?ll>r9 z0R@v)AvytflaL`M8~pF`a3ui%08jz|01*HH00000000000Dys5lhGkQ0j!e{A|nP~ I9{>OV0LglD82|tP delta 777 zcmV+k1NQv7M7cz;E)WGL_HUE2vojD@0)H!T^p}8OpuMF8nKIg9*g`2=G`26~UoEg2 z=c<}!lJ0iZY3!R%`go;zkwMF#yYx=41Mhd>UI(tp;)44fc+i1sG1j=NB^HuBG#l>H zK4zZ9H4?=|aqctD*i)&5oaUf4SywIE7~1B?UY{Rv<>J2GQMF(8`+>4|aO5=R;(tTa zsAUTNY@&Xc91SLU%+EE6j;`f~7S_2=ZR35Y>aAn;CnfNUY(EB<02HF2rGQg-fu6~Z zF8`l&Zvs2TPWYqfM-q83sRIyekmeEJ}CJ)gcr^#qkUq9o)+hMbp>jt>8q>WYNX9{?besR?~cPyzq|5dZ)H0000000000fPq(&(jh(ptCJBTBL-a` H00000rE6{X diff --git a/summary_results.json b/summary_results.json index d6407cc82..74c87e1f6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2903, + "Performance_Result": 10.1121, "Performance_Units": "Samples/s" }, { From 4c0f9474e0077f53e3c41f32a4d5bc58d40a0e0a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 05:43:44 +0000 Subject: [PATCH 012/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8762 -> 8763 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 36d3fcac1..450ed6ae7 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 01:28:54,184 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 01:28:54,184 submission_checker1.py:3270 INFO] --- -[2025-02-09 01:28:54,184 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 05:43:39,702 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 05:43:39,702 submission_checker1.py:3270 INFO] --- +[2025-02-09 05:43:39,702 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 01:28:54,185 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 05:43:39,702 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 01:28:54,185 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 05:43:39,702 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 01:28:54,185 submission_checker1.py:3286 INFO] --- -[2025-02-09 01:28:54,185 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 01:28:54,185 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 01:28:54,185 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 01:28:54,185 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 01:28:54,185 submission_checker1.py:3307 INFO] --- -[2025-02-09 01:28:54,185 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 01:28:55,458 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 05:43:39,702 submission_checker1.py:3286 INFO] --- +[2025-02-09 05:43:39,702 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 05:43:39,702 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 05:43:39,702 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 05:43:39,702 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 05:43:39,702 submission_checker1.py:3307 INFO] --- +[2025-02-09 05:43:39,702 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 05:43:40,636 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40413 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40942 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 427950ea4..fe4d93ae1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -639 +640 diff --git a/summary.csv b/summary.csv index 7afb3a3b9..75717eaf0 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40413,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40942,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0101,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4b5db379c..9bb8c0f55 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.40413, + "Result":0.40942, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 8051c28845df691951ef426fa34906b7a5fb4e89..88d1d102958bffb52761a7bca1c6c1af38424eca 100644 GIT binary patch delta 1409 zcmV-{1%CRvM7u<=%>xRKj+6D&0{{TF4U^yp8I$k>2Y*d-;y4h!@2^m~a!9HMe*q?; zz*7^FOg!FO2^XDIpRm1BDn{sg&i= zaYU4o9DfU&m84KJ3FA4I>Xok@QF4Ognv`db@4A7LV_H~^g(3eP8!Sm^M6Os=<)n~} z5l%8JRrn&MrLe+LOx2uT58@;lTVK54#1_tE)WF zSf<)AN@*OEqIMfqLbCjOGv)0KG*mCxb3NC4Wnb7{&kg*({ye*k>^V;3wT|!#OB{|k zdqKSRAWE!1lsELwl}eEweb&P|HfKMuI1C`Q4lU*;?2ep`Hmv@XtyhCSY|wA<-B*6& zp?{6&!5+G>hi;4S86Mh*KJ1|fd+4?JzTu&b2w)F=(8HI0i$6H_aM?zjA0sAh#Kke< zs*UiBk8uq^3^i{6h#|*U0Mb2%+yao^G2{+_^p7F;03Z z#YAG73F~a+BtSKI{)jcH_J*j?>VI#4$M=iC6!xj{P>j7@S0K#Y@W z0zk~110ZJ3wIQvX10d!hKX@nS0Dro9y1zreADB634om?D#`!ceIgbFx(Dw;|7~;(b zADeX5sw!mFQ5~)}u{{so&cELO$-j?S-uwC2WS8?x3#%3rQ+@W}tm(R?x}6~LvuW=G z)!zHA&eSMbR2e~XSE_cHY}Iw64#z_AHfcI}RC3BVmAfXj_06KnaG10BG+AO==z{*N z$mvOR5)4nG;OrCf&(IjD;0{m~V(q1FwSln0QHj@NhWVNn0%asoJIl3|%DlN7)X$QY z^^>aXD<)Oje|SsP1&MI|rh3?fF?o2=t^MnjEBqI;$OoPj1aJ~dNt0?EAb)PcFc5_A zeTvBU#7+WLYb8NyD^5kKRH~xhEPDZ~@sDh`fw!*{Ab`ZF$Mwv9JG*i<+o?AB1?Q#F zHOe!B642ZTsW&zHSkBW2l=#53V6D+mqdj;uYwjv;DK`$@oU!171fN8Z_LSQiZ9}jW zW6!ssn9rg^M{#YOVj-4pgMTgK-)sZ8Amkn^2u!fRa1TuFNJK}Z;3LxhxOM;tj-dqw zI(VFCIX-&>IOT6P49bO5$*{L@(|Qs|_s+}F*mYgjm4h|r&+%&U`aZ0hO5LA|18OS4 zDR;nvaZQCUi+H$}`LKv5UP};9`>5Y|J1%nQ1Chi4^bq%?e3s8I%Op8!3Q|l{Ql#W@ zNv3pC(h2$MM_%=xf$BKq-^3H(ko)=*G=H?~cPyzq|5dZ)H0000000000fPq<)(jh(p Pt&xR$2d2@}0{{TF4U^#q8I$k>2Y+32+By(@@2}8!@{r6Fe*q?C zz@3I9%?Bied)rQ*7-7p;Q`>SSnZUolk`1{<-em6Fmpp{n(mA`UW36^I`n=6Ev>{wj zR*Wsrb}dATh{d#6kF9TiUw6(eBqT0koUww8tsN28=kUYmneksDB}5`+pb%p#m9iW< zj)+o{V}D_@k`!ttVLZoDz4EmqN=|TGlk&{*T{m!YObe^AFy#Nm21^nekxLd;IVogg zgp&+Q6~0JmDXef5Q#Gg8gE&dX))#L$`7yAZa8$egPRXFk|b6FSV*sI z#n>-Bd8>w8O?6t_>pH4AM=?oomB|Hrz9Dp-N`GafuPG7BME#3$s-*QTl;iE#>M9R3 zmZ>(3QX0pksNF`DkSzbvOnG|)4b=~#=LUXXf1X`N_8h12T1R+^B@RcN zJtJOw5GB?h${Tv;N~K7TKIvf{o3kHS90rhDhZb`ac1O-e8&-eH){8+OHt4tb?hC*1 z(0@ksU=Ll`L$}5E3=eHYANJ6LJ@i_9-|)~z1h9ub=;2Gh#UC7dxM(BJj}enL;?ptW zvW@VJk8uS+3^lI-h#|*U0Mb2%+yIc?G2|A2^p7ET03%I2ENv|8B_%bh$^zqu-j>Zv3=S<)gQTorb*ltjp-s8%@@x<-oSrf<8Bp+uzK;OJ5N|$s z-=wQnRUxa6>TtD*?RoHe{`LM({=LWY-p;=!yPQ8-Shbj#>az!DP1h~e?F5P6O?w}x z_TG1Orbfx4$_SFXQnkZmtF9Y$I2MYxNz=)rl2gX1+%>7KZx&UC!<@yZ5m?hg7xZ^U zPEVqfV0aP*XCIM&hQ>$*cYvx8YcF-H4TKerO1vgB%-6IKC?kp5S+1>A=FQ!pewM7P zpHyXEF{#@A!&|B@NQCP*)x#!?$-|3o?O(23;Xkv<2c8rJ{O|H`lWQCxe`>=p5QOi2 z3ZuI%*&!trwgYX+sewXWT6z=GCK0uMkkmSU`zlW2B;ZqztC{_Fc2T+9t2X%s=cUmV z$}@rz(A)^Aw-tJCR_OvIJ}@m|vlL*qDa$BKY2$o{( z`3@BGS#;(FrN|h_pYh9RPx3XhDGv z9_Lw(FWvx7`N@VsxpFEQdJB`*vpBl#}Y(SY!Sif39ENhE-Fk52xaQ>QZpZ z9k5_rUE=E^9#XKcNN*)@L z(_%sA^RL6moBj(>ore6Icm^DE-+qGn53{WugaQS+2d2@}lQtk*0mze*AVmR|lh`0U z0U(nVAwdBJlUE@+0e6# Date: Sun, 9 Feb 2025 13:02:30 +0000 Subject: [PATCH 013/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 8763 -> 8763 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 450ed6ae7..0fab75d8b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 05:43:39,702 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 05:43:39,702 submission_checker1.py:3270 INFO] --- -[2025-02-09 05:43:39,702 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 13:02:25,528 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 13:02:25,528 submission_checker1.py:3270 INFO] --- +[2025-02-09 13:02:25,528 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 05:43:39,702 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 13:02:25,528 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 05:43:39,702 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 13:02:25,528 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 05:43:39,702 submission_checker1.py:3286 INFO] --- -[2025-02-09 05:43:39,702 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 05:43:39,702 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 05:43:39,702 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 05:43:39,702 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 05:43:39,702 submission_checker1.py:3307 INFO] --- -[2025-02-09 05:43:39,702 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 05:43:40,636 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 13:02:25,528 submission_checker1.py:3286 INFO] --- +[2025-02-09 13:02:25,528 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 13:02:25,528 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 13:02:25,528 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 13:02:25,528 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 13:02:25,528 submission_checker1.py:3307 INFO] --- +[2025-02-09 13:02:25,528 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 13:02:26,632 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,9 +25,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73955 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1121 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1297 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432787 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index fe4d93ae1..02a6b26c7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -640 +641 diff --git a/summary.csv b/summary.csv index 75717eaf0..7fad5a662 100644 --- a/summary.csv +++ b/summary.csv @@ -3,9 +3,9 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0101,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73955,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1121,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1297,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432787,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9bb8c0f55..0e922330b 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73955, + "Result":1.72821, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -189,8 +189,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1121, - "Accuracy":"acc: 75.200", + "Result":10.1297, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 88d1d102958bffb52761a7bca1c6c1af38424eca..5b5f379c50bfed24220fc6ca4b7af261ef93d71e 100644 GIT binary patch delta 930 zcmV;T16};PM7u<=*9QgL&R2Vyv)l*R1b<7n^*6SrjK0Ty%+!stg##|%HUF?8r8W=st_i5cu9!5}9wgPkN8 zaq>rPBVLo$EnT z+{4P-b;l|zO@&!+ZCvT+pvR1Oyx4?JV#Z|9Nz7m;2?jZd8SEs%ASau3P7)O$Cu=I` zZw|gX2}=edMNXXMU@9`kWBrg`DQiX0xtCwiT?C8FF-{ch+4aU*bFK&^!LG^Y_?niSe z`&qqrV;=Rs>snEnHybYlF6GY&@5uZq5pz{!D@)U-S@p(^l%#%~`uTI!dVlfC&oXek zVLbgdzVe?didWh=!8s$1Q5;?DWCo6h|xlL9^y ziE4@^OKJ5EPx(2Ilk$@xOAm3L$KlJj1$U=BExw7-oyBqf@}(G(&AU+De*sWS0|XQR z000O8002J#=W+t$T>$_9WV5agfdK`Dn2_|bvosJ@0tA&B9sRS85@P`r)3&V0v$Zgvm9T%0i5!a4TEy!R5Bba zOj^(4=-zud8oRE`x^l3_{5k$wzP=Burc(E(;((e;aLOI9U|dt->mnYmWj-w9i8m6& z(;?~;Z>L3$eISw;fF9$XmCy3|Wwj(gO+kwLloTm>T;(Mt1ubUle&kL61*lF#{!Kgs zj=66?LGuT*tsTk&1>4S7dzzDcAX@>Mlh`041%;T9^s$rwAQb^4lNBMO1dm0QfRnr- Ee@J}0?*IS* delta 912 zcmV;B18@AhM7u<=*9Qgf;F@)qv)l*R1b?gP7;gzl-(x>!_@27KppXRPKp`=Mg(Mgx zByo=)2}v;U{>+9Ug(NCKLe^B!XYeV)6)G7Jji9?UC~8PVVIc{|fbm0CZ#{Bh^dqKYhM(FPI&=~c_PloG~>lmY5&g7^&lzk zVdd?*W0jSr!mPK}N%}eHF(V!?HldT4F&T6cGuTOjK~7=@J4rCe$!49CL^H1DM3VblP_8@n`jA7n zYzv5~ZoMxO@^sF7JP0z$i^Zr^p_|g`C7Lliy06|hdBsJ8aWwUdTV!rfy&aSL(Ok-Y zR`1=IN4@X5R#fKA#>;?9`E$ZMGJi_MTou{M()4Loy>TNYsUN3){#>CtZ(?+3ah$(=DTZY8E>!nlvzZQ<0R;`f z*8sD#HV{<;1wBj6L+!JV5@P`s+G+ags<#?xGhMs&rSG_{{*4&@3$q3kVF(0p5=%*w zYaAedZo)7SgztTd$oIrf0#$1zL24^bMXFS)qTVcf0ju$kY`1~8uM;4E#Hq*i%zit& zay8qjHu(kTrO`FYGlCM(+z6>RHTqc2(+8CJz_ehk(NLp3cr~r^ mmy_5aA_Wb=*8sDV{~#3tArO;45EhdxA&dlY5=%*wydi&_n!&07 diff --git a/summary_results.json b/summary_results.json index afcbad7c3..bd51433f7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1121, + "Performance_Result": 10.1297, "Performance_Units": "Samples/s" }, { @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73955, + "Performance_Result": 1.72821, "Performance_Units": "Samples/s" }, { From 41c13c8e81cd7e6069ad851e8411b93e881a43d3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 13:03:03 +0000 Subject: [PATCH 014/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 8763 -> 8763 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0fab75d8b..6494644f2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 13:02:25,528 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 13:02:25,528 submission_checker1.py:3270 INFO] --- -[2025-02-09 13:02:25,528 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 13:02:58,637 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 13:02:58,637 submission_checker1.py:3270 INFO] --- +[2025-02-09 13:02:58,637 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 13:02:25,528 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 13:02:58,637 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 13:02:25,528 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 13:02:58,637 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 13:02:25,528 submission_checker1.py:3286 INFO] --- -[2025-02-09 13:02:25,528 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 13:02:25,528 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 13:02:25,528 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 13:02:25,528 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 13:02:25,528 submission_checker1.py:3307 INFO] --- -[2025-02-09 13:02:25,528 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 13:02:26,632 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 13:02:58,637 submission_checker1.py:3286 INFO] --- +[2025-02-09 13:02:58,637 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 13:02:58,637 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 13:02:58,637 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 13:02:58,637 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 13:02:58,637 submission_checker1.py:3307 INFO] --- +[2025-02-09 13:02:58,637 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 13:02:59,717 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 02a6b26c7..3006dac01 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -641 +642 diff --git a/summary.xlsx b/summary.xlsx index 5b5f379c50bfed24220fc6ca4b7af261ef93d71e..19fbe9fa2e093a41d7ecd5e77a05675d30bb8372 100644 GIT binary patch delta 311 zcmV-70m%NlM7u<=XB-4k92pdoYaA?pCZ!R!18vEvfkIPSdK1zn5w(7h)H;6qDvskM zkW-JVnf-QlQMK5sHu(kTrO`FYGlCM(+z6?+HF{q!(+8CJz_ehk(NLovJX$n&6}OZd z2d~apa6y7kB1n76ZH;yzScT;(MgB?T>J>we@-{{^T{L;g)X1CF_GKSA>cv#lKq1O$&o JmVlGIA%C{llLi0) From 8a0cb0a481bdcda2a5dfcb3b1b75d91e24cb4244 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 13:05:27 +0000 Subject: [PATCH 015/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8763 -> 8764 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6494644f2..92730e7b8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 13:02:58,637 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 13:02:58,637 submission_checker1.py:3270 INFO] --- -[2025-02-09 13:02:58,637 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 13:05:22,226 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 13:05:22,226 submission_checker1.py:3270 INFO] --- +[2025-02-09 13:05:22,226 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 13:02:58,637 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 13:05:22,226 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 13:02:58,637 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 13:05:22,226 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 13:02:58,637 submission_checker1.py:3286 INFO] --- -[2025-02-09 13:02:58,637 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 13:02:58,637 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 13:02:58,637 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 13:02:58,637 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 13:02:58,637 submission_checker1.py:3307 INFO] --- -[2025-02-09 13:02:58,637 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 13:02:59,717 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 13:05:22,226 submission_checker1.py:3286 INFO] --- +[2025-02-09 13:05:22,226 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 13:05:22,226 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 13:05:22,226 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 13:05:22,226 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 13:05:22,226 submission_checker1.py:3307 INFO] --- +[2025-02-09 13:05:22,226 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 13:05:23,280 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431199 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1297 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432787 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3006dac01..fb8477e53 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -642 +643 diff --git a/summary.csv b/summary.csv index 7fad5a662..4719181e4 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0101,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431199,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1297,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432787,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0e922330b..9131113e3 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431199, + "Result":0.433783, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 19fbe9fa2e093a41d7ecd5e77a05675d30bb8372..c9317fbee7db18e7114dd8140c06253169e023bf 100644 GIT binary patch delta 2029 zcmV*9QgduimhmliUY3e_x1zDk05<5JG8^YM+o}4`9`|+J+?me#bT; zaL$QCtCgtsC4upGX6DTJW5$=x=ch1`Ry@u8IO@tw(PW87ZtVHdyer?`o;7S)%5o?2 zoFI;PSAOQ1{JHm6=P^!yWDCx7NmLMJU3rn`$&spN?t+I-ro;)4#F1H?hE6WNf2MOa zOH%H5Wl$KXx~7?G==hOb)o_&VQ%5|T`7S?=-DSw5ys9MSfs>2wXA3{cud!4fFd!IjMTc4#Om4`U~QGB`dy0TWFDq_#dZj4jO<1=R&sY3-O>|pfEzQ)8Vm_V&zK&6=R?8lqI=R#9r17JamMvt7Q_KuI(seN`Zc+NR zC@%V@t_vnF1(L|47`;`m*-=+T#cSdl47r5`cd+1ot3Va~*>p;$JEfyOe`vdXm|&s> zkZ%OSc9B~G(RYzM0s-T3Pav%gvXgF;PPd6F03DrR!T{O6OWeLo+`UWO*AY9Jw&+Y- zC?1UW1QRWo>ifv;`^X(eOpt7CB|F&~WVXi!s=(T9;sg`+F44zCdzU!H1OqC3ArNqe zoe>C_nsWjHDqIjqYZtjBe-JP=0|Ehab44IvY=;B_2KSmk!03)JqJz=BArLUSV*&xA z`;|bN^x0<8XPb#nS1@WP7%{-~^f97?Q9C6NFlt{g(gdS+h7l80bNO%CmgDSn^rEme^&l`qhH^uez9tcp|qM!%YH*Ti(HicO>8(9XJVYvcf0%e@ZJF3=E1I~{p+&T5 z5p6utK%o!MYN%a#QcEu z`UWEwK*sj}N5n&MkHXBr4{f)N9(nthBQ6A1*Gh!JEW zMogg6dAr6$KO!8sa9I;RXai5KV8M_`0m=qrM#L1reJDmOQ0=&VSlL3yjk8AJlr=or z=9YFY*6x?Lq=(3EpDk&7*FUpulSd3Ke}m1zjf$vbNc9rlQ$SH8BFcg~QVdv<$6MV$ zh>*(%arw7g8wqU8S7CXDI9)M?wH5u>eLP5Zdk8sMcdRm!DVA>QZ){B&eUJT^;d|-^ zgF+Ha8w!aTEF{4oA&GnZNJxT#_h&W?DI`$=60)X(0fSEwu29L4XawD*K~WlQk6#HV0pwge602)Y{NV7&RiI;Ip_Jg0XDqr2eNlxw}w$oG0RZ zOfz0AmG}U1fjd|4nu4_eQ?rgFQxRgI9yespkM9fu@tt@#@v+9i-DM|e} z_4DVd_2QMEW#II}*!x&^Y8m-Im;B=g_k(%#!8JeqE$N@6t|W4QK}B|DE5!$@TfI)= z%=xvG&iyEp0zMOoYKkRGlW-0d6Fi<4-^A$7;y8c#QVhxFU8vr_vzZRg4Fss>Dx8yR z94vn!r4_aVZON&DLS0&N6VfITwSJJ)I)3{qPU0lsQ;(~e{dRUyx!9{V`32{t(G|)w zf)dc&2&uOf`e>Hv14?{gTCmn=s8A0cE$X|HTgr`tH)kxkAi*aQq&?-fLc0(w#n|&5 zDCV>1&{5nNr&x%k+hWW3H`@X(2)V}!0uz5MFnj=}b|j(`Qt%OJe_T5N1jo>V0v$Zg zvm9T%0i5!a4TEy!R5J7yCaq_2bnm?!ja}De-E6SN{5k$wy}l2trcxhH#R1i&;FLRH z!MM7_*F`*B%Y0bH6K^Dlr#|WvZ>L3$eISw;fF9$XmCxDpOS43EL5g`wij+Jy`HUEn zIW3CyVdPE!1*lF#{!Kgsj=66?LH!4_tsTk&1?{iiu$z;7AX@>Nlh`0U0wfTVKM)p^ zEFmfZ2a{JJIsti;k|8D=sOBo1B>?~cPyzq|5dZ)H0000000000fPq?*(;+?qu9FfX LBL-j}00000K5fIQ delta 2028 zcmVtwG!37BryKY%$ym2%=ps&{1gV#il><$M;)0dnk@0ii>H1x@5pzzXAN7HvfPcP zZV*SjBR}&@{@nem{TQb|vIXb4Br1rqj=ae8avcLpK*+f77{| zB`J5OWl$KXx~7?G==zad)o_&VQ%5|T`5r%wy=BOwys9MSft!o&XA3{c_oDWAE^ARot% z7u=sOaxrX15#q%`R{oVjUyP+Le@mhJ)RCLA^yp9XMNzS|@TXH=474nH%Pfz>@6{o; zuAmBK<~o*XY!mG$tfiT{QOw74z}GQq)oR(pQ#W_J?KFOr(z1mtaf_K@N4hSC#Vbm` z7R5#1)OEr5r9cvS6r;E5Hrwi|sCZ3$gCV!D;0_ktZxyJbKbubJbfAM*e-HQAo?zHM<8H4?g^x|L3Yw@(&;u)1)!s2Oc)^BcZu6~iMw}+`#NGL(-xg+ z3&n%+9%G^fQ+*$~eIL2QhzXLdtz;)#gUt5WKowZKO`KrD-X(gNaCV7POfaCr7Xkri z*cpLBTX=BXBaWz3HO2!zlaw^3-C;< z3Gcrgds(5bV8MXsA2`FVVa$-20?_acMl6u_=(XRo?!`Z&FXO@GfAH#Kxf#mDZfNf3 zi{*pj#i1Go;%_K+I^t8p)7iw2W;_-9H9pCCmQS+fL+EG4)+d{+3`JA(l=E_vr51ZB z^?J}-$EQ3KTbN8;?`0aBnkQb8G`wWlh@V$;e+f7i-?Z}jV1)h||UF_c!5F~^~$vuNooJmc^;9wP?$!RuiJNk<@1 zI*cIc2n3v<=T?n$L^xOhE^EU3INXH4>)YYLiviI(7|3f28cEC#=8)|QDaN72NU4lSZh zi)iDC1`3^E!~myX4P(-^%L?RrDSQFmI;YdWgMns}@D54ETB<2UK z*Eblk05Y=wKO!C?l|4kfw|ADrrlA|`jU~~cO?2=~13^wOVt`e%hY@5Vfj|>6f=tAS z2~;|FYD^@;J%J0CHQ{{+cya{`21E)_HW)J`rU33kF=BygN6uko3tcbH8i8BZ@MN1? z+Id*JU*3`)BD;OIq|UB?rel*y3@v~C&B2X|sANF(65dllQ9~lif;v(RSdvFu-9Lzs z%Lj4!w_FGg$+n!-b;YsYdgQ|BM@+{IKeaJ*=p<%L4LXS#>?FY;CozMa zBpBqRUgsoH0dlgYg8t^yCE3MhED2#nv=T=<&^V8oR4Y7 zi>1>3ot^7JQryGJ+jYk(D@}ieS#NDz>F1!wjCj1*gid0{WY9^>U?&L%If)tUB*7pj zn{`eS6(A>TD(G(xzB&m@21Fy42AzaaLm~=YM^ZJz3(2kS|B+5^t&xS4x~8z-u<{T| z?sGx8?%3-?4&AaXAf~$YzDUT^Iq&fx$RsZoqf&)#N~@P>#_Z_6df$KK6&DT0(bO+) zk-0(jc1-R^b1C~-y?0|C^}g#`QJFUzF9R;+&k66y{3#K0Rb(qm)2CVW#*LJuew_OG zbJcqB%Fi-zyJ0;2Saxa|`9GKZ;|K49dGx_EKm9G~pQMf?a(_WZc4aHY2dZ1$cH++Y zwVTfUD3by{6NzeyB}?5lFhqN-G2eInhwql1W_Cr6q9Tm zEPp1Y5w-(u$*F-tQ(Afx(k2nLevs5Ue)}qp<0Oz%kE@yec6L#<*sC`A1?Q#FHOe!B z642ZTskb$HUoX=Kl=#53V6D+mqaHk3Genv;DXTLGDq*&sXuBM_575Ehdy zAu0g|lUN}-0eF*>AtoD992pcP0RR9{0ssIJ00000000000001hfmxH(AwB`ElM^B% K245cl0001~=ei#N diff --git a/summary_results.json b/summary_results.json index bd51433f7..25de7fc4d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.431199, + "Performance_Result": 0.433783, "Performance_Units": "Samples/s" }, { From 303d89063543a081ab39384902bfa7304801d1f3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 13:08:47 +0000 Subject: [PATCH 016/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8764 -> 8766 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 92730e7b8..f594c7c60 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 13:05:22,226 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 13:05:22,226 submission_checker1.py:3270 INFO] --- -[2025-02-09 13:05:22,226 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 13:08:42,952 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 13:08:42,952 submission_checker1.py:3270 INFO] --- +[2025-02-09 13:08:42,952 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 13:05:22,226 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 13:08:42,952 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 13:05:22,226 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 13:08:42,952 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 13:05:22,226 submission_checker1.py:3286 INFO] --- -[2025-02-09 13:05:22,226 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 13:05:22,226 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 13:05:22,226 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 13:05:22,226 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 13:05:22,226 submission_checker1.py:3307 INFO] --- -[2025-02-09 13:05:22,226 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 13:05:23,280 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 13:08:42,952 submission_checker1.py:3286 INFO] --- +[2025-02-09 13:08:42,952 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 13:08:42,953 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 13:08:42,953 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 13:08:42,953 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 13:08:42,953 submission_checker1.py:3307 INFO] --- +[2025-02-09 13:08:42,953 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 13:08:43,910 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,4 +30,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1297 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432787 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index fb8477e53..51e227a6d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -643 +644 diff --git a/summary.csv b/summary.csv index 4719181e4..f77e47ec1 100644 --- a/summary.csv +++ b/summary.csv @@ -8,4 +8,4 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1297,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432787,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9131113e3..9485d8adc 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.175, + "Result":6.24741, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c9317fbee7db18e7114dd8140c06253169e023bf..4c8df5a8dbd39a546fc3c09ff20c682968e1c1e7 100644 GIT binary patch delta 720 zcmV;>0x$i%M7~6@*9Qs5hOcRz1^@t2v)~7n1%G1P=@y)gDZmYrHg{?8@<_`&^P9({1lPk&4LC#frm++R?U zUD-H#kEg{q zF}kxj&R@P1L$Y}rs`oFmnhwql1ckVXaFc8tAb(E7Fc5_AJVoStVkZGqtR$5H^+Y8k zia^{fd)r#XKe7#J-X14y(^l$<<9cSlon5(_?^K)og7ebo8s!;5321JF)SDW;uNLVH zB|b1MSZg%Ys0WYc%}vEE<;KCQGZtKs;FAc_o^o5GZ3vcP?D-ZH^I3H0D6WlDEX2}n zuzzLzn{5CWgxq5Vfe98E9)PKxi0FV6d_vkE*Nytw(Wk@4TFhUDst@Ia*`>9Dgof-o{l^sfSZ>Kusk$NPNBQ);S|p&RAjMruij+L8@{*Dny)D0nkr(|ZpgIis zH}MGA=f3;|%^$O_9m)a;$A+(Iody5^QImlnR{@-p*&sXuCJ>WA5EhdyAu0h0lUN}- z0eh2^AtoDzxQTEj0RR9{0ssIJ00000000000001hfn1Z+AwB`HlM^B%24Wuo0000j C-Aj`I delta 696 zcmV;p0!RJ6M7%_>*9Qshuimhm1^@t0v)~7n1%J$XYvW2k2R&xQ}U1fjd|4nu4_eQ?rgFQxRgI9yespkM9fu@tt@#@ zv+9i-DM|e}_4DVd_2QMEW#II}*!x&^Y8m-Im;B=g_k(%#!8JeqE$N@6t|W4QK}B|D zE5!$@TfI)=%=xvG&iyEp0zMOoYKkRGX(#m#Px(2Ilk$@xOAm3L$KlJj1$R827T?6^ z&f++K`BDtY=3S`Xzq6bU&J6^p<|>?%ZX6(gYQr!PgztR{qq{BHA*B_z18vEvfkItc zaud=f5w(7h)H;6qDo)}g;8TyQnf-QlQMuTwHu(kTrO_42GlCM(+z6?+75ZqF=>tl9 zU|O)&XsA#R9xdv-l3U7+gEwa^xFEqN5u`okwnDoQEXCOK9Vq6r=+IHz7^hf>rQ2eE z%lJ3j0xk%-#|i=yEHHckrgkKv6H@RIX@6Wh00hU-f&v{p&a)g}yaAl@lMRD%fgJOLz=7$HFc2a{SMIsti;l_4es esOBo1lfNMr0a}yRAwB`FlNBN(24Eim0001SAxPT* diff --git a/summary_results.json b/summary_results.json index 25de7fc4d..a5a1ca686 100644 --- a/summary_results.json +++ b/summary_results.json @@ -320,7 +320,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.175, + "Performance_Result": 6.24741, "Performance_Units": "Samples/s" }, { From 77e9eb967e42b5907d20e55b9f4badcdb093810c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 15:49:46 +0000 Subject: [PATCH 017/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 8766 -> 8766 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f594c7c60..e5d838763 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 13:08:42,952 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 13:08:42,952 submission_checker1.py:3270 INFO] --- -[2025-02-09 13:08:42,952 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 15:49:41,779 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 15:49:41,779 submission_checker1.py:3270 INFO] --- +[2025-02-09 15:49:41,779 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 13:08:42,952 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 15:49:41,779 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 13:08:42,952 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 15:49:41,779 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 13:08:42,952 submission_checker1.py:3286 INFO] --- -[2025-02-09 13:08:42,952 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 13:08:42,953 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 13:08:42,953 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 13:08:42,953 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 13:08:42,953 submission_checker1.py:3307 INFO] --- -[2025-02-09 13:08:42,953 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 13:08:43,910 module.py:5487 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 15:49:41,779 submission_checker1.py:3286 INFO] --- +[2025-02-09 15:49:41,779 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 15:49:41,779 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 15:49:41,779 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 15:49:41,779 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 15:49:41,779 submission_checker1.py:3307 INFO] --- +[2025-02-09 15:49:41,779 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 15:49:42,750 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,6 +28,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1297 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432787 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9844 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1716 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 51e227a6d..fc056efe6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -644 +645 diff --git a/summary.csv b/summary.csv index f77e47ec1..05233957c 100644 --- a/summary.csv +++ b/summary.csv @@ -6,6 +6,6 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1297,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432787,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9844,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1716,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9485d8adc..70e9a191c 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432787, + "Result":0.433634, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9844, + "Result":23.1716, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4c8df5a8dbd39a546fc3c09ff20c682968e1c1e7..58ce7a33349121d56aba77c02d094a5981d6c669 100644 GIT binary patch delta 2038 zcmV{N7oTln8{+qjofn4~vtR ze$9*XzR8<{>9#=NdE}$F?l-&gI_N5G}H&B+J`A7 zS^)V*AZiu4BM_~M+!F{Gj|T#2?T|{kO*-8sssMC!iU}QL`z~?!E^+@Z@lZ!pGHubB zwop76?SVSjI;y~|HgSpxt4a(o(XJ9_m{5TVUkC)8 zVdn$_rsje`fC`rc(yAhVR|Ep4W=J4lZmtOgjO~a(z~J5x2pHWlMl>+Gw*&%4cS0aw zbiWcvlRn!F`fM}s=?X^e6eBvAo&iQQFluK60!Hl%Mw(#M&M{)Z6K;zTzlfJa3-C;< z2_J5cZC9vkSTG{`2hOk?7&9iO05p7y5fh|6dF}VSfBDbk%XD;qHNO5>Y=%<48(MhT za`h-VQ6L9C{~Pd~4*wLhWIpr4IZOC{jm$@yMN zz8MWR@d-=$7ABQ9dzr?r=9v@64JTeTqA+|;R$<0BQ;iKZbE0tWEyTY?mu0r+r{brs zJSUdxug6Gq$3b0 z9Y&CJ1OiUb3$sQ#A{?v$S2f{79B#nh_5E<*#fWGf4CIXojU?s=^!OGdCO{_U5sh3d zXSS1H&NTSdOIcN8!F`NhvM6zuv$ZA}lG1Qis@6Kd#ABp?a)62V=AMbIwxsYY9a=<- z7SX~J4HP=Xhz?G_0Y;FB1Oi3G2ojM%pomtDh(tIPu_kUDWVw z8GG6}Si4`|k{%PRtQNuKO=|0qJP9{J_pYGWj@ zF5tKT+{LbzOx*5=rqlXz{el{xe2%4b6M60@Gjo?|{&dA6e9KF`Z{ZiG1TqQuKy%GUFj zepbHS4`diRG2|a=5zD0Rb^F^HIak|`Q3o>7ldH!`8dDVXg%3;XAi6fxTef#TL zHX(4%i9@TEsP-j+@poqC%=lx*m(J&>FpyR}&HOm(%1qH@iAQei`O&;9-`$=yY+1^3 zC-R&ij(Aso=9&Dt_gCjJPJd(z&T~mr5M^C?k>|;gs%Gwjhfb!%36I2)S)7J{PAr6QGwIQ~(5x%9fSR-h_k&&qC$Q_15qXBp&^ z`0;}K^F=O(%_u_LILOMsQs|3+vD9TLbe_6$Q82 z*t&u$l$q;Trm;75&+CN~b%2rK3J*yM35o zq6Ls|1j2TaTLRH{kvjqb<8ec9Y#!$Y;7ew*&1ZF#|Emv+HK+l6ZS6A$3%OVIK>15DtsXj zaE6@`2$-640s$&q5J+o(7r7)5Ff{`L0dsRjAYg2V1Of*4nn1wljxeHw(Y+xMFuG#` z0i*ksK$`T~X3}SyiBDHBY9|;m!1VMnqJvR8B@i%bUog@HqjrW76P|D{2=R+}L9_tR zw3_h2%dwXg>IxPNiT;5z>>9?5h$#RK-(bW7X^&s~J?mZkGyXDv8D5UAK9-xIOzei{ ze!f^fC~h38VIcm7Vy7cMB|M!?{bB>+vHBUJ&H(6@2 zmr}2X{dIiGGqHuq)b(Dbv8j3LCP~9hmW?=yp3-HMi_KJHO-QDGYKx(?noY}oLpqC=&cZVef8#M?fFHa*Mv!y_ z0;R(Ul8!*Y33_hTNJoT&72vWae1OAE_`ALx4!jr=t%HHQwxE&3{D2y*lH`9xYD6T zv}qA-Jkda*6O0((^y^~;iAW$&M2sL22?UC0*N8}jLlJAj2RIywXxE5Hv<^kIp@_u% zfc5$YBNjl$_Wwu3L!`2Yi1+r+lGrqKV{1>mV`7^&v5jXM2y%iE1FV{Tj35&U1e%Bu zWFkgPpwfANyT(K!9Jp{<6Fz7IPp)9WkVpZ_24hCV6u^BbMl4Y6xP4gJLdT7>M&Oh+ zJlW=!b}rWLm$#&c$Znr4X?xc{vu)XL+;w0YEnry=^a8D%Pz=2FKrqNbih&Xm46=}5 zOz`7>zF{a2sYc)hVO<4-&B2X|sANd>65dllQ6nOM%7Qvl3|NxKTirj1kjn>g`L|pf z32e+)VR?l(T``5V75&$JJVa)XEqEeBvAnpvZjIogHI8zP|1*J1l^@UQ6nM>3rR2*6q5ZxLY7P9 ze|+12<{$~~?FY;C-pif zi3*UDH5Cjt2Vb3pB|~b|+R#ZDH6o(mv$z|Av25t1{--&)yHI+ZC*pieGhQr}_V4Um z50c^@R^F~VR#|B(ddq6PwNlbgLXR2ofUyaGt;CGUpp}@xRuT-d5;NFJf#QUy zKvvdNFxVV?wGx&LiAF39S_z{@L=?J`q-utjl3U&XBdy$8B?~QeO<}*`W zvDe2Ox@B8HOm*vxk&vfz-seG(Np37gr3&4YRp!#8jK^)FK&`KLG^xr zOzuZ>Df?Nyd1D^+zw262nLC>-11{yy3Gd4MDG_s3WGhSF)2w>uMoLmYPW}A3YQ1>r zXBjxXF!nx{omxiz&n5r(!Tn$!eQ?cBe@pr&sVj-xUr>=<*-G(&>Q=9lICFmOq;o&Y zq=3&vqMBmKQd+&mQ-03lr2J&a(nFjX=W+P*ZNVLnr^Pohy0bXWU%nJWvUwY-_p_T0 z&kY2HxQTF+ZX6$f%}&EG5QOhMMdW*8CjnHfB$WX5L?tAOK-?^Q+gijwvJGk89w%+n zR_cl4dS<_!UAda?RGa*Q^U~-VKnJ}@mO~ozc z#=)yI7F>|vlL*qDa$BQq2$o{(`4$xOS#;u zagWMJ`SiRcTA-#N#a&8@lsv5Rl9Cy{Ex(457yT!oIt=+Y@d()GzWfBuAG5F>$^rz( UhOcR}ejq&o1ckVXaFf3we}{3~z5oCK diff --git a/summary_results.json b/summary_results.json index a5a1ca686..88f527a9e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9844, + "Performance_Result": 23.1716, "Performance_Units": "Samples/s" }, { @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432787, + "Performance_Result": 0.433634, "Performance_Units": "Samples/s" } ] \ No newline at end of file From aec0abf15eeebae4aa530365daad15b281ad61c9 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 15:52:36 +0000 Subject: [PATCH 018/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 8766 -> 8767 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e5d838763..16abf9973 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 15:49:41,779 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 15:49:41,779 submission_checker1.py:3270 INFO] --- -[2025-02-09 15:49:41,779 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 15:52:31,543 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 15:52:31,543 submission_checker1.py:3270 INFO] --- +[2025-02-09 15:52:31,543 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 15:49:41,779 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 15:52:31,543 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 15:49:41,779 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 15:52:31,543 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 15:49:41,779 submission_checker1.py:3286 INFO] --- -[2025-02-09 15:49:41,779 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 15:49:41,779 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 15:49:41,779 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 15:49:41,779 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 15:49:41,779 submission_checker1.py:3307 INFO] --- -[2025-02-09 15:49:41,779 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 15:49:42,750 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 15:52:31,543 submission_checker1.py:3286 INFO] --- +[2025-02-09 15:52:31,543 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 15:52:31,543 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 15:52:31,543 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 15:52:31,543 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 15:52:31,543 submission_checker1.py:3307 INFO] --- +[2025-02-09 15:52:31,543 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 15:52:32,651 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1297 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.86892 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1716 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index fc056efe6..bb33380ab 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -645 +646 diff --git a/summary.csv b/summary.csv index 05233957c..c316ba084 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1297,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.86892,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1716,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 70e9a191c..686ac7b0b 100644 --- a/summary.json +++ b/summary.json @@ -189,8 +189,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1297, - "Accuracy":"acc: 75.000", + "Result":9.86892, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 58ce7a33349121d56aba77c02d094a5981d6c669..3b1e3a93881c3d18e64a372c4f611ded51b3773a 100644 GIT binary patch delta 764 zcmVM_h9eB`zYcbZit0fkaJv1Bc z(mrOM#WfPeMRD#k&e&6_gq-G}HCb0J+8EmA$6lWwapmH^-BGn)_WOadcW~r1=6~Wt z)2L+%{%oRtm>dlzdCbo>iH@%2h8EVjPHp3TsOqg__9rFqi)=pzmjD!^prwFQc!8eD zjxPV7bZ-JX@nOeuKU})Yxt6)#l*)gthNit@%D4LAm1w^n{+LhaV{`-8N^5}D_m8OW zgj(+3MAbCChzDK7qX+epe^lrH9e>~GlCk!=txF;wuLh_eT3)D4&X)z#!xr{}Hw%VX zgb@Gxe!v~H)8ZJP_qgYUb zHXf0aZX6(gYQr!PMfd#*qrENLA*B_z18vEwfkItcvJ=uI5w#vjYMuOj@dJvSr;5`?D=5(k99mU(1v7+c3vX5% zuwHt7@LWR zMkHqw()`$_2XKa=0SU@GoFy5)cs*F@W*d6t$|;3E8JM-6#mRkiVluXEo3xAG8uDlO zYxDZvuNn*0or(df3eE_#K)tqAfv<~jxQ05v2`AnO;7_NZ&%BL`41K_(5P%GE&&ucG z`K4YYqbg7HWt`@5`dDX6vdqaM-*zK!`Y%8^4*56n3>b3XeuC-`v#=da0|h7cZl_4es4mKW2#3* delta 764 zcmV%jXRxYvPevbf-W2Of0bT8uUBYKetp56yQ`>kSs(R~~{YeS@BHNF_B>;seXerypUFs{!hVmKSQ1^JT&Gu!X(g&4M8o zA;kZ_Uof`3z#$0T%$Ct&wp0kw=IvxLUqzeA-C8Xjm#Bplgboe^7oR=|UC*a4R;(6S zfIUA5{g8N0ucy`O{Aiz8p{K=pUR{CHwz}%AM%ql*Zhh%HZmWMIM*jj(OS23VUkC(@ zhlXgAZyX?hPQx$|MfZF~;KGM89C2(5ggPlA;I{(_E;!MDN>m{D30osp7QJ3QBYYhgQ{H!3<&A!mHH= ztQX*-08)-HQ=+~1hG6X29wc>1aHt^ew3XC{($?62Q1(r0zM1?}V>1!a zfaGjKnjhPA0M0NpAVGPDvn0c3uLmpL&4y07a7y8i25wqMadID=n2c@PChe@VhWr`+ z+`PPXtHwf|PQ?IK1!sg=pkCXmz?VfhTtl7TgcI)s@W*4&Z@dkQ^nJji5P Date: Sun, 9 Feb 2025 17:36:05 +0000 Subject: [PATCH 019/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8767 -> 8766 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 16abf9973..71ff626dc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 15:52:31,543 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 15:52:31,543 submission_checker1.py:3270 INFO] --- -[2025-02-09 15:52:31,543 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 17:36:00,438 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 17:36:00,438 submission_checker1.py:3270 INFO] --- +[2025-02-09 17:36:00,438 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 15:52:31,543 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 17:36:00,438 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 15:52:31,543 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 17:36:00,438 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 15:52:31,543 submission_checker1.py:3286 INFO] --- -[2025-02-09 15:52:31,543 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 15:52:31,543 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 15:52:31,543 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 15:52:31,543 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 15:52:31,543 submission_checker1.py:3307 INFO] --- -[2025-02-09 15:52:31,543 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 15:52:32,651 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 17:36:00,438 submission_checker1.py:3286 INFO] --- +[2025-02-09 17:36:00,438 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 17:36:00,438 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 17:36:00,438 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 17:36:00,438 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 17:36:00,438 submission_checker1.py:3307 INFO] --- +[2025-02-09 17:36:00,438 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 17:36:01,379 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40942 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352565 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.86892 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bb33380ab..8026d9d63 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -646 +647 diff --git a/summary.csv b/summary.csv index c316ba084..4488da315 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40942,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0101,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352565,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.86892,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 686ac7b0b..395405bb4 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.352519, + "Result":0.352565, "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 3b1e3a93881c3d18e64a372c4f611ded51b3773a..f4d5b404be358eacde665d401eb88d990671dbb2 100644 GIT binary patch delta 2081 zcmV++2;TRdH*fWQj*^?D^5WE8pFoHEdbR zawqbfAdYxfe&(6{x%XG+F;0JE3(j*%R1jrdd6DPIk*a3yf`?9~#0ihYky)IEe@-sG zrgJq*Qto(VP#CDXrkQH!_>o-IaFp&-M?9PPEpQR#|hdBOGe7W?xvR0rfV$aHMj8n^LR1Pvg9taJPyBC zhuFG;DwLV)Sf;T}bX#F9&D4!zKAr=U@uQTMEo6yP%nUoybulb% zQTnwgF8Zdf3nniGlE|YNy;ZN-QCCI9YvLOWxrGIHu;6~HKo$MjbV{c?f2E^7XuExw zV4?+(Zv?`2ky`@Mcab{+0poE`Agv9ulWvnvw}~nM9i3pp0NK7v+`dcPy-VEJ5j&Z- z=uBHE9*p+{6D^qP`^fG4$Q?#ZkZf%wJJ}j!w#Nplz}ju%1QYfy(Z@u4mpH`)11fwW z5O9W_5eS%?a{>V>To6cWe;2tV5HK|Z0s(V#MIc~ohXeu!_nJV!=#DU=gVDVq5HPx9 z0s*7@l|Y*G*=Evbn~6_XFlr|lF~IcnF`|P}J0%b>YF{wY1fzC_5fh$pF9`9ActNxP z&$OEG!OO9i73vBW42k}MGwd41jEE@!4c}nI0%?z5`#tMj{4@SCe;Hnmu0EEVp-k+C z=6=3dJ}7P+s$n4hhGM59J|#SzP5o%bQ?Xy;)0}7dG+RD|epYOKvgyiDG&N5-FE?3g zv6oV>hy8VY$}_Qr$<*~;rm?Ac>Ly9UO_q%~ik{MCl#9(&V@*xnIGXu$`ES|dx#I_! z{HdePh~@h0AyN{Xe^xHKR{ndVU*D>Jv1*HM8nz+)T zMYL%VZ9LIHp%aW4;PmTb1c^u>P(+L%5eWo}XxE5HghLT)!Us4UiD=h|NVE<`w4sQ^ z{DAfP1|t?g#`gb5#6zUAhluy~&XU+Pbi;h(s;5nx*v2yr1UbQo0andEMv#dF0!_pS zG7%#tQ0cr~e`6vM4qUjb2_Lk9Cs(jwNTdK|gE1pw3gA8zBNnK3+&-*qq2tC`BXG(Z zo@{eVI~QyB%UjYzWVg?jw7u(}*|zLA{9_s|U|A0I09!HtTjWJvWA-cvwPBO=Oze>zeOSdzzE-9Lzs%Lj4!w_FIQ>C5=jl#nBk{3h7O&?jHy8JUq9DH>WmJF#;YeOet)QE_J&*E+f#HHH0#mxoAlpG(Sh z$F?4Gm`(VGg1Ys_NXXMU@ADwYBsUhLQiX0xtJi2I?C8FFMyUyfKgZ-*v61%$-e^0hjXUgm-2Bl!&=1vXv$8X;!^+BPFRHr+)riwO+jR zvkaVG7<(VfPAwz<=aPT?;C?WVKDg$mza{;X)Rjc;FQ~|_Y^C@>b*tA&oH@UC(zzdH zQov^-QBARADXre(DL?0NQhqXI=^@VZ8903Tw&0G()8d;L-B}#xFJFow*}M(a`xmpE z4$ln)=>f~rlW!a!f6Y$AFc5_AJVoStVkZ>^tR$5H^+Y8kia^{fd)r#XKe7#J-X14u z(^l$<<9cSlon5(H>{XNeg7ebo3gsC=321JF)Y}TZub1gPN_=2iu*PVpPzN3@>YI{V z%8i3pXDqlN!6y-|ASyAUkJ*z+AI=CkO~QQR1(Scs+De`3q{H`@X(2$^FAfe98E z?t!VDi0FV6d_vkE*9-u`F*Kk+2aoeC$0u(9r~K81K{<0O89EDBtw(Wk@4cLiZQEw; zY_P`sIsRO|ybY_SQun9gfa+3k${ny^TwUVxA|9?`KCI%2Hxk5S7xgP|hegId5J?O` z$GAu3WA^mCBwnJrAjNG;ij+L8^E*15Q$oJ_kr(|ZpgIisH}MD(j?VBA;%uTs&BOoN&fwg zZ6NrZ6NgqSQSD0txGM+cz<`^utXuv z?9jFSC}chHg{9)>{$JgvDEW~t8OsD-L74W$WtPPqSx%iL3+z;iVixiv^C$`Ie~f=k z7IGRV%yx^Qz?U^eG33DZLb0r&lN?eAKa^?0!B}Zk%o)_JiB!bK4_R7zu z(bFaK7R!teo1TX_k)IZSg}~!uf2oN=U_bZ7rYJmlZnn%T7M7mtvV5RL!C9qQ6nrla zshbMQP}SJPs*1XcZcA!QF*H4&j~Ab9V&t;bqK9X8X7{^E^duxj3u$cUGoyAijSq{H zmwwHQ^S;TOg6XzE;CbYux9&H)@;a}0LwtiFcd+0d7Ch_~$h<$hPHA+fe>BtwrP_xn zCRzabMj&bxxg!v*irf@@6Y3-0ox=lLWCaM5*bczWbWcx00_bze&F7Z%DR5ESR znYK_o81E@2S}@i3k-PVidyE($+1g4f+3IArCpxOYtTu6q39CvBFww3OXP8id3SS5W zoMGn#0;cAIK!6ID1k$P^e^&$ore;VWU~aAn1dQ#7K)~SM5C|CEF-9~ny0-)ZMt4FW zV06C{NRvL>4Ek&{@aYOh?Gz(An4SSfG%#vs1Oi6w3r3n?)Xp(tz!PqZ5Wk3*L<{gt zs|g=&k8M||YgjNM`UlRi8yGVtrT{d2ixCr~J$dc-ynp%6J}TSZsz;z8hM2 z*>d$LIZ+@7KK~o=oeuvLvt&N=!Z}O$evQpCmS(eb^%!_*zV%6GYh6<0EMctJWXbtn zO1>ElHt`8d`4%RXH+z}JuI8B&#|MVDo^=cnSQ zt~@7}>#xU1iEmmNfA3oH?~Q(aEBmFETMVhyY?{^^(wVe$CZ2Kl8;=nk{NN2Rf}|r5 zC>=(SbOZuU&rg}sib%{4 zSg&s}Vgh7h{eMI}Mk)u0_+V9*#HOz4dwb%FiEY}%HlAr9$SFp2uxbu4f=nb3Xd*_C zi5M|}N*C=Ke-nvt;KEf+_^=H;xrPNJA_XWLj2RPC0QaF7F+sJH_HktkY$r+^zFpMt zY#DpnIas@2-jW_8yF<35?W%u9+qB-e>!2#MfJHgb3$$`dG4R#{!5{}I21-aU$U=fK zz>ojMj-foH8i5ytbrlSE2X`u>k`dKQcuxUEjfp4|f9gmvU`d|rb^ja)XLbxJBvAnpvZjKe3ZEieqmmKP2)avyqQ*oN7Ls60D5Uxa30W+W z|M6{`f1@OLkdx}3lUh?!&AmIsRVV33OeZRSYNKn=Nz51ubP_YzNrFL6Vg@@&Fvv-* z&Pk#IPF5F6m$8_ik4eh%rPBJH zo$FCjJiyAkO~*mQDY(sT}e_k!%NA%?*EZi?yZu!mYO1|zv1OEl04*+ za?`QB#~elzzM&xRy)hE7WWff^_fx@%_^6bjyVCMCnyK1(DBn1F#d(8K=z950GTSfT ze~-z$a3Q2W%QtU~&fvSIB?t9_7bz6$ev?9S9!Lg;6BgGcW#6@@uI}bUdq<< zmwr~h-47!7W6`Nq=>1%=j~|>5M(2ZLeEM6^J_$X6=e|XGc10`s2dZ2BZfq~ujh!sK zFco|@=ZPwkDGEvX7Ekdxi{j#wE((uP8kR-D_S=%#E=%%ne01kglx@G{L$Z4ts{b#u zn-0$n1P(SHk&|v5AAieg!!Qs<_x%c^y)D}zr4_aVZON*ELS0(26VfCRwH`=no&0^} zN1Oz_>T-1NoVhcoSRG{({epEus}f~NiXu=uZwWFhC06qC*BF*Pp6>Iyp4+teZZp- zfDCcZ%ID(wr6gXXDo^ueoaS-*SZ7PJ%*i6(b|Y{4FF-jC`8V+l7;@i!g6gxc9m@g* z-Z+=fos)kcTLGMt+aNpwCJ>WB5Ehd!Au0h1lUgA<0eq8{AtnS4HXf0azabU@U6a-! RJ^`_l6(S=BV;=wj008al&Vv8| diff --git a/summary_results.json b/summary_results.json index f288d3e58..2c3ea2956 100644 --- a/summary_results.json +++ b/summary_results.json @@ -176,7 +176,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 0.352519, + "Performance_Result": 0.352565, "Performance_Units": "Samples/s" }, { From ff87e87ef99c1e10987757df83843f6bda7b8369 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 19:28:57 +0000 Subject: [PATCH 020/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8766 -> 8766 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 71ff626dc..ca6312463 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 17:36:00,438 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 17:36:00,438 submission_checker1.py:3270 INFO] --- -[2025-02-09 17:36:00,438 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 19:28:52,143 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 19:28:52,143 submission_checker1.py:3270 INFO] --- +[2025-02-09 19:28:52,143 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 17:36:00,438 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 19:28:52,143 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 17:36:00,438 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 19:28:52,143 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 17:36:00,438 submission_checker1.py:3286 INFO] --- -[2025-02-09 17:36:00,438 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 17:36:00,438 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 17:36:00,438 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 17:36:00,438 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 17:36:00,438 submission_checker1.py:3307 INFO] --- -[2025-02-09 17:36:00,438 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 17:36:01,379 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 19:28:52,143 submission_checker1.py:3286 INFO] --- +[2025-02-09 19:28:52,143 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 19:28:52,143 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 19:28:52,143 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 19:28:52,143 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 19:28:52,143 submission_checker1.py:3307 INFO] --- +[2025-02-09 19:28:52,143 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 19:28:53,075 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40942 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.0101 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7891 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352565 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8026d9d63..775971ea0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -647 +648 diff --git a/summary.csv b/summary.csv index 4488da315..356a1095a 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40942,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.0101,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7891,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352565,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 395405bb4..59d05c3d3 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":49.0101, + "Result":48.7891, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index f4d5b404be358eacde665d401eb88d990671dbb2..0183bc2610668322dcb13f05bb8f24e0b6abe9b6 100644 GIT binary patch delta 1133 zcmV-z1d{u{M7~6@*9Qc4VtZ<{+XuD-f3g+JZZld$n-rg}+ib%{4Sg&s|VgY1q|9?b0L@IlTcyI44 ziA_T{%r~xj+O&ynJkvms6O0&O)$C&gnMfeeM2sL4F=7Ih&f7I665+sw%bM^(e;asm z1q+5m3Q#r}Ga{w{?n5zRfojL?!^##qZk#m&r>x=0Hn+5Mv39?_B|SuT`)oIjn(f8Po8NR1(FeoI!w4sog!9o%Y5|X&bkAx%`cz(1%u7OS0`b~kQ%i%bP`65 zh$#3h?uKA28#<}~X-@7glpg1aI3LrD7fYr6J3H5dq_~Hbx9g5oR+@_5vRZGgl=PF( zV@5n+Y(gtBV=`zZX0Vk6e}k;V47QSBkd@6kD~Sq_l{FO%HV0p=ge60w5le$s!l)4u zg{~y2n&GA7R`>r%E4Nn3LQ7p!*l&1wh$Q#9q+EAw>oJGfgl{OQTW^enJe~7C4}wf` zV=*dK=%%!Kjb_4*?yEOWUUAW29C?0mlgtUK_hWKDnoHTw>dhPTf2jXm*NV#A*<=}T zDSu9QSLRQNn5!aNS@ND{)jKy*lKOG#=g(E^#Y;cS!0Cmt_p$8MGV*^e`Nt3L2lMEI zYkvA$(mzREN#y>5itNf(iVsw`dY#0X^J^!a`%xwZd?pgr6ib%U>Mfq~a~>z5Q)*9#c}@fr5KXU+fcoK0kfM9&kY2oJ5ex`ZX6(gYQr!PgztR{ zqq{BJAte#E18vEvfkItcaud=f5w(7h)H;6qDz@V!kW-JVnf-QlQL#G8Ci(^IgjOZW zk`zUtnC3$5OY~8%;{}SGr;5`?D=1M14y~$(f*HcJg*U4WSTDdu0i+yZrbGwt4Z+y4 z14!zU;7~!_X)CD@rQKtHL)kao1J2X*5lirtQ%~^;7@LWRMkHqw()`$_2XKa=0SU@G zoFy5)cs*F@?l$zwl~W4e8MtdbirA;Tr1vCY*RDfIoFXzw9Wq2 zBwvtuzCDe+>AwKwION~NGhoPl`w6N)v#=e?0t9wqdup?OAUy#DraMtElfNNAYQIA`uQ;xU2~uf3$%o zSFm76qyS}uF(YCM;64;17N~aIKCEn^hx6hWez3ZRZ zw(K|jV;U`BSq}69t(;H{y!AjZ$U%yM5)urukYG&ku+pL8GVoanBjZs27^KpOdAS`87w5hAR&o+{76WGf%j)N3@Id00TQyN zf&qh15w1|lkZ1(mr9n|6A_@yhFcuV&{Xs&OOXPoi+vXq%?&T!ga#C+9f6UstL%i!G z{fOz9;ioo+4xPk|sX-?(gPkN8 zh=_vE;%*4WvZ0gupXTK5Lg{gyi1RVcc(GL4zq4~aNQ!${dAsgdWu>X;Evxm`N=ZKn zJ!ZrM#wN59GbV#pVg_4De=x{O%wQ`C23gsxvy!L)Sy@xTU~}-*N?0-^8nHBJC5##o zQRqsNsu^BNZgu~Uv~p{eEVR@$h5d$?he&dtOUiY}wjOhsP56d_y7k6L$kRFR^B~A1 zHx{E(g>Fi#*Jvi}=)QX6psD5Stj?SsdptUy32wybaa+7qgoV&kY3W0n5{qZX6(gPQx$|gzr2> zqfbB_xVK+$?+BTEsuH4QbvUCu!4G>WSleX1|?Xxm@g3ll+47(&!51 z89@nXZiLj^3catF={-t(U|O)oXsA#J9xdvdl3U7+gI8xPxFEqN5u`okwnDoQEXCOK z9Vq6r=+IHz7^hf>rQ2eE%lJ3j0xk%dV+DZ;78veQgF&0 zuwYzW;`1UNu3f~rlfNN Date: Sun, 9 Feb 2025 20:32:59 +0000 Subject: [PATCH 021/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8766 -> 8764 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ca6312463..083402459 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 19:28:52,143 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 19:28:52,143 submission_checker1.py:3270 INFO] --- -[2025-02-09 19:28:52,143 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 20:32:54,867 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 20:32:54,867 submission_checker1.py:3270 INFO] --- +[2025-02-09 20:32:54,867 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 19:28:52,143 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 20:32:54,867 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 19:28:52,143 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 20:32:54,867 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 19:28:52,143 submission_checker1.py:3286 INFO] --- -[2025-02-09 19:28:52,143 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 19:28:52,143 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 19:28:52,143 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 19:28:52,143 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 19:28:52,143 submission_checker1.py:3307 INFO] --- -[2025-02-09 19:28:52,143 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 19:28:53,075 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 20:32:54,867 submission_checker1.py:3286 INFO] --- +[2025-02-09 20:32:54,867 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 20:32:54,868 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 20:32:54,868 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 20:32:54,868 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 20:32:54,868 submission_checker1.py:3307 INFO] --- +[2025-02-09 20:32:54,868 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 20:32:55,743 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,5 +29,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.86892 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1716 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 775971ea0..75a2d101f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -648 +649 diff --git a/summary.csv b/summary.csv index 356a1095a..d703aba0a 100644 --- a/summary.csv +++ b/summary.csv @@ -7,5 +7,5 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.86892,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1716,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2127,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 59d05c3d3..8eb757cf5 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1716, + "Result":23.2127, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0183bc2610668322dcb13f05bb8f24e0b6abe9b6..c238fda89346494eb2de17e11de7971253556b94 100644 GIT binary patch delta 786 zcmV+t1MU32M7%_>*9QgNu}vJBv)l(U1%IM5-Fm}G`VrGH!%uAt9Xg2_Q-e-o20KYG z$Vtp#CkX~Qsnm6CoEdd!Fij7?}IW=saH#0<8QV1JO6 zn88*O46?FWXC+Yqva+Uv!RFwrm9S(;G-7GcN*FaFqR^EjRWrPl-0J=xY30@`S!k(i z3i}N&50T_Pmz3*{Z9V2NoA3<snEnJDV&6F6GY&@5=ls5pz{!D@)$fta|50 zN>V>g{rtIVy?E(o892Q#_CA)KT1NiQCI9%r{a_w_aLrGDOZq3ND~a4+P?25PO7Vf} zRauE+ZX7Fr*}A1$Ez$>+_`tMaticNMpk z8wYRBSa3mtPa;Tr%59ByAy|sB=Q~i$XVIafxG_$#5KFhkmho@41zZqvj}-)eCRkwj z08H&jL?@)+Bhvo3b^r*Dp#=pxc${ZBzIX#Tore6Icm^DE-+qGT53{fx$^r%5 zu}vJBlYbyv0hyEAAUpyi5R*X=7LzX_Dgg(RS|K_Cd6ShPCIn;ZvTBpRAr=8zlhz?V Q0j`r3A|nQ19{>OV0QDPnApigX delta 805 zcmV+=1KRw&M7~6@*9Qf5VtZ^N{lh9*EJYZ}>D=}j-XeDN_m45_- zti%kql3?w3 zT~pX^czK8<_qn88cWmo1huMU0D5zU+jD$R$^F9xPOmbr}DplyFw0ezZ!jA5%H%?x0 z(O?{TesPn`399#FazC0&+0W|D8-Met|6SLL%G}vx8E`3oPIy=5Pl=eTB3oJVo@UiM zH&T-Naq8#KRqMq|Kg+=Bg|YXs?9?*ye=hmQ5AFx^=!0v1`diXJNnJ_g{(_3^%2tXG zRJVGa#F_JJC!PCICIx&Z64ew-meT4ip7L`ZC*>zYmLB3fkHeR53+{M4EfKzn(VfL{ z{_>?5lFi#ty?+6-nGVkl1g1MtFq3Q?D}UP|B@wm*ZON&DLS0&N6VfITwSJJ)I)3{q zw&Ns_Q;(~e{dRUyu{z2o`UUHRRwc@k6h)wz=0fdD^ii+l1&W-fiql3bC{YIvt*VED z8N#%MH>(X;FTh0sq#R+ULjE2$5q-D5-9H{Aoy)ASKb@PCw3 zPw@#Dn~8`Oli46V0woZWKM)p^Eg>oa36oeMIsto=lOZM>raMtEB>?~cPyzq| j5dZ)H0000000000fPq|-)FD0ru#*!aBL-q000000Dsy() diff --git a/summary_results.json b/summary_results.json index 698602ac4..af54a04b8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1716, + "Performance_Result": 23.2127, "Performance_Units": "Samples/s" }, { From 37b3835163bce2fcacfeebd54b2f22519d6cd3a1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Feb 2025 20:35:18 +0000 Subject: [PATCH 022/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 8764 -> 8764 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 083402459..627fcabbc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 20:32:54,867 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 20:32:54,867 submission_checker1.py:3270 INFO] --- -[2025-02-09 20:32:54,867 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-09 20:35:13,255 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-09 20:35:13,255 submission_checker1.py:3270 INFO] --- +[2025-02-09 20:35:13,255 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 20:32:54,867 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-09 20:35:13,255 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 20:32:54,867 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-09 20:35:13,255 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 20:32:54,867 submission_checker1.py:3286 INFO] --- -[2025-02-09 20:32:54,867 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 20:32:54,868 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 20:32:54,868 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 20:32:54,868 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 20:32:54,868 submission_checker1.py:3307 INFO] --- -[2025-02-09 20:32:54,868 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 20:32:55,743 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-09 20:35:13,255 submission_checker1.py:3286 INFO] --- +[2025-02-09 20:35:13,255 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-09 20:35:13,255 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-09 20:35:13,255 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-09 20:35:13,255 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-09 20:35:13,255 submission_checker1.py:3307 INFO] --- +[2025-02-09 20:35:13,255 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-09 20:35:14,297 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352565 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.86892 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1333 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 75a2d101f..0e332519d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -649 +650 diff --git a/summary.csv b/summary.csv index d703aba0a..54aadd323 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352565,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.86892,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1333,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2127,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8eb757cf5..c50db9483 100644 --- a/summary.json +++ b/summary.json @@ -189,8 +189,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.86892, - "Accuracy":"acc: 75.800", + "Result":10.1333, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index c238fda89346494eb2de17e11de7971253556b94..391f3d6fe1abf3dd3f8cba3c2f20f97e63ad1018 100644 GIT binary patch delta 1286 zcmV+h1^N2CM7%_>F%SeU;7Hw*H4q|y+iu%95Pk1g5ZagaK{wjE8IB9Xb^@c0W7x?e zeG+JCY-?8{Rg!X(zhC;6rlOQhTfjizGvpl3%$XUd|M{uN(H^+S7`-xF%P|npgrynX zUKtPf*JfxSAqh=M&L~_N&mfG?gAaWnBvMzYxH5K9mOa}R$qou4ELH+lmJQ>71(E72 z-`b+&KvJ;-kVS4|$MNlgWYj1 zIDxAKDB&5a;}0~Gq7E)WRJp99c}@y~&A?fkQB*hHZ8mvEp$^}(e}M{HW0;N`9-^W_ z(=qx8*49EQ9cWmknfz2gIbu0~<6q!e3$a=knUuimhT)!MIaz0UCiP3tLPAkT=PHHT zn-`oVAOzF=mw#!DNdh@=BK5;WJG^2W`JHercXKB7M?45f8w;!`D^3zEz%fV_z2;}6 zeAF?|a!ki@+!h?y@svoSWTi50A2e*=&PJ>#n5y@77n4M)4ly}wfU5?70W&EpvwT(= zNp|rbTfTCd!iFfJ#R;REY-_(Mus6x7wQ7RO%!$gFjGsckowuv=@-5NCPEMN1;f1Gu zqSf$nwQhLHN`pa*KTR6y!k_N*XxNOkawv_)gz-pYUYMN=b z*;I$IZ$9bcl@w(T4TJ97JDnE1+k$&7xF(A;9<<L7kBNJs{OIwca*(_qaX9Z!eo2CkLX04?tyQP+vI+`o#d zZn}sEoyDUEb;&0k?c;j0DXTfXlHKlTg8 zmKQocLN}9nJe$lF;%oDEG@CBs)#z@imX7PIg%m~(jzSkVA3g|OPsbN4Rtqe|UJyn> zEM71?Q#FHOe!B z642ZTsW&zHXcy@NN_=2iu+C_x(EuLJo4blz%8i3JXDqlN!6y-|AW+Yl_p*z+wY z=CkO~QCu6RScs+DV9WS7+W;;IxyK3u6Mrl)ya%RsBBCQw@Cj*uTsHy)$IyWS9X!sn z9G|@boboptM&-h(WEd>mw4TJtz4LN1_I;oA`LK*9UP};9L)34)9Tz$Dfk# w&CAt(I}sE$MaO*{b(xvxJ#^9Qr89ZdrSF5pPrlm8&P1a5+EDwDk-e?Tr~jsO4v delta 1286 zcmV+h1^N2CM7%_>F%SeN_HUDuH4q|y-*4hL5PrYE!pgnuKIE4nPJ$>a1hhg+DT4OY zdx}glK<*Md7dveG-=870+l`Z6_f|rx_{N^kGxN<19e#PrGjxPfq?}zDuH_gASj>}@ z?XQf7`)e~WkdTxmG~*1ejAszWm(g!SAtX{)nYc0zQWgW-7V!acDlA?AQcf`d2+&kGhTj$XD!5Po2ODj*))ugG|lKX%~GjfdKMD$Cc0E9 z)ZV-*c??2u&42N`#u&$tK}n^4m}-YNd?&xrQp??vOZ^cK0+QAOE6S>*u@>MIq>5hi zGf_U8m}fbr<2Y^yj+=NwB~`Lg8MjXwwr>{`p66WEdw+;%ELDe?CG4PoR1E@VTvTT9 ztTGZGq9eA5a+<)7DxpO&XS;N7zbmkJ$*Qetg38RO%9xCv0@BUf*Lll5)x=I~&1CS( z(>&2>c)i**yk@1vphci{j0!#bwEm19fwR-!Z@!89q4%f>;dY8?|0x{2d>HDg8LnK(1B|)*0`&GB^HuBG#l>HK4zZ9 zH4?=|aqctD*i)&5oaUf4SywIE7~1B?UY{Rv<>J2GQMF(8`+>4|aO5=R;zQG@WeWam zqJEeh4JLWa&ozmTuH}Xn*11k?<9(>=tz-5lCGd-EKL(co6r!M|fKzyZp2?0b|DSYk z0z2_x$8tYhy34tLmbu@Q%73kfroCdyxBB6gXulr*m`~?pbOYB)Yk=1GkErj2TJGOO z)ik|`2VKOY2lbMFROkO4-{_LD_PMP~A|I~?s2^Hhs7=n71=GV8_JTJHhFF9U|NDNy z*zy90AapZZMvK`}Aw-+Elf`@$Z6r^#qkUq9o)+hMbp>jt>8q>WYNX9{?besRWwH1?Q#FHOe!B z642ZTskb%ySS`{Al=#53V6D+mqaHk(H+L1clp6I>-g`M3yROT+*$eaEPP@RVSn|KBsbKib~<`1*39ZdrSC-!fXlm8&P1Y_#5YLmSoe?b6xg#Z8m diff --git a/summary_results.json b/summary_results.json index af54a04b8..fd02276ac 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.86892, + "Performance_Result": 10.1333, "Performance_Units": "Samples/s" }, { From 1f75bd7dda14413ce6663f82cc27ef3fbfa5905b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 10 Feb 2025 05:45:25 +0000 Subject: [PATCH 023/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8764 -> 8765 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 627fcabbc..280cd779d 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-09 20:35:13,255 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-09 20:35:13,255 submission_checker1.py:3270 INFO] --- -[2025-02-09 20:35:13,255 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-10 05:45:20,339 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-10 05:45:20,339 submission_checker1.py:3270 INFO] --- +[2025-02-10 05:45:20,339 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-09 20:35:13,255 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-10 05:45:20,339 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-09 20:35:13,255 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-10 05:45:20,339 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-09 20:35:13,255 submission_checker1.py:3286 INFO] --- -[2025-02-09 20:35:13,255 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-09 20:35:13,255 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-09 20:35:13,255 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-09 20:35:13,255 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-09 20:35:13,255 submission_checker1.py:3307 INFO] --- -[2025-02-09 20:35:13,255 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-09 20:35:14,297 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-10 05:45:20,339 submission_checker1.py:3286 INFO] --- +[2025-02-10 05:45:20,339 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-10 05:45:20,339 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-10 05:45:20,339 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-10 05:45:20,339 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-10 05:45:20,340 submission_checker1.py:3307 INFO] --- +[2025-02-10 05:45:20,340 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-10 05:45:21,262 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.40942 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7891 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352565 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0e332519d..25e13f24a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -650 +651 diff --git a/summary.csv b/summary.csv index 54aadd323..04e0327d1 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.40942,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7891,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352565,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c50db9483..a6a84d80c 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.40942, + "Result":0.391627, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 391f3d6fe1abf3dd3f8cba3c2f20f97e63ad1018..79ce5c0d1de0ef77ecb90970784117361d457096 100644 GIT binary patch delta 691 zcmV;k0!;n9M7>0?%>xRk!4V$S0{{TG4U^#q8ME*M*8+ccFTB7Xz)3fNlWx$?1!I?7 z0ubYUkCbZtm0=>Uj%$`9U2I)H8-@9)s>2WHZl15?0(aX`&X&LhAv^nC&# zhIsSA$7Wr%stQ?kREw)kY|n$Y6R`Jx67VCI_kIF4+2#Dw!>Yx^R3AP#Yuaw9t|v(R zZ2J2^_4j|it5Y>f7F9-&+?A>yCR=sisKc>PyiJ-`9+jLjPUWu2ZGE+V?AKp@RLn2(isU|jIOdeizZ~qB;=L-J?P)h>@lWrgyv*!na6a*d) z*^`rO94vpSOA)pMZON&DLQ`6D6Vk>JwSJJ)I(hplPU0lsQ;(~e{dRUyHQ%W&`32{t z(KX65f)dc&2&vl|eXJJgBT9T=TCmP&sL>ugnm2b9x0D+PZ_Ze7L4r>rNPEg{jkY0J zim~TgP|Ro1p`*AlPO%V6*J8`~H){bGggjscfeC*W7(M_~I}*_eDfo!AKdu`9f@A1F zfes$$S&lE>08aVIhC#VZy06oS%E1$FHm(>C_1u4pu6lqRY zq@)=4B`wJMF!HAV0#v6V|0bRR$K1D{p!ox{t{sE|38=vl9@PT?0JoDnAXfp(lanAt z0h*K9AUpyj5R*R;7LzR@Dgg+SSRpzAdXtkOCL10O*^?y!002+|000pH0000000000 Z004l2Ta(lwJ^`%>xdOj+6D&0{{TF4FCWb0FzH38ng8S(gJ_?+>4$M=iC6!xj{P> zj7@S0K#Y@W0zk~110ZJ3wIQvX10d!hKX@nS0J?d)zeB$tm^o(-OaTYR`7|>*j{wKe z_X&U);>`yin{?HxDrD7B9j-R9JrCZ_zuy1JzmHhn`}x;om-9;ts}>VeefHq2>AI!5 zogneEY3~Eo-ur*9&eSMbR2e~XSE_cHY}Iw64#z_AHfcI}RC3BVmAfXj_06KnaG10B zG-6ulg8r=W|O&={%U4p0?h?WJzDfw00+iPvO?`I;62Wh7BM%e9rt zyty0H&ytn(ld9}1CRN*icuUmEM7 z1?Q#FHOe!B642ZTsW&zHXcy@NN_=2iu+C_x(EuLJo4blz%8i3JXDqlN!6y-|AW z+Yl_p*z+wY=CkO~QCu6RScs+DV9WS7+W;;IxyK3u6Mrl)ya%RsBBCQw@Cj*uTsHy) z$IyWS9X!sn9G|@boboptM&-h(WEd>mw4TJtz4LN1_I;oA`LK*9UP};9L)34)9Tz$Dfk#&CAt(I}sE$MaO*{b(xvxJ#^9Qr99fSf2j*gS{)B^wjwv#*{R|3fg zlTRQOld2#u0hyE9AUpyi5R*U<7LzU^Dgg(RSs^+Bd6SeOCIoJRZYq<$Ar=8zlhq+U Q0j`r2A|nQ19{>OV0HrQDc>n+a diff --git a/summary_results.json b/summary_results.json index fd02276ac..ee90265bd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -32,7 +32,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.40942, + "Performance_Result": 0.391627, "Performance_Units": "Tokens/s" }, { From 38d1b93b80319c3073a17c33d7982e523db5513c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 10 Feb 2025 17:39:15 +0000 Subject: [PATCH 024/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8765 -> 8764 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 280cd779d..8b972e433 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-10 05:45:20,339 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-10 05:45:20,339 submission_checker1.py:3270 INFO] --- -[2025-02-10 05:45:20,339 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-10 17:39:09,949 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-10 17:39:09,949 submission_checker1.py:3270 INFO] --- +[2025-02-10 17:39:09,949 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-10 05:45:20,339 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-10 17:39:09,949 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-10 05:45:20,339 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-10 17:39:09,949 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-10 05:45:20,339 submission_checker1.py:3286 INFO] --- -[2025-02-10 05:45:20,339 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-10 05:45:20,339 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-10 05:45:20,339 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-10 05:45:20,339 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-10 05:45:20,340 submission_checker1.py:3307 INFO] --- -[2025-02-10 05:45:20,340 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-10 05:45:21,262 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-10 17:39:09,949 submission_checker1.py:3286 INFO] --- +[2025-02-10 17:39:09,949 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-10 17:39:09,949 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-10 17:39:09,949 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-10 17:39:09,950 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-10 17:39:09,950 submission_checker1.py:3307 INFO] --- +[2025-02-10 17:39:09,950 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-10 17:39:11,073 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7891 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352565 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1333 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 25e13f24a..eddc11122 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -651 +652 diff --git a/summary.csv b/summary.csv index 04e0327d1..d00abfe31 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7891,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352565,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1333,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a6a84d80c..20147ed40 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.352565, + "Result":0.352127, "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 79ce5c0d1de0ef77ecb90970784117361d457096..16642d34c2d4a53f0396e7abe4043fc0b2785ba9 100644 GIT binary patch delta 2081 zcmV++2;TR-M7%_>*aroc;Dq&=lide4e{6$*Dk05<5JG8^YM+o}4`9`|+J+?me#bTt zIOoKn)k;+RlEC;oGjnGAG2=_;^K;-!E1slY6n15%XtKmZCvv@T-j(m~&TEz|rI{VN zwjYJOE5Gnm{@nYk^AsgN(go+4Bq|8guDr;yxUH(Gv*3Z9DpAZsaby-Hft`u3f5}`; zWF4D&*5j0vkZ8cm6asiw=>cGbm7IR-0QfWIF{!l zC4APEPguMEoz-Qv*D2b5@c2`@^I6JMd5oeT`IjrVD{DEbBKExK#yF8&KC_p8Hi@1t zxi?>AV%VB_h!go~@mC5wF_yY4e+Bk)SFX#_ljmlOykcqLxh~HKT9%w;nnl6)@(^2B zP=+#N9m_PfjczHdp&5EJpN|)xuVd7*)uM-Ic4qfFN%SNoMGI+c=QG3Fx-N#r$xFZH z#d+VgYAGsqCeIL0e5HKDO1k%_dd+FBcbnB=B(9sDdnjqVEiMw}+`*(?lDq=6w2AydG z#e?ylV4?w2eIL1dAGybf0g|n(WG~w$ne9mvRbcKnaf%6RpXg(vwNIR3f&mr25C}NK z&Itre%>{t~6)p*+v5#C4e+ZbG0fB(Ixh4=WwnG8|gL^|DV01?q(ZT565(pUGF@b>5 z{YoHp`fM}kv(3P#D;Tv?j5NXY^f97?Q9C0LFlt{gQU{}Uju8W%a2tg9MZ6?hfM;4o z_+WEvvqD|Nf+5jAaE9H$m=Q4rpy6ALm>})(Yrki`%YVjSCc~@IfAz;=Gn9(m(A>)w z%SXkD0yXf(-$3ki#HW}iv#A%(cp~;|e46nzo2JXhz)SP3PdZ&S6-~_&&WlZ!n(w95 zn_+()pYT*{VN!Lym#J-Qo;q<{bK+$!3d84Q8D?TLRa;Y2CkkiYT>e{jd1iZlDu3#z zGh(^^dW@9Brj?7Xe-;1U=-0QhU##3>D2;mEwBC@;q@^?QjKkk}j5NUyULPY!Is$>x zVFXD>Am9YOFe{`Z!odn~RS`bG;RgI&-wg*|42jmkK;D?pNMe3Kk8d$z0%U9+(a8B? zYCHMmOif(9lvUN{!pHa}ixOusUFnLUXf zXyJ(l3Y}u42~NL0Mv#aE0!73K5|Kclh*pJ&L^u?&B7A_uk%(4>h(zm9L<@>Y%nw+v zZ!uy5WNiI^L_9_+2Z;D!?JbG*rp|PeHnByU*upan1Ubb>6Retjj35&U1e%BuWFkfk zpwdOF!bBn*f4Fc}5k6=EPp)CXkVpZ_24hCV6u^BbModucxOH6F0^5nwnr|01JX^+& zb`I9=m$#(H$nKCWX=~p65dllQ6nPCggR0Te^`>oJKaBukjqDL`L|pf32e+( zL2-pRT``5V6#dtIJW6&42sv4ItTd7-rf%tPY)u(`kNt$kybP_X$2A#wVc9LL_lbFFy5)5)uuX2*806AGv z!C-sv)k#<~q(-d;orF;%A__i>yCoRYf==pxnv?qrrOSCN&c`I>`BG{9&d&8HDIQ?u z-MV9?m4>1>%*I(ENf7z|p8e;I5g!5}L!gRLYOWM#d|N}>W}Wkm&p z?ZH8?OqVM zAB#>cL+|I3fBfKlFxnp+XeBKHj{vMXB2KTzH3bz*zYZ|r35g{kE8nMhPq zOj%0Gw|I)rSriwanzHm5Wmyz#zAd=z6Y?bgCPsG_McL*{J|x?>p?d!UvziXi4FpyI zQ(2R393OwnYQr!PMfd#*qrEN3p)X-O(3Y$kC^V&^J0VRHQR{)E*2&*ju@fhOthyZC zJ7?|;s-|1rX20NrvZh9~ASeSPtW;)QqmRWbe?VD?+(_P912x(~K-1=?5{?P$;munI zJ}L-V0-3;stI;M#$1o0J1Db~-IZP5))@vS9>DPbQaq-R9fJ;K|u!hJbj~wrTxf_Y- zfRtiHx*y;604XrEpuxm|X+iPH8^P;vwV_weoLa@5gR9n~IJ&n%jmEC)if+=l7a~jJX+sW_`FDmYk7$CbmEl)`Ls*=mAAtpLm$X21zG0YmP~Ptg1Uv#%Y>0tJ@fg!P$| zejr-`nUmTeJOU#SlRyv_lP)1D0SA*=AvythlawJQ1Xci3S(CmY76Dq5)ge9su9FlZ LBL-j}00000_r~GY delta 2099 zcmV-32+a4qM7>0?*arpOu}vJBlide4e_x1zDk05<5JG8^YM+o}4`9`|+J+?me#bT; zaL$QCtCgtsC4upGX6DTJW5$=x=ch1`Ry@u8IO@tw(PW87ZtVHdyer?`o;7S)%5o?2 zoFI;PSAOQ1{JHm6=P^!yWDCx7NmLMJU3rn`$&spN?t+I-ro;)4#F1H?hE6WNf2MOa zOH%H5Wl$KXx~7?G==hOb)o_&VQ%5|T`7S?=-DSw5ys9MSfs>2wXA3{cud!4fFd!IjMTc4#Om4`U~QGB`dy0TWFDq_#dZj4jO<1=R&sY3-O>|pfEzQ)8Vm_V&zK&6=R?8lqI=R#9r17JamMvt7Q_KuI(seN`Zc+NR zC@%V@t_vnF1(L|47`;`m*-=+T#cSdl47r5`cd+1ot3Va~*>p;$JEfyOe`vdXm|&s> zkZ%OSc9B~G(RYzM0s-T3Pav%gvXgF;PPd6F03DrR!T{O6OWeLo+`UWO*AY9Jw&+Y- zC?1UW1QRWo>ifv;`^X(eOpt7CB|F&~WVXi!s=(T9;sg`+F44zCdzU!H1OqC3ArNqe zoe>C_nsWjHDqIjqYZtjBe-JP=0|Ehab44IvY=;B_2KSmk!03)JqJz=BArLUSV*&xA z`;|bN^x0<8XPb#nS1@WP7%{-~^f97?Q9C6NFlt{g(gdS+h7l80bNO%CmgDSn^rEme^&l`qhH^uez9tcp|qM!%YH*Ti(HicO>8(9XJVYvcf0%e@ZJF4z70YfjT11-` z(Z&-E6gt6(0ZzX@Mv#aE0!73K5|Kclh<1&LL^u?&CVYUyk%)GUh(zm9L>r1o%nw+v zZ!ls3WNiO`L_9<)dx&^%?<|Q;LpRJfu6o+EiETX7K#&uR7+}@xV+5HrnQhB{!#}3c0+!`KFVM;f#lTw+1cMx;7$_mZAPWh`1V8@g z8;0_bY6M;o)>Sar9NeggN`_P~;XMTuH6o%cs3XOIe9+pH)|Aor*pC^$r*1GPB*Cm6CoEdd!Fij7?}IW=saHf5Z&7l3snEnJDV&6F6GY&@5=ls5pz{!D@)$fta|50N>V>g{rtIVy?E(o892Q# z_CA)KT1NiQCI9%r{a_w_aLrGDOZq3ND~a4+P?25PO7Vf}RicNMpk8wYRB zSa3mtPa;Tr%59CdAy|sB=UY(BXVIafxG_$#5KGs8V$1k9YXKL8JYWTZ2^JVW08={> z(FrN|h_pYh8vuf1=s6)o&m?)x1XT-1GBCj$^r%5u}vJB zlYJmt0h*K9AUpyj5R*R;7LzR@Dgg+SSRpzAdXtkOCL10O*^?y!002+|000pH00000 d00000004l2Ta(lwJ^` Date: Mon, 10 Feb 2025 19:30:28 +0000 Subject: [PATCH 025/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 8764 -> 8765 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8b972e433..642bec793 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-10 17:39:09,949 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-10 17:39:09,949 submission_checker1.py:3270 INFO] --- -[2025-02-10 17:39:09,949 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-10 19:30:23,733 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 +[2025-02-10 19:30:23,733 submission_checker1.py:3270 INFO] --- +[2025-02-10 19:30:23,733 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-10 17:39:09,949 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-10 19:30:23,733 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 -[2025-02-10 17:39:09,949 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-10 19:30:23,733 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-10 17:39:09,949 submission_checker1.py:3286 INFO] --- -[2025-02-10 17:39:09,949 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-10 17:39:09,949 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-10 17:39:09,949 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-10 17:39:09,950 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-10 17:39:09,950 submission_checker1.py:3307 INFO] --- -[2025-02-10 17:39:09,950 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-10 17:39:11,073 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-10 19:30:23,733 submission_checker1.py:3286 INFO] --- +[2025-02-10 19:30:23,733 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-10 19:30:23,733 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-10 19:30:23,733 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-10 19:30:23,733 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-10 19:30:23,733 submission_checker1.py:3307 INFO] --- +[2025-02-10 19:30:23,733 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-10 19:30:24,747 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7891 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index eddc11122..109144d20 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -652 +653 diff --git a/summary.csv b/summary.csv index d00abfe31..b1b4da751 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7891,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 20147ed40..4974cde1e 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":48.7891, + "Result":48.9664, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 16642d34c2d4a53f0396e7abe4043fc0b2785ba9..17ac7536eb7fa2cfa16615b0485185e1807b0885 100644 GIT binary patch delta 1197 zcmV;e1XBCFM7>0?*arpVC*E|Lv)u=_1Aj`(Fy7chTeOH4o@k)ZDMp&$^y^~;iAW$& zM2sL22?UC0RftH0LlG;&2RIywXjO7pB*K9UR~6xd7Ju;M z8Ws$R6rgM{W<*Q@+=pVs1l5jP$CWLxohYsOc2UE#W$b9@VC{Z+OL~m#4%w2n_Wd(j zruD{M2d2>i7Ue)M(8?*rz*`RlgB+w7C?UZh3kk*mKmHe6hVqbV1YQtURWR5d+^UF5 zhEy-%Jp~jsBBD&FBgKFvdA!s8qkjmwd=!^|%e9ff#%vW7SBTRUQ&>yUf8EEUWOsm& zlXb^RBbj3A*3RvneIe<4>?aK0Q*SaTB*C zT}e_k!%NAX?*EZi?yQozmb#{}-|+GnNgi@Zx$fA`V-BMZ@2;vlZ;S*yne#sP{Zw)y zF)C%~wzPbWX2RMJir9|nGVkl1cFv%)RSx+AAikG!!Qtp?>t51 zdtxUA1XhwtfO?`55=B(pEPLBp#6PkPY2F?uY13BfiQ{@^znxvVobOeW{DSk+=nCZ- zK?!JXgw)##eXJJg14?{gTCm1ws89zU&Fh1&{5nNr&x%k+kaxq_&3`EE(p2D3IY=>Fx&%EI}yaLOI9 zU|e0|^CBLuVLmM5i8m6&Qy29sZ-+(3J`hO^K*zX8<#YD@vL#xex*)|}N{TcmtNfA9 z2rX{E`jHp?C!jhE`8V+h7;|5Kg8C1$t{ut(1>`5*befZWAX@>Nli46V0wfTVKM)p^ zEg>oa2$NVLIstlf>vbIB>?~cPyzq|5dZ)H0000000000fPq_+)FD0ruagrZ LBL-m~00000n&u}} delta 1179 zcmV;M1Z4ZYM7%_>*aroc;Dq&=v)u=_1AmHXwHPg;MT=rg}sib%{4Sg&s}Vgh7r{eMI}Mk)u0_+afV ziS?$=bdxr*MVr{dGYtee#YhvZnthBQ6A1*Gh!JEWMhu|RMXSO@A{@AIRS`aD0e?@f zVZo3{0m=qrM#L1reJDmuQ0=&NT-gHKiPD;H7d1Rv#*TIl*6x?Lq{qnakS%Fz-#?>e zT5sHSU>Yr8Q4aJ1t(;N}y!AjZ$U%yM5)urukYEh(2GXJ8GVoagyDPYO$LP|m=+WgGgwH1K|&Ju_>qtV1Mkml8B$210wiQb z1p@}3B3z@AA<+oBOM{|DL=+a1U`!|^`-6lmmdOA3w#`uzJjhA5 zK8w307}J7I>VKM(`wOMZc`VMyB<1;1Y5mU5^(ZMGVCCJqW2KdbqBqROTPr2~B=m$4 z4;bsvO3WAxT8SBKCBYyoF@J-tBp766y~;|W0%T=H1%vIuS1V!3kZ8ozpp`IcL`0!0 zNvdXeDY?`AKhnybRWjF7*A(^}ULGUKLoO-T9ou=#VbtLp3hK@qBLPq5yw80EnlOVu=Ydw#>p!#8jM2M%Wsm|e))b(=7n=9{aLC0d%$ZG=K9{nW zn0ICV9E-Wivy~PjN_4JxuLTFF09-RgB>d(Lm{WbTEj#+qa>5{{pj`4$ln)Rsd63lWiOyf6HpaFc3xe{R*SK zEyS-&e5{CxNWG9Njx-?hLA?Tis^A;DfTJMzkO( z10$?dW?iF?#Vmh7S%}<7-dY1S+Cf0m=B5&k3G3m_TL(TW2w4J|z=W&OCPv3F4q^kE zhax#l5?9u19#iSpf7o&H&DVfSLhi7J$R&>)?}51+iRgfoVnn(h-}V40Ftnh-#DHl* z@yQ#(>u|N9SI(ST#hrtz)}uJOw?U1@uIq|!(pyu0ikI`(_kPt}nf<8*pr(?72@gD4 z-&FX#NQY~Ai1T#fl>+&+OZt_!!y-c;$Seh5L)@eCIeC6rCCpG$lJYhuWlqV0-m}Rg zqvUHp@}mC))Q2JeCLRGp?#oZm`~kDC9m)a)mf(c-nUj7XTLGDq+8{gvBM_575Ehdz tAu0g}lUX4;0eO>@AtnS?08?3$z9AL?T9efwJ^`+i6e1%AU>^Vg008<2Be?(o diff --git a/summary_results.json b/summary_results.json index 3eda07600..b3e9cf8f0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", "ColorKey": "availableMLCommons", - "Performance_Result": 48.7891, + "Performance_Result": 48.9664, "Performance_Units": "Tokens/s" }, { From 856318daee3558eb837118b0b88704c642d69f16 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 10 Feb 2025 21:48:17 +0000 Subject: [PATCH 026/719] Updated results summary [skip ci] --- README.md | 33 +++++++++++++++++---------------- dbversion | 2 +- summary.csv | 3 ++- summary.json | 34 ++++++++++++++++++++++++++++++++-- summary.xlsx | Bin 8765 -> 8861 bytes summary_results.json | 38 +++++++++++++++++++++++++++++++++++++- 6 files changed, 89 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 642bec793..23e644ddc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-10 19:30:23,733 submission_checker1.py:3263 INFO] Results=10, NoResults=0, Power Results=0 -[2025-02-10 19:30:23,733 submission_checker1.py:3270 INFO] --- -[2025-02-10 19:30:23,733 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-10 21:48:12,142 submission_checker1.py:3263 INFO] Results=11, NoResults=0, Power Results=0 +[2025-02-10 21:48:12,142 submission_checker1.py:3270 INFO] --- +[2025-02-10 21:48:12,143 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-10 19:30:23,733 submission_checker1.py:3276 INFO] Open Results=10, Open Power Results=0 +[2025-02-10 21:48:12,143 submission_checker1.py:3276 INFO] Open Results=11, Open Power Results=0 -[2025-02-10 19:30:23,733 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-10 21:48:12,143 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-10 19:30:23,733 submission_checker1.py:3286 INFO] --- -[2025-02-10 19:30:23,733 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-10 19:30:23,733 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-10 19:30:23,733 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-10 19:30:23,733 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-10 19:30:23,733 submission_checker1.py:3307 INFO] --- -[2025-02-10 19:30:23,733 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-10 19:30:24,747 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-10 21:48:12,143 submission_checker1.py:3286 INFO] --- +[2025-02-10 21:48:12,143 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 +[2025-02-10 21:48:12,143 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-10 21:48:12,143 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 +[2025-02-10 21:48:12,143 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-10 21:48:12,143 submission_checker1.py:3307 INFO] --- +[2025-02-10 21:48:12,143 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-10 21:48:13,121 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,9 +25,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1333 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 109144d20..6a1183329 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -653 +654 diff --git a/summary.csv b/summary.csv index b1b4da751..3bdf9cc70 100644 --- a/summary.csv +++ b/summary.csv @@ -3,9 +3,10 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1333,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2127,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4974cde1e..ba843a7be 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72821, + "Result":1.7438, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -240,6 +240,36 @@ "weight_data_types":"fp32" }, "8":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":23.0543, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 17ac7536eb7fa2cfa16615b0485185e1807b0885..05d4ab1a68538e8142cecf034881394fad7fc055 100644 GIT binary patch delta 3217 zcmV;C3~uwiM4d&j*ar^ivhTj|1^@u)C;$K$0FzH36_fi13V%uv$KTJVu&g|^Qn9_h z69H91nhPO>(j?V+LXKI$s&BPTF5mw6LI|FjIJ7zvRbLXk{?F{ptbg9M*PYK#fiFDp zB=w@OD>6wD1s=N5%nKJ?@%HAlWs5?ZIpNIlqmXySXP%0md;fGEqvS`rnj=o?8bU5!px zLk>Wo6Ra>mw(lx#-c{VbtGH`Y)H2oSOm$=r>OH{<9j5xeRlXTgB!6IN&Ik!W;hd1@b;$)G0Yft&Bw%bV z2??m}kdT1lUJ(*d-4T|kpt{$D1XOoSNI-SJ5)zHR+D!UtGx6mLs&`>OH@#` z6G8&2_618cP_|v8{dFHH<%?FHhgSZ3Bd%{%T&!AR zNV=w3_8ZVyG;|goad?l%5(9AA#}Wh`A%W0g34)H0fD81@YJiR~2Q$D$gZTiPn;^~0 zZF6vk91^{QyVsQki6q7cM0|}U7H~VZ_DJMnId$CPai*0&y;QDRi=4;!Ig1i^Ie&dn zB~wyb?ux1U4kYd)l^saDwYEsqZK-XVZ>*th8blipG`O~ovBUtZ_OS$kNJt=vSb{(# zBoIWq0Yt(ag4kd_z~%@*(=N>FUB!3bTNFtUX60yVt7xP)WfkeU_+*B?a%m;12$t7$U z5-C8~pv{OF0=N#v5(~sVZtoYiz;UCr|r2+8t2@PV9(m)6a4Wf|HnBeArw$V@=Qk}pD!X^iU&40-ail}8s zbYg)_u8`G;vVx0HN&{*=-U@#&K(6ld$G_E@NI+xu5R^}d(+4K8wxs?#k9)~(2OuZw zz$zk{Wa*my29NYB_9KR0sT&OPh&84HdBhsbBcVY&60i6XkAw!k58P--9tj7C#|8%j z1~(BdQOl638hlBEtVV}h|58|<$A^+#MHhW2MCnVVxlBy=Xy+&p94$~3C zn>L0DnZz2CK_;;VGf8L=lURe9Bs7RgwTVf>0b;Vj!C-Upl}Xq#Bs#GeWD;792rIZ1 zcTH$)1v07rX-wAV$r@RANN~|$eNF~-_DhUmu5^FG(ga%QmHBm`8KvXt37;H|yQVCmzL?@OCsf1P|$_mYt zga*x&+MiA3)+||IsVWlt4KDYQ5N96tzpF~&%$-kGJ{Pj*n0H0~6z5}AWPd9Pv!{9WDVz`|UX*y* zbJctCE#As^dO|sW0|XQR0JEVEkpT+s%jY7z0{{RH4U;?&8h^D{+iv1W5PjdTXn9{|9()Dg5MhLX zL}&;>kgPIKR$~|7UAx=sZZpZ>Pjky;Ty`e2qm_^%PIcGmsybCg`k$WijO?KlDQ8!j zZCDxs7V{)!+biwi{#p+-A|z!A%{YTA?HPpjY4As12uajcCa$!dlts@pMZANY3WFEG zlz(Nz%bZH}wcMJbD1jzo2O#szbS%p=bDA=Z#Jpls(fW=?Dwh6R!RUkpwl?UCbkLWB zTb@AHH|3yj9)J_LihwhOlJmckOct#;s>pYbb%DQ2=r)fslX_iWS z=~+m~>*!pi5WRU(@)(5R$ba!CVvJ+RprjHXrs(jBZ{$~6qTJ27#E*CokTe!pQC205 zQGjERDthE+q1IpUAxpz7(c((<2TW}wk2Q(*wqb{cgSigvGr37DYYF@H_R z(;>NmYo%@=%lk*twL+APS5ehX7rBzNTyapBijz~t-|$V&8Ef7kxJLPK(Ief^a6>f1 zFRQNO7>*mfT6Lb`d7l4$(>ade29`(2&14?UCUb>&X!4I{(?zrz-9s(a`m;T?8iUYs z!oYS8A40CDK~fHH?s&7VF(1;S&2=PZ5$wfYQr!PgztR{qq{B3X-g2c18vEvfkIPSaud=f5w(7h z)H;6q%1+`W;8TyQnf-QlQMov%Ciw;DrO_24S&kCW+z6?675Z2&(>Y3fU|O)oXsA#J z9xdv-l3U7+gEwa^xFEqN5u`okwnF<5EXCOKJt*e0=+IHz8mCx@rQ2bD%lJ3j0WR|V z5i1Bxu)uH+OzlWSC#2vb(*C$+00@qu0R=jEOfrHm-T+Sd$%a9>aw-`*3zOEfIJysB zj>fiavvxLEV}63aRE& zv#%Y>0t)D|@4oN`008JHlY<~u0q~RBAR-*@%jY7z0{{RH4FCWX00000000000001h zfqD>=KoAy_E+HxbXp>nXIsw{~lp!Vr+F6NBlfEGq0lSmcAwB{R9+MR!A_l=900000 Dtr-IB delta 3130 zcmV-A48`-EMZH9@*ar^eC*E|L1^@t2B>(^z0F$pE6_fi13V%oxfZz95SXLieX|X*9 zY=eL*Auxj!1y~eb7uT8<4foBbKpxW zo}^wBc4ek$vcy9ta=mcgmGAG)YnCjfnH{>eABDUtzwlK4-21EZ6eT~>1?QP0DhSiA zyvVY+t*WWB;D3RgDpAZsaby-Hft`u3$y`n2gxhWr6!@yHX@(luUMQC}w39>Xh-Nd- z;b)Pv40xE8l_cD^GtvEY;l-)k>$sjcmggfSeAbmuSiAq7)n&EUDcXMU_*1&`S;|v+ zjG`a;mn*j`YdNYS_PprEIFVdFvzLB0iJmUGH(z98*ngUNh!go~@mC5wF_yY41@?1S zuFKMs=Vpt%Vrk*IF3$&AmYijpMZx#-5L;JJhB9Lv%QUu)ZYivx8G19Hj~Ab>W7M+M zqK9X8X7@Tt^du!k3u$cUGsD`tE{4U)OTXsDdEeA^!DLe)i9GVrTlMN4b(L4VCceRt zJ6LcJ3x6JV3RKabZKrg)Q#$H{w%>;dCK>?wMj&h-xg!vLAGs$GFdh#C(%2$<>DK9V z>!{t~6)p*+v5#C42$-4yfq=QW zCJ->TLjnPVdqW^#bVnG`!RX!+2pHWlfq>EdN+5OmY%}Py&A_KC7`0Q3G{N-rF`|P} zJ0lP3@o$ zXfP(+L%5eWo} zXjOP$Cj z6I-;2Ej-gekW-8_!K&HE2r`jCpnr)NK_+6v04iOyDoiB8feTj^;e!_Nr0 zFlIzd0o;dT#01rjTgR0xu$?Ha`F2smvt{gP=V0xAc}seX><-zIw)XuqTBh~JT?eMo z0v6>!FVM;<#lTw+1cMx;7$_mZAPWh`06+d0TZZzGY6M;oR#hekNfoqZwcd+a9+-&1cgC?vtOppclsLJ|xTlDNl@gd`Yve`d>&LJ}1qAuB2vF!&VV z8kG!*M$lav6g48Eu#f~}LVqFIA0%Y4ME=LOZH|)QK~AzAC-u7W_9m6lkC;vvernUy zp_7;~H0UH|u#*IXoWu-vl3995mE42+%3VF z7Iaeo)12I2C|%BDaXuy~&zDN;cXqBvN$~(H@75hFtuz$9VK&}cDSzoFp(l)Zz*vV? zV#Z+5O3Yv@2?kk-8EhrNAS>%tRuUB;D=R7(Y!AL#2}_1VBc=wegi#|R3SCK3HN#8E zo$mjUR_?5lxt6-7u;1|V7)c&-NxAOW&SMUv4)3n2J8z5xJel)8_x)6IA~7mu=(ehXZhxh(e8iOwY)NCHd*>y%3fmLmHBfl z<|@xtmfYuA`Ob|LCtj3z*-P1a{?gCVw|hb4ek?k*485OA{_%tJ!DxSQj8A_{`X{L? ziQG4+$gXH5|3G!C*NN>pzp<0K7p9WWXChHeF=Z(!-{L7gXChHtd}_+lW0Yl4u=%#& zw#$?Jn;6|$6lI$)`H*bihU)zbv+xd>0e>#wNZqmn004vx000yK004MwFLP*Na%E&w zbaH8KXLBxiZEU4i+iu%95Pk1g5ZagaK{wjE8IB9Xb^@c0W7x?eeG+JCY-?8{Rg!X( zzhC;6rlOQhTfjizGvpl3%$XUd|M{uN(H^+S7`-xF%P|npgrynXUKtPf*JfxSA%6)? zNzN!-8P6b$&w~$rAtX{)skkzBQkFg27Re3@A}m${RhA9o1(E72-`b+&KvJ;-kVS4| z$MNlgWYj1IDxAKDB&5a;}0~G zq7E)WRJp99c}@y~&A?fkQB*hHZGSd-MxhShvwwjKTVt4x8y=#fLDMn%2-em@DjjH8 zrJ4LxKRIGK<6q!e3$a=knUuimhT)!MIaz0UCiP3tLPAkT=PHHTn-`oVAOzF=mw#!D zNdh@=BK5;WJG^2W`JHercXKB7M?45f8w;!`D^3zEz%fV_z2;}6eAF?|a(_(6aoiRh z*YT7{qGY8qZXYyk-_Ay?D444Eb{CUGstz$ZY=El<0W&EpvwT(=Np|rbTfTCd!iFfJ z#R;REY-_(Mus6x7wQ7RO%!$gFjGsckowuv=@-5NCPEMN1;f1GuqSf$nwQhLHN`pa* zKhi{0{QF=7@1X?czP92AxXshPA6( zn-aIYd$^gnJ>TR6y!k_N*XxNOkawv_)gz-pYUYMN=b*?&}rv2Q-<L7kBNJs{OIwca*(_qaX9Z!eo2CkLX04?tyQP+vI+`o#dZn}sEoyDUEb;&0k?c;j0DXTfXlHKV9|<#+Da4K0-H>c|4oU z72<32b~KwV;??MGsg{oGtA!Ls4vs<>A3g|OPsbN4Rtqe|UJyn>zk5W%8i3JXDqlN!6y-|ASyAUkJ*z+AI z=CkO~QQR1(Scs+DVt>o{H`@X(2)V}!0uwAS+yhfP5zzrD_=L1St{DJ=V`xBu4j$)O zj!)hIPWh`1gL39nGISQMT94x7-g`M2+qTWx*Q6-b(AXWjIlR6OV0DC^megFUf diff --git a/summary_results.json b/summary_results.json index b3e9cf8f0..9ccc6b924 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72821, + "Performance_Result": 1.7438, "Performance_Units": "Samples/s" }, { @@ -331,6 +331,42 @@ "Suite": "edge", "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline", + "Software": "onnxruntime v1.20.1", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 23.0543, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0011", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "Offline", From db4d19d720c31cceb803e4f1ab8965f0cfdab404 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 10 Feb 2025 21:49:10 +0000 Subject: [PATCH 027/719] Updated results summary [skip ci] --- README.md | 45 +++++++++++++++++++++--------------------- dbversion | 2 +- summary.csv | 1 + summary.json | 46 +++++++++++++++++++++++++++++++++++-------- summary.xlsx | Bin 8861 -> 8950 bytes summary_results.json | 46 ++++++++++++++++++++++++++++++++++++++----- 6 files changed, 104 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 23e644ddc..c34914373 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-10 21:48:12,142 submission_checker1.py:3263 INFO] Results=11, NoResults=0, Power Results=0 -[2025-02-10 21:48:12,142 submission_checker1.py:3270 INFO] --- -[2025-02-10 21:48:12,143 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-10 21:49:05,542 submission_checker1.py:3263 INFO] Results=12, NoResults=0, Power Results=0 +[2025-02-10 21:49:05,543 submission_checker1.py:3270 INFO] --- +[2025-02-10 21:49:05,543 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-10 21:48:12,143 submission_checker1.py:3276 INFO] Open Results=11, Open Power Results=0 +[2025-02-10 21:49:05,543 submission_checker1.py:3276 INFO] Open Results=12, Open Power Results=0 -[2025-02-10 21:48:12,143 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-10 21:49:05,543 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-10 21:48:12,143 submission_checker1.py:3286 INFO] --- -[2025-02-10 21:48:12,143 submission_checker1.py:3288 INFO] Systems=9, Power Systems=0 -[2025-02-10 21:48:12,143 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-10 21:48:12,143 submission_checker1.py:3297 INFO] Open Systems=9, Open Power Systems=0 -[2025-02-10 21:48:12,143 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-10 21:48:12,143 submission_checker1.py:3307 INFO] --- -[2025-02-10 21:48:12,143 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-10 21:48:13,121 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-10 21:49:05,543 submission_checker1.py:3286 INFO] --- +[2025-02-10 21:49:05,543 submission_checker1.py:3288 INFO] Systems=10, Power Systems=0 +[2025-02-10 21:49:05,543 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-10 21:49:05,543 submission_checker1.py:3297 INFO] Open Systems=10, Open Power Systems=0 +[2025-02-10 21:49:05,543 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-10 21:49:05,543 submission_checker1.py:3307 INFO] --- +[2025-02-10 21:49:05,543 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-10 21:49:06,516 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,12 +23,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1333 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.39588 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1333 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6a1183329..6412531f4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -654 +655 diff --git a/summary.csv b/summary.csv index 3bdf9cc70..c1e6c47ce 100644 --- a/summary.csv +++ b/summary.csv @@ -1,6 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.39588,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ba843a7be..b6944f8b9 100644 --- a/summary.json +++ b/summary.json @@ -60,6 +60,36 @@ "weight_data_types":"fp32" }, "2":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.39588, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,7 +119,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -119,7 +149,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,7 +179,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -179,7 +209,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -209,7 +239,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -239,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -299,7 +329,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 05d4ab1a68538e8142cecf034881394fad7fc055..980e41b5454e472481bc584d642f8dfa07082be1 100644 GIT binary patch delta 3280 zcmV;>3@`JYMfOFo*ar?Uz*c!d2LJ$nEC2u)0F$pE6@M}=cx`Ounp3LR0(O55JFO#q&iQ?F$-Art+vVK+n=>f2%Z@nT2-Pt=OwY%|CycH^^aZe z?%~I$FpwU2n)z|mm6_6(B_4V4)Q{#}`S#{>%aNrlccZBr#1Zex&peYq_WnA2jMMMg zg7aJw4u7JoD=+dqIZ)NiTkz1$lsMs$cruIA(9OkdI#;tK-B4yV3&R^%h4eAbnZ*g^jrGi9}RSk`^# z^T%xCS}IbxkK^yf&H1z|w+pF?w4ci`u2X5sXMgT8$j9;H8TaRlTr^wh-HU^){40gN zXiH6&LiedF>$3FdPxD3LSX%hgDK8pYmb_(_$Kki?5nDN^OqsQ^Wo@>$?kLP?Tbfz) z$8*3}HfmLC8R3bWyS>9Sew5O(hAeT5o?!=?CYr@7O1~7vMQrM-VEj@bi9Cwdd+6zh z>VHGwc%}FXO>SVpEiAa(C{RT_>!36`C=JEXuE#LO3Ij;K5)xLI+z=A2F1aNnpgryg ziLsW{($(p7b>sj9I>rhUWc#M#=1s-zn~J+8MJ-c<&eTBmpuNXfVZc=1mfXB8xy2F- zB-`kumaR!%jt`R_(eP;YJhKAgZbd)*~0)HDg zxr8<&q6>h{72y`7ef=8utatX$^{4Uhe01@?oD5}RHZ=G1#qwV9;!q6(@i!DR9dVWL zbT;v$8BfK0jZbo(<&$iAANpA_^~oj=rqWjPl=E_ur51B3^>Wx>*{3`cQE~8vbrdli2#D9yUnLn5RmZv;-{UDP+9I7*7xc;(_l*FW!i_pq{ zTjKg!#l@;AhGOWt?QDV0rlGTOkHg=1EHQzkk0l5?LIR<~5(FI~0WavO-2fe74n~0U z2J-<#3iQvly+iYRv_Y z$#Wj3-eU5gDVEZ1c}u1lJCL}KRCXZo*4`k|aFmW^ZJ9$mG>8uFXn@c$mY8t1g(V0? zLIOd=5(FY4fgm~!AQI*f#0K*LHb)>j4ImP=Ll7MZBGEr!yuKo$0*!y|{C7ayM=Cpj zc<0oHMBUVwX452gXc9ZPr+)z;$5>(lNgqoPiG&1_h$VoN!hU{U zx{nlhnU{9z5Oz$*p&4Xt8p5(1AO9xYkfCQVt-D zO%4X@hie>B$&ktfj(*5$L|7pt2@PU$Bm6%RN-!4wG>>~pZxJFA%wp>~yb7!$k}0;K zZyO@%O{OD;_iaoK@`yE-26@C9%p;*eJYo&zkRuAu2}x}ONljO_e}6hMdMEUV5gUv;WD;vE z9WseEm`Or|n8X^)B%wh}>P<`%4iJ+K4hHLouS~*{AyJ8KKqjHph_He?Mm!_`!NO1=$Z&ra-R9cE=*c)FSbt>sy`y+;T!c7BGN$jT^ zkV<0vA5lqY5Py|egQ+Amh)Sc0O2Pr6vcbV%{qU7aSTZCkL908+YD8I~krHe0NNN1p zRQ{GFp1mZw(`aebN6WTpY}tEcTh5m-d}hEjA(vQVnUG83>;dAE&>$|c26IVh5SL~X zmxKevWrKsk`r#{=uw+P7g1LlNBf<(V@(`ZUB9Hlh;D2%-N$#}NW7d{>3|+T$xEHBz zoUaUdI_G^J1exT;qE#x>b!m0zbj%L!s`Htza#3L%P5oj^$qlO0q`4o>rR+y_PSrZ- zf79B9GjBFt23*RY6W*2iQzH7R$X1r7PqXTTs+6RDocj55ReNznb{V+6FrL0IgIY%Z zk0pQq&QyD69lZ0b4}VMA2dOKG++U<3yRw$zf(om5n7DI(>85i(%A|nLM51lQmZh{h z^IBf#aZ+B1CcKaHJPu#(7Tlfkw73(qky#w)FE>S#tj`Yj{spt}4wwN6jeJ8b!~*~T zn3ItZ8v(78tq>c3vQJiH7sR`Ex7XcflE2^PmdUtmCbL-yDdJRjpRVdtbx8lqbDohM zlp^KqO0x}1Bfw&wq-=YoJ=|aGfkuR+ETI`^aHYL~(7p_Q>kA=?+RDV0_9SJ|Gffdc zK~9Ci3t%d;;bl&x`dV&HQItRv@dP0A%ycZvGjp0Ujl{fvVp7otjz%h${!_u|hy)#N z&=={TF9)|gfvj)JLEqd1M{pGbqh-qL_ybF&sDn!oRVM3bmeHI#x^J!ZFsujfHk&ME zP>1jN-@t^47^dTfi>PeKbWA>hu{DS)2N_mLDnHj(j(En)ukeCWtk!udC6x8Ra7WXO zuG1`)c+-o2kdW8Wxhf&L^P=Q22*J_$#qWqQjv<4RO1zk&#Vfv%-)M;zd;?eu{RE;i*Ft*ia?3DCTT`lWxt60=r07tyL3LdPY^nWb_<( z?Yv#gTW%>5J3VS9gIAvViq^pE)_UNzDh&aY0qPi)diH7gnLGk#`@i3O-RXwzBND>x zm`oObj)?Dj9pZKM5ifzXi}zR>a(t2t(JqcQ6>fR=a5J%c#Ip_Ga%@5lfYT!#-v~WF zJOML*1%yV&vwcGD@4imvt7tJVAqrZG!@C8zk}Y%SKhnKRC;w}LA5rFMU4s|a#rZRT zyR646rp{JBMical>W}GoIwUu6t<()Ap8wr*9mjA3%Om7wGLL4Hxk5bTL8IAp z5v@jdOJ&5ir%XBsEhh|YXa6DOdOAKYM;&GAf#do?=!c$bbvo#+4wqJ*9lEGc2Rjdx z&tp43JAFgw+CFajj+^RVtHHmsXcS=x1W|NPM3c=NAb(E7Fc5_AJVoStVkd{XNeg7ebo3gsC=321JF)Y}TZ zub1gPN_=2iu*PVp&;dMJ)Hfx!lp6=H&RB3kf=?nyd&+Hvb|F}bvFAHb%xBS|qqs3n zu@FnQ#ebIZZ?*+o5HiOK0uwAS+yPTN644PU_=vPWuIT}SV`xBu4j$)Oj?dlzPWj1( zUb%2884eaEttWAG@4Xz2ZQEw;thdJeIsRO|y!ET5Qg^4~fa+3k${ny^TwUVJA|9?` zKCI%2Hxk6-A?g!v$3=!d5J?O`hqx!@WA^mCUL>NrAjMruiZmzdBB!?xl+3@nkyrg^ zpgIouH}M1*a$kRf`VX@Y9?Aj=Fu+!MK?eW;e=L*ZAXWlF50g6~AsdZ+LoLJu005W` z000yK0000000000004l2%aeB@K>@*&xgk0NGZ&NfAteM+bWcQ+9wHV36djW`B0T|U OlXM~@1{xp$0000Gofky_ delta 3229 zcmV;O3}W;4MV&>k*ar^ivhTj|1^@u)C;$K$0FzE38h_oITW{J*5Xaxor?9L%v{JFX zz7qjeLYfO9gwiC{c|wj^z^ZSxO)lU5_(BMtnK-mM5>;Oky#CMZ%&dRjwbz}`Pk}Ey z@Fex3uq!f25d|K)(aZ}MUGetjv}KDznmOUj@uQG;#b=(1pL_pw9;4((y5u|)atC4B z6_;5SAAibn>MnWUq*4^~Fh4VolEBIG!(<_+al)NhX%zUfswk!$I9@1L9uAXTd_?oP z=kk-tT?IVMDkll|oh%POU3zgU_Bt~!KUd@A=YQT}ndN0O3KKW-)AFwnczG>VQ3#x; zuBeH^qc_Wzg=1mq&1SqPv?#c%G>d}o)hV`iP?<7wZOat4v2IIDS4`C?#^c%NYa6-h zwT$q@$(&v%i5`Wd>>-VvVr1B%s^(>Ji`K72a}k@oZkW6@2zeew^&Wa!M}8kmwspEnSUH zS3?d!pcAYxK(_BHZr)YgzN@%vQq(fl=}dKG59&R^3LU2UzU1b8$t{+cAlXJIwQLPC z+am)xuOsYBTZW3aWODB?hQ{A4^nF zwG%=Ds`dp-G*GouEHS~Qa`qx2?uh3^5AaHBFdw{}dzqmwVZ)G!4`g`-ZAQcpfPap! zvBUy#k6+`S_s;(vf0+y~MwcJT#Za2Bh8AA7T-{4<6v%;}{|)k$PW~(A$$aXCbDreu zH9pOFnoZNyec+|V(kGoh7?L7q3FqY^OD@(@^3|}vwoiDPFJV%7y_RWhJWt&?Zn^QQ z6@}qbvI?_&G1XeDrfw9@y@mKmoPY7m@%&W$*pcVNbp3T7Ddme+o`+Wcdn2xIRa~rE zVo18CS@s*ySu}JO9&vb&#}Wf@*~bzD9U+0xVF`kckbn#H%xZv+Fb6ZhMT7YOo0}lb z%WZRTha3{UgS*$21&Jia2Sj|0B^Gcyw)RNmVmWo(;&G;xKfP40T8o^=_Y*8j+jKEUP(M7se*qIU?Q4M8Nv2h4ldSYm-2<=FmvK-@7cB(`Z1+jyh_Ajeo@z<=i+EI}j^5=bJJAQG{}1Q+vJyMaW)9Nbhc8q5c6z{w?S z7!oN!*r3gb7y`Hs#S#m|J#Oz8w!m?twBOsCj`FF>yD^T)r{nn*xn_7Id$h|>oqv9_fCI*)tFZU-PI z>%b}^nPlmj{RWTpEA}IXU#S}m@`yF20(rz5%p;*eJQAX*nsv3MrgMX|>gcatI(6k|s><{9xoFV_`w>Eo8a3>_$7Luway}d?d^bXSz z!<#mS3Yo+jlR+l21~W-$5R+JgnItrbNwtYd!U1Bk!NFj2@|8*0G9)^&7-SM!jR-5a z6?aW&Yy~o@{%K6s=gAr8aeh4}DKBP9`*#+udr5HzDsR?-Rew~PlB!!5b4M zM(i(YkV>pERY)b)U@8d>q7rK`m4pUSsWnkaI6zc3I2deBzETNWhD0Zp3aNxvBgzWR zl!OM&l-i$7<<=}&V5uq+`wcGlk>pOZrB3nBekeGq3J_P?u2;mn;+Rz4T9=a_dz{(ltbV^w4;3bUtq^(mYXCtj3z z*>lx<@h#rUcX~lI`%nh83ca5z{^7tqFb@x0^W!H${U~&WJoguAo?Y2X@q-Gh*NL44 zzjBg=7p8*G=Xs)vWQjsjeOy=m&Z4;dWr)Iklx0!yaoWe%G5vQTB3Blw|Xf zV(&jtOS3}{mH`g#%jY7z0{{RH4FCWX0F%)n76G-B{tz2~GEY`x7vNpH+v{#K$=^?N z%Vb=3CbOfJkRncX*XgP{RYm%rp7MTeqWeLqV zgDdSBg!XChM_&j@)Kw;~w4Ibi&oo86gPaP37r>Nd!^@mX^|joZq9}nTVh14e%ycZv zGjp0Ujl{fvVp7rijz%h${#(K5gao!W=!-R_i>K63V(^xTk4G*J+kYeCb(#NXYBxT%{1bc~SBhgy6`3@h4)8 zW5}SS5+A1M@QQEbS6ZUn&AG&nco2{@7FbbMC5=&lV~{F(>{)ERX&5H~g z2Hm-LIxTp&1$SF;B#Sfdx8Okwj$%CFHcIRnM>7GEEz}$F+*W#?#U~_+i{jXSoUx~W zQVBfG0X1J+jhYym=E@e&mAJBtM){%WFrm?$+SxzCR^ ziT1W(hlYc>4(;S3HN1!F7}zgL;5XTR3@!mEL_tddhwuWN$(HW_k94mBJMcM{VgJto zzKgL%Ea!6#8>lYMpDn4ny<*C@`U#kSpx-fnO~=zAxq)k?ZXnD1N7A)Il#EwV)lC<< zlCxZKP?w66Q^nu#P0kr>-XOR}`Eb!A-OzADG{Y~euHzVv8@yU|p5b|(|9#Urj^PHD zN65`&9?d3mg?MQ4k7mM$l)%vqNwHkxaa>BrN4j)3Ur{jy&QL8j?Tt5gW{m^r* zP6yTM_-M5$Ll?z)T%v*6u6T)F_G^Rb+CE+Sj+^Qqn!z`-!4zQ#1ln1NO_L@aAb)DZ zFc5_AeF~$yEz4<35Viwt$*F-tQ(AHp(k2nLevs5Ue*4N!;w0cxkE@yec6L#@IH)H1 z1?Q#F6(U)V642ZTsdp9nSTEB#N_=2iu*PVpPzN3@>bsI#%8i3JXDqlN!6y-|AS z`w%R}*z-Lo=CkO~QQR7*Scs+DVSmf`H`@U&^867i2u!fRa1TuFNJJ;3;3LxhxMlzd zj-desI(SSnf-l|xPWj1(LAi1&89ED-*0VUe4_=POwr#U^HdteRg1=U;@58F8)cvVA zpt=;CatACJSC{y@h=*&K536|MtpxGZMSbG!w8*g!L=pqgW8AayIeUItFCf7ZsDGArKteS&2<00RR9{0ssIJ00000000000001hfx8@&k0Lz* P50lLzBL=}A00000`WFD= diff --git a/summary_results.json b/summary_results.json index 9ccc6b924..d80146003 100644 --- a/summary_results.json +++ b/summary_results.json @@ -186,6 +186,42 @@ "Category": "open", "Suite": "edge", "System": "gh_macos-latest_x86", + "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 76.951", + "Nodes": 1, + "Processor": "undefined", + "host_processors_per_node": 1, + "host_processor_core_count": 3, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline", + "Software": "onnxruntime v1.20.1", + "operating_system": " (darwin-23.6.0)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.39588, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0007", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -216,7 +252,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0007", + "ID": "5.0-0008", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -252,7 +288,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0008", + "ID": "5.0-0009", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -288,7 +324,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0009", + "ID": "5.0-0010", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -324,7 +360,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0010", + "ID": "5.0-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -360,7 +396,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0011", + "ID": "5.0-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From fc5827d2e169bb68bb05bb626c65c50d2c6157fd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 17:47:52 +0000 Subject: [PATCH 028/719] Updated results summary [skip ci] --- README.md | 33 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 3 ++- summary.json | 36 ++++++++++++++++++++++++++++++++--- summary.xlsx | Bin 8950 -> 9038 bytes summary_results.json | 44 +++++++++++++++++++++++++++++++++++++++---- 6 files changed, 93 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index c34914373..ba633861c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-10 21:49:05,542 submission_checker1.py:3263 INFO] Results=12, NoResults=0, Power Results=0 -[2025-02-10 21:49:05,543 submission_checker1.py:3270 INFO] --- -[2025-02-10 21:49:05,543 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 17:47:47,271 submission_checker1.py:3263 INFO] Results=13, NoResults=0, Power Results=0 +[2025-02-11 17:47:47,271 submission_checker1.py:3270 INFO] --- +[2025-02-11 17:47:47,271 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-10 21:49:05,543 submission_checker1.py:3276 INFO] Open Results=12, Open Power Results=0 +[2025-02-11 17:47:47,271 submission_checker1.py:3276 INFO] Open Results=13, Open Power Results=0 -[2025-02-10 21:49:05,543 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 17:47:47,271 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-10 21:49:05,543 submission_checker1.py:3286 INFO] --- -[2025-02-10 21:49:05,543 submission_checker1.py:3288 INFO] Systems=10, Power Systems=0 -[2025-02-10 21:49:05,543 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-10 21:49:05,543 submission_checker1.py:3297 INFO] Open Systems=10, Open Power Systems=0 -[2025-02-10 21:49:05,543 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-10 21:49:05,543 submission_checker1.py:3307 INFO] --- -[2025-02-10 21:49:05,543 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-10 21:49:06,516 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 17:47:47,271 submission_checker1.py:3286 INFO] --- +[2025-02-11 17:47:47,271 submission_checker1.py:3288 INFO] Systems=10, Power Systems=0 +[2025-02-11 17:47:47,272 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 17:47:47,272 submission_checker1.py:3297 INFO] Open Systems=10, Open Power Systems=0 +[2025-02-11 17:47:47,272 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 17:47:47,272 submission_checker1.py:3307 INFO] --- +[2025-02-11 17:47:47,272 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 17:47:48,332 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,8 +28,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1333 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2153 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6412531f4..39adda3c6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -655 +656 diff --git a/summary.csv b/summary.csv index c1e6c47ce..4a659dd7f 100644 --- a/summary.csv +++ b/summary.csv @@ -6,8 +6,9 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1333,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2153,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2127,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b6944f8b9..fff2cfa9c 100644 --- a/summary.json +++ b/summary.json @@ -219,8 +219,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1333, - "Accuracy":"acc: 76.200", + "Result":10.2153, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -300,6 +300,36 @@ "weight_data_types":"fp32" }, "10":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.426752, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 980e41b5454e472481bc584d642f8dfa07082be1..4b7984875c350a75811420506e84f66760ad029f 100644 GIT binary patch delta 3429 zcmV-r4Vv=yMb1XB*ar^xs~okE2LJ#RFaQ7;0FzE38h_oITW{J*5Xaxor?9L(v{JEI z--&=KA#D;uNJ^7b=LtDx0ju~@+vM`?&)Oyg&kPQ&szcP51lIqVomuUqhksM<&*qs}HZwQzqD5N#l>%R^r6x;( z``D3nS$gm%**teF&Hc%Q=L;=M-ZIUi;Cp$9tsIo5BHGfHHQCy_r7)vuX=Z*to)&y% zqn5K45gxml+ifS&gOn6Aq_LZy8Frv)Vp+U=^lLtx$EL0ZM$ZG1$Rl69`>x(r?{mj1 z#eX+wat#M=;K1$1fGXly2c^+LX()zfHHHyZ7(nukkg%%cnviH!$qgX^>v2m+jJ2ea zu1=?`BL^VR5muNW+gBCWuPSa{RovDoDw!H|rUtSH>pj8>1E%`AtTroR_&OOfK~g9B|2EOFIZxMiB6s+#2xXJm;qjCHRk>2W6wL(IUE?^32bn+Uw=TG zA#n=8#FvCykoMI}+|%yqzgM3}gR|lJ`=S_1MKv_@v-$E)@uENt7UFLpDjo4D=E-#I zhf|)2dX0}Wo@V28c^CL;UizftdsAtuS;BcyWT|;ArCtnrEBl0}qJ&A+RV~w4dmejn z-0L7KLosyScD6ug)6m)YjKh08mYDD+k0l5?LIR<~5(FI~0T<|rT>~9q4wyKrG4Eq@ z8{7rYH_gGZ0Wmw+IWBBSByoN~#Fs==U}dlDJrX&ak6kbSInxloUP@PunSbChe#)Z6 zn~(1`#ZsCLZ^<-c2NL&@$_^yn*c&7oj?%KMEj6@5gXrKB4G=oQ5)+*09+n^w2?+!d zOAv^J1cK<)fJm4_5Npi)*c^fA)PP9L4ncGvh{X8;`}HN3IAGbXoWBReeWbDjh__B< zOVmw`X*Nw_izcy!&olt!2!Bf~5JnG65Q&5Yl87aUL?Ra8<)l?ZA~v^y#Vas$JnnBj2AuPthW(OuYA~Y7b26}`BaY$((h=c}_NW}wDTz_jQ76}Ik#ySW6 z^}#icXkm6Qqz^~KOGtUCV%vh5ib~Z$RyTS24oUz zFq4D^F^M&pNkW5|)a#fe93UoZ9Q4-*Uzmg=17Z@)B(xe5R=8}%8eF#Oe>;==NN}gZ zRj(AThGy9tyId7Y`lbCL!!N>3142o>PB$Tx#M^%aC80r3Vhx6p&>$#{Iw%PT2+A4< z{q?~YDB;L}n195!Ae7K*NLazPl}ke77!XS1twC8WUMHN#;(AQd;sKBITfZmwk>U~aEn%Ccv85q^ZE3?{ zM3W8d!yz;lY{Vfnn9ChnaR?3OlF%S7>$-6$2Z+nMejJ7xa?r?tm;`f)dU6OWT*6`v zE@AiCmh-=H`Bocs80yrqbl4!EZZtgyJelzxUo29|i^Qswrt8tN6MV!DZp)_W7rB@) z3MYR4o`2U}lr7$wAI_xoXW2Y%9rV6y&D@zc9W57J%AR80k@;gR&Q+eREKMG#W!tzE zCw`Rp*;6@t-nYJ7xZNO{yf1=UhW^hbfB(*VXC1urtPdY0?Ss^jMDEX0kzFxM{(%as z+m78Czi^Y8AEwfRPer0l#g?U{Y(y_UXHi^y5+Ap=yC}<|;Q8B}yAz(|-^7oEX%uD8 zU-BhcH}rS^1G7O7m;nwPkBUgn0{{RY4gdfY0F%)n7Jt20ZExZ@5dMC@!s5Q{e#k52 z4N+DIXoZ$i1nsH!DKg0bxl8O^?6B>>KSOA@mr2;}^>h*v;u+hIXXcslF!=J6rQ`sG zNI1LdSi0FEz#<+eYFn5LY;Rp%K*=gaVyfe@0YtxQ~X_EP43!w}IP zGAeYQ1AkMNEiW=E)z4yQh`azAi#>qMQo}Y)*T`tXIwayHlZrO5JEUaEUnPty#CJNw zfk=h}IlSdDqys|^2gVVoz;y(S774H84=jdlu*2;L zbou+3c%IuOZciKY0!X`ftI&`tB^RtcY=3PE-0JS(W@`0`Yw4b8TZEhdyHC2F9=Kj` z2IdL~^sZ}pgxueKoi5hla=HrFqub?tdfaVdIvPcN;yHTH#5V!K0NteknK0U5*gzp0 zG`7p-9}Tc-=PH|8oNTw{vF&S5xV_RWPobgFoolDtg7;c*rv*o{IOARm?ziB`$A1d9 zP+-q^OHZ&+PdMYqUx;^5FWhq(YFQdpNEqe$sa`o_PlYOoBm-2eZ8>dXXzD{d+=rsl zvU^5L_CI{?JLJ>Ck+Xv#cHlb^l0uW`Kh6#;Xe^a(1}BU>)@{ds^Y**C>$=|er{A`9$2VOa6Vlq z#6?fxcs^T(>+#)6%>$OJW(7Ym?ZCI}2iXTzrJXM@FT0x0AaA;ZG6wKsITX?hDfb{}g6hO-5%#>*Fy~#0l zY!8yUBsf$Mx7teT!_|N8u%Yam?f_?LdXFV|%BiQg2gYV1q7lj2gfu_4833GNXh4GU z4wHo7v)6-_Znj`hE}T;M&cLkoBu?&w6O*xR+oW9#){vjz&HD9y7&R8EUljvX6`bYF z0`=Nf1-`t5#WmFVby)FM0DtO&KJzvXa;yU$g#h_6?n(JvJijE=D^z7^b{D5vOh`>0 z^4tD=^)0XJ&pbPyzq|5dZ)H0000000000fPrn3jv_t*z?01)BL;FH H00000XBJF} delta 3342 zcmV+p4e|2MM)pOp*ar?Uz*c!d2LJ$nEC2u)0Fz)L6@M}=cx`Ounp3LR0(O55JFO#q&iQ?F$-Art+vVK+n=>f2%Z@nT2-Pt=OwY%|CycH^^aZe z?%~I$FpwU2n)z|mm6_6(B_4V4)Q{#}`S#{>%aNrlccZBr#1Zex&peYq_WnA2jMMMg zg7aJw4u7JoD=+dqIZ)NiTkz1$lsMs$cruIA(9OkdI#;tK-B4yV3&R^%h4eAbnZ*g^jrGi9}RSk`^# z^T%xCS}IbxkK^yf&H1z|w+pF?w4ci`u2X5sXMgT8$j9;H8TaRlTr^wh-HU^){40gN zXiH6&LiedF>$3FdPxD3LSX%hgDK8pYmb_(_$Kki?5nDN^OqsQ^Wo@>$?kLP?Tbfz) z$8*3}HfmLC8R3bWyS>9Sew5O(hAeT5o?!=?CYr@7O1~7vMQrM-VEj@bi9Cwdd+6zh z>VHGwc%}FXO>SVpEiAa(C{RT_>!36`C=JEXuE#LO3Ij;K5)xLI+z=A2F1aNnpgryg ziLsW{($(p7b>sj9I>rhUWc#M#=1s-zn~J+8MJ-c<&eTBmpuNXfVZc=1mfXB8xy2F- zB-`kumaR!%jt`R_(eP;YJhKAgZbd)*~0)HDg zxr8<&q6>h{72y`7ef=8utatX$^{4Uhe01@?oD5}RHZ=G1#qwV9;!q6(@i!DR9dVWL zbT;v$8BfK0jZbo(<&$iAANpA_^~oj=rqWjPl=E_ur51B3^>Wx>*{3`cQE~8vbrdli2#D9yUnLn5RmZv;-{UDP+9I7*7xc;(_l*FW!i_pq{ zTjKg!#l@;AhGOWt?QDV0rlGTOkHg=1EHQzkk0l5?LIR<~5(FI~0WavO-2fe74n~0U z2J-<#3iQvly+iYRv_Y z$#Wj3-eU5gDVEZ1c}u1lJCL}KRCXZo*4`k|aFmW^ZJ9$mG>8uFXn@c$mY8t1g(V0? zLIOd=5(FY4fgm~!AQI*f#0K*LHb)>j4ImP=Ll7MZBGEr!yuKo$0*!y|{C7ayM=Cpj zc<0oHMBUVwX452gXc9ZPr+)z;$5>(lNgqoPiG&1_h$VoN!hU{U zx{nlhnU{9z5Oz$*p&4Xt8p5(1AO9xYkfCQVt-D zO%4X@hie>B$&ktfj(*5$L|7pt2@PU$Bm6%RN-!4wG>>~pZxJFA%wp>~yb7!$k}0;K zZyO@%O{OD;_iaoK@`yE-26@C9%p;*eJYo&zkRuAu2}x}ONljO_e}6hMdMEUV5gUv;WD;vE z9WseEm`Or|n8X^)B%wh}>P<`%4iJ+K4hHLouS~*{AyJ8KKqjHph_He?Mm!_`!NO1=$Z&ra-R9cE=*c)FSbt>sy`y+;T!c7BGN$jT^ zkV<0vA5lqY5Py|egQ+Amh)Sc0O2Pr6vcbV%{qU7aSTZCkL908+YD8I~krHe0NNN1p zRQ{GFp1mZw(`aebN6WTpY}tEcTh5m-d}hEjA(vQVnUG83>;dAE&>$|c26IVh5SL~X zmxKevWrKsk`r#{=uw+P7g1LlNBf<(V@(`ZUB9Hlh;D2%-N$#}NW7d{>3|+T$xEHBz zoUaUdI_G^J1exT;qE#x>b!m0zbj%L!s`Htza#3L%P5oj^$qlO0q`4o>rR+y_PSrZ- zf79B9GjBFt23*RY6W*2iQzH7R$X1r7PqXTTs+6RDocj55ReNznb{V+6FrL0IgIY%Z zk0pQq&QyD69lZ0b4}VMA2dOKG++U<3yRw$zf(om5n7DI(>85i(%A|nLM51lQmZh{h z^IBf#aZ+B1CcKaHJPu#(7Tlfkw73(qky#w)FE>S#tj`Yj{sps;510XejeJ8b!~*~T zm<<2`6aWAKcx*3oXkl_?WK(oMn8E_iKhtykM_;z$sE->+zSUuGVB1>X>1gn&e7 z2tkmnvQJiH7sR`Ex7XcflE2^PmdUtmCbL-yDdJRjpRVdtbx8lqbDohMlp^KqO0x}1 zBfw&wq-=YoJ=|aGfkuRXq%5HsXK3t(Wl)Fj`QN~V zi5RBihKs0d$aG9Tfw47+DhC->Nh&|rSB`kb%dhZ)Qmoc_DkYTlz;H*?jIPrxm3Y&O zkdW8Wxhf&L^P=Q22*J_$#qWqQjv<4RO1zk&#Vfv%-)M5 zJ3VS9gIAvViq^pE)_UNzDh&aY0qPi)diH7gnLGk#`@i3ReBJ4W?jsVy?U+m!e~yUn zdmZ9+^${C){trgBBd+c*Jc~*c09| z6KpgSAlX8_K~L?b=UIG2qPQpyeaRVnE|u`p98goXyaam}}qIF4EW}GoIwUu6t<()a~;QU1Ir`iW-^avlet1X)w2rVZJY-j%= zqfbB_xVK+$?+BTEsuH4QbyVCu!4G%8BE8X1|?Xxm@g3ll+47(&!51 z89@nXZiLj^3catF={-t(U|O)oXsFNuJX+K@CAX9t2d~apa6y7kB1n76ZH0CrScxE#{4<{T)n*YtEN(Sr{aL>QgF&0 zuwYzW;>#i)u3frqPif(T}p~HC+i}o zw-1!ezq*lE{b!&$4*56n1Q>E(e}eiCv%nt80tzs|R(U}O004h1lawJ=0YQ`NAtD@& zd_yh70{{S+4FCWX00000000000001hfy)q+DI!1u!4#8gA~^vwlcgdi1yOWQL?x5Z YA{7A?ljb5m0cevYBO?YHAOHXW0AC<0aR2}S diff --git a/summary_results.json b/summary_results.json index d80146003..f91b094ea 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1333, + "Performance_Result": 10.2153, "Performance_Units": "Samples/s" }, { @@ -330,6 +330,42 @@ "Category": "open", "Suite": "edge", "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 76.951", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", + "Software": "onnxruntime", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.426752, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0011", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", "UsedModel": "bert-99", "Model": "bert-99", @@ -360,7 +396,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0011", + "ID": "5.0-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -396,7 +432,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0012", + "ID": "5.0-0013", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From ae08f4423046af0c0ab2413c772a833a599bcd8c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 17:51:59 +0000 Subject: [PATCH 029/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9038 -> 9038 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ba633861c..9e95e5644 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 17:47:47,271 submission_checker1.py:3263 INFO] Results=13, NoResults=0, Power Results=0 -[2025-02-11 17:47:47,271 submission_checker1.py:3270 INFO] --- -[2025-02-11 17:47:47,271 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 17:51:54,043 submission_checker1.py:3263 INFO] Results=13, NoResults=0, Power Results=0 +[2025-02-11 17:51:54,043 submission_checker1.py:3270 INFO] --- +[2025-02-11 17:51:54,043 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 17:47:47,271 submission_checker1.py:3276 INFO] Open Results=13, Open Power Results=0 +[2025-02-11 17:51:54,043 submission_checker1.py:3276 INFO] Open Results=13, Open Power Results=0 -[2025-02-11 17:47:47,271 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 17:51:54,043 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 17:47:47,271 submission_checker1.py:3286 INFO] --- -[2025-02-11 17:47:47,271 submission_checker1.py:3288 INFO] Systems=10, Power Systems=0 -[2025-02-11 17:47:47,272 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 17:47:47,272 submission_checker1.py:3297 INFO] Open Systems=10, Open Power Systems=0 -[2025-02-11 17:47:47,272 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 17:47:47,272 submission_checker1.py:3307 INFO] --- -[2025-02-11 17:47:47,272 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 17:47:48,332 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 17:51:54,043 submission_checker1.py:3286 INFO] --- +[2025-02-11 17:51:54,043 submission_checker1.py:3288 INFO] Systems=10, Power Systems=0 +[2025-02-11 17:51:54,043 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 17:51:54,043 submission_checker1.py:3297 INFO] Open Systems=10, Open Power Systems=0 +[2025-02-11 17:51:54,043 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 17:51:54,043 submission_checker1.py:3307 INFO] --- +[2025-02-11 17:51:54,043 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 17:51:55,157 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,12 +23,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.39588 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2153 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.81708 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 39adda3c6..717d11435 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -656 +657 diff --git a/summary.csv b/summary.csv index 4a659dd7f..db3903681 100644 --- a/summary.csv +++ b/summary.csv @@ -1,12 +1,12 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.39588,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2153,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.81708,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fff2cfa9c..cf7a4a0b4 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.39588, + "Result":0.394347, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -219,8 +219,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2153, - "Accuracy":"acc: 75.400", + "Result":9.81708, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4b7984875c350a75811420506e84f66760ad029f..85eb43a364bb7ac8d1f100788b73b2f6fdae9aeb 100644 GIT binary patch delta 2431 zcmV-_34r#_M$SgC*ar#Mqn>k+2LJ#Sli>$i1lje^9O7RVvT*HAI zIB>f$po)0bL1}bQ8j7J=jbVfp29SIsB&;gACL~%_azjYKdfXBcV=bwqtJCS~$N>m+ zgcT;p_Ep98tBQY{R~5H)ib|#iovDHB!FrFd!hos1F1dbPa)TunNVe8VC0mos_RvHQ z?CKOpSm9I^J*;R|6~|b?01BT82{^;P5E8I8Cxir`a7svws^pB2fUW5h60kStgaoYZ zfRKR2y&xoDb%$7@fz`bvBw%%~2nkr-uY^RW&o+xb+bn;4x`I_Z!V(j#S`SM!uxiJI z1gzR;EYZQLeZdk7Omy-rA+Ct0#0>CEt1<6CAA7z-ox_0vp1=lY`vtTa5?cT!z9ig& zw69*`o_0_Fz4|;FoDI+47sXI2s-c;m&6jtI7X@mt5Pt(v>4;A;Po`r(obp7}YkZvX zG#jVOyTE@>^U^0B->B6@bHKz|jd>rN+u$mAzG)7Q z4T#yno#VoWL=yW0BEBS|0xNrE?~%yaeC&Gp&zXk!^-{WO%mk0|Qx+xOe0;AdmeOo^ zOQsn+khqUjb|CS_-XPI%l$K>}si7SjLL@dC|NvnoLY;FU|S&ezW1voi}0|R0L0L}%p z84_Cn>rgCl;E$Hpeqjq-FG?E=xA5@jSR0c)3>SaUjV%zeCQV^64(@hfmLo!Afs>#| zXb^~$29ii<5Q3o`ceBK!;jWY29d-XOM^&a4MvjCAS8)L{Rl~{v6~P{ zR)>EiF$sK7Q|F-1;LC(_G%}!~L`XubA!TJlB(cWPAd+mqNG|WhZE(3_67ZPa2gRK- zzeTc^qWuPu`^atwB1fyhN+OwJ8@jWFNBS+(A;T|jObzmgHI@!}#2U;ap+P)i4d#*1 zz~}U94apQHM-ojb%V4u?90qXb_WFgP9~Wh)KPUNx}hQvc^Gweei`z zI5Hq6!AwG{Az_8fR;QDqQtS;c94>y>XYTLP@{1KVvF;KKKG992pRk*cOBmS`7&+*tc>?XdDAV zX}mQktHtYt^H`jZNm@MNaenLfQi(N| z1*s%D4-l1v22qJMm`Xx}s5I-SBpiPrDr+3{*9Tvygd+oD67TksVTDUrtidJhKKpY1H!k05rw&7{I+hMQ1k`_xuIGR! zGv4EiMJjobSe4RrJzBPckJ!O&*){zl7ZXO|#Lr*!x{I>MJM+Vtl>RKc$E}0jcdeN_ z^QNQaf=k&`%sVoFjKyB%*~-%7aa#6`OL5{yiJv`{v*(TL%Z1wwqRIOrsAcH?T=Mtt zym!{YJJ0&?QPMt09ZBT=EEPl96|>|YsIa>2*q!kUH<|fiDlPa_B-&JLSxU-I^x|_C z#lIr8Y`3vz5>ViD1IEk! z`)A4i(er9d@{pbUJ33|sx{bMx1@9oezK81M*iVY?|7H6?92>G01LPv7g*eH?$BAoc zhyRl9U9$XOJpa2wIInAXlIykbqgT6bFPS=9&BQSp?0%a~W+QR~*Q%I+tnPmwNzV*W zFy44|!9>mu*O$B$5IGkR_WMN?q!LB*lZtbcjh21V3v?$yP5$NW@7lWU_;22RS9e|4 z`~LLXw(j_*OUTW15zeOzg}CS`9M5OVa6P_Tsd>P1)vVwLrXBc}ef$t|J)68(Z8cl? zw&VGM7r2hu?V?;nfx~SwU0c^Hvk03(18LSJSG{zCvO0!{MCj*Iddu*b{4K$kK*LsdN~>UzR&vEV2$|+ z{#?Gi4XdV7?@z@6HI?9$J7B@Mro!h%JY2_oSjH2tC5Xpe)UUi978(0MBryOT;~tfd z+0*kSYJr*}FUmA8QbJbbo|c4`w_p2_7yT!oIt=+Y@dy}mUw(q-53|4?$^r@3qn>k+ x2LJ#SlanEO1*dwQjm?u1B3A(!lWHP50gIERA|?ghtFw?Llh7g+0c?}zB4`?@fg=C_ delta 2430 zcmV-^34!*`M$SgC*ar#ss~okE2LJ#Rli>$i1X=s%9kV|O0|9@2KAskQWuum}77-r1 zncHnA(Swu}Go-PbpBZ+bX<}KteDrHRoX4iF21d^VlE@=pz5A}-R_}AiE5$cxat#M= z;K1$1fGXly2c^+LX()zfHHHyZ7(nukkg%%cnviH!$qgX^>v2m+jJ2eau1=?`BL^VR z5muNW+gBCWuPT3TURB)IDJq#7bfyNf2kSk;3InG4y5#zG$qklRAlX_cm26Ei+d~sM zu&YxXVTDsw^su5;RUBgl11NkZB;X4BLP)@=IUytfg;PReR3&GG1e}^aApz&+oREOE z9S{<*xEF*3tnLs?G_bmtgaoYa6(Iqu`<0OB^wnn3SDSx@FITW?M_6KlRqJ7i23GBu zkbqVDj3qi)wJ%s=fr(C@CBz-^l$ZftX*K5k=VQ-1)Hxg&;0bJSwO>G+A#n=8#FvCy zkoMI}+|%yqzgM3}gR|lJ`=S_1MKv_@v-$E)@uENt7UFLpDjo4D=E-#Ihf|)2dX0}W zo@V28c^7~9Xcr=o;O)m1IiSbH9Oaoq6YWg`m1 z$7C61qL^x|RAVm+r~XX-C{K9i`ioTl&{n6!cKvl9DT$($i_nUHTjKgw#>L7KLosyS zcD6ug)6m)YjKh08mYDD+k0l5?LIR<~5(FI~0T+MhiCqI7VGfu$t1<6ma~s?R&o|A% zu>mnV*f}n2NF;H7K*X0sRA6PV>^%}Wn~z;D|2fkTzg|jLjhWyve#)Z6n~(1`#ZsCL zZ^<-c2NL&@$_^yn*c&7oj?%KMEj6@5gXrKB4G=oQ5)+*09+n^w2?+!dOAv^J1cK<) zfJlFsLlA4s``8?T=+uBn%nm_xAc(~I0sHkOmN;P9uAILI#C@c)1BkayWlPjejcGPb zVv8oRh0inqbfO9GXGaq#-QE!Da_0IU+O`xCVNJ z260GfAc%wpkx0b@QCw>%76}Ik#ySW6^}#icXk;==NN}gZRj(AThGy9tyId7Y`lbCL!!N>3142o>PB(uc zl*HSA1SO$CP+|>+lF%S1jXEd^2MEd<2mSTI7bxM#fSAO#Ae7K*NLazPl}ke77!XS1 ztwC8WUMHN#;(AQd;sKBITfZmwk>U~aEn%Ccv85q^ZE3?{M3W8d!yz;lY{Vfnn9ChnaR?3OlF%S7>$-6$ z2Z+nMejJ7xa?r?tm;`f)dU6OWT*6`vE@AiCmh-=H`Bocs80yrqbl4!EZZv;A2Rxba z9$zd{$&19Ql&0&^vJ-s74sOe)=@+?}FbXGr{+`!elr7$wAI_xoXW2Y%9rV6y&D@zc z9W57J%AR80k@;gR&Q+eREKMG#W!tzECw`Rp*;6@t-nYJ7xZNO{yf1=UhW^hbfB(*V zXC1urtPdY0?Ss^jMDEX0kzGPDOa6fhtJ{v<8NYConIERof=@-FO~sa_q-;blK4(!} zd=j^|yC}<|;Q8B}yAz(|-^7oEX%uD8U-BhcH}rS^1GA71m;nVGkBUgnvn>%;0)IJr z&%`$Y!2sQ*0GTk_VAwz*8#K1dj3+}hz$j1t|P+-q^OHZ&+PdMYqUx;^5FWhq(YFQdpNEqe$sa`o_PlYOo zBm-2eZ8>dXXzD{d+=rslvU^5L_J2Qo?mOhu!jaL4i}tSeCQ9MK(rr_>J{qkbFkbH8 zKTGzHo>ya%hwS9v(J?E~ZOnBncn9hAJya*heo}P*FWU#=*pRgtAQw3;#7QPTPFzbn z{Fij^lH~{E`QH`7d0oSkT(5;6z1nqq$<*0uCXUfy_uFhT8<88hR>cHlb$|azdS-xv z@y4qQCUSPTzT~BV$hm;9-!Gydl_;8@RGg!1wCs~!pgRF-@-J_H*Vb*vfAjXcy6d{$ z_ov^sb;mbdLT;vua6Vlq#6?fxcs^T(>+#)6%>$OJW(7Ym?ZCI}3ZuO(%jpYaJJ6P_8YtAIp*tZ>5>e}cq}K7@S9TmH0j~<$ zNcYZ}JA;blK{nAZSSPe95lK=Mfnu5qwJXs_y^0@EciE4?y#Zko9+N-X?l+(c*?1#xCh2& zBBBw=*@QGdwiy7NVQ4^t@(z=P;Ir3*m2S3RP%fNO_|Cwr^(0R2gA4AzjJ z;LZB=eHb+ss$UfYR27`%%mVe=Rt3JigvB+~`E^+FRsetMfd!zqHu*R41UTlt{sh$@v%nt80txr49JP=K w000z|lOcKq9FK}f&XW=%R{|H9ZIk9AXl8?bo&W#< diff --git a/summary_results.json b/summary_results.json index f91b094ea..02bacdb97 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2153, + "Performance_Result": 9.81708, "Performance_Units": "Samples/s" }, { @@ -212,7 +212,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.39588, + "Performance_Result": 0.394347, "Performance_Units": "Samples/s" }, { From 2b78956207efe3275e2aa95ab09bd6ada44c3285 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 18:41:05 +0000 Subject: [PATCH 030/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 6 +++--- summary.xlsx | Bin 9038 -> 9039 bytes summary_results.json | 6 +++--- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9e95e5644..e3c6ec29b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 17:51:54,043 submission_checker1.py:3263 INFO] Results=13, NoResults=0, Power Results=0 -[2025-02-11 17:51:54,043 submission_checker1.py:3270 INFO] --- -[2025-02-11 17:51:54,043 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 18:41:00,582 submission_checker1.py:3263 INFO] Results=13, NoResults=0, Power Results=0 +[2025-02-11 18:41:00,582 submission_checker1.py:3270 INFO] --- +[2025-02-11 18:41:00,582 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 17:51:54,043 submission_checker1.py:3276 INFO] Open Results=13, Open Power Results=0 +[2025-02-11 18:41:00,582 submission_checker1.py:3276 INFO] Open Results=13, Open Power Results=0 -[2025-02-11 17:51:54,043 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 18:41:00,582 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 17:51:54,043 submission_checker1.py:3286 INFO] --- -[2025-02-11 17:51:54,043 submission_checker1.py:3288 INFO] Systems=10, Power Systems=0 -[2025-02-11 17:51:54,043 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 17:51:54,043 submission_checker1.py:3297 INFO] Open Systems=10, Open Power Systems=0 -[2025-02-11 17:51:54,043 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 17:51:54,043 submission_checker1.py:3307 INFO] --- -[2025-02-11 17:51:54,043 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 17:51:55,157 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 18:41:00,582 submission_checker1.py:3286 INFO] --- +[2025-02-11 18:41:00,582 submission_checker1.py:3288 INFO] Systems=10, Power Systems=0 +[2025-02-11 18:41:00,582 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 18:41:00,582 submission_checker1.py:3297 INFO] Open Systems=10, Open Power Systems=0 +[2025-02-11 18:41:00,582 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 18:41:00,582 submission_checker1.py:3307 INFO] --- +[2025-02-11 18:41:00,582 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 18:41:01,626 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.81708 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.91849 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 717d11435..4932818c5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -657 +658 diff --git a/summary.csv b/summary.csv index db3903681..9b2709fc8 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.81708,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.91849,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cf7a4a0b4..8d0b5e9ac 100644 --- a/summary.json +++ b/summary.json @@ -219,8 +219,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.81708, - "Accuracy":"acc: 75.000", + "Result":9.91849, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -229,7 +229,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 85eb43a364bb7ac8d1f100788b73b2f6fdae9aeb..ff29503aea8d78c41809e147df0dbf61306dead3 100644 GIT binary patch delta 1337 zcmV-91;+Z$M$bmDC=msb;8)?!lPwV;f7@>2NDzJBuV{H+W*&USS40>gAQ2iu5G1SY z6Kd=NylZ!R-EAiM`)w}CWL-9s*{oJVLY(UE(^YkJs~7fTba1hc2X8y(-iRzaw-g708^F?e=lM8Wwd0os~ZYdHwJ!vL` zSDyNcR>SMoy5Th{4F==^su<;Z_G$5%JOXEj?{9(bxRLjWgm60|{GCIB;5ORO_ zbv#?H=Hta`Ik=rq$A{e(rlWD(C4p-GkwJoPj3{8D#i~CSq*^X@TtXUYE2^J+|TpCA1@IA%wNgSid`?;*XuhwA9q zPm1pUW&1!J8?u%F2o$2Ar8vqY#FJ1I8-K5uI$O=e2^#Evn~Wv{as$_@n1C$qABk&4 zC>U?Nx?mz_hwDpT3W%Hw2>bma3UU%f^OK5mln>@z;zouSp(g+G_IDh^@xnK6zi;@y zAAEoM9mnuO%O~V!JX=l2Gllr*DI88G^VM>Aw@~wd?WvRB;+O{3g@2ecJRLH`PD9{@=416=4Vj zCTeO=p5QOi23ZuI%%V|pxwgYX+sewXMT5=Q8CK0uMkkmSU`^rw@B;Zqz ztC{_Fc2T)Fs3!RZ=cUmVB3X_S(A)^AcNO|rFVi_nd|+Cz#%QQe2OcfzyOLYVje|F5 zEVv-SClRDQ<+eim5G=*m^F1i$v*^%K+#07?h^5aFWvx7`N@VsxpFEQIt!E5vpBjBUXI4LZL@YZSYv*I zzgDmB!>Xy&{i!&hx)hvp2P_y@m-xDfhijM*t9atA1o6~Ged6u3$gvMZ5(Cg<+_UmI zdwy9jBvD=D#Y37GDIsezr}u>BWYdql>AwKgX~@5cXTUM{?I)=J0JFdzO#=my;8)?! vlM*6V0UDEPB02$#lcgdi1SV=~PLt0f76EOO<{~};!ILE;BL;II00000H|cx= delta 1336 zcmV-81;_f&M$SgCC=msxdYz5UlPwV;e{FB#I1v7Rzry0a?0(2AT0J%%-TR7tjA;2OY zCv11sdAPsU{0TPP1ne1!a5}4C6kIausft=$zLUmE5vs?!+}VK13A3qF{A@S z4hO~&sK9juj1~#6;}0y6q7JS=l&P$vX-YF{Yo58$f}n1^+iuf@K^?y5e*zN*Vwg=D zE~304vkCbG`c5Y*9As3+iTqq&e>vtUFTTPvab7`9XiEsQwZCOhMzz%G)N8&w3QmQocl89w=LJ8uv37CVZ>PAko% z@XAwP(Q0_zS~t8#rNMwaKoz50Pd}|blSkm}`2Ee(>|WqJA|c#P$aMMpn0TJsC2mg} z^8!e_c&pHmDkT@JJ#1|Xf86Ts;bv;}iEHVeXzq9gT{8b{G$O@?ObJ3 zi<9lPJhpx93Ab08o=e+ujwZ|MmZ>Ir8Y z`3vz5>V(cabGL@7L2x^3##N2B!v#>@TtXUYE2^J+};ke&QHI%WmBjk%5m?;yRthw9|m zPm1pUW&1!J8?qJy2;?HCg*eH?$CFSL8-FjEI$O=eF&gZCn@wgTas$_@n1HPAA4$&) zP%z$jb-_f=4%e5w6c9NV5cc~;6r>VG^OK5ml#P~s(hGDaKu!MT?eE&U?f7rreph#0 z*ZcnT+qUlbrc21pbP>*{3x&AoDICvd%WyruTd8@#a@DNh2c{kPmVNvXay^^8STAig zTllu)`GFU>j@j*^T%9iMxq9HBK%J%v-)x#Hs)568GF@BMrti3+{^1S(2D2I!VF(1> ztFw@ke;puyPQx$|gzr2>cC4j6Xl>qfbB_xVK+$?*WTEsuH4QbyVH)+#W>WSle zX1|?XxtecPm;8eB(&!qIEJq1wZiLirjow#_^Z_M4FfCYTG}LGZ9?hGZid)K!gI8xP zxFEqN5u`okwnm!}EXCOK4JhWb=+IGI8>d)^rE9T&W&E49fQvl8!wLcuEHJzWrgkEt z15)q_X@6Wd00hU-fdU;oCKmEe>+V8OVi!skUiT*rJ^#uKk4h{s*jue===8T&vaF#sLo9+i*T z)AMR1ftn&O$}}%hLRRFSmV}nKU;B|4{U@M04EZZlh7g-0c?}!B0d4YlO`i0267+(0002S41(nV diff --git a/summary_results.json b/summary_results.json index 02bacdb97..e3b60e38f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -55,7 +55,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline", "Software": "pytorch v2.4.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.81708, + "Performance_Result": 9.91849, "Performance_Units": "Samples/s" }, { From 6c300893a3ea334e7500ee58b6b735973e89ef95 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 18:52:52 +0000 Subject: [PATCH 031/719] Updated results summary [skip ci] --- README.md | 39 ++++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 1 + summary.json | 40 +++++++++++++++++++++++++++++++++++----- summary.xlsx | Bin 9039 -> 9137 bytes summary_results.json | 36 ++++++++++++++++++++++++++++++++++++ 6 files changed, 93 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index e3c6ec29b..5af576c09 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 18:41:00,582 submission_checker1.py:3263 INFO] Results=13, NoResults=0, Power Results=0 -[2025-02-11 18:41:00,582 submission_checker1.py:3270 INFO] --- -[2025-02-11 18:41:00,582 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 18:52:47,282 submission_checker1.py:3263 INFO] Results=14, NoResults=0, Power Results=0 +[2025-02-11 18:52:47,282 submission_checker1.py:3270 INFO] --- +[2025-02-11 18:52:47,282 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 18:41:00,582 submission_checker1.py:3276 INFO] Open Results=13, Open Power Results=0 +[2025-02-11 18:52:47,282 submission_checker1.py:3276 INFO] Open Results=14, Open Power Results=0 -[2025-02-11 18:41:00,582 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 18:52:47,282 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 18:41:00,582 submission_checker1.py:3286 INFO] --- -[2025-02-11 18:41:00,582 submission_checker1.py:3288 INFO] Systems=10, Power Systems=0 -[2025-02-11 18:41:00,582 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 18:41:00,582 submission_checker1.py:3297 INFO] Open Systems=10, Open Power Systems=0 -[2025-02-11 18:41:00,582 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 18:41:00,582 submission_checker1.py:3307 INFO] --- -[2025-02-11 18:41:00,582 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 18:41:01,626 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 18:52:47,282 submission_checker1.py:3286 INFO] --- +[2025-02-11 18:52:47,282 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 +[2025-02-11 18:52:47,282 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 18:52:47,282 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 +[2025-02-11 18:52:47,282 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 18:52:47,282 submission_checker1.py:3307 INFO] --- +[2025-02-11 18:52:47,282 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 18:52:48,358 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,9 +28,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.91849 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.91849 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 4932818c5..fc2f3d0ab 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -658 +659 diff --git a/summary.csv b/summary.csv index 9b2709fc8..ab614010a 100644 --- a/summary.csv +++ b/summary.csv @@ -6,6 +6,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.91849,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8d0b5e9ac..f8171581a 100644 --- a/summary.json +++ b/summary.json @@ -210,6 +210,36 @@ "weight_data_types":"fp32" }, "7":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.406474, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -239,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -299,7 +329,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index ff29503aea8d78c41809e147df0dbf61306dead3..d442c78a2f47403de28b6f6dfb6304aa953e8891 100644 GIT binary patch delta 3360 zcmV+*4d3$5MzKe**ar^t>AV={2LJ%6GXMY>0Fz-M6@M}=cx`Omn_F+%SP;kG_fuF_ zA6jX#IldDCRYKY(gpib6s(nI^Ie=Asscmxk_Iqp-f@cPYcB>`oO9IFLnKN_jpFP;) z_Q%IyA>H#N^`o#OGo>j@JoKW8AI>`R&GqMoBTH%Kh7)%Yg}fs_@l^iU{j2>DCEwFI z=b0oNgnwyAo@ZHnpsJ}i=Yg9lQOrZJWf~=cn~ATVGa5*(iEA;r?u%iDff$6E9k%#a}7# z#ae2z6u6HaS(l{;f0E5}$I{%NOnAP~vg9q(EDFAro7l=hY09jXEo-v1bxUDJ)6&d* ze>^St%0?|`Eh0R2Gq>ALq6aA{W=LZ<-!tq$)5NlP`RLbtIFC(T4UC=#B#}qHdiPzu zt$*IsJ*wuPScq6qQU3I#UDLgY_O^g#lB2U2^@p$7lZ_??j<1stNWFZ==9NM(MOww4_B~i zM_6LQ;|)tRAUP%^VAVchi4IonGnUvu^5t1VoDolm8Q_stW8Qz>_I!mpg98I1J|1wK zoa5)w!vmLL+b#D*lc zYDgr^A&E8S{TATl3=Ry42|!tY4sC|SE`W6?mN+2p%abh(8GoOzOZSoDE_G?E3SrC8 zjV%zeCQV^64$4k2%Mqcmz!|qkXb^~$29ii<5Q>BbvAEVyFj5X6jdc$C>y2w3(a3^>;2l=&r+wG{0ah}=hZI}kZq1y&Nt6x-09Ej-dsnGP9# zaARtaN35|7$RpNZ9tjQN5o<7yga+}b)$vF;Ks?qs=zp&_zVHY~2E-)j<|(ooQdWpZ ztZ_`pqxOb)EUuBi`EKf765I(%Z39V7SGNClWb~uZLqD(Ws{iRs?jymS3Rk^SxEh*e zZ(QZ7P=C@-?GG7#5^fq0O5%CC1)(G!|05^~4T2JDFqDJ_L21-MNjN}I);Q>|H@-j# zM+U?sHhf5ptcHXY>|41YG>!$KG~ODN)#7!+c`S~{BrWdnIKT9Javv$~faUcnuo6p4 z(F}XzyjrDlOLqyw9RsEbsl*z~hEx)r2Z%~SgMX;R8cZdjK~$P`R1yvll{F6f>y0l| z!jSHfVE*T4B5$;3=Ze`?CYzRFLnhUY zHxUA!%y^G47OCV#V$Dj^^=SEe#)uu<@ly7uc z2fc4vGk4}qN6Q75vZt7LWd0b7-IQl5OOwZG`7VbPCw`Rp*;6@t{xZpO;dX;)^1cXa z8Tvn#{QW!coptcevp)PSX&?R{4kXkd@2%cDl1~G zlJf14;&T?o#V1j2+(lUy1<&8++@0_w|0aI8PopS%{*o`r`YoI8zq61Jm;ni6GHV>! z0{{S}lQ9u1e+7I&zz6|}&=7(k8D*cW#xB6ScDL8vW|F_(=8{atWi!b}yHcbmr@H%e zojz5r^u9c1DcM6I63+UXZCDxs7V$V?+rD;xccuFp5t6c)rkp`vdjg?-8T{50LK3x= ziN3azGVhwEh<1=sVelN7ifnk1QK|kcwx-Anpt0Bif5L+CQQUI9oJk$WkaT8@(GNsL6jV1SjLI`TwOWhDKEal6H2jM zXNiGZLUiXv!6OiYqx19M5n~iV3I&yTF-414d?UZp z0%Vu& z&oZvwd%Fv1Bvpmz1#F;D1p<1MmwNuB6p41>f8H@%b!ZG5s*Dy!oNbb=dC_1O$;!1V zf>KYZQcQ-AzT3>(#=OOrBD2$zXHt0Ksjg@YylkxoUR0?GpbSvOsMOO>%g^KiI6M6Q z?&(fD=sX}J+>FU&@yCdGp4%dBTOaWPNV9oQ(vZ`WT!^-Dv`)C??fvz{?h@BFJj<~O ze>nn9m$W=1aJ}FR%oGq9E!Xx4xx4*3nXkgdWErl8H;dWiu-n3PG>W>!Ya49~-vk5$ zbe95T!e~uk4TY@n*fy7c*1)QrtJl=yWV0y`ZC`)H?UiPE3N?-HTsy4>yxo9z8gOKb zGwwCueglqjJmEGH_KY|51RM2)Gv1J|e~t2mcnj5nK9{GSrO^oqqdY&>F=y3t?L--`qLKor>L}@w%Oo{51;!6TQzWGG~%MU+r3gL+}nm@8TLoj`UT_VzW!da zpL$`1N$#^_euu>Dz;H0vp%HYDkKaOdOzfSe`+wO!pkvLe#Q?d;X(5g_@$thof3(Se zN%tmOeo)W1bV#n@N?jPp^6r7O ztpG{?O2Z2wa+bHcrvE_7v|HvPa&^?&BzAF~=2 zVF(1TUSVgGe;puyZo)7SgztTd$oIr?C@r#*Ahi{zB2_9?QE!&LfYta%w%d@muM;4E z#Hq*i%zit&ayj3tCiw;DrO_24S&kCW+z6?+75Z2$(iuv8U|O)oXsA#J9?k2!l3U7+ zgEwa^xFEqN5u`okwnDoQEXCOK9Vq6r=+IHz7^hf>rQ2eE%lJ3j0xt6W0V@bhu)uH+ zOzlWSN2K5*(*C$+00@qu0R=jEOfrJc-T+Sdn+=0<;Z!no7H(Qk;^^LcIU3ux&DzOe zjrj>)FJIq>Ra2?^Q*l6bDLCa0STL?G@nsPY*DxQJ@x&Vm;;D=Jjkn_>hdvNV3_uTY zPs-=y`DL{vKy{H9(=;zqLRMr(rv<%#{OU(u^`C+2ION~N6X1~h`V-WDv%wz90t)l# zycp*P005~ola(P>0q2wKAtD=NGHV>!0{{S}4gdfY00000000000001hfpL>5B0&Ls qlWQV60q&EgA|?c{USVgG&>|KA&y(jOJ^~{jlP4o02Gt+{0002DTRKbt delta 3315 zcmVk+2LJ#SFaQ7;0001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`OmnpK6(Dj{tWLP$!JRObmfW&x}CQrqP6?a$gK1kVf( zt*S%Rmju@TnVs47&mQb)e|iiS(mhX7KMFfCQ<}2GLob^6;jDin-&}ubII@&xZa8rl zQOG;;6Hnz&-GAB-QSu|5bDl}UL6~;rd6vZos+xLp9=NFz#XJ;8rcn~OnfRK_)HF`G zJ1LBUg{n21mKwNzD3=}%l3jd6)2Z+A7x7GXH@k;Rxnq0$y8#r*gF`$Zg)P#TJ%S&dc{~Ibc7Wq$o5sm^{a}TR~5H)ib|#iovDHB z!FrFd!hnCNzAm|bU2=mZ7D%?%NhMp8%=XYk4(#d_M_A!h6+NtIRTalr!2k-M2?;pE zz7P_yH7A4wpm0h^jH={}kbte}6B4jD=Y#~T?SPPg#l0XTV0DLBqJh=DBqU&UuLucP z-LHg1r_VNvKHDsOx`I_Z!V(j#S`SM!uxiJI1gt{ZXDrdds(rx{3ruwKEFrFlr^F2K zOsg^PKOcL(L!HBc0iM7HXZr=T84_CnCcY%xg0!z*;+}R-|GoM=8k`N!-;>B6@bHKz|jd>rN+u$mAzG)7Q4T#yno#VoWL=yW0 zBEBS|0xNrE?~%yaeC&Gp&zXk!^-{WO%mj~r@lzHh-h6zoDVEY~cuS@kJCL}KRCXZo z#@-;&aFmv1ZKW5)ud^mLL!b2?Wup0g*6=Al8`ou{i?K zsR5Cg9fIgU5Q+T(_v=e6alo=&Ie!m``$%O65O1BzEm1c$rr9)!Et6gx0P9dJao~@Z z)_!3NTrWx+3%Bs_=vW)rfqHa&&lIhxuwSoB_mSc*b!n>#Vaw2sEfBIMO<^$(?sj07 zBSK?=lb}av5Qvlpl1OL}ic~z1#kGcif{}25XsmP4Umslah(-ofCh+!$tcHXYVv^7x zC^y1?i&TP};V%QZm-KcaauwJbNvmZ$Tb>3o`ceBK!;jWY29d-XOM^&a4MvjCAS8)L z{Rl~{v6~P{R)-`p34Bmf=b+Ev%Y<_@GN7VFNJ6V2Wo1JovBuFLl5D?7F7L#DZE(3_ z67ZPa2gRK-zeTc^qWuPu`^atwB1fyhN+OwJ8@jWFNBS+(A;T|jObzmgHI@!}#2U;a zp+P)i4d#*1z~}U94apT2Ma3>_S4J0*P+5X#+(XT>(4;k@>QHM-ojb%V4u?90qXb_WFgP9~Wh)KPUNx}hQ zvc^Gweei`zI5Hq6!AwG{Az_8fR;QDqQtS;c94>y>XYTLP@{1KV?XdDAVX}mQktHtYt^H`jZNm@MNaenLfQi(N|1*s%D4-l1v22qJMm`Xx}s5I-SBpe_rYaH~~2Vba!BLiX*^vo4m4Jj*B zz7h>TsC+fwHkE(m!RuaslH93;HLE3T(=@hp1h6fAIE?7BfsHtX#)6$Vga&iDLoW`Y z!CVp=#ARJO4&?xGS=W%mP)80L84#0TE>TksVTDUrtidJhKKpY1H!k05rw&7{I+hMQ z1k{bL=YS_O-s6i!DtVDumC|%QTDF3Z*uibtHT@zN6Gq|0&tLR^x{I>MJM+Vtl>RKc z$E}0jcdeN_^QNQaf=k&`%sVoFjKyB%*~-%7aa#6`OL5{yiJv`{v*(TL%Z1wwqRIOr zsAcH?T=Mttym!{YJJ0&?QPMt09ZBT=EEU-mv*aJBu)6Koo$(7dnfYNVE%;O<+Ei>= zO3F_3;&T?o#V2tdX}gQEEDD~#&AB__N&ZdzNSH=Z_WUJZl66Oa_dm1e510W7k>FS1 z&I14dACrg?D}TjTL>M6;5gI}eB&+NbYU~2MYj=CyZ6^8qZ7#`VT{e^1tX4uooa*k= zRduSm=zV$0GqQ(Lq?}!8wqa=mSj>}@ZLhS4`)fVah>(;eG~*1ev}X|7m;Nt3AtX^- znYhw+QWjm)6!8vnDhyr#Qqq}G6qJ=l-KbGmP%0v7a*!k*3m4ZId$~F zTI*3%H{NYFS<0Xe-}66#2@^3)MhzEH-jKwkv9o~9XHr&%iTre`4`ucI>+LUiYO$zu?Lqw|Yj5n~)f1|^kvF-41)d?UZn z62)%DCBDRifTS_Oin1zcj2s++RNkX=hU$zu<{6f5S(e>`J9Rvvk}6rLh?@ruo43;; z&vUNwz1^*7ELDZ*C2XKn1p<0pRC@8OA`x-6g(l1eRkH za(@JzF6jhD+-W&d&XOOf{l8@8Atv?vVVno;hxJ-&$IZ1tm2|L)+=Z1sZ<4#=75T| zEvHQkO?_yK`%qljj%&7L|HJ3LLq07WIgPn!?`m(NwA$N-V;S~GqxA#E%l-Rj$^Oyv zYD{vUAN@NxW=DpDxef*IA-%qb>gd={ithhq`#>BUvX%f8qM)TX$|S^zYiWo7l7H@9 zvix8?|GPpsuWNXc>$UKsSG#Vnm^xd{#0eVgew&OY19Ah`s+fQ*?jMP3MJO0=yt-f_ zXNT)cUJ8hu3kduDA_{U6Me~!2bCeI}UE)TD7ojHq^7eNe!|}p5Z@+K&z8`#l`W?sc zLdz%QW;|O>$1{ca=qVgdC-c>Ecty8R^MLKESs{!pCkk!n@FC=SGJ3H(YPJX+F9@R` z@;$55LAg3!I`ECiLxDO@6`>k8@KkXcIQ%BlwSC(39XHiKy#C*_d=+5`1SV=~PLtss zAb)DZFc5_AeF~$yEz4<35Viwt$*F-tQ(AHp(k2nLevs5Ue*4N!;w0cxkE@yec6L#@ zIH)H11?Q#F6(U)V642ZTsdp9nSTEB#N_=2iu*PVpPzN3@>bsI#%8i3JXDqlN!6y-< zJ>|AS`w%R}*z-Lo=CkO~QQR7*Scs+DVSmf`H`@U&^867i2u!fRa1TuFNJJ;3;3Lxh zxMlzdj-desI(SSnf-l|xPWj1(LAi1&89ED-*0VUe4_=POwr#U^HdteRg1=U;@58F8 z)cvVApt=;CatACJSC{y@h=*&K536|MtpxGZMSbG!w8*g!L=pqgW8AayIeUItFC1=48{4yy?FH)oIATiD$qu_w6UB{{XWiAIbs>*Q1_ukOu$&6)=K`Wi6bHgb07c!002$i6wLqt diff --git a/summary_results.json b/summary_results.json index e3b60e38f..2513d597a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -466,5 +466,41 @@ "ColorKey": "availableMLCommons", "Performance_Result": 0.433634, "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0014", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 76.951", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline", + "Software": "pytorch v2.6.0", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.406474, + "Performance_Units": "Samples/s" } ] \ No newline at end of file From 3c77dcabea47d70119a148cb8078f3182647f24e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 18:55:05 +0000 Subject: [PATCH 032/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9137 -> 9136 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5af576c09..17b0f2103 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 18:52:47,282 submission_checker1.py:3263 INFO] Results=14, NoResults=0, Power Results=0 -[2025-02-11 18:52:47,282 submission_checker1.py:3270 INFO] --- -[2025-02-11 18:52:47,282 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 18:55:00,799 submission_checker1.py:3263 INFO] Results=14, NoResults=0, Power Results=0 +[2025-02-11 18:55:00,799 submission_checker1.py:3270 INFO] --- +[2025-02-11 18:55:00,799 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 18:52:47,282 submission_checker1.py:3276 INFO] Open Results=14, Open Power Results=0 +[2025-02-11 18:55:00,799 submission_checker1.py:3276 INFO] Open Results=14, Open Power Results=0 -[2025-02-11 18:52:47,282 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 18:55:00,799 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 18:52:47,282 submission_checker1.py:3286 INFO] --- -[2025-02-11 18:52:47,282 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 -[2025-02-11 18:52:47,282 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 18:52:47,282 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 -[2025-02-11 18:52:47,282 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 18:52:47,282 submission_checker1.py:3307 INFO] --- -[2025-02-11 18:52:47,282 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 18:52:48,358 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 18:55:00,799 submission_checker1.py:3286 INFO] --- +[2025-02-11 18:55:00,799 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 +[2025-02-11 18:55:00,799 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 18:55:00,799 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 +[2025-02-11 18:55:00,799 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 18:55:00,799 submission_checker1.py:3307 INFO] --- +[2025-02-11 18:55:00,799 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 18:55:01,766 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.91849 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2496 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fc2f3d0ab..e250077b1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -659 +660 diff --git a/summary.csv b/summary.csv index ab614010a..b590d1665 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.91849,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2496,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f8171581a..6af682589 100644 --- a/summary.json +++ b/summary.json @@ -249,8 +249,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.91849, - "Accuracy":"acc: 74.400", + "Result":10.2496, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index d442c78a2f47403de28b6f6dfb6304aa953e8891..3a818cedd514b643226088267af86f8ce9235afa 100644 GIT binary patch delta 1323 zcmV+`1=RYnN3ch*e-Q*cf*=T!gAp-*-w-fDKq54RAV@~pC)C&lc-QXsy4y_h_uE{O z$+&DL*=Sda6y;QRpRUuV%9ZYy$1Ej#C`7_pPqPh6BfugaCv4l(?(eSjKqEp@7Soh7 z=xI+Nv@iYNxy#QI{=w~rKV$9 zo|(~vX(Zw$lReF`9F3GL`MZSS2?>0y-xW!}EBiM*hO}$Se%CwzCvX`7qea52_&rOc zsDcX+Wh$#^n$nCqx^J!ZFsue|H=8tJP=)XKU%-Tk7$&2di>PeKWJErJu{DU2gAB?z zk)Nw8hdkxQS9n4xmg_8$5{hbnV6dl2O4mu6NWAGuNXV+_Oi76DJTG_zLU43`{ySof zB1oa25-+A`@se-kcUmCrW?bS++zW_nA6QYA1&vUGBakY4bk0znQN>)t(k;uf8*rzJ z$5c{fE2X%3@UVF^9r7&W>bz!-guO zts>4g$=1ARu#05nS`|U5r&K8>SC4_$lTi~Qe~ZuL0XRGS{_g8eJ9HnA5pG6gJpW@z zeBWykudNSx0i@ZyCuzv(NiIa&I9ey%;`aV}Y>wXQJx>`m^1cRs6t6HKtlW`Onf6#C+*P#)($j5J?Iwtl`)BV3}AJDPp z)nb5L(1?xZT*b&)a-XZPK ze{e(esd=72T8`nk!K(?xGd$1pe+(dwW4M9k5pq4At)}CdLOk?=8crwk)pB^dP*aKR zsfi^BEhh|Y=kPb<~6tIIbUre(1SYtA*a`cxm4=LKg{joNWR%^x)a%H1>FP zp)32e=?8AA|1F321JF)Y}SuG^=!h5+9frtTh@c)PqOM`k~~O za)0CC%^3?WNbpGnX-~PW&@Kc^G4^~1iuo)$bQCwnDHdYsw%9WM!?u8nJb%Or0uwAS zd;q3)B%%{i@DXW$T{{2-$IyZT9Xuu(!541;r~G8YpjscJ#doM?0*L7Jp z8>}%u!C&jw_hHpk>cgoxpt=;CatACJS4Wrlx`>BsnGfrD;*A9H)JJ{d?X<|T4@43J z&|}=Q@;Q5cX;!E%@?xImMM_9R7IZ$RIr%<}yy?FH)oIAx#53TS`}Pym|Fa_>O#=ix hf*=T!bRxP1qBgxJB$G2EQ~@HBh$ABg)F1!=006n|bfW+O delta 1324 zcmV+{1=ISlN3ln+e-Q*@GHV=@gAp-*d_lkn0g2EMf*=`XpRC3%z`J(0*WG54zu)GP zOvYt1$ws?Uq$sDl`*fW?Rj%~DJZ34`Lm?8*`kHN68UYsZIAPnqc7J!J`x+6FvY4iv zL0@|Up?w+r))PV!wUvp!wv#gNnx=?$kWpdq9GHr1c#%=5{w%hp$P1vc*a66YEHxd= za?Ok;Od}C5ne1zhyV;}(gDQN-{{kjV#4sJ#TtsCp`@o8_ENFxh9D`KZqjN^;j4IYKEZwp!y8(Bq zcuXZ#wo;0l2M?P!vk}iSuHJjQ3uz=(h3Exrpil(@dX$%X{-hL%cH!P2F&J9 z7Dk+HlC62sU>C{CwJL&APpMK&hL676lTi~Qf6LG00XRGS{_g2cJLo(hBixM1Wbwy{ zc%IuLZd)Jm0!XuYPtuUnlU#_lakNgjW>!Ya49~-vk5$be95T!e~uk4TY@n*fy7c zf7ZaNovYW><7Bfb4{cw6#O;-4c?va+?p!;q2E5&XcN%bHi!<&u;C=&+ay;QS684NY z^aLCAgfrfduZ{ABcnj5nK9{GSrO^oqqdY&>F=y3t?L--`qLKor>L}@ zw%Oo{51;!6TQzWGG~%MU+r3gL+}nm@lW`Onf531s*P#(~kdNO&bxiD?ru%={KA>aG ztHl7h$Y~*tHu3SpHMGfpN%tmOeo)W1bV#n@N?jPp^6r7OtpG{?O2Z2wa+bHcrv zE_7v|HvPa&^?&BzA5cpJ1QY-O00;m8vxgN{2!F3$VP_=)002+|000pH003lfV^DH$ zaC0wXZ*pZWcx`N*lFe?yFc5_AeTvBU#BnGsvXUUR6{jLqDpgT$mc4-0_(!(ekhiZB zAb`ZF$Mwv9JG*i@->WA11?Q#F6(U)V642ZTskasSSS`{SN_=2iu*PVpPzN5(>${R$ z%72Z6H)kxkAi*aQq&?-fLc0(w#n|&5DCV>1&{5nNr&x%k+hWW3H`@X(^85iS2u!fR za1TuFNJK}Z;3LxhxMlzdj-desI(SSng3sOnPWhV+gL2_iGISPhT2JEW-g`M3+qTWx z$zYB730^N>--lIGsryrLKy@iN Date: Tue, 11 Feb 2025 20:35:40 +0000 Subject: [PATCH 033/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9136 -> 9137 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 17b0f2103..eac218a4c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 18:55:00,799 submission_checker1.py:3263 INFO] Results=14, NoResults=0, Power Results=0 -[2025-02-11 18:55:00,799 submission_checker1.py:3270 INFO] --- -[2025-02-11 18:55:00,799 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 20:35:35,634 submission_checker1.py:3263 INFO] Results=14, NoResults=0, Power Results=0 +[2025-02-11 20:35:35,634 submission_checker1.py:3270 INFO] --- +[2025-02-11 20:35:35,634 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 18:55:00,799 submission_checker1.py:3276 INFO] Open Results=14, Open Power Results=0 +[2025-02-11 20:35:35,634 submission_checker1.py:3276 INFO] Open Results=14, Open Power Results=0 -[2025-02-11 18:55:00,799 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 20:35:35,634 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 18:55:00,799 submission_checker1.py:3286 INFO] --- -[2025-02-11 18:55:00,799 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 -[2025-02-11 18:55:00,799 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 18:55:00,799 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 -[2025-02-11 18:55:00,799 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 18:55:00,799 submission_checker1.py:3307 INFO] --- -[2025-02-11 18:55:00,799 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 18:55:01,766 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 20:35:35,634 submission_checker1.py:3286 INFO] --- +[2025-02-11 20:35:35,634 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 +[2025-02-11 20:35:35,634 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 20:35:35,634 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 +[2025-02-11 20:35:35,634 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 20:35:35,634 submission_checker1.py:3307 INFO] --- +[2025-02-11 20:35:35,634 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 20:35:36,592 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2496 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2505 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e250077b1..ace944308 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -660 +661 diff --git a/summary.csv b/summary.csv index b590d1665..7b4245243 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2496,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2505,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6af682589..85f0a5beb 100644 --- a/summary.json +++ b/summary.json @@ -249,8 +249,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2496, - "Accuracy":"acc: 75.800", + "Result":10.2505, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 3a818cedd514b643226088267af86f8ce9235afa..b3382b4bdf2056091f57418537d443102867b481 100644 GIT binary patch delta 1183 zcmV;Q1YrBHN3ln+e-Q))D4^z(gAp-*d_lkn0g2EMf*=`XpRC3%z`J(0*WG54zu)GP zOvYt1$ws?Uq$sDl`*fW?Rj%~DJZ34`Lm?8*`kHN68UYsZIAPnqc7J!J`x+6FvY4iv zL0@|Up?w+r))PV!wUvp!wv#gNnx=?$kWpdq9GHr1c#%=5{w%hp$P1vc*a66YEHxd= za?Ok;Od}C5ne1zhyV;}(gDQN-{{kjV#4sJ#TtsCp`@o8_ENFxh9D`KZqjN^;j4IYKEZwp!y8(Bq zcuXZ#wo;0l2M?P!vk}iSuHJjQ3uz=(h3Exrpil(@dX$%X{-hL%cH!P2F&J9 z7Dk+HlC62sU>C{CwJL&APpMK&hL676lTi~QYRk{$0XRGS{_g2cJLo(hBixM1Wbwy{ zc%IuLZd)Jm0!XuYPtuUnlU#_lakNgjW>!Ya6bU-4iq{I}JFp#ToY+aK8aZIi7GE z346vHdV-C5!WnPK*GBn5yoG8(pUYFv(&&VQlRXqKBoyv#!?6teqiX$v@p4~(FWFDM zu)-ww*)hLEVs>CSnCs98I+KtT83Fi{suV1L>3BLM*KnmS3}kutK-yM-q<^L1g%CN* zTV3*8uH;;KS%?ZD^+=u`7NfwUaM>G-cE5Z7>B*ZVPmIF8Zr zEtin%$vm7*<_dAq2Wm8%F2dF5cB!Tk+f@^bA6QP{+s@&KkgMtVIqIkh$9FoOA9#T$ z+p$_L^j619d#(|5kWk0j##ciRo^4KJk6RbIvQL|S;HLUNbMOzdd=+5`f3qDz(3ZuO(KS(ITcAza;HBhKa zOLsz=B%;;>Nv-3*uk1KZ0$F7_x_8dp8C0$gvW@=0I-ymCvLr-Z5x z&Qrx{s})qJ2ZvVmL&*$bf7-&E)ds8=;GzIhjxbZ9z4wM-?ARV8bxCljAa1pl)Q8gU zu%YaS?f~a$y2KJZ<-uwhqENZXRik<-OYwkxo}G1 zdjmJECvkQkoS2PW*CkysT0{N}e{Ejh$5msYhEp*>b;%iF7O2;@NG|bZ5f0Z<=QrWR zTLJv35BiO_X^~?e@F)Zz$G9iuvv_`K)~L?Yd=aO4oMla(l43!M<@Yf1s{ag>(~!G~ xC%`fH^(UzR0ka|>O#=i4D4^z(b0WG0vmHXyC6h8EQ~@KCha)2f)gS->0040pFXaFL delta 1182 zcmV;P1Y!HJN3ch*e-Q*cf*=T!gAp-*-w-fDKq54RAV@~pC)C&lc-QXsy4y_h_uE{O z$+&DL*=Sda6y;QRpRUuV%9ZYy$1Ej#C`7_pPqPh6BfugaCv4l(?(eSjKqEp@7Soh7 z=xI+Nv@iYNxy#QI{=w~rKV$9 zo|(~vX(Zw$lReF`9F3GL`MZSS2?>0y-xW!}EBiM*hO}$Se%CwzCvX`7qea52_&rOc zsDcX+Wh$#^n$nCqx^J!ZFsue|H=8tJP=)XKU%-Tk7$&2di>PeKWJErJu{DU2gAB?z zk)Nw8hdkxQS9n4xmg_8$5{hbnV6dl2O4mu6NWAGuNXV+_Oi76DJTG_zLU43`{ySof zB1oa25-+A`@se-kcUmCrW?bS++zW_nA6QYA1&vUGBakY4bk0znQN>)t(k;uf8*rzJ z$5c{fE2X%3@UVF^9r7&W>bz!-guO zts>4g$=1ARu#05nS`|U5r&K8>SC4_$lTi~QYKzb00XRGS{_g8eJ9HnA5pG6gJpW@z zeBWykudNSx0i@ZyCuzv(NiIa&I9ey%;`aV}Y>wXlRXqKB(&PwhGQA_N7ecT7suV1L$!Ibl*KnmS3}kWlK-yM_q<^L1g%CN* zTV3*8uH;;i{o~@?inL<4Dff`OH^VM>AyHHb!?Wu_+2rVZJZ0GPp$kk-@9Cg%$6F9CPgnsBJ zxmK%%-s*U1-!noN33Z%p0yXsD+2%C%cy*yG`?To?ZmRz?`~Lv5d=+5`f1)Kr+x)hvp2P_y@N0<1zh=*&L59@g1 zjRf)3M}6Y$w8*g!L=pqgW8AayIeUI-R;VuWVxHzjN=QQ%bUvp!`96%i>AwKgX~^Bg wGvJu}_7l|qvm+l(0|Y#RAPAFmBDw{lHoYe#lQSb!0V0!#BO?aXAOHXW05Z5OZU6uP diff --git a/summary_results.json b/summary_results.json index 51c753c31..dbe39bf61 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2496, + "Performance_Result": 10.2505, "Performance_Units": "Samples/s" }, { From c625416b2d3abdd0a485f89ee1a921da476b99fc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 20:38:03 +0000 Subject: [PATCH 034/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9137 -> 9139 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index eac218a4c..8b47d12e4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 20:35:35,634 submission_checker1.py:3263 INFO] Results=14, NoResults=0, Power Results=0 -[2025-02-11 20:35:35,634 submission_checker1.py:3270 INFO] --- -[2025-02-11 20:35:35,634 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 20:37:58,215 submission_checker1.py:3263 INFO] Results=14, NoResults=0, Power Results=0 +[2025-02-11 20:37:58,215 submission_checker1.py:3270 INFO] --- +[2025-02-11 20:37:58,215 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 20:35:35,634 submission_checker1.py:3276 INFO] Open Results=14, Open Power Results=0 +[2025-02-11 20:37:58,215 submission_checker1.py:3276 INFO] Open Results=14, Open Power Results=0 -[2025-02-11 20:35:35,634 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 20:37:58,215 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 20:35:35,634 submission_checker1.py:3286 INFO] --- -[2025-02-11 20:35:35,634 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 -[2025-02-11 20:35:35,634 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 20:35:35,634 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 -[2025-02-11 20:35:35,634 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 20:35:35,634 submission_checker1.py:3307 INFO] --- -[2025-02-11 20:35:35,634 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 20:35:36,592 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 20:37:58,215 submission_checker1.py:3286 INFO] --- +[2025-02-11 20:37:58,215 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 +[2025-02-11 20:37:58,215 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 20:37:58,215 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 +[2025-02-11 20:37:58,216 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 20:37:58,216 submission_checker1.py:3307 INFO] --- +[2025-02-11 20:37:58,216 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 20:37:59,164 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433783 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2505 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ace944308..b4d5aee0f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -661 +662 diff --git a/summary.csv b/summary.csv index 7b4245243..fc7960342 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433783,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2505,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 85f0a5beb..3bbefaa60 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433783, + "Result":0.430616, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b3382b4bdf2056091f57418537d443102867b481..a6dceca76e50ddfa470536ff0099126262dbf171 100644 GIT binary patch delta 2445 zcmV;833B$aN3%z;*arp6oT41+lide2e|>x>0;+_xO$Z?=xm5dv9CHAxzSTCleEao< z5Ii$Dv|BAvUlKU}&zzZK|LnmYcRoG_zHrZz)QiHd$RtG+c<4q`FPwM9o9oXlTNKjF z38#)9g}f_1@l^cS`>XR1CEwEp=b4Z@2-B{($g=oAmQ#1Z11FWDn1}h6S(F4$f0lnu z=5iV*+?kd}fiJ6yV#R zovDuO!FrFeLWiloF1dbPa)Tu%NVd^QEn9=k_Rv5Mtojs3SYg){eXMBL6~|b?01BT7 z2{^(&6B4j%z7P_C!U-YKf9sM{LIQTpfRKQFb4EzO+71Z`Sln|$0#W>7&iWhbvgMBP=oC@rETTkQ@^duxg*ML<6h#8A~i6`SL6w&WI<( z4Dd*6FdsZ`d%i-Q!GR$WA2?3Wq0NZc1z_R}EU`h_moIV8dME!}e|{PdPe*6(%VH?a ztD(7Kyio)S$L1bb3B$90GB?N zAm|7Qgbqs(bc6&PpkJ&8=m>Lg1vqUmA7FC}FnhLX4$hE6Vs=QR1&Ji~4~X~zOKf=h zw)RNmd@*s{;^#~&|MgP2YRz*V;%4k5qOb@y6O9QMaYG zX>O^ZZ5l)y?`VL~5tf)BiawSg5D5tc5law=gam?UH-Jc(Ll7Ix2iP2eXg7dJ%nm`c zA&A8O0aIRJi4A7HwEr9s_mRpDAl}-wOQL3|Otokd+cb%7yr%&mM_6KlL$r@2h(tmH zNyHLFB9>T?f5dhJiG(>MvB7-M2ArJ1fgv#gDC^Im&4}0qunxr%8^nFt-Y;x{<3?%A zcS;YBwz+{Fs7ELFOi~qz{d`@zj}&*QOWSn_+q$N2fsiRQh2=OXJHaeRgvJDC+&-Z} zAW|AgBB4Pj5*ozfT0_A|Ie;`aIT)-ru6aZwLn;$^fB8dJBf<(XNoWw18{xl2D#69@ zr-9r{db<$03T%y}*|zL0cLN!HtNoDSTk8gcNMentK_syTBS~lwlEkfkge2Bj@V?+$ zLn28`0&mnbIT$ebFyRc145=tBh$OTcQC4k;B-YqEM3U_n$>m*s8Ca zG>AvFiATZ#;<3TOV7>8$M>sMhCP6n(k=2N@LOf!PZ9pE?H^gIkjr`4bQ}>eKPDrX7 zNUEB&{kJ2dAB7$=;t8V$nZz2?fJ|ZyW|Gh#e~SFJTo!-_nM|w)EjJf1=9^UMF_u{M#@4Ig8@*Q(kV|MOhXF&)*i@ znewFgmj7^{MN#(rrC5^nTQZ=1QY-O00;m806zfdasuOB0RRAGv;Gg?4FyK` zAv+|K;vFh~+aYZbwgYX+s)0gXTDlX`BoVb9NNSz@edT8o$g0cHy>sTypkjTLP4o-a z39U+$B`JzPG0lbAm*~CT#48jzPZg()R#2i7IJB&$uly;8|W#4oUI8W1iEWuNMPCdmPFg7z0jY!UBr1`N;58w<# z0}_;XI7>2o_Ij|=-E8QU3#SzRWZl8;Tr1vHk^1TfIpsse&cOiWatAPg#cuTJ1HNFr{{Wusyxk?ahk_j zR_7@gSv-*C>Z==h)qe)camc@k6JW@F{Ryf+vm+nM0tL&Qq8#dz@*!IR>XS|)JOOl* ziy}b*f0NB3Isx&M2qPvNM)x5*Bmn>bPyzq|5dZ)H0000000000fPvDJNh3Z1C6kCF LBL>$X000004&rmm delta 2429 zcmV-@34->sN3ln+*arpk>AV={lide2e>uJr0aZfUCWMfbT&jISjyZr;e5q}6`SyEk z6M|<3hjyza>PrI0|Cuv$?4Lc@h5GCK!Ip>)q9E52{o@ZHnpsJ}i=Yg9lQOrZJWf~=cf18P~ z$xKb-gu9c%C|IakvuUY;>xXjb;UL+?M>L)K9zTw}Wx&I%bdvCen~Ct#xgV!;w>|O2 zwmcsx;nR+M$PRknm@TW_b}{#@&mYo_&r+VsT@-!Kzno4wax<5zNc*`6<1&#ZeCjS2 z*(iEA;r?u%iDff$6E9k%#a}7#f5lpAvJ|+F9a)#92Y-^ybH~!$pGlBqt z4LVZ;*@N{SVTA!xeO+?>y5t5+ERbxilS;NGneCy89N5(O(QR#?JF;AvrKb-PJ)N6d4@iZH!%e%l&^U^0B-0t?i zj*vj;umnLzNWcO5#jb&lFb7wF(;D+WHn#z@XPf5W3^^cXheXZCU}gWvMBN9<9ki9lxD+QGR@e5f5d&HvIB`X_6CWDqqHn* zOAYPNAUb$Q1B8yS!~#+DumpifNFaza5)w!v zmLL+b#D*lce`-i1%pr+2=KU7n5ie~{IXutH1{8U*D=_-~O)a54O8 zAor5qE<~;ZTO(<;Y-h{eKt|tcKV1=e;0_{M|L|9Ia&o)63G|H@@%)M+U?s=;kT18d6q>N33y7$fNd#cr31wzxi(JUJ~31No@m3 zO;@)6c4YLU&_hN%Vbmd$SYw%xNvy$45*oxLf7W0o2@PUWuVa#MfS9at&|hzSVG@oE zh)FP$&}v9n;j$HLaM`N==}hh;!JP_My;8Uunq_ZX<*HE9PwfvGeiCjP5K7{Cx&@&m z9{(dK2@Qf0YcQ0A20>}mK}k43P}Vr;uQ$Fx2}cIRBsP3VjjV=*73^ENAT*8zp)}qa zf0WhYb;5Zpj>jY|?(jIj^m}q2Dei#f^(wFuOH0uVd*i%XrE*Jm3Bw%&rU|LU8q0=M z5}gN#Npmx!j=_htOay2@T@1t{sPR zfViw{$YH1>2aODfNidhFDTlDaB`ndFgG$(a_T~Ikt|YCZR!K*0tjtS~GX%O-IWGm$IjrcVzw;i`|rG zD@&8dY56XP6eoU^_}NoAd;T)XWpd$mgJ|-;2x=MnKbHLcJMW!!@XoV7{4Hr8q>dzV zf0l~uidpgxR9M}1?9TYPo6P($l@@#|5^XACt&;NXkm7R|#lj| zB>yIUxKE=fd;XFy$@(pu?!U9@5B3cOvmHXyC6nbHDt|vnD8hE2Em<{Cs7p(CLYgF^ z)&ohcBo7D!a7vQ1*QjRcFp}qHpVC>i)By~w}s32~&mDGpQ?y#ZkhwcF9 zX}ZJ`Jb&fXQ#=4;GZWE-qfL3xL>B*SN~2P@sphEch2O5uA0H?1de zb|0LWja}CzT`^ii{tSO@Uf;)6W1)sqF+g?68DSQv*S0S4Wf2b7Qs+0}#9IOUsSo;% zw`q}MAMhvyAjh~T<+FHxY1XLD(|i%9d7NcUo*9y2L5k(~F!HMZ43yK5yNM^jG57T+ zsQ&@8B_GNH1@q~=80VApAzK0GlTac&0&x+OkP#M>B_a@$v?3Y-e3Q>2IsxvJ3L_>P vvmHXyB>?~cPyzq|5dZ)H0000000000fPv4GOCvr3Ba@0FBL>wV00000(xPq; diff --git a/summary_results.json b/summary_results.json index dbe39bf61..36136550e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433783, + "Performance_Result": 0.430616, "Performance_Units": "Samples/s" }, { From d1af8d93587c26a7fc7c3b708f2d155032d6441d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 20:44:14 +0000 Subject: [PATCH 035/719] Updated results summary [skip ci] --- README.md | 49 ++++++++++++++++++++-------------------- dbversion | 2 +- summary.csv | 1 + summary.json | 50 ++++++++++++++++++++++++++++++++--------- summary.xlsx | Bin 9139 -> 9233 bytes summary_results.json | 52 ++++++++++++++++++++++++++++++++++++------- 6 files changed, 111 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 8b47d12e4..a5ebed261 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 20:37:58,215 submission_checker1.py:3263 INFO] Results=14, NoResults=0, Power Results=0 -[2025-02-11 20:37:58,215 submission_checker1.py:3270 INFO] --- -[2025-02-11 20:37:58,215 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 20:44:09,464 submission_checker1.py:3263 INFO] Results=15, NoResults=0, Power Results=0 +[2025-02-11 20:44:09,465 submission_checker1.py:3270 INFO] --- +[2025-02-11 20:44:09,465 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 20:37:58,215 submission_checker1.py:3276 INFO] Open Results=14, Open Power Results=0 +[2025-02-11 20:44:09,465 submission_checker1.py:3276 INFO] Open Results=15, Open Power Results=0 -[2025-02-11 20:37:58,215 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 20:44:09,465 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 20:37:58,215 submission_checker1.py:3286 INFO] --- -[2025-02-11 20:37:58,215 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 -[2025-02-11 20:37:58,215 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 20:37:58,215 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 -[2025-02-11 20:37:58,216 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 20:37:58,216 submission_checker1.py:3307 INFO] --- -[2025-02-11 20:37:58,216 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 20:37:59,164 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 20:44:09,465 submission_checker1.py:3286 INFO] --- +[2025-02-11 20:44:09,465 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 +[2025-02-11 20:44:09,465 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 20:44:09,465 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 +[2025-02-11 20:44:09,465 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 20:44:09,465 submission_checker1.py:3307 INFO] --- +[2025-02-11 20:44:09,465 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 20:44:10,409 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,14 +24,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2505 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2505 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index b4d5aee0f..5b1d0e2bc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -662 +663 diff --git a/summary.csv b/summary.csv index fc7960342..fc54486c7 100644 --- a/summary.csv +++ b/summary.csv @@ -2,6 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3bbefaa60..b58d9111c 100644 --- a/summary.json +++ b/summary.json @@ -90,6 +90,36 @@ "weight_data_types":"fp32" }, "3":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":18.5738, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -119,7 +149,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,7 +179,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -179,7 +209,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -209,7 +239,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -239,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -299,7 +329,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index a6dceca76e50ddfa470536ff0099126262dbf171..f232f79880ae9bd48c6a17ccff31fb046ae1b351 100644 GIT binary patch delta 3634 zcmZ8kcTf|I&u*DzmkJ66DwNiuGG!0hm5m^)ML=jlKz0iXGRj^uTc&_)5M-1M3Ni%+ zWeX^K^CE}{%3BBi@Q?4_yDyhNlFKERJb5m6Y0^vAr_M2RGGx1Jpg{nDrXm2q2cWRQ zp`^t;Eb*g0cxd^=;;Chje)V~SQh{s{mvw}~V9ttY%gqdNCNng1dUUE!w~F+Q#bo2g zs?4iW3Q4Dnveze5-Q8{DE?%U+)4uJuhIZ0(Cc|0P{7b7YjH<4^`?j4W!D_G@Z84Op zy~JG@2oKvk4fq%QYMH{gU6rN`qNKFzPmwkw#8ICknN8tnBiLQNyXJyQ?%(B~jJ&zF zHQFrA8iKXrvEUSx;xXc4K2%pP`S6xVGuWu3gH@DMa(6qPv}^8zKLoH*~q2rw;em4hs{ErXdR}KLV}rg zDBdai(9GYFAV4!Xv;lQ`N?MtN@`qBMCiF!qnq26>JFS{jsujAK``Q>l8JCB3Jok&_ zT(&-DUG~H;X64l7RHqlFq6+K1HX3Jf(J0ehf3Ec%t@uYUayg}t|9(;RhvxR6PqUrM z7xOy3`7{(kSV7v};)50)n)-uIlC_-{#beHwUckEzy5Xsor&V=aWGTl~t`aL_y`xD$ z>tn14Ci>IW8?kBOw00!o$BkPoHl_XV)99tgWvuyxKicrl)?e{fDVo{P0)utPLrz`>_stO|<2~r@IoJEnWr;hifSk5eN5n$8L z%rv*-_jf04@)D$Mq&1@Jg9&G#RIe!Xg4qn9;TGx zIzaf{l91L|mg%Z<^>m+a)=F8@x`K~fcX?|@%7jtnQS4}+tKjJ%K7l26q94K_BoZ<1 z#1fg37S4A3$ckRbiPJi~33{Zi(6(~$qg8nYr+Lh2ox;WjG049ty=Eld5aH}BufGu@ z88BXfW!GW>NWR2e2>g(2I_HM7|m z9Q~Wur$o)a1vNWct{RoxeQMm|2AP*j@CvCQ0fo=j7U@wFhm6gG^Vj^m&&p$`7sT9YrmL$H3T;$++Rnxk zgKBs~;h0e#$d3y&v@Ze5mEh01JH>u2CYI{P-1e}Dg4f1J8g=Wrf#WimBK6(Cq=aau z;`RwM@mrk2NdcTc@}dL;%}2s_8R!yfVxxJV%Om6zFj;!H`~TpJJrRSUpU= zMeqph)G{*RE0V9rCp~i3EFdsc)Xmz#2X2dHZ`x+;P&9rwT6c&!CwI8H0y{ z4QhJzA<$^Vj3c69a8iVPPl&9G`a*Lrh&tU`VIxyrG?N=S(xa_)`dD4+GX^8|tKY#p zo{yaGd;i#=2mkOQkB7@k{l?a^Q@V5$-L{*%&tY>a4>H9=U3U!+Vqr2Q9K}3J4VUb$ z4QpUJIs+sX6BUGz^Hs!`iUc79g<&3o$BN8%0;479OU^Y{M*Ib3D}B6Y{!Q(w;uyZ+ z#_5(x$_=^&HUlp<11rjXjY+5Dm%q9DgI8lYa}oEaC0--U0pU4ku$&EI41o@mISe6I z0W#S2qHg7X`w%2gesF=~rQ-K#)IRsCGb_zjf2MLh;XPdtn=W$uZne9YOM#WxgN$rx zyK9boV8a?2rv#SbUNZ=k0XgF+#=;Rf`80~`8(vi7H@++w5&>ytk2m#X)MlwYYg8Qd zqD*ZbY`9*lg*9=g%$I8VBx9QBs`g!K@umXz8Vuu5Phz&lxf@n{d;PAnu39jU?T?2G zVB&P#j4|;#Zqk{-8EasuF|t_8a>i=ij1!z0@J)%GccUl^IKdM3HSrz+bT$7a1e)Kybcp+G+oSq+dZC&?o?u%;*sS;@Bdj-V@UP z%NA1n$n!{{;WL`lImh)TKK@ISyBvNASBPho`yegKVliIrO51-{zb0qx+coSzFX)Uk zm2DA}*I zMC-@iwj*s7gL^BjA-y~3Cjoph7E<*IgkN6+o=&rEYkw^bnHW~v3AM;v$W;}n50Nap zX|%*mTfe4i64YEG9h#q6k6{^^dxVn;y%IdOd}vEERgAAVc%)UUeK3^RP1=5GCCjzc zu@%?N_$_LjyK%0t9P5vkqA}mYEcTbnZeK7}JL%K?LKA1ROmAGSZx3;Q_HX`;ku{6^ z0=^f@-FehVw|!lQ*MH?*Vr>#LEbQK}Xbhs&{*-Fk<;nrFHkI+}&C>$_l3bJ|Zb6cl z&zchWG_1RCIP}BsG#gCN=Q`$`=1HX3ac?OKb=KGq>Ebi*q^2Y3pKFW_1X3*Pq?S`j1fmk*y*Iw~RI2_jg zx22nL#aMP`n`D@9m+*>9m!NBOH{t1&}V|B=Q$riH3C?o7)!hBDqTdC#S{ zH`GizhEKe^D~c_7OSY*oBs*Tvj+6Dy2MP|cP|uaJTCF2GcMI7Z@cM(oQtk!D`xDp( zbF|qGi&fH_Z>0wcKfUnXv=n&5)Gb+-&VQ@w+~hhvYoQL9Dw!gX*{i4Hl$#@C8$c!1 zFw&8KrI0@?4eUMW-1u*(uG*LvcUx_0AZHo26DeP2{D#p(;P$6@((R?CJ2&?v{k-~{ z1oP6)t@s;^$kNTSjrJG=BPIx5scA3f>nXH*T{+^j&sN{z6qG-=QdOa`_G>L1vwe!d zUce%fnP1ixKc^|_A=f&xaZ#RnZ~IE#Lmr>DR}K50Qro*G<&26dCJH<%--oved@-d@ zarPVRVp{O&F;7BzlSFZ2fjT#-<~NnrISxObshC6jeKUz;;)h*#NY-nWw~ZE|#6cL* zO4E<^X?vcW*>-WA&%W<6E8Dz#9>=|)la4>y-W=+;poED9#ybto>T|JpeucZAFnU6W zs6XSY^OHyPso58E+Ejk(S=0rRyTqwze@8D83< z^k!fv!DDAucD+ZAZ5oblCa#BN313EHG@KmKW&P;vX8O`|MT!SOJol7i zrw;EVYuBwlcdR=Jg%AWw*KbRT%u;9%6>_f*prtn56zpqkO6SX=sh&2;reTiC5k9Iz zTp~78hfVLzOS2CA)il1)wc%TA#cVba<^B<^L|AhP42L6@@uiU zc}Gv1#4OE@9K(?6(h?XCIZHP#n8a=#`Svg=Im*UWyj{2##a(uNvsHT@EdFZlNJ4C@ zX=mJgfv*x1U+zJ5`IRRh&ZhmD8rj3Z#(vT>ZK~_9$*L~0pX$LJzQSi~RunZ9l(kM# z47WJF*I$&coXH-{W>c%@M>y!rPqLqNS@fh1{!%`*>+0lqR&nd3axHxKmhZ3-)zMDh ze(eYjt8-DWSxZoa46Li~+AMKCN(vPw^M$AQr4A46{S(xs8=#dqpTXn)q@GQCn`j7o z`6K{g7@h>@B#OOe`pQ(^|8Mu$|?s8uy%@0Qx7Vz1Zs z+N*S+R*Ux2_xtf(*LQxLah>O!=Xvh?y6*Fw7noYjs%ct$f94yWZCU_;ECT>=0001- zyF{RuubZEab7|P)T&1E*<~%M&r|H z9zQ^7>%^_b(9^>hh>{Gl&H`cZ5Q_XYoSAO-`*X*&KxJi`R_|fb-%mO#C_?INxGfDM ziOfKJcgD^z!h&sV09^E(H>UZ9@}BY8^l9mL2yAhjdNZAbj#6`SC&ehRgG%J( zo3z*;Ddx$S^iuG;8@^m3^XVxXkrJf|wtbPSF}CZ=d^6pevKc|*D$nN~+CP-}CV4OR z?(dGrTq&}xlsVk@P*+y%bh6qbUOi}&(oPP+XZe1$NL0z^&WRCt`CjxVHdH<^sQNv&!v{?^i-_KOU64fglsD3|M1mycdpbKa z^n$8K-Ykia!|7X1GN$`b4t)bJ5k}LqO$aF~DU-ZuUVY_K zbw-!ZY$1=Y7Q42>J*9rubBNoL?o+L90{MsY%vw&fCtrMH=(*9&F5CE+O3r&J!0yh_ zsyS`J7>X=PQ*iAA#|?kJMggHR@kU`GS1VsMragEb3JpV$3ra^nxl(|1{!iIyC`TsU zhouP__u1d-COjxq$Ra=g+!35utr||tfjQ|vuv9RT_qIxw>pi#E39I4NUti6BukZ@23hf z4UC3~nf=3XF*{BoMogS*VPtt}c z^ZnR|^AE*|bTRH?+$4FGyh2)4vpORAS$L6p!?}ZT)6~{04Df2yY^sFMaIna4%Wy7c zeNCrt{kEcmrMZo%h?Kx2Zff79O+2A_tXHL5DP6bO`y;6tyGnXmG$UlSK#hHnto&(h zfygGWjg)u-Z!9iY`)-5BJLK%jO4t|%ne^99W{Y_8yq(9*By{ddnLjqi>eQQ-#B;~U zc!~c)=ml-y!o|Y3C^9R)=Z|==udCk}M&J1D5x-NC@Wcf6INvRy(Wj7%avB{m$0dI7 zQ%d0gq$;QIZP9_FnY1DP(U*8$KP;o!qVw~nMtVr{TKh3bhBV+^n1+aG+W<^Hb-d6K zbKD6?+a33_;Rv%9D&;f3vZ-_9_n~2<>!9DY)grjoeD@mmrOsWKM(x6FMdG_e;{d+B zkF$IRZwne+JjuBpTx#6koFhh9QquSmE?h&v;7cspkhtheQOKd1vJ81$N%NnTKAg=a{Qu?iEm-HUi*z>u}-}+bWS$jqqzvkr)g#zg|174 z#i^-+?sh@d1X9e1T>(__(egso9ibyMnVA<%NdZN7{>xMqQ#X{mD{N*#!aZ_T!=*n^ zi#f5OG4g7$rpU35A$$CJs8v%UTOe6`?h&4DuBp%~l6YMl0UO5-V zXUK&S$VG`lxw=*!pgS<~5wf@BIm+ZJg46`jVyw6yXj8#Xda1k4kYw)VHg&O%?%kru zdF|l$kVbmo8=0=W2# zG6cEp&YE(~TqI>rtSSK`Y~JMPwfrKy&!4hO;6}$NUARMEtETymafT$C5rTwd<238z zH9LVT)g2wyEqgJVNNFKqlpJ3F#Y9|<>V>(-oof?vU3Njw&ISXGj+2)3FeTiWl2>)- zZw2#sW3EZBfb(e5yST<9bZq}}Od8=6uBMT>$c6NPl@v|YWltb<1g|oXyzMbMQR&ft zph+z^BlMeo(RcFY1%EYu=UU%sa$nbNRqTFjQn)LgvP<$t$KJz3hsSm4V0$OLs6R-W z5psoO>9ktFTAjeCntvmu*7_4^^%C0E{-!{g^AX*Y#CrMvt(VhaH0z1;e?W2hg48p)y#AEUpk+l)=T#q$Y%&S>~YeX9<0nj!iF8a zIQIBM!@;uTn|_*GD-bT$fQxQmov8&GP2+cwLaOc;WHMEkgAtkueO2E14;+nZy7JNN z;x_ROJjR`0Yb7@EtNY~v+mh2IdFlAs6eeK9OzDSN32SH;vdTPmB(2IKcEnmTaBGTR zmP1FC(C+R!lgq;=nei%s<1<+vEr@o79?U@xHz1}QI&TzdBO5o5GN;(As^K-mL9Ij7-b?+jWUUw#U%3%+ePz~osf$BAA05p94xOu%n2^i(dq zr!;$u=c|;+DGU&P$(dZwY^>cI+f+Sb`(8oeT@96EzfhzDAJdk8{;v2dFzk`>Ahubj z<{%3T(OpbMA<*A?lG^Z_tuo9DjT^CT)Q5x-_Wk*sf}ntPcsH$B8CE1<<81!!1~!de zIcGgy%&^t;?qLAthy0Tjfqf&a=e)Z4W$NL@%k$~``6nsAy>rwJj?N0qk`66ywcvrZ z>X0-bo?7(v#tanzP{0HLu>H4A@N>5Db+9+`_jU1f^7~UGwA`QbT){xX7Au{D9y(qN zr4B{4(*fg0AW?jr4mo8X#oONpyq)^TF{y2FBT46@gG{f+d&7LuqNY>-EM=35gH|QS z>oaJ6=+Mts9%fv?%X>(hxz+%DV8JODo+YU!tuPbyFa(qv#(6f$stT=t8Q-V6)4Umf((iVL<)`Zz!ZNysn-GQs6hQ_WKS#71j}Ckx2jS_I%wr-@bPd_EazK zn+qWuLvvgH-Lh2vZ6l`O4InbIl#T-d#dd!)7f@%nj1o06S07Jy{Co7aTRANXe-rn9 zCLc@?3+?$064vRGygFs6=727=+Xp`_M*I2{mx%XebZCcqY$JWBaKXeo_klPYwJ%Ft7xt12btbOvs>R&XB8R8J4bC2zk%o zB4iS?MeY9j9yKU9IW_oKK3jai@`9A*BWFhLlBbt7GCh*YgGQ~d^!C+@MH%Xo4RszT zw$2UAeQfSd5vUy}IT_-X$ra9w&F@ELs<}>o*@UJpH|~lUFHjV)NgF6_Qzu;Y zSalNTp3$%3W<4=9;kNCAubD}XZKzaPInt_Cig~3GSdnE)wcy#V_!qLKoAa2J4*AF(9Q7=?lhcbbrSU_xImQ*0({D69wiO+1kj=Z3CUay~ta?}aj{5w;3PNCx& zskVy;+Zvg+GeW08>G8@{5I)iyKW}sA9jV@!u#K=>tAl7}oSZCk&91fOhma5|#hZ96 z-M2K^1tak0qniX&{eJjOMdMYNkyB}}lE&)#+fO(Jp(*bIaYp-~<2$pa5&8In(Z9dccsWWo)4c+Dup3a-d0nmQW41K zqkD){;!dwprtSv4&nT^{Y=(H}#8v>g5ISeV`h?d^Z)t95IhI)Z=|q0J7~B)6kj;yW zoU49-I56$e4yj=AC*JFs`FJ1Aob91!muPXNsYX8|e5sK7r;6V$bL~944IN(Nm|sx{ z2mssz0RSu)Qef|8chA?$+fTyI%h%zL4rC?uyMB;_s2`ux=d~c__&p#ER35!0k*Zb? znpnq#Sn97~T(i!?)}5e~lK#;azm1{SwqeP-2Mm+oa;;?8^@KZ8gE6zITP+J+H`wI$ z*^8c;-h0D}tgHA+??^X}zl<%vW!(a}LQV9IPKRD?mb+xv!^B6{jzUxW6qV;ICkG_{ z<*?|_jMN_wfp-|`-R{0Bc}s@n8D_{a|3|eh^AaPP3Yr{<)RqoC3a%Ra$9?N1LlN6A z`=!E1P^aV3_?hVjk1_FJH%iX(@RxpzHdnZDpcqZD7*cQmbU16ON!<$XT;|$7yOoI( ziBQRo3SafGwBMq-76Y)4Q@qamq-ZmkP@*)L>hr3{+;sE0xe{@GyOcX<0#DQDLurd= zwWfrfgG*Fu3A}wE`v3jENcROyLEFrvN&$ZE|5w`!l>coJk^ZC|0Td8VLI^5=_O6gz e1Ytb?&&U6a$F7n}1aAXjq%A=~nkl|N8~*|Jw2?0W diff --git a/summary_results.json b/summary_results.json index 36136550e..c2b4a289e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -187,6 +187,42 @@ "Suite": "edge", "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "undefined", + "host_processors_per_node": 1, + "host_processor_core_count": 3, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline", + "Software": "onnxruntime v1.20.1", + "operating_system": " (darwin-23.6.0)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 18.5738, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0007", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_macos-latest_x86", + "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", "Scenario": "Offline", @@ -216,7 +252,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0007", + "ID": "5.0-0008", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -252,7 +288,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0008", + "ID": "5.0-0009", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -288,7 +324,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0009", + "ID": "5.0-0010", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -324,7 +360,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0010", + "ID": "5.0-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -360,7 +396,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0011", + "ID": "5.0-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -396,7 +432,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0012", + "ID": "5.0-0013", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -432,7 +468,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0013", + "ID": "5.0-0014", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -468,7 +504,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0014", + "ID": "5.0-0015", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From 81f9df143c3b6a2834976d9b74c22fdffedf5b68 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 20:58:06 +0000 Subject: [PATCH 036/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9233 -> 9232 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a5ebed261..372af6b1f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 20:44:09,464 submission_checker1.py:3263 INFO] Results=15, NoResults=0, Power Results=0 -[2025-02-11 20:44:09,465 submission_checker1.py:3270 INFO] --- -[2025-02-11 20:44:09,465 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 20:58:01,804 submission_checker1.py:3263 INFO] Results=15, NoResults=0, Power Results=0 +[2025-02-11 20:58:01,804 submission_checker1.py:3270 INFO] --- +[2025-02-11 20:58:01,804 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 20:44:09,465 submission_checker1.py:3276 INFO] Open Results=15, Open Power Results=0 +[2025-02-11 20:58:01,804 submission_checker1.py:3276 INFO] Open Results=15, Open Power Results=0 -[2025-02-11 20:44:09,465 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 20:58:01,804 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 20:44:09,465 submission_checker1.py:3286 INFO] --- -[2025-02-11 20:44:09,465 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 -[2025-02-11 20:44:09,465 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 20:44:09,465 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 -[2025-02-11 20:44:09,465 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 20:44:09,465 submission_checker1.py:3307 INFO] --- -[2025-02-11 20:44:09,465 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 20:44:10,409 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 20:58:01,804 submission_checker1.py:3286 INFO] --- +[2025-02-11 20:58:01,804 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 +[2025-02-11 20:58:01,804 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 20:58:01,804 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 +[2025-02-11 20:58:01,804 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 20:58:01,804 submission_checker1.py:3307 INFO] --- +[2025-02-11 20:58:01,804 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 20:58:02,789 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2505 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.92319 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5b1d0e2bc..6b190af01 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -663 +664 diff --git a/summary.csv b/summary.csv index fc54486c7..4569a8aeb 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2505,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.92319,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b58d9111c..4061ad464 100644 --- a/summary.json +++ b/summary.json @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2505, - "Accuracy":"acc: 74.600", + "Result":9.92319, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index f232f79880ae9bd48c6a17ccff31fb046ae1b351..68b9e872008169aee52e81787cc14c853f384b2f 100644 GIT binary patch delta 389 zcmV;00eb$CNRUXd-4O(w?=j7@;SpH^0pGJb6aor==Pw56Bmn>bPyzq|5dZ)HWN%|o za&K^RFJo_VWiEJaY@L$LYQr!PgztR{qq{BHAteagfwtt-K%pruy$NaKh+02LY8}6Q z6(?~L@F~aD%zit&s9f$%x8YcMjYDXeEAq5|i_SdxoKyVB#DA2*#(b#oe z*3AZM%%9`0_3QhvYAW^NR2)!U3QoBL7L2P$OMG3#!?nzZbv*G_f_UnqKJj*1*_?pgVqJ-=*Ls4hq`Pf3yH`KBOrzM!P|K8(ESzW~)~$lb&<;F$aN6V(5+gC9); j1f1_N&6Ct3x&`Mi2IwS{lOt3Cf|KYYBL*}f00000&8)E_ delta 390 zcmV;10eSwANRdde-4O({q!mZA;SpH^0oSuT6aor=oMf$;B>?~cPyzq|5dZ)HWN%|o za&K^RFJo_VWiEJaY@L$LPQx$|gzr2>qfbB_xVK+$?*WTEsuH4QbyV zCu!4G>WSleX1|?Xxm@g2oBV?F(&!5189@nXZiLiLh2Gc8^d2QXFfCYXG*oC09xdvd zl3U7uje}QbEVv-SClRDQ<+eiG5G=*m^DQXmv*^%K+!&`=h^1?=W&E2pfD1zIu!6t@ z3k>(b)J{ZnKngw~?T>2*fZ!NfP@sdyd6wgoH-JQgF&0uwYzBUE=d19Q~+li;R6B zk{E!FagWN!?CE*EM0G)m+msY(p05i+=W|LPzWR|D{U@M04EZ Date: Tue, 11 Feb 2025 21:12:41 +0000 Subject: [PATCH 037/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9232 -> 9231 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 372af6b1f..e10fe5839 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 20:58:01,804 submission_checker1.py:3263 INFO] Results=15, NoResults=0, Power Results=0 -[2025-02-11 20:58:01,804 submission_checker1.py:3270 INFO] --- -[2025-02-11 20:58:01,804 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 21:12:35,964 submission_checker1.py:3263 INFO] Results=15, NoResults=0, Power Results=0 +[2025-02-11 21:12:35,964 submission_checker1.py:3270 INFO] --- +[2025-02-11 21:12:35,964 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 20:58:01,804 submission_checker1.py:3276 INFO] Open Results=15, Open Power Results=0 +[2025-02-11 21:12:35,964 submission_checker1.py:3276 INFO] Open Results=15, Open Power Results=0 -[2025-02-11 20:58:01,804 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 21:12:35,964 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 20:58:01,804 submission_checker1.py:3286 INFO] --- -[2025-02-11 20:58:01,804 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 -[2025-02-11 20:58:01,804 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 20:58:01,804 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 -[2025-02-11 20:58:01,804 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 20:58:01,804 submission_checker1.py:3307 INFO] --- -[2025-02-11 20:58:01,804 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 20:58:02,789 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 21:12:35,964 submission_checker1.py:3286 INFO] --- +[2025-02-11 21:12:35,964 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 +[2025-02-11 21:12:35,964 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 21:12:35,964 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 +[2025-02-11 21:12:35,964 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 21:12:35,964 submission_checker1.py:3307 INFO] --- +[2025-02-11 21:12:35,964 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 21:12:36,974 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,5 +34,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2127 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6b190af01..7ae8d441f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -664 +665 diff --git a/summary.csv b/summary.csv index 4569a8aeb..7e1c42fe7 100644 --- a/summary.csv +++ b/summary.csv @@ -12,5 +12,5 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2127,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4061ad464..82bc28728 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2127, + "Result":22.9842, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 68b9e872008169aee52e81787cc14c853f384b2f..63ecebd22f3dfabe4d41f219b1fad6d1bbb95309 100644 GIT binary patch delta 2510 zcmV;<2{HDNNRLRc*arnxzTy`|lidd;e_L~!S`>ia&#z#ZKD0AOU~@MU?IfmcVvMoo z(m78K!L2goVgYmc_lJuyveuH=p3WiDmqa(;+Bf)Q1B=6tkAW}T^EmOMup=@_5d|K) z(bNm)9r5P+bITTmBz3~6<3}Oyh)+BbKX(5*e2C)j$%6A#$SMevj<`tE<$)|Gf9`?@ zP9jB19%fr+Q5-mF_BEc%$uj27vied>s2GtcEGk-G|bn3k2q z+;`Hf`^myvCSvz+>Sf#V@dz=Wb;Kif(EG-0QSKfVZQpwQA=&yYFjO%xuyX}ZWO z78c%g%JYd91$ULCQShzY#MTv*q0C&zGKFoT+Y-|iQ#JDa@#OP$j9j)_^zg(@%eAsa_Nd-q-KP`=MAUK3X^f8-h#+`xj{ ztpYjg&!$r<-6<9IL8|r zFxpp%>sN`JSBcv?qB2sQ9;uGv!F-P~p~IoRj$FTv++f56gRQNkGFpQi?U8{hu&PZQ zW5TWyJxsK##0e%CP~j7Se}E(GGl76z^Mycw3a133SCKOU0lTJ8AYk8|69|~w0fB(Y zy&w=UyF-kqV0JGF1kCP;K)~#NB@m51+D!UrGx6aHX6+aw1{iM-BPy7+69NIXe8PwZ zX6-XZEO@{@Bg8M_DbWHv(rUu{&)c5QQ0K5=hq-@d3v#u5YcJ%A2>=!GS7?Q3j=GHGw zMLLU?&cb^f{>Ed(1l{Oi1W89AP&$kt=?DZIpkJ&S>4)_0BVL>B_{R4V@i4hwhBdb9p=ZlHs=9e?A?CPbgsx{Aie_TGLQS2@z_o`${O3Pg_ zRo}zJCWf+yi8t1kiMlPdO>@T@+NMRc@s0)x9b?1u4MimO4>(_6V#Ef>$o_LgY+@*ThVim*w3#^n;7CA5{}zj5;9vdEX}4J zWC|@|Q4TKkKqJQlV}e7!M=;1kih&{$3^I{mkcu0Ie{zwk0LoZbL4UJxqa!LAP>nz# zQPhx#LYIaFV}sN3Xsi2wtdxz6ai6Qjb;ma6#d;Hf zU@r*P1E@PWcPDj85CfvZv!H6)_2mzcp`vN!D|Sfl>D zl1&V6e=j9jRY|jL**k91Gx}lE5yQ`K3>7Mg8PkADVg@ToFi1(vU?nkQ=}<|vPD!E> zc>lDng8pXXi;}QpK=rZ>m4s14A_}cuDaL|Is?937y30<4tF@EB$MilZ&XxH|lC>rE z7mRFTbo(q_*Bx80T_sE3xpbwUh8{8EA)^L;f5ePwLLV`MeIyvJ%sw%bZ ztu%+Klk{`@BZi-a8#;87c%p7WCy4|AicNXnWD`kRd}Qo@n}(TJ@;DPh!*h=S>4msHI*l+t)>QdSqQ zQ_h#!@fat?Js$g)t|yxq;vQOFuRB(1e`!jpZf*UpR;}EThQo*?8<>YfFeXgIAsB4s z9;rA4gRLYOWMy484pjlNvMwKop@bY%G9VhkcV1CW4iSZyuLOgZuT7@q>|y2ax%K)+ z6LrRHJ5xv1B*oBoWXG}XOdZyK@e=2G7%)|bU@VxcLonFOJ+gHO275^`$jiEfe;uj< zjj~J$8+A}zMlwgl+9Tgx+yK+ zni{i%+wujY7d&e)3a4J46Xp2j>rkl|&V}Sh`J$D1(EFw;d1dZwyz;q_J}r4iXjy#9E(7kOG>wAi zZwu~Bd7OXKL}3<1>GPL-N;WUQb^iskMG*K61evWC0Fyx;D}T$Op$OZ7w&c`6p(!oB z32EbqT0cl?oxFWz$8i$ysmImKemlFUTpd)K{DSk+=n9c6M+s1&{5nOr&x%kYp`Yf zn>B!oJYQl3fqw}W818|oorq{e3O*t2k81~j;22s^po7OGBlzMC;FO$boBX9aIKs65eH}MQO z<-Yv{^&hi>AIbs+R=(mFM3Ye>TLDCqts*=D&XejQK>^#7EF(GrO_OIMCIp$S766lx YBNhQJlc*y;0fCd}BO?YgApigX07pr)RsaA1 delta 2511 zcmV;=2{87LNRUXd*arm!muWvmlidd;e_3uWjVA*J6YxPU>_I zyg476ylxmjHwalA`Rv_ywL|$ncf3|ye?gOL*l+_I zZnqlbY&@Gmsq~;!GzO(QhA~#?KypP$SXFXONYtw2hLC{yxFsa|Mp6k^qr=sZ12E7r zRu~}KR~6Tk-658!V0JGF37FjxApx`dm5^xk)n?LHn~5)1Fl)zHVt{yiSfYYiJ0T>% z%O@<+z^r}75({2%&l2Jn@s#KRUTHPv{pVxPE7Un`7!czFSJ(x#e;E>|0CaqbB{m3q z^fK;Q_w=99r}5xyc>cafhLS8BntSPDbtkz|AP0W-H^?%b?9-CRvxyhZc%0>He3J4c zog}Nfz)SMfCz;$Ek|L)u=S7kw=ed-8G3c%BW1eIwOd_vynbyYh#9c01?sCBhej@-(lCCM{ z)-O#3I*W$R!e<=*#$$;IhS9?k1RWuP&|wLJj*x&0^ovyk9bpbufU_F&J~p@D@A|em z;9@}Z4pxo}3ld43A28xeEU|%PWHm_Sd@*s{{C1|5-My5qe_Hd5$K_KR#qMHquS%w* zwA>X_^*u;zB9uKyys@@O)NQG4nmcl6n+DOwCmJAhj3p+xLVH+(KqMp(L@Yrd5)ufa zT>~Ov4neFj?_+ZWqFn2dvkZSYiXo$o_LcY$B9BK)kgpOQL3|Otokd z+cb%7e5L^)f5%v2f@`dYC5S{q0!hRYL?V`00HrVO8WIU}z`|LLdA|)fIfo4cq62sv z18s)b#)f$)ApzqawVQ=4aNHR$2$?Mz zmS)ooGKGe)Xa~1?pp#=lV}eV+M`#d-lm>!GXb_2ne+Hqr(NHW>4j_zm4*Hve8yr!~ zfa(MSiL8c%6}mMfG&We4M_a@HW1(y$#(nM<*8|(Ejcr}icdU(!e%yY<@Z)uZK`ybz z)FGExgSjL$h)d#eKjIQ=Fqf>3OTq!-vc^H5!3~9T)H0wVfmtcC8WL8RORT|MvNz2o z$WebDf5|4o+Y3onMbd0r_Kt`2jNWWIVtD_?P$807V;T@itiebU8iXX)U?j1|(jk&+ z9g;*R@cC(-gZ}2=3zD#9K#j5uk%U%5!U|=tl*WQcs?8#~y2}>A)tX7bV|pJHYh}Jj zvbLoD0+CHbw@>Q29@si}l`MT{>Pqj19x-B*e^G-xVvT7+9aG;lX# zqak@D93UQR9P~E_UwDKq1ELeOr-7`7gcatI(7@Ztt>HJ%W3fj5=1q=90^AEpZ3{_N zlXgGSVf4QJ5yN}oh7Oq|cGNA%BvAl>m?SiaNvy$45*oy$UdJTi05MtPpuaiz!X#`N ze-NEuCZW}ku)=98*5I^N|I?XlBEY>8SG|(B>Z)mNWt~+hca)bfTryx75K62uVTlQ$ z!B7$!1SQsBC=5mi}9YTY-Bs7T2f4YJl z$^qiCu4IS7ikfrOGN5J+afvE(2rHbz616!fg>ASv=l|jIjh5|PvF*B@yn;?MwH<{! z%-#VeuKO@x=?VC@c}!C>xDy+dd)n1lv_Sy#YAIY406mGCfFVRMdJ22>|#O9WNw z5LOsWqE-ii*>JJWYheE8J&`wBf5CIbswF(SV(J!TQr>z&DB$s&_qgvTf*WOXR+?^F z%eSV+?BKS1!RSSvbr^+HFRzJm{PK0E)C=cA@}qpw$~@?OQ%x8Y7b#wiwJ=^AVq z|6vW_f{;h7ATWQy0>cMjYDXeEAq5|i_SdxoKyVB#DA2*#(b#oe*3AZM%%9`0_3QhvYAW^NR2)!U3QoBL7L2P)d|kxDwakZgJn>e7 zc Date: Tue, 11 Feb 2025 23:36:11 +0000 Subject: [PATCH 038/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 6 +++--- summary.xlsx | Bin 9231 -> 9231 bytes summary_results.json | 6 +++--- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index e10fe5839..e3c8f6da8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 21:12:35,964 submission_checker1.py:3263 INFO] Results=15, NoResults=0, Power Results=0 -[2025-02-11 21:12:35,964 submission_checker1.py:3270 INFO] --- -[2025-02-11 21:12:35,964 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 23:36:06,833 submission_checker1.py:3263 INFO] Results=15, NoResults=0, Power Results=0 +[2025-02-11 23:36:06,833 submission_checker1.py:3270 INFO] --- +[2025-02-11 23:36:06,834 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 21:12:35,964 submission_checker1.py:3276 INFO] Open Results=15, Open Power Results=0 +[2025-02-11 23:36:06,834 submission_checker1.py:3276 INFO] Open Results=15, Open Power Results=0 -[2025-02-11 21:12:35,964 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 23:36:06,834 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 21:12:35,964 submission_checker1.py:3286 INFO] --- -[2025-02-11 21:12:35,964 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 -[2025-02-11 21:12:35,964 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 21:12:35,964 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 -[2025-02-11 21:12:35,964 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 21:12:35,964 submission_checker1.py:3307 INFO] --- -[2025-02-11 21:12:35,964 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 21:12:36,974 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 23:36:06,834 submission_checker1.py:3286 INFO] --- +[2025-02-11 23:36:06,834 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 +[2025-02-11 23:36:06,834 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 23:36:06,834 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 +[2025-02-11 23:36:06,834 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 23:36:06,834 submission_checker1.py:3307 INFO] --- +[2025-02-11 23:36:06,834 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 23:36:07,935 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.92319 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1461 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7ae8d441f..7cc86ad13 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -665 +666 diff --git a/summary.csv b/summary.csv index 7e1c42fe7..6e190502a 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.92319,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1461,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 82bc28728..438f5daaf 100644 --- a/summary.json +++ b/summary.json @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.92319, - "Accuracy":"acc: 74.800", + "Result":10.1461, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -289,7 +289,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 63ecebd22f3dfabe4d41f219b1fad6d1bbb95309..791078e1cc7e081d30863ffe22f9751d14b27268 100644 GIT binary patch delta 1361 zcmV-X1+My!NRLRc+z|zFEUix3li?8|f7@>2NDzJBuV{H+W*&S6d_jZ}0urGi1VOUO zK3R=jfOqX~ue;48f4|KwlXcn5W|CbA32~~stLxOM>Z14MDND%?3XyPjrP+q15nvII z6Sle19`3JoUn4?N7Soh7xYC|MXkYrj^n{Q^ZDrz0+e(>tO;bc$$fz)Q4oq3rf4s=3 zR6mQ2De?knEVckLOHIeJTr;By(@4ZiCa*Ndb~IA5}>Td&iEK^4B|e*zOG@-P|I zTtr1fCL{6*jEzAQ9b{0(iTqq$e>vnSFTTPv3b9;eiIh-O4TBv`Qo2ggMB+`)LPAzW zXNp2}=Xt>+5Q3xg^Iwt2D1sCUD)C~97BBf)exn8AZpJ0P#DjpiHo?keSBX0D^&!go>Ik_44-_r*>4;BEjAR1ogOul z!i%5kibliB)~ex!l^O$z09A|%J^i%!Odf%={r5LdciKVc5eeaTM8@;qhs5*T7IEA9 zkQYFj#e1ZN98+?^+Q!y8f8rK*4>x1GOI+LVEXO9~061OJ@{GXsf)g-RKwz|7+au)u z?(2BA4CmuTxE$Qhr{n!@3%jFH)Fqy4sD6lU0)hd$O93)rw8pT4Le^+(lgmG9VAal5 zHuX4JugiVg*Pn2ErCFXrO`|*2POAZLH{hKH9LeH@dkwhXfTI|Xf4Gf^J>d;K!A3pd zgg3-%H^ghBcp=_Ewa`yxt7mC+M8YV~5B1F%dn!}`B^jXNZqh)Vhq}JC!F?+#ZKrKE zsO8<~{tLM^_{eC)MRON?#ZtJlRT&vJW@}$Z9VCPQ(Ba$C^8=5UADFi~`$to*c#^y9 z5Z^v9J20GgGKpWXe>bZ9|Mm6`9&54|1LPv7g*eE>$BAp`*FWj*cT)4_m8A)1&I1r99}Sy6TQ_X=W->d za>a3 z3ZuO(KWJKn?Lb?yYM@Y;mh6NyaYU^Ll3FK!U)fHa1hVRKbnl$GGpJndWf%Q|bwaBO zWl4%6P)u{7+6sL%tN0N`&Qrx{rxjG_01hqdyOJ5gw1qdT4OlP0MFFH7VWvVm?+wA& zu^mY2lHgE5+-fVS52bB?v7zjnwt(|AU0?~Ga_T7_fU%j0=!oQOMw%bnjR4LtbRa=_ zhqENZXRik<-OYwkxo}G14+d^pPvY#}J24ykzEApMw1)f{{#w7jkE_N)4X0v&>XI|U zEKsj)UE<3k9Im6zufvJA0{GJ*=r`Vui%fmMqY!{haZk!;@%+*ytWce&`F)(`ah5fC zL5c-QA2!3ttNt@k9*6v!cmhnhuRlTk2eW}6O#=mSEUix3lhYzs0os!*BRT<0lV&3( T1>MC71tpV_BNYKHlcytSo;;L; delta 1361 zcmV-X1+My!NRLRc+z|zw?=j8Wli?8|e{FB#I1v7RzrxCW+5M1LATJ15A)pmnN)fcD z-X~;|0dkkvx!7Ube}9J1ZZDIt*KO}4B*Zhe$MeiH-6k=EDtyQP1SU-6VLGn4 zh>C_x$K(?jJA)`X$gqfF`MJ7sf5a1>e}yL$VzEhMDIu>Kh6frabQ34B#G9UkgtUsz z6@}=|i=2lb1V`s*zao!e2npm=;>8p#Uh%E`Msvj7oJ)L(djU~xf|bi6ry&Y(3{pjp z&KapQs#wRcbjz~r2HdIQ5tUTQN-=I8HEiC@Mm$Zq%J*)+rlC|7qUW%Me_Ry^=wVjq z*^^=<+^-Lg;i^L;*it35b;#K^-kBE#cF|d}QAJSb2~~{A^`q}L`)y;t`Hmv7(^4}D zy!ffEXf(WRtr}ifsWG4kP{pXwlTXXfhLT(K{QW|p6+y!5;v_9B|V;MGP>sUt}B!mCZ;oH*l1CN&)=B?ek`XySqbNL zH4{p;@_sbLRP6;*XRE&m5t?&F321JF)J=unx9ju)B|b1MSZg#?=m;LI>bsI#%8i3pXDqlN!6y-|AS`w%R} z*z-Lo=CkO~QQR4)Scs)-f3Rizn>B!oJYQl3fe98E?t!VDh-gF#J|XRoYX^Yf7+O%E zgU2Kz_~H%Vl%H)Flq;u_;b>vjdKM@5!OO|mbzRmi25Zbu@aN{`ZCEvxx<3^MRF{HN z?tlg3>JncU@o+8kVG~chlOP_CQJ;Am7diETNMZnbihEW*7EjOHBsHpwytq&EA|+&7 z5K0QVSbp^*Z~8AlH4gbV@eDZSzWoICAG3iUO#=m-?=j8WlhYzs0o#)+BRT<1lV&3( T1(~fD03?% Date: Tue, 11 Feb 2025 23:47:56 +0000 Subject: [PATCH 039/719] Updated results summary [skip ci] --- README.md | 45 ++++++++++++++++--------------- dbversion | 2 +- summary.csv | 5 ++-- summary.json | 52 ++++++++++++++++++++++++++++-------- summary.xlsx | Bin 9231 -> 9331 bytes summary_results.json | 62 ++++++++++++++++++++++++++++++++++--------- 6 files changed, 117 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index e3c8f6da8..ae06116b0 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,39 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 23:36:06,833 submission_checker1.py:3263 INFO] Results=15, NoResults=0, Power Results=0 -[2025-02-11 23:36:06,833 submission_checker1.py:3270 INFO] --- -[2025-02-11 23:36:06,834 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 23:47:51,613 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-11 23:47:51,613 submission_checker1.py:3270 INFO] --- +[2025-02-11 23:47:51,613 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 23:36:06,834 submission_checker1.py:3276 INFO] Open Results=15, Open Power Results=0 +[2025-02-11 23:47:51,613 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-11 23:36:06,834 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 23:47:51,613 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 23:36:06,834 submission_checker1.py:3286 INFO] --- -[2025-02-11 23:36:06,834 submission_checker1.py:3288 INFO] Systems=11, Power Systems=0 -[2025-02-11 23:36:06,834 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 23:36:06,834 submission_checker1.py:3297 INFO] Open Systems=11, Open Power Systems=0 -[2025-02-11 23:36:06,834 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 23:36:06,834 submission_checker1.py:3307 INFO] --- -[2025-02-11 23:36:06,834 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 23:36:07,935 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 23:47:51,613 submission_checker1.py:3286 INFO] --- +[2025-02-11 23:47:51,613 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-11 23:47:51,613 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 23:47:51,613 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-11 23:47:51,613 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 23:47:51,613 submission_checker1.py:3307 INFO] --- +[2025-02-11 23:47:51,613 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 23:47:52,582 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.369785 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1461 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.4205 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 7cc86ad13..a24ddf934 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -666 +667 diff --git a/summary.csv b/summary.csv index 6e190502a..42f2d5be2 100644 --- a/summary.csv +++ b/summary.csv @@ -1,14 +1,15 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.369785,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1461,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.4205,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 438f5daaf..1d77fab8b 100644 --- a/summary.json +++ b/summary.json @@ -39,7 +39,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.369785, + "Result":0.431415, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -210,6 +210,36 @@ "weight_data_types":"fp32" }, "7":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":9.45684, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -239,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -279,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1461, - "Accuracy":"acc: 76.200", + "Result":10.4205, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -289,7 +319,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -299,7 +329,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 791078e1cc7e081d30863ffe22f9751d14b27268..38d452ea9e4034d1cc7f6333d37c419346b42ac7 100644 GIT binary patch delta 3708 zcmV-?4ukQJNb^Xr*ar??m~U*C2mk=nIsgC|0Fxmi6@M}=cx`Oun_F+pD(c0YFE zE+78#G#M)oGD)L&+E9g7R~0!8<9;+9Hq_gjuQg9q(kz(vgK<2S4fR>3>Mut>9X`g% zujxq2On=E8Ow)!s%CgykuBYKhPJ&d6XL6b!8N|sX$nvkrP)}!x4EhVB$yhh)bw{5B z(NtY}I7oK!5f27YC_l&Hd?KgW(n%u6L6#3c9YwQLJv!`1`LQA%C6R-MdMpl_KZLL9 zM~92P?;`n_ZhTgXQ10XS*W$}rzoFI(sSet&i+^EUB}!iog84Y>#*g1*G#q95wAqD8 z7?0D%Uu6>Ib7`o`BzS75rm8$f{cKb?Rz^|3FN=v*m2jSB@#M$ykXSibnhIxSE9zox z?Q6oSJBD4HkLR&m+33q&7bE-}WWmv45p7)CxVVbS`%XfDR4uNt~9 z4Sz}=M=^U3N9LjaP&i&GuBpilZMdZkcN-0QKA!cU41Q1sjGf4f=w|QYvGP~ayiOH`v zhhJ?DE>|*ZCn#}9Fin&gWY#`25;ALFP-2pfzM{k>OXc)M!dwx*F+I?gR$<FsZ)EWom2By>K?Gg|m4ro_|iClKC{tld0ND)eGb4AR4Mat9_XT(KuB<9qI$- zcKv-HA>~P{%!ju4w`E+{%W;WiilJF%-PyR(l%R8Y=v+MG^cs&6hb)RFN&q@WLZL$m zK*vbPSWevv=ooWy2RN%RZ=tzMVmRM4Cs)We(>p=u!lgtq=ZB2=5+xqFZhv06dn9r= z>IGr(bEcO6dRe-v4Rao6&sm&=quzs|Ia<9I&V^y^K;k|^*@47ccY{RB(|pI-l0$nu zh#sD3g3t*{9J0ikC;<=|2?Y@)03suyAbJ%bGUgP-3iB441BhM)h)nMkM2~{VoF8(( zzC?*fuBBJrzX!y9gt7yOcYj{#mT20BFkGHQpC{4BGffuP2}&HYe48i%5*Z045hXw( zN?fv-PyGrK8FMnuvkLQ;PdGWJ4Q=cI@=$8iVQeUDjD(E)%HJ<+lOT-K+BjHvc=DXT zJ15;ocsl^t^)~?YHOqFqEq6ju=K)-_lczMKlM_bckR{S&G{7LIp??rE8laHTxFicZ zU28ZFIR_NPDhI9g!8MA|(q=mGh@T6v>Tp&RM@~a|+!+4*0P^)dNMP(e2+gQ#;*UeQ zml$`Uay2lqdXS{quIX*Tl7G^EEbz&?Ehv_#acqhuY7k3C16VRo`Td21=w8A(w6vK{5PwT*)nTmQX$|Mur&x;r0G9iRahC@;tAVZ4Ro^nLE$OP@H=B+H z?%&u3;0bOVl8js6j4`DlQoZz-5Jl*81Qpm$aqLMM8flf>npHLM~B* zTpI71OY*bvk0ZH{@ODDdAP-fACARFF34TNLSTOsHCPfl8j(UIN?+Noxa1 z!_a(hqw=WCRR84W;LRf@wo04S@9aRgN3KueqH#4{{n|(F)H?M&r}^EwRs<92UxQ z^4gbjmaoSoT|DFQ{?zZucUq%!BQ}e49L=!Yjq7Te%FQAj;lEOeB_1}d(qS|%Ez@B% zNaYTNI)97?sbn-jWmT;X=K!dzs@4&-nC2W>+Ds?-{*y)PRk{!d;>bA9|U{!cqxWtMNn-=Xb8kbh>FdF1?hq@g`gIqEi;IgW8hjRd2 zRu%6ET3d4tEp4U~a%QPKxTLpo?1x&0C zv1tttqj6~w52HaacPQgwGzcc60Whm7c{m3EW>qbZfMOnKX>*;xmI&1IFjfdAQ_%xp z?zyDrEinJ`p2$0`>A4nunaR3ixs*wLUOop;4$KzB9<9yDRrt8+_i;P`yaJPKt z?|)UEcNkCmQBkHCjF)dHX3=!0q`xiSeRK|*Ka6_eEF5&_W2t1%Gucq((=0ztMYO8Y ze;O>`dQ@gf6em&kyzISr?Q%X2jwW&c<6=ySPt?)>n&QIS0u~#C>Nizc((YI|2n^qvn-yxd>hH2 zFO%Y%sVajw&R)J0Q?h;^_2_rAL=cz(f2Nm%pX&nv0M-ux02BZK0C;RKb7*05Wn@!y za%pa7b1ryoY|U5OZreBzec!Jjv@h+0ZnSj^92bV|1V(c)oTNyf3|bo7+LcJHNV&=1 zFMUaxY9zO5vcLic44)x+I5TI4q zW>d%U?VP3DAhD>pIyESz2B~=ZPX)t05>cbylWD)F`j;YstY@o!&)x!i@FoV%%2d?x zJDw_82WKFwOx4jWV>zQ{;4I82f2td=m&+{WP={~D-@v7f7$&2Ji)e4iWJErIwX%o~ z2N_gJsy^0N4n-!)&+v#`tQL8y6qI$tV8hakEz&Gic+;a)kk`>G9YS>HX(?imQlRsT zKM`XbLk1;NcrinZZ^Tl4Vm~l~=#iNc$yq<5UxSp3 zG#HQvsA9D1*@xLjau0%Uf4{#4ChbPvJrcs@h>WLSha?F64)MF@P?SL0#k&s;*{9@) zwTrDyfty|5osZoU;=5MhP?wM$Ku<^~up&Q*4#0&5BCF%O0U@{7pT}1>^XYgtzZqOk zFUH&54yI#qd_n@x3LShCkQ~rmN>C|h4Tc4jszGC`LjBbM>vpcQe`zM^a#?NLzWIRL zE6a-v8XDc9b~-J1w*~iFa3qTZ9<<Srt5&*I9ZUAsj^Z$9@IWZ1%y zvslQN-SL%?+SZnbf28rNWIwjz7ew=oYf;B?{|AJx)51>}kH2!SCHvuJs4=NczVmPE zm>XI2jgiF9=zl4cH?zGVjtyB$018>KQto6D;>5K)8ox>R{&ny!jPD!M->ihgx`r|7 zW#y5EDXF12e>QB@?G@Mb`maoa=J79+(PTi*;Y|AkWOjQ`f4WYDvhyO$M@;09f%=lC ziYAAO#-VvqDcP&Ec@n~+$p_OD(v2)HLT~fa$kw42^}-h;n{WBPAAFzKD7C!M@d-H} zU(GMZR~qrri$1)VOy@Vl>zN+oTwjlPVdT&#bm{g($k}A{WTko}45=4{Q4sl_)9IjG z?I%7h9X#>v7yQwt$U`yQ?FyZyU1#pz@*TJJAM^gdv)UD52n4)-=Ju069w2{C!!Qs< z_k2aBtJMaq7vQ1*QjRcFp`G`J zVC>iqBy~w}s32~&mDGpQHrRhq_DvhWd79o~37&H5DIS2anThCtqf zL3xL>B*Q1K2P@sxhEX|lO5uA0SFJ~JcJG~-ja}CzT`^ii{tSPvU*5)5W1)sqF+g?6 z8DSQv*S0S4c@YlRQs>v<#9IOUu@Cx{x5FY+AMhvyAXD6<@=-iJZ&oCz&eMDmr+J)Z zo4g>m_hgZO4I?l5Pe6GX@^9i1Fy+4d1oa=Yfgj2O3SO9RY?lZC0Mj~?RU%damy@g_ zA{?fdgP-dI007nx000yK0000000000004l2Arh17B0vH@7LzO^IRUVfW+Ns9yng2P alaM180g;oZBR&D;ljb8M2A3fK0002fMen2l delta 3628 zcmZ8kXHXMtvkf62AiabZ5C}z(5;}@#fY4iLB0W@*-iv^tL#Psv@KPmoLX+Z4Q4k1S ziXf@P^gVSlYmQ_AC&!5?(oM72 zPetSW(>R3U;+*1dXY!Q3jErchg+rx%d?Qpwhlzr>J=^i zA4x`cneGSwxf7AJ{9`!g&}>MIDl5HBdwMMzEiSSfbInjdLyyc1lRmn%gowBJu( z^vIhNfw79QA@zkJqvUS2Z!Q7V$!g#31|tkud0#*77`g=pe+?Wt#DFP6o|hT#)h!Tb z*|JOC*$m3Wd*SDqr`b&94=(ms4}(%|URnLzBkC0HIzUvY{NAfI1skS^&f=}NchPG3L~L9umpWinQ=ke$qEvBnw$h*NJpvxGBQ70Yd90g|;w^d?cd?#OD2hrU&T5~Y?!-vjSXx%Zw; z%oj}{NsTD@PX>qsJIqyk8x5S?6VmjAZNJy+e)MG;tyw$}+q`sg<|v)*Oekr`Nj?a{ za#Q$!dxVblEYFIxItW$R@8!&BKH@4ayQ?EIyeYYdjo8T2-=)*ZoX$OT%oHMdh@Lo3 zS`QB19nOJaUfYefD99TvtTnAkTGZX7y|H4R>zYzMo0r#StaSK&8NIliU71c2Sx!_B z(=4d(R*)|9d_U4$&-N$y=T^-&`z;&?wqEr6tdxPqos^vM-f+m}fD;k)W!(X!qs523 zlghV21%kY=ujSfE^7W-uB2fQW@x6Xo?~_O_MTyw@_Rh~i6^gkae-eRV;^u=ARH394 z%*uD1Y~A)|d2Nk6dLqK+Ub*qMr0{W#v##!_t+e@KA5l1=DG{SMn}g|)Xf+5JxfAUqv!h9`x3SQujRN@Qm62A5V<1jwB53d-19a$Wps*w8pCqR1RX@vZ zodlTZZT@okH(KPcTyg*@C)7K}!B4(Cx_mDdOM{!6DAKuwBo;;8LtDlqcHTY-^q&wk z&-^sepqJ0F%*&iUZ_O-YEh9cw!Q)IL;b2TJc?5?Jf}CR1yR~3L>dDap%(OC@sAtJ& z?ganL8on<63h6Dj2bLEn@JcW9m!c^niK&m( zZb-*Q3UaJMT^ln%4 z5;P|BZ>snYhb;6J3AvcU|8S79mX5{Mofhvpvyd-%EY(pR@71|&s8%5bm(859Rn z_gyZmKMvKH4T>!z(&k`M&$2ZOvk?^9BY8BZ`Ds!ny;!pMNp)~hhbsYNHk@&0V+z?* zp&lj=*c(j8ai8}&MEX^D&v5l3f49WSu0a|7!2PjqjkniD{e!xN%!IxqLGIi{b(pX| zk)OkJjkj9q%D)6ToLy`c=*`KRnj~)|mc&8KWcavxLhOtU>6zkj4WQVZ27`*VZHKiV z?MJ7YEI|XF%BET~g8h>r&Vw&^^m%5V?lY6ZIAG?9DDhl^I_%S^N>dOOWw;J(XcXNM z$5R7bb+H|xkL8pnRXM}QxDJ?IzB3w!QtPdLe<9@AH+=g~2F^#$iHRgh1iM4v)O2pufv*mlRT!(;kT}m_PdM4G# z2Y!5GE~1{Exp~RC@FVejVgm8i4NW=0MyS|ZPp-C})Ct?7D2W;j_`1?nOy!+DX35zh zDzTWy)f_v2KvBwm*8@gVX;;Lu_Iqs$S6z zj8RX;wNkeDjg{Jne%*%ycC6tWrk|f&rhgm>=}?u6v=4M=dvgE5`GxXvVVK$8xcvQd ziK-iM>Ce(GIL9+csWy$=vrMy%3fb{)=o}M7caKSA1$}GT>J8J=Wcmh|e@-Z8LNR5qylS7~-E9c6hpL%?r?X|-rixVE>`%LY z#yvxLxh(6>>;4mBV2ZFFs)B{m>!U4aUk_E8-A_pqJpj6izXHhU84Nbr$d*!1YZLW^ z8{`IqwigdLriw(*R00?vJuaYBFm=N+gLo#MMiQP~U5kIL zE4#T~)H)UW!u_>%!*rY~TNeopWD!1qan9HvG}MKm2h^!)s;oTD$N|hb+Q#~ zuPn_=8y8GA(u^*a$H1gRcM7p=FrlXQK;u!eS|BN`2g(>zdU$A`Ie0UMl}2peF&Qj% zV)e9?V%d8@sEZgI)uzxh;$k^TMWF<&#%qyiXE@DkogQ|$SO$~tn(gQ6);O1QC?0bu z_*AMFx!A0QH`6^9G>>0LQ`D9qDS%J1aZfI5IWVu5#-(k7+-N;&;=q~(by;a)#whl} z=eNv3s%KL{0hR6R}d z2F8$e=#j}dv#edNfs*;uMJ+E+LvBQk0QQJ}leT7aPEG|^4|^>h z!Pq~OOY>8Q|mKJZ)Su@{z8@h{eHzl+Ud--#jr+!rk>Y&bJ6dqC-Xyi$3X_yf9L28jfH{@2W8{i*2^J=B8R@WUt3r04+YPxuU^Yx`TeT- z52L>dXJK7BS8GJ&%1JbP8xLVdt^JOX3Am_?e*dQFq$-3r_~gNtdXF==ll1&e2A@AZ z&G*0#ROm1L)D%Kw000sQ05Aan0B0X3q`!}EfRvMuzl&t3muDe%)V&YJsBwHjQ~E~Y zzMECt4&1uQ{RI-2vmGbrhp4-TwGu(we>N-tHEsF_{wQddt+Ls5VZVq8QX|l~=p+ju z^#e-gXd>kZznL_~P5U2LvYD*|=q0u~v7Ei=mUu-1-9xFj8f0YCFYxy?|GMJW^c4Nbx223x`hG?ukM_@}YLSJNCy@o;{c_~PRFf%m zZYcB(5iVtsyou-ep+NoihWn})*P7REi~wo=d$eV>3p^l0WHCYoAph4~um1RdBPx2F s>B?&1z&W<8XVBYZ`dK4TVebDg$bY*fvaSo>Ba6<`6@*gC^8bDR9}Awki2wiq diff --git a/summary_results.json b/summary_results.json index a9d17ddfa..f9e712027 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -55,7 +55,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline", "Software": "pytorch v2.4.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1461, + "Performance_Result": 10.4205, "Performance_Units": "Samples/s" }, { @@ -185,6 +185,42 @@ "Availability": "available", "Category": "open", "Suite": "edge", + "System": "gh_macos-latest x86", + "Platform": "gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "undefined", + "host_processors_per_node": 1, + "host_processor_core_count": 3, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", + "Software": "onnxruntime", + "operating_system": " (darwin-23.6.0)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 9.45684, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0007", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", @@ -216,7 +252,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0007", + "ID": "5.0-0008", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -252,7 +288,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0008", + "ID": "5.0-0009", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -284,11 +320,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.369785, + "Performance_Result": 0.431415, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0009", + "ID": "5.0-0010", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -324,7 +360,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0010", + "ID": "5.0-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -360,7 +396,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0011", + "ID": "5.0-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -396,7 +432,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0012", + "ID": "5.0-0013", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -432,7 +468,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0013", + "ID": "5.0-0014", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -468,7 +504,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0014", + "ID": "5.0-0015", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -504,7 +540,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0015", + "ID": "5.0-0016", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From 0df43e74f8dcc96db332bd22b53ca481c51090c8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 23:49:18 +0000 Subject: [PATCH 040/719] Updated results summary [skip ci] --- README.md | 56 ++++---- dbversion | 2 +- summary.csv | 22 +-- summary.json | 314 +++++++++++++++++++++---------------------- summary.xlsx | Bin 9331 -> 9331 bytes summary_results.json | 6 +- 6 files changed, 200 insertions(+), 200 deletions(-) diff --git a/README.md b/README.md index ae06116b0..1ef532dab 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,39 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 23:47:51,613 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-11 23:47:51,613 submission_checker1.py:3270 INFO] --- -[2025-02-11 23:47:51,613 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 23:49:14,087 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-11 23:49:14,087 submission_checker1.py:3270 INFO] --- +[2025-02-11 23:49:14,087 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 23:47:51,613 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-11 23:49:14,087 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-11 23:47:51,613 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 23:49:14,087 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 23:47:51,613 submission_checker1.py:3286 INFO] --- -[2025-02-11 23:47:51,613 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-11 23:47:51,613 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 23:47:51,613 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-11 23:47:51,613 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 23:47:51,613 submission_checker1.py:3307 INFO] --- -[2025-02-11 23:47:51,613 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 23:47:52,582 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 23:49:14,087 submission_checker1.py:3286 INFO] --- +[2025-02-11 23:49:14,087 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-11 23:49:14,087 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 23:49:14,087 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-11 23:49:14,088 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 23:49:14,088 submission_checker1.py:3307 INFO] --- +[2025-02-11 23:49:14,088 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 23:49:15,149 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.4205 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2233 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index a24ddf934..1b677c9c1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -667 +668 diff --git a/summary.csv b/summary.csv index 42f2d5be2..77a7e3d2e 100644 --- a/summary.csv +++ b/summary.csv @@ -1,17 +1,17 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2233,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.4205,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1d77fab8b..9203922ae 100644 --- a/summary.json +++ b/summary.json @@ -1,54 +1,24 @@ { "0":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", - "Scenario":"Offline", - "Result":0.391627, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.431415, - "Accuracy":"mAP: 76.951", + "Result":18.5738, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -59,7 +29,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,27 +59,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":18.5738, - "Accuracy":"acc: 76.000", + "Result":1.7438, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -119,57 +89,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":48.9664, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":22.9842, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.352127, - "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Result":0.426752, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -179,26 +149,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7438, - "Accuracy":"acc: 80.000", + "Result":23.0543, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -209,27 +179,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":9.45684, - "Accuracy":"acc: 76.000", + "Result":0.433634, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -239,26 +209,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430616, - "Accuracy":"mAP: 49.593", + "Result":0.406474, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -269,37 +239,37 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.406474, - "Accuracy":"mAP: 76.951", + "Result":0.391627, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -309,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.4205, - "Accuracy":"acc: 75.800", + "Result":10.2233, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -319,7 +289,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -329,27 +299,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433634, + "Result":0.431415, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -359,27 +329,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0543, + "Result":9.45684, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -389,25 +359,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426752, - "Accuracy":"mAP: 76.951", + "Result":0.430616, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, @@ -419,27 +389,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, + "13":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", + "Scenario":"Offline", + "Result":48.9664, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + }, "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":22.9842, - "Accuracy":"acc: 76.000", + "Result":0.352127, + "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 38d452ea9e4034d1cc7f6333d37c419346b42ac7..293789fd71c31e462c5e511ac0d2c34c8174f232 100644 GIT binary patch delta 1402 zcmV-=1%>+aNb^XrFA@&T{#!}w0{{Tl4*&oZ0FzE57JtoGZExZ@5dMC@!peQw{g5{U zc|pLc0$QP^6w!9o{e;|Ppt(!zTd1_2(4IN|HQaeH%a1_qIe@tCE8L*IA;X?z;|(UVdUy_L(p@t{=RwQU(a zK*ppca)033vJyqcl>S+)ZJ8IqV)+1|vec%I zN!fjpCJJwQk_xgaI@2LUcb*p_0x1PLKmQXkMiHb?FohR0wD?-A)Hhb3*v*8(m$;P> z*Ctp~mIaHDgCkJddvwlFpHam+mSZ}O<2K;5ipNYbEh`;yd#hplayk@QCUm~n4@(v) zU4J2F0V^nUfq)t1rI|nJh(r&|4YfRdXbdZ+g|>_YUnOh%q`*#+mG`;`N;73TVruyq zc+I?R%v-D(5WW6g%hsylB`P%rM}?RxrQ@sZqt;M?zSzDe6*=MD+s zazw`SuS4SdUW<5bb0`WR&EnmMhU`;v!hhPv*1Etgu5K^JZkKqj)p6e5GbM<*VyI#)6i{Bi zdniST?35RtxW72Wuk(w!~rZ|5#Y*p!5w0;u8qRGzYU4POJtxkyE z=I4>EMJ?J1jz%`m@;uM~KCw}1bpppD@6aWAK2mk;8KeI~~R0w~t51d*V1P2&^QP0QE#AB#J=XEPLBp#6PkPY2F?uY13BfiQ{@^ zznxvVTrQ2f5_&3`EF7o^iD+o-mz;F*t?LaLOI9U|d68;`1UNu3C(Pe64T@^9i1Fy^!1A5sGZ&Hh_SlQtu|1j2eB Ipp*0?e_AlAvj6}9 delta 1402 zcmV-=1%>+aNb^XrFA@%>mxG_{0{{Tl4*&oZ0FzE57JtoG+iu%95PjdTAha*-gKo5S z3mg}Q?F2@1F`T4GpA1?W+uD^#tw_1a-!FYhnrbAsX|liq1`MAec{np?hU4C+hdd)2 zC}k@6so`3VL4e01N%`v3xVt?wLxV`gc)~Kl;na8pX?*Jc(UVdUy_L&TW35ziV%sua zL(Zfn3V-0*vJ_>`l>S++Y+01R61fIYd1h0`@$H2xH>f`r3R^Z`cDPJJrYr) z-;-&-r}~#7fvjh%e$USqEnzt4!6=EMqyNX5cK$D5@K;m&+{W zP={~D-@v7f7$&2Ji)e4iWJErIwX%o~2N_gJs((J#R}Muc%FpnKT&xy(suYxU!(hYG zj4jeERd~~*RFK!vD;+{~=V>WokW!%Yi$4)#976^rQ+P2$i*LkIePbnx-IY-I5_b}k z#sq81s$?;8a0E(wkIotDGwPUUIi}+{ZVOK9c)}FZveFT^w;HxDFNPw|h0gbCJ!i4f z6@Oxuu!K?<2$*qEnZ=`yNW7kJsO9TJ6Ie1Ww0SJ}GF{n61$LCITIeFE%#7)Xsrf_b zxAS%}Z@FSf>};=@44!%FD_RZDTkD4Bs5BUm2dHAS>)D6dM{*B>Z@<3WLSha?F64)MF@P?SL0#k&s;*{9@)wSSAPO@W(T-<^-$6XLs8;82&49Y9Y=C$J(v zhz`Jo1|qBDy8$7$*Pq8%H}mOuHoqBMPA|sW-43Q>aeP7o&k7xU6ObIxT}n_XXAOo0 zl&V2vt3v(N0PA+HvuP&ja#?NLzWIRLE6a-v8XDc9b~-J1w*~iFa3qTZ9<<;Z4-2`=gh2fW2!x5Zz##a|cs3&{%V4d77DW}d}+WF8mAuI>fr52Y@wGzV1l zZQ5vJXzFJx+|T05rCqy4MQ=X$7i8GNk+WFHm)-G|k=oXlhoteVWIwjz7ew=oYf;B? z{|AJx)51>}kH2!SCHvuJs4=NczJK#?>zEr^^o^0k&**1E}ShAF9`IDa;5)$JA6^!l$%g68orlhI^A z&f!e^1Y~x5Pr6QovhyO$M@;09f%=lCiYAAO#-VvqDcP&Ec@n~+$p_OD(tnLCFG6qg z)5zAL7WKjxBb#sez8`#_*eJEU(D4a5A79Nc##b8g(ThI3m`vw4!|RzI<6K{lcwyww zD0J!eL&(`=^kk)aBn+t+gi#Rrp3~`|T@6aWAK2mk;8KeI~~R0w~(e&+Ti0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0 zE_iKhos!E=!!Qs<_k2aqfL3xL>B*Q1K2P@sxhEX|lO5uA0SFJ~JcJG~-ja}CzT`^ii{tSPv zU*5)5W1)sqF+g?68DSQv*S140@p%yr*HY)#;lx`3{IL)EmAAtpQy=gs1Rzt~qw-Na zJ#SX1&eMDmr+J)Zo4g>m_hgZO4I?l5Pe6GX@^9i1Fy*u0A5sGZrk8`ClQtu|1iXIc I_LKA@e{&4282|tP diff --git a/summary_results.json b/summary_results.json index f9e712027..1b8f21f9e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -55,7 +55,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline", "Software": "pytorch v2.4.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.4205, + "Performance_Result": 10.2233, "Performance_Units": "Samples/s" }, { From 5452686e805bc0e0ed77679ec8011fdd5e01d5f8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Feb 2025 23:49:52 +0000 Subject: [PATCH 041/719] Updated results summary [skip ci] --- README.md | 56 +++++----- dbversion | 2 +- summary.csv | 22 ++-- summary.json | 308 +++++++++++++++++++++++++-------------------------- summary.xlsx | Bin 9331 -> 9332 bytes 5 files changed, 194 insertions(+), 194 deletions(-) diff --git a/README.md b/README.md index 1ef532dab..13ca5d8a0 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,39 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 23:49:14,087 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-11 23:49:14,087 submission_checker1.py:3270 INFO] --- -[2025-02-11 23:49:14,087 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-11 23:49:47,670 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-11 23:49:47,670 submission_checker1.py:3270 INFO] --- +[2025-02-11 23:49:47,670 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 23:49:14,087 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-11 23:49:47,670 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-11 23:49:14,087 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-11 23:49:47,670 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 23:49:14,087 submission_checker1.py:3286 INFO] --- -[2025-02-11 23:49:14,087 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-11 23:49:14,087 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 23:49:14,087 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-11 23:49:14,088 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 23:49:14,088 submission_checker1.py:3307 INFO] --- -[2025-02-11 23:49:14,088 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 23:49:15,149 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-11 23:49:47,670 submission_checker1.py:3286 INFO] --- +[2025-02-11 23:49:47,670 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-11 23:49:47,671 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-11 23:49:47,671 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-11 23:49:47,671 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-11 23:49:47,671 submission_checker1.py:3307 INFO] --- +[2025-02-11 23:49:47,671 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-11 23:49:48,643 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2233 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2233 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 1b677c9c1..5d30d6f4b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -668 +669 diff --git a/summary.csv b/summary.csv index 77a7e3d2e..767262426 100644 --- a/summary.csv +++ b/summary.csv @@ -1,17 +1,17 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2233,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2233,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9203922ae..42458331d 100644 --- a/summary.json +++ b/summary.json @@ -1,24 +1,54 @@ { "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", + "Scenario":"Offline", + "Result":0.391627, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + }, + "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":18.5738, - "Accuracy":"acc: 76.000", + "Result":0.431415, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -29,7 +59,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "1":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -59,27 +89,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7438, - "Accuracy":"acc: 80.000", + "Result":18.5738, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -89,57 +119,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":22.9842, - "Accuracy":"acc: 76.000", + "Result":48.9664, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.426752, - "Accuracy":"mAP: 76.951", + "Result":0.352127, + "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -149,26 +179,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0543, - "Accuracy":"acc: 76.000", + "Result":1.7438, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -179,27 +209,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.433634, - "Accuracy":"mAP: 76.951", + "Result":9.45684, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -209,26 +239,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.406474, - "Accuracy":"mAP: 76.951", + "Result":0.430616, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -239,37 +269,37 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.391627, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.406474, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -299,27 +329,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431415, + "Result":0.433634, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -329,27 +359,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.45684, + "Result":23.0543, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -359,25 +389,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430616, - "Accuracy":"mAP: 49.593", + "Result":0.426752, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, @@ -389,57 +419,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":48.9664, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.352127, - "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Result":22.9842, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 293789fd71c31e462c5e511ac0d2c34c8174f232..3cbfdef31fda68c12c027755f45117f00a452cc1 100644 GIT binary patch delta 365 zcmV-z0h0dnNc2dspB{hBc;BQZ0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!E= z!!Qs<_k2a3vX5%uwHfo$=G#W(iMX>O9Tw z<1~-6tjP;9dnB{@S3mNq{|uDlkbe_TfMf3KPf-5>v*jNP1O?4_-=rp!^&?aP=95Y! LBLt51d*V1P2&^QP0QE#AB#J=XEPLBp#6PkPY2F?uY13BfiQ{@^znxvVTrQ2f5_&3`EF7o^iD+o-mz;F*t?LaLOI9U|dLD;`1UNu3C(Pe64T@^9i1Fy_Ae1oa=Y<{t|L1;TnCpe2*`BUAz9lS?Ec K2A3fK0002$h^Y7g From 1fa47174cc7be70967969f18cf94413e5d995dce Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Feb 2025 10:38:05 +0000 Subject: [PATCH 042/719] Updated results summary [skip ci] --- README.md | 56 ++++---- dbversion | 2 +- summary.csv | 22 ++-- summary.json | 308 +++++++++++++++++++++---------------------- summary.xlsx | Bin 9332 -> 9330 bytes summary_results.json | 2 +- 6 files changed, 195 insertions(+), 195 deletions(-) diff --git a/README.md b/README.md index 13ca5d8a0..228a28c64 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,39 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-11 23:49:47,670 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-11 23:49:47,670 submission_checker1.py:3270 INFO] --- -[2025-02-11 23:49:47,670 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 10:38:00,744 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 10:38:00,744 submission_checker1.py:3270 INFO] --- +[2025-02-12 10:38:00,744 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-11 23:49:47,670 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 10:38:00,744 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-11 23:49:47,670 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 10:38:00,744 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-11 23:49:47,670 submission_checker1.py:3286 INFO] --- -[2025-02-11 23:49:47,670 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-11 23:49:47,671 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-11 23:49:47,671 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-11 23:49:47,671 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-11 23:49:47,671 submission_checker1.py:3307 INFO] --- -[2025-02-11 23:49:47,671 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-11 23:49:48,643 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 10:38:00,744 submission_checker1.py:3286 INFO] --- +[2025-02-12 10:38:00,744 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 10:38:00,744 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 10:38:00,744 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 10:38:00,744 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 10:38:00,744 submission_checker1.py:3307 INFO] --- +[2025-02-12 10:38:00,744 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 10:38:01,794 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.391627 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2233 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2233 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5d30d6f4b..51ea4a54f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -669 +670 diff --git a/summary.csv b/summary.csv index 767262426..e16606ab8 100644 --- a/summary.csv +++ b/summary.csv @@ -1,17 +1,17 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.391627,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2233,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 42458331d..df5f6761f 100644 --- a/summary.json +++ b/summary.json @@ -1,54 +1,24 @@ { "0":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", - "Scenario":"Offline", - "Result":0.391627, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.431415, - "Accuracy":"mAP: 76.951", + "Result":18.5738, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -59,7 +29,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,27 +59,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":18.5738, - "Accuracy":"acc: 76.000", + "Result":1.7438, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -119,57 +89,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":48.9664, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":22.9842, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.352127, - "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Result":0.426752, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -179,26 +149,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7438, - "Accuracy":"acc: 80.000", + "Result":23.0543, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -209,27 +179,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":9.45684, - "Accuracy":"acc: 76.000", + "Result":0.433634, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -239,26 +209,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430616, - "Accuracy":"mAP: 49.593", + "Result":0.406474, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -269,37 +239,37 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.406474, - "Accuracy":"mAP: 76.951", + "Result":0.387358, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,27 +299,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433634, + "Result":0.431415, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -359,27 +329,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0543, + "Result":9.45684, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -389,25 +359,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426752, - "Accuracy":"mAP: 76.951", + "Result":0.430616, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, @@ -419,27 +389,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, + "13":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", + "Scenario":"Offline", + "Result":48.9664, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + }, "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":22.9842, - "Accuracy":"acc: 76.000", + "Result":0.352127, + "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 3cbfdef31fda68c12c027755f45117f00a452cc1..53b0629acd98f2756f0a975d4b24118c5a03cf6d 100644 GIT binary patch delta 1374 zcmV-k1)=)%Nb*Rq%>x8b^TubB(E}!b+j8PK5PjdTP`UDuR1LlXhM~Yy6Ov3WkP!AV zwNI3=Wvm+8@<=j)zhB9Q>>zhiTlGdE^l8N=z8%@6tEGHb-uK%LsMf~!NBwqhcG7?D=D*-H|SGH#Chn~DsL$0Pe zE$&Sn)qfJ|hQ?(|D~TyuuQPW6oX>uRVwo>ks9P zyrEJl(xcCMSjXlZ2A0PGq}HLu+=kt;v(<*xpR)C8(1#8BEx!B8Z#=Yr5k1&L7xvI? z@jb&s8_|b7^k5IY7T-5Kv=IU9p$~fa+Hdg(ryefbh~X(>+D3dnMO?KJp7AlR0f?dI z4FECZ_y$0_r;u9!(mRFR0g(PF!X z4*mW(cRUdv%z+u;z}TmAlk?c-7+>HCuxiM=_&4k%y#0G|y`0@WKAhK8Q>e1Jp)#%3 zcEoZg&(v?ON}_ry2~SqENC;Pjovb7gawV#DPKB^Lo4s5q`D2A(u6$#x~pRAZjOfz9!jGY9i z2G1X{Ce_{$6Izyv5*Sv#wfKg{(TN z#nmQu!{F@%?ERkv{D|efpMXtvIluI{ytFsz3=K&jgn=R z5hVAe>W9fr-8brREEMmOrjbPyzq|5dZ)HWN%|oa&K^R zFJo_VWiEJaY@L$LYQr!PgztR{qq{B1rX>j5fwtt-K%pruy$NZPh+02LYMs1&6+3Yf z$f?KG%zit&sGje1m;Hhd%9;kzf}jkHuu_>#gWgw*e1@_RxsklH1{$=7fadL8EgTbn z*2Al}4t!J)vIH`L3D=-)jE-R(#1=FUMRJ%VuC3QRrqXY))Ec(lGpt?^}%4%hJzm+8c71@dv9^c!!-MaDjmSqi|$ zxF_YKdU{?hP+OAnJ||^P%M~T8nlVDZ4kNGn&p>?~@^9h^Fy_Ag1nnQQ<{yLt1W@zF gXR~f1g8>EP>#EHpllCK20pycQBqIivApigX0FG*xF8}}l delta 1376 zcmV-m1)uuzNc2ds%>x9e!4V#l(E}!bO>^Qn5WVlOP`PqQss?`nCZWJn6Ov3mAR)}| zPVI>jwv1I{TOLU!@b6c$Av?&O)YM*b2(hL2x?9g`b!+r_n`dZ4xS*^UTb}J&h!hcv zX|Wz#fBkvWIk%9IxQKDa3Np5KL|C80KSnQ%{}3r55-9_P7+a~7<fso7rsOB8SB*9fC7wqMh&~++*m65)tL@X2aFUqNs*0)fOw_~fTJkVIC+AvCK9FwAU z8&yKG{ChLy?F}?kFW7TE*L!7O*j~>K{J#D?yNv8PPUE$X@Cr*DjyQWky!IeUtUr`D z^v;z^ksf{4!#Xx+Kd?9qAhixH<|gcpoQ*cD{*3>&5i$@!_|`m4kV{dmaw>F<6w6KDb{)xo38|~Gql>OHn@;w8 zPJ}9M!da(F9O)|CQAM>e)rrcgQ|=_=QM%mtc3``meKA}{tVrnE`eemKVwwr-Y~&W?i+a3R!hj zi>pm+&x5xUu=jrw@FSM@egZbx<^0mas>Q@qA3iv1+HR??CrJEk`ujlj_r9xBHA)s$ zMv&Zb>FDNu~58CnpPf_oH9=3uE}kEwWu;2<}5y1jhGg?pg${edJ>%k!;>gD z`-J>6G)5}815|}rd#QVEAgpjy;x(CJzNUph8A;U6a&4tDZ!QP*vt(ucq)PjWN!9ls z-cof#B3!?zCN^PA9$s{B|9a;N{{^$k2h|pT&3NCWCIJ8dPyzq|5dZ)HWN%|oa&K^R zFJo_VWiEJaY@L$JPQx$|MfZF~@0iQTEq|8hO~c=lcs4a zb;WXg@0__aa=F;cHu?qYgjN;Gk`zUtnC3!lEA-JU<2j0)r;5{7E2z)`99q^)4CBdPBxY1TpA4- z7@LWRMkHqw()`$V0C0w(1qsSKoFy4Pdp%g`ZZ-_cg;NTDFmTg)5-0cGiOJY?UD6eU zHRR9mdiDA~tQrf|pNavhOU?+hK)tp}b%`&FaJZH_zX~Vb2;fhLpx<~K7diF;k3s-) zjC)c(i|3bSiRwJf@8dL&v#iMrGJ7Pm`By*ks{ag>|Y>rYVs0kh;EgaQPp i!4V#_Y$AgJ1 Date: Wed, 12 Feb 2025 10:45:56 +0000 Subject: [PATCH 043/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9330 -> 9333 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 228a28c64..21e2757bd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 10:38:00,744 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 10:38:00,744 submission_checker1.py:3270 INFO] --- -[2025-02-12 10:38:00,744 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 10:45:51,418 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 10:45:51,418 submission_checker1.py:3270 INFO] --- +[2025-02-12 10:45:51,418 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 10:38:00,744 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 10:45:51,418 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 10:38:00,744 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 10:45:51,418 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 10:38:00,744 submission_checker1.py:3286 INFO] --- -[2025-02-12 10:38:00,744 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 10:38:00,744 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 10:38:00,744 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 10:38:00,744 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 10:38:00,744 submission_checker1.py:3307 INFO] --- -[2025-02-12 10:38:00,744 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 10:38:01,794 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 10:45:51,418 submission_checker1.py:3286 INFO] --- +[2025-02-12 10:45:51,418 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 10:45:51,418 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 10:45:51,418 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 10:45:51,418 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 10:45:51,419 submission_checker1.py:3307 INFO] --- +[2025-02-12 10:45:51,419 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 10:45:52,367 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -35,5 +35,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352127 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 51ea4a54f..4e07e6b3c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -670 +671 diff --git a/summary.csv b/summary.csv index e16606ab8..4abcc3cc9 100644 --- a/summary.csv +++ b/summary.csv @@ -13,5 +13,5 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352127,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index df5f6761f..421d9d6d4 100644 --- a/summary.json +++ b/summary.json @@ -429,7 +429,7 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.352127, + "Result":0.353536, "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 53b0629acd98f2756f0a975d4b24118c5a03cf6d..294c71d4a50e0253f8cce02d6dcbf294724deb50 100644 GIT binary patch delta 2392 zcmV-e38(h*NcBjt*arpg$gi%Mlidedf2)x)(0;4BtKE-Xw9ALTJWa;RgG|yWo;FmW z)m24K!?+(!hYj`i=4;JUl{5>c{a_qVWkY?Isrt*&Plu0j@@qPhGE;H~)3l+EvTSys z>uETWlOWaNnVjZF25~Y8vixf@)YDlagZ{#3GS-cH-O(pOG*y=#4w7Ab#DhT;f6C8s zIG@OAwsexnaggQ1Pe;)#RgVt)QGTq5M@i(Mp&pBa<`3bk`qAN{@4HAorW>D?B9!|$ z{tYyJiPD#YU_Q>e@#8lc4M$l%ZFXT2#^ZGHSD8flTpFq}37#6N zsVa|AKN}T}l~L61%VMHcC7h>Oe?0lIJS0{QmZrj4*^0VYTl<=@>W*O-=i_-SS2p^x z*To1w2U&1*n8c4tvgje51;v>W2ZoVPOIWmiFPe+7>8pnBOM{ZfQOw@Mk$I>;6pmMl zYie>s8*XXC-A03+k7qq7gCCRuW2l$M&_#tsB-e~YlqENe#3)N{83~z>e>+BEttF*! zO+H)`9FT!_QDKv4-&EYZsknVpaaW}%MQZVpT3}D+yNe2ohWfVT=55I>N*oewrIS*$ zHXH4+4G!FL7bmFj%8Di`{IcRRDg;5{3nL*b>?|QbwGP_rdgv{=DMq=`-&EZ#@gUgl7+6hV=5=;{% z2AQ?bjD*bE7nGQ!qpv7&$x=CekuX=pZ%hw#rB#@>UXH!oq0VVT8$0kw2p80*!<+)r z@g?JyEdQ(5aSx8Z{qO2aw|&+*|F}qo(mWd)M%ie7uZ8hMpN#Xrf0I1Z$v@3xGU!Fq zflTszP4+UGX1#QNKZ(*J^+|gVwpQ1(M9M{yr5CxBe$j5O>=T*hDNL%ba+%uNb1$6D zYT;~Ni>K44WIoOEWU97O^}={Mh=%IVYF}nSG)~n|hx&lIU4P$4NO{sK^Pw&N+cK`} z<+#K$#n3FX?rdCXe@f7~JajIeae9qMi9;4e6D0s0Bcaft1fXLiWGts{1$2x#xdWV4 zn77c}B{7_Dnv*MJo9UgPbKz1Vne#(Ne2Ef|TsN=WJrX$_^@6bYIaAAjy)0eThB=S3 z=PXXbQSZUf9Iajp=fbddAaNg|>_FnJyFsGmX};rZ$)PhKO0z{^F3Zh3rWX=z{UtglcBiGU^@81LBK0?_6 z#5=EaOEm5L-iHwAjh!P+XB`#Uar+x*Aj5!(SS%rDa zC!CzqhBkHpe|adi=`c1FHbz3medX^Lwn-4iX>A-VJUn^M-<^~0BfK2|?D`u3`kG}s z-j+L|sPh0W+Q}{r>EwjbIAn=584WPVX()t@1}J1SF3G}9*BXvP&H)9n%0X*=aE&6g zw3$vk;^zXaI-C{7k<(BfH-`T{fPB3V5*T|ALNn@`fB552?j^=us9X(9tTvJ~+cmu{ zSn@mV#{zfOZ9%a_jbl?RQG-}A8o-j-=?5%PgIJ0xEExxYWrc&5pwAM{p{31qf>=_k z4r2woHJoRkVk!OuSneanT{du516!x7zGZG~z?8YVMAf1(DtG^)5{8~~RU4qEGjuUyiWHWvx~p$JwT#tOMa4RUF`Yc9#p#y^haKEm4x zNrP;v3QKHxZYKB#qQ`=H&S+93QRBE2Nz@>cj0TWI4I+se*QH3BRY)?O&_^0o4qEGj zuSn9CHq(h`P$a2UhqHnV7BzmIB5Cdy$@zVLXCs`im?S)A50k~cvbaeKUo-v$k^6{l zr`*-t%w09t+M2rZkL`~IJ_@%j$|LhYoj%NFG{_^P0Ul94ad1M>_j};DD z>w~X6(v~*U32g6xRfn?z9vKb%qO{%-kN?CTlMoKAf9&yif9m(-JFU^V5t~Iij%HZy z#&xw!j+v*a}F(S zrW4$Gg`ynJ3es0b1L^CY>vG<IxbE-m6=GzjJnWju@q!DKW5W>qB*=K#R0 zs^t+-%mXcLt`m490`)wM6@tlB^Z=N9F6ns-f6Tx9Oyr%`^jr(S%w*lMT*{=r@y^~v zCPUej<8i8laXx2D({=0eMaHf;xLdyS_bSgjjHmsmC{ql^%QqCWXgXBV-5I0qj>=hM%M@kwbYdF(IJJi0|M#RqO!M~AauC@+F!7)?`UEC+d@buG^; zlI44pi_cj+TYSoYo!`e<7EfNjjbzZ5N%758l|dY5FJFo&S-+2Z^gFYc5C{$hfl?2h zC6lBcDu2tVOA)pMZON&DLQ`6D6Vk>JwSJJ)I(hplPU0lsQ;(~e{dRUyHQ%W&`32{t z(KRAjjuOz^2&vl|eXJJgBT9T=TCmP&sL>ugnm2b9x0D+PZ_Ze7L4r>rNPEg{jkY0J zim~TgP|Ro1p`*AlPO%V6*J8`~H){bGdH#SE1b-%2VE6z`?MOr?q~Igc{hQ9;((e; zaLOI9U|dt->mnYmV?He7i8m6&(?04GZ>L3$eISw;fF9$XmCxDp%W8p|A}`7`FH%yh zNEuG=OInikVdPE!1*lF#{!Kgsj=66?LGuT*O8Bt8M>lSw2a K2ALrM0000Fq-U-G delta 2415 zcmV-#36S>nNb*Rq*arn(m~U*Clidede|wWa`>pD(c0YFEE+78#G#M)oGD)L&+E9g7 zR~0!8<9;+9Hq_gjuQg9q(kz(vgK<2S4fR>3>Mut>9X`g%ujxq2OvxQg(}p_Ave|*I zr{PFWf>euVa+)6*#K|Pc@~_EIPiKh?`U|7USU2i*N1p`IR9$*FNOtiN4+c>ve?Q0J zd?KgW(n%u6L6#3c9YwQLJv!`1`LQA%C6R-MdMpl_KZLL9M~92P?;`n_ZhTgXQ10XS z*W$}rzoFI(sSet&i(yp7)CxVVbS`%XfDR4uNt~94N4wIF?$b3=Ar&jI9@5P zsmTp(xTOtu8x49sp7o#%eozLCpf4f=w|QYvGP~ayiOH`vhhJ?DE>|*ZCn#}9Fin&g zWY#`25;ALFP-2pfzM{k>OXc)M!dwx*F+I?gR$<FsZ)EWom2By>K?G zg|m4ro=%^V`83OusoF}_3*+e^8md35eVGN(I8{F#>I3F>{e2%Hi|hqm~)Wn90#*|^e_f1q=D=v+MG^cs&6hb)RFN&q@WLZL$mK*vbPSWevv=ooWy2RN%R zZ=tzMVmRM4Cs)We(>p=u!lgtq=ZB2=5+xqFZeF>2Byu?F1!3`Xrk4MDS-PqXa~@~U zS)7EU-h-hzTD=y|g<cZR`N@e^6@EVQeUDjD(E)%HJ<+lOT-K+BjHvc=DXTJ15;ocsl^t^)~?YHOqFq zEq6ju=K)-_lczMKlM_bckR{S&G{7LIp%5|}ppenHBnvxTYd8)$2Nc9A2d(wNHHy&E zW;*eRp9`?+a8?vYPD6Rz82d21=w8A(w6vK{5KC&+ zVXWY34d>aXSc?Aumivftmj^hjfvwY3-!iQ&>8jv2n~nwU-`ED_5;cxPxkL?e$!LH} z)F79re?cydDlQoZz-5Jl*81Qpm$aqLMM8flf>npHLM~B*TpI71OY*bvk0ZH{@ODDd zAP-fACARFF34TNLSTOsHCPfl8j!Th54I;^C07=v!lBjWAilkYEB-07q(x`IKS|5Bx zlD4#&PCSDmNv%4Z6=ble@#_>xbH7N=@ADhse|*Ix;W2xdEbf)XO;Y%p@h6DfM|3;o zuI6U$s=3zI)Ro`1KNh$ZZd;T`W<#BBW-}V(k~&y{?zZucUq%!BQ}e49L=!Yjq7Te%FQAj;lEOeB_1}d(qS|%Ez@B%NaYTN zI*bOXWHdl!Rjm%^0I000))BOr<{VnuOegr{6^e2=D@b1%4WzGouFH9g%D?B)>pQL3 z>59#g9mCY>wzZ{TRd`#t#EK7_7VR(^msagCe;VX+hq@g`gIqEi;IgW8hjRd2Ru%6E zT3d4tEp4U~Oso#EX$=pf zacL0`qd_otDC1!?2qvQeFsmwgI0pb`RV|NzVjgH|bDhAJ2-Nd1RtP3j(F0)axuoYU ze=z^@p2$0`>A4nunaR3ixs*wLUOop;4$KzB9<9yDRrt8+_i;P`yaJPKt?^T|6 z7*G3AQKlG-mv1O$(R8S!zb)T=bPk$7jC$cL9CYVnsbtSH*-+)vEI&;}w5rm78Z6&> zRAxyOCsFpi?7evHay|}@CUO7cVo>vGUG&>re*6%Aa1K6%&ZnOhySPt?)>sp>yB+K_G7oW3uw)m9)I=_#zES|i48_A$Alj57H zDuX!AUcMAlvVI@+=y$W35C{$hI<)eCfUM)~tlJY($WlqZ#B^a!lF+#r%Bd_|;Kz$tYZ{i6s=Dz*}?H{w} zAIbs+UYKuemy@<4TLG7o5+ghTA(KxdK>{#8 diff --git a/summary_results.json b/summary_results.json index 0f12fa079..4b312f991 100644 --- a/summary_results.json +++ b/summary_results.json @@ -176,7 +176,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 0.352127, + "Performance_Result": 0.353536, "Performance_Units": "Samples/s" }, { From a86e05d905f3163ac4f6703f02064e533991b93d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Feb 2025 10:53:50 +0000 Subject: [PATCH 044/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9333 -> 9334 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 21e2757bd..d20a4d6d1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 10:45:51,418 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 10:45:51,418 submission_checker1.py:3270 INFO] --- -[2025-02-12 10:45:51,418 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 10:53:44,568 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 10:53:44,568 submission_checker1.py:3270 INFO] --- +[2025-02-12 10:53:44,568 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 10:45:51,418 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 10:53:44,568 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 10:45:51,418 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 10:53:44,568 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 10:45:51,418 submission_checker1.py:3286 INFO] --- -[2025-02-12 10:45:51,418 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 10:45:51,418 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 10:45:51,418 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 10:45:51,418 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 10:45:51,419 submission_checker1.py:3307 INFO] --- -[2025-02-12 10:45:51,419 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 10:45:52,367 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 10:53:44,568 submission_checker1.py:3286 INFO] --- +[2025-02-12 10:53:44,568 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 10:53:44,568 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 10:53:44,568 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 10:53:44,568 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 10:53:44,568 submission_checker1.py:3307 INFO] --- +[2025-02-12 10:53:44,568 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 10:53:45,708 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,6 +34,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9664 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 4e07e6b3c..04226cf3c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -671 +672 diff --git a/summary.csv b/summary.csv index 4abcc3cc9..1c751f8d8 100644 --- a/summary.csv +++ b/summary.csv @@ -12,6 +12,6 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9664,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 421d9d6d4..9609b03dd 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":48.9664, + "Result":49.3058, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 294c71d4a50e0253f8cce02d6dcbf294724deb50..3a91d4681e41c1d3b354198313f7ffb658c6d24f 100644 GIT binary patch delta 1517 zcmVE0|skm-S&78ldTJ`f88We=K)-_lU*9p$qA!z z$P#HX8eovqPzV_fP{?Rpl7*eFH5`YW0}5i5gVy@s8bxSnGo5(E&jna@I4g=Hr=dJ< z4F7!q`FbBDF!mmVX4EzD$D!OyjJr^|8kksZBx$y5Zb?uDztes!aA(~X6id`NHpLP( zh$W)|ESa5tf4~woh^45)l5qf7Ryb%0`YhobTG~t}h$Xe^FjlZz!+G{8mf}BvL=7UzXaGsnAd;wYU5cbxg(TAneWX$4ptU~uiX?4mGo5$_MUq-|I4j6t zQRCMslIDJqoZsg+!ug6x!ejO@S==j&o22kH<4+K|kLY&FUCqthRdcPasVo23{#f9n zaND9he=-l$>BDSBgFG@C;1M;*Bcq{DwAUJzN5%p0SmB_xKKRNbZD})|!1fMUbvP^F zk?OdRkhC_CGz`u6HY$(GOl~PI5m+!_+muPvI6h^PDLeos84WOr z8f22u0F!nVlZ*pkvcf@Yeejh@+R|n^K_;nHe}}Qcw3YD;X{-HjXL26_?v%LN<;2x+ zTzeaoTWW9wmUq~+2#3+QvG6lFt=D&2vC|csB|C2DxN3z-3kG4(9;4tSa6Sf3&vd99r5;C&(q#=rC58!ZMyAh23+R&i{wY zcUr)6Ey@)zPpJw@KDxy`D{?lOj)}t~@qBx1N=VkB3YnStJa5Rbg9~Xm~Pov-F z^5ci_gLCj9bUyv87@w4elE?lcN6n*K^iq7_hIMo}3x@I{NQTigRmO6V2U^$iydqh? zN4fZ%#k0ky{MY$?oMrLk<=aRGeVG*BOjQ}garW}1n3DDTs7Jp8vzZVG4g|$mgm9Ck z9v^?pYQr!PMfd#*qrENLX-W~c18vEwfkItcvJ=uI5w#vjYMuOj6~B^zS6zBI zRb!#fr(%Grf-}M_P_J!O;OinBuA$Cv!ijeR_|qxqGjGEpeIM{B1R#CfQTbdvztkmb zROM;DjMF^M@;Xb&azXBw+w;hq{tHkJL;g)10e$Y?~cPyzq|5dZ)H0000000000fPs{g T4J1AR=#xw&BLNcBjt*arpg$gi%Mv)u>E0|sfn<7~;HldTJ`e>GFV+Ff4oM)e6DgFak?jy!s zHgHx0Tc@kOWo~T1l)2;|HXRFmeq$SyOVl_HC8Gf@QG;Bf2Dvnw~Xc(v~(C3H_l6RvpF)xkL?eX}oJL$=Cin-U$AWpz zXi_9m)v`F(yPoUfQ9JZ2A*#l5n)NeW*x{sfWxh;FCc)!fWoHP_mjy7G_hj|Dyo zw=K#efAc_{KFnq`$Rncx9#MllG8+0sd#z!4WE=pG6%JbKgReZ&mNwG~Z0~?ohqD46 z84diRwB8Yq|HK~8UIN?+Noxa1!_a(hqw=WCRR84W;LRf@wo04S@9aRgN3KueqH#4{{n|(F)H?M&r}^EwRs<92UxQ^4gbjmaoSoUF`9Af9m(-JFU^V5t~Iij%HZy#&xw! zf8}P8j__Zp#1apiR_QPrmzL=;8l-ZELLEkfR5BW%vZ_{xa{yFURqF^^Omhw`ZKf04 zd4-}J&I;03Mg!^Vp6hbnqVn(AdVQxAJ6*9^vSXN9-L|$AtO{=nmss&()1n+PpvuHY0(%+WvJ~{`@A4a`! z77n`eu~f3>nQW-?X_lX+B3f1HKMj^|Ju0&#ijydNUiMzRb~zshN0Ye!aWSa*H2Q5W zKYj>5I0qj>=hM%M@kwbYdF(IJMm)MjFU1FLSVxDmU??wwWEf3TWh@7IpmiRFy#-XD?rhDOtaddh|Q9nh*#M1c6cyos*^> zAAiki!!Qtp?|llRyDiJ9OA)pMZON&DLQ`6D6Vk>JwSJJ)I(hplPU0lsQ;(~e{dRUy zHQ%W&`32{t(KRAjjuOz^2&vl|eXJJgBT9T=TCmP&sL>ugnm2b9x0D+PZ_Ze7L4r>r zNPEg{jkY0Jim~TgP|Ro1p`*AlPO%V6*MDNm_%~|-7kU1G6$B<&VE6z`?MOr?q~Igc z{hQ9;((e;aLOI9U|dt->mnYmV?He7i8m6&(?04GZ>L3$eISw;fF9$XmCxDp%Oz@o znj$aCG%r$8tVmApOInikVdPE!1*lF#{!Kgsj=66?LGuT*=pV`g1@FkOu9=g!B3l8O zlM^F60V9)8BS8&7761TvY%gZk0?~c nPyzq|5dZ)H0000000000fPs^f4kSJS=aWq&BL Date: Wed, 12 Feb 2025 11:02:04 +0000 Subject: [PATCH 045/719] Updated results summary [skip ci] --- README.md | 56 ++++---- dbversion | 2 +- summary.csv | 22 +-- summary.json | 314 +++++++++++++++++++++---------------------- summary.xlsx | Bin 9334 -> 9334 bytes summary_results.json | 6 +- 6 files changed, 200 insertions(+), 200 deletions(-) diff --git a/README.md b/README.md index d20a4d6d1..36c837015 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,39 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 10:53:44,568 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 10:53:44,568 submission_checker1.py:3270 INFO] --- -[2025-02-12 10:53:44,568 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 11:01:58,997 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 11:01:58,997 submission_checker1.py:3270 INFO] --- +[2025-02-12 11:01:58,997 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 10:53:44,568 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 11:01:58,997 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 10:53:44,568 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 11:01:58,997 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 10:53:44,568 submission_checker1.py:3286 INFO] --- -[2025-02-12 10:53:44,568 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 10:53:44,568 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 10:53:44,568 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 10:53:44,568 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 10:53:44,568 submission_checker1.py:3307 INFO] --- -[2025-02-12 10:53:44,568 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 10:53:45,708 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 11:01:58,997 submission_checker1.py:3286 INFO] --- +[2025-02-12 11:01:58,997 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 11:01:58,997 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 11:01:58,997 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 11:01:58,997 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 11:01:58,997 submission_checker1.py:3307 INFO] --- +[2025-02-12 11:01:58,997 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 11:01:59,960 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2233 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1858 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 04226cf3c..f12f8d578 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -672 +673 diff --git a/summary.csv b/summary.csv index 1c751f8d8..9cc44ef15 100644 --- a/summary.csv +++ b/summary.csv @@ -1,17 +1,17 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2233,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1858,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9609b03dd..45fb49ceb 100644 --- a/summary.json +++ b/summary.json @@ -1,24 +1,54 @@ { "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", + "Scenario":"Offline", + "Result":0.387358, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + }, + "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":18.5738, - "Accuracy":"acc: 76.000", + "Result":0.431415, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -29,7 +59,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "1":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -59,27 +89,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7438, - "Accuracy":"acc: 80.000", + "Result":18.5738, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -89,57 +119,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":22.9842, - "Accuracy":"acc: 76.000", + "Result":49.3058, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.426752, - "Accuracy":"mAP: 76.951", + "Result":0.353536, + "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -149,26 +179,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0543, - "Accuracy":"acc: 76.000", + "Result":1.7438, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -179,27 +209,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.433634, - "Accuracy":"mAP: 76.951", + "Result":9.45684, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -209,26 +239,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.406474, - "Accuracy":"mAP: 76.951", + "Result":0.430616, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -239,37 +269,37 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.387358, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.406474, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -279,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2233, - "Accuracy":"acc: 76.200", + "Result":10.1858, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -289,7 +319,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -299,27 +329,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431415, + "Result":0.433634, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -329,27 +359,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.45684, + "Result":23.0543, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -359,25 +389,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430616, - "Accuracy":"mAP: 49.593", + "Result":0.426752, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, @@ -389,57 +419,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":49.3058, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.353536, - "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Result":22.9842, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 3a91d4681e41c1d3b354198313f7ffb658c6d24f..ac7882ca75ec8ccaa9073cabe00c44078157bb8a 100644 GIT binary patch delta 1380 zcmV-q1)KWzNcKpuG7=7^mxG_{0{{Tl4*&oZ0FzN87JtoG+iu%95PjdTAha*-gKo5S z3mg}Q?F2@1F`T4GpA1?W+uD^#tw_1a-!FYhnrbAsX|liq1`MAec{np?hU4C+hdd)2 zC}k@6so`3VL4e01N%`v3xVt?wLxV`gc)~Kl;na8pX?*Jc(UVdUy_L&TW35ziV%sua zL(Zfn3V-0*vJ_>`l>S++Y+01R61fIYd1h0`@$H2xH>f`r3R^Z`cDPJJrYr) z-;-&-r}~#7fvjh%e$USqEnzt4!6=EMqyNX5cK$D5@K;m&+{W zP={~D-@v7f7$&2Ji)e4iWJErIwX%o~2N_gJs((J#R}Muc%FpnKT&xy(suYxU!(hYG zj4jeERd~~*RFK!vD;+{~=V>WokW!%Yi$4)#976^rQ+P2$i*LkIePbnx-IY-I5_b}k z#sq81s$?;8a0E(wkIotDGwPUUIi}+{ZVOK9c)}FZveFT^w;HxDFNPw|h0gbCJ!i4f z6@Oxuu!K?<2$*qEnZ=`yNW7kJsO9TJ6Ie1Ww0SJ}GF{n61$LCITIeFE%#7)Xsrf_b zxAS%}Z@FSf>};=@44!%FD_RZDTkD4Bs5BUm2dHAS>)D6dM{*B>Z@<3WLSha?F64)MF@P?SL0#k&s;*{9@)wSSAPO@W(T-<^-$6XLs8;82&49Y9Y=C$J(v zhz`Jo1|qBDy8$7$*Pq8%H}mOuHoqBMPA|sW-43Q>aeP7o&k7xU6ObIxT}n_XXAOo0 zl&V2vt3v(N0PA+HvuP&ja#?NLzWIRLE6a-v8XDc9b~-J1w*~iFa3qTZ9<<;Z4-2`=gh2fW2!x5Zz##a|cs3&{%V4d77DW}d}+WF8mAuI>fr52Y@wGzV1l zZQ5vJXzFJx+|T05rCqy4MQ=X$7i8GNk+WFHm)-G|k=oXlhoteVWIwjz7ew=oYf;B? z{|AJx)51>}kH2!SCHvuJs4=NczJK#?>zEr^^o^0k&**1E}ShAF9`IDa;5)$JA6^!l$%g68orlhI^A z&f!e^1Y~x5Pr6QovhyO$M@;09f%=lCiYAAO#-VvqDcP&Ec@n~+$p_OD(tnLCFG6qg z)5zAL7WKjxBb#sez8`#_*eJEU(D4a5A79Nc##b8g(ThI3m`vw4!|RzI<6K{lcwyww zD0J!eL&(`=^kk)aBn+t+gi#Rrp3~`|T@6aWAK2mk;8KeJ92R0ssLXxUGbrXC-E&1%Ci5QOi23ZuI%%ORx|wgYX+ zsewXMT5=Q8#u2rCkkmSU`zlW2B;ZqztC{_Fc2T+5t2X%s=cUmVB3X_S(A)^An+kob zm+1pad|+Cz)@Z0u4<0S*yOLYVje|F5EVv-SClRDQ<+eh*5G=*m^BpMWv*^%K+#07? zh^1?Pux0$4HGqpezsCv!6D%-%0H$^%q7zc^5ov#1I{*a7(1HRTJSG{z7jFQk{A9zR zTsf5ty@g5ZSsdMaFGpk7by+tXtT8{qU#r*mVbxUX!>Kr+x)hvp2P_y@m-xDfhijP+ zt9atA1o6~Ied6u3$gvMZ5(Cg<+_UmIdwy9YFHv3O#XQZ6loV@1Xin(-adQ}X(|-Y~ m(~y4?&wyj@+fPvcv*#a90|cg*gP)T*Bf12%XxUGb_alFH?WXzw delta 1380 zcmV-q1)KWzNcKpuG7=8W{#!}w0{{Tl4*&oZ0FzN87JtoGZExZ@5dMC@!peQw{g5{U zc|pLc0$QP^6w!9o{e;|Ppt(!zTd1_2(4IN|HQaeH%a1_qIe@tCE8L*IA;X?z;|(UVdUy_L(p@t{=RwQU(a zK*ppca)033vJyqcl>S+)ZJ8IqV)+1|vec%I zN!fjpCJJwQk_xgaI@2LUcb*p_0x1PLKmQXkMiHb?FohR0wD?-A)Hhb3*v*8(m$;P> z*Ctp~mIaHDgCkJddvwlFpHam+mSZ}O<2K;5ipNYbEh`;yd#hplayk@QCUm~n4@(v) zU4J2F0V^nUfq)t1rI|nJh(r&|4YfRdXbdZ+g|>_YUnOh%q`*#+mG`;`N;73TVruyq zc+I?R%v-D(5WW6g%hsylB`P%rM}?RxrQ@sZqt;M?zSzDe6*=MD+s zazw`SuS4SdUW<5bb0`WR&EnmMhU`;v!hhPv*1Etgu5K^JZkKqj)p6e5GbM<*VyI#)6i{Bi zdniST?35RtxW72Wuk(w!~rZ|5#Y*p!5w0;u8qRGzYU4POJtxkyE z=I4>EMJ?J1jz%`m@;uM~KCw}1bpppD@6aWAK2mk;8KeJ92R0sscScGtsrXC-E%WA_g5JmU>3ZuO(+i6M>wgYX+ zs)0gXTCx+;BoVb9NNSz@eHFiwfLC3P?wvDt1{JHLY@%PVPH0u4EJ;xWifJy?zC<7O zI(|fv^Hg!#XayxYfkUh6u3&~RZQ;#o1J(<0Q2;4Nm?_c0dqXgG>;RIwBsf$MciKwo zLuvPa*iiOO_ki;>eZUet<VKzXh4GU4rfV*FJ2E;y4i+K zxpGS3PX=bKqd2*bPE5wOZIgDlJipW=YgFZFzKqj6&hk1-$#OyNm)rBmoBj(> m4nzJ;907gq+fPvav*#a90|d?fTS=2SBf12|ScGts_alGw&!NBo diff --git a/summary_results.json b/summary_results.json index cd57d9593..3341afacc 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -55,7 +55,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline", "Software": "pytorch v2.4.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2233, + "Performance_Result": 10.1858, "Performance_Units": "Samples/s" }, { From c8f071c0d25344c3564bfcea624c49b8b344bffd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Feb 2025 20:00:53 +0000 Subject: [PATCH 046/719] Updated results summary [skip ci] --- README.md | 56 ++++---- dbversion | 2 +- summary.csv | 22 ++-- summary.json | 308 +++++++++++++++++++++---------------------- summary.xlsx | Bin 9334 -> 9337 bytes summary_results.json | 14 +- 6 files changed, 201 insertions(+), 201 deletions(-) diff --git a/README.md b/README.md index 36c837015..e1c9d5e9d 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,39 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 11:01:58,997 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 11:01:58,997 submission_checker1.py:3270 INFO] --- -[2025-02-12 11:01:58,997 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 20:00:48,036 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 20:00:48,036 submission_checker1.py:3270 INFO] --- +[2025-02-12 20:00:48,036 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 11:01:58,997 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 20:00:48,036 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 11:01:58,997 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 20:00:48,036 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 11:01:58,997 submission_checker1.py:3286 INFO] --- -[2025-02-12 11:01:58,997 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 11:01:58,997 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 11:01:58,997 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 11:01:58,997 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 11:01:58,997 submission_checker1.py:3307 INFO] --- -[2025-02-12 11:01:58,997 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 11:01:59,960 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 20:00:48,036 submission_checker1.py:3286 INFO] --- +[2025-02-12 20:00:48,036 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 20:00:48,036 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 20:00:48,036 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 20:00:48,036 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 20:00:48,036 submission_checker1.py:3307 INFO] --- +[2025-02-12 20:00:48,036 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 20:00:49,184 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.45684 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430616 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1858 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9842 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0343 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1858 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432561 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index f12f8d578..25b3fa08e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -673 +674 diff --git a/summary.csv b/summary.csv index 9cc44ef15..a1ff7d53b 100644 --- a/summary.csv +++ b/summary.csv @@ -1,17 +1,17 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.45684,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430616,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0343,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1858,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9842,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432561,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 45fb49ceb..749716fd5 100644 --- a/summary.json +++ b/summary.json @@ -1,54 +1,24 @@ { "0":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", - "Scenario":"Offline", - "Result":0.387358, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.431415, - "Accuracy":"mAP: 76.951", + "Result":18.5738, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -59,7 +29,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,27 +59,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":18.5738, - "Accuracy":"acc: 76.000", + "Result":1.7438, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -119,57 +89,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":49.3058, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":23.0343, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.353536, - "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Result":0.426752, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -179,26 +149,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7438, - "Accuracy":"acc: 80.000", + "Result":23.0543, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -209,27 +179,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":9.45684, - "Accuracy":"acc: 76.000", + "Result":0.433634, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -239,26 +209,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430616, - "Accuracy":"mAP: 49.593", + "Result":0.406474, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, "compliance":1, @@ -269,37 +239,37 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.406474, - "Accuracy":"mAP: 76.951", + "Result":0.387358, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,27 +299,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433634, + "Result":0.431415, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -359,27 +329,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0543, + "Result":8.96557, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -389,25 +359,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426752, - "Accuracy":"mAP: 76.951", + "Result":0.432561, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", "notes":null, @@ -419,27 +389,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, + "13":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", + "Scenario":"Offline", + "Result":49.3058, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + }, "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":22.9842, - "Accuracy":"acc: 76.000", + "Result":0.353536, + "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index ac7882ca75ec8ccaa9073cabe00c44078157bb8a..086f323a7c00dd7622fab6ef79da83a5280c5fda 100644 GIT binary patch delta 3609 zcmY+HXEYlO7ssQtXe;)P8L>CDM}t~18`>CE9=iCqX+~4`%|A%|d{ce0>-0+e1vVdEZD;BdzYl3o}R#E?T z+8WDL+3x>w(x|06Tag@H{knRdQMQwm+@nQw-GElu<(lzSk$0+u#at+8(Y~a|5no$3 z@uvB-m}q$m(WX)zlG)kD_N%@@k+U1TCRnM&c#~->qi`K<@{u%XSZ$~UDJe$GhUN@F zog&SXAFG(Vk)QJtT^B27P<+}zT*~Wx|Ew>oVD9TUt7wZ}J87|#UK7bdsMB>_-dmGq z+8UpYh>eQ;ki7V$$I`$8-GurbqvWsMd%`ErYNPIqSbf2)&m5BMdP=8Zc;H<9R~N@} z)BC`sfxOH->#03bAkG$2 zRb6#FM87So?DQ*O4KYl%F8NMZDck=O_A{^nR0MDz}(!|vroo{;P8-~c2IN0Z9iVo z`TEhQLa?<9^DycLn7J^A*YGcIQ(ta+ELzj>uUZk_PX&<#Q8hv8f8lyXC%W;3aCpFJ z#cG{tBQu_8^dMFnyC*9t&9gI~Q`in^7Y{IQ$g@zvwa$8?ef(E^Lj&Tj`oSj*5_|B+ z=~tmv$Gyfw+~4!Ev&z-oL}(KI();IfixgOzlw}O{rX@Cuva>pTlvoryIMWl}`_e+L zU>9MAdhS0`v$JN;#*EE=dj}mikG`A4z^?M5@PR*tU80$L6tu}owY^)&6LIZyYYMWs>i-dHd(B&W;vKw>vN8 zQP}SHZV#l!-}r0PCQjPPV9iP>vm$ai__^C3u3?4+d8%})qp-9SYH(e?9qP@%vnT-} zc9Ux9B$abN5}}TBjU|muzuBxWl!mdGq`$SAI=LK{tG52q6G!vHRfJ6=7Z9ADkiIch z8Q|0}2xQFG?&dmY)95z~R2_&@7LsEpptuTiw?Wd^FKDxOZU|7&tsbgi^pPVY>*E_7W7H!tz%#!S%N(bl$#x?gQ)jc;rH=W$w! zygR{4g%g74^ebTo42Ms6X3DheZM77O0qFn8)Vh#4Nw2j;RSopGjM7r%A(XW&F{97> zl}g^w5Lk1jd2*8QAIu+;e0x6Lb`)-Ke%mI2sSJo55rKyR%oKT?FB}koGgF(**_vHY zG5?{O{X~z;$B;BV`pfF+?6vz{Nab#5>4Jwj z@Y;s^;oQ?e`y#ZL3Tj~O41YTO$i6HHJe?K{8f|cV2+PPs$WfSoOXVR}T_z>@4A&@% ze1`EhU#u$oJhMyvd6k z<*x|-H{Fw|#dQd|SqV5I;^Ea@pHD1OI^OmzeV&f?trsv7-%cnOL*TGy45^C3H%8@e zXWIq#^chwEqdf~&IpI~lOKwTwOZfX zzV8pPzt5BDJ>N(e7em~zC*XaRDgPK%jGZkB{Nfp{{hXs%1LV??RyuoJ3J*=tI7Hho z_T=dggC#HX@z#A+Y>j3<3_CTmG@It8k!!Bb?_v6kEvQZYUaPDV@^fl@Cs7kse~9Zt zQqUucMCYzBeFe0rKI9bTbMU1si2TcC3g+?Yg(fcyf+`tFl6+o4RLq;GGClWd2-~&B zR#@j4o_xjqIBX@&s;Q-7eh;3!aK~)7V}BhB{aOJ<(iL0kN)3pWm$EhX7^iJ-zR49J zxbX2=#Fuu_QCyUVa~nZyvD2->^~Rig)?>O`!@8#qQJXwy z&br+F5ed6D=*QnQn+uqpQ2{#xprAc!0N^Jh0Kf*oSc5rGqj!BQ*VLGPNB8uOM9rRM zskVpfmrG4?*!W1;#fpf92Rt{Af#d*dK0R=7fZBMCwBH_^o4Q@=$j|%8l-;OJ zW=(Sl+hTyCa&(x+y~k2?Nf|O}EC}H}fKAElR6wMrhlHxJv}9c-gDD;=S@IkHuF?=! zq5v8SR`f_?#^~2gL5|| zKhU%H?yBDk#qj!M0O?kXDpEuA`P(^!L2`VNjR~qsNA+%|f#^v6evD&{?^hcM@xmmG z3`{As&>i1f$p12~D`hN%Zlaj8{0Y=Fc1$kXC_Ughcdpqd;&qvBwueUV`B9(&#i3+!Ju;qk<=ADh0! zzsgZk{n9QB<+4ojYdFO;TE6UK-`4W{dPZ~S(H8QZ#+BV**BKNL=J^G{W!-#*dQLw_G=4wKfDYKp`WGx_T<|o zsl)hfz_Yhvd%I(gU zi2?HTxC-GpF%Nbcnw+{6^~G|iAs+0}m^I$H#tUw(mSHqm^B9iyCXPS3<1mBE-1yB% zw_K5Um+OA?#**pxJ`&ma_B<+bZ+T2XwP{-uobr6){I>X>dxR0lp(!80*5q8iKoaRs z*$q>U&i=BtyQ0sft5VT}pid5waix460n5N|!veZhyx+L{mEj59xZj@*!A2oyTT7rVy6;CN^0rh*lzngS2+ESf=%s+2Q^L~T zU}K$;%ueF?fX4D+lXP-}zE5ITyJzbd>O^jJGL?Xl$4VK4mCl7LUD7F*w09ocI>WM?{mk5GHw z#dv<%3Pw%C1^@u)noT(nbTpKGFcMS2FNK=*f-$vG`wZh__2Bk_tQ1k^*66G$jp)-6 zi&c>oIJz2p78+{Zp;jq+a(SzKf_+3Oj)lBHhkwNW6CpRLZW`%q-!R}=X+_H~v(uU5 zFp?3k>LE=Kd?&{IQhTFK@03ZJ`0=Z@_-6Aq*?NKJ72-+xybI}Ab)HI=F1-KhB^0Z) z`~o#xlVyTl;`m96ZXGuu&eii~ERcW8E>ms^U~w{>1nmfvBn;{Zv8$g77o$NsfGvkcBz@3AB2l`cf+Ry%ZuWVO=Cn8r~ z3=%vxg1-N1a>Bh9B9@%)-LQog~hOfg?az~dS1Bq z--w9?W30eX1_KTNAk14XC^+2PEeIn7X2Udrc`mhaVaB=GF%U2l?;o2XQ~-e0B>;f= af72H=CiW=?D568Pi!m1gUnv&))A~07jKfm^ delta 3601 zcmV+s4({>!NcKpu*arpO8YpL)lideNe=uvC9RB^c{a_qVWkY?Isrt*&Plu0j z@@qPhGE;H~)3l+EvTSys>uETWlOWaNnVjZF25~Y8vixf@)YDlagZ{#3GS-cHf8Eh1 zK{QpD9uAUSe8hu66w1$WIG@OAwsexnaggQ1Pe;)#RgVt)QGTq5M@i(Mp&pBa<`3bk z`qAN{@4HAorW>D?B9!|${tYyJiPD#YU_Q>e@#8lc4M$l%ZFXT2 z#^ZGHSD8flTpFq}37#6NsVa|Ae?J=)j+If=@5^GMRVAFKSv>i%JS0{QmZrj4*^0VY zTl<=@>W*O-=i_-SS2p^x*To1w2U&1*n8c4tvgje51;v>W2ZoVPOIWmiFPe+7>8pnB zOM{ZfQOw@Mk$I>;6pmMlYie>s8*XXC-A03+k7qq7gCCRuW2l$M&_#tseN*oewrIS*$HXH4+4G!FL7bmFj%8Di`{IcRRDg;5{3nL*b>?|QbwGP_rdgv{=DMq=`- z&EZ#@gUgdk3LAe?Ien2ZSHy2j4|Jtfn73Yzz1*SBX+s-3@JI+3)TYCn0@Cp%PxqM);a&UNQTlp8yZI0XnwDS@kF1D^S_fk)5$;0WHRVQ(}7I#d`)NcshuN>d$IlW=UWSf8KouG5!QX-l2Lq>dw5|3OruiQNnIUMzZu=qJs%YVHrUDbv;kF)12PQp>| z!O$G7UJK{Kuy!DEAEE3(;;p+uqUCF5-S&78Jsw04Pc%X31SJkxVoj6)h>V1Sh!Oyi zkx&r53J@7{3SxzM3(WyUuL4A-cM76ML1fMkxnF-@qQoQD(kt)Z1L8hH*#X2muXIZ^ z?fl=CTcXdC=;N6ti|Yg>4q3iUlmLm0gp!C7AQ2@lSnVo*X5;cgWsKS4eaR69WIA{s_Ea4nl+Ds>iCAI1>RVMAG8*6#HOM7u zkV~VAOU40kS>d3yKKRNdZE16n&>xCm)nTlVOVl8j#=GW{{A~Q=NbV!Noscxhrm76E z#Fpn~f`1@-ESTq$Y7HEJeWX$4ptU~uiX?4mGo5$_MUq-|I4j6tQRCMslIDJqoZsg+ z!ug6x!ejO@S==j&o22kH<4+K|kLY&FUCqthRdcPasVo23{#f9naND9hG7r@0!)!)_ zJTe;K5jDsoqoGf<*BX{b#sTnH;h?oX_{t+~X)~R`_6}HeI4j_Pk?OdRkhC_CGz`u6HY$(GOl~PI5m+!_+muPvI6h^PDLeos84WOr8f22u0F!nVlZ*pk zvcf@Yeejh@+R|n^K_;nHhq1!6mGKN|tNm|havuTil(^dE#MN+IdmEHnYH$RWci6ND zhtasS3Ww1klsnXa;V>G6lF+EzEb*{u zl@6nEX_*eAK`M7B)L}G8C8Gf&P2S8<2wT__0H0RKN(q=lromVKz;jAEiWi*h! z?zt}KEh_(>t=D&2vC|csB|C2DxN3 zz-3kG4(9;4tSa6Sw6^9PTG~t}$R*V1FjkntGM*uY-E*1F|A)(WTEKHH$`w48X*%Y% z3Icx%m{=WuV$&KPM&r^V9!7&;?oh_VXb?5;*{cZVw?xSzP zpJw@KDxy`D{?lOj)}t~@qBx1N=VkB3YnStJa5Rbg9~Xm~Pov-F^5ci_gLCj9bUyv8 z7@w4elE?lc&7)iNQheZsb#yojhVmjvhS4-t#&VDcTG#TtB3Zsix%iyLv&E@lkp-Nv$qg>0STs;gP-dI007pL zIuad!+iu%95PjdTAha*-gKo5S3mg}Q?F2@1F`T4GpA1?W+uD^#tw_1a-!FYhnrbAs zX|liq1`MAec{np?hU4C+hdd)2C}k@6so`3VL4e01N%`v3xVt?wLxV`gc)~Kl;na8p zX?*Jc(UVdUy_L&TW35ziV%suaL(Zfn3gFs*vJ_>`l>S++Y+01R61fIYd1h0`@$H2xH>f`r3R^Z`cDPJJrYr)-;-&-r}~#7fvjh%e$USqEnz zt4!6=EMqyNX5cK$D5@K;m&+{WP={~D-@v7f7$&2Ji)e4iWJErIwX%o~2N_gJsy^0# zR}Muc%FpnKT&xy(suYxU!(hYGj4jeERd~~*RFK!vD;+{~=V>WokW!%Yi$4)#976^r zQ+P2$i*LkIePbnx-IY-I5_b}k#sq81s$?;8a0E(wkIotDGwPUUIi}+{ZVOK9c)}FZ zveFT^w;HxDFNPw|h0gbCJ!i4f6=Ifuu!K?<2$*qEnZ=`yNW7kJsO9TJ6Ie1Ww0SJ} zGF{n61$LCITIeFE%#7)Xsrf_bxAS%}Z@FSf>};=@44!%FD_RZDTkD4Bs5BUm2dHAS z>)D6dM{*B>Z@<3WLSha?F64)MF@P?SL0#k&s;*{9@)wTrEPO@W(T z-<^-$6XLs8;82&49Y9Y=C$J(vhz`Jo1|qBDy8$7$*Pq8%H}mOuHoqBMPA|sW-43Q> zaeP7o&k7xU6ObIxT}n_XXAOo0l&V2vt3v(N0PA+HvuP&ja#?NLzWIRLE6a-v8XDc9 zb~-J1w*~iFa3qTZ9<<;Z4-2`=gh2fW2!x5Zz##a|cs3&{%V4d77D zW}d}+WF8mAuI>fr52Y@wGzV1lZQ5vJXzFJx+|T05rCqy4MQ=X$7i8GNk+WFHm)-G| zk=oXlhoteVWIwjz7ew=oYf;B?{|AJx)51>}kH2!SCHvuJs4=NczVmN?>zEr^^o^0k z&**1E}ShAF9` zIDa;5)$JA6^!l$%g68orlhI^A&f!e^1Y~x5Pr6QovhyO$M@;09f%=lCiYAAO#-Vvq zDcP&Ec@n~+$p_OD(v2*CFG6qg)5zAL7WKjxBb#sez8`#_*eJEU(D4a5A79Nc##b8g z(ThI3m`vw4!|RzI<6K{lcwywwD0J!eL&(`=^kk)aBn+t+gi#Rrp3~`|T@6aWAK2mk;8KL8<JwSJJ)I)3{qPU0lsQ;(~e{dRUyx!9{V`32{t z(G?1&{5nPr&x%kYp`Yfn>B!oJio^Z0)G=MFnj=}b|j(`Qt%OJe_T5N1jo>V z0v$Xi8NnBC0H^$9!=PL_l?=UwN$Xi0-Fq)bW7lBsnGdUY;;jVn)JJ{d?X<|T4@43J&|}=Q@;Q5cSuastKN1#`Y9lHEL6emuIsvnj&?6=rv}oB+B>?~cPyzq|5dZ)H0000000000 XfPs{g5hOkV=#x++BL Date: Wed, 12 Feb 2025 20:14:55 +0000 Subject: [PATCH 047/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 9337 -> 9339 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index e1c9d5e9d..3f05f8a51 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 20:00:48,036 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 20:00:48,036 submission_checker1.py:3270 INFO] --- -[2025-02-12 20:00:48,036 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 20:14:50,315 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 20:14:50,315 submission_checker1.py:3270 INFO] --- +[2025-02-12 20:14:50,315 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 20:00:48,036 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 20:14:50,315 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 20:00:48,036 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 20:14:50,315 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 20:00:48,036 submission_checker1.py:3286 INFO] --- -[2025-02-12 20:00:48,036 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 20:00:48,036 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 20:00:48,036 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 20:00:48,036 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 20:00:48,036 submission_checker1.py:3307 INFO] --- -[2025-02-12 20:00:48,036 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 20:00:49,184 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 20:14:50,315 submission_checker1.py:3286 INFO] --- +[2025-02-12 20:14:50,315 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 20:14:50,315 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 20:14:50,316 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 20:14:50,316 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 20:14:50,316 submission_checker1.py:3307 INFO] --- +[2025-02-12 20:14:50,316 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 20:14:51,463 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,13 +24,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0343 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8636 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1858 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 15.1819 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432561 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 25b3fa08e..a5b667a16 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -674 +675 diff --git a/summary.csv b/summary.csv index a1ff7d53b..370e9a030 100644 --- a/summary.csv +++ b/summary.csv @@ -2,13 +2,13 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0343,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8636,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1858,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,15.1819,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432561,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 749716fd5..a98079c37 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0343, + "Result":22.8636, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -169,7 +169,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -189,7 +189,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433634, + "Result":0.412678, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -199,7 +199,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1858, - "Accuracy":"acc: 75.800", + "Result":15.1819, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 086f323a7c00dd7622fab6ef79da83a5280c5fda..904a5808b51107f64515d6c9cac56c677c03371d 100644 GIT binary patch delta 3586 zcmV+d4*l`@Nc%{z*arnx_ndN@lide3e;Xg8kX6FWWI_nZki+P^LXK&`T5Pkn$>HCx z4TM0wg~OgcqFs%Yf%aR~U2Q+O++9BWHZdByOrgtE-BPgJclKqo#U$^R?!x zN|yWaz@H?sY^u*PQ-3-7>F_a0f6c~H=1Sop&YJ2t&!-2vo&{qW`I(kXWn3H?eSAT>X~L}AhE<-A=ZRd}=!;(GBmC^={?TEYJSyqDhivMX zXGRs?u1c{HsmVubf<2k<9x6;4>f4f=wT(lIb$T`)U+81IXCBwgv@P+k&wx~U?gOA zyC^Zp>|QbwGP_rdgv{=DM$+J~Hk-fNY`k2_tev35Cc(5&Vvt$;%t*+reL+csbo3P^ z4!KlLUnI;g;y0!TdZksEw_lFE+@a2CLkBx>iIWRz(`8NpvAJa2f0DCw^*Zk1(YOCy zed%@1y5}F~$xv2gL!&Ss&+fG#iS%ev{EdoCr}#9L>98NhLzx!&n(XH?%lp~vJ_@rk z^~w4VmR8sERLXghrI)#se$ibD7%8b3d3)Yr%9@OXBz`oyB>POx2dE zevrh&aHRgM4rK0!f0Io8bf^!R+x7Q-gj6K0vKZR@zb)gsUW`jDQVh*(7?v$~=o}t8 z2hTYDjYo-1E{YaP06IoOp+gBk$4JOnPMr$q7;|z5IIA#kqq$24c)o5Yb2VU)~H)952DM1=;Dbc2%VtBCYM+XB>*BLp&+6JKx8BoM7IJ&#+-s! zVcte_0MV@gk?Ea+=u!}w^F!{}m&~Zhw3_WR-?=a>8hA z@_XE3G{7LIp%5|}ppelxjDev0O;fET^$0K^6Q;`?0{4bxTkzQDa*a zOVl8ij0Uh|R{8-;)F76k3QNWTU|HdyE$CjtIka?`P7q6K)n%+;wT97nb&93<4`8{E z7t1f2+ ze>^f8c%n4l5s&}G8qZz=+zCl@4N1e$TzlIptJqRpBCueqR=YW|)q-rA9}r7wHJ^l}aq}uxOPIqj6}N z4x>RTcPP|hG)N_*0V=C%bvOq=WmUC~pv5%j(9&T#!If7i%HgaaePuL|zV5j$=PfG# zAD*SxcUrO26YC{AMnkJx=9Yq0;cnp)D?Th*w8Ll|lROU}0(6*@T@M_8DT+LH+j@_; z1x&0Cv1knsqj6{v52HaacPQgwGzcc60Whm7c{m3EW>qbZfMOnK>2RIEmI&1IFjfdA zY7oplm-M^^=3m|ud8ai!*TUOmvTmCWeMd}RduK0_=}5NZWRfXCQq0-Hbk(|ek+CNZ z?iTO-y~>LYlXwu8Ws3fPWbuY#9>yaj`)%>=qkYi&Vbn`!!LT=*NF{%s%BCuxrp0M0 zqg9o`({S3HV}R9R}kvU?t^Z$bu%0<0^13U&Bd^j zB7HI#Y3!(#NL8e~$={d0BKIL-KHD&J4%>FOPXf)=iE(>#W`+ilit&VHg2RdN1k(62_^mIcB6=&AC&q(PMbEZn`~W$VmMDO0f6G#o zIaB&)xw2(Z0!!orfXXwQI*xDWEae7?Ma9*LK`Avz#nXQ(819g$V+{H-9rV@UQY4V| zZ8hlI8(;^{W8kbzMIFE8sgiYY2C~Xj9nCV9GinCTy%|My-JO zN_&sa8R|3Ym}fbr<2Y^$PV0EW6w|WO5w|xQwl6P+BF}}+_v&H6Vx=p@e=K1Mr7jRK zx`uMbUN$+XZGvEa*eWgivTQL^e@7eQraOh-&D9z(yKw~Kkp6+>cY zJI!S9!c$+-YIxaNH@rlp!GJtK6{B6xKFvOpI}m*H{VgzQH}dX~5H3e#JpFS>g23+( zziSRf38Y=TyU>tbN{(2&f7sd-xY_mX`Pl6d-?ajVx`b>2+9RF7iu@qj0~Z>Itd8#n zgxp+z9be5C)A4LEKfRn@j5oU-OvmE5M*`2%{Se&*BnNbt5>(1rgW(=Z)u6Feq5f)s zbvxJDG?R3>tTt`me8laQ?f_p7ElEoemTJW$1e@8yK!d_k#1sQWsX511kD9Z8R}7^|KZ3 zXL04yuHB-dcc1$cGHl_tYorNa`Uu=g_WO9e|w}GSzd(R@8_|uLoMos z2VI>-g`M3yROT+ z*eIHg$rS4C~0oA48lsjO-xVpsGMLb-~d|1U3ZzPDPL)0hUPKzA-KqN5$ zJ;psNpR?zeW{K*86!Vl6X z0Y;ORBRT=Nlh7k38~!1rOeFyT08jz|01*HH00000000000Dys+lMy670qv7eBqIi& IApigX07yN%^8f$< delta 3563 zcmVL8e)hOd6`t>Z&3qVbYH#!-jf$^R?!w zN|pzcelSiZvY|f9O#S8Pr^Cl2{WTj&nJa~ZN!CzDc|JYR^(-98ILNeQDksH}e?gMQ zL0)`Khk768~5n5=#dQQ{gOaMP00{eN9+($FR%wcpl58jlSr0KElsI9vmH}$)l3ad&s6i zxiaFwFp6mj%hvB@b2&DB+0cDyP>MLp*?TxL5A}!A@ltV3O>StzEp514e{0Z-@vH`A z@Pjg74E4=1bWvdu$u%Ppo01zwVr)uo83~z>J4RxyBpczHe7Gh!AOr29!Y0wasknJl zar>s?u1c{Hsl`WXfjyb;E-EY<>f4f=wXax3Uy8!+Sq|doLo?w4zmQr=8|zse^%-0b=-raZ~wdc z(rur0&OgqRp{&S;hEYD6-D_bI>+!hw8yA^Q@o6g4K`)vNWLo5FvX{#&?`5<5ILgY@ zC+j`fT3ydmDd$O+UglEzMZ3AQPi0o5Fqyv0Woj$Wy>L3Mh0|FrnM|J2*(5KLsoGN2 z3zNwp8md35eVGT*e>hV=9qI$-a{YZDAr(ohEQU7!Z_Bu@7vmC(6hpI2!*&D@oy$Y# z;)>JXc$7Hgq-df9pkpKyI+Or(jD(Ek)UAMyF(+4mvkLPTntNn`=j-O=7qZRtPOcmm zE+vv#A2Q-gW>jPeuiQNnIUMzZu)LkA6?ZQSSG8fmN#(W?NF>79b;Q4pE+A=m3mW>jSIue^T`i2Dd-2N3VPjZ30w7ylfdM4u+RC_X}Gbgh^H#2XhZkp7VF-r27bO2LQYN8i1bWJGSle{E0db;JlqYr6HZ1FdB#a z9yb{cFvw{rgp39#WHc_hFFIXmI1V`n6vQeAt<}L5e~Qr3W;*f6#qk2HI-C{7k<(Bf z*M|Q-fPB3U5*T|ALNn@`_~TISCB|K-TnzU1^=Y|Sm2X&TTm=fVMAq6WD%s<>nv0GAaG zTC0PvT+)^{7YV&l1gj2XgDB1x?}oE2oS zsPXAG*xK;-g5>PJxDd{kOcEaRhj@OiEH9G6*Ni_wVbTgaLAdid&ctj2Ih#J?UJX%#eGMxaA6%JafgReZ&mNwG~Jl+AT4rc{C ze=-`lqqN=;kN?Cop1lOP6Oz^%l7^|(O>4_;zt~b-BCueNh+-FS= z3*~0=+Lv-#oR4WXf5zkesr$)yTBCC#){Ar;&9Gc+OO1|LFVYeIE0tK{VbdxdM&r^l z9Y%vx?og=1Xpl-q15{Sk>TnK#%BpG|L5pe5p{31qf=^zdD2KCx^p(*-`nu=3oVTd_ z9eW)v{>S;CW|Qc*nf&-6{NNmX2%S$qE5;|Kp%k&d zNQ>y^y_6rgVI3V#gQ2_#(qS~ol(8HXf!4JmuSgf~QO-Z-$rg0}skl48Px3s8U%riG z(3ffX%~X{^vyTv@0R+2oK^&7e5+Z+XPvbZc{(is0%6-}WkTvfiLXu?Km&{nbF>vUtkD6kjFs#{$Im652nxuVnXw6$q?ib{(Cd4MWLyPkboekL~{`0o2#VA6i%-5?>HkI7{5=ZFM>-y?qC z9ElQ0w|MK&kUAwVSo?q2+7`Iw#r4_5JtDqq1rBuy*#q>5^a3mLgXjRvG!R)m-wg=4 zy7)SoU#=FDriE7+Uw-A@ zO7_Fc&|p&AeD8nX&M`N#=sP2cpV9wPDDP%_M;u$SmH-s8V5Qv4B*cm9cr<>I?&Is= zLl{3croUJThjlGu((}qAEmKlUalSTey=HHtskpwP{=1W)nf;IHczQz4;8gn!WO;Q% z`c8y0_$(DKn8+bZ%_WajPYzX&!}q9UQY*QATEfE0PZobiq#s#cgx>GRv93og>V?n7 zI^XhrKlnb?QEGXi;}dc=nXhJ(xkh~S>W^mA#p-f&vDBlU>+3Nvj2s$;F5P_yIh~Fl ztyGVVA@zbV3L@WgdOehjdMecN!c*biA8m>}6vO?l&}rLs>eiO;xUK)Z5B~*FO9KQH z00008006U57FGxZ((of>ldB#re@hUy18vEvfkIPSaud=f5w(7h)H;6q%1+`W;8TyQ znf-QlQMov%Ciw;DrO_42GlCM(+z6?675Z2&(>Y3fU|O)oXsA#J9xdv-l3U7+gEwa^ zxFEqN5u`okwnF<5EXCOKJt*e0=+IHz8mCx@rQ2c4_&3`DE(m$V3IY=>e=ytwQ#%sT z2`Tu9v_GyH0D@y^K!FY(=UI*~-T+Sd$%a9>aw-`*3zOEfIJysBj>fiavvxLEWBwd} ztzO@URa2?^Q*l6bDLCa0STL?G@pTao*DxPe@x)sR;;D=J#M^0+V;_hl2B62dXXSJD z{IXu6x*)|vN{Tcu)&-%27ts6prXP9Je*vn~kbe`;fMf34Pf-5>v+Wwv*B$CL7Z5BV;83002+| l000pH0000000000004l2my;7DJ^|~KQY0e=ogn}K005|vwwV9` diff --git a/summary_results.json b/summary_results.json index 17a416181..d385518d0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1858, + "Performance_Result": 15.1819, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0343, + "Performance_Result": 22.8636, "Performance_Units": "Samples/s" }, { @@ -487,7 +487,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline", "Software": "onnxruntime v1.20.1", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -523,7 +523,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline", "Software": "onnxruntime v1.20.1", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433634, + "Performance_Result": 0.412678, "Performance_Units": "Samples/s" }, { From c5630b9cd84fd9423ac5153b720d0da70de420b3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Feb 2025 20:19:35 +0000 Subject: [PATCH 048/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9339 -> 9338 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3f05f8a51..55c39463e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 20:14:50,315 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 20:14:50,315 submission_checker1.py:3270 INFO] --- -[2025-02-12 20:14:50,315 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 20:19:30,244 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 20:19:30,244 submission_checker1.py:3270 INFO] --- +[2025-02-12 20:19:30,245 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 20:14:50,315 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 20:19:30,245 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 20:14:50,315 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 20:19:30,245 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 20:14:50,315 submission_checker1.py:3286 INFO] --- -[2025-02-12 20:14:50,315 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 20:14:50,315 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 20:14:50,316 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 20:14:50,316 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 20:14:50,316 submission_checker1.py:3307 INFO] --- -[2025-02-12 20:14:50,316 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 20:14:51,463 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 20:19:30,245 submission_checker1.py:3286 INFO] --- +[2025-02-12 20:19:30,245 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 20:19:30,245 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 20:19:30,245 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 20:19:30,245 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 20:19:30,245 submission_checker1.py:3307 INFO] --- +[2025-02-12 20:19:30,245 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 20:19:31,182 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8636 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a5b667a16..c5d89a6b8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -675 +676 diff --git a/summary.csv b/summary.csv index 370e9a030..33aa4e38a 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8636,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.095,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a98079c37..6fe164d47 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8636, + "Result":23.095, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 904a5808b51107f64515d6c9cac56c677c03371d..13d353262f7302d26316616baea508d72b6b0ba3 100644 GIT binary patch delta 1067 zcmV+`1l0TcNcu>y*arz|lUz)i2mk=mv*8Ci27jZW)xB-E$St)c0_z1Viz0~{+oMP_ zbq9bXqX8sQgGe$3F@U61g(TAnAX(v{y*l`cByH(1ogk9bs>@hm*2)y~K-Oyg+mYNy zfIH=_)@I&n*v{H>;3kw?N^k^LcUZIrhtW8+2#3)ilslB+FdBrC(a_2-QB{e6lwebi{g%j__Wo#0n3K7U?h=hgRt@8l-ZEIvqxXR5BW%vZ_>va{yFU z73&CEOLGn_9i|gpc!ioA&I+lUa?bVu$H-{URAhT zxWs}Fi`MKg8iy9`FdF1?hq4_;gIqEi;IgW6hjRd2R@LqZT3T}sEghy4pp$XjNtKG+ey&s7%u^NyGek(R=yU@U(Hx_K|<2X0tLhf{wfFZ^^A#+fpa z!y?eSR^%1w;x)?o=RBFtKNU~s_eq{7(aX27^anC6zco~4m?ZhjmvTy0uLPqW{SLF6 z5DX3k@!&ZBldB#ee@??N5QOhMMdW*8ClLs&B$WX5L?tAOK-?^Qn_9#_vJGk99w%+n zR?3OvdS<_!UAbE9RG0jM^U~-Ve`WlewSWsk?y!Qu1Pcrg zz|>AebVLe1A?=UrMu6ZLI#8g4$9b0Hvp0ZKezsv$E}Tk+y@grpNu1m}FDGN)_gP<# z)|fxXpR1R*an)4n;Zz(@QwdJF0~U;HDtuYQ!*$GuRXp)Vf_U6Vedg`B$e|BJ5(Cgf z+>`QAK0U9OB&aD!ahsAN&5LzGX#PM;@->XS>OTY3amc@kC%_^1^(Scl0JH5M$^r># zlUz)i2mk=mlfWWZ0h*H-BRm2m5|cp^7L#lvDgi~4mm@j>x0BK%CL8hKIR7O9002+| l000pH0000000000004l2n3EGEJ^}2LQY0e=o*@7L003ux?Hd39 delta 1068 zcmV+{1k?NaNc%{z*arz#_ndN@2mk=nv*8Ci27klQTzlIptJqRpBCueRR84WG{5>=%*oCAQe zs(%m@rzL#tcnmV#B` zZs8IuJ}g?a!)P2@wZmwT%N^==7!7jCXn@PA(jCqLa9LHnBWP{SIka?`PLNBe(P6AG zg+&dfuzN1k`TubFP78Rh#byPMDT+LH+j@_;1x&0Cv1knsqj6{v52HaacPQgwG=B&t zqX96hDtR~u0A^J!kAPwxXz6gBz?KNq^DtHjCTbAOJ(u*n1?FGg6M3gKJ=en9WU_9X z4t+;VUwdaSlIcjcyaj z`)%>=qkYi&Vbn`!!LT=*NF{%s%73ORpQgoWDx+1E!P9W@)}t~_!z2y!=SA=3YnQW$ ze-tHykMlvz;_$bb{P-dGU>|%4>`y-{#wVqz6tTZZi|FRPlpnZZ9UV^nk-YHJQ5a{+ zL=KBU>spalq>J|`=b!UrI{#EWo!=*UoQ1pXnUOp~e}Ab)DZFc5_AeF~$yE!k;G5w-(u$*F-tU0QMz(k2nLevs5UdHc#v z;w0cxkE@yec6L#@*sC`A1?Q#F70NS$642ZTskasSXqM?CN_=2iu-0g(&;dMJ)ORJf zlp6I>-g`M3yROT+ z*eIHg$rS4C~0oA48lsjO-xVpsGMLb-~d|1U3ZzPDPL)0hUPKzA-KqN5$ zJ;psNpR?zeW+aK~f)w+V6lq>G1)=#pos)Gx@}~a+RHq^TCY}Mu+_#^g{sXh@AIbs= zR`;B8n+N~^)04j diff --git a/summary_results.json b/summary_results.json index d385518d0..2640edf20 100644 --- a/summary_results.json +++ b/summary_results.json @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8636, + "Performance_Result": 23.095, "Performance_Units": "Samples/s" }, { From 7ad068759da0d2804c3996cc8979c3d1a9324ff7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Feb 2025 20:21:48 +0000 Subject: [PATCH 049/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9338 -> 9338 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 55c39463e..4418a8e30 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 20:19:30,244 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 20:19:30,244 submission_checker1.py:3270 INFO] --- -[2025-02-12 20:19:30,245 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 20:21:43,034 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 20:21:43,034 submission_checker1.py:3270 INFO] --- +[2025-02-12 20:21:43,034 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 20:19:30,245 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 20:21:43,034 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 20:19:30,245 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 20:21:43,034 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 20:19:30,245 submission_checker1.py:3286 INFO] --- -[2025-02-12 20:19:30,245 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 20:19:30,245 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 20:19:30,245 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 20:19:30,245 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 20:19:30,245 submission_checker1.py:3307 INFO] --- -[2025-02-12 20:19:30,245 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 20:19:31,182 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 20:21:43,034 submission_checker1.py:3286 INFO] --- +[2025-02-12 20:21:43,034 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 20:21:43,034 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 20:21:43,035 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 20:21:43,035 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 20:21:43,035 submission_checker1.py:3307 INFO] --- +[2025-02-12 20:21:43,035 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 20:21:44,063 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 15.1819 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 15.3608 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432561 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c5d89a6b8..2da17b67c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -676 +677 diff --git a/summary.csv b/summary.csv index 33aa4e38a..7b0603681 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,15.1819,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,15.3608,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432561,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6fe164d47..c3a4ef294 100644 --- a/summary.json +++ b/summary.json @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":15.1819, - "Accuracy":"acc: 75.200", + "Result":15.3608, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 13d353262f7302d26316616baea508d72b6b0ba3..48440c0724122ea1a7f9e37139c4c8f64abf28ca 100644 GIT binary patch delta 1303 zcmV+y1?c+vNcu>yG7?GahI5bG`q)7I-pXa)xK}Fg+O~}DA!E`KIdE-%S&JfL zO8+c2w#*A)vAhRRS!z?q@$8Hx+#r!Cx#}B~QiGH{`KN^80SQ~ipeK_-PYo_b3~A3+ zgPy$u4&ZeJoE3?v;R>iNc#6rGl)A z&U6UTo#%y!KuUqm&wodZQ3NRzOyR{0Exs0O^^FxMb~B;yC2l3ewF%agWx*ok;0ToV z9-TAPXH>C{<(Q7+xD7b1;xSW9%SuPw-f7sroDM~n37zlF{gOpWSBP1Ez#0l&AYev$ zY37eQBGLVFOD#_y8pE1tp)Di9*U83yQD85Ul`CBYrI|7vF|~XMyk_1u<}Ef1iJcuZ zlfpAkbw#7$d27}19F-aa@&Hwgc0K*H_)P9V@ZI+}-=yuZbBBa*IU?ivpF`sNUW<5b zb0`WR&Eh?Th8$Azg0+o*t#yH0T-{!b-7fK5%Xg?t$R40w((nP?VAs{y|OG%p{CItYp2zKw;S+I1CC^I#Qg?5Xuy$wj|bdEfj#04J;6mi z;fOc*>o)l7Hu&oze<9vLwE-N<+04@DfGndt-`Bn1{Grf=m1Ka5zDXN(40Zi%gZo)j zy0mRKsOa72{)7x0I5HLqdD^_snorsVpH`tMGRX7)cOqsbY$fOG9Pkj2d% zX*(gx;7KZ8Fp*=Hs!N`#o*b(lhwn+rpaWzJpcPtN2%2b9FLHT@oYIA&ottpSARI2%$L{0tA!r*Tu+aILFmvhaOv(t$oXXS zWTkp+3~0v>LO=96POF7-ahM9Vyzo@G_eYz;4vOJ^SK!p`I(HA2@3^i1ybt~bP)h>@ z6aWAK2mk<+cq0?DV-`aQ1RQsV)RU?nDg@%h0fdv-9xZtl9 zU|O)wXsFR1JX$n26}OZd2d~apa6y7kB1n76ZH=}eScFgyTLI}y~UoQ!?nXMH(ZWBvkvu3p~8Ra2>lQ*l5|B{<~{STL@s@MRGX*D)Vf@x&Vm z;&C7KnYZI2hdvNV3_uTYPs&I6^t@i8hUDZf%}H92HOVO{=xzD4*dJj71b0e$3X?q} Nx&$0|ht!k%BY(OqeINh; delta 1303 zcmV+y1?c+vNcu>yG7(xlkW|IO13>ZE`@^EI(49ER1k9kJcP|8&B6T`I}g8+|3 zlJeDwaeH%Sh6a&}@q}f9!-??(()cp?tuLh_dMlSF#)DEt&$ea!06CMED1d8!%Tkm% zQ~GDQvSm>MOXLH9$}^ifj&J8IUe|9P03m_#3#i5yNEE za1rednT*IMuvQk);UK3~lB&;t^_4@BiSjEvAs4IrJXH$Hy5V%q(v01wS*q}+C#fK> zqgOhF=+4tp#2}?W=NG>t#yExyN~Z8)h8EAoQhj43irtk^_!74glEwsU%Bo~Ba&QDn zdymc;>NDz?XE~v+Nx)3VYLw>KKLFE557&xOwS>S4iRr7OgLEMW}^6rojE=OcM{c}iy!0!;h zYYs&Tq+Ptb(2!k9j##^Y*xD4h+4b%D*zFPDwE~B_glqxYBb~sC{2L|6m(D4a5A73pl##b8g(W^ham`oS*;q^?9dakd>z%X)X6uNZtA>?c_ zdbUzMHipy-!YGJ*&*^kfE_PF)mKUB1xBh5TtCP)h>@ z6aWAK2mk<+cq0?DV-`aQ1o7ZF|C6d7DgrE9Te z{F}9a3qtO&g1`g|3=hE6PDFG>3O*t2kLyN&;21hkpo7PGmgBQGfKz_9VN@=hN`}3K zS?fuh+&fe+Cu86DSznIUm_Ns#tCzQN)l}-?R2)!K2~N2K7L02ud|AZ9b Date: Wed, 12 Feb 2025 20:57:10 +0000 Subject: [PATCH 050/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9338 -> 9336 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4418a8e30..17ddca6f2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 20:21:43,034 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 20:21:43,034 submission_checker1.py:3270 INFO] --- -[2025-02-12 20:21:43,034 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 20:57:04,884 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 20:57:04,884 submission_checker1.py:3270 INFO] --- +[2025-02-12 20:57:04,884 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 20:21:43,034 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 20:57:04,884 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 20:21:43,034 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 20:57:04,884 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 20:21:43,034 submission_checker1.py:3286 INFO] --- -[2025-02-12 20:21:43,034 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 20:21:43,034 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 20:21:43,035 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 20:21:43,035 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 20:21:43,035 submission_checker1.py:3307 INFO] --- -[2025-02-12 20:21:43,035 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 20:21:44,063 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 20:57:04,884 submission_checker1.py:3286 INFO] --- +[2025-02-12 20:57:04,884 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 20:57:04,884 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 20:57:04,884 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 20:57:04,885 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 20:57:04,885 submission_checker1.py:3307 INFO] --- +[2025-02-12 20:57:04,885 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 20:57:05,880 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 15.3608 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 15.3939 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432561 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2da17b67c..d078f4c5d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -677 +678 diff --git a/summary.csv b/summary.csv index 7b0603681..5b721163d 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,15.3608,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,15.3939,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432561,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c3a4ef294..4713f97bd 100644 --- a/summary.json +++ b/summary.json @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":15.3608, - "Accuracy":"acc: 74.600", + "Result":15.3939, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 48440c0724122ea1a7f9e37139c4c8f64abf28ca..27541b541c2cdb3f7cf6ea44dacd78e20a2d550a 100644 GIT binary patch delta 1396 zcmV-)1&jLnNcc#wG7<%>A0i&@lQ$AHe+mJu&{B$^J@r0CCK(`iiJglbw*B{K2<@_Q zLbuzVbP^Kc8QYI%=9%%(`|_A3Xb(A$8SQJfVQC0x$fB5T``Z28mF{VX3qm83FbaL` z3ApxU@LP{_f#g=o``S*3tZSM)+(AmX!7`xIvSE2jg#4LrO`hdIBEAC*&M z@FEdaG)YKGuEcq zD@QD0`B!*STrAdUECl3L!*EaHe}t^#Bo^wXCoUkZqH`HS<<5(og}^ye&d+{V7{d?} z$ca!F6J_y=ZNxW{E3un1pJ*Szst@7quqKKW8nn~b=r@Eri@UpdPc!^4lLGeJT z80mWQY55sF0HcTRZ?2Bpe&<0E!p#^>7JrVA>pCsuwDl3o0X2)a3=Jw%a>3eGt#yH0 z-riqN>@IR_!?m!D&=J60f7Egf-*NpDFq439v>e+-=#c(W-EZC`&>+bc=41Zo=H zsdicoc)J1bG~kLXPPp5EdkwhaV~N{JU{82MPq3AqaKan>wHy4ke;fR@6@MYxLbU;$ z%2`j7utb6|%Z_!=D1FRjVZ|vZMc<^2I)=J_wpIICSlGC2HmK;`=l+BY8#q!DGJf72 z-x#SwZFLlB{4Cjzt@w;6*xLrS4EujT_%wtaA*&h-I-*Hp^c^~`>P)h>@lW`*&vtt%R2n3GGB&d_B9w2{e z!!Qtp?|llRyDixvZ4tHuZON&DLS0&N6VfITwSJJ)I)3{qPU0lsQ;(~e{dRUyx!9{V z`32{t(G|)wf)dc&2&uOf`e>HvBT9T=TCmn=s8A0cE$X|HTgr`tH)kxkAi*aQq&?-f zLc0(w#n|&5DCV>1&{5nNr&x%k+hTvq_&3`EE(p2D3IY=>Fnj=}b|j(`Qt%OJe_T5N z1jo>V0v$Zgvm9T%0i5!a4TEy!R5J7yCaq_2bnm?!ja}De-E6SN{5k$wy}l2trcxhH z#R1i&;FLRH!MM7_*F`*B%Y0bH6K^Dlr#|WvZ>L3$eISw;fF9$XmCxDpOS2?IbwP@G zN{Tcunu5^z110nIVdPE!1*lF#{!Kgsj=66?LH!4_>>o`71*{(;9_^DpBUb@Kla?bo z0kxCSBPJV;$|R^I0RR9{0ssIJ00000000000001hftHgJBt8M^lTjoi2Am-P0002t CXqbZl delta 1419 zcmV;61$6rONcu>yG7<%MN_qq0? zTwOU7sVKg}BXY4^Wra4IwH~ia!V~we;*pdnrWde zBf;0n#(q&?FOrojT?D0>G959sdv(p(oR1gF>$A)GbiCW` zU^*5>UE;Sb&%rkV$pPJ^0F`i7V^~3iHq`0Fmq+4-axeh z9Lw3v(&&IJqdec&z2N+z(1n#`fQr6J8+8nI{cMB#SyZ~TZ8xas-RJ&<3>!Ey772OU z9p4zKU2S!cG=7%s$5wnowA{KDbu9OPK=?K-yu$eMEB98iA6|wEliFr`|8|bKp+(;r zN&JldmqK|r+dJY|f0MNsAeT8SrG2 z3bnlORJiv?o5Bu?;eJ=()a^QV50>w^t^d3a{smA=0|b+ABO0?|7DEUG9CwG*lc*jb ze@??N5QOhMMdW+p!~uk?B$WX5L?tAOK-?^Qn_9#_vJGk99w%+nR?3OvdS<_!UAbE9 zRG0jM^U~-V6Sa3mtPa;Tr z%59CdAy|sB=UY(BXVIafxG_$#5KGr$f6MqcYXOryzsCv!6D%-108={=(Ge;5gtR}d z8v%l2=sOTY3amc@kC%_^1^(Scl0JG{JO#=mYN_q Date: Wed, 12 Feb 2025 21:17:35 +0000 Subject: [PATCH 051/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9336 -> 9331 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 17ddca6f2..ac84391a5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 20:57:04,884 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 20:57:04,884 submission_checker1.py:3270 INFO] --- -[2025-02-12 20:57:04,884 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 21:17:30,419 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 21:17:30,419 submission_checker1.py:3270 INFO] --- +[2025-02-12 21:17:30,419 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 20:57:04,884 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 21:17:30,419 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 20:57:04,884 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 21:17:30,419 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 20:57:04,884 submission_checker1.py:3286 INFO] --- -[2025-02-12 20:57:04,884 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 20:57:04,884 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 20:57:04,884 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 20:57:04,885 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 20:57:04,885 submission_checker1.py:3307 INFO] --- -[2025-02-12 20:57:04,885 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 20:57:05,880 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 21:17:30,419 submission_checker1.py:3286 INFO] --- +[2025-02-12 21:17:30,419 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 21:17:30,419 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 21:17:30,419 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 21:17:30,419 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 21:17:30,420 submission_checker1.py:3307 INFO] --- +[2025-02-12 21:17:30,420 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 21:17:31,359 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 15.3939 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 14.8987 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432561 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d078f4c5d..44c2d16aa 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -678 +679 diff --git a/summary.csv b/summary.csv index 5b721163d..d620db145 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,15.3939,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,14.8987,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432561,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4713f97bd..77d07b024 100644 --- a/summary.json +++ b/summary.json @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":15.3939, - "Accuracy":"acc: 75.000", + "Result":14.8987, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 27541b541c2cdb3f7cf6ea44dacd78e20a2d550a..b3816cbd068a08622838156e2d3e65afdc8a0429 100644 GIT binary patch delta 3620 zcmV+<4%_kgNb^Xr*arnHkc}jnlide3fBWnxWR);8nGixUL8e)hOd6`t>Z&3qVbYH#!-jf$^R?!w zN|pzcelSiZvY|f9O#S8Pr^Cl2{WTj&nJa~ZN!CzDc|JYR^(-98ILNeQDksH}e?gMQ zL0)`Khk768~5n5=#dQQ{gOaMP00{eN9+($FR%u@jRAG8-3C1e1xBaJUBW`lSd_;_mEA4 z^2~?>!ziXDEL*>q&E?qiWkdI+K`G)WXYb+2Jk%db$4kXEHMyY;x3uAIf2~0;#gVyQr{esBcSd-j>{=#38{}I@yTUW}`i}!GXKk#R)3BO+^zG z{-)wHDg;5{3nL*{*jGkEe@@LQBOxe!VDY7Q+%4rbkK_?1DO{2n(XB=%X``EK8~_7 z^~rhbD7%8b1$4uYvFWOOD2=2bT-M0WU97Q z^}=K_h=%IVYG39-e>Be2Plx(|S+2kDBcvi}mBrBJ|7{u9^uiY zTs-6SHy$MpxhR?_0q7VBg$^YE9U~!QIdv+QmPIC(-9g^zlp+ zKu%ENkjuA;e-a>(kx&v*0wkiuB^UFlUqK>cP6lySVczlyC+D=GjU7N9N^Lrf4T4wQ(@_@Z>pvcTT#G@OA*O>#qUmX})9IF3+E+^8n7<$u14))N)RvpF)c567#fTj2kV7ZSN zciF&M7!GWeuKJc~ZCft+hfT)fkGnw582-0^2)a)#0pwM`rl2L|X5N$A4mvXDtmS{cuPr2TJ4avuTil(*WOd8^^LYxjYhP;M!~5m?<}(;6H`>35K2ZvE5AflB@X8RpscFJ5m1Z+Ep4U~&!SL5Jq}|<3p_6wjZe$B#6C-M zSbr!tbJxC<)8cwev-uW}_or?r-)V`?jaaYIaWunntt}-wV!cL3_^(uAg@;XxbQq0G zt8^F*Qn^E&4x>RT84XZbRjR`|04l4Bbp)-YIfs@u(+O_8LQM{51=%Yzd{`>?T$b|| zm4DC9>pLyj>5BD=oic;9tu6Jc!rQ_n7Jq!$v}T9VxU^`8(IA&Ql;qslRu%Ga4gk!mQXT=dJkZkSI)Nu5P|m|xA%B=m z%@2UN=Zc=U!2HXPMBZsh&$aM3nXEgOOJ5Pw*IwC+Wjd5iIUZ+9m=triFkQ7S-ec^F zgS*8mf3NbQ!(`Hr$|}WRym&z|k0wJU`)%>+qjS*wVbn`!;h;MkOC^7v%7!YRrp0M0 zqg9pu(_rz^qcTmSB#rXtMepTXmu$0fa1u`St!qVIkuF}NoPW-f>HJf1cYdGbc@n>T z8_A$A)AF0CDuX1+U%r%6vU(l$=yy;{0|b-sA{w)}5PAU#?o-Gv=>q@&-;+BMEPq4Z z5U@f(E3}j%XivRQkx2%~U1H~Ahi(7;nU{9iIHB8ZPdW(+@r>=qGxN-N=ze+3QnZ7D zCye$q+pshQG-7c=H$CnC?n?JH#08--Ng0Kn_5@t}(*Lc?xj=F&ox^N}Bvr!tjU$SnGFr z((j7?4T~Y|nxfw|_rMWcMnFlCuqu8}6Tz$C0(hBpSW!6M+CDd*?ED~wSDDHKGgi;1#$$=2c$AhRvJlkYyQ@`QB_p5(!x$ zdI4)FWPyMl<)xlK$%sVTaDRslR~{O}nn4MY3`wi=fm~A|ockN8fGc zZL7S+hA3huN6n=0!c$$*Xn5ILHM~Tn#-MnhRE%^z{j~Uu9)QvP_cu?+?ZA0Zgm5!L zv*;d=i^1V9Nf&O_o>BT($ihl8P>Q}u z8+8nI{cNN5v#7Lj+iXzLyU+az88&caBx3x$JH9be``U68Y5XkNkFEHODBRfwwha4! zK=?K-yux_-m3u4M4=+Q7N$j$Ne|yL5z`*Z}r235hmqK|r+YUS8Sd+CFAm=$L_(3MV zlW!F#8cF39<=>rHnc4rCj3xtg4Oh}{K#RKv)V2bXuoV`6qmt{VB~@72VBSIPz;FWP z{eB+nTG+sje>T>+hU>cC_o)tJ!|^Q_q3iK1oQ`J_ag|qpIGxPHW1t^c zIPh(}{~&ZV89iIE921QY<1aU&SBVHQIO1nR)Fgp;TqAAieg!!Qs<_x%c^y)Db34}|SNTe51PP?whO zgfvM+tp}1?$A4eNcANyV>T-1NoVhcoT<%qy{DSk+=n9c6M+sRD(w=f#pNZGW+4 z{F`k77kPe%6$B<&VE6z`?My@yQt%mRe_T5N1jo>V0v$Xi8NnxS0H^%bhCw-VDj9kU zSFJ~JcJIBMja}De-D0rD`~-ilU*3jQQ>hQ9;(+Q>aLOI9U|e0|^CBLuWj?Iqi8m6& zV;}V^Z_^@UABZFdpkv&l^09b&ZY5TzF7o0w&5M*2O+hHRr;GgSF!G}R1XR@ zG56&ssQ&=7>L1Di1uT$_B$<=BB3l8OlNBQ(9PU%dF6jdR0N)P)02BZK0000000000 z0Dyrb5|cm@7L#csDh)ms004MwFLQKxY-Mwk;Ug52v?CeU- z8mcr!loIgat21xs_U2}1?ss$l+ufNv(LU3zog*Xl#MtWRV7`*wN3(GiC7bt-x~P#4 zCu4Uy)Dk_rt>F#L!)q(=sUuWAL}K%m3hffoLVwwOw)0kQU9wXmYzpNF(n^f=LVPOK zs;Xk+!(HeGib{i?pEPY%>SkPi?|uWRgI7*Sn4)Dbc8~ArpvMMGwshCH$8+{%Cd$a# z^GTgq&$Bc1U2WM}(fpqH?fx&wV)tpYi|{sM$*F{(ud}?$@t-53!jb=Q4f}y^{Rm(f zyqnbEx7tCR7#qRX%3;uyc|qTF?dYx*cskXI|n35$ttl3Jf zv~){`oA{uhmuIT0q7`N8e_6WzPPdM}Vpz*7Ofmb2^R7O4Z!z_C3tx+1olb4KfmC7B zoJW#~-zxm6e;CCx({bG3p(al05vu5#sW+L-5|vzP$v4M90IM_~A3nPpj{xUVmJGciIkD=e_OY?`br8 zlx%f5sy$R4tf~`^{K?Em(`IJi02viqmo6m>DlyM$aY+2IR50!dBa`=w!BBclyBYxN z+FX{_4YTV+st$kHH>Vh0p}Ad{&Q_bqDE zyA{~wV-RvjUjGO`?!RWllYIiyd@&p!o7*ewo}`{Q5Uec7gIr38lkTLwG$I(ER0}%%w7q#2RCp}16 zyN9q~(lUUdY}u|a0Y6Tav>o@A$;ZkcuP4FUGC|J@5VGQRw%VVdWp#Y2%mm%by6*HR zdPY!VXB#HH8SaeuTeU707r?*ELJGYBprh5=Tag|6UfD)umI0lf*Jsyi$EH_Pflp#f z&S)Lml0eU_x>>Dg;OXyz+^|beJMhx46G;zOEN^Wx+;4t>;imov{cCSkIJ27mbx3^<7p=A@t&rON5@7^U%)4vN2jJem)WJZ{QHj7+`{qiF#LX3 zEB$zeJ_F}Tlt;*WG}Je>cRF5K6|YpdDD=#`di!0Dv(E#&bMH&%P@A<39h8gI%&wbM zkt!!E^>H*9Pk%pA4<%U@=pqN?6#lEP1FBerqm3*2W3aLS!749*5~dB6zd3MUD9tDdYdJ+Up{KdQ%7hEf zK;vtTe74>Z3zhYsX1Sg4asD<`H0YXU$YZOsFLHkTYB$S97ugq-ka%og(MLfW$FeI9 zji_Hl~(hnmFm-||Ng%Y{FV ztWd#ar1FY&_I0LiwHa+C_U5fNt?0fm+PK;M8z{?xI2ezJs=tBSN`P1ED-kzPTZ_+! z5II_bynTZu{=p%o0m4cfsAo5sC6&agWVd7 z^`IN&s0;*AhYfoGMl01g$AK@c%yGPLGsvvvYz1n0i8dAD(ad2nV3L3dW<}r7^UN0Y zWlx=nremq>>cvu+1O4cm`qSb% zSORM5`T6W78oth@F}%9*!l*%p;LWcbLasDN#zk~FJrVm|$MdenzSt+8c9hsAnUAGb zlqnh_MnEzRWJ{!Ac_!N}@XWjG_i^Xj3fbez%!b>)b@BzKMq28Al1Yesyl>{31vwwd zMCe+)FXFtSG14O@VFCaEYY%!5rk~t2l<;SJYB?eXwOHFMi8TEeP*yb!sR=ULC)07lGkM~jbqM4yeBul zzZJM84Gk-Hk5!#e0PwX^gr+LEnG^B{!XEbU9rMteOX_Ir)*f0v{`$LWU~!B;M#yYU z{+G0o&t0l%|1phXX%9KgH{Ek!?N`Lqae^#ws)V(mPUjB;O>CjFo=_6JIU4I{xE@ix z0b$uI6%ggFDGXp$+Z>O}jqH{+9iIq!2*T1a48<7z!|3`w*1JzSRk-f;zXkT|`7pEB zjyhI&h#5P3c=EFFc08WdyxQ5a-PP(Tabi`VI!Q`>D!nuv;#IwxCPo zjx3xzlaweZi#g~QgqPtk46P`|j`AR3mZP&8S=L@SbzI-%6&RzNp!!{<0<;_S!}FX~5l|F$^|*J)1Uomo-IGKmc7rl@RmRw@_^Mi>{> zdGlXahPujU&d(GDHVDypJD|-Lg>@Q8yue>8o?24}b^wngS7V_Dt&1<8Q@5@=$1LhP z8V14)Yt!@ndf5o*CN{#~!HmAMpwz`qc|T$DBtYkrYR=*X=N4}@Q(dA}&ZI;- zdjrqYp9{~^ioqgv_NR-`WP|XP?xXnPi}>qIHI(!sjr_VLriq+;gLU}am_Hxm!z=(+ z-DKWe$u&G&U$Y52Yt zKKU6*fcGQdKm7(zUwnBcXHivxFU+OLYp*!oNs>J0;CnS_mv7$k4JRtH;%|D5;v!HQ z={2aZIFO?gagcJpo0>Kv0s!c5s@%ob*$Cn57a->Bi*OYU^oHl9j(K6l=^~DTDT+Eo z=I?mZHxqjgSA?qBStA%f@zXbjB+tl&T;4TUhpd_=Rb^j2eQMk`S0!~$x?M8GjQcl? zVRDJQqJ{awBV(6W_l2W%?W{++5iO_4UPr184xJD#?GA0_?h&9bR@`h>>!5@7&z*1x zZZ+&o8ZUv$ETc;nAB~(zFA|B8w+&JdF+#;(07{=i3K>m~={?c5@&J;-$uJ@gh4BV< zhfZMb4-9oUV$)3GCkua#yFc>i1ABtq7uT=w+5v*>we^QC+Nx3$E`xsH=aSVldB>hs z?e@Me8Va@}FD&~ReIe0hN_&jHWg3aW+^L|KoE!P+6Z0o}4eC!+V^O-Xl43bwljp>I zC#O#M2Y*P*4Pe&~6o?bse3ezN$Qm}ea76$8e{1Bi0p?)nczM|Ww?)zd0Dt@F|Jch= z2m?l$fd#Y4Lx(}~f`Kqh60aPv34`P1XZ!z8@_(B$Fq9A_;2uU7!h0+C&foZd0Cr8m A6aWAK diff --git a/summary_results.json b/summary_results.json index a88d620cd..240ac42b9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 15.3939, + "Performance_Result": 14.8987, "Performance_Units": "Samples/s" }, { From eaaff8f8134928dde872fb844cb578503c6869c8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Feb 2025 21:21:21 +0000 Subject: [PATCH 052/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9331 -> 9329 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ac84391a5..5499cd3cd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 21:17:30,419 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 21:17:30,419 submission_checker1.py:3270 INFO] --- -[2025-02-12 21:17:30,419 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 21:21:16,105 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 21:21:16,105 submission_checker1.py:3270 INFO] --- +[2025-02-12 21:21:16,105 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 21:17:30,419 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 21:21:16,105 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 21:17:30,419 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 21:21:16,105 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 21:17:30,419 submission_checker1.py:3286 INFO] --- -[2025-02-12 21:17:30,419 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 21:17:30,419 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 21:17:30,419 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 21:17:30,419 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 21:17:30,420 submission_checker1.py:3307 INFO] --- -[2025-02-12 21:17:30,420 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 21:17:31,359 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 21:21:16,105 submission_checker1.py:3286 INFO] --- +[2025-02-12 21:21:16,105 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 21:21:16,105 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 21:21:16,105 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 21:21:16,105 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 21:21:16,105 submission_checker1.py:3307 INFO] --- +[2025-02-12 21:21:16,105 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 21:21:17,046 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,4 +36,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432561 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24741 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 44c2d16aa..0dba26ea9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -679 +680 diff --git a/summary.csv b/summary.csv index d620db145..25dd57918 100644 --- a/summary.csv +++ b/summary.csv @@ -14,4 +14,4 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432561,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24741,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 77d07b024..93844fc47 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.24741, + "Result":6.29568, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -469,7 +469,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index b3816cbd068a08622838156e2d3e65afdc8a0429..8f7b2c243ac43406717a57f5456b2a5228ab9b72 100644 GIT binary patch delta 2312 zcmV+j3HSE%NbyLp*arptFK%v_lidel1ltc?wzEPA2mycfcpl58jlSr0KElsI9vmH} z$)l3ad&s6ixiaFwFp6mj%hvB@b2&DB+0cDyP>MLp*?TxL5A}!A@ltV3O>StzEp514 zYtW1FtOjN9gEC+Y_02JKQDG6uH6sz5k{d>1Y)Wn!37L;OMq;fb8{wLKxF$Ft1MQ;1 zCegmBxOsn5ar>s?u1c{Hsl`WXfjyb;E-EY<>f4f=wXZGhdQSXZS24!PA;fThgkw*bIG_Rt911` z?!nQw|6P6Qw$D1}ALq$XR%AoNC?C!4wJ?eGcwGFAi%h5ZG?nR~7fl8-E%G(l%Vn1L zve|!q9A#zdll2~Kt*+;(l=CD@FLNpVqTO8Dr!p&2m`q>hGPRZGUO1iB!s)D*OeRn1 zY?2qrRBfs1g~?GOp{zxWppG&@9ui z9l=BA^3b`s;`BEjB@Q_$nkWJ27zu?AB>;aNBOzlsbt|A_%*hULR$<;kbB_%0eBGS< zLbjRS$co02YXoAoQN*r>EHBkZ}G7<_RN&rMgLP3A@ zDnMk+DTo#3Ei?xZy$TSS-YJM41(8`FvR_{^qau@k<^6j=+(#%ofOzL^Y>B2_{Bw8` zeV#-gSDFBFf)aiHwAjh!P+XB`!IcPyGrK8FMm-vkLQ;PdGWJ4Q=cI@=$8i zVQffDm(0wNabNlSg)I)kB&&^sxrcuz&-uG^(tU)t1AtwB4M0!x9ou$!{zRPzaNbVt z(vVJ07>z@IkDH7J800h*LPi4=G8&g$7oDy&9EY3(3SyOm*6QF2MQCX=op@w(ya1~X zXGL-3=7{pRHvIPiVMAq6WD%s<>nv0GAaGTC0PvT+)^{7YV&k1gj2Xgry1mDkPas=p&6P2d&k?S0rgmo9V*<d9vKbrh#KS( zHLgc_w5oVyIsqOl9JE#ke_wf|Ep4U~xV-~b9nK1PWQGr0r1g$?{3q`5>?OdRkhIp2 zG)%4TZ`&fb)RqXW7qD%LBx)R=BFWSp0FsObkVFk4$rQu@l6Do6OecV3g@e}W;46}} zrOkALNK&f~V})5O;~9{&|LsWbBfy>VR(msVH5_;CI&c%pEhRVtf2%ueT7$!ATv~*~ zXb{RB%5WGBLdj@o<(H_c#NiwOlvTAj0*Y~5!f#)To@oD*%*k?%& z3*~0++Lv-#oR4WXzs2MIsmsZCTB36!)@yVe&G2n!TlTtMqa*xRDzU=DrbRl8#-&v{ zj0UOPp-zX;f2gb~)!`ffl~u($g4WWULra_K1aG`TO%7)T*()=A*eds2mh%>s zf6txQcUrL173&o{Wd>_oTk2JXw}neA_^@fs4x@2t(GH_QE_W!~VKm4kqX90fDt9;s zz-3kKj-aJA=g`t-IzcX>M2E4$6qfM}T<*C_=l{dyJFVZje-@hsJeDZ(*lio8zXePz z4zXzo52JBu4G*J1Fn6fpVKfLPqX96h3VApO0A^JwkAPYpXlZkuz!MQD=V7c6Os3`s zz}$01&s$*rnS{LszcE!Qn z;+4NwdC_4qf9XeMm0~bnyr7szlcAFRws`f?IcWYc>ZP-A(4CE?l0Q#nLzPd{VwuWl zRi*zlSiJP8Ow%Yyqx^Z%d->MoY#bcLN&n+~P_s$&+f07^5PonDK7`JvpB3Yi(ol-n zU!+BJ^Ipmi+^~)gr@>HO1nDrEWXf0$ia_gHkyoUPDAy?GpYvoo|5RL^-zRyV#4q1Q zGU&^+{AQ}kAW8C9CcogkG`d10%TWTF8zJ?sLhsEweL#s1ObgZ;4HfFaqg8!Za!Y@? zaq#Ml1s5dvB!aZ3+*W8Gf~6RHz6Zs879Bc@TjLZ9@#=QiGXBkWfQvj|Vg-Q-78o9Y zshx@Fh!lKA+8@`B0KqY|pg;$YNk;J58^9@lvtU#%oJxk?!cFT*oZSa6XJglOS+^Li zF+ahdo0qq7)KuzWRUA-V3QoBL7K}%$OMH2Wi))z=o4Dew1o7BM{l?pIkf{zt5(CgF z?n(JrJUusSR2O-1pXNnMil!hm9OZsPAIbs+{4Z{9 zn3J?3TLGAp5F!ziXDEL*>q&E?qiWkdI+K`G)WXYb+2Jk%db$4kXEHMyY;x3uAI ztwArwvl^7a56XZs)Hlb_MTJEq*NjALN^Tg5u_?J_BxFAB7>Tu#Y=mp_;hNxp477_1 zn?(Dj;^u!%#qFDlyDG&-q!u5k1@>gVyQr{esBcSd-j>{=#38{}I@yTUW}`i}!GXKk z#R)3BO+^zG{-)wHDg;5{3nL*{*jGkEPR%JJAt-!fB-W|QYvGP~ayiOFAW4u5~OIe58}Svx_ALxO3d#2~ZwnURoL z`+^dabo3P^F1b`rUnI;g;y0!TdZksEw_c9DtWf8)p^Y7Q#K{G<=`g2&*jzGh$yvI3 z9rxhq+yAbbD7%8b1$4uYvFWOOD2=2 zbT-M0WU97Q^}=K_h=%IVYG39-G|tpdhx&k7uD|aiq#|jR#n9&eZ5h|~Vq9X8VrZ6W z*pA?#b9v}oJmd5?9wiRBD4Hk%=okrw4kdp89U~!QIdva}nt3~L7x_YuktB;LAfBwD^^)@_dm(c?k%@I(`YPEg{IORR|!0FjYU5K#gkG7^6Z zqE`VTV@^S=FmItbfaq0#$n;J@^eBkT`627|B{M2A`B&b*2gH4ZvIB^B-o}z>+QmPI zC(-9g^zlp+Ku%ENkjuA;5+IS0P!drBB%;J67xSrKK_X*L260wl-tq}2=d_`X9Y7vR zZ90q%iRqG=88YrGf4{KBL6~HpvcTT#G@OA*O>#qUmX})9IF3+E+^8n7< z$u14)Tp&RM{bTNk88t!A3(mYg9OIjgV2n+CjK~-dx>!uDwhKj%Z((>cFipbs^AuP+K&b9 ztlNTOi5kbISfU28WHf*!vy<2iAAfdhIM0Bk_zz&Yj~I8^z*!D#m9F}hX>D6B`G-x% z0-xX52IUepjzhUb4RXn7fJ@XMm#9H5jVdk~2f$^8gVyTcE0?sT%|$|Q6v3*)SRt3F zK`xDV%_X^Q{NqUOBfOoEG{~l^u*8<L=7UzXnz1n)F6_m zab1d}S%oCi34Nqd<)F1X_=+TLX)~R8CPk83bvP@?U{T}ODU#-Xk(}KZ3*l_ZB;hfC zi05l%xkw6MGyVjT`-pC*+|^vqT{YL*n!56j?T-aM3b!rFBlAF=KFnq`$Rncx9#Mll zqQ>>35K2ZvE5AflB@X8RpscFJ5m1Z+Ep4U~&!SL5Jq}|<3p_6wjZe$B#6C-M zSSUAh*S?g~;(AQ8`4*4&r*0?TX^GB_Sg+A>G{bVOEhRc)y+%j)uT)}%hfRxg7>!G- zbQld%xkH@}qd_Ve4S!HsRjR`|04l4Bbp)-YIfs@u(+O_8LQM{51=%Yzd{`>?T$b|| zm4DC9>pLyj>5BD=oic;9tu6Jc!rQ_n7JS&WW{1(Zv}lLXAeTFo?JyeTlFRu%Ga4gk!mQXT=dJkZkSI)Nu5P|m|xA(%|f z4}iJnik`Q?{L7C--f2nCweUBYtUH!VUlG&SUfGLfI+RU09%o9J6mzyPU9~RWW9*89 zyTvPiukxb9WPj3+$|}WRym&z|k0wJU`)%>+qjS*wVbn`!;h;MkOC^7v%7!YRrp0M0 zqg9pu(_rz^qcTmSB#rXtMepTXm$Pwj6esE+}53oPW-f>HJf1cYdGbc@n>T z8_A$A)AF0CDuX1+U%r%6vU(l$=y$W25C#r^>cF&wB>?~cPyzq|5dZ)HWN%|oa&K^R zFJo_VWiEJaY@L$JYQr!PMfd#*qrEN5p$~-ZKwGkEpiq~V?u0Z+M6CysTE~B1#de$o zvg&el@0__as9f$|vlL*qDa$BKY2$o{(`3@BGS#;QgF&0uwYzBUE=d19oD@7{{&Rikbe`8fHC*wC#e4bv*RDi0tGCP zjU<_qvLagnnUfA9JOU#UlRgp_lVT$(0X~zBBRT=FlgJ|`8|uKcge3t008jz|01*HH g00000000000Dys!lL;g~0p*iOBqIiwApigX07`&OUH||9 diff --git a/summary_results.json b/summary_results.json index 240ac42b9..18a1bf31a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -451,7 +451,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline", "Software": "deepsparse v1.8.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.24741, + "Performance_Result": 6.29568, "Performance_Units": "Samples/s" }, { From c9ab402fe88861625b353e09e2f0b95d3c0e75b3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Feb 2025 21:40:38 +0000 Subject: [PATCH 053/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9329 -> 9330 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5499cd3cd..cda103bdf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 21:21:16,105 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 21:21:16,105 submission_checker1.py:3270 INFO] --- -[2025-02-12 21:21:16,105 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 21:40:33,455 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 21:40:33,455 submission_checker1.py:3270 INFO] --- +[2025-02-12 21:40:33,455 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 21:21:16,105 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 21:40:33,455 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 21:21:16,105 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 21:40:33,455 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 21:21:16,105 submission_checker1.py:3286 INFO] --- -[2025-02-12 21:21:16,105 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 21:21:16,105 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 21:21:16,105 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 21:21:16,105 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 21:21:16,105 submission_checker1.py:3307 INFO] --- -[2025-02-12 21:21:16,105 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 21:21:17,046 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 21:40:33,455 submission_checker1.py:3286 INFO] --- +[2025-02-12 21:40:33,455 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 21:40:33,456 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 21:40:33,456 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 21:40:33,456 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 21:40:33,456 submission_checker1.py:3307 INFO] --- +[2025-02-12 21:40:33,456 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 21:40:34,404 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 14.8987 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 14.7479 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432561 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0dba26ea9..72295df76 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -680 +681 diff --git a/summary.csv b/summary.csv index 25dd57918..4863857dc 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,14.8987,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,14.7479,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432561,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 93844fc47..53dce8af1 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":14.8987, + "Result":14.7479, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8f7b2c243ac43406717a57f5456b2a5228ab9b72..cfcc0777f05af933acd8f6ce9766dc66ebcce341 100644 GIT binary patch delta 364 zcmV-y0h9jmNb*Rqo*sXOxHBgu0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!K? z!!Qtp?>t51d*Z|agsdc$0QE#AB!WQPEPLBp#6PkPY2F?;ZPQlDiQ{@^znxvVTI^Jp z{DSk+=o%GSjuOz^2&p$UdT*EM14?{gTCmP&sL=o(Et;E(TgrcpgI8xPxFEqN5u`ok zwnp0!EXCOKEhy%*=+IGI8>d)^rQ2Z3_&3`CCV7636$B<&V0aHq?LA78^hUDZf z%}H92mK5|hr)BxIA9>Y(2CCzbe-lrDL+jE&7C0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!LJ z!!Qtp?|llRyDiJ1p$OZ7w&c`6p)M`G32BpvT0cl?9lw2LJ8=@osi2EAv)|4xDpv>9 zCcogkG`d10%TWTF8zJ?sLhsEweL#s1ObgZ;4HfFaqg8!Za!Y@?aq#Ml1s5dvB!aZ3 z+*W8Gf~6RHz6Zs879Bc@TjLZ9@#=QiGXBkWfQvj|Vg-Q-78o9Yshx@Fh!lKA+8@`B z0KqY|pg;$YNk;J58^9@lvtU#%oJxk?!cFT*oZSa6XJglOS+^LiF+ahdo0qq7)KuzW zRUA-V3QoBL7K}%$OMH2Wi))z=o4Dew1o7BM{l?pIkf{zt5(CgF?n(JrJUusSR2O-1 zpXNnMil!hm9OZsO9}5HpWQ>i?B$M Date: Wed, 12 Feb 2025 22:03:55 +0000 Subject: [PATCH 054/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 8 ++++---- summary.xlsx | Bin 9330 -> 9333 bytes summary_results.json | 8 ++++---- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index cda103bdf..8ddb12efb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 21:40:33,455 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 21:40:33,455 submission_checker1.py:3270 INFO] --- -[2025-02-12 21:40:33,455 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-12 22:03:50,646 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-12 22:03:50,646 submission_checker1.py:3270 INFO] --- +[2025-02-12 22:03:50,646 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 21:40:33,455 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-12 22:03:50,646 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 21:40:33,455 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-12 22:03:50,646 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 21:40:33,455 submission_checker1.py:3286 INFO] --- -[2025-02-12 21:40:33,455 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 21:40:33,456 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 21:40:33,456 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 21:40:33,456 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 21:40:33,456 submission_checker1.py:3307 INFO] --- -[2025-02-12 21:40:33,456 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 21:40:34,404 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-12 22:03:50,646 submission_checker1.py:3286 INFO] --- +[2025-02-12 22:03:50,646 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-12 22:03:50,647 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-12 22:03:50,647 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-12 22:03:50,647 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-12 22:03:50,647 submission_checker1.py:3307 INFO] --- +[2025-02-12 22:03:50,647 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-12 22:03:51,597 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,10 +30,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 14.7479 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1402 | acc: 74.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432561 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431789 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 72295df76..328845479 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -681 +682 diff --git a/summary.csv b/summary.csv index 4863857dc..944133de5 100644 --- a/summary.csv +++ b/summary.csv @@ -8,10 +8,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,14.7479,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1402,acc: 74.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432561,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431789,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 53dce8af1..a1eda1a1b 100644 --- a/summary.json +++ b/summary.json @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":14.7479, - "Accuracy":"acc: 76.000", + "Result":10.1402, + "Accuracy":"acc: 74.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -369,7 +369,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432561, + "Result":0.431789, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -379,7 +379,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index cfcc0777f05af933acd8f6ce9766dc66ebcce341..d40c3c8ad9803552587257c79984bc5ea4801c51 100644 GIT binary patch delta 3561 zcmV|;oq+f zgh0K8!=65(U5%81_FL6mZ9jC`T|WHfDVit`GR?vyZmL47tBQ<+WDv%qrh0qxwdSfy zmizI*pCqwts?Rc0e>wW;@G(h$&BjvZO5q^Rn(8>urw6*81!EccnU+jtTpSsHCTZm7 z#n*JCXVX;rgSk;O(T#fD)+0ZR)rE(HbQd4Va2N*ia}vxV8RrWpshs$EG5l;CPBZoB za1a*9%6ODi4x8$+IB5M4o~j=m&ilR#~yCP6ZP$>x8RC@kjEP?gAkYN`!Yc?<{nxOA+H!@)q76Rj%2 zEX$MV$KsG!I#`$rduc1`VrA`V!mQhdRi2OMiCo&~i(cm={Osrc(P5fAD(Sq3Z0eV1 zMjRMMF)cya`n_x}$EGhEdM^!15l16FGH|LCm%x#B}kjcGZBxH8G zC^5+FUNRCgyH|{a%xK}myj^c5uz zxl~SHB+M`3H>L-ArB#@>Uyi-3Q0KIvgB`fU$py9PGN*vpTrzHd$yvI39ry6)+yAb< z^g3tV^N;gnC@Zp|QJ9Zs_gauddNe8iMn$Gme45I1*bn2OOpAO?_H&u#{cLt0g;|;U zWc>$AtLu3xOl{@4A55pUU^=TMar~6d;=D+vYD-l= zNaA5QQh!zlGWWxONv3`})Q8M+{e2%H6-lcshBp6i%ebx=;}VM$Lo*w~aXlV7hlkF= zGfscwQDT#eqJ`qiIc?}*2atzSn=WHRV!C7` zWZYNYeqoFJAjxVIf9~PQwg2v%bRXgE0ASBs1JKnx+p-*`CCy*jvmLJ?XzOeZc`952AC%UMwzISu7; zZTRm4$k%m{z}SBfno-xpABS=;G44X;a$sWlAW5^F#+C$C@K4&01wL7~1jQ0Hwnec- z4Pwb?088dcKVXR(#8OmY$v6NkD;%^1-Ag!!mJZViVo9yKj1@esVKiQyVk!OuSnean zT^`_nEC;qqS3R>~Zd)$-&8B04`!|+BxkQa^Q!Y`1TrwKq5;e#rYLH8#ic7`;a9QD? zy*l{HC2i?&k|u zMG`f(Ly<%cBFSg~Nz@>csBs*MWTOg6rW3k^tR*=D> z#-rO{Ys23QlC%3_A)GClBs}I1(R{5e7fIo1#-AW^AJOfUyEfKySIsfErmp<9{jtEU zaLc4TG8^i2Gn>&MkBkO*L=EzY8powPnpHe9odAy&4%(}OuRPM04$}!d-T|vFX9YZe zG8(v}G~W@A|HLz%y#%-ulI9wcMnkK6+m^^JwIu@U1uTmqi5lCZNHTQ?fFz>@h+(&>r<*n9c-fGy+ z+I`?Alv_%01Xg!gv<8RKIJ5|d(IAw6JCxxt8ibP3(8@1SRf)qn04S?!aRe0OKud?| z#5F0DP>;h{(E`s)M&lV2%6*pPuuyL1t^+Bj#r2qG^G7`HpSqoVrzJW!V!cMk)(p?G zw`H&EH9Eq3r4lPVELx<)XdGIl!)TDo9qM!#4N}QyfXb>;9nJwzSyikfXf4e+Ika?` zPVm7i)Z}nhki9Y*$X@qcmh%>sf6tTGcUrL16YCW_Wd>`RTk2JXyM;?E_^@cr4x@2s zlR6I{Fkw1DE}=w+vBDG1w5uG^4M()rnd!5EDo_~2@j)j zXp@N#9e?R?oxqL=l=CoF2qtO}%sp51yanc8-V%AIB|X=|+hnqCn+|Z5(o`eD>d zXTh*Hn@A;pp30^wpQgoWDx+1E!P9W@(xWm>!+#_V^XEnHy^y_6rgVI3V#{gJ%z(@_{_%0v!}K{P?cel!DOtUadh|O`O9KQH000080000# z0JEkLiUA2}XAgbt0{{T}lQ|MBe*<|zzzPAa&{B$^J@r0CCK(`iiJglbw*B{K2<@_Q zLbuzVbP^Kc8QYI%=9%%(`|_A3Xb(A$8SQJfVQC0x$fB5T``Z28mF{VX3qm83FbaL` z3ApxU@LP{_f#g=o``S*3tZSM)+(AmX!7`xIvSE2jg#4LrO`hdIBEAC*&M z@FEdaG)YKGuEcq zD@QD0`B!*STrAdUECl3Lf5UK3;)JZ@Bo^wXCoUkZqH`HS<<5(og}^ye&d+{V7{d?} z$ca!F6J_y=ZNxW{E3un1pJ*Szst@7quqKKW8nn~b=r@Eri@UpdPc!^4lLGeJT z80mWQY55sF0HcTRZ?2Bpe&<0E!p#^>7JrVA>pCsuwDl3o0X2)a3=Jw%a>3eGt#yH0 z-riqN>@IR_!?m!Df6x)YUDR?7-*NpDFq439v>e+-=#c(W-EZC`&>+bc=41Zo=H zsdicoc)J1bG~kLXPPp5EdkwhaV~N{JU{82MPq3AqaKan>f3+L@wHy4k6@MYxLbU;$ z%2`j7utb6|%Z_!=D1FRjVZ|vZMc<^2I)=J_wpIICSlGC2HmK;`=l+BY8#q!DGJf72 z-x#SwZFLlB{4Cjzt@w;6*xLrS4EujT_%~Z51aPLFE&h-I-*Hp^c^~`>P)oBv7GDT|UbK>dBmn>bPyzq|5dZ)H zWN%|oa&K^RFJo_VWiEJaY@L$LYQr!PgztR{qq{9zCZ!15fwtt-K%p)zy$NZPh+02L zY8}6Q6+3Yf$f?KG%zit&s9Nq-m;8eB(&!o$S&kCW+z6?+HF|GX=>tl9U|O)wXsFQu z9xa=@id)Kmje}QbEVv-SClRDQ<+et<5G=*m^BpMWv*^%K+!&`=h^5<-V9I^{37S8% zdBmn>bPyzq| j5dZ)H0000000000fPs{g3M4)O=aWe!BLO4%)BtVO*t3 zUk-xVIPWHp-()l#<;AqwrAe57jI;S)C60=@G*l%Ho*JsDDvwb=AC-=kQPl6ta-vlw zoMm|u|5zLnO9u;6;Vf-MU97BqO;~luu*>y$9?PYTzUXy6!p}h-937^~qms^h$fiNL zGUC86ifIYU*6(F=IW~RS(0yr8ia5&IdpI%=^@q~&QgKa9ZfL_TZMa*1YtW1FtOjN9 zgEC+Y_02JKQDG6uH6sz5k{d>1Y)Wn!37L;OMq;fb8{wLKxF$Ft1MQ;1CegmBxOr1? z`=;WqO0f~C#Ybv^J(=$=Dl8i6+mf5NCATPXNU)VoHlnrJXpe1h;BIztf(ma_(L{y6 zsrZZvK~VU@NXQxXm64Etr8#9J1ch&m#M+dcF%q&gEk;7t=A4m`xotBNGPxIwgv@RS zB?g(@OGZLw_ll8_+5OH)O#WTt-`$Za_nV?I;Rb7?7$;VE~rh1Sps5n$+#tdt911`?!nQw|6P6Q zw$D1}ALq$XR%AoNC?C!4wJ?eGcwGFAi%h5ZG?nR~7fl8-E%G(l%Vn1Lve|tcWo7D< z^&V`kuIH(g^CU|zb1D6z-CWwIGAmM;Okd_QwUy^yIGxtQ>8zGaCQs>Xk{8KTZK>*o z$z%`>)t}YA%!6ouoT;A<^#QY8f8R$)MbavZq0Rr>GOp{zxWppG&@9ui9l=BA^3b`s z;`BEjB@Q_$nkWJ27zu?AB>){GA!9jpE1+Y{$qsNj|}j9-JJYFwwd0^&T-*V zBAN9eBfextMV9c&-6N61Q7;I~%b8kn^|EkP8x}lHpYtSt4M)8PLvyrxEu0C%+JVG< zgt7yPx9%E=mamz0+v7p>co02YXoAoQN*r>EHBkZ}G7<_RN&rMgLP7K@KxE7*h!y57 zGzSp93J{szDTp2gky#(IUtcn#B9njR{d+*%M<_dhc;{_wiKborb9fSco+je>WM4bn4-cIh)kWNk*jYEEq zn~Vk+fj20MQCX=op@w(ya1~XXGL-3=7{pR zHvIPiez>>Ms4_KlGu@qHUG7bRC3I{DgpCz0_OPlEgv7}ZV#tQD%aGn86@gKl)A2IH7 z17|sZuvNP1Tc)*byW}4>9SeMZV;hu9)Hn|15;e#rqX8~agIuBpxiqS{WE=pO6%Jaf zgRflDmNpj&y-)_fNk#)mq6U#fjq6e*%_=0BPUs_ljVcGN)xlRJX-k{w#4{<9)T+Z-K?aK& zzfO@f_lxB0zSsz7OC|}A`9nP4E6Yt%_?q!2h}=hXJLRtCdhV*Z*4EUOe{6p&@KLyJ zQ68BG>hxhYqd^`S4e*E>M2E4$6qfM}T<*C_=l{dyJFVZj7Mle;mMHSrZ5yV)1xzdsv1thpqj71I ziw_-tX>*;x6A>uqVXP2LrsfC0+;c_GTVVd>M1(g-#WEer zrW}tmB}|GrTbQm|7w<84#lhX;mA_Yc(P1*_M`e{_FkZZ%m`9VLlKr-L_0c(K{xIsL zvvAOzjir)5Ph~@uPt#(V%4k)k|1?;<^r%dK(R@Jq(yY|Udj*Lu#OI=!BAcV=`fmP%2*DHK&cq@|Q2=l&oGyJ^CF`O9KQH000080001! z^&%3pwh(#&4(?OPF6jdR0N)P)02Ba|P$L$TR1y(?L*5XuLO?6Dlp<(Py-$%z2FP7v z=VFI#|NWVlcG)H}ht;@MUax3LMZ7W3HF-;zAAtT&iIZ$a?vmzrx{wy{o&kGT{8Pg4hy+;ccX`t9ivA6YA?=!?-!=EZ5nM(< zNs+KBeoqs@tKb57nTjf!rX(X+_pFs31XbhhdYvW|s_-5A8z?swhRLYrBGMZ)8KF;L zYz!pBfd*xqh|krPLzc4QD?BMKma8lg0*b1CVXz}fN>)jl2zApF7m!uanGB(F=Xt>* z;G8Mv=f5kAQ3NRzM5v32vUthX;u|TH*v*(wFLBQyu1&C{EDI7T4vs)b@0D|g@{B6x z7?y5Xmfe8kDjpL-q^xAb&Ao=ro9U2c8I$?mY(o+WSs{7>Yba!afF9+gotthI6xc3Na(f#)~Psi=Rc~FFKGeYC}pF`w%ZVS0>eaH$x&Eh?V z1|3s!!P-`>b%9&l-d~UH4svb7v#^bS&;h_5)bfnL^@0;Hl|W#$T-!tF?)K|=whZUv zMYtT?%%|i1ZcC*jQPe@6yrrey1ULodE(M5$k{ZJb3Q?o6O)mbbfn__F+0^4?y)O4{ zUw>5FE6MT{Y8u_Cc3KU1y8(9^a77j;+-t!723+y+h}%kFPk2L5u$7*0!W;a5wHy4k z8~n8ue<9vLwE>*US^3 z{RtU1aAYK6{JcB9F;e^5aujL&EZL8(_>3sr*#@=@`+q?AHZ8ouc=?rkE7=b(LxoA~ zvV(tn$Lzqs?~J7SjQ*EGc{ke*JK|WAwHP4hIVt!-Cccwx6(<@= z19S~n(r-YEy9d;^0+X&47Js9X>!&4ESlM9SLG8eB0_FXF9_w1zz>a@5*13l3y59Gx z4r9adEf=Bd@hqHZM^>=bTt`0Td^D)ee8IC z;03N@wOUFp4wsfwVc;l%It+}yRW~qR*+;ZUhL9p#udvcwA%!K6?W=c{v&TzR&t{w8s1e{#?Dh zjjN_o?@z@6HI?9$J7B@Mroxv+JY2_oSj7{sC5Xo%>N9W0MGk!+k{Ey<;+~X`^69x< zB%+4orc@90kh{H$^r!ZFK%v=wIY8B z?o-Gv=>q@&-;+HfQvp4bks~?+uNafh7!{KwBM=*gxHBgu0RR9{0ssIJ0000000000 a0001hfsm68Bt8M;lT0Kd29_ZJ0002}k*F5{ diff --git a/summary_results.json b/summary_results.json index 4c6541100..3255cfc9a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.000", + "Accuracy": "acc: 74.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 14.7479, + "Performance_Result": 10.1402, "Performance_Units": "Samples/s" }, { @@ -91,7 +91,7 @@ "a#": "", "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432561, + "Performance_Result": 0.431789, "Performance_Units": "Samples/s" }, { From a157a53e072a0b86bcf38bb7d2bd11e6fdaf56e4 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 05:06:28 +0000 Subject: [PATCH 055/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9333 -> 9329 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 8ddb12efb..ad9646624 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-12 22:03:50,646 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-12 22:03:50,646 submission_checker1.py:3270 INFO] --- -[2025-02-12 22:03:50,646 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 05:06:22,853 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-13 05:06:22,853 submission_checker1.py:3270 INFO] --- +[2025-02-13 05:06:22,853 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-12 22:03:50,646 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-13 05:06:22,853 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-12 22:03:50,646 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 05:06:22,853 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-12 22:03:50,646 submission_checker1.py:3286 INFO] --- -[2025-02-12 22:03:50,646 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-12 22:03:50,647 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-12 22:03:50,647 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-12 22:03:50,647 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-12 22:03:50,647 submission_checker1.py:3307 INFO] --- -[2025-02-12 22:03:50,647 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-12 22:03:51,597 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 05:06:22,853 submission_checker1.py:3286 INFO] --- +[2025-02-13 05:06:22,853 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-13 05:06:22,853 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 05:06:22,853 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-13 05:06:22,853 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 05:06:22,854 submission_checker1.py:3307 INFO] --- +[2025-02-13 05:06:22,854 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 05:06:23,953 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1402 | acc: 74.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431789 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 328845479..8ec1add55 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -682 +683 diff --git a/summary.csv b/summary.csv index 944133de5..3026ce44a 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1402,acc: 74.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431789,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a1eda1a1b..341d13d00 100644 --- a/summary.json +++ b/summary.json @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1402, - "Accuracy":"acc: 74.000", + "Result":10.2819, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index d40c3c8ad9803552587257c79984bc5ea4801c51..10ee1800dbc33b26e0ae81b2d72eae69bf59ef4b 100644 GIT binary patch delta 3610 zcmV+#4(0LnNbyLp*aro)d{6I}lide3fBWnxWR);8nGixUL8e)hOd6`t>Z&3qVbYH#!-jf$^R?!w zN|pzcelSiZvY|f9O#S8Pr^Cl2{WTj&nJa~ZN!CzDc|JYR^(-98ILNeQDksH}e?gMQ zL0)`Khk768~5n5=#dQQ{gOaMP00{eN9+($FR%wcpl58jlSr0KElsI9vmH}$)l3ad&s6i zxiaFwFp6mj%hvB@b2&DB+0cDyP>MLp*?TxL5A}!A@ltV3O>StzEp514e{0Z-@vH`A z@Pjg74E4=1bWvdu$u%Ppo01zwVr)uo83~z>J4RxyBpczHe7Gh!AOr29!Y0wasknJl zar>s?u1c{Hsl`WXfjyb;E-EY<>f4f=wXZGhdQSXZS24!PA;fThgkw*bIG_Rf2(x$I_|;IxBp#z z>9)^0=O5?EP*!9^!zdrk?zJ$9^>|$Tjf+gD_%xO2pchRBGA;5o*~?{?_p;f29A#zd zll2~Kt*+;(l=CD@FLNpVqTO8Dr!p&2m`q>hGPRZGUO1iB!s)D*OeRn1Y?2qrRBfs1 zg~?B2_{Bw8`eV#-gSDFBF zf)ae~FBQl86!@5hX4;nNR%+5*c$ch_edwmQOf2rwwiF0P;|3(_w5#Oqa~e zka1u6`-LqI!X&GWgSm$%&-uG^(tU)t1AtwB4M0!x9ou$!{zRPzaNbVt(vVJ07>z@I zkDH7J800h*LPi4=G8&g$7oDy&9EY3(3SyOm*6QF2e?@3%Go5&3bG!hn4rfJiRYC@ZM)C8Gf@QG;Bf2DvnNHQ8g5;ceq*fiy3Nl#K z_;re;xnCq__r*pyTQW&_%pcKmdi~}ugrW4Pi zP(nQpV?_%*FBy$b%eTZnOLAB!H*?p%l+)sTOtbkd9`8?GPQKF;og1-UqvL3XZ#&zv z*Yz45;lEOe6&^M%(qS|%t4$x#zN+x2XJk?!3O!f}O5duh=OwSlilCuPVGPTw=k8O>1@-jZ2Gm z7!7i{L)i|aK`t2$a9LHk!#Myht7>-yEv-3+mNwG~atS3mj1{J^jA!6-&s94AA1>c% z{m!-6Ea0(3k;iV^F#RoHVsVH~e@l26jZ15I7!88CLlqCBK`V^M_F{ zorQz$Y%G=hc`6&Ke3}-^R7R^R{ing=rAK9&MoAjw&x_v6w=QSn;3!V|ALoObO`_js z^5ci_gLCj9bUyv87@w4eQpEluEux$EQhwlub#yolhVmjvhtVWc#&S>uTGxuaB3-;j zIscp|)A^_3>ij;*^CW)x8a9$aU#8_ZQ&k2@lD~W@r)2dy>e25|O9KRx?IIepwh(#& z4(?OPF6jdR0N)P)02Ba|P$L$TR1y(?L*5XuLO?6Dlp<(Py-$%z2FP7v=VFI#|NWVl zcG)H}ht;@MUax3LMZ7W3HF-;zAAtT&iIZ$a?vmzrx{wy{o&kGT{8Pg4hy+;ccX`t9ivA6YA?=!?-!=EZ5nM()jl2zApF7m!uanGB(F=Xt>*;G8Mv=f5kA zQ3NRzM5v32vUthX;u|TH*v*(wFLBQyu1&C{EDI7T4vs)b@0D|g@{B6x7?y5Xmfe8k zDjpL-q^xAb&Ao=ro9U2c8I$?mY(o+WSs{7>Yba!afF9+gotthI6xc3Na(f#)~Psi=Rc~FFKGeYC}pF`w%ZVS0>eaH$x&Eh?V1|3s!!P-`> zb%9&l-d~UH4svb7v#^bS&;h_5)bfnL^@0;Hl|W#$T-!tF?)K|=whZUvMYtT?%%|i1 zZcC*jQPe@6yrrey1ULodE(M5$k{ZJb3Q?o6O)mbbfn__F+0^4?y)O4{Uw>5FE6MT{ zY8u_Cc3KU1y8(9^a77j;+-t!723+y+h}%kFPk2L5u$7*0!W;a5wHy4k8~n8ue<9vL zwE>*US^3{RtU1aAYK6 z{JcB9F;e^5aujL&EZL8(_>3sr*#@=@`+q?AHZ8ouc=?rkE7=b(LxoA~vV(tn$Lzqs z?~J7SjQ*EGc{ke*JK|WAwHP4hIVt!-Cccwx6(<@=19S~n(r-YE zy9d;^0+X&47Js9X>!&4ESlM9SLG8eB0_FXF9_w1zz>a@5*13l3y59Gx4r9adEf=Bd z@hqHZM^>=bTt`0Td^D)ee8IC;03N@wOUFp z4wsfwVc;l%It+}yRW~qR*+{OThf%DSn8s!;5321JF z)SDW8w9E7nB|b1MSZ6fUXaJ8E%|pd4<;KCAGZtKs;FAc_o^o5GZ3vcP?D-ZH^I3H0 zD6WlDEX2}nux0#*Z2%X9%&~&Ne*_B*?}4eEi0FtEd_vk^*Np(dF?66n2aoeC$7gQ< zr~J)^QMqs`83qeCttWAE@4TFhecxw&Ia*`>9Dl7|-^W!`srRSifSO8h${ny^TvOr8 zA|9?|KCI%2*Am3j5cL~x$3+f(Ad(n>9^#&q&+_@DU81HS#VjR7nwKq^8BsE)MftrS zdDVXgs^gHmi6_7z_w^@e{sFV+AIbs+w0uwRmy@<4TLG7o5+foT?o-Gv=>q@&-wyx) z6aWAK0000000000fPo>CPa{DAJd=?lIs&d3lg}6xlO!V$8HCx z4TM0wg~OgcqFs%Yf%aR~U2Q*f*HZdByOrgtE-BPgJclKqo#U$^R?!x zN|yWaz@H?sY^u*PQ-3-7>F_a0f6c~H=1Sop&YJ2t&!-2vo&{qW`I(kXWn3H?eSAT>X~L}AhE<-A=ZRd}=!;(GBmC^={?TEYJSyqDhivMX zXGRs?u1c{HsmVubf<2k<9x6;4>f4f=wT(lIb$T`)U+81IXCBwgv@P+k&wx~U?gOA zyC^Zp>|QbwGP_rdgv{=DM$+J~Hk-fNY`k2_tev35Cc(5&Vvt$;%t*+reL+csbo3P^ z4!KlLUnI;g;y0!TdZksEw_lFEtWf8)p@SW`#K{G<=`yE)*jzGhf5}<8dL8%h=-dCU zzVte0-Sdz0WGE}Lp;4HRXZKo=M0zwS{zgTnQ+%4rbl4B$p-hW>P4;t{<^61SAB9<& z`egkFORMX7D&;)M(#u>*zv#4<_NmN@6eiP`xlC>4xgSiYwO~4{C2{gCrfN%7 zKS<(XI8uLB2Qv4=e@Ui(I@E{Ea{YZDAr(ohEQU7!Z_Bu@7vmC(6hku`!f`zwI){hO z!81;O<56Ohi=u@RfR2$+=uiUCF%mMCQ>Owt#+HRglP-lR=zSn72K`$vJK4U@kX961f; zac%hT1IX8Pkigh~5Smff#2<%pFEQ>y<#J$R`5;NNoW_;}Rq#*Rj|Dziw*H+Hhn5c031UgDx{MV(tzk4?onk5e16b}O z#$6uZe=G;KN>@FzVQyP4`OT(df%`X>LAgYYZBs5$gIqEi;1V^+C2EjMql!z$0dQI2 zpuIZy$|Y^-aFNg(MX>5JR>&o4kW1rTb4hL+|2UHS2yZ7O4f;^k6kGPq1ivAAESP=9 z21ODzwnLFb4I;^C07=v!lBjVUie#e-Nv0FJf2C37puIZyiX?66FrBy!iX^q_a#oPR zqQ;}!U~9wQ3zD<@Vj-L@nIt^s57B(BEEh@PX~v%*av#y{l)E<8b63qVx2CTAw*9fd zt#HevJTe>VbTgaLAdid&ctj2Ih#JSGJepNJGMxaA6%N{~gReZ&mJZViJl+ATE@uTi ze=-`lqcqA>jf-}B8eK?qewD!2Y@7_0VGj_ zNHPU6fTUH0B-05XS>d3)I{1ntZRs$bAd=Lo%UEI7$`tcJ)@uFRk=#dsJLRp`X5MPp z&f0z8CX`!Ba0FI&ShNO*(KxgShtVLEe>;@nFdBrC(a_2-QB{eBKcDlu(buSkVH{OGe`v6v};;oq#Ud!-U9JSt>t zm`?D)E7at0R*=0i8pvMvT$b||m4DBZ*LPa5(-Z3zJ7orInOo{rg}a4IEcmc!%?_h+ zXweR%K`wVF+hH`wC8Gf@t15Rm2f$@j?T( ztmBG_+@zSF*O~WJ&^XEnHy^y_6rgVI3V#{gJ%z(@_{_%0v!}K19?Hf3IVOqQi`BG^*%)=86bCwor@i|{r6`G?Xqz~x7(g{5)$GW+mC1F zneov3@|Y%Q4>^w+?Q6DSX$WY@qL^;`+Wp;??rDe%LL-te3VrPfxb|i6TaR;rz;C92|p?-Ye&fiAs$@@j$5<>3Z^M`58R`qlfQr zu8!M&=Rpy|%@|Db%9&n-d|7bE^=(cwXltU z&=J60)N&2qas3l8lYnou9NR_c?)K|sz6utTWw09FEM}9#ZcC*jVc12kyoaUU1ULod zE;)#pk{ZJra#5qPZ6^Mzfn__F+0>(WvndX3Uw>5FD@n5iY8u_Cc3KU1y8-Vs;EF6x zxZ8kx4Y=ZCiQ7tGPk2L5u$7*0!W;aP-xLae+CsGfoXS~GldwdBFw2g0&nSJ&WnslB zC`I3-jXH+9ezsNnSy!F75;A_?9p4zKLv3{wY5XkNkFEHODA?Nu zwha4!K=?K-yux_-m3u4M4=+Q7N$k_3e+S2G-@xyTr235hmqK|r+dJY|leGvS;~B{W z`B5gGlWY|ye?jFH<=>r1nc4rCj;BL(4Oh}{K+C%a)V6#jgJ-F@U_z%XRhK+hJvvpr z8op;Gqf*KB(~>HzbhzlEwr_NN<^6sh>sr{r9q(+ca}39E-0xEz#zx1p9E7eX^I$fa zOTZ(-lF@!^Be)pY!9F2!jN&$}TP6aZ~$JNY!JG-b_?p2rkg7ebo8WmZN642ZTskb$HZ&&F9N_=2iu+C_x(EuJTo4blz z%8i3pXDqlN!6y-|AWyAUkJ*z+AI=CkO~QQR1(Scs+DV$1k9+X5zeevcIdCRkwj z08H&nL`S6HGt&OJZUhL9p#udvcwB#E1wMNNIOT6PjLL;m$uL;BX+4Rvd++6J?E60J z%h4M17x;7i@;0uTN_{vL2h>!8Q|^ETkLHALY|?yFv}g$s)~3T9B3ynwNA@d>uw!^`C+2ION~N6JW}H{Rx^sP)oDrAIJg) z@qeC&n3J+1TLGAp4kIEPX=e|8?E?S+`VRmA6aWAK0000000000fPo^DOCv!6L6eOm zIsvnj$Rj2jUbK>dBmn>bPyzq|5dZ)H0000000000fPs{g2_!xN=aWbzBL diff --git a/summary_results.json b/summary_results.json index 3255cfc9a..87e4ff255 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.000", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1402, + "Performance_Result": 10.2819, "Performance_Units": "Samples/s" }, { From 54578309296e3b5766be0ec0c536946bd0dee280 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 05:42:35 +0000 Subject: [PATCH 056/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9329 -> 9330 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ad9646624..07443def0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 05:06:22,853 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-13 05:06:22,853 submission_checker1.py:3270 INFO] --- -[2025-02-13 05:06:22,853 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 05:42:30,454 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-13 05:42:30,454 submission_checker1.py:3270 INFO] --- +[2025-02-13 05:42:30,454 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 05:06:22,853 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-13 05:42:30,454 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-13 05:06:22,853 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 05:42:30,454 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 05:06:22,853 submission_checker1.py:3286 INFO] --- -[2025-02-13 05:06:22,853 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-13 05:06:22,853 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 05:06:22,853 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-13 05:06:22,853 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 05:06:22,854 submission_checker1.py:3307 INFO] --- -[2025-02-13 05:06:22,854 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 05:06:23,953 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 05:42:30,454 submission_checker1.py:3286 INFO] --- +[2025-02-13 05:42:30,454 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-13 05:42:30,454 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 05:42:30,455 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-13 05:42:30,455 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 05:42:30,455 submission_checker1.py:3307 INFO] --- +[2025-02-13 05:42:30,455 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 05:42:31,474 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,11 +29,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387358 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431789 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42205 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 8ec1add55..9084fe1a8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -683 +684 diff --git a/summary.csv b/summary.csv index 3026ce44a..0a1fbea0d 100644 --- a/summary.csv +++ b/summary.csv @@ -7,11 +7,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387358,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387262,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431789,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42205,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 341d13d00..03bf89244 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.387358, + "Result":0.387262, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", @@ -369,7 +369,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431789, + "Result":0.42205, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 10ee1800dbc33b26e0ae81b2d72eae69bf59ef4b..223bfa75f9de2993c93f98b4f53130b1aaed181e 100644 GIT binary patch delta 2910 zcmV-k3!(J!Nb*Rq%>xAt+}&f;v(f|i0)OBKKAdy|IOzuMTrhUYB>*vwswn_5lMaBG zN!Nz7k`92Fr~KfZqyy;Y@%|3|eqbh@IWPkp7zfnc#C4d zN42=x#BLb8oq)anlYk$wy!R8Z$u8%Y9#$Fxjd5MjeiY;$70T@~GsLaVqysZtJT>mEkaF@p;6w&;|Wjk@K_YEEt_d z!Nn)!U!Vz6!5yJ0#M(>UYXf10;}UPk9PnBy(*G#Iu z|M-@w8xrCAO*OFzWAgZ-d;8ZrR~h&ZP)h>@6aWAK2mk;8KLED{yh)h|007aG;Ri5( zar^8jWR);8nGixUMut>9X=-Muh~e- zTqzvHSwkJ=`ShS(&%%+M1eumhWn3J886@c>$cwM(u%1m*8T98ylX2b9b*DZFqF7yc zI7oN#kqib=C_g9RY$D@);UtygATNfWjiPC$9v$|h;#e7vlFC6tJr)PeAHrAbM~Cyi z?;`n_t$kL?Q0|lD*Ye9*zoF`-v<}*@^I=@2N?#6w**NbekKbf89OcEd*`-O74+#Z- zWfB#0X{gF1cxtGosys&hd{jDCMp3^n%ZXN%aF*rCYiBxG_g7zvr(4oVC%yO)fF%*d(X3Uy8! z+Sq|doLo?w4s!~K%_ZZOoTaOO*KrSyzWwj&OSgU2IsZ6MhO#0X8bI8PD%F> z-VOkE{WSnR&3A0u<@pmj58%9=Y?EOO7k}Iqovt(-hnxcnVwHo|>fj1RXlXN@cw}+B z0ILpXMRDZji1N5L{PzLm>pDna>^%t0&^7VLq1;Q1yHL3tm{{&3X|`)_Nl*p9(S9s& zW8D@MOVl_v#S%4$C8Gf>nT>wH5;cgWsKSzQ09aNyXbJi#;T&4pOecsXwdycduzy*@ zc?K-Se*nvU#JJ1;&2nI?bk(;^Yuj?kKWjP``1r;)D3_>l9LgnXkV{4bT%rcKL=AFj zRB_2T04^&Wv{naSxuh*^E)sg92v!}&3b{lLa%sG4F3D}zMt=iHq6U#fjq6e*%_=0BPUsVjDhI9A!B-?{OPlG$ zGbxhPs>4}928$YBr%0OnMRImuEQGTqlZ40oVKQGU%SBT7n(-%y+(&df<*w#>?y9-g z*3^}MYJV*7Nw{rM9+~Iq^jS8eK^_?m@Q51Z5jCzyd9*4lIaAHtZ>j;9ehQSwzQc}5J_s)VXQD~Wjq6t_P-s;eFV5u z-fC~=t%l>S-3M+$xupU}V1IFkO-pbXjZ15A7!5+XLlq9AK`0pwE&LKyg*co8fU>F- zM?ftOw6vK{Jc~jJvDH9Eq7r4lPVY+9tlXk1#Q!)TDo9qM!#4N}QyfPc!WQXS3#P+3*1 zBWNwnIkdEyPH^KDYH~O$$X=P@!&15DvYfZ5{Cjp@-)X^4SFBg;lo_mTZ7Ek3-WD#g z;KQahJB-GqMLUcJx!j>_htVLHj0U)@s@&lm0GCy@JA#(hoI^{S=>)li5*@}0Q&`3` zaJlCyo&OJ)@3emBT7PU7@K~bAW4A4s{uVH?IK-wUJdDPrH9U+4!Q7#WhtVLIj0V80 zD&*lD0GL&!JOXNYpry@q0#8JsoQJVOFqxVk0CUe3J#T^emmi6|(~_QR;cqgjJC;k| z5Ua1fu{V+FP&VawoGD>a%-Oj)`LCxamx0(F-A^hMRdR($ScytTPKwB&v`PPe=6?I?~^=FCNJMcGU&^+{AQ}kAW8C3ZuO(Kblg6?Lb?yYM@Y;mh6NyaYU^Ll3FK!U)fHa z1hVRKbnl$GGpJndWf%Q|bwaBOWl4%6P)u{7+6sL%tN0N`&Qrx{rxjG_01hqdyOJ5g zw1qdT4OlP0MFFH7VWvVm?+wA&u^mY2lHgE5+-fVS52bB?v7zjnwt(|AeZUet<2 z;+~Yx;`yZ|S)n>l^F^HIaaJ_xg52k%$Tq{stNt@k9*6v!cmhnhuRlTk2eacJgaQQ& z+}&f;lWZbE8@C0#Ntp-$0MR-C02lxO00000000000DytXlgJ`P0hp5xBRl~jlS?B( z0X>tABRT=ElgJ|`8_SeuawP!(08jz|01*HH00000000000DyszlL;g~0pycNBqIiv IApigX06@@MF#rGn delta 2968 zcmZ8jX*3(!7EW+YtsvJtRwU-2sVQ1R%yWb4HLHe8Rf?dcMa+rm1~Js!n&(+bYbG^> zQe*M7W~v%WJ#|a_+`HC$cdd8U+Gp>z_qV=t*0=wBhbCW48pj#g(MPYnW*Gs1b`J6? zi(tzE;{pSI@-ydZsc^5Uy7zW;ler?07;wuHNX!WW3|hQYE6u#fH#S-Jp`yC#q?1QG zCyduVjztNs^lCiYlG^585%^OZ$fe&&UBR@}JN_0|t65E*m3la^@|kJxyol+HxY<<{ zY55S@^R?0E>eX2S=bPLYPSu~-z&d$9w9)5X`@8sCDe0b>+|ZMcY zMY2Qmx1RcWOQ4OwZ7t4DRbzQH)VYF#gHF{{_wP2x=R@tcmQz{h{i+iz1;|7E%S%1) zDMI~t0NR|9i3b1xuwH}?pf&8~e)dH)nwK?8oGveDgq|Bz%qFVbA;UtbdUCfKBJDm!k{XDA{N$)@tI8~kg&d)5CDr^~ zn`Uvoxl=JP1U*+3HqDWbi!q%;8;5FZO&%+VjA#tEddNvr(@^QJQLbT@gvV-@o|8qw zRLpYaJYG}_RGjegz(19~CMkHi7#(5N@1P)k+HWd1gmS&CBWyTrp{2QJOl?Al!7^jw zA1i=zbz&QKjR{BH`;gP8b>X*1(RA|0{2|Su=lPs@6=<>H$lbZj%ncO1ps_DS`rx4= zvPo@SC^LPU+;Zv$`y-K}atj736soS0BU$U<#WVlNw4T|g;fn2G`JSQRpY|ePbCyfC zs|r{{m)hEJOZ(uVM%r6Q;h{1Xx-=@E(9;JDl!MCVi=8_B@C|+_`)_el^J2oqGhY zD#@LfE^Kty*X)5HTMp^E(cL;Yhm^(@#?Qt5Qg6GfIznv&zL&4nn>BG)QH_0~ zv~uS+ z!ucB$iyyv$-&)4|rZdMD*~~=17QbE~+9ocri~Jc+OG_GL0!FHEe?r_NxO=W@Oxrsc@i`$IIw%Yl2(e9mIMceyMZGeu$rJ_O?uzk_Ma}{A74ypZi{b^pz*go^ao_b z$LVd#3CAJMXDLY*72i}cKR)BPWJbUbRTpTUStrd4;Ub=%yX0dr-@5wks*EG3xl*9) zdGWIIVg*9p=Bb6-7NRY4+f}1ruK#0gmW9;KkG|>rBe$?syKy5PTpY7Sv)yAc-uYEU zpkrW*3;u(#iQ$x7()013OL#>6Cu+waggU8U+fg{7zkN7K=~(-!E(QZ3WpibKI|6KR zAQ(VYQUZevmVbsB8v*&>c7LxINVI|-WHsajFRi1# zhHz;)dq+=R$c#%=wPv`20JB5P00u^SsIPLm9%HgZ-YvKa3s)CASI&D`qS!BK4&u#j zZio^H8NYe{ME76DGS?$Rwz0kmi;r1M%N-0k~)|A-~%)1CWp$n@$8@;f5(9 zb8eoqg9QT-M0^YTQPxcj4UNuQD} z@aQ=Xsod0=R=QA6fuVTM3jU3WdNhYcH5sH}Y!$3y>=8n}>6B?s`(oxM9y@yk2tZ?J zfuo9G`TFL0ADNm;Z;QTWr_cEp0?ogO_u|B_(s7Jl$VR)DyH4e8*jF3kYND6IA~3=k z=J**6&?Z=Q7*+yQw2pBuW~QiAcX)^ZvUx#{H8Bm&Q(0>KY+Y!k8W1M#@9BHh?2YCx zBuNVC;Gy2sGj%(7(nu7ALneyz1tEaL3-l% zZXvzvK-~~ir$r0tLesR1PrhNtBkX6s1t*qL+@M}3R3i;1glWi__}Zzf!-Mqx`D7i` zhIa9i>?76rz~9E(UX<6LVZLLcNd4HP!{Hm~3eFZmj>BQ^NaL9+q{;xT(GokV zJC(I^bKI?2TOy7Zs)wxe`rtI=hh1}42X0fgoAcZ8m1j@ro!!&X1@-m~?_{spFD=?M z$aTV<@AeTa?>`hsF;99Zz=5OaFMvu=ih>;e4tA6QBtWso=MfXU4oS#Tn`XnY!8DV3 z%NvuuR&Wz`nm@%AKioTi)AmkbrO6;|=)scmIsDGu0HURN$OYZ~5CM*8iasxA2})Uz zBNFdmH+;dTTw@l>G;^qHp7)8Vkf@8~#Khe}sOkW=3gsWaXE;*Q0f$z=wC4Licr?@P z6~hfJ+@5$UWqm}eZHDB_y4#E`^astp3?VhigXnw>XI-O7AhM~BTn$gmm$hH0H*zm$ z?zK~ENN-R_1Z$;&xEQ*xLgrbhoN#p*K9`Sn%cw@~yrOTd%-A#6$W&-^BI!M?UD0cO zMaJq+@$Hyg%i<5eAFdM)Vge5&g0#rncF`K6MgvnNT9Bb|}yxnhk z%Eb)AXAqeZU_bx>%>V#!U$_+yKMWe@=O3tq@xys4hG2aQ(#L%Ul(}N}!`Vw-!55^S zvo1xw*qG6Mt_bg9yGJHE>X`V5_Js3F2Y`oVhi1njuD_~u+`E;17IieWBeW@qV)K3w zJCi3A-b(w*4bxVJ+k2_Zcmqg$r&iv%?4_@I&N6Zfqe4XouAv@culZv&-Tr>Ys(Z)N@Cbp|I zh*&*FyLK&M9y~bBbu8>RWC#ksk}eP_wpN%iad=whQe1w8ssJ<0&4%Y#BPt9}yTn6h z@yylhsjm{GdnHberM-UuTQ Date: Thu, 13 Feb 2025 05:51:00 +0000 Subject: [PATCH 057/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9330 -> 9328 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 07443def0..3d59424a8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 05:42:30,454 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-13 05:42:30,454 submission_checker1.py:3270 INFO] --- -[2025-02-13 05:42:30,454 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 05:50:55,779 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-13 05:50:55,779 submission_checker1.py:3270 INFO] --- +[2025-02-13 05:50:55,779 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 05:42:30,454 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-13 05:50:55,780 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-13 05:42:30,454 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 05:50:55,780 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 05:42:30,454 submission_checker1.py:3286 INFO] --- -[2025-02-13 05:42:30,454 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-13 05:42:30,454 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 05:42:30,455 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-13 05:42:30,455 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 05:42:30,455 submission_checker1.py:3307 INFO] --- -[2025-02-13 05:42:30,455 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 05:42:31,474 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 05:50:55,780 submission_checker1.py:3286 INFO] --- +[2025-02-13 05:50:55,780 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-13 05:50:55,780 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 05:50:55,780 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-13 05:50:55,780 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 05:50:55,780 submission_checker1.py:3307 INFO] --- +[2025-02-13 05:50:55,780 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 05:50:56,717 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -35,5 +35,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42205 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.353536 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352675 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 9084fe1a8..d4d7ff298 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -684 +685 diff --git a/summary.csv b/summary.csv index 0a1fbea0d..4a1e749b7 100644 --- a/summary.csv +++ b/summary.csv @@ -13,5 +13,5 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42205,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.353536,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352675,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 03bf89244..3a2d4ff7b 100644 --- a/summary.json +++ b/summary.json @@ -429,7 +429,7 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.353536, + "Result":0.352675, "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 223bfa75f9de2993c93f98b4f53130b1aaed181e..a66328652b0f89d608976e99c57166bf80aad574 100644 GIT binary patch delta 2527 zcmV<52_W|JNbpFo*9QeRVVC=tliUY1f9*CtMj@+&naP9@k|BrDcZD3&fVCL2w#nh& zuMLDiy@kV`KBC>51ln&^cXj)r%Xazjm#28FJjgVQl1W1qnxQIk5+?m zs$_XE=?CLvA{*+n%+z0wemZk7O{2LisrfXR(~*3n!@@2YE64Y!pp1 z_2{r4701eWlvEBH>ajRz{t&*ZA05v7zKi5zw)R;mL%C0qU&}9N{f26kQXRBk=fk*4 zmA)JVvvJ-{9>2+GILeD@vrCgOe;H@d5o%hvB@b2&DB+0cDyP>MLp*?TyuAL;L zvy^bWvdu$u%Ppo01zwVs1)q83~z>J4RxyBpcz@`EcvtfDE*Y3Y$dxrsC#J z#qFDlyDG&-q!u5k1@>gVyQr{esBcSd-j>{=#38{}I@yTUW}`i}!GXKk#R)3BO+^zG z{-)wHDg;5{3nL+C*jGkEf0pKykq{KVF%oN2a>hu=(zF-}S(|f4Lgu#3NXX<~FcLDm z9h8`4b}tzTncXW!LT2|nBdPOeo5P=N4xX-L)=p63kYJi9G0CibW+Y_RzM!N|I{Jzd zmz*l6FB0Y#@f*_vJ<}@8TQA36cBpgO(8dlt;^czbbeJU|HkXWBf3iwfuj3vZef!_l zmu~y4bN+Fj3}r<&G>r1m>|P6#SdYiW-?+$hiceFS4tmjKAk!jWlf7JKc`uvY$5B?M zK3VU<)(kyQrJN^OdYMb<7wzWKK9yOK!esg~m#M8h_rmG47EWiiWHNb5XOp~0rfN%7 zFH9zbXsG_I_GKPKf8$L3bf^!Q?fUyZLMoD0SqyFdZ_Bu@7vmC(6hpI2)7bJ0@AAyK zxZw0B9wiPrCz>b$<`@ZO4kf@GBOzlsbt{--%*h6DR$<;kbB_%0eBGSt1SJkR#gn)TC4a0B*{&~{QIW~N^8P&_?jw{P zK)mxdcEq}EI_{P&(dS9@ais|$Cn#~q>DxpJkjO|Vi6{XQQR0%5`P8oBJ+O;{{lCI4g=HH%FAmwc)=HAYb=E0%Pw%Xr`fwKMv(yV%&ww<-o-9K9Xj; z^(_gi;BT}a3%s#z3yLLb9GhZ^8pM*(0G7;+ev{e^9e;4MhVu+qivIwX`-pLu`!~yh zt*a#3q`Q%FjmMVYLH9wU2{n;8~-?x`v`9*B+boxs&!$BEmx3&e;#@)n8%EDiX>_r zmm-N8M1PXe0FtOdBvIqK6v=uOl1wM`iAI%!*6QFZlC-7GbmG-1lGLihSwRMi8sDHu z*7u9#?7r9tXGaQuvzrCy3lfbUWp)_4V9UbFHnZEC1B~Sm2Xz+oC)& z&(rC%Y(|4TG8*6!HOM1sT#xc-Rq@Dl0z6hYXn(B^zVb+0+Ds>Kdk3sKoE7lM3?H^g z>mBjIxb2<8q|Jd6gxWHbO~RUr@O0Klv&8f?{8e>-++%4Yt zdzBX*CX;?tRw)MK#ruhQG#M({Z;Lk{orC5N()Po7Ryvdt1A7c z!Q!1qWtv7w8s*Q6-piLRXXD@~PWm6`gI}6WqTgon?nXhmL;F5aS?f6kNX{8MptexKxd62E*K$)GRO z@>^Y121$~?d?}}7^)~9!@3WW?1`dDj>?#f=0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0 zE_iKhos!LJ!!Qtp?|llRyDizqp$OZ7w&c`6p(!oB32BpvT0cl?9lw1QCvg(UDaX~! zemlFUp6^tb{DJe*=mzB(K?!JXgw&e`eXJJgBT9T=TCmP&XwU#2&D)2XTgrcpgEwa^ zxFEqN5u`okwn5tvEXCOKEhy%*=+IGI8>d)^rQ2Z3_z&9vE(n=n1%U|`7~TU@I}y95&H%3LWN#92cf<@xlWUeCgjoCKMcOl4dge;Fj{ zB*=@e>9C$nQyKKLi38GkCcsNLR@sSJ$Q7Atr;cOz~eBmUO;~+1FpN*nv zrXC&kqvBW@kCMtkLp>G;%^$*7>qm$4zV9OWn5}(Q%24i;5gHS=i_-Smp1i9uk#Up4)WmWFijqnblyWY z4azek4h*B1mauI7UN)Cwt1lb6FAYi&M>%^BN9JMup>(`dTvL-9+Hgx7f9}>A>cx0g zgEII*888NYa|~TnSVVHoNW`Y(hLISXl3PYX=Hrf$SS!g!xF#R22@c3WyQr{9v~MbI z-c;Paskp0BY(#4Dky>C+=DUjui-!8PbYKBGbq6uvMLa)o_meX zQR0$I<@80u{33p1dZ1TYg?a1c*vkraP8-_Tfk&KNP@4{O3W&`mf8&;%rK{I*501Y5 z@9ImpebzbuI8TPMA{!b;`Dk{pg~_Bo85e&iMW$1Hn#y$0i{gPyi+oM?a+&45Y<541 zvNH9_dJndy*Yi}$d6HExbE*18yScPaWmcpxS$&zy)K;E*;dELHr?XlT$4}`j&WmKK zwp8`PBpyUV^=Gv&fAb(3XX>ZJ`hZ!kzwaZYB59Sy(B}Vb8Q1kxA~yq#>)kWNk*jYEEqn~Vk+fj1RXlXN@cw}+B0ILpXMRDZji1N5L{PzLm>pDna>^%t0&^7VLq1;Q1yHL3tm{{&3 zX|`)_Nl*p9(S9s&W8D@MOVl_v#S%4$C8Gf>nT>wHli3U$f3R7@c?K-Se*nvU#JJ1; z&2nI?bk(;^Yuj?kKWjP``1r;)D3_>l9LgnXkV{4bT%rcKL=AFjRB_2T04^&Wv{naS zxuh*^E)sg92v!}&3b{lLa%sG4F3D}ze?|jHq6U#fjq6e*%_=0BPUsVjDhI9A!B-?{OPlG$GbxhPs>4}928$YB zr%0OnMRImuEQGTqlZ40oVKQGU%SBT7n(-%y+(&df<*w#>?y9-g*3^}MYJV*7Nw{rM z9+~Iq^jS8eK^_?m@Q51Z5jCzyd9*4lIaAHtZ>j;9ehQSwzQc}5J_s)VXQD~Wjq6t_P-s;eFV5u-fC~=t%l>S-3M+$ zxupU}lX4D7f9o;L=36}8pSqoVrxiLkV!cGi(G1^qwq>vDH9Eq7r4lPVY+9tlXk1#Q z!)TDo9qM!#4N}QyfXb>;9nJwzSyikfXf4e-w6vK{aN`wfayToDOVNV7A~>i!=^PmjK-x!e>;o@x!j>_htVLHj0U)@s@&lm z0GCy@JA#(hoI^{S=>)li5*@}0Q&`3`aJlCyo&OJ)@3emBT5J~ZSfa>dw=J0d7BI0m z#HJ-YjK-xkJd6gx+@Xqx(IA+N2EeQ;a%-O+G?me+ zO8;rFc;``>rcshc`SYUp@}R($ScytTa@$9c`}`UD(=qjlRQr*FW*Ko z=*zVHW~#~{N%EI3<&>=6Mm_o+vz8DB4g|}TXL6IE9v^?pYQr!PMfd#*qrELZno@-A zKwGkEpiq~V?1VINM6CysS|@*B*-o4Uvg&el@0__as9f%47yW{DLaPd8Ns1y+Omm^y z3Vk%I_z^|UQ^jeg6;$W|4lV1uk{QCZg*U4WSTDdu0i+yZrb0XK4Z+y49Z2eu;7~!_ zYAdM^rEPz)q3oNsfb%qcz!E&=)KfeFV>1)c5y{z%G(WZ*0i0pzK!WlPXGw<7UJq8f zn+>CK;grH34BWJy#M!-fVm9`DpY+9O4f!+twSIjcSB-@lPQ?J#C1-?LpkCX$#Fs@l zTt}T>hZAoF@TWu2Z@e8Bnfic7Apn`;o|Mnx`K2XUp*m0VMV#hwRy65?+~=goHp9rP z{xeV>hy0s(0!+EDKSBKmv*I7h0tL4Pyh)jpu_9Xmn3D}7JOLt;N+Ur5J(G+hIsvbf i$0H^L%amtwlkp=K0g#gjBt8M;lSd>Y29_ZJ0002goSvir diff --git a/summary_results.json b/summary_results.json index f742d6ebb..9c51d8528 100644 --- a/summary_results.json +++ b/summary_results.json @@ -176,7 +176,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 0.353536, + "Performance_Result": 0.352675, "Performance_Units": "Samples/s" }, { From 3b9c0620e11f0b73c9f2650314ed650d88c050bf Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 05:58:32 +0000 Subject: [PATCH 058/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9328 -> 9328 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3d59424a8..e8b850fbd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 05:50:55,779 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-13 05:50:55,779 submission_checker1.py:3270 INFO] --- -[2025-02-13 05:50:55,779 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 05:58:27,773 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-13 05:58:27,773 submission_checker1.py:3270 INFO] --- +[2025-02-13 05:58:27,773 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 05:50:55,780 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-13 05:58:27,773 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-13 05:50:55,780 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 05:58:27,773 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 05:50:55,780 submission_checker1.py:3286 INFO] --- -[2025-02-13 05:50:55,780 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-13 05:50:55,780 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 05:50:55,780 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-13 05:50:55,780 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 05:50:55,780 submission_checker1.py:3307 INFO] --- -[2025-02-13 05:50:55,780 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 05:50:56,717 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 05:58:27,773 submission_checker1.py:3286 INFO] --- +[2025-02-13 05:58:27,773 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-13 05:58:27,773 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 05:58:27,773 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-13 05:58:27,773 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 05:58:27,773 submission_checker1.py:3307 INFO] --- +[2025-02-13 05:58:27,773 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 05:58:28,806 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,6 +34,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42205 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.3058 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7236 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352675 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d4d7ff298..dfb2b8f67 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -685 +686 diff --git a/summary.csv b/summary.csv index 4a1e749b7..f6006291a 100644 --- a/summary.csv +++ b/summary.csv @@ -12,6 +12,6 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42205,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.3058,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7236,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352675,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3a2d4ff7b..afc1fe94c 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":49.3058, + "Result":48.7236, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index a66328652b0f89d608976e99c57166bf80aad574..02eeedc90e33abbb4cfb0cf2b19768b0cb99daf9 100644 GIT binary patch delta 2065 zcmV+s2=4drNbpFo*9Qb*;T@Qh+Xpp&P5bO9WR);8nGixUpD(Za;L{E+78#6pxh$nPyQkX{bW0tBRb2Nk5tl8|v-N*P5p) zSsqOK!8n=7hWacs^_Qcc4j+^B*K8zZt`rU?SwkJ=`Sd{7vv4HiAk&hmoD@eD21yzR zdGR$J>e)1vL4R%(k9DJ7cax+DOfFh4r8;Q8&WCZ8Dt$Q!X5+k@JbshWaFiF*W|t;m zGS23Il{hNq(omH+cxtGolO72me|534_BCPE9m6iy<9RHXHu|F1`3OG;d2n=?CXY%w z?;)E8<;sWy!ziXDEL*>q&E?qiWkdI+K`G)WXYb+2Jk%db$4kXEHMyY;x3uAItwArw zvl^7a56XZs)Hlb_MTJEq*NjALN^Tg5u_?J_BxFAB7>Tu#Y=mp_;hNxplf4Oae;8zT zFBu7$-77{yX7@WIG5NF2;mr^@Mz zg!x7M#`HkXvYO&Tu>+4dxu7;3W(kPRCF7Q?($(v@2S?xjclD*)KI@!+ zoF_wBkqr%_d^Ee)!X(z?aq%}Ue=?oo(^RH|UNjlVw8+TCX+!l zRDV|cG7qA0rhYoq2h4W;eIFqeNvkY|HvhL}T-S?niA9Q`S%y*H@(b_slP?Qi8bfom zdM%s@!`cDFeT1?Dh_~(Q|7+n3F-ARhYMY!pS*pXk!PEhEkgjV?$!PWM+np`^w)hY;h1K zS#2E5Jv@2N-<^`~BfK2|?D}f}dYbRpw#)M;>O6q+c5;)3baKLI9P)eIWHi7ar=bv& za11zqMqLws9Ll}KxC@obfr;gPB+YhBZwr?EjrL=KH`Z-Iu|$nyQ!G(~STY*GlDW|j zSfU286jfL<4gkvv2Q5J#C7eS`o9P6xq*fiq3U1bLo&ihoAHZ@SG468zW;w7`y6Rh| zwQalPpEVr|e0*aYluOh&4&@Rx$R(ozE>VMjT%rcKG^)5{8~~RU4qB^&uUyiWHWvxK zPz0+EV})Fz2DvoeHJ9YF@sA_9kMMRv(%8JGY6?qixq=k@^U!0#JZ3a0lBjW9iX>_f zNk#)mq6U#fjq6e*%_=0BPUsVjDhI9A!B-?{OPlG$GbxhPs>4}928$ZMPLVYCi{$Ko zzSsz7OC|}A`9nP4E6Yt%_?q!2h}=hXJLRtCdhV*Z*4EUOe`GWAP zqd^`S4e*E>E{s z@$4nQoshKFkTguK?r+;7x0IF$EElkUZHgpn9G@b|lpO$)j0TWI4I;_Z!vKC7x+){xfu(-pf zB{+=6r8PK=2BF-c3Ww1kl#GTJeu=6=9L@niSyhT7pcV&O+Ds>&MWKXp9L9=&)^}bq z8lP5giG5b&uuyL1u6-$|#rc?K^IJULpSqlUrxiLkV!cGi(G1^qwq>vDH9Eq7r4lPV zY+9tlXk1#Q!)TDo9qM!#4N}QyfXb>;9nJwzSyikfXf4e-w6vK{@Ww0D@XUa7VR(^O09;np?g(01a}F(SrW526N^}@2Oko+%z~!E+bpAhFzSH`hYq43tV~HY< z-L_%+TfoHP5Sy0pFdCQE@Gu$#bB8J(MuT878UVAZkcV>sU{;m#2&m;HftEJc2|N*j zavsJC!DMQF0L(pC^t=V;Uw$O=PD^^Og}=#U-IK@Y!BL#_Kh6g=n?%3O*#PA4CO_T4x>q?jOCyRw5}C-MY?#4a{f6_rt?q5)%ksr=SlqX zZ6t%fOcKj)rm75*B!Br*PRZ(R)T7@4vz8DB4g^?l$*7Z{9v^?pYQr!PMfd#*qrELZ za0$Y8peeG1;v`N2US&DDch1}yR4wuZ zI8+d~+Dhs}X?;!^%Ng~v6+bIgyd{OnqS+E0M0OUAVGPDvn0b8 zuLmpLY{RHrIi>J}fm!QWoZNdSCS%|CNnecCkUzs;>(}>j)mW&*sTiQ9;*2m0)N9*R z___#(>!|bVaN?~1{xk%A=Iyk|u@86@0+3_ev+`LyzicH|sL9j3jMF^MicMOQazXON v_hIBs{{<*dL+&P?0mt08pP=~%v*I7h0t8~=9hkGQB0T{FSZ~Rwlkp>e6Ts?_ delta 2065 zcmV+s2=4drNbpFo*9QbQVVC=p+Xpp&?KVC}A*+O$$%GJ+A&1d-g&fm>wHULu$>HCx z4TM0wg~OgcqTQPW+HX~Nb^D>qcKPs^r+BP9$TW+RNkbKyp(=6`CjDqKY^b+4Uu&MK zWO*>@2jgTS8|t&n)L)K%I($shU$c>vxl%ZoWDRwc=hFjS&%%+6gG@`Na#9=>86;^O z2+GILeD@vrCgO z8E5mqN*on)X{t&bJT=t1lO72me}-6D`d5o%hvB@b2&DB+0cDyP>MLp*?TyuAL1Zc1($37L;OMq;fb8{yXZaO>cJlf4Oaf0$%; zFBu7$-77{yX7@WIsq<%>!=G&qo~~rpPEg{IV45f~$*g^5BxKgUprlSZ`ic^loGPa; z66P228`A?l(<;ncFUMYXsB_xT#tuB<{Jr9vpr9-_@6H`>b>R zah?oiMK&~y^3m*G3zJxn$Hm{cf5>!-Pg9u=deLMc(;{D!y&x?dH-xm06L(Wco6fsjWQs!s)aYPG_}bGI>g8le|c#YD-lwOeTY9 zsQ#?>WgbN1O#O7I518%x`#wS{l2%y^ZT@e|xULuD5{nc=vrN<2@(b_slP?Qi8dGyL zqZZDDY3%^wK0?_6#9Ma_M9bIehVAi_rwdAd#JX)d?v^dl=SlQ&r3oM>C~?T?+e8VF z$Ve!OC;<{t;*yj3)UP0sF(-pKt1xf*gp+gH(8dlR4W%|6#)ian$;=EH_m#h2*y12e zvf4P9dwBAkzdI$}M|e8`*!9-{^fceGZI|ay7(9UUc5;)3baKLI9P)eIWHi7ar=bv& za11zqrlE;H4&`2A+=a^Jz{K)Cl4iU0EeWdNZ?qo^ys>TziY00sn_`I?#FEhfmduTQ zz!EixrKrM^aR69WIA{s_DB&Dh+Ds>iCAI1>R&cY1^9)#u{{WWzh;f(uH_L&o(pBH8 zTidov{#nzpz{fYXNx4Lg<4`V9gIqEi;1V@|$R%o!OS6hg#sP3y;h?oT_{t@1X>*a# z3q`Q%FjmMVYLH9wU2{n;8~-?x`v`9*B+boxs&!$BEmx3&e;#@)n8%EDiX>_rmm-N8 zM3T_}lBhu>QRBK4$$Ax%Oege-MwNrs>fkGqw582-;?*gV)T+Z-K?aK&-=Ij=_lx9z z?7r9tXGaQuvzrCy3lfbUWp)_4V9UbFHnZEC1B~Sm2Xz+oC)&&(rC% zY(|4TG8*6!HOM1sT#xc-Rq@Dl0z6hYXsr&u@ zO-pbXjZ15A7!5+XLlq9AK`0pwE&LKyg*co8fU>F-M?ftOw6vK{Jc~jJIxb2<8q|Jd6gxWHbO~RUr@O0Klv&D4m6a?rbcT{CO%Hs(hLj z%Tz|ID*dOy;+;ojnnp<)<hH2FVhk7TU}KKNs_;ODW_!hHtNyuvz8DB4g~J(Dh`vN9v^?rYQr!PgztR{qq{BH z#-Rw?fwtt-K%pruy$NZPh+02LY8}6Q6(?~L$SKFw%zit&sGjdsm;8bA(&z@|89@nX zZiLjE27Rm+=_5*fU|O)wXlT#?9?jc_np?_^gEwa^xFEqN5u`okwn5tvEXCOKEhy%* z=+IGI8>d)^rQ3gC%lHr504@lbVFiH+78u?GQ#%pS5h?hDw7;$!0fJ-bK!FY(=UI-= z-T+Sdn+>CK;Z!mV7H(Qk;^f|WIT`!D&-!w-#{4<{TE4!ItEN)#PsIVXwcwOHV8OVy z#+OArT*rJ^#uKk4h^HayH{Onm9Qr^cF#tWpJt?2%^UEb_f!cx;RZ5C9FIS|Z6`|Ga vdq483{|r>eA$Jo`fJ5%erx*Ie diff --git a/summary_results.json b/summary_results.json index 9c51d8528..e5f0bde03 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", "ColorKey": "availableMLCommons", - "Performance_Result": 49.3058, + "Performance_Result": 48.7236, "Performance_Units": "Tokens/s" }, { From dd6a2306bb59912924ecf4121ea402e84d2dfe67 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 09:21:10 +0000 Subject: [PATCH 059/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9328 -> 9334 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e8b850fbd..e56f2e938 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 05:58:27,773 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-13 05:58:27,773 submission_checker1.py:3270 INFO] --- -[2025-02-13 05:58:27,773 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 09:21:05,211 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-13 09:21:05,212 submission_checker1.py:3270 INFO] --- +[2025-02-13 09:21:05,212 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 05:58:27,773 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-13 09:21:05,212 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-13 05:58:27,773 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 09:21:05,212 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 05:58:27,773 submission_checker1.py:3286 INFO] --- -[2025-02-13 05:58:27,773 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-13 05:58:27,773 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 05:58:27,773 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-13 05:58:27,773 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 05:58:27,773 submission_checker1.py:3307 INFO] --- -[2025-02-13 05:58:27,773 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 05:58:28,806 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 09:21:05,212 submission_checker1.py:3286 INFO] --- +[2025-02-13 09:21:05,212 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-13 09:21:05,212 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 09:21:05,212 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-13 09:21:05,212 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 09:21:05,212 submission_checker1.py:3307 INFO] --- +[2025-02-13 09:21:05,212 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 09:21:06,184 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,10 +30,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0256 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42205 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434305 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7236 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352675 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index dfb2b8f67..a3888f7a6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -686 +687 diff --git a/summary.csv b/summary.csv index f6006291a..09ab6094a 100644 --- a/summary.csv +++ b/summary.csv @@ -8,10 +8,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387262,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0256,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42205,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434305,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7236,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352675,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index afc1fe94c..417490ba7 100644 --- a/summary.json +++ b/summary.json @@ -279,8 +279,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2819, - "Accuracy":"acc: 76.000", + "Result":10.0256, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -369,7 +369,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.42205, + "Result":0.434305, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 02eeedc90e33abbb4cfb0cf2b19768b0cb99daf9..50d515fd2ec12f5a3491e4eb87ec6adcf3d05a3a 100644 GIT binary patch delta 3321 zcmV0~2Xn~nC^ z1_$nD7bmFjHWf`&_?wE)s1O8&FN}m-VP6>uIW?z@grM+^kyx9OGe$yAO^cC`b92r} z$lSIW37OmrMnYz{gA#+x?j<83vwOuz$n1V+Bqo2gIsAXs=HTT@X6*ze4hg1-5`)az zXGTJ1?F&jw($QCxxa3kfeUUJ~h~JnV=#^Gs-g-IqvO=BHhBkKK5hoYaro)^9Vspv3 zC1>gCb=-raZ~wdc(rur0&OgqRp{&S;hEYD6-D_bI>+!hw8yA^Q@o6g4K`)vNWLo5F zvX{#&?`40p`#8$V)FVF9^%qnObr8vT#)!7CcU$^CS&N zy$3^cw0bR^3B%ff#C?Ra1Btip8i|&txrXWRAbLEA9-e4|&WXvgu73M882N1mq5SiX7h#m!zIX`5*zGOy4CjZL&_kg&MP<8ot!Wlhx{Hl84WPVX()t@1}J1SF1ar{U1>NDIR_NPDhI9A!4-c+zk88t!A3(mYg9OIjgV2n+CjK~-dx>!uDwhKj%LhrC?V8>eEct&Y z?Z*P2tlNTOi5kbISfU28WHf*!^Q0fJL=9pos<31n0G1UFT7vE+oI^{S=>)N)RvpF) zp4KoLzfQ3f{{bxb5#ufoaFzpGrK`SWTHBUOezWOV;QozmP%cs9IFw7&AeW2=xI_(d zi5le6sN#}w09;l$Xsr&ua!Fg-TqJ+=MiHz!j1_W;8syS=*Ibg@#y^haKEm4xNrOC8 z6_(g?2Pyas(PP2vGny1h)Hp6h5;ce$$7uT3b_Be%t<7;8wV8Q68BMb-J0&Xpl!n13aPzc|?utQ68--9+^&n#|j6n)xlRD zX-k{w1Rn2zRfn?z9vKbXQCjba$A97(&t3xD2}x@WNyF6Y{M&NAwKBy#khR+X zb|m)^;7)m~y_vU`8z06ND7Td02(0d~X$=meacL0_qd_QlD8pei2qmMTm0zN&5{GjD zP*&CA2q?yZmNwIgXHh7j9*41_1)i6T#y2RG`z*;}q1?<}`%+Ge>oI@L=8t&1KXp6# zPD^xd#Cna6qZz*KY|CEPYjlMFN+ni!*tAH8(YUlqhtVLFJJjhg8l;lZ0F_mxI-CQb zvZ`1|&{~>vXlXN@;DcAF$>FRZdu23`z3#az=PfG#o+q#Gv|y(z)+=_(4A!=`)T;__ z3zt~%Vbhu&M&r_=9Y%kHT<%b|!)TC8Mgv?{Rqk*OfXk}d9YITL&Y`8vbb?$$i4J3h zDJ*I*h23+N&i{wYcUr%5Ej9~yEK%gK+ZIfJ3z%3OV$%{HM&r^N9!7&;?oh?UXb?6TbzV^yqEYqQE%JDc;!lan9h3Tqw@g8GW9NaBl`FoWY9VU~0R8}bl8(ik`NYuPE#4Js`Q@*i3^ILYBq^}o5_zK!Vk{D zhtT=-vtoQw8cGrSi?oPt-b?v`8`ja`G#JW@ARR`NOc~2T5old2@``lv8s+?Ro=oST zio5gsB+rxh<=aRGeVLZuOjQ{qN&fPsoRZb+s7Jp61GAP8m;nj9aX}pK0{{T}lQ$9` ze^28$5dMC@!peQw{g5}ec|)Zu)D~22DOG_J_o*_;wB#WuvT)K# zm8w2t`|->?GoB8<+~*nDLMcy85%??#uJta4#&m=NaM@! zw}F(3=&f8H8+S?-N472FJII-|L;+k|f7YVRnbJSYjV+53SR(HLRG!(?aeO;xDK|(g zDz1(VN~u9Ap8iw8s79imF&xNrI8eiLkw7-E)o@_%fEv7vfwM9dP5hdtN;bhI$SPA! zG|O1ds2Mo7W)wAz7wdJFa%jR=;&0&6Mhw$&%SE&|WI86Fz}i?uhl89{Nvb|Ke^-t~ zCd#kyfLyF@^HeD)n}(AuOEY$xW~suP9;AZ2iOzKh(VZ8ih(St$&M$sPjByMZluY5p z3@yGCYxRwlD0Xw9@FlJ#B&`Y7lvT-MT_8y%x(q}X=&vH!1aoi4^Ht~chre&og zZtpZ~pU*}j&xOwS=5EDer7OfNe_;)!E)X!|qB4sI9g+BMwWXG?4^3drw9rLJ>Lxoxw`l|nP09JljZ92~=etj>Yj2 z2|O!w@J&E+KzAuYrJS`GZlP2y8ru}=uNGLhbDd2yN!ROY*Y?eO++JB;WYE&+4z<(k z!22Dz*MTEh9Ppq64?A$=e`AfiD6j{-qbIniCmiq&f87p$-41_U9E7+-$n-b(hv%g|s_+kEfef6g&CvgkV_iJ#H` zQYi0cdq*5wvX%f8vS6j$%Ou2!>v%MNk?!N`;6oTcHm1K=35RtpW76}=BP~->OL4w7 zY`tc0qp7&QqW-&+pqc%T>3DiV&frx04P<$BL;6mHGWaYNFPO+7OU)&ZRZk98kHhz< zWKt`+eOki8%1;(Yf21E-UWDH7$FZ(QE$W5O#ya2feLwg<)lq7Bq2m*BHkq$xletEG z^y-gh)5Yp?bg|T5^1@T$ z-XCp>JQTzIuFz@Qb?VlZ@3^i1ybu2cP)h>@6aWAK2mk;8vqu(H2n6l=+zOMR9v^?r zYQr!PgztR{qq{BHA*B_z18vEvfkItcaud=f5w(7h)H;6qDo)}g;8TyQnf-QlQMuTw zHu(kTrO_42GlCM(+z6?+75ZqF=>tl9U|O)&XsA#R9xdv-l3U7+gEwa^xFEqN5u`ok zwnDoQEXCOK9Vq6r=+IHz7^hf>rQ3gE%lJ3j0xk%-#|i=yEHHckrgkKv6H@RIX@6Wh z00hU-f&v{p&a)g}yaAl@lMRD%7C9yJ^ delta 3372 zcmV+{4b$@WNbpFo*9Qq=;T@Ql2mk=lli&wj1a3cc*|R_h2mycB<9RHXHu|F1`3OG; zd2n=?CXY%w?;)E8<;sWy!ziXDEL*>q&E?qiWkdI+K`G)WXYb+2Jk%db$4kXEHMyY; zx3uAItwArwvl^7a56XZs)Hlb_MTJEq*NjALN^Tg5u_?J_BxFAB7>Tu#Y=mp_;hNxp z477_1n?(Dj;^u!%#qFDlyDG&-q!u5k1@>gVyQr{esBcSd-j>{=#38{}I@yTUW}`i} z!GXKk#R)3BO+^zG{-)wHDg;5{3nL+C*jGkEmgba^5EQ;K5^Ga(#z@H0v=|9ln{!4& z=C;j9$mCuy5;D6Tlo(`oFBu7$-77{yX7@WIG5NF2;m?0I2TxZrYbPjiNH9&57-ZHy zGZHduUr=I_j=rMAC8x^ii-h?_{KoV^&$J5j*2}S%9qODmw6Oz^IJuxU9cBrL%_ZZO ztkTu%xCcky{&)4I+dk`@f1D>nS&#*Pu6>|wYr|CQqGesz09Tbi*|EqpUSLAVKRN0%hXn$d*O6i3#YSM zGMPN3vq@egQ?;e47bcTIG*o|9`!Wxrai)Gc)CbIV{e2%H6-lcshBp7VWn9;bafwBW zp;?Ae-|`FZ^31uo;PfXRB@Q_!nkWJ07zt$#CBT0iBOzlsbt{--%*h6DR$<;kbB_%0 zeBGSF^+WJcu4HG(qSDB@Q{onkWGf83_duB>*BL zp&);H6(BO^6vPVi7McTyUImCu?-WFjg2=27*{&~{QIW~N^8P&_?jw{PK)mxdc0|)Q z9Cyo>=<_7{xY7iW6O=gQ^lhR9NMt0GM3exDC~?WjeCk(_$e5EsoK={&e8R~&ZD?Z$ zkcLv54r4=Nx@2aCjQh&pFKlrTCRuG9%sqcRdCuRRlI|nC9RTe5YXEwh@7T7>^C#*& zfb({8lZJG1!e|`wd)#C+z#ylg5HcE|kkPp0vgmZB;W*?RP!Ovev{naKC_+n{>BJ+O z;{{lCI4g=HH%FAmwc)=HAYb=E0%Pw%XhvNVe;mra#JCHU%YljIeI(6xO>Ya9{EdJ1 zV}Uo;Z9%a_jbl?RQG-}A8o-jd(GOUn2C)=XSTYU(%L)fAK_4ZYLra_K1hJ%69mWc7 z)^MHyOYtATavw47a{p#IuvNP1Tc)*byX2oW9SeMXV;hu9)Hn|15;e#rqX8~agIuBp zxiqS{WE=pO6%JafgRflDmNpj&y-BKWBlGLihSwRMi8oy4FH1~_-?7r9tXGaQuvzjCy3lfbUT0LuI75~ zs=3zI)Rli~e=P7xxNT7$ndj;BSvI3V9vKbrh#KS(HLgc_w5oVyIsqOl9JE#kUwNc0 zZKe~ry#rPq&I)*Bh7Vh$^^SP_C+_j=CBU7KwAPR`Os(#3+akA=mIy2tux*MYY8;;; z$&?)cl8gqBL=7Uz)WZOhb`^h;OecV3g@e}W;46}}rOkALNK&f~V})5O;~9{&|LsWb zBfy>VR(msVEgyUsTcF%hfg`ZE!=@!TjK-xkIE)6N+@T7G(IAwJh8BK_szMyj0YF(* ziX)&F2U^-pC!R&2gmN6liq>~tG8&&&Z;5?YzWoP4Jh zIyYjyM90w#-*&cTuj@5B!hfX_D?Dskq{C=jTBXBikjfqEbQld%$!LJes!|=!0Z>_0 ztRrYG%{jERnNIM=E7at0R*=0i!-uVM&t*ApQTg}Wd3~n^J6*9}u~TNSwzZ{PRd`#t z#DWi-*6c7Emlo|X8svX+hq4_;gIqEi;IgW6hjRd2R@LqZT3T}sEp4U~6HXzV^mmEYqQE%JDc;!lan9h3Tqw@fu@S9NaD5_8(Ck`NYumZ^+ZRr*hZ#XFD6G>wup%AXg#moHt;#=%jX^gqrAHJe1g&E&@q;Rom7 zL+E_^Sus8-4W)?vMOs8R@1^{}4eRJ|8Vuz{kPf3sri|sF2(+#hc}2Q-i*o)sPp0!v z#nt(JlIKbM@@*u8zD&z+rm75*B!Br*PRZ(R)T7@80Z>Z=1e5L}8nd(zdI1jZQ^+pq z0{{Tu4*&oZ0FzN87JtoGZExZ@5dMC@!peQw{g5|9-Vm@tKr6JAB4|&&PmxIm$X#OR zVux-2{h60`**KxwZBIH03Gs~W$20THc<6q4%u=+2f+vjjG~2K=1TF-aMPp7sP>`_lid%eg>uE9E_HD@5KgO&)C_Bivv)P=9Gzvmzrx{wy{o&kG~Tt+}i zk+3R$PZPnb-~xD=iYl6>BqLb&td$-FRpae?ohB5j@E!XbC^r>`$*ATc(i=1xp-*6J z3?##W24$Rx&wtgGLzc4QD?BMKma8lg0*b0(up>!IR!N!&b<-0UkX6x{454!8dBGy! zoGIt$zblMU1Su3msEdiRc*)k{8!437&6rRxanB*HO|Yaa3lb>~jzCE7m2-yjj4I|B zmTp;=-GJjN9uq;NtYpN^y@t)3>5yd^llk6kLlOyDA%A)SYba!afF9+goJ^i%!j2?i|{r5Ld$L+v*P=s(ZLgV?LL*#jG3%PB5$O=Hs;ys22 z9aD0_+J9E9b%9&l-d~UH4svb7v#^cO0l*#9@{GXsf)g;6Kwz|7+e7H?_Um}I4CmuT zxE$Qfr{n!@OQj=G)IpxSrKR2kI0fY{1&D-_8p8?-QKPX4WjmMI)Z=8mF86I; ze^lEm$?_Cx8r`XOS`B!+0e2d3MHVOAYry>mTz~QLh}%kFPk2L5u$7*0!W;ax8~n8! z{IwN-A>Kf>0i4QN&(i3Kgi)R!>Yh>hSjfUkGEj=XNgH(xb^UCk_Oqz8aocQA(Yw$6 z2^ltUWF%tzygR-zQv2F+6lwe{*^jOGj40gM2DS|Qe?a&)Exf{b`IUPs*$*#6g-PtP zgMWW}$Lzqs?~J7SjQ*EGc{kfT;#iZl7$E04DfmGqzM8m(N8=aiKE4hJnUZRX^R;2?HG3URN#zyg-Q+5eb~CIfU0SJH1li@OKZwgM%CXQ{Yg zLZ>WMmpoTJI#ssr{rj(;}RxrXby-uI~v zW5e++7oqF%ES!#K5^k1Y!{zXHAxAyim1CeESUB))y#F9{H5olyu^bzH?0A0Q z1+HVYT1qYsmzGmu;3$DQ42-^2H!xn=N6UBIl>gBC{{m1;0|XQR000O8002J#A+t~x zQV0ZCZ^@{WrXC-E%WA_g5JmU>3ZuO(KX3`ccAza;HBe|uOLsz=IHJ}ANv-3*ui_+5 z0$yb~x_8dp8B{IzvWxz}I-ymKvLrJgcrs9k+3)E}dRQS3GhwG^G>u}<&0RA)tedg`7$gvN26atWA z+_UmoJilxtR;bC-yo}R4&WcT1l5#=v#rI+4P5%WbPebk|o&m?)x1XT-v*#bn0tsT_ z9hjF0007aGyCQlC?o-Gv=>q@&-;+KgQw=*7004MwFLQKxY-MwkAtMx%vLhM-tr(Nf z7!{KxBM=){Z^@`70RR9{0ssIJ00000000000001hfsT_7Bt8M+lT9Qe29zNH0002| CY)JqB diff --git a/summary_results.json b/summary_results.json index e5f0bde03..81b4fea3e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.000", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2819, + "Performance_Result": 10.0256, "Performance_Units": "Samples/s" }, { @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.42205, + "Performance_Result": 0.434305, "Performance_Units": "Samples/s" }, { From d1522f2a45538dea72d2e6049210bd26274e6490 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 09:22:32 +0000 Subject: [PATCH 060/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9334 -> 9334 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e56f2e938..847e49368 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 09:21:05,211 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-13 09:21:05,212 submission_checker1.py:3270 INFO] --- -[2025-02-13 09:21:05,212 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 09:22:27,472 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 +[2025-02-13 09:22:27,472 submission_checker1.py:3270 INFO] --- +[2025-02-13 09:22:27,472 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 09:21:05,212 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-13 09:22:27,472 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 -[2025-02-13 09:21:05,212 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 09:22:27,472 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 09:21:05,212 submission_checker1.py:3286 INFO] --- -[2025-02-13 09:21:05,212 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-13 09:21:05,212 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 09:21:05,212 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-13 09:21:05,212 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 09:21:05,212 submission_checker1.py:3307 INFO] --- -[2025-02-13 09:21:05,212 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 09:21:06,184 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 09:22:27,472 submission_checker1.py:3286 INFO] --- +[2025-02-13 09:22:27,472 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 +[2025-02-13 09:22:27,472 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 09:22:27,472 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 +[2025-02-13 09:22:27,472 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 09:22:27,472 submission_checker1.py:3307 INFO] --- +[2025-02-13 09:22:27,472 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 09:22:28,572 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index a3888f7a6..d780a02e4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -687 +688 diff --git a/summary.xlsx b/summary.xlsx index 50d515fd2ec12f5a3491e4eb87ec6adcf3d05a3a..1e08eb6c59322ee8e2ea59f217c73edb86235032 100644 GIT binary patch delta 294 zcmV+>0onfcNcKpuq8C5Wdk>Jx9LMUH(Sk{Ey< s0onfcNcKpuq8}6D%-% z0H$^%q7zc^5ov#1I{*a7(1HRTJkGNmU%UaF@{B#5Uz>Jx9LMUH(Sk{Ey< s Date: Thu, 13 Feb 2025 09:25:34 +0000 Subject: [PATCH 061/719] Updated results summary [skip ci] --- README.md | 45 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 7 +++--- summary.json | 54 +++++++++++++++++++++++++++++++++---------- summary.xlsx | Bin 9334 -> 9445 bytes summary_results.json | 44 +++++++++++++++++++++++++++++++---- 6 files changed, 110 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 847e49368..0f13a64ff 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 09:22:27,472 submission_checker1.py:3263 INFO] Results=16, NoResults=0, Power Results=0 -[2025-02-13 09:22:27,472 submission_checker1.py:3270 INFO] --- -[2025-02-13 09:22:27,472 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 09:25:29,515 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-13 09:25:29,515 submission_checker1.py:3270 INFO] --- +[2025-02-13 09:25:29,515 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 09:22:27,472 submission_checker1.py:3276 INFO] Open Results=16, Open Power Results=0 +[2025-02-13 09:25:29,515 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 09:22:27,472 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 09:25:29,515 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 09:22:27,472 submission_checker1.py:3286 INFO] --- -[2025-02-13 09:22:27,472 submission_checker1.py:3288 INFO] Systems=12, Power Systems=0 -[2025-02-13 09:22:27,472 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 09:22:27,472 submission_checker1.py:3297 INFO] Open Systems=12, Open Power Systems=0 -[2025-02-13 09:22:27,472 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 09:22:27,472 submission_checker1.py:3307 INFO] --- -[2025-02-13 09:22:27,472 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 09:22:28,572 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 09:25:29,515 submission_checker1.py:3286 INFO] --- +[2025-02-13 09:25:29,515 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-13 09:25:29,515 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 09:25:29,515 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-13 09:25:29,515 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 09:25:29,515 submission_checker1.py:3307 INFO] --- +[2025-02-13 09:25:29,515 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 09:25:30,511 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,12 +28,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0256 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.431415 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434305 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7236 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352675 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1939 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433269 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7236 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352675 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d780a02e4..e4d4fcd0b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -688 +689 diff --git a/summary.csv b/summary.csv index 09ab6094a..8fab9de11 100644 --- a/summary.csv +++ b/summary.csv @@ -6,12 +6,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9355,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387262,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0256,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.431415,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1939,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434305,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433269,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7236,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352675,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 417490ba7..9e910f2c7 100644 --- a/summary.json +++ b/summary.json @@ -210,6 +210,36 @@ "weight_data_types":"fp32" }, "7":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":20.9355, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -239,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -279,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0256, - "Accuracy":"acc: 75.800", + "Result":10.1939, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -299,7 +329,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -309,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431415, + "Result":0.27177, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -369,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.434305, + "Result":0.433269, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 1e08eb6c59322ee8e2ea59f217c73edb86235032..fe8432bb6206a8a1c00b6fe5cd7f6325e38c5111 100644 GIT binary patch delta 3828 zcmV3{s7(MbG>485ct&E+gRGK%8ROS7->xRK0b=?#mZaN5wCO{)=l!AxD&a1ifPM>HA* zzWf~di%`zeWhJqkdTG}EWD?91wS71YvSayplvs{h>XA4&`BiJGjrL*D_FW(!la0?x zK9u_?`Yr!*Hf*Vdv;o?$MK`WuWhh79Vt<F{%W)EawW5N zj1r6V^aLdunYGW1gv{C(l;~vEzM{k?-93GgFu#c3m=@@jRubNMIrg$boqyAUE;ith ztNnt;^q5mX8op$zC6jmc+V4^O+yAb<^t)%h^N&R`lw{e^I7lap`??>6jc}U%4YN!q z`!tvFXb{XsGS2cfIY?!a4wA)v7$kY>lMEirdb5$nQWi;8BhRH87u}Opd@PeJg-IH# zT&A|JdEn3IHGjUSMYGvcynmRbSu#~ysRn*D8wF$aXLTr3FPJ9kr^CjGS+2iVF{CVM zm08z{e>?hhz3i7*rWkcY*EP%Kp|g4DY&_%iHy$MxVgCdr039Qt(4hpNVhs40p&(oNtDcV_l|ovT|J5lt||MkRD&6#3j>qWmib#cz-hR{QP#NmfgK9 ztE!DN9_P<#6#J9GgI2fd&6>Xunz09oRSabh5^wDd5)G$rYr4gQ=4`5MFc4$Bf1zzu_m01{mZt6hcM=6n`=ro3wSh)^Hp$6#$53 z6?E1I*C;|smurOla9u!DkBg!>G8*72i$#mNcrzM1R3<4Wn^&ilz7uU|Gc&_u0T%b!?rkx`uA-S}ysAO-BNs-5E>VMAYVVp$a@+Xl zk*s2Pdm*W9A*tEI-bEz;JoHE~j~R7JBx)@B?2OSMk$;Q^h(rw%i5lCbMCxTkGL6tD z8f6u9)(2mSq$ORZ5eJB*Q9UjS5?Iu@1|?FjmdM3@whk^^=hp<>G^EFbqb?pa$ ztYUO~rLOvB>RPvrotZ2D)c#1|lW^0ZJTlMIUCJZ#_#b#=G{7TjkVn)wP0FKD#v{`R z@K{noXMcV0l}B3A<$6hXcOa_AMFEef!5d2B9r5^2Z1Gewz`c+(Hjvcxdehyp$0~MM zO9a{lOp_wX_)nS?Nyc^nNHQ8g5;ce<<8}cg%`zmJMgYl@3Oeh9uSn98F4s#yl1BBI zC`?)zFApTG=D!`uDh9Y$+G=j4tw#RbL)!u64u1!ZKy!ylEjWzErXCzdgHZ0_!eKNB zC8MFnFHvU1;VJ+q%d9v8yf{$OWg2lT3MJTam?-M+yks=4NujLLk;6i{ox2XDoM+c# zoD@4e&JW#9zEg+Jjo7s4Sar=ctzFscrbS1%uT-MJ!=xS^Mq^Wx4x>RT_ps?O8l;lZ z0DqNbP93fSpt8)XBdC?;97?)OBe?GhmK-h$vRB5G1KDfEuAH~1{ChTD->F}xFE$N3 zc?N45JM5~$*}*0HJxprZVKg@N>@XVSau3%Iqd_hi4RBdz+~Fz!F3YStf;wx?p`^<+ zf?R?_hl#=zmhtF73R|&B=l{dyJGJjzi+^qZjv=x2i(0_A9t~m?#93@$&$f6&rfq0`o6F5qYPM zo@?Q5GugBZd&e7MT9ao|WBfYQ)5Bgi!Lz$B-kOfni^lF`6^k5=aC;ToY#mVZ^y z(ZJIKC0(wUo}RauOh-A6;)m`jD;e89xvRG6<;m86S2yYQ zYh&XD+EB)0c_OFNMDe3+x|Wga(&byleQ|KNd;#}Wp0yXvhC%Kb_NL3%h0|a*R+8VB zFCtsTi^zFp{;0p0N+o@s%a$si=6~54$_J||!>7^m6=Y=|2T>fP&&$^Hcc2$juN_9i zk42{zv*7oI{P@BDU>$t$txrEI+9#!@WJ7SyKLr3#O9KRx z?jjnqv=Dj$4vFo5XafWQ0F@8`02Ba|Q6m+zSUuGVB1>X>16ak6Q z5Q0du%08jSE)(zC-ClQ_N&bGDOEQehCX-D@S_uhps=H5D)v4;D|K%~y$OcN0a(1TK zhNTf;5sy>0KGW`QFZ4hoLQ)pfj59dXo4G^dX4TPr;b>&ELO$x;S&_?G_- zOqhsaGHSSp@`g-C&g!AFL?w)aCCn0J7SC?$e^SWFQ#bm4Nv4ZTB6v^xWt#Z6A(8hSW#9b zjgW&Qkji^>&QP6E$6UkGEz7c7aHo#PR8l1?6>)Q`Ve@J_+8c9`uA$kc3 zl&U~LkBUk!o>WAlhvmjGJauRc2~|Q{Mw}(-+B_++lVsIO6+xwER7FfK9|NzQw~Kkp zHAP~ld(C9d>* z_`cU6URNLT5=gsv_n{&Cl$@}CcCoc7aEt4^%dy=fo^AM+V-vCioF3`;M(FwB0hlTv zG&-K`6LNd~bv(OS&c}=8&H2@QI^OQKFddDe9`Rj6^+R+M5Dd^=N{}g|4TcqzvO#0( zLjKhNt9GuksmE!ORNJ<%KjQXE^CE+WMt7*4P7B^`!QB=d$>Mf+HXIxQzmP zz*~BPje5cXZ}Hb|@z-wg*KYCGM*c#)hI(T-l)0W~(H>bwMX~D#oUzAJ6w&)|G<;GSuZrJ|= z!nbMR6~@!A-&@Ikcrj{!OmdU&{M$NahlcacNaE-9R|@6bZ10F;L)Kz|LKL(VJDCJH zaV?L@f28~PI`|OAkB#Xstb_(OnQV#dhSar1^02I7PCG7f(lDkqbb;5Vz1QqbG!;`< zRR6bP^rzvE$!Ky;F5yB27|7!Go^-7cIe3)w6DD$yLw(6}dm)E^_JYIrTry-|GU$MW z95T{8$l;9S=kp%vhK3uW(c*dH>^O$w21gU8XLz3He;+s<$8ZD7Bjj>CTTaI_g?MNn z8BQnj<<0PVp{7vVQxj?sT22_)&h|sd#boqsb<{)~IIbUre(1SYr-QPzA6S*b@xZ$C zN0~wwMR&I=u$p!XUD$iecia?04*s)17GeklZ#nf~lcpXkf7vFb6}AIy$*F-tU0QMz z(k2nLevs5Ue)}p;;v|q$kE@yec6L!U->EM71?Q#FHOe!B642ZTsW&zHXcy@NN_=2i zu+C_x(EuLJo4blz%8i3JXDqlN!6y-|AW+Yl_p*z+wY=CkO~QCu6RScs+DV9WS7 z+W;;IxyK3ue-kV)ya%RsBBCQw@Cj*uTsHy)$IyWS9X!sn9G|@boboptM&-h(WEd>m zw4TJtz4LN1_I;oA`LK*9UP};9 zL)34)9Tz$DfkI}sE$MaO*{b( zxvxJ#^Rwq4%K{1@aN5xF2mk6qB+e8Ui92lg}6xlO-b%9B(=GU?l+n q08jz|01*HH00000000000Dyr5ACnIxJpxxClTIWe1_>en0002Lmp_*P delta 3697 zcmV-%4vz8VN%lyv*9Qs(6s^CP2mk=mI+NiCA%Ab;NE3$d_g7ffT+B#|P5bO9WR);8 znGixUpD(wja9eE+78#6pxh$nPyQk zX{bW0tBRb2Nk5tl8|v-N*P5p)SsqOK!8n=7hWacs^_Qcc4j+^B*K8zZt`rU?SwkJ= z`G534*RyaW;~>+Lshku?21yzRdGR$J>e)1vL4R%(k9DJ7cl0=jChEe&LAr~NWH5+A z`8f$^v7F=!C#f6uIW?z@grM+^kyx9OGe$yA zO^cC`b92r}$lSIW37OmrMnYz{gA#+x?j<83vwOuz$n1V+Bqo2gIsDb;;N?nY?F1zb z38sk>gUs4zMnY!o3rbAV(N~nXowHh*^D5hoYa zro)^9Vspv3C1>gCb=-raZ~wdc(rur0&OgqRp{&S;hEYD6-D_bI>+!hw8yA^Q@o6g4 zK`)vNWLo5FvX{#&?`5<5ILgY@C+j`fT3ydmDd$O+UglEzMZ3AQPi0o5Fqyv0Woj$W zy>L3Mh0|FrnM|J2*(5KLsejs1)eDo!AR4Mat9_XV(Ku5-9qI#Sx&FS7kcy;L7DJo= z+cK`}#kj;G#n3F%GV5D@;$0p(7tc8TjYo+?E{Y~f06IoOp+gBk$4JOnPTdOV7;~}$ zoK={&(A*;fJYP2_zmRRFcd~L^xRgld{E!h}GNU4=@XFmIk;73h2!G4lnObr8vT#)! z7CcU$^CS&Ny$3^cw0bR^3B%ff#C?Ra1Btip8i|&txrXWRAbLEA9-e4|&oG==P{2n(M4KT=QD1?j#C}cD)xqmM@U1>NDIR_NPDhI9A z!4-c+zk88t!A3(mYg9OIjgV2n+CjK~-dx>!uDwhKj%LhrC z?V8>eEcqwx#{!?M+k#?=8pozsq6V>KG=L@Zq#v+E4Pq&(uw)znmK6?Kg6<`pLra_K z1hJ%69mWct)_*V>zfQ3f{{bxb5#ufoaFzpGrK`SWTHBUOezWOV;QozmP%cs9IFw7& zAeW2=xI_(di5le6sN#}w09;l$Xsr&ua!Fg-TqN{H5v)3l6>^ChV>gGi#rbt#f&6_QLRbW5Yk zL2GsJ6-nCCW;*dqiX^q_a8{7PqQoLvd zk9fR4bvyY^OLT6;dX0{w8NTgo%U;)ObcFv(C02OYv`B~1xU@=#(IAyO)afu9q>|A9 zm48*GI-CQbvZ`1|&{~>vXlXN@;DcAF$>FRZdu23`z3#az=PfG#o+q#Gv|y(z)+=_( z4A!=`)T;__3zt~%Vbhu&M&r_=9Y%v(?ohVFXpl=r16)>B?r;u(%c|NPK}&1Sp{31q zf?Pt04r7HWENU=?-E)=B|A)(WTEBBGHh&9vEK%gK+ZIfJ3z%3OV$%{HM&r^N9!7&; z?oh?UXb?8f?{9%EM=+$~=DdzBX*CX;?tR(~l5 z8(iADx5d52Ic>3kTiVSStDRR5n!kG%Ze38Lg`Hp9YJU9+hbtC25pD zFM2QEx}1%Jqd4h*oDXU?iGG{Oj~~Jh&cTP!`Si15d{P=p5&Mg@h;H6X`GFhO(cv^0 z%8MW!Mw3h#%Rv!nT`Tg6bnzPH{3UaqOy{49yYu@b&y)D&+eij|nU>#7RT(5n{_>@q zlGW>|N52EJ@er5+4!dzd9Pa}F0QwIA02Ba|z#|ra%~x$t<2Vrhe!s%XecAnxH@A61 zr7P4HRBb6$ffM(sGRd^$F0pg5(}jP3Zc|t`lfbfY(n*!7K4bgw%sexm4!+#y8QDT9 zQ^Ai7*K!O3JQhjHH^;{H)u|a8L@LG;mI)5W#sf&>%kZ~>l#1xBTpk;DN)<=8E#o`L znY2WI0bESk9;!IJagLHH{bRb(V5y z!dK#N;L=76({am1v^QirCZE9CSVV_|oK#7Fsy;VYjzlKPuke6etZwsEDJYwUlPya# zcAI9Y!kZqXg1m{&bqLX&7o~_nN`cNVen*UP3>lP6;l&Isz7%Wqjg=^NbD{7ht|cU` z3D%TV$ztT-7?k!NoioyBG%?R|OviED4xBdegej(Fr6X?dG;E*GMk3FJ&iCeS#bTv@ zE5s~e4W%v+Fyo>!iw7N%_-?hOmah*@V9m79RYN{az`fGS42o_$(=CO07X?)zI{(thOKAR(NO z$z<{8hy;P(BYxi;i4sV+chk1#F`MjmJD85e@ev6;D|GNpKypBLDM6*2wHR)p zR4p3Y6zZ=QShsVXO*2W?>uT5b&3oKlSzct&(&!Gg)9b+d9k|zlBUv2qpaTzoJ8T-0k)T5s3>oG8l z92$i#-F*l-osJ)^RF91z^@1=8BHwd*J(P=jD%A4AQ{mnpZHhb;!~L$%Y1?(`)|T(M zt^d3a{{>J>0|XQR000QHxfWOm1SxHnnv)eDD}P&|B^9;rNPEg{h4vv>im~T=P|Ro1p`*ApPO%V6x5Jk4Z?*$W^85iS z2!Bklz;F*t?MOr?q~Igc{132X;8wTadsbuIZOj^(4=stKk z8r!zb+Sy=@`3wBDdVL>OO{MNn#R1i&;FLRH!MM7_*F`*B!+co96K^Gmr!MLfZ>L3$ zeISw;fF9$XmCxDp%X*1wl9T&1CuuQT=NONa5K88oe&kL61*lF#{!Kgsj=66?LH)B< zAj<*@1r)8nmk0m=(mInXBUS;HlY%268@q8q9Pa}F0QwIA02BZK00000000000DyrY zlfNTD0YQ@iBsv1K7?V9DB^)VjmYO92002+|000pH0000000000004l2lpd3QBs~J? PACtW#A_kfv00000 Date: Thu, 13 Feb 2025 09:26:09 +0000 Subject: [PATCH 062/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9445 -> 9445 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0f13a64ff..ec24d5ba9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 09:25:29,515 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-13 09:25:29,515 submission_checker1.py:3270 INFO] --- -[2025-02-13 09:25:29,515 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 09:26:05,024 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-13 09:26:05,024 submission_checker1.py:3270 INFO] --- +[2025-02-13 09:26:05,024 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 09:25:29,515 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-13 09:26:05,024 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 09:25:29,515 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 09:26:05,024 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 09:25:29,515 submission_checker1.py:3286 INFO] --- -[2025-02-13 09:25:29,515 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-13 09:25:29,515 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 09:25:29,515 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-13 09:25:29,515 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 09:25:29,515 submission_checker1.py:3307 INFO] --- -[2025-02-13 09:25:29,515 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 09:25:30,511 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 09:26:05,024 submission_checker1.py:3286 INFO] --- +[2025-02-13 09:26:05,025 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-13 09:26:05,025 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 09:26:05,025 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-13 09:26:05,025 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 09:26:05,025 submission_checker1.py:3307 INFO] --- +[2025-02-13 09:26:05,025 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 09:26:05,974 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index e4d4fcd0b..01113a992 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -689 +690 diff --git a/summary.xlsx b/summary.xlsx index fe8432bb6206a8a1c00b6fe5cd7f6325e38c5111..a3f43067f32afc4809c40b9008a8910cad66a2ef 100644 GIT binary patch delta 311 zcmV-70m%O4N##kf5g!BtBk!@36(1~rA)ytv18vEvfkItcdK1zn5w(7h)H;6qDo)}g z;8TyQnf-QlQMuf!Hu(kTrO_42GlCM(+z6?+6?$)0={-t(U|O)&XsA#R9xdy;l3U7+ zgI8xPxFEqN5u`okwnDoQEXCOK9Vq6r=+IHz7^hf>rQ2f5_&3`EE(lp*1%U~F78pJN zQ#%sT2`Tu9v_GyL0D@y^L4gh)=UI*~-T+Sd$%a9>aw-{m3zOEfIJ);fL*vTin5 zWBwd}u3z4URa2=Cr{aL>QgF&0uwYzW;_D(Fu4O)~A6{RHTq~5=>tl9U|O)wXsFQu9?hG(id)K! zgEwa^xFEqN5u`okwnp0!EXCOKEhy%*=+IGI8>d)^rQ2Z3_&3`CE(p2D3IY>vZKSA>cvsWMs1O#t6 J^ Date: Thu, 13 Feb 2025 17:36:41 +0000 Subject: [PATCH 063/719] Updated results summary [skip ci] --- README.md | 64 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 4 +-- summary.json | 20 +++++++------- summary.xlsx | Bin 9445 -> 9461 bytes summary_results.json | 20 +++++++------- 6 files changed, 55 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index ec24d5ba9..88afce143 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,40 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 09:26:05,024 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-13 09:26:05,024 submission_checker1.py:3270 INFO] --- -[2025-02-13 09:26:05,024 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 17:36:36,518 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-13 17:36:36,518 submission_checker1.py:3270 INFO] --- +[2025-02-13 17:36:36,518 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 09:26:05,024 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-13 17:36:36,518 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 09:26:05,024 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 17:36:36,518 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 09:26:05,024 submission_checker1.py:3286 INFO] --- -[2025-02-13 09:26:05,025 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-13 09:26:05,025 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 09:26:05,025 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-13 09:26:05,025 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 09:26:05,025 submission_checker1.py:3307 INFO] --- -[2025-02-13 09:26:05,025 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 09:26:05,974 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 17:36:36,518 submission_checker1.py:3286 INFO] --- +[2025-02-13 17:36:36,518 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-13 17:36:36,518 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 17:36:36,518 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-13 17:36:36,518 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 17:36:36,518 submission_checker1.py:3307 INFO] --- +[2025-02-13 17:36:36,518 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 17:36:37,503 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1939 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433269 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7236 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352675 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1939 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433269 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7236 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 01113a992..a79760c54 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -690 +691 diff --git a/summary.csv b/summary.csv index 8fab9de11..e9c137b77 100644 --- a/summary.csv +++ b/summary.csv @@ -13,6 +13,6 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433269,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7236,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352675,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7236,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9e910f2c7..af2a6d70c 100644 --- a/summary.json +++ b/summary.json @@ -432,14 +432,14 @@ "Result":48.7236, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":16, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, + "accelerators_per_node":1, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -459,17 +459,17 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.352675, - "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Result":0.352299, + "Accuracy":"CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":16, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, + "accelerators_per_node":1, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index a3f43067f32afc4809c40b9008a8910cad66a2ef..7d598f17d9545297a6dbd1e94dd221051705bc3d 100644 GIT binary patch delta 3172 zcmV-q44d=iN%cvv*9Qe>hmn`?v)l)#0)GZBXiSee1*G9il(=MOu3q~+YJdCR)t7$v ztatvgNQROu8yW}cWN}~jqp%T9v%g`M>13bgG9C?r*+|A&z9t8$OwvKJxDSIQPkoZX zgIRAj(pbtO$!g@eRO6z1vWkyolBF<7W0lL))-@0O`Ml=O7qw_Mdx{scG)tyxD}U9% zk7lD_tp2PHW$FdfME!Kw7%|KB_bP^zC9N{+TJdj3zpj`463Y~$Zs@vZxjb|>51ox? zoc_k6#3JmUpah^}BosQ70CbFmT%f0R33M3l5Xo6dc!vom3&?p%c$eusSvf9jN+ffB zNRyW+ap}*uT_KU<$-wjT+nHK+_kXgisy5DeoIj^g>`w*{THUHQYyLuL#vUYAF_b+> zytOw-G@QDv=@t*7!-MGHi6#ggqr@WD!U;+ML`FhELnPQNxpC{oOff6~o&Dz`nZypi_4()3ka1M3V=wC?~r#q>*DrW0BwR z6Gj6JavBODqX7yTjZNA*U4Lsh4w(u7#Ig!H>w{|)p`^<-LRUVB>TyvNM@9oYZgl^B z0QtHO5*P;$qOLXT;?F}_$r$&ca@8@h+DNLKw(jh}lHX}R61cN&3W_CaEQ?}^8pM*( z0G7;7KVXR(#8Q-Di4`E0B^7i8eU@+zC0(u&z>-Gwm?+q-VKlBmv40f*0W7N+<31ZW ztB$SHRoBpsUCSl^u<1zP^BYs6T%yLZDVL~0E*TAQi5lb*HOQq_#wAvOT$WVOSs#4m zl9qJ2kpP!8s>ehjm#9H5wRg=Wxo!OONLDety^z$lkko8p?;?_a9(p90$Ba595;Ycm zcE)IsNJaxhq6UdXjeqS@BK0yNnMUXnjj{?l>w~XE(vmLIh+|PAX;hDkf&>;du1Sg1 zt0i)ApRI$76_NzU^dT&k%6yF!Ze9BUAgdVNUa70TnYz|(V`t{dKeay+_$1skD38qZ zbeHnTJpKnB84d7=8srf*PLuLzl<~+k0z8&f&{-dR<&lBFWee07*szNTLRjWZW)* zq*;a}(+D70QbA{Z@D)i~(&c&yNYbbt6NO1D?3S;YYNN?XmXwAIiod*e26 z8_FFH9D(KzlYd%p7>!LmIE)6N+{1;#Xb?(9Lycde%!tEP08o}$aRhjAprp$*;@A{Q zu;VaM)Zcl@Xk3dzS*0U~g>pN09ZEURuE#hjc6gj0x|@8b4xJmZY0fm8CHns3D8U%9>6Az<7Fc}SiS?1&6DgZFcoIC=oJW$f* z8UZlD&BH_?n2etXz^vHN^A?zY`H9Fob@W^dcbmzkW!O935Yw7GlN#gKp`IQ_V^dEL zqX8y44P}zi0F#Uc_ItE4PY>4!WwNY-js~6{D1Yg4y`=96K~#^6qDd^Hfh1O|*wgbC zlj$hOQT)(7WhGmoSbJyyv`CwIL z_b1jY__65JVix?qkRLzzAFP89zV+#6Mf;?*lx*lP z(rk1^EBObmTkXTSH0|XQR0JHcIkpT|*ALn%y1ONcY5C8xa0F%HY7JtoG z+iu%95PjdTAha*-gRa)aG8`9y?F2@1G3=yBpA1GCIog#-tw^OwzkVs(NnMTPG@BII zC&On*9?r~};b{2jAWAA~#LOaYQf! z@C?-t(M>=wKzAuYri`{2ZlRPd8e13guNGLfbCpd!PLrgn+rIvQ+bhkB3|bo9u6BAI zxZ8ny9XOK34i7r;et!p!eB9zT3hWN==m|FJ2|K*QU%SI!yTf0*!(SWu3-KD7jbT^j zdY(mFWEB;~rXO&|9!gbkX%46vbZMrIp{>uYai5DS+i}efb-nrAUyx@9M@}Oy_PgdQ zBURU&ha~f>WIwm)J)+gUt!mt`{|AJx)4~gkhhM+flKu2zG=G@neZKLpcFYb9=Z%rX z&+G3L%A48V5XY9R#Q=pUXel-_>Epz8JSP8x$%V zS<9TZU*e=?Ol#=^FHL(d+1qI9A#d^c9T*8o5PH@PvHceYy^1NX)F)buzB9heCbMI5 z3MVQjKo(aw#DBFy6q7xf959icf|^U7IuhA+BpmUlDkIw}LkA>em&Eqj4<|7{o)3r{ z8eWK|k*6WG=NOLH-y2eW!}tB*$BgPYhS#@zLQba_tF!5aLVPsCjL&BC)pC5fP=l-O z8@-_Kt5Y4%asxN)5ppt{Jei$(3hsq{l`_W)y*@f?J00VyqQEom#vf%0Jrv!|uD;c_ z>%`t#e&D7Ma`Z2=uohtm1c)DgXpT-1NoVhcoTt?Ijy8N#%MSE~(JFTh0sq#R+ULObsb!Pv1KNa~W{P(j>kE2$5qZGW+$ z?3=cL^E6#x37&H5DLw*YGZE1V$=QT7KeihHoMGrdg7OY$Nro?84_3O_hC#VmnSkqt0)_iMImy z<00rXZ>L4ZKHyOZK*qRd<)e6dZXedD&eQxpPV+b`nrumm1u2$a$B{Sv7oa>1`8V+l z7_(6z&;kW#hmn`?lO`iu0q>K7BO)95ALn%y1ONcY5C8xa00000000000001hfq9d^ zBS8VclK~_;0Wy<3Bqke(AAV>h0RR9{0ssIJ00000000000001hffJK`Bt8LWle{D& K1{op%0002+zSFe; delta 3145 zcmV-P47T(2N##kf*9Qe4aN5xFv)l)#0)MOhg2wciQ$QNNWU3{TclFxuQTyBfuDKABBx@n*9y4Oegy^m+@#2%tkWK@-;a~Ws(k(#eEngdFqo4 z9?W{Pk;YOMNme7zr5YFAlT~~ylPrZv8mnBUwyt^L&*wFNzNkgB*;BljrCBmnTYsqr zel!~eWA$fsC{r((ChDid#)w(2zgID&ENPWl*NT5T`gOhRmsqA4bwk%R%jKc7dFX6B zlx9ZKBzYvb- zWe*Z>?F|wQr*3Py#e?YZAUb%W2|~vxvBf)W6ckx&p(0w6LH3Zhd2A`?zQED7&m zIDqJsfXK8?L3AjH%=sbf^(9JNnnyYR9uTV-${rxzIa^DjZfcgjA|s(Bq6A1piA`4UQ@4af40nj+tR%eS5>C!(L6>QO=AksE$Hb7yyF`gg zFu!uEg)Q{_D5*`oqJ}5O`nz+|Du%ZQfPHrZK&S3nrfKv1i6#$VQBHPgNF&FL#v;Gr zCyWLdmE%1Xw#50$HqiPc6@-L!RQ2bTO!`;ow%byH9*QDa#YOVl8i zj0Uh|cKQKJ)F76k3`?v4u`H>eBj~e)b13O@jR2N3s>ejZZVjVxb$^Pb_zz%N#TfV5 zz*%){ovyluZtPkv`G-wM0-xWQ8s!o-mQA@t4RXn7fJ@XMm#9H5wK6WT0_3u!g3kKj zE0?sS%Z-HIC_+?^i9#+>gIsFwnoDxq_~((VVt9KYscj*t*}~pMB>z10NHC8XbxI^^ zEc)z>(IAnG28cur5`T#r+oeS6WkfQK&?g#Y6?E1IUx}n8U8WHSh@??HE(#J@)VKyE zQm>ZC#eKF8E>=hq9MgxeSSs^1Qn+>P2Y{?%bbF<)`ey1{w~d{dEC1B~NZ^xj)1W*u z&(mGXBlGwlcw{udBWjRG)HqGbqfy2q(+KcbQbA{Z@Rdhe(tqW8Nq2W3s>ekEkEp>L zO5+{z_)l!{R5HN5kTf=s)bx7O-Lc0ic34XU+67FLBFXqqniNUKb^u5+8bA^?h$Q27 z0VK^bB$-A4$&v~>>w~XI(vmLMOF)uF^_VD3S{W}7B(3JZ9my&NxL4Y0Zl$e8{@g>` z0p$({jzDvVNq;RkjK-!O97cms?%~2=GzcZ5p~f#!X2jtt04U3>I0C#lP|{@@aV!cY z*m0OB>hHW{G_FaZtkRLgLb;v04yBxD*JGR%J3P)0-A%qzht7@IwCGrM%{8rE+3Ti7 zN4T$4qQS$Y9vwzwQg%&Q})mF65ux=bUu?+TV2 zE()?&#*_ouYsIddx2XJkHeTPUU#BlN4Lf-TYZ^Q3s>0d9CHg%~YT02lHudZ<8su^h z*AAmWE*TAQS!UefDgZ9atUH1_YtEsh%QS*qfGo@5;|1DK#$$ORr_)67qinjCk?YdsTg82G zaJPH`_f?*?7tMx2?iu!`%h!d|U^Z5g-<;q5;FP)h>@6tnaYlK}~d?SE(k1ONb)laLZ0f8t0Gec!KW zd0%E8dfi!ImB~7qWi+Rb?prH8 z4C}`0B*{_+b@-P54NRDbVKQpCi1LO^M&uJ1YlEn8kn<`|<>&g!e<9C!`4yg!i`6Pm zrG&C>IN#7TqpLJaCEoNTB;<8;rb3ABJTG|!LU43`@jGISBFLbm5-+A`@eNPpH(H|D z&A7yuxDyaJCRkBcC5@1SBaq5_bk0znQO8`v(k;ufTX3h2$5c`!D;05bt6}qMI^=oI zRle5`OBzX4A$kc3f0U{~K#z(_FP>CHqKD3<*_2TSlBE>DoLgu#;rfN)@)Ah9c=w?p`;?rpcCoc7e{hTIyUVfNBc5&emSYpL z1Dqb|_(tgY;Q^Q`AT&Cj?Gti){dGLMSq7q30IPPcvZ=>ul2qHauRr4UO7kLvhDLX&olXnhZNc3Z9LeH< z`z?6Tf+HXIf4GeTd%#S{@z-wg*GB$AyoP#XIFz}bXVD&6Mn$pf z2b{6TQWad918N3snrUKa>T_${=c3AXx@L>I-hJ**$g_nbrx6#&UGt5R+SZ(lB=fUm zKep&2qUFX`HE!7d1H!jy;T6WyuisnAet0ozOmdU&fBf4zW`~CJ&Pd|t^;Zhz-E8lO zV?)+rfI<|s6g!y&IB_kH$$zB#_&WFy#*dBZFRX+HHkoXR?1t2}Me?w$VNN?PandlR zHFSa3roGqfO*9o#S5*JEV)UoskI86qPA=g>1sKTU_MUXD5IK01^Ajd=kVAdRb9*6& z_JYIre_S$TUoz-`gd8%`JjmgU4t_IqS4}c;_Ntv;|50)r)PMc=YJnK9mjA3 z%Om7+JX=o3Glh6)AQ?_4^X1L(dZDIJ+fx&25L!+c*v|Gt$i-yzY<1K`8#t~XgnsC` zR;Poqv>#ZN!tubm^GBIN7e#lsE3le&UD$ie2Y1{QLJs}~v%nT%2m}Ko@3E65A0L0s zYQr!PgztR{qq{BHA)ytv18vEvfkItcdK1zn5w(7h)H;6qDo)}g;8TyQnf-QlQMuf! zHu(kTrO_42GlCM(+z6?+6?$)0={-t(U|O)&XsA#R9xdy;l3U7+gI8xPxFEqN5u`ok zwnDoQEXCOK9Vq6r=+IHz7^hf>rQ3gE%lJ3j0xk$yUaw-{m3zOEfIJ);fL*vTin5WBwd}u3z4URa2=C zr{aL>QgF&0uwYzW;_D(Fu4O)~A4?Sp}HW& zJS9b%&l>VTiv=a~ufxcj{tHl@hWwj&1{||tAkYE@AaL5y@{=|rTLJNtk|QD;iS2)A z0|Wp7l@I^`6aWAK0000000000fPs6H&m%zru#*uaIsqb+Oe7{710(OTB>?~cPyzq| j5dZ)H0000000000fPn*(jU+w+SCh;nBL)c~00000ZBXk# diff --git a/summary_results.json b/summary_results.json index d5aa37ab8..5ddef05a6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -120,14 +120,14 @@ "Scenario": "Offline", "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node": 1, - "host_processor_core_count": 24, + "host_processor_core_count": 16, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, + "a#": 1, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -154,16 +154,16 @@ "UsedModel": "stable-diffusion-xl", "Model": "stable-diffusion-xl", "Scenario": "Offline", - "Accuracy": "CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Accuracy": "CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node": 1, - "host_processor_core_count": 24, + "host_processor_core_count": 16, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, + "a#": 1, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -176,7 +176,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 0.352675, + "Performance_Result": 0.352299, "Performance_Units": "Samples/s" }, { From 51e4dcf8dca4fd1534a9c70cc2a59a1c8aac32e0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 23:26:44 +0000 Subject: [PATCH 064/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 8 ++++---- summary.xlsx | Bin 9461 -> 9462 bytes summary_results.json | 8 ++++---- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 88afce143..91727b2a4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 17:36:36,518 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-13 17:36:36,518 submission_checker1.py:3270 INFO] --- -[2025-02-13 17:36:36,518 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 23:26:40,057 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-13 23:26:40,057 submission_checker1.py:3270 INFO] --- +[2025-02-13 23:26:40,057 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 17:36:36,518 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-13 23:26:40,057 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 17:36:36,518 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 23:26:40,057 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 17:36:36,518 submission_checker1.py:3286 INFO] --- -[2025-02-13 17:36:36,518 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-13 17:36:36,518 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 17:36:36,518 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-13 17:36:36,518 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 17:36:36,518 submission_checker1.py:3307 INFO] --- -[2025-02-13 17:36:36,518 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 17:36:37,503 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 23:26:40,057 submission_checker1.py:3286 INFO] --- +[2025-02-13 23:26:40,057 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-13 23:26:40,057 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 23:26:40,057 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-13 23:26:40,057 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 23:26:40,057 submission_checker1.py:3307 INFO] --- +[2025-02-13 23:26:40,057 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 23:26:41,025 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69388 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1939 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.88715 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433269 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a79760c54..c63806778 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -691 +692 diff --git a/summary.csv b/summary.csv index e9c137b77..d416fb7ec 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69388,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.095,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -9,7 +9,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9355,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387262,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1939,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.88715,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433269,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index af2a6d70c..162957f80 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7438, + "Result":1.69388, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -79,7 +79,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1939, - "Accuracy":"acc: 75.200", + "Result":9.88715, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 7d598f17d9545297a6dbd1e94dd221051705bc3d..6ccb3fa66d7825df631ebdb0fcab93b77978b43e 100644 GIT binary patch delta 2315 zcmV+m3H0{$N%l#w*9Qsjr+H5D2mkrHouwM6XjmI(9z#_j@0nq^2b zjeh`=B^7km2Vaq-C0(wUfFzCTF;SScGG-pgTFrkul2r_Fue{aV%3BT1vN!Gnx1ro& z!4c^0FsTQJ(b&|4!)Or7J#09P2BBm$)cGaKoH$$s0A-mMM}QdzO1exVj!mHiKMoT` z4W5^b#p_KFNdW@4|i+{)Yq1(xKYSFn7n;spjuDPbQD|_Aa=m__f zN_2Rb)TF~`Z0gcsG)UzhJ{?AbR5BW%vdpT(RRC0$nRNv9(wsv{muUnyUcr;YMM3t; zxN;zSt=N|H7L|X`&g(lh?DWN^V<*pGO=E{&RX97iM8k(kJv)rXrluW6gIw-m+kas+ z$R(ozF3X%dTm`^onRiD}Yt1>7beTqwOR(rLQJBIqCLKs&D|YGpf4F?7{+(;FZQwCP zmdEZ|Fx?$sqB+E*79K`pQx6ZLK`{4l@h}<$lhFW}Wkw#Z0synj$|Jzb10`Lq5dahH zJWLdV$ryS7%!(a7Z-M!jABntEOMlO`aJQLkT86#j6)~;JGpR9#9ct=fG&VK$FdATz z(@-WE4KT@QV82HzGxczdP$tVN=xAW-fs!uQOZuV^MD@5Rn#3|1NMf~$O+9ZhnT~QC z#Sh(6Rx-AIa#wBB%#*GEu5Qxp*T%*hw4sd0@xYFF`M+UOSA2AB#>cX2I_Z`SFAQ!8-WhTc3Vbv`dC(IVkZ{F)_FG{Gf^GL=m$QhJ-D ziNu>8g@mk$UZ@bFJ1+_zf)E^?pZ|#%!w^y^sKkpYTD;=1{6-5Dy9+MyC9VZTtqE3? zWkEya;1s0t9-T8$XEZU_uyo6^><-*%;t`cp$x21stTb$%pG|m{ah31Q-I|6{RfvCH zKn#T{5YWTC)bmFbk??MP?--ssG=i8ap{+yC;$&k!E3jwD%3DM}<$C&I`H|cJXVvdFUw8U}dxL~7-tWNO4jjp%#{CX_(1Cv=A9uKo z0$bx9J;6pjp~gG>wLAQ^JN&gf{I!w45N)8@80s?Dvozcx>oCu^{eUy}P^f}SGC<9s zOEYZ@ZGCQo`&?MsPT%ZM*PGA%1$lOGWHjXBuxq|DQdP~lNHV`l_H&CqAX?wss>Tib ze?a&;Exf?^{Ok8xvY%dz29tlh&$j+mj@g0XyfKpadHtP2c{AG^;@Fb42p|_ZEyPwP z1Dv>y$K*fKy?-6N3*-C7^fy*Q3!989BD*DZU6HJpwajUUB~Dt#w3aUL(zN%Iy^W?C z@|KU^fsugtfot6m+k0WqE1B|2eWFF^JLAiIIzJ|-aH4VoWO;Q%`c{8{VsappXH2A4 zP;<$?Baylz;fU|6jO?lm9T1Z`iS4r=PGWYv7?OTqxB;3*_CspVF&uYrFr<2h=Xw5* z8P#zNcVKyhoX#%RXR`~1cxZ;1oXr>O)#P%i23OlNdj7yur#i0H_xnMQkdygzZ+5CF zxEBmm${Z_j2k5N*jB6U<8F%ZCG6gP*?snI}YTI>U?<_xXQwTZ!7qh??VF(1i)PC2K zB_Aw*+E9e;KwGkEpiq~V?1VH)M6CysTE~B1*-o4Uvg&el@0__as9YXo8~uWHLaPd8 zNs1y+Omm@j75Zpa@gs_yr;5{7E2vNp4lV1uk{QCZg*U4WSTDdu0i+yZrb2t~4Z+y4 zJxJ=3;7~!_YAdM^rQKme**Dz*&eL>(C3wnzsi$}V#%3m>Ba*WjX?|=w0yx9af&}Fq z&XNqDy&kM|HycLf!YPIC4cxSz#MymtVm5YNmvqHw4f!+twSIjcSB-@lPQ?J#C1-?L zpkCX$#Fs@lTuYr_hZAoF@TWfLH{OnmOntzk5P(c^Ps(TU{L-vYou~OjoaS*>Ghy0s(0!+EDKSBKmvu7a60txS@c~0^O003c=IwMyB@spAxA{)Ac zQn3^S0077k000yK0000000000004l2dy~&2K>@*&5hOYRGm}gtCL6xge%B=d002+| l000pH0000000000004l26qAi4J^^Tx%p@ZQ8X^Dy004TtRyzOy delta 2335 zcmV+)3E=kjN%cvv*9Qq_hmn`?2mki(BsIO>bayyQ#13nTK)ZlxQY0DwNs}VU*bV?mMgvHq29adkE`X$2h9uJn zAb(j>L1%sN6-ips<$4K7(x@I2g-I*p<$n|!AZog1-f(Xr~9Yg)Uq*G-F#a9^oJ zgNI2yI*i7qCLKnDRPJHZVKhi3qX8<*oH|?uKxLU%M^G!xIh1snMsVL1EIC{hWUq`V z2eQ|ST{&-2`S)zRzEi(WUu+t7@(k8AcGy*gvx7_YdzjR+!)R>k*MX#}|hhYk~kDJ2kfK?+HOvkBg#7ETe%WR;$?4^A?loD92I! z&^=`(W7{Wp)i%96+4}G5Cf$B*Y`j1l%6KeKiMP#dZ5jn5SAN3bgsie1Rdzq_mW5=r7W2bVV!q z2d-P~!?`z>7hXILW{EPDqc3cr&3cyR$IF+Ri_d8^FFu*7av!B>6ux|$NN*_P{F|;S zqbN#WzT{J~emT7T2ea`Im;nX(ALn%yla3N0f7@=`I1qi`uOPH9?Srn?#WEZhf$aoF zb203sNS_Qw8adjPNUcbvNxyz6+euxG`1c> zOC!J{9;a-5q+MT~=zWa{Nm)!Y&frLU1fhKz{V^0m61A0yBkfMgVqlsgx`UhwgBQS* ze=fi@b7bTpKs za~?xBH05Y$*1#5AM!;y9@+N-GQYo6?0z{R`CYoh5r;Z+2w|W>hjh9K1r3{+z75^KU zFcHIS(sB{y4Vg{I2QbzKQQ;uRRh-I?f6bL+p7HWCJR%pX+dP#L%BJD?o~9YSO|w+u zO^-rC-b61{2+^JAC67P|j?ORsM2t}c8I)Aw#S|@G@* z5pr+>QhATg8LKmzm}gkJWm$Fy?lke3N~&a~B5u|iHqXz-JkPnx_xf%{BdIDxe=i|{ zQWXg3QBmo|ql!p$x4L%>UmY4lLY2@~5obxdHV+EyAX#;*ilEXnsv;&=4}HIz*TuZ$ znj*2&t!6TK=4r0zG(2x@8lI!lVn80Ciczj-9~K|U4RBWfehYNR4ZRyAg!2iR&cBXH z5coafyZV@yK)S`d4Gr0*s4O~?jt2Ba4lp&x`h;7kFb z(ev$qkgLnj(~IS5K3%Mq$LI62X}#ORbToIpl%!(Y3@U%SI!yTe}_`3vzHnvG#s=6aq*TVxd##ik!{#vV#laA^*x8FXo; zjiIg2t#O}=D%)|*4t2fx++UDq2S-jLF7~_TDS{y zuhYT{jE7&p*OLA8VlSU_5?a_~QWM!Nsq2d5ZduEmwqN3;WlU@70xwN_FWK8@>LG9O z_#GGtNf3J04YBExln_v?Hj$I@2gWC&vFAd>=AM@n>?AFdJ67^ zeU&oD3cWr$Ydhnr8=}B7?#3Tw3Oy9v&91)Hw(G>+T7KZB5OVY{v%wZ&2n2{9erS^> zA1r@pQiSb5Te51PP?whOgfwwPtp}1?Cx2hXPMie1>T-1NoVhcoTt?Ijy8N#%MSE~(JFTh0sq#R+ULObsb!Pv1K zNa~W{P(j>kE2$5qZLy*3o3?=SG+kf`o^pTcDLw*YGZE1V$=QT7KeihHoMGrdg7OY$ zNro?84_3O_hC#VmnSkqt0)_iMImy<00rXZ>L4ZKHyOZK*qRd<)e6dZq}&I)BHY8^EfM-Y)Kf3 z1u2$a$B{Sv7oa>1`8V+l7<1o#g8C1$Xduc031^3qm+%Mx0AQ0lBUb_MlanJN8~GpS zbrl2v0LTyk02BZK00000000000Dysclh7kU4Zs!v0C;RKb98xZWpk6EBNUS|BpL!T z8Iw;L6_ceS5F3aeerP2D002+|000pH0000000000004l26O)c4J^^Qw%_JiR86p4x F005@?OOpTq diff --git a/summary_results.json b/summary_results.json index 5ddef05a6..aa6ff4ad6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1939, + "Performance_Result": 9.88715, "Performance_Units": "Samples/s" }, { @@ -343,7 +343,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.7438, + "Performance_Result": 1.69388, "Performance_Units": "Samples/s" }, { From 3f12aa99898b0e211625342a512cbff3d8544c31 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 23:27:16 +0000 Subject: [PATCH 065/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9462 -> 9462 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 91727b2a4..b29b05058 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 23:26:40,057 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-13 23:26:40,057 submission_checker1.py:3270 INFO] --- -[2025-02-13 23:26:40,057 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 23:27:11,493 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-13 23:27:11,493 submission_checker1.py:3270 INFO] --- +[2025-02-13 23:27:11,493 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 23:26:40,057 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-13 23:27:11,493 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 23:26:40,057 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 23:27:11,493 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 23:26:40,057 submission_checker1.py:3286 INFO] --- -[2025-02-13 23:26:40,057 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-13 23:26:40,057 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 23:26:40,057 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-13 23:26:40,057 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 23:26:40,057 submission_checker1.py:3307 INFO] --- -[2025-02-13 23:26:40,057 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 23:26:41,025 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 23:27:11,493 submission_checker1.py:3286 INFO] --- +[2025-02-13 23:27:11,493 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-13 23:27:11,493 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 23:27:11,493 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-13 23:27:11,493 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 23:27:11,493 submission_checker1.py:3307 INFO] --- +[2025-02-13 23:27:11,493 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 23:27:12,485 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index c63806778..2034434e4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -692 +693 diff --git a/summary.xlsx b/summary.xlsx index 6ccb3fa66d7825df631ebdb0fcab93b77978b43e..160f20fe1fdd82f08da3b19d7e3b0b2d13ab67a6 100644 GIT binary patch delta 312 zcmV-80muIKN%l#wA|C|SNLMA3CLb$*Tc!_$?Lb?yYM{`RmhOZ!aYU^Ll3FK!U)f2V z1ib2Ubnl$GGpJhZRG0jM^U~-V6Sa3mtPa;Tr%59CdAy|sB=UY(BXVIafxG_$#5KGr$%lJ2I0h2tx!wLd_6D%-1 z08={=(Ge;5gtR}d8v%l2=sXS>OTY3amc@kC%_^1^(Scl0JCTy3j_q# KNLMA3cO-w3VviL7 delta 312 zcmV-80muIKN%l#wA|C|4)PC2KCLb$*KiW`)?Lb?yYM@Y;mh6NyNkpv&l3K@qU)fHa z1hVRKbnl$GGpJl1WE=g0bwaBOWl4%6P)u{7b`|<)R`DZ>oTrM@Rx7Ab4-PHsyOJ5g zw1qdT4OlP0MFFH7VWvWR?+wA&u{}uYlHgE5+-fVS52f8QebT=DD<-#e2?+x6vp2XRGaAG!gU6*vl zXbt%@{I!04A6Jcq8cxLk)g@ Date: Thu, 13 Feb 2025 23:29:17 +0000 Subject: [PATCH 066/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9462 -> 9461 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b29b05058..d17882522 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 23:27:11,493 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-13 23:27:11,493 submission_checker1.py:3270 INFO] --- -[2025-02-13 23:27:11,493 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 23:29:12,704 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-13 23:29:12,704 submission_checker1.py:3270 INFO] --- +[2025-02-13 23:29:12,704 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 23:27:11,493 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-13 23:29:12,704 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 23:27:11,493 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 23:29:12,704 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 23:27:11,493 submission_checker1.py:3286 INFO] --- -[2025-02-13 23:27:11,493 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-13 23:27:11,493 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 23:27:11,493 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-13 23:27:11,493 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 23:27:11,493 submission_checker1.py:3307 INFO] --- -[2025-02-13 23:27:11,493 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 23:27:12,485 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 23:29:12,704 submission_checker1.py:3286 INFO] --- +[2025-02-13 23:29:12,704 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-13 23:29:12,704 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 23:29:12,704 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-13 23:29:12,705 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 23:29:12,705 submission_checker1.py:3307 INFO] --- +[2025-02-13 23:29:12,705 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 23:29:13,783 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.88715 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433269 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7236 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 2034434e4..8a903914e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -693 +694 diff --git a/summary.csv b/summary.csv index d416fb7ec..a5528f020 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.88715,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433269,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434065,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7236,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 162957f80..98d3cd21a 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433269, + "Result":0.434065, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 160f20fe1fdd82f08da3b19d7e3b0b2d13ab67a6..8a2ea419ab9584e29902fb096f9adc11aeaaf352 100644 GIT binary patch delta 1850 zcmV-A2gUgIN%cvv*9QdGQDKXd+XrF;r{1))KnN`Ye|N-hObhf%D+%ws9D7-z&S^mx z8*oSi7c{2FoC4DDB}!Z}Ggq(u9<{&y@9Imxd)7PuSR_M9mJN-Abh5aw`%%~kr`g{y z%XG3&a~Y2Y!E7YsEMJp@R3_;lS=@&~lBYh&;K8gn8)+f_Q?-?9;77AjFjjw7hcfkoX`+5QY>b%Y`g;{a z%92)@b*=cfqhHs{eu-s@Q8)A~q1thaw|VGnJmd5pj}nWpe}WQ#j*(F4Py)~~5^{l_ z+9l9oxI-joCE*<=oGc*cCE;DB^JL|?uqlzufB7LzUZTXMw{N>bBFB?~=jT6XYT2)s zWmUCt#^d}sjbeW?c+l!ry;<`YLNoRtv5KMWLE^2wL89T*ZB4g$5FH*w2TwFX=olpy zxfV`P0w6LH3L;7XL`FhEbV@+PZ~(C+yu*Z35KF?lOy?9^}#iYP}1cZp(`Im^|&aCBclNxH@g2mfP7sC35UCIluOhgmy8CuL=AF@8st(d;}R=CE=wxttPj3&NlUuiNPtTk)nlTNOVl8j z+PmhG{A~R5NLDety^z$lkko8p?;?_a9(p90$BdI|4WEC1Q5x?E$bVvwr;-8g#iX&p zq^8%K?vCepFj3UtdC6#8i$YnY zC5MG_J9!;SInS=gI4QPxoFDo*`A#i5H)7MHW7Rd+w032$n;sqEzEX(}50jd77>!L` zI*bOX+{35CXpl-q15}n-b+`(E$}+Q#pkA7DDCvJPjo`*BcyhQX$X*#&4rH$t+j8Eb z^6%MseW!+SjMCSDQv|qo&OJ)@6^9@Ew&9jhRE{RT??kW157lB znACs5!)R>k;bAlg<{mB{MuT878UVA*$ir0tV3t{V1bBI%q{}q|V1k{8i9#?LLl1yi zv7_fLF#qx+k#}n8xfbp=lTFL8cf2B|HF+jA#;`+8J&eYtrXEHEOmZ5^B%=W)84c|B zXl14zt`W*)Sp^*pOg&K2<$6h96oRN87e#-QSVjX$tX8q9=Pf4FQI4bdq2H91jBTIX zRogW4Wb41Hn{@lNvGE3NDC4m_k<)3S_)#`p%gA-<^0nf=IJjHBf%_`Y+KXnxAU6$r z)8+fZX)qfr$?wZIk*(rQX!#DZGLM5O4$^Js zW$XD%(2J?p4x{16qEm}m@cTl3{NR7E4nFwSr=Jz=lhRVMp}$D8(G{)aAGmI{59i)k zUU>01m?g?ojsgu~FW+h|KBv*V_++ZeeUzqA`0{Ncy`hZrZ@Q|Cq9}d&l26I{ z?eO*=v+)oy4h3mqe%BRDEp=j;5<$5 zumn#z^%VEO*vv#UAvv3o=Et@JfHMp&NKoG4EXnZM>%mI5*f1y;PAUA+z@qge&hCQ~ zv$5;Cq$>t%$e-cQ&CA=cYAjTLDh8-7IU~#h_1e}YzAVDwTI&2JoOmaIKOTd=@HQO9SF<1~-6Vw)Eve;`@$)sMXDKLh17-V9b5}3F<$y oXduc01lLhvi?cQ(Jpl!2Vt&^olXxUl0cVrVBqIhHA^-pY0D0nekN^Mx delta 1851 zcmV-B2gLaGN%l#w*9Qder+H43+XrF;M{l*WKnN`Yf4_*|m=@@jRubNMIrg$bozsFY zHsFv3E@(`TIR&KQOO&`|X0BfQJ!*gZ-_@6X_pEpRu}FrJEE^gJ>11(V_oJ{8PP4yZ zmg!`l<}w})g4syMS-vI*sZ7#AvbYa}Bu{;k!Gl?EHquziBFSpxxm4q#d$Nj;Ws;>Z zNn@4Ef7I4B5B&MO=Fbj95V66VE4rS^E(?tDr*cdU(_4g`< zlqIb)>ss+|N58I@{SwO*qi*QBX1P3cHV>VRXPo}Vqr@WYpP&SwVkn0^Yh!8T6Xuc ztg1H7c$`0{QS46!4_e)-H*5YvXvQ8SRxy-4NW8T-NHmz@4j`6ZYc-JC;P3C(*?-O#nGYiA64k6O;gnjD(Vi5+D&JHd(<>-4YTp z+#wh3SxI=uC7hhof-chl%|mHSkBK2ObBPj{`~;Ib3^9M-@DoM@400L@A)^5b8I4Wa zI$dix4w(u7#Ig!H>w{|)p`^<-LRUVB>TyvNM@9oYZgl^B0QtHO5*P;$qOLXT;?F}_ z$r$&ca@8@h+DNLKw(jh}lHX}R61cN&3W_CaEQ?}^8pM*(0G7;7KVXR(#8Q-Di4`E0 zB^7i8eU^W44kcZ#5x|m0^_VEwtzk5-L9rD70W7N+<31ZWtB$SHRoBpsUCSl^u<1zP z^BYs6T%yLZDVL~0E*TAQi5lb*HOQq_#wAvOT$WVOSs#4ml9qJ2kpP!8s>ehjm#9H5 zwRg=Wxo!OONLDety^z$lkko8p?;?_a9(p90$BdI|4WEDSD2;anM>E6wK8TN$Xd;RJCaolaId`8+{#-G&9XP{1Gk~vVZjmT?l7qbhtb&7 zgu`eM$~}Kp#(n;6GaW4myE`>D3n!N za#$$0lh>h?^Xz(zlVXd<`Jvm%cWTkO5t|+ztFF1GwJUqw^ymopl}dDYnAD`hXl&}z zVKhkP9zGpLgH$papt8)W!&LxOmYH<~_0pU}Ntb_V1UFv6lfy+p_R6?&AbYLYmh%>s zf6vbAJ2mX|#inB?&tOeshhJ4VJGex{he9Y%v(?qS+AU9J%T6YM-p6oSbZdH~Fd z9X)S>`IjGwyi-fhwQ#qYY+8oB;}tQj$up@jh8=3^VKg>1^)MP>lG9Ko84WPWXkfob zD>LVc9j*Gu}M5JdI3D4Ku7G8#x?wTewWZ!wvUava4E-BVUFwtaF} zZPUz?t^clW((TvA#v8PujK}gsPN#|DN7-~OBiE(N*NXe%;BNT_?yEd&FPaU5+%)V> zm+uRw!ECG~zc1fJwu(2A^UC~De=(Ix`aG8{RX)wLGn5ZjRfbQa-U6FQ#5QjD{bJPAz7^?+f|yga5%g_~2Wgepa+kN=wOx{vyprSG1CU;JVd5oO@$= z;l<-%mMBv>$_Co3XK8-Ce5<+moJRBFlc_5AQJO~K%eRU2hBD5->8dh{qV(lUJ|*k7 z!`puVv+)oy4h7aoS0yErCm$bwYQr!PMfd#*qrEL#rVoVeKwGkEpwN_-?u0aPM6Cys zS|@*B*-4xPyy|jv@0__as9Nk)m;8eB(&!o$S&kCW+z6@L8ojTV=>tl9U|O)wXsFR1 zJX$n26}OZd2d~apa6y7kB1n76ZH=}eScSN-T+Sd*@jWMa4H%07G|v{adPjx zoQ!?nXMH(ZWBvkvu3p~8Ra2>lQ*l5|B{<~{STL@s@MRGX*D)Vf@x&Vm;&C7KnYZI2 zhdvNV3_uTYPs&I6^t@gqqK4$;HqA*|lxtE_a!(8LHH^IKKLgcq$iImvz#;ebCusfv pvu7a60tD};c}}x6BRv5H)<{<+C6jj~Q~_v{%p@ZQ8X^Dy002F0ZVvze diff --git a/summary_results.json b/summary_results.json index aa6ff4ad6..6ca28305a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433269, + "Performance_Result": 0.434065, "Performance_Units": "Samples/s" }, { From 98c5ec900095930ead5bb2f86556ee16c21408b5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 23:47:38 +0000 Subject: [PATCH 067/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9461 -> 9463 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d17882522..09fadac77 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 23:29:12,704 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-13 23:29:12,704 submission_checker1.py:3270 INFO] --- -[2025-02-13 23:29:12,704 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 23:47:33,198 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-13 23:47:33,198 submission_checker1.py:3270 INFO] --- +[2025-02-13 23:47:33,198 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 23:29:12,704 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-13 23:47:33,198 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 23:29:12,704 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 23:47:33,198 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 23:29:12,704 submission_checker1.py:3286 INFO] --- -[2025-02-13 23:29:12,704 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-13 23:29:12,704 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 23:29:12,704 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-13 23:29:12,705 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 23:29:12,705 submission_checker1.py:3307 INFO] --- -[2025-02-13 23:29:12,705 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 23:29:13,783 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 23:47:33,198 submission_checker1.py:3286 INFO] --- +[2025-02-13 23:47:33,198 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-13 23:47:33,198 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 23:47:33,198 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-13 23:47:33,198 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 23:47:33,198 submission_checker1.py:3307 INFO] --- +[2025-02-13 23:47:33,198 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 23:47:34,184 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,9 +27,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.412678 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406474 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.88715 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8a903914e..e0f6509a0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -694 +695 diff --git a/summary.csv b/summary.csv index a5528f020..b3bb4a603 100644 --- a/summary.csv +++ b/summary.csv @@ -5,9 +5,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.095,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.412678,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9355,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406474,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387262,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.88715,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 98d3cd21a..9a5e4fa7f 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412678, + "Result":0.434726, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -249,7 +249,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.406474, + "Result":0.409089, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -259,7 +259,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 8a2ea419ab9584e29902fb096f9adc11aeaaf352..5b13ed3a649a4188028774efe707eee000287b1e 100644 GIT binary patch delta 799 zcmV+)1K|AiN%u*x*9QgkpKhA-v)l)q2Y>nV5N(H*PPjY3M01EqEj*0IrXC(fgJACA z;$buhCZho`%Zxl+1psE5l}CV=2THnJBLF7Yd6*~!lQHxFm=!yE-U9P4KN5MTmY!?j zZZp|*o6ZhXoz~=;)EL7KHT5tWo0@tU4KT@RD3go^m}E4t-=me8dbma?lVufjG=DJl zKuMSDC4Er{qIz5uO=1}hB(Yk>rk=N$Oh-A6;)i}yRx-AIa#wBB%#*GEu5Qxp*T%*h zw4sd0@`j;N3#Y+stR%lL-$b^G zH<9zo{84`~l}h?Nmn~I3&9gI<4}VrwhEJpAJIKmB4x%_npO>xYFF`M+UOSA2AB#>c zX2I_Z`SFAQ!8-WhTc3Vbv`$k((e*jQR1GDrHE)E66 zFAx(YlP4b^e`>=p5JmU>3ZuO(Kl(!04zwk!1`2g)=uSwJMAUj9sdfDKm7T;%z^g7t z_s*F+gUZETw$U$GC$y?imZT^G#WWXcTcM9;89$=Pd8#;VwSo%u;LxJJDVZTmTX?hD zfb{}g6hO)mW-7Gv-Vlr(+kvDm2@Vy+jkc2dP}(gvf0TXGE#N#&@390=IrS6|z}QSg zbVPDCAbKa4mIy6;8Ynz@Pe{&%7NMIrIUKLI84zds04& z=a*)QBrYVs0kdZy$^r%SpKhA- zlQtt;0rHcQBRm0olg}eT0m73JBsu~#8Iw;L6_cbR5F5lV5ECT<002+|000pH00000 d00000004l26_br5J^^Wy%p@ZQ8zKMz005nXbO`_e delta 837 zcmV-L1G@b8N%cvv*9QgHQDKYnv)l)q2Y-gh^4MJqrn>`7G>4ef!oz56>fvED2<9Fx z9!7&;G8zE0%*ex40AQ9`c?5WQprp$+0$_rjhlxTk8AA_%S+S$%EinJ`BawG%>A4o} zHj_=uuy?#7rZsscHO8<*O+AdprluZ7159!n$|R!!CK(Ov_h@CN9GHMWzBssBzJdEH&)SP-!yq>ed(-9n!f7xYE6MN6H<7L4 zP2{{Xf7D-0rIJ3+WlNP$^Xv@egMU?(;nQgO4ze)~BBp?UT|{vZ237v(Xi;gk>)AM$XBkDZOZ{swNvtpYUB!3`T@zsyK>OTYJH00mJ6JX4J{R!$nvuPm80tMGm zVTH*CYV|08jz|01*HH00000000000DyrMACoa36O&{l5R-Qx5DNeR00000004l2 PXOqq(BL*2F00000SvP^D diff --git a/summary_results.json b/summary_results.json index 6ca28305a..1bc03a4d4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.412678, + "Performance_Result": 0.434726, "Performance_Units": "Samples/s" }, { @@ -559,7 +559,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1020-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.406474, + "Performance_Result": 0.409089, "Performance_Units": "Samples/s" }, { From 7a8cf0ebbb1573ac91763d42233ce4ed767b3a2e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 13 Feb 2025 23:48:36 +0000 Subject: [PATCH 068/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9463 -> 9463 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 09fadac77..392956fe7 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,28 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 23:47:33,198 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-13 23:47:33,198 submission_checker1.py:3270 INFO] --- -[2025-02-13 23:47:33,198 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-13 23:48:31,354 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-13 23:48:31,354 submission_checker1.py:3270 INFO] --- +[2025-02-13 23:48:31,354 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 23:47:33,198 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-13 23:48:31,354 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 23:47:33,198 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-13 23:48:31,354 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 23:47:33,198 submission_checker1.py:3286 INFO] --- -[2025-02-13 23:47:33,198 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-13 23:47:33,198 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 23:47:33,198 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-13 23:47:33,198 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 23:47:33,198 submission_checker1.py:3307 INFO] --- -[2025-02-13 23:47:33,198 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 23:47:34,184 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-13 23:48:31,354 submission_checker1.py:3286 INFO] --- +[2025-02-13 23:48:31,354 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-13 23:48:31,354 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-13 23:48:31,354 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-13 23:48:31,355 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-13 23:48:31,355 submission_checker1.py:3307 INFO] --- +[2025-02-13 23:48:31,355 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-13 23:48:32,347 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.394347 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69388 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.88715 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1221 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e0f6509a0..1e308d8d6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -695 +696 diff --git a/summary.csv b/summary.csv index b3bb4a603..28e60dda7 100644 --- a/summary.csv +++ b/summary.csv @@ -1,6 +1,6 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.394347,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69388,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.095,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -9,7 +9,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9355,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387262,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.88715,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1221,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434065,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9a5e4fa7f..61eaa82df 100644 --- a/summary.json +++ b/summary.json @@ -39,7 +39,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.394347, + "Result":0.32368, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.88715, - "Accuracy":"acc: 76.200", + "Result":10.1221, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 5b13ed3a649a4188028774efe707eee000287b1e..ec1441173f9528b70a9ac349f819e33227544927 100644 GIT binary patch delta 3156 zcmV-a46F0^N%u*x*9QqaJlosE;ith1}NY4Q>!F8%qoD2e&TlHqmUkJ_EgTyL^vImK`_6CWDQ@1tU;z4wH5FI?x z1fgS;SmatbK?#7!NGOOX0T3Ap1<@%15yJt*lJE`_PC+aQ?=qdIAUYI8=KPT5^b#d5 z`JudW{(n6nRxy-4K)iFdmPFmuG}q-xba@h8Jktb_W0Y9rVmLtwkjO|Vi6{XQQDT!7 z{M0QW5yKsF(VmrrcU;2BIW6ci4bVK4#`KsNGBcMbami1>m0K-rq31_QZR!;@JUQ0i zos(8EygdNyyBh#Hb=NXYo99n7c>s%YvP(l6Ie%s}7WoZ7VKl%Xr=bus8laHT*rct~ zwT9!6sQ^GMtDv(!xJD65x?Cf4<%6gm7e#SoG{ECV_umJQuj?RzaquAOTC*kN*KkMgu^i27iG> zjnkw+8f8E-jR24(6?E1IUxB10U9OjOcL$<+ToeF_8oZ%2-Vu=h#1>B_1Kf*AV}nUe zuQ%Ntj~&GhYl%R+fN4@B8UIO>BFWee07*szNTLRjWZW)*q*;a}(+D70QbA{Z@D)i~ z(&c&yNYbbt6NOnT?3S%1X<_sUz%t-RIHEPLZVa2v`U4jh5z4wG7N7>!Lm zIE)6N+{1;#Xb?(9Lycde%!tEP08o}$aRhjAprp$*;@A{Qu;VaM)Zcl@Xk3dzS*0U~ zg>pN29ZEURuE#hjc6gj0x}AKd4xJmZY0-5E@VJFXEO=E{$RX97iM8AhgEjx_Hrk))}gIw<6+F>-vC8Gf@%Zxi*1;AyQ zbw^NV%{i2GnMROHaOf~mn18}D9vw(wD>mu;f4F?7_ML07?cXtU!`5~<>4dujO!S7B z)WO4OY--_QGzjJ%CLTtEU@{s2v&_fCRRCa?Ie7$Fd7z}rH3DFQn}>-)Fd07&fLXDj z=PfY*@*|OV>gc%^?lzN6x9RNg)M-tgNsaOAP)`q|v8ktr(EyX2hJP~2Xn;vZ1N%K% znWu+qgfdxHK}Q2m50rGdUefo3Agaej(Il49KoYA}?CE)n$#j(CD1PXkvXZgwle=o0 zUY>0IcXg9)zcw~rpbcd_mM3yLO%y-MrfV6wE?vG=+!qIT%NKB8Xx7=ecaD@@bx(p?t8aGJF~>UqM#paS+8p`n+sC ze+POo_1a-H{8)5qF$;cQ$d4cV57xm4-}>~kqJ2_YN;dQtX*Rl|mHY$Ot@h#E8_NqX z9tX2TnaWW%&}KbL^W)`9&Bf<5niro;Rk@GSGzwq7O{6!JaTWeeSCvr|r7vIdDOtZ9 z-u?qnO9KRxZzCGB5D|I-1wScqt`(Dy5+Q%vZreBzec!Jjv@h+0uGYme92bG@1V$6b zu#+NvG8k#(XjdY&B9+Gd`lT;PU5(^4n-th5!)Hhy&dizNsQ>9P&&UImBIWEvvkgll zz#<-}Y;~gDT%YM(jR;9uOf$~lL|cQV9I}z z@G_@T{Vi9fC`zEQxCfAVW;&MTn>kIHMj~D@d7?QTMfjtimB~7qWi+Rb9$0sJ7}kwfNs^@u>hLxH8<;Q= z!(`NO5#bntcd7^~J)wAMLT2}6~(q6aK;`>Rd8tzs2Q|rrir1c z&#iEuiz?f3%@%dN`P^TSXA4J8BQ6fR<|`w$sW}fx=2yvnZqWxs%Lm(VEW`dE5WY?e zFEAc|{a#D<(~D7Kk`I6R*1wHoc4#|C$8l& z`Hyt(UkC5P_`Wgyjg`>ACX)@3-H^JrNbZ+4%xQ-uP8!CvhA!~ZwD*#|iDon8&DY<7 zk&pzTXWbIpe__z8nDR<}qQ&St)+{qBD}S32A<8+XFbm z(1HZz9nO*rU%Vcybh8b;a^;l59}Ub}M{#l=oS2MV*Ck!iTSNW~f308N`&DD1PN!mk z>XI|UEKsj)UE=E^9ImC#ufvIV0{GK0=reEUMTS1$Q3ybWxTEq}JilyKsLs>;Ax`r+ zD>ivS815IO$hN1EH~kl&JP-LdaRdyxZ$Cl(2eWA)$^r>IJl71PHyddzWszhx@?5HM(LGtk z$1=%Mn541FWq)exng{-TUi0URS~QzI#fw>*B~!JPYT!q+Q7~42R);e6f@z|DI&6%X z<@$RSL&}m?nRTuBx1(Rz%YKPvicvT8ETP(Qi??~`Y&_%i9*+`>uz!LQfR2$+=uiUC zF%oitp4uhQVYoviXC>hsCY&rF=Oy7?rt@UwxUeaa%zybIOHx zXKLB6mt|G8amM5PIgMg}GI-GHR=rvC7eX`kAhC*}>_OtKy+NYk)NM_-cn}>PLEJ1XPN+Vj1r4n3@0c75*Z045hXw(N^G)% zpSmR^Vz@&t+Ov}Ij!QT>rv+W60h))>m>v^DX66zlF1ZCc3{cxv>yrFSvLj65;c}Zu|y4G$!GveW~U#pL=9po%CN)=5X+JZ zI)8#bOE`y;F4qWPNuzp96ztY88rPs$ivIwXRg7_;4V+cS*6FHi=*F()l7HBAB=Gr- zsZlObW7(8T)F79P2Dn5Ga)}z`QY+&UD?lzwD(I{azH&)Ry4*;BOB&T&rl7Ak0B!8I4j5;L}H5Pq##%Pd8Mgv5m28l$C?NTE3 zG9sBq=o5{y3Oeh9uSC+4F4Kr(Q6gznkBfo?7B#L(iPWnla&e!ngNqfC1jqCtESAcA zjTCNO`vD-U7~Ni}tG=1K)-6|ecBQWTWBVh4kHSrZ0?9m3cPWs}^M3%6(EyOBL4P1o z<1{IdMj4PyBLHMc1)cT5S0HIgm+K|n-hrqd7X?6~27gf+?+D0$Vvnbi0q(`5vB9LK z*PHH+=Z<2Bw?v>{z%(h6i~*%dkz{-afFz>2kdUBxzKSiNdUvG4nvyYW~}itbbyFd*!X>R^Do8mc8*ia2v`U794@@4wHIt7>!L$ zIE)6N+{1>$Xb?(9L!Dou%!$KQ08o~BaRiugprp$*;@A{Q@Z&I1)Zlr^Xk3dzS*0b1 zg>pN29ZEURuE#hjws@Q$`Z@VdEjl-1)1zb6HP^IuWv`na9pS!Gi4G5wntyZ{jZIxT zj0UOP!>7Y&kV-}aRF+wFxC(&EGP91LUYc_#=`xMr#w&PoxG2b88CMQuuNB*J-lFpF z*?E1ZhMm6HbnN6AtZD4?0tvQF1F4G8d2^JkD3V%~r#-sx&Y{f2}{|}e%)W35rwhcV_^AK%^l}@-jz(jM1 zNi95##-<(~MuTAP;o@O52qvQeFw2ZQTm=ATnUzO?mj_C^Tq6J`*m;;J1d}oJ0GJg! zdfo!_FFz7_rJ zm6>|DMkteI6?8N(^*~9N>m_|r2%>si6is3o4J5Hz#ipLOm`q1Gj^c-YQ&uvzeR5ZA z)6A2t|E_M*?bpV}8?>Q}$MQr@r-|Z6*>o)<*QLwXiu>Z=Zuti8t2}Ejnhk^8H0({6 z?+d5FY^)@|FW*GAihnne^UC~De=(Ix`aG8{RX)wLGn5ZjRfbQaC{sDg2HLD=X@0zXtGW1`M)Ts6sVetTnnvNvw~6$IG7`?e>8dh{qV(lUJ|*k7 z!`puVv+)p^0R_5)Qn3`1juIh%O>f&c5WVlOAhehEpif(pWjHPZ+X;;3!?2SgJsFHN zafpd^c;jTB8K_203v9)*OgiC(A> zqB}1N9)b`YouB`S7{d@!D5%7XDO$YZvHV606uS#9@g=SWM6C%{lx0Cfheau^IC=xr}X(ojy zp5}^9!_(HL;VCLD2IK*%80C8UVfm5V0B6P}iH!{RMe;aAY*(;;?JJGE!B|xkxg=IW5FiCIg(fj>qIb(!GBj zybI&|#`HH^@YI*IMGA5LO+ycm*xV7LLAM)pH$&oLZ#a4@8LhUa(%6PsRmcuGkX5OQ>Qwv)%W{BkC2o3bZ>U5DYzF5RLUGHa0lqD z{fuiK;Td=9k1_==itcvTz-rrdV(%?;!fAtg(z}QSgbVPDCAbKa4mIy6;8Ynz@Pe{&%7NMIrIUKLI84zds04&=a*)Q>O9Tw;xv!5qR9(#82dnq z;%gXr)qe)crYVs0kdf!$^r@WpKhA-2mk Date: Fri, 14 Feb 2025 03:04:21 +0000 Subject: [PATCH 069/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9463 -> 9462 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 392956fe7..c02e35d3b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-13 23:48:31,354 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-13 23:48:31,354 submission_checker1.py:3270 INFO] --- -[2025-02-13 23:48:31,354 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 03:04:16,800 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 03:04:16,800 submission_checker1.py:3270 INFO] --- +[2025-02-14 03:04:16,800 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-13 23:48:31,354 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 03:04:16,800 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-13 23:48:31,354 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 03:04:16,800 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-13 23:48:31,354 submission_checker1.py:3286 INFO] --- -[2025-02-13 23:48:31,354 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-13 23:48:31,354 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-13 23:48:31,354 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-13 23:48:31,355 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-13 23:48:31,355 submission_checker1.py:3307 INFO] --- -[2025-02-13 23:48:31,355 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-13 23:48:32,347 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 03:04:16,800 submission_checker1.py:3286 INFO] --- +[2025-02-14 03:04:16,800 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 03:04:16,800 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 03:04:16,800 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 03:04:16,800 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 03:04:16,800 submission_checker1.py:3307 INFO] --- +[2025-02-14 03:04:16,800 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 03:04:17,806 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.387262 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.416623 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1221 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1e308d8d6..701ea1ad2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -696 +697 diff --git a/summary.csv b/summary.csv index 28e60dda7..8fdf6f8a5 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9355,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.387262,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.416623,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1221,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 61eaa82df..34cb4c66d 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.387262, + "Result":0.416623, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index ec1441173f9528b70a9ac349f819e33227544927..32d4175a14a370388f3257fae1492e63e0ce8da6 100644 GIT binary patch delta 1437 zcmV;O1!DU5N%l#w%>xRzbm&>r0{{TG4U^#q8I$k>2Y+32+By(@@2}8!@{r6Fe*q?C zz@3I9%?Bied)rQ*7-7p;Q`>SSnZUolk`1{<-em6Fmpp{n(mA`UW36^I`n=6Ev>{wj zR*Wsrb}dATh{d#6kF9TiUw6(eBqT0koUww8tsN28=kUYmneksDB}5`+pb%p#m9iW< zj)+o{V}D_@k`!ttVLZoDz4EmqN=|TGlk&{*T{m!YObe^AFy#Nm21^nekxLd;IVogg zgp&+Q6~0JmDXef5Q#Gg8gE&dX))#L$`7yAZa8$egPRXFk|b6FSV*sI z#n>-Bd8>w8O?6t_>pH4AM=?oomB|Hrz9Dp-N`GafuPG7BME#3$s-*QTl;iE#>M9R3 zmZ>(3QX0pksNF`DkSzbvOnG|)4b=~#=LUXXf1X`N_8h12T1R+^B@RcN zJtJOw5GB?h${Tv;N~K7TKIvf{o3kHS90rhDhZb`ac1O-e8&-eH){8+OHt4tb?hC*1 z(0@ksU=Ll`L$}5E3=eHYANJ6LJ@i_9-|)~z1h9ub=;2Gh#UC7dxM(BJj}enL;?ptW zvW@VJk8uS+3^lI-h#|*U0Mb2%+yIc?G2|A2^p7ET03%I2ENv|8B_%bh$^zqu-j>Zv3=S<)gQTorb*ltjp-s8%@@x<-oSrf<8Byh*)W%3;Mev zrzg=#Fg%HZvyaF>Lt~_ZJ3v*4wU@fr2EqzQC0>&m=4)CAl#xX3EZ0^l^X76;KTB5D zPpY)9m{fiL;Vo4+B*OKZYGM<{=lyb7xSw*sCV_1?Q#F70NS$642ZTskaq+ zUoX=Kl=#53V2#mGp$OZq-AcO)1w{+-P)06TdTLH+EP9sGD@RO7yJOO%>(IY_(!4?1j zcx*3oba`xLbCX~s6q7R~8UiyJlTaBIlVl_i98hwlWhDUs08jz|01*HH0000000000 r0DyrMACoa36O&{l5R-Qx5DNeR00000004l2Xp_z)BL*5G00000G6a~= delta 1419 zcmV;61$6rMN%u*x%>xPy+}&f;0{{TG4U^yp8I$k>2Y=gg;y4g}->*=)@{m*wz5#}z zz*7^FOfHZR_A<3kl(1#28r$+nGJ(He$%gD8cT!vXl7|pmI;XpJtX8+iUv_zhwuB4H ziizdfu7yYuv6vQ{iS_3nH=Ut{gv3RRGggp^wI{;*68<)RVf=?k36V${D8$4{r7TB| zBchb#Sbx~8B!!wu7|*d(uYBW(k`o-)q&#zc*A1K;)52;jjQD@C!IFeVa|6GxKhHiR2aeNtts}g`5{F~X zUJ$Q6h!X1$<&C_dQYq4-&w5zL<{So=#{s0)p~c*W-LbRPhSi_4^=i($G359LK)R=pTL98Kh1>y<{wd@hfCQ(I2LNJdI|CqwxN`tvsQcK4 z_=dVq0K`yt0YD6Op8?1lIT^saEod7sw12+@AcnA00Ai?i1wahduG^5Vq1p`qF;x3@ zgaFxp3($?tc=rzd{y29$5g*Kf8Q{R!r*o6@*yb2t;0ds5$h-JA>?FMXdvU#--90{> z*Hu%fvbmu$t=4wLawpH!Z>~zBdMXJ|R_3_fzOFh#Z5SybcG{bWjm^#C4dN42=x#BLb8oq)anlYk$wy!R8Z$u8%Y9#$#IeT;V@_Md0E7?&;|Wj zk@K_YEEt_d!Nn)!U!Vz6!5yJ0#M(>UYXf10;}UPk9PnBy(*G#Iu|M-@w8xrCAO*OFzWAgZ-d;8ZrSNIRJ$Ol>%1SE|Qdy^&~D}O)Q5`^tQ zTe51P(3FT-1NoVhcoTpnZ_{epEus|saFiXu=(X;FTh0sq#R+ULVNEG!Pv1qNa~W{P(j>j zE2$5qZLp#2n>K*+G=0PpJb&fXQ+xu(W+I|9lCueEer($VIK$9_1mzvhk_=zG9;|e; z4ZU*Zl)@hk%vwiravz+Sj9u3yUC~=Z{tSPuU*G#xW1&u`Vu0$BGr}xTuWeo8>mnSk zrOvOziFX3{(=q5XZ|6mZKHyOZK!&)Z@>x8;Y*whw)BGV$^EfLuc^N_O7o^Cxr;#`P z7oa>3`8RO{47qPVLH!4_Xdr|F1q|HXW7L!JB3l8;lT9N<0r8WQBRm0nlh7kU4Z;=x z0C;RKb98xZWpk6EBNUS|BpL!V8Iw;L6_ceS5E~?o4|^p6002+|000pH0000000000 Z004l26_bu6J^^Wy%_JiR8zKMz003X@pY#9# diff --git a/summary_results.json b/summary_results.json index f80ee538d..a6c9aa3b9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -32,7 +32,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.387262, + "Performance_Result": 0.416623, "Performance_Units": "Tokens/s" }, { From 4ef5b1eee6e55b22503739682b6a3b85b2436640 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 03:12:23 +0000 Subject: [PATCH 070/719] Updated results summary [skip ci] --- README.md | 64 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 4 +-- summary.json | 18 ++++++------ summary.xlsx | Bin 9462 -> 9445 bytes summary_results.json | 18 ++++++------ 6 files changed, 53 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index c02e35d3b..1cb0cdd78 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,40 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 03:04:16,800 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 03:04:16,800 submission_checker1.py:3270 INFO] --- -[2025-02-14 03:04:16,800 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 03:12:18,627 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 03:12:18,627 submission_checker1.py:3270 INFO] --- +[2025-02-14 03:12:18,627 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 03:04:16,800 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 03:12:18,627 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-14 03:04:16,800 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 03:12:18,627 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 03:04:16,800 submission_checker1.py:3286 INFO] --- -[2025-02-14 03:04:16,800 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 03:04:16,800 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 03:04:16,800 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 03:04:16,800 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 03:04:16,800 submission_checker1.py:3307 INFO] --- -[2025-02-14 03:04:16,800 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 03:04:17,806 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 03:12:18,627 submission_checker1.py:3286 INFO] --- +[2025-02-14 03:12:18,627 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 03:12:18,627 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 03:12:18,627 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 03:12:18,627 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 03:12:18,627 submission_checker1.py:3307 INFO] --- +[2025-02-14 03:12:18,627 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 03:12:19,551 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69388 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.416623 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1221 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7236 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69388 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.416623 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1221 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 701ea1ad2..679fa6b42 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -697 +698 diff --git a/summary.csv b/summary.csv index 8fdf6f8a5..ff032d940 100644 --- a/summary.csv +++ b/summary.csv @@ -13,6 +13,6 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434065,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7236,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 34cb4c66d..a26f76879 100644 --- a/summary.json +++ b/summary.json @@ -429,17 +429,17 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":48.7236, + "Result":48.9647, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, + "accelerators_per_node":2, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -462,14 +462,14 @@ "Result":0.352299, "Accuracy":"CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, + "accelerators_per_node":2, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 32d4175a14a370388f3257fae1492e63e0ce8da6..172fc621b85c9d7371e73ff7f6f9fc548bae10d7 100644 GIT binary patch delta 3643 zcmYLMXHXN2vJC-5Y9O=(2p~xi5d=eT3W(H*A~kdj9i&M|2uylWdCUznH!)vE6JYzMjn{e*wF^0``$rlDrn7S0$h2oK^1UN?>!+&%*(scza7i zNJ3wNKZ}-oD$QsSW92-qo=ZY3wY;2*DKB{Isz!cSO7=aR9Py{Ep%$XplHhwciz?3K zDIZhBp@}pvl8{ddL@?=wd>}jL2d6d#a}MGJYG{tH`I2%SPIYNoD0jSaBR$5Ysz^fH2ufPn4~Q zts*imTjUWl>=mjn8WPzb)WrC1zjZVvr)9qST58@eX;itPbg_f9JQt_qOBQm!X6Zkl z#zI3T>BeOzhm6P8bx6%nU#0+&Fb(M-(9_H6A4!I7M&?sV+@>Bn^xqrlxSVj(nUg2F zPVb}aYe2yU5GysWQB5^HuO7|%yfqR&>9iS(=u>|TsTumy?A8OyE-D~3s!sL!Q zsc>?8415St!3K`p;AgZ#eIN|B2JRl0T<+p<##U!bb<3yyiqP08S4SQMaXokmDDJ~VG;Rjk6H z#W&E4CSOqv4@N0*H6^EUcA^>M*?H@VTExP3;f&zh_|Nt+7Tm?~m^j==^d$ z{t9GGl1){ek`2X`UhmT~;lYDq2SnSkdrQ;E2&rnNxI|ufSSTC3ip!HlaeekIKi{Jq znDt5OlJ#fysl;&Nc!#w6nnB6ii824GKVGW$aiAex=$_-dGJSCRk3{sa7&e`0ep39E zYFpnkioazZgH8pYMx-#+dKv6I*x!vc~j6j_12Dey+qS z=avR~onM$Kd{@6rk$R0AQP!3tPRCA^#nlXyuT$wt+S~@LgI1{s0exNzq+UpWeM(qMiQT*_^ zH`KfekB}7v)jcm9A!&Y@Y2X9Y#!l=ezu8(yb6N88ahL@U-fg%GK0z!>3MGyF^iBZ| zD*mG3hcW_UxTwHVxl z)l4y&!9z?iK~zwxec9Vj(U(o30tws&7HQ#aFRnj`MPL;G2q+_e43`?gvlT$FhZ4~6 zP+KkP%+YRRgp*dvHj@93d^DSVWqb-vt937dBHs<__tr+`66bD}Dy5FYiSHnokZ(7a$4%l1`4 zaSXJi+o2ydmN3`2Y2q^~<>$8VYUUkzu8rhUL~l*lNJ>k66Xz=M0;MzMs|Z6GA7M78 zzX$__huh@CO`XPa3x*}b&#SBlzvS@;*`ei<+KqOG0^)``9ajI%JoCYwJz1zG+Eo|T z^i&CFz|8YL^%Rg`>RA2pKWw*-J$A*p)(8`T6uUlH74UQINDI8AZi7erk7ol&S?^RK zv<>tXQ$aO*y7#ZU<{kRyV+}%Qehf?1QZ6*9-~_rP|U^}9gFpP7~$6J)tydaIS7h#!w(iMc>DTq)eDXn^qu&W6h}?WO;7bW zhHB~FOS6^iLga^C@>Y&a^+>6viInH$hZWOkx0s6!?sikF{o5==euv`He9{oBq~v}( zEgr?w3tG}QLNx|^89!YZCRY8fa+}8$VxMo)N+_b+Gw7_GQ9S;g* z{?R*nV4SQ65frnky)oZbx5J!4LHqE>uVEk+0FX??gZXf)(#*ko#>`v8-y^DKdQm)L zzy;cPAD$kq*u%k`LVUiFfFb@s5NCBps4#}hr3=y@4(>wVNs7&6i{!=+#-?~8GUe0puMx0Y0G{fm3f4x}5r!!T}yhfB~k&-6n5UsBUO47I4 z2d35{+HfPK2Dssd8?j(V5nh08R$8LnslBd#5kA@&o#mw4KcU$>i4%r=$G@Mw5{zd$ z^YZA$4xAP79FB@s)3^&**P{bqo!C+~9}kMWAzRoeima~zsz~O2>9guXu)Pr>|9trO z6+!enaj2BJTEIO)_yKKvgNlLdKmdF3!L#6XT#`fE1>M7eUgzZE$Ep~a40lKJgdw>Q z>>daPAIQt=UsLnTN>Kz}la=p0f!$VTv3>M7hEu=(*VNq6%Cn%fvY>7)Cw)z!>lWrH?$O} zn!mb0ajj|As=qFM{dt?oa)#At@#?M{{nUie7QRcuocBeCEVJBo%ae@w(73z(s=0l#Z6GIi0_I#i72Yb^JR1(j@a*SM?=^lN=vQ*X<#+M! zld`Pyvpl-YE*k!m_)g~6xyye}z^bp9dJU4E-p(0?!%mF>b;?F4StZ?`3?ZK_avi@Z zsfEqD>dm%}U(yCNdaOyVtTelH+Y3k zu~99bWjq&gaZTYjt}$^w_e+}u;UwTPSv5drL1ASssR5G1OfA~0D`=6?OzK~|I~n{O z_WbH3#d-quHf0FxQ8J|9Qo?j0c`m^25j!AB!_#xz6j z?6Js5wlODgjI%oQ-{yH8Btxfdy4nhf>vaysf-BPp@Df| z@O^_tk)Q?BcXw+UotAm8J53pF#jFO}cV=3=$=EX_UWEUg+@!;cXwW*kMNTi!1y^u< zqtt$GZCc&uSn`O5cXT_QU^bKxA>$y@%-?;Tr9f%DQMHv>q<8KoOlYHS^S$08q}U|7 zZ~^yZ_(Wy_cjDx;A&zn1+~0C8WsL<8K8Sw&KTjf zVSw`Hs4{)EvbG!~2VH0!{Wx!(a7t?_X7wq`P`~tC-&5J=hU_|%iFnnm9|0_&S#dtn zk9XX(E2VK~$xS^1SiWe)l^pD1{()Rj2nH$o$YQP6g3Nj2jm5fzHk~$a=NX>!@_LRu z!>P=VtB_|_A>n7BI!FE?;QxJXGH^4B10oSF!2O@?$_xOo0sb4IVb6*0;VO*P8~}j7 ztJpK|09U(b#9TNV5ryCa3P6buq3py$I5YQO6Oa@Dz#||4!17-_AT*4Un25MTVN4uF Lz^~8>{B8XYF(bPH delta 3647 zcmV-F4#4r{N%l#w)&~VVJlHMJ4Nc@Qmy`&z^~+R&O!f2$FC z!AxD&a1ifPM>HA*zWf~di%`zeWhJqkdTG}EWD?91wS71YvSayplvs{h>XA4&`BiJG zjrL*D_FW(!la0?xK9u_?`Yr!*Hf*Vdv;o?$MK`WuWhh79Vw(1&$8RziPtt7K%sj-8 zrb+Qv34?4dHB||{r11(V_oJ{8PP4yZmg!`l<}w})g4syM zS-vI*sZ7#AvbYa}Bu{;k!Gl?EHquziBFSpxxm4q#d$Nj;Ws;>ZNn@4E)Ydf*{Q11* z&lk05HhYQ}vouSlYAe;ik7lD_tp2PHW$FdfME!Kw7%|I#_4g`ss+|N58I@ z{SwO*qi*O~Lbc;3-sYjR@r={oc$8R#{S%Y`bc}>ThZ2B}k&p}Y)GmPz!yO_yD+%u~ z;bZ|hFA485ohK{Dg-wZM&JStw5+yGE`L-)0ay%J$ettVs%kEy5Rn^8BkMrj=iv7vp zL91K!X3bv}2+i1o#43ie2Z^`#28o7Kw>913L6fcv9)A(T0mPE<4iip6ED7&2ou?o= z6h!9ykmd9eB`*1)ymJ0MAXYJyJwUv3ww6TQ)HK)SNpyJaB475vmKArZqJa?zfZgm+xR$vG|PG7Zo?l*aU!7&0@LC~?V8z?EAqY@z2z zNp0#CHGe!g*593zRx!Li0PMRP06KNoGEJN3Pc(S|i*m9{LmD||G#2>{KVdY$Ag7@a zG8&+e(b%M|)3t`RPid{ydbGjBy_-R~-|pjikD1>&^}=`JMJ7fqy&erl44&#exD6&2Afa zVaY#hIuiK!#?&a5sIhE{C29~$Mgv%)2C+m9VyTs3i4`E0B^7km2Vb$IC0%YLfF+IU zF&0sXC29~$?OkI@?i&9*kyQ+DFCw)qlU)rjf2ZC#eKF6E>=hq9MgxeSSj-*Qn+>P2Y{?%bbDp4`ex=@w_M%XwM6ny?T-XL2{#Q2 zB=bDor9d)|{{cuw13;n%fkchdq(B;FKr)R0kR=s#)(2mKq$ORhmvnarqIz5u0ErsB zf1xzq5s?4H7EdJu+>1$LgGo)VH{BhN9mNi7i9ox6X;LH^|4EY~$=D76Nk#)mq6U#< z+%ABmS%xIj2q0NfL1%sN6-ips<$4K7(x@I2g;^`(<$pN2 z9ZEURuE#hjc6gj0x}AKd4xJmZY0MX$1FO!IHy8LH5d+av*!H*p>4Zm4DC1>pS)9 zfAqzsVJFXEO=E{$RX97iM8AhgEjx_Hrk))}gIw<6+F>-vC8Gf@%Zxi*1;AyQbw^NV z%{i2GnMROHaOf~mn8Gq19Y|p-HtGC-xO}Jfoolh}-!XK<)^<4Qgu4Sw^oE$!!NX{5 zYT;ot2<9Fp9!7&;G8zE0%*Vr30AQ9m3V8%rd7z}rH3DFhtPdG~?lzN6x9RNg)M-tg zNsaOAP)`q|v8ktr(EyX2hBC=$fJsIJ`#oBjr-y5VGFetZM*~j}lytdX()WZQs>emq zB$m-Y60240>3NIEbd=*Le(0XElCkZRyK0+Wo^1Vhb(3ztHa1?M4P`u*CvrMX6hF$Q zYZ-e6XrAd>SoZL00B*5XC|Iylg#x2YNB}+F>;OSafPJ3w~e7k01OG*1-qg z`t-A+eNtLVHuM*1HoBsf`~%mm_Tk(c%L^|a2eU+(%275d&}KbL^W)`9&Bf<5niro; zRk@GSGzwq7O{6!JasEwLl~EL>FJJO0S--RH5S#%HKPhpp6$AhP$PfSk6abUJBNl(n zSKDseI1qi`uOPH9?Srn?#WEZhf$aoF6UVTVB7HI#Y2;{EBDErw#{K%GFG*dE_oE-OC!J{9;a+|qTO7d>0ONoNm)!Y&fr8_gU~(= z{^$!KiQ3AsqO3|9AqPhwmG|hJp*o|Ed4{E1mSwl#P92Y_q)JvQ z;^s!f=H49iS0C~cNV|Aq5UzIHk- zxZ8qzEjW_J9uHdZZVP{oeB9wS3hW+l=?OOK346T7U%SO$yTxC-#a|ow3-Jo-jbUHr zdY(l)WEmC3wjXfD9!phlX%46vv}vY^p{dWUaG#4R+i}enb-nrAUyx@DM@}Oy4!hbnjmW@51=LG5w8|(7+~>4UyfDy0%E} zmo?03hb2xL#xGIT&f_DO7>{csZV(^-$W zq2YyS8hIL0JC5Oa-Gd?3H+r@&h-8kb{2#v#=In2n0}arDc;9A0L0qYQr!PMfd#*qrENJNgoK?fwpATK%pru z-3e)uh*}RMwT}P3vYj{yWYy*9-Z^t;P`TKvCiw;DrO_42GlCM(+z6?+6?$JU(+8CJ zz_ehE(NLidJX+K@CAX9t2d~apa6y7kB1n76ZH0CrScIiRywBw<#&oe7+_#O6D}b|LR9x^q+w0Fy!CF zBVf#Z`3dSjvsWO>0tGxg-evNWCL>z`@RNcgA{##`ajq2v0077k000yK0000000000 z004l2dXv8+K>@*&0VFyBGm|_dCL2(4rDY`n002+|000pH0000000000004l26q9`< RJ^^Txyd)zA8X^Dy008s1&p!YF diff --git a/summary_results.json b/summary_results.json index a6c9aa3b9..280bfa2c3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -120,14 +120,14 @@ "Scenario": "Offline", "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "Nodes": 1, - "Processor": "AMD Ryzen 9 7950X 16-Core Processor", + "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, - "host_processor_core_count": 16, + "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 1, + "a#": 2, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", "ColorKey": "availableMLCommons", - "Performance_Result": 48.7236, + "Performance_Result": 48.9647, "Performance_Units": "Tokens/s" }, { @@ -156,14 +156,14 @@ "Scenario": "Offline", "Accuracy": "CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584", "Nodes": 1, - "Processor": "AMD Ryzen 9 7950X 16-Core Processor", + "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, - "host_processor_core_count": 16, + "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 1, + "a#": 2, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-51-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, From b41a1b04d2ef4e3a2d4f3c21cef526db47f36772 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 09:58:00 +0000 Subject: [PATCH 071/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9445 -> 9447 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1cb0cdd78..93d31492c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 03:12:18,627 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 03:12:18,627 submission_checker1.py:3270 INFO] --- -[2025-02-14 03:12:18,627 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 09:57:55,324 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 09:57:55,324 submission_checker1.py:3270 INFO] --- +[2025-02-14 09:57:55,324 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 03:12:18,627 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 09:57:55,324 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-14 03:12:18,627 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 09:57:55,324 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 03:12:18,627 submission_checker1.py:3286 INFO] --- -[2025-02-14 03:12:18,627 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 03:12:18,627 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 03:12:18,627 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 03:12:18,627 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 03:12:18,627 submission_checker1.py:3307 INFO] --- -[2025-02-14 03:12:18,627 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 03:12:19,551 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 09:57:55,324 submission_checker1.py:3286 INFO] --- +[2025-02-14 09:57:55,324 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 09:57:55,324 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 09:57:55,324 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 09:57:55,324 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 09:57:55,324 submission_checker1.py:3307 INFO] --- +[2025-02-14 09:57:55,324 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 09:57:56,272 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.416623 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1221 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 679fa6b42..764e10e25 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -698 +699 diff --git a/summary.csv b/summary.csv index ff032d940..b3a78076f 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9355,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.416623,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1221,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a26f76879..c2a94e04b 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.416623, + "Result":0.385187, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 172fc621b85c9d7371e73ff7f6f9fc548bae10d7..4ac2908bc59ae530cf7357155e4de5c09d7fa9d4 100644 GIT binary patch delta 1400 zcmV-;1&8|ON#{wh%>xP#7^#2M0{{TG4U^yp8I$k>2Y=gg;y4g}->*=)@{m*wz5#}z zz*7^FOfHZR_A<3kl(1#28r$+nGJ(He$%gD8cT!vXl7|pmI;XpJtX8+iUv_zhwuB4H ziizdfu7yYuv6vQ{iS_3nH=Ut{gv3RRGggp^wI{;*68<)RVf=?k36V${D8$4{r7TB| zBchb#Sbx~8B!!wu7|*d(uYBW(k`o-)q&#zc*A1K;)52;jjQD@C!IFeVa|6GxKhHiR2aeNtts}g`5{F~X zUJ$Q6h!X1$<&C_dQYq4-&w5zL<{So=#{s0)p~c*W-LbRPhSi_4^=i($G359LK)R=pTL98Kh1>y<{wd@hfCQ(I2LNJdI|CqwxN`tvsQcK4 z_=dVq0K`yt0YD6Op8?1lIT^saEod7sw12+@AcnA00Ai?i1wahduG^5Vq1p`qF;x3@ zgaFxp3($?tc=rzd{y29$5g*Kf8Q{R!r*o6@*yb2t;0ds5$h-JA>?FMXdvU#--90{> z*Hu%fvbmu$t=4wLawpH!Z>~zBdMXJ|R_3_fzOFh#Z5SybcG{bWjm^s0$lk*6041J#fh#}r$ z@UdA}t*b&-9o6D$6T4yXb^`YPPXd0#^4?FtCcB(pdRVoXnCim^XHDBJ)%664pG|)s zsQ%t}b*e_mvdRdO`%?A8WT);MbvPD^cS+OAqmom`soXcYt*;hUhQpl2=UEZcLKpOB zMb6KnvtV=<1s9)?e}N`Q1$Tt15Nj`WuMLD1j!V2DbIdoi5GW&w+F7oxROZd)pnjID zte;eAUo)xt{^MJ!Zb*ddH`T-@jLGAR?(JXiT;V^n$OlOm1?^mpIVO`9A0B_pPQx$| zMfZF~l``B_xW9*je_pwTK_G4Qc)!Cu!4G>Wby~-Z^t;X}R{iztB zy5x*73)E{{m-xI0hij6k+BbW6atVj?onCKo?jYP3smQ6Uc_l0 zXSb{Lkrel&D8BlU7yT!oJhMw6kOBn{7^#2Mli(s-0m+j*BSiu8lY%2W0ezFdBS8VN zlK~_;0V9(xRzbm&>r0{{TG4U^#q8I$k>2Y+32+By(@@2}8!@{r6Fe*q?C zz@3I9%?Bied)rQ*7-7p;Q`>SSnZUolk`1{<-em6Fmpp{n(mA`UW36^I`n=6Ev>{wj zR*Wsrb}dATh{d#6kF9TiUw6(eBqT0koUww8tsN28=kUYmneksDB}5`+pb%p#m9iW< zj)+o{V}D_@k`!ttVLZoDz4EmqN=|TGlk&{*T{m!YObe^AFy#Nm21^nekxLd;IVogg zgp&+Q6~0JmDXef5Q#Gg8gE&dX))#L$`7yAZa8$egPRXFk|b6FSV*sI z#n>-Bd8>w8O?6t_>pH4AM=?oomB|Hrz9Dp-N`GafuPG7BME#3$s-*QTl;iE#>M9R3 zmZ>(3QX0pksNF`DkSzbvOnG|)4b=~#=LUXXf1X`N_8h12T1R+^B@RcN zJtJOw5GB?h${Tv;N~K7TKIvf{o3kHS90rhDhZb`ac1O-e8&-eH){8+OHt4tb?hC*1 z(0@ksU=Ll`L$}5E3=eHYANJ6LJ@i_9-|)~z1h9ub=;2Gh#UC7dxM(BJj}enL;?ptW zvW@VJk8uS+3^lI-h#|*U0Mb2%+yIc?G2|A2^p7ET03%I2ENv|8B_%bh$^zqu-j>Zv3=S<)gQTorb*ltjp-s8%@@x<-oSrf<8Byh*)W%3;Mev zrzg=#Fg%HZvyaF>Lt~_ZJ3v*4wU@fr2EqzQC0>&m=4)CAl#xX3EZ0^l^X76;KTB5D zPpY)9m{fiL;Vo4+B*OKZYGM<{t51d*V0+1XhwtfO?`55=B(pEPLBp#6PkPY2F?uY13BfiQ{@^znxvVobOeW{DSk+ z=n9c6M+s1&{5nNr&x%k+kaxq_&3`EF7o^yD+o-mz;F*t?Lpt?aX@t` zIOPslFs?4~c@YoSFdvrj#2X3Xsf+rRx5FZ1ABZFdpkv&l@;Q5cSs5)*UF5}Gnina# zUF9=M3Q8Wo`jHp?C!jjBO(2j01-EqQS<{o^B3l8-lRhIw0rHcCBRm0olfffF4X_pf z0C;RKb98xZWpk5IBovb)BpLxClRhLS8~DoscO?M;08jz|01*HH00000000000Dyr5 UlYb;W0aug0BqIh1A^-pY0CuO3djJ3c diff --git a/summary_results.json b/summary_results.json index 280bfa2c3..8ed0cd037 100644 --- a/summary_results.json +++ b/summary_results.json @@ -32,7 +32,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.416623, + "Performance_Result": 0.385187, "Performance_Units": "Tokens/s" }, { From 3e036cb7f92ca98f4978f18d9f67a256cb7eb6bc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 10:04:05 +0000 Subject: [PATCH 072/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9447 -> 9451 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 93d31492c..3ad9d4e93 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 09:57:55,324 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 09:57:55,324 submission_checker1.py:3270 INFO] --- -[2025-02-14 09:57:55,324 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 10:03:59,805 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 10:03:59,805 submission_checker1.py:3270 INFO] --- +[2025-02-14 10:03:59,805 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 09:57:55,324 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 10:03:59,805 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-14 09:57:55,324 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 10:03:59,805 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 09:57:55,324 submission_checker1.py:3286 INFO] --- -[2025-02-14 09:57:55,324 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 09:57:55,324 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 09:57:55,324 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 09:57:55,324 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 09:57:55,324 submission_checker1.py:3307 INFO] --- -[2025-02-14 09:57:55,324 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 09:57:56,272 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 10:03:59,805 submission_checker1.py:3286 INFO] --- +[2025-02-14 10:03:59,805 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 10:03:59,805 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 10:03:59,805 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 10:03:59,805 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 10:03:59,805 submission_checker1.py:3307 INFO] --- +[2025-02-14 10:03:59,806 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 10:04:00,839 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69388 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72556 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1221 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96557 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.44039 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 764e10e25..a4890ce57 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -699 +700 diff --git a/summary.csv b/summary.csv index b3a78076f..f4794f900 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69388,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72556,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.095,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1221,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96557,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.44039,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434065,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c2a94e04b..fa5016714 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.69388, + "Result":1.72556, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.96557, + "Result":9.44039, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 4ac2908bc59ae530cf7357155e4de5c09d7fa9d4..635dd0564971801cb701521f621fb2b5c51bc21a 100644 GIT binary patch delta 2716 zcmV;N3S;%>N$W|l*9Qp;%0!O&2mkYA#^Sr84v*{G@B-h8dOs*6_0#NYJZdEKSo(t^XgX==nr1hq zemGN?H5`A$`_vH)hhZQ;N5Nt$XX&z%SWf&j>wYp0=ZV@n9E91id^}1lhfVcJ9Gv{B zYigr)ShRf?%Ex5mvyuH&C z38G0-{8gr5HkZ1pO#P>(YN*O%I7r8N#mYDw3}k;k(W(+Gk~EtBx;!LS6)Z!Ay^0l@ zSVwzxVQRK+<>%vhB3ChuWvfLGKl`cQI*g-7B`#V>=6-%=#DT77(-P#R-}B?KAPrwK)so4(dhPeH_3eLGUwWOh?)k?e8A`HjXcT{@ zOwfGM((xT*kwGI2+11%hzN-l}Xx97WdOI$y1-C|6tX%MjA_5Bw39- zmug&ePFC@;OtKUvX{>UY+Pda`FrU|g`JxuhW>4{AmS)LRZKdi5(QFuw)SuOXO#N_@ zsGklSL*{nXbmogcBytO2XS1?vj}}-wY?mI!x>2&T-*TBAN3;dVGlzk4)Q@ zQz4O~ao-Q}$C+C8^s=m~Hp+OMKc`U~jQbCI-L7l3U?FsK4-%^w${r-%Ival^nr_{* zH#ph?(d9vO@kA4Zj!|NhYvBYX03suyAfg07WF!<^Fp>tYRp8fOzL_-4YE;*FBFX(c?+<@Jtgxj!|Nhi{S(%Kq4cd zB%%aJM2SQ0;HO>*i5Tt@$yt9%c-teKoYR61(*VswX-t=iA(MBB5|3bhIdJSprn9c)#M@Hk2woca?j!32gK(efY_WIx&N+{`YjgX(P3yA7+Q4~x@17v@0bpL&z`Fckr zSoR-8UDxX3&*NFi8290G)iJSJQmR{y;qJhaUvobaxW;Y?iY01nn_`I?#FEhfmdu(z zV2K*UQj}qd6(E)+6|@E2RXB%|4%Y}^Nu#<<6s+4Y8qc6uivIwXRg7_;rJPmA)`_cb z*t@Xgcb$#|Zsb@x#S(utwnMQ*4Pwb?087*$mZ(82^)f860>rYUg7*60E0(mR!;OSK zF+xH zj0T8A4HAhO$D>3VWkfQK(5;QK3fk*~uSC+44%3JWMAE1(7X^R$D{4HG5@}RRq5_Z;`^Q>puWw6{FiLb2Tz#`gk9T4hKw zjR2A*6|~m}Uy-CG9j=#vB#r7aQJA$dt{%u*t$#a`RSa;iyw%#uTTR_|Hl72wq1@rc z5oq(Us2PXRIMj{9Xb{Rh{5XsTp=31F0w&5VIa~z*Wto2|M}R8_N;*sft2&?dW@4|g~$D&$H{l<)wvOyW*xh(d)Ceco=vlk@Ls7z ztA|D1I*i7lb{$58RPJHeVKhi3qX8<*JUd(kKxLV0M^IDEIh1smMsVE~%sE^XWUq`p z2eQ|S{W*VcQTg{QyuMTCPETxFck&F@GItnPg}Z}GbbeUWyu)Z5>fT{A$mJgX9Y%v( zG8*8r%)-M}09=-tcm(y>oI^>6X#}|huMQK1DJtUi0Ovc#*U{-AHc?-?;uB$m-Y602A2>3NIEbeQ8P ze&~OhvXZgwle_AhUY_j!?->?der;^LN;{SDNS?^aBvFDWo33T#x^($|aZeoFEnmfb zm1pfmvq6}W3LVo-Ze6SBb1oo$&75$UaRI;JJNVCxu zt>hoLZnX~Q{zzW<@hF@n%0v#cfokROLQO(`fqgZ7ltPjPq}X zstltjefg44$@(?%)*nzy0|b*_BO0?25djVb&(nrdC6gB)9)Hbh!!Qtp?|llRyDiyC zOA)pMZON&DLQ`6D6Vk>JwSJJ)I)3{qPU0lsQ;(~e{dRUyx!9{V`32{t(G|)wf)dc& z2&tP2eXN)1BT9T=TCmn=s8A0cE$X|HTgr`tH)kxkAi*aQq&?-fLc0(w#n|&5DCV>1 z&{5nPr&x%kYk#n1{F^m^3ql^Sg1`g|3?G209f|0K6nsS5AJ+~5!7;R;KnIWWEXNmb z0H^$9!=PL_l?=UwN$Xi0-Fq)bW7lK#_BKapAH|x z=(l(xWul}N%;T0iNs`4uqY?WPIrHLrxRCSo$S90vUXp%|#*KIpNpF}3&87{l*|Zun zFPN*V8V-M=ed-8DqrjJ+Lw`Aw^JG;?B&S}Ic0Zm3i&$+R4ukYqHXbFCqn3Ik4o-g6 znrfqcn74fw$j5l=vyu(vJ`8`$zMKtPYA$Vn_G{jat4JBjk++;C{qXUd491froi;NI z@xy7H|5au|I+vQN%)FGv<0PE@x;iA*6|6#qwT=}{ zv59u;!f0BWnVpa4sa(f2R;}he{Ol!O`!EV0l_+l^UU=D=5eJ%k&si|QYvGP~ay ziOyeb7Js!_c)5~UJ4T5`dU}Eqjm+9-MnY!o3rch{YhO`flkT3rNSI&5Z%hmHN-GKP zyc~O3q0VVR7aMTM)qX)^ddw*x4PP?VlF7Sz?f0nt?SEHa`rWhM`NupNiqmXp93+2} z<$c``XN}o3{X0uDo%GW}Mx#M6AIT`q*W@6PaWaUP_p>0*QlEJ6VAh+BB$6^uvKm<~ z)wt-Mtm7jYrzuR_Sm!deP0a&;v8efrWi6b~pQ7bFNt3DCS~c*)`6w8xKdVETc)>JQ zKOHtk%yRv`iXo**t4zC=|J%{8>s5ch#45$88+w{h?f8kedFX6Bdl(J6q>OIiB$|`4-#+fEfRkXr|w!? z92G!xcn}>t(FCDmlvw0iI6(=3$Ve!NC;<=|2?fz90g(x(AeMx8FdRU1NmE%1Xw#50&eViSeBj~e)b13O@jR2N3s>ejZZVjVxb&93<4`5lv828!0S$Axcu4=lq3rqf4 z(~-c(H>O6hM2%%rEKz@hSTY*G5;ce=Y7k4U3`?v4u`H>evpM*RB`xW4Bcb<+5Y=O% z5KGh`mfE|NlUs+BMuNrqk3Ev3d63IWcKN9#P+%zbV%=2`Y0?9o72Ot>@ z0Erp|5;abf0%?>1$ut5$mQ>K$9DD_mmUOvZ(%l`1>Tyv3Bx>-6(s)Ne{u5g~l?-q% zCXFp7HND<+cRYV~6g#XX0__5(Ns(myCryeZV>m?vbqk2pfX042u2eMZ4-;QJz1KcZbH4AyGp;`9UeP9vF9S$6U<_?ou za2SnEJvfX8q1?lT!)OpnMnjEXqRfcHRRB*T4~Oqq{}pd`>tTg;i4dWWlTAcy;khX zd5g-wXXAhMo%(h9V%xBjWw55P!>%fv9bBT{!=#oSMq^XY4x>RX_i*hn8sw7E0GDOP z9j*f4vdp?8sI%rAO1exV$R#**m?%tP8IKO6uoath{y$v4Q~S=fDEfB{-LN%l2bk^- zFwq-gQU?#Cv8jcJ(IA+6n0Ocsg2`wA%rYMjR{?*3S?1&sVC8|5F4qX`zk-{Gi9#?L zKM#Oev7zTJF#qx+k$39oxfX7b$)?+Mc6jQvCeNhC_;skKhtb&7)5B9G8+Fm#x z2AOBro336LPJ;PZiGN?ch-~FAB4?HPqyBO#mE?ILTdI6oq-Q7_tf~y3Mypqll|>YU zQII^ZTF>5rUQWICEF6B!JGGn#zc1y-5B^pM>)?ZLefn9^J}E6F9r}wj9bMi^_JQkG z`*7in<%JiGgL$k>T;{vIc3(^l$=<@nw?b7$mozL!n(3)Ts(DwHKDia;^Vh1yo=W3`AMP~<#S zoHklPg*tF(Uf-0=5T-4>S#7|20WJz4|?OcvICIJ8dPyzq|5dZ)H0000000000fPn;)gCsryS(CyfBL)j100000{w(LQ diff --git a/summary_results.json b/summary_results.json index 8ed0cd037..ce1886494 100644 --- a/summary_results.json +++ b/summary_results.json @@ -212,7 +212,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.96557, + "Performance_Result": 9.44039, "Performance_Units": "Samples/s" }, { @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.69388, + "Performance_Result": 1.72556, "Performance_Units": "Samples/s" }, { From c812fdd80e8cd9e6845f3c9ee7e40bf7325a3846 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 10:08:32 +0000 Subject: [PATCH 073/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9451 -> 9451 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 3ad9d4e93..f0337d988 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 10:03:59,805 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 10:03:59,805 submission_checker1.py:3270 INFO] --- -[2025-02-14 10:03:59,805 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 10:08:27,306 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 10:08:27,306 submission_checker1.py:3270 INFO] --- +[2025-02-14 10:08:27,306 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 10:03:59,805 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 10:08:27,306 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-14 10:03:59,805 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 10:08:27,306 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 10:03:59,805 submission_checker1.py:3286 INFO] --- -[2025-02-14 10:03:59,805 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 10:03:59,805 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 10:03:59,805 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 10:03:59,805 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 10:03:59,805 submission_checker1.py:3307 INFO] --- -[2025-02-14 10:03:59,806 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 10:04:00,839 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 10:08:27,306 submission_checker1.py:3286 INFO] --- +[2025-02-14 10:08:27,306 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 10:08:27,306 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 10:08:27,306 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 10:08:27,306 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 10:08:27,306 submission_checker1.py:3307 INFO] --- +[2025-02-14 10:08:27,306 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 10:08:28,307 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,10 +28,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9355 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1385 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1221 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1486 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.44039 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a4890ce57..ca1898553 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -700 +701 diff --git a/summary.csv b/summary.csv index f4794f900..33b0f0523 100644 --- a/summary.csv +++ b/summary.csv @@ -6,10 +6,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9355,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1385,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1221,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1486,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.44039,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434065,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fa5016714..7494b5095 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9355, + "Result":21.1385, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,7 +309,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1221, + "Result":10.1486, "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 635dd0564971801cb701521f621fb2b5c51bc21a..9533a1ff6fba37607ca127c6ffdc34981c9ea511 100644 GIT binary patch delta 2652 zcmV-i3ZwPwN$W|l*9Qa|y=)Pa+Xpm%iiKKtxeO`WGvhJF*n=;r^W>661F8TblJMo< zM-mvrUK^QAk|S3#4<2Z(-8bn&Tir_szdTLG%7aYOD4sS|q3Nn3r(xWWro*OsbNzMC zQ!*XqIQcCdNtr2m1=F;tj})Vd%Qkm;}*OUDU9j>{3TO7(}7`9EbCXoMwwk5;+dCy!+`W znx$&%pdaPOit#9k95mHKv48xlp{tG7LD}|gBp=hY&q^_r`#Aos_;T8Bs-?66+OK6d zE)%6M2f=)t_2S2GG8&GueA?`PA|#B*Y57-~MEP7Asxk?lnyRTPk5NAx6%{L^sNa{x zM5{_TPqTRP>*A1DRa9v z&Vu61h<(GzrzI>(zZbmpXP+`%bzAd?aTXKUEhYYr|l8w>Y>}U^dsKDK9;s_Pq zrs5bC{-)wHDg;5{3nL+aSJ+oZLQc&IBOxe!V76hpJj zJfYh16YuiSxp>CuZ#+sI!u~Nz06IoOp+gBk$4JNpdg4|<$Al9mPAkIO819joI9m@V z$2v^w5IRDML#~BmlmLi~go20?0FjYU z5WNZznQ#hXMR*&-0Yt9?M5c8LqDMhw&JVd?U!cUNd6f6>0kMvu>;U4ew{c4}ZNu<= zo?-DFc}Syxz_#nf#&NSkzjfEAT&eQ#Gl8rmND+a=dxpBxum4ouIX*Tl3#N_ z6u8E23yLLb9EW0w8pM*(0G7;}KVXR(#8OmYi4`E06&17v-Bmb)k`C7hU`eC8Ocbo! zFdE-~q*#jo0G4%(ahIi>Wye;Dt6@6Zu;h204h3%H*apQCHI7TML=9reXaGypAeN{> zER8BGu>!=hqJsA7;47B2q{EGbJ~2X6mx)3wQG-|-?;1<;{P^dItYdgP5ov4?X}H4O zvZf>WjnPBF>^GW}NYprV6OGXzk&Fh2L=6&ui5l0ZM4DAZGL6u!jj9UTtAnpZ(vl9- zhzCT{s4f=;`73ICixO$pOXU1MzYWfpND>^ghe>&-EN+p)*Nh(kvX0U1l)0MgnXBd+ zw(V>IlHb2S6u2jDTNFrUXWgejGDZLZlF;|X{|A7n40cy*?<&VOeO-21GY_(WSl5E zMUpWf0FsObkVFk4$@pFXNxKS3rV&80qJsA7;46}}q{HuW zy*k%o-K^tihHr0O;8{282>+Ezw0hXot;1+sYS&>jNaYTO9Y%vxG8&+=%Cp0NRRC00 zxpo9K)to^|hiL@YUBR5gMM3t;*mEFzt=XUR7L|X`!s|PA?)1dEb*IQ+ZEK5hRd`#t zMCXT1%{z?7rS2U@gIw<5-(fV!C8Gf@t1LWR1;AyMiAPY6%^8$*m`0FG@aiy8n8Gq{ z9Y|qow(IOP2TD3zBe4Doz8)qD!DO600A|hBp0~jK%UdGv)Z23<{7oixU-!1S z>kOS|(qQ~L)YHRgTERloOjcFUHo((=10@}T`32{t(G|)wf)dc&2&tP2y|0()93?(5Em&(bRA>(#E$W++Tgr`tS7$7^Ai*aQ zq&?-fLfa55#n|&LDCV>1&{5nNr&x%kYk#n1{F^m^3qtO(g1`g|4EMm)PDFG-3O*t2 zk81~j;22s^po7PGmgAE*fK&cz!=Riwl?;0eSFJ~Ja__vHj9u4d-E6SN{5k$yy}S*p zrc(E(;(+Q>aLOI9U|e0|^CBLuWj?Ipi8m6&<38$F-VTe5eISw;fR1sG%E#>Kc_qC> zbwP@|loV-xyUqzEb6Pxn^&>C(Pe64T@^9i1Fy_Ae1oa=YT_DN=1RA|;5wj~JJplwb K6}&!^Y$Sh>K^4RR delta 2633 zcmV-P3bys@N$W|l*9Qa(%0!Nn+Xpm%3ZT|qEYA#^Sr84v*{G@B-h8dO zs*wYp0 z=ZV@n9E91id^}1lhfVcJ9Gv{BYigr)ShRf?%Ex5mvyuH(d4+)}4Qv6k>VK$e#s!aW-rfR6lV>n31dBw^&91LVW z(W(+Gk~EtBx;!LS6)Z!Ay^0l@SVwzxVQRK+<>%vhB3ChuWvfLGKl`cQI*g-7B`#V> z=6-%=#DT77(-P#R-}Bhu=scAD3a&FEU37OjtBO#M} z!AQvLc2S~}*}Y^WWOlC@37OsRjKtutHk-fNY`k2_tR17oCOthtiB4wiGb16h_5~#d znYFJdaY%PhUnI;g;y0!RdZm?ww_lFE+@a2CK?fUf$<=;AW4g?LDIg7BGS!mFyL#>S zu=VYKS6_Oav+nuFA{k1uY-kjwOwfGM((xT*kwGI2+11%hzN-l}Xx9 z7WdOI$y1-C|6tX%MjA_5Bw39-mug&ePFC@;OtKUvX{>UY+Pda`FrU|g`JxuhW>4{A zmS)LRZKdi5(QFuhj?|ykflU2ylBk~!8$;%H{k@7IWl5{dx>o$RqhHs{eu-s@Q8$e& zq1y2i@9@w$c*f~(JW6cB{s~F|Iz~dFLkU2~NXP|x>XbmogcBytO2XS1?vj}}-wY?m zI!x>2&T-*TBAN3;dVGlzk4)Q@Qz4O~ao-Q}$C+C8^s=mfsy51aoIj^g9E|%9dfl#T zwO}E1a}N@$7|I?b-Z~p3nr_{*H#ph?(d9vO@kA4Zj!|NhYvBYX03suyAfg07WF!<^Fp>tYRp8fOzL_-4YE;*FBFX z(c?+<@JthbK#oyjlZ)X5B|su0p(LUNNJNQ4?%=0h35gi)63JOfc-teKoYR61(*Vsw zX-t=iA(MBB5|3bhIdJSprn9c)#M@Hk2wocc78jeV&0zk5?g7*608cHbXaE*|k zunUOla#0jaMgwGSbpL&z`FckrSoR-8UDxX3&*NFi8290G)iJSJQmR{y;qJhaUvoba zxW;Y?iY01nn_`I?#FEhfmdu(zV2K*UQj}qd6(E)+6|@E2RXB%|4%Y}^Nu#<<6s+4Y z8qc6RSc?AumQ{>#pQW5t$JU9fZrHo9iz-wnMQ*4Pwb?087*$mZ(82 z^)f860>rYUg7*58K@Ags>hBs$^8EPciL7FHdl9K`5ve=E*|DZ0_>Ivc!R$90lt|Rr zbQ6uyAd!p)h(rw%i5kbFL>gs8GL6u!jj{^b>w~XE(vl9-hzms0s4f=;`73HXlM-oE zOXT7{yA3W@ND>^=hiP%A%x{sxtLr}iWEG>^D|0nAGuOJKTb8|l14w@V{z%}SxMflx znVofy0?8Nw07ymyK%xeLM2)LaAk8u$nMMG}k_y`EgRel+k`C8Py3PYpT`mfML=8Sr zn(qk6e`2wxk^%0;q`ASQZqzkz#|EU>VKNbD9Iz~kB;!QUD3Xi;0gz-gfFx=VNyhgA zNLpn`GK~O|B^9)P*9Tvbq$M4$mw+UV>M~K7wKA?A$Xcy`JCaolaId`8+R9r^-F7yf z1Gk~v;l&YX^RTEHhtW9Hjl*aV%02uzj0T}(G}Hnn$}Blt1psB4DMx@S2TD3jBd$%M z1Y-^pMV+3PjK;GllvR3jSSYuX*MXGt?0SroVui>3p~uO8ck0!-5u0WmyRLiI&IO)L zvySjysYI)XMcq1##-Vl{MuSxDVc20bNF}2ID$6`OTm?X7nQKQ-Q_VS)beKkP-4)C^ zToh!lj6Dak*NXi)Z&CU8EWEx`=T1*-T6gjc)-rb(SB1NSOLTr%)V#xJ9O~X-G|1&1 z{vAexTrwJe;IhoZ!&LxWmYH}2_1K(4Nr!0!xdg8c6NM=(#PQ5+X!rNwlQu8!-hr3SKcqVnmuR}dOjK-m! z9!3L9avI7cqX8xv4XpR*Wu6|c5z1s)1#KNXJy6o&dPyD+E+DGQMbRXd(LfTbSM2F| zi^+7D<0yXUnX;0x?UTFen_iym{_hzUU4CtByh=Nj@kpM?$s|#ND4VWj!#eJ1$?M1Uen7fDl$@0bFG@Ol;@6lzNuTGksmiB$c82o7 zs> Date: Fri, 14 Feb 2025 12:50:09 +0000 Subject: [PATCH 074/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9451 -> 9453 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index f0337d988..2b3d0141f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 10:08:27,306 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 10:08:27,306 submission_checker1.py:3270 INFO] --- -[2025-02-14 10:08:27,306 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 12:50:04,422 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 12:50:04,422 submission_checker1.py:3270 INFO] --- +[2025-02-14 12:50:04,422 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 10:08:27,306 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 12:50:04,422 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-14 10:08:27,306 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 12:50:04,423 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 10:08:27,306 submission_checker1.py:3286 INFO] --- -[2025-02-14 10:08:27,306 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 10:08:27,306 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 10:08:27,306 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 10:08:27,306 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 10:08:27,306 submission_checker1.py:3307 INFO] --- -[2025-02-14 10:08:27,306 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 10:08:28,307 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 12:50:04,423 submission_checker1.py:3286 INFO] --- +[2025-02-14 12:50:04,423 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 12:50:04,423 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 12:50:04,423 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 12:50:04,423 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 12:50:04,423 submission_checker1.py:3307 INFO] --- +[2025-02-14 12:50:04,423 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 12:50:05,468 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,8 +26,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72556 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0543 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0086 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1385 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index ca1898553..88eafce37 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -701 +702 diff --git a/summary.csv b/summary.csv index 33b0f0523..b7ea8390d 100644 --- a/summary.csv +++ b/summary.csv @@ -4,8 +4,8 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72556,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.095,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0543,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0086,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1385,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 7494b5095..e701a7420 100644 --- a/summary.json +++ b/summary.json @@ -154,12 +154,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0543, + "Result":23.0086, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -184,7 +184,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 9533a1ff6fba37607ca127c6ffdc34981c9ea511..70ff114ac63c514e6c9a977321f60e720884ea87 100644 GIT binary patch delta 1836 zcmV+{2h;fLN$p9n*9Qgf@kg-xliUYl1EaTEvq1`qO?eT(z^#+(;HbLWtn6(id?F3-Z@^zCo;`bn6$CXW%gDz--WZ; zUO1cY#edW3Q!<}sc`~)PRNaN~bPx^IpVhw1f@qwopAH%WX1V@e$B^=*RpwnQ|J%~9 zt3|)WBE`@wGf$|t{KUIFbS|E8`WufDhp>N)5`d18Q0Pzs&@mEnfu6V(&@thJiPMVk zHimm-CeGHw$*~U8I$1f+T}mW#en^imP~wwmyMJ_RByu>q3&P@dW-q^cSyZ(*%z2zW zXK@ma?j8)y(e%A=E(~i266+Ys4kX^VYb08p<~wT~ZGhBqG=n3@AD-3Jc&M@X#&U*N*r=A9HRtCWF(YClmLk+amfmP;#ZJ};U1BkR)n{G z!pRve=r9e?Je0p!DKW*=34jP2b!-dBEj1qi9e5LEo0n;&t=ENa!X0G zUDMlwCBNr>C~%M678Fa=I1a@UHHamn0W6t4f4~woh^45)5-UI~D=KIU`c&ZzN`E?B zBY-81>M~KVZ^LMOlVU0U16bBE#$C2@mK|Fqu7>Gs!;*jMbSUsaj%`pZQRBE2OVl8i zj0Uho4PuEJ#L}q35-UI~D=KKO4!&YZOFG<0=p7?Ob(tu{5;cgW@vgBX_m6*`$U26% z6OqOSk%lYWEqgkGe=vF|nCFcqB^weo4tx}7ptb3Jp_T*J1VEkN?m?+*n&6Spl2B=cmG z;SDB#?(;xYmx}@*QG+*>);j|7pV;iFWq>;|X{|A7n40cyc>pQ4m`ns32W*=n$v9DT ziX>w|03;a=Ac-18lJUI&l6Do6Oe26~MFs8E!B-?{Nr&qtAW5UTOcZ9VjH?H-R{P(M zWE}(CDQ~qm@>a`m+_n3_O(?f`aRk~tY-+}TVKgpv<1iY8atA*Sqd_Pc4Yh!YDoYMm z0YF)0$`Rnofszi>i04o!!I;BDQK#nxqw#GDWu4v}7Rt@!wJ+r?zaEpc+~M(l=yvj* zdUdYFx>?834By_m!Lx4G5&kQcX!WqETZhrO)ULy5kjfnlJB$XYWK1)#R91O*xJH0~ z$|~26pr)ELDCuy$gnd^q=WtPwy)yP3$X;vq=e$Mb-?Q=hPMteFv2NWdGFaW)c30_b z;S!x6HZ|`s8kf3v7!7i{gMWw7AeW4X2Dq%U@NkU)msKVnK|MBSP}1Rg30#6#hl#=z zmT~Jq3R|;X=l{dyJ2moLiA^VuWm>L(u?be>6r@#|1e52JCZr-#u1lbnV!$!LH{MgxnbQRV638lg;9RnRuT(*q?g9j=$; z_TU_%x?B`ZVi^r2u}00Fp0}9H1_h4dhwdqB8QU(otFi9oF@z>ek8ZOs5aEM71?Q#FHOe!B642ZTsoNTTtQP4bN_=2iu+C_x(H=aSH+L1clp6hnNvU- zzF?{)lXv;r??LO^|1Q7uI;Y*Uk7Y8H=Go9N%0}~hEsQ6P$vFQz$uph&(@Z9VyJ$L) zNuICCyG*9pT{^#?L}`)wq<0UtrZ=)g$}-7n6uDI6ymP#aPh^^>Fll3%%j~Ucz6)ow zy>K?)i+`unr({0O@?>gnsk#f}=^z@aKdXJ21<^QFKOHm%%2Ewr9K!xFN&q@WLZL$mK*vbP1$yFEK*xj=CQd8D z+ZgVVnK)YyC&xNW>*UUH?ouL|^Fw-kffAog+kd57Bay?=T@V(JGkf{d%c82iVb0_1 zIg68UboXFrj;8O0b75FJkXXl1b|CS_T_e%*G~Zd{Xahu#2hqb5O%OUli9@c1W0U}h zjD&)S5&)5rP!PQe5SefaVnuix!vRFE0z{^D3Zh3rWX=z{Utgfar+Jk3?*Xxnq3i(S zt$(+1OEhi6@O_>{pC{4BGfeEGO$Vq`4zTBclNx8I4QYI$3ErBAE&R$*Kz4 ztAi^jp`^n#LVm)|A*#zoQ7{<|kh#|V_krf?9g$#p_aHPw*TkR4vz9UL!soJMV!5QG z*{IFRgMX3^ z*9c%qqqfkGuw4}q0gg!AsRF{cDEK!448t)oQ^8EPciL7II zI}vGY5NWu=-Lj@5_>Iv+!R$Ahlp9FYICK+@(IAnG28cur5{Vkur$m}nL^6%gt&@-q zAThrU&X-6M9J7Z>d8aIHk;2!E9{{qB(e0GEn(LXX<{Gx`YypzrzdsbXCvIC5NM>h~ z;SDB#uJb@tmx}@*QG*YZ);j|7pIGdvWq>;|X{|A7n40cy*?<&VOeO-21GY_(WSl5E zMUpWf0FsObkVFk4$@pFXNxKS3rV&80qJsA7;46}}q{HuW zy*k%o-K^tihHr0O;8{282>+Ezw0hXot;1+sYS&>jNaYTO9Y%vxG8&+=%Co~&0900g zxpo9K)to^|hiL@YUBR5gMM3t;*mEFzt=XUR7L|X`!s|PA?)1dEb*IQ+ZEK5hRd`#t zMCXT1%{z?7rS2U@gIw<5-(fV!C8Gf@t1LWR1;AyMiAPY6%^8$*m`0FG@aiy8n8Gq{ z9Y|qow(IOP2TD3zBe4Doz8)qD!DO600A|hBp0~jK%UdGv)Z23<{7oixU-!1S z>kOS|(qQ~L)YHRgTERloOjcFUHo(&ZB^|CTm*nx_ z9HP2h6is3o4J5Hf&7PjOn9K$Rj^c-&DQg+qF1f3*?&UFrCQOemvoR3p4g@$Aygrj1 zA0L0qYQr!PMfd#*qrENJ=_3f+fwpATK%pru-3e*ph*}RMwNC!NvYj{yWYy*9-Z^t; zP`TKtHu(kTrO_42GlCM(+z6?g3catF=^Q0KFfCYXG*oC09xdvdl3U7+gI8xPxFEqN z5u`okwnEzwEXCOKEhy%*=+IHz7^hf>rE7n%W&E2pfD1zIv4X$^3k>(b)J{ZnKngw~ z?T>2*fZ!NfP@sdyd6wgoH-JQgF&0uwYzW;`1UNu4O)~;)ypB#N$5dSKbbbjC~-I7=Vs(kIKjF>3Jo+ zM0G)myOb1Ze!I>IC39LleDxzQ`cFW081ir85isVy`~>wMvtJ;}0tFhqY!UgBE+bn3 z_>+nwJOY9elb{k7linjL0k)F~Bsu{olSCvY1UMDEK9g=F76A#9g(N-!U6aHlBL)v5 H00000T6ko= diff --git a/summary_results.json b/summary_results.json index 9031972f0..f5b3ff099 100644 --- a/summary_results.json +++ b/summary_results.json @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0543, + "Performance_Result": 23.0086, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 74bffd7caff257e2f358f66671e85798e31c983a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 12:54:54 +0000 Subject: [PATCH 075/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9453 -> 9453 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2b3d0141f..0e040c04e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 12:50:04,422 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 12:50:04,422 submission_checker1.py:3270 INFO] --- -[2025-02-14 12:50:04,422 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 12:54:49,147 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 12:54:49,147 submission_checker1.py:3270 INFO] --- +[2025-02-14 12:54:49,147 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 12:50:04,422 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 12:54:49,147 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-14 12:50:04,423 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 12:54:49,147 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 12:50:04,423 submission_checker1.py:3286 INFO] --- -[2025-02-14 12:50:04,423 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 12:50:04,423 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 12:50:04,423 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 12:50:04,423 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 12:50:04,423 submission_checker1.py:3307 INFO] --- -[2025-02-14 12:50:04,423 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 12:50:05,468 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 12:54:49,147 submission_checker1.py:3286 INFO] --- +[2025-02-14 12:54:49,147 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 12:54:49,147 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 12:54:49,148 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 12:54:49,148 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 12:54:49,148 submission_checker1.py:3307 INFO] --- +[2025-02-14 12:54:49,148 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 12:54:50,111 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,10 +31,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1385 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1486 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2376 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.44039 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.428664 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 88eafce37..0a1c47444 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -702 +703 diff --git a/summary.csv b/summary.csv index b7ea8390d..901acb85e 100644 --- a/summary.csv +++ b/summary.csv @@ -9,10 +9,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1385,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1486,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2376,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.44039,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434065,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.428664,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e701a7420..50454cb57 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1486, + "Result":10.2376, "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.434065, + "Result":0.428664, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 70ff114ac63c514e6c9a977321f60e720884ea87..a186d44a3a49bbc79fda5175ca3fe362ef43595e 100644 GIT binary patch delta 1880 zcmV-e2dDV$N$p9n*9QgebqZqov)l(%1AoJFoGnYY%R}el8K=MTC~*k;$0z~l7zu?A zB>){GAs6V0TLB#tPMA2Y2ybJ!M`q$|J)9itFs+l7_Fm;yGEkrX}+_@(SHVr z9uK02Cz>F1gc65b3&$t{5E%&t5hVa3BcULA6(BO<6vT?~HiiR;UImCu>l8$fg2Kq8150tX+?P3C!Czof`1Ov0L?>bOqYovlXrmfj1WDCuyGke{$~i0X1t6ih|~WUh7peW3ZeB7YJr?;eC^ z=$iQRc-At;UHDvfOf0vQG}|@3Em-nR*lj_vM2+K6EK!44G8({=+4Bc1QG-~D zDlD-A#ImA-wxCZH&Y+~jH3C@Ds4f!)`!?*#ad0{QgkjGjZFZKr&C(eF`LF1OOlz z4FHK61QIo#PJy(lfMgm0AS)_puMWNfNlQ9hFX=uHM0L3+01`ELLutJuApeQYo>~UD z6O+~&lZL73{uYmk*neU&5ojE+ZHgr0MA0ddi~#|VWHf*zY7j}r_X0@TRY)?80Fo6I zv{wgTk)$OZu9tu$jp{N{n6)yl9>`kle>;+O3~;Bs)!xWkEyHow?gKZW+~UO%X!EeC z8Hdrh)Q!Vv5Xv3=IE)6NWHi(QCaNqsTm=ASl_^JnD+fwCOn)PuL!ks?4iiP4o)?V9 zw<(l$dUIGPH-l~uT-Md!=`Q> zM&nYu4x>RTcQEWQ8l;jj&A?Jw<=NpH0V=CpJA#^O&Y+~j^%C}7!JNZILH5enb0B-I z*`Mwi0S?)1dEb*IQ+b#L2UrMHDkbbi>>yu)Z*>fT{A$mI_H9Y%v(G9DV> zvdY54H3D2#nRo>C*qlL0hwCM930@s03R76dtph1+&32vt50~%M$a5t&ojjIlxyCjy z{ViakOT?yL9!BF*GY_LdFn6%?Fd77t(Lfqo<>=um0Dmy6JUs$TJy6o&8iD;+@bxfJ z2qxp~0WfQ}_Phn=Uw$O=PQ5)>!rx?4_jPZJyUx&gCJn}~Lp?o=#-*MfMgvT88pK(2ki;4_dwSktG8+^)iXXbC ztYvJwD$m=C zr~RmK4+rDLi^Ewo9V+SXi&v7J@|EPGvT)FwkEN15&ty}TPqX|C6@yik{?lObGO{vD zqBx1N=SAzq+tBlI(3-^kk7cLk)9ClP{P-dK;8g5?2%S$qE5;|KspLa{k>;Z-TPZ$p z-D(}of}uPQl3_GWm9ZS;1J$)W%}*AuHd zd-+mK$?7%n)*rL_5FZW&`Dc0bCX*f?AAe56Fc5_AJVoSt;v`KGSV<}Y>WNB71W|Fb z>}_ii|Hw9^eS4fVOS#7|20WJz4nq*Y{!7Sg8I~3{YKiMwkWawXI8hS%ky2)cI97@kRiD zIt2a3+qlTF4|o&;kYn7F@;Q5cX(g7Z&eMDzr+J(eO_q~+L5jz(e&kjE87Rjg|0bRQ z$K2PSp#B51Um(f?1@3hUV)~OVBU=IalZqoe0)rBhpb{37-Xkgjx04AZIsqw@L?k8! S`Dc0bCX;U@6#)v9g(PTweT07i delta 1880 zcmV-e2dDV$N$p9n*9Qgf@kg-xv)l(%1AjA5sJ8sXyF7F*o^kpcj}nKle~c1ln%oAl`Z#OQLBThVSzv`aFp~o@oNe5lS3#F&v`=NMt0GM3exD zC~?UOe&Sb1IcXik+X29yzXqVMS(a}4`pZATlPGi^#j>32*O2Cp7>z@I&5s!k(8y^hj*JF) zWHc^m>tvA&+ho~+WMZsh=K;~Na-v^qnD}N%v^6o)s zhOUV}k7q4o+=b6&$Ha0=NwZzk+kz#(=YA+~kKGm&OVl_H#S%4$C8Gf>nLU5N5;cgW zsKOE}KrAaNXbbvO;S5STTqA%bjp{N{uy4a?e3N1+{sUOnF~(iCa+V!iC9a0)Y{Qa& z>U1dZL5^)uEK%dQ6id_~mVb-}utW`Fi5kSxsKOE}KrAaNXs-^wVo6In+(_shBSdwX zD8v#qh^6tau_X78f1b!XhPM-u#s-mwE8H!6I)Z;NdMKFZjV2`$H4c4<#%Pd8Mgv5m z28l$C>r*1lDk7Oi=%bCQ3filKuSC+44%3JSMAE1(7X|q%YJ7_lX@AyBp0`i~O?5Smd zJ27dkF=?2Z?r(VjDSx(@OavMSY?~sag4fQc$g4p#v{S!K!*;L3rL4u8{#=TIoYn8QR-r{@Ku z@ofrao!%T4%FX1pFXb%19+R}(;qiXxcJiHib*{y_S;x@~-`={xvu@TA{wtMe^{}a1 zhtasyuES`M${h?lj0UM>Of#@lR(W=~Mu5sH*N&j3nlmWraJ__mS1{*rQINed_8iDw zYxd{7MdjbK@qhYGojX0TZrv#|Sl!!pSLtox5}hA5HSaJQm%4Wt4RX1Ie}~Z^myCx7 zxU919aE$<$RVE%mJvL`h(&2guT!L4JiNX|?aqB<|TeDr~|HI`wHS%1EO(&0KTCTAT zOn(cQ=n}E1mxs}~)Xc+Z5X>FyJd6gxWHgY*RylgO3V#61Do>99QxBAMxJF?A6?{ER z6oSb(djQOutvzpn`IjGwyi;$_mGC#2)P3FC;;u7vo=Jo8>rhV*qj9OHhtU9&oQ5*V zXn;vZ1B<0m<>}!Xp-fg)&^Ex+10@}oKM#xa#gx`zqlv%Zx^rPzRL6V z;%PrB+{3|m@#1h6O@~VQ`{I>kr+g*3s4N`x=3}X3&okLn<WvtTICgJc*@Q)Mg%`9O6oPxF(->&@loES{C0Y*o3Bvn-yxd>hH2FO%Y%sVajw z&R)J0Q?hzZy!8jO`4Art1t_^M6D5-!A0K~e!!Qtp?|llRyDizNOA)pMZON&DLQ`6D z6Vk>JwSJJ)I(hplPU0lsQ;(~e{dRUyHQ%W&`32{t(KX65f)dc&2&vl|eXJJgBT9T= zTCmP&sL>ugnm2b9x0D+PZ_Ze7L4r>rNPEg{jkY0Jim~TgP|Ro1p`*AlPO%V6*J6Lm z_%~|-7lb@u1%U|`7(M_~I}*_eDfo!AKdu`9f@A1Ffes$$S&lE>08aVIhC#VZy06oS%E1$FHm(?T!H3cckloV-xzsd_*5=zSTVdPE!1*lF#{!Kgs zj=66?LGuT*Um(f?1@G}ku=|rPBU=IblZqoe0)!HipAr_6-XkgjxRVJaIsqz^L?k8! SD7h~aC6jL?6#)yAg(PU0RC4tI diff --git a/summary_results.json b/summary_results.json index f5b3ff099..dbe5e701d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1486, + "Performance_Result": 10.2376, "Performance_Units": "Samples/s" }, { @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.434065, + "Performance_Result": 0.428664, "Performance_Units": "Samples/s" }, { From eb84399b766149649a11f2a9b2323b8f04b7c713 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 13:09:51 +0000 Subject: [PATCH 076/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 9453 -> 9456 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0e040c04e..7025000b4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 12:54:49,147 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 12:54:49,147 submission_checker1.py:3270 INFO] --- -[2025-02-14 12:54:49,147 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 13:09:45,788 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 13:09:45,788 submission_checker1.py:3270 INFO] --- +[2025-02-14 13:09:45,788 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 12:54:49,147 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 13:09:45,788 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-14 12:54:49,147 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 13:09:45,788 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 12:54:49,147 submission_checker1.py:3286 INFO] --- -[2025-02-14 12:54:49,147 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 12:54:49,147 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 12:54:49,148 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 12:54:49,148 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 12:54:49,148 submission_checker1.py:3307 INFO] --- -[2025-02-14 12:54:49,148 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 12:54:50,111 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 13:09:45,788 submission_checker1.py:3286 INFO] --- +[2025-02-14 13:09:45,788 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 13:09:45,788 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 13:09:45,788 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 13:09:45,788 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 13:09:45,788 submission_checker1.py:3307 INFO] --- +[2025-02-14 13:09:45,788 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 13:09:47,051 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,10 +23,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72556 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.095 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0086 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1385 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0a1c47444..ed93b7a16 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -703 +704 diff --git a/summary.csv b/summary.csv index 901acb85e..60d47f799 100644 --- a/summary.csv +++ b/summary.csv @@ -1,10 +1,10 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72556,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.095,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75124,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0086,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1385,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 50454cb57..98877df4f 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72556, + "Result":1.75124, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.095, + "Result":23.0035, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0086, + "Result":23.0893, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a186d44a3a49bbc79fda5175ca3fe362ef43595e..6a1bdde63f8bc4900a9adc860b055d9c70e5d979 100644 GIT binary patch delta 1214 zcmV;v1VQ`lN$^Rq*9Qrr3U1>42mkcLd}= zvDs700C!^2T4T~MHC;Eim`ubLmx(~zICGdN zYW2KeG`>xttkaysLb;i}_NAQV*JF~FTYo&>58Y3`Q?t&sSa<6LXY z61^Tawd*h%m-=-W4N|#-V~5com5gf!mdYyA4%Y}!S!LT1)Kzl^B^|Dpu<;7+94-pd zSH_1)jfowum`dv;#msdc9(*1bDL3hVh>oU6jy!X;WiZ0g=&G%mI8FdF1?2Y&+( zqd_hi6Af@#<>BEP0WPatJc62R&Y+~j^%A%Qvknu5DJ)~xffTl8zs~=M%XjMJxe}XJ z9?P^`!`T9+zXeRRiP+T4!)RRU=3z7l<_>-yMuT878c1WSEInKW0A`h`M}VsbN;+I4 zumKCk9wrLGWUM^^X3gH7x4`_%&woVTsk!G$_?t}XzV2B?)CaWrF8(`{zk`C8Pa(i$NQC%*ICb5hLl31f=Q_ovW zW`hDp@k95NwTx|-+|^h&^B6)CrboA58*A^z4r;b6RYb2y8pLnZxv@lLW+zLQ*377lvzu~f3>nQW@^X_lX%Vz8>x ze;O>_MpkA?6em&kylB069eO?vT9df{vFy}*8vQ<(A3ua2oc#}>^XX^B_@p$IeCRLI zd~{_i#Rslit%F%Gl;=S*j5MaHGM0mUpt_c)`N`t_=JInE&&p4>s@%s}7EfNjjbzZ5 zN%758l|dY5FJFo&S-mIT`UA815FZW%F%&x~lN=u`e=QYQNh$&AiAqQmQE{{EZE6w! z$Tp;Td)%Z=TPY`w>zVy_cI9GukWKUp)(NdjlqD&OKrzjQs!Q~-S;Y@1a-J$q8?B&3 z9XPbC?h0lI(-z*WHekH~7X^@VgqafUy*C77$MztpOM*iMai^`MK9sh`hO%#31J2WQ zfhBm#f2pVV2#n20L?@&*JDl zI58UAwoTeuZw>h~yj{P(_p8Q29Z$soRRw2+S)g9qs=(JpI9x-WUxyR#1n{RW=o4?J zMTS1$Q3ybWxM$^a_WZJ0p(;=F`#842mkj zV*~&o84Uo58UzwGo=$QctdHuBOw2Y z&7N8YxD%7s8k2^p>HZdviP&N?5ojE+ZHgr0MA0ddi~#|VWHf*zY7j}r_X0@TRY)?8 z0DqDd6|`3eUy-CG9j=#vB#r7aQJA$dt{%u*?SDIxbqsK)yw%>wTP?$J*X{#1q1@ud z5oq(UsTqgSxYUiqXb{RB{5XsTp=31F0w$^~Ia~z*WtAyMfGY<|I!q&;L!ks?4iiP4 zo)?V9w<(l$dUIGPH@T`8UZS+TswlAYR;gf!}SvOUBR5gMM3t; z*mEFzt=XUR7L|X`#_KzE?)1dEb*IQ+b#L2UrMHDkbbi>>yu)Z*>fT{A$mI_H9e+lH zTrwUS;IhiX!!-h2R+)GN_1K(2Nr&qta0y-=CJIwn#;pS>Y|VC^{|}e%)W~xsHk~|{ zX}QKWF#RoHqD#c4ULHo{QZo;uK`?i)^Dr6&lhHsLTjl8CDgZF6JUs$TJy6o&8iD;+ z@bxfJ2qxp~0WfQ}_Phn=Uw$O=PJg{USHj<9QulRli@VOyc_t0UuR}dOjK-y&9!3L9 zavI7cqX8xv4J?*Mm8XYmgfdxGLE8XN50rGcUXt5`bBOA4Q8bBVG?2s^HG6vAVlo>P zIEo*-r>tdcyX3CMx|hcgnlL@O{n}W2m3AVNp*)u3ajJxIK3$8*Rq5jW;(wmlzg@hF z`zp`di>Ljla1RIL#f!sPG#x7G?~7NGo${6BqOx$%n~$ZEJAY_mHyLU z@iMY9OQJZ5vgbwX#oN&HanPE?{f}j*=F{l+x%~Jc{NU_=2%S$qE5;|KspLa{k>;Z- zTPZ$p-D(}of}uPQl3_GWl`*j#xq^phVSEPqWASV<}Y>WNB71W|Fb>}_ii|Hw9^ zeS4fVOS#7|20WJz4nq*Y{!7Sg8I~3{YKiMwkWawXI8hS%ky2)cI97@kRiDIt2a3+qlTF z4|o&;kYn7F@;Q5cX_lzY(|jJMd7KqZmKc+HL5jz(e&kjE87Rjg|0bRQ$K2PSp#HOA zAj<*??sW=c`Un63YLhi1R{{BxjUzk(gOkc5K>@du3?w=MDU(MeCItCsdGwQUBo+Y* UlZYff0bi5HBqIh9A^-pY07=q2uK)l5 diff --git a/summary_results.json b/summary_results.json index dbe5e701d..18201656a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72556, + "Performance_Result": 1.75124, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.095, + "Performance_Result": 23.0035, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0086, + "Performance_Result": 23.0893, "Performance_Units": "Samples/s" }, { From caaf5fb62a6274bd8875834d3d9fea2b8e1be7b7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 13:10:55 +0000 Subject: [PATCH 077/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9456 -> 9454 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 7025000b4..e79b8640d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 13:09:45,788 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 13:09:45,788 submission_checker1.py:3270 INFO] --- -[2025-02-14 13:09:45,788 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 13:10:50,643 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 +[2025-02-14 13:10:50,643 submission_checker1.py:3270 INFO] --- +[2025-02-14 13:10:50,643 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 13:09:45,788 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 13:10:50,643 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 -[2025-02-14 13:09:45,788 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 13:10:50,643 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 13:09:45,788 submission_checker1.py:3286 INFO] --- -[2025-02-14 13:09:45,788 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 13:09:45,788 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 13:09:45,788 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 13:09:45,788 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 13:09:45,788 submission_checker1.py:3307 INFO] --- -[2025-02-14 13:09:45,788 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 13:09:47,051 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 13:10:50,643 submission_checker1.py:3286 INFO] --- +[2025-02-14 13:10:50,644 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 +[2025-02-14 13:10:50,644 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 13:10:50,644 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 +[2025-02-14 13:10:50,644 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 13:10:50,644 submission_checker1.py:3307 INFO] --- +[2025-02-14 13:10:50,644 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 13:10:51,668 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,8 +32,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2376 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.27177 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.44039 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.428664 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ed93b7a16..dc1933269 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -704 +705 diff --git a/summary.csv b/summary.csv index 60d47f799..01ee175f0 100644 --- a/summary.csv +++ b/summary.csv @@ -10,8 +10,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2376,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.27177,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.44039,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.61982,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.428664,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 98877df4f..f6b428fca 100644 --- a/summary.json +++ b/summary.json @@ -339,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.27177, + "Result":0.38162, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.44039, + "Result":9.61982, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 6a1bdde63f8bc4900a9adc860b055d9c70e5d979..13110b490899a76fc8705bb1a288da88182d5032 100644 GIT binary patch delta 2668 zcmV-y3X}ElN$yFo*9QgTL&(+qliUY21-gZ9F58n82rYkmzdVgc+Jj2cD4y)=!Z3AB zO~SYrO$Ph=&GpwEPuJ2cnDl~CJW>1lvr6?}4u0BujFaEep;DQaS1?KU^i~ z9IA1U8u3(3@+18?83$SZH5oM0X`+H&88jX>q-i>haS%=Pc@4YCHg&}PeiW+DaX1^R zNj9$}QKNq#%e$WrqiL!i?DeAjSTPz>1<+D}{M%3@8Q7OYdjPcFBDfa zQVyTr(26F1cYOWIk>gNpmS#8?MC< z*MffvNJqPm$l2b-PPECuE zkaKg!NXXo_83~!(b4EgDw}TQ%X7_@TklBB|WF%yEzcUhxzuFxBYIE>%C9`&f5{LBk z7$uU-+Gj>WX6*|~EHZ0fQR0&Bp1eqyU&L=r3-n5>2yeX{ds(5*Xh9nr@W|DEPGdUE zDIg7BFx8UDyL|0;|KQvIF28i!r=7EpWipiJ+0Y=$hO>JkjK_`fDE~XoGoAd?R3(4? zyJ*r^NuIB%yG*6oT{^oTM`@Azq<0UtVK%ZvsWQoG6uDI6ynVcgPgI(xFll3v%j_&` zz6+<*op3tai6@h%WH!n2WNK%jx(nk;KN{#i>phhP(J0kF?KS$$a{axIA>~P{%DYzn zx1nEG^L~kWieWUR=QtadZkLD7#WR0Sf8$Z&5cZEz0?;uM3LQ!SIz~b+&=a=;IwqVj zaas}H!f=nw#Mx>%Io4)cCo9LfONnI859#p*N_;YHmu`(j4u*F@SlrI+u$4!IVN zQ34<`5(*+p07OPYLG&s>WWp(k72z!m2N1mq5Si8~h#m!zIX`5*zCejj^C<7%17aOR z*#g8{Z*57mY$<)8C(-9g^zlp+K#ox2kc;6MB|su0p(LUNNJNQCR`3(QfnZ9Wfe*{F)y#8laKWP#hTz@W^Of($>jR!x70;07zC< z&{`f`LJ1{pt`YJRb`DV;E{cN5Xn@R>?!ON-Uspte<=ulYq-lshk7s`^W88+%MaRTq zOUbZZ%iDk@zvq4^aF5*<6id`N4#g5Rh$W)|ESWujz!EixrKrLZD?ltODrgD%RN)Lt z+FT=mC5`GZQLt~rXnc!eDgFak)-lFywsIC7TPCj3ayDVfKXp13_#nrY6id`NF2xcx zh$W)|EK!44q6V>)Rak#w1&C!u1+C@5S1f5sn;QweV}z&<6NOl!2CSR#dQ$R7Z* zj?rzExmv54tKr&0Zg|=u_~-YB0-uT7O$sFQWZkDgGDZLZlFSlQPLT5P(TW15Ba@nPiMFz@%NpB-02mSy4f2dGM7s*Ufw~k{--`=>xv+C9n{wtN}^{}a3htasyufu4N$}Jo_j0UM> zTr;p#R+)CVMu5sH+m4{FnlmVAbG?L(S8(TWQINhe{v1eOYc}Y-MdjbK^ZHJ$J6*Br z-6>L7&)Ba>@sp)MRr8 zC2g*kz$KVR2{y$v4Qzy@rShw;tt)?rT4Pg2kz(kvfP0c)v#-(l^ zMuT8(;pbsA2qvR}G`7mp!&LxaR+)MPxO$+Z%{6}l8?a#PVWJRB#@Yj5*6i(h3(UX# zOyr%Kd#;4P&ZOy^-UfS}GVc9r*GqDHa1K!&E{Z0xj0TceS+l9!#`$#3BbTM~7mT}N_jdj+?yEd+FP`+G!af{~=5G#X(PW^d zzt7)EcFK2>i^{@&cQ#U5_B>Vlx_X-CXQ&vguJxY!^S6<;X%fXrls(T|FJ6b9je>)5 z-1}H|YBq^}pQ(=@!Vk{whtT=-vnD@j`&vmp^cQJ9y0Vqx1J|vCy=gE|=Rq=vCaE@3 z{d}ONk*E2|{Qc(ga~4m_Pqwbz$5|GSU%m}h&{IkA&C<1g9A__IiYZyXCw}k;v-}Vr z4h6R(kn<&zARixpYQr!PgztR{qq{B3Zb}ih18vEvfkIPSaud=f5w(7h)H-?l%1+`W z;8TyQnf-QlQ8nMIHu(kTrO`DaS&kCW+z6?+HTqaB(npl|z_ehk(NLoUcr1(VIa!_1>-}= z)%=q&BU=IclZ_)h0fm#wBS8VVlMEy}0V|V7BqkfTBarhY0RR9{0ssIJ0000000000 a0001hfee#~Bt8LPlgA_@1`;9w0000C75T>i delta 2659 zcmV-p3Y_)sN$^Rq*9Qfn3U1>4liUY21X_2w43icJEq@2UJWa;RgG|yWo;FpX>8c{9 zVcd_V!=`$3{dLb%l{5>c{a_qVWmA2YsrpOnr-R2h`7IqunJIY%)3m9MvTU~BXr$pt zPJ&d6XL6b!8N|sX$nvkru#wIZ8T8Ab$+%(Yy3?2h(Nta3u%GNwM?4rrq5K?&^NE~h zi%JqX4u7(|`{^i}rE2S-ALYl2@hFKLG}S|~fBdVVtBuw{+4gNDAJetZN->oCIR35p za@udIrL+OsuVptb6QwT)!F-(c;>T|?8jiAj+Uz1EjK^vDSD8flTpFq}37(p&sVa|A zKN}SlE2F62m&HV@N;prmc=GGwkXTl*2o=sUR)6SX742)n(jCJt&d2juE@K*tR?8lK z4zi$iki?HlQnrxJg5u1GeZ$D7B`iw67sW;28q0#-OM#M)qnN#imU+;4C@Nkmu4u?L zEx4ftw`&EByg#c>8GNS<=!3r5haM^{BDrEDVpDR>NQ_O%4I?4*amz@om1JYMCO=#g zDt{my?V-XZqkU6x{ifpPO~q}MVq>HhKT-?g$$a-vVbP(!ExCSMa)T0w47Re8jnUfd zXb)|uz};-(2o>I@;usbFrs6Xy1VP~oBOzDVS4Kik%?TqRD12ii)~4i?k&sidc)5~UJ3@&=dU}i! zgUs4zMnY!o3rb8fYhO{~lJ1_oNSI&5Z%hmHN~;KOzZ`p6q0VSQ2OIFn)qYN6y38pc z4PP+TlF7S#?f0Pd?SGeFdY#km*~c;&O7m=J7-ggRy%xrk#$=rTo#dHL{%IzY!GB#e z9mpil*W_I$)9fyt-%p~nNPW_~2V2t{St4bbWHpLhs&U>qUdAUf%~P1PvCL)mRyE&+ zv)NuaoA1Tb=~FVFW_dETw^ZGQ@pKRk)t}YA%z|i~s-F%T17^AYUdNE~q*dl!EC1Wl zud7AB#3IGeEW>l0Elan{L+9cdr+>fkC~*k;$0z~l7zu?AB>){GAs6V0TLB#tPMA2Y z2ybJ!M`q$|J)9itFs+l7_Fm;yGEkrX}+_@(FTYf52A-Bnjmz95`TwW3&$t{ z5E%&t5hVa3BcULA6(BO<6vT?~HiiR;UImCu>l8$fg2Kq8150tX+?P3 zC!Czof)3LF%|mHSmx&>hcYlErpJ0CJ*9+Su2;+2b9F#RYdCuRRlh!f39RTe4YXJJ1 zW$C7`zx)$Ci9+X5EX&D$4QcL((KzJS{Fu=Ijhu$!$Y_8^M&pvUPF5O@NTvcnvZ{ji z>fj1WDCuyGke{$~i0X1t6ih|~WUh7peW3ZeA`&d`9)xD-n)vg0)_*d_UHDvfOf0vQ zG}|@3Em-nR*lj_vM2+K6EK!44G8({=+4Bc1QG-~DDlD-A#ImA-wxCZH&Y+~j zH3C@Ds4f!)`!?*i~s;6qX8gMgFvFj(m}Xifv7GQ1wf((Zz!#I1mr)l*;C5^cVg07W704+T{pLwOvDzK zi9qLoZBr&0D}RbknPeOYz$BvqCQ*Y-GR7BR(yn5XX#|+8sGz+%_{t2SRSCTUcc ziNds%vGqXOYX94rtYd&XC9d{H;%XU=yLKbE3FQ_ujzFJ>P2D(*#-(-~MuSl9V8~%K z2qmMT9xzem$>AyhD63pK0&F=@(qS6$910~kbC@V<^?$rzG`>xttkaysLb;i}_NAQV z*JF~FTRh$m-A}$#v(B|xck4Ks;oDnxc-GxI!hfX_y&g8T>o6LZ`gIr$Qn`a;htVLF zjB5s#$|}wo!MoU6jy!X;WiZ0g=&G%mI8FdF1?2LlhIK`t2+4RBfI;o%wqE~{KTf|_j3 zprpg~61W7j4ikkbEMwP!6t-r+&i{wYck1N15}Q^Y%d}j>*#f4&1x&Pw*woC!Xk6;% zVKfNl4t^d+gJ3cmNMoxkJzNCT`r0yv5W?iSfgfB&s$7pg91nKL-&-mjBS_P)mS(47(x@K zM}N0p8*A^1W0Gq)RlFeCRLId~{_i#Rslit%F%Gl;=S*jHanF zmVROy@^co?%1^ec+{alKPhP%_WYCvM@y%3~K^$izVy_cI9GukWKUp)(NdjlqD&OKrzjQs!Q~-S;Y@1a-J$q8?B&39XPbC?h0lI z(-z*WHekH~7X^@VgqafUy*C77$MztpOM*iMai^`MK9sh`e}=MeS_970bb%#!%BiRL z2#n20L?@&*JDlI58UAwoTeu zZw>h~yj{P(_p8Q29Z$soRRw2+S)g9qs=(JpI9x-WUxyR#1n{RW=o4?JMTS1$Q3ybW zxM$^a_WZJ0B%vx#^ZPi><7~dkW+Z(i^TpS34lP@D%0sNDTBRm3z5|g147L(s2DgnBa3M4uKER#hfCIvARJ1HiUa3mE04U>i> RJ^^Br#UvvJ6e0is008xg1?&I- diff --git a/summary_results.json b/summary_results.json index 18201656a..db3ec7cce 100644 --- a/summary_results.json +++ b/summary_results.json @@ -212,7 +212,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.44039, + "Performance_Result": 9.61982, "Performance_Units": "Samples/s" }, { @@ -320,7 +320,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.27177, + "Performance_Result": 0.38162, "Performance_Units": "Samples/s" }, { From a5634548844290744893acddbaa557b32aaa5f43 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 13:15:58 +0000 Subject: [PATCH 078/719] Updated results summary [skip ci] --- README.md | 55 +++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 7 +++-- summary.json | 64 +++++++++++++++++++++++++++++++------------ summary.xlsx | Bin 9454 -> 9546 bytes summary_results.json | 64 +++++++++++++++++++++++++++++++++---------- 6 files changed, 130 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index e79b8640d..d1ff2b0dc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 13:10:50,643 submission_checker1.py:3263 INFO] Results=17, NoResults=0, Power Results=0 -[2025-02-14 13:10:50,643 submission_checker1.py:3270 INFO] --- -[2025-02-14 13:10:50,643 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 13:15:53,171 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-14 13:15:53,171 submission_checker1.py:3270 INFO] --- +[2025-02-14 13:15:53,171 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 13:10:50,643 submission_checker1.py:3276 INFO] Open Results=17, Open Power Results=0 +[2025-02-14 13:15:53,171 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-14 13:10:50,643 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 13:15:53,171 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 13:10:50,643 submission_checker1.py:3286 INFO] --- -[2025-02-14 13:10:50,644 submission_checker1.py:3288 INFO] Systems=13, Power Systems=0 -[2025-02-14 13:10:50,644 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 13:10:50,644 submission_checker1.py:3297 INFO] Open Systems=13, Open Power Systems=0 -[2025-02-14 13:10:50,644 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 13:10:50,644 submission_checker1.py:3307 INFO] --- -[2025-02-14 13:10:50,644 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 13:10:51,668 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 13:15:53,171 submission_checker1.py:3286 INFO] --- +[2025-02-14 13:15:53,171 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-14 13:15:53,171 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 13:15:53,171 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-14 13:15:53,171 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 13:15:53,171 submission_checker1.py:3307 INFO] --- +[2025-02-14 13:15:53,171 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 13:15:54,267 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,17 +24,18 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.1385 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2376 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.428664 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6851 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1705 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433437 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index dc1933269..c00f282b8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -705 +706 diff --git a/summary.csv b/summary.csv index 01ee175f0..1f94b207e 100644 --- a/summary.csv +++ b/summary.csv @@ -2,17 +2,18 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75124,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.1385,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6851,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2376,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1705,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.61982,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.428664,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433437,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f6b428fca..47a04094d 100644 --- a/summary.json +++ b/summary.json @@ -90,6 +90,36 @@ "weight_data_types":"fp32" }, "3":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", + "Scenario":"Offline", + "Result":7.68088, + "Accuracy":"F1: 80.0", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -119,7 +149,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,7 +179,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -179,7 +209,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -209,7 +239,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -219,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.1385, + "Result":20.6851, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -239,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -299,7 +329,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -309,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2376, - "Accuracy":"acc: 75.400", + "Result":10.1705, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -399,7 +429,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.428664, + "Result":0.433437, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 13110b490899a76fc8705bb1a288da88182d5032..577c137a2564ca094074d41c2d55565a8852f1bb 100644 GIT binary patch delta 3887 zcmV+~576-LNy<{9LX+VKAAg)%bK1xjh2PJwP^t2eR1HGiLN}N3 zlzq6|&|15@cS|4LQZG3C^>H%R z?p2aT@wBN6!_+l34dZ?^9X9ow>(4b$*U~JQ_JeUeRZaa#rTVW&|2ce!li$;kQkj-l zFn>*(`Y6k02laXyj?^Sbjd-S}`H?}KOoA+bn+)seEKxy!5i}XsrD;0#Nf1r-Weo?( zK6S){K@_TwaX6o-X|}8+QR5)XyPuAtS*jl$_M`k*k&l+BK~p~#2dBSDq1TTN7j550 z>LJ~Dtre-<#qsaOo3H(*ZWhuyXg@EyaetL)eKiQ?1Y=fHp%u?#r3O-n^zULWs0p#8+@h>h$rLSMMZ4b7$hkRZBxG#cjD!sC1tTG&+d+vWqkG9n$mm`%5;D5q7>UJSZ4Q66 zIe58}Q9D72L%MN_5=lnwBO@WB_6a2x8MV(SaY;jG&l2Vn@e9)ez0yj;TYt~To@c0Y zTF}M@JaV00(3lQ$3P{73DDmmV^3w0Y(U<>Ted@NqcFx}~7DH*i8X88~Xntpe@uWT( z=YJ>pN+*9cQ^}wgO$RE;*K4YmsWj`Q^Seou7E7PBcW)bJJxi2YEVAmwTB?50K3&Bp zD$SQLX??YpsjX}7g|k^LoPW)0@pSr_%%@qtn5wN*y)d2*qM`na-d9-=jZ^)@VST_% z*Wap0DPOdzylad9cJ%A}vR`7k#4s9`)v#P1I+use#WPMn<5A*}kIhq*0CbFmLWdH7 zj**b7@60WMjtM8Jd@TuYVYo+_J>Lu`A5CqhbxNd5iDb?X>G35>e1FpNm0KZ^!%;5? zi`$u6e)qDhsy57doIPc65{`QJ(r^s37S4ri>_K7`sq8`Gjk`f&!!vxRu>(Yp2hqb5 zO%OUki9;^GQcY5-nRw-{(p6c@lj*(*%$clsM!9Iz00Z|<$iav0>WHdej{%WK9Zv)Mj8L_~!uhcBR9FtV_ z?uFsylj2{7vyu$=p!0gwu_d0a;drL!?SPVh;(jdf3A-&Ql&EoB3MFa~N=5@vGEe*g zO4J~fq6|u=0)K$Aq=J^9dkW`J(&id*DU>v-!$iSs$!X|xxzYW%2<66HSx_lWL;L`i zRphwOq`B&tSUu-8Y}eX_CBNr%EN~mgmK00WIP`%%qd_bg4Pc2H#1b`#r7XjesQ|Dn zsi3t!_<|)ZX>*MLmNcrvM8U&UM&mmaOZk6*WfeK@^M8DG)v@*6q_mt}Sn_+L$AZ~% zv?!LSaeRs;Y7k3C16ZO4u|y4GX_aBgQ~+3(RM1)^oh3wdm?*>&HHfA4 zrm-Xsb^bb$Riw8Uk=8bmp4iv|+$NIW!#@_dFK#y|k*IM^N+e?j03sO;5Q!Qj5;Y!u z-n!PXM1L}k(EY};3R>%fFGSLkHrGqKWergsCJNWDjK-xzHmW6ZewR;!^A(Z=$LxNx zm??`XQuv1a0U)c$Zm*SVV{_$dxVDfx+(TlA%|xJez_uxnj2A^vAQ=k+0Lf?oNYo&Z zjPr&1P1YI~NTv}0vZR96`rr$Yw4}}T5~fHR)qi245J*M?HkgxFsdtCbAeDQ#cNh&)$v9|$$}$5F*9cHq zX5kUkVRH^8ZLXK_;1!%YOcX9(84WC7D}OfYyhi2EdGh*3Z9HAE>EkJuu%5rePA>c% zT%tw9rbZq{<5DLNqd_kB@bWMkr)8129&Ks_Dcrc~OgDDxu4t4b~8kf3y7!82QX(*VC27kch zG!#r(=IY@ppkS6&(30Tlfs!`Y2)(z2s16eaU@{s2Q&#Nic@50JyC+gfe)}*f&21)4 zKc6jkFe!F0$ryI1sfW?H)YQXhkjXtvJ&XpKWb87)WSOalYXq1qGxbO?^*~9R>!nYb z1XB+a1&dftLl?2KVpGp+OlE@uM}P4{_mq|7w$IvCZkl=W`QPz2401~x>le> zkGXpPE_~-4ybGNVzi9G<*3|OUpQU+ri&lykT(^!6XTeZi1j#U(rrKBy@ zl}TVTZkx*jPZXym9i7(cc4zecuE@zPR3hW-K(h@?Bfw&wWNdSwU0xjNp+3T(7uoV7zrVX+RDU%b|YmuG))oTKtYAUOJFLJfAXrJQvF+POi@-q6LA9|i`;Z9 z%Qp*}F^$B$X7WIDTt_1{%l@fhvPGg$8;?Xb9?9`3Paq$eay&8{U<=$2 zWb#{kf8~Vdy!rw6NMe0mWKu%a4jkUnET`95o=LpvUPvg~=$T3(y7QvqF$lrY`Q;bH z7{`!9MI~NL(c%?P3m>CSywbh0;eFAdUVc2ozccT!_qCwvU_l+ zjVDx6Wh<3&v+=NbdNScf!PR?jZq_uGszUS%e^RJafq)*DwO-z*V4Q&u_x zNCB!CB|ZPT{6?;Tv*z!wKzH28yFx}dos!w&&j|?ve?WX!pYRGuzj?Q*A=@W;ChFp7 ze^=s`=a(TP@Ql#HHvz!_-K7GVG1?KhhDvsLY*Wg=I$+h#)obcWmZo*n_Vqj5UTIP0 z(9!62wKM3!-5%WQ!I3R?c+i7~Jvh?1f5mN-*d5-}6KvEIc6g6oyGO6xqu1`yYxn52 zkzPnP&~6yJ^4E(z-XiO`EFby=XY8(0g_sq9nna(Mx){3p-v;-;xV9bF>~Y<@&wYh$ zdpHUjbFtq|-l(Z7CzO{`IckiBB{(o$HvKl(aGVITlfA=cF zw>j?x#^=AmZzcQfyVYWnx5b0LMwlHL&O0-SKk}bx%XhQABafe71~1Wld>woU|UgMoWw6u94>~$jSMeB z1J2{zJ8%rg3-{(;-|&4u_;2iW9K#DOpOE9(+4^L5rVt;EMU#{HV!fK2FV!q;`^F#$ zeRZniS#ID)1454G)5oyW4Az4vj6BP+A}>T|ZO3Nia6C32sSAf4llFs75w#`qP*NXG zYxeA)W(pz4{{pkQ7GVekZ()Ni~U7diBSNMZnb zh8tNt@k9f$mzcmf=9Uw?x353^t($^r`b zmAr0I2><{9LX$KjRsv8FlZ+!F94HNCRtW?E02>hi02BZK00000000000Dyto5|g14 x7L(s2DgqT3lM5s{0f&=CBqjuJmM{R5ZzL80Xp@E{J^{Rw#UvvJZXy5x002TLSvddz delta 3836 zcmZ8k2QVBA(>{bKQSOM|InnD8L?;(SJG};R$`L(Ug2Rax?N0RGT_hs9sIPuRxsybq zh9d+)5EAv1_n+^dfBv1F*`3*)XJ(%*GdrrkqgOw3otmgU^6`uk0Ps@FSia7K%(n|% z8Df66bRcH_GAsw1^z7X;AIUk9IXR-t&}Wns_Fxrrm+U2gAG5c%bPvSZ|9~Uc03%=} zOV91`dg1o>@>bjXPkU5S?#-g~3BpC&TYb_&hK7A@EQS=PC*!|nxQu5A;yH&gFcsM8 zNu&NRN4m(_r^y*hgi_PoEGua}^kQq-`^zuu}Pf&)AJc&*X$#+q6xQmnh+=;)W}Ea1mX14niu@{`%Qlfvk` zpTyfYY2$3vyn}(Dq4K7)+LZEe$;!GERCUAxp?SC_oA!B9v-5{7?8CAkiGoTSm5ZQA zU2)Nzcxrc`n8DlRBU(I!3&FBZ469(4NeQcUrB}ah<5!*eSStAM~-FBCIXT4OIZb~s(L00pa=ci)kGGm~N#je$YOCnSe(le)1 zckgEC*0Cm9XWPqx+9lha1>tHwY^fNKky7xM2>Tw=R=Jq|f8mN7x0Nx2#ft>DV5*g1yR{zu>uNO`F(!j2TqR$nZHjJuq-ccVxbWjVgCO5USHOX=H%Nsbr(sQyF#);-=>y6(p*XD z(0Vv67w-P7T%JLdTZi5lqPal{rfX+Tg^}{gf~fiR0Xk4E%|lTQc%MkZ&1H!j*BQE~ z8FD{~Byf|XmDJ%~-5QC;2g18cWDmKtzmN_f?R>rUnjVQn6yL(H>mB|yVAM!8SxJAs z%nH`v7bq1hyQZs?c;9l5u_s37#&N=G6MyF#qtR8v@)1$rN+DJK1L?SqD@f^{!>woD zoU#7z?P=F6HYeTy%7(4!Y7<*KM-dQ(6zLfvgNkdq5^x@0rwe$>0n%vNijH&sP~yWJ@gPq zE1j6N1(b9JB0)XPUBYEPZu0K%z{C5vdWW&OC~(huq_AJw0yES74~leyAo&DvIf;0# zDzf*=t*q$-iD`(7iF#0-m@nTz>d)E>M1fN_rD65H{T|`ZABtR%S$~ol@@@%W=CTv2 zBLpZo?I8G?a zM3#M%gwdjivyL>XVKh^qH;WAHT?L#hHHndnPb{;Xq=6FWm`0VymBe3ceep8JA|h3V z5bcA+^QrUIJMvrh&n;S0U38Zg+xii6%?3B66Gz*yTfV%H^v8Y832f0sTTwH<0P+!3`JS21Pn+uu0ya#?ej zeXCEgX`KSaG7tTEJT8!3dBBp~@$fkKLUN~8#aF86u&hswGx?h@ojV?0QrR=Pq!&CI zf_XmLhl~tSFtt5Wu{eGAB=4Jh4eP7AUqvSet>|V1Xi)Ek>ZG02gZQG97?l6~sdccY zLSkewHjSr`eqtwn>d#@LkUHYjR_%}lq4usJbA@c?HL43CZL+Q@n;?W$Fg*`1sX|x( z!JVCd+Ye+m9y4yKTNt)sPIK(0wU{!~`#DZ%rj6uumvT%JvFW}Qxp%$M1eimDH2!%p zMtThZ$YcNjm;eAk=p(5>59a_^xY^?XgpYfmWT^L}ajQ8Wq5?z2t`)=P*bb`h)1WM; z6zLpQau8>~UeY11q!L|j$!~$)7eYFoU|v}XDCG;$9OCxaF!6GMZi<`8u=jEe(WVsz zg@_hs#5L;y;)~alO&xPIUA0IcpU88>5L*Gln72s^wREDTTh=1$U!|&Rb7)|uL!WNu zyhobTAlUgHYmOXi7D)nmh;swrMY*Z2f831Bs?Z5m21N*?VZ81j0V&G5gPxk9h(Obw z1o#cQjGR<)9*#@YU-#vAYBC7p>T$s`1^JPbjl#LcW%JDL(Z9;%M#cpZgD`Lrlydmk zC54c|fHa%Dy-t|6kib4waixiowRHE+Giu|JLC?%;Q!Sg@1->46c#FJB zkna-^XkS^`j87+|C|iY;Pf@w&H{9d~P=Ljtz#X1{WOo+^QZ@s)5b z20HQ&PsaV{5{L{i$t5*u!|@blbO`k{;@txo@-xaU36(H9-LC~lA$a1 zn!W{Z$-TRkEXJbz`2GMQYqv5aE>^TuLSO>(*t>}7vtEN|Wl{-2?j^racA7&^;mly? zS|0z~=`J_f&}Hl!5BbhVuQ?u+5u7x*OOjx9?#0Z-I90zEdXV_ixs0okHU zB|*)vAql&i+7Cz(EW2tDcrk|2lEp5cw6(3fgL6)GWe)13q`-Hl_dD9iDK?UUI|@sx zRZg5E;cK079B#~C3-7dEEN~pJCjRiDnZ^rmqI>1+X_l^CBYT_;f{%KjoXIO6O?`fs zj+F0xSy=lBx#^!L__5{3R$5yZp>w396wl>ZJsr~_(KZt!V8gjrLcZJbb!1r8r@WVY zFRRF@yvVQD?yUKL72U(|rgV{&&b4mcrs-Z547Pnj;=cXXoILwmRGwzOTOMoP$vpdU zdrbq^%p=%L$K*LxSc~5(WM!qzt6Ro&4w-k54v);*9%hcvX`25W=fZY$8MWta8~W_$ z;nuNb1J8qG_Kf+}JC`k~UrLudbZqMCW`JCR&TEajsF=!;@P|%h&ysA!! z2K^AzHL61RK$&vEp$|BW|Ze`4)Z-N(L%Rx$9<3if}8cXzTq zkov&nc?!L?ak6PX+&JG$dneZ304h7_=6;6|+~>1llX~-s!{bkpP8QU|WLpzFuGFeb z3k!91ADg!N7VWx;RSW@pQYQDwu4QNnvNy`HXj5Km6-YnGc)}x&^w9^lka^><(OPD< z0gMD8U~^>p2j#HSQKJ>XuV!hrx#z*bF!`lgg+Ju$m3Vo6n^e%$JSDbO{=`~>aPP(M z-X^Elc$0FnP>Ao2sE^2Py2XOGUt_Kn>!csOm}CE;RnO!y8B&rnS*Cr@-=*LoUzX2v z3F6WtnzoUH>#umFk^hkPt4cQu01MM&(sWOWvn0HVam-8`UuAHV-JxWPYqvNk)<&^# zMeCfVqCtGuaR4OPAv82?eQlNLarH`6p1$8GSG|+H+42@Ycb%)6I5F`y zXkeOelhL>OZfXQChBb4H6f7wK diff --git a/summary_results.json b/summary_results.json index db3ec7cce..159984525 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2376, + "Performance_Result": 10.1705, "Performance_Units": "Samples/s" }, { @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.428664, + "Performance_Result": 0.433437, "Performance_Units": "Samples/s" }, { @@ -222,6 +222,42 @@ "Category": "open", "Suite": "edge", "System": "gh_macos-latest_x86", + "Platform": "gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "UsedModel": "bert-99", + "Model": "bert-99", + "Scenario": "Offline", + "Accuracy": "F1: 80.0", + "Nodes": 1, + "Processor": "undefined", + "host_processors_per_node": 1, + "host_processor_core_count": 3, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline", + "Software": "deepsparse v1.8.0", + "operating_system": " (darwin-23.6.0)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", + "ColorKey": "availableMLCommons", + "Performance_Result": 7.68088, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0008", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -252,7 +288,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0008", + "ID": "5.0-0009", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -288,7 +324,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0009", + "ID": "5.0-0010", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -324,7 +360,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0010", + "ID": "5.0-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -360,7 +396,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0011", + "ID": "5.0-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -396,7 +432,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0012", + "ID": "5.0-0013", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -432,7 +468,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0013", + "ID": "5.0-0014", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -468,7 +504,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0014", + "ID": "5.0-0015", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -504,7 +540,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0015", + "ID": "5.0-0016", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -540,7 +576,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0016", + "ID": "5.0-0017", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -576,7 +612,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0017", + "ID": "5.0-0018", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -608,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.1385, + "Performance_Result": 20.6851, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 67a28b7aa018fb7e1847b1c0b48915be5bdb15d0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 17:22:42 +0000 Subject: [PATCH 079/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9546 -> 9545 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d1ff2b0dc..dfaaa7ae8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 13:15:53,171 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-14 13:15:53,171 submission_checker1.py:3270 INFO] --- -[2025-02-14 13:15:53,171 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 17:22:37,327 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-14 17:22:37,327 submission_checker1.py:3270 INFO] --- +[2025-02-14 17:22:37,327 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 13:15:53,171 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-14 17:22:37,327 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-14 13:15:53,171 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 17:22:37,327 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 13:15:53,171 submission_checker1.py:3286 INFO] --- -[2025-02-14 13:15:53,171 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-14 13:15:53,171 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 13:15:53,171 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-14 13:15:53,171 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 13:15:53,171 submission_checker1.py:3307 INFO] --- -[2025-02-14 13:15:53,171 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 13:15:54,267 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 17:22:37,327 submission_checker1.py:3286 INFO] --- +[2025-02-14 17:22:37,327 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-14 17:22:37,327 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 17:22:37,327 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-14 17:22:37,327 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 17:22:37,327 submission_checker1.py:3307 INFO] --- +[2025-02-14 17:22:37,327 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 17:22:38,342 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6851 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1705 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c00f282b8..8b8e485dc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -706 +707 diff --git a/summary.csv b/summary.csv index 1f94b207e..f1c6566bb 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6851,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1705,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 47a04094d..33eedc2d5 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.6851, + "Result":20.9922, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 577c137a2564ca094074d41c2d55565a8852f1bb..3aac8aeccb44c5a1ec46977cb4c55ad57fbc6e7a 100644 GIT binary patch delta 1586 zcmV-22F>}(O36yF*9QeVca1(!liUYp0%|R@LJ<^4M!wb0Y$Q`g4X)r8cHZ>bB&NuyMU+;6Gb;1FBy$bfWO-4{@XzFWkxKp z>?<|PFUKTRy?bGJ`K0)l;jARXJ?Ok%b!>^JYdD_ic{`xwcifK!?y%c}LWvs3rBI>< zp=2}we& zSjsXinF;{Qk_uYugD+Upk~Y@}U`eAoOcZRUe=-{1p;*fQ11zh^ai9IwRmawMlhSf_ zVaY!mJr>L(M~h;K8po$tq6V>KG=L>)5KGh`mR1>-Oa*{tNd>L-!51uPNt?@r-dRFa zhlxTgQG-}oZyHOosq@!~tRlU=h_tqe^u)#^z-=P=XZXhgpNrcKN+fDrlM>080f0zG ze*;9K28l$CNB3LT8kR_=5&FEbtb*41;0uwoq|Nn`KC*_W4ikmzS4QJfA{*5bIls%N z!TAbFf@5|+SP{U&P-3nbGB09jH&e`|g41xQ-b=6VTJB#r7YQ3xcXfg4Ku4FUO2 z?7&u%;9gAH8%#>eFimTR^Gob7;|TP5*wl@~Xk2Q?VKm6(9)=u7gG@4(7htl?lfyLv zOqRKF1lV$*q|Nmbm;`4I6NSrG#+w7n)+)_8{|b|D)v6^#a z0F`A19%2zg&)Io>qc)zd*!1xf zOIXj}VJ8><4ldCmVpAgzqj9N|htVLHdw6*m4RXo2Wq`{vI}g_ga9L*P5#Z>7k~Y^% z;1WzdOcX9*8Cwr5VJr6a{5xE}QFqUGv1#uqEaweZIy{)tavBPzEOYg66;LqCDriY?^*~9RYlPlgLR5!|0x%g3fGI0>^}Gh=-+d;N z91vrF_a)uwsc(G2eWH?~I#uIws)cbrT+7IH>GE6uU2$-`{DS<8JZ~?a_M^hnAB>k@ z=g*?)P)q+<{!oBZ{7`^dR2B}p^Rd#hrT#Bzp(0sb>pu>bKM|nKk|<81>}lD0 z@f!v6ad0$=`|lT>nopxY=IZ^s@SSt;E_6Oo{G!PZT2sqYf0pLiEm|pFaNRmOoCQO5 z5hTNCnrdS;$P+b<{PsIp{^-Esbr#PSuWVhri?b}AJii;Mps$kRouzAoIL@Bm6hpH9 z@rI*60kcpMCJqE>VwZxGdLJKu%TB{E5JmTVMdW?rBn67BB$WVlMI|JHK*Fy&)Jowg*XF5*#XsTWuxvp|m@HY$*GtJHUCG z-eU=#a_T8Q0b?@}(SYP^LYg1jbO6pUG$28FhqENZXRik<-E2drTsWohM+39gQJmZd zCnjUtwn@9_tRa7fKQ}LL-Kw!rr&BRNRlyly7O2;@D)40y4%blUH{rxv0sQe8^qIF| zk-iUj6atVw?x=h$o}TL^HLCJ7zl+m6&Ti{$N%EX5mS3llSN&(89ESXxI0E|I*Po#J z1GBs!$^r#Dca1(!lZzu;0Z)_XBRm1wlO!ZT0Th#FBsu|xlc6Lg8)#yef+Ya}08jz| k01*HH00000000000DysKlj0;k0lSkSB_jrHA^-pY0PdLAF#rGn delta 1587 zcmV-32F&@%O3F&G*9QgomAr0IliUYp0#YxtL_(A@_}^j0R}rG!#ch z13WSsk7&-;8jeV=0*Yi=1+DeLHIz`&<{BZRb^%cxCW=0Ayks;!0sd;E`)>oyml?6Z zvai%EzZ{cP_3nk?<&)xHhO?3k_n`B7)v+a>uHks5=k0)!f8u^D@CmyuD3qviTnZ&> z5K2Y^e^4?{`~gbTAe5pEN~QvUvZR8RpnD4EP}1faaVeBEs>4LVY{_Zpbh**}w+Q9N zTv<>lO+)+umR01q&!oBPm{>jMHEh?~g(bh|bS!Wi$CeaJ)Hw8kJ)=P^84X~G8pIMc zh@~vUlBoc&EUBQiKKOzqEopO&0G2eW!$iTue^f@}I}}U#e}H8bIqvg(b=9%;-K4af zU0CvaqsM~TaJMAAG@*mbAG{=$$1* zb(ko`5;cgW^`@~T4|V=JkyWI(7m?OBk)GJt0^BB&-@`u^xG!!uD3PdfO-dwV1^^-% ze+>|c8YB`m9(~@r*04k}jnMtZvI<)3gD*tVk~Y^%x@8Sf9VQCbuZ+f}L^i4=a(fR8v^p5 zcmi8Vf_pJ(Z!jq>!!)fO&M&dUj3dzJVN*8_qj9MnhtVLDdl+&U4Km4CUVzCmPY%}z zFj?lx5n#)Kk~Y^%U=o};OcX9#8E+0OTdOqZ{3}epRjZC5ULEPU_O6xdmRZMqe}NLM z9yWFBFdCQIbr=moxrbqg(IAwJ2B0kS>~IwTlx40RK}|L1P|{`^!R-(*=P*&Y9m3di zU^}E@e-1+V^Y=sEsBh<5Y#MhQL;Cj4Cp_K`D$)00Q|k_+ajAEQ(IAz3xOW&0Qpq@I zfXXrh57!7#S!Ur8)M0ZDC2g*kfAHWHoH|StE?*f9EMF@&>%2zg&w29tMr}M@vFYO} zmav|`!%i;z9bBSC#HL0bM&nW^52HaY_we#C8sw63%K(>Ub{?(~;IhonBf!xEC2g*k zz$KV^m?&JrGPWLA!dC3-`FFT{qwb#XV$Vc9r*9g70gs2V^1z<8708>`%>Uj;!zq==s z91vrF@iq){OC0MPUvQtOWT;Nnc${isoDbJBa$UOo)_+$V+%CT$|02)Zi>Ljl@bm}c z<=6SMXgbu=KbAif;1oX;U>22ygYJB+wCri7n!0+NIXtN}WlPG&y zwqE>3!F(JXP2&FhMW^P|=#RO2|1Nyz9J~usoe#fg@`KjY^3?SPk+-O(VbkPL@A9uy~!tv&Ac0*Y4shizmf6Z#cFc5_AeF~$yEz7P;5w-(u$*F-tQ(AHp(k2nL zevs5Ue)}p;;v|q$kE@yec6L!c+o>-31?Q#F4I){N642ZTsW%P!SkBW&l=#53V4cy> zpaDFZwRbhQlp6Mef=@{M`LKv5UP};9L)34) z9Tz$Dfk-z$Wh6QQhm)WrCL3>-FaRY1002+| l000pH0000000000004l2Xp`Y2J^{RwASELPZXy5x006o)<~9HT diff --git a/summary_results.json b/summary_results.json index 159984525..c130a1204 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.6851, + "Performance_Result": 20.9922, "Performance_Units": "Samples/s" } ] \ No newline at end of file From c0123d110d6eb806c96a331a852586f5080769c7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 17:24:11 +0000 Subject: [PATCH 080/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9545 -> 9549 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index dfaaa7ae8..7e5468965 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 17:22:37,327 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-14 17:22:37,327 submission_checker1.py:3270 INFO] --- -[2025-02-14 17:22:37,327 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 17:24:06,333 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-14 17:24:06,333 submission_checker1.py:3270 INFO] --- +[2025-02-14 17:24:06,333 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 17:22:37,327 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-14 17:24:06,333 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-14 17:22:37,327 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 17:24:06,333 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 17:22:37,327 submission_checker1.py:3286 INFO] --- -[2025-02-14 17:22:37,327 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-14 17:22:37,327 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 17:22:37,327 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-14 17:22:37,327 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 17:22:37,327 submission_checker1.py:3307 INFO] --- -[2025-02-14 17:22:37,327 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 17:22:38,342 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 17:24:06,333 submission_checker1.py:3286 INFO] --- +[2025-02-14 17:24:06,333 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-14 17:24:06,333 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 17:24:06,333 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-14 17:24:06,333 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 17:24:06,334 submission_checker1.py:3307 INFO] --- +[2025-02-14 17:24:06,334 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 17:24:07,458 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,10 +32,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1705 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9695 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433437 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 8b8e485dc..9ed20bf29 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -707 +708 diff --git a/summary.csv b/summary.csv index f1c6566bb..2406136fe 100644 --- a/summary.csv +++ b/summary.csv @@ -10,10 +10,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1705,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.9695,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.61982,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433437,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 33eedc2d5..099c489d9 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1705, - "Accuracy":"acc: 75.800", + "Result":9.9695, + "Accuracy":"acc: 74.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -429,7 +429,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433437, + "Result":0.432318, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 3aac8aeccb44c5a1ec46977cb4c55ad57fbc6e7a..99db1a676cdd04cce6b97a70d57f24124f31eb6a 100644 GIT binary patch delta 3315 zcmYk9c{~)}yT@l_i!U;{ma&_~HnKD%yT-mmmcC@ql0Dg%%m`tO!ps;;WDv4v56Lpt zgoK2WEz8*VBwXFs?{)8e{yOJ%KCkzAKF@QWKTf~Fpuw9720E=0yPM{OZwv|;@{0i( zS3=k2p0C0U#Y0zNIC_gq38}Ht=V)j<_u*8Z5&XJ;4uOH;o>$0`?`nZf1zm5O+HKLP z)%9IrcOhQvOqEA6&C{B$ z_Elc;Xdv360^VU|eA<<;b-h zOc%))_vAM>9j+v+u*~4T{LX0kNxx#YX!u^}6}R5q$8}aLyuY!Eo~Q7q_C5mA)9+cv zQ8fqm2F#mq`9Jn7|2=l#Kcz552kCk3R(A|!iQy*-MvEc`;G2PAUwC`my_s&;EjTzhA zJ3bE0dL3jGG)}FF7d#z%D9}3+!zpc}w}eK!b)^y~-_VAwHojBbvI(;L6mjiSjmqvw z%fm0v%Z|djy}Cod{>IF{8265iyPEQA!$s!!En=)%MoZ*MUjA+94<78-G~4T-Cdnoz zsEdj(kA5U%QopgM*d~TaPdDQA8@+}qiAo2{Z|_RqN|~*@+*`h0fl}ew5|E%G$0uMD zRffK%_X#e{l9T;k?$@eTlW86_V$Chywj^>&ytC`pwP1|y6xF#^iFq@e6q}uOUFVkR zuTDK`3%=^_RskZsyp)}qQoSJql6%XQ6r1GB1;Zz8!XUy05#4%yD`;n^{F!I%aoJ$c zKw8q2xPkZQXPgF5p8Dhq?UX2StnYQC4i*JgiXh44COBkpuB6?)GJ{q-R6*s8Mzbo+ z;@9@0djd1fF&g0aT3B6aQaYOLg~M6ILCM7Nlq#mp_8-`!r#nZ{LU#_k)+jX)f12Ya znX^7_+f6|$@|G&vRF$Ah@pZX~%_1kfbR_<))iq-rdDpY4uXM?)gsNR!|2)|eFXe=% zhD(`Gj(uZW+B1-Gu*jcWm$m=+*<04qy6m?mk~at(o)SD z+%2sqNaq`?6GxV9U7lE{F|V421D@(jITY0LEKhOnj#zbnuNyNAzz@m$lt)W;hAvW- z1DMFpo%#|E%$?KRn;^I0MoPb~dAvssXw_6Ca}O2%7z;|Sd;F8Ax9I-uDCSRa5&nB? zIi|EUNlswd#Djxj;j9(uO(~~k7G7gWvC@%XNC{o=PPGbQ8J?MUXV6M#80o%hd__qc zYkaMZ(*U5Qb4yd}a#Ww5Sb{Gw!AOy=TIk)gIY7P*Kx9;jZlq91ZaE$qc5x>(=;zs1 z)L&p!=-Q?cR^HHHctL+Z6r`#$N{dM`Cw^U|@yw&ubAb}IhybkEBR#1H_CPH6rWxqM zioo1rHfLZ4y_h$$2q!Dh>fr>9?Uk&u{aS>+0!zcTmCudTV-Bf`i+5~Egj4Y6Z{h{j{gKYN5x}$TxcNT!C|#We(BPyFg}onRsga5(CAZ#u z`vhd5WD9W_yCzgRpf$qMYpAopK3mn$e6H~6aVC2Jg27<0c0l&F#koDVuY&lQtD<55 zHbALrg5PFtmAk>KIbxHWu5AG=;urrN#9=V@mGvn|o)K066wBntL|rTk#iH>g05ak= zS!(6_(*Utmzh{aZ1{m8g#7VEKSL0L*G!Z;3j#zLPNIszu(Rl%Xa{+I?`70VGbpZ#EkQYWe z+$0Kiyb}xmsJGW7@v%)h*oAlp+z|IVR6`s;C=iXtge_VXSmpN|ijiLbKb~s^qhS$E zdks1IaA}sARQHPFdTR?Id;t{xtMhsYHmoVxo4_IKZbL@)V?3M7G2IWFB4eFeTrYPiMhYnYBO@&L*=c19rhzzXK4o zU-DJ5JJ$52LE1@|VEmQo?NG&l3^CgVxfwpI29n3n@xK|5%kJ`PzEk=fTplSo!C(9Q zmlL5!>0w@<$F&-8T>&3}Aj08XoXR92YCwB{DB z^t`;3j50V}79y&jhxPFuY}n#_7M}a)4NtXD7B=%hk%FA{n=;_|@DOemv#_X$_z|JB zq!{|w2)XTx-lu7E3TW~xESr9w3%|@;H9O{%?|1yc-SPNdjrFp;1ZHSxjkkVhzpDY0 zEb7qNJR_jow0-D@TJa#WJ&n^f8dDDEkSJyMr!IVZ#3>O>8QpppT)Y{bB~ z_B82^)C=V0fv@3?mu+gQmVeCcK5LG6kMoHz@x584{CTN^2xlvD;Zq(Pr?~S`sZK}$ zyg1KM4h$fSn##DQmc^GvMO#~FVr-(z82%34{tPI;*@z?RqK8J-HD$~4GUuDuQ_Xgj zWA!~Y1-k^+CFJ4NKfKH5&z(p3wKB#j#5RNlawS~01q?YJLm;41v ztA2oHglC9Zb@_vv6VhNW!tI^q_?HwD1)GZzutwCEO8@DPgh9;tOVQQaCP8b=o96$O zY++~&$8u^-`c;VBC@gz;dek+tL1o^R-Ysu+S6~bDY$`Q?ow%9S#MZy+^$QI&$goNsaBLiGDVDawAga(_#p=Zjt&5`%t$NOXFSCq z#!f;yluuUd98EF5vNdDRSp|bty zO8agQcrd*RVb_BDjLcGN`$qN`@G`9v()kS zRQ@>-Np9>@x;7B6*1^+KW+CiGM~06qV+yauRh~6=oJv>v^vXxr9&H9VI86&YkM6Nu z4;j7TR$8LXX1!YP($=_1OXh%8(3U^+fpmRuDJGo@PQNUV8>;^2B;aRj;y|?-nmeV^70|gxz1Ur9K-#qOySiv|F zJWwtnDs~R=kO8Hl?o*H`KjpqISE$F*WAPbn^Xoy{2W8Tl>UQX%;yslAKe`ef8YHfN zyXr-tep`dphq`TqiXXX?hQVkL>9BQmmkEri3ao`|RX}tH#}SVcegvW5K1J;*9^d^$K!^!4>|fr;JYqdvZGDq3c~FD2q`lx*&URUwYWk7lW!6ldP* z+i5d3yhKaTfP!n~xKNCA0i5$oa6-7C0A}zuMdnKEmWuXb#5`1r!&+YgNW~iH(cXDf zyYAz~e%`0>Dl8(q&e%p=^!ZxmX>jn*`(Y`@`^;Z1R~Qgs;)xovgHNZ@RtoI9lg0dB zO5EnHAZzg})|GEExiOA%&Xg-SwgR~5Qv72xK^v_%Xq|fxg-Q&gQFM>WDqg7(gE8?e zu0H}96ir4M5FO^XRl8MX6d>$5DcR5T_MBe;+j-*Xp<-KpB$r?F2abg=yt#>z#T@Ui z%Xp~b3QlIb+`Xq2JRZczFAi_Jo&HuU2Q!c7CIU;Dh8Sx#S{p2%RN?&I#woS2iq0_2 zn%I33ow<$UJBiX22HfmXmCUVOPKSp-EmaxZT0kb(hUG}J=`2w}e-Shp*QIGX^+^y-^<@nQ$v$<&gFzIkk8wDksA;yWBvIoa%e$YB zqFJgR9rmOASdovGs6kUd76+%lNuk$|4i|0TM(QEmc&!zw+{N+l#hb7FrfwF}I%q#H zx^b0geKiQ?b$?cB7!4 z_h;QH$#+UZAI$AObWzbDlJAT}Y)h^giQJalFcLBzw~VB*mTaYK@#$Jn0qJNL6*kHC zRmJtIiknvzw`Gd0OdEWr4TvY>-9<%%ruw?%`gO?-N*t1GX(e0P+HAJRHdNqlH*tar zZ(DJS3V&Pi5fy@<@QIO-e=F=WBO#~ejFAu&zA%!;w&W`#A*ZIrNXWT4XC!27+l+(^ z?gb+uquW7=B%^!DNXY13F%mMm-x!I-Uu_P5wK;gXl2JQBi9@<^iV{gi?IR;0qxK0U z78$kAC~-+cXU`Jm6Y&ev0=?2o!duVBo@c0YTF}M@JaV00(3lQ$e+o#$mniY+#q!ec z!O@rhUVZAezjn^wFBU^-z8V@v*=T-egz=<48Rvf|`AR2$HB-r;7flB$$=7SDm#H-C zrSrQj|b7i6HO30L5V{yzf+U|h>V1Sh!Oyikx&r5 z5)hei3Svok3&R0KuLMM!K1u4z``-bvid6Oh@z&d#5-nRw-{(p6 zc@lj*(*%$cf0Q`n0y;$rkjO|Vi6{XQQR0$X&ioP*G2EjhmV~!_!pS);Xfq8^*l0`# zV|;qk!bk{@SAMmyO@c5^YvW*1!=vZ?>@{f>>Fohv*WUopHyq)6(tQ3WcoK!lqqrz1 z`!%Gw6Gr2Z`^Hm712l3PiX)=|9vO{CG-qoKMbB&NuyMU+; z6Gb;1FBy$bfWO-4{@XzFWkxKp>?<|PFUKTRy?bGJ`K0)l;jARXJ?Ok%b!>^JYdD_i zc{`xwcifK!?y%c}LWvs3rBI>rf?1=ZLSfQ zLP?`Ke@qn2mYjx8mmA%Gi%@RNl?9d3G{g^JSw)WfOq#2XiPb)@VY}8YEcs_n#{wVY z*pgz28i#J!GaAH_(EygHK`c>&SjsXinF;{Qk_uYugD+Upk~Y@}U`eAoOcZRUG8*5Z zSjztcEUUKG=L>)5KGh`mR1>- zOa*{tNd>L-!51uPNt?@r-dRFahlxTgQG-}oZyHOosq@!~tRlU=h_tqe^u)#^z-=P= zXZXhgpNrcKN+fDrlM>080f0zG14N<*iA0S@_gmK*mPn=%`n<8Mg4X)r3z4*>&GnK# zf3k+C4ikmzS4QJfA{*5bIls%N!TAbFf@5|+SP{U&P-3nbGB09jH&YklwqNLte7 zdI?h`jp{H_2qdF{8%p~P0r^ktz*dsre_l-58%#>eFimTR^Gob7;|TP5*wl@~Xk2Q? zVKm6(9)=u7gG@4(7htl?lfyLvOqRKF1lV$*q|Nmbm;`4I6NSrG#+w7n)+)_8{|b|D z)v6^#a0F`A19<4ldCmVpAgzqj9N|htVLHdw6*m4RXo2 zWq`{vI}g_ga9L*P5#Z>7k~Y^%;1WzdOcX9*8Cwr5VJr6a{5xE}QFqUGv1#uqEaweZ zIy{)t!F^J&Xpx^}Gh=-+d-hNq+kgPy%gA-<@>~C1ad5l*g8Yj- zZ!ezqqr%f4jF(^M&!Xv2OaEB@P=HhXP=Hxf77n`evC^`qnQH3lah9KixU$opbOmbUyr|P00^hQ_E9-mgdPeQSb<_6sJKSDrQRdLm=ytU;=)v6{-0Q)SEp~X&gNHph(zwNKl-M2K z(-Ume6LxryUb{!H-J{p;(QEhUwUJ&(HqdSuyYknIJl-PfxGW#~1ZV87QiYfmfSN>~ zmw&n#y87P+_rJKd9oOt}-Mi0yg>8E{3L0~<-%a0`siqb^WTn@V{ocA0fKrsS60p9t zjSzS5o?8BYY$2X&z1Ko!nZl^1;*#U!EYt|?Yq@tlDEZ!zDAfG8O}R1i9hn6 zY0G!By(5pGUj{GHeS95!2;;}*^b;$ggMUq?4UygPbbXoJE$f&T_e-2~42vCI;H8=K zrFa)j&D0guuf7ERbon`-&JW2k9H|5YSzcTb*NTw9y*J0&AN@|Yet%{!+4-ftl``B_x7C>@0iQTEq|8hO~c=n^#*YE0*JX=gggvi`79k(Jxph zv?@`Sq$mQ#G#6@DqW5|oKcL8YsyJ=5f)X9Up;dKLFhiKO@M^UI>jk(dfRrQ5lxXk0 zAs9Qh2T5HL94d%gZ6)=gv^#%nDEp>6zM1?}V>1!afaGjKnjhPA0M0Np zAVGPDvn0c3uLmpLY(uA9IHmAM1GCmqoZJT|CS%*SNxSH*A%BKHH!pA9s2!v%et90tGvFjXqG5jw4$EPm}5+A_OQ6 zWmc0ABo+bLlP4rW0Th#IBsu|xlcXdj1!!WIf+dsBBozT?ljS5n0lSkVB_jrHA^-pY E0O|xRumAu6 diff --git a/summary_results.json b/summary_results.json index c130a1204..1949e4702 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 74.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1705, + "Performance_Result": 9.9695, "Performance_Units": "Samples/s" }, { @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433437, + "Performance_Result": 0.432318, "Performance_Units": "Samples/s" }, { From b7452fd5ef36b91ea2b151165e32661c821af369 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 17:24:54 +0000 Subject: [PATCH 081/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9549 -> 9549 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7e5468965..d498d5a5d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 17:24:06,333 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-14 17:24:06,333 submission_checker1.py:3270 INFO] --- -[2025-02-14 17:24:06,333 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 17:24:49,011 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-14 17:24:49,011 submission_checker1.py:3270 INFO] --- +[2025-02-14 17:24:49,011 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 17:24:06,333 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-14 17:24:49,011 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-14 17:24:06,333 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 17:24:49,011 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 17:24:06,333 submission_checker1.py:3286 INFO] --- -[2025-02-14 17:24:06,333 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-14 17:24:06,333 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 17:24:06,333 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-14 17:24:06,333 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 17:24:06,334 submission_checker1.py:3307 INFO] --- -[2025-02-14 17:24:06,334 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 17:24:07,458 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 17:24:49,011 submission_checker1.py:3286 INFO] --- +[2025-02-14 17:24:49,011 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-14 17:24:49,011 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 17:24:49,011 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-14 17:24:49,011 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 17:24:49,011 submission_checker1.py:3307 INFO] --- +[2025-02-14 17:24:49,011 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 17:24:50,094 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 9ed20bf29..ff18e5227 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -708 +709 diff --git a/summary.xlsx b/summary.xlsx index 99db1a676cdd04cce6b97a70d57f24124f31eb6a..c1f84c734269118592d58e1e48c25a1467da8edd 100644 GIT binary patch delta 324 zcmV-K0lWUqO3g~Jcpn5qRAPyfd>3ZuO(+i_pQcAza;HBe|uN_Rq< zIHJ}ANv)H=uVN=o0$z1Fx_8dp8C1@9vWQebdwFea^aN1?+r{^ zPvYp_IWZc$u1mUEZw>h~{JDI6?^lh5I-H6Ds!PrYvp~JJb%`&FaJZH_zYHhd2;fiq zpijIV7a971MA=C^U0$62w;?nz#d;_>S+@~Zy~l*b|eCY}I8 W?(0ub|Fgaz3zVy_cI9GukWKUp)(NdjlqD&OKrzjQ+Lh>|Ud4|na-J$q z8?B&39XPbC?h0lI(-z*WHekH~7X^@VgqafUy*C77$MztpOM*iMajUJQK9qKUhYe-l zbO$(3(+4cUQ%*g_Juo&S5uK2njY#ujn*qQXh6W@k?{Jo6_~P|orJHORlq;tczB4ds zJ&U9J;KXQb+cs%ugEi#O@Mit`KCBuG)t`z1stV2svp~JJRe`UIaJYs#zYZtf3gAy& z&?nwbiyZraMEf#&dDDLZ%F~d46VHHS W?%PjL{j Date: Fri, 14 Feb 2025 22:02:54 +0000 Subject: [PATCH 082/719] Updated results summary [skip ci] --- README.md | 66 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 4 +-- summary.json | 14 ++++----- summary.xlsx | Bin 9549 -> 9567 bytes summary_results.json | 14 ++++----- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index d498d5a5d..b8bf75ea8 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,41 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 17:24:49,011 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-14 17:24:49,011 submission_checker1.py:3270 INFO] --- -[2025-02-14 17:24:49,011 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 22:02:48,748 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-14 22:02:48,748 submission_checker1.py:3270 INFO] --- +[2025-02-14 22:02:48,748 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 17:24:49,011 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-14 22:02:48,748 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-14 17:24:49,011 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 22:02:48,748 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 17:24:49,011 submission_checker1.py:3286 INFO] --- -[2025-02-14 17:24:49,011 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-14 17:24:49,011 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 17:24:49,011 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-14 17:24:49,011 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 17:24:49,011 submission_checker1.py:3307 INFO] --- -[2025-02-14 17:24:49,011 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 17:24:50,094 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 22:02:48,748 submission_checker1.py:3286 INFO] --- +[2025-02-14 22:02:48,748 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-14 22:02:48,748 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 22:02:48,748 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-14 22:02:48,748 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 22:02:48,748 submission_checker1.py:3307 INFO] --- +[2025-02-14 22:02:48,748 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 22:02:49,857 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9695 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352299 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9695 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352822 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index ff18e5227..923475f93 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -709 +710 diff --git a/summary.csv b/summary.csv index 2406136fe..670ad5a36 100644 --- a/summary.csv +++ b/summary.csv @@ -14,6 +14,6 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.61982,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352299,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352822,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 099c489d9..2f22d4454 100644 --- a/summary.json +++ b/summary.json @@ -462,11 +462,11 @@ "Result":48.9647, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":16, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, + "accelerators_per_node":1, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", @@ -489,14 +489,14 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.352299, + "Result":0.352822, "Accuracy":"CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":16, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, + "accelerators_per_node":1, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", diff --git a/summary.xlsx b/summary.xlsx index c1f84c734269118592d58e1e48c25a1467da8edd..b029718bfc99bb3b8e73b307f487fddcd10a48e3 100644 GIT binary patch delta 3817 zcmVp;l|b{;!Xdv39SLG>WGOx-d*#Q`0c+N7Lbfetq@1;p$qN z1=D^oj;HECe^ROb>*0U)AL8Wqbfi?KSG+vCu*84D@oKi$nx~lQ8Y{S z!~K4gA1nH!C2DY>ABnw_-=xr+hx?1RZzA=OZoJltuH41(@5P(1{R7=Bq)pI%UZioE zXni#Z=HsjvKYUTqaFpf4W)&fSVLVP3f3-=JkEPVLN$_}}w{-0x>Sv>(Vr>-l`>Ggd zT?^-F7EgX#9ulhxmZ8F4#R^lbqdh~kOT+ zpqLr4CuKe?VNv?6C@ykqt_ph31zO&ZV)X70Tl>xXqT-d}I}N#_1=qBH;AW$sndh@k zO7clb$idvsp@)h#k$h()Vq0>>NaVKUnvsz4xM3viwPdTiExx-gsDLEeLxn|p`>NvV zRmJtIikmXUR!`e}Pumbr#=D1#Htp)`lB?Gx*C??`XG<&D>aE50_Q-+?ob4u#QQ>YY zPEg@(D?Xw^5EMQ!5^{!reP$$NYEBsmLE#G{X>UuuG7>U19Y#Xt=8Tb$vGo}V8QgP5 zLPocX5=ln)f{~EXy<{Y0biXl@7Js(c{MlyX=}JcJ7$r7I;{+v=jM_&=LPqTqN?K&p zKBL4T4V^wqm`}tnObhf(D+%vBAA7z+ozVgx8*s^Rp3|5vGX_WC|`{@*W(1 z`S0bYp8vIb_I|M#O7qpwFv>>rJ0pxI&B-|bJIPl%`Ky^q2Dj04pptyOrfxHpX1D45 zZW5)%(kH#Ww+yqHB}y$8SzZ%F*{l)H=8bqdeN5)l zEMH7DR;t@Do(`gaq5g~BS6L8^Q~kq!bHH4#ztz#De9@}%)E58k$m{zuFR@%=810s9 zZG1*-LFe$$IXL6=Gae;2`Pe)`2|&k4D0Cz# zxX&a{!p|K_Br`uG;|r8{L~`lWNaS#I8-&H}Oe4R0Syt7580I|Ap0YRzN4NLVunn^j z&V_7OAhC|FR3P!%*&wm)8lJtu(H4j<52A|`O%OUpiA^pUCny0B83_duB>*BLp&+^? zAYwRxSQ6f0!YPO);Xac*1<|D-GV?0*pW*P)DrOu?mBFKm+_jMK(ASk&<7+CO_uT1WRP0NC?30Q3x7c&;>`{|TN%Ve%+0 z%E^8WY3`WO*yIE|VKhJ^r=d798sL%9xI}Zh)^J3BaurY{%PQ!s53Zqv5}#{?Ed0+Q zs>?*t4aW;cLml zUDMrh$0+z6_alKj?3SQVqQ-V8l&C={84W|I#$&qj}b1oOzTMX^MU?NKaIgIF>ez!EixC2A1M zRvDH|1%PEq1)cT57c6Os&-H|E<3LoGi9#$o6LGvVviU(IAwJ2B0kS>~Ix-0F-5} z9YIYsXHeoZjo{-DFy}B)_&9{I=fLBTn*BKl<K!V zdgjKy`!IxE0}s@&8D8$n9K$Rj^c;zDQoF}Ta~q|+%)s#*MH9g3wm?o3+@w@4AqGm zk5es-^Wj=Xu1lBS`tOOoo8=eeU*vgv@w6Wmp8jCG{5pRYO@~_g$MS~)?Ba(4%%ZYz z(3_8ymOahXfvz5B`3x1E)wTZPVEGdP+AN9UB+8zatrx#hFdqkpleqtWk<@$|{V`Ws z@85;*?7erP{oxl)e$WoIyz9@>ymyOMiWgj3hx@Z&sLq3A7)?`ctOj{UO(VbkPL@A9 zuy~!tv&Ab**Y4shizm{CkE?ciNo@>+5R=Id7`fz7yW0t-A*oR<3N z^v#{YxBEOJ8z@D}*-*0$OC!J{9;a+Q)UK~i^u9)fq%5WxXE4+rKxp4azYT;`PDjejvic|Tud2-A%UVeuMB(b{9 zQ-3L;Y#NR?G|lL3nxzt7dJq!wCVH+yh~B&?c?3dmBWPJNPl#<+Bk-8!}Haq;W;WT2BZLWjFO&xS$-uqz*+tG zN1!`y=-r?~IGd2^;?FS&0>4LmS0D2dNO$pWLqoPHdBN&pYg^!!7uTm#`-u3q5r0^Y zO~@nQ9FbmNgnkh2fVl!fqvzWJAy*gQr{|Ze#dNv4JU&~@r}b?M)6pn8B7tkDd5B&D zf&qF<2{L80#c&IyY^kwzA^&QD)i_t#)Z;Wss(S3}_jtV0yvU$s(CvDs*MYkoxYvQB zve@B42j1_%k;W}6)H@%)P_+i}f~s(ZKJSCnlBM@}Oy_NVC^BUSgJ zhf3+SWWSE?7@!aZEd{JLw$aDayQ7!?AKRX+n$EEd`*Y>JituffcYyKbZ-4Mx$$t56 zHJIckf7DkCvqQsqXC(1Q{xi$+-E8lOgrb+ckhmHMF=qhC9J%qFvA zatbFZCqR~0H^j9<6qCI&!GGa;Y7w$+kr(cgU5Pc1JdG@}8(GMnkX;hnH&C3!{CIIh z+|cksv=cq8%00($y#C&*>>IxC2mfu#j$?Rz%O~V?dcK-Z&lTdMO=>)!EmoJ~i=|qh zZQtkxeP4NXJj)H-ut&(rZ1QAw>Rr4S_EpLpEA;xv*LKrZO@TM<6Bza2e#>(Hs8d}D zJrvYOZ}pvhZ>A7(^e?j)7hwnmN>bjzC6j$09)HbF!!Qtp?>t51d*Z|`2&^QP0QE#A zB#J=XEPI<;#6PkPY2O|vY13BfiQ{@^znxvVTrNPEg{g|;DBim~TgP|Ro1p`*AlPO%V6 z*MDHk_%~|+lRUq}3IY=>Fx&%EI}yT(us>$-VP(GIm{;b+f@5^B4GY_3}2Xno8ZDiUX=k!6|pZf^l_;&x?4tmie%X zC*DX9kNc=!c{?mJ_JK%Z06NA!Dj&0_=Oy(L)g&jkX-?8&z9xj`gw7wn`jHp?C!jhE z`8V+h7;|5Kg8C1$zaYv21av4`Jd=$he+ijWpeh^$002&t6(myuDwAm>Isu%Mq$DN< fN>bjzC6mu26#;#d8O3g~J*9Qbdsa-jf+Xp&-FlCd(3C7qVwx`Y$C5r}B1Bj7?FaLfdz!>)0 z$V?_Pa-EY02U=@)_ipKROWe zvw=T}W~!k-sYL&^`Jdy5DE>VeE0tk3w(!d}M;Q*5F=L)1-MTKV~Sny7V5b=B&!hadgaZyv|dgBCAaNalWi zX2g+{*{}q8>9@Q%?^|_U(0eY>vOMz9yKmNxtM_@uYsGgOa!m_vXu<7&RzWrE&!$t7 z@05f-n7e)Gp`uPC-x-P6m0U9txhuJ0BxF2p8A*L3*-5v?r(1&xNJo39ut>JADz0Bu z+`Ou|EmG`cTIVyZLp&Mp9xCcI)z>B0uS;%FVv}SGE7{4`VzWK5paN&NiBnX#yNVVn zyj{gdR0x8?Cq_c9u+NNtgq)f)MnX{d!bs}7lCO+}oSHTxA?N0tk&vjM^zmY|@PuN+cPzkBo$j+9#CM$f$ir zi9;GXdzLVth+mi%=#^Fw-hMvzJVTw+f(|y|lI!$>#&nreKpMV(M2SZ)mY04Hn_vEW z^{LnS+C6{2Tnr`IYG@Ru+mML+9Wbr=RgCvB}3~3nc&@Bcaft1fXLi=};n>^Fw-ki4u>reC3o#j}g zA?pW_SVk%bka**4kyv*P&)(u_2Sk?#(Zv%@5IRMPO)kF{N&rMgLP10cfXGNFh;9Lh zOgIIxAiRy?0HRv}BGWnr(WM|V=Z7?Ti4u<_b>;r=fLKN<2Y`6%?o5d_OG?k?*t z4aZAH;}PJmwz~f|(0rK@b1Vl+&9lofPE`M17;ZKx{$)5z$?yO=uh$)0;prH*Yr1>x z7zMxMej;#(-4YZ^)YuM%5;X`VqX8(H9e;olH3+3Bf|98Kpe(4ME$B0ab13O>jW`ra z8r5ZgqF}bi%1Va%-;4sg$N6egMlday(?xTz5>Y_jwJ=sqMp(f97-|@G*`h zDVC_Q>4rU{K`a>!V2K*U5;cgWEW(nh0I)2mpuIWxf+a2KaE$<#G^)!)!DcF>@ob8v z{6E05j2sWyUtM=>b2lk#_C756XQL;AdE{7sqgbNG_9&L9K`a>!V2K*U5;cfrtq4n| z0>HAMg7)U%3zoE`!(~G6EFr4PL?M=_K`d);8cVXN^Vf+iBfW!&tnCu%imgY0yF~KO z@J|Fj7q{w^NYpqcC6X}%0FjIah(rw%i5i#gw{A2nkxV1>d1Fxp?ajd#B56s7>m_}E zWDQYWCJNWDjK-lv*2^VwahFYli#3u2$Mk->oGJ4uQh0{^0U*oB?x2-xeS76+Y=HLsEw4}rJ z5~fHR)n%d(NJaxUl-3&p@}Jm&EhWK!gP63on3OfcG;4dDUt*6LN1)HcqHY{U<4`*e zqd_JQFyt^AWRkJG0FyyvdEPqz?K6g9j=$aBsg=JC|tHO-W*uAmTAuUSD1XO zRvkgSI?{HmeJj@;vyS-!C0ac!>egX24z=qr8ieuy!w#cCC>aevS>)N_DgY>di(ET` znrhCWq{B3Vk3+zm!$jfZ5XPPZk3&lK=OC0n|9Qw8_3d1XZR3t@NYC24!{hFu5`7;Q zweBz)hkADy4N`f4dxy~=m5hT1s4Oz@aE$~5STY{?xN;+I4^xhJpx=a*+$!GvfS+c9=H8B6~Gm%pAJA_GT?lNh5 z*=)IoNwJ4X#;`+8J&eYorXEIvOdeqBVKm4jW0wIYi%dOSBfw;lsYimT2TD3zFFnd6 zn0lBfSj2J~x`>q}n|fYjG9BhPiXXbCEG2)pL)NZx+su>A|DGr5PEU2~3+_`Dk5o%d zCW#hA*>J5QH>Im@{rAMt?dl8iFY>ItXf_CQPrpA|eVsoIXCp27WA#G;cK$;FW?oq^ z>@6lrOP}Vdp{vJvc82m~b#3rCT>V6VHjl$74%4Sq>-ldKEGB+)8V%krJGGdFe=J+n z`**=R`{-R@fA~d{AGC&+rT#3ynJf8a_b>YXOaF%EjHOvwk(+92?PKD z82_!I{jSK#EmR`o>_D>(OC!Kyo@8uupj}=Z>7ho1q%5I1XK4&iQ3A< zfp#NhIW$cX-#|fy!AoE&lJcscQvF+POi@-q6Mu07AdB2|EXy|wnlX*Uyk_!1b6iIw zHOu~~VX{S{P#cd#HXh0GDNi6DnQ}Zb8(<5rVqmn&cpJZDnG|hs38Kzr8_jcCP)84} zYdwnE!SgiDGX`z=g8vOnn22FM?YM|iL*`TR6^xBRR659Eon-P`d*y`Zy!rw6NMe0m zWPegZ)eaoq(k!RfS)NI}>0U@E+US`|A-eOT;xP!p(fQ>U#2CkrLq#QCOwr;MPvtLK zq1>Hui7#;}AnANyMOjxgMgpfGm3nl}M4i#bJj2p0%d&fLr;R66Qe`WZakKHTd3rM8 zMZwj3Z*JB!ma0PZ3R0+4fq)*DwO-zs!b0)u9QbR2gj@bCzZs^I3yEOIBa2 zBB=G8s*K6?UFi4ox|p}xP-J$x^-K;=Jna>|fv2tQz*AN_0!RU>7$rUby8K42fV1ZB zuRwR)$h$&DIGvK&;?D^Q0)Ie!SD)|-NWXcvsUh1Zc_!-OXjkHv=a1bTp2KBybJY57A9P zFhF;yKxT|~1g@cy9Uj}1@~;k9wR82FdXlAS-L!rE4!2iYlsR-Xx?SxIdT_S~_j+(- ziya>H;9(DrG;VPlC3c7R^aLCAgnu30qu1`yYxn52d-U2pdTpc^k`1&Q#;*MJB9FJo zIxfqHKEWBgt5hLo1)wI;=cO)&uKu^d{V%R<$2EIg_wI9FVcQ;#g2r6zchfg!s;NZ} zS?RT8zqjrLpcEyo1gvjuBgEair+=wOp+Lu7Y6U0)`5 z%Q~jT{Sqe~!(vAlcxmQ*Dc(g>Gj&Dvt1m%6U4G7|^FwkBM=HTUmKRsVwIU>N@6ECH zN57M;-=En_c7ACudF(%A*MEOdJSDpq=^iKXixh{8A#o$ai_n1cIQI@5!|}qsxz{&* z-w*y9dmYE{Ldz%Qcy_ivnVl)bM`O|CWWHFhCg)2v3){Xi2tr?->UfqLxY2-+qxtkP z>@i0 z1w&L~i6)bfA0B_pYQr!PMfd#*qrENLabLoApe$07eFo&ZDc>rYVs0khE{$^ryL zsa-jfpCf+>>k(+92?PKD8b`;UpCSYLoXQJ^{az NHYFnlaUuW!002jDI=BD; diff --git a/summary_results.json b/summary_results.json index 1949e4702..15f8be250 100644 --- a/summary_results.json +++ b/summary_results.json @@ -120,11 +120,11 @@ "Scenario": "Offline", "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node": 1, - "host_processor_core_count": 24, + "host_processor_core_count": 16, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, + "a#": 1, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline", "Software": "pytorch v2.6.0", "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", @@ -156,11 +156,11 @@ "Scenario": "Offline", "Accuracy": "CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node": 1, - "host_processor_core_count": 24, + "host_processor_core_count": 16, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, + "a#": 1, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline", "Software": "pytorch v2.6.0", "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", @@ -176,7 +176,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 0.352299, + "Performance_Result": 0.352822, "Performance_Units": "Samples/s" }, { From a8d2e48da707cfaa5279860e6538ad5e28ed27e8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Feb 2025 22:13:05 +0000 Subject: [PATCH 083/719] Updated results summary [skip ci] --- README.md | 66 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 4 +-- summary.json | 14 ++++----- summary.xlsx | Bin 9567 -> 9547 bytes summary_results.json | 14 ++++----- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index b8bf75ea8..351825cf1 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,41 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 22:02:48,748 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-14 22:02:48,748 submission_checker1.py:3270 INFO] --- -[2025-02-14 22:02:48,748 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-14 22:13:00,505 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-14 22:13:00,505 submission_checker1.py:3270 INFO] --- +[2025-02-14 22:13:00,505 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 22:02:48,748 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-14 22:13:00,505 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-14 22:02:48,748 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-14 22:13:00,505 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 22:02:48,748 submission_checker1.py:3286 INFO] --- -[2025-02-14 22:02:48,748 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-14 22:02:48,748 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 22:02:48,748 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-14 22:02:48,748 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 22:02:48,748 submission_checker1.py:3307 INFO] --- -[2025-02-14 22:02:48,748 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 22:02:49,857 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-14 22:13:00,505 submission_checker1.py:3286 INFO] --- +[2025-02-14 22:13:00,505 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-14 22:13:00,505 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-14 22:13:00,505 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-14 22:13:00,505 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-14 22:13:00,505 submission_checker1.py:3307 INFO] --- +[2025-02-14 22:13:00,505 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-14 22:13:01,492 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9695 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.9647 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352822 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9695 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1596 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352822 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 923475f93..b4b6dc348 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -710 +711 diff --git a/summary.csv b/summary.csv index 670ad5a36..9328439ef 100644 --- a/summary.csv +++ b/summary.csv @@ -14,6 +14,6 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.61982,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.9647,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352822,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.1596,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352822,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2f22d4454..6f693ee81 100644 --- a/summary.json +++ b/summary.json @@ -459,14 +459,14 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":48.9647, + "Result":49.1596, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, + "accelerators_per_node":2, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", @@ -492,11 +492,11 @@ "Result":0.352822, "Accuracy":"CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, + "accelerators_per_node":2, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", diff --git a/summary.xlsx b/summary.xlsx index b029718bfc99bb3b8e73b307f487fddcd10a48e3..0b7eb708560a95e3080bfbf4e18c2f8b9f54c776 100644 GIT binary patch delta 3843 zcmYk9c|6qL{>O(bAw>3_F{tb&BFosvH@ zvo^A4c;8*>Fp7_N%DS`r3~wmQSNX8uL7$_j2iu>wDxxFo)ml&SDeQ6IblzDE$Vz%F zEzf&Yj>zFHsz?G+`|#QaeEQ5bE#AV}k;mCA`f&Hox#ib0yK^mm_y?M2%`nn#kN#Gx zg$8bFnZ1^M#-rzk6wp*i-DpdVuhz)9W7vvd{pGZNSeW{ZCjH?(Q5UPgK7$3^LR@;< zPLrD`8jTX4Qpxw_zb3@LQQRRq)>_#VWf8DfIrr9}4(svc%Rdx5sFsuR`(Mo+Gws@> zn>7=4F;dkiaa(NPAu*PIeM;<(+4uEp6g zk9DI~`SYPla%^r+ke*i8TarP~tb;78JVEWbfjl8IIW~Ed9V}dYf=(YkuIihsRb-H{ zlJ_i=s54L+mqd*++M#fuNQA&MR=>&T9WB2msZ1y(kiVavzg_AWiR!$9~J zt&JOE;2ClS?ofFAWQ9n-w|4lY=K1J&i(D3U~o(W_!yxAA>Sss%>M z88aRsg%}%F-Co{%rr~H_G`241@OjW%&H`CQ69wf3s9k?^J}QUHWQqNXpC3=i#uW4I%2so)|aQ-{Xqj)LFpcOF)OB*v-h-Lw&)}pV+etN zLSd5N7Z+eH4?;?bgsXWquh*CorSBFc0ry^l0-v!a*};6CgA$WtF$t_fED*(t#m`bg zES>yDE}CK8klfs%80jID^(aS)LxEYt~Gg4r^qxQ>NZcvV9KW_z|g7hI%K5oYCp8I@7$`kj2n z+s^>*ez?49bpMYB-IihF79_b`wb104nE(xzrcPgadWffFj#eU1EM@@o{Bw#1?7RzS z)9f_|nR1waW!cZ}P<|fVtOt10L(pq-X0o<{JSz6S56+~tnr~PwnHv>fivp44165Ij zcA+NKN8wea_A~L2v(KwKv#xvyxTKlu0Ar-y6Ki>%53Dy5AfA_vk+Bd2g%i?L^Ip(4 zwXx0MA`gY6hk0sZOrN>lybYW2F%a#>tt?RAM%ete|OI^9ZrtFc9_RBg_y6-ss4qzb>QyZMb7X2>`Lj zW$vfn`0Kb`n>zG7l&qsSjM(Gg)R=%MVe`UlHqQt*ymo{1w{O*Y*4!!Zhi>&x` za$bNn(p!wO=0d6AV^oVo3}vtNWsrg11y~WyV^prpuwVVa8<;#Alt%qGTgT{&)+%^yputr&}2yN@A5(^A(mvPOzEv`I=C1?XA+|jpV;AceZKFpn^*QyF7ULs(UZ(V ztO83-Og08i$^GDxF)bo@miY4Pxgy@3gCv+`*Jb~6Fm!=e{5ZXr_RSI$z@&A&mml9^Hq(#)rnqTsvHCJHG%Q!mylAVlWaKY%>zsLw z*?&uxrvwx574$-vMgYfBP~X!5F7N!yw3Bh3XU|nJVUu>`W|VX^R{YVTxAn;;m(4wb@!C3IOnE-^ni#99L!u?SAmTm5Xz(t3+G}C>}_(3CW+i)9=+&~M6IyQ zt~|y_j~dUiny8&4ow;q9C7q2A4^D5{wVH)~39a+l>Xlk%wA~CK7(Jmy$fxkHt(_bB z;6RoKgdH-KN04atU1dcU3QEcwiVs)RN!1Ug84iYX4!Kv3R6b{VRcK-bvxDf41?Q; z{{l0m-Tqypq>D16J@FIfwpy)Rc1j@MN9HL+24+Io%c|K2Sn6E@MyVMhe10%&7C!5) ztSKGR*(>&LvP{nzBscy!nOg}tpg!gdA8~;4(lnRNR;`c)zj%eI110}He@6>b$8#trKkH48FLzW9{oX>0qiots&ZadULN4%##1gugZ_ z*y&gWy+pDq#ZBpY@I3vpCH9H8@+M$4iZ^3>{l#xrce zSq-_j{~OoSD`hS+zYc3)GzNcgd4=*0DzJCpTP@U!Es-o#zUN;s_2g@7>KPZEaN_tU z;^Ppon=sAp&+Ej1b8-6TQy6a>Dx1y6pRTCijkhnz##EoV z+Dq-(c>5l-Pl3kgXUrL+Nmhch`bJX%Tzd(z%btwZG*{_y&O`0mMlQxKxBQ|wGH`ih zj{@LYWG24pLCLaF_WvKq%tQ6Yo_6Q;YW8|?2Nm(^^1Kv=3!8cPhNZ1q zSmo=K8J-s)+JLSFAGryeKZX7J=NGsU!vrhRp(P+g9>!FPbRke2%3ni(P ze*DcU&fn>x&l%>i^ziDmE<)jf?m?A|riOh~f7zaP;3fTQ?XV0^DQ&q@2&Q$^Nog}vSB;f#XXVTZ`y_-i&e4RTT@0&-J9?mfT z;f>_)-qs?CFNB4>RnN}5O4N59SU|N}*NQD0-(Z+%mAk0rn70;|!P_(tomPCjFSibT zav($x{t{T-!!UrHs6#6ArS|EsbvSpI{4PNFB&t3ijvsodc2Y*2t<|&B)^Lr}`6AWU zlTs`;Rq#4+lRCCq^abcZ2O=5+o0N>4upw&HGkQ!5C>ah}t|iV#xVPomH`{Gmif|{4pre+-6?v+~XYOhk@uuGf8UL1TCXj^E1 zF~ep3n2Bu<8N4Fln~abo4*Jx&o#yyZaE3>vvQq(K)=ob`CL1O?SK!;+9)R5~5LNfr z7)pLsG46J8KTz2zIZjhK=o)tfgH?@wK*-knT(teD+my?ju2U)i=Qtf_fv8|En~Pq9 z`+wqp<78()wg&s5-@atRe(V<0tZW(Z2vf>p0*5 delta 3880 zcmZ9PcQ71KyT;cFf)G~p&h8SHh>{2^L0BxSM56aB5>cA7#Gqd1e9-uE|g z6RcSg#cAcc4*!fM1lj@aCj42Ri@nCuX2x!a?TIp3U1|*)pPKpd!a{cLVOY!p!FDX;JATlA9~wu>6y%cTKrS z&K(Q7x9iV21_kWWM1<9d(%rlJqdJo62W{s8iQK{`IS@OSOz!X|>|E|}dgy6ntux<2 zoKxd?MA>x5&UQz7`8nwfEEhE#V3zwT^@YFG^_9gE=0|=y1eZk2a+?h!uO(cQ8B?Q{ z7@J3|QTyYSS?9jVyKbA4ZRL3D;PeyA3sdUVmrPm##$2FcDqPk#vkofVk{r3m)hJ`5 z4nKJGm{9&_2aovqE*@&yL+p`)I+I8)J~M^2Qo#Bl%u?s1R8Z8_zKqIeh|Fj<)@pz2 zrt#^(Y4$?Wf-DW@b2iyT^-r6e%+yu9CURFzt-E!4Fe^0TUc8?z*vihW7;pVLnZ>(4q4iX+m+JM^@E_-Xy1)9UJE3?2 zG9g&0W~(mzeZ92f@89lVs`s1F7@c`1VNLCMN8yLrigdRy5AT9p-}&2s(RP}wlnV3I z{4br=zlwvzr8p6aD=Sg)ag_(Gl&sTk2t`C)BuEW$7#tr*XhMV0?#*`-m@u{9&%WC% z{z4))Ss?e+7Oz1PqNY@?w3`e_BQ)`0PCGBnhllDZTJ3iO?Ozm8khH7e~PH_HseSWsUFOfJzqR-q^5MFJe!YKT2SrNBUB> zM_=Z+I2B1=M{v(f@N3FrXlkmvflq}XMVJf+tVG7(hgy0&;mAFR z#*|5^HQVnr=eF-9^n)wWUNZg~E2A)T4Jo}(m>*&pwAon~k+b@SbkVx!Z2@jACS}cI zggigOr(Gh`kbB3Ia0N>FE>%HyL6S8V z$_ENsA3$qj(-6~RBhaknw;q*CL$&%Lu-3OBP9#|35qR zW?w_SF_snhU`CZRrApo5NCgd^e(K2K##*;AgKsqZ|Mn`{|Ptpuhp(VN+k3@ADo zL^w*6RX~u0j+K&(9s=Ow&*Eu-@K{L)8vJv;9R`_yh=r38@_5Ae9+O`JAzbk$4VSV! zYz062d?{^bp#jWuh=IeVb_m ztqA3|R|rXk5V4dX6Ku8i8? zKB2Xen`K!cppW(;u(KiFPkl-Rnl5U@V*GLawCbf}=CP|XtYSl0s!yv6{H*Tqo6BD0 zg84fJOYn&EkUvOv1DC0t-RUWqSg0V#N+aL1aYXuWsmz+F zK?bl^BjZ%~c>BEqCVWUrvF?oL%B^)QVRX1}8FAuF-y3ajt&k^4vqNeyPUsS19(ZPD zRWoF7plQ#HJCi8j4~l&SQ>#>pM%~|FrLpRH0Z=FM;0hIB)~q-UvcN0&hl5sT*vl(6 z&|suclz}Y0D5}#<5Vv~?=#x1yu*P^KO`bSCsogNe80-!qVz2FWS1MoeHg#Y+Zw$oQ zT0l}^5Ied)z5mZU2yt4Jf0`Vw_)k|mD~Utvu9hLitpA%%p@9k9}Z{@|=>a(h6 z=`eAZ?X(UeoF(nKHLbZ0toh%k=FdQ~{n9{xE+&sfF}BdDS1dH*hOKdJFXGfXcYzu> zZ`XQ6V2`+d48WK+@yG<%0+x155j3e4pXD&99e+7^te@gntjDR)C^O<&Vk$f#f{!@{ zLX6~K>*JP^v^BrQ*g~p;Ekg-BiBaU47}(y13nL)Wx{Ol+C1tCZ85go zgjd6WBx`c6GWyJ_{%`z+Z7VmBexmA=UStKd3~&4N5u|++X#+ClI>&CFW^;wnm;!-& z{_qaa_<`qS&)d2;>eXQe#O}5i9P3M#lHJy1cW&k=h_cq*a51!5oo6Ev?Ykidz0xL zXhD{RI2!;vsx4-nS{zy&7>qI0Q&0&mra=0Y(VwVZ%WlxP6?iqz8ix4dRVlH{Es1!e z6aBYpFP$V_`mA(*J-vsuqQFL3hNk(jL_bE!MUkhLTpdHQ8~i+(1lp$|XTsltPAyUi zF1jM-(+}lp1Q&%y0-?ewKInn`jER(fa6{zl@V=il-$pdrK3TAyR4|w?*nL zhNK%~;cT(GB2jzuMu;-?#g+ei*B{ShP51Aen-NFMxY9aHY)6I%3=uq1gJ%fOw)G z+xcDn?YXzx8%JwWFutui>Rv+x$no4t)v;}f8axF-6@=(T~N9AyEdLioxg>TtH42ySXQpN{}mCE3(IG<6(e%VznDWb(}ct+ znOa*F&Zrr3q~?ch@f3XfNhW5k=9Q{jbRja4FNn;n#MG0BN!nT=Xfz9f;2vc+||AFc{h1J2TecQWNnB$B{@{1T`avCu2jT}r#uFBPVoqFEY(No9@lug_Dh%rgi?Sj z8wPYSx3e7BV(v>0tU0FTfy{mAK)X<%GR3gjMnUguiY`mdN1S&N^l0rT;Wu^Nj}ioS zDMWNO20ypQ)-%!7K@D{DAklFxl&X1L$E?EKsD269Ocm@4>FYEv4ahU=JU*v!DTj=K zF8I*r-oq0oW&R==r8~SqPo%D|>FE6GcoU(Daj%<_epmj!Gfw;2i{dam*fjkP(_|!i ztkf`gddKPU2wg1&Ip zPXy&uP0lM&%x<^DcZxXWv~2x}NwvsH#Z2=`FvNPt7 zaZkXplT^wQ6hqXf#y@WJwgvB~TuAio&Nys`Ew0wJ@;1b7zqx67HklG&B~V?_lMAgf zelo$>GgFCkeAV2c)>kztPDZ==K6JxT^5D~Fzaz@15b$-m-|r?bf;{k6$fT1jmK<|g zOg|L=+?7f|NI)K^f4{v(@fNm8ycIL=V@F@*7|*YtJ3y+3Xu^V=0#F>JA+}D2J ze&+Q`X?pmNVl{(YX|G(DOXJ1trw#V6+Z30My)n(hrYlssdmMJrp6jNK%zvNtatQCAbWeDW>Y;+}Z!)kP{0!)G zy(rW*rIfoxRB=A|;P0GI=emCf&%|tb{TlMgY%A86TNu+Wa=iqc1UOKFfFtCmVZP}` z?T@NxUe0j|yd2eA4`1}LY5rjP*<|Bs`dRSa$Ql)+VGL+^`g&11Ra1HLFFB;2g;B-$ zi^5?pxDoL_q%M)2qI}T6MIl&=Pt~{A@og=yK}6-Mv39>LAuFf;u4W~j`h-Q_(S3UW z7T(-O1;jAG#HLAgW$6hdw~s=qAC++ef}g)o2%_a;HVo351!x`@)4XwAc`CjkVp7A^ z(bKf&D(-2T?167(+haa*JJ`Kfl7^i57(x0;z4i9%mf%+&;X{#L^|<4T2!UBuy4ce~ zb8lIg_}?H^D}YR=oDes@VD$hr1{SD$sX1cx70Md25;DmB^(3D8Tp&~C#_|1~>nv{O z0l_CX@l8Cv!2kWXM)4byJi)H<^RfT;jimJ?5Y`qfL-HP*4TiG+ j%NRle0B8dNfGhcbD=^3(y9j Date: Sat, 15 Feb 2025 02:09:57 +0000 Subject: [PATCH 084/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9547 -> 9545 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 351825cf1..090d2157b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-14 22:13:00,505 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-14 22:13:00,505 submission_checker1.py:3270 INFO] --- -[2025-02-14 22:13:00,505 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-15 02:09:52,846 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-15 02:09:52,846 submission_checker1.py:3270 INFO] --- +[2025-02-15 02:09:52,846 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-14 22:13:00,505 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-15 02:09:52,846 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-14 22:13:00,505 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-15 02:09:52,846 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-14 22:13:00,505 submission_checker1.py:3286 INFO] --- -[2025-02-14 22:13:00,505 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-14 22:13:00,505 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-14 22:13:00,505 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-14 22:13:00,505 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-14 22:13:00,505 submission_checker1.py:3307 INFO] --- -[2025-02-14 22:13:00,505 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-14 22:13:01,492 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-15 02:09:52,846 submission_checker1.py:3286 INFO] --- +[2025-02-15 02:09:52,846 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-15 02:09:52,846 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-15 02:09:52,846 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-15 02:09:52,846 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-15 02:09:52,846 submission_checker1.py:3307 INFO] --- +[2025-02-15 02:09:52,846 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-15 02:09:53,868 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,14 +27,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434726 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438954 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9695 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.61982 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.60213 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1596 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352822 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b4b6dc348..d7e9848be 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -711 +712 diff --git a/summary.csv b/summary.csv index 9328439ef..b3c91b331 100644 --- a/summary.csv +++ b/summary.csv @@ -5,14 +5,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434726,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438954,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.9695,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.61982,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.60213,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.1596,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352822,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6f693ee81..5d2a89fac 100644 --- a/summary.json +++ b/summary.json @@ -184,7 +184,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -214,12 +214,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.434726, + "Result":0.438954, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.61982, + "Result":9.60213, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 0b7eb708560a95e3080bfbf4e18c2f8b9f54c776..d4bfb4ddce7f093d98dacfdc18336100c34eabd0 100644 GIT binary patch delta 2792 zcmV024-T}}?(W^vN4HuFj(>feO_h5YCt);isiLl_ikt`0Fr1HD>dp1% znxiU7>d%M%G@8qn`Xm$e*Yy|hc7Z5Ptt7If6P22h^EQvuQCg>vD8&%=0CPnLscHaVLHhxRwm(a zDD#06Vq6?y6WLaKlR(karB_X zs}_=lpPw0Vr0dzR1bOMVyg2V$V^h$7E>N;O^3l6*8^?|NyyA`GI}N#}e+4(R;C82= zk@aWWDV^_>4t>z}`_M;4lSsZZ60tA2W+eK)}A^IGi^dV8Sg$Snl#ndCD*S@Zct*8WD6_V%hqJGJu#sId%uZO zR5<&J4l3Mz#Ya>Kg2E?8e?qRX&y0kenlna1Q24?~n){NkjD(z;E+Zl5=A4m`vGo`U z8Qcp-LPocT5}l0hB_koDd&Nk|=ze1)27k3#{MBaRdU^2MZ z>sDQ>1xukfOORMaDkVs~v3E#pX16w`Yw;jDJctgSXoAowN-T2ubx;BzG7<_RN&rMg zLP2y2KxD!xhy~$Y3DB&jRse+R@WQYiuAt+O{J8m8{- zELHbNba@h8e>~F!kW-Xcy7I;hp z6gC>u!x)#|v@jCFaf+$p26$bDmn(EyE{hT_O*fJa8-5Y5?Ef5Q>URX~v}s-U|)xP=l*JgyNk zY8MdIW1{GW<0YeU3Gi1t-G3WszRZX@mP08Q+2t4~a&Rx|PBtn2WjHIzumqjgn~tsV zwCk3mIS1|-1;68dB5;S@6ckF-ST==re_L zDDk*Pe{2dRjp{K`Fk5mOI$iE`|1CnfGgszR>RMg=0G3tcSZ31PbWCjadF!Ta9Kw=+ z=5!+PF^;KIEKy_84SPm|STY*G5;ce=Y7k4k2ur2{z_Or%?)KmdmbAp<8UZY6RF8>* z%~VF?S`axAmIy6M>VZc;a_Ls;_9e@0IP^T^SlSfa*qDVC@~EEx@8i5kQb zHHf8Age6k}U|CQ>cYE*!OIqS_nb12+i0Uy>h$U(eOXE#rNj7!M>Eceq}T^C9+v9k;}Vm8eDFWBsiw`v(-$QPm#i{>puWw71@=?wNJ87bwx{VN$maqp_)7htVLEB@8=^2BBm$0A-P9e}}68 zpe%Cj2x_W1hZ2uz1RsZhIfseD$03Y82Ofu1?9V|cfBy53H|pED7Q4nBtFF7|!5tpw z0F~(bFsXHi(b&|x!)TDo67C&FgH$papt8uo!&LxO7Fl=%b=aIkiN`d88?WHhVWM#P z%4lHuTCrK@H7bA3&g&bs@$|*6e~%|$!aD8&JGpQVaETTXlNxy#jZK|Aj0U+Z;pJg8 z$R(ozE{p6uTm`^ok)cO`qX$YnrV->4Og&5#E@2s44=iCT_VxTbT)t6v&v&tF@5wFa zj%yy+@6k0LOr3G;P*)G5v8k(v(Eym7hJwjx08CCp!PJXfJzND8%%Tdqe>%8&pv2=E zq4$;$)nlRnOhyA>>J__sUIX**J`<@VzcNhf+CGz-o6VL7m=p(?WDGmh)Wc|OYU*J$ z$Ycpq52HaQ8M_QHS!C+r8UZGYOg%c7dZ5JPdg)Ro!PLVY%{G!AWuk)wje5@pYtbZuL%6}+8%PR{;{pD0D>C-~CRQb5b&QPALstg}T z>z@cv7I7HGVfwUgJ^zh@<{hMhFSu^Cj~D(}Uik4ioF~dujF~+(l^`&7R*)q(78# z{?1U9Q52=mZ}K78{&++CvrrKy4g^NUcQ2EBA0U5f!!Qtp?|llRyDizlr4_aVZON&D zLQ`6D6VfITwSJJ)I)3{qPU0lsQ;(~e{dRUyHQ%c?`32{t(KX65f)dc&2&uO<`dBT} z2bB20v|z2#P@^6^nm2b9x0D+PZ_Ze7L4r>rNPEg{jdmeeim~TAP|Ro1p`*AlPO%V6 zx5a;!@o%;TTo7`P6$B<&VE6z`?MOr?q~Igc{Zy06oS%E1$FHm(?T!H3cckloV-Ru1G=2BQ0m^!^oTd3s9Yg z{F`_N9CP1(g60piy&%d01t)KhSWuIUBU=GaljtKn0oju!BtZcblV>D40fv*JBqke1 u#dj|y0RR9{0ssIJ00000000000001hfoGHBBt8MVlOiP}25lk$00023J~|oz delta 2794 zcmV{MGjg4i2M1bfclU1Tqg$;7$G<*?Q|(^GNf6DNx~Lkure=OL3})k|esle~ z;^8l!!;zR0)Mx(%2A0vMes#&_KBvw-|&AOjV zf_b91j)y^ZEYC-a)u^eTh@}vGrpfZJ76#c^N?i-R$EIG_wTEDsPV$PiNiZC$ ze4usBUnFT1{1HpR9}}|zb?5!iA9nvtYjx!lg;+TgbM84CQeb| z>?+!*aCa3SQ6UHlpBM?bf5JX95^`$J7zsh)3nOXlO1?4@a%wt^gq)jmMncB6%Sg!J zUN90ex;>OgGP;+HgpBSLBO#;vjgi#(tIguCHVZFTGHRzNu}C-CD3N5;J~9$AYM)S2 zC!_WmB{pg3>{-HmB7R|7pjTQ!c<1@p^9*%P3%b~VL$1>c8q;G=e*tOu5+yFZSYG-) zYJK_d)u(>s*-e& zEbhV}$(KIK;NGknwKP_0xyY*JYpL2rx4n*!Rgx`XlG=JLQ`yu!@aOZ2KVMX$+3Yc1 z%+hQzRavVBel!~ee`Ea@eW+3|m?rv%2Vh1{#ht9?`PCw&OVv&!{Hc9|GMna)O2|&k4$klgd7eL2^lT^MIgm*C9A*BLp&+6JKx8Bo zM5h2mCY*v;5Z=LX0MRJ`k!hWR=ui-u^Fx}vM2Sn1x^n(^KrADb13JEu6Poj%wf0_VtiV}-lKy8!&iHwAjh!P+XB{r$$%q<`h!yQUuL3qa{oSf5wF4F*o zjmGpa#-%qcjD+xb<(3Ou==o7nnR?3_9v$mvuSv^D?*IV%?iPS<)e^2Fjpu)YCs7zY zipz4cUqhNZWi%GKZ)`Ifppny192pJp$Y>m*IooJBe5G{O^SaR&QdZwfX?f6$5wdS zRm(A)J$HM>C;TXGsYU2b*%Eke09SLReoqbhy?%QA92WYS!BOsw~LtEOGwhb8~a z=|tdT98*#(QDe~!dq#s;G8(`VHHal@5KCEvB~t-lSx`Y|bMOUATGHhj0W4`$kBNfK zR7T@k6ifMkfMppu9?%3vTQr4|~Sn|(Ce@_JS$gxhbM2+QAEK!44G8(`VHHal@ z5X*WImP`eJWkCg<&A}HeX-SvMgx*;~RF8>5EK!44*55RiWK-v_6In)j2N7A{CDIYr zp6wmMKf^x}_*~p1XK@&|w{BfEoEu8r-L zYt=S|+~Xb+du%2Gtplb>fn>ZWf&$4{5CBL<13;n%fn=O7)NiuUus|}60FVV0bT$WH zfTSf|u9q-H(x@I2g+MYIxS=%P5Rm`Ge-3OZ2_D3xxy7WcR}G`S$N44pm~jO9JWT4w zVKg?i<1iXz@&H2)qd_Ja%L_1BvE91?9Wowz{ zoPUMMw`$c9#H%AM+uXNu-7)JJFHoY@!=!ERR4>0U78ibP30F*_Ze;uv@ zfU?N7BdDq797?)OBltK3%sEUHJ`Q2*Iq*27WPc7q`SYKLyiwoIwb(Z9SXJqodv|!8 zJyfFa!=%<7Mq^X&4x>RT4{+}=8l;kO&;XT11|F^vpt8uqBdEjX97?)eFJa>qoH|St zE?*f9EMH4D>%2zg&)Io>qc)ztf7tf%%Pfs!uQOW+brJxmlXVHsNwEMZIb_53?rzEOA2cd>2n z$t~v%TkdhXN`nVeGL9YU>R~iCb@ebB0F%>DFc}Si$!RE0vG8f22O%K(!_rXH>lV6w>6Bf-=IC0(wUE@cu-JxmlVVmS?6#LALQJ+CpD zj&dBu58YFilG`C`SGjHGf63;5*98lDZR-o}p^C?CR(bJf(<<2*YpNQB0!tRK@iDbZnci*-dJ6D@i>?z+Ek6QM2%{8`yH=-bYS^9jpoZ&rmo#ZX&Qyk?|vlL*qDa$BKY2$o{(`3@BGS#;QZpZ9k5_rUE=d19GBsu|zlb|Ff w8}EarlO+KF08jz|01*HH00000000000DysMli?&j0lkwTB_jrJA^-pY0NjTuOaK4? diff --git a/summary_results.json b/summary_results.json index 43950a1ea..3f34039fb 100644 --- a/summary_results.json +++ b/summary_results.json @@ -212,7 +212,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.61982, + "Performance_Result": 9.60213, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.434726, + "Performance_Result": 0.438954, "Performance_Units": "Samples/s" }, { From 24398aa86ffbca0ab5d7181adaf89d94a799c7e6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Feb 2025 02:12:12 +0000 Subject: [PATCH 085/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9545 -> 9545 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 090d2157b..d37def9e0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-15 02:09:52,846 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-15 02:09:52,846 submission_checker1.py:3270 INFO] --- -[2025-02-15 02:09:52,846 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-15 02:12:06,842 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-15 02:12:06,842 submission_checker1.py:3270 INFO] --- +[2025-02-15 02:12:06,842 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-15 02:09:52,846 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-15 02:12:06,842 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-15 02:09:52,846 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-15 02:12:06,842 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-15 02:09:52,846 submission_checker1.py:3286 INFO] --- -[2025-02-15 02:09:52,846 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-15 02:09:52,846 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-15 02:09:52,846 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-15 02:09:52,846 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-15 02:09:52,846 submission_checker1.py:3307 INFO] --- -[2025-02-15 02:09:52,846 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-15 02:09:53,868 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-15 02:12:06,842 submission_checker1.py:3286 INFO] --- +[2025-02-15 02:12:06,842 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-15 02:12:06,842 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-15 02:12:06,842 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-15 02:12:06,842 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-15 02:12:06,842 submission_checker1.py:3307 INFO] --- +[2025-02-15 02:12:06,842 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-15 02:12:07,884 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,10 +29,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438954 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9765 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9695 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2517 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.60213 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d7e9848be..a8d5a83b8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -712 +713 diff --git a/summary.csv b/summary.csv index b3c91b331..9020c732a 100644 --- a/summary.csv +++ b/summary.csv @@ -7,10 +7,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438954,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9765,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.9695,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2517,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.60213,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5d2a89fac..1de38dd9d 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9922, + "Result":20.9765, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.9695, - "Accuracy":"acc: 74.200", + "Result":10.2517, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index d4bfb4ddce7f093d98dacfdc18336100c34eabd0..2a8d464be6086b0b442f30bf74c47cdbf91e0bed 100644 GIT binary patch delta 1427 zcmV;E1#J4sO36yF*9QfD1%=d6v)l)z2nE^}-F|Jeg%Dx_1)ADcc?gs25;1>N$P|?X zDm7eE2~4q1DPgyRcWudQ$qfAc$|PYkZkx*jPZXym9i7(cc4zqgzR1ZPR3hW-P_qq7 zBfw&wWNdw?U0ojQp+;ub&_x#@pcmTwj`V;YHh&E%oxxQ<3@mi<%1c!NZtHX4d-G?b$= zog{8$}!J*^#dM|#QLVlq=c#+IJ$qMSx#@VJd=3S zgOE_P(Q}nTbmw`+V-SL)^UE)YF^(aJib}kgqQy&|%3riXxjW|)U*bwY()qxOvaV>1 z1WrII_2`_jI-`wwhNWAUW%uAt8&9aD%2q1lX5(S=>~zeFf~)sl->zsZRfXskq)@2> z0X;5jy?js^iEmeTj^Tf+Lla1;GTJKUEX~&Dvj%&XtiDl2Q0qBW8I!B~(C_DUF>ke| z$n13EnH-*Y+ADekPg~o8r>t}YkOEXON_zfv@r_&qXU*SVf$q4Gca4m2HX+mbpJNgP z{($(dKIRpWe)DcpLpD$HOw`5EuEZ@au1==*0r71kupFC^N5FqMAcMe&{20-4!I-8$Po8119X=PWX5Pm;07w$;jwio z|LTBMJ6Er%Cs~@-P21P+aeJjjnL|gT+t$vY2X}jLuLnoA*y2GC9`@i!;|8};Vz+ot zPq0x>*y25U?H+%VIq8|Ki$qT(ie@?>_exw(a34Xw1caH+^HKnp*Ubm0nBsd+SaBN>S2E!0OI6 zLfpN3YWe@M?a6BB9LumjSKg}#-{!m*7@z+Jzm@E_?^b_{N!}HY`Wj(&WH|54B>u>M zrY+yi_KrM$ei^((_wjY`A&eiJ(@(5~4mO!KM0Us1^<{FqtYcc-FLBZ_EOvB(muAkF z;$1W~Q&&{K`V#ch<>zcNJ0d4=tP%`lad}N#D?$SI-W+S+`JHV1{>)yo^-FunuK$p2 z|3UGTY+rw*dz{2CQXI_>h#MJRga(}5+&gd##|!u7Uf=M2KlpF#bsWPBEuWB+>G|q( zdae*3jYZ?r*?hGeUo6xtZ2QI_2z_;`<5_OtMgu~QXOmsnX$I>-6h@xqSdkZ^vvy;% zg~#S&b>XmM(tgybwnQFE>f>q6p8eBIA>`;^P)h;>1d~7{8nZGNLkI-A^lt!@c^@o) zA#D-118vEvfkIPSaud?V5w(7h)H;6qDo)}g;8TyQnf-QlQMuTwHu(kTrO_42GlCM( z+z6?g3Vp1X=_5*fU|O)&XsA#R9xdv-l3U7+gEwa^xFEqN5u`okwnDoQEXCOK9Vq6r z=+IHz8mCx@rE9Qd{F^m^3qtO(g1`iS3k)BCsU3;vgcN*4+8@^r0KqY|pg;$Y^DM^~ zZvdzKWW%6bIh72(g-PpK9Nl{_M`PD@SvMQ3F@KJ~RE&v%4V50tJ2rh15}#iX&SAP?P2(A{(08R(S{n000{i h000yK0000000000004l2+LI$BqXfG2Zvd0ZB!5SG!RG(~ delta 1427 zcmV;E1#J4sO36yF*9QeBZ;x0|v)l)z2n4!dL2t8$5Mlxa>k(+936t#-F@LF$DJlt6 zYPh5lm|~w&!fpre+LG6j8Tk8^NnkT>o67=E6sIK}o!04gXY~E9$jL2KBIE2pvkgll zz+#?cY;&MpUL5J6Muem+p*d%8pxuMezK{PH2_cEv%EWO%dNfL50CfU@DUG zs-RN+TWw5HRzMSR10ajsbbl<%Hw&6Ejl{fW@<4N3Me$?+*q zARn1>JTe<#3$9{dw90rJzhs#dZEy*q&Se|Tb6QYG53FlFirT^RG|e*xZTN!!4NRDb zVLt7+h*Cr5Q}PvzjX_j8$YGsi@>_f5gy+2a0ryB^eO+WyLe&l&-ha|8r`K7YNxbP^ zNGRIqnMxtL^P=K02*J_$f&fu;+E%^$1{6KeA@^t$0p0PO+YX}cd0;TjCKUBp^_aQ+m!OJ z4p_Bw^_qH;rD@%?efV2*{PiM_x5zp!%ZEO}8M~`gA!Y@jCei1m zE{3lDx551{u5HIPdtCSKb6;WG9*%;>TUk4w;_^~0%MfIyMK|fu7&ZqN3atuc*!9bQ5SH!gcU~ir2U{%ZHYXT)Q8iWJ^QDbLdfyIPy%+*K{tHl@hWwj&1{`zWeuCx?v%4V50tF{;k62KXiX&SAPm|^&A{*-wXrl=P000{i h000yK0000000000004l2*^?tAqXb69cQ2F5B!98+v!nn3 diff --git a/summary_results.json b/summary_results.json index 3f34039fb..1c2fad47a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -46,7 +46,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.200", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.9695, + "Performance_Result": 10.2517, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9922, + "Performance_Result": 20.9765, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 7ea4442b9a6f68e88754e6284b16bc072a9150df Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Feb 2025 02:49:38 +0000 Subject: [PATCH 086/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 9545 -> 9544 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index d37def9e0..6a2237289 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-15 02:12:06,842 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-15 02:12:06,842 submission_checker1.py:3270 INFO] --- -[2025-02-15 02:12:06,842 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-15 02:49:32,677 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 +[2025-02-15 02:49:32,677 submission_checker1.py:3270 INFO] --- +[2025-02-15 02:49:32,677 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-15 02:12:06,842 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-15 02:49:32,677 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 -[2025-02-15 02:12:06,842 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-15 02:49:32,677 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-15 02:12:06,842 submission_checker1.py:3286 INFO] --- -[2025-02-15 02:12:06,842 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-15 02:12:06,842 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-15 02:12:06,842 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-15 02:12:06,842 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-15 02:12:06,842 submission_checker1.py:3307 INFO] --- -[2025-02-15 02:12:06,842 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-15 02:12:07,884 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-15 02:49:32,677 submission_checker1.py:3286 INFO] --- +[2025-02-15 02:49:32,677 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-15 02:49:32,677 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-15 02:49:32,677 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-15 02:49:32,677 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-15 02:49:32,677 submission_checker1.py:3307 INFO] --- +[2025-02-15 02:49:32,677 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-15 02:49:33,856 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,16 +23,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72773 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.438954 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9765 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435098 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9484 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2517 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2068 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.60213 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a8d5a83b8..d995b75e1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -713 +714 diff --git a/summary.csv b/summary.csv index 9020c732a..cb8415642 100644 --- a/summary.csv +++ b/summary.csv @@ -1,16 +1,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75124,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72773,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.438954,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9765,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435098,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9484,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2517,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2068,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.60213,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1de38dd9d..9759fab44 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.75124, + "Result":1.72773, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438954, + "Result":0.435098, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -244,12 +244,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9765, + "Result":20.9484, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,7 +339,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2517, + "Result":10.2068, "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 2a8d464be6086b0b442f30bf74c47cdbf91e0bed..3990225da91f68dce69573b99b5c068fe7196932 100644 GIT binary patch delta 2689 zcmV-{3V!v;O2|sE*9Qf@L$2abliUYjf8KqOKDyOfaQy4zY^vPLI0>V9OBHoZRpdN~ zhT(kNQg5z5*Bn(zQhz@5r_o%t)F+v!zqbE#{1C;zCle`CB}*_*TIwWC7e|do5=`XG zPwLS^&a)$u!?c-sX9n1Tp#B$V9PsCB@H(jWW_VKFj+fY6vJFk_zD|b=!d;aF@ zu%&9bv;o@Bt7crq%21B{*sjJG&e{89Ssyu|l zbdo1lCgE@>^MO{CV40-R?6>tHf3Zoh4i(lWR%l`y?bbz8vve~*A5T-ciD|5BUA6F| zpZe|NIC@axRSn6)&(DlF()Daug1q!wUYxhBu_@?37bsaj^3l6*8^?|NJn=^HorYY~ zf*V?JyHn7}+Outx&NoVjHfZ~8=%b=ZB;Of{*q2;05`ABC!$`<@+%l5pe^#>BU4!qg z0SQP$`=~HUZ(mhhzpA)-RdHLS*z0MN?`ad_$$0ls(WG5{U2^@p1<(=z22H^ zZ%<4}VDDFPiVA06(Lsf~ulR@xK~VU_NXQlTnURoFbH+#r3SSsWb6@h6k&si zoHG(KwjLuPgL}b9$msS^f1;Doy<{Y0bgvi*8QpJ;#Ne+si@(||yj;nsoub4d&FG*+ zC!_X}k&sdQgc5^{+Gmv5q@uHD3G<2gg{gsFX$9fk=VQ+^)HyBiumXo%rx!G)$D9IE z@g+)Jda=B;d({5&->Xl3?`!Y;{c16kWUHZZm`;{=^&pxxX4CBNe=J++WUm%79u30z zNXFTEO%76-q=RI6Hw%+|>5~lZ&AQe|V<}gQtVX_;YFv1oO?)hqYzdPzHfx#MHuE4@ zENa1GS&QcL$9OqUv&B?xqZ$O!d=!q=U(}&Y{cxJ7AC4O%X1e}XMVGQgtIV3V`tLxy zzOUOQ)=P|f)6k8bf6s_L=xiQ38_ziXj7Nz@J~lfj0q7VBg$^YE9U~!E->U!D*{zM~T0DpjK@XyXCz>F1iV}-lejStm zh>V1Sh!Oyikx&qw0uY&S3SvQc7sCNWrvOBzb_$|HL1fMksqzvfF6q>j^S=XP6Gq-?540kAr1>s$naB@xyJf;E)8;$8c*nIA++ZR)QwJUZ6T zUXxbQy%GTS-5mhkx~1vD(4PMZoLFe_RVQW0?y5(rjfjdUQ@3@}`++jBbg%UNEO`${$Ldj?V zN@m9&phOKqDT<(E5&)D13A%zlQ#gkbkE_I{P|~O#69uy+r=ip3PV?U)lsj`}PNjdY z)x{5BSw$bqOq!d9iOoK5-L#EESn|)DP6R&2F?EV1YAm{8&u9=!Mgv%)2C+m9VyPEl z$s_ddESF-58pM*(0G6mhEKz@hSQ$s$*d z09y`}cw8%iNpR*cQMhbnyg9IJt*UkTswlAYR;j=V=BSNAz;p7qVRul2xHHI#~~H_ za}dg(|2*W4`gX3xu5rhz>#lilhsQZUCHg*0YTaQpHudf>8ldxy~=m5c_cEHdzL z34qEX3y+`zS#Bg6ql2jje@Z;2633-Xf~kjzf<-K+p^I3(VpGp+Os1n8NAW}V zl$G?Y%-U7oHS=WizuR=^7pfaOUvQtvcq}_|I!%-y%7$wlxh-9P>%T9KZr5Lsf01YP zMe|{pd;0z9`s@5@I3Fv?AL}0qu<{=Y(DG!#sK1;_C4E}RmMR|?*%`_^f2%6P$I<#H z0+dA@Msb)vt!vMJqhLAp+p}o+e$}YuJp5xR-@gmqSx4^z>%%XK{y}LeS=XPXS?^Z0 z?HZGLA0L0qPQx$|MfZF~hQ^jek6;$W|4lV1Ok{QCZg;%Q$STDdu0i+yZrb0XK4Z+y49Z2eu;7~!_Xe+4? zrQLsGL)kao0?yO)4omQqQ%~^`7@LWR1|(+_()`%A2XKa=1qsSKoFy4Pdp%g`W*d6t z!YPG67?`z=;^f{tF&VqAOS+=BhWr`+T)({atHwedPsIS$C1-?LpkCX$#Fs@lTuYr_ zhZAoE@W(^YXWoWIPJO_m5P+QGj><>z^xP$^P@SjwB2M!-TQq4-ZXZZdd>uz#^`C)q z81ir82sq`w{si?Ov%4V50tLQ9uHsOWiX&SAPm|^&JOSC0BP2lq6O&~mIst`~pd=<6 vS~Tq%B>?~cPyzq|5dZ)H0000000000fPrR{;Uqo*x|1L!BL-|D00000|Gfa& delta 2711 zcmV;I3TXAnO36yF*9QfD1%=d6liUYjf9~$x(nq&i3yyz%oK2N`87E;hZ>gfLsfwHj z(J-8kTk6gA=bED`N$Sss{xq7)mii<+la@M3)5TGvkpvSt z^OJhCkn`-wD2iu(n!Sz3jbsr^f4B;oO&hwVS&f+=&ee4dM{$`tqR}V}vD8&%=0CPnLscHa zVLHhxRwm(aDD#0z!~O{~zwHrlO=re^78em4t>z}`_M;4lSsZZ60tA2W+eK)5XVYjTjvBpoEnyIGjzOP^$LZ`QR&8cVraWHs`&RO7OF-_F#syO(uUwQdU^2MZ>sDQ>1xukfOORMaDkVs~v3E#pX16w`Yw;jDK|F{Ko@j#5DM~DI`E^hN zATkmPB1!;6MnXY!3P5DSDToE(T?_{hodOV<)+vY%1(7*Fq{&N^xFo47=YI#pDpHg7 z3o3stXKn$B817IK3&OiD;pChacuWHnHX75z7?<9(FcQM!m0K-rGe3xu+SFgw@aR}S zdrewJdL;ntyE_27bxYHQp*{Z-Jc&Z%QCyXi{TkBTDWkE-ePf5w0F9i6;>c)#M@Hii z&DmDN5y@3Rku0j9yFIvt5=uO-5i)8Q5Y>NUqUeU>C8Kc(@K-zCe;a7N%!oObLn#;8 zz1QA2ksaJzvF%)aEIL#6iU=sHiZ&32qmKd zD4899fD$zbr6_`usQ{oXsGuw8Glg>~@wi583MGx|F;OsEavC~a?sWexLb)?n=2U;` zT3!4AmR00fX42erOl!xiS!jgaHbRzIEj;T{DQDe~!dq#s;G8(`VHHal@5KFxX zOQr(AvY>+Q_TUSaw8Y~Y0W4`$kBNfKR7T@k6ifa80LvDcyeQa7waSn|(C zPXzPG(V$qO#&Ri^s6i|l4Pc2H#1elsh^0}4B~t-lSx`ZDd+-HITHM>D> zC29~$<4t2pHg*0wkyWHuib!LhNJm%)ws!>o4F5#nb8)jtiA0U9Q6d>L01(M&fJoFJ zk*IO#e(P4l63H||pEnj&(A^$yFK^iauu$3fO zib->aN!_SxnsLDSB@UQz1p0qGOzOsAG&Z&4FdAgCgdvB~Ad`&c1(+=Igbkj9$LBXnRT=mDADR+QnwDH zv8i2$(IAv13_FYlp=2}wWszrxs{o)Za_tCesyT-ek7)!Shk!YUiNb%!A&fl-9*0!y z&p{}E{_~JG>f5;%yT%=>uDj;J9UkWZmFW90sdb0Z*wnkjXpqVh?j1&hR5BW%vdF;0 zRRB~LS$G6>*qlR&$25W)ui(^SqHy`jXkhtTv03LeDu2$->l?N4^u?}^Ctt!k?g2Zw za1U^a77>#gc^HjNoje|l2DvQZ-Z$B)>9D>e@b&nw!m*2bdHGm}Cq))YQXhY-;LZG{|HL zQxBs-CKy~m{jLALIJ#YbLHo`0;W3?E19 zp9oMEaTvv6`m}C6|BZs>)NjwC;rmsmmhnDEbGbrDUl;OS9}& zt>iDbZncjW{#ai4@i?3(%2bZBM74T$`yH=;bYS&5jTWm{rmEaUX&TL*-%X@HlyUyf zP?b>>rO$8jA=&Xy&hf{GtbtyRI4p=a*F7b5{57#mu zR`JAJ3F4`b`o!C5kz*f-BnF_zxM$^a_WZIXUZT1n#XKcNn$Opypm{;b!{#vZrvCy| zry>6)o&m?)x1XT?1GBmy$^r#`1%=d6lZhi+0Z^0WBRm1xlOiNR4HOpu0C;RKb98xZ zWpk6UBovc`BpL#S8Iz$I6_d3j5F5JmZvZ6$002+|000pH0000000000004l2XOrM0 RJ^{OvA0;COZ6W{w004dm7%l(+ diff --git a/summary_results.json b/summary_results.json index 1c2fad47a..14299121c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -68,7 +68,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2517, + "Performance_Result": 10.2068, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.75124, + "Performance_Result": 1.72773, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.438954, + "Performance_Result": 0.435098, "Performance_Units": "Samples/s" }, { @@ -617,7 +617,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9765, + "Performance_Result": 20.9484, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 8e536df004719f6b2a06c926357dae259bda52c7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Feb 2025 08:58:45 +0000 Subject: [PATCH 087/719] Updated results summary [skip ci] --- README.md | 67 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 5 +-- summary.json | 50 ++++++++++++++++++++++------ summary.xlsx | Bin 9544 -> 9762 bytes summary_results.json | 76 +++++++++++++++++++++++++++++++------------ 6 files changed, 134 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index 6a2237289..2815de191 100644 --- a/README.md +++ b/README.md @@ -1,41 +1,42 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-15 02:49:32,677 submission_checker1.py:3263 INFO] Results=18, NoResults=0, Power Results=0 -[2025-02-15 02:49:32,677 submission_checker1.py:3270 INFO] --- -[2025-02-15 02:49:32,677 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-15 08:58:40,204 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-15 08:58:40,204 submission_checker1.py:3270 INFO] --- +[2025-02-15 08:58:40,204 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-15 02:49:32,677 submission_checker1.py:3276 INFO] Open Results=18, Open Power Results=0 +[2025-02-15 08:58:40,204 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-15 02:49:32,677 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-15 08:58:40,204 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-15 02:49:32,677 submission_checker1.py:3286 INFO] --- -[2025-02-15 02:49:32,677 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-15 02:49:32,677 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-15 02:49:32,677 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-15 02:49:32,677 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-15 02:49:32,677 submission_checker1.py:3307 INFO] --- -[2025-02-15 02:49:32,677 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-15 02:49:33,856 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-15 08:58:40,204 submission_checker1.py:3286 INFO] --- +[2025-02-15 08:58:40,204 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-15 08:58:40,204 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-15 08:58:40,204 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-15 08:58:40,204 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-15 08:58:40,204 submission_checker1.py:3307 INFO] --- +[2025-02-15 08:58:40,204 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-15 08:58:41,213 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|----------:|:----------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72773 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435098 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9484 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2068 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.60213 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1596 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352822 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72773 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435098 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9484 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2786 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.60213 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1596 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352822 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d995b75e1..a20e4d18a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -714 +715 diff --git a/summary.csv b/summary.csv index cb8415642..09121a719 100644 --- a/summary.csv +++ b/summary.csv @@ -9,8 +9,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435098,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9484,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2068,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2786,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.60213,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9759fab44..d66078d1e 100644 --- a/summary.json +++ b/summary.json @@ -312,9 +312,9 @@ "Result":0.385187, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":16, "accelerator_model_name":null, "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", @@ -330,6 +330,36 @@ "weight_data_types":"fp32" }, "11":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", + "Scenario":"Offline", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "number_of_nodes":1, + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":16, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + }, + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -339,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2068, - "Accuracy":"acc: 76.200", + "Result":10.2786, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +509,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 3990225da91f68dce69573b99b5c068fe7196932..c2ee3f4b1ecffaa4276a63139b8f9686c047c442 100644 GIT binary patch delta 5205 zcmZ8l2QVDY+t&MWh!Tfzj_552LJ%d~=|n^cf<%qp%jprF<(uJz7m?uZ9$7V}!B-$}ggePKnb>Mfn* z@zEDW_5``decdzAT<(xMo*zLU;r`nH&ayr*1O|RRkS1vc>I_ywkZP=ukHmo&o&ko-hN|YYPdt0=%Qi3Hr`){QbPu$O)2EK7k0Tg3?%=`6GEmf$=)@b9yiJD@x`+^IQk10LvsKdYe&R3c%S^5xFoPM-38p*fzRSKTd4sL-` z%M`esBj*M|TBQ{Qdfmlq=1JV?;(7=U_(DV6)5XWfD=aQFFnQ#WN+#q z9bnZ#UqV3Fflb@r9lO8WB!;+8<9s1 zz3t~N0+C{gsWgUh_0i7|cNpq0Y@A0J7#IZiRSE+~aq?W15Ceno6$S?F{fN5_A7UGy z2EOZ*-ACQiw$v7oz22!jS>()OSGcJY=MgTzhES?6miu$?@=SzL zIL@Qwx1T^ZTg5i-wIh4*Jbtl`|JV2K~(MJY=vpC z?$4=3C0*SKi-Y}9>Lbq_pN79zBHCoADQh0SKfGbSBTH$ddQGt~kn)v>Sj2!X2wsrY zmX!N;tn)2ZRF7rde2D}klnr%ME&{0b;c&6tma22snr8eVBR))Ny=u>|j*gnDHIB_4 zJE@;!t5mHZqB;}Tq5VL)J1IX=k(cl!`ukVCULlZ}|RV{ZMC*Pr7^Qg{o|sWhcP4VbRLMY;~3d`ddax z1kcosdOGa-D+KL)H z-Z<#z&J@;s<+81ESK|B=mj4>+W9WF(v=#!}bl_30(kjI*maNjna;pzFErT<4N~{pc z=w1*&)~?Yw5VKx59tGAd4HNUI4=x4PB1cTjLOF z_!r+>S*1F9y5KE$hqq+emZsvUTh;13)ibNI-zHLNfH!G8Gul#bl;+f^-6r@^(=V0o zeieZtg>oc(3wu1y%^02(L-ac#k*6Q>>re}1>&JThMnAJ>;eQ3*502XiztJzg zlo0}ju-wr2WNJ#KK`y^^_5cT`mx|aq^lOW3EC%k*B6hB<3_jHM6g*e443l)K56p5B zbzakcu}+biR`}H5M>sG7N3*V?h2_L}YA=sR z7@&~otetvd-owzfad=@&-&n&wKKfLdr`h{mUQwhV%HA z1mBsOaG3%+w19?0;AqRZr2jedW(o5F(9eBmPZ|C-pLMZx@-OLEMz*n4NL=+@63oVP zrIR&6jwuU*MaUWpGw@tlrHUc=fl~6)1O7TBc>1X(Yv6_Pjyl~0wtzDO5o8T$;5qq2 zQ)`DIzmon2t z2n+{wlUuS!ORtU>I^@X6W#c?TW#fDs>H)-;DyBP8PhxAFM6KeE5(Lu643r+W z0Yr@QyV3^yHIM&#M^H|-(YQQMrMeURmY}VRHL`QsuH!B4GB3O!nHYh!q(i_!Qaq=rv_hs%dgsYLm&2ZBbZm zdN&GR*Ax7iGE3Ith-|vRh0Jcgr$y>SYs(vO-`iG+Rt<6&rcyE&CG9NE~} z#5l9=shM*2lJ{_RjHM$(T#ximoHaed5ha{Tuv6*826qqh`SPpGobFN56*22&*l0n>E@2(8wH14R6B&foX4a4 zt`xgQ7X=yRhxmsuqlC-kW8pT`%!J}u&ke)>H8keI(665~ArCAFKpGWjPsC2AF1TbI z9THW7!i5n*y(hj+X4z8(t>FpdHZKs1~KVXbr|WWJsT{^F(yQS&_*l*1vVnx4ib2` zHb_30()kd=VTjX)AA76)AG!+r(($b0a%r^Ajd%d9YwoV@)l}JwD2CNw=yGosk|EQf z;1e9*LNtNZ;Dusm0S;ZAE05q_CQd{@cqb}zIOW;YIPSNDPvF5U?CgtYGs zgzU-kgyM0@UiW}5uQI|b3K&B_m|+2)YU$YQc|;DjT^Z-bUG__JL>jUKM(oeM|EPsY zsyk+csO00kKfxPLkp_o?h` z@7%rpz3h6bf~N79G7HP$a>l(bG~gjB~OuPZORF<#ttW3Z@Elay0a=H;n5O^#2l zO1yaGQQ0KoPG9EBC7s%9^61OfMA7+mFA+6mn`9*3eqx}W!Qc-m>g}86rc{5Z+ug)+)CsU_RERs42!$ezYJCkLp073t}U$yn~|V~iG$TpxIgEL)Vf zq*KXQicFAFceow?S!?nrZ^Z7%unJA}rCNnN1(Qg{b?D^7z3_9UKpaxA!HqsLddIGE zQ3=Yk`a=PU+I}4C9C_>kZ0lMBG^&fVBc-J+x^jfl|KJwE#B#$mkSH3&zV{=&uXUeU9hkLlOb!KFU z1OuRZGom{Npdvt*B3yyX$C8<={0`KXU$|^*kNHo(9B4=|j#B@;G=VY1)-?UIoRXmA z%tdkCE?B85{uQY8gOs16mPv++_)E=@u#c1{gb}s=xjLH#VOocc@D|5GK^@;DDZL`$8C5URS$ar2^9RZTkf4VisBFv zALhEDT=2wUh@-`S z&$dFA2HupKCFpo6tQCI!!*+0K$&gTIK$%C260v*c{@b#AQ&_^*7L+cx#m`R8F6JsC z6MkhRtr2=XTKgGzb|S!qc{-)gVlIC7QlWJp5VIjs{SP@I+;-$+^rS|qAq6jQnOgn@ zhR4hS4~XRiV+P-18I-m01z^wVKT7L9^Llgp_s+?+qgVznGV%4N;|a@_dCe_GL$ZpM z8UOS7T5BBt-Rm`0jkw~&7gw0f0i5l64G1RQG3Ywj1ZR!Es=`v%|^$rQtMVumOqn~ znVAM&NxF{)a#h+mln~4;OYQ>CYh+7mI`Q_u(wujNd;RW~Jp7&RUP-~%Wo>a@o)6mc zptg-Q*M3B18?MpBVQ!@8r0%I>#Oos9?6O)r7nwse509EfxY(Ut1=Q@W^eJ^+e0GB8 zX}a((ahgkp8rIbkiiCgFVuU- zPB~_|;XAbN-C*=CgJlec%Z=n0VqP>u%B`eTYT+ZBc(kQ(_+`^>(4LByiM%=YQ8zu; zNUL2n>My~!_DTst>F{FEnr>Ehm#Sj{SdtoVj=Cqkr9i3VEUs4}I7K0EhVm!9;rT)# zqoJ>ml(ecvLzW=3WaN0ANpr_={NCgA4fKng4>h(8{;Z#N5nJdZXHQ;owdCTS@73rd zNAJpl4eJrw3;(Arb7g-nR7IsX-93LF)vys>O#{3lbxayBug~Vt@7boAix@y$U6pNs7H}b}W`=13+ zj4luq)(d+Y@lA+9y^ZxMrYs7^FeUVa+^Fi~LHOmvy7n>wYG6gLe%nY_X5FyLjTh>J zpT@Fs8U$1;fv|dbH}cS~4%Yy9as*x&Vzz8t4kntyF+;|r? z@(V*1oeBtaT+tS!=U%8u??SblI|{lQCc0F2&>e4FIGrAel%zlxGs3W~I6FAp+5;R^ zg89PT8<6OVu&2Kifze+ImfS%!VrTw}A22*4yfj~aR2zZpuv!sm#&d9t|6_m<-5OJWT!v{}adg delta 4985 zcmZ8l1yB@R*IpK+lyH%5+$CfcL>rG?ohg8X;=w?1*8#{j-^2X0R^N4q@;U6 zx=ZroeP{mp{`t<#opbJ-Gtb=T+&gFHxp~Sxid6%+#AGxjh%sCMppgVMfy;#G(w%W# z79tN=sDynvjMpM*#fivm*Q7{Peil!D%2%(JRAp68H*+Y@7u;sj8?nverD1rP7)fp@ zT-5AQuG9F##M39DA?$wNd*tdACS3kubt+W`^5hS*U{-E;Sf@LmI>o!<#M`pKZN?{6xpYEzKj)$OA!JqrEiCUFO%;HPxsPW%lQmMI9YD4qad zZYzSz67ks$U|xJV8F3!bqz34OdErPr@W{dGp?^i&-u%>-9>H7JhQ|_7rFmE7c~9cO zIA$gulRJHft99ua2AD$J{}nsneMWkHBjhFekojjY?PB|6ksIo0a*f7hk*LbPr}%G{|DJpPd&2QVgReK}^&Lbp? z`Sf*e-zFg+0i8mng@8_FVSqWi5&ekbozFfD^+F_)e#+>+6gj>OPIF&tf8cD&0J~#H z==>lc2N11Up5ZI{9 zJRLv5L*}R;n2T@by*x@~du#d#6E8JRIxLHJ={ZE`QG}td!h=$D@)Yj51ECh1Kn+U>!N}T%z$&#yVHvM~ zlC_~|kOC5ouZ0_x5XN&G2yUl2o$g6jw#Eu`kdfB?o)1V6ZyGmnzS>%GES>EaB0~Ep zQF^FoV@0DPmi%WmoG`B^7_=2M5aP#0!|;YNji0tbL5;%>HY4D!?R|CHtf&zOa&=6tc8vq&oKU>8VwTF-e~IpN9n(J892**EHda(~t-q$`)JMEN9By`^?~c6` zJ&--fihz%X&>J{6YyRq)u6>&{D5w}IV0-y?9i68DUZ6!zeuTdO1(>*Y$oF{9@+Ge6 z)JzzK?qP6<;YwBc=~pMA7l4D%`;4l6wAHdXX?ZGMgc~y*G*V_2-Sw8 zS>o6EbVnyOWqH#*U4+YfVjGj;H{s2d=TXFOVCS9kJ(QH>}=)>5MKk+W7bf=Z}L6sr?5QK@K;N)rF$g}5$&8)Oy^>yKy=$Qv<2QA(#dNKCu>3$V35?(~j zU)Y@q)Hr}B!hNe->iouT4j0PQ0`?sCuCDE$Cs~%|x}>V6c%87MzqB6R-h&!$SXNT; zF_C=#EB|B4SA|p~EkD%KNANz%e*^VkccAIFyaktTbh&J6iap#rOT{0GawJ0W3pk`r z3mmcNLG8aG%=M$!j=H^HsnvFAy+zaE4f{We8#futRlaskFJ_LTe{9`!qa*XNMyBj% zdc(z@M00E94CJ>`jPK~L*sP%_v2{l#VfrDV>littjJSXZC(b$zQ8+0C!1$O|cOlIP zz(}FmbXd>Q61_xfcIq#f>Cf+@yptrK0qV_U9$8jGBz?|b<8GE@>qZ4LV;f)8Z|_=V zl?n`c!$k|!8IP@+ht%*Qo>xx7mr)@@1##BY@hJk1lDOp}9M>>o9dr&ky zkTD4m6%!T`1p*`_z{s!-_QNuFPC7dso{7 z8;BV6&7YG(GY8R~rd8X)U=i7Gd!KU%XUzofg@azEsN_nMNcP$S6s^0NKi!2~i1(i? zL4c%tL`@;$nnIDF4`rQ&u#sXWihH$?qTJC{)uRQrBoQ|&)dI|T7n!NP40FG7dPW3t zK-_7rE>=qKa41t1PyEMokM|IJXp`BHExD2`u3jE~wKR!&3AI@-7y!Nh4-=EBdN-~p6qxm;?qI*4aHFfa5(ey%@Oi7zMb}X9x94ATDy;kb-#>b1bc@4Mwr*6C?BM3L zYS8F0BfdXt!cdh)SVx-5elPJLkB}H_@dm&`O9DXB;U&7X!rFRuI)J@eKnW`F`a7~# zfP#`JCKY9b`;`Rb zLR{H?&WvB+WS5sNzb4Eta+oDK2EQ1}x9?08)W>%J!za=VH_n4pMduaDOkw3m&rE1Z z5Kt3tfvpzV)uAb$0JK1M>*UD+R!Og0%-wK8j?ZHxKB^6MBuIJUPE5h4tu>%{}iqVSQ*g9@*n(+hJi*il*5Oe_4D2(@nO_%IrP>6ogr?SwpJ46h~*NI47jGegM zk{?iLTwV6(CGYL-iv-0B-3SGv85<3^+NnbTY8ilvF^f$1rHNHh5sSf3gPPp|_khtj z?9s|%aG+^T&loziSoO61;p(3urDoHGT5e|%Bo*E&x+IZsLO_;;U!Ba<+Mb&Rju)?rURiK0wB4WAm)iqwb@05ud1J?=Kao?Qzo%1=<@sz~|AG2O z&-na6D3ha4E3PKa{$%@9#Q1i-eba;PsZOP`_Lt}p{&can40#0S%2ihG6Evq{&H`4d z=K3I_cHm(;4srUljt?Oo06_l`us+? z?}MxBKCsz~cNzUawMRt|#;o5Po;K&#tv$VrT5K!Zuv!wO$L;3XmW~0%Mn+6qxw2vT z(A-xx&c$Luzgj=s&=9jCr~5i{J-MxXOw-v~!$XKeKelUuSGC_1Tr?VF?}t#N}Z@#7}lCH;qGR zv3opPxEI@a+V_0O=LDB2zVo862az9E$|%%?)>n0^5k+~AhQ`C{hTW1S-T(M>S&)aI zkNSC{p2x99zE^BX&FfOi?lY;A{37xPWU{O%G4i+leGSGo22WMu$;ovl6pTST|AghN05x5~`K zo{L|BOq!G&dgk}wCju`&=Sl*(hRNkOWK5@v@+gJ8!YHC&LYKX~YG2b1gubv-_ zo8>C!&=a!0tol;TW{&+PJcr()<;ykx+Z37Amx`3;Iski6fbpjJ`X<6*_I7#N6JFcx zu_t~;*|4+3Y5TJ)P%yxvFx}udKG|65)Ne}_pr}5g*uFU^AUXTh&;IXi^-iz-IzxZe zYu))W!iBXKYMPu|d_-IFcH^E-w8A{)hYLD0PfMY_pWvGwMB&qgfzvqAz(Cp->9Yfm zebyM}j8V1nFj9du0cp?f(Gx4z=5h;zSss^>=TJ}g;E&dWAD4~zUux=VwOxRYM_2zi z@`&+wC>{Y!4>bzP{i`ww+H4BfWDvOc8v62R9 zCdJJayMgTMQDr<b2!#ibW>0|+Ob#iv&v2b#=dglGs zF)OiOwS$*j=FDFP8?Y?O3FKCQNEoRTAz8>p`m-CT4x}1bcNSI(6{Z#Ss~HwVY<6Sd z&(3OM1HTF9YKPZ=Wgc=Ceu2#pe-$h{XObDz?6NDP)LI9SKHbVlr2USl3)d|p(c@{C z!O3SZE0h%3Bl$L2%_zT_xCnbQYs+0iRxoESH6=I)92`9+RoYL|R6BS`lYxxF2is7L zTun9*I7j5(BkB{$gwTrniRo=v)BCP|=L&yKlUg6QjGY&GPTP!8YRYSjxT3v*x3!#< z{g7l->jI2JY{K+E8PCWyb$*l)f-7a2L{d26?RQK4vYC&NQ4MHElLuuk$Mwk_9M6`+ zvWot)sU8U#^4qDtW0qIaArOZDy|DjQ XTQj4YA!5KL7SsiV4d0RNpUi&%z>ilc diff --git a/summary_results.json b/summary_results.json index 14299121c..ebc4a104b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -12,9 +12,9 @@ "Scenario": "Offline", "Accuracy": "ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node": 1, - "host_processor_core_count": 24, + "host_processor_core_count": 16, "Accelerator": "", "a#": "", "Location": "open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline", @@ -41,12 +41,48 @@ "Availability": "available", "Category": "open", "Suite": "datacenter", + "System": "gh_action", + "Platform": "gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "UsedModel": "mixtral-8x7b", + "Model": "mixtral-8x7b", + "Scenario": "Offline", + "Accuracy": "ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Nodes": 1, + "Processor": "AMD Ryzen 9 7950X 16-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 16, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline", + "Software": "pytorch v2.6.0", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.0148696, + "Performance_Units": "Tokens/s" + }, + { + "ID": "5.0-0003", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "datacenter", "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -68,11 +104,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2068, + "Performance_Result": 10.2786, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0003", + "ID": "5.0-0004", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -108,7 +144,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0004", + "ID": "5.0-0005", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -144,7 +180,7 @@ "Performance_Units": "Tokens/s" }, { - "ID": "5.0-0005", + "ID": "5.0-0006", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -180,7 +216,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0006", + "ID": "5.0-0007", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -216,7 +252,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0007", + "ID": "5.0-0008", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -252,7 +288,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0008", + "ID": "5.0-0009", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -288,7 +324,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0009", + "ID": "5.0-0010", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -324,7 +360,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0010", + "ID": "5.0-0011", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -360,7 +396,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0011", + "ID": "5.0-0012", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -396,7 +432,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0012", + "ID": "5.0-0013", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -432,7 +468,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0013", + "ID": "5.0-0014", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -468,7 +504,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0014", + "ID": "5.0-0015", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -504,7 +540,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0015", + "ID": "5.0-0016", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -540,7 +576,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0016", + "ID": "5.0-0017", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -576,7 +612,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0017", + "ID": "5.0-0018", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -612,7 +648,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0018", + "ID": "5.0-0019", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From 2bdcf1662e16e77340abe22355a7262811174953 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Feb 2025 12:20:56 +0000 Subject: [PATCH 088/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9762 -> 9761 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2815de191..a2959be0b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-15 08:58:40,204 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-15 08:58:40,204 submission_checker1.py:3270 INFO] --- -[2025-02-15 08:58:40,204 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-15 12:20:51,463 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-15 12:20:51,463 submission_checker1.py:3270 INFO] --- +[2025-02-15 12:20:51,463 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-15 08:58:40,204 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-15 12:20:51,463 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-15 08:58:40,204 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-15 12:20:51,464 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-15 08:58:40,204 submission_checker1.py:3286 INFO] --- -[2025-02-15 08:58:40,204 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-15 08:58:40,204 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-15 08:58:40,204 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-15 08:58:40,204 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-15 08:58:40,204 submission_checker1.py:3307 INFO] --- -[2025-02-15 08:58:40,204 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-15 08:58:41,213 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-15 12:20:51,464 submission_checker1.py:3286 INFO] --- +[2025-02-15 12:20:51,464 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-15 12:20:51,464 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-15 12:20:51,464 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-15 12:20:51,464 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-15 12:20:51,464 submission_checker1.py:3307 INFO] --- +[2025-02-15 12:20:51,464 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-15 12:20:52,454 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435098 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9484 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index a20e4d18a..b7b33c442 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -715 +716 diff --git a/summary.csv b/summary.csv index 09121a719..8e13c8a50 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435098,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9484,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index d66078d1e..2d66a76c0 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.435098, + "Result":0.436634, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c2ee3f4b1ecffaa4276a63139b8f9686c047c442..be81a43e9a3b62f52ec54b8f52af9b273d1109eb 100644 GIT binary patch delta 2384 zcmV-W39t5|OrcD$SqKDv8iIV2T?kYHd5)8_2*!V){k$B;WvmU<&|gf_tLWjY3Psqi#(rEhIYLi$eScM9E z9V-m6iT3KEY1q=rkH^zQtz#OiUY8^M=%;@BFpeIyc-ccT_wyqo4y4R>OOUsI%bWAD zHP(L(SI-SvR!6?~?%U>J<33NkR(z)+*R9CldrT1 z@sd=Eil#?OeO+?>y5t5W^tcu#*{RlIt37|RAc4Ev#W5<#Fi%h+2n!z>2|0g0F%q)l zKQj^n{uf3>52y~~6gq&s_MnVqG86zP(x64S#-aSW&Bzw2VNXXv3U?gPkUNRE0 zcfT=`CV#rw{OM*x0*e%Lj1rsd`xBIyq~4E=gw*?qk&t>nqr@dyzC25qkHoJ`59EIw zIV}k9Ja2oRqRwbT7d!CC?mVY4J&d7q&IOZ}9D>W2aSz*H|9koAs(acyd%s)_CE0Rl z6sF_FT|J1Vjp-!&JI$6l*{ivVhy8FiRB^UoQ~gvWX+K%qO~WK#`y~B)t8O&XSgGYI ztC26J8t2`Ub$qOnYz>n%){B|iCUbv3n9pm$d{K*Lv&VQbOS9EfZLR7D(QFuw^k4LW zO8sz>=pPOnLuS7IRz*qKs#Rq}TmH9aT;EsY600>vy=h8Q?3ueA9y$l1Bd6LiN^Ekb zo}dJvV*C3WG;+*-ls_)66lWambKP7!ANOB&T@%XH$I6iFJ@*amtUH1_YtEph%XEUzM8KiLMBy_L)G*IP zDt750l|TQX$QyOei&)p& zcVX%6;ga#{P&*H!aj2h%(IA&496g){xMVcIWs#|eO8{IJ*?I){dZ4Atbb?%hwTFqq zMJ#HVMQp|Po_`0-H)`_vF1B4hf4TGAw#_|X_mBoprbMHNP0c-w#--*SMgwGW8cHUo zp=5FzN~SC__iza)nMDaY63ji&(&aj#&qW}r$3y{{j0VY+6`OlrBlGV*7^$SbGEhom z7fQo3+&y3UlVT5)=o7K2w};WV)Z4>o5XutX9!>*LG8%xg$lJpu04R&Re?1bsJrcbkNK5`${bGQf|6+iV zCkuvGi;2?Gr@3nB>T#YOK%u-?T^l?OSHBgY&EqhN!}Mv@d;Vhui;3T!MuYduK`mzC z9}D&VUGUC6co*0oe$nIyt)*q9KTEUfmc8UJxU-NEz77Sm*WEoNlT#onf6Kw82-|_S z$JNY!JG-b_?p2rkg7ebo8j&nV321JF)Y}@p zx2yC4B|b1MSZ6fUXaJ9v&0WPU<;KCQGZtKs;FAc_o^o5GT?m$9?D-B9^I3H0C~k~X zEX2}nv1R<5Z2=c~evcIdePH|7l$KvU^U7@DPi!#lNl$0$g7-*5xl6)OTUiF`W z>Nw=z#1ml3efcB#0)-@~<`xvwbObZR$U^bW_(J!a+LD6KmseFi`o9*0o@fq|x-Z)h4k{unHCS zI#w8B6YbSS)3BwLACISrTE{e2y)H-i(NF#MVH`bZ@v?_x?&n8F97vh%mLPBamN(~P zYpj18uAUpTtd4x|-M7ud#(kc6t@utuu4%&!ZMfZPXk_Er3`*ibNj3&!cMMlt(QJ}o z0*SRNxdsw@S8@X+&aUJZNZgHNr(6@4YqA7_l*<)loZnwnT)(Qgc~x;+q}Zvni7Rcg zcu6Ydil#?OeO+?>y5xpS=y5GfvQw=Et37|RSORyqi({@J!#v>%L0I?*B;@@01SDj~ ze+Ci){udx2(0v6G0^KQ)kkhOKB;??n0SVc;T_7QQ_nb>4*}FX;A$#`%NXXv31QN1$ zzX3@TpKdlj-E5Y?BE=kYiB0zX3743p-j6^+>iq;Hq~6b5;*u<1o+aQT@hj+ooFjjy z1>v3NZO>EG8ExqD4m`3u&uL7L$Iv3DHh529&fI?4V{v!zb|MraQ`)XWbwZ^D7O=*ffbGL(`b2xP5R6FJpo1Cd9 zT*9CO5(*uVQ0Rb!e1v~-3ZUcRE`_cjyaU3?C-T{LIH|7-22Vco&mBr6kD-6-w+k-u zh~(0#kjT-v?+5wyOf9>7S*5CtG9KqoX%q+J{=KZ*b)yz6glv`|v5Hbkka**4k=V?x zZ7grk($d8sx*SAu{v30OP0pVaE@2=72?Y^ID2PBpL39g1hNv;UY&YH1bwP-##*|d6HihqX;cqI(diOt=zO^~{I>|@)?}Ge zDUG`L0W7PivCOQw9++6)3$9y^xerVH)agj@2RW8avE&*@QY^WKV+k}2OQ2y`at+5) z7GVj05-=01*yegtn)W^} z@#E1W!Jj%>4&{<-9Fua%HJnSJVO#^*mQ1b&WxB>1y&i$2>18mCE-t>W5^M}ZbKb8K;uwL4$yEWOPF$yhA|0jIgH67Uk>Vo zFaj0hpXgHN6Tsug^s013eb}cgQpahJ{BI}Nz&YCmU(gmIHXClm@1ETmd5v~Ex zL@IXaa4LWPLy9(NCxz@0-4JV4`62M^G2EK7KJkcMG@2{a7L zA{!4%z_2Vb@(Aj)Ib$td&QFlm&^Xl312mk=5{@3EVO#{1iegZf@b<`xy*;lnnGSOt#SdLoR#ID;)vMh0^kfSFuj$gSL^ig*={{BQ zNS&z3B+-H>+g+>3P3!9G|5xJRcJ)p97kSoSG#iAux8I+vzSp0IvyqnkvHHaTJO9N1 zBTp6#uND)frB8F!($(WaJUc>pvAQ;R9Ik#VK%2*56o={4s`vcI3KkQ;J&gwMmxEf& z!ao-3{k!0ueef=@Km4M}4_ZshN`IDS)h&C;U!b#)5xx!uzOAqAC6iMiDu3CjOAxjL zZON&DLQ`6L6VfITwSJJ)I(hrbj^iYdQ;(~e{dRUywb-jR`32{t(KX65f)dc&2&uO< zdS5ToIZAwBTCmn=sL=sDS~PbRx0D+Pug+L-L4r>rNPEg{jdmeeim~TAP|Ro1p`*Al zPO%V6x5bw6Z?*+o5b}T(1b-%2V7LdSb|Rt?Dfoo6Kdv1Bf@5ewfes$$S&q-%08aUv z4TEywR5Bba+_av=$-Vb-GIm{;b+f@5^XK?;_3}2Xno8ZDiUVpY!6|pZf^kiSFN=7% zmie%XC*DX9kB6w=cpDcv_JK%Z0D6pjQa)x+&+8>>3R09QDbl=LlNmWJ=k%U@^&_wP z&p Date: Sat, 15 Feb 2025 12:24:25 +0000 Subject: [PATCH 089/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9761 -> 9763 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a2959be0b..31d4373b0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-15 12:20:51,463 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-15 12:20:51,463 submission_checker1.py:3270 INFO] --- -[2025-02-15 12:20:51,463 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-15 12:24:20,564 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-15 12:24:20,564 submission_checker1.py:3270 INFO] --- +[2025-02-15 12:24:20,564 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-15 12:20:51,463 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-15 12:24:20,564 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-15 12:20:51,464 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-15 12:24:20,564 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-15 12:20:51,464 submission_checker1.py:3286 INFO] --- -[2025-02-15 12:20:51,464 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-15 12:20:51,464 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-15 12:20:51,464 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-15 12:20:51,464 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-15 12:20:51,464 submission_checker1.py:3307 INFO] --- -[2025-02-15 12:20:51,464 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-15 12:20:52,454 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-15 12:24:20,564 submission_checker1.py:3286 INFO] --- +[2025-02-15 12:24:20,564 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-15 12:24:20,564 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-15 12:24:20,564 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-15 12:24:20,565 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-15 12:24:20,565 submission_checker1.py:3307 INFO] --- +[2025-02-15 12:24:20,565 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-15 12:24:21,506 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2786 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.06 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.60213 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b7b33c442..05c8b2c09 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -716 +717 diff --git a/summary.csv b/summary.csv index 8e13c8a50..a4aec6a0d 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2786,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.06,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.60213,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2d66a76c0..17e8676e3 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2786, - "Accuracy":"acc: 74.400", + "Result":10.06, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index be81a43e9a3b62f52ec54b8f52af9b273d1109eb..9c038c4b750e08e7a91c6f023e8863d0c73df367 100644 GIT binary patch delta 879 zcmV-#1CacoOruP&XA=eJzwFavvuqQZ0)MU{?+--|;1m==N)Ra}9flQ@qNBXlh4`lf zme-$5E1LGaC&gJ2UoJO!TlaQ=)sj(wm4PA7j;+}YMPrO)(7veRv`&ZMA^gN5U$TBL5hqlEi-ITH~q&X-(tA8(- zx){2;@mlT1QH5>C?8&+}`+Y&$_Hg7RVtlu`zf!5X7F{L9UP|`oO%ekXydWiq)>4&-&fNQtbc?KHj&gsxZ~*hlx&uD46nN-jyh)7jxO-r2>hJA zi)JQY2id2^XLJLM*1tb}-F8CvroFJDG8Xw_j+JNMx7lQNgihdCo&=!9)eUm2P*Gwh zIaHAz3j=KnW6xgHBvyOLW1~V%qf+b%YL2dZ?o^L1KbjvPCp6qpnUJ1FX=~fC-C$=* z_YBYT{GCzSHr&9Hv-ZjKe0e%Omx!m#RpZmye0e#(Sk&{iXAJznlU{Ata(pKoAap#N zJeuwL888R~`IfP*&4iA2)=kDcpXL?dgaQgRopumwVuVny$fnEc3oF=lir&06a2M!eeYMz zl{uVB0BR~JnDD@(^-YDZi*&e_hqy>5UMrAK`=pP&ofbLvfy`0>c8q&gJ}1vF%QeG;a6VHHS?%Pk$`~kC*Ax#4X=)dgKW0U11R{`#m zJtaB;V3T+yCL6Q_TTLYa002+|000pH0000000000004l2L6f;9J^_`J_$4C-M!R delta 877 zcmV-z1CsosOrcD$XA=cYcFpu+vuqQZ0)LL-SgIVrDJX)JAW}*?3@a!_M|rIa@lOXV zuRobwdYmRnRbP|(M!k(lUS!bG`M#A|5xucjO6c@}MvWmFUoZHrO5DP>F2nfDCi{)ynVWPe|T#Wt$v+fsR z2nDm(-903eR3IvU%fY1x+kv*^)IgywExie8lZaYBNNOFweHA-#63D5?)y#f7yQo_3 zRhRsN^U~-Vkt|0EXl{hm+Zw&MtMmaSJ}@m|vlL*qD za$BQa2$o{(`3@BGS#;a&)xt|`I`-+a^X}m3>I!$PvY#}dpR5XzR&u_XpQ*^{#?JjjjN_o zA5O&qHI?9$J7B@Mroxv+JY2_oSjQ7@B#6f$>Nnnwi%fkWk{EzaaZk#};_10vp{B@- zGR=#Wlr1S3Xpz&Bd>uw!^`C+2ION~N6JW}H{Rx^svy~xD0|iZX&GcfE=OkAF?UO$x zIsspkdLbPyzq|5dZ)H0000000000fPp}hyCpsWlau--BL+ny00000 Da1X8| diff --git a/summary_results.json b/summary_results.json index bb8ec779e..8a2b5fc6a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2786, + "Performance_Result": 10.06, "Performance_Units": "Samples/s" }, { From 1766b86108e1b5b180d44c229035c1c920394d4a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Feb 2025 20:54:16 +0000 Subject: [PATCH 090/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 9763 -> 9762 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 31d4373b0..651827882 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-15 12:24:20,564 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-15 12:24:20,564 submission_checker1.py:3270 INFO] --- -[2025-02-15 12:24:20,564 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-15 20:54:11,305 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-15 20:54:11,305 submission_checker1.py:3270 INFO] --- +[2025-02-15 20:54:11,305 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-15 12:24:20,564 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-15 20:54:11,305 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-15 12:24:20,564 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-15 20:54:11,305 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-15 12:24:20,564 submission_checker1.py:3286 INFO] --- -[2025-02-15 12:24:20,564 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-15 12:24:20,564 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-15 12:24:20,564 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-15 12:24:20,565 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-15 12:24:20,565 submission_checker1.py:3307 INFO] --- -[2025-02-15 12:24:20,565 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-15 12:24:21,506 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-15 20:54:11,305 submission_checker1.py:3286 INFO] --- +[2025-02-15 20:54:11,305 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-15 20:54:11,305 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-15 20:54:11,305 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-15 20:54:11,305 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-15 20:54:11,305 submission_checker1.py:3307 INFO] --- +[2025-02-15 20:54:11,305 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-15 20:54:12,591 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72773 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75139 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.06 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3089 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.60213 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432318 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1596 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352822 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 05c8b2c09..ea5a0a500 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -717 +718 diff --git a/summary.csv b/summary.csv index a4aec6a0d..2183c827e 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72773,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75139,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.06,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3089,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.60213,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432318,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.1596,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352822,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 17e8676e3..ed34cb596 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72773, + "Result":1.75139, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.06, - "Accuracy":"acc: 75.200", + "Result":10.3089, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.60213, + "Result":9.77691, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432318, + "Result":0.431573, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9c038c4b750e08e7a91c6f023e8863d0c73df367..733e744d69338079e66016449c895c1af27e957d 100644 GIT binary patch delta 3298 zcmV<83?1{MOrlJ%SqKCb{g4ckT?i(BTXW*Z7KPu>uTZJxVWuhwwS;aiL(2Btj4}3j zY^Tl>C5r}BL5N7gmwz7#FowMsvND-Ou5A#l?DKjNcFiYEi>LknN z2hC<0Ol0JzT0EDt{KzOyB0tODCgWy0PozIw21V1Rq3d=t^23?B%5acW$q|o6VIV)o z!6K5gY?UODQ$NdxpH9Mgs&)>CVScQrM@i(UtsaSkliv)XHamyQzHdYMkZ!$Jic;?4 z`1j(?>9DQpg|x}EpO?e9Oq8L29QliBb`?K-mEm}j<)_UmLV|djF8@`cFh7@uszm-{ zTQybXAsl9tBC#?FheKJMXjKUoX%o%n*Retun`loHE!{S(;`4Z#%5_Y0 z)$4MEAN|bl947IDk}P{j=YH|Yhy%mOPfJj=ek+=bu{GBXSI-SfUPp0%_U=38Ve`I7 zyjFatA=k9whBn-8H8k__YzAfEK^bfe`tBI6xT4h}!vqp*S8@#`_O9dxNSs~CEs(ey z$xgW@F4trU1Syv*$T+{hsyO%&h&h9rLY2nMw#+RGT5?G{|V=l4D zxj*3&lhpeWNJzb(fP~chnM+)f<;$}K{7L)@dLY-xX-RnZ`PlP+6m>=$db|UVoX&F^ z)8{dC&ba_-$wzScGVW35>whmlUG+};XYZG*p)_9(jl*oRxYL3-YDUxi-zZ<|4<-L+5bl$fb77B{sQIPq>6Z2P70aAfeC!3HcNL#VLV~ zhr1NIlJG7FCx4O8w!=w%JurCkH~-wBMDiHAe!Jijk4P?mof?T8PX>Na+|M-f+m}_U z#yID3{*=W@Fd5t%nyu-LU?GfF1rqBhr2>gJ&K8NS{NBd$_AD)345G_HB-hU|m)PX` zIpGopB9KrJfrNqxBosuq1VkRrKr9LGf^Z6=TLK~&JO$CEAo7?NiMiksk4P@v{~ZwP zD5V03x9-k=lxSLpyS19#A<@GmdYnXZ^&fMIO>ca-gpmj<dt*Byx#MZ+yHG5j5hRm2W%ZmWBMS5u0w%@d<2(Xy|6`o5T}i)zs&IH+CO_uT1R;m0KD?H z0Q5A+u|1(b|0ghsLdPgB+X;(gnmGm1ejAyL+~K1>SQ%5`2%{vM7~Y;|NM6*sEt$0u7@QXc(1T!>JTy zRDuM5jLMP(T|u8JoUxW3=)^N9l{Bi4qUdyqG<3e)8va{^a%-|Is5Ep<`~a49)L3QK zTn|jF_kuOcG52AKpE?~0{vgM)DVAL07!*sc;aCCF1qEr)W+ zHI7NSVqiGC3vRIxisH2 zmt9f48o>lOjo@`Y4Ja$u&Z!&#AY@Ukj3pyL=*CteGS{X7|x@ zt}G@=;c3PX5Lricl@_n9?ZvC+Si;<6H4%HfCW6}sERPb&HI7Az48o- zlgyn1qWHpj+}lF>J@$w|3oe?C3940zVG5>;R2JJv%^u!>O#` z+CdsdCD5?7YngEeC16yRS$70=)||1H9_WNW6JZV=5XGN~a1D4SQnO2kQ~C2Bio8+x z&b8RK@7S8*S$lVR+&x$VcMdi10F6T(JV3*-tl;568ipm%Ff7Y#JSYLfvdqXMsL$q% zwe&zIeD{_4bx;(WEP;kCU~9I2?7T+h&)I!_qh_8fvF+w57O}3k$KNWvJzN5>4z=?D zjYIuBK*PDL;OIdb#wE}&F3U_kC;{WL%+@2AuZOktKqs6_X6*q{d=bkvU=drhz31Nn z^NpH(zKd;_PvJbbZFA4pJq#U_X>g;6P0c+(<5F`E&@eKQhLVXiluV?5p=26m<{p%Q zl3A9ZYcO*UYw4j*=yMSk)dx|GOrYUp8a11HUL*7GJ{YN`zA8`}`Yx2Zr@MRhVGXed zO70V}skaAcT2&%IekF_VnZn0I%iJIkLI+P4`G9V|gN{(^LuK{B*4% zH?6C$|6hrN+toMaU*vgz@oX3t-hO|&`d)t)&c;gm$LbdY?BW*#^dea>x>`)7l0D63 zTa}OV{1YmQRh8l6X!TnG$~+0M#A|5xucjO6c@}Mv zWmFUoZHrO5DP>F2nfDCi{)ynVWPe|T#Wt$v+fsR2n2OCa5a-tAS-{{X-W~c18vEvfkItcaud=f5w(7h z)H;6q%1-Pg;8TyQnf-QlQL#G8Ci(^IgjOZWk`zUtnC3$5OY~8%<3|)ZPZg()R#2i2 z99mU(1v7+c3vX5%uwH-7@LWR&PdKCr1`PU0N@Nm0}_;XI7>2o@p`b*%{C0ml~W4e8JM+>;^aO$ zF&W#oP1?m^4f!+twRwFXR*i+~PsIRL1!sg=pkCXmz}H1MTtl7TgcI)s@TV^5GjHcb zPJO_m5P+QGj>_la`K4Z?Do^ueoaS-1tn(R^EbmE{Z~Kuq{THA-5BWE71e|i;euC-` zvy~yr0t6NPkPMS!B!2~g)?JNaljkH?0qv7NB{~6LlX@j48+A2sH6;N608jz|01*HH g00000000000Dyr≤B80hE*aB_jq#BLDyZ0G(+`S^xk5 delta 3299 zcmV<93>@>KOruP&SqKDv8iIV2T?i(B?Q^2Y7RJBdzk;IX!%WqxbR%!6=vI<>i!mmX zNwRf6Sp-|9R>cL(%YVP1#^`yDgq@vTQuoe>j5g0X-3R#D4Lo}I>*I8y-K#hWqghKA zbwk(GEQki-Y}C?ku0PjYT}x7bHt;9WOtthUmFT~=|8w{d#lI(GrBW?RFiTp0`Z!JJ z2aQG&jMdan>d{=yvMs|Xp89F_HXb#Sd93`wGH5z!NW-ujQ$L*Ps|*KmnH8WcgQ{hS|Q9x;FJ6 zTe_)h58)sk=ZUp(I2fpWN9$U!NYZHf+iH_oCs>6FdmSqbv5EHTqG{OD%8$p>M6F{Q zt6rBQ{OG5C`!J3kw0PM=GWYW%BMzj@c1w`Ae#@Klu{G8WSI-SvR!6>n_wL)~VdFke zyjFatA=k9whBn-8H8isEYz8IyK}i^cu{(w$S_Y(AqWc}83{RmJ~0xq<3BSJ0{$09LZJJ~NC8g9$JA1!e4JFxfXcVU7#a%s!rj6+&`#a5+I@zna ziiiDhHdJx8U{n26C22od+)cwIU;8Bed#i3V(pahGDyxw%rW)tnlXZNol57o=G}eom z+9q>9n9pm$d{K*kX0yk5F-x=6RBf&52hnU8j`UyjflB>wlIR}}8$)Kk{#Hdv*{W4# zLtFm0XI$S`;}WYiM!jiDQ|y_$9UeLdp(Cf-F-mN5rk*C3WG;+*-ls_)66lWambKP7!A;W$Z05!oQCqqXk0oqZZsT`Tmp(@ zQG(89;|5A-=`x*oB+ofS^_VDn$MJ&E2!kAu%dPRh4K`n<#GK24QuFM5j1$$r7j-wA z75_4wl~h;)&+GNTR(v{j+cn%hmyCkH=YAyc9=l~xDpBJIN+olz9;jqAKqaFADp7+} ziXtk1nFK&(L4uB;PZiFfrOR~UNlGP+>TywYy5ux;zT6uATZD3JvdpQJMqT^>mQ~bP zX4YH}Osww(*Dc4~hb8~i=}6#%9LuIyqQ;RFOVl8ij0Uh|G=L>)5KCEvC6fTKEJ)DV zY<$6zwse_J5K9`>W1`@0Dx(pGpj^uT11_t7sIkl)*7d+P*OSt;_i@QT9z7EH)X{P% zm#A?}$|Y)$OGX1+G8*6#HOQq|#3hpexGYG}*=&5_lD2fYO6ZLoi0Uy>$R+bk8@V*! zG?(PQ&R<8eitG7yKN9$C+@jC68I9AVNTLRjWHf*zqX8sQ<60Dd z$z~CfOegfFYf*yEX5$Nzw57{*;+Ygl8r9>X07=vcgFdI;8h<57F7C34aIt2R@R;6D zmvd!4NeZtne}Kp;sw=g4ZEi1K>yC7`X0Kf$_jpYN+6OF;5{VkeqC}zwiDWcDB%=W$ z8H0>XiL{D{WI6#N3lelT8()Z|EnTL66Sx5jQ9UjSh-5TyO=-O$ApeQGu$2^8ib-pW zNom#%!?gA=xyO(r7`qL1KG8laNVz}mIQxWgp?DvPW;f;wx?pry-ng3m<2p~FPsGZEA<&qOMA=^&Lq|Dnhm zb?;n@ZTpU0m!7rvE|0qhOU9i;4Lpp-p$;BKgIJdE@NgQylFFwc?@#;`J z52JCYpNG*Pmn9rMoCdgLG{9w%sfSAdTo&1S1o(QOrOR}JT!OWSiNZxJYM4cA#rB?m z2h2BW^7$^dT|T+<+_udj$&?kFdtM{+?>-o*q`opxN@EvF!!z7HU-^?_ z50vN=v8lI*(YVyx!)Or765bw815h#=fU?Nj!zBPHi@ZG&ygks;WjgVi6iV>+a8Uqd z32%?A*xU0Olj$(WQT)(l zuH8jx8cm!8asJNKwP6&c&u{WwviTK<_MfxQ5tso5=)dgKW3y}%n*x6!?+--| z;1m==N)Ra}9flQ@qNBXlh4`lfme-$5E1LGaC&gJ2UoJO z!TlaQ=)sj(wm4PA7j;+}YMPrO)(7veRv`&ZMA^gN5U z$TBL5hqlEi-ITH~q&X-(t1o|-x){2;@mlT1QH5>C?8&+}`+Y&$_Hg7RVtlu`zf!5X z7F{L9UP|`oO%ekXydWiq)>4&-&fNQtb~6KHj&gsxZ~*h zlx&uD46nN-jyh)7jxO-r2>hJAi)JQY2id2^XLJLM*1tb}-F8CvroFJDG8Xw_j+JNM zx7lQNgihdCo&=!9)eUm2P*GwhIaHAz3j=KnW6xgHBvyOLW1~V%qf+b%YL2dZ?o^L1 zKbjvPCp6qpnUJ1FY1?YB-C$=*_YBYT{GCzSHr&9Hv-ZjKe0e%Omx!m#RpZmye0e#( zSk&{iXAJznlU{Ata(pKoAap#NJeuwL888R~`IfP*&O3dkR^}_Ot>0tV{{DTAhw`+D3Ze@ac#ZkF_nIU9T(qx1DKZO z1J)3Kx#W@K12A_35uK1y3`qCm+a4eVh88rK7%(XazIY>e9Yz~^<;tm5+&dVxp2fkv z3u-WST~~CI-kS0g{Iz&}?^n&0Ih;xWYAPw1@W7+>O@*(Ebhws>xJV~nE09n7q>sFv z7CH8T%u)b$jC)o-C(kd-IcjK0r+GAJ^_`J`6VL;M Date: Sat, 15 Feb 2025 20:55:33 +0000 Subject: [PATCH 091/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9762 -> 9762 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 651827882..44821f3ad 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-15 20:54:11,305 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-15 20:54:11,305 submission_checker1.py:3270 INFO] --- -[2025-02-15 20:54:11,305 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-15 20:55:28,192 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-15 20:55:28,193 submission_checker1.py:3270 INFO] --- +[2025-02-15 20:55:28,193 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-15 20:54:11,305 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-15 20:55:28,193 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-15 20:54:11,305 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-15 20:55:28,193 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-15 20:54:11,305 submission_checker1.py:3286 INFO] --- -[2025-02-15 20:54:11,305 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-15 20:54:11,305 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-15 20:54:11,305 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-15 20:54:11,305 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-15 20:54:11,305 submission_checker1.py:3307 INFO] --- -[2025-02-15 20:54:11,305 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-15 20:54:12,591 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-15 20:55:28,193 submission_checker1.py:3286 INFO] --- +[2025-02-15 20:55:28,193 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-15 20:55:28,193 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-15 20:55:28,193 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-15 20:55:28,193 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-15 20:55:28,193 submission_checker1.py:3307 INFO] --- +[2025-02-15 20:55:28,193 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-15 20:55:29,193 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index ea5a0a500..0315ecf59 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -718 +719 diff --git a/summary.xlsx b/summary.xlsx index 733e744d69338079e66016449c895c1af27e957d..67329249976351a34110ca479395aed03e12ba04 100644 GIT binary patch delta 311 zcmV-70m%NMOrlJ%P9OvqRVKibQXnjU!6gXWfwtt-K%p)zy$NZPh+02LY8}6QWyf(6 z$SKFw%zit&s9YUn8~uTGLaPd8Ns1y+Omm@j75Zq_@e)POQ^jek6;!APhgS7N$qZrI z!kg6wtQX*-08)-HQ=z^0hG6X29wc>1aHt?|wUyL|((bUK?1%0E=V|(gC3wnzsi$}V z#%3m>3CY=vG{3eT0i0oIL4xuQXGw<7UJq8fn+>CK;grJn25wqU;_N;+F&n$COS;8q z4f!+twRwFXSB-@lPQ?J#C1-?LpkCX$#Fs@lTuYtbgcENC@TWfLH{Pa2j(xzR5P%%x zo|MnU^GmZvb)M!$oaS*>GqV1OyjV JCcu-UC4af-l>z_& delta 311 zcmV-70m%NMOrlJ%P9OwzHE=bPQXnjUX-W~c18vEvfkItcaud=f5w(7h)H;6q%1-Pg z;8TyQnf-QlQL#G8Ci(^IgjOZWk`zUtnC3$5OY~8%<3|)ZPZg()R#2i299mU(1v7+c z3vX5%uwH- z7@LWR&PdKCr1`PU0N@Nm0}_;XI7>2o@p`b*%{C0ml~W4e8JM+>;^aO$F&W#oP1?m^ z4f!+twRwFXR*i+~PsIRL1!sg=pkCXmz}H1MTtl7TgcI)s@TV^5GjHcbPJO_m5P+QG zj>_la`K4Z?Do^ueoaS-1tn-u@EbmE{Z~Kuq{THA-5BWE71e|i;euC-`vy>qV1O#<8 Ja5a;oC4XnmlUe`( From d4937c071bd46c0d96963162788ed97134bf48a0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Feb 2025 09:28:52 +0000 Subject: [PATCH 092/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9762 -> 9760 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 44821f3ad..fac21a5df 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-15 20:55:28,192 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-15 20:55:28,193 submission_checker1.py:3270 INFO] --- -[2025-02-15 20:55:28,193 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 09:28:47,516 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 09:28:47,516 submission_checker1.py:3270 INFO] --- +[2025-02-16 09:28:47,516 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-15 20:55:28,193 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 09:28:47,516 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-15 20:55:28,193 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 09:28:47,516 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-15 20:55:28,193 submission_checker1.py:3286 INFO] --- -[2025-02-15 20:55:28,193 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-15 20:55:28,193 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-15 20:55:28,193 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-15 20:55:28,193 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-15 20:55:28,193 submission_checker1.py:3307 INFO] --- -[2025-02-15 20:55:28,193 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-15 20:55:29,193 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 09:28:47,516 submission_checker1.py:3286 INFO] --- +[2025-02-16 09:28:47,516 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 09:28:47,516 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 09:28:47,516 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 09:28:47,516 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 09:28:47,516 submission_checker1.py:3307 INFO] --- +[2025-02-16 09:28:47,516 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 09:28:48,507 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,5 +38,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1596 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352822 | CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352765 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 0315ecf59..a1708fb1c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -719 +720 diff --git a/summary.csv b/summary.csv index 2183c827e..091138169 100644 --- a/summary.csv +++ b/summary.csv @@ -16,5 +16,5 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.1596,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352822,CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352765,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ed34cb596..29d56784c 100644 --- a/summary.json +++ b/summary.json @@ -519,8 +519,8 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.352822, - "Accuracy":"CLIP_SCORE: 16.378625825047493 FID_SCORE: 237.9398950209584", + "Result":0.352765, + "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 67329249976351a34110ca479395aed03e12ba04..339de32d2b7228304e3da8b6b7df15554fd4039f 100644 GIT binary patch delta 3980 zcmV;74|DLMOrT7#SqKHyk!IpdlU@ike}Gy-HfJ z+;CJSN&WfIpGI@pQJ-X@{@VS|;X@Svo=l`nl`O$L>8O)5T^zJpNidN!KWRn_e>u;N zjG}nvr`g+h+)5U)^oOgU*|eo=n$?>5;apv3IEbs{h(@C@kRPL9Ig|5rog|i1Kh1`p zOu|K?b`OVPb}X+)iRGxH9*KjK-*ln2x`(U2Z$tTz?7UX;QtqPY_x#Q2u%l|Zw8gZa zSHrlBm7yH@%V~NQJ$#knc#>wPf6dH8f@qqo{#9mSb}n^QnfZ?$)lij(aF|Z=#L6Tb z4rP9#RV7#^X*Bz7eMoE)tV4yhi4~gIM!QYX)-2u3KaZ!W+{Cojy{<<1(NF#EVH`au z@v4Vp;pd->IMDU%v;=wUx4bzYTWiyB_1vIjb>wI7zH1z|?(@VO#djKVe@z>1Xv6JJ zLn|B4c2GJVl+MPW?T_J#E81-`Odv7$CD%Y=?MrTe#NLAS`?Y5_0`~e*zM6;y(ik0sjk-5a_-F34!huNXTW@0}}G#oB;_rxgL;^ zvwO}ZIyt+2AR%Y>0!YZ&y#x|+cE15h8((e~zT7O9z$C>SbBRUH{Rx*Cq~4D}LhAhl zB&6QYT;h-{U!Eo4PvTe51Gz>{OTv54$DXICGuq(s4qS3N&uL7bf5*@{=K`c9AHn6z zxJTWu|GoTl<(>A=-mg|eNwypshv{T_*9@XrYc|dPon=d%?A1cXqd_W9PD2+Y^t>L@8&waRR0tN#^@>-&0KV!g&_ zwhdw028PbY(AgY1a;Y72iAAo|6E0!U0SSc;NGNnbLjHt*u}h%i;SPnaB)kX0$zSBN z-EdN$2L@06=AYY?NFGDiZx>wR63L}qBa!3Dzz_2KnMQW|f3i;17-u{#p3*1|CWCvu zX*IP*uoQZ`0*Q5$Qh~%9dxyk!c5h?41xrf@gXnM&$@O#0B^J4UPPl}D2qY9lAfX@v z2?fz90g;C@5KF>)Ae@5elz<2ZPeF7jh&-lEVlKGEC6Y_$e+R@mN~r+it+O{J8m4Yr zCMMCvB)XhLe{%I7bBRT7e7J;>2qct5AfY63i9>IE+!7K&IJsy~OTv3D;pB`qc)SC) z4y7@D5JT6YKteu(OSfLwW_}PQjj6xN@aR}SdrewLc@+S>a(4i9o3?GaLVNyCU=oFf zQCzhX7RfYo3^XeSTc-$-SGxsCGpRt=JrIKrGL8%1K>KT9xrMkUv9Dn%KUAOWMYBtcKmU4=8& z;(<xJVOf%(w>|iRC2jFQCmc%})dx}R zX)4eNO;9fN{{t@TsIkfu*3H1S_mjF|6>*8Te@Bl5-*q%?$|cv>2IZ1#IF~@fxC9!; zCD(8+jWRAl0>))Yg5LJv3zxLTLzU16IV`FVqBxgePn&aTylF1UbDh7AWF6&ILekhr z(h-(kfF$0>KN5Uz+@yQkKx4Nll3c@)1R91U&@d#q#xW_9?J^`mC-k9fS%Tj7;0uzp zf5iixxCTX%M)gq?Ly~KRM)#?A#$OAP%e!nMTyB^oJf`=v)m)iRlEQ83KR{$1)m2)& zws#k=P1_Vkfz?D5cufSi511|`l51>}63I23NT6Xv0u3V)7-TF;q*+EJ=!6kjlAyOe z_(CLY@jxf+0W6E^qbNos(6D<-^9=#{e@{Gxt);+9Oqx4P>PA!3jDmd-QDDdsz-~hw zIY47mOAgR*CM%e7kcKe{Y&nd{GG7krgfUs>%n{6*!&*Gh31^bIb3hbdxN;3xxYp^= z`B$KPt9~6p3_H4In?-BaJ;#prLM8CyP|FU`*wnKFG@Qx`t{tRdR00iKyOtSue^3HO zWtnwHP-o2!8V+Rz zZx7NiD1nASS?29Q2^f@R-X5KKdsvGHI&s?+O6KiBQ4Goo-X6VXZ_jH?rlTB3@k4i& zwbWK+^{VfBda?z8+ji(2f7#mkru$6BV|gN{(?kiP>~yUox2@~1|6hrN+x0i)U*uVT z(R>)@-hO|&{$76?&c{mf$NCooto#=Pv^-fbx>`=9l0GeDN0pC@>=Vk1Rh8l6X#HCO z$|4S<2V$4-(O*6UuGV1v70L@9ig_M zYPeJdM$A*?CMP93Nt}%xrp&KjhX$q-Cj}OekSe}&d_JG6PljJMS&HtU#Jkk*AN#pk7&wrIMnWeYhOlx4LKJ`9?kiob}K}2V46IIke#`_5*TmD(tgK$0en!Xf{Egz<*d9NX7~sRZ${7w>8HsW#w16SKO~wSt10K zZNbqUO;Wl_(nP3-?zw=hjh@T!DtDfjECkM(V!!xXVGKh^p`=1pri$_<`|Z4Sr+cWNe@0~;}I29mZHprSr@)}c7Hl% zS;k}?tZ$bz6tb=K5@IN2SAiZDm0sM-NQJk{J7T!fXaq5p#kmYw9w%#aZ&AKC*}fv* z%T=vpmsWa8W!A)UE-{&w&PY!M=S$>ta~pad|l-|!?=GVcrDo<&uEKD z++`2`)sEr7Aa9JMx^sMFS-zR=4RQSVHh7Ni{rliu7~eOhAJ_>UY=0uIiEu~i`Xbq= zbu6#DEsi>N*N!Rh+zR}hyo+WgzYfw*i_ho=SYH4B^mJke&P`jf%B3&zi#bw0`@YR4 zvm}>9H};wlVgcMNMI~RUQWwY6g`ek5QAl?!8k@U3N4- zKz3j_fwCbztsKC^4H=ND%Pe=n8ZI3@uA08jz|01*HH0Az1tP;zf@b1!3Wa%C=fZET&A%}&EG z5QOhMMdW+pB!wziNh$&AiAqQWQE{{EZEF$#$Tp;Xdz>^)Td60G>zVy_cI9%hmu>V5 z)(NdDlqD&OKrzjQ+E(bJS;hs5oTrM@Rx7B`0UTP?cO^4~e`yPERvWNhfQtf1Il@eZ zcHSF;v12=s)Fr{8g1FIEQXfjY#fGwPx&@r4=^RV&lv7V}4~)%3L?e>332A<8I{-Mt z(1HZz9nO*rpS>QebT=CY<-#e2KNz@aJ&BWh@5E&6x-RKvgEi#O@Ot(7KCBuG)t`z1 zs!PrYvp~JJNOg%Xi*UG>I=>1h-U#4NhoIkh8y7kD0gplea*TUYK4;G_%@Wmlnm@#8 z9%u6=eI$87?u)N}I}D90iHCY}Jt+}EF={>&$b1ONbZ5&!@c00000000000001hfmf6KBtZe{lRhOn0bG-LB_Ihf3y6^ zC{7|j%ikvBW;#!#KU@Yy)25;8b~EzBnYzkwkW|SLk49l2KgPi#lCx};B#~1;%ZHy% z!g;E84u@fWtf)sxt?Gre$+VxB z!?;Y8p&a>(X?7Jqe3jvNlI5q(e=0(Pc$zN%RiZFImxiiD{$pD;RplWZW|JbZG6{!6 zS)6E92^MJ~q zdr0Sg@yUn-!^lrdP_%w4nv1bD*9}+C4N6`|arW*z=3(=`NW4~jryYz+GD7_PXY)gr?L5^Gm-4J7ui*Uu*)e<3ISGmsGQzW@n=?kkWG=uUxzTxMM$As@~ekdTwx0}^s} z&$+}PXSWX|V;y`O-D z)ccuBT$1I>vjqG}{0e#?*T`u}c=!3(^AvSP8+yD0kDShP8q?=7e{{~d0BOlbaQQOs zQRnM_FF#%NPWxx?m#d*PUk;7KY_ho1f;ehM)BN8kU+Uzq<}w)#!r4eB`GQRjGMQ$B zba5AjX|eW62ltkyH?u^_5X6^j8+8_>nNoHi8sy`iLLzJ#`5+oEnN(v%Rwa9&oP(S2qct5AfY63iA!&Myb=;YIJsy~OTxPz;pB`q^mqqs z9ZF;RAcn3(frNYnmtMWFMSc*cjj6xP@aWn^0ol<$p+Em8Fo{CP zC@$Lxi)5NP1{#M9=>%vPjYvarL>kH?(71GJ+-MMye<%S(vMfP&b8rJCYw3YbJd)>} zMfE`x-Eq7Cn*8%8Xl{-FZLs+=B^F!`rJU#2W0K0jz0llzR{YC!)>2^wJg?USTk+{= zwyV2)wtEHMb3YP%kKM8;m0aTpN+sB+H0Nx9@2&Lz+=E`f$| z$u*oyvy4lSfN@!ppu0Kv!X<6#p-SjR4vXr8D9$B#rp>uD-!zwGU+1qQSx0%5kTiFZ zbcMZVhezP&_(y_28@K4QZJ=>l6iKe(NCFK*5@;BbT;p04$yOPXpcA_3T9%-@IrxGk ze{Jc3PCSz$Nu&BGiXq80LZ{EEx5i%!l8d{1B3!JQBs^yK(Q>XVCQ0FG#t#r#M|G7J zudVIHtL9k3++#Hnd%Pxs+XpO<63I1=MTz7ZP9)GUB7uex2@EneCDJM*5_H0dEJ@Ja z9DE^?w)8+JYy*}>^-&Zf5@^^xrS*n@fBYwQVQVR{5|h>zlZL73x@qlUa*rWL0J{x! zERuD8eED!e^h0JQloh-^M$O)y*O<&k1&-o} z?ka1kt;*`v*!J|~3jnX>f6_U!x%Ex=NG4->BB#?-3F7>8ts*zAtFQlGiG$nKH|1aC zd4KV27#7}sf4cf!e-_ThO8Up@7X$3#7X$PnSunaE3F9Qpo>sjVKUT1q`kg2qzF!V%F$@1#$oKDpclN=%!2a-yPhxyf+Dcyfvox=6 z*-P;P4XblF_s8((z=Id7`hRIlVfCZjXIW6^ZA9c4{!!MgWLw8W}l+i;C z8ZbV5kJ=plG zw(TNGvXnv_zG6Rsa#LZSO*%eOhKgnr^a+fAwSlBo=%|WQ@wvTn%raJfg?lCZYL%x# zK-o4N-H|jSt29f6y6K(^$lK_-lvnxYdC4N+oGJc`zZJ$Pf(%L`)Wt+ue902=laz{f z=S-*{am^v_JbX!6l_XLEoPdzgD}Kh(M;mhuOSdcw_uzI9K4{}H5k#hzBk#v zB;U(btz?x}dPd}_iRC8n`e_}NwpF0N0e_yBp> z@GTo7^Z?igXy6;6=Z6h&DuK`#c-Tki>f-D4{Bk*;E|!-^XYM#A|5xucjO6c@}Mv zWmFUoZHrO5DP>F2nfDCi{)ynVWPe`GeXx8x(Mb1;j zX{!}fs0W8u^+U-FVcNo*)ds8=;GzIhjxbZ9z4wM-?ARV8bxCljAa1pl)Q8gUuz#WK zhwcF9Y5IsIc*?1#cmT#`CZY++*^D&5wjBYSVQ4{u@(yQ7hRI}D5oKJ6HkC+ z?(0ub|Fe-H%K`-z{g4b!lVK!V0nd}YBqAGu)?JNa1ONbZ5&!@c00000000000001h zfmxIKBtZe~lRPCl0bi4MB_ Date: Sun, 16 Feb 2025 09:36:44 +0000 Subject: [PATCH 093/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9760 -> 9758 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index fac21a5df..4a712a1c0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 09:28:47,516 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 09:28:47,516 submission_checker1.py:3270 INFO] --- -[2025-02-16 09:28:47,516 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 09:36:39,135 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 09:36:39,135 submission_checker1.py:3270 INFO] --- +[2025-02-16 09:36:39,135 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 09:28:47,516 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 09:36:39,135 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 09:28:47,516 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 09:36:39,135 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 09:28:47,516 submission_checker1.py:3286 INFO] --- -[2025-02-16 09:28:47,516 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 09:28:47,516 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 09:28:47,516 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 09:28:47,516 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 09:28:47,516 submission_checker1.py:3307 INFO] --- -[2025-02-16 09:28:47,516 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 09:28:48,507 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 09:36:39,135 submission_checker1.py:3286 INFO] --- +[2025-02-16 09:36:39,135 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 09:36:39,135 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 09:36:39,135 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 09:36:39,135 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 09:36:39,135 submission_checker1.py:3307 INFO] --- +[2025-02-16 09:36:39,135 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 09:36:40,109 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,6 +37,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 49.1596 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7386 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352765 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index a1708fb1c..6f210dddb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -720 +721 diff --git a/summary.csv b/summary.csv index 091138169..d2a91200c 100644 --- a/summary.csv +++ b/summary.csv @@ -15,6 +15,6 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,49.1596,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7386,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352765,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 29d56784c..80e421667 100644 --- a/summary.json +++ b/summary.json @@ -489,7 +489,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":49.1596, + "Result":48.7386, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 339de32d2b7228304e3da8b6b7df15554fd4039f..321eb4d0bffd7f98a19fc44f63373fb9057dfbef 100644 GIT binary patch delta 2790 zcmVs4;rZBVlJ}m(;!UA*0Q6PWM56c1u5c`0L|zqTI_QjpA8b z6`HOpau&veXf|rAH`kvVuBxP2FdGDucqZHGlT6iLJO4R+h?C#bv6PvTCzz#ee|4N? z^MhtH4aagCq*^?ev;4>~PNqSYKTSr>be_myund|`nuf01&1n$L)K!Lqq)Lu>IE+I1 zF%B0~Im=c_5;+O7eE8`&nx|^#a1iCkih7hp4%_OHI5_#u5Nfk?xa|8jk`L+DW2GqN zE{=aMo}3Qas$NK&O#68`jLSqBf5>65m}FP+!&e!N##w&atRf_gC+YI9GL72Iq;Vx3?WD(rQv(8VU&(?m$8F)|z8-u<(hAXaUwa74m#M+fy1BtyWxd9SqS8@v^ z?nbgxu8GSvSpq@I{Qyqm9|*CK`P~nmPbl`U2^@psp#*r&Fz_ zmsuA`$cu9ZB;@4!Ktj&$IhPpZ?Dl|!oZSl`A!qjzNXXg!1|%(fx!L$~vsnU*6m!fa zHaYhvTw;=XKLQD<_Y;tidOvfCOR{`BVjhvQ*cb|_ve@{_ow87^cc;s}R z)0iHQp>xg!NK0OV%a?HvJ752M`RU3(?VY_}u7=WlIW&s0@#0Pk<7sm`$^TCCrB426 zE|XzDnhj-=FW6*1lWEpZ7kASrE!IA1|K8H{W|l~~TxB(j#Z>d$KUv2oGR@a8X>+}p zX>2n0!}+`s&KHe%e>Qte7PBm0O*Pi4ei+Y&(MbJ89mp(*CaL=2usH&Ym8kbnDF|?K`EXTyqIT$*JLq{&PV=l4Dm3qP@3_2j8&;bdB4oJvb_!p-H zIv(y)=t{!7Ae_7-pKXVe`g|~W@}7V0P$GE@UB6v$iAN-te@=}=j>i2UEI!XP@{cd8 zRE<&2H#A$*8{tA2tqLU8QA!08Z=5X>TdwAq?w+Nki$Qcbh~)Y?<`SD+ zKPOzmKm-yBB9KrJfrNtSmVn5^8Hgp}T@X$|bW1=4gQp<66ht1=A~6?S;t|QE`@aKX z9i>zN@z&j$e-cg0aBK^c=wT8)P9nMbkGaIAAAGokkq9J|L?EFga*0bn_;@8Gf^c%t zo|c4nJ;KQuZSZ*qY#mBtdLV|bLxF_61eadDuuX$7P8*Y8nc>m3fA*TRj`Au1c;#&Y z=xL5)dqRKyCoqXZ$0#n_35#T!IR+Ys4Cw@D7>!6ne{n<_$|KOYbZXpa5RoVWMY1eG zcXMz9C2R3PCmzXj&Z2rCir#U&0Gj;uCunYs|821OG9?yV4y2st*JF~({=LxLd{+F+ zbkW($F>W16bBkW0hHR zJutDp7pz&1xerUc>vSaeMvi4uEV;%pD3)Bqu>=~1CD1S|xrSqDlwk=HFf2fil1BAV6ho40giiOVx5i%!l8d{1B3!JQBs^yK)8$-Q zOp?OWj2|Gfj_N8cUR&FXSIx16xyNcE_IOPMw+~nzC6a3#ixSB-oJgQyL;?*X5*TD` zN~BdrB;^20f9jzqMkLU%&y?010`i}@3tLNpm6)`)m^4gH z*UdfqAYzXpM*zDGb>skzLoGQ#!w0_qt-{;GCGhG{I}gw} z)XxJnoXZN19;9Jh0uAG`%+!MtFfPk%J%agqSc?xj;aoCn4~XK6Sgrw!*qZG<{|=aM z)a3JBY`c64=ecd0d%o^r=$K4{8%1nt?g1K?ntOnTk%=^vOr)V?A`KJQloh-^ zM$O)y*O<(P1&-o}K2_FIe_NH+tFi6r$rk`#%cXNjv&|8H4h0i1rqd;pP#_+E%WA_g5JmU> z3ZuO(KWJ!#?Lb?yYM@Y;mhOZ!Nkpv&l3K@qU&V2p1ib2Ubnl$GGpJl0WE=g0bwaBO zWl4%6P)u{7b`^SW*6{<1oTrM@Rx7Ab4-T#ByOJ5gw1roz4OlP0MFFH7VWvWR?+wA& zu{}uYlHgE5+-fVS52f7^VMEzB-2u+ibb%#!%BiRL2$TLG7k^*89;|e;4TEy!l*0E0 zX02y&avz+Sj9u3yT`^cg{tSO^UfzaPW1)_xVu0$BGr}xTuWeo8>mnSkrOt1{iMImy zV;}UHw{ekEAMhvyAg8!z<)e6dZq}&I)BHY8^Eg{H>5>!+QY^oYBX9aIKsgTiH}MQO z<-Yv{^&e161GAMO#{va&T&j#qlVK!V0nC%VBs>9Dllmk<0qK)GB{~6HlXoR18xt_5 s(Or%VeEWteKf2fl* zT^zJpNidN!KWRn_InR!aqIl+~+1q&BN*1y7hpV93w54mB)tdR?TwP~4h^yp?Mx!v0 zAERJ7lk;?)B$iV@&4!;$!bPHX4~JoPEU!n2<*1_`iG!2hbfLDohpWDCL-~;GyjJp3 z?xN`T{LSgGqiVUd#k8MS!?=uwP&CElBXqv44Rc2v!E_GFz z`HvmdP?d*pm`?J<$|M{PWqzVnC0HhDH2ZCRNNf_ULxr`86`I&ayG_y7EZxjMkEf~J z#I)AEu15ILPyOy;96c!Us)uCZ=bwx?(Dm%J1bOSXyg46RYtwM`+@NH2f8=NHzH1z| z?(@VO#djKVO&e}#!|hH(D;v*tP&yuz&c>kakKu|d+HEpSATjqP*Fa+JOKyO~-k010 ziL;gLm22Q~4VFNVa=C(x^ZToc>sJ*wuPScK6nmAnaiwh*uaio-qV1AWUzc3JF1g_n z`ni@S*{jxs)gGBFfwSMme=%2(VV-b>AS`?Y5_0`~0uplKKLZH?{|k^1=)M98f$kJY z$Ys_867u1k0SP&|9*~f;d(I^~IlFxzA!qjjNXXf}1QK#~zX3@bUv3t@+$@&BB*h$a ziAB!+36~h8-j6^+>iq;Hq~6b5;*cy~o+aQ<;#bfExkgS)!h6rhf1anPGuq(s4qS3N z&uL7b$Iv127=45C?UHqHK>WlNpx z)k4OjK{y}DI9ssEK`N7UkSy&PF<#EoY&F%`s0KkaABAJ}7j-C8Kb$7&hr`wg%-7%QC@EXD z%4}$>{}qhu`+8hry~b#^4Pn{_hR(*&*&I4@sU35PMXuBnE@99C355fT6 z7P)>-xP*ZSBoss-p&$YY1<@%1k%u!7OTv2~oPy|-fCvUpL3AjHJf=-zF1W-cl1t}* z2gEu`sQ}`wf3r6w8m4YrCMMCvB)XhLa`hi`iA8UGxP*}iB$PxTp(JvNLvMWC5)wf; zxoA&I!h0^^Et+!_DdVDn{4%()y&xyY`^IFW;U(R8v| z@h{U^ONABiyxt6K&8OY89L*_s+$-=i_anidv708Pl51>1sRYmJ8I?f8s012DCD(8& zMH!VKe*vSiBtcKmU4=8&;(<*8TM~?*Gbu?|t zCD+&n<&tYSmq5d~1RBOA*KjV4GA=;^#$`!@-uB=Nm$b!0mCy${EUFKpIG12gn{#Qr zX)eigoxhG`9pzO*(%47R5td$nB;LnA5`1smqg5 zWGqUgSwVoF7?x!=9+ZG#S!U!B)MstQV(=!A30tUVx#FJid{EMjZ6_xw9x zzEP9Ucd_g8$(`qxWfXkfL)S2wIyZ_~)Z7C!4mI}x4I>k2D49q@$wV4Tf2Lk$?m-DC znPmxjIy3jM77uko_eEG#A4D-SfrgW**KF>2jm*EhF;Yu?RiMY@Nj z?h~=7w+Cn(>g@p<4rK*z57ICwfrdd@=IucV7?fq+9-Vo6Sc?ZbaoZG1=IudI49W`L z9=&F7&udJkqZ~)^LwA+6f7DiG^{VfBda?z8+ji(2+1mN0`%K1Tc_OFNLH25et)|DUVj?S$4c_Y`WFMN{1*eXJXtWhT27^sJ}qQN zm5+<;6UvKKmEq%P{aXRbA`YWCOrO@h=Ra1koci5aGS-Kv-T1sYcOaN&>TxgU?id7@0^C@WNJW;fvR`qu|muhVF;dS$B0 zU6iKL?D^e9`a>D#?+jHLMN#_vCO;+HUvcRE3A4-*ehvjMmEJfelTRQXf6Y$AFc5_A zJVoSt;v|JCSV<}Y>WNB71W|Fb>}_ii|Hw9^eS4fVOHc|q=ruYTlJ{~0L9A^#?x z0LR?dpP>E&P)oCtA;_NMIwd**T$6PrCIv5* d-Z& Date: Sun, 16 Feb 2025 11:54:44 +0000 Subject: [PATCH 094/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9758 -> 9761 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 4a712a1c0..4a7ca709c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 09:36:39,135 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 09:36:39,135 submission_checker1.py:3270 INFO] --- -[2025-02-16 09:36:39,135 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 11:54:39,719 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 11:54:39,719 submission_checker1.py:3270 INFO] --- +[2025-02-16 11:54:39,719 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 09:36:39,135 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 11:54:39,719 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 09:36:39,135 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 11:54:39,719 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 09:36:39,135 submission_checker1.py:3286 INFO] --- -[2025-02-16 09:36:39,135 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 09:36:39,135 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 09:36:39,135 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 09:36:39,135 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 09:36:39,135 submission_checker1.py:3307 INFO] --- -[2025-02-16 09:36:39,135 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 09:36:40,109 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 11:54:39,719 submission_checker1.py:3286 INFO] --- +[2025-02-16 11:54:39,719 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 11:54:39,719 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 11:54:39,719 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 11:54:39,719 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 11:54:39,719 submission_checker1.py:3307 INFO] --- +[2025-02-16 11:54:39,719 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 11:54:40,660 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,10 +27,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0893 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9484 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409089 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3089 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6f210dddb..55a98c62c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -721 +722 diff --git a/summary.csv b/summary.csv index d2a91200c..2edad1a79 100644 --- a/summary.csv +++ b/summary.csv @@ -5,10 +5,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0893,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9484,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409089,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3089,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 80e421667..1cf802999 100644 --- a/summary.json +++ b/summary.json @@ -184,12 +184,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0893, + "Result":23.3541, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -214,7 +214,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.409089, + "Result":0.406511, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 321eb4d0bffd7f98a19fc44f63373fb9057dfbef..c673dc7e57751a6fed0a0733e2f3455d601a0181 100644 GIT binary patch delta 2674 zcmV-&3XS!iOrcD$SqKHXSpS+%lU@j5f4chsKfCFE^zhfm=|s7gNgBnowkkATRpcy; z2hnWQR&TC9H(XUovtTv|Ch<(R)hC&%zjpp}_z)+*r(-EIC9hzXw$*W#%@3N*G#txm zkZSQ<&hjI}IGF}n{x%sk(|IC;!7^w%X&Sn2H>W`~Q&%+{BvtB&hr=k8ALDQ_f0eUr zRY@WzL6&zv9Y^z2?HmrG{8*8XlE`6OJrV~ezZpVpb`FC<8eR7L)8Me)uY*(KyRbn^lB_@g!aTRi;sXE)7+g29IskRF#Kl zkd2Fqm2or}$l^q+O1Ma~c>3Gwe~?&LunHCSI#%dn6YXiDrQ3#8T#u)TT*ow5tuA}` zG01|>VG=(m$+CrX9u!wb92iD^TEe3ATTxu}t+_6^dM;4%Jc_e--!TuH_eI5P#djKV zO$%;l!R=N-Gw;u)QwHvo!TO-@_Th>vS}oE|AhC8O*Fa+LN^XF}*_GS^e~G)1?4)br zbWK))An9@i>F4)X71ysSZeCT~mML~JZQ)E?EZ!iQaz)D{slG0`eqD0ICG@(MRFAA266&S-(p8}P{KJf|@|9z(~R z3s5b&1eY)U9(KO|_wv(~f7&~Hznl%F`E+O$W#h%27RJ-&bdvv_f96x2{MB40!+tay z$|Rq#$$lo&te-CKrcqkVebWBDrRmKqk#af9Y8I2J=DB~ej!$Ho&tcN$dNR}4)Z7o} z^F}ydG~(IpFJ1JHOJ6erm!3ne?#YB=o}6mxz&!j#3py@370VFfP_K^BosOzAs^vioD%4GxJ#ic z3Gae%@`-%59ZvG|LFdV5{<%YmY2=SDtEw8KoX7c7 z7AN7je{X2ErZ>WcFj^HztRs~QB;Gh%B(_}5G2J~=OBaLae{vAX{d3GEHo1RJxP*ZS zBoss-p&$YY1<@@5k%u!7OTxP#oPy| zj28I30X7e%e=$7}L+7DDLN39jS1)YSAdJ(-Bv{t)=-NMfPFhEL6#%^QwgB`r$FV)3 zKmQY$M4@98m*s>-GRzzUjYGP00yK<9q@g$>4doGNTskyvG>Ax40Y$Q`g6`(v21-`q zgGM}3&pC_gfhfA-cmXu|^(Sa<_5W?K`7$IHTn?n1f9LmOlFI(Q(A<1f{L6IKl3@ir zuh$)0@#$!`tGj!adj(!|KN5V6-Lfc^T;m8zC0MIxR00j75@;BeT*IjpWmJL+7?mXz zbOqg2IAbL~Xv8xpl{Bh{qUdmmG<3Y&>i%1Va%-?Gs5Ep<`~a49Y*Mc?6 zG52AKe|MdZ1mDQ9Y>Fk|Mzh9%c1kQK)9DL!DmiQ}=8~-I{BSJBxzI+MKL6~M(A{(daM7nAi2292g1dgNy1}xKV6QM#ULp>&G-Q# zf9uGu(&V+ZJ$cm}OPG7CCSs4*L~#3nDCWkiBT z7?C9vbTSxXoM}mvZx-4VnhNBd#1GB5Rm`GDr_wYR$|iHV$v`*T{rjagNQwb z90BY$)R6-;4z=U}4QH}~DF{`^gmH|pNG7Tfk6e_Jy= zYws$Ly9Z0)&Y=b#pmC^!2WU8!6+Aph!>|NC8a7y#*?3SR49hYjkDxxAGgjh*M)>M0 z^Xs4}Hdq1;o50p=*?Eo1pR@YOqY#F3W5^6N34AScwlB;aoCn4~XKESgrw+lRpqLf1Su0W#%4K z0VT7ng08{LJ*>n>jnI7&7S#h$j7*^6WEwS_dtM{+?{19Ll3x`l4Sg3%-L}M@;nEO$ zpyWOgn|gbI#--jKpy5zf@b(}LgA!;Mlx5x?Q~`ss%-dryZx1W+K_gy^Ldm>6D2hQ@ z!P{fh?Cp7t$!u8QD1PXvf3lX`s?1)EZBI`=0q|Nb9V44t-*lhKWF$}IWRfajoS&{$ z_5UkzaJ%}Z{EIwqFP;se!rLEAR^RK-qS;7E|5*KEfL;7zfL>G<4zCsysbo)c z*;eJ_JikIkvZ^w89Ik#VK$#~|oJ85vs`cW>3Ko-~GmQuDmz`S7U7|l0^8LH;oqg~w zv_Jf!7$20jlBfPG&9hs!QoKOj>Kx94kvtEQQ8Y`Hi5%vM>RSE)o~(X-VEH{Xlmg7ebo3Xv>F321JF)Y}Su zG|Ti6B|b1MSZg#?s0WW0^+U-m<;KCAGZtKs;FAc_o^o5ET?m$9?D-B9^I3H0C~k~X zEX2}nv1R<5Z2=c~KF10Ie-kV)d;q3)B%%{i@DXW$Tsr^+$IyZT9Xuu(!541;r~KW9 zLAi1&8F~wMt!Hs`@4Xz2UDsvZWU$8k1b?kw--lIGsSl^(fa+3k${ny^TwUVpA|9?~ zKCI%2Hxk5CAN4zLr$vr^Ad(n>9^;;s&&l&kvqW`~7qc`kQZjD{8KJW&o#yMq$eaEP zP@RVSn|KBsbKib~`VX^^A<6;;x>*03PLp3GTLI6LyCggTS(ErAK>_TOIwd**UXyht gCImooF1C}OB^CicleQ&30h5#UB_jq!BLDyZ0M0!3Bme*a delta 2688 zcmV-`3V-#XOrA`zSqKGlT&j#qlU@j5e@^#7es)Vgdid+(bfVnLB#q)(TNRqFDsmRa zgJ?Ett2fu58?LIPSuh&}lXxcE>XS^>UpxOfe2A0Z)3KD9k|&s@ZFQVw^MhtH4aagC zq*^?ev;4>~PNqSYKTSr>be_myund|`nuf01&1n$L)K!Lqq)Lu>IE+I1F%B0~e>uxm zNfJ2;vV8dIIGU$w=Wr0^$BKHCL=M~PkvKT{%@As{bGYpLHj)qN)?=k82Iq;e`1|r6)Nm?tkA_K+S5c!w+*X!J)S0V9n)O(x*Xxh zAPYK&N&KKB%O28sP`onYz%cUD5*DrBisoW$&2_`obAyuCQJlT|j(OO;FA}d6-)YD- zZMdNgw_6R(d_0>$8F)|z8-u<(hAXaUwa74m#M+fy1BtyWxd9SqS8@v^f9^)IQ?7~2 zHCX~d%H;|&&hM`(u3uH$ysEe@Q|wgQ!j-mIyg@4Eik3%8eO+?>y5xpS=<8aVWT#pS zR(oWz1nzDZ$6P^%dBPQfu<#K`$o2CHNXUu*3?u~nFF-<|`wAokx>Fz_msuA`$cu9Z zB;@4!Ktj&$IhPpZ?Dl|!f1KS5AR%Y>5=h9|{RSj0e7V{9axg! zNK0OV%a?HvJ752M`RU3(?VY_}u7=WlIW&s0@#0Pk<7sm`$^TCCf2B_TXfBgsKbj3? zk}ud~Ka*+JPZxL7C@t1LY5(5R^k$Ywxm;y6i^Wv)+&@{zCo;{~FllqWm}zV>_rv+T z5zZHlcs6@X7PBm0O*Pi4ei+Y&(MbJ89mp(*CaL=2usH&Ym z8kbnDF|?K`EXTyqe>oUBheJm$wPP-^$(4G-B@8+sq0j*dg$_u_Tlg2J1Ueq>Qs_#; zyC9srBcE-Dllpuxc=Dcq?oc9m3|+rnaEV7GmrjjDj>i2UEI!XP@{cd8RE<&2H#A$*8{tA2tqLU8QA!08Z=5X>TdwAq?w+Nki$Qcbe~9GzIpz|ZTt6pV!axKP z3L=nD5P^h(=$3%U!x@Mr;aw0;L3B$%1cRp_x)ekn(;_h!T;dVQrTf1FVjZPa0P)t{ znG#LQaBK^c=wT8)P9nMbkGaIAAAGokkq9J|L?EFga*0bn_;@8Gf^c%to|c4nJ;KQu zZSZ*qY#mBte|jK>u0w%@yabnCy|7J#FisnjV4303wSV@Sw2tyB0C?qX0qALtV|zk> z{wFYrLdPgB+X;(gnmGm1ejAyL)c;3VhG~Nbq~?mPM)L8b?qn!M%D$CD1S`fre4ZHJnOOMkPqVs4Pj) z6?9kOjJ5cn6VISj(x@JaqSGbP(D`y}_-_%)t;w>W($F>W16bBkW0hHRJutDp7pz&1 zxerUcf9rH4_(qOpQ!Kg0F({T?!?6S!h9%H2EV+hbX_R3J5-==F5_C5QU$CStKInvF zNuzooirq~G8lelyrSX5jWgRtExx>01*yi)3VcPq+#M`4sg6}$74&{<-9Fua%HJnSJ zVO#fil1BAV6ho40giiOVx5i%!l8d{1B3!JQBs^yK)8$-QOp?OWj2|Gf ze~#)ZEnZvOi&xFDgt^CRBKCMq1h)@Z9wm}%9E%dkHJnJGVMGEABN7;7Y)Yh6MkMHj z5m}O;yE*tmByI6QC+r3+i|U~$MkLU%&y?010`i}@3tLNpm6)`)m^4gH*UdfqAYzXp zM*zDGb>skzLoGQ#!Ff4(FVOeJ5K?xX^Wkw!BeKu#T#Rr}6yRXcz zgQD1E2{dd0TeD^7H7bA3-Pbp2=D8BvZk}Qh>w0_qt-{;GCGhG{I}gw})XxJnoXZN1 z9;9Jh0uAG`%+!MtFfPk%J%agq5Lk;3I^kS0YY&Lxi&(A!i<3eSGJhr0C^Pq<1eDCO z1YLufdsvH)I-&a_EUE{h7@0uB$uw#<_q;~t-`yCgrM@aq8u~7jx~IE)?!y{l50u;| zVpDGq(74pw12i1U3f>;1VNe1MgR;!qgAy<(%e*}X^Y*Y7A9UiiD3r|GgQ6Ie6}&x0 z&EB5Zn9PO+j^c+tRe#n}Tb0$TvF+)}7XV(%rE_F+>znRVnT+I#oJ>+BjPujAirloW zzW#qD4sKW9lz)-u{l&9ERCxP?$?ALkSu`6d=^v|K46ut|4A6^Y;qYoPkxKS7mu*!( z&huBOC{|SlkHgh(1t{|*ijydNTJ>K1Sixcvbf)p({c=!?SzYwULcV_&zOxVBh4zPE z6yt-^R`SxHrFnJBUWx~3Se?UpFp}p%GKyxYGLgf)P+iMEfG4Y8A6P!l;`#ECr7CxE zmc`TOXJZ)*WKujcRb?2*+4Ga)lx%**q4OuR&k=qO1QRf()00skD}O&|XoT%RTe51P zP?whOgfvM+tp}1?$A4eNahwFa>T-1NoVhcoTpeT^{epEus|saFiXu=dT-Y8 z1B#rdiqlprs8A0Mt?Ijy8N#%MSE~(JFTh0sq#R+ULVNEG!Pv1qNa~W{P(j>kE2$5q z-C;x7H{Aiw({zC)cz?>Nr}zkr%|t{alCueEer!7cIK$9_1mzvhk_=zG9;|e;4TEy! zl*0E0X02y&avz+Sj9u3yT`^cg{tSO^UfzaPW1)_xVu0$BGr}xTuWeo8>mnSkrOt1{ ziMImyV;}UHw{ekEAMhvyAg8!z<)e6dZq}&I)BHY8^Eg{H=^2s~3sNk Date: Sun, 16 Feb 2025 11:58:14 +0000 Subject: [PATCH 095/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 9761 -> 9753 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 4a7ca709c..006940489 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 11:54:39,719 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 11:54:39,719 submission_checker1.py:3270 INFO] --- -[2025-02-16 11:54:39,719 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 11:58:08,982 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 11:58:08,982 submission_checker1.py:3270 INFO] --- +[2025-02-16 11:58:08,983 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 11:54:39,719 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 11:58:08,983 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 11:54:39,719 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 11:58:08,983 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 11:54:39,719 submission_checker1.py:3286 INFO] --- -[2025-02-16 11:54:39,719 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 11:54:39,719 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 11:54:39,719 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 11:54:39,719 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 11:54:39,719 submission_checker1.py:3307 INFO] --- -[2025-02-16 11:54:39,719 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 11:54:40,660 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 11:58:08,983 submission_checker1.py:3286 INFO] --- +[2025-02-16 11:58:08,983 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 11:58:08,983 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 11:58:08,983 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 11:58:08,983 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 11:58:08,983 submission_checker1.py:3307 INFO] --- +[2025-02-16 11:58:08,983 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 11:58:09,916 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,15 +25,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75139 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.426752 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.428606 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9484 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3089 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.4239 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 55a98c62c..db01d14d8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -722 +723 diff --git a/summary.csv b/summary.csv index 2edad1a79..9bdde9e68 100644 --- a/summary.csv +++ b/summary.csv @@ -3,15 +3,15 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75139,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.426752,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.428606,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9484,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3089,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.4239,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1cf802999..379e31337 100644 --- a/summary.json +++ b/summary.json @@ -124,7 +124,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -154,12 +154,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426752, + "Result":0.428606, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3089, - "Accuracy":"acc: 75.600", + "Result":10.4239, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index c673dc7e57751a6fed0a0733e2f3455d601a0181..78ecf50f5dea1583bfa0a48de01496b0d0f3f373 100644 GIT binary patch delta 3956 zcmV-)4~y`jOqoouSqOiZ)}V?@2><{GLI4050001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`OuoLg_=$QFg)&#$noe(074(>05i79<=>ADguq@K z$J5g@j?U?aG?drgRonQZ3fF?eUmvFvNOagkUVM}vVZPPD3o zi!_U;zpW05b%Ir>aMrOx7n^8b6PE56cJX>VP2@VJx$1v)Il_-Y7PJqO_(4gQJ*4xX zcxA+aVdSSJELy)6&BfT7>xQf61|_efID7YP^RRheBwj1N(~xW0a6=nzw;G!Hcs7GF z_(2&k27Px7SE#ToGE7Ed?@F#2iL)!YVI=OZUsYVc zs-??*;L>ixt>NWGs? z;*l(0o+Zp%;#Z~za*dppgm<2gJx@_*w4sX~_~dk+)0iH{&^hOVNlRXW%a?Hv+h6~C z`RS^A+BlK-9NOP&1DTqeVQG#kn!U$Du3Cey5+F7BpL zTC9J4(*C`z>CG&Wa=FTC7K^FodG}-;pU5;{!=%miVy3al+z;pTMmS$I;@RvmSoHY73SVt)pK)m&KrbN>=T*u}~^m!6} zB#~VG$0%{=2OpFGiHwAj$Ve!ODDmhAAHRe|CY)TfrzPPXpKx+U8@kv5tV3x`kBOn{ zP?T7t>`T91*rq`kr;SOl%<$+rKYM>oT1R;m0KD?I0Q5E2b$p>e{}VikLg!Ijwi6b~ zG;_>oTr#8+Mguf*8j2&Qp*%7gk4}vn4M!xGfFfCzptCu+ff8D}Oea3ca}H5GCW_v1 zyx=mEH^mrsg+8O~ZNtU%}WdSEL&UCn>-bZ^hy zUcujTKN5J0-L@%|sBr~_lDSn6P%;{TlFylTfv&`n)|TiH=T|I?&H`F z#S%5HL9s*)V##O#OGX1&q6UAlG|I4K5&)JZ2|AmDFIdu+F4GBONuzp96x>W@H2GSI za%ub@a9KxK_)AB zayShz$!LJdGFuLp0GKQ@<_K`+KueeD1epYL4ikk7SJW^I*E)X z9DR>XKPcjsDcn|B<|@a?@fdAvPTGTt0&-(fT^_3tnm zq_Tp8htmL+jEjH9p;VTcc(_i0$}$^|pe~y;Xz4PY;LTTX>u^ypS2C6!Sijb6*m;f0 zpL6r|jaqrG#I~0wU&ETVwa-A{@8Obh>QFNeqj9O5htVLH75qG$2DoG_IlyI^rHAVT zxGXdE2ypd4OPA>cxddYm6NPJ7)G%w&v~-zHe2YQ})*dbjpsZl+F>1E}l0|@lyqhNzk6ggZIlpEoRXl3;F(C_|7?a7djt)QH&2tOUX-rmgdzhdnq1p!)hPS zgONNBl2J5Em5ChYh3Z=V0X$j#_Q3LS7SETDY*o37vn-xIKO4(nAd}*msVc)b&YqtX zr(_268xHM10kh8$m;njjx@ z&c!FgFPkhycTn;qXNMX#EDZq*^C)5KL+$$NSobx=1!WOUa|VanJ#g*I=&vE?0?Dn6 zA8NNk6bGis!&}HGH}V3Qw8VLtQ6c{=*CsDYpb@_X5Ls#x%W}<(CQL(NUNLc~5y#O` zf5no2D;RGPY!PiVChCTp;-f4@G=BO%Or2(*DMjd4K9FJsc55VN;67y z&syn0&^BhtMwB$w{f)P3+b-fbO&GM{tNaHrZYu1vNyjCnt7tYspTJlfNNRm8EwolEZwp!+=G)I ze9*=tDyU3F840saeDmydoM%}s^I(0uq@j>yrI!#xDXR+fu&DInUMdydF7Jrpf67B6 zh^b7@Wtg)#S(|&4^1aFSCHY>iY9*_*(o-s+Hy^mVx-%=P2q{By`tCf zw6$$`5~ahSc%bCi*#s@_e*r@Q@&d=YLD<#Dc?qbuHtH?Y`S&sMJa>Rx`je#PDuKWlxY$GJ>f-D4{Bk*;E|!-^ zXY=463Xw5VKhYZZNge7cQHJq(B z#SYqW328h)o|MW`Fxp{QK`A=QYh8$cI$-(y$>h?bB#x{4f0@)b>T5)^ zB886L-?R<82X}gKB^w*u>%sjVT=8X#V@1FQ?`em)ryXLY9X5E6zqrR=+~Y6q@fY{_ zixqz%T0^^kHQh+h(r}9`!=iX-TRCH!Quc);1Epv6Wd-J(3 zNZTHcjD|ViZSJpBs;)&xe@U^IlKuITL;wXZXvtxDhYeqSOnPeh|FP}3Rnrm6z#r@0 ziwIwrdCxHJp9o${_Qx~YViI@RgMYPSI55Z?l~iYrk4(!qv%Nvbk1vDg=-$5$-i7gf zHT}R!=wK6ZO@uo_*QaE&tYdiHEpgN_yLNPe=SJY?>|Hc7`E`(fe_DJ-H;}XX_ot^5 zJ8*8=3o9mlkzdS_^4a%oHklov6F8QU0JONeLADhrO6(+uD$-+NplxC7*^8RQYA<v}t*G%+0ClC$>7^n7_b zJ(q~9%vIyl*?f69K)zVi^R+9ddOxs8;A2w%A#^;OJX(qT3h;^J`GFU>jx`u4;XIbN z@;m142WRBm9XN_|4(FrN|h_pYh9RPx3XhDGv9+QmV zi#LE%{%*sdTsf5ty@k8hvpBl#}Y#SYv*IzgDmB!>Xy&hf{GtbtyRI4p=a* zF7b5{57#muR`J9e3F4`b`klAaBF8=uNen=banH)<<{GlV&7W0nC%WBqAH%xlRYIm0alZEB_<{ILI4050F!Yf8h_=S?Q^2Y7RJBdzk;IX!%WqR zG{{>jx|L+!Vu;CPl5E{i7Qt4jRdE6H^4~A0F?!A;VP|KT)V=c|qs?==`v53;O^ z*T?BZxtB>A#k005G+kBXEQ|-yY}8h7u0Jm- zDKjOnV1Jgj)p3^151P$19Ls5tYVlmo@*~4InFd+@HW@Y3c_M?sGH5z!8oF*br$IDR zS2Y|YRqBX`!zh#=<8U#RvussKA}2wXcRw9R^Hl8|4x;>6k&lwdVOu>C2PeN7LTz>q zmu=rh@*&-NtrV%;#qsaOo6|vC)eC8pX+JN!aetX8133&9lk6&f_$s5(ILl9)RfL4` zBwhYhrcr(_4ON*2k8Radm4|4Mjf;wvaWojn;zX-TxJa{j`rGP|SXZzL74|w-=wcJ? zX`-duhE-gTr-@w0G*_)Id-yTPg3e(QKPbtvg>)ViS4JEdMt)ktqV!u)T=cEEF1UIw zP=E3~inDj$F%O&fMa65ycN%g{3vOt^?N&iE@6V=F2JV!>`k?Rj;fgC-Ez(UOv34cb zKw|GoZh*wumD~b}yOHdqYvOcGR)HYtas}z<_g59yuPSa{Ros>-b~0_@Oj|78AenMS z%Ok12F1dbPa>FI`x|UY5ldT1_J+fE@?tg9*$6P_WdBPQfu<#K`$o=yPNXUu*3?u~n zFF-<|`wAokx>Fz_w^oJ$OHc6&fV&h7<}kh6OUB;@RV1CkcL z-E4fj*{lMKBy-FqHaYhvTw;=ZKLQEK_Y;tid_QxEOKSP@ECC;hUqK7x9yu)u?|(iY zdmf_BXo1fg@W|;rr!hSqL&uy8P%XIxmoNPucE0}i^3#=n+BlK-9NQ=RDmld0yp zf3l8GWSY-m(&l;@RvmS5Xt>>%q2Fte@?iBfe0iNL?EFc0tp4tEdh~-GZ0I{yC9r`=$3#8I!{4#DTq9# zMPe?v#3PbR_kRb(I#Q_s;(x8XGbEap;n)@?(ZeKqoJ4Z>A9IOKAAGokkq9J|L?EFg za*0bHe7q78K{&Z-PfNnP9^vGS7WljYHV>sSJrG0Zp+G_|!KGI(Y||i&)5auN*6`@s zKYLDEM|u?iyz;gH^fbq@J)uAU6PQGyV-%O=ghevU90QF*x^x0GjDJR?p*SK9FH zIy7!Hh)7fcMY61d?&jbIN><{7Mm$o_Ig9FnD7xZ!0W|sbCunZ<|821OG9(sU4y2st z_hXXE{=LxLd{q3)bk>q#1w60U9b56~Xtt}ndzO0zUUNSZe2v|*D3x5}2udYbt7lXK z4Wklh7?oVZsT5^Yf`1Aal_eE)1>IFRV1R91V*KjP2 zGAuy_49k)Vx|@S9Ske+7G{Ui@Q9TgFR#SmS=z?--{2y>xM}Ll0R#?{^+dNMiroE3# zyghm(_^zYnP%gQ~F)5c^!?^?+#wE}&F1dztX_j#bDqvieRM6cVeBqLo_$U*)ki(*S zAc}Jd_Ov;d=9}h{tn2)BBoqq+&x9Zms#IR%7jl51VRP3q;|{8TQCVi)5!6|8#!7t92;UQ74jmB1_e8h`?1|Lu z(&1G8{7sQJ>fX5)+x8t>GdyeWDv!GdOW@9-1|FbssDlS+IF=PWJV?W^1U?!zSeDs% zP$LY>G9!MQf>peQz20u7tM)_-i-d5y}Sv-n>jnI7&7S#h$j7*^6WEwS_dtM{+?{19Ll3x`l z4Sg3%-L}M@;nEO$pyWOgn|gbI#--jKpy5zf@b(}LgA!;Mlx5x?Q~`ss%-dryZx1W+ zK_gy^Ldm>6D2hQ@!P{fh?Cp7t$!u8QD1UzFsj`;bs?1)EZBI`=0q|Nb9V44t-*lhK zWF$}IWRfajoS&{$_5UkzaJ%}Z{EIwqFP;se!rLEAR^RK-qS;7E|5*KEfL;7z zfL>G<4zCsysbo)c*;eJ_JikIkvZ^w89Ik#VK$#~|oJ85vs`cW>3Ko-~GmQuDmt>t< z%%VRQ^8LH;oqg~wv_Jf!7$20jlBfPG&9hs!QoKOj>Kx94kvtEQQ8Y`Hi5%vM>RSE) zo~(X-VEH&nWKoi@54G#7W8K#f7c`G(%5pf=?tyDxMt==C7f2q>`Jr|zL~&r6JiLXB za)T9+OG(Vij0*X0xi)!G0*&}BfXGslSe9#MG|4p-vMLvc8gU#AReyQ%Zw2Emf~|oz z8uDZ`6r(d1K{_i))CwPhD^oF+N6;Vbq7a&9W@vq{G#rK@N*L7%`_8%V|q9aT{xKDRZ;EM?_axL4e- zR#_qhlx@M$9ZgcYN`KNssE6*kfUJ$4%kV09o|h~H&Y5Dr_*-ENLr9^dLRF@U@+FJK zPg*LmJ7+?@#5ISgGx?ITDru-VH~}HOSL}?XjW*^OmTp-V?!ie9K4{|+6;zg@%!FAN zzIk>!W?9B$9jtGcG!(L}^b%qyWmkb77L{Jy%SeT{%R6GY(tl_KF_p!+3|Ss0YjbZ= zzBk#vBHznZtz?&0dP-&1#B$@i{j|19Tdt|njC5PKDLnDCHF^b4N85s@pmYcn50n}^ zo1n%0FUV1Vyuh(;5O(!3D*^TPMtx*D|2{^Z=MIo-Kcn6R32k$;M{29mt`IIRu1}`; z0J+%kEP@ew0Ds5<8hA$FdO-u6N+2)>F7^<*y7)Rhzg*6zi{<6f+5B``pSD!GN7Dmr z*u-~#n|2J}C-!g4#5Wud+ntuNLktU{HP8GXGBMj0mat{jaJJeMJ7~uxr0@WFj^SEr zIDqG%1f&F!Ly}*SZk@binfYlf|V+NgP-8Gk>XX)Yph+MG76WzZn~L5AO8f zN;NjP*Ms{#xZ=we$4USjyk{KZo^gnkaoFHJ{^A~gagV>a$6wszFIN18XbtW8)l4Hj zOT#U)42$AnY_UAwlyWX487MQWua>$Px~B13oyK8>iEZ{&-J9*cplo|MG8!_zJKSFx zsk#>(rGLa;O7`bV5&;yvpe2Xp9X5RRG3n{$|HroDR!v7N1AnZ0FCu(hY;9VHsH>Mxh2_0-Au8D9* z>iQztsC6u_yDg47cGr$6@Z1XgoV<%>Cch5SPk)Qg=muC`|NiuJVh7GmTd~TeFY=2y zQa<~>%_g%WbOOgR6Mz<1H^{aEB@#Qup$6%(G0?U#_MAmcVYO8r2Nh}tl_HN(le+G` zQ%zlVG(SLgU^s!YAw8|q#2}8pv!%O+>$={~DoqT>x8$yUGCg0OPR}LcDtp!VbT(gJ zj!7>T^?vQjt=T-1NoVhcoTRD(w=f#pNZGW+4{F`k77kNI%3IY=>Fnj=}b|j(`Qt%OJe_T5N1jo>V0v$Xi z8NnBC0H^%jhC#VhQ9;(+Q>aLOI9 zU|e0|>mnYmWj?Ipi8m6&Qy=v^Z>L3$eISw;fF9$XmCwoZOA)g~b&(gdG%r#zZwR5Y zDV^rCcOl*a3A$MSnobD-011;`Bv%2?leQ!x8`JC|3t Date: Sun, 16 Feb 2025 17:43:43 +0000 Subject: [PATCH 096/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9753 -> 9760 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 006940489..87a4e3a0a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 11:58:08,982 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 11:58:08,982 submission_checker1.py:3270 INFO] --- -[2025-02-16 11:58:08,983 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 17:43:37,788 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 17:43:37,788 submission_checker1.py:3270 INFO] --- +[2025-02-16 17:43:37,788 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 11:58:08,983 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 17:43:37,788 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 11:58:08,983 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 17:43:37,788 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 11:58:08,983 submission_checker1.py:3286 INFO] --- -[2025-02-16 11:58:08,983 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 11:58:08,983 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 11:58:08,983 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 11:58:08,983 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 11:58:08,983 submission_checker1.py:3307 INFO] --- -[2025-02-16 11:58:08,983 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 11:58:09,916 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 17:43:37,788 submission_checker1.py:3286 INFO] --- +[2025-02-16 17:43:37,788 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 17:43:37,788 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 17:43:37,788 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 17:43:37,788 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 17:43:37,788 submission_checker1.py:3307 INFO] --- +[2025-02-16 17:43:37,788 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 17:43:38,917 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.4239 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.211 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index db01d14d8..67adbdb35 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -723 +724 diff --git a/summary.csv b/summary.csv index 9bdde9e68..342e6d942 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.4239,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.211,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 379e31337..2b541d135 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.4239, - "Accuracy":"acc: 76.000", + "Result":10.211, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 78ecf50f5dea1583bfa0a48de01496b0d0f3f373..586d5a942bc35a8085abf9f19dae99fc3136b90e 100644 GIT binary patch delta 3941 zcmV-r51R0qOrT7#SqKh~NF?n{2><{ILI4050F!Yf8h_=S?Q^2Y7RJBdzk;IX!%WqR zG{{>jx|L+!Vu;CPl5E{i7Qt4jRdE6H^4~A0F?!A;VP|KT)V=c|qs?*I8y+{+}5;#pf2nyxBx7RG~UHfpOk*Pk1%s-#&k8w8VhCfn+hOx0gI|2ce!li$;^ zl$nwzn17{hb)04MgJv@g$8s8^T0EDt{Kzm)ra_iJO-9Xhp2%RZ44O`whOXPqX%NlS zRfdD4N{)Cqj6(S_4i{57%T`GeISI0S_~|&Br)uYL5aq{;dXz*C+v<@xIQh*GYO`~= z?E5y759!upr6}btj(;znoDSNmUPzlv`*}Hx%YQ@}$YHRUWLNRSR~e1QS$^8AA|#9_ z>GH2Kjq-D8sLC{WY^$cKJVb+RTqIV;(O@8p6Rj%YBF*CIZ>vLMonRFz>~*Zr#U|R* zL`$~~t9U)0CUPCqT=lvf;m05gI)_R8pd`y4(s@w4GUC86^3xI)t>22~Vrln zM@oHNa{apGhD+${TAF01S_@WtWU&P9ZhsfYTtS9;!WDwB@DWJJ_45fx$cg_9Bn12~ zKtiDV3M2%&Qy?LiSr440tu=26OfR4KXZvovV3`#fVaf2pa*h|oR)-lpMQ@% zPf=&I!RH-#!RzbIt`wOJ0J@mvIj}U;lgg>B>LtoxNYKhSGdFG>WqE;!X?W zX>&Ts|4#FzPX1^vlVLxa4P}xq*knJGY1U5{che{>);?+f-qQ4DmPol=Wi^Y%RP)?F zS;r?b&DSt#bG?{pY%=%5`MeR%7k`a-HhWAKvn*dtHP)(r7|({$Nc}|}$SjB^sruot zIRx|dw>nD7SFJK1+Va0W$HdS%7&?bTM=rHvF0sj#dcq|PIv}CY z0SSc;NXT3G7pDX|9_~`;O2WG!oV+8SZHJTkd@y+Oo`3F8B6$p5zg=*NM}H)jPK`v4 z#{D2HKF>7rk1wlKjZx0y{3(l*aNNH)G+Waf;X)X#3MAH1N(B;coGlVtuI8BTo~5OW zL3BBY|*3 zF&A9o5y_?dzXM_&rBnd%)_>iZ5>3l+Yzvd2qct5AfY63 ziAz8DcqJr)aB|U}mV|db!pRwJ@OcMp9ZF+*Acn3(frPvSmtMWFO@lB_8%vPjekf(aYP!*Bha{X zYTRfLkthL0vMfP&b8rJCYw2^wJg?USTk+{=wyV2)Zubg&&;3a7d+e4)spJ|*P%6Q_dPXJC zFe-tDQOPx&N>N57NPobnEJ@H6bXVbwwfLYD&!AM&s2+-<(brz<&tY0lXA&5oJ*i#TmlW_l504ZW*L_t0pqeHL3eZTg-hDvqe|$F92V6B zQJhP#r_H%E-!zxxzRq7qvX1g9A!+U+=?dG}gCyR^KN5Uz+@gEiK;yJ1l3c@)1R91U z&@d#q#Ly~KRPWP#|#$OAPi@SUx zT&$TSJZAUPMAW>Tic6Q&9Q{J$7&+>cufSi4_F>0l4~4`63I23 zNT6Xv0u3V)Xc&=J8Id3XBeEnxcXRNCNZR6qPS_1t7Jt=4QH)5SVV^0jHw5HAaTm6h z0xL0TZ82$>ny#CB_CdrRLyiD;8|ugb8i!hPfQB4rdXEQcu*${ z%Q7R6pgx;3*5ZRs_}y3L*FjNivIH8ofUViG^M4wZKj-f28#VJ>iETGezKAug);UM?=PMWqQcuBOjh6P&!X8#N&i^=Vt`%zVt`&G z3x`*WiBz(uxooTQah|_IMX{ND?phiQJh5C)2jF4#|jpcpfimJ@0WvG%psyb z7V`bO@ST0|F0?=Vq8J~Pwvw0rEX}K1_EJ1R!|EK9j}a9K`oQvW7SETDER(Gf8VM^7 zoj(CkO9KRxkt7JfSY35$DCuweuM0CbaW6F+}RoQ)l(%&%XE2Bs4y1s0HyD!y}k zKA)>khF>;WiteD~3Cj;PY*-qCJY-RluMf5Bt7F~Q5EnF$Xv%Up)b4?6Uq*ioITuJC z&H15rD@1W%nmoLPjBErP9qHX8C|G!&yV7C|~R#b{{Oz!tm=A*W@++W2*z2;K%4z^hcW(KMwQ zCAw#=^dM*pvt%Pmn(F?>TeW2uahxVOwBalE19EOE?6XP7C8euqHbI}jSQ|*j3LRBZ zB0je@$1G*#SGZT)uU1(i1e9&T(H%`vx=Mf2M5u@Exqz&Vp3Cqmcb=Cl1kRaazxZ2W z3`0nvq(W7uit;6k#ZOu)u{&o%y~H(#s5AMJvMOn)I5+_zy;tmvrHwY`7?y5X7Vg1G z4?bw)5fxOHqRfO@7ruFRI%ZkMWF4$;moyZzt@ILNC}mfH9u}2e+{;LXx63s>Pu!2%_wAZ>2|8&6e`IE(^M@bx4^)r8|Z`9X_ zWaECYY6 zdoLnXp2eQWe@(cC&MHj|$G7CJeKI{? zo=(pt;wpR9_;fa3UXDpG7WIDZ%B|iHEE4#b)PD#a&nAyjBEJHB;&^`G1+HTa21+`& z`?YKZb-zBA7xp_=@&`ZVk-$-c`e3a-vun-dT#Wt$P)h>@lMp2uvjP}H2n8vi?;0kP zN+2G8%TB{E5JmTVMdW?rJfKuol1hNOq7o8CRO~E!np(sU*@iTKkDIhx5Pn%90dCpqS=DH5K|;FXK6ioTrM@Rx7Ab4-PHryOJ5gw1qdT4OlP0 zMFFH7VWvVm?+wA&u^mY2lHgE5+-fVS52bB?u%YamHh}XqonZ-{a_T8Q0An)|(Gkhn zgfu_49RQqRXhDMV4rfV*&t4B!x|J24r%u1mUNu!j5@-mG5V zhgD;t4yR&(>XI|UEKsj)UE<3k9ImC#ufmD90{BxO^c!!-MaDkhQ3ycBxF_Yacz#(X zFHxPR`9qxMaW-3LkL11}#r*3q@~Zy~l*b|eCY}Ie?(0ub{{gd$A<6;?k4Pl#ObGx0 z36owVR{_nFwj?4Oi+fS9VFUmGbP@mn6aWAK0000000000fPq(&^CUq5>XS7kIssgh za3v-iDWLBfCIJ8dPyzq|5dZ)H0000000000fPp=eu_ZnMl9TWyBL+kx00000a{5zT delta 3937 zcmV-n51#O#OqoouSqOiZ)}V?@2><{GLI4050001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`OuoLg_=$QFg)&#$noe(074(>05i79<=>ADguq@K z$J5g@j?U?aG?drgRonQZ3fF?eUmvFvNOagkUVM}vVZPPD3o zi!_U;zpW05b%Ir>aMrOx7n^8b6PE56cJX>VP2@VJx$1v)Il_-Y7PJqO_(4gQJ*4xX zcxA+aVdSSJELy)6&BfT7>xQf61|_efID7YP^RRheBwj1N(~xW0a6=nzw;G!Hcs7GF z_(2&k27Px7SE#ToGE7Ed?@F#2iL)!YVI=OZUsYVc zs-??*;L>ixt>NWGs? z;*l(0o+Zp%;#Z~za*dppgm<2gJx@_*w4sX~_~dk+)0iH{&^hOVNlRXW%a?Hv+h6~C z`RS^A+BlK-9NOP&1DTqeVQG#kn!U$Du3Cey5+F7BpL zTC9J4(*C`z>CG&Wa=FTC7K^FodG}-;pU5;{!=%miVy3al+z;pTMmS$I;@RvmSoHY73SVt)pK)m&KrbN>=T*u}~^m!6} zB#~VG$0%{=2OpFGiHwAj$Ve!ODDmhAAHRe|CY)TfrzPPXpKx+U8@kv5tV3x`kBOn{ zP?T7t>`T91*rq`kr;SOl%<$+rKYM>oT1R;m0KD?I0Q5E2b$p>e{}VikLg!Ijwi6b~ zG;_>oTr#8+Mguf*8j2&Qp*%7gk4}vn4M!xGfFfCzptCu+ff8D}Oea3ca}H5GCW_v1 zyx=mEH^mrsg+8O~ZNtU%}WdSEL&UCn>-bZ^hy zUcujTKN5J0-L@%|sBr~_lDSn6P%;{TlFylTfv&`n)|TiH=T|I?&H`F z#S%5HL9s*)V##O#OGX1&q6UAlG|I4K5&)JZ2|AmDFIdu+F4GBONuzp96x>W@H2GSI za%ub@a9KxK_)AB zayShz$!LJdGFuLp0GKQ@<_K`+KueeD1epYL4ikk7SJW^I*E)X z9DR>XKPcjsDcn|B<|@a?@fdAvPTGTt0&-(fT^_3tnm zq_Tp8htmL+jEjH9p;VTcc(_i0$}$^|pe~y;Xz4PY;LTTX>u^ypS2C6!Sijb6*m;f0 zpL6r|jaqrG#I~0wU&ETVwa-A{@8Obh>QFNeqj9O5htVLH75qG$2DoG_IlyI^rHAVT zxGXdE2ypd4OPA>cxddYm6NPJ7)G%w&v~-zHe2YQ})*dbjpsZl+F>1E}l0|@lyqhNzk6ggZIlpEoRXl3;F(C_|7?a7djt)QH&2tOUX-rmgdzhdnq1p!)hOs zkP#IL_Q3LS7SETDY?H1L8Uq^+?X$}fngI#lx@&c!FgFPkhycTn;qXNMX#EDZq*^C)5KL+$$NSobx=1!WOUa|Van zJ#g*I=&vE?0?Dn6A8NNk6bGis!&}HGH}V3Qw8VLtQ6c{=*CsDYpb@_X5Ls#x%W}<( zCQL(NUNLc~5y#O`f5no2D;RGPY!PiVChCTp;-f4@G=BO%Or2(*DMjd z4K9FJsc55VN;67y&syn0&^BhtMwB$w{f)P3+b-fbO&GM{tNaHrZYu1vNyjCnt7tYs zpTJlfNNRm z8EwolEZwp!+=G)Ie9*=tDyU3F840saeDmydoM%}s^I(0uq@j>yrI!#xDXR+fu&DIn zUMdydF7Jrpf67B6h^b7@Wtg)#S(|&4^1aFSCHY>iY9*_*(o-s+Hy^m zVx-%=P2q{By`tCfw6$$`5~ahSc%bCi*#s@_e*r@Q@&d=YLD<#Dc?qbuHtH?Y`S&sM zJa>Rx`je#PDuKWlxY$GJ z>f-D4{Bk*;E|!-^XY=463Xw5VK zhYZZNge7cQHJq(B#SYqW328h)o|MW`Fxp{QK`A=QYh8$cI$-(y$>h?b zB#x{4f0@)b>T5)^B886L-?R<82X}gKB^w*u>%sjVT=8X#V@1FQ?`em)ryXLY9X5E6 zzqrR=+~Y6q@fY{_ixqz%T0^^kHQh+h(r}9`!=iX-TRCH!Quc);1Epv6Wd-J(3NZTHcjD|ViZSJpBs;)&xe@U^IlKuITL;wXZXvtxDhYeqSOnPeh z|FP}3Rnrm6z#r@0iwIwrdCxHJp9o${_Qx~YViI@RgMYPSI55Z?l~iYrk4(!qv%Nvb zk1vDg=-$5$-i7gfHT}R!=wK6ZO@uo_*QaE&tYdiHEpgN_yLNPe=SJY?>|Hc7`E`(f ze_DJ-H;}XX_ot^5J8*8=3o9mlkzdS_^4a%oHklov6F8QU0JONeLADhrO6(+uD$-+N zplxC7*^8RQYA<v}t* zG%+0ClC$>7^n7_bJ(q~9%vIyl*?f69K)zVi^R+9ddOxs8;A2w%A#^;OJX(qT3h;^J z`GFU>jx`u4;XIbN@;m142WRBm9XN_|4fFc5_AeTvBU#BmUaT1k-Fic^s)HB~Eamc4-0_(!(ekhiZ15J2M8<9cSlon5(H z>{Xlmg7ebo3Xv>F321JF)Y}SuG|P035+9frtTh@c)PqNh`k~~Oa^v9584E5*@JR$| zPr0qoE(A+4_IwA5`7Ang6gS2x7GmkPf7mkq&9;DxJfC3&fe98EJ^)iY6441M_=vPW zt{nh^V`xEv4jz+?;EOkaQ~qwlpj?~cPyzq|5dZ)H0000000000fPpuYxFtRTi<9>yBL+Pq00000#XDgq diff --git a/summary_results.json b/summary_results.json index a9f244dbb..d103678c4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.000", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.4239, + "Performance_Result": 10.211, "Performance_Units": "Samples/s" }, { From 1813f9358aac8279e8529859885b2d7b4cf08d13 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Feb 2025 18:17:00 +0000 Subject: [PATCH 097/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9760 -> 9760 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 87a4e3a0a..67380f511 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 17:43:37,788 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 17:43:37,788 submission_checker1.py:3270 INFO] --- -[2025-02-16 17:43:37,788 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 18:16:55,458 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 18:16:55,458 submission_checker1.py:3270 INFO] --- +[2025-02-16 18:16:55,458 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 17:43:37,788 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 18:16:55,458 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 17:43:37,788 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 18:16:55,458 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 17:43:37,788 submission_checker1.py:3286 INFO] --- -[2025-02-16 17:43:37,788 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 17:43:37,788 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 17:43:37,788 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 17:43:37,788 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 17:43:37,788 submission_checker1.py:3307 INFO] --- -[2025-02-16 17:43:37,788 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 17:43:38,917 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 18:16:55,458 submission_checker1.py:3286 INFO] --- +[2025-02-16 18:16:55,458 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 18:16:55,458 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 18:16:55,458 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 18:16:55,458 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 18:16:55,458 submission_checker1.py:3307 INFO] --- +[2025-02-16 18:16:55,458 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 18:16:56,454 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,5 +38,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7386 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.352765 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 67adbdb35..f8c40655e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -724 +725 diff --git a/summary.csv b/summary.csv index 342e6d942..e4ef12ce4 100644 --- a/summary.csv +++ b/summary.csv @@ -16,5 +16,5 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7386,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.352765,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2b541d135..1683d7458 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.352765, + "Result":0.358519, "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 586d5a942bc35a8085abf9f19dae99fc3136b90e..941efd8a71fd4dca5a94b405c3df3756f3aab047 100644 GIT binary patch delta 2666 zcmV-w3YGPsOrT7#SqKFbpFSi_lU@iVe_M0n#ukO&&#zFa=3%BP2(^T6EB-H z@z_qCCrTC#r~-&c!k2#^2{4Ae7qT*$M6Pr4FoD+EyL$_L?pABT;jfR=iE=NKG>T_! zRcN}Z$XOTge-dulfxT=z7!E6vr;+br#Pcl`1?fmEPAx?fz$5LiWo?w=?f7Nl8 z%@3N*G#txmkZSQ<&hjI}IGF}n{x%sk(|IC;!7^w%X&Sn2H>W`~Q&$-dk}5gk;V=s2 z$2eR}rL#WNp;j-`BNIs-nua%;d zyEy*6cyl^vt9l`AGVSN(FfJ2ie;|j!Vv=3O4_{?88fW=wvx<-~o}|lvm1&fpOG8zr z!DCxBRplWXWaA>SGL8lVS)6E92^VP=Pk&n-66*x3P+_lQg)TPHo+etlZCJ(U@idX^ znC7b2TRmE+YVyDs;uC&GC4N@srv^-Mk>yqo&B{y6` zKiASEJJnjS+9Qi4aCf^nf947@%oDBV;y`O-D)ccuBT$1I>vjqG}{0e#?*T`u}c=!3(fAbV|MjL$Ifk#f~ zIgRP@7&_-%fVAWzxO^G+u=Dl5m!Gcu)85(pEdo0rN!DO?cZCP-pmpym#eI1v6yO}`zP!8M5g%~CT*@4 zGmTB=emI{u!ug^Rf6r!*$zqn}tEt9X)eqy@FdC`9r~{b=(Iizr95#nwzW!E6N%^W( z=0jWlZ_l{Cuf`=-YYeSr3d=DubPk5j;n0yw?U+k!a;2Vd34;zuD0DzVp#u`~C;W?3 z0v!)`DRd>_T@X(GBA;!Cllpuxc=9*@+@VDB7`lGD;1Z8We=eOGi5!jlL0H_+H1gY* zRjS4)=W+g&#Ys5s-y52(>5Xt9j8+8_>nNoHi8sy`i7i)iOn1-H(#0UU97J;c9CL|H zuAdVwVITqt1rbOnh(JO?bW1?w;S9u*@Gc0aAi5ObZZo8I_v2_q3mD2YHqN#qij-jkjT9|3%m zz6>NoPhb*-j!|5;6Bfxda||>N8PW;RFdC7D;)pbqN1$=()VR?gB2fa0WLbjl=HLcO z*5ZRsJd)>}MfE@w-Eq7Cn*8&VB@HcqwtEHMb3YP%kKM8;m0aTpN+sBf9@ZVmq}Lb)|r7E~I#CVl|R zI%=#kYpw?-)_cL4<(T`h#7~`$1b>iY*%V8zaSVzj*KjO>hG7Xb3`?%zSQ=%2Sb_u$ z%aR1$&A}HeX^Rg!;aJkB9*APQsX!xiLAf;k54fzO#wt6k>w#_VCk@ly$0dF|dL;N$ zN6VpHa*bnBF1dzt2{epLpkZ8c4d>D<;}RrbT$Uv0ZVtY1Nn3nW3EjwHQ9TgFxdhL& zIhW>}=928|{BkR?^9Vq12hh`j+}lFTsug^s012DWtnjYC16yRS$70=)|{~xA9TW> zi7m5Mc$};=UQyrcWlk@ti8KD?j9_GJBJ#0fX1N?9-!e^ zR`Bp34Z{-nXl#mQnT-c^!mune@(Aj)Ib$t8=!EaSGQSRrVv{A%umx;?&6b_lsQfv* zuW!`Mb0xOjJozHlv|9Tt1l}Glfmesxd4R^DejcFVTvl-OAPwUZm~t4GWu_j~3FETN z)+3m&hqd^i6V4^G_JAn9h~*lvh^^V)^Y4K9Mom87#kR|*aGu+?x##O1hK|WJxKYHW z<{qGNsksMe7@0^z$wV4|N+#0Kg{)C#?m-DCnPmyO1~d1t79Vv&pNp`l9*AOO0u3kA zsM*}}8kv9h!ALFjRe{pbccIj6OYE604Y3DG?h~=8w+Cok>g@p<4rK*z57ICwfrdd@ z=IucV7?fq+9)o#%Sc?xj@mds0=IudI49W`L9;0S&&udI(!vaTt@k4i&wbWK+^=fQ; zdh!K;*K+9`+1&c3`&1?)c_Js1R0-q!bgd#ct*fv9Ux|a;)i>o|K6m-;uiz-B3U@RT1=#pJTmBHh1^;-eTJc;5Y%AQue z7e7|8m;{|^Jb1rh9MobK{jrem--Yk&gLk3*;TOgDptO~|^k->a-LjYB1sYc8a2|~0 zd60~vS*lFrFfUZs@*D7E_3H!6*I7JYzOq#1F3z%e`uuJzgMmzncc!Wg<2ZYMQ=F2` zuQ+u61hdT%ehvkcV>}-vlTaWYf6Z#cFc5_AeF~$yEz7|s2-|_S{XZig7ebo29YdB321JF)Y}HVub1f@B|b1MSZ6de z=l~uq+J~B3%8i3pXDqlN!6y-|ARyAUkJ*z+AI=CkO~QQR1(Scs+De`3q{H`@X( z^1Q+d0uwASd<3R;B%(7?@DXW$TsHs&$IyWS9Xuu(!B=kpr~G8Ypxii>3_QNJS92-U6XeuCL5GvJRcNF?n{lU@iVf9-Rk$QH)G-@k&Q=EF?YiZsYuD!P?q-eQQ! zWRh&%PZq&esa0_S^YY&>s4;rZBVlJ}m(;!UA*0Q6PWM56c1u5c`0L|zqTI_QjpA8b z6`HOpau&veXf|rAH`kvVuBxP2FdGDucqZHGlT6iLJO4R+h?C#bv6PvTCzz#ee|4N? z^MhtH4aagCq*^?ev;4>~PNqSYKTSr>be_myund|`nuf01&1n$L)K!Lqq)Lu>IE+I1 zF%B0~Im=c_5;+O7eE8`&nx|^#a1iCkih7hp4%_OHI5_#u5Nfk?xa|8jk`L+DW2GqN zE{=aMo}3Qas$NK&O#68`jLSqBf5>65m}FP+!&e!N##w&atRf_gC+YI9GL72Iq;Vx3?WD(rQv(8VU&(?m$8F)|z8-u<(hAXaUwa74m#M+fy1BtyWxd9SqS8@v^ z?nbgxu8GSvSpq@I{Qyqm9|*CK`P~nmPbl`U2^@psp#*r&Fz_ zmsuA`$cu9ZB;@4!Ktj&$IhPpZ?Dl|!oZSl`A!qjzNXXg!1|%(fx!L$~vsnU*6m!fa zHaYhvTw;=XKLQD<_Y;tidOvfCOR{`BVjhvQ*cb|_ve@{_ow87^cc;s}R z)0iHQp>xg!NK0OV%a?HvJ752M`RU3(?VY_}u7=WlIW&s0@#0Pk<7sm`$^TCCrB426 zE|XzDnhj-=FW6*1lWEpZ7kASrE!IA1|K8H{W|l~~TxB(j#Z>d$KUv2oGR@a8X>+}p zX>2n0!}+`s&KHe%e>Qte7PBm0O*Pi4ei+Y&(MbJ89mp(*CaL=2usH&Ym8kbnDF|?K`EXTyqIT$*JLq{&PV=l4Dm3qP@3_2j8&;bdB4oJvb_!p-H zIv(y)=t{!7Ae_7-pKXVe`g|~W@}7V0P$GE@UB6v$iAN-te@=}=j>i2UEI!XP@{cd8 zRE<&2H#A$*8{tA2tqLU8QA!08Z=5X>TdwAq?w+Nki$Qcbh~)Y?<`SD+ zKPOzmKm-yBB9KrJfrNtSmVn5^8Hgp}T@X$|bW1=4gQp<66ht1=A~6?S;t|QE`@aKX z9i>zN@z&j$BN9!^aBK^c=wT8)P9nMbkGaIAAAGokkq9J|L?EFga*0bnlb;M90lbsH z3?xG*Fo{CPC@$Lxi)5NP1{#M9=>%vPjYvarL>kH?(71GJ+-MMyC;>&XEJ1g3a04Z4 z@j)jZ$#c%4dLWA4al8PU{PmM24K06e_X>Q^{Ydb8?3P8TT92VbzHEk5XkV@acWAd1~h1sb6X%BAstz-1jZR=LBv9@ysdq+#0oxWwC|M}qG< zS`OusYaEku$u*oypkZ7B4daq)IG1J_mmmS-vLr!wbMS>r+Tx>1=#3l})dNwSOR%TS zxisH2m*l?AUq`Zz@+u)|?jnEb3ftI&B;LnA5`1smqI=swo3zCbwd?H+|nIt@B z_tWKESxl0`(~KV=vX1I1EnZvOi&xFDgt^CRBKCMq1h)@Z9wm}%9E+3C4kCXISQgbo zQH)5SVV^0jHw5HAaTm6h0xL0TZ82$>ny#CB_CdrRLyiD;8|ugb8i!hPfQB4rdXEQcu*${%Q7R6pgx;3*5ZRs_}y3L*FjNivIH8ofUSSovhy01Kj-f2 z8#VJ>iETGezKAug);UM?=PMWqQcuBOjh6P z&!X8#N&i^=Vt`%zVt`&G3x`*WiBz(uxooTQah|_IMX{ND?phiQJh5C)2jF4 z#|jpcpfimJ@0VeNTFjz97V`bO@ST0|F0?=Vq8J~Pwvw0rEX}K1_EJ1R!|EK)gONNB zl2J5Em5ChYh3Z=V0X$j#`oQvW7SETDELFLSvn-xIKO4(nAd}*msVc)b&YqtXr)2Xh z4xK*%v(6EI4h1Qo?;0kPQ6L_F%TB{E5JmTVMdW?rJfKuol1hNOq7o8CRO~E!np(sU z*@iTKkDIhx5Pn%90dCpqS=DH5K|;FXK6ioTrM@Rx7Ab z4-PHryOJ5gw1qdT4OlP0MFFH7VWvVm?+wA&u^mY2lHgE5+-fVS52bB?u%YamHh}Xq zonZ-{a_T8Q0An)|(Gkhngfu_49RQqRXhDMV4rfV*&t4B!x| zJ24r%u1mUNu!j5@-mG5VhgD;t4yR&(>XI|UEKsj)UE<3k9ImC#ufmD90{BxO^c!!- zMaDkhQ3ycBxF_Yacz#(XFHxPR`9qxMaW-3LkL11}#r*3q@~Zy~l*b|eCY}Ie?(0ub z{{gd-A<6;;k4Pl#Op{?GTLI0Jy(ByVSCjfAK>_NMJS92-T$6VtCL1ZB?;0in002+| Y000pH0000000000004l2J(IX4XuDzq?EnA( diff --git a/summary_results.json b/summary_results.json index d103678c4..01003dc85 100644 --- a/summary_results.json +++ b/summary_results.json @@ -212,7 +212,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 0.352765, + "Performance_Result": 0.358519, "Performance_Units": "Samples/s" }, { From 022a6bd602fedba790970c8cfa0d9c5a38e18ae5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Feb 2025 19:29:30 +0000 Subject: [PATCH 098/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9760 -> 9759 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 67380f511..aec9d7dbe 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 18:16:55,458 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 18:16:55,458 submission_checker1.py:3270 INFO] --- -[2025-02-16 18:16:55,458 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 19:29:25,147 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 19:29:25,147 submission_checker1.py:3270 INFO] --- +[2025-02-16 19:29:25,147 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 18:16:55,458 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 19:29:25,148 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 18:16:55,458 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 19:29:25,148 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 18:16:55,458 submission_checker1.py:3286 INFO] --- -[2025-02-16 18:16:55,458 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 18:16:55,458 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 18:16:55,458 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 18:16:55,458 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 18:16:55,458 submission_checker1.py:3307 INFO] --- -[2025-02-16 18:16:55,458 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 18:16:56,454 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 19:29:25,148 submission_checker1.py:3286 INFO] --- +[2025-02-16 19:29:25,148 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 19:29:25,148 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 19:29:25,148 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 19:29:25,148 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 19:29:25,148 submission_checker1.py:3307 INFO] --- +[2025-02-16 19:29:25,148 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 19:29:26,122 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,6 +37,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.7386 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index f8c40655e..d825567d1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -725 +726 diff --git a/summary.csv b/summary.csv index e4ef12ce4..2761428d0 100644 --- a/summary.csv +++ b/summary.csv @@ -15,6 +15,6 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.7386,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1683d7458..a7562ab97 100644 --- a/summary.json +++ b/summary.json @@ -489,7 +489,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":48.7386, + "Result":47.3803, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 941efd8a71fd4dca5a94b405c3df3756f3aab047..2476ec54d9c00e3b50e83df9f1c0b14691dae1ba 100644 GIT binary patch delta 1845 zcmV-52g>-MOrK1!SqKF**Q8ENvt9_#1Aj*|U40LTE(X!%Ad>6nm`iMO{hV+K0})6l zh(JO?1QH6OTLK~vXCRh@cR@G>(JcWH44#7MQV@Aei^N=TiAN-t?*9&mb(B&8#9Mb~ zN;ECk5_@iYJWQg;NhDYQF_+l%#)nH7i9kY01QJRjm$>xC$15Qbgp-T*v?RRi5r0n3 zXoJr?VCzsC(*rSd9SS7mBe?YHg>4#yaoU&!%M6dM{j=Akb(B{Dz$cV+Y|cp ze*%*zbd2J%ov=uznPZ@F$dFEehS7*L6i1|?JOYhNr^by25s4B|B+C+XHwQORvKAk7 z;*mV(EUE{h=#JwB(Bz*#L33;TZ-0Z$mnpH}avHsMPNgWL5+q<$mL%v3 z`c&bJwfLYD&!AM&s2+-<(h z+=nH8>U1RdgB;7ISaOYHP%OEIV+k}2OQ2y`at+7QD8mvYU|5zU=xz?aU`bni&_HMg$3GJM*|Vv;Y0!rBNAvBkwC+Uw91GC2^f(j z3A&quFGSK7A9TVtU|CcTMKL0QhTT(IZwSbLVi&fS0xL0TZ82$>nyznMoV!f!G2{qf zx1o+4pmC@r2WU8x6@N@QNW+)}8pdRqF9#)HOqMxw1heL_79VuNnPl!95XBd+Tmu%a zbvktZ6)4}TUq=wbj^Wz+9t*12aqQ?XR02N^wd??mLp?h{!>O#`+CdsdCD1S`%ZxiH z0i&|ax+AEw=8U!YpcDQ~ggJCT6n`edHQ<>@%`P2I<qCop3IhwFgA;MJ(5VMQqLXo_`0-H)`_vF1B4h zh4b9D%{^cDFmz0&!Hps|HTM9GOU*q%!^lJ$N+!}!GLeQZWQ{U&4@yAEEKATen7N0w z_^1>5T!cmSKolbrXgHZh&E}rh$o#txMrx_A3Y3Pv3xB0_gmP_Zz=GHger!pDI6FHfrN*L#-YZbX^U48xkN*vs- zzA67A&wu-iXM?El_6L*I_xiJFHd4|*R=*fv7rz*w7s}f9Bs(hU1pHNY( zstg{7tKSMx=1CMMQTDXzz4)<$#U$uV}kA-~yE_`PnybJ9QzbM8BrLE+p zKTGrKmc0}&(6Bm(^I#;;gJcxVQe`5Cd7-+N-z$J8t6v{jzRu$L@|C43cX5`*)8}_% z84P4nyfam07{}T3o8pvge#N2lC$r8Gehvh88EUwbP#_3ZuO(+aaMf zwgYX+s)0gXTDlX`#1XX~NNOGbeHFiwfL8@=q% z@r!GSb$h?!0|)lld41t+9Au~i7We?!5O-2OR!`3*)ee#uXm zoB9h-j7|PcoB%`a+fPvb0ke`J$^r#6*Q8ENlVK!V0nL-WBs>9Ellmk<0qT=HB{~6I jlXoR11a}!~xRaqJ76CnzxFtRTk(2i&BL+hw00000Fp6YH delta 1846 zcmV-62g&%KOrT7#SqKFbpFSi_vt9_#1AkX@On1-H(#0UU97J;c9CL|HuAdVwVITqt z1rbOnh(JO?bW1?w;S9u*@Gc0aAi5 z{!d^Mg^p2Nwi6b~G;<6z4jIx3&@dX2hT@1clt-X(>D0K1ejAyL+~K1>SQ%5`2%{vM7~Y;|NM6*sEt$0u7@QXc(1T!>JTyRDuMI%8~?K zL7ys|u@)b6;u(}m8r4Hlbh<FJ znESBAPo0hge~@F@6icpg42mV!a4dm_VF@%0ORnKq8f9351PsfP1l`TS7c6Ou4?5vk z(x@JYV!Nq8BXmKzH2x2`tfR&%JFM%0ZSE%x)85A=emr_4_)|yApYB!B2`4!&?nTYOXr-N<24JrKpY1kbcNm*$)1lI-jJbtLO3 zuM(2xE|RXWjXg->=lDm0KO48`vu&VpS`q)|N-#gOD0q0{HoTjQ?<$;Dkh5iZtD5+1Yr>3?#rEG9|eX~qu_ zSx0r17O$=C#jEC6!rZgxD)x9y1h)@Z9wm}%9E%dkHJnJGVMGEABNAvBkyaUzAORz? zBtds`@P$a);)71u1}ux}p(sWq(6D<->kR?^9Vq12hh`?*fCsN-{TVyJB}Uwg-YPZp_Uz>aj0hpXgHM>Tsug^s012DWtnjY zC16yRS$70=)|{~xA9TW>i7m5MStF?d*@ng+jnfu@T|SN zJnkMWfjfs9c!0*C4j!Q4SXS`xAPvJ3_-Jg3Wtoi!b;7VLGx7-PvpHieKInw+zB0cK ziei%`(69w;&6b_lsQfv*uW!`Mb0xOjJozHlv|9Tt1l}Glfmesxd4R^DejcFVTvl-O zAPwUZm~t4GWq+m~)CuFV%+@2AuZOkxpcBp|v-W@}zKG=-u!ybM-t+H(`9@7X-^I4e zr*NLzwz=o)9)^y|G`La3rsf`?ajCfnXc(DDL&-!MN+#0Kg{)C#?m-DCnPmyO1~d1t z79Vv&pNp`l9*AOO0u3kAsM*}}8kv9h!ALFjRe{pbcYmSOZAo|a-LjYB1sYc8a2|~0d60~vS*lFrFfUZs@+=$hWcBL<%hy>vU%s+b4Tq zEQ5heig%`}4C6R^ep8&1&969g{sgnl5q=H?lw&*}lTRQYf6Z#cFc5_AeF~$yEz7|s z2-|_S{XZig7ebo29YdB321JF z)Y}HVub1f@B|b1MSZ6de=l~uq+J~B3%8i3pXDqlN!6y-|ARyAUkJ*z+AI=CkO~ zQQR1(Scs+De`3q{H`@X(^1Q+d0uwASd<3R;B%(7?@DXW$TsHs&$IyWS9Xuu(!B=kp zr~G8Ypxii>3_QNJ0&^+ kU6XbtCIpmYJRg&wB^CibleZ;40g{vUB_jqzBLDyZ0O7xF5&!@I diff --git a/summary_results.json b/summary_results.json index 01003dc85..e0ce14620 100644 --- a/summary_results.json +++ b/summary_results.json @@ -176,7 +176,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", "ColorKey": "availableMLCommons", - "Performance_Result": 48.7386, + "Performance_Result": 47.3803, "Performance_Units": "Tokens/s" }, { From c5fd6d2d88d455c877e5b8df09328e11efa83e59 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Feb 2025 22:29:44 +0000 Subject: [PATCH 099/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9759 -> 9759 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index aec9d7dbe..bf2b47604 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 19:29:25,147 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 19:29:25,147 submission_checker1.py:3270 INFO] --- -[2025-02-16 19:29:25,147 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 22:29:39,641 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 22:29:39,641 submission_checker1.py:3270 INFO] --- +[2025-02-16 22:29:39,641 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 19:29:25,148 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 22:29:39,641 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 19:29:25,148 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 22:29:39,641 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 19:29:25,148 submission_checker1.py:3286 INFO] --- -[2025-02-16 19:29:25,148 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 19:29:25,148 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 19:29:25,148 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 19:29:25,148 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 19:29:25,148 submission_checker1.py:3307 INFO] --- -[2025-02-16 19:29:25,148 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 19:29:26,122 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 22:29:39,641 submission_checker1.py:3286 INFO] --- +[2025-02-16 22:29:39,641 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 22:29:39,641 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 22:29:39,641 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 22:29:39,641 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 22:29:39,641 submission_checker1.py:3307 INFO] --- +[2025-02-16 22:29:39,641 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 22:29:40,587 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.211 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2374 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d825567d1..5a9a14151 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -726 +727 diff --git a/summary.csv b/summary.csv index 2761428d0..36c9cb4c4 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.211,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2374,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a7562ab97..2f65da465 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.211, - "Accuracy":"acc: 74.600", + "Result":10.2374, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 2476ec54d9c00e3b50e83df9f1c0b14691dae1ba..790e3a771210a7b66bf5531a7d11dc3453e8fa48 100644 GIT binary patch delta 350 zcmV-k0iph%OrK1!WfKIX_hON=X%n0R0b8>M7DWmKdp%=)lTRQYf6Y$AFc5_AJVoSt z;>3jttR$5H^+Y8kia^{fd)r#XKe7#J-ySDv(^krf<9cSlon5(H>{XNeg7ebo3Kdz7 z642ZTskaq+UoX>pl=#53V2#mGp#yldsBcPcDK`#Yow4A81fN8Z_LSQS?Lx2=W6yV> zn9rg^M{#4EVj-4pe~T^S-)sw*7x;7a^471KO5L4`1FB2GDR;nv zadn9=i+H$(`LK#7-bfIShp11y9Tyq;KqN5$9pavpkJ;1nC3=Z!l9RhMCuuQX6GF)Y wz0JS6kyrg^pgIouH}M1*a$kRf`VX^_Ax#4Wr1xTxlj0=01baPWeUqRie}M(2KmY&$ delta 350 zcmV-k0iph%OrK1!WfKI8dr`2nX%n0R0a~*L7DWmKcNuE9lTRQYf6HpaFc3xe{R*SK zE!!cXHMRq7$*O@uU0S*m(!>$99!P2(|9us|l7LqQZKQkW%$-5`_9U9%7mVdf7AT2g z6o90PaakAWz1oHMD6o!5Mj9odKpj}LE$?!wGpY={8l}NF4mR*0X)~$|baYN<7+ZP- zL2Tq5(u)UW1abaqe`>4={U$ZwG>$h|fFq1Jg3rKE&qOpL8J&^($22{FQ49?Tkj~;H zO7O+&zzDlo&?{F?!Cj|e(K?B<`(*iSY}+fCfO`HHj?%PjL{sFU*Ax#4Wi+fS9lj0=01a}!~xRanIe>y3lH2?qr diff --git a/summary_results.json b/summary_results.json index e0ce14620..02c6f21e9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.600", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.211, + "Performance_Result": 10.2374, "Performance_Units": "Samples/s" }, { From 9ab6a75568231c8d2284067d594a8e5d0e4191ea Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Feb 2025 23:20:09 +0000 Subject: [PATCH 100/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 9759 -> 9747 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index bf2b47604..df48f5292 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 22:29:39,641 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 22:29:39,641 submission_checker1.py:3270 INFO] --- -[2025-02-16 22:29:39,641 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 23:20:04,712 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 23:20:04,713 submission_checker1.py:3270 INFO] --- +[2025-02-16 23:20:04,713 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 22:29:39,641 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 23:20:04,713 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 22:29:39,641 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 23:20:04,713 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 22:29:39,641 submission_checker1.py:3286 INFO] --- -[2025-02-16 22:29:39,641 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 22:29:39,641 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 22:29:39,641 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 22:29:39,641 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 22:29:39,641 submission_checker1.py:3307 INFO] --- -[2025-02-16 22:29:39,641 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 22:29:40,587 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 23:20:04,713 submission_checker1.py:3286 INFO] --- +[2025-02-16 23:20:04,713 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 23:20:04,713 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 23:20:04,713 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 23:20:04,713 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 23:20:04,713 submission_checker1.py:3307 INFO] --- +[2025-02-16 23:20:04,713 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 23:20:05,662 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,16 +26,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75139 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.428606 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436634 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425368 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.42599 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9484 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2374 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.77691 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62305 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5a9a14151..f4815f7c1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -727 +728 diff --git a/summary.csv b/summary.csv index 36c9cb4c4..4a33f6c92 100644 --- a/summary.csv +++ b/summary.csv @@ -4,16 +4,16 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75139,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.428606,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436634,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425368,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.42599,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9484,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2374,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.77691,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62305,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2f65da465..ef2f6d092 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.428606, + "Result":0.425368, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -184,7 +184,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -214,12 +214,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436634, + "Result":0.42599, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.77691, + "Result":9.62305, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 790e3a771210a7b66bf5531a7d11dc3453e8fa48..4688c02cd06b2afa4d5a97b83f0dfdecbc1c4c76 100644 GIT binary patch delta 3966 zcmYM1c{CL4_rPZ~#fXuyjMp-jW+K@}BW20{+JzTc#=eZnI-%^7i0m?Elr39@m$EY{ z%Gmde$T}G&B}+pQk*|K=bH2alocrhVx%YGLbDneVV>6;ztpf)HrO%qw1p@%5bO8V% z01<@{ASf$@2lz7!)~~N@mOr`#OrOzvh@&)UEJO@w?^d8uS;IlbA#wqBFaj>sl~=^&^`s>)S*KOv(1h#k(UoYUaeBi8LJq|m@z6&;EN7+Rg!5?WgMx%dgqLO0KQ$5Z6C(3Jzch7mwqqz(pC$MaseJ&gA_CMe{g-2+edlxPBY){!(w%6T- zOG~7?!7M<0XNl2N4@Z*X$&m<%MeN0URV_TQ7<$i^&jWgTOf zrdLC4G`1@aCrE>mYUl%>Zu;+6EfQ8KI_Fnqu$%1xVynB}3=%_~oAQ)fwQ=!QnT}#l z13V|0-MbzoH0#;QNbw7!>C#GX7CKY+69P?aa>EL%2 zW8b!W_}`RS*>1Zh<`^8YJHOoex$l1Y#1wMsg=t`GLi-DnadTCZdz0ZMg6YSA+xQ^? zuZPkYNsiOvaS5VYkMAUy9bhFSKN5gYw)yK4l4FHz;44FF+PVt`7bPT#A^`BC`49zE z|Jtp%grSX-Sy|JsK~Uv3`knG*9#wYnK!~Xu(IbP>B^g5KL<&~!p3TaN)B=JlcQ59K zBL;J^;GaeZoRJc}eK-R`_{v_>3UW2=638w3`?nI5GhA<~;IslOd3nowE`Dd~$R+kA zr$rlf6V_Hz{Q<(g6piknm3oz4>*=N&*S$XvANErF+)ax!OXbdAV=50uO&S1l=?~q1IPxr_F8@Zg z&1;!Q5Jt&<>P&8gQ0B!S)RL=!uRmJUOk{7>)CeUt+eLHjH&BgFk^WPiV-z8^ zTs=cblcxf{??GfIl1ojZWDE$t2Qlh1{0VDeQ-g0j7|o=me1mXEi_c*57t7lLTtx|w z+j5NHdb>d|41+){RhSn}7R`usw<#J$r_ge}t9oib;&*bSap=|g{JgTi(tQ<%KEV-_ zRs(RvLaPYklJoFZnwUhSkbY*oQO4sxxmh8{x%DMVB-p?wd5vW5yPUkQQ4nX(y8v9| z*MGsLuN2N%YN)e}GKYq|MP7CPG5(7Vtjn<80k8;@vyM>t*@o^68eE+CIRx7OjxdCT zXtEZee@kUqLgNy$`;c%=7VCl$XfagWk#~Uwmc}+p0>s8*WYWKE2|bQ}i2;BYS>rEY z&Bj7V2TxH2WsS&@(FNk9)dC#gvL^!A&+~Nro922%MR1-dYxR|u%RaFv!OO5K5Pffo^SM1 zd5RZ2lD%CA9PpQ)K3bbUJ$ym1yMqfWFR-Dr^5ce^sfP0~iz}%2y-S=TMI0QPJl}ji z2eZV`ob?U zfm}!Ox|BLQvQ1v%xmduU+KoBvI}Ep3wZciiX211LC_PjSdT=#k-BOOyqjH{%q1J(f z6OZX~ltfRQojVw+BzFNCI;q(1X6(c^2JX%NJ4}ba?I9<+Q%NTG(-AN?j4;tUdUEpp zo5^Ryr_415lrE+TJ$PESZmmG+X_CpsP$?kco@1f{B{2kNi$5kR{FfMfOtb;N&Y+gQ zlt>8y2jW5$YE*}jl;6CR@#e0#2mQmK)vVHiY^L zWouKjmW4;35{xeWn*}OVLj0F1@u$w@AZrB924+wvCO5^QXNk!FA$6tWf?(%figBlB ztd>JHi79nHdiVe+^|d1K#jsyU$aWQ_x%QUx=V^Bp-Z@RTUpYq4@c|}?sbHzNL|!mD z&8XO2f5uk-E-0-p< zb9pX(ob$PvByI?3;Z|7&q??Le(WX9{qY~rMX~xCwW_HszE!ld!NGL-kEW{72EnQ&j zGXk`VUK&twTXKA+v*H@JRWlO?VtswMvq9^Zr)IEpk+n;fy^yiiiTV@hf&oo}nYt0R z0wnymHz|l6>1JbhTc+hcnUJKd6*^x;*+QXT9c}@S zrnTI*{<2;S4G*b^VfV$X(t*MjOgIEy?ry`zT~`{TE~2y=bp z#BNY`cwwye&~G@gaH`^fa0TW{uOO|M_(w*hXjXfxc0&{n#V~BoAS&C(!t)y1RLUH6 z^jO%m*d1f%5P2-@O{z3hy3X@|!lpiJ$HF{=DsDVZ4U`o+a68>Gx;@oAbJRn;ZE)IC zSjddnrZ2j>1{BUzi@K8-myG5&D=%@A;yy{MzPVFO0Pzo1fy9VcqU=rYsFmC{u(yHf zqm<$j<8hPz>k-^T9Z^o@GfjAv zm!DDOak!fgKfGGr&QjS;HcQ&l&**&wMRteEpPi~1-2CmD96!sPT6SGFAoMqi=l1*# zesy3?fL}{8+3DR{VJkR}G-U^LYhLD)Y^`B0Q&MIFLrgZ+mh^kcZBGw4AeT-=k31&7 z&h4;K5>Hw;9w&4NO+*VHp>j7J=mwhw@wXprFT%PHiW_rEzhiM$^QF04Kw-OI!GaKG z#6!pu?r9pcSY~1@WqUJpk9H!pH$vG-qu!RO(7)}qb>NSnwwn!N$yHq02tus2MgP9E z=nR3aVVL(J5d4c3o9zk$04#}td=dn;GqGA7odXfmF@4>E&nrUj&D^bGS_|0MT_fe2 znT(vXuvn}0CLH2@-P-Wh9|2J7Zq=RWB9`h>|F^+&rtidZH2SAY61h$F-gIi1eAu^a zYge0$;l}GY=+g?`mfYgO{8HZc>!WDjys(rsQ*)*}4!8A9PQ9CGb_D_T$$4G%n2Tj#n&rdmW}-K`@MxR5Wjbe%6Ua$lggZi6IOUxKTt^Mv?dv=F zM(`C^+&ks|*`F$=prH8EA8lo4eDlWBpz{rOk!Kpy9qM_J=3#`TnwjR*%NBg60)EOK z0n_q1fBBE&jsB9&Gar*ynmk3wrsx{l=G-4$CoO3V555R~k>FZWug@Vfn^>lvR~||# zPqtK@4*@!eXy$&g9nnzSK680g;HaC<=_#bki4<7%sFG_R@?;UoCL1FmC2&Tb-bBP+ zcD*dF0(2QDS$6#(?MddPPeSBC2f>ZE#ynSjmv{wiOsp0X98S|U)WrxZ4rEiY>Y!XS ze|`0LsE3TVZ5&*Xu#>swF{AcG^FWr+a+DUPt(k7c1&z~1Ng=91E#PT(^9-ZVm*>_s>>nZo^ z;w#AFg_(*LvlV)8-!@?mcyk>~RyEe0ZJYLNvh};p34={Z{}X)&;~y^gmO~S;4B*x{ z8~nN$Xn{N{h!D_PD0L^Llt-5{vlxo8QszH213u2fyj zFUD+A=+Y9ajs9KU`dub@sVer|kMw=vchyEEWxJ{Y)Qz`=aEz5_e3WJ@J^n(SKfW4K z`!;Guas!uSk4xfS;8Wa4hV6ExMlrfISsqB2`;%iEuI8%a4d^+k!CD#kjNQnGr#Uf_ z{n}p~VjNdJJ`+}Z{F+3P1kN>2m%UW|6%*$lby->XQ*;ZZL;0UDd9{=-7n7Y8Qz(%C zRWUM&DRwFP2kJ{D^QgC(&(7RX?gYU^`Z@#q2@D{ zq3~^vqdK~ApkM2-@n>3~1op?~wQJJ1KF&45?97V(iRCsNke+Xu_PUi|@zLJruQA5z zscE}neM$)H1{>TUv$>Ss08=T&DMt9~R^W6~H2)rM3LVW^ZSpeQKH6`nsn>;tPln!4@7cpYYznoAGjg2R42@UgTOvoFs}e&{ z`#cS+Fn12n>4}hM#A^}~gd0?)GYOk9oMmnD(~>0|vvJMK6WS#T@~^=-;(bRgqZ?u! zF#^(f=(qE|?}uX5+bS>{w)qEf+vA@(nSbekeFGCFW;tWf|96vSkGByx-w-E=B(<{*2l?Lqjr9YR0L1g8BX*EJqcNANyK3IM_qI7gxD)RS73$l@=PgLbw{jOQu15fe$ zh*QPAtypHr&cS#!!qPd*qbOTy7-o(tJ6YoOJ4PsiMgUgPvf7fdZ&17I(iKO*h6j=Z zY~cT`#zZf~bs&}a3Lz!(zt0;A0Pq9;|J;c)C+;D1fD1%*q#DqYcnf*%zb!9D3Ib8Y T7UWgnJ>m}X>?zf=e=hw84!?P* delta 4034 zcmZ9PcQD+Ix5ig5!LmeICCYAyPFC-|1`#!g-aDHRBwrzVw=5Dph!Cr+h+d+XRg;M5 z%aUlX5~4+Z-aB{hoqNxpXU=@i%z2)3W)8}*$gs|uk`6IjW@JbO0#T`hK&&8?Hw+38 z@@<0g1I#hY6SXGqHXJJiDL=gQ6V(_wSx9!>IrwH&z^~6tTS*+R`zZ&wMI|2F!O|d^ zr-=zZ6o2vcdt3O8XL}ajJ*%~8GYLUrC`6*WuenES)sQtSxOA?vX2}-tYp%+4>+zvC z#OL``8%vr4yO*2$JVwur2Tk4d3mpJ-)0aRr4*Uv-;28Bx_d2j0KjiC`%_FzIl4g^d zdCD8zBk3XU32pV0d8>waV97?8DJX|u6{jW)%Gw$dgl*t@>#(82)wgRNg;fQ;YFW30 zw(djZe5!TuE9){EmALWMhJWUDmCI~uvrDtuWUo3xgXKM{!+3lpcQ*;*q5z{ScU+HC zrd>bS-W36LKl8*Uri&MrjRM)8Sw#jK<`q58Hd5vU9;nM2Z@pwHIf-??hdC9CC$lkN z!wK0Ytu}G2b&e60o+`Zku+`V9fAav5v7Nh7HA|pWA-A5CGsIKCtxwX&qwy#Pir9=I zbe+c^3j~X^H-;Tk$esnXr{Z@OL&kbHo@IpAoEm9hJbP(qug z$+hLEs*mrM-uvAy2)~-@w_oBc{jN>=8>Zbc{CTQhpw=%5U06Hm>tEN3i{_J38*lDZ*UNEX`dI zyaaGL8gB&H#r7J-3#H$pE|)z!KM&jETqSpOuqB3vWUqJJC2{9J@qWBlNkoP zxduK#<%I!M(R8{3NLBR`638L{Wzqv;PH@N_Bz)D5-sp7)_XzRh4X?h!qq;IM51ogtAxa#F*t#rm}s;|qNm z_&t|&lqL=H2RmE#ILnf#Tgd;0s&O#KJmdKhK#USVCgU3l>U1meoQng8(v2cxr+|0_ zlTKNQB5{hr+kq5vh6IQSY!rf&p^!0YV7`UKXDDt8S5cJyNL?eFTrW~l3--SBkH+QP zsb2}=CMb4sq0CPHiP_k|;_WN2XlkLAryX415{-4Rc8iPHMWOw|i;R`<9kkZFb*PSs zV9z6^HxLijY}MY%b*NFfz#9k*a4GN-Uj$Pfwl&d7t_- zjMNM_6i-h|9gtXkz3-fD9jqi?zjWiP@Zdrm{P`9Vb{fqwT45EG;%p+^66;KbpfMO2B zOWJxNFb)r`z_lQj5jigQo3(t_I#09H5g{4K7~dT=h{~V}FdzhQr#wCB%QQi^4@F%=_0O&1?HIqmOn5|gr#hi2*ko>DGQX}ECR&j%Ago1b+b3S`)>S!L;B zRyB!f`L6qAsP@J;j>MBvv1Y)jT3<(tX-;2@`$^it8ZZNEN$|diU3~D+pK=IrVrfDX z3rTqTN_ghJ!uZ?ew;Aq9tW!FCEJ@`k4y=Szc*vq|8p`SR=8{4OT=K*y;0}gpxnNT4 zM%}-_JeNHV`hFISEeSK`u{`qXroeKQy$*eE>gxA%t{YX~0(X9hnGvhdeb@n|XecSD z_Pzt&uY6a7YQrbR1xJ$=1c=@=jJ5JrVmJQF4OfOQ<&dgO!0k^`D|xy&QBEVm1R!vz zy{ItjG~em$9qYckNC^DxK(!$Extjz*_jn&fJJ=8vN8!ne?$&XBAUx`$SjiacnwOzt zQiNmkK2YP2_?>-`#okPXegg zXITE z4Axg27B-la%591Ja9YWDZu?T}+Nykv9a5t#8|8jBP7_Ga zcqNmdqWF6qN~&w7{@}>~0in?F^jNj)6&R892kUni3m=`Zj?MQzjF?QAEj78$B|4c{ z?4Phog@-?KmSks^O}y0#RKG8860Ej#o}WR9f!fgqn?oC4WftE`vJErJ<(28PZoc)t z+2U#U?9X3)js;BToxi*j7fJ5XmHQ_Ys4u_Iu^!T<(<&BStPU()ti>$DpH{y(@n-LM ziq2O> z`;A?2m%so;dChvciowwqx`Rn)qFTGEPMj^wYlzh%`qjz2rh zE*A_5MEB2ZhB&j)xR|!zHFsqloI84r&+>WRUvk86Vn9NQN`gVLs|;a8bY;JtG8{=k zAS46?VgjM|V9Y3hkqnS!dcJJ$=xXoA6;%IPNRYFnso1O8j7)p%0pT`3TtP2Cg|lL; z);^BBf{~w$IXzvlGdM!IsMczo?-y>Y z>ElAmC32STYO}^EL^&J0)xG8yx5o{v3!nb0t*Zf>53VeM`M2d7SA|YUg8T$`uzG{= z@@fxhwW4k-c3ohAn$!F~ukuH!L19;dWwYf%t%Wr&X_jx#6z4tDR{d$s%DBCj$oDCQ zqx*s(g*n2c<9SY9r#6l>8oDKA>gE-Rhw!9+mD9YM%BWmB{XIp|R%T_YU^I-zhWK$-6viUgMHu5vK3@wJ&I`?N4Kq zz76KxbTZQB;~m9=MyW6|W1ab^x5Y?GCU6#1;dOEUAF-kPk?q~}e)La{C4`8MPqd%8 z$p2N=F5IL8(!EvQERjRQCng7Ipv`>ZuJYRsW*Pyvk zA~kV&OHyK<8C~c4C7|g$lY++}ylfxzv}x7GOEwbNEG8`@%~)^LsEaa1HqN%v|3!8m zo}v27b@VtakMF+o|2!M-rbW$(?P6np0^a+zqAw-gJR5zX)Qlfx9@Gc1x4U!T3j^b` z1HY{(UE#Taz_hOBLHmU@F^kI>58BOac&~vvtlpSNT}DwoN2e{>g0Wo&5#s%l!5v zx7%HG)5b{?Kgg{f{k{5ybfOr)XH!2?E_RqCXR3VBiMx&ELlO^4GDb36$_zCT!yyX& zCFDQrB_^9BPV%+xIeswbqPXcrp3R$fAJw|o@!+1_6NHND69?HB4LnLU(|z2jUD$G$ zw%F~~rE$S~G=m@4G0!BG>Sa00AZSoZfc8SH2 z5(rmveg-E3flP=&AO;W!r;@4GwKzmfBBE1)_OR;FNXR( zEE{yJ35O-BE(nLuKkQR!p!Hnl;`f}f7)e+QbL%RwYq8xiM_oo8P5+?2X`Kq0T%dgS z@n)F4*i3lVX?LST%9Ke7NoZrGzrNocK!)y7LH$Y;X_037N!C2|u6iAl^F&B-?nJ4E zPM~wX6A#)iNn~x==SJ@O8HW(1VO@6sh2gHQrcT)k-}F^;1qEel2@#omsI3c`ayW@B zfuB8$@adh4tW|DfVc%58?;u2z@9PiPCFl{$Y4Gvk*V&g{cv3 lp~T?wM7Ag|_-*$8ug3q@Dxs?3%0wBcEjS;!2H(H={{Ue Date: Sun, 16 Feb 2025 23:23:10 +0000 Subject: [PATCH 101/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9747 -> 9744 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index df48f5292..54f3214de 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 23:20:04,712 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 23:20:04,713 submission_checker1.py:3270 INFO] --- -[2025-02-16 23:20:04,713 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 23:23:05,442 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 23:23:05,442 submission_checker1.py:3270 INFO] --- +[2025-02-16 23:23:05,442 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 23:20:04,713 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 23:23:05,442 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 23:20:04,713 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 23:23:05,442 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 23:20:04,713 submission_checker1.py:3286 INFO] --- -[2025-02-16 23:20:04,713 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 23:20:04,713 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 23:20:04,713 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 23:20:04,713 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 23:20:04,713 submission_checker1.py:3307 INFO] --- -[2025-02-16 23:20:04,713 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 23:20:05,662 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 23:23:05,442 submission_checker1.py:3286 INFO] --- +[2025-02-16 23:23:05,442 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 23:23:05,442 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 23:23:05,442 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 23:23:05,442 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 23:23:05,442 submission_checker1.py:3307 INFO] --- +[2025-02-16 23:23:05,442 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 23:23:06,684 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75139 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73362 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425368 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2374 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1823 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62305 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f4815f7c1..1f276db26 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -728 +729 diff --git a/summary.csv b/summary.csv index 4a33f6c92..a7eac4d75 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75139,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73362,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425368,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2374,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1823,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62305,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ef2f6d092..e867dfe10 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.75139, + "Result":1.73362, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2374, - "Accuracy":"acc: 74.800", + "Result":10.1823, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4688c02cd06b2afa4d5a97b83f0dfdecbc1c4c76..5b9e95baa2beb2a9a1035e25c7c1a39df85ec1df 100644 GIT binary patch delta 1888 zcmV-m2cP(pOpr{lSqKFtWqaa4vt9^V27i{dlgJ%b6T#gBwoi%V8rP;oat$XEXc&<| z!-xbLMxVhavB+#&PO8X4~`A=Mgt);-dn6x*T zG%d|AHV)2hCU^L81aR9>Lk`fm)R6-;oXI^rIY`5p1RBOO|l5WkLT?>?Q}GVB;HR02B=b?g9*OD#JQcE zw1YH^N}yp>R{3^N0!C$(b4O5P%^7P6Kqvf`2s7w_D1J)>WM;QSYWC=GD1ZJvkvHny zxfYw|9Y-^Ldnc9N4l02+huU|5#($;$9iZV0zt(Knd5y}SbMf_!T6wO-rkAJK!&-Jz?BLSh!6k6&P%{tE zxYW%9G@Q#l{5(j*xC9!;WtF7|C16}unR*0s^{|!zbi%o0#vTyG_pn?8_J6Q7n|uBp zFyE-f=eyYS_$>CZyPQ3yfx$GnOT?kh9-#54vj=Dxm`FpxL>dYv(oitXDrXN$K*6j^ z&@q{_hqVN#6M9#KMRh?G0~2UCm}bq^p4Y(qySGJZsc#=9O=Fu$!?DGV)zTC@pyVzQ zhdO(J#-q+2py5#N;p{;g27e{cFes~>JtzT#vdYlMWJNQ9u!4)u}DLA zv1ZNAp4XVn2PKZ;hfXSMscoOltGVguDb|0#>Cx{)>Kk8kpU8A5Pvm%ne`LCvS}A9MNsUG&a5co#Vz zeo@R1N=qq9f0h>2EqW;*pkcKSXW>wuhv_h$X3AI&ib4&oIR8$UKRvK`oF}uzBU@GO zk~~i)&(B6O?8~%#W)rE(AW8D)C*_o^f5M^tCs0cR1e1#-8nfXMdI1IR983FNlVB5D z1I+l#v%V8K0e_aFya}@|eDmydoM%}s>tKDmq@j>)rI!#xDZ2{vu&DInUS=x1UEUGH zl};mwsVvTAn6o%pn|q7$y~*|!`ChJSCA+lJQz~ywEH}Q}Piw2R<(ewZNVj#H!V^z> zMX%v$YuoUYln#Rufl^~<6STPh1q=nq3moePVOJmLB?F+|-jjG0a(@<@c?sf(d&8n4x999EdvW>3|<+3yR= zwud95Va|7l`zte5_oAbe*h|U&d`TjJf)}*pu)M>DuRbO{z5M^!cEYOZh-Ki94ev#S zudBRg822lJ*OLA5jJBA>UG@-P9T*M_^2SW6n&TtO^37~-$baLU=wjoggdUTFO$u(4)MC%;;2Krc1(fiB(U<)DwTiwY4I7|K+fvlpPo+az`1EJ zte8Bf{2GpwPr7fj$?OQ7z_H9SpvBb`KM%2Nnr@OzJ;`j%SlcE0GG{CywU_Uf?>`V4z(0Ry(_0+WA-> z*zb_r4{^#Zfukh#!CSp&*PF??82txOO9KQH000080JG>9R|o_q4yntNLm(@E+o7Qd z+kv*^)IgyrExie8lZaYBNZUGj`zns@B#=`<7ing{on2Hd_PR}e!3SkcO^Pfh2^e9e zGTWNGub1fqNdj^sd20>SbUskTcBm0A~1$j zgGtlU3}fTq+-7ozA4dSU4K?HdjY}OlK*O2b!;^zFj7gwjOjg-)Py)tel`%&!XMYZB z2|y>DNoLLgQGDadHDKdfr$Ogmf%2`|bp-M2nD*||$t}Z<@j@lA<50&A(74pH12mk< zJxn`D!>9xrMrD<62PI%sRylVBHP)Q5mH>3ZZ;3F24v6BnL_lVCOQdFx4u|sR-xGPG z-koc)Y2I-()3FuBrcyp+I2Y+Z>>fZqxPURjB9;9JZ0u7_G%EW^bFeznOg@%vHdI1u>EVzhMm`_{5cn2->8-6N^E+0iao4lH^mMv{T*Barw%pq z0F6uCJV3*_+{4dl82VO&;OdQbw!WtFK%Fjo(22|y>DOJ?i=QG5@}HGg0aTeG?6 z-vRTDT715XO^?rFAG^!hV;UGtleg)j;k2-sRhJlGR6ilR{U?L3#)2wp#pac}m zsstUAIeS=3fI6XfMOai9L@_XdhJ$I=Z0&gs%)fhEq?Y>jVbV0VnKT?*>{u;Lu>(r( z5^<=r2WUL%>;W1M*h0u6(*%GrYwFes~>JtlMZu$BOH;#(9-=IlXHbQg;> zbQf#Z?Cg1s$$U`aD1PXqvX|mlQhnsmc5t1R4^Zh?Mc#qzZlef8viku@83o5oP&3f z^Whi8{GhayqV#8JQQe}K@&OuF`*0Qx<$0J6<7uXh<)A3k(2Dc#botW*i^q8~TRgH= zSc~SqtAhJ00g)mP;M1ZUfjz}g}2K)Vz|<21TmH3 zT!uM|leM{5lKs+FwLN>8c0HL=|IZa=N9(w1wg6eHd0Hiaji_KIG^)7G}( zDJdNWB?3j0VHFht+LLz`a(@z*uw~V7w%QasXvZO>@c?SE~X#%r}3hZQEa*`vBQ`+b3IdpI&0=6tugzcN#GEjo(CUP|`oOA-MT zyr3nAY#mh6vbw8bRuvWNKUz;IxY zH)c|uIX+UBZ)ST#9)CZ+44$KV|2lXV#`n$X2UbD{n}};7+;Me%nQWGI46nN-jyh)7 zjxO-r2&{ax%E>?dwD^o}AZPXOPfsUy;M}wqR!p8#eho*;C*8N%WOjs3;8|vlL*qDa$BKY2$o{(`3@BGS#;Xy&hf{GtbtyRI4p=a*F7b5{57#mu*73v}3F4`b`o!C5kz*f-BnF_z zxM$^a_WaVUP+gE>o{}QX7fmr4qXnU4{(TsE(|-Y~(~!G~XTUM{?I)=J0keW3$^r#3 zMDf-_lTjpF0mGB6BqAI8@qUtD1ONa~5&!@c00000000000001hflZU@BtZe*lPo1V l0Z@}?B_;$5)N`kkktG%ZF_WkzJ^_W3=OrTsHzNQ5006Fqe7OJs diff --git a/summary_results.json b/summary_results.json index f26660d28..8723a60b6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2374, + "Performance_Result": 10.1823, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.75139, + "Performance_Result": 1.73362, "Performance_Units": "Samples/s" }, { From 0b89b42e2a7500621c8d8b64eb30923ff27abd24 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Feb 2025 23:25:32 +0000 Subject: [PATCH 102/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9744 -> 9745 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 54f3214de..1a1d42ab6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 23:23:05,442 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 23:23:05,442 submission_checker1.py:3270 INFO] --- -[2025-02-16 23:23:05,442 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 23:25:27,339 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 23:25:27,339 submission_checker1.py:3270 INFO] --- +[2025-02-16 23:25:27,339 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 23:23:05,442 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 23:25:27,339 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 23:23:05,442 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 23:25:27,339 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 23:23:05,442 submission_checker1.py:3286 INFO] --- -[2025-02-16 23:23:05,442 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 23:23:05,442 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 23:23:05,442 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 23:23:05,442 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 23:23:05,442 submission_checker1.py:3307 INFO] --- -[2025-02-16 23:23:05,442 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 23:23:06,684 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 23:25:27,339 submission_checker1.py:3286 INFO] --- +[2025-02-16 23:25:27,339 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 23:25:27,339 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 23:25:27,339 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 23:25:27,339 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 23:25:27,339 submission_checker1.py:3307 INFO] --- +[2025-02-16 23:25:27,339 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 23:25:28,267 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1823 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62305 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432102 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 1f276db26..9559cd924 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -729 +730 diff --git a/summary.csv b/summary.csv index a7eac4d75..38600e7f8 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1823,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62305,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432102,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e867dfe10..70748337c 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431573, + "Result":0.432102, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5b9e95baa2beb2a9a1035e25c7c1a39df85ec1df..35774a7d0b956c531a5c0bb02f9429cb1adce8b1 100644 GIT binary patch delta 2347 zcmV+`3DowGOp#2mSqKGTtZ4N?lU@j71wnpxLqC(n2+4nk%`QWtWSlMjDwDW4mZqvq z!pD|ssmep#&qrlqWfb@OvOLhL63w$bnf$hVNvskqLxr=76^2+x`^vY%N1mt-(OW+zpA)-RdHLT*s8RND{ZoPlT^wTO`nwdy5#zG$qkp# z_q8&~R<$;)_Q+-lyzMTIxq=MygewGL;Ukcc^XGpPkdOoa8Au5DUx0)__Z3J8bf-W< zPO}b>kT>THNXWqrfP@^~b1pH-(d_~WIl31>LXPewkdUMM4M>{!baU|O=CA}dDdw0< z9CGYWxWpp$egqOy?xWuJVl++h5(F@o<20D z%VU4&oO8h?K9O9$jC;`j`rpe>SHWrb?EPXjloiXNVVsZVcUqK8^vSsRJ1Ler#iN-_ z2fcVYkZG}Clf7JKc`uvaP2#Lv`((X)TQl@Lm2$Dl(#yq^ejc2x;!~LwYnV)5EoK_) z%)MwfYechoBbiPg)A=+nR#T0Ysuv~GK|Fs{e^L7~594vBemK+zV7~rVM@hx1RTe{A z{I_FV-;p9X6Y%`qH7l6T&@N<_E$z$mH?Se~uQq-keBay>VFO16TnMQH>vP^%~ z7#2Lvp7JD(M!kDeb2Ot7&4thIOO~} z;SvTSkWdhTgn|ep6hyBAL>|sStO)Oba0;SV0U{VY1<|7*@|Y%xx!@9?uzKnJ?|@iG zDSLo;>upVmmhIVM$7PR?N%T31Y^w*TOtjeE;ok%7NOjjD@!U(LlZxMWgRu{Gij~{ zCRVqCHQTjzVTm`Ljs)Mwu^oyf*SIFdl504YK*O*E8ipm;a4gL#EI|T>WkrI{`o$M4 zX-fb);aJkBE{I|`Q-MYp2IbQHKj5;C8uz)ux*FK}a?-S%U0mYb(IbDsHyv%4a>+HW zMY-e}&Lz+=E`f$|$u*oytBOmIfN@!optFAQg-hBJpi1a{92V6DQJhP#rOmmt-ZYow zw$5KivX1ihLekns(i6^(Jr9Am@s9-G8n@}zHqf|DiX_)?B!Pw@2{a5zuJLS&WU~rM z&U;tP_rB>;b&_$Eb?Ms-mXTfuUTFa$-iSuc|FyJ8}oub3n}=J%7uTv<+% z!q?0nAhM3?_FB9)Hy5v(Ygs#q++j5l+&y6Xlt`{|ZAv89a3X<*5eYPmNT6Xv+Eqk? z1dPav1fBJZFGSLo0Cd9czp|(*3fzB-Nqd7y)6xuMX?<#$=T-M=)m&YY9LnoJnTR0a1M6 z$~9o&TBkwhUxD(i+I0l+>zMZL%gHUnj`2bzu;Wn24$!#NvI8`n$~{awNW-WE8b)Q6 zZwDn{R8~261T`DhoUxVwbi%hpm_Y|b@huUMnQe*G?9t&+{``}i4?Q3AmAQ3L6kERn z4O_p~Y}k2?%Aa%d^^ID2uEeI7r&yEt4>%Uruv`Pyur-@|{v9yisKw{I*!1`;){{mM zGJm=&!lJq$ih&6<989xjYtL(7{@rbnTI$<}Nz>S7(r|3CW41KK4k)=x#G%d}pz)}) z2WU8ydpLWLhCvB549Y5J4@$tGtaA34%-O?Q0?>(XQ7Dl*yvO)qE_K{Angzs(hRk?@(E+ zs`MWR%O46*W@((HasIUIz5Jzu`8aG(lK%U}pyt!~kGXvRE_&x2yo;OK46}574mMhqG`f&%<;WPcvmK2SuTVR$PCl%by-tJkFEZ;*qT?cS)Wnljmn6 z8TMsbKC@J1kRrNPEg{g?1rWim~TAP|Ro1p`*AlPO%V6x5bw6Z?*+Y@_d06 z1b-%2VE6z`?MOr?q~Igc{132X;8wTadsbuIaOj^(4=-zud z8oRE`y4hfj`3wBHet8>KO{G4ZiUX=k!6|pZf^l_;uZwuNmie%bC*DX9kA2i9-cE}g z`#>Zy06oS%D<89`=Vpa!l9PFwleAbgWEn=uoRa&m!^oTd3s9Yg{F`_N9CP1(g8C1$ zf+5NR1!1gc^+A(SBwGQ(ldU8?0ZfzXBtZe(lPo1V0Zx-=B_;%9o7^RnktG%ZFO#Sx RJ^_Q1=OrTsH6s83004h>Z~OoN delta 2363 zcmV-B3B>l1Opr{lSqKFtWqaa4lU@j71!?)&4Lp;@2+4obW|tvRGR_u%l}TL8rKu{D z@Uf*@s`3!`^HG^t8O8m+EGJr3qIs4lli!vviB*DSsBl)X!Vv3dUlUEkG41mGcpA%9 zjK1u3F~W~w9<~qD}|<4kT~0t8z6DFCAUE0 zttDIKTDV+`B@m=st{~(5{;J~oRmIJ#irXs1R;5i`X_Lj9q*AVE`lQs?CD*S@Zn%WL zua!x*sb2~ZFh0Z6=awvTp;f_(=Q;dLYNhX+?PF`L*XI>Wnr7V0`rOp)p+^ zL)U+t3oh}AL-BvDGOa&15?0 z#nXXIiw&FX)qR$q35ZTi(QspZl?6};A9n_%BXk? z;BoepCuub5-J6=D8I5Qz%;p{>)=|nHB;L3iBsN{m@{AoIdKg5HgGi2_V=i&X@pHl@ z3`8KIAOZ;m5lASAUImCeoPk&o-T~nhM6UuwFn9{0M?vH^Z;(0wS4>GBw#e7al!3E}b5uNSsS7$sR_94<0Edd|<@lh#q*9spkX8vy#6 zV_HVjU;o1-3In6KXeTU^1K}8GTr#8+pkXv34aE^@D33tn(WP;%K}4bi6v=<81fBJZ zYbaSu06Ou>tevx{E{LL694}C20{rF1@ZScSFDqh+Wnao!aX6-#?A;5^D;CAS3}-DB z?m_4EYG6w|UCr?fZ^zYMfiJlq34V#)wkecc;|dBTxKz)e1R4e<&@d>uhC?Z;pacmR zlobg&g5Fd(V=V#b#1|Aw8r6SAQFOIL8oFF=4F4@cxv^H3RGNk+egMllYTRehTn$XD zE(L40Ywf}k-*h?>{63EDP%OE|H7S-{!?6S!h9%H2EV+hbX;xtg5-=<)5_HxtzFA-gk$bN65qx@68zSb=jjoYM1at%ikXc&?}!;s_}&!$K=tB?ep zFeEDybk;AvAW2&S(20L!@z8&1-XW^QyU)wUfvlRujS91GZ0zW6c^FYY9Ln{FVqa=zu7GO9W(Qw?u08=x`{1{*#>#Js9WdXh#pk=&^!P0HlSdFT ze|lGhMRh?G0~2UCm}bq^p4Y(qySGJZsc#=9O=Fu$!?DGV)zTC@pyVzQhdO(J#-q+2 zpy5#N;p{;g1|`riD65=3C;@}A%GqNwXAf%$KqtOMp=8b;6h(KjNJDqAX3fr?*O<%) zC63~UPAY4uZJ*7nx#{RB)_=e0(eFd*e;Z$OpU8A5Pvm%2EqW;*pkcKSXW>wuhv_h$X3AI&ib4&oIR8$UKRvK`oF}uzBU@GOk~~i)&(B6O z?8~%#W~s^`N%H3><&?9$5rGZ_CJw2~lR+RWf7_v<2-|_SK^JLeznxuFE%v%ie!&N2O-+g{CkYr~r83)^ysww(14#mMBYA5L)Z_qx zESkGYI3}!zS8pA7R1lH~GJy$KlO3XClm@W_%|jL)CW;&DHOF}MTk5#@=3Ag;p3kWU zf8>%Qr$=DwCL$S-QcOtq7xZ)W^45=^#7EhRbuO_OLPCL1OWsmmn+002+|000pH h0000000000004l2E|aMxJ^_N0=p`ctG$Q~2006QfPjLVM diff --git a/summary_results.json b/summary_results.json index 8723a60b6..4e99ab7cf 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.431573, + "Performance_Result": 0.432102, "Performance_Units": "Samples/s" }, { From 8cca53a36a95384a91549de6111ea9cce8f46c50 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Feb 2025 23:55:36 +0000 Subject: [PATCH 103/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 8 ++++---- summary.xlsx | Bin 9745 -> 9750 bytes summary_results.json | 8 ++++---- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1a1d42ab6..8198fb167 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 23:25:27,339 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 23:25:27,339 submission_checker1.py:3270 INFO] --- -[2025-02-16 23:25:27,339 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-16 23:55:31,612 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-16 23:55:31,612 submission_checker1.py:3270 INFO] --- +[2025-02-16 23:55:31,612 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 23:25:27,339 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-16 23:55:31,612 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 23:25:27,339 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-16 23:55:31,612 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 23:25:27,339 submission_checker1.py:3286 INFO] --- -[2025-02-16 23:25:27,339 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 23:25:27,339 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 23:25:27,339 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 23:25:27,339 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 23:25:27,339 submission_checker1.py:3307 INFO] --- -[2025-02-16 23:25:27,339 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 23:25:28,267 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-16 23:55:31,612 submission_checker1.py:3286 INFO] --- +[2025-02-16 23:55:31,612 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-16 23:55:31,612 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-16 23:55:31,612 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-16 23:55:31,613 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-16 23:55:31,613 submission_checker1.py:3307 INFO] --- +[2025-02-16 23:55:31,613 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-16 23:55:32,628 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,11 +29,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425368 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.42599 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9484 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1823 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0629 | acc: 77.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62305 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432102 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9559cd924..dc3a10aa7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -730 +731 diff --git a/summary.csv b/summary.csv index 38600e7f8..fe4b1ef14 100644 --- a/summary.csv +++ b/summary.csv @@ -7,11 +7,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425368,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.42599,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9484,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1823,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0629,acc: 77.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62305,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432102,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 70748337c..527a9c190 100644 --- a/summary.json +++ b/summary.json @@ -244,12 +244,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9484, + "Result":20.9948, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1823, - "Accuracy":"acc: 75.000", + "Result":10.0629, + "Accuracy":"acc: 77.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 35774a7d0b956c531a5c0bb02f9429cb1adce8b1..98735190dc9a5fad8a1aec34f8acbaf875bfc6de 100644 GIT binary patch delta 3908 zcmV-K54-S@OqNWrSqOjRJ;+T$2><{HLI4050001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`OuoLh6^#ukO&&#zFa=3%BP2(^T6EB-H@z_qCCrTC#r~-&c!k2#^2{4Ae z7qT*$M6Pr4FoD+EyL$_L?v{GN;jfR=iE=NKG>T_!RcN}Z$XS0F52D$qt=?RJZn&zF zX2EO_OyZert4}gjf9?F|@F7lqPsdVbN}gbrw$*W#%@3N*G#txmkZSQ<&hjI}IGF}n z{x%sk(|IC;!7^w%X&Sn2H>W`~Q&$-dk}5gk;V=s2$2eR}SGL8lVS)6E9 z2^VP=Pk&n-66*x3P+_lQg)TPHo+etlZCJ(U@idX^nC5?~*X0O523gQKOyUP6S@w|5 zgW{7B2ZoWKmau63Rx}r5Ypxrvo*R_Bj^gaycg(})eUW&r_)bHvX~PX|xZP@K=HuB6 z%D{s%*ckNPFTRmFd8nPR8X7Ou3#;tf(MSF}7*>g$s0*CjVxLO<8iBsEdo0 zrNw{RC+*)`n%>M3DVM9PX0e!Rp8F^3_(Z1p8YXS77c-4b=6*PzH^TX%5zl6i$zqn} ztEt9X)eqy@FdC`9r~{b=(Iizr95#nwzW!E6N%^W(=0jWlZ_l{Cuf`=-YYeSr8bUWP zbPk5j;n0yw?U+k!a;2Vd34;zuD0DzVp#y&s@+bU@Qvw|icPVrw;aw0;{vw}khm-nz zFnIDe|J5Xt9 zj8+8_>nNoHi8sy`i7iJnUERbWx)?;4gGjEQV=l4D^>e}{3`8KIAOZ;m5lASAZV7*g zJe+}865a*j6hyZKL@;;?qDw*KF)b2v!6hD%T)O`|Al6Yz1rTrDohi|@Tubb^?eQ>) z9w(7p{l{El(;FWyVI%?xB@sv{iCp5+8y~NPL=a9c+S8Kou17dIqYXaqfUQGmOb^7+ zbtsUKkKod)7q)2-#%W^`EHga1_RoJ_lh#pQ1pu$SEdV{uHcY+cZT`a~3LT@kY$q&| zY33Mc95SR6pkXv34aE^@D33tn(y4KyK}4bi6v?s#-Oa%bl&r-Eop>bAIg9FnDEh?l z0%(LzK9I|;@xKi=U#7%@%Yl^h{CZ4M*}oT>o6m}Wna)}&tbphBdSEL)9nF7sb$8F> zUV)#v9|``9-Lfc^T;m8zC3sfPs012DCD1S`xrS3I%BTbh7?mXnx`OU1oUs-kbmAG5 zN*dKeQFOXQ8aiKY4gW1dxiwi9R2sS_egMllYOFGAt_LR8&w@3}G52AKcb$#|-^j6S ziY3=L2E~$VIF>-euml>0CD(s&ER8ZOK>~(lNrLX?;0u1kQT92Vc0PEk3G*KFDEFJrKpY1bf0NuKNcbtLO3uM&Tf<}Q-1u=i|w z2)vJfB>3LAMfbLW#%WO`xrQSNGz>|gVMua~Yf&UyWk`Zf=tI}C1l`TS7bIzm4?6Ko ziX@Hdp(utV*KBWgt&wcii{#=ip9mLgCJB$({d7537L%m#G~)+|tfRV0i`Uln;#G4@ zb1#v5ye5L%2P}^g$u)nDMTz7ZP9)GUB7uex2{epItBgpHfDu`epu0KvLL_bRK_~11 zEQ{))C`KgEuzO1D4FUO2JcX^Lz)DP7TTB|Jrt4c5=Pr|b3^@YWZKxv$XdG(E0UFL^ z1yc^vFeZV9FWzHPItU0X32c2*xnL7tW@r5hbfQ5f+oerIU1`U{o7k3%gxK;ux)4$yEaE4X%$hEWMLjLI_O4obkNEVJ$i>a00qEk5Xk z?};#n4v6A=B3uLZL~3^Fa4LWPrpOz0?_7&*`;M&{p0yWCcMq1pokI;gK;uvc572Nd zD|mR2hG7Xb49kBq8xKmruq-q32{ejOCWCQG1U3)q@1JFijsbDqAw zQ8Uk#*mm>ei&)cYi9K9;d$>j7y+lT$Y)7Py)tfnXN}K zUk_{XK_{F`X6*q{d=bkvU=drhz31Nn^NpH(zKd;_&vJhuyU*KW=on0c`$TN&?ExB> zdV7F|fr&H}Or)V;A`J!8DD(E91Qg7&1YLu9dsvH)I-%PlEUE{h7??oA!8B_2_Phq> z-`y9frM@am8u~7ix^0O)v!x;SK*?<)HnsKujZ3XPK*OP|VC_K~1|=AeO`$Ba_MlD} zlx5Z)gIRxjSc?xj@mds0X6->y49W`D9;0S!&udI(!vaU~LpPPR)K+EjYHVA2^7-HM z*cYptTVHga%48%@%Hi|5N%ma5#vSr$*9-;HH3kV)~*RFz>IXU}hnQ?mIHht8i+ zO9QjT5tRW6-hc$JDanx-_PME9(f9t3S; zmTW{xQ{CTqtG4YTj?*NEHhjf?K+a8teKzShqzo0!Cg>9wYXeEG&`}j7;&XfDn5C@z z3inF*)hbJbfU<2kx}!--SAR*G2zApv7m&5lb1AQK=6T6N;G8M`i@z1dFoYCJD%8bP zS$xT2@spN{cIQl}hq&etbzZ)ttV$Xx0Zu^3=oLR>>7$J~hNWAUg?n()gAdwxLn3aHfYop#Woqr!A&vOUJwVzquh=jIxv&U+y)-DMb z7uP3Ke1KeRcoxA3J(HjnN`DfTuw~V7w%QasXvZO>@jyyz$TC#q0G@*)NC_gzX@_A2 zrRXTHbs_%gfaUWilS_}1IIikvQs1br5zUGeI(mQ8HtZhU>A{t3Y;dm!_j_<9mMxAI z0UNxh9pavLh?REO;63r;o_KLjytpS`+!HTW;)Q4p?f%ttBRxyQEq}5Mi{hbeu{__D zvM(eVC_Sq$m%13by75}=#$kntZT4i{oBh5ZZF@K}8Zy4y++V3wU5k#AVlO58^CgJ@ z3SQ8X!}1OrzWSK-)bjsh+X<_tBbI?bHoO-RzAp2gVcb6vyq4^bXSBs6?y`sY>cDVd zkT)u+&Kw_^mTzWzgMW@6Uk1<7y?-6N3*-B0`hk_u!6xFG2zOjvpOVe8j^TB;#8JoW z+R+7`8-bs*chSt`*FpMe@fqC!%j@5to=)t*xoIz~av6*KVvdy0zHhV1>n)CTY;j)PI9OsJr)Mq7RH{vs7b8$lE+4cnntDAW7J$-_kY}}UR`!HKR|Y1IDs-D zJ&n@DAdbH?rMrgfy57zxO$^7k@6tm$MTnGeG{8~?wK_DN0%WA_g5JmU>3ZuO(KX53*cAza;HBhKaOLsz= zB%;;>Nv-3*uk0jF0$F7_x_8dp8B{IzvWxz}I-ymKvLr%L&XeX+QOUF2CNt0q5x8kFjJ$Q_l98X*bXFhNpPqjZnTxuhth6;v7zjT zZUN_My1)`V<Ts{ag>$02tUPk=-2 z>rc@91G9r6$^r@GJ;+T$2><{HlU5{G0mGB7BqAH%n&3KO1ONbZ5&!@c0000000000 z0001hflZU^BtZe;lPx7W0aKG`B_X0RR9{0ssIJ00000000000001hfi#n; SB|ZU&ljtQQ209}E0002K0YeJ_ delta 3921 zcmV-X53cZ*Op#2mSqKhctZ4N?2><{FLI4050F!Pc8h_=S?Q^2Y7RJBdzk;IX!%Wo* zG{{>jx|L+!Vu;CPl5E{i7Qt4jRdE6H^4~A0F?!A;ac5_j)V=c|qwRA}_d$MkLqB@> z>*Hjs+{-kJlW9v8nxQIk8YTUBI&7&o*Pk1ns$_XM?T6!JDqHH4%+z1o|2cd}(%-X@ zl(|wQn15z1b(H6`16|Lek(`8?mdxa|cx8~JlQ1uyrb9iOr84X?dgz;2e zW;jUq$&m~OaU?$`(R?DO`7%i=$6;O!KO4ofOl=?bO1kJFn}y{&BMwZnI4n`w`mJm($EL3uuAUo| zqJNI^=-s!iL;b!?yi$CpA=k9whBn-8H0Z^6)`K$fpiDLfV|xr&T+wWjVFHQ0Ex85~ zXIpXuB<{B47D&9cWUE{Ymus;Ef|Sb@WSrk$Rb0QSxOr7^Tcy~lw23QivUrnJ$`wtY zl=`~l`gO?-m(cgMGRao8Hmvr@W(mCQE`N@>f(-M7D+FQTBao2u=M#{S1OFLF2>4%s zgh2NdNC&H=L|^5!3}_f9Nlv+G0D;G0tq>~7eGRe?j?|rqx%g=n)q~c z@ag8T1U4z=m`faT>`%DFBK3X*5>oFcAR+aB<`R!&`SL6QABkT<59Ay?7enWA=*X#d%q0#vQ%|^rK?fuhIv}CY z0SWmC|Ke6a$HP7H`Egni-T~p{L;GwqoYWV9!ISWFmlDZi==$w~OMFt)rGHx^k;73h zjLPeoMsfMFOw||`JkFl-B#lPBdsA~XqY=%8+1!J~I!f7t#2a^m#HOoRo?&4SJq)79 zK_utTF_$>x{5jzg1|pD95P^h(2qY9luL49K&OodP?|^U$qE`VT7(4~hqagB_CW*P= z5}&Yo>HY73SVt*)fOzX|O@E1&?b%|-Wsi?Z^f`&->_6rbhhF$_2_q3mD2YHqN#qib zUikPGB!Y0d4y_39_=J-)+7N&a=sJ|fba{+VK3y(=gz$Lj*9+SujFPM|4i_07J?Cey zN$V(Y4*;+H4FG-3F)gF%um52Zg@I9Av=bJ|iEs=wE*a7Z&@dX2hJWISG?Yi6@#xgJ z)*vEL0*Yi+g3kKIHI%F+0G;^csGYN@E{LLc94}C20{rF1@ZScSFEe6^Wnao!aXO}% z?A;5^D<;Li3}-DB?m_4EYG6w|UCr?fZ^zwUfp57V34V*+wkecc;|dBTxK+=f1R4e< z&@d>uhC?Z;pacmRlz$ZoI)ZL0oUxVwbm9vNC5`H$C^}mr4V^AGhW{3!+?Xp%DosNZ zKY(Q&HSRNMt_CJnw}Lg>wRT~NH=T|I-^Z~XiY3>$CdHC#IF>-euml>0CD(8)%_=NG z0)}Npg3kKI7c6N@06O7V(x@(oVmDKPMi>U=()>T*vW^<}xqrdB8rb@B(zKjiT;ko) zBf&QvZI^P%HLgXuDD&TxJ`;A*Kj0(h9LY>H&F3V%t^2}80XL1+Eq3zD=Y0G;?IMUqB!Q50Lja*Z$qMY35hlJmP_BAl<7 zBs}K#lf_(FPLjgc%pV}Kj_USWyf!x%ubOLFJBi$3H4)rBVEdFvu5oQjB-d~vfrb$Y zG>k}~VMN+hM1lm2$chA=^@}e=(v|>p!tTGas4j@&M1KMeyQZ|?5Rm`GP1srr+>1$j zgGtlU3}fTu+-7ozA4dSU4K?HdjY}OlK*O2b!;^zFj7gwjOjg-)Py)tel`%&!XAWx# zKqs6@X3haoeBsJ9VBuP)LFZqA@~zr+1o7*b_U_BcEyIrSLM5={P{$6?xYV)(G@Qyk zOgl)!sDA_+MrD<62PI%sRylVBHP)Q5mH>3Zw?vph2So8L5s;Z}iPY@T;ZXkkJ&`x+ z-MJQ<<{d{feS0UB-VQ2(H;3AHfX1c%9iZVQFNe z(74pi12mk=J^VaK!?*+*#$}bI2PI%!R+)MPbM>&60Cd8+WX2v4#n-T01JdYv(oitXDrXN$K*6j^ z(0?(Rvxl_=s1v#?!lJq$ih&6<989xjYtL(7{@rbnTI$<}Nz>S7(r|3CW41KK4k)=x z#G%d}pz)})2WU8ydpLWLhCvB549Y5J4@$tGtaA34%-O?Q0?>(XQ7Dl*yvO)qE_K{Angz zs(hRk?@(E+s`MWR%O46*W@((HasIUIz5Jzu`8aG(lK%U}pyt!~kGXvRE_&x2yjP2y z55Fkp2c@MHr9VrH>K46}574mMhqG`f&%<;WPcvmK2SuTVR$PCl%by-tJkFEZ;*qT? zcS)Wnljmn68TMsbKC@J1kRU<5XFIM^6(Zi%8k4L zCM|JZW>m<3%eBdi5@^J40YsLX#Ijs7qY2Ycm{&|3YQ%9gRDZGL-wMWC1Y1NK4S6ye ziqTmfK{_5Z%r&WzS~c0tF+~sD$Pi@b(_KyPkTkL;c08z@RXDegA##KV`mez zxc>zV1;`5=>jq(0ALk{Y-rlIUOy}Rn$n)F*a_wi9H%UU@c? zsf(d&8n4x999EdvW>3|<+3yR=wud95Va|7l`zte5_oAbe*h|U&d`TjJf)}*pu)M>D zuRbO{z5M^!cEYOZh-Ki94ev#SudBRg822lJ*OLA5jJBA>UG@-P9T*M_^2SW6n&TtO z^37~-$baLU=wjoggdUTFO$u(4)MC%;;2Krc1(fiB(U<) zDwTiwY4I7|K+fvlpPo+az`1EJte8Bf{2GpwPr7fj$?OQ7z_H9SpvBb`KM%2Nnr@OzJ;`j%SlcE0GG{ zCywU_Uf?>`V4z(0Ry(_0+WA->*zb_r4{^#Zfukh#!CSp&*PF??82txOO9KQH00008 z0JG~CR|o`To7^RnMj#)5&1%Ci5QOi23ZuI%TOnx?wgYX+sewXWT6z=GCK0uMkkmSU z`zlW2B;ZqztC{_Fc2T+9t2X%s=cUmVDzY3Upt%uJZ!7fPtkMUR_`tMatSa3mtPa;Tr%58;qAy|sB=Q~i$XVIafxG_$#5KFgz#g_4JwgpV`e1R1N zCRkwj08H&jL?@)+Bhvo3b^r*Dp#=pxcwA%!zIX#T(^uoBj(>ore6Icm^DE-+qGn53`6N z$^r>ttZ4N?2><{FlUpQL0m75BBqAH{983FN1ONa~5&!@c00000000000001hflQO| zBtZ?_7XScwY%g?~cPyzq|5dZ)H f0000000000fPpWQt|dMJgOlwgBL+1i00000$XRPQ diff --git a/summary_results.json b/summary_results.json index 4e99ab7cf..8135ddf76 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 77.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1823, + "Performance_Result": 10.0629, "Performance_Units": "Samples/s" }, { @@ -653,7 +653,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9484, + "Performance_Result": 20.9948, "Performance_Units": "Samples/s" } ] \ No newline at end of file From fa0aae56fc99b145357cb3ee82828e261458b422 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 17 Feb 2025 05:34:27 +0000 Subject: [PATCH 104/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9750 -> 9748 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 8198fb167..b2dddf6c6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-16 23:55:31,612 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-16 23:55:31,612 submission_checker1.py:3270 INFO] --- -[2025-02-16 23:55:31,612 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-17 05:34:22,215 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-17 05:34:22,215 submission_checker1.py:3270 INFO] --- +[2025-02-17 05:34:22,215 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-16 23:55:31,612 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-17 05:34:22,215 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-16 23:55:31,612 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-17 05:34:22,215 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-16 23:55:31,612 submission_checker1.py:3286 INFO] --- -[2025-02-16 23:55:31,612 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-16 23:55:31,612 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-16 23:55:31,612 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-16 23:55:31,613 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-16 23:55:31,613 submission_checker1.py:3307 INFO] --- -[2025-02-16 23:55:31,613 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-16 23:55:32,628 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-17 05:34:22,215 submission_checker1.py:3286 INFO] --- +[2025-02-17 05:34:22,216 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-17 05:34:22,216 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-17 05:34:22,216 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-17 05:34:22,216 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-17 05:34:22,216 submission_checker1.py:3307 INFO] --- +[2025-02-17 05:34:22,216 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-17 05:34:23,379 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0629 | acc: 77.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1611 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62305 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432102 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index dc3a10aa7..8766d8a98 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -731 +732 diff --git a/summary.csv b/summary.csv index fe4b1ef14..918e9c347 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0629,acc: 77.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1611,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62305,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432102,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 527a9c190..7f8797d97 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0629, - "Accuracy":"acc: 77.000", + "Result":10.1611, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 98735190dc9a5fad8a1aec34f8acbaf875bfc6de..218ac73e820829f1fefbf6c3c214ed1c424f1184 100644 GIT binary patch delta 1541 zcmV+g2KxDyOq5KpTN4HRU=k`|lVB4he>fC=-(O*6UuGV1v70L@9ig_MYPeJdM$A*? zCMP93Nt}%xrp&KjhX$q-Cj}OekSe}&d_JG6PljJMS&HtU#Jkk*AN#pk7&wrIMnWeYhOlx4LKJ`9?kiob}K}2V46IIke#` z_5*TmD(tgK$0en!Xf{Egz*rkdf5r+ORZ${7w>8HsW#w16SKO~wSt10KZNbqUO;Wl_ z(nP3-?zw=hjh@T!DtDfjECkM(V!!xXVGKh^p`=1pri$_<`|Z4Sr+cWNe@0~;}I29mZHprSr@)}b~E?eWSicG%Hf*nElPzuzPT)2Un`G!Mz^b@4*#cwm4P- z*x)_m5ciBjtc=43@9`J+_=|h|#XbJw9)GdoFGOo-&#z`0=~)_Xk!4sE4`Ykv`KFX} zA<00QS$(zC#n3g4f7j|X4l7J-v#09bZ1)9a+ryF3kn!E&{>n(zz33<<_ENGxUy=x* z-~}x?Ebp-4tB*-fFaJNb9k*&aVj1{j-Fp$?>niUV#{C+mw7s*DgV|m?ean!N9 zc1(fiR^aF4T{JWKb&!5qd`36G^7{9urxQDHZrX}fE`5<-%#rfh_iZ+r9ibCAmYD#w zxVk~M6)2I|DGoJAkBx!0jj`t}Y6`2Z@;InaGpH1KjGEMS@11JuvZMI{vID~jlnv== zl_myp{GBb`V>MjY^>$WiVmQ7fckPqu`SNsnE)iGRtH!6Z`SNmnv8eZJS8nxwV3EMb zr2a$bcs6;I68RP26UXxdFK``eFi_ID-LGXUsQdM?ys+P~l0WzBtJMaq7vQ1*QjRcFp`G`JVC>iqBy~w}s32~&mDGpQHrRhq_DvhWd79o~ z37&H5DIS2anThCtqfL3xL>B*Q1K2P@sxhEX|lO5uA0SFJ~JcJG~- zja}CzT`^ii{tSPvU*5)5W1)sqF+g?68DSQv*S0S4c@YlRQs>v<#9IOUu@Cx{x5FY+ zAMhvyAXD6<@=-iJZ&oCz&eMDmr+J*+Z_))RZb_bh4I?l5Pe6GX@^9i1Fy+4d1oa=Y zh#^e_1^i$VDqxe^Bv%dJ7XScwY%g1ZyFVo#Uv0Lp~UBs rB>?~cPyzq|5dZ)H0000000000fPpfTt|dMJhLi0jBL+Al00000me1MW delta 1522 zcmVxC;Kfl7tdD(r)#cr;sbcNc2s^wA@Sg}u) zn@md1CUH*euw{S!acE$>aZ=y_5>mxC_I#etGvmqd%O*?F9h5v_`JsjlOGA){EK2hA zp>}D2L=o1)ge*;Oa&`}j7;&XfDn5C@z3inF*)hbJbfU<2kx}!-- zS4o-(b<;f;khRfsDX((odC5ZHoGJc`zZJ$XgcM3D)WuX;e92<*la`8h=S--FxaJUb zUcRKPN*XEwPC&@$6+dI?qm4O+rCXMTdvMZ&588M{1(m5NZ^En--#j}Vf3qxOG7r|b zOBxDUR(c6Bl(MQo4~t4K?xj-U?edNou5=nfOl5K|Lzc(M+T5Fz?@hKZ$@g+qD_NzL zo>F;hV!83%ep*|lE!R{jM!L=06rOn6D|!u2Tib>wQ92At1WJybP0-^07vv~FUf@_a z2)p{2m4JF{quw%|e;*^ye{%=OwVzquh=jIxv&U+y)-DMb7uP3Ke1KeRcoxA3JpkkY z4Ll=oy`TY3B@h?`7kda@U3{IMUoPj<#q#p#Y<@becUvmmqw4`SY~s7WT|0*F6Z^Mk z;v0^K?M~0wA%=y}nrHqGc`@4(mat{jaJJeMJ7~utr13yXYsfNGf8+q3gCa-?BFSlo zVFjh=D6e%P{^@|_^Cy!_kCHg9>St2lsIL*tiWE9}f73SX9^C1{m27NquLt*ga3z*4 zjuinLyr&)Fo_2_pcG%!O@#3C%aZkLsCtlnWFIM7(XbtWD)pR30OT#U)42$BSZLvJx zl(H`*87MugFPFL)f4aKyTJ6SRg^6wUWZj$nz94OTI5HYCzT4bisZ?Eyj*?<8CHwOw zi2w>-(2~RQ4jaDunDo^0|6|(;tEMBCfj>697ZJWL^PXYcKM}l^?2l)(#U$>shxqEi zaA1%(DyhyKADNbKW_yEcC&L~X`$G7CHeKI{?o=(pt;wp31_;fa3UXCvo^?dEhsooDP z68M=p5JmU>3ZuO(KX53*cAza;HBhKaOLsz=B%;;> zNv-3*uk0jF0$F7_x_8dp8B{IzvWxz}I-ymKvLr%L&XeX+QOUF2CNt0q5x8kFjJ$Q_l98X*bXFhNpPqjZnTxuhth7bf1&J$ZUN_M zy1)`V<Ts{ag>$02tUPk=-2>rc@9 z1G9!9O#=nrn&3KOlh`Cz0pXJ{B{~69lWQd=8&dpQPbC2W08jz|01*HH0000000000 Y0DyrsldL5^0f>|9B_jqpBLDyZ0CDZ&00000 diff --git a/summary_results.json b/summary_results.json index 8135ddf76..880a87f41 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 77.000", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0629, + "Performance_Result": 10.1611, "Performance_Units": "Samples/s" }, { From d9e0c8ad36b54e9e31f5cbf5ff5d24466d192094 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 17 Feb 2025 05:43:37 +0000 Subject: [PATCH 105/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9748 -> 9747 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b2dddf6c6..6339ddb5d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-17 05:34:22,215 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-17 05:34:22,215 submission_checker1.py:3270 INFO] --- -[2025-02-17 05:34:22,215 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-17 05:43:32,442 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-17 05:43:32,442 submission_checker1.py:3270 INFO] --- +[2025-02-17 05:43:32,442 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-17 05:34:22,215 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-17 05:43:32,442 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-17 05:34:22,215 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-17 05:43:32,442 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-17 05:34:22,215 submission_checker1.py:3286 INFO] --- -[2025-02-17 05:34:22,216 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-17 05:34:22,216 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-17 05:34:22,216 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-17 05:34:22,216 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-17 05:34:22,216 submission_checker1.py:3307 INFO] --- -[2025-02-17 05:34:22,216 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-17 05:34:23,379 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-17 05:43:32,443 submission_checker1.py:3286 INFO] --- +[2025-02-17 05:43:32,443 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-17 05:43:32,443 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-17 05:43:32,443 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-17 05:43:32,443 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-17 05:43:32,443 submission_checker1.py:3307 INFO] --- +[2025-02-17 05:43:32,443 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-17 05:43:33,444 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73362 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68088 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.20591 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425368 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -35,7 +35,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1611 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62305 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432102 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8766d8a98..aec11d8fe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -732 +733 diff --git a/summary.csv b/summary.csv index 918e9c347..c0ebd0397 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73362,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68088,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.20591,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425368,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -13,7 +13,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1611,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62305,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432102,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7f8797d97..f0f72e17d 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.68088, + "Result":4.20591, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.62305, + "Result":8.9953, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 218ac73e820829f1fefbf6c3c214ed1c424f1184..4f34dcf34d8a979528d2f9d9700f11de2b434e39 100644 GIT binary patch delta 2716 zcmV;N3S;$@Op{EoSqKgfJPK(+2><~7K>z?40F!Ja8h_=S?Q^2Y7RJBdzk;IX!%WqR zG{{>jx|L+!VvNaTl5E{i7Qt4j6>$Oc^4~A0F?!A;VP|KT)V*J3w0X|yKFBZK(2pMe z`WQ`>dzqwRJZq~$(^W;zf_NCt#%=ZH`g6lol{E8bLw_32WLtfbsrqZ@KZg%-@_Raw zGE?#dvwyU$PO@x%&}^o`L`Ht9#dA5!kBs6Z^0WMHGH#~xMEb*JP&92Cx^6ckKb)zn z3XA4&`OOe&vvauY z`!DFtdDCI7We=pvg4%@0;NSjRic{z;BM1L8|k-wN`SMkGF8IC7ee%h=eB#5W! z@?Rwi^K)saO5{JbRZ~?S!eKTk5-XE%IF!YSR+V6pW^we}>X29`ScM9E9V>LPiS{(n z(rv>kK98rVT*ow5y)H-i(a-$OVG=(m$+CxZ?iZhoI53R-v;;-#x1zZiTXWrT_1vK3 zb$=9R@4jOmHt&nXYsGgOa!ng5wml)=WJ?~dV$D_SiwOdzp#CD%Y= z?@Dff#MzbH0*Skk?38Qba!r;%kaD?#jPv`eitASuH?Jyg%M?46ws55_7H^PBxuWHf zQeT%`zb?7q68gE8CfTXhg4G^bEP=b*#eXqZkYS#1g&-_^1QK%nd;$`3;y(ik0sjk- z5a_-F34!huNXTW@1rqY%oB;_rxji5uXZM^-403k+Ktj&$1(1-ldkG}u?0y517QWnU ze7V^yfklcr<`SEn`x7oPNxdI|gw*>9NJzb(xx^(|zC267pTw`A2Xc*^mV|eokAFQ+ zQD?NF$2;)I={%<~eI7&SoC}bad<2&-;~sUs{`d0JRqwQa_I|kMSgkR%mT3sxz|c7uI)_6?F12GWvB{Ns!X*qkAfeC! z35528+@;W!gm*zW`HOtE9Zu@&fx(l%`R5KLlE={X+Xa_+M1OMW)JWub zGVp`qex{M%zN}I;#yOAkrz}o_$>845Y)x+j3t_Y>kXT146-c~swn%I_n(68$2GPYJ zx*SAu{Ty?NO|G94E@2=72?Y^ID2PBpL3B$%5}V%ma0w$3NGORwLP_Kj zm)`hzB_x7ya?zfagm*o{$r)|v@ebHJl*aTy3|)r;3Hb;vy?SAb{2)#nQ-7J^(Y1f} znzWAcDgb!pZ2{VqiynDGK=gibz?%dPRh4Ip3U!2-sil=J+0Oj0?x z7n++-g?|~zS}Lr-*c9t=+@Q;ze^>4(0(NNgLTWIRC0|YD3#y=J);t67?nW7 zsN@<>r6{8kBw$pQB!B1%x|MLoT6&-p&!AM&s6L9Kb0pHxNpfrWZxPC^8M2_#&^7S` zSk_Tvm8o()FtL6BtXYn^XX{1at)?TvcW*44V#zg*L9yf-jwR4AEP;k$$u%5HqYO)s zfMHpZpu0Kvf+cO~flfG0Ngn3>btLO3uM(2xE|RXW_v~~Cym@~l_=dPeH?)&s4jN38*0KcM&A}HW zX-f}u;+Ygl8r4Tp3`wrp-r-s!*{T=G#a%uTF4jyE9<%#sIad~wr0_K32Z*erx=M@J z*7o96b4+tDk$Vg#f_ny&#||Zb>48qzQ&$$%M^TJOpkeov)*AxypLq0IOM#V`w6>Tu zOikCfF3w#h_qcEb@Yqlr4$wH%hXXX6$qG&!q+v_~4P&y*jDr#|Cd=$Ng86Y+OAmCy znPiq65XBd+Tmu%ab=q?N6)4}TNkXS>-xJ{)uqRTpHHTCA^EXA_sAcC` zYl=0OT#0Q9PrisXt=7KHN^cLBz@S4tJV4`6 z6A#dEE-To0kcM#yG>pqKCl5-%xGeMX2xjJCEj`c)=aTt(KonoZat&C-*6iu|cffq3 z&YtgL+uE~S$a;IMb%u_?G`LU1rrsW)ajCZlXc(ACL%~EE3MSHjP%w=$Zx2d9!7NMA zHJG=Dwe(OYbX$Z)^+6N^6KFV?M$O)y*TDR{`y#c}SA|JK-(^y_EwN{|G{hb#xlP2T z)*hg7skH}aIFuEvJxIf#1mm$Olx5Z))Cq&K%-UlxYY%Jbflj;@g_2o&P!xl*g0;t} z+1m3Oli8@iQT)(bv||I2$YJAFE#pu!~;_(2Hcj=xQ;QO7=9DZB;(b z^G~QKR#k?Nqt)*NDDxzYlQ4T)^&hqF|f%28gZuI2aN$?6vemansTzI>yoMb1;jX{!}f=l~8a>zk4p!nB1~s|{E$ zz(oP19ATzHJMRs_*s&c*>XP74LELC7sSl;yVnf+C-2%?j^bSk#lv7Xf5g40^hz2BQ z6Vm+Hwg+&Ap#=%bJDepMK6^b_>1G>x<-#e2KNy&`j^gCrJ24r%u1mV2UAKn(8U9?q zy!ETbLLE=V0M#XDgjt|o+q%S;ML1kbonMC&Zv^niL(pg5hDA<&z@rd=oZ^nkNAdLB ztWce&`65p9IJOq5KpSqKi~J;+T$2><{HLI4050F!GZ8h_=STXW*Z7KPu>uTZJxVWuhw zwS;aiL(2BtjAiWc*iM}%N)`>M0*FY$mwz7#FowMsvND-Ou5QNFoY^z7&;N&+$sLjsd zvhUkSKBQZ(m7jbM%VXtF_E;iAg zCR(~}SjFe@G?D9==Bn4_2tNi{&^b)v2PIkdkj{hRlMx4ok)M{ZX#G|+7h`L#8?K%k zlz+UA;_Tgb%){n=k$A26PD8F~!wqe?-D+s&*6Rr@1g^xf&uAfgpLQec=AR*v? z0TKe;S0EwKodOBD%(_5AKAbZkAt%=d5^{FWxx^r6w+AHT>|Ou~IlGrYLeB0tAZg*t z&Bm9T%@SCom}4%n$+@1pG<-3VI;d$Z1J<_ka1= z^AvSP8+_h@M^5KCjp^|iI_F$~wB#eWd>Qw!^Yy=%pRWAV-r4)*YADT@L!&4gFYdH3 zo;IhG{NHK5)X87XWisqXv!P7#1)J<=GR^ww;%*wH#o8zB-&>mA%n~V=tE^_Rm};K; zC+qk`ruiBsZLSwHjZNl$IG;Dd`G2Aj&t{LwVwUBrsm5B>598S|8mYgi1DOTUBvn5g zHiuxo{#Hjx`KndsLtFlD&$zy?#wAv346S7vLN_pU4u;O*(2+~+m`iMOrJisJgAPb2 zbU;F(0}}El{EJfp9S?UYbS2?k5KjIgpKXVe`g|~W@;Cq7p+xc+x_-Oh5`T|KE}a^Q z9F6-ySlrJv^4phHs>UeiasHIWNjUD`8=9@@jc_51Rs|C4D5V04H_jG`Ek`q5-NYce z7(|zYNUon_F0sk=bHXJIL?EFc0tp2XNGOPI35YzLfmjmW1>qD#w**8mcnYFRLF6$l z5_7>N9+6zS|2rVoQA!06Z-3pLDbciCOYFJr@i2)VCy`wJ$6R958y_xVBmxN~5lASB zT;kFjAFqT&5Kb=I(~|J6M>sj74Lb)Juz2$BG!z2nFqquA*ERt#F7-$?aq!XZFG=Cxu#Sv*Jk3i$n zsd1x0M4|)~$+869&A|1R91V*KjP2 zGAuy?hGj{D?&jbNmbAqOop3B^R1ZY4r>Q_AbV0c^{tvjUqie=0PgvIj+uTnYroE3# zyghm(_^zYnP%gQ~F)5c^!?^?+#wE}&F1dztX_j#b5-=`H5_C5QU$~?#KB|O1$YD`E z5XHF!d)k~!^G$O}p6mQ|BSOo}9p>Y*rxB-dWzHPItU0X3 z2c2*xnL7tW@r5hbfQ4(F4xN7m%D3v*5yY@#So?1$cN{zV3zfi+LoGW%<4}Lk4$yEa zE4X%$hEWMLjLI_O4obkNEVJ$i>a00qEk5Xk?};#n4v6A=B3uLZL~3^Fa4LWPrpOz0 z?_7&*`;M&{p0yWCcMq1pokI;gK;uvc572NdD|mR2hG7Xb49hYb4@$tWEHm;5>a#gx zEk5XkKYe9>9TdeTOQ2y3*qVPWJFijsbDqAwQ8Uk#*mm>ei&)cYi9K9;d$>j7y+lT$Y)7Py)tfnXN}KUk_{XK_{F`X6*q{d=bkvU=drhz31Nn z^NpH(zKd;_&vGHV&)Z|@7)*ovL~QEq0UDQjdw_<4i8K^Uq@iFU4F!MGDD(E91Qg7& z1YLu9dsvH)I-%PlEUE{h7??oA!8B_2_Phq>-`y9frM@am8u~7ix^0O)v!x;SK*?<) zHnsKujZ3XPK*OP|VC_K~1|=AeO`$Ba_MlD}lx5Z)gIRl6iw`>SS`NDnOYhQJh5C)2jF4w+a@MpfimJ@0WvG%%WgF7V`bO@ST0|F0?=V zq8J~Pwvw0rEX}K1_ENk+!|EK)gONNBl2J5Em5ChYh3Z;<|DCLUd|>%Hi|5N%ma5#v zSr$*9-;HH3kV)~*RFz>IXU}hnQ?mIHht8i+O9KRxizFJe;1ORA1fj&|l9NLqHWMNc zJIkJ?7V$&2ABtJMaq z7vQ1*QjRcFp`G`JVC>iqBy~w}s32~&mDGpQHrP=1O&h>@n%-dvo^t9b9)Pi#iRggj zY(|v<#9IOUu@Cx{x5FY+AMhvyAXD6<@=-iJ zZ&s+z(|i%9d7RyE(gi7QNuGZVBQN?-KzSJQZ{iU!<-Yv0f+5WU3gkV=O+pC(00=^p zStM2g!;`QiJONFU?Ib|~-;*yTIss9WY9%Hcp~UBsB>?~cPyzq|5dZ)H0000000000 XfPpfTt0g`GhLh?gBL+Al000004Eq70 diff --git a/summary_results.json b/summary_results.json index 880a87f41..3e533faca 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.62305, + "Performance_Result": 8.9953, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.68088, + "Performance_Result": 4.20591, "Performance_Units": "Samples/s" }, { From 3ec24a09666db4e43fc2c16a8b3fb675892bd291 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 17 Feb 2025 05:49:05 +0000 Subject: [PATCH 106/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9747 -> 9748 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 6339ddb5d..c20d72527 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-17 05:43:32,442 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-17 05:43:32,442 submission_checker1.py:3270 INFO] --- -[2025-02-17 05:43:32,442 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-17 05:49:00,198 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-17 05:49:00,198 submission_checker1.py:3270 INFO] --- +[2025-02-17 05:49:00,198 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-17 05:43:32,442 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-17 05:49:00,198 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-17 05:43:32,442 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-17 05:49:00,198 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-17 05:43:32,443 submission_checker1.py:3286 INFO] --- -[2025-02-17 05:43:32,443 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-17 05:43:32,443 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-17 05:43:32,443 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-17 05:43:32,443 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-17 05:43:32,443 submission_checker1.py:3307 INFO] --- -[2025-02-17 05:43:32,443 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-17 05:43:33,444 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-17 05:49:00,198 submission_checker1.py:3286 INFO] --- +[2025-02-17 05:49:00,198 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-17 05:49:00,198 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-17 05:49:00,198 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-17 05:49:00,198 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-17 05:49:00,198 submission_checker1.py:3307 INFO] --- +[2025-02-17 05:49:00,198 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-17 05:49:01,151 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1611 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2506 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432102 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433742 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index aec11d8fe..b7ecf5c07 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -733 +734 diff --git a/summary.csv b/summary.csv index c0ebd0397..726aa6d93 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1611,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2506,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432102,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433742,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f0f72e17d..c41a88c70 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1611, - "Accuracy":"acc: 75.800", + "Result":10.2506, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432102, + "Result":0.433742, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4f34dcf34d8a979528d2f9d9700f11de2b434e39..5ac2b09a07ffba5eb2f1e8e588f257ab0f6a82b3 100644 GIT binary patch delta 3055 zcmVXMX1}i64|?*+V+_i%&)z7)E|tf}-_X(OitJ zxqoiBdTvniI*PM*-!TuH_eJ8h;yVqwrVTf=;dZN`nU7~PD1#rA0b|g2$8d#;R*MXi zkyyKuYer)4N^Tg5vn#n}B<@DCQ?ALEYeE7+%0&ek=l53?*RLvWURB(dDRwGt@s+k9 z-XN8tqUDiNUzc3JF1bMo{aj0v>{M&9)qfsYkigyT;usZVm?x+ZgoTfcgj_$L7zsJ? zpBV`O{|h4_(0yej1iDj3LN2o|BOxEo86zPlx5r4x**!;zLC$WUk&v@{!AQv2y<{Zh z?0#b;E&g(|`OD3Q1Qsdg7$r72_a`VZNxdH#390uJBO&#EMu|(Ze0i2Ie-ghkJ%5mE zSH08z+56>cD9x8c z<1m{n?zA9|n$a}>H_Def`K!51MuTuRl1aW`lY>mA*&toqMPXX3ebT|brRmKqk#f1p zY8H#B=6UaA9iPZFU&Ex$^@iu)vV1kwSgQs>JR5~$^%r$0 zGe4ZB>W9PTh?%dy)lpKuYL)rWmjCS;*Z0-9#A=P9wM^4#89a0j51oV1kxT6uB{sQI zPf!BTF%k+LBcaeS67nbfi&Fv}hPxEHlJG7QPW~dFZHJTkdd%R--~4lj5`T#?bp3XL z5|2nOof?T8PX>Na+|M-f+m}_U#yID3{*=W@Fd5t%nyu-LU?GfF1rqBhr2>gJ&K8L+ zM>AdBD3EcFjn9D`zs8pM*(0G5mf zutW`FX_R5fBmgW+5_C5QU$CStJ*E@Hl1BBJD0qa*XoM~(mw(3p0he{uSmm+mdSIKI zNW--EamnwA9tqrPv>eJMY8;bti5ldR(Eyi>2Dn5Ga%q-v$s_T92Vc0PEj_Lh z`t$~(`b-pZ$!ugJm*$)1l03}$>qyp7UL_>WT_jy$@7d`P{O0|UzzuPWZfG+ar$v!O z4I;^C07*szNPnWnwJ4IUG9;N!=#$p61l`TS7bIy*kLkoSDUvj*&qV=}s1Z8doZcFL zEl4iz@`-Sj0T8gG(e&@5!IHy7;ldR)%)+%!Th6}%J2oBtg-XVRLw!1o z#-T&@D!Oz1);UX3_ z%p$gCPtU&t<{NePd>7l+p5;Q;+heUWbRJ9teIhpX_AnZkdV3fRfXQhnn4E@!$!RE< zMt_;Nhf6@gEKATez}o{YJ+2eFEdo(}CJMl0Gzg|qv$y9pF#qnpNGaG11&wK6R$;~ z1Zxi$1yEM7_82u=dtPHQ8x=T;AG)cmrGK_6i&taY(v#2so(JEiZf*_22SK{Dy_3ii};?NEVE) z7E`HYPjlH;<>Ng6go#SauLrhX@ihwqnzTFk;f7GCoG zyWpLD@Gh`F{Gu2il(v$W{w&R_TlP}C;D*&Xocm*W?kD4LmMT*@$_v%C{Qf&x{o=s# zbr#Q;uPjx$i?b|_p5INRKa@%F&Qz6A9B0pOic_-r<%Z6mP)h>@lZzx8v)~bW0R%3x zaF?@Q6Pf~lvJovc9Kdr>0#brVa@rwSK`A=gYh8$cI$-(y$>P$ZB#x{4nbbGxYecgm zg^ta$6wszFYfUd zEB->XhW7kwrjeed;TBnjMe#7USe|c6ITw-)l$q6kS4&+CUDJ52PUEn`#5Q}X?#*^z zP_{iB84VfV9qzA;RNae?QerP9`|~A<0195vlEd;28@~FO^z`!oW7~17rX!YtKi0h$ z5x%bSo?+a-5xkb{k7u;SB<`{Y|7yo@V30RPQr$T|vMk@s_J%lqd>cGR_x^qGE{yLR z(+})_gbp?l*F?A@b$yX+)H;^e-4;h3yKBc3cy0xLPToZ`lV1nvr^RP<11zt9e|kEx z1Lvl#Smn|e`NbS5pMBqEli3kEfn%8oK#QvzWLtp}iJjt5gY?)KXxkWj&Z4HU+A5EO z3N?dDk;kY>UH9IprY<|0A0Rt0oIu%-o>pmTVi3pQ+0tFZbzN^~l_rMcTXNSvnVv6C zr{@xJmAz_wI-4&q#}|uwzjozT?*|qMd`#*;gpOyEM=6nC0X}g&Kkx$Au?7Pro!kA| z?b@%8<%Ru@mHfd^c_eU@pgvfu&+J+=ITxe<0JGi~VF-UBG7>r_0RR9{0ssIJ0001F zZ(~q$Z*X%jV{dY0E_iKhos!FL!Y~j;_x*~<`@~KteOO75+KN??DwT?=JIfw|)%YRX zZOGr(1PCCp>T-PVoVhb{Ioql>`32{t(G|)wf)dc&2&tP2eJtnc14?{gTCmn=s8A0c z&FY(yTgrcpgEwa^xFEqN5u`okwnCc_EXCOK4JhWb=+IGI8>d)^rE9Qd{F^m^3qtO& zg1`g|4DW!c9f|0G6nsS5AJ+~5!7;R;KnIWWEXOBr0H^%bhCw-VDj9kUSFJ~JbZ@;J zja}De-DI%F{5k$yyuJ^src&=t#R1i&;FLRH!MI7f#OFmkT+4h|#1pS2h^Id4SKbbb z?E63@F#z4iJu0io^UHFM>Vg#0loV-xza&$7`$!4-+Ks&EKLOQY$iIn4z&`ioC#e4b xvxOnb0t7TVBr%gyB!2`hvT&D^)+D+GA~F&>CXLnuvI3oZ6001D3wqF1M delta 3054 zcmVbDFznErM@xxacjwe}u z+N>fZh^OiDUnL6jb7`nbJg$jEe zD|E4m_B7GbZNn-)kEf|z$23>HE=Ty$&-~6|5Gat`pPzD~9!N#EPj^TsJ*wuPScK6g!o+aHTC4 zZ;(p4qUDiNUzc3JF1g_n`ni@S*{Rlo)qfsYEP=b*#W7cqVV-b>AS`?Y5_0`~0uplK zKLZH?{|k^1=)M98f$kJY$Ys_A67u1k0SP&|Js=@x_nb=%a(4SbLeB05kdU)`2_)p~ zegl#gzT9klx!Ej%MT$A*5}Ta+6D~1Hy&r*u)cXlYNWGuA#3fn2JWIfz#IK+Sa(|7S zmV|eok3COOXSAWmJMhTqJf|^z9z*Ax3y_w41eY)49(BI{_wv(K@3epRez_V-^X1Su z%qELFEr_FLG|m5w@}*AxYA%z}Ae@b4k}ufgAd_h}NEdfem=xCo;{~FllqWm}zV>4}XIByb;V7jd(VDOct{&UrjaEszDIXM&VffMIFk_ z52va6;jlRZ^YynnO3GKQG9TLVzdhsnz8aTUtueHgX$alG&^Z`7heJm$wPP-^$(4G- zB@8+sq0j*dg$_u_pYShE33NQ%rO=gxcR@J$i+r{nPU`D{!IQuF=ME*3$A8fE+Xa_+ zL~`lWNaT1j@Pp!hrjg&itWq_`Igj(FEKY*S;NH+|O>YDXVYDicSVt)pNW5{jNNhQp z>FOp1(ZwLT97J;c9CL|HuAdVwVITqt1rbOnh(JO?bW1?w;S9u*@Gc0aAi5ObZZo8I_v2_q3m zD2YHqN#qij-uQSWB!X~q(VmuscRj+%8ExqC4%j-B#`Hl9U55e*`3Nq(dSQ$FAWj=o zf0^OYwSV@Sw2tyB0C?qX0qAO;XYH8?g^u}Kwi5u!+;R*w4jIP@(0?!lk%lseG!#Oh zap`=x(I5;_0?J`og6`(vhD6rV1D$vz&pC_ggDCo#@d9XsPCk&!t?|DNAYbOe0>+_~ z^Za^DQaQL6nww9Be;LVIDy+cd^?G3I<*8=q*478VOC^5LekAyVb<3hua*ZP>mEZwA zqY`Ksl|aL&xoS&q49>qX$LrX#_3Z!DW)$u*8avE&+#CD1S|freqp zH5^N$3`>xJVOf%(yE*uRC2i?}PB@k{st=;rBUGRfx}aPd|9=Nu)=^`X$ExdrZEhkB z)85A=-V;3%e5=uND3@I0n3PMd;ama@;}U2Xmt4cSG|RXI2^g0p3A&quFI>`=9;$>s zy~sjcd4DAMhPXvHw1LKHQ6#yB zBMCGNNuXg!a(|6$Q6yVsNP^Oq?aacOMR5SNZ&?Z2JevFYe9R00zY_2~eO zLybB>!+)u)VAVkyMkUZND$CqDC;_9g%&#M;o#u?S^gt(kPlVZWKos8-;To_fQnNLO zQ~C2ZMc$}o=UQxgc5Kb?ti4#ed$0tS9O~Ku8i(3;fQDmP!MKAo3`?M4SeAKrPy&W! znR`c2bIloR>48r8(^qEFK~Ze71RA!0t=XsZ8h@2P=jrPkb?{t?Z3|Dnh&8R&zRgN+ z50}88Lp?k|<4_Y1&~Pp**m#hJaS1ex%Q7bqO2D`*^YRF0=3y;8&~`FTJTU&L|^ zSj5)s>G^lSe51~u?_%59vs}n}d#rVaj=?mzPsFC)9-wilw+Cn#m`FpxL>dYv(oiss zGJkImNUW}e_@lZ+%9v)~bW0R;SD z5-PJ^6Pf~llF?Gb0XzpKASH+-ryYV7l%k`()`j?|1D4O9EG|7t;<&1xNqwWfMl>r@ z=$QS@*syzWrw3Q6vBA9_-0#5^U$!__0@&a^;}G|ZL#&L$2Ji6~_xOu@{KY-~;vRpo z;x9yNXwR=^8tGXYZjohJ6c1yI<@u(Rb0NtI$|04W8HfZ z;p-~z8OHq^!E4F>ct%@H;x2peuXYRv260=T_k77G; zaBkX)RW5yzU(Av6+4pTWnH`}MIF^|Jw79xKwiPIm*eMP*NRN$ywvDmpENTj?t@1di zP&23$d5oIWb?=>O>awHx0kQ+b36u@#X_Y2v266nIE!{O-*Y$Q*X<|6OC3o$U>G|?> zdM*)H*{jB64VE4^_g93Cg)=GAG6*UVF-VcpW?eE0RR9{0ssIJ0001F zZ(~q$Z*X%jV{dY0E_iKhos!E=!!Qs<_k2a zE$f?-8Nz?Gg;%Q$STDdu0i+yZrb0XK4Z+y49Z2eu;7~!_Xe+4?rQKpf**Dz+&eQY` zOYoFaPw^2Ln~8`9Bxe)S{Mfb!aE74;3CcU1B^f?@Jy_{x8+zr!DTO~6n6-}L<>z^xUjaou~OCPV+dsZ_)+1El81n9Y Date: Mon, 17 Feb 2025 07:43:03 +0000 Subject: [PATCH 107/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9748 -> 9750 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c20d72527..cec5cce7b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-17 05:49:00,198 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-17 05:49:00,198 submission_checker1.py:3270 INFO] --- -[2025-02-17 05:49:00,198 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-17 07:42:58,635 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-17 07:42:58,635 submission_checker1.py:3270 INFO] --- +[2025-02-17 07:42:58,635 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-17 05:49:00,198 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-17 07:42:58,635 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-17 05:49:00,198 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-17 07:42:58,635 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-17 05:49:00,198 submission_checker1.py:3286 INFO] --- -[2025-02-17 05:49:00,198 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-17 05:49:00,198 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-17 05:49:00,198 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-17 05:49:00,198 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-17 05:49:00,198 submission_checker1.py:3307 INFO] --- -[2025-02-17 05:49:00,198 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-17 05:49:01,151 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-17 07:42:58,635 submission_checker1.py:3286 INFO] --- +[2025-02-17 07:42:58,635 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-17 07:42:58,635 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-17 07:42:58,635 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-17 07:42:58,635 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-17 07:42:58,635 submission_checker1.py:3307 INFO] --- +[2025-02-17 07:42:58,635 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-17 07:42:59,607 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2506 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2876 | acc: 77.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433742 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b7ecf5c07..5d0df6f29 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -734 +735 diff --git a/summary.csv b/summary.csv index 726aa6d93..6d253b1cd 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2506,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2876,acc: 77.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433742,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c41a88c70..bc0567762 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2506, - "Accuracy":"acc: 75.000", + "Result":10.2876, + "Accuracy":"acc: 77.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 5ac2b09a07ffba5eb2f1e8e588f257ab0f6a82b3..d4ff263761a8bac0aa435f0fb8827ad7594087d1 100644 GIT binary patch delta 1550 zcmV+p2J!inOqNWrS`!6JtpNCAlV1}ge>xC;Kfl7tdD(r)#cr;sbcNc2s^wA@Sg}u) zn@md1CUH*euw{S!acE$>aZ=y_5>mxC_I#etGvmqd%O*?F9h5v_`JsjlOGA){EK2hA zp>}D2L=o1)ge*;Oa&`}j7;&XfDn5C@z3inF*)hbJbfU<2kx}!-- zS4o-(b<;f;khRfsDX((odC5ZHoGJc`zZJ$XgcM3D)WuX;e92<*la`8h=S--FxaJUb zUcRKPN*XEwPC&@$6+dI?qm4O+rCXMTdvMZ&588M{1(m5NZ^En--#j}Vf3qxOG7r|b zOBxDUR(c6Bl(MQo4~t4K?xj-U?edNou5=nfOl5K|Lzc(M+T5Fz?@hKZ$@g+qD_NzL zo>F;hV!83%ep*|lE!R{jM!L=06rOn6D|!u2Tib>wQ92At1WJybP0-^07vv~FUf@_a z2)p{2m4JF{quw%|e;*^ye{%=OwVzquh=jIxv&U+y)-DMb7uP3Ke1KeRcoxA3JpkkY z4Ll=oy`TY3B@h?`7kda@U3{IMUoPj<#q#p#Y<@becUvmmqw4`SY~s7WT|0*F6Z^Mk z;v0^K?M~0wA%=y}nrHqGc`@4(mat{jaJJeMJ7~utr13yXYsmLQe~|-t4vHWph$N>S zh82{eqrBFI_@@Jw&!0>#Jxb!Zs-H=HqrOHoD^lp_{Y~4jdvK=*SF*9ey&l}}!IfCH zI93E~@Sb*vd)gsZ+F^tD#EX04#Xa%jo_KLjyjY1BqBXSpSJRF3EDg8FGAxRRw#D*% zQ_8-OWT5n{zFg{Jf9UGQYqcAP6(+XXlXY+Q`+~IX;mByn_-=E5rBZb*I!cPYl8HhKbOS7}e}8&9u>%_g%WbOOin zCIBt2ZjfySiV{1?p^Efa7-(A8x?9Am12)kb9LQwr+RhS(fk0}f#C$o zg!D8@6N5PZX3muE8m{YlJEJr)9N&_&_Q~{oc{)9ph^x$1fdn><>H{0i`iO*OF*iiOOcYyOWy~7eb<)+}q8`cFh%`U8ZULNO zs6m4A4rfV*&t4B!y2*xCxo}G1j|L{KgE+bmPK?H;X_98vT0{N}f39EN+ErtrPN!mk zs)94ZEKsj)NLAp=A{?%!&acCXw*vU%G3XO-{UTi-@F)ZzUED$Wm_0pjR;bF;d>*HH zoGmu#g52h0{_u4gdDVXg%6`bdi36a^eftpNCAlh`Cz0pXJ{B{~69 zlWQd=8;Zv7P$dBX08jz|01*HH00000000000DyrsldL5^0f>|9B_jqpBLDyZ0M0t* AEC2ui delta 1543 zcmV+i2Kf1wOq5KpS`!5>vT&DRlV1}ge>fC=-(O*6UuGV1v70L@9ig_MYPeJdM$A*? zCMP93Nt}%xrp&KjhX$q-Cj}OekSe}&d_JG6PljJMS&HtU#Jkk*AN#pk7&wrIMnWeYhOlx4LKJ`9?kiob}K}2V46IIke#` z_5*TmD(tgK$0en!Xf{Egz*rkdf5r+ORZ${7w>8HsW#w16SKO~wSt10KZNbqUO;Wl_ z(nP3-?zw=hjh@T!DtDfjECkM(V!!xXVGKh^p`=1pri$_<`|Z4Sr+cWNe@0~;}I29mZHprSr@)}b~&30c+i|O=BbI?b*1Z=IzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3 zkT*tB-8nw8EZ@xbhB$tF8$3t%{(bN+jPD!M5A1{vHWAlEf4C!ceUWU`I+oYn7DpYs zYsVCLZUuf$-bFK$UkB-@#bn)CTY(aZo#Ie~^w=0^+ZcP!qNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rL< zR%v1o$KTn~V_m~_U2kWVCWhl%a@RhYo-a?Q=Mr(1y=r_qn=dcN7mIqocI8&@2Nnr@ zOzJ;`j%SlcDUn|RK5;xh@B-Jd1_LFX+x^<@+OLo0h5e3|{J~FoByg0VK3J>I>{>H9 z7o-0Gv*8zE2n8ZC5;`W6M<5@6Zo)7SMfd%R$os@jD1BH-klKn>kt&snsyoXbg4OsT z+il3-*8~V4vFdVs@0__aayi?oHu(kTrO_42GlCM(+z6?g3Vkf+=>tl9U|O)&XsA#R z9?j~Tl3U7+gEwa^xFEqN5u`okwnCc_EXCOK4JhWb=+IGI8>d)^rE9Q%W&E2pfD1zI zu!6t@3k>gosU3;vfE0X0+8@^r0KqY|pg;$Y^DM_FZvdzK)rLVib1E5n3s Date: Mon, 17 Feb 2025 07:44:02 +0000 Subject: [PATCH 108/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9750 -> 9753 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index cec5cce7b..3e80f540f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-17 07:42:58,635 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-17 07:42:58,635 submission_checker1.py:3270 INFO] --- -[2025-02-17 07:42:58,635 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-17 07:43:57,850 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-17 07:43:57,850 submission_checker1.py:3270 INFO] --- +[2025-02-17 07:43:57,850 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-17 07:42:58,635 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-17 07:43:57,850 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-17 07:42:58,635 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-17 07:43:57,850 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-17 07:42:58,635 submission_checker1.py:3286 INFO] --- -[2025-02-17 07:42:58,635 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-17 07:42:58,635 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-17 07:42:58,635 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-17 07:42:58,635 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-17 07:42:58,635 submission_checker1.py:3307 INFO] --- -[2025-02-17 07:42:58,635 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-17 07:42:59,607 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-17 07:43:57,850 submission_checker1.py:3286 INFO] --- +[2025-02-17 07:43:57,850 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-17 07:43:57,850 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-17 07:43:57,850 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-17 07:43:57,850 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-17 07:43:57,850 submission_checker1.py:3307 INFO] --- +[2025-02-17 07:43:57,850 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-17 07:43:58,808 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425368 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.42599 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436249 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 5d0df6f29..a818abc4a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -735 +736 diff --git a/summary.csv b/summary.csv index 6d253b1cd..26c2b4b2d 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425368,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.42599,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436249,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index bc0567762..705e33f8c 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.42599, + "Result":0.436249, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d4ff263761a8bac0aa435f0fb8827ad7594087d1..392d3496eac4ee71111add6b4b390dd50e745c8e 100644 GIT binary patch delta 2561 zcmV+c3jXz$OqoouSqKhtIT8j#2><~8K>z?40F!Pc8k2ztFage!vIxe1O#68`jLSqB z%8|dAW>@jUR~e2cS$^8AA|!~X>GEGC3iESms7mBNwpCMA9>QTZDH1D_a5$93iB^?h zk!Er9+v<>5Cs>6FdmSruv5EFH(b8?hDn5^=sa(f2SG_Jr_|eb&&S4TiD9N&ibnX|Q zj5sii{Imo`>$jr07+Z6H-Ej5XpyYKFXYama9yafb#B0TO8gfk=ZfL{pRzou%&t^~t z9+bhxpzn_1iYr( zS-w0=z@Nmgpa*h)jhvQ*cb|_vPf=&Ip~pM$$mu+%F?}9G=bQ_WmV5-4FXJ9{zW(>} z(^c=ZfA)U48cOr!&^XK{i#sidqh>VC|BdpcPX1~xlhGiYjbxH9*yJFSX*Ng~cTt!Y zYoBy*Z)tinOQc+`vYN$Ws(Ic!S;r?b&DSt#bG?{pY%&jjg894=%omM#HhWAKvn*dt zHP)&@5YI;8Sp7vE%FGX^sruotIRf+bw>nD7SFJK1+Va0WIs)H=zxSm2P70aAR&Lkzc?k(@o<+yR}$U@;p8v!*>*UouLlNC z{^p-Mlt>n;1kFgXnS)$@O#0B{sQ!PPl}D2qY9lAfX@v2?fzD0g;C@ z5KF?lAe@5emVgKbPeF7kh&-l6VlKGEBa%z^e+R^WI!dVk;;p+gC7PCNi9NSH9wyP_ zB$BKDm`iMWp~pL5>rfig2QhRV z3MAwsxb*6UE%JjnZA|@ThDX=_*=y1|%Buk2mA3_;t9hQaXCf3j=5N_f03>tEG0-?< z94A13!w^Io${^BE2!Y0>^WjE=FhmI`hh+)6n}Zt?SxXOe;*mV(EUFKp=wrqUpbb->oG~?;9h8MJ{A6DBx|X#0+ZM4fvuOPnxR`;AN($r_(A)T z;1AXWl4hW=HLsK zw510+;aJkBK8RwEP=QA1f^up6A8=Vmja442t_QZci8M@mAD4Jf^hofnM$4gGa*bnB zF1dzt2{epLpkZ8c4d>D<;}RrbT$Uv0ZVtY1Nn3iTe-irihDG&36z39bWOFXfH_at^ znDf_>tfRb2NSeDyy29SG(;@KY{gL1s;uhV|1{$YDk>nbVB+xJcYL02{C326!L~zf5T92VaP!Ej`c)d+N%f`Y4JK2{i1U(t1Nc{u7U0Ybmf2lhziKhN)&S9Y-)f4r}RwPB@dyk^`dn z!j)^l!nICY&c6cXTQ%tj;?gm!{kM}lHXZ$iN?^jFJ{_QOs8I)KIF%KwI!ME)1R6$V znOg@XU{sd*bp*B3oUxW3=!EZyFk23Y;(H=o1NKB}w&rjufBvS(8@23Qi*3)2tr?!R ze-}%450=1^LtQ&S<51fU&~Pj(7L5!tfdD!f8ks*KM#oFi&(A!i`beyJ^v1vZ`9fIU2I!> z3cI;&?E6ZQp<^-)ZWOVpxd&)mYVH9VMkdlwGLeRoi8PcPVB5_An_?qMxG z)Ct`eVNrb$#mEF2PNq?_x#u-9|L(>}E%jA_($III)NM=bnJx{n2TJY}v8lHQe`s9l z?ExANWd&~!(l98&cx(z~nYRaZ!k{ek_883D!&-Wv6R$;~WZoVW#h|RIVXpc@oA+m_4m}FMgq5G4(r9Jbb?#)M6I?v5@cI1@G*GcY*!k7sdFXw3WQ{ zXK7yDvX|lo8dm3U?vLfUpNzv1e1;=8nfULVh#n7aHe!7lSd#be>^CytR$5H zbwwp4ia_ivd)ivW57~yae~*(UX)ATba(wTcxifOPJjf>c1?z-X70QwnMWC4GLhUN_ zzFEZ!6gf{7r;S!np(8l7tnW%@2-6l`tu|o202c+2a)g-*?Y%byW5@O&sY`-G1#zpb zq&}2(hYe-lbO$(3(+4cUe^X99#T_s<6A=wa&L*V!u}u%)3_}AFly^8wGJN)Wu+rUZ z=#>kn6#i)7rgao2_rZzD*tTub7QHp(&+zB^<*i>e7OFcH15}rs5oUpUZR-+W7U6IW zb$%UAycNJ7k3qljHY{@L10IC{

*QK8mO3%?j0dn$O}ikF)tE8l98-g3K0Q-N>u{ zGf)mg{!JVKr`*?{p#B51h#|@X3353S215w|0REF(Bv%2%ld~i|0ZxIt z=&aJ)M2VC2yAf;hSdmn>Np01`Bom|{A2Px&s$k!K$RWAB0pt7jk+12VYGUof&kF3> zhq{Yz7O%546tP{cTX;~KpP6!32_7wy3PcY+?DUIGv`S(n;B4OEy^*dv2EKgQ+RF{@ z&~0Rmc?E^&eUnD3TbG`H?a=z-c%`)OZOfVTOAn-dzNL{w=S0&A3RZBpoQMuBat&Ib zQwKxBriFL)C`Q=!S?lc9BHu-~45<>1nAnu$7zd3!KG3yqdHf*YIg5nYV7NP!uF)KM zQ?6D{LQJHeUqVc(z7y3osji3B>i5mVJ*y3EqfVRLpv#*PXsGO2KKMn#jNjx)M0c4gt8+L{VC`ebkc4c%TmNv~EzcHD&Wx)4t7FZ=dg zi>fIYHKx_?NO>p3;^d}+ZEsDO@~BNIgO-{=EP9_e7_Ov|Q}iXieKjGovSrllRy z(O~-d7XSF+=-UpjC;4xThCTV!f-bN>j~>JsZc%j`Zkd?U3@U(`g(UK+VNdEL$(%+g z?j3g{epimm_)I*{C!TQpsCS<7`6MtNjKL}doj!OBPC>~%hhPOw7$Qt(1`vDGM9bG? zl{Npf)Q|e|{4)YsCJXl#hCCAxit)LJNM>?E?Mkl=-&{@!u+TnNMQ3z$&9-Jj}V zR{=YiIrvKrD~v}~oS{=Eevlbqn&J}qP(`qTxPil?rKPlIT``2D18T>PB_x`E87{*e^5sPh}kUu%=X;){egnPPGQMUcS$wY5iTzdq4*sl-uHd z9r&zQwlkS5wxoh_K}3oc=d%Vtn8p6~-w1m<>f{Ds1twQ)4QyRB((CQSOI2Q_xfgv(npM2dNd+s_B@4P<)<$!6xr7w8z5;d6d^Cb$|&JN_}_SafJHiVow+g1k3m+w zh-=`5#ix=Ryt->BV?r_;6w`KJSqL=KRDCZI<|P>IiiLFGhXCox3{^U8uAZqb6evK( z5FB)s(W&a?ujL(h=J*Fg!Lbo{0Og*W1aUQc6|dHN^c6K` zb{humbA&p&>`ZQOoUESWTG{-Qd_9yP#kx2hHgSpvKp#nqDj>gs^Hzg@k6cZh5K=lCsM*qW4D}PlI60JZ05uJ7>M1&wKq6d}$vy~&t%7(lXTQ-$V4W$+VPRZFrB->$Rp;i7Wyhd0Ld{Vg;r z_R1Sfi=PzAPU{KRuLzyS8jxeO!&Q^X`A8vTw1rjG7eB}gYLRK~zCv<9Q`^;X&Jp9l z->shE0@`2TiVYPnaXHZOBCU$1X<*EkpV&R9_(;0jImS9Ll~jm%$7o2#Ydc=20!dV$ zjg|biAEcEEOn0B70-bGF=Qu|k{sV>`GfN7|F-saEX;~(+!c{BHjskO=o6Q&p(k4i| z%SkrFUR%>;KMwpTd_g7gzkEBWzCFxWgpFw(6e069y%eghT+e$onqIZvx^kGd0jjS! zk8-a#>5j=j3+<{v@Ie^w^jPl7&9P~8xl)j@5pB5mmHh*G2=P%UA6q2F6P}S17xgP3 zGwb6GHQQti2bEi=X{pzVt5KH5EL@Vg)eXT^WL8_hrYwKH-Hejmw{-b5_%L(C$8a-K zmaUr6pbk4RGS%+k;Dy<8K`^<~d$Sh<;hEnixVsg-*~dYews*D^TBoSUBCbl^d^&G- zUV2ljsslb{`_S8HMS+Vdi6V`!$_h!+N!IA;PlSQ=HYU%z{#qRV*q@u|O6;N%l<<3WTZkeJ8sl@mlVzgcF2MwOR(|7aF z!*}JXG)@ZmL{dq3;+afesr|vN_w#Y1*K+3{;-j% zlQv6#38(pp;PtXDIP8;d*28_b@VE1(hp9$s_k1aGj+6IU81+?-3WQDC)0l$JW#QTvAx)x_>I}D6$KuqrS3QM;Yqs_pM60hr@)6Lvm!V zZ16|Tth$};217zdF$~=iT`gGJ7d_#ppw}9;Nn+hJp#Ocx2)Ciuzz+lxR1IiE_zu+q vE)bMqsz7ssA52o<_i+LN08<(OfD`cl5&0NFsD|ADCJ=UE5)7&mzcT*<6r-Yl diff --git a/summary_results.json b/summary_results.json index f357e253f..4ed4c2cd1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.42599, + "Performance_Result": 0.436249, "Performance_Units": "Samples/s" }, { From fe05cdfccf59ae4160111115b5ef5e847e0ac4bf Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 17 Feb 2025 07:47:55 +0000 Subject: [PATCH 109/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9753 -> 9754 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3e80f540f..04f0b649d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-17 07:43:57,850 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-17 07:43:57,850 submission_checker1.py:3270 INFO] --- -[2025-02-17 07:43:57,850 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-17 07:47:50,208 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-17 07:47:50,208 submission_checker1.py:3270 INFO] --- +[2025-02-17 07:47:50,208 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-17 07:43:57,850 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-17 07:47:50,208 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-17 07:43:57,850 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-17 07:47:50,208 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-17 07:43:57,850 submission_checker1.py:3286 INFO] --- -[2025-02-17 07:43:57,850 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-17 07:43:57,850 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-17 07:43:57,850 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-17 07:43:57,850 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-17 07:43:57,850 submission_checker1.py:3307 INFO] --- -[2025-02-17 07:43:57,850 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-17 07:43:58,808 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-17 07:47:50,208 submission_checker1.py:3286 INFO] --- +[2025-02-17 07:47:50,208 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-17 07:47:50,208 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-17 07:47:50,208 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-17 07:47:50,208 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-17 07:47:50,208 submission_checker1.py:3307 INFO] --- +[2025-02-17 07:47:50,208 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-17 07:47:51,310 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2876 | acc: 77.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433742 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430312 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index a818abc4a..fdeeeba2b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -736 +737 diff --git a/summary.csv b/summary.csv index 26c2b4b2d..2e8ec3dd6 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2876,acc: 77.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433742,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430312,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 705e33f8c..5feb5a537 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433742, + "Result":0.430312, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 392d3496eac4ee71111add6b4b390dd50e745c8e..04a5c66c230cade2ace76bb612a4b8d9676d89e4 100644 GIT binary patch delta 2771 zcmV;^3M}=ROqxuvSqKF#i?Q=WlU@i(e^qe-^YY&>s4;k+BVlJ}m(;yqW@7W4(|wSi z-O!I7{`xqZD)%x@!f4)Bg{G^DoCnb`oR8b;&GqMot13zA&xigrn#;EOBop=5&VLRc zqWJe@B4w&%3Fb*#ouujFpxI1o#cs?NjMzJ{6wotuuRfu_S^c9*d$no3VRbP zbg_;0G||#+!^+=}r>We;G}pbZM)=WB{mx+=Jt*<2hh*XB?~FJwjO?@odF!{lIUied z({T0Npk#IAXYama9yagu#2dwT8gfk=ZfL`8p`n?LXFDi^ACv)O(09jhe}#%xiwu*I zSi6#IMq=+uZWxKPE4gJP?pCr>uF02cLIOd`MFkn>_g59yuPSa{Ros>-b}DW0m9`+> zAeEw`<&jcfmt4Otxj_kiUrUqhRBN%-9$ApU-Ru>$Tf0W z65f42_B=(M(S{y&;E~gLPGkBQL+6|eCM|gjE?>qy>U{n0<)^FOe`){h{c1IoWXqv( zm`;{=S`f{evuXD4EL-Yie=TG@8iezajI#xs9HcTy2g&ko7AE=HCmGyZn%+!fDOank zX13SQ4#QmvT}gPC2`8V(XT@+*Uym6)`OH6eD3KUL*KZdn z@rdNosgcO>WZ(z+{Y)dfeOae!j58h=PiYhflfk{A*_z%6f0n{%RUomBQYw&m;}l42 zIhyI}CJ&;^gXkiNAiA}Db6O;gmjD&*7NGOPmgo5anfQaD$Vo7+H38x^sB_J|` zry#l%M2u;Xm@(KXDn&(-2CPJa}{H@vvfMjktW;6~N#|fhW1UU_5kke2I z8I4Ql!>xwHkV`;0EKAVc9^8@$Ej^|akK{Rrs6G=#e;+emFdCtgH{`M~{ZFIdu+9@7b8Nu&Bq6g)y@G(s1YOXL55%Q|YT@>q20Ngn3>btLO3uM(2xE|RXW_v~~Ce)IlF;D)$GH?$dz)1pYC z29abmfFz>O~B1;lS?1Q^5&)HDejP#WG-uG#V>-b-5wPVjQMf098fH(Ve`aeA zQu*^YMc$}o=UNm!JGN$c)?O^#JyG8$OG*6h=Hjmn?%^!1H8c&)k1KMxayi&)e! zi`beyJ^v1vZ`9fIT@X{N+{0*GYVKh)KqjZ5WO5oxCa0lf z8fE4lE&(O8EJ4=*a}TuixK8N42t@UnC?J#3AelzZ=APHc{JR??wbWMye@a8&g;KXI zv1ht8#2zToCt_1?52JCZw};Unloh-^oCcs|#$!_`%e*~YCje!cx5of)547}{PP`U{ z61+WJ6hK+Q+hf%1?RkyKbd=*Le(0{UmfEVUUX7xsCtCn`9(+k$uiG$nqf7jz*H12et)|DGJhJ*$4c_Y`ey>{{AU96JXtWh zT27^sJ}qQhm5+<;9mMu)G?xHk}X3wuC(jUq=e`Tu5D2meO7x^jK{&Yj%NFX17&2GXl5QOi2ipclGPAE-fB|&N{PDQFzs-oU3djYHQk8HPrx33c*fW)cC z^~`=dyK*(#sW$lq=cUm#$}@rz(A)^AH#PcL&eKPf_`tMatx<3^M)Kr2~?tlg3nhIYQ@o+8kVG&QfmLQ(?QNQtaT;$LPB8dU$A?``} zoIJlKEa#{xNHI-Gk>=%+l=Qx&Q}Wf1yy`y#)p5wbi6_7z_w^@e{s6O!A<6;;E{n1A zM3Y)1TLHzBwj?|OPm}W`K>_5GH6=O$R+DfgCL8A=VAdu9002+|000pH0000000000 Z004l2Hs4;rZBVlJ}m(;yqX0&A1hcfQPO@x%&}^o`L`Ht9#dA5!kBs6Z^0WMHGH#~xMEb*JP&92Cf4Xir zBR`y}s|*K8l^pSC6bABR94sO^%T`GeIrX!A_~|5^r)uYL80N={dXz+t+Uk)wIQh*G zYO`~=?E5y959!uxr6}btj(;!SoDSQnUPzlv`*}Hx%S0K5Cs>6FdmSru zv5EFH(b8?hDn5^=sa(f2SG_Jr_|eb&&S4TiD9N&ibnX|Qj5sii{Imo`>$jr07+Z7Q zaP{1v5wml)=WJ?~dV$e=AxoGE5+` zb|u$9V(&_BfW+CA+yaTak?fRf;&M%vK#+2|f{gR~tBUJa6*sRcZp#!qm9}uDEf#N( zO1Yxtky2lmT)!^4;S&0}mL}P$)`Ha@SuBCO+r=?gkYS#1g&-_^1QK%nd;$`3;y(ik z0sjk-5a_-F34!huNXTW@e+3fq;hX^pIk`O`A!qlTOAK;$`#?g@?gfyLvwI07(S-w0=z@Nmgpa*h|oR)-l zpN~CHQD?NF$2;)I={%<~eI7&SoC}bad<2&-;~sUs{`d0JRqwQafA)U48cOr!&^XK{ zi#sidqh>VC|BdpcPX1~xlhGiYjbxH9*yJFSX*Ng~cTt!YYoBy*Z)tinOQc+`vYN$W zs(Ic!S;r?b&DSt#bG?{pY%&jm`MeR#7mau}drTIyEMHAE)~Z1e&qm=`{Y4$h%nzrj z`r)uS0`v8^I!ek{f2}ef+Va0WIs)H z=zxSm2P70aAR&Lkzc?k(@o<+yR}$U@;p8v!*>*UouLlNC{^p-Mlt>;!*KZeG;t|QE zQzMb%$-oba` zn;1kFgXnS)$@O#0B{sQ!PPl}D2qY9lAfX@v2?fzD0g;C@5KF?lAe@5emVgKbPeF7k zh&-l6VlKGEBa%z^e+R@mN~r+it-CWNnwD#cJ-0m`Ceh<0lB@riOKf`M!zGMFAfY4z z2_=zBTzccp~pL5>rfig2QhRV3MAwsxb*6UE%JjnZA|@T zhDX=_*=y1|%Buk2mA3_;t9hQaXCf3j=5N_f03>tEG0-?<94A1-5JVcvAkt6>fySlt z;YNcnLb- z>oG~?;9h8MJ{A6DBx|X#0+ZM4fvuOPnxR`;AN($r_(A)T;1AXFk|Mzh9%c< zER8ZOK>~(lNrLX?;0uaWfkxl82VO(+zf9KLH;}RrbT$Uv0ZVtY1Nn3iT68iLp zMfE`x=MrpWb1uy{%_Vu5^VgBAqr6H;n!8B4!rrsfA@Jt?k>DHR7TwSW8mC2(dK<}D2fpYH0++zdP6||6OUeN zDX48#$=ft zM=(DQYw3YbIFrnh1EToCm21GlwN6{kzXIi3HR%ZA(lM<4x05?I9sPw$V8Wq39iVZj zQ3q%^l@+WyNW-WE8b)QATL&dzRF?U51hvzgv6dd_gzt$kTMmfgdm>x|_C#v7f97y1 zfBvS(8@23Qi*3)2tr?!R7fW{!mcWujT{}SIP}>gBa4ahrcaVl*2{a7LGVcybz_2WH z?+9wHIb$t7&FwbX z7<8zI2WT8>;sF}YWd$1#(l9Q8e}-{c=Hx*M7?)*U9>L5!tfdD!;aoC54~XK6Sgrw! z*qS{({|=aM)Y3EdZAQGF1_$OIZrrctxG=QT3_?#4(h^;Ln=f6#ZK)NM=b znJx{n2TJY}v8lHQXk6;;0U8cv1#b`1Fet%zYzk$Ww+D5?pe*zD7|h$lT6&-puSKC` z-X0Xipse8SF>3bqyvAfUDsU7(bXQqRZB`#N=V>+9{2Ovds=PN%67 z#QEu3MQ&PG-}%212e+%Qf5*Sb^Zw%5Ff0uH{&e+a{w$o0mGqC*&ji@T&jjd2vS4(z zm`Wvkn#;B-ALsceR1~W!!^hF;2LhCN62?iGJ*|2#exYD7^*d2Ke7_vjVix|fkni6G z@9cwjf&JkZ#rUAKmAv$4Xl``B_xVK>@0iQTEq|8hO~c=lO|~^b;WXg@0__aa=ARn zCi(^IgjN;Gk`zUtnC3$5D)hct#S0WUPZg()R#2fMIJB(qN@fVt7GAA3V7&kr1(0%t znF{T_Hw0tH_8_TCf2o_Ij|=-E8QU3#SzRXyB%G6est=iOJZuZPFIKHRR9m=lbQXUo{r0 zI~4;|mz)u1fqHH05?>bKa1C{S9ZtLzz#or6zwtIKa_R#fg#hFfcT_%#r{^Zk3e|a< z&*C(Xv-u{Slly|q7GK@StNt@k4nzJ;908}?*Po#N1G9`F$^r#)IT8j#lUpQP0mPHH zBs>95lk_A(0ppW4B{~6BlW`>`8f~ Q0gIFIB_jqsBLDyZ07W1iga7~l diff --git a/summary_results.json b/summary_results.json index 4ed4c2cd1..400ebc458 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433742, + "Performance_Result": 0.430312, "Performance_Units": "Samples/s" }, { From 576231b356f3be104c62d7226e94cf5646e18fe8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 17 Feb 2025 09:07:13 +0000 Subject: [PATCH 110/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9754 -> 9754 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 04f0b649d..3f3bf6883 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-17 07:47:50,208 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-17 07:47:50,208 submission_checker1.py:3270 INFO] --- -[2025-02-17 07:47:50,208 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-17 09:07:08,447 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-17 09:07:08,447 submission_checker1.py:3270 INFO] --- +[2025-02-17 09:07:08,447 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-17 07:47:50,208 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-17 09:07:08,447 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-17 07:47:50,208 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-17 09:07:08,447 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-17 07:47:50,208 submission_checker1.py:3286 INFO] --- -[2025-02-17 07:47:50,208 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-17 07:47:50,208 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-17 07:47:50,208 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-17 07:47:50,208 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-17 07:47:50,208 submission_checker1.py:3307 INFO] --- -[2025-02-17 07:47:50,208 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-17 07:47:51,310 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-17 09:07:08,447 submission_checker1.py:3286 INFO] --- +[2025-02-17 09:07:08,447 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-17 09:07:08,447 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-17 09:07:08,447 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-17 09:07:08,447 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-17 09:07:08,447 submission_checker1.py:3307 INFO] --- +[2025-02-17 09:07:08,448 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-17 09:07:09,577 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,11 +26,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73362 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.20591 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.425368 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432377 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436249 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.406511 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2876 | acc: 77.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fdeeeba2b..3e40c3898 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -737 +738 diff --git a/summary.csv b/summary.csv index 2e8ec3dd6..639d635c1 100644 --- a/summary.csv +++ b/summary.csv @@ -4,11 +4,11 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73362,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.20591,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.425368,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432377,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436249,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.406511,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2876,acc: 77.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5feb5a537..a07f6a29c 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.425368, + "Result":0.432377, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.406511, + "Result":0.40751, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 04a5c66c230cade2ace76bb612a4b8d9676d89e4..e0212e5346e959967b6578996307af9ff29f6881 100644 GIT binary patch delta 1033 zcmV+k1or!yOqxuvSqKSDRUJ!32><~7vtkJI27jO~9m8~-Jyh=4bo3W0850im=`b3H z8g&>AQdz;O!)bs@MgvrqxplY%KxLU8MO46PH;~IY&lF6?unp=*%PVRnuApS z{7sQJYT3CKMbD0{8J@NGDUZ7cOU9BzT|11%p|%}HgIHED?r<8wlFW;N% zwlu^ZCebEhQ)>^SajCV3(IAr*tUa6tm}JIdQzpx-JzOWiWSOLcW7KTzc@4>Ql+!4F=%%uk+NvyGjiRL|oBurzzD?aMe7k)nhfy4+PwU?EA1GK({mv{JzF!S$IS>C>%J=Vr zclN=%!2a-yVti2AN>=)_G^=jaOa2Eptj^)WAIoz;9*6TpnaWXCsIF!A-|_kv2UdTl z(PH(Nr7Cw(nnttdR}<+EWt_h=K>GO;Hlx%;wq4OtDO9KQH000080JF~#b`Aw( z{5KROlS&{fe^5)Yk|4Dery^A<-)0C=q%i{p2X3;_i{A0ZJV`|!5Z`Dc)fgmA68AJ?oY)5 z)urH+J7B@My2O`7JY2(kSjH1?B#5Uj>NnnwiyZnuBryOz#62mWljoP!0@Vd6?o(2v z`E*5Q7?eyYnSJ#mulmnGbsX|<;t6obef$^r>ZRUJ!32><~7lV2oP0mhTJ zBs>97lk_A(0p*i6B{~6DlW`>`8)W=96eR%w08jz|01*HH00000000000Dyrwld>gf Dmx0}+ delta 1034 zcmV+l1oiuxOqxuvSqKR(i?Q=W2><~8vtkJI27e$f9mCpxJGo=i(O;-!OgPl1!)P38 z)L}G8Wd*AarvWM%4NzI;*5MKWm1TY%LG3hW(9&Z%!95YMNEn-cpwQsZ1+ruSe z(4ih4M&nQu52HaYE7*8A4RFb5fXgx`50?PAEc5aRF!Ml5kLd)t1V0ZGg^O6!FpJol zJw5*pm~YhC^Ia6JJ-OZ7Huim`$k2H*4SzI>*woy^Xk2RUVKhJ{r=etW8cHUop=26m z<{mBqC9^C+*8p=5wDh=6=)MR<^_eIjlhGiVM$P7)*U0?48zZ&UR|QH#--S}QEwN|1 zG{hb#(I;Y4Zx5q!skevGAe0rnJ)8!hWX5AtD9gM(Tqgi!nYYIPZx6Kem`=PFg?|#f zJzNw(S;5<5)a>nfjmdPB<0yXUuCkWes;pj(qNgWY0C*mJow`~0dizYqV|gN{(?kiP z>~yUox2@~%{9lQK+x6GuU*uVT(R>)@hJJs#{xW|W&c{mf$NFai?EGf}^gLNGx>`=9 zl0GeDTa}NC>>bLBRh8l6X#E2L%6}pbqc}{T*1hMyP_Uf(omn(|zZ%qX9{#bE@81RQ z?1OiK{oxnI_@K0vtn_DTR^6(X{10weox_Dcmgjyv4(Ew7m7}asUCVC3x<3^M z)Kr2~?tlg3nhIYQ@o+8kVG&QfmLQ(?QNQtaT;$LPB8dU$A?``}oIJlQ=cp-2F-=L4 z=H-%<8T7uSQ}Wf1yy`y#)p5wbi6_7z_w^@e{s6O#A<6;?E{n1AL_5GHYGX%R+DihCL8A=VAdu9002+|000pH0000000000004l2H Date: Mon, 17 Feb 2025 23:32:13 +0000 Subject: [PATCH 111/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 9754 -> 9756 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 3f3bf6883..0378fef32 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-17 09:07:08,447 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-17 09:07:08,447 submission_checker1.py:3270 INFO] --- -[2025-02-17 09:07:08,447 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-17 23:32:07,809 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-17 23:32:07,809 submission_checker1.py:3270 INFO] --- +[2025-02-17 23:32:07,809 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-17 09:07:08,447 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-17 23:32:07,809 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-17 09:07:08,447 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-17 23:32:07,809 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-17 09:07:08,447 submission_checker1.py:3286 INFO] --- -[2025-02-17 09:07:08,447 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-17 09:07:08,447 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-17 09:07:08,447 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-17 09:07:08,447 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-17 09:07:08,447 submission_checker1.py:3307 INFO] --- -[2025-02-17 09:07:08,448 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-17 09:07:09,577 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-17 23:32:07,809 submission_checker1.py:3286 INFO] --- +[2025-02-17 23:32:07,809 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-17 23:32:07,809 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-17 23:32:07,809 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-17 23:32:07,809 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-17 23:32:07,809 submission_checker1.py:3307 INFO] --- +[2025-02-17 23:32:07,809 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-17 23:32:08,980 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,13 +27,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.20591 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432377 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.3541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436249 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.123 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436249 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2876 | acc: 77.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2652 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430312 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3e40c3898..c128405c9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -738 +739 diff --git a/summary.csv b/summary.csv index 639d635c1..4a5174d83 100644 --- a/summary.csv +++ b/summary.csv @@ -5,13 +5,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.20591,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432377,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.3541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436249,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.123,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436249,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2876,acc: 77.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2652,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430312,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a07f6a29c..0c38efbfd 100644 --- a/summary.json +++ b/summary.json @@ -184,12 +184,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.3541, + "Result":23.123, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -214,7 +214,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2876, - "Accuracy":"acc: 77.200", + "Result":10.2652, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index e0212e5346e959967b6578996307af9ff29f6881..0b870f60b7752e8123d0e7316ad4f140e5459a2a 100644 GIT binary patch delta 3926 zcmV-c52^5)Oq@)xSqKSyMX(h~2><~6lVS)@f6gOeXJ?nxyGz{nCwt92@x#6lxlKS(ZKaJ+Htv<;_{k8L-!-pvTJ(){dq^(ZUbaBvZCc#9`{6vcua-JO-Me)o}v$yfMnJi-I4_866Y17bkyE*g2xw@|5 ze;}?>M>HCRf&3T+%bA>~>q=rd_0z2T$s}APYUgkmX2XA4&`OOe&vvas= z`!g$s0*CjVxLeFbyB|F(#Fxw-GRp9P6am*E@n61QIg3-+-irFE<-s zZZ@mHBFP+ciA~1-giB14??)gZ`F;WtlJ93OaY-#-o+aQT@hfP7TqCC?;oawB&r{SH zE$HzEJTjc;G^WpE=$vx_swJo3@}=LS&e#85e!A+N_Rrq0Rzpd)92$q|e`I;51<|ZI zn`VD!*-|HawUF^>5Y9(3&K7KPkjf+-B+I*5nB;4pWN>e3dNYlsT&=R2`C_Vh-aFaE z$1=&*FiCT>m}zWl9t4X;BUmgO(R}_GFXw5tnrduRgCLrZ!m;{`I+Up&P80RRVRHoL z>u+_Wl&xB2*0t5YJ^lK=f9{u9uQ9Zishfol#vSM!44uQFBbVASm)PV=J>e1t9gtAy zfP_K^B;+Ifi&Fv}4|geaCE;BVPCk*(is2-`9_T#z%s+Q1kvxX3-!8bsBa%y}Mk2?P zfgj}eGmY%_WnEQcobkALN~1WK4DJoh*7QcO6h^B8iFKq>fy5i9e?Vf((M(r2F^Db( z(d8hL>*ttDY;ygaa0vquNGOOvLO}!)3Zh#AA`fREmV|dfI0exy0TFbbg6L8Zc}$DM zTyTj;B$w|04v2N6QUSzUcV|j8E!PrzZhJgTqQ^-jSN}1W*!0GSOBjhjLP-P?N+OrI z^v1_4ArXX=i}ti6f4u7vPR?jSk2k>9p){rsV(2;)NXRL;^y-Ce<_A&InEI<49$ouq zuSx4juL6KqUI9Q?^E_+ML@0F3->RGdNamJfpm9h)PJo6Xh%}Tzq@fT3jZ5dltp;I; zDxe&eRnXlY+>*#jdY}=H)N{_F`XGuvX1o9zp_3DGS?K@Ue*p4j9?W4JO1a3c$2gIL zd!f16RQQ*XtR=$=OkQs~wppHPhHe#3{4SOFLHm*557sSZ2$+M-euml>0CD(8)jWR4j z1q{oQ3cA~aFIdu&9%zJPNu&B8iakOF8lelyrSX5jWgR(Id91qW*!CvUFztO@;yuwL z!M7SMhjPg^j!C)X8qOuqFfM_Hamh8DOS6niPyyqzf24x$_TUSbw4{eJp-*pER3Aif zF2P1N=hA%BT#|=5e;vs>(yN4|xr?ML>^(aj0&m_Q3BDn2(G6{&aat5fuHi@m4MP%W z7?ND$S`^7v8Iqt8`lPk2g6{U<3zD>?2O9BAiX@HdqbPh07?D;P zk)Q%bWJv|x?ZFo!X-N+>!k)Uas6L8fL;?-Fr?lP>kpIM^*IE*+#H3YV(l9k$FI=3v zOzv^v2;i}yHXNXFs1FBdIFl8eI7q{o1RBO48T0o(Qw$fGEBv!Zlz|q-JXlr}F1-io8+F&b26dc5Kb? zti4Zpf80G-0!t2c?EsBKZ9722v8-U+K^lf7FwfW&%QEi{YJ_1~=H3z1Tyw@sdY}>h z^p%-(P!yXifrc$$Yxe29M&-|W`uav4JXfM<;mH=U!r12^@b+K{{5jOZ12hix@Bj_R zvVw~TX&9Ekh{LcfGxDHD7?x#L9>KgktfU7Tf8kg%I}eEBYgn!UYuK7CJ^v1uZ`9cH zT@;-?`8w7%_I;zs&@q_?_lell+XFN%_4WV_BNJ&TnMgy)L>jt|HOjm_r~*o6Sp{8# zd3#t%4>dx!MOaiHL@_dfhLdU3?Cp7t%)h%YQcHeSpfvPdD0Qds1&&=N#U3WPOT?zm ze;%N5sj~-YIFl8eJxIft1pT42rBUYWL5(mb%bYz1bM~;39%#gCQ6`zQ2SqU^D>!?M znw>qbA(@VH8pRLYQ`VAOm9?u;bo6BNzvr&nWKoi@54G#7W8K#f7c`G(%5pf=?tyDxMt==C7f2q>`Jr|zL~&r6JiLXB za)T9+OG(Vij0%7GZ@D&kQ38$lEr7^UlUSB(W;Dq)6tXH8hZ=Dl4OMyaZw2Emf~|oz z8uDZ`6r(d1K{_i))CwPhD^oF+N6;Vbq7a&9W@vq{G#rK@N*L7%`_8%Te~3LRBZB0je@$1G*#SGZT) zuU1(i1e9&T(H%`vx=PYSsE6*kfUJ$4%kV09o|h~H&Y5Dr_*-ENLr9^dLRF@U@+FJK zPg*LmJ7+?@#5ISgGx?ITDru-VH~}HOSL}?XjW*^OmTp-V?!ie9K4{|+6;zg@%!FAN zzIk>!W?6s6WF4$;moyZzt@ILNC}mfH9u}2e+{;LXx63F7^<*y7)Rhzg*6zi{<6f+5B``pSD!GN7Dmr z*u-~#n|2J}C-!g4#5Wud+ntuNLktU{HP8GXGBMj0mat{jaJJeMJ7~uxr0@WFw&7c9 zIDmiWpai4@k>s>Pu!2%_wAZ>2|8&6e`IE(^M@bx4^)sn&)Yph+MG76WzZn~L5AO8f zN;NjP*Ms{#xZ=we$4USjyk{KZo^gnkaoFHJ{^A~gagV>a$6wszFIN18XbtW8)l4Hj zOT#U)42$AnY_UAwlyWX487MQWua>$Px~6~eTAjvWg^6wURNb5HzMyP-I5HYCzB}As z8L7G#9i_xxO7`bV5&;yvpe2Xp9X5RRG3n{$|HroDR!v7N1AnZ0FCu(hY;9VHsH>Mxh2_0-Au8Dte zN9y__*{F3aue&XdI(FBNDe&A1{G7atW+uN5(oc)e=muC`|NiuJVh7GmTd~TeFY=2y zQa<~>%_g%WbOOgR6Mz<1H^{aEB@#Qup$6%(G0?U#_MAmcVYO8r2Nh}tl_HN(le+G` zQ%zlVG(SLgU^s!YAw8|q#2}8pv!!UehU>cC&MHj|$G7CJeKI{?o=(pt;wpR9_;fa3 zUXCvo^?vQjt=T-1NoVhcoTpnZ_{epEus|saF ziXu=(X;FTh0sq#R+ULVNEG!Pv1q zNa~W{P(j>jE2$5qZGW(#?3*@#^E7?H5mnSkrOvOziFX3{(=q5XZ|6mZKHyOZK!&)Z@>x8;Y$aBx&eQxpPV+c>+~fr* za*{5#r;#`P7oa>3`8RO{47qPVLH!4_j3LSb34BGc6-fyI0Q{3*Bv%2-leZ)y8>IJQ zkzoV?0CW-n02BZK00000000000Dys1lk_A(0q2u8B{~6FlW`>`8-WdKnf~0gjXLB_jqvBLDyZ08&;#aR2}S delta 3939 zcmV-p51jCvOqxuvSqKSDRUJ!32><~7lVS)@f1V>@XJ?nxyGETy1-d2UCtBRZl(J-8k+v?5r=Z32)N$Sss{xq7)w)!L!_1DgS4j-cU_hcev zs$>b~Nn4$y>EfW-OoEA=`H2=S>RH8 zz76F=QutfROSy}p-}4uz!?vpD(k5s>uZD3MD?>T*m(%nrdiW~C@g&Voo0W$I(KK28 ztIWdeTpFq}^B>!)sVWcQFrDOyl}R`pf6Dwst4gp;(rEVE`jFTpSceLG6DxGFjrKIr z(rv@a-;bxM+{853y{<<1(NF!(VH`au@v4Vp;pgv+I53Rtv;=wUx4bzYTXWNJ_1vIj zb>wI7zGEIX@AJeP#djKVO&e}#!)>9VnT=;VD1#rA0b|g2$8d#;R*MXikyyKue``i! z?@DeMiL)!YWhCxavQw_fmuo@-LCQr18Rz#`71ysSZeCT~mML~BZSj@1Al@LAqN3%I zQeT%`zb?5!34LEnlk8M$vDF?~kigyT;usZVm?x+ZgoTfcgj_$L7zsJ?pBV`O{|h4_ z(0yej1iDj3LN2o|BO!0j86zPlf49d-$k{zdi9yb8pOKKWd%;M^*}Y^WSTW{WIP&#^O20R1)ChCGD!!?@@^I;`PwHL+*_L7Ok*imtE^_em};K) zPB!teOtLjh(%dX&8r#f+V6kWfi)AC4&mZIEJk3^9jg4v$MDtNNR)0~4GWEl0qJB7R zj+puSTOB23t5%r}ZS~)te{p?Zk4voA7+TBJ&B6!c4s;F=orBPkOYImXHn~zyPy)~~ z5(*t7q0liB@)7>UDS-~dT?$=Ec$Wz$pU7v$a8h5789e#SKX)jR7(>@@7bx+FBfEWBr)rEd9v4q(6bF;Ry`kBf-Uyb$XjLGwj#4U+e|X~*NNhQp>FOpA zqRWHmB8cSrIYx<1uAdW>0Emo)g2+fHh>V1S=$3$p;Q(Suc$W#MAi5HhD4SVt)pK)iK#rbN?nEwSge$Ky%#kVJCzAEU&kH$Er<5*Z04k&#dm zQR31YAFqT&CY)Tff2SqkU5{{bMjLwA0jxu5OrMFN>rh5Q-hxZ7Uf5=S5G9SNzsm6F z+CO_uT1R;m0KD=F0J@szS$ifzq4WH$+6jPUZaHQ&4jIP@qX7gt4P}tiPzV`~OXtI_ zhQp9cKshW+(A^%~k_at5rW23kIftk|6Gb00UN9P=lQ-nDe=z>H0p!a(n8P@fa*y-qsA&zf8}OiV)Fo4vmA5J){Ee`nvMkS-dHxp5;cxNu|y4G$!GveMgv%)2C+2C zuw)VdmL&;>e=KOUe>nN`hlIAXwuCVv)bO?U){z%}4xJ5U#8I9ATNTLRjWHf*z zqX8sQ<60ESRvD5^C-g~cS%U8N;0uzprN?yQnG{JH)#suBNz@3PZcZ1*Ukj4UyKEv{ zZkQxIf2Q}d)m)iRlETxBA0V=h>MAW>TgAnz=9uPQBKH_f1bPN6j}nO*$D%}{28m=e zKqR99A{h-3X_XPlBmg2y5_Go*Ux=hFJ*E?Q>IzYPE((ZbG;mL8y&)j~iAS%s6j+H# ztH7jTYPw#yICq)cz@HIE)6Ff2`od;WWS`qX8z%%s5;EV6x1PBfyUX zEj^|aWD+bnOcX9$QNt`;>$K(kD^R{wla8P+9m8~-Jyh=4bo3W0850im=`b3H8g&>A zQdz;O!)bs@MgvrqxplY%KxLU8MO46PH;~IY&lF6?unp=*%PVRnuApS{7sQJ ze`?ve7DdmFtr?!R_bHFN2TR71LtQ(J#-X+yMuS*ZFz#>~z>?7bmSx@@E&*U!=H3z1 zTyq93J*E?U`U)l;E(#_~Mgt4jnteL2QTcP8zP?ch&y^@zc(O&TX|?ukR(gB5WDGjg z!^3DCYT{uu$Yli^52pbx84Yk*=H%fLe*l+dULFBv9%$(?ogkOs=V7975sMmT5nHpT z=idSIjXHb2i=wqBU&z|VzONJ+I!~s7MiHBudl-#N%{`0;$mBGXOin||W;N%f3`Hl z9wyNyVpD4mqj9OVhtVLD6|6m+2AE{VV^b!}tUX*Oz+{=V#{g>&wDg!xycT5=tUX*5 zFj>LcW7KTzc@4>Ql+!4F=%%uk+NvyGjiRL|oBurzzD?aMe7k)nC>%J=VrclN=% z!2a-yVti2AN>=)_G^=jaOa2Eptj^)WAIoz;9*6TpnaWXCsIF!A-|_kvBnMW1r_o~d zm!&FqQJO}x=T{Ty4`rOcGF4?1Md|a4{FH2exwFR+n*jw&tpNCAlV=koe>xC;Kfl7t zdD(r)#cr;sbcNc2s^wA@Sg}u)n@md1CUH*euw{S!acE$>aZ=y_5>mxC_I#etGvmqd z%O*?F9h5v_`JsjlOGA){EK2hAp>}D2L=o1)ge*;Oa&`}j7;&XfD zn5C@z3inF*)hbJbfU<2kx}!--S4o-(b<;f;khRfsDX((odC5ZHoGJc`zZJ$XgcM3D z)WuX;e92<*la`8h=S--FxaJUbUcRKPN*XEwPC&@$6+dI?qm4O+rCXMTdvMZ&588M{ z1(m5NZ^En--#j}Vf3qxOG7r|bOBxDUR(c6Bl(MQo4~t4K?xj-U?edNou5=nfOl5K| zLzc(M+T5Fz?@hKZ$@g+qD_NzLo>F;hV!83%ep*|lE!R{jM!L=06rOn6D|!u2Tib>w zQ92At1WJybP0-^07vv~FUf@_a2)p{2m4JF{quw%|e;*^ye{%=OwVzquh=jIxv&U+y z)-DMb7uP3Ke1KeRcoxA3JpkkY4Ll=oy`TY3B@h?`7kda@U3{IMUoPj<#q#p#Y<@be zcUvmmqw4`SY~s7WT|0*F6Z^Mk;v0^K?M~0wA%=y}nrHqGc`@4(mat{jaJJeMJ7~ut zr13yXYsmLQe~|-t4vHWph$N>Sh82{eqrBFI_@@Jw&!0>#Jxb!Zs-H=HqrOHoD^lp_ z{Y~4jdvK=*SF*9ey&l}}!IfCHI93E~@Sb*vd)gsZ+F^tD#EX04#Xa%jo_KLjyjY1B zqBXSpSJRF3EDg8FGAxRRw#D*%Q_8-OWT5n{zFg{Jf9UGQYqcAP6(+XXlXY+Q`+~IX z;mByn_-=E5rBZb*I!cPYl8HhKbOS7}e}8&9u>%_g%WbOOinCIBt2ZjfySiV{1?p^Efa7-(A8x?9A zm12)kb9LQwr+RhS(fk0}f#C$og!D8@6N5PZf6kQd8m{YlJEJr)9N&_&_Q~{oc{)9p zh^x$1fdn><>H{0i`irNPEg{g?1rWim~TAP|Ro1p`*AlPO%V6w|~W!@o%;TToCer6$B<& zV7LdSb|j)BQt%OJe_S&F1jo>T0v$ZgvmBqj0i5zT8wTaVsbuIZ+_av=(Y^O_G`4M< zwUfab^XGWIe0?8QO{MNn#R1i&;FLRH!MM7_mqk2W!+coA6K^Dlr!ML@-j0hL`amQx z06oM#DW8+)mnGE#)deZ;Q&Ob)bVX*AOevXt^&_wP&p>q?@^9h^aL9fA3F<$yjv>kd z2~AZUOGOC)0R59;Bv%2(ler`!1xu{}_+yjcBozWs6O(Wg7LyGnDgot_I3+p(SCexk xCL3h@HxwlS002+|000pH0000000000004l2IFqy`J^_rA^Ccq&J|h4C0051;P=Wve diff --git a/summary_results.json b/summary_results.json index 2e0a1d579..d44c16d59 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 77.200", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2876, + "Performance_Result": 10.2652, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.3541, + "Performance_Result": 23.123, "Performance_Units": "Samples/s" }, { @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", From ca7761e2c3f5b56d6412439ab1df3b7ee0a6b864 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 11:54:55 +0000 Subject: [PATCH 112/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9756 -> 9757 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 0378fef32..7fd2f9030 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-17 23:32:07,809 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-17 23:32:07,809 submission_checker1.py:3270 INFO] --- -[2025-02-17 23:32:07,809 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 11:54:49,476 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 11:54:49,476 submission_checker1.py:3270 INFO] --- +[2025-02-18 11:54:49,477 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-17 23:32:07,809 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 11:54:49,477 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-17 23:32:07,809 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 11:54:49,477 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-17 23:32:07,809 submission_checker1.py:3286 INFO] --- -[2025-02-17 23:32:07,809 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-17 23:32:07,809 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-17 23:32:07,809 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-17 23:32:07,809 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-17 23:32:07,809 submission_checker1.py:3307 INFO] --- -[2025-02-17 23:32:07,809 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-17 23:32:08,980 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 11:54:49,477 submission_checker1.py:3286 INFO] --- +[2025-02-18 11:54:49,477 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 11:54:49,477 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 11:54:49,477 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 11:54:49,477 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 11:54:49,477 submission_checker1.py:3307 INFO] --- +[2025-02-18 11:54:49,477 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 11:54:50,800 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2652 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0619 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430312 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c128405c9..278c95299 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -739 +740 diff --git a/summary.csv b/summary.csv index 4a5174d83..1589730b0 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2652,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0619,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430312,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0c38efbfd..1c2485d7c 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2652, - "Accuracy":"acc: 74.800", + "Result":10.0619, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0b870f60b7752e8123d0e7316ad4f140e5459a2a..33d2c65f7ab2abc123991b58b1c12a4425be1923 100644 GIT binary patch delta 910 zcmV;919AMEOr1=yViN_rg^*QZvu6{T0)LJn-w!1mz;jRnQi4cw+96m$DLUF~U5I}= zVEO#X;?koej;s2a)HmvDM6)7=j@jRg4Z8<-dT^y08{F%`{T^KLWs74afDPU=4sp*o z#L75q@E(70kH5IbU)&30c+i|O=BbI?b*1Z=IzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8 zEZ@xbhB$tF8$3t%{(bN+jPD!M4}a{04mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$ z-bFK$UkB-@#bn)CTY(aZ zo#Ie~^w=0^+ZcP!qNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rLZ= z1QY-O00;m806zdBon9SB0<-%UO$Y?CNF2VCOdubB&1%Ci5QOi23ZuI%%c)BcwgYX+ zsewXMT6z=GCK0uMkkmSU`zns(B#={&tC{_Fc2Tw1t2X%s=cUm#B3X_S(A)^Aw>5fS zFVi_nd|+Cz)@Z0v4<0R=yNX-Nje}QbEVv-SClRDQ<+et<5G=*m^BpMWv*^%K+!&`= zh^5;b^r*Dp#=pxcuX>a&)xt|`I`-c za^X}m^cHSfPvYd>dpQ}quFJaFV2$|+{#?Dh4XdV7A5O&qHI?9$J7B@Mroxv+JY36s zSj7`>B#6g8>NnoTMUH(Sk{Ey<aECYY6doLnXp2eQWe@(@ z6aWAK2mk;8KL8<}UL8jQ0JHrUOb7&l4QiW{O&}kC%WA_g5JmU>3ZuO(KWH0-?Lb?y zYM{`Rmh6NyaYU^Ll3FK!U&W4{1ib2Ubnl$GGpJl1WE=g0bwaBOWl4%6P)u{7nhJev zR`CKw&Qrx{s})q}2o5dlyOJ5gw1qdT4OlP0MFFH7VWvWR?+wA&u{}uYlHgE5+-WPR z52bB?u%YamHh}XqeZUet<)+{qBD}S32A<8+XFbm(1HZz9nO*rU%Vcybh8b; za^;l59}Ub}M{#l=oS2MV*Ck!iTSNW~f308N`&DD1PN!mk>XI|UEKsj)UE=E^9ImC# zufvIV0{GK0=reEUMTS1$Q3ybWxTEq}JilxuR;bR?{60?eID6dW1u1fpF1DwUH~kl& zJP-LdaRdyxZ$Cl(2eXeMO#=m__hOM@li?&+0q2uBB{~6FlXN8}1c41|o0Fa;76Cew TwIx0Qj+68yBL+Yt00000d61>; diff --git a/summary_results.json b/summary_results.json index d44c16d59..5e467795b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2652, + "Performance_Result": 10.0619, "Performance_Units": "Samples/s" }, { From 516ae6384ada1cd01ee0ff861b6d6c4f8faf3438 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 13:39:59 +0000 Subject: [PATCH 113/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9757 -> 9758 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 7fd2f9030..ac2be2b08 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 11:54:49,476 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 11:54:49,476 submission_checker1.py:3270 INFO] --- -[2025-02-18 11:54:49,477 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 13:39:54,192 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 13:39:54,192 submission_checker1.py:3270 INFO] --- +[2025-02-18 13:39:54,192 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 11:54:49,477 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 13:39:54,192 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 11:54:49,477 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 13:39:54,193 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 11:54:49,477 submission_checker1.py:3286 INFO] --- -[2025-02-18 11:54:49,477 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 11:54:49,477 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 11:54:49,477 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 11:54:49,477 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 11:54:49,477 submission_checker1.py:3307 INFO] --- -[2025-02-18 11:54:49,477 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 11:54:50,800 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 13:39:54,193 submission_checker1.py:3286 INFO] --- +[2025-02-18 13:39:54,193 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 13:39:54,193 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 13:39:54,193 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 13:39:54,193 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 13:39:54,193 submission_checker1.py:3307 INFO] --- +[2025-02-18 13:39:54,193 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 13:39:55,175 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,8 +23,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73362 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.20591 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69558 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432377 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.123 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 278c95299..5588794c6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -740 +741 diff --git a/summary.csv b/summary.csv index 1589730b0..0a0ad86ef 100644 --- a/summary.csv +++ b/summary.csv @@ -1,8 +1,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73362,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.20591,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69558,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432377,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.123,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1c2485d7c..4f531b72b 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73362, + "Result":1.69558, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -99,7 +99,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":4.20591, + "Result":7.26681, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 33d2c65f7ab2abc123991b58b1c12a4425be1923..f106f82103f573455bbd7c934f3091bed4408600 100644 GIT binary patch delta 2672 zcmV-$3Xk=jOrA`zSqKC*S+-@9T?k!&W@7W4(|wSi-O!I7{`xqZD)%x@!f4)Bg{G^D zoCnb`oR8b;&GqMot13zA&xigrn#;EOBop=5&VLRcqWJe@B4w&%3Fb*#ouujFpxI1< ziJbX~7A@pFJ2HymnV)7a<8d=t#L^$Gf@agEq3d>Y=7)22o#7y^k|P?8!a#n1jDqD% z&eL_0SWf*k8-6kg7m3T)4+)}avietQTd$rCG+a5$9tiB^?hnWWKw?6>tHu}QEF74{}p=wch~X`-duhLyh`PgA*xX|8)+ zjqszN`kli#dQjq356QyM-x+aW7};qF^44#8b3V4_rs3+jLCNaK&)$8TrUR~0v}DsIaZJC(L@r7ae3kV?6t<&jcfmt4Otx#1G}zLqB0 zsn&wk9$74byW7PvSCCHa9(BmC=8f|yKYPDg4JFxfXdI@K<((Epv*v7nn*BS=mO9yA3mK0F z;d~_HY{4c6sZ7#Avb>vxNxt?;2KSbxH`7?k)her*FQ%I3y^~FRER$>vlQcJrnZ`Er zAXqFK!D88n=JUsRIZw0IRAZwW1krpHj@4h(p-la7ny4QRn@*O+=0%)&^a7Ba;Y72iA}E56E0!U0SSc;NGNnbLO#O3I3>{W zaF;??65a*j-+v8ypJx(II`j5H9rZ+xZ!bk)XN+OU@61l{sH$Gkoi6ER@w5KKE zU5{{bMjLv6yaTolr7?XFL)W1|Lf(Q)uU^<@eh?*%slUqb=-NMfOSx1dk9;YB@d6@Ipk*uSv*-m}vo@aFxI;2Yu=-OvUar$v$E z8jd8;FeHJ7A;~qaMUiZkAqhI6Pg=_obhigzkfbd=(1~YKBxzJ1MKL6~M(A{Nx-kA) zkX+tn6X9~hB;hf=pRMM~e3BHNX8Zt=byQbr@!BdbUNzgZtbLQ@9)pSCo&n3FL~@N| zQ6jm2h7$=ij7XqiL;?*X(kdeoBw$3AB-9=^gt(^NoL6bQGDUbHDKXdr!D7yUxD(insfwp=@_Qt?4feUrlY@52~0TD zrvo$&HR=Ejr?P@o2Wc3UK*OjkbL*f4jLI^e>MshuU_4hGSX5xPvqdOJJU{DVAk_ z-W}8l!?Mi1BdEFNjJ5PYC;aIvGwGlxHdz7NEnwN+WW8bwD>HvfAb`!aR2@a6WI zjK}gsPN#_yMA_+DM{Zl!-}t{02e<1l$G^z4{-XIX%f-q{E50{g=+it#~dD_QB!(yY2wFZmy6Se?U#KbGfyJPzlHGL@sOP+iMz zzvJ}}4y^u8qs8hkOI7ZoG>vA@uO`wT$~b>zs>&#e(&rcXDcSyTL+4Mk$`OAK1!vPN zS0@0hlTEq|8hO~c=leB3ob;WXg z@0__aa=F;cHu?qYgjN;Gk`zUtnC3z?75Z2&<2j0)r;5{7E2z)`99q$07eFo&aO+>rYVs0ke-G$^ry6S+-@f eT_imL1!vPNS02zdp{U%Ds$}Fq*ejq3Nn3 z=Rq_K=i|0|bN#vDs!Edj^PxYD=CZ9m$wd9N^Pj_qDE>W}NSP{G1@ok>PSSL7&}=5b zM9%y~ixzU89T`RO%ulno@wk~RV(AZ8L9=Po&~>{x^TWBiuHhiAQb#lzg@OEk7zN9j zoTuwbVmbBGtoz9%TqJ7ea2RIC@_dw7j@s&xI5_#u5Nfk?xN7@0ln+VawUVcD7e&A4 zZ%&78RnMhOrv1F?#$~Jw<;Y)7)2ry=s|?4JG#fT64+)}avihsc!fY%JRhjvZZPiqj zhj5rq@`{y7I2_7+pj9PUCTTQ(`)z$lY${lX3VRbPbg_;0G||#+!^+Rc(^PI^n(J0q zJ^bjWe&;Zb9+Y_1LbCAlGb0WRBO8_=Fa4Gm=Y4B#3a*|Dlq`>Y^zJ+6Ve>w(c%%4E zL#}DT4K27W6g0E`Y&&J(P8qBZ`feYtxT4h}-2@VAS8@#`_O9dxNSs}N$t{q$Tggtk zCQjF66$p|pSCD>we^qh)s^aEV#ci2lC({e-j5(3>V@q=hdx8((fVtH2`39CL|H#{Ps$ zOp@Yet_-mg|eNwypshv{T_rv=fhIh$sGe`ncJCwsM!@n{gv zM>5V9Y;uswBpoEnyIGjzYoBCrZ)tinjip?zvYPo~s(Ic!*~G^($<{DQbF-LfY-=6_ zi$x_T@X$_kc{G%h60%H!+BRE(X!%Ad>6nm`iMO{hV+K z0})6lh(JO?1QH6OTLK~vXCRh@cR@G>(JcWHbe@9fQV@Aei^N=TiAN-t?*9&mb)-@O z#9Mb~N;ECk5_@iYJWQg;NhDYQF_+l%#)nH7i9kY01QJRjm$>xC$15Qbgp-T*v?RRi z5l+r%L60|oz}BHOrVnE1IuuCADY*3Ng>B{sQPPqxHxfLC4tKv(lT zYtKX|bj;tXoB&AXmSdoCNIy=1h9QVFltHAS5CV-$=fkZAVTdZA9F|qk-5%VM$Vz&k z5s%b!&Z7Dtiaut%02-l_6LMMT|JwlaWgg6797?%=$gamYk%N1ox!F|smyxU`!wO7Z zZ#uSFo@$0}6;AvvmH0vXk>C&3EsIjgHIATEf(P`BN}yp>0u7^*YdDpnj7m@eqq3xe zuAo~9XRM?L8u1KDC5`H%C^|^(aj0&m_Q3BDn2(G6{&aat5f zuHi@m4MP%W7?ND$S`^7v8Iqt8`lPk2g6{U<3zD>?2O9BAiX@HdqbP1>Nnz7b0m%4>ZD_y0WM~ief|p4ZEkb-Vl)g z#G}_*60F3eRbbLEHC-=UoV!f!ap4Hyv7t5`pmC@V2WU8x6`VLo!>ud#K#8>F6(10uv7P z=>UyGjXFTXsjOhtK^jIS&@d{?+&ZWNMrD~_M^HP>87t|5M);lxv*my&z9+&pU{9oG zYYwOK=WmLAXhe&w2X#Mjbp?qG;jC7O}$E z=OFO*U(*rp_LqajCNhXgHG_Lq%Cd-_EJqC03 zu#z5V#A{I|nX?B)F(xZGdyJZ$J+C2|j&d5s58YGNl3SIvt5J0HWb?o0u`g3M3tw)Z z$#^VJJ~6IJjMZIsQeSwHM8YVQ%O5r|Ylsr{R38B!8@bB*4yp zBtXwA3r1JVsZ`RZg>0+xagm*Wp*&et89t8IKM$ZR;xLNC^l9CC{`&;Wso$AJ!}qIB zE$87MOZom?@XkJX7uX+uQH&2tTgg&?mS)+lTFGCaZgmb9{#c&-@i?3(%2bZBM0G8@ z{f^f^IIw!1MvK)eOI7ZoG>vA@?#s>&#e(&sn%kZgarq4OuR$`OAK1+qvS zz9o}RARd3sYQr!PgztR{qq{B3sY?*H18vEvfkIPSdK1zn5w(7h)H;6qDvskMkW-JV znf-QlQMK5sHu(kTrO`DaS&kCW+z6?+HF{q!(>Y3fU|O)&XsA&S9xa-?id)K!gI8xP zxFEqN5u`okwnn=UEXCOK9Vq6r=+IHz7^hf>rQ3gE%lJ3j0xt6W0V@bhu)y#EnA(Ym zMx@{q(*C%100@qu1qC{IOfrJc-T+Sdn+=0<;Z!p87H(Qk;^f|YIT^dI%evWMjrj@w zT)n&vtEN&PPQ?K=mEe>+V8OVi!k0xnT+4h|#S?ENh{rzaH{QlYj(s4K7=Rw*o|KQ- z)AJ>IiJBrW$}}%hGG7xy%X?bpUx$%b{b!&Why0s(0vvN+e}d)@vydUm0t9?Tuobgj dBs~EIvPc}hC6k~fQ~{5Z^(7+)K_dVF001qo27&+p diff --git a/summary_results.json b/summary_results.json index 5e467795b..14ae30b5a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 4.20591, + "Performance_Result": 7.26681, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73362, + "Performance_Result": 1.69558, "Performance_Units": "Samples/s" }, { From 594249ea06b46cffe7708117bb405be83a0831eb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 13:44:05 +0000 Subject: [PATCH 114/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9758 -> 9758 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ac2be2b08..335d26a47 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 13:39:54,192 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 13:39:54,192 submission_checker1.py:3270 INFO] --- -[2025-02-18 13:39:54,192 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 13:44:00,884 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 13:44:00,884 submission_checker1.py:3270 INFO] --- +[2025-02-18 13:44:00,884 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 13:39:54,192 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 13:44:00,884 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 13:39:54,193 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 13:44:00,884 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 13:39:54,193 submission_checker1.py:3286 INFO] --- -[2025-02-18 13:39:54,193 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 13:39:54,193 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 13:39:54,193 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 13:39:54,193 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 13:39:54,193 submission_checker1.py:3307 INFO] --- -[2025-02-18 13:39:54,193 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 13:39:55,175 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 13:44:00,884 submission_checker1.py:3286 INFO] --- +[2025-02-18 13:44:00,884 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 13:44:00,884 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 13:44:00,884 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 13:44:00,884 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 13:44:00,884 submission_checker1.py:3307 INFO] --- +[2025-02-18 13:44:00,884 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 13:44:01,869 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,13 +27,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432377 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.123 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436249 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0619 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2272 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430312 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5588794c6..3bee760db 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -741 +742 diff --git a/summary.csv b/summary.csv index 0a0ad86ef..cf0d0c86d 100644 --- a/summary.csv +++ b/summary.csv @@ -5,13 +5,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432377,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.123,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436249,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0619,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2272,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430312,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4f531b72b..623ebccaa 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.123, + "Result":23.1816, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0619, - "Accuracy":"acc: 75.200", + "Result":10.2272, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index f106f82103f573455bbd7c934f3091bed4408600..c978d47a214e7524466c206b96e564fa3273866a 100644 GIT binary patch delta 1421 zcmV;81#<~7K>z?40Fy8z6|;N@g$I8t_Iyo4czd`61|90* z0UC#zcz}j;S;5AGG>l8 >^PIeAbgjLR}Fk6>mV*3tu=a4wmj2So8jEZ2ZVY|WmY ze+SGr>g@S0iq@WdA!{4^zEWi9m`sBkMQm#B0UDQ@dw_{wo1`o-7z$EvHgR zpBA#M%Ev|a4&}wF%J6Zt{(S&t5r-8SjzYBf_L`8 zyTJbNi(*cEP})jX`m;2vZq-Zv2O3uAaN&>TxgU?id7@0^C@WOgvit9N{fh&uztd>3 z`pZ(4yC_Yg+4HN3^oKIeUzw^hilX%SMSe=QzueII6SK+@m;nW(_hOM@vu6{T0)MvQ zTWUCf=b!|n1d-&lL$HEUbhOvH5dU<*^7)g+rAJ8|SM@WgZ`9X_WaECYY6doLnXp2eQWe@(@6aWAK2mk;8KL8<}UL8jQ0JHrUOb7+(k2(J( zlTIKWf6Z#cFc5_AeF~$yEz3?z5Viwt$*F-tQ(Afx(k2nLevs5Ue)}qp<0Oz%kE@ye zc6L#<*sC`A1?Q#FH6mG#642ZTskb$HUoX=+N_=2iu-0g(Q4bz1n!Acy%8i3pXDqlN z!6y-|AWyAUkJ*z+AI=CkO~QQR1(Scs+De`3q{H`@X(^85iS2u!fR@Bx_GiHJs| z;1kmRxOM;tj-dqwI(SSng3sOnPWhV+gL2_iGV~U1T2JES-g`M2yROT+a4B^3cYleQ&j=5MQQ delta 1415 zcmV;21$g?NOrA`zSqKg_S+-?K2><~6K>z?40Fy8z6|;N@g$IAJ&q3hr!4mj$sD%e; z9O~f#8jfWJ7Z1`fEP)Y+VOeJ6L7gxx%d9+td3jh%4|Kw@WOg19#n-T01J48qX7G;t-dr%Z( zvVyb6sM*=`8j|TKr&0XSJ!LJmRav_lMMqCI|9c+$GIf8m@a6WIjK}gsPN#_yMA_+D zM{Zl!-}t{02e<1l$G^z4{-XIX%vA@uO`wT$~b>zs>&#e(&rcXDcSyTL+4Mk%n_IY1-gZhRbsPg6PW^kjv?O< zB^!0K{K?|dqa=>2`kB->>T5)^B886G-;52r2X}gK zr5YRD>%sjVT=8X#V-(2~RQ4jaDunDq4W|6|*6tEMBCfj`#07ZJX$@}6PbzY)Ba z?2l)(#U$>s2mflvaA1%(MpE55KC&#|%=U&jeta7|NB91H@Ggw+8`BSe?1T8HhKbOS7}e}8&9u>)dcSt%R__NE34BcIKZK5FlSe6$UjaUGJU{RP*RcizC7s*-TDF3^Umwd0`yDI! zgP-z9;3z?TuvVYhwPtcIM*jg&O9KQH000080000#03n@T9Y+GQ{TEFL1!vPNS0@0hlTEq|8hO~c=leB3ob;WXg@0__a za=F;cHu?qYgjN;Gk`zUtnC3z?75Z2&<2j0)r;5{7E2z)`99q$07eFo&aO+>rYVs0ke=H$^r>BS+-?K2><~6 zlVc=T0m_rRBqAHSg^*QZ1ONbZ5&!@c00000000000001hfmD>ody5^ diff --git a/summary_results.json b/summary_results.json index 14ae30b5a..574eacb40 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0619, + "Performance_Result": 10.2272, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.123, + "Performance_Result": 23.1816, "Performance_Units": "Samples/s" }, { From ca1c62248d42fbe43fc55c9f35c73bdd20ef76da Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 13:45:13 +0000 Subject: [PATCH 115/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9758 -> 9761 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 335d26a47..a992be8c2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 13:44:00,884 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 13:44:00,884 submission_checker1.py:3270 INFO] --- -[2025-02-18 13:44:00,884 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 13:45:08,923 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 13:45:08,923 submission_checker1.py:3270 INFO] --- +[2025-02-18 13:45:08,923 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 13:44:00,884 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 13:45:08,923 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 13:44:00,884 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 13:45:08,923 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 13:44:00,884 submission_checker1.py:3286 INFO] --- -[2025-02-18 13:44:00,884 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 13:44:00,884 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 13:44:00,884 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 13:44:00,884 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 13:44:00,884 submission_checker1.py:3307 INFO] --- -[2025-02-18 13:44:00,884 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 13:44:01,869 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 13:45:08,923 submission_checker1.py:3286 INFO] --- +[2025-02-18 13:45:08,923 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 13:45:08,923 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 13:45:08,923 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 13:45:08,923 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 13:45:08,923 submission_checker1.py:3307 INFO] --- +[2025-02-18 13:45:08,923 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 13:45:09,944 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,14 +26,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69558 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432377 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436249 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2272 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0106 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430312 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3bee760db..611de2536 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -742 +743 diff --git a/summary.csv b/summary.csv index cf0d0c86d..be01950d1 100644 --- a/summary.csv +++ b/summary.csv @@ -4,14 +4,14 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69558,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432377,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436249,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2272,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0106,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430312,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 623ebccaa..a9d2644a7 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432377, + "Result":0.433237, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2272, - "Accuracy":"acc: 74.800", + "Result":10.0106, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index c978d47a214e7524466c206b96e564fa3273866a..ae430c1d9398f3c43b212fca8fd8035266cdc226 100644 GIT binary patch delta 1589 zcmV-52Fm%KOrcD$SqKHm4xtZBvt9`L27hGtlg=J0cWgTP3zfixLw!0x<4~gx&~Pd% zSapzwQ3*7R$}+bOO2DWr^Xmv|r#WLSJ|PA8Jl8R=G{S^Ff7a5JAZziei%`(69w;%|4yisQfukU*D*M=SmbUJlP^vxMI)OG=#T@OJLBU z9v+}^sEG$?IF}V{JV?X11Wp{rWto!)b;7tT^YRF0=3y;8&~`FTJTU&L|^Sj5)s z>G^lSe51~u@1kh!$rrM=vF|HIhJTL9G`La3rsf`?ajCfnXc(DDL&-!MN+#0Kg{)C# z?m-DCnPmyO1~d1tmLBSa?u)RfK8Rvu0u3kAsM*}}8kv80W2Bb)sz7PzyHM&*;Ts&g zOo}~Ba+`=vtvx{F3d&>!YY);eCM#Hb(0J%_Kxa6VR&Ki0nzVCTOQpy$bg(baM) zmGo&L+p2tAWbaU3tf~wjM}O<_;v#s{UXWTih#v+7p8&30c< zwmlpf4H@4Z?yrng-HVP=VlO58^CgJ@3SQ8X!}1OrzWSK-^z#2>+i|O=BbI?b*1Z=I zzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8EZ@xbhB$tF8$3t%{(bN+jPD!M z4}a{04mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$-bFK$UkB-@#bn)CTY(aZo#Ie~^w=0^+ZcP!qNcFgDvyH- zHG@i#$EZnN_ui?dE<2hZAUiOeK-rLZ=1QY-O00;m806zdBon9SB0<--W zO$Y@8RUi;1lTRQYe@??N5QOhMMdW+pB&DcWNh$&AiAqQmQE{{EZE6w!$Tp;Td)%Z= zTPY`w>zVy_cI9GukWKUp)(NdjlqD&OKrzjQs!Q~-S;Y$!IZqX*jaE>i4jfumcLg(q zX$x;w8?au0ivmbF!c2+w-W!6kV|$R)CBdPBxYJfrA4*$ee?!?ftpVq0`hX>P%BiRL z2#n20L?@&*JDlI58UAwoTeu zZw>h~yj{P(_p8Q29Z$soRRw2+S)g9qs=(JpI9x-WUxyR#1n{RW=o4?JMTS1$Q3ybW zxM$^a_WZJ0C7~)$^Ld=+akkiGGjcyC>EqXNytVq nIssjibtNVR163dpCX=8g6#+hzwk19Rlauu&BL+ny00000D;DpY delta 1603 zcmV-J2E6&9OrA`zSqKHwdf*F7vt9`L27ef)Sd$0tS9O~Ku8i(3;fQDmP!MKAo3`=01u_=~i-W}8l!?Mi1BY&v5 z=8U!UKqvg^D>Lb!C^lIF4O_t0?9+LT%AfP}^^H1su0+wolPzL}EB1U%LwI|*1O^@I z;Q<7h>Oz6guzgD6HO&~P%1n$110k@IvXg&;cKfga+f0sWE=VK-LWBn@ucK#~?dY&v8T`i|l zNuL(7t;)wm_73I6s><+jw15760A&$}Q5>dE>)!JpC|FMY&MX?fUkz$G5C2%o_wRyt z_QAWr{_u-pd{Ej-R{FCvt8Udx{s$UX=WyYV<+&e^!+D}iRk5tH0A| zvHHtWmAfcSquKMTiS&mu&R?0TGK!+~`9*$8w!hra`4dn}0|XQR0JGE)kpTsy_hOM@ zvuYEW0)MvQTWUCf=b!|n1d-&lL$HEUbhOvH5dU<*^7)g+rAJ8|SM@WgZ`9X_WaECYY6doLn< zUFAK)xPK#fE!iK>Xp2eQWe@(@6aWAK2mk;8KL8<}UL8jQ0J0cN z2nFenIsYY-Q6L|GYQr!PgztR{qq{B3PD>EB18vEvfkIPSdK1zn5w(7h)H;6qDvskM zkW-JVnf-QlQMK5sHu(kTrO`DaS&kCW+z6?+HF{q!(>Y3fU|O)&XsA&S9xa-?id)K! zgI8xPxFEqN5u`okwnn=UEXCOK9Vq6r=+IHz7^hf>rQ2eE%lJ3j0xt6W0V@bhu)y#E znA(YmMx@{q(*C%100@qu1qC{IOfrJc-T+Sdn+=0<;Z!p87H(Qk;^f|YIT^dI%er!~ z#{2|-u3p}TRa2=Cr{aK`N^r^@uwYzM;maZ(u4O)~;)ypB#A6@z8*k$x$375A3_y=@ zPs&I6^t@gqqNd1;`!p|7GGCLD&So?xUx$%b{b!&Why0s(0vvN+e}d)@vy&mp0tM50 z;0sHWVkBDu%#*$(A{(UlVv%740049n000yK0000000000004l2R+IZAK>_KLJtaB; zTa$PtCL8IGIsYXA002+|000pH0000000000004l2Jd?R4J^_%E_$4C-LL&eG004sk B0jB@} diff --git a/summary_results.json b/summary_results.json index 574eacb40..b1e0dde68 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2272, + "Performance_Result": 10.0106, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432377, + "Performance_Result": 0.433237, "Performance_Units": "Samples/s" }, { From cd04fa869ee5b5e89907d27523b793fa8753340a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 13:46:23 +0000 Subject: [PATCH 116/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9761 -> 9759 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a992be8c2..951aadbb4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 13:45:08,923 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 13:45:08,923 submission_checker1.py:3270 INFO] --- -[2025-02-18 13:45:08,923 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 13:46:18,663 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 13:46:18,663 submission_checker1.py:3270 INFO] --- +[2025-02-18 13:46:18,663 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 13:45:08,923 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 13:46:18,663 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 13:45:08,923 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 13:46:18,663 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 13:45:08,923 submission_checker1.py:3286 INFO] --- -[2025-02-18 13:45:08,923 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 13:45:08,923 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 13:45:08,923 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 13:45:08,923 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 13:45:08,923 submission_checker1.py:3307 INFO] --- -[2025-02-18 13:45:08,923 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 13:45:09,944 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 13:46:18,663 submission_checker1.py:3286 INFO] --- +[2025-02-18 13:46:18,663 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 13:46:18,663 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 13:46:18,663 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 13:46:18,663 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 13:46:18,663 submission_checker1.py:3307 INFO] --- +[2025-02-18 13:46:18,663 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 13:46:19,625 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0106 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0134 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430312 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 611de2536..064c0200b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -743 +744 diff --git a/summary.csv b/summary.csv index be01950d1..a82dbc206 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0106,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0134,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430312,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a9d2644a7..6089e4ac7 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0106, - "Accuracy":"acc: 75.200", + "Result":10.0134, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index ae430c1d9398f3c43b212fca8fd8035266cdc226..71e38b5583b90d338ec8066c82d545ccbbcc430a 100644 GIT binary patch delta 888 zcmV-;1Bd*fOrK1!WfKMbU=k`|vuYEW0)LXxQo{i}2PGgSh$N>Sf)$jaqrKLJ_@@Jw z&z~$VJxb!Zs-H=HqrOHoD^lo~{mt00dvK=*SE{kWy&l}}!4+S&I93AK;639I_l!fV zjKc=+@fY{_i+lXVJ^tbzf3e~(L~CfzuVxzQSsHGUWmpssV~gearj&Cb$v~M|eSfvo z#n3g4*XlG5D@<&&r|RBp_XTC!!;#UD@!jG6%1G6{=qM%jQnEi^k_e#S1uZ!&@37&k zk4aB2|39`Jw`w|K8Tez}dlBL5D(@M_{Tsn+$^LjoTTJ3Ed+@Jz3!eUV?xk@DI1Z8n)5p%XZknECTRw7{GBb` zHC)&Ac2;R(IKCx!?UU*G@^pGG5m(u(#;3FS@^XB!sP}7EZuNd(k-*2K{zK?^HhGj1 z`4!+3$MXX(a2;zfP|~^GuVpK!`}MKBu-~zgKlmw+1dbBa2W$12U27)iV)P%g>la}N z1T#>DNRv|AW+Yl_p*z+wY=CkO~QCu6RScs+DV1LW_H`@R%@_d371SVKucn?hN zNJK}Z;3LxhxOM;tj-dqwI(SSng3sOnPWhV+gL2_iGV~U1T2JEW-g!A1yROT+a OllUbg216qN0001{BC(nP delta 893 zcmV-@1A_dYOrcD$WfKLug^*QZvuYEW0)LJn-w!1mz;jRnQi4cw+96m$DLUF~U5I}= zVEO#X;?koej;s2a)HmvDM6)7=j@jRg4Z8<-dT^y08{F%`{T^KLWs74afDPU=4sp*o z#L75q@E(70kH5IbU)&30c+i|O=BbI?b*1Z=IzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8 zEZ@xbhB$tF8$3t%{(bN+jPD!M4}a{04mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$ z-bFK$UkB-@#bn)CTY(aZ zo#Ie~^w=0^+ZcP!qNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rLZ= z1QY-O00;m806zdBon9SB0<--WO$Y=7RUiIrKng*Dgo+= zN=Ot@akK1gY7zg)Hl%rb+@wugDJPEWnf-Ql z#S0WUPZg()R#2i299mX)1v7+c3vX5%uwH Date: Tue, 18 Feb 2025 13:49:03 +0000 Subject: [PATCH 117/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9759 -> 9758 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 951aadbb4..f7cdba0a1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 13:46:18,663 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 13:46:18,663 submission_checker1.py:3270 INFO] --- -[2025-02-18 13:46:18,663 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 13:48:57,942 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 13:48:57,942 submission_checker1.py:3270 INFO] --- +[2025-02-18 13:48:57,942 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 13:46:18,663 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 13:48:57,942 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 13:46:18,663 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 13:48:57,942 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 13:46:18,663 submission_checker1.py:3286 INFO] --- -[2025-02-18 13:46:18,663 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 13:46:18,663 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 13:46:18,663 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 13:46:18,663 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 13:46:18,663 submission_checker1.py:3307 INFO] --- -[2025-02-18 13:46:18,663 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 13:46:19,625 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 13:48:57,942 submission_checker1.py:3286 INFO] --- +[2025-02-18 13:48:57,942 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 13:48:57,942 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 13:48:57,943 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 13:48:57,943 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 13:48:57,943 submission_checker1.py:3307 INFO] --- +[2025-02-18 13:48:57,943 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 13:48:58,974 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0134 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1588 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430312 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 064c0200b..5f6e60798 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -744 +745 diff --git a/summary.csv b/summary.csv index a82dbc206..eef94fc1e 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0134,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1588,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430312,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6089e4ac7..cb486a32d 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0134, - "Accuracy":"acc: 75.800", + "Result":10.1588, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 71e38b5583b90d338ec8066c82d545ccbbcc430a..a9048e5ec5defbbe2d3da6a190a023b4215affdc 100644 GIT binary patch delta 875 zcmV-x1C;!qOrA`zWfKH0vT&EPX%m_Pf3guRH5|ZmPy$kdNOIaCSV1W|+G|~ie>!0K z{K?|dqa=>2`kB->>T5)^B886G-;52r2X}gKr5YRD>%sjVT=8X#V-(2~RQ4jaDu znDq4W|6|*6tEMBCfj`#07ZJX$@}6PbzY)Ba?2l)(#U$>s2mflvaA1%(MpE55KC&#| z%=U&jeta7|NB91H@Ggw+8`BT$e}oP;5!XbxBXxa|Y}7iI*WDIJ9lLAC6nJh0eoo#+ zGm~Ej>8HhKbOS7}e}8&9u>)dcSt%R__NE34BcIKZK5FlSe6$ zUjaUGJU{RP*RcizC7s*-+U?q}kL88^j+OkuPkAJ8l%PIXtIzCOGdUNd{{T=+0|XQR z000O8002J#A)Q_wM*;u3ZuO(+i8jswgYX+ zs)0gNTDlX`BoVb9NNSz@eHFiwfLC3P?wvDt29>L$Y@%PVPH0u3EJ;xWifJy?zC!QY zbzGpxd8#;Vw1Ntqz@b%rS29DGw(x4T0qX_0D1ekB%v9*$y&)Job^u9T5*#XsJ8dQP zp|pE{Y$*Gtd%$^`K41x+a_T8Q17kB0(SYP^LYg1jbO6pUG$28FhqENZ7q15^-E2dr zTsfuiCj+zAQJmaICnjUtwn@9_tRa7fKQ}LL-Kw!r=Tk92b;%iF7O2;@F7b5{4%blU zH{rxP0sQe4^qIF|k-iUj6atVw?x=h$o}RZQYgFfHzKqj6&WdfeAom4X7GLL)H~kl& z9ESXxI0E|Ix1XT?1GAAKO#=ijvT&D^;v~8S;*YafC6l2gQ~{8a_9Y_*LL&eG000kG Bo9zGq delta 876 zcmV-y1C#uoOrK1!WfKJaU=k{`X%m_Pf0EHs!vQ=8B_Jh;B&QvM6_lc*z1D^JrvsMH zpDZptO5(VxpGkeAzD6`FQs|id&DgMeaHj`XswAfj6NE~3Ol-5K>fUVk1!ddAk@51=LG5x?!f9PNnaZQ9fQr8#BMy+Fc-EDEyvAcFmf#+7>=j2^9 zGx>Fpep-A+H^B1x_ot^5J8*8=id8OskzdS_^4a%oHklov6F8Qc0JONeLADhrk=Q8? zHAs(*fwqmY=PYUptF7`ls8BPg6nTuA)OGKjYU;A1`2n&6!wHlP>1mZFe+F^R%v25z9o0a&)xt|`I`-c za^X}m^cHSfPvYp_c{v)puFJY|u*UoZuNJTG!>Xy&`%`g1O(i(x4p=a*sqkeH57#mu z7V*Su3F4`b`i-~aB8NT@Nen;_aZk!;`TVjboTH}5i-$BXQZikVlHN}!dHmXsyy`y# z)p5wbi6_7z_w^@e{s6O(Ax#4W{9qC)li?(~1v5~ENG6k@B~$^Cll3Jd216qN0001U C6t6A- diff --git a/summary_results.json b/summary_results.json index 4de91cb48..c872f3e8d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0134, + "Performance_Result": 10.1588, "Performance_Units": "Samples/s" }, { From 986c263d900677ce9e40cc660d9b866ff9ff44cf Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 13:51:29 +0000 Subject: [PATCH 118/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9758 -> 9758 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f7cdba0a1..5413403cb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 13:48:57,942 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 13:48:57,942 submission_checker1.py:3270 INFO] --- -[2025-02-18 13:48:57,942 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 13:51:24,615 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 13:51:24,615 submission_checker1.py:3270 INFO] --- +[2025-02-18 13:51:24,615 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 13:48:57,942 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 13:51:24,615 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 13:48:57,942 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 13:51:24,615 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 13:48:57,942 submission_checker1.py:3286 INFO] --- -[2025-02-18 13:48:57,942 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 13:48:57,942 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 13:48:57,943 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 13:48:57,943 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 13:48:57,943 submission_checker1.py:3307 INFO] --- -[2025-02-18 13:48:57,943 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 13:48:58,974 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 13:51:24,615 submission_checker1.py:3286 INFO] --- +[2025-02-18 13:51:24,615 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 13:51:24,615 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 13:51:24,615 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 13:51:24,615 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 13:51:24,615 submission_checker1.py:3307 INFO] --- +[2025-02-18 13:51:24,615 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 13:51:25,566 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436249 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.994 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 5f6e60798..bf2f48558 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -745 +746 diff --git a/summary.csv b/summary.csv index eef94fc1e..7f1458b68 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436249,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.994,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index cb486a32d..bbee979cb 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9948, + "Result":20.994, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a9048e5ec5defbbe2d3da6a190a023b4215affdc..df46b9c6083fc5c723d57acb601f7f906c8d2bfe 100644 GIT binary patch delta 542 zcmV+(0^$9hOrA`zSqKh{S7dZd2><~6K>z?40Fy8z6|;N@f(U=~TaeAdm)mDD9?KIs zohC{UWoK(0xNTj3|vlL*qDa$BQa2$o{(`3@BGS#;CRkwj08H&jL?@)+Bhvo5b^r*Dp#=pxcuX>aFWvx7`N@VsxpFEQdJB`*vpBl< zUXI4D>#}Y(SYv*IzgDmB!>Xy&hf{GtO(i(x4p=a*sql3X57#muR`J9e3F4`b`o!C5 zkz*f-BnF_zxM$^a_WZIWUZSSRi!#lNl+4#;MoU7A^7}CIrvCy|ry+L}&wyj@+fUH^ gvyUOm0tt*)WOPgk008{6V<~7K>z?40Fy8z6|;N@f(U=?+tkg%x7%kj9?KIs zohC{UWv6Q$xous4<^M_?+^)YJ|02)&i{`^H_w)PH^>_Kxa6VR&Ki0nzVCTOQpy$bg z(baM)mGo&L+p2tAWbaU3tf~wjN9*4QP!@3*#bNri?mhp3g5}ij%%b7@)u5L1@Q3ZuO(+i8jswgYX+s)0gNTDlX`BoVb9 zNNSz@eHFiwfLC3P?wvDt29>L$Y@%PVPH0u3EJ;xWifJy?zC!QYbzGpxd8#;Vw1Ntq zz@b%rS29DGw(x4T0qX_0D1ekB%v9*$y&)Job^u9T5*#XsJ8dQPp|pE{Y$*Gtd%$^` zK41x+a_T8Q17kB0(SYP^LYg1jbO6pUG$28FhqENZ7q15^-E2drTsfuiCj+zAQJmaI zCnjUtwn@9_tRa7fKQ}LL-Kw!r=Tk92b;%iF7O2;@F7b5{4%blUH{rxP0sQe4^qIF| zk-iUj6atVw?x=h$o}RZPYgFfHzKqj6&WdfeAom4X7GLL)H~kl&9ESXxI0E|Ix1XT? gvyUOm0tv|up$|+6008~7V Date: Tue, 18 Feb 2025 14:25:49 +0000 Subject: [PATCH 119/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 9758 -> 9751 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 5413403cb..0352dff7f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 13:51:24,615 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 13:51:24,615 submission_checker1.py:3270 INFO] --- -[2025-02-18 13:51:24,615 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 14:25:44,301 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 14:25:44,301 submission_checker1.py:3270 INFO] --- +[2025-02-18 14:25:44,301 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 13:51:24,615 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 14:25:44,301 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 13:51:24,615 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 14:25:44,301 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 13:51:24,615 submission_checker1.py:3286 INFO] --- -[2025-02-18 13:51:24,615 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 13:51:24,615 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 13:51:24,615 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 13:51:24,615 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 13:51:24,615 submission_checker1.py:3307 INFO] --- -[2025-02-18 13:51:24,615 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 13:51:25,566 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 14:25:44,301 submission_checker1.py:3286 INFO] --- +[2025-02-18 14:25:44,301 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 14:25:44,301 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 14:25:44,301 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 14:25:44,301 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 14:25:44,301 submission_checker1.py:3307 INFO] --- +[2025-02-18 14:25:44,301 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 14:25:45,324 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,16 +27,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436249 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.994 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43456 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1588 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0318 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430312 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index bf2f48558..021807516 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -746 +747 diff --git a/summary.csv b/summary.csv index 7f1458b68..0adfd78cf 100644 --- a/summary.csv +++ b/summary.csv @@ -5,16 +5,16 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436249,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.994,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43456,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1588,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0318,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430312,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bbee979cb..e417b4642 100644 --- a/summary.json +++ b/summary.json @@ -184,7 +184,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -214,12 +214,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436249, + "Result":0.43456, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.994, + "Result":20.9786, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1588, - "Accuracy":"acc: 75.000", + "Result":10.0318, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430312, + "Result":0.431964, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index df46b9c6083fc5c723d57acb601f7f906c8d2bfe..f88d29b6524c9f8406d7a265b929c58f4ca8ffe6 100644 GIT binary patch delta 3139 zcmV-J47~H6OqWcsSqKH_l=E;zlU@i?5_+BkVP|KT)V*J3w0X|yKFBYV!U$IZZ<87c zO@FfUTFFbfi=yB2H>bn4s^`)sXg{xpaTzN^Ir5j&^eTGzD#P(4%}$$@hXm0yS^cZb z!t7icsxtE*+p4K558*JKn6%_0xO^G+sPpx|m!GbBr~R|{tJP4FEr-ToI$7RnK{RX5 zrrE!Ru2xyie19?3Jnx-s z;$xX)YnY_DS;t{t)?0q)gXxGqj0SLq7G&1htov;aM&C% z^YynnO3GHPG8@|Jzk+dnUyn#I!dWP;*GOIV$0D?S2uYOT^>Xi zK_u7DF-mN5{hXi#Kx8BoL`FhEWF!_nPZC4af$vXQJQ{Dx(p)pj;aN2VB-sW0l9Mn}Ka_A`R0n;*#GJJrcOp zXgQQi)Ho*P5;e#rqX8}%4RDDX&ZwSbL;?Zj@1y*9x+F{Z# zHC^AiIQN+>aN!8XV?%8?jK-lp97cmoR&e5Q8eo#q0Fz~A94-McS!Tx(;KzZM9@7aj z36>lt3Ky=ZVHU1++H(FCDBr3{M^Kkec0cJ9P=C2+)6rk3WK1~Jr^9F*YSdvgNM!}9 z4yOSs84XZb=GNg70F`Ba9YO6hXVB7PI>9{=u;nmOxF>=dW>2JMYYtNR^EXA_sAcC` z?0R-=&G4+kr#x-}mW(Bbx^@_iLv1^Z2C=MQ+~G8UC8Gf>%e*^W0>HA&y(6f(<_ubT zOn)c%^c74xTog=}j0P64HT!g4qw?oGeSM=2o-46y;mH=U!W9Ky(-2+(myAJ&dUzO( zLrpx42Dz+YY@hdtL+c@9vA#QePD&4SkXk0;=tl;e7G{9s9XAd_Xn=)DE?BO~ACd-^X1~_}5rN?yQwJ4L|?BSw-$$tvY z9;0Sw&ud7gqnt+ZL-&-m)K+EfYV10CvgzM*TslQIcfQ;{lkr%d$mujuf+#y(>&R{E z`WydO;^21u<@gtQ)?YLqhPj>JpRT{kpN8|XlKip$kpMgYkpMkU7L2ZzQ>mm+3)xoX z<0AWn@?uqG_&8esJbLq`{4Xbmw@W=ApkH_IWQKoW~6{>65?RUKX!GYE5G+L})S*mgu zrD-&Kem9Z+P{#Q?Q&mP$ls>=7Ps#R&8#;diP)h>@6aWAK2mk;8KLF=)0<+!`hXDmg z+ri0UvuYEW0)LKSTWUCf=b!|n1d-&lL$HEUbhOvH5dU<*^7)g+rAJ8|SM@WgZ`9X_ zWaECYY6 zdoLnXp2eQWe@(@6aWAK2mk;8KL8<}UL8jQ z0JHuVOb7&nR>8`XP9Q6P+eu0gwgYX+sewXMT5=Q8#u2rCkkmSP`^t9WB;ZqztC{_F zc2Ti9$|m{+>x5P%%90dCpqS=D)g}7auHz+&oTrM@Mk^@M2^?Bg4+S%XX$x;w8?au0 zivmbF!c2({-W!6kV+WAbCBdPBxYt%vA4*$eL)ka20q1G@h$VP`%BiQg1IA_~qBD}S z5ovyG(*roe(0~Nx9nO*rU%eizbdwFea^sZ3pA1Y|FXHGvIx!mCwoTeuZw>h~{Iz*~ z?^lh5>Q2P~RRw2+S)g9qs=&8JI9x-W--HwI1@Na+&?nx`iwu3hqY!`$aWBf}?D=K8 zMpd5Xi#W~WY`M)D<|JQ``Fz)nyz9RL<$1`zi5I|-`~DMDf3uJw$^r%Gl=E;zlV2oT z0mYNMBqAF}+ri0U1ONbZ5&!@c00000000000001hflrh8BtZe8`XpCuLnHIud_J^_i7^(7+)J0k!90030i&l&&# delta 3203 zcmX|@cRU+f8^>ctszzgnnu)e*6|IOBJ4RboCH7vGs|2mtNJ?sl#wbONS~aS)h*_(O z8rQr^Y3!;^>l)>a`@Z)*|D4a~jPLV1&+~gepObEqV^U?s!W3`m8el@UXE6f@4M;e7 zc%+XuJGeXVW{rd=y(G7@SklRnA)uQ$v%hV6j45&Xv%7r79ks8Qhb2cI+q2CpSP-!b za>d{8?QPv7&KqWC-?+*F%J-^1w9JT_q$ruYEos)?yL(T#-3igdb(QQ3PdSLONOjD3 z_2*jMPqDX#RqQ;!BrR5)1=qdu3#W*wts=MYQewO+L+ui_3SJ&&cfdnT=k9Uz;Mxag z1^gWOC%r&B;}3%UHENfNsH+sMp4Cr-GvVE8n3?4m<=9Q7h0Gh%L&Pa<9Yw*xho;QJ zDQzEfHxsF`SFL3fOhn&4giRowH;sHEIww=7>0=sG*HXOaus73?NmvRYe)p&>_J<{p z@*63iK5o_2THit$-UmrWl>EhHoS8$qZL2JW3O;N}lbkFaCyWVON#h%*x~^F!Ipz39 ziAsktg0a8D=)u@IO$HF%DQgnvN((&*zrl_A@~w?5q#w65eSDuHNT{E!mL`< zj6WF6Sq?E!^lurrHx72^6qhJ|8JfQsv=ERp@?3+I=Rd%NGM*vyrTdW(4yU09-(Nra zq8Zw1%?6vqdI}Yh7qfYFYnCvXzg7Fv5R(q90wiW?7<3-fCJ8Jx8P=+9dJ$#V^2?BU zKdnf41l>C(RhA)Fgv(rRCf^K`CS%yhO3HB=A8+i6qNO2i&XdtgyWK%5w`<1CT?!zK ze}-`RUAtCR8D=T<~8I!8Xt2mibtla$J7xNRM*Fib|t<@j+HuB)H z|5I>}%gW(8!B|b45^x`iRTKDI>#^p@Cm)CLQpkLKch_A=SDUNv1JBwbWkR96H4m!1 zAmto%2F?VwNpx>$h|vnT7A*@*w$<#mbteBb1B%WxsNT)@qNM%hKQ-?JR=*`qIV4d!i9@%mZlJIR|LST(-i6N^0l1oLHpcQYhZ=GuTf$_s&r?ej*kgw=w}2m^7pY$$67F3yzK{Zp2a z&O|4Nqm@Czy`tX!51mVNRqiTFITd{Py39nz+2*&HSQ?+X>zOw^4cSq}2qD5Ie5Df< zq_yaYH2TJ`Z>(wW3UNRL6+K*{5$#Me#p|ySuwKU^zGqC~G9TvJ<`^ZDP?7YY+k2?3 z3jMx3TWo@hnJk6qPt$6g>tu^%TOO942SXig)1WIp94)?vHX$Y8!#$$jPc80cFviti zwqLm)7W{m7BDuVv$q1F%h-zq`sTmR5TKb**O>mAE_jQOFQmmJ}07--n$wTo(v7YOd zddZknF%Tl}wsjvkpbqGe^h!p3D8|DK>SZpXcMVF>UkUJUkUz0bvYm0`qGCIs)N9Qz zE0%i0o^j{69|MvD%`LsQzevg2U!STe4<|1OKZ*I4L)TZJOOt;`oI>QH$w%cn>C&@F z;D)6nX!>>ktDlpmKbPy*XsX`NJKPdr&%e|Qe-a~(F9|n~n>oTHE9l9@((fVExoI~n ziPZo=VtY>oBkLx zBE9>A<$ke>3oQ?zSuUqqA6wL%n0xz7cmcD9b%S-o7s}eH3aRX>$j)3-;Eed4zSZ6Y9 zsEP^GuaP}ba;$ROVYcMSvJtn8+N8Kez4PFNQe@pzW5=O9VPP6IhDJ^2!Kk4+>KShd zeRRtp0|VRZrFVB&+?SlrWG+8D2u5-{bYj`io#SiY4<3C8D;UVx%=cXH;j6BFyLnZQ znAoF)tH0A(v_u1zQh=WP87Q3|U5Jt7@K99w9rq0_(B%v_Py@5#epQDexgluS+>Q8KVAZHro4Xq zT~3GZHq=kU35%nfpYy3lW4UR&TcpCUQ`sO7ktWtPj@HW`frS;WshY+$s}{`*xLl_| zcD@sI&e1kJ>r$_IVzqXlOylpw2~Bv_p=tteS*!FtO5Ctgrh0283U_(_IJH!&;Ih$6 zhlOI-qYk!gO*&I!21afG0Kh^{f%4SMUkIUNlCAblcdT*d%Am(pUy!*j^ugbh-i(04 zei1`!T|s>Dwu{Cg91o6r=7tNPiDicwqspt^x;5faiIcPOY>mK2A-;tW!Ra4uzlCkL z-Qrwe5o|Mx?0$pr&R#fXOBm-zYOU{fn4Xv0*yg)>qWzBe>-c_Aa!w(Y*NUjOEnkby zd0?c%_w0O1#u(1h*zUaDNKnPC`KtEL8=UuYbfX%0 zdcWJ(F-ck9?5m1Kwe;-gUqtxhFhMng{gd!5+4xPt5so!nB>RmE6T0;hf#J!RQAFCr zS6+jQjo#hdK|V~6Z_Vf_k{y8Gi@Q7m7^L6ea3P4d49i6o=*3~agD#;fFT-cu$&(il zc$W+iGOkR})dmla$(}QPviBFi=M;bFKz>ox7b(({bJw33C3lw?+U?Jj7Rjn+iioPp zXCWt(axA=X&Yaz|lf!l$fp!l(QYDG=HrNS`j2yY!#miv7)aLtPh!=8l-F$H&tq&J1 zHi5jLFq2iy$V3ma!IfkOIt@QH%`N(gx6BN%?5W>h*1x{=#F@AS=1(Gan0P^&S9G!xMm#2Rh zf@#RhpuLqwOMoQaew{9v#gA<-6nSlvzL8mmbFelAb}VVmYX|JyyH)`!mwn>zN_P(b z@z(A5=^u1_!6(r?j5tZOFCqDe#o0J_#!)6zOr;H@B+t#f{4X- zryyvqlyulR<(2Fr5)Ea}?XLG&^#65SZJ%HCDNdn3F6HxttAM_-cHGX9{7%s+y?L?N zBd);+myN00L&pp?F=D;+|NH8rp*D2= z0DvHjoPR(t#?7A`0plj$g+b_TAY@+%4>=OXDRh>27##p$N~^|&e{a*~2)vpo`Hm>p Z*&%Ye7$@t$dlC2~@&-(TSzF?4 Date: Tue, 18 Feb 2025 14:27:29 +0000 Subject: [PATCH 120/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9751 -> 9752 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0352dff7f..a82f5a23e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 14:25:44,301 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 14:25:44,301 submission_checker1.py:3270 INFO] --- -[2025-02-18 14:25:44,301 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 14:27:24,776 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 14:27:24,776 submission_checker1.py:3270 INFO] --- +[2025-02-18 14:27:24,776 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 14:25:44,301 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 14:27:24,776 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 14:25:44,301 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 14:27:24,776 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 14:25:44,301 submission_checker1.py:3286 INFO] --- -[2025-02-18 14:25:44,301 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 14:25:44,301 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 14:25:44,301 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 14:25:44,301 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 14:25:44,301 submission_checker1.py:3307 INFO] --- -[2025-02-18 14:25:44,301 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 14:25:45,324 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 14:27:24,776 submission_checker1.py:3286 INFO] --- +[2025-02-18 14:27:24,776 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 14:27:24,776 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 14:27:24,776 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 14:27:24,776 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 14:27:24,776 submission_checker1.py:3307 INFO] --- +[2025-02-18 14:27:24,776 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 14:27:25,778 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 021807516..895152c23 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -747 +748 diff --git a/summary.xlsx b/summary.xlsx index f88d29b6524c9f8406d7a265b929c58f4ca8ffe6..c45526ab6260045d1fa13633854cd0a835070257 100644 GIT binary patch delta 339 zcmV-Z0j&O)OqfitLm&kTlmhT3lSm*Re@??N5QOhMMdW+pB&ASUNh$&AiAqQWQE{{E zZEF$#$Tp;Tdz`dQTPY`w>zVy_cI9GukWKUp)(NdjlqD&OKrzjQ+Lh>|Ud0O(IZqX* zjaE>i4jfumcLg(qX$x;w8?au0ivmbF!c2+w-W!6kV|$R)CBdPBxYbrtA4wMFDW{&|9vGXEh)zh(Mx^<%%>dvGLjw|&cQ{KjeDQj)(oHrD%9T?J-x-*+ zp2g99aAGvJZJV^S!5Z>sc(ZQBW0RRw2+S)g9qs=(JpI9x-WUxyQK1@NaX z=o4?JMUH*IqY!``U?j<7zJR<-A002|nn#TYD delta 338 zcmV-Y0j>U+OqWcsLm&l%R>8_8lSm*Re`>=p5QOi23ZuI%+eu0gwgYX+sewXMT5=Q8 z#u2rCkkmSP`^t9WB;ZqztC{_Fc2Ti9$|m{+>x5P%%90dCpqS=D)g}7auHz+&oTrM@ zMk^@M2^?Bg4+S%XX$x;w8?au0ivmbF!c2({-W!6kV+WAbCBdPBxYt%vA4*$ee?!?f ztpVq0`iLcX%BiQg1IA_~qBD}S5ovyG(*roe(0~Nx9nO*rU%eizbdwFea^sZ3pA1Y| zFXHGvIx!mCwoTeuZw>h~{Iz*~?^lh5>Q2P~RRw2+S)g9qs=&8JI9x-W--HwI1@Na+ z&?nx`iwu3hqY!`$aWBf}?D=K8Bt})9=8HJZ<7~Oj<|JQ``Fz)nyz9RL<$1`zi5I|- k`~DMDf3u1q3j_s&R>8_8lbR(|0g03EB_jqqBLDyZ06&hO-~a#s From 9ebde07640031eadc8ff6ee554a4d7103223a689 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 14:32:24 +0000 Subject: [PATCH 121/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9752 -> 9751 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a82f5a23e..c04e26638 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 14:27:24,776 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 14:27:24,776 submission_checker1.py:3270 INFO] --- -[2025-02-18 14:27:24,776 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 14:32:19,048 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 14:32:19,048 submission_checker1.py:3270 INFO] --- +[2025-02-18 14:32:19,048 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 14:27:24,776 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 14:32:19,048 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 14:27:24,776 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 14:32:19,048 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 14:27:24,776 submission_checker1.py:3286 INFO] --- -[2025-02-18 14:27:24,776 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 14:27:24,776 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 14:27:24,776 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 14:27:24,776 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 14:27:24,776 submission_checker1.py:3307 INFO] --- -[2025-02-18 14:27:24,776 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 14:27:25,778 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 14:32:19,048 submission_checker1.py:3286 INFO] --- +[2025-02-18 14:32:19,048 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 14:32:19,048 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 14:32:19,048 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 14:32:19,048 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 14:32:19,048 submission_checker1.py:3307 INFO] --- +[2025-02-18 14:32:19,048 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 14:32:20,057 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0318 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.94887 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.431964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432418 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 895152c23..cec645708 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -748 +749 diff --git a/summary.csv b/summary.csv index 0adfd78cf..fca334952 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0318,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.94887,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.431964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432418,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e417b4642..f382e2ab4 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0318, + "Result":9.94887, "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431964, + "Result":0.432418, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c45526ab6260045d1fa13633854cd0a835070257..7b040adba5cc9551be65fe282a56df6b261cdd37 100644 GIT binary patch delta 2731 zcmV;c3RLx&OqWcsSqKF6uP!i?T?jXS%3CVBm1N#xh{5!B(jiaRKx4-!G^! zdd?$ZXJ?nxyVAj~@Q|7)_OXnWSMn>!?E0RYlH%co@#c9rfn=bHh=U zH1lUee;UtZM}3m1`fK+;hYxY`dpeOaQ}P6}w4+Y4Y<|#erolu;eyYWDIm?fKjN&Bn zv;1u`Zl?1@`om>VG;JEXZZ#u6oT;k}2T7G2@n{qV@?#t4A+b)d3KiBmR_J0A?P{W}TZUPD9#2!bj%luXU5@ahpZVRxBz{nmWe@4x zFFqM@U>Nyn35wQlMRPH>=DOkPxk1V6D9+w}w{_UOFA}d6-)YD-ZMdO-4Yykj&3rtY zK^b^Z1{;IEJBBN+Xt&8QfyCUETmy-%Q*H~F+hPd>DVHnAIKRKD zxPDb}^Qz*uOtDjG8&}$9@dl}sE7~q8^>xYh>yjHTp`UAMlAUTzSnZL?5;(hE9CHO3 z<_T8_!oo)&A=l3*AR#Az{xgse@V@{Ff$l4i5a>>Ugj{AlAR!;l8IX{Z>j4QlyXRbD zkh9wd5^{DgfP|dgOCTX<_ZyJ3@#SXW%gtg5Oj67-mssT7pKwWw)cX-gNWGtcgw*?) zOB|Br%d-UhN&E_WAlJxgNqF!1*z**1MjJfdflE&3IgRP_7&_;FT!6IXBe;AS_o(~z zzn7n`ywm>K`{imV&6h*tFq=X}(KP=z%9lF%tGP@@gK##INxoo{gG{E`AYI%= zVOp$x(!sr{>CG&Wa=FTC7K^Foxp%UTPh^^}VbbP$G1J&&9t87wBbYB5@oe^(EM{50 znrf_7gCL%b!m;{)i#n8LL3A*P4hNB3KgV2Rk?ZG#OBjeiLO}!)3L=nD5SzaEeLZM^+mhA*UGPfK9 zjZMaJ0yGRkq@fHV4TTVB96BFvGzdeKfO1%tptm`HxFM0Xc%T!PDHarb(&f8e32*!3I5}5@;BeK*Oly8cwAsqY@-wRF)*@3Hm7EjJ0^66W5?r(x^U) zqH`pF($GnAYxr*w%B>l)pwiGa@dH@aQDc>Ay-T>B2yR~QgM&L(HM}j}SF)fNE z*VqQdl504YK*O*E8ipm;a4d~7EI|T>Wl4hG=HLsKw8aCRa4cz5A4IV&RG<;Mpj;aN z2VB-sW0h^y^}seakw(kf$0dFydL;OxM$@K$Tyl-wqFizf=Mrcbmq5d~nbVB+xJib3ueB6diAi&Ri%FxU>H5~ixy$4p7mffP8*0M=8k_oXfQB@~xV51a;}; z_mlP>DtBx;`U{o7ghPEgKx0#*4$yEaD_C`qhEWMLjLI^%4obkNEc5FKYNt7WV=W%& zgg+Buwj2<}pNVh{cqUS_HHTCA^B;=5QOnM?*!JvLn&FyzcX^yWSOQB9b?pF+O>H|s z!?CPj+(8VqhM1}4yOFpZkM zJ+FcJcb|*YQePD&4SknM-QM~F$1am%50l&_Vo_%g&^UrJS;5(ZG>pj#&K@)#i!xc} z>_MF{Cd-^X26OhX77ujdwkea$*@L1OlNFpjM$OKi*O1Ic1&!i|?kQ`jt;*Wf*mm^f z)4yvwbc$?leYri7$ylC$$mujyf;c~2tH@33>Kp%8;^21m<@gtQ-d{W$hJ~HqpRT^j zpM|rrlK!#!kpQdskpR6&7L2YIQ>kQ6bJqfbB_xVK+$?+BTEsuH4QbvU zCu!4G>WSleX1|?Xxm@g3ll+47(&!42EJq1wZiLj^3catF=^Q0KFfCYPG*qYqj~4Y! z$t~r^!K*VCT#(?C2-2Q%TcKSDmSXJr4ixiQbm%BwMvx*_g l0tEH1E-5!B(kNaRKx4-!G^! zdY%JeXJ?nxyW{j~@Q|IGZZ>GETy1-d2UCtBRZl(J-8k+v?5r=Z32) zN$Sss{xq7)w)!L!_1DgS4j-cU_hcevs$>b~Nn4$y>EfW-OoEA=`H2=SvnVIhjVqE;UKP(BN~muKz@vZ>RH8z76F=vh!NWOSy}p-}5)8!?vpD(k5s>uZD3M zD?>T*m(%nrdiW~C@g&Voo0W%u1kp5E{j1Ev>|7eEGV>qXs;Mdu;V_-#iIqt>9LoGe zt4gp;(rEVE`jFTpSceLG6DxGFjrKIr(rv@aKaZ!W+{853y{<<1(NF!(VH`au@v4Vp z;pd->I53Rtv;=wUx4bzYTXWNJ_1vIjb>wI7zGEIX@AJeP#djKVO&e~1Xv6JJLo*xC zc2EXCC%Sha zDz0Bu+`Ou|EmQ1O+Tts1LA*gKMMcXarM@n?eqC~d68gE8CfTdjVyiu}Ac4Ez#W5<# zFi%h+2n!z>3AuhgF%oir;y*JI0{$09LZJJ~NC44G7?hnCq_c* z{frWqWcl(eVg4k3WqKgj$Z1J<_xafK6m>=$df0(SPUks|>0=CkopUajwB#eWd>Qws z^Yy=%pRRhR{j>M0)liZxhsI$#S>9tQ^|v}o%2us18`|o>f^mIck4voA7+T8| zo?QT)!$apFbmUSyMu|T$4Dr2jD-9N|KgNDhv6=Tt|Yw6gpXS?B~ zz8*7p@;Cq7p+sT~UB6wR#3PbRr$!>jlYt-P_cM*`_GO)asxi)ZTs);w983oHhGuJe zBUlQfRe{7hN~u8Njk7~y%h60%H+c|U9z+*GB-hU|N^EldoS+0iWF!nToB2VMG^YM4!=r2e>@{f}^n@xp(8Od5Itia^;W?*Y59nG+9R}@gm zKWIM^_+Z_#D3z#j1f`OBKo3+h8laNV0F|ggDn%KUOah>?Btciut%NgZ=`o#n2Bnfl z^|>g2I!AIEI!W#f|1CnfGehQ78oDNa0LwaRtTI(@1|~KSfHli83$|VaztwakaQDWt zDVC^l42mUc5KBe_STY*G5;cgWQHCXx0I)1c(A^$@d6@Ipk*uSu{3XtSC z?~ep-h+A|+o6$HeiX>_fNk#)mG8#YOMRP?t`2 zKj{=uxo6YSU#MhEIMk=ZXdG(PVKhi(1*;CH0V){{P+8{I;SvCqWqut&?KEe9(9&Z% z!95YMaCO+1VSxvXI0;WWS{qX90um0GFj&A z;W_~(%bYz1ID4R_$8_SgD3jpq;i7=a3eFy*W@pc9NT#ElM)5=Ul(p1WW$kM0I(o9{ z-*a3#MK*W7+&+`>Sf0p#=`>M-C_7#2$ZhNT8~<10;CB7x_!oKBUo;P`**=R``}$*fA~c)J}7M^EB#rTRk!L#C4a#Ut8=*U$MW2d z$KgCtrgD@Os%zQpcf9_=fz|6YTC84Ks&W^lX*7F&H-7@LuZPDsv1r1`PU0N@Nm0}_;XI7>2o@p`b*O*Rb5l~W4e8JM)5#nF9mVl=jG zo3yjR8uDj&vwnRaR*i+~PsIRL1!sg=pkCXmz}H1MTtl5-hZAoF@TV^56K|(Qj(xzR z5P%%xo|Vtp^Gm%#Ri5VaIL+g1QD++WB!49N{Hq^%(|-ZV(~y4?&wyj@+fPva0kerA m$^r!Fl=E=2S0p_F1qze`@FtU)B~$^5lkO!W20SAG0001SlpTuz diff --git a/summary_results.json b/summary_results.json index ed67b597e..d27b656a9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0318, + "Performance_Result": 9.94887, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.431964, + "Performance_Result": 0.432418, "Performance_Units": "Samples/s" }, { From 7d86451b2c6bca45139e4e8b109fe9b60ef31d98 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 18:25:41 +0000 Subject: [PATCH 122/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9751 -> 9751 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c04e26638..ad48be134 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 14:32:19,048 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 14:32:19,048 submission_checker1.py:3270 INFO] --- -[2025-02-18 14:32:19,048 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 18:25:35,301 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 18:25:35,301 submission_checker1.py:3270 INFO] --- +[2025-02-18 18:25:35,301 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 14:32:19,048 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 18:25:35,302 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 14:32:19,048 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 18:25:35,302 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 14:32:19,048 submission_checker1.py:3286 INFO] --- -[2025-02-18 14:32:19,048 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 14:32:19,048 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 14:32:19,048 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 14:32:19,048 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 14:32:19,048 submission_checker1.py:3307 INFO] --- -[2025-02-18 14:32:19,048 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 14:32:20,057 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 18:25:35,302 submission_checker1.py:3286 INFO] --- +[2025-02-18 18:25:35,302 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 18:25:35,302 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 18:25:35,302 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 18:25:35,302 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 18:25:35,302 submission_checker1.py:3307 INFO] --- +[2025-02-18 18:25:35,302 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 18:25:36,654 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69558 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72377 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cec645708..9beea1f9d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -749 +750 diff --git a/summary.csv b/summary.csv index fca334952..81b8ad245 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69558,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72377,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f382e2ab4..40488aa30 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.69558, + "Result":1.72377, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 7b040adba5cc9551be65fe282a56df6b261cdd37..21a410145531baa9f883869393b3ef0f79005874 100644 GIT binary patch delta 2727 zcmV;Y3Rv}*OqWcsSqKDfa&8}!T?jXS$XhD9m1N#xh{5!B(kNaRKx4-!G^! zdY&U;XJ?nxyVAj~@Q|IGrf>GD)L&)>eh4tBRb3@gSOw+Um{q=Z32) zX%@@|!6cr^w)!Mf_1DgS4jvnS*L^E}j;UKA!BOVT;P=1WV#Z=C+Rgy$bf-E0?I*#V4 z+BqCV`LUuNC6U9ndL#}`elvvH>>MupzK!HVy7gKqO1X>U--|b=gSM&{(k5s>FNblN zC<8eR7L)8Me)uY*(KyRbn^lB=gz+R@{;N!*{9GEUG7TQvs;Mdu(I6WaiIs6Q7|7y8 zt4g>?vv~U3>X29`ScM9E9V>LPiS{(n(rv>kK98q~T*ow5y)H-iG01|>VG=(m$+CxZ z9u%L9I53R-w1h?Lx1zZiTXWrT_1vK3brfgszGEIX?~BB1#djKVO&e~1Xv6JRLo*-G zW>5w{C zDz0Bu+`Ou|EmQ1N+Tts1LA*gKMMcXarM@n?eqC~d68gE8CfTXhVyiu}Ac4Ev#W5<# zFi%h+2n!z>3AuhgF%oir;y*JI0{$09LZJJ~NCKxRQSN!1UB%^@>if2*UUeAO!Rp)LR0Gp_Heaf#I$Lu;8v zOYE7u9UeLdp(B^tF-mN5rJkS!pkpKyIz~dFV02QhZ2b~bp3XL5|2nOof?T8jr&1Z+|M-f+m}^;s>UeiasHIWNjUD`8=9@@ zjc_51Rs|C4D5V04H_jG`Ek`q5-Q+=Zc@SL$kz7B=D6z@)bAl28k&#dk83_fEkx&rb z5)d&QKr9LGGT{_Nw**9H@DxOsf`~CK5_5qPk4P@v{~ZwPD5V03x9-lAXj-l%_T2V( zJc%BXNUr{WW0ctR#s?)pA|s(BG7?H6N?dy59YaWhCSyxb*6UZ5o7e+L#2(43Dn;v)80`lve@3D{l)xSMxk;&qOG6p1)-~0g%is z$Bf1y<2Ye7fFP%#400L@A)|5We7MnY7;*_Hhh+(Wx|@R=5~0OsI`K%JbBO9OQFNQ} zg3$<_d?1%w<9{1KzRZIKi~}j>`SqBjvVSi$H=hdsGLp4aSb@pw^}tq8I+|hIuGm8* zztMgqaAV!FD3z#j1f`PMpa&`$4N%EwfJ)RLm7U<;Me2whMv zjsF8K>!`8Hw(5Fdo0~|(wD)nzKNCF?_^8o;awwOmaZJi3YLH7t16(p1;1V^+rCG)$ zlK{9ZNzmOKeBqL|_*^A)_XeVROcZj-Jjg~a%{R>@+06OtNY+taB_z#VBwb+}dywQG z-X95kAa2nI+Kk3&Q6y1=NHQ8glF2j_tCQ0FG#t#r#M|G7JudVIHtL7M{ zH5~ixy$4p7mi>&Hq?g0XdLRpVKm5O z1t$)t0VWv@Fj;2C;SvCoWp*3^ejI4=nNE;Nu;egNxNt=cvv94`mh-Pb`BqIjg1U6_ z`^kMX<&I59f1#2w;ZUCrqj9KFhtVLF6|6d(2B>5-KxLU*hf4rdmicuAwbPt`L5t6H zg3m<2mcvBhGZEA<&qQjr<{*_n|Dnhmwd`DrZO@La8J@Lwm&e_MC1c5P}>fp zK`bj6cQ_4T$!Gw}GVczT0I)1`?+9wHIfE9T=>&IQ!KA}Q!DPv3U;$gRPvE|4-cbpsELQsAeR+vJe&r&WHi8KnUjZ009=-N zc?6hwpv7l8K`z11!$jdC7B$Qwwq{SyzXRqQb@qH0+t!|9A=|R{fNAJFm0T70e(`d9>_dQ235fXQeOOrvIR z&ud`*-RB~;)K`T`L*HdmcecL3vCE{`!z8*yZ0hV`G_IgbR&e%k8ep=5vxggxO_?ln z_Hdm5lV#2x1Drk3;xnCiEy^S~d$=fIvVyb6sM*=`8j{(tpi%tLJ!LJmRav_l+m4=m z`u7}{PLa*6FSk!+GLk2MaxzJkFwRfcDst1h`o{m2IJjMXIsQeS_ZQCwQDNr~CabUV zXVGk=q<^e_B)~3yBtS2cg~O}GL@L?ST((vDIL|+!qF7ZKJPubs51`DGC{Cj6Y1Mo2 z`vi+g(3!@A_sc;oX3-xD`Tkw_&OUe-+8=&Vj1Nj%$xDBh=G84ndnsOU!|EK)gONNB zl2J5Em5ChYh3Z;<`<<+QaA5g5i|5N%ma5#vSr$*9-;HH3kV)~*RFz>IXU}hnQ?mKt zhR&Z*O9KRxk0cti;}KvE1d8x@+>=KjAb)DZFc5_AeF~$yE!!c%2-|_SZXr~`-A^+U-FVcNp0)ds8=;GzIhjxbZ9qxXhj?AQ?`bxCljAnvu5)Q8d@uz#WK zn;rn?X}ZD^Jmu6=+yi4X5z&a`Y(knJ+YA8CFf<@Rd55zk!&k2dE8T3vpxij6@STBK z>qVU0CnqLj+qOx&7_1?GhCjD2Z^NpwQ2nVGpt|IYFbmXcTbKB@2#0H^^V@LZy#W5$ z1%2jiT;$vbJPHBGIqpUISUf%NHYBLd(|j4Hd7KrytRVT4ELLCr$h-b4P>w_XO}qfk hx$i$g{Rgv&A<6;-ZgOrPvsWZN0R)Qhc-)hiC4Zw>A9?@) delta 2727 zcmV;Y3Rv}*OqWcsSqKF6uP!i?T?jXS%3CVBm1N#xh{5!B(jiaRKx4-!G^! zdd?$ZXJ?nxyVAj~@Q|7)_OXnWSMn>!?E0RYlH%co@#c9rfn=bHh=U zH1lUee;UtZM}3m1`fK+;hYxY`dpeOaQ}P6}w4+Y4Y<|#erolu;eyYWDIm?fKjN&Bn zv;1u`Zl?1@`om>VG;JEXZZ#u6oT;k}2T7G2@n{qV@?#t4A+b)d3KiBmR_J0A?P{W}TZUPD9#2!bj%luXU5@ahpZVRxBz{nmWe@4x zFFqM@U>Nyn35wQlMRPH>=DOkPxk1V6D9+w}w{_UOFA}d6-)YD-ZMdO-4Yykj&3rtY zK^b^Z1{;IEJBBN+Xt&8QfyCUETmy-%Q*H~F+hPd>DVHnAIKRKD zxPDb}^Qz*uOtDjG8&}$9@dl}sE7~q8^>xYh>yjHTp`UAMlAUTzSnZL?5;(hE9CHO3 z<_T8_!oo)&A=l3*AR#Az{xgse@V@{Ff$l4i5a>>Ugj{AlAR!;l8IX{Z>j4QlyXRbD zkh9wd5^{DgfP|dgOCTX<_ZyJ3@#SXW%gtg5Oj67-mssT7pKwWw)cX-gNWGtcgw*?) zOB|Br%d-UhN&E_WAlJxgNqF!1*z**1MjJfdflE&3IgRP_7&_;FT!6IXBe;AS_o(~z zzn7n`ywm>K`{imV&6h*tFq=X}(KP=z%9lF%tGP@@gK##INxoo{gG{E`AYI%= zVOp$x(!sr{>CG&Wa=FTC7K^Foxp%UTPh^^}VbbP$G1J&&9t87wBbYB5@oe^(EM{50 znrf_7gCL%b!m;{)i#n8LL3A*P4hNB3KgV2Rk?ZG#OBjeiLO}!)3L=nD5SzaEeLZM^+mhA*UGPfK9 zjZMaJ0yGRkq@fHV4TTVB96BFvGzdeKfO1%tptm`HxFM0Xc%T!PDHarb(&f8e32*!3I5}5@;BeK*Oly8cwAsqY@-wRF)*@3Hm7EjJ0^66W5?r(x^U) zqH`pF($GnAYxr*w%B>l)pwiGa@dH@aQDc>Ay-T>B2yR~QgM&L(HM}j}SF)fNE z*VqQdl504YK*O*E8ipm;a4d~7EI|T>Wl4hG=HLsKw8aCRa4cz5A4IV&RG<;Mpj;aN z2VB-sW0h^y^}seakw(kf$0dFydL;OxM$@K$Tyl-wqFizf=Mrcbmq5d~nbVB+xJib3ueB6diAi&Ri%FxU>H5~ixy$4p7mffP8*0M=8k_oXfQB@~xV51a;}; z_mlP>DtBx;`U{o7ghPEgKx0#*4$yEaD_C`qhEWMLjLI^%4obkNEc5FKYNt7WV=W%& zgg+Buwj2<}pNVh{cqUS_HHTCA^B;=5QOnM?*!JvLn&FyzcX^yWSOQB9b?pF+O>H|s z!?CPj+(8VqhM1}4yOFpZkM zJ+FcJcb|*YQePD&4SknM-QM~F$1am%50l&_Vo_%g&^UrJS;5(ZG>pj#&K@)#i!xc} z>_MF{Cd-^X26OhX77ujdwkea$*@L1OlNFpjM$OKi*O1Ic1&!i|?kQ`jt;*Wf*mm^f z)4yvwbc$?leYri7$ylC$$mujyf;c~2tH@33>Kp%8;^21m<@gtQ-d{W$hJ~HqpRT^j zpM|rrlK!#!kpQdskpR6&7L2YIQ>kQ6bJ{XNeg7ebo3Xv>F321JF)Y}TZub1f@B|b1M zSYtF)r~{7{^-akw<;KCQGZtKs;FAc_o^o5ET?m$9?D-B9^I3H0C~k~XEX2}nv43U! zn{5FXd47)-1SVKuxCf?oBBBFQ@Cj*uTr&U!$IyTR9Xuu(!6$D3r~K81K{<0O89EDB ztw(Wk@4cLiZQEw;Y_P`s1b?nx-iB3EsryrLKy@iN Date: Tue, 18 Feb 2025 18:29:45 +0000 Subject: [PATCH 123/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9751 -> 9751 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ad48be134..0c401224e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 18:25:35,301 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 18:25:35,301 submission_checker1.py:3270 INFO] --- -[2025-02-18 18:25:35,301 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 18:29:40,114 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 18:29:40,114 submission_checker1.py:3270 INFO] --- +[2025-02-18 18:29:40,114 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 18:25:35,302 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 18:29:40,114 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 18:25:35,302 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 18:29:40,114 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 18:25:35,302 submission_checker1.py:3286 INFO] --- -[2025-02-18 18:25:35,302 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 18:25:35,302 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 18:25:35,302 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 18:25:35,302 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 18:25:35,302 submission_checker1.py:3307 INFO] --- -[2025-02-18 18:25:35,302 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 18:25:36,654 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 18:29:40,114 submission_checker1.py:3286 INFO] --- +[2025-02-18 18:29:40,114 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 18:29:40,114 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 18:29:40,114 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 18:29:40,114 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 18:29:40,114 submission_checker1.py:3307 INFO] --- +[2025-02-18 18:29:40,114 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 18:29:41,181 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.94887 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2864 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.432418 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 9beea1f9d..29988c802 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -750 +751 diff --git a/summary.csv b/summary.csv index 81b8ad245..8258be6a4 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.94887,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2864,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.432418,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 40488aa30..7d2985922 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.94887, - "Accuracy":"acc: 75.400", + "Result":10.2864, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432418, + "Result":0.434658, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 21a410145531baa9f883869393b3ef0f79005874..effe1fb0626c4350106c0e11efea37c772e70807 100644 GIT binary patch delta 2516 zcmV;_2`l!OOqWcsSqKF9adM`UT?kYG&XckT$A8m)UJm0jQ3i4tEGF4i{P0yqqj8p> zHme8;<4L;wSD8lnxinN|8a%dDQ&k?KK{hTDE8}P|kj05sm2i<}@$|RVA+b)d3KjM` zR_J0A?P;Q=+lEzq9#0dwj%luXU5@Z$kOiH?Bz{nmWe@2*C_Wi+U>Nyn35(WmMRPH> z=6|~3>bXJ5>nP6NeaAd(-WQ42itjY!nl{|fhTE-%W%L0I?*B;@+}1SI6d ze+Ci){udx2(0v6G0^KQ)kjtzKB;><60}^s_eIOxc_nb=%a&~(_LeB05kdU)`2_)p~ zegl#gzT9klx!Ej%MT$A*5}Ta+6D~1Hy&r*u)cXlYNWGuA#3fn2JWIfz#IK+Sa(|7S zmV|eok3COOXSBiR9eCt)p3|5fkD+tU1xQOig3Fh24?AD~d->_gKkc2pU#^DId^t3V zvhm_h3*%{XI?4Z?=1ZOY)m$dSel#1(Bww(}ekRkbpDymEQCh5h(*C`r>CG&Wa=FTC z7K^Foxqq^bPh^^}VbbP$G1J&&?th2#c_W-J8u4uQm@H;lzM5*RRsArY4Wp6zi#m{5 z5KU6`!(np>=Id{Dl$5VpWj?g!e|yIDeKjtzT4QJ}vt?U*=57Z==WyuArFP6EHn~zy zxP(CmBosOzq0j*d`4j%dDS?iMyA-;T@Gb}^f056&!%2NU7(Dr#f9_Bsd4CLDzg=*N zM5UJUFcN`;k_aS}L@sgZjgMDC zA_ylJ?P*DP*CU*q(FUJ)z}BHOrUzo^IuuCAM{w!Y3)?gZnN`R zfLGoYfUf3w)}Dz_=$OA{I{}c)EyqCPka3&<4MPxVD1%5tAp{zi&W9Tf!haAYpd6MZ z=xz>fNMtQO=)@y=&RJ9sMA2==3!o7?`9LnW#{V{ee3=Ie7za|$^XoB5W&d7iZax+M zWh85w&GHbTq@ZU9pEsywQFn_{O?rQ7XB{5tK@>LC>fJ8b&41FeeJM z*ElBSl504ZK*P8M8pb8pa4yXkR?2?4CaowXR?A52Wc3SK*N|UGvlBHjL9-Pj$nQq*5ZRsIFrnh z1EToCm21GlwN6{kzXIi3HR%ZI(#h{9_sx_$HXZ$iN?^jFJ{_QOs8I)KIF%KwI!ME) z1R6$VnOg@XV1HDW`E>-f)10vuA9TW>i7;CZh~m#gxCT5Eso9#tsr>m5Mc$}o=UQxg zc5Kb?ti8KD?j9_GC5O6pfX1P=9iZV@Rxs`$4Z{*>7?x$;9h87!S?1mm)Le7MT71w6 z-+g5!9TdeTOQ2y3*qVJhuTlAPc3PG_X;zt7XB3U@RT1=#pJTmBHh1_45GAJc;5Y%AQue7r#%im;{|^Jb1qx)M6I>v5@cIh41WxccJ~^7sdFX zv`>}1^k->a-LjYB1sYc8a2|~0d60~vS*lFrFfUZs^4srZ^@9V;*I7JYzOq#1F3z%e z`uuJzgMmzncc!Wg<2ZYMQ=F2`4>xrF1W-!@1e1>>8nfdOdI1Ea_hON=VH27H0=D6^ z{S`(E1WF=Z*po*fAAieE!!Qs<_k2ao)j>AVFIXqEDp8iCC<4Va7iw3c_j(;KQRF;ToHklPiH_jVs=6td zAxv9%wc3F70$db8$`NKtwD;Z+j2+v9q%H{#6~wK!lKN2E9e*~IebXJ_JWcPh1W!5j z6rX^xnTTjWayB8&k8L^tXBZlgpuEFblHs%0gOzT!p;IoLQuw2RS?efH?t>GPv2EL= zU3AuvKf|A!m$z=!Sg6yf7@(@)j4%t-Yg-lgvIvK3sPmg};;jJwcntc?+ptLA2RsS^ zNFR4pJ{C{U^(7ird79tFX&z_GI$M(bf!wBFr;%6vXP_L0{F^ue`rOx_p!x%|i6P1Y e1o&}srju7Be*~oWVv&>DB)SAjB3;;%m?eK*RjXG3 delta 2515 zcmV;^2`u)POqWcsSqKDfa&8}!T?kYGo|Cc&$A4%)FNblNC<8eR7L)8Me)uY*(KyRb zn^lB_@g!aTt4yQ(TpFq}4IbO7sVWcAAR8Bnm2or}$l^q+O1Ma~c>3GwkXR>Jg$jEe zD|E4m_B7GbZNn-)kEe-T$23>HE=Tw=$b!yc5Gat`pPzFCJ1ID25j^PRwtri(3 zBe8ZR*Nnv8mE15AXIFB|NZgHNr(BaS*MtOul#2>7&hM`(u3uH$ysEe@Q|wgQ;wx=I zyg@2OMav_lzAm|bU2=mG`ni@S*{RlItA9PRAc4Ev#W5<#Fi%h+2n!z>3AuhgF%oj( zKQj^n{uf3>52y~~6gj{A_MnXQEGe$yAuFpuw**!;zLC$WEk&v@{!AQv2y<{Zh z?0#b;E&g(|`OD3Q1Qsdg7$r72_a`VZNxdH#390uJBO&#EMu|(Ze0i2Ie-ghkJ%5mE zg@24Exb+D3g4_Ci|I8vwpg`n?`A|_DTEqmZmqeM9SqV zt640jn&IzEwUzJ^Jg>%~lClYhA%&gYGAzG%d=*<-SpW%+8Vu~zlNcs7hi>M!a* zWgJ&K8L+ zM>AdBk&@QXoHU(z&e!1^q3gB4rL_dBe?YHg>4#yaoU&!%M6dM{j=Akb(B{D zz$%k3q1+aDNzb2`GnU z3A&qu8xo<#XFBmno^y!mF;R4z@q*C^oqQmdTjPHlK)%d_1&jkJ=lS)Rq_Tf6G&i3L z|1y%bR9Jz@>-E4^P&%4n+pgF{CBM;rByeNhvM80PaRjB3*`Nn184Xa$Xn;!8AeEww zN+tnNS(2bD=%a))Xz`g&Jb!~yNuzpP6rCeE4V@&nhW{3!+?pW^Dh*u|KY(Q&HCCA_ z*8>yl4Pec3%styTf`8O>B=G5tWm7Cs;}{f6)F76O2C!r_fF)`WOQQ@+CIMhslAyae z_<|*E@tIB#OB&TKF6*eV%C_oyV4Isr!?gEt$$vi+Jrek+(Q+u4 zsBuinC2EjMMgv?j8sHK&$fa4vC6fTSEJ@Ja9DL!Dw)k8nboU0LdQ22@$vntLF3mU1 zCE3jR>qyp7UL_>WT_jy$8+(xCAKo7cd?0So2ilCrX;CCmgGe$OK$6h_lBjVlie#${ zNv0FJ(^{6GyE*uRB!6x3nNB>DB1xlqTofRQ8llsN(_7=O1 zRvD2@0wA&^L3eZTg-F`sGo8S$D@66UC?Jy2z&)k)hJgGhwtrr0DX1cpI)b`%^83ksGv$s=M}MJ`G2u|34x@3XQHRkWl@+WyoCc_5 zG(csUTZc;kRDYKFbp*B3oI#7vbb`-Bz?Q>A;WH7`FwaD4w&oy}KmVb~8@23Qi*3)2 ztr?!RcbCWAgC%3hp{^Z9<51fUqd_by7l=0OT#0Q9Pris1u7B9`H4WkI;gT`vP!A8Iaj1!h z(IA%xMVcIWto$QO8{J!d3gkwd7#B-IzcYM&%;FFA{I5wBDQ8v&%Xoa8+G=4 z7u(jJVjVJeUUhL~QEqVKgrF_AnX%lhaTzISmDq(@-#tGH(x;fPz_;plg7) z2U>it6My2@bM|nZ0F!0T9s`^`(BdwN+WW8rzPZeERnsmrjw*t$#1KPh~QaCvq}Ll`zgv*D7+;y86cd zl{mOveL4O`p7$5e22o+>4<@Ux@@LU(q@;hWek8yyek4FIl7+*o#Y8IE(_FSy`8dx% zp`ut-89WYGKM$bHlPFH2>}l0|@%sdeNzj?bgZIlpEoRXl3;F(C_|86f7up|wQH&2t zTTaPKf0pLeEqf_maKq{x&V!LW50X(dOO=Tn=7s87e*2xQesEyW#aR|l zpWlsTFpx>{&Qz6Q9B0pOic_-r;fBtiP)h>@laC}Cv*QtZ0R%_e!O62>6Pf}7j$yO? z6-EjKitu>clSd#Qf6Z#cFc5_AeF~$yE!!c%2-|_SZXr~`-A^+U-F zVcNp0)ds8=;GzIhjxbZ9qxXhj?AQ?`bxCljAnvu5)Q8d@f3TtKn;rn?X}ZD^Jmu6= z+yi4X5z&a`Y(knJ+YA8CFf<@Rd55zk!&k2dE8T3vpxij6@STBK>qVU0CnqLj+qOx& z7_1?GhCjD2Z^NpwQ2nVGpt|IYFbmXcTbKB@2#0H^^V@LZy#W5$1%2jiT;$vbJPHBG zIqpUISUf%NB{rze(|j4Hd7KrytRVT4ELLCr$h-b4P>w_XO}qfkx$i$g{Rgv&A<6;- dZgOrPlUF2v1V`J!$&=b7x&(^wc-)hiC4at Date: Tue, 18 Feb 2025 20:26:07 +0000 Subject: [PATCH 124/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 9751 -> 9749 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0c401224e..65c14e02c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 18:29:40,114 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 18:29:40,114 submission_checker1.py:3270 INFO] --- -[2025-02-18 18:29:40,114 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 20:26:02,391 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 20:26:02,392 submission_checker1.py:3270 INFO] --- +[2025-02-18 20:26:02,392 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 18:29:40,114 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 20:26:02,392 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 18:29:40,114 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 20:26:02,392 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 18:29:40,114 submission_checker1.py:3286 INFO] --- -[2025-02-18 18:29:40,114 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 18:29:40,114 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 18:29:40,114 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 18:29:40,114 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 18:29:40,114 submission_checker1.py:3307 INFO] --- -[2025-02-18 18:29:40,114 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 18:29:41,181 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 20:26:02,392 submission_checker1.py:3286 INFO] --- +[2025-02-18 20:26:02,392 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 20:26:02,392 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 20:26:02,392 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 20:26:02,392 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 20:26:02,392 submission_checker1.py:3307 INFO] --- +[2025-02-18 20:26:02,392 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 20:26:03,398 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,14 +23,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72377 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72116 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43456 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40751 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409084 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2864 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -39,4 +39,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29568 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27544 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 29988c802..a28912922 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -751 +752 diff --git a/summary.csv b/summary.csv index 8258be6a4..6d83de6ee 100644 --- a/summary.csv +++ b/summary.csv @@ -1,14 +1,14 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72377,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72116,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43456,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40751,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409084,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2864,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -17,4 +17,4 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29568,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27544,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7d2985922..b8aea5e5c 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72377, + "Result":1.72116, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.40751, + "Result":0.409084, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -549,7 +549,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.29568, + "Result":6.27544, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index effe1fb0626c4350106c0e11efea37c772e70807..101d6ded7b49b2d7482dfe1877a32fcde377cde2 100644 GIT binary patch delta 2774 zcmV;{3MuuMOqEQqSqKiX$?@Ss2><~7K>z?40F!Sd8k2zt6MxEED!P?q-eQQ!WRh&% zPZq&esa0_S^YY&>s4;rZ17T-pm(;yqX0&bmns^`)s(|%qJ<1$u;a^x?k=~eXbRfgk9nw>T?4}S@wX|noPnT6T8G*o5gKX%lX zsyu|lbdo1lCgE@>^AoKq!7@ps*>CGZVv}GUDy&Vc(8V^|)kIsj3^V^co~CjW(_Htu z8sSGj^}C00^q|D69+HKhe=_30FtXDU#%vBC*CN& z(~xW0aDPJ^Zg(1**?6{tGVq`bHU@ov3|CyyZj)gGiMcPi1`=yuaswpxzT_52oULT9 z+!ijk#S#coE?1Coet%VQ{i@>TRmE+YVz1IRuC&eK4N@srv|Uo_>yqo&B{y6`KiASE zd)1n-+9Q)CaQ3@6<_a>*6Rr@1g^xf&uAfgpLVr&DXCNWqe*qE#-B%zX(47Jaxy*V% zLOz@`AR#B$0}^s}&$+}PXSWX|v4(o8bfQh+Ll=`ciR{`n?pw~wPP-^$d!7+B@8+sq0j*dg$_u_SNIpZ1Ueq> zQ0Pj+dmx;ABcJVtllnX`c=DZpZc`$83|+rnaEVJKmv)Urjwb^@$nR$w+3m|ZRexig z@wj+Oqd1rh?hVb-^hU50M!N!ub(B(p#2b5u#I~)q9KD4>bTEhx2a#Mq$6R8O>*s_^ z7>GbZK?D*CB9KrJoe~gvI0LaHya&Q5h)xNJVDJ<~hl0ps+9c+JOI#wkbpCfhtfQ0) zAl^EAQ)0_>Oi^&#<6;tBP9nMbkAJzuqBlNV!bk)XN+OU@61l{oH$H9&i6ER@w5KKE zJ(qBDMjJfd0b7UCm_CT1>rfyeAHk(tFKjbEh?2(CUuAf7te?Fmt)sjO0A9H}06LoM zngtV~&@q3jb^;)oTaJOoCgV5(8ipX!PzI5PLI^Ytoe#Gfgds{mIV?-i+kYP1lE_*- z(1}a(oU^Dth@y`fFMvkq(VH`@i$gamYk%N1oIoVYBmyxWc!U{}Y zZw9u8($);iazp`@_(A)T;1AYKlTyhwwxCpk2lR|epkY)34Wp84IF+J|N|1n2S(2b9 z=vKlRYw9*Nfw!8D1mC?eEs7=A*apRtYdDrb!>|Mzh9%c~(lNrK+?;0uY>TUkj4UyKEv{ZkQxIruVbeT$xXj!qto) zAhM3?DlJ~yyNg%NHiRf}(})6tiQt|A)1^dmjcrmQxrP%7G>k}~VMGEABhoA*5+q)&S9Y-)f4r}p1C!9%U$pKM(;mS2& z;aaCH=U;*Ht(tTMb?IdHlXd}>do~^Yg-T$;p*|g;v8j`44<3IHbi(&Um@NlH@jVf) z0ed1fTXQ&-KYvr?jaqiD#ja<^(hS>d7dTdhQ-CF~}3+!E+^cEj-yG zRyc(>mTm!;z@P?0Jv=~TQ= zFffsZf{8Q~Or)V;8fD%dlz@U+mY`=aZx3tnP$zU-ghlm16ay1zIG9Gw-k#UM{JZ-i zwbWOINkiXfQg?NyU=?eK0w}pn#G=+7pm79+vVye-X&96htUYKv7KO6R+Jib_P?lML zdkkjnVJ#l$#BEb3nY9N+F(@lodyJZ`J+CpDj&dBu58YJOQd^b9tFde8$>x98cIX`0 z-1+PFnT*HsL{6uP5=7bQT1ReM*MIVVB@S-ae;xlK&-#nz!!Y;r`_uK`@~7c^tR#P| ze2K`rOuA4~cEUGUC2co$e7eo>4MN=L~`f0ky|t$N8{pkZ|n7yekD`|&uOC(2Zg zvO;w&yZ?^Yzc{dZokok*D^peOqBMH6s6B^@>8>8nfdOU=9T_11el3lSm*Qe@|-yF%ZS?{S-^?yU9i>5_VT;i>C^OT3UJw$*7^5 zKO|{azkTDXtHPcce>5~zz1bbjmmscJ! zyPyVR*L6iV>8&Y$iNBVw@BON|GKW(MKuskD6CQZ9zNzqakq+1L5SQu1YX$OYpY)Np z(;~+{kXZ`Aj&aY*=j8ciB(*>dCG;^TG%sf>N*J9oLcb3qZ~8AleHwB%@eDZTzWoHv zKeLG;$^r?o$?@Ss2><~7lUyWM0mPHEBs>95lkp@$4dE960C;RKb98xZWpk5&B@~lW zB^m-#8k2Au6_bP|5F9ZBDqJK1002+|000pH0000000000004l2G$4~vAQO{>B@mOB cArK1y000000001hfrpdsB_jqoBLDyZ05JLz3jhEB delta 2741 zcmV;m3QF~rOqWcsSqKjJadM_a2><~6K>z?40F!Mb8k2zt6Mx8CD!P?q-eQQ!WRh&% zPZq&esa0_S^YY&>s4;rZBVlJ}m(;yqX0&ZQPsu$8G(|%qK<1$eOau_To*;V}TRYs$6mY+7O2!9FVNxJ-3nMV1!G*o38JhoL+ zRUV>2HZBq?<7hCD#fesxaFJ&5^taU^u}-iG74|w-=wcJ?X`-duhE;qXPZPP0X|8%* zj__lU1)akreo&HS59vH8J{fUf82M=li`H*Nb1}B&y5Z`%LCNbV&fa~;JZ#<3_K`-b}DV*N?R=6AeC}O%Oj<}F1dbPa>FI`b1hA> zQ>_K7J+fE=cejgUt{}ra;R-=m_y{EA`uPMToJ$OHc6&fV&h7<}kh6OUB;@RV1CkcL+-!Wg*(`xYiaF*Io1FU- zE-^{HAAy9_`w2)$y`QBVjhvQ*cb|_vPf=&I!RH-#!Rz zbAQeSNJ~C~%a?HvJ752M`RU3(?VY_}u7=WlIW&s0@#0Pk<7sm`$^V_^OP&1HTqeVQ zG#kn!U$Du3Cey5+F7BpLTC9E2{=KE?%`A~}xyou5i>cu+_Gl&@N4KD6b3 zd&c#BH7>DQV`we2Wm|jZZU;l>aOlXTcFZLi2UEbeC-`R&UpRexiY z^EiLX;v^jR?+wk?^hUT4Mympeb(B(p#2aUe#FnF(u5MxwT@0ejK_u7DF_+ln`Z?he z1|pD95P^h(2qY9lw**8U&Oj^)?}Bg&qFVwY7(4~hr6BT{7Kypw5|2nO-TxgB>nNoH zh_~*}lxSM6CHCC*c$h?wlSr=qV}CBO>5UJUFcN`;k_aS}L@sgZjgMDCA_ylJ?P*DP z*CU*q(FUJ)z}BHOrUzo^IuuCAM{w!Y3)?gZnN`RfLGoYfUf3w z)}Dz_=$OA{I{}c)EyqCPka3&<4MPxVD1%5tAp{zi&W9Tf!Vo2(9F`^MZhsDLNMtQO z=)@y=&RJ9sMA2==3!o7?`9LnW#{V{ee3=Ie7za|$^XoB5W&d7iZax+MWh85w&GHbTq@ZU9pEsywQFn_{O?rQ7XB{5tK@>LC>fJ8b&41FekR?wXR?A52Wc3SK*N|UGvlBHjL9-Pj$nQq*5ZRsIFrnh1EToCm21Gl zwN6{kzXIi3HR%ZI(#h{9_sx_$HXZ$iN?^jFJ{_QOsFQ0C9)BNn!k>vSTMmfg&qTNe zJQJzen!~C5`42_jsAcC`YfMHqY-VxMXbH-YH&8G0;P?HW29RWU*ED#!hZS7gw&8`-X7NCqfY2!5f;@0Q4CC=;b0mydwX63^Y1|rfF=)`MLCYiGbMKLBTID3qmojtE1nGFjX#Sh(6)>2!QwX3o1=*g#l&vEG# z+1&ba`&1?)c_Js1R0-q!bgd#ct*dYRUx|a;)tBR6PG_X;zt7XB3U@RT1=#pJa-LjYB1sYc8a2|~0d60~vS*lFr zFfUZs^4srZ^@9V;*I7JYzOq#1F3z%e`uuJzgMmzncc!Wg<2ZYMQ=F2`4>xoM{sd4< z0|b+eBpS2g5nv7lN+Mm@C6h-W9)HVD!!Qs<_k2ao)j>AVFIXqEDp8iCC<4Va7iw3c_j(;KQRF;ToHklP ziH_jVs=6tdAxv9%wc3F70$db8$`NKtwD;Z+j2+v9q%H{#6~wK!lKN2E9e*~IebXJ_ zJWcPh1W!5j6rX^xnTTjWayB8&k8L^tXBZlgpuEFblHs%0gOzT!p;IoLQuw2RS?efH z?t>GPv2EL=U3AuvKf|A!m$z=!Sg6yf7@(@)j4%t-Yg-lgvIvK3sPmg};;jJwcntc? z+ptLA2RsS^NFR4pJ{C{U^(7ird79tFX&z_GI$M(bf!wBFr;%6vXP_L0{F^ue`rOx_ zp!x%|hat)W3HWhxrbGz<0Q{3$Bv%2&ld>c{0Z)_fBtZe?~cPyzq|5dZ)H0000000000fPpoWttCDIiIeOlBL+Jo00000G%gA- diff --git a/summary_results.json b/summary_results.json index 9ed0a3385..5ccc4838d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72377, + "Performance_Result": 1.72116, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.29568, + "Performance_Result": 6.27544, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.40751, + "Performance_Result": 0.409084, "Performance_Units": "Samples/s" }, { From 5df51a1ac12e649d61c4227abab3e168ec7f9c12 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 21:35:39 +0000 Subject: [PATCH 125/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 9749 -> 9754 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 65c14e02c..aef39bc1a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 20:26:02,391 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 20:26:02,392 submission_checker1.py:3270 INFO] --- -[2025-02-18 20:26:02,392 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 21:35:34,735 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 21:35:34,735 submission_checker1.py:3270 INFO] --- +[2025-02-18 21:35:34,735 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 20:26:02,392 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 21:35:34,735 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 20:26:02,392 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 21:35:34,735 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 20:26:02,392 submission_checker1.py:3286 INFO] --- -[2025-02-18 20:26:02,392 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 20:26:02,392 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 20:26:02,392 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 20:26:02,392 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 20:26:02,392 submission_checker1.py:3307 INFO] --- -[2025-02-18 20:26:02,392 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 20:26:03,398 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 21:35:34,735 submission_checker1.py:3286 INFO] --- +[2025-02-18 21:35:34,736 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 21:35:34,736 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 21:35:34,736 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 21:35:34,736 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 21:35:34,736 submission_checker1.py:3307 INFO] --- +[2025-02-18 21:35:34,736 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 21:35:35,681 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,15 +25,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72116 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0035 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0746 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43456 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432719 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409084 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2864 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1129 | acc: 76.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a28912922..c505461fe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -752 +753 diff --git a/summary.csv b/summary.csv index 6d83de6ee..f9927e269 100644 --- a/summary.csv +++ b/summary.csv @@ -3,15 +3,15 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72116,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0035,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0746,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43456,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432719,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409084,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2864,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1129,acc: 76.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b8aea5e5c..84fbbec4a 100644 --- a/summary.json +++ b/summary.json @@ -124,12 +124,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0035, + "Result":23.0746, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -154,7 +154,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.43456, + "Result":0.432719, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2864, - "Accuracy":"acc: 74.800", + "Result":10.1129, + "Accuracy":"acc: 76.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 101d6ded7b49b2d7482dfe1877a32fcde377cde2..10f58f0710bf9b9036531826af509643add9a8e8 100644 GIT binary patch delta 3756 zcmV;d4pZ@!OqxuvSqKhdNRI1B2><~8K>z?40F!Mb8h_=STW{jX7KPu>udu9s=$0nN zRp46`7$xaj2qEcolF@mJ98-WbfEnB5^6$q6LSXM@$J5g@j?Q`MhVt6GY8!u4xxB)` zuaDD-cCX?jjAm_J7^beNSr84v*{H2wUwz(lbuCH#*}$JfGu76gRHFad`Om>a6#t%# zl}fcN!GA1i>*F+??>8DrFjiAPF`~JeWk-flJoVG;Z9Hlu^H}+VWzclekfv!jrhYin zR~h!>DmkLzFbvekC|FF@EL|mu)x=M;;V0v8p6H!}L6{xO>(OF0Z0m<&|M)j4^hW1k z+4oJT9+IusT3*Us6#bsRIT^HdGnY1)_VaQW7k{xfP(yz)NiU;^uPPjk)9kcac}NgV zlI6eJG|bMW)U~Pq*w&l6_7D!zah_Nkhl7F2PqeNDizJPvzpW05b%Ir>u-CD|6q{(z z5G~V|R{nWBP1HK3vFdd>!jFFHcMjs{L5r6?By&IiWW>Id*=Y&#)^B-pKDNfX;qtjb z%YW*~&)$8fdC<7e6R#EDX~-3AxTXy^TMdnDJexsDJSfS=VD66Lk}Fy*GE5+`b|qIp zV(&_>fyCLB+yIHYk?fS)#N{?w0zt~<3Np^`uPUxyRb0QSxG7WYRNBIowphF*m2ySP zBc;ABxq4l4%_a16Elsjhtp%$+v{(Xnw||Qxt{}ra<_bYr_y{EA`uPMTtP3RM!#M>Ka&mh>LeB0Pmq>DU`#?g@?m3W((`E0ZfBJs88cMR|&?rpDi#sET zrj6+&`*)fxb+T7;6%TL2*-*vVf=%6~DoJmX#oaVa^0iNLdv6(LBaM|>uCf~WVybc0 zJ6^}fD#_L`Nn^d3+1q5k4d(N`V1K^Yi)OROcri<})zscvbsI#pVK~x%(FZE^!%3ol zIA{#PeEqGClCo8+%7(W5uV7r?SK|_^HHOh@wrs0l?shPA4u_6hYDZjRlPmR@OBi%O zLZJf^3LTJ;ukbHU33NQ%rO=gxcR@J$Mn2sRC-wEf;K_IXnL~->F?9WQ&VMBykz6=6 z5;+>*`aym_vzOhztWxcbG9KqoX%q+J+k0u)hPf9kglttHv5rzIka+EEk=SyKrfW7a zh%N@v0*!0GSOBjhjLP-P? zN+OrI^v1_4ArXX=i}s`>yz3E8PH97rcfi)6G^P(?=sFZg$VYJD)eGCy529pm;x98i zy7tdrlh#pQ1pqI-EdX7^^Q?l2P?(s%Wjg_o%q>SiLZCFpJrZb)P;J}#K8T{*jORcjO!9$TY>oeI0QoWx<}ePFnrGKz zoT%G-VYt~;_?MBarNRnKUabeVg3>XhZM&j?O1#m2DEP*@Wl<`*#u1cCutCqL1R6#q z&@d{whEplZs00ZZm477(x`IARIAtw8(1|B0l{Bi4qUaonG<1^O8va{^a%+ansg$N6 zegMllYOFFAc}3F0*x>Q71a9KxMIUGbjnkq?at%ikXc&?} z!;s_}*P=+a%72gqozR`uvIO1D!51WHOAmD7H7Sxbs*j==l3XK9`fz${{IwvtxXUKO z#hOXNV|qVb&XxHjDLg~|0FiZ6S84Iu+FraGjufK6O(O~nCW3ngERPb&HI7Az-l%2gN^E;}Y(qL$tH7};+yX3tC5O6pfX1P=9iZV@Rxs`$4Z{*>7?x$;9h87! zS?1mm)Le7QT6&-pzWd5dIw*=wmO#T6ur>R1UVo$V=j^_|Q3ubZ*tYOwi&)_n-dK7C zTmpj*_3!|VLrpwD!?~=K)cC5z94T z5nHpT=idSIjXHb2i*0L9zL1qpk+)8om`uryA~rSm0F6t{JwU_AL>fva(oiyyhLS1E z%zr&70VT66L02+!4{Pb6PUv$H7S#t)j7*^6WXhV&J+G1ZcOQ(@QePD)rMU~G>6vc9 zGFFNLD7jC>rrsW)aRr64g0}~07?c&fJ!m|1&qtPddr&6~$}(?{WZoXu(gU4%Eea*` z_Mj*RWd(1Ktl8W18k6ZT$5H&yU1cq`RexE%%56_iwgB)Pm#$tLTYug@Rq;q2tH~tM zf+#y(tH@33>Tmup#s1Ce&*NX@S%1-N5axz{f3o^x{xqD8wB(P~&ji@{&jgrxvS4_* zm?$lMnya?19_QI7lozXOgU8|O2LiNt97b`NKCODsf1zM8@jKIK@P0X{#Vq_|p@lZ_-Av*HnY0R`@4 zA>Uw=VG}NYlGtgksC0zdf~w(C6&Nv3m7AQD>?CnEc9=51ejOT^PMj22Ktihc&hhzt zu09!l*<>lYgOVpKKh%hUH3WIcq9k7*YS&lCy00NFXdcm&<#4Fo1J}Nc{u**FkUX06 zL+w_G;=nX{cncZj1}h+!l9-hl74qM5ZStZ78u42l0FkApg|Ta9G|4p-vMLvcn&mhe zs`BLD3dUQ6;FBv9DFn5253iG06p(*d#~70y-0HyxZ9Jla%2Je>FzdoM&rZiI%b2W# z_3e^|LbjD&LJXzsD$v8C(u;c;sql7rXBnuNR#?B^aasPh{augsh zaPSQxu0CcZpx)l7k4)#^$H?>C0dnnU)SDooZEp5RZB^P8!o|h)$&?%*mlz(l2tp5l zb$|w*5x8E^0H+cNjDbr$gsv{WPR}ow^XX!Fd2}{Eoz|zZO801bKn&aR-QT7i!}l%w zw`I#W9FN$YmWg8-7@;-K{2zZZG20fFFvc~Utv1CD+HnafJV2gn*jNn*@EnwYlpvCv zb_iBbijMYL7vi4|SU!KUxb!HAQ9O(- zmgk#N&V?icWoGr&QWrzlG+wLIIIM_en>|(cX1gya+a8XLhK%nH_g6-$?nOr_v6qtl z`I1Bc1utmHVR=UkUwuq^dinpc?YLFbS=b;S>)wk9Usrk0Fz(+7UQ71JGumPjciDq~ zwPPeOtT#qd-8nw8EZ;=T_J%lqd>cGR_x^qGE{yLR(+})~4mJ_jM5H5ieUWU`I+oYn z7DpYsYsVCLZUuf$-bFK$UkB-@#bI@n#(@%vo#Ie~^w=0^+ZcP! zqNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rLeG1>^M#WUIny~?wvDt1{Is5XoEj6mMdAJB#Kc0k}Ae!Q=*T08?I4c9g&Q* zNe;6c*nRF~-BoX#<}^ZcClbouAG9qUc;jG zEY9wu<+HKtx~R)WtIwa{uifkWIBLk{uqq0uDj3bF0pgUY3VeO>i))E>JHO&R2lmu^ zec^2yvxgzd0tsSBj_XJX0091zS|nEi$&<1qA{*{xA>UvG0049n000yK0000000000 z004l2Q_8HF(o<~7K>z?40001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`Ouob7X>$QH)G-@k&Q=EF?YiZsewD!P?q-eQQ!WRh&%PZq&esa0_S^YY&>s4;rZ z17T-pm(;yqX0&dp1%hNCJ; z>d%M%G@8qf`Xm$e*Y1A~AENm8WFlp%WC`X;N1deU;-J|~f{C2@i54y7JUcRq;+daj zZ{u+@S;W#Gu7YOMrlIRrbLNM0b)DfLu971fjlw{FjDqD%&eL_0SWf*k8-6kg7m3^AoKq z!7@ps*>CGZVv}GUDy&Vc(8V^|)kIsj3^V^co~CjW(_DY|x*Fj}KlQtZarB_Xs~(bt zpMNsqz%a7Y66CGl^5%SO%}vAAbAyuAk)OT$ZtJjlpC{fZzSEFv+HgY~Zg(1**?6{t zGVq`bHU@ov3|CyyZj)gGiMcPi1`=yuaswpxzT_52oULT9+!ijk#S#coE?1Coet%VQ z{i@>TRmFd8nPRWfHmg$s0*CjVxLO<8iBzx7Gu-YS&C2;n;IOYm6 z%oDB390up zmpCNLmuCt1O8g3XAlJxgNqF!1*z**1MjJfdflE&3IgRP_7&_-%fVAWzxO^G+sQdN5 zm!Gb@)Bf4})oLiomP6w(oh{8YXFO7Bh`)=0UJnG=jyl5zXh1@p7JK ztEt9DH3*{lC>*Q5s6(0h;WSY{95zQ_zW!E6N!hAZWv4(o8bfQh+Ll=` zciR{`n?pw~wPP-^$d!7+B@8+sq0j*dg${p6$XECmy97EO?ojAT!h0Z`d?TOjhLid{ zFnIEve{NGEc??~@U2us@B$sxLM2;r|KgjQA8rkj3I#pww@wj+Oqd1rh?hVb-^hU50 zM!N!ub(B(p#2b5u#I~)q9KD4>bTEhx2a#Mq$6R8O>*s_^7>GbZK?D*CB9KrJof3Z# zc{l^HB)kX0DTq!9h+yy(M2CXNW7;IrfyeAHk(tFKjbEh?2(CUuAf7te=0qCat5q3IJZYI{-SG>zV}(VH`@i$gamYk%N1oIoVYBmyxWc!U{}YZw9u8($);iazuXtmH0vX zk>C&3O_Nf|HMXEsf(P`BN}yp>0u7^*YdDpnj7pGzQCX6pC+Jqf8Ef%CC$2%Mq)~ko zMdwJQp_Am!@ZTboJ2PZXrJ-x$2e7Q8#wt_gW?*9T09Z5aR>9*Nfw!8D1mC?eEs7=A z*apRtYdDrb!>|Mzh9%cWl4hG_TUSaw8aCRa4cz5A4IW7s6ZohLAf;k z54fzO#ww3hHv`+=L>euth)cXDdL;N(qiItvxyEi$F1dzt2{epLpkZ8c4d=2|#wAF= zxGYJ~+a7%3lD2rL68iLpMfE`x=MrpWb1qwNnoII9=dUAKM|qWyZ0&y|=?KdxKoW1> z9|^u8Zqf~Hpt0K&Nv`2Y0u4hFXc&@QY>TUkj4UyKEv{ZkQxIruVbeT$xXj!qto)AhM3?DlJ~yyNg%NHiRf} z(})6tiQt|A)1^dmjctEYBDsbW2{epIpkYJ;4I|PlBN8NFM3yAzZ4bT>Nn1S7347|w zqWUO`5eYQxp3;0nK>ibtUTZ0^5|ic*lSWI^^_`1zpUDCjjsPATYQq5)&S9Y-)f4r}p1C!9%U$pKM(;mS2&;aY#EE$3f>@~xV51a;|T z_mg%3m3uZF{e?9xrMrD~>2PI%smicuAwbPuj77uj7 z_e7X22So8b5v~DyA~jocIF&ztQ{;_WcCN*)XUEbE+iVv&R)tf5C9vdB*ACFw)V2dO z9LoyE9i(Ac0u6t|vdp`K5-=>w+&hArYtC4U2Rh+TUzte&CLQZ4IP7NaG!`py*)tV zP;U>=FffsZf{8Q~Or)V;8fD%dlz@U+mY`=aZx3tnP$zU-ghlm16ay1zIG9Gw-k#UM z{JZ-iwbWOINkiXfQg?NyU=?eK0w}pn#G=+7pm79+vVye-X&96htUYKv7KO6R+Jib_ zP?lMH3}%1rVJ#l$#BEb3nY9N+F(@lodyJZ`J+CpDj&dBu58YJOQd^b9tFde8$>x98 zcIX`0-1+PFnT*HsL{6uP5=7bQT1ReM*MIVVB@S-ae;xlK&-#nz!!Y;r`_uK`@~7c^ ztR#P|e2K`rOuA4~cEUGUC2co$e7eo>4MN=L~`f0ky|t$N8{pkZ|n7yekD`|&uO zC(2ZgvO;w&yZ?^Yzc{dZokok*D^peOqBMH6s6B^@>8khF>;WiteD~3Cj;PY*-qCJY-RluMf5Bt7F~Q5EnF$Xv%Up)b4?6 zUq*ioITuJC&H15rD@1W%nmoLPjBErP9+Fcc{SCAw#=lU)>$f0k}p7Vg1G4?bw)5fxOHqRfO@ z7ruFRI%ZkMWF4$;moyZzt@ILNC}mfH9u}2e+{;LXx63UfV{x5ZV-0$F)IP}_C|eVI{!XKp63pbYd@pj1PN_(vqx&H(ykCLF0N0e z_yD=s@GOE6dH~1)8hA$FdO-u6N+2)>F7^<*y7)Rhzg*6zi{<6f+5B``pSD!GN7Dmr z*u-~#n|2J}C-!g4#5Wud+ntuNLktU{HP8GXe=;%K7M8GO)o`}j6gz0gC8Y2GdA8wO zYB+%Bpai4@k>s>Pu!2%_wAZ>2|8&6e`IE(^M@bx4^)sn&)Yph+MG76WzZn~L5AO8f zN;NjP*Ms{#xZ=we$4USjyk{KZo^gnkaoFHJ{^A~gagV>a$6wszFIN18XbtW8)l4Hj ze@nwHvJ8vjVQjHH-;{DLBpE0(tFM;27`mqMTAjvWg^6wURNb5HzMyP-I5HYCzB}As z8L7G#9i_xxO7`bV5&;yvpe2Xp9X5RRG3n{$|HroDR!v7N1AnZ0FCu(h$={~DoqT>x8$yUGCg0OPR}LcDtp!VbT(gJjxQGVe(lPw-VZDi_?Xmx2p!KRk5VGP z0(|0le&7YJV+{sMI=B0^Yykzc$rpVH1u+9ETqKiAARd2DYXUJ4#qa$TOYXbLMk*3^ zS7?i;3WZu)dJD;@p_@M>X;;5}# zjXqY3e1@_RxsklJ25Pj2fac9ZB^(ph!<)Abd{hv!1TujMSEFr=j$s_c7BmkJySVQEJM~)A`+zmu@LP{|p-LG$ZfD{;7&|qS~Wl`dbH-gt; zw4qn7oLa@bgHh{Q9NfF024mNCMK|fKDSwH-map&qs<|?UQwcy#B?S{6c(lH$@O6<6 z*YXgT>BMUV@@b#+k+;(#$3Bo*3c!wW&&ucI`DG-vKn*4IF())HXDdn=oiakd4HYGX%Q Date: Tue, 18 Feb 2025 21:36:38 +0000 Subject: [PATCH 126/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9754 -> 9753 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index aef39bc1a..239eff994 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 21:35:34,735 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 21:35:34,735 submission_checker1.py:3270 INFO] --- -[2025-02-18 21:35:34,735 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 21:36:33,415 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 21:36:33,415 submission_checker1.py:3270 INFO] --- +[2025-02-18 21:36:33,415 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 21:35:34,735 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 21:36:33,415 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 21:35:34,735 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 21:36:33,415 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 21:35:34,735 submission_checker1.py:3286 INFO] --- -[2025-02-18 21:35:34,736 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 21:35:34,736 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 21:35:34,736 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 21:35:34,736 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 21:35:34,736 submission_checker1.py:3307 INFO] --- -[2025-02-18 21:35:34,736 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 21:35:35,681 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 21:36:33,416 submission_checker1.py:3286 INFO] --- +[2025-02-18 21:36:33,416 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 21:36:33,416 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 21:36:33,416 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 21:36:33,416 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 21:36:33,416 submission_checker1.py:3307 INFO] --- +[2025-02-18 21:36:33,416 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 21:36:34,396 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index c505461fe..e29798463 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -753 +754 diff --git a/summary.xlsx b/summary.xlsx index 10f58f0710bf9b9036531826af509643add9a8e8..d4d86aab244ff06df5f3c355b6bea0b618d3a592 100644 GIT binary patch delta 325 zcmV-L0lNO0OqoouMj!=vbHOeolS?2ge`pB8cAza;HBe|uOLsz=B%;;>Nv)H=ul!5` zS!Fr8ch1}yRIZM)iT=Pkp;d*lBt;P@rnyl23Vm$X@e)POQ^jec6;$X14z22kk{QCZ zg*U4WSTDdu0i+yZra}kr4Z+y414!zU;7~!_X)CD@rQKsg*$>?V&eL>(C3woIf2X(u z#%3m>5y{z%G{3g#0i0oIK!WlPXGw<7UJq8fn+?5k;grIk4BWI%;_N;;F&o>qP1>Tj zhWr`++PuE^tHwfgr(%HWk~6|AP_J!W;>#i&uA$Cv!ijeR_|qxqH{QlYhCbj?2tbCo zlk!gOsl#jlS?2ge@F?&cAza;HBhKaOLsz=IHJ}ANv-3*uk1KZ z0$v5Qk?x%{cLo)kqiBOaFqSJ>q9lq@0Fo-kWmBS$dK<1$U>%W+v`RvWda!6yJrq>u zR2g_PN`rA6Y~VrC=2VyH;GE7ew)6mk*vLDi7x&5t;{4S#SQGj~8o+5BKVkune=y<* z9)O{qiD*JHIwSS3X-5E~7+Me@oyAF%;EUIR5q7a)RIZ$YyI#Yh^(@Zrqvf-)>$<4R zMyt=C;IG~5`#5UI<*+IWs45uEsR81YsS133@r!GTbvwV}JqPyGdwt<;8st<5EbsyH zQ{1!inLWSMTU4cSx(eeoOxAUp806W?pYNgNP5lKZrY3h2&wx|z+fPva1G9@E3j_te YOsl#jlba<}0gRLIB_jqtBLDyZ0B6ye`2YX_ From 257cca996e46a70863c95c8f39013490234442a0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 21:39:06 +0000 Subject: [PATCH 127/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9753 -> 9754 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 239eff994..1f9b18cff 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 21:36:33,415 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 21:36:33,415 submission_checker1.py:3270 INFO] --- -[2025-02-18 21:36:33,415 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 21:39:01,381 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 21:39:01,381 submission_checker1.py:3270 INFO] --- +[2025-02-18 21:39:01,381 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 21:36:33,415 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 21:39:01,381 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 21:36:33,415 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 21:39:01,381 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 21:36:33,416 submission_checker1.py:3286 INFO] --- -[2025-02-18 21:36:33,416 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 21:36:33,416 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 21:36:33,416 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 21:36:33,416 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 21:36:33,416 submission_checker1.py:3307 INFO] --- -[2025-02-18 21:36:33,416 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 21:36:34,396 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 21:39:01,381 submission_checker1.py:3286 INFO] --- +[2025-02-18 21:39:01,381 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 21:39:01,381 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 21:39:01,381 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 21:39:01,382 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 21:39:01,382 submission_checker1.py:3307 INFO] --- +[2025-02-18 21:39:01,382 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 21:39:02,374 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,10 +30,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432719 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409084 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405225 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1129 | acc: 76.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3163 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e29798463..20610ea72 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -754 +755 diff --git a/summary.csv b/summary.csv index f9927e269..e2273edf3 100644 --- a/summary.csv +++ b/summary.csv @@ -8,10 +8,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432719,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409084,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405225,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1129,acc: 76.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3163,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 84fbbec4a..e08547b04 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.409084, + "Result":0.405225, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1129, - "Accuracy":"acc: 76.400", + "Result":10.3163, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index d4d86aab244ff06df5f3c355b6bea0b618d3a592..f79d03b112ce09d620cbb42c613f56a93c66d883 100644 GIT binary patch delta 1617 zcmV-X2Cn&;OqxuvSqKHWcS*WOvt9^62!ClwX%#>z3ZUdZ5u18@fW{RR$_m~dq+w82 z@b;kb&^;en=Iue9FeuBsJ(78QSW6Fd;vpHN<`t_>cCs~-r^ z=5ZLsVfwV{J^zJ*#l-JSqrv;-pcb?6kA-^wE_i3}zYFXSzi9G<*4DDppQTxK%U<#q zXjq+txj#~8emn|ii8fKgtWeX)ZZE*&)lUvAU#HQ0`O4C@yC_Yg>GQj>@&_u;-!*k@ z7)9yxoBWh)e!8LaCs0cR1e1{@8nfpSdI1HCdr`1qlV%exe{!*#D=HnKwxDXbR0T%N zQ{^TnB|Ax+jUA@UuV04-rV}Rx7LbrCzH@v&pQ}%XUp85a?x5re%MUedSQ>&nWKoi@ z54G#7W8K#f7c`G(%5pf=?tyDxMt==C7f2q>`Jr|zL~&r6JiLXBa)T9+OG(Vij0*X0 zxi)!G0*&}B9e~JElUSB(W;Dq)6tXH8hZ=Dl4OMyaZw2Emf~}J<6e$EHx@WDET@;Xi zmTp-V?!ie9K4{|+6;zg@%!FANzIk>!W?9B$9jtGcG!(L}^b%qyWmkb77L{Jy%SeT{ z%R6GY(r5%RmBqOXSso{Ab8k_;H`%@--^*34WS3TYN@do>a^t)Gw6;oHuBpk@Y{B&BMwp6-D(*tbS#CLz2b`0Mq_HWC?HyjV!otCjf3=5$(&-@>MGBMj0 zmat{jaJJeMJ7~uxr0@WFw&7Z8IDqG%1f&F!Ly}*SZk@binfYlf|V+NgP-8 zGpTRX*NA3C3LUe*85?#F?)2bFH8!}{gZn+W;>#AtN&p+YXB^_5afp?1*x)_>;vRo- zkH5IbU)$Px~B13oyK8> ziEZ{&-J9*cplo|MG8!_zJKSFxsk#>(rNmxJ_UB6y0TjHTC5Po5HhlFl>FMSF$F}2E zO-C#Pf2?~iB79xtJ;S(vBX}*@AJ1ruN!(=*{?(4*z#wmoq`Gr_WLdsMne7d6{P;F_ zj_&>Y;9VHsH>Mxh2_0-Au8D9*>iQztsC6u_yDg47cGr$6@Z1XgoV<%>Cch5SPm9m! z23TJI{`7QWlPwn=f3^Z850{RB9Bp%y6(MGO1&{5nPr&x%k+kaup_&3`DF7o^VD+o-mz;F*t?MOr?q~Igc z{Bsm=CLX;;jVn)J1*b?X<|T4@43J&|}=Q@;Q5cStTz~ zUF5}mninaVuM0wFk2EKne&kL61*lF#{!Kgsj=66?LH!4_iy_Ja1-W-gx<`{*BwGQ< zleQ!x9E*EVuweuM0CW-n02BZK00000000000Dyr~6O(Tf7Ly7kj0IFx?&2krnz@CG+;6C*{a*FU9`N>d)g}de48MU@`GK(`fL1IjF@f{9~cszYE^k`|kq#!!MfrptZHE^k->S-LjYb z1sYc8VD699nIDhBS)xtUFe}tFvMd|$c=eM5%hzc%U%s+*?Ji2wX!`tato(tB^LI^M z8%9z3{3bsoo1bpz{0Xzj5tso5?qng~V3TDNE`O5PX|AYrgxZ3t;ZhYCF;A77oRsV& zaW;0CGQWNu8kkO;6j(q)s`$?F`FyTE8GhMhDY}D_CoDhIh=DZ(dB~z9Umt4MSI4@q zAuebh(Uj$IsNDnCzKs4FaxRcOn)5^LR*2%jGRIm0m&&rR*xu!=loQdl{+lc6nzRt~44! zOl5H{Lzc(M+T2@|?@hL^$oFzpE7_%$o>G}LvE2A>Kdr6OmTRgsBi+_*3Qs(3jb6di z(YD|zC>;XD1Et2!CTMa0e+zOHATMz64I-{SW+kBB-l&gE=ikT3^V|V)?Pt`RAfat; z_DF42+7-gZ#r4UQ93Yn%9<~TV4}f)m2A&bPUeExi5(tceOFV?GF1}9BFPHP_VtILV zHb0%#r?E=+XnH^l+w$GtrX9ogE&I1+%Qqa4*qxS%V;LBsHP8GXe=;%K7M3u^HJq(B z#SYqW2`M~4o@>}x4F~WXlz@~VlALx3R#1wL_F5O>pAJ|)f3mprD2d~$ekS#e`Wn%! zNTFl)H)F%@!JQslsm2EPdT_r7SA5yxLG7BJ_)B{H zC5pcgt)V@?nrWnGe`&ZymSIsmj4hVun^Mk&Bm-q;_0>`rL)SE3tJ65Fh-I5SRrhAQ zFDTm{j*Nzk?+*7@Myl>bM=7zFlKuITL;wXZXvtxDM+{$mOnQ3x|FP}3RnuA6ARp`A ziwIv=dCxHJ-w0ky_Qx~YViI@RgMYPSBrvQuMpE55KC&#|M9lVvIDUK^JV*Ebeef=f z?;Fz(?1T8HhKbOS7}e}8(q zWs@uy9e>7w5{aGSP=oZ?7--uVd(NV!u-Yn*g91@8d9A7Nz{o0jVy&qsJ z@QGFbA#^;OJW4J372sQr=LcTkI(RTp()qYwW3$N@eFz13bHOeolS?2Tf6HpaFc3xe z{R*SKEk9@o!gio7Sv62-N=tV_nk1su14*ruzpwmE0$F7_x_8dp8C0&0vWfn{I-ymC zvLr1P-n0hmsk>w1qdT4OlP0MFFH7VWvU{?+wA& zu>(lzlHgE5+-WPR52f8>e?!?1-2=|kbb%#!%BiQg1IA`1q7lj2j5NQt=>eQ!Xh4GU z4rfV*&t4B!x|XI|U zEKsj)UE<3k9Im0xZ^DUp0{GJ@=r`WRMTS1$Q3ybWxRdf(Jilz$B&g2Q{4q}RI9qP> zj1&t}6yM#*tNt@kjzjJyPJkiz^(UzR0ke!D$^r#qNRI1BlUpQP0m+lMBqAK{WFg;R z1ONbZ5&!@c00000000000001hfm0KcZW9)h3nh#NcyqxnB$J#aQ~`^V@g*Y$JtF`B F005f$4ygbD diff --git a/summary_results.json b/summary_results.json index 949231890..651032801 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.400", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1129, + "Performance_Result": 10.3163, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.409084, + "Performance_Result": 0.405225, "Performance_Units": "Samples/s" }, { From e3b5c9646d6a71aba55490e6e286224189b105c6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 18 Feb 2025 21:43:41 +0000 Subject: [PATCH 128/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9754 -> 9754 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1f9b18cff..da57787c5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 21:39:01,381 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 21:39:01,381 submission_checker1.py:3270 INFO] --- -[2025-02-18 21:39:01,381 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 21:43:36,120 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 21:43:36,120 submission_checker1.py:3270 INFO] --- +[2025-02-18 21:43:36,120 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 21:39:01,381 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 21:43:36,120 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 21:39:01,381 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 21:43:36,120 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 21:39:01,381 submission_checker1.py:3286 INFO] --- -[2025-02-18 21:39:01,381 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 21:39:01,381 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 21:39:01,381 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 21:39:01,382 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 21:39:01,382 submission_checker1.py:3307 INFO] --- -[2025-02-18 21:39:01,382 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 21:39:02,374 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 21:43:36,120 submission_checker1.py:3286 INFO] --- +[2025-02-18 21:43:36,120 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 21:43:36,120 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 21:43:36,120 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 21:43:36,121 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 21:43:36,121 submission_checker1.py:3307 INFO] --- +[2025-02-18 21:43:36,121 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 21:43:37,157 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,4 +39,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27544 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.30318 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 20610ea72..f03c497da 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -755 +756 diff --git a/summary.csv b/summary.csv index e2273edf3..852a3289c 100644 --- a/summary.csv +++ b/summary.csv @@ -17,4 +17,4 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27544,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.30318,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e08547b04..eaa85e783 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.27544, + "Result":6.30318, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f79d03b112ce09d620cbb42c613f56a93c66d883..f0dc6fdfe7a4494d8acce94470588b41d45131f5 100644 GIT binary patch delta 2740 zcmV;l3QP5xOqxuvSqKE44jUPhT?i(B?Nj2&7RJBdzao{I4>MJ3NO#DarEaP7CL%hH zx^+KUF*J}`AZ!wR`R_LYL^$V3%+Ahktb4!AK<7E9`-J@HPUn%mUmqu9?Or8m6i@ec zVVJt6reWNVro(;x`s(wJr)y~zO#8t&o~nKQNu~O)2mjf7h?C#bky4qKCzz&x`}!!$ zX1lFc8jjQ?NR4==rumUUoJ@i&f13L|8tm(bV)ytrDfHIC-lFfDNIj$*ueG9- zyEy*6cyrR<*UduOV%pD(VO%7CT3-!<`8d0bAHJ$+ILh+VW)~r0JWdz?YLh5Gmr~az z!Q;Me>Doio&qhUJZ4~wUsyNZQ7S7Wwp8U2vBvuKQp~6|k3RA43eM7WON7}{b@ibPe znAWn_#RxwJS#YqI#1C4s=pmg2#U~?nrOZ!DShRjCnv1cuRt=ZW4O(7*M{)M<53Ie` zeUW&j_)bHvXu~yaxY=lE<>OfoO5#CDHU@Kh43}KdZj)gGiM=hk0upCiat$Qzw&Vs# zytQPjTnm?Lu>^vY%N1mt-(OW+y{fo=RdG|L*s8RRD{ZrQNh;-vwogiZU2^rhBVjhs}3cb<=bJx@`mw4uv8@X6^s zqcJ@mL+6}xkd}M|7cb);9DM!n#iz^eN$>RiVl|ZJ%b{VEjplbo7*AT0asKZlU+Uzq zW-1xnM$>^x@&%i^%~YD*rt`Z=loo5B^!DC1%vP2twOD1fip5mxtb4qQPgI((Vba!W zF|)JId>hVYJK=19z7tQUkI8(R<*TWkmFhN(r-Nvy|DyL*7DVGz|FG8@fcg4c6D8%V zR+SHJ@n6ZfzAwimmTL^7ZM7Y{WbSq`bS{UETxv&L;*cx#m`fORKtiDd5(*uVkgxDB zZUuBa+@sJ{gm*wV`9?n73@7z!QAbJ!;9@8c<=Un0w$%Xg7 z17Z`U)By2+#@m__E!(q2$!(91N%T31a zjZ|2J$*a}CmQcEebR18VP>DC%4+Y;?w{1!#*SLaG2{z~%l|aL&1R6#q*KjID6_p?X zqp~7@K}XO>38$>33p(*7rIJSVP!ydbk%mr^8^eE#P;ShS1(nh?#1CNEM2&T(%GJQc zY6IA?U8`jKM&L(Hhk`%7u^oyf*SL~m$u%5HpkY`74a1UaIF_;sOOSwJS&^W#KKOzq zZRvteIF>Z32cp;(D$od1P%h>F0hdkGSZ7;*bv3Z{O{BD(GA{8m(L=!>HQFxal51Rx za>+HEOQ2y~0uAGmYdDuy6_+3ZB9v2}hP7i67n{3jRRcrVq4%#%)t1xrQSNGz>|gVMua~XHz8GRY-z= zPUuc+Rf5j?;0uzpr3*UoEs7+K>Y*rxB-aR&KAhedek}~VMGE~j6;dE ztB3@hFd{1wbk+x7h@>rD&Vz>_WycZBkHcEJpcBp{v*ds% zzHsFluyAeCmh-Pb`BqIjg1U6_`$@Ni$}O9Y`9dWy;ZUCr(74p712mk<8de>Dq+wJ7 z4O_cbxphziMrD;>M^HP>DQoG1PWUqsX3GIl{Fw;XfM+5NTXQ&-KmVb~8@23QiA~Rr zV_2po%656Y5-fowhq`uv#-+9$py627Fzz4?!xCs1mQ~&zlz?Gb<=zq0Tyx4=x}X!j z`^ro@D2h#%K*JWW4f}Lnqw?o}?7qHH2hXM0wD9DMSmBl4So$Sg0)r0q@BocVO*}xu zxvXL1K^n#-&@e8moIEH20lFkd2B{ zYw4m+=yMSk)dNwCOrYUp%7)E7uaWt8AB;3oUmYl=xecXhOKF!tDN3N^J`sm{dw|9h z6v`Uj9;9JV*6{YA@z6aVS>^3PoiHe?ygib6dss^sbmF%ul+4?Mq8OAlygjmEZ_jH? zW`hDp@k4i&jnr0W^(r@iJw5pXz;`{mdTnj|dHX~qLv^gi<5Uad{B$iN*R9LH`M(sq zH_Jbdf05_?#nXON82W?p@{jqmXgbu=KbAie;1oX-U>3>3!R36cwCri7_I33*%Rix_ zSY7Ks4wgRQSNHI4iRJX!waz~Xfl&laz2UAv33ES@~S z8>yhLlH#4EYlAq>p5GLwWc||(2Y&*y$Ps=H1gYVt+>=TmAAieg!!Qs<_x%c^y)8f5 z5QObOTe51P(3F<$gfwwPtp}1?$A4eNahwFa>T-1NoVhcoTpeT^{epEus|saFiXu=< zbD^3Fy>Hj?5=G8a#c8V*RHz4sR`p%U3}M>BtJMaq7vQ1*QjRcFp}qHpVC>i)By~w} zs37jNmDGpQHhGPvFo~|D+X)GpW)BV%iFMOEY$H-3{YKiMwkWawXI8h zU4+B6)cH*~@lF7L?1Mh@HZF4N10IC{%_W+dL!p1t}I^ u$B{Sv7oZ%6{F`_NoO0iOg8C1$j3LSb1fC8X8M9g>Jplx%;iue_nudu9s=$0nNRp46`7$xaj2qEco zlF@mJ98-WbfEnB5^6$q6LSXM@$J5g@j?Q`MhVt6GY8!u4xxB)`uaDD-cCX?jjAm_J z7^beNSr84v*{H2wUwz(lbuCH#*}$JfGu76gRHFad`Om>a6#t%#l}fcN!7OQi>*F+? z?>8DrFjiAPF`~JeWk-flJoVG;Z9Hlu^H}+VWzclekfv!jrhYinR~h!>DmkLzFbvek zC|FF@EL|mu)x=M;;V0v8p6H!}L6{xO>(OF0Z0m<&|M)j4^hW1k+4oJT9+IusT3*Us z6#bsRIT^HdGnY1)_VaQW7qK>fP(yz)NiU;^uPPjk)9kcac}NgVlI6eJG|bMW)U~Pq z*w&l6_7D!zah_Nkhl7F2PqeNDizJPvzpW05b%Ir>u-CD|6q{(z5G~V|R{nWBP1HK3 zvFdd>!jFFHcMjs{L5r6?By&IiWW>Id*=Y&#)^B-pKDNfX;qtjb%j(E~&)$8fdC<7e z6R#EDX~-3AxTXy^TMdnDJexsDJSfS=VD66Lk}Fy*GE5+`b|qIpV(&_>fyCLB+yIHY zk?fS)#N{?w0zt~<3Np^`uPUxyRb0QSxG7WYRNBIowphF*m2ySPBc;ABxq4l4%_a16 zElsjhtp%$+v{(Xnw~Hfxt{}ra<_bYr_y{EA`uPMT ztP3RM!#M>Ka&mh>LeB0Pmq>DU`#?g@?m3W((`E0ZfBJs88cMR|&?rpDi#sETrj6+&`*)fxb+T7; z6%TL2*-*vVf=%6~DoJmX#oaVa^0iNLdv6(LBaM|>uCf~WVybc0J6^}fD#_L`Nn^d3 z+1q5k4d(N`V7}OYi)OROcri<})zscvbsI#pVK~x%(FZE^!%3olIA{#PeEqGClCo8+ z%7(W5uV7r?SK|_^HHOh@wrs0l?shPA4u_6hYDZjRlPmR@OBi%OLZJf^3LTJ;ukbHU z33NQ%rO=gxcR@J$Mn2sRC-wEf;K_IXnL~->F?9WQ&Ltjykz6=65;+>*`aym_vzOhz ztWxcbG9KqoX%q+J+k0u)hPf9kglttHv5rzIka+EEk=SyKrfW7ah%N@v0*!0GSOBjhjLP-P?N+OrI^v1_4ArXX= zi}s`>yz3E8PH97rcfi)6G^P(?=sFZg$VYJD)eGCy529pm;x98iy7tdrlh#pQ1pqI- zEdX7^^Q?l2P?(s%Wjg_o%q>SiLZCFpJr zZb)P;J}#K8T{*jORcjO!9$TY>oeI0QoWx<}ePFnrGKzoT%G-VYt~;_?MBa zrNRnKUabeVg3>XhZM&j?O1#m2DEP*@Wl<`*#u1cCutCqL1R6#q&@d{whEplZs00ZZ zl_d#(x`IARIAtw8(1|B0l{Bi4qUaonG<1^O8va{^a%+ansg$N6egMllYOFFAc}3F0*x>Q71a9KxMIUGbjnkq?at%ikXc&?}!;s_}*P=+a%8&$q zozR`uvIO1D!51WHOAmD7H7Sxbs*j==l3XK9`fz${{IwvtxXUKO#hOXNV|qVb&XxHj zDLg~|0FiZ6S84Iu+FraGjufK6O(O~nCW3ngERPb&HI7Az-l%2gN^E;} zY(qL$tH7};+yX3tC5O6pfX1P=9iZV@Rxs`$4Z{*>7?x$;9h87!S?1mm)Le7QT6&-p zzWd5dIw*=wmO#T6ur>R1UZe7V=j^_|Q3ubZ*tYOwi&)_n-dK7CTmpj*_3!|VLrpwD z!?~=K)cC5z94T5nHpT=idSIjXHb2 zi*0L9zL1qpk+)8om`uryA~rSm0F6t{JwU_AL>fva(oiyyhLS1E%snW70VT66L02+! z4{Pb6PUv$H7S#t)j7*^6WXhV&J+G1ZcOQ(@QePD)rMU~GX-R1nKq(5KA6e$@L7gxt%e+03d3#t(4|L+SD3r|GgQ6Ie6}&yNW^d1H zOs2yeNAW{>@C0>E=zx_WJF{dxOT#UpjBCX+-9qU?07A~&t8zxlru z`!}mUkAIP8{YA4um>c^2$?A{!({MJ@l0Q~I6JX~*6JX}ag5l+2qO|mBuG+eKoM)d< zUaYPS9*3(R2+-zn7{y`wwCX+og@VPz?@Xh?`{ke(v+$3FdjBq0cxUgw3+xZSX!3*B z*0R!{rCD{$Uh)@cSe=8pKT>CYJPK!tHc`W@P}9h6z~j|V4lG}%(R}&J(zUxNO{3}a zyRq^ID$d_Eb!`|$>GPZXlx%*wq4OuR$Ps=H1XNY-;*&}sAAiki!!Qtp?|llRyDiJ1 zB^9;1 z&{5nPr&x%k+kaup_&3`DF7o^VD+o-mz;F*t?MOr?q~Igc{Bsm=CLX;;jVn)J1*b?X<|T4@43J&|}=Q@;Q5cStTz~UF5}mninaVuM0wFk2EKn ue&kL61*lF#{!Kgsj=66?LH!4_j3LSb1i5!fy0cm&Jplw%Rqo=Gn Date: Tue, 18 Feb 2025 22:43:47 +0000 Subject: [PATCH 129/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9754 -> 9755 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index da57787c5..00bcd0245 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 21:43:36,120 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 21:43:36,120 submission_checker1.py:3270 INFO] --- -[2025-02-18 21:43:36,120 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-18 22:43:42,228 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-18 22:43:42,228 submission_checker1.py:3270 INFO] --- +[2025-02-18 22:43:42,228 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 21:43:36,120 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-18 22:43:42,228 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 21:43:36,120 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-18 22:43:42,228 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 21:43:36,120 submission_checker1.py:3286 INFO] --- -[2025-02-18 21:43:36,120 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 21:43:36,120 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 21:43:36,120 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 21:43:36,121 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 21:43:36,121 submission_checker1.py:3307 INFO] --- -[2025-02-18 21:43:36,121 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 21:43:37,157 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-18 22:43:42,228 submission_checker1.py:3286 INFO] --- +[2025-02-18 22:43:42,228 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-18 22:43:42,228 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-18 22:43:42,228 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-18 22:43:42,228 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-18 22:43:42,228 submission_checker1.py:3307 INFO] --- +[2025-02-18 22:43:42,229 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-18 22:43:43,292 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405225 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3163 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1457 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f03c497da..6bf42b950 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -756 +757 diff --git a/summary.csv b/summary.csv index 852a3289c..d0ba265bf 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405225,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3163,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1457,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index eaa85e783..833ebdead 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3163, - "Accuracy":"acc: 74.600", + "Result":10.1457, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index f0dc6fdfe7a4494d8acce94470588b41d45131f5..266e7994d9aca74d07eea016cce729d3ce9e4357 100644 GIT binary patch delta 897 zcmV-{1AhFPOq)!wU=szpg^*QZvt<*R0)LJn-w!1mz;jRnQi4cw+96m$DLUF~U5I}= zVEO#X;?koej;s2a)HmvDM6)7=j@jRg4Z8<-dT^y08{F%`{T^KLWs74afDPU=4sp*o z#L75q@E(70kH5IbU)&30c+i|O=BbI?b*1Z=IzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8 zEZ@xbhB$tF8$3t%{(bN+jPD!M4}a{04mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$ z-bFK$UkB-@#bn)CTY(aZ zo#Ie~^w=0^+ZcP!qNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rLZ= z1d|aZ8nXi!LkI*3QrVr8N+2J9&1%Ci5QOi23ZuI%TTV+5wgYX+sewXMT6z=GCK0uM zkkmSU`zns(B#={&tC{_Fc2Tw1t2X%s=cUm#DzY3Upt%uJZ)^0vUZ!)D_`tMatSa3mtPa;Tr%59ByAy|sB=Q~i$XVIafxG_$#5KFgz#g_4JwgpV` z`~fQnOt8T40hro}h(@H~6Vm>;b^r*Dp#=pxcwA%!K6?W=?~cPyzq|5dZ)H0000000000 XfPp!avL!wNjg#>uBL+Vs000008)}}{ delta 909 zcmV;819JSEOqxuvU=sz4dr`1qvt<*R0)MvQT533e=b!|n1d-&lL$HEUbhOvH5dU<* z^7)g+rAJ8|SM@WgZ`9X_WaECYY6doLnXp2eQWe@(@ z6aWAK2mk;8KL8<}UL8jQ0JHlSOb7(2;iue_OCTSA%WA_g5JmU>3ZuO(KiUw4?Lb?y zYM{`RmhOZ!aYU^Ll3K@qU&V2p1ib2Ubnl$GGpJl0WE=g0bwaBOWl4%6P)u{7nhL#d z*YOfX&Qrx{s})qJ2ZvVmUC9h#+QO^V2CNt0q5x8kFjJwu_l98X*d8QxNpPqj?zENE zhtf8G*iiOO8^C#*F0ceoIrS7Dfw7s0Xhd>0A?~cPyzq| j5dZ)H0000000000fPpxZvn4(OjFa*uBL+Sr00000O@po% diff --git a/summary_results.json b/summary_results.json index e036dddd2..a5427868e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.600", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.3163, + "Performance_Result": 10.1457, "Performance_Units": "Samples/s" }, { From 408588faa8b3dcd07ca65fd0be68d8561e5778a8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 19 Feb 2025 06:44:10 +0000 Subject: [PATCH 130/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9755 -> 9756 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 00bcd0245..ac705c9fc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-18 22:43:42,228 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-18 22:43:42,228 submission_checker1.py:3270 INFO] --- -[2025-02-18 22:43:42,228 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-19 06:44:05,561 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-19 06:44:05,561 submission_checker1.py:3270 INFO] --- +[2025-02-19 06:44:05,561 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-18 22:43:42,228 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-19 06:44:05,561 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-18 22:43:42,228 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-19 06:44:05,561 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-18 22:43:42,228 submission_checker1.py:3286 INFO] --- -[2025-02-18 22:43:42,228 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-18 22:43:42,228 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-18 22:43:42,228 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-18 22:43:42,228 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-18 22:43:42,228 submission_checker1.py:3307 INFO] --- -[2025-02-18 22:43:42,229 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-18 22:43:43,292 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-19 06:44:05,561 submission_checker1.py:3286 INFO] --- +[2025-02-19 06:44:05,561 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-19 06:44:05,561 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-19 06:44:05,561 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-19 06:44:05,562 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-19 06:44:05,562 submission_checker1.py:3307 INFO] --- +[2025-02-19 06:44:05,562 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-19 06:44:06,548 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72116 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0746 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432719 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6bf42b950..52f555366 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -757 +758 diff --git a/summary.csv b/summary.csv index d0ba265bf..b1d61cecc 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72116,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0746,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432719,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 833ebdead..c06d42aec 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0746, + "Result":23.0058, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 266e7994d9aca74d07eea016cce729d3ce9e4357..a767ee9c3b265eb35107ca31ef19feda909e0882 100644 GIT binary patch delta 1066 zcmV+_1l9YSOq@)xSqKFC6pw$iT?n%Ve=PF@h!Pi$03I7^!vPwX`fz}TGg-rlgEWju z;KgA~R+({7CydD|JC0y}9M;kWop2_ZB?m?xoPPz%w`$T6)TNW(Pr4;k zZrOCq7b<}Xhx&AY#-&Cbpy5>3u<9TUqY`M?+O^89gAy<*tNc2G+G$Q%OBZy)f1imk zTMmfg&qTNeJQHcyn!~C5`42_jsAcC$Y!<6q_u8hAm(l_UXJv<l82VO&-@c~AnzWtEplFf$Kp z>4Hu;m(0%tqWB_~YrrD5VNcJ$1Lhlb_Iwwc)}CS^E8Q}0ois6-k{d-FYVH9VkD7ac zhLMRhluV?dWFieEQ&yRKPy$M3Rf3LW<{sA4MV-*+A}p#0q8OP#!^xBlf17(=BlGV* z7-^)wI#5b;8%oob(k_8glt9USA`bQT0F5Uolr_9PNW-A4;q5`=p?f~E%G-lFVNh0i zdnEJru$C_9#BWn5nYRZ;F(_+zdt}4jp4XVn1_h4dhwdsHsjbfHRc?BE@&$nJdUW;L z+W7PKiAsj*SdGW27RLGMe_BSaTbF*{fqe?mpEy4HUjEPo(Cn}{=HuXC z68GOP1~s2Xf6UeUci}r{_g(0G_(hW+w0$iv{aKnMTfx(KOY@ zYLFLd8u<-)vi!+`#p^7dEneBWb{A(^Jb8XMQbAuO#XC#a263D{zbQ`1`llNX{sgng z5q}N^J4C&HCX-AcAAfGbFc5_AeTvBU#7>}5Yb8NyD^5kKRH~xhEPDZ~@sDh`fw!*{ zAb`ZF$Mwv9JG*i<+o?AB1?Q#FHOe!B642ZTsW&zHSkBW2l=#53V6D+mqdj;uYwjv; zDK`$@oU!171fN8Z_LSQiZ9}jWW6!ssn9rg^M{#YOVj-4pgMTgK-)sZ8Af&_!0uwAS z+yhfP644PU_=vPWt{nh^V`xEv4j$)Oj?dlzPWhV+gL2_iGVCqfw4TJ#z4LN3c3qcs zlffGE=XkYveIHg$rS4C~0X3E2lsjO-xTeCFMLb-~d|1R2uO*15ebjHf9Tz$DfkT?n%Ve{GjBS>nPGz+*#gI6&i49}duPCTloxkcKe{ zyf}=>Dl-o1gfUrV#}Ukr!&Fe%Py$9}m0w3tJIyI;>4Hx9e=`wg z%K=gRnF!Z_XCe(-b2ybh|Dnhmwd`DpP0x;FSf(Y)c6q!KEP*A5x^{rZrM4ZQ;aJu% z?jQ}r5@;BfRo)$xfMHqX-VxMXbIMw}pcB6P%1k;aicOY4!xpd&`*dET^5^WnzEKCy zrP#Fa~xz(|%MK`h)TEkNLA`I@HoX zmOm5V6h9MS7RkcF<$SEP>}jUqed+`ee^Ko!6 ziTm#tgPKpHKj!NFyYQW}`z~}o{G!PZ+P;>T{w&R_Tl7-AK*KuNJ(~qXbrvMUXqswc zHOLD!jr;~YS^ng};&m3!7O!kwyNk0do;<%Bsi3ct;+>^ygE-Ef-xQ~0{nHHxe*&|} z5q}N^2~ydeC6i1bAAf4YFc5_AeF~$yEn7}Y5Viwt$*F-tQ(Afx(k2nLevs5Ue)}qp z<0Oz%kE@yec6L#<*sC`A1?Q#FH7c?kC7`(xQg3VYzFwwtl=#53V6D+mqaHk3G!8Q|^ET5Y|8y7kDfkxsvs@%S0R;(C*_|bmo+VTPjg#{wBL+Vs000008;JaA diff --git a/summary_results.json b/summary_results.json index a5427868e..e6408b90e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0746, + "Performance_Result": 23.0058, "Performance_Units": "Samples/s" }, { From 1a790818fb105da8380abcc2e980fea3ba251218 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 19 Feb 2025 19:09:39 +0000 Subject: [PATCH 131/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9756 -> 9756 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ac705c9fc..27cbfd173 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-19 06:44:05,561 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-19 06:44:05,561 submission_checker1.py:3270 INFO] --- -[2025-02-19 06:44:05,561 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-19 19:09:34,497 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-19 19:09:34,497 submission_checker1.py:3270 INFO] --- +[2025-02-19 19:09:34,498 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-19 06:44:05,561 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-19 19:09:34,498 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-19 06:44:05,561 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-19 19:09:34,498 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-19 06:44:05,561 submission_checker1.py:3286 INFO] --- -[2025-02-19 06:44:05,561 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-19 06:44:05,561 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-19 06:44:05,561 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-19 06:44:05,562 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-19 06:44:05,562 submission_checker1.py:3307 INFO] --- -[2025-02-19 06:44:05,562 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-19 06:44:06,548 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-19 19:09:34,498 submission_checker1.py:3286 INFO] --- +[2025-02-19 19:09:34,498 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-19 19:09:34,498 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-19 19:09:34,498 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-19 19:09:34,498 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-19 19:09:34,498 submission_checker1.py:3307 INFO] --- +[2025-02-19 19:09:34,498 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-19 19:09:35,516 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,8 +33,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405225 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1457 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.38162 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2594 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.422164 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 52f555366..e4978283c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -758 +759 diff --git a/summary.csv b/summary.csv index b1d61cecc..b799848b0 100644 --- a/summary.csv +++ b/summary.csv @@ -11,8 +11,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405225,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1457,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.38162,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2594,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.422164,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index c06d42aec..bb4f93e4f 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1457, - "Accuracy":"acc: 75.200", + "Result":10.2594, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.38162, + "Result":0.422164, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index a767ee9c3b265eb35107ca31ef19feda909e0882..31770ba86618c0ee46ab1afc8ea9d039dd9e9ec3 100644 GIT binary patch delta 2161 zcmV-%2#)uhOq@)xSqKT{=E34g2><~9lVS*B0s6DW2v`Jv^fBW(&2)jj7p$kRB{ccQdCh15-=(&5_AOJN;qXLUC@ayDU~#;hoa~li8OSQ+!+2_ zgmPnsEU1)!rXhX+%O+~9GgYnzCRPuC4coO!9^VMO)pRKM?v3qGEV;&&6icq*SON{h z5@;BfT*I-HRak-q49kiHo%O*NENM#@bi%QuQ9TgF9-#t_Fa_mO{vUAJM2&SGtF8vN zzKN8UQ^qCU6Fn4stI>8Tmt5moluNGRTmlW_5@;BImt4cSw5qrS2^g0Z2|DY8FI>`= zE~rHb>9_IXYB%3I&7LwK$l5Hu4Q?kD(@b3Mg;5*_r-O&ab zw@sPk8qOrpFeZV9G08QaO_^+0F$p@Mk6Nn|bk+x7n4~RT(1~wRCTUa;MKLD1MwoPW zdSm>5jW9XC%V)y*3Q7WGc0XB6mc=Y7d_(>Kl1)@sYxUaRT)i5u6r#jVBT5`5f|~|x zpAyM6u1$&L8crn8Fd~755eaND4kgmAA`*1Mh^$D^Ss#2MlD2d~C+x8+i|U~$MkLU% zn@al)0r^indu^n^T1?s-OiIfz%`!uPD6!#x2w<|IJ{+KNsSyWgIFmK3I7q{o1ZEt@ zWR)8Sb;6je^5Y0*$YCvA&)!?CR4+(8(l9Q8hH+VCd;aX}qfajnDqy&-Vv_r6hQgpP}x)A?#!1DQ%#id6{99Q)-sc+QR zh-O6!9kag~8+H%w^x#T0Hn`V=`#reg%NEB<02{n#9O9mFh?Q~J;64809)EFE-{&w&PY!M=S$> zta~pad|l-|!?=GVcrDo<&uEKD++`2`)sEr7Aa9JMx^sMFS-zR=4RQSVHh7Ni{rliu z7~eOhAJ~5h9c&`5iEu~i`Xbq=bu6#DEsi>N*N!Rh+zR}hyo+WgzYfw*i_ho=SYH4B z^mJke&P`jf%B3&zi#bw0`@YR4vm}>9H};wlVgcMNMI~ zRUQWwY6g`ek5QAl?!8k@U3N4-Kz3j_fwCbztsKC^4h#~{p(8l7 zs_#l>2-6l`tu|o202c+2a)g-*?Y%byW5@O&sY`-G1#zpbq&}2(hkp%a-*g8!Pt$uW z!Bb8>#V25FCL$V@~Q$qJGdWU>4@jlAi<0OdI3-^4Ru$bI_>>OZrN zA<6;?=H|iTNeKV||C3=PR{_eCxg;V5i+fS9VUyq_6#`TflWr3hlMN**0q2uAB{~6F nlXE2|8z#K*_ay-U08jz|01*HH00000000000Dyryle8sh_O<~8lVS*B0q3*D2v`Jvber)UXoN{Vkc*A+zYQQ? z=D`BSzEZRNdQ4JvdoK(xp9=pnl8sbYgUPGaz?M+DhIAZHlu(H`+7AWaShsCTCD*uu zQVBNb8I?f8s012DCD(8&MHQ7G0i&`aK}XO>38$>33p(*7rIJSVP!ydbk%mr^8^eE# zP;ShS1(ni&G{g^J*+h+Xrpnd8#A*ZBuwAQU`$phLO^1R%y|EpNCD*u;V#zfeOQ2y` z0u95GYdDs&3QLfHVOf!&vp)EOC2i?~PB@k{st2Oj7AnvPQ&2AD{{fdx)L3U*bv3Z{ zO{BD(GA{8m(L=!>HQFxal51Rxa>+HEOQ2y~0uAGTl504ZRuz{Z0pqeFL1%sNg-hDf zMU~Lq8y3|AQJhQgAe(b(y=gAVX3k$nvWfC)A!%(R=?O=cAc-H|9}50J+@=q-fyQl9 zB)Nto2{a5zpkYXIjb~FN+f_({PUuc+Rf5j?;0uzpr3*UoEs7+K>Y*rxB-aR&KAhed ze>sr>m5Mc$}o=SpmPb{xYpEm1aEdL>u_OAdAI0F6s+J3zy+tYO?i8ipm% zFf6OQJ17CevdX<9sJZ5pwRAxzeD{@^bWjwVEP;kCU>o-7yhi2E*?oPZ4xUS~Y2nEi zvBE38vGhy01O^@I;Q<<#ns|VQb6LZG#)CABOQ2y~Rylc40>)*Pmq#!&4{PayPB@p$ z&jX_PB9?2wBDP^q&%Xoa8+G=47n|0eVj(NtGH;zUF`1GZMI37G0UD2*dw_( zN^=`Z)0WaMfl`z}$$cUY_4WXbCn%IPygf+6pseBTLF1u&KC;T&gF0bQR(X3Q^Y*Zo zF6hK>Qz)6Y2SqU`Yj}HP!``0Pn9K$Rj^c;zDjTV-&gxZedV2B&fbV*A_1fC_^Y)2K zhU!?2$Eg;^`RQ6lu3MLX^M5ISc5jw{9{(cG`-`Xjs4(;g-!BF= zpGJSo)%$nhJ7@P@=zRD^lOMEwEie69npd~zrFemcb+9)JhUzRxhS4-M)y8U&7it>$ z4S2Hr$$`b|ES@c1*}8TYXIVUXem7De0R_5+ zkX2%{XA_wMe~uyF4<#JHb5H_Of=F`OAy`2vI@)Vph<`d@`TWV^(xW7ftNNMLH|lFd zvm%9#+24!}y9alAaHSd>-0Q*p9$fKdi(@5#4c;>janCr!$~bKB9)EFOX{zXOl-MkzWBmaXdfp0@twy10|i? z{aUtyx?dm53;P`_`GcSGNZ=?zeXv%a*|lbJE=K@6aWAK2mk;8KL8<}UL8jQ zv-=lK2n9Pty?!Q>O&}kCZo)7SgztTd$oIrfpiyfjL24^bMXFS)qTVcf0ju$kY`1~8 zuM;4E#Hq*i%zit&ay8qjHu(kTrO`FYGlCM(+z6>RHTqc2(+8CJz_ehk(NLp3crO;ax`{b zmvxiD8uRCPwRn9WR!ybuPsIT>mEe>+V8OVi!k0xnT+4h|#1pS2h^KwjZ@e8BIrM=@ zVgP!Gds03p&o9d*Icf@0+^3{S^T#DA>2yj-`PGlS>OTY3amc@kC%_^1^(Scl0JDxE z$^r@d6pw#L2><~8lVK!R0mzfNBq9a6g^*QZli(y30#Xx`a1$1j4J9f8=94%jIssUd mb0sDlJ4C&HCIJ8dPyzq|5dZ)H0000000000fPp!av?XZYVe?x6 diff --git a/summary_results.json b/summary_results.json index e6408b90e..b465950ae 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1457, + "Performance_Result": 10.2594, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.38162, + "Performance_Result": 0.422164, "Performance_Units": "Samples/s" }, { From 6de2413760b05dff89e16b9e41bba8c1a661a4ed Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 19 Feb 2025 20:08:41 +0000 Subject: [PATCH 132/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9756 -> 9758 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 27cbfd173..5b58585bf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-19 19:09:34,497 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-19 19:09:34,497 submission_checker1.py:3270 INFO] --- -[2025-02-19 19:09:34,498 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-19 20:08:36,252 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-19 20:08:36,252 submission_checker1.py:3270 INFO] --- +[2025-02-19 20:08:36,252 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-19 19:09:34,498 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-19 20:08:36,252 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-19 19:09:34,498 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-19 20:08:36,252 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-19 19:09:34,498 submission_checker1.py:3286 INFO] --- -[2025-02-19 19:09:34,498 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-19 19:09:34,498 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-19 19:09:34,498 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-19 19:09:34,498 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-19 19:09:34,498 submission_checker1.py:3307 INFO] --- -[2025-02-19 19:09:34,498 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-19 19:09:35,516 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-19 20:08:36,252 submission_checker1.py:3286 INFO] --- +[2025-02-19 20:08:36,252 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-19 20:08:36,252 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-19 20:08:36,252 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-19 20:08:36,252 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-19 20:08:36,252 submission_checker1.py:3307 INFO] --- +[2025-02-19 20:08:36,252 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-19 20:08:37,238 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,9 +28,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432719 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436551 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405225 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2594 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e4978283c..a38f20ae2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -759 +760 diff --git a/summary.csv b/summary.csv index b799848b0..874f91449 100644 --- a/summary.csv +++ b/summary.csv @@ -6,9 +6,9 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432719,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436551,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405225,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2594,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bb4f93e4f..64c8d7a9f 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432719, + "Result":0.436551, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405225, + "Result":0.412006, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 31770ba86618c0ee46ab1afc8ea9d039dd9e9ec3..04f053b658725daa03de0c1e07d463dc66017b0c 100644 GIT binary patch delta 2797 zcmVjx|L+!VvNaT zl5E{i7Qt4jRdE6H^4~A0F?!AeVP|KT)V*J3w0X|yKFBZK(2pMe`Z${^_cBhxXx>(Z zrmKpa2hlK`kK5|a_2-7GDoN_khyFC0%eML?6ZO~5e-0m_`1fQYWvXNe=1E(ff28T+ zpxI1Y=7)22o#7y^k|P?8!a#nE zg5^xk({++qPW?0+eliIciP||FhS{;a9wnBewt6HEPJT0l+Uy*z`o0b2L$dQ)$xFG5 zqTlm3r^B|Y=h7zAeqIgZGFFCif8;Nx=~eXbRfgk9nw>T)4+)}avietQTd$rCG+a5$9tiB^?hnWWL|xAh^hNw5wT_9j;7VjJygqNUr0m46;jQ@M$0 zu6tdL@S~slox?bKP~ue&$->V+8F643*=Y&#)^B-pKDOqj;p(|T$?C|@f8Kq^JZ#?Q zi8qSxG~}8#+|Y*GorY#Mp6#FvJSc;WLEj(46<4%cWSBr=?MtqK#NLd8#O0bSfgt5_1sUh}R~6TLeB0vml)*i_JM?)-3uTgXZI3F$l3h{BrSZo+4yp^Sptg`bIc_+ zIrk@AVv>440tu=26OfR4KXZvovV3`#fIo>}K@a2_IV}nAJ|BCYf1=K4Lyvdhk<)oj zWBNRX&N&w#E%^v8U&cM^eEsj`r>ov+|Lpy0HI!t_p>dc_mUmhZ&6=}m_U|lP>SV7L zG9C@W`AEjuf=v!mnWTecc{dA_eC?AA?k!Dkrm>W(RaP@!Of}DYC!6?KCfOP$X>JxX zjcw*Zuvj#L#j+92f9H?!a-L?Zsm4Y%2%`BY9IL;mLz(*FG*LesHb-E-{#Hjx*{W4$ zLtFh02QhZ4zS==$w~OFSaEe{^ajay%LML4H5e$ZlWO zsT$*q$Hh|`#ld87Z)morH-e=wS`|pFqm&9H-Z(oXwj9lLbrXZ=Vh~*pBDsE!xx^;d z&k2_>5P^h(2qY9lAfX_-B_Q%}24YEg7lcy~-4YPN;3yZ zrzPQCk8pBE8+yD0whpB+eGo&}p+G`Df=jPn*k*nZC5@@S%JAsgKYLADM|l+hyz+Jc zbT!Yj3MN9KWByj{1VA#k90QF*#&H5P3_+x!3?dDMe-LO~Iv;K|2t$;Ba#)t2yFIui zk+t+dCmzXj&Z7Dtiaut%02-l_59D%Z{BHxumw7OUaVX^?yB_024(^5KW>eu`MzWR) zD=>Mz8Q2<1M>A~O6$Mn{2kl3KKUlXcN+s7gf>H?{&@(E5hEWMLj7qNIREjbxK>|i) zNrJARe_IJ>tfdD!@eE2Ojq0N)I!7W6og{aL{}!R#nIUs34P6sIfMp#uR+%a{0~4DE zz?$Wl1&?n8-fB7$eD}t(DVAL07!*sc;aCCZFIdu+ z9_WN)Nu&B8iakOF8lelyrSX5jWgRtEd91n_f7tdW(lG5JF7ck|k>Fd6mP5JZ8potu zat-GaXc(73!?@%c&ZSw#B}l-yEJ@Ja9(>`Fw)9XX^yv+Y>VqiGCD_R3T$*p1OY$)1 zuOnGUd6kef_mFHEhOi6v7X{wEKN5UL+@d?$K;yJ1lU&1@1RBO9&@d*s#rS*n@{3o8h)>2?4CaoPN4O7$g zB13>Eu;B<`vY|d4pmC@X2WU8x6|6W&!eK-mPGtqJe-6?x zDuITrUCZn`C;_9g%&;SGQPY$*10F6U^J3zy+tl->18ipm%Ff7Z=J17Cevdq3CsJrHjwe&zI z{P8Pu>7Xb!Spp4Pz}9Tkd5y}SfAjeDjaqoF#IA=YTf_>t@W#?B;1W1=sEG$?9O~i$ z8qQ?}9}m(nE`f$|S!U%y2^g1UW*))ZJglV$I^kS0Ll21Ji&(A!i`be?J^v1vZ`9iJ zUF>>$@`bEz?byZlzZ$1wG7at&v8lTUXk6;<0UAao(oiyyhLVXiluV<{f8B!;P%_IB zbPeY2VJ$t>3EdcBQGF1_$OIZrrctxI=QT3_?#@Up^;Ln=(D$Jf`L~D*pcDmAa-)b% z%{@Tl3JPTfa}UxmC@YwI(0J%GAEV6NgF0bQmYI7DX6|7vJM-C_7#2$ZhNT zOaE8m;CB7}_!oKBUo;06YJk06k9@jINebsiaQ}*;eJ_ zBKw5$VpV1MI9mTgfU=0gC=S!7b?^C46fCEHXBG|LuLiZ8hkq>PTKjjwJNw{WV1M{U zF+M15B`f_|npL;zC4Yg2)j3@FV|nh!<8YoRQ#r~C)wS#fJYN6i!0L4xEmp5ARk@4O zG@3oXn@E2s1&{5nPr&x%k z+hNQ2H`@U&2$^FAfqw}W818|o9f|0K6nsS5AJ+^3!7((TKnIWWEXNmb0H^$9!=PL_ zl?Bsm=EiC z;;jVn*hPKf?X<|T4@43J&|}=Q@-cgQ-mFkvkm5ciMVdcs3K>Glg3jh&{m7gC3s9Yg z{F`_N9CP1(g8C1$jv>kd1vfq@+e?#OBwGQ@ler{30#*}~a1$1j4J9f8>617mIssdg zb0sDl3fC&NB>?~cPyzq|5dZ)H0000000000fPp-dv?V?PkdyNzBL+ev00000ywWQ% delta 2773 zcmV;`3M%!UOq@)xSqKH@=E34glU@iWe^cVf7RJBdzao{I4>MJ3NO#DarEaP7CL%hH zx^+KUF*J}`AZ!wR`R_LYL^$V3%+Ahktb4!AK<7E9`-J@HPWq9(Umqu9?Or8m6i@ec zVVJt6reWNVro(;x`s(wJr)y~zO#8t&o~nKQNu~O)2mjf7h?C#bky4qKCzz)DfBGoP zX1lFc8jjQ?NR4==rumUUoJ@i&f13L|8tm(bV)ytrDfHIC-lFfDNIj$*ueG9- zyEy*6cyrR<*UduOV%pD(VO%6ye_suP`8d0bAHJ$+ILh+VW)~r0JWdz?YLh5Gmr~az z!Q;Me>Doio&qhUJZ4~wUsyNZQ7S7Wwp8U2vBvuKQp~6|k3RA43eM7WON7}{b@ibPe znAWn_#RxwJS#YqI#1C4s=pmg2#U~?nrOZ!DShRjCnv1cuRt=ZW4O(7De{uHi53Ie` zeUW&j_)bHvXu~yaxY=lE<>OfoO5#CDHU@Kh43}KdZj)gGiM=hk0upCiat$Qzw&Vs# zytQPjTnm?Lu>^vY%N1mt-(OW+y{fo=RdG|L*s8RRD{ZrQNh;-vwogiZU2^rhBVjhs}3cb<RiVl|ZJ%b{VEjplbo7*AT0asKZlU+Uzq zW-1xnM$>^x@&%i^%~YD*rt`Z=loo5B^!DC1%vP2twOD1fip5mxtb4qQPgI((Vba!W zF|)JId>hVYJK=1;e-lrqkI8(R<*TWkmFhN(r-Nvy|DyL*7DVGz|FG8@fcg4c6D8%V zR+SHJ@n6ZfzAwimmTL^7ZM7Y{WbSq`bS{UETxv&L;*cx#m`fORKtiDd5(*uVkgxDB zZUuBa+@sJ{gm*wV`9?n73@7z!QAbJ!;9@8c<=Un0w$%Xg7 z17Z`U)By3uf7_Z8E!(q2$!(91N%T31z4clUJ*OEunM`={TM!p%Oo6KNS4Ix@}V`xyBWgO7MW5Q3*7RN}yp>at)_aR8a{M zFe)n&e{=-hN;qXLUC@ayDU~#;hoa~li8OSQ+!+2_gmPnsEU1*GA$|bMCTgrRRjvjm zRu6y;+qFs_-w3?bbSU`ljqOk@xyF?gORnKq0u93wXc(4U!?BcASb_u$%Zdb@^}!b` zX-gM$!m*@LJrKnnp#qIC1?5uyA8^@3jddQYf360$zKN8UQ^qCU6Fn4stI>8Tmt5mo zluNGRTmlW_5@;BgT*JAvs<;FR7?%|ZI_rZkT+)^;>e z=KOUen<%dqlGYZIZ7GFQvcD+s?){JFgd@=XTtdkN&;kdKUqwc z#Vjd&L;e7gO;lHF_1fNCy&A3*qQp)kN*pGFn+9y363I2LO^M_hP9)GUB7uex32ZSA zCDN`U5_H0dtVqyVAABK_wsb)!?6E70f9jzqMkLU%n@al)0r^indu^n^T1?s-OiIfz z%`!uPD6!!PV6ve;9H4Qj5eH~ElQpb3NW+)}W*o+3l^X|j!kDb`;|ONRVJ%(I31^ad zazGScxpECyxi;y``B$KPt1cZuZ94f4rCUPfmQTlgp%S=os8I)KT_TK^lf7&@e2k%sVIn!?Mc0BdELPl(lq0 zC;ag%bLpTcHdz7MK?xX_Rc0Q++&rwM3p(LkGD8oD;)__W0gKp%O+Eh(m~YhD z^IdFudy0jubj!?j(!^v+?i6vTy9a1I>h1v=MkdlwGLeRoi8Pc^6Qe+ej=RS7zh zxqDbk7j;56Mp#r2L@_dfhLb59cK5tS=HJ~JX{5e7P)c(fO4F9oE`d^%K*^0F4mI}x zjVCCSHOxIo!=S8T?m^?B&wON+xd(N^psX_YNM`O~EnU!w-=$HBoQ?!R9QYCetrn5*~iSi*PC?z_x5Pn%90dCpqS=D?JD%%tm6k1 zIZqX*tyWN>BRI6G?@DF}(-vN>HekH~7X^@VgqaHMy*C77$MztpOM*iMajUJQK9qKc z4Q1bS2RKjDdo00!Q%*g_Ctz$QA{vpLO-S=&+aACfh883!?{Jo6_~P|orJHT&l`E$d z{%ByTfb^7)ag_VP+f9Hm<8&!txJ4egu}Je`As9BllUY-0q2uCB{~6FlXWE~1SY)k b_miI`76Cewwk19Rj+6BzBL+Yt00000Zi_jo diff --git a/summary_results.json b/summary_results.json index b465950ae..a0e941a19 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432719, + "Performance_Result": 0.436551, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.405225, + "Performance_Result": 0.412006, "Performance_Units": "Samples/s" }, { From cab8998ebfbe294829632aeac2d39935db10e3ec Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 19 Feb 2025 20:14:42 +0000 Subject: [PATCH 133/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9758 -> 9757 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5b58585bf..4cd5266b7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-19 20:08:36,252 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-19 20:08:36,252 submission_checker1.py:3270 INFO] --- -[2025-02-19 20:08:36,252 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-19 20:14:37,951 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-19 20:14:37,951 submission_checker1.py:3270 INFO] --- +[2025-02-19 20:14:37,951 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-19 20:08:36,252 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-19 20:14:37,951 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-19 20:08:36,252 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-19 20:14:37,951 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-19 20:08:36,252 submission_checker1.py:3286 INFO] --- -[2025-02-19 20:08:36,252 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-19 20:08:36,252 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-19 20:08:36,252 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-19 20:08:36,252 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-19 20:08:36,252 submission_checker1.py:3307 INFO] --- -[2025-02-19 20:08:36,252 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-19 20:08:37,238 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-19 20:14:37,951 submission_checker1.py:3286 INFO] --- +[2025-02-19 20:14:37,951 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-19 20:14:37,951 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-19 20:14:37,951 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-19 20:14:37,951 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-19 20:14:37,951 submission_checker1.py:3307 INFO] --- +[2025-02-19 20:14:37,951 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-19 20:14:38,909 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2594 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2132 | acc: 76.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.422164 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a38f20ae2..72c2a20c9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -760 +761 diff --git a/summary.csv b/summary.csv index 874f91449..7c8cf2ea1 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2594,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2132,acc: 76.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.422164,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 64c8d7a9f..28ba50827 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2594, - "Accuracy":"acc: 74.600", + "Result":10.2132, + "Accuracy":"acc: 76.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 04f053b658725daa03de0c1e07d463dc66017b0c..78015233518ed84b7b3f868075afa34b02dca967 100644 GIT binary patch delta 1264 zcmVOYjQ$#OE|5H$^F!@ch~mIBd3XyMo} zg33~qnK0|ZH_uMTEX$azgZ1r_hC;TLUP26|>?+X1qSA|d8L9Agd1o1}G#WunWpOS; zmdDB3+*_3IO}4Md_i|M$*`<}9QkgZe-1u%kt*z3QYpOIO-PUaiPdsgnUcuARw%{o! z9RkGzrN+)CXmS643vv`7FL3Y;BCbAWC7|BksE1yVNNrWx z6~e{E^~sbRAeR^(wg^HGfOUWdo)NfS&;X|r2#kSCJcO<;zE00Cm-Fdjd3kg;Kb_X6 zu}b%7dO!@@^4;I29mDr6`?qDwHyn@HotBAX85p58&-@>MGBMj0mN3ROoUJy+4%%@E zDLg=)Yxr0V2k;z}fRrGToOTFSP>PQBS{LG<4p=^avbgjpiQ}q%CiRW_8qussp=0(p zW5e#jogQ4N#s>F#aK8sveA(he31EZwj6>2h4v8`j8@$I~(&I1b@t5@YOM3hzioX!8 zp*_EvX{2X=X}Cp}VNpDcEtcn-QqF}W17&9Q)lwHj*EC+M(>Sb%Wt%-!_h!2Zvb$yX+)H;^e-4;h3yKBc3cy0xLPToZ`lV1nvr^RP<11zt9e|oxQlQ0(@ zf5w3liJjt5gY?)KXxkWj&Z4HU+A5EO3N?dDk;kY>UH9IprY<|0A0Rt0oIu%-o>png zupED9OLq;|b-kTc+Al``B_x7C>@0iQTEq|8hO~c=n^#*YE0*JX=gggvi`79k(Jxphv?@`Sq$mQ# zG#6@DqW5|oFHz(?Rh%|jL5Ysw(5kvAm?2DCc(vMq^#WWJK*|wjO0@Uh5R4t$gQPAA z4i&_$wvzf#+8uv3lzr13;5<$5u>?;!^%S3gv6+ZyKyo%A&5vz50B0B)kf6N7S(4$i z*MpUAwxLrloKpCsfm!P)PVR#fld)~vq+N8@kUztpo0qq4)mW(0sTiQD;EXT})N5N6 z__7FxYpC;^aN?~1{&)=f%-gU?-v>Mj0Z1QrR6Z6@&-Eo5Re74<#c3XA4|SfB?3OH+ zU#F2*{b!&YhWwj20{Yz7pP>2!vydT80|iyz@;qRZ;v`oA=#x4nIssafbtNVQnt3ri alb&nWKoi@54G#7W8K#f7c`G( z%5pf=?tyDxMt==C7f2q>`Jr|zL~&r6JiLXBa)T9+OG(Vij0*X0xi)!G0*&}B9e~JE zlUSB(W;Dq)6tXH8hZ=Dl4OMyaZw2Emf~}J?6e$EHx@WDEU=)ylmTp-V?!ie9K4{|+ z6;zg@%!FANzIk>!W?9B$9jtGcG!(L}^b%qyWmkb77L{Jy%SeT{%R6GY(r5%RmBqOX zSso{Ab8k_;H`%@--^*34WS3TYN@do>a^t)Gw6;oHuBpk@Y{B&BM zwp6-D(*tbS#CLz2b`0Mq_HWC?HyjV!otCjf3=5$(&-@>MGBMj0mat{jaJJeMJ7~ux zr0@WFw&7Z8IDqG%1f&F!Ly}*SZk@binfYlf|V+NgP-8GpTRX*NA3C3LUe* z85?#F?)2bFH8!}{gZn+W;>#AtN&p+YXB^_5afp?1*x)_>;vRo-kH5IbU)$Px~B13oyK8>iEZ{&-J9*cplo|M zG8!_zJKSFxsk#>(rNmxJ_UB6y0TjHTC5Po5HhlFl>FMSF$F}2EO-C#Pf2?~iB79xt zJ;S(vBX}*@AJ1ruN!(=*{?(4*z#wmoq`Gr_WLdsMne7d6{P;F_j_&>Y;9VHsH>Mxh z2_0-Au8D9*>iQztsC6u_yDg47cGr$6@Z1XgoV<%>Cch5SPm9m!23TJI{`7QWlQ0(@ zf3^Z850{RB9Bp%y6(MGO3fC&NlT9EWf6Z#cFc5_AeF~$y zE!m+V6}AIy$*F-tQ(Afx(k2nLevs5Ue*4N!;w0cxkE@yec6L#@Jg6r51?Q#F70NS$ z642ZTsdp86->lLFN_=2iu*PVpPzN3@>${R$%8i3pXDqlN!6y-|AS`w%R}*z-Lo z=CkO~QQR7*Scs+De__k`H`@U&2$^FAfe98E?t!TtiRgqBd_>wG*9-u`F*Kk+2aoeC z#}{t^r~G8YpjscJ#2QNos+qPLd8>}&Zjz8BgZ^Npo)cvVApt=;CatACJ zSC{y@h=*&K59@g1tpxGdMSbG!w8*g!L=pqgW8AayF?)L6C9F_gkm5ciMVdcs3PQ<( z&gNhJ$eaEPP@RVSn|KBsbKib~`VX^@Ax#4Xi+fS9VUyt`R{`mhIVCy)Ta$DpCIkxC cDz%fIB^CiZleHy20g#jQB_jqxBLDyZ01OLOXaE2J diff --git a/summary_results.json b/summary_results.json index a0e941a19..ee7d049fb 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.600", + "Accuracy": "acc: 76.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2594, + "Performance_Result": 10.2132, "Performance_Units": "Samples/s" }, { From 237c5a27a6b3079a5def531ad2d23900c73f4588 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 20 Feb 2025 04:10:26 +0000 Subject: [PATCH 134/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9757 -> 9756 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4cd5266b7..3baf3f4b1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-19 20:14:37,951 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-19 20:14:37,951 submission_checker1.py:3270 INFO] --- -[2025-02-19 20:14:37,951 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-20 04:10:21,566 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-20 04:10:21,566 submission_checker1.py:3270 INFO] --- +[2025-02-20 04:10:21,566 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-19 20:14:37,951 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-20 04:10:21,566 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-19 20:14:37,951 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-20 04:10:21,566 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-19 20:14:37,951 submission_checker1.py:3286 INFO] --- -[2025-02-19 20:14:37,951 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-19 20:14:37,951 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-19 20:14:37,951 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-19 20:14:37,951 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-19 20:14:37,951 submission_checker1.py:3307 INFO] --- -[2025-02-19 20:14:37,951 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-19 20:14:38,909 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-20 04:10:21,566 submission_checker1.py:3286 INFO] --- +[2025-02-20 04:10:21,566 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-20 04:10:21,566 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-20 04:10:21,566 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-20 04:10:21,566 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-20 04:10:21,566 submission_checker1.py:3307 INFO] --- +[2025-02-20 04:10:21,566 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-20 04:10:22,568 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72116 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71838 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 72c2a20c9..ab5a97c06 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -761 +762 diff --git a/summary.csv b/summary.csv index 7c8cf2ea1..14786961e 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72116,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71838,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 28ba50827..458371816 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72116, + "Result":1.71838, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 78015233518ed84b7b3f868075afa34b02dca967..0d96088e93e7fae790d56e28a2df8cdbfc4290fb 100644 GIT binary patch delta 1154 zcmV-|1bzFROq@)xSqKDUuvZzgT?k(WeDCWkiBb7?C9jy4!;l51VQbeiyADdgs4O$=2L$E&EZu3{9TbZ z>e;y#yQUpmGj&rGO_pu}mcWxke{DNJ<51rY&~Pj(ICqeSVF@%0%QEv0O2Duzv+oG% zt~p~ZJCz_=_k^9bhVVJ$t-3Fnd-dO#Fkf5dVPSj5(B z>iKuTe52N$?_$^6lP_d#YsW6W|J67hlWB0Lh)vx+K;u$(5700&k%p3qG?YxFp=26m z?jDqYl3A9ZYcO{YYw4j*=*9?(>VqgoCeUy)jhfv(uaWt8cSdTduL_igz7M6yzeQXC zr6_=s8%1nt?g1KCP$(;ye|wOIL0Q4vgT_Oj`50y99@Ghgvdr9LFmn%U>48qX7KM_T zdr%aEvVytCsM*}}8k6ZL$5H&yU1cq`Raw0nyQZFO0pK|<#j&~b{q~uR$MQr@r->3o z+38wGZd=!1`o9tfx9ji6zsR%xqWLh)9sT}v{cZj-hx0_4%28IRu4Omi@%lFhR?D&;ARd3s zPQx$|gzr2>qfbB_xVK+$?+BTEsuH4QbvUCvDSK>WSleX1|?Xxti~E zoBe_h%9xtoaSfRtiFx*y+; z04XrEpuxm|%c8_5Zv?Ny)rL_yb7~cP2Uo2}adPj1nv7l772RyKru-%TT)w=GtLDlK zrxJjgN(v@C@MwKg;qxLLuH_*v(}~v#vk}yiYhLIQjC!jtI`8V+h*yq0d1kE3_k0Hte1Y@vQ8M9p^Jpl!BqkHTm UlbT?k(We`5$yV5boU4imvm1C~dLeK-m zPGtqJ4$?3xfrhPJ%j`NR0i&|aup_9S=8U!UKqq`tg!ytn6yFr#8n7u+vp0uR`SW*0 z-l%8iTI`y3Y|Ye7Q8Zb)1y}-4e-5?n0F6U^J3zy+tl->18ipm%Ff7Z=J17Cevdq3C zsJrHjwe&zI{P8Pu>7Xb!Spp4Pz}9Tkd5y}S^Z50RT6nI+u7@XE#0t0Y#?mX`5;%0I zi3eyL>f!+!&SeE357IC$frfEeX5~Q%7?)*c9>Lr^tfdD!;aoC94~XK6e^{;oi`be? zJ^v1vZ`9iJUF>>$@`bEz?byZlzZ$1wG7at&v8lTUXk6;<0UAao(oiyyhLVXiluV<{ z-GdTPGRqQl4d(7)Ej`o;-56m}eGtXS1R74JQM0?}H8TJ1&PXlwRe{pb_n{Q|w}=a% z6a`Rnqlit-JwW3M3S|Xze-F|yC@YwI(0J%GAEV6NgF0bQmYI7DX6|7vJ?x306k9@jINeb zsiaQ}*;eJ_BKw5$VpV1MI9mTgfU=0gC=S!7b?^C46fCEHXBG|LuLiZ8hkq>P`**=R z``}$*fA~c)J}7M^EB#rTRk!LTe}RV8Ib8T-dG5#KaGoerIm!yvwd@8wUjOF6>UA0| zR?nmxap7D#_6l``B_x7C>@0iQTEq|8hO~c=n^#*YE0*JX=gggvi`79k z(Jxphv?@`Sq$mQ#G#6@DqW5|oFHz(?Rh%|jL5Ysw(5kvAm?2DCc(vMq^#WWJK*|wj zO0@Uh5R4t$gQPAA4i&_$wvzf#+8uv3lzr13;5<$5u>?;!^%S3gv6+ZyKyo%A&5vz5 z0B0B)kf6N7S(4$i*MpUAwxLrloKpCsfm!P)PVR#fld)~vq+N8@kUztpo0qq4)mW(0 zsTiQD;EXT})N5N6__7FxYpC;^aN?~1{&)=f%-gU?-v>Mj0Z1QrR6Z6@&-Eo5Re74< z#c3XA4|SfB?3OH+U#F2*{b!&YhWwj20{Yz7pP>2!vyLIk0t7ccDBH7KBs~EInt3ri VC6k^dQ~{5Z^Ccq&K_dVF003?(Ag=%b diff --git a/summary_results.json b/summary_results.json index ee7d049fb..d6ce139c9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72116, + "Performance_Result": 1.71838, "Performance_Units": "Samples/s" }, { From d7eb4b7a8836b42d7313bea4bdc9d5b399b69afd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 20 Feb 2025 04:12:28 +0000 Subject: [PATCH 135/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9756 -> 9748 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 3baf3f4b1..db5901346 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-20 04:10:21,566 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-20 04:10:21,566 submission_checker1.py:3270 INFO] --- -[2025-02-20 04:10:21,566 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-20 04:12:23,561 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-20 04:12:23,561 submission_checker1.py:3270 INFO] --- +[2025-02-20 04:12:23,561 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-20 04:10:21,566 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-20 04:12:23,561 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-20 04:10:21,566 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-20 04:12:23,561 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-20 04:10:21,566 submission_checker1.py:3286 INFO] --- -[2025-02-20 04:10:21,566 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-20 04:10:21,566 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-20 04:10:21,566 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-20 04:10:21,566 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-20 04:10:21,566 submission_checker1.py:3307 INFO] --- -[2025-02-20 04:10:21,566 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-20 04:10:22,568 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-20 04:12:23,561 submission_checker1.py:3286 INFO] --- +[2025-02-20 04:12:23,561 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-20 04:12:23,561 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-20 04:12:23,561 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-20 04:12:23,561 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-20 04:12:23,561 submission_checker1.py:3307 INFO] --- +[2025-02-20 04:12:23,561 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-20 04:12:24,513 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2132 | acc: 76.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.65301 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.422164 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.434658 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433643 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.30318 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index ab5a97c06..18fced385 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -762 +763 diff --git a/summary.csv b/summary.csv index 14786961e..4458b8ca8 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2132,acc: 76.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.65301,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.422164,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.434658,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433643,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.30318,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 458371816..1c07e1f80 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2132, - "Accuracy":"acc: 76.800", + "Result":9.65301, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.434658, + "Result":0.433643, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0d96088e93e7fae790d56e28a2df8cdbfc4290fb..6510c0062882a6f768150dc1e1842ea80241a86a 100644 GIT binary patch delta 3948 zcmYk9cTf{tv&KVjMx;rToH-UhFkzPVEf%Xz|wzThVKRaacPAwv9O-)aOL$>p=QyS!8p=A%{-(?J4 zH;p9*I*?Ljhte=uktj-Te9eYEC`M2rK^}tg_t8~*;kc8lLC}OLc%V{ z;on+h*_l1xD+Qq7?5i3m>sMiZ?xfrR7*9V*uNFZtz=xmAu4h_HSSK=j2uKFMO*&9? zUQ2i8-tHUMYjCx#b<7t#<8jj_3nvNXxJ~P-u_lMXH~HZ)wjWFP?Y`Ql);B*SL_IX6 zs3qgW7UWrN5A{jSBz54sQWS8qY4JfVE;qyD1|(i6hst#{tj|A*c~G>+4Rp}3oSdcd zKHCRQ+Cn_!Novj7%<-YDyvWqb3^dd3JwCVm{H->5!-`AUGWjJ!y3?G61T?=6l`76T zXBF~(Qq>*$M0Uw8h%C+ z$6e!5HJA&yChODk*29*{)<(U76=tpCjg7II2QxYA?fRlJmw(XV>TR%et3KU{2|J46^S{Gxmv9sf@z z_H~z{nR%yo?(t~eMQ-ZGskgmLbML6sc(4=(J-mVoVp9vO5x{*Cv09f$0My_*I9Cwo zAG!Ili@-JNOt_7!W!zR+T;@0qRxg~Gbfq`Bxi?~u0QHXU&#EG#MBn-Ig92flz>lPu z2eMtLXq$=|{uPbk;zkvwzShI1KRr(z30Iz4#(xq=)M%5M-|LsIcvn{C$FA-QM>2}O zi%B;nWp0ppzyMJ(oUj5r^IwVN5_VCEwPu@l|9JhGBi*y1AJ?A=(y_Tpu?MBI7aPd5 zUkF?&fK<>wQ=T8r@ti4=Mm&5RRir0m2trw1upzERF2%D8XJwhP^D=?iz=9pSyOIWO zABp;p&DrMnnIx*3aA<*UwVdAHSe~}a`bqi&ojV1IetZc5ZOTk}yn>o!kJp9;kk|CF z`Tt&xE&6#HO4Ne{7j?O-Q)LcG-%jlXFR46$#R^y9E(P*2i8fjQkj#x^z4j@-e`kN}y7Ng{Uo%i0k?jNJwjS#2QLsC* z9McAgika=UtiGSiYkAX`lGq`~DxHgfJPUBnKGd)Mt*glN9`qFL^&=59~1D@9X>q4@mr1OEWa z?Mvz3xxu>s0AB?lyKnj=r4g#_^Ms=0+dD8uYPxKn0>_tRb8RS=x)gj=9H$K}!5K^s#)h z_45Y{rsTK0X{_M{ql2k<{R6E1_mvxqEN!Dgz{p*GS@zY`&I+dUkMZF z9?wi7pYb(g8l-~bwfl_&sXZao$a~WPv_}CYe|S%jPr&G;kGYcG&vSh+Erlq$fmZJ^ zwjX2iEe#D0&@f-M6PDamCiF=#PuiU{wN(0=I_GbTFfDLdT(eKE9r_Ny;@)uxZ8K`T zQ0MQu4)wb8dh`~H7%{vDdVHZjg>!D9?k=aPg*JR@aqs(zoa1y4s<6@lFZJUp1>j5O zlb6f1p*}~UDnQde4i7*#F6RTP>lC&b*Xvai^90A4Zbs8(T;`?=0=q89?fdj7y_~E* zQ#El~sE#9>`>Kai6zg{Mh^;VGg40Lv(g^OV;2$U}Ynir;4G>g2o+<^2N3ES zDU3T5t-nLQqg7|+Nc6Fukn@>(Dy?~O`T3)XsXYf@^~cE}b*{5uC2CzV5jHA8GJV8a zwp-nb{tCbrKFub2LH>%DMZ{wQZO+guS_hhT&C#(XHQcEv>hFb%MZ)9V_<#kE9mV#x zAEv_0odh7jXl{gA6q-PbFrx;a=v`a?kf8`UfOtwV^RgpFb;-qig zin@?wMg3dUMy1^zS)Q%=#oaD9WA97=nrxZ-Vab306fmHXL;?*I-Rt1KW%}kh?Mp9> z@UCkUE?3XrW^OMeZWo@1;r14}KOdDe<+lI!xb5XQ*O*UYeQ5nMK0+?rLt7#FL5)II zPrIqZ;G=HLH`&Wl7sI4uimaXG7D+0lYUFjV0#AQ=xcjRCxpX|4`}6hbvp`+434Vy} zITh1Fn?A&z767op_;El{fo}^Wp#r#;5^sV99en8Sins(=JqeR0P&xG42~A(gAwhRF zGZWhBdb)Q!{8^$$wm?ch%#;H1K=Ua{0jXH_AkG`QzfRVz4;4Srxw9Ds(W7TH{eKOv zz7`=<6YEojyjyiR87hfAcKr}#3ib1*PZ3x%kqhZbaalwOTNjG%ij6n>VBnWjaip%Z zqkKR$Ti;jH&)FzRo=0rHY<{q1q9+9I5kb2)%2|f9p|RUhpoI^D9xY8uv9$wbVy$XC z%e)Xb5Wyq1Yuckq4~Xy_*thevR_|FEl8O;e(_-xzgWN;tCf=|QTv6+(q+Q%?&2QkH znEndFRy-@&hl+TS@3iO`&}HkMbj7p6V$W)j!|u=VDmDd=F_aGC0r=Rl!5duLG6-DokX zs-tvrr>WU5oV}u_B)nvKYMNGh3q_?%MWOC%)1La)7r|8Bsz1*#Uy z>e;sw?Q(Gwd-Qb4`Td^T=9L~D&o4JF(3gf<62QJT ziiPz`pD)h@FnMWc73B<=uKl}C`=tm~ z_jvD}tih~cdg@cv1;4G=7OxugZyUbN&ptoS)&tPVdJ>%@&_JhXWTf};#oJ#5%W<~B zZlR6&6h^1~h!MH1Vn)Y12YokYT-d{=<5A77Dsuh&xfUMj8OT9pe1~>lhHN}}b-wHJ zeBy4j;(0e>^*%>{QG&Y8WR)Af&NUo0Cmk>$|2{N0jtu`n-gJQ7I1b5-v5kT&XW~x| ztExEE{U6S-NQy^iISpk|rVss;umU&^mAouK6QNDJg$sh%1fij0uFfTX&YROIFFtw8 zM)+D(ds!rF!i`qj$i|An(_ha3({YQm0|~!j2#$D_e}{9n_!>4M)yrV=*DM#Z@dw1; zQ4wmI`Oj*HESJWoqasv0tta-HC95KDy)pRVYJMRStWlt0n{>v!leBak$4y9AOeun@|-NA60(b~JO@ zdubcPJ8Ug9e#DwMox8Sy9An~a%`ur1Y7wVUV-o1c-Hcl7d1$Da;_RBzDVq5(=tt3n9mk;@%bCU`Sy?>Wlu7}nM_CmpPy^+r6u8( zk_bq44=BK|KX5Qn1!iqqR) z1$Tyvg$uy>BUc7$EWfMsG#7pT(81y@7wTXYjG2Ey$v^VhJ(+*~LNZ7t7cgOAF1HwW zd!vBaiF?gonl&KHxc#1R@qwXGgt~O#CJnX%u*q{+7iC=Jlm`l$FxA-((g{N3xeXu& zR)a%$BF0fv>*JXyj*(c`tyz=NGb((#$N=^K`}aD7tf=}h??9sb|My_C0|1Qo{QvQs3IH&r1_0RpdlV2Gr+{e!-=unq*#nEwtBCzA{SU>b BW-$N& delta 4006 zcmY+HcQhMp8;4`n9GRWmUfjZ!1T*3ue9DO#1bMg=ii zl(x1|TaDV|MZfm@&Uw%Io`0YFT+e-9=XdUNo@}!svzPV^Y#uLd`Ar!CfL}TQ03QG! z00E+xiZ(;bdU<->7iH+VkSJV%C5?d?+JtFA^3 z(zyBf zK9qa`FBoV(geSb1xmVhfd!STp0BCaWyKl}b``^~#9&Xq*e{f&$tG zi&~VgTtn{vL}C|=3yPJc*%{0H8c%KW?@tine|s%!H6w2#L7i^ueV3*2!K&FOIGdbX z`OiLYyg3kQFn-C#eJF0T>MpS9S5`iS%|U~1+$7@s>Y|;~em;GCT1--nviy!@@9P}W z;t#pSH{r*)`|VC-sAltW#r!%^rw_f}J2V?hMg+{R;8aq!;fn>z)5F-QVyw4v$>tLl zghXPR4n>&6ZTrf>Gf&%n)3h$WYc_WpfNPQ`(s4#ePe_k-EfvVbgO^Zr{3}Pc+++1p zK`zK?g5;r8;Jk^W%q%zuWiTLC-X=$kb4HP%`=PStjkYi7YBU<{s3Y3Z4?XI@B*Z+E zj@>Z>WbyQb7P8odzQjp=!3!1{j-}(Io{`MR7^r22ERxh{I8mr*ZQa2wwOqc%jl=k) zq8Vneh|dw{_1=l|OAy%@;>0nk$)5TlSLjEmvf9xVC-p7m=!s@nvgy0Tu40mO=^=|Zj(IC-)j?aUnp3P-4h}lz`Y30i z$((&2tEZa@rf7sHG3+)3h2THdiPIYG4RZbV9Jr85?!F1^{@@vW$G5pcC7~SVz_~pb^cMfMZQDWR9Vwd>X5MwTkBR(mC0m7vZE(O1Zf_H;2xf|c( zO}fc3!^!bsPV#{qT_*X}^^bJ9_w-R~7uy&+BS~)U_vrAW3jI-(tX1&Tf*YrXg(&ef zo!C{OqUqFmotD!?G{cgU*j28g>B5Ku7dgGEgw8vs2crxHqTbUqvz1?s<aWd6v+*Ezk3%DrZm*VB(7WPabdGbtF7=xkS0y9CxjZ# zqVuej1ue>YpTpNlT{uXIK`$GFvLifsr_lnrW-rjYL_C zAraVLJf+k5BB4^O&X=_?UP4MX`BEx0mka4T3YPNd((lmD-d5*~IqGSYBseYbahcZ5 z+flQUIDPpDPNW8S|4Uj%J_3~S?Np+?-3S4+n&e^p^!F?px^%!&=Kk^>bjKfLB)ax3 z|M;Q!2y9bSZl`25bIvi1phsK;lAk%DSXX}2aj^&e8@{T+p5JGV>DfMZBl#T zl?2|2lurxnfegdMi9N3+@GsB`;yPFMt^Dm1whrf*@vr`PFzbi>@z4VqV<_IT*iLQd z5zmd7Fl(9CHpZB3`_%L)Jciz{T>Ptp$_#VlEg5_=jX zGMz{`TA<eME}1^H12w#tzx;lAJ78t-t=n~0sq4J)W7Dk z!x{T083s!ncToWcJ?W&`MquLc{K;4CmwR-IVd4X4stNok zJ5%c8~9SDX`5la z?teh^S-+FIM2a~n5W-L zw%y03@baF4{$#nfUtd*)@P5IS56ovYv=^)+*oR>Xnb<5GJlpJ9DcTe0wFSz?WZ4s* z_`@RD{ntt@R(lx!1j|r-I9MUAU`C@p^CTj|@b>e1rfh5*dwFlWffJxy0+S_R5O8~o z3TSI7A)ObUfz`49FrC<=_xOa*YZ>%~F|qOEvuEbD z{hJ@#t+3ZcCeT=i-exIb-ia&O zyfsoJ^5;`t>TCl+!;n197turLvRi5g*P48fB4l9YDl*$Mh>>VHV`DLXoT z()MdZBeidl@+Zvj+QIow;ek*sQ1YIBe!rDXRh^g0+|H>Ay-oe7Z$p&V^M2>-iq9J- zGy11`{43jLNPYvW(=%}`=oBSr^^DQdl!g|c3IR4P{S`&UhNSEwZqWh&VVnQ}7XZHv z;X%K)b-b~l$`wa(b@Oud24#)=)m4QhO(s;&TXXYWy^&K`l)V87QNEj}&>klQ%(E!s zveVYBJ|5gTiK)}%kDKt6e4ChcMMmq(gLoy{$lfxDtMKhHTk_j_5jkJb=j^56su`NG zw$uwCN-hIYu&1|a^Wb|2Jn|kh|I#ICa4tG4FNdFdmx2o1Eu#hQI07=uEABV=d&B#b zxEiH+lM9wqfz`W<6s8`>q`7!y;I1c}>!~7-jyk;C*Y{fo&wCb;O0UkzESik6TeC@>>QY5bSYF_$~1{EZ=(PT`9me=?y>!2TZ z*a=+kVG=@GiE6%4BFlI!*|ykD*W1{O(q;#K_$*{@{nlhezbbi)h^b908Tcs`QIab| zJDBg@e(dPRqIbRYnJ%&-Wf%Ie`-X#5!!;k>)-I*wD#eqmMk&84RaGjHR@;r#@-*dT zK07iW9yGoFfj$sXme6mVoe%rbse&G|Aj=Ho)g~TnW?^-kb6j*;$~UBjNo6Asqxqlz zyc_AmUk3;3n0bZB=@H0%lhcmb+vd&ny+G7PBXWJrxSMI_b9wOI_w z=kBe!M@EJzYZZU1Y%n?UjnZbB+MS_P2$7Ct|9Rszpdy$)UQEDw@bi@_4V?)qT%F<} zzmMM=89eBS#X0y{oH@56c;1;Um z#1GO=fuC!ymwFtkc#yU}6oZ_e_gp%P3T~&jjhXna&v+~GC{=GnkqxK7Ch(r@I4iW`#(VC9D)$58j8+czGXuMO*Xgjlo@NMpbJc zcok|OkG$W>OH6?trD9XJ*^X=ttG5h{BpY(G&QG%R0d(@deQuFPEUvL;QBfzq8a7*y zqxR!q;UBXP8C-26hhSS21{c#qvf`vWXT(Gz`n3lHR;yxYJ)@@CC`0*ph*gb?5kMFn zUBv#BOWhV2W#33bvnl^IrvFSVJT^1ni#lO!hQ~vfC$& zMy4K(j(Y1wKWOgQ&|faU`%ukV^Pjd);!R><)MSOr7#>@d-kx|)T=93y#QkwT-zmQ6 z!Lo{%lTzu4OiGC+ndB;C=cC!@6>`@Kgu&NfMcNY?B{qHozApvQKBf_@W8tmE$Vml3 z;Q{A=JfqqW#U-X)$iwQLwSczwg03>_=aV^(t-`mYf8vn8M8Xh-2ou>8?m5RtEA%dN z>i1xlBPT7c9^vhp1m44IHoNp%OPL*338%MSXg2wOT69XiplPD;x-rn9oO5mu?I&LA#yS><%Sg;*d zc%H!CUjWRZ)8XfHD_1B4w$}5U(Ddsr8t#4rR#~xAeJU(8Q5COEw!I^CTm|-I^rXIA zXE?kCR*gc&F^Q*NQj-6|A7MbfkfYo;kk4TK>UK3K(MyX(FcraS^j z!d^Z3(1jT&>$-A>wx0AAYo!)?n>(7`_JvUhYCEAI)p?A2Cg4@Y8T7)b?R{F!az9HY zC#JxJNgVOhH}>ZIo~TgWl8D?WZj5Qs Date: Thu, 20 Feb 2025 19:17:22 +0000 Subject: [PATCH 136/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 9748 -> 9747 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index db5901346..f6259cfb6 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,32 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-20 04:12:23,561 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-20 04:12:23,561 submission_checker1.py:3270 INFO] --- -[2025-02-20 04:12:23,561 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-20 19:17:16,805 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-20 19:17:16,805 submission_checker1.py:3270 INFO] --- +[2025-02-20 19:17:16,805 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-20 04:12:23,561 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-20 19:17:16,805 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-20 04:12:23,561 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-20 19:17:16,805 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-20 04:12:23,561 submission_checker1.py:3286 INFO] --- -[2025-02-20 04:12:23,561 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-20 04:12:23,561 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-20 04:12:23,561 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-20 04:12:23,561 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-20 04:12:23,561 submission_checker1.py:3307 INFO] --- -[2025-02-20 04:12:23,561 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-20 04:12:24,513 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-20 19:17:16,805 submission_checker1.py:3286 INFO] --- +[2025-02-20 19:17:16,805 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-20 19:17:16,805 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-20 19:17:16,805 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-20 19:17:16,805 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-20 19:17:16,805 submission_checker1.py:3307 INFO] --- +[2025-02-20 19:17:16,805 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-20 19:17:17,804 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.5738 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.6142 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71838 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436551 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 18fced385..c8b655e57 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -763 +764 diff --git a/summary.csv b/summary.csv index 4458b8ca8..1ca3bff7c 100644 --- a/summary.csv +++ b/summary.csv @@ -1,10 +1,10 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.5738,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.6142,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71838,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436551,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1c07e1f80..e92226e95 100644 --- a/summary.json +++ b/summary.json @@ -4,12 +4,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":18.5738, + "Result":22.6142, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -34,7 +34,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -124,7 +124,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -154,12 +154,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433237, + "Result":0.417437, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 6510c0062882a6f768150dc1e1842ea80241a86a..8edabefe751da811c255554d1aea0a3426676f66 100644 GIT binary patch delta 1035 zcmV+m1oZoqOp{EoSqKI5qe9k2vt9@g1%EI#ZZs^EOag$iB0*QsorF_p=`o%728EJF z^|>fIMKT(gBe#bC7NOjlAPXuDT@ycmWfL{lnJL!;6YFQLnj)5KE&9OC|weS&^W-IrxGlZRs(cAeJ<$&qTo!R7N9ooquv^ z{2y@HM2&Txs;&pNxra1Nr({P|@LQsX0(Tm1mvV_3*Q8vc2DxN3z$K#rE>VMPShI>t zCIN6+k)XRd_`)S^>2a0Nhc^(_XQGfxW*-~5G~YCrXLd|{Hd^q5Y3 zlQK!8`dkz+i5j5`%B0mSlZ(52CS0tcBtT~O)8%AY%#y;_j2}R3S1GZ0zM2%}xB2j}xG8!O~(EyQ*28gt)h-4A~krfHLn}aVz(tnm7(+NCv zg{VFk1w^6-Zz}CK1mr*Q=(UjoYcXkW-%*;HuG?iymbh>P5-KxLD856}Y&x4Dx{5O6w=L!qql z_Hdm5lvUmy1H3)Z(qlUDEea)gd$=fovWBw%=!@!wZ|#7V%bf;Q5y6@C(LrB@2{9 zF$zFZ#ki~s^igfX2NYOGBqNQIP@oPhT9-FD)fspYFAMgnpA6a2m&VSb!spID&g%sAnRYkc`eq{bQN|z$k_W1W0Fb z5+(TTbzp>DEEtpvr{J#BuxLGrv-@cIY;4;$YFC5R=TGo%`}#hN8gkjMiUKNgMl))F zIAur59A94i;u>Pz)~|TafjxCzUwE4a8S8)rK0r3cJt?27=a*`O$}~=I!#EAoxJn+g z^;A zFeN$xP?KvVCL2=Qwl^gK002+|000pH0000000000004l2F_Ww%J^_W3>m?%wHzNQ5 F004!+z&`*0 delta 1036 zcmV+n1oQioOq5KpSqKGK$SRITvt9@g1%C*osDhG708myW=nA@%a0)FwrW4y7qQ*Ki<$7RZ{rp$6U9)88MesXKhXOZmY=>fr z8rPs$q6V>KG=L?e0W492SQ=GWG6?|7iUi%w!51uPOONRUv7}LbCJLUQG8&=llz&U( z|A5OTYOM2Abv>}nJ)~heWnA)GqK5)^8f}+yi5l0WT%rcKWHi7fqX8~agItgTr&IE$ffzFxg^hW{yLIPlvfK$a|cPwFoaXGy(sw2 z`$K^n;x^sTW;CuvnM4gT$!LH{Mt=iLqQiW|>^v*j$|`S<0p1>H=`o%77KIYLJzNw(S;O07H0a1RkZBI|W z0PwT-5J(SyjkRY3B>?~cPyzq|5dZ)HWN%|oa&K^RFJo_VWiEJaY@L$LYQr!PgztR{ zqq{9xNlOv718vEvfkIPSaud=f5w(7h)H-?lDo)}g;8TyQnf-QlQ8nA?Hv0u1lr=Rf z3xYB*!b)W}HTqc2^GB40$c^N!HBh5H1T<^zD&d%aupZvLb>O3dkR^}_Ot>0tV{{DT zAhw`+D3Ze@ac#ZkF_nIU9T(qx1DF!>fHg!edF1#2%-ujlC!`bu(*5|h2S|aT1q~(! zToxt1cq4cnMjLwN%BfY{I~cW|#lgJ`YA|+PS9Fu!n(~+UYw`NtubL}!IF$g@R8lbE zfk*2{n+jhS>2NI%agk2ERv@4DNgsJTEpqGwnWX^i827AvPM%+ubJS2m?{h+PN|t2G zO3LVTbr^Zme*x;#kbe`;fMf34Ptg3cgdxiU1z5-`jz*JIBwGQ;ldvQ_0a26fBtZe+ zlP@JY0a24`B_ Date: Thu, 20 Feb 2025 19:18:25 +0000 Subject: [PATCH 137/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9747 -> 9756 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index f6259cfb6..7e304a4c3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-20 19:17:16,805 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-20 19:17:16,805 submission_checker1.py:3270 INFO] --- -[2025-02-20 19:17:16,805 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-20 19:18:20,648 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-20 19:18:20,648 submission_checker1.py:3270 INFO] --- +[2025-02-20 19:18:20,648 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-20 19:17:16,805 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-20 19:18:20,648 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-20 19:17:16,805 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-20 19:18:20,648 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-20 19:17:16,805 submission_checker1.py:3286 INFO] --- -[2025-02-20 19:17:16,805 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-20 19:17:16,805 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-20 19:17:16,805 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-20 19:17:16,805 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-20 19:17:16,805 submission_checker1.py:3307 INFO] --- -[2025-02-20 19:17:16,805 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-20 19:17:17,804 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-20 19:18:20,648 submission_checker1.py:3286 INFO] --- +[2025-02-20 19:18:20,648 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-20 19:18:20,648 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-20 19:18:20,648 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-20 19:18:20,648 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-20 19:18:20,648 submission_checker1.py:3307 INFO] --- +[2025-02-20 19:18:20,648 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-20 19:18:21,693 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,12 +29,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436551 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.65301 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.422164 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99533 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.255097 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433643 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index c8b655e57..c099828d9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -764 +765 diff --git a/summary.csv b/summary.csv index 1ca3bff7c..bb5a0145f 100644 --- a/summary.csv +++ b/summary.csv @@ -7,12 +7,12 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436551,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.65301,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.422164,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99533,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.255097,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433643,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e92226e95..1f836d755 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9786, + "Result":20.5941, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.65301, - "Accuracy":"acc: 76.000", + "Result":9.99533, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422164, + "Result":0.255097, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 8edabefe751da811c255554d1aea0a3426676f66..2ab6e2f5ea7a7c655778229fa508357ef5768dc3 100644 GIT binary patch delta 4000 zcmZ9PcQhOR*T+NbB7_>TNwg(qLu0SjNT^*>(bSI7BGj(fqtuGgs#*I>?On4rHA+>L z+I!R1mr@@6o##Byd7jUo_k7O1uls)Ab3f-p(#_DV#LzIRBkrl{&;kILY5)KSfaDB? z5LYuST$lRU!skaTD$4>lQmK}Yz8gMH&UY0E%b(EET&I;Q1ULh3*AJ}D$1{X=%w6l7 zn7AD`;=37f^zFS97qO(;xpT21ej=J6K|;K65HrO$|I?4*0KJ_qE||j-pZffhWZ&t+ zq`UOmr9@xal=$Rw{XpP;pnji`Ju2&QDzW~vd+IPKeHg(#O#Jz!d%F5 z%1rnQhIUHh8wEej~0uT@mc@ucBv)v ziEbH!tzlZ&X@izAT@`@XE(=yk$58<>8F*7#c1JdS zq6H6lo1)ZUvCJl1T=r9&F#NmJ&e9K`^!{hM9+TUIFuqVYo=^BNGzspny{>jR5C!b& z*GHSz28yP~RLM*+n2j=^9uc+NkN0NQtbVj$_a5}2x8jF6puDueiO=t5<;pvLJ! z|9g+4yzxVIzhrRtTnZK#T_f5=2KF{1T!YSORvVW`56NK2)dpATq(}BEM9Oidvn~HL ziW%>iI4z>Y*jy7(O&^YQN-EIhxU^X}+CNw{X~)F5%PHO^wPR*?iTFyiUsAtOorz7E zoVF=YnT%HMFgrLb=#o;K#uFDDrs_=xU6I}f(4M-D1chbHiYFx0_>aJ-n$&Ho(r#ue zt~_EqKo0Su`f%y~wPi;S`GL#KHFW0rBv!a95WzJF#Z?Xj3Qv?}i|-X0J}i4ae$o1F za~PgTXc}~4iz!+2{nwT!yB~q;${+q^U05|R3tIo7iNqm@OVc}2n2*%~GLz<|zpWIx zG=+Y#KjW|APtV_q>{?^`I5od@^A@gdFaSSvc^tca=7hDb>m7qnc*0pU)~z?Nj66-B z|F&zkMVltM5fr9Q4P}+*7a)FUZ-k8{;D`5kp!RR zr-}Hc^vz9w)t{14ywr(Ul|JlJ!uEO?nj=b8U5x@vCzxQs)Cy+uMFOj!Ex-@Aq8`yk zJw9N)8z0hd0|$F_!ju!SewPaTUkp){Os=h NMx_h*^$ByBdf%nt<+ok^?(#K$~RF8)#O) z&|?f7H_}hY;M>JjgCol!EMqp!-;=vK_12O^3&Z+KK#NueMrz|zDF<;Emg!Z{# z*I$7=Xl1MA($?OrB`JP9-PZnM_y`6a=7BQId}-kI*q4s>-jAo4XKvVxlO1bhv%A70 zsdl}hfb>GAd}Qs2<03X-Pj$Y!RM>_KW1FOoMRN|ze7^2@G4Bg&UBb(;&Ap#rPMoa> z?rEc8F*3;%{X2op8Sog4(MkFt@T!<)!DMM&c>Wl6@%5SgfD$81ryj6gKY9f+Rd|=c zaN|nwMAfb+3W#RDr6yFnBMPoFHq9L%kW_zV@lKT}F=i|GrY3Z(@(t>Yf$UUY(f?gR z2TJ$bjCXQj;r6kz@V{T(C?uy1_!!3;KSu!(83S|=xbv-a$r)^_KfVkFO9~I2BSSa| z=4f`$53g;rxS4v)%CW!)<%Mtena)*DzPTfe@AR|{jL(F^4pt0i{^B(NuO2bF5uW`2%t7_XmMc8?(2hb#v-8M* z`1snT(->b^lVD?&lrF=XnN!8*M7em}d~x_X%EjO3=w8`Dch0fe@Y_FV*4C`9hL(0ciPKM&71vE*Fz^22vUTd+5s>#K zYu|ci#`t8I$A3t??Q9asLvnO0iqiP5q`GYA4H}b_PK-#3MkMRz=h_k>c6+?ZK}nd@ z92-e*kn`r-`y0J9F_Fd#!hZ^1~b;QpP?0z}zyz=uLJ{FVLl5J?4S56tw&gvh4ntnj4SeEwdZEvA3 z&6XCte*HON_npHja@`HjQ+ZX^Hk^I0CNb%v>USof=p+-+s?Ek&!NyWztM4&ww7RT# z8TP{3)b0R@{i7Zvt^XDoU0r0l-=oh87gqC!faWBq>t5X>!f`8S-ir$B3zi!JC-eNf+HrCGpL1pd7*juR1p@rkJl#a&Z!hJf7UvVp0SOlRk)Zo#niyF zl-s$69`loyGD}3dkRw_1tVOmzW*Bx=$PCax3Q`OdQnSMBN5-f%(>Y*80RVV{0RUD2 zX%EUq3Sof~buwP^6n?D0MN$>A+@NGjNDyiB2vE*0)sMi*VqWa!tk4dDWL(T4A&Rz! zNfn^F?NXcXQ%vlLc>Ev%`FStc8K+5PkYm^89X`$V620sGI9;(NJeCl8JprxraWIzd z)U`?0*O2_>iC9p-5y?E&*&Z<)~vqrPoiU$}$isUDqpLq0) zno-2@yTy@vO+jTZvxW{tytC5a6eF*#+fOX4=ry!+^VLlYUmU?=dh{QPR4F*9w|2`! z7T-Qisupo9Mk2`sTFrK13tq~uail)QugSH=I3O4cqQaw+YSM4`gFDX&7U&oryZRIbOHM7DOoA}0ye*VU> z>rnR{ZqQ6-YLy24wOpCeb3SRQ8Mf3)yGghDZ>$RVK3M(%z&Fmy!U-9)oH`*oXG;`(_i%Kjc5Pfa@~mltU7=K@Mbws9)-OsR2c zh^-F=;A(N;UdnQhN^$?XpmedsOLUKKZ|uX(E=xQ(563g?o=)ltDWSW>s_A2vtX!%T z=aUKIP|lmTODj^FIhqR(A7=Jig-1U*Hu?T$w=KuTA1MuyZc~`Wah5c*^Ar%fAEA!d z(O@#xmK<2*DvxZyFR9JV)yuy6v|0VEYxgJNRx6 z1P7nI`b?zZ85NE<5^N_ASn>&6pe(mxphC$voYlEdK5JJ_Wj|JUa73-Ln%H3&^84{S zRJUux5_kiiX$_$%ciI0RRT%001)p0I+qnF(A0Qc}Uo}670l%p15Qt4Lfy7 zGKK9wr^|aUHk1pyUJraSP!gW zZ18HqjJ%}Sg_t`*#(~vV2IU#3h|qAIZ4CE!e+ey{sb`owTxC)!ow(Aef2}4>wk}M# z4J_&IHv!pVOSRDfmfc--A0tf_41-c6cj@iuC9Lf0yVk8VLS9+eO zvpXhM3<$wI9wz9i92H)Yn-mFL3TKh^%{FA`T&Wo@Fi1}OKYwMCIn<17kW>j37B`JC^)zu&p%+!AA)@pBsn425v>vo0e5aH<6W@BsjT zFdv!F`+?p;9`5eJK{Ee}ymXka&vMCTQ27U*-mdwszsrC07*Eo9&Kc^q=M34U%M}{u z)E{5feQxPxY2W?&pTrWJ510U*^GW$jAJK5k4UYZx=fxB=?UwJ4%NbJ)@BT9&n>bs`g5@LKSTlwS(Ys~OI`Kl{dY8eDB_ zF^&b|Cwy*NWRVo%T(=o*6}Hq+y$yc77~8ky`*xq~)9YLPh*5r~yDF*8q4RQVwg)$>%fh`{d|oE{Iy7E5m&$cCw9h|5iD1eIIO}s0Z8*{E2Z@Y+BMxBZJ z_*#aa@(W8`hry~P6H~5qw>2mu_9$@g=>DuSDoU)wpC25Ec*F9R6mw6eE83>=3;(j( za9I4(RWBauDHGX9+e>X{a)^_6b6dg=jfk;haGkoMi^mug?V8i=mZt$x3s> zxgK9irBHs6QKdS8JNX7LoT5u_9IY%Yh&!QNqqj6!DWkUmznCYsvVsIG&#qU)dK-!l|Lz zC(-3AuDwCh*AI1GPlNu^pyUGZ^bjI?9#u?Q@600?M zX?4o!?!z{fXqIm5th8oRd&_y=pEoNiYM6C5&L{fQEWW@i3g|7G3$qfmB)pLU$bY@} z!>+Ry03OvfEi!B`Y8qv((iGAdv2rC|vzm~-HYMSUQ9fhJy+2tYo@}H~{QP;c)o3}x zL?46Cky|h=VjA3y&BQ9nXJ6DK@iuaxP9*)#m7_CC(5^-=Bwk#=02)ny9#yN zI{od41gE!<#SuJ@&^eTiji4p%O$3ufBPIWu>6QHmpDW~5PdLf<` z;gLj|+@Ts8hfA;3N5>Xb@W)~oUniPC zHRMP~U~SVGQToL6^Y$-)P;66DJC;9F8(+8k%l|PT*Ag=)^e@p>Wk-q$5*A5>UZnJS zOwUq4xS8kOW+$T#?&7VJHk}~VQXvP5LndkZ%h7{l3kmUXxFpVkaq%R$UQDUJw(-O% z^|W}hebH$t9JAxYekJD5-KHaH49`bB)KTA>6?F;8ih57X2Kn0FxFSdWqq|*RCgnsB z1535c`)ARA{}?c!mO=s#l>XJpd&BhkBbvvaYGGXh6D~ZbFS35lC;lus4aHLyc;Eki z(vtUj5AEsrt*L;-=4bslK3p~jttFp&@0omd&udeM!Ta5~Nf}o7*)ZvFSH{kAiv&-r z8fom6=lf6*=3X-(o6$_>eSdZ4Bv6}tyLo`)5f$V7OI@fv4FF(+^X9sNNx?l7d^*(V zgr$4RB}vPZl?8j_dQ3SSelOW0Qw-M)HQfcHu^>e_3l%d3>b$PKd9-GIpzw~?ox>!E z4*Nyje{*o9QIt&GU!N}Q)qazk{%L=Y-3O@BF7>09cj1L*qGwW55;CYK99^{RHeQ^A zfgjT)jJ2H|kBU`a9VH-=(2TBVK;4K(GNu!uEH>eTzJ2iWa=ix2*Tg))Wi_DH-rKKAI_j&?cIDEytY==_S}w!z+W)8pgy zGwkJ|wgiZejY3Ji;(OLF0gRq%nx(k|=F5O#O^ynhY}qf))>>Cp2D}}TF)DWtPx_a} z2;hv`BTV6h(wetUvshxr+3(`*iB*Cwm4b?4 znO(#_r>~1LTRXf5ztgT@L~XTSVHCa@jEwoG!F$TXJnWQRi_L!}f-hB~_m|CiwXKvK zYri4uWM6zNt&5~|$QaBDWuzx5&--pQTGTY?>SGLF6y%&9=I8)uWjy+w!?7$*(Z&&8 zhi5M~iI(FWgWbaGb7>4t1>qyITV)K6M!)*5e{taqor%Y^x*}yi@aI{eGct__m6|)X z`Z8tW$t!bRtaFJw(F&*CsFk~10VWBmH>axHn(JJ{zDWg4$h~?H97opsPTp`pTt5uS ziow`M=_zG3A01RxajE+IePO;P5uNQcl)XE1;Hy|Di04A)X9HS@FSXjPKv5fDENslx z`H8Rd#!On#J1?1VAB$>Fi&S+zla-g`TMEH5pH2WXaSJp933~_>S3L61aPAhUVLd{X zfRGcI&uRYEzZVs*qF#_(J7l>yJ`)wLtog=zg3@}eD&od-gYT~9XQIJs#cH-GCrsNZ zi-&Q%#0&*K=z>~)`>yJbEqe6QS3zWIAe(r2Cpr|Yq|8e=1DuZ+X9gPc*)}Ddv4}8c z(5pw5UH9OWiO?xEi>KN0FPe=Y8dAAxZR+N=KY;fvY zMAZ#VqN6cD2XI1uZ-m@UaDemWGFN?X7=P}ep3;2map>bStLSAFKS{p5?RR{nkq!!+ zX-uVt1?gS)ibc8)2sk;OkU%cqJ~xJq#HoikI)xR}0oP+`H8d4_)vuP-k_#YS@dJok z_j`~m<3yBD^U_>d Date: Thu, 20 Feb 2025 19:23:53 +0000 Subject: [PATCH 138/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9756 -> 9757 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 7e304a4c3..a85c69069 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-20 19:18:20,648 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-20 19:18:20,648 submission_checker1.py:3270 INFO] --- -[2025-02-20 19:18:20,648 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-20 19:23:48,399 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-20 19:23:48,399 submission_checker1.py:3270 INFO] --- +[2025-02-20 19:23:48,399 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-20 19:18:20,648 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-20 19:23:48,399 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-20 19:18:20,648 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-20 19:23:48,399 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-20 19:18:20,648 submission_checker1.py:3286 INFO] --- -[2025-02-20 19:18:20,648 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-20 19:18:20,648 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-20 19:18:20,648 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-20 19:18:20,648 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-20 19:18:20,648 submission_checker1.py:3307 INFO] --- -[2025-02-20 19:18:20,648 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-20 19:18:21,693 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-20 19:23:48,399 submission_checker1.py:3286 INFO] --- +[2025-02-20 19:23:48,399 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-20 19:23:48,399 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-20 19:23:48,399 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-20 19:23:48,399 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-20 19:23:48,400 submission_checker1.py:3307 INFO] --- +[2025-02-20 19:23:48,400 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-20 19:23:49,379 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99533 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96372 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.255097 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433643 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.30318 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.23624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index c099828d9..3e6c7a340 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -765 +766 diff --git a/summary.csv b/summary.csv index bb5a0145f..bef16953f 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99533,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96372,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.255097,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433643,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.30318,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.23624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1f836d755..a5c0099d3 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.99533, - "Accuracy":"acc: 75.400", + "Result":9.96372, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -549,7 +549,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.30318, + "Result":6.23624, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 2ab6e2f5ea7a7c655778229fa508357ef5768dc3..3d2339692be9620fe1ca73d2cb7574d19f220081 100644 GIT binary patch delta 1479 zcmV;&1vvVgOr1=ySqKE1W}sfPT?kMIf5UY4SyqL&2TS0|p|%~Maj90Wtn*gC16;V*>?nW*POAIF6e|meq}Bl6vZY>pkWKxnvFWIQTcNozrIlm&z0Er z@Z^hF;fX!p(-8h1E`dXbns|W5r7j+z;apbm@gNQ35@;BgWmX=PfN@!7<`K-zf5Te3 zpcBp|GxUHczKG=-u!ybM)bsCv`9`fh-^I4Kr|_FQ_Lg0I_p5O_Cez?f5r?{afX1Wl z9-v`lA`K-IX(*XUL&-GC+&w4(C9^C+$6)Rr*3w0t(2Wrm)dNwCOrYUp8a2CnUL*7G z?u^t@Ulk|~eHThme2aJwlwuE*f7~eIP;(E^c!EM%!Q6v149W`T9yA_@LRn_+L7gxt z%gj9nGxxBTF6hL!D3r|HgQ6Ie70f+G&E}rhn9PO+j^c;zDr>2&%IekFG4<%0?OSw? zY;JwOeJYcYJdu-0s)TWVx>k{!*43B(uf)Oa>ih98^1Q!zHi!yGe=u2nf15vxW+Ns2 zWA!@$PVqYddXX#~UM(h4$)4u2rOL;7{s|Sus>nxrxAz#_5au;V=JbivQmcc+K#XD0~hH;!dzbQ`1=C>Q#e*&}0 z5tso5x`mKcVzXxxngV|z-w!1mz;jRnQi4cw+96m$DLUF~U5I}=VEO#X;?koej;s2a z)HmvDM6)7=j@jRg4Z8<-dT^y08{F%`{T^KLWs74afDPU=4sp*o#L75q@E(70kH5Ib zU)&30c+i|O= zBbI?b*1Z=IzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8EZ@xbhB$tF8$3t% z{(bN+jPD!M5A1)04mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$-bFK$UkB-@#bn)CTY(aZo#Ie~^w=0^+ZcP! zqNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rLZ=1QY-O00;m806zdB zon9SB0syo87fc8Q{n#7@lT08Vf6Gq8Fc3xed`0Aa;v@xBtR$5Hbwwp4fv~JYFDE7dL1uO(e>RkT(;eVE zP4BS;PdW7zpMbHMh-g4^HX+TAZ8`vF7#fhEyu(?N;j`C+m2S47Q!boR_@jYY>nKj{ zgAbk>kR!=Ia%w{F!~sMDz!psL`EFbmXcTNU`S2#0H^^P6zutpNUb4EoI5 zut?trJPH9wA9qwf7EjOhBpOwDn%~7~9_MMDJ&=4sZkJ!Dkyrg^pd5z$n>Ygc+}EF= z`m>E8%K`+OW}se^TqJ)5x`mKcVw2z`R{`jgI3+p(T9b1nCL8_O90er-002+|000pH h0000000000004l2JCn2}J^_!D^Ccq&K_dVF008;Py@>z- delta 1462 zcmV;n1xfmyOq@)xSqKC|G(|wOT?kMIe^WQbzRA+tgC+3fP}>gBxYV};G#twc&K;y- zSON{hvdp}L5-=>w>^p+GYtC3p7j(iOzcQB&iei%`(69w;%|@NqsQfvPU*D*O=SpmQ zc=AQ8@Wh_)X$XH0m%yPzO*}y3QWp=Cz_=_k^9bhVe_<_M z&~8G1kzU&L|^Sj5(B>iKuTe52N$?_%5AQ~1ptd&@4q`_(udlWB0Lh(p~yK;uz& z5700&k%p3qG?YxFp=26m?jDqYl3A9ZV=#9QYw4m+=*9?(>VYUmCeUy)jhfv(uaWt8 zcSdTduL_igz6+%&zD2wTO0fq@e{K|UsJRDdJVBwXVD3R024w|v4;qg{p)51^piUT+ zW#%4(nR{4E7j)uV6iR09K~W6K3g#Z8W^>PLOlHFZNAW{0_ANR` zHn+atK9$Kxp2*20Rl+zwU8~4V>*`DYSK{Dy_5Ju4dEQ?<8$^YpKbWk(f6bpovyqbi zvHG0=r}&)!y+{@guND)jWKVP1Qsv`3|AdNSRb}uvT>V0TGEbs7iL$3v@5N6PEG9vF z8V}wt2ep_*e=Owtci}td;9cl^_(d^3C@m!~{aKn4VY6owngV}eTWUCf=b!|n1d-&lL$HEUbhOvH5dU<*^7)g+rAJ8|SM@Wg zZ`9X_Wa zECYY6doLnXp2eQWe@(@6aWAK2mk;8KL8<} zUL8jQ006W87fT2Pw}%#7lT9EWf6Z#cFc5_AeF~$yEz6-L6}AIy$*F-tQ(AHp(#8?B zevs5UdHc#v;w0cxkE@yec6L#@*sC`A1?Q#F6(U)V642ZTshbLYte5E=B|b1MSZg#? z=l~uq>bsI#%8i3JXDqlN!6y-|ASyAUkJ*z+AI=CkO~QQR7*Scs)-f3Rizn>B!o zJb%Cn0uwAS+yhfP6441M_=vPWt{nh^V`xEv4jz+?;EOkaQ+~2xP_CRxhJ%Gk>scJ# zdoM?0*L7Jp8>}%u!C$M__hHpk>i$$5P+ba6xdRrAt4n-c#KX1BhgCfBR)TmsM1A7z zw8*g!L=pqgW8AayIeUItBrj224VUyt`R{`gfIVCy)S(9`nCIq*K7F?5_B^CiXleHy2 Q0gjXOB_jqvBLDyZ0O-B8wg3PC diff --git a/summary_results.json b/summary_results.json index 10ba8526c..f2ce751ff 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.99533, + "Performance_Result": 9.96372, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.30318, + "Performance_Result": 6.23624, "Performance_Units": "Samples/s" }, { From 55ad6891fad53ee82b1c3da7f9966a1dffb3af00 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 20 Feb 2025 19:45:31 +0000 Subject: [PATCH 139/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9757 -> 9754 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a85c69069..5e53dad30 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-20 19:23:48,399 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-20 19:23:48,399 submission_checker1.py:3270 INFO] --- -[2025-02-20 19:23:48,399 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-20 19:45:25,912 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-20 19:45:25,912 submission_checker1.py:3270 INFO] --- +[2025-02-20 19:45:25,912 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-20 19:23:48,399 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-20 19:45:25,912 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-20 19:23:48,399 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-20 19:45:25,912 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-20 19:23:48,399 submission_checker1.py:3286 INFO] --- -[2025-02-20 19:23:48,399 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-20 19:23:48,399 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-20 19:23:48,399 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-20 19:23:48,399 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-20 19:23:48,400 submission_checker1.py:3307 INFO] --- -[2025-02-20 19:23:48,400 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-20 19:23:49,379 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-20 19:45:25,912 submission_checker1.py:3286 INFO] --- +[2025-02-20 19:45:25,912 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-20 19:45:25,912 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-20 19:45:25,912 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-20 19:45:25,912 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-20 19:45:25,912 submission_checker1.py:3307 INFO] --- +[2025-02-20 19:45:25,912 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-20 19:45:26,966 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436551 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433605 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 3e6c7a340..3efc617c3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -766 +767 diff --git a/summary.csv b/summary.csv index bef16953f..f14d5d2b7 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436551,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433605,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index a5c0099d3..89fc1a4a5 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436551, + "Result":0.433605, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 3d2339692be9620fe1ca73d2cb7574d19f220081..b81a9466dda800d7b29bdcbdf81b870aa7534803 100644 GIT binary patch delta 785 zcmV+s1Md8tOqxuvSqKHm;-{=dvt9`82Y+YfH`i@UrlDgp4ek_isJjPfJnHTN8b&74 zP%@E*l8H2wOry-*gA!0O%Mx@9=I&uFUDOHP7-3O85XHy@8cwEBv%BXtGXL(*NG?{ia}Yy++);i?s<*LY*^qZe(0{UmfEVUUX2}7kFMFiMZI>-t?#!_ zWipZ{axzJkFwRfcDst1h`qKZEIJjMXKmJ9Y_ZQCwQQ_zhCaZ7rXVGk=q<^e_C%`Fw zCqOTfg~O}GL@L?ST((sCIL|+!qJLOb89WYGzYw6zlPFH2>}l0|@e>7$Nzk6ggZIlp zEoRXl3;F(C_|7?a7djt)QH&2tOUX-rmgdzhdnsO^VYLtE!APD5$taqo%0v$HLUk>_ z0Z&%HIk0@4#q;GWTUGAjEQ_bl@5VA1$fS5@s>(2qv*$O(DcSsXL;FuqOS8-oeGUaG zlA*&UlTIKWf6Y$AFc5_AJVoSt;v}UNSV<}Y>WNB71W|Fb>}_ii|Hw9^eS6$ANn0r= zj_aBIc6Q}rd5}%?3)Ts(N|Yrjia;^Vh1!+qqh7@iC~}@EP8+SDL`QIFS=|-P5T-4> zS#7|20WJz4d!Bb8>#V25F zMk4BwoQ+8HW19}Z8HNTVDDQBVWccFsV5OUE=#(p`6#i&n(mIHv`{2ZAY}+lZoq0gplea*jJF zpR?zeCVGXcJk95En#XxsXOHB5PV&XqY2;1+1t|L=|0WKAbMD(uQ2hb3k0Hte1wBL+Sr00000rmJx| delta 798 zcmV+(1L6FdOr1=ySqKH2W}se4vt9`82Y;vVn>+TFU3~YeaXKc`;7$>Tx_f}eqwXG{ zVPql=B@<~VnMgy)G|JpPC;=t2EJ4R$?jF|CMV-)%5f;@0QH)HW;ba;$yL(O}^cn_3f50u;};!txB(0GDES;5?cGz`iL<{mU2heBCq?m?X}D1Xb$ zJq9!Pu$C_9#J4Dv%-n;b7?c&vJx0yup4XVnh6Rq|hwdtCsjbTD)z~rh=$h?YbdGFp zeZPGwlaV};lS!(Caelg1k(<`lm;SHB!R_k%@h|eczj!u?3P*o1S$&&7i)JGw{bThz z0Z#Ed0eX=v99}IZQpuj?vZcz$d4K*16~(H`;BmP6g#cxqL~#;jPpjUGpD0*Ng7!2X zyk8D#F^m3K$oKEUch14P(E0F-Vti0qN?!W2G_P*iOYs5?t9>{RM)EvJM$s%)CUTe; zs%!ZTc(VGz5k+5&$9v&IpB z4h8+#90etlN+2G8%TB{E5JmTVMdW?rBn4EgB$WVlMI|JHKPo2@Vy+t+tZ-P}&`THk5tS9pF4o@390=IrS8u zfU%i~Xh3o{A9Blk_A(0qB!9B{~6GlW`>`8~xZE1tkFh08jz|01*HH00000 c000000Dyrzld>f~0gsdMB_jqwBLDyZ0BdP-FaQ7m diff --git a/summary_results.json b/summary_results.json index f2ce751ff..c3776e3cf 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436551, + "Performance_Result": 0.433605, "Performance_Units": "Samples/s" }, { From ec0a702a78b245e4198d9eb60ede81364af49cff Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 20 Feb 2025 19:48:45 +0000 Subject: [PATCH 140/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9754 -> 9751 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5e53dad30..4283dc290 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-20 19:45:25,912 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-20 19:45:25,912 submission_checker1.py:3270 INFO] --- -[2025-02-20 19:45:25,912 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-20 19:48:40,344 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-20 19:48:40,344 submission_checker1.py:3270 INFO] --- +[2025-02-20 19:48:40,345 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-20 19:45:25,912 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-20 19:48:40,345 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-20 19:45:25,912 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-20 19:48:40,345 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-20 19:45:25,912 submission_checker1.py:3286 INFO] --- -[2025-02-20 19:45:25,912 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-20 19:45:25,912 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-20 19:45:25,912 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-20 19:45:25,912 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-20 19:45:25,912 submission_checker1.py:3307 INFO] --- -[2025-02-20 19:45:25,912 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-20 19:45:26,966 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-20 19:48:40,345 submission_checker1.py:3286 INFO] --- +[2025-02-20 19:48:40,345 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-20 19:48:40,345 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-20 19:48:40,345 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-20 19:48:40,345 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-20 19:48:40,345 submission_checker1.py:3307 INFO] --- +[2025-02-20 19:48:40,345 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-20 19:48:41,304 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96372 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.77098 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.255097 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433643 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3efc617c3..dc763b654 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -767 +768 diff --git a/summary.csv b/summary.csv index f14d5d2b7..a976d020f 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96372,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.77098,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.255097,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433643,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 89fc1a4a5..bc0be0f18 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.96372, - "Accuracy":"acc: 75.200", + "Result":9.77098, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index b81a9466dda800d7b29bdcbdf81b870aa7534803..42f554a978e0558ac995cb062bbf090045083805 100644 GIT binary patch delta 896 zcmV-`1AqLQOqWcsUK0iUU=k`|vttvP0)LXxQo{i}2PGgSh$N>Sf)$jaqrKLJ_@@Jw z&z~$VJxb!Zs-H=HqrOHoD^lo~{mt00dvK=*SE{kWy&l}}!4+S&I93AK;639I_l!fV zjKc=+@fY{_i+lXVJ^tbzf3e~(L~CfzuVxzQSsHGUWmpssV~gearj&Cb$v~M|eSfvo z#n3g4*XlG5D@<&&r|RBp_XTC!!;#UD@!jG6%1G6{=qM%jQnEi^k_e#S1uZ!&@37&k zk4aB2|39`Jw`w|K8Tez}dlBL5D(@M_{Tsn+$^LjoTTJ3Ed+@Jz3!eUV?xk@DI1Z8n)5p%XZknECT#|B{GBb` zHC)&Ac2;R(IKCx!?UU*G@^pGG5m(u(#;3FS@^XB!sP}7EZuNd(k-*2K{zK?^HhGj1 z`4!+3$MXX(a2;zfP|~^GuVpK!`}MKBu-~zgKlmw+1dbBa2W$12U27)iV)P$SO9KRx z5hWV40vJOG1*jLzX(f|NARm8j!Y~ko?|q8M_ry-16}6HewH2o#RVr0cZ1&{5nNr&x%k+hTvq_&3`EE(m$R z3IY=>Fx&%EI}*_mDfo!AKdu=7f@5evfes$$S&q-%08aUv4TEywR5ElHZdy;`=-zud z8r!zb+R0#z`E$HpzP=Burc(E(;(+Q>aLOI9U|e0|%OW1GVLmM5i8m6&Qy29cZ^uOr zeISw;fF9zWl+VfY%W5Qn>Vg#aDJjx|tnx=XozW@z>PKGnpMmN)-;E?&30c+i|O=BbI?b*1Z=IzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8 zEZ@xbhB$tF8$3t%{(bN+jPD!M4}a{04mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$ z-bFK$UkB-@#bn)CTY(aZ zo#Ie~^w=0^+ZcP!qNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rLZ= z1QY-O00;m806zdBon9SB0+Svk3bGhO2n8yVp~EJVNFX17PQx$|gzr2>Ir4?97 zDgo+=N=O7zakK1gYZ3p*Hl%%f+%!pBDJPEWnf-QlXDp{Nb_Tx4!{|P1|%r&aF%5F;`Lyq zn{4QmE2k9xXkgMhh@<=9#As~WHfd*_HRR9mX8ro!tr`n;Iu!#{6`T=ffqHGL0$&&5 za1C{S9ZtLzz@LsmpLpvRIrjmNLI84(J1C#C=a+gVg{nNw=W&|Hd0JB@mOB cArK1y000000001hfsB*xB_jqtBLDyZ0QQKfJpcdz diff --git a/summary_results.json b/summary_results.json index c3776e3cf..53ad27763 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.96372, + "Performance_Result": 9.77098, "Performance_Units": "Samples/s" }, { From 95f0dc4962daafa51671f966c5249e94df8dd9fb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 21 Feb 2025 10:04:39 +0000 Subject: [PATCH 141/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9751 -> 9753 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4283dc290..85e5fffca 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-20 19:48:40,344 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-20 19:48:40,344 submission_checker1.py:3270 INFO] --- -[2025-02-20 19:48:40,345 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 10:04:34,575 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 10:04:34,575 submission_checker1.py:3270 INFO] --- +[2025-02-21 10:04:34,575 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-20 19:48:40,345 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 10:04:34,575 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-20 19:48:40,345 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 10:04:34,575 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-20 19:48:40,345 submission_checker1.py:3286 INFO] --- -[2025-02-20 19:48:40,345 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-20 19:48:40,345 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-20 19:48:40,345 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-20 19:48:40,345 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-20 19:48:40,345 submission_checker1.py:3307 INFO] --- -[2025-02-20 19:48:40,345 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-20 19:48:41,304 module.py:5481 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 10:04:34,575 submission_checker1.py:3286 INFO] --- +[2025-02-21 10:04:34,575 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 10:04:34,575 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 10:04:34,575 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 10:04:34,575 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 10:04:34,575 submission_checker1.py:3307 INFO] --- +[2025-02-21 10:04:34,575 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 10:04:35,808 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,12 +28,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433605 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432662 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.77098 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2844 | acc: 76.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.255097 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433643 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index dc763b654..7fb7d4db4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -768 +769 diff --git a/summary.csv b/summary.csv index a976d020f..293127db0 100644 --- a/summary.csv +++ b/summary.csv @@ -6,12 +6,12 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433605,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432662,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.77098,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2844,acc: 76.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.255097,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433643,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bc0be0f18..8503c3092 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433605, + "Result":0.432662, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.77098, - "Accuracy":"acc: 75.800", + "Result":10.2844, + "Accuracy":"acc: 76.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 42f554a978e0558ac995cb062bbf090045083805..f0d4cce1ae08d2ef225e6aeeb8a0ba241b8251d7 100644 GIT binary patch delta 3754 zcmV;b4ps4&OqoouSqKGZ*bNFtlU@iVf9-Rk$QH)G-@k&Q=EF?Y25H_&MYod7TZ}Q8 zOp>kpDMheVYE@joy!`hIYK)%qK-k&YC3WwY8Eu|(x)1V8H}sE`^Uc-LTz;pmVMtu@*&-NtrVr) z#qsaOo0CCX)eC8hX+JNAagitke>n^mlk75n_$s5(ILl9)U4(@3BwhZiOr!i<8mcl4 z9^0y^Di6^h8yAU{aWojn;zX-TxJa{j`rGP|SSMJ83TGWFbg_x{HDT$FVHcmr(?qUg zTB}}{Bm5X-LFXWeACzR-Lpl$NPe$w;Mt)ktqV-$RT#T)?Zn%7IQ1Uv8f3tVrF%Mez zMdG#MI}N#_4cD~cW~-rC zTA5_0S{qh-XtM;~ZWl*fe?f+M%oT#L@DWJJ_45fx$cg_9Bn12~KtiDV3M2%&6Cfd% zSr440tu=26OfR4KXZvkvV3`#fIo>}K@a2_IjIQmJ|BCYf1*xlLyvdhlhb)d zWBNRX&N=5GE%^v8UdBD_eEsjmr_0_+|MdNGHI(Mdp;45L7k641Pg~PT{_iwj>g2EH zG8x`Rv!P7#1)IFhWSZTki@Rx*7HgmM_TJX?R+dP)TxGS2#Z>F8cf5{IWSXyG($;!0 zv$x578_wr@;e4?df6r!*$zqn}tEs)U>NbpL!)T=bq7GyhM3YqgaL^ip`TAQECFQGD znGbFGU&*+>uf`=-YYfdYO~*1ZbS{R@<*24QhOvzOn# ztWxcbavtYTS)7F9+j~QEG<`2z2*avDViTp*Ao1GWBGGa+)6+{p^e~7X2a#MqM_l5N z>*ttD7>GbZK?D*CB9KrJy$TR{I0LaFybHo9h+YMVVDJ<~kAlc!EE03hB|ec{c>g;f zHc?6q5O2Jle<{(lJzJFA_V}1YpOZ+g{v$4N=#3ATFcN`;k_aS}L@x2@jgMbJA_ylJ z?MX#=*C(8u(uN-IfUQGmOdrJ1btsUKkKn>@7Pe^+#_8T9SY~+ioS(fWZKAvy0ABiA z0D7A5+a(jB&@q3@b^;)oTaJLnCF3{-8ipX!PzI5Pe?kZ}9-R+28iXNAKsl^R(A^x| zkjPqkpc9|uIb%_M5Jev|o&$~0$p><=HU75&?)+#}g$~;s@=Afl5FVRPd~1E~ZF z7?l+Xf4YKhC7iOB9_Yk3D3vs-kD}-ti8OSQ+#3E{gmPE&A}HYX-f}u;+vF78r4Tpj7hE$I^CV#8h;~9F7EP~aIuDx0GZuS zmy=~NOA22zegMfPs;jknwYHEHnkziBY?dr>mWD<MK?xX_Rc0Q++&rwM2Rh+gGD8oD;)__W0gKp%O+Eh(m~YhD z^IdFvdkVj~;W#B5mWGbWG`Lg5q3#}_@u<58Xc(DDL&-!MN+!}!GL0&Ce-BDP$*fAy zHJH1Hwe(OYbYp}?^+6OP6KFV@M#Ju&*U0?4J0p$MR|iT%--S{X-y$x7Qj|c+jUo;; z_W+G2D3mqKJxIf#tYPj!<8dgIRpuVl34^lA++#3v4{PaxPJD|($;>?{ia}Y!++#Fs z?s<*LY*^qZe(0{Uk=p94Cti&mQ;)9MzD2!ut*!63Ph~Qa$8s`Bl`zgv*D7+;y86=p zrP#k&lNS*qRm;UhD%sOqwpIB!&p)A}SXCK34p+Yrpv;pfPNM8-)qC+11&c}0nZ|?n z%Rwz>(H{%>{$2Ra*?$*0AAV7c4@z6fOMjN;)h&A|UZ7!h4(7o~o&}SS5j6(qPf$w( z1e1>>8nfgPdI1D_Q^6gRViPZalGtgksC0zdf~w(C6&Nv3m7AQD>?CnEc9=51ejOT^ zPMj22Ktihc&hhztu09!l*<>lYgOVpKKh%hUH3WIcq9k7*YS&lCy00NFXdcm&<#4Fo z1J}Nc{u**FkUX06L+w_G;=nX{cncZj1}h+!l9-hl74qM5ZStZ78u42l0FkApg|Ta9 zG|4p-vMLvcn&mhes`BLD3dUQ6;FB#BDFn5253iG26p(*d#~70y-0HyxZ9Jla%2Je> zFzdoM&rZiI%b2W#_3e^|LbjD&LJXzsD$v8C(u;c;sql7rXBnuNR z#?B^aasPh{augshaPSQxu0CcZpx)l7k4)#^$H?>C0dnnU)SDooZEp5RZB^P8!o|h) z$&?%*mlz(l2tp5lb$|w*5x8E^0H+cNjDbr$gsv{WPR}ow^XX!Fd2}{Eoz|zZO801b zKn&aR-QT7i!}l%ww`I#W9FN$YmWg8-7@;-K{2zZZG20fFFvc~Utv1CD+HnafJV2gn zxL6Gb@EnwYlpvCvb_iBbijMYL7vi4|SU!KUxb!HAQ9O(-mgk#N&V?icWoGr&QWrzlG+wLIIIM_en>|(cX1gya+a8XLhK%nH z_g6-$?nOr_v6qtl`I1Bc1utmHVR=UkUwuq^dinpc?YLFbS=b;S>)wk9Usrk0Fz(+7 zUQ71JGumPjciDq~wPPeOtT#qd-8nw8EZ=|3_J%lqd>cGR_x^qGE{yLR(+})~4mJ_j zM5H5ieUWU`I+oYn7DpYsYsVCLZUuf$-bFK$UkB-@#bpngupED9OP7nS>v}t@v}HIxmb>=J^n7_bJ(q~9>{a8_ z*?f69zF5@zwJW!JKfqSt6RZA1=y*1Hlv?sDz_%RF54^y2@L-^%^Krk%>VADJFYI@$ z1d|RW8nPHe2n5;RjR%uQARvEg!!Qtp?|llR zyDiynN)fgLZON&DLQ`6D6VfITwSJJ)I(hrbPU0lsQ;(~e{dRUyHQ%c?`32{t(KX65 zf)dc&2&uO<`dBT}N0j)$v|z2#P@@BQG;i)IZYeho-kh=Ef&`yLkoJ_@8tp=`6l2eK zpqS61Lq~CAoMIuCZi{~{(1<`E&fWe0?8QO{MNn#Q`;y;FLRH z!MLWv*F`*B%Y0bI6K^Dlr$f{y-cE}g`#>Zy06oS%E1%`_%W5Qnnt~LwloV-^uW~}k zJuPSJe&kL61*lF#{!Kgsj=66?LGuT*i6P1Y1!vd|3P+PzBwGQB@mOBArK1y00000 U0001hfs2#wB_jqsBLDyZ0B~UEtN;K2 delta 3718 zcmV;14tepJOqWcsSqKHm;-{=dlU@iVe_L(>05i79<=>ADguvbl$J5g@j?Q`MhVt6GY8!v4vTMQNuaDD-axarKif1iV zXu7J%Sr`wZ*{G%7Tz_tOs*+~GY!FQ1nQW;~GF5+V|L5=_PJU0vQf5k?V3xMje{q)0 z51P$19Ls5tYVlmo@*~4InFd+@HW@Y3c_M?sGH5z!8oKT@r$IDRR~ZhHDmmieFbd_z zI9yESEL$Z>QNFoY^g`$;N&+$sLl4_vhUkSKBQZ(m7jbM%;jCkYE;iA=CM?}C?Beryn#grb zbJgo|gdc+}Xdfo=gOV(JNasQE$%q5P$WKdHw0=?c$g#f5i`M)aL#~)oZK#ukh6QvB?dXWJs=@x_X0@B*}Vi3a(2G~iG?pW2VZUuOJI{?j=97k z=l+CCOj7ShAR+aB0uoa1XD;zbmM_l|@F(#r=z&}#rzPQ?=VQ-Pf7BUm=<*JHayrjx zOpnLVIp+eTB_F}%%eaT_um8RLbk#lWoxNYKhSGdFG>WqE;!X?WX>&Ts|DEPbo&42Y zCc}O-8_FbKu*rTV)2yE^?xs;%tbNk{y{+lZERk}#%4!yispfh2WF4Q#G+)D{&GllY zvB}&I=krE5Uo_&`f9x??%(8qn)mW?gVLTf~BlQ<`AhRHvr0R#m<`B%+-|8qSU$x47 zXv_chjO+VqTw=Ax&@9t*EE7ZLV(45B9l6wwxx^t?>Is)H=zxSm2P70aAR&LkzqlpP z@o*UouL}lG{^p;%lt>;!*KZeG;uFcGe_JDwqj5h7i~E^Ie*3aY z)fnYG&Y!Y43CI0=Lvu8}5iW#bRUomBQYw&m<8F~?xti(edqDIsh#m)#TtCNL;*jg- zgi9ERKte$T5(*-aP!PQm5P3KQu_U|$!YPPe35a0u6hx1L$YU%LbHOD(kz9KJJ0R9k zN(B&ay`3r1f3!VY?78jnF^N7WkzD=9T;k9hA1+}e0tqD%NGOS1;?Wx)zl1~(PA=Nh zlJJgCI60#YUETp(htilHh@tCHAR!;YrC%>>(;$q~#w1u~c=Vi~y(X=ryb1tb`C9;b zn(y0tCPJZO{+8_oKr*)+1C2|@aRM|9L8PG!A`OKQe`q{9A8s@VLzIAWSeBr(Ik+K_ zwRAxzKFM>=qIw{TK4!cC8ljU9Shf89zrV=Z0KiEmIUX;cqI(K!-n=p?x{{I>|@)(lxtY3Q2x0W9mNvC34r9++4^ zbk!VT8GEqATTMrT@7~xB#gc1WgJQ`w97~{KSON{hl504YMj4hM0mHH+L1%OD1xwn} z1)Xp#X;cqHu}7#tBXmKzH2x2`tfR&%k5$(Lf7{$d8m6;nOH|-J(IdgP8f}+y$u+J? zx#SwoCD1S~frfF(HGC0kmT?IZFfL0HbT$WHxTGyzR0(~0!=id1igO7zvN@OLo92={ z%=zm`)=^$1B+VTp4cpeY9&_(9iFfah1m6+2>5ewgxE5uSYdDiY!HrO=vVvC!e`y$% zK*Ojkv+JM)jLI^@j-Y;;GuF}to$yT&=F0(5d{cyLz@|ve-W*Ql&)*e!qn@2>v2EIM zG{bcESyqL&2TS0|p|%~Maj90Wtn*gC16;V*>?nW*POAIF6e|m zeq}Bl6vZY>pkWKxnvFWIQTcNof4{y_3(u9<_VDD3SmB91-_sEO9xj1HhnjeR#-%PE zpy6Cr@bMrG;}U2Xmt|HSlz?$rX66yh&BI!{pcBp|GxUHczKG=-u!ybM)bsCv`9`fh z-^I4KXXQ87ZA_-2V=@iy6mh7#2WUL%?g1J`Celzck%p3qG?YxE%-w?$e^4^Z5_Ama z?qMxm)Ct`fVNpF0#mEF2PNq?_yXQ4B|L)F6E%jA_($III6velQ_dqH3K*^0F4mI}x zjVCCS70f+I!=S8S?m^>mD3oR99@Ghgvdr9LFmn%U>4Hvti$clFJt&GnS;5?6)NJl~ zjmd0S;3$6RuCkWes;pj(B^^_buGzjty>`v5@3&87GLk29GD(#%&QI4Wa?`r{(*Kn> zxLuPO5hGQr#Y8IE(_FSx`8dx%p`ut-89WYGzYw6zlPFH2>}l0|@e>7$Nzk6ggZIlp zEoRXl3;F(C_|7?a7djt)QH&2tOUX-rmgdzhdnsO^VYLtE!APD5laUcM2K!G?O9KRx zkt7khF>;WiteD~3Cj;PY*-qCJY-RluMf5Bt7F~Q5EnF$Xv%Up)b4?6 zUq*ioITuJC&H15rD@1W%nmoLPjBErP9+FBB;RCAw#=lUx*#f0k}p7Vg1G4?bw)5fxOHqRfO@ z7ruFRI%ZkMWF4$;moyZzt@ILNC}mfH9u}2e+{;LXx63UfV{x5ZV-0$F)IP}_C|eVI{!XKp63pbYd@pj1PN_(vqx&H(ykCLF0N0e z_yD=s@GOE6dH~1)8hA$FdO-u6N+2)>F7^<*y7)Rhzg*6zi{<6f+5B``pSD!GN7Dmr z*u-~#n|2J}C-!g4#5Wud+ntuNLktU{HP8GXe=;%K7M8GO)o`}j6gz0gC8Y2Gd6Lmm z!vQ=8B_Jh;B&QvM6_lc*z1D^JrvsMHpDZptO5(VxpGkeAzD6`FQs|id&DgMeaHj`X zswAfj6NE~3Ol-5K>fUVk1!ddAk=j2^9Gx>Fpep-A+H^B1x_ot^5J8*8=id8OskzdS_ z^4a%oHklov6F8Qc0JONeLADhrk=Q8?HAs(*fwqmY=PYUptF7`ls8BPg6nTuAf7EsF zooec`qxk`{1H%cF4e4o>CI)f*oh{upT-Wt>R%v25z9o0|UwUQvU6{jLqDpgT$mc4-0_(!(ekhiZBAb`ZF$Mwv9JG*i@->WA11?Q#F70NS$ z642ZTskasSSS`{SN_=2iu*PVpPzN5(>${R$%8i3JXDqlN!6y-|ASyAUkJ*z+AI z=CkO~QQR1(Scs+DVt>o{H`@X(2zkH?0uwAS+yhfP644PU_=vPWt{DJ=V`xBu4j$)O zj?dlzPWhV+gL2_iGISPhT2JEW-g`M3+qTWx$zYB7bG%-@z7MOWQun9gfa+3k${ny^ zTwUVJA|9?`J}l#jHxk5C7xf!&$3+f(Ad(n>9^#&q&&l)4Y9xW`f)w{DDbj+h@<%$I z(JA@rM_%=xf$BKq-^3H(ko)=*)PJ*!A<6;;$>OK1MUz@2TLH$Cwj?4P{9qC)U<3dF zbP@mn6aWAK0000000000fPqjGlWr3hlL{p&0pgQ2B{~6AlW-*_8>kn}X(a&w08jz| k01*HH00000000000Dyrtld&Z}0g03FB_jqqBLDyZ0N`2du>b%7 diff --git a/summary_results.json b/summary_results.json index 53ad27763..08ff5e4b1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 76.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.77098, + "Performance_Result": 10.2844, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433605, + "Performance_Result": 0.432662, "Performance_Units": "Samples/s" }, { From 9b0a4d2aa243f6705e8bb3dc519cece7473b1acb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 21 Feb 2025 10:10:42 +0000 Subject: [PATCH 142/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9753 -> 9753 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 85e5fffca..224f489fc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 10:04:34,575 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 10:04:34,575 submission_checker1.py:3270 INFO] --- -[2025-02-21 10:04:34,575 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 10:10:37,645 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 10:10:37,645 submission_checker1.py:3270 INFO] --- +[2025-02-21 10:10:37,645 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 10:04:34,575 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 10:10:37,645 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 10:04:34,575 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 10:10:37,645 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 10:04:34,575 submission_checker1.py:3286 INFO] --- -[2025-02-21 10:04:34,575 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 10:04:34,575 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 10:04:34,575 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 10:04:34,575 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 10:04:34,575 submission_checker1.py:3307 INFO] --- -[2025-02-21 10:04:34,575 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 10:04:35,808 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 10:10:37,645 submission_checker1.py:3286 INFO] --- +[2025-02-21 10:10:37,645 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 10:10:37,645 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 10:10:37,645 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 10:10:37,645 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 10:10:37,645 submission_checker1.py:3307 INFO] --- +[2025-02-21 10:10:37,645 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 10:10:38,600 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2844 | acc: 76.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.255097 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.433643 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.23624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 7fb7d4db4..361dbd8b3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -769 +770 diff --git a/summary.csv b/summary.csv index 293127db0..b914dee16 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2844,acc: 76.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.255097,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.433643,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.23624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8503c3092..436780006 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433643, + "Result":0.419926, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f0d4cce1ae08d2ef225e6aeeb8a0ba241b8251d7..d326473430c8d3cda64b3fed84a6186200b8d72b 100644 GIT binary patch delta 2787 zcmV<93LN#BOqoouSqKGmmTBoolU@ike~L87TPnJhWZq(o$z+mj-A@+5R;g8S0rT?T zFQ_qk&I4g*XP4ByUuLv<&gnkLFWu0O9{&0`n=1D*PQqy3R)wakikt`0Fr1Iu>dp1% zhN~(`>d%M%G@8q{`Xm$e*Uo^)f zisG4{W^dzhGg-vaAFhIC)25;8c5~*3b9J5JAg+=l8jZq0evE?UOwQAFl2}gtG#h?0 z2^WdlIUI)BvAiB7mZP?MBo0n~Glbgg9IpDl4dp|!^IFMExr?IT^Eao%wyNjSCewai z4dXIahH~UDr|DJn@KuK6Nt&HDe=83OqG_`FSDA&`xinN|=0CPoQ&k?qVLHhZE0b_I zl=+EPm0+2q(d@VNA+brY4i)w$R_J0I?P;Q=+lG~Y9#2!biD|BTU5)UgpZcA{IC@ax zRS(I+&p#P)U>MnH3G&u&d2>Ft=BDB5xk1V5$j{z=$2@G_=ZQCp?=<9^e>U9EhTENn zW;UMfpbR`HgN;GoAHx+_v|41CKw|Anu7Skfm)rn}voE;?5_c=vE7!#3nk<1J<#GiX z=l53?*RLvWURB(dDfTLD;YwR9-XN87Mav_lzAm|bU2?-E^m8puvRADIt39$<0(ZZQ zW3C{>JmCsKSojDe<60}^s_dq6_Y z?m3qj5=h9|{RSj0e7V{9aLE7a%S92rgg7 zJ?eb@@8zef-f92r{c1IoWXqv(m`;{=S`f{evuXD4EL-YiuNE>M4Z`_I#@T{R4pNz< zgJgL(3zK~9lML=HO>d^Ll&e)%Gha+K&wD4E_*f>{8YXFO7Bh`)=0UJnG=jyl5zXh1 z@p7JKtEt9DH3*{le<&QQzo5e|4(HIOB2gltytd8QdG1 zt?7+mDU4PH66+|X0*N=y4v8&CGhN-pAi5YtmxD;IpJOhu$@O!>B@9F$p&$YY1rbOn zh;9jpJe+}865a*j6hyZKL@;;?qDw*KF)b2v!6hD%T)O`|Al6Yz1rTrDy(!VOTuT(( z_IQ{?kCRBQfBs`GvFVKumoO56gpvp(lteCZ>5Y$9LLvw!7wu_Dc-JGGoY96J?|`jC zX-pr)&~+$~kdNTfs~5JJA4ExG>aQ|9y7tdrlh#pQ1pu$S9ROX;^Q?l2Q0SPyRXYKY z%q_=2njm(GV<4Z;v5pd6MZf9P%xZb@VC&3EsIjgHIATEf(P`BN}yp>0u7^*YuMbl)j%pi0!C#?g07%j31_UO2RiW# zN+pfzf1@ZmMVqiu2o-3A zE-07A{{feE)L7-Q>Skcun@Gd73${cB-V;3%e|)RaawwNv!z+R{Un(5E*nst=+#mtZ5Cb7{V5F3H23zm8-b^(MpRFd#e3lfRX8Zt>byQbr z_1fA&QfQ8F&7xVdz+ob|X~6O*kzC_glt`}OL;?*X5@;BaK*NZ%%7_FB7?C9jy4!;< zMADWX=!89XWl?<;#fStNc2jA+At3*We`l|?6j+H#Yxj=Q)O6h{VzR)7BY??<`fz~8 zp++2_;Y?Pr;vfxU5@;BcWo{glfH7I-#}Uks!&-Wv6V4>_R00O#`)j=9YCD1S`%j`NR0i&|af3PE{ zpXQ9U^gt(kQ-t|)Kos8;;To_hQnNRQQ~C3EMc$}q=UVKVc5KZs?IO#na0{>mo*Zi1 z0UC$;c7TRsS;4u3Gz?3iVOW-#cTfU`Wtn|PPu=ke*cL*Iu|e?F7(Sf0q~G*N;mJ6-F@ZR`3=|5xJRcK!YM7kSoSG#`e! zqu-ydzs;Y9^RbfrvHqO^JO7;kJx>;lu9j1&q)!XkR^{U&`-JjhRb}`%TK__TvWUYd z4%4S~@A*#@ET?{F77gF82DO}re=OzucfmXR;9X#U_(d^3C~YMxP5oJ#Rk!LTe}RV8 zIb8T-dG5#KaGoerIm!yvwd@8wUjOF6>UA0|R?nmxapNPj5f{GF*PqbN$B z-{hxc``ZnjKLJon0|b+iBpS2k5nm1kWtD{jB$G-Y9)HVf!!Qs<_x%c^y)8dz3C4Dy zEm<{Cs7p(CLYg?D)&ohczVibU+igDSL z=)K;B8x&YaBqOboP@*0z+E#Z3)j3rLUX9XV90wbCkhD3~B|18%bBrxLf*>~X4(Y{% zGJ-gNHGd7(gnp9-a2m%CSb!spID!XYs23udk&G@#{bSk@z$k_m1W0Fb5+(TJbzp>D zEf|$6r{J#FuxdSvi~D5xV(hvu>ax-5^C$Ro|ME7D8geVO44Kt9DiD<9d@b0ob(RT`)FVVs6(QYUflPqz6QTHe%O zfMRa)Z{is+<-Yv{)gQBqA<6;;be3u9NRwJ5TLH^FY|y*(G)FmlhR2fsc}C(6A{(kPy_RiWvsB4=Sdh-Raq6u^0&#TmCh3x433GwkXR>Jg$idKD|E4m_BCPYj$s#{$J0cvV_K_Tmm~ZbWI^X3i64|? z*+V)Hicd!D8%BOw!lLzC(OitJwQjh4Zcy?%inDj$F%MezMdG#MI}N#_e+}2P;byC$ zm5*mLC<715U}MmC$8gCNmPLjMB=)Z43P_w?$u*F;yOJ9q@ivm3a!p*W$r1=sE?1Co zet%VQ^{V3fRmDw}Vy99IS8B0%gH*~DmQPB3U2^rh--Uo}x}^Lyvdhlhb)dWBNRXf6h7QAT9X_E?&kx z?0o(2#iz^ON&ocyay69Z%b`(}jTd)X7*AW%N&fFNU+Uzq<}w-HMzf(z@&%i`&19P0 zri;62loo5B^!DD?^j4Ngxm;znip5mxtarSQPh^^}Vba!mF|)VHd>hW^d*OVs7tdyo z$zqn}tEs)U>NbpLf5T{`{-O?K7DSU&{czA4g8BMe6D8%VR+$fN`CrMnzOTk5R%;B+ zGEK)aF?245&gIaNOYMkD9CD=|a|weENGNnbLZJf^@+bU@TLB#p_b7A~;aw0;{vw}l zhm-nxVDRK`{+UaO*24QhOvzOn#f2>mNjdC96Pg$IVppxdY}`ZZIMgyq?2^f_X3A%!AC7iOB9_Yk3 zD3vs-e~+T*9Emh^lH3~pTZD3JhAgNwbWQvKmQB=HXR2HeOspTeYL2js5-jmn)1lzI zH?~8uxr3x( z+xpgH?p-GF?){E&A}HY ze`!k(bmE(oNgCBhQH)8h5jx$S-Wq=+OfK&7nQ*a&k^q_APnVNrF-r(l9Q8hH+VC z>eL*Io`6yG8)fl`z}$&DfoHTM9GCn%IP%soiMpsZo; zLE~{KlvU;))Cq&K%G_fxa}R6jflhpjLdncMD2hQ@!`x#uZ0>oD$!u8QD1PX!vXR>A ztX_>BQ;)9MzD2!ut*!63e@|sHlE-o~NtH0pPuD7P)4KZ7|E1W!S$#kLMV|K;&jwN9 z=np2VZ}Vr-Y^0=rtbQlJDSjtFFOr4B%f&=0+0$IMRrxs2KcS*nRT(@ESHBRT%#$ci zqU>qad+`$mi%HO##)J3EK`my{9}D^ZUHHz~e-}C*eo>4MN?XZGOn;W<)h&A|UZ7!h z4(7o~o(0J$nx)D_4)a2FEx!RzR=+v0e4WMfWwH1?Q#FHOe!B642ZTskb%y zSS`{=l=#53V6D+mqXT#}Z|*8?DK`$@oU!171fN8Z_LSQi?Lx2=W6yV>n9rg^M{#4E zVj-4pe~T^S-)sxGAmjln2u!fRa1TuFNJJ;3;3LxhxOM;tj-dqwI(VFCIlg!UIOQiB z2Ib1BWH?xuw4TM$z4vl7c3qcsHEobX~ Date: Fri, 21 Feb 2025 16:27:50 +0000 Subject: [PATCH 143/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9753 -> 9754 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 224f489fc..e353b158b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 10:10:37,645 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 10:10:37,645 submission_checker1.py:3270 INFO] --- -[2025-02-21 10:10:37,645 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 16:27:45,436 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 16:27:45,436 submission_checker1.py:3270 INFO] --- +[2025-02-21 16:27:45,437 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 10:10:37,645 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 16:27:45,437 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 10:10:37,645 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 16:27:45,437 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 10:10:37,645 submission_checker1.py:3286 INFO] --- -[2025-02-21 10:10:37,645 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 10:10:37,645 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 10:10:37,645 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 10:10:37,645 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 10:10:37,645 submission_checker1.py:3307 INFO] --- -[2025-02-21 10:10:37,645 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 10:10:38,600 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 16:27:45,437 submission_checker1.py:3286 INFO] --- +[2025-02-21 16:27:45,437 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 16:27:45,437 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 16:27:45,437 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 16:27:45,437 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 16:27:45,437 submission_checker1.py:3307 INFO] --- +[2025-02-21 16:27:45,437 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 16:27:46,453 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.432662 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43177 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 361dbd8b3..a01f8275a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -770 +771 diff --git a/summary.csv b/summary.csv index b914dee16..750079d78 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.432662,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43177,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 436780006..da0e6898d 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432662, + "Result":0.43177, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d326473430c8d3cda64b3fed84a6186200b8d72b..fcf6fc87c64b1751ced856dd4e7c31458f605775 100644 GIT binary patch delta 736 zcmV<60w4XEOqxuvSqKieZo~>l2><~8K>z?40F!Yf8nb~2&If;R3f8fPj=?mzQN*U^ z9-wilxd&(%m`FpxL>dYv(oissGII|~K*20a&^4I3hqd%jCv;zgMfE`x0~2UCm`2U! zp4Y(qyBj05)K`T`L*Hjo48qX7KM^|dr%aEvVynAsM*`|8k6ZL$5H&yO=T^$Rav|md!8O$ zv%D6y+BJ8+-aeD@Sf0q~G*N;mJ6-F@ZR`3w|5xJRcK!AE7kSoSG#`e!q2Hgbzs#S8 z^RbfrvHqC=JO7ygJx>;lu9j1&q)!XkR^{U&`-JjhRb>(QI9mTefU=0gC=S!7b?^Bv zlV1@&1^zo;|KyXR5h4lG4V^y$P)h>@laVAEv*!_C4h5xN3Fs!1OCTP9%}&EG5QOhM zMdW+pB&7veNh$&AiAqQWQE{{EZEF$#$Tp;Xd)zchTPY`w>zVy_cI9GukWKUp)(Ndj zlqD&OKrzjQ+Lh>|Ud4|na-J$q8?B&3M{sCa-4)CbrY*c#ZNPc~E(##!2s0(xdv6HF zj_pBGmjs6j;#ONpeJJgJ4jan8=?-w7rVA{=Q%*g_Ctz$wBI=QxjY#ujn-0Jkh6W@k z?{Jo6_~P|orJHQ%lq;tc{%By*I*6nD;KXQb+cs%uoi*go@Mit`-mMx7bvhLTR27^N zW`TNbs{&sa;cyLgejQG{6~Ld4L7#Z*7diIM+pD`{*zuLR{_YAwj?|P zQWKMJ6Bd&SB`N{rlQkte0alZ6B_<~9vtkJC2Y+GP1sj%zj>$B*Q^cn39-wily9a0( znMgy)L>fva(oiyuGItM3K*=mi&^4I5hqd%jCv;*cL*Iu|48qX7KM_Tdr%aEvVytCsM*}}8k6ZL$5H&yU1cq`Raw0nd!`;;v%D7d+BJ8+ z-#(M^Sf0q~G*N;mJ6-F@ZR`3=|5xJRcK!YM7kSoSG#`e!qu-ydzs;Y9^RbfrvHqO^ zJO7;kJx>;lu9j1&q)!XkR^{U&`-JjhRT5?RI9mTgfU=0gC=S!7b?^C4lU@-%1qM7` z|K^jS5h4NGv&0dB4h3bEg#sj#N+2G8%WA_g5JmU>3ZuO(KWGWYcAza;HBhKaOLsz= zIHJ}ANv-3*ul$IUfL8@=q` w8)cP+0we(d08jz|01*HH00000000000Dyrwld>f~0gIFIB_jqsBLDyZ08Al5l>h($ diff --git a/summary_results.json b/summary_results.json index db3b99380..eb5ae5d1c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432662, + "Performance_Result": 0.43177, "Performance_Units": "Samples/s" }, { From e624a36ce09d1e85e3fc7a31f7273eeb3853759e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 21 Feb 2025 21:56:00 +0000 Subject: [PATCH 144/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9754 -> 9752 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e353b158b..38c3d7e6f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 16:27:45,436 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 16:27:45,436 submission_checker1.py:3270 INFO] --- -[2025-02-21 16:27:45,437 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 21:55:54,918 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 21:55:54,918 submission_checker1.py:3270 INFO] --- +[2025-02-21 21:55:54,918 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 16:27:45,437 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 21:55:54,918 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 16:27:45,437 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 21:55:54,918 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 16:27:45,437 submission_checker1.py:3286 INFO] --- -[2025-02-21 16:27:45,437 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 16:27:45,437 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 16:27:45,437 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 16:27:45,437 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 16:27:45,437 submission_checker1.py:3307 INFO] --- -[2025-02-21 16:27:45,437 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 16:27:46,453 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 21:55:54,918 submission_checker1.py:3286 INFO] --- +[2025-02-21 21:55:54,918 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 21:55:54,918 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 21:55:54,918 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 21:55:54,918 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 21:55:54,918 submission_checker1.py:3307 INFO] --- +[2025-02-21 21:55:54,918 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 21:55:55,932 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43177 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.439989 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index a01f8275a..6d629c4a2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -771 +772 diff --git a/summary.csv b/summary.csv index 750079d78..d7ac0b67f 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43177,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.439989,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index da0e6898d..de022345e 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.43177, + "Result":0.439989, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fcf6fc87c64b1751ced856dd4e7c31458f605775..c8d4e825c5259419a962d0d0acc2a892d1bca72f 100644 GIT binary patch delta 757 zcmV<~9vtkJC2Y=_e1?DZ<~!q3=T}@^29rKq(5KV!dAX6`YV zxqpYX^gt(Gi$clFJt&GnS;5?6)NJl~jmdPB<0yXUuCkWes;pj(JyVaaSze2J?V3B^ zZ=cC{EKlTgnkYe(ovwA{wsrlb|0{8DyZ(Osi#+Qunh(R=(eF>!-{w!l`B+K*SpQCd zo&QdNo+k@NSIenX(x-)NtMYM?eL{J$s(&(k9IbyLKv~3L6o={4y7&Ai3YJs9GmD1r zSA$y4!#|et{k!0ueef=@Km4K?AC$I|mHsTvs$2Dvzd*z494`E^Jon>qI8T(R9A$;- zT6O~-uYYr3^*W6ft5=q)+(l^`&7R*)q(78#{?1gDQ52=mZ}L;J{q2U%pHNE!1hdQ$ zc@72JsgukklS&{Sf6HpaFc3xe{R*SKE!)AR7~6riWYs{SE-l>&Y2t`l4%E#${8Y_j(sTpujpJ8EKV-67^uwwz?~* z&Z#o+YLo`!IM~30q|K==(a|}bV{GXW1hJ8KNG~3g5ybhce`&BL^qVw*(>T7z0vut) z5j+4xy%5oiWOPC5AJdKiMlrM?Kst+)D8U!610(Ee!Khq01$VuMRqI(?+$YNyW7l<2 zw;8QIe}X^vFK^?hA(z9dD4?ofG^YlLQ>H5L^~Eo)CD!fziVqyvWAF8qw|S7M4p`s= z<~9vtJ}Q0R`KslguQOoF!BNij(mrBL+Mp00000454%q delta 772 zcmV+f1N;1#OqxuvSqKieZo~>l2><~8K>z?40F!Ve8nb~2&If;R3f8fPj=?mzQN*U^ z9-wilxd&(%m`FpxL>dYv(oissGII|~K*20a&^4I3hqd%jCv;zgMfE`x0~2UCm`2U! zp4Y(qyBj05)K`T`L*Hjo48qX7KM^|dr%aEvVynAsM*`|8k6ZL$5H&yO=T^$Rav|md!8O$ zv%D6y+BJ8+-aeD@Sf0q~G*N;mJ6-F@ZR`3w|5xJRcK!AE7kSoSG#`e!q2Hgbzs#S8 z^RbfrvHqC=JO7ygJx>;lu9j1&q)!XkR^{U&`-JjhRb_wpI9mTefU=0gC=S!7b?^Bv z6fCEHXBG|LuLiZ8hkq>P`**=R``}$*fA~c)J}7M^EB#rTRk!LTe}RV8Ib8T-dG5#K zaGoerIm!yvwe0>oUjO94>UA0|R?nmxapNPj5f{GF*PqbN$B-{hxc`_m1b zKLJon0|b+hBpS2k5nm1krCtf>CX-4a9)HbF!!Qtp?>t51d*URe1zAZd0qTiLNCZ)F zv+Qka5&y_GqP=%Zf6k0^4U zDoz`%phQP-Xj$DA%n+t6yjg9)dI2s9Ams=%CE9y$2*!@>K~k3lhYI3WTSqx;~*Xl&ayX=j}^JFc0 zfOGEKPf+~ Date: Fri, 21 Feb 2025 21:59:37 +0000 Subject: [PATCH 145/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9752 -> 9755 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 38c3d7e6f..2433bb6e2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 21:55:54,918 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 21:55:54,918 submission_checker1.py:3270 INFO] --- -[2025-02-21 21:55:54,918 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 21:59:33,044 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 21:59:33,044 submission_checker1.py:3270 INFO] --- +[2025-02-21 21:59:33,044 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 21:55:54,918 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 21:59:33,044 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 21:55:54,918 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 21:59:33,044 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 21:55:54,918 submission_checker1.py:3286 INFO] --- -[2025-02-21 21:55:54,918 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 21:55:54,918 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 21:55:54,918 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 21:55:54,918 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 21:55:54,918 submission_checker1.py:3307 INFO] --- -[2025-02-21 21:55:54,918 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 21:55:55,932 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 21:59:33,044 submission_checker1.py:3286 INFO] --- +[2025-02-21 21:59:33,044 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 21:59:33,044 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 21:59:33,044 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 21:59:33,044 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 21:59:33,044 submission_checker1.py:3307 INFO] --- +[2025-02-21 21:59:33,044 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 21:59:34,061 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,16 +24,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.6142 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71838 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.26681 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79832 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.439989 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43577 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2844 | acc: 76.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1486 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.255097 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6d629c4a2..9e7e0ab23 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -772 +773 diff --git a/summary.csv b/summary.csv index d7ac0b67f..f9a42a097 100644 --- a/summary.csv +++ b/summary.csv @@ -2,16 +2,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.6142,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71838,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.26681,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79832,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.439989,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43577,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2844,acc: 76.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1486,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.255097,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index de022345e..b36c19e48 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.26681, + "Result":7.79832, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.439989, + "Result":0.43577, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2844, - "Accuracy":"acc: 76.600", + "Result":10.1486, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index c8d4e825c5259419a962d0d0acc2a892d1bca72f..59b693cae4f1c1b448f8373cb9616c3231f79c8b 100644 GIT binary patch delta 3684 zcmV-q4x90qOq)!wSqKi%5*;K+2><~8K>z?40F!Ve8k2zt8-Iw&WRh&%PZq&esa0_S z^YY&>s4;rZBVlJ}m(;yqX0&ZQPsu$8G z(|%qK<1$eOau_To*;V}TRYs$6mY+7O2npj!y8Ks}Mt}LaG*o38JhoL+RUV>2HZBq? z<7hCD#fesxaFJ&5^taU^u}-iG74|w-=wcJ?X`-duhE;qXPZPP0X|8%*j__lU1)akr zeo&HS59vH8J{fUf82M=li`H*Nb1}B&y5Z`%LCNbV&fa~;JZ#<xg!NJ~C~%a?HvJ752M`RU3( z?VY_}u7=WlIW&rYvhm_h3*%{XI?4Z?=1ZOY)m$dSel#1(Bww(}ekRkbpDymEQCh5h z(*C`r>CG&Wa=FTC7K^Foxqq^bPh^^}VbbP$G1J&&?uYYvBb+Z9@oe^(EM{50nrf_7 z{V<*lqmlZHI*?fqO;Yv4VRH!P>u+_Gl&@N4KD6b3d&c#DeKjtzT4QJ};d#cM8O_1a zIUG82sU35PO|H}vE@99C355i(W*dV z9i>zt@y6MIBC+LYrmLG6L>GhTauCV&bIc_+xqeQ#gn0Hx()>r z@)2Bm^};p{!Z>YAf@OwB*Z$dS(mKkk0N|Ck1)!^Wp0#Hp6guW_*-ii?bIUQ%IAk0r zK*JD38prSsuNgD^x1D2HVUx|@R=5?PB6I`K%Ja~9PDQS>q61<(kcd?1%w z<9{0|K)%d_1&jkJ=lS)Rq_Tf6G&i3L|1y%bR9Jz@>-E4^P&%69xm%a{E|X9VAb-(7 zDnSB9Wl4grpj!!Nti=bNcm}1CM)gn>ogxJVOf%(yE*uR zC2jFRCmc%})dNxN5h~CKT~IEK|9=B6>!`8HW7YM*HaC%mY46z*6?jkdNbs#j%b{Fy zjbll82VO(+zU&NYaT!I9Q%aR1$&A}HgX^W34p-*pER1ZXPF2P1N=hA%B zT#|=5e;vs>%BzH=xr3x(S^Cyv?p-GF?){PAJK`4I(FPi)MVaIp&Lq$^(K zpDriMVwM!1X8Zt>byQbr_1fA(QfQ8F&3&`v9*2qGrUA>NL~@N|Q6jm96A3hoNT6Xv z0u3Y5DkBmkU__QA=xz?a5PwNqe9#Gd?8>5gD2fpYH0-9*dP6||6VG02DX8M>N^E<0@MU~V4P;)6~&m(0)uqWB_~YrrD5W>e3<1LhmG z_IwxH-ky9RYg*2pb*!OdFb!@Lv8lNSXk2RU0U8D-(oiswhJS*IG!#su%-n+#P%z69 zbPZK#=(|jc;!DJPpcH$cDP?mXnP$vw^GH;K;ygjVN2c38=3MKRQpeP1q1#gd0 zv$y9pCbMCIqks6Jo61^htFm}Cc04`0W_c|^aBOaUy?rW^kvx%;Nvec#e!5nXo7UBL z{;$Nr?dt3CFY>&G#khF>;WiteD~ z3Cj;PY*-qCJY-RluMf5Bt7F~Q5EnF$Xv%Up)b4?6Uq*ioITuJC&H15rD@1W%nmoLP zjBErP9+ zFBB;RCAw#=lUx*#f0k}p7Vg1G4?bw)5fxOHqRfO@7ruFRI%ZkMWF4$;moyZzt@ILN zC}mfH9u}2e+{;LXx63UfV{x5ZV-0$F)IP} z_C|eVI{!XKp63pbYd@pj1PN_(vqx&H(ykCLF0N0e_yD=s@GOE6dH~1)8hA$FdO-u6 zN+2)>F7^<*y7)Rhzg*6zi{<6f+5B``pSD!GN7Dmr*u-~#n|2J}C-!g4#5Wud+ntuN zLktU{HP8GXe=;%K7M8GO)o`}j6gz0gC8Y2GdA8wOYB+%Bpai4@k>s>Pu!2%_wAZ>2 z|8&6e`IE(^M@bx4^)sn&)Yph+MG76WzZn~L5AO8fN;NjP*Ms{#xZ=we$4USjyk{KZ zo^gnkaoFHJ{^A~gagV>a$6wszFIN18XbtW8)l4Hje@nwHvJ8vjVQjHH-;{DLBpE0( ztFM;27`mqMTAjvWg^6wURNb5HzMyP-I5HYCzB}As8L7G#9i_xxO7`bV5&;yvpe2Xp z9X5RRG3n{$|HroDR!v7N1AnZ0FCu(h$={~DoqT>x8$yUGCg0OPR}Lc zDtp!VbT(gJjxQGVe(lPw-VZDi_?Xmx2p!KRk5VGP0(|0le&7YJV+{sMI=B0^Y#{}8 zzdn{1_B&Sc2S4SJz)^zwV68s0Yt7_bjQ#^qO9KRx4<#D20vJOG1z-h<9VL@WAS!=9 zNNI)bKwGkEpwN_-?1VINM6CysS|@*B#j%|Pyy|jv@0__as9YXo8~uWHLaPd8Ns1y+ zOmm@{3Vm!=@dJvSr;5{7E2z*B99q_QB{PI+3vX5%uwH Ckh-P- delta 3624 zcmV+@4%hLUOqfitSqKR!l0y4O2><~9lVS)ue~igwl5E{i7Qt4jRdE6H^4~A0F?!Ae zVP|KT)V*J3w0X|yKFBZK(2pMe`Z${^_cBhxXx>(ZrmKpa2hlK`kK5|a_2-7GDoN_k zhyFC0%eML?6ZO~5e-0m_`1fQYWvXNe=1E(fr0L?I*-V0oocW0sE#y2qGK%7vpJs34 ze{nNe#L^$Gf@agEq3d>Y=7)22o#7y^k|P?8!a#nEg5^xk({++qPW?0+eliIciP||F zhS{;a9wnBewt6HEPJT0l+Uy*z`o0b2L$dQ)$xFG5qTlm3r^B|Y=h7zAeqIgZGFFCi zQTd$rCG+a5$9tiB^?h znWWL|xAh^hNw5wT_9j;7VjJygqNUr0m46;jQ@M$0u6tdL@S~slox?bKP~ue&$->V+ z8F643*=Y&#)^B-pKDOqj;p(|T$?C|@-hIbBY~JUIH;V5xcPKMEKo_9|`RN?R=6AeC}O z%Oj<}F1dbPa>FI`b1hA>SFHuBJ+fE=cfXT;3NU|rKtj&$IhPpZ?Dm0#oZSl`A!qjz zNXXg!1|%(fx!L$~vsnU*6m!faHaYhvTw;=XKLQD<_Y;tidOvfCOR{`--Uo}$iZLyvdhk<)ojWBNRX&N&w#E%^v8U&cM^eEsj`r>ov+|Lpy0HI!t_ zp>cnhPL_9C5Y3viY4-0dTk2%57BU_U!ud$X*@8_DQkkTKWO+9WlYH%y4DKyWZ>F)7 zt5sGrUraU6dncRtSSHyTCTVUKGmUNLL9kdfg2l2C&F7Eta-L?Zsm4Y%2%`BY9IL;m zLz(*FG*LesHb-E-{#Hjx*{W4$LtFhv4(o8bfOd&oc^UGzUZHaOlXTcFZL< zxl&KKgh2-+6gnWG&;beg6aK|1fsTi}6uOe|E(j-okV3uaq*N!aWEO&8=9@@jbJH^Rs|C4D5V04H_m?! zi7iJnUERbWx)?;4gGjEQV=l4D^>e}{3`8KIAOZ;m5lASAZV8AyoPk&p-UZ2qzcqX-R*0*CU*q(S{!HfUQGmOdrJ1btsUKkKod)7q*!n zL`h@nuQEKk_Rn6E)=^#s0I$3q0A0=Vtb&P9=$OA%I{}c)EyqCPka3&<4MPxVD1%5t zAp{zi&WBqK!Vo2(9F`^MZVzrrWGy|=iAVCBv#36ZqK_FbfJW%#1G(H8|Jx`4@?{>( zVH`@i$gamYk%N1ox!F|smyxWc!U{}YZw9u8($Ngtc0~b`Q4Jt})j%pi0!C#?g07%j z31_UO2RiW#N+pfzqbNE@A`P7+cZUBKq1>4vb1Ds86F-1u9W_>&DmMcYn}@ENEm}qa zmUye_Nbubo%cfXzjbl(OxrSp2Gz?3iVOVkv$I>Xn5+q<)mL%wI558bYTY8`qjwOxi zgDCb06=;MmD3`{6{{feE)L7-Q>Skcun@Gd73${cB-V;3%e5=uND3@I0n3PMd;ama@ z;}U2Xmt4aav1S>UAOYjDBtdt3@P$j-(nFQdr#CFB5284iU?ZDzX})PL$-|t#j$|F> zRYKC-L(;G;edjUvK9hL&{z&j0af|L~1C7(7OmYop5@;BIlR(3mF6(10v8T7>Hv*@L!CN6!>O#`)j=9YCD1S`%j`NR z0i&|aup_9S=8U!UKqq`tg!ytn6yFr#8n7u+vp0uR`SW*0-l%8iTI`y3Y|Sw3BFn09 z3$O&99BSJE8i)FJfQDmP!MTGp3`?M4SeBW0Py&W!nSDo4cg-1V>48r8<5%X=K~Ze7 z1RAz~fUVi6^BR>u=kecd_g3$riGv z=eY&uIzz`~8r&&jQ+E&0xYXSPG>lB7p=2U|4J8w4D49l?y9XtpWR@l98qD3pT6(Ax zx-r6{`XGvt2{fEcqh@!{Yh?c2osnAVs{*B=??Wl_ZxI(jDGH$EMiHBudw|9j6v_(b z9;9JVRxtOV@z@l~GII~=gh5$m?lG9Thqd%TCtiy}$;>?{ia}Yy++);i?s<*Lbd=+N zD1PX!vXhQ;)7$UWC?LR{3i;QQ@=BdhVNHkgIdnRKbG?SyWpLD@Gh`F{Gu2il(v$U{w&R^ zTlJE^K*Q=BF8r}P_v3LmPn4+~WrgZmb^{)-e{*2WMQIw%p5INRKa_F) z&Qz6A6s6B^@>8?CnEc9=51ejOT^PMj22Ktihc&hhztu09!l*<>lYgOVpKKh%hUH3WIcq9k7*YS&lC zy00NFXdcm&<#4Fo1J}Nc{u**FkUX06L+w_G;=nX{cncZj1}h+!l9-hl74qM5ZStZ7 z8u42l0FkApg|Ta9G|4p-vMLvcn&mhes`BLD3dUQ6;FB;EDFn5253iG56p(*d#~70y z-0HyxZ9Jla%2Je>FzdoM&rZiI%b2W#_3e^|LbjD&LJXzsD$v8C(u;c;sql7rXBnuNR#?B^aasPh{augshaPSQxu0CcZpx)l7k4)#^$H?>C0dnnU)SDoo zZEp5RZB^P8!o|h)$&?%*mlz(l2tp5lb$|w*5x8E^0H+cNjDbr$gsv{WPR}ow^XX!F zd2}{Eoz|zZO801bKn&aR-QT7i!}l%ww`I#W9FN$YmWg8-7@;-K{2zZZG20fFFvc~U ztv1CD+HnafJV2gnxL6Gb@EnwYlpvCvb_iBbijMYL7vi4|SU!KUxb!HAQ9O(-mgk#N&V?icWoGr&QWrzlG+wLIIIM_en>|(c zX1gya+a8XLhK%nH_g6-$?nOr_v6qtl`I1Bc1utmHVR=UkUwuq^dinpc?YLFbS=b;S z>)wk9Usrk0Fz(+7UQ71JGumPjciDq~wPPeOtT#qd-8nw8EZ;=T_J%lqd>cGR_x^qG zE{yLR(+})~4mJ_jM5H5ieUWU`I+oYn7DpYsYsVCLZUuf$-bFK$UkB-@#bTyY|WSe0e%Omx!zERpZmye0e#(Sk(KqE4O+- zz*gWBtNugics6;ITJkHvw;az8yufwvV4$S)alghK>VADJFYI@$q9lq@0Fo-kWmBT}dKW&Rz&ausX_bT$^segfC(9RO*L6|1 z8Ld8lf Date: Fri, 21 Feb 2025 22:00:44 +0000 Subject: [PATCH 146/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9755 -> 9754 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 2433bb6e2..e4751dfbe 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 21:59:33,044 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 21:59:33,044 submission_checker1.py:3270 INFO] --- -[2025-02-21 21:59:33,044 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 22:00:39,414 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 22:00:39,414 submission_checker1.py:3270 INFO] --- +[2025-02-21 22:00:39,414 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 21:59:33,044 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 22:00:39,414 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 21:59:33,044 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 22:00:39,414 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 21:59:33,044 submission_checker1.py:3286 INFO] --- -[2025-02-21 21:59:33,044 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 21:59:33,044 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 21:59:33,044 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 21:59:33,044 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 21:59:33,044 submission_checker1.py:3307 INFO] --- -[2025-02-21 21:59:33,044 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 21:59:34,061 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 22:00:39,414 submission_checker1.py:3286 INFO] --- +[2025-02-21 22:00:39,414 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 22:00:39,415 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 22:00:39,415 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 22:00:39,415 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 22:00:39,415 submission_checker1.py:3307 INFO] --- +[2025-02-21 22:00:39,415 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 22:00:40,381 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 9e7e0ab23..571c4378f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -773 +774 diff --git a/summary.xlsx b/summary.xlsx index 59b693cae4f1c1b448f8373cb9616c3231f79c8b..b41cd93ee6dc485488ac1f84a59c9d4860548b4a 100644 GIT binary patch delta 338 zcmV-Y0j>UB9}aJJOFby5zzrD#e{S}z8e8jVCX=Di2;{IiBH}LUWcmTbc^g;Fl^ISY0JXIgOnBhY`nJaBMLJx^LtLa2uN26~ zUDB_-9TwU5fy`0>wvT&MKB}kZB;_2nl+fFp(43Y_N*N)ndiWYfUi6=U`Y`0*#3Nvz k`|=aCf3uAt3j_tL9F}+_lbt100gRLKB_jqtBLDyZ0AU82RsaA1 delta 339 zcmV-Z0j&O-Oq)!wM<4}Y1&JLclT08Uf6HpaFc3xe{R*SKEk8(Uh3!CFvTC5vl$Pv- zG;u_&2a;MRe_zG1odmqjK0!XFLH zT1RnmADozsUDqXD(OX0Q41cX(-}_Z#p-!h_fa;Pn!Yoj)ZC&E)A{?%z&acCXcLMm+ zG3YaI!y@NC;86%b&T&WOvv_{lC9F`Lr}-jI^El5oc}5nGq$swhkvIJppd5z$n>Yf_ lxo Date: Fri, 21 Feb 2025 22:10:07 +0000 Subject: [PATCH 147/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9754 -> 9751 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e4751dfbe..961cd25d7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 22:00:39,414 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 22:00:39,414 submission_checker1.py:3270 INFO] --- -[2025-02-21 22:00:39,414 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 22:10:02,575 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 22:10:02,575 submission_checker1.py:3270 INFO] --- +[2025-02-21 22:10:02,575 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 22:00:39,414 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 22:10:02,575 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 22:00:39,414 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 22:10:02,575 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 22:00:39,414 submission_checker1.py:3286 INFO] --- -[2025-02-21 22:00:39,414 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 22:00:39,415 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 22:00:39,415 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 22:00:39,415 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 22:00:39,415 submission_checker1.py:3307 INFO] --- -[2025-02-21 22:00:39,415 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 22:00:40,381 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 22:10:02,575 submission_checker1.py:3286 INFO] --- +[2025-02-21 22:10:02,575 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 22:10:02,575 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 22:10:02,575 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 22:10:02,575 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 22:10:02,575 submission_checker1.py:3307 INFO] --- +[2025-02-21 22:10:02,575 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 22:10:03,641 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.6142 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71838 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71733 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79832 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,8 +33,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1486 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.255097 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2002 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.30832 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 571c4378f..1e2569914 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -774 +775 diff --git a/summary.csv b/summary.csv index f9a42a097..b000eb694 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.6142,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71838,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71733,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79832,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,8 +11,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1486,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.255097,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2002,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.30832,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index b36c19e48..60267133c 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71838, + "Result":1.71733, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1486, - "Accuracy":"acc: 74.800", + "Result":10.2002, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.255097, + "Result":0.30832, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index b41cd93ee6dc485488ac1f84a59c9d4860548b4a..e8864f4ba13f7e787da5fc9906aa98f4bf866ac5 100644 GIT binary patch delta 2437 zcmV;033~RLOqWcsSqKR-utq*b2><~7vtkJC1%HdAzJCRaH}8)G-w?OxhBnYREs7-9 za3q0-Aqg}LNv?4%ie#${Nze&>(pr|FyE*uRByI6QC!R@>q)|N-#gOD0q0`Ojt?}1_ zME(F6(10uv7P z=>UyGjXFTXsjOhtK^jIS&@d{?+&U-$qq5AeBdDF`jJ5cn6TTEPwOv zpacxdGWU+4=9)9s;)723(^qEFK~Ze71RA!0t=XsZ8kIli>FXPH@LY*)3s1g?6|UH0 zY8BodE`dRZdU$}wp(Y-n;apa*@gNQ35@;BgWlkQHfN@#o}l0|@dE{mNzj?bgZIlpEoRXl z3;F(C_|86f7up|wQH&2tTggj*mgdzhdnsO^VRa7Y!APD5$taqo%0v$HLUk>_|4vrF zIIw)3#q;GWOI7aTEQ_bl@5VA1$fS5@s>(2qv*$O(DcSsTL+4LWOS8)nmH`AGOtEs4 zWfL$KahfYC9ig_MYPeJdM$A*?CMP93iJgVchVG!`DWiuPHY^PRjaZ!0^`Uls zb*u*(;)2kaWQ@Y0b`M5XFIM^5_-G(t${Wg@^my5qcavmHZ;X(Xx6|M zyo`T;XD1Et2!CTMa03n&VaAG+2J!k#{6 zC7|BksE1yVNF7z$6~e{E^~n?;AP*b9Wn+XM0Q&$9d?WPy zumMgb5E=sy`v_fKe4U~vbju5DNdt$7~&kcrv0u!JqEhV#{?*g-okA%zFXcMQi;!vUOv5|9!^N=b)c1*Pa{ zuXQ2*>44?)CyPsu(C1#g~6Aj+Fp5 zc+WV*J>w87c6o<&<^85PCD*kY7!N;wzO z9F&>WS4&+CUDJ52PUEP;wqy2G-J9*cplo|MauPAVJKSFxsk#?krNmxJ_UB6y0~EX< zC5Po5HUjlA>FMSF$F}2EO=o{w2L4$0UPSo1%6o=!|3>gyvOk{D7L&NkAN;Ew!=Yin zF_P-e@sVZuX0|uP@#EXzIlA}ngLh$k-ohq$a{0sq2emqt>y!?zTAU*j+oO zz;i3`bMh{lnfyA)J}o|@8(_5l{pstr6S_BT#fnN_b=md^sCIBt2 zZjfVzN+foQLk-enW1wwg>^X~?!fLBL4l2|PDn*{4CUxC=r<%I_Xnug4&~QU#LwZ`J zZNqkhoh{unJkRrYR%zRC1557OC)4xg>GWJ8p0ZbsPiOPx<@jPz@7JE(>Vwd-!vNd$ zA412o$)nVkUjc#b`avA%hn{N<21+`&`?YKZb-zBA7Y;gB@&`ZVk29*Ut85m)uGF^i{mh=1(Wg&7Sd2bCgXbS<&+PhjfCai}yZyork zAY=(-0u!!5>lhuwIEXc99*X2JNnBa4c}%76uz%y?hj)M}ArDwXqidzR)tw(WkZ-SbP!!Q&>HCj{t5`QgT z-^W#RWp<|$fZAFLCOq(HeOu%6A|0;hAuiI1R|@3QHtAR14vXyjKxQcb+s8dBpVjlr zawCpfO6Yw~Xim!|rK}{3RNuRi7yT!oJ`A~;cm(WoUw(r2AG3@h$^r>8utq*b2><~7 zlV2oP0mhTJBqAIiOtErd1ONbZ5&!@c00000000000001hflw2ZZxa@i3neN6bPyzq|5dZ)H0000000000fPprXvL!wNiIedqBL+Jo00000 DfR~PB delta 2493 zcmV;u2}1UlOqxuvSqKi%5*;K+2><~8K>z?40F!Sd8nb~2&INyiq+wb5)?@BnCh_k5 zk>ESx7TwVX8mC2>k}~VMGEABho4(5+q?nW*PO8yA9TVWzcQB&iei%`(69w;%|@NqsQfvPU*D*O z=SpmQc=AQ8aK#>1tMK-42^>1q!~--Ab@2cV=dyy22Wc3WK*P8!v+|$>jLR}Jk6>;d z*5ZRsIG4=O1ETmMmTSNwwq{e$zXRqQwf1}$+unbkd?9OE&YpFwp<^%&ZWOVpxd&)m zYVH9V1}4%_Fp-9Wi8K^Uqs-ic5>PP95_An_?qMxH>V)o#u&5r0VqgLd2h*t8-18ck ze|KY~minqNY3RF5isDPed!Q71pyWOgn|gbI#--jKpy5zf@b(}LgA$C#rcjo7dr&6~ z$})d%kHNe>ti=bNcr6Mg^Y)-924w|rk5RL?=QSp?VS%Igp_|HDYOAt%HFi8bx@LJT zL2ztteZ74ulaV};lS!(Caelg1k(<`lcmA)$!R_kn@h|eczj!u?3PXP|S$&y5i)JGw z{bThr0e0~-0eX=v99}IZQpuj?vaQO;dH#P16~(H`;BmP6fdFNmL~#;jPpjUGUnp2i zg3dG^yk8D#F^m3K$oKEUclN=%(Ejj?Vti2AN?!W2G_P*iOYs5?t8+LHM)EvJM$s%) zCUTe;s%!cEce47)f#vHgo-bcns&W@+Sv-AyHygUI)4QM zP)h>@laC}Cv*Zza0R*J?Vv&<#6EJ_Vn=2|Ep|+rExKss3%v0qiCnY;coQ)l(%&%XE z2Bs4y1s0HyD!y}kKA)>khF>;WiteD~3Cj;PY*-qCJY-RluMf5Bt7F~Q5EnF$Xv%Up z)b4?6Uq*ioITuJC&H15rD@1W%nmoLPjBQj=Jg zYi2abH59Tc7l#^g91T@@@^1y>ErP9qHX8C|G!&yV7C|~R#b{{Oz!tm=A*W@++W2*z z2;K%4z^hcW(KMwQCAw#=^dM*pvt%Pmn(F?>TeW2uahxVOwBalE19EOE?6XP7C8euq zHbI}jSQ|*j3LRBZB0je@$1Hzk zUfjz_g}2K)Vz|<11TmGxxeQqzCu?(WQNB0Xz9QetRjp)~R(eWh*2Hq-yZyAbN?Wd} z(u{Olw<$dFv^9DKPes>Pu!4V5bhOvH5dU<*^7)g+ zrAJ8|SM@WgZ`9X_WaECYY6doLnXp2eQWe@(cC z&MHj|$G1%6u6;5+U!G3SCE_Z3)%bKaUtW$c7WIDZ%B|iHEE4#b)PD#a&nAyjBEJHB z;&^`G1+HTa21+`&`?YKZb-zBA7xp_=@&`ZVk-$;2))#IF1gjjDc#}vVAb(E7Fc5_A zJVoStV#g5(tR$5H^+Y8kia^{fd)r#XKe7#J-ySDz(^l$<<9cSlon5(}ZFQIZf)C1? z29*Ut85m)uGV2DtFX#C^%0lEu^3ED)&<+BcwKuhJOjr-E-a7D6LC6xw1SVXAHZeMe zaS$8OJQT@clDM*7^O#D%#($2BZ@vah3Aw`>B9}aJJOFby5zzrD#e{S}z8e8jVCX=D zi2;{IiBH}LUWcmTbc^g;Fl^ISY0JXIg zOnBhY`nJaBMLJx^LtLa2uN26~UDB_-9TwU5fy`0>wvT&MKB}kZ<~8lUyWM0m+lJBqAH6 z_hOM@1ONbZ5&!@c00000000000001hfm4(5BtZ@47XScwY%gbPyzq|5dZ)H0000000000fPp!auO&VKjFavqBL+Sr H00000)q{Zt diff --git a/summary_results.json b/summary_results.json index 41d62b442..c57ec99b0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1486, + "Performance_Result": 10.2002, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.255097, + "Performance_Result": 0.30832, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71838, + "Performance_Result": 1.71733, "Performance_Units": "Samples/s" }, { From 711d2a4fdfcd47d68e968764fd65492257b3a6dc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 21 Feb 2025 22:36:29 +0000 Subject: [PATCH 148/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9751 -> 9756 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 961cd25d7..4e0c1c381 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 22:10:02,575 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 22:10:02,575 submission_checker1.py:3270 INFO] --- -[2025-02-21 22:10:02,575 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 22:36:24,283 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 22:36:24,283 submission_checker1.py:3270 INFO] --- +[2025-02-21 22:36:24,283 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 22:10:02,575 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 22:36:24,283 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 22:10:02,575 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 22:36:24,283 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 22:10:02,575 submission_checker1.py:3286 INFO] --- -[2025-02-21 22:10:02,575 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 22:10:02,575 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 22:10:02,575 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 22:10:02,575 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 22:10:02,575 submission_checker1.py:3307 INFO] --- -[2025-02-21 22:10:02,575 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 22:10:03,641 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 22:36:24,283 submission_checker1.py:3286 INFO] --- +[2025-02-21 22:36:24,283 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 22:36:24,283 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 22:36:24,283 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 22:36:24,283 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 22:36:24,283 submission_checker1.py:3307 INFO] --- +[2025-02-21 22:36:24,283 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 22:36:25,236 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,8 +27,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79832 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1816 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43577 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43577 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 1e2569914..fcaee3d79 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -775 +776 diff --git a/summary.csv b/summary.csv index b000eb694..cef4c290f 100644 --- a/summary.csv +++ b/summary.csv @@ -5,8 +5,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79832,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1816,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43577,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43577,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 60267133c..07b31ae88 100644 --- a/summary.json +++ b/summary.json @@ -184,12 +184,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1816, + "Result":23.1988, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -214,7 +214,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index e8864f4ba13f7e787da5fc9906aa98f4bf866ac5..7d3e42781a490faa0a22ec02a2cfa037f32de8fe 100644 GIT binary patch delta 947 zcmV;k15EsvOq@)xSqKF@DomkCvt9^l2Y?0@+# zwyiz+Le{jLJ?mIQ$6y-VCt_1?574;O+XFNVOr)V;A`Jx-Y3MrEDD(E91Qg7&1YLu9 zdsvH)I-%PlEUE{h7??oA!8B_2_Phq>-`y9frM@am8u~7iqWBK+9w@~gD7j6pakQgv!zjH?SDa?FeuBcJqEM(uofS5;Y3X-`NN6Li@unit#~d zD|zY9(!9E5FU1Qqtj^&)7|HV>8AY>HnaE*YsIKMr-^uD12bQn1c)omPsmfiPW%2a+ z-B<<#nH29#RT;){_WY(eB?_BgZs`08P)h>@laC}Cv*ZzA4h39>u;3+=NgyhJTWLcP zwgYX+sewXMT5=Q8CK0uMkkmSU`^rw@B;ZqztC{_Fc2T)Fs3!RZ=cUmVDzY3Upt%uJ z?<(}MUZ#&I@quZ<8l#~?9eA{;?@DedHxAyMvEYIPpG1)Ml-mmJL$DNM&-b91&!R&| zaci7nA(n23E#u#82bkpf94iQaOt8Rk4@~VyL?@)+Bhvo3W&j9|p#cRtcwA%!zIX#T zC5Wdk>Jx9LMUH(Sk{Ey<6)o&m?)x1XT?1G9-C$^r#EDomkClUO8M0m_rKBs>9Blkp@$4d)jC0C;RK zb98xZWpk5&B@~lWB^m-*8k2Dv6_bP|5F1>Eu;3*D002+|000pH0000000000004l2 VI+L#@J^_xC?j<7zKqCME003-H#8?0T delta 918 zcmV;H18Mx6OqWcsSqKF(utq*bvt9^l2Y*1ruq^ZLpacxdGWU+4=9)9s;)723(^qEF zK~Ze71RA!0t=XsZ8kIli>FXPH@LY*)3s1g?6|UH0Y8BodE`dRZdU$}wp(Y-n;apa* z@gNQ35@;BgWlkQHfN@#o}l0|@dE{mNzj?bgZIlpEoRXl3;F(C_|86f7up|wQH&2tTggj* zmgdzhdnsO^VRa7Y!APD5$taqo%0v$HLUk>_|4vrFIIw)3#q;GWOI7aTEQ_bl@5VA1 z$fS5@s>(2qv*$O(DcSsTLjvbdv&s>E4h7Mx$f+cgO&}_NS*E23+kv*^)IgyrExie8 z6NNSzDeHABh638jX)y#f7yQrRRbf5i!56YSbl?6c=7-6L{U4uTB^ZXHIA#x*m zZw)kP3jxjAyIMFVtcN#m9r&mqWC>&f6Rtt)7#+hnh&5;)isUd!Tv@MqOr`IzFUIT2uZKe=T0$$5nG>cBc}6+FA-GJn(3JTjTR09j@mgF4Bot3gpu^ z=~vzki|qSAW+?#M$2}^a)$_}8j#^6SeNJdj%O#~4tR#$7-@B0){U@M447r(j1nhHP zeuDNNvyLIk0tGU#Mm|N8T_jrp#*?`uJOWS?lW-FjlMN**0ppW6B{~6BlXE2|8_}!C ssU!ga08jz|01*HH00000000000Dyrule8s10g03IB_jqqBLDyZ08#0sZ2$lO diff --git a/summary_results.json b/summary_results.json index c57ec99b0..97f63cc8c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1816, + "Performance_Result": 23.1988, "Performance_Units": "Samples/s" }, { @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 91a71b5873e8a88a41863b614da3b7c090163edf Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 21 Feb 2025 22:39:35 +0000 Subject: [PATCH 149/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9756 -> 9755 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 4e0c1c381..576fadb4d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 22:36:24,283 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 22:36:24,283 submission_checker1.py:3270 INFO] --- -[2025-02-21 22:36:24,283 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 22:39:30,916 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 22:39:30,916 submission_checker1.py:3270 INFO] --- +[2025-02-21 22:39:30,916 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 22:36:24,283 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 22:39:30,916 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 22:36:24,283 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 22:39:30,916 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 22:36:24,283 submission_checker1.py:3286 INFO] --- -[2025-02-21 22:36:24,283 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 22:36:24,283 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 22:36:24,283 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 22:36:24,283 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 22:36:24,283 submission_checker1.py:3307 INFO] --- -[2025-02-21 22:36:24,283 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 22:36:25,236 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 22:39:30,916 submission_checker1.py:3286 INFO] --- +[2025-02-21 22:39:30,916 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 22:39:30,916 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 22:39:30,916 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 22:39:30,916 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 22:39:30,916 submission_checker1.py:3307 INFO] --- +[2025-02-21 22:39:30,916 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 22:39:31,884 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2002 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2049 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.30832 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fcaee3d79..6e68a0f7e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -776 +777 diff --git a/summary.csv b/summary.csv index cef4c290f..0bd5fa6c3 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2002,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2049,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.30832,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 07b31ae88..e13241281 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2002, - "Accuracy":"acc: 74.400", + "Result":10.2049, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 7d3e42781a490faa0a22ec02a2cfa037f32de8fe..7688de0c0826a21c03fe0454d8fd330b7b9fe276 100644 GIT binary patch delta 1488 zcmV;>1uy!XOq)!wViN`YU=k`|lV=kz7O|TvDjlJ=plY~O1xCzMi4 zS&HtU#Jkk*AN#pk7&wrIMnWeYhOlx4LKJ`9?kiob}K}2 zV46I zErP9qHX8C|G!&yV7C|~R#b{{Oz!toJ3?Zjw!rJ(Co(SFs7r?7jw9z!B86~=Bt@I#h z3$tV+N}B5a##^;z7jc{>Ike#`_5*TmD(tgK$0en!Xf{Egz*rkd#tI!(Q6fIKHODMv z<`|Z4Sr+cWNe@0~;}I29 zmZHprSr@)}b~Ufjz_g}2K)Vz|<11TmGxxeQqz zCu?(WQNB0Xz9QetRjp)~R(eW*W!A)USf)$jaqrKLJ_@@Jw&z~$VJxb!Zs-H=HqrOHoD^lo~{mt00 zdvK=*SE{kWy&l}}!4+SBwm4P-*x)_m5ciBjtc=43@9`J+_=|h|#XbJw9)GdoFGOo- z&#z`0=~)_Xk!4sE4`Ykv`KFX}A<00QS$(zC#n3g4*XlG5D@<&&r|RBp_XTC!!;#UD z@!jG6%1G6{=qM%jQnEi^k_e#S1uZ!&@37&kk4aB2|39`Jw`w|nVj1{j-Fp$?>niUV z#{C! zeUV?xk@DI1Z8n*I9ibCAmYD#wxVk~M6)2I|DGoJAkBx!0jj`t}Y6`2Z@;InaGpH1K zjGEMS@11JuvZMI{vID~jlnv==l_myp{GBb`HC)&Ac2;R(IKCx!?UU*G@^pGG5m(u( z#;3FS@^XB!sP}7EZuNd(k-*2K{zK?^HhGj1`4!+3$MXXyFK``eFi_ID-LGXUsQdM? zys+P~l0WzUmKH5ARvEE!!Qtp?>t51d*Z|q z2&^QP0QE#AB#J=XEPI<;#6PkPY2O|vZPQlDiQ{@^znxvVTI^Jp{DSk+=o%GSjuOz^ z2&vl|y|0()JxY9FTCmP&sL>ugS~NEmx0D+Pug+L-L4r>rNPEg{jkY0Jim~TgP|Ro1 zp`*AlPO%V6*J6Lm_%~|-lRUq}3IY=>FgyTLI}ylkjaX?KaIOPslFs`Za zWf2e8F&|d(#2X3XaUb=Wx8ov*J`hO^Ko4tP>i4 zd4}$w_{k`!f1#$hkoBDCLLRtq{e5 zY4Ye6a>5N(0F{!2l{pde-*Rp8q68B2TL6(~rfpfCnUj=iC}I^AhnnrW8megeZw2Em zf~|oz8uD~B6r(d1LpC(UXlT~J7QBpqfRZw0ZTy<1g15m1@G28+G|Na%Y~8n3dKk8a zdAboL$@F03uiCPUB*{_=ZTO1)0Lo2;eKzU1q;wU{Cg>9wYXiwxp`$8J#pkx>m}RW| z3ipcp)hbVgfU+$(x+7^uR%wE-cP3v_RwaoP2PYt;_lljdw9&>~!_qCw!aca%gAdwxOazgo zC^KQ!g>Rmnj#-{FSqJOeC5eP=E4_pSO4(JQM@6L<_cBt^?efkxJZUtBgvjDtMvNxu z+T2@|?@hL^$oFzpE7_%$o)MXUHL=_TUO%m)(w1wYG$Yy8Z3a&~ZH->R)6urzDJUHR z#RH|r&L(Ja{|hJzksrF&4Z@y2W+kBB-l&gE=ikT3_q_q~oM+UVAfat;_DCI7+7-gZ z#r4S)A0Q7KzGY*C9sv6Q4SXZ?{ICH|B@h||5BmsRU3{IMUoPj<#q#oh=xlyEtxsDj z-J|INHXJ+few%iUAh4a^mhHfBee85v#;$Ex2(5V@{E&&+wy=aPtA_K{rr1F{E+K^n z$af6KQo{k9f)bDtL`q4AUl4{^@|_^Cyc-kJBWn>St2lsIL*piwrtue=|0m z9^CE0m1=BozXuO`aK)E@Esm7{Hh9lC#69B>E90=id;G;c{^A~gagV>a$6u`Y3-KD- z^Q)OgdY(mFWEmC3!`Nb!Zb~^9(j1hT)mKYh3|-TBtxn^p!nR}fRNb5HzMyP-IC2s( zzB}As8L7G#U8TfcO7`bV5(5;xASH+89X0~>G3n{$|HroDR!wJrTL%7E_g+N!y2^Wo zasNi}TCzW$(H4`q%OCu!9mAnvzcG^P&he3D`DV5^#PQ?X;5oYY?}K+?eBYRUU?+62 ziKHgN9jWV!WTV!xyzaI*>eyX7roeM6@N@Dmnwk7M$UZGTqZ?qf{{89ewiCKHZN-X8 zU*s2atbF!;n@whaN9Y8OWhMYEu5OTHg-RrLibD<3V`HFgW9&JLn!;+UJPs<<3@SyQ zpeA+Qd#9SZ{AhlFoX~JXWkY&erESA@gPkqiGd$1pcUEcJa05&3+9%WV<>~ZXBA&8W zjZbIu<>mNdQSaBD-0FkSvcmw|^&djVv&o~>mR|vZ?fO9}=!c$b4F*a&xBInh1$DnZ zmKP2>R`Lfw<&n@;g8E>sKC^4h<6K|(Qj(s4K7=Rw*o|Vtp^UHc9iE5IQhcqWCDb|G0*_@L5 zO+WIc{{mE}A^#?x0mt08pP>E&vyCB50|g&Uv2tOP-y~N7=aV-jIssXeawR4kT!ygV rB>?~cPyzq|5dZ)H0000000000fPp%bvn4(Oj+62wBL+Yt00000Ba_(r diff --git a/summary_results.json b/summary_results.json index 97f63cc8c..192309634 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2002, + "Performance_Result": 10.2049, "Performance_Units": "Samples/s" }, { From 84cd563121f457b1064dc42d1af0339a5b4ef972 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 21 Feb 2025 22:56:38 +0000 Subject: [PATCH 150/719] Updated results summary [skip ci] --- README.md | 38 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 12 ++++++------ summary.json | 16 ++++++++-------- summary.xlsx | Bin 9755 -> 9753 bytes summary_results.json | 16 ++++++++-------- 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 576fadb4d..4dd0a4863 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,40 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 22:39:30,916 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 22:39:30,916 submission_checker1.py:3270 INFO] --- -[2025-02-21 22:39:30,916 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 22:56:33,860 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 22:56:33,860 submission_checker1.py:3270 INFO] --- +[2025-02-21 22:56:33,860 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 22:39:30,916 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 22:56:33,860 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 22:39:30,916 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 22:56:33,860 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 22:39:30,916 submission_checker1.py:3286 INFO] --- -[2025-02-21 22:39:30,916 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 22:39:30,916 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 22:39:30,916 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 22:39:30,916 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 22:39:30,916 submission_checker1.py:3307 INFO] --- -[2025-02-21 22:39:30,916 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 22:39:31,884 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 22:56:33,860 submission_checker1.py:3286 INFO] --- +[2025-02-21 22:56:33,861 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 22:56:33,861 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 22:56:33,861 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 22:56:33,861 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 22:56:33,861 submission_checker1.py:3307 INFO] --- +[2025-02-21 22:56:33,861 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 22:56:34,943 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.6142 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.32368 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71733 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.6142 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.27002 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74502 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79832 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.43577 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440164 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2049 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.30832 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.347301 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 6e68a0f7e..451dd3f1e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -777 +778 diff --git a/summary.csv b/summary.csv index 0bd5fa6c3..a34a61ae6 100644 --- a/summary.csv +++ b/summary.csv @@ -1,18 +1,18 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.6142,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.32368,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71733,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.6142,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.27002,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74502,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79832,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.43577,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440164,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2049,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.30832,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.347301,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e13241281..f03642bb9 100644 --- a/summary.json +++ b/summary.json @@ -4,7 +4,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -34,12 +34,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.32368, + "Result":0.27002, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71733, + "Result":1.74502, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -184,7 +184,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -214,12 +214,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.43577, + "Result":0.440164, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.30832, + "Result":0.347301, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 7688de0c0826a21c03fe0454d8fd330b7b9fe276..718f9b54c82c319a0b3cab8ebfb2515ebb8159c8 100644 GIT binary patch delta 2796 zcmV<~9lVS)mfAX4&ZY7zw7-KS-BwP2BMX*(BRb0Tl z{PzoL44&sm*xA`7b?=uMZJu+w5AsVl^rMHrKF+4fy^NDEnzvP<>8c{e;}GBtACYQn4L>QRc8KUTWzVzLpV$)d17S} z4u>*7(W(+GlQf$Bwmu{_3D%*)-oy%BY@>y^f&}hv z7ssd|!#qKSAS`@jB;@+}#7M}AfB(!#2>4$Z34!h_BO%b8G7@r`br}iyaLyPBIk`PX zLeB0vN(^##`;3H~-3vxS&h8~6A!qj+BQg2Q&E_vR8xmNgm}8XK@g!z;BmFa<8Bc~+{?oR4IjE!gBB zl}S2CmUpu-$=5!~;NH^oW*SSmT4go!#Z>dWce06(WsV1S$Ve!N zjD&*dmVk)i0AfjamkFmJx+NeogQp<66hw?MNz4UGJR-St|93#Fqm&9D-nu(eV#{(Z zvFEnO<4N?8L~`{Xf1|{vH$Er<5*Z04k&#dmQR31YAFqT&CY)TfrzPQCk8pBE8+zCQ ztV3x`pNXOCP)0&Nf=jPn*k*nZC5@@S%JAsgKYLADM|l+hyz&YFx|-)%dnQ7m^Zc#a z34mm7Ic78t8OI5u0R%Y>WsuWQ2pNq_=fkar!;nirIV?-if88G3k_at5rW23kIftk| z6Gb00UN9P=lMm#wF#fjz$!LH|)F73jj7lZ}P+5|oE9h3j8MO46 zPCSEBNu&B)e-xb~ISrj83&VelP!?v$oJvF2#1CLuM~zjc%FV#U=Ao-*3v(Zq{8rPE zz}*|mrdXoJF({U(K`a>!V996zOVl8iMj4h&0>H8)L3exb1xwn}V>&@BX;hzyf=8%~ zM(BcaY5X5>Sx1dk9;w!$hEI!15@OsBtVxBx;aIMgv4L8X%I<0FhQ1kxT+0vLr!wd+>!w z+R|e>fyb^8)#svsNJaxUmDU>q@}GG2T1$bIf0(q2ca$wn*R4HfKC#D!BN&qn_2DoY zhZ=Dh4Ki85iobeC&(mta+oMwxuS+yxz_2+`B$KP zt1cZuZ90xwxVm(#r0F{ges4TPVa0!6QGQ*Cb ze}0-XXz4MX;HC)pa+oOG6hRHMDN?gH2dVt|yCQGYvvVzqrX5=|TJ}E6s&MyU$#`<8 zZHLh~)VITE5X%bA9ZmySG8(|L%)G-T04&SwJA%4v&Y-2obb^mx!KK4R!DPv3U;$gR zQRg)(f6n9AH)`Rz5=9SBwulv;yKfQef9>Itap+JJ52JCYi-*x5mlb?GoCdgLG{9w< zm4{0JT$Y)61h{#irN?xFT!Nv8iNZxJYM4cA&8D7z2h2BW?fEW>-kxkBYwE(@Gh-S$ zPo{xR5u3Vu7>!HaJ&Xp(W6$@Xn9rg^M{#SMVj-4phkq^O z-)sk%I>K6p7A+qTWx*9^;;skJ;1nW+a7bl9PFwlav%2Lg;)!XZK(I$eaEPP@RVSn|KBs zbKib~`VX^>A<6;?H+E!4M+pD`|C3-OR{_YAxFkFRQWKMK6Bd&UB`N{rlQ$(g0alZ8 yB_<~7K>z?40F!Yf8k2zt6Mx8CD!P?q-eQQ!WRh&% zPZq&esa0_S^YY&>s4;rZBVlJ}m(;yqX0&ZQPsu$8G(|%qK<1$eOau_To*;V}TRYs$6mY+7O2!9FVNxJ-3nMV1!G*o38JhoL+ zRUV>2HZBq?<7hCD#fesxaFJ&5^taU^u}-iG74|w-=wcJ?X`-duhE;qXPZPP0X|8%* zj__lU1)akreo&HS59vH8J{fUf82M=li`H*Nb1}B&y5Z`%LCNbV&fa~;JZ#<3_K`-b}DV*N?R=6AeC}O%Oj<}F1dbPa>FI`b1hA> zQ>_K7J+fE=cejgUt{}ra;R-=m_y{EA`uPMToJ$OHc6&fV&h7<}kh6OUB;@RV1CkcL+-!Wg*(`xYiaF*Io1FU- zE-^{HAAy9_`w2)$y`QBVjhvQ*cb|_vPf=&I!RH-#!Rz zbAQeSNJ~C~%a?HvJ752M`RU3(?VY_}u7=WlIW&s0@#0Pk<7sm`$^V_^OP&1HTqeVQ zG#kn!U$Du3Cey5+F7BpLTC9E2{=KE?%`A~}xyou5i>cu+_Gl&@N4KD6b3 zd&c#BH7>DQV`wemdB&a@&B4$)96EBT9dn6IuGAAQVbB2yg$_t4bU;G>gnw~LpyS~# zg{~yL3&P1?i(W*dV9i>zt@y6LAvE^u{tD6`^7lY_>5Xtp(%q2FteonZA zfe0iNL?EFc0tp4tEdh~-GZ0I{yC9r`=$3#822VkBDTq9#MPe?v#3PbR_kRb(I!dVk z;;p+gC7PCNi9NSH9wyP_B$BKDn14%bdgH?-j6@)zBmxN~kxN{946x!4h0hO5nOup!Zr=UIBiUVWrj!B{@H8NI?Ag6;FY%ppsRVF zwPzv}I_7WLP5>lx%Q4V6WE>|z!w^Io${^BE2!Y0>^WjE=FhmI`hh+)6n}35F5?PB6 zI`K%Ja~9PDQS>q61<(kcd?1%w<9{1KzRZIKi~}j>`SqBjvVSi$H=hdsGLp4aSb@pw z^}tq8I-29TTbKDRmH0vXk>C&3EsIjgHIATEf(P`BN}yp>0u7^*YuMbl(LgFe0!C#? zg07%j31_Uu2c38ZrIJSVP=6GiBawzql3T-ni%@ROkOh^7u8AMOvW^<7OqJ__iS|uaV?5us|-od34PL9 zmY};i_<|&D@j)k^Nq>=~Q9TsJkmMSn)6MCv@z;Xn;x3;E7i%U7kJkpIM^*IEj!#DAoRoFn<8)2vU4rAJv+8$nD#!$ zs&MyU2`o9(wF5K`we0{6$FhQP2Wc3Vz&vA9EX%w*s1t@|nR`c2bIloR@j)m2=_@no zpeQz30u5Wh*6h=Hjmn?%^!1H8c&@~@g(qLc3eVlQh=29=a0v`L)WZWb4mI%r4d=3g zjR$EMm%xd`xGZz>piUTL7gxt%d9;Hv-YqSA9UiiD3r|FgQ6Ie6|6l*&DNgR zn9PO+j^c-IDr>2&%Hq}7vGnMg<+TLCvAOl__J65NM)E{XCaDs}`RQ6kZdzAg`M(ke zx2tc*zsU3c;@Kc7{QSXW^5$@3r?MYB|y$YEZnuI2aN$?6vemansTzITe@??N5QOhM zMdW+p#1ROrB$WX5L?tAOK-?^Qn_9#_vJGk99w%+nR?3OvdS<_!UAbE9RG0jM^U~-V z6Sa3mtPa;Tr%59CdAy|sB z=UY(BXVIafxG_$#5KGr$f6MqcYXOryzrzXw6D%-108={=(Ge;5gtR}d8v%l2=sOTY3amc@kC%_^1^(Scl0JDoB$^r>IDomkC2><~7lU^iO0m_rMBs>CC z6O(Qe7Ly7kDgow`H6=O$Sd(xiCL3shUmGO>002+|000pH0000000000004l2Ig_y^ RJ^_uB@FgP#KO+DD003H@JsAK1 diff --git a/summary_results.json b/summary_results.json index 192309634..5dfd45c02 100644 --- a/summary_results.json +++ b/summary_results.json @@ -293,7 +293,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -329,7 +329,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.32368, + "Performance_Result": 0.27002, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.30832, + "Performance_Result": 0.347301, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71733, + "Performance_Result": 1.74502, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.43577, + "Performance_Result": 0.440164, "Performance_Units": "Samples/s" }, { From 406f01e4d3c0c29a1fa340d087a49e44c233fb59 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 21 Feb 2025 22:58:01 +0000 Subject: [PATCH 151/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9753 -> 9753 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 4dd0a4863..35b006657 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 22:56:33,860 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 22:56:33,860 submission_checker1.py:3270 INFO] --- -[2025-02-21 22:56:33,860 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-21 22:57:56,612 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-21 22:57:56,612 submission_checker1.py:3270 INFO] --- +[2025-02-21 22:57:56,612 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 22:56:33,860 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-21 22:57:56,612 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 22:56:33,860 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-21 22:57:56,612 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 22:56:33,860 submission_checker1.py:3286 INFO] --- -[2025-02-21 22:56:33,861 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 22:56:33,861 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 22:56:33,861 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 22:56:33,861 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 22:56:33,861 submission_checker1.py:3307 INFO] --- -[2025-02-21 22:56:33,861 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 22:56:34,943 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-21 22:57:56,612 submission_checker1.py:3286 INFO] --- +[2025-02-21 22:57:56,612 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-21 22:57:56,612 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-21 22:57:56,612 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-21 22:57:56,612 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-21 22:57:56,612 submission_checker1.py:3307 INFO] --- +[2025-02-21 22:57:56,612 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-21 22:57:57,631 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 451dd3f1e..7e1d92a39 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -778 +779 diff --git a/summary.xlsx b/summary.xlsx index 718f9b54c82c319a0b3cab8ebfb2515ebb8159c8..fa8e54c7420ef764c5e83119a7de7ce683084e37 100644 GIT binary patch delta 311 zcmV-70m%NDOqoouMIZzwecmLKNgynLA#E|X18vEvfkItcaud?V5w(7h)H;6qDt6)| z;8Q^tX=cBjT~w?NvW1GQC<;p3A?+wga&*J1hI58Q!u1mVb zU=8^*{Iz*~A4ZLZIW`TNbs{&tN!s1%${3fh;CxAcoL7#a$4KmgNk3xWa zjC)o-7tb&C8dW4E%Qz)5$!bCv^5sJ~+hfa{`U_B=n*5u128_9HKSA{evx^}M1Oz61 J-XxQnC4Yu+kR1R3 delta 311 zcmV-70m%NDOqoouMIZ#A`uHuANgynLX+jaU18vEvfkIPSdK1zn5w(7h)H;6q%1+`W z;8TyQnf-QlQMo**Ciw;DrO_2CvK%F#xe-$DD)hctr4K0afoZ`SqoG0_c(knVN^U7P z4qlzH;DQ97M3DBB+Y0SNuoPp@_n?^1qC-b Date: Sat, 22 Feb 2025 11:17:43 +0000 Subject: [PATCH 152/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9753 -> 9755 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 35b006657..7b0338a20 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,28 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-21 22:57:56,612 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-21 22:57:56,612 submission_checker1.py:3270 INFO] --- -[2025-02-21 22:57:56,612 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 11:17:37,762 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 11:17:37,762 submission_checker1.py:3270 INFO] --- +[2025-02-22 11:17:37,762 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-21 22:57:56,612 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 11:17:37,762 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-21 22:57:56,612 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 11:17:37,762 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-21 22:57:56,612 submission_checker1.py:3286 INFO] --- -[2025-02-21 22:57:56,612 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-21 22:57:56,612 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-21 22:57:56,612 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-21 22:57:56,612 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-21 22:57:56,612 submission_checker1.py:3307 INFO] --- -[2025-02-21 22:57:56,612 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-21 22:57:57,631 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 11:17:37,762 submission_checker1.py:3286 INFO] --- +[2025-02-22 11:17:37,762 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 11:17:37,762 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 11:17:37,762 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 11:17:37,762 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 11:17:37,762 submission_checker1.py:3307 INFO] --- +[2025-02-22 11:17:37,762 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 11:17:39,007 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.6142 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.27002 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.407085 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74502 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79832 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2049 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2598 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.347301 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7e1d92a39..d2d648e25 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -779 +780 diff --git a/summary.csv b/summary.csv index a34a61ae6..9a4801d72 100644 --- a/summary.csv +++ b/summary.csv @@ -1,6 +1,6 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.6142,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.27002,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.407085,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74502,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79832,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2049,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2598,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.347301,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f03642bb9..293b1c2ee 100644 --- a/summary.json +++ b/summary.json @@ -39,7 +39,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.27002, + "Result":0.407085, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2049, - "Accuracy":"acc: 75.800", + "Result":10.2598, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index fa8e54c7420ef764c5e83119a7de7ce683084e37..44827b42485d9fa5101be132a783987f884fff4d 100644 GIT binary patch delta 3273 zcmV;)3^wzbOq)!wSqKi>28hB)2><{9LI4050F!Yf8k2zt8x&$PnIv2HlSQyqYE@jo zy!`hIYK)Vx2pE4(C(6A{(kPy_RiWvsB4=Sdh-RaxU3KWH}7a4e@ms>O3T%a07>WEy1o+ho*C=ZOpk%b@9` zY3RD$oCeWMU1c~(s^o}=!zh#=<8U#Rvuu?lk&__Hho66rqj{=!4hKOA#1Bfc>>-^8#U~>U3?n}+VbS`n zXfDRqTsK@jHz;`>#o4>>v<{p1MdG#MI}N#}4L7vmcB`S8k7qL|gCCRuW6*cUaD@uf zB*SDR)~@85k=VPE8%E;nN^Tj6yOHda+v3Y@K>~k4%0&ek=l53?*RLvWURB(dDRwF~ z`ASWQH%O(ZFg;T0>yqo&B{wLcpKEE7ooX$%+9L}RxVv2(qk;_c1Qmj?@R5;_>*o_A zAt(MbBO%~_VI%~)uZ)C1cgje}W!7aR0)ccWoGHl(;0zmuCs{C-E!O z1Gz>{OTxR)$DXICGuq%|2Oc?{=QO5=F?7zkVA7J0;PPeM!_L?LUVggrPkU$Ym#d*P zUk;6;Y`nPB!g$)8PV#@J`BEo;HJ8b-AI*P;GRYTgvY*K`>!*vmX_OXgpR|8(X?inD zq+G7Dn#E$OdG4RA;}ebp!)T=b zq7GyhM3YqgaM&C&^YynnO3GKQG9TLVzdhsnz8aTUtuZuHc%HFmMss-R9E6TsYR7*l zvB{Nsf)ap^kx=Lu35AZ4kU!yHoD%3T+@;W!gm;;6@)!ARJDk+#GlM68^UobhB*xJ7 z+XYHIBDr*GByu$F2VrqP)5vdMR;e1JoX7c77AN7je{X2ErZ>WcFw6=h)=^3Y5^tO> z5=}>Ixq6ES(d9vO5kzwR9HYc0*Ux_mN&rMgLP2CC6huZsL3B$%#BczyB)rRnQxM$} z5ShVK5M2r)#+W4L0wo@iT)O`|Al6Yz1rTrDohh+pxt7>-+vD*hdPpL<`j1g!(;FX@ z0Evu*lE_FXi70XDjgMDCA`?z7+S8Kou17dIqYXZG0P9d1(_>=jI+T%+kKljOs~5Iu z5XNa^5-c-3y7tdrlh#pQ1pu$SEdX83^Q=7+q0o8$mhA*UGPfKv8i$PIgwX(koQ5*U zX()t@#-;P&M#EvqC7>LZCFpJrZb*a{pXtOSdCnoK$3)S`j2DbX=;Q;r+#3Je0P5024I`W~&TLCINt1 zlAyae_yQ(v@wrOq;~R+TF;NI6vy+Wrw%#iaCUu!9_5|h^UEoDp7b!(5CPwesG2*zbYjW~?Pp-vn|gG^TN z;&2*ZlFbyqf&w2j( zMm;=Ne`4FjlP_Y0=k8m?dV9EJEIQQ1!)P38<6$(&Wd$P-rvWY*4RBfJ<>3+lmt}4q z0d^i}@tIDLOK|isQMiaj4YP=?+12y!fcZwfJ>SK)xhG%9n!2#}%$SDGlWCw;#HRKh zM&nX@52FDxISnO~(@-)w4JFekv-fZbD4Asmf4Tvphl>IzE4X`%n%zCGF_{eu9K{daRn}5lmDQ`U>g)Vj zG#e@DAFCe~#nb0^V;Kx&QoL)a$}oE-{&w&PY!M=S$>ta~pad|l-|!?=GVcrDo<&uEKD++`2`)sEr7Aa9JMx^sMF zS-zR=4RQSVHh7Ni{rliu7~eOhAJ~5h9c&`5iEu~i`Xbq=bu6#DEsi>N*N!Rh+zR}h zyo+WgzYfw*i_ho=SYH4B^mJke&P`jf%B3&zi#bw0`@YR4vm}>9H};wlVgcMNMI~RUQWwY6g`ek5QAl?!8k@U3N4-Kz3j_fwCbztsKC^4hKnJ}@mW&E49fQvkTzzPC?6D%-%0H$^% zq7zc^5ov#1Hvk03(18LSJSG{z7jFQk{A9zRTsf5tdkd4+vpBkUUXI4T@3X!ftT8{q zU(46`VbxUX!>Kr+rV^ZT2P_!ZRQS4xhwGRR%Xs3A1o5>&$b1ONbZ5&!@c00000000000001hfm0KcZW9)h3MDE5 z=94ugIssUda3v-iZ+oG?B>?~cPyzq|5dZ)H0000000000fPp!au_ZnMjg#;tBL+Vs H000000&W)G delta 3253 zcmYk9c{CJUAIHbOCE3DwY?;xNu`9-!ZEPb_cA_xZmysd7^fWH>=FQ-a_I!*1WlgK#%>M-q$C zh#sS|u(L^b!l3oWDc>q<#Zw+(zn&?*s+m;tF3^CMDR5gh@>2MXov7oL`!lw?&R>{+ zP^T2--8~`3$EbqaMm(;b>P63Iy_>c&B@1L~(76tIcyj|A@ zmuJroF0gra3Ga}VoJ)IDNs1yV&9ZGs&|V!SF-yH8+X=oD0M`80a)5HmyoSz1-O%f! z9oc-?tw}Ak6MW;DiZ`=(XU)IFZGGu7A|vOpopKmT;x=2QIu>|TIHuL^4a7753=rzC zlq72DB}hV}JwTL0TCd4K%E_<^`+=)4N8C?YX{k?mWj<7Xs0arwXjGU3)}{^DAN>h4 zJA(@Qjd}cx%=KprmX<=_+ZUBqq799#D5#0@*+0*7&zYgj6_-3WT0a^vG3hK8=BV0N zF>}uTHq`W(hCRq#i*uH)056iM?jLVXiukYIVH1bpsjB4}O7}un88TmAwjt$9RNu#CNg6(UbY8(&=C^O0H}lYEBG@OUOSLy^oi)iA+@ zHz|^)bw~^I<2y*8d7Go>mF*9=w%>*~ret_#*WSrqK`KqmAjO%}jHK9bB(C%C67Pic4o`nwhg@=M9`y67IoZQ}|NX%EZfD;o#A0M$ zk;uqmX~;PZRFxQ-&?{rN&S3`7fa{mKg9J|F_%S-ztb`c!RNszo-m{0~N@=>QURB?J zriebG8xF}5Hkb|+f@~&J#;dXhHa!;DXh#U?j)x8vJ{T2oAxm!oHf(oi>1@F~JFZ7j zHq+ktlAy`kU`|lMlh3H#^E_okk5(@vQx0M96pOlIG=B~t8S^-Pzy$IqR&lY(i-5bP z5l=ZPB-_D~%8yP%z!0NkI?Q6Gb)oJ4N%*ywgd9%E`F72lcZX1N4b+zYWz(HZL)5MW zp|@O32g1N7!U4|8vf{h3G%hMp|JYFGne*$@j=5w6q2QtqW*A-?cq58aO21pd^>yOW z=JmH{kTK`L=~eFsf31Gejk0hwjuZsE-Qona7f>CdWb{k7>9t=tVm$(2NY%3<{Tu`a zW*E0;=7bfyNV++dSer=9G`|LentvjT&t%?=s9|EO$&6Yzg12W5+@I{4+#s!dp?~jbC9LFqo_9cH5DiPbyS5-% zmBCOmkWZ!woXJobD49J5$g&m5LNH4H*GPiNA0u)7b?2t2WwEE1)N((gMH#n>BlaB4 z<8t#}r-tXqSxnbzBXt`36QzZ*+`Eg4z$9R>Ft@TpcPi2Q-}mF%f?JupJn|)sW2M*} zA0RFcfFe+N5a+hCM2EJ|YeO`P@g;_zmf7HG~2M6 z&kve>>U@!(l<#`ayox(e!?HZ?I&WcnlyJH^SsTMTKvB&>TsI*73BoX$2rlu!fmea( zg`aUa@o5gSaNOw}v3Mv0K@*j;aK!aX$zcn=O)IQ3_Sr55fa-JXge-hP6MPws{l53C@dZg~T?-xAj-f&TRv+ zYmojM^iM9Ha?Lgloj}cUahE17G&bPYRj#jr#iR(RCFOnrL@x(l7fNXfJ$i8>8ywVe z979jj`mRvGL!2Aiu$=z7Vo9Xgn;zSrT#v<{R?$T+xZP0It?`NkznZP!tcUuM4b{rg zf7PO;t~vDH5lC8_Xy2z_4uacIyVfV;!l&Fi_I_g-!Kl~;CN|$a`CM=QDlAU`D z9jSKO&3?a-I0bV)aF6`izZ_Hqxd|KOoNE4uRqOW2O4oWQP|mg zS*E*~+rV$+=)5OclJ?$+YM>dLNW6IrEOa-%%Wh3b{(MGs_Lw!1L5mG62~Gx8h{X%$ zlo2{VCmd8T;XW2a<+aV0TC0mq9}?cdRb{PL^NvPlUPT2}u-R2Nba#r+ZFCE|wy4`O z&MaszpWDY6yu=V$H#UX#57MIOAGD8`#L*2)3WmO3AM268H3wiomZ!Y{k9*oTT6s>0 z6zN*N!$WlQyCMhfJQYjFLDVPl1!27H>5@-yKOhk?x?ltdR z88mA!9WO_|;yk?@X6t=d{T(>`2X>9Nb^n1puib1-Uqusb3l-f^x$kQiup8r+)z$y} z*kCo{YGe){O>lv~f161mNX%E!YP^kH?GXeljbRlzgW|@ccAgsBhEZRxRU#W%3pHvm6@8XE! zAMfLBj>(+Vi1~dvYzee*Q%}en`5y0b>*d+T^q6jqQk#@UkP$DWyv@oYMQ1&fo_v8? zwe)(5(`@pQLCt!do6Nm0J%v-OFNGG$0M_#N$_=A7ES@yDM`cceyG!F+Cv2@H>RP%J z8w0-pWV|0_fYA|yRTT&9m=Qz6i=cb_Sa0&QTOH?ty;|qpNGrlH)pE=4Hgi zSlwN{;z&(y<#y)c3m z(-x?!x7&n~8e;tauf!dYEyDn|6?EahvxNN$5@fXI#|HAB$DRRSW}slzz#0s;SQJ=B l^p9LH0{~!ks+;#z{{L2NoEo+XtizCo-2zLqYD@n){|}c+EyVx; diff --git a/summary_results.json b/summary_results.json index 5dfd45c02..890d6df91 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2049, + "Performance_Result": 10.2598, "Performance_Units": "Samples/s" }, { @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.27002, + "Performance_Result": 0.407085, "Performance_Units": "Samples/s" }, { From 1a8cf90800fd3ebb7fb4b89eda886237bfd5205d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 22 Feb 2025 11:19:57 +0000 Subject: [PATCH 153/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9755 -> 9755 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 7b0338a20..6c578c071 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 11:17:37,762 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 11:17:37,762 submission_checker1.py:3270 INFO] --- -[2025-02-22 11:17:37,762 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 11:19:51,986 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 11:19:51,986 submission_checker1.py:3270 INFO] --- +[2025-02-22 11:19:51,986 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 11:17:37,762 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 11:19:51,987 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 11:17:37,762 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 11:19:51,987 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 11:17:37,762 submission_checker1.py:3286 INFO] --- -[2025-02-22 11:17:37,762 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 11:17:37,762 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 11:17:37,762 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 11:17:37,762 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 11:17:37,762 submission_checker1.py:3307 INFO] --- -[2025-02-22 11:17:37,762 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 11:17:39,007 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 11:19:51,987 submission_checker1.py:3286 INFO] --- +[2025-02-22 11:19:51,987 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 11:19:51,987 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 11:19:51,987 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 11:19:51,987 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 11:19:51,987 submission_checker1.py:3307 INFO] --- +[2025-02-22 11:19:51,987 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 11:19:52,965 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2598 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.045 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.347301 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422627 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.23624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d2d648e25..394a6cf1f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -780 +781 diff --git a/summary.csv b/summary.csv index 9a4801d72..3045462d9 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2598,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.045,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.347301,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422627,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.23624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 293b1c2ee..27c446bb6 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2598, + "Result":10.045, "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419926, + "Result":0.422627, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 44827b42485d9fa5101be132a783987f884fff4d..8c55bf027a6732e90d862e64d80ec05549b366a6 100644 GIT binary patch delta 2369 zcmV-H3BLB5Oq)!wSqKD;WsyOXT?kYG^OLd&#(&AQpO?e9Oq78f28&5{6+e8H(P*6I zr_Cxt!g!J{|5c_@el87FnFfz-wWTT#(I6WaiIs6Q7|7y8t4g>?vv~U3>X29`ScM9E z9V>LPiS{&M>b7AOpU2Zgu49_3UY8^M7-T``Fo_?OWZ6SH4~kDl92iD^TEe3BThUyM zt$(?0xO#3-@;ZvMci(9pHt&nXYsGgOa!ng5wml)=WJ?~dV$D@>CN z6G*IG$u*GJyOJ9qadsuQK;mvBJLR@;xhg$s0*CjVxLO<8iBs=Q)k(@fbSiT!6IXBe;AS_ptNzzn7n`{L|jq`{imV&6h)? zC>t;Cv@o7Fr<45OX};9SU(IDQ>_@YqO!5Vr>}N8~`sw0s8l}bBC+*)`n%>M3DVM9P zX0e!Rp8F^3_(Z1p8YXS77c-4b=6`-TpEtt!q7lz#kI7<|<*TX2TGbEZ*)STZzo-M5 z1<@o`KO8oPV7~rVM@jjrRpvum{5Y$9LLvw!7wu_Dc-JGGoY4lKcfi)6G^Ph)=sFZg$VYJL)eGA+2;;Od z36>ciUHfORN$V)D0)SWE7J#nidDfnZQ0SPyWjg_o%q_=2LZCFpJrZb)P;KIp_FdCpl>4@A+&j2A#7bn<~*ZjJwK0QoWx7BCK^ zoafhLlFI(Q(A<0~{L4tzQeg!quh#=xLFs6Y=Wbo*yHw%_?MH$?Shp-nCD%BDQVAZ= zGb(|GQ3*7RO0MBliX1X3K>|i)NrJARTM1{Z#Rr{u2Bnfl^-vU@BaxGi4I_W?R@0H- zyEm3ivE&-ZpjdJZ#}a55mO#U>GocmP5hh8mC3U>ZIvm`-xbMOUB+TwqsO6cPo7S#h$989p2&B1KFX)wvtoWIUw9pzPGvbFnQ z$8=g-pUp0lc>DfH@GWtRZfOIJV^Su$hBFB?j7gwjOmdBDQ6|kYCP63kS!-E>?&jbN zleEPLop>$EB#r8!D8?k$2%TCOfpjrh~g_(t^q68I*mF1 z3Y2fvrXz??M>u9{k55PJ7mUuI5@;BeWquu$fKgfI z*b&rFbH-YH&}xK@1O(>%QF9tp!S+G*5ZRs`14n0(?L;evIH8o zfUVi7^BR>u=lSa!_3&JYZ4*zvh!vi@ZxQS5;SyMMsEY?^9BShM8qQ?}BM;IrE`f$| zS?1+I2^g1UZXRR7>^!W+2c2*(nWG0p@kK1xfJJQ0uAYAf%s1-o`7XB2J^4b`)P=og z#x!(HropWuHnsNvjZ5u4K*PvH8cHV8P%@E*l4+FLdr$&OW?6!+!R$S(#YdgcjS&{r z15u1jpy88^5FLNqDPmK1574;O-2*fn$_nltq+w8k@z@l~GItN^gh5&6?lG9Vhqd^i z6VIejGItM(Vo+9a_ZT(1dtPHQ8x}ZP*JR^3?7H8p9oOqNfakh_O$A~_>F?aBh+t>mRYOY`cMy%aCdvx*VD4g|amx2uy&AS-`MA*B_z18vEv zfkIPSaud=f5w(7h)H-?lDo)}g;8TyQnf-QlQ8nA?Hv0u1lr=RXMTs&n!b)W}HTqc2 z^9PiL$c^N!HBh5H1T<^zD&d&09^Sll;G=?&C6EbBxEgI^bPVGlwxD?^lEWl%ZN26( zm41UA7vFpXn3m-|))0TWfcKbMB8OoPX^+ESH=XpoJI#L74t3}kVlRV7@cSv>u1bx5oetU`so zjupDtM0=Vrb=$Ct&*Nz#*D=jiugejB46>ken8XiCvg{$92gN5N4h$ndEn(66t!OUB z)_+_#Ts=1^c^$>syYI9PoA*WHwc3s4zWJ>g$s0*CjV7p`UAMlAUTTwtw0q3lg}yT^yr=4D$pPg0S$Bk&x@>6C)uf z{xc&X;D2Ex1iG(`gg|%7NXTW@WhCUoIb$T`nF%nYmXOy@k%a>;f^C$5u(|-fG zMovq@yU)j-r>Ha9;A00KIi2S;riU?f&beUHl8@l>W!%Hg*Z*FAy7EtZXYZG*p)_9( zjiPM4xYNRT+MG`Ef2a9UCx11U$*>>IhBC<)Y_gxpH0!5}yJ?gbYoD}#Z)tinOQc+` zvYN$Ws(J38tm6}z=4+U=xn9gPHh-D>;e6f*=Zi)>n>{9rS(dM+8f#TQjAz4Wr2e7~ zWEMn|RQ+(+95VCuw>nD7SFJK1+Va0WP#V)?V(2=Qk&utz(yJG?X%NO~ zV-hSgJi7MJUX#{QUIhTJye$A-&GW216QR&~{+8_oKr*)+Ga84C<9~$F0D_!`GRSEt zgp9_e^WjFrVaO$*9F`^MZVqlpgchIa#3OmmA*#nj(Z`Gzj7I3>1G(H9|JwlaWgaYG z97s9Oug4^n{d=Lg`BeCqk*uY{3QS(F2eyLJ(Hzg+y3BW}3n3MOh0OhyA>G8zCA zH3(*_3``~gfLW5DyE*s*CT;P#e@f`%8;I&LQ3xiplZ{}u-ZYrxY0h6~vX1g9G1=OE zuwy!{t#w61Teb!o*pu0Kv z!X$0+nNGYGWs*kqxF}!}HA1J`(_7=Og~`QTJ{2z3P!b@s`{{DFET&1}e`&@KAX!Is zm6osO7Lr19OiSOleBEO)5$GDQJW3>L9E%c(8YGg@0FjIah-5TCq*X>FlK_Y;NzmOK zd?Avy_)I78+!dmFToe$=XyC5WdP6||6Axc&DXpmg}WlCVRl7oHs>IfKYv@~jhc3@#kOn5f7Xnaz0b2M+&x$_ zrX1?qVKfdk?l2m}vVwJo(*Tx?2Cyu1?{EnK%QF9tp!S+GXz`g&@cAp)bhs#(EEx?f zU~6{jyhi2EdH(uFJv>)p+r*PEVuk1KTf};MxMVCk)WySS9BSiXG{|KIBM+wmE*TAQ zS?1;85&)NFZXN-4VIFAlnNE;PaP%-yxQIm!vxu$P)${Lw`9{4x-^I4MCtt{#y0G`m zn1;@iX`ofaruH62<5GJMqX9BG4JDJ)P%=3UCDSOg_iza)nPmyO2H1O`#pgPq8zT_a zW1@geMuU@$5FLN$6tSthhtasy-NR@Q$_nltP6JRfil z6VIejg1d){0w^oEdyJahJ+CpD4GSE_58YMPQd^bPtFhzi(KX96=|*L9>kIBvnT+I# zoJ>+BjPujAirloWzV&}44sKUpkbjZq{l&9ER9O0h$?9S2{8=;`Dd``p9}2LG9}3Wm zWa02?F_B94G?#5vKF;$`s3=xd29LwlPXs9QB#M(Lds_8g{6@iI5_G2V;Qew?i&^x? zLcV_&zOxVBh4zPE6yt-^R`SxHrFnJBUWymovx*VD4g_y|p}&($AS-{%rfm?m18vEv zfkIPSaud?V5w(7h)H-?lDo)}g;8TyQnf-QlQ8nMGF8KxLrO`DaS&kCW+z6@L8hxx5 z=?o=4FfCYTG}LGh9?hG(id)K!gEwa^xFEqN5u`okwnp0!EXCOKEhy%*=+IHz7^hf> zrE9Te{F}9ai#&h83Icxf+4v%VaxF+agy%h&f|)l}-ksW_mf5}a}eEEv~R__~OP>zEJA zc;bx&@wAWn#M^0+V;_hl2B62dXXUeeepxM0Q{=^cnipwNtOyyQcvyCCj0tDLzh{Cg5Bs~EHZ+oG?lbj`gZ|rl& diff --git a/summary_results.json b/summary_results.json index 890d6df91..99d7f042b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2598, + "Performance_Result": 10.045, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419926, + "Performance_Result": 0.422627, "Performance_Units": "Samples/s" }, { From a6f6d27918e085f0d62586fd41e3a9f2559ec120 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 22 Feb 2025 11:24:44 +0000 Subject: [PATCH 154/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9755 -> 9755 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6c578c071..6147d396b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 11:19:51,986 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 11:19:51,986 submission_checker1.py:3270 INFO] --- -[2025-02-22 11:19:51,986 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 11:24:39,584 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 11:24:39,584 submission_checker1.py:3270 INFO] --- +[2025-02-22 11:24:39,585 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 11:19:51,987 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 11:24:39,585 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 11:19:51,987 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 11:24:39,585 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 11:19:51,987 submission_checker1.py:3286 INFO] --- -[2025-02-22 11:19:51,987 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 11:19:51,987 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 11:19:51,987 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 11:19:51,987 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 11:19:51,987 submission_checker1.py:3307 INFO] --- -[2025-02-22 11:19:51,987 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 11:19:52,965 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 11:24:39,585 submission_checker1.py:3286 INFO] --- +[2025-02-22 11:24:39,585 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 11:24:39,585 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 11:24:39,585 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 11:24:39,585 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 11:24:39,585 submission_checker1.py:3307 INFO] --- +[2025-02-22 11:24:39,585 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 11:24:40,557 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.045 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.119 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.347301 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422627 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 394a6cf1f..fd394dde2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -781 +782 diff --git a/summary.csv b/summary.csv index 3045462d9..5b8e97247 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.045,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.119,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.347301,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422627,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 27c446bb6..c2db40901 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.045, - "Accuracy":"acc: 75.600", + "Result":10.119, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8c55bf027a6732e90d862e64d80ec05549b366a6..74780e96f8062f7dd888d72c2d1b37563c789cf6 100644 GIT binary patch delta 339 zcmV-Z0j&O;Oq)!wVG{&L+ri1RWfPkM0%2RT0v1II1U7*90h3E0D}P%_OA)pMZON&D zLQ`6D6Vk>JwSJJ)I)3{qPU0lsQ;(~e{dRUyx!9{V`32{t(G@DP93`N+5mGl5`dBa1 zN0j)$v|z2#P@x_?TGV$Xx0D+PZ_Ze7L4r>rNPEg{g?1rWim~TAP|Ro1p`*ApPO%V6 z*I>)|H){ZsJb%Cn0)G=MFnj=}b|j(`Qt%OJe_T5N1jo>V0v$XqvI1Yc0i5!a4TEy! zR5J7yCaq_2bnm?!ja}De-E6SN{0076y}l2trcxhH#R1i&;FLRH!MM7_*F`*B%Y0bH z6K^Gmr#|WvZ>L3$eISw;fF9$XmCxDp%X*1wl9T&1Cn+K8q8XrMPUrdNF!HAV0#v6V l|0bRR$K1D{p#B51jUi0~1V`J!$&=nBx&$_W_yLogC4b0BotFRr delta 339 zcmV-Z0j&O;Oq)!wVG{(?>>&%YWfPkM0^wS-0v1II1iTEltCLG0D}PHNr4_aVZON&D zLQ`6D6VfITwSJJ)I(hplPU0lsQ;(~e{dRUyHQVVn`vo7AH8mndi83(4N@X@R`dH5M z2b6`#jpVI0P@_ErG;8iE;h3-<-n@0-qk@nnkO@q<8f{~A4C5fSpm`{g!z6KSz2-5M zeuEts-+Tj@mgPOx5P!Mkk>dj}cLNcfkWvgt_v70hAO(gNG?*ALDG0uJBX}J~8+zr+ zsa4!N7`2|o!MzJ=Fm_#6bd%ni@)P{Eczy3z&6PQvN&spqDVXrUqxDUNuZwiJmWQ}V zCtfR%Py3{gyqy*~_JPb&0CtRfRz4@sFUvVyx3zOa?x&*uox2uz!C4Uh-n>GLd diff --git a/summary_results.json b/summary_results.json index 99d7f042b..a44821520 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.045, + "Performance_Result": 10.119, "Performance_Units": "Samples/s" }, { From c962ed703428e7ff6710bfc7e25eb9448fe628d6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 22 Feb 2025 11:32:20 +0000 Subject: [PATCH 155/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9755 -> 9751 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 6147d396b..6093b3f43 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,35 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 11:24:39,584 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 11:24:39,584 submission_checker1.py:3270 INFO] --- -[2025-02-22 11:24:39,585 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 11:32:15,375 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 11:32:15,375 submission_checker1.py:3270 INFO] --- +[2025-02-22 11:32:15,375 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 11:24:39,585 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 11:32:15,375 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 11:24:39,585 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 11:32:15,375 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 11:24:39,585 submission_checker1.py:3286 INFO] --- -[2025-02-22 11:24:39,585 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 11:24:39,585 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 11:24:39,585 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 11:24:39,585 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 11:24:39,585 submission_checker1.py:3307 INFO] --- -[2025-02-22 11:24:39,585 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 11:24:40,557 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 11:32:15,375 submission_checker1.py:3286 INFO] --- +[2025-02-22 11:32:15,375 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 11:32:15,375 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 11:32:15,376 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 11:32:15,376 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 11:32:15,376 submission_checker1.py:3307 INFO] --- +[2025-02-22 11:32:15,376 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 11:32:16,533 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.6142 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.407085 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.36853 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74502 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79832 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440164 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.5941 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9443 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index fd394dde2..634a08341 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -782 +783 diff --git a/summary.csv b/summary.csv index 5b8e97247..94ee89f4b 100644 --- a/summary.csv +++ b/summary.csv @@ -1,13 +1,13 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.6142,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.407085,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.36853,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74502,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79832,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440164,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.5941,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9443,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index c2db40901..bd6462a7b 100644 --- a/summary.json +++ b/summary.json @@ -39,7 +39,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.407085, + "Result":0.36853, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.5941, + "Result":20.9443, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 74780e96f8062f7dd888d72c2d1b37563c789cf6..99943fe127c1129e2e717243d2ca24a306bd0eb6 100644 GIT binary patch delta 2674 zcmV-&3XS!fOqWcsSqOg!Sw&7o2><~9K>z?40001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`OuoLzIG*cOKG&#$1UzUZzRk&S$&qEkuwD~6bKI?2?zvIuUK8WjgHAOC(pjnTDU z3DeUvq|UiXbn~vY_W~aG23|e<^>I2;?q!lj@vN;1O;;5;3*&!5G#j~LV>t~{EuPC+zGWCE(;&;= zCZlFLPh>Dy22CeTL)Y!*G>B&ED#JliB}Y6QMxp!|hl{D4Wve8KoCH}u{B#`6Q?+wA zi1KYkJxU^nZS{Xh9Gv`S2({TcT=sn%$%l07wNjLF7stOBZ%zkoRWGDXrv1De#$}=m zvpWonRFz>~*Zr#U|R*gsIzxRUD6}iCo7tSG_7ONBA+wg3e(QKPbtvhjbnk zM@AeNM!s9ZqV-$RT#T)`Zn%1GQ1Uv8y^{e7ZGVqzX_B34Em-Z5#S*x?T^w@-8RiLB z2*ScgAR*_^Cm>Ugq&twAR!0m3`oe%^?`)!-E%H6$lmP% z3E8_BKtlHJC6JK4`wd7;e7f2AbhB9kixhLrB{tdjCtT7Z^?n2rQtu}qA@zRd5|?E8 z@_#G=ABkT<59AyFZ+o7i&S-_g zKkc2pU#^DId^t3Vvhm_h3*%{XI?4Y|^QBJyYA%yuKbj3?k}ud~Ka*+JPZxL7C@t1L zY5(5R^k$Ywxm;y6i^Wv)+&@{zCo;{~Fn?)ty_jiiGWWyzyb;b9jd(VDOct{&UrjaE zs(u*HhS5m_T@X$_kmAJ z9i>zt@y6LA(R8$ytG6(SE(X!%Ad>Uvm`iMO{+w_L0})6lh(JO?1QH6OTLK~vXCRh@ zcR@G>(JcWH44#7MQV@BJNn$Rz#D61_OZR^V#5zi;0OGB?GbOex*Aja!dpt~{$4Mk- z|1p=?^umWr7>PhaNdyu~BA2-I!pAEi5rmVI_Ov9t>k&@QXoJr?VCzsC(*rSd9SS7m z5L|lo!Zr=UIBiUVWrj!B{@H8NI?Ag6;FY%ppsRVFwPzv}I_7WLP5>lx%YQM@IAk0r zK*JD38prSsuNgD^x1D2HVUx|@v~5?PB6I`K%Ja~9PDQS>(B1<(kc9FWVc z@xKiqU*^FA#(|Xc{CrGO*}oT>n@@#*8Od5Itia^;dSELk9nJCFty-qsA&z<$7RZebZI5g}Dz){HWT98(+AjEk3G*-o0T_JrKpY1P`(~m#sI=CApdN z*O9EFyh=#6c9Arl*4DAvWfBTMy+0ECiMU0dXp>Y*scB-aR?KAqkge=STd?(&&%v4)ZWncYv9lVveW3QseB0LeP4tF(GGw~!Q?V_N#Y z)$1OIiQuLI%cDecjbl+FxrP%7G>k}~VMGEABho4(5+qic=UTZ0^5|h^U6=h4)b!(5APwcVb2w<|IJ{+KNs1XNfIFl8u zI7q{o1RBO#HFe=OJIw%37vdpj}sGsJHwfLYD z{!oPZazGS+D8e=1p-9c%98TrWe=71uJv-N8+q7eAM$6u3SrzUcEP*G7+IE1(p}rlU z;aFC1?jQ}r5@;BfW#%1}fMHo?-x1VZbH-YH&$B*Q^cn3 z9-wily9a0(nMgy)L>fva(oiyuGItM3K!3?BOVBl#yN9*-s1y2Nghlm06eAO8IGIMx z?w;4k{JT#^YN@XZl!m?wr6|5dya!6L2TE=fv8lNSXk2RU0U8cv1#=J5Fet%zYzk$W zxd(N^pe!@@7|h(mT71xnXHqDcxd%ltC@YwIjGE0ouQ8bo3mnA{T~*dnTb0$Tv43Og z(KXLBsd=Ee_5Jp#Oh)oVP9~`m#`*4AMQ&PGU;4ii2e+&5$G^z){^HpnDjfa6Wc6+S zESim!^pDl=1lYyz1n5PwaCo(tNF{rk%eE>X=lKyTidB`t<8bv00m?jy;v~wRR=pQL zQLvZ{Wwzq5a_(#rUAKmAv$4XxRM+wg@MQIy1IyQ0JYT-DROK$tvUvLZZY+a=Op13cRT;){_WY*UC7a)F z===$@$q{}I1z157uq2aAAS!=LrlAPifwtt-K%pruy$NZPh+02LYMs1&6+3Yf$f?KG z%zit&sGje1m;Hhd%9;j|qC^=OVWl#g2EDHq`2)&ArP5BA_ zT)w;wtLDlaP9*@fwG>Qv;L-ZF#+OApT*pIPrW3Cf$j5!sZ@e8B8T&wHDF7Seo|KR3 z>3Ow4EiLJNUecV>6(Ja`qKwSG4kNGn&p>?~@^9h^Fy_Ag1nnQQjUmbc3J6(6PDKd- z0RKUgVI)=o#*?@tJOWS?lW!9ilME#)0ppW5B{~6BlX4{{8(2XTup|Kh08jz|01*HH g00000000000Dyruld~m00g03HB_jqqBLDyZ0BZW~#Q*>R delta 2680 zcmV-;3WxQVOq)!wSqKh|WsyNh2><{9LI4050F!Sd8h_=S?Q^2Y7RJBdzk;IX!%WqR zH1e8?ZY7zw7-BM+BwP2BMX*(BRb0Tl{PzoLjGpsI*xA`7b?=uMZJu+w5AsVl^rMHr zK29gfy-d<5p0!n>>8c`UVLXUtqqcf;{kh?)N}2_;K`@DDvaLSJRQs&)kZ!$Jic;?4`1j(?>7cFZg|x}EpO?e9On;Pt90rR?b`?K-mC5Cs>6FdmSruv5EFH zVd}PF6`#k`M6P3+t6rBQ{1{|G=P-#Mlw{dMIuD9ZMjRMMepV;y`O-D)ccuBT$1I>f3pPqN&E_WAlJxgNqG19*z**1 zMjL$Ifk#f~IgRP@7&_-%fVAWzxO^G+u=Dl5m!Gcu)85(pEdo0rN!DO?cZCP-pmpym#eI1v6yO}`zP!8 zM5g%~e5UJUFcN`;k_aS}L@sgZ zjgMDCA_ylJ?P*DP*CU*q(FUJ)z}BHOrUzo^IuuCAM{w!Y3)?gZnN`RfLGoYfUf3w)}Dz_=$OA{I{}c)e=WyAnjm(GV94Z;v5 zpd6MZ=xz>fNMtQO=)@y=&RJ9sMA65L7eFI)@_}4#jsI-``7#d{Fb<@g=htJB%Kp93 z+1dAUZe8ZPRN@EiM}j|Cw=7B}*EoVw2_DchDuISkO$juN zO0MBliZUue0!C#?g07%j31_Uu2c38ZrIJSVP!ydbk%mr^Tf={gP;SkT1(k-bi66kS zjvA{>mFt0t^+Q+97Un)I@mAB3;JcHv4IzIVe8G~o_@EPxC5`HVDE0^yXoM~(m&X4A zmvz)w<+18|V4LTvx}&?+9x(By=#k)ijg~{fhaCeSc2frf#}H5|-V8JHje z1G6MScXRLsOxoh3O6cPo7S#h$989p2&B1KFX)wvtoWIUw9pzPGvbFnQ$8=g-pUn&| zlX(07NboIji*9L?T@D(5S!-E>?&jbNleEPLop>$EB#r8!D8?k$2%Tq(=!89YWl=p8#fSud8g^G{y&)j~iHEPX z6j+H#Yx|b6rRlo0$IU19_;3Vp*-#@6&^XkI12mk;3SJzfVN3!IW3tSSgAy<%%M3Y! zIdWKw4?5vYGE)wS;wx9K0V~%!jXD1clyB9hBZyB&IA&{)Pe<$+b@Uf1fenW`b%4g9 zRvn<>R8}zSAPu8`5@;BeWquu$fKgfI*b&rFbH-YH&C!^lJ$N+!}!GLeRo zX_VP}Py$MSW?6!+!R$S(#YdgcjS&{r15u1jpy6a1HQRe$BlGX>jMP$J6(|jT7fMll zjd%}~Vh@zuDPmK1574;O-2*fn$_nltq+w8k@z@l~GItN^gh5&6?lG9Vhqd^i6VIej zGItM(Vo+9a_ZT(1dtPHQ8x}ZP*JR^3?7H8p9oOqNfakh_O$A~_>F?aB@laC}Cv*ZzA4h1%V z_yHx8NgyhJTS-e1wgYX+sewXMT5=Q8#u2rCkkmSU`zlW2B;ZqztC{_Fc2T+5t2X%s z=cUmVDzY3Upt%uJHx>F=FVjbq_`tMatC5Wd!>Jx9LMUH(Sk{Ey<6)o&m?)x1XT?1G9-C$^r_FWsyNh2><{9LX%x2RsqSA zv?M$MQ Date: Sat, 22 Feb 2025 11:35:13 +0000 Subject: [PATCH 156/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 9751 -> 9754 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 6093b3f43..fc98d8b01 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,30 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 11:32:15,375 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 11:32:15,375 submission_checker1.py:3270 INFO] --- -[2025-02-22 11:32:15,375 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 11:35:08,428 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 11:35:08,428 submission_checker1.py:3270 INFO] --- +[2025-02-22 11:35:08,428 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 11:32:15,375 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 11:35:08,428 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 11:32:15,375 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 11:35:08,428 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 11:32:15,375 submission_checker1.py:3286 INFO] --- -[2025-02-22 11:32:15,375 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 11:32:15,375 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 11:32:15,376 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 11:32:15,376 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 11:32:15,376 submission_checker1.py:3307 INFO] --- -[2025-02-22 11:32:15,376 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 11:32:16,533 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 11:35:08,428 submission_checker1.py:3286 INFO] --- +[2025-02-22 11:35:08,428 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 11:35:08,428 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 11:35:08,429 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 11:35:08,429 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 11:35:08,429 submission_checker1.py:3307 INFO] --- +[2025-02-22 11:35:08,429 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 11:35:09,360 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.6142 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.36853 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.36853 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74502 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79832 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.119 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2317 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.347301 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422627 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 634a08341..3aedf24eb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -783 +784 diff --git a/summary.csv b/summary.csv index 94ee89f4b..0cdec4fc6 100644 --- a/summary.csv +++ b/summary.csv @@ -1,8 +1,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.6142,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.36853,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.36853,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74502,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79832,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.119,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2317,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.347301,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422627,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bd6462a7b..3ef1893b7 100644 --- a/summary.json +++ b/summary.json @@ -4,12 +4,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.6142, + "Result":21.3892, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -34,7 +34,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -99,7 +99,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.79832, + "Result":7.93669, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.119, + "Result":10.2317, "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 99943fe127c1129e2e717243d2ca24a306bd0eb6..2b33dcd002593acf0dbb484809148c3820585e28 100644 GIT binary patch delta 2707 zcmV;E3T*Y4OqxuvSqKFa!bK2BlU@ilf6X_k=v0#aiZLdgPBL|_EP`95M#TZl$G;y? zV|1-o!u0eEsdH`;-Mnkj7=W>>Be;LNf zG|2L|$*7sm6B!JaLDNao&~>Ld4WgO4%5acW$q^5SQ7Avg;bJOh*(ymQCqb4EKOIN& zRBay)qI_FXkCMn?OFa??C%+j&ZMF}Wecwj%A>Ded6s6q7@$bc((?Lts3u%*SKQD)I znJ5D}3>K5@Dt`DXqtQ6acbi>=e}wTQUH(<3QNAw?Rhb5lE!9+&hiH(Ei^R$}8VqEy zqg5qbq**-uZM8|P6RbjovyK(I*hKr9uyn_;i{tS$k?WY|s@LTRKL%OQK1|{VC0X{6 z&V%B}hy%mOcS~5bek+=bu{GBXSI-SfUPrO_?%U>J^S(&DR(z)+*RU81H8k__ zYzAfEK^bfe`tBI6xWcl?FoDG0m0Safvn#m)5_eZ}3nbn~vQw^!%Qaa7LCWO{GS2U> zDz0Bu+`Ou|EmQ1NYT-&P7H^PBxx(^Esjo|}Uzgl)2|cc*Np`BWVYNp#OW^Hxam*ED zm?vBz2n!#9gq%O0fQ0P$f6qWd!2bdy1iG(4LZCYZ5^|b#fP@^JGaw;5w+kd>@1Aps zLH2GBNXXv301~oyFM)*Y-ETl*;nU5*r<=nP*rb?aE^)}dKj9LS)cX-gNWGtcgw*?) zOFWY0%d-T0Bz^@wkaOg;B)s#y?RkniqYYi&flqelIgRP@7&_-%e}J^)5L~{Dd)WT^ z-^))|-P7LL`{imV&6h)?C>t;Cv@o7Fr<45eG+*lEujVot_M_QQCi#L*_A{Ag{d93R zjnZQ6llJdzO>bt2l*?6Cvsg?u&$}n<_(Z1p8YXS77c-4b=6*PzH^TX%5zl6i$zqn} ztEt9X)eqy@FdC`9f2aeQ1<@o`KO8oPV7~rVM@jjrRpvum{@XLI@2hc%)fz*y48t*8 z44sRib2)V6R6FJphn%S=T*9CO5(*uVQ0Rb!e1v~-OQ7T79)+$XyaU3?C-T{LIH|7- z22Vco&s|C+kD=?g3oh}ASY~+ioS(fWt)sjO0ABf90D7A5+j}NLp=17*?F2wFw;ThF zOU7{mGz>wcp$sAog%D^wIv;K{2t$;Ba#)t2v)Q;If04CxK_@=RbIziAAd22*yZ{=Z zlLK0VW+on`G>}S=fKgeJpd;v`gfrIC1)cZ?rIJSV zP!ydbf02exl3T-ni%@ROkOh^7u8AMOvW^<7OqJ__iS>Qi{Hk&5$nRo_A2l5b{`AIn zD3)B~8Wc;e;aCC@ta;-X976MBJuNw1LL8D3e^nnFJcfB+xJ>xyG|8lU5m%pc8tlwJbqrv+;#V+R_D` zfA}V4l1BAV6l0QWgifDMZ;iheCKq@4Ot@G>Nr24mr_0H*m?ed;89#tz9o1D@y;@sH z3eB}_ec$SJkHbW8(}3+$BDu!3DUn>mi3A!(B+xJ-frb%jmk|jPFd|D5bT%7bh@>rD z&L3lH5@;BeWp*8ufKgdy*b&rEbH-Y_lZg)?GUI55>Fl$t3U3dVz>`C5 zJ3!-7-wx1lEGsy7kcMFiGz`l!^A1YDuq?Ch24Hvti$clFJt&GnS;5?6)NJl~jmd0S;3$6Rsh$aD%sOqwp95z&yP@1tf~wihpS%*Q07S#CsFpa z>b>}hg2g0gPvgP+<)9X`=#Pbb|1Nyz9J~vi55FkJ2c@Otr9Vsa>Xy9}FVL{shx1@0 z&x2$X%~E9|hk2p8mS2D;tKS@0zRu$L@|CS3DtB>~#nb0^V;Kx&QoJ)&Wf;fV^P6Iq zY<|0;{U=aM0|b+gBpS2i5nv7klTmx5P%%90dCpqS=D9ZK}E+r%pr zIZqX*wN_A~GdQ%a9tvg%(-z*WHekH~7X^@Vgqac@y*C77$BrPWOM*iMaj&hUK9u%= ze+^|n^Z+%9T?J ze>N~{9mUCga$+(zO_MZ>&KmM(_-p(6-mMx7)t-t0stV2svp~JJRe`UIaJZT}zYQne z3*b-ZpwGMwi}ZcKqY!}faYyBI@%*yeB%mr!^JSdoah~t8j4YNUU46GBZ~8AlISjd* zI0E|Ix1XT;2eXMG$^r!w!bK2BlUO8M0m+lJBs>9Alkp@$4dxdB0C;RKb98xZWpk5& zB@~lWB^m-)8k2Dv6_bP|5F3+GfKwy^002+|000pH0000000000004l2Ig_s?J^_rA N?j<7zJ|h4C008zt{zm`+ delta 2680 zcmV-;3WxQYOqWcsSqKFPSw&7olU@ile~o;mqEkuwD~6bKI?2?zvIuUK8WjgHAOC(p zjnTDU3DeUvq|UiXbn~vY_W~aG23|e<^>I2;?q!lj@vN;1O;;5;3*$jF8@1J&>(32W zRnjb&4T4ELlWp}$rs}Vq{~SKV$?xe{%1p@<%+j_x&a(MIvzdltISo=Rp37Oje`Oda z(;&;=CZlFLPh>Dy22CeTL)Y!*G>B&ED#JliB}Y6QMxp!|hl{D4Wve8KoCH}u{B#`6 zQ?+wAi1KYkJxU^nZS_bTocv}8wb?mb_I(@4hji<;Qj~HR$G;bEP6usOFQiST{k$B; zWugq^Fj!2otN7uoj7H-u-)&YAe-g%%bop1AM)|%pRAm}Gw$+xZJVb+RTqIV;(O@8p z9jz+iBF*CIZ>vpWonRFz>~*Zr#U|R*gsIzxRUD6}iCo7tSG_Jr_%XGKUCA|&*t?P&AaQmjw?N`(?bWTtbg)X_B34Em-Z5#S*x?T^w@- z8RiLB2*ScgAR*_^Cm$W?A>!N zG05KS0SVc=7eGSx?j?|rz55MFOnkc8_;j;b0*e%L%q2G2_a|J^BK3X*5>oFcAR+aB z<`S1=`SL6QABkT<59AyFZ+o7i&S-W2aSuCR z|9koA%0KO$y)=w9A z(*yKz-;SvTNkWlD=ghB@-YAf@OwB*Z$dS(mKkk0N|Ck1)!^Wp0#Hp6guW_*-ii?bIUQ% zIAk0rK*JD38prSsuNgD^x1D2HVUx|@v~e-c@X4?6Kko^uw}15xxg;|0(N zog9$Mt?|DNAYbOe0>*)q^Za~FQrW*3nww9Be;LVIDy+cd^?G0{C>_o5+^y4mmr8u2 z{YdZ|>y|~S$E zemG-R;HCk~qeOCzV^Jcxh7$=ij7XqiL;?*X(kdeoBw$3ABv?6KhpV6ve;9H4Qi5eH~E zlNGEuNW+)}8pdRq8wVv|OqTg^1T*BY79W#74;(Vg)*hRV*zxJ;FH`~-4mIikjYFL} zK*Oo5;MGAIMkUZND$DFTC;_9g%&;S+teATncXM$6u3SrzUcEP*G7+IE1( zp}rlU;aFC1?jQ}r5@;BfW#%1}fMHo?-x1VZbH-YHlim*`4|De|V!b_F0*4MY@c@lO zlMxUdfB2vi&LuPSfGEC*$B*Q^cn3 z9-wily9a0(nMgy)L>fva(oiyuGItM3K*=mi&^4I5hqd^q6Z&9;MfE@wBNJ#inMTd- zp4Z6yyH7@Hsjmu@hQ14>D85C!2THLAN^TUff2p|#Xk2RU0U8cv1#=J5Fet%zYzk$W zxd(N^pe!@@7|h(mT71xnXHqDcxd%ltC@YwIjGE0ouQ8bo3mnA{T~*dnTb0$Tv197d zHP1Aud7!!V{r0I$M)E{XCaDs}`R-apZdzAg`o9tfx2x~RzsU3c;@Kc79R0y$^=X=lKyTidB`t<8bv00m?jy;v~wRR=pQL zQLvZvU%s+b9_231vUvLZZY+a=Op13cRT;){_WY*UC7a)F z===$@$q{}I1Xw{6u#-z5AAiki!!Qtp?|llRyDdwmp$OZ7w&c`6p(!oB32BpvT0cl? zoxFV&J8=@osmImKemlFUp6_&*{eln5ng)@gL>U-ir81iay{{Je1Ij|=M)J-YXwV)4 znzwhga7B+_av=$-N6| zGWLC6^wnTZ`3e49zPt^q=E@vSB>=Ux6ij&F(fYQ=mqj{U$3tAE6R#D>$9>Xoyd4)A z`#@$X02||;l#lA^c_g(!EiLJNUecV>6(OvmjLg0cBd_|;Kz$tYZ{i6s=Dz*}?H{v@ zA<6;;2w6o=MUz`3TLH$Cw Date: Sat, 22 Feb 2025 12:42:35 +0000 Subject: [PATCH 157/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9754 -> 9753 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index fc98d8b01..f5e9cbe29 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 11:35:08,428 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 11:35:08,428 submission_checker1.py:3270 INFO] --- -[2025-02-22 11:35:08,428 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 12:42:29,726 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 12:42:29,726 submission_checker1.py:3270 INFO] --- +[2025-02-22 12:42:29,726 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 11:35:08,428 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 12:42:29,726 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 11:35:08,428 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 12:42:29,726 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 11:35:08,428 submission_checker1.py:3286 INFO] --- -[2025-02-22 11:35:08,428 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 11:35:08,428 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 11:35:08,429 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 11:35:08,429 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 11:35:08,429 submission_checker1.py:3307 INFO] --- -[2025-02-22 11:35:08,429 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 11:35:09,360 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 12:42:29,726 submission_checker1.py:3286 INFO] --- +[2025-02-22 12:42:29,726 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 12:42:29,726 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 12:42:29,726 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 12:42:29,726 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 12:42:29,726 submission_checker1.py:3307 INFO] --- +[2025-02-22 12:42:29,726 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 12:42:30,913 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,17 +23,17 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.36853 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74502 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71113 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417437 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440164 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9443 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2317 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2707 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.347301 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422627 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3aedf24eb..34d96ef63 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -784 +785 diff --git a/summary.csv b/summary.csv index 0cdec4fc6..a64de4497 100644 --- a/summary.csv +++ b/summary.csv @@ -1,17 +1,17 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.36853,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74502,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71113,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417437,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440164,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9443,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2317,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2707,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.347301,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422627,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3ef1893b7..4a3ae7a17 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74502, + "Result":1.71113, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417437, + "Result":0.417899, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2317, - "Accuracy":"acc: 75.400", + "Result":10.2707, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 2b33dcd002593acf0dbb484809148c3820585e28..7066449f0c20089cb0f50850828a42406653cc2b 100644 GIT binary patch delta 3575 zcmV z>8c`UVLXUtqqcf;{kh?)N}2_;K`@DDvaLSJRQ|$C>MupzK!HVy7gKqO1X>U--|b=gSM&{(k9b>UJm0jQ3i4tEGF4i{P0yq zqj8oWHme8;<4L;wSD8lnu{2a=8a%euma05NgKS(RR>sj_Ad3U7D&Zo{;^}XzkHk8` zDpc6(SfPtew5JJwQ@0JPI3G_FxsGYBdR>n2V~_=%!z6xCl4TF+JSfhLI53R-u!KeH zx1zZiTXWrT_1vK3breVMzSBBv-WQ42itjY!nl{|fhTE-%W`HEd#N9}C%5C9tTP%Sf<#Gjo8Rz#`71ysSZeCT~mML~B zHF2dTi#JH6Tw!{o)Ym1~uS;&Ygr3*ZBsiq~Lq~1?JLhAj@B`(SGzg!(9qpNq858lfTGk+u@`>9}J%S%|CZ2kvxX3-!8bsBa%y}Mj}V! zeh?P-GmZTAWtFNi%6XhWWpNUY`}c-sYkDJG2*a#EVjZPaAo0f8BGGiTmaDfgh%N@v zYAf@OwB*Z$dS z(mKk2s{r7Yw*{c9d7iarA{092Z`n=&By-C#&^TlqCqTmxL>kH<(ohJ2#-;P&MuRX! z2`GnU3A&pPHzcwaA9Ui8Jm)N`2cqa>#tWbkIyoViTjPHlK)%d_1&jkJ=lS)Rq_Tf6 zG&i3L|1y%bR9Jz@>-E4^P&%5|vh6*SqQDP-+K&W(ux?qDO0IDPr4l@#XH)_WqY`Ks zm0ZK-#*GG22@)_WOA>Si-AXuPEk5YPGboibs)wTJ9Emh^lH3~pTZD3JhAgNwbWQvK zmUYxvWvW~cOst>F=68)-XMPt;yw!9h`0kBmQ!Kg0F({T?!?6S!h9%H2EV+hbX_R4q z2@)_YOA>T9AHHBoTYS(7$C5_%Koom~3N%6&luP6PfXg~+tnyfOJ+RG9r0woq#Ns{C zBf+;CEr)W+HBO6i$u*oypkZ7B4daq)_#(Dd#wAF=xGYJ~-F*1MC2jFhCG_bHi|Ta(sWu|=Vq5lynBBn_>Q$EB#r8!D8?k$2%YXuZ;iheCKq@4 zOt@G>Nr24mr_0H*m?ed$89#tz9o1D@y_#D{3e6Eh?3*R`I7|dL4Oku}l4~4)ixSB- zoJgQyL;?*X5@;BaRvD2X0VA>`L3i`v3z4+N2c59Tt}LpDq8O1t!)_|AHw5HA@$9vh z0xL0TZQoJ0G+np$nEAvW8;$@b8|uRW8iyKjfQBK2*>sIJ$~Ks>F6(10v8T7>Hv*H zojO3nsjT4DK^jIS&@d{?>^dj`qq5AfBdDL|jJ5cn6TT_Jd^sSBZ;EgY*c7SRo5QL6 z`MV--)U$IfwoN;>X0+^mmQ~^I!4i0KsBH&m9O~Nv8jfWJ=MK^^EP;l9VOeJ0K?xX^ zW%eCG-8E;d#Rr}6$FIz#gQD1E2{dd0TeDH;H7bA3h1v=m%4j^hLMRhluV?dWFieE z(lq}w9Rt?#!_WipZ{axzJk zFwPH?4iP7RE`BFKFOr4BtHner+0$IMRrxs2&rngUstg{7t6vCE=1CMMQTDXzz4(cO z#U$uV}kA-~yE_`PnybJ9QzbM8BrLE+pKTGrKmc0}&(6Bm(^I#;;gJcxV zQe`5Cd7-+N-+(8p-yB%J&f@vF)ZD(EZl>W9(>ToBPysYMVSe+E`0Osbev~dF6&@@yQHCzZKanGLn*ro^suP( z;$B87yj|WA!<9xOh^Z{jWtg)#S(|%{^1aFS75QGSY9+h0(o-t4CYBq2-|eThRoZe* zm1d;dx=rDUr>)T|cskk^JO!mgpm?Cv*x3Xv?tcM80rCRJxRn|)A{!? z@;rBdT>BaICP-+Tn>|unm3D=2adCYz#Rtg6hG!9s&;vjY(7-bS*9#ipR04r9aIuHb z)y3E8`Q>sxT`Vt;&gQ2_)B3cf(mk3UV8bT9``ff*_&%|JTPD8Yc-Zc=j2&WF2(5YM z|B#8If5<&14gITw-)l$q66 zOI-|I(|D~;&30cI(ii!~94ViD z-)5895jufmnF&CPs~conff9+G;!uP1*cfQr7<) zdcQ+<OX{zXOl-MkzWBmaXdfp0@twy10|i0`?aO+*T?e0e#c7w;HNwi zI7(0-tkq|Bt(lyQ(SJ}&0|b*1B^t8?7()mI54$@*lS?2Uf6Y$AFc5_AJVoSt;=}=j ztR$5H^+Y8kia^{fdz)IsKe7#J-ySDz(^krf<9cSlon5(F>{OThg7ebo8WmZN642ZT zsoNU8ub1fqN_=2iu+C_x(H=ZnG&dEulp6=H&RB3kf=?nyd&+H%wjo%GvFBS*%xBS| zqqs3nu@Fnwe`3q{H){ctJio^Z0uwASJOEQW5z!GT_=L1St{VY@W9UGE4jva-fzRFm zPWjn}QMqs`8TJ-tttWAE@4TFhecxw&Ia*`>0)MVv-o{l^sfSZ>Kusk$Nw=z#1r6<`}z|!f3u7s$^rzQX%4-US|oo36FP81V3Xb?R{`adHYGX%SCeri xCL0gCJ3k}=002+|000pH0000000000004l2IFqs^J^_o9@g*Y$JtF`B000pdpB?}J delta 3585 zcmV+c4*v0(OqxuvSqKCZ!bK31T?i(BU2~$?7KZQ7ub`;D=&l-(%{Qs&RFeLRF(#c( zGIg#jf?K6V#R1I6zaLOzbgfsy^z;m=b8Zsdyld^ffXBUoR}X)EoKBQ`nWRxXYpFuh zRYlIico5A-E%oO5bHh`WGz(^fU=q({OMQ~5`fK|?hYxY`dpedfQ}P6}w55(Fvuu9Q zY^LE@PJ>j7=W>>B8OF&p$nv+zsF}_a84Q*|(@E3Nb*Gal2_k=O9}c2?TTzda$YD!8 z5(g*08A5Hg50`!4M)D!udaV?t+{N+l#hcSXOVtZ$lW9LMhjE!G133&9lk6&f_$s5( zILmjNU4(@3BwhYhrcu5x4ON*2k1f?ym4|4Mjf=#}I2sIOv7=QbT%=h%{cW{LtP`w4 zg|m(oy4Xbfny`O#$FPgz@idX^nC7b2d^t!^n3_ShRjC znv1bD*9}+C4N6`|vG?xV=3(=`NW4~jry{M#u zN-Y*|kV?72@=2+$ORis++;9m!uBAzKsb2~?RIg@6=awvTpLcsq5Bm}yzKtiB91rlobV&Q+&&B3Re!xGq}m}4$+$i6?}5|h;X5lBeApMZqa`@1bifZ1wD{+ zb>G2pk=UjlaSKZUz+56>cD9x8c zqbM6M?zAwTHm8&P?=)ZPbt2l*?6Cvsg?u&$}n< z_(Z1p8YXS77c-4b=6*PzH^TX%5zl6i$zqn}tEt9X)eqy@FdC`9r~{b=(Iizr95#nw zzW!E6N%^W(=0jWl+cU23t8t0d8bh-T!!cY8or|G!IdtSyJLVFHoT(>V!k_~Z3LTJ8 z=zxTLgnxf=OQ7T79)+$XyaU3?C-T{LIH|7-22Vco&s|C+kD=?g3oh}AfbQ6Q5P^h(2qY9lAfX_7B_Q%}24a6ncn5@25WNx*!Qd%~9tDxdSS03x zOMD`^^!|51tfQ0)Al`aAQ=(~mw%Bvo<6{ziP9iz`kGaI57d~9VNCXl}B9KrLxx}Lv zK7I*_Ae@}ErzPPXpKx+U8@jv$whpB+JrG0Jp+G_o!KGg>Y||i&)5auNW_a|RpS>on zqr86#0ABf90D7A5+j}NLp=17*?F2wFw;ThFOU7{mGz>wcp$sAog%D^wIv;K{2t$;B za#)t2v)Q;Ik+pO|CqBt@&Z2rCir!|t02-l_19G`F{0VW+on`G>}S=fKgeJpd;v`gfrIC1)cZ?rIJSVP!ydbk%mr^Tf={gP;SkT1(k-bi66kS zjvA{>mFt0t^?lj=s&VVc?_!A`H6025^u~55mR#c+6icq*SON{h5@;BfT*I+6%CLU~ z2^f|o2|AmNFIdu+F6e|~Nuzooirqp58lelyrSX5jWgRtExvjb$*ybYA@%Aoa@iWmQ z!5=l+F6EMIT$6IiHJnSJVO#@ta;-X976MBJuNw1LL8D3e^nnFJcf zB+xJ>xyG|8lU5m%pc8tlwJbqrv+;#V+R_D`_$FnNM)gn>W0GrxPM=P1jlUKq7kBwg zxL89;fXwcv%gM5sC55jUKY(N%)m2)(T3bj8&9!WO-|BUb!$fe?fbCNvxyFCBDUn>m zi3A!(B+xJ-frb%jmk|jPFd|D5bT%7bh@>rD&AJng%qRBPa0D>fP#+G^xYURPG@Qu_Rve^ZOacvKvdoQx5-=vq{5XOc za#%|jbi$cro*WRxSFT(GR<3__`f~miDBr3}M-ZEia4mC>O-Jnbbo3W0feVKkb%4gD zP931(R95inAPu7uXc(1cb{&*}QCVi#5!6p}##*|d6aG+y`Eo!Me<;E=;Gsy(-W*Ql z&wncNMm;;%V%xOiXol(Rv#bhl50=1_Lv1@i<5J%a&~Pj(ICqeSVF`aU49ha}4obkV zEVJ(j>aICsEnUzFzx~QwIw*=wmO#T6ur(WXUZe8o+B<3Sq6CD1S~%d9*o0pqgF%p;hahqZJ;C!9-W=mAlD5z94T z5nHpV=idSIjaqxYi*0{zPri_~bm8ooF%2D)X>g~AL)|?<<571H&@eKQhLVXiluV?d zWEy4e9+ZHRS(cz7q{PgAo?h15u1jpy6a1HM@IWBlGV*8L6edDo`5wE|jA9 z7V#b^#U3cRQN*F<9-#54xd&)CloiZ9NW-86<8dgIW#%5#34=wl%-mxza}R6jf=+yk zLdncMD2hQ@!Q5liZ0>oD$!u8QD1PXwvXBQ;)9smPO42&8_dZPh~QaCvq}L zl`zhClMWFle<^+^KrfPo!>h$aD%sOqwp95z&yP@1tf~wihpS%*Q07S#CsFpa>b>}h zg2g0gPvgP+<)9X`=#Pbb|1Nyz9J~vi55FkJ2c@Otr9Vsa>Xy9}FVL{shx1@0&x2$X z%~E9|hk2p8mS2D;tKS@0zRu$L@|CSBcX5`*)8}_%85s;@QoJ)&Wf;fV^P6IqY<|0; z{U=aM0|b+iBpS2l5qbdyN87>4VUuPPLjp;hldBUSe+kPEHEdWKf;?nVlCKZ7>#Jkk z*AN#pk7&wrIMnWeYhOlx4LKJ`9?kiob}K}2V46IErP9qHX8C|G!&yV7C|~R#b{{Oz!tm= zA*W@+f7Ike#`_5*Tm zD(tgK$0en!Xf{Egz*rkd#tI!(Q6fIKHODMv<`|Z4Sr+cWNe@0~;}I29mZHprSr@)}b~Ufjz_g}2K)Vz|<11TmGxxeQqzCu?(WQNB0Xz9QetRjp)~R(eWh*2Hq- zf4lv(wn|&BsnU#eTem4Z@w7F11y4uYf~TN#2ow*L8atby#r-eHQGmR_v2GA{^)V{} z_4Y=6WIF#oMxN&mkZV7q-UJD4bF)WktJ1CzE-tQ5ruYE4*zhcZ5qbc~0UCHl;Cev= zoJt@t1}^pxy1Mu}J-=Mer;Fv~(b@cTMp~b?RJupg18msDcYm994BsdAZ_C6t91q)_ zma#(&3!yd7{2wwg+ZL9vWz}%D+7vrz$0elj0C|pKTWUCf=b!|n1d-&E4Hjo6u{__D zaxNqpC^M_Cmbw_artw;x#$kntZT3{%o9(`!YcC&MHj|$G7CJeKI{?o=(pt;wpR9 z_;fa3UXCvo^?pU|%B|iHEE4#b)PD#a&nAyjBEJHB;&^`G1+HTa21+`&`?YKZb-zBA z7xp_=@&`ZVk-$-c`e3a-vun-dT#Wt$P)h>@lMp2uvjZ4I2n3T+fK!u7ARm9rYQr!P zMfd#*qrENLAteagfwpATK%pru-3e)uh*}RMwNC!N@-qo|mF4K(Idf-Fu|CN<`UC5P zRwc@k6h)wz=0Y7x^s(E-D-=0T6{odUP@*$9w5}csW(d<3-mErYy#N;lkaC2X5*@ua z1Y^gJAgN1&Lj`fKt)xDb_JDs4Wk2))I8W0@EWuMwJ;g0BHWLvINX{mt`L%Ti;0!|z z5|np1OEP@%da%;XHgw9BQwo1JFl!yf$$fHSGB!<gs}@@M#K`}*Fk8Vl8)iUFz$ z&Iq$Wy|z_>uZwWFnmWG?C*BL-Pv@Y|ybX)=eZZp-fb?-k<#X};vfCt}Do^ueoaS+! z@3M?6mLy$$wbPyzq|5dZ)H0000000000fPp!au_ZnMjFa#sBL+Sr H00000nX8|2 diff --git a/summary_results.json b/summary_results.json index f14487974..703bd59c6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2317, + "Performance_Result": 10.2707, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74502, + "Performance_Result": 1.71113, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.417437, + "Performance_Result": 0.417899, "Performance_Units": "Samples/s" }, { From d9fb5dc904b7bf39b676aa124f114867a1d6160a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 22 Feb 2025 13:19:46 +0000 Subject: [PATCH 158/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9753 -> 9753 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f5e9cbe29..ac7f86d06 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 12:42:29,726 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 12:42:29,726 submission_checker1.py:3270 INFO] --- -[2025-02-22 12:42:29,726 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 13:19:41,066 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 13:19:41,066 submission_checker1.py:3270 INFO] --- +[2025-02-22 13:19:41,066 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 12:42:29,726 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 13:19:41,066 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 12:42:29,726 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 13:19:41,066 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 12:42:29,726 submission_checker1.py:3286 INFO] --- -[2025-02-22 12:42:29,726 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 12:42:29,726 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 12:42:29,726 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 12:42:29,726 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 12:42:29,726 submission_checker1.py:3307 INFO] --- -[2025-02-22 12:42:29,726 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 12:42:30,913 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 13:19:41,066 submission_checker1.py:3286 INFO] --- +[2025-02-22 13:19:41,066 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 13:19:41,066 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 13:19:41,066 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 13:19:41,066 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 13:19:41,066 submission_checker1.py:3307 INFO] --- +[2025-02-22 13:19:41,066 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 13:19:42,016 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.36853 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71113 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71671 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 34d96ef63..97e9e0c3e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -785 +786 diff --git a/summary.csv b/summary.csv index a64de4497..afa06ef87 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.36853,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71113,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71671,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4a3ae7a17..5a217b4e6 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71113, + "Result":1.71671, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 7066449f0c20089cb0f50850828a42406653cc2b..e9e80191e4e90d5c91ba652647a494e03323a8c5 100644 GIT binary patch delta 2439 zcmV;233&FIOqoouSqKG|{E|0ElU@iVe_eB;*cOKG&#$1UzUZzRk&S$&qEkuwD~6bK zI?2?zvIuUK8WjgHAOC(pjnTDU3DeUvq|UiXbn~vY_W~aG23|e<^>I2;?q!lj@vN;1 zO;;5;3*$jF8@1J&>(32WRnjb&4T4ELlWp}$rs}Vq{~SKV$?xe{%1p@<%+j_xf6lV` zL9>~LV>t~{EuPC+zGWCE(;&;=CZlFLPh>Dy22CeTL)Y!*G>B&ED#JliB}Y6QMxp!| zhl{D4Wve8KoCH}u{B#`6Q?+wAi1KYkJxU^nZS_bTocv}8wb?mb_I(@4hji<;Qj~HR z$G;bEP6usOFQiST{k$B;Wugq^e=t}~va9&vtBgkDEZ=Qb5fa9ebop1AM)|%pRAm}G zw$+xZJVb+RTqIV;(O@8p9jz+iBF*CIZ>vpWonRFz>~*Zr#U|R*gsIzxRUD6}iCo7t zSG_Jr_%XGKUCA|&*t?P&AaQmjw?N`< zBs=A{aJem(K#+2|f{gR~tBUJa6*sRcZp#!qm72Ixlf@gPQm!yPQtIoH>(?bWTtbg) zX_B34Em-Z5#S*x?T^w@-e;MWpR|vwwM<5~R&nF-uJN`3}5b(bM34!h_kPzrjfrOl9 zT_7O`=L|^5&h>$W?A>!NG05KS0SVc=7eGSx?j?|rz55MFOnkc8_;j;b0*e%L%q2G2 z_a|J^BK3X*5>oFcAR+aB<`S1=`SL6QABkT<59AyFZ+o7if6i!w&pYtQ?mVY4 zJsv~noC}ba9D>W2aSuCR|9koA%0KO$y)=w9A(*yKz-;SvTNkWlD=ghB@-PhaNdyu~BA2-I!pAEi5rmVI_Ov9t z>k&@QXoJr?VCzsC(*rSd9SS7m5R<+POn;3V5?PB6I`K%Ja~9PDQS>(B1<(kc9FWVc z@xKiqU*^FA#(|Xc{CrGO*}oT>n@@#*8Od5Itia^;dSELk9nEXm_MS;m;2Z5ng5OxT zEJ`KUID%3MZqPF-fre2DG>l5FVRPd~1E~ZF7?mXnx`IARIAbk7=)^N9l{Bh{qLYaY zAAj%5=2wkdM}8Me{HWq(=!D&NWl=p8#fStNc2Q})At3*Wd#|+=Scyq%`--xq z>AJPY%qRBPa0D>fP#+G^IMj#(G=H4Q3RWDXVN3!IW3tSRgAy<%%ltTk8FE;Q4?5vY zGEWYO;wx9K0V~%!eL4RMlyB9gBZy5$IIg$v_Un#MM}MIbxNxXZ2WTAX)Bze!Wd*Me z(l9E4hEZ8&*Fgywm1Twf!+!&SeE357IC$frfEeW`E^D2^g1UW*))ZJgmhBop3Ihp$A0qMJ(5VMQqKco_`0- zH)`$qF1Ecr`9jvzg}rCSG;~a+!JQ&Db@u>`OWi#{!^lJ$N+!}!GLeRoX_UEpPy$M3 zS%R*?+&!$tN1f0IBP^;1q8OP#!^t#icK5tS=HGoXQcHbRpfvPdD1SxqE#f^;iak(r zqlit-JwW49a}UsPC@YwIkcL4C#$!_`%gjBf69#3OxyNAU9@gT6PCS!B$;>?{ia}Yy z++);i?s<*LY*^qZe(0*QmfEVUUX2}7kFI&9NzDVzt?#!_WipZ{axzJkFwS?^Dst1h z`qKZEIJjMXKmJ9Y_mdzIDFKg@N)b~5Uz45@BnvLt{B}depMR+379dZH2%MO54@d)r#XKe7#J-X14u(^l$<<9cSlon5({?^Toh zg7ebo3Xv>F321JF)Y}SutQP45N_=2iu*PVpPzN5(>zk5W%8i3JXDqlN!6y-|AS zyAUkJ*z+AI=CkO~QQR1(Scs+DV$1k9+X61~{2nU^e@w8za1TuFL_`Oq;1kmRxMlzd zj-desI(SSnf=}K6PWh`1gL39nGISQMT94x7-g`M2+qTWx*--lIGsryrL zKy@iN2zdlYU%Dqg|D4w-d zq3Nn3XJI^uW}~)xbN#vDs!Ey#vq3P4XR@t6$yEKd^Pj_qIQcytOPMKof?3*Df5%xi zKWH}7a4e@ms>O3T%Rd>$$u!9Fx5=oP&J!67mO;}=)6jLhISrzjy2@~nRLKz!hfyd$ z#^GWrXW1%AA}2wX4?i78^Hl8|4x;?Cq8=rY!?t=P4o-eEgxc&JF8jWXZQPsu$8G(|%qK<1$eOe{vWsCfQZ|@Kr{mah4x8s|X3>NxJ-3nMV1sG*o38 zJhs)Asysx4Y+NK(#?fFPivz7H;UdlA>2Ir##5%z$RM_iSp^HtlrwLQH4XZdGPZPP0 zX|8%*j__lU1)akreo&HS59vH8&Wt!PjQp^KMeDbsxfok>-Ej5XpyYKFe@E}W(>iS4 z7m3%3?=<9^Hr&vL+pUIXKAz2>3_K`Si6#IAhCBPH$dX-N^XI~ z-AH!IZQ*iTEP){9as?UZ_g59yuPSa{Ros>-b}BV-r6!9vNTpn1dZg6XCD*S@Zn%V= z*U}_A)mpIHBa0<)ce^;|e+n|p6Rr@1g^xf&uAfgpLJs_AAR*v?0TKe;S0EwKodOBD z%(_5APR<#Skb~<32|2pwTw;);+XE7EbT5E}9NkMGAxHNckeK*#v+?C-vji3?=9o)t za_mpIq($oe2qdK5Pe4NI{mdmU$@1k{0{$d^1wD{!PgnkF@9h0@HI(Mdp;45L7k641Pn*+8{_iwj>g2EH zG8y)x*-$3=f=%`_nP&ZTaW{?9V(pXm?=4MlW{H%`RaUcDOf}E_lXZL|(|iq+HrI=p z#wK$=oX;EKe9?$!f3wGAG0XDRRAa5`hw*F}jnrS%fy{zvlBypLn?o>Pf2*UUeAO!R zp)LR0Gp_Heaf#I$Lo*G-HXIC{gQ0UcbmUSy<`SD+sV7{*paT*L9gtAyfQ0-B|KgND z$HQF;T}gNsgpXWQYVJ|7I8{LMdiD3LsduHP=W#3PbR4W~vTN8^4F7WXra{PvRv z3@-=k=a@@ua{ZG%3_(6_dpt~{$4Mkt|1p=?^u~uv7>PhaNdyu~BA2-I#>Xom5rmVA z_Ov9t>k&@QXoJr?VCzsC(*rSd9SS7m6qCFROn(nIB(fGCbmEab=ParRqUdAB3!o7? zIU$!@<9{1KzRZIKi~}j>`SqBjvVSi$H=hdsGLp4aSb@pw^}tq8I-1wA?LCvCzz^Dw z1b?t@S(HkyaRj9jJfLS(0u7@QXc(1T!{)|~22u$UFe*zDbOqf?IAbk7=)^N9l{Bh{ zqLYUWAAg_A=68)-XMPt;yw!9h`0kBmQ!Kg0F({T?!?6S!h9%H2EV+hbX_R3J5-==F z5_C5ozFq2}u+2@R?e1N~;yuwL!M7SM zhjPg^PK$EMHJnSJVO# zxyH39lV%x{pcDG2wJbq*^Wh7Vw8aOVcrD5#jq0H&#w6DWo$gL=jlUKq7kBwgxL89; zfPc*Hr_0H*m?ed$89#tz9o1D@y_#D{3e6Eh?3*R`I7|dL4Oku}l4~4`63I23NT6Xv z0u3V)Xc&=J8Id3XBeEnxck|&3k+j7Jov_EQEUJg17?D83ZYr%e1mr*Q?6sBxD=}$p z-%++SUAOj_`NSR@jsPYb>casVhZ=E!hJQ0z!HR=4j7gwjOqRKEPy)tenIA_mLk?^4 zK_{F^=E(t3eC5hDVC7n;FXvx@@~ygb1hMG|$MyC-e%RoFyMH2Y z)U$IfwoN;>X0+^mmQ~^I!4i0KsBH&m9O~Nv8jfWJ=MK^^EP;k$S!Uir2^f}T_8mdp zHD|2F2c7W8ugs-`qS#~!G;9G|vr*?YDu2%7*Eee6xf0tRo_rB2Ja^wB*4x7+aOhAI z570Q&#RD{)%L+aoq+whF4db%R%723rFfPl?Jc7A-Sc?xj;aoC94~XK6Sgrw!*qTi} z{|=aM)Y|i1YxLtie{zabm7n2|nC;`utNfA>4-;{OThg7ebo8WmZN642ZTsoNU8ub1fqN_=2iu+C_x(H=ZnG&dEulp6=H&RB3kf=?ny zd&+H%wjo%GvFBS*%xBS|qqs3nu@FnwV$1k9YXOryzsCv!e-kV)JOEQW5z!GT_=L1S zt{VY@W9UGE4jva-fzRFmPWjn}QMqs`8TJ-tttWAE@4TFhecxw&Ia*`>0)MVv-o{l^ zsfSZ>Kusk$Nw=z#1r6<`}z|!f3u4r$^r$RX%4+elUgKO0m+lL zBs>CB6O(Qe7Ly7kDgot_H6=O$SCeohCL0gCJ3k}=002+|000pH0000000000004l2 IIFqp@Xz`7J?*IS* diff --git a/summary_results.json b/summary_results.json index 703bd59c6..d921ee093 100644 --- a/summary_results.json +++ b/summary_results.json @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71113, + "Performance_Result": 1.71671, "Performance_Units": "Samples/s" }, { From 2c4bbffc9b61379f95afbb7d09dc423ee3931500 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 22 Feb 2025 13:23:41 +0000 Subject: [PATCH 159/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9753 -> 9755 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ac7f86d06..3d4f8ee8c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 13:19:41,066 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 13:19:41,066 submission_checker1.py:3270 INFO] --- -[2025-02-22 13:19:41,066 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 13:23:36,498 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 13:23:36,498 submission_checker1.py:3270 INFO] --- +[2025-02-22 13:23:36,498 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 13:19:41,066 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 13:23:36,498 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 13:19:41,066 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 13:23:36,498 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 13:19:41,066 submission_checker1.py:3286 INFO] --- -[2025-02-22 13:19:41,066 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 13:19:41,066 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 13:19:41,066 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 13:19:41,066 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 13:19:41,066 submission_checker1.py:3307 INFO] --- -[2025-02-22 13:19:41,066 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 13:19:42,016 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 13:23:36,498 submission_checker1.py:3286 INFO] --- +[2025-02-22 13:23:36,498 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 13:23:36,498 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 13:23:36,498 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 13:23:36,498 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 13:23:36,498 submission_checker1.py:3307 INFO] --- +[2025-02-22 13:23:36,498 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 13:23:37,599 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2707 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.92245 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.347301 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422627 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420243 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.23624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 97e9e0c3e..e791c500d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -786 +787 diff --git a/summary.csv b/summary.csv index afa06ef87..a7416115b 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2707,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.92245,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.347301,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422627,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420243,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.23624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5a217b4e6..c6402cc98 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2707, - "Accuracy":"acc: 76.200", + "Result":9.92245, + "Accuracy":"acc: 74.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422627, + "Result":0.420243, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index e9e80191e4e90d5c91ba652647a494e03323a8c5..d305d94447d18599d194c722f3f30f3dd018263a 100644 GIT binary patch delta 3483 zcmV;M4P^3}Oq)!wSqKCDI+2rH2vh>~E|and#(&U$UJm0jQ3i4tEGF4i{P0yqqj8q+ zHme8;<4L;wt4yPOUmB`14IbNSOI04CK{hTDE8}P|kj0Kxm2i<}@$|RVCb3Si3KjM` zR_J0A?P`HDKiMx^Pl-uIVZ9xJ-%0&ek=l53?*RLvWURB(dDRwF~`ASWQ zH%O(ZFg;T0>yqo&B{wLc$F(%cPPGPB`(SGWcFw6=h)=^3Y5^tO>5=}>I zx%wUuT^>XiK_utTF-mN5{+yr$Kx8BoL`FhEWF!m3fA*TRj`Au1c;#&Y=xUy4?U@LL&hxiyCjgSU<(SbpWE>}q27eIbG?YP3Lm^}| zE}aiI8V*A)0p+kPL3gupLn5^JOeY@6a}H5GCW_u>ykIm!CkNzmYy58m$d`GrfN>z@ zJU<_kRQB(M=H^r3Uq-T)3M(*qy&l*KN=Nfrw!LRk6#R|$BY`*8EsIi#8b?qnnH%&# zC8Gf<84Xa08kie58h@5bCIL`clAtT-ql7bP@tICMgHlPOdR!EpBRLJ7B)5kD7NOjl zAqy%ET@ycmWgRtEnJU)<6YKl3`BmfAk>ACVf7Em&@ac_ZQ!G*A7!*s?AeM{;1(*Q5xSsU8h`%>T-H%zmD{T8fo(1# zZFlb?mVYLCB=AwAY;ud|P&1f8x zGKm^wlFT98()Z|Eq^}K3EXyts2&#uL^2wJJ({{v}0>V%id>M749A^8BY$i?JydL`gRx%Vp+kt!)X9ZMgv%u znRmDZfMuC|M^Jao8MOFJCwTi6TsmA7OqPrW7Jslc8+Bfz^5@)seWMnhE3xh2$rrK0 zbN4M`y**qq4jpRZVKfeP@h}?XvVxC?(*T!@2DmJ<@^A@&%Q7>M05=b`_)I6rB^Y{` zC|ty%hFQecZ0h-Uz-r+rM@aq z8u~7jqWBi^9w@~gDA6clQ*#faajCh7(IAu+%sre2pk&5lQz*;KJzOUMWtq9h0CNwt z_)I6BNudOD4;KYcRxtM%HJf{0V=@~SIDd*Cx~i&V0TGEbs7iL$3v@5N6PEG9u`8V}wt2V}LF zMSm>h`*-0x``}$@fA~c)J}7M^Fa24XSGVk?c)<;;b2tx1@;pdJ(JWOaa+nvYYxxCu zvii+|#Jkk*AN#pk7&wrIMnWe zYhOlx4LKJ`9?kiob}K}2V46IErP9qHX8C|G!&yV7C|~R#b{{Oz!tm=A*W@+f7Ike#`_5*TmD(tgK$0en!Xf{Eg zz*rkd#tI!(Q6fIKHODMv z<`|Z4Sr+cWNe@0~;}I29mZHprSr@)}b~Ufjz_ zg}2K)Vz|<11TmGxxeQqzCu?(WQNB0Xz9QetRjp)~R(eWh*2Hq-f4lv(wn|&BsnU#e zTem4Z@w7F11y4uYf~TN#2ow*L8atby#r-eHQGmR_v2GA{^)V{}_4Y=6WIF#oMxN&m zkZV7q-UJD4bF)WktJ1CzE-tQ5ruYE4*zhcZ5qbc~0UCHl;Cev=oJt@t1}^pxy1Mu} zJ-=Mer;Fv~(b@cTe_EfmRJupg18msDcYm994BsdAZ_C6t91q)_ma#(&3!yd7{2wwg z+ZL9vWz}%D+7vrz$0elj0C~0{-w!1mz;jRnQi4cw+96m$DLUF~U5I}=VEO#X;?koe zj;s2a)HmvDM6)7=j@jRg4Z8<-dT^y08{F%`{T^KLWs74aGJp-a$6wszFYfUdEB->XhW7kwrjeed;TBnjMe#7USe|c_b`~Ok8Q&f5uZ&dPi;hxa zFD3i)C5ZqEUeJ=m@(vrm`k3_e^8aJoajT{ymVrOky%!O_uJWE?+`kdLmh6vbw8bRu zvIqZa$8cbfH%3z3IX<#1-^})gIDUK^JV*Ebeef=f?;Fz(?1TSf6Hpa zFc3xe{R*SKEnB7!gzZ3EvTC5vl$P#2oCLh;a&+&UxihF*>{OTh zg7ebo8WmZN642ZTsoNU8ub1fqN_=2iu+C_x(H=ZnG&dEulp6=H&RB3kf=?nyd&+H% zwjo%GvFBS*%xBS|qqs3nu@Fnwe`3q{H){ctJio&V0uwASJOEQW5z!GT_=L1St{VY@ zW9UGE4jva-fzRFmPWjn}QMqs`8TJ-tttWAE@4TFhecxw&Ia*`>0)MVv-o{l^sfSZ> zKusk$KUX!v2#($>$yd1`5q739PSWL33_~EOJM&m5s zZB`Ky#*=jUSD8lnzBE*28a%euma05NgKS(RR>sj_Ad4NXD&Zo{;^}XzO=6v36)Nm? ztkA_K+S7!o+lEyfkEe-T$23>HE=Tw=$b!yc5Gat`pPzD~9!N#EPj^TsJ*wuPScK6g!ogxKfkF z8>CXMFg;T0>yqo&B{y6`k85d?ooX#u?SGNQ61clv9CHO3<_T8_!oo)&A?MF0AR#;c zGmsGQzW@n=?kkWG=uUxzoMv4hAqVFSNXX9hfrRYcb1pH+-t7Si*}E4&LiX+@kdVFm z4MR%a?HvJ752M`RU3(?VY_}u7=WlIW&s0 z@#0Pk<7sm`$^TCCrB428E|XzDnhj-=FW6*1lWEpZ7kASrE!IA1|K8H{W|l~~TxB(j z#Z>d$KUv2oGR@a8X>+}pX>2n0!+-g_5zZHlcs6@X7PBm0O*Pi4ei+Y&(MbJ89mp(* zCaL=2usHtUzKNrBopC#@Qm#bhMVM zw=jq<2GQjplJn=7OKfufoNx&P5lASAKte$T5(=VQ0wND*AeMx8K{y4`Eddb>o`UF7 z5P6JAVlKGEBa%z^e+R@mN`I*U;;p+gCAKWr5_>LtJWQg;NhD|gF_+l%!iP&3i9kY0 z1QJRjm$>x8$15Qbgp-r@v?RRi5l+r%gU>r)>rfig12J?R3MAwZTzd7wHVwi!ZA^k? zhDX=_*=y1|%Buk2mA3_;t9hQaXCf3j=5N_f03>tEG0-?<94A1-5Pw7(${^BE2!Y0> z^WjE=FhmI`hh+)6n~fV1S&I)k@kpL?7S#h$^fu!K&2Ao=H*Q8|_De-&nUSN+s7gf>H@?&@(E5 zhEWMLj7qLybK^z>sec3s7?mXnx`IARIAbk7=)^N9l{Bh{qUaonG<1^O8va{^a%+Yx zs5Ep<`~a49)L3PzTn|jF@5|;_jax^47fbx8=}7RWHVYVB3l(UDE-07A|9=6Ob<|kpw(5Fdn~O-> z-Mfgz&qR*|f7EC>luNF0T9iw!;ama@;}U2Xmt4aav8^&LK?25QNrLWX;|rIx#YdIU zyEiPV2ckHa;6XO$vh}99BsX*ZI+AsiR|(11E|R9x+B!D7OyZ~aM}j{Qx9AgXpm9vf zB-d~zfrc>&G=Gdqu5m5Oq*=x!=!D*CElbedYT98()Z|Eq^}fgxz*!Q9TsJhy)sTQE9y) zApePbueB6diAihwin68Yy0yp5C-&HI1Tfi99}dts)QAH#oXHAS9He1P0u5ub%#DK* zFec0VID#2+Sc?xj;Y>154v69_SFQmo*E)SU{|c0E)ukhdO-DGcx9|4rj!#E_p%S=o zs8I)K9DnN60UAza1+Na$Fe-tDQCVi!K?xX@WriI={WNE+#Rr}6ha$|E1ETmt5v~Cb zMQZlua4LWPQ;|37*|`?mrX5=|TJ}E6s&MyU2|PK}wgWT{_3Z!+$FhQR2Wc3VK*O*s zGw+}T49hb6j-c+EGuGmRPWbIt=F&k?Y_bFzwts-F*{Jgxl|Se9>l?N3T#0QDPris1 zp1W@m>+RtZICQ9q2WTAX;sF}YWd$D((l9Q8hH+VDFt*3^Z)XT~&iOs2t|A~tpR0F6uCJwU_AL>fva z(tl7gk%p3Kl(~CQ0!n6Cg08{bJ*>q?ozMp(EUE{h7@0uB$uw$q_q;~t-+eMtOMO+K zH1u64Me!}-Jy420P;#S)P0c+(<5F`E&~PX#n0t_hK?%lVQz*;KJ*X1~Wtq9hVCEjy z;)6~+lS0YNJt&GnS;5?6)NJl~jmd0S;D0E7=&G`o+N!KxjU7{uu6d?O%>&J?@3&87 zGLk29GD(#%&Ue=;a?`r{(*Kn>xLtie{zabm7taP!;ph)0t8epX(QKrof2@8dz%G6# zKrfPo!>h$aD%sOqwpIB!&yP@1tf~wihpS%*Q07S#CsFpa>b>}hg2g20Oyj}(~#nb0^V;Kx&QoL)a$}o@pm2=&lC7m&5la~WRc&hs)4f%9CkU;M2w zh9RU-QlTnSe?|E+kHt?~DzQ7yg?fo=4pC?FC1q98P;qbqLVB;*8A}^&%rPw8vMk(# zlOBA~#v>}IEJc|Kvo3t|>~x%GSuX2feY>QgkZq-x5JM@u3iPn3^x|GdD!g6Z5yO>6 zBZ#Ri&SjXhI9Z!}i}Jn6_7(YFu4*N_w9->5vnG}sf8Xt=wN=`3O_gS(+qzBRiKng6 zD|kBE7CZ%|L!fw|)Y#bsE$)8-Ljm#v$GSn-)yH`WsJA!jBh&f!G4ecjfL!|-^(IJY zo0~mSTa|W&aB*>cGQ|hT#fE1QjL-u>4$#0e0@n)~;8X&EF>tYm(ACA)>G|byK3yy? zkIv?&f7AN3rP4i`9$>>JzWdv>WB5L?e_JNL;dt2Yw2U2MSO~3o=Kqk1*|xBREvtsJ z)uz}%J1!xG2gq{``F<$j0M0-ONC_fgv_r6hQgpP}x)A?#!1DQ%#id6{99Q)-sc+QR zh-O6!9kag~8+H%w^x#T0Hn`V=`#reg%NEB;vRo-kH5Ib zU)aECYY6doLnXp2eQWe@(IlQS2RACLRBrS8|q^1^<{O8(%d zJQ6rcP#>(-XLhZboQu(av*s6J2n6i@jOmk0ARm9tPQx$|gzr2>c1q4=-N`QKz z5)wsJ+$?+BTEsuH4QbvUCu!4G>WSleX1|?Xxt#A+ll+47(&!42EJq1wZiLj^3Vo~= z=>tl9U|O)oXsA#J9?k2Ul3U7+gEwa^xFEqN5u`okwnDoQEXCOK9Vq6r=+IHz7^hf> zrQ3gE%lJ3j0xt6W9xDh;u)uH+OzlKO2c+N=(*C$+00@qu0R=jEOfrH`-T+Sds||y4 z=2S9t7Oq;4;^f|YIT_ow&Dz;ujrj>)FJIq>Ra2?^Q*l6bDLCa0STL?G@p%ys*DxQJ z@x&Vm;;D=JmAAtpV;_hl2B2fyqw+a>epw|gP+jE3U78nZQLM;}l1F+=zWR|D{U@M0 z4EZ?~cPyzq|5dZ)H0000000000fPpuYvn4(Oi<9ytBL+Pq000003}AZi diff --git a/summary_results.json b/summary_results.json index d921ee093..021559cf8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 74.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2707, + "Performance_Result": 9.92245, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.422627, + "Performance_Result": 0.420243, "Performance_Units": "Samples/s" }, { From 29dff7fad5809ad2090dff96a24446f37b9cd12a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 22 Feb 2025 13:24:25 +0000 Subject: [PATCH 160/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9755 -> 9754 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3d4f8ee8c..42e8f1ef9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 13:23:36,498 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 13:23:36,498 submission_checker1.py:3270 INFO] --- -[2025-02-22 13:23:36,498 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 13:24:20,470 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 13:24:20,470 submission_checker1.py:3270 INFO] --- +[2025-02-22 13:24:20,470 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 13:23:36,498 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 13:24:20,470 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 13:23:36,498 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 13:24:20,470 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 13:23:36,498 submission_checker1.py:3286 INFO] --- -[2025-02-22 13:23:36,498 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 13:23:36,498 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 13:23:36,498 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 13:23:36,498 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 13:23:36,498 submission_checker1.py:3307 INFO] --- -[2025-02-22 13:23:36,498 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 13:23:37,599 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 13:24:20,470 submission_checker1.py:3286 INFO] --- +[2025-02-22 13:24:20,470 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 13:24:20,470 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 13:24:20,470 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 13:24:20,470 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 13:24:20,470 submission_checker1.py:3307 INFO] --- +[2025-02-22 13:24:20,470 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 13:24:21,415 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index e791c500d..cbdc9073b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -787 +788 diff --git a/summary.xlsx b/summary.xlsx index d305d94447d18599d194c722f3f30f3dd018263a..87fa22c9bd99cc6bd1e32f8acac6d3c6320ca33d 100644 GIT binary patch delta 138 zcmbR3Gs|a#y8`Q?MQfg@O!iTbGdk;c*g&A=y^ZUW>PJTyTh^#dW%%l%s?z0fN2hvr z)DinlyqWjq)N?O~o$7pCm9~Fw?9Rz^6v7yfPyVb>C9ikZ)9b`84b7cPRsKu4#eJRM p_{!qR{UgE;Cj9;-Y*N3uSCN|qYS&!V5XPR#pH!9E?35XR00204J%RuL delta 139 zcmbQ`Guvl_y8`Qsd--QnC;KSK8J*5LY#`wJ{V&(zy}HrM_*>pOL~qJkaAC@{cP%D` zE^FB*MXA@v-}IcrvgcJ%$@@7wjaN*bqY%b;c=Bh3Dh0E%o?a(*X=rL@R{l5b+4IVt q`D)>b{hjIu6u*B}&-ll%xkr(k1!~zG)ey$s$sbjf+3b}WfB*odRz0Qw From 848498b385673d3f47055fd46ead947ba2dacacc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 22 Feb 2025 20:40:05 +0000 Subject: [PATCH 161/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9754 -> 9758 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 42e8f1ef9..62a390853 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 13:24:20,470 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 13:24:20,470 submission_checker1.py:3270 INFO] --- -[2025-02-22 13:24:20,470 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 20:39:59,919 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 20:39:59,919 submission_checker1.py:3270 INFO] --- +[2025-02-22 20:39:59,919 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 13:24:20,470 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 20:39:59,919 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 13:24:20,470 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 20:39:59,919 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 13:24:20,470 submission_checker1.py:3286 INFO] --- -[2025-02-22 13:24:20,470 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 13:24:20,470 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 13:24:20,470 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 13:24:20,470 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 13:24:20,470 submission_checker1.py:3307 INFO] --- -[2025-02-22 13:24:20,470 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 13:24:21,415 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 20:39:59,919 submission_checker1.py:3286 INFO] --- +[2025-02-22 20:39:59,919 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 20:39:59,919 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 20:39:59,920 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 20:39:59,920 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 20:39:59,920 submission_checker1.py:3307 INFO] --- +[2025-02-22 20:39:59,920 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 20:40:01,023 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.92245 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.347301 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.437017 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420243 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index cbdc9073b..0c2b7810b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -788 +789 diff --git a/summary.csv b/summary.csv index a7416115b..2a462804d 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.92245,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.347301,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.437017,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420243,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index c6402cc98..39c676dd0 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.347301, + "Result":0.437017, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 87fa22c9bd99cc6bd1e32f8acac6d3c6320ca33d..8f0f157a144a63c5dd5d4fdc0ae312a312e204f4 100644 GIT binary patch delta 2731 zcmV;c3RLx)OrA`zSqKIDW2zdlYU%Dqg|D4w-d zq3Nn3XJI^uW}~)xbN#vDs!Ey#vq3P4XR@t6$yEKd^Pj_qIQcytOPMKof?3*Df5%xi zKWH}7a4e@ms>O3T%a07>WEy1o+ho*C=ZOpk%b@9`Y3RD$oCeWMU1c~(s^o}=!zh#= z<8U#Rvuu?lk&__Hho6q4d8&2}2T^{ks7FcUu&o}6gOlG3p*A~*%f4?T`H*hCR*F*Y z;`sOC&FP@6>V>q)w4ay5xJ;CRP8GKUCA|&*t?P&AaQmjw?N`e-j5(3>yO%&h&h9rLG4bVQ@1pG<-3VI;d$Z1J<_xafK6m>=$eBOaaPUks|>G2pk=UjlacD9x8cqbM6M?zAwTHm8&P-)X+o$zRQ7GVDjQp-l1xo9t&Y&HCx$ZW^V< z+9&PbTbka?5-FFftY)#8YM%RlC+qk`ruiBsZLSwHjZNl$IG;Dd`JxfeW{=5YmgTFd z##+@6fT6Ho1OIxP*ZSBoss-p&$YY1<@@5k%u!7 zOTxP#oPy|#BTb65yJ-0m`Ceh<0 zlB@riOKf`M!zGMFAfY4z2_=zBTzccO+$whlw65jO)Cug+5=N+(hD2?fX7`hGx z67msTdiBCK4Z=8WOoC;GN7w$@YtlN(s{r7Yw*{c9d7iarA{07*=5N_f03>tEG0-?< z94A1-5JVcvAkt6>fySlt;YNcnLSNS<>R)dNxVG2;c$2%UT& zms{h18$iCyg9VHODd+k1n543QFElrw3jZ>awNzMv$?NsNR!};c*Rt(BlcK;6+K&W( zux?qDO0IDPr4l@Upl4J94Wklh7?oVZ=EjW%QV9|;DoYY{1>H(GV=X@D#4{+BG^&T9 z=p2bObduZ}{#%4{YlbYSG;~e;0G4&sSY@hQ4@|6|%jS2DTc7+cmUye_Nbubo%cfXz zjbl(OxrSp2Gz?3iVOVkv$I>Xn5+q<)mL%wI4!&SXTYS)e3CEH~^*|JRgbFl57nDom z|A5OnYOL~Dbv>}nO{DGaUBu!&(IdgP8ZC!%$u&-ka>+HEOQ2y~0uAGmYxp9zRmLSq zz_=_)(A^w-;gYuas1o}0hDG&26z39bWOFWCZ<0YhxJ7rgfyOZ@lU&1@1RBO9&@d*s#pZMJXd1d!;>#!h3D>D#Cm(U1P&c);sF|mx_E$wb6LU1gEWjwpkZ8=S$R+b z#$}nAM=&=JYw-vRTDT6?~WZEsJ$kTrE-@0l?T9g}H) zaHoh(-913#Qg;u~Ffx&bl8H2wOr)V?8fEStlz@_1mY{1ecMogvQ73d`ghlm06eAO8 zIGIMx?w;4k{JT3NwbWMyN<-g;QWW1J-UFrB10^?#*wowuG%hvw01bz-g1HB27?fZ< zHifdx+=Dt{P?ni{3})_OEk5YPGbxmR%-n;b7?c&vJx0yup4XVnh6Rq|hwdtCsjbTD z)z~rh=$dD;?_@Q%zTZBT$w;2a$s|?6I6qyh$W80&OaE8m;CA)>_!oKJUpyN`g`+>1 ztiH{kMYEBT{;~R<0K52|0KG^S4zCsysbo)c*;eJ_JpY7>VpV1EI9&ZgfHF^$RS_~l zfriyNoChO$9weh^mMRlD%nQ}E{02N({pP^(br#Q;uPjx$i?b}AKEE5wU?7v?T}xGl zahyHBDNf1ew;MWt0<*{we+~qEg|!!xOCTzLV#fu9tR$5H^+Y8kia^{fd)r#XKe7#J z-X14y(^l$<<9cSlon5(_?{u5}f)C1?8kGe>85m)uGMgH`uNL_O%0lEu^41!tQ4ayl zo102FCai~7ZyorkAY=(-0u!!A+ZY|gIEXE19*X2JNnBg6c}%6>V8_Kb-vFkB++z)Y zkxL#q9)P)CE6O(Qe7LyAlDgo+~HYGX%T$6DnCL4T(wHG7-002+| l000pH0000000000004l2J(IE}J^_%E@g*Y$LL&eG005hF67v86 delta 2708 zcmV;F3TySAOqxuvSqKIFI*~U=lU@iVe_eB;*cOKG&#$1UzUZzRk&S$&qEkuwD~6bK zI?2?zvIuUK8WjgHAOC(pjnVZk3DeUvq|UiXbn~vY_W~aG23|e<^>I2;?q!lj@vN;1 zO;;5;3*$jF8@1J&>(32WRnjb&4T4ELlWp}$rs}Vq{~SKV$?xe{%1p@<%+j_xf6lV` zL9>~LV>t~{EuPC+zGWCE(;&;=CZlFLPh>Dy22CeTL)Y!*G>B&ED#JliB}Y6QMxp!| zhl{D4Wve8KoCH}u{B#`6Q?+wAi1KYkJxU^nZS_bTocv}8wb?mb_I(@4hji<;Qj~HR z$G;bEP6usOFQiS-eqIjaGEoL{O&BaD*;V}TRYs$6mhU#J2npj!y8Nq5qkLZ)sxl28 z+iFWy9-=`uE)px_XfTk)j#ia$k!JDqx78-GPOu6U_BvMRViWCY!qjcUDvp!S2^oJQ zBMuBB-z{O$`mJa##@1XnTs=1^c^$>xyYI9PoA*WHwc3s4zWJ>g#`!>(?bWD51x-G|5i27F+F+1qs~UE{;(_hIxVt zL0I_6NXYr~iII>U|Cx~x@V_t;0^L_eLZCZkB;+*fG7@rd&KL>VxjrKyd-ohA2HCqk zMnd-P1tTGQ_mYv2z59)knEdHx^QW5)2`o~~F-mN*?@v(DBK3Y`B&6O?jD&yG`xzxJ z$@1k{!h9rtWqKgz$Z1J<_j%j%6m>=$eC)s@yYrmJ^e~3bITuV?atJP8#y#wO{qN@iu)vV1kw zSgZPBJR3$M^%r#@vmlzJ>W9PTkeRQ))lpKuYL)rWmjCvQ>-%b4VztK5OhY%!77v}n zL+2oL*Uo&u0ct zKJ(8VN+ib6_1gtXJR-StY9w+r?gwFUJ=4f9UskCaqnyY2Qx+%TxPNbGwx&12g)qzt zB-T+%1rl$ZEfP&fYq|O!5M3Tb7eOTF&oN4Da{ip41VCgY6huZsL1ZKpM7IP)3*E>PkT$))?h17aPeQ~>eT-I)?wmTQSUmpvX&qK71s zv;P<+Hofpc36RK0D2a@Ol86$QUif$=Br@UTq&+PO?|OujGuq%|2e1yMF+C=Ru0t6K zIRuwpy|7J#FisnjV4303wSV@Sw2tyB0C?qX0qAO;XYH8?h0cHTw`?Z>lDXxW(Kuur zCyWLVxDw%&9^gtz}0V){{P>C9t8#fx3N+tnNS(2bD=%a))Xz`g&JcCk6qk3Ev zog+C7og}x0{}!R#njs4+4P6sIfMp#uR+%c-0~71}viVix){)=El7G~6B=G5tWm7Cs z;}{f6)F76O2C!r_fF)`WOQQ@+CIMhslAycU_<|*E@tJ>45K9`>W1`>|Dx(p)pj;aN z2VB-sW0l*g>w#@9B5il?B9?z9dL-~sqvcR8QRB2Im#9H584YmBXn;%9;3BqF#wC*g zxGYJ~-E4f}lD7C^`HU$wC*pq=eWJ~19FsDM8f22u0F#Ucm_&_hQ6|kYCYes?t=6&x-Oa`qCTWY$bmFxr zlQgQwMFEqj5juT3y*2(?m|WcDGvQ(lB>^(KpDriMVwM!1X8Zt>byQbr^=fV*DKtme z&ORddI7|eZ1}u*fi5kbEM4|?XWHdk|qX8lr4G@26l@ZA#03u5gbT=Dch@>q((+S*m zg{U4E1w=9$xTv(=5Rm`Gz1La_ti+_XeMQ;Qbluux<`a8tID#?RP#+GXai|f8(IAr* ztT>zom}E4-WSJX>O8`uk`Edjoa-hX$IzcAElfy*e$`v)t%C$~k&c6cXTXpFOV$%_h z>+OHL{kr4R(O;-!TsYLI!)P4p)L}G8Wd*MervWM%4NzHT*WnTXm1Twt zyL(Sa3mtPa;Tr%59CdAy|sB=UY(BXVIafxG_$#5KGr$%lJ2I0h2tx#|nP} z6D%-108={=(Ge;5gtR}d8v%l2=sO2{~AVK^`C+2ION~N6X1~h`V%yNvydUm z0tNj#kvB(^UnE-r$dkJyJONUZ_#{CA=94-lIssUdbtNVQqN1$ilb Date: Sat, 22 Feb 2025 22:17:47 +0000 Subject: [PATCH 162/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9758 -> 9754 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 62a390853..9b6bbc9e1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 20:39:59,919 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 20:39:59,919 submission_checker1.py:3270 INFO] --- -[2025-02-22 20:39:59,919 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-22 22:17:41,835 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-22 22:17:41,835 submission_checker1.py:3270 INFO] --- +[2025-02-22 22:17:41,835 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 20:39:59,919 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-22 22:17:41,836 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 20:39:59,919 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-22 22:17:41,836 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 20:39:59,919 submission_checker1.py:3286 INFO] --- -[2025-02-22 20:39:59,919 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 20:39:59,919 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 20:39:59,920 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 20:39:59,920 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 20:39:59,920 submission_checker1.py:3307 INFO] --- -[2025-02-22 20:39:59,920 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 20:40:01,023 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-22 22:17:41,836 submission_checker1.py:3286 INFO] --- +[2025-02-22 22:17:41,836 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-22 22:17:41,836 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-22 22:17:41,836 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-22 22:17:41,836 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-22 22:17:41,836 submission_checker1.py:3307 INFO] --- +[2025-02-22 22:17:41,836 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-22 22:17:43,029 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.36853 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71671 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69453 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.92245 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.6158 | acc: 73.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.437017 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420243 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0c2b7810b..17973640d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -789 +790 diff --git a/summary.csv b/summary.csv index 2a462804d..cdff8b58a 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.36853,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71671,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69453,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.92245,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.6158,acc: 73.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.437017,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420243,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 39c676dd0..0e313ece2 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71671, + "Result":1.69453, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.92245, - "Accuracy":"acc: 74.200", + "Result":10.6158, + "Accuracy":"acc: 73.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8f0f157a144a63c5dd5d4fdc0ae312a312e204f4..f293d94fcdadfcec47d465225b3d3e6177490fa1 100644 GIT binary patch delta 2983 zcmV;Y3t04?OqxuvSqKHbxQw?*lU@i@0q2vl2*!U2+Rw{jTqepu4ui!cyNVya%4jss z^3!G&Az?g8m;Wl$C_k5ms!W5&w%SsahiH(Ei^R$}8VqD{qE#haq**-uZFNYj6Rbjo zy^a;S*hG7pFm>CoiqGR|BG)m^RjrO0w)Bod?AyBMuBBKP_R=`mJa# z#@2saH(Wh8D0v;l*}Lzw4x9Hy;4L*t?P&M&j&BZW)QYk?fS);>&G80zt|}1sUh}R~6T(?bWD50NgX_B34Ew+E!BMTC^yImZkf(-Kn6@swvk&%$==My6# zC;l@dA>e;uBm}yzjD$dU%1Fp%)@3B*!#QIl_B+Hj)3G*lME7N}i zxkgS)!n@DMo~NiY+Tddc9yy)oG^U3!bk4b8(vpwh@@3q^&e#85e!B8cduQ*LtD!Vs z4vnI0ytvcCc-ov!@_(oKQYU{km&vdn&4x0`7i_Yh$u#Syi@Rx*7Hglhe{X4eGfSjg zuCkiNVyb!WpRD5(ndWPlw7Fi)G&X;k`{8`v2VhrW$KiKa6L?Xr%t4 z4rCTYlT`h1*c>wR^|v}o%2%y2AKLQ2J>&Ym8kbnDF*MWA4YS2V=kU-u2pzf9j!|Nh zEA<2=039Qt&@mDU9U~!s!oN5r&|$bsp(_dRGU4Pe^4WGcsn2HyPyXhgJCuJ&jG^nd z3zT?7a_Q7a9?oQZ+_7kMpN2PQr2j-q37KZ-fhBm=#E@qm&9H-Z)z% znvT|T^*tcEJcurWNUon_l-T6@IY9}4$Ve!NjD&*7NGOPI35XaDAeMx8nQ#iCTLL08 zcnYFRLBtr7#9W}nBa%z^e+Np$I!dVk;;p+gCAKWr5_@iYJf1`kNhDYQF-mND!N(3@9ZF+*OblIzGLyRuCq2*KvYh}( z=9Xhd%cfYO#xW?Cs6i|l4PeP=087*$mPQ$t zOaj2NBtds`@C8fS;xnBfmNcrzM8P9eMk91VxitO{xU8ebDvwpy1KZq0+V0*(EWamu zByg+IawwOmaaxp1)F79P2DoH2z$I#M5!))`l1TtumL%wI4!&?nTYRn(fBN(WqIyge za>;CDBbTi=%_Vu5^VgBAqr6H;wsw#-TaLc9sk+M~zk7cqa7WytJKBuKF)5R%K_(du zFv)0uNz}L&WzsBTlIes#YAs99-5h*jlD7CvCtiy(NuzpP6flVzq0`;zt?}2wj0T8gG(eicYUTZ0^ z5|h^U9c4??b!(5APwcVb2*zYXeK?H9p++1=gG^Si;&2*ZlF(YsPYx4>D_7JoE7v-GIsXcjZ`GwEh)qX0uD9>;>yA%Hf1#3b;ZUOvqj9KH zhtVLF6}&o}2B>5-KxLU-hf4rdmKk;g_0ya|i_dg|npf3o*kR)xCMMxEEF{5g+b->8M>N^E<0@z?+Jd6gptl;C}G{7aJ0WQm|JX`|cvdqjQz|8|Ke?HR*atVeWCJGm^s9_ed zHJf_=9WdXhwdcFo_V(lpSyLDGo*C27c`^-jirCcM!)RRU?qM`QCa0lfavDk|r=esT zW$qp>0VT66LDv9x548APCv;;3qIygekjZF}OrvIZ&ue7<-JOwI>Z<~!q3=Q|if<9` zfl};&5{)7@e>L|o8kd@T7!5*M!Q8`X07_;&Hifdx+{1MOP?ni{3^4aVi_di8nG{Me z_i#}FWd(DOQM0+{H72uRfus1LyUJQ>tFn4Ec1%6G=9%!FtmfAD+ov)a$rCx5q)Hg) zr)w3tXPImKUTjJU>Cm=pcl!);niXy zmF#IQ+p2t==bunftf~wihpS%*Q07S#CsFpa>b>}hg2g20Oyj}(<)9X`=#Pbb|1NxI zAG{0g55FkJ2c@m#r9Vsa>Xy9}FSucK4(Gv0o(IV&nx)D_4)a2FEx!RzR=+v0e4WMf zJ@;#xfYlqA{t1Y;dm!_j_=~mo1K!05*8fIK(~U5G&)b!F&A0J^tbze{qk$xW`|t_zTe*+ViWK zMtYWpTVxp)#lzTQdA=#-Tu3rdW>$Y+Ep;(;P2;sXjl&8P+w7^jH`{$d+4gW`G-P~t zxW6(|buT(fiM^EU&zB?uD0o3j4$C`i`08WQ)64&lZO5&ej#vi%SodB;_`1q_hH?K! z@LIAzp3xSQxXT{=s~y9ELEacib?5lVvV1e!8{+u!ZSWl3`}e`SFure0Kd^rjI@m;9 z6XA~3^+mE#>sVfQTO4)lt{qe0xfS?1c^A!0ejTKr7N5}#u)O~L>FLA{oSU{{l}lgb z7jvY1_I;a8W=H4*j%6kQEv{~mZ3Rjsc8Ws{(qm(wZDZ^?i<-h}t2_=W)C?*`9-}68 z-Fv5+y6k9vfb76<0%b#bTBU!9K^%W)OLq;|b-kTcni!66$zA(odcHiJo=e14_Nwve zY`(l4Uo7hV+Lc?qA6O*tF{%F$I-X4)r9^%O_{8!2zzbZ*8VrS-&eL1CxNWG9Njx- z?hGpDJJlw?;Jh@tLPeIN1T;57>ZU>;t3~>V5+9frtTh@cvkRapCuLnIg_>}J^_rA^(7+)J|h4C001~imY4tl delta 2996 zcmV;l3rqBxOrA`zSqKIDW5Cs>6F zdmSruv5EFHVd}PF6`#k`M6P3+t6rBQ{1{|G=P-#Mlw{dMIuD9ZMjRMMepGKUCA|&*t?P&AaQmjw?N`(?bWTtYwB(j+_8TCjiGBa0<)ce^;|3Np+St`LNUk3d4MpHDzS zPW)#eA>e-j5(3>yO%&h z&h9rLG4bVQ=$eBOaaPUks|>G2pk=UjlacD9x8c zqbM6M?zAwTHm8&P-)X+o$zRQ7GVDjQp-l1xo9t&Y&HCx$ZW^V<+9&PbTbka?5-FFf ztY)#8YM%Qi>-a>b`5Go|t`{?nP3C`oIG;Dd`JxfeW{=5YmgTFd##+@6fT6Ho1OIxP*ZSBoss-p&$YY1<@@5k%u!7OTxP#oPy|#BTb65yJ-0m`Ceh<0lB@riOKf`M!zGMF zAfY4z2_=zBTzccO+$whlw65jO)Cug+5=N+(hD2?fX7`hGxle`QkJmzoNP5>lx z%Q4V6WE>|z!w^Io${^BE2!Y0>^WjE=FhmI`hh+)6n}Zt?S&I)k@kpL?7S#h$^fBWF z(32nyGCaT!+K&W(ux?qDO0IDPr4l@#XH)_WqY`Ksm0ZK-#*GG22@)_WOA>Si-AXuP zEk5YPGboibs)wTJ9Ep>T4JLnhtLaGa-5blMSaOYHP%OEIV+k}2OQ2y`at+7QD8mvY zU|5zU=xz?aU`bni&+GLi*m^|oJ*i#TmlW_l56-PwpGR@NWi!(NzmOKeBqL|_^5vp`t*iH^*|Ko z5^Q91E?aM!OY$)1uOnGUd6kfC?I3Bk9DQq3b(cxJdw(SOj<`j4w1LJkDU)2onFJcf zB+xJ>xyH39lV%x{pcDG2wJbq*bMS>p+Tw#wycT7WM)gn>W0GrxPIsrb#$OAQi@SU# zT&$raKxX&TuztC$UP1d!A%2}M~UPb$D%}X z4JQ(47?D83hy)r&q*X>FNWh3JNzmOKd?Avy_@EQ^*p)@~P!uB)XxL4q^@f1_C!W35 zQeY(}t?fI?mZt009y6cVW5W@^WJ7&8K;uv&4$yEWD_C)mhB1E$G>pkIHx5d`m@M<- z2xiD(Ek5XkGs!$TAd0VCxdyCU>-6RPD^R{wmyRGd9pSj%zQ?aSJ{|pqO5nnwMjfDW zs8a`MIF%K=I!ME)1R6$VnOz4ZU{sbFb_Dg)oUs-kbiy}9m@fxJ@l6q~0h=N18ipm%Ff7Z=J17Ce zvdq3CsJrHjwfLYD{`i%-bWjwVEP;kCU~4w&yhi2EdHniDEj(9Z+ryJDVuk1KTf};M zxC9OzYT^MJhq`!xhI3iL$AdJCOQ2y~mRWgF0>)*TnMZ#xHxFy^K_{F`X6OM?d=bkv zU=drhspsDT^Nm`2zKd;dPri^fbz$$BF%2D)X>g~AP2D{}<5G7I&@eKQhLVXiluV?d zWEy4e9+ZHRS(cz{Fn14Y@lhvqV}wQZKolbrXgHZh&F-Gp$o#uIBem351xiETg;EsX zBHja~*aLqhH;UNQ+ygW&HTM7whq8jX2Wc3TU_3U3vdr9rI$=%!UPy;)m`kYpJcu>ebjW_2`;svhQRyx4z##mB~n+$jKyC z!Z<%&tH@33>P!Du;^21m{rDGo-d{W$M1`Y2n5=)k&7VcHk&^zg`ker~_?-Z~NEQyS z789vtPjlH;<>Ng6goS z0)Mt4-w!1mz;jRnQi4cw+96m$DLUF~U5I}=VEO#X;?koej;s2a)HmvDM6)7=j@jRg z4Z8<-dT^y08{F%`{T^KLWs74afDPU=4sp*o#L75q@E(70kH5IbU)&30c+i|O=BbI?b*1Z=IzOM3~ zVcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8EZ@xbhB$tF8$3t%{(bN+jPD!M4}a{0 z4mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$-bFK$UkB-@#bn)CTY(aZo#Ie~^w=0^+ZcP!qNcFgDvyH-HG@i# z$EZnN_ui?dE<2hZAUiOeK-rLZ=1d|aZ8nXl#LkI+Xg|!!xOCTSA%}&EG z5QOhMMdW*8#|4C}B$WX5L?tAOK-?^Q+gijwvJGk89w%+nR_cl4dS<_!UAda?besKx z56YSvl?6c=7-6L{n;N~Z7Wo6pLgYsB)*7f$4*|`an@Tt)tcO=`9r&mqWC>&f6Rt+v z7#+hnh%IOyisUd!TwAYsOr_s{V8_Kb-vFkB++z)qOCC8MfVrE9=zx@BLb@N{jsPhz zw4lMnfXkx9CvOC=!_|gSIdf_idk0soM{#oRf|`t7*A?Atw5I$e{#?GijjQI$45t!+ zno0^LJn(3JQ{nR>9j@geF4KwE3gly-^eb Date: Sun, 23 Feb 2025 11:46:03 +0000 Subject: [PATCH 163/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9754 -> 9756 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 9b6bbc9e1..357b66b93 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-22 22:17:41,835 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-22 22:17:41,835 submission_checker1.py:3270 INFO] --- -[2025-02-22 22:17:41,835 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 11:45:57,393 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 11:45:57,393 submission_checker1.py:3270 INFO] --- +[2025-02-23 11:45:57,393 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-22 22:17:41,836 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 11:45:57,393 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-22 22:17:41,836 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 11:45:57,393 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-22 22:17:41,836 submission_checker1.py:3286 INFO] --- -[2025-02-22 22:17:41,836 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-22 22:17:41,836 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-22 22:17:41,836 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-22 22:17:41,836 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-22 22:17:41,836 submission_checker1.py:3307 INFO] --- -[2025-02-22 22:17:41,836 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-22 22:17:43,029 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 11:45:57,393 submission_checker1.py:3286 INFO] --- +[2025-02-23 11:45:57,393 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 11:45:57,393 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 11:45:57,393 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 11:45:57,393 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 11:45:57,393 submission_checker1.py:3307 INFO] --- +[2025-02-23 11:45:57,394 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 11:45:58,656 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.6158 | acc: 73.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2146 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.437017 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420243 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422569 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.23624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 17973640d..4ab4c5d79 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -790 +791 diff --git a/summary.csv b/summary.csv index cdff8b58a..7e5f8bd3a 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.6158,acc: 73.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2146,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.437017,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420243,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422569,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.23624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0e313ece2..d85f83032 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.6158, - "Accuracy":"acc: 73.600", + "Result":10.2146, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420243, + "Result":0.422569, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f293d94fcdadfcec47d465225b3d3e6177490fa1..5882b2458fc083e604639fd26bbd0ac5af25dd0f 100644 GIT binary patch delta 3082 zcmV+l4E6JxOq@)xSqKGq0Xl(6lU@i@5%Wmc+1Vv^@0Xd_Jm+*DHE=Tw=$b!yc5`HEd#N9}C%5C9tTP%Sf<#GiX=l53?*RLvWURB(dDRwF~aiu1UH%O&iVS1$0f7d0~ zuS;&Ygubt(Np`BWV6{gUOW^Kyam*EDm?vBz2n!#9gj_$LfP|d*&p<-J{{kcgy01V& zpgRQ;a+!63guFRtKtfKg4~I2VlmY`fA>$;@rg|H zHB8!EFJ>B>%>8gaZ-nzjBc9D3lf^8{S5u9(svpL)VKh>IQ3o;$qDiWLIBX8VeEqGC zlJZrn%!ju8Z_l{Cuf`=-YYfda49nj08Sh}|91b11)Q-8tCRgeSmoVsnghB@-6gnUw zAK_n|66koiOQ9#rt5l6q&g1+ki<5BNzc(~n(;MMJ7-j_$>nNoHi8sy`iKe5qT)l-sbTNo72a#Mq z$6R8Q>*s_^7>GbZK?D*CB9KrJ-4YOaI0LaHybHo9h;9jpVDJ<~f0u&DV@wir!6hD% zT)O`|Al6Yz1rTrDohh+pxt7>7C3=`dkCRBQ{$nn&>5UJUFcN`;k_aS}L@sgZjgMDC zA_ylJ?P*DP*CU*q(FUJ)z}BHOrUzo^IuuCATX5;s3)?gZnN`R zfLGoYfUf3w)}Dz_f9ROMWjg_o%q_=2njm(GV94Z;v5pd6MZ z=xz>fNMtQO=)@y=&RJ9sMA65L7eFI)@`hY)jsI-``7#d{Fb<@g=htJB%Kp93+1bZd-gloU@Pqav!5^$!7NwGF96_lBe-G#xl|aL&1R6#q*RZ*9 zqk&X{1dPg(1YJS563$qQ4?6J-N+pfzp(r{>A`P7+w}$@~q1>7w3n~p=6F-1u9W_>& zD%S%O>*uohUE|g}zl$Z_YB~~p_r|g*mR#c)6icq*SON{h5@;BfT*I+6%CH0p7?vdo zx|@S9Ske|Be{{mJq)|N(#U7yojnD<<()d5%vW^<7JXT!~Y;zN7yL%V0cu(|5@U2G6 zpT92Vc0PEk3G*KD}X4JrKpY1RL3$ z%hsFbl03}$>qyp7UL_=3J4l)>N8j31-DMK*-X96RHY0A)9c`d-Ov)tJa3+C{Yrx92PG8Qy0_9tE=?G%e5svHad;GfN)6rk31TGwE)BzfY zI(2}CQ(3{QgEWjvpkY*&*>z9?MrD~{M^Hb_8Ef%DCwxgBIMlZTG#tx+3eFv*VORnU!?Mi0gAy<- z%j`RXx@*o@iw`>Ck6)Qf2Su^T5@^^0wq~QwYgGQ6$FFbH!gD3IJv{j$R(S5dMXa}n zOW@F7=I%iWD4Asmx(0Li zuofS6LN`WOR1ZWkGJ%GZY1HiQd5z4!yE9TteN~_|^j#=L@h###P>MZJa-)b%%{@Tl zQgaW`a40L7dys}f3C3enD9g+}s1pWdnYqVc<{sAKgHAkulS0YNJt&GnS;5?6)NJl~ zjmd0S;3$6RuCkWes;pj(9aE34d8SFXMVedRZ=cF!Bv0gIk}6@GpRQHprginD|0{8D zyZV0oi#+cyo(-bH(H~4!-{#Mv*+@zMSp80bUHndfUL*^LSBr^MvZuLhtMYN4ze7c_ zsxo*Su6`kZK$#~|oJ85vs`uh23Ko-~GmQuDmxEf&qCXb${k!m;eef={Km4K?AC$I| zm;Nlxt6TO`{DFqmIh+S0c^)LAXqGAyIm`>ywfqJ=S^eg~^6xC3FaNSs4Tm zEQ5heidQXF8OCw;{GvD|o8NBe{0UG?0|XQR000QH(h-jV1V`J!$+Kk>ngV~0VOwfA zfajnDqy&-Vv_r6hQgpP}x)A?#!1DQ%#id6{99Q)-sc+QRh-O6!9kag~8+H%w^x#T0 zHn`V=`#reg%NEB<02{n#9O9mFh?Q~J;64809)EFE-{&w&PY!M=S$>ta~pad|l-|!?=GVcrDo< z&uEKD++`2`)sEr7Aa9JMx^sMFS-zR=4RQSVHh7Ni{rliu7~eOhAJ~5h9c&`5iEu~i z`Xbq=bu6#DEsi>N*N!Rh+zR}hyo+WgzYfw*i_ho=SYH4B^mJke&P`jf%B3&zi#bw0 z`@YR4vm}>9H};wlVgcMNMI~RUQWwY6g`ek5QAl?!8k@ zU3N4-Kz3j_fwCbztsKC^4hK1d3@c)V@UT^(J1S z$a$(bZM1?Cb>Pstx+|C=Oj~%h+JN-}Togdc5oSts@ZJ!N9Xo)eE(s15#GSU1`cT?E zHk5tSJ>Wb|AFu>Ze>wFOpMbHMiD*P}HY3fCZF&G_7#fhEyu(?N;j`C+mF{LkuUt5# z@STC1)=8Y*M<-@u+qOx&=&d1thCjD2Z~dyVP^VKdKvlsRVHT*@wkq&t5f0Z-=eOa+ zI|2N$3;K<>agm`9coYJVA?~DnES{e04XW}qU&d)3=ZiYa8OZ&TELUHrkyrg^pd5$% zn>Yc6+}EF=`UA6!A<6;;cmX4VFUmGbP@mn6aWAK00000 z00000fPqvKlW!9ilM5v(0q2u8B{~6FlW`>`8^J~W+9d%108jz|01*HH0000000000 Y0Dyryld>f~0gjXLB_jqvBLDyZ05_zk82|tP delta 3078 zcmV+h4Egh%OqxuvSqKHbxQw?*lU@i@5$8zQ+1Vv^@0S^Eo^!en@=KG#2v-5ylNt$5 zf4x?UQtsmT_u|dzpsnhKvOA#1Bfc>>-^8f5j&w4h$ndEn(66t!OUB)?7DSJvS(M9mUzZ z@3an^_eJ8h;yVqwrVTf=;dZN`nU7~PD1#rA0b|g2$8d!T(`HDKiMx^Pl-uIVZ9xJ-%0&ek=l53?*RLvWURB(dDRwF~`ASWQH%O(ZFg;T0f9sO# z*CjV7p`UAMlAUTTw%Q{L61clv9HW8^^8^)wu<((Qkn86YBOxdLGb173e_<6V0)ccW;f^C$5u(*wCiPD{eO&&Qsps59E& zV+S5No#!;BhcR@{xnRIoMupiBaGRYTgvY*K`>!*vmX_OXgpR|8(X?inDq+G7Dn#E$Oe|heotm6}z z=4+U=xn9gPHkte3eBKD>i$*+~Jtm7;manE7YgIptXTxZu{-O?K7DSU&{czYEGV}Gf zI!ek{tui0l^1nUf`o0>MSgkQM)6fmG#Y5-t&^ZVlxzvtPVv{TN1SJ3+BcaeS5(*t7 zA%DWZI3>_wxJ#ic3GXuDf8;On*>*Uo&u0ct{^p-Mlt_%B>$eM(ctmpP)JWuL+z-Oy zex{M%zN}I;Mmdl3rz}pwasS@XY)x;33t^ZQNUWoj3MAe*TO^u})^hbdAi6w=E`ms| zpJSBR534q8*D2R-Ng2+fHh;9jp7!Dwogm;;63Zh#AA~Sdjf1*o4#2AyrT%g1w zl1ul02gEu`sQ}`wyE7%WEY}iyZhJhQL=QYAf@OwB*Z$dS(mKkk z0N|Ck1)!^Wp0#Hpe-t{;-?E(mNamJfM&poioG=awNzMv$?NsNR!};c*Rt(BlcL}sv>yq4ux?qDO4K-le^SXjpa&`$4N%EwfJ)TB z+_=%OR5A&G%8~?KLAMglpv7l8@eE2Ojp}hxbdKaSbduZ}{#%4{YlbYSG;~e;0G4&s zSY@hQ4@|6|%jS2DTc7+cmi$)Jk-*&>%cfYO#xW?Cs6i|l4PeP=087*$mPQ$tOaj2N zBtds`@C8fSf8sNpAeJ<$$3(#+R7N9oLAf;k54fzO#ww3h*8|(!MB47&MJ&H3dL(eG z(Q+u4sBv19OVl8jj0U)5G{7Zla1q-o;CD zBbTi=%_Vu5^VgBAqr6H;wsw#-TaLc9sk+M~zk7cqH*iPXqC47*#xW_As6i$f4KT@Q zfJxN27G=^bW0L8FK58vX(A^w-VUo7^OebE8GD)L)Tof>glavlKe`p%8JW3>L9E%c( z8YGg@0FjIah-5TCq*X>FlK_Y;NzmOKd?Avy_)I78*cGCBToe$=XyB&OdP6||6VG02 zDXwvx@A2!7Pe*^Dl5ycsqYk5S zs8fg0Ae9xoI-CZmWHdl!nO%oV092M4b_Dg)oI#7vbb^~A;LBm6a8m>|%%(`q-W;Uz z=kJQVQP0k`*f#Chn$fcNSyqL+2TR72Lv1^Z#-Y9)MuS*Ze{k+_8o-j#0G4Iu9WDW2 zS!Ul6)LnB1Ek4r;K7Iw44i^QJC8L1_Y|TcU*QoqCk6+)Yh386adwB9itnl1@i&$?D zmyAP)ns^wELtQ+K2Dz-@h57QKqjZ5WO5oxCa0lf8fEStE&(O8EJ4=* zcMr7qTqks61fqIO6p+bikW8aych75N{@tCCTI#C;rJ?UaDT;3q?}1Y6ff9`(HZ}J! z8kd@T7!5*M!Q8`X07_;&Hifdx+{1MOP?ni{3^4aVi_di8f0-0YF!yj#0A&Spk5RL^ z=QSp?VS%Igp}WdjYOAt(HFiury5^biovh~8_uHp38OaklnWRb>=cj8GxoKT}>HkU` z+^)VK|02)(i)VwVaP$Y0)wlVxXf{&PKUTjJU>Cm=pcl!);niXymF#IQ+p2t==bunf ztf~wihpS%*e^BO06em&kwCcV1iGsx>=uG3m`{ke(v*?e7eE%+dXCJ%^?GL{w#s{UX zJ@; z#xfYlq44?)CyPsuk~psFXHws&uMy3P6gp;qGdAoV-08uU zYHVjz)&p5mx;NW>LD}|jWHe-aceuYY zQgtsnN{PLc?9Z1Z0w{PvOAgCBZ20P9($mZTk8Q`TnvPfo{#f^3MEJVOdxmlUM(|p) zKc3MRleo(s{HqQzWO}|lot{g?Rrad!>1@8d z9A7Nz{o0jVy&qU4@G+_X5IUYs9;HNn1^C4A{J;xb#~KWjbZ+-+*$V1@eJn5RcdX0|XQR000O8002J#A)Q_wM*;w|{1;3J1rfSde%x8Y;90kLLAF$t~r^!J9J{T#(?C2-2Q%TcK?TmSXJr z78LVYbm%B>V|B+u`$g1~(b)J{ZnKngw~?T>2*fZ!NfP@sdy zMONUGH-JRa2?^Q*l6bDLCa0 zSTL?G@p%ys*D@cL@x&Vm;%OiCD{qHI#y${93_!=YN9A+&{IXi0n&jjz%}GkgiVzrD zJkVS6)sMXBKLOQY$iIn4z?l2;6V!jRjv>kd1-`h9w@8y+BwGQ=ler`!1c8&_CX?SJ z76MZflW-FjlMN**0p^o9B{~6ElXE2|8xguze Date: Sun, 23 Feb 2025 12:17:43 +0000 Subject: [PATCH 164/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9756 -> 9755 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 357b66b93..da46797ca 100644 --- a/README.md +++ b/README.md @@ -1,34 +1,34 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 11:45:57,393 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 11:45:57,393 submission_checker1.py:3270 INFO] --- -[2025-02-23 11:45:57,393 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 12:17:38,277 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 12:17:38,278 submission_checker1.py:3270 INFO] --- +[2025-02-23 12:17:38,278 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 11:45:57,393 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 12:17:38,278 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 11:45:57,393 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 12:17:38,278 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 11:45:57,393 submission_checker1.py:3286 INFO] --- -[2025-02-23 11:45:57,393 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 11:45:57,393 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 11:45:57,393 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 11:45:57,393 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 11:45:57,393 submission_checker1.py:3307 INFO] --- -[2025-02-23 11:45:57,394 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 11:45:58,656 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 12:17:38,278 submission_checker1.py:3286 INFO] --- +[2025-02-23 12:17:38,278 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 12:17:38,278 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 12:17:38,278 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 12:17:38,278 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 12:17:38,278 submission_checker1.py:3307 INFO] --- +[2025-02-23 12:17:38,278 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 12:17:39,282 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.36853 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69453 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440164 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434355 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9443 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 4ab4c5d79..16331dda2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -791 +792 diff --git a/summary.csv b/summary.csv index 7e5f8bd3a..0e18d8749 100644 --- a/summary.csv +++ b/summary.csv @@ -1,12 +1,12 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.36853,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69453,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440164,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434355,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9443,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index d85f83032..c80772196 100644 --- a/summary.json +++ b/summary.json @@ -4,7 +4,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -34,12 +34,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.36853, + "Result":0.436659, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -219,7 +219,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.440164, + "Result":0.434355, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5882b2458fc083e604639fd26bbd0ac5af25dd0f..595def930da16e2408953c9ba8af5b30b8643f43 100644 GIT binary patch delta 2750 zcmV;v3PJUpOq)!wSqKX1BI1%r2><{9LX%?%SARsC=bY{aesoRw52D$qt=?RJZn&zFX2EO_OyZert4}gjf9?F|@F7lqPsdVbN}gbrw$*W# z%@3N*G#txmkZSQ<&hjI}IGF}n{x%sk(|IC;!7^w%X&Sn2H>W`~Q&$-dk}5gk;V=s2 z$A36nOyw+FB}wEY$nxf=<7l3$ox?$tA1nH!By!kRkHo>rZ-!8tox^3_w~>5Ew_Ynn zSMK8Y_u|dzpsnhKw8^xem(93Llz|)ui%E7BKYW$ZXq@Gz%_>5|c#+v*^ z>zL-M)@2Jn23gQKOyUP6S=NxwgW}4F1H;HqOIVbCD~gM@HP;1K&jm`}kK*jzcUp(d z`y%mL@tuZT(}EjXaJyB|%-geRlz|&%ur}zsZMfnJ(71ysSZeCT~mMM06YT}-nEZ!hJ;f_(=Q;Y9RN>X-RnZ`PlOibw&$(UV%qW z=Q)k(@fbShT!6IX5?sEtd)WE<-^))|{%P;*{c<*x=F_23l#Lg6S{P58(|<|+cbZRi z@>g@24Exb+D3g4`Ci|I8vwpg`n?`9d_euNrmZmqeM9SqXt65B@n&IzEwUK8H!0 z>&Z-Gler(x=Z$c_XvDMGW3rfK`E07OR`tVpHjGBe1t9gtAyfP_K^B;+If zi&Fv}4|geaCE;BVPCk*(w!=yPe9(CEnSbt3B6$p*zg=*NMkmoO56gpvp(lteCZ>4T3~LLvw! zH|=Rjc-JGGoY4ZGSAW3fp){rkV(2^+NXR9)^y-Ce8iaA$m;}oVkFNc*=cILXuL6Kq z-WGta=6Tkhfl%m}zhyZAkc=(IK;w{hoB$0&5NRlbNJAk68kdfT8x6t`C7>LZCFpJr zZb)P$KB&YadCpl>4@A*r#tWbkI=LX1TkU@vK)#HF1&jkJ=YRSAn543QFElqF3jZ>a zwe+w8lh^Bpt)O%?uVwFhP84{d{YdbIb<3hua*ZP>m0*FMQ3*7RN}yp>at)_alu-#1 zFe*zDbOqf?IAbL~sKhfUl{Bh{qUacjG<1;MYW`b=~1CD1S|xrSqDlwk=HFf2T92VcOXC4WBZ30=NnQ9TgF!2~&G>l2EaV^TE zS;i!&gs!!gCFpJrzA#Bkd{BwkqD<1L9*Sa2a*fdG_ViZ!YhiM6mk))DHIxL%?0&i& zEsJ4Ncz>Gl14!1T92VaP!B|fNxt-G?Q9*SZ_0u6hrwB8Vq|HR^JEgh`Hq_zD>+0t~~ z+T-REdwe(oxNN8q2WTAX!~q)4WCbq{(l920hJP_xX2(GZ7?Wj&9Kjqpti%VEa3+~4 z2So9iE7yRTYn{fNe+A07YSR(Krz0HK+xI-VW7N@Ks020~>eK-mhgx-jhErL=tb;U+ zN}yp>micv10!C$-V@FU!%^554K_z@wgc)-{6yFu$8n7!;vpI)T`SZ6$-l%EkT5P*^ zY=6yY+50@J!rg-*Vq+wVB4a2g`y@L`kEX({mg4%1&ScwlR z;p?x=rh}r`U+$7CAZDq>T6574;O-UBp@Or)V?A`K-IX(*XSnY{-kpk$UM z=o-x4!%BQq3EdcBQ9TgF$OIZrrctxK=QT3_?#@Up{i_0{q3=Q|imwsxfl};&l7Blz zZ0ha-8kf3zfQCa^!QF#23`)=*n?hOU?m?9>D9hYE26OkY5+79JnG{Oq?mX z=lK;XI;$#!$KmQH0+e|Y#YvPst!giRqhK)!I@5UYe%YwSEc#<1-@gmr*$3}J`@=7a z@j+=TdDow%dGD6B6faP-I*0RMB+r9n6wOj)B8PcLbuE7YPgXxVuza1x^C9IcOI7aT zEQ_bl@5VA1$fS7JQk7vGXU}hnQ?mK-hR&a}$`O7J1Vx}L=95eyD}O&|8-(pZTe51P z(3FT-1NoVhcoTpnZ_{epEus|saFiXu=(X;FTh0sq#R+ULVNEG!Pv1qNa~W{P(j>jE2$5q zZLp#2n>K*+G=0DlJb&fXQ+xu(W+I|9lCueEer($VIK$9_1mzvhk_=zG9;|e;4ZU*Z zl)@hk%vwiravz+Sj9u3yUC~=Z{tSPuU*G#xW1&u`Vu0$BGr}xTuWeo8>mnSkrOvOz ziFX3{(=q5XZ|6mZKHyOZK!&)Z@>x8;Y*whw)BHY8^EfXySs6~UM^Y@dr;#`P7oa>3 z`8RO{47qPVLH!4_jv>kd3hE-_l1K>v002UhVkA}p$&<~9L6c($SAUt7$Jcwqawt92@x#6lxngz2#Fo|cftv<<766S`O9R~Oy`LV2Fsx7q-p57-JAx|OkHI-NUG$Bhr=k8 zAAjR;F_p7yl_ZgqAj^lJj-z?1b`A$oeypfRN#wAt9*KjK-wdHPJBQ1@ZzK7TZvCwk zrQF5w@5PJLL0i=eX_IL`FNblNC<8eR7L)8Me)uY*(KyRbn^lB_@g!aTt4yQ(TpFq} z4IbNSOI04CK{hTDE8}P|kj05sm2i<}@qhHU)giG?unHCSI#%dn6YXij)NR8m-jAn= zT*ow5y)H-iG01|>VG=(m$+CxZ9u)74I53R-w1h?Lx1zZiTXWrT_1vK3brfgszSBBv z-WQ42itjY!nl{|fhTE-%W*p(mz&KJSfrR^ zF0skEKjD%VsrMt0ka|A>390upm$)R$muCt1Nc;+VAlJxgNqG19*z**1MjL$Ifk#f~ zIgRP@7&_-%fVAW-xO^G+u=Dl5m!Gcu)85(pB>%>8gaZ-nzjBc9D3lf^8{S5u9(svpL)VKh>IQ3o;$qDiWLIBX8VeEqGClJZrn z%!ju8Z_l{Cuf`=-YYfda41des^BM18=o}6mxzvui#3onj370VFfP_K^BosOzAs^vi zoD%4GxJ#ic3Gae%@`-%59Zu@=!Qjbf{<%Ym#r zt5l6q&g1+ki<5BNzc(~n(;MMJ7-j_$>nNoHi8sy`iKe5qT)l-sbbm33E(eiZKgV2R zlk4Y%OBjeiLO}!)3L=nD5Zw|Gc{l^HB)kj4DTr7C3=`dkCRBQ{$nn&>5UJUFcN`;k_aS}L@sgZjgMDCA_ylJ z?P*DP*CU*q(FUJ)z<<`EG^Ph)=sFZg$Xjsf)eGA+2;;Od36>ciUHfORN$V)D0)SWE z7J#nidDfnZQ0SPyWjg_o%q_=2njm(GV94Z;v5pd6MZ=xz>f zNMtQO=)@y=&RJ9sMA65L7eFI)@`hY)jsI-``7#d{Fb<@g=YQ8@lFI(Q(A<0~{L4tz zQeg!quh#=xLFs5-%iedNDDZ>!Bf%f6TNb5~YaBtT1P|yLl|aL&1R6#q*RZ*9qk&X{ z1dPg(1YJS563$qQ4?6J-N+pfzp(r{>A`P7+w}$@~q1>7w3n~p=6F-1u9W_>&D%S%O z>*uohUE|g}zkiD*-fB7$eD}t(DVAL07!*sc;aCCT92Vc0PEi*o8oXggm z=8`8WkiAmjL4D%-Oa%lB58{c zI$@7pSyT^2F@GX~hTT+JZwSbL;@N901y*9x+P> z2x8L_j_d7v{JP`Q(O;+pE*xsq0UC!ob%2IbS;4D=G=GdrpkY*&*>z9?MrD~{M^Hb_ z8Ef%DCwxgB zIMlZTG#twc&K;y-SON{hvdp}L5-=>w>^p+GYtC4U4?5wGUztk>MX|{eXxIX_W~0t) zRQ{aDuYYgU!gD3IJv{j$R(S5dMXa}nOW@Ffw_QRePJ34bV=WeK_lbN8?oA9X@EMp#r2L@_df zhLdU3?CyDu%)h%cQcHbRpfvPdC`Iut;yqA`Jy3F^h)vBsK;u$#572NZE0}wbhCvC& zV^b*0%sr?R24$JK$6)3j*5ZRsJd;An%snWIL0Q4vW7KT!d5y_zSl}pr=&rJs+N!Kx zjei|ekFI&9Nw-CsTiNAwZcYQJh5C z)2jF4CkhsmpfimJ@0WvG%%VRQ^8LH;omPGDF0?=Vq8J~Pwvw0rEX}K1_EP+ThSfQo z2P1hNB%^4SDib-(3)Qv!20U5)=D_mrES@j_vQ*_R&a!y={Aw(NflP{5EmaxDarXS8 zI3=6kZs`08v&j*D4g|qP{o0dDAS-{{X-W{b18vEwfkItcx)ahQ5w#vjY90T56~B@| zR$Y$noild^73-sHqF=C1XjP&tNl^rfX)e^hMDO(`UZKc&syJ=5f)aJ$(7L)Sm?2DC zc(vMq^#WWJK*|wjN_6nv5R4r=fTS)74i&_mwvzf#+C4UuebYVQJWU_41W$iC^%S3g zv6+czL~=GG&5vz*0B0B)kf6N7S(4$i*MpVrW<#%BIHmBNft%JzoZUw!W@Fp7NxSH+ zA%BKHw=Zw~s2!vyCCj0t$Em zI)OCC6O(Tf7LyDmDgoz{Hzhg&S(9=lCL6&;{n{k~002+| l000pH0000000000004l2I+L>{J^_xC@+Bh%KqCME008Ap4BG$z diff --git a/summary_results.json b/summary_results.json index 8bf5f6e42..ca000a66b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -293,7 +293,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -329,7 +329,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.36853, + "Performance_Result": 0.436659, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.440164, + "Performance_Result": 0.434355, "Performance_Units": "Samples/s" }, { From 19fca7ad4e772f687621593a1f7ed4cb7fc4c9c1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 23 Feb 2025 12:20:25 +0000 Subject: [PATCH 165/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9755 -> 9754 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index da46797ca..dbc8f373b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 12:17:38,277 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 12:17:38,278 submission_checker1.py:3270 INFO] --- -[2025-02-23 12:17:38,278 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 12:20:20,488 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 12:20:20,488 submission_checker1.py:3270 INFO] --- +[2025-02-23 12:20:20,488 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 12:17:38,278 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 12:20:20,488 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 12:17:38,278 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 12:20:20,488 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 12:17:38,278 submission_checker1.py:3286 INFO] --- -[2025-02-23 12:17:38,278 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 12:17:38,278 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 12:17:38,278 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 12:17:38,278 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 12:17:38,278 submission_checker1.py:3307 INFO] --- -[2025-02-23 12:17:38,278 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 12:17:39,282 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 12:20:20,488 submission_checker1.py:3286 INFO] --- +[2025-02-23 12:20:20,488 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 12:20:20,488 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 12:20:20,488 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 12:20:20,489 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 12:20:20,489 submission_checker1.py:3307 INFO] --- +[2025-02-23 12:20:20,489 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 12:20:21,657 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,11 +29,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434355 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9443 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2146 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2933 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.437017 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422569 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 16331dda2..68226522e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -792 +793 diff --git a/summary.csv b/summary.csv index 0e18d8749..8509fb0d8 100644 --- a/summary.csv +++ b/summary.csv @@ -7,11 +7,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434355,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9443,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2146,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2933,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.437017,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422569,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c80772196..a79a14c1c 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9443, + "Result":20.8184, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2146, + "Result":10.2933, "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 595def930da16e2408953c9ba8af5b30b8643f43..493891d8676cc8f66bc672f713410ab3b6fcadd8 100644 GIT binary patch delta 2725 zcmV;W3R?A>OqxuvSqKD?HNsAlT?jXS%3CVBm1N#xh{5!B(jiaRKx4-!G^! zdd?$ZXJ?nxyfJ+;CJS z&HUNWpT;xUQJ-Y0{@VS|;X|DKo=&99lsv&K?WmJ1n;$frX)uwIpK9@3&hjIFqd1BD zEPtDfo9R4}{%{!-O`C?UTg}K1XX+}$K~g10JQ{_8{1^v|NY1iVl0;7ZEN^}~3FoQW zJsgJlv7$dpB1awdNF1E}W(c*}JzUm(8_I`t>$OsJJg$ipOD|E4mb~Vw~EyFCX$J11=`L50c7A_V zas8^|=2gXQnPR7>ZQRo~i#JG5xuWfouD&k0eqD0ICG@(MCfVt&3442FvINd<6~|mb znt8$%g0S!rNXY&32}sC)iT?~F1pF^RLZJH!Bm}xsAR)I|4@k(xIRg@May=j+XZM^- z403k+Ktj&$1(1-ldkG}u?0y51Hon~~e7jjJfk`@Z%q12%_a|J^BK`dcB&5HefQ0n- zGnY6d%a>;f_(=Q;Y9RN>X-RnR`PlOibw&$3UV%$a=Q)k(^B6jR=3IcZSKev=?EP{!l;+c+ahOdOcUllf&1jncjq<5Z{%S6h(IA|SWRg$V|sSED7&{a0;SR0wQQU1<|1(@|ZS>x!@9)NG_fK9T4m2N(B&aot+`EWjdzVGbFm0 zM3<9D?*3zcF0tr?50@|!frOF>B$Pxhap;4OTS6iTCpYbBNqEmCoSe}Dk5|Cvp){rs zV(2^+NXR9)bnAsJ@`E^SO#NksN5}fvbJ9AxR{_8)cMCvAb6s=KKqz#~-?E$lNXC|9 zps`6iPJo6Xh%}Tzq@fT3jYG%7jRs+e5>O7y67)8I2R9_L5)V}3l04@ust=;*GUEl% z2%TJz%dPgm4Ip2}!2-sil=J+4Oj0?x7n+j~g?|~zT6$Q4$?J8)R#4iS+p_jOCknjK zekAz9x@l4>xyBZhO0YoBs012DCD1S`xrS3I%BTbh7?mXndV+2xoUsxQRN@+xN*dKi zQFM%dL>f9sZZ-ccLb)|U7E~I#CVl|RI{H{;s9ZNptk-4pr^c--zl$Z_YB~~p_r|m+ zmRw^S6icq*SON{h5@;BfT*I+6%CH0p7?vdodYgkUSke*?RKl^OQGF1_mQaC4=z?-- z{2y>xM<1&!tF9ZiSy#=6OLO0Ni8n=$1mA0aG;Ind*VruzCf9H&G>l2EaZJi&yNpRt30-R~OVHaKd|{H7c%Tx0 zw?&zxQGFD}nB*Fv)9vZ4_SeGX;w~Qw7i%a9klB5-94(7sQn;G&14!1iuU`&=7as+eauo4ec!kJ{I91z84u3Q6Vu5}u7{uLP^T@hx?0a1KcgloXANX_OPPUX+v7I~wloolh}+OagFW$p8<3TF?Nz?4IMJ3wPo z;||bpEGt-dkcMFiGz`l!_YO+Huq^ZM2x_l6VNWFieE6KN=!NJGgq%IrNT0VT66LC;|J9#-O^O6bOa2#e~2C`KmG za59aW?LDuN`FD3lYUy7UC=GoVN>O}`cn_3f50u;~Vo`Sw&^XlH12i1U3ho}HVNinh zSQN@KcMqzBL0RVRF_^oDm3W{Mw@smB?j97ypse8TF=}@AyvAfUDsU7(^i)|(->S@B zjU88yt~tV?PnFHBFStj4G8xMgIi03T5a*|B6}c&0ee3^99NezHApauI>x*Z@u(0&| z)797cvv4+6(mz%|6kruU6rdN$g3;AtDwXVME<37xoaa}l=&Y&?A4jX72vFup7$;%& zw5q-Mje^D0??&Ybjo!W_1te z{#c&-$vB*)%2bZ>j_O+e0G_OVbYS^9i|5N%rmEbrNPEg{h4vv>im~T=P|Ro1p`*ApPO%V6x5Jk4AGQNb^85iS2!Bklz;Fjl z?LT0v$XqvI3vI0i5!)4ZU*VR5Bbb%vw+4;| z<#X};vRR>;oRw52D$qt=?RJZn&zF zX2EO_OyZert4}gjf9?F|@F7lqPsdVbN}gbrw$*W#%@3N*G#txmkZSQ<&hjIF!#J4+ zS^hQ|HPd+_gTXRrI%yiZZa1evG*ed@4w5Q4;^8m~<;OT&Oyw+FB}wEY$nxf=<7l3$ zox?$tA1nH!By!kRkHo>rZ-!8tox^3_w~>5Ew_YnnSMK8Y_u|dzpsnhKw8^xem(93L zlz|)ui%E7BKYW$ZXq@Gz%_>5F!g!J{|0>fcKbMB8OoPX^+ESH=XpoJI#L74t3}kVl zRV7@cSv>u1bx5oetU`sojupDtM0=Vrb=$Ct>+v*^>zL-M)@2Jn23gQKOyUP6S=Nxw zgW}4F1H;HqOIVbCD~gM@HP;1K&jm`}kK*jzcUp(d`y%mL@tuZT(}Ej+T5!8n(9GMj zX_SE*Ww18ryKT7Q3ezOb1QKgkat$Q*uH*(toL$K+khmMkPIp_lyDgSLknVB?Y3KJ> z71ysSZeCT~mMM06YT}-nEZ!hJ4%sgh2NdNC;f_(=Q;Y9RN>X-RnZ`PlOibw&$(UV%qW=Q)k(@fbRP=3IcZSN>`5?EP{!l;+c+QIw4rcUl-vo6|}DcbZRi@>g@24Exb+D3g4`Ci|I8vwpg` zn?`9d_euNrmZmqeM9SqXt65B@n&IzEwUK8H!0>&Z-Gler(x=Z$c_XvDMGW3rfK z`E07OR`tVpHjGAp>M!a*We1t9gtAyfP_K^B;+Ifi&Fv}4|geaCE;BVPCk*(w!=yP ze9(CEnSbt3B6$p*zg=*NMB$Pxhap{APS3)8PCpYbBNqE;IoSe}DpI5-LZCFpK{4sJ+fB|fOcBYDnQR1ZYaWyTAj z5jwdbms{awe+w8lh^Bpt)O%?uVwFhP84{d z{YdbIb<3hua*ZP>m0*FMQ3*7RN}yp>at)_alu-#1Fe*zDbOqf?IAbL~sKhfUl{Bh{ zqUacZi8OSO+-m+?gmP=~1CD1S|xrSqDlwk=HFf2&G>l2EaV^TES;i!&gs!!gCFpJrzA#Bkd{Bvh z*P=|)s2+-9OmdCT>Gt$i`)grxahDH;i#3!4$n1W)94(7sQh1v214!1T92VaP!B|fNx zt-G?Q9*SZ_0u6hrwB8Vq|HR^JEgh_X#H6+TNZHbK-P+^k6MK9(0=R6b5eH}->cjyW z&SV8I4$?3tfrc?zX2(GZ7?Wj&9Kjqpti%VEa3+~42So9iE7yRTYn{fNe+A07YSR(K zrz0HK+xI-VW7N@Ks020~>eK-mhgx-jhErL=tb;U+N}yp>micv10!C$-V@FVbL(Lg0 z@j)egSA-dJKos8<;To_jQnNXSQ~C3^Mc$}s=UQyLc5KaP+50@J!rg-*Vq+wVB4a2g`y@L`kEX({mg4%1&ScwlR;p?x=rh}r`U^!W*2bFLxnWG0p@kuP#fJto4uAYAf%s1-o`7XB2J;frd+1hde?0z>+$7CAZ zDq>T6574;O-UBp@Or)V?A`K-IX(*XSnY{-kpk$UM=o-x4!%BQq3EdcfVNpF0#mEF2 zPNq?_z2`MD|L)F6E&ZzkrJ?UaDT=QV?}1Y6fs#8#Z0ha-8kf3zfQCa^!QF#23`)=* zn?hOU?m?9>D9hYE26OkY5+79JnG{Oq?mX=lK;XI;$#!$KmQH0+e|Y#YvPs zt!giRqhK)!I@5UYe%YwSEc#<1-@gmr*$3}J`@=7a@j+=TdDovtrFrj`wG=N@pM|CZK08dswIT-1NoVhcoTpnZ_{epEus|saFiXu=(X;FTh0sq#R+ULVNEG!Pv1qNa~W{P(j>jE2$5qZLp#2n>K*+G=0DlJb&fXQ+xu( zW+I|9lCueEer($VIK$9_1mzvhk_=zG9;|e;4ZU*Zl)@hk%vwiravz+Sj9u3yUC~=Z z{tSPuU*G#xW1&u`Vu0$BGr}xTuWeo8>mnSkrOvOziFX3{(=q5XZ|6mZKHyOZK!&)Z z@>x8;Y*whw)BHY8^EfXySs6~UM^Y@dr;#`P7oa>3`8RO{47qPVLH!4_j3LSb1nMH< glCxSQJplzppeyDjlbj_~0gaRKB_jquBLDyZ0L;KA3IG5A diff --git a/summary_results.json b/summary_results.json index ca000a66b..3b3ecff0d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2146, + "Performance_Result": 10.2933, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9443, + "Performance_Result": 20.8184, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 443a373102bb2c387957e7d2bb5fde819cff8439 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 23 Feb 2025 12:29:41 +0000 Subject: [PATCH 166/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9754 -> 9755 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index dbc8f373b..f1e2a8cee 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 12:20:20,488 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 12:20:20,488 submission_checker1.py:3270 INFO] --- -[2025-02-23 12:20:20,488 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 12:29:35,492 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 12:29:35,492 submission_checker1.py:3270 INFO] --- +[2025-02-23 12:29:35,492 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 12:20:20,488 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 12:29:35,492 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 12:20:20,488 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 12:29:35,492 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 12:20:20,488 submission_checker1.py:3286 INFO] --- -[2025-02-23 12:20:20,488 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 12:20:20,488 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 12:20:20,488 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 12:20:20,489 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 12:20:20,489 submission_checker1.py:3307 INFO] --- -[2025-02-23 12:20:20,489 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 12:20:21,657 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 12:29:35,492 submission_checker1.py:3286 INFO] --- +[2025-02-23 12:29:35,492 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 12:29:35,492 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 12:29:35,492 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 12:29:35,492 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 12:29:35,492 submission_checker1.py:3307 INFO] --- +[2025-02-23 12:29:35,492 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 12:29:36,870 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2933 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1648 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.437017 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422569 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 68226522e..b371f5ce8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -793 +794 diff --git a/summary.csv b/summary.csv index 8509fb0d8..91a0d592a 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2933,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1648,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.437017,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422569,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a79a14c1c..d66428218 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2933, - "Accuracy":"acc: 75.400", + "Result":10.1648, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 493891d8676cc8f66bc672f713410ab3b6fcadd8..f731581de8f056afe03dff0b9fef2a1675697bff 100644 GIT binary patch delta 366 zcmV-!0g?WiOq)!wVG{(T_hON=WfPhL0=D6^0TxCI1?8Y2Cnb|iARd3sYQr!PgztR{ zqq{9zrlA$K18vEvfkIPSaud?V5w(7h)H-?l%1+`WkW-JVnf-QlQ8nMGF8KxLrO`Di zvK%F#xe-#gHTqaB(g&3Iz_ehU(NLp3crSN-T+Sdn+>CK;Z!p0E!?!8#L2z$ax(UPpY`Qvjrj}wwS0XaS52iJPQ?K=mEe>+ zV8OVi!k0xnT*rJ^#uINOh^KwjZ@e8BIrM=@VgP!Gds05j=a(hb0yQKj_i0X2QmzU@ z$s;Xi>tWvZKSA>cvy3540|cb^Vv&>GB)SFVpdlwElbj_~0gaRK MB_jquBLDyZ0Py3bXaE2J delta 365 zcmV-z0h0ckOqxuvVG{&L+ri1RWfPhL0*+y`0TxCI1wv``q9l_{ARd3qYQr!PMfd#* zqrEL#p`{4hfwpATK%pru-3e)uh*}RMwNC!N@*_?HUS&DDch1}yR4xyyN&dijX>^5( zEJq1wZiLjk3Vm!==_5*fU|O)oXsFN;JX+RwCAX9t2XD?;a6y7kB1n76ZH4wBSc Date: Sun, 23 Feb 2025 12:34:07 +0000 Subject: [PATCH 167/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9755 -> 9756 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f1e2a8cee..3064ecdea 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 12:29:35,492 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 12:29:35,492 submission_checker1.py:3270 INFO] --- -[2025-02-23 12:29:35,492 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 12:34:02,207 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 12:34:02,207 submission_checker1.py:3270 INFO] --- +[2025-02-23 12:34:02,207 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 12:29:35,492 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 12:34:02,207 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 12:29:35,492 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 12:34:02,207 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 12:29:35,492 submission_checker1.py:3286 INFO] --- -[2025-02-23 12:29:35,492 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 12:29:35,492 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 12:29:35,492 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 12:29:35,492 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 12:29:35,492 submission_checker1.py:3307 INFO] --- -[2025-02-23 12:29:35,492 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 12:29:36,870 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 12:34:02,207 submission_checker1.py:3286 INFO] --- +[2025-02-23 12:34:02,207 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 12:34:02,207 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 12:34:02,207 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 12:34:02,207 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 12:34:02,207 submission_checker1.py:3307 INFO] --- +[2025-02-23 12:34:02,207 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 12:34:03,294 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,4 +39,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422569 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.23624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.20445 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index b371f5ce8..114370ffb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -794 +795 diff --git a/summary.csv b/summary.csv index 91a0d592a..6dcb42583 100644 --- a/summary.csv +++ b/summary.csv @@ -17,4 +17,4 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422569,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.23624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.20445,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d66428218..f82cad125 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.23624, + "Result":6.20445, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f731581de8f056afe03dff0b9fef2a1675697bff..40aab58b477f9981101a3628b22691a22124f88c 100644 GIT binary patch delta 2302 zcmV9LnNEt4gp)vpD)~bx5oetU`sg zjupDtM7x@3>y}{_*W+m_*D=jit;-gE^fSMEn8XiCvaBJU`^A+J2ZoWKmY^v8RumU) zYp#C_uAU2&ydTBcyYIFRoA*WHwcC7=oEOPEoP|_m({m4j2e?Ku2(%;W0aY&Xg&l2V%@hg8* z1Gz^|OTv54$DW6%Gg{zb1ui+A=QO5|F?7tiVA7IHaQV{iQTOYAFF##*r~R|{%h^zx zPlv`~Hd)+hK^!%sY5q6Lr#ktoxlBfba5j=jK4FuCOs3f&UED=sTFiaY!M&;J%`A~} zIm>Dmld0ypce0L8WSY-m(&lsQa)>ydDE8v_O$E!s$F6=$I#k_VOo1W<82-~8=)h&+A&Hj za;Ki41fXLi6goyip<^WEBm9e90v(1s6uOe|9urPJkN8Pv znel?rlOGK-e}19;NZ`V{X;LatV+%?pvp^42G8&+g(EydGK`KQVl}rMlvLr!I(5-|s zDDjv|T!T_cqxxJF9V0mn9VEA!{}!R#8X*fR4P6sIfMp$htTI%t8z$E4viVcv)|KDI zlHY1N61aO~S`dP0|PAga$qA(+fgHiFrD(_oU-oWIUw9o?(M zWQ#Cqf9qSjs=G|`+xJHTx5Q1lrOjyUHf0hu$RwiyCK(Mdi5kbGOt#CIWGbO+tz`*% zn}aV*(h`rU#BEU~X;hz!0wz%-bhkJiBrWlnN?_d;qWWAE5XorZllu-Gf5v4)jW~?PrcNA2gG^TN;&2*ZlF*bR96N#Byd>kg*@EEx@8 zS?1p15&)KE{vAQ>HD^%bF_qx@E7)|nC>Sgm4NPEbcIv!F< zm?)gYqK28o*6ix}cffq3-k$Gb+uTzu!nRxHo)Oc~c`^;OidfX%!)P38?_o4RCa0lf zavDk|r=esTW%eE}0VT66LC*kt50rRZC3Is1qWVk}kjZF}OrvId&ue7t51d*Z|agsdc$0QE#AB#J=X zEPI<;#6PkPY2O|vZPQlDiQ{@^znxvVTI^Jp{DSk+=o%GSjuOz^2&vl|y|0()14?{g zTCmP&sL>ugS~NEmx0D+Pug+L-L4r>rNPEg{jkY0Jim~TgP|Ro1p`*AlPO%V6*J26F z_%~|-lRUr2lk*@Wf7ymnxo|2O_7-NXCvkG`yqt`E-)DU}T4VkKf39BM##K|Phf{Gt zO(i(x4p=a*sqkeH57#jtR`J9e3F2`d^_jQhB8NT@Nen;_aZk!e`SiSAqK4$;F3m|w z%5_0#c}sKhHH^IKKLgcq$iImvz#;ebCusfvP)h>@6aWAKvy~yj0tHX29`ScM8} z9V>LPiFP&7)-A&kc|VG?ci(LtHt&nXYsGgOa!m_vXu<7PK{Id9rcnlNl)>7d@3!HJE81<+ zOdv6LCD%Y=?MiNd#NL(M0*SMc>~yzdc^7I#_@N6l!O|BdpgPX1~xlhGiYjbxHf*yJFSX*Ng~cTt!YbDwl@Z)$ooOQc-R zvYN$Ys(J36tm6}z=5v^|xt`24Hkp41!F=8b=8Hx=n>{9rS(eYH8f(=ch-ag4tp1`7 zW#)&|RQ+(+9D(urTOD1>XRR`C+VbC?c70#9ORVM?TH7#8YtLuAjiIwSbmUe$<`RqC zsV7{*paT*L9gtAyfP{R6f3ZuT*VTp9dOGKJ(9QN+f@eq4T#3 zE^&$E(yo!n@nql!#q&%fe|%Y`YK(Ip=TBLj1e3wNp;?;V2o}O*z`a5O1BGA+cpTrr0wix|l?llSuCVV=l4igAbQ5 z5`l!02qct5E^+9Ck6S__2q!n~X-Rm`C7hhm0*_a~=AksE4`S#%6qCISCp^sGvYY@& z#+GBCu}M2lfQBK6G?YQ4p%4O%L&w9724RR2P!7uy^fm`KB(f3@RN|66=ParZqUbW? z1<;cr4Kjbc(0(NN!n$cvD!IlMluEEb&!_|%MkUZND!GPJDaxn>2^f_n33`HVC7iJm z4^-kBlu8=aM^SW)L>f9sZZ-ccLb)|U7E~I#CVl|RI{H{;s9ZNptk-4pr^c--zl$Z_ zYB~~p_r|m+mRw^S6icq*SON{h5@;BfT*I+6%CLU~2^f|o33{7@FIdtN4^+ajq)~kk z#gSw|nMEUT^?wpmxrhf8zcc!@Vfj|AUqG;Ind*VruzCf9HPzhaYElbeb z9DHGtmUy5Nw?&zxQGFD}nB*Fv)9vZ4_SeGX;w~Qw7i%a9klB5-94(7sQn;G&14!1< zyGqm7_7;*tv)uOHwVpi|6Tw{rrb~(B8rviu zU`&=7as+eauo4ec!kJ{I91z84u3Q6Vu5}u7{uLNWFieE z6KN=!NJGgq%IrNT0VT66LC;|J9#-O^O6bN2i|T_YMkdg3GL4$;J+G1ZX?J%O}`cn_3f50u;~Vo`Sw&^XlH12i1U3ho}HVNinhSQN@KcMqzBL0RVRF_^oD zm3W{Mw@smB?j97ypse8TF=}@AyvAfUDsU7(^i)|(->S@BjU88yt~tV?PnDDY5Q{*d zW_1te{#c&-$vB*)%2bZ>j_O+e0G_OVbYS^9i|5N%rmEbEM71?Q#FH7c?kC7`(xQnxkwSS`{Al=#53 zV4cxWqdj;uZ|*8?DK`$@oU!171fN8Z_LSQiZ9}jWW6!ssn9rg^M{#4EVj-5U#R-=2 zZ`J}Pc|OCF^dKXDn+>CK;Z!p0E!?!8#L2z$ax(UPpY`Qvjrj}wwS0XaS52iJPQ?K= zmEe>+V8OVi!k0xnT*rJ^#uINOh^KwjZ@e8BIrM=@VgP!Gds05j=a?~cPyzq|5dZ)H0000000000 XfPp!av?V?Pjg#{wBL+Vs000008BZz- diff --git a/summary_results.json b/summary_results.json index e0cac80a4..5e7dd721e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.23624, + "Performance_Result": 6.20445, "Performance_Units": "Samples/s" }, { From a4861f843bf7db19dfbf341dc8248d5565405190 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 23 Feb 2025 13:21:16 +0000 Subject: [PATCH 168/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9756 -> 9757 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 3064ecdea..3192d677f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 12:34:02,207 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 12:34:02,207 submission_checker1.py:3270 INFO] --- -[2025-02-23 12:34:02,207 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 13:21:11,651 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 13:21:11,651 submission_checker1.py:3270 INFO] --- +[2025-02-23 13:21:11,651 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 12:34:02,207 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 13:21:11,651 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 12:34:02,207 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 13:21:11,651 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 12:34:02,207 submission_checker1.py:3286 INFO] --- -[2025-02-23 12:34:02,207 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 12:34:02,207 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 12:34:02,207 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 12:34:02,207 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 12:34:02,207 submission_checker1.py:3307 INFO] --- -[2025-02-23 12:34:02,207 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 12:34:03,294 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 13:21:11,651 submission_checker1.py:3286 INFO] --- +[2025-02-23 13:21:11,651 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 13:21:11,651 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 13:21:11,651 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 13:21:11,651 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 13:21:11,651 submission_checker1.py:3307 INFO] --- +[2025-02-23 13:21:11,651 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 13:21:12,764 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,12 +23,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69453 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71605 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434355 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.9341 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434355 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 114370ffb..0d1dc8db8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -795 +796 diff --git a/summary.csv b/summary.csv index 6dcb42583..b39fc9172 100644 --- a/summary.csv +++ b/summary.csv @@ -1,12 +1,12 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69453,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71605,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434355,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.9341,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434355,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index f82cad125..230b1dc80 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.69453, + "Result":1.71605, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -184,12 +184,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1988, + "Result":22.9341, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -214,7 +214,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 40aab58b477f9981101a3628b22691a22124f88c..b7c0d08e65289b7f7148cf9092573e1174d0a5a1 100644 GIT binary patch delta 2792 zcmVvLMUBN0;*y~uKi%qnr2~)QXtGFId6S!4P?BW} z={zW|j5sii{IrBc>9?Y|=v#ALaP?fE(wS0M&fRDtlpapV|oR)-lpN~BcQD?Nk=M8w|be_|g9*?18f6fJ{mRy3%mwpdB zU;lgg>B>LtoxNYqhSGdGG>WqE;!X?WX>&Ts|4#F%PX1~xlVLxa4P}x~*knJGY1U5{ zche{>=00iv-qQ4DmPomrWi^Y*RP)?FS;r?b&F3&_b3K`9Y-;X@^LZnjFB*U| z&j+0+pZVtwC6dR``P&7TctmpP)JWuL+z-Oyd8Uy+zO1The~fY-=TBLjgya6bq1l?= z2p7UIE09=6Diuh)akfY_9j)c+EexWIL3BBY^<3m+TGREz7mUo*~i0 zBzl}ga`zu|e~C>We7J;>2qct5AfY63iAx`Ryb=;YIJs$0OTxPz;pB`K_`Cr&52Z0Z z5JTsoKte9TrB^R((;$q~#w1wQ@aWndYq(71Fw+-MMnr~=AiSq0tAf58ojti%V6c%+_l7S#h$beZu2 zXoOBK$mLf5-v*E`<6r^fK+1W3KPIW{-wVynhr+*%WGxw1VDft1u@#h#=C$m7&xryr zv>yq+ux?qDO0IDPr4lUAGb(|GQ3*7RO0MBliZUue1&qp)3c7-BC7iJmA2i|_lu8=a zLs4{$e?%HONN#oiEke09LKajSx+Z=A%Q|wbGE}ZRCf4h+`BUT8mEXk@Z#5kWzI$WY z6icpg42mV!a4dm_VF@%0ORnKq8f9353K*6p6?8WTU$CSlK4^qvNuzooiY=i6jnD<< z()d5%vW^_9EUT_Nwpmxrhf8bUc!@Vfj|AUqf3zG5Cf7JE3MSWZFoA}F2{a5$uHj&| z%D@B_FfdCh=xz?afJsYylnGtFVNpF0#lZwS*&NK)n+B7t=KOUg>qxH>lP$ueqi^l1 z?lOtD?~erE61V7VG(y)}%PQz@4!$r+OMK9X zf7hZ+(x@JaVoY+4(CPN{R{v{Za&earg^M+m1jy`ux*RQwVN!US@dHTKkzJ+ftGR`w z&>Ugw)*d4FSWEa z8e!|MEUJg17?D83o+_<31mr)l_*zSXf0dZDwjU{5nyy=W+4@Ur(4K?BbjYFL{ zK*O1=;Ke~2#w5@%Cd=$Nr~<}hnIT6oM-D6TK_i?=X37CkeCEnEVCGtkiT|EP;K-rdXD_cTgh?%QF9tp!S+GR^o$3`1&ie>7Xb!SON{3z}D>4 zd5y}Sv;O))p+r(oSn%6SLe;zKqJzN5d4t4PWjYDlbK*PDLVB|p>#wGCLFfPly zJg5=IWtp2tFgp(`@j)Y;OXlbSQG621HDD53v#aOd0rQP|d%lZpb5F4dYqqRCBc`Ea zG7WAOv8lZWXk2RV0UAao(oiyyhLVXibRuh%*?Uk0l+3aUx(2iNuo53Ne?m7#SX2*0 zF*1RMlWEj!?|F^Pzq>P1OMX?LH1u64Me#M_Jy420P;#e;P2D{}<5G7I&~PX#xO zK?(XphfAZ(-GdroP?ouS4Cd}(B|d1xGbxnJ-GibSloi}PM$PV?*O<(P1&-o}o+@j} zt;+1x*m3pfnk!uTO~~ffe;3@RG8xGeIhmwN80V*J6}c&0ee3^99NezHApauI+lyy| zsIc@0lhxPxvuHL_(mz%|6kr!W6rdNCg~O}GL@L?ST((vDIM1(8k*ul=9*3)+2vFup z6em&kv}(Qhje^A_=uG3m`(>vVv*?e7eE%+dXCJ%^?GL{w#s{UXO5~|OOY`iOtrRa% zw>pROU?k6jWE9O(Wg>@pqPmtpfG4XT9az53;`#EGr7CxEmc`TOcVih0WKz6qsmd^p zv*$O(DcSsZL+4Mk$`O7J1y}HBN+gp_ARm8f!!Qtp?|llRyDdwmr3l-Bw&c`6p(!oB z32BpvT0cl?oxFV&Cvg(+DaX~!emlFUo^5rP{ech4ng)@gL>U-ir84UVeJtnsBg#VL zM)J-YXwVJA zOj=Lk=-vi38vDL4`f9MI`~-h3Uf+jRb7l6Y5`fxT3MM@8XnkAb%OV}F;~_56iB}5b z(=O=~Z^uOreIT$Oc^4~A0 zF?yaOVP|KT)V*I4ZJu+w8~E8x_oIiuK1NgJUM6W6&pN8mbXAeFARdOZaYwzm{@id> zCC&WV(4WRL*-@Wls{Y#j&*4Ly{GLvv%#=LAEbXY1ESn!Rn`tnSk)LYuT+Z?%f1@~w z{49T)jGO5^k^XQQ6iu6ku3OE>4`=Es!$DFdM?4yZf&3T;i%8D0Rgy$b{VZ>OItk~g z+C3bG`LUuuN+L%c^++6?{ALKX**#p=eH+S$bnCTJbmcCNe=pvg4m+w|NSmPjyllp0 zq73E8Ure*B_~EMz$CE5SZDtV?f5g*t`B#a;{9GEU68VoEwWTT#;V_#NiIqt>9LnNE zt4gp)vpD)~bx5oetU`sgjupDtM7x@3>y}{_*W+m_*D=jit;-gE^fSMEn8XiCvaBJU z`^A+J2ZoWKmY^v8RumU)Ypx5fo(q(`AH~_b@3sz`_eJ8h;yVqwrUf^&f8chjpqaO4 z(C7=oEOPEoP|_m({m4j2e?Ku2 z(%;W0aY&Xg&l2V%@hejUxkpY*!h6rho`Dmld0ypce0L8WSY-m(&lsQa)>ydDE8v_O$E!s$F6=$I#k_ zVOo1W<82-~8=)h&+A&Hja;Ki41fXLi6goyip<^WEBm9e90v(1s6uOe|9urPJk+_~^Zb5HQaQL6nv)NOe;LVIdRT$U>vh9cP}-W?vi3bE z3VxyeNZ`V{X;LatV+%?pvp^42G8&+g(EydGK`KQVl}rMlvLr!I(5-|sDDjv|T!T_c zqxxJFe;p$^4IL!6n*SD|+!`SZDh*u|KY(Q&eXKH6t{W!S>$3S%7AL?M{WPBwzsdedN%)ttZ1WF6hB#AJ&wY3p0N zs=G|`+xJHTx5Q1lrOjyUHf0hu$RwiyCK(Mdi5kbGOt#CIWGbO+tz`*%n}aV*(h`rU zf5dH3CTUckivlK5BXqhwz199&m|WcDL*Zf#B>^(KkCvllF-!_qGkyTcI(k=W`r6(? zQfQXj-n-Vb$6_MTHDJ1wNYvOSB@#7AB%=W$84VE0Xn;twj7TN{5LuF-w>kJiBrWln zN?_d;qWWAE5XorZsnUEyK>ibpueEfre-e}C_9JCW({*!?n@{ZV;RwcMLyb6$#->gj zMuSXN@ZxY9V3N@QlVx@sE&(uEX2=oX$bk}%sRWq>Qw|e_Ggs6wGuJweIsXcjZ`GzF zh)+k@j=S%9a>uBnzfj58aHvy<(b&|g!)TDo3T7Qn15`2^pt8)b!zBPJ%N#p`e;R7e zpu}S;!CetB<}gvXD}owkSEOch4pRB^w?*ElY3EvOyLK$i(A!&|th-n`d$43oIn=kq zXl!cSVKj(k1?vu{0W28}U|Htg;SvCrW&RyO?KNjm;xU!r`YYIUxF{Gb84XNeYj*0q zM&-|0e|@7Ko-46!;>jnm!gcmde`4J|Trw6N>f&KEHns6E8sxHqk%!X&my8CuEc5bk z34qHoH;({250rRJCCDW>dYCAj#G;0o#MbQU`FFs4qu!qHV%ywPEW);1=AIGL(0MWq zw2D~N-ot1dYVTn*KqjZ5WO5oxCa0lf8fEq#E&(O8EJ4ozdk>U&TqSg4e*~iXOcapG zXpl^!W_!Bi;k0*aIaxMJ(#>VKfeP_b?iSvVyyZ(*TrA zdn^iNnY)Lp1fVQ)_ZZ;rffA3Y#BEb3!QI0}0hAToJx0y$p4XVnMg@-Ihn^~H>06cA ztFhzi(KSan^qY{)tuMGoe=-@%6FHrxN)YF#YZbXEU485SN*vs-z99c1&+Ch4!?3XQ z`_t9e`Ll2~R?F?a)bB>|@cpt;i&^-`LcV_&yt5A81=fdO6yt-^QSz=oNK5nHEo&)WaLwu- z&i%1G_mgorOO>e{Sa3mtPa;Tr%59CdAy|sB=UY(BXVIafxG_$#5KGr$f6Mqc zYXOryzsCv!6D%-108={=(Ge;5gtR}d8v%l2=s@->XS>OTY3amc@kC%_^1 z^(Scl0JD!F$^r#lFZP{DlU^iS0m_rQBs>9BllLS+0q2uBB{~6FlXN8}1s85?TP2g9 YB^3cWleHy20gjXOB_jqvBLDyZ0AZ6cBLDyZ diff --git a/summary_results.json b/summary_results.json index 5e7dd721e..056dc59fa 100644 --- a/summary_results.json +++ b/summary_results.json @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.69453, + "Performance_Result": 1.71605, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1988, + "Performance_Result": 22.9341, "Performance_Units": "Samples/s" }, { @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 0b2130c1857c4bd07613b60ff586bebd8a0b9201 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 23 Feb 2025 13:25:53 +0000 Subject: [PATCH 169/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9757 -> 9757 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3192d677f..46ecfa794 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 13:21:11,651 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 13:21:11,651 submission_checker1.py:3270 INFO] --- -[2025-02-23 13:21:11,651 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 13:25:48,130 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 13:25:48,130 submission_checker1.py:3270 INFO] --- +[2025-02-23 13:25:48,130 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 13:21:11,651 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 13:25:48,130 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 13:21:11,651 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 13:25:48,130 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 13:21:11,651 submission_checker1.py:3286 INFO] --- -[2025-02-23 13:21:11,651 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 13:21:11,651 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 13:21:11,651 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 13:21:11,651 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 13:21:11,651 submission_checker1.py:3307 INFO] --- -[2025-02-23 13:21:11,651 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 13:21:12,764 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 13:25:48,130 submission_checker1.py:3286 INFO] --- +[2025-02-23 13:25:48,130 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 13:25:48,130 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 13:25:48,130 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 13:25:48,130 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 13:25:48,130 submission_checker1.py:3307 INFO] --- +[2025-02-23 13:25:48,130 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 13:25:49,285 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1648 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.14 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.437017 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422569 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0d1dc8db8..34eafc3dd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -796 +797 diff --git a/summary.csv b/summary.csv index b39fc9172..d694783b4 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1648,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.14,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.437017,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422569,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 230b1dc80..d1f7dd132 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1648, - "Accuracy":"acc: 74.800", + "Result":10.14, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index b7c0d08e65289b7f7148cf9092573e1174d0a5a1..537fc034043e346fd0eaf742d8abd944b3d724db 100644 GIT binary patch delta 1426 zcmV;D1#SABOr1=yWD^7*OtEs4XcI6NahfYC9ig_MYPeJdM$A*?CMP93iJgVc zhVG!`DWiuPHY^PRjaZ!0^`Ulsb*u*(;)2kaWQ@Y0b`M5XFIM z^5_-G( zt${Wg@^my5qcavmHZ;X(Xx6|Myo`T;XD z1Et2!CTMa03n&VaAG+2J!k#{6C7|BksE1yVNF7z$6~e{E z^~n?;AP*b9Wn+XM0Q&$9d?WPyumMgb5E=sy`v_fKe4U~vbju5DNdt$7~&kcrv0u!JqEhV#{?*g-okA%zFX zcMQi;!vUOv5|9!^N=b)c1*Pa{uXQ2*>44?)CyPsu(C1#g~6Aj+Fp5c+WV*J>w87c6o<&<^85PCD*kY7!N;wzO9F&>WS4&+CUDJ52PUEP;wqy2G-J9*cplo|MauPAV zJKSFxsk#?krNmxJ_UB6y0~EXFMSF$F}2EO=o{w2L4$0UPSo1%6o=! z|3>gyvOk{D7L&NkAN;Ew!=YinF_P-e@sVZuX0|uP@#EXzIlA}ngLh$k-oh zq$a{0sq2emqt>y!?zTAU*j+oOz;i3`bMh{lnfyA)J}o|@8(_5l{pstr6S_BT#fnN_ zb=md^sCIBt2ZjfVzN+foQLk-enW1wwg>^X~?!fLBL4l2|PDn*{4 zCUxC=r<%I_Xnug4&~QU#LwZ`JZNqkhoh{unJkRrYR%zRC1557OC)4xg>GWJ8p0Zbs zPiOPx<@jPz@7JE(>Vwd-!vNd$A412o$)nVkUjc#b`avA%hn{N<21+`&`?YKZb-zBA z7Y;gB@&`ZVkL{D&7pxOnRVYhR6oF!z3$?G%`*s~KQRF;T zoHklPg-+nms=h0kAxv9%wc3F70$db8$`NKNbnxB~j2%0Gq%H{#6~vvklKN2EJ%2Wo zebYVQJWU_41W!5j6rX{ynTTjWayB8&k8L^tXBZlgpuEFblHrTjgOzT!p;NA$QuvdB zS?efH?xPcvv2EL=Ejnw+pW)BV%Uicno)AH~!2b|j7JJk1wzn#Xyu%?gq)$o=x`Jo2Xh0+hp$e-lSQ gpZoR`)PJ*&Ax#4WA55`wli(z}1aK7bVUwOEf3tzcH2?qr delta 1426 zcmV;D1#SABOr1=yWD^9W_hONgXcI6Nv70L@9ig_MYPeJdM$A*?CMP93Nt~0e6B>V6 ziteD~3Cj;PY*-qCJY-RluMf5Bt7F~Q5EnF$Xv%Up)b4?6Uq*ioITuJC&H15rD@1W% znmoLPjBTeW2uahxVOwBalE19EOE?6XP7C8euqHbI}jSQ|*j3LRBZB0je@$1G*# zSGZT)uU1(i1e9&T(H%`vx=PYSsE6*kfUJ$4%kV09o|h~H&Y5Dr_*-ENLr8z2q(W7u zit;6k#ZOu)u{&o%y~H(#s5AMJvMOn)I5+_zy;tmvrHwY`7?y5X7Vg1G4?bw)5fxOH zqRfO@7ruFRI%ZkMWF4$;moyZzt@ILNC}mfH9u}2e+{;LXx63F7^<*y7)Rhzg*6zi{*di(b@cTTA#L5 zx<}IkY}mwif17p;-zWBO%fvSv58IuVu|o_Cp*7F^A2Ko97M8GO)o`}j6gz0gC8Y2G zdA8wOYB+%Bpai4@k>s>Pu!2%_wAZ>2|8&6e`IE(^M@bx4^)sn&)Yph+MG76WzZn~L z5AO8fN;NjP*Ms{#xZ;1y7RO2e8@y*6;+}Dcm2ueMJ^tbze{qk$xW`}I<1beHg=h`! z`PED#JxjwavJ8vjVQjHH-;{DLBpE0(tFM;27`mqMTAjvWg^6wURNb5HzMyP-I5HYC zzB}As8L7G#9i_xxO7`bV5&;yvpe2Xp9X5RRG3n{$|HroDR!x6LECYY6doLnXp2eQWe@(%_e`dBXk1CG82FnS2xJE0woeV#i0i2u`$rLG4`BAO<}cF9tRa_29+X@ zQIopvy;DtHb~Hagc3?PxvLQXK(!?N+zq6&ghU>cC&MHj|$G7CJeKI{?o=(pt;wpR9 z_;fa3UXCvo^?vQjt=U-ir84UVeJtnsBg#VL zM)J-YXwVJA zOj=Lk=-vi38vDL4`f9MI`~-h3Uf+jRb7l6Y5`fxT3MM@8XnkAb%OV}F;~_56iB}5b z(=O=~Z^uOreIT Date: Sun, 23 Feb 2025 16:24:54 +0000 Subject: [PATCH 170/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9757 -> 9754 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 46ecfa794..aeaf1b9d8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 13:25:48,130 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 13:25:48,130 submission_checker1.py:3270 INFO] --- -[2025-02-23 13:25:48,130 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 16:24:49,357 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 16:24:49,357 submission_checker1.py:3270 INFO] --- +[2025-02-23 16:24:49,357 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 13:25:48,130 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 16:24:49,357 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 13:25:48,130 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 16:24:49,357 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 13:25:48,130 submission_checker1.py:3286 INFO] --- -[2025-02-23 13:25:48,130 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 13:25:48,130 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 13:25:48,130 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 13:25:48,130 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 13:25:48,130 submission_checker1.py:3307 INFO] --- -[2025-02-23 13:25:48,130 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 13:25:49,285 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 16:24:49,357 submission_checker1.py:3286 INFO] --- +[2025-02-23 16:24:49,357 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 16:24:49,357 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 16:24:49,357 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 16:24:49,357 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 16:24:49,357 submission_checker1.py:3307 INFO] --- +[2025-02-23 16:24:49,357 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 16:24:50,443 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,8 +27,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.9341 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.434355 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.9341 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437105 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 34eafc3dd..8832c4dbe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -797 +798 diff --git a/summary.csv b/summary.csv index d694783b4..329da0efe 100644 --- a/summary.csv +++ b/summary.csv @@ -5,8 +5,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.9341,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.434355,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.9341,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437105,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index d1f7dd132..66a5383b1 100644 --- a/summary.json +++ b/summary.json @@ -184,7 +184,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -214,12 +214,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.434355, + "Result":0.437105, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 537fc034043e346fd0eaf742d8abd944b3d724db..fb980fb40f3b0b786a84d70aafc076201cacf0f9 100644 GIT binary patch delta 2769 zcmV;?3NH1XOqxuvSqKFku&A3xlU@imf67}bx|L+!Vu;CPl5E{i7Qt4j6>$Oc^4~A0 zF?yaOVP|KT)V*I4ZJu+w8~E8x_oIiuK1NgJUM6W6&pN8mbXAeFARdOZaYwzm{@id> zCC&WV(4WRL*-@Wls{Y#j&*4Ly{GLvv%#=LAEbXY1ESn!Rn`tnSk)LYuT+Z?%f1@~w z{49T)jGO5^k^XQQ6iu6ku3OE>4`=Es!$DFdM?4yZf&3T;i%8D0Rgy$b{VZ>OItk~g z+C3bG`LUuuN+L%c^++6?{ALKX**#p=eH+S$bnCTJbmcCNe=pvg4m+w|NSmPjyllp0 zq73E8Ure*B_~EMz$CE5SZDtV?f5g*t`B#a;{9GEU68VoEwWTT#;V_#NiIqt>9LnNE zt4gp)vpD)~bx5oetU`sgjupDtM7x@3>y}{_*W+m_*D=jit;-gE^fSMEn8XiCvaBJU z`^A+J2ZoWKmY^v8RumU)Ypx5fo(q(`AH~_b@3sz`_eJ8h;yVqwrUf^&f8chjpqaO4 z(C7=oEOPEoP|_m({m4j2e?Ku2 z(%;W0aY&Xg&l2V%@hejUxkpY*!h6rho`Dmld0ypce0L8WSY-m(&lsQa)>ydDE8v_O$E!s$F6=$I#k_ zVOo1W<82-~8=)h&+A&Hja;Ki41fXLi6goyip<^WEBm9e90v(1s6uOe|9urPJk+_~^Zb5HQaQL6nv)NOe;LVIdRT$U>vh9cP}-W?vi3bE z3VxyeNZ`V{X;LatV+%?pvp^42G8&+g(EydGK`KQVl}rMlvLr!I(5-|sDDjv|T!T_c zqxxJFe;p$^4IL!6n*SD|+!`SZDh*u|KY(Q&eXKH6t{W!S>$3S%7AL?M{WPBwzsdedN%)ttZ1WF6hB#AJ&wY3p0N zs=G|`+xJHTx5Q1lrOjyUHf0hu$RwiyCK(Mdi5kbGOt#CIWGbO+tz`*%n}aV*(h`rU zf5dH3CTUckivlK5BXqhwz199&m|WcDL*Zf#B>^(KkCvllF-!_qGkyTcI(k=W`r6(? zQfRiYbaM}pdn_gbT?3{|iA0TUQX)}j0T7_%ZOwW0FfmLdYgkUMA8zE zsRY(tA*#!M>I*bOXtYFsRG(aVz0V>P*hT{5yi$YtEp=V=BS*SFq`DQ7~9C8koS=?9_RU z%Ad3T`bIrGS7O`5V;Gv-YKuKwf4Y0PWGp(=#lvW9YU5!v$Ylj152pbx84Yk*=H=lM z0GDNM9szb9DDjv|kV|m%Fi|*(MGZ5Ft=ZM{?|}J6y*=N>wz+4y2rKq2`4~D+rh!%w zi`sh_jYI7{j0VW$G?YwEL&@YcluV<{-oquJWR@l98DQ^$5|68dZj3-wf1im0G8qk$ zY1C}*d5z4!yE9Tt|EfS~=(|vg;%mfvpcH$cM5l;F-93!Pq3#|=gHTp*_i!43l4*}c zp)7OvaFqa*W$qpW+&xg@F_pM&3MIIExF~?Kg1g74+1>LRli8@iQT)(TWi5THGJ7?4 zTs^wx2#0Im@{a=ZL+tnB3U*vgx@oX3tmVSS_ z`Z|9W&c;gm$Lfaytm20P^dea>x>`)7l0D63N0pEB{0bGFRh8l6X!R2T$~+0~#nJP-iS&muDc-eIWfaHR^PA$7 zY<|3<`zN!^5q=H@cWtVFCX-Jf9)HbF!!Qtp?>t51d*UQ56(C3woIr}zkr%}7KiBxfVi{Me=kaE74)3CcU1B^kbWJy_``8+zr+DTVJ0Oj^(4 z=sq|x8r!x_+F5T6`7^v-zrOdY#zGxW#Q;?WXM|axUfZg`*F`v7L!DoT6Ym7@r!MFd zZ>L3uKHyOZK!&(y<#YD@vL;!fDo^wKIL+gHw#gPGpOg9H*Ky=c{{<*dL;g)X1BTqU zpP>2!vyUOl0tFqgsGCNUUL;!q$CJ7wJONRY_as39wBL+Sr00000{bevr delta 2788 zcmVvLMUBN0;*y~uKi%qnr2~)QXtGFId6S!4P?BW} z={zW|j5sii{IrBc>9?Y|=v#ALaP?fE(wS0M&fRDtlpapV|oR)-lpN~BcQD?Nk=M8w|be_|g9*?18f6fJ{mRy3%mwpdB zU;lgg>B>LtoxNYqhSGdGG>WqE;!X?WX>&Ts|4#F%PX1~xlVLxa4P}x~*knJGY1U5{ zche{>=00iv-qQ4DmPomrWi^Y*RP)?FS;r?b&F3&_b3K`9Y-;X@^LZnjFB*U| z&j+0+pZVtwC6dR``P&7TctmpP)JWuL+z-Oyd8Uy+zO1The~fY-=TBLjgya6bq1l?= z2p7UIE09=6Diuh)akfY_9j)c+EexWIL3BBY^<3m+TGREz7mUo*~i0 zBzl}ga`zu|e~C>We7J;>2qct5AfY63iAx`Ryb=;YIJs$0OTxPz;pB`K_`Cr&52Z0Z z5JTsoKte9TrB^R((;$q~#w1wQ@aWndYq(71Fw+-MMnr~=AiSq0tAf58ojti%V6c%+_l7S#h$beZu2 zXoOBK$mLf5-v*E`<6r^fK+1W3KPIW{-wVynhr+*%WGxw1VDft1u@#h#=C$m7&xryr zv>yq+ux?qDO0IDPr4lUAGb(|GQ3*7RO0MBliZUue1&qp)3c7-BC7iJmA2i|_lu8=a zLs4{$e?%HONN#oiEke09LKajSx+Z=A%Q|wbGE}ZRCf4h+`BUT8mEXk@Z#5kWzI$WY z6icpg42mV!a4dm_VF@%0ORnKq8f9353K*6p6?8WTU$CSlK4^qvNuzooiY=i6jnD<< z()d5%vW^_9EUT_Nwpmxrhf8bUc!@Vfj|AUqf3zG5Cf7JE3MSWZFoA}F2{a5$uHj&| z%D@B_FfdCh=xz?afJsYylnGtFVNpF0#lZwS*&NK)n+B7t=KOUg>qxH>lP$ueqi^l1 z?lOtD?~erE61V7VG(y)}%PQz@4!$r+OMK9X zf7hZ+(x@JaVoY+4(CPN{R{v{Za&earg^M+m1jy`ux*RQwVN!US@dHTKkzJ+ftGR`w z&>Ugw)*d4FSWEa z8e!|MEUJg17?D83o+_<31mr)l_*zSXf0dZDwjU{5nyy=W+4@Ur(4K?BbjYFL{ zK*O1=;Ke~2#w5@%Cd=$Nr~<}hnIT6oM-D6TK_i?=X37CkeCEnEVCGtkiT|EP;K-rdXD_cTgh?%QF9tp!S+GR^o$3`1&ie>7Xb!SON{3z}D>4 zd5y}Sv;O))p+r(oSn%6SLe;zKqJzN5d4t4PWjYDlbK*PDLVB|p>#wGCLFfPly zJg5=IWtp2tFgp(`@j)Y;OXlbSQG621HDD53v#aOd0rQP|d%lZpb5F4dYqqRCBc`Ea zG7WAOv8lZWXk2RV0UAao(oiyyhLVXibRuh%*?Uk0l+3aUx(2iNuo53Ne?m7#SX2*0 zF*1RMlWEj!?|F^Pzq>P1OMX?LH1u64Me#M_Jy420P;#e;P2D{}<5G7I&~PX#xO zK?(XphfAZ(-GdroP?ouS4Cd}(B|d1xGbxnJ-GibSloi}PM$PV?*O<(P1&-o}o+@j} zt;+1x*m3pfnk!uTO~~ffe;3@RG8xGeIhmwN80V*J6}c&0ee3^99NezHApauI+lyy| zsIc@0lhxPxvuHL_(mz%|6kr!W6rdNCg~O}GL@L?ST((vDIM1(8k*ul=9*3)+2vFup z6em&kv}(Qhje^A_=uG3m`(>vVv*?e7eE%+dXCJ%^?GL{w#s{UXO5~|OOY`iOtrRa% zw>pROU?k6jWE9O(Wg>@pqPmtpfG4XT9az53;`#EGr7CxEmc`TOcVih0WKz6qsmd^p zv*$O(DcSsZL+4Mk#}R%G1#lGcVI-4EARd3qYQr!PMfd#*qrELZni7QVKwGkEpwN_- z?u0Z+M6CysS|@*B#jhmbRhOfC=ggf!(5k*GnITMDc(vMq^#WWJK*|wjDs=GP5R4r=fTS)74i&_mwvzf#+C6_Z zlzr1Z;5a1>b0#)d|iaYHPrb{IPp#Z ze>??!=51J{?*kr%0HlvQDj&tu^L8YS>O9RCahk_@vCRsSFUbA!>pb$N{{oc5kbe_L zK%e{e6V!jRj3LSb1(5BcV@s1;BwGQ@leZ*10alarBtZe`lQtzf0b7%CB_ Date: Sun, 23 Feb 2025 16:36:05 +0000 Subject: [PATCH 171/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 8 ++++---- summary.xlsx | Bin 9754 -> 9755 bytes summary_results.json | 8 ++++---- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index aeaf1b9d8..f1f805c31 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 16:24:49,357 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 16:24:49,357 submission_checker1.py:3270 INFO] --- -[2025-02-23 16:24:49,357 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 16:35:59,751 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 16:35:59,751 submission_checker1.py:3270 INFO] --- +[2025-02-23 16:35:59,751 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 16:24:49,357 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 16:35:59,751 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 16:24:49,357 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 16:35:59,751 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 16:24:49,357 submission_checker1.py:3286 INFO] --- -[2025-02-23 16:24:49,357 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 16:24:49,357 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 16:24:49,357 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 16:24:49,357 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 16:24:49,357 submission_checker1.py:3307 INFO] --- -[2025-02-23 16:24:49,357 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 16:24:50,443 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 16:35:59,751 submission_checker1.py:3286 INFO] --- +[2025-02-23 16:35:59,751 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 16:35:59,751 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 16:35:59,751 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 16:35:59,751 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 16:35:59,751 submission_checker1.py:3307 INFO] --- +[2025-02-23 16:35:59,751 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 16:36:00,744 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,18 +23,18 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71605 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.77934 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.9341 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437105 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412006 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414566 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.14 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.437017 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2302 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409194 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422569 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 8832c4dbe..e1be92e7b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -798 +799 diff --git a/summary.csv b/summary.csv index 329da0efe..4e5dcef33 100644 --- a/summary.csv +++ b/summary.csv @@ -1,18 +1,18 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71605,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.77934,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.9341,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437105,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412006,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414566,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.14,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.437017,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2302,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409194,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422569,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 66a5383b1..f1183172e 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71605, + "Result":1.77934, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412006, + "Result":0.414566, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.14, + "Result":10.2302, "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437017, + "Result":0.409194, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index fb980fb40f3b0b786a84d70aafc076201cacf0f9..aa03c301eaad32de39078412f829f2ff83878c4b 100644 GIT binary patch delta 2747 zcmV;s3PkmqOq)!wSqKI1r$=Z>lU@j1e`aFyoYQ@fpWV=p9{&0mO_h6@q+vYks6x|K zMb3hF7|zBW_2&9>!%>wq^JhbU8qZ`$eUhpAYxh5g4{`E)I*~F{@&vQAqfWAHe$Z^D z!9+%Ws>O3T%a4rWB=WQTWioE2^F;c?Wl%J28oF*ZBR`y}s|*K8l^pSC6bABRe;h0# zIm=c_5;^s=eE8`koTqB{a2V#tih7hpjymd*I5_#u5Nfk~xa|8jln?3F-%3%+T^#>j zyf__pRK1WkLHl_*jLSqB%8|dAW>@jUR~e2cS$^8gA|!~X>GEGC3iESms7mBNcGQ-t zJcPq+QY2O;;czI66Rj%2BF*CHf49{ku}-iG71lad=wcJ?YND-MhFQEHPgA*$X|8%* zj_{+O`Q5`Leo&HS59!=5-WhRV82M=liq>yMb1}B&y5Z`%LCNbV&fa~ub=bTw60a5C zX~;EgxSCWHv|Uo_>yqo&B{wLc?`vt2 zooY?C+9MMZIJ;dOqk;_c1Qmj?@R5;_>*o_AAt(MbBO%~_VI%~)uZ)C1cgje}W!7UP z{OTv54$DXICGuq%`2QE3C=QO5| zF?7zkVA7Je;PPeMqwd%LUVggrPWxx?m#d*PUk;7KY_ho1f;ehMf7ATmC|~O2f6Zkw z8iccvO!5Vr9Aq-h2I=B13e#fklMe1pO>bt2l*?6Cvsg?u&%Kj%d?M3)4U;z4i=NiO z+@a8wg!h5X6^jCKVQ>nNoHi8uBZiEUeJIeLo+f6?JVbPz;x{T!pjBG=Ce zN&rMgLP2CC6huZsL3Bz$#BczyB)rFjQxKgJ5ShVK5FH94#XuokqIXk?P*DP z&n29k(FPAYe}Hu;jp;KnbREh_$Xjsf)(czY2XWe%`pXQDj`g$Gq;-^60l+JF3qVJ6 zU31SwD0H5`Wjg_o%q_=^#wO!9VKjgsr=bjT8VVt!ap-)w(Qp`Y2`GnU33{7@8xo<# zV>)q3o^y!mGg0(0;{~G;I(b7bx5ocAfP9$;3mAt|f6nvkF-hg%UT97}75-%;YpJjT zlh^Bkt)R3uw`J|SPZa!v_9KB0)=iUAi5go_DwzlLKqaFADj5w>i5jF*lu^kf04hro z^aR~XID;0C>BKcCl{Bi)MbSBu)6hwBYxr*w%B>l)pwiGa@dH@aQDc>AyelDBe zHEzB0f4f-nTTMp-cW+FKVu>2tpje^?v1BxWC8Gf>QG-|-Wmqx^0LzjDz0JWFENP3! zbb?sYs6G<~k5Czn&;{kv_&?yXjvA{xR$UKl^ISEbF3o-OCBG?pByg|Mv?-XVv0D^O z)F7CQ2Eb%A048b>%vKqgOacJ2BtdU;@C8iTf8ueK(8o6r)n}p*OlBt=!EC*0Fv-)L zzs_VG^@q~mc=wF ze_YM@0VM0FuF~?gy@jOEY};+^BXW<$M4)THbSaUju}w-OYLG}q14J?!Ad=Alk!Bf@ zOadUXBtdU;@P$a);xV1Tb61G!b5TGfqk+3h^9=#{Pdt3BrNBx|n%lROEltrmkl-IFdCaWaTpCUS;33LX@E&ae*;XG*>Si8z+{;rM}Q*-T0Eu`WD-m{OcbtM zQNyfU>on&4D^R{wn~oqp9br4}zWd1?qmKSUC1b;(P8~*LQ>zZ6K`JYlbvO-B$!LJe zGQSR&0H`c;>&I0z?j2C;jRd3m|c;Y%{fTr&)*h#qo$o}vF+Nie>6jH zZ+)`vV(IL`k}>5_-wva(sd0zVAeI%ZJDdivWHf+fnR|yz09cm!cLcT9oI#7nbb`-c z!KTAS!DPv3U;$gRQ|C1*f6nvQH|pWJ65A#o!_e~kO0kDacMq40MTfe07>!MBJd6gp ztYGBfG{7aJ0WQnDJX`|cvdqmRf56TIEgsVeatV$eCJGm^s9_edHM@HL9WdXhx97Xq zHun@8Beu}@%$SDGlWCw;#G>{dM&nR>52FDxISnO~(@-)w4JFekv-fZbD4AsmdIs2g zpvB`lp&KI*)n}rBOh$ub8a3N{UL*7G?u^t@Ulk|~eHTj6HmyD1Yc#|je<;x@Vo`Sw zqj9LahtVLE72G|X2B2idV^Ju}+&x?;0A-oG#{hQ^w0KM>Zks|0?j9}*pse8TF=}@A zyvAfUDsU7(bXQqRZBK1M!{n0ccXatemSVcEc|03-@gmqSqJX|>%%XK@j>Y* zdFjv6yt-vC#UI?Tx`%UrEYJO99L`c@Do1&tx|ZL7C#xSFSpJ>GBJ<^6rmEbx|0I)2ARm8f!!Qtp?|llRyDizK zrU=`Cw&c`6p(!oB32BpvT0cl?9lw1QCvg(+smImKemlFUUhGwu{DSk+=mzB(K?!JX zgw)#xy|0()14?{gTCmP&XwU#2E!w-9Tgr`tS7$7^Ai*aQq&?-fLAwwv#n|&5DCV>1 z&{5nNr&x%k+hTvq_&3`EE(ocxg1`g|3?G20orvg!6nsM3AJ>fl!7+58KnIWWEXNmb z0H^$H!>C+2l?;Q0S?gJx+DCm=CLX;*A9HI7EHs?X<|T4@43J&|}=Q@=-oLua_gJEl6>nk|Hh2bzaf3qJ(@M zM&9&afa)~l-^4TEnEUn_EJHYGX% zS(9-kCL6n0cmE^-002+|000pH0000000000004l2I+L;`J^_uB@g*Y$KO+DD004#< BB;x=8 delta 2751 zcmV;w3PAOnOqxuvSqKFku&A3xlU@j1e-dq;bGjS&*-iJOhrd2XQ{`SJX&BEss?c;* zk+UEkhO==;y}AC}a8xDD{Mpc-#xvPbpJb~3+WpVrL!A7cPNd9~Ji#pOsFN(4A2ge3 zFp-g;YVlmo@*|@-iTo^on~a<3Jdyrz85B*MhOS%9$PZ`gD#JliB}Y6Og@ODSe+P?5 z&azdKL{9xIZ+ws|?4JEI(~#5fa4Hbop0_!u(tssuKB+9krz@ z58*JI6p58dI2_92M5{`$NV7Qle{FS0tP`w4g|&_qy4Xa!nrQ2mVHVfpX)4z-%~h?- z7Jl?Izk8U(4@$DEA)Wihl@SMqk)M{JDE(Fx7j0{<3$C6El)N9s*}Lzy4x9Hy;;XVWNyZT_rf4OBO&PKA+ z-4@^779?@W+VjsFN}mh_mz&nR(7mM_l|<|FYdQvd8hre_siK(noozuVK!OZX+az{f1_#sH_E3v`K!51 zMuTuRl1V;clY>mA*&toqMPXXZebT|bsp-uuk#af9Y8I2J=DByWj!$Ho&tcN$dNR}4 zWF7?bc_WxF8u4uQm@H;lKAUQ+Rf8a&jl!||i#n8I`HHOJ7}e}-XNdp_fB9y%MLBe&WyN-T1xo}dJvVBcUKN5(*+Cp&&XXAYwRxSQ6f2!YPPO35ZPNDTodQ5o6jU<^m-ykz6|eJ0RB4 zl?ouN8Pvnel?r2%TJz%dPgm4Ip2}!2-sif0Xn5eoRt1xEGp}4~2gj$y$0? zfywK2!&Xq*n%lDWJtqo&q5VkU!n$cvDp6w#N+q*E4^%Q5ppwx5m8d}~MH!V$0-&-a zK~K=Fgfl4dm`YrOQc0uwTofH6ISm~ox0?SJq1+lF3n~p=6F-1u9eu1aRIVE)*6XtQ zQ{&c^f8WKD-)cG%xO-z-6id|D2E`IJh$W)|EEx@8i5kSxD8rIT09ckJ=xq+ZU`b0n zrV_-GM)jE}SVCnqLKl=vmL08G>%n5{A}nFIi4NrK+y;0u_vf5hW@LYHqKs?S6rn9NQ#g4uf0V3O6G zzs_VG-K)f8i!f>HTf3^eO!C|JM*_FRO}eGcXzVs+5;e#qqX8xv4KRrs$D~ZQ%a~*; zp=+&W33{7@FHF)BkEz6MQ6_0rpNj$}Q6qG^J-yZbT9{ni8X%I<0FjIah&0QH zWD)?8B?)?)gD*tV5|60_)?Fd0&qV=|j0T=6%{K((Ke6~)O9v}4X>LDKwlrNg_qh4Q z9v_ZiTsG8*!)R>k#9=hZWCbq{rvWAze+@8MX2;pmg}WlCVRl7oHs>IfKYv@~jhc3@#kOn5f6@%S z-QG8Wb@pJ%m~yCZhtb&7xWi}=%L>*VP6Jpn8o;v5y~8B{EX({mg4%1&pu}S;!Sz?L z>2OgnSTY)zz}D>4d5y}Sv;O))p+r(oSn%ioNJzTnbxMVCk)WySSY-;0SG{|KI zBM+wmE*TAQS?1;85&)NFZXN-4e;z3Dm`adKaP%-yIEh6KGl{L))${Lw`9{4x-^I4M zXSoO~_AU7sI!~s7RuPNZdl-#F?LCYJ$mBGXOin||P1OaH1sY3RF9isEa;d!Q71phTyLe?{FrjK-nv z9!7&uR&e)l8i0~%k42#@bN6tS0F-6!9s}GxP~tI_xNQn0xO=!LfU<(S$Eexe^BR-c zsK8PD&{JhCeXBBiHFjJ*y5SunasLNzfrK5`rRlVzF#(KF$@1#$oKDpch40AKbGfyG7e{{GL@seqq>$qfG4XT9az53;`#EGAE_#LahAo=^Sg=k zhcYSNwNzyk$Jz6n;*@NDyrKIiv&s>E4h45@s(&VvO&}kCPQx$|gzr2>IEfrWv zDgo+=N=Ot@akK1gY7zg)Hl%rb+@wugDJPEWnf-Ql#SbWQo+?fot)N65IJB(p3T6n?7T&BjV7&kr1(0%tnG)^2Hw0tH_8_TCf&$ul(xoyhO%#31J2WQfhBm#si*h|jLk?yCnRSh()`$_2XKa=0SU@GoFy5)cs*F@ zCL4O?$|;5K3`|C96O(Wg7LyDmDgoq^ zHzhg&R+DliCL4Ecs(&T{002+|000pH0000000000004l2H Date: Sun, 23 Feb 2025 16:44:16 +0000 Subject: [PATCH 172/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9755 -> 9757 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f1f805c31..5f4fd0a5e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 16:35:59,751 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 16:35:59,751 submission_checker1.py:3270 INFO] --- -[2025-02-23 16:35:59,751 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 16:44:11,086 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 16:44:11,086 submission_checker1.py:3270 INFO] --- +[2025-02-23 16:44:11,086 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 16:35:59,751 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 16:44:11,086 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 16:35:59,751 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 16:44:11,087 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 16:35:59,751 submission_checker1.py:3286 INFO] --- -[2025-02-23 16:35:59,751 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 16:35:59,751 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 16:35:59,751 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 16:35:59,751 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 16:35:59,751 submission_checker1.py:3307 INFO] --- -[2025-02-23 16:35:59,751 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 16:36:00,744 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 16:44:11,087 submission_checker1.py:3286 INFO] --- +[2025-02-23 16:44:11,087 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 16:44:11,087 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 16:44:11,087 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 16:44:11,087 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 16:44:11,087 submission_checker1.py:3307 INFO] --- +[2025-02-23 16:44:11,087 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 16:44:12,182 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414566 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2302 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.89323 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409194 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422569 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e1be92e7b..5ae5aef84 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -799 +800 diff --git a/summary.csv b/summary.csv index 4e5dcef33..11aca536d 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414566,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2302,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.89323,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409194,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422569,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f1183172e..92229b6fb 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2302, - "Accuracy":"acc: 74.400", + "Result":9.89323, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index aa03c301eaad32de39078412f829f2ff83878c4b..be63e0ce34f23dfe8dc0aea717e77c70fce7b3c0 100644 GIT binary patch delta 1480 zcmV;(1vmPeOr1=yViN_rg^*QZlV=kz7O|TvDjlJ=plY~O1xCzMi4 zS&HtU#Jkk*AN#pk7&wrIMnWeYhOlx4LKJ`9?kiob}K}2 zV46I zErP9qHX8C|G!&yV7C|~R#b{{Oz!toJ3?Zjw!rJ(Co(SFs7r?7jw9z!B86~=Bt@I#h z3$tV+N}B5a##^;z7jc{>Ike#`_5*TmD(tgK$0en!Xf{Egz*rkd#tI!(Q6fIKHODMv z<`|Z4Sr+cWNe@0~;}I29 zmZHprSr@)}b~Ufjz_g}2K)Vz|<11TmGxxeQqz zCu?(WQNB0Xz9QetRjp)~R(eW*W!A)Ua$6wszFYfUdEB->X zhW7kwrjeed;TBnjMe#7USe|c6ITw-)l$q66OI-|I(|D~;&30c+i|OZrX!YtKi0h$5x%bS zo?+a-5xkb{k7u;SB<`{Y|7yo@V30RPQr$T|vMk@s_J%lqd>cGR_x^qGE{yLR(+})~ z4mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$-bFK$UkB-@#bUH9IprY<|0A0Rt0oIu%-o>pmM5Xaxy(p|%KU2kWVCWhl%a@RhYo-a?Q=Mr(1 zy=r_qn=dcN7mIqocI8&@2Nnr@OzJ;`j%SlcDUn|RK5;xMKkx$Au?7Pro!k9dwt~7} zAIl5-9V_{RpYllHC_#O&R-f6mW^yh@{{ge-7hwnmHW!k_C6i4cDu2sKLlL$EZON&D zLS0&V6VfITwSJJ)I)3{qPU0lsQ;(~e{dRUyx!kKZ`32{t(G?1&{5nNr&x%k z+hWW3H`@X(@_d061b-%2VE6z`?MOr?q~Igc{tP>i4 zd4}$w_{k`!f1#$hkoBDCLLRtq{e5 zY4Ye6a>5N(0F{!2l{pde-*Rp8q68B2TL6(~rfpfCnUj=iC}I^AhnnrW8megeZw2Em zf~|oz8uD~B6r(d1LpC(UXlT~J7QBpqfRZw0ZTy<1g15m1@G28+G|Na%Y~8n3dKk8a zdAboL$@F03uiCPUB*{_=ZTO1)0Lo2;eKzU1q;wU{Cg>9wYXiwxp`$8J#pkx>m}RW| z3ipcp)hbVgfU+$(x+7^uR%wE-cP3v_RwaoP2PYt;_lljdw9&>~!_qCw!aca%gAdwxOazgo zC^KQ!g>Rmnj#-{FSqJOeC5eP=E4_pSO4(JQM@6L<_cBt^?efkxJZUtBgvjDtMvNxu z+T2@|?@hL^$oFzpE7_%$o)MXUHL=_TUO%m)(w1wYG$Yy8Z3a&~ZH->R)6urzDJUHR z#RH|r&L(Ja{|hJzksrF&4Z@y2W+kBB-l&gE=ikT3_q_q~oM+UVAfat;_DCI7+7-gZ z#r4S)A0Q7KzGY*C9sv6Q4SXZ?{ICH|B@h||5BmsRU3{IMUoPj<#q#oh=xlyEtxsDj z-J|INHXJ+few%iUAh4a^mhHfBee85v#;$Ex2(5V@{E&&+wy=aPtA_K{rr1F{E+K^n z$af6KQo{k9f)bDtL`q4AUl4{^@|_^Cyc-kJBWn>St2lsIL*piwrtue=|0m z9^CE0m1=BozXuO`aK)E@Esm7{Hh9lC#69B>E90=id;G;c{^A~gagV>a$6u`Y3-KD- z^Q)OgdY(mFWEmC3!`Nb!Zb~^9(j1hT)mKYh3|-TBtxn^p!nR}fRNb5HzMyP-IC2s( zzB}As8L7G#U8TfcO7`bV5(5;xASH+89X0~>G3n{$|HroDR!wJrTL%7E_g+N!y2^Wo zasNi}TCzW$(H4`q%OCu!9mAnvzcG^P&he3D`DV5^#PQ?X;5oYY?}K+?eBYRUU?+62 ziKHgN9jWV!WTV!xyzaI*>eyX7roeM6@N@Dmnwk7M$UZGTqZ?qf{{89ewiCKHZN-X8 zU*s2atbF!;n@whaN9Y8OWhMYEu5OTHg-RrLibD<3V`HFgW9&JLn!;+UJPs<<3@SyQ zpeA+Qd#9SZ{AhlFoX~JXWkY&erESA@gPkqiGd$1pcUEcJa05&3+9%WV<>~ZXBA&8W zjZbIu<>mNdQSaBD-0FkSvcmw|^&djVv&o~>mR|vZ?fO9}=!c$b4F*a&xBInh1$DnZ zmKP2>R`Lfw<&n@;g8E>sKC^4h{XZig7ebo2IUz+321JF)Y}HVub1fq zN_=2iu+C^`&;TAS+Pj)t%8i3pXDqlN!6y-|ARyAUkJ*z+AI=CkO~QQR1(Scs+D zV$1k9+X5~Ksjz~;e*_B*AAqTyi0FhAd_vkE*Np(dF?66n2aoeC#}{t^r~GWgs9ZUf z41sg%KdoL$r-}hNxj@Fnz$Dga0w{g`}>cgoxptcsAatACJ*Vg#Dh==Q#536|M zjRf&HM1AJ%w8*g!L=pqgW8AayQ9eDdm#8gBai5YREy{IX7}2t#gnS)F-t=F9>NMov z#53TS`}PyGf3uGvO#=lVOtErdli?&+0q2uBB{~6FlXN8}1-n>x|0I*2B^3cWleHy2 Q0gaRNB_jquBLDyZ0M$I#qW}N^ diff --git a/summary_results.json b/summary_results.json index d9724632a..a7191cd85 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2302, + "Performance_Result": 9.89323, "Performance_Units": "Samples/s" }, { From 4317f6d725b221a8a75ff06859871aef6e7aedf2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 23 Feb 2025 22:02:57 +0000 Subject: [PATCH 173/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 12 ++++++------ summary.xlsx | Bin 9757 -> 9762 bytes summary_results.json | 12 ++++++------ 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 5f4fd0a5e..6d28bdc2e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 16:44:11,086 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 16:44:11,086 submission_checker1.py:3270 INFO] --- -[2025-02-23 16:44:11,086 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 22:02:52,262 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 22:02:52,263 submission_checker1.py:3270 INFO] --- +[2025-02-23 22:02:52,263 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 16:44:11,086 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 22:02:52,263 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 16:44:11,087 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 22:02:52,263 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 16:44:11,087 submission_checker1.py:3286 INFO] --- -[2025-02-23 16:44:11,087 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 16:44:11,087 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 16:44:11,087 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 16:44:11,087 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 16:44:11,087 submission_checker1.py:3307 INFO] --- -[2025-02-23 16:44:11,087 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 16:44:12,182 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 22:02:52,263 submission_checker1.py:3286 INFO] --- +[2025-02-23 22:02:52,263 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 22:02:52,263 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 22:02:52,263 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 22:02:52,263 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 22:02:52,263 submission_checker1.py:3307 INFO] --- +[2025-02-23 22:02:52,263 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 22:02:53,331 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,20 +23,20 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.77934 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74497 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.9341 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437105 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1644 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437105 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414566 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.89323 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409194 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422569 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.20445 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5ae5aef84..8bce69f50 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -800 +801 diff --git a/summary.csv b/summary.csv index 11aca536d..60bddca80 100644 --- a/summary.csv +++ b/summary.csv @@ -1,20 +1,20 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.77934,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74497,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.9341,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437105,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1644,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437105,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414566,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.89323,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409194,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422569,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.20445,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 92229b6fb..55c8d1c34 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.77934, + "Result":1.74497, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -184,12 +184,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9341, + "Result":23.1644, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -214,7 +214,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.414566, + "Result":0.407536, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422569, + "Result":0.420903, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index be63e0ce34f23dfe8dc0aea717e77c70fce7b3c0..19c5fd3371e7a4fc4e5634a6cb7b9b12253a0c3e 100644 GIT binary patch delta 2766 zcmV;<3NiJaOrlJ%SqKG)=<6m=lU@imfAX4&ZY7zw7-BM+BwP2BMX*(BRb0Tl{PzoL zjGpsI*xA`7b?=u%o9A@*0e*JV{pjJZkJE{AFOxKiXKhtzx~j-o7!RV^sIA^ye{Q&{ zl4ikd5KQ8kY^zT)Re$aL=kOs;eox0zW=dYcEN!dfESn!Rn`tvLMUBN0;*y~uKi%qnr2~)QXtGFId6S!4P?BW} z={zW|j5sii{IrBc>9?Y|=v#ALaP?fE(wS0M&fRDtlpapV|oR)-lpN~BcQD?Nk=M8w|be_|g9*?18f6fJ{mRy3%mwpdB zU;lgg>B>LtoxNYqhSGdGG>WqE;!X?WX>&Ts|4#F%PX1~xlVLxa4P}x~*knJGY1U5{ zche{>=00iv-qQ4DmPomrWi^Y*RP)?FS;r?b&F3&_b3K`9Y-;X@^LZnjFBr2>gJ&K8NLqqW?9Ai5YtmxD;|pJOhu$^CP}B@9F$p&$YY1rbOnh;9jp zJe+}865a*j6hyZKM9_H(qDw*KF(!$*;1Z8WF5Ukf5bH>#0*JTn&XCx$TubbE?C~&( z9w(99{l{Elf71saE@31B2_+FoD2ZI+(gz=}ghUWdZran5@UBNVIim$WZ-C81X-p5q z(0M43kV|ms)eGA+2;;Od36?cHy7teWlh%=51pu$SEdX83^Q=7sq0ljZ%W?uB8C#Bl z#v%PU0UCxN(ohDGhC&E5E*%dy8iXOLfO1$?L3eX-e?uZG@j)XVspp(U^*|I|X1o9z zp_2=8xz+!-0p!a#Sim@ta-QFhNh@Gm1-ONJGgyk2)~1*M~TE!*BRCSHDrr;? zMbR-5e`)9-xz+u*2<6rYSx{-{n)m@M>&UUnP`U1ySg*_GPmNnweiuu;)pR8I?u}(r zEV;%pD3)Bqu>=~1CD1S|xrSqDlwk=fU|5z^(A^w-!IGBvpb?HGjp~6YwuA~aLKl=v zl)y zU=(=!{z&jGaf@zg1C3)+Cb@<)2{epJpkYjMjcZXR%`zrIBXq5`tb*?5;0u$q#0QOd ze=W)+jq0H&#w6DWoo-KW^}iM-7kBwkxL89;fXwcv%h9qJCWWUNKY(N%*;Sgpnp;Q; z%`wfbou^$Q_gG8>cMVt`C6a3#ixSB-oJgQyL;?*X5@;BaRvD3?0!Cy>1>McT7b0ni z4;o?Xt}LpDq8O1t!=5UwHw5HAvG`g`e}a{mw6-59Tbiz0d)$0tj}J!xmkl-I0F6VP zI6%Xhtl-5#8pb5hFec0FIH&@~WSJpHFh>q6@j)Y;NoL9cQGDjgHDKmir!nVWf%2`| zbOiC~2*>sIJx}f!b@Uf1fenW`b%4g9Rvn<>R8}zSAPu7uXc(1cejQW+qq5AgeiHNvng^X~|1uQ_8SK4^rmzcQN+ieiH$(69+? z%}$-ysQfwWuW!`Db0xM-JozM6f7s@}r5a4sttd60&23A{Lr z%Q7zyYJ_oF=H?O1&cjN4&iKuTe52l;?_%5BQ*4YlLft^X@=aJ%|~{EIwq zFP;se!qOj1R$u4OqS;7E|5*J{fL;7hfL>G<4zCsysbo)c*;eJ_JikIkvZ^w89Ik#M zK$#~|oJ85vs`cVG3Ko-~GmQuDmz`S7qCXb${k!m;eef={Km4K?P9K!ElBfPG&9hs! zQoKOj>Kx94kvtEQQ8Y`Hi5%vM>RSE)o~(X!VEH1&{5nNr&x%k+hWW3H`@Xxd47)- z1b-%2VE6z`?My^Rq~J5s{&wv^^B4GY{qi=hno4~*6$jK*f>Z8*1>>3uUl#Fj9rIxwPrQ*J9*3ykcsnjK z^?^uY06N7zDIew2bGt$f$;l$kNlMC=5Eq&gx*%VNkyrg^pgIouH}M3Ra$kRf=Ch6= z%K`<6=<6m=lU^iS0nn4WBs>CI6O(Wg7LyJoDgo}3IVCy)V3TwuCj`=*K_ru(B^3cd UleHy20hE*UB_jq#BLDyZ0N;QpBLDyZ delta 2750 zcmV;v3PJUvOr1=ySqKI1r$=Z>lU@imf67}bx|L+!Vu;CPl5E{i7Qt4j6>$Oc^4~A0 zF?yaOVP|KT)V*J3V)LBSeUP8s(2pMe`WQ`>dzqwRJnN`J(^W;zf_NCt#vS$M`g6lk zl{E8bLw_32WJi6Hsrqa8KZg%-@_RawGE?#dv$Ug5vTT0PY^K3PMt-Wrb2-b8e~jWJ z^0WM9GH#~xMEb*JP&92Cx^6WiKb)zn3XA4&`OOe&vwOJg`!DJ#$QOaE$|6aT}9d=Z`kTya4c{z;B zL>bDFznErM@xxacjwe}u+RP#(e~72)@?Rwi^K)saO5{Iw)Rw9|gu`r7BvvNja43rt ztt!DH&En{{)giG?unHB{I#%dn6YXlEty_j!ydO_fxsGYBdR>n2qo4WR!z6xCl4TF+ z+%Mi4abOtvX$gwfZ$)!4w&uFw>bXJ5>nP6NeYbVkye|^372j#dHEp<|e+{==4b6N! zn?V`;pbQv;zB`61RJ7Y(?bWD539bX_B34O}5%26B0PPT^yr= z4D$pPg0S$Bk&x@>6C)uffBrKgA>e;uBm}yzjD$dU%1Fp%)?*~(%{gNvHa9;9&g0dTWilFsvyn{l1)CgXGR+3* z;w}o)V(pU-?oCZ^W{H%`RaUcDOf}EFlXZL|(|iq+HrI=p#wPP1n9m!*e9?$!v&UpH z%ktG!W33tl@oW^1f7M^qq0IbnnyMcTnhqYvlh6Efn-YmJbp3XL5|>CW?HY+3PX>Na+|M-f+m}_Uf5tfHasHIWNiZ4Q8=9r* zjbI^+b_EjaD5V04H})2ZZCh(OdW#3q;X!l|L~{Kcqr@WD&k0HZL`FhEWF!%1FptaOu_yTjU3E+L-#w43CcWv)80`lve@3D|ZV(M{`|s&qOG6p1)-~0g%is z$Bf1%<2Ye7fFP%#400L@A)|5Ve7MnY7;*_Hhh+(Rf186F5~0OoI&n#!bBO9QQS>q6 z1)~u82M^XoB5<=|duPCga>Wh85w&GHv^BS7?YmDD z{Dbx*fe+SAlTwKqTTm*Q2lPNCqX8-z4N!?1q*9bo$s_sxV&`EM@_-_%)tr@bQ($F>W16bBkW0k3LJutC;E}P#qZoTunSn^vKG=L?e0W492SQ=$mG6?|7k_5fY!51uPi^p_=SkkCI69tb@8I8~d z<icee)&1DS9Muf3MNBDVV6STNF&xAef8>z+^N4CTbAO zRvDN~0syllL2q;L1x(uFah1@=HxSilq7Y1GCmX?Ry=gGX)11G~WF6&IVzRZ%r0xoL z&jzF5x9^VxZi$<8OPkTyZOSBSkV!@ZOfni^5;cxVnQWIa$#g=WwU#C5Z4SOLNn1Ro ze-pPwnWRyDE((}LjnL`#^w#)mVRCVoPlby$lmy7^K3dL}#WX2g&G-Q%>!_~M^0mE% zq|j{JZS5m+kHtiwYru3Vk*KjvN+fEKNJaxhG8!O~(EyQV8Ieo^AhIMuZ*%a4NZR5t zoxpQfi0X4uKqRAqyGrv70r^ine66Lxe@aZ6+qaZ0P1nsmZa%Tcha(u54K?C08k;(C z7!5L6!HdIbfJsIJOqSVkxCFptnIT7jBL`YMrW0fmOgT&xu3S;WtX%6f=KL#AzEzu! zAU+*oJMO;w$sMDP{z4^V!=X+cMq^W}4x>RTE0}dS4N%EwfXXtz4wnF^EOYD#e`=^X zgBFkJ1b0Qin8QTjt_W(FU6Gp2IY{Nt-xhhJrk!iC?b@+4LvL?=vhHH(?7@;TA3eyVy4O6dNP9(D%%khR%~| zpjE`8_8vy#PjMP$J6(|jT7fR7Ktv%msG{hb#(J5k4cMqd+sJn;JAe0r{J)8!h zWX5ArD9hYETqgi!nY+gTcMr69Oeb!eLJ963E()Nm;O;SMcK5u-WHu^r6hCxVSxaqI zR~xzvtd|R z`u*wZ>-^>rfDQx~lEfvGR3Iu}%Sl5KwgYX+sewXWT6z=GCK0uMkkmSU`zlW2 zB;ZqztC{_Fc2T+9t2X%s=cUmVB3X_S(A)^Aw-tJCR_Oyud|+Cz)@Z0u4<0S+yOLYV zje}QbEVv-SClRDQ<+ehT-5?r&F7kYV6$B<&VE6z`?MOr?q~Igc{_HK zJ|#K Date: Sun, 23 Feb 2025 22:19:21 +0000 Subject: [PATCH 174/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9762 -> 9761 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 6d28bdc2e..34fdcac6c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 22:02:52,262 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 22:02:52,263 submission_checker1.py:3270 INFO] --- -[2025-02-23 22:02:52,263 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 22:19:15,297 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 22:19:15,297 submission_checker1.py:3270 INFO] --- +[2025-02-23 22:19:15,297 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 22:02:52,263 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 22:19:15,297 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 22:02:52,263 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 22:19:15,297 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 22:02:52,263 submission_checker1.py:3286 INFO] --- -[2025-02-23 22:02:52,263 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 22:02:52,263 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 22:02:52,263 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 22:02:52,263 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 22:02:52,263 submission_checker1.py:3307 INFO] --- -[2025-02-23 22:02:52,263 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 22:02:53,331 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 22:19:15,297 submission_checker1.py:3286 INFO] --- +[2025-02-23 22:19:15,297 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 22:19:15,297 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 22:19:15,297 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 22:19:15,298 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 22:19:15,298 submission_checker1.py:3307 INFO] --- +[2025-02-23 22:19:15,298 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 22:19:16,485 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,14 +26,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74497 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417899 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.406142 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1644 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437105 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.89323 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.4723 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409194 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8bce69f50..227b54a06 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -801 +802 diff --git a/summary.csv b/summary.csv index 60bddca80..c622d556b 100644 --- a/summary.csv +++ b/summary.csv @@ -4,14 +4,14 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74497,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417899,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.406142,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1644,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437105,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.89323,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.4723,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409194,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 55c8d1c34..4b41410bd 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417899, + "Result":0.406142, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.89323, - "Accuracy":"acc: 75.200", + "Result":10.4723, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 19c5fd3371e7a4fc4e5634a6cb7b9b12253a0c3e..f0951afc4f3988bbb6c309c1a0d166423446a398 100644 GIT binary patch delta 908 zcmV;719SYMOrcD$SqKCfIX2U?T?qXK2k5pi_bv47vtbVh1O@zH5-MP`Z4;RSf0EHs z!vQ=8B_Jh;B&QvM6_lc*z1D^JrvsMHpDZptO5(VxpGkeAzD6`FQs|id&DgMeaHj`X zswAfj6NE~3Ol-5K>fUVk1!ddAk@51=LG5x?!f9PNnaZQ9f zQr8#BMy+Fc-EDEyvAcFmf#+7>=j2^9Gx>Fpep-A+H^B1x_ot^5J8*8=id8OskzdS_ z^4a%oHklov6F8Qc0JONeLADhrk=Q8?HAs(*fwqmY=PYUptF7`ls8BPg6nTuA)OGKj zYU;A1`2n&6!wHlP>1mZFX$Eopoh{upT-Wt>R%v25z9o0WwH1?Q#FH7c?kC7`(xQg3VYv09`LDDi=5!CIrC zMhEa{-rQB(Qf?f)Ib*>E2|kG+?J2i4+J#^##-8s$F`q?;j^f5R#X>CI7F))@*%mO# z^BGnUn15h_;U1XUk%&%6!AGS1aqR#Q9778Vbnv*y3ViVfaLP|M49b;L$#AeRX+4Xh zd++6F?7A-N%E21*7x-)W`aZ0hO5LA|18OS4DR;nvaZQDW-0Q*p9$fKdi(@5#4c;>janCr!$~bKB9)EFOX{zXOl-MkzWBmaXdfp0@twy10|i?{aUtyx?dm53;P`_ z`GcSGNZ=?zeXv%a*|lbJE=K@6aWAK2mk;8KL8<}UL8jQvKUPW1vb*0K_rt> zAS-_+r3l-Bw&c`6p)M`G32BpvT0cl?9lw1QJ8=@osmImKemlFUTJBYs{DSk+=o%GS zjuOz^2&uO1&{5nNr&x%k+hWW3H`@Xxd47)-1SWr2VE6z`?My^Rq~J5s{&wv^^B4GY{qi=hno4~*6$jK*f>Z8* z1>>3uUl#Fj9rIxwPrQ*J9*3ykcsnjK^?^uY06N7zDIew2bGt$f$;l$kNlMC=5SkYg zx*%VNkyrg^pgIouH}M3Ra$kRf<`1)zA<6;-i0JDklVK!(1-gZhRbrFnBv%3MlRYIm z0brAOB_ Date: Sun, 23 Feb 2025 22:21:41 +0000 Subject: [PATCH 175/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9761 -> 9760 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 34fdcac6c..e8cac39ed 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 22:19:15,297 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 22:19:15,297 submission_checker1.py:3270 INFO] --- -[2025-02-23 22:19:15,297 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-23 22:21:36,258 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-23 22:21:36,258 submission_checker1.py:3270 INFO] --- +[2025-02-23 22:21:36,258 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 22:19:15,297 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-23 22:21:36,258 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 22:19:15,297 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-23 22:21:36,258 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 22:19:15,297 submission_checker1.py:3286 INFO] --- -[2025-02-23 22:19:15,297 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 22:19:15,297 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 22:19:15,297 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 22:19:15,298 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 22:19:15,298 submission_checker1.py:3307 INFO] --- -[2025-02-23 22:19:15,298 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 22:19:16,485 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-23 22:21:36,258 submission_checker1.py:3286 INFO] --- +[2025-02-23 22:21:36,258 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-23 22:21:36,258 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-23 22:21:36,259 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-23 22:21:36,259 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-23 22:21:36,259 submission_checker1.py:3307 INFO] --- +[2025-02-23 22:21:36,259 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-23 22:21:37,319 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.4723 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0895 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409194 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 227b54a06..25e1ae45f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -802 +803 diff --git a/summary.csv b/summary.csv index c622d556b..5ce50dbd0 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.4723,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0895,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409194,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4b41410bd..ed4a2b312 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.4723, - "Accuracy":"acc: 75.800", + "Result":10.0895, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index f0951afc4f3988bbb6c309c1a0d166423446a398..b36fdb6ee4c614612cb99ed242ebb1dbac0e6382 100644 GIT binary patch delta 837 zcmV-L1G@a7OrT7#XcGi3vT&EPY!jLSf3guRH5|ZmPy$kdNOIaCSV1W|+G|~ie>!0K z{K?|dqa=>2`kB->>T5)^B886G-;52r2X}gKr5YRD>%sjVT=8X#V-(2~RQ4jaDu znDq4W|6|*6tEMBCfj`#07ZJX$@}6PbzY)Ba?2l)(#U$>s2mflvaA1%(MpE55KC&#| z%=U&jeta7|NB91H@Ggw+8`BT$e}oP;5!XbxBXxa|Y}7iI*WDIJ9lLAC6nJh0eoo#+ zGm~Ej>8HhKbOS7}e}8&9u>)dcSt%R__NE34BcIKZK5FlSe6$ zUjaUGJU{RP*RcizC7s*-+U?q}kL88^j+OkuPkAJ8l%PIXtIzCOGdUNd{{XY>7hwnm zFCb(_B$HAgEq@5xfwtt-K%p)zy$NZPh+02LY8}6Q6(?~L@Tteu%zit&s9Nq-m;8eB z(&!o$S&kCW+z6?+HF|GX=>tl9U|O)wXsFQu9xa=@id)K!gI8xPxFEqN5u`okwnn=U zEXCOK9Vq6r=+IHz7^hf>rQ2f5_&3`ECV9TV3IY=>Fn@dirgkEt6H@RAX@6Wd0tCm< zfdU;oF0uk&yaAl@vkjwiwAfj6NE~3Ol-5K>fUVk1!ddAk@51=LG5x?!f9PNnaZQ9fQr8#BMy+Fc-EDEyvAcFmf#+7>=j2^9 zGx>Fpep-A+H^B1x_ot^5J8*8=id8OskzdS_^4a%oHklov6F8Qc0JONeLADhrk=Q8? zHAs(*fwqmY=PYUptF7`ls8BPg6nTuA)OGKjYU;A1`2n&6!wHlP>1mZFX$Eopoh{up zT-Wt>R%v25z9o0Nqz@?ZfoZ{7qoGCz@MzxLRoqf;9K1PW!37CEi6HGMw>8>@ zU@69)??5r1MTd^!#yG`7EZr7c#=qGXFv;^7RuGtAfq&s1nA(wuPDsH=r2TR201zBQ z3kr1bxX228@dj|pPc{t7l~c)ZurO&oi=%t*F6+v{8uJ(UYx(*we@-{{^T{L;g)X1CF_GKSA>cvyvfA0|fkF5-OA9B)SE$ox8gwlcFV5 Q0h5#WB_jq!BLDyZ09(zaKmY&$ diff --git a/summary_results.json b/summary_results.json index 5a3a8f036..6d3bec8ac 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.4723, + "Performance_Result": 10.0895, "Performance_Units": "Samples/s" }, { From c0380eba51d8d6f012c2627845c800fdd7aabafc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 06:39:28 +0000 Subject: [PATCH 176/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9760 -> 9764 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e8cac39ed..d7884c860 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,28 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-23 22:21:36,258 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-23 22:21:36,258 submission_checker1.py:3270 INFO] --- -[2025-02-23 22:21:36,258 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 06:39:23,964 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 06:39:23,964 submission_checker1.py:3270 INFO] --- +[2025-02-24 06:39:23,964 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-23 22:21:36,258 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 06:39:23,964 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-23 22:21:36,258 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 06:39:23,964 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-23 22:21:36,258 submission_checker1.py:3286 INFO] --- -[2025-02-23 22:21:36,258 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-23 22:21:36,258 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-23 22:21:36,259 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-23 22:21:36,259 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-23 22:21:36,259 submission_checker1.py:3307 INFO] --- -[2025-02-23 22:21:36,259 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-23 22:21:37,319 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 06:39:23,964 submission_checker1.py:3286 INFO] --- +[2025-02-24 06:39:23,964 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 06:39:23,964 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 06:39:23,964 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 06:39:23,964 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 06:39:23,964 submission_checker1.py:3307 INFO] --- +[2025-02-24 06:39:23,964 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 06:39:24,920 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 21.3892 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.1596 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74497 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 25e1ae45f..14b76cb9e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -803 +804 diff --git a/summary.csv b/summary.csv index 5ce50dbd0..79a14d810 100644 --- a/summary.csv +++ b/summary.csv @@ -1,6 +1,6 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,21.3892,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.1596,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74497,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ed4a2b312..85b00e009 100644 --- a/summary.json +++ b/summary.json @@ -4,12 +4,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.3892, + "Result":19.1596, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -34,7 +34,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index b36fdb6ee4c614612cb99ed242ebb1dbac0e6382..9baa61cef11fa58af1838ba31f3a12bbc4ef6a32 100644 GIT binary patch delta 2734 zcmV;f3Q_f-Or%V(SqKGrm@76>lU@iue`7M4BwP2BMX*(BMO?tV{PzoLjGpsI*xA`7 zb?=uMZJu+w5Aw4c`iY0XK1NgJUM6W6&)TZcbXAeFARdOZaa+B){@id?CC&WV(4WRL z*;b!qs{Y#f&*4Ly{GLvv%#=LAEN!ckESn!Rn`tnSk)LYuT+Z?%qd1BDEPtDff1Bw% zk^XQQ6iu6kuG`JX4`=Es!$DFdM?4yZf&3T;i%8D0Rgy$b{VX4TItk~g+BqDC`LUuN zC6S}HdL#}`elvvH>>Mupz76F=y7gKqO1X>U--|b=!?vmy(k9b>UJm0jQHFBlFQ(a5 z{P0zV<4KmEHme8;;%U14SBb*>e_R@>68Vp9wWTT#;V_#NiIqt>9LnNEt4gp)vpD)~ zbx5oetU`sojupDtM0=Vrb=$Ct&*Nz-*D=jiugejB^fSM6n8XiCvg{$9`^6_C4h$nd zEkV)xt!OUB)?7DSJvS(M9mUzZ@3an^_eJ8h;yVqwrVTf=;dZN`nU7~Pe<%YF%3x#A zcgJwW6{bms2_)98xYh>yjHTp`UAMlAUTTSnZL;61clv9CHO3<_T8_!oo)& zA=l3*AR#CIGmsGQzW@n=f9@-g5a>>Ugj{A_AR!;l8IX{Z+XE7EcF(!QAZND^B;@Q~ z00}v}mq0?!?l&MY@#SXY%gts9EKLAtn$!n9cXq=S1) z)0QH8WI8D_Le}~Nxn6JOpQBuBYmHE(?|Lqyq_tm(>YK@_phVJPt44s3ab2xP5 zQak1nn_Q_UT*9CO5(*uVQ0Rb!e1(5;N}%K6E`_cnybHp~H}ctbIH|7(22Z~8&mBr6 zkD=?g3oh}9f4!mEn%)Q&!Z0h4SVt)p zNW5{jNHiU-m3fA*TRj`Au1c;#&Y=xUy4?U@LLj`>@*69CEFatt&M8OI6GFa(i? zGKe%3LZET!e7Mmd3{e8gVOfIi=HP}z*3tu=cqGp`e~apaDEgT30%(LzK9I|;@xKiq zU*^FA#-Wt+{CZ4MIk*>^n@@#*8Od5Itia^;dSELk9nEXm_MS;m;0NtTfl5FVRPd~1E~ZF7?mXnx`J*coUxW3=)^N9l{Bi4qUaonG<1^O z8va{^9dc`iET}YeP5c0sb<|j8s$36Dte@^`!t(ZPNC=a#4IqCKFf2${nMQ{z4_N;ZUaz z&^Xkp12mk<3TA&Dq+wJ74WqKmuY(dWD$5)@f*NYhSW6Fd!gocOF$YBPT@kJUyCOB4 zb2ybhe_P~@ns%nN0^pvB?r>*aEg@r_O(CRQ{ajuW!`Db0xM-JozG4 z*yg@RJ>DKJfklV9c!0*CHXfkiTvjmhAPwUZcySn)WnLcC3FETN%_Eqdhqd%TC!9;> z=mAlD5z94T5nHpX=idSIje2{&i*0jHu`%KZeb0<(=$K4{TSaVY?*ST@+IxV8k%=^v zOr)V?A`O3C$Qot#9+ZHRS(cz{FnbSc>7h>O#t4h*gD6HO&~P%1n(aNWk@j zKX8v^GL|QDI!%=z&QI4Wa?`r{tN$x;aJ%{g`4@TKUpyOzg{9x0uKt`q3uj{`{bTh* z0e1020eX=v7+o!C|FGWP81K{ zF9%b#n1z2V@0iQ zTEq|8hO~c=lcs4ab;WXg@0__aa=F;cHu?qYgjN;Gk`zUtnC3!lEA-JU;|CNuPZg)F zR#2e>IJBtmN@fUu(-z*WHekH~7X^@VgqaHMyf*}6$95p8OM*iMaiguIK9qKg4Q1bS z3ph{HIhNolr=H>-7@LWRMkHqw()`$V0C0w(1qsSKoFy4Pdp%g`ZZ-_cg;NTDFmTg) z5-0cGiOJY?UD6eUHRR9mdiDA~tQrf|pNavhOU?+hK)tp}b%`&FaJZH_zX~Vb2;fhL zpx<~K7diF;k3s-)jC)c(i|3bSiRwJf@8dL&^I4P5N%2VX*;haEs{ag>|Y z>rYVs0ke`J$^r#>m@76>lVK!V0n(GbBs>9Jllmk<0q&DLB{~6MlXoR18->}VcqRb= o08jz|01*HH00000000000Dyr(lei^50hW{ZB_jq%BLDyZ07X$4`2YX_ delta 2731 zcmV;c3RLx^OrT7#SqKFgIX2TzlU@iue_}G3BwP2BMX*(BRb0Tl{PzoLjGpsI*xA`7 zb?=u%o9A@*0e*JV{pjJZkJE{AFOxKiXKhtzx~j-o7!RV^sIA^ye{Q&{l4ikd5KQ8k zY^zT)Re$aL=kOs;eox0zW=dYcEN!dfESn!Rn`tvLMUBN0;*y~uKi%qnr2~)QXtGFId6S!4P?BW}={zW|j5sii z{IrBc>9?Y|=v#ALaP?fE(wS0M& zfRDtlpapV|oR)-lpN~BcQD?Nk=M8w|be_|g9*?18&IPEJT!PD&e|`@;U;lgg>B>Lt zoxNYqhSGdGG>WqE;!X?WX>&Ts|4#F%PX1~xlVLxa4P}x~*knJGY1U5{che{>=00iv z-qQ4DmPomrWi^Y*RP)?FS;r?b&F3&_b3K`9Y-;X@^LZnjFB zr2>gJ&K8NLqqW?9Ai5YtmxD;|pJOhu$^CP}B@9F$p&$YY1rbOnh;9jpJe+}865a*j z6hyZKM9_H(qDw*KF(!$*;1Z8WF5Ukf5bH>#0*JTn&XCx$TubbE?C~&(9w(99{l{El z(+3|eVI%?xe@Gm1-ONJGgyk2)~1*M~TE!*BRCSHDrr;?MbR-5Y3Lxi z)%~{!9_7{uSx{-{n)m@M>&UUnP`U1ySg*_GPmNnweiuuVuMHr7DqvWaRM6cVe8G~I z_@EJvC5`HVD7J(OG(s1YOXL55%Q|wbvaGu9*k)ZdA1Y*scB-aR?ZclIZzZNDJ zcll7bSVKvG%a8e!|MEUJfpq8O1t!=5UwHw5HA zvG`g`f|Z!GwjU{5nyy=W+4@Ur(4K?BbjYFL{K*O1=;Ke~2#w5@%Cd=$Nr~<}h znIT6oM-D6TK_i?=X37CkeCEnEVCGtYTU-!ozwIwsTLRuP-pdw|BJ_8y>NWFieE z6KN=!NJA%ovPPM`2US4HEUTbvFnbRx@lhjmV}wQZKolbrXgHZh&Gw$x$o#uIBempL z1xiETg;IB{)}HS*8e$KW+$myHcMs6G)ZGI#9Lfsr9;9JVg8tCq(kOHHphg&!W$qq> zxqDcN4;t}I3MF&*peP1q1$U28v%BXtCbMCIqxhkJr^;G#t1^2vc3eHW<_ee2UYlEA zaG%O#Bv0gIk}6@GpRQHprgZhK|0{8DyZVCsi#%^Ho(-bH(jQD#U+2%F*+@zMSp86d zUHnjhUQ`wiuND)jWKVP1R^{V7zd}W_sxo*Su6`mwnI}=4MA_4-_2M@Q7L%YejR)_S zom$LMqCXb${k!m;eef={Km4K?AC$I|r~WL>vs<=Oyg=RR9L|H0JP(plG)t9<9OjAY zTK)i@tbTN0`8tc|%U71F+{IZIPoLk7WiXIQ@vfyR!?VQ^iVlA-AY?`)0RR9{0ssIJ z0001FZ(~q$Z*X%jV{dY0E_iKhos!LJ!!Qtp?|llRyDeL$p$OZ7w&c`6p)M`G32Bpv zT0cl?9lw1QCvg(+smImKemlFUTJBYs{DSk+=o%GSjuOz^2&uOrQ2f5_&3`E zCV9TV3IY=>Fnj=}b|Rt^Qt%0Be_S^L1jo>U0v$XqvI1Yc0i5!)4Wn}9R5A<}X02y& za__yIjD6o{eK}fV{sMolU*5)5Q>hQ9;((e;aLOI9U|dI2;p-wEu46u|_TOKqWc>UXyzzCL1pxWJV+b002+| l000pH0000000000004l2Ka;#AJ^_-G`z0d=L?ZwI008JE2QvTw diff --git a/summary_results.json b/summary_results.json index 6d3bec8ac..933c953c2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -293,7 +293,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -320,7 +320,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.3892, + "Performance_Result": 19.1596, "Performance_Units": "Samples/s" }, { @@ -329,7 +329,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 8828aff7df7d598386f959741a959b825f89b80c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 06:41:52 +0000 Subject: [PATCH 177/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9764 -> 9765 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d7884c860..a64b56f68 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 06:39:23,964 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 06:39:23,964 submission_checker1.py:3270 INFO] --- -[2025-02-24 06:39:23,964 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 06:41:46,700 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 06:41:46,700 submission_checker1.py:3270 INFO] --- +[2025-02-24 06:41:46,700 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 06:39:23,964 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 06:41:46,700 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 06:39:23,964 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 06:41:46,700 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 06:39:23,964 submission_checker1.py:3286 INFO] --- -[2025-02-24 06:39:23,964 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 06:39:23,964 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 06:39:23,964 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 06:39:23,964 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 06:39:23,964 submission_checker1.py:3307 INFO] --- -[2025-02-24 06:39:23,964 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 06:39:24,920 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 06:41:46,700 submission_checker1.py:3286 INFO] --- +[2025-02-24 06:41:46,700 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 06:41:46,700 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 06:41:46,700 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 06:41:46,700 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 06:41:46,700 submission_checker1.py:3307 INFO] --- +[2025-02-24 06:41:46,700 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 06:41:47,729 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0895 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2895 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409194 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 14b76cb9e..cc34bb397 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -804 +805 diff --git a/summary.csv b/summary.csv index 79a14d810..e6adf75c4 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0895,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2895,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409194,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 85b00e009..8b85accc3 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0895, - "Accuracy":"acc: 75.000", + "Result":10.2895, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 9baa61cef11fa58af1838ba31f3a12bbc4ef6a32..bb714d9aeaf5e186ed730cb41c9a21c60e9e6902 100644 GIT binary patch delta 916 zcmV;F18e-GOr=b)Y7+&Edr`1qvu_ib0)MvQT533e=b!|n1d-&lL$HEUbhOvH5dU<* z^7)g+rAJ8|SM@WgZ`9X_WaECYY6doLnXp2eQWe@(002+|000pH003lfV^DH$aC0wXZ*pZWcx`N*lFd%TFc5_AJVoSt;v@|P zSxG7Z>WNB71W|Fb>}_ii|Hw9^eS4fVOKvI_k zhYI3GTS#XT@K6A_I_&L*V!vF!lh3_}YNly^8wGJN)W zu+rUZ7?caA6#ih~ru8IF?!6O}vFo~|n+?{GKf~+Q>-(^3EL49g2BTzcfo!=V^W)r+J(gO*$t-0Q*p9$fKdi(@5#4c;>janCr! z$~bKB9)EFOX{zXOl-M zkzWBmaXdfp0@twy10|i?{o3u?uaD)0{f?FV!B2T4aFn1vSgX(MS~EEpqyGT2?-yYR ze}&nkcqRb=08jz|01*HH0Az1tP;zf@b1!3Wa%C=fZET&A%TB{E5JmTVMdW?rJW2&t zl1hNOq7o88RO~E!+FHa9*@m=#kCUcpD|N+keD9pOGjh4u%QpH2>x5Pn%90dCpqS=D zZ7cNAEaL|hIZqX*tyWN>130v(?@DF}f72G;tTtf102c+2a)g-*?YuVxW5;$NsY`-G z1#zRTq&}2(iw$MpbPG67(>a#lDW{&|9vGX6h(;u56Vm+Hb^vgOp#=%bJDepMK6^b_ z>25X*%7s%3e=u;I=>1h-U#4NhoIkh8y7kD0gplea*TUYK8xpqNFx9M006m+sICA2 diff --git a/summary_results.json b/summary_results.json index 933c953c2..3aa6a06eb 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0895, + "Performance_Result": 10.2895, "Performance_Units": "Samples/s" }, { From a60bef974c2651c4c2d1832ba2ba41a42e82bcf9 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 06:46:00 +0000 Subject: [PATCH 178/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9765 -> 9760 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a64b56f68..e64bc8efc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 06:41:46,700 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 06:41:46,700 submission_checker1.py:3270 INFO] --- -[2025-02-24 06:41:46,700 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 06:45:54,940 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 06:45:54,940 submission_checker1.py:3270 INFO] --- +[2025-02-24 06:45:54,940 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 06:41:46,700 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 06:45:54,941 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 06:41:46,700 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 06:45:54,941 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 06:41:46,700 submission_checker1.py:3286 INFO] --- -[2025-02-24 06:41:46,700 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 06:41:46,700 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 06:41:46,700 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 06:41:46,700 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 06:41:46,700 submission_checker1.py:3307 INFO] --- -[2025-02-24 06:41:46,700 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 06:41:47,729 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 06:45:54,941 submission_checker1.py:3286 INFO] --- +[2025-02-24 06:45:54,941 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 06:45:54,941 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 06:45:54,941 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 06:45:54,941 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 06:45:54,941 submission_checker1.py:3307 INFO] --- +[2025-02-24 06:45:54,941 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 06:45:55,908 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,4 +39,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.20445 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.12384 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index cc34bb397..85fd1d2b2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -805 +806 diff --git a/summary.csv b/summary.csv index e6adf75c4..c4087c927 100644 --- a/summary.csv +++ b/summary.csv @@ -17,4 +17,4 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.20445,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.12384,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8b85accc3..c848f4f08 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.20445, + "Result":6.12384, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index bb714d9aeaf5e186ed730cb41c9a21c60e9e6902..c15737aa1a1a78acb48b7e5df109121350f348b8 100644 GIT binary patch delta 966 zcmV;%13CPqOrT7#SqKI0t5Ty)vt9^M2Y-aoa`)X*x_ht$rX1?q0UC!IcYuatS;4x4 zGz?2%pRp;HW$qo+3B$6?zayx<=8U!UKqvh9E3@gKC^lIF4O_t0?9_RU%AfQ6^^JOX zuEe&9Ctt)0+uX;cw}(q$(V;FLpmC^;2WU8#6^uMc!?*-q9L8mtmj`vixGZz?2!Ceh zVJ$t-3FneIdO#Fk#BvQ-#MbQU`FFs4qu!qHV%ywPY>YTUw=kK8j>$B*Rm7(D9-wil zy$5I*nMgy)L>fva($IyhQD*N!2`HIm3AzTe_pp{8>V$5Lu&6$WVq^jhC)23e-t!un ze|Kl3minqdY3RF9>WebkG_2dfx zSFpRv=GGs$M=}}96FHrxN)YF#YZbX^UH#Snl{mOv{ek?8Jnt`_4a36H?|)BMf6kwU zv$2x?vHGC^yZE61y+{^}t`<|NWKVP1R^{V7|AdNSRb}`%TKze{<%Q~6egmGYeso~@Iwy=p5QOi23ZuI%+lfOFwgYX+sewXWT5=Q8 zCK0uMkkmSU`zlW2B;ZqztC{_Fc2T)J$Ts=~>x5Pn%90dCpqS=D?JD%qtl~!$IZqX* ztyWN>9voWM4<$2%X$x;w8?au0ivmbF!c2ws-W!6kV|$R)CBdPBxYbrtA4kjelv7Xf5g411h|WmPMx^<%?Ev5mLkkj=cQ{KjeD!*;(oHrD%8gSB-y4{; zUc}LTaAGueU6*vT!5Z>s_-p<8KCBuGbvzXVRF|9)W`TNb>k{7<;czW=ejQG{6~LeR zpijJ=7diC-k3s-)ihEH$XU{LqBns7enip}J$N9WT7i39Flm8?^0qc`NB{~6JlYAv68%!)meI)?^ o08jz|01*HH00000000000Dyr$lf5NA0g{vaB_jqzBLDyZ0MB*QSpWb4 delta 963 zcmV;!13dhoOr=b)SqKGrm@76>vt9^M2Y(FR-1=tS#nRn_B{1bs-wx0?)VKpQ9LoyU z9i(Ac0{e_ju`F}%piUT;W&RyO?KNktr3X6U&tI8M2Su^T5@^^0wq~c!YgGQ6=dW+n z!*eCJO+5J`R@mmgM?KyiE`dddx_E%bp*9|%;apZQ@*oZ45_oYKmt|fa)CuFV%zw=z zn4O2U^gt(^OXlbSQG5~0HDD22v#aOd0rQP|d%lZpb5F4`;s|}ujA`hYOoLlRY-;ZT z8kgF8fQFHYG?YxFp=2TrUC0_`_8ydgl3A9ZYcP8cYw4j*=*9?(>VqgoCeUy)jhgK} zuaWt8cSdTduL_igz6+)9Sgk!a0)Ip7fs#8#Z0ha-8kf3zfQCa^!QF#23`#H_I$auN z?jF<$gR;!sV=#9QYw3YbJd;An+&w6YL0Q4wW7O>Kd5y_zRNyFn=&rJs+N!Kxjcr#? zz5sBAOINSWtv_&&WHOc~aym_wAkI(MDst1h`m6sdad5l(1Nj$u-d{W$hJS^n-=D7j zoIeX^V2f$ zi{B_%O#Mz258p2bwU~u}Eadxl!8`llU0{FsMKL}oZ6z=LS(;b3?4@{thSfQo`(t_T zC*yFIDpNVi3)Qv!20U5)=qJGPbr#Q;uPjx$i?b|_p5INRKa@%FuB9rYIL@Bm6sKhK z;|-lZ0kh5#ehvlJ&aNOPlTjcae@??N5QOhMMdW+pBn<^wNh$&AiAqQWQE{{EZEF$# z$Tp;Xdz>^)Td60G>zVy_cI9%hmu>V5)(NdDlqD&OKrzjQ+E(bJS;mhja-J$qTdkl% z2XJUn-<8Y|rY*c#ZNPc~E(##!2s0Jhd2a~Dj_p8Fmjs6j;znCZeJJe~e;dla=@xLF zrgJR8Q%*g_Juo&C5sgUBCZzeX?Ev5mLkkj=cQ{KjeD-><(%ozrlnbX6{$Svy^(0R2 zy%UqM>$;?y4c3r9!|T=S`><*(RDUW4s4h7p%mVe=)+N3y!r@x#{3@JyBY;00f_~#| zT;$jXJPHBGG44tEoISrZB}-K2X?`E4d7KwbIwwU&iifX$I}D90iHCY}Jt+}EF= z{sXg;A<6;;d6+9UQIlaLTLIFOy(ByVT$B1FK>_cRJS92-VUu?yCL7kyt{^4>002+| l000pH0000000000004l2LX)^9J^`1L_a!3+Nh1IN003Hgzy|;T diff --git a/summary_results.json b/summary_results.json index 3aa6a06eb..c09540e4b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.20445, + "Performance_Result": 6.12384, "Performance_Units": "Samples/s" }, { From 36498f53d47c99ec268ef32cbee4d7802212bff0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 09:38:11 +0000 Subject: [PATCH 179/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9760 -> 9760 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index e64bc8efc..0b2bf4804 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 06:45:54,940 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 06:45:54,940 submission_checker1.py:3270 INFO] --- -[2025-02-24 06:45:54,940 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 09:38:06,721 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 09:38:06,721 submission_checker1.py:3270 INFO] --- +[2025-02-24 09:38:06,721 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 06:45:54,941 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 09:38:06,721 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 06:45:54,941 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 09:38:06,721 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 06:45:54,941 submission_checker1.py:3286 INFO] --- -[2025-02-24 06:45:54,941 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 06:45:54,941 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 06:45:54,941 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 06:45:54,941 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 06:45:54,941 submission_checker1.py:3307 INFO] --- -[2025-02-24 06:45:54,941 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 06:45:55,908 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 09:38:06,721 submission_checker1.py:3286 INFO] --- +[2025-02-24 09:38:06,721 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 09:38:06,721 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 09:38:06,721 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 09:38:06,721 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 09:38:06,721 submission_checker1.py:3307 INFO] --- +[2025-02-24 09:38:06,721 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 09:38:07,785 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,11 +23,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.1596 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74497 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74777 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.406142 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1644 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.2869 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437105 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 85fd1d2b2..882e31925 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -806 +807 diff --git a/summary.csv b/summary.csv index c4087c927..be325987a 100644 --- a/summary.csv +++ b/summary.csv @@ -1,11 +1,11 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.1596,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74497,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74777,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.406142,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1644,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.2869,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437105,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c848f4f08..c02061a54 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74497, + "Result":1.74777, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1644, + "Result":23.2869, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c15737aa1a1a78acb48b7e5df109121350f348b8..44a6b6933f2e04603d9ad853e207b60293d2e2f3 100644 GIT binary patch delta 2780 zcmV<23M2KPOrT7#SqKG7&$p6HlU@ikf2uUfTPnJhWZq(k$z+mj-A@+5R;d+n0rT?T zFQ_qk&I4g*XP4ByUuLv<&gnkL&u-`^9{&0m&Xju@CqXpts-mf>ik$n=D40*W>dp1% zhOH_|>di;qESk%%`Xm$e*WP~)AENm8WGZE^)f zj^fZuv$yf2l`LZEjaEV7tfgz3*$Ta2uC6m2#8q-c<8k22kCDF&hO%j;2MIqs@Q;^5>rU8t?z;i~W3Kt3cpua&%%yD0iSe{(wOs#-2>G41Eo zFfL~g{~UC&O7pSOO?oAa@?HVs$L4N6u=e)jHr?Zei6o_M49PD8F~f5Q!JxZP=J zW#icnO2>oJ*%-9_F39m)vj({aj0v>{V;PYL5(-z~1lT zm@CLIPq;!57Cr(AxqdzYe+fD9pMivc{{=`0bYFplKz9lxiq;H zq~6b5Vv{Uio+aQb@hj+oTqCC?;r-`h&r{SHZE$%94mq9YG-kkKf9RZZ0n(C>;PPeM zlXW74Dw$#a9Eo3|%2J^9uvjv+RrZP!~ z$?`4?l6>ux4DXGm)=Fb3SF5a6zL;vAyC<9YSSHyTCTVRJGmUNLp}$x({Kc{n&F7Et za-L?Zsm4Y%^rQJWf0(Ghs3Vzr!7Ncf9Ja<_zW!E6N!hAZWv4(o8l&0K zHAib>=qwDK#i1jY+A)`y_eGpE*k?Dl1yf2uLbcw9WCQS48L_j=Q8 zY7KuW^iBm5>nNoHi8s~`i5;uiwu?ZtF^Dz?kz7B=Tw;>z=Y&fbh(JO?1QH4&kWdiq z5)gSf1FyZrzPQihj4O68(iK2TZht^ z0f?dNP#_^6!KG6#Y@z2zNn_@%GCbPm&t8+(QCm79Ty&C}hcFr0!734ynojs)Mm zF-(dj*H}8ml504YK*O*E8ipm;a4hvQEI|T>Wl4hm_TUSaw8aITa4c!m07S7zs6Zn$ zLAlib54fzO#ww3hHv`)~SIwqNqiDXwo1#a8fA2LK76p@QtTqLcYdDxd!@vX@1}4|= zRcyNqOpt(qS(2c?J@^79ZE;a0^zjXg8h|JcCfLd5V7A{hnB-~BUuUw8@+vXe-e*#C zgj2A=DDd|Ek>Fe62HnyI8mmK@Cb>pvbbESd{IxK-yvwG-V$Dw=H?O1&cj+<&~IeI`8U&L|^Sj5)s>iKuTe52l;?_$^7lW&YzLNhR# zx`xTrxmCoZ_8y?Isl5ki7@0^z$wV4TCeqM_tX^jCK?x|CWeNH^v-hwTe;0K^H%3_0 z07Nk|frgW**KF^3jm*EhGg3=^RiMb@u=bhq8jZ z2Wc3TU_5lX)XUsGs1pWdnY%}4?jF|Sf=-+cg_5~@P!xl*g1bkr+1>LRlj%6eQT)(d zWi7Q;S-tAJuAXcGU<-Cvf7#mk1NTtI6L}(MvqbTu>~yUox2@~H`o9tfx9dNUf01YX zMe|XRTl&4(`p@~(U_McjKh{4KVCFv*pykQ@@zruBmGo&LyQ+L#WS>x8tg4J2$LpU6 zP!@3z#X_4K3Fe71ljE#Vt(o0`$Lk**SiMf8#p;!zDtA$uM&a|jsq{uN&fm3F zWgJE6^PBvXY=69=_b0Q?5q=H@N&2eYCX-Ph9)HVj!Y~j;_x*~<`@~Mt2epzQwH2!( zRVo!#ca}W_tMNm&+rZz~1PCCp>T-PVoVhb{HQlH#`32{t(KX65f)dc&2&vl|eJp0_ z1SLK&Em&tX)MyJHO`DsFTgr`tH)kxkAi*aQq&?-fM(YqP#n|&TDCV>1&{13&r&x%k zYk#q2{F}9a3qtO(g1`g|4DW!c9f|0G6nsS5AJ+{4!7+58KnIWWEXOBr0H^%bhCw-V zDjBvGu3C@c=-zla8vDM_`f{+w{5k%dzrGKvrc&=t#Q`;y;FLRH!MLWv=S4hR$9$N_ z6R#wQr)|`)yd4(V_kl=a0J@KRRF>uQ%O+xmnt~K}DJjz8c0nF#Iicj?Yd7+u{{&Qr zA^#>G0sGvSpP=~zvyvgo0tHLYw~|bgVI*4t&6B+(JONje`XoUC>XSSrIssghcO@np iN&2eYCIJ8dPyzq|5dZ)H0000000000fPp=exFu*EB~R=C delta 2780 zcmV<23M2KPOrT7#SqKI0t5Ty)lU@ike~L8nnu=~EnYS2YGMOY>_mf4iRcb|Cz`XqT z3u=s>^GMj)*(G)Fml7y}AC} za8)JE{Mpc-#xvPgpJb~3+WF7nL!A7cPNd9~Ji#n&tCK97A2ge3Fp-g;YVlmofAS-v zIEnl$f18Y(={%ACa2XU$n})92&BzaD>MFxQQYA+`8ij%U7zc|;&azdKL{9xIAAULs z=c(E`9ESO^q8=rYqqce^4o-eEgxc&JF8jU>bn4su$8G(|%qK z<1$f(a^x?j*;V}TRfgk9mY+7Oe+UWUX}bJZiNgF`8mbcck8QQ3Di7f>n-qzaNjMzJ z;zX-Tut>8w`fYVctP`w4g}sgyy4XZ}nlN?Su!_&)X)4z-%~h|<5q|VDzjK(x4@$D^ zA)WihCnF9FBR?%c(fX}uF2>eeH(Wh8D0v;l*}Lzw4x9Hy;b$@cB`S8 zk7qL|0}skzW6*cUaK#m-Nrnj|)~@6lNbFt74Ujmyl3O5gH|Ou~IlGrYLeB0tATjafX5-7vW(h1(%rTeP@1biia1wD{!G#wPUks|>GK#mf9G6)wB#eWd>Qws z^Yy=%pRRhR{j>MW)lix*hsI$xS=?zs95th9{%@2ob@EqpnT!VEY$TI>!6pZpOtV3{ zxQoKHSo@@ddrQ-sSt8|fmDMa3Q_b_<$vQrfX}*R@o9o3)W0QFh%;$|@zG%d=*<-Sp zW%+8Vu~rR&cs2^hf9fylP-cEOP1O&F%@LTdztvGvzG{{E(3b!08Q1sKxWsCWp_zv6 z=`9SMgQ0UcbmUSy<`SD+sV7{*paT*L9gtAyfP{R7e{o8n*Uo zuLlNCzVpu=N+ge=>$eLo@rdNosgcO>WZ(zI{Y)djeOaYye~fb;=TBLj1e3wNq1l?= z2o}OHE09=6DHTY(akfY_9j)c=1JT7Gx*SAu{Ty?NO|G94E@2=72?Y^ID2PBpL3B$% zfLKQ<6+paocc#RaObZZf1BR;a0w$3NGORwLP_Kjm)`hzB_x7ya?zfagm*o{$r)|v@ebHJl*aTy z3|)r;3Hb;vy?SAb{2)#nQ-7J^(Y1f}nzWAcDgb!pZ2{VqiynDGK= zgibz?%dPRh4Ip3U!2-sil=J+0Oj0?x7n++-g?|~zS}Lr-q2}u+4MTe7dyu&6jvn^hof%e@4rpU~-MqqF{0j2NP%*m_Wn81J5_C5QU%;d-JyZ#Oe8ZypAc}(tcCtB`tv3xOd7AUrnXIF{N=&wPnbbYu z?b%=yc>DfH@GWtRZfOIJV^Su$hBFB?j7gwjOmdBDQ6|kYCP63kS!-E>?&jbNleDD= ze>(A6lt~)ZM^TJPt`R!jp57XNEle)%@~LpKhLQl8-ABvWvY00GC7$sENY+tZrRA%+ zy?oUi)7;v5+9h(2#YAw|faOslxyG?5kzB)x1R6#p&@du_h7oC%5eX77B1;liaCe_v}Uuo9Ej7L!Ix({*c)n@{ZV;RxWep++2_ zai|jqXgHG${nMQ{z4_N;ZUaz&^Xkp12mk<3T7RoVN?PQqq5AegAy<*f6E*@ zf*NYhSW6Fd!gocOF$YBPT@kJUyCOB4b2ybhe_P~@ns%e~SthZ=W)hGSX5x`Q+fOJJX|DVAmK9n=ZKvdq6DsJ-Tlwe&zI{P`=h>7Xb!Spp4P zz}D>4d5y}S^ZfOVdU&qHwuvWSf5Zyg+{dN2hf845p)MYvaj1<4XgHS@j66ugxCCAt z#$}n82X(@@EOYY+X6IooJ$m zs6L2dWC9H*)2P|r^BS3dcW0!Q`l>)_=(|wrj@8;@BQV4sD7jO_rtTh~ajClpXgHJ= z+&xIcpakQg)1^`7?m?X}D9hYE26OkYmLBNDGbxnJ-GibSloi}PM$PV?*O<&k1&-o} z?ka1kt;*`v*mm{g3jkNJf4j=&)*rY>G8xMgIi03T5a*|B6}f3${nh`KIJjN?f&7a+ z?=PMW!@|<kQ6bJpYbYS^9i|5N%ma5#vSr$jn?#18vEv zfkItcaud=f5w(7h)H;6qDo)}g;8TyQnf-QlQMo+GHu?qYgjN;Gk`zUtnC3$5D)iB; z;ztxYPZg)FR#2fH99q^7B{PI+3vX5%uwH2o^?I<}T&o3p-3e|a<7jc@$`MgONWL}V>*c?aR^ytbsIssjicO@np iOe{xzB>?~cPyzq|5dZ)H0000000000fPp@fxFu+F Date: Mon, 24 Feb 2025 09:40:49 +0000 Subject: [PATCH 180/719] Updated results summary [skip ci] --- README.md | 38 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 12 ++++++------ summary.json | 16 ++++++++-------- summary.xlsx | Bin 9760 -> 9756 bytes summary_results.json | 16 ++++++++-------- 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 0b2bf4804..39a777079 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,39 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 09:38:06,721 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 09:38:06,721 submission_checker1.py:3270 INFO] --- -[2025-02-24 09:38:06,721 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 09:40:44,420 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 09:40:44,420 submission_checker1.py:3270 INFO] --- +[2025-02-24 09:40:44,420 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 09:38:06,721 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 09:40:44,420 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 09:38:06,721 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 09:40:44,420 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 09:38:06,721 submission_checker1.py:3286 INFO] --- -[2025-02-24 09:38:06,721 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 09:38:06,721 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 09:38:06,721 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 09:38:06,721 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 09:38:06,721 submission_checker1.py:3307 INFO] --- -[2025-02-24 09:38:06,721 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 09:38:07,785 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 09:40:44,420 submission_checker1.py:3286 INFO] --- +[2025-02-24 09:40:44,421 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 09:40:44,421 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 09:40:44,421 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 09:40:44,421 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 09:40:44,421 submission_checker1.py:3307 INFO] --- +[2025-02-24 09:40:44,421 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 09:40:45,378 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 19.1596 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.285 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74777 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.93669 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.41546 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.406142 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.2869 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.437105 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.2869 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2895 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2069 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409194 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 882e31925..5af3be1c6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -807 +808 diff --git a/summary.csv b/summary.csv index be325987a..b50315b70 100644 --- a/summary.csv +++ b/summary.csv @@ -1,17 +1,17 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,19.1596,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.285,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74777,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.93669,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.41546,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.406142,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.2869,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.437105,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.2869,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2895,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2069,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409194,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c02061a54..457299d20 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.1596, + "Result":17.285, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -69,7 +69,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74777, + "Result":1.69756, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -99,7 +99,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.93669, + "Result":7.41546, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -184,7 +184,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -214,12 +214,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437105, + "Result":0.435308, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2895, - "Accuracy":"acc: 74.600", + "Result":10.2069, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 44a6b6933f2e04603d9ad853e207b60293d2e2f3..bd433a3a0653ba417290fc953495c5a8c141e58d 100644 GIT binary patch delta 3962 zcmV-=4~6icOq@)xSqOihPBczQ2><~9K>z?40001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`OuoLh6^#ukO&&#zFa=3%BP2(^T6E9~ycxk8Radm4|ScPV&UcBpeQ9KG3QX zER!^v{kFa&HVM|D!rsIRU2LO0O|*2|u=3C2X(~4{&2@jRs}_FrQ@?WP9k8}$7)TyaIKMVbjD*1qH#NbG&d4Ujndl3O5gx01c?nz*|rOCU&hxq`Iw`>TrU zR~0v}DsF$v6ni~w;hwfwyg_=(6)lf+^>xYh>yjHTp`UAMlD*zqu(wAROW^KTam*E@ znI~K!2n!#9gxo)$fP@VEXCNWqe*qE#-B%zX(47Jaxy`yjLOz@`AR&X>0}?X2=Uift z(d`2X8QlvYA)|Wvx zNxpyfNe1_prZ>}A%GD~XnJ=cA=e?6nd@Pe}4U;rCivoCt8bfOtx~H2M zItN4NaOlXbcFZLQLgh2-+6gnWG&;fr5`3nEylt9PBT?$=Eco&3|Z{)MxaMHgX zXgvAOKX)jRJch2{F1W-al1ryXBFB?~ALP$7jqLGdovJa;cw9WCQ5;ML_l9O`dLvj0 zqg8>#I=WJU#2aUa#FnF(ZV`ws2GQjplKbbFOKfugoNx&P5lASAKte$T5(=VQ0wRA8 zXCRh@cR@G>(JcWHG@gR!QV@Aei^N=TiAN-t?*9&mb#$cyh_~+ElxSM6B?=yUJWQg; zNhEjwF_+l%!G}v2i9kY01QJRjm$>x7$15Qbgp-^0v?RRi5l+r%L629!)}b_}4`S## z6iCQNaOu?x+sqH5q%rkZ86I8xXRm)r>*!tu0I$3q0A0=Vtb&P9=$OA%IRTK&EyqCP zkanB^4MPxVD1%5tAp{zi&WBqK!Vo2(9F`^MZZB?0WFKB3QS&a8n%Yg(OSZ473_a}3cS&N zB>2Xjpq+oIl2NP%*m_Wn8hFJRJ=9_k6*yEy`r8j7d-l-D)jM(A{2qVUm{gKqa0@ znWRyD6vde28lltE>7Dl1!sPNUn+lg3C<&11{cJT`=F_C`G~)-5tfP08RP}c+%#Z$lt_QBaV$zC*Ki_%h7k!gj7XqiL|SD;f&`4nk_6rD#TO!JNe@)Q zwq03NA4M@DfrdR)T5kx*e`4>omJU{8(%NCtFg0Db3e0?>z=k7$$%guHfX1Ol9H8M$ zR*2{3v4(3GZNGX2TmpvB<3Sq6CD1S~%d9*o0pqgF z%p;hahn4g|C7erU=mAlD5z94T5nHpV=idSIjaqxYi(P+jPku0BnR>zFrJ-Xo4ek`N zsk;YgT|^!wBG zU-PHoe5@pYtbZrK&VMIB&yxkCtL0QG>C-~CRr!Co$UdRGv#K(D9IbyLKv~3L6o={4 zy7v4h3YJs9GmD1rSB+ZE!#|et{k!0ueef=@Km4K?AC$I|b^TeI^=?&5{sJ|tbGY!w z^4yQd;XF~Ma+Gyc*Rlujc>S9LtJi6?SiQ1TCQmYsk4k@+jqp+N}`9fobyS z7IMN3RsfZfgq1lF^51f8@}dM1^IHIsXQpjgo|%)BYA9kA6^EMbx*Doz`fml}ErP9q zHX8DDG!&yV7DF~P#b{{Oz!toWe}IxQWo`VLrh>P@1@I~pZ8XbBPHf$`R(cq=g?YLW zCCT()ZxTJIy%_is*7;6K`SfQgTPQ~Z8=9p!y{0jGq z`_(E>g@CdxIJzTgMpkK-3iZ%E7m&Bna~WRc&hwH*z&TUw7k?{^Q3M&3e?+LtL{YwE ziTFuMC3fdbsF%3r5O*eDQdT926bC0Dr1y%Qv9!^~T*J~W%fdal-GdL>cuWM5r6@CD z)`f4LosLDt^| zlJh(1YSR_qtcdZqBJAf)@=q)JZ+6$!PC*U;3+5_0>uNR z#?B^aasLY_3XvbW)(yg*K4vAL-rlH>Oy}Rn$oIVg@| z6dxcD8@^>@gdPC<01bR2^!%^^P9+c;0}uNMU0r;go?kBK)5Y@gf9Py}I;~GzD&3>$ z0X7^v@P39z<$%_m+W`8p_oF3fm z!If%kaK8r+dT_;;e=Uxc05*8fIK(~U5G&)b!F&A0J^tbze{qk$xW`|t_zUqG+ViWK zMtYt_TVxp(#lzTQlx|8n7t$P*nblWIT?}2*c&$$3sKT~m_Eg=Q?Y^LFdpL3uF}^$8 zUm2;o7hR>qUP|`oOA-SVydWiq)e_ICrSodB;_`1q_hH?K! z@LIAzp3xSQxXT~>s~y9kVZSkw>dx_zW%*{dH^lMd+u%96_wR#uVSL}1eqbkbu!*E5 z!X2sWi)5qLvApiKIO^D4JEp*MEAVsjE}EJAI>$VfRH*Lj=N?+s` zbF6&!eVa{Ye@Ey9j%6kQEv{~mV}(j2c8Ws{(qm(wZDZ^?i<-h}t2_=W)C?*`o}eam z-Fv5+y8LK|AW+Yl_p*z+wY=CkO~QCu6RScs+De_+e_ zH`@R%2)V-w0uwASJOEQW5zzrD_=L1St{nk_V`xEv4j$)Oj!)hIPWh`1qjKg{GV~U% zT94x7-g!A0yROT+*=UXVbNsn{c^g+vr5;Yj0X3E2lsjO-xTeDAMLb-~d|1X4uO*1b zKI&KA4vXyjKqN5$-N!vDAG4?DCDj5o1u1S*Qlv$>A`i4Av@E}dkr(|ZpgIisH}MGA z=f3;|%^$OpA<6;@pH4JRNeKV||3Q;xBvt{+lf5J&8y`%ua$y7j0CW-n02BZK00000 z000000Dys1llmk<0q2uEB{~6FlXoR18?6_w^d$iR08jz|01*HH00000000000Dyry Ulei^50gjXRB_jqvBLDyZ0KTJoh5!Hn delta 3954 zcmV-&4~_7gOrT7#SqOhi&$p6H2><{9LI4050001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`Ouob7X>$QH)G-@k&Q=EF?Ysx-=5D!P?q-eQQ!WRh&%PZq&esTFYn^YY&>s4;rZ z17T-pm(;yqX0&dp1%hOH_| z>di;qESk%%`Xm$e*WP~)AENm8WGZErrAk?y7%B;^5>rU8t?z;i~W3Kt3cpua&%%yD0iSe{(wOs#-2>G41EoFfL~g{~UC&O7pSOO?oAa@?HVs$L4N6u=e)jHr?Zei6o_M49PD8F~!wqe?-DzlL39m)vj({aj0v>{V;PYL5(-z~1lTm@CLI zPq;!57Cr(AxqdzY2|4kffrNnn1xN^VUx9=`cM2rrGV22g`EbsFgq&O#NXXee=MtTq z-2sr0vwHy~iq;Hq~6b5 zVv{Uio+aQb@hj+oTqCC?;r-`h&r{SHZE$%94mq9YG-kkK=$vx_(vpwh@@3rP-q-(L ze!6l`2WRhBtDz)Y4o!k|y1Z-pQP>J+*}q}7)X82gWIP@Q^RbMx1)ChEGD(NY@-7UL zeC>ae4DXGm)=Fb3SF5a6zL;vAyC<9YSSHyTCTVRJGmUNLp}$x({Kc{n&F7Eta-L?Z zsm4Y%^rQJWn5e&~Bbj=^EKxrkw#Hz-{#Hjx*{W4$LtFh_eGpE*k?Dl1ysxirUTs);w>`#aHdedxb4Sy;0 zP6ZO{D5V04H`We`9jn>4i$Jt7h&Bh2TtCNLVv_6Ugi9ERKte$T5(*-aP!R1B5P5$% z1F%1oLsc0CE;QCpoPrGrfw!8D1mC?eOo}Df zSUSa$YdDrb!>|Mzh9%cWl4hm_TUSaw8aITa4c!m07S7zs6Zn$LAlib z54fzO#ww3hHv`)~SIwqNqiDXwo1#a8?=>101(R#6HU*PwIG8}gzyulwCfD#)Y`Y9h zkbr?%lAymm_yQ(vaZx4o@ePX_fG7?o*vaN#w%;_EFe62HnyI8mmK@R6UlKxBc%L~z%D;ZT1fxyCXmkzB)x1R6#p&@du_h7oC$5eX77B1;nVw+CN{q%AJ! zggtjcjyW z&SV8I4$?3tfrc?zX2(GZ7?Wj&9Kjqpti=VLa3+~42So9eE7yO3m1~{GoPPz%w`$W7 z#HXW~qN5j3xo6bTUZ?~%9O~2o8jD(WfQD09!K{Ncj7p$kRF?U5Py$9}nPW##L(Lg$ zaX}}1SA-dJKos8<;To_jQnNXSQ~C3^Mc$}s=UVK#cFd;G+jh|{rCop}Fy&C+4$xTC zxC1mC%L>*Vq+x$p0{e_fu`F}%piUT;W&RyO?KNkt#RZ-4=daABgQD1E2{dd0TeDN= zH7bA3^Vc`(;kgpKCZ22&tJ{U!l}-Vdz@kH4JV0Yn8xPQME-M&$kcM#yyf}=@GA|G6 zgmGEs<`K-!!&+R>3FneIdO#Fk#BvQ-#MbQU`FFs4quzg>?_$^7lW&YzLNhR#x`xTr zxmCoZ_8y?Isl5ki7@0^z$wV4TCeqM_tX^jCK?x|CWeNH^v-hwT7j;56Mp)DUL@_df zhLfq+Z0~uE%)h%cQcHbRpwzW}C^gGy7uX1NQ2-@(ikQ^h12i^u_W%utvVyw@X&97X zJaoF$%iMoGs1pWdnY%}4?jF|Sf=-+cg_5~@P!xl*g1bkr+1>LRlj%6eQT)(dWi7Q; zS-tAJuAXcGU<-Cv+1mL7_fWpzfxk!Sry^HGpn z`n}ou&-v3}K2efC);|x8tg4J2$LpU6P!@3z z#XmMChy-uUW>Xo4?cTt)~;q$wx^hPqy-?dd`97XB# zoBRiqY=69=_b0Q;5tso4i+fS9lV%ez7O|TvDjlJ=plY~O1xCzMi4 zS&HtU#Jkk*AN#pk7&wrIMnWeYhOlx4LKJ`9?kiob}K}2 zV46I zErP9qHX8C|G!&yV7C|~R#b{{Oz!toJ3?Zjw!rJ(Co(SFs7r?7jw9z!B86~=Bt@I#h z3$tV+N}B5a##^;z7jc{>Ike#`_5*TmD(tgK$0en!Xf{Egz*rkd#tI!(Q6fIKHODMv z<`|Z4Sr+cWNe@0~;}I29 zmZHprSr@)}b~Ufjz_g}2K)Vz|<11TmGxxeQqz zCu?(WQNB0Xz9QetRjp)~R(eW*W!A)Uta~pad|l-| z!?=GVcrDo<&uEKD++`2`)sEr7Aa9JMx^sMFS-zR=4RQSVHh7Ni{rliu7~eOhAJ_>U zY$C3Sa7XI;BH5^QEU&vQjyiVNjw$fm3jCbBi)JRj4$@DH&*%nNUjP2|bYch2O}>9H};wlVgcMNMI~RUQWwY6g`e zk5QAl?!8k@U3N4-Kz3j_fwCbzt;03N@4F*a&xBInh1$DnZ zmKXLrR`Lfw<&nTqv)UJK2n9*{s@*1&O&}hB%WlFj5JmU>ipcxKPSOXpk|4Det0Gk@ z6;*eZJp`-qL$=$%-`4~PAhGIleD9pOGjcWEs4n>h=cUm#$}@rz(A)^A+Zuf=X6Xba zJ}@m7kMk_YCvO0!{MCj*Iddu* zwid2hkK*XwcsUyTzR&t{u*Up3{+z$Q538n9?@q-5HI?9$J7B@Mro!h%JY2_on8y>Z zB#5VN)UUi97TNcKNMZoGk9$;><@3uXVuqT66n7~p(&BbO9%(tDK@}mC)REHt| zCLRI%+?SuA`2(|#A<6;@OV78GObGx0078>uBvt{ Date: Mon, 24 Feb 2025 09:41:40 +0000 Subject: [PATCH 181/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9756 -> 9757 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 39a777079..75fb0037b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 09:40:44,420 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 09:40:44,420 submission_checker1.py:3270 INFO] --- -[2025-02-24 09:40:44,420 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 09:41:36,136 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 09:41:36,136 submission_checker1.py:3270 INFO] --- +[2025-02-24 09:41:36,136 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 09:40:44,420 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 09:41:36,136 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 09:40:44,420 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 09:41:36,136 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 09:40:44,420 submission_checker1.py:3286 INFO] --- -[2025-02-24 09:40:44,421 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 09:40:44,421 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 09:40:44,421 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 09:40:44,421 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 09:40:44,421 submission_checker1.py:3307 INFO] --- -[2025-02-24 09:40:44,421 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 09:40:45,378 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 09:41:36,136 submission_checker1.py:3286 INFO] --- +[2025-02-24 09:41:36,136 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 09:41:36,136 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 09:41:36,136 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 09:41:36,136 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 09:41:36,136 submission_checker1.py:3307 INFO] --- +[2025-02-24 09:41:36,136 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 09:41:37,099 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 5af3be1c6..91dea1ec9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -808 +809 diff --git a/summary.xlsx b/summary.xlsx index bd433a3a0653ba417290fc953495c5a8c141e58d..75b3e34a28f9419c014ed54035a78362b60d35a4 100644 GIT binary patch delta 340 zcmV-a0jvI;Or1=yNFW7rafx>(lT9EVf6Gq8Fc3xed`0Aa;yg+PR+379x}p*iK~(H4 zd)ivW57~yae~*)#~{p#wOysP9T<2-6nctTtf102c+2a)g-*?YuVxW5;$NsY`-G1#zRTq&}2(e~S%e z-*gK&Pt!S;;3=n`;vN{AiHJrdXA{!=*meMLhM@%s$~&AT89sYGSm|yy49bO53V$$g z(|Qso_uh%g*mYgf6@xY8&+vNn`aY~03)P>B0jf*R2(v)Fwsnawi*UG>I=>1h-U#4N zhoIkh8y7kD0gplea*TUYK8xp`LHlRlDJMvD1YKk};o43y)Le-lrD mWA5uuQ2zn5jv)&K1#)qTcP5jbB~$^Alk+7b20|AW+Yl_p*z+wY=CkO~QCu6RScs+De_+e_ zH`@R%2)V-w0uwASJOEQW5zzrD_=L1St{nk_V`xEv4j$)Oj!)hIPWh`1qjKg{GV~U% zT94x7-g!A0yROT+*=UXVbNsn{c^g+vr5;Yj0X3E2lsjO-xTeDAMLb-~d|1X4uO*1b zKI&KA4vXyjKqN5$-N!vDAG4?DCDj5o1u1S*Qlv$>A`i4Av@E}dkr(|ZpgIisH}MGA l=f3;|%^$OmAqxZrtrxHKC6k{eQ~{2Y^d%z(KqCME008Clnydf- From e262ee1efe45589b256d53b8e17387fee29bb677 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 09:45:01 +0000 Subject: [PATCH 182/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9757 -> 9757 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 75fb0037b..98a7fb696 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 09:41:36,136 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 09:41:36,136 submission_checker1.py:3270 INFO] --- -[2025-02-24 09:41:36,136 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 09:44:56,127 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 09:44:56,127 submission_checker1.py:3270 INFO] --- +[2025-02-24 09:44:56,127 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 09:41:36,136 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 09:44:56,127 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 09:41:36,136 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 09:44:56,127 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 09:41:36,136 submission_checker1.py:3286 INFO] --- -[2025-02-24 09:41:36,136 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 09:41:36,136 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 09:41:36,136 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 09:41:36,136 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 09:41:36,136 submission_checker1.py:3307 INFO] --- -[2025-02-24 09:41:36,136 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 09:41:37,099 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 09:44:56,127 submission_checker1.py:3286 INFO] --- +[2025-02-24 09:44:56,127 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 09:44:56,127 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 09:44:56,128 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 09:44:56,128 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 09:44:56,128 submission_checker1.py:3307 INFO] --- +[2025-02-24 09:44:56,128 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 09:44:57,133 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,9 +34,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2069 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409194 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.12384 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 91dea1ec9..da1ccbb5c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -809 +810 diff --git a/summary.csv b/summary.csv index b50315b70..1ab6c284f 100644 --- a/summary.csv +++ b/summary.csv @@ -12,9 +12,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2069,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409194,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.12384,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 457299d20..602b1eef0 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.409194, + "Result":0.173499, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420903, + "Result":0.420446, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 75b3e34a28f9419c014ed54035a78362b60d35a4..470729f536376473988987bb1de5ce74d601b8c8 100644 GIT binary patch delta 2743 zcmV;o3P|;xOr1=ySqKD2CsGoVT?i(B?Q^2Y7RJBdzk;IX!%Wo%Y2Haix01|Tj4_!^ zlCAqGMX*(BRb0Tl{PzoLjGpH}*xA`7b?=uMZJu+w5Aw5{=7|TtK29gfy-d<5p0!n> z>8c`UVLXUtqqcf|^?A=zl{5=xgJ2TRWLtfbsrqZ@KL-zS@_Ra#GE?#dv$U;$jLZ4WgO4%CMi*$q^5SQ7Avg z;bJOh*(ymQCqb4EKOIN&RP7uLqWoADed6s6q7 z@$bc(lR;b63uz0qpO?e9NR)wp90rR?b{Ri>mCNx zOtMq0%~pG8LjrHNiz8HjkYOI9LJ$@{G7@tAd}1Wz#D8Wa1pF_Igh2O|kr3!k7zw$| zx{QQ;IH!z+oZKEGA!qjtB?dXWeMUmg?l~hNXZM1UkhA-Zky!lY=J1!B0|{(W%n?c) za_)~&Vv>44G7?hnCq_c*{frWiWcl(eVZIW-GCh!McOG?|2=b$TVNWq^$d!7G5`d18Q0N#5g^rPsukbH! z1#}qhQRphdyG%IwMn2sRC-wE1!ISU&GnW#HF?9WQjuM}LNG{w4i5!h@gRr=t*~@QV zR;l(zIgj(FEKb7l?Y*Hnn!XnFFj9qQ`^iA&BJqIYNm; zuAgI+0Emo)g2+fHh>V1S=v9D-;Q(Sqc$W#MAbJ%bGJ~fedK5&Au}I80N_--@@cwr| zY@(DJAl`U?J5!=*d$uUK?eTdMeI$`w{YNNq=#39bfJ8<@Nn|9HM3i{+#>cN9kqIXk z?MX#=*C(8u(uN*(0P9d1(`RDnI+T%+kKn>@7Pe^+#_8T9SY~+ioS(fWZKAvy0ABiA z0D7A5+a(jB(0Trr?F2wFw;VAVmyF|>(Ex&+hBC;1X()t@#-sD$M#EvqC7>KuCFpJr zZb*cd9@B|W@|;0bpNXQ|jOUC-=;Q;r*c$)a0PuG+on{a#ub!GW`iE6WHdk}qX8;W19Rg>!&1p4 z04ghg5_AQ9lyC|yJ*E@ipj6VRJ{LvjNKQj1$*tkPMJTss$bw2k*TfHC*+h+Xrpooe z#CnfdGd#P5O8!aHp}>bXwnM2zjcZUUQG--68laNV0F|ggDvc^CnFK&(MS||;;0u+s zrN?xFRMMzE69qe{j7I2!Vrl#zVA({Cb#_&M*8|&ZtD25um)t`N{+Z~Zz(;#gJ3cm0F%)Gn5e-;tXTynlK{Z1NYLFJd;ycT^tejs?hQosnJ5I4d611@nr|9R zvYGSOnQWrGT1=WdObXYu{4yr_r}u{fpNQM^i8iBgEy^TnkV!@ZOfni^5;dMpnY5~Z zm}EMkTdh?Ix|@S9OwyJf(}{0VCTUckivlK5BXs(7dTacRFuAzPr^3Y=N&;kdKV8n2 z#WX2=&G-Q%o2ahV>ebp_y=so{+GRJDB@PpTrUBchM54yEDUql_A{h-3$!LH`Mgv6J zRYWogfXIpj-Oa%lB56yH=>)c2A*#=RMFEkF25u_tHw5HAvG>|Yfwh>lx0p0cP1o%b zGoL82;RwcLLwz`m#-&CaMuSY&u;OqUV3N@QlT~gUE&(uE<;M|V$bps~(+M&Oo*X6$ zSFWgGR<2F@a{d)4->OST5Sxzf2+Js;a>u8mzfj4zaHvs-(YVy9!)TDo8eScLP6Jdj z8lbYuuEQk&Dys}Tg8FGrp{2)kf)7Q&m%~KiLlM+44@DaG<{*_n|Eb6u_3T`UZPSjU z3B&YCEUUsR!IJUhP}>fpaj9>I(IA#JoI9Kbuw*oVWtDk{O8{6_*>?nW*PKF2kLd*W zU%{orMZsjrXkY=`uuPENYlVY{RCWe+SGrYVG+h zw!J;YgAv=*OKvX>ohQ>kr-(z{J&eYq?jA-1WO5oxCa0lfavDmeQRVJ`;Sx|Xs}gh# zaQ8qb2U>beC%#3Y1al7;1yI&7_ZSVE zdtPHQ8x}ZURR1;&%e{B3U@RTuh{rJTmBHh1^$P*YJc;5Y%AQue7e7(3m;{|^Jb1qx)M6I>v5@as--Yj-{db}B;TOgD zptO~|^k->a-LjYB1vjkD!8{nrvmhBovs9VLVP2@NNf|Lud{f*d}XW3U7Tg{ z^!eRb1_PNC?@U!0#&P!irZ^>=-)`vq3A4%(ehvgT^+u_aOdubB%}&EG5QOhMMdW+p zBuy)@l2iiJ6P1t%0&%nKZEF$#$Tp;Xdz>^)Td60G>zVy_cI9%pmu>V5)(NdDlqD&O zKrzjQ+E(bjS;hA#a-J$qTdkl%2XJUv-<8Y|rY*c$ZNPc~E(##!2s0Jhd2a~Dj_p8F zmjs6j;znCZeJJgJ78}aG=@xLFrVA{=Q%*g_Juo&C5sgUBCZzeX?Ev5mLkkj=cQ{Kj zeD-><(%ozrlnbX6{$Svy^(0R2y%UqM>$;?y4c3r9!=LMyw_(*-sQy$8P+f9Hm<8&! ztxJ4agu}Je`E@w)MgV_21pUU_xX7^&coYJVW89PSF?)I@ZdRzy(|jJMd7Kwb`ap_; x%okt%$gBP{P>w_XO*{dPxvxJ#{Rgv;A<6;-M<-Gevt1-T0R%VoMyZpYC4Z?6KH~rY delta 2743 zcmV;o3P|;xOr1=ySqKE5PBc!FT?i(BTXW*Z7KPu>uTZJxVWuhwwS;aiL(2Btj4}3j zY^Tl>C5r}B0YoI>%fF8V7{lI6vND-Ou5(^yptbhy3;O6*_cDjSKF+4fy^NDEnzvP< z>8c{W0MGHC4u8g91=BL@)c-%}DvGj+lpxLx(=(^pU`Qcn$XE=zfD{aA=!DYGFFCva^x?k=~eXbRfgk9nhl$khXm0yS^cZb!fY%JRhjvZ zZPiqjhj5rq^2Ev{91dkZ(5ezFlQf$Bw!S1b3D%*)-oy%BY@{IV}nAK3{u(o}$iZL62A9k>Nb2 zF?}9G=bQ_WmV5-4FYO+6zW(>}(^c=ZfA)U08cMR|&^Syd%R4QIX3g0&`*)Ttb+T6r z8IK0xd?e#+!6pZ(OwvKJyqkqdzV=B5_m-wN(^$&YDyx|G&hTx z#y0aHSS%XBV%dm)=JUsRIZw0IRAZwW1krpHj@4h(p-la7ny4QRnGLu(njr<)i$2Sev@=*X>h%q2FtQ%|^rK?fuhIv}CY0SWmE|KgND z$HQF;T}gNsgp+UNv)ypgzaD5j`OZIgD3LsduHP=W#3Pb_OQ%L6$CH5{A9}8)t{amZO<&5r{4Z(d8hL`{$TTY;ymc za0vquNGOOvLO}!)3Zh#AA`fREmV|dfI0exy0TDEwg6L8Zc}$DMTyTj;B$w|04v2Mh zr2>ez?%tGtXj-l%3LbkrOrpn0BzONYm)P{dhf5fVKtf3b5=tVMxb(rtDD3F{%nzcZG4)p&9$ouquSx6ZUIhTJyd3~t z&GW2+iBRa6zg0N_kjyQ|K;w{hoB$0&5NRlbNJAli1R9slhg%K85G9}-mL=$JFK$U> zB|T7yNAjGrs6L3I+l&`LBXshCT<*00Z2W16bD4$0}3hreR{U zN30pHRX`;^X*v@8@W!$!m0aT(luEAQR00j75@;BeT*Ij}%BTbh7?mXny4#B{RML_j zsDx8Vqxv9n1jy_h|RoygyY`d*$+O}2j3@PxL=#k(@jg~{fpqy4#B{VA7Hv>IvPwVNrb$#lZvz*&IysO@m1` zbN)J$b#$*1ljc5?x+lDX^F@JA?~epO5x3}xHqbaN$|TotCV_@A2{epJu5m5OWUGvS zNl*#hYAs99-ClfQl9u#9C7wx{q)~ko#hBz8q0`gpo%Ywle;;|Gweqj!~7udUtHt7dzSRrF9<;4l%~G+=p@NUm`#N+j2CB7uex2{epIpkYK> zWkiAmjL4D%-R;E}B56qvRKm7hSyUf?MKL0QhCNhTZwSbLV(+z<4pw5)+F{Z#HC?v~ z%zUE2h9iKlVxrklz=f==Eo7tki$xPpc2j`^W=ai zzH;RnuyU=_m-DYc`Bq&zg4lF)TeOSpnJyADdgs4O$=2e;y#yQUpm z6Nc#)JyN;_SOQNDwe0|nLw!3y!?CR3+(8-vRTDT6?~W zU2ji*Fk+c{!Q-W&V=@iy6tSth2WVXC?g1J`Celzck%p3qG?YxE%-w^35>PVB5_Apb z?qMZ8R0%y8VNrb$#mEF2PNq?_yXQ4B|L)01E&ZzkrJ?UbsXLZgun%j90w}pr#HQvR zpmC|W2WU8y70f+I!=ME1u_=^g<{nfDgR;!rV=!|ME9rqsycUI$nR`$agR+9T$Eexd z^BR-sD92I!&{JhCeXFv6dNp=UJ=p@l73`_9x%2n!GZ~NNiJVRoC5W=&T1Re6*MIbX zB@S-ae;@xM&+3cj!!URB`_uJb^QYl_tR#P|e<#4seC?LQ{3i;QQ@=BdhVNI6TF%2imh$~syWpLD@Gh`F{Gu2i zl(v#}{aKpzZdFVE0yV31xbVmF+>giMJW-}{lyy|svIp>Z{hI@;*J-p^y|Ps0E=tpA z_WW)l{h^HWcc!Y0q9}cSlMl)Ew;MWt0<+2yehvh3afx@6OdubB%TB{E5JmTVMdW?r zJW2&tl1hNOq7o88RO~E!+FHa9*@m=#kCUcpD|N+keD9pOGjh4u%QpH2>x5Pn%90dC zpqS=DZ7cNAEaL|hIZqX*tyWN>130v(?@DF}(-z*WHekH~7X^@VgqaHMyf*}6$95p8 zOM*iMaiguIK9qKUiw$MpbPG67(>a#lDW{&|9vGX6h(;u56Vm+Hb^vgOp#=%bJDepM zK6^b_>25X*%7s%3e=u; Date: Mon, 24 Feb 2025 13:36:09 +0000 Subject: [PATCH 183/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9757 -> 9755 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 98a7fb696..459d8189c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 09:44:56,127 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 09:44:56,127 submission_checker1.py:3270 INFO] --- -[2025-02-24 09:44:56,127 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 13:36:04,177 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 13:36:04,177 submission_checker1.py:3270 INFO] --- +[2025-02-24 13:36:04,177 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 09:44:56,127 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 13:36:04,177 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 09:44:56,127 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 13:36:04,177 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 09:44:56,127 submission_checker1.py:3286 INFO] --- -[2025-02-24 09:44:56,127 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 09:44:56,127 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 09:44:56,128 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 09:44:56,128 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 09:44:56,128 submission_checker1.py:3307 INFO] --- -[2025-02-24 09:44:56,128 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 09:44:57,133 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 13:36:04,177 submission_checker1.py:3286 INFO] --- +[2025-02-24 13:36:04,177 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 13:36:04,177 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 13:36:04,177 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 13:36:04,177 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 13:36:04,177 submission_checker1.py:3307 INFO] --- +[2025-02-24 13:36:04,177 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 13:36:05,119 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2069 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0242 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.12384 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.1507 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index da1ccbb5c..f8ba02fca 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -810 +811 diff --git a/summary.csv b/summary.csv index 1ab6c284f..2c67610eb 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2069,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0242,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.12384,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.1507,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 602b1eef0..fe841f0c4 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2069, - "Accuracy":"acc: 74.400", + "Result":10.0242, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -549,7 +549,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.12384, + "Result":6.1507, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 470729f536376473988987bb1de5ce74d601b8c8..77f629a3017bc727b210eb17be6453fb15d177b9 100644 GIT binary patch delta 3635 zcmV-34$SeLOq)!wSqKiI0yj}f2><~8K>z?40Fy5y6_b1jH5Bte*xA`7b?=uMZJu+w z5AsVl&7+gW2(y2SX+JNAagitkISdw)>@t4%Dx=Xj%TJqKgoN=VUH+>~qx@VNsxl28 z+p4K5578hS7m1Z|G#JR@M5{`;NV9nQ+v<>5Cs>6FXB{hav5EFIVd;)x7oW$|M6P36 zt6rBQ{1{|G=OBq6lw{dMIuD9ZM(i6#epAR(7o z7f8s5a|$HnDjI`dbqv<*Qbi z4{iBh$+*6+#wAv349zlh%PN`CTnwGdp(B^t5tlgRN01lmlA);W9a(roJ)Knxo{gKax}gT!s32rFTZ_R zrP>?iJkFo8I0?tM_lD+Z`d+vYhE;>aCQ7M6;fe0iNL?EFc0tp4ts{oOQGY~7nyC9r`=v9CS22VlsD2P19A~EM&;uFb*_rHGw zViTp*0P)7#nG#Lgvqi~mkB>?8If>-zKjIRH-uQ3{BN0d_i9kY0YW)eZt8pZRqh1*gBNP^g#?=hXM)t2rm3)VVeeFobFA6Wrj!3`PpmICd#V;;HAF> zpr`r1T{0009rL$rCjgSUH@K=oytj!>9xrMkUv z4|Kw*q)~kk#dc7EM(BcKY5bG$4HSR;O!QFjM~${i!Q>j(q+oIl2NP%*m_Wn8mi3A!(B+xJ-frb%jR}l#kFd{1wbTq& z&34bWUd^sSBKNR5_@KB^-Zw{yO=RXyBqn@2Bv2EIMG-2y*36)+6mB5liT{}SI zQriyDa4KsUcaVlr2{eq#D(?8GpqCCl5-% zxUBN>2xjJCEj`c)=aTt(KonoYat&C+Htgy7cffq3&YtgL+uBn+7qLyf7W$r-V!TQVNrb$#mEF2 zPNvbYx#u-9|L%j4M(V2rrJ?Uask^pWvJGp95-7P(#G&3Epz)};2WU8yHM~7Y!=MD? zaVV5k-X7EmgR;uoV=!+IYw3Ybe2YTKygew2L0QAwV>ImTd5y_zSl}pr=%%uf+Ul%c zjcrd)z5wuqN55nT+1mPg`&1?)c`PTBR0-q!bgd#ct*h_+UyA*k)z{-+SqF+;%5T%B3U@RTuh{rJTmBHh1^#cLQJc;5Y z%AQue7r#)GVG$`n8dm3E9*pE!kc^^Ps!ZfCFI3m^`|o7+lLO1wSv+69vQ_0S&a!y= z{BA6RflP{brm76%ID39moRZB?H+234v&s>e0R=9yaF<|{XA>_Lv70L@9ig_MYPeJd zM$A*?CMP93Nt~0c6B>V6iteD~3Cj;PY*-qCJY-RluMf5Bt7F~Q5EnF$Xv%Up)b4?6 zUq*ioITuJC&H15rD@1W%nmoLPjBTeW2uahxVOwBalE19EOE?6XP7C8euqHbI}j zSQ|*j3LRBZB0je@$1G*#SGZT)uU1(i1e9&T(H%`vx=PYSsE6*kfUJ$4%kV09o|h~H z&Y5Dr_*-ENLr8z2q(W7uit;6k#ZOu)u{&o%y~H(#s5AMJvMOn)I5+_zy;tmvrHwY` z7?y5X7Vg1G4?bw)5fxOHqRfO@7ruFRI%ZkMWF4$;moyZzt@ILNC}mfH9u}2e+{;LX zx63F7^<*y7)Rh zzg*6zi{*di(b@cTTA#L5x<}IkY}mwif17p;-zWBO%fvSv58IuVu|o_Cp*7F^A2Ko9 z7M8GO)o`}j6gz0gC8Y2Gd9o2LH5|ZmPy$kdNOIaCSV1W|+G|~ie>!0K{K?|dqa=>2 z`kB->>T5)^B886G-;52r2X}gKr5YRD>%sjVT=9Qpi(@5#4c;>janCr!$~bKB9)EF< zzqrR=+~Y6q@fR!pLbQhV{A#9=o~7XyS%yXNFt%8pZ%R2Ak_?oY)mKYh3|-TBtxn^x z!o)Uvs_xBpUr@F^92pH6-yQC+j8xr=j#6STCHwOwi2w>-(2~RQ4jaDunDq4W|6|*6 ztEPV=mVrOky%!O_uJWE?+`kdLmh6vbw8bRuvIqZa$8cbfH%3z3IX<#1-^})gIDUK^ zJV*Ebeef=f?;Fz(?1T8HhK zbOS7}e}8&9u>)dcSt%R__NE34BcIKZK5FlSe6$UjaUGJU=M# z0@twy10|i?{o3u?uaD)0{f?FV!B2T4aFn1vSgX(MS~EEpqyGT2=NDlJ1q#g6;U$wz zARd3qYQr!PMfd#*qrELZ(olr$KwGkEpiq~V?u0Z+M6CysTE~B1#de$ovg&el@0__a zs9YUn8~uWHLaPd8Ns1y+Omm@j6?$*h@dJvSr;5{7E2vNp4z23Dk{QCZg;%Q$STDdu z0i+yZrb2t~4Z+y4JxJ=3;7~!_YAdM^rQLsFL)kao0nXEOi6wZ-si*h|jLl3$6OywT zX?|=w064?Yf&}Fq&XNqDy&kM|HyZ}!!YPIC4cxSz#MymtVm5YNmvqHo4f!+txp{dT zR*i)^o{9mgOU?+hK)tqgi7$(AxRyG<2`AnP;E#RKZ@f*5jD5hP5P*zvPs&H}^xP$^ zQJtsxeVpcTzG$+76ibpWzK$cW`p-Z)4f!|m1Q>H)e}eiCvyUOl0tuo5H&ICm0091z zVkCM6F0yc!V3Xk_R{`deIVCy)Sd(-mCIt%2)ZrzQpCuImIg_;|J^_uB^d%z(KO+DD F001tX$j1Nx delta 3667 zcmV-Z4y^H;Or1=ySqKS7CsGng2><~9lVS)@6z4$L+1Vv^@0S^Eo^!en^0S-fiIc?$ zvwsV;pO?e9NR)vb28&5{89#iL(P*6Ir_C-x!g!J{|5c_@el87FnFfz-)l`*-XpoJI z#L74t3}kVlRV7@cSv>u1bx5oetU`sejupDtMEjbsbjPrZ&*Nz#*Dke zki-v4vg{$92gN5N_6;LHEn(66t!OUB)_+syYH9>t@|SJTJfERT+xPW z+HkYg(8|ZN8I-{f%78KGyJNUSg=LXpG7@`Na>YoTUCA{gad#y*jKteWcFHyRa!p7e zNV%vW=A3AxO=jD&nRr;LQ0 z+#Vw#XZH*x206QZMncZ+IU^xw_kxj-v-^#aSp4PY@Ryqd32ai#5lS3#?vGJol6pTf z5>oFcMndZSj1rGz`SL7bz7oGOJ%5mEHFnsD9x8cqbM6M?zAwTwx*N(-)X+o$zRQ7GQ5puLz(0Y zHhG)LG`md~che{>);{U&y{+l3ERk}#%4!vhsn%KVcpaa}G+)D{t@UDNZ-0~dHk{A* z!uet^p3NSU#VpHLQ+sRGZ5Ypn(MbJ89mp(*CaL=2pfzOX>u*hzl&@N4KD6b3CFA-u(+Su%Wq#+srE)WkMpN2 zPQvl+y`edpz85ZpVbvh9iBf8icV1S z$Ve!NjD&*dRe*@$0AfXWmkFmJdKDltgQp;R6hw@%NX$7(d?LB<{(pBsY@(DJAl`U8 zQ=(~mwkWyn@p%$`B#~VGM<{XVjSotIL`FhMWF(YClz8;U$FCrf2`3lrNkw?qC!Czp zh8}hR>rfigXJY6&l#!5+;KFYfwrLQ?>E0w*W_a|RpS>n+qP!XaUiw=AdYbRsB@?00 zdH$B|1VA#k95EV~jDO>p(Ex&+hBC-$D1?m0qx0cL!(qrJpd3~u=xz>fNQ9Oi(}_>= zoIzBdiK5$#=Zr?^@?{<@U>ry}&#%WMmACgo^YW?iFC*DVg*BMGS`Tam zrK?%OF1t?@{6_nsz>Rg=rc|QF6_iS5gC3}4G(aVz0V+`gbARJT!&1p404ggIbOn8s za0)FwrW4M2#D7R+Ff=WZz#1CNEM2&T(%Jsm+dXHE$JiCNS z{z=oJz=t=sL#afKYfvgtgH$pappwx5m8d}~jVdac1VCj)g6`(v3zf8`$8>^J(x^TY z1v{vWM(BcKX@C45VA({Cb#_(P1KVt?nvP?a+(QcfndqUwM~${i!9sC24ikZ<0o$iUqQ)c2A*#|BX$(~hGF!}LlltHLY6 zlJVqF+YX~~sc(nTAeJ?pJDdivWHf+fm3fCt09aPpcLa6UoI*>F=>+#*!KK4R!DPv3 zV1EJIuul?N3T#9WEPrisXJg01{$1mZMap+JJ52JCZi-*x5moPU$5_Ao4_drXJ>x4cS zfv7$c1!OWBB-3cv-SZlmfA`5qBlXpR($III)Lq*w*@rbm36y9Qaj3b6(RkF{!)Or7 z8s;8O15h&KaVV5k<{qvSfU?TmV}Q8_T6#<;zD1z~a}O5@~0e6+Ht-o)d%48&usjVKT)ul1f6L-c#~)mB0x8+&cQqw$+I9CMYB|y$YEZnuH`r2$?7); zmansTzICQmYsk4k@+jqp+N}`9fobyS7IMN3RsfZfgq1lF^51f8@}dM1 z^IHIsXQpjgo|%)BYA9kA6^EMbx*Doz`fml}ErP9qHX8DDG!&yV7DF~P#b{{Oz!toW ze}IxQWo`VLrh>P@1@I~pZ8XbBPHf$`R(cq=g?YLWCCT()ZxTJIy%_is*7;6K`SfQgTPQ~Z8=9p!y{0jGq`_(E>g@CdxIJzTgMpkK-3iZ%E z7m&Bna~WRc&hwH*z&TUw7k?{^Q3M&3e?+LtL{YwEiTFuMC3fdbsF%3r5O*eDQdT92 z6bC0Dr1y%Qv9!^~T*J~W%fdal-GdL>cuWM5r6@CD)`f4LosLDt^|lJh( z1YSR_qtcdZqBJAf)@=q)JZ+6$!PC*U;3+5_0>uNR#?B^aasLY_3XvbW)(yg*K4vAL z-rlH>Oy}Rn$oIVg@|6dxcD8@^>@gdPC<01bR2^!%^^ zP9+c;0}uNMU0r;go?kBK)5Y@gf9Py}I;~GzD&3>$0X7^v@P39z<$%_m+W`8p_oF3fm!If%kaK8r+dT_;;e=Uxc05*8f zIK(~U5G&)b!F&A0J^tbze{qk$xW`|t_zUqG+ViWKMtYt_TVxp(#lzTQlx|8n7t$P* znblWIT?}2*c&$$3sKT~m_Eg=Q?Y^LFdpL3uF}^$8Um2;o7hR>qUP|`oOA-SVydWiq z)e_ICrSodB;_`1q_hH?K!@LIAzp3xSQxXT~>s~y9kVZSkw z>dx_zW%*{dH^lMd+u%96_wR#uVSL}1eqbkbu!*E5!X2sWi)5qLvApiKIO^D4JEp*M zEAVsjE}EJAI>$VfRH*Lj=N?+s`bF6&!eVa{Ye@Ey9j%6kQEv{~m zV}(j2c8Ws{(qm(wZDZ^?i<-h}t2_=W)C?*`o}eam-Fv5+y8LKluYYhfUI=B0^Yz1|{K9&~_I#%)rKjo3oRf76ptv<7B&E#B+ z{sXh-7hwnmH}yuTCX-Ac9)HbF!!Qtp?>t51d*UQbE3lGO0@M?gkO%^Cv+Qka5&y_G zqKvI_khYI3GTS<%D(9qaGs_M zEWuMwJ;gmRHWLwzNX{mt`LXQ);0!|x5|np1OEP@+da%;nY#5XarxgBR;HLE?PVT)E zldzB7-)mW(hR18pEaz>a1>b0#)d|8CUwbc1_IPpdRe>?>J#@o2a zu@86@0+3_elkzcpdM0jGsLs=T9;bPn7ft#=ih|4+U;W6d{xeXHL;g)X0gkz^KSBKm zvyCCj0trVaQW8lC0094!U?h44A55`wVUyn^R{`gfHzhg&S(9=lCL1^PMyVzN002+| l000pH0000000000004l2I+L>{J^_!D@+Bh%K_dVF003*M_hbM7 diff --git a/summary_results.json b/summary_results.json index 46a2290c5..0f421d0af 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2069, + "Performance_Result": 10.0242, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.12384, + "Performance_Result": 6.1507, "Performance_Units": "Samples/s" }, { From deb760d51454b1c7abb5c18d0ff9202075a9796f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 15:25:35 +0000 Subject: [PATCH 184/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9755 -> 9757 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 459d8189c..31e6c9fb0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 13:36:04,177 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 13:36:04,177 submission_checker1.py:3270 INFO] --- -[2025-02-24 13:36:04,177 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 15:25:30,551 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 15:25:30,551 submission_checker1.py:3270 INFO] --- +[2025-02-24 15:25:30,551 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 13:36:04,177 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 15:25:30,551 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 13:36:04,177 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 15:25:30,552 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 13:36:04,177 submission_checker1.py:3286 INFO] --- -[2025-02-24 13:36:04,177 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 13:36:04,177 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 13:36:04,177 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 13:36:04,177 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 13:36:04,177 submission_checker1.py:3307 INFO] --- -[2025-02-24 13:36:04,177 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 13:36:05,119 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 15:25:30,552 submission_checker1.py:3286 INFO] --- +[2025-02-24 15:25:30,552 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 15:25:30,552 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 15:25:30,552 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 15:25:30,552 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 15:25:30,552 submission_checker1.py:3307 INFO] --- +[2025-02-24 15:25:30,552 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 15:25:31,558 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,8 +27,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.41546 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.406142 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.2869 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0657 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index f8ba02fca..7438337bf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -811 +812 diff --git a/summary.csv b/summary.csv index 2c67610eb..515bf1ded 100644 --- a/summary.csv +++ b/summary.csv @@ -5,8 +5,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.41546,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.406142,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.2869,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0657,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index fe841f0c4..76b89b658 100644 --- a/summary.json +++ b/summary.json @@ -184,12 +184,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2869, + "Result":23.0657, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -214,7 +214,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 77f629a3017bc727b210eb17be6453fb15d177b9..8d24dcbff2f1826597e41ebd47ff7f5cbe1ba45a 100644 GIT binary patch delta 1620 zcmV-a2CMm-Or1=ySqKI9nXfQQlU@jA0ra!P2v-Du^fBW(&2)jj7p$kRB{cQ8#fwAB}l-itVqxmbSvSMwe&zIzCo#^QGFCe=SZZXljPR$-y)P- zGh{)3rJ-x$2e53S#yV5wdSGJxj94=~yM#)-({w2K=8f%8D!IlrD3x5psRSBECD1S` zxrS3|R8a{MFe)n&bTfrc^3HJ(kGw5pf{ozO?ERSCMAgD*_dmLBNDHz|`es*j==lUyTox;wpp zHU37JT-@bT;bILX0W!OvE@#VPniRff`~Z?oR99>DYHhDxHOF`DvYW~hhl${(0o$iU za*bNn3iL6ZY7ZMfFh>BNAxXO{M*Y zfcz((y*5%{Ehg?t- z2^f=AejLFJIjp4zI^j$*PY#IUD_5=oE7vA{IsXcjZ`GwEh)qX#gk_Xax#QE(U#J8w z9BR}78kahCfQD09!>fZdj7p$kR94w_Py$9}m0?FvKg}s?>48r8rU>)pfGECyDZ({i zQ>0;U4yW?x?~1%p&(4+DHtjf?uywbDO0R@UV9BAb9iVZkZ3k#Ll{JhzNW-WE<{5`l zS>@e9oiHk^+&hArYff294|KvGzcP~!iehsm(6IGu!#L5!tfdD!;aoC5 z4~XJxSgrwU*oHkl{|=aM)YPU$5_An_?qMxG)Ct`eVNrb$#mEF2PNvbYx#u;1GXL(zNF(*t zfzr@-q10X5EO`uTh!QBdPsE|#9-#54w+CoAlr_9PNW-86y|%>eblx^yCWwPk8iukgct+ zw@+m(H{%>{$2Ra*?$*0AAV7c4@z6fOMjN;)h&A|UZ7!h4(7o+NS+1BD4M0pL=N*p zbuGXDPF6oTuza1x^W`gBRqo;}i>J@;#xfYlq zkY9ZDlS?2gf7@wF5w-(u$*O@uU0Sje(j*bJ9!P4P{CyR_l7LrTj_#c^cLo)!qimvI zuuf=IqAW>K1d3@c)V@R?^*Vk;k@Hk>+GqtOI)Ov0>aJjhFm2(@Y6I2_a8Up$N0=$m z!Fxk6cI*I>x+FMM5O>;2>O*Pw*iiOO_ki;>eZUetf92Fud~q0Xmb zfU1Hs!Yoj)ZB^jwA{?%v&TqnrcLMm+Dd;nA!yO&kGz?%PjL{Q3ax|@S9VA7Tz zs)X*|u&6$W;$VUY*&IysO@m1`bN)J$O_W!QNppuu;hL6T#w31ve<=79ahpET1{&9* zOmYop5@;BcK*N~i8qcOoT2)MfPUu!^Rf6v3;0u$qr3X6kP0A#V>Z2&eB-aR?KAqlw z8h;~9F7EQFaIuDx0GZuSm$PLtO$uK#egMfPs;jknwYFEUn&Z26*-d4M!$fe?fbCNv zxyH3AkzB)x1R6#p&@du_h7oC35eX77A}bPfHwRycq%A$r3EOsMQGFD}hy)sTQ)#~; zApeQI*G3Ag#iYH(q+x2hZkL$(M2QW5M*x!z_2B@GON}@{!tK2v! z0b{btk0Y2Nhqd%TC!9&<$pKM(<;pc+<=Uh#=U;*Ht-5psvFYfJu#6HacYHef3zfiy zLybB><5H&%&~Peicy*A5Q3*7R$|}1KO2DYBGVBQIr#WRUJAc{YK6yX~1 zP^4jR4yW?xKNWeSo}DYPZQ5}(Ve4)Qm0k&zz>-5bkcLqSG>pnB z?+!}9sH}4D2x_i5Wi36>3EzHYCLI*T=1QPp>(_>TI@hH(irjLRw~4@$tetn%^*X69inJZ=2# zq3=SeyS7=f4Qq%JD7jC>q23;#@u;^4XgHKLygf+6pakP_D3n#+9@GhgvdY_IFmDfQ z>48psi$ck~Jt&GnS;O07H0a1RkZBI|W0PuuIzX#dc`g;3R zCL?()CzDhOsjVzfiE41f6L-c)uLf zVix_ekni7x@0|U2q4VJv#rUAKmAv$4X>C3t_zsi*h|jLl3$6OywTX?|=w064?Y zf&}Fq&XNqDy&kM|HyZ}!!YPIC4cxSz#MymtVm5YNmvqHo4f!+txp{dTR*i)^o{9mg zOU?+hK)tqgi7$(AxRyG<2`AnP;E#RKZ@f*5jD5hP5P*zvPs&H}^xUjbou~PIoaS-9 zXtEiC6ibpWzK$cW`p-Z)4f!|m1Q>H)e}eiCvyUOl0tKQ1H&IEGUL;!q%9FYzJONab u_as39=94)kIssUdbR{MP3e42ulb$6O0XdVkB|ZU-lk_Dc20tSJ0002)iuNY} diff --git a/summary_results.json b/summary_results.json index 0f421d0af..c3e935578 100644 --- a/summary_results.json +++ b/summary_results.json @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2869, + "Performance_Result": 23.0657, "Performance_Units": "Samples/s" }, { @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", From e37066035a6030f0a3c163fac3408f192d314d61 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 15:26:59 +0000 Subject: [PATCH 185/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9757 -> 9758 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 31e6c9fb0..07b457565 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 15:25:30,551 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 15:25:30,551 submission_checker1.py:3270 INFO] --- -[2025-02-24 15:25:30,551 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 15:26:54,635 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 15:26:54,635 submission_checker1.py:3270 INFO] --- +[2025-02-24 15:26:54,635 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 15:25:30,551 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 15:26:54,635 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 15:25:30,552 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 15:26:54,635 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 15:25:30,552 submission_checker1.py:3286 INFO] --- -[2025-02-24 15:25:30,552 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 15:25:30,552 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 15:25:30,552 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 15:25:30,552 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 15:25:30,552 submission_checker1.py:3307 INFO] --- -[2025-02-24 15:25:30,552 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 15:25:31,558 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 15:26:54,635 submission_checker1.py:3286 INFO] --- +[2025-02-24 15:26:54,635 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 15:26:54,635 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 15:26:54,635 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 15:26:54,635 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 15:26:54,635 submission_checker1.py:3307 INFO] --- +[2025-02-24 15:26:54,635 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 15:26:55,636 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0242 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0152 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7438337bf..4dc06e60d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -812 +813 diff --git a/summary.csv b/summary.csv index 515bf1ded..ef81f7145 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0242,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0152,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 76b89b658..766b1b19c 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0242, - "Accuracy":"acc: 75.000", + "Result":10.0152, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8d24dcbff2f1826597e41ebd47ff7f5cbe1ba45a..e8b23554356d373ea2d796db41a7671f3ddb40e9 100644 GIT binary patch delta 412 zcmV;N0b~B1OrA`zWD^Ctg^*QZvuP8V0tJpC-w!3T1{TBtA6mA8x?dm53;P`_`GcSG zNZ=?zeXv%a*|lbJE=K{*6bLP&Va&?qV^b6JrttylyDT+Wb&4t=m=zY76A5i2xRh%|jL4{7> z(5k*GnITMDc(vMq^#WWJK*|wjDs=GP5R4r=fTS)74i&_mwvzf#+C4UuebYVQJWZEa zf~TB+dWt(>Y-S=Fk(|v)^JAMHz!`=HBq;B2mSp(s^a1>b0#)d|8CUHPrb{IPp#Ze>?^K#@o2a z&<8vU0mu+{Qa%<>&)YSs^E5BwG>`N9ZB~#Md`XJps~dUMe+J5N$iImbV90&_3F<$y zk0DJ11-gZhRbrFjBv%3IlQ|_i0bG-GB_;*SQF%-xlbtoB|ZU=lk_Dc20|kM G0002UF1%0x delta 411 zcmV;M0c8H3Or1=yWD^B0vT&DRvuP8V0tB)VEj6K1d3@c)V@R?^*Vk;k@Hk>+GqtOI)Ov0 z>aJjhFm2(@Y6I2_a8Up$N0=$m!Fxk6cI*I>x+FMM5O>;2>O*Pw*iiOO_ki;>eZUet z<$u&ud~q0XmbfU1Hs!Yoj)ZB^jwA{?%v&TqnrcLMm+Dd;nA!yO&kGz?%PjL{Q Date: Mon, 24 Feb 2025 15:30:15 +0000 Subject: [PATCH 186/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9758 -> 9759 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 07b457565..49b15d437 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 15:26:54,635 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 15:26:54,635 submission_checker1.py:3270 INFO] --- -[2025-02-24 15:26:54,635 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 15:30:10,431 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 15:30:10,432 submission_checker1.py:3270 INFO] --- +[2025-02-24 15:30:10,432 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 15:26:54,635 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 15:30:10,432 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 15:26:54,635 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 15:30:10,432 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 15:26:54,635 submission_checker1.py:3286 INFO] --- -[2025-02-24 15:26:54,635 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 15:26:54,635 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 15:26:54,635 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 15:26:54,635 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 15:26:54,635 submission_checker1.py:3307 INFO] --- -[2025-02-24 15:26:54,635 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 15:26:55,636 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 15:30:10,432 submission_checker1.py:3286 INFO] --- +[2025-02-24 15:30:10,432 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 15:30:10,432 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 15:30:10,432 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 15:30:10,432 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 15:30:10,432 submission_checker1.py:3307 INFO] --- +[2025-02-24 15:30:10,432 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 15:30:11,386 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0152 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.52785 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4dc06e60d..7ab878c68 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -813 +814 diff --git a/summary.csv b/summary.csv index ef81f7145..4adf3d9dc 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0152,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.52785,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 766b1b19c..6adfda2a4 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0152, - "Accuracy":"acc: 75.200", + "Result":9.52785, + "Accuracy":"acc: 74.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index e8b23554356d373ea2d796db41a7671f3ddb40e9..b935e8bc592ad5f2bcde35c6a471aaab79ca2fd4 100644 GIT binary patch delta 365 zcmV-z0h0cnOrK1!WD^A965#H$XcL+O0k*RP7Dx&OxKHiLC6iDf9)Hbh!!Qtp?|llR zyDiynN)fgLZON&DLQ`6D6VfITwSJJ)I(hrbPU0lsQ;(~e{dRUyHQ%c?`32{t(KX65 zf)dc&2&uO<`dBT}N0j)$v|z2#P@@BQG;i)IZYeho-kh=Ef&`yLkoJ_@8tp=`6l2eK zpqS61Lq~CAoMIuCZhwm{(1<`E&fWe0?8QO{MNn#Q`;y;FLRH z!MLWv*F`*B%Y0bI6K^Dlr$f{y-cE}g`#>Zy06oS%E1%`_%Oz@ont~LwloV-kzshH{ zBs3@Me&kL61*lF#{!Kgsj=66?LGuT*kReS21mY6l?vvppx&^pT?a3vRpe0lRk(2c$ LBL+hw00000XbY-B delta 364 zcmV-y0h9ipOrA`zWD^9sg^*RVXcL+O0gkf+7Dx&O%TakuB$H4e9)HVf!!Qs<_x%c^ zy)E0RLlL$EZON*ELQ`706VfCRwH`=no&0^}XA;P&%hA1a=FXsUb(Br?3)Ts(DwHKD zia;^Vh1yr>eY=hyP~<#SoHklPg-+nms=h0kAxv9%wc3F70$db8$`NKNbnxB~j2%0G zq%H{#6~vvklKN2EJ%2WoebYVQJWZEaf~TB%iaTIzW+EDqoXtq{W1Akp8HNTVDDQBV zWccj$V5Pg+&?^^CDg4R6P3t7i?xPd4v2EL=UG&zFKf|A!m$!b^Sg7t)3{YKiMwkWa zwXI8hS%kwi)cH*~@lF7LJO%y6+qlTk2RsS^$Pjl@J{C{U+axur^E5BwG>`N9ZB~$c zNs8jD8+p}#2Fh{BzljrI$bJ0@>OZrQAx#4Wx`mKclj0=01 Date: Mon, 24 Feb 2025 15:35:15 +0000 Subject: [PATCH 187/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9759 -> 9758 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 49b15d437..b129c9def 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 15:30:10,431 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 15:30:10,432 submission_checker1.py:3270 INFO] --- -[2025-02-24 15:30:10,432 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 15:35:10,331 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 15:35:10,331 submission_checker1.py:3270 INFO] --- +[2025-02-24 15:35:10,332 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 15:30:10,432 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 15:35:10,332 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 15:30:10,432 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 15:35:10,332 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 15:30:10,432 submission_checker1.py:3286 INFO] --- -[2025-02-24 15:30:10,432 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 15:30:10,432 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 15:30:10,432 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 15:30:10,432 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 15:30:10,432 submission_checker1.py:3307 INFO] --- -[2025-02-24 15:30:10,432 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 15:30:11,386 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 15:35:10,332 submission_checker1.py:3286 INFO] --- +[2025-02-24 15:35:10,332 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 15:35:10,332 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 15:35:10,332 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 15:35:10,332 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 15:35:10,332 submission_checker1.py:3307 INFO] --- +[2025-02-24 15:35:10,332 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 15:35:11,440 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,9 +33,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.52785 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2346 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.9953 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.63549 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7ab878c68..5fa2986d5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -814 +815 diff --git a/summary.csv b/summary.csv index 4adf3d9dc..ee6b1963c 100644 --- a/summary.csv +++ b/summary.csv @@ -11,9 +11,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.52785,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2346,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.9953,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.63549,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6adfda2a4..2d1231fa0 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.52785, - "Accuracy":"acc: 74.200", + "Result":10.2346, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.9953, + "Result":6.63549, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index b935e8bc592ad5f2bcde35c6a471aaab79ca2fd4..51cf6abbb7f259136a3ed37eb3619ff4a1d3bf9d 100644 GIT binary patch delta 3246 zcmV;f3{mr+OrA`zSqKixNN`+B2><~9K>z?40FyB!6@M}=cx`OuoLh6^xE6)q&#!Q~ z`k}jOVl3aC3QQ>L>ZPbdss_lyZy!*8pa0lgqy!8wi2DmmN<}&p36?OE<`COWML; zBufhpetn!ylzW+^Q9Ns_Leo`6&cb*Q%|>nY`s(wZrz&X{%m%?Ep2@cQBvbX*&VLRb z;^g;qEPrLDZ*qQq)r|2a2SR1V;nA~a+a+sN#rER^6sbOXr8K_gF%#EEAmkiIc%$kV*mIz zL#VCJ!Lsd}NIs-nuazQ|yEy*6cylsnt9l`AF@NpnWj8JoWgv&aVv=3P4_{?88fW>i z*+obgPtxVT$~4Ny(omIY@Yq&ORe6X8*|?}!8ApSGECyOt!bO_J)8AH?#JYl2sBqS? zLKmB8UlW$@7}rnPEy*~5=P7IY4h_(4gQEu{0H_+-SsVdTRS7Ny^c;-YV@ zb$`L-bAgiQQHx8Msph>w~`AhfA)oEYeLN zv3Dg`K;rC5u7SkemD~V{w~_3mYvOcGR)HYtas}z<_g58HuPUxzRoql5b~3earWT7g zNTyt2`6SiXC0DOYuDOJMu9cPSWNX804}Wb|fw$Yl5m%6I9&?2tEPMnKa{qh+5;E|g zfrNnn1xN^VUx9=`cLF5jHtPZj`EX8wgbZ#ENXY1(afv}jw+|#_bkBi=jP3=HkkS1H zBo@Bi9DKVutOA=PbHpVM8T(@{F-g83frRAy2}nr3pSi>%wS0M&fIo>}K?~#_IUuPB z?>=99o}x}^L60}!li@t0F?}9G=bUp;E%^v8Uiy<43tWGx`8J%-_rm#NFP_aFlf^8{ zS5tdy)omEhhS5mzqtI1^cR@J$i+s8rPV(!4 z&Xd3SXD)vwlE={X+c}r`L~`LaNaSdI8-&I4%wGQZvZ`utl=C=$%Hkv(-`*RVqv?C$ zLKs#J5}Qb+28q}17KxUtnVxQ95Iqc{$3Z0b&k>h6mT5bBRwR7v6vW4v0;pQUk;rZ)Zw0ZO;}Zk3Bvn z(dQ(RyZ?ww9QxqHC5%KMp(FwcC6P-!`rzYNkO;!bO?y%i-t`G5r?jBQ8(`~D8q)_c zbR7yL&GPxW9 zjZ1$zatt(#L8PG&A`PVwXgoS0ZZwEPQ~?FCs)Fw3;s!-l(gTh7q@FVt)dx{@pYa@M zgibz?i>>~@4I*DA!h*(ul=J+4Oj3D!FElTo3;!~cjbvB@%Byw9R#dv0CG4^XMS*wP z4+Y;@w{1!#*SLaG33liil|aL&1R6#q*RX%daif7$f(jUw6%}*^Jxe%cB|XrHZ%`^} zR3AmrNfK%3EVGM44h28Gu^mb! z*SH3yl504XK*Oj68b&48a4L-|DnSK|%8ClQn~N`0(vlu%gi}eQ`XGvJp#qK21;u~T z_&>n1i5%-}tFAk?*;h3k$1YpM;zQ9x!Ot3Pmx9SPu1Ue<8V)AVFff6Jfyp&|5o=a~ z2`XSr6J0UM(ig9VUfqT7DUm z`1t-%@FQ`X9%%!OYf&b-hBFB?j7fi>VN7z3XHzDvDkecAbg#9lg6`(x3zM{@2O9BB z$|Q~IqbSBC*9e^+PjB_V5hfRR`Bb=ALrH+l?x)MyvX~}?uNgmpWE0udTD@A^t5?nO zUAyd|vczH{xNE@nDUn>`+LTDH;Y0!rBNAvBkwC+Uw5x~&6)++zD(G%5z7T&&OM0LY zw(rWK`Y4JK2{i1X(tblg{u7(8jU-r$NqdV)!_;)$E^+gT5+9BLE*omZ0UDP&ae#(1 zS;LEiG>l20VN6!paZm+}$tpvRV2&JC(gTffCYdP*MDdj?*MOC4lg6BX1gX?20vir>>HvR@ORYLU!>O!c)t5yiwE6mDqOeIGV6^w}eWsgi7Gap|%~Maj9C<3Sq6C9vW!E~~6Os1e3x zm6=B{HxDc6fkrr&%+Ld(_!^dLz#6t;Q_sHx<{P#4d>7l^p5k1@HuaLHOGC$G8r&%2 zP;(E^c+}hjG>lB7p=5s|4J8w4=sMP@GWVbgD4A6ibPZ=c+}eiG#ttr-X5f3P=fx@ z>C&k3_Mk=>lvUmygL!*cNe?vQTNFy>?Lkot${OAtqhW8)YfOJ;!vaU~Ll2dWy|mN*L$EwTj%7uDP?4;v3?7H8UkFg-O%|Hv&<2g0R_|SAq!!%X%m?Oe~#f=YB+%Bpai4@k>s>Pu!2%_wAZ>2|8&6e z`IE(^M@bx4^)sn&)Yph+MG76WzZn~L5AO8fN;NjP*Ms{#xZ=we$4USjyk{KZo^gnk zaoFHJ{^A~gagV>a$6wszFIN18XbtW8)l4HjOT#U)42$AnY_UAwlyWX487MQWf3KFh z7`mqMTAjvWg^6wURNb5HzMyP-I5HYCzB}As8L7G#9i_xxO7`bV5&;yvpe2Xp9X5RR zG3n{$|HroDR!v7N1AnZ0FCu(hY;9VHsH>Mxhe+eCIBCd&WN9y__*{F3aue&XdI(FBNDe&A1{G7at zW+uN5(oc)e=muC`|NiuJVh7GmTd~TeFY=2yQa<~>%_g%WbOOgR6Mz<1H^{aEB@#Qu zp$6%(G0?U#_MAmcVYO8r2Nh}tl_HN(le+G`Q%zlVG(SLgU^s!YAw8|qf5ae;zq6&g zhU>cC&MHj|$G7CJeKI{?o=(pt;wpR9_;fa3UXCvo^?vQjt=BIRb!#fr(%Grf-}M_P_J!O;OinBuA$Cv z!ijeR_|qxqGjGEpeIM{B1R#CfQTbdvztkmbROM;DjMF^M@9S(y7E6+)+w;hq{tHkJ zL;g)10e$Y<~8K>z?40Fy8z6@M}=cx`Ouob7X>$QH)G-@k&Q z=EF?Y25H_&MYod7TZ}Q8Op>kpDMheVYE@joy!`hIYK)%qK-k&YC3WwY8Eu|(x)1V8 zH}sE`^Uc- zLTz;pmVMtu@*&-NtrVr)#qsaOo0CCX)eC8hX@5U2hjEc8133&9lk75n_$s5(ILl9) zU4(@3BwhZiOr!i<8mcl49^0y^Di6^h8yAU{aWojn;zX-TxJa{j`rGP|SSMJ83TGWF zbg_x{HDT$FVHcmr(?qUgTB}}{Bm5X-LFXWeACzR-Lpl$NPe$w;Mt)ktqV-$RT#T)? zZhyFZZcy?%inDj$F%MezMdG#MI}N#_4cD~cW~-rCTA5_0S{qh-Xn(T=-fkC1TtS9;%oT#L@DWJJ_45fx$cg_9 zBn12~KtiDV3M2%&6Cfd%Sr440tu=26OfR4KXZvkvV3`#fIo>}K@a2_IUuPB z?>--Uo}x}^Lyvdhlhb)dWBNRX&N=5GE%^v8UdEFb3tWH6d>hW^d*OVs7tdyo$zqn} ztEs)U>NbpL!)T=bq7GyhM3YqgaL^ip`TAQECFQGDnGbFGU&*+>uf`=-YYfdYbjvE4 z(Oe9j%b_Ee+7XvH5b z5QbHQ#3o9qLE^Q$MWW?url*@2L=S`LaS+M%bHpVMxqgnhgn!n+`xg6LI%2nJ6<^eBit#v(E2T;dbSh4+8I17Z`U)By3u+nEwg+p|TObNVhu-*b2_q3mD2YHqN#qib-uU zoqQk{TjPHlK)%d_1&jkJ=lS)Rr1JJ&XkI=Q{$(T^sjvo}nV^z~}?2>y(f%ilY1>b74T?!`GxF!XYYdDxd!@vX@1}4|=MXXr` zCP=`*tVqz^9DD(jw)9XX^yv+Y>Vqf_CfLa4V480lO!6@2uQS<1d9|1{cbF8eY58SL z;@$g0!FR-Mx}yy=u0@&T8qOrpFeZP2hB3)Co=usws+a_w&_}IR3A&quFHF*w9_Yk3 zDU&p+kD?fpTqAV4JH0jjMwndOrTvD0{3o8hHd0_MChaXI4O7!~yTr^VN^Cd+m~5yI2WVVs!~q)4 zWDP40(l920hA~;?#z6@flU05k!3;U9r3X6UOfpXnh~g_(t^q68CVe^o3Y2fvr6Y(< zM|Xr}lu)_j)6rk31TGwE)B%4QmpXNThErL?tAjL*N}yp>R@rq>0!C$(VMkCu%_(c? zflm0Q2=nEDD84DeHDFVuVQ&tn^5^f0yiw22mDo1zIGV6^w}eWsgi2t^p{^aEaj9(w zXgHNMj5|ofs08L2hf-PP-9eo&Dy!T(f|_eiSxXOe!XLjflMaevb0vS!u=Q)hKAqQ~ z{5g+b->8Gg;6 zL(M%v<56=D&@eKQhLV4YG?YxFq3c+q%G`qzP%^6$bPZ3EdZAQGF1_$OIZr zrqQsu=QT3_?#4(X_0@sW(08HKUE3^q3~PuID7jC>q23;#@u;^4XgHKLygf+6pakQg z)1^`6?LnO|D670Z2J`l?mLBNDwJQlr_9PM#J8o*O-6Ih6Rq|hi)nxsjbfH z)!6p*}l0|@e2iuNzi4P#)J3EK`my{9}D^ZUHHz~e-}C*eo>4MN?XZGf0pLeEqf_mpkZ|m z=D|pw1<5FyrOHGO^FnnkzyD5FKRK{`oyGIzD_d3W;w+1&&+o=E7|5h}XR69Dj-0Q*p9$fKdi(@5#4c;>janCr! z$~bKB9)EFOX{zXOl-M zkzWBmaXdfp0@twy10|i?{aUtyx?dm53;P`_`GcSGNZ=?zeXv%a*|lbJE=K@ z6aWAK2mk;8KL8<}UL8jQv;7xM2n4uK?a7l)ARm9tYQr!PgztR{qq{BHZb}ih18vEv zfkIPSaud=f5w(7h)H-?l%1+`W;8TyQnf-QlQ8nMIHu(kTrO`FYGlCM(+z6?+HTqaB z(npl|z_ehk(NLoUcrmEe>+V8OVi!q-JST+4h| z#uINOh^Ir;C*Dqr9Q!~dF#tWrJu9E(^UEb_ftrF8vy>ERalguEv?Meq>we@-{{^T{ zL;g)X1CF_GKSA>cvydUm0txq-uP{pq0091zV Date: Mon, 24 Feb 2025 19:07:46 +0000 Subject: [PATCH 188/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9758 -> 9752 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b129c9def..334a2aa01 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 15:35:10,331 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 15:35:10,331 submission_checker1.py:3270 INFO] --- -[2025-02-24 15:35:10,332 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 19:07:41,565 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 19:07:41,565 submission_checker1.py:3270 INFO] --- +[2025-02-24 19:07:41,565 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 15:35:10,332 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 19:07:41,565 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 15:35:10,332 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 19:07:41,565 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 15:35:10,332 submission_checker1.py:3286 INFO] --- -[2025-02-24 15:35:10,332 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 15:35:10,332 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 15:35:10,332 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 15:35:10,332 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 15:35:10,332 submission_checker1.py:3307 INFO] --- -[2025-02-24 15:35:10,332 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 15:35:11,440 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 19:07:41,565 submission_checker1.py:3286 INFO] --- +[2025-02-24 19:07:41,565 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 19:07:41,565 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 19:07:41,565 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 19:07:41,565 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 19:07:41,565 submission_checker1.py:3307 INFO] --- +[2025-02-24 19:07:41,565 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 19:07:42,552 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2346 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.165 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.63549 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5fa2986d5..cd93ccc9d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -815 +816 diff --git a/summary.csv b/summary.csv index ee6b1963c..932720bdc 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2346,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.165,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.63549,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2d1231fa0..5b72a816f 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2346, - "Accuracy":"acc: 75.600", + "Result":10.165, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 51cf6abbb7f259136a3ed37eb3619ff4a1d3bf9d..e4f99dffcfecb75a285c2e704545142311ddf901 100644 GIT binary patch delta 3928 zcmV-e52x^+OqfitSqKhJGW}gj2><~7K>z?40Fy8z6@M}=cx`Ouob7X>$QH)G-@k&Q z=EF?Y25H_&MYod7TZ}Q8Op>kpDMheVYE@joy!`hIYK)%eK-k&YC3WwYnbE`^Uc- zLTz;pmVMtu@*&;&TPaGpi{sym7bkz~2|2kvMncZ+8A=RtcKeKkoZWLqLeB05BOzz^ z8zZs!%gy00HwO~fq?jX=ION<44re%nY83BxOf@&u=Dl57oRSBC;ij+%hgbtFNa1^ zHeTFmVLWY3C;7kAe5sTFHJ8coHku7(k}ufgZ6?#~HeK9JqqJE2q__9Brnj<0%H=An zRV=1jXT9Tfd?M3)4U@LkiW72YkeRQ)HBnN&YL)rWmj9KE>-%b4VztK5EJL@fk{QkAp>q*Ba;Y7m#35Je zF-ibBMna)uBosPELO#O3xE0W0xJRL@2=6lC*ok14!M4gQ34<`5(*+Cp&&963Zhp5B8CHq72#bboPy|8fXED< zg6L5YF~%Y>=P2=sN%WCKa`hjf#GyAnC;<`~ z2_=z{P!dt%(HkGXfB$LY#qjAYVj(-^qFvw{rgq(&_ z$Y?w|A#OArhg<>*VpW3f=HLcJXz4MX_$1F6MD>{{y3csdWhNhq7hA)B8$iCyg9VHO zDd+j+n56RdUT9uE75-%;8>z4clUM73t)O%@OW0-iiGtr~KNPsJZrc<})VP8|$!yRA zl#B+TWHbOJYG7{MXn$BJnFIi3MS`xNj}lIyrN?yQ8x%?!)#sw<9LZ>4lH3~pTZD3J zhAgNwbWQvKmQB=HXR2HeOsw~aHN&$@sN|nC9SVGSV>^^e)VKzv5;aIAqX8-z4N!?1 zq|&IOl1TtmRwU?d4!%%HTY5|UvBKiFlQgQ&MFEqj5xSsETFo-K zxXY))#TrTiWOhGY&X&bADSXZN0VJEKuGZ?++FreCj_=xKHFlrN?xFOoAteiNcjDYM7O4lfImP1F6(1GAFsn@aS++FjX=}9$3CMY=6^v4a%Rh`}#%=JeOkI!BZ?@9lMN6 zzl2N1pF=G?jK-xN9!7&))^PD~8sL&K;sBRbMjoyc;IhifBf!f8Ej^|ay}EVKg4K_AnYClhaT#ISnO~ z(|^!qtWjm{;Sx|Xs}gh#u=YSpkL!d!7J;Zf69r^48YI(b*xU0OnSb}WNF(*tfzr@- zq10X5EZK!MLewdXY^vtfaw_bv||G#e@DAFE#paEf0E z(2Hc@@NzMcO7=9DZB;(b^LMByR#gU%!`1HtDDxzWlPG&y^3oV=gYrrRk@3^ES^5U8p~iHlj4=BD#JL=o?jHFWb?}noj(Dy%n_IY3E#R! zC|m>p09li56F+~6lZ_pw%&%XE2Bs4y1s0HyD!ya;^ZCxjC&MqBEJb%v@+4=68a6Bq z0SogeVe3Qf`s!HsHN*vF5lwRjhuS@G?aSz|A?E_gt&AUPw?Y&Lrpd!w$S61R0+_VK zd6`im|1H-hFG`>hzXcFkY7)zG&5R~YLt$Pqai|f;(NKTIl7A~0ZxL(}Z8YS`XedT! zc?9Xu6r-V816%Mi1V+mwZ{yc25xfm9fLEz#qiISrN_5X!=|RvoX30jBG}Zl$w`$ui z;y6thwBf7#2QY3b?6XP7C8euqHbI}jSQ|)cg^sEy5ue*D$9bBUU*TSHzglI95Ky)a zM|U(y=_-Fo6QORp=K``edM@Qv?mRE^5IE13^NYU~#xR5wN-EUFR9SqP$Kod~746P* zp}IOhp+9vrc^T>~x%G zSuXQneY>QgkY%Np5JM@e3iPn3^x|GB72YoIh~a-79`IiD_;mq%yw(`miiQt2L#9$>>JzWe)V z$MAh(|2~=chT~zo^JMH0!$N4yGyjJS%(jFjY*{s&tv1CD+HnbKJV2h5%2MS3&Oi~Q z1d%Y>VOT*aI?8KZh<`d@`TWV`(xW7ftNMSL)HmvDM6)7=j^5w24Z8<-dT=Eh8{F%` z{T^KLWs74)zy|MWhq$L5Vx=86c#prh$6wszFYfUd_xOtye<4~!yMHy^NYB!6i!8&U zcxYQWW1CX;g(L%|XZ7V$7eiM!UaQ?WtT3_7o~(QGxi3iD9*&HLIp1yWuT-k8MMr;0 zv6qtl`I1Bc1utmHVR?rQUwuq^YWe@M?YLFb5zD|I>)wk9Uzd5$Fz%lSUQ71JGumPj zciDq~wPQFi$QzYZXO53d%Qv&VLC23TgXiepzYgAo@qIP@z)I*~6LC$1J3`l|WV5Ve zc-<{=)G@nubb;qa;OFdJG&A{ikbZw!d`35rv-Iy^J4F=V{j^NOux+$0;Oj~%f z+JN-}Togdc5oSuX_udeU9ovJXE(s15#I3fH`cT>(Hk5tS9pF4o@2~_mZKqgA=2%X_};&wbqb7 z!=J0y_jc7-sMDz!psL`EFbmXcTNU`S2#2ew^Q&;;tpNUX4En@dzev{yJPH9w7k5xL zv*(xf5>iC7q|^lUoGd+q8ZvGDwY5p=_j;qnGP<;8xgcRXMkaNmtI)OJaLf%qm}$kcRH z`O%1>Qa{e{W2Xbe@FGh{aF$7G;Il$AH5}!}q^0gi=VP{EJ@Z2i=E3v`fkk+EMUbJXM9BxG5vfB9({lFW#nn0iO_dXzKFehf>OoaY2`4~3Z#E;x8yT6`{nZ|+NloL z_RKi0yHTQ?&X1H3rR5_XH9MI|jY7LhGMM~Y`RvVOA2}=~;Ig~y&KgEZiok&piRy4l zx!ud^fPsS^99PZgBKb@wR@(;QgMAZdvdgy*R>qe8(R4G7r8fNfPlyt9w=6q#GkPW$ ztkS!i!z@cZr>bx8W?~&U({}lMzx%5huc#E)xjvfdO48=Rv!onpqtYY`_j8Y&yIUPs zx;S`>7Gjt4J<1<-tojgB2y15X^TB;;*iXFNmD~$1{oIm`#Xu!HV0nvXrU`%JO}eco zt+P>v6;%=w#v;{zP;0H_ZG#G8_b2ikAg1Bzb|G#VZ)`;r+3*H#to^j+wyuO#I9SeE zi2KUbc^31;DItsP9giXJr=ixBh80cWAr)luwll8Fjn&4viK|GFn7i)y_@3}5-hh(xyLG9y3VYIZ@vZI{n&HGBl4m|8eTA zkosjP&nz{8*Y6gesof;%>?_=1bEWc&9RIvtZJyB{FtcUYIpb4t+-sLq9$e0k!}+Nn zR%ys4drVzykHuD*UrVB?b4I1(#8iN6ZNrOW#jMevoyPFi-)Qjf%t_02%sdw0Ik z$Z_PUu28>3wAhQ2rCbHt{euQE^Fix!xvwAh+EPM-9S#i3ruZ^2>uHv91xo6Kj`xxxas?YC5Z<%@_RiT+%}ZC0)O}^b;4(yTVO}7kU?Xtp2A>r(M<%j} zE@pa{Cnk;ZEc>Aon>tf|pEl&@D0{l{%GvcV9CYAhG#fEIi~+)>Q%S8OJfc;`@HNcy zLcTQ>cwG%RULdl^7uTKT6JrYDoj+%ZHZC-$=_UwxJTQQ}kMcm1)HTG$*B~jc&!Y_m zPZqnN@j4=T5Nbx~Q$w|r#n0Bs44nhUf#&S<2aH!rUgh=)+}%QhCh2AILKh$_qK)4& zBiupZUd>93X$ooqBcpdnicjw_1EfzS0YbyFJ2md~dG`S6tKnL5O@dSjhAQ4n@ObUQ zJOUe-xrhy`rgHfqJQ`s^1+0D>$<|)=Ny?{~cmv?jI-(m(IC(E=NjmwxJoMJnFNuXwHCc*U;+{{XVh+7i=5c4WQGF zVdjj!>-=P`u2Nc%9(1~Dz^j005q5!(K<4Jg1JwOR6qha9Wj*7FN+Pqrdc?kI_a7S%4qW7ebaYHo1`AO0O|{TpK3!1?PWfNFCoBO5 z(+0*5#$IU7jFYyQND;<{-$WIhh{U?*Q74Q6dnS%F+iPM3w+L>O3f1}}>R17>h>gj}5F?2L3PJ2RDYXfgb+NZkW% zE_%)^XCIkiC1$ai;o~i%KZ}KH# zaeAHagFf3Qm>UvQ|aGcWq zBZx5>80EK0DYYJkd`};Z7#6>_LDW0SW03z6dNt%~r@|t-(9wsbWN$4ew#966g#5vG zM>(VFQA4;vWw^oOKwiC2uGb3>daN>8AxjncM`XMFQRcRodF_Nh;BD2gW}JyFiWT13 z;PA|lUbni51N-5m&aq$wM?jxt?My4VE;Vt@3THbpZ+*bnpFr)YqgWKa|7r5JnpRg+ z+0tiiP%pG5=9l;G_N>HAq=F7(>fDuOw8vlO zEDc<65Hv4b=!gzG8{3#D-8;_7dw|)(fWfRe&}Jf zkVJU8w$oFqZJ~VfWMFglw=G;ieB%;jfzdN|fpIKuuybkE2~6I0k(zxOl(m+Fep>dD zNjQq)z<~aO;<{pX_{#OoQA}75WZmd<$1h9?Pp#v6Qbf?XRM6UX+r{UsZC88gi^^AC zXd3Qsr%`xC8|NQ1OO~Mt*f}l<7$nxu z$=SR`-V(#tH?yDVgj8(%GJMI$n1()LJ6qy`OL=8=~G0#$P!0Xj1eyqfR)I4DB~3 z*`(p_!v_?&YGAxkATL2?OM?HN2k^6GSUFJiyCBXp7q+*VR*f=#IChc@w z?f9Yu#V>qm4{dpNJ5a0|*w5_T7HRo!!)DHz4{@W?}6+!s)IwlU}u$c z4ZZ9&mSmh}!8|RayHBmYD`ldBO-ic5wD<_n^x>OC7<&MK&`$+=thX4TO_OCHB10#2 z$kyC^q@vztKT;C6x;j4d8-FXO-lAE5lE&>rw;v(Xmucb3p~q0Sk|68s^R#LN)5Sc6 z(chQkWM_HOsxa?8>pya-V*8+l-vIuJOSfE=L)UU?3u&LOyh2oknd4tgH?y7N+xL!1 zJw^H`LF)r~!V8ag)6w^JF9_AP@eBB}?*3RbP*7}~e)3Dbae0WR&j29M@<4Do+bf#v z^tK+Uj=%mU-v+y^*N8VV7hLQ{1wnTeaFks<#w`|EfHtGOd}f zx%m67=e>>*G46*JpE7q^bKL@!A>)HFnrFIdI_`+tNAZOdw zXFJSZzmF6jC+?p9TD;A-pj3S61EY`|xFW_=fAa)9Q~F4v^0j25itluy__w(Wzq4B6 z?u)gtOqdelM%U?V?wp_fz&p{87~5437D?~LT{qYG)jBLzD~hM?lj98Ku~zD@r-pK< z_TK0xRw@(;28V{Hk*UK5ElQ<2_Wh+N0N8ecEmoc5=5|qZT*3Vjq7-^IwBu zg@S6S_z(=VkZ!qf5>iBh;9|gDPvbuDl6d3xts2e=2z#9{()Ic8VD7e*Q0v=?yU*qQ z=(s5bZut&XCRodS36vRatc%pUR-H=+Dj_;g_bt3WVu4TyK!l9%HXx-LqHMwD_t9Vz;qvnX`q21i=srBSokMMv@a|?1UGIHxKGncX zI9#)S1CZs2)e75Vd4P(PzQbzOJQHcsNHoY}FxTH8CE z#i~nd{G{5**DW^`ACF;5(d>SI=GjFw3*yXJkSch0))ak8RU3PGkecz|FPL;2Vomjy zR0jbI{_D*g000-@{}D>G;v{)-@{kyzFy;+j0N}Bkv=83j4dX*1Lb*s5P!P2ph~x$0 pCPhHm1pXcXN(BI*sR01?|3d(wL^VOZR~SnB`) diff --git a/summary_results.json b/summary_results.json index c6995fd51..57fefd8ac 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2346, + "Performance_Result": 10.165, "Performance_Units": "Samples/s" }, { From 117ce893f6385424a4fa040c2763d874a032e280 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 19:14:20 +0000 Subject: [PATCH 189/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9752 -> 9753 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 334a2aa01..fef0dc55c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 19:07:41,565 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 19:07:41,565 submission_checker1.py:3270 INFO] --- -[2025-02-24 19:07:41,565 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 19:14:15,291 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 19:14:15,291 submission_checker1.py:3270 INFO] --- +[2025-02-24 19:14:15,291 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 19:07:41,565 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 19:14:15,291 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 19:07:41,565 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 19:14:15,291 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 19:07:41,565 submission_checker1.py:3286 INFO] --- -[2025-02-24 19:07:41,565 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 19:07:41,565 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 19:07:41,565 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 19:07:41,565 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 19:07:41,565 submission_checker1.py:3307 INFO] --- -[2025-02-24 19:07:41,565 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 19:07:42,552 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 19:14:15,291 submission_checker1.py:3286 INFO] --- +[2025-02-24 19:14:15,291 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 19:14:15,291 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 19:14:15,291 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 19:14:15,291 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 19:14:15,291 submission_checker1.py:3307 INFO] --- +[2025-02-24 19:14:15,291 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 19:14:16,414 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.41546 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.406142 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0657 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cd93ccc9d..ae645ab7e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -816 +817 diff --git a/summary.csv b/summary.csv index 932720bdc..847574467 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cp MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.41546,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.406142,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0657,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5b72a816f..ade8cca99 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0657, + "Result":23.1113, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index e4f99dffcfecb75a285c2e704545142311ddf901..2980c7d1a7c2e1aecbcee11593a06caa0b8c269c 100644 GIT binary patch delta 912 zcmV;B18@A8OqoouSqKH)A3pv|vt9_E2Y-_{sD+2oxYWbLXpqYq zE*?$;Trx%+;Ihid!*v2&R#|xjczK|u$8>^Rf}Mwn!X+$fm?dn(mY#nH%r|Q6`7XAd zJ;h@Y+tf>LE)AV0(?FYuL#;iG#-r9AMgwGW8cHUop=5Fzx{NidtUX);N@i7pu73g6 z9%$)tozTZ35Y=a*fJ{b%WEu^7dtM{+?>-l4q`o>(8u~7jx@(&yyRe2Rff8*Z4z>0$ z8jo6g7!5*M!`j1X07_;&bh&wDg!xe2YQ})*dbjpsZo-F&eh^ zyvAfUEN~P*bW_<#ZFN?!#k{!*40=3 zFU9`N>f7-z^1Q!zHi!y8e=u2nmp_YUBPIP~^(z5R@hbs(kt`fuE+$gRp60Tx%Ex*B z4i&|!%HVOh`h5UpoK6x=e`oQ0`IoIKcX5`* z)8|)X84P4nyfRf~7{}T3i{g}Qez~FZCs0cR1e1^?8nfvUTMh(U9pG1!NFX17&2GXl z5QOi2ipclGae%6|k|4Dery^A+_`tMajnPn{4m_IIcO|!!8wYRBSa3mtPa;Tr%58;q zAy|sB=Q~i$XVIafxG_$#5KFgz#g_4Jwgp_|`8`$;m|%h79+=vZh>l3XN2L96%>WP_ zLjwwQ@R(!-pS=N`@;4g><-)0C=q%i{p2X3;_i{A0ZJV`|!5Z@uyk5S(538n9_ow23 z>QZpZ9k5_rUE<3k9Nw=z#1r6<`}z~qf3u1q$^r%5A3pv|lUXEN0nC%NBs>CE z6O(Zh7Ly4jDh=cp004MwFLQKxY-Mwkfh81^R3#b$RvMFW8Wod;B@i219pG0b0RR9{ m0ssIJ00000000000001hfj5(|B|ZU*lkX)X20bGH00024)1#mO delta 885 zcmV-*1B(2aOqfitSqKGAGW}gjvt9_E2Y*v6VI8}SOTUCm#-BqiJdDPr9v()6T-I>$ za2nu}G2#H1RYo4J6X3GS$|Jzb11&wK6XX)?JWLcWVNt^@VH>ve{5xR2QDe_{vF+?B z9*fwfUUGA3=scMQ+C&^`?O`+?we~O?Ad}NjGC2(;lhe>;tWjm{;Sx|Xs}gh#uz&VI zOONY>J{EzfJ`)9GG8!b)XxQ8H8kv9hxkw}R)q&E`ccIi>+br3IHAD%NXcKX$wTIDo z)Y`*n5Xu_X9!>*LGUK7srBP+=;W_~*tE@c+SbLzQ$8_Rb6iTr6a8Uqd4Qr3lu(jtk zCbMCIqxhkl%0_CdvwAhQEj{@Hz<(1S{TgIz>)Y*9nT+JIoJ>+BjPujAirloWzVd%5 z_HS0-j(?Ho{l&9ERQUOW$?CiOSu`6d=^v|K32=&E3DApV;qY=XkxKS7mu*!(&hvMu zC{|SlkHgjP11R$(ijydNTJ>K1K*3@Xbf)p({c=!?S@g$3zJC|KbN1hb&Q^zC6yt-^ zR`SxHrFnJBUWz}sVRa7X!APD3$taqo%0v$HLUk>_|4vrFII#RXi|5O~Y*o37vn-xI zzZ%P6Ad}*isVc)b&YoWsr)2ZX4V^y$P)h>@laVAEv+5CB4g`zH>-3XJARm9rYQr!P zMfd#*qrENLNngTtpe{*6bLP&VVsVgl^b6Jr ztxA+7DT+Wb&4t>P=wrQ%A5i2xRh-sZL5Ysw(4x92m?2DCc(dAo^#WWJK*|wjO0@Uh z5R4t$gQPAA4i&_$wvzf#+8uv3lzr13;5<$5umn#z^%S3gu^EY|M{+hI&5x~H0B0C# zkf6N7S(4$i*MpUAvY}NjoKpCsfl2Eij_!jKqp@k4q?xtWkUztptJn8-)mW(0sTiQD z;EXT})N5N6__7FxtEuy=aN?~1{&Wob#9O~e*9SZb0Z12jP&TvYm-QtQRe74<#%Uhs z^L6$}(t9$`zfL2s`p-bw5BWE70Cc&pKSA{evx_0h0tHSo{as3vS|nQm%agVwJONdc z^CUq5?~cPyzq|5dZ)H0000000000fPprXu_ZnMij(jq LBL+Mp00000D%_|H diff --git a/summary_results.json b/summary_results.json index 57fefd8ac..4981ba29d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0657, + "Performance_Result": 23.1113, "Performance_Units": "Samples/s" }, { From 84d38f6140cbed97b7c443ef7f7edc1d990fad36 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Feb 2025 19:16:53 +0000 Subject: [PATCH 190/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9753 -> 9759 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index fef0dc55c..85a576a26 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 19:14:15,291 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 19:14:15,291 submission_checker1.py:3270 INFO] --- -[2025-02-24 19:14:15,291 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 19:16:48,343 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 19:16:48,343 submission_checker1.py:3270 INFO] --- +[2025-02-24 19:16:48,343 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 19:14:15,291 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 19:16:48,343 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 19:14:15,291 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 19:16:48,343 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 19:14:15,291 submission_checker1.py:3286 INFO] --- -[2025-02-24 19:14:15,291 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 19:14:15,291 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 19:14:15,291 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 19:14:15,291 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 19:14:15,291 submission_checker1.py:3307 INFO] --- -[2025-02-24 19:14:15,291 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 19:14:16,414 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 19:16:48,343 submission_checker1.py:3286 INFO] --- +[2025-02-24 19:16:48,343 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 19:16:48,343 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 19:16:48,343 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 19:16:48,343 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 19:16:48,343 submission_checker1.py:3307 INFO] --- +[2025-02-24 19:16:48,344 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 19:16:49,331 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.165 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1038 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.63549 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ae645ab7e..68c7a4170 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -817 +818 diff --git a/summary.csv b/summary.csv index 847574467..b86ad453b 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.165,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1038,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.63549,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ade8cca99..9c5842610 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.165, - "Accuracy":"acc: 76.000", + "Result":10.1038, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 2980c7d1a7c2e1aecbcee11593a06caa0b8c269c..7b9a8516962b70b63351c6c85641a34d2df304b9 100644 GIT binary patch delta 3957 zcmYM1byO1y8^*Uu4QZ7aDQvrdw3LK^qjS^%5hf+w2nd5QO1j2CLb^L7MNmRYS{NlI zB_bfmmFDw)-#Pbt|9zkHJLi4QbDneF4DDR)Ix`9yQ@h4%9WVfJtport1Mr?u2zDde z+l1@2usyPn>d z^z%_IsR3uL`EE2+>n2TbP_}MrU`CO?oO9))DPxt!x!!JJ{UU_0i&rnrWe;e{v} zJzEh|GM&E!5yaOs;!)3bDk3k7blHD;Fh?BPunCoTrB(B5tV0J5_e#GyZCy@llgISc0@rx+3xz;jM7PT;(0NG>XZk5cw zsby4pZ+ob5na521Bc?0|^sTf#X5XJLBlVv^~*VYVY*X$XW4P!j$msEsgyZsb%#QhE=#K+ z+4r!{g_l}2bpn$*T=jl16UDV%%^F$k}rlUn{7 zwDF=5pRZVxM+sxBlzPb%t5fJS5oS`WRnrz8Ttgu2+B)~R(7K{QynRpw%7S|L!f05|ZG=QMIyrIS>*ru~_0HV3UY;h*i?k4h zXtdJJb&dz+6#Ea+*ub0uT}P!vsTKUHT%l+X2VyJR8!327A!K zW`3d4)Z3oeHO2H9eXsKCNYJy*thJh)2@q{;DL<9W9{b&WinSKb8R5Y92@IPU20Bl> zGLJ=0s9&nE4+uoT%gp5^M_q2!9nO_t*RO3OuOwgW3j?EW|?$^ znp+c(kmtbbfKMaI<*&q zrSy*s2Q@< zi^-yidyLU(l^Hp1)^zez1^r6U-(&RY((6|bHW|pclaO?{@K6fqZRJ{!GRLT56~%t2 z2W5c?kb7H>bfS>!kU6$D+dEnh%CvM%6QxsRaHAK?>Ta(o?mET@OO$^IpV)$?yuFUn zVmn*ufyF6vpZOo zSrM*~aL;xraGHc%!04DIUNYH|8X$Zr2w)!({n6k`o^Jz4-w0O}Yhwe(Ysq*~iN`4w zpo11t-k7_ z=0?N2LqFnDk1m3>4PFEaiwo!8ycWa9+nU}Os(jv$GHTC+?KY6B;QZ!kNtex9Ts2|D z)=&em?=}lTv8Rj#q6#C{4SE*hj6x}yv4lH!d`RWBr0rnx<5BKNen%rtl?e?7$8CYO z>B|<1jhWkKNXPBW6QDrWK>g^Gw!K%T?hMY6_UXvz#7r7eg?IJr|M1C*Q=Fy$#e49_ z^Ybf_#tp?#wr9l(8%?HgID{K&6rOR#xaNc6$4UDpPZZuaM6-Rzx>VN4HqV(zu=fj@ z_pTu(5Ff@MuIZtJ5;Ynh)*&?TaS;(*ByLawL+^~jf%%_4MH!8Md4d7r(|#f0KIf4t z$4a-!Z)Mwhi;PH7!{RC+baQ6v-VC?tA>(ZbF@O8d?E)^v(N2*n%_O7<#byZvJF2LhvP0{z37Rq*uX_hUD?raMifmmst?*QVps$~>=%B!$e zGfIB~F)qSic_C8(xoftlTsQv4tAu54(#v#yy2 zt$23}Izx+GrM`{9VfL)as&Ld9i@eJIL6mY@ZM(g|k%OVW7tSj>1?~#hS{cuWbNqqE z{(+`TVC2+$T1ilBAMFSPa*6KNDzPJq%}uL6nWflDVF^&YjqdTHHtmsg**W!lk13qV z*|%O;FwBmfEwMcGaom2T9mcA{=ClWdBxVMNL?%azr)n4F+j3J|VEZA)n;b?!SYt(K z&4*9&RYfl$`CHrq6|@D+8HzoF;t7s-oDIag-e z`SFJt@P~;o>sbgBr_vnfl(B#qFL8XP2x}Nxun}>m{l`j7?4!XBdEOYMKPnsX-5)RF z2VG28c!RgG`3=d7J5`qPu6lZ%fP-+gE{eJNv*2?hIL`}#zwk!jub5U;a@OQ@`jv3s z)yU5IPvOIkcm9%5lxkQJv!E=6*k?SjOPga^pzcpuV1T}vbjK6zq7?ZNJ99$f(ZwzP z6l21NuO7!YfkSf9YtW$_R5g==e%&jQ*N$@jUAv!ctQjxsllGS*_Ombj@{d-T`cCrR z5af>S9do`k5Y|X{P+Lzq;?~WQ;tI?vwAV%SjEIL@`?kkfH-?F3a_G(m;|-zF2LH1Q zH-Rvmc8p(11=UySz+gkF!Oa#{87v5?b%O+-423lA&<6m)w`4pJPe}j(AH3@=1FSdZ zVQ!^fBwEreW-o7DW0Xe1#S{W!+|=)$c{zVPUBBlXb#sV2bCZguDR5;zeFBXbB~4Ld zQTXf@!Aql!f7z0x@3u}V!WVrcR`}@3yhg_|Rgow7CRf8}Xxc_5i_WXa?hflUBu_$N zehlmyC9=dl3vnpCtEY+(l^Nzy$G$9KrXGRJhNj;Yc?^HS+Vw1Y#nl7Yp{c^EW0I(7;V%5Ur#4|r3b2J@~0Mis&8U~a27h`LY%pEtyIx@({Jho|_yVl@pmljHm4?w^TK z&r~Sc8@pIoeQEc8tzb1JBwLAL=kl#2*Q zUo^5=o{fdtyHj=%%~IaZ#9rKMsrmPo76b5hM;`3u2D>EY7%Tei+s1W&^H-FPn!pWg zRiH38#$rRhrxnZNFF40J<`Ao~Xh(ZBov`|z%RR2B-5X(m!GE@_faD zd0DE2a~90*!d(;Xp|*3zJy-Tzp!Tg`f{ZUQf$w0E@*=w<_8Gj3W>PP9Y@5srb$#{A z$C`Xp$GqNp63cYj$Htc%wEPS|M=yi3$p98ToJ-0y>UXcoN<@VPvv|h5MOe1G;=SQ2XSJV2t*TLU@9`1YV6V84Y~sDW`A%BG!}LfW1yU zZz|p+_8)c!_iJ9Ql*%zkYa%(TrtN*JGqPhRu>UO+IvAu8#EhUIA(=cn!)tJHVuy8m zMCeq0V^r3I)}VsM^(jW6&vyt(yrBBVT}LN+1zXn1D|t6d*Wwz}g#Fj|{ZxfeX%|f$=?Dwj>*hmE_VfQZ;^~h2~ zQAOhqELMzV#Af~jJ9-)-a!YEpR~C5jlld2`dk=RI=K(JXSJ0yV=_`48cNnp3XzJpj z?vZq^dn0*@-~i_IIpoDOmTbsspevduZ!G8%T%phydv^!~{`UdKTS85MqxfbhH|xKp z%m4t~2K;|QaHjYps1k4oF9DMQKEZpy;H>|64g&%JNDu%(|9>eU1SgNLhdl%);}2jw KWC)&rR{sH93S8g- delta 3923 zcmV-Z53KN?OqoouSqKi@A3pv|2><~7K>z?40FyB!6@M}=cx`Ouob7X>$QH)G-@k&Q z=EF?Y25H_&MYod7TZ}Q8Op>kpDMheVYE@joy!`hIYK)%eK-k&YC3WwYnbE`^Uc- zLTz;pmVMtu@*&;&TPaGpi{sym7bkz~2|2kvMncZ+8A=RtcKeKkoZWLqLeB05BOzz^ z8zZs!%gy00HwO~fq?jX=ION<44re%nY83BxOf@&u=Dl57oRSBC;ij+%hgbtFNa1^ zHeTFmVLWY3C;7kAe5sTFHJ8coHku7(k}ufgZ6?#~HeK9JqqJE2q__9Brnj<0%H=An zRV=1jXT9Tfd?M3)4U@LkiW72YkeRQ)HBnN&YL)rWmj9KE>-%b4VztK5EJL@fk{QkAp>q*Ba;Y7m#35Je zF-ibBMna)uBosPELO#O3xE0W0xJRL@2=6lC*ok14!M4gQ34<`5(*+Cp&&963Zhp5B8CHq72#bboPy|8fXED< zg6L5YF~%Y>=P2=sN%WCKa`hjf#GyAnC;<`~ z2_=z{P!dt%(HkGXfB$LY#qjAYVj(-^qFvw{rgq(&_ z$Y?w|A#OArhg<>*VpW3f=HLcJXz4MX_$1F6MD>{{y3csdWhNhq7hA)B8$iCyg9VHO zDd+j+n56RdUT9uE75-%;8>z4clUM73t)O%@OW0-iiGtr~KNPsJZrc<})VP8|$!yRA zl#B+TWHbOJYG7{MXn$BJnFIi3MS`xNj}lIyrN?yQ8x%?!)#sw<9LZ>4lH3~pTZD3J zhAgNwbWQvKmQB=HXR2HeOsw~aHN&$@sN|nC9SVGSV>^^e)VKzv5;aIAqX8-z4N!?1 zq|&IOl1TtmRwU?d4!%%HTY5|UvBKiFlQgQ&MFEqj5xSsETFo-K zxXY))#TrTiWOhGY&X&bADSXZN0VJEKuGZ?++FreCj_=xKHFlrN?xFOoAteiNcjDYM7O4lfImP1F6(1GAFsn@aS++FjX=}9$3CMY=6^v4a%Rh`}#%=JeOkI!ILjxg%BlH zI^maa$@p`qg@@6&)WgGQkjok_9!>*XGDaNWvdYNAbpl*gS$PC_d7!1obb?%horj6S zB`j)~C2Yf%o_`0-H)`zpF1DRL#bXiM)JtwI4V@>`K%0m|tv!s!qt+fq17vae4MJJN+QVr8N@hHCx-_b+JzOUMWtFwZ0BaAl z^q5Y3i$V$39xe)?tYPgj8n*Vl#$+}uaDNm(bW_<#ZFN?!#}RVJQ&HdAQ?roRGG+OUZ}3+ z_ut9t7YCMqXYqXbm#r#yahAo?=T~DH3}jNgGF4?5$Jz6X;*@NDxuNqXv(6Ei0SVu_ zMJQYZ003E&Zxd2~WBc>@&c!FgFPkhycTn;qXNMX#EDZq*^C)5KL+$$NSobx=1!WOU za|VanJ#g*I=&vE?0?Dn6A8NNk6bGis!&}HGH}V3Qw8VLtQ6c{=*CsDYpb@_X5Ls#x z%W}<(CQL(NUNLc~5y#O`#gcz37;h155p6W&$!I7>XL$sF>ChCTp;-f4@G=BO%Or2( z*DMjd4K9FJsc55VN;67y&syn0&^BhtMwB$w{f)P3+b-fbO&GM{tNaHrZYu1vNyjCn zt7tYspTJlfNNRm8EwolEZwp!+=G)Ie9*=tDyU3F840saeDmydoM%}s^I(0uq@j>yrI!#xDXR+f zu&DInUMdydF7Jrp%0nZFsZ7pgn6o%pn|qV;y~*}}CHY>iY9*_*(o-s z+Hy^mVx-%=P2q{By`tCfw6$$`5~ahSc%bCi*#s@_e*r@Q@&d=YLD<#Dc?qbuHtH?Y z`S&sMJa>Rx`f-D4{Bk*;E|!-^XY=463 zXw5VKhYZZNge7cQHJq(B#SYqW328h)o|MW`Fxp{QK`A=QYh8$cI$-(y z$>h?bB#x{4nbbGxYecgmg^u3ev<%sjVT=8X#V@1FQ?`em)ryXLY z9X5E6zqrR=+~Y6q@fY{_ixqz%T0^^kHQh+h(r}9`!=iX-TRCH!Quc);1Epv6Wd-J(3NZTHcjD|ViZSJpBs;)&xNwJrb{rQqa00l2-$zgei4PSkK zOnPeh|FP}3Rnrm6z#r@0iwIwrdCxHJp9o${_Qx~YViI@RgMYPSI55Z?l~iYrk4(!q zv%Nvbk1vDg=-$5$-i7gfHT}R!=wK6ZO@uo_*QaE&tYdiHEpgN_yLNPe=SJY?>|Hc7 z`E`(fT6{(~khA*tr>7G;aBkWQD<*w^kzdS_^4a%oHklov6F8QU0JONeLADhrO6(+u zD$-+NplxC7*^8RQYA<v}t*G%+0ClC$>7^n7_bJ(q~9%vIyl*?f69zF5@rwJWE3Kd?yPV^aSibUd3aJX(qT z3h;^J`GFU>jx`u4;XIbN@;m142WRBm9XN_|4+_`tMajnPn{4m_IIcO|!!8wYRBSa3mt zPa;Tr%58;qAy|sB=Q~i$XVIafxG_$#5KFgz#g_4Jwgp_|`8`$;m|%h79+=vZh>l3X zN2L96%>WP_LjwwQ@R(!-pS=N`@;4g><-)0C=q%i{p2X3;_i{A0ZJV`|!5Z@uyk5S( z538n9_ow23>QZpZ9k5_rUE<3k9Nw=z#1r6<`}z~qf3uMx$^r@9A3pv|2><~7 zlVl`U0nC%UBqAH%xlRG6k0alZC hB_;%09pG1!pd}UoH Date: Mon, 24 Feb 2025 22:47:22 +0000 Subject: [PATCH 191/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9759 -> 9760 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 85a576a26..7cb44918b 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 19:16:48,343 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 19:16:48,343 submission_checker1.py:3270 INFO] --- -[2025-02-24 19:16:48,343 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-24 22:47:17,446 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-24 22:47:17,446 submission_checker1.py:3270 INFO] --- +[2025-02-24 22:47:17,446 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 19:16:48,343 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-24 22:47:17,446 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 19:16:48,343 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-24 22:47:17,446 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 19:16:48,343 submission_checker1.py:3286 INFO] --- -[2025-02-24 19:16:48,343 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 19:16:48,343 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 19:16:48,343 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 19:16:48,343 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 19:16:48,343 submission_checker1.py:3307 INFO] --- -[2025-02-24 19:16:48,344 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 19:16:49,331 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-24 22:47:17,446 submission_checker1.py:3286 INFO] --- +[2025-02-24 22:47:17,446 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-24 22:47:17,446 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-24 22:47:17,446 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-24 22:47:17,446 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-24 22:47:17,446 submission_checker1.py:3307 INFO] --- +[2025-02-24 22:47:17,446 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-24 22:47:18,396 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.285 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.41546 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1038 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1622 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.63549 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 68c7a4170..b3fa96d20 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -818 +819 diff --git a/summary.csv b/summary.csv index b86ad453b..0e2b9ab05 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.285,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.41546,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1038,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1622,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.63549,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9c5842610..2bf799315 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.285, + "Result":18.7241, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1038, - "Accuracy":"acc: 76.200", + "Result":10.1622, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 7b9a8516962b70b63351c6c85641a34d2df304b9..d811cc00d68a3ae2f9b0f8df1bd73ba1e38bd442 100644 GIT binary patch delta 2324 zcmV+v3G4QsOrT7#SqKh3Tz|Gr2><{9LI4050FyB!6|;N@E(L$@0Bf#ci4rXFS<|85 z$2Yb^vE&-ppjdJZ#}a55mO#U> zuVT$AFhKsx|@qHVA7Hv%7pIUu&6$W;$VWKY!0URroklJIe(qWCeo|Lq`AYS za81iEV-la=9}0dZZqqYupm8nAB-d~zfrc>&G>l2E@odVZRmCJ|gl@K0RnXmBd|{H7 z^gtuNNtvWkeH6u*sIlgO`Jyn)?Oa!+L*ghqaYh0TW$u*ovpkYJ;4I>h07?E}rk)Q%bWJLwt&BYfY zX-N+>!Zu!6R3Ak#B7ufIRoZU|$bVw@wUGpCF==lxX_%U>+a-2BQDVdqz-B|8I6&i4 zD-O_bCTo9~agc^F2{eq!DnAaYfH7I+$Pvtv!%BLf5zZuY<$x%@a^)JZa&6L?^RGbp zR((2x76K6kOgYrI z12ir*?f?y^vW9gBX&9BjKI2d-tK2)N5k_T|e@9Sz%_%GCfkycDE3@gKC^lCD4O_o9 z?9_P;%Ad3S`bIrGmtxz*ldoZg5G8Ip;g@g;EIQQ112itR@c<3yvWAfdX&9Hli^I6A z^74P6Mi`e>ZXUtxJglS#8sS_rM-PbNYgn!UYuJWeJ^v1vZ`9lKU2L0sigOX$)JvW& z4IPtdaHohv-913#QFjl}Ffx&bl8H2wOr)XfSfk3_gDRk8R#nh7n7fCS^iU)8T!cmS zK@=ksXgHZh!|tBf$o#tpBaP%&2TDWVg;IZaZL?$>)(|C7a-)bt%{@TlQF9N_a42h- zdys}f3Hn2)OQXu%gBoE_R+)PYX6|7nJ%F_{eu9K{bk zR5p@Zoz<(cZR*Jv0G{yZHz8YFUvQtwWF(K}WRfajoDbJ3a#Onc*8ip0zgc}j{zZSD zw-?U_QDNy1CabUWXVGk=q<^e_D8MOxC_pbN3x}7BiBz(uxooTQah`udMY5_gcpR>N zB0!lZQJh5C)2j92HwqS$pfimJ@0Xog%%VRQ^8LH;owNTgbUyr|7$20jlBfPG&9hs! zQoKOj>Kx32kvt2MQ8Y`Hi5%vM>RK%R{GF_RbYS^9i|5N%wyNC4Sr$*9-;HH3kV)~* zRFz>IXU}hnA=&(RL+4Mk&Jmaa1xMS#$zhXg6GH+?oRhEfF98iG7zQIf9@ zwd<>6-PaHoG>>S?ayZoPfooqze+@YoNFL4kp>``oabTJ}yoHQ%gB6fVNzBTO3i)rj zHhED3jrc8q$WoJ7mTP7-$u$(RDi?`1{c7qRJ74Fr5PoEFiWfyUrCONd> zEA|6&ZYu1vNyjCnt7tYspTJlfNX7~sRZ${7w>8HsW#w16SKO~wSt10KZNbqUO;Wl_ z(nP3-?zw=hjh@T!DtDfjECkM(V!!xXVGKh^p`=1prhkg^C5y#RS}L(SXF|QiHHWA( z`I53KX{b0j0U^Ct?2M(2Hs%$Px~B13oyK8>iEZ{&-J9*cplo|MG8!_IpB9aOSYH4B^mJke z&P`jf%B3&zi#bw0`@YR4vm}>9H};wlVgcMNMI~RUQWw zY6g`ek5QAl?!8k@U3N4-Kz3j_fwCbztsKC^4hl3XN2L96%>WP_LjwwQ z@VLkdeD(%#%HM1llnbYlp|fz)dJ;$X-pkS0wr$o<25Zb;;PvwLeONV>x<3^MRF{HN z?tlg3>JncT@o)|EVHr=nkszMBsNZ-yE^_Dtk;DM>5ci~fPM%*SRtr>Nw=z#1r6<`}z~qf3uMx$^r^LTz|Gr2><{9LX%}Ac?Cz?!O3Bh u<0MxB>ytYrIssjib|oeRj+R(6lb|IQ0X~zrB|ZU?llCPe21FwO0001_dq8dg delta 2315 zcmV+m3H0`$OrK1!SqKVPWwZZH2><~9L9=5BM+JXx5^IKMmr#k1nhpg&y|EoiCD*tH zrIKqnl|aL&1R6#q*KjJ0Dk?z*jLM1%x|@qHRML_jXoOQqqxv9n1 zi5%-}tFAk?*;h3k$1YpM;zQ9x!Ot3Pmx9SPu1Ue<8V)AVFff6Jfyp&|5o=a~2`XS< zR#bn`-CTSDla};QCUpCTMfE`x2NRrRb1=;}4JO&m`RhzJkzOq(%^fC%Yg&F8llb`l zQ1Byhn;vNcjcZXRxrQ?dG>l20VN7z3XHzDvDkecAbg#9lg6`(x3zM{@2O9BB$|Q~I zqbSBC*9e^+PjB_V5hfRR`Bb=ALrH+l?x%mt*|L}>g|8VufMgTd)mpt;+pAa2@m;&@ zp|ZqcBDib7_9>BEh07?D83h_tJS1QjqMD=O%2F1`>+OM0LYw(rWK z`Y4JK2{i1X(tblg{u7(8jU-r$NqdV)!_;)$E^+gT5+9BLE*omZ0UDP&ae#(1S;K#e zgEWjupkYi_*>O+>jL9lPj$n=)R?-8Fa3+~42So9eE7yRPYm>&De+A07YSR(Kr=vT< zGD@i2G3w|qR010gb?N|(ORYLU!>O!c)t5yitGC&Xw49?KqmSb+?2{uY^kA$)UC#pmC{h2WU8z zHJm$0!>9!A8HZ9?W!^!JFe49V%ytOoQv3|Uh;Hl=$K4{8$}#y z?g1K)ntOnTk%=^vOr)V?A`M-~8dc^VQ~@Qks)DY;%ss56hZ>>hA}p#8q8OP#!^t!n zHutuQo z8dcsN)ChyJ%G+ZwZx1W!fku3bLdm>6D2hQ@!`ovt?Cp7t$!u8QD1PXnvXR{CtX_?6 zPfxx8@PtRd2-({De*07=BY7+*lT-=ge7IJTo6^;n{x8M;&FcH{FY??&Wm}bx^ZXMkl2w(#<8bv00m?jy;v~wR zR;?F5QLvZz5kI)4JQ&Jmaa1rs`OLtv9@6GH-tlasI$AAd>C4mE688UhyPQNq@T+V$13?rVq( z$|9QP3=Xw>;M$kbUqj9Xl1CXo)NX|+4os7Ww~$e8OP? zIxZ<)MY9R|1jgDxGFIrQiW2d;tvSxqy!;CHiu=_nON4;3EjYTPNlI5qnh5pKJr|I* z(Q_GI<<9dm4}tSsv0wbHFoq$dP*R~PQ-4MIGLOYiS}L(S&xLx4YYtIo@+DiK zy9)HMsPy7qMk>5r-Vwu(or9+^2pw!se1TF4=0Yd@u0>`>R*wx2*38=R> z>Lb(n_c8K3cYs{`8TBSeXq%foQd^aFg>Z3keKN%d$i;?d5sc6SKn~ErGXmEO8sJm{ zfiZBghtSo<*XjA?az0%wFOSaVr$^KJw58HLnjT=oCcgXIv}5=_v42}8zTtS-?zD^@ zVps^RdFKC+iP^TWge|Lvv(={9K|3xXg$KxU4f%d3;Q-D+2}lVdVUrOSXC~#0ZAv*8 zk_?oY)mKYh3|-TBtxn^x!o)Uvs_xBpUr@F^92pIBlb#lhe{xp;{`7QW2hL4fv0~B} z`NbS5pMBqEli3kEfn%8oK#QvzWLtp}iJjt5gY?)KXxkWj&Z4HU+A5EO3N?dDk;kY> zUH9IprY<|0A0Rt0oIu%-o>pmM5Xaxy(p|%KU2kWVCWhl%a@RhYo-a?Q=Mr(1y=r_q zn=dcN7mIqoKX&C-?*|qMd`#*;gpOyEM=6nC0X}g&Kkx$Au?7ProsavqrS8|q^1^<{ zO8(%dJQ6rcP#>(-XLhZboQu(av+EaO2m}Pu^ZJueARm9rPQx$|MfZF~;KGM89C2(5ggPlA;I{(_E;! zMDN>myhM@nRB_s91tmIyL#yhpV1_Vl;nivb)(db^04Yb9Dbe10Lojx150bhhI8+dK z+Dhs}X={IMDEp>0;5%mHQ zv!PQioKpCsft%J*oZJT|CS%*SNjvMTA%BKHH!pA9s Date: Tue, 25 Feb 2025 10:16:50 +0000 Subject: [PATCH 192/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9760 -> 9759 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 7cb44918b..c33a7aee6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-24 22:47:17,446 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-24 22:47:17,446 submission_checker1.py:3270 INFO] --- -[2025-02-24 22:47:17,446 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 10:16:45,280 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 10:16:45,280 submission_checker1.py:3270 INFO] --- +[2025-02-25 10:16:45,280 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-24 22:47:17,446 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 10:16:45,280 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-24 22:47:17,446 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 10:16:45,280 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-24 22:47:17,446 submission_checker1.py:3286 INFO] --- -[2025-02-24 22:47:17,446 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-24 22:47:17,446 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-24 22:47:17,446 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-24 22:47:17,446 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-24 22:47:17,446 submission_checker1.py:3307 INFO] --- -[2025-02-24 22:47:17,446 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-24 22:47:18,396 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 10:16:45,280 submission_checker1.py:3286 INFO] --- +[2025-02-25 10:16:45,280 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 10:16:45,280 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 10:16:45,280 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 10:16:45,280 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 10:16:45,280 submission_checker1.py:3307 INFO] --- +[2025-02-25 10:16:45,280 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 10:16:46,254 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,11 +29,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.406142 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8184 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8158 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1622 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2172 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.63549 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b3fa96d20..58621e61a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -819 +820 diff --git a/summary.csv b/summary.csv index 0e2b9ab05..3df68a9a1 100644 --- a/summary.csv +++ b/summary.csv @@ -7,11 +7,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.406142,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8184,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8158,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1622,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2172,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.63549,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2bf799315..97932655b 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8184, + "Result":20.8158, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1622, - "Accuracy":"acc: 75.400", + "Result":10.2172, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index d811cc00d68a3ae2f9b0f8df1bd73ba1e38bd442..0f4ee57b78fddbd1ace21f1d5ebecd7f9439a5be 100644 GIT binary patch delta 2217 zcmV;a2v+x?OrK1!SqKHS$mOX_lU@j81Rs*66|=<%DguA6#IK+Qa*v!;gm<5>Jx@`m zw4lcu@X2tV(U?Aup>xhTsFr*L7cc!DcE0}i;?rgCq<{K;xf)9I<#7sFLm-)bD0cpquEd<`GQT}W-`rg)5YC1N{h8mdV6ncdMit$T&}WO#bT;;);nIu zCo;{~Flm2ly_nhC)O;Jx=X>FNu@}!~kI7<|<*TW^wdyvEXTxZu{-O?K7DSU&{czA4 zg8BMe6Dj4ZR+)Eg`Cm!DzOVWvR%;B+GIYx-nbBMfoy(ykx7rbxIOI+}<`M=SkWlD= zghB@-+_WbZ;a#6_a!Lz&yaBcjr7?XFL)W1|LOy~EzggI(K^UidlVDlHqv!nW zHE9#+)d29)-vZFl9Mdk_b`UxSa9K_WB$I#35zx4#Bga6)7(^NhA<|F^fySc~;zolw zL={jFt19SjE^bg{B|XrHPwF{iQGF0a_ZiQDM(E@Nx!CIe+aU5~A}nYeNIB2%$0U`v z_d@gXx$rMD*+_;ppuAdlY(=H3S;8)RP!xEl{ZR0ob=#&?a*Zn}m0*XSQ3*7RN}zvX zRB{cQ95)(BC8&T=Sy4e((6fY7R?-8F_y(ntM)gq?og|Tl&XQZ*e~VCVO_2qahOUVp zz_N)P>&%twj*0aSu;v<;D8Uk+H603md}BKlORjMZiY3=@EP;k$2{a5#uHjf3Rak-w z7?u?kbT=1Yu%snD&BNAxXQ>Fcefcz(RUmHoV z7L)cClZL73x?N)D6D39*0c1D4v69_SFQmo*Cw4g{|c0E)u$tfQAc-#Wt334xlfj7p$kR8|>wPz8+2D$9uw2^UI~@JltXJoy?{2vOpu6MhMo zz@kH4JV4`88xPQME^8QhkcM#yyf}=@DlZRegmGEr<`K-!!%BLf5zZxZ^nfV7hUFTt zhHco@^Y4K9M!h}X#kRSpI2W-^z2xcA&@q_?cZxXF-2*fpb@zV&4I>k2D49q@$wV5u zjy0;>J*WaoW>p1UgSmTHNe?wb&qY{NA4D-SfrgW5H0In;`o9$WH>*F8f05_y#j`>d*PJXf{&PKUO~!;1oX;pcj>e z!^_1)D%sOqwpIB!&p)9eSydT44p%=Bpv;pfPNM8-)p~338wHC=(3!@A_sdQ#X3-xD z`Tkw_&e?w#Iv;*fj1Nj%$y0xp=GiSJ@;#xfYlq;5tso4)9fJ&vuYEY z0s`S$vj-MM3Ir~JgS(SZARvEg!!Qtp?|llRyDi(PLlL$EZON&DLQ`6L6Vk>JwSJJ) zI(hrbcH$)9Q;(~e{dRUyxjM);`UUHRRu#&U6h)wz=0Y_Udf%?&2NXF^6{oFMP@y9@ zw5sn)W(d<3UadA@y#N;lkaC2X3hli&1Y^hcAgN1&Lj`fCt)xDbw!wdfvTxb|&eL>> zC3woIr?>~kW+I{!lCueEer!7cIK$9_1mzvhk_=zG9;|e;4TEy!l)@hk%v#Ule$t^UPlU__C^MZnxpA^;X)Wz4HBD3r3Q)D?iPubtC^l|Vh}wH zqQ^lb_s|sStO)Oda0;SV0V3!;1<|7*@)(Q6 zoO6GPPb3%K{|<;vq*4RK8*gVyG;PloC67HmCei04lDq$iOC0*(!zGMFAfY4z2_=zB zJo@0{SC9z8$xVAw5#IF)C#STa#~WblP#V(*F?1aYB;+Hw@SBBg8iaAWHwl(CJbKR0 zUXwPFUJU>*{Vf0;%`xq=Z3m%a0GH*2Kr(;19083>I&us&j6tNK5F!nw5NJF)A#OB? zLsS6;v8saZ=HdoLR?-8F_@tgQ7S#t)bf57YXoOBakc+MUzYQW^Cc=Woft2(7eoRt% zdoMIEp9}volZ|9p1Inv)$5vFjnkDSA2StH*+7AWaS+{LUCD*uuQVDkG8I?f8s04o+ zMkUv<$#J8BRDuc^l@%3q1wBhRWhFh(h;L9TX;dFY(Mb|%=q$O_{kI6^))ZM#Y3Q2x z0W6!yvCdq%?wDBb0Bf#ci4rXFS<|85$2Yb^vE&-ppjdJZ#}a55mO#U>49toOx|@qHVA7Hv%7pIUu&6$W z;$VWKY!0URroklJIe(qWCeo|Lq`AYSa81iEV-la=9}0dZZqqYupm8nAB-ej%CV_@A z2{epJuJLTjq*cWvXoPOIR#nj5Tzp}Ymh?a)zDb#+QGFD}nB*Fv)AQ-A{x`zp;x3;G z7i%a9klFooIa?Ofr0_N42as$cyIRXvYkT>sIlgO`Jyn)?Oa!+L*ghqaYh0TW$u*ov zpkYJ;4I>h07?E}rk)Q%bWJP}k-Oa@pB56qvG{QDsSyUfIF(QG6JyqIo2*`h8_qCA( zYcXkWF=?2ZuG=McK2c)C5x{0coj5?_QY#M7a3*V*agc^F2{eq!DnAaYfH7I+$Pvtv z!%BLf5zZuY<$x%@a^)JZa&6L?^RGbpR((2x79xrMrD;@2UWnRtg`F~>ZmzoB|XpxKNVrl91z7%MYsl>iZtxb;Z*+o zW05!N+PM6K6kOgYrI12ir*?f?y^vW9gBX&9BjKI2d-tK2)N5k_T| ze@9Sz%_%GCfkycDE3<#;peQz10u5WgHtf`S4a%Rh{rW~dJeOkI#FMXKg%BlfI^maa z2`oC)#RD`hwebKA=dy;82Wc3Wz>CAUtn%`pMi`e>ZXUtxJglS#8sS_rM-PbNYgn!U zYuJWeJ^v1vZ`9lKU2L0sigOX$)JvW&4IPtdaHohv-913#QFnh2&@eKQhLVXiluV?d z>sX`8-GeHiWL8zsHJH1HmGn>}^jw5R^+6OP6KFV@M#Ju&*U0?42P2K-R|iT%--S|l zZL?$>)(|C7a-)bt%{@TlQF9N_a42h-dys}f3Hn2)OQXu%gBoE_R+)PYX6|7nJNB0!lZQJh5C)2e^<;x`Hwlb|z=2k)1iTFjz9 z7V`bO@SU^&E_6Qpq8J~Pwvwm*EX}i9wo<%6-Rd07gONN7l2J5Em5ChYiRxPZ{GF_R zbYS^9i|5N%wyNC4Sr$*9-;HH3kV)~*RFz>IXU}hnA=&(RL+4LWO9KQH000080000# z0OxW7<6X1h5rqK+N87>4vuP8X0s>)Mvj!GL3IvXpSTmDPARvEk!Y~ko?|q8M_r!^z zrLvMBwH2o#RVr0cZ{c+6z5FA4T3Uu(e$O?S+ z25`#XY#5Xar;?$waMOAcNB7>#(b%?a)=ma%%wOR3^7VaKHI=$Q6$ez8f>Z8*1>@=x zUl#Fj4fA0cPrQ*Jp1P>tcsnj~=mU|&0Q3;|qPKGnpMmN)-;E? Date: Tue, 25 Feb 2025 10:36:37 +0000 Subject: [PATCH 193/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 9759 -> 9760 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index c33a7aee6..7b083b189 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,39 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 10:16:45,280 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 10:16:45,280 submission_checker1.py:3270 INFO] --- -[2025-02-25 10:16:45,280 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 10:36:33,005 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 10:36:33,005 submission_checker1.py:3270 INFO] --- +[2025-02-25 10:36:33,006 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 10:16:45,280 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 10:36:33,006 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 10:16:45,280 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 10:36:33,006 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 10:16:45,280 submission_checker1.py:3286 INFO] --- -[2025-02-25 10:16:45,280 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 10:16:45,280 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 10:16:45,280 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 10:16:45,280 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 10:16:45,280 submission_checker1.py:3307 INFO] --- -[2025-02-25 10:16:45,280 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 10:16:46,254 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 10:36:33,006 submission_checker1.py:3286 INFO] --- +[2025-02-25 10:36:33,006 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 10:36:33,006 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 10:36:33,006 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 10:36:33,006 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 10:36:33,006 submission_checker1.py:3307 INFO] --- +[2025-02-25 10:36:33,006 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 10:36:33,985 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436659 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.373682 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.41546 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.919 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.406142 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419809 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8158 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2172 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1393 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.63549 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 58621e61a..60d194b07 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -820 +821 diff --git a/summary.csv b/summary.csv index 3df68a9a1..a53611937 100644 --- a/summary.csv +++ b/summary.csv @@ -1,17 +1,17 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436659,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.373682,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.41546,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.919,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.406142,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419809,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8158,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2172,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1393,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.63549,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 97932655b..e6cf1894c 100644 --- a/summary.json +++ b/summary.json @@ -4,7 +4,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -34,12 +34,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436659, + "Result":0.373682, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -99,7 +99,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.41546, + "Result":7.919, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.406142, + "Result":0.419809, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2172, - "Accuracy":"acc: 75.600", + "Result":10.1393, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0f4ee57b78fddbd1ace21f1d5ebecd7f9439a5be..13f4a471239275fa77bf3744be8b2d490e0e3e9d 100644 GIT binary patch delta 3017 zcmV;)3pVthOrT7#SqKgf9Ll>)2><~8K>z?40FyB!6_b1jHUYMivk1n2f%fyV8yAT( zki%dx$u8rEuQD2qvwYa>A|#9_>GEG?8s%eYsLC{WY^$cKJVb+RTvV)#qrpHH1Fb6I zBF*CIZ>vjUUBN0;IO|xUi%qnz2}^ejyZAhwCUPCqTD7|D;m05gItNMopd`x{(s@vP zGGgB_@?i;!(r-m^(YMxry5RD;K*{qcM(@639<=U@ir0$oG~|jFT+@P^t%6qGpG~I> zzEcMDLEr7eB`Pe7bd!cOG?|2=b$TVNWq^){Gq0liB3LPUMU*TWe3g|H0qtI1^cbRbVjeNQtPV(z9 zohRS*24V3$vzI@5b z5QbHQ#3oXyLE^Q$MWW?url*@ch#n83hai&s=LjVZxqpsP0w6LH3L+z+ATkmPqE`VT zh69Kd;aw)2g6LI%$aJ29=ur?c#v(E2DDjEp!u#J#0kMfxYJhm-?M#WL?b)K_vB&30 z^pQkz_aC9ep$|SN0TLMrC6SR(5>evO2OqzJL?)cvv?mqeU7v7rN(*|}0IWl4OrMFN z>rh6My$mNk58$$#5J)DMBSzzrjvO-@V35;L2ssU9VUfqT7KCYmOs5e6nG+T(-Un*<64wS)F6|L2AE_t zz$9usn=)xtG08MSw_2+z=x#2)FiA^#Oe4NYnWRyDE((}LjnL`o^j7~HVRCVoPlby$ zlmy7^e!843i)m8$n(+fjHj!Pe<*T*5eAOJ^wMz~kMTx^iplQJNDSwfuacxQ@YLG}q z14J?!Ad=Alk#-f4Oa(w>MFri>#TO!JNsnm+wp}5r&qV=|j0T=6?KcGEKe6}PNP@MP zw6~ZvOikDA5;LDDvEc~DWJ7&8jK-x#97cmo*0ADm8eo#q0Fza29IgUjvdWJmz>ota zJ*E+45(_>TII!~s7J`sm{dl-#Jy*-Qu$mBGXOin||oD$!u8QD1PXnvXR{CtX_?6Q%}AC@C59@wzmGfeJYcYJeHG5 zs)TVqT&u`U>FRI(FU9`N>d)g}&30c+i|O=BbI?b*1Z=IzOM3~ zVcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8EZ@xbhB$tF8$3t%{(bN+jPD!M5A1)0 z4mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$-bFK$UkB-@#bn)CTY(aZo#Ie~^w=0^+ZcP!qNcFgDvyH-HG@i# z$EZnN_ui?dE<2hZAUiOeK-rLZ=1QY-O00;m806zdBon9SB0syoB7fc8Q zLr9@olTRQYf6HpaFc3xe{R*SKEk8&o!gio7Sv63oOG|e`nk1su14*spzpv~#P6AnF zIl6bw+!<7^4zi8@z&fE-g|Z|?5h$j)P`e6!H0$^gMb1;jX{!}fs0W8u^+U-FVcNo* z)ds8=;GzIhjxbZ9z4wM-?ARV8bxCljAa1pl)Q8gUf3TtKhwcF9X}ZJ`Jmu6=JOE=e z6VZg^Y(|=2+l~OvFti{+d55zk!)LDtE8We8QMqtR;d=u&ttWAIADoztUDqXDF<{9LX%?%PXU*cvk1n2G41DNH!c!oAcw(Xl3m6R zUu85JXZf(%MMxM=(&fL(G|I=)P?c%$*j7zdd58wtxTsheM}vVZ23l3ZMViIa-&U8z zx`I`xaMrOx7n^8b6PE56cJX;UP2@VJwQ6H<v;&e?`fgtH}1?lJaR~1*UDz0Bu+*B!c zGPQ7~7K=AXrd(n9B-Pg?SFcO1xrBbMm6hydYr|}R4{cU~x7)-KSCDQVbA=!*d;}75 z|9k=xGVq^)gn<7ANCjDY+a87}Q3~mod$mpJNi9trU4g@23~!^^P$v0;P2OfQ&2H1h-84#zwNH9`Z)oUdJag&DSt#YrUA++thr28_wr@;e4?d&t{LwVwUBrslB!8HjHP( zXr%t44rCTYlT`h1&>DjI`dbqz<*QbicWwD!Nx#0Y`XyFt49zlh%PN`CTnwGdp(D52 z5tlgRPCe!l1|5)4=zxSm2PEVx{EJ%w9S`>?bQR%U5Kg|4Pq)KKem&56@|}O?QX+YO z3|+sSbBRwR7jA<@j>fk^SUk_{<&Q6`s`f@XkMpN2PQvl+y`edpz85ZpVbvh9iBxKk zcAe`K^Cl%pcpKx+Y3wpc(whpB+eGo&}p+J+q3@1DW za9K_WB$LY#(72={$3VjvL>dYq(ohP4#-kJBMuRv+6;KeXD(G%5Zct<;JN#Uk zeGoat)gtHyTJK zsDM#fQ9)PGvxHMt(gTh72Bnfl^-&a^B$0;Bl3U$>i%@P&kp-28u8AMOvWXn)%$4ho ziS-V!<{FkL!4jV}9SVMYV>=W}u5k^DCD(8)freoTGz?3BuHjf3Rak-w7?u?kbT=1Y zu%snD&83h>Vqf_COFFGV480l zOtPKx*O_d8BE4EnnmbGi*R=dHCh__Gq2OoYHa*h@8rPysat&t^Xc&_~!McX7ba;*4>aPNlt~)ZM^TJPt`RytpWf=RyLQ=AWr@c`aNB@??NcJT#RvC6s1&qon%Z{Lqnp0NN1C8)g5$4PRQT$YdYrv^U!|ohT<3GFAr*jaarZ&5zNlRN_wCX&Lwm7fGEC(jt|HLBb_r~*o6 zRRvvxxqDbi4>dy1MOaiHL@_dfhLdSD?CyDu%)fgu(nx-FpfvPdD0SC1OSWMRQ3545 zia6BV12i5r_W%utvWB?_X&98CKXkf(G^)%!s1XKbmAS`Y<{nnk1C96=g_4otLsS?KdaIGRY zrK`XCzZCm7t3Qx`k>~Bjvq4l?`h&^p&-t@xHd4|*RzDQr6h9Q87nOy>%f&>0D%sOq zwpIB!&p)9eSydT44p%=Bpv;pfPNM8-)q3$81&c}0nZ|?n%T6t3(H{%>{$2Ra*?$*0 zAAV7c4@z6fQ-7A`*)3ZsUZ8Gu4(7o~o(0J$nx)D_4)a8HEr0$_RzEthe4WMf-5tso5)9fJ&VY6xzngV~}T533e z=b!|n1d-&lL$HEUbhOvH5dU<*^7)g+rAJ8|SM@WgZ`9X_WaECYY6doLn zXp2eQWe@(&$u zl(v7thO%$k0M65Ni6wZ-si(LH#%3a-6OywDX?|=w064?Yf&}Fq&XNpYydJD{vkilC z<&?r74a{24;^aO!F&VqAOS;8i4f!+txp{dTR*i+~PsIS$C1-?LpkCX$#MebQTuYtb zgcI)s@W*4&XWmYWjD5hP5P*zv&&tQ*>3Jo)Ms=R%MV#hwUTm|JWJ_{ieDx!5`Y%9v z8uD-A88GI){RH(Nvyvgo0t&Ur<*7^w0000&lV&7&1=H*y3t^MwBv%3IlRPCl0bG-J hB_;$efrGn~p(PdpJ(IX4J^_)F_a!3+Ln8nH007;MknjKi diff --git a/summary_results.json b/summary_results.json index 4504019eb..ddec525d4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2172, + "Performance_Result": 10.1393, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.41546, + "Performance_Result": 7.919, "Performance_Units": "Samples/s" }, { @@ -293,7 +293,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -329,7 +329,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436659, + "Performance_Result": 0.373682, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.406142, + "Performance_Result": 0.419809, "Performance_Units": "Samples/s" }, { From 4669e7ece2b4a13d243183cc2e64ad22d60af006 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 25 Feb 2025 10:37:13 +0000 Subject: [PATCH 194/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9760 -> 9760 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7b083b189..4a84217e0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 10:36:33,005 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 10:36:33,005 submission_checker1.py:3270 INFO] --- -[2025-02-25 10:36:33,006 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 10:37:08,887 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 10:37:08,887 submission_checker1.py:3270 INFO] --- +[2025-02-25 10:37:08,888 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 10:36:33,006 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 10:37:08,888 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 10:36:33,006 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 10:37:08,888 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 10:36:33,006 submission_checker1.py:3286 INFO] --- -[2025-02-25 10:36:33,006 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 10:36:33,006 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 10:36:33,006 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 10:36:33,006 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 10:36:33,006 submission_checker1.py:3307 INFO] --- -[2025-02-25 10:36:33,006 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 10:36:33,985 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 10:37:08,888 submission_checker1.py:3286 INFO] --- +[2025-02-25 10:37:08,888 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 10:37:08,888 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 10:37:08,888 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 10:37:08,888 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 10:37:08,888 submission_checker1.py:3307 INFO] --- +[2025-02-25 10:37:08,888 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 10:37:09,866 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 60d194b07..1a3716f46 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -821 +822 diff --git a/summary.xlsx b/summary.xlsx index 13f4a471239275fa77bf3744be8b2d490e0e3e9d..042e2f911edefebc0d33916f073b6eb586b99a9e 100644 GIT binary patch delta 312 zcmV-80muHJOrT7#OdtdyDEF_EP#`OR*&%%h+kv)Z)j**sE!_!el89OlB(;wJzOtP- z31rpf=-xSVXHdD=t0wsc=cUmV$}@rz(A)^Aw-tI{FVlOJ_`tMajnPn{4m?`aHzl`} z8wan>Sa3mtPa;Tr%58;qAy|sB=Q~i$XVIafxG_$#5KFhkmho@41zZqvixmWaCRkv& z2c~u+q61R!32A>^GXMm~(0~FRJkGNmpS%H_@>d%M<;vMM_i{3}ZJV{T z!5Z`D_;dC0HmsUT-JgmBs!PEscff*ib&1c5c({i7u!<+%NDz-*)UUi978(0MBryOT z;~tfd+0*lSiRywB^OO{6F<<8ygwF0LdHCu_Ui6=U>M-Qr#3NwLefbIMKeLh{3j_oq KDEF_Ep(TGC-Ix^s delta 312 zcmV-80muHJOrT7#Odte9NTFJjP#`ORKS(ITcAza;HBhKaOLsz=B%;;>Nv-3*uk1KZ z0$F7_x_8dp8C0$gvW@=0I-ymCvLr-Z5x&Qrx{s})qJ2ZvVmL&*$b z+QOUF2CNt0q5x8kFjJwu_l98X*d8QxNpPqjZnc%vhtlq_q3nn50Ox7C#1cGz<-uwhqENZXRik<-OYwkxo}G1djmJECvkQkoS2PW*Ckys zT0{N}e{Ejh$5msYhEp*>b;%iF7O2;@F7ag%4%brWH{rxv0sN^C`i-|~kz*h5C2y#rH7ss{ag>(~!G~C%`fH^(UzR0ke`J3j_p1 KNTFJjp(THof0LpB From 6c5734a4a1e7710adaac15b1731000cb2510ec95 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 25 Feb 2025 10:43:51 +0000 Subject: [PATCH 195/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9760 -> 9759 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4a84217e0..c748fc804 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 10:37:08,887 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 10:37:08,887 submission_checker1.py:3270 INFO] --- -[2025-02-25 10:37:08,888 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 10:43:45,875 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 10:43:45,875 submission_checker1.py:3270 INFO] --- +[2025-02-25 10:43:45,875 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 10:37:08,888 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 10:43:45,875 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 10:37:08,888 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 10:43:45,875 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 10:37:08,888 submission_checker1.py:3286 INFO] --- -[2025-02-25 10:37:08,888 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 10:37:08,888 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 10:37:08,888 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 10:37:08,888 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 10:37:08,888 submission_checker1.py:3307 INFO] --- -[2025-02-25 10:37:08,888 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 10:37:09,866 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 10:43:45,875 submission_checker1.py:3286 INFO] --- +[2025-02-25 10:43:45,875 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 10:43:45,875 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 10:43:45,875 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 10:43:45,875 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 10:43:45,875 submission_checker1.py:3307 INFO] --- +[2025-02-25 10:43:45,875 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 10:43:46,871 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1393 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.63549 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420446 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.1507 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 1a3716f46..20f9720bd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -822 +823 diff --git a/summary.csv b/summary.csv index a53611937..5e89e1b66 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1393,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.63549,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420446,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.1507,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e6cf1894c..9d5efdb7c 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420446, + "Result":0.421108, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 042e2f911edefebc0d33916f073b6eb586b99a9e..7b7668236a73f1c9ac4a08fece3c27b4712f719c 100644 GIT binary patch delta 2775 zcmV;|3MloUOrK1!SqKGU5ucn(lU@iVf9-Rk$QH)G-@k&Q=EF?YiZsYuD!P?q-eQc& zWRh&%PZq&esa0_S^YY&>s4;rZ17T-pm(;yqX0&bn4s^`)s(|%qJ<1$u;e{$q6r|DJn@KuK6Nt&HDD-Q{xX|noPnT6T8G*o5g zKekm{3&lW;hc`H5DQV40-R?6>tHu}QEF74{}p=wch~X`-duhLwLFPgA*x zX|8)+jqszN`kli#dQjq356QyMKN)dg7};qF^44#8b3V4_rs3+jLCNaKf6v~1$2@G_ z=ZQCp?=<9^Hr&vL+nt7HHlFRE3_K`-_9|`RN?R=6AeC}O%Oj<}F1dbPa>FI` zb1hA>SFHuBJ+fE=cfX5cf36_IJmCsKSojDe?@1`-1P7a$?feFYK%-6@ce z%d86|BVjhvQ*cb|_ve@{_ow4uj4@W|;r zr!jpVL+6|ekd}M|moMWUb-w=h^3zrCw14(~wHivY<vxNxt?;2KSbxH`7?k)her*FQ%I3y^~FRER$>vlQcJr znZ`ErAXqFK!D88nf9CVYcsWnA)l_4n8U)dN6pqzj)S*oMaGIze4x1w|Uw^Bkq-@nH zv!Sj2D;U@J^|-`(jiI#+A@qW|+riK|96EBT9dn6IuGAAQVbB2yg$_t4bU;GB!oN5r z(D87WLRS*r1>xiy`D{0w)Yk)pC*S$!4kePu(DmB|mv}^Sf9cdnLv!!#UQ#IL~{KcbBRr^ zpA#-&AOZ;m5lASAKte%uOF-n|48)S~E(oU}x+Nfj!BY@j3L=kbk(diE@rdNo{oetx zj#4Usce~G5$TB6{#$HOFgoJ4ZU2R9Rs*(Cj^qo^o6UuPnaNrz ztN`WpW?*Y79nCA669wL9KN5Uj-Lfc?T;mAJB-o#4OacvK5@;BcT*H|ZWlVwujLDJ& zT|u8Df1I(F9_Yj~D3dg*kD}-li8OSM+!_8`L~>_>%$YQFP5c0qb<|j8rrZonZ1#UO z$7qQHEb)`3Bf%fuST@CyYaD}O$u%5HpkY`74a1UaIF?2kmLLJcvLr!wd+-HI+R_7^ za4cz5A4IVoRG<;Mpj;aN2VB-sW0hUi&A_(ze~^x8@9gdGgNdJs9tr-a(Q+u5T;rG& zOs?Tz0u2KbXc(AW!@)Gmzyt{xm?a6i+k-D)(v}{ogznz3s6L3|V1ftP98B{~gGn}X z{yLL&lvjyKbB{^km@Thp4U3=N9|`_M+@epkfyQZ3Cb@<)2{epJpkYjMjcZXRTV+gw ze@^IDYgvNs_TUSXw510+@l47jjq0N)#w6DWoj#r38GkKIF7L9baJhk!0GZy;REG!Av@01c#G=e;AcO z!>BB?>!1XT$}+=_pnjS&*3tu=@Mj{-mjj~sGZC%<&qQkW=5Q*1{zH*B>e;y#yQUpm z6PE52Q0W#>2`o9(wF5K`we0{6r?P@^2Wc3Uz&vA9D$Be(s1rtInR`c2bIloR>48r8 z_A4{#peQz10u5Wg*6h=H4a%RhfBpJK9XwZJ*TR#nVTBL{UOM3wa0v`L)WZWb4mI%r z4d=3gjR$EMm%xd`xGZz>piUTlB7p=2TrB@=1rI@T!jfA*jRl+3aO zU4wahSW6FeLZ6GUs6L2dWC9H*)2P|o^BS3d_rXXl^;Ln=(D$L#9m_1(hBZV1l-wv{ zQ*#f{xYXPOG#ttb<{qSBP=fK$>Cz}O_n=M~lx5}~gPD6+OAmD7wJ4O#+=HSRloiZ9 zM$P7)*O*L4Iga9oZYpc3f33>u)z~%lWD5XSupQXu&Y!o>WIUE9aym_vAj(eHI&#~( z{+s_Rad5l-^Y|Be)?YLqhPk2NpRWIyKMm(&CHZ6hGXZw~GXZ*@EEruar&39u7P76% z$3^xD<;AMX@Nu;MfdFL@hfy4+PwU?EUnp2k{mv{JzF!S$IS>C>TgvzEf_L`8yTJbN zi(-6G+DcaXvox!2)l2>Y4Xbmw@W=ApkH_IWQKoW~6{>65{dc_n$${1DG+L})S*mgu zrD-&Kem9Z+P{#Q?Q&mP$ls>=7Ps#SD8#;div&|8I4g@f(=Ri%yy|jv@0__as8}6k6a9j9LaP#GNs1y+Omm^?5`Apf z@gs_yr;5`?D=5(k99mTm1v7+c3vX5%uwH1%bfaGjMnjhQr0M0NpAVGPDvn0b;uLmpLWJ9mq zIHm9>1C!Q^IJ%EcjK;QYlXliyL;ehZZC>B|Rb!#LQ!zkQ!5Lu|sMoeC@NE$e*HGs- z;lz6Z{OJ_*iML^qb06?11R&?Q7v*#I{IXr6Do^u8oaS-9*k%|hna{|4x$8#W^)lU@iVe_M0nxE6)q&#!Q~`k}jOVl3aC3QQ>L>ZP zbdss_lyZy!*8pa0lgqy!8wi2DwjEDT&p36?OE<`COWML8lBE?6etn!ylzW+^Q9Ns_ zLeo`6&cb*Q%|>nY`s(wZrz&X{%m%?Ep2@cQBvbX*&VLRb;^g;qEM=zT70lAMe>%>x z`F^XFhGRJmQZ1g#S$<_0C(|Iy-zKA0I!|OUSO!feEkoCx)-;G_>Z*qQq)r|2a2SR1 zV;nA~a+a+sN#rER^6sbOXr8K_gF%#EEAmkiIc%$kV*mIzL#VCJ!Lsd}NIs-nuazQ| zyEy*6cylsnt9l`Af%fyV8yAT(e~`mqG086Dhp#dkjkA2%>>?zLC+YHEWg6vUX{gFH zcx2Iq`VqL*1R5O|o+fe~ z(^|E||>3nOYEUkW5iw`6SiXC0DOYu2Dii z*UCzEvbEW44{fNx+il_qe-)&g$EXm5g^!Gc+&`Zf2^sj$jD&#yg^>{GzA_R5-3cQh zw^^5wkPqjSk&wadF%mMmXDBhq==K>28QpV6LPqz3k&w~-#z-vwc60dK&4CJRlFSiG z95VLDC^1RC9~lYB_Y)%_`F=);M{4=FvF(>8&i0a=FTC6^p6XS?_oqpU5;{!=$bC zVrFks^KCev?}hWle_lMBJtm7;manGv)~eeuo(-ds`inY{SrAQ9^}|7H$jsN@nn)>M zwaUC}%l}IH^?lVZv07tjmZ4i#$&BXm(76a5xz&zP;*dM_7$pE5BcaeS5(*t7Az$HN z+zRM0+@sJ{gm;;6@{N4D9ZvG=F`Xyh`DZRA5@YE4?HnaOf010c4H7vT-v(jvJhPWS zzO1U+8|6ICpRza!$G7)}=4kp}xDbX_gTy9MsX^kkyG5eqYNn@~Jcu3-qK6=o`{xKH z4!M7hQ34<`5(*+Cp&&963Zhp5B8CHq72#bboPy|8fXH;7g6L5YF~%Y>=P2=s95WhVkke2Ie>n}MkkNQ_LfmLL4!H^_h*cGI zHy1Z3LP?Kl#3%KfK~$fKqWg^Jj7I3>1G(7h|Jxw)Wg;wS97s9O@5dySxA#Kx^11LY zGucRnHK4p&cWgzatNCSfqTu)04+ZY4+csqqHLjpcGW+wuB%=W)84WOr8e~#bG09W_ zOjcCTe--p3;S@@GOe4NQnWRyDE{aZ(oQBSkTit()NN!D#1(SxZi64Nni5%<9l zb`_CK1wdp)1>McX7b0m%k7)$9T_LK^e?otaJ*E+45nAJ^yU03P`*`{jvzK2;d^@7+I7dLqrXtexNxXZhtasysl#ZH${JoBe@+8b zG8&+=%C5sz08~~Pb_Dg)oI**DX#~$iz?Z{B;h6|(m@|=vy*Wtb&p#A-qn@2Bv2EIM zG-2y*36)+6m5e2ax^@_iOKm%h2C1xJ+~G7pC1aj(D3w*-9j+0evdX<9sJZ48N_tEq zxcv$y9WDyyN=5_g*M@yMuR-~9f3{!WsDtNHY+HEpHLMV##7igq5-u5o4)yRb8kd@Q z7!7h+!^XpDfJ?@S16)=)dALS^%PKFA05cDi^q5AFOYrkBQMiUh4YP)A*wgdxfcZw9 zJ>SK)wI^T4nzpHzJY5<(Po{xB5r=wv7>!50J&Xp(FRI(FU9`N>d)g}Sa3mtPa;Tr%58;qAy|sB=Q~i$XVIafxG_$# z5KFhkmho@41zZqvixmWaCRkv&2c~u+q61R!32A>^GXMm~(0~FRJkGNmpS%H_@>d%M z<;vMM_i{3}ZJV{T!5Z`D_;dC0HmsUT-JgmBs!PEscff*ib&1c5c({i7 zu!<+%NDz-*)UUi978(0MBryOT;~tfd+0*lSiRywB^OO{6F<<8xgwF0LdHCu_Ui6=U z>M-Qr#3NwLefbIMvydUn0tF8o%DYUHU?f`s&6B(&JONje`6NLB>ytYrIssjib|oeR eAt?8+lb|IQ0X~zrB|ZU?llCPe21FwO0001CFGRrr diff --git a/summary_results.json b/summary_results.json index ddec525d4..e7b9bd9e2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420446, + "Performance_Result": 0.421108, "Performance_Units": "Samples/s" }, { From bd7cb81df965d65fb29b987621eb46660198fa9b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 25 Feb 2025 10:48:13 +0000 Subject: [PATCH 196/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 12 ++++++------ summary.xlsx | Bin 9759 -> 9763 bytes summary_results.json | 12 ++++++------ 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index c748fc804..908f2603d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 10:43:45,875 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 10:43:45,875 submission_checker1.py:3270 INFO] --- -[2025-02-25 10:43:45,875 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 10:48:08,115 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 10:48:08,116 submission_checker1.py:3270 INFO] --- +[2025-02-25 10:48:08,116 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 10:43:45,875 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 10:48:08,116 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 10:43:45,875 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 10:48:08,116 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 10:43:45,875 submission_checker1.py:3286 INFO] --- -[2025-02-25 10:43:45,875 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 10:43:45,875 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 10:43:45,875 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 10:43:45,875 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 10:43:45,875 submission_checker1.py:3307 INFO] --- -[2025-02-25 10:43:45,875 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 10:43:46,871 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 10:48:08,116 submission_checker1.py:3286 INFO] --- +[2025-02-25 10:48:08,116 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 10:48:08,116 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 10:48:08,116 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 10:48:08,116 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 10:48:08,116 submission_checker1.py:3307 INFO] --- +[2025-02-25 10:48:08,116 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 10:48:09,076 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,18 +24,18 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.373682 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.919 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95343 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419809 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8158 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8403 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1393 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.173499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.63549 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2508 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.29297 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 20f9720bd..2a83cc214 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -823 +824 diff --git a/summary.csv b/summary.csv index 5e89e1b66..4d46c8602 100644 --- a/summary.csv +++ b/summary.csv @@ -2,18 +2,18 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.373682,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.919,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95343,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419809,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8158,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8403,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1393,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.173499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.63549,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2508,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.29297,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9d5efdb7c..63a82fd00 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.919, + "Result":5.95343, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8158, + "Result":20.8403, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1393, - "Accuracy":"acc: 75.200", + "Result":10.2508, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.173499, + "Result":0.325687, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":6.63549, + "Result":9.29297, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 7b7668236a73f1c9ac4a08fece3c27b4712f719c..aaf094625090c0834b4907ba4073bacc694bc8e1 100644 GIT binary patch delta 3221 zcmV;G3~KYAOruP&SqKhcs$PFk2><{9LI4050FyB!6_b1jAb*I-WRh&%PZq&esa0_S z^YY&>s4;rZBVlJ}m(;yqX0&ZQPsu$8G z(|%qK<1$eOau_To*;V}TRYs$6mY+7O2npj!y8Ks}Mt}LaG*o38JhoL+RUV>2HZBq? z<7hCD#fesxaFJ&5^taU^u}-iG74|w-=wcJ?X`-duhE;qXPZPP0X|8%*j__lU1)akr zeo&HS59vH8J{fUf82M=li`H*Nb1}B&y5Z`%LCNbV&fa~;JZ#<xg!NJ~C~%a?HvJ752M`RU3( z?VY_}u7=WlIW&rYvhm_h3*%{XI?4Z?=1ZOY)m$dSel#1(Bww(}ekRkbpDymEQCh5h z(*C`r>CG&Wa=FTC7K^Foxqq^bPh^^}VbbP$G1J&&?uYYvBb+Z9@oe^(EM{50nrf_7 z{V<*lqmlZHI*?fqO;Yv4VRH!P>u+_Gl&@N4KD6b3d&c#DeKjtzT4QJ}LkNA(-0fiK z91b11)Q-8tCRgeSmoVsnghB@-6gnUwf5N{wCD8G3mqJ$(-UZ?0FY?)TIH}JEgC~FU z&mBr6kD=?g3oh}9Xa4B)bz5J)DMW1w-! zKu&;$F^Du2LZqP-0*y;2#Ek}Vh!Ri`%Mx@q2RA6P79Vutkv!)tst2OzbH)px5jy!m zF1N;i|2BwxnFtFS2U5=S>oG}X|6XWrJ{SIFCTpp%0+iS5fvu>tG|w_yd)B7{KWRS_ z{K>jyQ7XB{5tK^sgq~3eG>l51VN`Mrr&5$r2@)_WOA>Si-Ag!QEk5YPGboibs)wTJ zB#AV1mfRZtTZD3JiY%x!bWQvKmUYxvWv*Nj4@|6|0BerX5__=3dre1zZ{L%&4Indf z@C8fS;)6~&mNcpdqS!N3pb@&DTpIreT-H%zmFKGKfo<+09n;==y1xr1-W5F(e6y1W z4j_Mk9*E*#f~{;0runA9B#(3cI+JyjSBXh;mr292-Fn%L z@I7&h?r8&!)1pjr4QCQ)7?VK5nB*GQqD;2Rm;{~BhplA^x|@S9Owtw~bmEzmNgCBd zQH)8h5jx$U-Wq=`OfK&7sc^A|k^q_APnUnQWid?(Pcwc1$vUd5w0v!CFJCp=bF4jv z8nMS?BDih9@+gs9<5-kPuHi%i4I>h07?D83h_uRx1PK_CB?-ElgD*tV79VuN9=x)s z9*SZ_0u8&XwB8Vq|HRYRS_-Vhq_xGQVQRW=?Xd)iJw_Y>Y&O)112hh`;s6b2vVwmZ z2Wc3SK*N|U^W&fdjL9-bj$oD?*5ZRsIFrnk1EToKm21GtwN7WwzXIi3_2~#=)DfPm z@7a|UJ5C+_g-YPVp;jHBai~`ZXgHM>+&W0Zs012DWtm|IC16yRS#|_<)SR&vA9TWZ zMVK=OMDbk_t^vCuHM?^-l|O%5l8&#bI2Qd3k?OCydK7 zH;-U;9@gT6PB@p$(F3CR8kTFo8n$Lv&%Xoa8};^l7u)8Zd>w0A=03x@p<^-)?i8`9 zy9a1o>h1v=MkdlwGLeRoi8OQ_Ym~WrPy$M3S%R*?+&!$tN1f1p5f;@0QH)HW;ba;$ zyL(VpV1EI9&ZifHF^_IEk{S zRqw@b6f7n|XBrRQF9)@lMSm>h`*-0x``}$@fA~c)J}7M^Fa24XSGVk?c!7r1Ih+S0 zc^)LAXqGAyIm`>ywfrppJ6Zkc!18q#&zG+(Rk@3^ES^5U8_QrIlj5DJD#JL=p5GLw zWb@+E-{&w&PY!M=S$>ta~pad|l-|!?=GVcrDo<&uEKD++`2`)sEr7Aa9JMx^sMFS-zR= z4RQSVHh7Ni{rliu7~eOhAJ_?h9c&`5iEu~i`Xbq=bu6#DEsi>N*N!Rh+zR}hyo+Wg zzYfw*i_ho=SYH4B^mJke&P`jf%B3&zi#bw0`@YR4vm} z>9H};wlVgcMNMI~RUQWwY6g`ek5QAl?!8k@U3N4-Kz3j_fwCbztsKC^4h9^;;rkJ;06C9^_xL5g`winN$FIidFpN)}&-kyrg^pc;q# zn|J~ob6xBvt{@le{D%8%Nv0$zcQl0CW-n02BZK z00000000000Dys8llde;0q&DKB{~6MlXfL01OeBfVUwUG76Cz%wqHXp9V@kac2)$=YKkdzMI+v6L)Xk}W2RnMn!xWv#I@wh@h`5D`Nj zqasU*ge+;Ikv)>^;>mb<&Uw#!&U^oPpX+?S_x=4|*SXJKjHlzPFY@qv3exi|;UEyl z2m}%XQNvK;)JPr)AhFXsJ>aX2NQ=W0dm?ysWlXK?uh1SY4`1Kx!B_5A>TdVU>A&l} zo~t0u+17fXxT$X4>c@77EvbmlgMB=amohT{VSP5PLeYG__Z|nA`@Vh~CA>b|1Mtff z{oeGJ^(ps;&zQU9Kbo5;VBw=Ed0P|_l_@=4_4o-I{Yu~@K*U~b@0Y2Hx-w8w_~Ryd z<{mjXe;iqrE+?MUY8a6iHI45X2u^ph-oZ&ksKZ4mU$iCTST!%D(OJdyp(uXq2yD!k z1<9)ICd9NcW7WCSl9^C6wyIg$R$o5bFFz1*2{Ebe5J#F)E1k1nTe6RvG|$b~P!fO_ z5g6N!v{-<}#9i>2F{rnkD*bN0?L^!#aA z71_?t9tBZq)tKBux0F~)R0Ccy`3=Qu;}$N;Y-A?s%zL@VO{&ZYH&PpVjjUqDaIEC6 zIK^;++-j!Y;XMI_8J_N0)Xo;0TG)t;$@_W`dCYfH$m|_!=KxA2oXPr4s=WwFI85$F zYiMsOxO%3C8bIJrHic7DLk-9jOGfzZth>+JpB2DJNqJc@x9BOSY|i=C6?W(@y_?f3 z{p*gj@oe6~)U0r5w%Vk0a1&>9Wx)Pd9k3DcT-<^&-KPcM72fNr+`saHh5P4LmP#tMLH>qw|KJd&#ZQYXxr2dUGORvhkgt{oIK<{Scu;V$}N-Oe;{|l9jGXN6fmS% zTUg0a5*r3K*BjbHlM0{n9b1cHjj_7$ z>GDW%HC$DIrgu(@`AMp$3Kw|r$s%G85iqfall^dFg^^?t^l&=^`t@gvO8nEuFr8(y zhh&i<*tr|@Wr$ylwSMaR{N-i@AnPvQn)kLaBVi#lQCPb{H21-WSb)~)toRf(y?*7g zzoOI!rCecdwy%LuxcgTIhUz-W(}ns>_r*gHUP69iRM$4zVIO=HG_f-s>Sjbo<<(?K ze{aWPsJQZ!qVI<(xEsl%M>#GgB;TlR8t#VGA5cc{pp#F@C?7sx13;YY@S{MKm#HOk zEi!B>6M@8rEpc0Z>F3H1QYKuGF{zQdP$1Y?;3~cgbNzJAE7NW0!|VFZ)MUc1g`dBo zDRhc7oKr^S<3|D`(Q23bv!u&o7TMKh`u8U;N5NQ<_*JWDa}6>_hZ>yWNHHg;n2+bE zJo7^&(AhF@j>%||ObeD+a51+V&x@g#^DnGT|{djW@9AAKuc8*BuqQ>VP5F8tb0Rrix4 zUac1bpFTuT~ z#m%;GSFOP^E*yXGJP1+Bh;oDx#4-)r%9Lw=6hITZ)Z3B^B66{x_F`G`mF>>vD*BBT ztOEC%T-}%nQUo~0X_~-1iz)J=M9M`tYw@t{oQN%4S$pvL9`X6le~H9>;%hxGJH_`e zb<2cJeQ054ZOI>w`p-pL|K3o-&J|!Z93cpo=1&U)w-H_R+MR7vDLqHe;Z|y;N9-Z) zOFH96g5Z2QgKc}vlgl8)EF&sukD2|N($=6%UI7ZM#j<@aNw;+C?YH~5tGa=}{Rw}k zDzN|O+*7A^`EvpXS*vaf&S8w;rv%!Wn`Gk-Z}CwDPrE>YzJ|dOE?gr1QK7di0xeAV zuJZ}Ax={el=w?9OqHl*eSqcQ7rYG#Tm8UlZp)gR!8d(*m0h=vmD(m)<{e7E-gMid~ z<Pv-(@~Ht@+EClkjk9HUq5<`^Jmz0g};J6P~d4i+5xP+81PrS!Zx zU4&8D)p2G6KK-V=Yl&FLi3Irw-a98*V)KK&>Wz4)kcW%ApbVd&Sz&Ss? zbg`Ty`j~(Vi_lEt*uLG;wX-)pm!aR>Y16q;Dxm{FYCj6*fVmrEQ|&!#uM6iwfMa6< zXvx=PTgdq+Gt!r1PDaOdzWBS{2XroP{ppm0hn9VnvBuR;0nFSE7PVRVxoYMwT0;o z12we{(S*?Eo6`=zU?T8v{JcR-vM=@Q@icd^K2gtL2{K%BfD4^Jvg2GI)#xts{@_1S zN?XzN6|-*`qdV0Obk_4LBvsR0aP6X~2M=(f;`<^R3H#Wn?YH;D{!buykhyz@0_RgT9<6 z-T0Nq#FjZp`3Ubs1)r`E@-dH>wZ)SQ=Y{SB3@^*Y=at3}m?~jD!X=&U1-P*>0O4Su zgGBYEy9p38?N&U6pDhtp;{{6{M7I=1H4HnPm$7j1GG=L3f@cYh?LvCfgLb1}FRakV zyHf!sxiTYbpq>*rq|xnrwWv17XE3F&-p0>HwH_}ia8W-cXf>u-C%&z}M!osT#hAkf zT)Q;(kG)r6qupPI_AN&}M9PNi%@mie3Gxry&T5Xghk^gh9bTjS?U6>7S;9b(vkjut zPcXr|B2DO>Tx*)v&`!P@n zC Date: Tue, 25 Feb 2025 10:52:05 +0000 Subject: [PATCH 197/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9763 -> 9761 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 908f2603d..5c0bc0f16 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 10:48:08,115 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 10:48:08,116 submission_checker1.py:3270 INFO] --- -[2025-02-25 10:48:08,116 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 10:51:59,773 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 10:51:59,773 submission_checker1.py:3270 INFO] --- +[2025-02-25 10:51:59,773 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 10:48:08,116 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 10:51:59,773 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 10:48:08,116 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 10:51:59,773 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 10:48:08,116 submission_checker1.py:3286 INFO] --- -[2025-02-25 10:48:08,116 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 10:48:08,116 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 10:48:08,116 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 10:48:08,116 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 10:48:08,116 submission_checker1.py:3307 INFO] --- -[2025-02-25 10:48:08,116 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 10:48:09,076 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 10:51:59,773 submission_checker1.py:3286 INFO] --- +[2025-02-25 10:51:59,773 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 10:51:59,773 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 10:51:59,773 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 10:51:59,773 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 10:51:59,773 submission_checker1.py:3307 INFO] --- +[2025-02-25 10:51:59,773 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 10:52:00,832 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,11 +29,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419809 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.8403 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9538 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2508 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0737 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.29297 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2a83cc214..637bae312 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -824 +825 diff --git a/summary.csv b/summary.csv index 4d46c8602..74b024c92 100644 --- a/summary.csv +++ b/summary.csv @@ -7,11 +7,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlco MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419809,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.8403,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9538,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2508,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0737,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.29297,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 63a82fd00..ca0763d98 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8403, + "Result":20.9538, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2508, - "Accuracy":"acc: 75.400", + "Result":10.0737, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index aaf094625090c0834b4907ba4073bacc694bc8e1..7085442f7e9ec3ffff63e17e904138b3b3a57b5d 100644 GIT binary patch delta 2498 zcmV;z2|f0sOrcD$SqKDe;B!WkT?k|Z&u-`^v&9H20)JQHSI`5wMovq@yU)j-r>Ha9 z;PVbVayrjxOpnLVIp+eTB_F}%%eaS~um8RLbmgD+&fYIqLutMo8b#T7ai@jxv^kyR z|4#FzPX1~xlVLxa4P}xq*knJGY1U5{che{>);?+f-qQ4DmPol=Wi^Y%RP)?FS;r?b z&DSt#bAP>2n0!}+`s&KHe%HhWAKvn*dtHP)(r7|({$Nc}|}$SjB^sruotIRx|d zw>nD7SFJK1+Va0WgJ&K8L+M>AdB#2~sDM3;j| zuAgHrvB~vw!X*quAfX@v2?Y^ID2Q$eh&-HuSQ6d^;S@x-1Vk`+3ZhFvE3A1+}e0tqD%NGOS1;?f%* zuY^PpPA=NhlJKraI60#YKJS37LupJ8#L#spkdTky(yJG?X%NO~V-hSgJi7MJUX#{Q zUIhTJye$Ad&G3w^<;V_yLdO6u+X;bWa(_7n8ix$z1ZWt8NJAk+8cHG1xO76?Xb^`e z0R^!vL3eX-gCcA3K_?!`bIziAAc{U`yZ{=ZlMm!_Yy5A6$d`$*pm8AOJii{3RQB(M z=H_$ZUuLqF3M)W)y&l+#N=x%Bv$bb^D)5u`Bf+1nTNb5~YaBtT1W)K0l|aL&1b-Sv zCD(8&MH!VK0i&`cL08bdgfrISgHAkyQc0tFD2h&!NJD4It>M2#D7U7_f=WZz#1CLu zM~zkH%Jsm+`U$Y+7%i~}OT5=~B>47?Wm7D<#xW?CT*I*h8ipm%Ff6%-V`-FO2@)_Y zOA>T92VbzHEk5XkV@acWAc{Rh1%Dc$3(BSOf52rOHCB19x*pi(F48gWt*86DVB%fT zBf&QtEr){1HI7NaVYT@ zCfLg6V480lO!7GAuQORkd6k$ncbPOS+ue6(iTCf11m6?4=$& zG>l2EaV^SZtBgs|34PdFmY};i_`)P@@j)k^NtvWkJru>5((AifY@Wi5x{0coj5?_P%945a3(95agc^F2{eq!GCvMVz?dv^l51VN{kGc2ELFWtnA1P)E%fYwQ+jeYCSo*#@N_P*Hz?4IMJ3!-5;||bpDl1rbkcLqS>@zl{vdq1MI$>0n z`F8}h*PO8yA9TVWzkf2D4vJ!PCD5?-Yt2rb*P#44k6+)Yhv!Ocn|SgytPo<~Zl$+} zOJLEVE*_w9sEr3`IF}WSJV?X11YR7*Wto=;b;7tTbMpvh=V2{A=!A3096cb4uVJ|c ztYK?*_53?vzEN+_cd>2m$=9)_W$rVa8#*S_;7$>nx_f}erGM@opkZVp4J8w4D49q@ z*Re*Ky9XtpWR@l98qD3pT71+A-4|g|JrKpn1R74JQM0?}H8TJ1#z-yoRe{pbccIiB z%iQxA))0H3;ZRmE_aF^}5{!pVmqwYn2X(@rEHn2Q%-q9Te9(#4 zqEIq(4~k+?R!uPX7&V)FUSl#F7C4F@x~Z(CwkoSvW82h|F91Btq;q6*>kr(gG8xGe zIhmwN80V*J6}f3${nh`KIJjN?f&7a+?=PMWqQcT2OjduhF%dWd1^i$VDqyp26PW^k zlF?Gb0XzpKASH+-ryYV7l%k`()`j?|1D4O9EG|7t;<&1xNqwWfMl>r@=$QS@*syzW zrw3Q6vBA9_-0#5^U$!__0@&a^;}G|ZL#&L$2Ji6~_xOu@{KY-~;vRpo;x9yNXwR=^ z8tGXYZjohJ6c1yI<@u(Rb0NtI$|04W8HfZ;p-~z8OHq^ z!E4F>ct%@H;x2peuXYRv260=T_k77G;aBkX)RW5yz zU(Av6+4pTWnH`}MIF^|Jw79xKwiPIm*eMP*NRN$ywvDmpENTj?t@1diP&23$d5oIW zb?=>O>awHx0kQ+b36u@#X_Y2v266nIE!{O-*Y$Q*X<|6OC3o$U>G|?>dM*)H*{jB< zv-$FJe6gtaYgcadeqfQn$E5y4=y*1HloI(B;1kF5121qLYcNpKx!td2E2#VRvAnR~ zv64UdDUSq>64VE4^_g93Cg)=GAG7WkVF(4w%?~jolT;upUcsdZ+kv*^)IgywExie8 zlZaYBNNOFweHA-#63D5?)y#f7yQo_3RhRsN^U~-V|vlL*qDa$BR5-5?x)++ziS2^JVW08={?(Ge;5jI=+l z8v%l2=sI!$PvY#}dpR5XzR&u_XpQ-E{JDO48&^%G zKAegJYAV4gcff*iO@%Luc({)Fu#P9*NDz-h)Ni~U7n%A%BryP;;+~X`#nW@ULQO%6 zG9^VET9j>0XjxD~z78X=`p-ah9P)4C2{7fp{she*vy~yr0t9W~b4HV6B!30`U=k`| zljkH?0qv7NB{~6LlX@j48_UfPF(d&108jz|01*HH00000000000Dyr≤B80h5#Z MB_jq!BLDyZ0K1x}OaK4? delta 2500 zcmV;#2|MBVjhvQ*cb|_vPf=&I z!RH-#!RzbIt`wOFn|jmvIj}U;lgg>B>LtoxNYKhSGdFG>WqE;!X?WX>&Ts z|DEPbo&42YCc}O-8_FbKu*rTV)2yE^?xs;%tbNk{y`|~RERk}#%4!yisph$VvW`z= zny+Ef=6`xI)7WJ0hx2(OoG%*jZ1$KeW?8($#`S$QF0oo;Xe~nseb3zOVCWnU9l6wwxx^+{>Is)H=zxSm2P70a zAR&Lkzc?k(@o<+yR}$U@;p8v!*>*Uo&j*7ie}D7O9ZDpRq3gE`F7b%u(y5Wi(YPOk z#r;epzkOMyYK(Fo=TBLjgya6bq1l?=2p7U=RUomBQYw&m<7|=Gax~M`O$?%oL3BBY z|*3F&A9o z5r4_0`@aKX9i>zN@z&j$5>3ms#Gcz850mI|63Nwn%q2Fx@!=9iB9KrLfrOICB`&@3 z@k&Sp;pCz{EeY>>gp)Jc;PVdHI+VurKnz`n0txvDF1>nTn+9Q=HYUL`!=r2e>@{f} zpmE4RPJo6nh%^*Jq@ff7jY}uQjRtXu z5>OD!5_C5QHz=|eA9Ui8Jm)N`2cqb6#tWbkI{82@x5ocAhsjvc+*Xx0;sI)ZCGFyArrvg7|KN9@Ox@A!+xyBKcO7Mi9Q3*7RN`IhX zRB{ccQj}2%5-=)D5_ARKOE_aKKIp_VD3vs-hoa~ti8OSU+#3E{gmP<&ET}YeP5c0s zb<|j8u3QgHte*gDj?of(u*7>!M}lwPST@CyYaD}O$u%5HpkY`74a1UaIF?2kmLLJc zvLr!wbMOUA+Tw#wIF>Z32cp&f3ntzb zJraDg(Q+u5T;rG&Os?Tz0u2KbXc(AW!@)Gmzyt{xm?a6in}aW4(iR_8LZ9ETs2+&o zV1lh|4yO60!6c7!{yLL&lvjyKbC*fOvfX`mmU#dENbo&zi|%Oyjnkq`a(@kH5@;Bc zK*N~i8rPysw#t|UozRD^WeK{QgD*_d79VutnUqNy)k9H?Nv;t(-JjkXe=STd?((T{ zv4)ZWncYv9vt=<&3QseB0LeP4tF(M=Z7*Ln+jFcvh8nTQV?&q%9U%t%C$~s&c6cXTlMJ(V$>0ytMA#B6gy5G{eOi@;KQL-9iVZj zR|jY~l@;7NNW-WE8b)QAVFx8(RF+wG1a;J$u@)b6!gocOGY3TRT@kJUyCOBab2ybh ze_P~@x^}L`wr$7Ogr)DhqjdLB2~0WEw*xc|HSPcnr?P@|2Wc3Uz&>MBD$CqEs1rtI znSV!6d(9bZ@j)m2@qa6`>7Xb!R{{-Nzt-&3c@4^+^Z50RdU&qHwuvWS!wMnx?N)kw zxC9m*>f!+!huV06hI3iL$b&SDOW?&}T$XuxP$!JbGB=N4b{^K^gHAY?%+Uj)_!^dL zz#6t@SI@r#<{S0)d>7m1o_rl^TIN2(xuIh+4ek`Nsk;YgTz~5B0UAao(oiyyhLVXi zbRBDyxqDCoN@iJtuEE?rti?y2(0vgW)dNwCOrYUp8a2CnUL*7GZj97YUlk|~eHTjI zvCKV>VGXedN^TUfsksMeTx#wC8V+Rza}UxmD8YE>bZL~Cdr&6~$})40!OT6Z#Rr{u zEea(w_n;^SWlaThk5RL^=QSp?VS%Igp_|HDYOAt(HMUJX`2xUgvG0O3x4z&$mB~n+ z$jKyC!Z<%&tH@33>RbO;;^21m1^E|w-d{W$M1`e4n5@3DF%dWd1xMS#$zii@6PW^k zj$vDBIDqG%1f&F!Ly}*SZk@binfYlf|V+NgP-8GpTRX*NA3C3LUe*85?#F z?)2bFH8!}{gZn+W;>#AtN&p+YXB^_5afp?1*x)_>;vRo-kH5IbU)$Px~B13oyK8>iEZ{&-J9*cplo|MG8!_z zJKSFxsk#>(rNmxJ_UB6y0TjHTC5Po5HhlFl>FMSF$F}2EO-C#Pf2?~iB79xtJ;S(v zBX}*@AJ1ruN!(=*{?(4*z#wmoq`Gr_WLdtM?G17K_%?Ws?*04VT^QdtrXSdU2_0-A zu8D9*>iQztsC6u_yDg47cGr$6@Z1XgoV<%>Cch5SPm9m!23TJI{`7QW2hL4fvC5?{ z@{2iAKKs7SCbJ`S0>?5FfEHIb$hHC{50{RB9Bp% zy6(MGO$={~DoqT>x8$yUGCg0OPR}LcDtp!V zbT(gJjxQGVe(lPw-VZDi_?Xmx2p!KRk5VGP0(|0le&7YJV+{sMI=B0^Yz1|{K9(2u zJ67@sKjo3YQG)tltv<7B&E#B+{sXh^7hwnm0oS2nC6iMiD_&_z5Viwt$*F-tU0Qk* z(k2nLevs5Ue)}qp<0Oz%kE@yec6L#@+^aVE1?Q#F70NS$642ZTskaq+Z&v97B|b1M zSZg#?s0WXh^`9Xfe98EJ^)iY5z&Ygd_vkE z*A4)|F|?pS2aoeC$7gQ9^;;rkJ;06vqE)2 zig`+k9kiG?IidFpN)}&-kyrg^pc;q#n|J~ob6 Date: Tue, 25 Feb 2025 10:56:32 +0000 Subject: [PATCH 198/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9761 -> 9764 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5c0bc0f16..38628d38a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 10:51:59,773 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 10:51:59,773 submission_checker1.py:3270 INFO] --- -[2025-02-25 10:51:59,773 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 10:56:27,355 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 10:56:27,355 submission_checker1.py:3270 INFO] --- +[2025-02-25 10:56:27,355 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 10:51:59,773 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 10:56:27,355 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 10:51:59,773 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 10:56:27,355 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 10:51:59,773 submission_checker1.py:3286 INFO] --- -[2025-02-25 10:51:59,773 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 10:51:59,773 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 10:51:59,773 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 10:51:59,773 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 10:51:59,773 submission_checker1.py:3307 INFO] --- -[2025-02-25 10:51:59,773 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 10:52:00,832 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 10:56:27,355 submission_checker1.py:3286 INFO] --- +[2025-02-25 10:56:27,355 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 10:56:27,355 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 10:56:27,355 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 10:56:27,355 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 10:56:27,355 submission_checker1.py:3307 INFO] --- +[2025-02-25 10:56:27,355 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 10:56:28,338 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -35,7 +35,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0737 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.29297 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.10767 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 637bae312..a05e0eedc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -825 +826 diff --git a/summary.csv b/summary.csv index 74b024c92..d0ed85abc 100644 --- a/summary.csv +++ b/summary.csv @@ -13,7 +13,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0737,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.29297,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.10767,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ca0763d98..858076eed 100644 --- a/summary.json +++ b/summary.json @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.29297, + "Result":8.10767, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 7085442f7e9ec3ffff63e17e904138b3b3a57b5d..1d698dc64a32a3b6844ff19998e2ea6cadc8f384 100644 GIT binary patch delta 2821 zcmV+g3;Oh-Or%V(SqKRzU8(?62><{JlVS)ue~igwl5E{i7Qt4jRdE6H^4~A0F?h}+ zVP|KT)V=c|qs?==`v53;O^*T>mZxtDPgM)S5RG+kBXJcx$jeB4%Vu0JZ{&V;c#lI&LDN`k@V4k$qNt!MWn$0Ab$eEvL(L&C%Bcmvu`Dykx ze;zlJMJ)Z{Drh!s8oF*bXMQ+W*EJl(RqBXFqcD&kqhL9c^K@NFET?{&bw8Pei$v`l z4#VtNo{tjCQCmF{2PeN7LTz>qS8d;h@*yd_R`OKtqUiVh&FQeM>bbPZw4YbqxQvyd z9Qn&>dKEo?i5JKZPo=6a&gXpgq++SkdU)`&Lsvp zyL})bXZHd~$l1LF5^{FG0Z9wrZZ^K%Y*v9qk~!uQo1FU-E-^{IAAyAA`w2)$zMr|o zCAEBcmVl4Mub>5TkDQi-cb|_v4^d~dpvN2V$mu+%F?}9G$D9jLEx81je=q$Wb-w=h z^3zrCw14(~H5*E@>CiY#C(Ao6h-S^%H2XWtraIZHg^Wjoa6Xc8Her*4R3_;lS>DaU zB%k{vgL_NUn`tcNYL?Z^CsWPy-pM9DmPs~;Nt&Cwby#97AgvLg;(O zZU;l>aOlXbcFZLQLgh2-+6gnWG&;beg2>;@gK*z&f3SCKf7le~f3ms#Gc0<50mI| z63N|v%q2E`@Zl0he4O+L z4+Ro(2`;^QVVn6ulr*OPs)k3`{@HWVI?}5E;FVVZ(A9+Q*pB}EPhb*-j!|5d6BfxZ za||>N>Cy?%FdC7D;)pbqN1$=((74qgB2fht$+8N%+k;yue_2TnG~$tZ&RJ9+M9~$; z3!o7?xgeK?{=W@2Uxvh-%b}Eu?0$?BIk*>^n~jQpna)}=tbphBrekY9EzPsc)}EQF zz-#VDg0Hb#7NwGF96_lBYxRsupkY)34Wp84IF+J|N>Bl#vZR8ppt}lZtfU7T@eE2O zjq0N)I$RZd`zL8_u z6icpg42mV!a4dm_VF@%0ORnKq8f9353K*6p6?C@;U$CSlJrC5J`zK@XCJoCj zEFbPNiFfjk1m7LE=Ey`r8j7iW4U3M+2pu0Wz!Xz!} zfkr%&e=2o>HTaqTIR!~@HFEGkgOxSO4HX? zar&y+o@4E?)`&fB6TuAxmPd)?8ponUat$XEXc&<|!-xbLMx<3nB&dK9SyDlFd+>!w zTG9iJuq9X))kjf`NT6X)mDU>q@}F3TttG)qe@t2hCJj^5b!(3~K|rH6&y=9W$v?{ z8#*S_;7$>nx_f}erS2Y}VPql=B@<~VnMgzDu|}D@2US4HEUTbvFn13t>7hpGf4&Hd z>VqgoCeUy)jhfv(uaWt8H%4m7uL_igz6+)9SmvH(SVQcAk{d;AYVH9VmzsNkhC^Aw z+=Da>O3)uVTpDHO9@Gefvdr9LFmn$p>48SP7KM_Tdr%aEvVytCsM*}}8k6ZL$5H&y zLuD~U0q`u7j*-p6e^=dSG9JqlIh`g-5M`%p9l0%CfB*kV9NezID*qzS z+KcAHFgN%6)Abko({Mgkl0Viz8DQr>8KCEt1*5CwR4VDyLbg@;xX7+ho~)`2A4lsS z3s4qu7{y`wv~E5BwSwi;@64j%`&Flw^YD+QeE%+ZXCJ%^><_;v#s{UXN@S@&OS9}& zt>iCIw>pOle=N`acpT0XWhzHmqPmtnf5+>eA6UIkqs8i#r7Cw(nnttdcN6IkWt_h= zRb>=K>GPZXlx%;-q4OuR(Gh(Pe|Ao0FeCv008jz|01*HH0Az1tP;zf@b1!3Wa%C=f zZET&A%WA_g5JmU>3ZuO(+aYNYwgYX+s)0gNTDlX`BoVb9NNSz@edT8o@T$wvy>sTy zpkjTJb@U6?39U+$B`JzPG0lZKl<0l8i62npJXM_5T0x1<;Ly6dE0`fne_MF9+JN-} zTogdc5oSts^xhDR9Xo=gE(s15#J#qX`cT>fHk5tS1K>POS6G6loO+5|U~DEL8jze# zNb_Us4!{|P8YC$1aF%5F;`Lyqn{DWnE2k9xY+%+pij(`~#AIxmCTSL(HRR9m=l12T zTQwG{Jrx5~6`T=ffqHGLM*?3L;czu|ej84_7r-CSL7#aW7U}zdM|JON&l`y@dD@RL0yIssyncqJwqc1~t6Bmn>bPyzq|5dZ)H0000000000 XfPq7kxg|aUmXr7;BL+w#00000ywErG delta 2786 zcmV<83LW*NOrcD$SqKho;B!V$2><{9LI4050F!(q8k2zt8-Iw&WRh&%PZq&esa0_S z^YY&>s4;rZBVlJ}m(;yqX0&ZQPsu$8G z(|%qK<1$eOau_To*;V}TRYs$6mY+7O2npj!y8Ks}Mt}LaG*o38JhoL+RUV>2HZBq? z<7hCD#fesxaFJ&5^taU^u}-iG74|w-=wcJ?X`-duhE;qXPZPP0X|8%*j__lU1)akr zeo&HS59vH8J{fUf82M=li`H*Nb1}B&y5Z`%LCNbV&fa~;JZ#<-b}DV*N?R=6AeC}O%Oj<}F1dbPa>FI`b1hA>Q>_K7J+fE= zcejgUt{}ra;R-=m_y{EA`uPMToJ$OHc6&fV&h7<}kh6OUB;@RV1CkcL+-!Wg*(`xYiaF*Io1FU-E-^{HAAy9_ z`w2)$y`QHa9;PVbVayrjxOpnLVIp+eTB_F}% z%YV3sov;7B{B-4?_RiifS3_yO92!O0cyXtN@w7Red$KUv2oGR@a8X>+}pX>2n0!}+`s&KHe% zHhWAKvn*dtHP)(r7|({$Nc}|}$SjB^sek(6usH&Ym8kbnD zF|?K;guZ9)b})1fhmKro$6R8QEA@m+7<52Fp#u^M9gvW(@GnjYbUfUp(3OODK{)wF zKHCl__4#1%qD#w**8mcnYFRLF6$l5_7>N9+6zS|2rVoQA!06Z{3|K(X?Dk z?78jnFo_-~kzD=9Tw>E3A1+}e0)Gi55lASBT;kFjAFqT&5Kb=I(~|J6M>sj74Lb)JwrLe?rFq zF53x#WO6wM8ix$z1ZWt8NJAk+8cHG1xO76?Xb^`e0R^!vL3eX-gCcA3L4PM6$#c%4 zdLW8EXS@I!p_32fa%=o=gUFYOu%K}uhaCeSc2 zfrf#}H5^Q{3`~%KfmxEEyE*s*CT;OiCG`0Xi|T&G>l2EaV^SZtBgs|34PdFmY};i z_`)P@@j)k^NtvWkJ%1F%nB*Fv)BWkK@z=uS;x3;G7i%a9klFooIa?Ofr0_K32av3z zx=PE}*7ovMvpvV!W2g~(JSKwM1}u*f$u*8eiR2niB+xJ-frb$YG>k~Aj7X4x5m}O; zyE*tmByI6QC+xv1i|U~$MkLU%yGrW~0r^ineXXUyN=#Z?On(}trt8)oOMuv8#1X(| zL!CH4<4`LO&~PRzm~oJXF$pw`$ud6l51VN{kGc2ELF zWtnA1P)E%fYk%=UCwy0gIdec1-xc8+uq#rtJBL&G^S4FbsB7n1Y}} zE%jA_($III)E&#*^BC3;d!Xb-5u2KOfX1cf9-!e+RxtM<4TBPlhfbG9nYjmb!k{cO z_ZZCF!&-dMiPxe~GII}#Vo+8v_ZT&sdtPHQ8x}Z3tfjUpt5;*&)RQj&JjV6VGEbs7iL$3v@5OHvEG9u` z8V}wt2V}LFMSm>h`*-0x``}$@fA~c)J}7M^Fa24XSGVk?c!7r1Ih+S0c^)LAXqGAy zIm`>ywfz1&S^en1@^u!^m#-{Uxr?(bo<6@D%U~dr;+?4~!#K{K-xQ~0^WzPjKLJon z0|b+tBpS2w5nm1j%gqlllT{!ef6Z#cFc5_AeF~$yE!n}P2-|_S$JNY!JG-b_?p2rkg7ebo8s!;5321JF)Y}@px2yC4B|b1MSZ6fU zXaJ9v&0WPU<;KCQGZtKs;FAc_o^o5GT?m$9?D-B9^I3H0C~k~XEX2}nf3ao!n{5FX zgxq5Vfe98EJ^)iY6VVYV_>8nat{VY@W9UGE4j$)Oj?dlzPWhV+qjKR?G7J`OT2JEa z-g`M4`@YZm#b}NBbNsn}c^g+vr9PaB18OS4DR;nvaZQCUi+H$>`LK>B-bfISL)34) z9T%DUKqN5$o#LL9kHyn-B)dXQL5eaZMOu_?PH0(BLcR_oulmnGbsX|<;t4S2zWxNw zAG4Mr$^r>(;B!V$2><{9lW8PZ0nwAeBs>9Ill~+@0qv7OB{~6LlY1p58_UfPF(d&1 o08jz|01*HH00000000000Dyr&le{H90h5#aB_jq!BLDyZ0BkiJWB>pF diff --git a/summary_results.json b/summary_results.json index 2b4615941..cb39734e9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.29297, + "Performance_Result": 8.10767, "Performance_Units": "Samples/s" }, { From 7a6304a8718f5a9ab33d56a78763bf04e269e9a9 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 25 Feb 2025 14:08:11 +0000 Subject: [PATCH 199/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 9764 -> 9765 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 38628d38a..2a0aecf46 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 10:56:27,355 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 10:56:27,355 submission_checker1.py:3270 INFO] --- -[2025-02-25 10:56:27,355 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 14:08:05,861 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 14:08:05,862 submission_checker1.py:3270 INFO] --- +[2025-02-25 14:08:05,862 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 10:56:27,355 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 14:08:05,862 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 10:56:27,355 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 14:08:05,862 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 10:56:27,355 submission_checker1.py:3286 INFO] --- -[2025-02-25 10:56:27,355 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 10:56:27,355 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 10:56:27,355 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 10:56:27,355 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 10:56:27,355 submission_checker1.py:3307 INFO] --- -[2025-02-25 10:56:27,355 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 10:56:28,338 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 14:08:05,862 submission_checker1.py:3286 INFO] --- +[2025-02-25 14:08:05,862 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 14:08:05,862 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 14:08:05,862 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 14:08:05,862 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 14:08:05,862 submission_checker1.py:3307 INFO] --- +[2025-02-25 14:08:05,862 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 14:08:06,885 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,15 +25,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.373682 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95343 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0058 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419809 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1102 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9538 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0737 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.246 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.10767 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a05e0eedc..9ebf09b35 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -826 +827 diff --git a/summary.csv b/summary.csv index d0ed85abc..dd3355ce6 100644 --- a/summary.csv +++ b/summary.csv @@ -3,15 +3,15 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.373682,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95343,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0058,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419809,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1102,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9538,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0737,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.246,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.10767,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 858076eed..54a31c1e0 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0058, + "Result":23.1102, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419809, + "Result":0.419348, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9538, + "Result":20.6061, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0737, - "Accuracy":"acc: 75.800", + "Result":10.246, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 1d698dc64a32a3b6844ff19998e2ea6cadc8f384..0179781ab8a640afc95479547523925885df77db 100644 GIT binary patch delta 1437 zcmV;O1!DT7Or=b)SqKC=@#9jnT?n`aP!q*Z%Uve-SaJj~+)z&r&^XkT12mk;3bq`i zVN3!IW3tSdgDPN5mU(joGv~099%zI!$^1DWiqBlR2FzUR^yvI6P`*{ijv$sD;hFpF zwv%QLpnq0#Sg~)n(%Zu&FzZl1570Q&&;vA_%LVK@=ksXgHZh&F-Gp z$o#t-BempL1xiETg;IAcbI&rYA@)GYjUqNR_W+Gc%{@TFp{!u;K^g`n=now(jWTl& zYJ@>qX6`YVxrdeXKqFp@LdncMD2hQ@!Q5liZ0>oD$#j(CD1PXnvXf-q{E50#^IOFN*O&X)9Uk&(bWrRV(=m)UD3p!XL|X zKOTqkM48G_mZ+{}&)@O-=Lc4=(`d1JWvR+tl%~<_`Q1eNLmB7qOjQ|0QTqHQKPB6r zap?RBP)h>@lbj?Pv+@yo0R%3xaF?@i6Pf~lvJovc9Kdr>0#brVa@rwSK`A=gYh8$c zI$-(y$>P$ZB#x{4nbbGxYecgmg^ta$6wszFYfUdEB->XhW7kwrjeed;TBnjMe#7USe|c6ITw-)l$q6k zS4&+CUDJ52PUEn`#5Q}X?#*^zP_{iB84VfV9qzA;RNae?QerP9`|~A<0195vlEd;2 z8@~FO^z`!oW7~17rX!YtKi0h$5x%bSo?+a-5xkb{k7u;SB<`{Y|7yo@V30RPQr$T| zvMk@s_J%lqd>cGR_x^qGE{yLR(+})_gbp?l*F?A@b$yX+)H;^e-4;h3yKBc3cy0xL zPToZ`lV1nvr^RP<11zt9e|kEx1Lvl#Smn|e`NbS5pMBqEli3kEfn%8oK#QvzWLtp} ziJjt5gY?)KXxkWj&Z4HU+A5EO3N?dDk;kY>UH9IprY<|0A0Rt0oIu%-o>pmTVi3pQ z+0tFZbzN^~l_rMcTXNSvnVv6Cr{@xJmAz_wI-4&q#}|uwzjozT?*|qMd`#*;gpOyE zM=6nC0X}g&Kkx$Au?7Pro!kA|?b@%8<%Ru@mHfd^c_eU@pgvfu&+J+=ITxe<0JHEH zVF(2P#5|@YlU5)ef6Z>fFc5_AeTvBU#7-y(SxJ!Eic^s)m8umt%U-~0{3F|K;O*;# z5J2M8<9cSlon5(H98{D1g7ebo3gsC=321JF)Vm6Ote5E=B|b1MSYtF)=m;Jy>W7kB z%8i3JXDqlN!6y-|AS`w%R}*z-Lo=CkO~QQR7*Scs+De__k`H`@U&2$^98fe98E z?trNsiD*CyJ|gXpYkGj-7#dKZgU5N6 zp5mUB&&l)4C3=bKf)vw~6lpPC=Z}=kDVc4$kvIJppc;n!n|KDCa^HS}`VX^~A<6;- rI`QLDlVl`+1TM00my_otx&;8lJfvE&F~xS^gLpmC@v2WU8x6>K?3 z!CyRDpnR*29YHKR!gKXK z`;(Jp51@atIjj(3-)^P1hf84Ap?)5qaj2mOXgHS@EImlWxCE{o#$}nS2Q|XDEc5jU zX6#`lJi|T_YMkdg3GL4$u zJ+G1ZcQ;0A$*&5OhQ14>?pWrYWmrS(fsz|VY-;WS8kd@TfQCa^!Q6v13`)=+I$RoM z<{s1tgR;!rV=!|ME9rqoycUI$nR`$agR+9T$Eexd^BR-sD92I!&_iV{xmB6H8bwo2 zHUWR|ER&9r&B9mRXEGkk6FHqGN)TnIYaO{QU4Q@oN*vs-zbgMC&)SRT!!S4Z`_uIo z`_phfR+2x~KN(=>KN+Csl?9`#@lbs|Qv+@yo0R;SD5-PKB6Pf~llF?Gb0XzpKASH+-ryYV7l%k`()`j?| z1D4O9EG|7t;<&1xNqwWfMl>r@=$QS@*syzWrw3Q6vBA9_-0#5^U$!__0@&a^;}G|Z zL#&L$2Ji6~_xOu@{KY-~;vRpo;x9yNXwR=^8tGXYZjohJ6c1yI<@u(Rb0NtI$|04W8HfZ;p-~z8OHq^!E4F>ct%@H;x2peuXYRv260=T_k7 z7G;aBkX)RW5yzU(Av6+4pTWnH`}MIF^|Jw79xKwiPIm z*eMP*NRN$ywvDmpENTj?t@1diP&23$d5oIWb?=>O>awHx0kQ+b36u@#X_Y2v266nI zE!{O-*Y$Q*X<|6OC3o$U>G|?>dM*)H*{jB64VE4^_g93Cg)=GAG7cm zVF(3wPG&G9lU5)ef6HpaFc3xe{R*SKE!!b!5w-(u$*O@uQ(C$c(j*bJ9!P4P{C(wT z67Z_a(Y=V{&fw6xx+|C= zOj~%h+JN-}Togdc5oSts^xhDR9Xo=gE(s15#J#qX`cT>fe>RkT(*xikhyfh8iR&?{Jo6_~P|orJHT&lq;tc{%l~@I*OC~| zLdrG( diff --git a/summary_results.json b/summary_results.json index cb39734e9..865a2272b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0737, + "Performance_Result": 10.246, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0058, + "Performance_Result": 23.1102, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419809, + "Performance_Result": 0.419348, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9538, + "Performance_Result": 20.6061, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 4125cb5ebf718bfb2c644272b66f905500456bf8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 25 Feb 2025 14:10:52 +0000 Subject: [PATCH 200/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9765 -> 9766 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 2a0aecf46..9f5b7f1f6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 14:08:05,861 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 14:08:05,862 submission_checker1.py:3270 INFO] --- -[2025-02-25 14:08:05,862 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 14:10:47,490 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 14:10:47,490 submission_checker1.py:3270 INFO] --- +[2025-02-25 14:10:47,490 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 14:08:05,862 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 14:10:47,490 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 14:08:05,862 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 14:10:47,490 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 14:08:05,862 submission_checker1.py:3286 INFO] --- -[2025-02-25 14:08:05,862 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 14:08:05,862 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 14:08:05,862 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 14:08:05,862 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 14:08:05,862 submission_checker1.py:3307 INFO] --- -[2025-02-25 14:08:05,862 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 14:08:06,885 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 14:10:47,490 submission_checker1.py:3286 INFO] --- +[2025-02-25 14:10:47,490 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 14:10:47,490 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 14:10:47,490 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 14:10:47,490 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 14:10:47,490 submission_checker1.py:3307 INFO] --- +[2025-02-25 14:10:47,490 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 14:10:48,454 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.246 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0269 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.10767 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9ebf09b35..9babd2833 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -827 +828 diff --git a/summary.csv b/summary.csv index dd3355ce6..bfc284545 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.246,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0269,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.10767,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 54a31c1e0..87e9d4b54 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.246, - "Accuracy":"acc: 75.000", + "Result":10.0269, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0179781ab8a640afc95479547523925885df77db..6e02cb1a77cd9631402d28e626ecdf4ee507da45 100644 GIT binary patch delta 888 zcmV-;1Bd*jOr}h*Y!d~f_hOM@vvCud0)MvQTWUCf=b!|n1d-&lL$HEUbhOvH5dU<* z^7)g+rAJ8|SM@WgZ`9X_WaECYY6doLnXp2eQWe@((X;FTh0sq#R+ULjE2$5q-D7`4**Dz-&eQY}OYoFaPw@#D zn~8`J&nBSzX0Vp-0Q*p9$fKdi(@5#4c;>janCr! z$~bKB9)EFOX{zXOl-M zkzWBmaXdfp0@twy10|i?{o3u?uaD)0{f?FV!B2T4aFn1vSgX(MS~EEpqyGT2@fTqT z1OUW5rju46Ab)PcFc5_AeTvBU#7-y(SxJ!Eic^s)m8umt%U-~0{3F|K;O*;#5J2M8 z<9cSlon5(H98{D1g7ebo3gsC=321JF)Vm6Ote5E=B|b1MSYtF)=m;Jy>W7kB%8i3J zXDqlN!6y-|AS`w%R}*z-Lo=CkO~QQR7*Scs+DVSmf`H`@U&2$^98fe98E?trNs ziD*CyJ|gXpYkGj-7#dKZgU5N6p5mUB z&&l)4dL)VJf)vw~6lpPC=Z}=kDVc4$kvIJppc;n!n|KDCa^HS}`VX_1Ax#4XF0yc! zV3X=3R}JtN004MwFLQKxY-Mwk(Iga;VkH^^Vj7cw8Woe(BoG?_#5|@Y0RR9{0ssIJ i00000000000001hfkTtNB|ZU{ll>(l21z3T0000Ri><)` diff --git a/summary_results.json b/summary_results.json index 865a2272b..884644e64 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.246, + "Performance_Result": 10.0269, "Performance_Units": "Samples/s" }, { From 115c1d1c24f56ba937cf5150bb8135b195266f1e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 25 Feb 2025 14:15:57 +0000 Subject: [PATCH 201/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9766 -> 9765 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 9f5b7f1f6..ecfe62e1f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 14:10:47,490 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 14:10:47,490 submission_checker1.py:3270 INFO] --- -[2025-02-25 14:10:47,490 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 14:15:52,084 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 14:15:52,084 submission_checker1.py:3270 INFO] --- +[2025-02-25 14:15:52,084 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 14:10:47,490 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 14:15:52,084 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 14:10:47,490 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 14:15:52,084 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 14:10:47,490 submission_checker1.py:3286 INFO] --- -[2025-02-25 14:10:47,490 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 14:10:47,490 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 14:10:47,490 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 14:10:47,490 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 14:10:47,490 submission_checker1.py:3307 INFO] --- -[2025-02-25 14:10:47,490 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 14:10:48,454 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 14:15:52,084 submission_checker1.py:3286 INFO] --- +[2025-02-25 14:15:52,084 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 14:15:52,084 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 14:15:52,084 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 14:15:52,084 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 14:15:52,084 submission_checker1.py:3307 INFO] --- +[2025-02-25 14:15:52,084 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 14:15:53,276 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,4 +39,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.1507 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13056 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 9babd2833..57557509c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -828 +829 diff --git a/summary.csv b/summary.csv index bfc284545..dc06753d6 100644 --- a/summary.csv +++ b/summary.csv @@ -17,4 +17,4 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.1507,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13056,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 87e9d4b54..36b381986 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.1507, + "Result":6.13056, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 6e02cb1a77cd9631402d28e626ecdf4ee507da45..bbf0ad51786b787313dd61ac190cbbb9853ecaba 100644 GIT binary patch delta 2690 zcmV-|3VrpaOr=b)SqKiO)V};v2><{KLI4050F!uTZJxVWuhw zwS;aiL(2Btj4}3jY^Tl>C5r}B0YoI>%fF8V7>m6ZvND-Ou5E)unKI1ICEc|J-kM{V^;9Gv`S2({Tc zT(x~0%7>)zTFFzni=yB2H>bn4s^`)s(|%rc<9{+%hH~UDr|DJn@KuK6Ntz9tm4^h; zG+F(t%))Fe4ON->k8Radm4|ScPV$PCNjMzJe4te&SSD#S`)z$mY${lX3VRbPbg_;0 zG||#+!^%I8r>We;G}o=Jdic>#{mx+=Jt*<2g=FF9pNu##jBHqfy!2aMocFD{DY$wr zPz16(^3jt53T=P=TuUq2$<~6|9$BmccejaSt{~k!;R-=m_y{EA{`mwXWZ*vo2?75L zkPzs;0ttcc6iCQz)&&ys;hX^p8QdO_kkLKo5`&CxA4tgPUH}Oh-Af=Lqx%g=TKIOe z@$F`_3M`V$F_+k6>`%DFB>8>>5|ZyHAR+mF<`S3G^5uV70=^Qzf)>a@)2CV^n29#`rpe>SH08z+56RMD9M&X<1n2p@3bJA zHD}Z8-&wZQ$zCmFJQ{@ak&Lqin;fJvNe9XDZWbo_+9w&@TbkZXV<}gwtY*HLYM%E_ zHu14cvNeB9(%dX&8rzx&!D7(}7RyF7pFhUSd77=J8XMIhh~}ektp1`7W$K61ME!8s z9D(`zTOBE7t5%tHZS}uB{rbM{msqbcw3Z=+zGv=sFmw)wj@)X;Tw;?u^@K|pbU;F( z0}=`ykdUwNFHQ+`Jlv(wm4tUeIQd3CD~6N&dZ2&vOFKhHF>$Cq_gjd8}~;wg>dU^2KjG+Waf!BQBl3MAH%N(B;coC1k0M>AdB#2~sD zM3;j|?w?~WvB~{&!X*quAfX@v2?Y^ID2Q$eh&-HuSQ6d^;S@x-1Vqqz3ZhFvD)A1+}e0tqD%NGOS1 z;?f5nuY^PpPHx)MlJKraI60#QJ>CFYhtilnh@tCHAR!;YrB^R(Ge3xu#?)Wc@aWn< zdrewLdKCb?@(KXDn$R8F(Vzblm_(st6jy)cghevV90QF*x^x0Gj7FrPI3f+@5olaG zHEuPCNK^quvaEvc_Tm;wR?-8Fc%+_l7S#t)bjR@mXoOBakjp~<-v*m6Q)15LP|8Ji zKgNk1+zZXkX2riuXDu04!1H?3u{EET=2>QI&q`I`J@+HQ_t-6qQpq)rpj3jrdPaXG z&@d{2hEd5ioJvtfC8&T=SyDk)&{KsoR?-8Fcm}1CM)gq?oi34v&X1R91V*KjP2 zGAuy_49k)Vy4#B{SkjUnXoO=)qxyd!itVNXjnD<<()d5%vW^_9?67V+wtb#-OuMkp zzY8WlA3YNM*wJz*m|WwS6ilw+U;+&T6KEKiT*JXM%fJK`FfdCh=x#5*fJsYwC=qxH>ljbgyhGiGF4|kcwC;3N$pN?DfbQ^zYoEBx0 zYdDiY!nWRyD6vde28llsZ>O%i( zVRCtwO@+%1lmy81ezuw|^J!9en(+fj){$MM&v5EUBQoz4$^TE$M+q*cL2{>Z2$|B+#&@O6v^) z`A_V_){$^jbAWCdFe(l920hA~;@%s~|} zCd<4zf|+wzNe?u_nPmPP5XD!nTmx3Fb$WFE6)4}TV@D9nj_}NVc3Xe3mRWaD1&qov^Nyh2nlo0?1C8)g5$4eWQT$YdYrv^U z%|0DY<zVv{A%umx<*rk&TQ{5kutZ`96nC5nEY)go5x zJFfKha0%Qx)X)Po4t4YZ4d=3grw3^mm%x_8xGb~vphg&%WyT)CoIR|h2O8mAGII}z z;)__W0gKq04L<)4m~YhP^Ia5uKG{Onw9I|pb3?~u8r&*kQ+t08(74pz12l|Gq@iRY z4J8w4=t9;gv-hA1D4AsybPZ+(D9hYE26OkY zk{)QpYf&hfy9a+oF(@mzdyJahJ+CpDj&dBu4?R`Zl3SJ4t5J0IWD5XWFYMH;acma; z)_o@9u{@E}X`%#CHeBn-ZRz?40ItNr?fP%!U*uVP(R>)@?tXu|{>%O}oR5{{kM(Z` z*!gb;=y_$q=xRBYO8T^rZB;%lvQH>aR#k?NqxG)^D2snMjN&kTTDPA6T)}eccV^M> z{i;*TdHBatzJC|Ivk%?{_J>~-WmAfcSquKMjiS&mu&fl4;GK!+~`At3~+uw2M{0UG?0|XQR z000O8006V#5poU%D+OUmB$HPl9)HVf!!Qs<_x%c^y)E0ROA)pMZON*ELQ`6@6VfCR zwH`=no&0^}XA;P&%hA1a=FXsUd6Z4`3)Ts(DwHKDia;^Vh1yr>W3!4MQRF;ToHklP zg-+nmvc4;sAxv9%v)X|50$db8$`NKNbnxB~j2%0Gq%H{#6~vvklKN2EJ%2WoebYVQ zJWU_41W!5j6nDVb%tSOIIh&E@$2L8HGYkz#P~PDz$?)0h!Af_tp;s=PQuvdBo7PF3 z-A5;8W81b#yXdVUe}=!-ukZb;u~6Nq7@)f3j4%t-Yg?E2vIvK3sPpS^;++8gbPD>7 zw{el74|o&;kRk4*d@i0}HY6)l=V@NVX&&drCc7tDL5h6ajlAkV1LZj6-^2+p<{JLI4050F!+r8h_=S?Q^2Y7RJBdzk;IX!%WqR zG{{>jx|L+!VvNaTl5E{i7Qt4jRdE6H^4~A0F?h}+VP|KT)V=c|qs?==`v53;O^ z*T>mZxtDPgM)S5RG+kBXJcx$jeB4%Vu0JZ{&V;c#lI&L zDN`k@V1J&p)k&Hz4w}s*n8=x*XwgE>vm>J@p809^HXb*VMJ)Z{Drh!s8oF*bXMQ+W z*EJl(RqBXFqcD&kqhL9c^K@NFET?{&bw8Pei$v`l4#VtNo{tjCQCmF{2PeN7LTz>q zS8d;h@*yd_R`OKtqUiVh&FQeM>bbPZw4YbqxPOe5p&a?kX?hhse3jvNl4hsP%0q%^ znymgRvoJfChN{f`$F^#!%0oCzCwaxnBpeQ9exg++SSD#S`)z$lY${lX3VRbPbg_;0 zG||#+!^*G6(^PI^n(J0qJ^bjWe&;Zb9+Y_1LbCAlD(wS0ehmVl4Mub>5TkDQi-cb|_v z4^d~dpvN2V$mu+%F?}9G$D9jLEx81jFZ~{MzW(>}(^c=ZfA)Sg8%nb2&^Syd%R4QI zX3g0&`#Z~~I@zm*j7NiTK9X@ZVUvSYCg~tq-p#@!pZg?(drQ-sX)NVxmetHBQ_b_< z$tFIQNj86nNt&Ch%q2FtQ%|^rK?fuh zIv}CY0SWmC|KgND$HQF;T}gNsgp*I?vtl^OuLplRPd@X{9ZDpRq4T#3F7b%u(y5Wi z@nql!`SVO8dwf|})fi_yE}qgT4km+pL$fu#5iEt#sz72LsZ=2G#wn24ax~M`O$?%o zL3BBY|*3 zF&BSa;t|QE`@aKX9jR0R@z&iL5>3ms#Gc0<50mI|63N|v%q2E`@Zl0hB9KrLfrOIC zB`$sN@k&Sp;pC=0EeY>>gp)H`(BloTc_@wPgBUsw1rl-zF1>nToB2VMG^YNlhDX=_ z*>lo5(yIXAl~(}J)r9WYj{f{lU=oFnQCxqO6BfxZa||>N>Cy?%FdC7D;)pbqN1$=( z(74qgB2fht$+8N%+k;yuSxFBx;*omJSyUfH(G|xFpbBl#vZR8ppt}lZtfU7T@eE2Ojq0N)I$RU|^P1(A^$<0h5;WP$qOG zheh>46bBP*YI88nHw`9P*!k;B){$N%Ce2+Y4a+VpAMP@Vck+(}-yOH;?lynWI4#N~ z*Kj6*hA{~=j7hF>Ey`r8j7iW4U3M+2pu0Wz!Xz!}fkr%&GD)NQD2g%3HA1I5)rJ1o z!sPNU8w!^jC<&11{cJT_=EJ1$G~)-5tRuTh)7MsU`l{KUW9_llh&^r-!3_kKM~UPb z$D%}X4JQ(47?D83hy)r&q*Z@LB&dK9SyDlFd+>!wTG9iJuq9X))kjf`NT6X)mDU>q z@}F3TttG)qOj-pd4O0`vPs?2<_gHcSFx*g24$wH%lmj%J$qKd{q+v_~4P&y*nS&}| zOqO|b1T*Kbk{)P;Gs*lpAd1gixdzN!>-6aSD^R{w$BrPD9pRb#?6!Ym$FrlqPzfA4 z)U*RM4t4DS4X3h#ZwF}@l|aL&EVJ&Q3K*4T<{d%3HD|1(2O8nKBFv)$qWG=|*MMD- znteK)%Adb2@<#nT*P>|Pu{B}o`<^J>JyZgF4t4MVjYBOwK*Oo5VB$d(!{)Cwdv;!f^5-nSzEL;Nl_=VIR&!Xf zZ@1Fh!zD25P(KgQIMmPsG@Q!{mL8;GTmn}Pg~AP2GPzK;u$(5700&k%p3q zG?YxFq4QXy%-w@3pk$U+&^4I5hn4hDBXnPcMfE`xBNJ#inMTd-p4Z6yyBj05%sqc7ia}Yy++);i?s<*Lbd=*Le(0gHmfWh$UX7xuCz}A+dSRhvjbpR$ zRri^U$MQr@r->3o+38wGZcEqS|GyFkx9hLUzsR%pqWLh)&Hes#{l)$?oR5{{kM&Om z*!fQe=y_$q=xRBYO8T^rZB;%lvMZD)t183C(fY>%ltq6WMsb)vty|B3tzbFzJF{r` ze$}bvJp5xR-@gmq*$3|e`@=7a@j+=TS?bTyEW1@J`3uyo&f&rz%X2>-hx0_4%2Af6 zu4T{P@%rZnR(X;FTh0sq#R+ULjE2$5q-G5_4**Dz- z&eQY}OYoFaPw@#Dn~8`J&nBSzX0Vp401C5eBsc*DTRoBcC6lKmQ~{Wi{3Rm>N+SRO004&a4nP0^ diff --git a/summary_results.json b/summary_results.json index 884644e64..4b67e588e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.1507, + "Performance_Result": 6.13056, "Performance_Units": "Samples/s" }, { From 8a8f404ef23bc6d341f0f2b615e73365b52d6f8b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 25 Feb 2025 14:58:15 +0000 Subject: [PATCH 202/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9765 -> 9766 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ecfe62e1f..d63016aef 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 14:15:52,084 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 14:15:52,084 submission_checker1.py:3270 INFO] --- -[2025-02-25 14:15:52,084 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 14:58:10,409 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 14:58:10,409 submission_checker1.py:3270 INFO] --- +[2025-02-25 14:58:10,409 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 14:15:52,084 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 14:58:10,409 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 14:15:52,084 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 14:58:10,409 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 14:15:52,084 submission_checker1.py:3286 INFO] --- -[2025-02-25 14:15:52,084 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 14:15:52,084 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 14:15:52,084 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 14:15:52,084 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 14:15:52,084 submission_checker1.py:3307 INFO] --- -[2025-02-25 14:15:52,084 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 14:15:53,276 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 14:58:10,409 submission_checker1.py:3286 INFO] --- +[2025-02-25 14:58:10,409 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 14:58:10,409 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 14:58:10,409 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 14:58:10,409 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 14:58:10,409 submission_checker1.py:3307 INFO] --- +[2025-02-25 14:58:10,409 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 14:58:11,416 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0269 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2436 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.10767 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 57557509c..e3240710a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -829 +830 diff --git a/summary.csv b/summary.csv index dc06753d6..7909d06cb 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0269,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2436,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.10767,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 36b381986..361ecd21a 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0269, - "Accuracy":"acc: 74.800", + "Result":10.2436, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index bbf0ad51786b787313dd61ac190cbbb9853ecaba..3c654dee16bce4eca28c4f51873460bc27c3608a 100644 GIT binary patch delta 875 zcmV-x1C;!wOr}h*Y!d~#g^*QZvvCud0)LJn-w!1mz;jRnQi4cw+96m$DLUF~U5I}= zVEO#X;?koej;s2a)HmvDM6)7=j@jRg4Z8<-dT^y08{F%`{T^KLWs74afDPU=4sp*o z#L75q@E(70kH5IbU)&30c+i|O=BbI?b*1Z=IzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8 zEZ@xbhB$tF8$3t%{(bN+jPD!M4}a{04mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$ z-bFK$UkB-@#bn)CTY(aZ zo#Ie~^w=0^+ZcP!qNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rLZu5Xaxy z(p|%KU2kWVCWhl%a@RhYo-a?Q=Mr(1y=r_qn=dcN7mIqocI8&@2Nnr@OzJ;`j%Slc zDUn|RK5;xh@B-Jd1_LFX+x=R$g1TQH%M1G*EBS+;@<`w)L4B}RpV_r$axO;y0kiQJ zVF&~g{s)tjRv;`5X-W{b18vEwfkItcx|5F}7k^@Xluh&t)(NdjlqD&OKrzjQ+L!3P z-oz^uIZqX*jaE>i6F9W4?h0lI(-vN>HekH~7X^@Vgqac@yf*}6#||K=OM*iMai^`M zK9qKk4Q1bS4>(WL2Q0x;PCdmPFg7z0jY!UBr1`N;58w<#0}_;XI7>2o_Ij|=-E8QU z3xB5+{$${$brNUy(TUmEwr$cbdTYp^;m_^MTfb^7RCg){s46%k%mVe=Rt3H+!r>a~ z{5G6;CxAbmf_~#|Tx9429)$p8h&w4Ci>K#$gQ`5umvNfM`LfRL$#O-q^s5_r)qe)c zamc@k6JW@F{Ryf+P)h>@6aWAK2mk;8KeMDEKm!H3g^*QZlj$T^0rHbUB{~9R8k2t- z6_eB?5E~Ny2a_ZL002+|000pH0000000000004l2MU%ZHJ^`4M{3Rm>N+SRO0086m BjaECYY6doLnXp2eQWe@(@ z6aWAK2mk;8KL8<}UL8jQ0Fy2y39}FwLkI*b1z|~(S0F46sY?;I18vEwfkIPSvXhV? z7k_ejluh&t)(NdDlqD&OKrzjQ+E?gfvx*;4Wb|AFu>ZIrS8Gz}U=0G$J{h zk>+N~)=8Y*M<-@u+qOx&=&d1thQHRY z@BON=P~E8*pt|IYFbmXcTbKB<2#0H^^XqWpodEuH3i^$=agm`9coYJVA?~DnE}maD zD^%xcUc_l0=fx(wCs{#?eA|t@>OTYJION~N2{7co{si?OP)h>@6aWAK2mk;8KeMGF zKm!G&_hOM@ljt<8 diff --git a/summary_results.json b/summary_results.json index 4b67e588e..b5926b115 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0269, + "Performance_Result": 10.2436, "Performance_Units": "Samples/s" }, { From 2066c3d20be58b3d8682a9fec50851035e3b7aeb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 25 Feb 2025 15:10:45 +0000 Subject: [PATCH 203/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9766 -> 9761 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index d63016aef..399e5335a 100644 --- a/README.md +++ b/README.md @@ -1,28 +1,28 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 14:58:10,409 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 14:58:10,409 submission_checker1.py:3270 INFO] --- -[2025-02-25 14:58:10,409 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-25 15:10:40,511 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-25 15:10:40,511 submission_checker1.py:3270 INFO] --- +[2025-02-25 15:10:40,511 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 14:58:10,409 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-25 15:10:40,511 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 14:58:10,409 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-25 15:10:40,511 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 14:58:10,409 submission_checker1.py:3286 INFO] --- -[2025-02-25 14:58:10,409 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 14:58:10,409 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 14:58:10,409 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 14:58:10,409 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 14:58:10,409 submission_checker1.py:3307 INFO] --- -[2025-02-25 14:58:10,409 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 14:58:11,416 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-25 15:10:40,511 submission_checker1.py:3286 INFO] --- +[2025-02-25 15:10:40,511 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-25 15:10:40,511 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-25 15:10:40,511 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-25 15:10:40,511 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-25 15:10:40,511 submission_checker1.py:3307 INFO] --- +[2025-02-25 15:10:40,511 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-25 15:10:41,525 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.373682 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95343 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1102 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,9 +33,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2436 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0988 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.10767 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e3240710a..b9d5edd3c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -830 +831 diff --git a/summary.csv b/summary.csv index 7909d06cb..bfc036a24 100644 --- a/summary.csv +++ b/summary.csv @@ -1,6 +1,6 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.373682,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95343,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1102,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,9 +11,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2436,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0988,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.10767,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 361ecd21a..54b498f54 100644 --- a/summary.json +++ b/summary.json @@ -39,7 +39,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.373682, + "Result":0.298773, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2436, - "Accuracy":"acc: 75.200", + "Result":10.0988, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.10767, + "Result":7.83029, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 3c654dee16bce4eca28c4f51873460bc27c3608a..af4403bde18cbaf64ca2533ab92b91ecac4aa046 100644 GIT binary patch delta 3411 zcmX|EcTf|`*9{O#P4f)~G~p-TE;aX|+uEF#ExG)# z@1nh{Yx+pNt;@UMz7_S3ey^-cB1e13=aM_}#?AMpZa*z5i+#a7DEA`iJEeDfDY8t` zfYKjzgvjcy-GlIKkM&{LMe}JpPU6n5oj{8i1DotuD|5NyC|t~o-57r91yvRK*tyIc zmk0Ac3Ra(oL@RJE!$o{3oVQZ9b&9WZ$H{U>En8^2-i z?TK8cfCX1-gQLh5>egq$)*!I&4MDGaEuMLKnF+2cFb0k;JI8^bcJEfca()4cWXq3u zcQIlRR7rDg&Cm<`dD1YjOVNiUVI-zcLTk0{9-6mqm&HD};Cqj8j?Ff-sT<;tQ_?R) z4{qw0RvJ~a%8xelY zsL$eWi;d6>VudWouhN5dz5Bu=UQCE2Cs&8`L79|4hKgNq-J#d-7_SF%`vx}B8Jw~8 zy3#P=k=7*MDunTixzTqBO3!#Tm87XnIRrvpP4%Pa@NnCg&>5)>sbwtsVli}fa>=pZ zY(&6wd4pZm_iDczDyMxwd5QiNRFwRqyY6~+<#GLi|ES7(PQ7h;B;G?A!9Lu%X?!N% z%I`|=QMR*D_S&d+TY(l*v=eDl)u|;(@K1(QUF`9dm{2^qzHp@%k>*>|b$>CgbTOFf z#3R}bv1x(y_Bs?1dsSKZqySMm!$9H0cD}9zNNw^-Byh7$k-3ifu6GK z!~8Wd+6~Q(Na(Pd{qT# zT45T)9bUFyO2PB-?X70uRAXfIH--7ns2>9`I@YsljN zl{^m8JOvb+`k-Q#??y`d>i2!#o?FdWoV1rj`dp->j&9ZmavPq2ag#n5DyiQE#fZH? zzdKRvPLj8;p5UWT47KcqmqR&q#_}|^)p16ZcfY!XaY7SH8@qljf71xT+_1hL%m?_i z$qKld?>;|Bw!$eMz*#+5X_claBNlxndvK5Aly8^VisaK3#P2o3(X$ zo5m`o8C);$HOtUw3tnC0iV2ncWxSsd^23cm^gU`m-{MNkmQmgdU%}`$5O9rp^RPARN-?r7CPn)$41(N5NC}lW-Ya8g;S zRIs`poSK2a1O)}2?UwNX|CEo(nHB{dp(gVC-(lO_Zc30M6+no1gIs%*Z-VEH=<)1%E`YBni1YFP{?pqsSQ#0PNqQFHGvD0ZEQ9m!wvDj7T z84Kn5l<+a%LlXFUkoqkdKyaH=rYJMi4wa_t3#?wKN)uU*Q0PP}i^W!=Nf@R#=vDv4 z11Zx6^PEI6J>l8ReN?SOduc(VDd;hBz>(fmpyR2kId3Y;G2ksDg$z!P^a1rC0s`+ht={K3>)K_{E1K#op zS?g+*DZ4>A@guiv`Cx3J=L{!_J!;*SI%`^`NazPxYZGYl_iYJAb@;-Ts``BJGKDh-ZtialD9t_-ZmaZ|kst4;@ znA8HQwwsN6!R;=yvJ06iE17CuL}B1R_JhbO`!jd!Ff`rE>M&2J)S*?D`Fjc!gN+IL@E8*s(xFJNATH|wl3~c@&yHv6OW|+f1y-@75 zzsi=aL1${j03^JGfU7sT9?>x#*l|nRRr~N{(qr(0w_4~_nW19PXh^Xzj_+B1LH(Df zv|@&|{v5cBn&DjC>l}m7f(AW=)UD-=!``n|A>PHz7O#oqdXWh#na{Qsd7J+0jM_Zs zPQ1>W_!cJW#Z{hBzmC&HY|~r3-k)3hb5QY{|>v824kh=$YSzPR&fK zP)yB8>wsdy7zM#Dh4QE_rc8P4Fgky4Nf}Jun4CT8(jTo~hC%<_daGfJXUUymlvajQ zt7glR3`}H?HEM<5-@oc0Ky#c7;hJ5Qqzbeh3X`3H-~OKL=TDtxrZ&?WE*~}-W0viBd~QUj+!{OAsaNh8Jn88(Li`wr!Yvl`ahE4vKTLGE3V$%^nFY$*>Sh<+77czX=;CUq_h0+ zO7Ju8UA`Hf_ALfkZ((;i(;?DirL$DY{xWLHj2On8-cC?7p3O$6aE{;kW3i z19$-d0Lw{e04`+T6~0an01#xMEY+OnbyyhTpKTVQ1WPdyOz-^`InD@bC~Lt->F!b^ zu384q&Qh$lnYAy^P#afGSn|l#;C)HsImnDzQd#C;1^)0$l)L))Q$GnMgigC2dQ`O?05Q_G(^2MTr5wd`;aP^>UBXp%XtUW> zagho~b@+UY68Gm}bqddeV%@}0G5q;(k@=UY{o`BR(EHddaoH|}>`*>B1&@C*McB4q zxN*!_Vyts?LzN+s^gu&vi@7@ELKi*g|NkuncZdbu0HFmUF8Fu2fB=9$Hvn)-_A-n_!D46~(m=F%;BaA{gh5iab2LPC!$iwyD*-gkEIQy?A=^Kju+jb^rhX delta 3408 zcmV-W4X^T{Or}h*SqOiq)V};v2><{KLI4050001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`OuoLh6^#ukO&&#zFa=3%BP2(^T6E9~ycxUUlO# zR)%uqFQ@5M^zc=N<4Kwgo0W$I(KK28tIWb|EDcqe`HyYYRF#Ktm`?JFl}R`p%6yUz6ltAt*(0b(NF!(VH`au@v4Pn;pd->I53QCSc1ItTV9;^t+^?< zdM;41Jo3@I@0f?p`@G_f;yVqwrUf^&;I>fE%=)wKlz}^Cus-O!eYoO^R*Q5KNUUAS zHIUf5k{cj#b|tq!;%+57>6$oQlT{!{x?Dl}`TbSJ^{a}TR~5H^Ws03lTR77ei#JH7 zT+#AKs;^6~Uzgl)3H@A4E7{4`g4rHftO9qpiDRxH-8|t6L0I?*B;@}21SDkOKLZH? z{|k^1=)M98f$kJY$ZggI67u1k0SOu09*~gHJ?9dGjBX!D$mm`G2^rl>AR(js4M;t{t)?0q)gXxGqj0SLq7G&1htov; zaM&Dy`TAQODP^lxnRRXTzdil>zV4S;uQ9ZiA%wnX?shPA4u_81YR6n+lRNc28+@;W!gm*zW`9?k~hLikyp!4KA|JrfyeAHk(pFKjbEh?2(CU)AvF z+CO`LOh>+pZ^n>M4@98SLK97GR+(VjYGP00yK<9q@g$>4doGN zTsk#wHHb)50Y$Q`g6{U>7D`sq1C4m3o^uw}2T^p#@d9XsPCk&!LjT_en=eyh&gD?b zMRq^Ni5%Pu&CO=Tzf5N>8CJmadegBrpO)r-S!Qd`N>$)J_anjg*e#1v$u*9kRD!*F zMkUZNDuISk$u*oxQAQ=GfKgdeL08aIg)>&t1C4kFrIJSVQ52mnk%rEfh3>yaC<~Ki zPNku1;s>y-BgZPU=B8s}vlpy6Mr+?>iBFx51V6~JY>Fk|Mzh9%d3 za4d~7EI|bf%aRJZ+lwz)(vlu%gkwpg`XGw!rUH%71?AHCKj5;C9INcGZaTJoo^(vR zu+P5>CO#iM68zZFawwQw2AE|YsKIRY4Ns3!+#9BRq|8qQ<|TMp7NCV_@AS?0_^ z6)+~tyg7oIb680aG{Tu={u~g0#aFIe16HnedUXC3DBr4MM-a=7@XUR7Te0KW(O;+p zjvQ**0UC$8c7TRcS;4o1G>l51VN{k`cTfe4$};ngpx&A@R?-8F@KX`y(E(BXRD^55 zsYuN}9Zu!XKNfkT{+(-4H1OD(F!j9)Sa%PWz@I}cJV4`64-e3AEGxKwc#wu+37j-G z#j?!EgBoF2mRWfOb=sV$`*&T}P-exB7L zR_r^j^!9KG+&a|I12hhG^Z*U#vVx}vX&9HlmczI#v-O}x7?)+n9>JVFtfU7T;aoCv z4~XK6Sgrw!*qRMK{|=acZ`9`VT@-yj*+SN|%zfT-L&szq+$v&Idk@gK)ZPO$j7+4V zWFieE6KUu|)+n?0pb99NWfgP{X76DoJ=6$27-3O;5XHy@8cwEBv%TjvGXL($NG&R{C`Ue26#KGC-E?+>hAr_o~d%2JiPC{3f;^Sg=khceFJnW{1kilX%SO+F;s-*M>t3A4}< zm;nX4g^*QZvuzWZ0)LJn-w!1mz;jRnQi4cw+96m$DLUF~U5I}=VEO#X;?koej;s2a z)HmvDM6)7=j@jRg4Z8<-dT^y08{F%`{T^KLWs74afDPU=4sp*o#L75q@E(70kH5Ib zU)&30c+i|O= zBbI?b*1Z=IzOM3~VcfqFyq4^bXSBs6?y?8}YR7P3kT*tB-8nw8EZ@xbhB$tF8$3t% z{(bN+jPD!M4}a{04mJ_jM7Se$eUWU`I+oYn7DpYsYsVCLZUuf$-bFK$UkB-@#bn)CTY(aZo#Ie~^w=0^+ZcP! zqNcFgDvyH-HG@i#$EZnN_ui?dE<2hZAUiOeK-rLZ=1QY-O00;m806zdB zon9SB0i6F9W4?h0lI(|;CTtu|o202c+2a)g-@9lSRL zW5*64sY`-G1#zdXq&}2(j}2wtbPqUB(+4cUQ%*g_9WXXC5sgUBW~BMCO%LD<(%o$6l?$g7{$${$brNUy(TUmEwr$cbdTYp^;m_^MTfb^7RCg){s46%k z%mVe=R!0TCEW+U$>ijmGcqf2Ao`QbkZCqsN10IC{WQaQ{AB(5wdV{Jw&6jbS$N93( z?#XgRvh=GPdDVXg%5liQi4$PRef<{KlV&7W0o0SeBqAHS zg^*QZ1ONbZ5&!@c00000000000001hfnJmQBtZf4lRYIm0c4YSB_ Date: Wed, 26 Feb 2025 13:12:23 +0000 Subject: [PATCH 204/719] Updated results summary [skip ci] --- README.md | 62 ++++---- dbversion | 2 +- summary.csv | 20 +-- summary.json | 338 +++++++++++++++++++++---------------------- summary.xlsx | Bin 9761 -> 9760 bytes summary_results.json | 14 +- 6 files changed, 218 insertions(+), 218 deletions(-) diff --git a/README.md b/README.md index 399e5335a..617dc08f4 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,42 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-25 15:10:40,511 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-25 15:10:40,511 submission_checker1.py:3270 INFO] --- -[2025-02-25 15:10:40,511 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-26 13:12:18,368 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-26 13:12:18,368 submission_checker1.py:3270 INFO] --- +[2025-02-26 13:12:18,368 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-25 15:10:40,511 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-26 13:12:18,368 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-25 15:10:40,511 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-26 13:12:18,368 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-25 15:10:40,511 submission_checker1.py:3286 INFO] --- -[2025-02-25 15:10:40,511 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-25 15:10:40,511 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-25 15:10:40,511 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-25 15:10:40,511 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-25 15:10:40,511 submission_checker1.py:3307 INFO] --- -[2025-02-25 15:10:40,511 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-25 15:10:41,525 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-26 13:12:18,368 submission_checker1.py:3286 INFO] --- +[2025-02-26 13:12:18,368 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-26 13:12:18,368 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-26 13:12:18,368 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-26 13:12:18,368 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-26 13:12:18,368 submission_checker1.py:3307 INFO] --- +[2025-02-26 13:12:18,368 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-26 13:12:19,552 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95343 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1102 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.435308 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13056 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407536 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0988 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13056 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95343 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0988 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436808 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1028 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index b9d5edd3c..8e6742924 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -831 +832 diff --git a/summary.csv b/summary.csv index bfc036a24..6ed6c1b6e 100644 --- a/summary.csv +++ b/summary.csv @@ -1,20 +1,20 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95343,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1102,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.435308,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407536,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13056,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95343,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0988,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436808,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405291,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1028,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13056,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 54b498f54..cd90a8771 100644 --- a/summary.json +++ b/summary.json @@ -1,5 +1,35 @@ { "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.325687, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -29,7 +59,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "1":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -59,7 +89,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,55 +119,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", - "Scenario":"Offline", - "Result":5.95343, - "Accuracy":"F1: 80.0", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.1102, - "Accuracy":"acc: 76.000", + "Result":0.421108, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -155,20 +155,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.419348, - "Accuracy":"mAP: 76.951", + "Result":6.13056, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -183,60 +183,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":23.1113, - "Accuracy":"acc: 76.000", + "Result":0.385187, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":16, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.435308, - "Accuracy":"mAP: 76.951", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":16, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "8":{ @@ -274,22 +274,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.407536, - "Accuracy":"mAP: 76.951", + "Result":5.95343, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -304,81 +304,81 @@ "Availability":"available", "Division":"open", "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":0.385187, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":10.0988, + "Accuracy":"acc: 75.800", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":23.1113, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", + "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":10.0988, - "Accuracy":"acc: 75.800", + "Result":0.436808, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -394,21 +394,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.325687, - "Accuracy":"mAP: 76.951", + "Result":7.83029, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -424,22 +424,52 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.405291, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "15":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.83029, + "Result":23.1028, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -449,25 +479,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421108, - "Accuracy":"mAP: 49.593", + "Result":0.419348, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -479,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -538,35 +568,5 @@ "has_power":false, "Units":"Samples\/s", "weight_data_types":"fp32" - }, - "18":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", - "Scenario":"Offline", - "Result":6.13056, - "Accuracy":"F1: 80.0", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" } } \ No newline at end of file diff --git a/summary.xlsx b/summary.xlsx index af4403bde18cbaf64ca2533ab92b91ecac4aa046..8a60fdb5bce92e4232dcfe4c8f8d113915d6aa42 100644 GIT binary patch delta 1104 zcmV-W1h4y{OrT7#SqKHJC?Oh7vt9^)27fTd*px`Cj7ZQ4BeEnxcYE-KNZQf^ov@8p z7S%^lj7Xqica_!~0`i~OeXXUyN=#Y>CJj@|zAX1x0>mC8jsP|r>cjyWhgxxfhBH~g zjDs|cNnpoeOqTg^P$!JZGDnVJmK@g71D$XtnJWiG@s%srfR$^V&YXV*%D3v%5r4#} zBRq4T$yV$*b@Uf1fe(jTb%4g9ULBy}R90~7APu7uXxQ4d%&>zJFe=L|JAyiD&R9zi zbi$vCFlP>k;!j1m20Rt1*`33w{P~YX-l%KmS`=+NwkAw{?*i7{gC%g~P~#5JIMlfV zG#twc-W{Z2SON{hvdq4N5-=>w41YX=`fJWuOAmCy_g|S$2Su^T5@^^0wq~o&YgGQ6 z{ns~Y;<*w<7f-&36=L7^tG9-r+rM@aq8u~7jx@8n@V0NJtd!Xb_5u3VufX1cn9-!e+R&e(q4TBPlhwk|p zW$qr-34^lC-D5C!4{PaxPLtOVCx3OD;@B+wh5Jm#V|gN{(?kiP>~yUox2@|x`@a$g zx9h)x8 ztf~wjN9*4RP!@3*#bNri?mhpJg5}ij%%b7@)u5L1@Q$I)@8?EYJOT9L^JEDo0tNx|ZF5$Ln7mSiMf8#p;!%DtA$uMziO4 z6X_3SoWC^Cz>=5q%B>EjzjSlTjcnf2N@owgYX+sewXMT5=Q8 zCK0uMkkmSP`^rw@B;ZqztC{_Fc2PCot2X%s=cUm#B3X_S(A)^Aw>A1$Ez$>+_`tMa zt|i(NrLY7;0uwor3X4;8?P*? zkD?fnK*R1Ttv3YZKe78-OM#V`vyK?N@IPm%yV#Z9G8ZP#+J_a4su2d60&23CuW*%Q7<$>V$DwX6F&i z&%;`JpcBp|v-E%{zKG=-u!ybM*7NUx`9{q>-$l{glPzRT%iQNUH*`#VqgoCeUy)jhgK} zuaWt8pN!N}Ulk|~eHTjIvCKWoSVQcAk~>9g>h1v=m%4j^hC^Aw-Gej?N-!QeT^eQX z9@GhgvdrCMFn14Y>48qX7L(f$C4cq8PR$y}X5lZ~XEGkk6FHqGN)TnIYaO|5UH{qt zl{mOv|AqXEJnJu-55wHk?@!l%&!2|#v6B3;{-ppr|D^yuPZo@>mQ$&uPYc;r<>Mmz zgz{olW%xK+|3-kah{Grj)2DUs`HvJVr+#M^4d1T@wVa24Eam%m!8`llT~lCx_(d^3 zC~YMx{aKn-x9TN-friyNT=-*o?#JVBo+wi}$_mxB>;^nu|LVZ%bs8;JuPjx$i_$cj zJ-?ese<kXYh0kh8$eGUXH#9QW*P#`RSsY?;I18vEvfkIPS zaud?V5w(7h)H-?lDo)}g;8TyQnf-QlQ8nMGF8KxLrO`DaS&kCW+z6@L8hxx5=_5*f zU|O)wXsFR1JeoIm6}OZd2XD?;a6y7kB1n76ZH=}eScaw-}2 z7ACD{adhvz9F2Y7XZ>uj#{2|-EnnY0B5BF_Ip zPm^IJTLI9My(ByVT9f)DK>_TOJS92-UXynvCIl?RTjrCYB^Ciclei^50h5#WB_jq! IBLDyZ0Ic!`ApigX diff --git a/summary_results.json b/summary_results.json index 666d78f03..e78198735 100644 --- a/summary_results.json +++ b/summary_results.json @@ -437,7 +437,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1102, + "Performance_Result": 23.1028, "Performance_Units": "Samples/s" }, { @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.435308, + "Performance_Result": 0.436808, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.407536, + "Performance_Result": 0.405291, "Performance_Units": "Samples/s" }, { From 5005367e5dee5bd518498a56eee7be1c62e80cb5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Feb 2025 13:15:34 +0000 Subject: [PATCH 205/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9760 -> 9758 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 617dc08f4..d547673ff 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-26 13:12:18,368 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-26 13:12:18,368 submission_checker1.py:3270 INFO] --- -[2025-02-26 13:12:18,368 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-26 13:15:29,539 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-26 13:15:29,539 submission_checker1.py:3270 INFO] --- +[2025-02-26 13:15:29,539 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-26 13:12:18,368 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-26 13:15:29,539 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-26 13:12:18,368 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-26 13:15:29,539 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-26 13:12:18,368 submission_checker1.py:3286 INFO] --- -[2025-02-26 13:12:18,368 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-26 13:12:18,368 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-26 13:12:18,368 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-26 13:12:18,368 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-26 13:12:18,368 submission_checker1.py:3307 INFO] --- -[2025-02-26 13:12:18,368 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-26 13:12:19,552 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-26 13:15:29,539 submission_checker1.py:3286 INFO] --- +[2025-02-26 13:15:29,539 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-26 13:15:29,539 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-26 13:15:29,539 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-26 13:15:29,539 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-26 13:15:29,539 submission_checker1.py:3307 INFO] --- +[2025-02-26 13:15:29,539 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-26 13:15:30,521 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.325687 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.145277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -30,8 +30,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95343 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0988 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96611 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436808 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8e6742924..2db3ebb3b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -832 +833 diff --git a/summary.csv b/summary.csv index 6ed6c1b6e..79ff40654 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.325687,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.145277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -8,8 +8,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95343,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0988,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96611,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436808,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cd90a8771..e88531a97 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.325687, + "Result":0.145277, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":5.95343, + "Result":7.70465, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0988, - "Accuracy":"acc: 75.800", + "Result":9.96611, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8a60fdb5bce92e4232dcfe4c8f8d113915d6aa42..a65c2fc2920ec3285ab37f16f2ce387aef03d086 100644 GIT binary patch delta 3896 zcmV-856AGJOrA`zSqKIFf<;zLlU@iVf7^4S*cQg$&tE}N{m@-CA{*qEikV8%w-{s6 z=_FI<$s)K_YE&G+T>kq3Z_%|r2-DLuq|SNi*3Gxp-V6D;H=8dW{`xeXC=W7D!f4i3 zg{G^DoCVP!oQ>M*?ak+gt13zA&j$V^n#s2MEEDzDqyHQ}M)B{-Sjtq%63mjee>zUn z`9ZUp1YMFxQTqQ>|9EO4X z7zK-|oTaNIv7Gp6HvD89&J*?Ma1dt4@_Lk54%_OnI5_>y5Nh-2aM|}=C?AuZ*GgW> zeH8tkzd0MURXvwBnfCK?7+0|}e~?3eF-fnZ$FDLRjnnM3S$RkhO_Jrm$~4T*rJ*WQ z|EaB-s`3~P(s7yn$-CAVBc zKiASEd(~R7+GC3)aQC}7f8h!;%u}uqgoTelLav`rKtfLZXCNWqe*qE#-B%zX(47GZ zxy(91LOz^xAR#BW3nb+1UT}#)&TbD#$l1LF5^{F0fP|dgZ$Q$*mz#|*H=8A}NHHf| zVv}=!$|WYL_al&ydOraPsrNINxFpM$jRbrpeg!>{YvimXyt6sBf0?4rX+xKH;E~gL zL1TJ6hR!*cAT9X_u3pAHJo@_It54V6v)=jpM$*Ci&VY=|5PS-b`aDm#eI1zL;uWbWhjuu}rcxOwwF0 zW*Xbf{a`+C1oK5Bf11sn;>9e@R#T0&svktNVK`EMQ3o>h!%3okIBX8VeEqGClCo8+ z%!ao7uV7r?SK|_^HHOwQgwPA-ZU;l>aOlXTcETk#xl&KLgh2-+6gnWG&;beg3jgAi zK*z&f3SCKf2ZWPvzt@z&WPvE^u{tD6`^7lY_>5Xtp(!X-Aj zeoncBfe0iNL?EFc0tp4tEdh~-GZ0I{J0P5b=$3#822VkBDTq9#MPe?w#3Pa`_kRb( zI!dVk;+?xUe46x!4h0hO5nOro!Z!7TC}~XmWrio${@H8NI?Ag6;I+2{ zprg5_ZWQn*bPV9Koe)SSmlL3I$UshkhB1gV6hfq-e-r|ZODDvw262cIP!P)!bhZb# zD6*C==)@y=E?86#MA3c5OP~=t`9Q9A#{V{me3=My8V6F&v+FTVWdA{EZZ;SGWhQH> zumY4f>w#^kbo7>O6;O$H+K&a_S+^`oCD%BDQVDkG8I?f8s012DCD(8&MH!VK0i&`c zK}XPMe+lQTr3*Uo3`!-9>Y*q)Ng@rMC3lAZ7NOjkB6BJYT@ycmWgRtEnJd==6YCvd z%`sY{089L==~(c`H}ne_f>ExsFq?pCj;7(PP0MHd+n^lWQE4 zg2^=;OrT+40u2L`YdDx@8JHje1G6MSXM6AkOxn^#mC*eg7S#h$98B;in}cb-X)wul z&R=J;j`AuoY3?y8T9)BB1x(`S_s4=i6SwFyZJ=>llu54POacvK5@;BcT;p1l$yOPY zf1ne(*;rlZL5f zUzP=y08wDX5x{0coj5?_P%945a3(95agc^F3G6tG$ud6<>Vz>_=ExDulEYfMpcBp{ zbLD_2zH;RnuyU=_ne(qe`Br^8f*5s#XBL@k#hz0~f1wiiaHv%WXdLR*0UAzae+9P= z(l9E4hOJ%83_BSKyyC+-7nzmcyI5%`mropWuHnsNvjZ5u4K*PvH8cHV8P%@E*l4+FLe|t~@N@iJt zj=}6btfh-Op$|q_R1ZWkGJ%GZY1C}*d5z4!`(&h+`l>)_==)IWma%gKvk#>xfRZ~! zZ0ha-8kf3zfQCa^!QF#23`#H_y60n*xqDD249YThkHOqMtfdP&@mds0=I%jJ49W`b z9;0S=&udJk!yHHPLwA+6f7DiG^=j<8da?z8t=kmG=FVTZPh~ukr*bk$lpxAZ*D7+` zy85&KYjJS5`V09NdDdSv8-%&1-=D1ho<9v|BPID`^-BSE{!0ORo-7z%FD6n+pXaiz z%BOku3FXDA%HV0Z`i%f(9*0pJrq8S1^B*Z#O#Gv1GkUVL0<+H%m;nS6I&ediYZF8QiIbDD6CZy` z&JHzfSQ-Kr=261dhuZbkvF>Y#3(6vz<_r$Cd*Ira(O*N(1(HV@Kh$o8C=N`MhqsVX zZsY|pDT(tkqeA{$u1#K)KqGz&AhOgXmgSlmO_+wlykg=|BaWk?iY5P6Fy12A8fc>- zPewyAI?E$Sho%?}%^KK(mmx4(CV77wzh;TxZEyj+N<|w@Q<_ntd)7)1g0?VAHln1d z?r*$RTXqr0X~LilU*$i5aZ_QRO*$?qT}87A`UJ+>Kr&Y7sEQKtxve?Q)4co&_lo<~ zDoccbvMo5eqe)6vNty`t&^;HBwb64KUggg7G7o|CT(MvLtuTfmq)<|!DpP+&`7)2i zPg*LmJI{rBiE9o~XYwUwRnky#Z~{VluhtKDmq@j>)rI!#xDZ2{vu&DInUPdasUEUGHl}00osVvTAn6o%pn|q7$ zy~*|!`ChJSCA+lJQ!2A2mK%TH?WeU>+Hy^mW~AG?P2q{BtRx`x*5nNNAgzJyKhhc7SXAz9h13(VYz%v5Z3mV{50)a7bv4_yr#n2|8&6e`IE(^M@bx4^)sn&)Yph+MG76WzZn~L5AO8f zN;NjP*Ms{#xZ=we$4Y+y8@y*6;+}Dcm2ueMJ^tbze{qk$xW`}I<1beHg=h`!`PED# zJxjwavJ8vjVQl4$ZAv*8k_?oY)mKYh3|-TBtxn^x!o)Uvs_xBpUr@F^92pIBzB}As z8L7G#9i_xxO7`bV5&;yvpe2Xp9X5RRG3n{$|HroDR!v7N1Al+4doLnXp2eQWe@()dcSt%R__NE34BcIKZK5FlSe6$UjaUGJU{RP*Rdl810|i0`?aO+*T?e0e#c7w z;HNwiI7(0-tkq|Bt(lyQ(SNh+7hwnlg>=>UlTaWZf6HpaFc3xe{R*SKEk8&o!gio7 zSv63oOG|e`nk1su14*spzpv~#P6AnFIl6bw+!<7^4zi8@z&fE-g|Z|?5h$j)P`e6! zH0$^gMb1;jX{!}fs0W8u^+U-FVcNo*)ds8=;GzIhjxbZ9z4wM-?ARV8bxCljAa1pl z)Q8gUf3TtKhwcF9X}ZJ`Jmu6=JOE=e6VZg^Y(|=2+l~OvFti{+d55zk!)LDtE8We8 zQMqtR;d=u&ttWAIADoztUDqXDF61JqIssdgcO@nSg>=>Ulc6OR0X&nqB|ZU=llLVf20|kM G0002a*j(8F delta 3881 zcmV+^57zLWOrT7#SqKHJC?Oh7lU@iVf9-Rk$QH)G-@k&Q=EF?YiZsYuD!P?q-eQc& zWRh&%PZq&esa0_S^YY&>s4;lXBVlJ}m(;yqX0&mTL zMbYp1o6})i)pKc+X+N)qaTzN^e>w7()ATBO_$tHkB+X8nm4^h;G+F(t%);zk8mcn$ zAKR*_Di7f>o#cs?NjMzJ{6wotuuRfu_S^c9*d$no3VRbPbg_;0G||#+!^%I8r>We; zG}pbZM)=WB{mx+=Jt*<2hh*XBpNu##jO?@odF!{lIUied({T0Npk#IAe`oK$V;(l| z^TZp)cN%g{8*XUBZK0u=jb}S30}skzW6*cUaK#m^78xdxSi6#IAhCBPH$dX-N^XI~ z-AZ=KHF3EnOCU(OTtUY9{Z+;FtBRXf6}M%Iol0A{(iV$1NTpoS@<^$#ORis++;9o~ zTuYPeRBOR%k1UqJ-RJNDwA}OEbnGvlCOP|!M&yF%`}#BwaRMdi>c;$?_?7n%OqRFB+boY zrm@XD2o{S*uvj*tfBF0|Ue42OHPzUt20=6*g=6&>btqFmoF?jr!{!Lg*Wcxiy`K%aD>g$2QlkfayZ zrzPQCk8pBE8+yD0whpB+eGo&}p+G`Df=jPn*k*nZC5@@S%JAsgKYLADM|l+hyz&YF zI+|4dn|AP!Li3SwD;?)Kmo zMb^>-op>bAIg9FpD7w#h0W?A_RX0nzF zD?oX@8Q7XiOYHsMPNgWL5+q<$ zmL%v3fBG!pjJ5PYC!Rs6q)~koMJGw5p|fOR_-_%)!W5ZPY3Q2x0W9mNvC3Sz8JO7Y z0BerX5__=3&zg<|e|%%v6icpg42mV!a4dm_VF@%0ORnKq8f9351PsfP1l{ey7c6N@ z4|Kw@q)~kk#r9BvM(BcaY5X5>Sx1dk_Ek3ne|x@*G(6XF_RW|0spyg54;w9qg2^?G zNx|eA4kpkrFoA}F$u%5IvkXj-fPq<(pu0Wz0w!(gp-Slf4U6i7C=Mogl+D33-!zzH zJLj)6Sx0%5m^62Hb_~ldY#Qz|iJ#ve3I0snqR+H}#%WO|xrQ?dG>l20VN7z3Yf&a! ze`QR9PUvQ9S%U8N;0u$qr3X6kOv)sU>Z2&eB-aR?KA$d(zZNE!ciB|9+(1cyOz&r_ z*)pFdg{K)mfMgxjRa(Bbipy8c_8e=Ep+@ZSml504TK*NXx8b%~A z#@LidtBgp{2_v#3L3exbg-F`c1D&vqe^(aOM^TJOpka5F)*AxypV)n^rNBx|S_LKz zQ_H?A_gDhN9wUwbHXG{10UC!|ae#(1S;35hG>l1L$6-vC`EgJujL9-bj$oD?*3tu= za3+~62So9eE7yRPYn{%Ve+A07>eCU#s3SacpUGD2ICb{XOAmCypNcSN4v6AUMYsk$6{*>s!>RoFk44_7 zYv)=NZ9BFmOnvVH*4=|8aOF_r4$wH%xdSvD%L?8dq+wVB4a2g`zJn4lEXxc$g8FOD zSW6Fd!uMa9PX|S@$r5PT0=8zWf6i-E{+#{SH)`U!5=9qJzK9iK-}bAwhfCnmp*9|% zaj1_6XgHS@oIFUwxC9!;Wto`=C16~Z*?9!>^RSj4=!A30EIlBKFJid{EMjZ6_53?v zzEN|}cTsfrWD8l-cK12X4IPtdaI1(-?L9!_QhN{3Ffx&bl8H2wOr)V?e;Q@>9+ZHR zS(cz{FnbSc>7h>OgAo?h2T_bnpy6a1HQRe$BlGV*8L6edDo`5wE|j`u6mDR4p%i48qX7KM_zdr%aE zvVyzEsM+1~8k6ZL$5H&ye_drQwN+WY8bw!6wg9koo8s6k{Du2W#$$ORr_)3UqU?07 zBe$*VKl{HD2e<3LkbjY9{YCR(n0xyD>H6>a({Mgkl0VkJ6kzAS6rktHg3;A-DwXtU zA=|2aTx6e6UaYDNA4luo2v8Pr7{y`wwC+9sk%Hya@64j%`_-V9Ui0verF{P`cxNBH z3+xZSD8>h+tz@M?OS9@$z2q;@usVkee=N`acpT0XWhzHmp}LmcfXC}!9az0iqs8i# zr7Cw(nnttdcN6IkWt_h=Rb>=K>GPZXlx%;!q4OuR%@LRZ1pHtUDwAmwL;^{ildlsW ze+kPEHEdWKf;?nVlCKZ7>#Jkk*AN#pk7&wrIMnWeYhOlx4LKJ`9?kiob}K}2V46I< zg^Y586_86w%*u=k`ER*4c~Jt5_$`3QQj=JgYi2abH59Tc7l#^g91T@@@^1y>ErP9q zHX8C|G!&yV7C|~R#b{{Oz!tm=A*W@+f7Ike#`_5*TmD(tgK$0en!Xf{Egz*rkd#tI!(Q6fIKHODMv<`|Z4Sr+cWNe@0~;}I29mZHpr zSr@)}b~Ufjz_g}2K)Vz|<11TmGxxeQqzCu?(W zQNB0Xz9QetRjp)~R(eWh*2Hq-f4lv(wn|&BsnU#eTem4Z@w7F11y4uYf~TN#2ow*L z8atby#r-eHQGmR_v2GA{^)V{}_4Y=6WIF#oMxN&mkZV7q-UJD4bF)WktJ1CzE-tQ5 zruYE4*zhcZ5qbc~0UCHl;Cev=oJt@t1}^pxy1Mu}J-=Mer;Fv~(b@cTe_EfmRJupg z18msDcYm994BsdAZ_C6t91q)_ma#(&3!yd7{2wwg+ZL9vWz}%D+7vrz$0elj0C|$p zQo{i}2PGgSh$N>Sf)$jaqrKLJ_@@Jw&z~$VJxb!Zs-H=HqrOHoD^lo~{mt00dvK=* zSE{kWy&l}}!4+S&I93AKf8agi5ciBjtc=43@9`J+_=|h|#XbJw9)GdoFGOo-&#z`0 z=~)_Xk!4sE4`Ykv`KFX}A<00QS$(zC#n3g4*XlG5D@<&&r|RBp_XTC!!;#UD@!jG6 z%1G6{=qM%jQnEi^k_e#S1uZ!&@37&kk4aB2|39`Jw`w|K8Tez}e|r((>niUV#{C!eUV?x zk@DI1Z8n)5p%XZkf0+QZxVk~M6)2I|DGoJAkBx!0jj`t}Y6`2Z@;InaGpH1KjGEMS z@11JuvZMI{vID~jlnv==l_myp{GBb`HC)&Ac2;R(IKCx!?UU*G@^pGG5m(u(#;3FS z@^XB!sP}7EZuNd(k-*2K{zK?^HhGj1`4!+3$MXX(a2;zfBT&-0-LGXUsQdM?ys+P~ zl0Wzwe@- z{{^T{L;g)X1CF_GKSA>cvydUm0tKunAsSATUnE-r&y%|(A_V+k5-OA7Bo+Z#llUY- r0qc`GB{~6JlXWE~1T8zc`IDa|76Cq!wk19Rl9Tl%BL+kx00000j|XYg diff --git a/summary_results.json b/summary_results.json index e78198735..7450d1008 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0988, + "Performance_Result": 9.96611, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 5.95343, + "Performance_Result": 7.70465, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.325687, + "Performance_Result": 0.145277, "Performance_Units": "Samples/s" }, { From 72ad3bf0c37892a03d2edb158bad912bdbc03534 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Feb 2025 13:21:55 +0000 Subject: [PATCH 206/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9758 -> 9755 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d547673ff..7409e0b27 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-26 13:15:29,539 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-26 13:15:29,539 submission_checker1.py:3270 INFO] --- -[2025-02-26 13:15:29,539 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-26 13:21:49,986 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-26 13:21:49,986 submission_checker1.py:3270 INFO] --- +[2025-02-26 13:21:49,986 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-26 13:15:29,539 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-26 13:21:49,986 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-26 13:15:29,539 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-26 13:21:49,986 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-26 13:15:29,539 submission_checker1.py:3286 INFO] --- -[2025-02-26 13:15:29,539 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-26 13:15:29,539 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-26 13:15:29,539 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-26 13:15:29,539 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-26 13:15:29,539 submission_checker1.py:3307 INFO] --- -[2025-02-26 13:15:29,539 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-26 13:15:30,521 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-26 13:21:49,986 submission_checker1.py:3286 INFO] --- +[2025-02-26 13:21:49,986 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-26 13:21:49,986 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-26 13:21:49,986 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-26 13:21:49,986 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-26 13:21:49,986 submission_checker1.py:3307 INFO] --- +[2025-02-26 13:21:49,986 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-26 13:21:51,238 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.145277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.69756 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73748 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13056 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 2db3ebb3b..6006f5caf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -833 +834 diff --git a/summary.csv b/summary.csv index 79ff40654..0284db881 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.145277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.69756,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73748,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13056,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e88531a97..08b5a3163 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.69756, + "Result":1.73748, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a65c2fc2920ec3285ab37f16f2ce387aef03d086..7d2190a5c650f897c2d7ed396626fcfd195afb41 100644 GIT binary patch delta 1116 zcmV-i1f%<&Oq)!wSqKH4i96*=vt9^Z23awkBI}SS@R$g08?Zb|B-c0=C6a46kwC+U z1R6#pFvi%FNUMxU&TN9>k6*$6$ zTYx2S*0VORnU!?Mi2gAy<-%M3h%`fJXASxXmm!uMa9PX|S@ z$r5PT0=8zW&TCZuoc-4~YT~&TyDpx55i3N|_N!OGCGhA_8xPPp)W-ufoXZMM9;9Jh z0uAG`%*=xlFfPmNJc9XoSW6dl!ntIY9uUPBv0MWdu{GOz{v9yisJZ96*md`03t7{4 ziyY^Mj>$B*Rm7%$_8y>dsl5ki7@0^z$wV4TCelzcjWT-=NV!dA=I$|=yN9)OK_^~|Ldo2JJt&GnS;5_7)a>qgjmdPF z<0yXUuCkWes;pj(T~|-G0I+qN;@I5z3-_svNAgroCW#V6+38wEZd+G>_J1u7?pA*x z|02)&i)Mo`_w@Ud)!*}{;cTQNf2@8fz|MauK+lr}!|TOFD(UlFwpIBw&px5NSXCK3 z4OhPrpv>ccFp9(UdDVOVBL$0ze>9B-@0WvG%)&nw^8LHuoqg~wus{5w7$20jl9j%Z zX4Nfw$zPyh9Uad7k-YHZQ8-JKi5zBy>RNUK9s@z9u8cjFv#?l|i zIDcoV$}o!3&71s`Y=6Ds=uc2f0|XQR000O8002L;+7WOL1)AIMyd;xPAS!>`A)yG{ zfwpATK%pru-3e)uh*}RMwNC!N@-qo|mF4K(Idf-Fu|CN<`UC5PRwc@k6h)wz=0Y7x z^s(E-k0^4UDo$&yphRbIXk9%N%n+t6yjg9)dI2s9Ams=%B|3U<2*!>bK~k3lhYI3e zTSgs}@@M#K`}*Fk8Vl8)iUFz$&Iq$Wy|z_>uZwWF znmWG?C*BL-Pv@Y|ybX)=eZZp-fb?-k<#X};vfH34PxED*=5fB-WeXURXJnavwvRD^55Q<0k8Ih@L$|5)UWx^`~Fu5HKGgsB%U zVBG>Nfh&g^cYwyB&K;oPSXS`vAPvJ3Xc(4d_8pXfVOeJ25!7FE&VO3EpcB6T%6vK~ zicOY4!xpeLTXkNe^5^WozEKm;wb*s>-^H%GCtJvx zwp-*lH*`#9g>h1v=m%4j^ zhC^Aw-Gej?N-!R}=VO$)dr&6~$})G4!Q4Hpr3*UoS`@6aWAK2mk;8KeN~oa1I59bk+DJlT9Ehe?Lel z!gio7Sv63oOG|e`nk1su14*spzpv~#P6AnFIl6bw+!<7^4zi8@z&fE-g|Z|?5h$j) zP`e6!H0$^gMb1;jX{!}fs0W8u^+U-FVcNo*)ds8=;GzIhjxbZ9z4wM-?ARV8bxClj zAa1pl)Q8gUu%YaS?f~a$y2KJZf92FuJOE=e6VZg^Y(|=2+l~OvFti{+d55zk!)LDt zE8We8QMqtR;d=u&ttWAIADoztUDqXDF Date: Wed, 26 Feb 2025 22:29:51 +0000 Subject: [PATCH 207/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9755 -> 9747 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 7409e0b27..e9d7622ad 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-26 13:21:49,986 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-26 13:21:49,986 submission_checker1.py:3270 INFO] --- -[2025-02-26 13:21:49,986 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-26 22:29:46,601 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-26 22:29:46,601 submission_checker1.py:3270 INFO] --- +[2025-02-26 22:29:46,601 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-26 13:21:49,986 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-26 22:29:46,601 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-26 13:21:49,986 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-26 22:29:46,602 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-26 13:21:49,986 submission_checker1.py:3286 INFO] --- -[2025-02-26 13:21:49,986 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-26 13:21:49,986 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-26 13:21:49,986 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-26 13:21:49,986 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-26 13:21:49,986 submission_checker1.py:3307 INFO] --- -[2025-02-26 13:21:49,986 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-26 13:21:51,238 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-26 22:29:46,602 submission_checker1.py:3286 INFO] --- +[2025-02-26 22:29:46,602 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-26 22:29:46,602 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-26 22:29:46,602 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-26 22:29:46,602 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-26 22:29:46,602 submission_checker1.py:3307 INFO] --- +[2025-02-26 22:29:46,602 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-26 22:29:47,772 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,9 +31,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96611 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.89347 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436808 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.427381 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1028 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6006f5caf..d50bf78af 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -834 +835 diff --git a/summary.csv b/summary.csv index 0284db881..ab0c294ca 100644 --- a/summary.csv +++ b/summary.csv @@ -9,9 +9,9 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96611,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.89347,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436808,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.427381,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405291,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1028,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 08b5a3163..187d439da 100644 --- a/summary.json +++ b/summary.json @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.96611, - "Accuracy":"acc: 76.200", + "Result":9.89347, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -369,7 +369,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436808, + "Result":0.427381, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 7d2190a5c650f897c2d7ed396626fcfd195afb41..4373551e87813fa8e438662e1810e32f80777f3f 100644 GIT binary patch delta 3957 zcmV-*4~p=cOp{EoSqOi)&=&Va2><~9K>z?40001ZY%h0ja%*#FWo2}8FLP*RWppww zcx`Ouob6NM$QH)G-@hW2nh!HoE2KN`mb#_Rn~3N*>el^a#n3=%fv`#N<-gwq5aB#0 zF*`fEvF`mc1D)rb?i2E(JDo@Netn#bwR@GMQ9Rw(Ma$4NH4T5`el#8K>(^JGcRXE7 zvtZf}#_?3`>rX1xe?9om-b0-Ho{p5tv^>Ey-PcE1Hrs8t({Q9FLE4IEYMLJz#K|Pc z^0&#boz4;!^cO*saa$UO)1Cy;R9|M;P3q)`2ZJb7ALDR7QPXUhBvIoa%ZHziqFJgR z?DeDkSW%CbsKI}}ekgX2f0IIQAM7pqzKPUBy75{oO1X>U--|aV{e9giq;1fCUJT!>HoFK3<8iw9SDQrnxsD5QRv2O(?YD$wIMOaYkEgL(#k7Byy)H)hG01|0y(E6nl0^^c zEGRx1u`6YMTEe3BThUyMt-Wfvd~VS4I*PM*e_-yl?~BAM#djKVMH{YZ!_7uRJ0H(_ zP?8^%gfSS~W4J_xWszYr5_?;6#YmiO$u%Q!w#9K?Y$~F0NO-LX}xu_uH{Qj!q z>Q%+{tBQY{D#cc%7GJ3a@sd=E3d<*@zAm|XU2=^Q`ngsn*{arNt39+Kfw$eo5h}o*Fn-qU@gc66G`(u=tq~4E=gw*?qk&t>n zqr@XwzC25quf(rR59As-sR-{pAA6pnPH96IJMhWrJfkr^jG=SRIg^%r1Q##k9vpoA z@5QIf?n&?T{bDtg=F6dBl#S+htuUUnC*%CzNxsy{U(Hl9xQ(U*mE;RHb(^U)yG`eJ zlPG^J);{U&z1=d}S)$ZpmDMg5Q|+_v@hU!1X}*R@+pERQ&N}mLIGgQ+v-wUuojxY> zX_l|1c2=s}FrE&gq5g~BS6L8^Q~kqUd%(=s-6i9@cRW0U}hjD&*7NGOPmgo1zQ zRe*@$0AfXWhY6=3dKDltgQp;R6hw@%NX$7(d?LB<{&zrZqLdmS-gsM6qG@}!D7o$N zc@ljjkzD;pC~@eG4@!VUMnXwsB$Pyyc=X1{uON{LCl~EWMR><2oSf2zE_MLxP#V)? zV(2;)B^D|B!fzI~Nf5^A&Nx_Pc=UgqpS>n+qP!XaUiupVx-HK%WC?%5-~n8;69UQP za>Qs{GLU0N0}OH+3L&SV6fzo*PKav_$03)1f>@QHvp%>+5n8%TCqBt@22njGitaO> zbD7Ck;>E`B-v*E`^I!pEU#VGsIVP#Py%#Mnp9=pnl8sbYgUPGaz@DLW4aaevi5i4bR6)rk04OUGbOe2ra0)G5rW0RMC}~uW zi=uNRqk&0sWB6|o%8ePapi&ww@dH>kQDdE{ay2lq+5m33(h?cv7}KwCJMGt8I3Ru%BB22 z;IfGt>ujs82KIatDSgj%OSW?a|4{T$;Il^CrC_4QH7S^=K`BpF)1utx^4-R z{NwvWfse#(`beA6xE5s+HOM5R0VWv@Fo_z^rc7E@OfsF&z1FG(o%O*NCTUBT>BKiF zlQgQwMFEqj5r&{lTFo*!zssk>`3gz`WOhGU%$CJ8Dg2iF0VJEKuGaF^+FZW2T+=Oc z4v7+ri9pwY?NcIAnSHjLU`^aTtwDoj8mJnXKW( z;WWS`OO2(5zZ99y{rM?|T zgH+aV?r<8QlF@$vl~v{)E&)(kW#19hU2_U8U8WPQN*F< z9!BF)a}T2dGC2(;lhaT#ISnOKR+)Ra1eDCG1RV+H9%$)uozUkZ5Y=O%fJ{b%WXgu! zJ+G1ZcOQ&2QePb?rLhgAVN1UZN>KtO8busx?qM{ZpitH@_i!43vWB^b8xP&`kyYj% zt`mT=%G`e=!Q2BaU8WP?qELdlhl>IzYnXdv!{(mXn9K$Rj^c;zDjTV-&gxZentJjD zfMYlm$M(kGw@*|uRL5#OPPH)3PuDVX-Majv|4XrZv;6z`7kS=aJnct?qdyog|C&FG zrb8|LWBEG)PVqYdMv*KWT+YWz%bsRxUssQ_{1bmFiq*CL<6!v<0op8y;v~wRmc180 zQ7|6|2a~w}ele)|H2Pz%-oFdqIlJ#d=ff|W{Gjb?dFjv6yt+j%#S3m&2Ya(%sLq09 z7)?`ctOj|ZMk~JoPnN$quy~!tv&Ac0*Y4shizmhzXcFkY7)zG&5R~YLt$Pqai|f;(NM*be=8Vo5o{4{G~~%>C`M;_1b^w!6r-V8 z16%Mi1V+mwZ{yc25xfm9fLEz#qiISrN_5X!=|RvoX30jBG}Zl$w`$ui;y6thwBf7# z2QY3b?6XP7C8euqHbI}jSQ|)cg^sEy5ue*D$9bBUU*TSHzglI95Ky)aM|U(y=_*MR zp>De80}IOhp+9vrc^T>~x%GSuXQneY>Qg zkY%Np5JM@e3iPn3^x|GB72YoIh~dgZBZ#R?&SjXhI9Z!}lk&aE_J1Y$Uao2-tF+Qn zDx)Tr8{h4xwN=`3O_gG#+q_NTiKo4y*YLEpZFmx;!=QMe-79`IiD_;mq%yw(`miiQt2L#9$>>JzWe)V$MAh(|2~=c zhT~zo^JMH0!$N4yGyjJS%(jFjY*{s&tv1CD+HnbKJV2h5%2MS3&Oi~Q1d%Y>VOT*a zI?8KZh<`d@`TWV`(xW7ftNNMLH|lFdvm%9#-ruwhy9alAaDOEm8{F%`{T^KLWs74) zzy|MWhq$L5Vx=86c#prh$6wszFYfUd_xOtye<4~!yMHy^NYB!6i!8&UcxYQWW1CX; zg(L%|XZ7V$7eiM!UaQ?WtT3_7o~(QGxi3iD9*&HLIp1yWuT-k8MMp`omy-SYl0*On zFKEeOd4~;OeSb`PYWe@M?YLFb5zD|I>)wk9Uzd5$Fz%lSUQ71JGumPjciDq~wPQFi z$QzYZXO53d%Qv&VLC23TgXiepzYgAo@qIP@z)I*~6LC$1J3`l|WV5Vec-<{=)G@nu zbb;qa;OFdJG&A{ikbYWxMmLbN`uC@&6FYEj+6yZteSeW(%#rfh_iZ+r9ibCAmXQFo zxVk~M6(~yVB!?=}V_~3eVeHw9n#5`^d2CduX;g|mMos9t=T0?r+0py}*@58%%7pYZ zN)v-P{?3%{8m{YlJEJr)9N&_&_Q~{oc{)9ph^x$1fdn=L$AiTn!iiR1Z!7r2f!7%1U9mbdad=I#e)WSleX1|?XxtwiPoBV?F(&!2mS&kCW+z6?g3Vkf+=>tl9U|O)&XsA#R9?j~T zl3U7+gEwa^xFEqN5u`okwnCc_EXCOK4JhWb=+IGI8>d)^rE7n%W&E2pfJvTDv4X$^ z3k>gosU3;vfE0X0+8@^r0KqY|pg;$Yi>$yWZvdzK)rLVib1E5n3s$nJx*TaIss6VawR4k0&BpSB>?~cPyzq|5dZ)H0000000000fPpcSvn4(O Pg_H6nBL+7k00000b#!bH delta 3981 zcmZ9Pbx_o8*T;7WDQTpeT}pBlSXW{J5m*|eQ;`yhrKEq7(gI7Di?m3WNQfZd5-aIa z(k*f$A<{4R%scbU^PE4gIrF)`=X=gva~+I!u6C_CIZb}-{WTN?2n1FEfmlHpR|q?> zmN)DDc8a-g;hU|p!NrKR3_r(eMqD;(tcBHnYztzvf#d&R{Ol2Dm(}GN*kiOwk$imo zPBA`U>-%ua_j#(5R=HeB?#t7=`NFg?-EQaaCQ+r8ae1Vpym^T;+rNf4qANu;x4+%E zgyi6wPsJFI2t`+%iVuJPgEJd=gku;2QkxOF<0r}u*(CP&ju^#r2F?ARvi2G&4bS%# zIl^9-D4E^R5zG)+9+?h{WCR}0Bx~SQ6h{%f+j6$XYAvlp2t8Ne2Ee^WK>NT z8TA^eQvF5`siJ%C#LKq`fEkE>nDX+CbomS)P zbO#B8mtW1x$h;pO$g%Fs+(b#aTxhL^w2;*kj5ja<6|6WGNKOt^0#j5R-^u&{8O%&1 z35JG}DkQtEzK66o0QZ`+-{M_VGhZyGj0Y&&2g*m|?`86epO9DE^eM=no-S1C5q>nx zpPa2d6UJqz7D;hQ?l$q}$d+Mt7F?Pqa5lpZ@;k2@3KJSgb9c&MnrFD>Pt(dZtRx-TXNNBA6g@}|05yS};ezE}v0)RV#HM#V z4;miB4FLl(*inz>+fev(n$N)(RY|G!#+5_S&F3|CRtu&p43#p`-(is>RO{|!svDl0 zUrYSDp|Y)pdaaSlWDq6QIStw0V@+wT=}#7k@mM|h;b~~gaK9_K5buXh@$HmL73}toG}>l8p(@ zCiiA1GsHLil-7NErR?l3%W~-;=vq-rlqUBx%&pw#FvIPZe(Z~x*Q5*PEn+Ar0UmBh z;kU__j_fHXVSflEc#5jArSpP|>LJSNz=#vY#`_W~#P6FZXySIEFhY}`Iv@HX`ii-5 zE}P$5l-in6>7@ggj7j!w0x{!HF1ulAi?<-H^Y>e3d;M}5I(4h;{a7V5`i)(m7% z?J9dxi9s|}qpMa-b7Y!3_;YHrkFpS_7W*Wz!E@SM&XE$rnSEE_Oy3DJ2U}kSpmx~0 zc&RnXsP>J(p3$iNXC7G#8d8iaCng*x!>NsZY)3>*jB$l$IkIK(f{7a-%1%Lhq}nrM zYUv{v9L(2HPVGJ6+z6LW7b%<{-qxu;X3~YnK-)@N8R=BP_2BujQC>$q^lTj)AR`#OL zmI@Pp_FUBJeIW!{N;|&xJX;UbJ)XdWtWVk&6}Po)8#Eo3&gcJ`EWDRR?#@vwkAe{3 z%+ORDLuRY0Qj~=GP{g0?d=D1_>J2$1l>qa}e#n znF^^Y=|X8++}CC=%?L4PmNvKhLf_@6nh!^==-;Lp@f5tBg-(u^rr3ce;9yEL+mX`d z>tE@IQhO{uVj1xe@3oi0onC9~wO&=@*#|Yq6O#}`IIC}@6Ei+7NSEs(3boi%Xu^d> zmz$C~!_#j8a|n%p$D-L!{LT)RBw+wHn1oHr@&0}-Z?BAhD4Y20UnBZzu%6m8zYR}O z;L<)tZhNMczO9+dlCjkN7Q8#k@+H)BO@D=E1U}J9e3x^Oa-OUBu^AC{PyJA51cQ{= z;7$4PQ&+sX7K=~!v2C^?+^vo%z457<$HWBRZ}7!X8@ zLgK`PC%|6NZbvGcukT7{wOuong>2$gaB~X6 zamwbOf7nm#>;@#_e6m)ALpQ7IfsC1V?`;yTP4${VwZ6H_Q*$R&#}x_c`;T-5V;LSy zZ&|3hTbcrY0CAdK6M*lE|J!N2V}#W4;TdLf;h=2kc`*cdA1cy?2SkwHFr+;{dmo5y z9auFRpWOYXSlDi6%qCB5`iUMpv?3MPa~LogbSXd@9}C<~yzx(FgvC&|of7l@McZP2 z$eu*4()n9fp0wuF^v)E=N%0_@E%q@N7ycrpeQ(78u9X(c=@_+b;YZ9}6Y6Vsf=FB+ zG|hN7r|!b8Bll0=7|&NY;Ph@lDsF5z_dM-6(6O|sr>{j!f{BN)H>}bJf@v5Lo~m{v zAdn9O2*d=!>_V6^;f!*Ccd+7{O1)@XS@XDE?5ZZ7M&`L0I}!7`LHGApGe?tkyRI=* z1N=GbH))!J7H2X?ZB_B4sp@P>6Yf#MG}@R~Ey)J%tE5sQu?Mgsl}pQN9h)?yKqwV& z{RAXqEr(4fE4sV=$+EscQc+?fF z>dj`ehChJOh8+M_b++eKQqg?H`i9rPEwvNa>s~zz(YpKJq5@-GX$+k9p$^7l`e;ug zmB5-QHZ2afA!)H6^3h;23dB^1;T^IX;O4nO@%tUPKevDro9buKw#G3d(FVZ`j)h-k zO~x}`I_ZwZvmbSqV#}&VZ~G%=nB_@JRI&cpiK%H4@m&C{3f?E`|EN4Q*&jer-Xgax zFn)3@y)XabrPUeRk)VciZmxo%Wx8LKCmJ0B>m~cz8|_4&FhY!^8R@!1M4)0{d_)u` z?OFF~WD@8lY@MumDO!t3zQ(VYjh{A_j49-B+w}-`p3OGe|5dC5ECoB4{HJys6S$t3 zkyiz*UnR#*H`hjrS4KBXmQE?(m*9&c`|3@75-Pj|!Xqg(DEJ>Ys@8@Z`ZN+g(p~v< z?Vn(JOVl+vO^@zWcuWvt9Mvef8a}ac_|fhjE&?bS*$;$>GldV!gUtOX5J$Tc=W+o$ zCLKMssBpQ=4WM6t`p|DXw<2kB9_38^>ts?}KtZcVy#8kV^5f@g32vx+EU7osvL{Wo ztZ!P6`&a3%)OLT74SKgb>}RXrtiZ`e?cI5Q&X%p>BMVo_7aP={4J`&RQ#%&=YK==a z_y~~R+FHBnZ(cFcQ5Pcy-UmtY2V1Wx^n3sW{KcnvzB$Kh&Nc_fAkK1 zYTlXsiyD3YSJppi_u_2Pnsq_0lxK>9>p6dQtf%_+DgSi2uUJi+coM>okR-A*YY(S%!RQrBFM?urfuvs91_L_1<5@5wS5mT{B|h+z-5VY0|yy+`mKNpT4#^f6v=VzfKW{I3@lWX#yQcWm3att*daM z4(n#WycYH<5Y=_9!ai{FbhcJt8lo!&#M{l?x&pxxl)P)?cc6$s5AwbwE@i_^l2??= zHkO(p!oR>A1EHH>&w|q1VxNt46?(;`WEttHHSmJPCr!R&i#JuN8PZ0JJn=C9QYk2p1c9s(QK?9Ruk{} zAqVk^T4ml__&NUjy}q{i|KnwtG3FOU8N7j! jgd)J^m}gKyj{oh(e{vC+N~j_@5wi^yxS=HQ@A>}$8YyFb diff --git a/summary_results.json b/summary_results.json index fc2ab915c..d0b2c6eaf 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.96611, + "Performance_Result": 9.89347, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436808, + "Performance_Result": 0.427381, "Performance_Units": "Samples/s" }, { From 5f0b1c135e858bbe82a272cbb2064812241a88de Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Feb 2025 22:32:46 +0000 Subject: [PATCH 208/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9747 -> 9758 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e9d7622ad..398040d65 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-26 22:29:46,601 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-26 22:29:46,601 submission_checker1.py:3270 INFO] --- -[2025-02-26 22:29:46,601 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-26 22:32:41,985 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-26 22:32:41,985 submission_checker1.py:3270 INFO] --- +[2025-02-26 22:32:41,985 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-26 22:29:46,601 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-26 22:32:41,985 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-26 22:29:46,602 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-26 22:32:41,985 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-26 22:29:46,602 submission_checker1.py:3286 INFO] --- -[2025-02-26 22:29:46,602 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-26 22:29:46,602 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-26 22:29:46,602 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-26 22:29:46,602 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-26 22:29:46,602 submission_checker1.py:3307 INFO] --- -[2025-02-26 22:29:46,602 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-26 22:29:47,772 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-26 22:32:41,985 submission_checker1.py:3286 INFO] --- +[2025-02-26 22:32:41,985 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-26 22:32:41,985 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-26 22:32:41,985 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-26 22:32:41,985 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-26 22:32:41,985 submission_checker1.py:3307 INFO] --- +[2025-02-26 22:32:41,985 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-26 22:32:42,961 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,14 +24,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.145277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73748 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13056 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.89347 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.90597 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.427381 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d50bf78af..2986e5b80 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -835 +836 diff --git a/summary.csv b/summary.csv index ab0c294ca..892297b6e 100644 --- a/summary.csv +++ b/summary.csv @@ -2,14 +2,14 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.145277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73748,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72551,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13056,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.89347,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.90597,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.427381,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 187d439da..a46c359fb 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73748, + "Result":1.72551, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.89347, - "Accuracy":"acc: 76.000", + "Result":9.90597, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4373551e87813fa8e438662e1810e32f80777f3f..0fcad22717c0c2c3c6996079fe7dd3c50693237c 100644 GIT binary patch delta 4003 zcmZ9PcQo8xx5sA~H9A2?XGX6vgwZ9sC`m?dVJ3QXLGUvONwi_q=){Oev?xImy+lux z$msPU2+^a2EAP5%z3bld=U#h#_CDXe_gQD-4f71EEh*`0D15sl=cN+ZxyT8`Q--vR1z}aSVaR&DKTCYku zHYWReIQYl6q4(eBuAMa9%9i53I7Q?N)891eaQSAYb>IBD;Fg0&b5zQB zKJB?40`GZfDY#LkbEy~_zci1pNi9o(X994~fE>a;>0N`!nw(QT*;fOWfoS&b`)WgT zJpvM0y`wzhQ3U3pF9Fusr;CQR7=`EME6-y(rExsP4l-%A{%NpuX0@T1{P@n`B5p3a zjx7tjB&LCf)+($NrL8wKClcv0!segt$q<7pR!U}czV4i4*J@Z#_?7k4+ zoTqVmAzUiqX^Sp7k4lI7Z%4YMh+m=z4+AA>QCX+Po)Zt!dcARp`(6DCoHXe0A{5*L z=8E{^8KC=?`K{)&Pl?x?ziU6e`*qZ_vtKP~?S)OROSxT!!1;RV#n*?NDx$mZV$s`c z`SA+U@mqC};LZ0R6?w879yiV*ge?KC6XH{aoAmXu%DYy(xiK?eVyvgU-;WhwmWg!z zZBaMBvmgACvGLO?*NUNneQQ}E*WC*1lfWsQr6}?GyifV$d)KD2kBRc0q&dzf96bA{ttQuvltm|H)y(eN`r+jZ?GbMWx{@zAYZE_U$bi%57$|(-+ef$AXSfsu= zc)yY%`dN~?W@Wyyx~9>&GcVr)0A?bYHRt8~k_)D48bqpVs^<9d@(I1a23y&(^M0W* z{Fq+%Gw~CI$v3bnB06W*OMSE>BRfIrRyMp^S4POj5$^}|!v)KZ1#q|!H854#v1~>o zY#<{65(*C|Q^C2fyoG(NRRkKciiTZpXFOR*9t+lR3Q>+3R?Og)IH9bx>rqiYJ)JK% zo}8_nJ2_i*6YfjDT`bKdwVkbjm>}S{=3N`c`|O4=%3HCuh4Hmy*;}PI_0Ia1!l8mE zQ$jz~EbiJGIEiGGi%ZxWy0XoV1%!&@J!tVRo85*^*-boyYY7f5F2J2y;{A`AB}BJ> zYRnF!T;yqVHl#Mx3E>KQllWR_L-j! z>zs64P2)z5U4rHtA{o-_5z(3yFcu^)pZ%hJ4%2#2qd}n<)lP)>UP5RKhtbAnWIKN6 z{8}tipVRv3&lCe?exS+M4d#p3KGO(S8~gm&-mDB-9N&B52kmZeymUbPB+OB=cDSH; zYZC8?;(|7}aNm7j5z4dK@32L!`!e($`gOqNo5GEnqOp`PmOH$Kyd+d)UnRiFv9b&X z9ZI}8k?ZLZl;p-5{vywn30jVlO;gX+rclC84iz`JcS9b}D#`{fuS>>t@5Kj1kWpAbG^E3X^ zli}nhH*E!CUPC1`1qFkDZHor5n-j;Mwo6NnXXeb>0YTbZRhtIjHv4%R&z7tNf zS*Fy7e=l@?Ktfy0P28ClJD^)!#c=BCu8~)XWI(7Bk;S0Whvp@G(pwxG z^Ng%+g$c1Y4Y#ms-R$?+a*=K+G7!c|Q8f4T8Rm>Y@r+hF10fcS4&pE%Vq za%ibm)(UhO_C;C{8za*vSE-fdkp{?6Aq%rBFjUCtC@I6y|0p4X|EmN$-bTRxG&IEd zyk5GgafHy(4b>+>pGw;fr=K81J`#3Gb-$U~maMwFKRJ0-VJA`Ua$>b0!#E@F5S*%8 z=_9TWoAl<##xPPL;&mP3YebUGZX_2lYoqldhmRJnCD)Z)Lq6oF0>uA@N>=aO7Ol7i zSy8JwvogS@W!42|A_gBFVJsb}7|66a!5;GCFGvw|^-CaH=Xa4eC|ud`ZHb+k;`_@5 zM(lV@B4@j};@d0YqTx+*y5~z`=9OTvNo20#C3ULTM5(}YQ=Onk$$e&9iU9{~qFDM^ ztAL1}LOZU4Mc{#%_a^V(&iFo6KUf{bBrQ51t`v!M8@BvgxBQpV&R0BRtwwU?>t)2; z&a(59B(1qO?rYjxQ-NCq>45S!M#&ixajaM@^vM--_=;&^_G_D$w)3jl(lJL|sSyt? zA1yGB+xrb$H5V|qT=s_|gV9N ziPedn3c4byau+Pxn0fE<(k@>SGcb?1QzLvVAFVMCj&%M74#R5?B7|F>uR^PzR}jDL z??jwv52?&|SAGzBlA}bhu3mLo&oqp(*79)M0YfvUo<_tZzd)oKzAi-b)53u<;o9Q@ zNb2A_2v-#Gsf7VsfF@7;-??){lh6u{y&=x+L$iS6K3$1J6n6B>apAZ`rY7WKYd ze))i7P=c6n>h&2F-Ohry3)>pYg-=zrvp!G+bQFmD9kUP4{wKvMtwQuRH%s?l7a6y} z?&%k6>{ssVaA4-06dGTJny zioSlc+-Y`_fen%5((kJt&wP6`9X?)W&oQvUcX-Z$zx&OmRlKc4!; zt4TiClqBJ|M8xNNc#CxRM~FT-mUp6Tu}~jQS`-kx=S8+G5m|wihwh;fEXA^{8c0O9 zi_5n*){iuNrI?PX-vN1)bL4DojR^A-o$P1tOEy-9%%_E#&U!W!iF!*up(og56*W9MLRwy@e&T~^{nl-%|GvVDZ#iHcW3xE9!6r9L6PY>s8^D`m27pdzz z#b==*f}C=zy*gm{-gIc7H7|8U#v zV&a_O5wvr%bUq^rSKnWcU)+FCQfp(c{min|_AZ;)g=SsM(YYHTIj>}q)iOqFSGvI> z&r4sCErljEYr@>Xmd180sKRKux}!_}=|9^;f%Ya1Dx5cHZ_IOTSH(Y=jY|aXIZDCL z_Teu;)OKG|I``-}T23<$4)!jcrhUo@0_j+_Z=cv0TPpYR9?rBfeoWs=a<1!!ZNH;@nPJDjwC`KVXltWqP_#*6=Wc&`t82wf>ojl{@okoFxuw;*MWcuVR zs}9!(_ga!wPBS+@3_RL#nhGA>B zC)PjmmDIij*Uqs$qy+kmdgtL_5XgiC1Y!h%Kxi)~6JIZHKPe|KUyNjsr$=tekXxrT zL-bA6+Yvpeu_Gi2?_yosp~Z!oTtRhjcJv>M0JrGoa%vVLA^CT3N2$}) zKW|sFIE@?^WR4Ww*70%AapX&X94oOr??I8da>6M_X;|InO>g*9_m0r(1%c6v#z!E^ z#6mC`GROu^))GXjuqwnszS>`gUKcmC8d+Lb4N`Na;xYAM@vhFWwpd%>TVN^UTFm1-r1@usjxa4+YjjS%+-LSDF~;A0U& zB3V^bg|eDEeFUE1F#!C}7!F__iE!GUHa59Z^BG@13H5)sWxOTK3_O6ZhVgU!-&g!f vtq}E<~9K>z?40Fy8z6@M}=cx`Ouob6NM$QH)G-@hW2 znh!HoE2KN`mb#_Rn~3N*>el^a#n3=%fv`#N<-gwq5aB#0F*`fEvF`mc1D)rb?i2E( zJDo@Netn#bwR@GMQ9Rw(Ma$4NH4Wo_G#&2i*H@o+JY7q(VA>DH@l@^WPb$@aJ^0Vw zL!A7cj(?QOv^>Ey-PcE1Hrs8t({Q9FLE4IEYMLJz#K|Pc^0&#boz4;!^cO*saa$UO z)1Cy;R9|M;P3q)`2ZJb7ALDR7QPXUhBvIoa%ZHziqFJgR?DeDkSW%CbsKLH|D0Yv3 zlR|GF>@E7fiPS^7@mecNxr^i9i#I3zecdReZGX^yUJT!> zHoFK3<8iw9SDQrnxsD5Q zRv2O(?YD$wIMOaYkEgL(#k7~bE=Kq<$by5tB!1A6MGxsLC_WjnD`kFK!lLzC(OitJ zy?<)Bd~VS4I*PM*e_-yl?~BAM#djKVMH{YZ!_7uRJ0H(_P?8^%gfSS~W4J_xWszYr z5_?;6#YmiO$u%Q!w#9K?Y$~F0NO-LX}xu_uH{Qj!q>Q%+{tBRW{#a5*jU#SK0 zl2nQc%O|D2F1dPLa*Y!DxmG6Gs@7(!J%6+zfw$eo5h}o*Fn-p_|5{I1oW0aVr-j9rg)cc8%ka|C(#3Nb0JWH6b#IH;bX_l|1c2=s}FrE&gq5g~B zS6L8^Q~kqUd%(=s-72zEwoO~mnZibWky3F9ocmA16iGRcxx_&!H ziBBXKZi7S)N4G&(+|TUfw=c_7JHwpE*;5uL;pp~Www#u+6V8RSYLM7ODK$vEb~i}0 z+?MGXCJ&;=gXkfM6i9@cRW0U}hjD&*7NGOPmgo5Z*fQaD$Vnuj|38x@>6(BN$ zryzP1M2xXW%sEPYBDwJXcYi=^qLdmS-gsM6qG@}!D7o$Nc@ljjkzD;pC~@eG4@!VU zMnXwsB$Pyyc=X1{uON{LCl~EWMR><2oSf2zE_MLxP#V)?V(2;)B^D|B!fzI~Nf5^A z&Nx_Pc=Vi~y(Vp@QHvp%>+5n8%TCqBt@22njGitaO>bD7Ck;>E`B-v*E`^I!pE zU#VGsIVP#Py%#Mnp9=pnl8sbYgUPGaz@DLW4a>1hpyW5&4+UkQDdE{ay2lq+5m33(h?{EA1fBK47c6N@m+1ttq)|O43bs%gjW7(#rTjnOvVVyh>ujs82KIatDSgj% zOSW?a|4{T$;Il^CrC_4QH7S^=K`BKiFlQgQwMFEqj5r&{lTFo*! zzssk>`3gz`WOhGU%$CJ8Dg2iF0VJEKuGaF^+FZW2T+=Oc4v7+ri9pwY?NcIAnSHjLU`^aTtwDoj8mJnXKW(;WWS`OO2(5zZ99y{rM?|TgH+aV?r<8QlF_G-x1VZa|$h8rW4$L1(yyN1#=~1<$?8U!+%Dd*QoqC+pllb!gDD$Jv_x4R*16Q zSHFZy#-T$^JdDPrE*?gMT-NaMa2nu}(EyiKRvs<^a9L&M5#Z*5mM+r?atVeWCJNWE zsA1Ny4V!xY9WdXhwdcFo^!5~fbJz4r9#~3)CsU$P#G&RMM&nU)52FDxISnO~(@-)w z4Syw5R+)Ra1eDCG1RV+H9%$)uozUkZ5Y=O%fJ{b%WXgu!J+G1ZcOQ&2QePb?rLhgA zVN1UZN>KtO8busx?qM{ZpitH@_i!43vWB^b8xP&`kyYj%t`mT=%G@Kt+ygCLrW4%mxLH;)m`k8-J;-&gxZentJjDfMYlm$M(kGw@*|uRL5#O zPPH)3PuDVX-Majv|4XrZv;6z`7kS=aJnct?qdyog|C&FGrb8|LWBEG)PVqYdMv*KW zT+YWz%bsRxUssQ_{1YmQ)wTZPVEGFH+AN9UB+8zay%#@GFdqj8leqtWF{t@8`dnkK z-oFdqIlJ#d=ff|W{Gjb?dFjv6yt+j%#S3m&2Ya(%sLq097)?`ctOj|ZMk~JoPnN$q zuy~!tv&Ac0*Y4shizmjHY^PR3-c&p z>qG7O>R9(R#06y$O>+i^+C6aX%jmBm=K{&Cj2~*ZLKFw4$-`U7C^zx~n6$)snNcDC zE!QS5N}v(H1rS+k63cSUj3!J&VO}wDs1e7}P=Cdee=8Vo5o{4{G~~%>C`M;_1nJNe zqoG*?TktXjM$05`DX-YFnbkADpLC`j4$wrhk)%}gPYTGX2I87L| z;j8=yFm5XBvq{G#rK@N*L7%`_8%Sz}j;bgTpW7?Pd777B;a+jST4jk4P__+6cQi@q zDt}26p>De80yi@z1dFoYCJD%8bPS$vtt;wLQ??ap(dUgDZV z)ERt9S(P+Y9GrlV-Ye&fW9(>ToBPys&MHvaRPJHw1bev~dF7sf0 zyQHCzWu=!8Ln*5Y^suP(;$A8h-Y)Nm;eX0QBZ#R?&SjXhI9Z!}lk&aE_9gjVu4*N# zw9->5qb8Oc-|eThRoZe*m13mZyiMVWr@f-r@U*pUcoL<Ii6)A{!?@;rBdT>F{jjYw!4n>|)rwRTCkxVS!<;sfMj!?OrR=zjqq z2Wa3Kf$Iega4Lbo7`WI&=<4F@^!#!;pDvb{M`!cXX}#N0=^l?BV8bT9`}=6e@O@(c zKAHH2<6*n=Wb6>bLTJr1|A!3BwuB{YSv8!kHpLFwaS3TWK%SJ!Qsn^7KoO(_kucg} zSV1W|%4=PSe>!0K{K@3fqa=>2`hS_!H|lFdvm%9#-ruwhy9alAa3vcX-0Q*p9$fKd zi(^H=2JdNyxThUrr5!eSkH5IbU)PAzDMbe>L4m&(d&q_2p6*LsvImtKB%PFtN>^tb6mhFG$-Sj*Nym-)-)%RI08;M}JANmy-SY zl0*OnFKEeOd4~;OeN1|4`Tw!)xK+~;%fKJ&-irudmwC@H?w<%=OZLYz+F}xS*@J(z zV>mF#8sH?zG#$B!?A=jh(Q4&H_FeKq~SO6Xt{aZQ9fLf5Bcv#euy-7Rs{ zF}rqjf#*ix=j>fHGx>Fpet%kgMmLbN`uC@&6FYEj+6yZteUV?xk@DI1Z8n)5p%XZk zkpQ%~xfdn><>H{0i`irC5Wd!>Q~+li|qSA zBryQp$2}^m$@9x{B#vs5le;u0DVZ(_q2!U?-hb^zUi6=U>M-Qr#3Nvz`|=aif3uJw z$^r_w&=&Va2><~9L6c-8RsqJ7yCfnU-?~L8Tm%3BSrPyM6aWAK0000000000fPqjG zlW`LklMf{-0p623B{~66lXWE~1OjWon3JC+76CDnwk19Rg_HFqBL+7k00000(rRoR diff --git a/summary_results.json b/summary_results.json index d0b2c6eaf..c1dafbe8f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.000", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.89347, + "Performance_Result": 9.90597, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73748, + "Performance_Result": 1.72551, "Performance_Units": "Samples/s" }, { From f57399e6e2237ea7f6985a216ea098eb4ae86063 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Feb 2025 22:36:29 +0000 Subject: [PATCH 209/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9758 -> 9758 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 398040d65..836186adf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-26 22:32:41,985 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-26 22:32:41,985 submission_checker1.py:3270 INFO] --- -[2025-02-26 22:32:41,985 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-26 22:36:23,962 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-26 22:36:23,962 submission_checker1.py:3270 INFO] --- +[2025-02-26 22:36:23,962 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-26 22:32:41,985 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-26 22:36:23,962 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-26 22:32:41,985 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-26 22:36:23,962 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-26 22:32:41,985 submission_checker1.py:3286 INFO] --- -[2025-02-26 22:32:41,985 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-26 22:32:41,985 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-26 22:32:41,985 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-26 22:32:41,985 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-26 22:32:41,985 submission_checker1.py:3307 INFO] --- -[2025-02-26 22:32:41,985 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-26 22:32:42,961 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-26 22:36:23,962 submission_checker1.py:3286 INFO] --- +[2025-02-26 22:36:23,962 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-26 22:36:23,962 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-26 22:36:23,962 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-26 22:36:23,962 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-26 22:36:23,962 submission_checker1.py:3307 INFO] --- +[2025-02-26 22:36:23,962 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-26 22:36:25,004 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13056 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26456 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2986e5b80..cb93c35b0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -836 +837 diff --git a/summary.csv b/summary.csv index 892297b6e..607e4f445 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72551,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13056,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26456,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a46c359fb..6f8751c68 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.13056, + "Result":6.26456, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0fcad22717c0c2c3c6996079fe7dd3c50693237c..43035031ffea2a854deb8240541b90f1597dd0d6 100644 GIT binary patch delta 2550 zcmVudu9s=$0nnDtwJXMoIb>LP$ED zWOSa8V+ybaFk_os{{7fM2<*M=czSxq(K#>OP+ogiZR3wBmsdFW^>I4U?p2(G(X6eD zhM{X}7DR(^HfrlP*Pr)XT}x7bHt;9WOttkVmFT}7{^#H!ihobWN~KzsV3xFh^>Lcc z_nXZm7^|tDG@`khWk-flJoVG;Z9Hlw^H}+VWzclel!jq9r+zroR~h!>DmkLzFbvek zC|FF@EL|mu)x=M;;V0v8p6G`MgD^Xm*Q3R1*w&B4{>g7r=*`1}W#6}i;^RYG84Oh<%T2@DYe)jGUTL;bi zJn>rborYY~h8x;&yVcOl#lq7HCenQm2!pYky2lmT)!^4;S&0} zmL}P$)`Ha@SuBCO+r=?|SCCa8((fVOJI>=j=97p z=l+CCTBP2OKtk&M1SF*1&s^e?EMJ}_;4ASf=z&}#rzPQ?=VQ-*Q`8x4=<*Icayrjx zOpnLVIp+eTB_F}%%eaS!U;lgg>8g9$JA1!e4JFxfXcVU7#a$zarp@Uj`*)fxb+T7; z6%YI2Y^dUF!KV7DO45F^xSNJazV=D__g2Gbrm<4XRaP@!Of}EDC+qlFCD|G#X|5MD zdz;MtU_Rdq=8L_5Xf}I{7qc{5P3^5!{UDkR!;$`rK2WJ2P7?jYL30S^>u+_Gl&xA- zHnin`1>^d@8kbnDF&d^6!YG)#9Sohrp(B^tF_+lnN(5Kg|4&$h!!eO)kk@|}O~P$GE@UB6v$iAN-VmrjjDj>dgI$nR(NvfGzc zs=ZOh;(%U%?c#eQA!08Z=5X>O{dXvjTQ#c#UQ#IL~{KcbBRr^ zpA#-&AOZ;m5lASAKte%uOF-n|48)S~4hW|px+Nfj!BY@j3L=j&Nz4V8ctmpP{_lWT zM=2FRymfbfro@)zTB6{#$HOFgoJ4Z5|Liqs9pzO3@XFf) z&}q0WLl*ES3=H71oe)SSmt&xD$UshjhB1gV6hfqbp%em*ODDvQ262cIP!P)!bT$V! zD6*C==)@y=&RJ9sMA3c53!o7O`9LnW#{V{me3=My8V5?vv+FTVRR3Ny+-xrV%S_f% zVFf6!*8_W|(lJciDxebYv>yq+vu;_GO0IDPr4sDWGb(|GQ3*7RO0MBliZUue0!C#? zf{vhn&l1j9OBZzFNlGP+>Y*q)Ng@rMCAWtE7NOjlB6BLG(GWj?WgRtEnJd==6YCw| zh9gZ;fF*v`bR_uW8_T9xa*ZP?mR!TJ1R91V&@e2yhGQwqumlMhmL&-~n}aV{(v~jh zgkwpgdLWAJp#qIC1m#lxA8=VmjaBwl*8_WhzKfKe>o^7bIRZZwJrew3qvcRAxyETx zFu8_<2{a5$pkZKg4F|JT1|~?rz${77*&KWUleTnGC3OFWMfE@w2NOKX=3utoG?-*N z=dUwaM|qW)Z0#^9OiMaW0h9Ro{gL3$#4Y+v8)zJpGRZZZNuXg&0u5u5Yg~&mX_hg6 z2|A&htz`*1n}aV*(v~jh#A{I|X;cqIF($c281(t{*7$2-a&ebUg^M+m1jzJ$x|}WZ zX;OF%`2$GSQC+3wtGT^=%~r3LRkVC9@R$g08?Zb|B-c0=C6a46kwC+U1R6#pFvi%F zNUMxU&q@}Jm!t);+9Oj=t^%2p%$ zvMjI!hyo*y05%)y!~q(IT5*7eGg-lmgEWjuV8>xhmici|CydE5M~+~Y9M;kWop2_Z zD+fgJl`GePm1~{OoPPz%x9Zao#Hb^@R*}h8>^OCd7b<}dhgx-j#-Uyvpy5=LbPpH- zSd)zp9tc`&+ji`Rv`v$<4=Vx2liv>>1C`izk`POOw!0^HoI5SIV9Q4um`us7A~v=6 z0F6uSJwU_AL>fva(oiyyhLS1E>^&#}C9^C+M>2a4Yw4m+=z|d!)dNwCOrYUp%9`yx zuaWt8pN!N}Ulk~&u?wYPNv{Y>Q2-@(irCcM12nFnP*!mFAPs}Eg1ZNehwk~vGItN^ zgh5$<=I)Wq-NRbCpcBueP%?K9iegY!aQDcX-94`{nGSOt#Sh(8)>2!Q)vMfg^<)bG z+psB)&8@$1pQ?DIPSj+QXhD>nu2tlwb@gZeS7QHm^%wFl@~ppTHVAW1zdu?1J%1X` zMq2X6>X!oS{FefZJXtWjT1=FdKFw8ISC8|5>=Vk1)wRLnaP=Dj+B^=UI82{bz2`qt zu$cIV(`fL1IjF@f{9~cszYE^k`|kq#!!MfrptZHE^k->S-LjYb1sc}j!Q3CIb3Y!1 zvqYPyVOFTo$Zo*n)vpdLU#HQ0`O4C@yC_Yg>GQj>@&_u;-?emY7)9yxoBWh)e!bxW z@K3YL5q}N@IH!(dlTIKke{M|?wgYX+sewXMT5=Q8CK0uMkkmSP`zlW2B;ZqztC{_F zc2PassV?~i=cUmNDzY3Upt%uJZyNNmoTrZ{@quZ827Avmd`KC zIciBx9@3noq*@X}7|V*1`_*COP5%X`PDB1pJOhroZ$Cl%2eXhN$^rylj`Di5UL-vM M1URRTW0RjHe~%u^LI3~& delta 2552 zcmVn=c;z`ZS#=4>C@|Xx3Im zQ&$x^3!*_d8@1Kjo6ikbRg%=74g5(olWp}`ChD(8|2ce&;@^|8l&O*>m?dq0b)2U2 z15HbUv7GuzGn&g;c4QdEQ$Nk##v?76$I>4xgQgSB&~;mz`r%AnWjKhd$ki)ADgyrxZX4SQtIoHo7W|`TtYwB z(je-j5(3>i9yb84@k(_y#x|+cCUbhoZW9g(!!UUjW0KwC9p^_CtPBa zbAQSuCaL!$kdS&m0ST%1Gncp|%a@G=d?kJbJ&oW>%~lC zo4Fs%=Z#>#XhgGr*;BljrP*q#u~zkiXf_N->M!a*rhYg{)DMT+5X{%#>L@8&waRR0 z%l`_-^?fxiv07s^TZRyN!QAa&=o}6mxztX$#3onjDVH$lfP_K^BosOzAz$HNoD%4G zxJ#ic3GaY#@{N4H8&2x$g29vT{0oN?$z$mH?UGA8BDr#ZY9w+r?)yQ0Khwx=UskCa zqm0M-a~j3Lxc^`@?WW!c7Q$#%AhC{8Dv)^V?2y=Unx?Cp7(^F?=yDLr^>e}{Ho1OI zxrBiTBoss-p&$YY1<@@5k%u!7OTs%KoPy|lXKe8mL=$H4{lLp zEnU!wNAg^-s2+%-`;3=BBXshCTJTyRDuMI%8~>f zL7ydmoU@iL=)^N9l{Bh{qUa=vG<25S8U9;@a%YOnsWkMa_yH{IsIkgixgMBU?*KO) zqa_Nk#Lt?J1%G^F*%V8zaSVzj*KjO>hG7Xb3`?%zSQ=$mf&>i9k_4UY!51uPOBZy) zv7}Kw5XJUTfkxgK3t52@)_cOA>Up2VcOXEnQRz-M?W`JrKpg1dp;gnC6=XlWgbw zbtda5uM(5y9+RSF8IDuHBz}H>Eci2Vi$2o^8mC2>k}KjIk+^ zRvD3?6Gmi7g3k8f3z4*?3p!yNuPmy6hoTsfK*R1Ttv3YZKe78-OM#V`w04*@%x3mw zSzrke1x6eJY&O)112hh`;s6b2vVs{0X&95hj>DKN^W&gS7?Wj=9KkF(tfdP&;Y>1D z4v69_SFQmo*E*d!{|c0E)u$tfQAc=Yk;zuTd?J0=$K4{TSaVY z?*ST@+IxV8k%=^vOr)V?A`KIPf$(`D=3r|+&xIcpse8TLF1u&K1P|l z2X(@re=KwN7|h+nTDqVUuSKC`?j97ypse8TF=}@AyvAfY%yASybXQqRZBxwA{=$7Kd-lS!fkQFgjkk=xeQpZ#BpgS*vV$iK+5{-W6+%su`7WcBy_ zX*e4x$sem<3b6BE3efXp!SH%9kxKeJmu*!(f6cQ`C@)r322aD)Zv-gwIE>;jeO~pR z|46}N;vY?;!TaT)7PIh=g?#@mcxNBH3+xZSD8>h+tz@Ndq*-;#Uh)@cSVxC*e@Jp?WjB0gqR|IOI7ZpG>xX4cVp=fWSqYE=y-O18h= z190>wv&#{G4g|?g$KR7qAS{1rLlL$EZON&DLQ`6D6VfITwSJJ)I)3}gPU0lsQ;(~e z{dRUyxj3jM`32{t(G@DP93`N+5mN6e^s!#1k0|khX~7z!p+X&aw5ab&ZYeho-kh=E zf&`yLkoJ_@3hhI%6l2f#pqS61Lq~CIoMIuCZig-7-)sk%132X;8wTadsbuIZOj^(4=stKk8r!zb+Sy=@`3wBD zdVL>OO{MNn#R1i&;FLRH!MM7_*F`*B!+co96K^Gmr!MLfZ>L3$eISw;fF9$XmCxDp z%X*1wl9PutCn=e)385G>Lhs3@A9>S%0jkrGe-qDuWA58eQ2zn5kRi$f1hEGwShHRv OJplyCPRHMqpCy05Gtu$@ diff --git a/summary_results.json b/summary_results.json index c1dafbe8f..71da5d25c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.13056, + "Performance_Result": 6.26456, "Performance_Units": "Samples/s" }, { From 988370c0658e67b8eb98d345ff83723ecc17ac6d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Feb 2025 01:18:46 +0000 Subject: [PATCH 210/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9758 -> 9757 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 836186adf..de8a2c3da 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-26 22:36:23,962 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-26 22:36:23,962 submission_checker1.py:3270 INFO] --- -[2025-02-26 22:36:23,962 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-27 01:18:41,140 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-27 01:18:41,140 submission_checker1.py:3270 INFO] --- +[2025-02-27 01:18:41,140 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-26 22:36:23,962 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-27 01:18:41,140 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-26 22:36:23,962 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-27 01:18:41,140 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-26 22:36:23,962 submission_checker1.py:3286 INFO] --- -[2025-02-26 22:36:23,962 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-26 22:36:23,962 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-26 22:36:23,962 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-26 22:36:23,962 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-26 22:36:23,962 submission_checker1.py:3307 INFO] --- -[2025-02-26 22:36:23,962 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-26 22:36:25,004 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-27 01:18:41,140 submission_checker1.py:3286 INFO] --- +[2025-02-27 01:18:41,140 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-27 01:18:41,140 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-27 01:18:41,140 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-27 01:18:41,140 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-27 01:18:41,140 submission_checker1.py:3307 INFO] --- +[2025-02-27 01:18:41,140 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-27 01:18:42,342 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.90597 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1764 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.427381 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cb93c35b0..1cc5be4f6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -837 +838 diff --git a/summary.csv b/summary.csv index 607e4f445..ec8e78587 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.90597,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1764,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.427381,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6f8751c68..40bfa2c52 100644 --- a/summary.json +++ b/summary.json @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.90597, - "Accuracy":"acc: 75.200", + "Result":10.1764, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 43035031ffea2a854deb8240541b90f1597dd0d6..d54902f5ee13516b29873f2d3d57b9fd643cdf1c 100644 GIT binary patch delta 893 zcmV-@1A_dXOr1=yV-p3_>>&$bvuG2V0)OFJYB+%Bpai4@k>s>Pu!2%_wAZ>2|8&6e z`IE(^M@bx4^)sn&)Yph+MG76WzZn~L5AO8fN;NjP*Ms{#xZ=we$4USjyk{KZo^gnk zaoFHJ{^A~gagV>a$6wszFIN18XbtW8)l4HjOT#U)42$AnY_UAwlyWX487MQWuYZ=h z7`mqMTAjvWg^6wURNb5HzMyP-I5HYCzB}As8L7G#9i_xxO7`bV5&;yvpe2Xp9X5RR zG3n{$|HroDR!v7N1AnZ0FCu(hY;9VHsH>Mxh34a}IBCd&WN9y__*{F3aue&XdI(FBNDe&A1{G7at zW+uN5(oc)e=muC`|NiuJVh7GmTd~TeFY=2yQa<~>%_g%WbOOgR6Mz<1H^{aEB@#Qu zp$6%(G0?U#_MAmcVYO8r2Nh}tl_HN(le+G`Q%zlVG(SLgU^s!YAw8|q#D5@;zq6&g zhU>cC&MHj|$G7CJeKI{?o=(pt;wpR9_;fa3UXCvo^?vQjt=3ZuO(%jqKs+kv)Z z)j**sE!_!e;)q%gB(+ZdzOtP-31rpf=-xSVXHdD=sW$lq=cUmVB3X_S(A)^An+m;Ydka^sM{#oRyqt_(*Ja&ou*UoZf39BMhE-Fk`%`g1btyRI4p=a*F7bI057#mu zR`J9e3F2`d^($|OMaDi5Nen>8xJTt<_Vm0ZUZT3li@P*0(&AyA6H4avmVEUiFZxeF zbr|w*;t??BzWfCBAG44lO#=nf>>&$blj0;-0qB!DB{~6GlXWE~1plT~N|T=@76Chx Twk19RkCXK!BL+bu00000T-l|t delta 893 zcmV-@1A_dWOrA`zV-p3sg^*QZvuG2V0)HXj4<#JHb5H_Of=F`OAy`2vI@)Vph<`d@ z`TWV^(xW7ftNNMLH|lFdvm%9#+24!}y9alAaHSd>-0Q*p9$fKdi(@5#4c;>janCr! z$~bKB9)EFOX{zXOl-M zkzWBmaXdfp0@twy10|i?{aUtyx?dm53;P`_`GcSGNZ=?zeXv%a*|lbJE=K@ z6aWAK2mk;8KL8<}UL8jQ0JHrUOb7%xr;cNjO&}kC&1%Ci5QOi23ZuI%TW(DewgYX+ zsewXMT5=Q8CK0uMkkmSP`zlW2B;ZqztC{_Fc2PassV?~i=cUmNDzY3Upt%uJZyNNm zoTrZ{@quZ827Avmd`IG%Q Date: Thu, 27 Feb 2025 01:23:41 +0000 Subject: [PATCH 211/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9757 -> 9757 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index de8a2c3da..ae1bd7fcf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-27 01:18:41,140 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-27 01:18:41,140 submission_checker1.py:3270 INFO] --- -[2025-02-27 01:18:41,140 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-27 01:23:36,061 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-27 01:23:36,061 submission_checker1.py:3270 INFO] --- +[2025-02-27 01:23:36,061 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-27 01:18:41,140 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-27 01:23:36,061 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-27 01:18:41,140 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-27 01:23:36,061 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-27 01:18:41,140 submission_checker1.py:3286 INFO] --- -[2025-02-27 01:18:41,140 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-27 01:18:41,140 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-27 01:18:41,140 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-27 01:18:41,140 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-27 01:18:41,140 submission_checker1.py:3307 INFO] --- -[2025-02-27 01:18:41,140 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-27 01:18:42,342 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-27 01:23:36,061 submission_checker1.py:3286 INFO] --- +[2025-02-27 01:23:36,061 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-27 01:23:36,061 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-27 01:23:36,061 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-27 01:23:36,061 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-27 01:23:36,061 submission_checker1.py:3307 INFO] --- +[2025-02-27 01:23:36,061 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-27 01:23:37,021 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26456 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1cc5be4f6..097e2a6bc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -838 +839 diff --git a/summary.csv b/summary.csv index ec8e78587..a33d40c4e 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72551,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26456,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 40bfa2c52..699d7762e 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26456, + "Result":6.19705, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d54902f5ee13516b29873f2d3d57b9fd643cdf1c..ebf70ba84087f6bdae4d5d67f156944724eea255 100644 GIT binary patch delta 2729 zcmV;a3Rd-{x^A(T?i+CbKNcJUu<*)HyHRAg?WH3x7zKRyh3iX*y9JWSoT2tfPvi zt}1dCM1ycP>ZrFjpBs**WRJL%?YmGuCOfZ{JeB(> z`aOShHt48&F4dU!^RgRPu`-Zo%n*0Dkt+i15b+PYDe65eJ5m4NH)he#?vVzG>@%>rH`@<&lqn-osJrPBhu&EgG`DOa>zlIrV{o7W|`TtYwB z(n|KSHDR{LCab{NZ{mc1D@Zp_xk3;YJ^~53e?9>T8TijYLcsq5Bm}yzKtiB90}^tZ zb%BI@IOjk@2G;`;GP)OBVvy180SOu1OCTYmdj%w9biV;f8{cjgzTGTVfk~1%;S!6C z{VA8UNWLF|gyj1PNJze)xx^v0eA!6ASK?RD0=Y-dO2WIFYnv&5>YNsMyaAUC=LL=F z@fbSiT!L!JM{xDh@8Qwc|6YB%_Rf0e@0Y8gBwG%R!gRd2ZwAp+n@+NSr`b{`do`Ew zupiEbGR_ukvY*N%?I(--X_(||pQQg_HgzqHrChGEw0tq8U3jPK_*f>{8Ya=!ibGMD5vpIC+Ry*Mmi`=QFT*9CO5(*uVQ0Rb!e1(6pOQ7T7 z4u!5HybHp~H}d&zILXfgohRS<7d9o5$I$iLC6~BFa%IdYq(ohP2fySW|;#Pw=L={jF%PQz@FK$s} zB_3$RCG}jes2+%-`;3=BBXshCTW1PtTgJ?S0T=JTyRDuc^l_eE) z1wBiDIAHb@Ua%YOnsWkMa_yH{I$g#>?x$c-)?*KP# zqb&-s#Ai*%f*;?Q7R8clY=dISH5^NzVORnU!;)(_mPQ$tpaOl20VN7z3V^SvDWlVy9 zM(Ad1Sq0tg#TO=Ni3b{STa-x})k9H?Nv;t(J)hp`e=STd?z5?Iv4)ZWnLbRHvt>R_ z3b$$e0FrfNS84g$-d(g5V=PLf zSwl1L$6-vC`EgJqjL9-bj$oD?R^ow1IFrnk z1EToKm21GtwN7WwzXIi3_2~#=)Ddp0$Yd+_oI3gomB5EXtvWzsQ?Cxta4IW*xOI?* zQ3*6`?OJBoK@~76%Pc#BI%>{Yi3b|try|Um1ETn;2-kp9k(%8(oXVenEb>NOJ2zt2 zwqs?>b6YPu7I6x&1g;!v+yNS!I(LAEV_CtwgES0FpkY{+*>_L{49hYDkD&gVb5`Pk zM)>|K^XZ@{Hdz76a!p4@S6x1544A4A7v8r&*kQF{;2IMm()G>lB7p=2TrB@<~VnMRp^y$4l5$t>S5LM8uyl*!sO|iP`&7mwc`7H9Ljt6vJR@?Q$j^U8wZ^F4g`=e{!5chAS-`AXj+8rKwGkEpiq~V z?1VINM6CysS|@*B*-o4Uvg&el@0__as9f%47yW{DLaPd8Ns1y+Omm^y3Vk%I_z^|U zQ^jeg6;$W|4lV1uk{QCZg*U4WSTDdu0i+yZrb0XK4Z+y49Z2eu;7~!_YAdM^rERgH z?3=cL^E6#x37&s)>M0(8v6+eJh~#WWnjhPZ0M0OUAVGPDvn0c3uLmpL&4y9Aa7y71 z25wqU;_Ti#F&q27Px@lChWr`+TED)JtHweNr(%HWk~6|AP_J!W;>#i&uA|Pc!-=;7 z_|qZiH{OnmOntzk5P(c^Ps(TU{L-vYou~PIoaS-<(4-j|$qP~}Hp9rP{xeV>hy0s( j0!+EDKSBKmvyUOl0t7q$H+i#NBs~EHkTCvBlb$7i47(uv delta 2729 zcmV;a3Rd-6_0#NaJZdKMSowox&~(z2hG93SemK)t8TR8UIile(4AjRc zSWML{T_uUt#80!~C*yFQ=!XY`Fguplqs402){n&g$!}8V&BKFb-?yQ9NVZ;Uc`0{M z^n3p1bkNp~T-s#X&&y$4#@axC4gJL=y^0>bs&F(;v(skfAwe`rmj7zgFgur0*QWks zTW{&wLpVssd17rG4hAYe(Yh8ak~EtBwmKx%309%PUdIYUY@)q}Fb!K;`RDO8QR|rI zs@LTRKl-VEco0VqTD0!YZ&y#x|+cE16Mi7z)BUv4%_V3A^uxx^;t z{)9_fq~4D}LhAhlB&6QYT;h@}U!Eo4EAcDnfm|b}CE=atW6x86)ERB)@(w(5I?riL zkH^qC=K`c9AHn6zxQB;d|9koAs(acyd%s)_CE0Rl6sF_FT_cF5&FLijcbYAAvR88z z5BuS4sN!tFruwN$(tfhIn}$ig_DTBpR>Nqfu~N%bRx@8rHP5>z>-bnD*%~Hkt`{?V zo6P-SKHm%Gi@j)nHhYX0vou>x?X6Y)Aes%sk^YN5P^lkI68*zLa|q_^Z*`QEty)z! zwB>&Ve1t9gtAyfP_K^B;+gni&Fv} z4|geaCE*J#FFq12&W*rB_M*qQxIJWB9Adi%mtTtL~`l=?|@iG zDHTAxb$6zJ#Fph+qTsg2!z6l~L~`{XbBRrFe7J;>2qct5AfY63iA!&Myb=;YIJsy~ zOTs%I;pB`qba@AC9ZF+*Acn3(frNYnmtMWFP5mHB_9p%^!=r2e>@{f}oHDL|6VlQY%cuEOx99i z1t_oA1AC^@F-+Skpc3!29|^v*ZdsH{u5kpV670}3DuISk2{epKuHjUQGAcm=MrBEY zj-bzf63$pl7j)uDN+pfzp(r{@A`P7-w}$@~q1>7xb1J3L5I=xr9W_>&E7t=P>mA^R zBTZ3&C4Sa)B>3YS%cfXzjUy?RT*I*h8ipm%Ff6%-V=2q91PK_HB?&s4gD+UpmM-Xo zV@acWAd2mw0*x>P71a9KxJn7E&5CwXdII=$u*owpkYh`4P%mPT#GVkmN5x` zI-#4bWeGZ)gD*_dmM-YTYf&a?R1ZZlCb>o!^!fDG_-kQuahFYni#3!4$n<`?oGtTd zQg{vd14!0UU8Uu#xxIYNRl504TK*NXx8b%~A#@Lid ztBgp{2_v#3L1%ODg-F`c1)Z>sR~FTOLs5)Kpka5F)*AxypV)n^rNBx|T3bxYRwMhe zEU*NK0waz9HXG{10UC!|ae#(1S;35hG>l1L$6-vC`EgJujL9-bj$oD?*3t!?a3+~6 z2So9eE7yRPYn{%Ve+A07>eCU#s3W{qk;zu{XOBZy)pNcSN4v6AUMYsk$6{*>s!>RoFk44_7Yv)>Q z+ji`Rv`wqP5iZ;UEP*SB8h3!kq0Swk;aFDi?jQ}r5@;BfW%eDEfMHo?;1SecbH-Y_ zpcB6T%6vK~icOY4!xpeLTXkN4qw?qMzrIlu&z0DA@#KqGA&RzNy#g+QM~B*YfX1Od z9-!e|R&eql4dW7M7?)*c9+ZG_S!U-E%+JGGx}X!zCA0K^D87j08nB41+1B&#fcZwv zJ>SK)yC-*?J1w_h%SRfROv$YxHnsNvjZ5u4K*PvH8cHV8P%@E*k}1o7>^&#}C9^C+ zM>2a4Yw4m+=z|d!)dNwCOrYUp%9`yxuaWt8pN!N}Ulk~&u?wYPNv{Y>Q2-@(irCcM z12nFnP*!mFAPs}Eg1ZNehwk~vGItN^gh5&6?vc#h!&nu2tlwb@gZe zS7QHm^%wFl@~ppTHVAW1zdu?1J%1X`Mq2X6>X!oS{FefZJXtWjT1=FdKFw8ISC8}T z6UvL#wZY?X^&0`&JPxBcOrKW0=RZ=gnD~d&Xz+eHsKqS&W1-$yzYE^k`|kq#!!Mfr zptZHE^k->S-LjYb1sc}j!Q3CIb3Y!1vqYPyVOFTo$Zo*n)vpdLU#HQ0`O4C@yC_Yg z>GQj>@&_u;-?emY7)9yxoBWh)e!b!Fv&s>F4g~+ER7#UgAS-{%=_3f+fwpATK%pru z-3e*ph*}RMwNC!NvYj{yWYy*9-Z^t;P`TKtHu(kTrO_24S&kCW+z6?g3catF=^Q0K zFfCYXG*oC09xdvdl3U7+gI8xPxFEqN5u`okwnEzwEXCOKEhy%*=+IHz7^hf>rE9Qd z{F^m^i#)%_3IcxwMvyUOl0t8-;@_MsfBs~EH|E5$*lb$7iEb=K& diff --git a/summary_results.json b/summary_results.json index 7b310aeea..1d60425d8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.26456, + "Performance_Result": 6.19705, "Performance_Units": "Samples/s" }, { From 1b5ff914ad699e57fae85b5659d13261e7915047 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Feb 2025 07:56:47 +0000 Subject: [PATCH 212/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9757 -> 9758 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ae1bd7fcf..27a6edc05 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-27 01:23:36,061 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-27 01:23:36,061 submission_checker1.py:3270 INFO] --- -[2025-02-27 01:23:36,061 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-27 07:56:41,796 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-27 07:56:41,796 submission_checker1.py:3270 INFO] --- +[2025-02-27 07:56:41,796 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-27 01:23:36,061 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-27 07:56:41,796 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-27 01:23:36,061 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-27 07:56:41,796 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-27 01:23:36,061 submission_checker1.py:3286 INFO] --- -[2025-02-27 01:23:36,061 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-27 01:23:36,061 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-27 01:23:36,061 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-27 01:23:36,061 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-27 01:23:36,061 submission_checker1.py:3307 INFO] --- -[2025-02-27 01:23:36,061 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-27 01:23:37,021 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-27 07:56:41,796 submission_checker1.py:3286 INFO] --- +[2025-02-27 07:56:41,796 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-27 07:56:41,796 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-27 07:56:41,796 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-27 07:56:41,796 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-27 07:56:41,796 submission_checker1.py:3307 INFO] --- +[2025-02-27 07:56:41,796 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-27 07:56:43,251 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.145277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1764 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2254 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.427381 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 097e2a6bc..4ba0ae72c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -839 +840 diff --git a/summary.csv b/summary.csv index a33d40c4e..aff0aad48 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.145277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.289104,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72551,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1764,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2254,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.427381,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 699d7762e..954579b78 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.145277, + "Result":0.289104, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -309,7 +309,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1764, + "Result":10.2254, "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ebf70ba84087f6bdae4d5d67f156944724eea255..f0ca08b94324a5ea8efcda0aae849aa6cd23c6da 100644 GIT binary patch delta 1267 zcmV0#Ukj6q z`)n#)tf3@8rVrEQY?)7!!fhHqfMgxnRa(BbcbBi(>eV!hmahdK6Txi*rb~(Be;V7Q zL~;!$5@;BaK*NXx#u$qdX_gTQ8ev41RM6dCd?AvSc%Tus@yeolD2fpYH0-I;d_zF~ z6T7dqBv^?_bB9Tz)y%#u3oHSmz=$J&&4xO0fX1d)9H8M$Rxslr4Pz46aTt?jejL;Y zW3tSVBbX(Jm3W{L&Lnf?fGEClf8`pma;?*u^RGbpR((2x7f8WKfyC-*?+bySH%g4|$nFhCtSk&GFG!C`*01YD(X(*XU zL&-!MN~Td}??Dw%GRrFH8qD6qN<7pEJs4q8JrKpn1R74JQM0}0H8TJ1$w)2vRe{pb z_o38H!!3eR6hO(HA{KS`0F5Ikloi}PNW-A4;O;^Fp?f|?nY#xyf5M>X!nn{Fegs zys}_;y_iTPeV)sXe=48m*(a1It15%1;p#U6lzALRahN`@TF-x^U@`HJrqSU2vQvv$ z_{T!Ne;2&74&DXUhhG%qgVIs5)Hl*ByJai+3)HQn!?{0_7k)enXNfYA!z@v~nLU8V zt6v>hzD}e0@|CG7_feWg)6KiF^anD|-?da_7)9ykO+F;s1z&GC`V+Iu5q=H@7CS^b zCX-Ge9)HbF!!Qtp?>t51d*UReEwYkS0@M?gkO%^Cv+Qka5&y_GqP=)GRY4=8e;Doz`%phO)ww5sk3W(d<3UadA@ zy#N;lkaC2X679V=1Y^hcAgN1&Lj`fGt)xDbc7KNrW#4oMI8W0Bmf$I;p5h)Dn~{i4 zNX|y2`LWFa;0!|p5|np1OEP@%da%+>HVn#@QwrZ1n6#e7(S2}YG`4MuZwWFhC06qC*BI+k6q9w-cE}g`+!Fw06E4zD<89` z=O%iMsyxl-ahk{ZvQC#|z99GcS3mNm{{ob!A^#?x0mt08pP>2!vyUOl0t917^}n-S dBs~EI7CS^bCX=5fQ~{8a^d%z(LL&eG001K_VB`P* delta 1266 zcmV{x^BET?p_6e?2Be+ca#ufJuCQe=PW!xJl2nfyQoACb@<) z2{epJpkYjMjblR_3b$$e0FrfNS84g$-d(l1L$6-vC`EgJq zjL9-bj$oD?R^ow1IFrnk1EToKf0b*%%C$~s&c6cXTlMJ(V$>0CtH@+4_MAHU3zfi! zL#;YMV^gmV&~Pd%xOI?*Q3*6`?OJBoK@~76%Pc#BI%>{Yi3b|try|Um1ETn;2-kp9 zk(%8(oXVenEb>NOJ2zt2wqs?>b6YPu7I6x&1g;!v+yNS!I(LAEV_Ctwe}gm(OQ2y` zmf3eu1q{nF1COBonsZj-fkycLEA#1~C^lIF4O_t0Y}I*<%Ad3U`bJGW*J9VjlP_Y0 zDB6B?3%CRx9ctqN8k_offQEBf!O4R(j7y+lT$Y)6Pz8+3GCPl8ejZlhfkrr&%+dp* z_#&2Tz#_J0ThG4(<{LHle|#6a?w;InZnvC*EgwV2WE$KmVo`e!&^XlI12l|Gq@iRY z4J8w4D49l?y$4l5$t>S5LM8uyl*!sO|iP`&7mw zc`7H9Ljt6vJR@?Q$j z^U8wZ^tf~y2hO6HQQ08$M#bNrqYCZpvg2lu?nnr{7%T6t3 z;U5e6{$23SI(QdYAAV7c4@yVLQr}3k?3S(MFHpCR4(I+zUik4SoF&Rc4zon{X7&Ic zuYPr4`8ti}%U7nV+(&5|O*ikx(jUk;f7eo#VHBmCH~Elk1%JKa=uflD5q=H@kTCvB zC6i7d9)HVf!!Qs<_x%c^y)8dzT7>ODTe51PP?whMgfwwPtp}1?Cx2hrPMid?>T-1N zoVhcoT<&ET{epEus|saFiXu=E$h3I8N#%MH>(X; zFTh0sq#R+ULObsb!Pv1KNa~W{P(j>kE2$5qZGW+$?3=cL^E6#x37&H5DIS2anThC# zB*SN~2P@sphEch2O5qO%Zdy;`?A|*u8~eUb`eL+({2BgQ zzrK&F#zGCJVu0$BGr}xTuWeo8%OV`Eqt36xiMImy(;?_L-j0h*eZZp-fJ|{u%4hNX z(j}}=ou~PIoaS-<(4-m33sNjL!^o@tGf*Cf{F`_JOu4T=LH!4_kRi$f1UvpWd9z+5 cJpl!fF#by=lb|J30gsdQB_jqwBLDyZ0D{U}ssI20 diff --git a/summary_results.json b/summary_results.json index 1d60425d8..6d3972671 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1764, + "Performance_Result": 10.2254, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.145277, + "Performance_Result": 0.289104, "Performance_Units": "Samples/s" }, { From c082e572f3cd2906faebabc894fc4dbdd0eca47c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Feb 2025 08:03:23 +0000 Subject: [PATCH 213/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9758 -> 9755 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 27a6edc05..da5c6c21e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-27 07:56:41,796 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-27 07:56:41,796 submission_checker1.py:3270 INFO] --- -[2025-02-27 07:56:41,796 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-27 08:03:18,213 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-27 08:03:18,213 submission_checker1.py:3270 INFO] --- +[2025-02-27 08:03:18,213 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-27 07:56:41,796 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-27 08:03:18,213 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-27 07:56:41,796 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-27 08:03:18,213 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-27 07:56:41,796 submission_checker1.py:3286 INFO] --- -[2025-02-27 07:56:41,796 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-27 07:56:41,796 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-27 07:56:41,796 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-27 07:56:41,796 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-27 07:56:41,796 submission_checker1.py:3307 INFO] --- -[2025-02-27 07:56:41,796 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-27 07:56:43,251 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-27 08:03:18,213 submission_checker1.py:3286 INFO] --- +[2025-02-27 08:03:18,213 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-27 08:03:18,213 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-27 08:03:18,213 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-27 08:03:18,213 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-27 08:03:18,213 submission_checker1.py:3307 INFO] --- +[2025-02-27 08:03:18,213 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-27 08:03:19,246 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,12 +31,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2254 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.119 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.427381 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1028 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 4ba0ae72c..0b6746cd3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -840 +841 diff --git a/summary.csv b/summary.csv index aff0aad48..5774467b0 100644 --- a/summary.csv +++ b/summary.csv @@ -9,12 +9,12 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2254,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.119,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.427381,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405291,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1028,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0414,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 954579b78..79d60e65f 100644 --- a/summary.json +++ b/summary.json @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2254, - "Accuracy":"acc: 75.600", + "Result":10.119, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1028, + "Result":23.0414, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f0ca08b94324a5ea8efcda0aae849aa6cd23c6da..696868a3760d499aaf05ea379b119ec84f93e0d3 100644 GIT binary patch delta 1657 zcmV-<28Q{bOq)!wSqKHlc%d^%vt9_Y27lDsqG*;ZFyaVcv!PBLps}eH2WU8x70ftD z!zAd0VCxdyCU>vZP)D^R{wpN=3#9pSc$ zOtxarsiVJ834A!zssl7O_38i(r?P@u2Wc3UK*QFqWriJ80i&|avLmRY=A4yypnnm5 zD#DyOAc~)ga1A&Wso9;wsr>oJB5%~Sb0c8K8B9TG`Lm7qV^u3 zaj3lqXc(DDL&-!MN+!}!GL15O52}EYSyn;UVD=tX;-N<9!3c}$fha~M(0_0;jhgK} zuaWt8Pey9VuL_igz7M5t8g3Dkq5w+n6tSqg2WT8Yp{(HUK^g{S1$PhX58d-I%G^Du z5e8+MyT@Se9#-OkM%*@qlDT_O6oayYyT_>6-SZlg=`hDp{LoWnExA=$y&Ai&o@@bN z=@!LN+xZLksf;tzZM5~tG|$ck!S5ivq6}9`u)l3@A=bk zHd2y5R=*Tr<-Zi5=amJ+>%~MW>GNE6RQWW|KA}8WRT(@DSHBUU%;PYM!}NL8dj2B? zi-~_UjRxymm;nX+U=k`|vuG2U0)LXxQo{i}2PGgSh$N>Sf)$jaqrKLJ_@@Jw&z~$VJxb!Z zs-H=HqrOHoD^lo~{mt00dvK=*SE{kWy&l}}!4+S&I93AK;639I_l!fVjKc=+@fY{_ zi+lXVJ^tbzf3e~(L~CfzuVxzQSsHGUWmpssV~gearj&Cb$v~M|eSfvo#n3g4*XlG5 zD@<&&r|RBp_XTC!!;#UD@!jG6%1G6{=qM%jQnEi^k_e#S1uZ!&@37&kk4aB2|39`J zw`w|K8Tez}dlBL5D(@M_{Tsn+$^LjoTTJ3Ed+@Jz3!eUV?xk@DI1Z8n)5p%XZknECVvKT{GBb`HC)&Ac2;R( zIKCx!?UU*G@^pGG5m(u(#;3FS@^XB!sP}7EZuNd(k-*2K{zK?^HhGj1`4!+3$MXX( za2;zfP|~^GuVpK!`}MKBu-~zgKlmw+1dbBa2W$12U27)iV)P$SO9KQH000080000# z03n@T9Y+EH0JHuVO9%zXr)b?JlTIKXe`>=p5QOi23ZuI%*`aL^wgYX+sewXWT5=Q8 zCK0uMkkmSU`zlW2B;ZqztC{_Fc2T+5t2X%s=cUmV$}@rz(A)^Aw-x$mmgyWNJ}@m< zYcy1-2agu@UCAxw#=)C27F>|vlL*qDa$BKY2$o{(`3@BGS#;(FrN|h_pYh9RPx3XhDGv9_Lw(FWvx7`N@VsxpFEQdJB`* zvpBl#}Y(SY!Sif305MhgDOl52xaQ>QZpZ9k5_rUE=E^9ore6Icm^DE z-+qGn53`UV$^r$*c%d^%lV2oT0m_rRBqALAU=k`|1ONbZ5&!@c00000000000001h zfm9QdaT6Ak4<#xA=94-lIssUdbtNVR$fs!CC6k~f6#+Suwk19Rjg$2yBL+Vs00000 DzkMD_ delta 1659 zcmV->288*WOrA`zSqKGVNcF!;vt9_Y27k=HEDJ0FqQHnFfX#+Fae&6ARve(=Oja=C zAPr*@*l`$>Wqusg2xGF$kt3KThn0As5zZuY<$x%@a^)JZa;?*u^RGbpR((2x7SKyyC-*?+bySH%g4|$nFhCtSk&GF zG!C`*01YD(X(*XUL&-!MN~Td}??Dw%GRrFH8qD6qN<7pEJs4q8JrKpn1b-S%rctxK z=QT3_?#W0k`Bj0^(D$L#O~WmMQWQYRogx->_W+F}D3lf4JxIf#tl;iJ{h@n4Mwz<@ zHNv1QbN3j`-NQ;e(1_cnP%?K9iegY!aQ7HByL(<^G9BhPiXVEatR=T9t5;*!)srm% zEZw3wYCC`7K9%uEp32E2QGbFc8?II4wsiGp|JUN+ZuJ-PFY>ItXf_CQPrpA|{XKsg z&PGb|$Lf~?to)Y(^t`fQc)gfNC4HXDjw+w#*(a1It15%1;p#U6lzALRahN`@TF-x^ zU@`HJrqSU2vQvv$_{T!Ne;2&74&DXUhhG%qgVIs5)Hl*ByJai+3qaJZqr2=1o2%+h1=u z`V+It5tso5)9fJ&VY6ownF4=~;aX}qfajnDqy&-Vv_r6hQgpP}x)A?#!1DQ%#id6{ z99Q)-sc+QRh-O6!9kag~8+H%w^x#T0Hn`V=`#reg%NEB<02{n#9O9mFh?Q~J;6480 z9)EFE-{& zw&PY!M=S$>ta~pad|l-|!?=GVcrDo<&uEKD++`2`)sEr7Aa9JMx^sMFS-zR=4RQSV zHh7Ni{rliu7~eOhAJ~5h9c&`5iEu~i`Xbq=bu6#DEsi>N*N!Rh+zR}hyo+WgzYfw* zi_ho=SYH4B^mJke&P`jf%B3&zi#bw0`@YR4vm}>9H}; zwlVgcMNMI~RUQWwY6g`ek5QAl?!8k@U3N4-Kz3j_fwCbztsKC^4h}_ii|Hw9^d3&6+O)M^s_qJA2-6l`tu|o202c+2a)g-@?Y%byW5@O&sY`-G1#zpbq&}2(hkp%a z-*g8!PtygK;3=n`;vN{Ak%&%6&PJs9vCRPB3_}AFly^8wGJNrRu+mL749b;L3f~!+ zw4TM$eQ;tlwr!iVv%wnjXZUmT@;0m*3)P>B0jdhl2(v)FwpD?zi*UGxI==}g-U{H4 zUC<}qPKzA-fJY$!ImSIJAG4?DdL@mjJk95En#cLFPM2i9AouxKKk}yk0+gpA|0bRR z$K1D{p!x%|jUmbc1!G9{zet`002+|000pH0000000000004l2JCm~|J^_%E@+Bh%LL&eG F004go6_Ee{ diff --git a/summary_results.json b/summary_results.json index 6d3972671..d5ea3200f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2254, + "Performance_Result": 10.119, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1028, + "Performance_Result": 23.0414, "Performance_Units": "Samples/s" }, { From 128d3a50c264d1391df997f4838bd3b3ff0b261d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Feb 2025 05:51:21 +0000 Subject: [PATCH 214/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9755 -> 9782 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index da5c6c21e..070d7932c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-27 08:03:18,213 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-27 08:03:18,213 submission_checker1.py:3270 INFO] --- -[2025-02-27 08:03:18,213 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-28 05:51:17,041 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-28 05:51:17,041 submission_checker1.py:3270 INFO] --- +[2025-02-28 05:51:17,041 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-27 08:03:18,213 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-28 05:51:17,041 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-27 08:03:18,213 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-28 05:51:17,041 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-27 08:03:18,213 submission_checker1.py:3286 INFO] --- -[2025-02-27 08:03:18,213 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-27 08:03:18,213 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-27 08:03:18,213 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-27 08:03:18,213 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-27 08:03:18,213 submission_checker1.py:3307 INFO] --- -[2025-02-27 08:03:18,213 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-27 08:03:19,246 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-28 05:51:17,041 submission_checker1.py:3286 INFO] --- +[2025-02-28 05:51:17,041 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-28 05:51:17,041 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-28 05:51:17,041 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-28 05:51:17,041 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-28 05:51:17,041 submission_checker1.py:3307 INFO] --- +[2025-02-28 05:51:17,041 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-28 05:51:18,011 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,9 +31,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.119 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9238 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.427381 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433081 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0b6746cd3..dca34d14d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -841 +842 diff --git a/summary.csv b/summary.csv index 5774467b0..fb0416030 100644 --- a/summary.csv +++ b/summary.csv @@ -9,9 +9,9 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.119,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.9238,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.427381,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433081,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405291,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0414,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 79d60e65f..48dba7c46 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.119, + "Result":9.9238, "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.427381, + "Result":0.433081, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 696868a3760d499aaf05ea379b119ec84f93e0d3..583de0a24fcb4ccb2acbe6b1b95b1e0502563130 100644 GIT binary patch delta 5204 zcmYM&XD}Sx*8uR<7tuvothUQW2+@1*C3+XVZ7e}nqJ*`GF4$EQ5-kL=iQa-}u?SX= z5G`6nd64MkmFNG?Jom%>a%X<$o^xjI%(*WO8w@`Tk7ke=l_A^DiOx_pPDkYqtg?icsQrD-s6lKT6rgmYyD_;G2cR+^5Q{v&ZoC!kEn z^GkL`)5GZZK!#Y!w1dKC#-a^-7CLJPiaHp}0?9UZ9jH;o+c(q&ck0qxmI&87%pSfI z)(xy}Ab4?CtNoCr>}Q}EOkOFH?$QMT*_EB3WHLI@u=EiNWpe&K!ui_oKDu^z3VG%) z6w`jsf&&Mi%^Wr(CQ7pgP+BN%Z*r=kq?oPIP-F)>DoNbTVrNK*7QbnJiquA7R`qrH zxwI(Z1C>tv2JZ0tj8z+j~MD%z|Vz+`-EG;HDGHmhh>bg1dQg?f)nb*a#@SWCoIsB;Uum^-*4 zC$)I$Q>aDkbbOd@ZX5fDl17wu_MM%vNyV?T;8OV_8dm~CzHg7APVoEqDwut6hDge8 zBrHgiRr5rr2f^xkdq5iOVgI9*N1tmc8g><*^D`a{_Mk}Fx7T?O4PK1{l1_vrtE$%M z07<{hyO9zLOj8mQ)&62e2hXNSkP^>d$`KR2=}_fq^TofR4Nwab1fobSj;PR)n~wbS zCf_+oi7UdAR8@m^>G2B7fBvCd|4FX8O&6p~ZQ*;(wqLU=fBkmBYrk}yhSWBGm|i|d zdtG6J^`+O#Kj)0JhDZuulgiupW^Gw#91l?JhaW1AD|Ps?Rq55e;rF6PmD1=zWtZG| z%VzIb$=VtV)TL()MVd94%bsGtTbH|_##)sB-St&IgQf>m$LH1Kbt0X>@^5qoLF`kL z?jP4Ax>o&$t5zR?V=Ah=@pp=(CxJ;drXPU=U6bddzM)2~)$>qT;BAw*M$?VyKaN!HML0(wBRBpQaC8@&0oHK8;+Pkg>{FTFGW|BCohem|%ZKjd-qjtb7ivQDy?k)*&-pNDK zuRe!!GD*Qz%D;xnklJ4{$$BpW`AcdXrS^jAl>Kt#FKjPvU2@>sR}WyO)b?pp6cl{@ z+2mKYuT6q4_M&}mySXTen7hVjt4EAtQS0jMe3=z+&5?j`HWy~$LSY8c`X@oMW~H65 z;?ank7P*2d1~nb5J0TZs1chVQCW_C5w1kHR;;jP)L3WWbQXu9|i~|`WcV$jjPnfCx zWnj7MpFQm$5pUD`{f^;k&kAPN{IsIt#exJ?{M__=*R(TN{!DoPrZzDkA!P;t7yv*3 z0H6X0HVf1-pacM})Ua+qKGZin%d>@%_a^20X%;jES%e{voEp7lHl{7vhO%KMXcDJ2NenSC?!2AP@s?YY$yhu25yzR^0d*daGkhB0s&Dt!X%@-B;TyC1^ znWi`M^G|Vd6_}EF#D-CEE~0!y+!W<|wSIgz5oIi3R;q8+DGV>iq9pUVr9cTP>4n)7 z#m#vh*4n0=q3}a#unZ zj<4AEx`zzT)dQx3nsLcFI+dJkRRaM_Tooj9-fm3Qbx?K zAP?_J#2RB<^2Llcu`+!}Bnw_5(2;TeyTh!Z32R*es*l6J-#_Bb-EgpmSDt~1NV-GY z@34u>{e?>7>9&K*-m_IZSU-!#51xGConH~m(p~VwhdCn18tX^wspOpq$PYxb07bds zm|9H}T{Oh$1QJbAi<26&8CV5Xri;D-Ifdm1Jr%|x$gq-A2D#rF*3c@ir_WD{ScW`} z_gu43o<8N@?4*lf^VNsM^VSYQvzs90w_DZu`mku!`uc+BgPH7XOUwFz&p*!PQe08u!GvE{ z_bX4R3uWkdsuL|}dG@B}d8^^mG&&Mu@wDWlOFWqM^$Y{yePLu%3>pYw*5oPVA*Li7 zy+xEBCqe&j+^CI=NPIVYJS`bRPnv*ea}V@rec)t)=&uLZ8=iT|NHcy{s}se_&5GR! zWTez6b8t4A4XPbzzw;3(&|Lf zF>_J38|}WTkECM0P1sNUvtpwXW@M)oN&yJo-T`<|b7_oIA2Wf-{cZ07AR zM4-<+V!;Yt3TQ?*2I-%W@?c@6f!HW3$UY?b~*X>O+s9kzf?}d=5)5#s`t_+Aqav zP~c)iOw*0=kvLZe>()B(sX!>-v1u{fT&y#oM6GiLL@PiL6GO+MuX)4b8-^xCB()_`JBQ2E zIdvrzAeS>4;fmBEwwy(tm(lQ#aIyZ!!%ki`iN9%G$G&=v1w5M)c4mA%{~8zp^OMie z!&Se$-j)oBY1|mS*|3XmU%x^;luFYh0F@pHqbdtp+m5>+B0tgE4$0HO=T+C!P?aY~nFl5@CLMND9j zQ!dIj@NXNta@}#9^u5+N2NElsrTnAIo3MonV;%b9=l&_4Xj7dnxu7~$Av9f_T~Orb zQb(?WB@6K4lp07I1tomV7!dEKt%x+W^_*dtFn!diA6mc=A=H&8$CLOv(Vs^4X8G6r z?(*}KKE8(&^^f=;`zly9C^ps?xiORh3>F6pKXVmSmp`KLmd&CFev%gJLMeG;Q2gv# zbg}hZWSDUJpV)pi@Khd_DB_Hc6>+}Zu=!DYbz}yG3NIFTkMB;y8qsB7+U)~64oBS3i=RV^>ylC zH}XV#NNJ%L*D7d_2gR;2BThCz0_N*h3`yU@B0m=%f(u;S`Dma_a9FOcM*qa%meN}k zvA(A*+5P;Puz$-)t8HV!tz{~Xj>VQ+BX&blzI23%o9RWwsh0bknLH@nVNtt9Rj_i8 zA*toBxC1U&G665{sDa3#8+6|fMhEAV-ZGGTQK7MwW9|&Mp`CV0mnWIJ7wr#fJULlB_;Viw?)dxr0g~@$2{AJ^sd&P(*tMu8i zN$v}$q~g}eg}x4a?P1Z37+&dO9{Feb94^LjyPvyqf%3!t0wrO8;C%Ftt5D|ekURDj zPhWUFff-v_#u^t-iju+(WaF24!X4{hU1@f@Zq|2uA4klT_X;1o@h;eQ94y%C>^}X~ zEt+_#2ES#y$pO#$UXyuFwrO=1ypytOEefWn)yy3*&3bD!(jM7Yxxi`Dey{jsKeJru z$n=Hw(%N^;P!0C=-5ZwdTBa=_Si1hhMviYN;@Sa*93oQZKu)B(3 zzNBj7IZ%6B!o)v~^DXe((iV>%v-iQf3MifSc0a|u$@kW)ReZ*R>PCz)rVmiRFd_(w z$q1MwXbE~`;75is-Cpf4bIkd;ugV*g#1z40n~U}Ir}bXwhL5+UrAdua2eE$3T_>Yc z1L96bt|r*J>CIqoCs@sW_b`3?6e4vcj{fD`%XX?0cDE91guUOA$r6Y>7trSxaW>DM|N#Ql!A9DLT?4!$n7Dr{orwBdyMnezGtL-CC(|p2aO9 z+J=q@SsF0Z4~Bi_;;LLK`Wtnpw^Rj^8`!2y@GQ4{15b>p-_2`T>S|~71lYE{{%qx= z93;qwT)antyJ?dW#?`cd_R*PNvKcf_d4iOk8(sP^we;X4zvCQ(=uSNAlb$4n;AgCs5+^yDf7T6)N($YaNp%B5Md{6JM zDs=}Rr=vt>9*Vj}>&<)6?OCFiK0zLSDN)vb>TcVfyG;{v9?Y(&b95;m%=-Cg=qH6x z;WMMpsGAIa%`G^?sX@E7(eQ-gtbn>V)QU2b%3cqPbIZ{x7j?(Uev!4CXK`L>YKwD{ ztXYP=MVDMU-&C4AJh3yBr&MK|0ss-5AK?>io#EEaxK|}QE(Q5wv|Cz1E>hWw(n1z8 zLPOx<)6H*v1{a$(*b@d2X(^L8)7W^Aq2cl|Dy+N7rv>+G+vrOV@~M5wgi{%7W$n*r z>JGj*(CeYK%GKjQ)09yhX#V1p06JCgkJ<)OI;4(KYU`|L@B9hb0~a_JBQ0Al2T+ACIFAQ^w` zDJKML$OA?VKIj&s*E+hODsB^*<@Kd+Ao1<5K&#)kYhfCLFw}!}ALO5huX-`!D+Wi~ z9<2=*xQ1sK{idFxtkTbb-bz%xJM`>Z_D0)$&uwNoDKDn6v~# ziteyy$w{>l-Qo=m62^=tvIgQ$4?{nUeMf9$QkOAbx-XUnF?t=1Cd_Vtp1 zG_pNt-|;}$;)xLXTOnEgFT{VoS&_91cP)W7&!zK%1tV2nMMeDZb#UJx0Y3w{$1B|8 zPAXdug+x~<4rK?t=sT&nm+hXLzO_`z5i*G)9|*vABrrMS!!IZ+RBBUrV~DB#`^m#{ zKomLt8=wDv?ESYQ(H_`9h!)Wrwh6-bzh{3Q!b0rNg!N})!QOxh5figvWubCJNNgZf lfaBjz5CH%t*X?2WPXR=tEwQywWumv(eJDS<9sl2>{{!L7r$Yb$ delta 5263 zcmZ8lbyO43+a9^mU87@2gD_xpii8qMhajPJhvXQY0vikvkrEgoV}K)6Lb|1;L_h^W z0SRFee)v1*J73&$?)~Gv?|t4p=icYy?-bo>94Dcy6@BS!NeTdbW z1&lp<&GBKGddswsrc{ESf!@sVr(^V<_iGadBBpwxF{hc)E=NwJ@e#ob`Yhk@92pxu z5!Rd~ozdCpDjHs1>7>eQ=evvpjqC8>qU7}-!Vdx?pjWRO1Q-^?$!DbA~eZUBo*dUTGsLD*{<>)=XvAL-(1+4 zh2EoQ)AED9Z|50!fU*S{hF5n+mf}T-)lBz3sF_<(hU~ucoyRTiCK0L-7}emLCBmyY zmoI6s40QV~Qj8pZFlADH5r>x_@bKEIMpiWnKpk^zo?&n$&OT?QR|) zut$)0kvPjRkkI!^^;uF!LzDcXCP38AOrKb{0$?XM0;g$4w@cHE!W_E``YMtLsMhCg~-^Jje4 zvi=37?En>eu*Dom*#XZsA44W;xF$UM}lY(@3iqn&GW{T9SiO zwlrh0y*$;jKN6N|T@+b%=-U6oTmOs1KA7abnG=|YpT2FRh#syJ3?faVMQBd((~ETU z&`$9Oi*#@wj}kF+dl&H2mTvQH4Sb1p+n?7x{!FZP#+$Y$7x;%}N{t$?yVa()rgHH) zcx$uv=&k{RjWuy?66@dor|gYqH@c_Qe8?QxnFtdqHkJ)}AuA`^q_+4qb*?cV->4u7b|k`@3$qh~9_PIiZ7R1K`gl=%A7m1A#j?#@(eJFXlaDHo1gGa%Q!- z@)*Gzsh=4`brvLjc9P{nEPA!qeTW@Hw=gORidrv`q9VXZlm;}K&(TZ_0`NWGag z%ly8MI15bUJWiW!*}YYCsl%;+u=PtdXP4b)Zl@`k_i|Q4Iiun>MX%J-$JUUp^yA*HOcV`AVI0Y|kPzb*P5 z3Z12A6ee26H)xQeRxJ+5on2(n0;JVmTVr9`@OP8^L;Rp{UH78jMuIq8U(uvpo1Rsj zn4%r7PDH_RHmRxI{g+sXv;(WDU8+LD7+4g7NQi=MX=;-%kTA-2u-Q9jPyM{sTs7k3 zuZScQat-bxGT>bEW$j#*utXRjmDU;4GcHB*Tl7155eZ+;DfIO_OWOPRUq>zKcUx^( zeIRbXJz@C%O_5iT=nnij>yx)c`P5rvncsi0D{tAyc@E$9KbhWNpIZ3)AiCrCM8wv9 zeqEW69c||O;J0ml4gLMZ!t0@igAQ;hVe%x@L$g8M|Cv=wsUml zi!xj_T^N$WKkGtf8eE<_GLG;%!AkErvE~NJzv9~qrxsm(o|#kLoZ8!DL9Ll(%}T*v zELEDYrFuWCcu4St4NsC3-&iBCDL{~e zA5NAb`E9y^ZVUSGFYYU+@fYD$IclBtwVoNWJX@0<%)rg7_^Fm>5rWpB3I{}=^1T|( zjaTnCEV0EYN;#($-f_ zr*8jVpOn$?8jP{MuZ&LJmm-z#`AW-U$ht$mf1ojy>L-9*@^k@DO4+U{A3twTo}IM! zeT@|4K%e9M!53?d$h)?;BUk`$L&E?qMaD*olxIxJ%B` z{L`kC2>3u5(}ra{Oqa!!epGDRSK8Oi6~g1*2RX5C#PFn)pJ(K23I-n{?0+8r^`R%o zGJJ#5zI^`p)aCimRlK|MqWxKZVLham()3fk3chT5ei?0}3Hjuq!5Z~=6)Ku%??Pn= z`B?VYoS3G9(VacdzPEjo7#E6>$TYm6K9s@>A0@9Be;Gs)f&xl?r(xX*KhjOgA|qq;FF&2J21cs66suhi z&D7}odq{6Da*fRwk)yn~E<%KN^|6eoSJA@UR@-DZGKfpoc&TE9eW&GoN^#4%1`X?X zh-qA|(}1$yEXW-W-;DozVl$KmmTJi#GqT`XGuQiGEVbfd4g89w7oxW1xWksF{w^(Y#`M zgMBSzooW!2Y)~;byIk&$mXO&1IW?CVbCu$1Sb**@6w4S)zqf!ilxx7XkQXTbzCa@A z3+U1C!|Ng@$PL>o@ugwy+sW(gcZ0uOh?{m=lsx6r<6g(7-N8Fg)>L$vQbfUKYe}FA zCp!R|>ZUk5Al!pJPqTNv!7Yi~oT#E{tVs}4@$GruwSO^U@1I}em8AL8z*uS)i+gJ> z2!jHCv#)~$Lw!^%@v5yS0He<%Wdq#T4hCI2h-P`KqTky{3Qlq>j0ReCH%iDI zfOfD-|GF$;ze~1+Hq99Y`Orw_J#JaNfVohDpJV-I`+_7x9aI+uEJ0>1mpiQf6spRz z1OL4I1)$k#ibDZC*z+}e+a%!FLNHYDU3M+xSzivq^#NP5A$G?CB z#}qyH2vVV|YVDgVzDi_v>Pb(Bp=}m@<-d7K12FG^?(Bv7I>WkPL*Z#t*-Y6;I(Tg? z(#BG2!Ra@T=# z<1lZ4?p#`B^`G~&z~pXZVRSomt4B9SR@5*Ixz9pvJW*U zKquA9RBbFcRB5cb7)r-BWzu?&!{;Kd1j*7<;m}O-{rqCvvG*FRhnH{cJ{N}^NHwZo zl(X~ZK1Eh`<~&-Iv>SFT_2uTCxXgKbuxSP}%1z~Zl)5wigGlwkdf*An- zW&nx-!U}(FWoNPpV~)9Ie{PPCcmi=n;$qxo_)^a{b27cL`(K4`KvWE#XL47KVI7mn ztC(((v1VtBbp}PLm(-agIV#ws?UjC^7^Rc_Xv%Fupy_xWztx77UAZts`QLSBGA zk+XHznYT)z?k6#`0xl=W}3K~A!xf-y0?Zy`HTv(}PUHmsmps(1EW`jXUWu3c% z7QCd}np1y(lH0;eK)qFdP{Qr2Rh!ic*3w3RG}otRiu>-p4#R0(O!8iPY<=d7(F3ut z7x@yT6NS!QXLb*%b@a<#YFkvL9YWIkO>D(*Dv!0_^~on=PJRp zrouONw66{8UXKBty89*ri#fV{EWw7Y6CNI_{K#g&mAl?AAIjYH>X^}4#LWu9!AE?e-1b`3St z8Nqg~qN=}I2kp;MWy|h(nDskZ@Y26^r0s=#feUS>qWaN|AnjT zz1X77_R^^ML`vZk)O_2sZn}Y9N~*!6>I~5S@jxPuAp$T&1b(9TIYNu%xgnnnxzum^ zj*c^BwJ!UK>h!N)rxq`QZA zGu&vaEis6$=+uLX-`g&@9RJuD>0b@Bs>SuZ<5}77VR6L6EQvpUR@_BP7?|3&??|kw;C39Qu&6&#nwYq*aqCfX;TLkj^UXz9+5>)z`Oj1c=zj3`T%fLkX zuLhpG)l=D-l3{&UY@cJ`lC#Ha`09zC-JK1o$OCLBHIO{qMQE;gQ*iWf+}kf@ zbFj51D0qJjxc~mF^X6r@@iyFvDd28`%Dn})VZB!_;j2CYv$!{*cMw~apSQj^K;;M_ z*)g_JdWzkLe@?5bnN<9Jm(nbxqH`R_at;?yeH040%a~y3LO@GfhZdeiBgzI)^elfwfpiX~U&?H5i$ zEj^rJosh7fsPBZs;0P%OtHrv3>Sn@WU=$8>>Sg77662K9J^1oUXZwjlcrGJ>LWt;_iofJBF32!+c^Zs8R$Y1<``!+EE zAcN9`0Gs*PbwFf8)$Xehlsg{}eB85FhCX3GnzG`JnFZEzIyb zCVN&n=3LBpL+JDUth&O>&``7Xk97(JvaQM~mJzu)#t$o?npT!S?&$Aa5Oo9dP2?>v=$Th zhprjJpFsS-Kl2oXRJr~?y}oAH{~56eBM=G!0;4cOVnidT86g4A|9chuw;})#i!evg zL$qkVvHgdg2L*?Fy9J@vg_u!+5HPVd7?llXL46gX=lV~*hyVc7>oPL@&-rTrB|cOD cAJac26rP`+>c9755t%3&VL@^Y!GDtf0&xDbBme*a diff --git a/summary_results.json b/summary_results.json index d5ea3200f..2ec58b950 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.119, + "Performance_Result": 9.9238, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.427381, + "Performance_Result": 0.433081, "Performance_Units": "Samples/s" }, { From fdb12f67974e3f0b68a7673671a3ed8692f39f5e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Feb 2025 05:52:31 +0000 Subject: [PATCH 215/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9782 -> 9783 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 070d7932c..30dccdffe 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-28 05:51:17,041 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-28 05:51:17,041 submission_checker1.py:3270 INFO] --- -[2025-02-28 05:51:17,041 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-02-28 05:52:26,582 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-02-28 05:52:26,582 submission_checker1.py:3270 INFO] --- +[2025-02-28 05:52:26,582 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-28 05:51:17,041 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-02-28 05:52:26,582 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-28 05:51:17,041 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-02-28 05:52:26,582 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-28 05:51:17,041 submission_checker1.py:3286 INFO] --- -[2025-02-28 05:51:17,041 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-28 05:51:17,041 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-28 05:51:17,041 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-28 05:51:17,041 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-28 05:51:17,041 submission_checker1.py:3307 INFO] --- -[2025-02-28 05:51:17,041 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-28 05:51:18,011 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-02-28 05:52:26,582 submission_checker1.py:3286 INFO] --- +[2025-02-28 05:52:26,582 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-02-28 05:52:26,582 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-02-28 05:52:26,582 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-02-28 05:52:26,582 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-02-28 05:52:26,582 submission_checker1.py:3307 INFO] --- +[2025-02-28 05:52:26,583 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-02-28 05:52:27,703 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index dca34d14d..07dff4f8d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -842 +843 diff --git a/summary.xlsx b/summary.xlsx index 583de0a24fcb4ccb2acbe6b1b95b1e0502563130..92e315699422bb70c04d1edb4fe9e138f85d708a 100644 GIT binary patch delta 339 zcmV-Z0j&PEOt(z1V;}{jc~CValV~6wf6Z#cFc5_AeF~$yEnC5D5Viwt$*F-tQ(Afx z(k2nLevs5Ue)}qp<0Oz%kE@yec6L#<*sC`A1?Q#FH7c?kC7`(xQg3VYzFwwtl=#53 zV6D+mqaHk3G=EEwUcq2hP z_EEp_HZF4P1Chi4^ceS~e9WGnCD%*TkerlhPEs;o=Orx(B@bVRkyrg^pc;q#n|J~o lb6VGOtwt0V;}`1hWZ>NlV~6wf6HpaFc3xe{R*SKE!!b&5Viwt$*O@uU0S*m z(j*bJ9!P4P{C(wT63D8{(Yi6F9W4?h0lI(-vN>HekH~7X^@Vgqac@yf*}6#||K=OM*iMai^`MK9qKke+^~d zbPqUB(|atzQ%*g_9WXXC5sgUBW~BMCO%LD<(%o$6l?$g7{$${$ zbrNUy(TUmEwr$cbdTYp^;m_^MTfb^7RCg){s46%k%mVe=Rt3H+!r>a~{5G6;CxAbm zf_~#|Tx9429)$p8h&w4Ci>K#$B!j9v&6jbS$N8#Gmt>id?BS~$dDVXg%5liQi4$PR kefNler~S0jQG>CL;z~BLDyZ07562+W-In From 30a432ef2d504e89a9549b13139a8766516fa218 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 1 Mar 2025 05:35:25 +0000 Subject: [PATCH 216/719] Updated results summary [skip ci] --- README.md | 68 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 6 ++-- summary.json | 12 ++++---- summary.xlsx | Bin 9783 -> 9762 bytes summary_results.json | 12 ++++---- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 30dccdffe..899fbd08a 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,42 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-02-28 05:52:26,582 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-02-28 05:52:26,582 submission_checker1.py:3270 INFO] --- -[2025-02-28 05:52:26,582 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-01 05:35:19,524 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-01 05:35:19,524 submission_checker1.py:3270 INFO] --- +[2025-03-01 05:35:19,524 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-02-28 05:52:26,582 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-01 05:35:19,524 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-02-28 05:52:26,582 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-01 05:35:19,524 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-02-28 05:52:26,582 submission_checker1.py:3286 INFO] --- -[2025-02-28 05:52:26,582 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-02-28 05:52:26,582 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-02-28 05:52:26,582 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-02-28 05:52:26,582 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-02-28 05:52:26,582 submission_checker1.py:3307 INFO] --- -[2025-02-28 05:52:26,583 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-02-28 05:52:27,703 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-01 05:35:19,524 submission_checker1.py:3286 INFO] --- +[2025-03-01 05:35:19,524 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-01 05:35:19,525 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-01 05:35:19,525 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-01 05:35:19,525 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-01 05:35:19,525 submission_checker1.py:3307 INFO] --- +[2025-03-01 05:35:19,525 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-01 05:35:20,700 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.385187 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | AMD Ryzen 9 7950X 16-Core Processor | 1 | 16 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9238 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433081 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9238 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433081 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 07dff4f8d..874a6f88d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -843 +844 diff --git a/summary.csv b/summary.csv index fb0416030..908c752e7 100644 --- a/summary.csv +++ b/summary.csv @@ -3,10 +3,10 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72551,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.385187,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"AMD Ryzen 9 7950X 16-Core Processor",1,16,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.9238,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 48dba7c46..4a9d059da 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421108, + "Result":0.41903, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -189,12 +189,12 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.385187, + "Result":0.378015, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", @@ -222,9 +222,9 @@ "Result":0.0148696, "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD Ryzen 9 7950X 16-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":16, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", diff --git a/summary.xlsx b/summary.xlsx index 92e315699422bb70c04d1edb4fe9e138f85d708a..0184aeaed3bf6d3ac91500e1924496a9da7215d7 100644 GIT binary patch delta 5179 zcmYkAcQhPKw7?gOU97SM+30p9dRsMmiyB?DMWRQE-dT%?-d0;BTB1Y{B`X9;^xkEo zB%(%(o*;KD> z$8BF_Hn4+Bjd@aO^x3+0TCG;4);-XUVnC$p2wq}<9T=?&999`lV2 z90MbBDn*y^5Llm<=6x4)CpgnvI*m!C{{1sP-gh!EwfL~|Ls91VXX{cGiV=r;ZpcdO zD{LB5hrdUh-GQbNlma?HoEtmnuDlueYu|`^HsP`BLL=X+P$xAw^`MFF<`Ah?HkzL^ z*^dG7g10UrSs?wTzaBTG%`+#R5KXRnqu69qs5-oEAr`@e41L8e<3=JebMV=^FW}p3 zd-)VzzPZe1%u4$+OZ9I{^&n`R9n$W&$1aQ92Dqgz6HdrBrI^`>hMCn{Y-2W^KBfg% zK2k`vB-ysOX~WIN>4=xG(|WQeghnmAtPXw?D6i|UcB{LXTlRe%E@l54-CaZc{v9jW zB9Y-%oKW0hbKI;#_x(%{xj}qypmL^cC%NHYM$sucdAUJq*jjUnjn4jHDjlfugc%d* zpic|RZLa_Ub!)|e#7+bm;)4KACJ?agQU^$^jee>W@;=RtbvbJ5J}sycy$c~GnrT<( zC~ge|0ZmTWFp*md9BDM&+e`GLQ;^6x_U$yr%DHQ7*fqBQgyC8kyXNctxCLsUMRE9v zBnzF4*GY~^Zl6i8-~B~ahDBs5sH>AwEkwpZstLp6W%FC zSM4)Fc0wcZkBy()3g41+tt5JWX};CIN}mJcObx!B?_L@B6{SzH z$L-d?=a!v!J-6A5t{Lq!{jvRu1kzbg?TUFCkfc<+MGG>yroK|;uya;8IjN*BiFOyh zj#^lyELvJ*g0<;k1yH70mRtaSjs3!SR#e^UW4K=G-cRG|xIA;Bz-V{=0kkhZc}_w! zdLc;kJ5N5)jjnFS*&|%rIYw|G_XV$%3g5OV{o=jtmDsAOqM4%Q58t<}6jGsY-;lb7 zdvRohx$d+70zW0NE4JXp7<;G05V1l0t4#92Xu(x}qpCcOV;JEXoyI5VAoRd2^uaT3 zM_7^=jMYpzEBLSs{)~JHLr?^@mkGy)I#5Ayr(}GM)omsfq?+bs%5(w)dLOg3Mw2tU z#SGu`=J1)ZKQSRhG0tS|%rKdN@;r9U;Bg@1z{3^Eauat?6RmIh$r_4820A1_1^|E- z00IC2ih#k^{a;#O0N?@}2I54|`_5@F^e&!1a6txZI#IYF{EnDq@mT3zu>h9&LIu$V zb0uRX6KKc9N!GiYqi66bWR2WU9pdV}f_6>CfZ5C8C1HJR{UOH+JgHpJbXvkReTrl2 z+0;Xw_VzPtUlWAZrRyU5J>yxUBFECdlYck&MHA5a8b;;X*>-tN(}w80QEFH+oOvL= zXu1qn;PimZQ0`!A$3xB?-e~^kQhnTEVO(P5vF?S}D7nRsNX;s$CAD<%NXq## z`_YS z25t&HNDGJvHT6Dvwfx((2Zei7QK7dm#gj)o4o*uWItRglU-XqG7qY_0pq*2^X|&Hf0N~)L(a0F8aB|K5%ez0Ig-NyK za0fr($e1KgCe8j08)ERyulyXGW#>0E1ui~XoMIZyiLnY^){oNf26ZCHHuvsva9&;l z;V(WEwRCgZqg%5hX}X1r$qthWvSI>16F$>V%b*4KmCiLunULi}_8UpQ0|=p;xayn$ znmOBu_ka9;7W$7Z3V&A7!ZX8SH(erZX-c*1#YX=YI#3qw%;@$;+eowaOIy_r*zjSS za)$SAKg_5L%lH;;L4dgmtUXGe>CZ7rrrdM5!9E1H`UFp&F)GIoEKoW%T8iWoN;i*r zpQH7s+yzaoszQ{JtTC#;oZku?E3(|)de+qo82)Tqyh}J>P(O)Y4fg z^|bnCuFNfQXW#XHY``#r@k~>03EIn&X0RR^BSsEiJ(GK@lS{I+T6X>#-qzK{3XSMs zy{`a~WBUegKI$x2ieffeL*%@3(5>B}z@S-DLL$=8xGKj*RCd+lE~>s+Us%tl%9CP3 zO{*E@y2_JXqDMDSvn1VCp_JX_D5(N2T2`vkE@k-Gm}rjqs)h6tl#DCQfEE)I1%flE zRRGm!?<)&{_x8QWN{xxRXq-7@8PsT1ou+hzya&kFvW^ys>Jw^rh?WZ*#kpbSX#0ey zJ$WIXi8VMhHl?MTi>3>lBLc<5WPsqAYC_(c>J^^gc#eXIC0_8}yBMsm<#!ClhFE5= zUvQF+#yV89|K7(eD?xX9#kh>QFSZUrKCju^;Wee9p@5*Doz$`)rIj?fid&FnT0}8NHe$nF$kV_9!B*M*Qk*?;E9y?` zKv29G{cs&V?@BT<3`g$Ed~P(_&?kD|VNZFw(4FsMNYiYxOFp;FLc$&LvPMF%5un{R z6ahc2ZFM!e)6!dhj#26=0m8@-<05xjVpVwui-BO~Oaxdq>N$5oBNR1?I0H$Ye{g5V|aOH4qSIMz`sYIJ>>GP|jpB+E;{)r3i{p4Ft7E>ELn`ik7z zY@#nb!1Mvc3LQUI(g_t@kA4E?st*NVmluQIlT*Wu2R>>RTy=-AsLV6 z$BzP9AnjT+HEZkjwBZCVE=(*P7@4G5>%B4Cr$G`A%o5?!f%-`@*~Yyq8=W;&F3aZRX(pn!NG7MWAl(1W~d@20H7IRqB>IZDxWVVt_ zLGcA9$<9$`eT-Etl)a`^U(@(bM7B8~=rX8_yS5|FmK1GmS!JvG> zUCpjaAWXv%7yK`AHW27f_+MhPEl!7Az5W;f_?usDJsO3C$_vqXblsEbPG3R1yxc6z z3}o1JzQyPTW|5ulCP7O29{?afd}!EdCPjoWvECL=Ysz5e97iX%PYZId|4A(&T*kQH`)_Xr|5S*Zl`=>-xbwZsVk!cFgb2dAkz zpa#<}w{hB4MuYF*t}{SZ{N&9)UXQAqn=q*xieX~0RS$M{gOi6pgo+qxUH!%w5duo4 zb-xNm88IjgBy%xd!;)1;qdU4jlCG_s&vBMA2SQB#ouU)_O1Ey=k7#yu8ckP^}qUl(%+<6i)Trt zN5nNOS!#G{=b#VJg4Da&XqGFkv8}}NOOe3i^Od!t!-^%pN;p~o4gWZZhARzWNu#&V z%=tEtbnzx<$e~?>Zx*JTijZtYn&RPQk%X?}V2jYd=fI=-#pi05Cy5#(6-(^CXQ;oZ z{ZZK5LCbBahifFG||>R~}01W!AGeu+D7FGVKSQMUx~F zT#;67Vy0^f+EXqK5mFfuw!LR}RZYnZGiw)DRU6KhVt*I?^XkIZ!C{R+Xb2IQDD2El z_qrTZ#y+DLK!g3PF zZ?VO7*&I8;_(^gTLN3Qj-Mt^xlLiqGWif}mA!VXtXwtU0^?}#u)BJaqpu%{`JV#_j z_x{f;x>yhU=bmcn;@n?OtMN6K#yzv984j*Q9;bPikx^EI@r+1e@eL8(K32Mcw&G3Z za3Wt1E;P>EJbk0WR(aq|Pu4omrx^8q&kVzd4#TdO6JKh<(;BO^0LoOvJYO5$n!aIgQQUEz?$ z9^K8S^oMw^oXq3NYUGSz%~5rJBUnL&O)@;+E{2WBC+1Jb&5($9B%=u0yl|Rx5Xv^Z z$XY17aUiTkx+S+zKkqC4n?HL#HMq^T79|dfI~o!P>k|6zS2*uLr!-tfj=n$By~JxG z+~*+$b&_jrbKW3VIh78FrW?kjZ2)x%^o_WtV%FFjCZ%POpI@F zmDd|uj@JHVh3r1|Qa8K1K=_U!dyvxBQ=d`6TD$3HK1FylNeFnyWAi$m$!)nOFR1=n z?4~?`%_X}K%#!31@%>_aLw$d{5*`^35bOIIbn~2`lRRON-x#P$oT8Klk8=*80Qdox07Skf_n` zCOpjJ7k?OK0xE&;0)IT-(GR(2)z1DUF(AZMICx9Zg@9%VjS7&Ol$UrVl~FXDl188B zSxKcZ7A+zge`{yt%k$0P=O3g%^ivRb8Qvm2p`0IBo%o=^cSzZy(d@*DnR{)x6nPme zd5K;p0J|PNzUg=2fbfZ>iQ6VGVGpVOy2sz#RG9Lzb7u3@giU&bxsrV)(@N!m)>c`w z(^!X8qOa%$&A-=+5CYc7!K?}*6$iw6!LcqJeCS!8m!NqJesx-@gjcW~?23)ERMdRT z`XT(rU2o`qRo~2T#NGFmmTV^Z7g0x(8?d#T%3v4InCbVh_9teWB-&wimTx}0R2tH- z3Ff~~uo_H`e&8(dj^op9x)<_mc#U&ffu1i%@^{u7H^vPYU=`+ZFXtZ)ol4CU34vPr zD2UPY0wQ6+@&K++q3_|HBWKb zv&X^7itQH~4-FOMpO{thYSwFN;WENh8JU}`$D_hn?|nPAjod=6E_RFC4TqnD^AsDa zIAe%O|NWK2ioumx{;vvF1^}3^>*D`fB*qzp#eiVgD!AVNyS1<2#*kem03gU+*w-)E z-O(3Yj9|d}B4EUJFl-v^2KEJlmgV1FAOZjkhyeh)|A5zHBu1G7>&-#`&mwFyCoScF SbCH-#EHw`|nFjYin*RZ~(|yta delta 5140 zcmYkAcQhPK_s3UXgw>*3ZI_J_B6{y#SiMCTHfm%gN?40%A=p(DL&Tsy>^T*8n+_~SGGw0k}XxL!*VTgq4i${gCKPdnZ%z#A!*|5J! zC{VM$%koTNDvtD8VV-Z%_7QY+;;+U+ere-1tk|-$2{p#MhGz}(JQh$GyaIUO?I4uD&*?Lzm8?SftK=_VB%k zZa_^v!GpU>?T0Mo06op;#FawnPF;2&o3bO6OhzXPmO5gtOwPYYIA8nI2Wmqkk*EJc zG46LS*mLk%&tcP|BQ>i5B?S_8CZ`%oikTYqg|_TR#R;1kZ1nL_5;x6Gk=mG5l)f%M zmlh>ru-uW~zzuPKygzciU`g9xW1XadW!0dw3M3Z(S zV1b$}nkPEl-YhP{gVJDkyB{q)`dmviB zB>gh)N=hs+O-W2t^NR@`G@Ejllz9G9j+p3eyDCqsPybumM{0qBKvHwm5fwUO(}ADP zPD?4Y$ zSu}aZNY+%FUp_!pQ>0jtIqxYBxOTb;YOF>2-&tSfGiZEBb$nhmUMt!G#C-!91hP#{ zx_w-`+qvpHT(SBP99>r7*?+rGdJ_1e8vYSD*g1JV>JwtrQZ;|E&O7wO4H15apDz-! zy4LY2A$4eb=WYPzf-|+i0=3&MRq-QWlcs5AU*YA|u)_qZM{9agi=t$yfELkV0YU8GW~@Fi@Qd+t}x?B+ff@}7RCQ*M0b~jIOoKV z^sCq5oXm@$3gurzrAY0sm_)r-0sO_)4pMu8waUI(@)tH2w=OyGZL0?`IJI5M6a@v} zKqmQ>%^Q=Ti@hi>VOM8GQ8SmgO!e?l?7BLtjW4|np*iv>jMbS*q(FpTtnO)`tZ7LH ztY|d+rg^rYia~Wd%TDk`D?#DdrIDhSkP`2gFVQk+5NI0_Eyd2%fw3p^&R&_*)e~W? zdmVst`MakbDC!BnKj09i_B?-P%~vZjPCQUh#n)B8XH7eO&A8Hc=5>h4r005-D zHY$K1(*PXOKw z)J)^s`T1vfxiU=gy!VDtQMNa3L;{ZTxmrKIlYlZ7FfGxy><~eqa9GJ)ZYlP7mDGaF ziK3<)cPni;X9(g@8Z3he7HM6QfaORdjcV8nSiVeH(CF;(25J_;7e6PbwgXp1?prW? z#$`VEZYBV|z-=tOj|voqNQr=_TU5?ft{wMVTSn zKV(D9m))rc=05Kx%8vc@wm$#($Ic<=sG{n2<2)1027H3rP^wM((W~LGXSN^W$nHz4% zP*rv9BjJ>k(QOh+w(kZ!yhWj85I1ByQT%gmS!xx3=JHR)AkIwd71}3_-8?7Zqzsr{ zK_1?d@HK|m#EThiVr9Da2aq?EDI6lI}zt`elhCtF0chC6jk}Lw;4=}dHyKuP${_n&{Tf>3&Gh+6A@k4! z<2{s1#se?hH(ROSMKdtg^7eNc*&!X0a1~A&ZKwK6OD+AGR~8TM+g(}_DtF7n<0q+# z<|m#LIB(=871YQfSKtb(3@@GuJZX$pqku5U^Ar$>D1%Rjw-`2p8Z}hr4LVzRa>DVu zG1%lNFv!o8&qp5;$6GT5&1{6=gj>}4`cPQ(`uf6yhclU(78Z4ndVieDCApv^f(XAX z@0XuY7s!BksuIj;dG@B}d8-i9G&*<1<7ml8mv}Ji>uCle`y$B3XfzPQq{&mjLrh6F zdW$GE_AcGOaU(aw_l?ME~#rd&4uY8ED4uptL$zxLL3p z0SuHHrS>kVu?1bw#A_IYm=c<5z&S?9wo~T=3WBzS^@66MIrfA|uwO)l2EvfLJEc|( z9X%H*++h1veIyz4ZNhHq_lmVjsFAH!2n8T$dk5e-&80C;eay&C?r(Ds2yx0GPBM}o zBVzC688ZQ<#+U*nX;HX!?{YQHLMm9C`%Pp6O9QML1NQ8G8%iZo1}faR{R2t>Ce5Xk z=U%qc@Hq-}=JXb%9c%?BGRg(f1q!~VzLLPEy9EK#WPZK$c~@Ik;?8Ume>7uEx?m+^ zqz|3`iqxU=lucLddPmBx4W0HE(qc=`C)Re?BfGC=%y8RU4+WB3-b~YHpVWaPF`Df1 z20O?rQT3*~T5IN;w)f8g85S!+{q{{eS=PJ!(>|@Ys6KQH8VN>n&u20BV7$C@T?V8$ z4f36>iD|kpUa}b>Y#I5)G1vI-+dV8I2tANwXH_vgjIQU<1g-5E8GyUu_P5b`8M{@E zSExT>#{G|kLrX!v@T;`J;okApAvCmcr*V77!LE5 zPt(I!y}mX}hQu_k44$mm#dmM=y#1t7^awyD)I*W-{Fc_^PKfAFw6=YegkI4pETO`= zN!wH2z1l_;`{K-7$o_o+M70F86j1-ARBvhOFSlxS!WOMV#u*ZVKy`@rg2|)hp|7MW-Zw4Hd|7L)?-qdFC$`9C& zayhEQkb+#s=A2DsHf_zAhA?E>oJ8Kz6Y(UH(sj!K8jpRzUD`2m0W??a9h=IvIwC;v9^H` zy(JD@`Ag>D#VIw^^|K&+O&gTxqAiPn+dP<|pMXE^&=1L{4;Sjpk>g2tli*JydlUCH zw+nZE(#PjFQTLetiI0M1y<$UMp(}kUz+iE(pqDGZ3ip`8Q#OMl=xIufGo|E>&k|?X zL}!}^i}VwY|0T9v4L*~HC5SqqV?>>V>o-4Yua2N*P+>&^d0th&mIs)t)Z+@btr{W> z7Do%D!Pm&-Yh)&cY;a1w1<=ydhjiRcKz%zYfRoJ8|(sR>z4E{zJ*5g794`}o!$6opo|Duwyws&#OE!P z(mP`P2R3B)b7w;Tm62B4+MHVpE`g50;;g)PgOk3rhf0|0g~zJmyv|G>mh3RA-J;50 zxkvw^`5)pAIDg3mytt#rP7d7weLooeJg4-Ip4>gVMaN?YY&k}f)M!KneSDM;YfjW+ zZ%&*Pesc7&*G+!#5tGWFifRmMj@G7ivjv~$P2_}N?y0IeINB~q?5pN49oO6|@@}+D zogJIxzHod|)H1oy*WO=qSU4lzuXHhw{5yS)5a$pc;4WXF{II`3N!TAeAN}hhl>R69 zwq4n?R~}Dc#+DW_#zm82q_6|oxMiL&hq}Bg%?_8%x-QS-@EKf>$g!&(??U^*f{o7Z zvtM0e372Y!TQ-{Y*m>E~pdmS;gbNvl?3V2T>e>_K?OJIj%_h`#a#PV2UNMJM~| zIH4o>EA6GV@0uYRZ0ox>EZDT*&B0jEz+nT&HxzNr8hsWKrBeWPfYWsuh{VP+ilAtM z%VFhI8&p1ybhV}Pf^HbfAs?4@mr{eH={II$w>boBTk9DZmf1qVBSCq4;^);vrbvqC zy0Qv8&f`10PQ1ws^>RD+9dp^kL6mwnnr6d`jZZ;^NA7?6On~tdmI5zB8%)=9GN7Gy z{qMwsX$@^$9Nf&y-t_b`1QSUzG@t+y2BI#r`1H}}zb5?#0?DD>a1p4%rKs~qjSt^< zlVbSTXjV6JbW&(V<=Aa+ejv%@6}N?ajCZWjOb}=Uz)Py*OKuIp?kI-(kgAPmLG5e^ z6Tchgn-Rj&=8vDS^&vX*DINEAKSjUI_0+3Te9qkEJ;snaNd3x)ASf;)V2X;@5_C`N z9~nw_&D&q*nDcU5l{YAkE=0&S73t|u>%Gzq8*fcXks75AWcil8P6ko~;!j4dCRn@Z zOkwXPSj>F(Fn#+JqP4{i{y6SsTh$5MTM5-7p6|$HiExF}=ZWdb0&7|I&a=`xpY$l^ z{g&y|%_l3GDa%dc1H+})Y*4)xXm4LD$u#{aZK69&`jJl2!HAV3A+x3J&6ipD0CqJ) z3uvYK@UojL*ZajI0fpkYDEn1gy=F<%GG8NnecP;&0s%4Gt^G9cbtj|oW3AF9gxC1* zv^3@@X0)a2I%93)73B@GernD$a_4E*17ed>=`zvrlLgn6U|;U0D3nLWw4QoPi<}eJ z##GPh-|1V*>bIqpUOPpD0Rhpcd2EdiCL|tIz%Oktl%nys+l|CB%1@A0ws|=tBPEl* zEwh2#uSu3$+U*recCj5|Bi%AwAdsz+YiyRZI!Efs!X$KSrOas-zv$gMbVSI|fc>D~ zXDX`5rlPx1dwNS%0F_wJI%U$&e9I@`#F+a1yq1Noc3Sr%o7OkImR`z%f~?5JdldMa z)=8mUjSFZmo%to}&t^$ak+O56OCP3|9)9F^n4=fliDP-%{X${6$Mz$gPyP&Mlibw} z(WqhRoTq~18~$2vVrncw_-0}?PTA-xQx^jkJqfq>J4BsDZf-61d-nH)w}-oz z2o1>D#3B1uyo1E}p50$n>IyneMTyS% ziMdAU&AWs4EYM4zAbwwpm9?L_*|cSE(*&Ofu_@{tUCIZs^d1fUq!222Zq$2|-nXe4 zZ#ebYc5O5)z6h1^sP-+jqRgbShhI@P4y|%gdz|PSQL}j#>ye_iI48-HVc1i6$))p6 zrK$Y^c82nls&rETAd2@Te4?#2+`1W?SFGclpDRwgr4{Hbm8mE#WG*8#1TH$={MKi1 zv005hp=T#8Vf17i8}BwWTs{u%YV>Nx|JpYC(v5s(mt-~JSjtjf^Ygj7y$_!K&Cpu; z>Tv)(X%x>sfAL8Govim)Z38JCT+1M}b=JLi{)yVYZ|A7)*Id%5KGi8}SQ+hBgW){r zQJE;Z2>Q3fuFhlOH8h&@f}&=J#TiIe9Bd*5#hUX7q3k+*=;}#)25@L~GusfxF#-Cz z@RSMp@H3&8KOiY)$>rG>PoBW*zLra$kgdNRl@hAvgyuKVoBDbEX>%xfoOb-F)~RCD zqzJ5NEBW+hLmCG~<*>ECTBI?bZs?#-(i2|mgsOD~lP?~l1x1Z_?p4tlH1uTb`PGvd z?w^B&fhfZdpMDc%eT$FNE?#6DyKH;{0KP0BBa;uZcOz2{C6QhAlfE(d+-!hg*N9Mx z$A<2%x@|Sz{`DYAYEA6-`sS*SwN(14zYy&^9BJts5p}QB%{GzcCEK+wp}Zh|HN``C z`=0defGv{LZonVbqvB^@^75t)CQ@?J5cfPqLfgOV{EF`g-1Sn?D^{;V$R2o zsK_-?O&V`BG1 Date: Sun, 2 Mar 2025 02:37:31 +0000 Subject: [PATCH 217/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9762 -> 9763 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 899fbd08a..9b88eda25 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-01 05:35:19,524 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-01 05:35:19,524 submission_checker1.py:3270 INFO] --- -[2025-03-01 05:35:19,524 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-02 02:37:26,577 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-02 02:37:26,577 submission_checker1.py:3270 INFO] --- +[2025-03-02 02:37:26,577 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-01 05:35:19,524 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-02 02:37:26,577 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-01 05:35:19,524 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-02 02:37:26,577 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-01 05:35:19,524 submission_checker1.py:3286 INFO] --- -[2025-03-01 05:35:19,524 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-01 05:35:19,525 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-01 05:35:19,525 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-01 05:35:19,525 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-01 05:35:19,525 submission_checker1.py:3307 INFO] --- -[2025-03-01 05:35:19,525 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-01 05:35:20,700 module.py:5474 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-02 02:37:26,577 submission_checker1.py:3286 INFO] --- +[2025-03-02 02:37:26,577 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-02 02:37:26,577 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-02 02:37:26,577 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-02 02:37:26,577 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-02 02:37:26,577 submission_checker1.py:3307 INFO] --- +[2025-03-02 02:37:26,577 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-02 02:37:27,517 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -23,7 +23,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.298773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 874a6f88d..af6d5cfda 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -844 +845 diff --git a/summary.csv b/summary.csv index 908c752e7..2f137ebe0 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.289104,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.298773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.296426,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72551,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4a9d059da..1aeb56e72 100644 --- a/summary.json +++ b/summary.json @@ -69,7 +69,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.298773, + "Result":0.296426, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 0184aeaed3bf6d3ac91500e1924496a9da7215d7..93268afc544b321fa86688deed90b2c1ab189a79 100644 GIT binary patch delta 1337 zcmV-91;+ZKOruP&bqEBFQ&-Bfc?g>Yf2&z^mLc$o=#k(Djiyb(1c#9Z1(Rzym_Wn8 z1R4e=*Kjb6GB7~`24+cu&gS9^n6$(Lm9Xs_7S%;j3`}s0&A~L@G?-*J=dUwaMfdh% z(%5CvafQ|_U=p9*9|?XWZaS1nuCbexNv`2c0u5slXc&{=pr4*=Z#0lePzgO~e=JMT z*<5^Kl9qU&5?7~8(x@(qVry8g5t^V(HmhZFexFT+^A(f?$n;^dm@V^ZQn+>fJCLlR zcY7^go7>CRZ1rlIMa$O$kBQ)>0n?>Ka*b_LBDsbW2{epIpkYLEjYH2qHX2AIsDu$& zlAyD>_(CKt@lY%2zAKCBf+$WTf6%aRmF61)@}Jmzt)zo{F==iwsT*~zDT-#v0vnD1 zCL8L*0UDbcae#(1xrY@8X&95h?82BVbK{^&7?Wjw9Kj4Zti(gDWK1$o4v69_SFQmo z*D8HE{|c0E)ukhtO~=@>aqLpLW7N@Js020~>eK-mn_6{%hEutRSqEttf0aOEQ!2~+ zIw%37vdpm~sG;VZm3W{Mekj6>IUtH3ih#`QP^4mS4u|r`pNhOu&(4k5Htks1>fF|f zPDPvoDuE}5+IE1(roJ7Z;Z*M7+(8;fCD1S`%gj3{0i&|az9Xo+=A4yypc20Q%3L}q zimhLPhOJ*KHtM`a<&W8Ze|@7Co@=q~;mOvpLWlx0op1}d1P&c);sF|)x_E$wbGe6) z2Wc3WK*P8!v+|$>jLR}Jk6>;dR^ow5IG4=O1ETmEmTSNowqjGyzXRqQwf1}y+uojR z9cvhxQ?TQsYnV)(8$~Q??g1KyntOnTk%=^vOr)V?A`K-|FEjU`e*~1wvIHHSnR{4? z2P$FbA}p$lq9~a_!^zYucK5tS=HERSsic4VK&flHP->>`7C|WrpyWmoi<*0Y#-ZjO zpy5#NVeUa11|?_@-SW}P%sr?Q24$JKM`z|9R^ow5T!TW%%snWIE@P2~E@SnI%{{L% znGSLs#dm$HtfX)If2>~hZBtLS0I)QR;@H^wEi*H>JzJ_`enh zcgsJHf01YPMbmzmJNo_c@=y8Ga5_|yKbAidVC6p&pykPe!S#GBmGpTgTdI7TWp^m= ztg7^%2FsrZP-bx$#bNrqtUdpIg8A5QPon<&MWg1^@Q=BCS^qA0XC1r?tPejc`Uj<@ zWL;lNv)(Og$zPymwGU_hP+s`)Fq|gJSPrs|YW3_3c)a|WwH1?Q#FHOe!B642ZTskb%ySS`{=l=#53 zV6D+mqaHk(H+L1clp6I}sE$MaO*{b( vxvxJ#^9Qq(A<6;-jZ;_3vtlGY0R`R3eq1G!q$N}Vm6Q1;BL+t!00000U`~N{ delta 1344 zcmV-G1;6^EOrlJ%bqEB?y72Y0c?g>Yf5)~9wsQnN5j_(8pwYA`nBXwdpkQ(h2NP%* zm_Wn8N+j2CB7uex2{epIu5swu$3_E*1eGu% zOA>T87hi~^B_3)e-FIbCT@b~Ie*_x#tib(ua$IgFDA_`CUv8(HAT@ZSzyBv zz+^*xI6z}lBM#7TCik%7APr*@m|Yl?Wo{f)31hO%k0Y2Nhn0A!m5fQ|$pKM(<;pc+ z%RF*k*1U1y0vl0(f!Vg85F$YBPLlKaf9g0-!&EZh~_*0QL>e;yw+ol~WTbgB*wnWJG@QykoI6Ovs012DWtn*gC16yR*>?nW*POEw4^+apUztk> zMX~iO(6IGu#YUaisQfY8f3I)U!gDRQJv`YORtQnF`|1{O2^>1q!~--ob@2cV=W-7p z57IC$frfEeX5~Q%7?)*c9>Lr^ti%JAa4wml2So8TEZ2ZFY{jOYe+SGrYVG+Zw!J;s zI@T~Wr(nlN*D#qnH;P!)+ygWYHTM7wBNJ&TnMgy)L>fw_US{q=e+ej=WeGYuGxx9( z4^+a=MOai9MNu+=hLfpR?CyDu%)fguQc3^zfl}9Yq0~&>ErL=MK*^0F7B%+(jYG{n zK*OQj!`y>33`)=*y5*ynnR`$r49YTdkIu|Jti%JAxCVuinR`$aUB)5}UB>Dan|oek zG9BbNitqYXSxMjae_6fi+oqmu0bpqs#j&yV$L$jt59Ns*j}s+`vf)}rZc3Mb@qaB2 z?v{TX|02)oi>CcBcl7(?<)8AW;dH1Ze=L6_z{-ClK+BT_gX{TND(Uk~wp95v%kEI# zSykyj4VFI-pv>Yhio^7IS$qEb1oN@qo<#lki$=|-;U9DPUH)D0&N_G(SRZ~?^bbl) z$-2IlX1!a~lD|OBY9G%0p}g?pVK_~cu^eO_)#}+7@Ob%y1B=&bG+VqfRpmZP(`d4O zH5=#yRGXREgB$HGiD}N@X2-|_S z|AW`w%R}*z-Lo=CkO~QQR7* zScs+DVaxb8+W{^JxyK3u6Mrl)d<3R;BBB#g@Cj*uTsr^+$IyZT9X!sn9ACTvobs~` zgL36mGV~T^t!Hs^AH1B5UDst@Iap)<9DlA~-iB3EsgI}PfSO8h${ny^TvOreA|9?~ zKCI)3w-Ur-AN85H(;{OZh$IG}W8AayQ9eCyR;VdRu}Ddol6*rJv=>~^f_xoE-t=F9 z>NMov#4}*beftTTKeLq~$^rz+y72Y0V Date: Sun, 2 Mar 2025 05:40:38 +0000 Subject: [PATCH 218/719] Updated results summary [skip ci] --- README.md | 68 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 8 ++--- summary.json | 22 +++++++------- summary.xlsx | Bin 9763 -> 9782 bytes summary_results.json | 22 +++++++------- 6 files changed, 61 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 9b88eda25..ff249df3a 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,42 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-02 02:37:26,577 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-02 02:37:26,577 submission_checker1.py:3270 INFO] --- -[2025-03-02 02:37:26,577 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-02 05:40:33,740 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-02 05:40:33,740 submission_checker1.py:3270 INFO] --- +[2025-03-02 05:40:33,740 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-02 02:37:26,577 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-02 05:40:33,740 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-02 02:37:26,577 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-02 05:40:33,740 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-02 02:37:26,577 submission_checker1.py:3286 INFO] --- -[2025-03-02 02:37:26,577 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-02 02:37:26,577 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-02 02:37:26,577 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-02 02:37:26,577 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-02 02:37:26,577 submission_checker1.py:3307 INFO] --- -[2025-03-02 02:37:26,577 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-02 02:37:27,517 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-02 05:40:33,740 submission_checker1.py:3286 INFO] --- +[2025-03-02 05:40:33,740 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-02 05:40:33,740 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-02 05:40:33,741 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-02 05:40:33,741 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-02 05:40:33,741 submission_checker1.py:3307 INFO] --- +[2025-03-02 05:40:33,741 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-02 05:40:34,847 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.6061 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.9238 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433081 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.358519 | CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:-------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2026 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433081 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index af6d5cfda..5559899bb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -845 +846 diff --git a/summary.csv b/summary.csv index 2f137ebe0..da708b209 100644 --- a/summary.csv +++ b/summary.csv @@ -7,14 +7,14 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.6061,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.9238,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2026,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433081,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405291,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0414,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.358519,CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1aeb56e72..8ce33c53a 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.6061, + "Result":20.9481, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.9238, - "Accuracy":"acc: 75.800", + "Result":10.2026, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -522,14 +522,14 @@ "Result":47.3803, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, + "accelerators_per_node":1, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -549,17 +549,17 @@ "Model":"stable-diffusion-xl", "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.358519, - "Accuracy":"CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, + "accelerators_per_node":1, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 93268afc544b321fa86688deed90b2c1ab189a79..eaf6629aa1b4190891930611dd67f6f35f364ca0 100644 GIT binary patch delta 5192 zcmY*dcU05OwhbgyAylQ;Bs6IP(z}3E=~9i9(1}z55%>{6q)G|B_o{>DGP#lW!=w5PL#Epg>wIK+1$Y0+Rv7 z()1xKk1+-umM*lyTBPyGab-K5J3=3xH0sk3(2au}-KR#nIarb(_ylD!?dy~VYxM-J zw61&%?bXfMx>W!ZIfQ&@>FKA03SyeHbe+10sF+aS9fT@^ZPe#4b3k@fnLZk2c zLNB8!ncur;zH%|&D4~w0(WqA*I(n9zw#Z6VY@%p<&KNfUJIxl)kpES7mwl$@N((Oz z_@kq2&XvVz$!YO~8Ii;8T08zfj#Q!ZF|IC)Q?gZZj%sjdmG0>Fsf?CcbRwV;7(+W5 zH`}d|H$2%^mP8@~Yk(<~Bx$DD!nBH`=o=Sm*TOavKERyN1%TDE6uh{jX!= zfku(DQ6!p~-7K{w%~I*Y2f*aC{M-V&qioR)yK$OM^nmsK0FS6Qd~yLPZ1IPod{5Ol z)XuF2f*9;>4-2z9n0;;M)Mi@^=L2Fh_M+~|53@6GH0L38j)oJcNh{7Em{3##4QWOv zFDXQ~#u!5AILkzM+{Cp-3hB7K0U^`|qLrb|TxPNZMANz4)CNm#0EDpWdAPFjQx6)_ ziUT%6g6YotoE%ZJ{k%ElETBfENiH^nMlwQzb9NYY;^E&(9yhgU_Lw9!ic9wXwhi;6 zZv~H^y)AUD!#yGT97>!XgN7}B~>%AS+ z2d9XUr5vkj6ir0c8)v+5(Vc1Xkt)+J^5H(z@p@Bd(~{-kpV-7(UTtFC88kF1Szf0$ z&VxgnepgH~_>fmavaW zMtOpZnokx&s6M;GB^v=%Zw?Qce_0d`HeSN;$PTqm$Z+mi4G8cZKDq+<+CC2LP^q1b zuh8c1`vxB#g})hlZ@RTt1*Ysvg6@RA@C#Ee*rg^lVI#y;-Q%=zQatsqfEI?ki};O; z1XsvPivri2*_K0Oq-041;muK|bGOd(tJ_>HZWg)Xsazb^2G104*_?Hg9*Iw0fCFqebbwy;xajKqDl`1rpg#{xy=@IXAx|V8XA={$b7V4swk}7$4oV1qh14U(!TKo1y76q zjBb5cw;5F2b?fbG9IBzKM0GsKCq0pqTXzYMTgWh~Yc)hXlapk=f*9z4A@m>+h!g~( z04afv78ujj$UvY!*bq`Kz{+@G=+=vsk2r(MlSGqvbmKu$C_g7zFJC0J&91zm>XU`4 zI>7|E&-oebJhJmF`3NNj+s#QIc=BdRYRAXJU!>z|&shE;J1R>Zzpi`}t!KIp*Znnt z3Mg2%2A;H5C2#bWaKvGwI@Npaz? zb9FaLTEsLbrH5jhcWSYE#!TP0RE}GU-LjUV!<~zSUFUTCD`89&MFt_deHMbe=S9gT zufjXFLnLF$G6P#y5@Oso0#Zz%&`+>B|C4OtPh+cpI`$Vd6&+b{sos2NS|`uXH(vt< zuBC7L-{x;Ewdke-U96A>uaH#vkfaIE6ox6sR4!YwvCS(1FO$*I8Ue2(-vZ{U6UHB> zh8A^VF~5Q>9=xrJXb{w2Te+1!P#>mVQJY?;O1I~X6Vm-K^X4Q(K4C+wtik!|a({vf zJL^F?zBqZzYN$-*nbSaC4Z{>|TItI0TD?cEL+^BzGXFb6U~IX+ux{lU-)*Eo*HogL zrRTjHPo&6Y)q$DZNw0F>84`3mm6^4zWl6TOorg{hW?(*T5ft4n)JjC_AvDiigL&yz zOAcWYHcJc%0ZhFvtJXG62hq89ZY4bImH4%%Vn#JbLz<6Gt-GsQR>=d6_jc!)D~aw0 zqK=9PV`>jB+Ep!R7z*p`Gex#m*Y)dQ;IP=y{R>z1# z+KNNNh4k?7C~y1o+Rv7}`lUHr=DA|2yFown(~c`gV<>p`1dTIi+BVxgWzjFDEN*A& z<^hxQa1rmy8Kb8RJXjS86Tt97jH<}1EM)GGqQ~{}O*Gd{nQFl{$tzsZ;}wQW+E|KW zG`YUGs1MX}Y;`n=cRxeN{UqsnvvD$m;ZhzC* zmpT4{P1)U!PcAgzt9z)kEV};<;lv?D<7OPLRUfv5jVOvf@G9nsHz9?l?6<;^y z^;L?2tRrF4a?Y8E3;7((wh^@oh095pj=VP@ymZAzoy|B5yFx?Neiyp_1%jU29$@A2eda7=-65w0#g~p8YRw*LV&<2= z>kwzgaLs#MYRuYP<1Hn5Q$QoJSCl{j=Zxw2qsP|8L)t;(QTLna%}uD(CiLFjD4=0( z% zOZ9xI)?dddRSH&32q}wcof%WD`zz_6nuL)vrry2jOQfJF?SxQt9ss8JQzfci7HxA! zOZcWPZvU~0@V1W@to+p2^pij^oAIl#+eW>;U2FWJpRcirO*B>3D~M8ioqG&oR$6v2 z*-bi4xxW^|9Eq6q6#OA8s6s?Bl11LO7b{{uNy{P8S65=FKt0%GfYERc^RB$&@SGoD z^q+tD$j|+UkkOfY!!96t?+a`K)~nAf^fHX>0gRHIXVy~`6Nw>{pwO*ZwPAj0bDeU= zb;`30k9$h4Q;yhQsjXm-v9k&Dafg;2m-WIIWPn^ePh>M4_MIfQhF78?Oe-h=fd9;F z*)Za1eRqc+Yu%p-VSWc67v<^IQc{>hx^ZF)XaXnA%~SyY;NursmTTb}^5J4y#esY= z8@1LjiCi%k$&vMMUFPIYI(%e_=lA3v3ynA!1)wiU65Z{5nVQTsP3l1K{^D)l?un8@I2 z;O#PX;?=-`8w|V{t>qT3wdCwpR$+;_9VAWI;$A^R;>@}`K)^h<4n?w-*y<8WLGA0u zUWGH;>dr8ELwwe2|FWEkDr2P{6gR6O$aKTL%K?ann_Z`9M$7Hpe@Zu1E_&}eiVk?!JT{#IGUVYgU5G{%lhaJ&}MWLrJvX7VTJa#qT9trein7S7|2zNu<5 zw-aI3i#XagV4=E(d=Hk$8DYzj*X(b(R?Z2kfzAg}HKU#=1YVgI8 zYa0?E{I3z0t`VPJBfj(|@u!*;H+^AeC-2P*NRe0z%uvOZ^jWm(Il*WNr(F_ef` zXU=-JOWFP$X)nf+A|1o=>*4&+oF5+fL(dG*PXYz-Si$62fMxa(Uq1cyfE<2XBi&hU z??-Jvv#)N~`M3W({mJlb0x7n0zqDM5QRGjCQUcFC#D)E_IDy}}IDw^wg>*|+d}*?h z{su-Mzig0AfTUXBvb=YVgfPX08R3$mBw{H76xv8aprFNk@uzTPCkwtnWa=chP#_$RH}e?gx481{U(Nsf;KF9ZnHwOqfdo*g2@-_kwF~s4 z1%c>6*gx=FK%<$S@mFc;&O@AmgNSFX(D{PN5PfPyCH><{NYk9LcHPE+T+@0Zis$!5 zs+Nsd+D#jbu>h_#WcOkB!}cq*@%k5xiM;*yw=Z?)z!C2R@!Pv^{r#UmiBRYvDkB?X zLX>yj?|Z&N41AYiP<~SS`)%{BBKbN&cAOTb6|b#Y3LqRzF)w~wy%9t{gzyIGMe%u$ zM2KTGvIa2kTJ$dI)y!eWuG-BR2Ey z(3vNIC5HyNRTRG|BV3Dyc)F+=E|{ZO3|ALPrF!ep)ep_R{IF%uG_+W{9F#~m(!7gg zgokr{0Y;wUS5YWr4+`=ss#Of5D=1x!)n#oqgIbJL-8iX8oF3t(3o}9Y&x6OjJ4R1* zE3(;JIn`M@)BFo_o>|sqc#hFuZRC^Nvtn~h%qwE+>I24%j_J&_jc}O;R9^43CN21r z>uWq;L!`M?HE=(fv^mhD;)Mj)*UsY1mj^6>er-?vlMOxdcUKu{A-+MohivR@vI|O zbg_EddoGVsn{?a*teRz>t-9dGnUPIRlpD20CD8L#&RcM z6lVq#&HJMOQTZa*;~&peO*($VIh(n*GYS0Ata0qgL07^A*-;XcG7G^GWRACoY;=gK zh|QB9GvpgY5AuBObEhj$r0nawU5MY^PFAV*!sW(r-C5}6;~E8V3iMgJlbl2+s5Lh) z$tjyU^rlE3hu9CRByGk%V&SxGZ>oJjIzl2Zv`73=MTS5hzhPAwH+OMMe=R(9n|i#8 z{>r!K7^DUVeeX&qU*(|;dWC};t zG_@HmxLU&&8YKpQ*miqVJfOUfv43~g{#{OdT64@;``a5-w-Ol{7ed5q5Whx{S@=u9 z8Gdm!_fMbew!3o?j`C11;;_2IjFC!d{}6Gpui$crmMq#~c6y9)Ft=f96W;Xj*2NyuOi#x_c+McX?`gyT|5 zzD-AG;hf?_-)KO;5w$bBi99>7u$gV3e@^b#Jg4AN)oh-lryt6ai7cVe0Yv&6(4Ubl zQt20EhZnueNAB53(hc9s;GoVH*2g*nW*VYY6e)^(Fb~!v3PsNH z^ff*@T0#F5?h_R{k&OGOKJ8j~^~vbgu4b^XKC7SZsy=2|D5)Iv18>#fw30p8%J`e) z|M0+|Zh(+&rpu!Di&A)B;GDwDmnXAsrwn+Cft0 z$6W+7IwErctfdGc@5eEOZ1Q6btofY;IVVyq8XJ9;C`Q4>w|N8j&Qou(e}eOWY&gfC z`}}SVe8ys?SGt@Z19lbY<@fIi^U`}E5*yI+>by3F7pX&?aPN~x2OERTcZ{YTC@|`| zk>YUqmS1#M82xEV3O~Q4SUIt{>d5?TNJBVR8X?(#zwEHwa0QnCT4fawi1FG%|7V0^ z9I%0Kb%GUa51gCj{~z%Gi2;E^F)r9kxC$WwBUTw9L*RoAMDVcuWgS5P0vQm3Ks5gw XdmU_oZAT~)RA7H0V8oWNzf1oDJqC<@ delta 5166 zcmX|_cQhMr-^L??#1^x$N&IT>S-U8;_g1wxwPzxzJ)$8Yc0YA!|e z@@cWnsZXVB-A%0sv{iP<75Qvvhr&3hrU^~}c>|Z`*2(xaEOJ}tgZ)3WwepO;Th=VS zOrwE~=;<1H*?7QNUohy*sgE9>2?@SbglVP4Rv$~UrKN1i*Qgq#*4{Ax zROcgIGbkl=3|N>v?x(RG@$1lxb}8M*d!!o(eug zW{Kohg&FcwIi+~$N9*#nm?dXfLbc(~R+15{GBNesioT?>3rFuAM#4WW^;8o&R5~l2 zrtR;(=VzR*WO!EGZYhCBm#)Yo%I3_#2i;pRSi+kBUB8+wAcsB_?2F#7G)oPVz_16ad^`nwn}2UlPZ>a z+?l$hI%t&VuRPu{9HEh?)K6jhmqn7mprSlZ3)}3>cG5o_&tU-9p0J_g-HhqMg*`Q3 zkU_IF2;HBYJS{J`D;;ve2W=Q}!wX46-=k zLdWl@@?hu&ch?xF2sH7_+`AZ-+U09(>@{}ugz365ea$!g$_{R&cjO6`$Pzo5c_ue4 zw|gebQTUB^xy(u)6Os}1#zBMLdHCZM)F6z)YM#bj58R zlTa`yz4`T>P%vkD0NrIhl{N*1z$;S2P)tZei2D18R7;aj=lfSCp{2P5?$rI3*ZfK! zyw9Bu5}T$*EWhs7lhX9J(0ZdEhG(jm?a+fQuBr96d7M2}PfluS%M<;?uU9R$R!Knu zHOtw4zXqh(sYD?Hzs4@IoRv1b3O3!U@EfJ`c3)pOQDt#H?*ku7&z+N!OkIeQ{Kl(9 z_%b|O@br(n>zO1vR`^s&~iz3=mmy=o5h#d9+6xFDWKvEGN= zzaS5Z+-hANQY^y+DMWIl@E2BnbR2qQ6=R*k?+(k9f^k}D z!c!>L&_p$GPo;Qrj2ksg(J8r5TSK=+4VkV@r3Qn@nBg0o-c(lJpp@xr!2%&G z?m!D-0?R`F-U6!y81KJt1y2Q=M;vd+Ra^K4Sm=E+&eBmMG0`UlF#!MoFzt1r2FNnE z9@m2a02kO;FfZyu$g&R8@anm>mrRtd2c;Jx^aq<#2UcNNDxBj(iK=9)jk>wI1+@3# zB>&}&sWUi1rb+p`KCt1SxJOqleCcv>P23pUa?G;<&#V@;oR_uCCGhN|5bo>u^qe__ zSRnK+y;r&Km@k=?x>x+2`@4N8nT|5nF{{3dbH=yNo1*Yjw6H8V+gMuZd}UFwhc&sW z@)2RrU)c}dX7lG#d&X^LM)sYL!9`HLm^8=W`Il7kL~aTzaK7okOQ62&{u&{{L{Ci>EBMr8n*iISaK=4>5F4mPQ~gEdCyB+{pK7w0qG>O zA!PL@81;F2WIOEVsdn;!DW2wWQ3_~Q!S-_VM8O&Ak(M z)tP$eBvuB0@v0kc1;Lf~+YM0~`c-^<<>iA5*gDyW60w%da&O9DLH;!Fi(|W!bGN&e zx9IGwgn{DK6B>O?0rFkOj}BQYGU6@}v3o}~8b#;7@%#w$kCR~UEQZ}KNS|uIgA!&N zBbPPzpfdD5xql}O(P36bPZ~YAoWBA{>k;lEgL+yAQ^5CiS_zJ%5kzMQXj{x06ON3r zeDI@w{kP?iW06Zujo}JGAcLr9+|`dziTh6mC;ZJ&bfSe|&iMQ1=$Df`<$)1(P1OE5 z1cMnpS*vzY?!7AhW}lo`OP zJNnHD2)Xf5n1{FWoLN`o;*ISoy3GcNRSj}*$-NcXk09SZxWmJHc?p6)?JMmXyoj`R8P2m%M2Mqh}|e^C}s*=40gW*Ic1RiT8i_#_)w!=a;_7?`s7IDUU_)s0OPY<%{{~IjJiKG~mjY(aHL z@4U&k7xiOQOMSGwc+V{|xRJXD-c%721Bm<{K;lK@yiK4wWrUFjnHquNomXsLesdVc zvisD0G!N;$Z$3n+Z(x*b*lE`35oJ6$#5X^&024O^SM6YxGk!ezllg~Qv zS90;OMHC@y=Vnu_Ba%jL2O9GwenJ-$x>j=oDuvy)vVJsGO|qhGfVSr)WSL$kGDN_qjZ zSE8C-vbVP^cI+HuDR7oQAC%SwFG#d3Yd3$0HfM59n=Z-*slAdEO-B$Hm4+VqD&W)y zo`Yb?2x5@PFDVfXV6wYj!qnC}6|Q}dG(Skh+k#cAh0~&p0k6|Fe?{S7H9Ha)ZfQ+p zk4l?&L^ark;&>0!zU6+V$f`Dc3rfi)C&T(G6sixrxmGPe43fx_lF$H>d;CAM@L%Th z@9h5yrZx!5eD_x7lT4wT{MTpFw%RAZUFH_u=N>$KFX9X>{m2bL!UL4@^qN~9Ul(fz zKpI!57aR6-S?LqsKpBjl2}H#UC89sG`h2j*|F1CL7= zW1F5P#z=YxNY=53H3_DBi~-rR0Rrqh10=A=um|zU(p@Rc6y9vuH+%hsA|V@PciFJ+ zNRGGEij9M}o7@bfjA>KXg{_-Ogo#3NidU3*feTgqg@z-Gtq=Chx?JaHooYdH96R&Z zG`q0)Wb>{R+iMysU)|MQXyQWYCP`)MWD9?7`BriM^->^j_Gc>cq;sdoPwtsKbqS3#D%375BZnLzr@;=>2yBH{h{wHNQJbt*km$PimVt;K#1EzC*)PyD`Z~ceZ z%U}EY|B1bq;|@wU)0^(0lbh}d%k6aQZ00N>QE`2S#VHKu`v%tGVm{qR zy2bwDcP!(FPfz?fecABl&o?T43k^!xwMBgPNnk1XnE2$}G?_eur{!*ZGzfEX>60=P zC~vYA#92}lW0OW9bpI=#NPR}d%_8-Iv3cR}lWv`@RTZ|nr>JxFI^WkOKGpjy8iJH% zUvwBUu3^LFTU=#pCj0mIHLmZM7v0C>*@;P&S}3`DQP&;Mt^a6rNYosgRJoZh=9-%< z=K3rBoys%GBHCb%p@_JK#;1Yjs79w>{^rIVfafw9>x+wjHh!mNrW|W!kEB7hmV!k2 z<*O$8PWxCWA|}+bQMsPKvBm=4PF2i+)z0mDGwVD|*_L26U4k&c} zMhw^VY0;6yX~&p!C>^u0?Sqwp>W{Zgv_y7}`1 z3lhVu*d18$51krOAsDg){^UeVO+0KWYWSe%u1&da|_mPmg=s{WKZe z(d0Nu6}H{syX;J!WqB*NO+%r~NjrF$FodBIk>s$Uc}~X4z|?-%mc38V?Cpoo9FZky za(H){n!&^G`3%Yau1^BAw59nMPa8U#?97LjEFZafllY(FFXI#J$J1D3#HGKXBn(D4 z8OFNHw%OuHLj3uPY(mXO>Q0xP6kTpK^bRCyT0J|q0M{-jV`K0hq4^$lCh{slBvI@= zc@gEsQC2Nr1<&-WW8;UC={$+)lwp#VEaEnN$ho_EI|#-d-T;NclDoNiVcPtNH{!-K)bWSE1VZrfx_>9dvg+cJR1Z58ZfxlYzF9XHi0H-5?1J z`7caWM-8+iCSqKo=e(&Q`FEXk8g=JhgqghMHVm)jqYYjVww-&CYZ~S2(gjggX{wwH zXJ-GnBbVs;4PdI0uT*6ej_XErD6258GNr?!`ICNo=9RK~xrI;|naJuA=4 zVW`nFlYds-VrnedSJGn(8wyL+hRQ4MGFwyzC=_ss@HTte#6rP$@Lfh^*y5Hv<~>A+Z%+T{3=6$kEQk$T=*!@Tt>$6vfLIeSsy z@5-T*_Nm-v&zs0oZ)RR7WFe8D+Pf@VcP^`f0D^@=(a{Q%?Qt1>fV|Jmt zC|gAlOHppv55R6E&TNNXxFLd*=~8zo%DJPPKOP8owwGjA^)GCnT5u_RW2@!f$g|h9 z*4wG<^qB6I&j^vcplc&xyAK9^978%_J$OV=)rJfmr2b=dXiamwV8%HCh8IZ8yb|(! z+-Mz+G0$lzx(p9Dm0fL+J0tsAH7h$VodBI#0oQfP{;?6AyOVeNrro(|)})e48808f z+?mHt5jbTNq>*6EryDcwpB2t)liu?6mdT{&YxQqa1DlK_fE>Ku+bUHjdIBlHy}|cAd@sI=644EA3AP$9hhJrNSKx z!Zy9K&O$C;#Kv)38&+`{4Ux#^f6lw*{Ptyae}T5L!$hpjFk>J*wMKPE{!?}a#~*dL zBtT_QMX+@9$Ak|WpTKk?C}?ihsH;IfaYNK&_kJ6W7YBs=@Ba-e3s>RzpX=2C0NAd5 z@P8v7?Fq)B!7ywsT!>@~OMvsS|L@I)0s#N|=Kn@KItWXT(4g681pp%b#6v=({M=Z(kBp*v7z)yaM|KHUA E0gJ|oDF6Tf diff --git a/summary_results.json b/summary_results.json index 3813dfb96..61b2d9722 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.9238, + "Performance_Result": 10.2026, "Performance_Units": "Samples/s" }, { @@ -156,14 +156,14 @@ "Scenario": "Offline", "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, + "a#": 1, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -190,16 +190,16 @@ "UsedModel": "stable-diffusion-xl", "Model": "stable-diffusion-xl", "Scenario": "Offline", - "Accuracy": "CLIP_SCORE: 16.36890298128128 FID_SCORE: 237.82578979650336", + "Accuracy": "CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "Nodes": 1, - "Processor": "Intel(R) Xeon(R) w7-2495X", + "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 2, + "a#": 1, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -212,7 +212,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl", "ColorKey": "availableMLCommons", - "Performance_Result": 0.358519, + "Performance_Result": 0.342746, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.6061, + "Performance_Result": 20.9481, "Performance_Units": "Samples/s" } ] \ No newline at end of file From a9c1ae6d587ab3f5cab0f6fcf3fcbc654b02594a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 2 Mar 2025 07:12:12 +0000 Subject: [PATCH 219/719] Updated results summary [skip ci] --- README.md | 68 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 4 +-- summary.json | 14 ++++----- summary.xlsx | Bin 9782 -> 9765 bytes summary_results.json | 14 ++++----- 6 files changed, 51 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index ff249df3a..3380bba74 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,42 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-02 05:40:33,740 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-02 05:40:33,740 submission_checker1.py:3270 INFO] --- -[2025-03-02 05:40:33,740 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-02 07:12:07,811 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-02 07:12:07,811 submission_checker1.py:3270 INFO] --- +[2025-03-02 07:12:07,811 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-02 05:40:33,740 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-02 07:12:07,811 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-02 05:40:33,740 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-02 07:12:07,812 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-02 05:40:33,740 submission_checker1.py:3286 INFO] --- -[2025-03-02 05:40:33,740 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-02 05:40:33,740 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-02 05:40:33,741 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-02 05:40:33,741 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-02 05:40:33,741 submission_checker1.py:3307 INFO] --- -[2025-03-02 05:40:33,741 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-02 05:40:34,847 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-02 07:12:07,812 submission_checker1.py:3286 INFO] --- +[2025-03-02 07:12:07,812 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-02 07:12:07,812 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-02 07:12:07,812 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-02 07:12:07,812 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-02 07:12:07,812 submission_checker1.py:3307 INFO] --- +[2025-03-02 07:12:07,812 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-02 07:12:08,817 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:-------------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2026 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433081 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 47.3803 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | 13th Gen Intel(R) Core(TM) i9-13900K | 1 | 24 | NVIDIA GeForce RTX 4090 | 1 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2026 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433081 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5559899bb..0b45dfc17 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -846 +847 diff --git a/summary.csv b/summary.csv index da708b209..81c62d456 100644 --- a/summary.csv +++ b/summary.csv @@ -16,5 +16,5 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405291,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0414,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,47.3803,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"13th Gen Intel(R) Core(TM) i9-13900K",1,24,"NVIDIA GeForce RTX 4090","1",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8ce33c53a..fb61acd61 100644 --- a/summary.json +++ b/summary.json @@ -519,17 +519,17 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":47.3803, + "Result":38.9895, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, + "accelerators_per_node":2, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -552,14 +552,14 @@ "Result":0.342746, "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"13th Gen Intel(R) Core(TM) i9-13900K", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":1, + "accelerators_per_node":2, "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index eaf6629aa1b4190891930611dd67f6f35f364ca0..f5d33e6794f251aaf33125137983b7a64fff066b 100644 GIT binary patch delta 3975 zcmV;24|wplOr=b)a|j6yrm2fe2><~6lY9svf7^4~Mij>1`&Vc*c}QjoXmyvz(6ODw zF~-<&Ya$GMUs^vMKE}!K=}5{<$t#$oO?8xI zf763TBMn9}^3!@em6QC+AWkAb%iku$MmkNT-=7CX#!f@*NX8di5xW5BXMx@t1i?=>u}!oT__*Zjn_&s zl>0dTy?ArhZ>m}$Z7}WUc{i>Sr7s8me{7sx$B$oRI2>j9v{^++5RcROze*J5bE&IJ zG2)9y#Nw2yImk5vwI07WOlEBgv{<&AhGcSXX6LXW))cUf#VXJ z+`$trG01p70tp%KCmGz=Z<+rO(*PXNO`TKb;l;+9MFw91?`+5*ZjcA!q{%C`^mYC+$60b*+&lQqFU%Mv+Q2E;=X6_(Z0829q|HsZ4EM zb1#@qYr%9@izkz(WH!n2e_X1zRP};*G6;w2&uU+0emG9m4~LBbSgpTSF{C_am3i0Z z|Ly44w?)6iBEzVg!qW`QoP(KjICJDlJLVFbd?cQ531bdOD04tUnFA8?e4aWb%z<$7 z(S24D-sa&h2|wQqC!Yo#9&VD`dErnXK@4f~l1n@?Cs$5|Kn_Pee?KU`%+&JlE{m#a z!yL!ya~3DTsP~}P?YdSAW;dAfvjL*v);)LQQ*jGK7lY_>5Xtj%%q2E? z7EZW?fe0iNL?EFc0tp4tEdddPQxHqS+dQ0s=$3%U!x@M!1rfxM)%21}Jn}(#<^Fd- ztYRp8fOzL_t%!!He>>)mG}OZ+dYnYU&@q?T<3Hb=P@~VX`@`E_7js1BIPpHh_Ft2MZYcQcm;7F-c|bLDb!RDg4Vw zRx-jpn7mnbYz3vGnf8wKRN!6qBf)poEsH|QH4fbm1R4${&@d>0hC#_S97<6JC8&Tw zSyDk;&~t=ye^$}~jd*mQk41G+6kQ{chAxsD-G7TvZmf_6mAY0J-+^TnW87z{Ty{(> zcaiIkZi*dP;&Y}W!H;e%n_|f|j!v=U8jdBap$f(jUzB^9*S7hk}nB^}TR+rD8@T@=N@1jpDMOyf<1 zNp^GoI+IllZ!ac|Z6;k$Xyy(k@!9>6;78(?OPS;v$D~Yh4QCQ)7?VK5m;?v?^jv$b zflPu%f9OGDSq1I&#TO=NNe49I>6A$t)kRS(hvgcf3Cg5dEt9kRd?}nQp(H?N57B(J zES5>()%EW{vWn5|mA;yr>1&?7TGp=gb%)19aMOV0Q6jmY6Ebt&%%zI0Be#s1FBd9BRY?8qVY%Rve^ZOaikDW3tSRgBoE>mici6Gvu(64(cUi zl6i7K6wh3_24t>P`f~miDBr3}M=+a?v0>xbrgF=uqrFfGY&g`Z12hh`>HrO=au2f( zf6_22fySX!micv11&qov$Bv+ensZjt0gdoO5oXK*QT$K@)XWY=D)#1ZD1ZE^$Q$+S z+=xxnj-6-cj<)Mm#N9z9@Z?b24$wH%w*xes$~~MrNW-WE8b)QAc?VU%s4TPZ2if?`D?{So!6-Re=*yyZ`8tbEjB$oc@8Ut*x_mw-VQE-Lx-Ap zfX1ON9-!e|?&0G>8pb8iFfPliJg5T3Wto{rFgFh?>3~K!m(0)uqIeF=H6Vwr*wpjy zfcZwPJ>SHpw+(D9g+}Iy3jMk`8FZGbohI+=HTM z8jCbEjnykQ_q@hrHYjiu-}SAsf0D87lfCMjrk*?j@JzNwHa7maJ(9^#p2+byRf0I5 zu0`a!bnzGe*W%!A@yGEm^1Qux(hmzqzdv65DSsADhD!R!;zt7P;zt6sqOxFcJsV3U zd!EXsDxaqL9V!N^D*dOy;^zUBX%fasm_09AFMgk3HuhUl+3qKi#lT;bYK|WBeo__&P7C$&Jf1SnC z`729R?&Bz>_1hd}}m;ni01m4R{1ONa+lYkQ) zf7?>yI1qi`uTW)QW*&Tz;wu$0MJ0hs4VP2`TkKOxvD?8rw&k^CCj9-%BrqAv3b4Ra zDyOAB>elI2d+=?OXXqX(p0eV&hfTYOpomzU7VG2Q&Go4f^bi-ch-tIq*6Y zZ8XbhPKn{$DFloKszUS`l?f0&Lt zB9eJQ(=qx2=GsIuvFN0ZQ}MOEa>O!LeTN5ytX}1*5Ky%ZC-*eX=qk-pp>BHM0`fL` zDepo#^Q>YKaLyF}<(~>;6hQ_R73yNDEMBlg{G^qV)JrDRL)>tPJC!0S>xxE7fMXCc zdd1I3`e!Qzh2L`H1B?lLA@aEomfV4H^|BP{|Sm zBPwg7e2|%n?w0q&^rX`m5-OFvj98JRYwMtbKbq`NMI7X+SF)sQBct-x#BvjOdRj-N z?J739)oljPJna=)!}Hd*;W;TC1|*d?Zo(3

e-P4mgnZX@Y{gUoFF;XE1tKkIhhYVk=qR9dDgNnztA9s>yBb`P)eiSHc%Z?RBX4o6JwSJJ) zI(hplPU0lsQ;(~e{dRUyHQ%W&`32{t(KX65f)dc&2&vl|eXJJg3?)7=Em&tX)MyVL z&6~T5Tgr`tH)kxkAi*aQq&?-fM%xf9#n|&LDCV>1&{5nNr&x%kYq4ego3(%oLLRV! zzyyB_3?G209f|0K6nsS5AJ+{4!7+58KnIWWEXNmb0H^$9!=PL_l?;0elh(61x_4fV z#=h^fz8tJEe~!PFukXXEsnmy4aX?KaIOPslFs`ZabrBEOF&~!k#2X3XX&?28x6>lW zJ`hO^K#y_H%4hlfvRa^~AjN%3%9Io<@<s{}=|J`d}>)!YMJkMUcRHsR&)|ZNUf1WKvl^O)Pz=l%s09Hox zgEyWp|Ap7DJWMi9@MzpEiV)(a?iP$;u-TR~P<}dJRVR`N^F2N0JB{f$PT6-r@@?m2 z^gVsMDEZabGXT+k`OQeql*=JYjj*nG-9y)89j5bZ+##@F$r^atT9rcVF1hsz8!M-u z?^t#be3F0u5q2)mu8F&?Mvnci2yhZl{qU27p5#|iZ*p~rB`rve38}&O=C8F_U89>n zHkA%qirumnW20P(#GcG*2UPOiq%G19*Xgkk5jZVMF@7D@z7>8it_&C4vYZ&_Mhr|f zhCrtH>H-e4#imAAFWPtJHSRmHFnx3B3!vCG(N=OMPtiGO4d#VgEe-glm|0SIIAG#^UbVJi((jRL z=T^ecRY_QThBU0%AJlkkV*R12WtAq_=-c)zw37Tmu){%!R^F;Tz|C`f4%Qfq4g)ih zeiz`r(+tnYMDP32z@p0$ipqHf65KpS%j8Txm!qw!0uA5DsBw#UMP;XEN|=K`A|iTZ z7oL#`GF6B`vxiM8<_1#iPnT+Hb}Al2pM{hBLzHAs>DpA5NwL)3SiVU(f4# zZ$i~iUHq3J*cRr*c;E}~xDZ;`QKluHpA2{7YgYTPWBRgV!l@>Er zL(&&{mxa&n(RMvKe;dnlUAkK2{k==rg$-%$6a8 z60PfDq_&(7AhvkP$&k%91mRwKZ*{xKcKfwOatsihshWt@V3XNk%9f)Qu3hL1~-PIMCf5BQSF@ zC*;c9J5SPe+mlSOd;M~|(G2em@SUOs4MF*l}H9oHSWR=@~t<;h?_2iR3RMWW#eVM68oN10BC zyGrjC?L>bM>)b>3a2B+q{oU0{i*ZdRO7JBo1*TA|#d^Us>n~$;N(HONq;$nhE^KMm zy_GD_j3a5-((YdOBbV2Za)#e`=>sN((oj_|ine%TQGRI)TNhT*KK8L9l~avPCnO@- zY+uCOi1qe%tqBYMen!SNvGiH5!3u44?s4!LDVhEhH>q^R-dZp;20r5@@>51ciJW#g zi{|~ec!c=`(=Aj_U5SA_Lw}P#M%^XSr}FZa*IXZ4z?|tLfA^oFhR5y=+km*r3g0+i zw;ojVMI`k@J~|ry882~642Bv-t5dUT1AS(5B{}m-@(k7)PwN=0oKoUGELAfA2J>an=^3PC8YQ zdn`KaXc*{mPMPFx=O3~l+N$WJt)By+3nRCHY+&JQy2zPTd3T#zL>G)FI ztbycVo18|+jZk3TZq+ki_%>Q(`q9@>P+Ohpn?7LQ_1Zr!;~dI382Tm5 zYDjQy*nu2?e5m=!B;!cAz59t&Q{{rswhO|T^cJLX8z*`VV zdltiKOj2AwTF+NeypfZ3QotB4+>;~C9hV>HMJILLbvC!R^7rXSUeBpUU(tIWvi`|= zvX-CgURq(`UPS#bkTI#LRx%XFFlW20#=oWDm>WiE(pCFLVxMXoFa4*=qW3-^^C59| z)FNZG;3nJZQ8(j@obwsUCu^+$Q?^(hZ|rqt{3RsQb86w}k^B3QIMr!!RI}ai zgBfw0Omp1q)Djs=tnnB`$H(j{h|^FZT=iZV+Idgux<2xoFtD{fvebLkbyTu!wg&Pn zY6LL!VcAk`$L3w+Y4$1$)G@fD?V9vV2NwkPf%|?Fh|JNP-lkM=h6FJ{xM1B?cnf-wTiT|ktiKT^wR7+MuX^Mg#5hI*m*3T(SSuK2C-n~Xin(7LLyQV52 zED=DV%{>U@OUY&UQso@%mp%XThil7oA?AC^XT6BH3ngj&xI%nM|JKIaRD^yr}VzknfsyMn0TLX1VeYQ6ABQMhq;BqQ80p;=a7)a zrC;xAzP>gY7@G@&Hjpy~lRzZLDdA3F5Xhei1Y!ZDz;6JJX1YdSq!>E(@cNDjuUgU5 zdE-HrwCGBf$CcowStG4FVxMf&dZPpX@3S;b8)W)*8;p@KzBGK>^n+>JrH9e_3dUH@ z{>Qr)I_yU5F^M{mN*J0A3eEK>kKq*)c8mi~U%e4|LNPJ|1u ziD@NhDVG9pM-$BR-&Su$(57I4P~De;KEp9%%``dVTrVwj`(6L~2d1st0JtF+c&TJM zl=7XUI@zr2qSwQ;X@8lLxpxQY2-YB{2lia}1KD#J(VImH8`5I69$+t5RfBnR2X=$i z1uDtzdXH*Tb8mlaS&+IWd#9rU6*1kXNLp+t#}8oRFMj>f0o~;Qe*Ll)iO~^}s>bSY zG@C&zMyswJRwR8M=4FmF_UN4>8}(@)In=4h=4$0uWA8{0D9j16tjqKoWw|8g)7W!h zbBxU^;_K=IM-2~{&9n^hIDLBWkD3z}LMin%UT?rsyvpkMlbc$%JVqo6Nv>Xu6csio9bbn+cf;w92(Ip<{EU+>zbNyxW=oNHwlLZPQqvem-}SKL*^7a z3qpM6ILgX`^7css0V1@d!SXbbkM3Rehi9kYNA7E zeiavF6s$@s+L$Eo*|Zz{5r?1N_ox~NA6>5DkMgfN4Cp)4S;5Z^nUd5E$vb9UkWcRM&F z+X|N&qjY8<=Z|aTC1^d4Gn{2n9iZ0SykzHWhKTEko?FPDR>@jyJrrW;**^3;fK)Up zFQQB0QAMV3kB~uC882^fOK&YKZHr;7isjO;>+o(W(H>@zHT|jD)jc)6Ve&(Dw>(`< z>SqW&DzKtEjnC7&WZm_J+b`(EkGmy58aZoXSzXuEvI3L&(Iv60LI)`hYJVrV%$UC7 zO>7?I6&o(U=oMTvHe9h#O5G^F4dBFLYMR~~&Ofo{n{Pz*|Fr$!S@DqW0mgpdxb26m z#Al6BBdzam9X=>fQ}e(P-u;9%5?tXgVHeohp?PhHB3Ojr7 zvmJT&JFzd;ggW`fS#bYQN>oDVA6ul~aynDw`hcdukB4K`HhswdDKf*etwrLd08j$anHKs2d_^*TQC|fhB~- zzj;>PwW`@XM^`U`9fvNV)qaZTZSeRT-6ENBR;~ukE6K)aUp%!BknFpZ~qx{G+ zn0bPqu`xVpN+$eF_~9BX$tbZJckb_R&^TKuc1pfkI(~OZB9durfvQsSZm?&@S2C?o z8_U{Jmx^%^muOa7oYhc5l$oAcPG ziR3&;jH>k`4Jk7qBHm6}=FeNiXSh%83Rp{$f6R|#$B{h|?iLE@iuBffj))Izd3{$6 z5q=$O3x|^UVngBl?0;N`lYl__q#zLEe-45`5g21^2mC%s1@1L diff --git a/summary_results.json b/summary_results.json index 61b2d9722..c54baa899 100644 --- a/summary_results.json +++ b/summary_results.json @@ -156,14 +156,14 @@ "Scenario": "Offline", "Accuracy": "ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "Nodes": 1, - "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", + "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 1, + "a#": 2, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, @@ -176,7 +176,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", "ColorKey": "availableMLCommons", - "Performance_Result": 47.3803, + "Performance_Result": 38.9895, "Performance_Units": "Tokens/s" }, { @@ -192,14 +192,14 @@ "Scenario": "Offline", "Accuracy": "CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "Nodes": 1, - "Processor": "13th Gen Intel(R) Core(TM) i9-13900K", + "Processor": "Intel(R) Xeon(R) w7-2495X", "host_processors_per_node": 1, "host_processor_core_count": 24, "Accelerator": "NVIDIA GeForce RTX 4090", - "a#": 1, + "a#": 2, "Location": "open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline", "Software": "pytorch v2.6.0", - "operating_system": "Ubuntu 22.04 (linux-6.8.0-53-generic-glibc2.35)", + "operating_system": "Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "Notes": "", "compliance": 1, "errors": 0, From ef07f1750be43234aa7395887a4b354c281e263a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Mar 2025 09:26:44 +0000 Subject: [PATCH 220/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9765 -> 9766 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 3380bba74..d8dc3e033 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-02 07:12:07,811 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-02 07:12:07,811 submission_checker1.py:3270 INFO] --- -[2025-03-02 07:12:07,811 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-03 09:26:39,249 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-03 09:26:39,249 submission_checker1.py:3270 INFO] --- +[2025-03-03 09:26:39,249 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-02 07:12:07,811 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-03 09:26:39,249 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-02 07:12:07,812 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-03 09:26:39,249 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-02 07:12:07,812 submission_checker1.py:3286 INFO] --- -[2025-03-02 07:12:07,812 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-02 07:12:07,812 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-02 07:12:07,812 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-02 07:12:07,812 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-02 07:12:07,812 submission_checker1.py:3307 INFO] --- -[2025-03-02 07:12:07,812 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-02 07:12:08,817 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-03 09:26:39,249 submission_checker1.py:3286 INFO] --- +[2025-03-03 09:26:39,249 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-03 09:26:39,249 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-03 09:26:39,249 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-03 09:26:39,249 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-03 09:26:39,249 submission_checker1.py:3307 INFO] --- +[2025-03-03 09:26:39,249 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-03 09:26:40,349 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72551 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71285 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2026 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.433081 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436673 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0b45dfc17..4e18fc9c1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -847 +848 diff --git a/summary.csv b/summary.csv index 81c62d456..91a669269 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.289104,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.296426,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72551,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71285,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2026,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.433081,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436673,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405291,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0414,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fb61acd61..280234066 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72551, + "Result":1.71285, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433081, + "Result":0.436673, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f5d33e6794f251aaf33125137983b7a64fff066b..58a0f6b6ad2e933bca8ddee7a6297d1506383a25 100644 GIT binary patch delta 2690 zcmV-|3VrpZOr}h*a|i|ZO-Ni#lXwU<0~eu-ldlLde>m12RGb9Sq^XOlp=)a5NBv+j zZ0ff+pDT{8C8;;*d*f)Ln)|!@5xB1RLd%uBu#ykrqhF3E%8Sx^pa{c zRg>(>Ac{jT&ECetS~885*PjQ4-|RvTONxv{((A`jI#|`Be(N);gTGeHW<5WaG7#59L0Je$U^W^_#kp zOKVK~dESkySnI2SHyfwd(c>2t3`c1;ZH+v{kH*RTUo8x>xs=EZs6YRiJ_Re_d`BcHv8R{gN{kXO7^e4`;ZwBVK& z+-(%pvi__)C2^-D>w~e~hik4d8>E{+Vr@%qfW+RG+yaTSEx7{{cP-f(ZXFM|&MFXO zGq_wq`uXiu#m%dV+gBBLMT)JFns}rpicutYI!PEyXc%O<71U%8B9`JrZSav%{_lQ zt@zVfC7Mj0;@Kq4a;eHv)$^mtAQwT4a!8p-B9M%S4wfLv5cYY z0pgvrwIbF{X`4IJP#2TvauNwc$6R8Ohv9@v7>PhaNdyu~B9}O%p;NbjL=aBDfAAE9 zw_U=?IW6db251^eW4b)XB{OpgB;+ID$}JbR(DS3DGWO;*JUP~nPD#rc-W~v6yBh%7 zRi|#q9s3T#zyQw634!GH9|Mg|I&uOuj6tNK5F!nw5NI4y&*@r&I7Agt5Q{2kuP?4q zWF;NYh)Y*K7S#n&bd&KC)l4wIf7cm?$2d{F2T^shrSLB!S;`3a zVDe_!u@#iIVOl%VQ-OEcj|AUUZ!{>BTw~MyK%n7J0u6%_Xc&}S!=V&KP=X2=lm!*E z1wBVNXC)obh)ehRSX389(KQlj=pwn%{kI6^#tNBJDUGW54lK(U<33B}f3jm@xr|Mzh9%c6+xGZCg`|PSNJGS}~DP7Clb(SIUiRh8w2aOGzf(Z^I>l93` z;a~y{0~2T%m|VlbtQUa^e=1;L7F5t)Uwi?RmUKWPZ2N{qbx{-p6C7i6Fzas`OtPEv z*O@G1czZEf-)7Qrg<3~LDNtvWkT@=M~SgsL zZdHB;@&0YdDcW!-xbLMkLob^z37; zfkc8v7?A}PwAUA3h@>SQ)JwYW%A&d;iW3Pm>|15y4FUO2Y`&H>VU905!=)Q1B!XErtB01any4=WDRFeZW7g)v#=#zBoRCX4(yf*EpHNeA_kG08kR zAc|+MTmv%KGJQG!3Y2fvr6ZV4r@mq1*rsyJsAIfP32ZpjsRJ}Nwdw#3r*aRo4$?3x zfySm(7Ws8h1&qoflb#P4e@gb|a43KLsmL4k?A(Y=(~gy8=eDsce|2_H2|PK}wgWUa z_3Z!+r*aSH4$?3xfre38WZpp)Fe;1eJA%4v&RIzZG{U!EnM(&nvHTTiSpHhFQRg)( zf6Vsl8@2FUi%kztmct4mcDP!FyMs&M(4i(Cps}fo2WU8#d-!;ee}-`hG>pq4D-WuG zaam;M5zNiQN;;qs&LuPSfGD2Bat+8~OE&fVJ7B(1YtJ{a>Fvq#*t%ueJIIsY^x0CeUy) zWy$WI*U0?42P377e{UZsrLhgA(U9&A7l9NzpyWmoi<*0Y#-ZjOpy5#NVeUa11|{eZ z-SUw|<{s1tgR;onBbm8}m2^NOZk{V`>da?w-H64m$ZR3yILlqCzi5ia+&5yF_T12i(7k}}8e=QE~7JnT7BG1~3CjB6H z^n2sQpYo@{WT+*7EPf=w%6}xl$Sd;)*R!$G(&wpa>gs8l-JyK2y4HUhEPft9o5n#D z2kG;o_5Ak0UrDpk&0EP|pl-De zr`}Lqc=0fpI3(Iw4YGk6)$9v+y!gR^`Rg>A&R;cj?LJD=C|tc8DX*{M{9Rqw22qr* z-sDrV{^5q!pHNE!1e2X48nXowOb!IUzdd`CRUjXK%WA_g5JmU>3ZuO(KWJ!#?Lb?y zYM@Y;mhOZ!Nkpv&l3FK!U)heG1hOhhkK*Yixmlw+PxJdYEy8KiCCL}0Sbp^_uj9flm8?^0r8VTB{~6PlYAv6 w8^6CjdnEw?08jz|01*HH00000000000Dyr+lf5NA0hp8gB_jq(BLDyZ0G-_PX#fBK delta 2709 zcmV;G3TpMHOr=b)a|i_urm2felXwU<0(F;@um~}KG*%vDl7{i5sfxO$DsmFU{ctjD zs<$_vYp$xKnLp|K<9H&Q>a$GMUs^vMKE}!K=}5{<$t#$oO?8xI(}PAM4MsBZ(|SCW zll;mcP9i_c-zLLGI!&bCp9e+bhOTLLBl5$Ex~SnG*{6 zy!+{YD4eEh>#!f@*NX8di5xW5BXMx@t1i?=>u}!oT__*Zjn_&sl>0dTy?ArhZ>m}$ zZ7}WUc{i>Sr7s8mY@A)kk6&as9A){mSw%y0%)N&S^mh^p7-fL1Vf+hORi5 zT;h?Lxq9jMp!Maqt54URv+nu(c`lUZ$GjR9D#zg97%JZF`8*XIB2=-0P@ zMZd%%!>F6W(+td=|p9UQsZj#%1;ZPt!3~BO`OFS|sS5AdM4o5vdD89_p^6xH-s%pa=$LVtx zC&8%qpx5oXRtsiAH}?RsilOWQ;;plP0ixm7J$K_%aSKEjgXnS)$@6o}B{q2$PPl}D z2qY9lAfX@v2?fzD0TF~#5KF?_Je+~(mVn5^8Hg?g5yX(y^pZ<=8iPf!z6l~M8eQ9m)PWCIN=gTB9KrLfrOICB`#^`)GHwogwrp7JSE|6 zk8pBM3p$_ynugMtE|2lZ%v=Hq`3Si3s)a4`gE*~?{do;fuKlA^(kh0x2Y}b!27pf8 zH8g$4zJt&(fb()fAi4d=K;w{(oB$1D5NRlcNJA+E8kf{_y4D~LQ3VvlvI^Ski)$2F zNe49I(Up%ybwL!}WV}Q*6U?uFHoE^dfP7g83mE%SPV>hxNoDUr)ZKh3{L4sIGQvHW zyjgZ^1*M~z_Kx&a;9d43!FSayi$cjY4&4s~8V)7UFerhBLCG~7N>K(SsDMFPQbAkL zbA)qN(gBTlbf1q!bx{;uBawzKk{jKBi%@Q?kOh^xRu|uaWff!GXQ^C&c1$dHk?W3b ziXB+ubEYG~k8UiRV#zg*PO;<~jwR4AEP;k$$u%5Hy$nlG0mHJSg7*623zoE`0~+C2 z(x@(oVmqinBQ%Y2ss9hStYVD&?5Zw1w)zsOd$zgjEJNTE(IdeR8ZC!{2@WF-3MSWZ zFoA}F2{a5$uHj%BWnhAT3K*Cr6|~nEU%;d#9nc8dzF|>a6ve;<$JiW9<4uD}c60tZ zlT{3FFD8v`CS6Zx<_;$D+5M5=N8*-CndBPBq)c)RXA)=_lR(3m1PA@}Tzjp7OoB$} zL1S43?e)bMCTU3rG~(%$NgCBfQ7nh$8lefwq**PKv-^B0oGqb$BtT{l(R{TmmPz5& z_3uEkiqY+rzM7lqYo5JY)~@t*hsQ*4(}3ksBDuz~D3M&li3A!(B+xJ-xyGeuA8QRH z5;VeyEUBQqzW72UE$N_M(tTGJ)df+UNT6ZgDy=sJLp{6d2&D$ z&s@0%WUf{Ea{d)4->OSTFq@9CVdL1Qa?7Zry-*2kIMk^FG!C`u01c;d53>%^Fe-t@ zp;VUnbx;M2$}*Fn4;X(c_U3RXfBdP)8};nmh)vUuooDBcw(C^H-9aVr9xrMrE0K2UWnREVJ(j>aICwB^}TR-+pB-9Tdg#SD<0}YsE&L*Qoq4 z+pllb!gDP)Jv@01D}>nLY8BoNE`dXbns|W5p)MYv;au+F<3WEK#wE}&F3YSur~<}i znVCm0HxDc6fJQi%%+Ld(cn-@oAcw8k)bsCv`9`fh-^8Z3C(mOILvwfR_~;rYQ|Cqz zo0@xo#--*SpkZVp4J8w4D49q@$<)itJ*WaoW?2Pootb-BNe48-&P7;M7e!GrfrgW* zSM2V2jm*D$Fj9ZX`1XNP*S4Y5EZy4$rPu)_H;UNQ+ygW&HTM7whjI^d57ICwL4W9$ zk6vc(L5(md%gjAGGxxBP4rs(PD3r|HgQ92}i!?Ni)hjmlyvAfUC~y?t^{uj!vF($+ z>YJvXJOS`bwnjEK{vS4sM8%rg7p30^wpQia8Dh8`6{ing==K+*y62?iG zJug}>exG1A_FGZhe?RZkY!d!4lkeXJ@9cwjf&JlUMgO2Qm3-(cX+FAnE5!@ct=8ex zAIb|q8HPENR2j=bK2WWme*sSxKR7UdoyF7nD@#@G<1CA#)w_}O`!Xrs8LBdf<81Y& zn3DAmH?;l)P)h>@lb$3Rvj!4O4g}Pr;Wm?2ARm9tYQr!PgztR{qq{BHrfm?m18vEv zfkIPSaud?V5w(7h)H-?lDo)}g;8TyQnf-QlQ8nMGF8KxLrO`FYGlCM(+z6@L8hxx5 z=?o=4FfCYTG}LGh9?hG(id)K!gEwa^xFEqN5u`okwnp0!EXCOKEhy%*=+IHz7^hf> zrE7n&W&E49fD1w%u!6t@3k)BCsU3;vgcN*4+8@^q0KqYIpg;$Y^DM^~ZvdzKWW%6b zIh7213zOEfIJ$RUj>f+4v%VaxF@KJ~map%_s;Sh6Q*l5|B{<~{STL@s@O2Rn*D)WK z@x&Vm;%OiCiMP`t$375A3_y=@&&p@{{IVrlpr#?~cPyzq|5dZ)H0000000000fPq7kz9l{Z Pmy`V^BL+z$00000hhYo3 diff --git a/summary_results.json b/summary_results.json index c54baa899..c4b4662ab 100644 --- a/summary_results.json +++ b/summary_results.json @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72551, + "Performance_Result": 1.71285, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.433081, + "Performance_Result": 0.436673, "Performance_Units": "Samples/s" }, { From 58845e8c4149460ac3e41c7bedeee7c1bcaea728 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Mar 2025 09:33:05 +0000 Subject: [PATCH 221/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9766 -> 9770 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d8dc3e033..d86fe8df3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-03 09:26:39,249 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-03 09:26:39,249 submission_checker1.py:3270 INFO] --- -[2025-03-03 09:26:39,249 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-03 09:32:59,997 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-03 09:32:59,997 submission_checker1.py:3270 INFO] --- +[2025-03-03 09:32:59,997 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-03 09:26:39,249 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-03 09:32:59,997 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-03 09:26:39,249 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-03 09:32:59,997 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-03 09:26:39,249 submission_checker1.py:3286 INFO] --- -[2025-03-03 09:26:39,249 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-03 09:26:39,249 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-03 09:26:39,249 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-03 09:26:39,249 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-03 09:26:39,249 submission_checker1.py:3307 INFO] --- -[2025-03-03 09:26:39,249 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-03 09:26:40,349 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-03 09:32:59,997 submission_checker1.py:3286 INFO] --- +[2025-03-03 09:32:59,998 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-03 09:32:59,998 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-03 09:32:59,998 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-03 09:32:59,998 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-03 09:32:59,998 submission_checker1.py:3307 INFO] --- +[2025-03-03 09:32:59,998 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-03 09:33:01,063 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,13 +25,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71285 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41903 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418157 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2026 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1944 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436673 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4e18fc9c1..f7c0118e3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -848 +849 diff --git a/summary.csv b/summary.csv index 91a669269..79ef5c0a6 100644 --- a/summary.csv +++ b/summary.csv @@ -3,13 +3,13 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.296426,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71285,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41903,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418157,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2026,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1944,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436673,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 280234066..e5032276b 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.41903, + "Result":0.418157, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,7 +309,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2026, + "Result":10.1944, "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 58a0f6b6ad2e933bca8ddee7a6297d1506383a25..bf3db4b32221081f7778148ea78b4700ed8312a1 100644 GIT binary patch delta 4021 zcmX|^c{tQvwKi!u@f<*EkZkI$u}D=)?5@zBtYguFRJUhHGiq?w+Zo=)WW}_`0*;)#>pAs$o-upN=A!Xf> zHF;Nhug~4SMzl(X8srC@v7VSIFb23IR(y~@{8`Tmjx)-AOMa*5S|j#Csqgyzi`b!@ z{s_(cswFHXhhnZBERE+p_%oWGY&x)&r*Giy13D(*eIiH?XxcA3{io>2MSU1#GAdD1 zvuOYZ8KIqp(1}l9paVCro*_iekU}AJv||Q3aCw)GaMBD(N^m~@dpbL;3RqX515drg z2^V*?!5|j92twR~hvk`i`ofyn zip0WX(`0UrU78+!zq)r$fM*r)srdRdDKsJIwLOl_xqtQuaT)pWBqBnd*XgtUnw39Y zZcL9T`)~;DjQ0(X>ch8yJFDLd zIp5{$RyV!gcqR_d7TTpQO2aJe#z_u3{x$4e9;K1z?%x^yNS^uq;HF#4!Kl`Q0ZPxI ztDX+TaOiEqt8-!nGe@2ar~?&I?b>5nitn|7#Yz5<=Be$qG|NC^Ak4u&%)uednGbm? z)P9j2#wkcgDA5+ihru9s>pQi>V1jo8phrzCx=?6;2fb++DYP_H^L<;gC7#x~eu58s zlu3o^?2gTH6sh>DUbR2p@P{lK;NsJ0`-ajC>_kk#8`{>?6p+xZ|6@`raN zMl;SF(2h%=4MQk7nq1WI{c5Kw=7-ZmnYOcEvuSv+r)c{~$)u3>?3n8WZY8vJ>wZ=C zMT1o`edYp7!bx3*#0%QD@s{jH*@yQp$o2V(Xsl2GW0AScp(#vB?sN4tS>_ykPV)4p zPF2ARROh^LuP)}A5%MGS$2peW7c|Fjg3)rH>D`M1A$4VgDViOem`C=--Xp%1whPd@zR|nJ803tgkj?+wP$hKc1sS*FW#bDLXD( z&L~Oo`8Fy!4tO7j{o#4jbzZyt+QnMuEX;402WG53EhwIv#zz*V)eqeD_pbuj9qJE% zY!#Ynz4!9gSfmA+=rY|Ot31xFAkcR($-aHyQmn4m?N=whQj%qzc8FPxl|NINLc{*J z^RY1+Yg@MuKQa&~Je0iS=!<+e^NMB4t!dMCI4ML(w`)1@-EH{nV7WrY5%y=lW!<*L z+@rLK)o+`r%+Ov;fua;EQweZY{-ido*H5OrKJBQI=s4Xn(2$GRy4&LNVYNECB>Z9? z+8c8m9&8{jWu3?z#Vd8DF@2||2jl0xxcv22S&m$bZtWH__P2T0Egt<0TWp|6{{CJ* zJM`KOD#t>()7Jcpz!s10?HbMD{>)LDc5|$C)S#53|+DGFUopu zp0(ZmVZB$uDvxVLob+KPi(fdn72T#eA7H#5*Jo$kO0*m3ebUSf;gzu>#l}Wi25;5O z?^wQZ$aM8B6Gv84=Fz+ttRmgZo=OHdT7C{;P^;Dr_e-0M7gKBcqk^8^kRy?{V(C!s zJ?eB!;GMIiyqT;~O+YJBm4Ty&$}KEnHO5OKRR@BZRDflBQOb{2LgL6e1Bvu;4aAf` zYb-3%9L0cgzu85Xl6Fa%-ziA3_H_xjK<PQqahJAip*@cDKMOwsSvhR(-Xw z>(R27Va^$NPndG&Q^xVIr#aQbDk>?zE?;Zc2a)C3W(oYB8Lgp z)m$}pD_>H%e(_4V5BD0`Bf2h2I>FFq*rKATzMbEsGTT|TR)3f-GEL3iG=v9RQI$C^C?s1gX`EYzl@Ru7%W=Nq9CwVmu7RsnZ71VKc)?!vZ&h z7)$IxqO6|69h^FB+BsBXnax4yV)nrtxbfcOAK>|-&KA^r)0Mo-zwoT&Z^0Z)+JEH$ zsb3O}c~UgB3?UpwqPfq@9-BXnJ4ODCJe7m|vhX)DdJJp1oFMXRW_~+_;}?6-Vm8an z2iB4@`CrQl*!G=_cZS0}G$hC|iXWtE*%mM;$cjuPfD3ca(ot4482OUI79d_c2?p9C zI($YVW6#STQd<~CVxh%&3TOs?>L8DK_>>2gZ(lzuY-^}}sE&k~~^eH_Eq#w); zmrZJ{oWyhreANjT(wTaj)6;d&{esx%1WJL&z;(PT(fg;t6^7W6Skxc4@Zl1CNSd|~ zj1Hc};l*=bv$uGG4^b`=&tdVpSh}hh{Y|e&Manxt54&HSp&O3pbP&q~h%#CHvp_07 zppK-HJyCV{=j*0>a_h|&GMB}3GBRt^F>eskn|jpDf1{LgOdA*Y^sh;`6Wy3FXN-$o ztV7@f*SK7mB%Q6fEXHhwqPw$lp5ept{e{mB=VeXNsMSN0zSu zS)QU$XjtrGLI-!_`0<;Xw${QZ&&2oma+>bBXYZj-d1Yw-v1H)<1u~{4p~M<7tkA=D z->}(U+;jKiHDx<3!#ES6{;er_C-FO0*>ePU-qIPV7zHKjbd6Y9=W2aO5=AFyI&Sf* zXnTL7M5cXj3WQG&H70wyEsqlNg`+qyJ;|r0x2XToLS6JOb(~nP|C_IJre^QD`y_onQ^!fy2+J-6njS=wGXmiNq zh@!61&Qg#Ba9BUczy0HLvMDva$YVLZQ3?o;z%yZ0CR%=y{M>$wLwVr^()b}w*S5v#3Z0P*>`#~d=ms;4q+8yosg9w z5QA;p7jfrgWl~45@$I*)|Hj?!PDHZ33AL^VtN2kW9s0M16Bx*h6e@3Ri((-WV-%V!d^QZV;TY~0sz=kW5Wl3s>4)=$kEgUxWeTovTh zr9k-1Dx^QoZt>*<5pHGoH`nOPF|Cg=(kp90>;{eWf06$qA{LQL$TAmv{PCE+deh+N z1U?r$SAG=gpWTKmMX$bEbBX+R0IEBV;qvz4lQLG^ebMj!X*8H~F5Jpy=)(@haj~NM zyfFw%EmwZ8_-a=D`le=4)wslXyhK60?m|b{X7-71a8TQyn}?}b-%AgE_k14znZY%; zc%_%3X?$#bXcSj;VicEY#r6BCr&xLUsQJ{`6cuw>d(S!i$?l}np?aKR8qh0&pf2mK z?y+)?=*5N#Y|}k*$Q0fP-aA0aTr0rst)sieyU;2=pf*koAO0i-R;J0a{+jgb9+ww2< zQ&>${-t)l~RO9qhmn!CSLcabx#mgdNuwe-CzS90=Tn1t+w-XJ$yCQoOX7)y38_e4%mw^4Q{w69#VOd?hQznIHRf7Z<8z zpMkPiJ<$)o`SWFAn!~h*gNn7=u>er-NF!b~uRgF0)-**cFCwgE>#H~R&@%NN>UHyX zl{osoihQ@CRp9zn*$mHU4m{UiERDX6a=diaoHVb9_Q3l7(j3m>_%+q`qDliiHMua? zC+0j7v&Zy6qd89`k&fX%zhtrq3McTtR*D0Gcusrje@7g_6GkS$_{e!EN$?uE7lj2! zWTA2J9c~aP((h7e*eySgQ1Up6lkAG-qx0e;1AOPmQz$n6|HJ_Xf$(%75c_{=r*j;^ Uko*d*4lX1gp~V=@#Qq-rA2wZ9Bme*a delta 3990 zcmV;H4{7kKOr}h*%>xSMKWPSV1ONbf5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`CC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%iWw?t7H*` zs;y|rE}}8j3d^chz6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*%U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0t*!``xOyC4ag;F1o%hIxr27bVvuT@(2Wq4uL??ArL4!1Oi3Z z-yu4L4n?s!PsDDPeWe<-@o$mtJ6_t4)Go$Yje@9=}I z&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)g5$U zrvy1SItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50nYt*0(9)QYipgR z_3m3bCu~I~B+IxFQdzKu`8L{|H_V*Dy!jUY1y>*QMU<+QEy$Xa1CbUhga2rbD7?Ys z(Y2MaFrHNzF~n;rKk3O!DSzLQ%GETYm$*5NFiMtEmWtQLyE+}IvS6IkXul4sD$*Yn z+20HIyun`R9egp|17l>!%r{V(UG1elE56fUht^>A83vOg2mWkSJxmK~}e51Y= zw5+dWv%aK4(0p^3kRT#V-N~=pm{PI1Q6+ghHX8o~P)h>@6aWAK2o3-M06zftO-Ni# z2><~6lYR&?f1!)V(6ODwF~-<&YbEzaD~_%usW<6+<7lFq`m;*(Us^vM zK1T8H$w;YG%PN>8O?{N6(}P+q@kc83l4>+nlkCbMe~Lpd&ECetS~885*PjQ4#Kn`8>iRN;};bSM`<=~ zjXcDUf5yrDUo8x>xsw~e~hik4d8>E{+ zVr@%qfW+RG+yaTSEx7{{cP-f(ZXFM|&MFXOxLiT{`R!H3&8v#rR~2_fimj2Fc%&wa zmt>?|;gXTQs?4p6U_Swgxh?q& zecutYI!PE zyXc%O<71U%8B9`JrZSav%{_lQt@zVfC7Mj0;@Kq4a;eHv)$^mtAQwT4af5AA> zKOEKuV72~Q#*ng{Rb^e9|F@%G-xmE6iwvV`3fHJ(=4{NI&6y)l+A)_{zNp9yL10b&_L*#pE|djrI}Q+1t< zPsJ?|9SowwK_t)5F_&26SvcVm1|pD95P^h(2qY9lrvO9{PC+aPZ}V^lqEi4O4`(1c z6hsh1R?|x^amfeemGj>Lv5cYY0pgvrwIbF{X`4IJP#2TvauNwc$6R8Oe~00OOBjhj zLP-P?N+Opyq@h!{fJ6{Zzwi`64gvFzuM^j+W_)q9n4|u zD>cm?$2d{F2T^shrSLB!S;`3aVDe_!u@#iIVOl%VQ-OEcj|AUUZ!{>BTw~MyK%n7J z0u6%_Xc&}S!=V&KP=X2=lm!*E1wBVNXC)obh)ehRSX389(KQljf9N8)(fzjw<;Dt` zQz?zA_zo<~7~?)m<+5X9xr|Mzh9%c< zEM*avpaO6+xGZCg`|PSNJGS}~DP7Cl zb(SIUiRh8w2aOGze}V}PBkL4QuHj$;4FeNs7?@nc!K@d72`XS<7F5t)Uwi?RmUKWP zZ2N{qbx{-p6C7i6Fzas`OtPEv*O@G1czZEf-)7Qrg<3~LDe@U67QC$?pa#*erhM-KE zZdHB;@&0YdDcW!-xbLMkLob^z37;fkc8v7?A}PwAUA3h@>SQ)JwYW%A&d;iW3Pm>|15y z4FUO2Y`&Hs7-PyH?2^HXH#=Hq?g$G&VKj01any4=WDRFeZW7g)v#= z#zBoRCX4(yf*EpHNeA_kG08kRAc|+MTmv%KGJQG!3Y2fvr6ZV4r@mq1*rsyJsAIfP z32ZpjsRJ}Nwdw#3r*aRo4$?3xfySm(7Ws8h1&qofJI9WohMIF$(gBU|LlI`o0a5%= z1k}t9MN0POa43KLsmL4k?A(Y=(~gy8=eDsce|2_H2|PK}wgWUalim*>e+M+ew_lk{ z2Su^`6=+!gTC!2+H7bA1_Ujw9@LY>c4^NiE3L$p5T7|oVOW@F?{e~PBDNJGNieqi#kK02P57mhpj}y(0vgukxu1gnx@qaB2 z?iPO>|02)YizfXbcl3MX#h>!0!DOf3ZuO(KWJ!#?Lb?yYM@Y;mhOZ!Nkpv&l3FK!U)heG1hOhhkK*Yixmlw+ zPxJdYEy8KiCCL}0Sbp^_uj9flLRF}0r8VYB{~6PlYu2B w8^6CjdnEw?08jz|01*HH00000000000Dyr+lforF0hp5kCL;z)BLDyZ08-jkhX4Qo diff --git a/summary_results.json b/summary_results.json index c4b4662ab..2bfaad330 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2026, + "Performance_Result": 10.1944, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.41903, + "Performance_Result": 0.418157, "Performance_Units": "Samples/s" }, { From a17dad4b4d778eb23e46743db287393a203e6219 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Mar 2025 23:20:12 +0000 Subject: [PATCH 222/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9770 -> 9773 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index d86fe8df3..6cfec8250 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-03 09:32:59,997 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-03 09:32:59,997 submission_checker1.py:3270 INFO] --- -[2025-03-03 09:32:59,997 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-03 23:20:07,339 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-03 23:20:07,339 submission_checker1.py:3270 INFO] --- +[2025-03-03 23:20:07,340 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-03 09:32:59,997 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-03 23:20:07,340 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-03 09:32:59,997 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-03 23:20:07,340 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-03 09:32:59,997 submission_checker1.py:3286 INFO] --- -[2025-03-03 09:32:59,998 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-03 09:32:59,998 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-03 09:32:59,998 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-03 09:32:59,998 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-03 09:32:59,998 submission_checker1.py:3307 INFO] --- -[2025-03-03 09:32:59,998 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-03 09:33:01,063 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-03 23:20:07,340 submission_checker1.py:3286 INFO] --- +[2025-03-03 23:20:07,340 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-03 23:20:07,340 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-03 23:20:07,340 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-03 23:20:07,340 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-03 23:20:07,340 submission_checker1.py:3307 INFO] --- +[2025-03-03 23:20:07,340 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-03 23:20:08,291 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,10 +32,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1944 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.1113 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436673 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436673 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405291 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index f7c0118e3..8163eed6a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -849 +850 diff --git a/summary.csv b/summary.csv index 79ef5c0a6..160881b4c 100644 --- a/summary.csv +++ b/summary.csv @@ -10,10 +10,10 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1944,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.1113,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436673,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436673,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405291,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0414,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e5032276b..de1f1e16c 100644 --- a/summary.json +++ b/summary.json @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1113, + "Result":23.0348, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -364,7 +364,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -429,7 +429,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405291, + "Result":0.402086, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index bf3db4b32221081f7778148ea78b4700ed8312a1..fd06f103cfdad21c52a340f385a4dda8254b7e11 100644 GIT binary patch delta 2795 zcmVJR~y(bcE2wW9ZmU;uvG> zIJP@|Vq|duO%bA!@a5lE;%2e;l1$UI^0Y5D9IbtB(#P4-S#bEvQ!rH?WEA`1tgVW= zrYdsgg+qTfZmYL9pKG?N#ECl_y3=qb+v>B7)n7V49X^K9@9{*+L`f@{#cg$xeD>q&r*&1=EJEX=WpE{h7L|;ULjG>X2AhunHCCI#y_68|~DEp_zIsJ0H(exsGY9 zT3z<=qno&$!zg@IqGb#5+|AC6IMDTUTD+|EYgU}~t+6h+-V`Y5II`J$f9Nz18xL8< zYsEJjazhJlX~A8tppo`x+bJD)N@so0cKdM66-JA66G+Tm$qkTLyOLWVv3DhRK;mpA zJHu__;Wk+Xf((}{NI$>5s&`u1v8rQUj0FVDUN`DOWgTq^~M&URB(31wE-{ zb?l6^1xI?+;^&lH!^c2Ef6n^}kdO)c2qXm9Pe5YqN;kbB{rG+(~SgtAbtTYkZa_uB)q#hwppOge`!Gv^p7-fL1X$n zhORi5T;h_WD9$Iy+kwj81bF4;|N;NKeC+qk~#%TrV~jIt6)8|Fmx7&j$CTTTw;>X#1k%I&;bdB4oE0;Kte9iQ>z3z5Kcb3 z&q~6(JlrPX=lO8*anR%8205P>79|qIkR~s=#36HXWmQPze|R!*z3gtLmfpOqs;Z4s z9_P*ttDOmZ!p za0vquNGOOvLO}!)3Zh*CA_%7-mV|eCI0Mlx0g;C@5N!$~h#|}AC6_qlgYwG$?|@jv zQ1$@v&fZxPf18GG83lQ$gGqEaiG-nJE-}f)aKa^wL?EFg0tqFNOKj56sZ&BC2&Z>E zCE;C%aB@xydY}QChtilnk8#M%TmlLC2)J^pg)MNsFs@DAWercZ`J;2vDu%ZQfY(kA zK&x&yHND`$gU~R5%W^^>IsL~#W08)W01aahX()t9e?ut*8k^K}y44^KQ3VvlvI@G} zgIg3?Ne?vQ(3Oux^+6PU%6N%tCYWF4y8kwSd|3xG7>82M)5|f6<={co?Q|*p%Scu- z!abP0S$Av$rKK5WL3%3iWA-D#AFH=o6iTkK=mSBZ;ZOn%gA!;Mlw8B16lG9?3K*0n z6?6sNe@HlIB|XrHLm%|9s6L9KYb4UpMKag@w+Ll!h0LhbwYvBYEUOsfK1=1gV`BXn zxo+u(D8LeLG#v@PchfQ{mRw`$6icq*SON{h5@;BfT*I-{%di9$Ff2)6ve;<``8@J=9>nS zJk9y*Oja?xy_jt7GHE+PGYXi*oA*b8?}@i;$|ToV24#|KIFmrbm;@TeB-rVv8{1nA ze`FFgLU$U=D(G$xzA#BkdY}^&d2$rj5 zwoD4Au73xTRg7-0^wr3xuXRi4cF|pBfyqR0*FejmL~@PQqC|2HClY8FkwC+U+Q2zK`kvHnu zxel5%o?&ZcnRy2_!l*2>?+EIy zIcFt3&1BQ_sHx z<{P#4d=q(ZPnyRzO*6M)vHQh14U?&Jqlih(JwRhqa}UrkGLeRoi8PcqX716MxrdeXKqF3*LdncM zD2k@BNJGpPsC;n`##DA=QCBV#nB|yt6^G4T; zsZ^5ZxooTQX`Y^;Y_O^_d>XBOA3&K$ei-@5^Q!gi2MQKbw-bcJV)x5VEoS~73;F(? z_s%?c=b0aVR`d@_TS5_ z#wix!)wS3%{>@sz1tE7>L12Ofh6iA3Cn7o`1)q@i$8{q>a10$N(81$8%kkM8z$rgl zFe(>LCBxprto0;L?wyyDvG4n=FGp+4pX1Nf%iB0=D)q1`4ydUFr`!Px#x)hbyu`(I z%!gH6@kWAp+&D*l=IuDhp$WTLJHr$0R%fd6NVsK>_%aMkP7{YLkH_ xCL6+=>Fy-~002+|000pH0000000000004l2OOwJSJ^`PT0VX2`Q6m5V002vSFuDK$ delta 2794 zcmV1`&Vc*c}QjoXct{QhK}tdjxolL zW4qHQMivWb3W!R=m;b&JH^Vs}GELLU)4tfS`p({@AA6+z;P98H@ko16aS}$8hAygx zuBk~7^}@-Zq2JzouDH6Er2eGmkD`fc=+7$Ae`)@7_!z~%Cqt!DEvsOXH1uJbe@+i- zwImp-v7c0yVb{ifIMEk19K`$75%v3Fpgu;y zY^)~fqLNsR{50!+G7P7Q-aPDu*|B^)TCDmF{YV^~{3?ZBYaY(qz6;f3vhiBWhjJf9 zzvpkxdJWykr8TDgJnzO;to2mif1i!g>*(=|3J1e9n>H&C38GOl|ErC|Y%Zm)js2&F zUe~q9u$K<=inU?b>#2OAbuE}BX*B+IaY!sHScD3D87mC2j`pg;G;C?*=i_;#mNB(O ztMeXy^i#ii7)Os{^Wwa3wPnHeszA%ekkf3tpAd&nzZ zD!$Q>8(MHn3+^@wYFU5QoszgylJ&vZ?!z@#m=@_Kkl5Rj8z6DECAUE0ZcFZf#9K?Y zhFizOt+NUQ87^0letvsZar3I;_Ep7Qkz#A4CLXED;w2d=S9oNkuPSa{Rorp~J*h=? zY>m`{BR#VCIVIQdF_4h+e|`caWWqiI2?6#KkeJ(&&p<+WIt3EK(-$BiSI`-dkg0D0 z2|0h~Tq4Qrwt&9+{b|mwxw~Uw*s#blpDdoWGyvLP?ek4Z?IdyRQb(xHcYTf5%y(lf9a%xZe#Y zeHCZvn(C%1NxRAHejFxw=96?Etg2BCe`iH|>AFS42%NSCYv#P9X^M5<~ z^=;8FvB)s0rf`L|V?A>)bPk7(Tx!Q$Vw2Cr6E0!U0SSc;NGNnbLN3r#rvN$-PCmQO z3c_1F+$G`Xo8jc+pv}Wgay~B{N+gIOO;dAPyR{_Ne@*F_JMvHuljw002}8$RVv~#Egi9ESKtf3b5=tVMxTK*|uYg1lPVaaM z!do8Uotmvu0Qv8U8DyBy<0bst34&6dKyjASVz z+=I!RWye-fI)-WQNKXZR%zh;JV|B}-P;!k!9|!^shZ1NQlt9CvQ`k;?Rbx;&tBawzKk{jKBi%@Q?kU5posEY5vvWzkAvs5lSCYFzp ztBy3q4lMCT(~;nNH!S|5DDl9seV zBOFT_)d5lL5h~CK!=PNs{{fd}jB%gGs>_b8e{LeBXPdh=GX&leJraDU(Q+u5U^lW( z!Q>haCeSc2frf#}H5|-(5tyI?24+D8t@XhdFlk8}G{PR=u&54-Vqk)OYz}7qO@m3E z=KOUg%NX8XOxCxVbUk61JD9|q_eX;7iCZpZl4~53GRZZZNuXg&0u5sl?DW%(?X?Cn ze+e3)JB>vZwAKe-n4~3b(1<4~lQgP>qF4^gHNp^-NwZugXZP7sI9ozVfJ`68^VKq6 zCWTj(-+^QqquVQeH8<1Osw1Sk>#lN#$wYA1faOslxyG?5kzB)x1R6#p&@dvo#-)27 zYYik4G{T51sGzkz_(CKtX`^1!=dLWOe*>a8kwC-lDy=sJFt5YbMxd%nj zG!|)S8Y@dS_q@hr+Rt$mf8TXiS<2Y<$zJ8AsV7SSJkzB();7M~K34HSov6_$(Sj(O zu0`a!bn%t{YjJS5_;&n@JZmqS^upZH?~fMWfx6W^ocaTG;m3n;l4v8<&jxB#vm5Yu@rwiV*J(7J zzp`}gK1$PQym~iOeow{uySlFRqbOay$){xf%MHyx0kiKBY7PZIBK!spt^ zaX?KaIOPslFs`Za`6Vu{Wj?IpiZ>F(<00x--ljpuIuJ<=K*zX8<)eIhBwjC3Q;^~| zC1pHo@<7Xy67tozyr@3`)zswQ#3NwLefbHRKeL}9$^r!t%jKm|lWrti0qv8@Bs>9k wlLsY10rrzgB{~6UlY}KE1wSJE1|*ZOB^3cllf@-I0i2TrCL;z;BLDyZ09+U=`Tzg` diff --git a/summary_results.json b/summary_results.json index 2bfaad330..32cb7a90c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1113, + "Performance_Result": 23.0348, "Performance_Units": "Samples/s" }, { @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.405291, + "Performance_Result": 0.402086, "Performance_Units": "Samples/s" }, { From a04f2b159193a9b83fef35902ca027fbde0c6b1f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 3 Mar 2025 23:25:29 +0000 Subject: [PATCH 223/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9773 -> 9772 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 6cfec8250..b78cbe9bf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-03 23:20:07,339 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-03 23:20:07,339 submission_checker1.py:3270 INFO] --- -[2025-03-03 23:20:07,340 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-03 23:25:24,245 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-03 23:25:24,245 submission_checker1.py:3270 INFO] --- +[2025-03-03 23:25:24,245 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-03 23:20:07,340 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-03 23:25:24,245 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-03 23:20:07,340 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-03 23:25:24,245 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-03 23:20:07,340 submission_checker1.py:3286 INFO] --- -[2025-03-03 23:20:07,340 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-03 23:20:07,340 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-03 23:20:07,340 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-03 23:20:07,340 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-03 23:20:07,340 submission_checker1.py:3307 INFO] --- -[2025-03-03 23:20:07,340 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-03 23:20:08,291 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-03 23:25:24,245 submission_checker1.py:3286 INFO] --- +[2025-03-03 23:25:24,245 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-03 23:25:24,245 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-03 23:25:24,245 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-03 23:25:24,245 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-03 23:25:24,245 submission_checker1.py:3307 INFO] --- +[2025-03-03 23:25:24,245 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-03 23:25:25,508 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,8 +30,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.70465 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1944 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1537 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436673 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8163eed6a..ee3de9906 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -850 +851 diff --git a/summary.csv b/summary.csv index 160881b4c..05c77feaa 100644 --- a/summary.csv +++ b/summary.csv @@ -8,8 +8,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.70465,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1944,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1537,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436673,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index de1f1e16c..142bfc641 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.70465, + "Result":5.17796, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1944, - "Accuracy":"acc: 75.400", + "Result":10.1537, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index fd06f103cfdad21c52a340f385a4dda8254b7e11..f4b8b31243f8f82740c9f05f647acbe501dc1b9b 100644 GIT binary patch delta 4459 zcmV-x5tQz&Osq_>%>xS62Q1`q1ONbf5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>9DDfnCC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%da`NR>>j= zRa?=LT|{H56_!=2d=-?1$>Pavdu_Y5{`xOMTEywf{9;B#1Zl?2gP|28AGiC@$Vv8d zM21%XeGu2IiOXp|$n8$wx8JRLTYsYK9q65?LNQZRbDvv;*=nx1L9Rh)(Lm*Ie z{T-r1=umV!=$CycIyf-Z9cZG1oEx13vc5&Eg7V(=|A(UcjGX>JbPxS4(b-Pd_YOb! z)cLsTd|h>*K{(PO9k9?&5C~Kq0)eVSAW(G(1bms#{2i)8=pfsR9rQ~ds(%g+Omzoc z*eOBIjm`lJ$t_|PwnhHus;*jAg{WF`Bdtul_}<3??EV)4enxtKAi%l*PJoWxc6uGS z0khuirE|hoWJ0ox8zGeiYnX4N&3VJj8O)o#_%FEH%okCrR<}3WyBD#rQFh!mw!^eBbBRZL@#l38ex1=cs9RBhcO_Y0?q+vX5szNhVMNWfw5Kc!;_2&9>%~h2&^QQxU5>I7QeUhpAOY5h@ zhdB8?9ZQ)hc?Hw7sgAR3cA)ENFqV;@YVk}?^M515IEnl$f18Z-be2eeun3AKx@j1; z9{J%^UDj}r>{CZP9EO4X7zgu6PP1htiJbUZ-u-kO&Qi5?I0*A&#dwrN4x8$cIB5TB z3RQ0%F512gwdsT7$s^aEV#ci2lYorZ4(guq+ z$w;}vBO`rPas8^|hAZevEvsW|q!t|Mk;Ts`xrUE{gq-&_kdO)c2qXm9Pe9VxmVbN( z62j97kPx1}013H*PJx6>eFsR$`8(qhlgw@xNXYD-0|}Yk9*~gPy#Nw2yO%&hX7?+Q z*!Y68@danI3M_iTafwaNV4F+oWV|1NgpBtSkdX0y<`S38{mHWg+z`Kj7RWVnS`yxQ zKK8sooza3W=pSj|oW}Hc3|(<9xPQbWGjsXU?_ul9ZHIDV(<1Xp`}dY+=vgA=BFEB;R7yYZ zwpa0qO!EvT)mN!ZZC!Ien9XXzY+j3})5l~!&GKBTwo>(jcsdM6>d)#xW`BM-N!1UB z`Vg$vU#l2Wp0molYm0w7`t@zuFR{!pw1#knwPQVVFmw)wj$CTTTw;@ZVw+1CbU;F( z0}=`ykdO=X#3_Legp+&sX-Rm8hr1;FY%`qP4!S(NLC)v7Lx}`2q{#~|@yMK9Iu#N* z8u$I6_%u_?KfElfs*Q3UXMaywoCM?ky{XxnQ48k6Z0tc|6+_vB#2aUW#Ja0_?#69# z3q%)#=yDLr^>fT6Hn|qsT*5#E5(*-aP!NHHg6Nij2*N3dCE*<&&OmfaK;+>JM3;gH zV#soO!6hEKQC_9aB@Zqx}X7?htilHkMYRNTmT8V1zdX7!WQ{KoYp4( zqJ~G;{?R#U6~o&Dz$O10t#YT1%I9O!8MAkqzfAH=*q{UdLW8EWxPN&6U;9+y8kwSd|3wz7za|$ z^2;$vW&d7iZoU-$Wh5&Z;T}w0uR6Ab($a+EcsmwFfgiIU3I15!vM7{X4HW$mNcpdqSzx;pb>^axitR=Tvjp0eIBc>I`;e_()8@c zuFVXAw?vNw-+yVe9114bjjU5JxrT!YGz?6jVPJ9%2eV!VCa8dcSyDk~eeeZLTG9oL zu*Wwns)wQ&m|!29gIRymV3Maff1SxHhPM}!^=&3yPZ*6IOybS^Bfk-{ni;d<7)|GP{o!t7WlF3Qse?1Ia2zw^#bw*i2tFN0{!8O%JidWFok0!15@O zT;o`jNUq^T0u3V)Xc&=PRDhS;@A?(pFV;Ig4c9H4Qi69;HGlY4k^kcKe{+%AmC zGCK}xgfUrW$Pvtu!%DiSmyAhf$^lV4bLAS4xmIb+`B$KPt2P}$eL7x!!^W{q<(5&$ zc%c&5aHvxUXdG(Q0UA!_9%db+VN?Q*L#ZtD>kgm_7?ov?9YGB>XRM?PldTUNe-@jj z9Xrp?9b;Gi>h7Qtcyg$12WTAX+W{I*y_FgFh?>4HW$m(0)uqIeF=H6Vwr*wpjyfcZwP zJ>SHpw$bhIVX^(iI0KVua-)b%%{@TlQgaW`Ffx&bl8H2wOr)WCtXXF6K^0Ik z%PQ!Y%-q9Dx}XuZFT$dFD2kE^G@MMcVt3DLWd7ZakxIt550s{{4W(fkf9{UWN>l8B zk{d;AYVH9VmzsNkhC{iBxd&+&l%PL!xirhnJ*W`|Wtq9hWab`L(glrpbqXaj_n;`6 z#v%<(W6g@qJ+CpD4GSE_cYUg?WNiCnujZzyCr<#phD%L7`o_20Bbkh3TTUja62$p* zEhE>Z%dh-jiG$nax8q;re|dZHbPyJfet)w3E`JtIM@ss~@>c@v;#UHUqOxFkHJ?Z& zdz#6nDj#S087c;=Duc)2^7jFhSrW!cm_039FMgn4KJi;oJb1t8)O;HLF_-V(1@G*G zcY*!kXT|)WG?jel&(eH!i&lyks9UYWnLm=}eliNDsWOqne4vJwEPnt`mcKZ#c%8+w z#Vboy?&2(qqvv;H=?`R5ysN9qFpjh5H^r2!f4QOcC$sMnm;nS=*o=v@ixZ*(f5~Vo zrV4lgifSqlX+b*-E2uL(leo_x z<7)!Lp-JAENp*PmOj*90?Hzgi{4#in?&Is=Ll{3cr=M5}9c&_Lh;YZ%f9W#0Th=j! z?w2^~7(zR`z)Q32OZG0Bx{ybdpSrQ~S@zd-JUu~Ya4HiFXnuW*96MA3+`Dr%_9wlg zt=7E+Z8_7>*N5~0HH&h0n=Rug5#0~alY|r#O&)*w_ ziRlKmoRrTdm&^0Xr9?bsPFxzDPiM=;=xW|f$DW+YgCO+#c0cR~fhYH#PRCDXB8T{Z zxPIUg-*)?fI;(BRV<~ZUJU*2t4mu|4$3W$l&{eYf=&jka@6F;|4F3bO_!nUa1+COZ zY$KCiARd3oYQr!PhVOj}qq{9Hq!eR2(3YGUDAc8;Hz7?NQENd`>-g;}FL4s^sW6w( z|IarIDz+!l2ESk|SF%J&6r%tnRgBA~MDO)3d_aM9L^9GU2_@>mqHT3oP@Pj{;MFJ% z#&NKL2T7Y#U819NI>*@3BM4$6?~q#{wK-#1T9IL%k5u zjAV2{>L1gN07fyiAV4~clPJL#uLC3OYQd;nIR$sUhE?lXT-+zi7h~6TQJ0NYKYxNh z_b+ebsv(!dsVJbTU^J%&h*PF2@O9!3*AnaY{=^3k?6LRy%G6ULd3I?eKQ_)bNhLY~YmxRqcyPgT1ONbf5|e%i8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Ec`~Gq%&q-;bR@2ex;aYF;Xx7f777_AOraj{9F9+4U&g6O7g|P{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!` z50@khcz=@?B$rQO%CbPnTeh%x!AKBQq%5^u+xDz1NOH5b&|@F5LE|_H$uSMfjO3y= z!blnj89rYo1viKND3Q-pJ_aN4!2IU)MvuN}4f~bre-rY|H?9p4%pXWfLO~)~0Mk&J z%_;q%9$v^HTTv~Ids;`+py$qFXJw$KI^K%w*v=dKu<5jI&+Vw!^O}+s$Ev+n5grF3 z81@-`Hdy6>7lC?FPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!XgpNa^4JSW@V}T&3|MM zAg>C-;#rc%gvm~Z%mm?L#>;t@aMcv>*;A)!TY?c%wI7!1GOX+IXvJrQ%iWw?i{&f` zRa?=JT|{lF5f){mcomd|%h{9L^qO{KwfoOPn#ak){9;B#1WC%xgT56bo7??o5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iCTqHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR5dC4mVy* zEkA4HTRJ0bK}IA^xe-!Xu$uWcTAw#eoZhtl7XJ-bAM<&XsFf{9>yrbKIJP@|Vq|duO%bA!@a5lE;%2e;l1$UI^0Y5D9IbtB z(#P4-S#bEvQ!rH?WEA`1tgVW=rYdsgg+qTfZmYL9pKG?N#ECl_y3=qb+v>B7)n7V4 z9X^K9@9{*+L`f@{#cg$xB=dttBladTaN~M7m$QHL$S8~gH%Z?{<3>D>q&r*&1=EJE zX=WpE{h7L|;ULjG>X2AhunHCC zI#y_68|~DEp_zIsJ0H(exsGY9T3z<=qno&$!zg@IqGb#5+|AC6IMDTUTD+|EYgU}~ zt+6h+-V`Y5II`J$=rj)-4_U=)#WxyqLkoXyX~A8tppo`x+bJD)N@so0cKdM66-JA6 z6G+Tm$qkTLyOLWVv3DhRK;mpAJHu__;Wk+Xf((}{NI$>5s&`u1v8rQUj0F zVDUN`DOWgTq^~M&URB(31wE-{b?l6^1xI?+;^&lH!^c2E&ie_FkO}(;Bm~$`Kw^LF zN;kbB{rG+(~SgtAbtTYkZa_u zB)q#hwppOgX+aP4k2G*WWBNRXt~h^}T;h_WD9$Iy+kwj81bF4;| zN;NKeC+qk~#%Trd$}bP$sTF zjnxl_jS*O_zg97%G-s7**OvbZ`t@zqFR{un>V~jIt6)8|Fmx7&j$CTTTw;>X#1k%I z&;bdB4oE0;Kte9iQ>z3z5Kcb3&q~6(JlrPX=lO8*anR%8205P>79|qIkR~s=#36HX zWmQPzcrtLk>~5x(-n^`;s*QhB9_P*ttDOmZ!pa0vquNGOOvLO}!)3Zh*CA_%7-mV|eCI0Mlx0g;C@ z5N!$~h#|}AC6_qlgYwG$?|@jvQ1$@v&fZxPn}%)~1$n50Npv`ggrR?9E-}f)aKa^w zL?EFg0tqFNOKj56sZ&BC2&Z>ECE;C%aB@xydY}QChtilnk8#M%TmlLC2)J^pg)MNs zFs@DAWercZ`J;2vDu%ZQfY(kAK&x&yHND`$gU~R5%W^^>IsL~#W08)W01aahX()t9 zLn#Cro78i<)gTU01r&e8vI@G}gIg3?Ne?vQ(3Oux^+6PU%6N%tCYWF4y8kwSd|3xG z7>82M)5|f6<={co?Q|*p%Scu-!abP0S$Av$rKK5WL3%3iWA-D#AFH=o6iTkK=mSBZ z;ZOn%gA!;Mlw8B16lG9?3K*0n6?6sNNH}LDJr?F*GFu`tQlY+@L9892LU;+&TlWRDb%`z}S1q{rR3cA~aFJRJ= z9%zI;zF|>)6ve;<``8@J=9>nSJk9y*Oja?xy_jt7GHE+PGYXi*oA*b8?}@i;$|ToV z24#|KIFmrbm;@TeB-rVv8{1nAWD+z&cN)to=xz_bFiC$)dY}^&d2$rj5woD4Au73xTRg7-0^wr3xuXRi4cF|pBfyqR0*Fejm zL~@PQqC|2HClY8FkwC+U-XHcd0PVX^zgI1Q7jbEAk!%{@S4Q*#f{Ffx&bl8H2wOr)WCtX^jBK^0Ik z%PQ#V%-q9DdY}=uFT$ewD2kE^G@MMmVt3DLWd7ZakxIt550tvL3#F!Oc7MTUr7jAf zXwS z=ecaF@@bx)p=_|KGJG1Xejh-YM}8Rj$@8l9><0=KQ@0a@!}rThEoS~73;F(?_s%?c z=b0aVR`d@_TSm-~n zU$s=_K1`A@*u0xacPOLmT~k#?VVG>*WK**J<%Z6mv+xm^0R&wH-pjL$6QTlt*K};f zQ~@tQQB4IREog^f1(oP1pmiz!>44>HDCJ|sX_C~T#?ld_ceH+ z!IdL#ajfKNhim#8*7P;3^tBzXi5F|)#hQ4rCSI(G7c22XyoPoo+O-uU&!R1|jLPz{ z|FELiRIx%7xZdse73$P*p(D15KR3Kr5xz}%FEAc{+TTj{+k4nz68HII zd`)0DG|4+NsSXdHDa&`Wy(5pGUj{GHeS95!2;;}*^b;$ggH0q25$?EuI$b7r%Q~je z{SrqVLuf}Acxje>$=*d%7xIYmQ#V#V%l?{1??eUCo>6*poAP5QKi;?uY##@Z{dp>G;V^7^vJ5x=L0by)}FGy;+=#;eWIF7hwnm!kg*t zC6iwu9)HVf!!Qs<_x%c^y)D_M4}|SNTe51P(3F<$gfwwPtp}1?Cx2hrNt^_{Drh6! zJ7?|;sunxdCBNXjG`dE4Mo<>M~mjB;+Asb;MExm zE=cf61ZhvXt5_#wix!)ql0vGXBk4zy%?9SV3Td1%?M;Y9}H( zA_bq2_Q!Q2KyVBlDA2*K^Bq9V|1m4S&?<5uhd6Nbu zK>_%aNF_P}YLkN{CL6+=>Fy-~002+|000pH0000000000004l2OOwPUJ^`PT112K| JQ6m5V000q?RZ#!{ diff --git a/summary_results.json b/summary_results.json index 32cb7a90c..e8ed0cd65 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1944, + "Performance_Result": 10.1537, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.70465, + "Performance_Result": 5.17796, "Performance_Units": "Samples/s" }, { From 564f919029af9cbfdd2da8f9465b9141ee5e9406 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Mar 2025 04:18:51 +0000 Subject: [PATCH 224/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9772 -> 9776 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b78cbe9bf..1e7c3b3b9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-03 23:25:24,245 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-03 23:25:24,245 submission_checker1.py:3270 INFO] --- -[2025-03-03 23:25:24,245 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-04 04:18:46,200 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-04 04:18:46,200 submission_checker1.py:3270 INFO] --- +[2025-03-04 04:18:46,200 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-03 23:25:24,245 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-04 04:18:46,200 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-03 23:25:24,245 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-04 04:18:46,200 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-03 23:25:24,245 submission_checker1.py:3286 INFO] --- -[2025-03-03 23:25:24,245 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-03 23:25:24,245 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-03 23:25:24,245 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-03 23:25:24,245 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-03 23:25:24,245 submission_checker1.py:3307 INFO] --- -[2025-03-03 23:25:24,245 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-03 23:25:25,508 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-04 04:18:46,200 submission_checker1.py:3286 INFO] --- +[2025-03-04 04:18:46,200 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-04 04:18:46,200 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-04 04:18:46,200 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-04 04:18:46,200 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-04 04:18:46,200 submission_checker1.py:3307 INFO] --- +[2025-03-04 04:18:46,200 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-04 04:18:47,240 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436673 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0414 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2039 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418465 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index ee3de9906..a7ccfea89 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -851 +852 diff --git a/summary.csv b/summary.csv index 05c77feaa..5250bd70b 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436673,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0414,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2039,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418465,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 142bfc641..671e8cc99 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0414, + "Result":23.2039, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419348, + "Result":0.418465, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f4b8b31243f8f82740c9f05f647acbe501dc1b9b..8b3c277452e585aaa01ab01bf570b37edaa312f2 100644 GIT binary patch delta 2798 zcmVTqH9EX$AW77&M!k|4j!Qx3tI)r|PnXgJhpN;^8n1-JWgv(Ce|(Z%#SdR(I2vd9w3$Un5Kq#@ze*J5b7`nbtc=6KKo%3ND#1L>;^^1qC9$et87iz*tkA_e+O3PGZW(5AKb|IX6{9U% zUG(sypZV>>Bz{nmMGNW7FYb&uFpPX!f}-?mQC##*TNPYA7by8SirKqwe>V=b`=a8N z;u{UQrUf^&;C7=x%lotLlz}^Cus-P9eYoO^rb)U9B-Xa%8c6JI$qkS=+mc%#ao3Wq z;WqGa8>|9BhRYSCpWj|pT)(Qgc~x;+rq~*36OXjX;teuVu5ignUsYVcs<`0_dQ;2l z*czz`M|x!PdrF?+V;~{-f4u`FWWqiI2?6#KkTka?pMiw%bOI!Vr!PQ4o}g18AyeN4 z5_12}xWpi{>j4Rw-E$xzv)cm_GP@T*LT2|8NXYDd1riHCa29^xELMR@A2=?t$Q|r( zNrQ~{Bao2segYCQ-p^d(khwp3mVh_JFQ5hTjGUH)cb~64FHmQ+f4~F%BMqF>m>!Rz zE6xR%xMXH7U-~_4fBEh5)0KDHJA1##h0;738im<-epe6TNQ)-?j1 zZP_oe%rNRr;RtibdS+wjYz`fH)Q-8tBJYVEE@99C355n76;pFYWZItsk zd&=S@829gux>eU}!CV;4JxHu#D0`52V{eeyaO$qJ@wT`HqJu$nIEdu=Ipz|JJPREz zVITqt1rbOnh(JO?bV@)3;S|J@@GcK$AUY)=@^A*CLqP;FWI4Uy5|_MDUON9B5UUu< z9w6R2TT5c2e`(mw9eJpWNpv}hgrQ?DvB<;F;Sxq7kWdnVgp$Z54r%DbEg=zv(@#7l z;a!(-az+b0&;ZRtX-tpDxMXH7fP}mST)Nf57WqM()+YXpW&@q6EazY@v{l`FKla6$NhB1gV6hfq-e-r|ZL+UwMYY>O10t#YT1>N<< zHHxgn1C6+JY*sQMj{PeBsaSM7NOi&Aqy%Ey)M23%PPjW&r-SSm{@Hi z*KMOIc3_DQnvMiNyD=?_CD+&n#gc0{mO#U>1R91V*KjP2GAuy_49k)Vy6cNCSke*? zG{Ui@Q9TgFworja=sM-n_#bdt#TfV5R$X=MfB8eC;abgIhZzDNi5>}l(rDTgOmG_6 zpkQ(h2NP%*m_Wn8a|e_7@cu~fGjY?QOmdCgq)c)RXA)=_lR(3m1SkFUV0*2B ze@uc#=t*N)1>N<<7baZLDV(pMBtT~O(PFhM zmPz5(jqgCRiqY+rzBV`0*SakXXUCz3*kLje+%;gjlt`|zO-dx!a3X<*5eYPmNUm|{ z*~eM~i3E)>B1{F%rhJgGhHeV|l;9g9c8%!FF zJTq_X+J4>P!x6w`Lyb52{5q%tMrD~}M^Hn} z87uKXBm7W=8FN4sKNJBqvqO=Jy*V7pAAc(HMm;;%V$-x^<=MHd@1oM#K_&3yP}>gB z*wnWJG@QykoI6Ovs08jAwpNyzcTgjY%6~HZj-c+EGgjh(M)>wCbLpTcmcIfG%U>%t z>byqfkJ)~GqZXbkvFYK-b6CBxVS3!=(%r!&aOhAI575}u#RD{)%RPKNNW-`UP8r5! znUx1M!niCm^9bhVVI>}DgmcLZJs^tbuv`Oj*osX({|=aM)Y|h+Yn7jj(eO z7S%&hluV%EWEvH_dtM{+?;ea)GQNGFH1usKbwhV{999})2bA0>Vo`Gs&^XlG12i1U zJ_plNVG~zZWl+4_NqG%e6G&GGhDmM4L#$+}u za1`J5sj`x>?UTJ4o2H&T0dSiRHT7s4-)@g&GLjuRnWRb(=hL-}T$e7t@_!``ZkOMV zf05_y#nVApIQsp`^1J+5I2|eJAIo0}u!>&^(2L4~;njR1mF#IITdI7V<$rgm7_6!c z9*4`{2T*277$;%&v~0cjfr9zOZ%6Uq{i0LzY52!nzJC|Ivku+`)`y=JDFg2p%o!hmu>$je|F5EVv-SClRDQ<+eim5G=*m^F1i$v*^%K z+#07?h^5sGQ$c26D%-%0;YB#q8=&ufV4lZ=>URbXh4Au9_Lw(uigMo z`O$_>xp68Pjuu9(7jbYOyc~>e+h%RiS!4bjf305MyH!)EPp9I5>QZpZ9k5_rUELYLcBIiC3Nen>GaWBee@%*wSUZQ$JCXXp8QZikWDb43}I@_E^ z-t}LBsvq)i;stQdeg6sSKeL`8$^r#*r<~AMlWinh0rHc{Bs>9plLjS00sE6kB{~6Z zlY=ED8bO;5NDGP^GlX(awe_M0fRuqNb`zth>JR~zk(h=z9F?4K_IK~(| zj_ppL7+D-ZQ-r7_eEIj40AtvDA=5OiJnf4OM{A#(^yzGL799Ta7)_LWnWSMnZK^^u zR7FmMco0rUP4(vbbInziH1nqee-ck+Q+<-D`b+Dl!-qKeJsnG#DR~9cw5g7>e{6Q3 z>uE5Sk)LYuOiuG7!#IijEPtDf^mLX;f3OIOCc0@DwjTN6R9)6^knB@OJRF9B{1^xG zNKUh5C5fE)S>FA09L`d;bvOv~W5sxsL=KzkkvM4oY6?|v9WL6w4dp|+@meW{au>(H z7jI4nP1Pu*I@5k$bmKBn26E`ne<#^h{P0DFqj8o`n^lAa@g!aRRiZGTOH)-M|FNmo zRplWZWaFY@WgHF$vY2R93Fc`QN53u)iB$#5P+_lPg(23_o+cWGZCb_oc$&ynjJ|Aj z(Zi2^=C=-$_(4e)Eu=HQI5XnFH1lZ*iqfw|anUz@RdDrOpycByX79dLe?Qdki;7o@ zZ#3kZ7TnN++l>M}@6WnZChnBU`e1DL;fgC77U?FC*xQn8AaS-OH$dWUOKyS0TT8Zv zTgSt#vkC+mE?1C#etT7M{i@>TRmE+YVr!%gJkkb>H_1r3!XqPnRdM~Q;)W~eNiC~m zYorz&>5;|HDY=G^frOm*e>RYi3Ht~n1lUhN(%6=K1`@*436Kz;z5ofif=+>iOnnDP z$oV_t5|hks7f8tLo&yP)-5!vT*}VV~GP{>RLT2|Xkl6Twv+)IIvkEMF!EuRA&S0BM z>SVkhfrO0r6OfSce&!OF%>Bu;1l$n6fELI#a#|AJc|P{MK%LQoe=g`BY2cj3^mq(i zaW1&TBQtaP((hsG%Ws#TuDYkav-gW!D9w|hQJ9VAcUllfdNj%ZMtP!>znaNp*bk>e zndIr3>}N8~`sw^G3ezIG|Ol@6rKbXyG z!E9cOr_;w|KF#u6f2y`p^@Dgi3`gqE>Of|GI7!tHhx!n#)?cd_Ql7KQylabpJNor) z*)OroFtmnng|%Znb1-xchmKro$6R8Qdt#eQ7<52Fp#u^M9gvU<^u#HF4uq3?_i0IZ zhljf){A@Fv+zz@tyg|xQ}G!Gka`fQxcMAUXZVK;w{(w1I{(h%^*Jq@ff7e~nA(IazBEho}MyVp#>9^}#iY ztfUJX@#xCOqIw{TK4rW>H51G)H@g2efP7g83m6Ad&hpDKNoD_DXl}j~{$(U98Q~sG zUavZ~gwoQ4<9It3MS&l)9|`_g-Lfc@T;tFOfuhC?aJpac~#C`&5n z2)dDQf5u9>pb?Kg=wnem6h+raq@j!CM)%($lp8B#L8WPE;ybXcVvPGNm8*`4)njDM zF&knBmUyG-NbtQI%cfXzjbljr5>&vjEUBQgKKOzqE$MZ32cp;`RG<-tLAf;l2V7P$#(f^Et~&Poe<9NJ?8dIm41u>qj|AUov>Xa1*o~}H zFu8_<2{a5$pkZKg4F|Jc1}3P0fmu>PXMOMmOj^SJCTUa;MX?-~YlI;vlZ|SboZsb3;d})p0W!Of7OQ2k zObSmkzXQoCMz>e`+Sp8AHAk54j!h4-!(<}3Yryg-kzC_glt`}OL;?*X5@;BaT;tNc zkF^F82^wKUmQ>JLAABK_mUK}s>2p^WOVtBWoJgQypDL|41mr*Q@U@Zw?!~0F!K7K& zjE2~?O78IC2;j1zMjW7Vs1pZhIFoyLagc^F3EVD>$uc_*YJ@RaX2=oDk;6*5lU)xO ze?fgZUVX#Hu}$TcQO9_p64-F4QwL}qYSjT6PURkE9i(AY0*yndEc5H23K*4TjvYY_ zHD|1(3mV~@BFvZrqWGo=sF`hwRP4>+Q2zK`kvHnuxfYwI9Xrp?9b;Gi>h7Qtcyg$1 z2WTAX+W{I*4HZ1<5%X=K~XG!1sayW zR&3OHjmjVM`1Oric&@~zho{J4cdwS-4laR1hnjeR#-T1Apy6EZ;p0IX#wBpdFfPli zJg5=IWto{rFgFh?>4HW$m(0)uqIeF=H6Vwr*wpjyfcZwPJ>SHpw$bhIe_^rx z#W(|#X>y~8P0c+(<5F`E&@eKQhLVXiluV?dd8}Dx?m-n$GRrFHn9SV6O1hvCwlBh> zdMJvL2{fEcvtoD8Yh?c2jgd;mw-1!2u??kR8t#tGN>l8Bk{d;AYVH9VmzsNkhC{iB zxd&+&l%PL!xirhnJ*W`|Wtq9he`Mw!R?-EHcy$UTGxwkc@v;#UHUqOxFkHJ?Z&dz#6nDj#S0e;FzUt15%X z;qvzZlvxtSNtit?TQ7c~U_S9%Q9O9R=+t}~{xO&D-v#gNgLi@b;b+DCpfr_y=+Dx8 zbcQcL!+fBImVW?GmcKZ#c%8+w#Vboy?&2(qqvv;H=?`R5 zysN9qFpjh5H^r2!f4QOc0VlKa5o!(vt<*+rBa>ku9)HPd!!Qtr?|llRyDcxI6k|Kk zmYfJywkOdBzhEp^vP4M~qW~mTjLW7( z@AWQxK!J5cGSVsuCF;SVZFN^rol|At)hG?daj=00Nt;t$qN8&<$Jo*%2x24ekX}3} zBZ%{T7z0vut)5j+4xy%5oiWOPC5AJdKiMlrM?Kst+)D8U!610(Ee z!Khq01$VuMRqI(?+$YNyW7l<2myK3Ge}X^vFK^?jA(z9cD4?ofG^YlLQ>H5Lb>a`# z66^N<#0L)SvG@AQ+q}qB2Q2Udg&GK~P<7;SnQ-1-9 zxyiqYXTUV~?I)=I0JEYY$^r$IDGP^GlW`9nlL{q40r`_mB{~6XlZPcH w8?DqvY$E{x08jz|01*HH00000000000Dyr^lgK4L0iKfwCL;z=BLDyZ0EUAw;Q#;t diff --git a/summary_results.json b/summary_results.json index e8ed0cd65..aab6405f6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0414, + "Performance_Result": 23.2039, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419348, + "Performance_Result": 0.418465, "Performance_Units": "Samples/s" }, { From 94e314ed9b12d151a69e804863fc8319e6478b12 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 4 Mar 2025 04:21:50 +0000 Subject: [PATCH 225/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9776 -> 9775 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1e7c3b3b9..306854bc5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-04 04:18:46,200 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-04 04:18:46,200 submission_checker1.py:3270 INFO] --- -[2025-03-04 04:18:46,200 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-04 04:21:45,003 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-04 04:21:45,003 submission_checker1.py:3270 INFO] --- +[2025-03-04 04:21:45,003 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-04 04:18:46,200 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-04 04:21:45,003 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-04 04:18:46,200 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-04 04:21:45,003 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-04 04:18:46,200 submission_checker1.py:3286 INFO] --- -[2025-03-04 04:18:46,200 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-04 04:18:46,200 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-04 04:18:46,200 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-04 04:18:46,200 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-04 04:18:46,200 submission_checker1.py:3307 INFO] --- -[2025-03-04 04:18:46,200 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-04 04:18:47,240 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-04 04:21:45,003 submission_checker1.py:3286 INFO] --- +[2025-03-04 04:21:45,003 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-04 04:21:45,003 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-04 04:21:45,003 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-04 04:21:45,003 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-04 04:21:45,003 submission_checker1.py:3307 INFO] --- +[2025-03-04 04:21:45,003 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-04 04:21:45,982 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1537 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0637 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436673 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a7ccfea89..8a8c012aa 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -852 +853 diff --git a/summary.csv b/summary.csv index 5250bd70b..09202072f 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1537,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0637,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436673,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 671e8cc99..135aeb918 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1537, + "Result":10.0637, "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8b3c277452e585aaa01ab01bf570b37edaa312f2..9a6823e68383bf724f7f770c6bfc4897a8de41c5 100644 GIT binary patch delta 601 zcmV-f0;c`2Os`C^%>xQgoQ?-@1ONbf5|e%i8ME*MwF7@tpx1#LFzel3Iwx#JCM3(a z5mH&OhWR$yoHxvz!Mxdv|AMQ{d=aH;Wec+AJwSJJ)I(hplPU0lsQ;(~e{dRUywb-dH`32{t(KX65f)dc&2&vl| zy|0()14?{gTCmP&sL>ugS~PbRx0D+Pug+L-L4r>rNPEg{jkY0Jim~TgP|Ro1p`*Al zPO%V6*J6Lm_%~|-7lhnn1%U|`7(M_~I}*_eDfo!AKdu`9f@A1Ffes$$S&lE>08aVI zhC#VhQ9;((e;aLOI9U|dt->mnYm zV?M0ni8m6&<38#WZ>L3$eISw;fF9$Xm5=i2dA%e=O+kuzO3IYX)?`MDoX+N7hmkk^ z7oa)~`8V+lIOe|n1kE3_q9KF=1y7ug2XK=sBwGPxlZPZl0r8W~Bs>9olL{q40s50n nB{~6YlZPcH1mwev4wJDZ76DC@$R$1jp_2zDBL-6=00000+A0yc delta 623 zcmV-#0+9W$Ot4I_%>xeA2Q1`q1ONbf5&!@g0Fyx^8ng8SumgYHc6uGS0khuirE|ho zWJ0ox8zGeiYnX4N&3VJj8O)o#_%FEH%okCrR<}3 zWyBD#rQFh!mr}kXm8)q)FL84kVU#STEETVfcXc{aWx+V7(S99NRir;EvcDJZd4s*s zJNRO_2gb;dnQum*GP~MKeO!QqH6E71id+Y5l@{E{NGubz+j6a9&G|-sEofO^$!2{? zg`oN7E+Iign7Wg@+n7?Zxltu~J2o2s1W-!@1e0MT8nc54*cSwn3~CsYUmzfVZo)7S zgztTd$oIq!p$J(?klKn+il?OYeEPh<<#SPX1|?Xxm+Anll+47 z(&!5189@nXZiLjk3Vp1X=^Q0KFfCYPG*sva9xdvJl3U7+gEwa^xFEqN5u`okwnF<5 zEXCOKJt*e0=+IHz8mCx@rQ2bD%lJ3j0iF;t!wLcuEHHcmrgk8r9x3>Mv_G!t0D@y^ zK!FY(=UI-g-T+Sd(S}aBaVi;(7DlZXad02J9E@$-W^K_~WBwd}tzO@|Ra2=?r{aL> zQgF&0uwYzW;@ct~u3 Date: Tue, 4 Mar 2025 07:32:05 +0000 Subject: [PATCH 226/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 9775 -> 9773 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 306854bc5..2e01412ec 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-04 04:21:45,003 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-04 04:21:45,003 submission_checker1.py:3270 INFO] --- -[2025-03-04 04:21:45,003 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-04 07:31:59,589 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-04 07:31:59,589 submission_checker1.py:3270 INFO] --- +[2025-03-04 07:31:59,589 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-04 04:21:45,003 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-04 07:31:59,589 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-04 04:21:45,003 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-04 07:31:59,589 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-04 04:21:45,003 submission_checker1.py:3286 INFO] --- -[2025-03-04 04:21:45,003 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-04 04:21:45,003 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-04 04:21:45,003 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-04 04:21:45,003 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-04 04:21:45,003 submission_checker1.py:3307 INFO] --- -[2025-03-04 04:21:45,003 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-04 04:21:45,982 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-04 07:31:59,589 submission_checker1.py:3286 INFO] --- +[2025-03-04 07:31:59,589 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-04 07:31:59,589 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-04 07:31:59,589 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-04 07:31:59,589 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-04 07:31:59,590 submission_checker1.py:3307 INFO] --- +[2025-03-04 07:31:59,590 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-04 07:32:00,777 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,16 +24,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71285 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418157 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74482 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422257 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0637 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436673 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2039 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8a8c012aa..ea8789a64 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -853 +854 diff --git a/summary.csv b/summary.csv index 09202072f..390689c48 100644 --- a/summary.csv +++ b/summary.csv @@ -2,16 +2,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.289104,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.296426,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71285,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418157,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74482,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422257,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0637,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436673,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2039,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 135aeb918..8697e5e01 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71285, + "Result":1.74482, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418157, + "Result":0.422257, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -364,12 +364,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436673, + "Result":0.440243, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9a6823e68383bf724f7f770c6bfc4897a8de41c5..f08c363c1be7b2e6448b43218cfa94886dec5cb8 100644 GIT binary patch delta 2782 zcmV<43L*8cOs!0?a|i{vd~bG*i>;f65?= zCw`i}jfb^l7AwEM2%3y*(lG4W#1E(XGQ&ZRV)$`fm&u-{kt zK++IVC0K?Edlf4Tv5xkt!Zd7Y<@e)htX46#Wvz=Ae)Lnnc^F3zTD+(s znfdvh5eHIc!xH4BU-RO;ZM9Xw)pLQC^&=m>`)2*HcAqC+DZbH=Yg%wae+zCm3Tjz< z){T<5QIfU6*lxoWSC|%QCXm?Ml4~Gwwk0<};%-ZBfy7%&wz^x#-L10(f^?TFNISp1 zsa(^nPOuPSc1g5K1k99uoLU{8-Meox6Wd<-Py zzPEsc4A@5?A;5kD5_4Pfe;G&!PbWY^c=`e)GBvl<6Llwe@8~<@}=E_=9k|t zKV5ZByJzngYoR1t3=P9{G{36`(WEvRXa7#Jg--Trrs6>_oDNi+E!R{pRY}@Q=691Y z$yYu}@7}5!wKP_0vBs+9OR3slG#t>WDObV;qi(4SN7(|zYNS>c#F0sk8(BcvXB9KrJfrNqxBosuq z07MW@K`aPw^Kb^DTL2;tXCS&1L=Z!!(+e)~$Oq-6``-bvjIQhf;;p+iCDu*pm^;>? z9wyP_Boc;>f4Rga4?~Mf7>PhaNdyu~BA2+Nq7$!xL=aBD@f3u&J;KQuE$Dy>=sJ|f zba{+NM&<%Y$Vb4XS1xQ5KZugb*k5FLbnPF#CM~0TdjNRlZ2;(0-MS%nY&-}91Gp$B z1d`i-3^WdDNDF8fgGfUmL>fvV(6}Ve$y$RrLx*j?SxE;};?bFpMRh?G z-DSK$nF;2X8_jERwsUauOqgwm=C$MJSd ziUM!59|^v#Zdnvcu5svwAkc6qfrdc|Gz?0v;ZTYqC_w@SWkG_rpa%(OtfT`f@#sb$ zi|V2%e>z7Z4V@%6n*SD|+?XMADy2~s-+^TrecWfNTs2IrwvnrjG{p`q@j=s(;Ac0M zO|j$}M^Y@ghGPjd3`?M4SaJ==QWjwe5-=DlJ4!wi9sM2`eNe`&NF3MM#>tWz+#hJy(-3{0S5U~&xy zvt9%yNWj1>NYGwid;yb|bU-C+`-VkzQ4|9coMUq^>u(xNvYYeQnJlAwdofwxX43V9 zVeViOAKo7cekN|Ylu52}Ov)tJa3+C^(MpDbp}e3}$qRelGOW%O>Z<*T{5 ze62dBxp4%(P2>)fiQujQ%cDecjbl+FxrP%7G>k}~VMKC`OV2*m8b~Clgb`VgpuN8M zLL@EepjOg-R~FR;QJhGiVc#mPHw5HAf3f*mN(c91(%N8B*0Ys)eb@Hu4j+yHE*omZ z0UC!oae#(1xrY}AX&95h?ZTKWvg4pi7?VYY9KjqptfYfl$(UrO91z7RVb(z!MkUZVl*%H%e-28( zs4Q~q2x_P~V2|pBJ#vBmE4@E#`b|_M^H-|&{<4;B2sAuO|Y?^lLY<2D!yQp+` zPzgLa)V2dO4)yH-4X1Js=MK^^DuISkS!CWp2^f_{_8mdpHD|1(11jO$ugs-`qS*Qs zXxRF-WTVb&RQ{Ol*Eee6xe}Wmf1Yd&Yt%PPkK0^&JGcZ69ctnM8i%@gfQECqhmQwo z7?(i9xGb{rpahJ|A~TO*ZXQhfHiE%rk;NX%r|Q7`6f2KJ=r?e zG^AO_WJ&{*DY;R^rsf`?ajCfnXc(DDL&-!MN+!}!GG&pu2PL3n7A0s)e`fAsB^^)+ zI~QS5T@*#h1R74JEZN=j8kv9hV5F4(?E|GWwxKkn;qExBlwt>z+$dsGa}UtC)Z7C! z9Lhb+JxIf#1nr?)KC;N%gDPQA7MXh_GxxBP4yeSdQz)6a2Sw3kEYi?rtSs5w^BR-s zAjeUB*SE@2`nJ#NRc@Mkf3gLDXS&qXQ``9K_KAvzs-?!`L<^#9xR#OY(&eB0Ux|a; zeC#(TQUCp-QS)i|$6URC7re6%Ro(^mho3e1L2GDP z*Po?X?-sS>FHp0Zhckbu&i!~8P7`gc23bdqYW4*@UjE|1;&mF$7OyN_yNl8^nmoT7 zDZj7c{9Rqw22qqgzsZMW{mTu_KLNAx5o!(u!0s7llV2bof6HpaFc3xe{R*SKE!pl1 zVLQ;4tQsiPrJ*|^O%hS-fuz>S-&b)GCjqaz9Njx-?hLACTh%4M;Jh@tMtMe10-75k z^}0qM?L2)%i4RN*))@^o+JQ&2=BDD7a^v9584E5*@JR$|Pr0qpCIm|{_Iv}1`7Ang z6j#P67Gmkvf7mkq&DMYmLhiAGzyu2n?}4cuiRg$Fd_>wG*9`!{F?66n2aoeC$7gQ< zr~G8Ypjaj+cwq_hHpk>iwxWpr#U>atACJ*Hrkj zh==Q#4~ux>l?3s$i~7XdagjqGh$IG}hqx!@vwVJOCFiIqNO6~vG9|Yyd7x!M3HjQO zyy`y#)p5wbi6_7z_w^@e{s6O~A<6;;xO{VaRFiNdTLJHr%p^Pkd6NkxK>_%aOC>r1 kYLkW~CIrCl8D^8PB^CiolgA}K0iTlwCL;z>BLDyZ0CfZ#O#lD@ delta 2784 zcmV<63Lo{YOs`C^a|i`lQn=S8MzT6x-=#H`-4U(&-Kc3(LB(4bu zRnp9#4*W?xl`Zv2rs^;4pAH}5M#SdR(I2vd9w3$Une-Ka7#lK1v=5uMNO5{Jb)P|}&goA8cRIH4{!9W%h ztt!Dh&En|SPc`Rjkm(I@+y^rfwN#aX+3WauuU3TV3?HHCa2u=wL59l}q@Ukj zRb0QSxOr7^Tc+3=X%mmM$>I$%Qm$~xNMBW4zpA+53VKt^>ew2o2}gQl@_S02;bR~n z_q_uoWWqiI2?6#KkTka?f1iPb@N@zsgr_e+LY|;gAR$xV1rl=q&bY)Nv+Dr~ncZ_B zA+y^95;D6NKtg8s5=hAGegzT>KX4X);4D^wNgp^avB(|la7lxV_al&y@qPjlGTzTz z;*hyNd6s}T#4n%)@{F98gm<5>Jugsaw7>)XBMqF>m>!RzE6xR%f4F33E?@dRY=8Of z^3#=f+B28uf@~pV=|v+c`j93 zsro@Y9fl+IXLTSme?Oe0>W4#Z2v+N_RSYT5S!Ld}#eX~c^=;WNvCJ^)P2mW0$9iUC z=xh!hdDM=%#3JvB9WG(e0SSc;NGNnbLLSf)y97EAPTspuOTxQ6+#%s-o8jc`z~kXf zazD>)N+gIOOZItske|yT}BpCPajk;CWYr$L? z%{@r0Vkmo%cw=vn*l_Buv+=gL1)_sNbU29Q`8nnii#!V*E@2=72?Y^ID2PBpL3Bz$ z1mP6KlJG7MXCOKyAo6eqqC-IhF=RQt;1ZX-QC>R#9T2M+${rxzI$KL(qiNX99eJpW zNpv}hgrQ?Df3e8J(BTqBB9KrLfrOICB@Sum#4RBagwszvCE;C{aB@ZqJkS8mLupKp z$GBu>E`Wr*1zft-!WQ{KoYp4(qJ~Gu`q4RQ6~o&Dz$O10t#YTe+Aw3#Wjko!~>1Ebme1FJrG5A z881-H1oO*{?!OHnU)I3_#(|Wx{BcZD*}oTcCtnKxGLn^ya1SQ0R~=hIY1V~pyE_&| zfw$R@1m9LSO$sH~*mOe>XgHKW!=MBj1|`>UC`B2RpaKSENd;X&4-(E;i3b{S=|&%m z>Y*sQe?}q=T_iWU{}!R#SRo544ZSYD1IsGLxX)6#>X=w-euml>0CD(8)jWR4j1q{oQ3cBlyFIdtN4>ZEDq)|N(#kNp^M(8@_ z()b^6S;ZLl*;ZY3?D<2a;abgIhZzDNi5>}lf6{2$6ijd$*`Q!@4F?lw7??oAz~mYZ zW}^&DPyqw8q=N4H;tQCx!~>16?Hd-=Ls1M&aE{HvY`keO$!^YHXR?an?Zsqcn@PtN zdUFSp`0)Nn@H27Kp-gg(-K0!%4QCQ)7?VK5m;@*N^k93fflPu%=t*N)1>N<<7baY*r>!*Y$#1!b~XEtB)Rd?}o-pd>(M_t9dtES5>(){XB#vWn5|mA*DN z)7QE!3}?rohuC2<5!^Lkx|B$+u}w-O*Ki_%h7k!gj7Y9==-J0w1BnEUFd|DT=&mom z5J^is)JwYW%A$H8iW3Pm>{F%rhJgGhe>Pt$8Q@+_nj1_SjXX1N?Am_a;lmNYWkZcP zKx0!U4$yEW_weE%4Pz3xT^N&Pb{y0QW3tSUBbXzHm3XL^j7et70Z}}22{5q%te@11Q zV@FU!%^554KqLH6gc)-{6h9OJHM2vJioH1;${&9!@9!A8MaoInRieljLI_mj-c+EGgjh(M)>wCbLpTcmcIfG z%U>%t>byqfkJ)~GqZXbkvFYK-e{)#9v0-}L=F;84C2;6a6A#eX)WridoXb6YJV?X1 z1Wp;oWto)+HNvBC5@;tU- zSsMoy+h2^+F_{K8idfX#12hgb_W%te6KN=!NJGg)8k)x%W#%4K0VT7ne}b;T%ss5c z1C6kA5f;@$QIt%e;ba;WyL(?UTJ4f19SBJOOZ<4mI^?8{ck^WHOQ+IhmwN5a-jij9iy4zw&=2 z4sMsP1ivx?lvv{`n%T$%SILqSb z`PEqZ1DO=B8mcmktl9U|O)wXsFR1JX$n&6}OZd2d~apa6y7kB1n76ZH=}eScaw-}27ACD{adhvz9F2Y7XMH(XWBwd}u3p}TRa2=Cr{aK`N^r^@uwYzM z;p-wEu46u|;)ypB#N$5d6K|(Qj(s4K7=Rw*o|TXC>3Jo+L`^}8c}mKZ%+_Q^i=58p zUx$%5{THA*4f!|m3^?Y#{RGV)v!5Z#0tIuYoX}R2ZX{a)@sr9VJOO)?2PHuP`jbf| mIst5xge4{f Date: Tue, 4 Mar 2025 07:36:43 +0000 Subject: [PATCH 227/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9773 -> 9779 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 2e01412ec..29bedb9a2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-04 07:31:59,589 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-04 07:31:59,589 submission_checker1.py:3270 INFO] --- -[2025-03-04 07:31:59,589 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-04 07:36:39,022 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-04 07:36:39,022 submission_checker1.py:3270 INFO] --- +[2025-03-04 07:36:39,022 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-04 07:31:59,589 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-04 07:36:39,022 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-04 07:31:59,589 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-04 07:36:39,022 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-04 07:31:59,589 submission_checker1.py:3286 INFO] --- -[2025-03-04 07:31:59,589 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-04 07:31:59,589 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-04 07:31:59,589 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-04 07:31:59,589 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-04 07:31:59,590 submission_checker1.py:3307 INFO] --- -[2025-03-04 07:31:59,590 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-04 07:32:00,777 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-04 07:36:39,022 submission_checker1.py:3286 INFO] --- +[2025-03-04 07:36:39,022 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-04 07:36:39,022 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-04 07:36:39,022 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-04 07:36:39,022 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-04 07:36:39,022 submission_checker1.py:3307 INFO] --- +[2025-03-04 07:36:39,022 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-04 07:36:39,965 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,16 +25,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74482 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422257 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421564 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0637 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0423 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.83029 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.22502 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2039 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418465 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ea8789a64..bde83afb6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -854 +855 diff --git a/summary.csv b/summary.csv index 390689c48..7125778ed 100644 --- a/summary.csv +++ b/summary.csv @@ -3,16 +3,16 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.296426,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74482,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422257,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421564,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0637,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0423,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.83029,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.22502,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2039,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418465,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8697e5e01..bc8e50d0b 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422257, + "Result":0.421564, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0637, - "Accuracy":"acc: 75.800", + "Result":10.0423, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -399,7 +399,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.83029, + "Result":8.22502, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index f08c363c1be7b2e6448b43218cfa94886dec5cb8..e4cb98d24a373a96597e1780e35363556c14d9a4 100644 GIT binary patch delta 4493 zcmYM2c{mhM+lLKPGm~sXc9YBy8e$r2WUMi=B(j9;`%c-7L4yjz*d_a(C2N+*zD1ZU zQOK68+4p?<-uJt{^Vd25+}HV?`#I;jo>Tgu>OsW-)fE(%l&=Sng2E3<3Iwx}{!oE% zQ!aDTOo4Jw8I%L<^6@DA-oXfmwdS+8+Nzute35tFlly04(lMY~?-U19!E_eY zH#Pf0ru)$+PoIu-`0!jRKyoB&lErD_ULdo-S(pW7ycG{1&+_zSbDOiVC6|Y#Jq% zCl$&MBWRer(THv&@=;vmq^f)zrsvO5+|qZ>eYjvoQNZhv4yv6rGKcJOzSx?6*?HbV zimNdPPqG{_EG$H+k|I9~m3C)eQMQSwSetFceo*~#rA&ke3bN+mTRVC))QQtkg>xxG z4BJYc!t*Mt%j|Po`#(9U(V?O0))NLJ{5*z>;>dcl-zIQ0s3+u*%31=a)9LkIMUwFTVnqO+H6BNY!$_aW-V-9AOFcB+Qak5tX z;~764f_?-@HO1ws!N^paI*!Hrw*zd=nU0Zb__FA7#GtrEF}z8l<8i!@OkZM`Hzr;> zn>zaxZqY#f{+cyMOl9eo34LDlANhEk(t~FmH(=JFsO=~B%Xxhk?}LF~{lhSr%5E_5 z^E4xnl5bpq5>T~z74I`e^aN5)|Gh*B$ZJvJ4xb|4;+hLtiDm#7;x@!60iz$~xicH; zz(CytT72-zG@R(&vHFv7kW3RidHpw$xo{GU4-SZ70QVdKFD6(pFNTtU5C z-LmVv4Nhv5s$=dqq%W?`zR?&lsSE$yuku&WfI-++%Z2PX88x~gM{USVesR?KIs$Ch zHW1o;4W2f2X1jnmk>XJ-`8yrysMEdpmBXWIbZ9r*uI-f(O9m+& z4E=znuU7|d{z)$#?AH6WT1-XL7E5c7fAKsFlet0<*1ez>D{`CJ-rMmmq|FL-x_xnK zfrZjINs%$GYV!i1l!P=0iMYT{Fdt=K|n3-sW& zs}(u?avagN8U-_X!bUx>mO`)Cw6sln+k{)#<|;DQfcr(RlELuX{I?2yNe5$JT=+$z*s*nc1uMSl$p=&)_g050biV?ZgxRmVAp8J&Gf*e5Jrj`H=0g3;Xjted5$?Q zs|c5huK)7AQ)W?GaE$xw6mNPz=BchEYyP(iLtj<;sJNr2&YU)u0?(Y+nEx6Y8Uf55J^nK~xQ5CXucxCiU=ybUt zXh$R(R^wx;h}zq^1EAB&3n@G!2Xb;6}xOh1{~o{u;&701EtUdvmDnR8L!2qrzeat-IJ=YCQ= zSEl-Y^bzC1SF5eK-S}HFBV_$+66Jy}^&xM+W^+D~tufPzK?Z5|iUnLnEHq_MK-el` zFbctEK*Un*!W|<0TTOh}pWREUJ=eT0BdnI6U=#*tq>@xo4bI#*a{K`ZE3dm41(gG+ zYR|GM0K)d89Ahz0J=n$aW&j5@*bSM7roCxp6fPqD4FW<(X0$L0$0sT3#;E;*TaXVo zf&gMf$Ax#?O*PWBshNZJ@*LGMx^D88(lUfW$t2^s*w}hMd+(N`#kvx$@%5^~qmv^e z{$6IqP!X=djS_iuS=9Y+Gw!uRy`~y^-!}=Q=9g8AsVn({YK^^XK{!snqrRKJ?-}Uj zBz9p3#tmu$1oRs1GLZ9@30Aj~b8C(JbjMN}TfK}R`BhkSicDvfT54&Ypt_z!1mBrk z=J~q=-IeehrE@of)zELpj%+4^lTfZBRLo?OK2Y&U@hHM42DfSh42aXY?0wRz;P z7H!9=%b@u5H6xX12_|wqas#@RbhlsWCdhZq?@C3@nN0)|^%@;?LsE!KFht^Tf~;!O zGjY{-je1{xd=PTMFI&)?(DKv=Mb~%mjkiIPMCkEhAz%a(PKi5GIN##p?8M4sh_LAYPCQVdCzReA}U zW_Kd<&qC+7TgsqNNf-qefWB#c zMbsHL7AFYDz5Ur+gWFB5e!Z~{n60S;WIjn4LFK+bz9|yf_KAP6AM$EJLGm(tIQlox zjx|bE(OrqX4?gq(@#VKsXoM1kKTa89v>NKjZ>YwhJ2xyvRDOnG3Y_+dL@~~mi>IE8 z4+W$!cq09=8dd$Gnz{$_x<8zBPW#;wN7`kVaZQHePWS@7Yti%T$*@a7vq6YMs^|UL z+8$Ag*J&te!l8Do@tF3^1$Npz{tzg_Oq&^D2WiN28^lg#LSyc2TR!AsQnppk%_1+V z?9A}Qb9h>7=Anz{?p7dkVon~U_7fr(sTPaj116rl(;m{$0G94bfkfkZ)t9{EZb&?i zU=BfFRWJK#(8q(6rzBX*hm0N1QYTjD(g^-E7%8NUu%LT{S7E)psPmWZ!#|{qnTfd5 zp1Y1;ZN1|Y#o2*jcZa zy<*X7DGTg_h#?U^JV6w)fBKvGz=$dZm@p*UoeeDAYh~fH|99?ceTb-)O#WKJBD*=b zFV|3*RCVGc_(muTfyt+vKXOXZBDIzDZDJ~4Pp{Vmh)E=s>vf|g#BuF)k>AZF7{1Gy z`Npf3xg)WNaW@`3fdPVkeX;y4++#!)2}}^-`A>+5ix6M2$H2sYj&DNkCJL~tG=L*5 z8G=w=sK2Pavxvazx4x$4b^zaVNe|$kDQ$JyA<3^KD)gIPSz{^H_GDl)OP5uwC`hj@ z8Ckgm#w`t)|3h{ms(uCYH+w9yS9l1Kvv;# zKNTWT+u)TdDcgVMaHoBz7v@Tz;t8cs)oa(9R2E}=PmfgwWX3JX1Ef;(sAIyJdz!i{ zDCtv);<^lGUR#cC$W8dbZI_#7{-TVNl=ETn!S04us|=sV+y9mCAa50J|6-RH&MF8( z-==pqgO5}+?#*=M_uqWdok~EjCO@z@L-9(yocQ{k3N0*lKtTg%wC7JzRqRz_$L1qZ z2bsY5&)Gns-p{h{E-Kdiga=t1n$4erY445aZS*&V?`*ofjvr|W%#806k0+qNbEPHd ztOdt$C_6c90N9gKasI&s#rp7ikUeXBrtEHP%s(cUHC}lM_Fv}K3KQ05L;j%`+bgse z+bwnRgQhO4$uI6qGkZmq@y+X1ehF#$Z=C?MDZ*lVU&Wh$-Fv)+NiQ7c9}eN?C{tc& z@Y-5DcXW5FJA8ZkN~Yv?>TcWjsm&PXnH8<}UJOqCufeHlaQeAvaO_R$smXty*J916+DivAOr4>NR5)CC<>H)}lC= zC9^P}GM2fzy&?qb#Wd8wbnf+OTP%KCbPlbyvNVbo$M%04ZOLghp(*Qe2tQ$$8Jg{j zt&@DbUEMAgNaJ|S{`h#3cD&=2*p`JxlL6!Hv5r>>J~vDi8yVIoU&h)FGaIT{A5Gv4 z9y$pc&f3#rcZSvSUuBh)mf=Qg*+-Kl6IF~i_fM`obCC347j2%&n@KLpUocj-DCa#n zQ;Ps#O-Ej~Z^P2?$AWg){qy(hFGhzLraCe>rc1yx`AsaWbaLyiyeI3jPNI<~r=rb# zBAlrU=_$J5;a}nx8^2A*c7M4YnzVecnf6}TmZj%$WWrD@UmOAKZH!A7CN)}fJz5Tm z%!$5HokQJ(?r*l*%r2t#RXQr-xRqgz)95BMdR;C+wFWPZ*Y&(t^F4HKk}W+w9F2=s_OzV;Vp@oPL(~GMhrIAS*~MCgDSBk*#EuaEQq}U%Lt~BtgNVE-oLT>zF_dQ|5XN=Br}cdAEsp~C=y z!-#6RGB%13OFImn7WT_9T5Fm@^QeR-)MH5(<;7fP6f?Ix?JLDTRXZ?gQPFiO_1Czm znXV*X=b&INctyvpkE_DW*ko>p2Ucc&4>=b(4()ivvUkfhTzNl`JB~GYV;+{Ew{=>{ z_a?_rfzMk&K*ysIaZsH+GPhN7-FFyAS=>&%#_-fIADQYY^{KpqLm2fwTj|A+e@s}h4y delta 4579 zcmZ8lcQ72>*IulwURGaawbg=%5Tb9i)jL77sH@jRi?w+57AsnW)n%ivwn|8N!xB+~ zRT6>_B8Uir;FI6XZ@&4y`^TMo@0@wgJm)$0&N&xvSZ~-gMnPjzoCyj90sxU<42p^i z^Me9}vJa?GWQ|#NLYqSEkYH&&s*IfU>aE@_sWJaH6J|1&34n+1htWPLcaCvLbP?B% zLDgfu{^(`G$~aHd??c#ly6Bs5{j?`rt$rW)wGkDq_cGq~eXWsNmg}TS6&v<~>ug+K z*&yvZ4S0^cI!Iu0K~PXa3sacZ`&+^bvQ<8#{Ooe>s-=jJQV0ahkF3Ik;Fg%5?}Ggn z@jv;c-o1Sjd!9lM8A2j5k@wasnKGDlS~P}#MC9asLXL_-!#116FeB#!v%Z9O{tahG8})f@TogAz{JLra(eXZUXO;EtXR zDkWPdG8H_PPU_Ps8JX&;ege87gcnk+e1gbz5z?zjW^G$+T8-Pt8WZyL!ZPG z=RRxN3H2M_c|kfTdLlpM-80L~2NL9D#WUHQnyL@VFfNauDg z88<#$fyvtYiAeKAQZfip*vH*e*`bh+lHQ$~QJGuh^H3_MeK5h?>o<5`H#{J@UQ9VW zmp5ZCRxC^ls&!&N7|rqE`iL~2yUkiRzdr9uycjC2U^`h;WrPp1-cjOc@M9#4iK_M( zh>ms3Vx}tS6{P|iHdp}3JxE;S`|Sc>senD_v_NuwREh>ehk%XJAmz++K_*kwCm$d= zE-GF_BP@WKs&chec-@Ez;qU*b1=a9vSmhBp*m-yrby+Da@Tj=w*>`;6_OirVGzuG{V2PQpUt zB2fA)!Ac^>d!7O5zh3xf4CT(q->Cm)RoiYbfoPm430!&b`lWkG@C@sZe=QpQ?Bkft zJD_>rWb3xK+{Qc;MjP*RSQ7=Qtn4Rq9JE|3UUwHw?T z`SU7GK6D1noE*AJ#hx(Mq(we9T0uOBqF~4euB)cP!UO+M~+1*UGoI%PBtm zKqQ5JmdMh;ZZc7^@{*&=HTg|L&w0x=H%&v8H}fBhDgCAARUTi={hXrZm3Cd+&!xl5 zr1G=$;tP#X#&fC3d1KNsgZl63lg@9^4q;fh;#KIHCXAV=EU(~jS8!oe!05v9KfE~2 zP5#9+-d_e#9EyR#Fbw1{Ck}s~@t%y9DD{A?UHjrIuEEawpIWwKz2xQEA7(?!Lz895`{V1-uN4dQO@afg(ue}@GpC(hbR&E)?B#ihQs=; zk*_n-2W8E=rAQL9#R1c#xV*!szO(t3R7C;1V6axg3uQughB(_ZZmXOQ*NS};Qv(Vh zD*ynXV!WEv0iDqyQI^yIz&ZTtUE)VMge>T=ChnbGmn*t2e2cO*O-(g1(+E7o={nT> zX^TxLjZ%(M3$~Jla-x<=3~_WTs(xD2tLJ=Jf`?8GNOw2h`LiA*JG*ODoG{s1mWC_y z@rG!f2P*k)Dv(OAH#ERrsPCMudEp?Z82w*PMe4k2TKjuM#0$$&upuTC~9< z<>Djmt|taQhG!9z>7)0yU!qa!&>EY`0S-0>UF2~_jLDSO&+VT%_h9!DX`ewMIFUo zO=kPp1!~gzE6{k`M$KmAbeiR~y`J+QV6!P;M~Y<6E#~;}2erc#G*j>q(UR-C4RnP%$wH?tsW5IGf?Y5uuEBCs})CU_jQlo@_XWY_sIgh<+ zo3lxlWQ+;%Ry3XPmshBc*aW&cU4DW77;P&0phCu8d;r_$Q} zz!Y7bjh|JY;y3b_rPUq6jJrMeIDE>4?dj+TLgo5iUYo4E&Eb`r2WgYD_HKTQTQ_*I5K+^Xnq`$!@0_#XEpB1!xl4(+ zg%yllpA0?a9Irtk+w*XBmc9H%3~$OV@Jlu?pqVV^11^35kL^bHP;q-at`3=Y`LK0+ zvBlv5;Z1FkxxUX(^oH{$4+C$+bS7a?q%=iJ+GT=_`X2^_Sg6Kbz6q8ZfD)6jTDJ-9 zn?;q6uocf|D__>I0|Q3E>f^ln!pQd1m2M{DCsx7N z1hY+<&YEmZD>ULm0IM8Vcq^J8);t__2it9IVZR9&>d*6h*U*X6d-)v53p(ZsY~FI zuL>0fMW>i3FlP}(C&z$9(0XykBQyoHP!N5AiB92hqFubA7PVNiHub`Gnh&e39>{Wa zvM@_K49@z=y+@;vdLy5E48^VS>RtSu^v6{m^J|=4(hOU8un_lCMUdkiSbvMhFBNp zV0sYV5h#M_@0UbOI66g>Ve&#XV7_OG+F@p_6fDKAAu`(m5W|*d2c$yDtGM5^UG|td zOmNwC21A#i)G;Fdgj~IbAj1)1QlJPr@g}ugx?bRv31d46GK>CLhj}&LcqOA} zpo9gRn2R7SIkcPQB}r78rXpj(g~iM67Aw(|Ff!TRQb~`|=pt4}s~Av@`!D~-^_K)f zbiGnFAqe=&=nDfT<#PksXoXgAhO3z%A(7?E69VgUwKW)5n}r~*d@#XpUBOgh?TfMx=uxxEL;AW2F= zv5B_`NgN;~C#HYrO@U;~8djeZqn*g}n&(bkWxO4+z^$5x>0g_M`xMpDr{;XBeZ&j@B>$g8TUl&;R~27Acgp59ciQHLs(9~X zc~P)KQt)0OfS>V|_|p4KD;Q=i-j7u71Tye?uU^sa3Xe?;AtYs8(azTx=#*^vi3O&Q z3f{_c_UE&nB6}JrU04Rv+P_jPR^FFbU4jC=Q}Et75xL!~I;8lQf??gRvcSM&*X1v5WeyiJ2Ucw&5Mo>{IsfSZ zQiwWsuFSpIp{SB;FO#cW*;TB|eE`b+DE7c2gmI#C`XhxL&xqE@1gs>D2^UH;}YdjQmOn2+{eq7R9a=b$KLkkhAF-%=@zQ? z{dqX2F^gZ_HCx>b?;}bT$BWhe)iB5yd!wX7ciNO(kxC!8cwgh{9|s50HM3l`K4OyF z`6&IEI@#X}4D(VLf@c9ok}=rKrKcEOHHLS=001ih!vJH)H?zc%qljMDt7*c?QoSXZ zkZ@(X>$O_CWc3F7CLl>CO>wA@F{aYr(a5!;wb)Qu=_o6x6I2rPh$JGI)}1k z11kr8?$*WGweZaj<@eGoBgGXtc=5U%(=&k?WO8CgGWw|tg6Z-{L zuPC|K%vIYMSM7EQU39!h&)aPDhp^I{`onJ|3cdg1!-aLr^3ni1rgI#+=0iSErngk6 z7Z(28(Dv1{9o~pPe`Lc`cGtE)YjUdJt|`4o&Fd?OOb#5HL2bzr>}PNOvzxE@jq|Q@ zSHA)NyTOrkQCGHZZfRHZPun9>sER7wGf;ws&MW3|8jrFi`N!@M`6JQZiuxtS@7l_} zt@$d4(AI7BR8@F4_f`pHhMQ>Op*u`%q+ofi^eeoeV37XE!*~3b%J?#OLdw>)_Tt=e zk^IpEVwIMPyYYRc6C42MdJiShiu&K|0N2cboeON@t1JOq<|$9Bei1}{H6 zLo2=X8*&v73R6*gD$1ZF@@+_CJ#WIQ<#e-t9=|uXdY|r+0)NioLWSzz zx#;B=(7O1I>eakp)Y|7UE?Q~C4?Q5#git>z0yMaJ6itzOBB>P0YV%f{{lV7kob1Ke zY%J~F-R8J{AT8Y<`UP*wW-@EE)&i774|shu6srNsI*=Ul3;@*paT~ziSA0sROpBSt zD0X!?X1;y3hkwOWcN3MFpGG=%nX&wK{z4I34K5A_Npg}9mcLt)leP;O)52Gu0{2_f zP6ro*o@hdog|A_gIz`v<(J{!cwc?1N{2WC3PxHO&i%hl3;@7+71#fcShPDPRal#B4>KVw$o+qB+J9{TfLL?{ sh8_kd`;9SyDUo?&Qeh(8|6i&9TF0UdFrzRvvQi9 Date: Wed, 5 Mar 2025 06:49:25 +0000 Subject: [PATCH 228/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9779 -> 9779 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 29bedb9a2..61eed1f17 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-04 07:36:39,022 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-04 07:36:39,022 submission_checker1.py:3270 INFO] --- -[2025-03-04 07:36:39,022 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-05 06:49:19,834 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-05 06:49:19,834 submission_checker1.py:3270 INFO] --- +[2025-03-05 06:49:19,834 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-04 07:36:39,022 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-05 06:49:19,834 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-04 07:36:39,022 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-05 06:49:19,834 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-04 07:36:39,022 submission_checker1.py:3286 INFO] --- -[2025-03-04 07:36:39,022 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-04 07:36:39,022 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-04 07:36:39,022 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-04 07:36:39,022 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-04 07:36:39,022 submission_checker1.py:3307 INFO] --- -[2025-03-04 07:36:39,022 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-04 07:36:39,965 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-05 06:49:19,834 submission_checker1.py:3286 INFO] --- +[2025-03-05 06:49:19,834 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-05 06:49:19,834 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-05 06:49:19,834 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-05 06:49:19,834 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-05 06:49:19,834 submission_checker1.py:3307 INFO] --- +[2025-03-05 06:49:19,834 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-05 06:49:20,815 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,8 +32,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0423 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 23.0348 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.22502 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2039 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bde83afb6..7e48d39d1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -855 +856 diff --git a/summary.csv b/summary.csv index 7125778ed..5d5b5243b 100644 --- a/summary.csv +++ b/summary.csv @@ -10,8 +10,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0423,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,23.0348,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.22502,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2039,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bc8e50d0b..c6e905fa1 100644 --- a/summary.json +++ b/summary.json @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0348, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -364,7 +364,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index e4cb98d24a373a96597e1780e35363556c14d9a4..360bbb17c955ca8d5380f6713547608992a4a010 100644 GIT binary patch delta 2674 zcmV-&3XS!%OtVa|bqED_qEtRvlX?hVfA%K*vF)L zmBeb~r&;%tVK`0n=3y_)j^*RgV%2ZxN8;e*S1I&b^KjnwU8o+Djn`T}l=~?9J%4l7 zYv@KUtugKAc{i?Nt*83_Y?NL{k6%AR-j>_|iL))a1rm2#at9>de_FCN+&Ug^ zomC*naJho?^V_S6n^zULuPW|}6k8)T@kmV;FUd%`!XqPnRdMsG;+8AuNiC{lYorz& z>5;|HDY=G^frOm*6CfcI_7O-3u%Cd$+?IR>62j9dkPx1}013H*&VYnWeG5p)`8($l zNoKbVBxH6kfP~C$2S~{5e_jF!ncXWOA+!4xNNjw;+4zF9Sp^oo;JCylXYhne>SVkh zfrO0r6OfSce&!OF%>C&~0&a+3Knvs=IV%Wnt&Xi0sB>D-2K^%qT+o;fkD)8hC6{<) zX0BfP-EV&R?dsEY`>b>Rex3^@Su!*T)8XvC8bssTc$EDeXNgYse`>1Yem9)-Rh*@3 zs++1L?IyGPahT+pPttv`szxo1m73>RwLF!oU9?Y@@v%y>3?`{9Q<=)T=58>ZR)XoQ z5=|yg@obW2xm0DT>ITuI9}e`N^`1)oaFpmD4r_g|T7NBLNLkLRvaZel+tIIYi++hk zhEX-8uxt}U=V0g@e-0hF)Q-8tCilb>E@99C355<3AqJadF8@3_Jb&?jQn{GPptGk_+J&XdKd|6QE%O{R6%Qfa1A9Z zX@f>Qx(2eS4v3=98!u7K1o*3s?!OH*Usl8%%brrxf9!IM6V-hXRX1A{|1zAVjBpP+ zZ9|`_|-Lfc@T;tG3gFwTf1R4e<&@d>uhC?Zepac~#C<`iR z3A&|l&Pv*#5syCdV^JLxMORCtq08k)_unFv8*61wr8KJIJFqNcjQcE_%Z`cV1LdkC z&0T9HlgtWz+#hJy(-3{0S5U~&xyvt9%ysDOc4P(f>b@C8g- z(guyN2RSUNgQ6IiU`LySS%1@DlIJ>qoyjtWw-=N3e{CjRPZ&FPJp|syKN5U*+;STB+D4xUg@j3nZ8yX)7;n$-zIX0-9&Kz zfaOslf4Ro7D3M&li3A!(B+xJ-xyGftAZraI5;VeyEU2KhKKMc;Eoq}((kHMisso}p zkwC*fRa$Qd$baH7Y$*fWi%DyPNmnsZjt z295Ad5oXZ=QG8Pb)XX+TN_OdRD1ZE}$QyO<+=xy4j-6%aj7@T%g$?5 z{+P$FZ`90lEjHbMJW^I&bHm)S&84@4OW@U^b{?Q{sGkRDIG1}kdXR>32|P25%OX<` zYJ_oFWa|;k*TYKMpb^d`v-W@}p2Knt$YD#i_xw9xzEP9UH?isR$?{m!kme3DrGd$m z+$dsGa}UtC)Z7C!j7+4VWFieE6KQB3D~rrMr~*o6Q3Wj^$;>^hqzxKj`ywo=gQ6&z zK*PzDCA)iGBlGWWjFd9Ilfn=se?N4<tzq)ChyJ$lN2DxrddsK_gzBLdncMD2k@B zNJGdc=r)hSE^1EmEgSkunvHJWO8x?Mt9dx}2kOF)2jL{qMyj6;)Tm}3z~jYl56oYu(RBXG z(zW|2O{4MZ-B9^G73c5j6S~%qqIC5ppOW=&I5htRP)h>@ldB{evl9|a4u7{G7@s8p z002+|000pH003lfV^DH$aC0wXZ*pZWcx`N*lFe?yFc5_AeTvBU#7_}gPbE45aFdE9CL6aP g7@s8p002+|000pH0000000000004l2QIpFhXac4H761SM delta 2674 zcmV-&3XS!%OtVa|bqEFC`Ni&7lX?hVf6iW{pLWrHaQMs9c%(h3I0>UkLl;#;*VH74 zdf{Zy&~I-(S6p37Qh(C(N6|zz^k4mn%p+zrCusc~x=ys^YFlvDH%(_ta$ZlJt}-Jkrxw6*sRcZn=V<)S?_)J+)v@ zk1T#p$u)coB;>rG00|kek3d3z{RAZDw&XLA5S~tfgz)qQNXQj*1|($YTR=k2-#M2^ zGP-RbA)|W%BxH0uKte|Me-cQ@=w1N{8QrfyV&e(Tw$RC5e@#`~?}n4UinHaK z>ZU45yUFZ+947h7C+R*|Ril>1O3l|;wR|a6yJ(**<71U%E10CVT*_3|nY+PsS_!7J zN;H`~#j{D8t)(hURX2zx{cxcFtoKyvhoeOQa9Hbu+4^f4UCP$1Dr?&Oza8!RwrH1F ztT3vk6qapb=o}24f5V|8m)bFx*yNsg!X*qkAfeC!355iAQdfSMGlY#4@_F2Z(p> z)|6N`rDN_`hkBSqkCR9kI_46aTnr~%!bk)XN+OU@61l`B6`gtoB!Y1IiKigEDkFcM;mNgs^qRDc?(G5KwYLGFTb0r> zq_O%ZFp0vzD9+0Xi{wH$1{#Mn=>%vPjYvarL>kH?(6}Ve=~{z`LR zNgGt+(K(PsbwCt--gt>J6X35ln*TP?e3=n*EPF~#f3wRmPE_|nRNZV+{L65b(!)LI zyjeDEfu~g!j^pi^9tD2jekAwf)XTPP!=R; z3A&|l&Pv*#5|2LeV^JLxMQ2N-q0{9?^WP$r8*^n&r8KJIJFqOHkNZrT%Z7>N1LdkC z&0TXPlg@fkqex6+xGbZO`#fM>Hf;5A zQhK(zYgl93`;a~y{0~2T%m|VlbtQUa^5->0e613I_U%;d# zZBPk&ki()nD2jmzcCYymLhUFSz z2+E{cE|atSY$}{Bp(H@2599f4nNO3#tIF>{vW(vCwR|-WRWokRl=AovgQcp&0!^N)Jn!Av*&;)2EKCT8nAMcaSt_r0!C$#cSlfb%{eP+ zgG%_O2(##bD84BIGP6yQl3h9+${)Wg@a#g#C2dd%fBee)Iw*>*Ux9|LUrV;^yhi1Z zdHniD%{zGVwU@|2) zirCcL12ir*_W%te6KN=!NJGg)8cL=tGWVbal+2<8Ey>ItJ*=b+Dq;H~EUJT|D49UR z$&@9#dtM{+?{18g(!Z0$5G8*-^pTG&GWVcL7?efk9?8r-tfUPp@#+*xX6`{zbQy~@ zbQvp4Hut>7WZKVh6yNo!vXs8m?i#)3@n)Jfl+wYGS-|A1p$v{i~So~gqo&R2dktcr(`q#6O($eRt zYUt`|nw_D%v%1!M>MwpNK%2&46o={aqW1i!3T7j}IgWbo=Z%_8!aru}{k!0ueef=@ zKm4r84_ZUZy1tTTy_?sPzd+4u9!~v%y71#cI7zgT>SrA_s@Vtdc=6i<^VexKoxiel z?LJD=XuNtiRDMsz`MVUluJxlRUA@VNWc?cs%|8K9O9KRxt0Wq;6cS4ge^{i@KPCYH z08jz|01*HH0Az1tP;zf@b1!3Wa%C=fZET&A%}&EG5QOhMMdW*8r-1@il1hMjq7o8C zMcgcV+gijwvJGk89w%+nR_cl4dS<_!UAdg?RFnLI^U~-F|AS+Yl_p*z+wY=CkO~QCu6RScs+D zV9WS7+W?*rGQ|o46D%;?15-N^(E%y=h_pYh832M~Xh4Au9_Lw(Pu>7d`Kt|sa^_Sr zbQZ2!kK*Xwc{v)}w$0jNu*Up3{#?Ai538n9_ow23>QZpZ9k5_rNnPUeA|9?`J}lyi z*Am227xgP|heh^%Ad(n>?&BVnRq_0?oTGX|CU+?*QgXW__q3SO;_<5=dC`9Ys>6_f z6OVv>?#oY5{{ge5A<6;;-ucDuSd(-lTLJZx&?Gzof0GR*K>_`fPbE45Z Date: Wed, 5 Mar 2025 07:00:40 +0000 Subject: [PATCH 229/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9779 -> 9775 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 61eed1f17..bfb0cc660 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,37 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-05 06:49:19,834 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-05 06:49:19,834 submission_checker1.py:3270 INFO] --- -[2025-03-05 06:49:19,834 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-05 07:00:34,979 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-05 07:00:34,979 submission_checker1.py:3270 INFO] --- +[2025-03-05 07:00:34,979 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-05 06:49:19,834 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-05 07:00:34,979 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-05 06:49:19,834 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-05 07:00:34,979 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-05 06:49:19,834 submission_checker1.py:3286 INFO] --- -[2025-03-05 06:49:19,834 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-05 06:49:19,834 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-05 06:49:19,834 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-05 06:49:19,834 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-05 06:49:19,834 submission_checker1.py:3307 INFO] --- -[2025-03-05 06:49:19,834 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-05 06:49:20,815 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-05 07:00:34,979 submission_checker1.py:3286 INFO] --- +[2025-03-05 07:00:34,979 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-05 07:00:34,979 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-05 07:00:34,979 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-05 07:00:34,980 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-05 07:00:34,980 submission_checker1.py:3307 INFO] --- +[2025-03-05 07:00:34,980 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-05 07:00:35,955 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.289104 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.299277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74482 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71999 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421564 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0423 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1017 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.22502 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7e48d39d1..bc7466b57 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -856 +857 diff --git a/summary.csv b/summary.csv index 5d5b5243b..b663d4e1c 100644 --- a/summary.csv +++ b/summary.csv @@ -1,15 +1,15 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.289104,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.299277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.296426,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74482,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71999,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421564,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0423,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1017,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.22502,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c6e905fa1..e2e185054 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.289104, + "Result":0.299277, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74482, + "Result":1.71999, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0423, - "Accuracy":"acc: 75.000", + "Result":10.1017, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 360bbb17c955ca8d5380f6713547608992a4a010..71eaa4728ba78627555d982c1e0e499d3c890fab 100644 GIT binary patch delta 3081 zcmZ8jc{CJk7oQn4qr%uVV+_V1TPkE{?6QqDYqBpHWXnDV*(Js{60*N!-$PWk8OBzU zC6Y>rNV0`b-#PCe-*?Y>?z!hV_uTtD_jk7@{W|@6D$V)r$I%b((Evu{6dVw!e!WOA&e%d^0(HwHfz@S5v2?*)nycXQe`XQK}u1Bs}Df(fTU7^mTgM z{1_NBnSeO+R_X zM+RO+9Vart20e5hdRTrbg{CrVHL4D6g=A#Ub2F8gDVrW}rSze89$(K@JS>;yc~^U? zOTLc3k4w07pn8qvJPR3=8%OcdUA@1%rN%gu#5NbTy+v1X?CLLNAXYTbb^iJy~NoSs_ zT%+aD@lcbv=>)J2sRv$P+IAxcFR?i}rTBi1$5p=gg5x-IH?GfCJJ=_oMpz*@6OsBO zN;pshsc~S_7tZM@N|E4kxwYKMdlRt~BaBbZT~AO~r0~GLv=!Xe+oGgFA(eZyged1U zW=L+g5QJ90)|3|HI(-SW-O9HDq3t@pKnuEwPgG@Wq9^ABx3)xJwJF3X4%y z4fJ7#RBj?bz^QI+USz`bYoYuKIG{ykk`IAxp$7pEcu>$sKmG~@{7}iqqcTuuK6&kL zcVKqki|oA%aPM|RWhJK&0hpRvzLj%JQ615|y(yhX>_Ljkr+b^uGTrSRvvqT{5=XJ- zA{m}Dj2_-!NPnn!7bf1io3B1;TKC104dcXUmmFVKyA=3Mv2FF6*5PcTnN8Y|*=yIH zt*<&$-C~F9!(mIkfI z05#?*I||u$G&ZFB+g~Hj;xqy}{VZ|VLwRV!aItN%Ce#LByuMt#zFti8W=khN;G;;I zYSA|+gbe{A6sz-M1C9~J>ff*d3g2=fgysK!&%HP%AO3`<@rCBI=${L_r}4!{DfuU= z0Zhl@6ElX>dyFdkZx7nn!tDZ!FtTT%s~ljC~xBO zQnq3u5@BXusv^?Z!?_s}(><=R@7N9=7F|NCN`;;vI`=_7%Siec3f>)=y0!Q8DFV~Jf`quM^inCh^(#hg z?bEO!0r%KVIS%u2Yh<`XMs_m>RZyxwS;?5>C?_@zQO62W{Yd#A-1L^b=+VGy5nN-p zhQmp0x6LEL87?W_b-$+O_3f)4ZgnBNSsPw^^S9%Be+sAD;+HKmb!wM1Kny4rcdm5X zX=`e?Z zX`#nuGb#Klj8##_R-ZpGzqR%sy*XRN{NnN}F?rr3pQNa~nQ|Sn1cLEP+9G>M$>tMy zgU4b+)A#wSLB^|_MJ(<9m4v@9{0M{@e&ON4^OQQ6fshO~qRC#0mCRZ$>FL_oLaLmo zM~v3~@~yAa7-KO!9W1*i+iNBym4e*a-%ZU6sqHU+tWBx9>z!-U+VLyMm%dB>?I?{D zT}g-y<5%B3Vp8E{Q6&rKjKCoe22E$tAr5Y_3Wc zt13d1I^wg=lEBM8Rx=RGhGun_B{e$VAG_;(@*y?dWT1^HbMk%M^2qYExlS2Uv4Lio?Sl zCaNd1*EQfgJ9dw4ard9b%!UnOH^sozC>s!5u4kaslKP_~&0G;Hv8!pmQ@#&A9-8A!|ac1qpXVbdfP@|F?;pPNR%P5)cdx9Q%Q5a z-LV2}e(Gb=v!M36moyBu+#$(MXd?yNH`Ddybq-~98@>izmaJFa^(9MhJD!P&oS&qd z;S;qrTp=8|bQiPiwBva8{_+gEg0DqV$S2Ovxt`CiS>til#;J);9(}vsJ$2ZMI8+7m z@{!$PD#`QasJXyz>&C^xSgxYa=o(qE9}Sl(>!Qn@Pmy!CEg29E27fx1x|6nWlrX~f z)|(TH#>Iu#9K`l9!-R4B;CN^eSNDJk2eq@fBswxPTX8x|ESKTB2!UlNssX2 z_ek&^oWieO>#%uu^}%M2>>8Jv!t-9ertf;cCGws>(#|Y=-tg1v_jG`w62{F}g!Q6( zWJEHzf*EMO(_i|xPd=nZa|ohw|y zL=Tar!-9`oDL8?XIJrCG!Ov5f&c*KvmWj2RaZr%4^kTKl0o_X3 zgf!)PG|Turc_0JS}OzYS;kPYwOS6_GzV4M1AHFh3MD!ZUJ*=(-RDJUTT*f-`fB{xz#V6 z57?3&^9$cT=Swk?xk$+}C6C+WHW&0ybnp2|1=-~U_TNc8@c+{i9)YwlC{xZ)qVW`Y zdHH^C^gSJe2X)E8?b!$QDMAZ6P@{;qLJNk;+()VcR{=@`N}`4RJHwuYyi264%a=_| zYKLghu^e9xC4ZKmg3K!TP6Z2qgY#{LMF%uZWqh$9y8kZTL5{J)cq9smHV0RUWQ zas8K~2(H8=0ZrgGaa@2O{=b#>uMPkZMF=A@pfSKhq7hmi=tWFI3&Q^&seiSj2nNJq Sv@-B1@w_1FoGI$>{eJ*0O{FdX delta 3009 zcmV;y3qJI(OtVa|%>xQD9XWY&1ONbf5|e%i8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Iz&dGqKal-;bR@2ex;aYF;Xx7f777_AOraj{9F<(!_Wo1*a?< zm`>9+4U&Z{qS<0#exIH;eAD0}$f6)&85x+bgqvT7|MZ_(@q;f35r#C7@qxJ%BJWui z50@kjcz=`SB$H2KR-}QDx5dKZc|n4xBBhDt+O}t<0nNS@>W!PBKC?Z+qHY~Yd>0|xKSsAFQj<@1Ew)4h5Y&vb*b35wwye4GDv1+eXgvWsh zhW&y)8%5=T=Ye`rPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!h(urId6t6wbDfXrhl>r zkXJb=;u+0iQpiq*%mm?L#>;t1xoQgd?5WeVEm06swI7!1GOX+IXvHrGm%BN)7Ry-> zs5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iA-qHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR0pPd!wbT z}xaaluLhs;<;T{+RLuS5)%IqpH)o}q5*07%k3vwM43z~5wA+b!PA>o#UgtZ!6F-j0oizW`860|XQR01N;K0000#0C%EP zK3TJQ2=WDg^=&3yPZ&FPJp|syKN5U*+;STB+D4xUg@j3nZ8yX)7;n$-zIX0-9&KzfaOttBDuz~D3M&li3A!(B+xJ-xyGft zAZraI5;VeyEU2KhKKMc;Eoq}((kHMisso}pkwC*fRa$Qd$baH7Y$*fWi%DyPNm)1vdQK>1b;JAycNglpP6i~?fIvSYkZ2@E;Zvja2^HSGWmr*aS5 z4$?3xfySX!7CCoN1&qof?~b6>nsZjt295Ad5oXZ=QG8Pb)XX+TN_OdRD1ZE}$QyO< z+=xy4j-6%aj32|P25%OX<`YJ_oFWa|;k*TYKMpb^d`v-W@}p2Knt$YD!= zw)gxyV7^h4&o{B@^2zd8(~#y4GNpmZl-wv{Q*#f{xYXPOG>lB7p=2TrB@=0A9xIE? zJ*WaoW>Ez#$;>^hqzxKj`ywo=gQ6&zK*PzDCA)iGBlGWWjFd9IeV~-aHk5`m+#Q>h zQtW_|8%1nt?g1K?ntOnTL%D~!2Wc38l%PL!xs*lb9@GefvdG*cnYo9Rv_T_YokGdX zJt&H%u}DMHSXr{U=QSqNevYH~u1}SvjBTInRc@MkvIM|0U25v7ZG6vttm1(>QKM0! z1yMF#i^z59;>-Wn;^1!aJ^2@T)?PH}g}JxiA1%JspN5lxmi)2!y#PD^y#OPBuPo?a z&qhj1pQoy!tEXvphVsGcTJNd9_@w}C8i!FFrq7Gk^Pei1jr`^~>b;+LYBmY~n5p;g zf_L`8yTJbNvnD@i4J{k`N}7#s-b(%gb*p(e^#|(0j|br-(MGDD4b-S+AHd_qZx760 zr_pr&%F?y_C{3gB>fKQJJr(B>@9MhNkD_$-CZCe^Z#Xpn1herGm;nV_uw5G2%2y>gLrnHcHFc)xV>`T@9|t zYKQw8Jka3Ek+(Qj^0dP>eGO~+8dmz+4%ftsHSuCiyjT-2*2IgIcp+XxyAkc$ijim0 z7FkAR`PhG0QEVz%;Asv@^U+DBi=nHnu7B0G8r7IMmPTCf_WKHTYB+KlF}~lW-56$ImZ=m*_se4nBnOV{`h6mC(T^l7 zWF~Tm4~XjrF7a)*AE>k1c0BgF#^Y0Y;-F)qehgG@30)lguO{8(XmN^-Kf+074P~02BZK00000 z000000DysjlL;k30sfOqB{~6clZGWG1h*g Date: Wed, 5 Mar 2025 07:01:57 +0000 Subject: [PATCH 230/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9775 -> 9776 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index bfb0cc660..78324483b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-05 07:00:34,979 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-05 07:00:34,979 submission_checker1.py:3270 INFO] --- -[2025-03-05 07:00:34,979 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-05 07:01:53,045 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-05 07:01:53,045 submission_checker1.py:3270 INFO] --- +[2025-03-05 07:01:53,045 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-05 07:00:34,979 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-05 07:01:53,045 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-05 07:00:34,979 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-05 07:01:53,045 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-05 07:00:34,979 submission_checker1.py:3286 INFO] --- -[2025-03-05 07:00:34,979 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-05 07:00:34,979 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-05 07:00:34,979 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-05 07:00:34,980 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-05 07:00:34,980 submission_checker1.py:3307 INFO] --- -[2025-03-05 07:00:34,980 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-05 07:00:35,955 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-05 07:01:53,045 submission_checker1.py:3286 INFO] --- +[2025-03-05 07:01:53,045 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-05 07:01:53,045 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-05 07:01:53,045 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-05 07:01:53,045 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-05 07:01:53,045 submission_checker1.py:3307 INFO] --- +[2025-03-05 07:01:53,045 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-05 07:01:53,998 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1017 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.82784 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.22502 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bc7466b57..45a52f567 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -857 +858 diff --git a/summary.csv b/summary.csv index b663d4e1c..c48c1856c 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1017,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.82784,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.22502,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e2e185054..76049a525 100644 --- a/summary.json +++ b/summary.json @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1017, - "Accuracy":"acc: 75.800", + "Result":9.82784, + "Accuracy":"acc: 74.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 71eaa4728ba78627555d982c1e0e499d3c890fab..aa7ae995ee46b37f7ddefc3ba6cf0b88ea23d905 100644 GIT binary patch delta 1054 zcmV+(1mXLyOt4I_%>x8Ny|ZYO(F0%tK6tY;1a1O<-JDyiWD$g_t!T+EqA}G9%c@nr z3d+J{@#MC>w%uCq{)>3=szV@PXFl_Hs1BinY%g}uFMX&wI55>6bYZ6iIX5~7EF`yxRoE8! zpR2lRSrwvc$q#8|;>Gtq7GU?k2=Ft~`vU_~ktz$uIgR$~psFJMQIY+f-_=tnsi6 zR^&QhtF+)oMq-($-Ii+=YtA<&>T5yE`bswIODY7-H+KmMBEr<2{JM=P6`LDXlDA`{ z@lQ}o0|b*{BpS1X2n7oT@KV|#PP2{^p#p!$l+TIMpn(^lsHOsu7PP~#f=YB0(7F`= zbind8l=3m+G)d~_)igFr%h}bxqrqJbuE=VK`x-pZ;L4G=I9Bqs!!>;kYx){i`q~cH z#EUiYVokhQ6ED`pio z)wUYdm^hY3T<`Y#3Uz8YavCwd-=*J}six#yMPjcd`~69X0ZLxdio^0An}PZ&XiE0~ zvF(M`&=K3jpBvt*2;Zi>7Z?ve?QbRf?LBNUiTnI9z9ujnn&h3CRELMpl;ykG-jT=8 zFN2rpKE4h3wcEO zsT(VwWq(b_(-U+Cr!v8S=GV8#u|p-my*pQ9f6_bJ>ixi8w9`v_$y5J9yZ)ot6STXL z?x|6|k^E$Kgq+ZHLuK%J9)yWW++c6U_Ds+7{JlY#m~LRpN%?GYxjdg-O2ki7#--8u zbhccKuI9~j?8%ut2tvPa_rrb=cyjOQbo^u{a)=Lz>jy6JZMPq&v)Xn%mJ(OT<5PL! zpktzb3{-9jT_vlJ-kLr8-Ym|=@ISNq7hwnldfqf8vtJ-90T($fi}hjTP5%X`PDB1p zJOhroZ$Clv2eY9egaQOYy|ZYODI~=O@KV|#PLuN_R{{HzOC>r1ZIgy2CIoukG$oU; YB^CirlgA}K0iu%zCL;z^BLDyZ0KWnbcmMzZ delta 1045 zcmV+w1nT>+Os`C^%>xASn}Uv$(F0%t9DB1f1a1Om4$ZM(Jp`Y%FS#OccXVn##+X~xZip%o(^xBJh?N%nI@hF1T55ZA1U%V|Ey z?M~mf->rIEqU++7Ne)9^@#bl@tFK%nRl2oxOxfuchoP;~trqC@CVbUWymeJDCO zFx4G@XrhCh8=V8PzD2Bp^4|6Thobw8oc=&`5B)9C*-qE@4nO$R`MBzQU3H*AIMN{< zu+UBr2vi*cfvQ6wP<03de3{Su9jZgs13R%OHxuch45lb2GyBbBRZL@#l38exVZ`tr&S0ZINYEmXH016~(5K1)k=hG#{O0x){3J>RNwot5J=KV`;?o zZojWkr-mb^5##$^`i+@tO3qaz_FA&vpM)5ohz6-kC{tc=$|NzMJhGdHnn`c!}=g>)=Be zKQ^bISP30$B58<57-_nuD2Pi7*A_<*>6;1b_<`++*EZO3CNadkXCl_w55ChEsP<(AM@ zvij()*|YD>;#>^>1GD=VVF&~lyrR^zU?3|27lan&`Y`gQ{{mE}A^#?x0mt08pP=~z zv!Wq{0tE4!f{v3aB*g_+*o=uylk_B40s50nB{~6YlZPcH1Q)!b)RVC#76DC@$R$1j Pp_2zDBL-6=00000D>nbI diff --git a/summary_results.json b/summary_results.json index 51638c9cb..020dfa757 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 74.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1017, + "Performance_Result": 9.82784, "Performance_Units": "Samples/s" }, { From d123153fcf5ef454acebdf335054958099b4327e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Mar 2025 13:25:26 +0000 Subject: [PATCH 231/719] Updated results summary [skip ci] --- README.md | 38 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 12 ++++++------ summary.json | 16 ++++++++-------- summary.xlsx | Bin 9776 -> 9771 bytes summary_results.json | 16 ++++++++-------- 6 files changed, 42 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 78324483b..ed171dfdb 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,29 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-05 07:01:53,045 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-05 07:01:53,045 submission_checker1.py:3270 INFO] --- -[2025-03-05 07:01:53,045 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-05 13:25:21,584 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-05 13:25:21,584 submission_checker1.py:3270 INFO] --- +[2025-03-05 13:25:21,584 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-05 07:01:53,045 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-05 13:25:21,584 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-05 07:01:53,045 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-05 13:25:21,584 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-05 07:01:53,045 submission_checker1.py:3286 INFO] --- -[2025-03-05 07:01:53,045 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-05 07:01:53,045 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-05 07:01:53,045 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-05 07:01:53,045 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-05 07:01:53,045 submission_checker1.py:3307 INFO] --- -[2025-03-05 07:01:53,045 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-05 07:01:53,998 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-05 13:25:21,584 submission_checker1.py:3286 INFO] --- +[2025-03-05 13:25:21,584 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-05 13:25:21,584 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-05 13:25:21,584 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-05 13:25:21,584 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-05 13:25:21,584 submission_checker1.py:3307 INFO] --- +[2025-03-05 13:25:21,584 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-05 13:25:22,789 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.299277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 18.7241 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.296426 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.1529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71999 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421564 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -32,11 +32,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.82784 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.440243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.22502 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2039 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418465 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2039 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416896 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 45a52f567..b75c874ec 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -858 +859 diff --git a/summary.csv b/summary.csv index c48c1856c..b3ea65b69 100644 --- a/summary.csv +++ b/summary.csv @@ -1,7 +1,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.299277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,18.7241,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.296426,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.1529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71999,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421564,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -10,11 +10,11 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.82784,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.440243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.22502,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2039,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418465,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2039,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416896,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 76049a525..0adbfed69 100644 --- a/summary.json +++ b/summary.json @@ -39,7 +39,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":18.7241, + "Result":22.1529, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -69,7 +69,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.296426, + "Result":0.353561, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -364,12 +364,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.440243, + "Result":0.436959, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -454,7 +454,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -484,12 +484,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418465, + "Result":0.416896, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index aa7ae995ee46b37f7ddefc3ba6cf0b88ea23d905..37af73cbd13fcf14630d9d5a57d9f809f02eca68 100644 GIT binary patch delta 2811 zcmVTqH9^XczS5F?4K_IK~(| zj_ppb7+EZ!DIh8dfBgGOfH9o&BGWXjJZ&xLDUbY!-jry{kh`mT9W$HzCVtps-ZurME|Av)8RuD|DKGLO0_J(G->Fge>9yP zRI5oaQWHNhqM4dzM+Q+m@zd;WJgg?OSo!@$&}3YdrfF9vemK>a84luoazukc7^sg? zFrTPtx=a$Qv7ctmPe$P^(VK_;Fgup_M~l^YU##&zufBgA4y^0>bsBkz+vthIHkRTc-i@(|=%*Imc+Qffs z=rvt?2>aPdxRje?@I@&Wt-L$2ZpO2@pTE$eC zwJuut(NF#6VH`bZ@uG%g=I3Wd97vfBOOTg-&5QH4RaXU9&jni6k9_p*f19<#>V2Mg zrT9ieu4%yyEx6q%sAlb1H%j70N!A8)yA4-dQMX7lfyCaHTmy--Ex7>_cUy7`B;H!G z)!iEIZjB`nq`O=}+WGBO#r3O-n^zULMT)JS)^ShkEMAhHa)n2F`l{mkRmBZg(34t} zW2>hY?CFuk&ndZvkAZ}ofAAR(js6-aD+!P)qNvsnU*UT|DulQY=j zk{apnM<5~n{RAYWzn{6pC1Zc`ECC;gUqB7y8aXWpZ$BSEDjxL0 z=|IKVa!vJ8m8897em4n|eC3n$?k&Ttrm<3sHC8oWN>$H0tyO%il57Q&R98!x$~to| zn9VA|Y+i|`)5mx|e@(NsRAr^=1<`a64)ve)zDoUYoai48s{=4we=Va+*_u^lOsgTOhg^M3;j|uAgHrvB|a2 z;t~cTkWdhTgn|ep6hyZGL=a9vEC_G&a0a4V03r`(Ai5Mp5JRTZ3oh}<2j!*v-vP0V zuIvHgt-CcPf7a^Ksqa{adYD9ylSmjk<`SD+3@t8UBmxN~5lASBT;h_7PP_sVK{&nR zDF|hA8b%}1e^4BehVlqBF3EGU)*vEL0*Yi& zg7*608cJ5u0hM@k4rEbX5JjIiUZBhb_{)vvzYR2BX2cxJzEZR7a*PwzyBCI=O^SaR z&Qf}~2c6fehAr{54BK?yVrO0MBhiXtdM z0tRJ4e}cB4TMB2aqysAP=p#QC)kRTswnQ2_U2Zi0Eke05SLReo(-7Z*Wf^_kXVP3X zOspQ_8p4u09@h)J<#Z(YE{xMj!Wif55tG*!p(Ts#*4~MJ(PNJraD+(Q+u5 zU|+IE!Q>haCeSc2frf#}H5|-Z5ttwW1G6AOdwuW)Oj^^(M zpDbp}e3}%VA-@C3GJ3by^0mH!q%a)edApXcJM1Qc`v)wK63I1=MTz7ZP9)GUB7uex z2{aBR(kdbnBw$1qBxtV>z7R=EI-nBve*~6AbwLy-5@^_6rS*n@{3jm6meRq!n6x%; zDQnrvyta$U9gZ9U{5I5*12hix}+-Jm^j$$~`iHdj+;v< zc0kFEA~rSm0F6t{JwU^u+{4_1Gz?169%@FHMdlt<34^l8+#{K}hm~|dC0>m}$;>?{ ziY{Z3hAv}e$>yHdm`n#be~#k2?kY>^+divTxn=4x4X^G}??83qd+rkz4^>Nz$B7n1 z*>Ei**QLuZ|6hrN+vWGBtJMaq7vQ1*QjRcF zqrLZrVC>i)By~w}s37jNmDGpQwtv`A_Dx&Bd79p137&H5DLw*YGZWDX$=QrFKeihI zoMGrdg7OY$Nro?84_3OxhEch4O5q0si`KI^yAMvx#=h^9z8I|`e}+FdFK^?ju~5fT zF+fel8DSQv*S4wfbrBBNQRg?|#5)1}aR~at+i8)h4|o&;kSXq2`6!;Aw7D*G=H_!ZOeH!2X_JQ~CL0Hc^;{$Y002+|000pH0000000000004l2N|VSXJ^`JR N2PPv1Pa^;T005~|GGYJ# delta 2816 zcmV+b3;*=1Ot4I_bO;6N9gWmglX(awf7^4~Mij>1`&Vc*c}QjoXct{QhK}tdjxolL zW4qHQMivWb3W!R=m;b&JH^Vs}GELLU)8@f})pzzL{j>+|2Zz5rjYrypijy#!G;~ol zbWKfys25HK4gL1!bH&xQB=sjfe-uqrLw{C@{!8AA;UM0pj;P-c1NAWq zW@9x;7nQ_n*|b@CNDz&Z`Cn}uW^*ZZZR|fa z^t!G+hP`x{SF8=gUQgu{t!u$7Nu%+vi$h{r!6H=H%UEHEb+lI%reRAfKOfH{wT!7P zTAlasqo4ZC!#H}>;&}_n)X&e1IFK@%mLM}!n8;?fyCaH+yIHQEx82}cUy7?B;H!G zHQYKLZk<&i$Z)xW^z+-Riknvzx34PhiWFNTHStJI7B9(2xxynOeN}Pus^XR_=t(WA zV{4=q9O;q8&ndZvkAZ}ofAc>fW+LEd^547N~Ptf6xZ~BMn^8m=2Gj zE6ydCcw}a-Ui#f{e);X{({=l-bN+sw3nf`HGzin-?7kXA7B<&`%`*E1$nNQMvu&PEajg^|`ShYNrs$H~CmhrJlvJ57vEmN7wy5?>$ zomPVBtP)KoPw{M$e`dKr3o?vn8H&2Vx%X!CHBoX-n~5(#2Rlb2lLkvX|?N+fbHf9(1}{%NL?eRx?^RT*SF zPM^~#4u;(aS+%Q1C720m?m=Q1L)nAGTW5pBx?AH zYkhDHB`axzMm)L(vZxM-qR$&IQOyMStBvlz4K!a?#2m|>Qq$~mj1$#;5LGu@6#p`u zrHpV7I&YR8Ti|I`h2wZTmPdgfxE~4rfZehvlw9M`M}t7ap#&NRCD1S^xrRe2il77) zFenQue`pE1rEtzl+Mp4SKJsHx9TY`ZOQfO8w_;? z(vmi4gkwpgIv|QYOa&TY7?ex-Kj5;AG4At#e|6ch)yGNc+2*co4S_dDj|AUyv>Xa1 z*q5wRFu8_<2{a5$pkZKg4F|Jc1SY6}fmu*NYklwqOj^06u5nDtB-d~zfrc>&G>l2G=TEn| zf7cqwBxr>0ITlsWS|5C2l9seVBc7y8(x?uKVmU0=2t!aN&2pKX-DgYTYzZX+GJP1& zSIc~v6kb(+2a;usZm;y!+)Q7qj_|x4ZWFP?ZX&pU!15@OT;o`jNUq^T0u3V)Xc&=P zv1#A2v+Ue4c2Vi>pc1%qsDTG)9O~c!8cyXN9v-A&R01CjTPurfJg5;yWs#9b zP@l~?D`|s9_~TdR*FjM%e+3$rf4`P&*?Eo1AM^P2jhcC`#ipA_%BpK_m|M2F^mcFw zygJm*12hix^8gL!at}uj(l9Q8XNGZEWa>eUFfNO1J%agqSV7e`MOvaTMS6sj`%@?UTLAO;b;n0C=WLO+B@Z@41gvJWwZU zG)lA}%BE`(xh`FN`TtrR+%3K*|02)YizdA=_xAgv#kcy?a5B)6KNi0iVCTOVVC0nr z{p;CCY3cJ+HFWhf&CXChSY7Kq^%uVspiSd2io^7I(R%(<1+$UgWgJJn_w!E8CgC45 z_5NM(&OUe-*dKn@X*6EF8!Er2;{07**ZNVEuHNKRvi=Q+=AX0e5o-r zNPEg{jdmeeim~TAP|Ro1p`*AlPO%V6w|~W!@o%;TJR{^DD+o-m!0-W>+L4G(NWn*> z{c-I85FA4b3Uu%|&vJb625`zxHVn#@Q_0XZy06oS%E1$*l%Oz@o zni-juDJfD?uE+x=IW3FzVdPE!1*lF#{!Kgsj=66?LGuT*ogvBs1?nA*)K!ygBwGRT zlgK1I0eh1LB|!oElSd^w0d13lB_*e0000d6*24p diff --git a/summary_results.json b/summary_results.json index 020dfa757..7ea65f743 100644 --- a/summary_results.json +++ b/summary_results.json @@ -320,7 +320,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 18.7241, + "Performance_Result": 22.1529, "Performance_Units": "Samples/s" }, { @@ -356,7 +356,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.296426, + "Performance_Result": 0.353561, "Performance_Units": "Samples/s" }, { @@ -437,7 +437,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418465, + "Performance_Result": 0.416896, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.440243, + "Performance_Result": 0.436959, "Performance_Units": "Samples/s" }, { From 43815db499c799b286c552a39c10739c6e4c4f1b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 5 Mar 2025 13:26:31 +0000 Subject: [PATCH 232/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9771 -> 9772 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ed171dfdb..8130ee814 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-05 13:25:21,584 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-05 13:25:21,584 submission_checker1.py:3270 INFO] --- -[2025-03-05 13:25:21,584 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-05 13:26:26,640 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-05 13:26:26,640 submission_checker1.py:3270 INFO] --- +[2025-03-05 13:26:26,641 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-05 13:25:21,584 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-05 13:26:26,641 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-05 13:25:21,584 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-05 13:26:26,641 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-05 13:25:21,584 submission_checker1.py:3286 INFO] --- -[2025-03-05 13:25:21,584 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-05 13:25:21,584 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-05 13:25:21,584 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-05 13:25:21,584 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-05 13:25:21,584 submission_checker1.py:3307 INFO] --- -[2025-03-05 13:25:21,584 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-05 13:25:22,789 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-05 13:26:26,641 submission_checker1.py:3286 INFO] --- +[2025-03-05 13:26:26,641 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-05 13:26:26,641 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-05 13:26:26,641 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-05 13:26:26,641 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-05 13:26:26,641 submission_checker1.py:3307 INFO] --- +[2025-03-05 13:26:26,641 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-05 13:26:27,656 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index b75c874ec..a47c0f27f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -859 +860 diff --git a/summary.xlsx b/summary.xlsx index 37af73cbd13fcf14630d9d5a57d9f809f02eca68..6b985d194c7b96bb8beb71548d03835478a65d9f 100644 GIT binary patch delta 326 zcmV-M0lEIGOsq_>SRe(b7T+l)lU*Pye@I${?Lb?yYM{`RmhOZ!aYU^Ll3FK!U)hPB z1ib2Ubnl$GGpJl0WE=g0bwaBOWl4%6P)u{7nhL#d*YN|2oTrM@Rx7B`5gc08cO^4~ zX$!Ab8?au0ivmbF!c2ws-W!6kV|$R)CBdPBxYJfrA4=O`L)kZN0Ox7C#1cH^f7Dal z17kB0(HY6vgfu_49RQqRXhDMV4rfV*FJ2E;y4i+7xpGS3j|OI~qd2(_PE5wG>yoY* ztRa7fKQ}LL!>X}R{iztBy5x*73)E{{m-xB}hij?xn{eWt0RDIk`pnyTky9V=CJUwsMsLs=T5vN6*F1A@g81f~_@4xzyH~kl&JP-LdaRi)l-+qGn53`*i3j_tH Y7T+l)ldUCG0iKfrCL;z=BLDyZ0GUOV2mk;8 delta 325 zcmV-L0lNOIOsh<=SRe%li1l0~lU*Pye{d-Ygh&Qrx{rxny_0Ebr1UBwJx z+QO^V2CNt0q5x8kFjJ$w_l98X*d8QxNpPqj?zENEhtjs#Q1(q*z?~}e5 zts#GgKQ}LLAwKwX~@5cXTX&E_7gOJvz{Rf1O*3( X^;{&At|e3fos$D5BL+_+00000aT}G# From b423cf0cc924858bda005bcdf8d79df2524de44e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 6 Mar 2025 01:50:44 +0000 Subject: [PATCH 233/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 9772 -> 9771 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 8130ee814..a675808fe 100644 --- a/README.md +++ b/README.md @@ -1,40 +1,40 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-05 13:26:26,640 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-05 13:26:26,640 submission_checker1.py:3270 INFO] --- -[2025-03-05 13:26:26,641 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-06 01:50:39,122 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-06 01:50:39,122 submission_checker1.py:3270 INFO] --- +[2025-03-06 01:50:39,122 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-05 13:26:26,641 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-06 01:50:39,122 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-05 13:26:26,641 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-06 01:50:39,122 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-05 13:26:26,641 submission_checker1.py:3286 INFO] --- -[2025-03-05 13:26:26,641 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-05 13:26:26,641 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-05 13:26:26,641 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-05 13:26:26,641 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-05 13:26:26,641 submission_checker1.py:3307 INFO] --- -[2025-03-05 13:26:26,641 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-05 13:26:27,656 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-06 01:50:39,122 submission_checker1.py:3286 INFO] --- +[2025-03-06 01:50:39,122 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-06 01:50:39,122 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-06 01:50:39,122 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-06 01:50:39,122 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-06 01:50:39,122 submission_checker1.py:3307 INFO] --- +[2025-03-06 01:50:39,123 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-06 01:50:40,147 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.299277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.1529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71999 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421564 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.82784 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99715 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.22502 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2039 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416896 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a47c0f27f..9761bfa44 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -860 +861 diff --git a/summary.csv b/summary.csv index b3ea65b69..cec1e12d2 100644 --- a/summary.csv +++ b/summary.csv @@ -1,18 +1,18 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.299277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.1529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71999,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421564,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.82784,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99715,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.22502,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2039,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416896,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0adbfed69..2106ac600 100644 --- a/summary.json +++ b/summary.json @@ -34,12 +34,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.1529, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -64,7 +64,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9481, + "Result":21.0953, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.82784, - "Accuracy":"acc: 74.200", + "Result":9.99715, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -399,7 +399,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.22502, + "Result":8.44052, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 6b985d194c7b96bb8beb71548d03835478a65d9f..503e2ee2c18fe33b482cb5946638e8887c0bdde4 100644 GIT binary patch delta 3250 zcmV;j3{CT_Osh<=%>xR&qSzB~1ONbf5|e%i8I$k>2Y=gc;yM(4KVM-*eQ2eDT!0V| zFq+bP>5Vd{I!`Dh4n&j0nb_&&@5fG{Q?_@RYF;Xx7f777_AOrajt5_#(}XF$KYs8Gb2O-2{~OaM{^ptpe{oL}8VDN70vJSP zwqWdsdU!5}EJZan?nxEZZ9zgB`DG%e?CFfg%YT)SMmmZTu!L8?B#mWh9fPF)^U!EX z4{<0~$}m~QVMw#eZBTN-(uewyy*5x&ZFkMJE&G+-Yuatgbvo+xyd-qZF>9|?gh#&c zM}vYrk)razb6>qEyYE=CN`V@^Q^P7Yb3HKK3=o-zD(2&;H83BQ;WgqOQf6-kOzG{_ zrGJCFbTkjvwuh;XXoDDZL}wc@(-H0~Vk^oP9%W18+aO9E(FIZd5i#3^*ov}^N7>f+ z4v11mcp%C@B4#?Ghez2#qx{y=6P1v-cQ*fXQ`&K)~!eMIi1r zI!nM5e}Fu2m;Kw+uvbFqqF~RkEfIK>&yMBP2jRmSjJ*iE}8*L%`}m}sekMN z;m%BMPm#euS zsJ5aZyNKFUBgo4}{vucrtmcnS(`{Of_3l3pSQf=ghP$2!}eP1s2*d0)eVSAW(G(1gZ{!fSvi&+o3vy4zfMpLBH^z>VM$CM0e1J zof70+>m0C<+#ptATjYJN>dHl#iLxO-q?L(h&wXEj?Y|M=XQcN50nWTP0_-(=y{@+D znQy+OQ(7$Pm?jA)0h0x*nXjYudBfE1&+2dS-*ELYpM|kn+5DtFIS_HSB>ZQ6MB(!g2V_WO<{PNYuJTeH7a(Dc2D!haSAMaKGfom3$waNDT&s%ue51M+ zEU&I)v%X+Luyk{`qJBsVbtk`WBgVw$MwR6C*l6?%P)h>@6aWAK2nqlI06zc&vI>$? zvv~-A1b^yv(-PkLA0|C&5Ef|YJ$u$mrGzc^tN}yp-0u6(bYuGHg z)I=ImP?uB8mCT~r!51cJNe5KoNy;RR>Y^yNhUFSz3d&@?Tqfst*;F`R zK}mmrOz$U)*)pFdg=fg`K(dV9?X`TZZ!TXAN9=yov%_v8xPQR%D3M&_Sd>Vv;Y0!r zBNAvBkwD{6BCR4KK>|i(L4x-B;0uwoqys8pPheS87esL)fri~xT5kx*f8sH0DIMI4 zNo#{iS<6=DwOvf^aO4Q!x1p9CpmC@t2WWpdlY6*wkcKe{G>pk2V-8Bdm@Km92;R2JO*=rt zsocZ1gEWjvpkY)NId@P3MrDzAM^J0c87t|4O8BM-v*>^*z9|ASvrUnbT{;}fAHNbS z@k2D49q@$wV4TrYth|pahi6q6BTp%ss561Ai)E`ywo= zi=rr*K*PzDCA)iGBlGWWjFi&9eV~-)Hk76`-5ocVQtW_|8%1nt?g1K?ntOnTL%D~! z2Wc3Tpgq)#E{n`Ps1gQck-0}Qa}O)&fJ(d?g_4mIJjMYPyR)o z)fY|sVeaks$IEZ^r{Q#{C4Ve`FTl=!FTl)`1%s>kSZV3gOf_`%ILppZ-dSDiKMt0^ z6rj!GFp9(UX<2*zQw8&}-<(AK_lri&r{N!S_5NM(&OUe-*dKn@c(lnYpzZ)sPuj2e& zP1godls>=7hh+U54$VISv+WU>0R>msjEPONiW8v%f5~VorV4lgifSqlX+b*-E2uL(leo_x<7)!Lp-JAENp*PmOj*90 z?Hzgi{4#in?&Is=Ll{3cr=M5}9c&_Lh;YZ%f9W#0Th=j!?w2^~7(zR`z)Q32OZG0B zx{ybdpSrQ~S@zd-JUu~Ya4HiFXnuW*96MA3+`Dr%_9wlgt=7E+Z8_7>*N5~0HH&h0n=Rug5#0~alY|r#O&)*w_iRlKmoRrTdm&^0Xr9?bs zPFxzDPiM=;=xW|f$DW+YgCO+#c0cR~fhYH#PRCDXB8T{ZxPIUg-*)?fI;(BRV<~ZU zJU*2t4mu|4$3W$l&{eYf=&jka@6F;|4F3bO_ZMLZ1YF?vRFhmFAAiki!!Qtp?|llR zyDix!p%u0RZON&DLQ`6L6Vk>JwSJJ)I(hplPU0lsQ;(~e{dRUywb-dH`32{t(KX65 zf)dc&2&vl|y|0()JxY9FTCmP&sL>ugS~PbRx0D+Pug+L-L4r>rNPEg{jkY0Jim~Tg zP|Ro1p`*AlPO%V6*MDNm_%~|-7lh2Qg1`g|3?G209f|0K6nsS5AJ+{4!7+58KnIWW zEXNmb0H^$9!=PL_l?;0elh(61x_4fV#=h^fz8tJEe~v#_FK@%Dsnmy4aX?KaIOPsl zFs`ZabrBEOF&|d(#2X3XaUb=Gx6>lWJ`hO^K#y_H%18P1yd_?urXa;EC1pzHYm(C$ zq2$?2><{GLI405 z00000000000001hfn}3}BxnU!*o=uylkg;00rrzfB{~6UlY=ED8(iS`R3!lb08jz| k01*HH00000000000Dyr>lf)%H0iBZrCL;z%>xQTy|ZX>1ONbf5|e!h8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`CC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%iWw?t7H*` zs;y|rE}}8j3d^chz6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*%U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0t*!``xOyC4ag;F1o%hIxr27bVvuT@(2Wq4uL??ArL4!1Oi3Z z-yu4L4n?s!PsDDPeWe<-@o$mtJ6_t4)Go$Yje@9=}I z&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)g5$U zrvy1SItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50nYt*0z7Q{Zr|?$ z`C0G2rE|hoWJ0ox8zGeiYnX4N&3VJj8O)n+@n3NDF<(TfTG@iEIXMt%u`>9N=7_=@ zOdefZ84KfCl@UX{mhzLHynmGP9jRPRBYKIO(+HzvDP^g6ZM>_~ktz$uIgR$~psFJM zQIY+f-_=tnsi6R^&QhtF+)oMq-($-Ii+=YtA?7 zYeCEUN;d0DDg@0pcL@n1!qlDox{WCnn;TVhA8b%}1P#lql@(468$#b&SARK?yVrO0MBh ziXxMa4L5%t;u^w|J08~yyybKx_%4oRQ!Kg0krYd=;aCC~7c6N>2UNnbq)}ZE#U7>tjWA8hrTiaoSwhaCeSc2frf#}H5|-Z5ttwW1G9f1L3@4h1x#Af0hO=^IV`G+q8ON9 zN1KCLd(&W&=Q@9#$uhdP7n8LuCZ*>|XU8tSz}xsog71!7E@hHyoH}KaYdDiY!4P+8jLiZet613L`UznsN9Z-oUDU&p+i=x;XmTQD5D3kSanVjEcQ{j9C zB>{gjy`L;*%Y2#?o*};j$ufGk*YdT#fut}T;d#53uRH7}g8K(7j}pl>jzx*&8crn8 zFd~755eYO7CDJM)5+q1n4v69_SFQmo*D@_S{|c0E)vzOoV@KHT&gZXNmL2njN?^#Lo*kfZsA&gi zIF)d(XcE<{LHnd=r~4pKKjlvps9a z-K8`!nUWhtY-;WS8kd@TfQFHYG?YxFp=2TrB~uoedr$&OW>JE+Wab`L(gA;!uze90 z)kRU1OrYUp%97nZuaWt8H%3b7-#$=Ea~n!in(mI9ODT3h$&Dg5HTM9GOU*q%!=c>6 z+=Da>O3)r^MwdnA9#jc~vdG*cnYo9RbU-CujY7%HJt&GUW08g~V`a(ap4XU62RV-7 zyY4DW>DxZ5SGi^CF%7ToQty92b>n;P6BQ3tOO3~g7DU-_EhE>Z%P;?5iG$na_vByX zS$)y8ALibEf4uxwe;Q7QTJp#8_X6zv_X5m3SunVokCm1_%~V5IkF)Fy<(<{F{^MZz zO99#}4x>0spO&@fKUFXv`^`zzf4^wdd>Z~SSMT2i@9cwjf&JlUO@2+#8d}!%XKB{E zMJ@RY)U4*=%pasF?)=|^QZouQ^Zx1Y9r_pTj%F?yFC{3fu^ShDq`zp@g z)pTtTMd|aKd`Q;6;n4gOv+WU>0R`|<+96J}iW8v%f5(*1iPE5f7oe!70+AN9!?1!% zbQI9K6#sO<@->w5G2%2y>gLrnHcHFc)xV>`T@9|tYKQw8Jka3Ek+(Qj^0dP>eGO~+ z8dmz+4%ftsHSuCiyjT-2*2IgIcp+XxyAkc$ijim07FkAR`PhG0QEVz%;Asv@^U+DB zi=nHnf3DTG8r7IMmPTCf_WKHTYB+KlF}~lW-56$ImZ=m*_se4nBnOV{`h6mC(T^l7WF~Tm4~XjrF7a)*AE>k1c0866 zSI6U1dE%gBqJ9iiZV6o_tB>BAJ^S7)&c*OQv-THZ2n44V-zk$@ARm9rYQr!PMfd#* zqrELZNLqyLKwGkEpwN_-?u0aPM6CysS|@*B*@>M5yy|jv@0__as9YUn8~uWHLaPd8 zNs1y+Omm@{3cYXF@dJvSr;5{7E2z*B99q?PB{PI+3$IoiuwH1!a8Ohm%G(WZ-0GwfHL4xuQ zXGw-HUJq8f*@i*6a!TQk24=0JIJpl_OvbM3lCBu6A%BKHH!p9)s Date: Thu, 6 Mar 2025 01:51:38 +0000 Subject: [PATCH 234/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 9771 -> 9771 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index a675808fe..cca2c8553 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-06 01:50:39,122 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-06 01:50:39,122 submission_checker1.py:3270 INFO] --- -[2025-03-06 01:50:39,122 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-06 01:51:33,591 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-06 01:51:33,591 submission_checker1.py:3270 INFO] --- +[2025-03-06 01:51:33,591 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-06 01:50:39,122 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-06 01:51:33,591 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-06 01:50:39,122 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-06 01:51:33,591 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-06 01:50:39,122 submission_checker1.py:3286 INFO] --- -[2025-03-06 01:50:39,122 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-06 01:50:39,122 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-06 01:50:39,122 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-06 01:50:39,122 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-06 01:50:39,122 submission_checker1.py:3307 INFO] --- -[2025-03-06 01:50:39,123 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-06 01:50:40,147 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-06 01:51:33,591 submission_checker1.py:3286 INFO] --- +[2025-03-06 01:51:33,591 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-06 01:51:33,591 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-06 01:51:33,591 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-06 01:51:33,591 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-06 01:51:33,591 submission_checker1.py:3307 INFO] --- +[2025-03-06 01:51:33,591 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-06 01:51:34,533 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 9761bfa44..b7f96dfa6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -861 +862 diff --git a/summary.xlsx b/summary.xlsx index 503e2ee2c18fe33b482cb5946638e8887c0bdde4..b4c7cedb7a86751009c0badcc69694e4546376d4 100644 GIT binary patch delta 69 zcmZ4Ov)X4vxB^S;{W#~z(F!`uc}{|pyA-S#nI|7rI3N+}cl!7G!zX6{;|`eH_Gfkl Xn@sWjx~6CRo98NWvp_VhQLP66-o_m| delta 69 zcmZ4Ov)X4vxB^SegYO}eqZM?R)fS3Q?ozN~w4HoZ;edqX!l#F?CqD80&$`0T`Jb;1 XbKlN Date: Thu, 6 Mar 2025 23:18:02 +0000 Subject: [PATCH 235/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9771 -> 9773 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index cca2c8553..f1a56bd78 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-06 01:51:33,591 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-06 01:51:33,591 submission_checker1.py:3270 INFO] --- -[2025-03-06 01:51:33,591 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-06 23:17:57,545 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-06 23:17:57,545 submission_checker1.py:3270 INFO] --- +[2025-03-06 23:17:57,545 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-06 01:51:33,591 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-06 23:17:57,546 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-06 01:51:33,591 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-06 23:17:57,546 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-06 01:51:33,591 submission_checker1.py:3286 INFO] --- -[2025-03-06 01:51:33,591 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-06 01:51:33,591 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-06 01:51:33,591 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-06 01:51:33,591 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-06 01:51:33,591 submission_checker1.py:3307 INFO] --- -[2025-03-06 01:51:33,591 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-06 01:51:34,533 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-06 23:17:57,546 submission_checker1.py:3286 INFO] --- +[2025-03-06 23:17:57,546 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-06 23:17:57,546 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-06 23:17:57,546 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-06 23:17:57,546 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-06 23:17:57,546 submission_checker1.py:3307 INFO] --- +[2025-03-06 23:17:57,546 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-06 23:17:58,572 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.299277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71999 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.6926 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421564 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2039 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416896 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8103 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416896 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index b7f96dfa6..64a158471 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -862 +863 diff --git a/summary.csv b/summary.csv index cec1e12d2..bb418bebf 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.299277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71999,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.6926,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421564,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2039,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416896,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8103,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416896,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2106ac600..10e4b6f4e 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71999, + "Result":1.6926, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -454,12 +454,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2039, + "Result":22.8103, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -484,7 +484,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index b4c7cedb7a86751009c0badcc69694e4546376d4..d645f0ef21679d3111e6099d78b6a3cac85dfd18 100644 GIT binary patch delta 2735 zcmV;g3Q+Z{Os!0?a|j8SbjLAN2><{FlY9tTe`mj>hdt81aQMsPWUSq*I0>U^Ll=gr zYib%q{ct*L=r`A&E3U33sXy)e<7lcH`jblZUz$H1K1A{F$w;YG%PN>A4SkfRvx90i z2}Ww-Cq^_=)9lC~iYI=Wy^V*}WELyGzX+O)tI{;>>ckJH`m%I7+i=v+|H28Yhdt+9b^8QtH~oe{ASA zU3&=o=_s#Q8-@M8$|qXaf_aiglV6vIf5fVSWvH-MvBDJVXwMLJ)0S3#KAy&E6;oZd zy6E9YKlPi3arB_Yix!fZpPw0VAZ0c!L0(9DV5_d|nKA78mxZ;YsMY;(j_O|32NStlS4Uo9ol3O70f7X(% z;nwhQYpeo6hRYSCpWj|pT)(Qgc~x;+q}Up19gnom;w2d=S9oNkuPUxzRorj|J*h=? zY>m`{BR#VCIVIQdF_4h+-U1RbVIP5n0Q(6@>f4gfKtgyr0TRN~7a$>5&?%6Rsc!=b zIe%wdBFXG_fP~EMIgpUq?E(pzf87fpA+vi4BxH8K0*Q?;I2&JZHmks*7aW(^7tjK^MotUD+t0_I7pOB@&;k7;4V=@ME{~xr z&IOlvWM(d3`aNiV`R(%4Rp+#O_I{BIC0Q~w4Aas4&IqDObu!NWPO?NNe|t4k@t_w@ z2P)3eHPuU1lJ=7M-6Tx%%qQvHTZUOpW2F{3Ry9whs^^{7Dn3?8mcb;|RVq_i*W3$c zvq~_VSEA|kF`iG;ESIXRRJ|aY4#J`Sv))&!AC43K!(nv*R_m{23@OW5Ro1n|e>?j1 zZP_oe%rK0)6qa4b&^Z`7e}_XyF12GWvB_s*i%S@EKtiDd5(*uVkPGz0DS!@ylh5wc zg77vEcS-o!W;ppc=kO;!*9Zx}c z+asKu(SiPLoA^PLRL1_IhDX=_(K%@u!`lPED{lip z*QnP`OL*&lm_%V>6c^=$MRFk=1C2wv)B+ktBhpYDk%saJG%l&Vhcxyzv6nOn|@K=>FS4^JPWMvFs}~f6Fe%I8nWOVYu0%_?O`< zWrTasdA;h`5>Lwzj^piE9tD2jekAw|^MNxFML>jtWZgl@GLb=ID{&dybYv!36t~H3}xza4>;}feADWOs?TcY^?}PPyqw8pn~@L z;0u_vqyrjZ4{}&k7ez5J!HzZuv-YOJB+qsJI+JA#Z!acme_KpS&y&uMU3`JJ@s9-G z9k*P{B-c1~$|TotCV_@A2{epJu;)*=xYruUBxr>0ITlsWULSm6l9qHpBc7y8(x@(q zVmU0=2vbld>*X>zzsr`w`3gz`WO_eYtd{vQDLg}d2a;usZm;yUzL~xnwzq5VU+i$3 z2yP#+JPIV&e>fHel504SK*N9p8U`fSxO5X_t${#-Mi`I<6|~m}Ux1_~9n?$u0G35{ zK@ems(up?}D z=i}Ec$By|zCGg`=%MQ>u)UyLLoXS01J4nN*1R94@S!CQn6)-A`tUH1_YtC3n2Q zM3_SdMDaZlP&3;TDcPjMq5SciB5%~Zb1gRAJ9d_wJLayZukH>ifjNizcYwyB1|Fc{ zRPJHne?b~XCD1S`i(EXY0!C$#k4I3O%^54{fJXS!S7z5iQ7nH28kWD7?AUpY${+Le z^^JOYuEeI9C(mK)?yelx+rcHU>QFZi&^Xl212mk=Jq$fa!?*+*#$}PG2UWnhEOPY- zX6s=k9nc8pk~w=o6whI~2IR0MyLl!^Rd#>f2Wyh=<0EnouPcNy4HUjEPp6Ko5f)ihw0O@ z_57C#=3~D(iTdvsotjU>Kj!NFyWpLD@Gh`F{H)0jT0_f*{w&Q#w`e7Qfx6W^ocTj_ z?#IJ$nrLG+$OdW}*$sHS{ON(k>ol4zURkIq9}cS36oFB z`X?Nke*&}Z5o!(u!2>?~lUpDkf6HpaFc3xe{R*SKEk8&agzZ3EvTC5vl$Pv-G;u_& z2a;MRe_zFMoCLh;a&+&UxihF-9%LK+f^|Zx3S~)(B2Y|op_&SPY*z6CMb1;jX{!}f z=m-ui>xYsV!nB1qs|{E$z(oP19ATzHd+!aw*s(oG>XP74LELF8sSl-Xf3TtKn>K*+ zG@WA!o^t9bJ^^Dh648LSi#VNc@`7ZKWVYCzM&9*bfpQq~Z{h`T&VBz0 z>OZrcA<6;?mvqN5R0#k829t0kR{`&n$Rs=gd6NYtK>_%aM$?2><{GlY9tTf9LFn^sp=K3x~fvPR823ijy#!HgsW_ zx~8T<)DNe_hJJJXx#H?tlKRuWKaQrVp+Bia|E2lU;X@Svo{W@AwJgCjY3QRgogGxG zNib3qKQW@2nr24^Q9SX}>}@=(CbL-i{YB7ZT$QG2S0{cr)t4C#;(c;NgFzUmk5Mq6 zf2e7?OcJZHpJvTZM&T^cn}_`{JC^rHi`Af^ABlt3uTtpM=Ha65+fY3u8?UvzD|b=! zd;aFM-_XrmT4mbLi)LKLT3-$P`8d6b9=@n>I7+i&v+|H28Yhdt+9b@zQtH~oe{ASA zU3&=o=_pUEjlzCkpe`1wj87k~mtT4qo+A~Dmw564wkEgL(#Z;HI zE?W4}PyOa$96f09qK0JV=VwM7NSO^wke7bVi}SWsR|Qwk1zOgReDv;{wZrOto_MAB zMnkS?!3{0A-6*JL?O8WU;zmi<26MX&S6orINHc-N-j-YgiL))a0TOpxatkEhe_FED z-5Tz0jU^DIyIeuq`R!H3^{a}TR~5HKimjg3aZl?kUXq@2g-3e&s^a=p#SK@`lUkHx ztEU$1>5;|HDY=G^frOm*7Lbqu`v@ci*iS%G-(V}J520UwB8Kn>&?IV}ipKOcLZpw4JP2egk=a86^oJciCV z7hK|zk-2C+G9rp+tfhQso7gcw|g2of3&0j(UEO-_2CAo0ny($}r<` z_LN3(FzVe)!#2!HFc-4E2Z?2LWe*Z>oDC9duHm^GAB$Tcx)?;4e}hP_pJOhu$+ghp z5(XlWP!NHHf(RrOM7ID$5KciX2ygRn2BKR4A`fREx)ekZL#ERUF7e0*<)!=I0kMp( z>;dAfyEP@&>e8w2SciI;M30k57&_(>n_LVnE@31B2_+FoD2ZI+l8R2e0un(uz2hke zZ+nE3Gg{CA70`7ke~sz#7>|t11(1-BfJ?7j*d~4uC6%$i$nfaeKYC4CM)&pr@XFf& z&^79H(-PkLA0||^MNxFNL>f9>ZZ!WbLb)+l=2S}45Z{4i8GYPm(p)u6 zte!a=u4C`Glgtewe+d|t1qs^ggD+Upk`AbZV@adBAc{Rq1sY+RluP+P;IfQ9?(=|k z)v)#Lq*b$Qu>(xJIeH}co}=YZFu}fLje^NF9892LU;+&TlWX`QwpIitNWj1>NYGv% zd;yb|bU-ESK@N-Rq9_I?*wN--*4{Ljfrc>&_WbD<_gVv)1eMS|$D#!7^}!b=X-NlE;z`ORjq0K( zwua>zVG7D*y<8^eciB`pUqMNLOz$U)*)pFdg=fg`K(dV9?X`TZZ!TXAN9=yov%_v8 zxPQR%D3M&_e^``AuHi%i4I>h07?D8ZP$I1&B0&O1WI=-V`rr$Zw4?(nVNYOLR2M{X zB7uh8Ra$Qd$baH7Y$+Ywi%DyPNmpk2V-8Bdm@Km92QcEwu3Z`N}yp>7CCoN0!C$#cSlfb%^54{fJ*qL2(##b zD84BIGP6yQl3h9+${)Wg@j$$~`U{n?vc?9*@oUxJ)sDwX$WqutL#n!Jt!`81QTXtTf^2a=WeWPZc zE3xV3kTzWgW1YR9#=K&gr`gwqcbGe732Wc3WK*P8!GWDPYjLRZhk6^wY zR?-2La4wm(2So8TEZ2ZFY{~YXe+SGrYV!Fee>Pn{**dmnd)AJ-OKD;k2D49q@$wV4TrYth|pahi6q6BTp%ss5611e$rA}p$lq9~a_!^xB-yL(HUVILppZ-dSDiKMt0^6rj!GFp9(UX<2*z zQw8&}-<(AK_lri&r{N!S_5NM(&OUe-*dKn@tl9U|O)wXsFR1 zJX$n&6}OZd2d~apa6y7kB1n76ZH=}eScf+4v%VaxF@KIfS1)hFs;Sh6Q*l5|B{<~{STL@s@O2Rn*D)Vf@x&Vm;&C7KiMP`t z$375A3_y=@&&o&n^t>frqNX6lEG1=1=4+DESx(FQufxcj{tHl@hWwj&1{`zWeuCx? zv!5Z#0to}M3X)O@000M*awJy)?UTwRJOOu;2PHuP_LE5^Iss^tge4{fUEf|qlddHe W0ZEg^B|ZV2lLRIs22UdZ0000l)Do=# diff --git a/summary_results.json b/summary_results.json index 7e501fced..38b1bc620 100644 --- a/summary_results.json +++ b/summary_results.json @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71999, + "Performance_Result": 1.6926, "Performance_Units": "Samples/s" }, { @@ -437,7 +437,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2039, + "Performance_Result": 22.8103, "Performance_Units": "Samples/s" }, { @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 6b0556aed7f318789a1c6a7c8f008db21c6a0bb3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 6 Mar 2025 23:20:57 +0000 Subject: [PATCH 236/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 9773 -> 9769 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index f1a56bd78..f9960c11a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-06 23:17:57,545 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-06 23:17:57,545 submission_checker1.py:3270 INFO] --- -[2025-03-06 23:17:57,545 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-06 23:20:52,718 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-06 23:20:52,718 submission_checker1.py:3270 INFO] --- +[2025-03-06 23:20:52,718 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-06 23:17:57,546 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-06 23:20:52,718 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-06 23:17:57,546 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-06 23:20:52,718 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-06 23:17:57,546 submission_checker1.py:3286 INFO] --- -[2025-03-06 23:17:57,546 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-06 23:17:57,546 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-06 23:17:57,546 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-06 23:17:57,546 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-06 23:17:57,546 submission_checker1.py:3307 INFO] --- -[2025-03-06 23:17:57,546 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-06 23:17:58,572 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-06 23:20:52,718 submission_checker1.py:3286 INFO] --- +[2025-03-06 23:20:52,718 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-06 23:20:52,718 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-06 23:20:52,718 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-06 23:20:52,718 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-06 23:20:52,718 submission_checker1.py:3307 INFO] --- +[2025-03-06 23:20:52,718 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-06 23:20:53,680 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,12 +31,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99715 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.95046 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8103 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416896 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8103 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414229 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 64a158471..3750a1291 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -863 +864 diff --git a/summary.csv b/summary.csv index bb418bebf..c7cedcc5c 100644 --- a/summary.csv +++ b/summary.csv @@ -9,12 +9,12 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99715,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.95046,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8103,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416896,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8103,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.414229,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 10e4b6f4e..22b646a77 100644 --- a/summary.json +++ b/summary.json @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.99715, - "Accuracy":"acc: 75.800", + "Result":9.95046, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -454,7 +454,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -484,12 +484,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.416896, + "Result":0.414229, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d645f0ef21679d3111e6099d78b6a3cac85dfd18..0925291c909eb0cdea0918a8135bc902ce461690 100644 GIT binary patch delta 3418 zcmV-g4W;s}OsPz;%>xAHhNJ+q(F4^3e;un0b zK1cOw1j&n-6|s1!y{praGVw<#3-_y_$}Ij_(*3<)&+YF8?!g!09FQTAnQx#nQ@hGb zbzFdiH5%mpl3w}6GR`hq22TClvjlFj;p3Bl6M-HQ4lE!3U-x{VkU zn;TV<*JGp6FHlPZ1QY-O00;m806zc(Y;_$`lXwUve_M0fRuqNb`zth>JR~zk=p4|^ zW9ZlLDUbY!-jry{kh`mT9W$HzCVtps-ZurME|Av)8RuD|DKGLO0_J( zG->Fge>9yPRI5oaQWHNhqM4dzM+Q+m@zd;WJgg?OSo!@$&}3YdrfF9vemK>a84luo zazukc7^sg?FrTPtx=a$Qv7ctcPe$P^(VK_;FgupdM~l^JP##&zufBgA4y^0>bsBkz+vt_gLkRTc-i+{CA zn60JMwTb`O&}+K(5cbnio>&`&{l3ZZD~)%!g0O7V?`T+@ad+HkwkP|e1(9+bp`l57m-_86|XqHd940*SpXxdswvTXF*= z?zZF>NW8UVYq~W&-5N_E$aJ}ajPu*8itASuH?JygixgWkt>c;2S-d1Oc>fTX@H`3xk4rxPF{JbeKY zas{0N30e9!kdX6t#wC)hZU;!n>Yf7$S=}y>kk!2a60*9NKtfjcE0Eaug0t}jXR`zr zz2LaSCTFn4B{eeNk3d4^`w2+Md_QxEOVwAz`##EC#3`2k9c6z}j9{HfWbpJabmNAt* zK)iLgZi%(Jbm}|mP!E&naS{nb$6R8Qi=o9Oj6@)zBmxN~kxN|C(TP_;A_%A7cnZSX z9^vGSHgrG-v<{^)T^{3+mAL>C@)2<9l?&U%52B$&wFVK15>O)1QKKMK6%bR_sLj%8CUxyF$cORnKq0u93wXc(4U!?Bb_Sb_u$%Yp>$^}!b` zX-j_xbi%QuQC$$l9;O0~Fipy({2y>x#vJ!~z`7dP`s<`svuv>gOuRXIB>0}ARRF4QCQ)7?VK5m;`(N zbc=hfflPu<=$>Oyg7*603zM{^13K{}Ws*j9Q536Txki|RGFdN|$@yJ&E1a*OBtWM3 zlf~UKzfB6ykl%r18ME7~e64SmuZHdI+WQwf+$MtC2P}^Q$u*8ef#ezvB+xJ*frftp z2{aA`(kcQHBw#=mBxtV>z5q#EI-nEw0G35{K@kaYvPdtS!WrBMlX>A}W zrQr&*wu8wXh8zLxHq?;=G!C`o01any4^s}(FeZV9FMa~?-tU0Wu13KYM zGItJ$;)N^MfWoy*ht9tOgBl-n`WN8hON81YFKXvm%yq+-8?|! zP&*IMa4z>S^dJr65@;BgMV=m%fN@#m>JiM=!&*9^6V4@b_JAl}!*UI%VM})R{5xR2 zQIF3zv1#(j>e!m?Sv%@jX=0Ok5FUR7l+2<8ZOObntfd1wVcQ}ss*9p1nLxwIlqH*c zUL*7G?u(Q%zkQ&T<~Ee3G~FGKVWrpsCHINg)Y}6zF7@^R4To|MZx7NiD8YEB7hM*4 zdr&6~$|7%%WZoXu(gB@#H3}v3_Mj+Q#v%5;-PJ&RVa)r~)MpQw1KT53E_v>?isYZV{aKpLZqZBr0u8HqIP-_< z+>eLhG||RtkWJJyvM=EA@}~zDuhVF@cxCC@U6iKLh@$lQO}-@S zpKxgY3A67Jm;nV_uw5 zG2%2y>gLrnHcHFc)xV>`T@9|tYKQw8Jka3Ek+(Qj^0dP>eGO~+8dmz+4%ftsHSuCi zyjT-2*2IgIcp+XxyAkc$ijim07FkAR`PhG0QEVz%;Asv@^U+DBi=nHnu7B0G8r7IM zmPTCf_WKHTYB+KlF}~lW-56$ImZ=m*_se z4nBnOV{`h6mC(T^l7WF~Tm4~XjrF7a)*AE>k1c0BgF#^Y0Y;-F)qehgG@ z30)RD(w=f#qiqP5V(j@A6!Teh=qRp@Q!K>N zZGW(3{D*A-7lh2Qg1`g|4DW%dorvg&6nsM3U)PNQ!7+58KnIWWEXQYW0H^%ThEch2 zDj5a~H?1dea__vHjD6o{eK}fV{v3a;Uf;)6Q>pi-;((e;aLOI9U|dt-%OW1GV?M0n ziPsXu(-8F=Z^uOreISw;fF9zWl+W_{r6paWrXa;EC1pzHZBf#K&{_GtA9>Y(2CCzb zyNM^jA@}tsX#N4SpCN<-1m=dM0Fx&qLInhDbsbTYZX{a)?32nQA_ZHp@AXWR@gx-i wc9RDsK>_rWNhLY~W|M>^CItD4q12PEB^Ciklf@-I0h^NqCL;z-BLDyZ09c@yivR!s delta 3469 zcmV;84RZ3SOs!0?%>x9yqSzC&(F4^3f4#1@>6vf7rBhlg>6j)7CjpZMtC_E(^?Ae8 z?$7FP@!xRuF`tF8TG{-hJ~!g2V_WO<{PNYQ?Bw- z9Ty;BjRv{Dq*s2ij5AIW8p%Yhrd+Fv`h26h7A&u>WV60tLa=mmx1xSX3w0;IZX?FT z=0=s|_1I|i3s6e~1QY-O00;m806zeibjLANlXwUve_eChMihqc`zth>TqH9^XczS5 zF?4K_IK~(|j_ppb7+EZ!DIh8dfBgGOfH9o&BGWXjJZ&x?y@5xB1RLd%u zCJlX*f2Om8YBdQ)YT_qGG*i>;$RLU*eww|Fht*^jE5E-8nvAQ`H0|od52yOFhJ$#Y zI-wc##&zufBgA4y^0>bsBkz+vuU&PkRTc-i@(|= z%;r+++Qffs=rvt?2>auApqb<>ts zemFK&O0qtf+kLp=in>L*2_*KmM=2aBHjrL59l}q@UkjRb0QSxOr7^Tcp?;X&sNW&f+B*DOY%8q^~Nj zUsc?21wE-nb!?5)f+Ib$_&Ful@G+2(fAih~5;9>QfrJ422}tVOlFvXwcsc~?^J%XUcs~IN8SiHmw04mE?@dRXny(a^3zr4w0rh`kqaeRGBga+(frN`qDgf!&i+oa zL??SSQ}Lh|P6sN^(lymfRg(6S`Q0Q;^2{gc-CKrPO=G1NIaW1KrK;zh)+#<$NtVGR z)m17}S=ZbPX0u8#n^&Ug^f8`Kf72|Ns;pGKAes)sq5iYpSE(P46aB+sbpTfDuVoA= z%UM;{wZ(rs`t@zuFR{!pjJgz-UB}Ql7&?bTM=rHvF0sjHVv9={bU;F(0}=`ykdO=X z#3_Idgp<$i(}M6e4|hrU*=9KTIOy>3Iys-`4kZ%AkR~s<#3OTZ>6A$1C2-X9gZyr$ zlHI&4tEvn$9%oN!6bGZ;y)wAz`#!!wkZkL}6kS7v+RSav>Z8jYGQB0vbjm(oh_chVlqBE~)2atwBVh3Mi6A6|~m} z*HE&O4rs)qYaom2f++gD@dDLMfWO@6{@XzFWkt-f>?<|PF2^`gy?bG}*`oND;Vfl@ zd(e5k>ev!b%Mgy^?N}ZKe&Bv2_ycy!qEK>;Lmz()0u6@}Xc&}0!=U6Ewo0xw5K2%1 zgR-E4wxC-IXRM?H8u92OKNi(RQFOIL8oFF=bpI_vxv^H}R7%qj-+^TrW87!aTy;#W zo;e$?WAC_s6nM+&Nbp@8%cfXzjUy?RT*I*h8ipm%Ff6%-V=0TU1QjqW3o2-@558bY zOFDm`5soE|>VhctFcoNoX;Lob|A5Of#<X&Wx06=Qvc(QC@#g4};CqgiL%{_5 zk~In@*Kja_hJgt*3{0-!No=hMOi%#>v!H_Z`rr$gw4?(XVGnXxR2M}tFu{&C2ebC3 z!6eUh{yLLo3~w(cYgiefn|*9cQkChO%gIls%6!ubkH0%UqW zS*(`%GATSmeg~3ejBc;=wZ56Y8n(ArS*n*{3o8mmNLM-khC_C zl+tj8S=+(n4nvLrb{p!*0UC!|a)5?2xrZqSX&95h_QIGf^5vjL7?VZL9KozPtfYf_ z$(UsB91z75SFQnxYncw6e+A07>eqh}#IPf5cjx2REys@eLM8CyP|FU`IMlNPG@Qyk zTsug^s012^Qdwl&K@~76i>y0>I&02YNe48-_e7XO2So8b5l}PR6DirG!=e1~n<8)2 zymKu!-8*)coIB>Or?2h~DuFqN`gefFp#~nH;Z*Kn;XxWkCD1S`i(EXY0!Dvjk&j1E zo6Q+3>3~M~(^qEKK~XG!1sayWmh9MhjmjVM^!1H;d9K8!nJ3R->+Y@`*4x1)uMHRFq^Y*Zk4rqjJi?FCJilSr!4JT8U zZ0>oD%)h%YQp))Dfl`{=P@2+ocidb`u>(r(6S1kc2WVXC?ExAN#$-CkaTMQmS6Rx~fA+~< z<))`c3d8g)n!Q#xzUDqr@ldtYc${cKlug$%a$UOo?*ElIxLtlt{zaa(7ft(NZteHS z%P;k(;dH1ae=L74z|Mazz|1QP23Pa3($c4yYUt{5mYtz|u)5ZN94vn*K%2#36o={4 zvi1Cz3g%QSIY8u%Mc)a}SfyL`Ink`;gx^@?(X*79$H&T9I#reCMt_`9neSVWq z$@(W8ntuXNO9KRxp(Gl!3KDt&1y|UNiA}SK6QKfs$!II43U~pEYAO(EK|2g9s6
pzM; zLAx92o*LB~$xmiS$O%n1R0f~tL714t4fbYi&-6Uc-y4L9=?1o(l+PxY%k#;lL_B3u zTpFEEXUoOtYTiu8o}9^pAoTloKkNsAC-?~lUX1if6Hpa zFc3xe{R*SKEk8&agzZ3EvTC5vl$Pv-G;u_&2a;MRe_zFMoCLh;a&+&UxihF-9%LK+ zf^|Zx3S~)(B2Y|op_&SPY*z6CMb1;jX{!}f=m-ui>xYsV!nB1qs|{E$z(oP19ATzH zd+!aw*s(oG>XP74LELF8sSl-Xf3TtKn>K*+G@WA!o^t9bJ^^Dh648LS zi#VNc@`7ZKWVYCzM&9*bfpQq~Z{h`T&VBz0>OZraA%p@1yrS3>lOrTT1($TkF;tUk zBwGRRlg1<>8&}wjiA@9m074P~02BZK00000000000DysclLI9|0r- Date: Thu, 6 Mar 2025 23:23:52 +0000 Subject: [PATCH 237/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9769 -> 9771 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f9960c11a..b979117ed 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-06 23:20:52,718 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-06 23:20:52,718 submission_checker1.py:3270 INFO] --- -[2025-03-06 23:20:52,718 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-06 23:23:48,088 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-06 23:23:48,088 submission_checker1.py:3270 INFO] --- +[2025-03-06 23:23:48,088 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-06 23:20:52,718 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-06 23:23:48,088 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-06 23:20:52,718 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-06 23:23:48,088 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-06 23:20:52,718 submission_checker1.py:3286 INFO] --- -[2025-03-06 23:20:52,718 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-06 23:20:52,718 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-06 23:20:52,718 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-06 23:20:52,718 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-06 23:20:52,718 submission_checker1.py:3307 INFO] --- -[2025-03-06 23:20:52,718 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-06 23:20:53,680 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-06 23:23:48,088 submission_checker1.py:3286 INFO] --- +[2025-03-06 23:23:48,088 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-06 23:23:48,088 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-06 23:23:48,088 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-06 23:23:48,088 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-06 23:23:48,088 submission_checker1.py:3307 INFO] --- +[2025-03-06 23:23:48,088 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-06 23:23:49,054 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.95046 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1535 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3750a1291..0914cc157 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -864 +865 diff --git a/summary.csv b/summary.csv index c7cedcc5c..3a8a05b74 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.95046,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1535,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 22b646a77..571e19ed6 100644 --- a/summary.json +++ b/summary.json @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.95046, - "Accuracy":"acc: 75.000", + "Result":10.1535, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0925291c909eb0cdea0918a8135bc902ce461690..9eece5cd5d4dfec6785c0a8e32bb961f6a1eb2a3 100644 GIT binary patch delta 2079 zcmV+)2;ldrOsh<=%>xQFwrU7*1ONbf5|e!h8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`CC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%iWw?t7H*` zs;y|rE}}8j3d^chz6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*%U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0t*!``xOyC4ag;F1o%hIxr27bVvuT@(2Wq4uL??ArL4!1Oi3Z z-yu4L4n?s!PsDDPeWe<-@o$mtJ6_t4)Go$Yje@9=}I z&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)g5$U zrvy1SItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50nYt*0(9)Q)9b)Z z&wBSQofEbq6Ov`z2&pVs!+aZU&KqXVVBUO-|AMQJ`65cy$`)kJ$$?0VmBD{BM-<** z^61*iSQyW$j2Pmzl%MqErGJ#~Nabo8(M#N%Mi?baDNDs`<6WJOR9P_2X|!JlRTb%v zitO)&d){C#^bWol?tw8fWab;F%&zuQ9~U5DjfZ8hBG&<1r3E)K63ayGwp^=NbG}hu z3tHA!vRPkJA!xq2OGpqArtaj|ZA_`y+^CYg9UF~*0#Hi>1QY-O0Fz%N7PEs00Sg87 zJo5levxyU-0)NMJY{gUoFF;XE1tKkIhhYVk=qR9dDgNnz ztA9s>yBb`P)eiSHc%Z?RBX4o6HL5XjERDF{?e`Vx)Ntf9 zVtl_#zcEuy$+?QeUQ71-lMn-xyrdO}@lOQD;vl19X2n2lHEBTXIAS{1wLol`jZON&DLS0&V6Vk>JwSJJ)I)3}g zPU0kxQ$ZJLX1|?XRIc{2i+;g6p;d*_grEo%(_E;wLhsEwUZTi(syOYmf(i}b(5k*G znJJjI@M^UI>jk(dfRrnksnE`QQ()}a4kUF+aHt?|wUyL|t8KBN?3=cLGeQ%mHQvtUv#oKpC~z)kB(oZWjTW@F#? zNuN*FkUzzro0qp~)L5u-RSZyFa#k=4)N5Op`0^4K*HPy;VZ~bk{Ba2Sjkn_A6{>IwRRbO!Am4nk*PE!nt35jV-V0&p>%>@^9h^aL9fA3F<$yn<0b( z1vIv52yv4nBwGPylYk^e0qv8AD_OsPz;%>xSNhNJ*+1ONbf5|e%i8I$k>2Y=gc;yM(4KVM-*eQ2eDT!0V| zFq+bP>5Vd{I!`Dh4n&j0nb_&&@5fG{Q?_@RYF;Xx7f777_AOrajt5_#(}XF$KYs8Gb2O-2{~OaM{^ptpe{oL}8VDN70vJSP zwqWdsdU!5}EJZan?nxEZZ9zgB`DG%e?CFfg%YT)SMmmZTu!L8?B#mWh9fPF)^U!EX z4{<0~$}m~QVMw#eZBTN-(uewyy*5x&ZFkMJE&G+-Yuatgbvo+xyd-qZF>9|?gh#&c zM}vYrk)razb6>qEyYE=CN`V@^Q^P7Yb3HKK3=o-zD(2&;H83BQ;WgqOQf6-kOzG{_ zrGJCFbTkjvwuh;XXoDDZL}wc@(-H0~Vk^oP9%W18+aO9E(FIZd5i#3^*ov}^N7>f+ z4v11mcp%C@B4#?Ghez2#qx{y=6P1v-cQ*fXQ`&K)~!eMIi1r zI!nM5e}Fu2m;Kw+uvbFqqF~RkEfIK>&yMBP2jRmSjJ*iE}8*L%`}m}sekMN z;m%BMPm#euS zsJ5aZyNKFUBgo4}{vucrtmcnS(`{Of_3l3pSQf=ghP$2!}eP1s2*d0)eVSAW(G(1gZ{!fSvi&+o3vy4zfMpLBH^z>VM$CM0e1J zof70+>m0C<+#ptATjYJN>dHl#iLxO-q?L(h&wXEj?Y|M=XQcN50nWTP0_-(=9jguH zXTJHCPHC~EW11wK1WXpJX1bK1cOw1j&n-6|s1!y{praGVw<#3-_y_$}Ij_ z(*3<)&+YF8?!g!09FQTAnQx#nyUI&-T!4f%8sz?xUirl`&NxYEBonooa;+-r^Ns3S zu)MmG&H91~!P3p$iuxff)SdjgjTjS~8mW24b8P)h>@6aWAK2$NzY6|;m00Sg6N zuw5G2%2y>gLrnHcHFc z)xV>`T@9|tYKQw8Jka3Ek+(Qj^0dP>eGO~+8dmz+4%ftsHSuCiyjT-2*2IgIcp+Xx zyAkc$ijim07FkAR`PhG0QEVz%;Asv@^U+DBi=nHnu7B0G8r7IMmPTCf_WKHTYB+Kl zF}~lW-56$ImZ=m*_se4nBnOV{`h6mC(T^ zl7WF~Tm4~XjrF7a)*AE>k1c0BgF#^Y0Y;-F)qehgG@30){OThf%DSn8s!;5321JF)SDW8w9E7nB|b1MSZ6fUXaJ8E%|pd4<;KCAGZtKs z;FAc_o^o5GZ3vcP?D-ZH^I3H0D6WlDEX2}nux0#*Z2%X9%&~&N1b+(*?}4eEi0FtE zd_vk^*Np(dF?66n2aoeC$7gQ9Dl7| z-^W!`srRSifSO8h${ny^TvOr8A|9?|KCI%2*Am3j5cL~x$3+f(Ad(n>9^#&q&+_@D zU81HS#VjRdO6F}*(ino!S^2#mdDVXgs^gHmi6_7z_w^@e{sFU{A%p@2=7yvIaFZq^ zTLER0f+R%&?32hOA_ZHp@AXWR?<5rgc9R7qK>_rWM Date: Fri, 7 Mar 2025 04:41:05 +0000 Subject: [PATCH 238/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 9771 -> 9771 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b979117ed..fc24b8b4e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-06 23:23:48,088 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-06 23:23:48,088 submission_checker1.py:3270 INFO] --- -[2025-03-06 23:23:48,088 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-07 04:40:59,917 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-07 04:40:59,917 submission_checker1.py:3270 INFO] --- +[2025-03-07 04:40:59,917 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-06 23:23:48,088 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-07 04:40:59,917 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-06 23:23:48,088 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-07 04:40:59,917 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-06 23:23:48,088 submission_checker1.py:3286 INFO] --- -[2025-03-06 23:23:48,088 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-06 23:23:48,088 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-06 23:23:48,088 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-06 23:23:48,088 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-06 23:23:48,088 submission_checker1.py:3307 INFO] --- -[2025-03-06 23:23:48,088 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-06 23:23:49,054 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-07 04:40:59,917 submission_checker1.py:3286 INFO] --- +[2025-03-07 04:40:59,917 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-07 04:40:59,917 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-07 04:40:59,917 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-07 04:40:59,917 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-07 04:40:59,917 submission_checker1.py:3307 INFO] --- +[2025-03-07 04:40:59,917 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-07 04:41:00,954 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,8 +24,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.299277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.6926 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421564 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71361 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421099 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 0914cc157..47eff0704 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -865 +866 diff --git a/summary.csv b/summary.csv index 3a8a05b74..ce70553ef 100644 --- a/summary.csv +++ b/summary.csv @@ -2,8 +2,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.299277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.6926,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421564,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71361,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421099,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 571e19ed6..b83ca7f5b 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.6926, + "Result":1.71361, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421564, + "Result":0.421099, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9eece5cd5d4dfec6785c0a8e32bb961f6a1eb2a3..658a8d20a71fabda270045ad60f0a2e0f3041bfb 100644 GIT binary patch delta 2760 zcmV;(3ODtuOsh<=bO;H8<&UFL2><{GlYIywe_eChMihqc`zth>TqH9^XczS5G49w- z;uvG>IJP^zVq~#^rhup<{PFKA0mg97E19Ng<#96B?p2(G z(X^!t!_+l34WfQH9k%qF>(4b;*OJtq_Wf}*RW1EVCHgN1KkYw6@$bn09ti2cjgqVl=5`ydxT0y1W&(-5Ex85~XIpXuB<{B4 z7D&9cWUIRk+}#FCAV_z)g0%D7tBUJa6*sRcZp#!~J#FHiHd(wRJ>?3I^z>E5^{a{- zuAnEiEXP()E!fjTi=R_+4Icpse>v~RKtcxWBajeaKLJT|Tk;u52u~+KLU{TDB;*P@ z1rjp!Z6G1%?~F?%8Ql($kkLH{5;D47AR(iB0VHH}FM))N?pGkO@dan&3(jTF*~XA^rW#B`z8JlV=IIA$|chkZa_$B)t86?0JGZf1?E*&^}VZ zIgRP^7&_xzaEV7o=JKW8gM%->U4FXioOaLNFV;dywip_Q>1cju1kt2E8E1bd*+M6K zHB<4R7fuH%&X#Mcm#QS~CG)#UnB*&;q<3!_W<8CSTCB0^`BJKW-Z@^y$12HIFiCy2 zl&P&V_k!817R=_gXgYn2f9KOQTT9hes$LLH2jNivS?{aV566lAVZS~Av-Q_1x|FS1 zRo1k{e+BLOwrrPJt}u+I6sA{jmv=C94u_6hYDZjRlY8PZmoVsnghB@-6gnUw7wCyo z0v!k^_wLh@@HP*3N%+}jIJq5kczBbX&vSiI$bX{MHa zcv+^Z4Kp5RPiYhfqu#wVY{RStb0M2MkXS`mb|CS_*&wmu8lJmxTigQC#UQ#IL~{Kc zafwZ?g<~#ZAOZ;m5lASAKte%uOF#tS6vUG7HVfaxolp2_q3mD2YHqN#qijRCMB% zkO;!*C!UhkH?(6}Ve$y$Sm zLRNe5Ko(K(PsbwLz;-gtpB6W}j5n*TP?e3=n*Ec;5$vdb|}RPSCG zZZ;|YWjHJ8;SO|OuNt<*(=vqPcm>m=zz^IH1%JS9SrkgHapDw$`LU=jilVb6($ML0qxo+U%8j`)r&5}R_zouHj$;4FeNs7?@nc7qN{pFhK$aW=Vqf`rr$gw4?(nVGnXxR2M}t zFu{&C2ea{}!6eUh{yLLYbZ;jn8(U0D&y!BUF22Cq_=ke;j$1Bel53nMWs+++lR(3m z1RBO9f7tVClglQgP}qSzXiYlJB%lg(oFxnFCW8A1ERPb&HI7Az@+z+{0VM*zPKwd4SeLp?b_!)&SHAgUS z4lC(^N;s3uo&%!z%9U%t%C$<1&c6cXTQ%$m;@A;QDT_8mwk$j53zfi-Lp?h{<51HM zf6#C$cd+ds4Wklh7?owt9h87kS?1jl)LL`KN;;qtzA3^iIv|Q~ih#^)Q>0>-4u|r` z?~1%p_s+H0wC~v2>fAAl9>2N;R04MnHShq9LmfOo!>QcC!-F)8N}yp>mf3hv0!C$- zkw;LU%^54{fJ*q|SLWA2QEdGRG;IA^f3aofH7bA1qQN*U^9-wilxd&(%nMgy)L>fvaf6`Dg zWtq7LC7@)MC1^`#?qMYzPzl=?VNqQaMacvjPNuBb-SZlme|KY~lK$-ir8KvpG^Obl zn0ursfRY2B{E9rnryat7m znR`$aUB)5}UB=3a%{{L%nGSLsf5mrws;s1MyR2U2rm05?!}Kh=daZAK&wZlep*mLM zaiRrLHeAcdb?NfU|5sw~cKJQ|7kO4+H0_7Ex8ENxztx|H)1j99vHZONJO8}^Gfx%_ zuI6K-w`a>)oQ3`~_;(!T!u2s&hXchSNkFt3lRL)5t!6$IIUy zSiDZ7+2WO@Yj;taMw91vBjxv1oWE=6+8~P3=QsI~tbfDd;7_yd5o!(ugmc;4lUpDk zf6HpaFc3xe{R*SKE!j>V2-|_SWYs{SDJ|UzX_AOq4=lyb7xRB z->EkF1?Q#FHOe!B642ZTsW&xxUoFxHl=#53V6D+mqaHk(H#Zfxlp6=H&RB3kf=?ny zd&+H%wjo%GvFBS*%xBS|qqsIsu@FnQf5DdVZ?*wk5ORkV1SVKucmSq$BBBFQ@Cj*u zTsr~;$IyZT9X!sn9G|=aobp#2M&-<@WauqiwI0REz4LN1c3qcs<{FlYIywe_M0fRuqNb`zth>JR~zk=p4|^W9Zl< zaf~r`9NV2fF|s&-rhupLDUbY!-jry{kh`mT9W$HzCVtps-ZurME|Av)8RuD|DKGLO0_J(G->Fg ze>9yPRI5oaQWHNhqM4dzM+Q+m@zd;WJgg?OSo!@$&}3YdrfF9vemK>a84luoazukc z7^sg?FrTPtx=a$Qv7ctcPe$P^(VK_;FgupdM~l^JP##&zufBgA4y^0>bsBkz+vt_gLkRTc-i+{CAn60JM zwTb`O&}+K(5cbnio>&`&{l3ZZD~ z)%!g0O7V?`T+@ad+HkwkP|e1(9+bp`l57m-_86|XqHd940*SpXxdswvTXF*=?zZF> zNW8UVYq~W&-5N_E$aJ}ajPu*8itASuH?JygixgWkt>c;2S-d1Oc>fTX@H`3xk4rxPF{JbeKYas{0N z30e9!kdX6t#wC)hZU;!n>Yf7$S=}y>kk!2a60*9NKtfjcE0Eaug0t}jXR`zrz2LaS zCTFn4B{eeNk3d4^`w2+Md_QxEOVm#-LH=!~l6`qu zrm74x9%oN!6bGZ;y)wAz`##Hto@y6L8vE~|{yYaEO1)_^VbUBFR`Z?wj zn_LSmE@2=72?Y^ID2PBpL39g11mP6Kg77vEXCS%-Ao6eqqDw&pG30i7!6hE~puBYd zJ0O-Zl|4Ybb+>Mbf3>=F>O1OC50mI|5(z`cTw;@pp~WSPL?EFg0tqFNOI*^?iB~`( z2&dn83c}kS;pB`qbU+8R4y7?&9^;Xfxd0OK5pe013){pGqNFnR7a1O1`$yNLWlV1m z0I$3a09~VAH!b0<|HC8-6Qj6jCoGZ+;TUKfGNcyJFdC7Df8vNVlt-X(NuHCn1`&x8 zP$Y{IwATmMP_mW|=)|MnQS^D^1j+kTFS8A4Bj&Y)T_rh?q zo8n)Fvy>U`LFe^qU`sqLLpYANM}*gL*I3cTfXB=|0lWm7D<#*q|DuHje$4Z{*>7?xbav6Mwvf&>i9 zf&}gL!51uPO9ynqv7}L55XBy*0*x?D%BB1ta9PG2fA@L7x*FK}>!ek)Y_S7Oyg7O# z_@1NXP%y#1WQ~H!H5^Q!VPFCc1Cwib5nC$)6C_|@79?n|559m&TRNZ<_8^Bvbx{-p z6YOYnFl%obO!8dkuQOT3^!8%1w#B6MJn8J%#TR%R|48uNam%Gla*b1`OmYop5@;Bc zK*N{>e|!FPi+insOoC46o?}sh_WIxpleDD+I`JfBl16n=6suvmMwo&!SudB#`CWD^ zoUfoHK&JPT#oaQ$O$yJD-+^Qqv)ikDt#6jEhVAXz`xiUhCW6}sERO=oHI7Ar_1gUKC+90BY$)R6-;4z=U}4QFx>Qx4KFCV_@AS>($>2^f<_&K$w4 zIjp4vI^j$*cMgc+g)7&9!nI6?&c6cXTlMP*V%QONDR=FOY&mw!7b<}thgx=k#-W}a zf1u%1?%~=&8b&41Fe;0TJ17C8vdFq4sI%sbwRAuyd{2ZqbU+l}69Jjoo=C|i9S-G> z-xPVH=ACP?>E5xk;@mNJJ$-d|PzlUA)V~8X4mI!q4X1Js3lGvTDuISkS>)nD2^f_{ zJ|01BHfOA*13KYPUzuG8MX~x7XjuJPf3joeH7bA1)7LlZ<+&1@W}du;t-HHwSZ@cH zz^X&tJV4`6I}gxsF846>APwUZXc(78o*tBdaarW*5zN-ZS~{Q;&Lwm9fGA$Wat)|q zOLq7CJ7B(1kIy%;Y4XYH*qZHGJL*_zVlpN7iP+TJ12iu6_5ckd6KN=!NJGg)e;P`r zEb{iC1eDC81Z~N@J*=ezI$_%)EUJs5D49UR$&@9VdtM{+@9v9~GQWMGl;$>+rZn9h zk71?Q0VVf|*wottG%oe_01bz74{s0BFet%zs25!pd3#VN49X&Jk7V8+*3to;cr^+o z^Y)-9TE-#`En{WL-k#T(Ob0oRf8x8oRhBZheX3Wv>FJTeFg=S_uhoq|bDyYqs9I_~ zPP8D(mTMWgZe9NE|CKnnUH+N;i#+Qun)buo+V78-f7G9b)1j99vHZCJJO8-=Gfx%_ zuI6KfDcq;WW|4YLHFTG_o(?@$#n! z7O&H2ws>Xf+Fg{U(d7BvNcnvg=kIE|Hi)A1`Axng>z{CF{t2_~5o!(ueBCSglUpDk zf6Z#cFc5_AeF~$yE!l2EFt!72$*F-tU0Qk*(#8?Bevs5Ue*4N!;v|q$K^JLeznxuF zuJ*Eve!)7SRfW=opa>MxT&T7}@69@1qR4rwIPJ883Ju`Us=h0kDVVnKYPA9D1-K}H zlq;C2(9U~PVC>iqBy~w}s32~&mDGo;e{Hd$?3=cLGeQd#ajXVaR~a2x8op(I^aOZrcA<6;?1Z;I3Q3(J52D5M^ OH~|EF-7EQ%tR;Wyv^*&Q diff --git a/summary_results.json b/summary_results.json index 1a7b3ef3c..13aa1c812 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421564, + "Performance_Result": 0.421099, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.6926, + "Performance_Result": 1.71361, "Performance_Units": "Samples/s" }, { From 779404b6b83644f2c6d68f67739d2ab1f7bb7e06 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 7 Mar 2025 04:42:20 +0000 Subject: [PATCH 239/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9771 -> 9770 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index fc24b8b4e..e8662fe55 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-07 04:40:59,917 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-07 04:40:59,917 submission_checker1.py:3270 INFO] --- -[2025-03-07 04:40:59,917 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-07 04:42:15,144 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-07 04:42:15,144 submission_checker1.py:3270 INFO] --- +[2025-03-07 04:42:15,144 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-07 04:40:59,917 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-07 04:42:15,144 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-07 04:40:59,917 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-07 04:42:15,144 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-07 04:40:59,917 submission_checker1.py:3286 INFO] --- -[2025-03-07 04:40:59,917 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-07 04:40:59,917 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-07 04:40:59,917 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-07 04:40:59,917 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-07 04:40:59,917 submission_checker1.py:3307 INFO] --- -[2025-03-07 04:40:59,917 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-07 04:41:00,954 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-07 04:42:15,144 submission_checker1.py:3286 INFO] --- +[2025-03-07 04:42:15,144 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-07 04:42:15,144 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-07 04:42:15,144 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-07 04:42:15,144 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-07 04:42:15,145 submission_checker1.py:3307 INFO] --- +[2025-03-07 04:42:15,145 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-07 04:42:16,103 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1535 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.285 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 47eff0704..6966b4b37 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -866 +867 diff --git a/summary.csv b/summary.csv index ce70553ef..edb7c7669 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1535,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.285,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b83ca7f5b..dea8ae0bf 100644 --- a/summary.json +++ b/summary.json @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1535, - "Accuracy":"acc: 74.400", + "Result":10.285, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 658a8d20a71fabda270045ad60f0a2e0f3041bfb..0e0448e673bf7459fe9d8677071a5905856908e8 100644 GIT binary patch delta 1526 zcmVxQC@;{Pr1ONbe5|e%i8I$k>Cj_hk%wUrn1RH-#jVDra%JPxv zv~ANMdB`G~uSVwI^V61Z8e9Z<6r?OCBlDGT^XvGp;WI10^Mnv#NCP<^nTZg^z_NIl zkSySBR*+miiCLKiLf)1uix(vcqMDSYmTTLdl?6068w-Q-Pi(L_rXe|IVU>|wG)792 z2116<6IyU{JdCJ(rt*I=N)nIEZ_Z%y=$qDfSiAm*l4t(z+7Q9=futl9B$5R%4VBrF zvG3~Pg&eXK)zY}9byRny5s^5kQZZxC=Y*~jA&vADC1feDe~paF(z>RR1urABBR$Ya zB+9Un&?q8#?KZ5qVA*5y$XOd`sE)VhI=1u1K5RQ(+jD#B^}H0OWX-V}uXTjSfe6OK zl06$`?SU7wNCh4NX%!VowQPqhv$9nFX0iv6R|P5K19I z!Mynt{{>gy@_9tn!WN{>!GWOp%HThm6AEuId30@MER1JWN(}K@$S->GQpi`NQZo=B_xPQsqW;*ZOoY1 z+^CYgof?gQ0#Hi>1e0AP8nc850Sg6Nuw5G2%2y>gLrnHcHFc)xV>`T@9|tYKQw8Jka3Ek+(Qj^0dP>eGO~+ z8dmz+4%ftsHSuCiyjT-2*2IgIcp+XxyAkc$ijim07FkAR`PhG0QEVz%;Asv@^U+DB zi=nHnu7B0G8r7IMmPTCf_WKHTYB+KlF}~lW-56$ImZ=m*_se4nBnOV{`h6mC(T^l7WF~Tm4~XjrF7a)*AE>k1c0BgF z#^Y0Y;-F)qehgG@30)RHTqc2(?^u}z_ehk(NLp3crOTY3amc@kC%_^1^(Scl0JEJTgaQdM@;{Pr1ONbelPV-v0cDedBt-%2 zlgK0@1zWK1^-PoRBozU6lLaL~0rZncB{~6SlY%8C8()*md?o<^08jz|01*HH00000 c000000DyrOV delta 1551 zcmV+q2Jrc+Osh<=%>xQFwrU7*1ONbf5|e!h8I$k>Cj`&{W^9ui1RHf)N{f-{ zv~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw=GXCG!)MCA^Mnv#NCO2QnTZhPz_NIl zkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ@*pkD#=?O8i47XZX-JM~SmmS;jS)t& zK*;cUl9t>Y52I8*Q~7@wjKm}Jn=_a^`ldA=)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QEkVbln64H#RR2QMSDBR!;% zNR(kCNu!7qwcD`bg65CSBWG=(p*r50>)6g4`>^eFZO`qg*Ygybku}F^yw(vO2O<~` z8GSZb?SYrGNCh4NX;mtcYS|8HZsnQ$&1DZDuS&w=MOws!$xenW1mR-At7V>Y)fDi> zQ?G4Xf)P@;AC~Getn2Ay&1Zzm-JDyiWD$g_t!T+EqA}G9%c@nr3d+J{@#MC>w%uCq z{)>GSi^i9ZO$8J z&S2hri~oYFkNF}>)yfuR&B=jCi631v#WncgS#4Bk<||OHF%)Gl_PI)tmJ8jYx)}2 z^fj#XwH>aB7i;3hns~7$UaW~1EAc|ShIS*`wG|`JqAjwF%JQ-Qu%g&hvcS_El;)$8 zOcz5}TU~#vZ8fSfaV(9v-tG4l>eO)LG-7AFdlx|-%9q|d)Q(U_xWRdO<*`Q$vZQt4iBFx z%XhQABafe71~1Wld>woUb^r z{YSAUXm=ysQ=@t#`N`}EIicx>%HZ=n2osaI!QPDRnV#qQdxJ1B-N2TU^4a8ac|N(6 zh^JDFOQZAYY`GX+&70}ilQVe`gnr-dhy5V%$3*=YsN52|N>(4eHGB5GS)7aEe^5&U1d|~p8nY4@LkIV2-|_SWYs{SDJ|UzX_AOq4=lyb7xRB z->EkF1?Q#FHOe!B642ZTsW&xxUoFxHl=#53V6D+mqaHk(H#Zfxlp6=H&RB3kf=?ny zd&+H%wjo%GvFBS*%xBS|qqsIsu@FnQf5DdVZ?*wk5ORkV1SVKucmSq$BBBFQ@Cj*u zTsr~;$IyZT9X!sn9G|=aobp#2M&-<@WauqiwI0REz4LN1c3qcs^|fh0u%?UTnOA{+HQ^8if*002S~000yK0000000000004l2casDqK>_xYMkP7{ zXp?~@CL4rv+1w=o002+|000pH0000000000004l2Nt41QJ^`JR0VX2`Pa^;T008DV B-EIH? diff --git a/summary_results.json b/summary_results.json index 13aa1c812..28f822529 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1535, + "Performance_Result": 10.285, "Performance_Units": "Samples/s" }, { From 85dd5efd05df5c74fea400a2c5b1ebce95309482 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 7 Mar 2025 04:46:17 +0000 Subject: [PATCH 240/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 9770 -> 9769 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e8662fe55..5f5f4c2ae 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-07 04:42:15,144 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-07 04:42:15,144 submission_checker1.py:3270 INFO] --- -[2025-03-07 04:42:15,144 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-07 04:46:12,539 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-07 04:46:12,539 submission_checker1.py:3270 INFO] --- +[2025-03-07 04:46:12,539 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-07 04:42:15,144 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-07 04:46:12,539 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-07 04:42:15,144 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-07 04:46:12,539 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-07 04:42:15,144 submission_checker1.py:3286 INFO] --- -[2025-03-07 04:42:15,144 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-07 04:42:15,144 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-07 04:42:15,144 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-07 04:42:15,144 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-07 04:42:15,145 submission_checker1.py:3307 INFO] --- -[2025-03-07 04:42:15,145 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-07 04:42:16,103 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-07 04:46:12,539 submission_checker1.py:3286 INFO] --- +[2025-03-07 04:46:12,539 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-07 04:46:12,539 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-07 04:46:12,539 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-07 04:46:12,539 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-07 04:46:12,539 submission_checker1.py:3307 INFO] --- +[2025-03-07 04:46:12,539 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-07 04:46:13,530 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71361 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421099 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421136 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 6966b4b37..21dbb491f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -867 +868 diff --git a/summary.csv b/summary.csv index edb7c7669..e1d038742 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71361,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421099,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421136,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index dea8ae0bf..4134cfefd 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421099, + "Result":0.421136, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0e0448e673bf7459fe9d8677071a5905856908e8..564e4b95a9bfd526ef834b03885f02da1af886da 100644 GIT binary patch delta 2198 zcmV;H2x<4KOsPz;a|i_q#y|K_lXwVX1BYE{v)u?P0)O~G`~qqq*T`u}c>DR-^8|H9 z3p${Eq=IuA)8#RA#<}1UkBrRaOS=aLUw*s%bk#ZSp1ohJg_3MBGz`ceqX0SwL+5bl$fb6~B{um?JmwMx9gtAyfP_K^ zB;*1;aY~>A;pDUXv?RRE!(9@7wi!-74mv!%Nq^4gxkHHrF{H{1F7e2iTsjpJIUM!; zAitZbWj8O&RJCEoSy` zfApHPitg63H5RQPxAx%048b%}1P#lql@(468 z$#b&SAR|l4~6LXb@;Plz%|O zpadEQCD*W7a;i8k_2r*w-nA;Ne5Ko(MNtPs*9rNY>6~F zrXjuq%PRV~%cQw#m{>h?HeAOpxPKIQ%jr=~1CD1S|xrSpY z%di9q7?vdo+UtWaSkjUXsDxulqkp;}iaksP8ey80OZh+GvWhKu=VYv)v#<) z04ClXJrsP;(Q+u5U|+I9!Q>haCeSc2frf#}HGC1l20VN8NOf4arJ)<7mfC3MfREJ1sH@P$cQ(gBrtk}^r7x+sdRVYx<_f->2x zmdW{DHWkiSP!b^1`^jRq%%@4=8S*=jtfF^2Enl0P%U8n@jlx;7z-}VAf57r6kzC_g zlt`}OL;?*X5@;BaK;uv%t$#8iK>|i(NrLwJ;0uwoqys8pPheS87esL)fri~xT5kx* zf8sH0B^}&}No#{iDGgVcjRGbM9618`ZKx#&XdLRv0UFNa4z3)eVN3!IW3tScgAy<% z%d9zqd2?7v2UNnDWcC~o#aFIe16Hn8T6F#uDBr4KM-a!3Xi8bMDSxtM*)dGgi_8mGDgwX3+srd{YEuW}6}v zyL32|KYmx_jksO#*>(`1cJFijsV;;Z0Q8Uk#*mU#cYuKh+ zw1)KxxCCAuYUcqOhx&PdhI6@tqX%gimq5d~EHm|>1dPivTaRGA9#+x;m2fVZwFgA; zH7wVFHEhN9o_`0-H)`_vCN^C@**dmidse|ZR+^Yh$&Dg5HGlU2jZ4ivK*PvH8cHV8 zP%@E*k}1o~JtzSsvn)YdGII|r>3~Yuz6guzq9{rx&~P$k#qOTh$o#t-BbD@T7bvB< z4W%hfx4_&ZMFEuDC}LA{574;O+ygWm${ox-NW-86?V)CLS!V7*l`tsF%srBsdss;a zRN^%#l+4_NqJQWz7HQ}*R#t58d5y_*kmD%6>#nkrzU{Jlm7As>DGbxI=<2n;@jds6 ziihf0jmL==MA>jHBiE(NFaKYOz1!vYs9ilLRF}0rQhaB{~6RlYu2B8+Ub;Qziib08jz|01*HH0000000000 Y0Dyr;lforF0h^NnCL;z-BLDyZ0JeGBvl<6LlwM@HuIrQL&rFTY)Wy6T*E&)zTALP@q58iwg;erE*Hq&^vE ze<#^OCwnzh@t_w@2P)2%YpR#3B<&^hyGfYjE1#ryZy9Dijg?xgvFiC!s(#)%Ud6{M z$yP8)eSfu-sjV~jg4wJV%;vRdI(>}i(==O4)mExr5KRZ+Q2$x)tJDw2iT+`~J^-`z z*DAV{tyxvpw8eh~?fSNCmsqYajHVQ(S8$hiFmw)wj$CR-Tw;@Z;xU&n=zxSm2P70a zAR!m%iBkd{2q*XM(~|Hu4|hrU*=9Jo9dvkjlYgAgbB7WMVn~%2T;h>2xpXQdayaVw zLH=o`mVJ0xrm77y9%oN!6bGZ;y)&zLMOStp@y6L8vEdq?yK!6G0@1}F zx*SAu{Ty+LO|FGwE@2=72?Y^ID2PBpL3B$%1mP6KlJGVUXCS&IAo6eqqDw&pF=RTu z;C~X2+$b;I{|<;%bY%w+Z{4javC))HvtS+SVG=z~B4Ox=OKfs69CHaH5lASBKtf66 z5|>nT;+2pH!s#cTlJK@iI60#Q9Z&&XhtilXkMYRJTmT8V1zdX7!Zz`PD5;J8MTSS$ z{?TjFD!R7=fLGoIfUePOnwId^|1gQd#D6F*$_b0)LO22%hcxLJXc&!1LvchJ$|KOY zB+toOgNQ^4D3WCf+UtXBC|OAdRN~P&kVSPt6n)-!fie@|FE^V1Hqd;T5pyj2O3kv% zF-}zPUKnmRDgI?RE9v15bY8C-w#3sigyVPx)1$x-+z$nRz;0O-O0IF}qd}nIP=5jq zgA!;Mlw8AR$+ZST2@)_UOA@pN-BLJXB^^+SM<4mIs4j}4vnA5d>2jm_ZxPCkxiY6x znuhoeEUW0_E|cb}VPf^n*>D}Z;Pa!vTTX|9@8VcC#gc0rNwMS_jwR4AEP;k$$u%5H zS%xJ@z_2Vy&|V*W!IGAAKqVYY8h_OVQS4zV&8pb5p^QT+fYYk))R6_S0%M!HL2Va<^B^^+SCn=LOs*9r78kTE>DJYZ8 zYMGqhWmDmN1tkG8y`L;*%Y2#?o*};j$trrc)AF^sxqLMo(I}iH3+yI>`v)wK63I1= zMTz7ZP9)GUB7uex2{aBR(tj!=5+ql20VN8}8b5H`t zWSKQbFmDbk>3~W&lgyq2qWH>{Yrx92N{i0F0_9sZ>4McMdi10F6T(JV3*#+`+?xG>l51VN{mc zcu)dHWtovjP@l~iD}U*LO8Dbf=GQ?{Z2bx}Z2elXW#=_2f6U|8H)`g&5}R(Gd=1-l zi`KAS0hhq5L+w03<4`{j&~PqyaP%Mz;}U2Xmu03Nlz?$rX6q5m*TYIWpc2j{v-W@} zzJ}!*u!gPJ-t+H(`9@7X-^8ZNCtJrhY|koK$4V2EDY;R^rhn!hpmC|W2WS|XNJGg) z8cHV8P%>qixd$bnWR@jpOJ?q2B^^)++ZSO`T@*#h1R74Jtk~W28kv80W2BP)?E$G#J+CpD4ssmDcYUg?q;I>dUgf5#M+(FAEV_EFZ+y>v zqT-=CR^xG^1yMF!%gA-<^2`5MV()hOJ^2@TR$nyjhq<@kA1}YvpN7++mi)2&y#PD^ zy#O;$77VWDW2L1}Gu6`7<19Nvd1rO4|2SCwQh+v#!+$6a)2C(a`A-$h$Ns@2>c3w! zYCaABn5*~if_L`byTJbNvnD@iEiLQ%vo!17qL%yxYSzL2%pasF?)=|^Q zK7hx|-yT@JPNUi4m8ENUQJO}R=XWFJ_f?#~Yv|e_iqhvd`H-xC!{OjhP)h>@6aWAK z2mk;8Ka;8?5wi&rOAZ8IlgxaRSRgBZP6BOZB|&N{PDQFzs-oU3djYHQk8HPrx33c* zfW)cC^~`=dyK*(#sW$lq=cUm#$}@rz(A)^AH#PcL&eKPf_`tMat z5ci~fmd`KCIcf@0OjA;(Ck6O)k>7Lyt!DgpG9MI|}`W|M#=CL3Rq%zP#R002+|000pH g0000000000004l2N0Y%NJ^`GPCL;z;BLDyZ0J*LJs{jB1 diff --git a/summary_results.json b/summary_results.json index 28f822529..4d9b6fe70 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421099, + "Performance_Result": 0.421136, "Performance_Units": "Samples/s" }, { From edd2edeef818a15ee1ac6afa3b01e34fdeb236d5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 7 Mar 2025 14:22:13 +0000 Subject: [PATCH 241/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9769 -> 9772 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5f5f4c2ae..7f3497d66 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-07 04:46:12,539 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-07 04:46:12,539 submission_checker1.py:3270 INFO] --- -[2025-03-07 04:46:12,539 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-07 14:22:07,736 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-07 14:22:07,736 submission_checker1.py:3270 INFO] --- +[2025-03-07 14:22:07,736 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-07 04:46:12,539 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-07 14:22:07,736 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-07 04:46:12,539 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-07 14:22:07,736 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-07 04:46:12,539 submission_checker1.py:3286 INFO] --- -[2025-03-07 04:46:12,539 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-07 04:46:12,539 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-07 04:46:12,539 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-07 04:46:12,539 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-07 04:46:12,539 submission_checker1.py:3307 INFO] --- -[2025-03-07 04:46:12,539 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-07 04:46:13,530 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-07 14:22:07,736 submission_checker1.py:3286 INFO] --- +[2025-03-07 14:22:07,736 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-07 14:22:07,736 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-07 14:22:07,736 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-07 14:22:07,736 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-07 14:22:07,736 submission_checker1.py:3307 INFO] --- +[2025-03-07 14:22:07,736 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-07 14:22:08,877 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,14 +24,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.299277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71361 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73041 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421136 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.285 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2234 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 21dbb491f..978c515a2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -868 +869 diff --git a/summary.csv b/summary.csv index e1d038742..e153959e1 100644 --- a/summary.csv +++ b/summary.csv @@ -2,14 +2,14 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.299277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71361,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73041,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421136,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.285,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2234,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4134cfefd..9e6afdbd9 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71361, + "Result":1.73041, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.285, - "Accuracy":"acc: 75.000", + "Result":10.2234, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 564e4b95a9bfd526ef834b03885f02da1af886da..96e640c4b09333bc4dc1615fdaac401b301e4c23 100644 GIT binary patch delta 2147 zcmV-p2%Pt+Osq_>%>xShYGjvk1ONbf5|e!h8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Ec`~Gq%&q-;bR@2ex;aYF;Xx7f777_AOraj{9F9+4U&g6O7g|P{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!` z50@khcz=@?B$rQO%CbPnTeh%x!AKBQq%5^u+xDz1NOH5b&|@F5LE|_H$uSMfjO3y= z!blnj89rYo1viKND3Q-pJ_aN4!2IU)MvuN}4f~bre-rY|H?9p4%pXWfLO~)~0Mk&J z%_;q%9$v^HTTv~Ids;`+py$qFXJw$KI^K%w*v=dKu<5jI&+Vw!^O}+s$Ev+n5grF3 z81@-`Hdy6>7lC?FPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!XgpNa^4JSW@V}T&3|MM zAg>C-;#rc%gvm~Z%mm?L#>;t@aMcv>*;A)!TY?c%wI7!1GOX+IXvJrQ%iWw?i{&f` zRa?=JT|{lF5f){mcomd|%h{9L^qO{KwfoOPn#ak){9;B#1WC%xgT56bo7??o5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iCTqHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR5dC-Hl0I z%g@^Qmd*%UkP%5!ZiG}8tY*HA*5?fqr#G#?#ec)q$9x_oYGn)3`s6?)`NH5o>mv%U zH+pn!Wh{&*Wl9Y3TF6g&@_$muccgMPjp!w=Pa}+iC6p!Nwf3$~N6Iu9W;EKbf-3Xm zXG!+=!ac9I7kUR@4EMko7&7xURAyIssg4Vfu!j92Sdi<0Es~rYDT!sGc2llZtUlkU zt_3ZsE7`2isSq?<-z`ZH5vK0s*KJIxSl_6Uyd4`2e*sWS0|XQR01N;K0000#05##y zWl^(u2xSHYX_(ETv)~S+1O->vjEPONh!dd#f5~VorV4lgifSqlX+b*-E2uL(leo_x<7)!Lp-JAENp*PmOj*90?Hzgi z{4#in?&Is=Ll{3cr=M5}9c&_Lh;YZ%f9W#0Th=j!?w2^~7(zR`z)Q32OZG0Bx{ybd zpSrQ~S@zd-JUu~Ya4HiFXnuW*96MA3+`Dr%_9wlgt=7E+Z8_7>*N5~0HH&h0n=Rug5#0~alY|r#O&)*w_iRlKmoRrTdm&^0Xr9?bsQd}CH zPiM=;=xW|f$DW+YgCO+#c0cR~fhYH#PRCDXB8T{ZxPIUg-*)?fI;(BRV<~ZUJU*2t z4mu|4$3W$l&{eYf=&jka@6F;|4F3aAO9KRxASD{J5g0=V1sg?H*(H-%ARd3qYQr!P zMfd#*qrENJ=_3f+fwpATK%pru-3e)uh*}RMwT}P3vYj{yWYy*9-Z^t;P`TKvCiw;D zrO_42GlCM(+z6?+6?$JU(>Y3fU|O)oXsA#J9xdvdl3U7+gI8xPxFEqN5u`okwnDoQ zEXCOK9Vq6r=+IHz7^hf>rQ3gE%lJ3j0xk%-#|i=yEHK;yQ#%pS0V()|v_GyH0D@y^ zK!FY(=UI+V-T+Sds||y4=2S9t7Oq;4;^f|YIT_ow&Dz;ujrnu@xq5jUR!ybuPsIV% zrQno1V8OV$#OFmkT*G`=#S?ENh{rDKSKbbbjC~-I7=Vs(kIKjF>3Jo+M0G)myOhjQ z^03ZtX;Dxz|LR9x^q+w0Fy!CFBVf#Z`3dSjvzsA=0txzRWS4RT004TEC?q`_HQ~=? zQ3(J52SNY<7ytkO0000000000fPrU|fFwl$?vutOA{xQC@;{Pr1ONbe5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0TZkN%wW^YzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`B}fwbPvK7_RxTkegccl@LIH*!FW6$S=u746Cjr0^HWGSzIjf~3Dx~7o@FC()f zJ0|58)fZ*7lC?F&cL;0l|nUouZC4@)_P#J88BoXs+dpX&d_>NhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgcTJ@wQPqhv$9nFW`D8= zkXHpM;|0xQQp!$-ECk_V!K-CPxoQgd;;GlREm0Cuw;z`3GOX+AWX&%Lm%BN)R>>j= zRa?=LT|{H56&6*iconP+lf{$U_S$x9z56dhmdA8uela5=0-AF3U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0t*!``xOyC4ag;F1o%hIxr27bVvuT@(2Wq4uL??ArL4!1Oi3Z z-yu4L4n?s!PsDDPeWe<-@o$mtJ6_t4)Go$Yje@9=}I z&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)g5$U zrvy1SItMHyw}@5P7Wtp6x@uYFqH4(xX=UQY_dXV2_rD16Gt&D50nYt*0(9)Q>%%?I zdiN=vk#a>QBu%*yGFh*N_%_-cH_V*Dy!jOW1y|qlc|_I17NpI=fuQ-y;6IuZ3U4rZ zbZuoUjAvC!4DnjXFM9G)$bVO)QZ@6aWAK2nqlI06zc;#y|K_ zvw8?+1_Kd|!n5NJqXY$8uw5G2%2y>gLrnHcHFc)xV>`T@9|tYKQw8Jka3Ek+(Qj^0dP>eGO~+8dmz+4%fts zHSuCiyjT-2*2IgIcp+XxyAkc$ijim07FkAR`PhG0QEVz%;Asv@^U+DBi=nHnu7B0G z8r7IMmPTCf_WKHTYB+KlF}~lW-56$ImZ= zm*_se4nBnOV{`h6mC(T^l7WF~Tm4~XjrF7a)*AE>k1c0BgF#^Y0Y;-F)q zehgG@30)r?N^U7P4&I!x;DQ97M3DBB+Y0SMuoPp@cc7ThqC-b< zW1M0kmTrHGE#u#83wT1v3@Zpsu)uH!OzlWSN2K5*(*C%n2MCU#0R=jEoM$;cdjmM- zZ#MMGg;U9JuyE6Q5=Zym%hA}jZPpgOHRjLpdina^ubN8Tor(jhOTj63z=Cmgi7$(I zxQ6+#j3?em5Ko7w-*`JNGW3B+VgNeCJt?2X^UEe`f$9mF+^3{S$-|0F>2yZ(;;S2Z z)qe)6l0RR9{0ssIJ00000000000001hfku Date: Sat, 8 Mar 2025 11:15:38 +0000 Subject: [PATCH 242/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 9772 -> 9771 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 7f3497d66..9f0fc1489 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-07 14:22:07,736 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-07 14:22:07,736 submission_checker1.py:3270 INFO] --- -[2025-03-07 14:22:07,736 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 11:15:33,493 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-08 11:15:33,493 submission_checker1.py:3270 INFO] --- +[2025-03-08 11:15:33,494 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-07 14:22:07,736 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-08 11:15:33,494 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-07 14:22:07,736 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 11:15:33,494 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-07 14:22:07,736 submission_checker1.py:3286 INFO] --- -[2025-03-07 14:22:07,736 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-07 14:22:07,736 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-07 14:22:07,736 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-07 14:22:07,736 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-07 14:22:07,736 submission_checker1.py:3307 INFO] --- -[2025-03-07 14:22:07,736 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-07 14:22:08,877 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 11:15:33,494 submission_checker1.py:3286 INFO] --- +[2025-03-08 11:15:33,494 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-08 11:15:33,494 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 11:15:33,494 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-08 11:15:33,494 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 11:15:33,494 submission_checker1.py:3307 INFO] --- +[2025-03-08 11:15:33,494 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 11:15:34,450 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,14 +24,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.299277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73041 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421136 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70326 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419444 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2234 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1034 | acc: 76.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 978c515a2..51270dafa 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -869 +870 diff --git a/summary.csv b/summary.csv index e153959e1..c8c924c9b 100644 --- a/summary.csv +++ b/summary.csv @@ -2,14 +2,14 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.299277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73041,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421136,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70326,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419444,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2234,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1034,acc: 76.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9e6afdbd9..10e642456 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73041, + "Result":1.70326, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421136, + "Result":0.419444, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2234, - "Accuracy":"acc: 75.800", + "Result":10.1034, + "Accuracy":"acc: 76.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 96e640c4b09333bc4dc1615fdaac401b301e4c23..daf3ba20fed66a6aba041c591f9cd2d2b8cbe3b0 100644 GIT binary patch delta 3310 zcmVx9VWxg)6(F4^30qm2f2hIdw(Gwe!c?c$dTXWi06oudWD>Rxs zBr`?m9CY;?1^r4N; zGW)+gjz`+PO42BvG<9K^x~3*!+>0iIrhaq%x#sCwngx?yFp4LtsXwVy|K;GP{f9XD zJsm2QX?cQ4+SG@CSvK9P*VAyQ#zAVtQ#Hwt^y6e4Wck}?OP8i2MC0R3GDTHdd2tkt9*0Aj^lJ4x?$RAME#{{8%v`Em8fZekk^if0aV7 zAMDTjzKzsFy7pQtrg9g@zZY*#drjReq;;nKJRinoqV-gNKbVcOtN7uIiUz|hUpBi4 z3FA>Z|5qDF`C3X{8wZa~y`gIlQ7;=7iM3(W>#1U)buFBwSv>xAaY!r^EJB5|j1{I> zMf--ZOh?+q`FI+sWlVk1>wJVCgDg1MPvQqHnfH)RgW}AHJt^~L35(XRMRPH>`m*8b zxk1b4QLNs7{Xt{DeqSVBD!$Q>Yua!_8*bMc>iKw9gOYeql8wRK9K#h?ST-3ZkT{!? zYanqqB{x9gZAxx|#9v7^rrW^NZLkD_OqVOjIKREBxPDb}^Qz*uOtCRj3(wSI@siAx zD||B3R~6T67Yfe1@u6!k<*g!*7LFF z8`K$pZD@n>kq*vjOozwNJI)1{_+(`+U&h@(`10H3r>pj9=j{Ew7E1GCXb@$?*_{!_ z8m6!*HkxCY1U0=cjG86Dxb7_ZyRPkOO%?|SoNZms-L%y zm+^^8^9m-dFH4!)DswlSPHW+GR*NT-$7D8t$?{sNwp4Y)c+!ss`prm_QxH|`pV4bSkswU5OO z5Iqc{$3Z05&k>h6;}feADWOs?TYY@-ZJkbr?%lAyIZ_yQ(v zX@gGKgB%vsK~W4$u%pevY`keO$#b2*&SVwS+lk4>29whFrMqPpU*K*0L&0~)ZI3d^ zHLgXON|Q zFa>4Os+P&wU4AQ^EukbpX7}Uy-Lkk%3g3|5fn*i4+o^oD*2`DJH5=PD|HT%&iQxVL z+owcwjcZdPxrP%7G>k}~VMGFrONq40hy)23ktGRQtAj5@(v~)V=!89iWlLqPr$k6|mB;7&~1YfMUMc*1OKVRDNjM*zPKwd4SeOFcP2!)&SHAgUS4r^(HPB@dyo&%zI<;pdna;?&$^RGbpRt-CXI(956w{41SSa!@8 zDuE$~dUk-urKTN!py5>RVB0|&MkUZND$ATZC;_9g%)29~wdRbqv_U6)Q-oP`Kos8; z0h!sRNX0H44&{&E6?voXoolgf-*NKl+%>m7e)YCc3EVl+k5^UFyE-j=bKn}`Q&wM!}0Acb*wZonUWht9BS?X8jqTLfQFHY zG?YxFp=2U|4JA{SnR`$IN@iJtmSpB0*3t%@uze90)j?5|OrYUp%8K1RuaWt8H%2O% z-!4!}a}!Ean%UHtL?mDsyo{GI%ZJnt`_^rFJsAB+}%)t^O^ftLQU z_`Lw9_`LwLNEY_5W+SC#PgB*@)#EfjL&aout@qep{8E56O`Qevx^g<0)Nj8Y{gUoFF;XE1tKkIhhYVk=qR9dDgNnztA9s>yBb`P)eiSHc%Z?RBX4o6HL5XjERDF{?e`Vx z)Ntf9Vtl_#zcEuy$+?QeUQ71-lMn-xyrdO}6;Op{z7AAiki!!Qtp?|llRyDcjwB?#Mrw&c`6p)M`G32Bpv zT0cl?9lw1QCvg(+DaX~!emlFUT<%qy{DJe*=n9c6M+s7OXWI zD%68V%le_@mU83Z%^3?WNbpGnX-~PW&@Kc^G4^~1iuo)$bQCwnDHdYswtv_%{=>F_ zXLBsnGfrD;*A9H)JJ{d z?X<|T4@43J&|}=Q@>x8;G$kuk&+^$k&5JZ&G=xwxr$zC77Qelkg;00rrzfB{~6UlY=ED8=IiA sOeFyT08jz|01*HH00000000000Dyr>lf)%H0iBZrCL;z%>xAbYGjwQ(F4^30o{|Q2hIdF;m>80c?c$dU31z-6o&8nD>Ry1 zBr`>57xd*Z?$}P^7-Q@>wmZFIWU+vzfT$$=@$V}E#&FInnWkywX>;Mg>OE&aq=#K; zU)cZUaWdBKRh)#;w51Ef)HO8?qJB6Xw)C6p&ox)qlGLB}{c$u^E&WL)`Y#7R?LS2E z@5xB1RLc@fla@YzO4He1y`BUkHSrT8nyG1aWDvy@Kh568!+J7{mET_kO~!R;ns$BS zhf{r-VK3e#M>H6Of%+H)^NE_K%OtTH`)StvWE9R4{b0WzX2`9poOOTg-&5QH4)mH^q z&jni6k9_oh?hhLK_4_>WO7V?`T+@OZT5!8jP|w=4Zj{80lB^Bpb{np^qG^$40*SpX zxdswvTXF*=?zZF>NW8UVtGf-{-3Ch_NO!q{wDa4mitASuH?Jyg%M@EZZQ`CbS-d1Y zDR- z^8|H&MhiNieWZeO8q?)5bjG>h5|50`{B+eh?Vi0~tc8+nF*FR*(frN` zqDg%+&i+oag--Trrs6>_oDNi+E!R{pRY}@Q=691Y$yYu}@7^-ZdKxRWSYy@mrBwaA zbG(X=Rg$e>lKN^XQ(I^51+!T#n9Xa^bov;7&!=g&ma45(y&#$n!lC}N-dCv~juZXE zetiID>#tRGDOt#^dZMjpAU`yO)M-n6+RoWOD}+tLVxOB;Gh1BsN^bb2mN~ zw?K3;h%N__Tt7!#Vv}p(m`fOlKte$T5(*-aP!Qb`5J5Nvu_V0B!x@Nf35YzLf#^~Y zK@6ErFSx`bAC#Bwe+R@Wy0Qa^x9-+|l-OuWr&%y1dYD9ylSmjk;u4!&498r;NCXl} zB9KrLxx^(Eop>cAf^d4rQxe|x2q$N>paUwP>rfigCaFfodYa>63H5RQPxAx%048b%|3(oh_c zhVlqBF3EGU)*vEL0*Yi=g7*608cJ5u0hM@k4rEbX5JjIiUZBhb_{)vvzYR2BX2cxJ zzEZR7a*PwzyBCI=O^SaR&PsZ?1D)5ahAr{54BhI%8~?aLAMmnSV;#|;?YNbEUJs5=xm8Jbh_MV{#%4{ zW3J4pl%^rR1IsGzJN(f zI-nBvAcsYDQ4|9c>}Yc^8*ds+@?7VyGg(FVc4D%z#iaB+=@jhZ3%regDERKUl20VN8O5J%75zz1Bb`K_zt0u`EG*eei`zTG9cPc#<+nqq-=Htzo%F zn1V9dtd`07T{acYS5Oil)BDL{w#=tV;TiHfkgTG2J1t+Eo6A?jX_(ETS+c-xBDjCR z@+gs9<5-kPuHi%i4I>h07?D8ZP$I1|B0&O1WJ!Yd`rr$Zw4?)nDq&AxSyUH9aUy|+ z-Bns|2*`ipF>ECr+=)qRgGnh3SD1|gCJP)n0{CsHB?o96>d65b&g2fR9He1P0u5ub z%$S1`Feb~aIf8j}SV;#|!kJ|D91z7e&GrhnjYOfQD1KgKYfiwyPUQ|B9;9JZ z0u7_G%*KNfFe=N8Jc9ac&R9tYRKg#>GQSRrV(V9+Ve8j_iY+^@QTbyYzrIm3&z0D8 z^Wfva(oiygk%p2f%gj9}0VT66L0d9&4=d?_O4z;#i|V2%N+!^7GG)c?p4Z6yyBi~w z^luj^rMV5IDNVP)+#^K+l-wv{Q*#f{xYXPOG#ttu%soiMpakuqW^`F*?m?9>D9g+} zl9_v0Ne5KoH7JzK+=HU%G8Sp*GFDb>?s<*LbdckJD8B2ivXZ{-vU-)9rXDE_)3fO6 zwZ8E^_lb&!>R64(i55iJa4jR(rOPk>Ux~fj<@e-Y)c)et*3DR(~2!hg$N- z^7jJl{PzOPJXtWfnva#1KFw51SC6yo4CS5Gwf^H^`AY%XEDobMOrMsu=RZ|2ANvQB zsQ-Rp(Wv<}{9~@(zYE^kd+!4K!_S)hptZED>(A1xcZ*u`7pPeW`!j#2&i!~8P7`gc z23bc*o=uyvx^g<0)NS9E2au~0g7rW5NSa>3@fNaM**!%@lOXVUqdM$BTkc~ZeC4e zqqLk|{W}`m)!>S(cDS#>0}ZYmd5dEuPdi-G*RZCqVWqF_a80~e6ED`pi#73LO}to% z7veRv8_}+<7KH;hy1+}b>`V48n!1ojl%Kk>@>%xRbUZylXK*SL z3}}9RiyS*t0^GZEHTEaHqpjW#>_t1hw3j^fAGGT~iakNQ8|j`J)f>rAW=F^gO*d2q zpXWiCn8XeCW^B*&JkQ@7go)_}ww#pDCYQ_e$)!X*Wlmfgolj@W#pr6@Ovj#_$%7#D z`*uI<2Z1N|o=(S4W+I39fVh6(65n?FfjX;g$73mRbv!c>Fkme5tQ`sl6M zv+vE~TnzsMv-THZ2m~8NR@swVARm9rYQr!PMfd#*qrENJ=_3f+fwpATK%pru-3e)u zh*}RMwT}P3vYj{yWYy*9-Z^t;P`TKvCiw;DrO_42GlCM(+z6?+6?$JU(>Y3fU|O)o zXsA#J9xdvdl3U7+gI8xPxFEqN5u`okwnDoQEXCOK9Vq6r=+IHz7^hf>rQ3gE%lJ3j z0xk%-#|i=yEHK;yQ#%pS0V()|v_GyH0D@y^K!FY(=UI+V-T+Sds||y4=2S9t7Oq;4 z;^f|YIT_ow&Dz;ujrnu@xq5jUR!ybuPsIV%rQno1V8OV$#OFmkT*G`=#S?ENh{rDK zSKbbbjC~-I7=Vs(kIKjF>3Jo+M0G)myOhjQ^03ZtX;Dxz|LR9x^q+w0Fy!CFBVf#Z z`3dSjvz;M?0tEVMWS5gABtir=;m>80Yb1XKSJ;e+O_T2=R{{5vM Date: Sat, 8 Mar 2025 11:21:18 +0000 Subject: [PATCH 243/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 9771 -> 9773 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 9f0fc1489..d7f29041b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 11:15:33,493 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-08 11:15:33,493 submission_checker1.py:3270 INFO] --- -[2025-03-08 11:15:33,494 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 11:21:13,845 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-08 11:21:13,845 submission_checker1.py:3270 INFO] --- +[2025-03-08 11:21:13,845 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 11:15:33,494 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-08 11:21:13,845 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-08 11:15:33,494 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 11:21:13,845 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 11:15:33,494 submission_checker1.py:3286 INFO] --- -[2025-03-08 11:15:33,494 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-08 11:15:33,494 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 11:15:33,494 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-08 11:15:33,494 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 11:15:33,494 submission_checker1.py:3307 INFO] --- -[2025-03-08 11:15:33,494 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 11:15:34,450 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 11:21:13,845 submission_checker1.py:3286 INFO] --- +[2025-03-08 11:21:13,845 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-08 11:21:13,845 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 11:21:13,845 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-08 11:21:13,845 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 11:21:13,845 submission_checker1.py:3307 INFO] --- +[2025-03-08 11:21:13,845 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 11:21:14,784 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,13 +25,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70326 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419444 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422043 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1034 | acc: 76.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1321 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 51270dafa..54b472ca7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -870 +871 diff --git a/summary.csv b/summary.csv index c8c924c9b..b1527d213 100644 --- a/summary.csv +++ b/summary.csv @@ -3,13 +3,13 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70326,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419444,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422043,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1034,acc: 76.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1321,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 10e642456..5df7e0698 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419444, + "Result":0.422043, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1034, - "Accuracy":"acc: 76.800", + "Result":10.1321, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index daf3ba20fed66a6aba041c591f9cd2d2b8cbe3b0..e11cbc310f8db753983e46a798d13d27940a419d 100644 GIT binary patch delta 3420 zcmV-i4Wsg_Os!0?%>xALYO7n5(E~*VuqKH!v6CAF8h_Hncp?R-EE||k(>4u~g)E}k zVqkuso;G~b;3CMPAYmC9n6HGJUx)wnpIPyPF9{KbG?4LuxfCMrSr!kMBn^0zA|~wloY2KmNFyCZ30cCcUn8Zmw61BS!OOsGNe?sq3 zNCh4NX@ZJnId6t6wbDfXrm_c+S2-!-8O>r+$WDgL1mR-F%XvzNa~XVL9>Oyf7VA7UT^g1+R9iMPs)TC;x(6_^yDR%??~lp8qrH!pGFvY zL0Lh?YwcZ~j+99-Oj)#F1yyGBlMx6XN<3JQ>!4WBj2j7wWukUdu2n^SzENEZmRDD@ zS)Vf@Sh~Jjk{}|5x|3hGF=JwVqe}92Y&84@P)h>@6aWAK2mk;8KLBoj1)NfodI&ZK z(9L7$*d&v-2r7SvzdTOH+P#XCFq$@WVVJt6ra{yXr^AMRbN#vE>ROWe)4o5BrmCSo zsYL&!`P1P;6#t%#luETM!8B>;qcoizRI5oaQWHNhqM4dzM+Q+m@zd;WJgg?OSo!@$ z&}3YdrfF9vemK>a84luoazukc7^sg?FrTPtx=a$Qv7dit!%s%xEYX{X{V+S0&qs^Z zprId$gVwK7=+)-oqVL;KJtP~iwR|dfQS^KM=Ct3?&0Jb#+Ruw&T*g{o4gC2yy^0>b zsBkz+vt_gLkRTc-i+{CAn60JMwTb`O&}+K(5cbnio>&`&{l3Z!@S~sl&BHi)(Bef8$;{8sj5v@oTb3Yi z{hBxDW2>$juAUpTY##aQ-8XB8)%!g0O7V?`T+@ad+HkwkP|e1(9+bp`l57m-_86|X zqHd940*SpXxdswvTXF*=?zZF>NW8UVYq~W&-5P&OAjovNf{gRqtBUJa6*sRcZi^IK zGp*y9)>*tHGvx}8%=A^o^{a{-uAnEiD96@JEjZI7i=R_+4IcvuIqxkXAq(~qNC>c> zfTX@H`3xk4rxPF{JbeKYas{0N30e9!kdX6t#wC)hZU;!n>Yf7$S=}y>kk!2a60*9N zKtg|3_bZUt_=2OR4I4r?rZYRgzUONp)4qRMwe$!E9CuX7frkoj%6%X`0nim6fU&MAJby)PL6d zD)qy0qJKE74#3^|YZ+6@YF3pEZSlVy7<52Fp#u^M9gvU<^u#HE4uq4>?$d(sHV=16_}OMS`8eqC@H#o4=ME(j#E>p8 zxWprCa_N*vjH&EF;*GOG zV$C%?cjIGm3q%)#=yDLr^>fT6Ho1QmT3o_F1QH4&kWdhTgo5Z6fC$1Vhy~$o9?n2? z3qa)I3`CcL2x7?X^nyz~@xx^+H zLyJooi9kY01QJRjm$;;(6R&_o5Kh1G6oj`u!pRwJ=ztDr9ZF-mJjNp{a{+%O@PAry7rH*Nz0ht9spi>8vwdSy>42I6OV=W!fiANv#v8XPJqIXNA zp|{J8;lD*FH|~`=mC`iCcVJn@9QV0tt_CJn&zud{v3Gob6nM+&NbrAM9LuIya*ZP? zmR!TJ1R91V&@e2yhGQvw_;?(v}YBgkwpgx*&=@Oa&TYnv_fVKj5;A zIqvg-bv3Z{*Ga2p*_LAHi|V2%1}50i=3v&|G??VM&R=J;jOp#gWNnK{>3P!Gv5PP8 zHvW;|yW^HindBO$PMPEy&Lq$z7R=UI-nEw z1eQg0K@=wvXxO((>kR?BkmrQr&5hc`g%aO4Q!x1p9CpmC@t2WU8x zd$@9thA{~=jLCl@V-8Bdm@Km92pn38xKmrs4Oz_2`{5mL#)vrLq>erGjJFijs zV;;Z0Q8Uk#*mU#cHEiA8Rl|BaxCCAuYUcqOhx&PdhI6@xqX%gimq5d~EHd?=1dPif zTaRGA9@c--0iAFznY9N*@fwzEKn+{6z31Nn^NpH(zKKnjPgcj)Y|q+J$4V2EDY;R^ zrsf`?ajCfnXc(DDL&-!MN+!}!GG&pu2PL3n7A0s)X6|7v9ncBe7hzFd6h+Ad8cwDx z+1>LRnSXa{GIzm#Y5Fn<8h(|QMO#m$aU-TkN>a4!R_+zrE9Te{F}9ai#&hC3Icx< zEHHckrgkKv6H@RIX@6Wd00hU-fdU;oCKf+4 zv%VaxF+agy%h&f|)l}-ksW_mf5}a}eEEv~R__~OP>zEJAc;bx&@wAWn#M^0+V;_hl z2B62dXXUeeepxM0Q{=@%nwM!lTM-#T3qr~L`Y`gQ{{mE}A^#?x0mt08pP=~zvz;M? z0tD)6t6P&LBtivle+8UUlWZhg0q>K@Bq9Y~-p15TlkX%I0eO=JB|!oBlSd^w0cw+i yB_0RR9{0ssIJ00000000000001hflHIaB|ZV4lL96q22mpb00013Dq_U| delta 3401 zcmV-P4Yu;FOsh<=%>x9VWxg(x(E~*Vh$e|Mwv!tK8h^6Xcp{7^G#{8w(>4u~hcrs^ z#lZYNJ#F}=!9|cqK}vHnFkcBbzYhQDKU4ODF9{KbG?4RwxfG)4Sr!kMBnx^zRirGnT-)}nEJ$**w$NiAu|eZF3CS@H%Z%irHo{062pK+KCIvT# z{V0*oRDV7OBk{oe=JZC7zG)5nmFs^K^2|4`4H3*ANJ>IMB3S^_P?^ms{h=OS$RS%% zEsc9xMRmuFh{QpeiV1x_C&^+dq>+xIgf!*VuaPCPw61An!OOsGNe@XRmddcPOrnV7 zmD{l7f@Y8PBWGoxraIn=>)6g4`>^S>ZO`qf*A?@ck`>3Qy;cz(2O=2u8GSZb<$)Ko zNCh4NX(AEJa^4JSW@V}T&14TCuL{EAS(3+u$xepM1mR-F%XyY?)fDjAQ>STLf)P@+ zAC~Getn2Y;#b<=e-JDyCBSNKkFk3uQz&hZDlNsCuK?u@mk1Fdh$}pccgMPjp!w=Pa}+i zC6p!Nwf3$~N6Iu9W;EKbf-3XmlMx6XN+MX0>wqnioEs^LWukUdu2rl)->9wyEvhTo ztk0@6aWAK2mk;8KLB9S6B|*JdI&ZK zboCf_Y$ubp2r7U3zdVjd+PzBBD4sNRVVJt6CSlx*CWEGabN#vI=~|ivlU^{2C#tDG zsZ{^v;HUkEIQcytDwSz@f=Sxchgmk=tJl+TsK!BR#8WlNkM!eY9Ax?1WKd71i3)o2 zpz)|KP1C85gJ_~JGVCS0^~Uqac6Fho26kX{sOW_oDn*F&`~a z{ic2>_Kts*La!g}&-=cO)I+-VS}Ue<7stOBZ%%tn-7KVaru{r0#$}@QR6m%Fva9&v zi;4!rEMGRe2npj+I{#N2NBLSxT^k3FO}(LO4^b~07KycC)a$8Yp>-{srCB`wb#X{6 z6D&f7vy6WgrdUP$hOkUW+Qs>J8mVPWebMWDgdc+}IM`3(2Q8WRkWPc*%!oZH^JNK( z)~`i#F}C`$;p(|T%jZ$7-u*#izkXjNUMjxPkZam-LmO__8tVCYR)dmwP?C+o+#JId zS6DU~CXhIrl4~GwHzhYf;%!Q9fy7@)Hm2La(`|pS1cFSLE66y%y{fo=RdMsG;Eo=$*-@bm>p$Q5)7BxLDZKtj&n8J9@1x@{mKt9uS4WOX}0LRR+zNXY75 z0ttUv-LF96;0w;d7o5Wq*z|(q5{I0@V=ifs`F;cvGT%=?LgxFKOFXjnC(jb_f%paV zK(3L~lJM5^vF97q8Et5T@sSSBX-tR5&^yirm-u95E?>soKlt+7<)^FmY3Ja-<3!VJcR3-gxH0ghh!+6q<2Kvu> zPh~+gO7#!>^**>;f30FldCjWwq0RrdWnAAD;}VMs!?1*JSr&%Q#n8DNI&!HUafyFJ zJ`<0*gh2-+6gnWG&;bd#Ku_Ef=s-C6>^?0CZ}D)CgrBX4laGTo54XtqJa;LPAck~# z!6iOflS{WkA_v265EkENYWbI!MXK5$=W+U!#Ys5q-b=$V%vv}T(%ONSCaq$6I{j zZv)MjJ7R%lPpN5sIVP#<-V1-j%WsN*8O};(xC5Qn%YiNMv<>09{+8RLzz^IH1%JS9 z+Z0N!ap|K$py5yg4TBPB7?fPY?vg7Fgc2lRP?jWU3A&|l##-8-6Q4fvV^JLxMemkK zLvNRB!+(oVuH7pODy3umY{DDalkq2PbJIJQHvvCYLuakDecElDi@#g5E;Cqg?OTh&Dk_`$b*Kja_hJgt*3{0-!MQo!COpt(q zS(2c&I`{%6ZE1r}*n@u@7S%yf3{0@2&B1KEX)wuioxjdx71P^^$;Jkg()XpiWfx!I zZTv&QcgJmyGRZZrMVaIp&Lq$bmB|O zB#r8zC|1LAjW7je(yErp*u^e6`lgSHm?M z+cy8j7Q2by{sG&kL~@O5QzE&B6A3hoNT6Xv0*y(xUUPK>1b;JAyiP zEGf5bifmYR%oi$wA%}W)fX1b!9iZV-xYbI?wxC~Zr^e8>fC=dw>^IKwonP&In=-dG%j`U z01c;d2M-U@Fe-tDQCVi=K?xX@Wkw!BeKu#Tr42gak6)Qz2Su^^6=+!fTCrv4H7bA1 zpqKQx8hOxGb~v z2uoc^T{v9yisLAJ>SajpWVEd;sqMk!TvND zsPiBhM3Yn-seV3D)5yPoCyU=6n7_{A>HHs+t!sC2mc`@ecS9BQR8qWa=vqIHv*$O( zlB|Bi;o!6G5t#u6?w5)WOtXv=p#lQW46`p5sR19S@z_?!<5PL!pktzb3{-9jT_vlJ z-kLr8-Ym|=@ISNo7hwnlo1n5xlU*Pye=8;>2-|_SF321JF)Y}SuG^=!h5+9frtTh@c)PqOM`k~~O za^v9584E5*@JR$|Pr0qoE(A+4_IwA5`7Ang6gS2x7GmkP*fRdZwt#1O{)iO>ecgoxpt=;CatACJSC{y@h=*&L59@g1jRf)3M}6Y$w8*g!L=pqg zW8AaySv6;Op~r976D0<#U(xgos$G6BL+_+00000h~-;( diff --git a/summary_results.json b/summary_results.json index ecdde7071..ce80a332c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.800", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1034, + "Performance_Result": 10.1321, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419444, + "Performance_Result": 0.422043, "Performance_Units": "Samples/s" }, { From 6f84f549d876af4865add092bf49a852c5aaa851 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 17:20:14 +0000 Subject: [PATCH 244/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 9773 -> 9772 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d7f29041b..47b5bd696 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 11:21:13,845 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-08 11:21:13,845 submission_checker1.py:3270 INFO] --- -[2025-03-08 11:21:13,845 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 17:20:08,085 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 +[2025-03-08 17:20:08,085 submission_checker1.py:3270 INFO] --- +[2025-03-08 17:20:08,085 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 11:21:13,845 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-08 17:20:08,085 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 -[2025-03-08 11:21:13,845 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 17:20:08,085 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 11:21:13,845 submission_checker1.py:3286 INFO] --- -[2025-03-08 11:21:13,845 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-08 11:21:13,845 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 11:21:13,845 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-08 11:21:13,845 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 11:21:13,845 submission_checker1.py:3307 INFO] --- -[2025-03-08 11:21:13,845 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 11:21:14,784 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 17:20:08,085 submission_checker1.py:3286 INFO] --- +[2025-03-08 17:20:08,085 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 +[2025-03-08 17:20:08,085 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 17:20:08,085 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 +[2025-03-08 17:20:08,085 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 17:20:08,085 submission_checker1.py:3307 INFO] --- +[2025-03-08 17:20:08,085 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 17:20:09,613 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1321 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2534 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 54b472ca7..ee3d8abaa 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -871 +872 diff --git a/summary.csv b/summary.csv index b1527d213..972c7351a 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1321,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2534,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5df7e0698..59ec883bc 100644 --- a/summary.json +++ b/summary.json @@ -309,8 +309,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1321, - "Accuracy":"acc: 75.600", + "Result":10.2534, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index e11cbc310f8db753983e46a798d13d27940a419d..42a8b246eedac135342540ef046b6a8bf4fb7f7a 100644 GIT binary patch delta 1477 zcmV;$1v>hzOsq_>%>x8*&nVH8(E~*Vh$e|Mwv!tK8h^6Xcp{7^G#{8w(>4u~hcrs^ z#lZYNJ#F}=!9|cqK}vHnFkcBbzYhQDKU4ODF9{KbG?4RwxfG)4Sr!kMBnx^zRirGnT-)}nEJ$**w$NiAu|eZF3CS@H%Z%irHo{062pK+KCIvT# z{V0*oRDV7OBk{oe=JZC7zG)5nmFs^K^2|4`4H3*ANJ>IMB3S^_P?^ms{h=OS$RS%% zEsc9xMRmuFh{QpeiV1x_C&^+dq>+xIgf!*VuaPCPw61An!OOsGNe@XRmddcPOrnV7 zmD{l7f@Y8PBWGoxraIn=>)6g4`>^S>ZO`qf*A?@ck`>3Qy;cz(2O=2u8GSZb<$)Ko zNCh4NX(AEJa^4JSW@V}T&14TCuL{EAS(3+u$xepM1mR-F%XyY?)fDjAQ>STLf)P@+ zAC~Getn2Y;#b<=e-JDyC)qKx|W}{@hzPZwjd*trrZdrELhEa8?Db9CQff!e~bTy ztB?6SO4Q00r1i;xNb-fjf7VA7UT^g1+R9iMPs)@S;U^yH;kYx){i`q~cH z#EUiYVokhQ6ED`pi7Z?ve?QbRf?LBNUiTnI9z9ujnn&h3CRELMpl;ykG-jT=8 zFN2rpKE4h3wcEO zsT(VwWq(b_(-U+Cr!v8S=GV8#u|p-my*pQ9f6_bJ>ixi8w9`v_$y5J9yZ)ot6STXL z?x|6|k^E$Kgq+ZHLuK%J9)yWW++c6U_Ds+7{JlY#m~LRpN%?GYxjdg-O2ku5#--8u zbhccKuI9~j?8%ut2tvPa_rrb=cyjOQbo^u{a)=Lz>jy6JZMPq&v)Xn%_PWO7Q+eW` zW1@ZxRBj1fC999#nmzm8EY8L7KePB3VF(217XtK?T_7NTPQx$|gzr2>qfbB_xVK+$?+BTEsuH4QbvUCu!4G>WSleX1|?Xxm@g3ll+47(&!5189@nXZiLj^ z3catF=^Q0KFfCYPG*qYqj~4Y!$t~r^!K*VCT#(?C2-2Q%TcKSDmSXJr4ixiQbm%B< zj8iPc(rvMSW&E3M0T+baVFiH+78veQgF&0uwYzW;`1UN zu3xALYO7n5(E~*VuqKH!v6CAF8h_Hncp?R-EE||k(>4u~g)E}k zVqkuso;G~b;3CMPAYmC9n6HGJUx)wnpIPyPF9{KbG?4LuxfCMrSr!kMBn^0zA|~wloY2KmNFyCZ30cCcUn8Zmw61BS!OOsGNe?sq3 zNCh4NX@ZJnId6t6wbDfXrm_c+S2-!-8O>r+$WDgL1mR-F%XvzNa~XVL9>Oyf7VA7UT^g1+R9iMPs)TC;x(6_^yDR%??~lp8qrH!pGFvY zL0Lh?YwcZ~j+99-Oj)#F1yyGBXG!+=!ac9I7kUR@4EMko7&7xUMpR~3d8v*Ikg$gR zJXnzHpjgn18wrVJqIOfRRYiTiQC$m`S68xGpEDs?y1rYIAR>jjlV7(nV`6=yO7eDW zH2einO9KRxUL+c`g$M!*1zz69)J?OD6QKfs*Ys?~Q~@tQQB4IREog^f1(oP1pmiz! z>44>HDCJ|sX_C~T#?ld_ceH+!IdL#ajfKNhim#8*7P;3^tBzX zi5F|)#hQ4rCSI(G7c22XyoPoo+O-uU&!R1|jLPz{|FELiRIx%7xZdse73$P*p(D15KR3Kr5xz}%FEAc{+TTj{+k4nz68HIId`)0DG|4+NsSXdHDa&`Wy(5pG zUj{GHeS95!2;;}*^b;$ggH0q25$?EuI$b7r%Q~je{SrqVLuf}Acxje>$=*d%7xIYm zQ#V#V%l?{1??eUCo>6*poAP5QKi;?uY##@Z{dp>G;V^7^vJ5x=L0by)}FGy;+=#;eWID7hwnlMIl(MlUyJme`>=p5QOi23ZuI%%Wg{$ zwgYX+sewXMT5=Q8#u2rCkkmSP`zlW2B;ZqztC{_Fc2PCosV?~i=cUm#B3X_S(A)^A z+ZuhW7U>KnJ}@me`WlewSbE}f5Zv`6D%-%0H$^%q7zc^5ov#1Hvk03(18LSJSG{z7jFQk z{A9zRTsf5tdkd4+vpBkUUXI4T@3X!ftT8{qU(46`VbxUX!>Kr+rV^ZT2P_!ZRQS4x zhwGRR%Xs3A1o56)o&m?)x1XT-1GAnXgaQQWYO7n5CM3lLUf#ykO_T5>R{{8wNF_P}YLkN{ xCL2W|Sga)h002+|000pH0000000000004l2OOwPUJ^`PT112K|Q6m5V006RO&)xt4 diff --git a/summary_results.json b/summary_results.json index ce80a332c..8fa8555cd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1321, + "Performance_Result": 10.2534, "Performance_Units": "Samples/s" }, { From 43631fd210c9862128e704e1b949c1cd6035c5f5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 18:14:26 +0000 Subject: [PATCH 245/719] Updated results summary [skip ci] --- README.md | 35 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 5 +++-- summary.json | 40 ++++++++++++++++++++++++++++++++++----- summary.xlsx | Bin 9772 -> 9872 bytes summary_results.json | 44 +++++++++++++++++++++++++++++++++++++++---- 6 files changed, 97 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 47b5bd696..7bfc3c878 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 17:20:08,085 submission_checker1.py:3263 INFO] Results=19, NoResults=0, Power Results=0 -[2025-03-08 17:20:08,085 submission_checker1.py:3270 INFO] --- -[2025-03-08 17:20:08,085 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 18:14:21,924 submission_checker1.py:3263 INFO] Results=20, NoResults=0, Power Results=0 +[2025-03-08 18:14:21,924 submission_checker1.py:3270 INFO] --- +[2025-03-08 18:14:21,924 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 17:20:08,085 submission_checker1.py:3276 INFO] Open Results=19, Open Power Results=0 +[2025-03-08 18:14:21,924 submission_checker1.py:3276 INFO] Open Results=20, Open Power Results=0 -[2025-03-08 17:20:08,085 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 18:14:21,924 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 17:20:08,085 submission_checker1.py:3286 INFO] --- -[2025-03-08 17:20:08,085 submission_checker1.py:3288 INFO] Systems=14, Power Systems=0 -[2025-03-08 17:20:08,085 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 17:20:08,085 submission_checker1.py:3297 INFO] Open Systems=14, Open Power Systems=0 -[2025-03-08 17:20:08,085 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 17:20:08,085 submission_checker1.py:3307 INFO] --- -[2025-03-08 17:20:08,085 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 17:20:09,613 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 18:14:21,924 submission_checker1.py:3286 INFO] --- +[2025-03-08 18:14:21,925 submission_checker1.py:3288 INFO] Systems=15, Power Systems=0 +[2025-03-08 18:14:21,925 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 18:14:21,925 submission_checker1.py:3297 INFO] Open Systems=15, Open Power Systems=0 +[2025-03-08 18:14:21,925 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 18:14:21,925 submission_checker1.py:3307 INFO] --- +[2025-03-08 18:14:21,925 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 18:14:22,912 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8103 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414229 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436315 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414229 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index ee3d8abaa..d6cf73b61 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -872 +873 diff --git a/summary.csv b/summary.csv index 972c7351a..c6148bf33 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8103,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.414229,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436315,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.414229,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 59ec883bc..e0f2eb72b 100644 --- a/summary.json +++ b/summary.json @@ -455,11 +455,41 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.436315, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "16":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8103, + "Result":22.948, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -479,12 +509,12 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 42a8b246eedac135342540ef046b6a8bf4fb7f7a..8d87cbfbf6059b860b8f0af59bc7cd2b6b262773 100644 GIT binary patch delta 4080 zcmV<}7MU#IBAb;I++DI0~-}kRjsq&CijX^EYm&^8) z<9r!o>^Qck_Q|C{1F8YUNWyXc`;h=+xaS~~Y&(a# zOn=ftbC_k*{dzs^4)r)pYw=W1@-6*18HZW^H5t^?X`;j4JZL_P;_hs$C)pxNqDNtt4?i77)6_gV=tcRqq8=mB{X_Fu?VtQ%+h+afVBYs_ zq`#$WpN*oFuW|fm@#VC4Xj+A|4%!d%VSijEMo;&{*(ke;zkSltV3_5*%_%~<@hF}D zG{#ZBFKyEphYyEl!!*7{y=+({Hil8Jr;8nJ8r@l%#p6E~o76JFB2>A{SY@eIbWl@G z%e9^2czhq}WlVk1>wJXo!Yn*GNaAlsGVdXshQ*Ol`+4i*YwJgU*r4Ft1*ui5e&VS*zL?Vv+36Y2$_Krvdusv%3j^7zdY_WISL?ZU?IgyCH+aVIMcQ1%U?A=Qu5qtMDk+}SXbNLD9 zLIOvga42!b;W3{h&?*7rIe_y`8YM*w_UeD)3X+9YmMA>lmwbqTt_3o`h_nNNCm@6@b%mS{bnW7UhPRQx5|9iolf_<(|_4sJefQsvq_fErS_JpyKX$`M+5UWv!}B#8l~o&gL$Qa)$Z`OxP7Z5h{>#kkaBhEZ$Uw$*HK=sXUchtP?$*f~NaSF6 z7j}!gnZ5kxWsz!ckn=eGp2bObcz18t+?uu5ohiGy1Bq3XvIB`X-WrJwzZUp`!$I^p zh(3ZS&Y!m^am9f?K?#6JBoag-ksuO@1ko=6k%UVSOTt?i4j}p^AYwRx=t~eujM(`X zC{dQkd+Go0kXS`2JAaUP>u+2W8%^75ZkdP%9K`@p6cD{di6@3}f)YTHNF<6xB2h$% zFYkPU5)?_eoQRf$w*mpmnQUld2QU$pF&z>kCi)jBQI^Pi8B|N#IPAvh-YA@Bc<|j{ zJu9uEyd4O<3f2hpYt5$Rs9^Pvqo^#-;=EnJC>VK5G@g78L4P!WBhyG8nMMLhG`_qz zt~3lumOwIDmY}uTxS|qT+N6^}Tqw^WszajW6UPgpQI<%3xip zlT_c`tD2u*7Jrz|N-Ep|&+FyD7JNE2<$1xD3#8)D+>aGLV|N@$C2BldQc0fG1C>Mr zR1ytPi5j?0u75O?N|FGmEJ@H(a#i6BTH2(OpdqQ0Q5_ZqRH6nim}}#IiBzs#EekAd ztEPT}%PMNz<+8aPm|8w}uKAw3D>;rUxkQcEkX)h$xg;9kl4yWS)F793 z8J8pha9NU|wc7Z^rEF=FPLN9()ge*vK$U2e)sS4;|9=NuR#D?F4_TK3TisDQ4aZem zz~l?0#|qaS9Z!Nu)+U=0Ow=HlL<3+F4S7lHOtPwtU^ZSfnBvjSk26_Cc{?%L*kIBQ0^8fNlCStu{;|Rpa>tiUqQ-M1lc+%^ zi3XS?8h>DttowVC$z~aoq!VDWBtdJn@rg;<(k7h*P06H;>aZx7#G*!7aO5zx43c?@!UvT4$*khmqRqjFNEFUoQA1{~RoZm^5hh=%X-5gyj%wQWwq=nG+m7`_B{Ahl z-+vC#c+$8-G)UzR)*Yq+Dv1W>u4V2WmH?heZLF%zw566WEGLD5ga|c@w(*T%617MapdsqShv&`G0z}y2ZZPE#X3H}}u zg|k@Z@PS!u#TK7`2Fw>~^!cLJoj&p&dP$)?!0A-oOXSMMON^tm)P6Ag#2@W3?CGRka2JbM}#{VLr z%=!h7>ZhLgR8re6H&_(=YyI}qQ7SP^{ehHcmGw_ee;`PzcCJt zy!1zDUfsNx;sYDj(ZMtv=sEK+8AOxR80mgqs8#b!BU!v0Wd1per}IyaX?%^dEFM38 z8|tv9lj2*$H2QIzJ$@;6$?7F5M}Gse?-7^*3GiR7U0MVH0PvHL6B~cMSKD&pI1qi` zuTZfsGY`H<_?jY2rGQLPlM4mOR`!V!cAI#|x3%O9`TLOr$uO2gl1;Kt6sM&=>QfajS-Y}>PPnif42l}*8qd&Cng zO_BcFz-WtL+wBbpEFBK`@Vty68(4feuv%aXE+Qysot9nvx=1SVB@d4wt7t^XO~Hmr3G~1tMWS(%#u#Dpm~3lfx<8yDCCgb zjDWFH@?ZTaF-8$&P*W~1rqbd?neZR9mXdl=a`_O~3}R)a2+F3WkrdzpnIe5_rE{yJip&=a**^n68f|%Iq-IN|8dw2R)zrY#f+`FvsJTKw>x%!QfW>Pl%EyS)Bx%}L z)7VHYXQzKhgS#4BDytptYw$pWOGn<~Sjy85*Yq{4>1$Z(Ydc&MFV@71HSuCiyjT-2 z*2GJscp+Xxw?XaNjge>37FkAB_0X5fqS(|z_-TI*Qd82El8QmqZr5_VjT%fGOH*`j z_WOdW)o|oAD%pPber2XwDY`GZ3KiM!Pelw+v5M9VmUq|;PtaF56$u73zq&z=9ZCW2-MJk5qr}iwVh8r3U5R#=JhnWvYkAMPz%mO@(C*UI zGp>AT`N`}EIicx>(wO!%wiA=M!QK~vXL@3u=kJZ}#B>8&%=KR<7t6EBg+M%MxEq~K zXUoOta^BAJp7??Yg3#~V{jeVdp4fXj9Y30h_~ZzP>jy6JZMPrDljU}p_f*6DsW@?< znExLF6yi!na@3A6VXVF&~WPCsdrTp%BR%WA_g5JmU>3ZuO(+i8mt zwgYX+s)0gNTCx+;BoVb9NNSz@eHFiwfLC3P?wvDt29?XBY@%PVPH0u3EJ;xWifJy? zzCs_HRa~IRd8#;Vw1Ntqz@cS*S29DGw(w@P0qX_0D1ekB%v9*$y&)Job^u9T5*#Xs zJ8dQPp|pE{Y$*Gtd%$^`K4J-;a_T8Q17kB0(SYP^LYg1jbO6pUG$28FhqENZ7q15^ z-E2drTsfuiCj+zAQJmaICnjUtwn@9_tRa7fzt*qs-Kw!r=Tk92b;%iF7O2;@F7b5{ z4%blU*WtuF0sQF{^qIF|k-iUj6atVw?x=h&o?kX4D^%xc{t%~&I4w3=L9%<2=iBqh zoBj(>4nzJ;907gq+fPvc0kfVV$^r`4wYAKs2><}7MU!$QRsuy5lgcC^8}MJPU0MVH z0Pqq302BZK00000000000DytdlLjS00$Ui9NhLV}&69&ACL0J&KWQZa002+|000pH i0000000000004l2u9L(iJ^}(FlLRIr2DBpp0001|?zsm5 delta 3995 zcmZ9PcQoAF7RU8Y7=q}Lnb8FqGGz1;Mu-+Yf>FbW3<;wAwCIA2o`^1l=q01OIuTuz zL?@$n?l6hx-uKU2Z?ARse`oK{`R=mTNz^NQQ2By}&I6!)Z$wK%Lal}cXaoQ&uUS*J z*9#Y7uuKH_jwYUGWC(?y20xFJe#tYJr6gSq4=xHW1_uz%QbApAl$&BejSoPjRoPCR54D!WN8d&Mmb#ZZd_Q(FCzn4u zY^uTtKe@h?w>M{?2->m~2mfu7D-fCf=q(yd#cF34a zmzvwfyr^}KYIM~VQ|5n&s*5R+^nV<;pPzj`)CqqzvaETn|LgE@96V9RDplaw@65Zu zP^INuRzxnu?m~ znONfV-=Q@Jl+HC2>@AC5x}eUL-4I!sg#;C~^$S|6UEzeXNT~c=Yy>l+KGs113`${vGb7v`$AzSe8=Hkt>inX_S)ki9=&t6+Qt>U5K7^ zKFTtN#WXAI#*!i$paddTxn%k%CumEslwC)JOAc z=s0fQmO(V0CY0+8rqA<{*!XdiG-|bYSpH>o8L7ht)p|avZw}`8OL_$Epe>PO2NPrj zx>;O>6I52-B81!+1woJ;ZFY!D#kKNW2MOMh==*yi;-BdA@JBuL$r&U7PMX*dvOdTw z-CJETA>PEZqH(hOddcZ-cN>GV$ekZy-?k>(Q++KXJb9k=?iemo@7@npwrvongM5b{ zj|z-<3Wr3f(!v;!<#s;(byRA!ASq#h9#l*D(+H0GHyuBrs#r26&quc)w_z}TsZCS( z?rw^F${4K6$JU~U=M3jsP6ag`e$V*5$|nF|^BvsK-ha$9SY(cgza^4t-C0G!6#*HM z630cLL}Du`XgO0UX=k^>Zkvy#GU3BfI{A$PF^Uf;x$H8NbqUIg)p7E`?osdy*~*je zI;BoC_SEUVUdP(|iuVU|Z9@b5xhS!2@N|{Vb_y;k==XiWdQ%Fhc6+JPnZfiJ-Ti?H4jT)#eI9?TphQu)9rwc)!FbHo2iPuy)XACX|ZV~RX1am z>!bda`vxA7f`&v$(*mc&4fZoj@(hZs5yjp-ik*`V4ydaxwfC~l066|CvS}wKUW(GO z+C~z4)U<61f1;`r{9sQgw#hN*9ovVSZRGtjP7P)R%LVZ#jJ;($vqX-2+tAPpYVib6 zzw8^$=v6NLt6YL15iGPYCdinmAP8)$!4AHvaVxWql>(5%dPn(^5i@(7NT~6WBtEi60h)Kas7fQCS-fM zc=bBX`jFjK6=|K{&UmS?RaoPwOv)WCz5iSnUif5ppSkxOAE{!pdmOwpX@RsAgMTvkY~u=GywYaV&t-$gWiU*$BB zFE>1e0b+R{_lrI2SiOrkS&x3b!EGIsHiaP;CiSqs~|N zzEIfkKK(_?6?yI#9N~qF%%^tcOjGInA}({1LlHM9Xq8~w$~QGen&l~(QAe$Dz?JHz z!e~|9@@s+gC_XcwHCiw2p=s}w`|zw`FOn52f-~$YTE8MJ!BM+#W@xMTYj7a}=$L_w zV-k)UlH+hSdAd%!|7tkLVQ2W5O|zeZOTS2b=I0St2H=A#NG9x%7s48HVTBvJz6@gB zOv3*g5}xdz?w@!yVLj?Y`0BnNVG4Ez>>2*xJ7<+w-QQJeD4c<#)*O{#cXhcTrYm>W z+~c6o2Rvt6IQd+ijN3788PGFMV2jwWsD1U>gP3)p(XBpdwDLTU(y-oou`PMpkRVqxC#pNy2{^_++be`VL!sX&)qAOn?zl<@0Uy^E1?YWEY*onZz8~ zTKCQ2G543THisN8=0lws$|k>p5DXuq&)hiR^ZHm3l#>APn`Gyl9t7;%VkEXT|M zM{SgA4;_40kz_BB`;YXZwE(VevA5;k&4F8k%cH!j$?wX-)ht1#YbJedle5c?l(2Ol z62W0{`?(J(F+~)T1N-kS^NkL(k zlxn#k6ay3A&+gxR2uQRoad zgclt$%Jq1)NOmaqN@Pz>{;15J#q%56w-dSmbVB`y zXVkqEL6NN(Eo3l?S>{#uu=cqNs@#>k9*ttuhxxX}_nH`#SoyJ<8bhYK2~lhESTc(R z30JTKBiD7XK|hG%mZ)2PKw?JBZt7XraltqzNHnE6&OE8WZ7U~KIdj`UlV|mlS#^Qi zt^C8OE)1*hE6e+ZJEC%plW2fy7_oERvh!%Feo0=0mkL5w_|y?UlSVM}rY~~|0?w(P ztDTv$taNkgIROcYd;Wo|@;y_EQoq*~fL-vNvNX1yE8(ogov{SWbBy%8Y*-VzWlKqp7v$S!jkRQ_2eR~sry)i0 zBbmQCu4SB~laSN(-0&yp0ma%C<}u&!0E?&f_Md_|a*B({lGtlID?^dqY$J_qm)`A% zvdM?i%P^hI^+}iOlA0L#v7Op@~`yM>k`VHz+ubQiT)bY(&wjp`J|=9YdgpRDJa zOoyjxS?`@*(7kvH$8brvE)*`LmlUm9YuZ;pE`IAoks@uzVmc4`-i4ftx*$(4t9D*b zj6rKO(W^F!wGiA20P>0}-!6CA(kzK|;g`))Tdh59I z^&#)={tN_-THCX;&LC!A%-)7K(E*Byx)!xXEqQ=$;4iM3eTg`2>{&7PTh2k{1u>X- z&_0HgCd_9lUd)Ss>)uy3&?Tq%XVK7H9c7Z6SvI^iXwHW;6nGA@V{E$sKQN@r@PzFvRInjWCV^n^K`_EZ+DOWSP&hrVmj+<^que@?<%X*Bn-hPB7h8~UK z>DF$j(zo4;7udX z0p3bm>&MGSf_&w6N@dH@cR^h!juWwGk(xb9%sLDzyo^J_I^f&*m#`)8g4o09FZU?m|px&MEqUkUxM w1&8=ypF&j0&N#52A@ZcI*k2Ine`RNZvXdgQ!cZl$4>z$cP(dn#TmRhn54pmNqW}N^ diff --git a/summary_results.json b/summary_results.json index 8fa8555cd..8011eb3e6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -437,7 +437,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8103, + "Performance_Result": 22.948, "Performance_Units": "Samples/s" }, { @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -618,6 +618,42 @@ "Category": "open", "Suite": "edge", "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 76.951", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline", + "Software": "onnxruntime v1.21.0", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.436315, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0019", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -648,7 +684,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0019", + "ID": "5.0-0020", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From 3ca02b057e3bcbafcd1d5e198a034d0d6536327f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 18:17:11 +0000 Subject: [PATCH 246/719] Updated results summary [skip ci] --- README.md | 50 +++++++++++----------- dbversion | 2 +- summary.csv | 8 ++-- summary.json | 86 +++++++++++++++++++++++++++++++------ summary.xlsx | Bin 9872 -> 10059 bytes summary_results.json | 98 +++++++++++++++++++++++++++++++++++++------ 6 files changed, 190 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 7bfc3c878..5e04c9f41 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 18:14:21,924 submission_checker1.py:3263 INFO] Results=20, NoResults=0, Power Results=0 -[2025-03-08 18:14:21,924 submission_checker1.py:3270 INFO] --- -[2025-03-08 18:14:21,924 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 18:17:05,305 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 18:17:05,305 submission_checker1.py:3270 INFO] --- +[2025-03-08 18:17:05,305 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 18:14:21,924 submission_checker1.py:3276 INFO] Open Results=20, Open Power Results=0 +[2025-03-08 18:17:05,305 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 18:14:21,924 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 18:17:05,305 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 18:14:21,924 submission_checker1.py:3286 INFO] --- -[2025-03-08 18:14:21,925 submission_checker1.py:3288 INFO] Systems=15, Power Systems=0 -[2025-03-08 18:14:21,925 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 18:14:21,925 submission_checker1.py:3297 INFO] Open Systems=15, Open Power Systems=0 -[2025-03-08 18:14:21,925 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 18:14:21,925 submission_checker1.py:3307 INFO] --- -[2025-03-08 18:14:21,925 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 18:14:22,912 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 18:17:05,305 submission_checker1.py:3286 INFO] --- +[2025-03-08 18:17:05,305 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 18:17:05,305 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 18:17:05,305 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 18:17:05,305 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 18:17:05,305 submission_checker1.py:3307 INFO] --- +[2025-03-08 18:17:05,305 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 18:17:06,834 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,15 +29,17 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0953 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9496 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2534 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436315 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414229 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2095 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0168 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436315 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414229 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d6cf73b61..ac44007bb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -873 +874 diff --git a/summary.csv b/summary.csv index c6148bf33..44c274256 100644 --- a/summary.csv +++ b/summary.csv @@ -7,14 +7,16 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0953,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9496,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2534,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2095,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436315,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0168,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436315,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.414229,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e0f2eb72b..6f8ee7e2d 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0953, + "Result":20.9496, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -300,6 +300,36 @@ "weight_data_types":"fp32" }, "10":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.437693, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -309,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2534, - "Accuracy":"acc: 75.000", + "Result":10.2095, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,12 +479,42 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":23.0168, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "17":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -479,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +599,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +629,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "19":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 8d87cbfbf6059b860b8f0af59bc7cd2b6b262773..0cfba8ca6d13208a8e2460c53b64d967823a974c 100644 GIT binary patch delta 5506 zcmZ8lcQoA3+FwiXBUr0MiC`_!Tl9#Kuyzr>iy(SgJ)(bu1gqOcw20n<=&M9alpuOU z5Jc~NMT>al-t)fqo_GG3Gjq<&XJ($~^UQqaNqE$+Svf#V;Xm%z>p=nn`9pC4IWumL zm<%v;%afrGp0fziW;b<&3Zuwlu`z|~E$c!9u60_pM0CR-JD0KkHg;C5rdMD(^R{|H z5UM?Ju5o@y!T02ddno4iM_;w*#PwSA7>5cnuhuyBOWSIZ(40gAc@*zw2PMRc!2HU@ zzQrf|zShH+5Gy3{HTE2#yJ`<=R;y?5!H&H_x!LaKW@_G=_}}+GR1Uq9cJ7~l|L(#y!EZ( z64iSLc#e*=JXI8jQb2pbso{MtsP*SZdk?55qnsRnjLDVWwNr*umuh@jJCsEk0hk1Y ze-v~yW}*#|)i>Hym`EnVRl{{JKM|Q^#f8d?q_3NAKefUOuiwwp@!yF`KydCm7#9wEyz0NlIfu#TYa08cb{+9L3qBgfm zQ?(Wchy8WKCnCv^X>F{P`FQEqfbh=fxcv9&TXu4}luw6gThN`4Rejwe%X#nmCb7lt zzTkbP%&vTD)){!?sX(9bEnB0-W)3yB`7mA}I(0Mhp?u#hmSud_6ZO5mcnEp%F-gb^ zn|NCC)Rx=iBpMZZBw)JPGsf7aKJ=!dkRu|8n zvLRl!xo$IH|FVB3)9jrM?aR_~hhE`av2lw2QiE2CzK=HhOZ6@FGp4(Lq7xWBn#9{u zX=#J7{+=tbDM?V0E*UnCfF9_4%Og~R8? z)Hz`;B0j?+LB(>wziehzL?SCKMPQG};m*afiqK9ZAduP> zvBHa5IY09M!Fp}X^)cmYhu}SN@;parR2$-xzkGbCE5I!A=4+j;?2+Sh`8!6ETdKAj z4xw3Du)oCLZ4a{8`h_jjDF! z;b+9|pJET~wjur}0t)>J&L-|$l&*M+4M#xg&FBX`mNU!kU-J|UkIcMr$V~HI$;5_c zEGul^ha0cF;0C|DF6G-qRji2d*5N!mJvHe%s)@?5XTYmOz0t^!ALHqCPUPbcOkA&t z+O%hoE>5^zR~+|71Dy`6SH-UCKmO!~QO-Q3gV7TIibIlAw$Y-zj1IDpBwFl|%6o)F zWG92S8vN;+nKEo}j#;giX+|nBa^&7^s8|ylj5aSo3I5bu1fv|Ya~tMya&2{waxjF# z#jhi$la@Fj6dV5em@P3f5a5C)?gk|JF36qi@p867?qeC1`GKNq^W{C(@pNC)1Rbo2 zTaojRyS+sbgEBXGby6Bugx|_3$ef+*_Rm#J6}_pM=K`h0PcBXab})crMyhvyBER*fEydIDy5XFJ_ME0@DqxO!vKv^02D!K6o3 zDO1>L;mlvC1`m{+R!G0yJz9xwahD2+WUPUkYiySrJiqEO(F|C82~P0t!F><((q>Os zKhtC2D~SHau|0LrmKhxM;vHw#09Z7MyE)j6!_M(obcvw;$mfRmJ92@!^g8t#u(hH}|$Q6}-K z1HuI#0BVduzHVrznVY??i$vY@7n7*7n=0%&^pF-6jDT=8T@olt8K!fE2~(5zunpd$ z)b4b*ARN|md|0N`9<)))kl=WoD_$5T)&~``5REN98GX-)cf{Z54#4eHhm#GRPHv9! zj|znROm2IXZR9K^WOkYXWUAPDFuewrLlH#1X1N|+l7SB-+PTJ%9BBwBN}`o1mxh(f z8f(CjBUqYUC)>~gRhufK6^nTLP(MPsHzMwaa-N{h(X!#C1!;yB?rFE}ee#v{Bc`s7 znU+$&^$#aA-iBDKf8^g!u$k>UP9$xf$==Cdh@XjPP_Vs(xJVVlJrs~7(zm$_afu)#@*>)GyJ8h7) zm%)0(4_BJ3oi4T-R~j`3&GQ$`w0bC>d%38FVpt|pEOSXa;4C1V14!d5M`{zfMj5E% z3cF7Bgrrc=u?7LsU>@fLboR899g>#SgpV$XUK=bKugGpTS!V>OHCx@~C(~#}`Ioux zj`(n=+5n@8|Bd)lu3m?Prv%jg-;nbE6vk{M@RH!RT)0wWFZo2rY07QheXgQUjb#(f zpqkkC@=>Ip8@q4VVBi`6)%e;_0g%v1lJks!Qk97Wk zR%1#<1B=M#%PtQ6w(^q>SDioLzpl<>d*#N&X?1|-0G{Mir!Cx??=}DQQ2~c|<6GgP zewQr$;76AroPmweM)&A!YgJjJA=gk=p674bs0O!ur3i<|>%*KZ%GEDbBOF1fh0qab zFD!%FnIu6_e1q40LlKp6r0bfXc(n*EJdHf-MBnjyBJvMgGdvk?9My-ujQb`D^#7=$ z1eW81di+D71MG2xT}bS0DtYBF?Uc$rQ7T20iv-9vF*-sLtj&1pn^ZfQsdR-3%_eE4 z#(b<3xU~bXE2Zjz_HThSJ z*O&L8CRQtw@sbOxmY8r_)ieLdZd~`RTqQwuC&4};l z;zJ2Vo-w3&8UouZfz>cy+P7jribwN1IPRTX*-`>*^We&q9pWht?#%B5ex3X_YA=KL zimRKnpbvYcRBz4=n)Ep(Kjbk*++088WYIbcj3n(9oIkW;;lRl_EKwB-ADmVqUVMSQ zjVFCea}`-4%N5+pSCIt_rs>01HO#AqRs%+U=LGeuVFmBlZ=mzyUB3!0FUg;c$VF&K z#LA4WJ(S3jID=4OO+K!Dy4q~jFept0!Tyar!Ni6zc{3137o$-2T9Jn-2uVMfW*ehm zc5z7-cB=bE!GB^MH<(71?De*URn{igxie9dJ$FBUEW#n-3lz{~FC-GjkQ&ygXtEm; zC73J`Mt%}Puu_gB*RMgwt#lOan2X{ozONtNOuH<}%f|jzQ{sE9kW&94b6b7pO9+&W zE6)4$ia6#9e~IIM`jGfCsvB{+Ka|$nXx7Xj&&iL?gnPp^?wLWpYDp zd#>UM_z12wO=*AO@yqDlOz%drL9efr#KL_ekx!o0udPf>Y+E|UpF){>@qX?Oq%?$bh=h)b(Vf2tI*_--o+&DL_$q1jJ$sowF0R>0*^wO;h($&kw4nZa&y&1LE8htsZM zx0r`qk1ll5?eH$#=j~0;yhV4Z!wy{~ZVo-QhF%|4E>Cx68QJJE?H@wPSOix;T2xPk z&lC2idDni0@t&vrX=QzBb#TJXiXEWri~I9`67=5+bk!5QXifwI*?>Wy8VhJJ5fHUk z9p-%XS^le2|4LO~7ZEQ9D^FV+J#SA(S9>pEU$je|(X{7+3?yjoy>^wz9hz3;PqQe| zWH2_D=LQ9~e2b2IOKLw3?I)DR3AA}~Fn3oYt2wpki^>(`%6C$lF44FG2P$qYfx*P~N`k1f|YhJOOAFXEZEA>g=NO%hXu-!Kxhj{#`@bXAV!}99iqp)>N7jzIcqP zzIGb^0AALlR`#x+vtOKt&gQ5xJ10TXAqn!3PaP6dSsb&VgWwM|F2k&=714I;+a17c z{4&eR!k7bHsvk0VB)l{2!&by3_36=$P%;R<_ID}KMbqC?b$x1 zS6-9R94)4xBrJ6=!}4Vo$TPGli@D)RC9yC!B-Q*z0Qt;pAiKrTjj??fvHQ*GPvSPm z>t~-4_O8eWCpYPw?HrIGK~aLOa( z;y(wE^`6{T3nR@6!%A{T!i$8$xs!4WUKqYQDY}+|&(UD#Qd?*!b$eu;SDR^maV*i0 z*O6R1HJ?R#EZI=KEEJb4v+e%K;!}EScyep~CrSS*gvd8wt&63_IPZot0UG-xcSN1O z1yY_pIqb9D`+3woTJU48?}oMDkaU3yJ&II~Zmi%5>qE1ECnejmWuogQ>F|rnIcb~5 zaIQSg#q*gzZxTz@Cv@U=%5B;#glH8zC`~<$%c|Dj;iC%)m+aG7W$AI>dHy{1bAtt1 zthC^0@sGa)J+|_{dHqBt-K71(P-Qi@gX%*no^*G01@x9@FChQQo+zZh%}0V=WeDnXV)Qw&|sZ!O3Bz1R`W z3T5A|Ywy)ceaxg9Is=^bb`D0gFVvupXp z6jpKYlkJISVf5*qo#BvlMP_N>{k9w;~fr%_F#E>1VNA#^s)-Ghor>XU9D)^SW z%d>Xb*GzYRnc3yu&hbS3V`JMB}ZNq=+#lK6t@bS@UlrQEvy|!8~ z!q9go6gqsBJb2FFKUX#i=2j3DGPah`Un9RIwAqX`>x&IDvKMUN=(tUre{Z?|!8t^* z>-*t7-j$lQVVxf=MaGc@(|}d~iS#s47HxGq1(;4yG!R_q1JAw1cM@59zhWH}YVDvH zLVau1Bu-=&qa229moDetc=Kp^I{xiyt9_P%h2xer)m2=C2BUy24+}!_m2`vvJ*pU;*$8(RU P1!vF0Md}9sJMDh}kGNal delta 5287 zcmYLNcQhOB_l}5F1hF>}N=vDximDnxVpD35YK<6G<6{%6DrN{VTC}NErS{&nN9|F9 zeyY@}y=(p2-}n4}-}~1+=e_57&wcLm-gBOJKo_S|_J#b~i(%1GPY?hQ0Ktb)bK-Z% zsW7DHi}EbN4{ey#f}QfhEJJZnTX^2_7`MIdKiT?7o@f#5otp81Z;4q-R5iXC&ek`w z*pY9lk0h**63%RFDkl7(ao6Ge>AK@G^vSQ_ntNSs=ZX*9PV(E;a2m>*UeZ!FH1}G; zS8ER+9YsjeFh#*}`(gDg#8opkhBO$h9|q3`OVhLKuTbi>s44SnM=~1cidS2bj*7*R zo|QEtPx;H0f5_7G-KHOiTO!JQMsiVeD%yxp+3o#pChq%wm z)~eiwHw<Gm}|+ikcrY@lioJjyQv5Oqavh!vYL+hpX*VRqh6qMj5+KIonq7Nj>8$ zO4sXn{4KyR$|RZ!nem*?bd2;`3&ZZa=FIoYje(hWl=Aj!^pK2Qkxz$9(Eg`Wkk6hfe$PUqRM>A>gBmmu7^$RG z(L)UyTwn$PMW)4oWR)9qVSXf>7YMj;NdY9wYefjWBH?cGE{3lsFi{s`w(bJSCYzN7 z-Zs=ygN%;BVWH~_LO9>fjql7ufU3}pG1#|I$g)ng}Q<< zOUPR){+7>(ou(6E|R5 z@MkE~fad}o+QI(A`4`MH27%IRpBK%Ndm{UEqXt3W?Yam22`?sDc4i(Gpe@3g46FGG z>(a#=yG~t>kpuH@EMk|%sf{g0aaIVoA|^Kb_kOG?F9&Auq=QVKr`2b}Dw#cfpU>^o z5g=6mmxvZtq1U7f$0gXA98~q=%EuKRBg1e{zBu{mA-dQ z(l{03cw`=uwOJLsy^~ck)MLD|QAAGvISGsod+8IY^md(*+UO5zk*a{LqvF1AA^2T{ ztK=Wnf(zv_vQksLRZj{)a`)t^#9@D6zp|a?d~EQv{z-K0WkB06b{#LU+n%|DPB_~4;In@0EKFQHj@Bm-fI@g*7fbURR6;s&M014VkB+gDCI{G=hI)Z zH3dg~c}nC(WSz**9+ziHfTwjU0(e)SrP(hLh+|t#Tv!=f>*vqVq8B_I2 zw;#U{w86QYa!3mZy52z;Cm29^tJ#J1;^e zCbt*2T#K*IKv6`ECj^Fvyz1i<3i7LJr!_=Aq0tM~ebddH^Cu*Si&fi&W4GMaJeQn! zVoIsUx#R9~m9LZe%PR0J!~PSDrHH!TamS2kV@zxC*r;v#a^n{tSfl8L5rl@ zIL9l7R}LNU7PV$qQjGRcQhV|-&Aylk z*}`w^*t8f)=_PZSx!x#JbyR?2_T*bvMN#f&R0&8-(ZRf2^VBqE@GzY(xSI(cD8 z`G&=xs&AZ{uI0WY1oUW2E-AX1BnT<@m5fIGn%9Cd1s>l?symPLI;B}r0cvEw5cTN# zE~kuAoj6NCBAU0ch>0^Y;g;ULT?3U6!4T+!Xs@LzSkqx0W^)(#j|lhdS7iY`e$MfB z4*#xt0d&_I<`lklIa+|RX=!!N@fornUQt0Sa&jSNUK~|V#rGWUieAYLU6(&C_@MjS zcQm&YAJmY)EuH1yyY_2CRx>t3FTi(uqBwxy^dRTyWb6l{i>Ba;I6d#oVcW6ao;NCM z`-7&lbdMXxF*DW*849pU6k#~;h)~q{TwzUveE(i$L8TN|-p|(&0Xdc>4^0dV6&)u} zM}A(paZaao)lJw%8Mw^WOuD{URRCwb8o^dBNoZPnbn58?N0_jY2}qbX0;)TbXieh{ zvrO?chb(;FQR2Z>9=_Q9SQr0Yhbd$W1x3Yi>{4)IQcrX|*Nl-%wtnnBv5gtc=RfU} zoZWHzvKQH1C|^Hc_vyLJ>gg=8#(HwBaY_X__TB;)mcEtkB&_7B@( zipn&hNEieGO%i}$L8FpnNs(HNTAwXAzyiO81?*%)SG5>XabHi_MGXkOfooUWI8!yT z`xuOd5{rAhV^oDM}$k~Sl}t)8tTLd<7# z@k*AW!-7Ss#3kk)TA~2~8C)-Onb!Vu{BHTb6$OXzhx~Y>Z}T5Bb8Of!=|h?~=E#OO zmfP5Z%KmMiLc(#;ipKb>Yb zF7p}g+7~>1c~dUoW<4T)8fQtQtcP*^o9q;-H6f7Z!6YnlgV6v6lR+hu-Pfh;wO)Gz8N39FI*0)6ilTU18Ebg@;^51WG`Da$oo-}P@w-*dws4Rk|Oowq{Y?F)60+ah3IPtm6wiPgC zm+DVpYY#xW0$4=+iYC+`0s*0CM8TC{&OtViwG<|lJQ-9A zQuUZSCmNF?;WkF25c#qY91DbI=utnvnZ=G+!u&wF3R7%>@+v9w+@`~;mw)Jp)Yf^j z?9L%Az@vU+X{zwvCGZeWKA8yi9#y*dos}}(&_3zF1Rw@nRfW3V3p4dbvJ6=os5*Qy zp7p+D60s>lYtT$5IL{u{*eB7G5)Ki@N)nUS(ve_NEn z*m&0#1-bNQ{h}$$=h}Na;Gx2!0D`?W9|2G%?zJz9uK5pOPHQ;&l^Pjg5K}|W{52-{ z{_vAi7rwR@eU0H%^V)+YylLy2yGMPIEp7TLn@3*3E2kjzx*!+Gi^_sw9Pn2T1i3&{ zAfR%P^uBG5{4a;$G?#pkmzAbJcf!nEyV!Ctl!NTf2dBiTEX7mpn(vfDgp6W04{0}e zY~9$us_+*02_rqO4+Bru&QE$K9SJM9I6#;)F^H#eOkWgr9bli);aI&(qxs?lrwcSup#k%=32#eb zDYvDjyRV?{27gl|t`}sV7#Wgo-6I?|R2HO@ffAa^i%XlM?TYbXilgrhoV|Yp`AR@d zN-(jtUX~v&NL{hw*-QSI-CTB7OdxXtTj_i+v9AqzW!^N>Az{7xgtf2X;^PdGd*=9q zBK)ue{!m4&JbyHBRnmI-i8fwTm-y?9c)Vx@tu*k1Cj^$RL}cm^TOEEMe%!k0du2Qw zy#M-1>uqp+(}4!sdcG-bk(_&Kv>nc~Dz`7iiNH^Atc6$?8t-chnKkh|yOfJ9*w;%y z2gO&4^F0jVUN0-aFIsfKe|x-n5F+N&w=R5B14CZ94M``vwqsLbW&X!!?m+NKEL@mo z6O!kmEMxGuQvGttI}9zzlT4^~UZ}PKo|dBT@evl{^V+Ycwt9)~^eljj)cfZ8h{wQsz3D6OCOhYB-QFwpxWXZ|;En41-8XZi@b@3#TJ=q9*&e-( zt1s(l*9I#a;zG*}_iQf2ZmTmr(|>eJGJkAQEn|njijW*&;Cae8O=oAHN8O*FpTzPn zM_sh_-W`ZyL%PdslpD!3f|(w(u)TPD*=VAqj^?8=_MZ@$8~aTx6csI2W6`wqTz6sm zv^3#N#_|JemCh z%r)|jvGLD0WFoF-Z(S?-j&_;eNnv@Z9HV4dLjB+jeiirpt@!LW{ubpon33kBnBAZ_ zk;+z$-s5J)J)WmVQ=tdI*?dYj_odLuq>}xCjZ)^INCns5GVRn4087+|!FLYMUr@cF z5rCiZ9WgaDoXgakiaX*@Jv}@X@4v2UbZg#Fl;MT=wOTehFfy*BIYQDuCGzGu{(_9F zJAK*vw-I^qI2Ty6$u?y28sp4h%%j4Qh?C59(8}jBB{Q6yXd4f|L3gvTUPWmm=IVWD zmy2kBPxLqpJ!>-l6)E9ZfYFF(e;ho0aJP8P#dO`}z9qBd zRA=nnT|1p7)4!=!uAiW_YOV6-YGy6;}H9NbFl`=hHI=d+j?qr`sFcGbOhh8bW77JLd#d( zHSCYIG+dc6u6~lxAgkpE9eyW$7>OD8?PssB-Lub$qBi7YD(CA_skIe}M`Y-_%r=j?sw$YVRIBcc`7CrzP7p~}U7 zxV}xAR&Qi$Y~s$RAoOY9FKW4WYf)qANAMbEd|mqjdzfQ@UG~a${AQ;&xy9h2&2{#H z%D$IM3g|lSEFH&zu2s*4p}p*raO&`pZPG;p z8uP}GTrrU(rht^V6A z$Z_8XfU<({+=#yKCVcp=DiGr;dX^v9=AWr(N0aTjE~ZHS^bXT#=#r~SCO^p=FeC*R zQ1*XS3F}}lrMfxqs#;(vm$!4BymHFm&78({k*A~@RO4!~ASSAHs&sq4s9b;L(QT!F z`Z%Ak6;9XO+Wm(KJgvVn{k`)(+4&mFK}PM9q!tE079dV>t1#$RwvCMNO`{d`C1XLX ztb~l2az$YB3TeOh^(C!7SoKnehb(xDq^EyHR@n@V1XBF(mmpqUSeg63790TJ_+!fd zFG9l{@kC)YGBy0FFqG%Ns~rIN3Lc@lc@ggEhKno7MlZYJf3m5*qh%nFJkqDCk f0Q$hcUjM%l8um#5za;X242Xwb7ow~Z`g`<0k6r0# diff --git a/summary_results.json b/summary_results.json index 8011eb3e6..cf16cd4b4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2534, + "Performance_Result": 10.2095, "Performance_Units": "Samples/s" }, { @@ -366,6 +366,42 @@ "Category": "open", "Suite": "edge", "System": "gh_macos-latest_x86", + "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 76.951", + "Nodes": 1, + "Processor": "undefined", + "host_processors_per_node": 1, + "host_processor_core_count": 3, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline", + "Software": "onnxruntime v1.21.0", + "operating_system": " (darwin-23.6.0)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.437693, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0012", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -396,7 +432,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0012", + "ID": "5.0-0013", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -432,7 +468,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0013", + "ID": "5.0-0014", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -468,7 +504,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0014", + "ID": "5.0-0015", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -504,7 +540,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0015", + "ID": "5.0-0016", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -540,7 +576,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0016", + "ID": "5.0-0017", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -576,7 +612,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0017", + "ID": "5.0-0018", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -612,12 +648,48 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0018", + "ID": "5.0-0019", "Submitter": "MLCommons", "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline", + "Software": "onnxruntime v1.21.0", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 23.0168, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0020", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -648,7 +720,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0019", + "ID": "5.0-0021", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -684,7 +756,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0020", + "ID": "5.0-0022", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -716,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0953, + "Performance_Result": 20.9496, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 835f4d5805c89441a552403496892d93676a2b5a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 18:20:35 +0000 Subject: [PATCH 247/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10059 -> 10054 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 5e04c9f41..c02a495de 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,38 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 18:17:05,305 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 18:17:05,305 submission_checker1.py:3270 INFO] --- -[2025-03-08 18:17:05,305 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 18:20:29,225 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 18:20:29,225 submission_checker1.py:3270 INFO] --- +[2025-03-08 18:20:29,225 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 18:17:05,305 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 18:20:29,225 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 18:17:05,305 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 18:20:29,225 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 18:17:05,305 submission_checker1.py:3286 INFO] --- -[2025-03-08 18:17:05,305 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 18:17:05,305 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 18:17:05,305 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 18:17:05,305 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 18:17:05,305 submission_checker1.py:3307 INFO] --- -[2025-03-08 18:17:05,305 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 18:17:06,834 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 18:20:29,225 submission_checker1.py:3286 INFO] --- +[2025-03-08 18:20:29,225 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 18:20:29,225 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 18:20:29,225 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 18:20:29,225 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 18:20:29,226 submission_checker1.py:3307 INFO] --- +[2025-03-08 18:20:29,226 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 18:20:30,617 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.299277 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.358439 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70326 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7143 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422043 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9496 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2095 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0308 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ac44007bb..94cf1ca71 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -874 +875 diff --git a/summary.csv b/summary.csv index 44c274256..d66c3972d 100644 --- a/summary.csv +++ b/summary.csv @@ -1,16 +1,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.299277,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.358439,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70326,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7143,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422043,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9496,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2095,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0308,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6f8ee7e2d..8d570101d 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.299277, + "Result":0.358439, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70326, + "Result":1.7143, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9496, + "Result":21.0866, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2095, - "Accuracy":"acc: 76.200", + "Result":10.0308, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0cfba8ca6d13208a8e2460c53b64d967823a974c..c5e9c28a1cf9e56b86a3787fc22f9175596fde9f 100644 GIT binary patch delta 3129 zcmV-9494@zPR35K%>x9`jj@=M(E~;VtO3kmlN$sZe@l%gQgX`jk?FK;(;#`sBATy8 z=HK(vmTwwd1bGysEGHxLm2mUx_^;tJE5GxE5Mf9IIUkvc5XHcfwbPvK7_R zxTkegccl@LIH*!FW6$S=t`Z@Q^b{pzDX)KxjLOowrjZ3NBeNqt&`2c8u#wOxB6;mL zthiv=WAn&a8)&GGx8^#w^Ts}GJ6+pzd+PPP6sBa&u^O*+gvWsh#>0|58)fZ*7qdtO z9sy|;6-l*hhb*(QRQ_hN2as0N2eB>153>375M$w^qp_2vu9rl3he&sudPht9TWx43ou^+xFUaYrXp~LYBvLWqvUu zA_AInlg|da0p*j52O58*0z0-3f-_=tnsi2 zR^&P;S2X8FN@AI)-Ii-r*_?0G*Mb%Gm2B3RObC{3?h+D2q*QnE>o#UgY;IIZ-j0pN zKLJon0|XQR000O8002J#H;eX?PYM74CYh=R~mGP z28n+xVbNh4Ad;A1v?Q{~rNbHlB8z-F3XD2X(q_Gs`y&w5AyI@#CV~e(Rr+-#k$=Y~ zNhJxEf^yrwqjk;S@^GfMB_>S4$&Z#B^*3V15gqd4M16B;$e*d zltnfkC0#aWP|{|-1Sr9+L!xl$%KSR8bgh5bu=Dpb`9f_xpVhXHC%vImO|A15uU-z7 z=mT-2m4|3N>E$6Bq_TvYhiQOHqH!gaMTQ<$0Z>_F=~2>ga|R`C(g^NYf^&yN;f^Kq z?!b;^#pWHP^5<_^zED%owc2*|IO+1-weogG{2VONFXBjB57Bti*F!XjWeH~w(*S># zL<3kBnR{3TfMt=rM}fZwO4_6m+!+Cj4~Zfx*fJ&`Rk6$GIWB+B-pEV!`lxNQPr8cr znqJPmrM5Vk8to#EwD=H>CoMii17tFdB$H_*nM@nI(KaOao*R2Mv%}Y4RZ>COmhiL#x;?aTWvdHAa8UZMa zOgm=%nzmuvHrC)o zgLj*FOa+y7D^J3zRbpLT4p^3dF$q=AvP+BUJ#ZQZ8u!qfahYj|1O zG`u9G#h^r>s8)uaewltn_rUV{_or_VCv@+XV!Rrn(d6#|@_nz1JjWP+uo947K-<)y z?TH+)I;wSgb9XVqC&qb)LzisGq{u{__DvQZ=%C{0XPdTk7C?Rlktw&$qA#IZC*`eDCs zsB#TQMkB`eyZt*eRZG`Zj_$2wzh5meK*0-Ia+t5N8K}35M*9DcZ7-~vj@Tyt-0JUjK-^O-{UFxv<6`1!(njqdo$ zI|}1*bNVLjeQJ<@M^O7ahJj6b~ZUdPH4KJ@^pO}Wr#`KU~fwDOwaRT{Jl|zm~LRpS?6MOJ--}X zOT<$iyo1Z}WIh|*OzZi^lT%d?gnrlVhTS0W7OXQGYBYdH%jT})mU83Z)fo#eNbpGnX-~PW(Jll_G4^~1iuo)$bQCwn zDHdYsw%9WM&9;CGLLRV!z<&e_3?G20orvg!6nsM3AJ>fl!7+58KnIWWEXNmb0H^$H z!>C+2l?;Q0S?gJx+DC zm=EiC;*A9HI7EHs?X<|T4@43J&|}=Q@=-oLw=2{Xq_|H>nUY1DFBoV+Xi2^fBX9aI zKy@1OZ{itn%zgU_nm@C>B7_12(2cQ}lSL&$2{()Ol1~Z%0OOO8C07CKljtQP1i~YZ z!IKUq76EmWB_=@u5|d{pIst@}q9!IA@vc?XBmn>bPyzq|5dZ)H0000000000fPrO` T<0d`x8_)p^O2(E~;V&;Vv^lN$sZf6t63!gxxHk?FK;(;!7iqqJC! z%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8uxLOzKp z%L5^A*~;Q2BSBP?^2~B=+q3c@EzHKkfc=RL8pml!j%irsq!5h}MzTQ2@OhG!+#C<1 zR6bMre;AC!BlDXxm^}KXH6GTk|4GR+e|K$&VEI5Y5(*N@0+@!%Y)R>N_3%Ot*@|ju z+|xR$J7z>A4ysJd=<_*ASBa2DdWsU#jMu+Lp32g?rjZ9PBeNqtq>)IJVIxVSh!nNk zu;PN|kIf@zZJ?n#-kR&!&Kvu%?R0I=?Wx!E6q=DW$7;OR5grF37!MhJHdyU}m$OI( z9sy}pDw1m14ry-Xnf%RV4N2eB>154kgv;HWTdQOdgsQD*$u6QX)e6h1RlW+!!esH}w!OC9TJQdgkQQ;eGQXG+ z5kZ=9lg|da0Tq*q2O59uwrd~4P0xDwEu9m#A`_Bj+z6>GSi^i9ZO$8J&S2hri~oYF zkNF}>)yfuR&B=jCi7+e;?$B$Rfw zP%gftbKzo5r64l7pDw4%Y@SqpP5%IsRb*G{#?s!tv8=gPBhQ|ra{M`pxNoFEhiF{s z&>?>sM6!fOhiL#w;)2l<$s(H$YXp!iGU_OB>Oe`G^-^w-KvaiB5hR%ip7>O0*pW#7 z9lIozBv^{dZU2teHGj+8u?yvneaCu&63riu^zRUjCk;GAgHV>R@GuQPNo+I#Ws!@A zH3Co;`FNDH*_=U1oAnZ)1iKE2!nG?i?7)B8wPMH4-_PU=_3?aG+eV)Bj#4$X&fBu| za;QWjh$Fo`MB_;_578i%CG0#*15^@?E2%7U^sowm$|6sXl9rn@C~1>MaLW>`J0uFX zESY%+wk#`l?;w>wf6wxTx_YkFwynoW*XOR4w>9GDV2OqiNBVk*#*@Y#qCqT6SbKk% z2CyU=z_Q5Q!zutQi~Kzb3_eiOCXL|M2zY!*6j{NRarvlGL5PPx^d_ z2FWbp^I;kwlQ?MP4W=&g`LITS%p#wU2A>a^Kqj%!0GWS9MjzG) zkXdB(*&KW#6O2Brmp~>MeMl6}nPrSVTmAnBgZWaOK6=;blg^n9m2;n|8#!d6Ys8UG zAENQ3(}!q~%o0u?rU5dE23E60P9IhQkXhvP(ctufk~V1s3tn4M0guIxt-pxqMh7 z0A-QOM}x};O4_6mWdW67^Pyp|l4T-T$yV(1c@E2DkP)eV=y7Z%xs_VLZkv44`)0oh ze?n55- ze-c9h#goGlBLWM`v*;3=0R)QhvV)U{6fu9`TZ%B10+OO87YdTC>=PyIHt~*ac`cbC zf4_1d8OE|mvLR0tr>#CZt?rh(|80|{XbmM#Sbo~Urrkl1hb&6+)oJJM_T2D0hzpuW zG-WxQb{>Iuz7789b1slvn)B1ngAm1uW%2LKrSsYD>EwO-*RQ~q68Z82LOMO zr53Sm*UD&;cTmWxT%2}@FIlQQ`L}}M7QuGV8T5HF=!?M>iy-Y=V$ip0U<=Me$Z46d zCVrPEf;Yh#@G2EeG)-wniQ(BxBM6$tEZK;XriQ=qmQ7o^qKdPdI8Kusn(!_A0Xesn z9nEQ0q|joRj9L`QxS+`heF1Z2A{l>JbXG-)_}ct($Wm5*hew61T4sq5P&N%`Ynr5V znWTwO7d>(TSrfgMXQAwQTCxy0XNv#gPlYiIA%&6(^<%1hJZG`^NlPWD*G#CJxZ@DD zW{RY&N*XExM>!W7t`@MOdtQI3?K!M4=~8rPkru6?G+n%hNj@TytTzPLId|&3hlM%OE*vDl&{3v-Z*>5*_gGsEj zCw;Xr9GK*TXQw`UKC=)$%=UpeetzM-MtA(mI|}1*WBR7r`&56CN2vC93#KCIYdv^%u+cfWdQ+1d02^#aogl&9;< zC__x*_+Ov)#B_XH&N>(4>&4~xS|YCU;2mC0ri=OTW>(KPuAHiTKk&MC zH|YAlE7zV+M$cv<$1tBbp6?LPcDla$+I}8zY<0jnmpk@bW}+vda!KGQfq(K=ui5uz zaV`e`0Z>Z=1d}}`8nZANLkI;uwIwnolXxL2e>tJ45w-(u$*F-tQ(Afx(k2nLevs5U zdHX7k?Ihq+kE@yec6L#@Jg6r51?Q#F70NS$642ZTsdp86->lLCB|b1MSYtF)=m;Jy z>${R$%8i3pXDqlN!6y-|AS`w%R}*z-Lo=CkO~QQR7*Scs+DVaxb8+X0>tvcL)g ze-kV)d;+F+B%&TE_=vPWuIT`RV`xBu4j$)OjxXK-PWj1(PPuX_8IBevt%Eqa4_=PO zwr#U^-dSV*9DlA~-nvy&sZXckfa+3k${ny^TwUVpA|9?`KCI)3w-UtTG3pa<{UYZ+ z5J?O`&v6IkWB&BKS)qDHX7?$Xr=-~A83oNBC|P`+M&9&afT|zzZ{h$r=f3>}^&hji zB7_12dewQ!lRzax3CikV4O0pL0OXU4C07COlj9{K8;bC{W8 Date: Sat, 8 Mar 2025 18:24:27 +0000 Subject: [PATCH 248/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10054 -> 10055 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c02a495de..02b649050 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 18:20:29,225 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 18:20:29,225 submission_checker1.py:3270 INFO] --- -[2025-03-08 18:20:29,225 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 18:24:22,218 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 18:24:22,218 submission_checker1.py:3270 INFO] --- +[2025-03-08 18:24:22,218 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 18:20:29,225 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 18:24:22,218 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 18:20:29,225 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 18:24:22,218 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 18:20:29,225 submission_checker1.py:3286 INFO] --- -[2025-03-08 18:20:29,225 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 18:20:29,225 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 18:20:29,225 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 18:20:29,225 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 18:20:29,226 submission_checker1.py:3307 INFO] --- -[2025-03-08 18:20:29,226 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 18:20:30,617 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 18:24:22,218 submission_checker1.py:3286 INFO] --- +[2025-03-08 18:24:22,219 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 18:24:22,219 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 18:24:22,219 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 18:24:22,219 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 18:24:22,219 submission_checker1.py:3307 INFO] --- +[2025-03-08 18:24:22,219 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 18:24:23,174 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0308 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2137 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 94cf1ca71..f141a7bc7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -875 +876 diff --git a/summary.csv b/summary.csv index d66c3972d..148cc9238 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0308,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2137,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8d570101d..e2eb4ea99 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0308, - "Accuracy":"acc: 74.400", + "Result":10.2137, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index c5e9c28a1cf9e56b86a3787fc22f9175596fde9f..a5e5aa2057f005591c48882177fdc858ecb347ed 100644 GIT binary patch delta 1219 zcmV;!1U&o3PRCBL%>xAD_=Ag+(E~<*&;Vv^)62gf8f)N{f-{v~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw=GXCG!)MCA^Mnv# zNCO2QnTZhPz_NIlkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ@*pkD#=?Mq{fP}4 z$7x88X;|f?5RDN=vOvi2d6Jgg91o*ZK2!M^jKm}Jn=_a^`ldA=)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QEkVbln64H#RR2QMSDBR!;%NR(kCNu!7}6t&y1;)3Rn%_C=RprJb6n(Nrk8~d>BbZyV=sn_$G zku}F^yw(vO2O<~`8GSZb?SYrGNCh4Ne^n}yYS|8HZsnQ$&1DZDuS&w=MOws!$xenW z1mR-At7V>Y)fDi>Q?G4Xf)P@;AC~Getn2Ay&1ZzmuQ|6?$s!0m4$ZM(Jp`Y%FS#OccXVn##+X~xZip%o(^xBJh?N%nI@hF1T55ZA1Uf6Hk; z$n8$wx8JRLTcYdZqU-CT1Jm$Ghjid7k3gX45C{|<0)e7KAW(Gu9il_%P;@)!mwhNY zI55>6XrhCh8=V8PzD2Bp^4|6Thobw8oc=&`5B)9C*-qE@4nO$R`MBzQU3H*AIMN{< zu+UBr2vi*cfvQ6wP<03de3{Sue;ukr=pfsR9rQ~dstyiJbq8J8DM8MS&H)R_En*e6 zMgHfiu3A=ws9JI(txUZ5-p2y${ucp$MtXlBz`6fUfR5dEoetc9S?~7JIbka@Az8+a zkjjEJ%(v0zykX`H=FMLG7hG-TizrnqTaYy;2O=$22LI6!%r{V(UG1elE56fUht^>A83vOg2mWkSJxmK~}e51Y=w5+dWv%aK4(0p^3 zkRT#V-O1f;OsUx1sFJ)L1RITi0<&fa9Sj7t%1M*6hZLg%0%l;dEEh)#1#3*ah$WMF zAs&CrYQr!PMfd#*qrENLX^RoI18vEwfkIPSvJ=uI5w#vjYMuOj6~B^zS6zjk(d zfRrQ5ROsNnAs9P$07+dE94d%AZ6)=gw0nPSDEp>+zM1@0V>1!afaGjK znjhPA0M0NpAVGPDvn0b8uLmpLY(uA9Ii>I?1GCmqoZLqzCS%*SNxSH*A%BLy*01l~ zsv$!IJ0tDjtgNu_uCB+1^ h%1M)x2`0J)YfQX|C6mY|Q~|k@9Va6OYa{>w004LbP=Npd delta 1218 zcmV;z1U>u5PR35K%>x9`jj@=M(E~<*tO3km)62gf8RR1urABBR$YaB+9Un&?q7^dF?i=xM101^T=5nXsC|2<~p|X#y)I2UE6be>h-*) zWX-V}uXTjSfe6OKl06$`?SU7wNCh4Ne-#x;wQPqhv$9nFX0iv6R|P5K1u&D z-0t*!``xOyCAvN?y1p(tFb$7%NC&R+2n31_fk4qA5GXnX0!7!~Av%N(MYn@~*@vQo z15@3BCOXKu(K#UNTf{0T?_K|YD7w$c=?_Hr(BBfB?R0(b@Pn<+$5rR+ssjzekq+sA zg?55Kpz06^R2>3=szV@PXFl_Hf2ad`f8)nX6-h7Myf~$}DJfdo43)1G~K+t?;@E^?)g*TWyy0$VFf5x*aC5Ct{ zQun1Pj5fwtt-K%p)zy$NZPh+02LY8}6Q6(?~L@Tteu%zit& zs9Nq-m;8eB(&!rH89@nXZiLj^8ojrxbb%5dm=>%v8fr9vN6Y4};+Asb;MExmE=cf6 z1ZhvXt5_#wiwJ>9&8^GXBlBfD1w%u!6t@3k)BCshx=EgcN*2 z+8@`A0KqYIpg;$Y^DM^~ZvdzKY{RHrIh729g<0!aoZNdaCu86DSznIUm_Ns#>zB82 z)l}-ksW_mf5}a}eEEv~R__~OP>zEJgc;bx&@i;_%=Iyk|u@6KN1JGmKv+_|sJ+~w) z)D)z+Pf3}QMVl{ZL1;<74kK^+FFKmY&$ diff --git a/summary_results.json b/summary_results.json index 8be05b225..92bccb3e5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0308, + "Performance_Result": 10.2137, "Performance_Units": "Samples/s" }, { From afbf10c2b8b5257af59211d688b1cb2d0931280a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 18:29:37 +0000 Subject: [PATCH 249/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10055 -> 10056 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 02b649050..fa4354bfa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 18:24:22,218 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 18:24:22,218 submission_checker1.py:3270 INFO] --- -[2025-03-08 18:24:22,218 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 18:29:32,914 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 18:29:32,914 submission_checker1.py:3270 INFO] --- +[2025-03-08 18:29:32,914 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 18:24:22,218 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 18:29:32,914 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 18:24:22,218 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 18:29:32,914 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 18:24:22,218 submission_checker1.py:3286 INFO] --- -[2025-03-08 18:24:22,219 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 18:24:22,219 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 18:24:22,219 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 18:24:22,219 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 18:24:22,219 submission_checker1.py:3307 INFO] --- -[2025-03-08 18:24:22,219 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 18:24:23,174 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 18:29:32,914 submission_checker1.py:3286 INFO] --- +[2025-03-08 18:29:32,914 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 18:29:32,914 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 18:29:32,914 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 18:29:32,914 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 18:29:32,915 submission_checker1.py:3307 INFO] --- +[2025-03-08 18:29:32,915 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 18:29:33,918 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,8 +25,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7143 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422043 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.19705 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42114 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26386 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f141a7bc7..659400d33 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -876 +877 diff --git a/summary.csv b/summary.csv index 148cc9238..e7f681de8 100644 --- a/summary.csv +++ b/summary.csv @@ -3,8 +3,8 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7143,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422043,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.19705,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42114,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26386,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e2eb4ea99..eb9171663 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422043, + "Result":0.42114, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.19705, + "Result":6.26386, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a5e5aa2057f005591c48882177fdc858ecb347ed..929411d64b764995ece1bce0bb585d05c2d2bae0 100644 GIT binary patch delta 2794 zcmYk8XH=8f7KTG7C@u6zC^3l<0x}p7Q7IB2kv^grq97$SgOs3j0yqevntXIoFcd+` z(4+|H&_Rko5EYRT(9lFd+NH^bS?jJl=jT3WJ^S6y+CNUWWshZLfFO9{XIHYd5C}94 zgbTuH)NwpC=EtU8v27#%Sy$fep^Uzb)ERS#OCqd$W7MHhFfIjZkcg0<{X2zMHeplV z75R`D>`rAEEH;1It6LjVTc?XeglAeN?kL4~PkkG*)#}4W6Hk3hac?(TXI!OeB!~{!F#9(b*Bj0V6@6-u9=t~kaTPOR z-)2T6-Cqw<;QRDT(?hr9R-gX@`+JaEh1dI(*Q4Sh{4);A>fh9gS~0{c5I*2G^RB)^ z(ri_5z>wn8?&`2aw5)t~x7Yrcz)#pkQ-QuZpYGOoI-4JQe&2%Jr@1;>?!*?Q5f<}+ zM_?FjU%O}~0Q20_yh$|HtsVO^d8Zd$`d&6^XHV&v2YXI!8j?ciktoPt-m|C#jf9Nh%Zya9^gbT?jl$(2J z&sQbQ+Foz>q2qEkCE(&b8_9Uy%2(n9Dha3+av4CV;01H(+A1!` z0$+6(UD@OAIY&~_X4k11Rxm~R@ZHU?K6Xp&!4HdHoza$dM(uu@_F*OKZJyz}g#E_= z=?ImU3)|anUdXUl`_0Ru++RPDG}I`sEAvDmW1N)Qz>+zE=9-JcT{J~Nbm8JMa{~1s zU5I+-1vNn*r-oDhI(@N6Fype5l03oRFTn|kb1Qt8QjsS6BjpczrsgU>_T6^h7-`D5 zawW7qwR@``krt4QC<890oT1_3KH9kQ|&SK~iSu zpuK-f3`B{6N6>L_0s=x{*%`pW83^e=>NVu?YeM)AU!`FH!Ib^Y$uSQ{^hbW(qvFf{ z;f0W>&iN&uILT?nZ*11Yjk$&oXw&d>hTuFr0%8njpjjOfb{v6MAe!;fO=EqlBW?FanBkv%?6~+-dtQDI?}j=xpP5&`6||g_+kz>Fw?G6Pw$Adl*w=V| zc>VY86h+wX?#i=t5tnFs)Mh0pnf~k)b&n=T(X)t`%h*%OeEQBsQXp{i8K-b{I*?5y+(#}q zI|NcXpWe7!vG^lXS&Ft0QMP=t$=giDW=0#C_QmspH34gY90sKoN74QBpV&I{;mL-4 zYRFBg!MuV2s2?&-^O+ILx2*tv)`bHJqF>__^(C5l)83AXMiPya$((^Z-J{#X)B)-v zSRf<*$9b6QM1K_wjPklpil#$(0S}Nli%Y}i-pvKLMin=2On|LpwEI`e-tT&mA28g2E$F7S?a;V<829D$TEg+=94VGAM(8h` zFjvhMt5wqK#w%KGugZvOr|V_&bcdR%#V2n>R_PbdL@k-2Y-U23gi$2`$72o=)}TWj z38*DFig0{H!rIpG7Im3UAda}TtRaCzuRTjhiD1a3CVRAu<4B`#RnhNaSrb+nY_hhM z%Bvz;WUorK>Kzx)Ss%=;3a{_k$iN1w$-f4KjQ84uDcZB9AS^2FAnJW$P{1!l_0gzl|rCCcxVbYP4pk)X>Msni1F1* zvbo+f=ZrVh>%VeZ7#R!eHo-MkFXGp;CzC_Xy!|w_#*O7YSCpYv_F-}SJ+4p!ijCLI zGGUe^VD!qk32rugYQoZrXivZ-(mZbA0s2}OL|EPBPN#J4K20by4MB7}S-Ps8OY!Mg zS%R4n%kMs*CySeohaGs-R%97!n!OLpcMDkduVJM5FTQ=Be>%h zD$z8IX> zm@BbKL~={(TW5%DZ+)<@2fyg*+EH?Ew#mLzWe7O^x0Eg?tmVsc z`!|zqKqTbi$>;WcZwNY$5!DtmW|@eowUv+Pxs7}1o%COm!fhJCyG-F;&bq59jjLu6 z5+}m@#YF=S1S+f2^gaAymJCSMUU}9T>EHgq5^&`LO#5z{W2a>fZ}ozIz>l29)5Z4R z$TEA;cP^Gf$%J^gwK?@koz}KTh94b1TQXhC7V^SI8YOTlobPLRPYrnSj2-c{0i86> zEEygQHPX0C5n#SR=-7Nak(*(b$P?sBub(~ov=NUmc z!>!Xk6vg`$Hpo1h+oDhwaORlF*S3=}ibp4EA9s9c33S%p2+v*X^bp=Uz56^QT+uEm zMmHy(%S&ZYu*RUJ&LZYif^S2z?kk}alY*bEjD~!gVKQ0Uk*3O^tm((8Z>F~jZ?I5S zt2;Vs)3uZuIp&v?0`yoc!*0(#@4U#?nHAnle@9w0N3^C9{+jz=|I9?B9oIMD3-UbI zT|gOSC@did0tHb{Q3HZ0WGYaJk_7IdpxiOXfV^W;KrKp4{!lTJ3k0&}27$yu|38sY gZpuKEvc%ydpc^iBSn@ zt;ez8yDLBJOKqC4kNF)Y>RS=caml#I<1rG>9+imhG(!)sUR6%?&)pO+($^n~La;1i zHM>bUcDN4e(&+E944u5gK$(|*D3k2A^OjxyQJZuVsU&49hz11A#n7W#K4 zaC#zQeZJ?seuZvSMpe3Om>LL2c49TC=`U%s2;A&0H}r$TfnlG^B6A(czqJJmDW`C4 zm=~c2Ns&HOd7Ye*IL++{mHMUMyZAQXO^esZS=iFp50hF#Xx{8TVa})ix-G>gr^^ODjpmE#)zDgbq=u z;9>vMY43GPCTPZN7qCt5I1?PSqf%7%v=Tm8Iu!Ct1v zrNnyn`l>gmjI{W~?*8Am#x69Q3RrH9#-<(ZT=xlf8)<#BP_)IjwndlLOrMSRc=!iyBDZqCp%Yw(Wqx6Mgm=?e~lOl?5Mv!EN&ELwV&9%!kuv%O7x$z{i7U6XDwiOXSz3(FtQ!Fy*eR42^Y zTz=t8M`!;y2$AI7OvLh4wMpUv?3uN~j<_}nLxv1m{o#Sh@@`Vn!I2E4JBhayJ9)V) zPz%$qCyTI5Yn;{0J&n@a*;?uSRPp{{4tuQkB_X>aCT7~TcFSt%?(j|^<5xD*K_r0M zmVG&T}uOckOdClTyR57}3kyT`; zgYPvfFP*ya*TlFTMH3}UDTx*nx`t-WK^7nm4YAxuLd*>O$+y3Y5U8DgP8;jB%Fn)t z5c*xuW$7s2FVHTDI{pfD)JZF$bZe+mrh(ER+ZD!Ku89#C*nYP(q&}n(@j12aZl1lT z?oo&BENZq1*ta1hhgwtu2D-c%@B{6K3^O2A9>hv2s zKxXgDB#+Z_jcsUX4waTDi!MZ3a=+O9cH+;nU|#Ja8w$=YaOGZhOtd_{x$tajBG8WE zB4kiGcF19sGR9ik(LYf%l{AqgjdE-7#=$&3loT-xDg@N&{H0KE9_XZ&rH>BFtPJo7 zS(;Zo)Ad}QRZCQ@D6+x?VWrH|ZJ^g)m-9J7(j11#UB&fOk41s@GDx@f`fM}FWAAIz zY?FEam7*yJi*Df?0bZt&bnsZ3a{+$`6bxXx1Mnlt>CJ-pGC;>ZyiR|MuLMb9V*q&h zQ5DPYgj9Fd7c))nZ{U5~nTxafOU!Kzm>sYSh_29DCJ)u6+iO% zbiE}X{1Kr2zeoe2l6#BE2tMXlFV*yf-$%4Y#milv^i)fwCmhns5*=F&2tF(IY1q)N z(bgvngdb_F;5&Fy5IzVHO*Fgi0tgSdv*JNfAc;v2m-j zXkWSG4g>c``phPSZXi^A9R^@BY03QhIf8T2&nx7eUE z3#Mp*=Out-Mb8plr(6;MEECrNuO&O~CAZk>m$?V7xdTqVZN6u4G%?JEyNn z?f=+wp1V`(&9-Qc<7PVcw-ndwWq`jy5^J|464Z{O(7dkN2~Z{lnt@m4jF`wM*+oFv zrIBg;@#3snoa$LPdjANJBs%~7>gKEQ1Jnc5*JcbPn>9MFnd{}-x%w@;xXe!9wSd%_ zqN`M}TGU5#Pks&3RVo2$(<3k+ofjcl;+}jx$eL~Us_nD5EUuN$C8Pa2p3KIXT<;%| z9>3Kp+o$xtz6{FI4k-D!xjm2Rvoh~5R4NbXK@Y^LhNJe)$rT?|&Vb!Q;dxy_Q~}Xs z;>`NKI~gXZ+`bMq?T)z@p+c^a1Am~zt~vJTZ>m=MvC;X$8wsd|>~{qH9~sRl#_&vx zB4tAbYyl3B<>w$>HumGgbay`K;JV==z0blZiDHxx6egzK0$_Pa=74M3@tyg&1k^yv zb}m?0ol_~rdOky<^iq3QpDekFvE1eSi#Yo}Q`S_#(KXQ=8U_pn^!B$5WFe1s=>e41GfG%y7bKZVkU zbz);Wa9SCim}7XFr*zYiD=lvq&AxrA_z2`eEyhKz?HT%Snm(D}EGF1;LUeVe4DDB* zU$430LL`z~I!3R8L95KHch1gB|)v6HfvbF_NVD#;B5`Xcqzk*`L_&o8?VwoW- z8q{%0yktGEH|KGLk@cmTgF{fsy2DtHoS)&+ZHzniR}z8>iQQcdG*~9cV<{VX3~hng86PB6lVj zfV%=xRF6I54ddq5hIt`&OTyZ3W}>bNJ)mA+Jc8mSosb;7Gvj5HsWcu4=-1LwK6jJG zhQ5%k6wsLPGJbejzxcZvFZ+#U_6JinIBC2d_jJBYg&NcLrgCNCaW!Gi`uN!(j=YOZ z=~%tyYLKMZ7-4Q~vhxh^=YrHmMuSUAt9XbMPcmw`^p@rWCtVGjIm$1Ihw*wxb&2xl z*1{NbhWbEeOAX>}JR-p8vd=2{3!$67vhp02suCz|V<-{_ KD!(W6Pv?K8SRFF} diff --git a/summary_results.json b/summary_results.json index 92bccb3e5..a4226989c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.422043, + "Performance_Result": 0.42114, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.19705, + "Performance_Result": 6.26386, "Performance_Units": "Samples/s" }, { From b0be390e8140fc921fffe2ab72fd201763a5a2ce Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 18:30:35 +0000 Subject: [PATCH 250/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10056 -> 10055 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fa4354bfa..263460f0b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 18:29:32,914 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 18:29:32,914 submission_checker1.py:3270 INFO] --- -[2025-03-08 18:29:32,914 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 18:30:30,933 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 18:30:30,933 submission_checker1.py:3270 INFO] --- +[2025-03-08 18:30:30,933 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 18:29:32,914 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 18:30:30,933 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 18:29:32,914 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 18:30:30,933 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 18:29:32,914 submission_checker1.py:3286 INFO] --- -[2025-03-08 18:29:32,914 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 18:29:32,914 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 18:29:32,914 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 18:29:32,914 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 18:29:32,915 submission_checker1.py:3307 INFO] --- -[2025-03-08 18:29:32,915 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 18:29:33,918 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 18:30:30,933 submission_checker1.py:3286 INFO] --- +[2025-03-08 18:30:30,933 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 18:30:30,933 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 18:30:30,933 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 18:30:30,933 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 18:30:30,933 submission_checker1.py:3307 INFO] --- +[2025-03-08 18:30:30,933 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 18:30:32,101 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 659400d33..cbb3e1816 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -877 +878 diff --git a/summary.xlsx b/summary.xlsx index 929411d64b764995ece1bce0bb585d05c2d2bae0..d6073e66991d2acacd1ccdeafccf8c6c20723ca2 100644 GIT binary patch delta 338 zcmV-Y0j>VXPRCBLbRh-u&lJ@plX)Q?f6Z#cFc5_AeF~$yE!n0e2-|_S$JNY!JG-b_?p2rkg7ebo8s!;5321JF)Y}@px2tr45+9fr ztTP&FG=N9T=C0zFa^v9D84E5*@JR$|Pr0qpE(A+4_IwA5`7Ang6gS2x7GmkPf7mkq z&9;CGLLRV!zyu2nAAqTyiRg$Fd`8+I*Np(dF?66n2aoeC$7gQ9DlA~-o{l^sSl^(fSO8h${ny^TvOr8A|9?|KCI)3Hxk6- z5cL~x$3><-5J?O`r?@BOqkMX9Bv+^@NO7N%G9`;PU(k}!GXFY^yy`y#)p5wbi6_96 k`}z|!f3vzG3j_u7&lJ@plgcJk0lAYOCnE-HBme*a0Qo_iwEzGB delta 339 zcmV-Z0j&PVPRLHMbRh-gV=y8mlX)Q?f6HpaFc3xe{R*SKEk9^+h3!CFvTC4EmzL~= zG)Y9Q2a;OHe_zE;oCLD!a&+&UxihF*>}41If^|Zx8f8g}B2Y|op|&;pXqRz`BIl{% zw9^V|G=M{k=B{FfFm2(@Y6I2_a8Up$N0_P6&U-^Jc5DZdx+FMM5I5RN>O*O_f7np= zO}BvaG=0DlJmu6=d;rE~CZZ#fvl(fAY&QZp!_a{Q1h-U#4N zL(p%$9T%DUfJY$!nc|+5&*J%|C0(K>PxJdYE#kCnvy$YGq$t*hkyrg^pga!wH}M3R la$kRf<`1*EA`1is Date: Sat, 8 Mar 2025 22:02:01 +0000 Subject: [PATCH 251/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10055 -> 10059 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 263460f0b..67ef53e9e 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,30 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 18:30:30,933 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 18:30:30,933 submission_checker1.py:3270 INFO] --- -[2025-03-08 18:30:30,933 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:01:56,695 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:01:56,695 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:01:56,695 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 18:30:30,933 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:01:56,695 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 18:30:30,933 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:01:56,695 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 18:30:30,933 submission_checker1.py:3286 INFO] --- -[2025-03-08 18:30:30,933 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 18:30:30,933 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 18:30:30,933 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 18:30:30,933 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 18:30:30,933 submission_checker1.py:3307 INFO] --- -[2025-03-08 18:30:30,933 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 18:30:32,101 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:01:56,695 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:01:56,695 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:01:56,695 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:01:56,695 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:01:56,695 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:01:56,695 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:01:56,695 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:01:57,645 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.358439 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.260917 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7143 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7252 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42114 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26386 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index cbb3e1816..180bd2482 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -878 +879 diff --git a/summary.csv b/summary.csv index e7f681de8..db365bb06 100644 --- a/summary.csv +++ b/summary.csv @@ -1,8 +1,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.358439,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.260917,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7143,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7252,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42114,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26386,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index eb9171663..6bd758cf7 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.358439, + "Result":0.260917, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7143, + "Result":1.7252, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d6073e66991d2acacd1ccdeafccf8c6c20723ca2..c263a59020c9792422209ed2f3722fc67cb4f27d 100644 GIT binary patch delta 1484 zcmV;-1vC1`PRmZPbO;4-5g_AJvv~+s27hylN$ps^+TpTMJIp5v-2}ETnM952NhVQ) zOcD(+Ni@JDYJ4S`w2PP|jYv;GQ3b8d!6znVNt^Z3mrTm24v8YSmrUeIDD7&YTzpCA z!o`|OL1c12T~3$TJgNMe{sAVd$gb3lrJcXAthqY3xwuW_4tI_s-WzGpAsScubAN~i zi7esJVHzNkcwn?7vdE;v8UZ4UY&r^jI#AMPy_EYS5Y-`3gh(cW2R>EWbtI90$0kW7 z36_E~@7~e6=I6W}+fZ(~cdREU(fi>@`wr1~(!WDA2xSQe57PjY#6<&87MXZhBLHQQ zjYmnB%^8%mSuX)faO;pLT)Hy94u33ND>m%>{Y<`48_#Ez_wl4Rl&Y!qt{qEn2bJgp zaio=pXgulVAsVEzgqw$HfJ&lqC6z^n9##QRS!C%^(s6SJC2i6O?pT6zheY9yCG+mU zj%CH>9i;N-Z&|)jQ_r=^yLz1Trn2Gf+OqU_V2OSaN7{Ob#*@AtqCqT6IDdPX2CyU= zz_Q5P!zutQi|joL{5??8CXL|E2v~eb6j{NRG5M&9T|Uoo`E&M0UaHqe<;_0nD%NXy zJD$L5i<7C*F5*aw57Bti;zKk*CeuhVnMRVyG?GkRWbt7YNHU8mXlbzcKuMc40%Sr| zheW|LmS}Jpt1Gtn{8zxNB!9^=OIe+F`DA+|j&JW+%4)TPOmvJm(&j@np0xQ84U$>H z=EF2VCNa>+8%$kf^I?qunMF1q4K^PrX|rC+O-qRCkSLrpsYuS5x?-Ep_oDej+m@9i zS!T}Ed7qDN)GX!fc=V(74l>a*;z*wl(RkA5Lo`Tc37-$s0GY%=1Ak-|`FvO-KxUE8 zXLIn0Oz`=zUILlm^C3|)2J@vFee|}`C!I4JYKH?)H+GPTrV&RPeTc@B zMjxUkY0JlfX#6&4WWTc$lYWC`;Bf`VWKEOR$W2G!CP2@@>_6_7<+iq&UsTFl0OE`ryudas=QS3T#=FNS_&92#lrkJ2=|Wh>bS)~%z1d3P9`ccbBW78{db zkS1!?T+@hFuPD`8eon&q@{?^EU&15_r;p!8LAM`7*|+rfZV-ma|AWyAUkJ*z+AI=CkO~QQR1(Scs+DV$1k9+X5~KxyK3u6Mrl)d;q3)BBBu~_=L1S zt{nh^V`xEv4j$)Oj?dlzPWhV+gL2_iGV~U1T2JES-g`M2yROT+a9^;;rkMikxy+lnx zidjm^l+4#fK^RF+XAfV8kyrg^pc;q#n|J~ob6!HsM^NB(?f$d8sQR8}&Nz@>d zL<39`4KRrsUr8qIA|^>A(i2crL2GmHiAh<~X1(+!lQOD9qR8zf6L}I!yILq0U(&g7 zv8GZGncPp8(`7bKD!-#NPlDr zhYr&Kk;DU|C6PrY9o7gCS!B~u;M0MUHtVI_AAzV2i6TTY5j^mz(yk+k{5v*DDoL;u zlzI1#)-^xp?bwEL%e`YgL5bcEN7{FY#*_XXqCqH2ICz)_pd>CDfU?NM!x{l7i)=hf zx@^v%q|JH>P=Z^BMB&nv`E_9FT7R)&=kI6oh1z&NtGtgVy`fZ1t#>V6y&Y7d55$pH z9-{H2mxpMO$`WoKrU5F6#+6hS8G2X+KxL7oM@h%c8I-h1Be-J;&K(kkJC@A513Q)# zn|F}PpTA}KLQOr_D(~uXYT9XdyS6O-9ay4Y#F4fhqVc4!hiDMX63!l`0e>us2CytL z_pk~8%OZP^0)G#bv`HhlGXfSL5=B<9WlTP*VwcZzT>hNBk(cWAQF*gZx{CFh-i~`q zZE-R++C>~`@gW*dT6~BG$YdHxCeuhVnMRVSi!45@0!e031uYF0A1G;)Mu1F+>X0Z{ z#u5!KV|B$ApZ^M&l_XhaDSxZ;E}v{~#PRJNOIfXUkco~FN7{Ud#*;Q5qCqlC*nF4< z$Rq|Dd4s8oY(A_JAhXEkqrv6_C2iJAxoHVe9TJ6eCKbs!Q&(*B`Cc@CXxp-qB+Ja1 zI`8w*jhdyL9glvr-a#gMMjYw$AsSEme250gEaCHE8X%K6Xn@QjpMMW)1jsD%`D_k8 zkqJH@)=MB0d_E)!=gcxbpIra{!C<~rqmSM;`lNGaL+xBbH+(KO;nqYu$|(&$4p zNM;G657PjdL<6hYBBKwh0LUyd`e-ovKuMc40)7IB>aZvvvy9P4S8Vioj?DkmP*6#J zrJxihA4^-i9{%X&4u2@oFycs)57Bti>kH%p%PQI;L&)&kdm~@Y(VgL2A zQ;XU7&qeV1RqvIv|ElM_`Nhz0j6)+${ZX1`w`?W*z`AvGFz*h7^KLX8&thW|4AMlc znrj-->a|+S&m~DXUw*Pp<4c$%;q>v_DCqWsDEpTF-VMSqdHj+M$>#NBM}Gmc?-Ga( z1@g}n)g+UAAuE5TB?#Mrw&c`6p)M`G32BpvT0cl?9lw1QJ8=@osmImKemlFUTJBYs z{DSk+=o;l2K?!JXgw)#_y|=4$ff65>7OXQGYBYdH%jT})mU83Z)fo#eNbpGnX-~PW z(Jll_G4^~1iuo)$bQCwnDHdYsw%9WM&9;CGLLRV!zyyB_3?G20or&m(6nsY7AJ>fl z!7+58KnIWWEXQYW0H^%ThEch2Dj5a~H?1decJIBMjeXx|eK}fV{v3ahQ9 z;((e;aLOI9U|dt-%OW1GV?M0oi8m6&;}G>5Z^uQZJ`hO^K&QAT<)eIhZda%&NO7N% zG9`;PU(gql&@%rzjJ)bU1J!ZJzlkTnl>7P zc9SJ0K>-t!XC^uUg_EKtCL8k46xAdF002+|000pH0000000000004l2W|QM4J^{It NA}1pTYa{>w002IirJVo( diff --git a/summary_results.json b/summary_results.json index a4226989c..9fd18c3e0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.358439, + "Performance_Result": 0.260917, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.7143, + "Performance_Result": 1.7252, "Performance_Units": "Samples/s" }, { From 749cccf90202991a558f99a51524b2df4f6dc126 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 22:07:39 +0000 Subject: [PATCH 252/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10059 -> 10056 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 67ef53e9e..2747be4c1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:01:56,695 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:01:56,695 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:01:56,695 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:07:33,570 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:07:33,570 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:07:33,570 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:01:56,695 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:07:33,570 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:01:56,695 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:07:33,570 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:01:56,695 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:01:56,695 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:01:56,695 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:01:56,695 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:01:56,695 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:01:56,695 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:01:56,695 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:01:57,645 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:07:33,570 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:07:33,570 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:07:33,570 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:07:33,570 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:07:33,570 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:07:33,570 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:07:33,570 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:07:34,811 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,15 +24,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.260917 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7252 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42114 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75359 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419877 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26386 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2137 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0526 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 180bd2482..74ca4fd3c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -879 +880 diff --git a/summary.csv b/summary.csv index db365bb06..d94a9fdc2 100644 --- a/summary.csv +++ b/summary.csv @@ -2,15 +2,15 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.260917,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7252,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42114,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75359,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419877,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26386,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2137,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0526,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6bd758cf7..bcd0cdc72 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7252, + "Result":1.75359, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.42114, + "Result":0.419877, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2137, - "Accuracy":"acc: 74.800", + "Result":10.0526, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index c263a59020c9792422209ed2f3722fc67cb4f27d..54e26155631e9ac28beab55461b50dbf5953a362 100644 GIT binary patch delta 3547 zcmV<14J7i*PRLHM%>xR#_GfW$1ONbf5|e%i8I$k>K>|K_voi#40)O3{TdQOdgsQD* z$u6QX)e6h1RlW+!!esH}w!OC9TJQdgkQQ;eGQXG+5kZ=9^I&Mj$mVwc89B**j>ypJ zzYpS?HE}u3hTQJ-ef!<2wdD&&cTyMEB6&5}oaIeedvtt41fHf)g5$Urvy1S zItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50e{Z@cLH>z0()AgX}$ZF z&IwzQ3CS{Ugj5!+VZMzv=M6JwFmJxaf5Fwqd=aH;Wec+Aa) zZ?G472VV^Lz+M;`GV={oW>Z=1QY-O00;m806ze`g9}Je3IG7) zlYIyxe^c8?8inuoSEy9ENU8=~-L}3cV2Y4Qf-!~=U}~=@$Cf{8Y|A6bg#7!FY-8;A z90?_xS+TRZNKEUz-Tk)p(5>zh9sK$*of!8)6pzE%p{Z(?X#}%g*dNb^hvv=o`#sMz z;-ow4cPHU2I5fWnvH9!KzYe~J(I4?B2ofW$e_$3LnxiC{@7L>bZxl?saV?w&v-HX! zjHcZr{V^KW<9QTx`^%u|q^>Q?sZYD(nYpTAKPpp4I2eq3!Mm`xm~x=Gso zcr>2J=FvfaoL(X9Qk%xK`*3JBOyldgpNz7K zjnTN@53+$ajouzMkg)nyOgb(8MV zK@@&9qGb#5yqkS9YG3PgSbACMr>r>Ze_MTBaP?SVqJBV-x7)V-cN``4A?s&5y1XNB+XsPdm<4$eIOFS(?=o^chD)3h@o#0iTM7`P@=`? zwuwZH?m3Z&(d`h47~KmZ5up7(Z#`anoS@ERK^q(JMGEIKrh_qZ#<@U=vcx@i z`PA{j(Z_#Z{&v+q?VP<{E{5WCH8dP2qs5n6FPzq=ll1Rty3$EM&4XxgH=YfGC|$1w zcS#T@ck$xObR1_(pZMgVl~b$k@W=@KTcuh%krf18@`dh_{S zZ@$<3AAJc-RW2lWA&u0K_gQo3jj(ylH4$?4bU zRln3~iBW4R-*a;&G?!E7B6Z?kdy5iRbmIgiKpl}t>WD;AM3H z4EIG2XWQX|i#CQg#Xz3Be-cR&BYJ#+5@m_ezjP}^ayYu{_Oj=hz4Y;ARn^`w1#`{$5YMJgpo zy!Cb_#YR)R&77sE&r$RdMFG)Ul(^z1J3$GcNF)+PB9SPf#1nVs2fqMC4EIGpPYc3Z zz5wM+7PLtNaw(JF3?6?jts=b=2wwSH2zs?<)3TMn`NLUM7Kd?JE@%{xyd@e}+`}hC z13WT~1d?eakwoK*A^Nb$8WD5&IYf0xl-zW@AR1+fsxP z+mcGuxJ^kV*{%mFi3X@78lVz2Fi~zaluE1ssVu0VrR2fF8I-hHBd(-UMs-LOPMB0A zXUwexULuxT(`Ck`wrc7Jz^o$2GV|uTV`{zkT=QHf=P6S0qo-qqr#ZGG!9(Ar#l0#lZ>St9_ZjOvgm*iS|V9w z(_xJOl0`-x1x_6(X|rC+0}_bpkSKyA6Tyy8m4+RO z?$~#%Cn(YU;Yj}u(Rk9pLo^6w2@4O?0F=Z=15g&Zcz;+U0A-PnM@gH_8I-hHF9Axh z>yRj1yE4NLtX(U1?EL*qzEB^}XSHqQ$vkyUt@EZzFNaDrf;iI4Lo}W=^AHVES;Ef4 zG(aWMxRS~uM-Qt2s4VjIC~3JlgOWCB1dlAixhNHk(X-rQQK~xbQSA0y__vrZE-R+`b8Y+@gW*d zdVGim$YdHxCeuhVnMRVSi#$H80!e031uYF8AAcxmlSY6{i0Y6iSjG|!E@O4Y9-sdT zn3W`1W+|(;Z9duIh~ww&`e>CyCR#=u>GL5PPx^d_2FWbp^I;kwlQ?MP1EwzW`LITS z%p#wU2A>a+N2R>0hM6$p<%F+ zWg=L~R_yb64$EYa5vhJ?JGPSCN-bcwO+MKgz;h&#^{tmUO@n9{oCK3eZ1lo(xPMlW zo6^;rt*+Gm?dl~^Px7?AaMmAZ|7*27S-r9<8PA4B{O9UrSWfmbEGw(5H@I3%0wei0 z4-U=XVV-_NS+b_le;BM@gJsO4aTtx0Z>!d`_i!yH-J@yPf4%I~VmAJB5xjoYd*$rE z>N#(IG4vbb&`48%l&0A&Tgg7KZa^I!%)7(jyc-S2v)GsfgEUdA=9)&dda>5>a}v&% zpKR0k5++GFef((@bo)V+{gnQ`8-!u<_(L`%n-`QF{ROk{5|{x58#9?1vx^j>0t36I zW3w<9M+yWiswWSVdm$fx%WA_g5JmU>3ZuO(+3q6<+kv)Z)j**sE!_!e;)q%gB(+Zd zzKWeV31rpf=-xSVXHYfYsxJ8j=cUm#$}@rz(A)^A+Zw&E7U>KnJ}@m7kMk_YCvO0!{MCj*Iddu*b{4K$kK*LsdN~>U zzR&t{u*Up3{#?Gi4XdV7?@z@6HI?9$J7B@Mro!h%JY2_oSjH2tC5Xpe)UUi978(0M zBryOT;~tfd^67acwLnclio2AQDVeQ`f|3VX-hS;zUi6=U>M-Qr#3NwLefbHRKeN3e zgaQS*_GfW$lSU;$8@q!GNKpy^0OU*n02lxO00000000000DysIlcFU>0qm3LB_aeH zGnpBa4ki`>c9SJ0K>-t!XC^uUg_EKtCL1iOCl4h7002+|000pH0000000000004l2 VW|QM4J^{LuA}1pTY$N~x000|mjI{s& delta 3593 zcmYk92T&7Aw1z{d;-w@MF%U`!MY;$ms0b1u0Ytdc0@6E|9+VacMIoVwPz4l25fD&{ z6e)@*RayX%E})?o6@*aa#rx*Id1q#4XJ==>J$wGMGw1Xe_Zz<)U}RrBh)oP&0sz9m zqzJb2q#Z_Wpip?-B_>}W?uB}zMK6ra+-4-*7U5jMmu3DO%a`%6cd^$e+^6N$??c6# z*6&yQj-B)nOrBi>u}}HLMH-KWdQiVGh&YSW2=0kWLcgbG&!+Kw<9t<^4S9WuyuMCm z9Nj_22Yr!A)hb-!X5)ndqscm=CP61qvd)T0(AAaf2x*NgO5;$R2RSxhuz<0)UAHpk z$K2j&JozXo|F>2U*NObdr18W)=Z%9gTGJZV_Fkd(6^a=2<*+d~n;D%~LFe6LW5VVR z9YZU0Rz#;0p}U466`Y(#_gHN|U(g;*9%MM%EAPbDp!<8dqT1TxVVQNHc$HD>!CW_R z5VvZtT5 zAWoJgyB&oM`O*$Kv^kiWT^%FGOO$`)3q)y19$B`GXZt z)k>Fw4)39*D>xK zO{5Z&=T1zQ3L~GGw_0Gc#3pLL=<&}`+1EUAY6XaQ?hOy}+y*5kGQW3ViW36b32^*M>U z-k}{&--8&#W76Y#_Mt>P7AFoSZii=tENUF=NQ>RTi=fWJGDmSSJD-+ny3ec`@z>Wd z*cbaNIBN)qyXT+HkT$%F$-WI05sEn8|K?}8IXXDG{>@3+*ixr6K-n5wzW2^ELIZXB zxKy@ack~*bC3tC46i?Z+_RnSt2t23QZqyynuuIt9+I(bwn=ERjP`fs1T@g8_iSE|d z&R1|;JPebsYb?jnsxLp;-lHbA`KyKF`Rm}eMw?aU!Ss~?*MR-U$h4qt(q}BjL?mtX z@DAUFlEhDCm75Zn<7EupGQh!5Nh{x(4KgJ3PAOGsJqs(< z)BV2gzbGx)=H}s@8Qru-cv4tVOSmn@>yw?|Q%d-wWmF|B6AYhgQU?p#4eSTrXtaTc z^VWfNyl+P>g{Rr1`&uadK6#HZYqVO`WODy3TX5Z~k~ePA0LlN$#8qJ8}-3 zEL*+6LuL@p)dTjbq$A1SKX6BU_7EUd0?)XbGFJ_BaAcWK+>-I?$3kQcxPd07RTSb% zs8Gb5;Ldw%-by&1eIlCkY{HqBHF&>*TJm!-codC_lh|ps3-ESeBl^wi9W8R@FcEHAOoH&|Fa011%DRIbn= z`@buOsXJ4_Ap|RGX@8QT&-fTu6k0dMl{#>*d+cxoGk__81``vuuR&#JKGs0bNP6A1 zi6KCkNFD$qGJ}$7EY4vKrc?uDsw<57ME$~(Q%i?4D7l)&BRVdblTEW^o5xOB?^+O@ zbGs|oQ+&2AY{qStK^=Zvr#L^fNo|nrfKQWBn0Qp#Bu4-_2u24o%S25wObNvdaMS)1 z9w{YPRG#W0>R+T-j*GKkhxhUA=<-Mp2^-y%lD972-qDlZO~zZmX%>>-s)+2&g~scD z+_e0EZuV|zMor2ANoD{CezV~zAuRAtRy|65C!de3A;`caB~mNd%=<-e`qEp6xf`g=a3l0@(_b(A;>KZJ3>U$LmcYiwLQh15C&MHw+1^_gD!hp#*sMp zxR~h!q+|@Mruo!8I1$$^iIO;z6{Ea8H}aag+zOpG@of&Sl#(7fT0kCgj)YL&=%}!{ zo?+!01!TO}Wi;W>4Z&H716+y8`l#%I^BMJMMFs=^^)%_bua)0$%Gt`QZ5*E%wVMbI zxo7TwFTR{nQNC9VHS}Z6Twx3-&W!iCw)#yBQe68WFWxiCfJ{#e-8f+j1{1W)swpJ^ z!y@5BVcW8b!NemLQY3zEy-jibK!q0nSoTA=o7@?C)z=c1kJ~M+UD%we-3@jiM1u6E z!thmscz!B%r9 zfQI>{XIC+y|7}2XRp!$_VOJl&^-3g_k0>l66S?TmUer<1k{m@U=jx@?Ka!EH%XddU zq;jH+hv!S&ffg!H1?Y0~Pu3}CdHA1yF57T$Z27hl-m7egE(oT%go~&6ZCmwuy^1vo zO1eU!>Zm-fr*U(nqu)_G>D-IMJ%V)PPse0HuCAI0hnk4AXPsT&EkpcTnsol9a^zs{ zYp1-SeKJjFIx7>7b{iZEeUj=sCiiV4WVZ;n$EiGWX%YG4kDtprDoD38Pw4XGzSm}S z;iqA(#I@XrL++Rzg3%7`r5!N$%TF=WEACm}X(^(_>u_LSoQ^lYi9xbzYT?SmPYm(+` zr=lT@%kO7#F$O0~+&}#Fqr9pdk&t+fXTEdveNY{K6^Y^+Zan_s07K)VyU3E`E zr_Et}S)?}h{yv#?j%t3=TV(q{tj?v7oM-ORhM!apxnjo+UnYgBa25sp@EH&L#b5oQ zI9!F}=tXvc^~WxpLd?XAuow9!me-dD6yqYDg&>}5sT7qKSJ}y%te>iiVnc%k@`H;O zLihzq4r!8J<9A2#7B9xFBaPqDevo`1_V3_lo~6#~X*Y@>CL4G6eYxX$m$JQ)YA>X& z9-gh+0Ov3;)%T%Db$DJfknyUUL!JCNeRd|qu5ueGL`(!X;MJ+b{a`zSy{JWgQqqUJ zfzhY8Jo@Gx>o6XQBoKye74F$wFduT^<3NNEmhUNgq zv^Q{|~(<`%|_5Dct|FsOV4$G<}YW_9T6g)MRKQiAagF_`&~d{SWX?o2>u< diff --git a/summary_results.json b/summary_results.json index 9fd18c3e0..8c074af27 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2137, + "Performance_Result": 10.0526, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.42114, + "Performance_Result": 0.419877, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.7252, + "Performance_Result": 1.75359, "Performance_Units": "Samples/s" }, { From 9f87b47b4276d0577e9da20d87a6fd0d5f3a45c4 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 22:36:27 +0000 Subject: [PATCH 253/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 10056 -> 10055 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 2747be4c1..2ba22e5e8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:07:33,570 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:07:33,570 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:07:33,570 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:36:22,066 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:36:22,066 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:36:22,066 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:07:33,570 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:36:22,066 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:07:33,570 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:36:22,066 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:07:33,570 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:07:33,570 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:07:33,570 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:07:33,570 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:07:33,570 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:07:33,570 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:07:33,570 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:07:34,811 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:36:22,066 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:36:22,066 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:36:22,066 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:36:22,066 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:36:22,066 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:36:22,066 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:36:22,066 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:36:23,276 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.260917 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75359 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73407 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419877 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26386 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -32,13 +32,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0526 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2831 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.44052 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62118 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0168 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436315 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0168 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.429845 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414229 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 74ca4fd3c..3df31cd1c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -880 +881 diff --git a/summary.csv b/summary.csv index d94a9fdc2..d0c743fb0 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.260917,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75359,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73407,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419877,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26386,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -10,13 +10,13 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0526,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2831,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.44052,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62118,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0168,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436315,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0168,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.429845,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.414229,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index bcd0cdc72..1418e98ed 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.75359, + "Result":1.73407, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0526, - "Accuracy":"acc: 75.400", + "Result":10.2831, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.44052, + "Result":9.62118, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -484,7 +484,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436315, + "Result":0.429845, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 54e26155631e9ac28beab55461b50dbf5953a362..eec82af88aebf64aeefc7c252fcc86eaccc18121 100644 GIT binary patch delta 4167 zcmYk9c{J4D|HsGJNintzvJHu`CN%bpWsrR*OH7um+1IzF%#5_+*4cXWH^f~AEJ)hUP=bn4dz4vkM^YwcD@p|>@_vlv*kkeiJuwMCy0t5<$5F%(e z2wUXTz=YQ4hqFeNd*m zoyZGz?VP%OF}wZ56MOrS$BM%-J#fyefuqti17No9@%s+pdAYKRu|6{{1$kA^EjO z7P88CTr@okEW$gfo)YErd*H$*S-9`oYRvY%TSwA^jZ*8F>hR)iKFS2DZn$&+Vovhn-TlQ8p#2^qi~8Y22U4q6?Gl1WO2}BHIbf(}MbjtM zy!`FS_%d-hM)(|?v2n*kW4yc2<6$^8vP?jcae;@)gxeRZHLg~8`epbbt=$n5m~8JF zA7k>m6qt;3KoK3)7Z9)dejTCCyCFNz6&=+8M60T4q4XY^+6{H^1qCW}f?Z)8S*k0P_5OEkqGzx$GTY-sy|$+e&H|3lj?PgVAwl z3>>*c!>w2NlzX>-kF*yWNlD$^6f0Xgc5D6zFPEeRBSnU9P+Lc0+|v2v3%zc3Ig1ms z?Dy!rDr{FxjvRSfmK#4{0*sUQzRF=6h7a;v_v*(DQY_r>?hk?bjj?Qvo0Pa^67y@_ zRrJQt8DHmyz4(8?*Pt-zRO#*CH@Obua zc7dV~Nz7hcy8|@iKQ>eZu8$8Haaivi=-Uzy;>}aZ*{>^IJS#$N0OHy#acz}IKDwoi zsr@OEq?*6XNW;PnjwY(}8v0Li6V;au{S}te?+GL1CBJaH5@T~XUy#?dYE;B*PX(M~ zh$qi-lzE#l8)?h%=3kANO^<+Ul?Ve1Zdb-ce{FhrtpEks45OOA%1C3%4JK3A$!t&$ zpy757tB#FaQ5yr=qpLUa859J28;5T@@HJ{we5mg|u^oH9Bh8FfkJJ`rmdHJ61x)|~ z?}xNE$-z0HoZ#Bm#dDq>+I6R&61-PZ+2|2O)Cbgi=XUqS3D<}z$?CAtW1CYs+lWst zjuqyzDrE+jkvU>C=T3k@zva$l`ichC}9OIqQHO}^wDWGPT96c~^1~aR*xx>>MI8@Z_*#4*5&(467m@Z| zrg##gPI83W_nbMWnt43yK8pmB8EV~j*?b}TfIYYI8l8E;N2?wYT~Fd!mb-OgJ!$8+ zC#1q@^_6+LQux=qOh+}cgi?qv1X zf!$xWLGlkybLiAwosA|;eFuX~)5m_KL2U7g5VwP$R* z{UtRA<=KUe(%vqL=OL8kj}JHdrz^jgydZt;?=sA(jEkFdt_2Q_*Pe_X`{V!Q6pOyt z9aY3q1AF)fc>h<{6@8UK$er1^%5P2=dYgWgt%Oj_ht$MW7a>N~H zk3+>s4P{4_a(SD3i%(m8WW%s*B-lN@U$4!AE_PS14A_5)O!n^~d<*w8#53D@xRc1t9q= z|GDBt767wV4YOvP`ruUdKfAU}A{tKi#~duzkqNgxK9L|Ep- z&4NRpLM+7J8iHXE94ti}hC}kBcImWXGx*M!Ew(svoE;hN>Sb6c9tjb6uuh*UtSkQV z9dH?ER%9I0Y2)o;>>*P(@yROwm`j7lh=s9511Bn8!<-6=NAeq8Ao+C^ebJFy^oE^2 zfc@@HE$_Ppdc#3ywQL#QbfR=|tkg${n7w2|>EXy5wg$Hb_U`(9!tXh{%X& z^v~yQPx9^kYK?RE4h?+So_9IgrqM$wVcxiI)~{Q%!=NNRfoNwa3O0OrAFJ#k^9*Gk z{kWzy=F8$wDqUeWl4d7YA($9D^O1?U=PzN29 zFTUKF4H~sV#R+b`vkUU*D!FG;xwiBvIbJ1D#r!a1vR?3csc$x3XZ!87QW!KJ?;54^ z`8~36*ifVzTvU@ElZNM{cQG7!knk3-!!gEZB=2aPk|Wgs3C3A^-A(66|6X1O@PJ71 zT>YQ2*AmP2NK08uh}%Ia@!m?@qCWF+Ns{sZmXSB@Hog9)Y}2R74xzH^0xP10&n0$Vw(LcZ>Fo{XCxPIJbUkAC9*j_`CWNEPfI%G~w_5Rf z-!=5`nngGA{|zKF;X-@pimuRK@hKC!Lff#^z&bRBu`u7!r>;?r0 zag4f_i`cn|+qzb-AN3;pu>ujw$T?Krm0{jY!Ej|ga%Tubveo>KBP%FVLz#y`nMc@- zpy}`xLPWsmmh?pMe#MpFmUdcP|NPU67DT0G-rTDn){=8id-UiUL6nIDaE~UE+g;tYiga&^xL!S6O;3yn2KR<7GXf-WKnhH3UsoT8e2M=yaQ5R~`!~S68@q}{zW}HcJ(<{y>J^i4vbx`66Pg68`ZK{-wjFpbpF}M5Ec0{ zpGBNU*Cyl&0Qlde*71KPKeC`Mlk0!P$Z*H%!$OA)NKk5}_XXtM{I+8PC{;t8he7;b zWvP^=F$0*mcAYewgP=|I(9&%0ItHjrPB?H?Dz@G z(7RvAUffX^|0A_>$|K$*AvnodEn&L*oO1o$8u*yPU!EvSLu$o$NQ@) zDr%;Fx=YbMo1ajWmx;xMgvQl4-z_|sv}~d3&(NBagkGOEf9xL}EZ1F-Qv65wF3F}IAK@^GKx71A+C$`^i7wSrv{Y_H^MP9l zt|EK)UoR}dawJs7i+>C1;eCz|Jz4xu)KVjKfoX;hk|C#fhQ38gv=%@{T8CtiQPh4! z5lC1TBA8s^fdfg5U5kz)s+rr05X{E|47hhlGCtmnpet+^vnl4InTW2R|EyXdF4jWp zM!;IDXuET_MX+A?h4kL&8T%Be&|GT0Zkm1?y)&{5WwlC#{PTmROzzgPYx)Y8pt7Wx zrHjmGS5Ok$fO?nMLU()iO*Y%)p9Gqb+($Nro-}Fq*N@m(cC7rDg)0+G;GH9$Ko$BF z;|U@t<~dSE)3RNiA~#WB8fLn2__(&Ml9MdPX<0S;@-=p==%Fc){!unV;iDN-=}T;0 z)&{flQz5rGPFjbKAJulBQvR`SX-#@nY`kgEMRx1?&r}76_P50!(lr!qd%`WdnBNFu zOZZmhrV^iW9jn5mKuVoTH}V#CbOoD{41Il0S?O-d-n0b^&mh|^JRxHMRry?UGw%9( zzJtB%u7Ai#P5h6+l>av81a(0*u78VRIuK}H4+Oe;Q6B%lMMb;N5Ta?IgeE}|vNghj zAV23{-#@(Q=>OKz7vp~x6&*m}7gA$@aDYI;9uj{3As%QyLbVVJ;js`D{Fswaz{yIe p6Jq528!16D5XcY=0x|vXIS3RLjVAmQQYLF8@LU(5dI8V#{f4ks?mEh^u9Z=47TMQC{wOvtT}>Y@(&ebCv3hl3 zb#nVbn?q0I?kljJ77Z<5JRZE#TH6iWvJ#(EY z+u;}M``j9Iif7{Lbkch3WVVtCj)Z8Gs&xH)L45!9ALa}bCF4Vmh<3r<#Ora2N7*vG z6GeYe-PbWbt~lF66)7kKw3|9Ew8sUt=Cl9vHfTEB+3D-3Lf*~CYH(1VZr}G^dDMLj zK3c;s96Ay)*`$%&Gg6yPppy_R6TFp4(2B7WL}lPuD`rX-Lsp~u1sz;d(-h_sC{qL6 z$0ZwHu7ex7le-B7nNH{W2?#G8jDwjoP}XcCI5ybNaG<#Y{qDAmUva3Z%Y>h8C8wl4 zZ(f}K(!h|PQRpO&PAk5NyCk#fMl42rb7Wj`YKHf*LOSEaAxML3yOpM|JFZaVrf&>S zGad!oSfIS0Vt zAp0l?eO!YuJy5sE7)bqil9PJ7Qh-bktUqN0Qfp)IDxfNX`wH!};~!vP1I+9bAa$NU zRz>BxJA^)GlZTpWyg`$n4>#E+ob-_!P@y~`z++Hxk(%m|R}dV%^>>gA14||j$b#(! z6g0ova#{aKec+btIbOSP~_fP7&gs3wHJIil{E&0A6F&%vg>Y(sU;m*N?q z`0*=KQKIuS$jNOb{-hs+X67%}Ze3KjXn)nM7m5TfJZ#BFCqEl!{n=q$>}nYTF{mmNl3(j=&X(@n&Fq9;>^M+jw>@a0|9O4n zn1*WL)Q8$DliJ_nqf^82`+at2^N6$Cq1VILO$roO(Xk>(gsm7&kiynQy9itwMPv$e3$nOX@J;4ZKR!r7n#deXN zW*(A*Ed8Yzn-{*Pss8Jr^_F|dWk0$HmkydaIHh)(Odj9&&y8Y?F8SOhWk#R`6^?{( zbK=qI-j+qC1@iMb#k69StJ4J%xnSw}uLwAf-q>e{BAadE;%0)HdmH&#naV(i2FMBk z0O%Rc8)ZNpmQl-)V&z@H94;5~*@SGe;e&9GzWsPc^PRn(E?*uHfnc^D^ z4O{KsKfX06y_k8E;v>9(l#_0v{d7nfZUQxi4)bk`&_e|aZQ1z}GT`xBlN1kkR^2)e zu_U^(y&nFE>{?W zrYKh3TGW?QU+Lt|)nQdkl1=Z?4fA+dJ-$6!bcc5>1f97`ozHN&JQSFW`5>t9}DCtmVwq!_swiGU)X2(V8-Nf<{j9I z*;3G5707A=ZIvbMez|g@p3~eFroa;Kux53ZynJRwta*6 z!HZjQ_JT>OT5S&a^yH*q@yNWRzbt$P)}fbuH`$40v7(EWt+8v*ykoyCoZF;Jq9W(* z2VK{MwNH`rxz%CwQ2g&XeVp2*shcoJ^O**`-@4l$22l*;hqjDr!XQ3FR|vQ?cL@IX z*?>WM$9GI6UM*r%o`4(P0OL7k8DWqNe-!+omofz3<}MdD{}g0m+sh3?yyci;iIG-o z+`_3XA9E9M0rMH{EC(%NnimgW|0b048_A2&yWg_$Pf8uISLNpZsm?%_%En%M_PIs{ zU)p}l=WuMhWw;ab;LLkmX(Oq@Mm%)UOGf%1VN6Cxtc66<#7ENJmQ(ErO1U5F`1D3H zQ;3LlF|7{@EbOzr*z5I=gHFot&AB4;eKL3$p*+P(%ADk`3o@=r|I z^4mSMac`VrbSMjRyrwhKF*axMe3H=tyFGN$@q-TKqloXnLAOs8U!-hGFsgNA8YvR{ zgXvzIB%3ykIpYnVh|YU@3Q5NrlVQ&nZT#Or?|Ur)5h^?ct#zE5ktTc*bo!0mK(m#$ z{3Gpx*rfw?ki*PE$Q#+v8yzaibx;&k&@UhuipCE&d0ka|CRa=ORZ5O;*xN>%j?YQb zK06b;m@)T#(`GU$p<>c9XD@B_1#7+*CJPvTW0O6csd&}jvdP^?Yflc(OB8WStrGEA z31LL2zzAAUyc$spPE;!nQak4bB)z+xT=aLgXxxHIx-bfU?cwLfbB~vedXCA_X6m8{ zt@?d$p$-k>y8fiBoI%4qh&t7g>t@6V)^2Ju2z1C-w8(30M- z;_+zdyQaf7a})M34|8!x*64Z!*Zr?;!fG^%QjtKS6P%#7^hV&QWLQ$joEF-X&R~)F zh8vld+0O;>{4KBcTr1`)#;cy63ysf{s5<&1H5HJ+dR_cjRZ@SeQv|fRY1*$t?%2vM zBVN+^_M%~PDqO!V+iG$!UVv@hTlrpfzuOE^d#w5{3Nc+oCyKw21LsPG%19czNylgx z0CJHzsaeY{t(-CC=7L3;MjFqxSoPve;0}F-KpWV5%Te^_tV(ChI$d)CtW#=twf2#7 z!-3VVl`#2=(RRkiN3TE1R|>YMMLyAyG{o9MnSxN#03m0vt~F5TUXU%j$8)6Ggo}pz3=JiOZx0l%{tjUwG%9g_Bv!X42ZV=3#p*7jL^hn$tO8Kv z){X>iRr_bk)_Sqiin%8VXH-J@T}5NqtcFutbI(y)mpLVux5}E3gGlM`Bos3OLj%Q~ z8yEs-X?d&vE|dqqJnx}xgbK;Z`TA?i(R#P4kEhOHbIPYcSY-HQLF*4_*mtD)jx;^` zA|vxCAf{b|#(=%X*Gv$ACPZnX68ryFT!Kowgc2=cRVUR3DzLul{o~|Rx6y!a(4oS0 zxm)r1-NM~$ZHX{#=@7Y*Ud$coWa(WnlaYDrK%2A*Vz`>v1Jh*A2jn1k_C%a&Vu_3T}U}sHF(;(S@_$1AJ@U4?LB%pA`XGnk(MyO2^xF5*griyC|rI{YjVN^k~ppOi*Q;n^Q-OF*t3Ujt6=Lf9`DXiqp*(_by$->V!q!S^ScCUZ z4<(sYP`V0|*6cL_|JJrw6WdFt{UsBErg~?)n`T!+M8nLw@BP?)0qV7nhUC6j(q=o$`jK6MPtl zKTj%dnm3e+at?-SoRHViW_rJ*kXQ2yZd6<7QVSl+GK2cR*(JZcV_up)jNvG-!J_*z zJUAjr?)ejC2T#R>W;2|oFbzM(9?G71Lh4_R`>*_HT9N?%SJ(Efo|~z&Y$^_jl{U;@P0+Yn?)Pb8#}hgbo8mJ18!Cj73p{Z;rzO>x)0neH z?pJm@zwgMx6EDZ`oYIz2pm9_{?(6ZKWnrx8)HHS3kIImjH5q`&JYi;%j{qO0*SSp+ ztbTCHkkKGEy(vgP6I^LOu99{|Z1$qlBF=&tUZHo^82&--qW$>F_c?h_2LqY;Jn=(X za@Md+&-D=K&{z808Loz6_O(k0B}p+e2k9|KI^O=o#Bj@=NbG$FWHo<_FeLNVN~K03 z7}+tk%OtW|_I=3Eiu08wZb`m3%&tKeKAAXUfku+S2c! zl6m(n=B8~+b#^ORUD2vD!0a1{}S>k;cwX^mDQK52IHM}k+ z=1P2FcnFcsV(#UJrp;bT(7NREe7m4+U*O@z1m&_C*ehz+Xz+ee{uMnjF)u* z0FHA&{x5=p(WGRAI@J%-G6Kf^*Y+pppZUKa{gGi?| z?MW&TVkKdPxTs%qlTx|aNaaG{fBqH;K?ML9Py+yv|FX~fpy0;>q+kJ-zne&%FfikP SYeB)aBwk^`3oh`#Gyeksj=2E< diff --git a/summary_results.json b/summary_results.json index 8c074af27..fc9e39bb9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0526, + "Performance_Result": 10.2831, "Performance_Units": "Samples/s" }, { @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.44052, + "Performance_Result": 9.62118, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.75359, + "Performance_Result": 1.73407, "Performance_Units": "Samples/s" }, { @@ -653,7 +653,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436315, + "Performance_Result": 0.429845, "Performance_Units": "Samples/s" }, { From 76e92269cc21ba9dd6450453fdb32d51b4208dd0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 22:37:12 +0000 Subject: [PATCH 254/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10055 -> 10055 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 2ba22e5e8..51d0f8b91 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:36:22,066 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:36:22,066 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:36:22,066 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:37:07,471 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:37:07,471 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:37:07,471 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:36:22,066 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:37:07,471 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:36:22,066 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:37:07,471 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:36:22,066 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:36:22,066 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:36:22,066 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:36:22,066 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:36:22,066 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:36:22,066 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:36:22,066 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:36:23,276 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:37:07,471 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:37:07,471 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:37:07,471 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:37:07,471 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:37:07,471 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:37:07,471 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:37:07,471 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:37:08,610 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 3df31cd1c..98da32e0a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -881 +882 diff --git a/summary.xlsx b/summary.xlsx index eec82af88aebf64aeefc7c252fcc86eaccc18121..8bc57edc7c9f235eeda74607d5cccaa8a51da494 100644 GIT binary patch delta 206 zcmV;<05Si^PRCBLb0GworF40dcOffH*`{p}wgYX+sewXWT5=Q8CK0uMkkmSU`zlW2 zB#={&tC{_Fc2PCosV?~i=cUm#$}@rz(A)^AH#Pcb7wHToJ}@mO+kwLl$0r%wM9Y8M@nX^{m85SGf*9e{F`_J9CBZOg60pixgrY$1ev9D Id6UQ{e{aiJp#T5? delta 206 zcmV;<05Si^PRCBLb0GwPyB%4RcOffHTc#lh+kv*^)IgywExie8lZaYBNNOFweHABh z638jX)y#f7yQo_1RG0jL^U~-V6RmJt7|j1^!ySlVBnm z0mYMhA{rz_4av!4nwM!lZwaC0j1uy_A9>Y(2CCzbyNM^jA@}tsX#N4SxgrY$1c198 IS(C^nf2r74i2wiq From 86fb0aff44787bcab43f9aa71a240b2f0c21aa02 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 22:39:40 +0000 Subject: [PATCH 255/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10055 -> 10054 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 51d0f8b91..33c630764 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:37:07,471 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:37:07,471 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:37:07,471 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:39:35,232 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:39:35,232 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:39:35,232 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:37:07,471 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:39:35,232 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:37:07,471 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:39:35,232 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:37:07,471 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:37:07,471 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:37:07,471 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:37:07,471 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:37:07,471 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:37:07,471 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:37:07,471 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:37:08,610 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:39:35,232 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:39:35,232 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:39:35,232 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:39:35,232 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:39:35,232 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:39:35,232 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:39:35,232 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:39:36,408 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,14 +25,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73407 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419877 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420835 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26386 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2831 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2182 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62118 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 98da32e0a..d1935688f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -882 +883 diff --git a/summary.csv b/summary.csv index d0c743fb0..5ee0cc7bb 100644 --- a/summary.csv +++ b/summary.csv @@ -3,14 +3,14 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73407,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419877,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420835,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26386,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2831,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2182,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62118,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1418e98ed..2e935031d 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419877, + "Result":0.420835, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2831, - "Accuracy":"acc: 74.800", + "Result":10.2182, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8bc57edc7c9f235eeda74607d5cccaa8a51da494..5f9e9039d1b37d773b7a3040c0dfd99b786892e5 100644 GIT binary patch delta 5341 zcmY+I2Q*ww*vA(uHbmPXO0>0$uxdn)kZjmRbfOciF1qMfNwlcDIzfmMR`0x8bfO1Q z5{VMM3qcYe?|0tsoNvxOGtasA&TpRk%rp1Qe|pt=)GCHZ$k}`Dl(>RHAYUjRfH2|r zNXP&q=R8Tezy-5l4R%92s2~OsgNrU$Z&?=@dS0teOLUV!^wE4`u#KGsr{?aT&a|zP zAAo81UugV3BIo_`HwuWuo)=cXs-Cp*k# zC6S6ruuy~^s5@3j7H~>gKTAq57E@HlU3(Zda?{YoT{gPNzu-Vb85;M|dD#45En~`C zzxi)C&gSpVEKSEj@H;pi97iflnKXl|+}-&8>UI_V_AQj@O=DI(Mh7VLEjPk;Kw#5kz<};RueFiMHMX~iir0WiQ z;yU&8%4!#xat2x|Wi%oiX&$m2G*Kx|%ypBoO zimf#g{XEBJEuBkYF-qIw)Mc#X?Hp0cE90HW z8nYY3>xp7Vof>ud-?HEz5M;wXU25h~X8j(@3%pLFtIWaDNj|;`Bb`#bn*?1Mg#cb046T> z3ze7mbf$$A?XVIPO|~d;ut!XH@?;jWfNCF%bFymGUMD6xW#giW-@P)jU^K!R!{Rhn zoYG2b=FBErIcCqZoTd)>C50zhJrqjII9IQI4{8kIXpe3_r}vVTyExgg5^rf}oGqUN z3!cBEr^nPfh9j%1bpBccRb=r`N$)I?4q0jqsD&ZH-z_>yu!@cLOo9k9XLf2E8rRw$iVGv!TF`Nm?vY4Jhx5S&Ca0B0spF5uQa%edhdYIhnfejSl zf)S^?9h%#A_xK3Q7x8#(AL0 z$_^mlD5_xjF-{s@X)y8Q=><;dmsF4ILjz~Rb3;kAzXWqj_ce}5acmlO@$v4OTmZaH zBYo@CsFTqJ=$mbifZe^&(&1j+)y-lO$`1)t>p`!)LgaHcXdpVQ#KFanILvM24qp^e zy$y8~@){KiD3S*LjoGt8u#|)}84_L}q5GNPG`pn9-e4uyzMt9-yV!jq$6$Wa2HCqi zeojI({O1L+$5!3uFTwcipF2jVI-x8UMF)$_~-A{J@z&`JuK$QG-au5!V8KG3TQw2 zJ_BS~*wQ7YcTZ#|BJeweg*Pv&zm5K8==xEvbRqqVSrO>U<3LB(03incwy`*?R~{SkK_EI!s%Z4BL^`;?;-Enaf-qO#>6ODJKL-GuDfn$piV)oi!kF!$ctyr zB9zws#W>zn4}8t}zO8KBP+sU?$(H05OH{|VQCYonp0>>bg9p<3bpbL|tvyEBF~}sL zXTOw=S84!T!?YSIVD@D2xv`?+14fTq_+`G|XCjQ$N*HNJFNSdnp8W%LOKG)Ut&EZ)Mzn6EIk59_-nD)uHFd^%E_`q=kRYRt5s432pn`}t>p zH&f4wC$ZEEp7@Ca!MfE7o@CkGT&(S*qi^jyGU)zU9mS&4 z$6i|$`$p9%9ayi7X?vgA=^Tmry~BfoUE5+!fS3bZxF2Q9_Y)vlax{_O*0~ZQm^_r> z7HdW*SYX@TL|c(2Cm3y>pBmi4Lk0y?JPAQ}t(ih28NaBb7Wb~NUS9H~b(t=_u3ha} zIqx_R0+iseg@p*z@;MtKE0v6$4Y4fL759!bVF0jG@ZQ{?T_X16rkMR4B%~#oBElM- zz&}i?oQU`_zoOksro?^`a5|@ZUgLi3+sG6(@nzH{Gn&E7zQx5bTW$P7;kp{nA2I6O zjxXNgkX(;8hIxcS=6v^Sp$XSar+~#~zh>)6JTD|UgWy}2Qk-3{KDDn&Q)rah(E4?7 z%gNHdb#rLdujSN2D07G{w;PF((DrkfG!e?A?0Ix}tC}7DWgjlSaQhRPRpVcs$I9yk z_t4FCzs=Oz>Q>6O9=qHdx@^+G&T7JZ)#d*vLf4Ub2w96rL1t8V{O}8c3z;;Z>>e?-0vfRB1DuFZg20xcj7O zkeN+()n-?v1YHgedHR(kMeT(sEC!NcJiP|WJP1ZI6HU( zYUwrSS;d$zw*e`1AeY=Q<24KN?YkjnT8Cd!C{J{v|C+ag*Z@@5?D@|NYz1&y!)tAP z9BQ65+{;U~v!gqtGTpsfP>s8x^3A7Z=W~>LbAj>nhyf=(uoXzy6KQr*;nQ3aMCbi`nqb#7jJmU6Ct zuJ_>4xaBLGxH|q0_$og3{{c@bV_?DHi@YF~V4#a1>|ivK32*<8Srg_kp%N zMvr6j+a0%pW*i%ar@(Wh`0RCdep$Zy``5Ycjh}`@GK2B9Ak|zV;r@NOpcy1FUI%2( zpffsaj4!9+1uAEXZl6RxlnLHe;nrO8#bG{i;87&1P)gxQW-|=;HjEp?HBP6Yqu}7a zABiy-w{AH?;ID$#w~e^z166J|phc55ubwi9tyAM`D*%G}`KtImE$y5a z^rz+(Ck7sg_GafroGUVWMD!$4vF0kkRlGe?uL1FJK}LZDWYv0wBB&ph7fW17l~`jS zv_f?KG7;z$aFO34)swx{r--=pc?ZY zZ+|2P!Ct4Jo&Y6wDUxU~GZsm?|idpC)Jh-zt`69KdL~h z2K$Ijt z_WC9#vpTb1#I;J~v+ZZh?KoS|AOL$oT>`2=9-QzB|B}aKh=nL|ecR{Mv?4R=S;^z8 zJC*HS8Iii}^WK`hcKaCfqSAC?&<9*1pjM#R7QuC2?$XIP>M>B7Dp$W1GlqKtE$W%j z8?MXIk1Wo~?^q&1IE_Q%uWPbGN$2!)JDdn*pO+5Jh^le$K}YP*x*;Aw=C-T9j17N@ zE>)T>lk2;Ya9i2+Lr!LR^~;fPyP(Uo5q<41MF zqCfXdYAbs0ih9?fiAJYf8ovT-!ltd?DH++rO|sxINyhnkMy zdUM4p{FjyaA8Yx4S!(Q*YV222(yjs0WMv?=N_IV4i+UqBl5m5j}W7kUdv{SEa!}^;!;`o7G1v_qDMcfi|eBZzZo|z~NYu(i9x6MmMaNInU*Wy}QQF^)$c zJR0;8GR7w}O3Oa|-QGZGs2|j{xJqJ)!(ak<>L$Z@4x zHxXOn`>{?w0%@E;n;QpHPZhd~Q=P7`RBl5rJGtq?37>CEiBQ)Mob4$1q7WaBR6dxr zTirRH-;;@;NLLcWN@aSn1;1#e{JZ8Z7;l_@I58Yb(;bQ3-y;$oc%mmSC?LXrmv4E& z4ZEKUX5J?Q0SWmGl}^(GwzwA>eZ|I^UJ*hD#j>aMA>SD~yhfRhd_>n?chJA)*fz2l zt+zL)7=F}emsY_Yn8Fhx(ZRIGNB>0ZiJfBP_Zbx~NO}))CIVLw*0!Jg#HcYhsyahb z4Szd*PvD$Lsy>l*$WHIws6}eDAq?lEfTukp{xN_O1uEAFQPBO`)0Oyq!f&cLa)a8B zP|h4j=HV$Z%(^J%=37DEW#TGsUxAT_Glw-+oG!z37v0(9mW&U{5%e3qI;Y;l7W?>Q zWv%iw4GGs?PWw_ij6sr^4)N{a_%Z&#uyLiE?y7SYPAm>CxgE_XNgiv+) z8k@f9#owbF;}n^G`Hba?wz7ZxB4kIlVf8jQ0E6pjp&Tds(PI>qzF)DYm-`Em9-hAL z0-KxBqc`Q%G^X#8OjCHZ=Yxr>H!Y~&&SmT_ZJt_4X9@hUc{(xZ+A=%Uykk+V1n-UXO9-LWkE9_vUEK8iEIuUCq??@7G zb|oy^g*%)vPaWMO9M!UxMc#5)>`wQt13Hyy=wTEsjtYcUZUzW^Nlo=tjmiSMt5op66R(fumJ*MO!=1@)+jwnK|2H*+j1+P!{to24D z$})@nAM7=GNvxf<@=s@}D~?#hdWEwkvvz4lyqa3iX1`KNYLRXPxeLD7s{e>m4<0}S z=fVH>8?{(3WOO$&{wBLpmUuaC=4xT4M-cE{NX%{g3r==m(7scc=JClN^1K#-2WZTz zZF!?A`y7pw-Ne7S8cd?>}EFJEiz`cV)+x|`LKi>i;041sJg%P zqW5*4+J`qmPc0j9kzHtcef%k>KAkjmrsGGQICc8jK8IWY>Xb5M5 zA;EHO<-?s=9H&*e>eL0u;|6SlFo`EDpnbWRLi&2c}5Do zc4AbU+W&fAwn4#GocLhQn^!O4dto#b|2+!|Zosqga9y*9 HU)}j1vk3Yu delta 5329 zcmY+IcQhP8x4_q`A%rEwve8yAK}4_7S)HiSMen_CkYFY1>LggbTQy3EwptKGL=Vw> z@8U_$`QH29{4rt9a-cmGWC8LDSR>3(} zKd2k;9ZEQ-my1m&12L^6LvGf$f5AW8#J3%Z4k|okz>UGf2(<_W9N+i+S!HrdGj7}F zuJ%R>o^y{Fhu8+|K8io{>8Y$lYrJdBLN3jVC$#UbeVjMwl1X?i5b>Ui~p zP3LK^JC7^RC&wUsxe_LE(iJ)~9f*6B;*?C@^^Z}30Q}X+vK(!b7E{H=dC7EebBHL<;9LTj2mp;ej#I(%!D51jW1D zK&*++vh+F4X9UZTgXZ%$p5E9wr;4KaxTS*6>FF2l#1AE8?glzIz*pl_;4m(^H3A4M z?;YfL@TB%6AOsg5(;O~XJn?LKmY$YZpA;v}-M76fHe&5|?gI=%C8U3)m-)peyE>|; zCQArACBvQ(t^=RK?kmfp9OW8jH0o227{*z##0>g_Jtrvkr*w+!(@{Bf4pOiQo3zQ_ zqtRxXJieYTJSk75G@a#TeaPxd6yj53uJx)J4Te8v70%Rw%jV#e)m+h37%Hax&Ny-x z+T$UDso7mACf2DDEo!5aGj$WBC{N3Ld}c+Zt#f&RF&wqEgxcCfVUO=CMb`b|h?CA) zCncbO07Fr-^qO9C5R~k?rkB`y>Pt3JVcs7QJ5+cU%?Iq7Hu>_f!+G!9NYvH)EJ^z2 z$la86u9lsq$UP_T$CaWQ`Vc$PST7|at~w?hpkAbO&L$~=E(C}rc9h;I>rDW$4n&8C zY{*Q&J3`UBImBYjeNCex<_t~p<(&EpQOcdlU#vSER%h&j=XueAk-T{Q}HUTh^-Ei>qYN6|(zez21y#={@p| z0>4%n`(!Cfa-p~qcd(3es(d7!l~EMP*W8fhXMc~ z0uTTI04adx%n=*XkAM}fGou84yLYBvtlOkiif266(8gvS9j_e57eQ+7P{r1pBIV#T z$khMl`YOT6YWs}#9&@H4y0(elXRCGYrs4P-_lXRCyCjSi6HMCk+s((WG{XkmQBHN* zTf3O+?|Bn4@-#+xWB77TS(u1q$W_MaPrz|ic$LMmqBvn#w-P5jttWDoO?ma|Bw&^X zcihEXHk$tee%C8ohR<%GCvfn%yJ*U^K;c}Z zY^?>@toOz`Nzds$<|ZVbxX%8V(jBA@H-l(z5x&SBFzn?}bw=G}I2gq?)E*r;gUYQo z%U+~P1Wl?nE8=UNPei3~cOSXLpLTOmqx-(d${0-u~r@MrTNb3BH=?dV=tsq*315937=XLX&4WDm>cDm zT%lqcf_}}!#nK2bEr!dB3&Wn1+UQK%hmG>lA7Gi{<1F2d3G5s}2^+Q1Tec5{isM=9 zi(@~)oHI7y=$JKKlP*_;T-F&mBN_H-tP)OjJDIYl{^3U@91XfCx&7k=rS6AbDp1RH zV>!m*1!|SLM{|M+QKJ&9Aiib@BUxUc4Aj!=2}17uE-Rz^Iqup%(QqiClYd8UHyL(d zSYkxq86+Vp2+jmb*b7YZTM|9rXJcq5l}e`qf#gTtc=d!cHZc|z`!Fcg1 z3=31SX{CHx<;fP66uvy)8(gTIF8WYAIoP9_RT&ZS+q&*td+Y7^rB~vgtU}HY$K&Er zc)q`<=%dC?jNDRj+&!qCw847a8;%isyE_?r8Yp@Lo|?@mqtcLhNyA=6nJgZQrON>B z+Qt%#i%OyFy_3~5NZys&d%KAn&cX=UoP2hJjM1HIi)LV|R5_wHz z3U55Q@`;p-L#{I*mVg28K>9qNG=WeM0-B%%MaVHK9}_A;7Zba~_9-K<5vEv(Mfc z4#}t@?rsyZPo&xHW_Em3N8h?aqzNCkmdO;LPet(;hVu=8xXpNDiqFTYC>!k?sW5@a z-I@sek&CIVF^)0L&|k^zZ*$B%>$E?foNKsKCihsFq!58|p{@wb!^!=7qkuRVQ>Zmq z89I7y871i?@Lu^v=$o42upg_x@Ko9CYZba_LrAnBAZ9vIQ_`3kxrtQc8#tWWNm4DE zI`xJ=t!UDU0^j$nbhY4U$#>jPIf8k=)zsIir|6|l<<{EQ_-H8~sTb#Ia}CVN#qODj zDuRNy=m53Rw1ZzWx8SiQC$(!CYoa2)3DK?+5YE{lG~^zOgNbn%0*r&S1He#N zyjtPrn~c0@AWm)0Ut`GAYVwD`SiCm6k#ZffdMK?F&Zh6T_$DtBtkT!Yw_=@l*>@s# zmMg3-ST@j|pA~14%@c2w75^V_>!$))T0Rh9^xwz>Xc~X1EQ%CXQyXJ0_^0TLu2fxO z1!J$axuDM%CqWb2E#M3;a&Lo$){eEr!C)e(Ne8|p1&!Zr)bdWqi%@WEdZ%!t-pyGn{PvxH^metM_R{9DUzMV z2&|LFa6&}o!j)3?*LjKI^vWi@MvR8m;}$W1M0o`Wt~%r{5=;k(K~bCWY++FH4)Sac zVlhyEyH#p_B95w7)6T+Z^QvpQm{0C3de1;Z6Lry*@CSlA$PbOQBOL%yJ}$&!L&V_a zaCsb5D*y%<>EaRvGoSv20+Y-q8MHq6*hD}KT3&4bytqn-2jxnB9C4WGL;7!K7_Au; zC8x?VgJozpRv5q<8U9gtC0rJ37Jp;NMA;@GFH#SnOX(kwR5zO?qns%aW}IR^cSp48 z|0-5(V~7sKFS))5OnLHs!n2~mSwcNPe`^2mla(k2=SGd@ z=#vc7VI~7-cm8LW^0XTVcS;5XMd9_`T`HDwCH{1U#FBU7lI1|Vjru(!qH0*-(I+<;#S|P=!#};6Eq}}VUjCl{r0DWBimrf*b$%K0f zbYldGcRlE>=PT;Gkr%hO9p-JiMRD`qmjd;B&pf&OniFJMniXg_{<1{>u@?WARrHVL zacSpvDShV>U<5E~vzm>*_Wih`{`Rqd?AP1xTAIos1vwNvbgIVw58z%;@m0Lu3y-bH z3Vif}Uyu?%H|$*L5&$ssZS;Kry<9%Li^L>(9y(&4f959P>V^$PMs%p8Ak3MK(Iaaj zTu2#({L9>4bn@~G2doz7!~P$Dk4%Vi*U#PdV7abe&@`B39#vsiee`Tn<-C)3!KthM zlhA;~qMf(3e}u2fCU&h)21>2PX}|~B@Um13k*UGz^75MbUk-ft-Y?IH3k!rt`UgbR zSgYmT@)~@`8%$ID%?oB-cu`@yw`c@wRvfRs4VWtFak^e(^t&lN8<>Jmc+Ov2t_xvu z6AJ&Zsnr@cBnF$$iHfHAMt1lMfn#maeE0C|Q7?v`#n_!#R;;W+vz|yz6&^6d?kBok zo;y=`UCAT`tJY3~im^!Q29%MGca`VgoR0#4WMH907E+MG&odZd1+Vjq``?CHb6&~a zYhjp2b5}7xUEb3tfp!^CDhnk^jx|(F0?~M1Psz441&w0Xcp+dc8cF{Gk3~<#j>%HAx(yVSOKn zA;$?k{6j!_gHf40mnUEuSi=&)J@S0&DBqgQZJ6S=C!5fULWYp};byPyg&!tmiIqD! z-}Gs&%TlcDtPy)+4TpU`jxY{Gp8oay{5Y1?j0TGt!5_cm!hJLSV7STNoEn{_dX_I= zk{G1G`3&jHB1K}}e9zJ91gWsg@S;%s(TeFM>lSS+g{&Vi)%i7vYh;6|{R5%~9lb1K z`BC8}6VFwe`L#JT)V>OZ78QBxP79((o#zVoJIAk|JNg+;3$#KDdUE7)ieE&03`q}> zKl^fcc~;~~%IdY3Y}pD%jV|wD9R^pu6YqTluLi}NO3!@H*+I}&i3~nvi5Zq zx{|;DMlP+cIVmJN^1$~?0nYQW;Hyb)QqOYsvtbjAmVbBz01|JxKj0k9lZw=9BIqGV(~PLQ@wvzEilojdOldo)P<#<x#Xhiz%3|(=a&h~ zqi@Hjk41cnaX$I3I&6=i6pXd#4Bq*m{C3F81N?Wp>?R{0UV(Nbjx4V~Yel5Fm=7ARS$avd)4o*g)_L3-K6qd)Niz%Ey1*cH z^hn=?OzuX^(eE!$A06yZ!ImFUl1@G_S(#gj1Syu=7nBr7hFg@qZIM=!iM~ooVo$G^ml;%4c~y4PSN7+rCM5DY z(+^1NVFnLdt!-p{3&j2RSBR8hmZAAuo2wH70Lw4{;L%;R{-5?b)E4=HSsrT(`J0)M z=KsI&|B(RzUWa-k=~-lmLDT?%pVJdhFMlT+PvikJ1u~2U417a_%%*vWJYgoK`RgNQ gEC4_g2mp}%-~Ky*4RW4E5~~RbVP(Q|g#KOn9|J)9SpWb4 diff --git a/summary_results.json b/summary_results.json index fc9e39bb9..87460b794 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2831, + "Performance_Result": 10.2182, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419877, + "Performance_Result": 0.420835, "Performance_Units": "Samples/s" }, { From fe25a265d4c477aa2aaa3acd96118aa1166a6362 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 22:40:49 +0000 Subject: [PATCH 256/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10054 -> 10054 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 33c630764..e10bb7003 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:39:35,232 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:39:35,232 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:39:35,232 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:40:44,489 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:40:44,489 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:40:44,489 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:39:35,232 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:40:44,489 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:39:35,232 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:40:44,489 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:39:35,232 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:39:35,232 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:39:35,232 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:39:35,232 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:39:35,232 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:39:35,232 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:39:35,232 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:39:36,408 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:40:44,489 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:40:44,489 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:40:44,489 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:40:44,489 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:40:44,489 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:40:44,489 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:40:44,489 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:40:45,448 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73407 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420835 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419881 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26386 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index d1935688f..5b6e2f9e0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -883 +884 diff --git a/summary.csv b/summary.csv index 5ee0cc7bb..3f98a9ad4 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73407,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420835,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419881,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26386,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 2e935031d..2e2330d23 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420835, + "Result":0.419881, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5f9e9039d1b37d773b7a3040c0dfd99b786892e5..1bd9a011884056fffc1381e770559405e02d4c75 100644 GIT binary patch delta 2887 zcmV-N3%K;gPR35KbO;2N+G7fncnBwdQ`<-yh41%Qs8qQ~ss>x#w!SD}ijYZyF@_Le zYOg5AmOpB2%OlBz{QHq?W9;`F2_>6Zv9q}#cI&*|{kHVbE%iYMzdlSS#(faQ<8XFp zs+wgQ!K@ed$Ft#~d2{`K&ohlU>CXDyNjM7*&2K?${(AJUgRf!qM?4CG#7HZDn8kZiGoQtNxL79#`D-bI_QtnV_7~%6buf{W3_+syKb8Gql0DJx8vYz zy!F}0Quz{we`H@y`-i5LN$a40eZTC+Wn}b&L3c4puEMV$gYj^bq{C)sA-!-CFMk@- zaXOaTG^X8$L$hHTU&sAqlvQku#{GVf4YXk3w(%2~%MOKqb4nrd2( zwzKo`Z4#_w>Z?|lJ$%t_`=sE{YMsE(aXZJOzSv5n_cT*GgPM4a~%A`t`jjz|QszY$4uSMr`n1WzA`MDX;H zNW>L%N+e?FTSOwx-x*4@7~M9Jh|xVK5;3|RA`zo|K_p^yFNs8q?k6H~_yyF_Z^Xle19Vnk?(tyc;Y(v@F*d_5I>R@L@K9$1>vp7V~-QmnJj2y z1HMS%T*h=TM$R}FC{dQU<}RN)J~;aL@5|q=+NYhf*UQCFoUVq3<7Bk>QtO4&`gD^1 zoTe+C^wT_u26yAxAc)fST5y*HadHK>rh!(RXT}B`HW&}hU(9}xbjOp}d~;AA zkm>qU6)B~Q)*$WL^1qyZeO~oTt(F+Irt&>caq3)7or~0oYwaycT+xjalmK-^BB>)1 zNga`hEA)d~KphDe(4H2Aw=monIh<{W3ohCi-V_6Q?n)$oNsQ?61xl19M*q^S5Xs@_ zuG`D*XZF(DmsM4J!xYH*w{Vil>BAo14QnG_pM?KX3kqCQ8_M-&A_Z&Biki|hm?fFhAd6p2Kl zh!Rg+nIHTD6fxWv{X8uQZ}|e0Gg;6k4alXajOk#EvP8=lOd~43^sB{f+USA*Bq^sXFdMQS)eM5!?-MJ7c>e;-V%)~uHh4+0Unt~0?9Oz zNTTt@5PjHa7?Z3531v|Qtu1k3&vkO{ zA{F0yI##%wljaQ|0l||84kv$G$MQ|mV}*NVDWIr=*5=?7ld`1Edg)6hWmJbmk=siq@+6dYwNNg;q;uh7 zO{E|*xt}hl%WR%heocS>0FzZ@SL(*n-oCM{xs9fkhh&aFM-lgpH0ThGD;+vSgGiR} z=r9c+Nn9{mB3We9VT}NiMMfP3P8}#|vtG{T-MD1syt!4sb<4LcIazhjrAk_1ap zx$WQ4y5?`0J9eSmvF}(E`QGE$V)Z*sBO1Tx{CFhUd|&} zZE-R+`b8Y+@gW*ddVGim$YdHxCeuhVnMRVSi#$H80!e>nQ3Wjx9v>)alSY6{i0Y6i zSjG|!E@O4Y9-sdTn3W`1W+|(;Z9duNh~wKiOIfXQ$VAJCBYi$Z<4K*F!6#$tlGga&mtj{b-#-CYnYZY4jl)Pa1uQ2FWa8^kEtxlW2g0XMjsM|b7mQ%&sP8c!C<~rr;pxs`ZTrgHFGR@x{*UBx<(x7^dTBgI(>)+$t>aY zVHzNlXn@Qjrw^+D$SiXDXmI*KNt-kRh60G{uqb~Zvy9V6SM2n8j?DkmQBX;KrJxip zA4^+#4}Wws2TF8|IMU@qG@f+%5Dh|E!sWv>03|W$$ej{hM zlm%3R&4-4;N|uRWC0nu2=Q%8sK}Mwdp~tb6OWC*M}BXYb)!Ou9$Yu>X45sl{yk z=OTFhs`tv-f7Nr|{9@=g#-Wj>{wPheTedWkePG=>I+%Bd!Fe|tj%Tqk2?lARR?Rhy zX!T;P<>w@vFF)C)@g+=>aQgUd6mT-1NoVhcop6^tb{DSk+=mzB(K?!JXgw&e`y{{JO z3?)7=Em&tXG-wYV&D*=0Tgr`tS7$7^Ai*aQq&?-fLE8{4#n|&LDCV>1&{13)r&x%k z+hEK1H`@R%2zkH?0)G=MFnj=}b|Rt?Dfoo6Kdu`9f@A1Ffes$$S&q-%08aUv4TEyw zR5I)>+_av=$-VP(GWLC+_2poV`E&fae0dvIO{G4ZiUVqE!6|pZf^lt)FN=7%j`^^R zCtgbskNc?KcpDcv_JK%Z0D6pjQa;M3=hXtW1u5=RQl@0KDi{h{5n5GWhmlwPXP_E~ l{F`_J9CKfPg7&kwBFh2(aYg_D delta 2887 zcmV-N3%K;gPR35KbO;2n2ERL#cnBwdbJ|E6h41%Qs8qQ~s>Yyh(3i{hl;cbiV~icg z_S9av2#AjwK#U|D=iiS67{hyxWF?zfd1vP$1FiFR_uKT7ZmCan@aw~LV%!H&JPv1v zrm9({5zKmFe>@u=nm5<)_dL^xlkTkForJUC(EJv}=C4QpI`|q!f5f98NQ|_9f?0fM zj*?`)U$4i#Q84YswQwHH(kp{7ns$@)Yc#CK^C;-{mqF7>U0aq@pLWMHb5+BBRHlw_ zFc|lOcVTZa4Q9!zk|>yTleGKsXgrV2ql5l9y_V%;M8V+DJXZTBzw4%1KRQ^peLD`m z##^6_ER`=|_(%5Tw0~$?nY0dn+V{(DTt-Ge7<3ntF=vcYF)u9R5|NdWvNZHUsFxX z(RTKGe47O8nEI;KWe?wVlkU+$6n-_LWef4VoBc9sU+Z*OdRghGtT^j`TYX({^;lq} zd1Rw^f7Cdr-)9xC6`y6uwJf-i1-DxT^|U{mPHEmL4Sle7`*4MdrY*WjB+jnnnn>JT z$qkWsyOLWX@i&s4bQ?U~22`L#x~LHS{QRuq`dP)zvx?gy#ZIP8o@o=}wa63|zR2`h z#r3m_8&t?2wWyArOl_Kf>9LK!shGoWiA4PFCqyC!>>ZH^V1FZ$=C0&DkqDkX5Q*UF zBaw(1bV?**=vzc0e!nx6Xfe8NA`zo|P9$P5@_k1nBH!PLMCAJ(C7zfEA08#-FT{_e1(C{sX+e1F@!I18btVhi z*nlrmIF~UUjFBtO1xl19=G^5|#|K9r|9$z}Rr|Da_IkM)iqqZDaGZ=5UuwN@TAxnR zKd0$VC;c=JqQTvGHVC40zZTpjL7d#hi!al0oNayLyL-E4)srX)mYb}4wwJ1(w@=pb zQ4ptFn7F>)%j|7`YQF2u=X<^RVlSM{9-_r8NjFn_Yt>yZoDIf9^B1!pB;D~OHs2i7 z2V}YaR7FbZrZq^rw)|gCzdo<}rB+*vT2ouz)^ zkmzv|JtR>~ptmS-<%SC-Kq8Sy5{X2TNFtBlAWLgP$UwGB9TZG zQR0c2`N1zh5yO4a&(nhNmM=g#lLc+kfZU48m=4A$OSF8!G@{~5zgpa;-Ch{)O}fh( z9z5q~?@OyluLOcu{uY9M&CyDE*5iMQv#2Z%1v5g`y zNd*9AK?SYN#V0UjNt-p|NibzpheRQmNzRI!Ls2ul+LOX%&OD0j{dXh=hAd^G` zOcD(+i5g!?Cha07Nh8t}P*g!{bMc8uS<+^`^d*xrszaj4?Ija=5=y&TC>LMSwQ#Yf zQV^NkPnXMOwoWR)rhk8c$ttodbz^C7-&oe%M$^heGRL2zi2Ftwbcn{44jrOFBujX7 zmM@W>LZJ0uE^ESY%+jw~y7 z?;w>w|IG4*x_^4E)wZq2N$)BfUf!9dpMxbDMjYwuAsSB_dx!?HEMe_o8o-ih0LvnG z532yMEb{j#F!(@8n>2z)BjE8NQDg^O#^s|bw)s5A<r10`+J2%K6%REI?2nn^`+&D0h9e7+aWA3C?JB*`*srrtLC z=tj*_PJhm$AFXr9MAL{PjXp%5-5)F`9Wb|Pb0GUNbpUuT5GQsFW z8bLC_=tH7#%`9W|+3Np47|fUI^wGOcpQiS`W{w38-Xk@c;YI8B3S7@P!? zNq=ng!gRP+k(<)ho2{QkaXa8!oJ6XN5DjCm)M*QdMWmr!3GAt{r ztT(t?OadeMHV+QX;9;KrLRqq=(SI1MUV~-Kqj4CGlW(imv-fZT-1NoVhcoTpeT^{epEus|saFiXu=dT-Y8 z5=G8a#c8V*RHz4sR`p%U3}M>BtJMaq7vQ1*QjRcFp}qHpVC>i)By~w}s32~&mDGpQ z?y#Zko9+PTX?l+(cz?>Nr}zkr%}hiSlCv3Uer!7cIK$9_1mzvhk_?}{9;|dX8wTaV zDTVJ1+_av=*?n+gHg;W?bj4r|`7`{vd3hUFjfFa%iUF!i&Iq$Wy|#6UFN<)vmO8%) zC*BI+kA2W@yiJRYeZZp-fQ)fZ%180^+^kWZr}-jIi#T02c^FQL2U0A)jw7%7&p Date: Sat, 8 Mar 2025 22:48:54 +0000 Subject: [PATCH 257/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10054 -> 10049 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index e10bb7003..c40e5b0ea 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,32 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:40:44,489 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:40:44,489 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:40:44,489 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:48:49,624 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:48:49,624 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:48:49,624 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:40:44,489 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:48:49,624 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:40:44,489 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:48:49,624 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:40:44,489 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:40:44,489 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:40:44,489 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:40:44,489 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:40:44,489 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:40:44,489 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:40:44,489 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:40:45,448 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:48:49,624 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:48:49,624 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:48:49,624 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:48:49,624 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:48:49,624 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:48:49,624 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:48:49,624 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:48:50,597 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.260917 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.351306 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73407 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419881 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26386 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26688 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0168 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.429845 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414229 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418495 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5b6e2f9e0..b9cb48f6e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -884 +885 diff --git a/summary.csv b/summary.csv index 3f98a9ad4..ed075a9c0 100644 --- a/summary.csv +++ b/summary.csv @@ -1,10 +1,10 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.260917,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.351306,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73407,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419881,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26386,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26688,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0168,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.429845,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.414229,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418495,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2e2330d23..80e6e6359 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.260917, + "Result":0.351306, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26386, + "Result":6.26688, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -544,7 +544,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.414229, + "Result":0.418495, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 1bd9a011884056fffc1381e770559405e02d4c75..1766c9701b93e0744f6688b4c884e5d5b4d917da 100644 GIT binary patch delta 3082 zcmY+GdpOhY|HtQ8_Kj?m&`j$+Eg8}+x;fo6`7N$4YCL_Gc{X;Ygyd8gtFBASwI}Dw>$~MO;!{+oym)2ce z3HDUIAIAnXu*0nqw4M2ZR+*Nw%kXaOUVAiws!fnt4bFSBp3i@5DpMRYm^{PNIIWZN zv9!?p+9%>EW>=i4ljON$=Z3Vu&kXPP*-~$daY0&P4kvC;yAEHnO&~ScK=;U*@)HYF z+9FtR*uGcao!rt7?rF2!c3LR>6;!)AGr+4^n$X_J3uqqs$G!zO0N>0X;ynubJgDaT zo~o4;tmvQM{#8ru+aH2e581TNtCYXWj2^?!WqtW~ppy6O;P8vm-`7j(!WR6o)?E=- z0uRebn#Zuh%Ui7neYbwjWjG&mx~+Z-_H5q^+M3eSz_H?cRF(H?Jj3Gpxp0G5H)7`C zu=atAXt%YDzU^Oh;?H8{14hi$*70j^x<*{6`Yy(Lz9iV@c%!Kw470S-?%&~~7=l&~ z@@!kUV0lWp-}}58JX^{`o6>dt_MKW}A|6<)_?%~b-?dvhliQd@Ml9=PJqRH;O&$c! zvs@gLgJWVY1aFspoFR1tGOs)t)T?|hyZlO5h4fi7=8c+@WM zQ%3+kelL)tVH=GR5TEsU>-O*B_w4oqqr4<5sm8v-S>9p(K z*u?z!FwjN4v_pAkyzrpeO*1hoKog^7&V0@qiQD;XM?s|{ysAB1!%Blabtz`GykQT4Wi)?cmRq= zWLiW2hC=aNpc-gO_+Z8&TObm}Xca(ad$QtBQDjdBfqHzs(DDkb$We897#(~EAplL2nOf(-pjt3(S%=i|yZo-6eP zH^nR3FHbCXfv<7MjK@%2JF-1KXjJST6({Bhl|D~X%0=lb*&x--)CthA>`5nV0z-4Y zJ=3wLM1}TBihk#FO8p-%3GE^N4Sn^P)Wj_j?fH`Go~C-0T<7<=UYnTU4wa6@V?b(tvbLOkwzjeE4B%^kj7i*avd{(OaCX&No;vA>RPw@R z5mtW3_8R(?g%pMxSMhJyJo6eqGw;-bk1V^sG!O#hdQt}&?kq_jpai1(f^3JU4Qv09 zRKtV*2YE66I6D#m^MKP6d%fqWVB1y86XQatxVQhFK1y$Ooo1D*_GZCO^{eub-h1w> z&R~F=TVk$FlYF_$4OkM(UC`0Si-I_Sxlia1DckWu8@xHaYk`9yM z+c9eMs7fe3@FeLegD6<7c^S=mYv&%3g&)RNctUkw0|OvBPRy3o>5zqgmVzIoVI;DV zyXGf_<^kCSp7y)yn~~kP3hJMOPZfGPaso76%1Y&N#p^>~!kGQ^9{*g^m!7b9e{5a3 z8c3}Uz?Rn%r=uJsb`GDJ)E|6kiAf9lDl3g3@z`ycSYQAP?bgYcA%LtIisACu@P|bN z!#HVZ@?0h=9+2a~3Js-CL%Ul&avn28PY~-D-k;8li$JaWgOG6Dxc# zU-5Yu{utLwiaG*=-X=5pn#9Nq-NqXkJ%z14n-;Fdqka=39{RQul)Gw#N)Q2)GB!AB ziK)P>$x2@#@%|E8BL_eXuSFFGs7hmH9i_ctM#u?^Cm)?&K0@@AAa{IW`_*n5b$#g{ zKO9l&4S%t))_tKmXadcZ@K?HTE4PiP6>P|+o}e0=;KV$IZ;pNyVA>w-#`GGQXv6^= z4f8JX^!>Y+G*iO~!_x4eN#<00^HmCegL7ZVle~`m2Ter%!-q_8z&qf#juM_MuXctB zTxi`dwC)3&z>7t6Xp*lrRyKv!-pu|k@xuS`s{Eyh#?afLY6rRvhRUepvn$QMJ8xUO z&}S|%Pw@lU>EnKfdNiH*dEO#SU#04zfh1p-tAZm+04IJR>1wPkOGKwP#Y96SgXAq3 zKOBXSOEGyY2MBinJlm&!wy=EpcLFb})D9jx`fW(hxX$(Z&#r(28MF!r*)Up|u^pUc zu5$t#oh$hoP*SCG`o0N6W_sDdGP0VX&Y?<*T5QgP zbJ8y>1;w#n1}K=((jer9C?QT$h(n#=M(vJ_8qC3hH1p5p2g0Gzgv+`7hldg?5zpT* zop4V>j%QQrUDzM$RNQvodeySd!@sngIw(q+_)M=9aUny;CnJ-GFv(UtyfzbXHtn$+ zRSy|zh^%>c@2^%@WAQnB!KwmZ^778bXFHy&ub*P>r#}DO|E{g-`>)GRXA8q*-Y<;+2oWyD>Z?Fn?s5jd6a z*QPD|=TbD7-H09!vXQM73+yh9vv6>Zlp9ooyokvdHjMeHp7sgIa7cf~+@?^7FUFqf zZA#1*4e0iqiif_P+*?|w8+l1<$iODb-MMPSw}iM~1)bM;huo8zaM4GDtI9hDd2IZp z_H3i9M#tC+3^l{~`j$9(=!sKu(FFBM?}q+_sOH{?mtavj4ecmUF$KYb9N+l!xoHIq z6z}0{9w(zROG@jX0@-cmODBYUaMTeT408VW$Inx@3=|tHdK*HjI)54LxHLCOKP$F} z+jWTO-kZ)o?at#Czsfmd;!q zx~iJF^gcxp=w^UcXjnvmS15fMDNm1AQxc6)rWYz7psyljm3Lc>6aj(Gi-JH9(0^BK V+$H*ynu!R94m);K(hJx<`9F_?zFPnQ delta 3066 zcmY+`cTm&I762MJ#N)<({hi@a`|9I_Ov2J$l5u9vuU7-O7t1jMq$mx$|agwDbBdG#YTaZM+{-n1uZfWQR2^ zwG*5+qqOgz$Cm&3LL_R>}3S4OIkSgFc${G1bCPG1nOOu-hGnPBaUTxc0;F zCmgTbHu-&^#=vdi>)+WebGmUIG6tTxhpFy13Dt~^HBV@%?glsyHGW*E*nBx)g7f(Z)a=!LS*(1E+33_={Z;H^+SDg=OBU7h~|MF_2MUzRmBMzhMuXwLS_7%xm}AF z37!6v2hPpBoq%DfXyV5DS*>ScSWHBS=pUP8Oe>$iKEh$%77c&*`KRkl8M^JN4D>rjUBqh4@r!biq3W!6=t0A<& zubMkQq{N!Iu2A^VGC&bzs;t8?k!L7H&ef5*5i@1i?ysc#?xKK{8NGH^Axjf=ac^g} z=X3d3Q3hkUr_D6IJUV*X>DA8l?~exe{3*ZE3uQAl2epW-_oqiii33|U-cMP4d<7I6 z4Lf|QH%Y52%c&M7xuVtzFBeB`o`p}Ma2*6ia3?+YPB zeg@*E;c}V9{o~Ua&feR9g+KA{ppJz4nTkAF*t@|eUGVs$=N zi_+`Y<1+kN05B;a?Bkd7=kuL9;wd%E<3*$ z=7UU9es(R_`_Fv1hT~o24`}5?s)C9_mBo!KAZI=edW!n6@Qv>-eQJI8d{^LnpMy0m zYttMrK@Ep$t66x|*d+Qv$!KPLE^G6U4&DL6=B)j@7D*)nHcbtT?=C(KkRoHAq4VUd zd`pfvYfIrB%6+(D;qi$ZV;tjMwjJw61CEbE602pK5&P7H{}YER~UZcQ4;QowAD!{cr7VYm z7&sP*63|<@;_NDo9hO2lH>&Oz4O#R(Yu*PW7}$A3CrUcAF8m7BBHt>uYxXpCtTHso zkzuB>!t*EqJpR-$w|xsiE!LJGu_hy6c&J(<#c)i=kfNI}on0LHarO%j^<{o<@PdKF z@1W#yz(8UhK?|rdRcOrjn{qHJ5LJGirBKx4k!c=obOfSWI9V;W4@@>zmcEWJOp0+z zxT44XGhTI`D_&Mu6CH)2|Cm?e5XIUl*}!p5gZA+tijJ-@#8_C$jd81+14>+cA{B|? zY39k2;kX3tYI1y&oV%7E5#!3$4dp*q z$RtIIXU&%LU8~Rqb;CsxDHqaIeyd;_ept+`Hh>T~kOQqy{af z+RRxOB|W`{glP*c%!t8EQiF)BC35z2_ONViXa>cpm3-P^X;op2ukv`J}GM z02}3ahd%OiKAYEPd34l_Z!|AToV>TETD7X?Cl^>pz9vzpE41*N()^p6I;MdCSVezZ zzI(3Tdsne(s^Dlaaj`;5lo2e& zqiXBs@8xs=l}X<$-I?k=*SkJ~GQ~-yUJb2oFm%foM6#hnoLE0)s!&&Cr!=d`!UTy>aYZt%JU~#-Eu8^)#6>Z9Li?q5b26nMhn-Fh7|9; z9Zz2WeQjEp0q~jQPoD+3#uem4~WWblC}tjU;fp%_p7Z<)W|uYWi!H;>=a^rw5T61M{piXSGkd z-hDWf#@g_Lf8h@#N=e?@MOD?6^IY$EeX4DDe3X@oT&-xcBzOMhnQHBEF1fam4SA_q z#{7FTJHaArV!=3o@7Ylw2Pv=xy=4dU$SH+_3IpJ|+|oM_UtK8|1VuW|UyV3@ZVs2w zHwiM{P2-62oibNmlAm~$+1k~x;j6qImbEp5kKgq>cpVxfXlYb{N{?lBztb#z|81m? zXT0L$z}ke}ia5zz23(2z#h(KO1vIuJFb<&LxRC3p?xH#52XGZcdgaFTLZ@mOb{lRR zi`y||t3#J;Sj<6MAXv!nHW0bZ$zZ^{!T Date: Sat, 8 Mar 2025 22:50:15 +0000 Subject: [PATCH 258/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10049 -> 10049 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c40e5b0ea..104c2ac82 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:48:49,624 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:48:49,624 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:48:49,624 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:50:10,337 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:50:10,337 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:50:10,337 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:48:49,624 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:50:10,337 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:48:49,624 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:50:10,337 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:48:49,624 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:48:49,624 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:48:49,624 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:48:49,624 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:48:49,624 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:48:49,624 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:48:49,624 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:48:50,597 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:50:10,337 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:50:10,337 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:50:10,337 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:50:10,337 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:50:10,337 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:50:10,337 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:50:10,337 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:50:11,559 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.351306 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73407 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70822 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419881 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26688 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index b9cb48f6e..87d0f710b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -885 +886 diff --git a/summary.csv b/summary.csv index ed075a9c0..2034abc6a 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.351306,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73407,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70822,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419881,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26688,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 80e6e6359..ee9b37ce8 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73407, + "Result":1.70822, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 1766c9701b93e0744f6688b4c884e5d5b4d917da..0f0a7dd3604a6c153d4b8b86409d5c4983af60d5 100644 GIT binary patch delta 2935 zcmXw*XE@t^7so}-nvGd2ElG-6jn%5zrXskFPAP7C#Htacioel{s2Kz|ZOCnw+FPqs zO6`g*rNb&!qqZ84dYXmpUB6b7R+DlZ8`DnZt)b*VHWW7-n;F+ReQ^Bx zPZVS9`{Z6WVW6t$;R?!!w~xnvSP9ol=6PqQ1c5B7kFIQ6SwBlY3?3s#`5l=Z&2{XR zp^h|mMDzhF-)GZml~zsdp$q8Bu?K3?tyJ>nw0`~S{6dnsaPJ;^PfbL9`o_#QzBqfl z8Y>btnY9_&(A-xJxzl+jUca>k5aK2&120OFk%!d)Ikzq0LaXFR0(X>NkX$k9+5SLI zXvvmos5(7p{k!e(w&xch&abSdBt_9~2emba3oI1PsAiwoF1EAMpZEybOdFh$-QOGF zX-|e~dEOO7uobK=7N~L1{IH7?B}d+cb-17IO2%-Ey(gFxANr8u?^zz3?7j7SFL45D zfd^zC^A&T}#BGVZ^c>4i4Uzbqe+bWSSoT+a7jTdZ;CP?f)VK~`fBoHs96O=jyHU+q zIbw?OQ>&fJ|GHSAXY}Jk-_(KmkL*We*p#$(ccS~ax0I-tA6G9W1S;L|A<*Ty=4+X#OYk*6-@dzxdB0f|lj$N*=M?kY zkp7ycM$M$$yd6lPxd@E*%X*oe*BPo5yPA5xn&!ZvbIXV8EeyQYK;|V}wp)otGW-@z zst$my5Rj`I;RZ~m5CYOluK+1&5`s6N&Id$+Z)Hs?Bt=&}Hi3Ry+KNVIO>#&AXI!?4 z2GsEbBvkF~7mv`^@x&IQkeH!|kG-iG4ZPVb1vX!KfRtPeNdVuoVueV+W`w?>Xpz#h z)zH`LflEFS8p&MNOD%k`tnPG)_77+08DU+#qn~C6TTm#-QOGX+&USUcR#1&_%;50H z%Ledn#W=mo?tS?S{g_4?+6Iu{cf|?yph8nhNB5hk3CT@?yH0}CaoUCEraBRQWkN7e z^TEy8y=d;ywj&>8*QfCbAv3e4XG(2-(zaquRF3rcYw#hj{5-lbeS%)N%&P21iq*G#R9bx5AK)(N(_5t0Yze~R zh1=N;!!#t5hRzUj;|-_+d#y4#@lQBhO^x1N6<#KY^!@r8@k}AA?ma;>07``F$L{>e z>T*OYI@7r}Q>yc!xqm6o#%82wt6e7X*#opN7d&9wxy?)Xab%w1+ab*ad-1&6x@+y7 zIb*(}FEW3JG+n0v-Dq4A(g*bB20=>5N*V+vAc+Q0B2qF5+6aRZz0vmQK-RW!WZe@a zCo9+*ElD_|yv1yoaz5?mvJRfAVOXU5?NyVqOzlMepn^qAkk}0#bAH}GG=)mO6owe2 zq&XB!kTkUNaSB~EG^2W(#;zZ4ZkpY!&+&R1?i`RtLFD5vb@6K1D8(5BFAP7Um*}uf zzrg~jUt4Q{t;Xv3U$W-{XJ>ggL~4st+%*p`T4c*cw$-}{shl>L0>?>}e&wTpBr^0F zJ5`7Vlr^f%Uiq*R9K*=&go)_e`XXrLx=?9C6$|Ay*8;;HX#C9^hd?fNf7-EO$uv#2 z2ij*pZVN4J3co*`eB)NX@OozwH*(XCS+h+2l;t{BHDx-GSvmJx-DHBmlyjXN8s?T& zn@jvu$|_4>DTIn-gH@Hxy;M^Sjl*t}px>n%D!f_btw#t_MJdPap7Oy3I?pQKYcZlC z)UBPZ{?WgFz>w}gWZ^XRiPP9~Sxemt=up@ZEZP|vtavvm)=jddv{yjW3-y!;EG|rwh20>4Iy#&W0#5(hKffgrm)w>P-jc3QUuI3F_ zE>ZmDK0~$c)^EFB0UKB9V2zZ_BW=N4VQ3WwxSO{)UG`Kml;M9w-~>sy*m~k?OMLFh zRJEzYoWGZo>TyhEtg1QX+%!AZz~Lce+*|;Zco%>2g{2qzh_S=nc#SEb6H)qa2InSD zV9-pF~A z0qJT!ix-Yy)%c4UNG=3gr3E^`kX*Qy;z*NqAmSwd0y(j3&?8NayGRP6c^5MNXx?c4 z8$jS&59?O8H_yAC%lbSe*AyH0wQQCV^*C-9gC<^5rmNI8T!y~S0 z`-?n;5|5XJmS?_8L9Bef)Ekwi9I$k)!Qu0z7um>ly!^pc_rmf4*>imIQneeqym1#pw2_gHcUO0n zxDwGSy3QH>co+ksJ6IdO6P%rAq!Dgy6up=vouyo4(3j!dt4&_Hv5w!8cbBQ`%EHdY zX*(C&NjKT5y8pxMNWoG``Vfjw-T>Y;vFEftBp-wzuzZkH-YaK11`oJAUL_bS?IoHd z0)NUevsgI9clwAma`)zQ?YYvy0?#0?rY~CwQPWH!})J&E{8tw5fbzM1_wlgn`A6X@A06AJfe4 zdIQI4bgBO`Eq0Zvq_71ua1$M zmU+w`Q!~l0KvtIE@g6NIb`IrmL?xRzc=F~KxQjP9OduMQqh%~*O9%H({q9x#XIKB~5RC@O!}=HhT`A;Y zfnt4jIHt3Lnzog2zXhZSx~Yg+KO5m0x&OxG-TTGVJa?OSP>@9P-2iN6gI zxZnd`mFXDn3BB4>XtjJNb3c5Y_WktiSv_`FuO~zF(;+IwxNtBeU9dz|M0O|=)xDQ9 zL-L*e(tDOe!C)25gf(q=^-+Pz=+ag&Ys{{k29967T}IG;p>_;$*j5|HaZREkqJ;_I z;Dm$~i4(p*;SWYm*o<;Bv|g&t)@4mtm+KIEz)-O7*p_W|C6)8dQ}hpv*yLkHl*6b$ zMqRYz5)l7(q->?_Wzi7+mXEH#%;i~b3#C$pHUBd_qX@{z8da42TT#Lum5xnvfJ8%C zvODTsX8pGIX(eXXiS{pu7(>Co%s&cW$$4}bBK&9&0KFVcY$CX{T-yG5q3{S)RVCTQ Sc=EZf3WsTd{yVUDaNs|>0*6rm delta 2935 zcmXw*dpy&98^?!H_C$;##k793*pP;ivx%A9O^O*(&c`At!W?dFY1jC6=n zw->n`lI^FKVZE5$_9(m-UUD@g|J8Z{=b?#YY0MC1j*UF2o$|iC$oujK!U=j$oQadz z>7%EIwZ6`c?Dg4H>x^+xu|gk8+M0D8xnP?>Y_@^!k})B3l8RC~BN#BnJ7xux&l z)oHQiv{dvfxM6jU%dY!ArL~bC*f#dBeLI#5;};CG9|V6KQuTeSnH+K;Ai@2!rs|h- zyj2gWv=@Hi|5O-0gq_a%^jDCA_x#XEefjSzWsPA=0T}C^h)Y3-B*n~P7}qL0tcQFz ze=cM=A9YGpBi~efvZtQ9rtmmuqVx_~>8&c;ptNb>n*Q_aF$*xoF0LcWZ7pM9>sL16 zXDR(2EoNr(*yUF}V=j6wM!LR4MgC-qiN7NH`$|_px6gr4)WKlS&ZYAfCk_sJpHYS7 zhHWYNhKmCwBqrv3$X3Ps zIbwGZ{nDc$-Rh?j%cW(q`wGi=#O_t*t%D9P_tqnO>`L}m@+O`SI=FORredSM9mT@! zCQibopKYH4mRx&p!lezPm7(lmE4Z}d>tmUevu_VWm-+V|bjkWO;(_;H3uUO779+T{ z&w9Ld7kRP`fQE#dNK-kQn1Tyebb*?WqRCLw-3ghTr`Wdd6zs^)gPD}bx5Zui#@WeM z;{3#oOrElhK1%$%q0-H}_g{Hh$esLw6`$8GO+Fs_qREu1-KBG#kN)wF+e5yv9X@Kb zbg#`#BPlCT2MrlsevdDPoh}&0tzFHp46AcoAuB7*Br-2b`7GAYYDi@9G>B33BwRZW z_w8F{64&;FvXe$#KHvFq#EcQKr^hS{bunu$62)xnjT$c+T;E*jc0(5!b=@ZBqe|Dt z9=F7x4<;Sil8csl7=yv_^mcI$)2mEI3HA-X5_Jzyzt!c(=SNv1{#357`rx&Fk6yha zkLX+9V~x!)x}-ZXX5we8fGh)e=7=M{@74-`^r!LI=$`dV!ztx3_>Jy8-28b$ zMR5)UF@qr?8m5!xgSXn>9Oix^1I^Y9R;-|Z=6^JDKvXOo2S9P~Ol#;HC=|y6YJt{- zccu)IIXp>(S_NcwQ4ahRR_V*YA6qZ7xTIL(pu9ZxhAvAA8upVzS{kJ>k`WNfj*Z-; zxDCh>o{4}`vC=A4ENzn;Du_$0qc0+yp)<>1giqZAyg>$@OgJ#yDepxo)XX~twDT6a zgBI3{Juko4JYmRM8+6MZBe;ozt6Ndvwxl4nPYUV3A^XevX0D)7XS7<=EO^8l>tkWy{=SYa8oM13m{z8pj?{f zaIV`t@tU+;bZW;%R$Tec4F&Q%$wM@Eh8P=A0A>4vY)57d8qSHS;z0i=lK&tbAycF=L+ zV;X_CTK6oPL4IxL9+8C`!BlxdbzTBoP&QV?mhs|j7Vb$3ZitE&%~9DgKQ0(EFsIPd zen*`j*^90Ed+4!TUw3YxhD$}cEVgug_){2tFuN}x&*Yh>;^?`}OMXG*+CWTY17SAm ztmyV3%W1u#`xfZ5u+LHwa3Z_76CDe1F;H@^c7Y@wWKBD8O%`+QehJC|c?&F6EtZ=U+MW>druM_;A^r#2Mg@gW6{u2d?dMinbVUggz*O}*%ktZ@CD15d+nM_FFt zV~Ws35^bPWghbP6xt`Hi)ZxQ7cQqRKpBnSfvz-dRt%|P(;n67*L*o|cD)gF^#3ds8 zAAz+B7ew>=rbGkPAXnBAWUUQTd~R4g>B#KzVS=Y9srys0e*@pJ=hNWip@?#CSpCvk z@A=x`DHKaIK;f>f%oedElxq8Y;k)`J4;0f;fI zR7jgD_7b_m7g&4K2YbLpe}FXs;shgT;#Jjsz52rylk+QWzT28lTDLlRQ;*kw>AT#flF_MR)wv1I1J6z5*o%%^j%(Xb+PCK9I8nul z{=_foLlwku)a-{ zyK~K$Zy8~)5-Pvt7O5{a!O;iFQf42OerWWm;Z%zhvU_4h5i!U7{F*Q=|Hz3_GIgnc zLvKo0V|Twx)+?mZe>CR@AKF_r@bm@Xx|- zPvD9lO!_e(?@_O#h8qD^1Uz__KE{P+2=Q|*#Rsy7EsbtL(E f%V5l0`!<*ZV>01=!hb&3%s3Szq5m!Hxhna;Fe-&> diff --git a/summary_results.json b/summary_results.json index 099f8fa28..1fbe9188c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73407, + "Performance_Result": 1.70822, "Performance_Units": "Samples/s" }, { From ba2203dcace1c60cb40c280d7d5335cd76ddc2de Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 22:51:43 +0000 Subject: [PATCH 259/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10049 -> 10049 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 104c2ac82..b4b00fb1c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:50:10,337 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:50:10,337 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:50:10,337 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:51:38,128 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:51:38,128 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:51:38,128 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:50:10,337 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:51:38,128 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:50:10,337 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:51:38,128 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:50:10,337 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:50:10,337 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:50:10,337 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:50:10,337 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:50:10,337 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:50:10,337 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:50:10,337 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:50:11,559 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:51:38,128 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:51:38,128 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:51:38,128 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:51:38,128 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:51:38,128 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:51:38,128 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:51:38,128 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:51:39,109 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,9 +30,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.17796 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.12825 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2182 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0769 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62118 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 87d0f710b..4fdb29846 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -886 +887 diff --git a/summary.csv b/summary.csv index 2034abc6a..fd04fe0b4 100644 --- a/summary.csv +++ b/summary.csv @@ -8,9 +8,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.17796,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.12825,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2182,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0769,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62118,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ee9b37ce8..27f458fe5 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":5.17796, + "Result":8.12825, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2182, - "Accuracy":"acc: 75.600", + "Result":10.0769, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0f0a7dd3604a6c153d4b8b86409d5c4983af60d5..2765fbd1b23f98a58df6be589e93c0eb803e7dd6 100644 GIT binary patch delta 4402 zcmY+IcQhMb*vDhlUNMT$D2bigYLrkxf(9*WRBP5KRjWp$F~7e3#bSe`paLgHuq-u|`z6KQnsu|l-Bybb z?V-5tKHfbK71S`sZ})n-O%k^GVZ9wKLDMxIV*1hg{KZf;*|QNwGExi^%}JR7c+ODg4ss!djgW` z8X49^4CgY=_u}j==Uft95OlXR2pP0xYxE7P8jXvM5Hjkm265 zVzKHOzgcM5`N)H_H2Dp^Px-Pme;azM{C*!OqO2(OozE#NG!2waQQn|c9I`#VcOIT~ z9G!Ni?#+A(8yZKG4w&G7M-OXOF;+eq8j8c$`N{^-EHuU!xWWsZ4-F+IdEf-gSE%`$ ze=>@1R}Wms#H8;-ZS_M`qd%sfjGhRy<*3p+TP`fMFnyhG;VD?6|LyN|C;9r>RkD}b z0a*T~+H-W!TVv~)P)pF-&xgvwu3AEno*LG>=4D}r3KMBJd()b~jYpk_6!l?Pb6gzg z%YQAZe`@cZg{KJ8xl8$byE#Z;qa97O8U9#w!&Lo6msIZ7rfBa1Z|1RT6YqB4{WIZr zKTptvGun%1gQQ`ib(Ae|{9^9I%^&pjp^{-QtfLu{53h(3it0x@sgPz1Hr&NsK24_1 z%`wq!GDdYUlGs?9`zR?Saqpe$*n4A|6!-stK*Xy8mD;)(QI1rQ39j0XBm0kmp&kV# zI{*NnxtMUkWHI@BydRAM_NhAdK~)~FBaX{%U9l=}pQxoCsaM(h|fb+fCt)AWPJ8LvBn=m3I- zJFx#mb0-y?+Ia1BZT1P*DQj#XTzZ(2@OO|`n22e77<+7ooruCdnVb5UqZ4K$w*g(5 zbTW=zj-LTZgy!PER!P*UZBWU2y|aOf;RSh_LL1I+tV`yeNti<;PEeAlaBys>zw0$V zhLBw|(^%HSR?Qu=jgKd#sgF;8TaO>O&0=G!AA-z9AGK`7uJ^6X4pzuG^HprVZ>@=r z4u-G8goq}8_p-6z=nhoCgoQa0p4Qg$u|p0ykfRXeyg4`Mn$$tEj*BL=btrN%oU9{K zwXv+ibGrxMuUDeW=uGyNP0d}{HohB=l>3Mu-)gvX;-eq z-%8bRRuf^gD|_Dg>~-X0JqA2--HIq5uULlz33&KIs}b{V5S1kbzgk9Q2BJ2fW#Qzc zc>c(|y^jz%DbSA2x%1GOZN4&}o#R9Pu6g?&ArfTV#at=UiOzvpta#AF?|CpbLV&_H@p8V<%?QNSZZMM4033mMppl0VKnJEG+m?O9fUy3j;mEdeRbimV z^E!k;|9@2RZwN0?koonO0>IQq&lDPGU6F&htGv3d9<~OC!z}o7I$kXQw^P4$j}T=% zBj0)x8#Jva=|)Ev_cA%O8%Ua38fgmG0iOpRPMR!Nyx0$@<%%5nJ}83hh?1;)saRql zyfWbYbr|)i@6^ffYH!%fgi_K;U0W=^&hL){Puuvga6_H@_u>HH;hAT=$-vm%y`6rd zvfB27H!T~9^d!0d&M$1;8K!~sQF6a!eEqsbFV0)KvfA|LUhB9NS8`(#1$>1H{{!P? z2*#}(-Q$4fM47_*Cbk>XJOUra=#?aLRpxF#xbLybi%t(6jd-#F)8&Ue_cuaqM!_>b*HWwq`$zQeUKaRYYZucM4W5r7ytZvE@&9jZ zbCM5eh0cnCMAHAhIKOuGHfQv^sOF*VYq1-k!CM|~L@Maw#hu6nSmB8=2`IiYWBy3mH?BESelJQ5Nq z;?e%@uvuK4wN+KRe7VSi8C`ckRr&XEN>#q{R2bJ(`#ZN=78(1J&-El7S1CzoodxN?G zZt%HNWPkHNlfFC4?VGs-9G%{*fBm%|kp|Vd6-8d0MUgYl;ii76_1|&ESsl*0V#JMr z3>?-eNH>d2%!q<*Nn5j^jS}gg5mJQ%(K`+f+oUVvbh*DnjTCL}#-~YF)A>c1J-zdu z`~5^=0T@7t^d74Wcg2FoM3h7IXfE0=xK;YMS;k8ApKYpRM2B?3jmPTlN zTw4lse$ToPY(X?NvJ;g8saPw3-q!mBZ-fvJc4Il7zN>yfE9Anf6B2>tp~O1@^s~sM zi2G_m;}Ya}2tbM+SaFm3V^ac*Z|F?6 zrcf`di=-#y3B@LqxMV)Spq5+OQ*<@#J>gd9_v5z;zNMUz2^Hm+j$Z8<+rQeAVpnTv znrQ{`I#G(OasQ+d?!>z?c%27P{z@m@8Gy?2fZ((g_!0qe5;ChI&sNp-!oA5s5SEK* zP@;rFRMbp(oy&@8%TYjGkE>SS7(imk+-ub-UiHyDBk94p-x ziMM>7$_@~-r+CGNXl(T)nTKkIdo|^d4zw!FBY<}Z3S&;C05kH8@~)BWPfNcTER;X7 zO_ft(*O($&V0w?IgJC_9KUI)R12Y6nX6c6~m2PWao=jBz{6-w^X$CS1R_*Sw#E8eJ zB0|j)%l4%htxW||GbV3KHFYo-L@>Twgh!!P$R1+J(b;7Y-*w)711S?17AN=8M=}8`_{= zfxOr&dCV2wNMUP{pv!cZ=}`>yZbyGy?m98xqk1=bpH6pf`%A^rFGV_US!!MH=pVyN znsTMyhb)Y@&0CgQ6D-#j(GU-F9$zrb0o_&#c zCAKbrW)y#GLo$sqWc{=03;cJ%BAm5?%sl))b89&v+{WCz{A|lb`cKT_w9JubtYGS>;mvC~o7%@PViXHu}yLZw{Rl5544nDkdWOGSy!7+O3e zO|1cRTptW}*c_hmZI#l$He;V>zY|oR-3TBo0h7q6GCGf>TnQoU+2q(IpDa12StSvZ zdm%{A30t$X6FV3RbOf9*uo7gqJeKszza(w>#G z{T|rPzd_U?8!2O$f=w#jo#2|7*wJ+xuz8Id7-nAyKW)hDmgnVWPv78KIfqx(m$@k0rvCM6eZe+&EVh{*U9;4pc4J{qhxmf{ zvW^JSw@p`VzfT=TfyZpEk2(%G(#^HHg1Z)$={M!SE{Jc6V52+xR#Gbt3N)^1Y;1B} zc?&z>RA40`YI4+klBWX4WFYGY(xWnh6%}7e+A*YcH#w6V)k%viqW5+l;fEF(teR@| zr834hR9DA?fwR-eMPJ}iJZ@Tn*070)`AUPb)umeIY%8$oy7UB}#c|yu)rxi3fRKRT z70T|BJ%@zrSRs#-dQqah6=`o@o7+~o0p^$<^=EJ169Z$t@4v+)DLm4S@sllD2Hi)S zG=TlA>OkIsGw+@Z2xmLW#@i(s@DBLCaa6H-6zJwo&09e3)1>rXW#95~R-XD2uE_{^ zzF6{*u-P~NfZ>ed|927B(;xEbkQ4a1@6b?EjXbBqdt3)&dt6$jShNmK>9Xr#aHMq0 z^9-1|ykWkcp6~4xNY)qys$`+@^Ew1gZtkG;)wh3&+Zc9G9(C5FoM{qM4$)#2$*nb< z7<)`OQ_1r*WjgE_N^2%nZ9T3eTcfHG2}%&`9d%l+zs{vnMS-#c@`}9y zxk2|bD=e|DBFCAQP5vorwzMBSS4FP{e@Tw`{btcsJLS>j9YDX7H@}8|I04e?_?cS# zXI~hq_L=S+HO0*mVUOpV=}9qxGs1z==9*^=*pf#xc_wz59rvWw@3eETuN#=l0f@+a1;-{ekpnoNH c@dI+%w_Ubw|2KkypW;D6+5a2z79s5a0C6r)6#xJL delta 4371 zcmYkAXHb*P7J%u!8F~OCkVHUw2Q{F^K!A^?C=n2lUZsRyUYcT%mIM$*2}VKMNAD;_ zLYLk_K&2=}Kzb3l=$$)rcV>6?*R#(#bIy;m&4$f}B|Wq(kT%HzZ#pWfU=T5kNszcf zdl4w4{$w-M-3SpP82X035d5j19focSovr=ahX~x?f$1k)H1+A)#4lBN3|$E75PMOK zTAbY)N_z;lqzmGCD}7OXE7fajWxx2V(xt|&7%p2~S`El^99L^oSzvmy^cRmi$(9Hl z<~jZKnY2q!`C-iD7b)3kZc~}5^vY^NtkJ-xk$Z`Hj;MyCZ7SYU8Kp;Hxc<10`<|EY zmH6DyoJ|EiP|~M|y*8UaGDZW<>rY|{PA6-VT+N%2Z=pO;0=+zY>I9*5efjGn;WFNl z!J2n|w*#`z;nj0Z73})H#}t7?6SjF^_L9XwLVVX?T@IdJPP||n^?W5S+&)MaKCIdM$hBVYujN*)A`k~I)rRpgRG`JdlFWc zv1pTtE{-Eodq9*+LX(c>=Gd&LW(>#8>ge!^SaLM&t?5X(W=HLS#)TCNij>5<)hQrR z*VnPciQMa-1-<%-4O`!c{hdEZSlf4W8fu!1($MNBX5)MZq=GtE%mUf9lheiFTk9Tz&t|WY zX+vr7h?ctdIDiJbTD7t-n@rkSv(OY@h%?ZA;UjKeX2ZVaQK9iLT_ZYZOXh^N2(h^J zBtKO~H-kufZ7a1IP}bp@Ije=qE^|PjFF!@#TV*n7dpJ~RN^4DEL6`Yd!KZY_ z`m}UBR8&+XZ?B3g4Fpd1d*n{cO0d74QZz+FZ<8p69==GaNDoAfzk}fuj zjV5tCz7{N9p!4+i!0YhBU88WdsoyAlIoo}m-9>C*uF&%e=E=PR zTT9erKV;|G(7e>|!vTP^jR?K!afeTl>D{m8cdD#|zAnp?g{NM*)!3hI@&<6s0}l{e zY`8z!&!Z%s=(T%vm^=wJkCh7HE?}!n*ahc%jOV0=ouAD)f#=k&`Kf#iIDQLYxgJ?p zI*;6V|HFwCKdIKUUBOm1YJ~Asty;|ax?Flqf2E^$=GbH^;~kQ`eq{8pGq8$3ZUH;A^D`R$D5 z|D+S1QZ)9vo{~hY4?J+-qf885Y^bjWqm*!=KxK!kqg&o$(4IXvL3;7y)!X@(^U6(#NmAl69HIyfQ$9nrUX*=4buu;P~u>*UYmq-}ASwQ0O z*s*S|K@_)VkLt&xEP{#()crW#^S*X&&9=54&D(90 zOn&+hEx-;B*mG?06bOmV)@vNTlJr0%`?mJ4w$9h%K0=upM`86h$lYix9_dZ};U-R; z-%^4aghLW^p#-ED9$E*361>n4(1DCCk;v*8BpV~h3N3~^r?gA2mm>Pi6j;-WrD*Eq zX@C2-UP-cQGG|EEJT6%HCZ`Dx*B_j08Fva*98$~#3c`u$S$aE!Z|E6Qyo}e^De4OGf137#d~=OjPgg z7e0OGr7}Z`a5#s_RS@i<=6$XNMdWh#=Y4C23?t%$wuSyZ{-qs()|2TEt~E>V_ouO= zrndA~iq&W>wp>(F<^t)J-hQi|PU4+$td>T@TwhkbC44Sol)?cFxlnKxNJZYnQzb>u zAmVl!^oK-ksTYHc)hJFpFXimHN8E7UPC=!+P5Kl?H7iHU7pNP@ROx=h<_qGWB26sUs@KdNuYJYgTCy+{%j+~%3~AeF>R&ynnkXyqp802ML! zSBm|hlpO&p_7AlKv*uK^kwI6QtWfsF=YKSBHr!XlUwRExEI;x&fW2aUCL~3V?>u%_ zv2xscy9r+8l*j1!r8bz~JCI9bZ};-O^V$l^`dcHe=Hc6GvChIv{i0xGS(Hw!3l*B| z4JA0Jilk8`ODio)cr9z|Kf}@pLAgf6P3X|@sLKdHus@Ulw9BGp=?1_o;17eY1(_l*LO=%82Ya(wqzprB&jPLeU zc1;DZK6aV%9I#*FwZT0;XK@((q%F1WJo%dRCG(JUh7u^asqE8q3y3Ps;p99$1LTjN zmmA#`1T?PDrl2PDc8@CQd(Cf+3~{-U)i;}-BrNYr)pV3aNltU8@_^g}{UpwTn!!2X zhO#z%NLTSWig4C>2IZgTT0pvr-5{*N#{S|DRze<;QOIj z+4`E1R{Bqt@e(hU@^pKjJN6&|()!J<*j*Vn$+E7OE{h2W#{yf4dK(qD7xeaI7fKom zLiWxch;>f>#Daq4P%GjXI;52jOX$wX15P0uI0N~^WW(e?=NYp=Ak1g(lrj3i^c7;U zXy~0suxI_3-K2>#JuC4l?BSW-tG{|)hyT+9hW`4!M8MU<{`gVfphoVP^CV+hm+VGY zw}TQtoerBA>xZRg@yJ42lt0txL{7*?fJDx4f1%sla5YD){y#HXti zmqi`BQwqt@&VwuGotsFlhn3dEvfj_xA_&b6)aZ!yd&`BRf{_#KaYsHhJ*jfPZ>ObX zb=B4!mF6yAch4|P=AdeW)9=(4AATKETBjN+@Gm4fTZVigK5BldZf{k?vfJghuWz6dOZ-!J@gg}n&G z!S9nPtOG1B*geKFJojh4`D~gt~e@|`~8HL2MPaJf3Q*>JO$iCY)7yZ)nDHq z@4-Lrnax}l^zMEXX4%$~MA#!7`xDjDDVia1#T;ym@PjWCd3~9EJRf5x5XtR*{4;LY z_PaAC0zgv6kebW-t>`OJld}qJ!WL^X?qWqaFa5A(#oD7n2i@^?yIcp9vx{GG0=Rf$ z$~oWCQ=wDTG-t69{-#r~F2eV@jduA|$Z$C=tm%W&T|0nJWisxt>0SkQs~gE?q}vyQ zsz(%#+f`hy*bQh5cAl8nrrV(zVorIt(k*er!1ddH$Ss0fUD3+V1Vsuq)S_3De{Q99 zVgH_3(dHMC`-VAB4>la{^r;oS%n#MtsPk9*aoBKaEYk?p=P1@LPg2iZ=j!vXYdD;o z;ZQfFUyksV4Op%IgfxomfZAi9{d!(6^d_Ltu|&M`&E=r4*8^S8XDL&}s3-VX8v%8=DwpXl#}IAd`U6 z?U*mPqNti&;5I~(jlw6N>AzQQ7aP0N8`vu sjAQ~6%VAQ~3&a_in9$#rI{V`IGnf4t&HlF{A|Dck;BWq$_m6PkKeWY2nE(I) diff --git a/summary_results.json b/summary_results.json index 1fbe9188c..939332806 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2182, + "Performance_Result": 10.0769, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 5.17796, + "Performance_Result": 8.12825, "Performance_Units": "Samples/s" }, { From 8696bd222231ce785fb53e250860a0f03698e5f6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 22:56:03 +0000 Subject: [PATCH 260/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10049 -> 10052 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b4b00fb1c..c91ae2cb2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:51:38,128 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:51:38,128 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:51:38,128 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 22:55:58,285 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 22:55:58,285 submission_checker1.py:3270 INFO] --- +[2025-03-08 22:55:58,285 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:51:38,128 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 22:55:58,285 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:51:38,128 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 22:55:58,286 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:51:38,128 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:51:38,128 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:51:38,128 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:51:38,128 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:51:38,128 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:51:38,128 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:51:38,128 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:51:39,109 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 22:55:58,286 submission_checker1.py:3286 INFO] --- +[2025-03-08 22:55:58,286 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 22:55:58,286 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 22:55:58,286 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 22:55:58,286 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 22:55:58,286 submission_checker1.py:3307 INFO] --- +[2025-03-08 22:55:58,286 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 22:55:59,287 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,8 +25,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70822 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419881 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26688 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422411 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29277 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4fdb29846..dc6a988e6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -887 +888 diff --git a/summary.csv b/summary.csv index fd04fe0b4..d45fdc1f5 100644 --- a/summary.csv +++ b/summary.csv @@ -3,8 +3,8 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70822,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419881,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26688,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422411,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29277,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 27f458fe5..754477255 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419881, + "Result":0.422411, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26688, + "Result":6.29277, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 2765fbd1b23f98a58df6be589e93c0eb803e7dd6..1d769a3e8c1617773e42ffc739aae060514d81c0 100644 GIT binary patch delta 2838 zcmV+x3+eR1PQ*^Ia|i{9)AM;vlXwVI1T5}3$dk_qxqox7jjWU}VfaV(=5%mqTA8#C z+V{(0Tt>#gANCfLk^o5r;FaA-D6<7+TT##v%x z91I42cA`zAzlf7?`upmTS|?bADtjHPEVYUDYN~13x|Mw%-zNS#roQTRIl_0nq<3@> zgcpJ%9xeZ=!0}?1v zE-J)0KR>IuepYewtm3vvu~li4SK5SlEhdfdX# zR9wSvi9~$vCqyDn*gGN-!2U)g&27nhA`v`&AQHjTMoXsyd8xpkS1&0z_oY)hTG(^4ch(y%; z8-J0Adf%hO71zOsM+y0b_>uG=N;xeEZ$BP;oS@ERLkBzXL<#3Iri(Fh#<@U=vcxra z`84q1(Z_#Z{&v+l?Vi0}E{5WCH8cv6@#0IZA5QDjN&0h|u5{8@b3Yp11+$?arRz2S zF7e~!E?#_@2649ZiSO=PHLIRPzQ0^#)qk_KRQW(z)yO?BsSk1)Q4oc{!~Rt>7vz7hqnB0$GASP z#-&zEj9OFcX6{$;7IhA%&Oz$Lwe}Vzju^%XN`N{dk<<~1q>f0$75c#`ppJwKXn#)& z!rK_`i5kxG;ev|}hBw8DJa;6LBu0$*0wu~4XaCZv5XsT_uGi1L&+MgNUskF1Mk$c< zZ%G*S$9MO-X4kB}{z9c+YMw{2ic(6Tc;ne+Utk$-lVt8@Zj1%dtF*Zd4DAkyz+7g zdNo@s zdvkDuCA4%{Ct?miho~-zk~bYMh(=i=^<{4SFN4jeNipMc;QRCRdW>TK?q1d0bYA>! zIxDHL1fJLHfvxy-Yf8I%$A4l~@!Rgl3U9NwT9QiCI88| z(w1PN2Eim60F!6{Ow`~ywowEoNdRCLBxr9AK7lD)I;;~{f+?fABnrVKJKG3m<9`ZF z^+PM!DrzhRW@8J?W`D!6a~Ie)n0!zFSm9QA%avfF#&IN=s6jA^2EZg50F!L*1DMSs zFi8RcvmilxbMOgF+0r4Mzar*R6Ey_P=8FbX+;#naCaWl~6q8LcS*nJ$;|8qS z;XYAlCun(+Nz^#5WD+&VB+&qqL<3Bs##55XRuPk=6KM)4N`KJa9DHI@wshDiJ;|hu z>XIn(^^%EP31zEVC>LMSxp1+jQV^NkPnXkWHcu+ArhkCRDyl2>#j=(EVp(%6H)j>v zMskNgM-lgpH0ThGBON+KgGiR}=r9c+Ni>c`vdE^x5&)7#MjZuC9cby0PGExsqPirC zAjw2<$EQlej(C=KVWb*Sws$V;jmX`;PSlC7M5M>E9t5R~mST2B9or z;b9tpl4t;VWNP$_*wW)eG_Lgc5Dk#YG?GlFkz_KBBvTi8d{_cW zW>JE+29FQ4bVw&aCPZ~f6f9$j2A8qAVvo;%1{>!pmqg*5Nkwwb z)D`=Dz8B3O+PAEv$TD-L&KrGnqh=|4$E_c&caVvu5nCF4h{lyhAEH4rOBj8a2FN5D zAhXEm!x8|QMMj^^!6!1o=tDX|GQsFWqHxYEWAw?5{~rwIOLh9_ZKqFDdv0^b?H}FP zL4PK?Mr`TyAsSaYeTW9hEaCKF8X%KsfXpJN4@&@K7CC)1IDMd{LplLN0Yr6K6p&fQ z>7y%l`aDPGf9fcxq`p#63YU+iEq8~bKsR?liH;Flx_pSnl`bEmK`2YOe3%BHBqkj# zp)7LwuucHVBA1T_mk+dbNGHkyD#7MM!+&5U%S5n}t=Q-D9G1y2BU1g)?bu3cE46^l zn|!i0fNM)4>$!(GP5o%(pZJqWZ1ltQbgd#ct*a+nU8()s)kC13*b&pv*6E#|N2$`mA(I}Z@>A)&~J=GBQ5<=npU^$C40e!b#yTA zjr{XoGzw<1G4Y3Kp;pZ?jcD~?t>x<^oG)LsOyf(KB;oY&-PrF9{3v^u{=OTAVe@6aWAK2mk;8KeO=?d=3S9e$SvJlX4*{f7`(&2-|_SWYs{SDJ|Uz zX_AOq4x5Pn%90dCpqS=D?JM-LUB^olIZqX* zjaE>h6F9W0A4+Bj(-z*WHekH~7X^@VgqaE*yf*}6#||K=OM*iMai^`MK9qKk4P`%c z4>(WLM=Zfpe@;Ec9WXXC5sgUBW~BMGO%LD<(%o$6l?$g7{$${$ zbrNUy(TUmEwr$cbdTYp^;jhi>d%tQdRCg){s4h7p%mVe=)+N3y!r>a~{3e`uCxAbl zf_~#|Tx9429)$p8h&w5ti|3c^8r6B47je3X)8#hL7)eo(V)@;Tyy`y#c!2KN0@)NkoEBsro zjnyqsMXR_5CLgzJSdL!onw#h;)o_t7U3%VF85tRZSwzCyEdO20RETWB2YkFr#$l41 zI#OFSaot&3Q7%hgdR*gMhU{Y^IZZ9v(!$$|*d$|`xYC!^--AcXNX3Gd>$+CtTN2*~ zFZHv%RkrkM&u@F11`RHFJX{LEFMU%Ktds3!)98MmJJ2#^ue}4VZju*r3Ut8ML7(of z4cou;c63_W$)%dtb%S0XN~N@xxmGenDy&7;y!4U`UC;=T8^u2F?mh~?Nfc(`7H!*3 zF|@0hU>-;qriOI5D$wMz_ zxvS*nHBLDAQtQDo#5+AtQU?S5tV$IN({@;Tf(2}W8JfPZr7-OBY6yt^PCVr#X>Ko(1PXjsU6P&DYW4Zp$>*!1B^lfDOtnNr0!fBY7yp}H&) z`Ng*qu;h;T6D_SWv)r(D(9qaS|696=!fw#UsO4G1x+9t5r>Lapq+cA=Tl zT(`?VeR#XBKhA!KJ0IVVEgIUcZi;4A`~9TIHjVTt)mGmbEC>K??07INY26<#to?f5 zri)&g@&ZZ>4c$(tIsdhKC$M%XRc*atWbx?LPNtW7S%uZ7wZ;*YbV6M`4`z;!H5}+^ zuE3nzym$zf6=8*u|F}|@ObxmgMJ$YyF3VoAxlEmxBc(hT2)q3qNsv>Il!yg*VT)k4c4q zsp@N(Ei~u({UCDIM{t(G`#PAzfWr8vg+yD<`lisc`+5FHnw1I1Lnfm*kFGZs`v3T_ zG%63WC1pfF!-c->-Cs3v78rwrGO-W_6ch&SMJh5L5)J8j2q1i9 ze2JQQhH6!;$F2_4%7Op{-T9P)FXZ`x+4_z)-GSJz}tuMqTynQb=1aUR6UK8>&>oU^4{~9LpH@t@On15(z zWT8*FT_uDz@vP!e+GS4Jg`%~8)to|JsftdqZF>-k{zT2Em52hMMi5Jhh6JU5(+uM9GsU_-ip{ zuiqiomws4wou6%9%H}Wyi4*=cM~1`ADzZ!SJ*Dz#;p09SE5D?gLFT2578inY+xLJp zrgJbMgG(&*W z)Ap86x3K9cX)}1Z#Z)B|KcL9M@|c1|P+_uWc)0aBQzq%<@ic_tJ(}lo7=BM^<|>q> z@rszyIz)tfz+ww;rP~DJXB(Om35Jdy7+ccw5xu-Oi91~I{M@2}!yO;i4|gQqsIs<7 zw}pCc>x5U{d1**@mYeH2DvM=5G@-iy@EKGn^PHA^93VzjV;*sL-hfE=;s!&-_T1pb z!$Z%YEoFV6Od9Ro2|SE1F2iHAF4MbfRKs`iY}JkNk{4;%Uv%W3GuSBt&%s{BSeUAw zDUUfw#DF|K=>P<@LlmN!T1h9J=VC}=gyMx5giWI)Zx9K)L zZ}*ekZJA?o%Dn>DiE0OXEBiW)G;tK*nS=d+XIP#~G%1~cE8J=9*8a?wGhcb{g1 zWAiSEhBY9GH$R6UJHkJm!Oe7!a{|+gUfeEoTPV5xvHa7Uw!V&fkjSy})itpwl_-7e zgDY{x>nIU>yk;{MO654a^Mci#(Li>W_~`AK?L%o+I*u8XrZ}`h1~sdj?i|wz zLMvo*X@nhURbAOA<~6v<=r(yk>*n8o^KjY2+!J99I;`;NIr@KG`81RNk_C z@W8Ju-JsQ?{+vAhiUiLslhYop*r6v#S_*nE;+(yR6JW_`Qyxc|)sk6nV zk7XG1boX~w7p(L67R%d|ta&QSq3qfr&fQ%XW41b?o1EYu``TNpD6`onK~Q z2>tVMHI8b-#ZN`Q)-Y(FBaG`+wctA_n4MV%FemcxN&)}?fxW^2W<9t$8z4VXKE8e*J{ZgAs4@+M zQ*ZD|Ly}LK=Mss&XA_|rA9(PEQ+2Nwp(JiHMdu&i_hT}bkiXdxJ8yNPHv8Q?hTgSQ3A zDL(DF)k6Vca!FK8a~$B%@RB!qo?GVN-dXg7Ku@E+x*Xy{MpKpUN2 z^Q(O7dW^5SOZdXibGlH8>a&y*9~CsN6r@fz+7YG|+KuN}-bin`sBUn+S;j#scc?sX znIbMaWHCGRr3uWn?IN%lRWSvjqbDo`3`D7B?M^?p`-;q2IU5jCgf?v*L_wJ m91i|#B^Ll-&J6&F@45Wn3J$Sjjlt1e92QvV*Z~T3|NMV9d_3m> diff --git a/summary_results.json b/summary_results.json index 939332806..69ef3f7ac 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419881, + "Performance_Result": 0.422411, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.26688, + "Performance_Result": 6.29277, "Performance_Units": "Samples/s" }, { From b409057b38cf9cb701aedb42a643a50d9fd56388 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 23:03:42 +0000 Subject: [PATCH 261/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10052 -> 10054 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c91ae2cb2..2804a3808 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 22:55:58,285 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 22:55:58,285 submission_checker1.py:3270 INFO] --- -[2025-03-08 22:55:58,285 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 23:03:36,949 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 23:03:36,949 submission_checker1.py:3270 INFO] --- +[2025-03-08 23:03:36,949 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 22:55:58,285 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 23:03:36,950 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 22:55:58,286 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 23:03:36,950 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 22:55:58,286 submission_checker1.py:3286 INFO] --- -[2025-03-08 22:55:58,286 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 22:55:58,286 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 22:55:58,286 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 22:55:58,286 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 22:55:58,286 submission_checker1.py:3307 INFO] --- -[2025-03-08 22:55:58,286 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 22:55:59,287 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 23:03:36,950 submission_checker1.py:3286 INFO] --- +[2025-03-08 23:03:36,950 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 23:03:36,950 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 23:03:36,950 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 23:03:36,950 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 23:03:36,950 submission_checker1.py:3307 INFO] --- +[2025-03-08 23:03:36,950 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 23:03:37,962 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.351306 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.26254 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70822 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.12825 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0769 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0479 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62118 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index dc6a988e6..a80d49926 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -888 +889 diff --git a/summary.csv b/summary.csv index d45fdc1f5..3792967df 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.351306,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.26254,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70822,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.12825,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0769,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0479,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62118,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 754477255..c47b9659a 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.351306, + "Result":0.26254, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0769, - "Accuracy":"acc: 75.200", + "Result":10.0479, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 1d769a3e8c1617773e42ffc739aae060514d81c0..8a834a45b13078333d00b96904c3eadcbf829770 100644 GIT binary patch delta 2881 zcmV-H3%>NkPR35K%>x7_0DWMy(F4{4f42{}HrBhnbWYfcOh}e-Bc!rm4fAcZId7Ob zgL$(T{{>f@`65cy$`)kJ$$?0VmBD{BM-<**^61*iSQyW$j2Pmzlv{f8Qp$Ivay5a)Z?G472VV^Lz!(`a^9@vHR#$te zj|-5n#=|mLk?Vl1(t;ZqiDja8Tdq~CIp3(S1ug3<*{mxCW$429yP)h>@6aWAK2mk;8KLFS`a)3_?0085&dwXJhEY_^fyVLnml zCTMvQNz^#5L=rWKB+&qpL<2~o##0i>RuPh<6X^*kO3>aMd_q#Tbl508iKL9`k|^@+ zl8Ic&WUE>x7hlq;aIuC`0GZrRm$PLyO)9UZe*noUsw?%qvX%c{S#vBm=M>sTa)&)f z5%Z1o=Maq}e+@cBgGiRJ=r9c+Ni>c`vdE>w5&)7#J{<){9cby0PGEloqPirCAjw2< z!>3BWjzseB*d(c>z*1D^?K@i6yqvdV8_F&Bj`aj3dOvJw-ys@T`ge#1p)BFxVH$vv zXaLG06Aw!OP!`#ElyupgK}&~pf>456heYAhmHBmGf9YDWVdw8>@`c)XKC8TsC;dXH z+~zJLk8*cVi9QfpT6u`Zm0ljAK`KkQd6)*MBpRTy$k4+Q0F_0S9wi+&XVB6io#2io zICn@C?pQMK4(wP~Y~DdCfBu%`3pMpztGuhnu4zv@JN|S-c{{K~zlbeuJw)S5Uk}kB zmL;4$e@p{d5)EKkWbR=J0LvnKj{<)Wv~)-(xHAG49}-1Yuw_g>s$!SVb6ozMy^)vd z^-+1VPr8bAo9>P~u-f8eYP5^k(&9rjuC(|N4Uow+l1!$NWHOB;Qx{o$SOQ6AQG&Jx zix0GPNGCuhM0H6NEMtiVm$ABHi_d=r%u0$Zf3uX;d6!SNH)4CO9e)I<)ebVzF=9)b z57D^N=0h|{W(k`Q(*T)717sH2d{_b?v&iP7!R7-k9nuMGT0&HpMB$uCMRLy672ABi z7tJ5qwydPcGIOTR`+RhxW+{8etskv-kcplVTl##6#+5!FqCqlC_cgJ6J>E;e7f6*{vOOp@LxYFc9Gzeu0lMmAXl*FT>C6q-b zAJz##S!D9jVDf>M4(UW$Kqa_*Xc(+ynFv<072ABC!!j9WM5-UU9a~9lr53Pxk59G+ zaBWFsJ@*W!sUMB}6Mr&^jeeM(u2tlwb@gDYE46>SdIr>!Jnb)>4T9{CR(q4xe-o>c zU^X)1KUdGfva@GlSy{6F@MKc zM!_sLCjKxj)T%kA5v`u9wS1j~B=hB~mT7zmlO&uzz8m|!fgfe>(%*N(FialbWT#~F ze6pjz0JG>4m;nx}k5Jib1ONa86aWAe0Fxgk7L%kD5r6oaB21-#q^QY-f@CZELhdlN<|Y*Q<_m?`1ZmG!=^DyHln1d5p4WL(^jsi;_PO*Op_d% z@GbiRIk%J@&1hDn&|(-5TOyKqLE|C%0_Ms@GJmn?tcnxywfW_MrL6o8j|y3}$Pyu- zY#PqiG)d_qNfV(idgKDKCVDN;LfP}AWD#)A6#vDa3S$&O3MCcl$5i=v#+KqIEtRBR zGofzcjzip9DUz}(X`}==1RAghNtjfB*oQNj{RDM2uzqB4p{nW^Yuz9yz8oyM@FvXbW!%a_T@I#|KSCOf2v zgIv`@O1d&qDo;(!H-V?8byV6mvC(bartrek{6cGZS=uzbB&EfmM4+fvhMs70j&!0PYWz>Ls>pXoGh1BeQO#UrRD7O?`UvWgDYjV!+i}NXmDlATO2ES z+TohMhBbW+D}8N;YvRS4c(Ep4tce$E;>DVHi6&m6iI*txLcD@zL)*0{BTJ(#GLMSl zsgJQd-;}aZBpE18Ojmks3~lXsrGK{PsKUgtG)4Mhzi+5=4M#>J#`nAZJ2O>F*Hw=0 ztz^GnEipjB3tDoRudx}Zw~I#l|Br1iteTG4CjQ*;-bDDm%6lgxZn?0J%Xs)v@?Nsv zZt@0`SZ7c1)q&yABp*CG_1W{8LijM-2lDv&!h4PG_{uv9<8gEPChdJ{kbg%|`#Xk# zW6QmTO)P67+)|&qOzu8w84~vU7;2djT3X#}qsD9YHkvAzTTF6@m8to!@o;>GF5p}y z7|`_g9yxZX1h{wRYV6OVMq5!ISc`U|cO7cw4^J4tHQHGdqV9QzOVst&f99>JqQy#p7%kgAB8{ACm`Noq| zRS<-J*Y1YhAn@ec^YQT6Oyn3A5Z4b};@fUFP+!~I0Y|Dp9dOR&j)Rt&=qXUSBy^R; zKY6Ry?0d5~7ybVLvk)0!2nA_8R1PJRb|D^r&1%Ci5QOi23ZuI%*`{p}wgYX+sewXM zT5=Q8CK0uMkkmSP`^t{vB#={&tC{_Fc2PCot2X%s=cUm#$}@rz(A)^Aw>A1$Ez%iE zd|+Cz)@Z2F0X&*FcNMpk8wYRBSa3mtPa;Tr%59ByAy|sB=Q~i$XVIafxG_$#5KFgz z#g_4Jwgp@e@_-ctCRkv&2c~u+q7fB_$;-AJ_fJtNt@kjYIxT zJOPfmuRlTa2eY;!gaQO50DWMSJ|#j4*f?^4PYM74?~cPyzq|5dZ)H f0000000000fPrL_+$KH&xRVx7ju@8;2(F4{4f8T>!8|&R(Iwx#JCM3(a5mH&OhWR$yoHxvz z!Mxdv|AMQ{d=aH;Wec+A@6aWAK2mk;8KLCi+^Lb4Q0088(dSp4DDw4^ ziChU~t6C@*U(&g7v8GZGncPp8(`7bKDzB!0fXOPVEA_>)mH%Q{b1XM!71~B}hd)OV z_l-2@5RD@pe>y~iNS5&EFbyC{G>$~F$fm;*0Fp&U9R*GuXz7qnV1oprx+IDq$wY9+ zr%J<)MDp+0C8?yqQdH*sJ6hMgoVjBg$}Rhj^#moFKWypWAsSa2c!&m}EMehc8i0~$ z0Lmg44@&@07WsITwAq|NONVrVP=Z~DMB&<%8Fpape_F9)=kI6oh5C3ttGtmX{YI(W z<}N3Xa(7UPMi5(id5Ff9W*(wJDofaTmhNRk(X-r zQF*sdx{7t1?v7iq+Tvtt^o!Wi<3lvA^!N}BkjXTXOs0`!GL0lt7kPYG0!e03g0=>a z543bhCqO1dbx9N~V~GZrvASZ9&wmBXN{TGAf0WgEn@_emVtcI}e+8)34l>a)VoRS7 z(YVs*Lo`Tc37-$s0GUJsWES~+SOOrk$mgTM=L0Pr(h2NZLR6PT;hafDa?aEh`+U9^ z%^%vgtfa^?bEeK4eRQK{DSOAQAFX$giKY=-8hwbyl|~<;K{87ieV7KwBpM*I$mqio ze*l?9MxV{WCo;k4Lpnh+!RSMxaLz1a^vR9?9}MP8b^7RSr%zLRZga=&AKlnNCb~v! z>GUBQS2}%&2FWbp^kEtxlW2gqby*aUS;pz3D|Y%k zN9KR(D5#{qQcwz)kEJblhoe9@cR-1be-T@{e2B)CE+3*nC`-6}m$fX$nHvNeEf zOCsyJhd53BXyl*xlSyp!!}N5mA~&t8CtF>q{oB<;pq}Juf8lHpWPi2Vo2(vLf0YEY zkrDs7dKi|SJq*jrlJ$pIi-~U}-{$_I=|9ZVPbe$aGzJgD)nl-Xc@%_EkbGP9o;`>&xQZ`RsWT}|Eh1l`Nhz0j6)+W{ZX1$x9lZ*!G?8oFz=20^IkLx zX0b8xhiRc!%`uH=^m-~nB44#k<4c$%;q>v{*zXPeD0`Rwz8i*N^7tk@C7TD7 z9sLEf>Jpd%4xq>SFl+<>00a~O02Ba|Atn}+rW6r>;9H6?l>(BYCKn2lt?Uye>^AX^ zZFw!3A%DMeAQ{H8NU|YM6sN5|>QJ(QoOT|8cfJk&=yNWRT$=OK&VvxeiDmKd0W!)>RzNN-F)K4F8f3GU$uJ6^kJ4TVl|+YG4b_ zL&#~FuqJ+&CxSP@8Sp9?nxuS}*n>bFB9GdVg z`vEz(lpW1!R;18kn2cH?l6gUs5&8n=%0x1MvFNOd67jY9<&dSU{0@%_S+&d(A)ss; z&ek+Z=`u+Zp)Pvl0g% zH*v=yYONGWS(P+Y0vv&m(JOw2(nk|>Oxv(+8*6Z)!Mja7qJql0l_z1YrE#_xe)5~$a+Sa@7VzWnl_m^wO^nKF% z<(c@V@b*@QHnAkQ)7b)qzA;5jINRa1gU za@t~8LMd7Vv?|0uEwH=|W%(FU63112YZ@D+;kYx){i`q~cH#EUiYVokhQ6ED`pi#72QO}s=CFHz!!Xa&uNwrfvDmWEqo5f;T$ zA7gpGDP^NbGEkbBuJqa%+S>DfN^Q?!g-OrS6zPZkzM;xB92pH6-|zPC%v3F1M>)E; zlKpe+!3;NF?5u|JC%ZAE=xE!v6VwNJ3><+%@|T_0BLG1?tk`@3H~wCrqpf_j1J z1j^I(Wt1T%as0h0$u(VL*Y);B8DcuVEoYsJ@%7?zd@T`IdGHP|C)34zcr&Z#8&^(M zz8`p9yBlP@xkzw5lIUW(d<3-mErYy#N;lkaC2X3LU&R1Y^ezAgN1&Lj`fCt)xDbc8`Az zWj}NeI8W0@EWuMwJ;fa`HZu{8NX}-Y`L#_C;0!|p5|np1OEP@+da%;nZ0MB>rxgBV z;HGsFXZO*G+1R#i(k^;y$e-b_&Fg!=YAjTDDh8-7IU~#h_1e}YzAVDw8tVKeoOmaI zKb?Yp<854I=mQ>w0Az?eDW8kym+d4P)p?p1ak_}pz+000CO z000yK0000000000004l2a+4q?K@AWY004MwFLQKxY-Mwk@g)?Kf+iXQf*X^c8x@oF yB@i2Ve$SvJ0RR9{0ssIJ00000000000001hfn$^2CO!eSlN~1`252Mz0002)Z(A+^ diff --git a/summary_results.json b/summary_results.json index 69ef3f7ac..fc3e9e051 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0769, + "Performance_Result": 10.0479, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.351306, + "Performance_Result": 0.26254, "Performance_Units": "Samples/s" }, { From 5ae416713fb4ec617b1b7b781c6c3c536a84ea51 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 23:07:58 +0000 Subject: [PATCH 262/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10054 -> 10055 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 2804a3808..ed15972d0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 23:03:36,949 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 23:03:36,949 submission_checker1.py:3270 INFO] --- -[2025-03-08 23:03:36,949 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 23:07:52,895 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 23:07:52,895 submission_checker1.py:3270 INFO] --- +[2025-03-08 23:07:52,895 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 23:03:36,950 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 23:07:52,895 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 23:03:36,950 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 23:07:52,895 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 23:03:36,950 submission_checker1.py:3286 INFO] --- -[2025-03-08 23:03:36,950 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 23:03:36,950 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 23:03:36,950 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 23:03:36,950 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 23:03:36,950 submission_checker1.py:3307 INFO] --- -[2025-03-08 23:03:36,950 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 23:03:37,962 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 23:07:52,895 submission_checker1.py:3286 INFO] --- +[2025-03-08 23:07:52,895 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 23:07:52,895 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 23:07:52,895 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 23:07:52,895 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 23:07:52,895 submission_checker1.py:3307 INFO] --- +[2025-03-08 23:07:52,895 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 23:07:53,852 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,10 +29,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29277 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 21.0866 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.12825 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0479 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3424 | acc: 77.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62118 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a80d49926..a6ea319d3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -889 +890 diff --git a/summary.csv b/summary.csv index 3792967df..39fde6c6c 100644 --- a/summary.csv +++ b/summary.csv @@ -7,10 +7,10 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29277,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,21.0866,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.12825,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0479,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3424,acc: 77.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62118,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c47b9659a..342e9d52e 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0866, + "Result":20.9761, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0479, - "Accuracy":"acc: 74.600", + "Result":10.3424, + "Accuracy":"acc: 77.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 8a834a45b13078333d00b96904c3eadcbf829770..81d9972ed397102df2a5fed73f607cc77a798be9 100644 GIT binary patch delta 5172 zcmY+IcQjmG*!D*sL>nb~88hUOQKLtc7`;XpWf(+{8lBO?5M|8h1R1@H8Z{(ZwCIA6 zCt9>bCkQ@yzV)tmz59=|_FCuOzk6TTIeV{t?|$6@-MUdC3WmBKJ#S(FAP9sD1+w7w ziO5ik&sSs_LKP5nDxprr5tiW?u#I@}*(95T?u$Hqb@n>GxCd{ihQB4hRwQfo&vLOA zczstrx9L>S`ZVbhftc(K0F#u$*fMpeB&o13VfcBZO;~=OWXRON;6XgMEelI#ZFCMI z_U1U>{2)r0oGwNja}d$W0NyiaqRD_z1)y;25S$*;q$mQ@()VE{P${wuTpYK)(Ck=D zFgmjExb`=iifKR`G5~`aV&fLnm9UC~`{&q=uk5G10{P`336Z@-M>$h$k1k3ic1>$; zir2C{EZM-B3Yh5VNO(P<49T??i{8QGm#`x_T5ip3%0nE31o;%)$GEnE` zP$4(}P>J9zu5YYyF6tTXcGI8{2WdHcF`K*xHeZ*7wc2diT@9MEl$9s7tjIPW^~<}- zx-a@X+&<~xv+sNBQfnMN5CM$2ihRVTvL5jxBIt?k=O%}=#}YxcF~)AQE;jX?BKDjW znR*{hz6BY?n4-zlv!35En_SG%Lf!S>a^c){r=jPVAQjz2k4YjccsrzeO;QErhSK}} z6;oyLl=wfq$TrHij1Fwc_095LH3=a>@q0LT>>}nq$4caO5*@G%^-F?~V5U=v zyP&NFNNjXQ9TC30%!~2w-T6*GHboV_z_o>8Dq8?Xgoh;40SC{BZ##N)z)T%SS71Hu zYhGWGL?OR%;knPVJLin-%sbQz1txPCpD536OC?&H{-rDGhn#sg{{CVth_9AEP2CAI zLO(>tJIHv%z?vrE*CnWWGK{IrKPxPwNP|J$m=WUd$o}C#%~ltR-@D~gje<{T4Vt*p zx5cV<4xIXgff?;GElk7av=z;Go_6$t5%0*b>a~Cwosgs6^C{Pb#F<|Tl;%uRFVFiD z&_JgTqfuRqyg7?kj%$!h8L)EY^>Uo6!NB@7n|H&^$Hemc?r-F#;LkhKX#7n#@S(|?`-+5&7}wNZCoEc`Pz-&B$ zjXcf)?`K$JtPpdedX!}$Y|@TE|;tIhO_f+z%63H|DF@(On@M|Q=ttQduz*eDxGS#gh= zZ+opJbo`z31ybaW*Kf{8lSdKB*Dv-Z60Xq$UHQz;BsdOo5tQ-AFr$(rPw6`s*X?!h zGcOwKjWyQvf;1Mc0@z3Fjy7#*iw5dL5(c?Avj<#m*8G>mchty+n>~nbTyuln?aM>R zKPuEql}0H~4Hdl@??U*kiiwe4*R^CqOux@1hl#NAZ`C!`W5T54GEj-mn51IKMCZNl z<8GP#b2gZS22^~amk9IkdBq%66dnKfh#g@1!A~&1jIO zsVu2l+d_P5l_{)bg*{9Qvf`t_0}{2_L&E`=Y$@U`(7ibL{&9plu^v|(JXfg{4vy*4 zfvg;%9xBH%6)xV!w*7#ERa>+mNMB(zJmD8K1wC;Ehe#@XcMJP4udk$)=XvjKUw2+X z5zk?$8U+098ydcL$qB8IrjeJ^9K5@tHW{hD-pJi?6|n0SDT^cFaOmPL9St$;uzD`jXSn&ADcE z>WundorlyaAa-9i%qur$Lbs4E9@T9-u?@B&Vkl9vut`e3x91xL@{p;MQUL?ayVa<$ zg{F1X!;6!^8R4d<$m#5-Qa*?ti)oo$S+a8a9}5<#(s(xhdj#8PxPcl3D+{J`+$1Xv zmjwwGJQ4(Y*uCTTmL86!6q03LW*`dur4G?Z*gPQ7Oogi6OqSDrn)qEa_qoRLbaFVu z*f_Y5Veq^Kl5nxw9&FX%I%|!3Gl_bY{o_d$I1O}fSz3mH?fIyW&$ku}NLw!at_B1D z_-%Uco|ho*6m>`(9IOEWY2Yms!Q>ebrsx(?c6DD$^NGdKHcydguLNIYlTY)@OA}>&ffK= zU?0yYXsWLnFOnLi5L+e`hn3ck`jB_PVJM#|hyLPbmu<4{D)4~V(MFq{ zOXyzfhU}@3$ocHl+Yd{*>4H7*#3`%|p0o$T3cxu5w5Pc<+ko0tfF4*`L8r`b;u6?a zohm^4FWS>E%_((VUFhV7Aj5dgY(svh%`V4v$fVF=@9Dsy&ZOPT;ABCyNR3l|cXB=# zT5S}bKqQW|8%|YY$EIR~r2vdXQY_>|>Ddfnsa<>1Y=&;#c`Wj2Ofqu*cRr!v)wpJJ zJiAuJ{1?K8dC%Ri^~*#Cq6eeVSJV9xM6A`X1Xo%dli2*MHk&_vJL`i{1vsf zjiaOCM%CYcVKqO5Fr8%|9+i37obuK7^ON2X-YXgSs(KXl1UD86lhr0t2PN{Rz`VEW zC;*?p`ALL@)TzGeymbnMPpm*H7<=%$?}o()HSoE=8`cPG^N+yMDMz)d_1NNDs`d2X zfuKlGCs%Xh9&O_NtB3F?{dCWL5h_IuaezY#ZCtD|K%Z$oAk}w1S4ksQ9M?_5F5X#S z^xP*SMXmK=8Jr6(SS<%%)fA?g;7)^nmD46N0#WIk+i%)30@75{IiVa04ui%xO9)_bq;)W{v)lO(gM*U^Q8a(&k3-U`ph&PI zc!~v9nEYP|AdB|anz{rit7ZRoyrBFIqsV=Y`Vj+@A6D}7%jlm?et&@al^phXyWVt3 zuGVBU!S0M-z$JsN!9YmaxWUCB7h4x2+={R?~|RH0dYU_aj?YZk*FCET5(lFbF$5 zccYe1A(!Wx1Y&QWx6EfO`Fb@sH#Q=OiU~Cjw$SVapPYx!Z#Mg>#U{j&T7@(d?hB#O^cCe)%P$TVe(==SLqQAQam1X&IV@@>e%&p!6or7@A3 zhxbW}Mb6C@y#4>+wWRzN<6`J6vP01QX?$FGrF&YLA(^BF5Ka!kH!!jD+ch!-#wC;D z{?j`|DA2R6{LuzF46nmhf7v)6kXP*fa+#a3XDJ|zp z{DY!LiNz&`Wm3-xcpJ6_)y9lZW+5V)r0+N}S@cZ*jL*4GK zxtWQHIVzwj^SA1u`!BBS>yL6*(e?H^cKafhk4T0y)s&b?KD8L{%-g+V!YxmTvv*kg zxrkC;TmG&^oFL{eYcEc75O=QnrNt+wxr)`C{f15I?^ao;ksU1$D(tBg+Ay)vqcZti zX4t3DCZA`XcIkdNKQtZ|?~2v{>2iHbyjt!~uJ_+MTN1g08<3jExaH~j9p7!(24xaZ z%W4NJyWAFee~(uT02~h=6Rw1T4)w1^yhs`$7SDH=0$aUH58ahKNOBJp%;G`4_vU8l zuUD_<(PJpBmftqXPj9<%#}zwKLG{eS7J5pSlrXU{&BGbzAHPn&?mt9RW~+%i$mIHQ zM)-G8U2pmbVJx$crpBUZ`{Eo9_6fv>bW9Y51V!&Z5Lo->?Qno6W<4MSpi+vN-n!2X zIbr?v2P-Ud{m{bZ6>{e-k>8nm{Ki?11I0F9^)SBT{%L_6Z*j4q98(^2&Z^@J%ixcc z>S5UzVAK)UaaN7{KCi_K%A1?+?;-EL!pL4LQ=fIQHy)!Ak;}mQ;>WXOp6F zWM;xsFg8#LM|VNEF$5Pwz1^&e2OTt>uVad<{!k}Vm^Y1pcnaKD$7aPfw!}2(-v|Y* z5!Uks34VS&f7EEt<2lA~)0a*G6sh}`R=-H~g zr^Ae1c((VI!~ycZ?nJMa_yGuNjW!u~b80`ew|9Q0!eskI?nE^GM~;DHJcOwD*4o;{ zH`R}7QPZdFCYi@Dos_UIdu$b!A5$hU2UpK;#FXrhf4_Kp*wTCHL|bedW_((5 z6jc5Y#-%Ly;IUR^u={kt=(4Key7wy$>J%}F8wnPjv2l{zVPc}}rRBmrj(Y^ZUbnJd zHk3`6Oq0#xU*EI7B(Aj1yA)YCvF+n5Tzb>={&Gfnc&voK>~5xRft(dWl?4eJ_?CmU zoO)qv<>8UaQ?k0~J=@BR5EdGk19_#9?d^MuS&hU>H3W5etsAXhM$9#}E6tr*wVsBE z1xeD&#GLmYgv16kPo2!no%(fqEt2N?wA#IQ;0vNK5+Y}Z9_TG^OI$9)#`^wMXrN=U z1rmSVLEgdn@xV}g?zGKB?w`sd9veOlqMz)iYYM|m%Z6PF=&nF8hRKy#SAKp^n!c0^ zos;}1o{$)N5D6Op3M^k_?X1n%Vu#9$2%0&HPdfpUnMSSp1lRj}DjzUeV*bSeC-b~5 z3f+O}$lY@$h9h&|&3n%i3}L;KO<%;WN1t|+@SK?`*+im9V$Qm!b>Y5n3VEKN$vdjy z|1!(!7eoyNrmr?gg&ST!`&ecvopvYcWzH;4y4BxWx4&zzp^Lp*6qTf4D(taZQ9gZX z=%uIwm0qVd6R9cRcc7!06Xh~F=w%~c@j_oDwyc6<~xcgvG z2hEKgd=c|n#Mn^n)nMiI1AUIzKRE$}?A{1eFfNJ%h zZ&(<*?RR|0Jru>$?yVUazwr?>%Au-}^U);wYs=aPp?UFo@(A89_DX820t>4Xdlntf z`kD`3gRL}(NT8`$@JefycS@Y>PXL2XP?pE?^%HSAPxUQwb150%_P@GfW~%o&guZsS z2VP*QSb7{ZVjN6Xa>3EGYE_C~c6}0(XK&Nw7|83NFhzB89i-n)kv+|qVE3_>wpih|H)_CbNhbG%X2rm$svKg)-+sDqbQkVHeYqu{JOkQwA92961OG}>8 z#6wQ3U8YY=Xg9%3xL3=$Mo!##af6sp6Tm7`*Krz4can_1gMsycA1{ari~K?q6+N73 z$qTmG2noiU)H$Hx6CFGmA6P*(3L~6sdNmY;1SjlV;JBTCgG?A$G<{GKY{Mz7zHZ6# z_;;3tQzqv7fq<0gIGejtX&L7l=|W&bFh^_D=Q9RRS^2A@Z5zp^`iAN98DinH*M|HF z&!?yzoK{lL2LM|dj&jZRMOAd?>t3BuLE;7Lwqn~%*KzvYF8y+hSupLRI%x7Qv69UV zn}KJq`^)dw8TG=*4b6rV9wVO?F(exn`!giI>0Ru__+dWRT%L+3qdPbH&aKsu#17p9 zA=wOE(7LmsFQ~t6$WWuJ0PsW!w%k^wfB8T5$WoHJPF=+}9Wc4UHC)j(oCi6vf9lXVTan5-$4n&KA1!DR zKuxXfLH2l(PiZiEkl;qIkTpZC>g8Hd8QSPg0os+TRJ-wbu7~jV6&YQ6wRYr~aJ zV2puf9Zj9&_s>o4I+GFF8p$uYa;q$<4lp0@JEzE|r>*QfhTn1+g?m=E#)KVMKzBD5(mRU%Rd7N4KxhKG~ud%1eC>hnw9D}p=ss4czM(;_0K2Z2E36xT)p zG%4j8i1Q)0M%JBCUd%?Hnvk`~Yx8tBLA#WsWW)^z#HfJpG-Pd#?)G!$E%R;_M}7Qz+~(Fd0D#M#k%Fx5#}cl?N%OLk|<(ztp0b^-NhcA z`jLA2sNc$5t-BTaml3zje||blqqYQ>!;}i1u02Zc{&?hKFR8usLR6#M z6tI4J`zGltPyBDjM1G>^#}em}Ni!5Zw}=S2=vpOOOD=Mk6E`OUvP?zek2 z<$Ae;-&jQ3PpQc6*uIaNo{*Lkx+po+|DF+D+U!*C>nJ)RT~+M`j7b&DKf_K$42dnH zC1<~c&X}vnu?^YjrO{}1FqLfGG0(D3&Dv|!K=s6l z2*UeNn7>m>L7}+*mC?(ClEqq$N<#;n@$_qNA%4|2Jp2JhFE2_KNg4K4g!-;$B8_}! z+lLIyJDjmd|7{*5OlMLCM%#L+1|Pum`oU;r12`aUqv|kP<8OTN;rjCk$leL?E*lwH zJ;Mok={bZ&X8aJ253lx9f-nAl0>MgsejnbsdMjfG+6hOF zow3G;`_AWdGEB6EsZ*T1IgbCsp?P13+*yI$FC%NvMukjf7fG=umLGY?wYxzPTh%Gy zbt1mIFx&%JTzZb*-^y%yA{+`Vx=V;tChRS0l|pqkM^UG4X=Uys#%>BMb0pQ-(7a8F zE8)GRW#K3OsiZG!|6q5v%_NF5kZdmQPb5Ff| zd3T;(4ZeWyB-REbpadJ;!>9p@EC$AF$yS_$Li~ec&9vGyaBf zA_$9wq2Le{EM5chgAjtMhgWhmMlUGh%@kpwMA$NTd}lcG+4ZprT6{ZNkMLJIFA@gV#lPxw=Bt;G0{HLf3GrbwMH)oNXWnk)SV^QpJxYue50(dwx5j8x<5I0k`^ee zAkLiF3@SZ#f1&x=x5wIWZ)UrP(z&`0ouD};^#H})@R-H`LZyciSLA+fiJ!ISO-cb3 z3Tk{aa(%jR6D?grr~Zg9=M$WgN1H$4u3cH++i<|v>C@NxF{1|;pcKO~6T3Nc8tu{@ zxe4>0ITFOYue4Ep`e+q*O2JDsM0<=Q1xaKHp$f#Z5bNIx!cu_01`}@$wDE}AYK3_a zTJ~KN2~ICU<_N007GBZRm*Bg_j!Y+rX8b=mN}&Iic?izK<&5f&CSBnvHo!=-C7!em zfU|&b_8{%CpVGAti4iD}B_rJ4nOZ)G&+Ix}HJBiY0CR2kW>)lrjJlMG=CkIj&Q_OQAhA z!uhT>z$pAz6)%WRg81udBn7GdtjEPGL@_UJfLa)!+o0gp{5E&nU-?n3`y{U%HaF^5 z0`LA-0@l3OrWlR5Wie9dQW^RPlsc>Xw%S(m0e^Wu0P5v05;$zCO1gSt>Lz$^jec6q zHB3&O_8|Jt8aSGZN*WoWkzBDZO2vT^k3m*_$?JfC5>(~cNYvQuH{x_G0b=|ocmzi; z(go=FpxF|G+~1jm7v{D;GFaV`K~FSh)IDYG6FQ1HBVwhIXo2(?r@Ry0r!thZ!rX)Pg3ZE^3nNJ<9+n&EN~^Vll+%li9i1?mhcUt1SMialK(>SnVdRiYnie#}9Ld}na@s^D5GqSkn{6u!P(REsmJp!%=&r$ovjM9D zgebd*!L_83|9KPcnxMx4O0Wto=&a0OEpiYO&!v z)>`Q_!}TMpD&iV;6zMdhpj0H04dax*2_Z-Im(`rMRU#p_5ei4NvcX&ALIz6Jb#RPf z#=rU20Cc*xB=?J$TmImgzk7}yz~|Rf3N7`D!)DT&o%&xhm51V$#I%n6Q%JwSS+2p> zu8F3q%D=z9#$s)2D$+{-EZ${MaOt2o`U-k;RZHS@9h0N0@8{&!vjzPzsf0*Vb9DG~ zT?CeZaME5ut3OsO960;Hudtl!BG@o82Z%ETWaX(y6DSZM0S{C9IMeR+90AEwS@l)m zId4I3^CLY)M~3fwi@#Lk=??nG4t^gD&3sm=bd z<$h2y^#;;Zk-OgCo~gtpmE5T6nr~)@7_%L+2>OZ5428*Ktni7|ttEBAplsxhZhk0) zMxWCGfEYIVRZ3==;c~vmp=Gf^4e;5QbPOlEC6YlE{5#gj4>wL%oM0&_wy0n{)LvEn z%*xcX>T>t_oxSMgnOOz#i!h&43fr!f z72VILEzK;h@qhRFV5Kr|AwRx zkQB%2O})Tj=|zF#G?;c?U~WP0n_wG(GbF`l&bdmVa_+m!JI2)T4iiR{dZ|Wd4syo2^0aFLRH6^YO%CsHu_5bTOZgU z8p-^+m!*~!sugUWr*fHp{O~}RwT55v`cVnXWN~_tUg_T2BuO3uy6r`V){h@knu7W@bzBMhYubG(~_0L zm>>DO?W5<)FT72s#T(&yU0KMi0;AB(SE;X5k6QOnk3KkqdE9rBtQ%SI!^=Aa`vK*V zvOP>d1tji?;>=jq7WQ_9bdiUzo7bIoo-ZBw&U8`V=oEGIChn;W(Rj7y5i3`3T0ygC z(syz;r~Jjz2A@4M#`QmJn*RQI+j^8U=%~Nu1u4PHS3ZcKjnheo!>GsB)8wVzjOTHM z^AQH^wU0Zdto46q3#G?((=E&7KNHA@cvEYR1Bo078MRD|dJ&8SrIdTRHw2wkOy$)w z1m7A4ykKF?TlxKG$KJ&_M8LMDBY=wC-o>`x?6!c<^g>#5s6+sjKmaD}#bQ^QcO~LU z3HH{1LoDJ69{ZMePq{JJPGzQshf6C*NGvvDu`sq$c0RG{A)J}_fwr+P<|by#Cl9Dy zD`@Ym6jwd9YH==brBYYp>OT5>A#>MIYr-&QyVRz|Lg8yqStb2xR7s(dUu zL$-f>((V_%ecUHQx%ATQ6tNZXJPuITJCg|wdfMOg=7Hb~*zBYCpR~8-Uob=do-sX7 zrA4EEyvXT(@mDg9|E-y}Yj{%ON%@|Of2i?dk8g& z^u&8_(Uti6YDYhD-gggRwhc$kS~_d)YY!dRoi{xU zO>wg9H(j%KmutEGSgTW?y)mSJ&qkhgR(tcL8>epwe*J1BknQUU+&wl$bA; zJRw=j8#<@Ped2ar;~xMudXTrR z+eILe^fOV$vaLS9{jI93dB49&H=Q#){DaDvb7OSm?K4IA9gtkN9N+tugZ}41X~FbI ze0=(1W(?d++}uLYiHk4Amv{A|RW2XM62vr=D<~b1RX4z16O#P*ZHH6lR$~1plyxaV zpe1b(i0N87|GxzW+2S&}RSACKmbqc9|2^o}-+cch_CJ;V{}veJj^jiqQPDDkK;Dib zZtgygwr;oz1U)VU!9p0qipyeUz Date: Sat, 8 Mar 2025 23:11:23 +0000 Subject: [PATCH 263/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10055 -> 10053 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ed15972d0..783023a96 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 23:07:52,895 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 23:07:52,895 submission_checker1.py:3270 INFO] --- -[2025-03-08 23:07:52,895 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 23:11:18,363 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 23:11:18,363 submission_checker1.py:3270 INFO] --- +[2025-03-08 23:11:18,363 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 23:07:52,895 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 23:11:18,363 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 23:07:52,895 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 23:11:18,363 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 23:07:52,895 submission_checker1.py:3286 INFO] --- -[2025-03-08 23:07:52,895 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 23:07:52,895 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 23:07:52,895 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 23:07:52,895 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 23:07:52,895 submission_checker1.py:3307 INFO] --- -[2025-03-08 23:07:52,895 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 23:07:53,852 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 23:11:18,363 submission_checker1.py:3286 INFO] --- +[2025-03-08 23:11:18,363 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 23:11:18,363 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 23:11:18,363 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 23:11:18,363 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 23:11:18,363 submission_checker1.py:3307 INFO] --- +[2025-03-08 23:11:18,363 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 23:11:19,338 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70822 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422411 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29277 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22432 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -35,7 +35,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3424 | acc: 77.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62118 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0168 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.429845 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a6ea319d3..2d0aae81a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -890 +891 diff --git a/summary.csv b/summary.csv index 39fde6c6c..64ea88dd8 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70822,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422411,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29277,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22432,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -13,7 +13,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3424,acc: 77.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62118,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0168,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.429845,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 342e9d52e..248da8ea0 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.29277, + "Result":6.22432, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.62118, + "Result":9.71972, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 81d9972ed397102df2a5fed73f607cc77a798be9..c106576cfbf7b1cd91ffc44826cbbc31c921bc7d 100644 GIT binary patch delta 2895 zcmV-V3$XOZPQ^~JbqEErz2N^&lX?hYM7R42{oIZESO>p8Oee;@AH_j9J2X|zG7W#$ z4+p_)bZFjOzu$9BBTjm=L2nYy{6q7bADh1({p;Xs82u5CeLpeM1d|^LFn^U{KPrgcpJ%9xeZ=!0}?1vE-J)0KR>IuepYewtm3vvu~li4SK5SlEhoXsyd z8xpkS1&0z_oY)hTG(^4ch(y%;8xj>1s#5H&MH1Ofk$A4e`cGWrUp1oc!hJWI8H8cv6@#0IZA5QDj zN&0h|u5{8@b3Yp11+$?arRz2SF7e~!E?#_@2649ZiSO=PHLIRPzQ0^#)w8u!{k(Is zj*t8}UBblm^;%|clliVcpYQeOi@k6*dx#dZBwbAHtyOpZa5fA^<}c>JPkO;5Hs2i7 zhh)0`R7FYYqJPy-hqnB0$GASP#-&zEj9OFcX6{$;7IhA%&Oz$Lwe}Vzju^%XN`N{d zk<<~1q>f0$75c#`ppJwKXip2m+ZgVN8qV_Jf{PA@H^qrOcO;S|MvV9ZCCUNqFX>Bh69PN zB$3338T$ey%92Yd_n$*z6{VCQ@z&j%6dO(LGC<=()qQnsw*$GMjMIwe+Utk$ z-lVt8@Zj1%dtF*Zc_k3M@^T1zHAi`lwjTfOSfDD4!?dV~tUj~~` zlVZl@!1w3r^%%we-My;0>Ad*ebXHPf2|TaY1Cx0TEPpWpQdy9ot>nhS8MJg*Cyu02 zMs-OPPMB0AXUtpyFA>Y!beVCft(y7)FsrDs%)GfCm|EX^uDOoA3rxQCbgXbUr)5hp zQRB2En5aQ8i3Y$V8UPbD2&OIq6B8hq1qs@lgHK?}mJaI#z?4y45(PI^iAGtD|N_vzDatlaPP6@NHEF%q%FZj4T4ED04C7@n5e;ZY@-NF zk^sOgNYLIKd;(LpbXX^@1XD(JNfd%fcD51B#{U$U>W5acRn%At%*Ga&&4y#=F0gGd z`JVoU(7?|Gl#2SZ>ZKw2kBrdyXRJ8|lv>8b=y*hz5}? zVbNh4K$2)2iDZ#Wha~_ci+nl?j5^TLA)Ubf2t;*B6hV@S;D%3?ejSPA-?2$jNr9!P z%-eUgu6a3c$2OE(?j7q1O7wo%(!N79u7C9J5Dh|E!okBd0431?ltm^UmH?nEvhgVC zvN?m64(SA;1h)=}!lf(o>%h{rV#Chg&*TfW@qAW!A5Z#)Qn}4tMjqwvpb~u`wzTpP zjVrx8M1xe8aPu$?P)RgEWs#wWB>*amEImp(ZqA^kLps47OK|RxDBQ7R-W}Mntbf?N zgH-bX{VSC3uOy4l#Zg7tP_iGC4V+Ion_mA)RLK`cu+dzc2WBpSf7$lSvc z0G37e9tHj$Xz7qnaAyQ8J|v2)V9S_%RK+fz=eYbidm}H^>!b2!pL7-LHr*X}V70}` z)Myv6rNxJ6Txszk8X%KtB$-Sj$v9*hNv1Bc_^5xu^fEHe6NF#14Chjapd0*LCe zC?KKCBafvdHA4!Q=xi9ny)ifJ%RG`Oq*}$ubeFWGlA$Jcng6%!pJ!bUU_^ z+Da{8^B$k{d$Z?&AGV%*hSSuKM*fLEnZ!mvOi$M;a?`qcu+^2?zg;~8>PepV7tRJj z_D8F|$?Az!NiZ83@t><_VcFTUu&gXue|WW+_(t+=?jM@|!#w?jvSLkR@Gx9G17OXQGYBYdH%jT})mU83Z)fo#eNbpGnX-~PW(Jll_G4^~1 ziuo)$bQCwnDHdYsw%9WM&9;DxJb%Cn0)G=MFnj=}b|Rt^Qt%0Be_S^L1jo>U0v$Xi z8NnBC0H^$H!>C+2l?;Q0S?gJx+B+ZAexytq&EGR+rlQ5aG} zC|P_RM&9&afa)~l-^4TEnEUnX;#|G59$=X{>$^W6KKbGz^M-mMR0VLZ<~vuI2TV{t^yM2Z}( zrXLo>V8ly#7iN>^C33~GD<5XXYHq~EIH;&8KdGJDYAxlF?wBZ0Wpz7#NgGZc#v~sH z{AV=V;*dJ!f@i18~(_pB7*h3W`k;3%a?NJ%6E33y+)(7}XOjxN9> z{q@)Wm}FzE1ApaaV0FW&)MQmfsq2Q-UT8$l!A9m`<(Mni+%9pJqRofBUg*m9vppkH z$rG~?V0~z_R#uJJu#J4+vuo{89O_aO42&Ym=iK<~`14)+8s%CDjLoglVUzhrWy)Kr zu}JHw$d+3=WEjw<58Z28%tq%v>gOP|Oc8A%KUIh|0U{bROcn*GwsP*HKYyy{4{f@K z$q7@CZmUwGJP+Ra#y&p&yuM#H-Mi!j5sN;q#`2^ZLtEvM1p#d7yeRe#blzW660GR3 zg+qaUix8BXVO#O2AHSZW&l!uyqjI&&P>|TSM$q{kK>c>SK+)7`Y}*P7qT752>J)Gl zhf4eoBj6}UD5#3&S8wcJ6DHbs@_b}ob$8|!6iaSL>OmoMb2!xE?~AZn1Q$}npigjK zj~b0$dM(j@6td|Xtwv%L^X}xmoI9AQ(9?Heg&sBlUYTAp?~T(FKZ&GGS+2Ye-wSIM zN&rU2KFOeaUR-U&B5I7K*XBIhsQQ@UBli%&p;&Bc?f9QJUCE?3AwOM(yC$g8Z{Iu_ zs}2Lxj{OLhA3vMz?tJK1LvAkwu=0+Ndu4SP4F@05BI-0To6pG;w_j(%)|@<_)U<8H zksaYmfFd)N$|m)Swp@UOj_j7nni&Yz0NBaKB|!afH+1Z3<9(;m?E9+zn0L0LH*(dO zD|lBXZBr42BGNK+>o}B|9+apC;c;7KE{jqFD-_(6XL;h>BJGD5jAK(!6Po2?2>p(R z>L;%LWHd;H>7Gz4xceywwc{Ua-S()18J3peMSOjzR?_wG4Zf`Ep`qagZFuzsPf7KAkeD)6jX zQ4}4(hVJr_prk>GLW<6q0+fy4<)veP^cfF2+h$90DysO`a#Bfoujwyt9XiS>Z62zw z&02tlf^{GSnZho9z-^hi4y41rkJ2<|`)eu>v}7m{fM)Q!EbH7RR3t~)W*osKT*oWRY*O6V5=2z1Ts0)-2%)=E18S(VQX4C@C|KOQ> zlwK)8O9>Apx@#z+#8rfQpjBG8Y%H|H{e5C!sR7oPoH&&<w|VD z7^WO!l2gx8=l;qXO{n{LrlYcTt^Hsoz>4U|s!=yCMhfGQ176V=HxxuS5k=VCOsI}{ z1kvEa7|aymXhz6Y^k;2H$@!1C_LMYvt}}O5Lwc2VmfM^?Bezhyj2E=>)+|cq1 zCiCgQBdk;vx{i-DEHBkqBAG2X7&P^S3_i_ zQ^rSsyW`vrLdN>8X(-vaEg>-Hdy;znla_B^S702UmtLDr1UEj4NKJsE~j&4 zE@%^B%}5+7JU=Nchdiqil*N5|UG<4DVY{C71bO5MKm3Y*!;qQP$^+!YEbd!l&>zwl zZVBLg)DR>mnJMR=I_f=GPuu_8Uy8r~hWS139h$u|;6Bwrgbog6scr30@YyAsV$Ims zv)WQ&y54I$lEU{6BXCUpa$k2Gg(XPtr>uOgDo~%?HqDz8Nnk~aQ(1_o`zi9_g?!`M zlM@FxrjnxfL?kkM(mzN2@d>cn!>L43cDhrDw0rzblPV!{HIsjfWrUG`(-`vzJwfM68ylVy;n3g>$2Nb3C+YO9()`l*KcmxSX)`y03nT;2fEvjzKdrczt=c= z*5G2~`~z-xlW|ZlQ%_rnvA@}JW5T&bfHX@{7Hzi=@>FCywmYc9>|!Q<*B$QVVAzj( z5X#h1@e+@qL8}JAYFj<#o!LIkEuA6<0TEsxQ;O6Mmf+f-fwPl(RVPxs^NfRj36eL! zah#0P8&2#nQm~{fFd_AbyNTgZf5p>V=J&&RsXaW6*pyK<>~Gn-^UxIYwl zG5XuwSIY>7SmLyX*H-*+<>4Av6u0`>CBi!Ii#xW2ZMm$}HzkOBJteraQ2wsf1Md6))$ z5Fd%aCqPn>lBENa9HrFgpNWw2Mfgd-VB8}AG?fknGNA{7ctHPKQBPe-Gg3&pR+6an LB_=QE*~xzZ%ZgY; diff --git a/summary_results.json b/summary_results.json index ca71185da..404366dc1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.62118, + "Performance_Result": 9.71972, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.29277, + "Performance_Result": 6.22432, "Performance_Units": "Samples/s" }, { From 3e31dd9bc48861cbed3bf67762ed54f18016e807 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 23:13:51 +0000 Subject: [PATCH 264/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10053 -> 10054 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 783023a96..82034e567 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 23:11:18,363 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 23:11:18,363 submission_checker1.py:3270 INFO] --- -[2025-03-08 23:11:18,363 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 23:13:46,612 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 23:13:46,612 submission_checker1.py:3270 INFO] --- +[2025-03-08 23:13:46,612 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 23:11:18,363 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 23:13:46,612 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 23:11:18,363 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 23:13:46,612 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 23:11:18,363 submission_checker1.py:3286 INFO] --- -[2025-03-08 23:11:18,363 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 23:11:18,363 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 23:11:18,363 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 23:11:18,363 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 23:11:18,363 submission_checker1.py:3307 INFO] --- -[2025-03-08 23:11:18,363 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 23:11:19,338 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 23:13:46,612 submission_checker1.py:3286 INFO] --- +[2025-03-08 23:13:46,612 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 23:13:46,612 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 23:13:46,612 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 23:13:46,612 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 23:13:46,612 submission_checker1.py:3307 INFO] --- +[2025-03-08 23:13:46,612 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 23:13:47,572 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.12825 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3424 | acc: 77.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1994 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2d0aae81a..8fee0ff38 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -891 +892 diff --git a/summary.csv b/summary.csv index 64ea88dd8..60ddc7779 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.12825,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3424,acc: 77.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1994,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 248da8ea0..d64fc628f 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3424, - "Accuracy":"acc: 77.000", + "Result":10.1994, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index c106576cfbf7b1cd91ffc44826cbbc31c921bc7d..19cf6c159f76a87c14fb932d714d1165de536337 100644 GIT binary patch delta 2040 zcmVxRcLhjme1ONbf5|e)j8I$k>7k>~CFq+bP>5Vd{I!`Dh4n&j0 z8QbaQ@5fG{Q?_@RYF;Xx7f777_AOrajt5_#v($Jbj3+c7noiR;4U&g6O7i8<{60Hr z_@=={kVipEb22nv2sgiu{uw+`_Jgko5r#C7^P#yCqUc)|4_71$c#{?+mrr8KvOvgN zwzPP`NPiGjq%5^u+xDz1NOH5b&}Sd9LE|_H$q^09jO3y=!blnj89rYn1vf{7D3Q-p zJ_aN4(ER4~#}B?~jRuwLe-rY=-&`9aSlpA8gn~q}0H&cbTTuE#Jv^5~wxU`Z_oRyI zmKhO=gEAFU`gBH;9+F6`lwo6)L=nj=w_(Wz z%^vDU&e}jtb-Xp#v7J|Tuj#aH&+Vw!^OBM^$Ev+n5gr917!4SGGFat-7lC?FPT#d< zl|nUor-oH*)_P#N88BoXs+f+EL>EN)N5pIwVk^ov9%WnOyC6y(;e#muh?wb!9v)=} zjq+PZuO)cphX@4JdyGIpvPTF6%-&-J0w&i90s*t@6oGi#kTV1V-u^iP0j0k{AiZtK zr4DgHbteb}RQC#jfa+c&5K!G41Olpii+?~sb?*?!JL2{53FqksKZn`fK#{1OgKbO&uvt1HO4MmS)uAP6wy zZhwazkIw$PJ)Ta^uP^tjHi64NVVQ_kxoCzov$9nFX0iv6R|R45Jjr9iWG6%Bf`4!^ z=j9?xxM~Xc{IS!tEx`z>+7C;08P?Tgyyi2)v| z)%?+IdQH2r-u>qx&EsTgela5=f+XeU!N7`<&F$_pa+3WVk)c(8@5D8&<8tZ^x!LOa z_M262OLSdabX{F^U>Y9kkQQ9!5t9K27k{D)MF$5ax&uvgkaMkbK-M>iRZ!mB?*CA9 zpOMoCM0ela5}oa|eXsY|c=nmSjQ-YjpodXt<8^kJXi~P@3UAZW8Q8wgyrbKIHqUX3ijW7z9P?m_7 z+PgX(Dbrw-(P+O4s?3w0CE4E#_fNe3Ug#ZsG28=VXvoYrP?=rjr8+J^!Ws>VU`eh5 zwoGzvq$HM!+D*AuvHEmphD1WbGITvM3}mhU$-%(VsoQP@_KAE`m<&T z9Sj9>pGN^}vxpR+0)HUSFWPhMEF-(2~PqjZI&@T{P1Fe{6eU z)pW!*@#lv3Cc^hs-a8p_%Y}Vh#>0=2_mcf~lQ)>eI(v$*4h#n-`QX{9&z{c|!iU*D zkjKv#-fMKnSAX777>}FNH)-!vgFJ%T-!TjvTkb7vBCd&WOMU7xx%;eTNZ9XVsAWcI zX?3rS8n4;gXsTRpG07oPrslsUqsbY%fODB(K(pI>)UyL6z`Zk9V}BMk+KT$XTC@|z zYoB1%%X1$_yFRSgW3)T8_IJN}XxZ8H1oZ;b36!Vn%WWt_Oyc-^Q<7`CuIufMGQ@Oz zTh2Nca^57j_PNs|b@Mc!eH?Ewjd_VBIb~otyzAM+BPe#vXBF8YFIG*ni z&vv@L`r6(OI8p`bfO9T)?6=HBPl3uMfukh;$y>c<-eG1Y$r|vS#>$Och1}y zR4(_jjefy8p;d*lBt;P@rnyi}h2A%-c!474sp7QN3M$ltL(BT6WQH(p;nivb)(db^ z04Yb9snE`QLojx12a>uZI8+d~+Dhs}X&ZlRDEp=j;5<$5u>?;!^%M`l*vv$9Kyo%C z&5vzI0B0Clkf6N7S(4$C*MpVrYQw0UIi>KufveV|IJ@^w%*L+klCBu7A%BKH*Dr75 zs*YpL_=aN?~1{@4fo%G+U)sSkJ*0+1>0QTZsIo;M{c zROe}a7pFy>E;e~VvVz<`d<`Qn`cFW481ir85isSx`~>wMv$rCI0tKK#?%HvaKqXrN zW|N#HMFHxQ;w2&+a-T;5YXkrQ1QY-O6aWAK0000000000fPr)rlaUk_lQt%d1?sZ} W7bTO&CR72qlN=`_25KY#0000iHQ1v7 delta 2047 zcmVxP)u)$Aq1ONbf5|e%i8I$k>7k`i$?b)Ha29Iz&d zGqKal-;bR@2ex;aYF;Xx7f777_AOraj{9F<(!_Wo1*a?9+4U&Z{qS<0#exIH; zeAD0}$f6)&85x+bgqvT7|MZ_(@q;f35r#C7@qxJ%BJWui50@kjc$4KMlTTt+q=AsP z#lqrwL4Sg%BBhDt+O}t<0nNS@>W!PBKC?Z+qHY~Yd z>0|xKSsAFQj<@1Ew)4h5Y&vb*b35wwye4GDv1+eXgvWshhW&y)8%5=T=Ye`rPS3Sv zl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@OBZ+PY98!u4-*~H1~KS} z&VM#ysw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@tT@a;?@IjP+L`-$WAs%H1 zjq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svtkaGkA-u?vw0j0l0AcxzK zD;?s3>W&czsO~ia0oA=hAfURp2n1C34u64w>fR%ecf{-A6VB5OfZ8AFkQS)g2!Vin zj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=(>kitWR@abogK)rHK@ecZ z-Te+b9-jYqcQP4Y++6KfZ335l!h(urId6t6wbDfXrm_c+S2-!-8O>r+$WDgL1b^XT z#>;t1xoQgd?5WeVEm06swI7!1GOX+IXvHrGm%BN)7Ry->sY9jkQQ9!5t9K27k{z~MF$7Qx&uvgkaMGRK-Ra2RZ!l$?*CA9 zpOMoCMEB6$5}oa|eedv{t)*ZBArvy1SItMHyw}@5P7Wtp6x^iA-qHM?yX=UQY_udy^`+si) z_!;SaK!8(!Lx7Il_Szf$YAru&<6Al<#e$4Tl5itrvS2myZL~gbm^i&@{Vo0*u0H0o zh^mz>Na~XVL9>Oyf7VA7UT^g1+R9iMPs)TC;x(6_^yDR%??~lp8qrH!pGFvYL0Lh? zYwcZ~j+99-Oj)#F1yyGBXG!+=!caZ0w-jjlV7(nV`6=yO7eDWH2kw= z2ptR#I!t>*Y6JiP1QY-O6abSUCKj`%6oCSN$TXT#Xi9?yo`X^~C5R-aErunOqD4Tf zLj2PL%iB&`YYD&}?YC_GDyfxJ4FWQ9SiAmgk#NHi{&F z1Eq=SO0SKftv#>Q_8eB2^ej!0e%S9Ds$9d7(U9@|ZvW0q)zWp8qkAja?^jC%Q1F75 z92RSA`s(eXk^cW<+Y76vBesb@H@r6yzOVA$$%tDn?Bg;Xew4hI?6;e|!6eq%Q+#z` zI55cv&rW^ze5MdS%=Upie!lQtqdUHT@{YoI+?>8id!HKQ5!C*UVc^(uZ($R0O@v$O zQ#%%LD_O-QJ^~9Vh|r zow*wOv#8Nl)Cbn0ohV-W1gl=2`!L$|VZ|P!-J!L=`_)6s&ZZ}*7nn|^Hv$C5(18LSJSG{z7jFQk{A|OhTsf5tgN0e^S)AN^FDGN) z_gP<#)|j8*&-KgOxN0i(;Zz(@QwdJF0~U;HDtukU!*$Gubv*G#f_NOFKJ#{3*_?pgULpPt(!6>5sSxKHyk%@=J^QbH(Md>uyK^k0DLH00mJGvJu}_7gOJv$!IJ z0tFPX!B29NK_y!OXOo>JMFH!R<0T>+I!t>*Y6JiP1QY-O6aWAK0000000000fPr9Va6OX(Rvu002(%%(DOh diff --git a/summary_results.json b/summary_results.json index 404366dc1..e5ce3737b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 77.000", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.3424, + "Performance_Result": 10.1994, "Performance_Units": "Samples/s" }, { From 775dd1a93341def09095c89344e71dc27ee9e982 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 8 Mar 2025 23:32:48 +0000 Subject: [PATCH 265/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10054 -> 10056 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 82034e567..4db936e08 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 23:13:46,612 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 23:13:46,612 submission_checker1.py:3270 INFO] --- -[2025-03-08 23:13:46,612 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 23:32:43,725 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 23:32:43,725 submission_checker1.py:3270 INFO] --- +[2025-03-08 23:32:43,725 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 23:13:46,612 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 23:32:43,725 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 23:13:46,612 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 23:32:43,725 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 23:13:46,612 submission_checker1.py:3286 INFO] --- -[2025-03-08 23:13:46,612 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 23:13:46,612 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 23:13:46,612 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 23:13:46,612 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 23:13:46,612 submission_checker1.py:3307 INFO] --- -[2025-03-08 23:13:46,612 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 23:13:47,572 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 23:32:43,725 submission_checker1.py:3286 INFO] --- +[2025-03-08 23:32:43,725 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 23:32:43,725 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 23:32:43,725 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 23:32:43,725 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 23:32:43,725 submission_checker1.py:3307 INFO] --- +[2025-03-08 23:32:43,725 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 23:32:44,675 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,9 +36,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402086 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0168 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.429845 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407143 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.429845 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418495 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 8fee0ff38..daf966594 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -892 +893 diff --git a/summary.csv b/summary.csv index 60ddc7779..0e8fa3c66 100644 --- a/summary.csv +++ b/summary.csv @@ -14,9 +14,9 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402086,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0168,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.429845,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407143,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.429845,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418495,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index d64fc628f..338b0c1b7 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.402086, + "Result":0.407143, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -484,12 +484,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0168, + "Result":23.2819, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -514,7 +514,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 19cf6c159f76a87c14fb932d714d1165de536337..206e9553c203e75a657bd646dadfe094a7eea396 100644 GIT binary patch delta 3017 zcmV;)3pVt|PRLHMbO;4_y+_+nlX(awe_d1CNE(Ij_gAP?xk#!8TiurRK><^QOcIPS zgaA`}MLD+oQDa*kNhajqk7OGozvoye+02Ta%>}Vr=k4ydrH5{*4?6huVLCDHgD4({ zvqMwWEYk>Py|6!?4G+zm>-T$}X~ao)*6&WjS#W563u5!vqkkQI4WmEeQ4l0Xe_FvT zJ~T&3GT*P)$&4`%6+K^RTDN%}P!*5i2;bo7=eL%c)PhrUad%?S~x0nXAWK~HNOu9+h{dhE<$L7&Nf1Do6@-d=daA+Q@{gdCdZPt$tmTljT zgRgP!vyr9pB@F+_zMS?CO)Hbue?j|x*^SG{=m&%DVv<~iUq1%p;V4Omt(k@N!b!aR zX-voISZdRlb{`JShG~2q_mfdpu`wF=`$0C)rqNr(NjUv|bx5r%ScNKQ9jh$0iS}#C zwjABe&d0Y&u#Ty(T3z<=T{r0-9Yo<*BU-i)&%4=~QTtk_!_v!2KV`*Pf8Xlsf~&^@ zBh4clz5AoaLH$0fc&+#>L#}1PjV!p$71Y!IY&xZRr!@4z+U~;@D(t4{CXqPXl4~My zw8 zfeM=Pf0peUaV}7zEOE_UK6QL>^zq-9zg@LYJ7=$#i=jAO4GqW1Xz``i3#awz zB>g!}S32pZc@Pcm#$S|@f2QWU-h95-n=kgl+3X=&%#w65wYOH?^}^X;JT!kX`$5tjPh#`U zL481`>rYjrlrCC>v}?=%cJ%A>s$Xig#HiU?+qqxCThzInIv1%E*VS;4X@^V zx!<>2Bzl}g4@ndk&|8$aa>0cXAdyHUi9{kvBoaxYS3n{Omn0U1w=f(?^a@DCa3Il> zB$600V_%>|e_3)V<^6L=tRj^XB;I;klVZcxuDxR^>T?u*L{UKW7A3B@$WBlKC=!W8 zkw_$pDDlLV`N1zh5yO4a&(nhNmM=g#lLc+kfLw~om=4A$OSF8!G@{~5zgpa;-Ch{) zO}fh(9z5q~uS=^)uLOcuehxvu<|^OS*5kh&3shxse;AkLf<^(!TcUBrHGD!ez$4R0 zAelxINi@C~q7NGlW0F-Mp)9JPwK=%K5=z>v5iy6KLsW-E$%l>?M58QG^<}RAFN4je zNipNH9|ZICdW_=W?q1crbYA>!IxESr1fJLHj;;9gYD#;0$6{6S$L_}pAG0@`l1kLL zwxp6gf362Ai3X@78lVz2Fi~zaluE1ssVu0VrR2uK8I-hHBd(-UMs-LOPMB0AXUtpz zFA>Y!beVCft(y7)FssP1%)Ghom|8!3u6eGr3rxQCbgXbUr|C#AQR6lxn5aQ8i3Y$V z8UPbD2&OIq6DvS43o2-B4nBb?OWLdv08>VFe@GNOR3#c^xspr$f52rGIhJ|Oy6)KK zp0e3!I=j}fe3SH8;of7@m0*(nNk@W-8U&MQ08F9*Fj0f+*hUeUqyhl5pn}%s;1ig# zq|F-fB$zU)L!uB&va^j~HvXr;R6n$mts=)#U^ceEv>UFIyTG==i0GMQZAHcMWz$6s_m<1KIHV2=;lqGG_2s~$ns1Az)Fi}Imv|lus z;;HNRGg(D?rI@t0nADEWJ+RqEa)Voi5g!?B%4J@ zl18K_ps0e@=HL^OvZT#==}RPKREI>7f4fU2@+6bZYMESoNvFca8cG3Vaz9DVeM_PG^#*{`M4JFrB*h$C%1MB_16UFbU|D4DVHE(DMfM&A{vIf4lSXi71S~!zimYJEn0!>lE}!SP{5g9gFV*X# z@@AiO73wq+$rmYFkk-shtmHSMWgWa=G%WTI!pkv<=y@ubg(XpqbjJ|Ct5GKqr*$Sm^t zuttE)BA?IZ;1ikP^I^RNGQsCVqHxYE{-=h5O7bfOr7-zeuCjOCV%j^PM8k+9O+G~9Ns|xJAe1FcK1>5p5|0i{mqjKY z)(AjZWb)Bq@_~{zX+&8-CAfTO7_4NO2v)Kc+kBqGG8tq?wox>EbMt9L*>$$!}tJbsE za4ja?qiNWGz3kLtHvV%FynfYt!L#z6(lkOoUe_L!Q z`=%}6JWU_41W!5j6d!=GnThC#B*SN~2P@sphEch2O5q0s zH?1decJG~-jeXxIeKA@?{tSO^Uf#x4W1$YGVu0$BGr}xTuWeo8%OV`Eqt0)_iMImy z;}G;4Z^uQZKHyOZK&H4S<)e6dCT`ZK&eQxpPK!8QHhDpcoGcb!hmlwPXP`U|`8V+d zm~vl#g8C1$xFX5|1$ezj+fb8-C0haPlj9{k0d|uhCP57o82|uyY%g?~cPyzq|5dZ)H0000000000fPrR{-X=Z)x|1Cz LBL-|F00000GP0tL delta 3015 zcmV;&3pn)1PR35KbO;5qz2N^&lX(awe_wOjNEXH4_fx1;c}S|ppl(2aT(+kiXOb9W z>^Qck_Q^#+fNB6Sl5m`FKN4Up?m5UxHnZ~1=D~s1xwrcY{oIZESO>p8Oee;@AH_j9 zJ2X|zG7W#$4+p_)bZFjOzu$9BBTjm=L2nYy{6q7bADh1({p;Xs82u5CeLpeMe+0Am z&>Sbpe7|0g`(uCFi)-QBpQT5JVKnU}>Dy>jkLQu!8!UsSle)GnyFTp&Gjo+;KPr#gANCfLk^o5r;FaA-D6<7+TT##v%x91I42cA`zAzlf7?`upmTS|?bADtjHPEVYUDYN~13 zx|Mw%-zNS#roQTRIl_0nq<3@>gcpJ%9xeZ=!0}?1vE-J)0KR>IuepYewtm3vvu~li4SK5SlEhoXsyd z8xpkS1&0z_oY)hTG(^4ch(y%;8p7(Z$BP;oS@ER zLkBzXL<#3Iri(Fh#<@U=vcxra`84q1(Z_#Z{&v+l?Vi0}E{5WCH8cv6@#0IZA5QDj zN&0h|u5{8@b3Yp11+$?arRz2SF7e~!E?#_@2649ZiSO=PHLIRPzQ0^#)w8u!{k(Is zj*t8}UBblm^;%|cf0OyHKcDaQ=Zn2?HhYK`vm{+i?X6XJ{ctu6M&>W(z)yO?BsSk1 z)Q4oc{!~Rt>7vz7hqnB0$GASP#-&zEj9OFcX6{$;7IhA%&Oz$Lwe}Vzju^%XN`N{d zk<<~1q>f0$75c#`ppJwKXip2m+ZgVN8qV_Jf{PA@H^qrOe|IF3Bu0$*0wu~4XaCZv z5XsT_uGi1L&+MgNUskF1Mk$cNqFX>Bh69PN zB$3338T$eyf69_eDfgd4Vil#7Ao14SniLyN?KF2RMLmw9hbRh&-lD`27ug9)07W8^ zC=!W85hbp;GCz0)C}OxL#(7#0-u472XR@I~I*?0I8PmlWWr?0Im_{VN^s2>e+Utk$ z-lVt8@Zj1%dtF*Zc_k3M@^T1zHAi`lwjTfOSfDD4f5W(J7c>e;-V%)?uHh4+0Unt~ z0?9OzNTTt?Df+O{FeX_731v}&_U7OQOK9n^PQ)C34pChaC2u-j5RI}#>dV~tUj~~` zlVZl@!1w3r^%%we-My;0>Ad*ebXHPf2|TaY16%Rw)|7Vjj>W3tx808w-ezyLB$cRf znvzO#f4d&2BpRTSXn;!8z(l#xP%1G2Qdy9ot>nhS8MJg*Cyu02Ms-OPPMB0AXUtpy zFA>Y!beVCft(y7)FsrDs%)GfCm|EX^uDOoA3rxQCbgXbUr)5hpQRB2En5aQ8i3Y$V z8UPbD2&OIq6B8hq1qs@lgHK?}mJaI#z?4y4e-Z^ZRf$Gfj^tARA8=Vkjb-k$t_QaH zPT6X->|N_vzDatlaPP6@NHEF%q%FZj4T4ED04C7@n5e;ZY@-NFk^sOgNYLIKd;(Lp zbXX^@1XD(JNfd%fcD51B#{U$U>W5acRn%At%*Ga&&4y#=F0gGd`JVoM!xsu6NwM;I)q*LKy4W$4wxt}g)%WRrd zUQPc1l2ue!>U(7?|Gl#2SZ>ZKw2kBrdyXRJ8|lv>8b=y*hz5}?VbNh4K$2)2iDZ#W zha~_ci+nl?j5^TLA)Ubf2t;*B6hV@S;D%3?ejSPA-?2$jNr9!P%-eUgu6a3cf5$eI zTkaj}2}<;S*wVg3G_LgT5Dh|E!okBd0431?ltm^UmH?nEvhgVCvN?m64(SA;1h)=} z!lf(o>%h{rV#Chg&*TfW@qAW!A5Z#)Qn}4tMjqwvpb~u`wzTpPjVrx8M1xe8aPu$? zP)RgEWs#wWB>*amEImp(ZqA^k0z*2J5fC4LUDLYR*tLT7c3_Eq5nI}Nh{lz^9-=`k zOE`O&2CyU=z_Q5P!x8|NMfM&A{vK%QkWO%C1S~!zimYJEn0!>lE}!SP{5g9gFV*X# z@@AiO73((L9d}^0#mUrY7qO+qhiF`B@gW)@lW8QGOe4u;8cC)uviPtBlFXt6Z4DNG zA86^2PJm2^>XIl}#u5!KV|B$ApZ^M&l@wWKDXa4?pKNc$_F6ms2vDmXWTIolmNp-v zaiz_NXpqbjHXo(|GKmJrEVB8q1VCny%}0aH2UyG>zEO z=tDHFH2M$?l3Bv&!!$r9(Eyo6Mjw^{$SgAYXfXOfONVp*=;n_mq5#yB+6(jTR1b<19|7i?HRM+fuX$UpBz zqhJ;r6MvW%YSkRmh*r6lAjs4!hkFs~^@4I0bCXa8jQ?hwJ z+0kFK>=K9$1nRQ|7n67)EKK?c!gio7Sv62-N=tV_nmD4?14*spzprd3P6AnVIl6bw z+!<6Z_p*(C!8)N;g|Z|?5h$j)P)&v2H>-GoBIl{%wABhK)PqCI`lgfcAtiqsY$*Gt z4d6UY@390=IrS6|z}U=0bU<=8Bh8O(M*wFST9BZ;!&#Eylh=cl?rOuRoH?cNy@9LN zqd2?wPRz!x>yoY*ts#GgKi4mBJgcy5x*73)E{{m-xI0hij?x>u}<&0RGqq z{mR>6k*N=O6atVb?os(Do}MN*D^%xceix@joGvzbL9&9}K70)$FZxeFc^L9<;t??A zzWfCBAG5k5$^r$lz2N^&lZho;0qT?GB|HIilOiTT4G|dt0C;RKb98xZWpk6UCKQu| zCK>{R8 Date: Sat, 8 Mar 2025 23:35:41 +0000 Subject: [PATCH 266/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10056 -> 10053 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4db936e08..9cad127bb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 23:32:43,725 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 23:32:43,725 submission_checker1.py:3270 INFO] --- -[2025-03-08 23:32:43,725 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-08 23:35:36,817 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-08 23:35:36,817 submission_checker1.py:3270 INFO] --- +[2025-03-08 23:35:36,817 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 23:32:43,725 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-08 23:35:36,817 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 23:32:43,725 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-08 23:35:36,817 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 23:32:43,725 submission_checker1.py:3286 INFO] --- -[2025-03-08 23:32:43,725 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 23:32:43,725 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 23:32:43,725 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 23:32:43,725 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 23:32:43,725 submission_checker1.py:3307 INFO] --- -[2025-03-08 23:32:43,725 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 23:32:44,675 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-08 23:35:36,817 submission_checker1.py:3286 INFO] --- +[2025-03-08 23:35:36,817 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-08 23:35:36,817 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-08 23:35:36,817 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-08 23:35:36,818 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-08 23:35:36,818 submission_checker1.py:3307 INFO] --- +[2025-03-08 23:35:36,818 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-08 23:35:37,791 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.26254 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70822 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73114 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422411 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22432 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.12825 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1994 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1884 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index daf966594..7c890ad89 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -893 +894 diff --git a/summary.csv b/summary.csv index 0e8fa3c66..044807e0d 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.26254,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70822,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73114,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422411,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22432,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.12825,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1994,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1884,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 338b0c1b7..e89521c8c 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70822, + "Result":1.73114, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1994, - "Accuracy":"acc: 75.600", + "Result":10.1884, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 206e9553c203e75a657bd646dadfe094a7eea396..473be543d7f0f190a1b25d325069c84ad68ff47f 100644 GIT binary patch delta 2162 zcmV-&2#xp1PQ^~J%>x8?UZ2d9(E}!bTW_N{6n@`dflwb>X%a3;2#u1Ars=)uEzNGV zPZ0qVtO3km)62gf8mrIu^ko|OeOHyaCs@=t8AIHn;vW?_|)Tr@^Xk_JMC&l6g3 zb3BZwe5Ud-N)nIEZ_Z%y=$qDfSiAm*l4t(z+7Q9=futl9B$5R%4VBrFvG3~Pg&eXK z)zY}9byRny5s^5kQZZxC=Y+0*5+RNB6eVOSuYZk<%F?=~kp(X!vm-swNF>Uzkh-*)WX-V}uXTjSfe6OKl06$`?SU78 zdQr~6wPlq;HF~dxRczLJV73`BWFD%RPvg$edQyhhhzCfSzZOu6F7DFRJlNYF zW;&t^V$c!2ZNywhcyEZUC_8wR9gXjRD0M_1MEP69d>3LX$}S#dSL3@NN*&>YD1VEX z>xe@<${rf!x1L^0@XC)62&nf2fq-O>5eS&QCkO;gu2Td8X4e@4@wOr72n4+S3j_j6 ze~Ca2w;@+L#0AxzA`no2-D?B_s(XV#Ky_~s2&nEI0s+;%M<5@F*TW~AryBsZKhhx` zP_+pH0r?&y5RmT)0s;A+A`mc}&NdKaAv#CsU>aTQpkHpDZPxQEI55>6bV04IA?F6+ zfVqMoz>K^56?Qy6|M%`>HodsH+OOLLF8hQP6-l*hhb*(QRQ_gvvIme?1u5eN&0|u^ zPKGQ5;bOt7Wk$Jb3i#rw*S0ND5>mGxmg+LB>*-|8FA0~sIk#5HA_!Gm(UM(6W2zMv zRjYUvtPGRIliT*%c5A)+FG7~bbY*@qBO(Hta`Rwl#mMG%{~0;SevZh{>c0=-nl*7b z&4%3W^nLr?s<$P7x;`$tzAid24Ucq42d?r61d0xUK+z!(C^`fJMc3aUI)n~Iw}XD! zhoXZ6Q{90kI>@=vIUws>#40H7UH^Y5y3fez4@CFS-x8hebbasegRRcTRp;xf0}aBF z4(Wh}c7i~l>JSK29Rh)>Lm*&hKJ#~|4xxi=FLux`eW*HrI55>6bYZ6iIX5~7EF`yx zRoE8!pR2lRS>>W?$q#8|;>Gtq7GU?k2=Ft~`vU>a{dWR%?6%|kT_8W}-M4f`$`zTA zH04IfWWgHd+h}v%Fmnd;=3D$1Tz$;v5mhT&kTxd=g61oO|7eaVyuswrwUx0jo>eI^ z#A_ix>B&ofA>Wb8)ik1)xH*k5ijuODir2=wIvuIfV4SgNzYeO(>5q!+?}dBbU@!Cz zz8LO-F*0Q48>q~#_EH}gAYqM%MX(~*LAjziH&PPIMD4a*tIFnlqrMibsIO$RzGOnM zY;%{8AR?u@lV7(nV`6ioO7eDWH2w)tO9KQH0000B2mk;8KL8D}T~SW6c?iP>1RF~2 zLX$!d@&o7{ShF_}O#}sP+Fna)vxyX;0)J03+KQ&`YYD&}?YC_GDyfxJ4FWQ9SiAmgk#NHi{$zrGJU( zO0SKftv#>Q_8eB2^ej!0e%S9Ds$9d7(U9@|ZvW0q)zWp8qkAja?^jC%Q1F7592RSA z`s(eXk^cW<+Y76vBesb@H@r6yzOVA$$%tDn?Bg;Xew4hI?6;e|!6eq%Q+#z`I55cv z&rW^ze5MdS%=Upie!lQtqdUIxj(@^<+?>8id!HKQ5!C*UVc^(uZ($R0O@v$OQ#%%LD_O-QJ^~9Vh|row*wO zv#8Nl)Cbn0ohV-W1gl=2`!L$|VZ|P!-J!L=`_)6s&ZZ}*7nn|NWe`EY8K?KTt~p z1QY-O00;m806zdBon9SB0+UZB3bQd8LkI-j&XO;acp)r*NE?LhKwGkEpwN_-?u0Z+ zM6CysS|@*B*|D7jyy|jv@0__as9YUo6a9j9LaPd8Ns1y+Omm_36?)&U<0Xomr;5`? zE2z*399q?PB{PI+3$IoiuwHt7@LWR1|(+_()`$_2XKa=0SU@GoFy5)cs*F@W*d6t$|;3E8JM+> z;^aO$F&W#oP1>TjhWr`++`PQ?tHwfgr(%HWk~6|AP_J!W;_D(DuA$Cv!ijeR_~R+) zGjGEp=RV+32tdwpN9Chx9WLhjm=(E}!b+iv1I6n#Hm5k-AyrGZ?45D+k$(tGKRGN(FE zC?pOE-XoPM}k^cbRHlDxDWdoVE5XUiOX$U!SwocqEJ`G#{Ey(>4u~hcrs^ z<I0?xS4aPODnKuvYLHP^A7S9Y)Iv~ADrsMqt7k~PPwy;cz(1tJ&?7=1EW<$)K0 zdQnc_wPlq;HF~FpRczLJV7eJFWFD%RkE7PWdQ^tjhYDF29< z>4+X4We1J&TSu=Yc;$x(1k`(sKtQrb2n5XDV*~;w*9igvv+ERrc-xRO1OndvIRXKt zzd#_pZOEk#aY1z_2n1Ar_X>f4>RuxdP~95@0;+q9KtOfx5Xd{?_3#Pj=>|aU4|PZj zRBenvK)y!^1mt^+KtR4H2n5Wg(+vb!h|Ul?m`3M2=og!3oAvw>4oq|hZBVN#$hk&1 zV6GqtFyn52haHd3{<}S%PR_3{_p3I6%RXV5h*i00hBULXRQ_gvvIme?1!3_#$z#G~ zCqw3fa53lQB1^bx3i$l7)3hzY2&vi+OLZC6)nvTpGs5L=&aLHY9)zl`Xvi+2Hq{7= zvQfMU%EHzB(QSH7yRqK==ONAGWNCgeBO-z%<>tY_ijmFj?lW?d{Tz{@Re$fqHLc@v z>J7Qs>iYJZRc}jwbX{C@U0rlw8XoG97F^{K2oxOxfuchoP;>|cimtmubO;@aZU_CM z3q=P9Cb|PnbdYncb3oQNh*ePD+wT8Rbf1ya2Sj(@-4dPcw0*Dl&Q|B+s`GW#fd=7F zhqS;#J4PT-bqEBi4uL?`ArP=LpZYsghtNT`=R4>ZK2#ll9GK`1+OSiCoNJu}7Lpsp zDr}4V&sANyD05LZ zO}PpE*|E!NFy#DyXwUx0jo|Y*w z#7iMR>B)0{A>Wb8)ik2#xIT?A3YJioh?m;CIvpv~V3g5lzY40%lb1}!h= z6xv3UKo9Z*>{`LIH4se%1#+K90c*2|6rlot&oNzFF_pk`P^zW`k>s?+u!K^y2xwJ^ ze_CL98_M!Aq9l&1`qnfyO3T^lzo)?+4X%{c4)-*;ufdfqZ*i>TX@_h28rJkRtn{@V zu89|G;>DVHu_j)ui5F|)C7O7NCSIb%3(*Ri4QH1FQ-eH$+TSq@99!-!Y$C3Sa7%sa zGP(P#Wk}fXW2j|DXlZq?jT*1n+i0p>ZZXLrQl{p=CZow2x`1<;U_i6md(^W7CBVHi zS7Uz`HQI{$z*@8u#cQ8n)ys1qM!P<&*kiOiwDxzudT80%^aS++(+QNP>&qx?LrmiM zdsC8Yx~}W(jWWb^d|S>s7vt;2<@j16uJYg=UQVWq`S4~|&o{1|s(e52x^_3{`o1gI zo=-;4W+KNhpE#cH5YKkHzWUnU4meT;>VR`DckH*!L{EXrC4r+P{>fXtX5X8|xfuKh zvk)0!2n6@X&NGvBAuNB=5`^tQTe51PP?whOgfwwPtp}1?$A4eNPMid?>T-1NoVhco zTB*SN~2P@sphEch2O5q0sH?1decJG~-jeXxIeKA@?{tSO^Uf#x4 zW1$YGVu0$BGr}xTuWeo8%OV`Eqt0)_iMImy;}G;4Z^uQZKHyOZK&H4S<)e6dZq}&I z)BHY8i#T02c|jG5oGcb!hmlwPXP`U|`8V+dm~vl#v$P_I0tBE!?%I<-B|-&wy+_+n zlZGW*0qm3FB_bPgpGN^}1ONa86aWAe00000000000001hfp(K0CP4uclVK)00fm#C yCMFyA$Ide)0RR9{0ssIJ00000000000001hfo7B4CO!eWlN=`_25ck%0000OL+i5u diff --git a/summary_results.json b/summary_results.json index fbf9c089f..03e784b19 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1994, + "Performance_Result": 10.1884, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70822, + "Performance_Result": 1.73114, "Performance_Units": "Samples/s" }, { From 2c7ffdf9bd998dde421918c6ebf62134247ef938 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 00:40:50 +0000 Subject: [PATCH 267/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10053 -> 10054 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 9cad127bb..5a07136d5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-08 23:35:36,817 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-08 23:35:36,817 submission_checker1.py:3270 INFO] --- -[2025-03-08 23:35:36,817 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 00:40:45,050 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 00:40:45,050 submission_checker1.py:3270 INFO] --- +[2025-03-09 00:40:45,050 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-08 23:35:36,817 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 00:40:45,050 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-08 23:35:36,817 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 00:40:45,050 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-08 23:35:36,817 submission_checker1.py:3286 INFO] --- -[2025-03-08 23:35:36,817 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-08 23:35:36,817 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-08 23:35:36,817 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-08 23:35:36,818 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-08 23:35:36,818 submission_checker1.py:3307 INFO] --- -[2025-03-08 23:35:36,818 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-08 23:35:37,791 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 00:40:45,050 submission_checker1.py:3286 INFO] --- +[2025-03-09 00:40:45,050 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 00:40:45,050 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 00:40:45,050 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 00:40:45,050 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 00:40:45,050 submission_checker1.py:3307 INFO] --- +[2025-03-09 00:40:45,050 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 00:40:46,212 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,9 +30,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.12825 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1884 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2268 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7c890ad89..208931296 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -894 +895 diff --git a/summary.csv b/summary.csv index 044807e0d..83fd70747 100644 --- a/summary.csv +++ b/summary.csv @@ -8,9 +8,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.12825,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1884,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2268,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e89521c8c..8da8029dc 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.12825, + "Result":6.60193, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1884, - "Accuracy":"acc: 75.800", + "Result":10.2268, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 473be543d7f0f190a1b25d325069c84ad68ff47f..e7e23c4ecb392b0a00285f2bb8a690e363df2c8f 100644 GIT binary patch delta 4093 zcmYjU2{aVm7oK54M7CtjSZ2u9(4-KJ-B_|@j~H8aDiac8`PsA0m|-YsLS!9eCrg$g z%aDYmWZ(B){`xQH{NFk6oO|v$@4oN*-goY~ul;Vv-LhU~8mMOF*yXJ`%Fz^>v<_N@3(EO~A5S0sNq=kS-BcW6_8$a%MEi-L6W z+>?wV2cd**PKf>0QVf#NHK<5OIZjKVrd+`qed>nA;$S&}EAsg@_M?tlT@UzN^Q-D- z#|S4KDxTxEJ?LkPI}Ldlzg}6sQuIHaP3xU)SY=%KZZnfzqj=Cy%W~Z?-bRZy3Hc|= zu<;A5#Au1xp_=RP*Qvj5OaV=1*XZ;YR(f+I>iysszg9&P6X(OZ{WN^|WJlOm`5Yv> zS*|#1{~ibH3Q*~d|8U7pMt+MckJla@-e-w1(GCE2PYTZ}PO&Co%A>Uoir)c^V*Yi8Cy86di^5?$xD01N1qi2@4&x6HadP>cX=;LZII(PF#WC;AHzX{1KE;c@Td6Ak`fSe*B$b54 zZ?rQMtm9Jo=+&>Q$yXoscOiG5m`0hO2D6S7#rAm1Pd|OPX0>MCC{5jYMBPDC!`uRL z)D?a4+F&C;22dPJ7=kLqDqi=8g(~DHUMH~Kqrxe`pT5Wfr>gcP^Z~6Vh0Rq1*A7qB zG1OqKyBKg?^dG{!RQ*zIw4f*YN{OPOg@7w-;+b|*l^h@%vuH&2A}fQb01Z%Si&(9W zW)N@+B1eQQs1CP;6|816DM7kwsS5T&wVFk(Rb59m!)84am(BD-bR;>hWgU?L^{1Ml zuTy>qGWf1Ga%6jb5a`?qJ^t)JImm;Nw(wIMO2vgz!f$-8rp`8?TrSS`R_|Y{IZW?w z(GP!p%U|Aihf?fG9bURH;|P0p=1?RY1Aw}->f3^# z=ArgnrUPUA&uQ)vDy>8Sc`p`|#RKwn;oXlRX@{qwJ!d7iUZN++y~@rNRKKUijt9^M za%*E}o@9&Y5iSwLaf1O$1*B`x+LP!;$O*zQbc_H7BQGJGV{h166S0gts#`| z^pLN`^Vl$~AjB1NAETL=+wAEX=62KO;hYI!Z9!1-yM)g(cKt6kCI43vyAli-lA*zw z0{#yrQe^$*C6b~PW3vV`a43XoANQ_gvft^_0^VWphl7cd;$E{0A4Oo6BhouO?^=pR z!V*%{I6{Y=8H|L7in>3Cq&ys4oo=lBQtq{szq)F!$`HG7`tpU& zk2fH%7%zK>pG&5k_$l0X8Oqu8IHXT560SGd!}lHsJNJ5te_)~r?DA`VzB%|>9bN*T zXJr<)Y9+OuT|=#5rwFI96atc^HW$QUiwI;UoaRV`by&z4goK7;=vB-P(V|#PClm=e zPaw(pOj7G+=NbgYjl669i^RmKN56EASf~MmAJyRu&gbbb4gpd%XpOjt0hZ?g&M67n z$h2Ob)JmiX1EdD6!zp7+w4*1KT;hBBCj@GjpDKA4f#hZDU#L-tz{ z7L$XmYAL(?f;sCMTjje39`$eh?HAUX*-wNMD|c8E#DbMIA}k5MyD(<^Gl88`YNO2n zmRzGjoclr9IR?Bfuh`1=lf}!!r5!hoFV#QK`n@@~j7exPElujq3?4KqdKjdX)$AUR zE1=3u>BL_=<2$HKoq$3wE;p$el|8F6k)Xpr`B1drb+$*QxPPo%IWPtPp9h=W#I)^bj`K#0_EbLtDHH?DZD>=hrJ8*`eJOW zLdA=Iv5`{@EdAQ>{M+gTm8xCkAWY|P!HI9G&D*Y)3uX!0ID5adKt<#RR-9OOrZfl|J z%p!^KzjO`G=o%4VXLJ$B+lnw-F5+`bku%p&v?D0=x??$P&KMy6=jIz!8P-|Yl49<; z70A-#l+0ETGXKaEQj524o>f*^2(DNQ7p%$(iXMW;M71Dc#7|n+^rJm4)n2u2{I;mO zGFkEixFc#f&h+r)(XMvk1Z6S7j1nLZpSokQ*7~^IA7y3b+}JYgz|Kvn4)k@UySTM| z!I_ewb&4u`hG8PeCUv0Tcbkf>`yaG@NcT(1L!IuF--{pJ@7G*EG+V=0oW7rHIA|a6 zd7&<1aAKU{Qtb&jYOX{3!PZzJw!Go|{SJ=o7X~$oGdy^yi;dPj{m5H(A?_I+X)t8# z!tZM1ffah=UFnnHn+$E!8+!7hnR_^alXL%R$QA4CSm3$JZY0hF7tIO7>}Lq;=Z5Cw zQzFBiB#$)C*9-k1OP4~fPp?_9h_+dBA?2rIrz(fFCl-fH_L{uPIg8&+pYJhL`c|pL zuo=^iY*JkLK9R+zO8-c8D?LvtdZk4%jR^nD0R3}# zw4`C{dou@HTNX4?RFpp}2=83)PD9b6w&aw5R zQMwf^TRNdiIWZA7!)m&9awZan2A^++y*rzC$K=UWuZg_%)}doHj{vJNc!Owed$vY) zzIjwu2r)!+uX*!u@0|~;nBN-Sp@Ek~on8ZO2A9U)?&QXlu_r%3jDN{qiRCX-e2;$i z#9y{mtg9{0hvUTn2jZ^L$W8VBi~h~IAhn7ml;E53%(b^mqtE3s`W`;C!1Wk6j(x3P zbr=!~+w1w@K?i&k5(o`jIQ-_e^TO6)jG;Jy>o~b!GS0NM%C>FP;okBa>CEH~j%g(q zd&xZZ02bY0Oez@rvWlC_D2@wQ%(!95B;})SrK0sxD$Oj|gO@LNW_f$f)z>0I()mMM zF!L2xU+13t{E~rVzcQMlZUi$+275(&OtohOkf9HXVlV$YjDaT z6VZDsEteEGRggr!J(*f=EXpHppl|Wa>w?!xU@oR=F2D5?8LoMV`skDYm|0s(q+_rC zMCrPj-iTS^YO&KtJ84-&3zPNZ^incUes#Ro;c1fBwW5r~Z_q#A*MDg2H+2VziFntR zCG`aC8*e@+pY-0O!&yEw+!zk~H6SW@TWM!>i>S5_8>ipgKWGg~Sl#bdV*LH+$su$l z*gYAeZFHm*73SE}7-=BoA^hFsb)EjIiU$w-?h&^;kFwxq(1E*W#`;gl$ZFICczt?oYh!&>f0~<%b>yPW)VTLeBfIf&p!R!&fN$SS zVg!LWf7Pf~)DQpZPGI54a!|#s%O|IR3RWHz14z;|@&yhpB#5!oYlCsNPo8i>*1QIL zV+mEXcx46FBJrlk=@KS+j1I~6W20L=nr|D~rT4h+Wa7y+$_V|a#z*^C1Md-TYDS^*mm$+umB?{9fkhSQ*b4%9VJh?MlIb@vazrE@jnp{YITR|kFs;ZV!pu85+M zzWLzQO8l4TLQFCF&2r|~(98dPN1e}lym)mrj7*G~kD2yV%McvD0O*6k!K#7LL!#{u zOX(#iJX^ADZso4O3bR^5zWh!v26Y^-Le}#a31Q;yRA2LH%O@M1yywoAo0D?n${(rv zyfFijDtXiA(-!AXv+sC$n^^rD>?p7<4(jTt5u0&5*oN468P$U0&GZSff%yy+?>4L8 z9qH1=eng<>rq9%Q*lg1u(2QV3weUYQ|M{TbfvAE0PS3bV2CxBs_7!<=vvP!(Vej$cfK&Rz6x{$FQZwR8Xg delta 4126 zcmV+(5aI8}PQ^~J%>x8?UZ2d9(E~0Sflwb>X%a3;2#u1Ars=)uEzNGVPZ0qVtdkf7 zAAgM}QgX`jk?FK;(;#`sBATy8=HK(vmTwwd1bGysEGHxLm2mUx_^;tJE5GxE5Mf9I zIUkvc5XHcfwbPvK7_RxTkegccl@LIH*!FW6$S=t`Z@Q^b{pzDX)KxjLOow zrjZ3NBeNqt&`2c8u#wOxB6;mLthiv=WAn&a8)&GGx8^#w^Ts}GJ6+pzd+PPPrhjD3 zu^O*+gvWsh#>0|58)fZ*7lC?F&cL;0l|nUouZC4@)_P#J88BoXs+dpX&d_>NhS!J( zNSVJIu%x#)moDzo)jZhS9%eeC3u4d_y=}x?M|f|DttdNqlpT%lfGBlDA4K_E#C#WG zE6Oe&Wmn_7AW9wKgD8KCnCpl`Jb%g_8s)d1UQ6)Gj}Qo`_XL4}WRDRDn7t`P~B?;0;+q1KtOeG5eTU69RdN> zy+oTnQAwLj7!9Z#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+c zIyf-Z9du!*1UWZ42P`DFh*j7Y`Jb!0YFXu?YRM02W#Yy6J{Dm2zX)p3>Mt{l`nUFN)M#yBr8s^(*bKWp>2J_}y{1;q(%;ynRD_f8@CkKM& zD}(=NjwrmrHyE+}I(qNpi zXul4s%IS}a?C*tp-e52E4!#)ffiW^<<{PNYuJ%$N7a(Dche<`SBG*B=qB%EG63ayG zwp^>q=6s{R7ObeRWV60xLa=OemyjSLrMi<}w=rX4bE8V~c5F2M2~bM|1QY-O00;m8 z06zc?v0YKKcnAXpfAnffdwR!WRq@B}#|j^_H=B}5)VQ{!l02>lDv1WDBpRR+H84?b zG?YrL0I4jfprz!-!WopbStG8bQbu)16i%2_BxlTA0xuEE+;o|7sjZs&0Whn`vCO== z?wDFXd#-t|vkOeV^>nOoH>c@HFj3<+C77r|Fo_1hBpLt{e>DiEE&>xPKrjm`Xl)KY zfhkMctPucHMs-LOJX9qbWx0||{eQq^6*-o9&bsc{=AN?IXga&rv3!&CSmEAd)0JS7 z{YgiHi5dixXaG#20WeX6>)1vSn4|&#v!H_3=HL^UvZT!#@g$fsszag>OtQ0$U^f1z zz*Il9lC2`ge^OvJw!pL-u9LgKw!!3k`o{{l%A1}96E&_Y!9)#$Ni+Z^(Eyladmq5G zi@+om0GI_8v^EEyz?3Cz(g-|fg{TgT0x(fSz_ed9nBuAH_cK{VdZn1Ox0uw9&ONZ% zMskPwM4_9Y=}RP0<9ZTF)F6^X14t4LAc-1ZNhF&^e@K!>q$i-Lg4X8X6Oyu|&3fre zBxO{GM3K8oCh{bc&1#ukd`YLm#TrThWO6@U&X(CUsr;J$0VJ!)uGD>HGk;%Ma~n$S zLUM;aM-lUl^yd(bD-AkCgGiRJ=r9c+Ni?oRvdE>wDgY#ld^!q@I#ALkjllj0M0H3M zL6V8!e}PYxejSPA-?2$jNrI)Q%-eUguK77{$2OE(?j7q1O7wm>(!N79p7ie!4MJJM z!NW8FCD8zsMJ66r0YF(~<5ALOa|R`C(g;EcZXFVZOIPOCfu(E3hMm8k$rozl`K!b2!pL7-L+1`#Pu-f8eYP5?u(&9rjp0xN74Uow+l1!$NWHOB;Qx{o$SOt>I zq6%6XEIv@uCXE1@5Y-`3u#6=dT*m5(Ek6GhFe^#2%u-h8T|U{~h~qbR{1Kp5JIF-G zh$C%2MB_=D578i*C2T%S17s2djh188tsa<619b}?s#F0K9qVc59hiH(@5BlP!f+0 zOqWF_AJzy!S!D9jVDf>IHfcmze?TR;d}tV~WSIz7vK8BWp2IR3WJIbTdK_CxZlxBm zd5=$e-|V~Khpp${;WQ1RVQ>;mCb7{A)8Sf0Zc0}#wz^XLx2tzRJ;~Gd!dZWu{n2W7 zvU+1xGM){M_|Mh5u$=5&SXNe9Z*aAk1V-|09vqs%!#q7hS+b_le;BOZe}ZMqqj4CG zlW(imv)6DfCf%cH*nhq3)M7UNa}m6L)qCaazv?+}elheL+QFvX zL6CbLnnyHcIh=MLfp@+Q{^)ZqkX)Mc)6Rns#ffF{@BuQ)O;$iI zEio%ID&*gCW$~f}8u14Jf03mYv2EANXp(nO$f{hNc8D)osyz9(g5egycF-C0c{1pW z!4-=j?OS5dw`yPu&O^v)nXo2)mnVWZ!5Q!>6-_ivX-0|R*-Ik`n#L^Ih?1s;zwwq$ zTe+f&vzs_hlN_4xE&BmEx0D^tX;!4rVwj9tB9eJQlM(s?=E_7ef3fJSiW2d)`Q?zM zto#m-3R$(x5+R^$8qU@Z)5R9;>jN(ydDtuV1iRntG5yVth@*-q; zoUE*a6?|;6Ly9=aRV}5YD22K z6xGVm(=W5H=pI;J|Nit0(hHn>r5LY9XgvLUh&<2jBDZG@e_07gFQ9E|(DptYX~+nev>>-l0jo-O8QSJTUJz1r5h z?qah?eD{}Y$Mk*D`{kMVrsLsWt7q&G(?)2;GyjK7{Z^zBw(T0umYZS^ZP|o0o*+*$ z+KQQZ|Yt1Eq=SO0SKftv#>QfA$(ExkrX#kAKR3KL5x%eT-pPnt zF6`qn9)6U(m+ZHjyul>a*;9OVU^p8i zd!HKQe-YIFj$z=~a&KW1aZQ9<>Qk4=-DfRB!hRn^Ei*z(t9xzKc+K8MQ{{4tNe+=R zHUBjkP0r8-oXZ3Qn%&-`o*gIw?wz?B`?ILgR@4X9qMayS`vj|Ap8GJ`^IfnVf@qCAPw$t_1*Y%~&ikzp4(?%<(&uK)l5 diff --git a/summary_results.json b/summary_results.json index 03e784b19..cd0c6b738 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1884, + "Performance_Result": 10.2268, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.12825, + "Performance_Result": 6.60193, "Performance_Units": "Samples/s" }, { From 5c05acec7550be322b752c5c09e86474218eb964 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 00:46:28 +0000 Subject: [PATCH 268/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10054 -> 10054 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5a07136d5..b5ef22c63 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 00:40:45,050 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 00:40:45,050 submission_checker1.py:3270 INFO] --- -[2025-03-09 00:40:45,050 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 00:46:23,686 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 00:46:23,686 submission_checker1.py:3270 INFO] --- +[2025-03-09 00:46:23,686 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 00:40:45,050 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 00:46:23,686 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 00:40:45,050 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 00:46:23,686 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 00:40:45,050 submission_checker1.py:3286 INFO] --- -[2025-03-09 00:40:45,050 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 00:40:45,050 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 00:40:45,050 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 00:40:45,050 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 00:40:45,050 submission_checker1.py:3307 INFO] --- -[2025-03-09 00:40:45,050 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 00:40:46,212 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 00:46:23,686 submission_checker1.py:3286 INFO] --- +[2025-03-09 00:46:23,686 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 00:46:23,686 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 00:46:23,686 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 00:46:23,686 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 00:46:23,687 submission_checker1.py:3307 INFO] --- +[2025-03-09 00:46:23,687 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 00:46:24,784 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73114 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422411 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22432 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.267 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 208931296..4564b1f0a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -895 +896 diff --git a/summary.csv b/summary.csv index 83fd70747..52b82c092 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73114,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422411,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22432,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.267,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8da8029dc..db534f9e0 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.22432, + "Result":6.267, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index e7e23c4ecb392b0a00285f2bb8a690e363df2c8f..e8b8b6d8033b05a3470dc745a12c314be69e310c 100644 GIT binary patch delta 3015 zcmV;&3pn(~PR35KbO;GCCz@$a3IG7%lYIywe_wOjNEXH4_fx1;c}S|ppcX=ZT(+ki zXOb9W>^Qck_Q^#+fNB6Sl5m`FKN4Up?m3c`Y-Z)1&4UB2bG!RC`q3@*K?lD+%%cpY)FoqVTI0ty+i|{p^!b`=*%=%OETLloe-ve`{_E zt{w}tG>>fb?vGjr&HJq4jpDNmxt0YtvfwsX&`kTY?UczoWkMf}-9B8Q!fK0d5{bPl zxh4{4S8_un?ylsPNW86NC*2lLw*?g_kuEAkKR-XKxPDe~^Q_{wOtF)x#WS@a-V~Xl z!V{T3tGIqvaf1r^p_bLLlWCh~e|p@;&s3blZ;3>F?k7Yd2J9V?2w;CB5^GoTo=5~w zABaTo^pQx!8FWe{V(2?WB0j$}l$c_4yF?;J_nb(?==O+2jP3=Ih|#?y5;3};h{Wb6 zoXt--8!Bka6AmS|7}yh(v_!t|h(zT38a zdEPzQ#7BOdE@9&4W-YU~f35j$uvqL37R$YGK7WXo^CVqN?QK+dgK$0y#`-V%&`7vz7ySDnTpkJTY{Zi{CM#C~qEB6(=L!HB^bC5c5uDwNxBf4>d z5}=MqBy~h0sUs3`hJJ8Ls3YM5+S8Ko4u*RohqHXR;G&D+mKex$e@7xoVnmNGP@)Vm z`j<|PNRB6W{Xup;vzK1JtgG4^r$8>gC1Es}+})cEyJ74NmMXodc^<_&QmKIAjgv#M z;xr% zB9TZGiA17^5?7p=AG{J2G29dVJS_?Dcmk9&S0yjAM9UXUBPzc1>cwr= zAB6GVw7;t1!L@((y0nh;Dj;~}GteKvf2Ze{oeVXcUmVB^pPZ!zV-o zJTi?0l4&H7MB|Ae`moh7CRqg%%CZVN+k;yyp`^(`Ck`X*ARifLTY5Rp!l2$JFNDbHjD)f~!cyx1Np_?&h>@2_|ZswgeM3 z2qw`0m_!3$q6Wb<%fQ465X_PaI@^O!V9Js%YXrcQe^EUW1vgcRMj4Le()>T*vW^_9 z+-Kc%YAbu8Z`Jyy8)*mfkCWPj3@V4?=WBpLvdXaG#q;5xQd1}3Qhz$~et zvpx6(rYz~QMqCM|jOvjn1e5G+BbcrKDKOO!tz_%Su@acA9WbqyV;3%E`JVoh-LOw_^dyopsz;*8f7K-uxsu6ty-Y5@q*LK?1El~mxu30O z%WRrdUc>wWBR}?BKdc0lGKu5B`Wjw9n)-hId8`< zf0R4!9pec~^nTdVzC$#w^zRT2LRrDV!!!UT(EyZXCLUG+Kv`zvQPO2|1|?n62to;N z9TJ60SLWA&rEATGoxh*S7i#1Atnxmd^ny~kR*{iMxdl|B55$&M9-?ujmxpMO$_j2C zrU5F62B<7E^sowm$}&rjl8&1*DCv?$e{jbVoI4~6cPyEA2X-uLHt!&nKYz>eg_?S< zRo>NOrx%q4Kip7W0hH(#v8AbpXk6*)AsU3Tg0F{Z07{|(D9fxptO9_t%-o~E-2)|E z(g^O0fWe1Ek@ag8i;t?=Og>BlWD@&~DakA|`LITS z%rcXY2__#X>9StRElY^%ktm!qf2l~$nP$x=@vE-|eF zDA6urON$TDxYFW7Gzet{ix1NPl*FL}(`A{(hcyCFmRWpEu=qepmo%abpb|VjGz?a< zOav?0noT~>VVR6FBGnJwe~hgqw@M4xyu&BGZuT5-!!~nIaGLqi*gx^7(^wmX>2R$h zx25aHT3xCA+w~Kmp5$qJ;d~fmKUwWh*Uzg;g85jB|6D%_%g&yJWn`5NMpw(JuO;6W z{-N$aEYeRXOIFv052N)nu(U-Kgi(-uTeqG)gljqNAI-wy>s6(nO7hqifOnsamVoNw`?O zYU|pUFiFDM<-)0C*ju=1J&B`x=jCYZx-RR=!5Z`Dc(r(aA68AJ?oY)5 zHI?9$J7B@Mroxv+JY36sSi}>rC5Wee)Ni~U7diBSNMZnbhNw=z#1r6<`}z|!f3vtE$^r>7Cz@$a3IG7%vx_A-0R#s$ J?|_rXCVyLjxlsTB delta 3015 zcmV;&3pn(~PR35KbO;GQ?*nj73IG7(lYIywe_d1CNE(Ij_gAP?xk#!8TiurRK><^Q zOcIPSgaA`}MLD+oQDa*kNhajqk7OGozvoye+02Ta%>}Vr=k4ydrH5{*4?6huVLCDH zgD4({vqMwWEYk>Py|6!?4G+zm>-T$}X~ao)*6&WjS#W563u5!vqkkQI4WmEeQ4l0X ze_FvTJ~T&3GT*P)$&4`%6+K^RTDN%}P!*5i2;bo7=eL%c)PhrUad%?S~x0nXAWK~HNOu9+h{dhE<$L7&Nf1Do6@-d=daA+Q@{gdCdZPt$t zmTljTgRgP!vyr9pB@F+_zMS?CO)Hbue?j|x*^SG{=m&%DVv<~iUq1%p;V4Omt(k@N z!b!aRX-voISZdRlb{`JShG~2q_mfdpu`wF=`$0C)rqNr(NjUv|bx5r%ScNKQ9jh$0 ziS}#CwjABe&d0Y&u#Ty(T3z<=T{r0-9Yo<*BU-i)&%4=~QTtk_!_v!2KV`*Pf8Xls zf~&^@Bh4clz5AoaLH$0fc&+#>L#}1PjV!p$71Y!IY&xZRr!@4z+U~;@D(t4{CXqPX zl4~Myw8feM=Pf0peUaV}7zEOE_UK6QL>^zq-9zg@LYJ7=$#i=jAO4GqW1Xz``i z3#awzB>g!}S32pZc@Pcm#$S|@f2QWU-h95-n=kgl+3X=&%#w65wYOH?^}^X;JT!kX`$5tj zPh#`UL481`>rYjrlrCC>v}?=%cJ%A>s$Xig#HiU?+qqxCThzInIv1%E*VS; z4X@^Vx!<>2Bzl}g4@ndk&|8$aa>0cXAdyHUi9{kvBoaxYS3n{Omn0U1w=f(?^a@DC za3Il>B$600V_%>|e_3)V<^6L=tRj^XB;I;klVZcxuDxR^>T?u*L{UKW7A3B@$WBlK zC=!W8kw_$pDDlLV`N1zh5yO4a&(nhNmM=g#lLc+kfLw~om=4A$OSF8!G@{~5zgpa; z-Ch{)O}fh(9z5q~uS=^)uLOcuehxvu<|^OS*5kh&3shxse;AkLf<^(!TcUBrHGD!e zz$4R0AelxINi@C~q7NGlW0F-Mp)9JPwK=%K5=z>v5iy6KLsW-E$%l>?M58QG^<}RA zFN4jeNipNH9|ZICdW_=W?q1crbYA>!IxESr1fJLHj;;7~YL2CHSJyU`{IUD7!pH2* zrlb-zt}Uq~e~;^dN}>TOi3X@d4NR094W$w*Kq?C=Xeqg|a0Vr9)`%;qlu;cLg%c(f z$r&@3z)Qq3H(h32YOAJx0L&_KEHiJeJEqppo@<`#?6`|meCz30;ciaTkzk_6ZAvgv zgJ2R3fJrm}CTb8&T?8gpfM6C>(Ape)0#lZ>St9_Ze~jvoD0rw!G|F-%m-_#J%PMj# z^PF|vvCTbYv(a>Rtz-En>9NAS$EGX6B>R(&1QRs~CeZ+xL<3-=2G_BTA}~n>0A@i2 ztHhTFOfuz>q#V0gGdq$AW1ZUBx-ym zk!%(rNg9!!fT9Xon}bhC%91wgr7w|`Q5_ORf9@`s$dgPqt7UTWC7lWvYbXVf$^CRW zTV~Uw@@x7BkgOuRQume3{C#E3Z78)1$sP6_Ma(zSpF=dRH0TfwB3Z(s!!&>-(YO-H zB9{)U0FW&5=_oMjKuMc40{bHn)ge&?NhX2^K2`d4B$9u}CP^g;mZCCm-_g3}=e!-; ze^73@cdREU(fi>@`wr1~(!WDA2xSQe57PjYL<3M3nRr+Q0A-PlM@g5>8I-h1BM2q9 zbx0I0U7248maY{WcK&`QU#N}ev&#E;(i=+U*}IH9%G*ID`am3Mkk4PZ$$fMt=nhgASr7TJ3g_D6)brWAafIyL_JG^5^W0yi~7`%A0-CRjg-wJD$L5i<7C*F5*awe-F`k z(&9rjKqk{jGMPq_$uyEoU1afL6-Y9RDrjl2_&`aUGy-HoREI>tGL~p?8LKO{`21JE ztR%@YOIe+F`DA+|j^EtzM}S)GAQK%UjBw1$8)OnweZq&4=c9E%f zkcplVNBVq-#*;oDqCqlC_!;FV*Oyw~aow_C0&Y;~(AFK_;3;9BK3+8c!O1hz7|lVf0}de;|`+U^QE0 z^kEeMnMFn)4Mra*X_H34PXJLJ76oLMG5Y9=jXuwj`JWmJD#@=Dl)~g=xys&ki)rtG z5)C7cH2DyXCrv&?gHV<*`7jMYNjy3*T^5;qSR(*sk;zAc$p=c>q!DESmEiKBVX%^A zB3Q{*Z1Z^z%Vdxdseb5je{3bWm0G~&JwEAuv+sf*ww`;3(=>>N!AUTg#6~YnhietN zDP6tT>Pqe3uHFIlBv0E5XZ>;ZN2}e*>Wx*&cs4ZRKUeR{b>Pstyvdo#n6~h0wE^n|xF~>>%a|$9 z-g}c_?ARV8bxCljAnvr4)Q78ot+Ao(o7R9Agxp~Xo^t9b?t!tHi0FvqY(knJ+YA8C zFf<@Rd56;^#b>VvE8T3tpjq(s42PYL?*nj73IG7(vx_A-0R$^l Jg^ZKOCV%lXvX%e< diff --git a/summary_results.json b/summary_results.json index cd0c6b738..22813b0c7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.22432, + "Performance_Result": 6.267, "Performance_Units": "Samples/s" }, { From bb30095ab0071fd92b732ca4dd479798a23b35bb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 00:53:23 +0000 Subject: [PATCH 269/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10054 -> 10053 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b5ef22c63..c00469f16 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 00:46:23,686 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 00:46:23,686 submission_checker1.py:3270 INFO] --- -[2025-03-09 00:46:23,686 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 00:53:17,650 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 00:53:17,650 submission_checker1.py:3270 INFO] --- +[2025-03-09 00:53:17,650 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 00:46:23,686 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 00:53:17,650 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 00:46:23,686 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 00:53:17,650 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 00:46:23,686 submission_checker1.py:3286 INFO] --- -[2025-03-09 00:46:23,686 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 00:46:23,686 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 00:46:23,686 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 00:46:23,686 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 00:46:23,687 submission_checker1.py:3307 INFO] --- -[2025-03-09 00:46:23,687 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 00:46:24,784 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 00:53:17,651 submission_checker1.py:3286 INFO] --- +[2025-03-09 00:53:17,651 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 00:53:17,651 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 00:53:17,651 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 00:53:17,651 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 00:53:17,651 submission_checker1.py:3307 INFO] --- +[2025-03-09 00:53:17,651 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 00:53:18,794 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,14 +24,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.26254 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73114 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72494 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422411 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.267 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437693 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.402228 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2268 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4564b1f0a..8a655068e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -896 +897 diff --git a/summary.csv b/summary.csv index 52b82c092..234e91bc1 100644 --- a/summary.csv +++ b/summary.csv @@ -2,14 +2,14 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.26254,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73114,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72494,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422411,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.267,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437693,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.402228,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2268,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index db534f9e0..98654e6a2 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73114, + "Result":1.72494, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437693, + "Result":0.402228, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index e8b8b6d8033b05a3470dc745a12c314be69e310c..4d69a0e57c72d8b06c1fc3176e3aba921b583622 100644 GIT binary patch delta 3080 zcmV+j4EOWKPQ^~JbO;5!t2Km8lX(awe_eChNE(Ij_gAP?xk#$Upl*adT(+kiXOb9W z>^Qck_R2*-eAED9B;h#!ek8yc-g6`?+04p2n+pe8=k4yd(L=Y?2Oa$SFr66pK@^X} z*`cXwmT3gDUf3VchKJ_O_4_@~G~%Q?>vt#NEI2g31+n?-(Z3G9hS4AKCk%yKya?2eb6ZAdIHnBz+qV>+w7ay8UI)bW+!r<ks!HtR#FmwYu!#yKd4wI*7urMzm}po_DiPM(t~z4ofd9{gf4Fe|@X33$7ju zj5LpI^zM%u2le}`;PkH;KgAm0S~v zyDPaN5^q;>OCp9qMx6iRa`%-xOrA_Tcp^@)aIGm5U)k1sPILm z&nm8;RotLLeyBxt>}1-cf0-UP@iP_Y@LM7gpZf`shyi;?Bm&sqh{WENyeAUD(+46E zJbfe*aR!|di5U78k%-Ul3?*8OZktHN=$;da7~Kw$h|#?u5;3}$L?TA_6OlOlgmd@_ z=RgHbdBUN@5d(XIl7`6l9g&E9e%ckXPgU^C`+7kmrorZ9DV%vbGFp78^}=a= zI!XUd)0Iy8Y92&`yYXxgMCp1hxJ!aKxr-NHrsFtU`owql&6-tDq99lOzo{zcfD{n7!S=~%zlt`$CKE6 zb5I|U>H1R@DW!|nAnn@nznp%3UiC|@mKZf#YkTV}c!xTdQ|BUe;#_-+5?6HN1SLQn zkx1%@L{dj2;tc)Z7EnjR1+=FH;VlgJMGj}%;ev}chTCEwf6rZsB#99{zCekx#OPnT z6(Ttt-F17}^~_#+`Le2NZn&7zQS&^CRish^#T$1E#fDe& zy{+%t9TGiGqK71k6X-2UT)E&v36Mx6l0+hrBoc`v(JLU4gi8_&!dn;)BzgrTVmOfK zNfJqnn6WQVf1)h8l=A*LBvz412@-F;ok_7_YuCDrhOWv2X?@ZPth@sgzM25`_~c70DTM zD}k4Y<<@kWajC7E`T;Pj$g#}4x$c-+-+Qiku9I^Wsrc5@vBKS)rX#^bjoXx9q6Wbv z8UT}M08G>%n7Rl|tN_6*sGzku_yndbX|qNEe@q$GAyIHsm1vaZN-p*P0hd+eSmr+K zx?`Jb%4VbKFG|FlAJSL?M`DXB)w6{7-?YerP3IMUJJwZ0vw(TUu)`2TZ=Ff2?q;yy;0W ze^KMQ5=_(}m_!3$5)FV!w)X){y9i8D0f1RhL2GmH2~1hiCXK*-R*34bC;$^R1Wfxy zgDLL1em|2{q*sbbdxuHw=&c(zyGZ7kPZYWdn!ZF5HLfR-L=7TIG=L=00FtQjl|-^x zgd}N1dIE|nXl)KYAt_7Rte3t-Qbu)1e-yd8WFk*8*{qhy#g}v{T&$rKKqmLo`Gl%Hn*=UYp%BaJS21MIf|HXq(6sfTxrlD8bq>$MTcnsNuqHjl0_~Z zRskScqsR3j!lwE5-dgKwtYwIns4VVe_eOn zJJu7F=>2e{eTQf~>E9t5gtCN#hiL#xq5&w2OgyXtfU?NOqom8`3`*Lh5rh)lIwT60 zuFS6kOV^4GJAXfuFVx2KS#A4x(hExE*?C4D<>gR`J`hJ*d5Ff7ULK-BDoeO|mXjWP;0wMB$uS#^tls z|9>!;FV*IwcWpkl_B}i2=8taVkcpNNN7{Ud#*;Q5qCqlC*nF4nSj`sMd{_lQ zW|7TDgUts@+N2S16F^jlMFE**Y(BbTo6mD({-<_=O7bfOrL_3C%FesQv~!?DyNDw# zK1Ab5ix1HtlqD=aOao97hYn1aMHU~{2tZk6@zG%Mfs!_9L|H&3czkFWtYn!8Rk6pY*!fcfk!?-+F@6G>C@5NidnjMlVc|AW+Yl_p*z+wY=CkO~ zQQR1(Scs)-v43U!o3(%oLhiAGzyu2nAAqSHiRgqBd_>wG*9`!{F?66n2aoeC#}{t^ zr~G8Ypjajz3o~Z^Npo)Q3}XKusk$NXAjK>tWlA2_gwR<@^ZD0d z^Qck_Q^#+fNB6Sl5m`FKN4Up?m3c`Y-Z)1&4UB2bG!RC`q3@*K?lD+%%cpY)FoqVTI0ty+i|{p^!b`=*%=%OETLloe-ve`{_Et{w}t zG>>fb?vGjr&HJq4jpDNmxt0YtvfwsX&`kTY?UczoWkMf}-9B8Q!fK0d5{bPlxh4{4 zS8_un?ylsPNW86NC*2lLw*?g_kuEAkKR-XKxPDe~^Q_{wOtF)x#WS@a-V~Xl!V{T3 ztGIqvaf1r^p_bLLlWCh~e|p@;&s3blZ;3>F?k7Yd2J9V?2w;CB5^GoTo=5~wABaTo z^pQx!8FWe{V(2?WB0j$}l$c_4yF?;J_nb(?==O+2jP3=Ih|#?y5;3};h{Wb6oXt-- z8!Bka6AmS|7}yh(v_!t|h(zT38adEPzQ z#7BOdE@9&4W-YU~f35j$uvqL37R$YGK7WXo^CVqN?QK+dgK$0y#`-V%&`7vz7ySDnTpkJTY{Zi{CM#C~qEB6(=L!HB^bC5c5uDwNxBf4>d5}=Mq zBy~h0sUs3`hJJ8Ls3YM5+S8Ko4u*RohqHXR;G&D+mKex$e@7xoVnmNGP@)Vm`j<|P zNRB6W{Xup;vzK1JtgG4^r$8>gC1Es}+})cEyJ74NmMXodc^<_&QmKIAjgv#M;xr%B9TZG ziA17^5?7p=AG{J2G29dVJS_?Dcmk9&S0yjAM9UXUBPzc1>cwr=AB6GV zw7;t1!L@((y0nh;Dj;~}GteKvf2Ze{oeVXcUmVB^pPZ!zV-oJTi?0 zl4&H7MB|Ae`moh7CRqg%%CZVN+k;yyp`^(`Ck`X*ARifLTY5Rp!l2$JFNDbHjD)f~!cyx1Np_?&h>@2_|ZswgeM32qw`0 zm_!3$q6Wb<%fQ465X_PaI@^O!V9Js%YXrcQe^EUW1vgcRMj4Le()>T*vW^_9+-Kc% zYAbu8Z`Jyy8)*mfkCWPj3@V4?=WBpLvdXaG#q;5xQd1}3Qhz$~etvpx6( zrYz~QMqCM|jOvjn1e5G+BbcrKDKOO!tz_%Su@acA9WbqyV;3%E`JVoh-LOw_^dyopsz;*8f7K-uxsu6ty-Y5@q*LK?1El~mxu30O%WRrd zUc>wWBR}?BKdc0lGKu5B`Wjw9n)-hId8`eg_?SNO zrx%q4Kip7W0hH(#v8AbpXk6*)AsU3Tg0F{Z07{|(D9fxptO9_t%-o~E-2)|E(g^O0 zfWe1Ek@ag8i;t?=Og>BlWD@&~DakA|`LITS%rcXY z2__#X>9StRElY^%ktm!qf2l~$nP$x=@vE-|eFDA6ur zON$TDxYFW7Gzet{ix1NPl*FL}(`A{(hcyCFmRWpEu=qepmo%abpb|VjGz?aVVR6FBGnJwe~hgqw@M4xyu&BGZuT5-!!~nIaGLqi*gx^7(^wmX>2R$hx25aH zT3xCA+w~Kmp5$qJ;d~fmKUwWh*Uzg;g85jB|6D%_%g&yJWn`5NMpw(JuO;6W{-N$a zEYeRXOIFv052N)nu(U-Kgi(-uTeqG)gljqNAI-wy>s6(nO7hqifOnsamVoNw`?OYU|pU zFiFDM6Ci-ZsmJxqemlEzHQT8+`32{t(KX65f)dc& z2&p$U`dH4>2bB20v|z2#P@_F~G;8iEZYeho-kh=Ef&`yLkoJ_@8f`V0v$ZgvmBqj z0i5zT8wTaVsbtt&xM@9!qkHG&XzaQ!>&n3z^XGWAczqvMO{MNn#Q`;y;FLRH!MLWv zmqk2W%Y0bG6R#zRr+w6Kyd4)g^nplX0D6dfQa;P)m*pfJH3cc|Q&OhnaY+cBOldLs z>PKGnpMmN)-;E?j0d$ieCP4ualVK)0 z0fUpACMFvPH1B{V0RR9{0ssIJ00000000000001hfn<~2CO!eUlN=`_25KY#0001{ C Date: Sun, 9 Mar 2025 00:59:25 +0000 Subject: [PATCH 270/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10053 -> 10049 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index c00469f16..14dff1263 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 00:53:17,650 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 00:53:17,650 submission_checker1.py:3270 INFO] --- -[2025-03-09 00:53:17,650 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 00:59:20,231 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 00:59:20,231 submission_checker1.py:3270 INFO] --- +[2025-03-09 00:59:20,231 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 00:53:17,650 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 00:59:20,231 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 00:53:17,650 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 00:59:20,231 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 00:53:17,651 submission_checker1.py:3286 INFO] --- -[2025-03-09 00:53:17,651 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 00:53:17,651 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 00:53:17,651 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 00:53:17,651 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 00:53:17,651 submission_checker1.py:3307 INFO] --- -[2025-03-09 00:53:17,651 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 00:53:18,794 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 00:59:20,231 submission_checker1.py:3286 INFO] --- +[2025-03-09 00:59:20,231 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 00:59:20,231 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 00:59:20,231 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 00:59:20,231 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 00:59:20,231 submission_checker1.py:3307 INFO] --- +[2025-03-09 00:59:20,231 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 00:59:21,281 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,14 +25,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72494 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422411 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421125 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.267 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.402228 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2268 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2354 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8a655068e..a2fa28f5c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -897 +898 diff --git a/summary.csv b/summary.csv index 234e91bc1..0fd4fd94d 100644 --- a/summary.csv +++ b/summary.csv @@ -3,14 +3,14 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72494,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422411,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421125,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.267,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.402228,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2268,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2354,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 98654e6a2..c7ad7c3f9 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422411, + "Result":0.421125, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2268, - "Accuracy":"acc: 74.800", + "Result":10.2354, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4d69a0e57c72d8b06c1fc3176e3aba921b583622..0acc0e93f64b7e2bae82c3ed51029562a2edf76b 100644 GIT binary patch delta 3632 zcmV-04$tw$PQgyF%>x92g$|^X(E~0Vflwb>X%a3;2#u1Ars=)uEzNGVPZ0qV&;Vv^ zlN$sZf6t63!gxxHk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f2 z5Mf9I1s|D-5aqzKc$knp;B8uxLOzKp%L5^A*~;Q2BSBP?^2~B=+q3c@EzHKkfc=RL z8pml!j%irsq!5h}MzTQ2@OhG!+#C<1R6bMre;AC!BlDXxm^}KXH6GTk|4GR+e|K$& zVEI5Y5(*N@0+@!%Y)R>N_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASBa2DdWsU#jMu+L zp32g?rjZ9PBeNqtq>)IJVIxVSh!nNku;PN|kIf@zZJ?n#-kR&!&Kvu%?R0I=?Wx!E zf0~gs$7;OR5grF37!MhJHdyU}mw|dw&cL;0l|nUouZC4@)_P#J88BoXs+dpX&d_>N zhS!J(NSVJIu%x#)moDzo)jZhS9%eeC3u4d_y=}x?M|f|DttdNqlpT%lfGBlDA4K_E z#C#WGE6Oe&Wmn_7AW9wKgD8KCnCpl`e>}<_8s)d1UQ6)Gj}Qo`_XL4}WRDRDn7t`P~B?;0;+q1KtOeG5eTU6 z9RdN>y+oTnQAwLj7!9ZY)fDi>Q?G4Xf)P@;AC~Getn2Ay z&1Zzm-JDyiWD$g_t!T+EqA}G9f6J;>z6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*% zU}(k2=63%XImv#G$k6J)58|3NaXHO~-0t*!``xOyCAvN?y1p(tFb$7%NC&R+2n31_ zfk4qA5GXnX0!7!~Av%N(MYn@~*@vQo15@3BCOXKu(K#UNTf{0T?_K|Ye<-@o$mtJ6 z_t4)Go$Yje@9=}I&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$ z&@X+cIyf-Z9du!*1UWZ42P`DFh*j7Y`Jb!0YFQPcYRM02W#Yy6J{Dm2zX2-nptasnie>q_*G9g*UjgZQMHO#ls=DcC%4Cc+Z_%FEnm@lGKt!zQo zoE(U>SQ-3Bb41|{CXcSIjD_*6%7`IeOZiDpUP}3nRIa8Gy~NFFgi*4TvQ)e_-qq}oIdaRCz6OL$lYD{>vMRa$T( zBe6`>Zp*cbHRl`kwV-8vC7bmn6@uoQyMzQ0Vd_qP-NuxP&5bI_+p*F3Cs0cR1QY-O z00;m806ze69dBkylX(axe^c8?8inuoSEy9ENU8=~-In!10aJub5{xl~08@KKIkx;! zV_P0cCgk6bWE*3@=SV2o%!-}O1+iP_?e4dwhi<74I{5WrIx+5pC?1EiLsQi((+Fn0 zus@y+56zqF_j{gc#7TG7?@q#5aA=u> z-YA%M<61ZmX6cbZ7)`rL`ZXHX<9QTx`^%u|q^>Q?sZYD(nYpTAKPpp4I2eq3!Mm`x zm~x=Gsocr>2J=FvfaoF2>aF`{5_XdbKmli#&%){hRBZQqWAukqGrBTMB= z82*ucIqe^sRwk{3fA;;d8<&yM4+h=EB)JN|ehkLLQIZZ@GYjd3lX&^ln2yu2)TS}* zJ{+12)A%~>C!?%lV>IsfgKVHpqqm5YaQge|kXl!;3RTWJR#|Ej?bnoTIl7shk8hJ; z9aCSmy6oY*ZqhwEh{CT%v}_@sce692_O(ujrI(d{%8IkTf7RCoSC0ionnyNz_eYI` z`h8aMTJc$iT+4zRS#Y~mP*3}_>6GT3($EKMw+~mSu$!WrMB?m9u8G9mmD~`Cw=20N z5`QDvNw>k%Z9oM|q>BpC&(F^)uAf!hJgc}ZQtV`E^Gt1s*CJC?_#)G171z%yZcrgl zYEd0KnKo&rf5%Ncr{WrZOC;jFpAd-{uy;fvfc=d~>|M!wA`v`&AQHjTMGk1IDcm-(PDJlL?TA_oJhpzc8Elb?gf#E(Y+)RF}k0K#Nij5!!I}oDrm|J4keBl z*b|gAM85BcMCAJ$k%)ZXqr?-}!G}i)`GxqAv>;MBe=P`aJsx|Upw47L8yoON3g1s#5H&M)bYX5$A4e`cGW)ZoV{KyhT?QJG#n?R#g|$yoYtq4^yf5P>7<|L zK{U7<&jvx1uGfOQB#4u{c=2UAjh2T6B4iOn|$^#Pf#KUI-Z zx@Zm3t}Xw|>DT8~ztn1pQM0vDTVU=`=W^;?q)uFGZ&BikZk(V5s3Q_d9g#@th(uhW zAKU`!NVtIZv>?2N;l9Y>Y&%?V(Z+CF4CJ{he~~0HqQ@5~QI;6}OSeKKhoif0FT0=F zOK)FRRqYK^Am`tbFzSu&?sd(nS$n;ON^feON3n`jN}zb-ZlT!lYQE>^kmzv|JtR?F zKyOjv$^{onfJ7pZBoc`vkw_$oUIB?DT#{H2-okJo(JLSk!+}Ikl1O62jD3L;Wyz(K zfA`NJv5Hhmka+9uOo|O#yLQe})aNMrh@yb#ElON*k)5CfP$UwGB9TZGQR0a!^MhZ2 zB8K~-pQi=kEnk3gCJWl60l5^FF&&IimT38cX+*`BezmwwyS*^pn{<~oJb2E}UYAyp zUI_%R{4E6inyY+QTaW(~XHi)k#$~yn2vI=tmS|jY4U-}bECHdDPYo6^Jw|bGcdu$* zIxl`Vot0!*0?+Gp$5wngHOEq0SJy6;{IUD7!pH2*rlb-zt}Uq~kCT=SMF>B8u6eGL za~G524K)_$+o`dy_5>iXd}Vi0ZH?024I?O#4NHDW1B1 zKa*9YSBgn{he_?|tp}6X4m$z6lN1jk1>ILRx9^iX4?O`7lX(wg0UMLu4^w}s%=yC& z<>x?&ZV^YCdWgo8t{$R6C`tn4UkFvGeBmM%ZD`rWEQ!6HV2=`1eXu%C6Ea&9}CoMii zgHV>R_%IDXNgO&bT^3n!J!#E%+oWJC2Jb} zhr#L{SjIdWhtW9swrV|l3D;uMJ(`C7*UL^VX5&8>!RuGOSI+*cp7Z7xL%%T&jWqQ~ zX`0=#mFxrS*3mw}ygLleyU}nwi;YPzNE5Ycu4zQ8w`wgvC*gegsc9Nt!Xyc&kKaZ? zw;x2=xAgbjAPkeoFWHc6-cEM(7qjUSm;nwOGnpA{1ONa86aWAe0Fxjl7PF-kf&v4( zrem`z7e@*NODuk_lXW32f29$&18vEvfkIPSdK1zn5w(7h)H;6qDvskMkW-JVnf-Ql zQMK5sHu(kTrO`FYGlCM(+z6?+HF{q!(+8CJz_ehk(NLovJX$n&6}OZd2d~apa6y7k zB1n76ZH;yzScx9Fqncrp(E~0VVWB>>(j;7x5E>;JP1AeRTbkW!pCSS#paINa zlN$sZe@n>|EqTK7kzuzjgU~!+VUn*##^3YPmS+$y{5g^Bd7Bk9mro*AX1( zSQI4zJ!V0b(Ofh}N}BpYhR@@q;Kq0uCi0oee}|Mb8X4d0!Q|01%<-^x{ZB%l`MYZ( z{N)2pX&`7Q3t$kH*^;sE>fwbPvJ}a{hCtkH$T;fYM(gki%`rl@4)0b*BgfRQDQzfa=~L5K!G)1Olpi zhd@Ae?-9rc;&t%}=jsMP?T>Uw2UKl>KtR672n6JNf#{( z7dz;en`fK#{0a_Cbq8Hgt82))K{#NpAP6wy?tXuu zTDAk0nOQ1-GuZ>ktAdu%BFQ6K%1(wZ1m$AEt7VpO)fDi>Q?G5AqNKELKTOqSnAg+E znqN{bcXMv8;)NfmwxT7wh{jYae<-R}@hVsu#EU1V?Y6DfdiP%hERT|v@x=(K@RO7q z2SYPLHn;oF$Vv8dREAdneGu2IiOX&__DK|3!eGk=`E& zaPGYmpl!7s$Mt~x%y-|?e;F-TbVAdVlYq&BHO#ls=DcBM59ZCc_%FEnn9svRt!#eU zoE(TGUlIPJIihd}lSju=#)4>8rId)*LVnWImqNZHm8(I*m#8_7AVrz5G7+zhcXc{a zrT#c$;eH)dl_x(cy1y6fxr4pHJ@`VL12Q5q^9@vHS9__C3y`qJOT)rn(QCh4B{?T4 zjbx%$Tdq}QbG}hu3s%%uvRPj;Ay~G#i>V*dQr*d~+lVo-xltu~J2o2s1W-!@1QY-O z00;m806ze{t2Km8lX(axe{b zZxl?saV?w&v-HRyjHcZreH#ty@jME;{bkT}QrDK{)TiC?%v{y5AC;*i91O<2;9b~T zOoLgnsw4^~-6ZXPJQ~kq^XQ;IPLE~z7*Q}dG>_H($?w`W>qiI6wr|J5*Ldr-k)`q_ z4FAa9oc0e*E0fkifBSyfjmyaB2ZQcnl3ayfKL+FBC`pH{nT7PiNxb}POvmY1YSWl@ z9}dliX?z{`lTlW&F&g*#K{n8)(ObkxIQ@NfNUbYag(_zqt1PvN_G`+v9No-5k8hJ; z9aCSmy6oY*ZqhwEh{CT%v}_@sce77M?Q5M5OD`+^loe-vf2*$xt{w}FG>>fb?vEM= z_4};iwc@i3xt0Ytvfy^Bpq}<;(<#k6rJ)bjZXd2tVK+rLiNx8JToZ}AE4d*OZ&z|l zB>qOSlWv2j+kgs`NEa2NpP!#qTtBP0c~)^-q}a*S=9$_MuSKS)@I|K2Dz2YZ+@L~! zs6}<`WZI;ee;zmSGZp9XTOtvk`w5YV0eeRz0@&Y(#NL&>ClbNa2O<$XeIycb2AvX# z82T2Gh|li~C0dMbn@Ggyo)d`}-42n6(Y+uNF}jyTB1ZQUkvRN>bNC78Km|>C!lA?w z1ABs!hRF9Fk%)YMBNCDCdz5(MJoxY^Azz3eNed#Cf762S*5k3q3F=H1w6OtSq;M`{ zIv68ooC}mFOPq6;PaPi|ef;<3Z&&Tp&e`kbVkk~mL&I@0T70SX!fAavN&imMl}`F< z9z=t?@oW%8>3S`=OM*DLix*#}<2YOT#CP}2npIDtAXqN4>e*VVe%?M=$45b&E@9&O zdM&fJf2sMdH=pnI=8L^>HhYK`vm{+i?X6XJy>K=d56xf9evov>lh}N7P#=)#`coAt zrHj@e?b`CcoPK>?^-Ham7&TjKd+RHBhdP&2=OT6DTziWWS9Ie9B|sgKNa~10Qb#1> z4E^91P)EWAw5J8(Ee!WX4rklpf{QkW+hQQke_e?ri4i@%K#8)%=wG@OA~_u0b$i+M z%wBr=vZ`utm;yQfmV{Anba$_7PR-iuEmV3@^E`@Gq*4OK8+QxEhF9~wt?%0%5xWAeW*trh_rc5-ndaji~t2uNJpyw-?5HlkT#H2haK0 z>(VOHD}msZzlES*bCvIE>+xUC0##WY48~=-piw~bmS|jY4wD`YEJ&u~1<@!=RDHSC z|Chn$)1;Vj*$;wwdOk*RaCfh2UOF#+H=UJaSOU-Mb;nkGIyJ{qTW8lUmHf8*vBKNz z&8DOhHLfkGB)5~24Mhmwd#-t|lXDf5-wibua8s3Nl;uh;_5T5vRpeOaKI^(;n`@IQ z4vHXsR*34bC;$^R1WfxygDLL1em|2{q*sbbdxuHw=&c)*)DAlVx|0wOA_ZMnHn*>n zHxE4lHMa9;!rJ|v2)U&~m0RK*^j=cxQSJ0maE>7%x-KIt0Pv%MTEfwnlA z8qFe(H24sWCk;MC17tFdB$H_*nM@@!-DS!D8IjR2WNCLawZA1GFPvw!RbI3%0$A}|cK1Ab5mk-e(nI&94Oao*R{|t~>yTr6}phUZfBP~8e<4KDT(IAv1EIv#FP!fj@OqWF#AJzy!S!D6iVDW*HHfcmz zKqYv5Xc(+ynFv<06`OpX!!j9UM5-UU8Cywir53Q;4xjY8*>}MWTi<$u(=>>N!AUTg z#6~YnhiesoxhY*e*6K>_->#kj^(0T*3upat_LJ4_Wc9qNWIP)h@t>TSlbdRQC|Mjv{i`n?k zMezDn@0GLvs^`4<#n5kzLnBT7QJQAAY$bcax^;9uFz*h7^KLX8&thW|4AMlcnrj-- z>Zw}G*GV{EzG|ArmoQ1f>EpXm(Cr6N_AdRt8-!u<_$C{Y&C|(_{sOb*5|{x5w8}}7 zvw;+%0s}i{V6!F{M+yXNOr8akaUm^#B^9;JwSJJ)I(hplPU0ls zQ;(~e{dRUywb-dH`32{t(KX65f)dc&2&vl|y|0()93?(5Em&tX)MyVLEt1&{5nNr&x%kYq4ego3(%oLhiAGzyu3_3?G20 z9f|0K6nsS5AJ+{4!7+58KnIWWEXNmb0H^$9!=PL_l?;0elh(61x_4fV#=h^fz8tJE ze~v#_FK@%Dsnmy4aX?KaIOPslFs`ZabrBEOF&|d(#2X3XaUb=Gx6>lWJ`hO^K#y_H z%18P1yk4TFAjK>tWlA2_gwR66|iA{(^INt0^?000CO000yK0000000000004l2bCVk; zK>-kxT_!pKf|HmgCL3%_o&_ZV002+|000pH0000000000004l2W0Tn?J^{Cr7$+kJ JX(Rvu005ISz>@#~ diff --git a/summary_results.json b/summary_results.json index 5c8188f8b..9c25e3cbb 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.800", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2268, + "Performance_Result": 10.2354, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.422411, + "Performance_Result": 0.421125, "Performance_Units": "Samples/s" }, { From 870abbbfcfb732012a2303690ac2582ee838ac65 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 01:30:47 +0000 Subject: [PATCH 271/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10049 -> 10051 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 14dff1263..6d79a3c1f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 00:59:20,231 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 00:59:20,231 submission_checker1.py:3270 INFO] --- -[2025-03-09 00:59:20,231 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 01:30:42,506 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 01:30:42,506 submission_checker1.py:3270 INFO] --- +[2025-03-09 01:30:42,506 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 00:59:20,231 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 01:30:42,507 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 00:59:20,231 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 01:30:42,507 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 00:59:20,231 submission_checker1.py:3286 INFO] --- -[2025-03-09 00:59:20,231 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 00:59:20,231 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 00:59:20,231 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 00:59:20,231 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 00:59:20,231 submission_checker1.py:3307 INFO] --- -[2025-03-09 00:59:20,231 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 00:59:21,281 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 01:30:42,507 submission_checker1.py:3286 INFO] --- +[2025-03-09 01:30:42,507 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 01:30:42,507 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 01:30:42,507 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 01:30:42,507 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 01:30:42,507 submission_checker1.py:3307 INFO] --- +[2025-03-09 01:30:42,507 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 01:30:43,691 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.402228 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407919 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2354 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a2fa28f5c..467921940 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -898 +899 diff --git a/summary.csv b/summary.csv index 0fd4fd94d..84e0d7001 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.402228,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407919,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2354,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c7ad7c3f9..e4612cc56 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.402228, + "Result":0.407919, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 0acc0e93f64b7e2bae82c3ed51029562a2edf76b..dfd6221e1fde3b57371056715f04333bfbd46e5e 100644 GIT binary patch delta 2991 zcmV;g3sCgIPQy;HbO;5*6%$NLlX(awe_wOjNEXH4_fx1;c}S|ppl(2aT(+kiXOb9W z>^Qck_Q^#+fNB6Sl5m`FKN4Up?m5UxHnZ~1=D~s1x!rwR`ng-`V;%hZFr66peiR4c z?9fy-%QXC1KO6+J(V=;B{eI6ijX3Ge2E9o*^AF8$er*1F^sj@jVf05l_Wi_2e=C^9 zhvqm*=KJ+}+#mbXUR(?3{wzH*45Mi;Nxw#;dOVN(-e4Itoz%5u+4X5Jn3<~@_MaoB` z^Tm zk{cp%wi*&wWyA*Oj|V5f8!ROQ*jNyB@%JoPl!Ye*gGN-!2U)g&27nhA`v`&AQHjT zMGq3IDcm-(PDHvL?TA_oJhpzc8Nrc?gf#E(Y+)RF}k0K#O4>A%`Z3` zDrm_I4kfl2*b|gAM85BcMCAJ$k%)ZXqr?^0!G}i)`GxqAv>;MBe=P`aKOTFWpw47L z2OIE23gxj>1s#5H&M)bZib$A4e`cGWrUp1oc!hT?QJGzyaO;!CX`PV3W2 z`g5ADbka|AKN{Wzv!NfQ>oxx_@#Ew!UVNDbaklh{@9tYQtDZ!@zg%S1v$a(HymPXS zkNh}Y!o>CUT4rxkfAd{`KHuxl7klAs_7E**NxGQYTdVH+;cOU;%wNobpY(!BY`!_D z56N`>sfv`+MXR57ZTa7hetll`ORbg|wWd}o2j&)a4yVpR>cqA77A218#tBM*IwFzO z5s9RZNW>NT!6~4QgbQd-3&PtN?ui`E^5KGu4u&_yK%P4ie@PM}dVGNrWr@+hbSgx0 zG`{Qgv-_F7^!8;{)!rxta{es|qyG5rUf1lJwbx&$^rq%{6st(31d2CK4#kFB^IUHS zi7qG6MH0mY^cE$KTyUWTNF)+TB9TZEiA0j<7LZ87C5Z*$Z43tz-2xIZ97uE}i6lnM z*cT{KmRw4?fBzg3t4O5;iMQ_7q}XU`r@3P(>Twi3L{UKW7A20j$WBlKC=!W8kw_$p zC~?J=`N1nd5yL&v&(nhNwkJS2lLZ~pfLw~om@dXBOSF8!G@{~5uUg!uy?z+)O?t~3 z9$fopuS=^)uLOcuUJgO8<|xn6*5g0LSyUE>aak^Ce-x0sB^pOu!zV-oJTi?0l4&H7 zMB|Ae`moV3CRqg%%AyL|n}ZuHp`^na5p(!CM0H7&eCT*VG|CcHU*`J%GT3~Y6f-Ud zzCTZ|$0+vi?p4iA=f&@)vyu!;;Ca37*osfPW?L$Eb!}70AG;qbe9YczNh(p}G$obf zaXnB;e>6ZP(EydGfr)aXp;Tf8NM%6Sp4C~|kne?+civQ;gUi!bR^xL89efK2YE%h@uUCY4vy zKY(Nv*_FDlY~}AOYmRPuyO7*r&r!sDBmFr<<4A)J(IAo~EILdBND_@Bkt}lQunGXl zBA<=|qYjjGNF%U60#RKOMUZ47c;Hi|Uq>SOcWjbWl3*z+^Y$IBYhKRVu?^*xe|yJz zf)c$SwzTgMjVt{-M1xS4aPTk-$XYz&Gcs{GVk0-sMRBm&Zkw>{Zs6-!#Ev-C6<4P|N(IAy2+&oMJR1ytP zS!C#86#$h*mL4S?H)l}NA&uaUFeNy5NEGf^GVc!TSXOM_K`MX#mgNgI^<1mGtH(}n zDtG+hhVphmiEa^FntF)Fm6JUX9RVSeTM!$6<*h#H8rE&PJFEoS;$&(xi`dfOLo}{5 z_z(?{$uyEorjcYajU-bS8GKj;lFXtC+8PW#P|_id0GSZgB~h@5B^q4B>WU3M{}nJR zNwUmBR_8rF=|Z+)d#xQiBU9AhPElY^%k|>-rsYuS5x?+>h_oDejo0gR%S!T}Ed6$oF)U>;6&r<6h zWTIolmM$Nnaiz*GAsSbcei0pi>5xXiO#o3{76oLM zvH9qVZ9dPD`JdVeD#@=Dlwk2u&0V*c%^gspUBs3aAEI%k#fNAR$`TeIrU594LkFhI zB8v}e1fVRk_-L^BKuL!*qAZ{iJU%oGRnB)3uv*u29h zy>Ip$aKqMfZ*ZD_`q9Wg@h6km=!fZWts*z2tJhjxsr}p48=#)#X?x*p5M+O{+MBH2 zSCs^_krDs7dJ~qNy$Q?8D(erg78Bn{zRmqZ(|?$!XDCb7GzJgD)jP0^c@%_EkbGOU zp1p)?G3gym!@=uirxvr|&xQZ`RsWT}|Eh1l`Nhz0j6)+!P5n`tX18o5`@p(&bTIFY z{PSKk3TCk}@rP-mR?RVuX!TaD<>w@vFF&v_*zXPeDEpTFz8i*N^7thi zlFi%6j{X9({2g7ebo8s!;5321JF)NPGEmh|AWn-DC;*z*l2=CkO~QCu6RScs)-v43U!o3(%o zLhiAGzyu2n?}4eEi0FV6d_vkE*9`!{F?66n2aoeC$0u(9r~K81K{<0O8Fm(~T94x7 z-g-G1`@YZmaj&YC5XZifHoFt>BAjMru%9K1VNlr^bZ_BU!$cz3HP#uQ+n|K6_xi3FK z^9QrDBFX{-evT_!pKfRmUeCL4x%<^&}H002+| l000pH0000000000004l2VUyV=J^{6p7$+kJXCwds0047<^QOcIPS zgaA`}MLD+oQDa*kNhajqk7OHTzvoCO+02Ta%>}Vr=k4ydrH5{*4?6huVLCDHgD4({ zvqMwWEYk>Py|6!?4G+zm>-T$}X~ao)*6&WjS#W563u5!vqkkQI4WmEeQ4l0Xe_FvT zJ~T&3GT*P)$&4`%6+K^RTDN%}P!*5i2;bo7=eL%c)PhrUad%?S~x0nXAWK~HNOu9+h{dhE<$L7&Nf1Do6@-d=daA+Q@{gdCdZPt$tmTljT zgRk+{XCq7HOBnunBTQdvkg_C&s z)0mFavDBt9?LHiu4b%8K?kA(HVq-M!_k(PpO{2GnlW_X`>X2GjunJYqI#yX~6YbZO zZ8^G`osVymU>#FmwYu!#yKd4wI*7urMzm}po_Dh|qxQ8RZvg+v+0!Pozl<;Yqt+qsIZ%&n?&O5O0J2- z-Id%BiMK1cB@%xl*-5v-(``TnN~DVl(a+D%Dz2YZ+&rteEmG`cYV%BOh}R-hRQMv( zXBF4aDsE6APij#eJDE0Vf2PMxJg4FseoG|cyq^$>7_fIlB7pskNbFt7dm<4$eIOFS z(?=o^SI{Yuh@o#0i8y~}DA8hc+e9Kp_nb(?=yr%ijP3=Ih|#?y5;3};h{WL+oWn0T z2P$aF3l1fY7}yh(G(^7dh(zT38p7(Z#^D+oS@ER zK^q(JMGEIKrh_qZ#<@U=vcxra`PA{j(Z_#Z{&v+q?VP<{E{5WCH8dP2qs5n6FPzq= zll12_UFoEs=0P;L8_xzol&;r;yCjH{yLjgVl~ zb$k@W=@KTcuh%krf18@`dh_{SZ@$<3AAJc-RW z2lWA&u0K_gQo3jj(ylH4%jwtWRln3~iBYq)Qd?l|Q0H>$T%=B1Yj08Fif){s1gIkt zNga_$>WD;Kp=>PWbN_Ou|ph2g%);cPoxaM8waTMXp6e=CtBF`~y8C{dOe{Y$q( zB!{EBZZEr^*-LLraVN`OQnkt7m{B#}rYiCzJTBwUhM5Z=OYAkixz5yOE*Pm)Mt z#EgA`5@pGyf0XynA+d^7N|1Q#?M#XdTf277Qq<=t`iP={=q*ZIagm*%1W+Uri6W6m z6j9=dEAxY2fFg$bqMxS);VoZ)awZGfqyf1Ul`$QRQI=@=f@ws>mwvUlO}o7?-kWrn zH9UCE&t8{SkzNS|uly|p{hF(MS6h$&6lYOc9L8n2f1pu7@|I{^aSfjk4e-b`5=f?z zL=ufJhUmja!DrhOWv2X?@ZPth@sgzM25`_~c70DTMD}k4Y z<<@kWajC7E`T;Pj$g#}4x$c-+KYOluu9I^Ysrc5@vBKS)rX#^bjoXx9q6Wbv8UT}M z08G>%n7Rl|tN_6*sGzku_yndbX|qNEOc~W7e^KyIm1vaZN-p*P0hd+eSmrtFx?`Jr z%4VbKFG|FlAJSL?M`DXB)w6{7-?YerP3IMUJJwZ0vw(TUu)`2TZ=Ff2?q;yy;0WQRBK2 ze@xUMm_!3$5)FV!w)X){y9i8D0f1RhL2GmH2~1hiCXK*zR*34bC;$^R1WfxygDIZ6 zem|2{q*sbbdxuHw=&c7fyGZ7kPZYWdn!ZF5HLfR-L=7TIG=L=00FtQjl|-^xgd}N1 zdIE|nXl)KYAt_7Rte3t-Qbu)16uG-(e`L8NHn;C9Yp%BaJS21MIf|HXq(6sfTxrlD8bq>$MTcnsNuqHjl0_~ZRskSc zqsR3j!lwE5-dgKwtYwIns4VVU3c6&f7TO} z=>2e{eTQf~>E9t5gtCN#hiL#xq5&w2OgyXtfU?NOqom8`3`*Lh5rh)lIwT60uFS6k zOV^4GJAXfuFVx2KS#A4x(i=+U*?C4D<>gR`J`hJ*d5Ff7ULK-BDoeO|mW5l5POh{ltXKM)-OCX-zd8-HzEebP0oXL~tT0&Q_JHJU{nY49N$Pa1rP z2FPR@NhZ@sGMPq_sf!FgtO7}9Q3Wjx1|KMClSY6{i0Y6iSi}+yE@E}X2A}^5n3W`1 zW+AJ$JwE9|w&D2AoIe57Du+z;i#XEcLo}W=`4A0~S;FMQG(aY?&uB?zk;#WO0)J!{ znS3;ue4wPwdMURoA*w^7aL%M6IcMsMO+Md?<_~RJR+40yIa6=Dd~~CxJ(WkM&LI;W zBaU?W5RE5YK173LmT>tn4UkFvGeBmM%ZD`rWEQ!6HV2=`1eXu%C6Ea&9}CoMiigHV>R_%IDXNgO&bT^3n< zSR(*sk;O-Y#Rp2-q!DESmEiHAVX%^AB3Q{*Z1Q;y%Vdxdseb5TY$dssTEK2QeA4@7 z-vu{ped`TQf72iu1}DK}5*xiR9j;a6rgZgMt1GpCyLtoElRRxNob|`qpR9H#tM^qU z!J!#E%+oWJC2Jb}hr#L{SjIdWhtW9swrV|l z3D;uMJ(`C7*UL^VX5&8>!RuGOSI+*cp7Z7xL%%T&OpP@4M`@bfvX$%u>(V z0v$ZgvmBqj0i5zT8wTaVsbuIa+_av=$-Vb-GIm{;b>(1<`E&fadU+dGO{G4ZiUVpY z!6|pZf^kSqg)fVExR&{_iYMMk5RZM-Z@i6*9Q!~dF#tWrJt-gM)AM?Xnt~Lwl$0r% zuL+^EIW5T7VdPc+8K}k~|0bRQ$K2PSp!ox{wIa#_1#um3W=WHTC0haIli(#h0dJEX zCP4uVlV2t}0e+L3CMFw8EPk#f0RR9{0ssIJ00000000000001hfnSr`CO!ePlNu)@ K24y4w0002=Vvx)L diff --git a/summary_results.json b/summary_results.json index 9c25e3cbb..8f79d0be8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.402228, + "Performance_Result": 0.407919, "Performance_Units": "Samples/s" }, { From d843e3e4fe4fa494df7bc2511eb2ac915bfa22dc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 01:40:46 +0000 Subject: [PATCH 272/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 10051 -> 10050 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 6d79a3c1f..ae6d42bfa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 01:30:42,506 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 01:30:42,506 submission_checker1.py:3270 INFO] --- -[2025-03-09 01:30:42,506 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 01:40:41,710 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 01:40:41,710 submission_checker1.py:3270 INFO] --- +[2025-03-09 01:40:41,710 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 01:30:42,507 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 01:40:41,710 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 01:30:42,507 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 01:40:41,710 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 01:30:42,507 submission_checker1.py:3286 INFO] --- -[2025-03-09 01:30:42,507 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 01:30:42,507 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 01:30:42,507 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 01:30:42,507 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 01:30:42,507 submission_checker1.py:3307 INFO] --- -[2025-03-09 01:30:42,507 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 01:30:43,691 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 01:40:41,710 submission_checker1.py:3286 INFO] --- +[2025-03-09 01:40:41,710 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 01:40:41,710 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 01:40:41,710 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 01:40:41,710 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 01:40:41,711 submission_checker1.py:3307 INFO] --- +[2025-03-09 01:40:41,711 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 01:40:42,644 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,13 +32,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407919 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2354 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99223 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407143 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.429845 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43378 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418495 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 467921940..4b02a197f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -899 +900 diff --git a/summary.csv b/summary.csv index 84e0d7001..c236df72b 100644 --- a/summary.csv +++ b/summary.csv @@ -10,13 +10,13 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407919,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2354,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99223,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407143,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.429845,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43378,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418495,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e4612cc56..d39ca5f99 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2354, - "Accuracy":"acc: 75.400", + "Result":9.99223, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -484,7 +484,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.429845, + "Result":0.43378, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index dfd6221e1fde3b57371056715f04333bfbd46e5e..6a12d85626b04cf4795729cafa7aafe0e4182b2d 100644 GIT binary patch delta 5277 zcmX|_cQ72>_x~50D6xxG!V+b(i$_+8Xc1+_>b*qoU7`zuWwl3-va1sY|ar;FE<4D0`lCA87c|=!AUS>J(bZ9d^2QvW$@yuLn z9(-OfWri6@;KIT}&;&qHo}jV7k*VZ|B-#3WBj|BO^j50?3_@xPE81BN8C_MzD&D^* zM{fAJ-28qap|;YopnZGPNr_4b^1$|s!5BBpa8(4^WU;Ap*=xvLQX1E=CSG4-dz_&@ zcl0gJPtuj#(6i#}3;HD`_K>k$3x0Vm-$mbCv!n?hXTw6x*HR^x^qF<6X)d=Z(<6A+ zewPSWXD2TSFVkq-?cPRiI&Y76VNIJeNd0FzbQNl4D8=(@V$wFrTw}gO5&l(}9X|m_ zYAd{yNotcDJmT=qclx1KFWh=3f09Nj@}MCyJ#J%3%X^f&&Fha*h<;!!3iYvv3<&oR zL4Dehlw-H>c*8+0&@xOD*>I-`#lHB{97e5q_5e6O_piw|^<7<`5ERPG6-!N4@{5<4 zXzHUZdv5bF8Bq6_C>Xy=_yQ{7-woot zP{)@|5rK8S^@bx=OP|m_qqpmtgR_joNY*Gq0FQoM>3`YZSY9}v*jza{I;(z8+qgrU zqFd1~_sFI+ne#w2SrQxcpXdDfh?mgn*vKy)w8E|KC{fc#p<7L9n zRo1g)s%NC6SkC59k)0&=b-neVe^+j>bFgdAXZn2os!5Y%8S4)k!gMaCA}KBe4G8Ho4C9%KwO6cR*UZwOb8AYwn{R^%?f zJE|O3>W4EHBE>60cpja-g-4Js1lCria#6o^O>2k|^jUP*a5j zNC%(+0LTCUfHWYJMVVQF0svUV2as`Kzk7aFWawHruF@<$NYH&P^F2t)q`sKN9>-|C zBUWF0)F+3+OaiRnOJdiTib3xi*A?{$>B&Cctb(KB$KK|7=FNptW9?N`HKp#m5o7Q) zwu~~n^l<52($GH4ONV)rwgpeM`_cZtWn7_ym$JVz+0xrsFSi$8P+!r!4v-eX;()(5 zMp=6Ht=i1u&#bZ2G0)pS=Wr>6%ZP1v_072H#xvwCFjs`wO*GddTcozt*4D&30u4A7 z1X$G1u3B>2vQB@$dISGc{85Z4P$(M-`@LZypcGel$e1TBArwD#)wkR1I2{kx3s_LU zsM1=iNlZCFkX+%wCmr)1Q1e^n(r5TFF?Dt02cS*T35Ubt zhneto?ac0qV>bs8)gRk7)bSIY-LC{0Pd1}OUSA1DhtBS9%Ra|j+$-Brs~}8@d@V06 zvfb7@^T%YKWu~1q&rFdm94^m~$uMGsKR=@y%TK)$d~UwIQu}9OTEo#3D_(zAw6yiU zU2#isWd2g?M{Zya=RCL6K*EbzxBJX4k3feu5_2++QrsZxYVRJmf1~YGDDjFrrhUSR zVl6mkn7a?60ln;uc(N+;stKf+i*72R{&8_pv3zFOHB=Kl2kTNVR5Np=U9Kykqt7n+ zW7u_A6V0sK%T~wTqh831HQIE)gVk{dwILw;EjiRtVwSs+XuvshqL2?_HwwM?Cs+}v z!4`!!kts&Q)B`dVJ9mA;A-uyb5sIC&l3WmX?-n#{_}8t($Qpko`0`F543cT?CepU( zq&SMZ%cAV$Zl$3)PJP&VAELGEiiT|k#6$E=NZD8sEBu}D1z1w8en**C=lfrN0KGVw z#lHk|DT}2(G>uJsG<-`r2~pWJP#jDZFLiZ(&`>_?c)WA?{2?8pJZ}un%HKA5)?Oi% z$%!4tC95AeE=6>-^kFt4`uv94VlKH`V^YlR@Z6?nXrJSE8PDQn7tDDHTbo8@f%A`N zJ0k*UgcdXV_lIR56K$X3tT)bv(;I5#DIun#BbkW)q)Na zmO^5{;i6qp50zaB58?i)L)oYB~|Kf=wWdX;_DB3@ZGW(m98|NgOZ zR`^9#n{J=l&~I)%EoE{#*DT6tP{Uf285nOwiNbOh0;3v!4HtZIcwU@dLw6_^&6>~6 z?iT}5FtOp{CZdZ%=PSY?a!HNL{TGY$xsglRjO#ZN&(Y>)^@W2sQG9Z?s&}oE5Pq!| z(8QDRa)t)ODmJwL#5_+&=!@$eFGu>-WYk{&0Q<%i%j)B=eis_<49Q)VCUtk9x8SZ( z*lwDcJ@PR?yz2d68>kX|?A0<|z?bKYcw}%Gfm_Dk-z^4F`_h2R3M?H_!WV_3y2l?W zZwNcIwY%UC9V%9OxazTZ9<$;{dH?#C+0-{1vkMpT2c^}(?3v8qcc-;4>I((+XZVsY z-k-n!+_Up8r2rzVuA{BQIzy6-Ceej}u$ocOH0}ndGJ+Gua|cqJ{LMftGzGlfnW)=; z6Yn!QK6)iK{hvjIReCy~4qx3f9ch3ay>Lpps3?g}ru~TU%zMvYF!}{QalOXO2?8Rm z#N7y@I?0}fCb58kf})^6ox8zNIjUSA4OI!A{X$gZ_&B?Nc0&)Nac!`q=R1&W zk>BkO|9?oCzf8Q-E=k%DOhAbkP{V$s`5w9I>^R4pPo8>_TLkW#kifyUnAf;lScJ z$^k_a!sH?x0ZRGqaGV_9y%azK@?eu&U<;)Z@sjxbKJ^qr3#h~yLMrtPd*%hlVS6~0 z5VunD1z#+THLuxzxYwfjtI?-|i+k+cQso>x);QiKD@0~XPD(WaNJ5inGNivUAPyq* zcE2qcSfex&FQ5&Bd=rOLq*e}Ak*Lf>gPvfe@%)qBzYT2zB>F&w9+|%KtgCC9QBu^# zM!u2V$!4tiGCV4czG=4^O5o63Lhon?1duG3!iu;jYCn1-ARa``f~_d;QFr0-yjluB z@W@3sMDnUhqJJpsQ;o4FAuy7vB$^Nq%FyPgB{`rU<{3}yLsk1yk*krP_%<6vZo0}A zDOF|_KjNu-mGveub?c9oVYvn&+$+;1E*WKla%K9OL$Jg!d&r&^R~ zvjq#KQep2yDvx5PE%t&_z?;C{u)j?wha6jk>FGP|omK@IDka)75X86ll^6)1xfiQD zu@1?yoM&=Qjt3l2{R~Xx?LMUCfhCk3?+`Qy!&9jMY#7opLTQ-XLRs-(UX{o#9Cy~E zQpMr)W-I*2asH`nW=b-hq22JLVQRd~WNl`HeW@1~oW8;MzHlJI!DRCW3$FxpQAU-yS%>?yNssJrhhFcaX?x)y|Xj7?? z;M1IDa-Pzg)?M7w8v5{L&<>om!SKGY<8NAf9MCXASjSDPDO-(>LHc14RNKH@q{oSa z+sWTv%SYxykWxaF3(iOK_neibm(4s?JT*}~&{FN_1i`rV^!@~%r zJiVcZuiY-S-%YKH8B|Esss2!*?rGoIpZl^4x)8Wbxd;%Y3_00X@l&qn=X!E>r)C|J zMg(^LYNqo1X8yc+_yOnucvemZ{98I2?4ohBKr%5dpYtvS?%fV8Vn<{@&e@kYodetE z!Oql8P$Y0@&=tRv+S1EOX})&G7dn6t8U}%1+l#)-Bt#&U52hT~e;s|>dlGpkLqW*) zah5l0uum)KYQvK+!6f7N#BeBeSCsAk9+6D8Qp zsazqV?M!<-^lCzCj`C4Uvnuz=GP;qok+`Dpw*3?}W)qhNOE# zj(P>7R%x+DFr2Rpp5}~rtsgCjeZNr=4cV_dT}>#gxB$mf7}gC!IC7n!!_z`an}SNT zd3=5=!~{-1-jS!XziVwc+=l6{yK*S38J|)h=(l=wPhSmN?c-CFwX4!KMcsQip1-40 zGE5fIC1wdu7<&*HKCVFPsXABf%>2SFzr96GB0r?RIho6HisyYLUj%Xhn}$r7>Ieyc z`h1j|BV1dVI<~qbc3I1g2V+nXwW(Z8)2XX>0JHJUs;CI1$2bvWqs4lk@fH?5xL#IMM<9D+B z3mL2A9V9OOBPB+D!W!%;v4-S|{JGB0M~~281OGBFZ%?0x-CVt0MfS9@qqHb>&6!6e zGk3f{6#|KCwymi0<}!DGY@b?*XY;Pv>rV{0fBZV#^2@450dt)CqUTS*`1xx3Nzsob zvvT_#>Ir#=u1D7qs+W2Nkoup{;gn1(D(BfSp=@>y!LQM!!0b?MVJspNUd$VgNXnN( zl|Q|hARd_0Oo^vKxSLEh6us9n8dkrS1pRVwE3EKtEN%|~RXX{`SR*u^m`u!67gbpn zR{eBS`(QxJKiwH4yl?fNJJzWpv=2d;Me2<@IkX(v*f=)IF<5Fz96n50&(s!)hLaTD zC|mV9@111TB=(3!FYOOfEiv%RE_0blN8(t*{-w+HukQ5e% z7fxqeKl2l>uC3?plo#EvkCGPsAWG9(buix-Y0S6tLy$Ev=|k#HL*|Ee`1Gs}#>+xX zppSG!J>UpaL@hOV6f6zZxXkK1t)gwj;oG2)KEzG1gJbEi@07#PYeg19sJ#M03 zZ$ulcdx6Z@A6Qf#qh|=K^03ExTCXe$T6GJD-vr7%t)}xtpe*D`9Fb&twJZ zS;)khOn8}Ey1FKD6#bad zrE9=UmSjL;=qL-`lLLw6C98lZ0}jy$cr@x8(tpmVQ&x$Z+-o%(qgvk>iwzT!{XILIz8}$E zGSPEMgW3nxoR1mb(2Dc7o+aMB>81Q$*|DOc~ VTj1w7rHE?rP_Fy8?cjem{tvgl^s@i} delta 5318 zcmZ8lbyO6_Z@xQqr|m=ghw{NEPyN5*56KQ8uAVPn^WTaZq&Kt4$P`*i*`FDrxGDeZY%~0~a2lBs!`E%uFhxj( zMMsKH=FPA5*YBI3r^--EwMl{$pJ!Q+y7iQ&VGS8KjH)O}X?>!mSUA19TG z=K3ER1#7=5t#s(^usshnXr^(Py zYG}8yfem-hfanTvgalqKt)@^1#^iT80_8<3I1s()OMrxgiya|$!679&+GL{~4Ry$L zJ9An-1)y4djFwWl`YAdZii(~f`skmLPDj9b)^qZN|dF(lVN$oOQNZZm~s6X2et zbe1eBrQOC{^{w-#X$k*x^TJg`?BVhd#^3n6Zmw(|$axc}&Kheui)UfGnS48Ff)A%D zQ)pY0(dh{6{Sd?sTs3YjHc5A!BtGs`Ewj<}C;V7Tow&#IbAJ~$XcpMNn5J6{BMdAm zG4JKfdNYaBU!>lI)7J(&*{Er(S=2eY4~r&osOM=;A|#YdaX=cHclK<#K9C9a`b74O z>^0-BrSLfj4hAh8o_XIa7S%G=i@`bTn>dBIaig5&u}V*|fuVSQ5~H*PEJ8qZndxO( zp+G#D+5n%~;H-crs*8S1LOpe(f;9WO9jq)z`VVlH`r)2keDLMSGU)qXXYZrqfRdqZ z_3izlC%A1fc)Pw|UHv68_6V@mDAE0jq-hMz#E@l;&=Pg zZFx-4h>eFBEd5G4Zz6g*O`G>@>cW6&Z{r82%!t$}}Oa#ub9V?9mw#zk2@=Dwik z-r)B(?uFFRO4uqDDZR95lV9I~CD*wF8feO03zN-}vRqxqo}Ru!al*F0I+Sml;=DS; z#G5hoobiSKR_dQXAfeIT2cN~R_|U-!*53Jw(JN+U=o26@006)S-~s^Rfc6(eiclN? zU^UJUn->1bX-=M`bNN!WK+QwGs4K%(0?u^z(s0P5wo%jD)Wx;7zQ3nw>bv zko{1wM}fcUbZD#9Z>nz5Tlcz7ckeUh?KSsz(n}v=tkr`NHNJk{TW#@Iz^c(q52rhy zVZI1~-9gMohyW6Q(9XyD-t(+1pv>q@PUPaQLmtkS)ET@b0NJ`Z_gtX988^^-wtdeo z%JV$if(z0to9!tQb-QFp#`}Gu+GO_Jf-5PxnEWD}$0e}cXykJF>s{x>55{5zI`|lL z?67;?kSbLUD}Y8caq~0y{&+-0p#9mBsap7T{mNcNi#)PN_UOLALAiOD)qqW`KiW#$ zp8lDw6UO;3@zf6%ORsUg%WH1zzlNGZo#W&WYMHFS%yKI-Y;~7BHlEh)^RyHLd}uaF(H|DLE3lx@Nasf0cX&fGaoNb?8zFmg!;q&ION(XrPO&$$w{1i38N~LI(MJ% zf%9a2gVVq(8PlQW3~_X*%U7HZZfe$pO;%N_-8DE>Mn>HV=(ly8pX->~slt)^aIxk6 z;g?@ltuF{TOd<72bGpTc*Q?f~;RBxr{C=GIL#&$w3-EiL?+QcX$7G6sxZ0qWBYnIL z&Y>lVq}6#EtXNwlE_nGvl6xK17bJ|4J6%r|e1+&@CI}HZk@6icu3KMcL@T3eNJjJcx_sbAh4nKHzl*=_2*)u;PrLOTZtS5VeO>IWy*#`E{RP z?>P{Qf>k8W9{K$4`Jjx>0z!aeWTorSzcAJU`up8e!hdN*ce&|>*ApueU8o@Ch;%kb#bx_GCy8cDE& zHm5mpbOR%tTu!f@8T>?qCjuC2!4LypN6=oEk8(3t^~m@ zka({hcS+tpkou>s>j$10uN309oLEibN@d+F7e|NcgYUOHkMgQwpF>wX8V_eemHwFO zvbwq0sP<`I>FCdAa{h;R$hNN)Mj!5-&b2E(qwDUrF-If)@jh?fYL1{)J~L= zZDG)27Nj^K!10vHl0Y#FjT?d40>p&>QNu&dfW+z}ORv17J|;SAafu>w{?T!C>jjs> zD%CmS!nk(Terrj;_EkL0@1NyKWlGQ-7x#s|m`$iJv)(*nvx>C!{1d%kG_2C$MSmv< zUY8HR8>NY5WEWC=A`gn7k70M*Eyn@$(*1};&&G>)mZvWlM}JgNCK^WPzM=}x{Uk?+ zaoG)YJ>ufMXtZZLRtECJMDV-CDcvv4CzWJrOJ|IfnIqgVP-Ip z`&RbMwX*_P!UjZf7DWF8*n~~$B~Pr~oLF&H5GpD*tb%fKl{J7DjmvG8i0zV3+0+aF z1kSC&EP2Cb##W?xOL?ydaHD&9V-@paU_22FsxQgBv@l`^9yb^HbFqJrfMkM6MwRjq zMsmH5y--}ShZo#zkjeomjcrk}U8x$QFC~=1^g4fOi0F(y56F4BDnc6CU59 zY$tyGu^RrkPYKKV+>yvho#l|*A8Z~y0OF+1L*qffF?-k@QRT9}19abpw^oRfS5A!@ zfjq67(pmOH?CYn!=MU7+x1iOi9h+1o)i1VA-SBqm#TiK^9a(D&BrUn-oL2~DFS`-a zh@2+Q+>lQfhJ1`mvJ5{pF8?FHXxWzc02O@_3|L|XYA4i2Y&Zt>W9}(yZy9mC;fczi zer?hqrE->jM(L(Z6Xv*?o#0*j z61sP}#nU?ag?ICl6dCCo7LWG{0SsO_m-=%_GXYo?L-pp%N3DL5A z)8L)-I6Nuej0HNyi;3SpDO?^7_aF{biT2X1PHzmfdb3OUzP)wdB%h>eW%%pEwfaLs z<3o{MGpcjME?k=a{hdmRX{|NmO-sFtGxt%%Ez*D&$F$^K*@o&25G> zF7ax`b!Za9>eKT zQ;AB**7f4_1 zeufgARHHV1bAR6AMd4^y_kd_KtjUL_cpwRS0nRAU2uk-!_K~@4IlaEjwXu&>UM8jDsy|2e%)7W9&@WHO+kF#xNP1w!B%UXDee_a20sxm|6?MU46+d4 zSWcuG{7t3jtC{Jc*&IAkp=*Uv6(Y;=l`0HmEbkRHJ#C)b{PoC%*K8NGbyV=mTpKmp z6*%I`${fIOAAi&<9*S9!fi7-HEt3g#bQZ0$yR0tw zQ&_+0DmsQF6r##b?7f0GSM-=jN6s#&1YHY&zumSh4~Da3bycXf!d$c5{}TQ5^f+uc zpALR;1Nw_sa<^DUrQ^ykDEYy>KJ(Pso6IBxvS4VZaH=?rG`?+A2}-gx?AP8lb{1`Z zr4Q{;rD_Q1KY@u+EGX`yy5dyT2?Bkl4}%tIjyL9zC&#mj8>A!z)A)L8^S1AlKh4jh z$rVXZ+7E3;1|%l$yjHGba8BqD_b8q|_);bC^6mjpg-d3H4Gi+w0>(Mgvcu7U8nFt# zo>?G}jM@VGmsFh&@k(5aWvX`;;~IO2bx+gg@A`EqHtOGQ{gGokTp2h{VzH!cwz=}e$cQe}o@^2@D zBn6#V$y`ZFb)wcbmOKT?ZkmR#9m~6J1r0b1mV?z*3orB6d{Y$c`zXr|^;B1pbX29E z#n{&n)1Z!V&OOZ~ys&U@npjfbNQbCCGt_NmT6P{yr@lg8ya*@c11VI>uD|TAC+wa` znKI}$)g2gK$0aU_#H#v57l{6UeiJ1bMJfI_lUD=)NFNjY|0%wHrg6!P=Kp<+V=`%? z;l&v+38DGMB{MNm{F9my4FFI^2LK5FXUJpi>t_@<$0UYU9Y@AYk7>s6uj{`6i)a;* diff --git a/summary_results.json b/summary_results.json index 8f79d0be8..88d37618d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2354, + "Performance_Result": 9.99223, "Performance_Units": "Samples/s" }, { @@ -653,7 +653,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.429845, + "Performance_Result": 0.43378, "Performance_Units": "Samples/s" }, { From d039d2ed4e353995183aed5b27701c1bce3a1263 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 01:42:09 +0000 Subject: [PATCH 273/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10050 -> 10047 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ae6d42bfa..cc228d1ee 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 01:40:41,710 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 01:40:41,710 submission_checker1.py:3270 INFO] --- -[2025-03-09 01:40:41,710 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 01:42:04,200 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 01:42:04,200 submission_checker1.py:3270 INFO] --- +[2025-03-09 01:42:04,200 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 01:40:41,710 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 01:42:04,200 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 01:40:41,710 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 01:42:04,200 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 01:40:41,710 submission_checker1.py:3286 INFO] --- -[2025-03-09 01:40:41,710 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 01:40:41,710 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 01:40:41,710 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 01:40:41,710 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 01:40:41,711 submission_checker1.py:3307 INFO] --- -[2025-03-09 01:40:41,711 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 01:40:42,644 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 01:42:04,200 submission_checker1.py:3286 INFO] --- +[2025-03-09 01:42:04,200 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 01:42:04,200 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 01:42:04,200 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 01:42:04,200 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 01:42:04,200 submission_checker1.py:3307 INFO] --- +[2025-03-09 01:42:04,200 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 01:42:05,320 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407919 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99223 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3089 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4b02a197f..17ec7611f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -900 +901 diff --git a/summary.csv b/summary.csv index c236df72b..d3ab42e9a 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407919,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99223,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3089,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d39ca5f99..fdde2c65e 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.99223, - "Accuracy":"acc: 75.800", + "Result":10.3089, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 6a12d85626b04cf4795729cafa7aafe0e4182b2d..a7559c5eaea3189e0104e326f5671fb275ab574a 100644 GIT binary patch delta 2451 zcmV;E32gSlPQOmD%>xNTQLV#q1ONbflhy+xf1@}Qe&1h#P#;=p5-v#yjgpL}>AmSK z&2F_%5djmd0nA|2%fBBRl1_7cni=g&rF}`jeBZg@&pvqg^(9MR^ObP(>-ewXGb_LIgb-m!134d=i4euWvUr$~EZ}Wc ze~?^0iCLKiLf)1uix(vcqMDSYmTTLdl?6068w-Q-Pi(L_rXe|IVU>|wG)7922116< z6IyU{JdCJ(rt&dL5|7Mp&S3KBo7Q+(yZ(ogXa4Tm5W(_+q$Ct1k_9jgmD!T9@9N=& z9I_SF(zvH}RClEjkvOPQF=Nl?gsu`He~t7MC1feDe~paF(z>RR1urABBR$YaB+9Un z&?q8#?KZ5qVA*5y$XOd`sE)VhI=1u1K5RQ(+jD#B^}MEJ&9NG#QO>}%WtBoTdas66Y}R^Uwiz&F9;%p6knafs0r{RH5HOq0HV|YXI!EYW8eQz5Uv8dl*7GYkFx4G&L9MPK z=LX?`xq=|TjJx|4c04}+_wHmiy|}sBuiFGJ`-Bx0NwsW;EVHsy{${cVe~?!NDdPpr zV^YdahAaf(V!^9rM!9MV_~NP8wk=T-Qnw$L>N2eB>153>375M$w^qp_2vu9rl3he& zsudPht9TWx43ou^+xFUaYrXp~LYBvLWqvUuA_AIn^I&Mj$mVwc89B**j>ypJzYpS? zHE}u3hTQJ-ef!<2wdD&&cTyMEB6&5}oaIeedvtt41fHff7KmyVW$K+H#!F_ zB)5oF*cSPptGa4g<)UiI4{2rM#rHlIVE4ZW@H5i;0|CzccLH?ewrBhB)5d!DEuE2a zMJ6Ooxe+p1u!i|I+MGAcoWZ>L7XJlTAM<%c)yfv6&B=kF`O4rwnj;EtFnM%sWh{(m zRZ0x;TF6g&@>0lmf249Xjp!wAP9uz>q^zXkwehY_N2)X!XDr&UgQ{}+qayoz;hs0x z3%!FchI?R)44L@`DzmG-)W-!#SmR+4tjKjxu4vAUl*BSoyDitMvN_+VuLUdWE7`0s znGh`7+$AK4NU84R*KN$0*xaa+yd4{je*#cT0|XQR000O8vvddx3m=%nzmuvHrC)o zgLj*FOa+y7D^J3zRbpLT4p^3dF$q=AvP+BUJ#ZQZ8u!qfahYj|1O zG`u9G#h^r>s8)uaewltn_rUV{_or_VCv@+XV!Rrn(d6#|@_nz1JjWP+uo947K-<)y z?TH+)I;wSgb9XVqC&qb)LzisGq{u{__DvQZ=%C{0XPdTk7C?Rlktw&$qA#IZC*`eDCs zsB#TQMkB`eyZt*eRZG`Zj_$2wzh5meK*0-Ia+t5N8K}35M*9DcZ7-~vj@Tyt-0JUjK-^O-{UFxv<6`1!(njqdo$ zI|}1*bNVLjeQJ<@M^O7ahJj6b~ZUdPH4KJ@^pO}Wr#`KU~fwDOwaRT{Jl|zm~LRpS?6MOJ--}X zOT<$iyo1Z}WIh|*OzZi^lT%d?gnrlVhTS0W^|nI%O5-SrUM4yLe3O|bCL5Fa(-9>B002+|000pH0000000000004l2U6a}- RJ^`_l87CtKVq<% delta 2447 zcmV;A32^qmPQp&G%>xQxlW)Ft1ONbp5|e%i8I$k>34h`^6n@`d5k-AyrGZ?45D+k$ z(tGKRvRmyF3W)>OBylEodinQbC%}~BQ)aX;m9`HwalY@|;?F+u;OlFe7|*2Olx0KH zY1*bivXDhITMo^?XJ-xHG`I+|C`edFhUOdL=GW0*gBMnO=PN>lAq`}FXs(3F`^i(?v+6Bd>!$wY0W zAW0x(_qFNgFtcvQsFd`BMWg@2R<$}=VN`FWr9YqOQ!mD2+rLwfHX{5pH&}>N$ zG!iRi*jUjhB3b1&EV*FmQ~k(U8>p#{x8^#wvt{?1PTTg}j(RIcWqgvP>nvQVHKOT9+++h44H>2=JTjEu%4CSHR2IcX72_p>22%M z#eZG8ng@H=!&FDKK@2*gvx}JN2yctniL!-9+0yt9h*C#%L6pBm%=RI6qHN<)wl%&B zqSO&Si1N3HnU3h;QFhQMzjgFlf>(ZwKtR362m~a1fE!bE`mkyfxa<>_RIJKHGi0fiCh|9xJ%50_ z%1IH=X%>@0b~0ox2p4l+E>g->Q^4oXou+Mxf{?2HuvC{}-Au-7enGh0&AGK)&4W<2 z6%E-%)TSC?UN-VK!HRG-e|DQ*({8MH|9Qx=m@dsPW<*3l6K)<2tQgtc9zG){+0PLf zTJ`UPxTbYnPQ4*FTV3CNx9aVPu78V*uB(dh-4zUW#d*A&ZitaOV`UBBDc6UT)J8j?VeX!N} zxaxdeb)Z2w)*&sh(2fxZR2>3=szV@9bqECP%;)|d)gg3{?d2Z&l@C=12Y)8IgEs7x zAm>)+fQ94^u?pKF|8rGWF3L=l4f!FhOuYKu#{z8s7Xf}odVe6mng33Jy=JfHx^8d# z2ZU(7`>IY!u_R-XB-{v@EM0A5J6fMSOr8F${wn_kSD*D+MAaG>B=s4BpxM&kKkCB@ zuRnfrZDlNsr)5G6@s`Vvdw=qp%eSSnHjU^tuFoZmyr8V0;;r_s&PmE77^N&atb!^t z`lBR=2jPL&KM1{}FNS+$3=LU>4OC`Vd8rN#kg!IBJXn&Opjgt38wrVJqIOfRT19=* zQC$m`S68xaUoat9y184CAR>jjli#^9V`6ioO0qpb8vO)NO9KQH0F!ei7_)>3_X`DW z+Fna)lYkU3f8bk+FqHz5q9zv#lCA6$CG0lwj%|4@nIV6_av&MTvPiNaPZX!EKI&GV zmb(9Klci`4B~Ms>+QFvXL6CbLnnyHcIh=MLfp@+Q{^)ZqkX)Mc z)6Rns#ffF{@BuQ)O;$iIEio%ID&*gCW$~f}8u14Jf03mYv2EANXp(nO$f{hNc8D)o zsyz9(g5egycF-C0c{1pW!4-=j?OS5dw`yPu&O^v)nXo2)mnVWZ!5Q!>6-_ivX-0|R z*-Ik`n#L^Ih?1s;zwwq$Te+f&vzs_hlN_4xE&BmEx0D^tX;!4rVwj9tB9eJQlM(s? z=E_7ef3fJSiW2d)`Q?zMto#m-3R$(x5+R^$8qU@Z)5R9;>jN(yd zDtuV1iRntG5yVth@*-q;oUE*a6?|;6Ly9=aRV}5YD22K6xGVm(=W5H=pI;J|Nit0(hHn>r5LY9XgvLUh&<2jBDZG@ ze_07gFQ9E|(DptYX~+nev> z>-l0jo-O8QSJTUJz1r5h?qah?eD{}Y$Mk*D`{kMVrsLsWt7q&G(?)2;GyjK7{Z^zB zw(T0umYZS^ZP|o0o*+*$+KQQZ|Yt1Eq=SO0SKftv#>QfA$(Exk zrX#kAKR3KL5x%eT-pPntF6`qn9)6U(m+ZHjyul>a*;9OVU^p8id!HKQe-YIFj$z=~a&KW1aZQ9<>Qk4=-DfRB!hRn^Ei*z( zt9xzKc+K8MQ{{4tNe+=RHUBjkP0r8-oXZ3Qn%&-`o*gIw?wz?B`?ILgR@4X9qMayS z`vj|Ap8GJ`^IfnVf@qCAP zw$t_1*Y Date: Sun, 9 Mar 2025 01:45:53 +0000 Subject: [PATCH 274/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10047 -> 10047 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index cc228d1ee..18ed5fe90 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 01:42:04,200 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 01:42:04,200 submission_checker1.py:3270 INFO] --- -[2025-03-09 01:42:04,200 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 01:45:48,220 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 01:45:48,220 submission_checker1.py:3270 INFO] --- +[2025-03-09 01:45:48,220 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 01:42:04,200 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 01:45:48,220 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 01:42:04,200 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 01:45:48,220 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 01:42:04,200 submission_checker1.py:3286 INFO] --- -[2025-03-09 01:42:04,200 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 01:42:04,200 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 01:42:04,200 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 01:42:04,200 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 01:42:04,200 submission_checker1.py:3307 INFO] --- -[2025-03-09 01:42:04,200 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 01:42:05,320 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 01:45:48,220 submission_checker1.py:3286 INFO] --- +[2025-03-09 01:45:48,220 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 01:45:48,220 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 01:45:48,220 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 01:45:48,220 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 01:45:48,220 submission_checker1.py:3307 INFO] --- +[2025-03-09 01:45:48,220 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 01:45:49,285 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72494 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421125 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.267 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25029 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 17ec7611f..ed8cc2449 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -901 +902 diff --git a/summary.csv b/summary.csv index d3ab42e9a..73c8cae94 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72494,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421125,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.267,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25029,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fdde2c65e..e6d6c6713 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.267, + "Result":6.25029, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a7559c5eaea3189e0104e326f5671fb275ab574a..23a4b14d8e53f77d5d781e550674c16db763ceca 100644 GIT binary patch delta 3017 zcmV;)3pVt>PQOmDbO;Ho7`b~!3IG7&lYIy>f1ody?J38ZB*qv!j_s*^auE<8HGmjN zIL?1R5@0OuImk*jv+~Z)Lk3#s-tH^(OSjY~I{5WrIx+5pC?1EiLsQi((+Fn0us@y+ z56zqF_j{gc#7TG7?@q#5aAN{*Xwa_6imBu ze=VE`v-HRyjHcZr{TdDH@jME;{bkT}QrDK{)TiC?%v@#IkILi-2ZM1hco+5-(_ogY zl0?Czo20{!N8@>H9v$?@>9MRHBMJtG=CRs8`CT{7`q9C%@7rIa{sHfxE3`+AsX&8gGJ%%e(G;J|VB5}4Q*F@rOOKyn7+m_rCiNBF- zmD}LuHXwl#<)T82^YgQc>t_`=&nj+<6kC-xd8JK=*P>EX_@dHh71z%yZcrhA)S?_) zmD;q@V;g@{aSgvE67jpA5Q#WpldK8|f43#?iA3=9fk*^TABjXMEePSQW8=}IU4G!LS|-FP+# zqIA6$+$BMr+{KG8({Y?Fed4=&yJppsCgq;%05q(fW& zZ^yVkug0ZTON?4mE0qIti#nH6=OT6DT6>EUR}A9>B|sgKNa~10Qb#1>3jN>~P)EWA zw5J8(Ee!WX4QKgq!9^Rxo8m;CyAnwfBSw6I5@m_Af9Y0;h{EeWIE=hC9Yg> zp#(@I5=kPFND_%elIRtXNWvwF1>r3W2NJyk5-}V|^dyNSM$Fh3C{dPNN_qbr600bs z1c|ra)}+{IYPY##De7|+e|e;-V%)~uHh4+0Unt~0?9OzNTTt@e<}K~(J&@i0tsbN zg4X8X21{sZvrfbuehyI`5+xrxUJ#A4MC!}j_+JK_Pm^NCWj_e!>Gc@J!QH*8dFj0P z-E>w`VF^61*8^Me>C_xc<*u%6D*0pgV}*~|ZCg@_8n-E_B#-NXN}>TOi3X@d4NR09 z4W$wjAe99PT1sv#f1E)}n|0z!DrHoMMB#)74Qw&5Dv*()UIy>$n72kR~R=AsEI}%LPxV8inH3%lr0GLDrV4?=W)J0%o0tB-lL2GmH z2~642W}N_-GO9zO;GrtfD9e>x>i+{StEjQebJq31Huscvf5Ucmtz-En>9NASN86QP zlKn|Xf{7XglV|`;q5&{bgX`Ev5tt+afLV~BwK@0%rfg}mPCN;wjOvgm1e5G+Bbbf< zDKOO!tz@gHu@sn%Eijvw>#H3b-HPw&A1mA{w>=3aYFt->i5dixXaG#20Wit-K7iRQ z0+S>FFbfj2e>Ml7z?3a*(g{3gg{TgT0x(fSz-+!~FvU~X?`N`#@=7t;++tEYI`_b4 z8_6B!6NPR9+m}e9#`PqUs6ix&29P8gKoT{+l1SP`NRm#ZC!i=nYjf}kN!ik7qx2<` zGO9zO$lWCqd6G%HS|%4?(y4H3qp{AZ|f0cLjI5llsde?Ive+QQ67jdMmhiE+M z>meG%vV^mTX#h*20W6EmJuCrWS!C}~;O~K!Ht7U+M!@1jqR0xijLAn;?DBbz%b&A1 z@>0D%DsT2lSFv8x+hHcq7AI4qUBr;Mxz zBaZa>5RE5&K173Hmhky74S-2B0A`WTha~_oi+nyBd_K_9CY^wt0HQi93cxJmfAi55 z`+S}Q^FQ?yR8n6lCc)#Qn!D~Wn>(OHzlb9}K1Ab5j}OrxlqEbqOao97i;k917I}PF zCje!U$47(52U^;s6J-IFVDh11u#jaUSjblF@_7!+WRMZ5e&}gzCAF1WzveAI>20&` zf*rP=dxg_9h=##QFqy7giOFf4 z*J9E=nuh(?%Rwz><3AU{>sP&3&i<>O^X3;rzcCJtwDd=5THUgj>;oItKheRwI}Fad z(QrJAjY%*_3$<#lX+*2nYArt};e7eYHjOV~l7!R8Z=;~w52EZ_`uE)+43oz%*(uq) zp6uu^v*Qwo4h0EZAA2T~a3LRmZo)7SgztTd$oIq!fr?s5klKnrE9Q%W&E2pfG339 zV+DZ;78veQgF&0uwYzW;>#i)u4O)~;)ypB#N$5dH{Onm z9Qr^cF#tWpJt-f>)AM>IiRuZN%u-UMrYVs z0kg3p$^r?j7`b~!3IG7&lY}K#0pgR~B|HIalNlyK0SS{^COQFola?kX1qoaqdnS{? LCKUl(lh-C_$Y-$@ delta 3021 zcmV;;3o`V-PQOmDbO;HX97-EU3IG7$lYIy>e}Fz*wx=9tk{Dy`IJT$u$wfeX)Bs{6 z;W+>ONPw}p=SWtvnU!~T9x~86_jX^QU%I6}(ZR0|v#D|KhtVXMADXIWnT9{_2ZPCc zbZFjOzu$9BBZ_+PpC?C#K{)Hh$=7gHj~1cd8?1t6)4H}SyFTkp=H@!Xepn_)FdR<${=1;RocZ&3 zoh0<9y*L?uG@dLX^XOnONsguU7@_H($?v*p){hQWecw*}uTkc+k(Tl$2>wXF zoDL36E0xwk`+hZy%g`A3!`^ZlUj<)3fBKWrI8IJmD-G!f(`falF`FdkQk%xC_i$)7 zOylch5RcQu#&|Lq`00r@js7xHw)&>w>aoE{>PXMt{ZZqfe}11P z-Y7oHkZaj+BO7ir4fSL^+d*kQC=Fw2V8xQ*jNyB@*$wpAd;Se_`*4L;(96ku-ND?}V1zAS6l}l9wp>2#E+y0QOapSc>D3#;{s5e$k3-*IEwbwATB?5DIoZUAev~X>qWWen zv$xHB*Iz95`itdWFrPn!f6IBCET;B0s=IzLA5KQ*FXq6Hdy{EozB#B5$#ngxijtB= ztDg*Q^}n2PeO`}Ct(O?JrdBEg<_>iZr_Mp@#I^PoC5{-z2}*!EB9YV)iKLE5#1;C% zDWHyo3usRZ!rK_`i5kwb;ev|}hBw8DJa;6LBu0$*0wu~4XaCZve-O#h_^#Ja?`QUs z+n05!y-@<>;#(Yq{qfzsuGuwfufJ5uP0jNtR#8d`6mOgiiVe5sxn2&5E+^4N62%4d z7A1~caG?Z9Boav?kw_AWM3U$hkVwKMi3QQh7RdKE=6Tb7h{wqdcI&9k@(W97PnciA4Gf8-YUa`Yya$ZX%*#_ zK=8`TAn4T`rByWkv5!dhu(EyK3eMrR1ytP zi5i$Fw;D<%enV4}uxB$%i{Fo_1hBpLvd zZ0`e@f6XE=Ndf?~AVGV3@Ci)W(jlF|b5@AzvM2x(H3ZD&iw09Xb^U%Ot0=D&lg%9_ zwXHJ`Y<7{%F`p=O6SO>uBx)R2B8eJAl4t-)q5&jP<0*+`s|ZQbiSz^%C1`ICJ|QVv zI&74lL{dg|Nff!eWFl8G*{YVw<(FhCTyCHge?Z3fv(;>wPLs;3=^sF{it0+;SGKbE zl{H5^X{4bX2?r0;0F*=nP!^eZSOS2u$i}0j%jOJPI;0bX65Ki@3YV_TuLDchiVZt|Ka(%i z#`9TaeLTqxrE;5jMjqwnP>DVeTUvRD#+6>ljAe1G1Jxl{o5)D9EWbI)I0LmhBj{PqKz}n{JMkKwF$ljb;&B8hnVxl?ET80Wz6JlF2lZOs0`!f9fKG z4@)4)EK1PUVDN#K4(SBQgs3iwf<-LR;38I6Z1DN7fLTeAWfrnJ>+wkzvJKm75xvqe@p;TT^0pk zmT~#$id{a>f%%`h2`Z_t6qDfaQO&$N%w`Uh=oYc1!-r^G>F^;MgtCOghiL#xV$jhN z$|8pk>ja=Ia`LCX1Pj@UJwDH284pt;)ek+3t)#Y6>({Ko zC%J9*95BPyGcRzO`QgYvfAOc&$mj>j=~_o_Ti0*3x>EbM>lZ*h$&>zq`CyX%i`CwA z{kp1nG9MYypX(Q4+3Ab0tTb7Fc(t7RM*MBzADaHdBKd{VVohW4FkHU^%UFbyAe_YC z*1f0i;95?5N3&q?dNru!eDdegfBmZe%HDs~x8M9?=r_ipk(B-@NKL9+^^$&I!#X-x z^hW-9FC0zgkumj$NugHFF^zEjQmxhJI9RMcwM^qn5XZsn@!Qz%4g4_umi&D;41)Oa zOL|JSFDE7d`Kt|sa^_SrbQZ2! zkK*LsdpQ}~w$0kvV2$~6{JDC08&*xF?oY)5)urH+J7B@My2R&2JY2(kSj7`>B#6f@ z>Q~+li;R6Bk{E!FagWN!?CE*EBt&&VirbXTQu44SIh_}j%)k1P7yT!oIt=+Y@dy}m zUw(r653{i%$^r?S97-EU3IG7$lY}K#0ppY0B|HIblNlyK0Sc2_COQFpla?kX1(W&H P5hatsCKUl)lh-C_HSw!F diff --git a/summary_results.json b/summary_results.json index a25cc3c74..974037ea6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.267, + "Performance_Result": 6.25029, "Performance_Units": "Samples/s" }, { From e695e8d08cc8b7c7801a7409efafbcc1f964a641 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 01:55:56 +0000 Subject: [PATCH 275/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10047 -> 10046 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 18ed5fe90..cd650c234 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 01:45:48,220 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 01:45:48,220 submission_checker1.py:3270 INFO] --- -[2025-03-09 01:45:48,220 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 01:55:50,930 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 01:55:50,930 submission_checker1.py:3270 INFO] --- +[2025-03-09 01:55:50,930 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 01:45:48,220 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 01:55:50,930 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 01:45:48,220 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 01:55:50,930 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 01:45:48,220 submission_checker1.py:3286 INFO] --- -[2025-03-09 01:45:48,220 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 01:45:48,220 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 01:45:48,220 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 01:45:48,220 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 01:45:48,220 submission_checker1.py:3307 INFO] --- -[2025-03-09 01:45:48,220 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 01:45:49,285 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 01:55:50,930 submission_checker1.py:3286 INFO] --- +[2025-03-09 01:55:50,930 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 01:55:50,930 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 01:55:50,930 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 01:55:50,930 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 01:55:50,930 submission_checker1.py:3307 INFO] --- +[2025-03-09 01:55:50,930 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 01:55:51,875 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,15 +31,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407919 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3089 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407143 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43378 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.434975 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418495 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index ed8cc2449..71fb6fcc5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -902 +903 diff --git a/summary.csv b/summary.csv index 73c8cae94..2d37b04f1 100644 --- a/summary.csv +++ b/summary.csv @@ -9,15 +9,15 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407919,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3089,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407143,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43378,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.434975,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418495,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e6d6c6713..6651aa957 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.407919, + "Result":0.307955, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.43378, + "Result":0.434975, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -579,7 +579,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418495, + "Result":0.421348, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 23a4b14d8e53f77d5d781e550674c16db763ceca..2e2612bc41c57a864462bb3bba320df9265a7836 100644 GIT binary patch delta 1525 zcmVo|5)Kxx$Q|XQRBK2 zOw=HlL<3+F4S-3u_W{gi5tt+afLV~BwK@0%rfg}GPT)B!M0HpcfQcFcX7fdZDW1B1 zKa*9ISBlBz7L(f1xd%4eNbWG7D0CCpzC;o=t|yU14I)W2fF#iXlBn^OMA9xol5`?H z0e?jaTAPDUNXnKr8>KIilu;cLMeZ({$dgRk)iSyGl1_z-HIxF#@nH!hnMDa&8Z180(k7h%nGn?>QLv098eGQeiY-3>6)-C)vdmIe=UqPO zQnumv_KrDIs~u#bW5kg*AENQ3&4*}^%n~*qrU5dE2FNV3`LF~)W|7TDgMZBjTH2%& z*tCSG4vE4!lZxb=sVlbmd@q_mv~5{Qk!9vgo%i|ZMooKa7nynondljDq|b+FJn8cx z8YHuX&xdJ%OrilYi+nyT0gzeb^VuAHA`^T*q!T0)d_E)!=gcxbpWOKW!C<~rqmSM; z`ZOC&-`(LS(2X5rqG`mDMt>io@ubm*XpqbjMjxgDGKmJrEHe7A1VCny(MN;P2U^;s z6Yvv2REI?YnPrSVx?-cxb7cOfhJs4!D+T4Y$){<#s=4bHv$+FGG>kaX zLRrG(!!!UT@#ts?Ws%8;bplWpnS3;ue4wRGI#Cu-2`(QR1}j-6f`654#WtVkuuKLS zk?MyY$5v8XsReA_$!J0O@n9{oCK3eZ1lqPbgd#ct*aMXU8()s)jOb` z$!}tAF0J*KjQ+-J@yPf4v;kVmAJB5xjoYd*$rE>N#(IG4vbb&`3*v zl%~}!d&xeqVI3XJyTjnT8x6;^*q8)^v{0+&nntvGuh#N&63&;OY}5D>CP_Gb{5A@@ z{UFM|rGMWI!Z3OKlAV&x`^k>}0#Hi>1QY-O00;m806zfda=p5QOi23ZuI%ImV?GwgYX+sewXMT5=Q8CK0uMkkmSU`zlW2B;ZqztC{_Fc2PCo zt2X%s=cUm#$}@rz(A)^Aw>A1$Ez$>+_`tMaticNMpk8wYRBSa3mtPa;Tr z%59ByAy|sB=Q~i$XVIafxG_$#5KFhkf0prYwgo&Rore6Icm^DE-+qGT53{i%$^r>&v?DM^3IG7(vxFr$ b0R^J44mu^1!6sAzu#*-iBL-q500000?k&^- delta 1526 zcmV@97^a+$y&{2_|Y> zSAvNe1e0h0OrilW$@V^g*(?H+BmgiA60|l4pTLwYZPE!mXN9N^ivloFL%?jlXfVZ7 z*Y9Vtiti3X4)8bA^?zLH4VMM#oP zq<<%%C_!s;@CixT(q^OdC6Y3#L!!vtB@=m)NxNDm7hlq;aIuC`0GZrRm$PLyO)9^p ze*noUsw;J0Y3J`NYp!nkyO7*r&r!sDBmFr<<4S`L(IAo~EILdBND_@Jkt}lQumpf) zkxxf~Q3qPuq!ZX5fv66NB1ke3Jn*T~uYV(v{5v*DDk-oOm3jM))-^xp?bwEL%e`Yg zL5bcEN7{FY#*_XXqCqH2ICz)_pd=cAvdF~45&)D%HXbEiHfPY%CY>OZ;MO5gxO8QH z9ay?nY}onxnS7x(p3f@p<4JEQmDk*5|K1iNYOA=G}oE%ZklANafGpvV5VYo@FXgH#Il65hiL#yq5&+6%sngtU|D4EQQ+@^mNw}GcSgYC zL!!tEwv5R~RqXP4j?163H}X=wK7T52_DNTQp^xQx{mTYUa2U{+FOnWe1G zyL{56Y{T*G9do8uJIF-Gh$C%2MB_=D578i*C2T%S17s2nkXdB&VF`fDB7d8Y2AdDG zv`HthX$ess5`}Xn70EeMS8Vh7UNnDb+p>}(%gmWN@AJ`(n)Y_>T57$6O!SO6(&s}o zp7i+;4U$>H=fgBWCeZ+yMLr*v0LU!z`D_k8kqJH@(g~6YJ|7Z=b7mQzPj3AGU@%{* z(MN9^eVUDiyW^RxZtMUPJ%1yP^!X5tCw)FdgJ72M`7jNDNi+avk5p5{r(OP!@T7SSJ8wk;g}a#|K*4q!VQUm0296m0G{%Ek5aOv+sf(ww`;1(=>>N!AUTg#6~YnPuD7P)4F=E)s@=6UA+S8 zNuKr>&ido*->h~gs~1)!n=IJk#6>A#( zhr#M4SjIdWhtW9swtwn9dkfcM(mk4n{nyJuEoS3C7s2aSy;siutDf`b7el`>4vn<* zM`>E!vX|@w8`jamygLleyU}nwi;YPzNDH-Uu4zQ8*J>?4C*geg$u^BIVUmQ?$8V#c z+Yh4bTl)9iAPkeoFWD*Cyq@gnFHlPZ1QY-O00;m806zfda#TpxQTlW!p( ze{RAs5QOi2ipclG4uOhVNs!u#Q;{l_s#R~6y#%ZAk8HPrx33c*fW)cC^~`=dyK=eM zsW$lq=cUmV$}@rz(A)^An+mQZpZ9k5_rUE<3k9NnnwiyZnuBryOz#62k=#nbb8C5h?@ znaomBq~u{uayp&S>HMo7dDVXgs^gG<6HkCc?(0ub{{geGBFX{@tQfg_MhXA`; Date: Sun, 9 Mar 2025 01:58:31 +0000 Subject: [PATCH 276/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10046 -> 10046 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index cd650c234..dfb17ce05 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 01:55:50,930 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 01:55:50,930 submission_checker1.py:3270 INFO] --- -[2025-03-09 01:55:50,930 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 01:58:25,692 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 01:58:25,692 submission_checker1.py:3270 INFO] --- +[2025-03-09 01:58:25,692 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 01:55:50,930 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 01:58:25,692 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 01:55:50,930 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 01:58:25,692 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 01:55:50,930 submission_checker1.py:3286 INFO] --- -[2025-03-09 01:55:50,930 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 01:55:50,930 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 01:55:50,930 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 01:55:50,930 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 01:55:50,930 submission_checker1.py:3307 INFO] --- -[2025-03-09 01:55:50,930 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 01:55:51,875 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 01:58:25,692 submission_checker1.py:3286 INFO] --- +[2025-03-09 01:58:25,692 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 01:58:25,692 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 01:58:25,692 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 01:58:25,692 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 01:58:25,692 submission_checker1.py:3307 INFO] --- +[2025-03-09 01:58:25,692 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 01:58:26,682 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3089 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2292 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 71fb6fcc5..debc39b5c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -903 +904 diff --git a/summary.csv b/summary.csv index 2d37b04f1..39db7049e 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3089,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2292,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6651aa957..e3c3de439 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3089, - "Accuracy":"acc: 75.400", + "Result":10.2292, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 2e2612bc41c57a864462bb3bba320df9265a7836..b16cece99481f9154c18932a3713ba62534c8e77 100644 GIT binary patch delta 985 zcmV;~119{wPQFgC%>x8EcUwl2(E~;V&;Vv^lN$sZf6t63!gxxHk?FK;(;!7iqqJC! z%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8uxLOzKp z%L5^A*~;Q2BSBP?^2~B=+q3c@EzHKkfc=RL8pml!j%irsq!5h}MzTQ2@OhG!+#C<1 zR6bMre;AC!BlDXxm^}KXH6GTk|4GR+e|K$&VEI5Y5(*N@0+@!%Y)R>N_3%Ot*@|ju z+|xR$J7z>A4ysJd=<_*ASBa2DdWsU#jMu+Lp32g?rjZ9PBeNqtq>)IJVIxVSh!nNk zu;PN|kIf@zZJ?n#-kR&!&Kvu%?R0I=?Wx!E6q=DW$7;OR5grF37!MhJHdyU}m$OI( z9sy}pDw1m14ry-Xnf%RV4N2eB>154kgv;HWTdQOdgsQD*$u6QX)e6h1RlW+!!esH}w!OC9TJQdgkQQ;eGQXG+ z5kZ=9lg|da0Tq*q2O59uw(B0cKz`P{Z|R({6`7DM<3>nj!5Zes13R%OHxuciE?CoiRZM=Dp-h+g96G{PuZ zN?9si8}I6Lq{@PEPNV%gsH#YRRAhfI-17!|p?C1ba1V@;Au~tcKxKBdm-@H>32Qto zgB7_B*eWf!k&##?YPaQD#hUYt`dZMkzLL%Qk_tid&0Ru*h%j|0ziwkn#pXtpx8OQLV$1(E~;VtO3kmlN$sZe@l%gQgX`jk?FK;(;#`sBATy8 z=HK(vmTwwd1bGysEGHxLm2mUx_^;tJE5GxE5Mf9IIUkvc5XHcfwbPvK7_R zxTkegccl@LIH*!FW6$S=t`Z@Q^b{pzDX)KxjLOowrjZ3NBeNqt&`2c8u#wOxB6;mL zthiv=WAn&a8)&GGx8^#w^Ts}GJ6+pzd+PPP6sBa&u^O*+gvWsh#>0|58)fZ*7qdtO z9sy|;6-l*hhb*(QRQ_hN2as0N2eB>153>375M$w^qp_2vu9rl3he&sudPht9TWx43ou^+xFUaYrXp~LYBvLWqvUu zA_AInlg|da0p*j52O59mwrBhB)5d!DEuE2aMJ6Ooxe+p1u!i|I+MGAcoWZ>L7XJlT zAM<%c)yfv6&B=kF`O4rwnj;EtFnM%sWh{(mRZ0x;TF6g&@>0lmq;fTl=p}AWBaEV? ztfb<#@vcrssx%m9EZVPws&e|HBKv#co;TPFy@M}?dti(VnfXTsDzmG-)W-!#SmR+4 ztjKjxu4vAUl*BSoyDitMvN_+VuLUdWE7`0snGh`7+$AK4NU84R*KN$0*xaa+yd4{j ze*#cT0|b+BB^tAY2=)sE8#9?1vwjq!0t36IW3wX{M+yX@unsztZXqClYQr!PgztR{ zqq{9R#-$au18vEvfkIPSaud=f5w(7h)H;6qDo)}g;8TyQnf-QlQ8nMIHu(kTrO`FY zGlCM(+z6?+HTqaB(g&3Iz_ehk(NLovJeoIm6}OZd2XD?;a6y7kB1n76ZH;yzSc;mho@41w13<9xDh;u)y#EnA(wuPDsH=r2TR201zBQ3kr1b zIL~r?@dj|pPc{t7l~c*kTbQ(-#nHX@ax`{bmvzNpjrnu@wS0XaR!yZooQeZ#D#0mt zz=Cm2g|CZvxR&{_j3?em5Kn#7C*Dqr9Q!~dF#tWrJu9EZ^UG=^ftneal_@Dw^0*>7 zElXPF>%+*K{tHl@hWwj&1{`zWeuCx?v#=tB0t7=*t;3TzCB+0AGnpBa0VcWxqOcA+ HlfNc^zj@aH diff --git a/summary_results.json b/summary_results.json index 164b5698b..db6d0c2a0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.3089, + "Performance_Result": 10.2292, "Performance_Units": "Samples/s" }, { From ddb9ceee809eb59e07c5ba81a81e83473d617f02 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 02:04:16 +0000 Subject: [PATCH 277/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10046 -> 10047 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index dfb17ce05..920f8fc99 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 01:58:25,692 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 01:58:25,692 submission_checker1.py:3270 INFO] --- -[2025-03-09 01:58:25,692 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 02:04:11,557 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 02:04:11,557 submission_checker1.py:3270 INFO] --- +[2025-03-09 02:04:11,557 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 01:58:25,692 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 02:04:11,557 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 01:58:25,692 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 02:04:11,557 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 01:58:25,692 submission_checker1.py:3286 INFO] --- -[2025-03-09 01:58:25,692 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 01:58:25,692 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 01:58:25,692 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 01:58:25,692 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 01:58:25,692 submission_checker1.py:3307 INFO] --- -[2025-03-09 01:58:25,692 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 01:58:26,682 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 02:04:11,557 submission_checker1.py:3286 INFO] --- +[2025-03-09 02:04:11,557 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 02:04:11,557 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 02:04:11,557 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 02:04:11,557 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 02:04:11,557 submission_checker1.py:3307 INFO] --- +[2025-03-09 02:04:11,557 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 02:04:12,595 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,13 +26,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72494 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421125 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25029 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28806 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2292 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1569 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index debc39b5c..7b8d08438 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -904 +905 diff --git a/summary.csv b/summary.csv index 39db7049e..a9d24b1ce 100644 --- a/summary.csv +++ b/summary.csv @@ -4,13 +4,13 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72494,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421125,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25029,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28806,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2292,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1569,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e3c3de439..d5fec5349 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.25029, + "Result":6.28806, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2292, - "Accuracy":"acc: 74.600", + "Result":10.1569, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index b16cece99481f9154c18932a3713ba62534c8e77..304acd7dbb3fb66bb2db4f0cee47bfe78f9e1310 100644 GIT binary patch delta 4671 zcmV-F62R@gPQOmD%>x8Mf0X`{(F0%t9DB1f1a1Om4$ZM(Jp`Y%FS#OccXVn##+X~xZip%o(^xBJh?N%nI@hF1T55ZA1U%V|Ey z?M~mf->rIEqU++7Ne)9^@#bl@tFK%nRl2oxOxfuchoP;~trqC@CVbUWymeJDCO zFx4G@XrhCh8=V8PzD2Bp^4|6Thobw8oc=&`5B)9C*-qE@4nO$R`MBzQU3H*AIMN{< zu+UBr2vi*cfvQ6wP<03de3{Su9jZg!%r{V(UG1elE56fUht^>A83vOg2mWkSJxmK~}e51Y=w5+dWv%aK4(0p^3 zkRT#V-O1f;OsUx1sFJ)L8;ySgP)h>@6aWAK2mk;8KLB3cHh+_N2qu4DbJ|E2#ozZ+ zs8o4Ks>Yxe=%35>l;cbiV~icg_S8PP2nbLOAVw07^X*3hjKw`ivXafLytDI=f!4XV z`wIQiE%k{GetnotwR=B`gK&PRtA?R#{(KM)gZcPSzqx+D=jvLV^ykC=G@Sc~`Zqt; ze?9uw!PhYQBcAwvqNRTc=JBCENs`5Wvl$O2{;VH2!i7IikBq`-)=$!}(YP5eBELUe z1D^JJYQ@~8bI9ez9s7O{SGFbvXTSv^|h zj}G-?wSV%v+18s!2dlns1OIEB`>bW9dQJJ(s4g&H4|AdP~>72E$~MCDtavaOh_zTGs~4I0ThD2>t0tQeAiF~J?CjkiwGNv1S>lc2vkbYG4L7pkHrLQh$Fm)j$p>Y^ z7>wO9T%n?EiD43ny(_sU5@%O(LnQ95L%N+jaccZfv%erG5##o6r=i8#CGL?X^^k4VJX zy&w{Cb}xxUoZU}EV)F~m<`v@QSUn<5%vB?B%oxx_@#Ew!UVfPcaklh{ z@9wRJ(M%%WUoEnl*;=Z3-aXmGM}C|xVdCayEwg{O&3rdlEcOPAu(O4BQjlos-vWI(dwr|Tm7$KT%XtDQtKr~qirgc19OKu zhg0Vub>dokixNi+;{+u@9g#@th(uCHB;pGF;FM5D!UeRaCE*;5=jyxMtp%1Wr(wX>C}kicyiYtWcM?B>Fvup)!sM-a`7z*qrv3v-fY+nV{foj z=}pb^DArL*1r%?b9EvTs;kjM`i7qG6MH0mY^cE$KTyUWTNF)+TB9TZEiA0jnNoHiMQ^~q}XbkPPTwi3 zL{UKW7A20j$WBlKC=!W8kw_$pC~?J=`N1ne5yL$(&eM|cjwe7llMP+cfn18pm>$L` zL-c&XG$Qe(S1)d}{veF^ru|ih2iN}D>(V;PtAOB@mqXBNILdQO8?Iv)+(jzB^>nOo zH^;Iin5c0q2_|X~OrilWi3Y$#4T5Qwfr$wa%#sA1?ZGE7WlNWJ0$|Fh9*KW~hpI%Q z3`cTl{vU8zM~zjUvu*~qy{EKVmR+=t<(s6(3ilo@M}kTACv6EPY7k7K0WgUMz(ftM zV_Rilk^}%|NrKMy;1ig#rOP^TC73d*N1_l+va^j~w*IHUR6n$mt)s?DV77L_G#t+= znlbsF{;|TXa?6!qqQ-F~n5ch2Fo_1hBpLvdZ0`e@?J_V)0syllL1%mL2~642C7r-? zR*34cC;$^R1kCn}22(tB{eC9vD6bNe?Hwjf+sr+%*+sIze4@}zV0jWr)Htq05;ce< z(EyS}14yFAQxZw53`x?7^aPY8=xh%@At_tBY?PivQbzSi6uG-(B3FMhY1PZ*@=H1u zE;mpLAd~yqYPQU#N#!-nA3(B>>MGq=TKW6RhGVw9A|wm!If|HXq(6sf9BI%Y8bq>! zMTcnsNuqHil4UL(mH?0}^XVus>Of1EbOQS$5Y;151W6`>2R?QBbtIC1$0kWF1y-Um zZ{IP^hL`hp>_WNY-Z6ikphWM7E$ur*<4XSy(IAu+96U?|P!bJ5S!Uv42>{A68;_DM zn=@$Xl1>mxaO;pLT)Hy94lG@3HthWUOukSX&u5kQ@vJO$twJi@0xHo5VoNIz(YVsf zLo`Tb1vd}V0F^`oRF)ZfSOTE3%+jNz|K1iNb#!OXl5y9m|@{J4ofv z-?Dt6rk-n+clFo}({+skQ;+frutdLzEp0tS<4Ru-(IA!;oIOkfSP~6jS!V8G2>{D7 zdyfKt543biC%7{L79SEtRh)21vroE;b=z)%nZPtSnI_sr zY-#Z!8dq9;hz5ViWEx2((?~LzMv`fkS$tRmNoHAsjtLeYXz7wpfJ}($ktkTk5)Ce6 z&6+Jf{}nK6DYD8^*37$n(xq(6_N;<`1Zb)PGSM+&OPdeTxYFiBG)QIzn-9|fnM4C* zmf3t*0wA-@=3|1*2U@zM6WFwbs2+*JIg^UyoN3l<^Z9>XG=FH@vX&yN%$a81=VP|g z<*X_q(<~qpJtMaC`4EjOeLh5kWLEI`Fb$ANG(cvV&xa)dGRu5E+k;PJg3pI^f@Ffv zheY9=S;gm*8~;BT%$I8PF?WqV?N-}!3j73StAI>2jo8xYLo}{5`VbA0S;6STG(aZN z0GVY*AC`Xr$SgDZm|*mQmM-Z8`~(oyV^Kh66{C+?v(e`{GXGOUK`r%Ff^yg7(>5H{ zF1p2R7eI-I5nGykh{lyBAEH4hE0}zk2B0J!9aBPCX7XX30F-4W9}`SI(9$KHCz z%c-v=-xmI%?msNjUnnb9*M<+H^;@vCMHGZlkbGPBp1p=^Iqe_K!r|-HpqBIC&!zwR z)!W|VwjfSIZ(fYkw ztItWeSbef|?Ms*>;q39-#P1LNDEpTFeK!ij(O1AGOJNgT=;u4qv1)#_JFl>{5 z6fl3_TZ%B10+OO87YdTC>=PyIHt~*ac`cbCf4_1d8OE|mvLR0tr>#EfR-cx-|80|{ zXbmM#Sbo~Urrkl1hb&6+)oJJM_T2D0hzpuWG-WxQb{>Iuz7789b1slvn)B1ngAm1u zW%2LKrSsYD>EwO-*RQ~q68Z82LOMOr53Sm*UD&;cTmWxT%2}@FIlQQ`L}}M z7QuGV8T5HF=!?M>iy-Y=V$ip0U<=Me$Z46dCVrPEf;Yh#@G2EeG)-wniQ(BxBM6$t zEZK;XriQ=qmQ7o^qKdPdI8Kusn(!_A0Xesn9nEQ0q|joRj9Ma+c|nsA`U2+4L^6M| z=&Xto@wNHokfp5r4vz|1wagMBpllk>)-*}!GD#DmE_&nwvL<>h&qCSrv}7T0&J_Q} zp9*6bLJB1n>c>?1c+O(+la@+SubEIcamOKQtrSUFl{8cW9D$J0D}IL3M-y{Q+pujL zYjC2$yG=Zzg37v;Ct=kpv92zMEX#kGERfa1f`&qB&?q5>Qc4hvu&9jUQD!Q9SgeWZ zN~aOTR95mLWO=4sNXvH)ChfMueq!PC68qSuR zVh?TEgfyNY&oSk7qBLmWIVgWsQ-Vlx+G1EjDOv=yD#Sl6u)Gar`4~|W$5nl68XKkM z?DXH$;Eo1Y%4&yu8r;|5%9gh{R`RsNHGK_h`WjaH+78#mi#73LO}tnWFV@71HSrQn zyhIZ(QR0PY1taNzc+0 z>4*Kkp~^KJ84VfV@AmJ^R4rXcIl8x!{eHDX00l2-$zid^rmx;E8tMN(w!N@wI%1pn zbHjTR;rlA@os78U!agqJ;YZ1P$$q=Z8%$!IJ;hfCh69s)@a)uQ&u0qZ!)zbO}-01dV%Q#%G32_lp!W@{Jkm3HC7d`Kt|sa^_SrbQZ2!kK*LsdpQ}~w$0kvV2$~6yk5S(538n9_ow23 z>QZpZ9k5_rUE=d198xJTu4_WZJ1B%rz=#a&8f zDS2Fxf|6UB7hnCzi~bW(9ftgycm#~OFF!&32eYsugaQOXf0X`{I3+>^Ufni-lYS+C z1)#_JFl>_oCRYIplUgP^0eq8{CME>DbqyPnza|y|U6a-(J^`_l6(=JGVuD|N{LB-093lSaZHM1$F!4=_rUxQz}qy^yG02*9bE!{+H?0m zI1DKt?$O`aDIp|pkEhrQ{Kq3$E_<7bBME#ar29&mxMP5C>t8QDOfgX-SbO zR(H!nkd%Zpu>hnYLVubL_qnv`o$+1dyFN-g8%)=!ES<$^E6L%*@e4wHVED?DzkH8N-8#OOzU&DZi^yrM91r= zi3;$-8W5g8Ci2bkJnV8Yp^T2_e7GcM<58D#{soWJ{;@#F-A0OOKEW^X@kY>h4QnP{ z>Bk?>O-5^45{Awb$mEZQ%%25Eh1v02g4g*iY z76SWHy=vm9qv2r%B9o29ef|^mhDjdjvtO3d5B5WN*}_o@dQ$=uByXClo@S9#4rT0| z`exa6s(Su7&}uh!u?@Iwb&SphSpCRzpJH^VAWA^G1s!{OE+p*$z2oXktMS!UFq0T{ z(#sKDumHJYBHzJ2CTqH~8Fkl1C$CKwE?@`EY^G!TS4oINT!p>pf74A^B$p+?+VV2; zI%1VqydBtBPkOeh%h>Lb5rwZ;&pySOOAMWlwbdNVxgI_m_7|_wZ3+F!K9e$WonvP+ zE)Cc&35UBfGG1AXnrkGK9`5v5JZ9nEvO}TbUP{7K0Hz67AE7!sjjxp9Z#u+UVlMi4 z)g7X{uL9ye*-mTR&C&aGr_~%2@CbJw|yv1mIn(<@ol?Yu7bc8bR*3`<QEJ`r+#U9IRNZBiZ1OxX7VXE;-A6N;JxCCEJYF{V5P(`)CRFS3>0 z`3CCICVKoygl=jb}^&+;!`Oj=RFlB?qJe0jqMw ztAOFKgw4BgD(&3xzb6xen+d=VSX53RaEQ|2G(E3QGGM0W{f7no7YA=x{E;_p`>vt! z_5E!(T7Q_1++W<6UM+0`-WyW93M7&j+nI!G6PBay@P9Bo5qy{|EGKmn+jx*!WdBWS zyy_=@ZDL&5{8xfb6Hda>r_cqPFyay1%+5GpKY!42(r1Oy%Eo@rx?g9XF#>#19bW(k zdNDGUMes-dik~z{e6})J7-x$NDmnx~&~btCJ;wLWdR1 z%`W|t>SGOTdtmgjKiAQOnM*NT6eQsW3tdBvIR>qJi7JQ(NBZMJyj8%l_szTFv;;(AecN*wf+=;@Xh~2BD9vJ_+cN6XanI)1aFQNL+)oa z1w2LQCmKfft$0tk)SDbdEW1g(Ff9wid=t_Cz^3Zm?EagVMD zRSa~@i!anur-5Q$E$~P<3XbQ zq{*VCN5`R`zF(8TKB&;jC%~73o+kyc!dI87RdrQ^@{0!NJZmd(QqDc%iKhFkzz(U} zNr2>di-K6z$pIDlGvoHQ4lz9LF}y@YC|fdKwm=P*cdJG!Bw2h5lmlv@fVb!KAMbo0 zue0hHUK&}wU5E8E@Ma$w*F{gO<~M|>odija_y2_*QIYE!w-0rQ$=$$`6vJ*{MFKS^ zyjx%XgLw+HH7Xzhm!1S~Pm7)!t78gX+;M}ztD@o!lQr`vTAFjoi&#pg#8yJ}lPJmY zAAgy@sK~91+h;n&0&kefit9JbDuJ3)-osNl^$ho5nR^ykP95u8rroQ#d>**RJ!>N~ z%-+Q>*^?wL@=A>=xAsJ6Y@-mxV?r2}D9SOfaf|{WCI_8R5lvG{xVd&3=GItCQb^iO zR!jo2e{IA|qnhH>sfy-iGV9)s6qNmQSA_q5&7iKqhPmvPZvD@*-!W2prL5M-*PZd= z>_}|hP`GW}AXuEe0ACbN)GJVfs{+Zri62r986q~B&7+yF4K2E{Pzn*bnpw#WOU6N3AcT`bVbz{mQWB;l+0Ou85yz6D`|hl{?~QeO(w9thUX6YpuSo(5|th zafc4oNT?`X-b z#?as-hc5BY8&w)ajPcvyY&{0Lrp31$7FPhoqQr&JndMHSr=(_xEkh;BDc|8X^XC znrDA}-M5_wqA1tfALWY#m#X-CrGmy=2hC>k4yDUPjvyp&j@17YOV)G+jMyyLj%oe$J+6t#C&iz%L4wmKKRCs$Uw z*M0i+O6H-yI$A$wzuda@mEZ%pw%aD}Q!A?(iyESo&i}+Z3YVqFOhd2d4;Nu)E&aYc z_naE5Vh6m=bdFxuEIAz!A&lNW$4&+P9EGqymiaYvoc8QYWRCdg4A<`aasRAehHS;v z?Hs!2|1J(t*1nJl3wk}!{Ew!9J=eVM=O&GPS$jtC?+b=^Y1Aer3--C)_P@o``BF?Z z-bW-A<0`Q(eqln(W>ALFlizGo9z~!pp6l;+N03UUI_ySK9!Re}nlc~n{SI4n_}vTe z9!2@pLM5Athq#oymTgnck-I;mO@!*ag|pg`#>lbNWwTCLs*}Zl@rITAlU5dU^-evu zZy^I%Ygy)bjU8M!LQkJ6GyuIHw!m??I(v*goY7ciprb-lnqlUgB(z`6&EYGT$g#Q% zuf>EUr)}QXZiKiebw2ejMeq8eBfYxqwa1Ymp-39zf*E^;MAFN{ajo_B`K90W^%>nP3tG~#`FXIjLYET@lB>RMtM?O zr_jkcCjTg%;J_2B;#4{O>@mjlC5@e6tu_2 z?wh<$(Ru=% Date: Sun, 9 Mar 2025 02:38:01 +0000 Subject: [PATCH 278/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10047 -> 10046 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 920f8fc99..f312dffd8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 02:04:11,557 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 02:04:11,557 submission_checker1.py:3270 INFO] --- -[2025-03-09 02:04:11,557 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 02:37:55,815 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 02:37:55,815 submission_checker1.py:3270 INFO] --- +[2025-03-09 02:37:55,815 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 02:04:11,557 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 02:37:55,815 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 02:04:11,557 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 02:37:55,815 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 02:04:11,557 submission_checker1.py:3286 INFO] --- -[2025-03-09 02:04:11,557 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 02:04:11,557 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 02:04:11,557 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 02:04:11,557 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 02:04:11,557 submission_checker1.py:3307 INFO] --- -[2025-03-09 02:04:11,557 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 02:04:12,595 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 02:37:55,815 submission_checker1.py:3286 INFO] --- +[2025-03-09 02:37:55,815 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 02:37:55,815 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 02:37:55,815 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 02:37:55,815 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 02:37:55,815 submission_checker1.py:3307 INFO] --- +[2025-03-09 02:37:55,815 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 02:37:57,094 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1569 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2173 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7b8d08438..71657e3fe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -905 +906 diff --git a/summary.csv b/summary.csv index a9d24b1ce..6452fdeb9 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1569,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2173,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d5fec5349..b874c7ef4 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1569, - "Accuracy":"acc: 75.200", + "Result":10.2173, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 304acd7dbb3fb66bb2db4f0cee47bfe78f9e1310..eb207043389f7ea6d3aca24e6be2db70bebad5f8 100644 GIT binary patch delta 1391 zcmV-#1(5o`PQFgC%>x9Uv(5aI(F0%tK6tY;1a1O<-JDyiWD$g_t!T+EqA}G9%c@nr z3d+J{@#MC>w%uCq{)>3=szV@PXFl_Hs1BinY%g}uFMX&wI55>6bYZ6iIX5~7EF`yxRoE8! zpR2lRSrwvc$q#8|;>Gtq7GU?k2=Ft~`vU=g&i!`+bnLe4^gBR)*1K=%oUj#{kSyay zNM*qq=G$m<-Y|0p^X6Op7hHYJ7g4HKwjgUx4n$h44F01zqVNWjN7q)y!gy9?#1OBg z{G=x@rF=&!SJQ}I;^s8MC|OEbDqb7!>U5;af^klx{W_?sNPkphe=pqg2794*@WpUU z4~&r^Gv7dEcD0xKxBv-jJS>A1xenMWEx3`9SSD(>^Nspi(6YXg&H9oGLG#UB zLV}1ebtk`WV@k#5MwR64*l7F{P)h>@lW`>)vxErt3kAt*Bq(aLe-xnte@{xyR!k-E z9F(dlK_oeCF)X1JEdp8<;-3~+-iES#j3|lYs=hUijnZ;<`tNCQM}sS6wZlCP?rU&m z%Uc{PdD`KczJ@h@4J&H0Fte-M*6{@#@2ny%}5d!r079p9F-&c*n8aXG%0h^st!hnJJ-Vm`c? z)$@%jrz+nMysq61y1wtqwda%3vzf>-%qNcLJH)e{uCKnfw*!vbHsGAg9s4aa(Nmyu zN#H1nfAUtZ+4p8~E(ZSrP)h>@6aWAK2mk;8KL8<}UL8jQvmO~u2n92Pgfb?Ra3LRm zZo)7SgztTd$oIrfB4}$RL24^bMXFS)qTVcf30C7D*=_@GUlSmJ#Hq*i%zit&ay8qk zF8KxLrO`FYGlCM(+z6@L8htG1=>tl9U|O)wXsFQ+JeoCk6}OZd2XD?;a6y7kB1n76 zZH+b|ScgpsU3;vh!lK8+8@^q z0KqYIpg;$Y^DM__ZvdzK&4xj_a4H#g7H(Qk;^^LbIU4)E&-!w(#{4;6EneS;Ra2?= zr{aK`N^r^@uwYzM;maZ(u46td;)&N1#M3V7H{Onm9Qr^cF#tWpJt?2%^UHE2j+%lL zla!Pxd0di$miKfr{o0Sb>OTY3amc@kC%_^1^(Scl0JE_ogaQPfv(5aIIVHsf$!jDi xYLf#dR{;o}Ta(u&J^`?k7AGSHVk7_n0078Hs=)vN delta 1390 zcmV-!1(Eu`PQOmD%>x8Mf0X`{(F0%t9DB1f1a1Om4$ZM(Jp`Y%FS#OccXVn##+X~xZip%o(^xBJh?N%nI@hF1T55ZA1U%V|Ey z?M~mf->rIEqU++7Ne)9^@#bl@tFK%nRl2oxOxfuchoP;~trqC@CVbUWymeJDCO zFx4G@XrhCh8=V8PzD2Bp^4|6Thobw8oc=&`5B)9C*-qE@4nO$R`MBzQU3H*AIMN{< zu+UBr2vi*cfvQ6wP<03de3{Su9jZg!%r{V(UG1elE56fUht^>A83vOg2mWkSJxmK~}e51Y=w5+dWv%aK4(0p^3 zkRT#V-O1f;OsUx1sFJ)L8;ySgP)h>@lW-*(vxErt3k9IZ`Y>#>e-xntf6p=Hb)qzA z;5jH&Q-Vlx+G1EjDOv=yD#Sl6u)Gar`4~|W$5nl68XKkM?DXH$;Eo1Y%4&yu8r;|5 z%9gh{R`RsNHGK_h`WjaH+78#mi#73LO}tnWFV@71HSrQnyhIZ(QR0PY1CtaNzc+0>4*Kkp~^KJ84VfV@AmJ^ zR4rXcIl8x!{eHDX00l2-$zid^rmx;E8tMN(w!N@wI%1pnbHjTR;rlA@os78U!agqJ z;YZ1P$$q=Z8%$!IJ;hfCh69s)@a)uQ&u0qZ!)zbO}-01dV%Q#%G32_f0Q96as0h0$u(Wq_4Y;?VmiJpXPt}j_2P1TEfH6F@D490)5Uyv zGppwtS58&FA9!868+3i&m21x@qh~XbW0+4I&v%GtJ6&IWZEpu0sRDJtIhQ;3TV|rC zK;@FaQ4;^;tzNV5&Ei}P{sT};0|XQR000O8002J#A)Q_wvlWSleX1|?Xxt#A+ zll+47(&!5189@nXZiLj^3Vo~==>tl9U|O)oXsA#J9?k2Ul3U7+gEwa^xFEqN5u`ok zwnDoQEXCOK9Vq6r=+IHz7^hf>rQ2eE%lJ3j0xk%-#|i=yEHK;yQ#%pS0V()|v_GyH z0D@y^K!FY(=UI+V-T+Sds||y4=2S9t7Oq;4;^f|YIT_ow&Dz;ujrnuDUcSB$tEN)- zr{aL>QgF&0uwYzW;`1UNu3N% zT}oytd0df#l3SV=U;W67{u59ghWwj&1dO>aKSBKmv#=tB0t7*Sl>U=ACB+4x$NDgA wlL97J0Sc2^COQFpla(eW1-x|)8zqy#CKUl)lh!6a0kM-6CnE-9Bme*a04*t Date: Sun, 9 Mar 2025 03:06:30 +0000 Subject: [PATCH 279/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10046 -> 10046 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index f312dffd8..3cc7af1a1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 02:37:55,815 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 02:37:55,815 submission_checker1.py:3270 INFO] --- -[2025-03-09 02:37:55,815 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 03:06:24,977 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 03:06:24,977 submission_checker1.py:3270 INFO] --- +[2025-03-09 03:06:24,977 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 02:37:55,815 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 03:06:24,977 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 02:37:55,815 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 03:06:24,977 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 02:37:55,815 submission_checker1.py:3286 INFO] --- -[2025-03-09 02:37:55,815 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 02:37:55,815 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 02:37:55,815 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 02:37:55,815 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 02:37:55,815 submission_checker1.py:3307 INFO] --- -[2025-03-09 02:37:55,815 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 02:37:57,094 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 03:06:24,977 submission_checker1.py:3286 INFO] --- +[2025-03-09 03:06:24,977 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 03:06:24,977 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 03:06:24,977 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 03:06:24,977 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 03:06:24,977 submission_checker1.py:3307 INFO] --- +[2025-03-09 03:06:24,978 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 03:06:26,001 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.26254 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72494 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74175 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421125 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28806 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -32,11 +32,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2173 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2647 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407143 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409825 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.434975 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 71657e3fe..5fd6ba795 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -906 +907 diff --git a/summary.csv b/summary.csv index 6452fdeb9..af9a0f3d8 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.26254,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72494,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74175,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421125,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28806,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -10,11 +10,11 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2173,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2647,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407143,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409825,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.434975,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b874c7ef4..e2da74891 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72494, + "Result":1.74175, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2173, - "Accuracy":"acc: 75.000", + "Result":10.2647, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.407143, + "Result":0.409825, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index eb207043389f7ea6d3aca24e6be2db70bebad5f8..fc777e496d114b7b2d29643558fcf4d1b28fafc4 100644 GIT binary patch delta 3990 zcmY+HXFS{O+s0$HXswuy5+h0N8?~xxRBEPHOU(*u-A1+c{247`Cw6JIu}4v2t4gU| zBxX<)wObwb{Oj|7UOdl>>-t=;zSnV{$9cXu+6>wZsz1}vc2|zBrjlsT)K`ORuhV)` z164N~%zB0BO|3{c8>B-SXNGBc2xr>!-i2O|V2_r%)5CvmTXwGX9ouUmXL!H8s^>xS>ZPp@f#a>nvtJ6F+THR$?C6HS?M|W zs3Y*20!BV3e0a@OBcGeAnV}-eO44I|fD1kdjMbsxYk}KmcfOIx`4Q z`r`LqY9`!bYnOb(PFz;dqAM|_FH(M>fwbIew|sf|^TT6qYmeo45mRxi6!~ixg(S@` zoRO{?2SoMeyNNs4sg%pw$4;1duJ!7R&SCq;HXq?DxI7u!L{=7CIZw(`ZTm0AP|WD{HTFM zbf{x^C zM4|Cy=4g!|iE{1uW73ji^ zxt+Dv)G%}>b7V>E_>jcXl?uD%Va6p(U;JymSm6@c+jV`a;>@e0!RNc1Gzwv1?*YCX z69)LiM^o_8NAab3^v|+j1*cA zO$I-(z6j-qFL5tulcPkw5n!!<0ydvSDyK5pY__w*bNjRSyLzsyP{aFJCuY8lv}0;0eR~EvKvSA96bUHZ?iK1_ko?UV=)c<&s=LZHd z`1@;Qyi|0KmuO1-V_hC5g#VsX zhbK>PRH5F7QI%wSKA`Zv`me6uHKZc9~&?9&~fX9BZaUqFb0Oz zfnhuuFfePJIe3=nB|wrCWb24PKEUbNcP4~JfE{}Z(~gD&-zAxdr37sJK*hbSUn5+~ z!1HU9G&$&y=d{@POO6G2ro(tz&@K_q_K~Z}VFj=^qB(pnV^muhK2Yev#EkINSA@f*QSPDuQ1A`dm;#sVi9B}XyqYU_}JB#PLXBko($bqjbtYgB) zzl#(%mkejN(; zf2>ax;^zF+%mQ-0)WFh9o=oBNp0@Cpz#y-2F5!Bk?Z~DvBQY5&6ARqy{H3msR|q~9 z*OCfMw03kj46^RSoCmC_{$cB%@yUMT3Srs-Xwc)v4(Wm3*QM*4_GY~JTIQ_^7OV`r z8U4K_tRgXxZX6nafe3?uQe8l(QGqO{D_L0}qJ-`+g+GM)Ui~>7RFtre6=^Khlk4+> zWiFplW%`bqe^cFXLQij{nf2GikG~5^-8TO^oBvHmy7#m|l89uA)RhAvNT93|favR9 zei?-Wwf|paCGEYxdLwoH>e&Xy&p5KyW6WyKsSE?-Qd!Z97I&^A=0?=a8cjW$z^GBB zBwBWP`oB0N4I1}76jW<-0ig>*XTZ)ot{Nd~)m*^dIK*gti;E78j}E7VkMrckdyC)rGxg*(kce|m7IRyiw4##YwcFH zoPR%`3Q#R?O?GxHb>lQy<1)givRYXN#&)GYi8lsjUHCg=H3P#PU{Jba20xrn z(Rr8*e>tB7SMt~4_+c))#k^@g?mnv_2ApPmyClD3UEW&W?U_~S#*r12_8=U7c*os! z%)GzCOF3>!K63nq5{JKzInfQ{edOt;^Ci1QQZ)5qEkXMVp zS2hz{?^h-F<%E(=E2jM}4bT=5d*w({_WJQGq(<_BrkpxrL`ODT3)d@$L)8u}bbxha z`@ue_>V5U|im4ynzMqUF%4WEDU2z+C|8-CmZlRAg&8Tq#X1}FMIa<{_a~`w3E@kXN zcE?J3s3I5ja#%^j|5_5B2f_zQF!of~Z_BtT9I9w91OJMY z6t?LZ{df^F6}`fBl3h!Y3I{U@*8_3~tcl`xRgqCfX3_`JOzOHS%0U$W9?yo;+@(^%yooqZCA@LWtp3FyyGMyUTt^ih@_O&Jv9m~ zd=6pKp1|*T*O^tNdc6zcb6lIsl-B)7S!}_~K@W|0rA|T>89SDK-SyL}AF7=N=Tgx< z&C{lGs|7gTPy<6UHdZp}?cPRZMYPx@!l&wICIosH(^$X~9mw1FTLuqHDnA^TQ_$A>e-VH| zJj8ciA~C*T@|bv)Ai>noBP0=5tlITEFmubPx_yST6c9M@zM`nmu)(#aMYRHawu`sa&h&oN`zEV zrd`UcJ!N_YO^TL3Ao6>!G>3alB7J5eg-&m~#zfJo; n!7B_7{CBHTsSyvT7*Z_}%2dIr=?KLC_O>0tNoAcnj&S)8){3ul delta 3991 zcmY+HX*ks1!^WqXL1fTmDG_OAO!j>%TgEbWDa(+bvXeDSh52Um!zg4MOCnqLZAOHU ztz>LNwrp7H%o(BxI6=Pa7lq4<&TtWb#QcnE+}YOkRw@-JDn3ZYl-9Rujvp7sS+%ueql z{LQj&@XBgvN%xn#^Gk&gSaU4P!n1$<32JW_+0NeK1&0)B@Y_16Yy{-9WQINz_wN-C z&R0^97~u82p9fWxS7H!BR7cz$=G@IFs&%#G$a4M4>f$inwxhynyVH1fRl08p@tu<^IS+P=SX=aGNYyA(8N7A>ilRz;t~DEoYeuBb#Nz_NugO4zndWo z23O2{l&zSxN*}L1WcIoFv6(|FTUqWwj-YsMkhYL{e}v8SoJ=#?QrA+*=JLv-Z%zln z+c5i3DZfw$xML59k8RKgW(KUbdEr=J#$J`4RCDlupS1S@#OQWYB$T6uWmrh`Lu{-C z%C-6@t}S&VY7f^-ZiwsJi(^vX=(=}{5Fz0w*yv%W+cu|_IJ-RTd~m^74+3Qefk0nTeD)r&@hW7nrh1?0uupW0$bt_}Hh3SX8X4+^&o2tgbUO-SV+q~;Yus3a zZ=7GmrESi)9Lc2ZhwsU&4g$)1Lrp^xd)Lzc;9vN69FG-R}= z&0zBP?h7;J)))9~YJPCPz7dt*?Jz;s$R74CFy)DvCkXh6tAXZ zOh9vSJV?vPxNEIi^x=t_x7EAhg32yoca4wNg*5*a6bd%;J}jQ5=UCSkrB;|0XbLM= zt;%x@ty&43Z`@~$jkvHKtphD@JdX`DUr*P88u3M7RySjHS~+}m5!2rSA}|MdC$wbS z2(Sz%D%8zm1iC$9bXrI1`QhIF;zUf^xDn>T#<}h}t~3~<$djhbR!+ffC{om9<%Yod zts^Z&*2E6lyUlNP%H-D3tplB>Y`JNq=K)t@lnsrY$eAyAA7?#(bg94PqT-D8Tg;{D z)Vd(QXti|O0lHf5X|-nHD^48T|W-CW7DPF*81hmT05zI{gLRJq6LHe&g5 zmt>E{)@EkKZ15>`-c_5#l|Ln6G<|`|e?-ZtBe=pR6Urm4+W0iF`_prdqLk{cAbAUs zZTc5KZ4@$8G1Mnx`k9+e=8TY2xx8>_Pzfw`@Xm}WZskzlOdOjntQ zoZ@BP!N8n$y_z(+T1KZa={Mk5_%*Gd5V%7G1z7kNG5gqOh|hjUI&fi`sx&a6Kyopde z;K_Q5&{gpTKJs%rM0`uCS!-mL%Rsfgq-L&&_bo|A9@Yg8c=bP7t6M0}ZDx9G<4IpN z*Y7bL)^YD%-JSI>f4(sBQLIQga9KmE+t@2ZkVFjR4jg7o zg;0{%Qfus;j{GacX2$r)*_SE8KadYnUW-YK8DwVycu=R3Yq6B42mL#_p7hJ5c3fkc zLJeEK#a{AX$jIb;ZxKe6E1Q1S83`i&Ck%|>LJA==vUSFt!U#$@=q34qZBMlc;f0uN zW^EsrBR(%n?Wnuy6c75U`n*aU>0V{@&(}F$n0Bs+Y7?;Ag6{uU5P+51SvlCoK|zjK zLihHM3(4eMs**P{S*lPwi%p+QiL$La%KLOjwKA?G`?IcXm4A~APmfTlID$zek(>Md*-NyJM*{zy;r$0WP)b63kaL#jfis)<0XM! ziEJ#OB^K56l6=AY*VqQGnLx290)vX*o*yC(_=3WU(wSvRc}_}pg(YqH|Grf-Ga;G& z7PAsAx_kpOqkGIk<4@q=xv38^hQrh>9jEUzf;yxw$Ha-fmUdoNgl)?r}=oSt}OtE4d{!4i=&@JkS$nrG4n5YRKp~(8;s*gHE3q>C#gR z@LO_cUv|#S95b#q=fYS8be@IS%&Rvy9>Pma?Yg1!vuBbVl=&{*Eiqtg7}oC>1vZUY zXDY`uMu@sApZNQSmBvrvNK@8_Sik|8t)=6i6N6N#z!)&VTIq`w{2&eH6$;V?0~u2fN#V-5+o>*S8xgPlNKP+#st>UJ>! zfI1g~rD&25^j@E0&z)d9(!D;xPQgIzNh2qU{FKf)J3<%3DfMvW^V2{no&+k*W$@v~ zu8j=fJW3`B6m_OJPR!d?_5Cas66hD4ZHGMnc<+k zYwSjvBVK0P=wLx+kpBMD+QHb-!L1i1rQeR0_jrws)+-ymRG7&?<@b!zNsB}Px>3P- zT!ZH#y{SJ(ZI6R#nNthOkdvH8nSWG8`u;%+n+=7rW^Y-QS8B39H+dCgwUb^S=BBvJ z*VR0=H1L+R@uMgAcyHiGV*RnJZR4({^rUk*4!4&rQ{pk8w;cjo`ejSw+n+yl=IOL^ zUySLy6&054M%$~$74=|0SOoKueV6r;5mUd{Mq5`{57GOiqhNWamf1Buf8DAT$^<;^ z4Bb~Z1lpAXuxEk+Bi$}DedH=nLg96Fgrd8oNG7`|7{@W4!D%rOMh<$3Ce9Q#$VNr= z`S|axxOi!Y3Ol@SQHK3-^KzhC@d!WpK9$-KCL0VB4t5WBn{G>eT!FGFCUX5}NkrLX z5tGnsw<-ya8l&$8uj#djlFz=FE{LhPIhjBD$V8@xB7~2#0fkja8=HhTiErRdvZl>9`lXbSjh5L7#!40 z)e~2P6L$=?#(eJcC9SPy&(_;Fj?cnS72WC6MUa*dtjc5mMasPNzpMAFSZCKZ{@vjo zUrP)C*gwzAZ5n*v|HdElb~}I>@`V(5?sbHlz}W4KTKzvNZkOQO`xjkP&f{?7ZrPvR zw&hZVlC1Q7A`%LA%f34Mgo*vIMqTXvy?tftw;H3@O6S0EhGtY%mP$G2GG^_&pvBhu zSU9beRe1a2GE2$fc-a-4znrpq{U=}W&xJjy4uCCQ;bTg$F7hQwBDog!6`f+O2^S`7u9-H__8!iJiKcM7v6L;@MjX81-tAfEqR;D6Ry@MWtL zf1}PvcC=7zmu^0nCfgxRfcK7_Eg0nv7MWjw=my#1%DbKOX)HHWWdk^>6D~7&c3XI< z6O06O0z2HJ1V0BULCyUw$`RoRDmm{b}Ay%#36WVzu2PuMDz=uJvD?RNV!${ddANU zst+FUmwpz{%kQZDcu{NG-!fV>6$ oPxHbff&a!T9*i Date: Sun, 9 Mar 2025 03:10:33 +0000 Subject: [PATCH 280/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10046 -> 10051 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 3cc7af1a1..40a1156d6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 03:06:24,977 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 03:06:24,977 submission_checker1.py:3270 INFO] --- -[2025-03-09 03:06:24,977 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 03:10:28,187 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 03:10:28,187 submission_checker1.py:3270 INFO] --- +[2025-03-09 03:10:28,187 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 03:06:24,977 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 03:10:28,187 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 03:06:24,977 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 03:10:28,187 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 03:06:24,977 submission_checker1.py:3286 INFO] --- -[2025-03-09 03:06:24,977 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 03:06:24,977 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 03:06:24,977 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 03:06:24,977 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 03:06:24,977 submission_checker1.py:3307 INFO] --- -[2025-03-09 03:06:24,978 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 03:06:26,001 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 03:10:28,187 submission_checker1.py:3286 INFO] --- +[2025-03-09 03:10:28,187 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 03:10:28,187 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 03:10:28,187 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 03:10:28,187 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 03:10:28,187 submission_checker1.py:3307 INFO] --- +[2025-03-09 03:10:28,187 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 03:10:29,249 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74175 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421125 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421691 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28806 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -37,8 +37,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409825 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.434975 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.434975 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 5fd6ba795..81b8c146e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -907 +908 diff --git a/summary.csv b/summary.csv index af9a0f3d8..599342dc2 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74175,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421125,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421691,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28806,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -15,8 +15,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409825,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.434975,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.434975,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e2da74891..f00efc886 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421125, + "Result":0.421691, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -484,12 +484,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2819, + "Result":23.1736, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -514,7 +514,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index fc777e496d114b7b2d29643558fcf4d1b28fafc4..5f2c20292cd35dbd8ae23bc3a77b18f13fbf0634 100644 GIT binary patch delta 2985 zcmV;a3s&^LPQy;HbO;4GXxnW|lX(axe^c8?7RBH9Q>avVNU8=~EnEL6V2Y4Qf-!~= zU}~Qz$Cho?*p^3<3HkOT*~ZBC90?_xS+TQuAa?8A?!GPk=$87RgI^zJQ|;c5;vk$K z>Z)Prnm-?e!(cu>)Niie@431bC;j=bKMm*pq5jQ}^$`nrbws7f8- zXcP?mci~_;^XJLBlE|O-leGKsBv{1y(ZMiCk7fC2kv}@rkJbLk?`B(X9v!UOz771Z zaqhF0rSc^V|H!_a4i9xBlQu#7e}2`C%SapgqyBQ5T!mjh`oVaTq{C)qA%k!luYPK? zARS9n*Jk~PL%pSIUxQ&X$tu<+!Eoqj1FdU=Wt@bw-`9uKrh;{-vNy5HP}^v)q1uLR zTG{#dHuX0#&2_7*9=_`*{iA~@{HjH(7UD%eJ2PtEG}B=jWTl_7;;e7Ye@(&FV}X|D zk&WK{QR|?2pH;k3e3l{CvfxG*+~x|JX@9nzGI^&==!3D_hbvUHEzwOPv3DicMB?m9 zZivL)mE01Ex0URq+v4fApaLb*MTO|+=Vuky&nj-7Ros>-b~0`AOxqA|icC@AiA*gnN^CK(Cn#x&eBTj?$oDrQ5&6DHi7T#y504V^3-KdqL8Nk8e-hq#JoY$2oymeO zHsFaA&SgvwW8{o;ff8kiYwq%?u=doJ1E%6c^B2lsIz1 zg%Ti;bV&npDJo-n7^4i)@&(g~iZ8u-ahvrAVZ1l(uWER3?Vr6ats}h(2wr(P z1igQTqddnn9{&|AP?f=9T$KwN1tf2Y#u3->3DE$LOe2A08c8J4cw&e?Y&DEYR)K`F ztb)$=;1)|L>9R(|9DWW_JrX4!I$jWsGDOvvx&FTlHlHTNjLV_#FVgEViv7EL)o{~! z@w@4)CBq7MUT-?K=F@K2hRR)CyHxVW?#F)$AG2GQq!KkwTT)3L*8`PA15^?XP>C9t zD7P9)C02k`mQ>JDa%15PO1i8OM^Y)HdL#-bOe&HyW-fu3h-GfN%(yg-hWY_8>&UUn zyt(O^+B|!1xQ<X9gTs7f@-a3q)J{{feE!xGddrGTi*+uJEzDatlaPQG_ zB$#A>(w1PN2Eim60F!6{Ow`~ywp9iusQ|z%si3nx_yndb>9R&swdkthU{>}(^L zt^X-7)eo&?>&USZn5|tf4ac*JW=wy+r+=()tK4!Wn5c0a2_|X~OrilWi3Y$V+xq}! zy9`WH0f1RjL1%mL2~1hiC5^yyR*34cC;$^R1kCn}22(tB{eC9vNUsu;?Hwjf+sr+% z*+sIze4@}zV0jWr)Htq05;ce<(EyS}14yFAQxZw53`x?6^aPYu(Age*LQ;R0bXhMw ziKL9`ktlL^$waPX(yEuq<(G6STyCHgKqmLI)ohtflgewDKY(N%*;Tr)wDR|r4X3S~ zqFJ)Qo}-BQM*4Gz#*qdcqCq4pSag^MkR%#MB3b6rVHE(9Wj-ARMja^Wl15;E1fqH* ziXh2E@W7`|zm7!m@7N@%CBc76ROanFrrGdv-i}=;cicP16O`!vu%&&6Xk6*vAsU3T zf`f-?07{|(D9cPdtO9_t%*Lan%jOJ9x}*_=65Ki@3YV_TuLDchnhiUDKa(%i#`9U_ zeLO2mU8|5vw}49Yf!NZ@Lo}}R@(>MDS;5W2G(aWM0F`Bi9##QRS!RFfQPOd91|?n6 z2<}*dbB9FXjwSQ%z>a0j<{hN+=WkiQP*cyf%DZ~(hUvOSfvHD%1z4hA#Fn-mqH(3K zhiDMX3eFy;0W66Iuq-q8unGXnGJB5#e-D&&Nh7#30u~<@gW*lT6~BG$YdHxCeuhVnMRUnmRWpQ1(M9N z3OXiOe4wOD8UZpPsz;(=8A~*{j5TYv`21JEtR=}ROIb7T@=2GnE!(pS{s_=i1!SUQ z#FjQ6qH(3ohiH(@3N|060Wyh!#*}21*?d?dKxUcE#{`=XlyrYtFXg5sMD<7%&Y4ss z=S;I^o6q;6`9s^5wIo?(&NTBrAG6g^POD%wYnlaQqG!aGJ|CiSrO$_Gkjx4`AEp5^ ziGv2nEc5xWMu5yRpU?K-6Pe)iVZ8)0!RJGwaL%ma^U3x99}MP8HTsylMxS=8?G^2m zn5_ab(KKR9qYr=4xYFoDG)QIzqYu*nnM4Dt*)pRKs{qI>Gy0fd^nsEtX$1TP5Y=N* zKxP%Ak6E+P=Q%R}Q$s;5`Bj2)*W}YSJl8COvRwcr8b)ks@*x^mntX@`p{!u?VH$vv zcywU8EHnAAMgYn(laC1|A1LXPMw9_mg3E`7!Ah2iU?qQBv(4u@ER#`2r23)9v9;t@ zX#tz}_@wvEo&$c^X6_wMGd~*pC;oIAYlAQyu65+Lbp2whE46>Seh1W(JZ&$W4}^CFKGxzt*YCozvv*+`S!ILK)pF`<$+v}nsQV9#^bBRm>e}#Ow0;Ygwupi- z3X*T@*0X=ta4o0(qggn7z3S9*9{joVU%wi>viDyN>^Hw?<{Rx$OH+T8rrE7p$v&`d z9UUzCWBA1$ zEz%iEd|+Cz)@Z0v4<5~%yNX-Nje|F5EVv-SClRDQ<+et<5G=*m^BpMWv*^%K+!&`= zh^5dpQ}quFJY|w8s27{#w4ikE^Cq52xaQno4lW9k5_rQ{l@Z9!0q2w3B|HIflNcsJ0S=Q|COQFtla(eW f1m*Q$DwDq^76DavVNUFx57U-YL_LSpH5@U=V z$M)1dxd;eQ4IoAmj`QtD0*u8yN3xR5th}>%aG-VW?Y=@kccVVm!LJXqsdn#2aS+ZA zb=5F*&7Tj#VK5&b>NnT#_gr0zlm2|zpN4b)Q2*w~`maa-I`|q!f5a2tPqZ|_e>^_a zCrPr{Z#Lt>#Gm!!M!4|j>5)+w&H73DHX1kMMdbH~tDxDmX&Q#zob`jbzRs{8Rml;K zM!~>;7Y>#)f1a$9MEZdjf z(sOC*+N}R@sJC?OYcNbESz>Jx42OPpqIGSsjFWKo`}&aDBv^+kdlRb+wT<=~s%_Y& zm3LqU$tn}L%ir`pN!f!&GfVkver*ob2hf-f2QH;u|Z4g z$j;vVQR|?2pC#TXKFg46*>EEpZgUOIbUfQZnS4+tjKSC)!xbvpmKY|H*t?QzB5`&l zH$>v@N^Xh7+e&uIZSitjkU)uYQ6a|p`B}yFvx=K%6}M%Iol4uh(l*4KqEb|NqS9v- z*Uu_$P$56mvK%{=TC~z*e+xfTaSgvE67jj85Q#Wp?}$VI`x}w8cO~zMMDX;1NCZzG zi9}pMr$i!7eTPWI=XZt@Q=Hu{k%+TfAOL?Y^aj}lj02Ol0K&NS=K>|l5ZB!0)4)eZAOC&%+g10pclLU<7>d)?&^SmY%P);VIBU+P z>CaiZ(n((}{AhF+%twBduGjp##E+A^c==@(#M#m(zPq;?Ml*?gf3?VJW^1YDdG}-! zANg^*go&G*wanf&lb8!FCX&)ctDg>S^iZr_Mp@#I^Po zC5{-A>I)u!8qV_Jf{QMOx5bG(cO;S|MvV9ZCCU(I|I(=u$?@c_Kghn%?4@5{)~WW! zDUgeANf-?#clTz)ZWw!mrAoilJda`>rBp!i#>t`BavPrO6_Dt15?v%wTtIJ8;>ZOT zN`OQnkt7m{B#}rYiEasrBwUhM65hdZAki%$5yOFhL|2kXV#JJnff8lNrIh>6A+e59 zDv)^V?o5iUw&}DBmZBa<(L)pkL~l{zh>Pq5C4eH4NEC@gqKFb#T$vxd5)?7q6XQHB z3Ga9Ulr!1TB^}76sEp}hj50*e7fd4(UwZZ8HtP?=vqr3_TUU@kL zy@sQIJjXO1|0&L*GB}K@c0r?nB#=xai6k0NoT3j~4P%lekWiK- z=xh&ev4oZ`>qN}q=MdE+QSzqa1<@!&q`u6J|7EcGG%03W4t;--UXM}i-`%T*o6d{h zO=m3?R>1RmGq5$EcEdJQ?&{j5lHYbeR(P9#-LfQ=sBzkoN^-j%s3aPol4yWR)WAf! z)le!i0a96#prho*!Wp!5StpL9QbzSi6i%2_BxlTA0WT5D+;o|7X&Mdn17OxsW0iSx zGcdKe_uOzDyWm@-;#*J03U_lXTY`xi$C6;82Eim60F!6{Ow=HlW*L~60KqIt(Age; zd;(LpbXg|=ri|*5D7dLgG|F%!m*)Qgmvz)wIx(O^#B8eKul}Mrnkt7;Gl4t-))OboFX_X;KI+31$vIL#&!6zhTOP7s*(vwKa zs2+(T-!7TRl}uXoGP(ScPKC=2lmf`)ezuw|vuRR!4f6+(tfRV0-z%;B_sWLTR!-3@ zSzymm#C#+DIYi?~gAUOkk`*jEOan*~jU$mPbLp@IfMl6ZM}biXTDqhY*dKwY9*H7I zG7;SHsnf3`k^DP0Nopyu5|w#>`;KWgyqvdV7s?&?j`0K~dOvJw-ys@T`ge#1p{(HG zVH$vvXaLGG6Aw!OP?p(vlyupgK}(l(f>456heYAhmHBmG=~}a4=kI6oh1z&NtGthA zWvOcwQt1{@i9QfpT6u`Zm0ljAK`JY_d6)*MBpRTy%+SLU0F`By9wi-rH)qh&C7s}o zB{+9T6z*6u?+)x()@meFf`g({4 zv8>?iVH&`aXaLJHa}P@ZSeDs)6!?3frAs=&oe{A3kSMZ(tzz;~HM@MCh|roqX7G|?_%ON$TDxYFW7G(aZPNHUp5lF2lZOtZ}5!xBg`%Mx@< zu=qermvjPTLR61L!7`R;a2adXZ1MT8fLTkCRhF`5-sO`nWm~pq75ovPsS3zM$A~R$ zK1Ab6n-9?-nH6k4Oao*R4Uk!8^I-{q%rcvg2{s>S>5@)h(-NY8dL#4}rimWncnt7j(*-DqQs)$UpfK2p^*wW`iG_Lgd5Dk)9!RNy?Kqk=unPom7 zmH@~s^Z9HKK9LDNAJPeu2|gbZg>z;VpHFW5|6nj*s?o>XHTtw$ZLesj#B3FiiKY=- z8hwbyl|~<;K{6|U7=4%q$Rrvdv&`tj5&)THMjsQ5KG4!7oq(SJqIxU}$gE=YF>5yZ zJV)k#YAC3szDiK;nta-Z=NA2$v)L|y5)C7^H2DyXD@{H`gHTp5`7jMYNjy5HgtE-! z!#V*d%S=8dn0%n6OFB^oPzf#{8U`y_CW4i0%{HIsuuMjO8IkITZpYSATcrhT-s6*g zZ}uGU!!~ozaGLqi*gx^7(^wmX>FHWWZd=z6wz^XLx9ev>J;~Gl!uc@B{%Ey7T|coZ z3Fc!h{&W2-EIWG^mXReJjINebUrW9%{6pP;Sfrm&R;;cKA4cn^U}=je2%{kRw(dQ9 z4A*knKbnPQ!`G`pE$6|XOaJw&!7F?J)xdu9i)Oyj4z;xOM`>E!s+a5q8`jamqCfV} z`_VX<$J*2%rG**|N7thDbG25llW?(mW$D_NFiFDMRHF|HC={-t(U|O)wXsFQu9xa-?id)K!gI8xP zxFEqN5u`okwnp0!EXCOKEhy%*=+IGI8>d)^rQ2Z3_&3`CE(n=p1%U~F78u?GQ#%pS z5h?hDv_GyJ0fJ-bK!FY(=UI-=-T+Sdn+>CK;Z!mV7H(Qk;^f|WIT`!D&-!w-#{4<{ zT)n)FtEN)#PsIT>mEe>+V8OVi!k0xnT*rJ^#S^b3h{qx7H{Onm9Qr^cF#tWpJt-gM z({sB-O+kuTO3IWxw4|gMWKN6O*M8(x{~4%`L;g)X0S>vZKSA>cv$Z110tG2F+|EXm zge6-6;*;PdJOONz9VS5m36ozYIsto=nkFV2EA52YB>?~cPyzq|5dZ)H0000000000 XfPq|-+a^8%u#*}mBL-q500000WICkp diff --git a/summary_results.json b/summary_results.json index 56a8cfafe..6eedb41ce 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421125, + "Performance_Result": 0.421691, "Performance_Units": "Samples/s" }, { @@ -653,7 +653,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2819, + "Performance_Result": 23.1736, "Performance_Units": "Samples/s" }, { @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", From c6a64fc1f85182287f22c71cccd5fe88ebdbefe1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 03:13:16 +0000 Subject: [PATCH 281/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10051 -> 10050 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 40a1156d6..ad3b2079b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 03:10:28,187 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 03:10:28,187 submission_checker1.py:3270 INFO] --- -[2025-03-09 03:10:28,187 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 03:13:11,238 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 03:13:11,238 submission_checker1.py:3270 INFO] --- +[2025-03-09 03:13:11,238 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 03:10:28,187 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 03:13:11,238 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 03:10:28,187 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 03:13:11,238 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 03:10:28,187 submission_checker1.py:3286 INFO] --- -[2025-03-09 03:10:28,187 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 03:10:28,187 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 03:10:28,187 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 03:10:28,187 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 03:10:28,187 submission_checker1.py:3307 INFO] --- -[2025-03-09 03:10:28,187 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 03:10:29,249 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 03:13:11,238 submission_checker1.py:3286 INFO] --- +[2025-03-09 03:13:11,238 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 03:13:11,238 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 03:13:11,238 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 03:13:11,239 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 03:13:11,239 submission_checker1.py:3307 INFO] --- +[2025-03-09 03:13:11,239 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 03:13:12,240 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2647 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0735 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 81b8c146e..fb1248b46 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -908 +909 diff --git a/summary.csv b/summary.csv index 599342dc2..26432bd86 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2647,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0735,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f00efc886..5fe31f19e 100644 --- a/summary.json +++ b/summary.json @@ -339,7 +339,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2647, + "Result":10.0735, "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5f2c20292cd35dbd8ae23bc3a77b18f13fbf0634..9aa93be055e2cc7e868d58b9a6b160421e64cef3 100644 GIT binary patch delta 914 zcmV;D18w}nPQp&G%>xQ{c=3C11ONbf5|e%i8I$k>K>|K_voi#40)O3{TdQOdgsQD* z$u6QX)e6h1RlW+!!esH}w!OC9TJQdgkQQ;eGQXG+5kZ=9^I&Mj$mVwc89B**j>ypJ zzYpS?HE}u3hTQJ-ef!<2wdD&&cTyMEB6&5}oaIeedvtt41fHf)g5$Urvy1S zItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50e{Z@cLH>z0{b1f=~?f- zrE|hoWJ0ox8zGeiYnX4N&3VJj8O)n+@n3NDF<(TfTG@iEIXMt%u`>9N=7_=@OdefZ z84KfCl@UX{mhzLHyp-}Csa#DXdWoCU2%}^vWvO^=ysOiZDhtLrjrQxHsv`YSk^Q}J z&l~K8-oY2cJx4G`hRl2emD$x^>f-_=tnsi6R^&QhtF+)oMq-($-Ii+=YtA?7YeCEU zN;d0DDg@0pcL@n1!qlDox{WCnn;TVM8Dlv6+czLUJ}E&5vyd0B0Clkf6N7X_Dfz*MpUAv0zXxoKpCsfko>{oZSZ} zW@FcNNtX}SkUzzro0qp?)L5u~RSZyFa#k=4)N4pvm-zA$7S~eeH(|v)0sQe8^o6%+ zkg*PU6ao}u+>`Q=KRs{PsLn`s8xR@uGHvp1ONbf5|e!h8I$k>K>{3mvoi#40)MYLw^qp_2vu9r zl3he&suh-1t9%udg~{T{ZF_CIwf_1qLR!S>%KTzRL&&Wykb3}$# z|9ueGtclBMKFIA(-?!hbdRwCF9q65?LNQZRbDvv;*=nx1L9Rh)(Lm*Ie{T-r1 z=umV!=$CycIyf-Z9e-$|gPa?k1G2tFtb+31_5X*W`;46aKy(lNEz#Ld*Y^%T_|*Bh z>U>>wpg}m&Asw*LP7nxG9Rh)>Lm*If2n2kY&-@*#L+Bveiyib!AF2)xOmzoc*eOBI zjm`lJ$t_|PwnhHus;*jAg{WF`Bdtul_}<3??EV)4enxtKAb-HQ|4x98-FCe$+<;l{ z_R=|FD>5Nj#*L85f;G&y(dN8i<_zY|Ui=qaZRU$8RV!PNH75rmEmj8q(Hv2DgUO?7 zD`R0it1@DU*HUij$xA8Uk;>IHqL;WijW9}#=ANlsj^_4(`dgAsw&bS71`el z_q@Sg=pB49+(-jsWXQ}nP?=rrr9Lh|!Ws|DU`4J2wn__bWF(e}+HJX3vF3cEz818s zuVk~nq(abqbC-}HB23-M-EB;%*xaa+yd4{je*&{*2niW~<@I1HB>?~cPyzq|5dZ)H zWN%|oa&K^RFJo_VWiEJaY@L$LYQr!PgztR{qq{BHrX>}&18vEvfkIPSaud=f5w(7h z)H;6q%1+`WkW-JVnf-QlQ8nMIHu(kTrO`FYGlCM(+z6?+HTqaB(iuv8U|O)&XsA&S z9?hG(id)Kmje|F5EVv-SClRDQ<+et<5G=*m^BpMWv*^%K+!&`=h^5CK;Z!p87H(Qk;^f|Y zIT^dI%er#3#{4<{TE4!ItEN&9r{aK`N^r^@uwYzBQ{l@Z9eA^#?x0EgVypP=~z zv$P_F0tLse)aY@OJtbQLW(bpk2pW^LB`N{uliwvg0dbQYCP4uXlU^n|0f3X4CMFx@ u^ Date: Sun, 9 Mar 2025 03:29:20 +0000 Subject: [PATCH 282/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10050 -> 10049 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ad3b2079b..8a633d7fa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 03:13:11,238 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 03:13:11,238 submission_checker1.py:3270 INFO] --- -[2025-03-09 03:13:11,238 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 03:29:15,228 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 03:29:15,228 submission_checker1.py:3270 INFO] --- +[2025-03-09 03:29:15,228 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 03:13:11,238 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 03:29:15,228 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 03:13:11,238 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 03:29:15,228 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 03:13:11,238 submission_checker1.py:3286 INFO] --- -[2025-03-09 03:13:11,238 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 03:13:11,238 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 03:13:11,238 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 03:13:11,239 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 03:13:11,239 submission_checker1.py:3307 INFO] --- -[2025-03-09 03:13:11,239 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 03:13:12,240 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 03:29:15,228 submission_checker1.py:3286 INFO] --- +[2025-03-09 03:29:15,228 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 03:29:15,228 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 03:29:15,228 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 03:29:15,228 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 03:29:15,228 submission_checker1.py:3307 INFO] --- +[2025-03-09 03:29:15,228 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 03:29:16,457 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74175 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421691 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28806 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fb1248b46..62bafeec8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -909 +910 diff --git a/summary.csv b/summary.csv index 26432bd86..6ab496613 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74175,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421691,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28806,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5fe31f19e..a21cd914f 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.28806, + "Result":6.2584, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9aa93be055e2cc7e868d58b9a6b160421e64cef3..f67bbbfd0983e77d18a2297c6a1ca2c2414e9ca1 100644 GIT binary patch delta 1150 zcmV-^1cCd)PQgyFa|j80IaH=f3IG7&vwR4x2Y+d~Mfa3m0hMSMv8AhrXk2OQAsVEz zg0Y8bfJ&kPD$Be*tOB62%-y5F-UB6F(g<#hfWwDGkp*lOkB_R^4j)#5B(to7jtLGQ zD1YgaMu1F+>X9f|#S#s!V$GTzKK~UkYe}-oO4iJqeA1O{%l52-zX3E=0hwqRv8Bt0 zXk6*?AsQsJg3E_#fK1|_F(sL0E+5tikXh#PF~Q{nC0*7_xn~JcJrad;CKbs!)2!L$ z^Sx;P(5_`INmiLN&AiRWY&DeADsZTqW`6;hXc@7k&4*}QY4af(B(s9ehiQOJVxR#s z%WOWZ5g@b7=CeKcL?+mLSTBK0u=$WEoHMJ~d~*H&2ZQ-ieLm){&!^pLdqo>1W~+cq z^o-cj=R-8E^!X4Cl3Bs$!!$r9(ZFi9%;&=@05Z#bJ|_5lprlJ00XqRi^;i^;S%1an zW7h2Rd5+Bg)K5@LewCoy_4u?6&ozsnY!^U@ei2)Ge2B)C9v`AXC@Xk;m3y^3fE~7(dxg`?kH-FqKb^+fAb(7UYaO{QUBB1rO6}jSUjg+bPumOU!yx;c)&6w- z!m1>gkG1&E^{cS#>{VDsR@q>5wVe7|@@?TC>i)wbJwsWtx;A_mtzUwrEutWdg5=w} z_3SNN%W3~;77kypI<=e!e=hykuLiH|{Z|9~%`ckyMmyBf)E}j3cB@vh4?(P3M+b}k z*gx+_<6s_ZQ-72uYBU^Oi`K8zT76E!#p;u#YhS`731^SrCVqeDN7=XZ_uVKAlgBUF zkZfO1cJvps<`Rewe`ly^^d$iR08jz|01*HH0Az1tP;zf@b1!3Wa%C=fZET&A&1%Ci z5QOi23ZuI%*`}cpwgYX+sewXMT6z=G#u2rCkkmSU`zlW2B#={&tC{_Fc2Tw1sV?~i z=cUm#$}@rz(A)^A+Zw&Em+1pad|+Cz&S|AW z+Yl_p*z+wY=CkO~QQR1(Scs)-v1R<5wSWskW>`UBf(3^6z|>AebVLe1A?=UrMu6ZL zI#8g4$9b0Hvp0ZK{$|6dTsV~sgN2*clQ_9|UQWio@3X!ftucR&KUXhr{IaH=f3IG7&lZPc&0p*k5 zB|HIdlN=^N0SuE~COQFrlbI$a8)v9#^d$iR08jz|01*HH00000000000DysCliDUe Q0ke}CCnE-BBme*a02e4FM*si- delta 1125 zcmV-r1e*K7PQp&Ga|j7JXxnW|3IG7(vwR4x2Y>0hMuDkEc?DRaU&NNS9-?ujuZL(5 z%L>jOrU5L82CytM_pk~8%QAb90)G#bbV(z)GXfSL5=B<9RZKprW|z-%T>hNBk(cWA zQF*gZx{7t%Zh@J=G&q?i+C^+>@gW*lT6~BG$YdHxCeuhVnMRUnmRWpQ1(M9N3OXiO ze1D*%OBw+(A*x5BU>Qp^xQsPxw)p&4z^o<7Doa^2@A65PvMt-Q3jPStR0U+BW5kv= zAEI%k&4*}^%nCLirU5dEfyR_%mf3t*BS2=E&Bp|r50rFSFXg5sMD<7%&Y4ss=S;I^ zo6q;6`9s^5wIo?(&NTBrAG6g^POD%wYk!&rWTI!pmOdY%ai!0PXpqbbJ|Ct5GKqr* z$Sm{uuttE)GM~@(;1ikP^I^RNGQsCVqHxZv;`7P%{~rwIOEvnKyGEaOtL+u-l$fmo zGSM_*OQR3bxYFoDG)QIzqYu*nnM4Dt*)pRKs{qI>Gy0fd^nsEtX$1TP5Y=N*K!0Wx zqmNm$(dRib|5HOjE%{Y~a@XY3HayoXg0fuzB^pL-Y4RZ&SDJi?2BEBA@?jc)l6Z7r zx-2vKutosNGLw%9CLbv2l17vPRD#QghQUgfiC`sLv(4u@ER#`2r23)9v9;t@X#tz} z_@wvEo&$c^X6_wMGd~*pC;oIAYkz|<9jEB z^&6{_U_RF3KiBWVva@$#8Chk6(baP5Yst5Tf2jKpi}Va-$?DqhVYGe=mbQq3Fba}y z>(;Z^a4o0(qggn7z3S9*9{joVU%wi>viDyN>^Hw?<{Rx$OH+T8rrE7p$wEG`ZXF#g z`eXmRAB}@~tWEtHTOT#b}hVT6qCHHO87DY?jfy%BN2u>95LUPsx zn-7v`w|{T?v2KH23Ast$_dMr4hm@;>Y@=VWPH0u3G$AMg#WWYHsnGj&9p9tKd8#;V zwSo#A!J$=sQ!-O9ZQ<2w1J(<0Q2;4dFjJwu_ol$uu{}uYlHgE5+-WPR4_ANNU_;qA zZ2)J4++hixa_TAWfw7s1XhL!}Bh8O(2LNXnT9BZ;!)cP@v)6-_Zn0ocE}T;Mqk%>1 zNu1pWCuU>UbxD^G){sBNpPQGrVboZtepL)mU2;}13)E{{m-zA$7S~eeH(|v)0sQe8 z^o6%+kg*PU6ao}u+>`Q=KRqRH*Qm}&b{msCCJ$SZ7il=@SKsog{tT2;lYbLWfHC*= zC#e4bv$G<~0tq>2+ign<0085Yh9y@4=9AtfJOOZ%8zw;k4U=6aIst!^m?kD0lV+O8 rB>?~cPyzq|5dZ)H0000000000fPr9>*(N>#w38SoBL-$9000002bl}i diff --git a/summary_results.json b/summary_results.json index 62ee8fbc2..8658b7abc 100644 --- a/summary_results.json +++ b/summary_results.json @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.28806, + "Performance_Result": 6.2584, "Performance_Units": "Samples/s" }, { From 26534576f413ade0275757297493f2a575eb9bb3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 10:56:51 +0000 Subject: [PATCH 283/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10049 -> 10048 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 8a633d7fa..6ff2e1f83 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 03:29:15,228 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 03:29:15,228 submission_checker1.py:3270 INFO] --- -[2025-03-09 03:29:15,228 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 10:56:46,221 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 10:56:46,221 submission_checker1.py:3270 INFO] --- +[2025-03-09 10:56:46,221 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 03:29:15,228 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 10:56:46,221 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 03:29:15,228 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 10:56:46,221 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 03:29:15,228 submission_checker1.py:3286 INFO] --- -[2025-03-09 03:29:15,228 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 03:29:15,228 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 03:29:15,228 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 03:29:15,228 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 03:29:15,228 submission_checker1.py:3307 INFO] --- -[2025-03-09 03:29:15,228 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 03:29:16,457 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 10:56:46,221 submission_checker1.py:3286 INFO] --- +[2025-03-09 10:56:46,222 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 10:56:46,222 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 10:56:46,222 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 10:56:46,222 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 10:56:46,222 submission_checker1.py:3307 INFO] --- +[2025-03-09 10:56:46,222 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 10:56:47,183 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,8 +37,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409825 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.434975 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438732 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 62bafeec8..8ad23df1c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -910 +911 diff --git a/summary.csv b/summary.csv index 6ab496613..a90ca3655 100644 --- a/summary.csv +++ b/summary.csv @@ -15,8 +15,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409825,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.434975,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438732,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index a21cd914f..dc3154a2b 100644 --- a/summary.json +++ b/summary.json @@ -484,7 +484,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.434975, + "Result":0.438732, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f67bbbfd0983e77d18a2297c6a1ca2c2414e9ca1..68a8207e9fc828246a18b09e12d596c14ff5a783 100644 GIT binary patch delta 3083 zcmZ9Oc{~%28^=e>kqpaGQ(h6q+k3VQ#53 z$G5q1Uxi2`=g1ZEb#L>;Jui}8i}_8Vexnsj18LZ)^nxjTWc;yN1Y1# zJ@w1ZhAMJ6#$eTMA$zAJ(`$FeZFplra}&bWm9&?4{kTk>kL&&O9PinqO~Q>U#|4}? zvFrBP8A~f(o@;f}ISHS|GZ@J(nv_Q*$=-larAj=uUMS9ssrD{u`x62~NNCx;oBRyd zJEJ%Wd-~^{cHRKL&2yH0s{+QCzWc<-A*qV7G&KsdYwfdbP%_#q$-GGg(U*%w6vN2u zuk)D}+YW_=`Hw2^vYs$VW&MMKwp78(4o0`u7pV~5wQVHFfny)*)D|l_Ow7FTqHt>X zsL856{&-B~#n`{B)qKxlp#fF!LDZxl#*f@In*Ie$d7nafI65({E*%A)rvII=y2^Mk z=PjnurQjDY>iAIoPW~10Al=e>liuSkHn=}EmwDeuKk4{zq=MArF8SUnK=3^r%xTCL zrf6PUBBKD?k|}D{@Ffar`B#)N&`gSga?&qEK}{p#jhi-Z$AT3*{W6W4hA+y218!YF zK|2=@r;r(8XxQZZT_`xtl?Zx2anjhWO#lQ>(kzoh;K`xmB9s>!8c>n7cSaPsk{BT> zctwD7W>%k+7U&>67OJzZssHR#top5qcyOmDme>7zLQoXglX7Em1wRp!OWk|7v+y!v zL@TVp>1gw;4^Gqca=db-i{@)Hx!LjFoB64CpYV21_~}RjE$UnZ7HQI^i<8ZdW;Hw0 zbva)foEbmSqQEvon~RQ^{;pBExlIJQ7mjsued9@5SAV<|y1M1!|87XYn-PaQi|J|Z z8*>OA6!4DWeBNoE=SAPvp>$~Qu5mpsxpdXSvJNNM^IUOn_-sj6aX?W&ICr3r#Sts) zobIbNepWLuMsE^hYU=Zw8^LJW(3CzAO3mMwY7Y0M5V;oc`XK_$2n(jYt`o7lWw}5z zg-t3!(VEC2t~n_C$$8g0F{swbufbu%yX`Ka+vO&e8rZ0(B+SPb8g=!9=L@|S*2EPW zite0lRxpu|noP_zDErnQuDr2*D;`BnospqTa)-u)ZIx=wxnW=mEX5c`K|#^Kju^wf z(VC){#bXaqT(~I5L9kF73VgQ}KZVUq(riDzhb|xfp$lu-_XYcE2lgN3xpWCdSlHoyYY4Ff@ZjhAC zo;Tvv_-cuku+&9WW(<30J{T`H{ZMb2)g?1C=rmnF61u~1k9vs&k*3O>KO>~?VAM)# zV$H}rAs!HZSg$dZ(jrkf!@SJQ@Zg7~8S*=1#z-Qc=XYp}1Z`@ef{oL{XF~MVSl}Y% zpefP`g6@?ECm!ueXA};)i}K#yc=;oLb@FzvJ^DU;`K^l(p}Ej12>GzSuBzxe^($3O z*Q?qW4fpv{kVn%YmGCN1cwWFDIpFgu7Jh1JCfG=8aTrd%wv5Q5=sQ69t0>Ll6!`uz zS+P$i&e?vzG^uJhdpY?O7u*H3EBGMK`Iqn(40L7P$eNq4iMI})o3;Dqm$YE4Vlg!W z#`Z~Ppboi#1tTdCpp`@vgn%B1Pcyw)Qdr+sas(d%-j_+@>Bv2-pK3eN|WV2Qh=p&Jbd@Y0VS z_%U%TqkP{U4lG0Ur2$$C5WPSKDIwS-OLVo_<#oAc7$jfFr zVttI*r-uki zcK9mX9z@nMvvSE>mwyM5TORV9qU*yjb~iJ4l3@^AsQvezYy5yu(Cj4cTu?;?TCVgG z_kM7O$sW*<39EcpBk0y`0?pdeO z81r=EvWlSIz9F6|lsGs55l_%DsUvjeM`Rpc7|VJKCi;Uk{b8Zk*ft+gB(1 z>R2nIqVN1I%h`n-aD;SH#mYXcK?fCMolaa;5j01u>3OC+--jv0BQKh0`~3@RJcS`H zl9CZ$$~?wS4VAqbdK+W?XGgJ3!slnL%^h2+Rc}@GTvpGxK77RRD}=wI`BUGFct9RA4yQA_EJK1wr{ZCcv1dMcK+pwBft#g3T00%6CEItk&K6;_FnMfRoU#D@KIuTSnD2FpAYY2-0^_Cwgcpf3nyDN+~W+K=T3RhJ)2V0rRbczjyPO$J6K(+fJ>Fa^AECos}!i`mD1rIv`oR4 zZY7&0A9Tdib8SQ6fm3_DFT#pqBM(OIVO*|B=m9 z)06E6JT}yoBBcZVL9T4!mbdgUnK!v_ag%?RBBU7!T}HO^t9O#x_xjAt(GmGMU)DF( z6vuln+)@q>;!T^82*_(aP|J~l+$+D=8ixm%T<`7%v!GO=IbaTQWfQl&c>ts%bHG?# zwWBuWSm0m7`+B>sJwB=m?H9V3_&Tr3QecSmwpp&hf8Dv1IS}-Q_=KlhMU!7Pb!g`P z?5u;KpM&jEWu4`Ab+N79#ZSF|KH8I>?q>HddaQkX>7gSD;ba%vUeg*bE`5cx_a|0n z057ljo<8}ck*TvATYmjY?w*QW1F$W_Xi5blH*VA5?f=aITW8c;x)-YTqM5M6hKB6g zyxd?7llp_kcCOE8v^-Bq|AV+v?6^*gaARs~XWWi!yrTh0@-92e&x3z%({Vl#x90GA znhkrZd}+9V1IN;czc+gkGwU&kZTZ~@4L;^~PTYJN!G-Y+oB$$SFj`%g7&GYBwyALg zV8-|O+TkdE@#Bk8MW-k;WYKWch}0km(0H~YM~|1fifr}v2Ndpf#Ol`)6;7;+`#G3R z&WzpBg4CC^)m<-b+0k-Qb{&tqYE`st6`&U^uQ?5}Qz=0!aXdwY234ifIZcxMlX$5$ zN&i+uP=qw?c;xBHthBDlEze55wR_kCd3no~8W#Vv&Y9i$VyQd*B%YU3$)saukp_!% zX6Acb3^zZOBg;$_ zS+ixU>|~2F$WmDnn(-aa`JMB-|Gm$7o!9$)UH3V6qfwJlx$i;lpbKUNII90aBxr9s zW+2i!wP{*p7GUs#$40?s7wr%ohb1XT2s^lEq1%)142~r0C-83lmK|+-zbV0s%&@9p z1jsyC9ABwdV9h6QXNTEKJDV;~Qf6c_rN6(pHJ`369p3Dqr8VGHI`yNvMnsOaRjw^C z$9~T4p={Vw*!O>`#;~L0>a$$kK<>3a##une?S|2q1BbyO%CYHpS%vy~w^SD87KUBN zV?^Vpq)SQ8{RM7f4}6D&nA z+sJ z5k|-iXG{%84iu2?7tzmGbQ+JjmOcL*amRXe>(9B3^U6KfbW~h4}ffim}L(X0H@_uOccQB8M?EOSrpHEfY!OaPsSw>$h1VEilX_LuRb?IN9J7u?!mn9OSQgetYtBo7>VB5{Kci7ebJ#%Qr zbK&sRgpELM8m7L%+DVmBX|`4yL?=`Js@c73>FKd{_Ob!jCLAh8j-5&lnhs_QE6ibQ zQa_Ist?*0-ABowpUw@vp9_CikaKSb9y&tPMHnC5ami6GyhP_UMR@qMekYSA8m|0b! z+I_*o4?5plDu0V0W9Qp=aYS`Q`~Zxc7i-XRbi3)q^Vsl1uGb9TX^5?mB|fZ9lb$F> zyl%bpOaqPu;j)3jds%xM{M<1Zng4FEC*fd$~$9GlAs z-V`7?{GpPBfuEs4i?>W~5|ALK>}a2j`N!u3ILr1mD3ML#114)UsJCf%)R#&5MeQc* z!n)924sHgap~IVP0Kjz-@#vhEqc@lb%J%E>Z(_9f@;rmg;zj(hTs|k{9RtO3`FmHn z`=g>y*9G3o*f?;j_WW|0yTA8xR-NKY**d<1Hv=y0uZzglY*&%mJ$lMGsp*z&5_m|Q z8}K5x!G?eG_ChFl*-$5OEYRzJJ4(w_H3n-D8cr`Eefa%F#1Oqe*K3go5j437M@p;T z6n=UJGni7%Zs0$cg=&rIdj8{d-=cCf>uu1WcYnaIw+ki6WIPri{p?MM6+bGPbNN&)BL!a5}5w= z6S;u^pB!F3;jFV_Xi5~}ZcX8ZtpZ3b(orJwGy>v~!>R`htawlsL~Xaj-m1930gZf*<&zUS-aMDuy(k zD$ZtH{IkE{6sI3s-aRIIKXu7dHudH138g`ymrfi{coGJyNxaPLiYM zHD#|>H1w+$bzD50P6~^3gWz(Y=q|iw$vz^zyDc}bi^*AsJXknWxi5R(8G(T#9h3*4<3J2~Bbpt;nZw z^xPJYi|5fv6_IX4j-Gs1?ZuLP+4Sy-+{Q_+J>cCKVsj)Wp4;;%h4Nzb+nPkOwu{HN zeCtPMlZ#Zv+Y?nbUyw=yMg`=%FdOyYsJ0|Q=^PMxCQUD)LnQ%U5bu;u5R|qE1sEly zSYAQty%vf1D9)fPTj2tww|mkhwRVpxd0^?CxS{Th;Jzf!uHPl32B?ZTG|g44BEM^| z!qwyA0E5)Uam3GLDI?Peiqb;|>Qk{ni_ z-zrZaAS^qEmR17V-%tq`!@}>3kY?!{yW)nI<_{-o@D}y)ofa zts`p+XeML+lgYSPp=DIQM+Hdun^8$TblKTb$AQEhvDr9agz>}hX&qmtEb0-zh^ZTA zhqVVJe`TB}nzEgni)_`B_`@D&3{lmpAKo--tN+2Mmw2fR#%kX?sU?i%za(_;TBrWjdhJahr$DCl5w0__ zdpIwzHIvCGW4))b&l*cc{>|{;C*flcl9)maI-+ym|NYHFkv7m->I6~~>O~bt=|K6Z swkUOI2sH(TI`&_yPzc0$9|Qt}{Qn__+EYhRT2Ka6R8E$M0PNlT5AD_6g8%>k diff --git a/summary_results.json b/summary_results.json index 8658b7abc..c0de1bdb3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -653,7 +653,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.434975, + "Performance_Result": 0.438732, "Performance_Units": "Samples/s" }, { From 3401fa5afbceb23901d2845a83be3f2cc15a927f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 10:59:44 +0000 Subject: [PATCH 284/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10048 -> 10045 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 6ff2e1f83..187e7daf2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 10:56:46,221 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 10:56:46,221 submission_checker1.py:3270 INFO] --- -[2025-03-09 10:56:46,221 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 10:59:39,187 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 10:59:39,187 submission_checker1.py:3270 INFO] --- +[2025-03-09 10:59:39,187 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 10:56:46,221 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 10:59:39,187 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 10:56:46,221 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 10:59:39,187 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 10:56:46,221 submission_checker1.py:3286 INFO] --- -[2025-03-09 10:56:46,222 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 10:56:46,222 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 10:56:46,222 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 10:56:46,222 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 10:56:46,222 submission_checker1.py:3307 INFO] --- -[2025-03-09 10:56:46,222 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 10:56:47,183 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 10:59:39,187 submission_checker1.py:3286 INFO] --- +[2025-03-09 10:59:39,187 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 10:59:39,187 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 10:59:39,187 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 10:59:39,187 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 10:59:39,187 submission_checker1.py:3307 INFO] --- +[2025-03-09 10:59:39,187 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 10:59:40,155 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,9 +30,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.60193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0735 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0899 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8ad23df1c..fbd4b2117 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -911 +912 diff --git a/summary.csv b/summary.csv index a90ca3655..0db394bbf 100644 --- a/summary.csv +++ b/summary.csv @@ -8,9 +8,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.60193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0735,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0899,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index dc3154a2b..9e01992c9 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.60193, + "Result":8.24648, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0735, - "Accuracy":"acc: 75.600", + "Result":10.0899, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 68a8207e9fc828246a18b09e12d596c14ff5a783..0cae9c8b501f0320a7d634c4a51c6e7198284879 100644 GIT binary patch delta 4599 zcmZ9QXHXNs)`k;AK$MPjq=nF>h)4}BKmwtJ9*R`yU3v>eflwu(2%BCuZrby{g+BKX_m2&iXQC=*UF-elw(3w~=bpjn{JT6H zO@D{Kz>l6nV~nrDs3*!X1I~~yn>uq%a}~knc)}u{K)~M^-+{&Y{uWG8%3Gr*qh}|I z#H(`Z`ChQ&RoC=<<<>I8_+_QVzO$p9@XR6O~1+WaTf(l1Y0dW?&`dPIks%4ABM!|PK4&jgUE`cdU_a2}D5a!C z_>o#p3MdaH4FPZWiP+w4RJ&X)tu!-!UTo&WQ-NqX`PECbamfe_R}V&WqCdUA(uqhm z%tMnkN&WcP>Z>yukAP)(^AyxtQ|!Cf%Xq$&i4Lk`yt--f>Xcjadq0EyU|%FGA^kI2 zQiJ}N_1@ZazbVsjX(H1?Lz;Q;o_>EofzQLFHC_)Loln7#5ok<~P z>Qo>_^*I>-nvw@K`)zHusVD|$h2f*EIKXy`kT{%J_Py*!n6)RkYbpwLmyK2FJFzKvE@e62{ab#Q|d1EP*6 zn*WXO_(pG_sjh?f$fjr%c=1D{@H@zQjKR|^8^!xL9+o>Dk9Y3>8RYvcX$i(edp{bC z$xZA)+1_J6=-e=UoI*nyt-5cznR{OO%Hd+odi297(l2@V+rI4qRhM*Y=F%La^4f2*(^Vx!%T97vkhY{~BShE~!S>nq2j}B8 z&g3?hU3{$lkHL24hAKaQv)~}YyA~M^b9C5uufs`ed%q>e?aN|sz4l?cD*D_fUmDz+ zQz@EQDGuG_3Chk8ehRzl#$K=I@kJsraL`cWQZ{VXy^kU_a0YA6y*q)e@L_G4B?0N- zEezwl!^4feFUl8IP~GmSwq;}Bs`^{IW`^4b! zApEam{@pIWoq426I9E7GAx%<)I`+9g`??Fol zMX1-o?p$WOWrMDqV-;gUKFFMcqSIIMN}r&HJG0NZQ0Oz3oqC6&rrPRQ=Zvd6H1o#L!W7k*hqTVuN2G_qtV%^Aq<*6lMQy&w<&=2k@PK#8 zGH&9wBznrB`8PgkUfY{{*b)Q6(D1Y=VQRak+I@wfKGGiuM|*#>S~rfKWVv|01j_u) zQ1|xq4XAR}8Js2z8u=US%f}m~=@WIc2Hfb5G1+fTnrku#1?NDi;Pi|m=4s77pP$om zrouoNC^sBL3&)!%ao@@WF+6J$V}l)#!@9iCoTE4pmPy>sDi4bC5m>%>3M5U`3TZn)?apzxwC;5#+5pQ4 zsd_l6kAR?^EYn-z(fym_ab20=4b9pgTKT6~0~nd_$*($AX_fI;5l5L^l0=0MqK(?h zDtZ;=7>fHB#O_buOd`TIK^Nm)Po6nNXcVO?NhE==PNClMkn)F#@C+ptR9apGq`8OI zY`UCIJT_O=Fc#Vuo6HD(E1Y`fV|OkZJ@^hTzFNycT(=RNi2J?u_ZTEeVVTIV#*=&ou>f#R0K_-WmmkOwhzM&2kQ2Is3KA(`ehmj0 zsQba_QXN3>j@r zOA!H=1b|(<*k|~Oa3Fx#ZQA-Dq`9BcMBIO1_&%p0*=e6!V_&-y*X=6aEIP|ice{oX z)zwfGEuS4#GUK+<>l{L}f=vpyz%CXh*FuT^3B&`%`i>wGQ^NlbId)6l@p&<20+AR} z^^;qhlUTRG#I|E?{GO)x)U2((T<~J#pcGc_DTk=EZLGo;puk2E+-RqY)-x=Pl@YLW}9kuMj8X%Mp8_XRd^ z$yDm-??ow&HYKhb1!FVHyk|;m&dXdZy1_wlBqhT0D>f{7Sf_}7ZdEERfFT1aIL&u9 zM_KEYZ|XY439s0VOabUxa;5tBI8yQU^4m2T0AP3+N{6o)uGKI%IJ1OF1P% z{-@V9P_f<>1y6mlf@pQ#A5F^kVU=V1W46snC&H6le@dOi=MI5J+hj3(>Q&LD${zL3168yZR+G))@0IkMWOfK)UWvp}Z>0gYDPinddHAMWZKe0a^Y`g70 zdDbJ816G#5qZ73&Egp>Y|B#VN_^}*PQTgq1bMLO6?p}Gj(=^roRoyhZsFi7Y%y@q4 z`TNkj8MtHc-6Nifsysp$$ZVsgHQuXFF@C;c1j^`r5BWRq zFyfRXsC?tWhe?5_W)>iH6m5K50%tHNl zjh}zU^^e=!Yw0WUmu?N{%1zeGk?qBG!`oyxXMet7RVMXT-hWvn96LV5voq9m$)PZG z6RY(!d^(64FI^*iPeDojeINOk!Y6(GiR7nE7N{2f$ewec6575*&26SmB72FQzDAJo z9Xl;7Ls0j!!n#_mn%)0Q;mtt^9^GvV3&~hJE@zpHsw49;1D9jwmNCDzy~tZM)#$Vh zvXQnYM;&CHMdDgg_<|A|w{x{|Tw*<;rto8Tug!U+0F?<=+opBB})kiwvtw z?HAb7cn#5Cb?1`X(JPbl-u=^ScpBJ~vA`#iUhw!qzQ;nc>a3XnwpP%&fLNNC>#fQ8 zoq`nbN_L~ojMR_c(CC0yfry*wenPq(rAD@vmXxL$XmJ#0S*%Rbvu}CG4q8nCWidQn zB3>h_WX+}xT*UXfr@EjPyGk>V+1D>?`neY@R}dF%@m)ppgM_}IuJ+eQk?-F58BR;p zjJp3U+UXcSS8)wQO-r_b3cK^5dBw&tZ^E;})s8>yogKgPq~`P8PPc7g!H+I&1NXwJ z66AZC(ABgr9xKg!%iF}TRm+z81wRcC@8J8=S>#EVI6|kSqc`zLb(kvPQz5ZR9l?ep zf9BQpTf$VRMD~!QqX}*R**g8>SLE?PgDVNpJ3I&+xqde3e3)oyJ55#*$Z-Cm zWG>#YqtUW+%GO{bU*y$`9=avDB0HfX+CWOp@8~p6Y)&H+gI+uXu!4M7_qMR7Dhi^W zBb;dz=E}laxV&+&?c`+=E%c00b2g!%>G|xon1^8$LSgQ)t_$7Sfi>XA zQw9Q0kb947=REdEaHfvByWb<{M?f6%a)(pM|qCvosuYfsHR#PGiDc`GvJ zrHAzZYSq?Pp`FcAtIObCOZ335J%n`@K{)Nf85mJN3!m7E`N_GnG`+vOGlf`UVxXR& zvRIh$c%)}FGXvBpRbuxVT27A6$XXN7dk^wW?@|mZo7f1clV-fQ0@RT++mPKS-Tntrv?H@*B>{JX=gP-O(@s9sTPKa`euCod|*CJf3~ z@SA7sve^MZmYff~DI1vMDAN{rLwcEym1McU#C}Z}VK%(51__j5nQ0uHwyo|tm$|fP zZ=onF(m#H;5b;OftDn8v+QM>yzyaz7*b@9&E&ugcB; z8uUnBbKpGgCoebazv}Hu3;_J|@c(T5|5`+(D~<)M3?#!@fMtQfxJ)pZ^*;v>1OW7i d005f*Exe8qk>t*?`aghGwLbs= delta 4568 zcmV;}5hw1wPQXsE%>x8>c=3Ca(F0%tK6tY;1a1O<-JDyiWD$g_t!T+EqA}G9%c@nr z3d+J{@#MC>w%uCq{)>3=szV@PXFl_Hs1BinY%g}uFMX&wI55>6bYZ6iIX5~7EF`yxRoE8! zpR2lRSrwvc$q#8|;>Gtq7GU?k2=Ft~`vU=g&i!`+bfg0N9k}UP@4lsT!d7HLvWy!c zl?7{bm@!W&E;U0WFo<5`svL%f#q zlb*bk@*SyMO(S}Vo6`uRWGQ8-cx}9^(~&9*#yO4l>!7M4{ZWzqy>QPP?1kRJ7sEYT zFh+*Vd;^u))n4l30wk>Qunbn@I$*1`;6_GbnW){CYZYtGH|lFa%lb+->q{yG%{O-m z2_nMOo&36uDHWR=Rg$-3qw!BrO9KQH000080000#0Ah_MY)F%M2qk~rQ`<-u#oza@ zP^t2eR1LPeZGBO|6d{uYV+|%@k7vU}^XB^fo@W|y(w+6YlW-Osn%{!h z{PpNx2VcYJk9ZUWiIIOMn8k1ngGkX|^6mwy}6ae6MbX-vBhhi1bxzK;9JC`)XN#{GVfooLhOE#f4c z{=Pb-)(KXj%2~%MOKqb4nrd2(wzKc!+ay@W)K|SONBFLrbdL_A@T(Cmdx+=V?3+>h zTBoO_m$iP%nzMhg)z=MIj}1mzM|Sq^j~WN{`z-NV@mYpk%Z3}-aGPtWr{mcSO7lT! z7=yJvhAUJwZ81zDakeGbMB;8sZivL&mfRAFzmaT}+u-FkAb}F)qC$-G^RtTUXB9Wk zDsGDuTa`9>rA>&}qEb}&qS9v-*Uu_$P$9q6q8wY5+O&VtV;jFyaSgvE67jvC5Q#Wp z?}$VI`x}uow-nbAb|NiEHliY2bsSkN>{>?W%p+ zIeWca48`ecXgE$ri!ZfaIIT}7>EF|IrIUV|2hreeJR1a2x?T(Jk|0j*;>DNgIL?+n z@!h>$v+79{1j|KMJzGoF&)X;K_$Y|eB}`mjuVp~?Hkt2w^Z8zHzSs+AvxjIgOVY*E z-dc6n3ulAz(EP>h2T6B4iOn|$^#Pf#KUGmux@Zm3p)LQ}F|N<6ajDgl)eAI#Sw38F z(Z=wmIFaYBM3Tga5nrH0S>o(px)mZh9Nl$$+5OC3di%0UwKq(GoPSHgs5iR1*EOeR z?e!Kay{UN~#VSfEf#QvuL$Tr2d@uL=c8f%hljtFd;sSb$5?3y`Py!?pi6oIoB#A^K zN%RUxB;k_8g76lG1BqS%i5L!lBzlrW5+i2p3zR5JE~UJG4vAHiQi8-=Z);L)G_~8@ zu@v<=iaw$!AbN`uS6pN#C;=3SM50I}5=E4F;>!Hs7odpYz8L3eL3qm-pq$BuHt9ev zMP*C}W0WO&zF-=W_|mTyw`sQ*#(R_QGQ)%C{OomU73Gyc@XF62=+|6-<-6K?{Ig?$ zsw@uUvR%+9AbCqPuDFIzhz59M8VMxRNFs^G7pLgMM#Gq72_%$730j+j8!Vxv%{mct z_&G#%NR)i&ctJGE5~(k9<9``!K23@lm;E4^r`KZ?2Y2_X=B4xEchgx(g(dL3UJq== zr&Dt*mAksOspOB{j}<?+HCT zRDA2{SmADt?MN_D9-=BKdc0 zl2lS)DJt^{_8qNje$Lym4ds)J4@Q4g-p8}F)HQZe>FuBreISms@(_(Dy*xyNRF-h_ zFbz;iG(csMp@$^^DvK;VN;+=NpruVZ!5vF*?vNhzCE7(C>FOaGPuhBj2B|Dz>|q+9l4yX+B5w~%08|#adlY}zd!VIF zI>C(*aQKiYvVbk)@lh34Gte@X_HQXOo-}`C|Jc34X$E!#SWkU3Ye7? zS!N}x^Cq8kCEIX(d&mC))M|eRnP?bsq|1kBJn8Zw8YHuX%ZF)zOrilYi(Eb|0gzeb z^3mY(ftEJu1okW;szai1&ZHtaXX=VwKHrPx5A9l3Qe>GqQ|E0yx=~YZV~0as>m6jG zWyFy-AENQ3&4*}^%n~*qrU5dE2FNV3`LF~)W|7TjbMT2wu=$WqkW7ED`H(1_Gt1a~ za^wF8gZWZ@K6=~d(`E7v4c$Xj5yNgLo}ZB`4A0~S;FVTG(aZN0GUNTAC>^f zEb{qi@cBSXn{)zp0*LCcC?KKueo+qAZ{iOg=OWRIqiu*24KuW*_M(J(j(CX?9ch3V;9MQ&PG z@3p#8`?srCKt0LR{=!**oc*8G?qv1Cs$@JH8u6d2S7ABXtFV8pELm@GwU`7(@@*a* zn!&?7{f4q)O{4!XSiJh})3E<~IjF^K{O2Ng{i^rM*?-k@ z-uzw@vFF)C) z@g+=>aQgVuDCiOPgDCqc{e3qG!{qUY?38R?v)vM$0R(cNM*)+46fu9`TZ%B10+OO8 z7YdTC>=PyIHt~*ac`cbCf4_1d8OE|mvLR0tr>#EfR-cx-|80|{XbmM#Sbo~Urrkl1 zhb&6+)oJJM_T2D0hzpuWG-WxQb{>Iuz7789b1slvn)B1ngAm1uW%2LKrSsY zD>EwO-*RQ~q68Z82LOMOr53Sm*UD&;cTmWxT%2}@FIlQQ`L}}M7QuGV8T5HF=!?M> ziy-Y=V$ip0U<=Me$Z46dCVrPEf;Yh#@G2EeG)-wniQ(BxBM6$tEZK;XriQ=qmQ7o^ zqKdPdI8Kusn(!_A0Xesn9nEQ0q|joRj9Ma+c|nsA`U2+4L^6M|=&Xto@wNHokfp5r z4vz|1wagMBpllk>)-*}!GD#DmE_&nwvL<>h&qCSrv}7T0&J_Q}p9*6bLJB1n>c>?1 zc+O(+la@+SubEIcamOKQtrSUFl{8cW9D$J0D}IL3M-y{Q+pujLYjC2$yG=Zzg37v; zCt=kpv92zMEX#kGERfa1f`&qB&?q5>Qc4hvu&9jUQD!Q9SgeWZN~aOTR95mLWO=4sNXvH)ChfMueq!PC68qSuRVh?TEgfyNY&oNzF zF_pk`P^y2X1d-&l#ju1@vA$DJ9SyFO)eiSG zxUa#LEpKtGH1F zQ-gmzg4*9P3>;hTEo>sLiEvAO>N2_etYt{p?_;QCMrdhuuZu@;y=LE=#km;#2eSwnVF(2~NAvb3lWrj&e{RAs5QOi2ipclG4k@U$ zk|4Dery^A Date: Sun, 9 Mar 2025 11:42:24 +0000 Subject: [PATCH 285/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10045 -> 10046 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 187e7daf2..c6c235fc2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 10:59:39,187 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 10:59:39,187 submission_checker1.py:3270 INFO] --- -[2025-03-09 10:59:39,187 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 11:42:19,547 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 11:42:19,547 submission_checker1.py:3270 INFO] --- +[2025-03-09 11:42:19,547 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 10:59:39,187 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 11:42:19,547 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 10:59:39,187 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 11:42:19,547 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 10:59:39,187 submission_checker1.py:3286 INFO] --- -[2025-03-09 10:59:39,187 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 10:59:39,187 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 10:59:39,187 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 10:59:39,187 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 10:59:39,187 submission_checker1.py:3307 INFO] --- -[2025-03-09 10:59:39,187 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 10:59:40,155 module.py:5476 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 11:42:19,547 submission_checker1.py:3286 INFO] --- +[2025-03-09 11:42:19,547 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 11:42:19,547 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 11:42:19,548 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 11:42:19,548 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 11:42:19,548 submission_checker1.py:3307 INFO] --- +[2025-03-09 11:42:19,548 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 11:42:20,704 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0899 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1784 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fbd4b2117..d6904a94b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -912 +913 diff --git a/summary.csv b/summary.csv index 0db394bbf..00f1bff93 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0899,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1784,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9e01992c9..56da3c798 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0899, - "Accuracy":"acc: 74.400", + "Result":10.1784, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0cae9c8b501f0320a7d634c4a51c6e7198284879..4a04d976a1f180f661cddab76ec551a9f83e1c11 100644 GIT binary patch delta 1592 zcmV-82FLlmPQFgC%>xR%KtIrN1ONbf5|e!h8I$k>4}W2yKD5##T#^tPB^gcAd(&H* z-D;mA0w$mV%wW^YzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`B}>T@EqTK7kzuzjgU~!+ zVUn*##^3YPmS+$y{5g^Bd7Bk9 zmro*AW`Dkrx8=&@MM?dzCS|GVSe9#Mev%uFg+cizHdquT0X=3xmC;-@MoOCcLWa-d zq~OMQ7$)+W%7>IR8X4d0!Q|01%<-^x{ZB%l`MYZ({N)2pX&`7Q3t$kH*^;sE>fwbP zvJ}c*4v11mcp%E( zBIY{c5RbBlM)|F$*Al$)BLo8KJwYHK*<%C(X733C0h8+#fq>a{hCtkH$T; zfYM(gki%`rl@4)0b*BgfRQDQzfa=~L5PwkJTLc2CdxtuuTDAk0nOQ1-GuZ>ktAdu%B7eyv zTFOp_E(GOb!K-DKaMcv>#Z#|snWCh$Za+-bWti8~$(mnME_ZWouHuCssJ5adyNJeA zD=4Z~@hVsu#EU1V?Y6DfdiP%hERT|v@x=(K@RO7q2SYPLHn;oF$Vv8dREAdneGu2I ziOX&_Ga7L3!``|3lGzMoxbqx`+Oj=q$VIxrZNYbsnxdPgfmi z5RP<62Q0J`1OiouK%nXn2vi*c0Xy@Vw?lOZ9b|j4gMR5j)xm+O?w|`hC4b1d(K%ot zxkaqPw#fTj)m6(X7gbAsNGlUBp8K%?yZ=RipOM}l2ypJb6QFIi?Y`Fq@-yFkOJ}rP z(FsjcP68$i)-c~joAZX5J(xG&;=kbPV?GZPwX*qXb8;Y(d`0+==7_=_OdcIe84IFW zl~N*J3;9V;Ukdq-RIUaIUw@+JG=dam!pcOvHs00gNR|5IjD`DkP*tA%sObJ)u;&i; z0{7qxaSq6c$jmoTnO*IrJ}yAQ8V?J9MX&vGmE@eHG?Iy0ZMjyJ&G|-sEm%=s$!2}Y zgkag`E~b7+OLZr|ZX?FT=0=s|?bvAi6HrS71QY-O00;m806zfe7d`Kt|sa^_SrbQZ2! zkK*LsdpQ}~w$0kvV2$~6yk5S(538n9_ow23>QZpZ9k5_rUE=d198xJTu4_WZJ1B%rz=#a&8fDS2GwIlV1tLB9Ht7yT!oIt=+Y@dy}m zUw(r653{c#gaQS-KtIrNlQ<xRa0nlA=1ONbf5|e%i8I$k>4}XDBA6jV=E=dTDl8mP5z3DB@ zZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>9DDfnCC`i}!gxxHk?FK;(;!7i zqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8ux zLOzKp%YOqQZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde5k|5=$nbfR zmfRcN_3%Ot z*@|ju+|xR$J7z>A4ysJd=<_*ASBa2DdWsU#jDOd^MxM&jx~7o_FC()fJ*1IHlwl)D zqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#)moDzo)jZhS z9)D&!q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbKyC6y(;e#lD zi`P~B?;0;+q1Kz~4WZxINn?i~UF)xAd`ABfk(C!D7n0JT5T zAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o)g5#}t*#;G z2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&1DZDuS&w=MSohv zgvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%da`NR>>j=Ra?=LT|{H5 z6_!=2d=-?1$>Pavdu_Y5{`xOMTEywf{9;B#1Zl?2gP|28AGiC@$Vv8dM21%XeGu2I ziOXp|$n8$wx8JRLTcYdZqU-CT1Ao)-NQZRbDvv;*=nx1L9Rh)(Lm*Ie{T-r1=umV! z=$CycIyf-Z9cZG1oEx13vc5&Eg7V(=|A(UcjGX>JbPxS4(b-Pd_YOb!)cLsTd|h>* zK{(PO9k9?&5C~Kq0)eVSAW(G(1bms#{2i)8=pfsR9rQ~dstyiJbq8J8DStuEjm`lJ z$t_|PwnhHus;*jAg{WF`Bdtul_}<3??EV)4enxtKAi%l*PJoV7pnnLrHrBhnbWYfc zOh}e-Bc!rm4fAcZId7ObgL$(T{{>f@`65cy$`)kJ$$?0VmBD{BM-<**^61*iSQyW$ zj2Pmzlv{f8Qp$Ivay5xCW$429yP)h>@6aWAK2mk;8KLF_C0h4_t4YPy@_6r2U zBaXqdeiWht0XwrH7fA{PL4q8#lWrj(e`>=p5QOi23ZuI%IVPnMwgYX+sewXMT6z=G zCK0uMkkmSU`^rw@B;ZqztC{_Fc2T)Js3!RZ=cUmV$}@rz(A)^AcNKcytkMUR_`tMa zjnPn{4m?`ccO|!!8wan>Sa3mtPa;Tr%58=AAy|sB=X+4hXVIafxHV3(5KFhif0prY zwgWsPQZpZ9k5_rUE=E^9?COQFnla(eW1VMrvw3ELk76Dt6)+RmyuagxgBL-n400000NKxw4 diff --git a/summary_results.json b/summary_results.json index 573c733ea..805acc5d7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0899, + "Performance_Result": 10.1784, "Performance_Units": "Samples/s" }, { From 3a9c7fb1862a97af2c77d5f398ed2ae74e64be14 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 20:10:20 +0000 Subject: [PATCH 286/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10046 -> 10045 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c6c235fc2..7f5d3cf34 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 11:42:19,547 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 11:42:19,547 submission_checker1.py:3270 INFO] --- -[2025-03-09 11:42:19,547 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 20:10:15,629 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 20:10:15,629 submission_checker1.py:3270 INFO] --- +[2025-03-09 20:10:15,629 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 11:42:19,547 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 20:10:15,629 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 11:42:19,547 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 20:10:15,629 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 11:42:19,547 submission_checker1.py:3286 INFO] --- -[2025-03-09 11:42:19,547 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 11:42:19,547 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 11:42:19,548 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 11:42:19,548 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 11:42:19,548 submission_checker1.py:3307 INFO] --- -[2025-03-09 11:42:19,548 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 11:42:20,704 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 20:10:15,629 submission_checker1.py:3286 INFO] --- +[2025-03-09 20:10:15,629 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 20:10:15,629 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 20:10:15,629 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 20:10:15,629 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 20:10:15,629 submission_checker1.py:3307 INFO] --- +[2025-03-09 20:10:15,629 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 20:10:16,846 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.26254 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74175 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72104 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421691 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409825 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438732 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.432436 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index d6904a94b..6cf44bc0b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -913 +914 diff --git a/summary.csv b/summary.csv index 00f1bff93..644d3214f 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.26254,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74175,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72104,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421691,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409825,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438732,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.432436,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 56da3c798..ae33e8951 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74175, + "Result":1.72104, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -519,7 +519,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438732, + "Result":0.432436, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4a04d976a1f180f661cddab76ec551a9f83e1c11..3143fdf6c200d600a5626153a0594f65d58c3125 100644 GIT binary patch delta 2769 zcmXxmc{tQv8wYSBOOnAPimc5r*^*~ycn}$xu@y-MA^T*>R@VGN%uEtytdV8x>6tPU ziV$N7MFt7UmMyY0Cc9qV=Y7u~=UmtQ$LHMF{XKu228()&m%)d*E1(S~M&z)=FktlR z2wJptZ2iTR@?Xi;0@by?_blIu$db8SkIKI5*Ry=A0`z}LHzx6IE*sMlQ^Uc1C&pY-s0bG^1(45d#m~SoGu!O&uM*sSeh&PzP0G6JhYNA%F~FmE zcBmBh=$^WlcDZ*~;E%qgP!Fp5qOp76gV?6mVsH5)5^!p!HW-RC)0{ccUbw2GnFQXN zTC=rD66TIIWZC}A+o5KA?9Mq4t^Y)Bh|lUtV9z);uOuzv`}PWO?o4gKYwRpO;N;0& z=TA4xNEIs!?an%V|RaAtLacn=1m;qi6{+|=h$+iq0yQ?I@` zl}TvMUW0bgXFi*PPHelPsV`;s$t}Cb)%4}55vXl@|MhxR;|*$da1xlZQX#AoL6{v~ z%(mRND=jU~sEX=*LI)}b1_iDs30$%>zWaMANt~Cxt-NZtij8;ZikBGj&-N}WofmRsD=%qCjCtP2rVP&Um?WC6rZ$_@o7z&Xvj3>;v*J|{&2zb z`1u}1zXTEcG|jMLbHX6S%6fys^gKSeU%evR*iSiSpEmpoV7f{)UJVdvgh5v8^Mr`V zo67_g$F@WYq6W51M6Ikvqq#0i6HyKZr6@UzdkN^4^^kaoO1ED&x@AaL0TOWcZQE=I{g@J{~V#_>~V=PrIQ>@l)Z0gfYtj|mgY*3Zicvm13?G8Rjdtr_#k?sUD? z(fS+o9~cp?E+ZRVd((lQ5ru^fIH3<_=HT?gUH`Xc)^a#&)A4r07k*DV-s_wxv$g-5 z-JL;x&)C&ZJFSbnm=GP}$Md{wP!{#VHA|~nF9BwO%AcWg)Silf@&WQ^eguHkbMWtN zTPZ=N%u-RL7J{%-J_-~6+PXoFQjec( zQN{Z5FsW8WlQYqiiP?sg69bWF*SGH`ppwXQvcyTw@C3+p)z?=zp%5Z81q~&l%waIp7@J6yP8cB8q~BLX1*FCp&k>#qtRg4JT%C2BBSI`8oSJ%rGu^_ydmq)W{O#pA$?d(9Bk#Sqo7Y~>GeKyg^=r&<}hF} z>iLT0xjJ7{MawBoSlagh-76S`KCG7`tt^@tSXf}~!dXO1N>Wo56yKICj;>6#T5q^njHMII7R;q(QQ4}+x-Nqw0iD}q%) znq(~4a(}M?%LCf$?>rdy=*??++B!bq4pJ9#lDX1RZ5ycI{**UK4ncTID}#u9*|L-Co? z9VguEvXVEo`?i_y={JnrLc->ytsp{M8jmE!0BQ=@24@~zbzKhooq3awqeCw5F=rw8 z)hmp`%L|;Ak`SY0H@VR9tHn;na+@JOBx@_@Gm5O=lK*(`b(g-hUy@wlGg*s5G8+sL zxogWp0vEB(O$-Fb{?t;9WInAlvb#9DaAta4#=kM(>9 z_^VbUZNj8`m{x;dFvT^oCTr11X6J$SOGv;9+x(I7Zw5y3`&yYYZ4EyE`d=+Xx#^7| z#lM{t7UJAab76t4Hd!^wPG#k@cGEX>EzC#0QMsQt#+tlJKr#pR!B_16f?xg*emejG z<-m3L%XNLz?BE!Q)8i|$mTn;Xwy8oz%}pKrbkC4@Sh@T%p6ivhF8CgR z`-90tWjbjfjC{G4z>b>fd z=xo8>Q09yw0`0+rev`BZByN}+R)jzRK zk>z^Q^*>oS&EOz~tVBsSif2oTo-}2(s0Z(r`~jjTT>?@k^1N66wfT}u-y@~RIJSq^ zZn)^!%VYZ1cq34jd!$~iOI}Zq+g@o@jL8or?&<3GQru?lw~5u>kCIm?PAw@?2V6Ve zOQ;ioNbP%YqLHyFJV8hA`LX-7_fv&UQ-@T1nMB%s6Sd7xYPKEf!q455?(QezuKu22 zePInzO&T^)ab}6;tu+jVG=c4V>YH)mD8@K6)6At%tfbWY_#51f`oSM2KgXz!AU5>! zHoxa?z;D2&jy%9$L0a|kY5}N@ano+Vi)I?lt-fi%1#+UJCd}W9N07B9(fLp#agX85 zhA6vAB{IBnBJJ6^^>NsGZrN=aYMAEvsmo%8`6Ggq z#nQF)QJJIUc4^vGTA1!ouM>R|LI88URg0#e_ChdZ2am)w#hei4v%1SOu zXJ6a7FE?90`D);ZVt=e**A~^S#Cfbpsivrc$G$d zn#vrsidX8Ru?yIoue=KKu7h_l8A+d$FSI%>46XPI2J+oh#I)3U_MH9-^hF#fP z{8w_H>|}@XoC+GFKbPy~{NG;`fZ2m4$RjWvkT>~+yaDJi*-l;?6i!Z)m;ZO5jq;)( Vd-7*_Jpwr((WJS4bv?(M!pKX*%gqJv)_CS&71h?6jy z9-69VnMN?}Mg4F(JTz~v-|u;*k)+*ezdMem!J+vrNX%c4{&ny*ivLJPL690*e}ZXp zXpYi!wqLI&y-_gfCbei5OtT|{D4ulF?8kUmPiAq@?Jt5R{wop5eI`q^H}Yl{H~j3{peuP_iY$_ zO*TFoc`09_=#Tt|)Bd4p<Vz((JU^c}OoBCyT$0 zNtm5WZ5osA!=c$Qjjv%p9p#CQQP}SX`H41--aJX8$?wZUYL#Fas+?7dU@-oyg46Re|^<(_1Iu! zb>wI7{-|+Kzt0n|6rW|twQRVN4YwN&^=v%rL1{iH4P&sj$8d#;rY(j^B+jJj z$qkWs+mc%%@z;{AavQwd1|(3TTvUj0etuSQ{jB2VS;cLUVyn_7ue1sAT2zV(UsU?6 z;`&*|4Jzc9T9jj}QkzzKe{AD-Dz4$TL?XWT6Cx2O>>ZH^V1FZ$=CM4a7EMB?xZ&fyoF z0|{(-!J)(vC-wv-4N>nqA`$ieMkJ!%_bBnib@1U)LVh8BBt3{ye@+X+TaU*cC#W;o z(8dmYQNp>5>0peUaV}7zEOE_UJ`H?u^zq-9zg@LYJ7=#Ki=iZ24GqI|H2+fTMU(nu zoc%q?Ryx_ISr8BI!s#H0v-MhVmj+3Cm(0IR!X#h%BzO0A&8nwy5G)p1^?WT=KX0F` z;^QF6mM}?uwU*giK4-q`&1QSO*?cdWP9Ng=G|d)Mdn?slFPaX*q4|s157KTpPRut4 z^#Pf#KUGmuwrCBqp)LN|F|NvA51I30{ z^SzDVw_7B7oJ0>v6c^B2l(=%ig%Ti-_qf_}|^RlcjO$3Ht3sLJ9nF4_f+0+P2x1^CpY*Wb}yB{l3e9Ug! zl1kLLO-UtrTn|(d4NyrmKqYEmqFiezm6!mjEJ)B&a%15PTH34=S5hgXIwT4wOe&Hy z=0*W85zCF~GUrlTHT45vR#9V_d2=-|wR)4<4J`t#50eQF9|IcL`jaaTE&VoNi={YYJ4S;w2P1=ok&kWQG(X` z;1iOvrOig^OC)7fheVOPOD6IplXkUC&c9?+;d}+905ZLwEN07mnpA#G{{WIzR9EW0 z(%!tUthr6)?wTcc*mD#y-$;KB(YVr}Lo|qF34ens`TqfB>#?0l1d6JMdhY_N9&ru^C|3>d&hc$61^XewC@m& zC;dA_gHV=m@GuQPNi+auk%@;T04R%WJW9H3&Y-1DIzcGGtwW-4>B{^%uyn21u=Dpb z`F}!fJfGF3k7r@2YwV=b+d(DzKpbi1AsSD5d58w7EaB#18laMBfXX664@&@47Fl|f zbljXlOPh3pJC@+wAyK$v$-FzTV_C6z2dVt|Tb3`>)N`#iT|G`s+up8wN`D8HXcuv$ ztA}VjY3m^xq_TvuhiQOHq5&$4yge)dP=8tE?onXxftEJu1UE*&;X|Ux0=A6DM^$X{ zc@E2;vo-Qktv+hg>ys^Fy{6aTWNM3(snIRsNQV#6c+%lRG(aZPNHUp5lF2lZOkL#g zVF@IeMG0CO96r#}CY=D85Y-`3u!BI~?j-?;sN`BaXEB5RE5oK173LmazFS z4UkDRKxUE6ha~_qi)=pYgHL3F&3}hzk5W%8i3JXDqlN!6y-|ASyAUkJ*z+AI=CkO~QQR1(Scs+D zVt>o{H`@X(2)V}!0uwAS+yhfP5zzrD_=L1St{DJ=V`xBu4j$)Oj!)hIPWh`1gL39n zGISQMT94x7-g`M2+qTWx*NoBF+K@=;HzLMw5OeTLI#e+9f;zY?BuzK>-PqStdFGdy|wVCIoH8Uc-~V YCKdr)lhr0Z0kD%4CnE-8Bme*a0Of5x(f|Me diff --git a/summary_results.json b/summary_results.json index 805acc5d7..30dd14162 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74175, + "Performance_Result": 1.72104, "Performance_Units": "Samples/s" }, { @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.438732, + "Performance_Result": 0.432436, "Performance_Units": "Samples/s" }, { From e393a6f729092347889f108dcc8d7b210cff2d77 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 20:14:34 +0000 Subject: [PATCH 287/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 10045 -> 10050 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 7f5d3cf34..368a53f3c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 20:10:15,629 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 20:10:15,629 submission_checker1.py:3270 INFO] --- -[2025-03-09 20:10:15,629 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 20:14:29,381 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 20:14:29,381 submission_checker1.py:3270 INFO] --- +[2025-03-09 20:14:29,382 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 20:10:15,629 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 20:14:29,382 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 20:10:15,629 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 20:14:29,382 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 20:10:15,629 submission_checker1.py:3286 INFO] --- -[2025-03-09 20:10:15,629 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 20:10:15,629 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 20:10:15,629 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 20:10:15,629 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 20:10:15,629 submission_checker1.py:3307 INFO] --- -[2025-03-09 20:10:15,629 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 20:10:16,846 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 20:14:29,382 submission_checker1.py:3286 INFO] --- +[2025-03-09 20:14:29,382 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 20:14:29,382 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 20:14:29,382 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 20:14:29,382 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 20:14:29,382 submission_checker1.py:3307 INFO] --- +[2025-03-09 20:14:29,382 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 20:14:30,319 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,14 +32,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1784 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2281 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409825 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.432436 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.948 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1565 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6cf44bc0b..ad2b6c57c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -914 +915 diff --git a/summary.csv b/summary.csv index 644d3214f..059f3f673 100644 --- a/summary.csv +++ b/summary.csv @@ -10,14 +10,14 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1784,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2281,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409825,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.432436,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.948,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1565,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ae33e8951..1b2cdc7bf 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1784, - "Accuracy":"acc: 75.400", + "Result":10.2281, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -544,12 +544,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.948, + "Result":23.1565, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -574,7 +574,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 3143fdf6c200d600a5626153a0594f65d58c3125..afc5775eda2c8564a0f2ff2cfbef651fec887efc 100644 GIT binary patch delta 5313 zcmYM2XEYqn*T-2aTC|Pct=%Q88qp$3BCHm@_Zl^bURMiN)Llf2$VRt%Cy3}oj~+y? zA*@6Y@yPH0;(0N1=A1d_GvDu>duPtM?N2{FtsWr)lIV{%VJ3)#m?N|~O`SnPs-%hN zgreQ{UBQv(4O+B>cW@XcASW`BSNFxxN$PIB`xIB;064o!d20nfmEpl+4h*C#fb;U; z(>?L~zpGa733TG4X>1Tg(wtc+n%LEaFGY83L~up=8+Xw89M6 z#k_Q4aYYN;Y_X-Av%RC94qb9hb@(2^7!jm1`mwb{! zH1@=!a6VrpPNj3Ro)AV`fgvFd2cz{iE;aVmC_dz?%>B5>vdAG0md&O-OZ6W^$uy)T zXTXSX`(#?u%ywQ%u3K8KgrFm-9QE)Jm=t{Njdtr$OF@e>#aKEDrLmp69vG;02^arrCWID=N9SZ>@&6L~?yhXuG8Mla;$ZJG7T*Z*Ezr zS_BGRM$^-)Hn_#Y>uUA?I@Xe;7!luHCmylW9eNrA2d>(7l{+Z6xRC!|E1hi=)B))> zac1m^;r4g!M})~bAo~U0+NpP7%kP?5dN3?$V0^EWdM*3Dm-I-?*Td7$%cZg=Xseu1Ylo1G2xYPl zf~4|<6c|qxp)kQs!zTj-JeghLmWif%(i9aoAB&A5(p!pv$qckiNOSF*_3-l@SzIIe zkga0_yVOc&<13&y-+e-leneG__8V;Pml08ZN~PKjf9V&gkiSPms?QFHD0|Fh{Y(z; zUqbaR%1zjBTsX8u=D(2H6a%FtWk}$VrZ|H+hV!@Otu7{;SeHR+XNR@kGdTw9vrf_< z_a-li2uH8{0X_$f`xjIxZ+{+|$?zaI<24Z3FL>;0tts*D_0k@h(ounU!M_xha$ESg zvUEX24u(h(pWcNOSD{d3=7Ig$os_xQY)ncY-O^XgR@PP!&HBeif z2uMdjKtOskkIgpzV2E%X)0S5q zb1KVrVClj!{_3w=eHZ9`6*hJ+QEGSeVo6nsf@;J=(d}>0X;oB}&8dnkd1McQPyoY0 z#)|Az?hcYoj(m<}7Ju`l!9n_`v%l;;qx%Oc4tt~|QC`9UjK3JBtTS-@s7O^w)i%!L zKKz`l%ue!@or#<>-EFE7y9VS*+w<(7@1q0x+*w6c>PuX%y@|6+FO--F)4VUJvN(VKTY_vx^ru1jFh|EeZ0UUz z-yavHVTQFp$p`mbtoD73gG_ddaZ>wcp%$J$jjam;?ce(&zets=Jfp9of9o*N0B=r3 z5;wI(1}(hREEYdXi;**43~he&WQt)40(xflLKAY>xDkW+;65mr`kQsou$40JIT61$ z*g`)pEG*2>>k7BFWzy@OYAY}W8>=TfHnoo@x3a&$tzDS(Qu1%83YOUk=;b&Dxvfjn z&;Bq2({5iyBYMKyc|e2*9!is6Gd}PlL0wZMU|QKw2*`{04G6__!H^n<_m2SyDmQ6D zY<pxi}SK`#zuB8A8Qmjuq!&j(c z?$_Q|!c$(kf}v||!EKJy7(UXBSGb_YjIy^)S~JHwG{x$f%^jA*2ktH%`y<=I?dP__ zxg#9dUbrfwUa;%5g>Wuq-(w`6v5ph6bPSPLVf#X2-}+boiP|pqKBBGhkB3U9T-8OP zp?ZR;oC-L z)w6TYt|zSBIksG57AOdc_F;ztswlJb(4sGU&3l%mwL@36Q?dh}QA& z@vm}yILNl5RfW3 z6#+sip?Bz05TBr^e`k|`+5sR@HpD3e1yTMy$H=FXVXhW<%?2t*6arCA%QFRYPA*L? zc6ax@=|Yus6p;UL%>z%h0em;4R0JfR@3<%Zyh~VtwmWoUY%20chOmM(hS+^F+&q3L zld6wTy*{K%9J{98Y5KKj$&%?vvvrt-Lw5UFm3ld%3K(e=My1%f#lJE=v);*0q3__u z^m;Q=nQ+hy>(zTd&Ge$#wIZ;y@%`kbf~-*jv6Ue6x?x>W*U$p_sR(RnVOriLRZg_q; zW{B3c@v33VcU2Lc^;VF^7p_btqW@$zo)La_q|Gb<5Y**I&TM_VgUqZg&}50&#?egM z=-hZ9%^YE*EDG4WxY968wY&?H0LRS?#DqU2+&GI z2yCAbNVSbrbu#W0NN4cR`6JyM=fa_=f`8Uu_zx=O*-utx4EDbE;Wx*yLO>Mem zX<~+|b~%v>webgPOGXRlmqXd*c-EUg>9Bg6&$c7=FB{gYP%+ktW)Q@{UPK5%ktwS~ zUYX0_XccM)ir;@MA%eQksJF-ibl5>pQtZJ2-}DNtRKVx{ZfJe9^<94aFZQL9yTRL2 zd4iWoKb4@_@@%^++$>rw0dcoV;Wn!_Ecxg;CuH~#n338QSOPye=5wx-jzUZ9L$%5L z*g-E{jU4A98eDsm-gI&fG48FO<4%A_eo*aTQWH#4`sX}2L-@^k)ycC z-D8$~W5`XD7Wiz-C`_r~(<0@zI37VcrFGR;)Dr|*8a#vcEUK_6gqUw? zZ>#~=;7!dHb|Kep!Z=BqLUt2H;&NkbmjSMFZVhimScirTcBaeWvGPOPGs`dmontpNxNSKYu?H$Q>cD>pe7H#zM|SmMVbsM1YIjb`aX zwlyqjZl2$XKkwX!)zI*J&-Oly@dlr)OZXXoX;?-o!h)39cJcpQhs50)gs*sBCN9N$Vtpn)+d41*A!rV(vpfZ@b4Yiyz%H z-bbFuhoc!otr-lb6?KS+&V=>Bq zw)ZTIpZ*ep3zXv|n@u$UJm(VBR3qPP^77unO{R9_ZyS1aiET|tCG-t88SKxyHZx!j zC&UjrE&W_XS+842Ej#zxU;MKAnJd}Jbdl)gR8T6fHg%9uVb5v&uqlaZYstoQIWcD4 z#7!eoF}uXTTa(AM-N^M^nrwSmtI;`x3 z${_^4c&kE)38fQ7elv>~Ne7^@Fh+88#laoufy8l=d!S$zH?s2u7js`NqLy2WuDC*O z&+u(V>$N+k(1{YNWgNE9RkWm-5(`&5nRebj{Pp$6gE)$86>$gYTtAKo{|?H(JKjPm z=GpkE(I}c;l*924La`xDLj@s0QGt8>8!Imyj?+ zE2CED)p`{46&=%Sd;;|xu+7HoW7j?_~OTlNpZOG4?UH1-i^!-8?!%mh;~I#3Zq zdjZ%QQW8Vf?BL=-#|`J3DTTN{RLSHf4WB{W`ED$uGvcbdVybuE2?cEcYGFZwV~^(X z_4eGJqjcB3Z^`YL9+5-o_xknE14eC+F&S#QRax4SUj5uI6?Cd5>EikTwuqDofv}iK zl{?-Vi`DL|uAbPgHceS<EMtPnqtHjCtfT&c_|2t45PhU6Qb- zDEbh$+Q;!y0|tVH1p+*UCtrGO*V}>?^3)b#60H@B zbd~|xR!)?{>Nzp53+kEDNa?1B#;1hqs&+VxEwBHc-kzk$4K8G=Qg)KP3XYQ<`wnh; zb{lCzh7itocNzbsL}eUY=IiI}FWSdD&|CEEPQv&dg{RuH_lRaG{5}=}0d@Pf)bAE^ zj@I|jZDsNVx1JeI4STgO%(NZa)~SS?y>{(C4V}E)%=%rlzG_+i?0{xU*{S#5b*#pf zVF9S|JIiQBjxDwOe6)BT*O;hI{*c<#P(23kY`i6&*mk-DVqks@c3f6zLOBWA>~y2d~s2W-3HP6hV69ZLS8pH zCa>`BB=u$J<8)UkF&43omG3KArKYV(FMK?pmsc(*0&>f7bhmAyrd7Lz_~bRxE!bw- zr$^#=JLU}b>C9mH0khZe%TV*a?qu{C`pOR@nb{)~9A8Csn*?_LZN79YZ;b9!?7Un` z{}|)4t53htvJO*n&XlC{kzN#1HgaJ_^fsH>p`aVM66L8Lx+m6hiE}{A9odyQC_6d( zRwc4j%`ZBYnLigDV&-?WNr=j&azghTMYhEM+E)Ye$U7b$#8!m{^Fx){heOd zx7O3c-ki#9a=1CW`UmyD{5yl3=I2j=KLJx@X~p})e2+EOeh2HJU0W+{_rE8K_4tIC zjmj{t@Aq#0HU(Fnd}FtKTJ-Athoj+`WMytyi2RRMKgpeo4uRP`E#)ysaKFe~sk|ea zF~8Q1i-lz>DP7{daBm_1gQm|)S`kCgi1(1cgJ$iHD>=O_On*q~NSSR&fu&^k_A&0< zT*#;0A~exQvQIXGZFz~6Dj?|uHy=GDUg0Y|c+s>+zLwH?nSOQ9uZ4wT_y&SaJP-2wPGk7ys7cXtW z!_CG=Z8Y%SIXpYxR^86H&R3>l$whz=X-hvj;0|2mE0Nv<+6ltDzoYBr zE13x*?VJ=Ms5rJwlC+iyo}QOeM!ne57xQ7LG)-RHSPqtEUThp$n72+m_x{ro6v%3* zS$L?`OE_KlE$xAIS95F6_7hq2?hxbNJ06g@exW_1WvW*Rp#+GkxcE5_)prut0ePafe5Ed^hu6OT1ok*$tDz@MCTRp4mA|Jsa_&_`td-vfw* z8WC<`7NFq&9&j{_fzT7f2va1a#F)Tj2$7gH80>$M?zq{q#!SQH2dsj9xWXclCkWo>AmW>LOt;i?>0)n!a zf-EZ_g6zG@UcT`A-Fxr<{<%vomrFkR=6RA_^0ewU=~jNf0%}xv@B=q^MU?Gfp8Url zm^GH3osUgtICR)5Yw}SJlnkDWV+C=edl*H6TX&Kl)`>*_kVQSjR;YDOk%rS;=tWOu zca749lee|%w*yXzTbZd-!N^jcA%%W*HDMKTF&(aU1SvTCIwD;UYUc2Y z33n=8HcxYR4##r5j8Qz|mo#VM1IjrorVd&o)U>jWm0|~G3#L1RSm8UFy>aLi1UuW* zTFczvop0NfPtNp|a*i6^p{Mg#bliAc1!z$8sbkox&6U3~Hwa9lBcF-`j05?o+~&+m zGpZK-vdZjvU+yyTp>9>6HPgC>WXO9?(~}32xv=`}ySLpa;CMszAfDbS=z`1)eIl9=r*`zAm))W0TD2zR$6cn1fC_!_fmz9fQ z--@bfK^F1-3^M2pHJ1BJ>DeyXd#BW3WB`SY{(JZCdPONzIS-}S>_&JD=gWbAP4_c( zZhMvQuS!Pt$v;eu|72NvB`woJ5PfItO85Jyv_WXkP0L>0SW(cteam~NH1AQStq!9K z&rE#5N3RFSQKt;za8H(YM@h2oWT0Df-_JPK^j{b~u?*AaB{}z`c++Cjv`J)e&u4Hi zU@+1xs6bi9J5`-W-=7ln92}`I$V)HKa1I{i^%rR1-0hQR;=GZu0!9of_P^{0T1-Jr z6@xdA|0$yf!_;)qJPmQ@X-mS5tMzf*9sykCGJ0m5uJo@iY$yDY5kzJZr;xisPif3a z2E0$otWyr4c6&(~c)D2?K+S1oI;?LM&{HR%&%G0_g!5MsRj z_M{5X_)j$=C;1OIrT0b?W3Fe-dNc9r&Yh_+Z5Q-M>p?0VBV95$%}=o@wQN#srd-Co zshKa+D-QB>j5UNKH$Pt<>{Hdl#D(})P9ztnHX9AzBGnLTOLL^DXGeYX%D9}I$_~GP zh+=YT?N#|K`Db)lBlURpMa)TnzRBCulBILo%0=)#*}*i854bzN``bPJOq=yWc~ON{ zb~;@ev}(nfZ|dCZ0t6Og=~-`@`u#{POvK#4uE#m7=NW2jL9{4bLpy-~i`UK0_64=n zx%FSJr;PgAS3pbv0D$r$KLEUkHEXF-0{}C)FiKwZq|t;1bH~&{g?{Nyk}+*|PMp)yZ=SSD8}xKHrt$jUeZ8i8Xa6-)GS+pIAQAgeJI&X`5=| zQA0XIf4lOxWhE2A$FsZUrilz!qm@bK>p7?8+0JK6sIh}}sYCWfc@B$PcCA~9YjMNp zd2UWaGOkm%<9yiI&QL#Wkl6QIt6eDB8+^-d)zwM1QfN#%nq$}}M6|{Y*+>`X+=gg= zb+q7u4D9uJvrNHHD$~4Pi!MQJPyAWdBguF!QbvvnEmX z!}4tZL@<8qJ%dPS{Nm(Vw!w*IadBaKO?Yosb=4@5)+~`$%~Jiz{zf7@#omeVw&k`( ztaV>32YSpq+r6ZCcI=uKNzsipx<)Qm$Qa=%j)eNu@)1>LytKXI+b7a~Loi>GJyIv8 z=OuX}A@g{p7bH^k^HrDY_XY&L5}2$~C4&of;)!@e<3s$A%XQ*Kt3L0S5&R9S)bWo) z4jf;Md}zP$LAIN6=rK|cH{(?RCmhL$I=)RbMrCU^QUj#NjZv{yELMT)h9zjU9|Xoi zMEbq5HQLAI_#r+|?yEriH?AhfSBGlCX4b-?kT^SU@R#Wu8mK-RFyAYwDt-aC_>d_E zOv%ALD8pW}I)=YInl2HtL$iHzSuw@elKxMK?7o!Zo1R$7C({X#ekUY_!`utMNC?^q zJ?2UP@k%)oR#+Izy$y5p+NjFAJW>c!f9VJb2Ef= zWc#@J7mZ7GoU6F@P{+t0%K##cOEh}>$7#nJg=U~E=KDR0Jz!?}0}}(o23OjlJduB6 z;^hPHeM(0m`9DW`x33rX&yUn=y!rI=55E0+o0KAzzB&vm15F-bzAXghBA_i=_!$kxn4}m+ zwOc)QH}pW%O#H55+LL}Rk4FpeT;i-9`2~PX7!|w*45FG6G}C)J&5|4cx0!YEVm*B< zxvwuAx2zzldg-?mzK~`Vd-CB>1#_KS9ZP2@ZbPCidhv8E!0q~cs=`+90PhmfdcJWy z?;mKQI>;RlA06D^w(LjkrMxdBsuAX4|$Xiz9WsCqL zwEzt!Sz|tSr+TdR8TzZ!^}A`o{S92d7Tykd{X+Mb2MABWg@+ti;nYBJuawZ z)m5}b^T+$;Y`D#3ZlT|Tb#hejN7iN+IzgBE@IbMieffwn;s1w-zC;dD$-2apfD_2X z43ximG`os*7!fyJ2+w!4#K(Ty8{U^W`0C8j!@0m1<3%=+u7U5|gJDY)VHolx7`p=G zZZlS6TvY?BTfmHd?}2?1oCct6~iSnZNbY ztDPiu1iuua(SRPocR~SnZi|~j?&bZDk)Ai}+ur8KAE_Pt#doHXbK(_Y+UEWlms6QR zE#`(EQUQ9PIj_j_!5=?r@>f0}yD)`nk|Q)|+NjV@9eNb#fi*Tn-4Z12bAE6??=P;j z+TdKJ(%KD|8fsQXyxrZTr&Y3OZD~UQE;dApDjLp}2FMjAYzr(Cq@&X$mqB@;IxwcS zP>8hpb*RR^t$%7@j;3b&scA6xz>p#Fmu6vIwAOB<)KKpwZHJ!L(4zHw+kdy0YU~B9 zM3}HEu>9%2ut4FKdS$$K7dAK$)kUx;fenvHb;5QfrHhvHHgdW;6X-wXSIfRcYkf?R z8XCPIe%$*H@z?(n|NMvOdmvDGprD=YAH{KVdW+Y<5opypn=dGNBpWwBFvcESYRa8@ za9ogJUa@>vDsz0s@sa-p@9zcA|B#+GsRw~Y79~4Myo4FCNIx|v({wh7N+XY>yRaad zUjcq`)|krdG2~`d4fb&Z=&Rqfo(orFM}u*C4Oma9=BC+yfJZgaNW=@Nq4`VX5j`!| zqILd4B76{#sVd5y2Iv;{J#zIvx|B#3nnpj$Yar!fnEd)qK!<3~rYb4!iHOiG!nfC@ zBuVbcl3R6EZF%K;V2MGA8MH<=-C)qrf-*wGnJSxZ7?7!Q#gTdJLL^Uh_d>Pwp$h!h zQw@HN%aW--)5$}=e*0f^%hOVM?xuB;o20)Vf96Y0N4!gWvOG6_yI>&BHR&5fp1TMr zfg$e}-Z`oSDL5w&s7DV_SdJGkRx&CW$EgA{ou0}!*yjxRIik;t!=toTgs#rlb``rF zz5Vldd+0zyZADfPTTqFd@ghKc22_d-upj7 zF8@phf=LN^Q><}nmdE*s4_;&1`;oM(OQ=7bQsj%1u2OG1I$^8@F_w3U2>hsfO5sdoam!4cEm@Fd_uYRnJ1Qfgp}MnQXWyCUNv(78F?+ zQBcwo6Xhs$qC(Zk`=?5zg741UrWqL4X3nfBIft3~II2FmGHiIz>`7qykT*xwr+5F? z$NQ8!@g17YGV8x5-*PBY_$ZPxiiG2a`)_Xb*PV0A^ab~f$;_H_JrZ{#YfIz=C|#Gr`Q6pD;D4xy;da(EZL})HUH5v0nL*~c`@DPy!r7=>0BZGH8)4z@!Y6m7oYQF0$ zUB*X2ooO@tBB=$ock;YuUMc^!yoDtQx)p$=K)m^8r`HM+;R{){4<{slPOC%*W(Og! zrUnS6yBAq}@navCtQz#3EZS-vK2>rLvKU8vh81?? zs^pfK#=Z^D3|Bq)x^sL`{Ft8Ke>2toGdq58ZWFi@R*@jv#fq+kB-tuVjO4CkZdA&= z3kdNG6mRA4ZYz4s_~Hkng04RCp7H=);MYQsQuXRX?!1ZY&HUA|rxIEHE-q%cK7*$5 zpN$*#!@Q9PeKqdnK%eknc;xc&FSorHHumF`AA*=qlS-$a8@JZlw2j#tt>uYiCv`H; zH5NGt6+wc)TBGO`9_;H{R%ZR@%)k%S+j=y@kCiRt)m{syn}oTua}>_6?QXhyn?(yb z)wG4ta=Cgt^;z8z3LgKP)f{^_j8-VjGtPacBP*y1Zd-<7`A-!Cx5Z=91-6wwM>#7` z)bQWb&KHqLd_Gf}SS34^LNI`_3FsirLOic|t_K&QYZpse+p7?&#}B_eF7c&RSG(DH z(0F$Lj)^wWHfV(0LvHT31iVeftFSs%Y+ z_9rS7O}t6UaLF z!`Wm7m*o>lX+>SvhP)mBFgB-n&EKfUT06)AlKp9uL2QN>s?TE4-e2y^4IAxtBd*WU zecO)m9KVsPN1Zq$6Z&W2#EsZVo~^lY()QLEa*mamo=9gsGvRel-)>?8sQylY)4PBE zRdibB(k=aZn15>fgW&hXwa{uQma~6=YFf63l<3bC)bII#0DwLj0KjnZsdaqppzr(G z+wZo+V_zrHAWx4s*zb?p#OR-GhEeA?ii{QI-L1Xi*;5*+WMt{fIMm5d6P`386MoXG zI|G|GQe<&+{_@7gSd@aFj} zEv|^F%=o2ea~?`r+!L<={bC<^6occ2VI3?L6hf08EhW~{g4`s+9(35P5)XVv%u61cT+_GlH5H>7ao4a)KRkvo6Ac%M4?FGyPfO{JzfHI$IdE Date: Sun, 9 Mar 2025 20:24:50 +0000 Subject: [PATCH 288/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10050 -> 10051 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 368a53f3c..e82f437e0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 20:14:29,381 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 20:14:29,381 submission_checker1.py:3270 INFO] --- -[2025-03-09 20:14:29,382 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 20:24:45,563 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 20:24:45,563 submission_checker1.py:3270 INFO] --- +[2025-03-09 20:24:45,563 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 20:14:29,382 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 20:24:45,563 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 20:14:29,382 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 20:24:45,563 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 20:14:29,382 submission_checker1.py:3286 INFO] --- -[2025-03-09 20:14:29,382 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 20:14:29,382 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 20:14:29,382 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 20:14:29,382 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 20:14:29,382 submission_checker1.py:3307 INFO] --- -[2025-03-09 20:14:29,382 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 20:14:30,319 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 20:24:45,563 submission_checker1.py:3286 INFO] --- +[2025-03-09 20:24:45,563 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 20:24:45,563 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 20:24:45,563 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 20:24:45,563 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 20:24:45,563 submission_checker1.py:3307 INFO] --- +[2025-03-09 20:24:45,563 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 20:24:46,711 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.26254 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72104 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73409 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421691 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2281 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.064 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ad2b6c57c..f0a9f26fc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -915 +916 diff --git a/summary.csv b/summary.csv index 059f3f673..ffbe74b03 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.26254,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72104,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73409,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421691,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2281,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.064,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1b2cdc7bf..fe517d0bd 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72104, + "Result":1.73409, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2281, - "Accuracy":"acc: 75.600", + "Result":10.064, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index afc5775eda2c8564a0f2ff2cfbef651fec887efc..7c3697aaea0ce8d46ddcae3dca8ae0379a8fc013 100644 GIT binary patch delta 4184 zcmX|_cQhN^|Hosi)=Jd~HDi_<6&|BOpNJ&(sG7A$N(6168X-oX5}Q~-O9icHQKQt} zgj%IlTalvnt}Z{GbH2ZO&b{aUaqs)QU!QyKIrn`#Ogc>}2N~E*+*0cO83BM`ATgX( zfVjgT%{T(^^d$;13gU+oq&tF8@u_Pym@!d(1i8j0@qO#c3x#>rde%6ZK5sqv*K1$C zPV72$diA&LKjv~lFffUb2wb$bs-Wx??FEco+ASzSc7Q<)JK4V|DWPXY+G|JYirCiqZSkeGTLtLJB-2 z4)`Hqq80wE|3gDbDvP2-wZzS$R7ARq1d<%fQ#)O`81*%IP{IpS@LY2)XZ`b#;Lpm< z2bkf_(usYZoK#5fy{1px!YtzaoC0BLHk*-I)d(hGI`nP1w0n~rXg?b1*PsWdXa%Zc zQQxlXG1cUWJmVMF_A-xWfggExx-Ba(_EFpxbt0MHR9>yGxet0h9+=IuFLUR9OnL21 zy^#|0?w5L^GuFUW-2UHj!y+ zUmmT^c%$^}wt#Y%k8JBI)lk+c74=cIFsoi8K?#*I?d`TTZpNovt7 zx>YNwr2YAc0^+?>@}Gx0khoI>TrUM=i)R{J9uX>D^KHFr8bWsdV0>>>KHJlP*<5?A zWO5?N#?&)lWK%IpYN_F^{0r#X6U+Oyo#758N#07Vq3w+?)$tGaH$68{$4x!_Z^JK1M>0@TMZJDL29DakNgFTjVj zuJ(BN9z^Rsf|^tmR)|FV|Ane{soqbyt7^illZebtELKZI9)B6}&F-0~%um{jOEmwy z&2plf3&%by8eKl7iPch^1t@v9Usa83PF0ka(7d&DMCNp#!B#SeQxK> zkPDt2dd)7Q&sCEw+XPFgGdCUsR#k)p!WhM{-i}8y!h+sb6ET+<}!hQ z_P#hbdHQTm10PHrew>ErS`8Nm#yI(Nv@Qs}Y|G>mLJ28XO3R>Qar4}SzGHQ9?voY+ zf^>N_XEJc_;;sZ*uPD$C&ovjKee{>!Q)-l&+Cn_fkcSbA+j7GFNT3J7XKSx?DVm&o z+VX3Y5;mm~+H7-4a`K)xe+l^{%T6(8TzY9^1#G|Op6{urU4QXKwCK<$pLRzOc+FAe zROJE@w?>?qoraV>>(Q&UasRkqYxTP+tST-!8f@PIKKy9siFMq^J7U$Q*9X6bjz0Lo zHT(koHhapxbSq@^0T;)h)BEt0e>wwPTQpvB_C5Gr8kb(B@00Qnvpb&@9(MQ0P$7%I zw!}KA;;)C-UkvGg>#9BCl`LQFxqyn$6i*%JCKM$hx~}|aS13q&e9_zVVuwBdR=m)V zS|<}9<`On8Qu~o)8W;~h_)ag`4KeV$7X**WoWZdQFtOGZJQ+hH&jl2s^7VwKyGrJS_Gm&I*+hw2m>3ODy>( zS1Yp+YA|ZR!vs5%i;Lf2l#CEhMeq<{g|R?W2a?DydLHE&zTesktk@I!^c%J#QtW-j zFB8_B6RKbXd`1#-EeB=Ym=Bx!)Vcb`mtWfNhaWZ6#2uhSgJ@XM)d>e$<5bA6I=4uQ zJjYU@1AatCG}utJy#9%4EOU@M=mK~QAx`Mw`}@G|8#rD6O3f#h37cEUT!{GB zo%^xhQe;j>cB8!0B`Ga4tEU6b`)G^z?>1Q4gl&dK!~5E1yXo|0iOY5umZB15kT286CjkNb%YhK)rdu) z+m}nIsaM>0EM&==U^ZzZxwIQ!;q;2!1Bq)YD;D;ULk4H-R5w!a{g2AEAx94T4qm;E zMT_>Py=;%}`x-5P;|nFd7%lZeTTf1!$LlSx^i8&R8Oku7(4Q1U`!4 z&3DW)ywho@>xZXwS+|#MzG|#f+}1-;jV@Y^pWfhz2R2{k$eSo7>YpNsb{KmUQWYRI z3dnxh{?M1qYGgqdVQZ%66Si}~$o7M0Vi53L)0|amauTj2cxvfP5wWA5`IJ=>oOZS= z$8P5+pFI2XrsH+sG|{=R=b>aa{-vH0fw}G6r|&jH_Mo^Dk_q6c$MpKJU0TTlnX5`a zY%50TEIec2Qp!~X8SRcA{v^E;Z?-T}tIHlwnhSESPF&{LC)`qx+%yE6FNFY+URPihhAI22dK zJiU1zJ#3U2)*Dg?927}9*yc)la%~$jVRef{hws{6g%m>!3goy4KSn3sD0b(-cGi_fe zQ^yOtTpSWMUNUL0o-=U($}_?Qs6eLlh0wcVAX7VBEOVbb2q`s&P$9w7O5^3BujrAU z=1dBBc3u$~G!Y(QbbeE8x`FKFt`EC;^H15bGr+cL6I>i(%A4^Fz-ewVu;k>nmc@4 zc=Gpj=DFt3crG;m*V%L9vy1rzK7@s|DiBaKd``VD1_D<<9zLTsDx_11$DNrJt?pZ? zhJJ@hea9gr5a%P$5Xl1v75sh55`AKk2o0HD9rIoNYN8^j)dw4kl}LobmXKu5QBjg) z0@R4X9mt+y>we&J1whh=i?GB0gKaYfm|x{(a?e!$fnSYFgbt^k-LQ1XuObbDl}ZH*D#u2~kv-S-L$akG~l& zBUtXS-0s!XyRF}v z1CN{z^VD1Uepyvp+UyNt%*^7IaW0TI;Mi&A4T{uerzF$`A+}6 zm{8En#6o{&-Cx#wHJy+!cAp<0>mO4b6J7C;U035Ko6Cxy&Zj`*CfH>~^TmI?nFgP{ z^)ZEKKCyS_=Iyo=aSLvd^&lfu%yRJAYcaoDp7%q91lR@U zw~w~pTPZjiXf)KN{SL2%0hkoMsV;ANN-E~AgVs)U-Zod>({#=Ke!nyH@|T}-+fNhf zX&un(OJ5AAq15n3s-ThU)0^FQSe`HtaXTC1Uc+8&Upjq7`j%EMZbKTDu5HWU6M6^N za%+xD^@Q~{xA~cq6pne-xF=xs#n6YOnXpeP;EiMDaTST0nuZB@;=~3<_4f6;>}4)l z%Y7%}=<+4Uws%HK1(TaMzfDF07v@N1Z*<~882vD3g(<>ft!d>q+B;6Nqloo}^3+wk zpN&p8Yc_mCqCz6q82iT#J<`Q|qym05%TghZ6Ng7|0awiy1+S;^yN8Q`I0f!I!8;s^ z(qrAYpc`c?f_F@Aw}=EeHVQrno6qnWmL_}MNOj9L6&ijN>!s!B6o$dF2$j-5Y}3fn zIrpfE%^+2uEjd3~ zygJ8R-kXIe#(Y@HQTj9p^LHhW+i+b$_Hg0f1_78nSFudLXoaPDUxs>7_qZ#JQZ~-@ zEG7Sd3neUexY2kAEdXQhNEH25?V)O+{ARkmJ#|55T1Tn-XZw2eO#(S|h337{aDBbM z9kjm^R2vi~Vr^W!W!g;#$yi9&a&E6jmULw4YdZ|!Y`QL2NXEVh8^<a)2$DozbfMkB`Ys-*gy02K2QOq}#-f52d~T0IF-mAJbp> z|CCSEle{T-p1e?O0Kk$F0Kk7%-2b(RNDm@OQlE}SoR<_A{O|gm{S2H(w)52XzZMaR cB??KAnf^N>VwaQ^T`f^ST7uaNe7@rU0Q*|i{r~^~ delta 4209 zcmY*cXHXNs)(wzIQCdVwsD>mU1}P#%k)jw9Kzi>;kt)5HAfOPGl0*;@5$Pp@(u;r* zLKP7ZrS}ek5|AeH!kc;D%(pYUvp?>fJ?GxNyZ5%~e$su{$H2m9{H4wp2mpkEiIGfv z#9s_zKoZ?&dm=xOAFG`tX6VD~})U?^8UQcRSZ=0jcI zub91hxs@8n;ecjP5`KZ>pnR+h$w)=cc$ubJtKckNfNo8tUw*;^XU@%hR*>#7o}C7g zQ{X=vyRGN>PO#fqlEWcYqO68DlZHV))xgD2N!VS6^jjJ9Q&8!%E1?t zcDUe9mnyJHPkRO%3-KT+(JIlKU(v9hclfX3(9oaqq_V3sX*sd4T1Ud-bixS!vfTSox~iwGB<@H73SbU#SqDKiot1=_CnU3`POdB%IPg z9Z|l%k}~9Ip@QN%$JXaqgWYa=khz-|>pWgJKjV>go zE@)m>_K=*@lmRoF{gFZrQPdiAsH#e}Rm444Oz*hRd>7f0P`@5)gqGOI@T)E#%I^uK0nkcFBtnxv3UCag^CNWox5s~^>|QS zLrmD%>$^pAThAV=n2$%+-?}rzJqZKfxAMCS+pJx9jQ-%&Es5Xf?>22@&3?cjp%1k& zPKb(%GW9(s%`I6_yzuuf4E#9g^k_EhwxK3mC~#$IC`9U9e%E^GX+f z{IY^_E+5CC+oPLB!E_tm8iNz*AH=}qAx0>tawH7w3wi~{@g8#H6R5$D!IG%DyRdr! z{Wy)M58~17E4Cas-brmJXWkJGEf7Uk`Wxe)LvLm4BktS&F*_&!;&!Aa3_3lXps{cy z1kbK#Q&HCHl4?)M2jP3LZt5ZHzenbQfrztXBhgZZiqB;PpW?560ckyhmpAklC9@~1 z{5#sNEBoWNys_&i!&OxFW>%3;vgKRYY_-a(eAS{=EwN)c_FVF0h&)qnKr{CQTs>=o z@USiR-0b>x=!!=Zv&1Z?Tq>s+ujW?tb5CP-*R~K!9A(0 zKA~I+62oH@?lEj5mBrd|o61uq44eE7lb;s&#^}`e&-jkkYVI|3Q|(`G%~q9)!va(7 zrwr4hFf3r6R5ffG%I;-?+)Aj z10V7o{1$B z<1`2>7gEum;JDMX8Nto;U>O1Q4h#oFeIDTvH%Pa}YK5K%fXmPYKwRC__xur8_qqnh zOKbjIks1bSr*QRMy886uG$cG4mMH$aJ?(%jrN-G7InX~8^DA9SO^FEd8jQA1?8#v3 z5ZA7bAj{>>X}4NV6i(XmZryF{g$OAx->=XvLstM}%%a%TTbCrJhriFa3Z6A~_T_z! zU5r7|bz9~7Qm#eZlKUt+gcf;$0%PP_cyrnLSse)`B5#nTIWvuQj0A;9+UgX@>O-vP6KtS7(S-*InT zUD%YUEm=sJOu3-qQs;$%yH6D66O_xbj1ssr#V|+?_?ooA1G#5fBtQwWIHzEt;}hSr zY75cwd_zjH#rCL$4Mv{lNv_Y&rm^B(B2SB)U=-2=erqTA?#$U$-BIu3MY1n|mr)$_$@sMII zxgJ0G^TU(qUI{-WH7juPec#Q3$#<`8*3DZmVI-fefgpH>6Lt`9pL>Q3g8+!G0G)3$ z*J}-ch!Vg@8r>SDePAKdtqP^gOv6kS2NE|tMKh94E=dowzjdPV2*&Ds6UoqE_%F}eJ-W{>oEiD_fU2CB?@Kf1Gb;1%LLM%YIEEbz zXd0L@`gm9iL)H$DbxiTLmk=+lblf$Ug+@F@8`>grzuvV!0%N|_XKR43CBOy%1Pums zZig8d)5L@sOxOW(nVc%WWk^#l6^Y9~rRf*kjn&9@8Z1xmraTXjuqM6XQbX(wEqcDV zOYEsw6;`WE+|Zgd8#_4a$t>uS1EwL3&Q#f+9 zp=R2~5=$g?3sFr8qLL~a^Oz^>1ArGgqTNmq3}Of^8;GaGn}~GNGNk}Zo}-B$WJ+bG zX$O^==aNlG-r5&cdmEIBOXk!Zi%94o$_wYJ%W98A zWSr@OK}4#$mf9o$vQIo!Af77vgNUS$g>Vg8B7Ava3|rQ*c~5&Y2mZeC>eV73uijt# z&@UFgJue?*{;dW(rH%+g39M32RYxS8=|;{{9Uvk^*XYmC*EH2#$n679HWfUxitr#4 zfwqDl`MdDF)+;|bv6r?h9nkdVkIZD9oWQs^GQMzy_~sA9yDGcwT!g@?;s)Z~qIGiV zWS=}$A<@$)uH=zd{wsE$S){ZrH~O;<>*S(56Pf37t*ESld-@by1N{as$OG1vWb1$nuQq)>0ELoa$D z&(792~o*EPRHRcmM?{jHPm9rLR56?nqhs@Xtx4xW z>rqZ?M=k(S2*F+ThAWO9Yhxvo_R(Lzv=E42sHVJ&GpuHEHA%o)+K2M@Y|G@rlFJUi z%gJl?j;^*Akp)@HpAEC^$<*0wNZ08se2Q#B3{n`CopW;VVQak&3 zraS^V&87@^u_-Ty6x_CqVwGdYre)3vSZzvrLd$VJ+L3Fr4M!wr^vITDu`|lmEwCa9 zQZYL2T5A13ru(8{;ZiFiubZzi+W&>UTGe>?w<-qs$L6bMj#%z%V~K|!EvwH{JYL%m z_xcN?np8+5j@37+{{**&xsHzR0C(w!n4cA`_ln=vai7}{HzIg6maqRzl5G!&u=;YH zXMUZsxNiw9-<}b))h&E-@XN)tU!gp!G(z=PW01niVaw$o*?Opc7ig#SYsKs>j{cy= zmcy|rHbq0wTD1R_;ElS^8hQ_V;1Ba)|GKT3U8Y}B8hHOQRic>7_)9r@`#E0;@$!5{=+kYJ=7`%;zOxX1PYk5-7M2E}J*}`rwFzYS zxMjO)C9B(%keJ($=^Ac6%%$~}O@sA@3Gr_Xb#R4ckx!l3^+DXXahB$a=c^1_5%!kv zI$VqO8HFU*UJ|T&ut~9I_mTA?WHHYCn~Sx#nmCY@5vrj0Qq{^=g9+Xu^Mr!&d%gP# zsUk4$xiNkE>QK=L5tzx~e^76I!-N)|&$89+a`h`YjpD25^^K zrpzr&g)8xm*Y=H#+9w_O|7{2hJ03zsMMjJS|IPX_Z+A6 zQp;c9u5eWiG>)G6|2lu-E%;4=(`HPI1pt`W0RVVuzWv{be(1tPe8>bL*1#p`mWVTO nDF6RY$Nw1u0MQS9h=Pbb(0{{3q#%sxs)-OW(K9ZvQ=9()Iz-rc diff --git a/summary_results.json b/summary_results.json index b60b06887..711f99cab 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2281, + "Performance_Result": 10.064, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72104, + "Performance_Result": 1.73409, "Performance_Units": "Samples/s" }, { From 213e8387a5922408f074c7bb1897dd533cc938ad Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 21:39:03 +0000 Subject: [PATCH 289/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 8 ++++---- summary.xlsx | Bin 10051 -> 10046 bytes summary_results.json | 8 ++++---- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index e82f437e0..bc3deb517 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 20:24:45,563 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 20:24:45,563 submission_checker1.py:3270 INFO] --- -[2025-03-09 20:24:45,563 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 21:38:58,308 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 21:38:58,308 submission_checker1.py:3270 INFO] --- +[2025-03-09 21:38:58,308 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 20:24:45,563 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 21:38:58,308 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 20:24:45,563 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 21:38:58,309 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 20:24:45,563 submission_checker1.py:3286 INFO] --- -[2025-03-09 20:24:45,563 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 20:24:45,563 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 20:24:45,563 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 20:24:45,563 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 20:24:45,563 submission_checker1.py:3307 INFO] --- -[2025-03-09 20:24:45,563 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 20:24:46,711 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 21:38:58,309 submission_checker1.py:3286 INFO] --- +[2025-03-09 21:38:58,309 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 21:38:58,309 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 21:38:58,309 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 21:38:58,309 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 21:38:58,309 submission_checker1.py:3307 INFO] --- +[2025-03-09 21:38:58,309 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 21:38:59,278 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,14 +31,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.307955 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.379315 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.064 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71972 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409825 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.81274 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.432436 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1565 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index f0a9f26fc..cdfbfefd8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -916 +917 diff --git a/summary.csv b/summary.csv index ffbe74b03..9968618d8 100644 --- a/summary.csv +++ b/summary.csv @@ -9,14 +9,14 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.307955,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.379315,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.064,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71972,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409825,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.81274,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.432436,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1565,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index fe517d0bd..ee1542d37 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.307955, + "Result":0.379315, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.71972, + "Result":8.81274, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -459,7 +459,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.409825, + "Result":0.414194, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -519,7 +519,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.432436, + "Result":0.437507, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 7c3697aaea0ce8d46ddcae3dca8ae0379a8fc013..466599d06621006e9bd39c31c1bb5e1aafa304b2 100644 GIT binary patch delta 2752 zcmV;x3P1J3PQFgCa|j8P$6i%N3IG7vlY9tRf0qoj&h75o=;v;ApV7gu50kNRAH+!* zO%F{~vrHqH_M(0`9UhuD*YEc{(@4_pwBH>^)8Nqj79{4cNB=tb8pVGkqaa9)tb%EB zXpYi!wqLI&y-_gfCbei5OtUM4D4ulF>}x!%C$l){_7_2uaa~)MQ=fFhsky9SKQ2>8 zW;7Utz2IHcn@@six~wD)#@#gQeliMYiFtI;53_4|K1Li24$Wh=fAYI-n)RcDMccPw z@HN@^Y~-nYiK0L9FQ@%O)5@iF(7s=E<1#k-!Js=Ir&rO}k3l#brP;9Ac}OoBCyRfR zxd|kHCAF$x8LFIBtg_TP+OMgmY|75x@q_5AdbEo@uG!f*3Ewz zwXbzHEWNz+Q(m0+t-dO_dMq%qJo3@IKWZG*@AHaRiqA6SS{B^Mg4>OPde)zHr!?=B zhCW!^eYiqJ(-z$%5@%a-O(gEN>ZH^V1FZ$=CxCx!5BH?T%bf*;-0&F>iFR3iJr#e%?M=#m7OCEn$-SYAv(3uKBJvo9*>x^Sx*~eTe7NG+RvV ztyFitXgUao<}YSHNW0-UG2a~22V}bbR7FbJqBY36w)oGEetll{OD&fewWd}dj;}a% zE~m~#>cqYF7A3Ce#tBM*IwFzO5s9RZNW>kJ_X{xv&og`3*^C(u4N(mHi z+zk{PUd{J5{@!kp=y4J~BvIUxVGL6co9qN7fFhAd6p2Klh!RiSlf(=_f7LvdA;<|- z7DsSVE*KPOyd@e}+_@)20~9ii#E@wuheYFxf%&l3FcMh>62+nlTI-8z7@?%i8WA)1 zIYf0xl-z2(AR1+fsxLSC|1yYtnhJ9o`#~_v?#DO@?(S91%VxvxX0nnDOF((O>ewTd zy6J2@OKelgH`|XDZm!$5f20yMZc|c8HtB&%q5&$22B<_0Opj{~r4lPZDhn!TDS4c5 z1|@CQh%2d-Q5_P6(P*F4wR@$9Jh zfzz?VGaTEIV4}vgC77r|Fo_1hBpLt{H3+6I0uw7hFbgVZtuH=-e<@4atPucHMs-LO zY^4&7vRuif{y*TdiX6-AuC6+^ev-5swzFezoFxoLVd^pmzLo}YW?GO!8S;Dx(G(aUW&j6K0-W}Em zP+8>OQPNy<1|@CQOP~@=IwT6$uFR$bYuAc>I)6WuFVwyBS#8>P7EZdx&chLJ2bF05 zaHNBWXgq1*AsVEzgo%e~fJ&lqC6z@!9##QRS>)tVf6{1k1|@CM2p(C2VTVNFktMV2 zz>#Ieo*ks}=bu@=P%qE5+BEYxHEnyl9xDAERH947k!BvE@uZuFXpqVhejcU)Dv1WD zEVA^l3V_NYQ;z~y50tb?BX}|b#vT$yRhMvU79Xu^UejxE zGPT9Yf7ECfaiqnEXgq20AsQf)X(X9UBgteMNv1Bc_^=8jnMD<}G+2C~q)i$DG9ju% zqF@zEG`Nb@6> zVHzNlXn@Qjn-8l1$Skt?Xt4P}Nt-kRhn5i4A|X*YXHt=zGj+u_pYKKUhmI{PNwUnG zsW*K-`Oc5q*kMeQCJ`k8zLQK5Ab+^Fwd2JAy0L>yG>tgY=tDG~H2M$?l3Bv&!!$r9 z(Eyo6MjuuIkXdB((O~p}k~V1s`~(oyVNpP48KaM`*y!^dng6MwppyJbLAh=6X{x63 zcWnOX<_;*)Fycs)57Btil<%wngsDMI0?q% z#OOuYa4jR(rOVe@U8()s<(r?L)dGt7><=d{0#xPKQSF=kg6$PW}cgE3d3K zxSEdxBmFiD4$a_UmiU((k#10EBOc3t)qikcNm;^<6$^WjBzl?618fsX~fI7 zX)Qjd(QNU_HjOV)nnshyZ=;~w590h=_V;cOMd{<0d`Q-BB|G{HPyT-1NoVhcoTJB{R{epEus~TlViXu=< zbD_31dT&>8i6ZBz;uZ zI8+cf+Dhs}X}8!=_D#2d^EAE35C*5P%%wo|KQ`>A78@CQtK4oECBV(B>H_O0p=w z#*tV3XP`U|`8V+dIOM+m1kE3_wIa#_36#fPRYnQ`0NIm>C07CAli(#h0c(>TCP4uS zlV2t}0eh30CMFvKX}-ZF0RR9{0ssIJ00000000000001hfn1Z@CO!eMlNu)@24W-t G0000+K^ag0 delta 2711 zcmV;I3TXAdPQy;Ha|j8zF6m`T3IG7(lY9tRe>^zQI=8!TOFwr@eXN6DA0}htK8TYr znjV^}W|>AX?M3}?Iy^LQuHWx@rjexGX}>#;roo~4ElA8?kN$P=HH!a8MnRAoSq0PN z&>W@dY`{O={6Bm}W-?Q9S9U+1GejPiAq@?Jt5RO0!|J^N?ONP8L6t zxd|kHA+@St8LFIBtg_TP+OMgm*i-h z?Q5M4OD`||lo#iHtFH>K9t(^tk9_p*j~WN{`@G_n;+p7m$lDa|{j zp%2z}AFfc*v_&_G#Mzcy6N$Smxgip7TXIW(B>q~mm2QKl+kgs`NEa2NpP!#qTtBP0 zc~)^-q}a-|$un(2ycU_F!WWr7tGIqvaf1qZQj6-?%G9Qr9@}_M#WnnvNW^(RArdiQ z?}$VI`x}uowiOBaiA`$t%M~Nq{gAb1q z@(b}JX+flNS`glPJoY$2oymeWHsFgC&SgvoW8{o;ff8kjYwq%?13Z~K|Ht%r-LBQ)@#9C8YJmmGXF9OlYHru z+}+zXtDeR|uvlc(^R-m{ynV8YkAoyz!X)+8T4rxu^IdN?+w0Bdd(m|I5YMM+wwT&m zsqT8wbPx{BU(9}xcEfREzB#B5$aMXwij=ZNYmjwq@!yVqeO~rUEteRzrdEy~uQ+us zr_M#{#I^PoC9dek2}*!EB9YV)iKLE5#1)hH3o!-vGke+XlP(Mw9Zk*iC{~e52^4SK z4HO$*&G$Bb-)@oUaS}ZwQCyQ^3{wz`>;xr%B9TZGiA17^5>H%{#SB1y*Iebh+IsxA zV}YtH4&$O+&?q2zOEj*yhEIqFcw`z0B-2PDiN+U0^kJ=GOtK0jltmS^)(6*ELP?u7 zBIfXOi0Y6i`Oxu#Xp|+YzTD{l%V6_qQp~yR2f-}69^)jqyH_zA zV@p20nr=E9cNN=I^2hFf#|j^_+qR?3k17KE>W0`q#)iJet_FVH^ zXUAQn;#*J03U_mCM}mnO*Op+S2Eim60F!6{Ow=Hlx(H0H0KqJOsGzkz_yndbX|qNE zOc~W7QSeZeXq4qjF7^KbmsR9g<~i%CW9xfLyJ0)K*0Fq(^jP8EqwPvC$^N7x!9)#$ zNi+Z^(EymJ!F6n-2uxA|fLTyMYklwuOj*)qjd&7F8Py?C2qxLtMlc)yQ(&qeTFF+C zV<|8j+hAI*Z||CaG5MbUvBIr#+mm3T#&so_s6jA^2EZg50F!L*1DMSsFi8afW7lHOwVoi5g!?B<&(3Nh8t|P*g!{eeel?NmHpP-cJ^@Wj;+Rzovfx$ttodbzf<3-dEP#M$_712U9!jIf|HX zq(6sfTxrlD8bq>$MTcnsNn(Q063HT$4r>IEEb{3nFzP@_oApxek3dw1L=hyJ2p;%U z>DQ4+{vDftB$Xssipov?Pc zbHv+0B^p5->E$6BPnvm%2B|Dz=V2P4l4x8>Ws#$QhgASn7I}J;jWm&O%2dVt|dzLTM)pM;jZ9Ps++up95N`D8H=ofLMt%qnl>FXgHq_Tvw zhiQOHq5&$4%ss3Ept8u`qrl$-C2i6O?u>xNheVMTY#Eb}s@Uc89F{+4Z{($VeblDe zN9&q@*Yp~kOl@&8HQGfSY4IT%Pg;D42FPR@NhZ@sGMPq_sf#Q=tO7}9Q3Wjx79S{S zlSY6{i0Y6iSj7?zu3~k?7N7qLn3W`1W+kgPT|U`Lw&D2pj;B6a?I05!BaXEB5RE5o zK173LmazFS4UkDRKxUE6hgAS%7TJ6>*nFTWq)i%uO-qRCkSLrpsYuS5x?-Ep_oDej z+m@9iS!T}En?9fX$&cIEVNsJG5hVdWlSL6Af9U2;G8;R{MAL{PjXp%5-5)F`9Wb|Pb0GUNb9}PwyC~1>Mz)t{C9To*-mNEM1ij6+ck@=q*3M$F36qMU0 zpQhz|ddEhIZtj2*4I_>;`4EjKO+G|}P?j+HFbzOSJUVitL>HNSSR(*sk;zAc$p=c> zf20v*0hQqLp<%F+Wg=L~R&4Wm4$E|q6RCdaX>29Am0G}VdVI3`X5R%rY<=S$PLm)W z1}DLIoEW_*8?I&Kx^($st1GpCyL<=KlRRrLn)bu|k5;?m!PR^m80oiJaA*b(v+NAz$(lz0e_^nE3zjj9!zd2ZZ_C#6*Kp0p-J?m=f4%6` zd>Z~a4_?3Oy>j+n^_(}q82XKIXk@8BO0(=1t>hnAw~h{G-C=OvjfdehF~-3lOVp~l zrV%gStF`!?Mzh5y+cdsJX&Ox)zm0-!KZx^h+3&kS6s3<}@*!EjpX}%_P)h>@v*i+n z4h3&4081v5Zy_Fk&2GXl5QOi2ipclGPC`*@B|&N{PDQFzs-oU3djYHQk8HPrx33c* zfW)cC^~`=dyK*(#sW$lq=cUm#$}@rz(A)^AH#PcL&eI2!_`tMat<-)0C*ju=1J&B`x=jCYZx-RR= z!5Z`Dc(r(aA68AJ?oY)5HI?9$J7B@Mroxv+JY36sSi}>rC5Wee)Ni~U7diBSNMZnb zh-buS|&OHf0LCaCIxRS081v5z$O&|V3XD+ RJ^{6p6(=JGXCwds000WM1=#=q diff --git a/summary_results.json b/summary_results.json index 711f99cab..e82ea3fcf 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.71972, + "Performance_Result": 8.81274, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.307955, + "Performance_Result": 0.379315, "Performance_Units": "Samples/s" }, { @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.432436, + "Performance_Result": 0.437507, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.409825, + "Performance_Result": 0.414194, "Performance_Units": "Samples/s" }, { From 17c4c5048a7ebea4d9147155ac2ce696f8f5e3fd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 9 Mar 2025 21:42:19 +0000 Subject: [PATCH 290/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10046 -> 10046 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index bc3deb517..da5da37c9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 21:38:58,308 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 21:38:58,308 submission_checker1.py:3270 INFO] --- -[2025-03-09 21:38:58,308 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-09 21:42:14,965 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-09 21:42:14,965 submission_checker1.py:3270 INFO] --- +[2025-03-09 21:42:14,965 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 21:38:58,308 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-09 21:42:14,965 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 21:38:58,309 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-09 21:42:14,965 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 21:38:58,309 submission_checker1.py:3286 INFO] --- -[2025-03-09 21:38:58,309 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 21:38:58,309 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 21:38:58,309 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 21:38:58,309 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 21:38:58,309 submission_checker1.py:3307 INFO] --- -[2025-03-09 21:38:58,309 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 21:38:59,278 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-09 21:42:14,965 submission_checker1.py:3286 INFO] --- +[2025-03-09 21:42:14,965 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-09 21:42:14,965 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-09 21:42:14,965 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-09 21:42:14,966 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-09 21:42:14,966 submission_checker1.py:3307 INFO] --- +[2025-03-09 21:42:14,966 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-09 21:42:15,945 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.379315 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.064 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1188 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.81274 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cdfbfefd8..6a21a128a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -917 +918 diff --git a/summary.csv b/summary.csv index 9968618d8..a146afd38 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.379315,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.064,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1188,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.81274,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ee1542d37..e9daf53bf 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.064, - "Accuracy":"acc: 75.200", + "Result":10.1188, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 466599d06621006e9bd39c31c1bb5e1aafa304b2..dbb6ef8841d9d0a105e28dd99abb143a5566489b 100644 GIT binary patch delta 2059 zcmV+m2=w>9PQFgC%>xQ-9!>&r1ONbf5|e!h8I$k>4;*2kKD5##T#^tPB^gcAd(&H* z-D;mA0w$mV%wUrn1RH-#$rCMk!t#+}w=ILvJYZpxuSUk-^V61R5H9>Y^i!78k?~5o z@pb&y@R^n0c}%Gw(m>8fMl3`zFijrBH1m0z6*QMmB35R;khkT^_EVL6y;5G)79A`a*`!jvuXTjSzVOGxl0B2M_P`5Yy(oL&Sh7li8ogJ;DmHUHFxw0e znTIOo)3`G;sH|T?*>fi?aifwyL2=U*0zV4j_86IbVP3(G1n398)7TU4jyGk zlA^2*>#3M+-=A?0s(LT0)c?iUm}phZOD}laX@va2n1C38i9c7 z-XIWA-CG0#s(Xh(Ky~jC$Oqze@d@YZ20-nPbVvtOZGu2RzQ+gzfZK)$C41k9$h z4Fp+;&Jlk)m_`>n=$D&koAvw(4or0iT~MoQ$hko{V6GqtFyroig&mF0|GhhzO)qY) z_Ukr*%RXV1h`3s|1D2UtDt|NC1IVj_meC@~BU;K%hAsrsXCA)~mR4acds#ftTSQ*5NC#UVUt=4+?Uj!_V zl9lnr2&wRslp6;_GeS1E`_ITp_H$H*R{wnv*Q|-lZZ_n0r|((sR=q9J^>NYlb|ciVlH5(IF5hy8aH)A#^CZ9rVjS6dfFx>JBu~LC%fN0a@Q7RzZ31 z`u~4J(S1fve;~St{+8%0yX(1!A8d6Vt~yUw9cU1abVvs*v=amZRfj;J>JSK29RdM6 z^O?6pbqF0~d$EIl=|R=OfvN7G3p*vqxzRabA-P4Y!nVl!T-8;}Di>8ten=}5FP{6c z0K5N1fS-}x9|&;ny%V5qwQbw;fc(sN-_m~>Emw3x)0C5d$$~Y^x6$UjVP+5J&A0e3 zxcZpS!$hrYe%hQIh$LST{-Zgfa0in|$5O_EXjY|^h}S}X($kkhz9W^ZLBf})IgKDi znXobuuZ?$gI#Q+nIAh^{9aNPkKPtMv7wox%y}&*ALYxCKA~N$0RAyIusgDbgu*N*Y z!e7yAzg#6bCn=3&qE=h3Rb_L&QC|yI)K{`uUos(Bwz-R`AJS6Y$*aB z7i;3hns~7$UaW~1YvLuEc!?%nqQnc)3YraV*Pe_l4Y$Z5EQ+T-#`1hq%0`i7pfoXm zUFo$kw6*7z+MdG-lb)q1(hvK6LzQbdG8!_z-|gR-sam>@a&&Jc`~7N(0195vlEY$+ zO<%oTG}8ZnYbV1*zaSgWkzUeb+3&Yui4vZs$6a{$stmv=D#MR$r-wUbD3a3v)g;rvjZi-y)##1 ze-<^`iu%A>v=hZ^pJ3I?b00>#KCIYdv^%u+cfWdQ+1d02^#aogl&9;+Ov)#B_XH&N>(4>&4~xS|YCU;2mC0ri=OTW>(KPuAHiTKk&MCH|YAlE7zV+ zM$cv<$1tBbp6?LPcDla$+TIR0QU&UOb1rx6x6DLOfyyO;qa^;xTfJuAo5i^p{0C4= z0|XQR000O8002J#A)Q_wM*;w|9vMss1j=E^lap>CAb(E7Fc3xed`0Aa;yfq>y_Mb1;jX{!}fs0W9Z^-akPVcNp0)ds8=;GzIhjxbZ9o%e=d?AQ(@bxCljAa1pl z)Q8eG*nd#=O&h>@n%-dvo^t9b9)Pi#iRggjY(|&r1ONbflRYI@0%i!4fe0Ftuq7%1;*;AYA{%m_ pM*(XD000CO000yK0000000000004l2Y?Bx!qXf!f$di-5CV$C}-#h>S delta 2058 zcmV+l2=({APQFgC%>xQfW01IT1ONbe5|e%i8I$k>4;+C|A6jV=E=dTDl8mP5z3DB@ zZnaMl0Ta*wW^9ui1RHf)N{f-{v~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw z=GXCG!)MCA^Mnv#NCO2QnTZhPz_NIlkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ z@*pkD#=?O8i47XZX-JM~SmmS;jS)t&K*;cUl9t>Y52I8*Q~7@wjKm}Jn=_a^`ldA= z)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QE zkVbln64H#RR2QMSDBR!;%NR(kCNu!7qwcD`bg65CSBWG=(p*r50>)6g4 z`>^eFZO`qg*Ykgxku}F^yw(vO2O<~`8GSZb?SYqpdQr~6wPlq;HF~dxRczLJV73`B zWFD%RPvg$edQyhhhzCfSzZOtA?$Xsf*xMdvI-(0=&=I|D#9T*sZ-}iZJ9v~G zjqiXcbwnRT`CG(%7h)^QE*@o95eS&QCkO;gu2Td8X4e@4@wOr72n4+S3j_j6e~Ca2w;@+L#0AxzA`npBYXkzS zdxJngb#D;}sO}vC0oA=nARma=!zY}l8vwOG(jgsCwFv?N`5q$>knafs0r{RH5HOq0 zHV|YXI!Ax#U>aTQpkHpDZPxQEI55>6bV04IA?F6+fVqMoz>K^56?Qy6|M%`>HodsH z+OOLLF8hR4Dw1m14ry-Xnf%RV4N2eB>154kgv;HWTdQOdgsQD*$u6QX)e3*hs#U%U%EDyvc0=-nl*7b&4%3W^nLr?s<$P&J}$bxE;=v` zk90@}uJQ;3iVlH5(IF5hIs^hm*WV#JgbqcwgMQhEqJsld-GL@L$hpxuAnRMiDk$$= z|9^idy3fez4@CFS-x8hebbasegRRcTRp;xf0}aBF4(Wh}c7i~l>JSK29Rh)>Lm*&h zKJ#~|4xxi=FLux`eW*G(Fx4G&VW$K+H#!F_B)5oF*cSPptGa4g6{2d%4{2rM#rHlI zVE4ZW@H5i;0|CzccLH>z0=+I!pY`rjIwyZ@MJ6Q6xDirWuZH+G+8j5`oWZ>L6#oTR z-||J2s)a4cnu7z87Au4QXig}+!Q|1km9a3MRT(kFYbn3z$xA6;kxJDxqL;Wij4(=; zQkII>#=ANksj^_4(`dgAsw&bS71`el_q@Sg=pB49+yi4|$iz2LnO*IrJ}p4P8V@_m zU`4J2wn__bWF(e}+HJX1vF3QAz818suVk;jq(abqbC-}HB23-MkK341vAIztc{?>4 z{{&D=0|b+BB^tAY2=)sFpvU?!Y_oq9qXK{Bb)qzA;5jH&Q-Vlx+G1EjDOv=yD#Sl6 zu)Gar`4~|W$5nl68XKkM?DXH$;Eo1Y%4&yu8r;|5%9gh{R`RsNHGK_h`WjaH+78#m zi#73LO}tnWFV@71HSrQnyhIZ(QR0PY1CtaNzc+0>4*Kkp~^KJ84VfV@AmJ^R4rXcIl8x!{eHDX00l2-$zid^ zrmx;E8tMN(w!N@wI%1pnbHjTR;rlA@os78U!agqJ;YZ1P$$q=Z8%$!IJ;hfCh69s) z@a)uQ&u0qZ!)zbO}-01dV%Q#%G32_lp%j6as0h0 z$u(Wq_4Y;?VmiJpXPt}j_2P1TEfH6F@D490)5UyvGppwtS58&FA9!868+3i&m21x@ zqh~XbW0+4I&v%GtJ6&IWZEpu0sRDJtIhQ;3TV|rCK;@FaQ4;^;tzNV5&Ei}P{sT}; z0|XQR000O8002J#A)Q_wM*_1Q8BGWT0%^X%lWrj(e`>=p5JmU>3ZuO(KS*(f?Lb?y zYM@Y;mhOZ!Nkpv&l3K@qU)f2V1hVRKbnl$GGpJhbWf%Q|bwaBeWl4%6P)u{7wl#Wh zS8<6V=c(ef(+X-dfJ4jXu40BTZQ<2w1J(<0Q2;4Nn5ogudqXgGYzLCMBsf$MH`+?- zLut3ze^B;Kw}A6By~h$f<O2ji+LOnu;^REKsj)Q{l@Z9Im6z zufvHq0{G()^c!!-MGk$yqY!``;+~X`;_10vBcUcw^F^E%ar)5a87WG#D8I&$SN&(8 zJP!Fc@dP;JzWxNwv#%nB0trrIkhpLJ004QDJtbEHWeAf{B^8saB`*QtliMXC8=%Mf oFl+<>00a~O02BZK00000000000DysOlNctW1OjQk!IQrxe-?ey(EtDd diff --git a/summary_results.json b/summary_results.json index e82ea3fcf..b26cba987 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.064, + "Performance_Result": 10.1188, "Performance_Units": "Samples/s" }, { From fc44ada96021147ac1dbccea41342948bebabf93 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 10 Mar 2025 11:06:26 +0000 Subject: [PATCH 291/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10046 -> 10041 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index da5da37c9..56e4840f4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-09 21:42:14,965 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-09 21:42:14,965 submission_checker1.py:3270 INFO] --- -[2025-03-09 21:42:14,965 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-10 11:06:21,603 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-10 11:06:21,603 submission_checker1.py:3270 INFO] --- +[2025-03-10 11:06:21,603 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-09 21:42:14,965 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-10 11:06:21,603 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-09 21:42:14,965 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-10 11:06:21,603 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-09 21:42:14,965 submission_checker1.py:3286 INFO] --- -[2025-03-09 21:42:14,965 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-09 21:42:14,965 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-09 21:42:14,965 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-09 21:42:14,966 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-09 21:42:14,966 submission_checker1.py:3307 INFO] --- -[2025-03-09 21:42:14,966 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-09 21:42:15,945 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-10 11:06:21,603 submission_checker1.py:3286 INFO] --- +[2025-03-10 11:06:21,603 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-10 11:06:21,603 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-10 11:06:21,603 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-10 11:06:21,603 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-10 11:06:21,603 submission_checker1.py:3307 INFO] --- +[2025-03-10 11:06:21,603 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-10 11:06:22,634 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73409 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421691 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420566 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1565 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421348 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1565 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41937 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6a21a128a..048efdb52 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -918 +919 diff --git a/summary.csv b/summary.csv index a146afd38..ff3f87a21 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73409,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421691,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420566,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1565,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421348,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1565,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41937,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e9daf53bf..72ce1c946 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421691, + "Result":0.420566, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -544,7 +544,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421348, + "Result":0.41937, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index dbb6ef8841d9d0a105e28dd99abb143a5566489b..dadf5109574503b8670315a5a894aeb67100e0c2 100644 GIT binary patch delta 2991 zcmZ9Oc{CLK8pmfaBH1RcCDX_-b|&w{*vVXshHTksDrC!^lw}x-Au==BrlOcEAzQXr zwuU4o%?OPNm9k5=Y80;C_nv$1y?^|E|NNfwIiKfyp68rjr(K&}U9^B;oP(F^$%6pE zTS|h!ar{5Fml}*eqbR-77y9dsn@(%-@Hq3PlTv5mf06(fTIZ}?8VYl-d{g4BiE96; zyWQqFc`^Fy5r^m@p^p0%8nHs1!T_ED_ z60=LB({c?ufD&-jtit8VEzThCt?J#u;N`G!7(9%6pmF&nBBQ(m`BU~5b-+mET8iK6 zXckhK9%vW$0eg0|4(eJ7lNF&Se$`skF6+_oJ_9~L@u6Nel}Xz`n;J-`vYU*cnj$yS zw(BQMxsy*eXPG?>D;&fRYDmJ^x^pL*3FYRG^gIat#DWJ*Yqv+p1gGHMdSY2s^Gp4B zdAY2SkhjZ5G^j24bJqlN=sD|6Q{d_c`>=(H@$j7Njf|!Ulr$gLY{>&pqA@qWk191i z#z{W9d%0TY4SXU@WiT9TxM$pHOSp`U8eMEiTfG*zIxnBy8MM}L-FIpbFVhr8W!G(P zuufl%m};5*^ZjDI+2HyMtJU3#%del}ht+-5O$Kv<-NRu*p)Cig?s(mMI#1I?i*e^Zq%X;Muz4X>z=dYDVQ4_{-9bSi&*g*jj zWix&GmZntQa?0{_U^8pLgpm>(EA{Gz`Q4gvLOar$)$Y6ICkyAYl-t@{j#&PYm4a(~ z5i}cIIeT@^7Xm$iX->sZmzsvA%4XYl<2%QTwZE0Hmyh`ju}*{>Z)$Ow+$v=wb}$AEyB7buA!b)}Fgg<1xwN%0SJ<(<=67avN`*IaS2OG5 z?vF21At>PswO#gX3&w`Z7|8}C5%x|~!qbQTy?J-)OAf{BkQ`o+sOUv@15Z5=Z`}^>*`fO*loY1}a)( z+fiZP*~~j(gpYkS?2|u6@&WPH{?g>@xKXadr=R@ty@1vrD1|4h1wE5t_&qfs>0}v6 zS|{sMPC)!|1XNenQlZMM#LdM#E-Hc-c01Y)c7E0*twM`P_RNw(mmYH1`s}iImY;Om z>73n+yud7bM_RdOvLh r9xmJ4QYSQmGw>#L-w!@ONu261~;9{y*!VF)nlRoJ*cJ zDdzf8j#enj-X{kn*}UoAY-QT3)$`CAXoO~Rzu-lC*14=!vW?8Bk^{=tk#twN1sqFs z3$-Qd=a(_;!4|g6U;Ew^VF#41cewefqJh+$ptq^LQOB`;fw;}gLTdQIHYVB1^*_VnGSMpXW zco1CXeEqY-R-9ZM*$HFyKg6ZCE;0o)9zY|F@NM?v;erMlU3N{eKR}{Yz zgeBl82#0{L2fO-@BsT>!Ozw3-UrH@bajTnG3@z!7iB;V)kDv0WFn2y!^NKQKSv?j= z0Y8wa=m_98nip@M$BuOa0sIO{R+wUj%8Cyp3rTU3LU6K_Ws|n45BGBK+By9R3S;egvEUa~&L4}y?Ng{$V4KGiQ zz0>9T3bFHMIG{TwWoq(=g<;vzT(K-=ZxeKiuS+-=A)JFjn)w$Frkw%!=C#G+vEf`Z zOd68sVxHLVg}S)EP=7F}-X_zTv!eGr#>l?8`Xld}xu1@hhE{dwhf+blC4RjmU1-S~ zjgc0RGjvcFS_af1^1sk%v5zy}$Jx7l{eciR!)O?kr8W%B&}D%tgGMIs z*!cT23t0z#dqFrFtm#2>ac10q{XzD3&8_5ES?z7M~boDPp!A-?s#g4 zQu)_ZP=as@ddZ3%UZbG*u&Tw~yK;v!nps@*$vuu4A~w5M9u(Ga&GwA!D@%{{xkZqh z-7b#?thU( zeY6dm3a;a|2;;TewRd-16XqD&RIa7rn}#>ghwd?hF6*Mzo>m7orc_X?eWl@f0Tf(; zQV`P1JJVHr#3RwJl)Fx;2|inb%qqJz{oFiEte4xeygG@ibhsO%pdN;GV*C|Zrq-yw*;r1CQf_Yd8yb1bw?nR3ni1@t=>3A=hx z7#Sc6WN_L&Q1I$B*D%PB3VB~aU!KE{%{kH#Ix1Z=;AqAt1jfs=_A2(_G#=OAr#B7I z5-dC?OgeQ`-vkh)C(aaC?Msa|?K}c-#Lb(f5P>6vkBxeS@els=`(CJ~jJN}OHOdi@ zs|4!JglVaS6K+n48K(K#Vf06BI%!UVW|uTUhmXcS>8x4vBZ-W!iRCZ!27}fkiZ_ph z_5bs^Nsjx$?&qz_pG&V@^MPsFdy>~pl+V}=WH?&-S8H^Mz-qUL%@X9+!#)ZkW;=KJ zRvK#+26_JLlbxdE3Y?;Fl#GA@6lu6Q5Jd5S8vw6Ra^diQ2ipyo1iDei;3mK-ilj1( L-&=7%dceN`kvx`t delta 3008 zcmV;x3qSO^PQFgCbO;HQ$6i%N3IG7vlYIy=e{Rq}m+dLXnIy&-JC5zCeR2^Hpc+7o zBpm14j|3RQJqKCIW>((WmkhMd?e5#?=WcbM(ZR0|ld*9h#7P)U4^36GOe2`~qJB6X z9-24T@Ao{@NYd`K-yKKO;L!XQB<8P2|2p^@#eXEDAV`g@f@yMSj?#3tU#};}x!%C$l){_7_2uaa~)MQ=fFhsky9SKQ2>8G#G@v;9b<4Pl9Q> ztRxP`-8AcdG74vjd34YZvuk-iMjQ+d&11EH^1E)D^`nDD+qYrxHQD%VzACtSEHJV>^3l6LY8=$>fAfl0 ziqA6SS{B^Mg4>OPde)zHr!?=BhCW!^eYiqJ(-z$%5@%a-O(gEN_-n~l zx(%Lg11eA=T~vsEetuSQ{jB2VS;cLUVk^@o&$J2gT4ag}Uu62M;`&*|4Jzc1T2#kY zrZ&y=*v8*f+{15)MEve2L?Q<4e;ttsV1FZ$=CNJPHxQR0dF;KQSY{6YLkS`ew67KFDRuRTssXR@G;4frC3e{&hr!5BH? zT%bf*;-0&F>iFR3 z(?Jkt>$Tu64U+UOnSYstNxt++?(XfHRZrs}SS+&Y`C6)e-ac8y$3c=UVUqf4Ewi_- z`K~vc?e%8!y=Xdpi09KZHCs&WtyFitXgUao<}YSHNW0-UG2a~22V}bbR7FbJqBY36 zw)oGEetll{OD&fewWd~%uQ-$43m;&hj)V(nPYc3Z819Q4&NjmZ7i|o0ih(?LC6XjY z^!NfL$`Yf0=~js3aCF!0<upVnjiz>+JC>q8N6|+V1w?OA;)1SNnXkw_GYM52fiPu!Uw`~nm) z+!y^kEeLP<0+cgZ&?XJYrKpVQV2rXv%NI-|D!%lq#ck51;epY*WcM+m97)uG_Yx5;bm9Qb{)Hfl8tQDv1WDL=8-j zYYn9mD?lm>DrhNroNxvuZPth@sgzM25{1(w70G#WBY~HQ<;EnLbE&PG`T;Pj$g#|9 zx$2l&?Ksyw*V*yxsQ7`?vBEPP+mT?R# zFFt`OOWLdv08>VFNEB?P5{UdwV|!KcqtRo9Cyur& z!6c`Wjsz1m2qw`0m_!3$q6XKojUq5f1psD21+DePCopA6n>FG|FlAJSL?M{uJR8Am z{7-?YerP3IMUJI^z-(-T>G}<|YsTa!`Ns+m$Zb!8i5l0HV4?=WBpLvdXaG!ds1IN^ zi@+om0GI_8wAL4&z?3Cz(g^IXLR5!E0hp*EU^ZVgm}0N>` zqdFvtJY6!8Cz-UXWpe%{n+oSECN|%mA^6xkrsU*QtRBoDfbfc#1ox43-rXA~vO0;}9(zQc0p0w={4N_Uc zxWhC+B{9zcl||ki)(B8p|89-{H2 zn}=wS$`XDarU5F62B<8u^sowm$|6&b0#^@|v`Hg)G6KdP5=B<9Wvo4_VsFoLSpJ-| zk(cU!@KKu^Kqk=unMFn)RsoP%Wc1Ns^nsE#X$1TP5Y=H(KxP@E zkFMD0^BkG~siB~f{7ON&ZSrZVrt)`e{^;foDA6$DNRtoIc+%uUGzeu0lMmAXl*FSW z4@z{A$%i!pP!^edG?;v#q)i%87ElR)E*}~OD_JIjm2AZ}pXabl2RV`IhjwEt$*t4^ zcGKgNJvaL<_+jfCZ*H0d@h~_E#^c23McHsIBiE(N*I8Yu{oCc6pPuAdd(pHX=Krf| zcf5R0RT@r*M)K$K4OmY81}rPDtT(utj{_t9HVY2T;9-{iLV2>L(SH~$-+g6&%;GSL z!}Qy-_59^q^KtiR67^p%IyIk$f6jx~uX?YX{Z~Eb%`b+2V;mY;>W|VayG1Mc2iC2l zgIRYNoOk14I8BUkFvt?MYOZO-%eQGQKBv)a@yRxgFHxFClgDqPpxY1P{9E?-ZV*N3 zaDAs&CrPQx$|MfZF~hQ^jek6;!APhnDqC$qZrI!mHH=tQX*-08)-HQ=y&rhG6X24kUF+aHt?| zwUyL|(l&qCQ1(q5zM0(8v6+eJfaGjOnjhPa0M0P9AVGPDvn0bOuLmpL z)rL_yb4uZR16Qp_adz*Wn2lZ6C0#LEL;ehZu3z59Rb!!sQ!zkw$r)i5sMoeG@p%yr z*HY)#;lx`3{IL)EmAAtpQy=gs1Rzt~qw-NaJ#QpdsLs=T5vN6*K5X)g+~y=(d<`Qn z`cFW481ir85isSx`~>y0sv^q*36#fPRYnQ`0NIm%C07CBlh!3X0c?{KCP4uSlT{`< z0eh2>CMFxoVaSsu0RR9{0ssIJ00000000000001hfn1Z(CO!eMlMp8(24W-t0001t CSfe8V diff --git a/summary_results.json b/summary_results.json index b26cba987..5c6baff67 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421691, + "Performance_Result": 0.420566, "Performance_Units": "Samples/s" }, { @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421348, + "Performance_Result": 0.41937, "Performance_Units": "Samples/s" }, { From ae87c9bf19f4c47bd355d450221748197b96fd12 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 10 Mar 2025 11:12:03 +0000 Subject: [PATCH 292/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10041 -> 10047 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 56e4840f4..a3cceefd8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-10 11:06:21,603 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-10 11:06:21,603 submission_checker1.py:3270 INFO] --- -[2025-03-10 11:06:21,603 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-10 11:11:58,576 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-10 11:11:58,577 submission_checker1.py:3270 INFO] --- +[2025-03-10 11:11:58,577 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-10 11:06:21,603 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-10 11:11:58,577 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-10 11:06:21,603 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-10 11:11:58,577 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-10 11:06:21,603 submission_checker1.py:3286 INFO] --- -[2025-03-10 11:06:21,603 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-10 11:06:21,603 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-10 11:06:21,603 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-10 11:06:21,603 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-10 11:06:21,603 submission_checker1.py:3307 INFO] --- -[2025-03-10 11:06:21,603 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-10 11:06:22,634 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-10 11:11:58,577 submission_checker1.py:3286 INFO] --- +[2025-03-10 11:11:58,577 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-10 11:11:58,577 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-10 11:11:58,577 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-10 11:11:58,577 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-10 11:11:58,577 submission_checker1.py:3307 INFO] --- +[2025-03-10 11:11:58,577 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-10 11:11:59,643 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,10 +32,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.379315 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1188 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1247 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.81274 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.05816 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 048efdb52..72f91c4b7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -919 +920 diff --git a/summary.csv b/summary.csv index ff3f87a21..021747072 100644 --- a/summary.csv +++ b/summary.csv @@ -10,10 +10,10 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.379315,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1188,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1247,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.81274,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.05816,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 72ce1c946..11d3611d5 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1188, - "Accuracy":"acc: 75.600", + "Result":10.1247, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.81274, + "Result":8.05816, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index dadf5109574503b8670315a5a894aeb67100e0c2..6bd07ecf9300984c95d178e64637a5a4eda76450 100644 GIT binary patch delta 5052 zcmZ9Q2QXZ1+kh8KM7IdxU3Hb{kth+}VwH$!(M$BOgb>2AN`lqxvT8Q08dmSo6QY+T z2!bGlAcE-allPzRpZWhYbIy5Y?laf9p8L7aGjpFN{dWDzAu{0aM?p~-BAGZc%z(?n z4J58j8IO)DTyI?$8}h7s$VkF83UKxqA86;|Kh^V9zU00$6hc>`H@LQUPx~<{Ni#48G!}=PYS+Z|kG(HU zz9zv}!>5>^3`?=+)6R=#uA8k~3SUhe;&XxLA;Fe6OYM2yZW1wj+`aw50{5W0}UgmqOZX+ z+Bquo8)VbOe~nEje3|7$I?Lm!pNulL!n>@r0=%Njgzg8VfaAAAh5Vpg&|}Lk1nU!# zesNBx$IH#!I^e|!A!Ka&Mzp#@KPUT21I|YGcYh)SW$_^}EYvZPkutqikP@g@ZUiKC zp1Mi8Q_u5*64-K14J6e;qEzS_cpl4lkx#tgWiXg?2a=WqMW`TE{Ja<`i+_PhNhVsg zxVfUIJ_=?RaRBO+$9TX7b=OHrjyd@llD7X2aJ&qXS;I06jy&>OYj*6`n>n_gIQZnA zh@9jE*jKfzoaYC{VrWx1cSl_FDa#85rOU%#jxw!{O*0jNX8R`0o7vWfcB2F8e~om;PLgIITZ*NT+=C z`MB{X!rCuKRqiTvm81F$l*U)Mk}I4aB@mG@at1C@U*n#Q1^?0IDxss(^SfsKzlNG` zAB^3z7m?yI{}i9pA1*!EKwNFHU!_?cvN>b5g0IF3nTS{>Nef=%6*Hx{B>wX#AfhkN zUBtm&p;YQg^pv6JdY|sCh2;K;yZAI@o^Xkp@&B68Fefe69ZUJ+|BS{T($$-oN0D$u9R0edqUqc6LQ2_u; zm|#j?q|q$wM)$&@QL*t$SP6xt*oFkT9;1}DIl-rxxxrQ>Xxt~hJ&Im1%Cjd5^F<9Dg2JP194mx4k?IG3&YexJ*3gs66c;fo^H7 z^oE2bY_;_)>(Oap2Cc5L#CZzqk6?FXIdwTFb**S3;?J){i)%xtElYJvH8f3Fa@p!{ zFG_PV_FcwrVf_1FDX}#NH>$mMiuDAcYDAg@K>m+?)FJcb-?6Z>ct^px8xI@ES%i|s zg`vvI%B77VrrvvmrFy+8%)9P8b=>pJ8Tb#Z8A|^KBB+iFuoG922X_>a^*i@j4u4zn zGA^86PmU%D(t?nFS%~hMNo^4Xb&g--WmE`;fV{800ioDVWRjz|w`uqb+2-hyqlt=w zpxf`Qc^UJ9R3T4%B~Z|~UxK{eN3=hFa}0-3dA$6X+r>y06pqIB9)gpj5lB8z@^(;4 zz>?C@Zy}y`24(aOsBjUIe{rkFJ(1~ihUl6dSsQH8>XYwn&0#rKp+}L8Kg8e4D=C~D zZ4WGz%NNbnObv7!pvxjP?d5F{Zh!LJp3C=L;G9L?Y>q2LNBJ*$uysubiZ$v+Kw5Qg z*cRHOC4$k-LCyAe$K@!ou>|Kf>{Fb-{^X8#lA~^Vev{3>Cj2R~ZDZs|(E0HbiOeC+ zrEbxo8%9Cy6V}R^v>%f%VwEVzY~-~6$X*-b$tYKzNH(l@rL8U(yS%kF^~0q7^HUVD zQt#sns^aJO{KO3ff7LYzd27h(8<2ixMCB*IAbwEZU!(M%*Zm+?DnS;Y`A^Smzdx%7 zi#M*>HS$yJeocj;k!-|e>J%aMxQhZyK6KuWb;Vodkxi#WiPDk-&6`PkUT(~XJ^qKz zAKlo3&R_R)KE-KOB^V3m1T1^ze%WPjYuG=15d}@etnr|hC<0?akHs4eNFg8;BpC)l zL3y?R=)fS0=q~hcnMg8}&20kQ>2hw{re4C}J=axwHUFLY7%qS>68uajM2G<#g`Sw@Va|N;MUZWz4MzWJbyilL6O7J=nkf_HO=E$xMAs#)6R`R$Tr)?Vcs+tgfu)0K{E3{sz+4WBd z-6LxdzbsfHTJt9U=Rny-JB`d+f$r=9RffTu)7$#t%|R2ro=e+BHCP&vZ!cSaec^js zxZUSu4{Tq*39t~`MeV5I4%&FoQv z1jS2P^%Wc=JDkAxOs8F}{XO>d+!{QJT)_kM+KDVU#hG#-Ci|iM{~=oS)c1ElSODo) zy0WZqJ5u<;lhJ~mDObsPZHLCdfi?TesUBBQ(waKz!i@%FVqD+IpF|eU)L9&m zNxEF56G|Jdjl2sGAOh0A2XJ-N{(s~cJ9{c^nu+eNc(dm4hoWvh7bX0C>ad@=`l#VnMQjo`VTP0WOUvNp=OBn zHIoQN<;VR*3+?*$L2-^_<;b+Y?lcc!D?%=_O5x%7ibuvnOutvUa;PJC+9a^Nw&AS^ zhv$G4)uvrZ2QDxASo7q0^rut`FkTfRV&T}140j*2a?z;BR*U!8{~z>cv>m^2!`6N0(%j7tb2#v6!VZ)lnW_BG`T`I8L-v4Z zigyINMSJ!fv}cY0p9_e=};f)MDA>M%+xve3@^~HQ19;` zOlLJY^XzZ6n|xz@Y(tW=M&_NL8s)06zQV))syaL?$_=1{ClMN;Q6!LtRu7Kl5OXIUch4+D^=prTovrW(Tt+NOs7OYw_lW3E z7;!GVs+340BAuca!p>bYxbjgLAT?m6B+!V+$ya9&yQ3R@wf5O}-ob`5oQK5UZfhX- zqV&bwwX6HH-8^RzyYXBNCf^KXhe)M}QcJKbdSbxXs`_1+GkF&MI3NQ`=7^m}n31bE zXAHSK-$ChB9w%ug{k>i>OTQrMPcG>x_6klAgVt{oCw$;}c85J4S9&-`t&_icO8@E| z{jc7SkHW|+KZKIN(Z7Bq_er3B=FCav-9H>AH}_s93|@YIXrLRCn)7x4$gJNmq;oaV zjYS0&!l)Jam&bX<6A}ry;^jbYfQ{75S9k2=0Vz(LTucf|24g(&LS)523_Wc`swh*Z zPQ7#5)uI%B%!ci&H&@ktH+3Uvh9h_Sa9P2uo0UUlkL4aj1Tbd%i(8UBuYanDq@H>*c0W@yVroEN%z5!`?6>{W3o{XyOOu6{ zd$G=KM7lcBeJwZaTy^Rh*~TQXvqhF_IA}*Gn|}tv=8R&eeLS@^;a-P*JMP+c=D0DF zCiEb0efi;Xjx6QUS<&XKVluiPS`CT1>31l`WlP~*)nPl!k zGWRv+^ic+rJa^)~Y36IHfjXKw^XWA-4T;ivO|WoAF&N0uy0~ZN`)1^$3|JL?NX(g9 zu`aorFDpO~dmVO||b(f>3a&I?~qVFP_pX&T3p{sB@0VC+M&XlWKA2 z2X>DboVY$oaO0?lJJ9Oi4QN1V85?9E8+IXt_d0~q0iFRV0Vc=&%ADLJY1b46^;^0O zah1h!OOG^#5oQF`x=s;em$CCMuj9*X0wLli!lPQ9#XG6gq90uzm(p*<6dS+&(=A9C zZdpZy&lmpwFn*@y9$-Fmw*^?xog-vWVyg47eXtE5WcTv?bo>rX;@PiM#};;E?#`cW zo?V58*sr+Tj{MxgByjbJdO@F#8KbhAtmfF;N~+>AidlBAZ~&k1_qdylHkD-J{0!;V ztU;7>^9U~65!Ue?kK4-4nKlU<6ZP|ctXYan&FxnA3%4=z5+hx*VERJ~MTe+#88s2y zv9jY@gNCsEZ26)`)2)M1>;o8b`TDMvk&U2E1O+YvEzKVdDH4n1Pr(<2n!Y|NBF8r5 z>T&VuEH##RJ#fgY&9**2yxW-9nOZxuh@&`^ZmeDrOGuU5diKEfV`f`qYFqtB>A)&Y ziOIEY_ExhzR`)-kcpE%Xm${XpEN6Q3#a3VM*t1yC5BPwe4x%Hn1s=>uZ3-Qx@d6tT zb;}@|cU$)elI!M~kn_p~S;wYGzC51g)47W`$tAi|j}m^BIkwx1F{*S@Tlkm}s@6*z zVhjIVa?RohnF(J6F05X9@gi(jTQQ6cR$qFY6wZWRN=$pnzKqZ&)CdOGN&b`mJ;J3( zDM+b?L22+HOfDp3u~pOG_N@Ot)v~w4N#V??ak5tNfj;df2M8`8fs+%np;#Sp*KSa&qhNajp$DgK-e<9)V0XtFh; z5nrznY$rCuf_cH_2?^nR=Hb_IFYW$;r88bI*GK!CP#AL8v|{?u*;}0AlH_kchLIFt zui?9?PD#F_5?#ZOapw_64hR%T*OS5f-i4_dS@Dos3-ZlKmDSC; z{f)J;IDK_|NT+OsZ^}&1=sYA~J_dDHib#m{bvV0{gcm%D}LM=&l!MY+A`R!H_Ziz`!hKZ3D zHY|{TTm`59Mx#y*O7TyJ0Bu`13BEPttv)L$AGl}-EwsFSxU$pZtf%;`=PlJk!rLb+ zw$F>J42n!{tN8VqcZwQRz^#`ybj_@E-`Ga6Yy*E6IgGl@N2(6{b>uPR$n}6zN6wC& zYSjF&&arYs4rtGtl(#;ZD|kK>^ZF^Fa_{c3tT!;(`F(bWU+sFQ z`+pnQ5dOy`%a}=iUXH&TjXMwk_}l3I?REcqgoeQ}OaiJTR2UNhc@hLBNkD+(-;9$0 h00yK00ONljyt;>mS!0L-N+eYn20=aw2gu*he*kNsb?yKF delta 5022 zcmZ9Q2T&8tw#OlWH0k(JBM?YHIvDAl2m}Mtq;~=7MT#IL6eR=&Ng|;sAX24w5NU>9 zO%O1kpny~nDT+bL#e3glR&{+CSwI3kTK((Q#- z6G_Yw%gztxGntB>a>)JUTL1z;3aQ6jr$)Pl1hA%|F}Zx(H%emRJu!>zUnaF8fByvE z80M0Rdb@{lYn4fQr{U^`tu+`}+@HeVVV6DDIIzS|VRnt%cf*fof8=B?Mj_kMxo!my16SMDjxRaz;>3q5OP{-+EVwSI2hiB2I4V6;L|-K*wKcOnijB zC74~=Gs)Qcoy{)~1sk}h-?x>8TipXQ`mESgWY@2S=2p3jq#baBU9TW(k#PJFN$tY$ zaTbP%xr{aI8<9ujUr*!>g9QmF%yN7eaF-fkz3geYJTGhvB+{Fo{-dz?@Iz|D zBSZ4^Z`KqG!${$gMbN6+68n<|lmVr>Y?kxQo+vF~Q&U%1n0%WS;I+dDpvG-+C&ZJ( zlXX4@45X1Jnese*X9clK-ffn#88l|HXzQZv@ zzZAa9xVCvgXZuN*5xrGFqK;12AOr16^k3uLZ>+SDh1M=*fZS*71lyRM3BjWC1!^7puhz_iJ;PW2oUxxkuVTS^I|T5g#sZT2lR_#8V)IWU;C0k=>(g`4aQRzLu?`hHHUemDcrBf35*ZbNsvH=*KtA+tId zIn+*6bros9UiG$R=#S&{tx-vyTQ}m2q`2hr|I|};oL!GEz$^$c1b^@3Dhy~2nBR{- z9=o?l;*+rn)0=vViyuzBHr7g9edFfEwmRPWTdwUWdy)XF_K%x;CYU1T!g2aiwJ4Y# zXEorPn2zS*iOr3t6t3CVV}oPLZQF?OwQ=Maikk?Jsdat4wii z$=BUH-`bNpdo+LRJ)=p(@qvPe zF;P){z{N6vkfC!%|MV{&T9#Wy8NC`P{O4$^LG!B$u8;TjV7+GZO<4fIiY5i~U^(Uw zhU#|Pnvr!wPUgK|V4CXCjz3CA?GB&rg#FkegcW_F58*Uu)OWge(TUmX#G8G>0ChC} z^N;4uS!|QCzyX9Pt@6H`4t=^s>}KsU*cBR9VX%;?bVcF?H-0W4}bS&Qp#CMYc#{p zE?AT~u$3v-4yp89F{%u)o>V`NlCrXGS%+X$IL21I3@D8C5wAM}s&tVud+-rwlq`U~?LjOLQFmq2OZl4eR z6bj;aIS=Mkc+CR?860Ab`qtw>oT$ZM*6uHY&;!d+@EEjovT!a`?l!QLK@Z@Pl}A>nMl@sRy~{80W~6Z zc)(5_rY)(Twoc+@3bw$w%d%XKfW(9gE6M#4*Dp=?chu}Y&l){p0OUnsVQduQ76cJX@ri)I35PVh;gFD zFh*i%XGZk-L&KJHVIWwTC?&!{NPykM8oBp9jy<1ssN^Z~a8jf`-AXbea%&rrDl-~Z zod_qcNgSmV#jnd4JBHw%7t)%Y!$24#Zu2Fy`hJp2A}IM!Q9&8_OMpxKT?n&UF?Bn4 zrh?OB^JH&Iq7SS;E3UK6B>F^xoOgh~rV%+Y>6|r#ai*iKJ(i*EOfFNTRZSTA%pf~T zmjp;_MZ%U-)ej$d$Lp11cooxDGQ49HePk6<9KXv*>lKp>wLd;;@=p<@qfzidJrf1c6 z&`y#RuRrx~$HV9GRz(BNVwoPe%jIJHg;?)+r1ATngEpd>WIdp`p*XPxJIHv)jO3Qq zhBfA!2AivxSYV!sw{k|JZ39iPs#&E(Gi@p_psIC_%q-Ay3%HiNrbaiJo+_jQl+qy7 z0~sN{jt+@vep;M26{3h*-aN@Q*VEOVj=>8~w>{o3F7F9PT?4p|oUbIY+N#Jr87b8c zJ!5OD5*?K6sN6~q$QRqjZB=wr?t)=8Qe|IpGog4R zJA#;hGvO=j1*+#47+W5qi5Bx^hZj+cp)X{>(rU%95q#KUjtyFg%EHd}-w%qVH|m^^ zIN^U|y0U>qYYZ1V$>AQ!k&#wyw)$7%5j$n0WW@qV=At~3YFMqI^1#t2NZUr~Bi@rc zyT>ZwiXq2nTG&_iG^qepSdx*o{Hi>QFdSL~YZ%KTNK%Q7epF@prfNnKN$aU#fQ7@k zb+KqyLYZk+I&5!8g}(yaYit#m7cpk7Pkz{;sa^_PMjJQ9sF0eG`q2EQZFUw5yWTbe zR0m9?%Jf_ST2q_ZtuC08978XFfg}bCMhKAep`VS{R6?`gxYqM-p%I?V1xi)Rwz>hq zBCfKBujUOvf_U=9h=z?AN$S;yGI-Llu}x7uh0&Pd$&t2nRGv8)LwLbB-_tN7(MWg3 zxO1L3eypAbDUQG95WzlDfb^i~Rcp-p&W|ob5%W^_OAnaKVYn30P7T;H{3;t!IP=ms z4Fru9C(F6Cb+=3BI`c2nm0p(E$T0ZfN*kjomMZO9sS<59K9j#zcdTkCDmVT4WeZTN zl>9@wwtpDPVp@t0Al6uw*uuyv^cNYA zNZG_n*!FR<$hwc`0=3RA$@{?@M)YPrDp5R`$@t}Yx;(8?NntJ)(eyA1dz!eD7A(ob zI+!BQk^zV6c;$?uZc%yQ+hY*H6dgDU3b=!3|9hV9oS&zwp4-nLu^ySK zQy*I>D+1bql|5O3xpW>yo_!c4p`zX5ag=w)7?YAvDYYW{!jN<$rH(0nOfhfreAnl= z?u(=e|838i*7X$spJzfBS8x2JWOrx@Reqf_=>7hLd@fH@H2vVOyx4!s`|-c>CjTvu z5`#qi_>aB_z~9|2+?YB!SVdmj7tA@UC@91qOX{|7?bbgnIZhlGnv#s?JOQO2{66Pt zp7YGCfBcqb(Sb{=l|E<~TPE;RON?r0K_X)ci}m@(T;lGliT>=XXJx)Y4fjoM@lP1o@2zdp*_xgLq(y71wBOhx05T^^Y^*;&yXBwgMK!!u zisY?vjx8BqmQN)d%)k11{K?@Fe8-!g#aT|hIr||}HgUH0sP@^3O|0eGxLhv9K>dBg zd!Z7$h*9f3wkiih`uzn7m7V^V!FX?+Yc|*y>ge>yMsCVJ&h#Z^4_EEyQUpyaeX=;9 z8^qp6Y5lhINj{Y{C`ZZgcQ%eV;Avdu%%-NjTVP0yLvq{EY+4uK=D7!vAZ(N_j^hZh zIs#LyU$08#+-;_O%_yllW_`kB*F454Qs~b=F$0BfD8jj3$lhN8G(hglOzM&Mn><8< zC%Di03z%;6>M?;YZ4O#dA}8E;ad=~k+8i_GkUjDJ)f{S`POf-rTN06^G#oeYQ>uY3D(xchZ8PkNsWIf+^5+cbK#_W&l${#P8dr8LF zR-reH%Q*V1toJ}3QO~NXk}k#gH|VFT4Be6`DV7o~*?ZsTzT9ztF5h?_qV&3Yp4&Mx z*TtJf?nYj6R&f*WQwHwok#|#ajkW8SZJp0f4=$3JpGBAO)*2waHBO>aG$yw}t#>Zk zF&fDg2Kc`F1!J|1eia@OdQV|Me5k+l4%g#%Tso#^vsYkLVe9ie?y8Okt^3FH*DFg5Rt7j<0iFGk& z%QpW6KF^&kfcyZ8%om^#WOHo=&qKLfdM4Gi8E8s#bhZen2iK|&nt?TF9>1mj+{k)}bJbrH zoEwRTI2ZLX6Pa>Z0%r3cOS7~NfuvK*vKcJ1oIH&BUeC2}(Lc`bWD>hxIA# zt|cEr-X7jsQ0EI7eHiOFKzM>Z#GXXpl(YNPqEl;GFVaIOpUD-I`IFplQDm)*!AYJ4 zzpz1Pn+0>;CwWuKK0B648i{j@3Ks<(zRUQ7YS6(qVmB4r|5SMzeb~8O?rGrd6JDFf zUrV0%e&rabFf3wGy4nqSIn4JuJ~YQwr*S^|eIpIHr}L`KZR9`K=2DM(?3=i7F}#5y z3#=*@ber*^viH8Vj=^pwjDQnfik%KSbb2l3^|$jL(>F5lxiQdPf%SWc;etgZk;A1g z80HgNR^IEpxFXZZ=z~kK*EaHf{wDU4dI;xG#o+WnB&HRLEgBmsznv5%t7S1(*Gb#yGww|!gXCI2~9nfxYVee zIQzwmQ0S=AoElB9TrD1QcPBCTaqJ>b>@{1ESpC8>`NWp(z z6|Fs*)QJ_`3tD27U+p@j-4<%B2R#7%=fg}Dtj8E_{@;}K0oaCm0rwFM68L++_|Z~P z{Y`EEX1D(>@d@5IK8P+g9nJxwMtu*5hd}-(;JuJb)V8=;h!%Av?jLc{3r=Exef|Yd Cbb9>& diff --git a/summary_results.json b/summary_results.json index 5c6baff67..5d947778e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1188, + "Performance_Result": 10.1247, "Performance_Units": "Samples/s" }, { @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.81274, + "Performance_Result": 8.05816, "Performance_Units": "Samples/s" }, { From 849144a58b98c8febfe50a0430b0c9a363ecb4ee Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 11 Mar 2025 15:58:51 +0000 Subject: [PATCH 293/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10047 -> 10049 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a3cceefd8..c883c5514 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-10 11:11:58,576 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-10 11:11:58,577 submission_checker1.py:3270 INFO] --- -[2025-03-10 11:11:58,577 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-11 15:58:46,093 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-11 15:58:46,093 submission_checker1.py:3270 INFO] --- +[2025-03-11 15:58:46,093 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-10 11:11:58,577 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-11 15:58:46,093 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-10 11:11:58,577 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-11 15:58:46,093 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-10 11:11:58,577 submission_checker1.py:3286 INFO] --- -[2025-03-10 11:11:58,577 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-10 11:11:58,577 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-10 11:11:58,577 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-10 11:11:58,577 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-10 11:11:58,577 submission_checker1.py:3307 INFO] --- -[2025-03-10 11:11:58,577 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-10 11:11:59,643 module.py:5249 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-11 15:58:46,093 submission_checker1.py:3286 INFO] --- +[2025-03-11 15:58:46,093 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-11 15:58:46,093 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-11 15:58:46,093 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-11 15:58:46,093 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-11 15:58:46,093 submission_checker1.py:3307 INFO] --- +[2025-03-11 15:58:46,093 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-11 15:58:47,106 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,9 +29,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.379315 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.254129 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1247 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -40,6 +40,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1565 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41937 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.374647 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 72f91c4b7..202802842 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -920 +921 diff --git a/summary.csv b/summary.csv index 021747072..3a8ef7c2b 100644 --- a/summary.csv +++ b/summary.csv @@ -7,9 +7,9 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.379315,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.254129,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1247,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -18,6 +18,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1565,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41937,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.374647,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 11d3611d5..4c8c6adba 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9761, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.379315, + "Result":0.254129, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -579,7 +579,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.41937, + "Result":0.374647, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 6bd07ecf9300984c95d178e64637a5a4eda76450..c6cab24ad0995cc9c3ce892559ce0af7ef1cdebd 100644 GIT binary patch delta 3051 zcmVz0NqAhKJ_2Ffo5U`q#nNDE=cEhGA-C ze-%uVLvxs>v;BHK=?=q5C#glVaGG7|NAaYSWfAYJw&HB;7 zqV3yJ_%+%1Y~-nYiK0L9A5MFRrj<+Uf1rK8=*DGi^um5;K2EQquOGwFV3=mZ=HwyW zXq+tmHYTHNEVXG&IuD0t!!*8*dg(B)*cgs_y)YkW)9B8VG@AUryrfnYEJKyMidB|c zM+Y@!TdsEU@8jDzT*cIvtuA`_u9J3-4&vym5ieRuW}W<-QTtkF!_v)5Kjp=Ff8Xk> zf~&^@Bg-Qnz5AoaLH$0jc%}F(L#}1PjV!p`D5z)sS$9hFPHE_awcUp+RM?K_CXu+? zl4~OIwk0=2;%`fCi6mG{w$g3zbQ@5C66vBs^z-wxitA?;H_s|=ixgX#+B{Pm;JBV-x7)V-cN``4A?s&5y1XNB=)xCJ&_2W zJ`joE=_8SdJLr^1#Lzd1M0|f|DA8hcTSOv8_nb(?=(dSOjP3=Ih|#?y5;3};h{WYL zoXc-G7bm3%eCl}r=;OaHf4gd(w$ENK7DGw48XAn!;rvUj z8%^qyarXBlTj^wqgW5Xkh+g_QJF? z8YkwPgL>;w9*BOP7KAqg0m_*yXpsiwQdGvYF-BRU6ZP(EydGfr)ahp;Tf8NM%6|AmYFwK9aF2l=bG=iJDwsHKYBV=c$(w55=_*1jsz1m2qw`0 zm_!3$q6Wd#MPOnD2xdVA&Gp46Fl9-LH3DGDs5Xg$e@#`QQI;pU)c*%uR*_?web!aS z*3Xnq!*O@5WBDQJvBI-Q$CF@^^GR2Ni5dixXaG#20WeX6>)1vSn4|&#v!H_J`r;Fq zvZTct@gi0GQ->AHcMWz$6s_m<1Iy*B76_lqD_F2<)>$RGUQsn5ZFO+AkVR zvFrN%OjeOzDJJc0CSAR;VY7|o4)cjZH-QsKBvIq}5=qn`l0*YY5)B}Unm|b;ogySj zBhnL4R6%on@d-&;(qg>~B$6_!O`^!tB@_9Qe@UlWCg)$Wsc^o6QUICWPZqOfK255i zrhfp*DzYo}TWC@E7(*Tl0<4GioTso`* zK(ffEqrj*GB`wkjoR2_Mn?w;LnFuy~s`TqfB>#>>l1dURMdhY_M>lH9*}2=ZW#6%$ zf2c(Bhb#R%MB_^X578i%B`iEl15^?XP+8>SVHE(CMLr%SZ8m36(jtu@m0;H)QMh(x zh8E!A4Hh3LX^}>NOo(ceC|Jc34X$E!#TK9c3Ye87S!N}x zH(fqf&2G4Xvt!pss~u#bW5ksN&xbUEWP;C!MB$uS#^!; zFV*Oyw~an_!*i@1yFa?IgG@AyxYFoDG`=+Y5Dk)9!sx>^Kqk=unMFn)e^vpIS!DFl zVDy2K7HI_h1Q69`Q9x!HqmQoG=<^(z|EZy%lKe_Rxoz^Xm92sun?Ks#0VNtnTxs$l z8ef`xhz6l7Ve(-bfRcE0w1l$APD2q%!8caS=(jtv03#bH_4-JErEEB;>wql#l zb6BSRoJjRUyRntzR%!vef9dhbo|^+-%l|=A-*|`9B#Z~)NjM%SMmNfaYZ`aMbYLWZF5iXa=I_F?^2)mXtNA!I(r>fy&VLnnbBWD%2ZAs~Nl!Y~ko?|q8M z_rx(N2w6#x+KN+=DwV1gH_KkYYWySHZQ$)|LI@z`)Z=<)znxvVTpU!B{DSk+=n9c6 zM+s1&{5nPr&x%k+hKpp_&3`Dp5*xqD+o-m!0-u}+JT6Aq~HV6{z(08aVQhEBP0DjALzMy(fda38!JjBVRyZP8g{euBSNukYQesnn-aaX@t`IOPsl zFs?4~Z4nRGFdtU ImC=@|8qw|joJQXDUxBJ0@^9h=aL#@I3F<$yu_DR>3Agk`s!0j}0OON{C07CEliVdd0dJET tCP4uVlUpV_0e+K~CMEQOgvb(#QfV?797U92!ikx(&X_?ovMWp& zTPaMoY}Hs&WEbjfiLE7Lr(=2+--2m4I$g4V2#fQ^Knm2S%cKAU;TiBIJv@WG#0J*AOwU{G$0 zsgmV&MC0~xC{#ux8|8 z-91eBkd9%G+W(RPHlJJmGF*o@lYntWY-KSi((PtpLQi1b2m!SdpgWI@^Oo-{*JxB@ zu6ED0hV9xplj5d(6`&Rke{cYAkOd|iRScs&9WHCt#OCAoh!soKu-CEVneY1C>%SBr&mKMeb0oT@}E+yKb>#UU3cyZoz zk&Mm-?Bwh1^s`&}X=2tUIy=d-Q(+_@mIzB;q(U=+xcoig{JyTOSG-CdTl57(K)-zS zcF~ixmFqo+Xu<9Opyg`AYEIt2_RtSTVQaiV!Gc|GngheEuMF&VoR{j5j9s;Wv|5t3 zI>yArq^KtRT)uga#3_O$s(rQ)zuTJ{^+H1xlPDB+DfM&Dk=_6W(iYVm~h+K&s$ z5+hstlgQvuOMvgw2)Sv1{z?_b>Di|&VH5_y0`@(IQRFs{Cy^HxM3MP&L*hwfuCXfY z@JkP6iJVX~;7Xtl1rz&SRXJc?Z0g6scM-ybJB?XQ68xc&RL0viL=u@uQh_BcgeHgZ z(Ca_cl=|D{KjVn5jy5xDy znq4xEp+0>cCXd7#lXf4Xg}N^qSS{=C>rzT%ne-(&RC$Y)v$TF4n(m)+uJ64;;g;Go z-WD#r^;eL#gX;InYPEn<`e*FqrD9+zxVhi2YLoI_k;ExMxegIc7yPd~5Zv;8q-`Fc zExDFb0vP|Zya(xX5ad1OB&-w#lTC)%eMwW;hDja%BD-Uxwcme>FZqbu2PLt$d4;hv zS7P-r9kMnm>UGPr&AtP(mTkki)x~3+Tj8rOE8>^c6+<9ZWNm2QTR+}H0h=MJF?#3q_fIOB`4a?V z=Q)SA>Y)T_KlWf%F6c#ie^9#oMM*W9E9yo1J&$y1^ z zAsLDINLJ+}?=|PXCza+4VXv47M|KO%u}?asoC2p7DtVsQf!mbLZtmE8OuC3fl^{^ItzQO8 zKc6aZAZCBC_1uD4_0}*kuw!I^3!-B9vDqyyFsoWot8Vf#B-#1HusiXj10&E?CybIC z`;DshGq4TA^y((DQrenR2YYa=TrPr>r}+bN=`r3$GZRYif661%WALEU*Pi?hSc+WF9fiD!A9Eep3(d83YfSr;lbv8mj!_nT~a^20yNlLk~a(3{d zLJuK?(a-aGB&gbhsV1u+(3o+n7$_G+O0!XE+gmwQ4_2>BF9up*QK4gKsvP)b4JU@s=Pe| z?>9*N8K+R~JZi7s#iz&B<6+aG+>q``^P^atKsvMsl!oE=VtgjL3Yg;3x~|_^q}Y^h zBw8f?zW10#pNLhPzo3Fkd7H#ekIaO=o4z@oYpsMmJ*I@^=5Iapzk2Kc)thQ0{q`fG zQS1Nuflq)jL2gosZp~9yKh7=PiSPW?YiDmAPJi-cW!<&iA)GUrfR{Fp48QmgKwG#!L^Boh$7saWSxs3X#fy(5$joatJl&lG3~TLQ7b^yYt_q?d^UR zzxF;ZwDgNTkY0Szl^)GbVr;*-E6t$Q3wONLn1w!oJ{(;sa@*Co{>ypf&eB6%9ap?kbHx(R z7%Hc4^Ur?da$e~RzYASd%RmkQaySZH%%z{#=TphZ?D4MK_L?0?QM1jN8MhmMqA$eT zE|^;z(tXg~s_;|k^DE}uv<`a$Miz%Gnv&VH03}7zjYi!A|15+F=hSL-pz@AY@%y$t zjX3Fe{-$Jii$rC3Vz1$yE%o?$VARn}7LTic?BS#hEsqQ57-l)&$5OL}q^R$QF2g zb0YbTf>`Hy%Yd5-8L6=z{Ya1D3+CaG^&g)v>=c&S7dRg_4Qg}aXxcyb_2A7~ySiIH zz7!+90QpgH^~3eyXtQ@g4LJv%7_=hj@3%Lu0qm1O4BSJ5uB%i(p+j%njQs4h{J-g# z{&jRiKLC<(Ya~2!6Ut^QxAXn4|9Ldh1w2k0MjC;AX$MheU}2gE$`DMXrJ_&=e>WBh g27&DPKp+Xw|D1af9<Y`&75H8G4_KkLeE Date: Tue, 11 Mar 2025 15:59:52 +0000 Subject: [PATCH 294/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10049 -> 10050 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index c883c5514..a08560bb6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-11 15:58:46,093 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-11 15:58:46,093 submission_checker1.py:3270 INFO] --- -[2025-03-11 15:58:46,093 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-11 15:59:47,587 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-11 15:59:47,587 submission_checker1.py:3270 INFO] --- +[2025-03-11 15:59:47,587 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-11 15:58:46,093 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-11 15:59:47,587 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-11 15:58:46,093 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-11 15:59:47,588 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-11 15:58:46,093 submission_checker1.py:3286 INFO] --- -[2025-03-11 15:58:46,093 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-11 15:58:46,093 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-11 15:58:46,093 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-11 15:58:46,093 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-11 15:58:46,093 submission_checker1.py:3307 INFO] --- -[2025-03-11 15:58:46,093 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-11 15:58:47,106 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-11 15:59:47,588 submission_checker1.py:3286 INFO] --- +[2025-03-11 15:59:47,588 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-11 15:59:47,588 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-11 15:59:47,588 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-11 15:59:47,588 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-11 15:59:47,588 submission_checker1.py:3307 INFO] --- +[2025-03-11 15:59:47,588 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-11 15:59:48,916 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 202802842..f55dabd5d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -921 +922 diff --git a/summary.xlsx b/summary.xlsx index c6cab24ad0995cc9c3ce892559ce0af7ef1cdebd..83eb112b0edda5ac62347147faf355d692e204f8 100644 GIT binary patch delta 326 zcmV-M0lEIcPQp&GZ6O6*BaP=KlX4*}e-*WoAhi{zB2_9?QE!&L1gr6nY`1~8uM;4E z#Hq*i%zit&ayj3rHu(kTrO_24S&kCW+z6?g3Vo~==?o=4FfCYXG*oC09?k2!l3U7+ zgEwa^xFEqN5u`okwnEzwEXCOKEhy%*=+IHz7^hf>rE9Qd{F^m^CwcyW6$B<&e_*%= zrgkKvBU11YX@6Wh00hU-f&v{pCKhdvNV3_uTY zPs(TU{IXi0dXi73XrYVs0kg9r3j_sR YBaP=Klfx!d0ko4CCnE-CBme*a09Xi@H~;_u delta 325 zcmV-L0lNOePQgyFZ6O6ay<`z3lX4*}e+XGgklKn+il?OYeEPh z<<#SPX1|?Xxm+Anll+47(&!42EJq1wZiLjk3Vp1X=^Q0KFfCYPG*sva9xdvJl3U7+ zgEwa^xFEqN5u`okwnF<5EXCOKJt*e0=+IHz8mCx@rQ2c4_&3`Dp5*xqD+o-mf57kw znA(AedZge3(*C%n0|<_x0R=jEOfrJ6-T+Sd(S}aBaVi;(7DlZXad02J9E@$-W^K_~ zV}63aR Date: Tue, 11 Mar 2025 17:38:13 +0000 Subject: [PATCH 295/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10050 -> 10044 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index a08560bb6..36ed03c56 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-11 15:59:47,587 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-11 15:59:47,587 submission_checker1.py:3270 INFO] --- -[2025-03-11 15:59:47,587 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-11 17:38:08,570 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-11 17:38:08,570 submission_checker1.py:3270 INFO] --- +[2025-03-11 17:38:08,570 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-11 15:59:47,587 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-11 17:38:08,570 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-11 15:59:47,588 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-11 17:38:08,570 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-11 15:59:47,588 submission_checker1.py:3286 INFO] --- -[2025-03-11 15:59:47,588 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-11 15:59:47,588 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-11 15:59:47,588 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-11 15:59:47,588 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-11 15:59:47,588 submission_checker1.py:3307 INFO] --- -[2025-03-11 15:59:47,588 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-11 15:59:48,916 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-11 17:38:08,570 submission_checker1.py:3286 INFO] --- +[2025-03-11 17:38:08,570 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-11 17:38:08,570 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-11 17:38:08,570 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-11 17:38:08,570 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-11 17:38:08,570 submission_checker1.py:3307 INFO] --- +[2025-03-11 17:38:08,570 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-11 17:38:09,577 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.26254 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73409 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71474 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420566 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -32,10 +32,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.254129 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1247 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.8409 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.05816 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.95927 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f55dabd5d..209fc0eb4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -922 +923 diff --git a/summary.csv b/summary.csv index 3a8ef7c2b..a6a73efad 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.26254,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73409,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71474,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420566,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -10,10 +10,10 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.254129,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1247,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.8409,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.05816,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.95927,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4c8c6adba..4c2201340 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73409, + "Result":1.71474, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1247, - "Accuracy":"acc: 76.200", + "Result":9.8409, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.05816, + "Result":8.95927, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 83eb112b0edda5ac62347147faf355d692e204f8..1aae40c6e10added7212e8d3b4471989a9279077 100644 GIT binary patch delta 4180 zcmZ9P2{0R4)W>6~l3FTMHMWW*RPD{fQ?*o@O6+@MFO5BxQqLZ=MQTs%YKx_m+9_>q zjisonC>qpWThXGZ?fLZ0y!UN8kh@5?j?+z*ifw#P~ts;?Rq_`2&P#i9bFzb53qN!hwkW~wI^M9ikL7T&vpD+|EC zDW@h&p6mWIe)n7cbI7q#@^OpzpupPWsg8@(8*->9-4HBODt*Lub(j%av02S-zMx`X zUO$6EIb@c8Nz=LDo}jequGxQp*#NDvajJWtiIepZS-?S-5 zKjN*cIykQ6<>BJU*F&Lc1gTz0=x6s3Vp6H?F>(J}N84^IYEz`whQ9=5PUt|;mA=#b z8Yev9$FIFb?f1=p1w2zy#XA@YoAmV%B=S% zuIXFH`_QK){g^rLNu9b4@xKbO4#I03D0+jch!|(|;bOt~jRTtrr)_($q1_#cqb!dj z&xNuOB1L_RtLI_pa9^<=;bW_v-@K|K%WSoh$g_(>V~MlNyD|e&1HD=YL(14A1ia8g zs=-w^i0ja)AJLk?vSMRf52z9Rn$_Sr}lS&Nv zxxr-_JaG7CE_JSsy*cHKMeR&bVSeV0^Oq|F*NY3dM*T%(4DVj2WFzN1b!3R06KU4m z#5U<>ILTg`j(OQNjMqGaDT)OS4Gs=U%H)|@-zaaAG4a5;ZR_G=#vmr9%#dN{3+aQB zm4>2ipK4Va6&|<{N}@T#p3Mn|=|Ja#)gd5x`|mMGz@b3$=Lymt5)yEr0gZf?h=gq9 zJ3_d%rs1H?&drA=Q{npz=*^P%>iJ+~sn_{k-MTGm-psBDoyULn5?;<%B&&TlU z<8&`NF2wmpfpEm&jia)8>chknwUc#<@8pfZmOJb)y4$jU%G5v`;$W!g&r8ng;&hDo zA)c?Bnmy#l-%SsaeuJh$ThNPk@It8F0aR&EpXyfP!!N^!9SD&gQC$~&F*h&QnTSW0!K94~83EI#HNJyC&$R|4N z2c}ePmT`QZ^g}V~EF_vd;N5ZSEEO-G^emTvBz49J=M^Ba4nGH5z{zh~Ua#zIFJ0T7W8G!%q}Bx{3?K_GM!vgWQX+;NIF+e z^0XX;%1keBj!$RtS~c#jMWW?rBv@i7;mgt(RUfiY0ka_#L_B5j8qLLC-_|mWghZXC zO=1>`!Qx2K)QyhN$Fsl1UP2 zW*=?stf{Opq0yL(W&_dZI+O+T`Ab6*I;NqmLnrB)3&W-v!^*5wtiO6UI>5II9HuA` zrbKSIcZZ>`->gTt>{H|GO{p3^*tCbbX;brtG70l3ps|SLL1RiqG7(Hc9EL=2hVU2| z12YddO*8#yzNI|Nj8UoJHZZc&=*5=}5X%$*M(M z(K%%UOi&NdydK2_P<*XwZ}H#iGqnkV;m|W?JZCXcTJB#O9K8+n?>B8 zZ624(#4^emX|>neAu=4Qb`CQu#9A=A;}g2Cf*h$%0=U$Uz$^KOka z24qed@YUhe^q%Jz5CFX@wdFspS3J}#6jgD-x3u{(?(^16YFls)1<=y85Ha`8IC2gH zNXU6KwExYY6^t6S?{f;aEALSeEcr~>)5#P#+| zaNm{5b-Kd(9RpL!R{}M)Qi!ryjE{Cn>H~HQl`u#WSxTk0H>!c*eF}RCq{S>AiIpT& z5=L$%Ny7TILN;fGjr;D((XnBx;ob-52G{8tu=Jq+Iz>#UEgVx%Xrgs!AzJHyoaZpc zc63q^8n7WiDo#znEYfr`v~}HbAw1)}p)vHa=;% zJ+ihQ$l0IgtGHz>q~vWk^hSI-dxIK_L@GAoU_hx+Tf? zRuy8o$~8z>b4{yWMckKpD5tmIyaxHO{Ekgs*HRm!lQh~K+qZ4|3)|WZpl> z?fnW_wPZFm5gIG$bOk;3b-Iu@h&nmxux~XjOq^0Y_$)nG1N0~Om51vh;+ts8r}`ld z&U28k9m}Ws!G{2uIFyJvLlV}B@N>xi=RX1-F`pm78EH6OC`_!RFK-KGo|F~jryWUZ zHsiNyQeQ+y4v63{Y7nEiEwX;=2N$7eN?gqd$zaKc3JXaWNL|~nMgjU1$cZdut`t9Q zu|7u0j;~r&dssiRn)HIcxGk8rA}4Nq{L#}xy4lRK-7_epc4?8&KDI?jLMTm5BG~39 zenszb<;U z*KYM4P-FL%5j3wl?`d`sC%RIKxI6e^^z7Cbi*705TbmNcjUG1$`=cR~*E5h0$71yd zo)xMu&3>~uw%D-RC~rMnyr|VCzZ);LbK5oT3nlgF;^>QqE!gjhs_R`Z3tb)JsCUy! zM}Fvv*|zGrZw7-0_}cy~*@R(+UABI$$gFX^RX#~MZV^Bk9sCj%A?;Z8a{kTu`^!mw z1q0VH@dPn2-_N_vK#t(WGOe2Dk-zZv5Fn&LO)~cZ<6<9Dp=P<_323un|7%il#Sv#B z5Zmw`B%1F5{xFI_uPC5-3S@#77;E7{(t|hY+x7QEy+81tbmajZ`ELSY7gl>r_X9pS zY?9M3##NbyNVlH>!9&m%miHV)r$#U+kh-P7YsejQVhEP@fdYM7@)T%7`N1ec5z|j!X?xZo#8H_t!nucN zs_ICU_VTG%aE*390FVCsqvB_SP$luA`&?I6Kn>is;`bJo6+ndBQaaBK0kb&pW4Uos z953HNq@Fxe>)r}6=|d$G?&TxoZB`oO>l>C*$8)Jba&GYz8rNZ{R;s(Q0$8DSdVM;x z%<@&jeLpYQ;jtSL0JQx`qB_Lu*|pdpXeR`Z;JYJ!or;@FQG4*WduCJDFZuL84z=#4 zCX)6@$Nprcj82u{Csmy6XW{!Z^s>R+LEg8~bhV_O!HZMwem%ApQ%3xW*~3b%-%TE> zMvhO*vkO{nNP9pkiQYP)s|wAB6*%qB-yKVFC>PfURq@~|`h?4KYft%JQR9-uHh5w0 zrB~s;nf4lW6TzSAm|*QK*DM?fe1D8TIcT@6=OKsjdx}nRDjT0!4fmD#YwhKGq(MX< zETRzlcdu28({y%MGyl=qI`&IIjY+BC3Cm9PS`ODKVR6@C^Ht(}kW*WZ zO|$iy8Tlx3o3;#0mb%U6;`!*)1hAzleMJzaCMSKz5n)=&l*qTE*DL*%(A`mfjn6u1 zhRjOKy>C@u!w$I7CjrCiW@QgE&>ZIXoa7WpAhQX~P&a14t)$QI~y6Z^f*`gU9 z+TTr10M>*8hBGPwx(bm7Qvm(_a*l`h7fluX_$n;D6f%+KHe delta 4125 zcmV+&5aRE=PQp&G%>xSXQG17R1ONbf5|e!h8ME*Mv;%*R-FDnA+<;l{_R=|FD>5Nj z#*L85f;G&y(dN8i<_zY|Ui=qaZRU$8RV!PNH75rmEmj8q(Hv2DgUO?7D`R0it1@DU z*HUij$xA8Uk;>IHqL;WijW9}#=ANlsj^_4(`dgAsw&bS71`el_q@Sg=pB49 z+yi4|$jn+dP?=rrr9Lh|!Ws|DU`4J2wn__bWF(e}+HJX3vF3cEz818suVk~nq(abq zbC-}HB23-M-EB;%*xaa+yd4{je*#cT0|XQR000O8002J#xAaA-NeTb}X{hni?WHT#v zHW!I$owvK+wjR3GeWHV3A0}htK8%x5G(9v`%`%N}+Kqan>EO`3xqiRrn?{m$roGNM znudqww=gk(J^I(d*C_rY8HQnMWPcS*lS6ZurnCKeJ?Rd^NhhgAvv8VS=|}OTlV(50 zgL*QH!%lAzG#S^mWx4f9XEZgJHSEV_>WKRNQ8#=Sb?1|Cnl3Ae!*M6gx}OY3v&1|) z=#8>#c|JxQ_7BZtwSV%vw$1v{!J_TkQTR34_-y2wlnqzv#wg zZ1lo@XFg7^qOTvr(O{Tn!{+26-DsRF{x&9~Y%H~DOgayTX2UeTj(X`Zuh(_^y+7jt=7Js}V0+ zNM@bKAHDmd#zFl)uXv^SEJLnk!Hq1q-6*JM z{aJTP^G<2#gSFj?x1R{lV8PmoXIpbWQL|NjVyL{?+|LEht zFMqpgowme}=bwncW&<|b#btGIsds-0Q#PC4maJCsP zxM*RxEe7)3lYdB(7}4VklqgG#{-swTl7r!0r<*^|>}8KH%c}MU8IZGYX%u&dclWyH z)~vnmTxAb6&!bpHDkV_7@itIw__e^__2F z**hl1fTI{7iUOjyDDlKic7hT>kw_$pL?Tf{i7)QV4?zKn7#@gzo)&~R0|CmJENGDi z7$@P~y{ct5@%PhNNrolxyk2$e zkxy;A8&4J6RPt^2V};x7jw7i=jb}?L$#y+ZNq;mzCD8ztsDX)ct)Wz61xRH<1x+On z7S5og#TxM>l`^VLqHw~bA~|DjB=8ck+?XzNF11xtKLBPGIhL6>R~=KUz2}9NAIN5_+3lJiMdf{7XglV|`; zq5&{bgX`Ev5tyU`0JETi=KA6jn6jkB8u2BVGOA6Y5KMBijbJwZr@&M{w34kN$5LQ6 zw!qYmt@MtAZpF{^j};!3JH7-HHJ&HIM1Ku}Ni+Z^(Eym_cpt#Di@+om0GI_8G}jlO zz?3B|(g^IcLR6bY0hp*EVA?MlOtI_w{Y+MoUMVK+Z6;m4v0<}~VoiJCx3B%LB8Nh8t|P*g#4eenrNS<+&?3?z~=s!gKE(Bvgiw@HOl0@T4 zB#T@+tO7u?$fu*gr~@S}(g>W7KvbJV5hR%iHhik|>qsR3jzf}45-dgKrhP{@YRcKU z+p}fgv7V?z^M@<_J4EA40}s(4l_e}ZOaoLB4S!HszDaB^p6o>E$6BUz&M{2B|Dz z=V2P4l4yX+B1aFa0H`eT^eAb$IfIfGX#|ff!Ma1D@W_&xci_mfV)qVG`SZ^#U#P3+ zT7PZYdfb|J{9O-~!44|XFXBpD57GG2*F!W&WeH~w(*Tu315_57dsqcPWs$u{fxibz zTBH#?83BtAi6Sf5GA18YvCHQJ5s7BXDR5QEd{1b0!tZIa60`^Z8yhf9TkyG>y2@=tDHVH2M$?l3Bv&!!$r9(Eyo6MjuuI zkXdB((O~p}k``$M`~(oyW>G+98GoaXuGr}F9GU;Ap`eodN8{-!9()^(4>Qi>AF%{-0Jm4vj4JM`@PbqC}Pa1MAk&!K^a~&pYv8G);_g*v}HRYMyDt%lB$6KBv)a z@yRibFHxFClgFQiVW$_y`A^yJyM7ds(d_WvG_?#C8*a-sGGRs5VdBCq^wFB zDgs9!qjN(y7DtuV1iRntG5yVtx@*-q;oUE*a8GLNALltq5t6IvEu8fq*Qxl7g z@9JrLDs5Y_(QV$Q@WRvlLTh+g+BCcbrNy8qP^wl+PruB*qI+O@{rl51NH1{im1Mjc zf1&a8?;-L$w~O4KF=QnmeF1GlgSIDfz}i!-vzxn%F+M>qHa(kQgq{F$g1Vj=xL&XW zE+r6{T^D-@-QIj3U(XlQ@oX_ayP95(>(#d2br+jG;=8|GJErfG-Y?I@Hysc6T0LWj zm^MNyp7}py=(j~GVcV|ZY`H1+(3VX|f8z=ATvJ{rN`nTTgOW8Rh$N>ih9#7urGQq2 z_@@Pyx1mfQBTC}9s&7qWqqLk|{r5DuqrsK5+TorC_cge(htIPGvvU&ETdhLygy z!!>%bMlaUr#Tva>qZe!R5{+J>(MuG)5Urru(01*~$kK3&EW)CA>SHX=H>GS8e@O;P z6Vs($8$(-rUa9RltT5?Wnk4Sc`w;-H+h3etg|P5wJ;o* zKxG1M|6w6wa{Mvd3(Z8TLbx0vJ*DO2-blhNc1UBI~vFreA(J?hzkB5?1_ zRqfA3jkZO7U@h7elXw_zAbuWjY<0jnmpk@bW}+vda!KGQfq(K=ui5uzaV`e`0ka1g zVF(3WBaP=KlWid%e{RAs5QOi2ipclGF)0KnJ}@mXy&{i!&hx)hvp2P_y@m-w=XhijP+%Xs3A1o5 Date: Tue, 11 Mar 2025 17:42:08 +0000 Subject: [PATCH 296/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10044 -> 10045 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 36ed03c56..2a9ac45b3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-11 17:38:08,570 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-11 17:38:08,570 submission_checker1.py:3270 INFO] --- -[2025-03-11 17:38:08,570 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-11 17:42:03,103 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-11 17:42:03,103 submission_checker1.py:3270 INFO] --- +[2025-03-11 17:42:03,103 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-11 17:38:08,570 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-11 17:42:03,103 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-11 17:38:08,570 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-11 17:42:03,103 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-11 17:38:08,570 submission_checker1.py:3286 INFO] --- -[2025-03-11 17:38:08,570 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-11 17:38:08,570 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-11 17:38:08,570 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-11 17:38:08,570 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-11 17:38:08,570 submission_checker1.py:3307 INFO] --- -[2025-03-11 17:38:08,570 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-11 17:38:09,577 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-11 17:42:03,103 submission_checker1.py:3286 INFO] --- +[2025-03-11 17:42:03,103 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-11 17:42:03,103 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-11 17:42:03,103 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-11 17:42:03,103 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-11 17:42:03,103 submission_checker1.py:3307 INFO] --- +[2025-03-11 17:42:03,103 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-11 17:42:04,056 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.254129 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.8409 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3282 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.95927 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 209fc0eb4..0dac14917 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -923 +924 diff --git a/summary.csv b/summary.csv index a6a73efad..c22cf5ea8 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.254129,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.8409,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3282,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.95927,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4c2201340..f8de7e5e8 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.8409, - "Accuracy":"acc: 75.600", + "Result":10.3282, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 1aae40c6e10added7212e8d3b4471989a9279077..39edd4997b389e0daafc5239266e65244bb065a0 100644 GIT binary patch delta 2075 zcmV+$2;}#?PQ6aB%>xRBU5iL@1ONbf5|e!h8I$k>Cx5H~%wW^YzaJZtPIG*k8SP7@ zeM!K4-?`z>K6v=`B}fwbPvK7_RxTkegccl@LIH*!FW6$S=t`Z@Q z^b{pzDX)KxjLOowrjZ3NBeNqt&`2c8u#wOxA~SjIHmtZ{*<0|58)fZ*7qdtO9sz$96-l*hhb*(QRQ_hN2as0< zDdPprV^YdahAaf(V!^9rM!9MV_~NP8wk=T-Qnw$L>N2eB>153>375M$w^qp_2vu9r zl3he&sudPht9TWx43ou^+xFUaYrXp~LYBvLWqvUuA_AIn^I&Mj$mVwc89B**j>ypJ zzYpS?HF1AA&4%3W^nLr?s<$P&J}$bxE;=v`k90@}uJQ;3iVlH5(IF5hIs^hm*WV#J zgbqcwgMQhEqJsld-GL@L$hpxuAnRMiDk$$=|9>dD&&cTyMEB6&5}oaIeedvtt41fHfIfHrgE&dCxKIZd?s+BEBo09`U^OeDWG)ENPVDjkN z%2U^yHO;8vg`PO9KRxZY3JCg9!Bt1d8yogOhv| zSpzz)?vvOQ$A1*bxS+`heF1Z2A{kh8Rz->U+Wd0JQdWM4M}@3fW{D6`HVtQMnxu4@ zq=`@$J#qnA6TOyaq3n5DvJg0DivQwIg)s~vg^~*OW2$^SXR-K6OC_k+OsJc<;}ErG zilnSc8Y%)uAf$T5&rtekVvcDWwryh#PBeJ8iAPjWnSZzPB&@nhtgFi*%Q7YtWc9G1 zp^!Cbln_HHOAw5(sEp!KMk;(*tcmGLrxCPvC(bbrtrek{6cGZS=uzb1f|8GC{U_aN>9JczM^|zdHws-Ge|FR z?v-S`8h@ej^zR|^JhzM7o-t%4AbkOCLxZ*_a=_YCt+SiEi!nYyE;c=zV1%9ka)P>^ z8Mt1s11=>Hm|Yip2;JU%A79TG)A4LEKf9Vt8+Tsx-klin}S#5Wxe z_gXz;hnO}(E1vm3Wazg=Dq-8M;cU4n_RyA1NPpuA@?2A1CrX0`o`aG#C5R-aErunO zqNRXVh4`lhmbalyA0tZQxTMOdtRySIjk`0S(+sMu-`XixrQU7A>;eq{+*GkOV?43?yY3MUo8

tGA1$^#32*o~)XV*e3p5d2b?oU*^4&5w~2}$7MYHD0wg0Z#Q{^NvyLc zeYG$gnB;?Jr#^c=vk*Sa_JKHle&M}FcVGO}@nvF1MKE5GhmhUz5?~3|+vv3^1VC?LF$* zfg*74%vJ5rMUA#aePAux6_a=vZXkXhaBOwJIhQ;3TV|psp>j#!D1m?SRS<27lWZX$f6HpaFc3xe{R*SKEz9l;VLQ;4tQsgZrJ*|^O%hS-fuz>) z-&b}LCxNWG9Njx-?hLACJJlw?;Jh@tMkLEo0-75k^`=H2%X#{U5+9frtTh^H)PqN} z=BDD7a^v9584E5*@JR$|Pr0qpHUvvC_IwM9`7Ang6xYTn7Gmi(f7mkq%{G9GJio^Z z0uwASJOEQW5zzrD_=L1St{nk_V`xEv4jz+?;FC9iQ~qkhsGK>K484V`)}uJNcV14$ zuIsX{9IY`w!K=mV`?zW<^>8WQNQwbSY+P^ zB8dU$KJHQZET3PNC3Dmid2yHKWlG4BJkZ;M=H=Hg@}mC)REHt|CLRI%+?SuA`2(}A zB7_17g-Mp zSSC6FdXtkTCL8K$PPruk002+|000pH0000000000004l2Ta(l#J^`18BPP|UA%>xPs|48$21ONbe5|e%i8I$k>Cx6fYW^B{TzaJZtPIG*k8SP7@ zeM!K4-?`z>9DDfnCC`i}!gxxHk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX z@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8uxLOzKp%L5^A*~;Q2BSBP?^2~B=+q3c@ zEzHKkfPejo4I0O3NRDY(<)jde5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASBa2D zdWsU#jMu+Lp32g?rjZ9PBeNqtq>)IJVIxVSh%*$m+pyw-=8w%IXKkRNI^LS=*v=dK zuN2eB>154kgv+lvw^qp_2vu9r zl3he&suh-1t9%udg~{T{ZF_CIwf_1qLR!S>%KTzRL&&Wykb3}$# z|9ueGtciciX+Fs9PT#lRt$JIc>*J#9>!JhG@JNSr;3|(mpy&_?6deMAqC+51bp0Ko zL+DU+JLs2vC^|SW)g5S}gPa?k1G2tFtb+31_5X*W`;46aKy(lNEz#Ld*Y^%T_|*Bh z>U>>wpg}m&Asw*LP7nxG9Rh)>Lm*If2n2kY&-{NKszc}?+lw9aOCPEZ4or0iUDzo> z&W+9i3&|~F6}CnG=c=w+R)wfqawDxwy!hV70_^@50e(h$e;~lQ|4x91ZNF>7osIQw zE1eUzA`_Bj+z6?xS3`UoZH^me&S2hb#ec!oUcQJ@wXg+Qb8sNiVrB3j%?X7!m^`|+ zG8TWvvnnHocrE3Qp1hRu6{%EBBYKIO!w92fDP^g6ZM>_)ktz$uIgR$~psFJMQIY+< zaL*g;h2Fsz!#yxYhD>||mD$x^>eB)wtnsi6R^&QhtF+)oMq-($-IhxgYmPVSYeCEU zO7`kYDg@0pcL@n1!qlDI+{ToO&5bI_+o=c9_$N?H0|b+AB^tAY2=xmDa-T;5lYJCf z1L{_vli3uZ)5R9;>jN(ydDtuV1iRntG5yVth@*-q;oUE*a6?|;6Ly9=aRV}5YD22K6xGVm(=W5H=pI;J|Nit0(hHn> zr5LY9e`q}Ydx$*G?IO2l3|R?CFQ9E|(DptYX~+nev>>-l0jo-O8QSJTUJz1r5h?qah?eD{}Y$Mk*D`{kMVrsLsW zt7q&G(?)2;GyjK7{Z^zBw(T0umYZS^ZP|o0f1V)EF ze`KIEF- z(2~PqjZI&@T{P1Fe{6eU)pW!*@#lv3Cc^hs-a8p_%Y}Vh#>0=2_mcf~lQ)>eI(v$* z4h#n-`QX{9&z{c|!iU*DkjKv#-fMKnURT~x7>}FNH)-!vgFJ%T-!TjvTkb7vBCd&W zOMU7xx%;eTNZ9XVsAWcIX?3rS8n4;gXsTRpG07oPrslsUqsbY%fODB(K(pI>)UyL6 zz`Zk9V}BMk+KT$XTC@|BdKhjY-VQiY1?qrvE_dv=%tTLt$|ZrLB>u@;y=LE=#km;# z2eSwnVF(1{Pv}pRZ6P0j%}&EG5QOhMMdW+pxDimSB$WX5L?tAOK-?^Q+gijwvJGk8 z9w%wjR_cl4dS<_!UAbKBRg?UJ^U~-Fkt|0EXl{hm+X}s}m+2fOJ}@mDE#1CJK< zP020g#=)yI7F>|vlL*qDa$BKY2$o{(`3@BGS#;;W&j9|p#cRtcuX>aPu>7d`Kt|sa^_SrbQZ2!kK*LsdpQ}~ zw$0jNu*UoZf39BMhE-Fk`%`g1btyRI4p=a*F7bI057#gsR`J9e3F5Jf`jxlCB4Zzj zBnF^k+@tbQJUy=^m#CiQv)eQ;QbN|`o)&YOKYaBgFZxeFbr|w*;t??BzWfCBAG5C_ zgaQc$|48$21ONbelRPC?0%Zu3PbC$TswFQ0;FH=VA_Q`uM*)&176EFL7bZag2a{PQ mIsti;lqMzw<4@>MlfEVv0a}yQCO!eKlN2W-24Exr0000S!|~<- diff --git a/summary_results.json b/summary_results.json index 8c549e955..c08296f11 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.8409, + "Performance_Result": 10.3282, "Performance_Units": "Samples/s" }, { From 9e2064452fcb39f675a3e160a9a894acf71548bc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Mar 2025 08:06:54 +0000 Subject: [PATCH 297/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10045 -> 10050 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 2a9ac45b3..7ddc25e8b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-11 17:42:03,103 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-11 17:42:03,103 submission_checker1.py:3270 INFO] --- -[2025-03-11 17:42:03,103 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-12 08:06:49,168 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-12 08:06:49,168 submission_checker1.py:3270 INFO] --- +[2025-03-12 08:06:49,168 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-11 17:42:03,103 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-12 08:06:49,168 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-11 17:42:03,103 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-12 08:06:49,168 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-11 17:42:03,103 submission_checker1.py:3286 INFO] --- -[2025-03-11 17:42:03,103 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-11 17:42:03,103 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-11 17:42:03,103 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-11 17:42:03,103 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-11 17:42:03,103 submission_checker1.py:3307 INFO] --- -[2025-03-11 17:42:03,103 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-11 17:42:04,056 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-12 08:06:49,168 submission_checker1.py:3286 INFO] --- +[2025-03-12 08:06:49,168 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-12 08:06:49,169 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-12 08:06:49,169 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-12 08:06:49,169 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-12 08:06:49,169 submission_checker1.py:3307 INFO] --- +[2025-03-12 08:06:49,169 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-12 08:06:50,221 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1565 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.374647 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1555 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.374647 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 0dac14917..ced7b1ba5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -924 +925 diff --git a/summary.csv b/summary.csv index c22cf5ea8..326cd3f6d 100644 --- a/summary.csv +++ b/summary.csv @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1565,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.374647,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1555,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.374647,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f8de7e5e8..cfba0f19c 100644 --- a/summary.json +++ b/summary.json @@ -544,12 +544,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1565, + "Result":23.1555, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -574,7 +574,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 39edd4997b389e0daafc5239266e65244bb065a0..bf2703c5f3ce347ba4148c040663b3af6c0d5896 100644 GIT binary patch delta 1296 zcmV+r1@HR3PQp&GbO;5MP?+9W0F}f%15}oIcUU7pWtn?NNpsB!lyq1xfl4sxkSJWcGMf&pU2FE~{QXeAP?O9L zCx3X7$}%4hs{p7hbMh!@v^jy24rv6BEWxltqVUL)S$5#avS!Z?Qu*`GEMKUX=TdE& zdEAC}{Gt_XP(UTRL|kd+AsS!0d58w7tl;Ni8laMBfXXsU532yEEHm{eaP>e*hctpG zBVgUF-P)mN5pxie3*veKx!RC*)3!p^9h$~G# zMB_`7578i$6-+)%15gr=jyx#QWhNii2tZk8^3h=OfszhsL|H&3xO`|Btbb&g2v)K+ z+kBqGG9Bhbsvp{ottGcg3)oGMPxjm#_*(uClIF&nn(b@x ztS;2<_43V6Px7q2XgV0>|5dd&UcRR)9ZheI_Bu-yC&SXN$He|Rw;herBs7Vewj z!z}xT@?=e8@GxAy`^uQbqkkwKrQepV=P%!yk9!A`Xz+T`srhvD=RAD^H|yPor@i=gG);_gILs2Y8lGvy%eQGQeov#>;y1@M zzC>vnO&&kpg}p%-=by5_cf%-3A3x+nvVJSs!Cz2I0|XQR000O80JH28eGUb+Le+aF zlWrj%f6Z>fFc5_AeTvBU#7@#8Y9&ExD^5kKRH~xhEPDZ~@sDh`A#Yz3Ab`ZF$Mwv9 zJG*i<+o?AB1?Q#FHOe!B642ZTsW&zHSkBWaN_=2iu-0g(Q4b!?n!Acy%8i3JXDqlN z!6y-|AW+Yl_p*z+wY=CkO~QCu6RScs+De_+e_H`@R%2$^67fe98E-UCxR644PU z_=vPWt{nh^V`xEv4j$)Oj?dlzPWhV+gL2_iGV~U1T2JEW-g!A1yROT+a9^#&q&+_?Y zCOJnIjjOevdo>Mz@7so9nuJ#ia=DC zL=hyJ2sV7`bm>SW|Bj=PS`w^8<)&#zw;IYR-0j&i?O0D#qUFPtt{tNBrEQ03kje_i z9i{;)i3X@F^X{+;fXXuWj*{k@6DaABMv#-c4?BMVm1RC2Rsm31=HyY*XmbK39nuIM zS%P7QMB$Mov+Tf;WzC))r1IyVS-wy&&!yTl^SBM|_(dz&pnytriMZ0tLo~j0^AHVE zS;5c4G(aWM0F`By9##QRS!U`{;Oc>r4rv5WM!?uZqR0xiinT}8?Cp6D%b#;L@=_f> zYSVw>qjkf#{T3%vTbxXdb`e)ve2B)E79XMkGMPq_$uyEorjcaoGK&wZK$2NjL0f~x z2TD4m5g-$yx+Dr#u|$KbSY5Nl=f47GElE~c$?8p)kJYeSZr~K0jc8RsCOSr3Y4af( zU)p?#2Fa{o^I;kwlW2gCkoHKRJHlOcB z^M{TtYe}-ooT)c`KKag%*D5fkYh6GldPZF7^C22v`h186$*kb>VHzNlXn@QzpAV}5 z$Sm{utS>&12|gdv2$Bgt9}2jktf( z=tDHVH2M$?l3Bs%!!$r9(EyocMjuuIkXdH*(O~p}k`8GE`~(oyWl=z86{C-?+352e zng6MwpqBh9LAh=6v6ZcYg3TXo7eI-I5m%agh{l&DAEH4hE0}zk2B0J!9W9|OGx@Ma z0Ln6xj|P(ulypcV$^t6Eqg|vhvFM!;ASiG}3RgaNi6cX4yBCCu(JMe%=gV&2r&8MS3=i%#D{a5bptG@f@7el`>_Khs{M`@PbqLus?)~$oRS?@MH z?ZvmFX=04SVV0=X@Ju6KzD;ZKdm7Cazd5GyB}&t1^7!d4>S<27B>?~cPyzq|5dZ)HWN%|oa&K^R zFJo_VWiEJaY@L$JYQr!PMfd#*qrEN5?h9c%(3Y$kC^V&^J0VRHQR{)E*74t0b`mFn zthyZCJ7?|;s%AUYCcogkG`dD4%TWTF8zJ?kMjy+0`iK%Am=>%x8fw&oN3-Up;+ArM z+7 z?os(HpI?@9)D(Gfm*!G0sGvSpP=~zv$P`00tHmc zBFROQgC$!5;gjDbJOOKy940{l2$NnWIstlS<27B>?~cPyzq|5dZ)H00000 b00000fPq_++9o~$uag-kBL-n400000bed-4 diff --git a/summary_results.json b/summary_results.json index c08296f11..2ef8a0c80 100644 --- a/summary_results.json +++ b/summary_results.json @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1565, + "Performance_Result": 23.1555, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From fc985f6bc7c00b692d58fa28f4705513110270aa Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Mar 2025 08:09:49 +0000 Subject: [PATCH 298/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10050 -> 10048 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 7ddc25e8b..8d0b0092b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-12 08:06:49,168 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-12 08:06:49,168 submission_checker1.py:3270 INFO] --- -[2025-03-12 08:06:49,168 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-12 08:09:43,974 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-12 08:09:43,974 submission_checker1.py:3270 INFO] --- +[2025-03-12 08:09:43,974 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-12 08:06:49,168 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-12 08:09:43,974 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-12 08:06:49,168 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-12 08:09:43,975 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-12 08:06:49,168 submission_checker1.py:3286 INFO] --- -[2025-03-12 08:06:49,168 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-12 08:06:49,169 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-12 08:06:49,169 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-12 08:06:49,169 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-12 08:06:49,169 submission_checker1.py:3307 INFO] --- -[2025-03-12 08:06:49,169 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-12 08:06:50,221 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-12 08:09:43,975 submission_checker1.py:3286 INFO] --- +[2025-03-12 08:09:43,975 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-12 08:09:43,975 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-12 08:09:43,975 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-12 08:09:43,975 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-12 08:09:43,975 submission_checker1.py:3307 INFO] --- +[2025-03-12 08:09:43,975 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-12 08:09:45,116 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.254129 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3282 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0677 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.95927 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ced7b1ba5..cdf11a8dc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -925 +926 diff --git a/summary.csv b/summary.csv index 326cd3f6d..69164ff1f 100644 --- a/summary.csv +++ b/summary.csv @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.254129,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3282,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0677,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.95927,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cfba0f19c..a79054a40 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3282, - "Accuracy":"acc: 76.200", + "Result":10.0677, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index bf2703c5f3ce347ba4148c040663b3af6c0d5896..413dc0aa419d7b7cbd559a1c6ed0d6e7cbccda49 100644 GIT binary patch delta 2095 zcmV+~2+;S!PQXsE%>xRsvOIHe1ONbf5|e%i8I$k>Cx6fYW^B{TzaJZtPIG*k8SP7@ zeM!K4-?`z>9DDfnCC`i}!gxxHk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX z@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8uxLOzKp%L5^A*~;Q2BSBP?^2~B=+q3c@ zEzHKkfPejo4I0O3NRDY(<)jde5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASBa2D zdWsU#jMu+Lp32g?rjZ9PBeNqtq>)IJVIxVSh%*$m+pyw-=8w%IXKkRNI^LS=*v=dK zuN2eB>154kgv+lvw^qp_2vu9r zl3he&suh-1t9%udg~{T{ZF_CIwf_1qLR!S>%KTzRL&&Wykb3}$# z|9ueGtciciX+Fs9PT#lRt$JIc>*J#9>!JhG@JNSr;3|(mpy&_?6deMAqC+51bp0Ko zL+DU+JLs2vC^|SW)g5S}gPa?k1G2tFtb+31_5X*W`;46aKy(lNEz#Ld*Y^%T_|*Bh z>U>>wpg}m&Asw*LP7nxG9Rh)>Lm*If2n2kY&-{NKszc}?+lw9aOCPEZ4or0iUDzo> z&W+9i3&|~F6}CnG=c=w+R)wfqawDxwy!hV70_^@50e(h$e;~lQ|4x98RG`;~8!+qL zUOFdiMJ6Q6xDirWu!i|I+MGAcoWZ==i~oYF&3qB1YGn(u=Hx)6#meA6nj;EtFnM%s zWh{S;XH`ZF@mk6)J$WhRJ5srtM)VRlrx8ZUQp!^C+IUx|BUKiRa~kc}K~+Wiqayoz z;hs0x3%!FchI?R)44L@`DzmG-)W-!#SmR+CtjKl1R%yYFjKnfgyDir$)|_wD*MgSy zm2B3RR0x`H?h+D2gsD5ZyNxLon;TVY#PqiG)d_)NfV(idgKDK zCVDN;LfP}QWFc_Q6#vDa3S$^T3MCcl$5i=v&SLSCmP%5unNT-z$02I16iHc?G*ki{ zfsoNFeumOV6LU=4ux%S_aH7GxO+2E4%740*Ct=kpv92zMEX$ZIkk!M2hC*u4C?SSY zN)U{&sEp!KW-5GGtcmGLrxCJOeOFfl&UE~Bspy{ETI%F0$LT~pB7l& zhO&H&D2d~$zBP@F(sFkC?`d#HgDYjV!#xe|Yj9=DTO2ES+TohMhBbW+D}8N;YvRS4 zc(Ep4tce$E;>DVHi6&m6iI*txLbQTrL)*0{BTK_AvIvXfsgJQd-;}aZB!3wwO-xsM zZ47Pgd8M}Ju)?HgX^Qm2e&0~#8jg&HjPG~*cV?=VuA>~?TgiUES|Wgg7qsNCSYy*y zZx@a9{~z04ST!B7P5imxy@~LBmG@3Y+;U+bm+|nUvCf|2s{_Ma*uFh45jv59IOlh4&iW@qd+f6vpG`^iA6P)F6+b_IC^e$Ci5wn}};7+)|&qOzu8w z84~vU7;2djT3X#}qsD9YHkvAzTTF6@l&Sfz$!KzhF5p}y7|`tY9`)=%32^Vs)!3g! zjkcmbuomq^@!BU?_43?@(XJ0G_89FBt^M7v9$I!bJwd&|bOPn+`e8E45R*9m-jw8; zuIqYxqYN<}-gcNMpk8wYRBSa3mtPa;Tr%59Cd zAy|sB=UY(BXVIafxHe9)5KFhgmho@40bCIBfE5HLSYWsZrgkKvBU11YX@6Wh00hU- zf&v{p&a)h!y#bu^HyZ}!!l`7~TexXGiKBbxF6+v{8uRCPwRn9WR!ybuPsKk0 zHI?9$J7B@Mroxv+JY36sSi}>rC5Wee)Ni~U7diBSNMZnbh-SrUM4yLe3O|bCL1ZpYQZJ}002+|000pH0000000000 Z004l2U6a}-J^`|m87CtKWF!Cp007Z0{!{<} delta 2090 zcmV+_2-Ww%PQp&G%>xRBU5iL@1ONbf5|e!h8I$k>Cx5H~%wW^YzaJZtPIG*k8SP7@ zeM!K4-?`z>K6v=`B}fwbPvK7_RxTkegccl@LIH*!FW6$S=t`Z@Q z^b{pzDX)KxjLOowrjZ3NBeNqt&`2c8u#wOxA~SjIHmtZ{*<0|58)fZ*7qdtO9sz$96-l*hhb*(QRQ_hN2as0< zDdPprV^YdahAaf(V!^9rM!9MV_~NP8wk=T-Qnw$L>N2eB>153>375M$w^qp_2vu9r zl3he&sudPht9TWx43ou^+xFUaYrXp~LYBvLWqvUuA_AIn^I&Mj$mVwc89B**j>ypJ zzYpS?HF1AA&4%3W^nLr?s<$P&J}$bxE;=v`k90@}uJQ;3iVlH5(IF5hIs^hm*WV#J zgbqcwgMQhEqJsld-GL@L$hpxuAnRMiDk$$=|9>dD&&cTyMEB6&5}oaIeedvtt41fHfIfHrgE&dCxKIZd?s+BEBo09`U^OeDWG)ENPVDjkN z%2U^yHC)8vg{dWe67x1&Z*pgKLw46j%c~t?rZD z6vuxQ$+)1&2z>!_Wg;0^bXG-)_}ct($Wm5*hew61T4sq5P&N%`Ynr5VnWTwO7d>(T zSrfgMXQAwQTCxy0XNv#gPlYiIA%&6(^<%1hJZG`^NlPWD*G#CJxZ@DDW{RY&N*XEx zM+X@+7RfO028PA>!W7t`@8rPkru6?G+n%hNj@TytTzPLId|&3hlM%OE*vDl&{3v-Z*>5*_gGsEjCw;Xr9GK*T zXQw`UKC=)$%=UpeetzM-Mt6Vw$~y|#KCIYdv^%u+cfWdQ+1d02^#aogl&50r%P2!k;`n=0 zl54uI>+Ov)#B_XH&N>(4>&4~xS|YCU;2mC0ri=OTW>(KPuAHiTKk&MCH|YAlE7zV+ zM$cv<$1tBbp6?LPcDla$+I}8zY<0jnmpk@bW}+vda!KGQfwTS?Z3qOlLe+bdZy_uh z(jsalL24^bMXFS)qTVcf0ju$kY_}nAUz4;UEq_rD9?hD&id)K!gEwa^xFEqN5u`ok zwnp0!EXCOKEhy%*=+IGI8>d)^rQ2Z3_&3`CE(n=m1%U|`7~TU@I}*_mDfo!AKdv1B zf@5ewfes$$S&q-%08aUv4TEywR5J7yZdy;`=-zob8oRE`x^l3_{5f7NUf+jRQ>pi- z;y-|zN^r^@uwYzM;maZ(u4O(f;)&N1#8V&j8*j%&4t*e!7=Rw)o|Mn>`DHmrO+ku> zl$2>+EXkCT3B7;(+OwY`mjVTaU5iL@lRG6_0%i!4fe0Ftr6drO!X+93<&)hdA{&bE zvV&^`000CO000yK0000000000004l2Z<87(K>-YtTqZgJev_9bCIq!Y)q9h{CKdr- Ulh`Ic0ko4BCnE-CBme*a0F{RE-~a#s diff --git a/summary_results.json b/summary_results.json index 2ef8a0c80..90ba6e1dc 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.3282, + "Performance_Result": 10.0677, "Performance_Units": "Samples/s" }, { From 5fb8880cedadf92b93e964c0e71ee85958e9c251 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 12 Mar 2025 11:44:35 +0000 Subject: [PATCH 299/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10048 -> 10050 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8d0b0092b..3fe76b9f4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-12 08:09:43,974 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-12 08:09:43,974 submission_checker1.py:3270 INFO] --- -[2025-03-12 08:09:43,974 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-12 11:44:29,806 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-12 11:44:29,806 submission_checker1.py:3270 INFO] --- +[2025-03-12 11:44:29,806 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-12 08:09:43,974 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-12 11:44:29,806 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 -[2025-03-12 08:09:43,975 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-12 11:44:29,806 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 -[2025-03-12 08:09:43,975 submission_checker1.py:3286 INFO] --- -[2025-03-12 08:09:43,975 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-12 08:09:43,975 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-12 08:09:43,975 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-12 08:09:43,975 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-12 08:09:43,975 submission_checker1.py:3307 INFO] --- -[2025-03-12 08:09:43,975 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-12 08:09:45,116 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-12 11:44:29,806 submission_checker1.py:3286 INFO] --- +[2025-03-12 11:44:29,806 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 +[2025-03-12 11:44:29,806 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-12 11:44:29,806 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-12 11:44:29,806 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-12 11:44:29,806 submission_checker1.py:3307 INFO] --- +[2025-03-12 11:44:29,807 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK +[2025-03-12 11:44:30,841 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,14 +25,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71474 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420566 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418488 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.254129 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0677 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1692 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.95927 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cdf11a8dc..866faa8c2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -926 +927 diff --git a/summary.csv b/summary.csv index 69164ff1f..25179b293 100644 --- a/summary.csv +++ b/summary.csv @@ -3,14 +3,14 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71474,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420566,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418488,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.254129,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0677,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1692,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.95927,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a79054a40..b313cb840 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420566, + "Result":0.418488, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,7 +339,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0677, + "Result":10.1692, "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 413dc0aa419d7b7cbd559a1c6ed0d6e7cbccda49..fa09798f781730f1604feb7cd5655fa52ac851c4 100644 GIT binary patch delta 2550 zcmXxmdpHyN9|!P_VU*3~pj?jIHk!Fq^K%SChOy;P$Vg|MOfE^-L7g$9U&!6u6Pf#^ zl9jZ6WTG0n%%uo9ZW9uj%H;Cfah~&ip6~aM?;o${^L;;`=lOg)-MZYGUrNeBQq~Ve zfB=9iO3XO$LFTHYy#zLveH3(2f<95|*BGF9wc&OU&xCb&SgxPoQm|x6>wWw!nL6#O685Hbcir0v$sJxLALQ-Lc@Swqz z>7MUcRr2>I;MJo8KO}+lSKBcX)K9H5S{%Fhw~xaYN}E-Rvr#incnVitFcuW`!7NHz zDDc}-I~~`r=F#BYsuSu4!TC1r->I}@tx7YU-+4dliuI;+E8MN#sxmOGra2=|4W^u# z7NENfNv{#&clG37Y&-~jQnIHD`)2o9zJ@zX&0{MP< zvC{~vW3spKFeOL5979&OT8tq17RYfw6la@7=o(`H=5VT1ykz7dg3x91^=aQQmHHTfbcndKS;d3P7I9Xy15<#ecFz7OyO>4d!>Z{{jp*>IuR$=anleKFUt6j6_V z#nh1f9}Z=K;FS`>%!i-}B2yO{EJxoB6dR*h%Dfq zn!E|9@s~!_13Le%lKxMb*-qpb6&432*NKw{4#xAz*W2H&oNvioM;;CiKf17(A%7zb zwhv_>X9IH6$-!mJ6JQ=mcqMrq7jyzFL;{{e?ahW|;0^3ZQp;a^3}6(O zsd~Q_5@!jBn^%fU>AadQk(?_q;EFA&zd(epPrOFo0m7DE+!=_fW{XabuI4SP=#t32 z9^H}p{+75I7e5GJCs!rik|>E+T1qRx7hI0=(N72{wsL0J$cX!-aLp6M;=r!!@Nmsj zCu<4LdZHN3jGQ;?aJC(U zTv&#hKY9L*@xquhknuEHzW|L>1_i72Xx1Mk%D|Ki7R-D zwnOj%#gzYOdjqpcfkHl6$jF5RgCNGb4FuMuS5^y~xCObWQ?sT};l4$@PE8`F~059w`u#exvCr)38;FF!2eKo9{M4i65V&v ziSN3zZ|N=fLsN`K*|+I2KKk&pDKo-QNNkOw>154=4V#R>v`Nju4W;T}^yMw8+BfO5 zrS#sN5Av0Uaa-9?-sbH989ICHH}-okt{ zNNqvil}7e$eic;52mJi<#^dXLy%|Xs=TU7sTm!`_o4Hn}F$Gl5h|t(j!2bBgKNr1EP07S_4XU;yCcp}s4l;7o)uI4Uc{{Ll%|gRz zuB$%a0BZbxSvRj;700zpbU}hj&DatR%6YHm0x>+=J|i+S&_MIHC=^{)~BD- ze4Z)ag{Rc&E?RuZOw-)7gB=FgblV&%pA!-u!|aadsC^GUQ&SZrTfshPEF6xX_1ZmH zm+7!`+!_dZbl28iD8p{iyDA~I_YGrO!R@8?<};H4SCI$=GZ|oYW;INK*#%bz&NCH_(&peI50O`y^1VUpkZXf{QE&%{2ibwpvD}g~~jv%nW`%EawaE2JzO>Q6bmC06>%o zB^Gj$vJS#gr;j!1_=iJpRMMb=O0D6&{mgdtE)87 z#JXux=z?MK7i0Qm+~yu_P~_t{mo&JMRLoP*(yES~PX>Erse z{~IvYB#IS{f(ip#JonNE^WUw!SacRIjW4l&pwZ0l5)xT`%^q%%~+csiSUkWf8L z6fX1k<)gn_Mn8DQTrezv^9HF$h6eg7r(Lx-Ikw@51u%d6(NNI}q=hzR!2O8wUMbOrH3343z0cj<;T z-W&uL0{{RJ&Nc*OspW%uqKdu=`*Tbd2kZ#RvAea6Ogv{**{l%GBM z?YI_dXKmCycPh)B8?2VV53X+=I~vL`NTus0qy{re&`T<;h_@r>WuYZ^35Y}s_&G!Y z34LrRsWCCDU-7Oy)qc#~!?g#3giR&qwOXI5GN&y&ES+S@o7=^7vJP@F%M*5a6S|Nd zr*OmPxQLL)&{%DWSRG2Ek0X#dXx}GQIoZ^{q5Q-WIUitX*V4MCOE#N~XnH^n7zgEG zL3Sei_!MZK)HKu#=@4f}wHtHT)Fmx~&Gc%7t!))b+Yr3vI7RZ8WfLwn;lXsA2HVz! ze;edEmF1l?8~ox-EP}T`4P%y<0a7Z1vZxBbw zjmIeh4pWl;9rJ6lU56wKU$J^iYYv3Jb7IQBx+M>LBu1E}U4W?WW zj2AUU*evae;3j3rSdgg*zf4L28i)8x1A&8&AH_RhqGL9W;+)1WcN{~IH8>siPas8i z$+&WNIkpGA@_J{2Eg=7m_3?UN1Iwxs(tQ$OK#yL%<@zDO;CF45Z?v)L@w{NMC$CI@ z7hpwnFcri zJI<_u)H2R6((G(l7vi^dV)XWCqLHi!GE+lZuo_UI9I`1-XgJ47NdRXmGwO_J>-}V> z@P&T*l~DoP-7llc9u>JHwryZRTB4h-bIuuWzGqNty|#;hvvP+4yj~Jew=^|N0e`v@&{5BC=9Ix{HxqH;cO@zDJ zk;bsEshxfM#RFJRMO4@LY(E_A^Pf@Cxz0Oc(x50K3WL`+>4=33Je5kWT%^70J10!g>k$Lh*vV76mO4E`2wkw6B-)Or=ebh#Tlkdo6p+PZ` z-S>z1i3w7ps*LD)cvgZrEmzmMbuEC2Vp0L1U7625LOFE<{gOycq8u{kBLw0CEi4j9I}%qG4PY)`V1<)J@{AWJ4& z+`_;Dgcr%oK*_Uw<)o(|Xs0WiEYHouim^^;WDC*nM2$~A#DT8$l6bF21NIJEB|Vq^ z2-$4LH)EM;{q>dPj`O}C`Q7uvLb=OC38~vlq#Isar2#F}tkvwY+rtu zZsO8FKIm=>tJ7J1omk$n6Rv5Qc0+eaq!D_L&^y>nP>HoJoqem*r`d@mKrN*VU*GP) zoM0Ez|GwEHZzgTHY($S$nq_#@vqh2wNDBS_3^^bGfad}LggFz(C&(Kg9CSTI%{wUA zS2ZFq;4kvj2yNsp9Jewdg_6ky|qqXNaGNE8b^mMM9@~dD;p$fEaiZ^GIB#s1) zI9|sjSj(D5GXv9`PC4Fr!gj$AU1i-bZPB)RB4pmopl$>;0bPYQ>LrUifJK`+5I$Abl0hG+o%2Z0rd^Ce{ymE`kqnF zBQX-c1~)GNFlzw-oZyK4&x(uprQ{=xfy1O8^ngT4zMP!I{}=MV VcW#t9IRhY_a`H5s#|LqE{||x_rm+A3 diff --git a/summary_results.json b/summary_results.json index 90ba6e1dc..dfbe7623e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0677, + "Performance_Result": 10.1692, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420566, + "Performance_Result": 0.418488, "Performance_Units": "Samples/s" }, { From a614f4e5481a14108e23144594ff173696b1ac60 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 14 Mar 2025 17:22:55 +0000 Subject: [PATCH 300/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10050 -> 10048 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 3fe76b9f4..a83360119 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-12 11:44:29,806 submission_checker1.py:3263 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-12 11:44:29,806 submission_checker1.py:3270 INFO] --- -[2025-03-12 11:44:29,806 submission_checker1.py:3271 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-14 17:22:50,532 submission_checker1.py:3253 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-14 17:22:50,532 submission_checker1.py:3260 INFO] --- +[2025-03-14 17:22:50,532 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-12 11:44:29,806 submission_checker1.py:3276 INFO] Open Results=22, Open Power Results=0 +[2025-03-14 17:22:50,532 submission_checker1.py:3266 INFO] Open Results=22, Open Power Results=0 -[2025-03-12 11:44:29,806 submission_checker1.py:3281 INFO] Network Results=0, Network Power Results=0 +[2025-03-14 17:22:50,532 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-12 11:44:29,806 submission_checker1.py:3286 INFO] --- -[2025-03-12 11:44:29,806 submission_checker1.py:3288 INFO] Systems=16, Power Systems=0 -[2025-03-12 11:44:29,806 submission_checker1.py:3292 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-12 11:44:29,806 submission_checker1.py:3297 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-12 11:44:29,806 submission_checker1.py:3302 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-12 11:44:29,806 submission_checker1.py:3307 INFO] --- -[2025-03-12 11:44:29,807 submission_checker1.py:3312 INFO] SUMMARY: submission looks OK -[2025-03-12 11:44:30,841 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-14 17:22:50,532 submission_checker1.py:3276 INFO] --- +[2025-03-14 17:22:50,532 submission_checker1.py:3278 INFO] Systems=16, Power Systems=0 +[2025-03-14 17:22:50,532 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-14 17:22:50,532 submission_checker1.py:3287 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-14 17:22:50,532 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-14 17:22:50,532 submission_checker1.py:3297 INFO] --- +[2025-03-14 17:22:50,532 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-14 17:22:51,584 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,8 +31,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.254129 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1692 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.264518 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1006 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.95927 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 866faa8c2..616005867 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -927 +928 diff --git a/summary.csv b/summary.csv index 25179b293..53c99f00b 100644 --- a/summary.csv +++ b/summary.csv @@ -9,8 +9,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.254129,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1692,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.264518,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1006,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.95927,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b313cb840..ff7a8d6ec 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.254129, + "Result":0.264518, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1692, - "Accuracy":"acc: 75.000", + "Result":10.1006, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index fa09798f781730f1604feb7cd5655fa52ac851c4..9d80737f9f5edb1857d3153d916dbc00efe918f9 100644 GIT binary patch delta 2743 zcmZ9Oc`y|I8^?Fuwp^=_V_j>B#iq?oD~p!Rkz2^MBC!(X$~wxKYuAyZ!%bF}MJW`b z5JD15`f3R|*SQkEYUca<&HSF3=kv#Bo_WoD-p^;=^X@QzZBFTD1*vdbkpkHOfN(G= zit7Y*jkSrD)stpPdcxVu-Y*J-V5nhStQ>@iG)1Rg25H~$E+;)rwH=-M9fPona|824 zV~_xt)<;gnvKj)z4)DgXFK>0pBXF1mB+JWc{!yG20Ctt@{&ot7^-^QhSyD);eG=vp zqDn*kT2m&RQmC-WuNi{|pLwX7iRO{PAfur}0AeL~7JB%kArUF_0DwE*FeXfRl#wMT z@u?3bbF$ax2#T6DPuD#X* ziHk&qSg2royu||*i6r{;btK`US!{OYByxG)&QQyIC}1$h_icDFSXuIE@G^g@s=p38 z&c+V0g5a^lyg!Y?<=sj+!(x-ud@Bn%R*1qRY=JA;NW4iMjY`yo*AtE2j|QgsW{-rd zM%oo@OH>aru8pZ^*KZ!FME^`slF&teGdsdJB$a&}tBYl|0jJ7W!3i7d6i{@pntVFZ zxIj>0A;Wm>9r&mY6qF9n!*Qm;{kE5+vdR@JGUgWaas>`k^e zq32SAVo> zw2_de{0CZCrEy!IIgmc4h$#1SLBYa}ofrNf_x>Wk{ulY^FLHxPR@Kks)|VXSBK|(X zcW-zfe>7-(c-ZdNW>b>>MSbtH@?ah4CLy>w8jDPQK0HtO{b1xCKM&EdeZCWkf(-%k zaK@5$zzj$4%^|nV7$UDP+dkW$-;VM=UXuQFe11bb@35-qan8P?L9;)jF1sc+wqMd; z7?l(+V4okE*qsZLEH+S3x*vI;<1rK>5oTH3ht($wMAyHkm$cJa!VErnI8}i zJ!%33vX@^IQ#S#p{XWU&Y=hV(vw&eT*T|Ch6OBvwhcRW}4Tl)5EBqofUDG2lKSfUJ+(*ZarI`0ul$`zk&ky@w z_A4vZZ9nIP4PTwwPK9ro_~uL%XYUJts)=iLTvgXy?s`_@>v6BsJ*RSX4J+%~cEN8w zLbPA#l~(z>oau-UV|PJiiKj#JFzZ*Vn6cH~uCyd^>rxNz0gI&>E88&u0KoOT&jP4h z;A9|shiqEAAL-691YtK{23rRc5G3I>D~vT2wqi2*JxC=e7eN3Iz{0M z3Gn$GMT>(PH;NHO;^v;-!_A0dLYK$+h@ANyvgqenDp!ybjGQz*oAmWUAgIOENbrJn zzLm%Bb_Z8y`6)B?;?gRCYx_ncc26s3mb{y>B=P(;<)1*UXIT>cK6a%aJVf{|$q>T~ zzDe(~%=Mzx2zB%1WWn`D=4^Bsd7n2O}Ie6I%}#7hZ=P!YN0o>IALSJ0Blgb%0Gz8X$^` z)sju}s^;C>VCC`;x+$gg#P0%qxjntw4()GYI;Dwm&ZFl{U#MMFw7BqIBc`Gv#AZUh zZpeSCe63?-|D0c_;{>t=TGsU#{kYOD{$W&Jl+n&>#@_w0MZV3S``_ zP0@WTLZE;%Tv4=-kC&6gDcVm$!f)I<)gk|`vm7sY?}H%9+;U9AaFFlT>oT@;)F1AW zMYMv|k{{!dYWV|KuGo|MtXd~NySj;dx!5eu6{ge;SVfHk0Xiz9Hnp`*TW zAw1Xiwx6h2+3eEhs!yPOoD#0F^EMB}hvX^@C$z+R8`2u(Wz9;I)zXt^s?zIpr?cv< zpeGbAVC}>GkNPi%l{I~@?08d$G}?22g|7_aF)^0y-g)^;=d`Wam~Glhjn}Ko%BN86 zN1TH0*Vdh=e3oLocbM+4O3qLF3_F-y`-0wW>y40?4QQsM_l53SFY%YBB+|ezYYOZo)hj|1e$(KtQ-&kuw(%M1b&-{7v9q{ z2p@Pu)e|3tQwjCI_JG)L-ihG9v#M)EkC@RcizY*}0b6J}XfkRN9zk<@bGDwx_X|Yc zcf@Wn>07w>s|PO4&P%qW{n(A+c~BLsMc**S62LB}r|Nxx>Q# zHztzwxP!i1aCO1ONbf5|e!h8ME*MwgUv?9lAi1sRzmh6*YA+OS5FSLQn{6ZuEGG&*1Wq84M2+W5 zBvFG%5)B|pG=L;(0ws}j%8(?DNGCv91?_+J#U~_XNr&|^kVwj?E{P&fmrUeKCY^ej zoPWut!ubkH0c3hVSMB_`_4$&Z$6^uJf15^_83{Y9--C>OY zm1XW7CCxP_P|{(&1S-L#L!xl)%4|BYcCFc`^Y=sfLft!`)uw&NYP3AtwTq4{{Q@e{ z{^3do57GG2!b3DjWd#!t(*Tu3<4J!i%X~bn0-&ROSNg{aU0t4i&n5f0hQc6fU$=}kriweYmchg+w&ZjKj&=Zr8<1nrp13p>xOUp zEl#GkIGGylBCfRf5RES_K12g#GL0mYX(X9UBgxcd79UoDB(to7wg!t2lypcVKqf?W zNffMNi3V4(x@L>de+A51lB}|l)tfFKt6{g?z$rKz(W-zARbWilx`0gdjJVS0Lo~kh`4A0~S;6PSG(aZN0GVYzA65a7S?2RuUwk4H zd_JTRBoll-Bns!uDn6f${{Mr)e5pnsy>0ZdTb^ST?EdIh0hwqTaixFJhiH6h^dTA~ zvx3owX@E?k0W!;sKCA*Dv&`tD!RP}e9nuK+2_UM=qJYdQMju_X(dRib|5HOjE%{Y~ za@*u%D_aExn?KqvfD#QOt~B`&jW10;M1xRPF!?YIKuJ70@}NYQnS59y0A-oUM}x@+ zN;;$wWdW7o@}Xg{l4XA)SjpCG^LY--beI#VerPwgmfR{WU^hKJ*>iK?YxzG&nj3F! znuPIfcodGuiP4X;;aWznOP8;+x=_2<%Qrtg$+PyN>0p%qSJmEl`JSqDG`%&FKbLR7 za`QJ}S$Sps;l+F$8tJ!LxNn9Jv+NtnlQoUO!*KcTD`OUqqIiFlep|MlzkF*x?j1~` z!Rtk*=F`!i^YHbn{wsI)Ro{K{i=p2b`$m@fqcqEI(MtXc>(;^Etalrp_Tt;oG%?2E zFiX^Gc%~6A-=?+rJ&k6I-yGBU5~XQ0dHi%2_6A{`f6D&e4WlT1{E!dH`mJOKe*sWS z0|XQR000O8006V_5{v-_$!jDiYLkH!Fn{1%iZGP|lA2jpO(7+ZIh*F4JA)le%isN-9eCtEK2g#Y3J_t-0(Yy3z|nX zWjUO79)Wkh4gTnJE|6TB^V80Q5XFgQ@$dmM%1u^4E-f)DGb-fYa%J(N1RC)N0DqCC z7O`#D%4m{zP{^uWoOXyWS*kqww}Rmo!FJFY^m#Jqi@_C(AnjXX(6?$}3(iBxX_>Gl zewQbLH^CY3DiuvMO=(7n;n_Y#PqiG)d_) zNfV(idgKDKCVDN;LfP}QWFc_Q6#vDa3S$^T3MCcl$5i=v&SLSCmP%5unNT-z$02I1 z6iHc?G*ki{fsoNFeumOV6LU=4ux%S_aH7GxO+2E4%DR;&Vbv~?TgiUES|Wgg7qsNCSYy*yZx@a9{~z04ST!B7P5imxy@~LBmG@3Y+;U+bm+|nU zvCf|2t1tt@fk{4icIva|GllSBwh!d-^M&^s-SL%o6vpG`^iA6P)F6+b z_IC^e$Ci5wn~0M(7$Sd^srj$TXmW-w;9Mpc(Cqde_3S_iaPQ33*q=p>wxT|;7VSjw z+9z1`^4y2ft`95r80`+N{oSt~T6Q))LA}6q0_ExYGRhE>IR4(00&;-nbq@+E2k>o54^734Z6PX%C$4+lhL!8$T7?(j^{hX zvz@N5zP7gmj@vfioXZ{iEi=(mpmIszD2adaR=p5QOi23ZuI%%T7xXwgYX+sewXWT5=Q8CK0uMkkmSU`zlW2B#={&tC{_Fc2PCo zsV?~i=cUm#B3X_S(A)^AH#Pcb7wIEPd|+Cz&Sl3XC#3yx z-3SmILk9|U@R(!-pS=N`@;4hs<-)0C7%behp2W$$^KvrweV_H^XpQ*^{#w4ikE^Cq z?@z@6HI?9$J7B@Mroxv+JY2_oSjH2tC5WdX>NnnwiyZnuBryOz#62mWIp|cq+^VNRjRsR{Njzj)UJOK{5uRlTa2eYywgaQQ)i1aCOlRG6r1r;@Q zF-wzyB^3f@2$O*b8k40Y5R<|s8Ug2%-6bL$$!jDiY6JiP1QY-O6aWAK0000000000 yfPrz78YV#j4U=3ZIst!^mnJ3zB568ylffny0brBZCO!eQlNTo=24*Ax0002d5e+B+ diff --git a/summary_results.json b/summary_results.json index dfbe7623e..3673973a7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1692, + "Performance_Result": 10.1006, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.254129, + "Performance_Result": 0.264518, "Performance_Units": "Samples/s" }, { From 823d892565c28b3fcb55a7774c5f721bb0ff44a4 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 13:23:04 +0000 Subject: [PATCH 301/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10048 -> 10048 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index a83360119..96a342c49 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-14 17:22:50,532 submission_checker1.py:3253 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-14 17:22:50,532 submission_checker1.py:3260 INFO] --- -[2025-03-14 17:22:50,532 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 13:22:59,682 submission_checker1.py:3253 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-15 13:22:59,683 submission_checker1.py:3260 INFO] --- +[2025-03-15 13:22:59,683 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-14 17:22:50,532 submission_checker1.py:3266 INFO] Open Results=22, Open Power Results=0 +[2025-03-15 13:22:59,683 submission_checker1.py:3266 INFO] Open Results=22, Open Power Results=0 -[2025-03-14 17:22:50,532 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 13:22:59,683 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-14 17:22:50,532 submission_checker1.py:3276 INFO] --- -[2025-03-14 17:22:50,532 submission_checker1.py:3278 INFO] Systems=16, Power Systems=0 -[2025-03-14 17:22:50,532 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-14 17:22:50,532 submission_checker1.py:3287 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-14 17:22:50,532 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-14 17:22:50,532 submission_checker1.py:3297 INFO] --- -[2025-03-14 17:22:50,532 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-14 17:22:51,584 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 13:22:59,683 submission_checker1.py:3276 INFO] --- +[2025-03-15 13:22:59,683 submission_checker1.py:3278 INFO] Systems=16, Power Systems=0 +[2025-03-15 13:22:59,683 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 13:22:59,683 submission_checker1.py:3287 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-15 13:22:59,683 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 13:22:59,683 submission_checker1.py:3297 INFO] --- +[2025-03-15 13:22:59,683 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 13:23:00,806 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,9 +30,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.24648 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.264518 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1006 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.157 | acc: 74.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.95927 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 616005867..63881d8a1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -928 +929 diff --git a/summary.csv b/summary.csv index 53c99f00b..ae9762811 100644 --- a/summary.csv +++ b/summary.csv @@ -8,9 +8,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.24648,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.264518,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1006,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.157,acc: 74.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.95927,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ff7a8d6ec..e38a01662 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.24648, + "Result":6.85051, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1006, - "Accuracy":"acc: 74.400", + "Result":10.157, + "Accuracy":"acc: 74.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 9d80737f9f5edb1857d3153d916dbc00efe918f9..f605931ee6e6601faa61711388553934b1c25bfe 100644 GIT binary patch delta 4659 zcmYkAcQhN&+s6}BjoN$fQKfM$CL+O=zs z+OzhnzxVv!^WJmrz5hJtbMN2$k=fy0i7O(=w;UDtq`CV|~AFk^8DXyl4h$_f~Mz~PU_*dB+Z-}c}HOj*GW)82S@Mp2v zmuh}A7ZL;Gx?{)Gs(*)18rG3k{<-S%gM3_-KXqxzjIMd5*F1kR5@bgIIale!Ro+Pq z*8!ZTl$u({|EATi+vA1DryEZ6MM?Bz3x@e z9qz50f1yxPs*N(TM@dfYlLqHK-|6t8F^^5+W0+NQ2&hdvtpSj`Dv7^hYNY-pCee_T zWGd`Zzhuynxe#}1{t_6ed56;kH28 zepqubW$}1QDC1)V#E}&DFwTrK-~}d^&`=@KQ~p>X|6OC*Gu69>TEp83pXQoM6yHV+ z`I$jE;}sF37xwa*Ldyf{p2=1{`%?3jC}-~X7r5D4_t6AuX7}0nuOA4V+nPDuX`-hB zS6&l};Brgzo2bn*4Y|8^`7B3@``#~@LM~Y2Nxy+PWJPG!G#Bi5htgxKJ`NX2ttoM4 zQVso8STFq6Y-=aKe)tuc`zT-W@u2hG*&f#1xbl|qmY9i|3m4aLvU}qMp)J94!Hdbz zy6)5JT$Ap^eTC@!51P%+d>Q_Go|>DkZ>ZpMk{8J=D^Be3 zn&(sIt8X=?-jJ*M9gv@!bLcdEyH~L=k7hK0r$Xz%8 zBBF7Z6dWjX+8Jc?k-YjFLO8Q5Z8T$_gARw?Vytf%y#qRUYwz8{MSj9n>-lD>|Ab0# zi|H+)aTjM0ENWk})&u!qhgv}WTqqLT}#S( z1;?Ya{ICAAel&h!yJlZ2Wm!w319?F%Zy&esj%HwG!TGwuPc~ryOjGhboKIbN5={4}x%i`8z zBT6JmPzvA0BC4OtdIA6NK=9_bCS5sH&Ug>Eb*k|ul{Bx@?4alt_4by3Z@F3UbYnNS zOODpZ7$aD2;DUSZuN~L0w%y|Z4CF0tRRBFt`T|4$_-=z9ksv)U3p+Gb%wcsfhLNDOBL76Oyg5NDXM^U4V3k<^BEl{ooZoej~pNne{$J@RX zrHG!vw=>9%lfwWS{7ItpXb7E9YlDm`$qiB4K;F}!>)PES5p}A{)nHGA>$44YvO1ts zCDS#HkZeYn$%3IcY>T&({}NrQ_NY5rR<=hUl=9%25!#*$w&*JVoXoj4NwiK?ggX%<+9lEi`$`rJ0xA z0bMF7cX}lqjm)I*We)Uy5y3#(sN9PO;0kxfBX zUzpAkztybW*YGhjy=X*!!BKo%fVamPFfu_+3`#{bMh4=4EJ&bzCftLp-aWhmrWr4qIzIPDjeI=B6%2Vz zS;+{ivMZYe&aVd2>-dPR!9e=2Z0M<8{)4)BaZQR6;fw$_%tk*gt&v2AsWqkott5t; zwNS=zXkRtmkIpN`^z|r{s%t96k51gSy~vx;NLBS zKH2kIORpLAES|$DD!v{3HW*Py7Id|Ulu5@P>yLqx;6?N{% zn0gf4iE>s#^zgAmsv@MK5CEuc$N)Wo-;>&vBvIUFMYM874T?E5$ca%i%7V}dW{}Au zOa0cg1+U`slVJ4X8cRh8EjgEN?|{uCw~T-?5wU^2IoKHWz~8~ThhS|7@&y$25Fn*p ziP(ra=((wYnnO~s7Sy+wExE>2G7JVK zz#YfK1bSkW6}6Z8pQ=_P^Pc9o)*_a@T_<**m)pGJGQz{#QyjVRyMpS{c%sl0; zt($VVU|Mbziy6}N2KLAGV`3Lq(HrEC17y&S2GTrq*}K&yVhOguj! z%QC0Eh@HAn7eg*LGgeu>3wxuSk%)ahcERe1-YCRN9XwDP?d z8K?P!>-^7ULp6N*hM6Xjtt`Df3Uy_I1Zm%*8PvhaqN3_kyC^BEKV0^Br}h*To@;IW z*V=y2tMC*7rO~9{#rICGwf*3K+kZOGknww;FOK)$sxR-9x_jbwaK&?L8Gi3{)jdC9 zdWX}sYb(KHgO}lvKn3`eHcm251@g*16?6)sN`!*d9Fqp3i~;t{Y$0IQklX?d6gyBA z2&pw=Of>6U&A%gis-(4yauF>NY%|dF$U?d~ytnCEPN;jXhK{6rl8KD&OcUn8tBbWUl#ufj;?wM^E&wnG9^gUmD%5^J5h>qnCM3vsn&2< zKs^eXE3>%YZZzKCsjZnFkweW)fyo!t1L@Alaao_lbljNFc&M~ZI7Hy;|WJ)S#y=gm5T~>+`*){68Kw`*%8$eAyacZuMK&e zB8Ab?SAw-kSoB-B-s5>5FRZ_Ph&S9{TkQ|}dun`~(sWh&r}d=c=VQ;!rRDa#VnmC z#T(mJsxgCIG$)B=v#@{zmg+_Nbl?rzV3{2RE{qX``fU*dN+H7Hg;^-DP&s(W3SAFj$T4wGK^}V9)Ks)U20HwPhnW){Bkh&uywm9qKu!|>Q z-@OSyB^5G%bRX+>#y;2kUTT>g5G`p|s&rBxKF9ne;3wNbkn}3%3loNS)57*=y^9TH z-^1^Sj7s6qG?8$*FKjzvOpm~i5Sr+@akv0kW~cagG`1+RZ8!CiMI%0@I!j&`$CYaL?>%3$nbL|VKBFFN}5PMvMz5gc{yc2v6Hz(KcF*Fduga5G6^N3N(|sR`_H+mBX!uuag# z*^rF3q}Nu8eKULB?!h+So>Dz#Dn|-sFT@)Hbbt7&PQ>T)gkgDgRue`_Mg6XcGe{omewLwKH!0Po%%|~xM(Sq#S+bPpo7yZDi?(qy zlKt&+Tr|h#%63udvJdX$oT%Eb)@z78o2-~~YrK8*Gxd@70{LEnmxWZyXJ5T4R1rzi zVogV7gh37C&r;f}n$LT($!m5HDCd*Ia)euQYW4f89Mb3D2CRwV)X?u%(kFGz9Utk7 zxhmFNO^3!R2gd>$G$lt@JJ?%H^I2REY~!Bd2jJirXgmv^3CuB3q0DB+?vbQn%J?`W zcIl)jIgX#8hOqrK>}u!i+JC1Or5wd1tPIB$@En~*ijR5OgYA)RlWj5v{%Z=|11%{V z$M1!RL<7~~)R=p+A%_72$(L7llQ`ocy2wI@=WX z1fl(p3O+DL6626N8^cchPRw7U##Ly$`X!skR^zRMw4Nz>=*$`gzZKgOHXv&Ba_b3=gkn}s?o%6u zusy#JHiu{^C(0A%^JvW9rlmzcGCkh2$85zG+0wvi4R1H*ZvNIo@RQ=1`LTY|nPQl- zee>zq7OB!s@urBI`0k*FM+$b+i7&dU6uj<<8;Nw-kSonz0Hkv80!W{eIg# z;(F9#mW}y>s-|RnMs}k@0URNS^XB70xqJiBMr?;r~4gSGqb7y3>E1H$Z!9Rs8Oxp0EXMGs)bQ=ub9P4&H!{z#v}C`eZ<&EDXT}T zUOqnSGRJOOQGUb$0R)$O5Y7f8G5Td5|Uq{HU(9@+wrHP8I$4}-EHR}pZAVF z{V|D1vg3&sB+cz=a9YYozVLV=P;y)Qqc${S86zu+&^LI{N-f}dDyinrzUMj5=Jg}w zj674dx{5!7i2VN?9Gt3<@{RxP@56*FfGfCZA%2ekeBjsq&OZy_p9}Ec@iNRC$1bc6 myoIwARse?L(u9RM{*!zl0H8+%0MPw6@Opn4W{vwL?EF9EaJqj0 delta 4687 zcmZ9QcQ71W+s2oOvQbx*5N)j`EH<(sIRI1{^3gYz6jr*zPXYwbad;ZU1=_&+5@A z(U!`@q@3=tZ*AZGY1!xEhIC(W?58j5Z4AWpyB^Ym5i?r{L8r5&9|Rf|#2Q=0WZ-mh z-WpVI_oyje(}yaJ@G%O>P>`#R&+*B;q*whA9y}RI2*08~8;XI-d}fB`maU zd4UmX9R9S%fIR*(2g=&?4%*%cuNdkzT=`Xcg{~ukel6scPnc59IwQ3K7kOx@3a|Au zh5Z-B^ts{A;yz!*gNkMD!M|~NKPjS+mXe`@K|VwoPOzS2mo++?vJf2m7#z^V?qda3 z>*F@+ojaqaSI96!=P$^;H|m~F>jr;sa$C!+WJ!yMEvRXr+c}n}?Y(}$y(ssW&Cl2@ zfcYljQC7rdM2C>6 zrm3(M=k~gHet(#UPTsjpaj6Ezz!oDuvrie2`(mftoxvuC zQ-b|fX!jTwKS)}phQ_Vm8{ubHh@m-AJlW9(6IP-=9{DO^Ww#Cy(w zZUE8YpvVzj_~CT<+tMl590K=VitC_f1TiDk_6mFQtG(^0Rc)82&yu}R%?Q(un3UQc z^{$(ChMqUMk7{v@z%uQOBlP6f3MPAh*QI?XR!W&rJm`7gV(RKO%K2arP1od+Btw{Y z|H?3@dVT`QEF91zy3c6UYze-|GqvH~pi`TymE+;3k33z@osuV6PTTEUyNw(j3Z11p zpSdoU1Y!w_dknqdL1X;|4=`U^AA2vzNm9;NO7Rr2jf}_5uO5jFMhy0;kVfvAo*|%l z&ahesZGVPSy8%RNEajS|Rh5Sn2W>}S&VxL~hokm;r+YXv!%AAiEny>5M=mZzqD#Y6 zUt65pf;-EYMP1|Se3R9b?8KYAcWTY{d})V!Zfcv3S#(;8;^&EME0`dsw?HxOYtgW{ zz=JIYvw*?VW?~YthPEN`s&VyC|Gb>E!)LR%24xF!8NU1Qi0M9&C1fCpu3BQ)uIXe8 zCTyE%0~BX1I;r`gd(1`os%HKh;K=asu&`LRp@n=|y_mi;+HqgoC2Ab3Z$JSad&ZJ7 zEc{lNukCBKOr6AYdvswWeQ*(xJ6H=s3{V7vZd-3hApxhH2@}(}VjROX@H*squeXd*RH{apErij^L!9i9>TB0fJbmk{V zJ(l&VS-mCu@iHFoT2!%ujUF#SIk6g$=SXk~Kgc68ct^9Cf1Z450kGW_TR0M$TbkAm_gB)gHg?gb--cW$7dOl~y3d<30%qMg zzw2P$R!63<=#}#)0fnsmIECuY)PB01wakJ?Rv;t|LVf5DLKps;SRw$61fw7z6gWWx zbPfWc>LFE-=9qaU!sGkkh%4AuNJ3ZCjZ<0yOg~QR)9$w)eS;Ka zUq#Oyf?jelDKPIsJp5Bp3B1UIF`{N8I2ymRY<2g{S@O&k>eN?Bo|sY=EB~m(Wn9eH z#2~QdTz7uCaX~{^Z($OrS&d>a0MX%-=om4Nl+F7VUNQ@LYdGKjUhZI zO*0ZE1?|h!a=H7JA;eTY*3fRiu=lHDZ^L|mYkN%|kxC~LIP8AYI1I}%oM26$Y=>|F z@b&=hue_;^wA$r>c38<}^}i_!VV7)x_Kj}>dX-63kQsaRpN{yqGe$}6hHSn+mb%nAyw7j6xr z3qc_OQ0cE!@J$Ig>M90chq}6X*ewW0qaqRHO=XPV(0TxBJcWydE9 zaD5+80{<=YwVb6yf_R$uI@epwA zy3{QU!rUJ5FiVUJyEx~8c%&7UR6K*&Ts2aW)){dfPIl<^CxAqG-g<8`#)!MfBEt=g z;9GDEoi6J09VK+yi^#j=2FVRC5)tE>N~p*~P%#Cb9@|kXQnU7Ps@5)hXpUj&+Vkx| zql_c2vXMRKaWR?dgKNde(?}66Ipl)YHHH!1Bo=Ks?JEYL7{L8^OSolhWISqLx=mWHN^g*hE<@Fpoc^2AGbDOz@cT-T_i%$$jVcYx#O)hyQj zB1!)ufBqMl{V#I=q+DDffkuI6lp1$=f?j#qv1AXcU0k$Vw;JTd{g5P0hz?f)eK6i- zA=-$T`q5PtL@R>CNQX0RU+qLFK}P^-Xf+-qvUpRwgAwb4P#nDjhT@FkucwvlSn}fD zPOk3XOuHz_%}G8JR&H?Ft4gW}3mfEdVN=3JFd3%@M)jmXc?ilvBB_C?lsOP?Ev3jqc)cNM*9%9t3#(j>K$8J(8QP7DaJaT?-*0=}8t-338 zUgs>2Ir$zqBb$m0AAh4WSd^!@GXKl?+<0rNto3x6O{MMO?V}jjp}IrzEFtNP?R!Od zi|Mwc+-CQOJO`WT&L_#m-*&Y5ty&*A?FMiRvbISV?h0s*+3)>c6WgHcxOb6w-omM- zNBYgjgF7_uqyIx*vjR#byV}7Y+wO#ac^!`VoetJ62+1$*dlhCYa*CvB;`>o0`cB}^ z;z1iTAS9dr{q}R+-J2*?%~dsXUbT zQww2{V?+EqLm&R|$PZn`X?X=t`7sjiRl&IBmDC#gfS(G`ef`nElMfcA%|f9)XJkqk z+Y(jh=^BajMXu{r!fZucAdO_TiwZZCB~}E znBBrv3p?af+@s*jGd`oukAp6!*FBk*{QeUe{vyLy`)bm2k%t|nhBlU#^d`xe+orr_ zkA1*^ zQP6uk$)=fuFtWH!wiop7ja)A~rVV=GPOR-06(%ykC=n66r(Fo*Cl@^jfCAH*MLB@r51_)H+d;c?A1G zL)SnX-zun8IhO6;D;8=+z@`Z8-fs@GQ<<(6bt5N#UI|NkuKs~#LLrTC!l|3c77m#xAq4J^1C)v#tr!X*4wkHo^#!!#G5>O zxce=1{tJXhPWEteKkfc+_%zkt?~{&zn62OavcLroFA{V!$T1$Hu6rsQ5&CqX<<$e2 z6L{uPcBA%|g3}Gq(J8xQ8k3pXtW#c((~)GlaEiIMXH-(j$-7~J#K9bVPI`)mZd)`4e z&{3!vYd6h3?csf!vlhLGR9EW((-j+U`L>&uT3!0w&EW&PwhEjx+P_YE@cM>~FN4Ro zB4&Aa7N_=icP6zL*;yIK87zp?u2Q;>r>Du(OO?3Xhn8Z)lhamj={7*Tle!fB%EmVW zYVNU}UjS-oZ#<#KOe=(j36KE*y5s-=)8)i!>-JRF)6LyW{HdEKTFlqQISV(a(+OvM zxh4oyOkv<=1Io%e$1dE1H4UW|ckm#ky47qU#wr^L|WBxbZVsJ#V`pQ@;FC-Tfv; zSog#}m(X&}>KDBk_9Bz0f@zz9-%``$VtZ%wBGlTIAKe+WH<3uH{GZIDf9mZ3)Uml_ zS7m&)CBh#50O*{Q(S>_18%TJwzBP*@F8&CRzA;@lI5lYUzp{DB>2W+M~dTqE+gxS6_O1bkJ-?X#DiXFMart zXA|`Jz0hiA;32;_>7mkd_B}Tu6MVV#jN*`|rW*W`oaVpj4qj0}{_6kC{-FZqWQ+J& z0Wjykzx!qO^EV#&n-Ki(3JG<^b3m2JuHh}9GGsyc6eyJQ|LgL9Z>{hXQ2YM??ycFD diff --git a/summary_results.json b/summary_results.json index 3673973a7..f2ae7d8bd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 74.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1006, + "Performance_Result": 10.157, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.24648, + "Performance_Result": 6.85051, "Performance_Units": "Samples/s" }, { From 8324b4783abd1bc8eb1290c71587d4fb97dc6b58 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 13:25:50 +0000 Subject: [PATCH 302/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10048 -> 10050 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 96a342c49..2913f9380 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 13:22:59,682 submission_checker1.py:3253 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-15 13:22:59,683 submission_checker1.py:3260 INFO] --- -[2025-03-15 13:22:59,683 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 13:25:45,549 submission_checker1.py:3253 INFO] Results=22, NoResults=0, Power Results=0 +[2025-03-15 13:25:45,549 submission_checker1.py:3260 INFO] --- +[2025-03-15 13:25:45,549 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 13:22:59,683 submission_checker1.py:3266 INFO] Open Results=22, Open Power Results=0 +[2025-03-15 13:25:45,549 submission_checker1.py:3266 INFO] Open Results=22, Open Power Results=0 -[2025-03-15 13:22:59,683 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 13:25:45,549 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 13:22:59,683 submission_checker1.py:3276 INFO] --- -[2025-03-15 13:22:59,683 submission_checker1.py:3278 INFO] Systems=16, Power Systems=0 -[2025-03-15 13:22:59,683 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 13:22:59,683 submission_checker1.py:3287 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-15 13:22:59,683 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 13:22:59,683 submission_checker1.py:3297 INFO] --- -[2025-03-15 13:22:59,683 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 13:23:00,806 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 13:25:45,550 submission_checker1.py:3276 INFO] --- +[2025-03-15 13:25:45,550 submission_checker1.py:3278 INFO] Systems=16, Power Systems=0 +[2025-03-15 13:25:45,550 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 13:25:45,550 submission_checker1.py:3287 INFO] Open Systems=16, Open Power Systems=0 +[2025-03-15 13:25:45,550 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 13:25:45,550 submission_checker1.py:3297 INFO] --- +[2025-03-15 13:25:45,550 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 13:25:46,798 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71474 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418488 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2584 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26127 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -35,11 +35,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.157 | acc: 74.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.95927 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1555 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1788 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.374647 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 63881d8a1..a04e026f8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -929 +930 diff --git a/summary.csv b/summary.csv index ae9762811..a35a631da 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71474,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418488,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2584,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26127,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -13,11 +13,11 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.157,acc: 74.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.95927,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1555,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1788,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.374647,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e38a01662..388039391 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.2584, + "Result":6.26127, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.95927, + "Result":9.75139, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -549,7 +549,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1555, + "Result":23.1788, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f605931ee6e6601faa61711388553934b1c25bfe..7aca9a1e57a9dd88382f4cbc3bda216e4f9356ec 100644 GIT binary patch delta 3096 zcmZ9OX*kqf8^DJwQI@7f1`TG&K9(tKq9)Clv1Q9X#n_cCAv1qQmLX|o7)vT4OH5@? zLi3`@mWHw`$x@aqk%r{e^W}Z6_k1`X&V60K`@VnoIoEYwS^mfJS=e47t-6X43tq{KI?X3zuP|VZC~-XW=Wep>szO+ z25B|AQfrDFHNE#O{FFcQoEFh$vlVEE-rfu9un5E`s4;G}m=UzK4 zne+p7+TiHT;YuT@x@cn3X649f?s(1G42M;o`V@VE zPl(14$$)tLkPEJ_{OdehRywUeaA?7(2x>u_whDrHd-~0d)NBNo)`u?!=%LwB?IU{9 z`^ho+*O)EnKHtr)xjW8|&xGyY8aPP$A6(SUAm|m}dysyWj+g0ot$nfZxzT;Z&>*?+ zsq@P~xtxtvox-0MI#LXh`=&DD?bZO`D&czSh_BNz?$>c>t!VB>RC_?Xk7TI&zAIj= zMF&%z`QBn%#ia}(qGrTaaDs5xk#RWK$1&6b{{6{RBKoCUhdT9_REOmY_4`-$saVUq z*(H-mB!`ggipd$L_Mn{0$8A%6{7qjfYoAL%XcPYl2pnE5VtlBims#kX0_s-sQUZRd zQ`l%~l8KJO5K`pPvq4}(!6qKuJ2}(Y(*#bWK``*L0LxrPqV67Lk zD0#DbEc|jfA{lk-y8<Il6Iy~}tj>pn2I83v{S64Sv} zu7c2G3R4o3=ljU$=^6n3ayxP%gls_PmkE3&p`6#1t?~Me5#86ZOvX_WY}CMj z{qwW;y2`?+1|iI*a}?DVA!Km=Z8ZEEO1=yR4oM&dV@qM5c}Mzb%O?}Ju!YZ0H#h1^S+)+% zBkUq;A!hJTDUq+jmRb}Pp#&xIW# zqHIjojwn2N{>IXpR|!l(?IKf@R@ySTEQINSRwAc&x1TzMiT>{3`eI#EI(O%$bhajj zg)ErFwz!WKH<2>|L+%F+owHkI ze`gCrt4HgKz$cz4l)=;IYGwIN`oH*!)xJG51K}h;DA)kY)|tK;pUpLTcJ^q@ zzp3mD8@R?kxsegk6_MhNYsN%Z1TS$S6tca5J}SOIfCYzu=)NG_`w4?OdjZ8V(5rtV zjf9GC&C!(j=nH|cZ03%#<_9sR*De&C&6Lr!U{~3*M3X0PY_f&xJA_$c(P07@XiPbZsq;@R!#-1$sFfQ#)Qqs z^=Q!eXAUM@;T_FF7rb7F(@IU@6oET%hza;aGYvgLK-0{Mj^|g9f6h!Cq*WEQMt{S? z33gd1Y9m&06%NoNns6G;Ubdevi#!5pH$Bks>59p!ZGy$BS}gX@}JeaC^|;gSV8aYGHqkc4$Ai&08p%5q=Rd{bw)g8s?BG zjWQ8ub&n)Nnfg?Txqu~SmX6geWit~)Q^ySb+0CASkqu==?EL2S#Sn4|lk7oujHu|0nz1}nOcIjp z0xe{2u{^E1j~MNZ!i$=KZG&sf=*dRIF@zGt8MeDaWp5<9%7uxzM-?o zWCDMHSU-q?+H;Z7aN{C>EFq`md>l*A+@R>d7=$NXj3SR6Qqh3gSNF{7YXEO&-e6B` zl?K(P`<*oE3JB4*h^PA%c(hTEK?_!ZB6gB*EUuX9rJV++`@wU(bJ$7tvK?yW`9Wm% zKSAtbu+J$1}pf_qdn}MRn zR3S5H-(K484o(R3K3v%Au(QL$H`|q@H_We9IzI_IH0G1b~v{!z~t&6L))(Z}n*Pr0UImV6;1?g+!$Y}W)p zgFN_j^J(6;BXPC|k;^qRd|&k*`p7Aw%VAZl!sWc+kB>@vOYMch3|~4vTfrac>Xzz= z7;q+87IIhV%m9x9WLk0hr`M)e#oM^`3mfB_DSd1N_fDX@b@ddOA`0Z*xZk3{0|Hs| zfZj+dZ!Vg+RQcz>%V2!J zi3~w81!>M~z@r_L$Md_!0g0;LZA_8;4^WffST1rA@#-8;k@TfvGs7hj)=;GkdX6?>Z93+!S|;`=sPf^$ z-MZKxNH@ID{m?mOW00rL0gh0|@J;0DIw_vKOJ7aMtD-)bM1;GT!;9fWn}$&H&w_iZ z2decNHY{OQ$v?H4ioT(|2_OK4fVW% delta 3113 zcmZ9OX*d*W8^>o9S%;adBik6vk!4I1LWa{=W=3TUWsS1V*b}LSK~9XxHe*-9kR?e5 zCEHs>6LPF&%dszo5JHYu=gWDo_xW&txUc8>-S_|bU;pcR-dy<4g{lyKfm0GFMJqu7 zV3g9n-aO;BubSt*&qVAh9|VmUqMtfD z9-}Sqndxb{v!%8SH-c|VMC?l^@wK95b=5={&8I!q`m^p=KOcCkvuJc8N2LFU!BTN+ z<285vrOh|g{PS$Xi+w)pd+SU`yDDM374=IF6gjzpH2?Y$ZcB3DbdUteweHozWW$A# zGY0YOXXcGQO4*_7f#%D;4@C%i+WToz^AWTs&jc&wh3;!4Q@^hyLyv9lHBO~ZafItL zc=f9P3S;NzZF-OY)@@i^AUYJLP-%VDurL=p6+oz@w~c0uiP2j$>s6R;n#1@PulxN? zc=1IH!QP&p9`(vcc4J0QUTWI-c~Um4{Nq2uNVZ2|{oWEEd(;E0m0AW04eC&#K96Y~ zpvaOb`A7nM>JCN;qT{w2j|J?>rhXb_ZewBLyC(3Ml4LAwC7%QXpPPU~*7hG3U>Uvs zV7l~p2n@^~RfXuhlET8wx0#l03z0BL=0uQI%cOJZe^T{M;GK1<)DYx&>I9hnZd(rq z-f6a=Yp>iDO@^!=TT>>R7G0+~ihZUwZt0soaCLp$$lO1I4+7B3alv(0gjV~I zxK-v(CKtiYH0F*skvL1_6ZaTqm(IEp278>;p8U|as#dYGaeeB!HAFi-5$erpTJdX^ zI$t8XM$PeJlr^OUjJU6l4tQe!4{t%KJB0X>6>qK%i4Ar1N(@zpiH0QyUl=H)Ou;n1BcMz zsg`g$7KZySWC@>Sw#9r?y?2Bj38YJd;JZvLMA_@BP#S_p&?^2(1JUh@Y+~A+XudXn zpGG)0HO!7AU5i2p#(yqsJFYh@fCretQV|e37Oc|z@)YJMFXCFb;_j_O?}U#cysPug zpeY2ROnQ7`g1$z@)@2>v+HIyu|m~Qe-{gVkYSLmV1>L+ia zq(u->izY4Z)2ITyI99KQJtB@uY)c`-4KuSQ1Gek zGuT>O+8Cu#5$Z`V*VfxTK*dvY1Vf;AuIhKLbm$ALuGn?=c(ty2RWpSNCbycFJ~8nx z<_w40IiGdYshoZL$+=h4mg>amVJ5yBCu&QBh*;y$CMp!S+s?j-bbFwUgS9D%I~{up zL4Iy-ge$&JVIdxr# zBuV5=>Q^2;emYP2HC7}BhX$zj0jzWl6fsoJO=lH1MX@+5YJkQyHtG$73^x4%o?ectDPP@ zx5oPE{>QPIB<hLk^EU( z3_jA=+DHt`uIO?C&Iwtjr|VajI(GF8aX~4FhYqJ0xx(mBEYk;oD+I!c#}V`*QY@T| zH2@~4`&4QhEB!`rtjLVU+Wgw`)+-N~D~h&g+d1=-peym=vbCUM{~q^RT~`TCV&J=| zqJvpjao)*?-Sh8rO8f$)n1)to66^ztPR98dkvo;3*y>oFI5GftEut5~g$06qkEUo` zpGQ>9o8TCmHhBpqwz*g^JeyyaT7F~E5`K`xPQ^ns%w`{xE%YewgTt1${IbI;Xlm-c z>y!A8pxz&Sb?4BQo&wXLen5Jg(Ggq8TQ}{MO*~WYSl~jsWzuTwwWIOK#T0~d11c9( z;c{|o@#nsEv-S1|?^^Q2bBah$*s_a7#@`_I-oGdD$3zc-Z~g%u7d-@Gr*Y~Gs!;_` ze)T&pyQp_g81GmFHmar_Z3WTaC=Ff-8BxwdA>A-;uSOPu=dy z*%lJo(!Jyh*NSuFd{W*+BIO?g)#-et8r9#c1f$-=NXa~xMbm05mBUdy?eY+_OI4l~ zXUnez_Jr8~{eY92x%7>gE2;3*w|QAlKYL`#gHpqI**A3#-CRTaAtsW-MKi4)r+T0g z;Up;ssy#2W>o9as4P)y*SRx=>Y*3Y0m7oH3e!G;QX?!4oJc+CB!e0t}MXksg4?5!A zLEG=A9Lrvi4leGF^P!SAl%Q}u&pzQAI&<*bzBYcad zQ;%!&X@#3fs>IdJLej?Kf>Y0}AG>zj=?w?vUz&H`{uy}xoHAjbnB)1vN9Ev;&@T&a zLo%`R55ti;s_pt;z3Gmpd(cy}qcxH(CB1%Yx$aQcLUf{P@}qlKOQhO&O1`eTU=sQ| zM0ZjuCh(kD2pvSLG(g9~fdGIt4*&oL0016=?$*}>gF>|31Fw5(h5P$GWOh?pkz%o{ z5rXUn&0*DI{?X{VnE{gmEzKqYGArH@V@Z)~iI7kWhI}jQ80?BPY$$bHvCI4!v(dKz zos}^a@Ck?=7=<>yc=8>nVvN+Z_eKx+0FpDhUEXTW(%N4?IVI_w@)wK0FYme&+l4>l z+SeVB_=dxc1w_Dol15vf|8viux&A<@*fDqBXaPxSV*tqz+$-?gkJuVar7SSUeZeFe zBtP$vr2QFiVQ2UrvHrTc44)8Zh@)S>edR)X^D3wL#8Vww66T|J;BNX;Jnq6fF8tM`38KO;%tkIDWTll$`3W5D(|e?|A7w%ckBAxLo?1<#S71_8dc3 zn@qI%y&0q&xT|dv3SAGJk_qaH*b~UXaB5)hdH(0`9;b2{IM13;G2#gpXC;YCvT9Vs xSh6x=LWfWM&xwi_W?fM=07kO1RaIsG--Ul~xw1x7je*rH88u}-5BRVCe*t-(v7P_` diff --git a/summary_results.json b/summary_results.json index f2ae7d8bd..8471aac90 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.95927, + "Performance_Result": 9.75139, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1555, + "Performance_Result": 23.1788, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.2584, + "Performance_Result": 6.26127, "Performance_Units": "Samples/s" }, { From 8315dca0c6ab1e7c4b4b4fe570f52e32b87f2178 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 13:29:38 +0000 Subject: [PATCH 303/719] Updated results summary [skip ci] --- README.md | 37 +++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 3 ++- summary.json | 40 +++++++++++++++++++++++++++++++++++----- summary.xlsx | Bin 10050 -> 10145 bytes summary_results.json | 40 ++++++++++++++++++++++++++++++++++++++-- 6 files changed, 95 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 2913f9380..e1213f6b4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 13:25:45,549 submission_checker1.py:3253 INFO] Results=22, NoResults=0, Power Results=0 -[2025-03-15 13:25:45,549 submission_checker1.py:3260 INFO] --- -[2025-03-15 13:25:45,549 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 13:29:33,396 submission_checker1.py:3253 INFO] Results=23, NoResults=0, Power Results=0 +[2025-03-15 13:29:33,396 submission_checker1.py:3260 INFO] --- +[2025-03-15 13:29:33,396 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 13:25:45,549 submission_checker1.py:3266 INFO] Open Results=22, Open Power Results=0 +[2025-03-15 13:29:33,396 submission_checker1.py:3266 INFO] Open Results=23, Open Power Results=0 -[2025-03-15 13:25:45,549 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 13:29:33,396 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 13:25:45,550 submission_checker1.py:3276 INFO] --- -[2025-03-15 13:25:45,550 submission_checker1.py:3278 INFO] Systems=16, Power Systems=0 -[2025-03-15 13:25:45,550 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 13:25:45,550 submission_checker1.py:3287 INFO] Open Systems=16, Open Power Systems=0 -[2025-03-15 13:25:45,550 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 13:25:45,550 submission_checker1.py:3297 INFO] --- -[2025-03-15 13:25:45,550 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 13:25:46,798 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 13:29:33,396 submission_checker1.py:3276 INFO] --- +[2025-03-15 13:29:33,396 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 13:29:33,396 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 13:29:33,396 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 13:29:33,396 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 13:29:33,397 submission_checker1.py:3297 INFO] --- +[2025-03-15 13:29:33,397 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 13:29:34,364 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,14 +32,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.264518 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.157 | acc: 74.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2361 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1788 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.374647 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.3374 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1788 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.374647 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index a04e026f8..11d171a07 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -930 +931 diff --git a/summary.csv b/summary.csv index a35a631da..06c07ce41 100644 --- a/summary.csv +++ b/summary.csv @@ -10,13 +10,14 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.264518,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.157,acc: 74.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2361,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.3374,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1788,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.374647,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 388039391..46dc457f1 100644 --- a/summary.json +++ b/summary.json @@ -339,8 +339,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.157, - "Accuracy":"acc: 74.000", + "Result":10.2361, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -540,6 +540,36 @@ "weight_data_types":"fp32" }, "18":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":21.3374, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +659,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 7aca9a1e57a9dd88382f4cbc3bda216e4f9356ec..b8201440b1b6dfa890d94ab2cab66d9253d5bf4c 100644 GIT binary patch delta 4878 zcmYM2c{CJW|HsF^gis+hWX#x?h(b(c>^oToSt9!o#=cAj*~T`*ShG&Z9%&HCk|kfu zPzjB(?~x@!^walup6B_TbI<+be$MOMbKmcK&$;g|!(yZQAsXhW<@X3*S^yxF6BogF z759TijT#!+Q3oG%nv( z*^X=u4z*n0JqNcB{mX%Wt||^hru2`lcU+v$mG@N~diHH=KjMU3NrbdQrCQ%fDcxjE zMQStD?lRIQvPG$q1lUECXsIZnTf8xF3} zy`T!V;%G1Fz)|Ul3MGY`pMglna$3bZN(>TUV!ZiRuA{=LRuA}<5|4q4?vLj`974Gq z4m%is+!{Nfp+XIx22mq7-%RgZzj1GaOhg)fP1M)M`w9{o99Z`duN8f=6{AD;B+o9? zLYI?7%*Ace>jJ}@WT6Km+;-U^9sAHDnQQL_U>Z+}{jPm%1~DZL2Wr!n$; z_|8OI2Cq!6Z~fX5mzt*iY90+8X+}u8!R-fJB6MPlj+Uha)L_}IID4YZi%0&Yy1Rj1 za%W7&dNj0`0RRBw`M3x8iQS7SVm=q#_*`(lzaslj^>f8iM(T`BLbNM;i@cEmlLdbH zc{05zPGKGewzRZm^z4F6eMg5{)ELJ)2z+M%6E1u4s#^`2{Psnv_sguN!;@2IuWxiF z%FN{_^5%CuHE({8JtZNV?ccWku3z|!QCKrPA^R$@9(Yosn-5MpQ3vm0N%lJkb}&lQ zx>IhZjx03bMI^PYn*BPB)C_Ou4QtcYAgr!k;`=OAxj zscb+@bY}&3+l8<_otp;uOju`g9)XMeQloMlERZf#HegVLV@e2>$(Gb~uj&(gZ*>&Fp!avAfCsH5bSG!-_d zVE5ZoUvtfBJy&LJW_u!6eqp+z@SDq(YsAvXd;RpSF8j;AoQ7vse5O^*_~MW74`OCD zrKhb8`oaca6@E%zr&!r5uK-ea5Sb8G&mBv_xRLHFZ@9Q>e)o9JrY9|aipsy@8sS;Z zEXBu+kxTX3-(t%ROMY8DpE;kz(vm@E*)SVgT9kg^N){cu`=a>bav;ByvEyV(9&FxA zPb#5@jJM@TAWF6faiSzAb%_Jxi2HQ9gd}Fth_sQIRN(_#m6y#@re2RdcJw9VCqQNo zeu2h?uVfjC)fx*Jd6)ogP&rd_{CdngJFuFVy z1`0lgf#;c(gg9mPB|#i3+q4jCjCY=P&uRf@0uPxjJ)Wc@$SKph0t0Ct-bz>ajrRXV zx1vh-rft@Eo_T;nHy1<+f`N7;jKCnuMUfp1dni=9|Khyx7#h{AnR21a=3*b8P{k$q zsL<)9tlgKGH3v+-@33)+zKq;TDvmyEYtAVoeLJ7M?$j0)-&Ak z&PlawW5k@0A!Qx?+B50T(*DL^gS|&fPle6LM&(qq6~A>`f}yRakIDSU!jKF=01Wg} zkW&Yk9xuLMYNHBd;=VW2M>i+Qa;UK(oHsX5Y)+oTE1(L?Yhl=)c)r47SWbz}ig&Cw zn~zUx`(~tzaC-#YqUn}?RDOg}S4B{mfBA+X`PHZzo--O6^JlmMJNxWMs zE3S2VRoZp5Xznw5bR@aSeEbgonOVNZV?Q2l)Dl|L7t#jdwRB5~rw?}KWde_DgA#hV z{YU$np2-Y|r5SUXk6QEFIx#^IU*cz1E|(~Qb)iUQLEguSle^)if25;_`gCL#8u)1& z_d_xBv&)8`+mUv^9hJlf>S+tIs!}A6kn07AsgMV|eOqzN{EgFZ2%Pe@k}tuj8&SHb z5-PC-`U)7UeugLO3+w~H1Fi?7rhZOyw7OH(9QLN7rAlCCVSO>|tz}JWX^z3>(z{nOU6o^7oXRvT7}^XGo~>&tj?7y0^hxUlHzEP-!LCz61x}z(gC$ckw5K$*v6=+Yq6+kv<5@v$ z1HpjTq)ZO4Y8Cga^Av*erxj&-@lY~juWrp&!QMnPUVlp{X-zgyIMaS&#GxT^Q}Oa|aI41shOKoR#w zyN^K5`12!1gD-Uo7K#Dyat@##zfiss zH8H*H;B5K|uqJO&Lw{d4FO^tQzOpVa!mx-i6w34f?HC7}o6ocge6nNx%5|kMZZ}pb z+fgr{hy;GhJ#=+7dURMiKE5EMp zUCz4a8n*aiD&kWaMQfwg6>+2Sl)+y}*sRp42bC_xU$R3OjB|UcUzYAHn+(RegL7K@ zm%7E$oX>CG*z9i9xv^;bV=CAZ(2pfZv$7VJo4ZGr@2kNBvU>1#_kEcSi}9WB`5iQ$ z#Nzbjk`>0*)a440$Ly?Fc-LC@xvfZm91TN$3#v5u;y!71|6ksrq(HTL_aXtMqv1F9 z>55H6WX6!|cUudrbB>OgPiO?!NMx?!?SLBpQ5Yp&upJ6p(5bsfx{(WMgX+_`ak3V{ z7k@fl0px1f6`uQ4u5&}3SHKND85c8G4e#A-On)z}zpNKtND>Chi-8)C2_=h| z>4|l&sdqbw4#Mg)YQC=TUB(aG5HwXmZA5uNS~9IWy^q7k?z&RO72jq}4GVAr1ylBq z*pr1Lps;8d{>RZxR(73msH$`NNSr)1jO!>|?pJux->c+e#>oWb=cR|qayF(eY{G>Q3zWz^j!_4d>C>v(+MB-~5-C^TC%j1DBn6p!d zBR?f4uH-JajBxum1?@l|=oX)T*V%?f`SLi4vN!v)YDV1k(0n9hFRz%wbE27h94$BG zP0~Gx3fOS)`2wUbC%Y-+%Judt@DLJm1BfoL!3nY?RCvy7M-2a2f}Zbp;|gaE6%%fjVH|n>b1B3nT$Vun$_44acuJ@kh($Y#pTL;j z8ni7;Tg$4SMJ~-c^9+Ipo++E;tI+8xi;E4usiil=$o4N0RQhym)>#DU6 zw}i*vrFV2KWBgc2hyp)hv8~AZjfqLU?Xpqkk}G5}3yV6O4%Sw@5+d&1RZ-Q`+|((9kgd)ES=_fC1mA&;T55iD|V!Mo= z&r$^DzAnvl_Lk-{1}(0n3r?~h`3QU91N22tST6;zhyqo9Ho;fq!?+DH69jv&#*BSE zn?Ne7sB=G&^5z`XIG z%>az2(ftox6nyRCfK9tAC9ji*|BRuL^{OKytuo_@HO@~!0;TO9`(4$h!J}w21HTy~ z=L@T0z*-BV>_5DEPTRKbhuqF@xm97y=X~uH9dN&j`LRwS$XMUMIF~uX%VLKnF~2-< z+(Nu!l;gVfmmtWk821Z47T4$|VLsKEaA5kyMf6-!wU@#hNBa?fi*EEen_b8fn2{?>pv2Ww6MmDmx`z>X3@w5J8 zt>x>-pP!9+I01P`vQa-{LDkq(PRyN(4a*-rUB}+)k}$4AJe=PEU9WvHT-TitMEz@A zX7!d&H)sVqYr?2H?5nlZ7#ulSZT!6a2xOR~VB^(*=2=W7h`cFJzjMl6bp6@yJO%3E z7N2`+rwom+K{3=!|IL?hY7mv{{|g@g0G{(W;y?8?%7qaZ#mI|$4H2VOzJ!~F2weZ~ z`+lBX{EaXEQ%|FMFXK`!bDj6HC&WEQHpdBMEZw!=DpuNGrRl8?mTDDIkRVGfA4hv(JAYvpgRnkW$>T`0Q{NpAvEmx z9SSva&cI4($|vNQ5AwA}-Mlo0=0iA3m~9a&!LT@pH94w#vD-b+y`lR27;@jN{b$dq zjV6rJ`E!57J61kW?a4qF;#WkXsWV?YPYN(6u74-4uM;UocHlAI z3&Jl|vsamDZg7!@5!Jc%ywA9Z>Z^L*imR!?f(mjHqg)s#Vnim#8;Xi%jgs)axr2)s z;z>g0xvDqgnb^>z&iEn2y+2=%>ehlReX`Z$hIzQk26dQd3@&*^mpmUD%AoXtQ?4TQ zRqp9{u6-n^h=Bp+Lv1-Br1mB83)$6fF&owzjq{bFQZu8d0y9rO5~}gfTqpU)IU8As zW&nm8^QrzqC-lXvZ-B>zebcnoN+hW((vDsBzVP}e)4b;E@6qYfX|m?K)*jWV5Tb#Y zo7xyBXKa-T`{@I(``_pVUY3#)qIOGqVtK5P^*MJS?}Idi4M$?F7MCUd5j52@6}IBo zUh~N934f-ObEZ&G1jiWIVIVWVr~=@3d&|Hpt0Lhw?DQ!9c;0Nlm-nTJJnbwb{;WsXvt}SzdKGb7K0U8FSo!aH6a`j;oU8*cTWQmHXOq1$(|e)WXoLxeMx+qj`uCD@GGI z*SUV~CD9E}y!0G+VX5Jk(mTIW6%g z!!3Ah&!8DL1UgC`X?^$P$1p~~X&%z>jNLT?JzHe_^{LQ$=WM3<1#wKFA7HNE-ag}$@X%2 zZ{EIqibr#eSX6)9_|{-GrKk{_^TDjqjVJwZPdewfwloJJ#OhcG@&$7;X2E8meACJF z_R;DrsG1?;(7(gA!;#%n?CKM{)=eWd>GgrUM*`c)w80X-CmvCBy*FpU%#OyM+Tg9y zAJOne%WkoxKOEgUAH}ksTou;iwKRwg2?;TJc2=^uY}(Ghiwcp5@=3$onxB}zVtVM!tydkl$gf0H0*r78Rn4x8XGo)2sFH8&~ zmn>(4*3}zkK5FJ;PTGD7$*gNZ4Z6+$gKhD_a5x;ZR0VE4^V@h9u7JP7<@Sk9 zD6={__&zhcJ)qDIt2 zA~M#i`{pot%MIs2&Esz&zloC@JXU?fZj#ltMzaSwlM2<{jMpG1?&t9% zE+PB)G4A!Ly_WEhV5Cvh|-6kgaY6VjY!?nlWw$6sJ+w4PTfk~W++W}{% zAr6p+U{k|t*`Cl$3^J$gUUP#ihfdqXI>f;D(J!SqpWKIJ9URZEb`ZvQnn?4tir|1u z7$f2a+n{-Joo8zlGj|dkjD_(cz|07u@qJ#}R4^g3R+1CGuLW=S;2y$*aknIGFTG;e zJ%yIeyx=8#DRkhu07bk_uaspmT4>GdMleiYb^j)RX8n}$72BIo|S-9a)i@2W4x(TXR1+_wni^vwMnY^ZuK1`2Jivt{XYeZ&9~KO>zBUw-7%kZ? z{7R2tI5@{SgQn%@8jDS98{jEI5`U<7UM1>B7aV&QUqB-I^AVCOo=lX#HUbjx3tLuq z8{MoaRPI!eo6*<|q~c)^2t$k+9U*r7`bxFJ~bA0R?(2$u`B` zYgV#4uMgD^L!(CFoZB&DN4PPrP(G@A7AAx434@E!$0xnFb%4?_+K&xnBc9Z-c z$~@7ZWtspK`zVHAe@8FHDfye0c>e{3_1uo2b;Gg1aZJkpFe{oE(Ps8&937w!M>q94LKoeGV{oR-s=RNI zszH`kz=Jk}08zVN4^Glc_?om(X5k30^{{?%+?8dPs3+vpsc_=w12C>Y4UDCF1qP}z z-)+Xhr|jW4&D=ZLCE>r97g%r=xorVIkzjj+R9I32QUJ3D#``v*#GCCNeY2r|50Gkl zEAH4ElwlCkc1i|b?qwiFvN{x6;(h{z?oSR zgpx$P{${jW>g@~i9EN0X6Ufpz4a^5|Lk)*r2-AHN!iD`YUygW)3{NSTso72=MSI52 zD7tiFnr)TzvCE7Pj-SVV##7xUstJpj_@z-5fMtN$6@49TKr-RpASJCSw=db3H>WiS zw~T>(n@VB#MQ@r_+OBNJ>_r{>sT8~Oy&Q6?C)l#4wWk=X@&na4e=hL=bu;{9sR~Qb zE02NMI428)eR@ueA;5s(ACO!%lc|P?fnvhV3FuiK{W|;ZKE*P7)5<{V!qcNb0_5wc z7pcZwS;yD7u#Z>tj9hd`Vi9i5IOI1xp$?6;zv9X`0wBOBObGRXCRal3(_0>dwNh&- zDKwbF+x?z1TluO1u-S0*kOS+BuS-e}#amI;0O=Mr}tp3PSB1 zsa^(6-aud)<=2;Ue3*pElce@P;R>;$AjR6N2l?-KrJlejUi)ZAv!rndw7zoH4&Pa_j3NT+Eb?X zquPzbS&6vePRP+KH!D5g{YpiabTV3|KE;RU4}9tcm?LiR7cn_$)dZi_42{*(GS3*AwF^U(4y> zIWu;|PRQhIlezkxjdBNV(h|&Wbk}8f(*%=w+4HhvOuUUWuz#jbJ-Z$ng~U9FDlv1w zGU>_CJGy4%dZ*`N_1Kg>$xK>lK*LB~SsvY8TOSFK(uYbQ_x1r_Z``SdPPQBx-EUp z#6YzF*~y4B9OAK+V%G?Kx4Xa1v!_%WKa(YivE%0sCUaMhsJY|$Y3dQF@^wSJq>7rj zjPh%AWEQ|P{9_h-omClyI6qUGC0iiP>|7AYdVpp=lDzlpcpCoCR z+|Uh9SUSo}PUP)Yb+-BV)z#L?bwH%rPC15EKpBb81^qsW79IDngW3hPOt(ls^!cgK zGt`u_aahbxF6gI*WQe~j6MW!1lze_+JB>FSxjxPcNf42XZ5 z|726(sRY*NckYLq(G>WgosF-K19+EPAk5>1iKV%J1Qs)BYJcR5;~ECcq9e~#qRkv4 z2}Xxg1-ltdiFrLsH}Raj_AckHQW=@=uF9p7Q9&D;9w;m7QEt7dFFLC)7^*2?EYWBn z`fak51o&^YIgQ7#1}6B%jGb>8Gm0s`&dRnzD}?QN2eaG9A~-QmwLG=QeVUr|qb<^2 zJnc1Iu?cFb+SZI1kLrW9eqxDl(J22HNHhR0#bBYAX|hgt4v#NOFU|<*tB3*%W;v~ew~eE&zne@m<6AM6ZtRa~C^G`HTJS?W zqL?+k$W2xA3YYA`WWrV?^cjpqjWg@SSa%>YDI3>3y|**#r3=iq?Agx3eeM?n?GbUj zdMK}VQ^2ODM!OVonfY-eIwA!_00yzw4RP%MR7DvV)0;p(X^`b|KF-rr+@^1xDCdH& z;TH7NW?V!H8Q;5?zuS0S6Z#8kiKfl+#HV{k-7 zL#!B$K2ApK*L!Mxqo$}FtCXuh)Ioi2-o`iIBJp6N?~OyAelIH5i+Loes@=i<_-4U) zP2rBub%t^MmGN)w=47YOXpZ})os(=qJA#zlopp{&Ier0iC3^ULfA)Nh%#_z9+_(G@ zTWr=bNwQ~b*-G?Bvb)#hmYdHg`$3f~e39g|{}p-g+5E~J|1SXm0JknhfqyMD%pU)T zUxR#=8GlcJhvUCZ`%*mks~`Mpp<$l*Kmj!}dVG_B0vQJXM?iq%Zyor_002F50O0z+ b2QSyqFf?BNjtY4dFW&zSFO>uM@816b*qz!l diff --git a/summary_results.json b/summary_results.json index 8471aac90..dc425188b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.000", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.157, + "Performance_Result": 10.2361, "Performance_Units": "Samples/s" }, { @@ -790,5 +790,41 @@ "ColorKey": "availableMLCommons", "Performance_Result": 20.9828, "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0023", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline", + "Software": "tf v2.19.0", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 21.3374, + "Performance_Units": "Samples/s" } ] \ No newline at end of file From 835f371dd2ea967c4683031f61978e05e865047e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 13:38:37 +0000 Subject: [PATCH 304/719] Updated results summary [skip ci] --- README.md | 53 ++++++++++++++++----------------- dbversion | 2 +- summary.csv | 7 +++-- summary.json | 62 +++++++++++++++++++++++++++++---------- summary.xlsx | Bin 10145 -> 10228 bytes summary_results.json | 68 +++++++++++++++++++++++++++++++++---------- 6 files changed, 130 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index e1213f6b4..58a5e4ca1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 13:29:33,396 submission_checker1.py:3253 INFO] Results=23, NoResults=0, Power Results=0 -[2025-03-15 13:29:33,396 submission_checker1.py:3260 INFO] --- -[2025-03-15 13:29:33,396 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 13:38:32,463 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-15 13:38:32,463 submission_checker1.py:3260 INFO] --- +[2025-03-15 13:38:32,463 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 13:29:33,396 submission_checker1.py:3266 INFO] Open Results=23, Open Power Results=0 +[2025-03-15 13:38:32,463 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 13:29:33,396 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 13:38:32,464 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 13:29:33,396 submission_checker1.py:3276 INFO] --- -[2025-03-15 13:29:33,396 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 13:29:33,396 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 13:29:33,396 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 13:29:33,396 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 13:29:33,397 submission_checker1.py:3297 INFO] --- -[2025-03-15 13:29:33,397 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 13:29:34,364 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 13:38:32,464 submission_checker1.py:3276 INFO] --- +[2025-03-15 13:38:32,464 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 13:38:32,464 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 13:38:32,464 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 13:38:32,464 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 13:38:32,464 submission_checker1.py:3297 INFO] --- +[2025-03-15 13:38:32,464 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 13:38:33,750 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,16 +31,17 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.264518 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2361 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.3374 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1788 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.374647 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.6841 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.264518 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2361 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.3374 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1788 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418547 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 11d171a07..e6c657ec0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -931 +932 diff --git a/summary.csv b/summary.csv index 06c07ce41..96dd18198 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.264518,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.6841,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.264518,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2361,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -18,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.3374,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1788,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.374647,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1788,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418547,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 46dc457f1..0da8c3bf4 100644 --- a/summary.json +++ b/summary.json @@ -304,7 +304,37 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":16.6841, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "11":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,12 +599,12 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -599,17 +629,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.374647, + "Result":0.418547, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +689,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index b8201440b1b6dfa890d94ab2cab66d9253d5bf4c..76723550f901e6eb70aba5b284b9bb2dfe762d96 100644 GIT binary patch delta 4096 zcmZXXc{mha+s9|@=4WeABKr)=nvi5mBV%8ZEJKL0FGZFxmdZ>fj6qFA#2{p9EX5Cr zXk_2lFxj`sFedw>=Y5~+_deHq{yEpV&bhw#=X;;~KG${fEVnF)AsnFYa~p>1AOJwv z78k=I2Aspa;t)d0%d~9x`{W>4S_;|tq&eGm7)DGOw+g7k1{tG$e_k@d$(OpWxb?|t z*;8~s^5QF@nkxb+&7*GSgPku=bVRx#wME2+-YHG3MW{<#)RJPtz+LgpV+@IR2a#L3 z34v}~M%tkG7C3(grFl%wcz*BMwFwc{{s08;LX(1yDhKk7_eEnQ=E<{gG1kdLRWB`X zul^9>{`GJ>C(n_T_E`@0gka@x{_T_fu9;hq<12wzW-&ccMfXIP1(2p=Wys5|?|`-_)RRLA&G2 zsgAPtkI&r|Ej6n`Z?Ckgiu&K3YGW7^lfBC#os)l=UtAIsh0R=)cHHLct{c7hX1mJ5 z(p1^^AWi-5ZQnAHE~*HoGI@1%QcRs!xW<7}MKM4=L9{=S9}TJ>?18tDPN&Z&^r^7j z593-WCilL~$SeGa3no4iA{jAtPD(T&*E^zqp_4aqq0T)qKjlVbgK?#{&DT}oIt9BrS1ZrSb@Q)t#-1<%bscTB%sWRqnV>Gw12+`E}!m{bFUU+RSHDmRwH zbqb6`SII{hLE?)TFsbleeFz1#3x`k!YEZB!WRYAiWwja&%bv7^Np%Gz!dpfU%pjE2 zoD*oxx?8B0nN!)#%9`xp5nB9Jw7jxrx|(5qAPP3HmkEc}Q>`dT_m2);3}Pf>By>=| zv$wB=uN;mXu!Noy8S{wBPq0)aB%Hs8Y~8_tJwdQ8F*aSz-Cuj7uwGrHrQK->=DVlI_FpHv^DQI~X*1aUT7E5C6OdAty5J8Kh8 zfr<%5fh?CvMpPDVF&L}^u}0`02M2XnSTRAsXb2V#LBphsw?J^nOj=9)uai%h(Jnx= z5Dz3W4F;B$8R(X?CiG-S9EpKTIR=KpGi2mz*KWMCSMIZ9awNg#3Pxm(odA3){^?rLeaG}w0f-vbK zL^ZGd&J~ukNg6J^k$#h@=|1wWD~;=8ZRjV|*HaK2QvQ zjft$52G8Fx%wZr!xNoOHQ>{)yz5ICsh>tQgmSFgMVvmSCoH$ESJ<usQ1(s*?XSpA*! zdr0>R&@>;YfMz4YxI5BCxWA@B01AJBE7K<&G#_VbAdkb*dc_}wdcDY2#dFMel!J1l zpF8p2VIyd)EOUC;S>MLv@9U^k#q1&&2-yW)n~lz*+mH7yRZEj$&sI!StrI-J+^NQD z`Z7MLX3j7fV}Qp=$gvmQ)U!@w1^fmCu9Z}k@OSMt-^~;+n^_IbvCuKE2QsG@4EhPGXTH3UHGZFPJxp6Z#jK$MSr$|`u?@i&2S*$KVao@q zqS?Mdoc8qzH~zsy{ugKV1!JG=3*|@5Dy7Bm2eQ4_I`Cuf{W3^;CHJD_K+jmKm5)u| z{V&tMHd6k->k-ztVxA->75oK`&y~6AFWfC>+wf)if7_Ui#T(btHA7F4&lDva@S~)W zU0+ z`R(;piJhCPZgw`VXV29r>eM>i49NLp5pl^jU%CZL5X;{vC{6TAZ3j1q5rF5=Y{d#6 zwuvnIp4d-D#?;QUm6gV_aAe^d6b)bMg9 zWGI+&%R4x~A6=kIZsKdBmA0zKWt}@UM5nrTE<2as?|~r~P||C<$0bn5+bi)Bx~gl& z>;>k2?cl+1;{HDr1$Oh0<3v)A5x4#ak@_FRn13VE_Pl=m2eFZt6WQ#58;y1yDjQkfWTftx z9q68l9H-(sU%vC7pY(OSe9be7UVNS4C*lO48k(&~L3N+V5_<aCy|MeY$6-Z7yvedW>)gBHHk(Y^eU0-~ub82`*Ul`_C53NA zMx)sB6lUm1H{sYcjM1TFr4ntIXaQ)Ksz_n0;QdoK?Uj%W*X=yFSm^z|&oF*Q5da>38lM>Y{oYZvwYBkg!BjaG1@y9rCAVc1a$AC7fn zze$y*u$BI=E)?rZ&vVmMGHISNV8T_-if16k&g>-^#*+% zX>_)QBx6^|)IyV+;>WC_DJwg_&c!KiJLMF$`Cd&-E17YFIyx?Y3g@%ltG3#R;PE*x za`cJArD3r?X#3Vy_RU72-Hc$bg2E}&{^iVi5AhLVGSB>^+piK4?l#iM3zmdRE7JN9 z#*41iKDPkLR@N(Q>~DBt47Q*mg;^(K_&oIrYgiLh%+`_LEXea!IYvjUByIi|K+vx% zr~4vU0022b+z$Z}r2V+fPffn)sR}F4TX)2wuoQVgRYPc;3HNhbCa2ZwChy0aE;p1z z?|zx8Fyq3#F9Vo{=W8m`d%Z(zCQe6<6FA58$rSC&Yi^UPlHcOBF(kS-b$uJL8Hoyh z`B;s7L32%qX(d|Z@VqNV`n~$?qI&kOB>raCCgy_lr3fHii3p3p+bA6>4ztOdyq0_j znD%z(>kq-Cw>t2$Mr3UCyG#iv+&FpU)Z~@u>PIIP3Yq|s2TMLJw6%w4Pgk^hsqQ^| zgcx3uZtJ@i!3lTPuj_d|`RokQ@k40L{jl>*vfr9&eZiO>etiq8sln(ry2!K*py5Xt zF{x**lR>7tU&`uuoCn97p+)0qv&j}l^bQmAE8I#d)IS<1+bwzn06{b;-1xOXb)V0h*g_O0n;=DXc;k48O1N&O~}qpE!X zah+U&&A+j)SFPiqE_yz6{}b&8>~}-QLs{wd@M*mp?y@u1P_aXwNafMMTwz6-2IJhV z!7s!L9Fq5%&7sm>NBf@2I6LL{PR60|=lz9ePy7lyVm)`$ zKG)BtRq2KCR<4M#Z|4(sU7X@QmAkpc)wz+kJ{T2~c^-d#uK0Gvtox4aH(KcUbKS68 ztvdPq< z(6DgHow^gU?=?=uMsCWhp4fcwF`DfPS8mVm+rf4&|r^ z_YrN37L*MJ|2o??*b#HNt{Aa?BWpiyvu_b3=#UKVncyg?5#05%TVjIhswmp~s@WE^ zr)1Kvk`?D#o3bvPM5IpRIQnw#IpsNXWV)~J@Ca-=ge=MhV6H2)^!X#-d+sOt1G{Pa zC*(QIh@XD}UyNZ<7Vsvcf`iaC0N60 zaZl}m^Yc&-V(UKZp<`nMZz;h>+~O;6uy8v4vTM`F+LmEMZN&E|`!@bS2&zbTnLn_| zFgaQxO(<6EXHJGCZNCp%W~KA}?xvco_xV7O0XZH_ix!ILM*`pWr`DnGem>7wlPx|1 zlK7?Dnf~uTd;w|)e2UYRkvj9wy959T0R9=VaRP$4C>ecVBd%UXi|PCs+>Q+N%%8MF ifdGIN6991HpTmIIIIt8>OI8PX2Ny0Y%_ao>v-d9nCDG^r delta 4043 zcmZ9Pc{CK<|Hp?ALY6{mOva3{%N7YKV;_5AkR|&vWS4DfkZsTmhLCl#l-(nHg(5tb zF-mBRHETSWM%KQ1&iVbmzw^8Q-g95)p7;Cpx#wJ(X|ZWtFf02{{BB$kI{*-2mJ!1$ z%s|O_!77N(lkZJ)iS#b8XJRU2;TC1<+uze;#5&hPk=y;)D24NvJgX^`_Lu2?mDx>u zhsUnI-uxCgWaCRIZ83 z7|_^fP$;xiAtWxgt3sgTT;#67HDf{+ysIUTNJ;nCYitRvn*hD+Yly;%3vNtsecmL2 zrfQVA==>YZfiimj3F8zYG0RRS-o!vsV9s=jI0xgr(!)xchuM2J#>F~zu+>GlRD17 z=n?Hx#V#w%j#WzcrEPNMMy9q`&1cP{Q#e{PnXMaUBT9>KbPtNe(9M^{=f8)7WX+r> zOY-3JzDBZ1A1DNSo}^}(Rxw_*%%ov+KN)q0*^rpRP8yLjm6R>KYp+q+B5UF6=Dm%O zNt^&%-u(q8!!BlA}G)8$cYZ6BW&u)FiX7 z1;Lc3!@=PPaL7Enni#JQGyaqsAq7<#vP7BB;aBHTIc}|97PiGyLH#gthk~&9FEiA8r<3A({7Z$^xm)I@) z#p`qV`?qy&#Gu__ozY0ECPl=P{Akneim=F_@^ur+x7DGnH-<$vI+9ScjzASN--rCD4lOzSv6KCK}g5&iecsLjbiC~7{9~wNk9e%Fo3B2|(oZ~F7h5`egX3jK~C!iju|!f*gkn6hpSiIl4Ix+IT3^ldT0AhjN%};g>6p} z+SFL~s5N@6U(19MIAOuWP@3+MRX`_}`8sZaf&xnz-2i9@RLI&hEs-VMRfr8j)(0o` z3Iw4?`y1Uc2nZOQeoPI;(Rj!e#gru)O zF)U$_Ok$~kb0RW*IA7xL0`BtP9kmE_nPWm5PF3+ZT1b?tLD*pkNxb5BT}C;c!PHfS z1&h3ppB&d-f2#kxqjfFERwJ?|vL+))pz_xja&b51$~4ZYq&EbTQ*rxR63cst5P(ma zwM>>V<1WDI6mAQOVzMOsg%kypVb`9ChgTRX8E(GG_ow}ySZYYVGN|rxKg2iM)C;YG zg8BC@<`EOv_k>Z_7y~-&KAV^~H==~$4;Ic@yn5N~dS5(Y>T%QJW?HB)~|M<1_Clqjmed^wmE z#WxU*h0M#82wFA?PrHtxK}-^CFYn7_?$)l_t2&xXBqBD&QdSl6VCXEzi4muUNQImY75*wy6HXm-bK?QiGIoq|HBk6 zxh-jw6Q09@u7KzY6*YY7>k(p*OvzU_6hfI45r-n!?qZx1;B)g?HlYuT|~%q>QN#~rL^;kvrrM<)84=hSDtY1)Uwuy zOeIRb)@i($Jc84VhJ`q%TJypXb!mct{m=ME7@Vpd2 zY)tS}_xd3s^r0cHHNgvDRKg&ga->nk_`c!OTnRt6$pV!2~!C>U*g${ngi% z-3!^bJR%ogPDPKGQT5i_JWy8~kIx2)!7NK%KHy|QCELWo1kcBaGMuYIDkQ@T^1SWS zQV&}4nd|A~jm_>nITefPQ(OgC62?uMG^=1`y|J@z{k{s)uV94i^rFeHTamYav?O)mAD9KH%oWBfD37PQ4-lp6i%^tQejl7ipwf+P}O#8IdaOo<*X8PT<)1 zMZ4&j%ps2-_Evb;=LVMk5z+i3=$PCa*F&mob{c|u%aoNODZ1abS(oFoR}b(z&)*LE)u`mc&wXskc+g|=qdcKIDdjotL1l5ezUPYsLk0!7hjw0-VW zShOZQ7EbtQ{~9N^L6oMJD{dq~nGw#vAEopws_4%sxm)tGX@Z1U;0iq4`}io$efYr$ zc;{tX6s>F!c0WrcZ+6Gax5F023-6f~7fg346)5Xz!rEucQdPo=Lu<2Z-6J#0f5ekd z#1HLuHpBP9t*%Nyp=~O9(0csFq}1BqRSh$<4>Y;3i-*!*Ynk^NX&i3-U$7^~PWyps zF8rz8o|#dOZwfkr{+gS@i0+eho$~Kw5_Na>=j$0MkG*>%k9PBlse*^Pxd*XILw+Pf z`je1#C;u-%mU4=xYHn|@svt2bH>BAeJ~%;<(S*(UZc6~WYMrm(<1!Nbn{PLR;FHb4 z%nGbMi*rWTA?p(?!bbv)$$U8s>>0CX$-ztjK;eZ9>5G@q*3*IWs@&1kTC;jN#q)3V zf8~M^FCP;t`r|RKQipK1cH@qPXw<{)2qsBrx-dFIoqKs&%v** zW%#^6;2E2YQD(Np(!77`bsKX=&dD62y*iK{Dz z)=0Z($bdcxR(`Ng@65znyC^+B*0@u%PQ$r)m3A-@WsB~6j&AC5*~m<)y=!gHndMt_ zqA0WJuCQH&Rcx)jT{z%;C;@QXy7^USh5r3;F`_=-`1|0;$i zQG$zv31`iUj}2*}ZhS(>zF0tt_8{J&YBc3yMExB(^A#w2C%iaYKSIP_yPc$rGBV0= zN{~j!mV~eQKO&%vNLuq3h~LW$`(>Zjyy+xNhnhJ*xRktjTt`)LeTNZ!d3%`d_#{mC zm8_Rt+gLOH_+&8-z%1q2@6_7W279NLO!(*bSGw!x;1A>5%L9>obB79BFR(2OBDZ80 z=F+qF@W=G-=A2(*(PP%0RZU7G3YEzM;OTU^G8PIB-2!@LP02#G^Zz7P%xpvKQ8Vct?-u!pJf?wr@tV$itvLt@h0TCpqs|TTM_K`p&n1%*p>iBja+> z8)i09lO=7#G~ZB+v}1Ghs?@IA09dWQvy)+%1ndjgE0ZpnE|BQ0OX>f)6BsDNt#`l= zx^Ln!_;JPEA>AQO@R#~X*r7XhE3!b8USQfc;tm#rr+=;S;G6jCut1%4!(Gliac3h$ z)?_0D>tlN7BR`c;v)FIf=|Row5(F`xoNat}$(a?c&U$DIWdH!o7y$qt007|X?_w6{ z|1e0-#XrzZHXP;q0zc^8bB$YP@0h)`Q|6YwSNsnxy9V!~(M-ZdoKgV1`YhfShIaf| z|AMJwEiia1zg4lqUfA{WF(%~BYj!t-WUv_`s5qBB+Sh9POmcpCGTBP1VwC5K?Uyj9 zVjSA*oWIrOI)}$8jQTga5%Khy%b0qX$Y9^P{)HYtDbcnz=EEp1kd(!W2y4!%?F*qH5>cfLTl@ioYqxsT>OwU3q$xr^-0o>VNC@@!3Xq(E-N) z`<+%3vtm%sz=?sc{P(c}04|(H{6FjQlimv%*DmsBEQv`n_-1g!)EH+jXCPqEEC0Wg Y|DJd!l93A2VBpIbgo(0zg#6j~4>R79zW@LL diff --git a/summary_results.json b/summary_results.json index dc425188b..18ff45ab6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -365,7 +365,43 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", + "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "undefined", + "host_processors_per_node": 1, + "host_processor_core_count": 3, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline", + "Software": "onnxruntime v1.21.0", + "operating_system": " (darwin-23.6.0)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 16.6841, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0012", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -396,7 +432,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0012", + "ID": "5.0-0013", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -432,7 +468,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0013", + "ID": "5.0-0014", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -468,12 +504,12 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0014", + "ID": "5.0-0015", "Submitter": "MLCommons", "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -504,12 +540,12 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0015", + "ID": "5.0-0016", "Submitter": "MLCommons", "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -536,11 +572,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.374647, + "Performance_Result": 0.418547, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0016", + "ID": "5.0-0017", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -576,7 +612,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0017", + "ID": "5.0-0018", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -612,7 +648,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0018", + "ID": "5.0-0019", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -648,7 +684,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0019", + "ID": "5.0-0020", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -684,7 +720,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0020", + "ID": "5.0-0021", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -720,7 +756,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0021", + "ID": "5.0-0022", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -756,7 +792,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0022", + "ID": "5.0-0023", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -792,7 +828,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0023", + "ID": "5.0-0024", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From 449ff0d176c11468c207cf3d312d57cbe4a6e601 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 14:13:19 +0000 Subject: [PATCH 305/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10228 -> 10229 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 58a5e4ca1..782665bbf 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 13:38:32,463 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-15 13:38:32,463 submission_checker1.py:3260 INFO] --- -[2025-03-15 13:38:32,463 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 14:13:14,953 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-15 14:13:14,953 submission_checker1.py:3260 INFO] --- +[2025-03-15 14:13:14,953 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 13:38:32,463 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-15 14:13:14,953 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 13:38:32,464 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 14:13:14,953 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 13:38:32,464 submission_checker1.py:3276 INFO] --- -[2025-03-15 13:38:32,464 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 13:38:32,464 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 13:38:32,464 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 13:38:32,464 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 13:38:32,464 submission_checker1.py:3297 INFO] --- -[2025-03-15 13:38:32,464 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 13:38:33,750 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 14:13:14,953 submission_checker1.py:3276 INFO] --- +[2025-03-15 14:13:14,953 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 14:13:14,953 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 14:13:14,953 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 14:13:14,953 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 14:13:14,953 submission_checker1.py:3297 INFO] --- +[2025-03-15 14:13:14,953 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 14:13:15,944 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.26254 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.408576 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71474 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.6841 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.264518 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2361 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.97085 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -42,6 +42,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.3374 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1788 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418547 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415051 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index e6c657ec0..2abc897ab 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -932 +933 diff --git a/summary.csv b/summary.csv index 96dd18198..2cd9151a1 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.26254,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.408576,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71474,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.6841,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.264518,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2361,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.97085,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -20,6 +20,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.3374,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1788,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418547,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415051,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0da8c3bf4..d95699074 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.26254, + "Result":0.408576, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2361, - "Accuracy":"acc: 75.400", + "Result":9.97085, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -639,7 +639,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418547, + "Result":0.415051, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 76723550f901e6eb70aba5b284b9bb2dfe762d96..7d09d618f3125d229623dc117c6add7c25e5cab2 100644 GIT binary patch delta 2761 zcmX|@c{~(a8^;I33}u;-eP_(bQj<`&F=LyFXk=u~l3kW=)`($rnbgG0AnHn4%aUa* zl@w77*+z1ax|V4|cs1F{OYi5t=Z|y#IOlwS&-p#i=lPtU=T^_v_VEE%JeQQ|Lws=J zMEJDPkwh~uh*e@xX^Ka(m>BXND`AeJTW)kt114fg@06LCV=s5vKXh!0%@6U^ERe$e z)wrhPxpoKSRmk`d2fK~XQ@UqAY#>Za*R%P-9^6402XBO0YlRCvbc|)LyNw-ONu(f4#g4alCZnyF7?!-yWN`_;@WXyP9@rn9usR-+PJHX zrolj^I?T*zK_*1F_x5a&!5f?`SO(>#&VLG=JzyOcBUgAd3I! z{Nf>}>hg)6eS>6K4=8)2>yF>DFg3Yr(NyFnyIztppDP2onW`({xD{nhV-sy~Y6@0@ z!9A5;YPNpC7l*q^oJV6cMH;Ru+s8M@62jDK_F?k(Db8d2$I*7tFZD<33Y=qV<{H8V z79WptX*B(=iq;DdMSfLjVd+F&7j}wNO1y*pmqz~_bN%@8G^fyy@!l_Fkba64d6~r@ z+u{jnqvQl!eydDvNwf1^;;|*xh{@j*ie*Mc!j}di1!0|Mv2iuzY7O_eY#qvSvPQI} zl;tM2GgRu5R*!+Azxzy!vW>!2s>U8UZ;q^}=nta&aL3lD$FbAFzz+$x(dc?C82J9P zFOU~LD#**jSU8drH<5J`$ou)1AumtqYqWA}Sa+$&ADCza7)T(l3#BB_d#Z`yC*^>= z6$!V{=;&p!ZsY0I{|IvC1kj73(YQl|uzl>SeeCQ3|JDAz_hrMvKv#0Vm{FcJ$M~Dn z{lNRHJ5Z6cQJ^q(ky(mBGlFaALYbX8*4i(+BqbHOJR04N`hhKKW#)=Rs+D zjC#Va0Af8T%bDM2#f0hm#t&ifm2~9Mi??g3`@G9h>6`HjPCB8cN`kS!wzp+a z`xci@UZw5IQy)1Impk>VXRgi*KOdXlTD$KQVM~m<=n-}CVwAV6;(V5*2O9!DqQIL{ zWTA|Y0s}Liwpc`g6*9s>yG4hrKp^-FKD(%_$m&S+$5Uh{{O5_KJ$zIlbr@tB`(+YZ zu-K4Tv)+G!GgHBh9!U|H!lyU1nHyUxBS4$-*sTLi!-4>A*D$ZV6beMT1M^Bk+9=dSS3%kHgdoo-&@@4w<)Vwb$6WUnMADnnqnRY$+ zKZUE1KS!0!o;B%8z8lt6Pm4P`i!i-*j5uq2{`o!YpYo&S5bJhm4|L^xf^)f=zwB*Y z*+pke&b1#5*mlCygV8UCT2qgwC3qhe&fgT!WK_N+bn)X{iZKvvrc+z)nSM%kr>;ZG z(NWC{g%N~NlH^EE=tn66_+Hr31sECZ82e+vpo|b2PB`P4NwIznz}xZxj{pDwQSkm` z25`mPcUl$#0OS~n;J*Zj#%JBi0$U6x)HJ&)iU;|paWx`CajYr}y>36j7Lj29y;Dq0 z;Zj9+U561hx1b29!(XNl6im0z9?Qm){zy;b$+3i0#{^c`aP4i_)0d58j(v0yg3%}D zuD*H?w_%U2ia)$whI$a$7&hw=?dI=8NMA+D^5xa~%B5eh60fI{6)4M5zd)X&ZMvPv9|3 z_F;YZ*A6Hk$I4oX-(Km!2XZOk6AE@%hyVQ&UoI(Q7e!_dtRFFP92h7aH`qNLy{J*Z z?Q$BQ^$3ZgoMz2i=sjrI5);0{)T7y!Q9X2tl-cuAu8A(ZOn(Pbu0W;eL3qz11$`7` z;T&5y$*St<-wRynJTw30dYWplw%=;gBzECNU|2OWc8FjYRP;wDHAZRw$&=|pVtvpU zr7|A9n*xK{_1m?;~{Jx{B<%twgfGkoMHNk1U!V1n5(JEouu0lH5!*04WUuUyIHG z*N>@GS#%`Z1#G0aGA?xtWY%w%O!nGjXzO1NH%`jUB;GTWQ-6{Fn;5*YPn1nMAo^_l zOYArx5+!Al^rVv>mf9Jq1d*oFZ%kD&cTmX77rw7=CUjpfODjCo8n(dkXMqx#ro~1Ts6Qb2fnq1o|iS2(? z9%+DVq+7tQaqjG=nsU5)pKO_1yM+`MJM)guQgT1r2(%w{1e-bnawG2N#($iKtKAwz zD$`sOle;;7t4suo2&E2;d@F9Ipyn!Gjsc6$`7TVR4=c+GAlm()a;-?yi#+nV;%uzwsgEAw|_VjT3p zR2ppZb4rKwRbB5u2S0(IH0y7<=&-Qa8*L5V)kFAVMU#7{VkMfh|$5bE>$6 zo-j&`ic3Zv#~A-UaOhF-vlI4pVz$k1b5|mo_)U7bpeki~WIJyM$U6k9YNW!iL|yA@ z#}c3#Q;EsuWAKiS6K?a#{Ij9Qs^^j>#) zCmsPKm!syzBg(DrBr4OSlNDzkK0|T!C0RRmN0N|k4|2vXX zBqZoa2}wE|uF2Cz7eg5F=+bQwaJd6})p-B_JTCwM`Sh7$PJ+E3Is?ZBcU|q`?jbJUR^U3m$#NQjp2Ay>>JI@AMs@qIh2Na5n+|eSgfavBFqU7>o2DFv*yb+r?JlTi6DL(nsD?!E3iN5zpWfLdnM*mgHWZecr zv%mXA%(rAQ>7GEs?ZAt(B@*c{i>JoD>ycSGZ)Kqmo$!v^C(^o<`30wQ4ZoJ1kso`q zPh_8_-JsA(+s3+ZQ8-&5H?c36xOii8-Bf%cB`EOiD6Z1L&rC(U(x!ibuTQ){OU|^2 zN`Z6I##t7HeVmpu23SkEPWi_Y2H_4_nfQ<{*)UoAK5i~!K-r)2pa}O@{_^WVnVs6z zFv`&Cqj7#d<4jM`A7e0iu+jn<*hOZ~({cy!cyoX_Jlr9g;h){a9nnA=JxpZEP z+I2Gpvu*dTo5#I1`Y`go7e2HitmJ0m)o7`?%k*=Gee%+?O@%*OulHq0Kq|ICu`xaw zFi3723Ieiu;s_G(8$T)Vu~Bgz0{Y|I5s-itE!kMSN$~=%ONe_S-3&T4bz!c3~4D9#Jd#5BhtI?`Hqq{0pH`&(;1dTq~|-_%TkFgpp( z>RCY(;(;NvdAkENUnru+>cx9=x2^=XZ6X>pD(^b_rmAn-;HgJQ*OMy)XZa7T69^tFqldH z6+S88W$#@0{z-jy^zGi^B^LC3{+TQK1Bq|fK2dk3Dw~kB7Cfe<9fQ*r%LpTYpKL%t zDdMrlBZ?AgIFNw(__QJ}LEOA4K6)alI9||kDpnQO+dQI=Sa$DHSL1o^)6`mqdKaN) zTf@`vd#hno-gxbs*$4Bxt_l`Wl&!wt4Yq)2gL4lPt zU4-{XWY0&$vmaj4_=Rj(rRz-}cZlXNf1^3HD`!^)nrU6RCQZJBE=5b#q~ExXbMSqB zwtOU7&dIsNNc}16_*!Iq+pxhG?`Pod-pstv2fD3qtf?L`31{SUo~L+#gR_sl?Ioa( zHcVcv_8!=_plSxVda?be#9I%x=P-&UX1Og)uO^*R?`sdOHo}9kdm{GMf*?r%004y? zOh$kq^^fLjq5yz&9Tjp{m=;HD+dO;yI9s;-w&_v2ay8EE1F=SY1d)Y&i?b0Vz>U(d z^hZja;>%{Z@DzBxWJLIb(w}hW{Ty{g+U2B$?ct2DH}@W?w;7zelUlRr@$91GDY{Ka z`Kv|U0#vrwxE*OD?@h!Yw=u(#>*?DZX|whvq93d}BJ`f13>}H!(Aej&y*?sZ&`|)> z?1PnR>WU#6sO{>raGn!-O;nS@5efD(bxJaWsU$;qXpPTnD~Rn2D`Xc3(kb;O%^zD_ zFN72}+eYd;6)Sf!%H)ebmQ*DLWcI<^<;#HT$-))rr=M#<#=eYZOCoRJRDC@WMLblCflr*&Xsbf$_w(#gFRgweDZER{wsVZz;o+ywbUFYUmg6@r}VR&)dS10 z9SN~w*B7gu*cjCjK)!j%f#opz>Q2DroqsKdK}}GR_-wSbn1-w8?-TPpMSflj74xZ) zHi_~a(36`U+&1v6zBX(&Pgh>WACy2VbOe!)T*2E22EfH0xcY37MF0aztwVF~1oKH- ziuc})Arr0#oTz2*; ze~P^CiqZcn(l&Mi*1tQlpdy{F%n2%K?D6Y0{#NF;%{TnmM!#zRdSDJ2PTDZW{XKqH25^VY9^YY4)EkK|-yRO(O3u z)ownR6og75Z1l+m9+KS-$jNB(0U42%2BxE*9&lrNm8vp?s}8udLokv-+AJa)tJHl3 zFoxJmOuLeO6{Y)CBbK2uRE~3GoruN#?y)Lpxe4}vwsC9~y(n^2iwmP*jJtyv;_huO zag(7N^{mTd3jVyP1zQ)bD4XuOBG`U&)U6U`n<=M1Yg2eWl}J*Vo_|i(d{$6CZSN91 zurMv=qLo3z&W)T+>t=5`^Cl)Wjx*Oruq!MN@gNz}wPnvW6k6^J{nwW27)P^!pmmdzoH~?LiGbjn7T>zT0;H!t=a4OO9+ zf8fM`^&y(ahFl{|=q4 zRaC`3u+CpOLEyh{BoApS^8W_T#Hn5YrZBTqm1GYCj06DyYEl5eVXyx_Vqztwm^xC& zm~*P?Ko%2*!U1)dHYin@!+4QE0Ki@V0DyhJeXz#F!kOtPJkXEXk5U#sfjm6=2h>0m Ang9R* diff --git a/summary_results.json b/summary_results.json index 18ff45ab6..592c75861 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2361, + "Performance_Result": 9.97085, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.26254, + "Performance_Result": 0.408576, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418547, + "Performance_Result": 0.415051, "Performance_Units": "Samples/s" }, { From bc0c50a7a699b622cd3ee2e95ff7d45294940d2d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 16:03:51 +0000 Subject: [PATCH 306/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10229 -> 10230 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 782665bbf..5aa2d3d39 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 14:13:14,953 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-15 14:13:14,953 submission_checker1.py:3260 INFO] --- -[2025-03-15 14:13:14,953 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 16:03:45,696 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-15 16:03:45,696 submission_checker1.py:3260 INFO] --- +[2025-03-15 16:03:45,696 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 14:13:14,953 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-15 16:03:45,696 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 14:13:14,953 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 16:03:45,697 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 14:13:14,953 submission_checker1.py:3276 INFO] --- -[2025-03-15 14:13:14,953 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 14:13:14,953 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 14:13:14,953 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 14:13:14,953 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 14:13:14,953 submission_checker1.py:3297 INFO] --- -[2025-03-15 14:13:14,953 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 14:13:15,944 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 16:03:45,697 submission_checker1.py:3276 INFO] --- +[2025-03-15 16:03:45,697 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 16:03:45,697 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 16:03:45,697 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 16:03:45,697 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 16:03:45,697 submission_checker1.py:3297 INFO] --- +[2025-03-15 16:03:45,697 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 16:03:46,807 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.6841 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.264518 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.97085 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2486 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2abc897ab..660008048 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -933 +934 diff --git a/summary.csv b/summary.csv index 2cd9151a1..27954c5d5 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.6841,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.264518,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.97085,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2486,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d95699074..c6c4f29d5 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.97085, - "Accuracy":"acc: 74.600", + "Result":10.2486, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 7d09d618f3125d229623dc117c6add7c25e5cab2..94ff425ba322c176c2c1ee5af07395326098beca 100644 GIT binary patch delta 2085 zcmV+=2-^4cPxeo+%>xAItC|&)(E}!bTW_N{6n@`dflwb>X%a3;2#u1Ars=)uEzNGV zPZ0qV&;Vv^)62gf8f)N{f-{v~ANMMM$Hx zSdGlT=cg^-G`I+gD9C6*M&>Kw=GXCG!)MCA^Mnv#NCO2QnTZhPz_NIlkUZdjZCa8- zK8Y#I10iqO%Hky>K~$6S%yMnpv+^J<%*Mii{fP}4$7x88X;|f?5RDN=vOvi2d6Jgg z91o*ZK2!M^jKm}Jn=_a^`ldA=)~^3a$uoa38+;LJrxA zYH8flI;uNnL?jNXOw8!>IZ0Q4iI7HmiW1U{*S|)d%F?=~kq0j$vm-sEkw}zbBT1u( z6t&y1;)3Rn%_C=RprJb6n(Nrk8~d>BbZyV=sn_$Gku}F^yw(vO2O<~`8GSZb?SYqp zdQr~6wPlq;HF~dxRczLJV73`BWFD%RPvg$edQyhhhzCfSzZOu6F7DFRJlNYF zW;&t^V$c!2ZNywhcyEZUC_8wR9gXjRD0M_1MEP69d>3LX$}S#dSL3@NN*&>YD1VEX z>xe@<${rf!x1L^0@XC)62&nf2fq-O>5eS&QCkO;gu2Td8X4e@4@wOr72n4+S3j_j6 ze~Ca2w;@+L#0AxzA`no2-D?B_s(XV#Ky_~s2&nEI0s+;%M<5@F*TW~AryBsZKhhx` zP_+pH0r?&y5RmT)0s;A+A`mc}&NdKaAv#CsU>aTQpkHpDZPxQEI55>6bV04IA?F6+ zfVqMoz>K^56?Qy6|M%`>HodsH+OOLLF8hR4Dw1m14ry-Xnf%RvWe*^)O2XnrTEv9O zPKGQ5;bOt7Wu9`?6!67UuWeg`5mL7wmg+LB>*-|8XN1e$oLj485rnF(Xvr?3G1Uso zs#U%U%EDyvc0=-nl*7b z&4%3W^nLr?s<$P7x;`$tzAid24Ucq42d?r61d0xUK+z!(C^`fJMc3aUI)n~Iw}XD! zhoXZ6Q{90kI>@=vIUws>#40H7UH^Y5y3fez4@CFS-x8hebbasegRRcTRp;xf0}aBF z4(Wh}c7i~l>JSK29Rh)>Lm*&hKJ#~|4xxi=FLux`eW*HrI55>6bYZ6iIX5~7EF`yx zRoE8!pR2lRSrwvc$q#8|;>Gtq7GU?k2=Ft~`vU>a{dWR%?6xZf2;^tI`nqu;FR2hT z-`pi6hzL`6^6NIHRBUckN#2f)#yuP7SW;Ygp0Ou+-NYT%i{$^kRixtk8=Uda*(;QRpQKy+omxDD)CZFNAAo zH^8QyX=xJdkY!L5Pkk@XvTZ4U8cUpl)bv!T+Qrb-&ew7~4=PM7Ly^=U_WOpaP;jI) z$l3nu@XkoprEW`C{Z_KypP~?;UJ_A?6*&i78Bp3Px@+M*w@JiBPoBEeP-o-nC%09as2#Be~s?= zrG6B~51~6E({^#-;s+i8`jGj-LA3(DjAuN5#tO(>Gy-brsL@ux`1)yEpC;-S$i|4_-{J zmzR@kfjH7f^5}9pTP{X7^Lhq!#N_FDzS}qZe&6#PvG;sBel`;^s(QqBJ)5|u-S^~) zWH$tgLX?NVb8%v?V{UvBDz^BxbR|#T>OK434958IKTt~p1QYT-1NoVhco zSRZ8*{epEus}f~NiXu=>PFu5Ux0EP@^9i9Fyy}d1l1q2XCs6H1m~-o6_f8J#Rd5B z)3$n(xh7Wu!IKduIsr42OeZEA6Sy?{B>?~cPyzq|5dZ)H0000000000fPoZ~jVC?< PXp_t*BL*5J00000z>4cO delta 2106 zcmV-A2*vmIPxVi*%>xASUhW{1(E}!b+iv1I6n#Hm5k-AyrGZ?45D+k$(tGKRGN(FE zC?pOE-XoPM}k^cbRHlDxDWdoVE5XUiOX$U!SwocqEJ`G#{Ey(>4u~hcrs^ z<I0?xS4aPODnKuvYLHP^A7S9Y)Iv~ADrsMqt7k~PPwy;cz(1tJ&?7=1EW<$)K0 zdQnc_wPlq;HF~FpRczLJV7eJFWFD%RkE7PWdQ^tjhYDF29< z>4+X4We1J&TSu=Yc;$x(1k`(sKtQrb2n5XDV*~;w*9igvv+ERrc-xRO1OndvIRXKt zzd#_pZOEk#aY1z_2n1Ar_X>f4>RuxdP~95@0;+q9KtOfx5Xd{?_3#Pj=>|aU4|PZj zRBenvK)y!^1mt^+KtR4H2n5Wg(+vb!h|Ul?m`3M2=og!3oAvw>4oq|hZBVN#$hk&1 zV6GqtFyn52haHd3{<}S%PR_3{_p3I6%RXV5h*i00hBULXRQ_gvvIme?1!3_#$z#G~ zCqw3fa53lQB1^bx3i$l7)3hzY2&vi+OLZC6)nvTpGs5L=&aLHY9)zl`Xvi+2Hq{7= zvQfMU%EHzB(QSH7yRqK==ONAGWNCgeBO-z%<>tY_ijmFj?lW?d{Tz{@Re$fqHLc@v z>J7Qs>iYJZRc}jwbX{C@U0rlw8XoG97F^{K2oxOxfuchoP;>|cimtmubO;@aZU_CM z3q=P9Cb|PnbdYncb3oQNh*ePD+wT8Rbf1ya2Sj(@-4dPcw0*Dl&Q|B+s`GW#fd=7F zhqS;#J4PT-bqEBi4uL?`ArP=LpZYsghtNT`=R4>ZK2#ll9GK`1+OSiCoNJu}7Lpsp zDr}4V&sANyD05LZSI2S61B1gX?=1al6-0KpY;)i*B?K)wlWsR(=sK7 zcq!y3J$WvF~NQr48f8Vn+i=nHXujO_gRG3(XBB?*@_YGB{;7DnZ zv;EiMosp_b-IlKUtz^GHMIk`J3R*H)Zm{mjuaTlv{6DrmSv4Iob^N*V-bDDm%zGyz z?s(uoRbnUMQ6(P!YI-l(Z=V`1Cca6Z^wq+!uagf(QvNRc%*y*P+Xv!*`1zIo8r|_r z{V0sbjp^}={V0sb75Rzf-oeJBnh1B~=aD1X5czK>YL<13OZyEIbxcbgJ@vJr>ot29 zO_hlw#uwLfog?U`a8yqH`s zFDKUmaiovr(dBfuT#Rn!^$h5U$;>2FZ-1sC^Z1HXBN}jycd-lB@6aWCT$QfD)1PnoL1d}Er zAAiki!!Qtp?|llRyDclOOA)pMZON&DLQ`6D6VfITwSJJ)I)3{qPU0lsQ;(~e{dRUy zHQ%c?`32{t(KRAjjuOz^2&uO<`dBT}N0j)$v|z2#P@^6^nm2b9x0D+PZ_Ze7L4r>r zNPEg{jdmeeim~TAP|Ro1p`*AlPO%V6w|~W!@o%;TJj?S3tROJK0>cMjYDXeEAq5|i z_Q$mYKyVB#DA2)Uk`a9I25`zxHVn#@Q_0XL3$eISw;fF9$XmCxe&WhJ#h z%`BglX(^b0F>S2l>h($ diff --git a/summary_results.json b/summary_results.json index 592c75861..27c0f4219 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.600", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.97085, + "Performance_Result": 10.2486, "Performance_Units": "Samples/s" }, { From 257a658b991833433658a6c97afa06591ca887cb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 16:32:40 +0000 Subject: [PATCH 307/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 10230 -> 10232 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 5aa2d3d39..2270d19bb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 16:03:45,696 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-15 16:03:45,696 submission_checker1.py:3260 INFO] --- -[2025-03-15 16:03:45,696 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 16:32:35,487 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-15 16:32:35,487 submission_checker1.py:3260 INFO] --- +[2025-03-15 16:32:35,487 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 16:03:45,696 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-15 16:32:35,487 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 16:03:45,697 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 16:32:35,487 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 16:03:45,697 submission_checker1.py:3276 INFO] --- -[2025-03-15 16:03:45,697 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 16:03:45,697 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 16:03:45,697 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 16:03:45,697 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 16:03:45,697 submission_checker1.py:3297 INFO] --- -[2025-03-15 16:03:45,697 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 16:03:46,807 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 16:32:35,487 submission_checker1.py:3276 INFO] --- +[2025-03-15 16:32:35,487 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 16:32:35,487 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 16:32:35,487 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 16:32:35,487 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 16:32:35,487 submission_checker1.py:3297 INFO] --- +[2025-03-15 16:32:35,487 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 16:32:36,805 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,8 +31,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.6841 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.264518 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.6841 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2486 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.3374 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1788 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415051 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0181 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415051 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 660008048..a5356d998 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -934 +935 diff --git a/summary.csv b/summary.csv index 27954c5d5..19d963e4b 100644 --- a/summary.csv +++ b/summary.csv @@ -9,8 +9,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.6841,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.264518,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.6841,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2486,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.3374,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1788,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415051,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0181,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415051,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c6c4f29d5..eae565740 100644 --- a/summary.json +++ b/summary.json @@ -304,7 +304,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.264518, + "Result":0.401752, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1788, + "Result":23.0181, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 94ff425ba322c176c2c1ee5af07395326098beca..ef6a1a2c9be50ee08ce2b35f5c5e19e4d149501c 100644 GIT binary patch delta 3138 zcmZXWc{CJy8^_0TMF^8z1`T6olr5KW?IRh?7%`UY5+NdF$rcl0jGYlPmWW0~WQmY1 zOU5$SzAr5z#!?cKtykx~=e+MZ_xI=Tch2Yge4g_>=lpg}nWirTIsYhDe4qV|6w3KG zerV7wyzg!$ng3lwS*X7Ne>UiH5>#wu5zM716fRFoFm+#|kO=Gz6ASb9UjiD&WHp#S zeEwGPIa17F|K6GNJXfW&U@xY_6{%6-=-4_N;AJ4Vw;Z#8ARiE3ZADrh{P?1CYBBn2 zs>k|4bvl%6NO*Nu?9#4ia;eVlt(MaBc)7z+e^=p$wb%0wg;PMoke%?DYeU{MLnWT7 z?k%)8_4JqEESdlhfzv|ET<)9~+tUu-%^GEDF?T!g1xY=3 z{bwo=Pr=5;orNcoe^Jy4X^yXtjL*t z*d3F16H|!1MO9&p`08}C_U-csyw7NtXm>16VBkDMbQdS|8=kXHm8nb=)O=JQ zLY0r7EHbwYDH<_6mwj@JlWZF+64QKSS2uE#yUpS*ZyOx=t)+mUsOF6KjYk0vve0(T z8Nnz>NSG-!`WYVzQdyXaf|x|)qPzBj6M*t`9|F2-`UnO6xPFg_%A5ZO1)*<>q{r7D z#G<_;JEjk#YbB##m(()Vwxjhh7Gno-XjF?Y5jA#j2KHEOZ)*#-^ykM@BQdWHkU|d| zwo>O!anEX===Dfz;F-{yO~O*Q^8UGCR&sf1-T8fIF<)Cji}u$2^~bFa@^MA3Du|q} zPoF2lbjT|1qf1-1H6s|L{GnfnGKKF*(YETRYBji9b$FZM4WwcVzAR5X&rMd_IU0wQ zr&m%z1sc-1%%Ri_nDN+Dh1+gD#dklk`#QdQj(hogv!8k@p42_IU=TFQx7RTQb3G&I(jW0$qV;x_!m`ELUAO$u<*}1tl<*FGQ60*w|h^G^Wamiy<-Do`8vKVu$(>NTBbQ`x2$xG z%jk>n%3F7VjwVHk&;;%;C*e`kTt*L2kaQ@J2u(*ri9Reus6&Pg|3Z(S#IU@CKxY)n zIs@&}^>AMV)+d;wytk<5a*moQ@oi?FBA6?`fv@w=DmrgCtS9&fs>V0Fj?ohdA*6pz6BXGB^C7f_~T){&y?F}4ci1WXKXqOsAOHd5( zKn&t>@At@ZXX6@nU|$jY#zu#H*3gJ#{sjwq4F3eodAsb>tt>mFm`1%H- zPqnt86?zr7eSNOYUxoL8dulUIW50$ricEoU&6{>5j|edQW@N=z^=3DWoY|)-Aa?uB z`Ik^odxi%9JJxR8IF6IUu(sDo=-T6H`LPxrt7#^g7r{rmHra8sizMO=`hewEvBm*o z@hT_e@ATGm)0hCf* zr62(clqJnJt*}L#dAY><1kOoOs=&T|-zbBXZ!n@O{c#xHQBl;RlxnAnXPaHHvb)+( z&sF}3T&;e)Vc4JUnjvpwok0XaNgfZ3i&&J*VzATh|6*pA!G2Ra0aqb!)oXvIUa`>j zRI%4jrY;Y--=o(t>Sz0;NLviB?3_Y_0|tIu%=G_|G$Ea|X&DF!iU%lh1M3=2fqOIM z#Yea#E#iVO!gLB$g-&_#i%qe-w0bOWdWNEGU*fNn3u<;>$+o=5KDMY?e8R1lei42=_B<+spJ5uYH|tqa0DWED3qV+nyp10@fENty_rQYggp zm8QkDoM50#wiAiw*Vth$A+kuf`tGJ-CR0^e0Qju@<6B;NMY`d*_;{bU^rlkd&85G< zWRz&Ue7+XylaVanumlN)AJ8&pGY0AZ9CXr`2T~F^k{6DeOd`{(OB7{1}xBOmk-Xg97@zEQv<)F-q|qBl~G1!wEJWA=V0D(lSZLz~m>8+vbt z%-h$IH!hY=ZN=Ce-#6`0{&DE=ykhlTrI-C`-Y*C0RXWvRGG6`#^2E8EZy1nmEY;`= zqQOSd3PI`PcmgcVK?vsI^>Y7rf1@?C`6sB({;jSMo36El7G!GFPm5gz*^4y;SFCPr zAnhjSt=wZ96HEGZ&n8RN*B$UD%Wfc1F&GK2_0Fk(k1g#@~?=e53 zV^i<#4OqWG|(O6XqBi<3+(7wo%} zOwxdWIOoayGfzuA?bc~Zz9`^{D&j;{dhkRQm!){3dKuFr)qo95 zpC$yc;)Ey>2y#RrRZqjr9c#|z0|s%}C^hrRxyjZaXvMi{851o{!=Uz>on6%?_O9IC8MZApt&#ElyW`7f@0$a^WrEbNt^yOOVlecX zmw4kH$2R_LhYhEy#eAlO=|1;z3}(NEun>4u@-9RPEHYfc2u9DWYC;DQw)l|NynTbi zh!5b2LwvLTHZQe%BB^8E?`Ri&Q@|wJ4~Tc#d)(UUuf8EZ7~Ro7UYPZrR)GEWIq{e~ z{FnY>g3xpH(l4GM)D={OuKaqM65pol}|Ki;U16r`p7`*=N`pA2+2*aEL5k z2Yun~y+V0g6`Y3=Huy@8TG0d><^`#(>f@9N>vcEvreOMta>Qd2KGW>$5x+b1u0OzQx-yRjrL) z%Xp8|o%56}lGFNS2$*0>clMyszXq&F%u|iggmYp@dp! z=bu%lWaqv0svOj*Uk%x6C(QEZPD#UWP6lDDcP7$wI-PPS_aet_N=M6t3!9db6uHN| z?6a9ooCL43@40>-^8v?!Jo^K)FMAvLLIhdU0_Mzq$~|_f_p)5^Y>gj<(1m8bMKB3@ zg8f^gy}8HSxHEd2@RSDuo3p04Uu?A=Qhlud{no3(?OBpZB)AsKpQItUE^ns@01)DR zDIhS^+bw|f1TIKYk_NG=h>=Xi&XK5ae$n6Ug|PqtI932a0Pw%_AWDdoB8_CZO&XMz K;t+%Ve)$hMxc5H* delta 3175 zcmZvecTf}9zQ+TEy3|mlND%@Ff`W9EZXlr(3nes>-m6kB5Dbb57)kZW8QpZK<-!r(Vp@uQEX+*+z4%tuP=l*1P}dUuBn=-Y!#?v}`2Ld3P54TfEz~tAA&WdAk@|i16|U_XeHJ z>8_1S!Z`=tH1xi0i(8@O)R(<`%}{G3SVL20!*Lpv48? zmI?G#)$((8jQOtjF@?p}?waJ8j&2miFHwp~MCMP5bws^`tp-b#+EYTR;-s=E7)Un5DV(r>W2)IzNpddMOgQ3#n9fj_6q4#j3`T zD>m|x_Z6SQ80%K`*q*68H(^{?&T8(|+`P_(2cY>D^9c8Sx;$Vd{_AvROu*ZlV+OZ9ov05yI)9diW z_RO9u>{ip`1Wj&XuH#hH^m4V#JHxTC@uPeMtaHPGkr93J)59$JLb9A8Hh9k4G;ZbR zBtbHfYI(YYjwAEPq*R%!Cm8rrc`Nr z2E8WreTvP5XEjvWy_sWaB>!YUV&YwpTTn|+m1{r6AnWF0)UsQ-om+4Nl98@*3{*6c z&Ot0=VK?BX)g^6+Il%vXr?!#BK6{CmX}-U7|FxcVC~sR2^Pn#eK@cCE_s@mi3>cc` z$vKbC3~TRPP4-jkOmeB9OOZ{6x*%TTQZv)b?KgFq3BOd-fuN2^D{hg7=}RI79y83u z0^;A1AVV1xNlmPGN(QI8bc5+WsxCSu=Z&Y(1_Aa^J1nH zpEp4<+d)h*Ae29>@nOEs5{wij^RSVhn`kd>Oc~#r9ich046_YAcDcevkp*t=V4g6( zE;xSbjJf;6%RuF2Be3M@%7Z$+V?AG$H!Ad5yfHa}K+KRVKHRp^7^SFALUsXidSY)dT@rJ!sZcbJkTz()*x zn#Cwv*_B={Y)0l?$ySf6@4qonN1)WP{_MPShJ@FxpdW;WsjQLL=01`VOLm-R_sZ6+ zH2TCUAbl`C!WnNAK∓C@}XS08RW%F@G(5Wg|6J*j`1F^L_do`T5wia9oRn3&uRQ zyXBukM78sO)hw8bCf|%TP|31+(}1px=$PIEld2+Lu*+l;b zpM{#1L4}%iCv569Rj}2VHK%!kPE!P1P`8bUw4+gHg?_X`TZD%#l2)-(D z`7dW4FmY-r>dl~%)%?V+@bg5nQE_xj=p6E%lZ)#VuES9j|Mnv;m3Z-(ddYavO3=;z5xWX@~=xnUY9XN&Db-Zy$ z`C^-)VY`b{SaFA4y#B3H#h#pcnbOVjy2k<8{h;?UWR{CKjw;oyy+)vkZ%zl?a{Hrl zbF(D`q4+EgAyV&w!*e<{h#tlLOn{+tHUjfrVjo;*%g3Z28)4^%nIZ-^hleZZ zI>&nV)}Te(y|?LeZb314J?ipxt>WfGLB5F77iUe*anr!h-8koxOG?|1Duk~>Wa_(DBpD{uUkcP+C zqhGYG87&31F74k(56LbE6{RYn;OCc^7bNWuldv44s>x>p zQ+5Q{C9ni!UcOL!y*tI$Z~uV{r(Ai$st~;+aYQRABo(>3cL`WKa zItBYYa@F0XDT5kmvhJ^D7q4iXIC+~Y?&e{lCRV+btq=4Ip3#RMfba^LT1SF-wU0)y zREK=^BXvF#%l8T=Ak;Zhk~0M0n?r-A(*7yM7e%m&I2uv^gFs8VqwM->X7P$D(bS?m zfvP~W3Q}po%*SZ43NGV^#$PLc!i z)(0fjscz_A^TBpy`BC;_&`HAXdJktj-BWKRi}KVk22!JoHdOjX8^NBqbzZ0TeZCOr z*O!pG{SK^7*x3HsC^r3ZEF7xR>bJAOQPT#O+zvQz@ony?nz_20mtn~aklgdU%-Qk` zf8qvI^W5>!J5oy+s`2DK?$^_zQa36B?+th~34}PqIk(r<_s8*R-&{g$Fs7&~m)Ky7 z<^MFn1->7>@}v#x|8ur!c1CN=##)N}p|y;j$A}3SRgrKT!C! z))5gZuY$f7W+v3i7TQ<+z|0-hE=&arwa6fXihd-`!wZ4GFGs~}6#k}J< zDaW@h>XGGcy z7ViFQg^Chy;tZXa{G<}*`;{LVOFU~Se%F^2b62A^w4gaYobN#M$8SmTD^B=F%XaSo zx7qz4x;*!os{UibT+i)#(U&yg4dD0QnsF!4Gkgmd zB3o-I5~7yTQC)}27N2)KQ=-mqg)=17=iVFxKgm*p6#7~J=kKqtT>FNg?H3nIy?d7g+oFG}=<2uhvRtH=TXIIsc$LV*8UDGx-5 T*$^X^0OBx2fkRU9?Bu@yK;`e9 diff --git a/summary_results.json b/summary_results.json index 27c0f4219..83598582d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -365,7 +365,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -401,7 +401,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.264518, + "Performance_Result": 0.401752, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1788, + "Performance_Result": 23.0181, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 198398c2ccaf13c3a6f4e62219dd32562ec10c56 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 16:47:43 +0000 Subject: [PATCH 308/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 10232 -> 10230 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 2270d19bb..6c7b06670 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 16:32:35,487 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-15 16:32:35,487 submission_checker1.py:3260 INFO] --- -[2025-03-15 16:32:35,487 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 16:47:38,874 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-15 16:47:38,874 submission_checker1.py:3260 INFO] --- +[2025-03-15 16:47:38,874 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 16:32:35,487 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-15 16:47:38,874 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 16:32:35,487 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 16:47:38,874 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 16:32:35,487 submission_checker1.py:3276 INFO] --- -[2025-03-15 16:32:35,487 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 16:32:35,487 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 16:32:35,487 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 16:32:35,487 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 16:32:35,487 submission_checker1.py:3297 INFO] --- -[2025-03-15 16:32:35,487 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 16:32:36,805 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 16:47:38,874 submission_checker1.py:3276 INFO] --- +[2025-03-15 16:47:38,874 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 16:47:38,874 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 16:47:38,874 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 16:47:38,874 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 16:47:38,874 submission_checker1.py:3297 INFO] --- +[2025-03-15 16:47:38,874 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 16:47:39,870 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.6841 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2486 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.79935 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.3374 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0181 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415051 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0181 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.40996 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index a5356d998..10c7f0b94 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -935 +936 diff --git a/summary.csv b/summary.csv index 19d963e4b..727d95de0 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.6841,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2486,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.79935,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.3374,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0181,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415051,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0181,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.40996,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index eae565740..773d70be9 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2486, - "Accuracy":"acc: 75.200", + "Result":9.79935, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -634,12 +634,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.415051, + "Result":0.40996, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ef6a1a2c9be50ee08ce2b35f5c5e19e4d149501c..82f0e833615b3cae9a185a15b62e9b4211905452 100644 GIT binary patch delta 4475 zcmZ9QcQ9OE|HpS%2?>@}lB}}IDnXQuAfm>KUKT+RBDyGFEd}dp!72$>w}RCYCCWFmFP}JGyd*7aVU5xM)MYMa;USZnxWMk-9vLd_D|C?I$+M8ku-J?0tRtAZ6!^4ykiJQ&7wv8h76pLz|Pg7|H@C$xuXHMy1mUXdvVqF|Oj=J*` zygJ;2G?-%+>4_&}G1c*!@)7$LvLBQKQ0?Y}k zED$hh_d;k$r+=7g^QwXd@4D;)Luklb;Hj!AS&~`Hi__<&I+xKDJp?=gG~VFNt@}bp zZMsVv8nXP2pV?^YI`!t$M|Xl+YU{v82kP8IkXxLmSjs7O?+>4HI*&y5qf+A_UWs5T z8r);rZ(kgDQfi8zVx3as{JebD=M@W6x)>d#=iS7#on}7i#PeA(eru)ss6>CJ+R)D~O)P3`q--4xV@tMJ}hh_Zsx(d{Sqg{7Cmc@9s zgI^I=XRes$>uL-~H7@GDpDv^!p(ddt3SYIwsas%A#? znQHD2W;!-*NGMUA*T`>*o2dT7$nW}(^LIoPi#05#7pE@hITi&7|{;rJ>I2`xEA`O{ICFolL>?|q)gQ;1-y zR+?68+Q?l7W}49t2O_g!3)U-Kl?ju)W_DH!*jLM48X351K9zzLoO3_cHe{`dWyZN+ z7Yy!ekb|sVt#7nB=NYh=IW- zd1=UmCS0aJxsAZQJNg-Jf5~1=<}#~Jo34Zknd}QlOec1Y@Xq?0=G;tZh`!AdoA$L{ zf%_6GkX-#w)7g@qxg5c%>2o`i$2K!OL41ASi?ED6T2WGIgJ}nimPNkWx+FQpR(w7V zA8`|fDZTWigs43I;uTGIsn~d#WID6>)MWV!#L$hY&Wed>>8e_c;A16O_z}cNmx`JV z1Oh?nPCFG)_0i7S##s;uj>poO0au_GCoKaXdgb*`8B4Cre5oJ*{BFC|1=a(-U*b8+ z0DObe;`e@3KJef3>@UBFnaqL)4BuTxjJ`j>tbNHp;R-s}qC%{qblgn3%fmhT(Ys;H zONMwnma|O^sN7iUOtfR}3-I2wmv=+Q4p11))JU%CSw&On+d^Ak6<5o z)3WOe?{9&l+4-4Dr=lTRm+A6IZw@#WFlJ>izcn*)20+0(I$cRa3BgkFW+!lxwvU8= zZ~FE+)BWDepB|{_4%d73fZ|1l!(ZqxamT2b@NusdbvOK7?vhpgGSrCd*V58d*ZX2e z%I^)2^!4ngR@@@I0=y|dcp2VboGpoR8PsjM_`1U1$8z6wC9UQfx0U_yKf%*?k^TkF zl${rk2j1zkda<2HEf~vMcSf%0sS8}}e_)ge)oH!ZGbK&PCpzNnU;wUBtQHA#Huup$ zuD@Ey)M_YqD4rCOd`nNCEOMUx9^w2mQ3=E?Yo(HR62_~hU*}TZ@Y=-cU}kYTr*D-@ z-raa|XdLUaYU(zq^X&4WLHpIiOs%%v;2X%fh(&cJiCYzel5^SRpTF^Q8o(1!__bLxm)UvxGnJ(u1 zn8C-9@5K?kmGjXVkVH~cDD;`~VermQ!&X(`zR&9mF-ao>fD7zdn(fNb-O+S?JmB1^ zBfM9$(K_4fp!}=Vp>c7#EtqW@-CaS_PL{wDv#d%zW#iY zQ5_wQ@?aEDGjihd(Z7l(%~Dc&`SSNS)@de7|15JA7+Bx0z|2L99Z?IaawlpnN2853 zk#lFU6gDHxmUcA@2QsF$oxC>0kx+FAAKpOZqBpgmFq%pc6;GYyicCb$;}j`x;UAH@ym{g} z1K)Ti@6li89>Sr%E7IBBfeGa(0KrZ#zqjU*^uYa8eNnfiX?$l6Vi&>~1Wp#sMy4F@ z9oi|1y{x3pNy`c?2fto<}vdy=gf z+mnzW@pG`6l*!xjPg0r4=#0%lM1;R>}10S8B#oKMt-fih_5zfOch z9lB2G5eb)C>yQl3ORGQolrD&yG&h~^5p@&adf|Via?PV~q-M3@8lHmZ5t|II;nEfw zC;ACyyih~oXg?w0ysW_0cgY?c|I0*YSSFoPWf%sK{Ddo6Cn7Qu}944^#w8p%;qnDnwr*Xb!#Bd@A?^OWVSuPx-3phpD%@kBEwxw8MS zBJ~3`-jZ%2NNo<9}LbYo`$A&u9 zWnR^6GBcuXxAe5iG$RM)lREiTpt>MEj%xwQGLwWGx>0_RgEZz5K@7Dvz~5rk9n*XL zy4Ep!R(cAXMNC&&v8LQgk(WdPGWsXQa{Vr)I! z^f*ZO!aa=e?!!A}gr?yUg){noPsfJV|D1xyA$rd{H!RX3!C5^NDw0!sa1O$8OFJtqh3&T6Xb(KtkMxS* z)E8gmxtNRVW3k%*qZs^G@%R5LQvNCiBN&7KRovyXlfRFuhxTN$PhgLEP&qHBmu@&8 zDU-{{7Ea%QMklc|`oB&MH%<*@4*wc%{A(C_YB(T{I&_gbRQlXSV<3jLx_WXbYxeT`t`1oK&t%E+1Rlx{&%)O3+w|oaiq2I>qvXQ@L_y zHfd>@@HX7#dX&~?7dQDAAKXv@8%>K7pZ#SA=S_+`6D{tuUuh8XiNE+6Z87w3Db*ko zXMW|`sUm&%L5cmHWV~U)E|Nc|v|rO1tK3=AFixFgD6qBM3S=Oj9!b}Y|)XOIILcxB7yT3nGJGwg6QyXUja`E&OP zV6jEv=#)$5u)0F~uywmzAYMbBzD4^GI~BOU)AhaPq0Bj2N%L^aK4C!OC{1!-a`osu z_FzOZGT4#J0Ip5m9V)21B}qObTb3WL;k5uPKgsV@L=_!NUHV*h{i=X<(^H@AYfK!a zhMkm}3N@}Xq=TG~!Co^_WW!F64^nh<2Z$k)YHaz^AOI%Cv!;V-R0wqfk4U2I`4HL| z<12zD^rX+{2f=&_chqNU<*U9(%PG^dTLoFwB?)_NKgW8}+6Vnc{P{`7Q-t88*^BCQ zH06!yCqnp3{5(LfL8}yV1VvF}A#koBkERTn0@_9kBEwWC#RDfS+f=F^Q zaPt99P1$#9E!G`bkHY+Kl)@afWB85qeDmH222_~;REi-K#|+;T6Bz_^S(?ewT#0Z! zS7F9kX%i9)(Yn^6hkNjiGpx$~dctSH4|!ql2F;8SI`u$%WVKfvjiT-_P3p|0;^%V$ zQnCUpj9Ps+3-+q?&&)8WQ^ArS%Il(nb^L1fP!ci+$Kg|i_&NWELJb0mu!BH-^a&N#?C}pVk|SrmM^0+7-TSGWZ#!8*|U@s5n~@?$(qJeipUnC2#uYL zWwP%}RK(a7W6Q($IluEf=lt#;_ug~wx$pb>yg%pj$Gy7-RD;SsI{Ky3Ebg=cIx(z` zZ~hIg&^h~XL$I|6NEQuDOid_UCN9hL`Fu3sq`A-!aP=DMX#(@7>OXjr#k+seI|t6hbNpg=tbAWQP)|K9f4<>JL{zvd@#yF1nMgKu|d-R1_R7KS7fe zumRq&4*TlzASMurmJrkJg=m)(Q)g((bIMXNvQiYkczzz`6l4(LVdo9GX1NyrJUqyV-2Bny*-gcuiWoDm(IAH!UO8vM5`w{Z za(|F<%vdY~Dzk~d@=fja=UCbG!4Z}5DM43w9B4L7a|;n*qOWZv?alj>go@D zIe|s%0<<(EL>(b;>{y$4&Rc%KN3}s=0mF}sv^4vIq8!N^XOovA9CF#@YaGtPaGhTc ze0Hm^?0sH(kM2m_kQ)*3*LYVhymWp(tS&;RIicZ@`;iLbWN+R18u3%zWW^M{>|vCd zWUA)`n}^TO%%eW63%gK-ZtH@Ub$b*^KSr8<-l?V3<)z;U*Jg)Fg||%vr*SPfV@1A< zvh3XF7S%J8V^iwP8CCMmog|z0&j?dbiCH64cdPT(j~ZgljP=If_~eWC>*bpWoSYh2 z_jerJJGvGho$xA=LZ5c^uQahQWGUWP>Wi6P{~dBTQ&J1X)+1!=8)e~gtV#YThVpG7 zeH>f3`hYNp1e~5$Yiv#!9>u0r8yPx*%ZsElnF$wzchwCj2~lj?B_%fOPOuX7U%Qss zk$e4fpciZYAsd^K@5o=wmMA53tgR_*%VE)vBGg~4Z~#pOXv0f1g&f=vI{`&(uOq## zKI*?3QUu5T>zU(QdZp`d25Crbtl9X*{oK-ePb=;`PZGO_%Ru@Q{v% zd>laQzgi=84*0{3c$h(Ev`Jwk$@@f^UMzELOuV+ls^BK)+t`6^8Ts;rv%u4ZxoYu; z`kRjKZ)#bhOullD{k>hJ6Wq+SS+=aZ4T83fvE2c0{1g*^L4EW^6H*J(3l=_ZmFw%2 zNb@YgCMzOK;(qIB?e4Xz`c_^FVz6h~Yv$pV-(fX%wFoXwVot8?Y?HT4p@h8~jNs?R zyW~BxDK{%qnk%X}Cqt2%D7#wB0h^nKLSlg?0o#;og{EFUrs&zHiJfXRh8Q|}E&u?) z4m|DD0Qqv?o^LS$0Q_koKyz#wj=pwucE;w{gW6#!IQ5UuHASCexh(pfwyctD#kSnN z$QEY=yv)w7aq7WcZLYn=$XOWS5chE-)b#MrFExg_h|MJT)x-BG5P}ZwX;MV*$m#z7^13BcUy83I`8{UmcibD_3}sNI=xbJFm?*A z#D+HOYS$;(EDQIu#(A&?=jqf`2Nd(05*8eGrw)=QA(lAF;0vXC9}{<1N?axS)dJ~@k%MyQ9|W*06}ekE5s+A?4P9A$`@}O_PJDw@ zofybhYT`;QsmJ>!Ewo!JnrEpaKdwiI+agJ_Jf2JWSxsPpOzc>mv1wr5pbq!*(^W&^ zrs2HH%FnDn4BlaGGP%#%BnH|d=5pX~IQqsS0f!kV>xR?-q)1?p0VLurJ5r?lRT5G} zKO`H~z84S&!pe|+aH#f)V=cJnA$RI| zMBQ?}N*EQY|5$bW)|SuSluK0Ic(VGk(N-oV&sh`3QPqL%mjVVxVV!?C<#c$=Z{)@2s?}a`m&c6hz^~JGSdx zpl>j3y2(Vpus~NqQNprQg8WbGboU>sE14EDKGnC?Vs{IPW--cReti-^ORN*z zS)Iy+*?e5bJI-g=ux?%W3_*ld8c?{`F8s352Yr1*4u2+w6g)muGDvnLI(n@r3CHFNC zFD^frdAOQ$i0ZhW`;>cq$+8A=LGh_53CWs@hN+7U05Xd-n9TOsy_i{@hV;OIcp0e3JEbx~W-n>svkKY=wQq^83~`ot_luR2f~1 zR6HLf%{@#nk4D}I8;O~4{V&b*V(6J_8{n2mt5WrT(oGXp4@DcT#Dc|M*GJ?Jl$xn- z(XGjeWo>*?wog?1mP1p;~W;sH2*I|hfDN% z_0mku&SW9LP|&DDoN5Z#ttxI?_a6^NF}ohpgrJdDB|{D)O67t~!!ZcQD7^Uv!oi~=Pn!&r(SA|4-=P0X?aK$6ex(@6K~ zT8xEw=g8(eE^s5sit=2*PlZ2QIi+POIwJxj-CpAB@^yE9`~xN+`D0~Xs35=VO0f?J zr9pdD^yu_Jnu##ZA`vxq3OzyE@3(}F!8{E~3{NoOd=X@k?^RXTR2!T|GJaZB zH_KQ5bwQgpqiF;lhn!z|HXeBIkM;Ycr3&13W__^ia5?d90n@H5@v`KAk`pG9ec;pJ zye16g!kEN9gatf9en&~hdm*H046U4h8-4p7p24;pWy#dMrrFwW+`Ni#xKcE}5o!7N zi9w6PpCj9nvgI%3cWst)PHYQqkqflPVrAYTPGc@*9{@bp6RmURBSD8yvJX>6aL{Di zi%|Ewl?P|`Mkqh$7%g!0YjhT|Y+s2ZB9g-QO?I)eQdg>aZ<@QUA*{z{%w41F;tRVq zE+mT9d^qGtlv+a|Bi%D959L^2p(vDO|KJ_$rK%Yn(I|kr>kv9~^6Z)z^6E5r?F^`LyFC!fx z!)LKP>>movhxe+j?*hfpc>U%;*WmMyxoDxxw8 zmGtC=^~%S9?cUEX{Ud*3aYqCC=UId?A;c8(SSgz@*&Pl#wGVb`pLpM?eTErwr}n9% zJ4I_T{wWi^EU zK=JZ2XozKqI}Zn`459wxLDj*J3(Knbt|iak16KJ4Rg%8fT7C_6ea0T%Tf~W;wCC!J z=7S+`?+Vm;+cj})+pgJH%)Our85}S#Mxqa@aI^l$gxlmOJ^vNEI(W`zxG z%sJ3L3YiCw9yMrfvleuWCbZ1lJKjaz;na_~2g2I#y=-jsQ(6<~i)iT?d6lt4%Eg@g zjQ`6Ve4;fM`X~Oiy#X&y+xUP8EA|<|9E7t{emM;A@8<^1yUQmf6szemDN^+NfMXl^ zwjoa=J624#mDlcad}&$z`}X}DKh^QT#Lq^KC4%}F_dDHdj`H}oeRaYG0?j$<)=*7Q z^%d@XT87E`K>au8S*r&x4bTY2-Au)4ZnTY*mSWo8hm`qN^ZUoDzj~hPR>-H54kR7TKi3qm9<3S?+_7#$#MHr0r4#gVc(6q~J5||L#~#Q< zoBUwfXkYrJeGT-k>5@ld?2rig;*=LUE{CgUW2h_pFS8+==DOI^um|f?28Jhe)$XEE zwEz3Xu?fA&_eN6>ht^@)p~zYIOw F`X3#~YHt7l diff --git a/summary_results.json b/summary_results.json index 83598582d..55fa0d52a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2486, + "Performance_Result": 9.79935, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.415051, + "Performance_Result": 0.40996, "Performance_Units": "Samples/s" }, { From 38f3bd70fdac148c028f9b1e686861c8590d1a9e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 16:50:41 +0000 Subject: [PATCH 309/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10230 -> 10227 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 6c7b06670..4a251140c 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 16:47:38,874 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-15 16:47:38,874 submission_checker1.py:3260 INFO] --- -[2025-03-15 16:47:38,874 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 16:50:36,089 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-15 16:50:36,089 submission_checker1.py:3260 INFO] --- +[2025-03-15 16:50:36,089 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 16:47:38,874 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-15 16:50:36,089 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 16:47:38,874 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 16:50:36,089 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 16:47:38,874 submission_checker1.py:3276 INFO] --- -[2025-03-15 16:47:38,874 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 16:47:38,874 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 16:47:38,874 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 16:47:38,874 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 16:47:38,874 submission_checker1.py:3297 INFO] --- -[2025-03-15 16:47:38,874 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 16:47:39,870 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 16:50:36,089 submission_checker1.py:3276 INFO] --- +[2025-03-15 16:50:36,089 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 16:50:36,089 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 16:50:36,089 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 16:50:36,089 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 16:50:36,089 submission_checker1.py:3297 INFO] --- +[2025-03-15 16:50:36,089 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 16:50:37,055 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.408576 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39636 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71474 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -31,8 +31,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.6841 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.79935 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -40,7 +40,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.3374 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0181 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.40996 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 10c7f0b94..df43bf46e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -936 +937 diff --git a/summary.csv b/summary.csv index 727d95de0..36ef3d347 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.408576,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39636,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71474,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -9,8 +9,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.6841,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.79935,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -18,7 +18,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.3374,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0181,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.40996,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 773d70be9..eae205856 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.408576, + "Result":0.39636, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.6841, + "Result":21.0461, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -579,7 +579,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.3374, + "Result":21.0076, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 82f0e833615b3cae9a185a15b62e9b4211905452..9f46673265db1901fae83621318c44a5e92bb8c4 100644 GIT binary patch delta 1790 zcmZXVYg7_u1IIy8TW5+ArRegOKn+|(G{p>L-VHBhnx?jBh-kDdQFCgrXrcLfgDJ`K zMp`;G5uAIyq^8Tec-P@&<;0m?6cf`h-LUt=KD^J@=bYd9pWpxgoadeIs;`*2>ys;e znUJO3002yyysJ-#VEw8!8d=1G?RbxaRr&nS-!8!x4>5k|B3R;1v4^ z-l%;<-8Q(j@+0h0gp9?=>*;1hz2@c3IiI4V#XGD}e_pDo4WG{5VV>e-vMQ~yo?4bt zOHyK@@_Vv;*ebvEi(M|U!BBShJrdx-TICaHqBT)Q-8-Liad(;(03H3}V>4 zl#-Dpa=_w@XHn=4@4wpSZ+nrbNykIH1&fa5O$|4+_yeOIF^?n!^MLp>DCtlC?NwxC zb|zhbgjv?%-WsN(@*Q0`P{C-@K` z$93F17f^=At{+@{jQ=6mU=dUf0)tcAo}#XLMus{uMYk9|tE8B+c`Q1h$C?j53Gmw|Pl9eF6_q+WSl!lAsqlR}YD$N=^w3C^mFaU{q>X@lpq^^l zUHS!9Ee7#=MWD<2ZWi>#ET8CSkhTUK*+)PIT?z`T8Cf`jVfDzu;jXOQ;Lewoa`#sa zQ$1)*{LmRZU3B@kT$+XGYN?z{H|&SdEFJ)Hsk(z`?4}sRre(|BIdZ*o4kdeTBJL)8 zn4qb_Q9)fV?>r{3zbGFo5ux)n$@==q?@gk!8J1r?((GJKQtk8g~y~fEM_WE z$SZ{|b}@r4b1AlO2rhG8jo=|mA^bIgtWAgL7X?$-DW3rPts66S8^N`jcLeelZ)Rcc z9m|{2qxIjJ`Cq~%O~SZB<~S35bosoLmBT@tb7w5B@AQ(KqC141UDu41PuFl-G_}z8 zy-&t%B=6J$JPX{}d{%JB(tT9jrlZ?me@CIpN}5mm?XJj-I&?S!>>q;n+p!mDAzCRJ z)7a~Q1ZwOZ?7pseMtWsk0e%H4LQB+ivD)b*mnrJYErGC_2)IwhRGLLlPaxx6!l%90 z??TE8R33G_Nw2C4QlCPFOdb;}>m}9ZJKH48WiZu->BxMINtfJcHAdgivqRfuA3iw< z=|s&Qap#PDY$p61&m&qrh_U41li6Jh3RO39I1!RPFt%`CN+@IoeCzPGK$4xXI@{rxJww!OluaF8<#(jExRRH+$qWVPU3YCBr5ht!>)sEPC z2qPui$|U?+`o?A(vGRR<3mqGc5jJmZ9`OZm_O?-G-^Cw zX!PirTfO=-y|l51iulP1%M6Zu%lDfWJ7v4-p`z}B320>tym(07Jhu9xMf#sTIR-lB zlGtuIA7O3I>Wym07`m=+?Y{rmmMA{d7FT2&e9VvTbu#`dp}lBa+p~(_QvIimkJkBc zd_%l!8g}MzoFsF*)uA|mBe8KcI{2X8FN{MN@7*wRmH9u7tlo>GgAHsaxk6Z5?GJVp z`ay?VU4bQ^Q67*&mi3?fUSsR=lK0$?G)+g9gG=AA#Ko_(yW* zF)ML)@3?9uG8m#qLE=XS-es2YZs!7sl?ErgE@A>DMl9mDTC7LS1X32%y1co2?O@hz zdpzo9wp|7IaHf?MjjZ@j;;|?T)yG2>AiGZ|A!ufxRLz9^_H9(1cB2A4{|5I7gRv-^ z=RfBBUr2McdvcHud0Y)1(2{s9nH-D%J|#!`yQ^~(ekC?T4uErf8-yp_8eyX~7;LNE z({9-pYpYIk>(swmeNU@AZ2Mf%$pR%^QQGQa@VVeYZDWjeZ}I!JOz)m!TQ?E=Renk36^xGl)oL7`<~zq9c|~1TVid9 zlKxO0D9=jd3hZ8t2c)cJV5+ie`;&r~ag}_PoSiB1xO?M^mkD+MPNSFtX1OD8r^3sl z7R}D^1Z(+?zS6lt%FWeXhheO!!4q2&`;*BaTc850o$iCv1)F=$37l`N#~sw4Xn4H) zi)HT{=k7_|wDvC(O9@x@?-P3Tp9p;(*U!Id_`#dc=wfnF^DdR-7x@l|irdw^%nZT7NKav!2n#8#5NSkI0(fE*-YA zDJjnJ{}Y#wh*X3W^)KR>ML&f)$ zeHBdUqYv{J1NU(JdcFtg_vZjfN8En>K)7-s#0rmuD}M=-4u!fHVGvqqe)wwIASnwi z+N!LK50~A)Y~9Nk{yOIFgr`)ODflABTzokyHEm&KRv%K%JjYqh;v1o(s=1oaaCRb2 zwAFzNiR0G3h;hk-o;OCPHx7HYDmsG)Gz%#~-`rqN;+k@q6N`dx@>kKCE@u3kV9h6a zUOglxj$Oi3w=cZB1Dw66$h0<;?8OTyQ!zW6gMYN%!JZJ)b5HQ%XaQ&U3iNjL zzU=*rvOA`iL?tT2?eY9AZdJ!sJ-)a>ePx1m=p3-{pptEhso6)Lep_#HPQ#}wD{R9^ z8dm4XUT$r)P!?zHmcFEhPABsn*|&Po5;MC9u4@a0IX?=3S!wcl(g5>$;a3zG1ag9a zK(ZhZC?M3|DJ=BIa6SLfFyh&}w}MM(uO0f(GKtHk)^jm4Rpp}f$UM-7l_oOYd{Q^& zz1NEiEmGtUD(CS$x53oeXi`_nwWl6SMESk=$cuGyM0}dMlU;aun`U&Q`>I(Qqc)9q zNk@AWtnB4>UQ|C3AfIS-Q)=>!O^4VR_1dLI{jL<>>~E6&R1v?Yi|%S_`vlzfJT$e zQY&MYUjzleHy{f*@U_4{!}r$4neY>lrivn06dAIlxaeCpb&%_@k3UL9QcpQNlkG-8 zElTey!T7tem(@W0*`ITN67rw=mfy_he7gPc)w^dL*xY8;9Q69D9QVDzGUX&;@m|40A;{nV72Oc!4$h!wqf8+%MLJp$Mkk_V=<6srxSI~uG6rMlwpu8qe};dM Z|Ivw!lc%R^;lM%kK`jk2xaQ&I{{YXtR)7Ei diff --git a/summary_results.json b/summary_results.json index 55fa0d52a..69e390e8c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -365,7 +365,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.6841, + "Performance_Result": 21.0461, "Performance_Units": "Samples/s" }, { @@ -401,7 +401,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.408576, + "Performance_Result": 0.39636, "Performance_Units": "Samples/s" }, { @@ -860,7 +860,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.3374, + "Performance_Result": 21.0076, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 3cf7f4d5f790e75d39651e493c8340db765dca3c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 18:48:26 +0000 Subject: [PATCH 310/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 10227 -> 10230 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 4a251140c..eb5e4beef 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 16:50:36,089 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-15 16:50:36,089 submission_checker1.py:3260 INFO] --- -[2025-03-15 16:50:36,089 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 18:48:21,942 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-15 18:48:21,942 submission_checker1.py:3260 INFO] --- +[2025-03-15 18:48:21,942 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 16:50:36,089 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-15 18:48:21,942 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 16:50:36,089 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 18:48:21,942 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 16:50:36,089 submission_checker1.py:3276 INFO] --- -[2025-03-15 16:50:36,089 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 16:50:36,089 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 16:50:36,089 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 16:50:36,089 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 16:50:36,089 submission_checker1.py:3297 INFO] --- -[2025-03-15 16:50:36,089 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 16:50:37,055 module.py:5248 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 18:48:21,942 submission_checker1.py:3276 INFO] --- +[2025-03-15 18:48:21,942 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 18:48:21,942 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 18:48:21,943 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 18:48:21,943 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 18:48:21,943 submission_checker1.py:3297 INFO] --- +[2025-03-15 18:48:21,943 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 18:48:22,961 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39636 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71474 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75007 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418488 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26127 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,15 +33,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.79935 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1046 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437507 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.441243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0181 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.40996 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8814 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.40996 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index df43bf46e..b4f71605c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -937 +938 diff --git a/summary.csv b/summary.csv index 36ef3d347..ef7a8b7c2 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39636,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71474,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75007,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418488,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26127,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,15 +11,15 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.79935,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1046,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437507,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.441243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0181,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.40996,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8814,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.40996,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index eae205856..5e992724f 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71474, + "Result":1.75007, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.79935, - "Accuracy":"acc: 75.000", + "Result":10.1046, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437507, + "Result":0.441243, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0181, + "Result":22.8814, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 9f46673265db1901fae83621318c44a5e92bb8c4..fc7b2e92c79914bbeff2b22708cbcc693e4c5a31 100644 GIT binary patch delta 4984 zcmZWtXHXMNw@yNUfKn3S6;Nsd;T7o}>4XxRfPge<0@90uU;slC2%Uu9q$pL2h9X^% zA|+G-5di^d(xi9%(7E58xpVi&?#}G&v*&rvo}F`c_wK>nDiRs>Fmhqp6AS=gV1(xo zX2LHrO02m@ffU2@S*u_@b~7iK02&gDk11SjUgamb*W85yuMGhlTu1v_*>B)=p9Ewu zZ)m*^M1KgFt)Cl|_dD9>9*jY@`)Nieu2#E^a;Rz)RGY;1w=Ng)&x*Z=MDc!dR8m{w zn_HULw)*&}w`Dhs-bRCroIM>!zxd&uUs{6fcdVNkURfUZZyy<_$7Jt3H(be(u+#t8 zZHlDM!S%D0tjU`7&WX1losCWieB~I|D{Yj8RhKsZB(Z8jsproTdLi^vMY_g zkHsaZVWMCoF%zw7IlUtdg^84ca3WmpRiZ|c4IEt%$xt&@H6OARPl7*k%gvLS$yoa` zaO1RU!`p3eqhw+?BT>BDxoI5AM#;rQ#~dhOvJspbjL{$XP-CC0ERLxRH*y`vSl2QO z*|C+T-CY71$gZll?Nq<{*=Q#dZkV?ABqS?&lF%&U?o% zi8XdBh}Q?jjyg8)2)J&`*DJtge}AEoLz8tblouPFwic-(+snqXSeIj|^{Y3Y9#VV= z4i2)5heFbtkr1$U#XT^{VS*X7UCX%)0XLmr0fRKLI7Qkz&igVQ91I&Si?Zp!yJw4>66Wj8A3de;;nzNi@H&|5`Z( z7B~$vx`lV1q;+y%Og|%G?ddowHQVP^%{sz*bi;3h=WN@`>~kKEG5qShSLtRR0yU^( zPx&eGW^L7u^eC)vI?KGw4jNWk?${%cFFZ!wS8CKk-P>-r{iC+IcG_&~B0Aw8uLhCU zG$>Rl!)w>tvGdc?)8EX}J`)km|%Nr%V{FYF+5_$%y<=vU}6PJTjl^how2z%sX!JZaw&o zPHbr1*Ng2x9h`LClFaI3BENgH!Wz2W(p>wjI?>vuYKs7clO42c0}5SPlVI7D>k7wM7+Olc&W)|>FN@x?nvl2e%eyQFu9 zO%gL>s~XAV3t4KXK6*xp7sa10EZ;BF94K#nkDg;=f~z>R1oy5x3+{UXVdH)#6nu|a ztBlM%;qbTGpWXXvqQsYP?APSl2Xbfbb2El4Wj2O^p*BBtpHFusMUi4}^-R4TK0@kg zk%1Wi004ygvQq#Uyr_=cCOm^kV7+}Na~@~$@pNpvIp&ezX0xuBC zI3VnGuQV_Y7UU{eQ0O4}-7<7tzOoOplwEDqPrff%Xm7J+>Bk%4dDHd?-bdRc zKQ>9)gYbnt6XSR&dBJ&C-dDL3ojbAPZ8Pt-`n-Pe4+KiQGiV8)W1pmBk2s|fCDbaO zx=Y6pJ~2@gT-f5X!e`~N;DxnJgq+*ce?%b2?H!MYGP;AW-I+!&rk@u~3$?^X^ys8~ zdCyE2q4YHWMwc}dIk#6g##y6VMQt=7W1Q}MZ9_8IUgqmSOZ*nWnswA{zv1Rw=JOGS zFVXEDYSrtUIfq0KwGHR&oD`+;6Khjy!=*F*s&p#N z^>=e36GNKaMCPQW&=DbuC;nH9imP0OwYj`%xtCG%?pgwo9aAY5FkEY*Yx306Vl8Jd zY%yOsW_Mk`^SB@?TAeY~KB`=k-?D^!+srVE+RE^xc>YYkll<0?RI`+aWd@nX7{>+Q zTNQH?D6W=6I2RO&U8hNeSkJ<6HT=)i5g|o~C@#MfMZ`Syi3Vc+vu`9S0Nc$wJiq=F zhsv71i{ct^kJsp!Jl96dujkO?q#CUvJLaxueH4_UgniiwQNi&EN+ngRHhV;(KA)y& zpqjVz=A)ndzOYn&&iR}}F4E=4L$u%W#g}ui&+Quxb^-a=(7Pgq5yFqVj$>f<)F@px z3VE5MU#C4&qnq6`+kK0+yt?;QCKi4;Y}ZIbipp0J=l&TL-XnK+IanUVJ}ThQoN#F~ zf_3y5H66*Yc2ABVTN-LI9r$n|*7cv9?OjU7i#CU4SdC?7@=&J}9mdI1J627|FNSY_7~ob-lGoAtlUL z=-c(V?qg4{lm;eu+}19EFA*V}-#5ssUpSQYUYra2Ebz$~yWOGaoz;!LTVn%0)IZMr zVQQh7W#-e~8A}7!dDTY1P8rRTp!pY>DsMN$0|)D*acsz86<8TlFw|vIahqz(S}I{BIU#aRF&~LW2-_;3{}GskJKW-Y8)tAU!Ee<~C(2C= z+i2Wo@Ohc{Br-o}MO@PYI1JSUsB$IpUgANyTUJ(@twOk{QYGcM=UAdzSIW%r*zPIM-W-DaD97f40- z{tyzFB98^(Cr}I2nsJd7es(NWh*7Kt7q092vl$Pu7<7uFvS~^#QTlxw%&g(nHT&LY z^z9ggw!F_-zd>Jw3bG!QLzzWKw53=Qf-J3)Gk3O)l{;frJB~j3jT_jI%MVVFnFD`{ z$xT&9SN#~ArelAOIp-FmDM;_D@Nd;8dLV!uK_pWbfOT zEE@NqD}u0sX+ED#7WLP#80*X_%|^F7s!w=kk#dIb&m9}_+V50X(IWbxb}*U*w3s4~ zb3*@9=5z!w!4!Wf&yixG>ndJ1sf0US2g(TGAVd zvIOmxN^G%b_iI)E+6S`zQlI7$^N>3^I_N}l>BAr~G;YAl{D)o}H=>Oq-bkzO-Nw0?}8pf*v z)B2gg)?Wu%wBI6k7PQCpExCC%HhkV}RbP3kZio%Or=CaEk!Z}Czy0cdgd=?qwL5De z5P@S?W9cwVw~6rKV!dNeB||NHXJ~{wo0rI+4xXi@A>+kq$ZY5uX3u0Z^ma~o3s}4< zVLHqdc7zW|JB1mOv^RBAMJqMdz}-sOFdhX(;2>MKT@E%~IgjtJ+l(7n zbKlxcVJpwfNhPv++*@VT!^P37XJ8_`2o-GkO1JoqFliLUh#8jLcl}>K$^dWm^~-5u zJmT_R_o-1ChD1c5=H`>%`+07g5o4FJl@UAX8U6#GYa6>o0$UxAe_mI_wLsOs`d`9b zT*C44Nd=@)xtl?Qybv$2%cXwc+)4EIqFUM6oATV0_;@ zFe{B3yW{1p@|=aWW5l#|Q?b%KBcb}G)}`kCZ|@+(8Ag-H0cJZb@5M>ouo!T3#Ho6HeLlWV$k zaLJk^Qa8Btoh;+~0cZC20q(GUN2aQtb$KC|ao59ReTkRVzNT7<$quBZS@pWiUjDDh z;SG@3u++RpBi*py$fk-|qyu>)-6$XfMdp|?8E67hB=&C>v~6O^{)DH}wRL&K)kiu@ z2f?O9hq}R8l_k-d)2!8|X=JuI|LgL6@^Ygf9y9futB0H{cTWN%DTd#CKeS;{BsgAL z)=H-Za7;N46gvvceXS(}ybna}n=mApc5W89)Sk+!E&X=pD}LRgsYhKF|1Qa@$~=Dq z#5*i8uTe%fj6E9psUilkCoj7+F3C>zaUCkvG|&VIuumUT4;`cY`w*JpHGiE934Oq) z2U`DYPMlm8zD>|qdxaT+PntQGRfmx71diN!G&@QzNI9$vO#BghXJl1Hti}6;o(5*x zw)#;_5y4DHZ@RQ+lMP6;*JY>C{p;%_CoL{)+p5aPAdRIxAlFO6Xi=KRZ!}8&{>%7* z(|~tI^{c)9y~dlQh*6FP+PIJ&gam~RrolKdb9`)#}fT|^i7#glysG}1rxRXX>L7~SiH4;n3lq33Fzl6PD4-h2rcZn{xE!|K#u;KdY+ zC2W<=5L539@SKM_Nd?s3{A??6@dVlwAnZ4{-ohkR`CNF9@8PV5R%>eE7{7mye3cFB zI_Y$xt?0S8Tux^fg6Z{(44pa~R9T(AfC6$Z;P~J3>;rLayeH=OsLl{Oh5o@kjJXi? z#~&9F>q)LNKcD@w`rlmmG8HDKT#EscOWyWWe?~+y7J#r{8y;#F!b~U(#*;oQ#5`p( z*4v2#nO#47No&#+&J|cK6qjH6T)d|I^W%(@I)?Y}2B$2jzru~TgiH*%2kG1y9YK8I?91iN zIkC*KpE3}`rbv;Otdx@XKs=wA%T6rSnog-W zDYt7?P~}u?qc#&Uaj)9!sZ7i_I#Z~XGZkkQ_BpHHnL!=3^!p>>n||LFjBJr}`rK?6 z&KSW-kC!a`JT2&#dt>cYl>|C9gA0Mv{@zoeDav{5(Q)zx%k;L^Id4pN?CN&(pONtx zw>Z<*RgeMk{jr<~pQJzbKsu-R#Auz;T=UV^VwKyk$y*cmzpj&vgLg59jcie3b-G7I zy@AW~JV27iR-*k0{zLN*e2s!HUG*9pG;4IAmUY)xqvf`Z5ikk_mDAH$2@wH7h% zOZ!(>B6~)61L`~u&RdqHZMTlS&s=$I4fi8AFcMxmjaxm}dfXulFL>#Km*+byyT2VF z3|+U~RyA{)#{3VOnPax+u>W%@h@H%45U_oZDwF`{;=@ikML}!H+WjTpK|^ zYuGfLv(Ew{dk71n(eI-Ce%h*v~i$Br($|{i*ILi86 zR)@U`d>1Rd&+sznZ92{o^;iN#p?l&c3d6kXhC>I za;8f)T>iU0L^1q5Vdt$M#(lhaU^IZaC|b%~!jxHQ^9p|1swnaP`J>#M5oeiF!0>Mq zFLN)*i1zR>5c%KlEy5h zco_kJKh@}eTTt*nj07RZe+c$G^n^Vg7GN8J4k-f^C1@gfZv1fuHxK~O0|5Zg|L(XP WgMt|d(MWlqBcT@wr(og!v-BS+IaR{| delta 5002 zcmYkAcTiK$+r<-_s6Zefy$PX*Dm6-pgx*0B1W}X{nn;&=Xh#uTDyL z!J?i1VPD*4KmNk~%z?-o{o$=}4tC!G8R{-F=9W=UbWmo7?2U*gGmhDrrO)Z&SlZ*> zzX7O>ihqB;^WqX8tz8hhIP%+ZUR%T!gWw;4ol@64F6K&sx205 z%So3pw+Rv6(yGz>T{q0Dt`faU~&^eMCHAdr6y_U zQ=>%}=kn#GcwWxP&2pPMxKrt`Yz*ZB5jEbjkm1Vcf_QWUt2fG#LwbN=Q|zHwJ2SVv z-rP8^E>|C=$r*$6_ZNqTK}4$lI0PUFdoWZ)s7?j~*A<`azi9J<0O(}aP^YsI|jNGF>12TPvcXzT5pJd9w)rT?W}d278*>K z8(FkGye*Li*6Wf@YA68K!SlBbW6|7>4B~b|>Pz9C(n||`aaGQjexH;LROp)+-~GUo zw<^oWhT&;;2yT$*lL-UN!f}waZ!EnIsZrt9cD#U{MtKM8J zgXv7`MFl!4he+4Qewj(Q^`(58lA}POcTQaRd+y)5%tweniUX_w!rClhZIwViux%V& zF)f;)mbFYz&BR3(Mo{N9_MYS-s4p9PUteaqCyY>(>f=HaBGO$l$jj@I#o@nx`2L9| z96wK2;jKR2OkRf9Y*f|xRU#mFGTs)^$x=rnawwU{q{+exNhlv`ibA@u(2T>u@oYpS zUAgiUvSB-G6_{>B`SnI+K*61m6$cXkeIj z%sBvqyK5E5*5zvIY%V_zSbXKa@`70pk*Cw7(`fhEd2tOL{6p%)G4aUiM8P_^)6urr zTwb-nhz^x5e({ebkIARgpAnBl9wBAL8FicWBve;cya0)*r1l}+8LwNJnu)bBmJG4@ zu^I%I2qW;K{7J*5!p_-Dfyt@M`x9q2)7(K2o0Dzt$77_+s!Rm@mF3w_$&B^L zDVRVY&}C}ULlxxSnE>6yJ*Achdb{ic+dbZ;(6@dp@b_e($Fr+FhjA`V@m!Q*4is;D zX5=uRdJV!haW~M}v=ib|_;{EG_yM899WIyp`W!y)DPc32 zcm)X; zYdB%7@ACsJr{0?MP2j1;b@OAP2B(@~MTEs(u=39ajnp2^yX_;qzxhvR=B7&>^7^UJ zQzcOz%xv^D~)e5u@&F>dv?aT!rff#WF+kV?{;DPA+1nD)mOE2KtXC*PJ8W13b2t zd1>m|W(uRxeR>USAKv?T-aT?!!&fMASv?wf5;FAw=9A;NwV&_b3w$zQbZ26bnKzZU zZi`yeSLbKz@ik6^=zd}CoRm&~L$Jr#LNRq>RWPXI9ZwC|=DYbc?b;H%w-bVrA1_{f zAN&Z?{GCQF$-A|{WHJJhd8G6$Z1 z^^{EM*=kz20T?oy1#%1-sHQ-K-i%3p2;(^r9LV2a&60)-WGaUKc((YI zapUhwO0SA9R9fTBmY%F~<``PLyf>MR5j&+2P~%F{UX4MSYQbhN1#dAKYc+k9+)ALF zaicjv{dSVKPS2%_`q+GM*p{S?W$jyVd?9Rx?oPjy1aQW=2b}JPI7n6nTfQCR5Kxn- zZ$*CRnVMLwzNc_-q?c=USwe4c9{NX|J95} z32Ky&B~xfyIuew(SNxeNkWaPS?#jZ;hy7#sS_6EqIr^c11w|jGG&NVJp{wk70(@eq zlHfDLw(M%y-?lkO{VQy~7DD}HW-qcz8j0k8L-7i$v%S$P6^f0?8Q9dny)vnjWgTL$ z*t8I(i?j>6Ze@U9xlvGC9YTid8Eo--SFXZ++wYbb@ztk}`S^s^pa;-I%Tr0 z&!vhNbvZ0s8#6~N85LEuRShn)mF4zDqph8Gud+{LonUx<{wh26wpu!6SDx8cp8?b>PVZ%VMZ{;WEdi6rm1DqgoN87w#f*7{diqg0)%Uc9zZ--&UX1 zsQ{4Fp22mmY@$P^o%DExG9b3p3vMHWl{wdflhvMEJd58BTFKUM+<;t zV<%KdO9>r59_<}|9Zye6eWM`pxUtx>Y;s9!G^qc_d5vju#1NjzVlg52Oq7CGFKRAE zd_x5_r&|N*<4qG-SJ5X&Lus=t?G9`&g1Y!>5~yk*72bfh5QRbMaiUbZT07<@fs5wbK z;kKvqQ^+;GBFGv=7QFljU3#RZ7wt{xOwNJUs>srm+Bxu8w41x``_pthO@S7?-qo(e zj_H;>AR1=QcaM7cbax;tGSp%IbMf9s^l?HNBq34BPmZK$>2+GjQOrug7D8%vNdsCD z1Am>dzl+s=as>2B@nV&NL%%J}@3sl`2SSs2hL+wCRWiMAKSum|U7qB>Gt&vtI>V@D zXlklz8;&46$E!oQ=BvgUY#Nk)B46*abY8&i-iFkz94ROKv{R8GsE)WNRtzX;r#Y3M zZ3ijs#r-~RQY}98i=F4tjV)n|sF^vMOlsjvr#G2Hi=pzup|uc^)*%yhMD( zhyhcordRVnC5l{mLk2uEy(VCG5l$}RqC4%Od7VZ~*k21=a{0{Y0^CMDN#!o9mdWjg zv@vsMC4*C|OB3ePrg!B&6*VeTN^to{KN2ydeYfml(Kk{$IKpn7%_6q@@IRa4-qT(poXRmea<=_G(sqGg z1i%M7ECi=)kb3|pC%Qn~=I`7Prk1&u-QQo&`)4Q3o?8#*ObjBg|JwPM)lJMi_L3qW|*V7xpENEkTp_f@BImg|H+1N0^ z#&0Ud*C=c`f-Khkf|NcvV$N* zh&-coVL-F5@~cmIlLzw{I|T~FJ@x4-#j+7;1(l0etb(knu|n>9uY=vG9tHj8`4+1R4Oj9Hblk1e=8s5*s49RTlm8$)^C)jRD z`j#i0H6pTaN0h@f7=-hxjPFIL40rry(C9Y%>h%7KEE+;5J{FzE)>QGILJ_TnWTOT+ zhtoC>X|+ZB`FokT_9}MT7&(C!D}dx~lxD9-`-bkwmXuN$0ZMyJJfUT&G0Tzjt~b-} zk0*e-rFMCd{?SC(7nb)I#3o(8SRhlz?+eunxjwyfA}){?70RD=2Zt+~Q7 z<7hK!HMcG{TjuvFLko(6pW#TNa{OIvq)Rd3DP3l9#7;y1I RDqwqDmk>V{i@^D({{xD|R)zoo diff --git a/summary_results.json b/summary_results.json index 69e390e8c..e59bbcdd2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.79935, + "Performance_Result": 10.1046, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71474, + "Performance_Result": 1.75007, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0181, + "Performance_Result": 22.8814, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.437507, + "Performance_Result": 0.441243, "Performance_Units": "Samples/s" }, { From fe446b8231cb79162160def82218d533f38cf6c8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 15 Mar 2025 18:55:02 +0000 Subject: [PATCH 311/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10230 -> 10232 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index eb5e4beef..23ceba8d6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 18:48:21,942 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-15 18:48:21,942 submission_checker1.py:3260 INFO] --- -[2025-03-15 18:48:21,942 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-15 18:54:56,737 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-15 18:54:56,737 submission_checker1.py:3260 INFO] --- +[2025-03-15 18:54:56,737 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 18:48:21,942 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-15 18:54:56,737 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 18:48:21,942 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-15 18:54:56,737 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 18:48:21,942 submission_checker1.py:3276 INFO] --- -[2025-03-15 18:48:21,942 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 18:48:21,942 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 18:48:21,943 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 18:48:21,943 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 18:48:21,943 submission_checker1.py:3297 INFO] --- -[2025-03-15 18:48:21,943 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 18:48:22,961 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-15 18:54:56,737 submission_checker1.py:3276 INFO] --- +[2025-03-15 18:54:56,737 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-15 18:54:56,737 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-15 18:54:56,737 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-15 18:54:56,737 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-15 18:54:56,737 submission_checker1.py:3297 INFO] --- +[2025-03-15 18:54:56,737 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-15 18:54:57,928 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39636 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75007 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418488 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26127 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1046 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1955 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b4f71605c..791a8be35 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -938 +939 diff --git a/summary.csv b/summary.csv index ef7a8b7c2..104b2e8b4 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39636,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75007,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418488,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26127,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1046,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1955,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5e992724f..7ed58a6be 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.75007, + "Result":1.70821, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1046, + "Result":10.1955, "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fc7b2e92c79914bbeff2b22708cbcc693e4c5a31..c1787bc6da6d59aa36865f85962b072341ec78f9 100644 GIT binary patch delta 1871 zcmV-V2eA0|Pxw!;%>xQ?>Z9Co1ONbf5|e!h8I$k>4;*2kKD5##T#^tPB^gcAd(&H* z-D;mA0w$mV%wUrn1RH-#$rCMk!t#+}w=ILvJYZpxuSUk-^V61R5H9>Y^i!78k?~5o z@pb&y@R^n0c}%Gw(m>8fMl3`zFijrBH1m0z6*QMmB35R;khkT^_EVL6y;5G)79A`a*`!jvuXTjSzVOGxl0B2M_P`5Yy(oL&Sh7li8ogJ;DmHUHFxw0e znTIOo)3`G;sH|T?*>fi?aifwyL2=U*0zV4j_86IbVP3(G1n398)7TU4jyGk zlA^2*>#3M+-=A?0s(LT0)c?iUm}phZOD}laX@va2n1C38i9c7 z-XIWA-CG0#s(Xh(Ky~jC$Oqze@d@YZ20-nPbVvtOZGu2RzQ+gzfZK)$C41k9$h z4Fp+;&Jlk)m_`>n=$D&koAvw(4or0iT~MoQ$hko{V6GqtFyroig&mF0|GhhzO)qY) z_Ukr*%RXV1h`3s|1D2UtDt|NC1IVj_meC@~BU;K%hAsrsXCA)~mR4acds#ftTSQ*5NC#UVUt=4+?Uj!_V zl9lnr2&wRslp6;_GeS1E`_ITp_H$H*R{wnv*Q|-lZZ_n0r|((sR=q9J^>NYlb|ciVlH5(IF5hy8aH)A#^CZ9rVjS6dfFx>JBu~LC%fN0a@Q7RzZ31 z`u~4J(S1fve;~St{+8%0yX(1!A8d6Vt~yUw9cU1abVvs*v=amZRfj;J>JSK29RdM6 z^O?6pbqF0~d$EIl=|R=OfvN7G3p*vqxzRabA-P4Y!nVl!T-8;}Di>8ten=}5FP{6c z0K5N1fS-}x9|&;ny%V5qwe7Emw3x)0C5d$$~Y^x6$UjVP+5J&A0e3 zxcZpS!$hrYe%hQIh$LST{-Zgfa0in|$5O_EXjY|^h}S}X($kkhz9W^ZLBf})IgKDi znXobuuZ?$gI#Q+nIAh^{9aNPkKPtMv7wox%y}&*ALYxCKA~N$0RAyIusgDbgu*OQm z!e7yAzg#6bCn=3&qE=h3Rb_L&QC|yI)K{`uUos(Bwz-R`AJS6Y$*Mb{^IVFj;2kQPOX73@u$YO1XUrQ9TkxkR&3!v#i;>vxVeev4>ep zft9GtTY9u^dS1?UyiKK7KqWdt9BJwy8c(`*VPY(5HnKG4!7oe)C-czsA18VZ=%2Mh%@yM2(# zpYJGmrjDO;mACvjP3t51d*Y-a zP+3VT0qTiLNCZ)Fv+Qka5&y_Gqa1>b0#) zd|iaYwbc1lIPpdRe>wzx;_bA^&<8vU0mu;dtbFFrFDA_r)w6U~#AzO9MU%}*aZifJ zuj9y@{tHl^hWwj&1`N4xKSBKmvuGoP0tIpEqugxS2C*$XE1ONbf5|e%i8I$k>4;+C|A6jV=E=dTDl8mP5z3DB@ zZnaMl0Ta*wW^9ui1RHf)N{f-{v~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw z=GXCG!)MCA^Mnv#NCO2QnTZhPz_NIlkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ z@*pkD#=?O8i47XZX-JM~SmmS;jS)t&K*;cUl9t>Y52I8*Q~7@wjKm}Jn=_a^`ldA= z)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QE zkVbln64H#RR2QMSDBR!;%NR(kCNu!7qwcD`bg65CSBWG=(p*r50>)6g4 z`>^eFZO`qg*Ykgxku}F^yw(vO2O<~`8GSZb?SYqpdQr~6wPlq;HF~dxRczLJV73`B zWFD%RPvg$edQyhhhzCfSzZOtA?$Xsf*xMdvI-(0=&=I|D#9T*sZ-}iZJ9v~G zjqiXcbwnRT`CG(%7h)^QE*@o95eS&QCkO;gu2Td8X4e@4@wOr72n4+S3j_j6e~Ca2w;@+L#0AxzA`npBYXkzS zdxJngb#D;}sO}vC0oA=nARma=!zY}l8vwOG(jgsCwFv?N`5q$>knafs0r{RH5HOq0 zHV|YXI!Ax#U>aTQpkHpDZPxQEI55>6bV04IA?F6+fVqMoz>K^56?Qy6|M%`>HodsH z+OOLLF8hR4Dw1m14ry-Xnf%RV4N2eB>154kgv;HWTdQOdgsQD*$u6QX)e3*hs#U%U%EDyvc0=-nl*7b&4%3W^nLr?s<$P&J}$bxE;=v` zk90@}uJQ;3iVlH5(IF5hIs^hm*WV#JgbqcwgMQhEqJsld-GL@L$hpxuAnRMiDk$$= z|9^idy3fez4@CFS-x8hebbasegRRcTRp;xf0}aBF4(Wh}c7i~l>JSK29Rh)>Lm*&h zKJ#~|4xxi=FLux`eW*G(Fx4G&VW$K+H#!F_B)5oF*cSPptGa4g6{2d%4{2rM#rHlI zVE4ZW@H5i;0|CzccLH?mwqtjp{H%B1(m8)&D>5Nj#*L85f;G&y(dN8i<_zY|xA-r( z`j{`GRIO}5)|?!Ov{)JZM{`8s4JMDSt&D~7tjdTXUQ78&PhLv-j#RFu5xvCCX@pU- zl(JO3Hs00gNRG0b)_6%+ z1}kzMuvJ=cBO|d))NaeQiZ$mO^|hd7eI=XqB^83^o4bSr5n<|1e%;2Dip`BG$=k8f z_$N?H0|XQR000O8002J#QiZgLvv>#&2Y)ur0$-}y^6{7tOri_KkwzY(@uZW7XpqSY zULK|aCW)U$dI(gRoriS-OqLmXl=RyiLra&9Qf^;DRF6avB#8*`ENk}eY$5qq>|xeY zU?nQ^mL9E}o|m&7Z&T?NP>GHZN1A$w#*?ldqCqMv_iIheYAxmAH6t@mjNu=dXwInVNgf)V90FGJU^j`RW%yiM|m>+Ixt` zll~r}K`1LYe3%BHBpO#jS!VKK2>{A6n~wsY543biC&W+yULO*Mh5}~x0YgE}ZXcxb z=Q|3XspIEdQ#(o*EMc|DxwC{2pAiLR$wd(*lPDq{f6HpaFc3xe{R*SKE!$~| z5w-(u$*O@uU0S*m(j*bJ9!P4P{CyR_l0a5nj_#c^cLtTKgKVQ;uuf=Ip)5&J1d3@c z)UHDB%{nen(e>RkT(;eVEO&_oXPdW7zpMbHMiD*P}HY3fCZF>M`7+R2^yu(?N;j`C+ zmF{LkuUt5#@J9nTt&=#r4^GU+uIrL+(OX0Q41aE3-uhKzp-!h_fa;Pn!Yoj)ZC&Ea zA{?%z&Tqnrw*vU%G3Ynm#zlrc;86%bhPadRv3PoJCDy3U(|j4Hi#S_0SwZd#lILHi zkyrg^pd5$%n>Yc6+}EF={sXgWBZLA4(I?~QaFg>TK^szqw21Kv000?O000;O00000 z00000004l2Ws^@PMFEVHl_op^7L(E@K@Gtf004MwFLQKxY-MwkVJ8%mG$$GXGm}pz xCL3kRMG++d002+|000pH0000000000004l26qArAJ^^Tx&nF`W8YKV#005&AT`B+o diff --git a/summary_results.json b/summary_results.json index e59bbcdd2..1e536b90a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1046, + "Performance_Result": 10.1955, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.75007, + "Performance_Result": 1.70821, "Performance_Units": "Samples/s" }, { From cf587787e9711b2fc1c77d0e5f204aa2d28d3b5d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Mar 2025 20:12:09 +0000 Subject: [PATCH 312/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10232 -> 10231 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 23ceba8d6..75db15c30 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-15 18:54:56,737 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-15 18:54:56,737 submission_checker1.py:3260 INFO] --- -[2025-03-15 18:54:56,737 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-16 20:12:03,815 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-16 20:12:03,815 submission_checker1.py:3260 INFO] --- +[2025-03-16 20:12:03,815 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-15 18:54:56,737 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-16 20:12:03,815 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-15 18:54:56,737 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-16 20:12:03,815 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-15 18:54:56,737 submission_checker1.py:3276 INFO] --- -[2025-03-15 18:54:56,737 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-15 18:54:56,737 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-15 18:54:56,737 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-15 18:54:56,737 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-15 18:54:56,737 submission_checker1.py:3297 INFO] --- -[2025-03-15 18:54:56,737 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-15 18:54:57,928 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-16 20:12:03,815 submission_checker1.py:3276 INFO] --- +[2025-03-16 20:12:03,815 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-16 20:12:03,815 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-16 20:12:03,815 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-16 20:12:03,815 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-16 20:12:03,815 submission_checker1.py:3297 INFO] --- +[2025-03-16 20:12:03,815 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-16 20:12:04,796 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39636 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 791a8be35..8421eae37 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -939 +940 diff --git a/summary.csv b/summary.csv index 104b2e8b4..167d3620b 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39636,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7ed58a6be..10e588871 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.39636, + "Result":0.424492, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index c1787bc6da6d59aa36865f85962b072341ec78f9..5be6b362aca341dd050cfaa480dea6f63ff92370 100644 GIT binary patch delta 1346 zcmV-I1-<(CPxnu-bO;I7aAyhe3IG5avwaAh27hfPwPpE5Ocq#m6uLkh>D3_`Pnvaz z2AQm2*I^o9k{Dv7Lrj%9c33CUvQw6zqrkKSEnU)yGTRbLFz(PWppuBB)$T;qY2Hbx zTzpL@!^Ij)fn;(!UCx);M5+9yegc%w)W>tG@3(u)gw^^Ng~2q%bLwQ zTS)#Do0zo}Sc%HKr$_6i=jD9I+f;f5RH7xsk**%1@uaPXXpqVZ#vY~tDv7xUs4VmL zuug!=GIx)X=9^<^>9SD*m0;o_QMh^~Hh&&mz1HmG`Rk#4rtY3IwQcXQOy4hX_b9&r zN;Hl*(%(Zgo;3In4MJJL;=?omCDFJN$}*P^O8`)o`Fs=@eW0aFIw6h%F#C`&bQCbR z4>$^Hw)-HJKi^XDOf5g>D)0Gmn%Ysi;Qv->lXI!jG~!6757Bti=|eQYCDTYQnSVxd z$uyEnUFP&*2_%GmNSM6-h1hiL#!q5(9^+&(M;pjqbj(ct!hmM-ap zpb2gt5(W!dmKqkax@NP_-$Ujzb$|Pu>Ac$~UCg!}-!Axnx>^^YiOvy6+I@(|lXf4X zK{PAaeV7K&BpN`o%D_=2V3YZMVAibJ@AIQIpRpa57Bti??W_*W(B_w(*T-818A1{eScU2K(oy6 zqrvY3EnU(HK@MN+FzDiIEuaB~nU$9%E?Op;X(Kq5quMg4q((6Ms2xSGY57PjYWIS@WM3;GeSSJ8w znb${y*9TgdDlW`JFV%3M!Ai55Yg2^N{24Q--R*{?5)%ze%)$Zl$)rk-Cw7+mR9B1!( z=ucKJdQ8T%Ya{-1^$JNRdxfN#B^!)R7n8t9zR!dGMsPPzzoD$yhB3SwtzI5!%%gD_ zjg#-I-m|xpAQM9Y*OO!uBm!RRvx*a$4h0mn3&kaqC?X$!YQr!PgztR{qq{BHX+sgV z18vEvfkIPSaud=f5w(7h)H;6q%1+`W;8TyQnf-QlQMov%Ciw;DrO_42GlCM(+z6?6 z75Z2&(?^u}z_ehE(NLidJX+LuCAX9t2XD?;a6y7kB1n76ZH4wBScaw-`*3zOEfIJysBj>fiavvxLEWBwd}tzO@URa2?^Q*l6bDLCa0STL?G@pTao z*DxPe@x)sR;;D=J#M^0+V;_hl2B62dXXSJD{IXsiqPif(LrP|8K3^Av<^?79n||a? z{{^T{L;g)X1CFy~BhUf~)^KMD@d^L{8nZnnH~|F|v(^b E06^@1N&o-= delta 1347 zcmV-J1-$zAPxw!;bO;GmJsNNE3IG5ZvwaAh27m2!tnai7c1Ki!Q%9i*#F17VqVc3x zhiDMV3T_>y0VIhZMjFIanPG=@A{{$r2|5ZqJJ8Z4ohY*{nFQYs4Ff2NNILCKRGr?P z6w1ZdbS_-1sT4#ex6|cxnaz{RZ|Wy7`Als*rz-E`QBBwMbkPEK%g19rFo`Y@M;dvE z#($Ge9-=`eD|mUB2ACv%8tEZWWp*Cc2{2h^=uy&da||tAHcGjD2~j-~MUW&SytAy? zyR(JlU$KW-OM#WB%v*Z2ZhBtMcDzlcS3o5?LL6!8AsSD*dWZ(8tl;Zm8laMRYkD=oxXO(T8X}Y4jl);F4)1mw!wn zxnvs2r7ko2umqCJvIHFsMjvSDl1^~n62f{kOiq_f1Z!Abv(x7>Fn`LXEK3awSzWW&=kFo&nSYvn&UD`FlP+f4j&B$IH(jj@&_v^iBi%kk<4Ly< z(IA=?+&)YLXc7&eS?2a(2>{J9x6kI_1DfFWA)VmPC4}{87_iCgJ}_(6Z1?$5HlMV2 zSxb?XX3f0cN4J~GE!rp1?E*B>IO0gV57Bti?n5+)W(B(s(*T-818A1neScU2K(oy5 zqrvV2EnU(HK@;pgBn)Rw=JtVEvu3x?-$Ujzwfmguyxk|AH7%vBq6Mv0fF@c;9O?HV z8c+Ithz8ND;P+t~K$BTeI2cF)Wi&Mx=VuBiUMNtF(g6JAKkSXW!EW zF9*?10g>d5lW`JFV$FxsAi55Yg2^N{24Q--R*{?5)$1Tn)$Zl$&4~~4w7+mR9A~e3 z=ucMfc}&K$Ya{-1^#(~NdxNBzB^!)R7n8t9zR!dGMsPPzzoD$yhB3Swt==7J%%gD_ zjg#-I-m{mJAQM9Yx07TOBm&;*vx*a$4h7;wUaKaPC?X$!PQx$|gzr2>IJBtmN@fVt7T&BjV7&kr1(0%tnF{T^Hw0tHb|9%sf(%RfziKSh@l*^@U2;a41?sh}OMG2~ z!?o1;RXFiR0Dn3Jed6u3$j}Eo3IWIv_pE&8&o9j%64kSGR>Wx@XGN3ENpVk#$FJka zoBj(>o`(FJcm@o!V Date: Sun, 16 Mar 2025 20:15:02 +0000 Subject: [PATCH 313/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10231 -> 10232 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 75db15c30..fed875d73 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-16 20:12:03,815 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-16 20:12:03,815 submission_checker1.py:3260 INFO] --- -[2025-03-16 20:12:03,815 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-16 20:14:57,283 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-16 20:14:57,283 submission_checker1.py:3260 INFO] --- +[2025-03-16 20:14:57,283 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-16 20:12:03,815 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-16 20:14:57,283 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-16 20:12:03,815 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-16 20:14:57,283 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-16 20:12:03,815 submission_checker1.py:3276 INFO] --- -[2025-03-16 20:12:03,815 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-16 20:12:03,815 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-16 20:12:03,815 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-16 20:12:03,815 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-16 20:12:03,815 submission_checker1.py:3297 INFO] --- -[2025-03-16 20:12:03,815 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-16 20:12:04,796 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-16 20:14:57,283 submission_checker1.py:3276 INFO] --- +[2025-03-16 20:14:57,283 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-16 20:14:57,283 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-16 20:14:57,283 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-16 20:14:57,283 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-16 20:14:57,283 submission_checker1.py:3297 INFO] --- +[2025-03-16 20:14:57,283 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-16 20:14:58,395 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1955 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.181 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8421eae37..3b2be1544 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -940 +941 diff --git a/summary.csv b/summary.csv index 167d3620b..849701037 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1955,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.181,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 10e588871..cb2b559c1 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1955, - "Accuracy":"acc: 75.600", + "Result":10.181, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 5be6b362aca341dd050cfaa480dea6f63ff92370..ddf692b8167334945a781585ea9be9a7a29cbc7e 100644 GIT binary patch delta 2063 zcmV+q2=Mp!Pxw!;%>xRXZ~6Lg1ONbe5|e%i8I$k>4;g__A6jV=E=dTDl8mP5z3DB@ zZnaMl0TZl~7z7`GjVDra%JPxvv~ANMdB`G~uSVwI^V61Z8e9Z<6r?OCBlDGT^XvGp z;WI10^Mnv#NCP<^nTZg^z_NIlkSySBR*+miiCLKiLf)1uix(vcqMDSYmTTLdl?606 z8w-Q-Pi(L_rXe|IVU>|wG)7922116<6IyU{JdCJ(rt&d=N)nIEZ_Z%y=$qDfSiAm* zl4t(z+7Q9=futl9B$5R%4VBrFvG3~Pg&eXK)zY}9byRny5s^5kQZZxC=Y*~jA&vAD zC1feDe~paF(z>RR1urABBR$YaB+9Un&?q8#?KZ5qVA*5y$XOd`sE)VhI=1u1K5RQ( z+jD#B^}MEkWX-V}uXTjSfe6OKl06$`?SU78dQr~6wPlq;HF~dxRczLJV73`BWFD%R zPvg$edQyhhhzCfSzZOtA?$Xsf*xMdvI-(0=&=I|D#9T*sZ-}iZJ9v~GjqiXc zbwnRT`CG(%7h)^QE*@o9 z5eS&QCkO;gu2Td8X4e@4@wOr72n4+S3j_j6e~Ca2w;@+L#0AxzA`npBYXkzSdxJng zb#D;}sO}vC0oA=nARma=!zY}l8vwOG(jgsCwFv?N`5q$>knafs0r{RH5HOq0HV|YX zI!EY#U>aTQpkHpDZPxQEI55>6bV04IA?F6+fVqMoz>K^56?Qy6|M%`>HodsH+OOLL zF8hQP6-l*hhb*(QRQ_hN2as0N2eB>153>375M$w^qp_2vu9rl3he&sudQ0RjYUvtPGRIliT*%c5A)+FG7~bbY*@q zBO(Hta`Rwl#mMG%{~0;SevZh{>c0=-nl*7b&4%3W^nLr?s<$P&J}$bxE;=v`k90@} zuJQ;3iVlH5(IF5hIs^hm*WV#JgbqcwgMQhEqJsld-GL@L$hpxuAnRMiDk$$=|9>cd zy3fez4@CFS-x8hebbasegRRcTRp;xf0}aBF4(Wh}c7i~l>JSK29Rh)>Lm*&hKJ#~| z4xxi=FLux`eW*G(Fx4G&VW$K+H#!F_B)5oF*cSPptGa4g<)UiI4{2rM#rHlIVE4ZW z@H5i;0|CzccLH?mw&OcMeb&2A>5P5Nz%8ii8dNsti(dM{e<_zY|r}!_p`j*cl zsus2&Z4M3u%~uBh(VS3tgUO?7D`R0it5RZ!*Ft{Llb1riB9*FXL@#l37-1A8WhE7_ zjdyi8Ql-H-W6^#cRF%^o71`el_q@Sg=pB49+yi4|$iz2LnO*IrJ}p4P8V`#$U`4Kj zaz%4)q$HM!+HJX1mCf-+eJxl~U&&s5$%J6p<}M*YL`rohKW<~j#O6ko-Wdb27Or2>C=PNW76ECVTPO5kxuI}9r*c}D@Q3;s_BEM7yAK3W(@QB}X1 z+E!{gP5oO6+*aUHS~a+soKTR)y~&) zI}a*MEJKmhANKo(s!(vGG|1Wh>+sG<)unDrSN&G9-=CropkM_p87wzg_vF_|(JKBQ z+n%hNj+i?BTzPLId|&3hlM#13@SiHNlklh#4}Ueim+ZGsjTRH%q)+;4Vc6Hn2O}we zmwjgCeVFY7ar}S$N`H;+_@#an#^c8H_{Dw{#^Z|o#B%Ro<55k7JM#0$k!*x2_hKV|+rH-EZ+R*iyy^E&G#1Z2RLh1K|zoz5q8M=UTag%`NxA(|0eM#V6a>?4C zy9e5Jj{|#AQ+=;3!Lpa`+nc^9I^L& zI({}2F{*mRc0HT8rrr1CiDWkfib9l!z;kh8uVZd}5-PU%wsa*=-s(O3-VDb0@IO#X z0|XQR005KnCKj{c8AAvKux>8XCX*;49)HVD!!Qs<_k2afo$=G#W(iMX>O9S7aazRLyvb9N-ILkFS3mNq{|uDlkbe_T zfMf3KPf-5>vuPuQ0tuUM`TB4K004QD_9a&VWs^=OMFEVHlqMn@`0>-WdISIfC=~zz t6aWAK0000000000fPog1(I%q>ux>8XCX;z5Q~_#}&L<-V93=n%001lR)hhr1 delta 2076 zcmV+%2;=woPxnu-%>xQ?>Z9Co1ONbf5|e!h8I$k>4;f*hKD5##T#^tPB^gcAd(&H* z-D;mA0w$o77z7`G$rCMk!t#+}w=ILvJYZpxuSUk-^V61R5H9>Y^i!78k?~5o@pb&y z@R^n0c}%Gw(m>8fMl3`zFijrBH1m0z6*QMmB35R;khkT^_EVL6y;5G)79A`a*`!jvuXTjSzVOGxl0B2M_P`5Yy(oL&Sh7li8ogJ;DmHUHFxw0enTIOo z)3`G;sH|T?*>fi?aifwyL2=U*0zV4j_86IbVP3(G1n398)7TU4jyGklA^2*>#3M+-=A?0s(LT0)c?iUm}phZOD}laX@va2n1C38i9c7-XIWA z-CG0#s(Xh(Ky~jC$Oqze@d@YZ20-nPbVvtOZGu2RzQ+gzfZK)$C41k9$h4Fp+; z&Jj9)m_`>n=$D&koAvw(4or0iT~MoQ$hko{V6GqtFyroig&mF0|GhhzO)qY)_Ukr* z%RXV1h`3s|1D2UtDt|NC1IVj_meC@~BU;K%hAsrsXCA)~mR4XWds#ftTSQ*5NC#UVUt=4+?Uj!_Vl9lnr z2&wRslp6;_GeS1E`_ITp_H$H*R{wnv*Q|-lZZ_n0r|((sR=q9J^>NYlb|ciVlH5(IF5hy8aH)A#^CZ9rVjS6dfFx>JBu~LC%fN0a@Q7RzZ31`u{_J z(S1fve;~St{+8%0yX(1!A8d6Vt~yUw9cU1abVvs*v=amZRfj;J>JSK29RdM6^O?6p zbqF0~d$EIl=|R=OfvN7G3p*vqxzRabA-P4Y!nVl!T-8;}Di>8ten=}5FP{6c0K5N1 zfS-}x9|&;ny%V5qwe7Emw3x)0C5d$$~Y^x6$UjVP+5J&A0e3xcZpS z!$hrYe%hQIh$LST{-Zgfa0in|$5O_EXjY|^h}S}X($kkhz9W^ZLBf})IgKDinXobu zuZ?$gI#Q+nIAh^{9aNPkKPtMv7wox%y}&*ALYxCKA~N$0RAyIusgDbgu*SnX!e7yA zzg#6bCn=3&qE=h3Rb_L&QC|yI)K{`uUos(Bwz-R`AJS6Y$*N`!L%F;`sT0mHryt@k{+EjK_`X@r(T^jK>xEiRIqG#-o}DcjV`h zBiRu7ZzpP&b&N~<4HI=tOC3G+wV~@ZdlyZWi6h1tgwpQ^e@(~JGjsvx;wAyjZ|{+1 z`jWuCO6UlA}6on`cf#>4HUdP<{BvfqiZRtv$yw!X5 zy%~)0;eSv|0|XQR000P+^d=Rv;2A>*1r)Rk#U+y{A|8LuYQr!PgztR{qq{BHX+sgV z18vEvfkIPSaud=f5w(7h)H;6q%1+`W;8TyQnf-QlQMov%Ciw;DrO_42GlCM(+z6?6 z75Z2&(?^u}z_ehE(NLidJX+LuCAX9t2XD?;a6y7kB1n76ZH4wBScaw-`*3zOEfIJysBj>fiavvxLEWBwd}tzO@URa2?^Q*l6bDLCa0STL?G@pTao z*DxPe@x)sR;;D=J#M^0+V;_hl2B62dXXSJD{IVrpqPif(LrP|8K3^Av<^?79n||a? z{{^T{L;g)X1CF_GKSBKmvuY!R0ts>Iqug->004TE_a#>XW(bpk2pW@WCMp4qla(eS z8#gR)M|lJQ04Nmz02BZK00000000000DyrPlhP)m1r)Rk#U+z^CsYAxlg}q31{);+ G0000U>*M_Z diff --git a/summary_results.json b/summary_results.json index dee2f7f34..908ef188c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1955, + "Performance_Result": 10.181, "Performance_Units": "Samples/s" }, { From e77adcb321bfe6cd367aafc3e2355fdcc8d7d1d8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Mar 2025 21:47:50 +0000 Subject: [PATCH 314/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10232 -> 10229 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index fed875d73..fe5f24102 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-16 20:14:57,283 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-16 20:14:57,283 submission_checker1.py:3260 INFO] --- -[2025-03-16 20:14:57,283 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-16 21:47:44,801 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-16 21:47:44,801 submission_checker1.py:3260 INFO] --- +[2025-03-16 21:47:44,801 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-16 20:14:57,283 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-16 21:47:44,802 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-16 20:14:57,283 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-16 21:47:44,802 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-16 20:14:57,283 submission_checker1.py:3276 INFO] --- -[2025-03-16 20:14:57,283 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-16 20:14:57,283 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-16 20:14:57,283 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-16 20:14:57,283 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-16 20:14:57,283 submission_checker1.py:3297 INFO] --- -[2025-03-16 20:14:57,283 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-16 20:14:58,395 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-16 21:47:44,802 submission_checker1.py:3276 INFO] --- +[2025-03-16 21:47:44,802 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-16 21:47:44,802 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-16 21:47:44,802 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-16 21:47:44,802 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-16 21:47:44,802 submission_checker1.py:3297 INFO] --- +[2025-03-16 21:47:44,802 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-16 21:47:46,011 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.181 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.75139 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.65762 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.441243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3b2be1544..34666ff44 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -941 +942 diff --git a/summary.csv b/summary.csv index 849701037..c090703a6 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.181,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.75139,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.65762,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.441243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cb2b559c1..13a1ef162 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.75139, + "Result":9.65762, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index ddf692b8167334945a781585ea9be9a7a29cbc7e..dfb77a98b811cea33ae29774a4a809ed5464afd3 100644 GIT binary patch delta 3187 zcmYk9dpr~R|HtPZS+eQS+!osiu`^RM_snIsNV3Q+yPFiwF zB$u`@afDPtb4l(>^jqhj-}m$H=ka-cUhmiQ_4(toYqMojAACqu5ONxoT^f#XYCPTupR72Owh%ab zB>-cc>A59ii+mR#_s?|sOf$zve)-g1hn&dFr>RwQ(!es?~xLD}ZS*5`l4%;S2mkKWBjr}t|cW7ple$eGA6{F4Y zlvrykC2L>5w6xhQ-Z-Y^7<{^Ato7q&N0;SNAN&1^NmN~EA>37+?a}wtkC_F$`!tD! zEn$_R6?2mQ%$vN1G5gCi)g45kHl5N{0fjpSnZVw+e5P6cO!s&+bz18QP&7wX`qCA=UwxNscffRAg~hw zT5DI7q+;9yQgDF1T+3?zemx~g{dN^l7e_=%^3srnlH`L9Q`o0HJ##AM5AP=g*r(+f zB~V~k6i$i#)mur@dneJdXJY>>>`3PaxtK29tmnDXa#qd}UH2+ZEs{-PjQ3A)N{i%V zkXzB-&JJi@WKDo;;;gJlc%irR>Z{7bYZF88S+IOG?N5WWY;Cr8j;b9rXdU@&xGQqptF!yZujtIR>vcLxoUOZ9D=X<56D*~9&PZq$>y%EY$(1=r zh{#d^->(r;Wsf<1)8d6*|=(;ke7ujABc*hS;A;IrEFLv z97el^zly&t(jAAZOT>wbfDG_TR7vpS?}UA<+b9y$(3}f#J%oT87yO*wJr>Jl7+MTl zND1NUY(I&mWSr+$<;i7R!v0oWGqYIT2T9>6%r@>x(bZl4oZryo`|dwX zd^T95x@q0y96-~;5&ND=dT+i@ERA3X9>spBDm5Jnz_u*;SeK!6;|IgK%8X}qhMDo{ z$_zfSeDkY)XqqWw&8%cc7O0#J&r;2WfAKqV^Gaf_CKs0 znvWCk00Fb85SZQFY&-~%4;RH2CP;V;@YEBtPIgDa#7in0M}rbR!!AAVlzxr9UcDY0 zYEaK&RZO=OGj!>z847AsAJPXj+RAr}p` zYYD-t42;r<$H&UuV-b=cizW^%?p2jE7u1yZS_OOQC3}$y?JW^AETfA|{LLtyJ!S`t zr-K*<{I*F;Cqp@E%5)e2#f#vyh+<#PDVGt^6Lv{kmNM*om<~IC>=$1iZ)jh3bIUw& zkaDrsWpnqzb24Md`pO$T5T!fBoy}K@&9g!pF1cKM3SE-X~z=+9gsJ8&m=G z?{NU)o@2tvu(L5m{k&rIRJ5}yS~NOS`uP(w&aNga%VTk~_|zlAQ2H?JVjeCr?aAA@(|&RpltNN#SgKk^iJMMt>bzy9j=|sBHny3 z_#IWKdS_%6l`ZECa=Cj#B?@!W#?&xLi`K}LUV4iH!$ehG zCMr+33Wi+t zS*l$ezE_bt(K0(J^h@v35%!E_`8l>@VM`&9k4sRQW-QUbf%!yB^H6T zkmK&~=KjH(d${_;jyuGd?gL=)4iW%9x!=j?UGnc*o$!gDNlyG&-M?&zyfNmt^Pj~S zp2eNLil)l@&GyY@l*eL2xYZP8pb@XB-4eslf@9)FL;)WVu*}hGm&b*L>dqAI{*TKQ z_1K3Bsr;x%yW|j7OzEq>pKfHK$D@}-tv+tq=B^(OBQ(1J;6HdwwoX9YB6;5KRe#uW zhs4s-EWYM`LDfkt36^PW{P6kq!V${p_PGd z@<-+Nvh zf0^M$c+TJq{Z_}%Q~QP6Iz-y7$X{H<7l{Pby`1-dq`g2s{gqWrV-4y{92{;+1K&L& zSoypFLRV2zr@+Xu)v!B&P%}8}Z{uZ+sDjHq5)(;iN@iiRue#^Upy<`2r`kac*gnJ$ zp`}C;e1IlpR+cw^_ANYergm0DO4sJtgLKk)#u2&8_TfU>iN1&HNV4^=v8nu)x-W>- z?f0_r4cDTwM&Vue;+Y)M1w|{TYo-%tYWNm>TYEtIl(|v6Xu)1D7z0delJt%7;|O57 zhhhciWFw>E|95IV3uXfEePxGQ*tw+6QmGh&HNIW#i>|;OBF{gj8)z56e>9&r&hu P^QOO1Q5HT9|Go2ngtF{@ delta 3174 zcmZXXc{CJk|Hfx%#6&U?O+}b7N;DX2jBR4Z8b!8hl3iqf>?18q2r~?0dt_vJP%e(!njfA4dyb6wZxdwiyI;1ni2E2kPwkW1}7-lE+MNg6VJoxIvU zUnA5<%h9suGC|0Crki?v5$l5%^VXUZQqE4gqC}58XlitTc9Seg(rK$?|IL-2AW@a$ z>rbawt*sISK(yVq7hg2DO|HZbHIA3FyL^7i_J%8$Thu?D6dUCii{9Z=B)`?(@luN; ze-z=O?KzM#pH85UOfO`>G(7`PDR3iJem%ISZrb2~OdDPRF}YebC|+q=2Da=~x1zeATvADKQvcFhpTH*L&kfkF)tsHoh+t@q#5O(9 zuUHh%-k?9wUvkgNrsxc9!##T2hHO}9A?VPNG_&jjA9QVHYD9o8Dq}A6@Lhs&DD+j zy=dOLMSB)&Ae8F#tkh8Ub`faB_Qo^tog3SQQ-91`K^JFIZ7KJPEi$Y^T&MrK_-b+( zBT>I0DItJZmo5RW*o8RIc?4;M7IH?n&q;L@2I$FST-W(xVy^1f|baS$gk##EFGt-kql zr(*rU-B($ZmB zx2b-veUyQyuc^#%mLW#rt3SFsc@zy5EcdIU!BhS`C#!}$s97e=;>NIg#k9+TO{0r) zw%sIu`AWLgExx8v*RXG!FXeejdqEJD+e?-Djx~E01~^u(5|$AC_D=!c-lf>5Xb?hk zyU_Pb!!)a_$S+CVr}Vu(QRdaEWHKNCBT*y?!JSKrky(9jr+bb^xS>vq^7t?0SK-O_ zEpH!6jffYK{hKV41e4G0>XSCl|CISoNg_DhZ&1u}L-7i0sq5lkZvc4aPbftHQAV-d zH~(~IFaAY^C6LgnZdepK^&~^@&5}}hUzHkB6w$8-X$7r9;jca{a>KPHlCTgWhJ=NY zFs~nDAab}%JZ!vb;(SVC%%pZM0tb_K(A_<3xQDpG7Ju{H;$mVD%hTkUr%|m{qs8Yr z>Ft=@$ay70dtkqSA>gV+lJtR`0K9&F{{5XO2?9^5iY9#W+?`0@188}i)q*MVnsMg& z_OX!oIs?s>g;vimVvkpP^BTIn=hU=9Z`m3I3!Ttd^W)w&8MgAAsN1Kh#&*si<;Fnq z9He2)G}!QY3|EjNlm|9|l_0hW2ai8M#o;L0x;B&?y4t%&h;2>YNvF=>6HOs7V{`!F zPq#%^v>u|GBl~D3Kiz?giR5OoK&M+92TR>USS_2)K|>aILFm3=PCMWal;&7PY}riT zH+~_sOO+=P?rv}3fXg1{q+mF$A$)iuH0^>4^cfrnu%^J~rK{=Qyt20#M1@(J7WYCw zQq4(0pqp9^&X!EA=27Ilo?)^^IHIsn_@=YFS~Sg8E8HvO)pc*NqD|(&i>BSPV@Zy= zZ2@pmv+${-hE%Urv$?11K0l{FRT2eM~-3=MlgDbPX$_anq@2)~W;dDw1$H zE#R~DtocF+K4Z+V*7M3$KdA|X=8bo|uC*lN@~bSIg83sy2wx%&r7cBpPwWmnO@#py zZAb_GY-;Dki{Mm+5`$)diA0kceK$e(x~7|wok4sGvpev3Pl3IOMNLE!117h`^V9V7wiri*Ujf$NJ3^R;@mS8OZPdE z2ca36Zk{~^Dy8V_*ZekRDhMt3q>W=N^&qq2e`Vg>Z2#ZTi&5G)60QQ%x=$a$f9Zo7 zzd`((HXbs+4VPN-l&W;vx@vsNIzX^S`z&iv9BP&x5W__+!4Kbs;z=S7d3wevYJt`W zM)`FLA&2&%g7aTUV-Q2+@+V{tDJE}l88hfMjw`YcGeI=BLX93e`d+!!n>vaq`ja2Q zk=*yW-S3}Y`l5sWCT7B=_^LP!CMVgI(8aTJlcni9t#8BRGjBf?Zd7&lNlDo=`?wrJyyjv#!8kA%AW*tbOc8 z2CA9#^i%~g2m_j;LWWT(Pk2_$C5Jy%w_IY$b9O5FTi z2wYjz{{7J6^wz?Qu>9q0#Y#{18{>(~-W$>l07KMsRmOxKDjT(Vf=}0`ajF?b0Kog- zK{^#u`HcYAC|hBIBH=kNjnu`Tb0BvTk^X|nDt=Y4``exKDOwX2KaFLz<}R|geA2&| z*L>f%awGc{^`oY+1^oR%43-}11AG80N7eW3K#`1YF<0;Mz*3!f)S;Q^PYih0_cr+B zHP^IEIV$&1S&egX-#z?Zaw(z9_E|~!^TFUMv$xMHeUEi}qAY>w^JjeW#tynOj!(8wGzW$IR$9kN@<0S^+I%Y$Go+__#ruR+gF%<)LGFf$F zh+Exn&rQf}5^8LPbBMhH24BKs+Q@-oxjHDBk?=IGPcW!9y?eQt00E>s1^(UBcofyR ztf$^sYIOKe@#L*={7}g#`#0@A+U<8aMM9=0J-9i+OC@DY?p^=qB@R@-+7!0oFgIHd z72^zQ;2!{_V>n$YTjACm2g+<(Bl~cycyvzwLBn`mOq@3!+R&YA|FyVwXmJ9Or6l{3 zP?`%WwTzSsF*xhCc`&Kah#0Q^68z>}s5aRZ(4)|Nz{QnvIszpUamWQ|2gIyd3ntpM zvS}XcI~w}T{U0)gFY67jfnO(-C*LWv%U$A-?pK+@zUJ@WWGv{RxSc?+54-5dbIZu)(9@qyH{Kf-T7qx$_D6fm9uizhFlno0GppHrLM!VMvbJx))bO_1J|o zuDDDLfF2y}Hw7lmq+B^9UGKkrf)8TT$ZAGu!%p*`usNWe1xRu-72`7%b3NQ3ZblM~ zEWPGPEOQko@i%#(APu{Z!=P{eozS}zR&H6%>J06)S{@`s51g+#{yeG!rp$FWq{b>K zV`y;ayxHLL?2kbr|F4P$Z{@Qt?2vggtIHwAZFCRGJ-;zcqHf(ZACY)6yLg;lbyH4i_mP`wSoPV@4$6Ce0(D{XL6eL1 zBC4+=tXdDIPYL(?U#! zZC%*ST30eAd57!|n0ylBGGD(QVuu7b&+J7nCf~ok5d8BFop>_!_>nR%xdAdvLWZE> zs$)emjI7CpKy+>_UJCtU+rmHkw1SX8&UcduX!Q+aXI3lvR*!TkZ-W*dTM2cN1h3|# zF?Bst+m_L~4(N5#H}Yun zO|4)5?DHm1ClXGYMc!LpFuS%7q`OFVa{Rx)fV$L8U>unwr2#xe?vv61HjxDo>Od6P o7$J4$uudr;0C0^101yED--&!EM2<(Gf$rpPge2El_~Ffe0DbrLk^lez diff --git a/summary_results.json b/summary_results.json index 908ef188c..f83fbf14d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.75139, + "Performance_Result": 9.65762, "Performance_Units": "Samples/s" }, { From 011fb7da6a15f3ce14d7fec7aac56ebb113e0e64 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Mar 2025 21:49:19 +0000 Subject: [PATCH 315/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10229 -> 10228 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index fe5f24102..035c925ea 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-16 21:47:44,801 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-16 21:47:44,801 submission_checker1.py:3260 INFO] --- -[2025-03-16 21:47:44,801 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-16 21:49:14,650 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-16 21:49:14,650 submission_checker1.py:3260 INFO] --- +[2025-03-16 21:49:14,650 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-16 21:47:44,802 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-16 21:49:14,650 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-16 21:47:44,802 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-16 21:49:14,650 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-16 21:47:44,802 submission_checker1.py:3276 INFO] --- -[2025-03-16 21:47:44,802 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-16 21:47:44,802 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-16 21:47:44,802 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-16 21:47:44,802 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-16 21:47:44,802 submission_checker1.py:3297 INFO] --- -[2025-03-16 21:47:44,802 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-16 21:47:46,011 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-16 21:49:14,650 submission_checker1.py:3276 INFO] --- +[2025-03-16 21:49:14,650 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-16 21:49:14,650 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-16 21:49:14,650 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-16 21:49:14,650 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-16 21:49:14,650 submission_checker1.py:3297 INFO] --- +[2025-03-16 21:49:14,650 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-16 21:49:15,607 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.181 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2056 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.65762 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 34666ff44..720219cd8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -942 +943 diff --git a/summary.csv b/summary.csv index c090703a6..fa59d3442 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.181,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2056,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.65762,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 13a1ef162..7acffb869 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.181, - "Accuracy":"acc: 75.200", + "Result":10.2056, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index dfb77a98b811cea33ae29774a4a809ed5464afd3..300d26f89eea3b8fce7d2f82f3d7d0647ee077da 100644 GIT binary patch delta 1588 zcmV-42Fv;NPxMc)%>xQb%yJ`e1ONbf5|e!h8I$k>Cj`&{W^9ui1RHf)N{f-{ zv~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw=GXCG!)MCA^Mnv#NCO2QnTZhPz_NIl zkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ@*pkD#=?O8i47XZX-JM~SmmS;jS)t& zK*;cUl9t>Y52I8*Q~7@wjKm}Jn=_a^`ldA=)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QEkVbln64H#RR2QMSDBR!;% zNR(kCNu!7qwcD`bg65CSBWG=(p*r50>)6g4`>^eFZO`qg*Ygybku}F^yw(vO2O<~` z8GSZb?SYrGNCh4NX;mtcYS|8HZsnQ$&1DZDuS&w=MOws!$xenW1mR-At7V>Y)fDi> zQ?G4Xf)P@;AC~Getn2Ay&1Zzm-JDyiWD$g_t!T+EqA}G9%c@nr3d+J{@#MC>w%uCq z{)>SQ-3Bb41|{CXcSIjD_*6%7`IeOZiDpUP}3nRIa8G zy~NFFgi*4TvQ)e_-qqG0b)_7P3D{>vMRa$T(Be6`>Zp*cbHRl`kwV-8vC7bmn6@uoQyMzQ0Vd_qP-NuxP z&5bI_+p*F3C$nV;ybKOfrhR021ONai6#xJf0F%%r7PCbagaUtBx@AhH5?BUO)|9~G zjCL4SQ1XrfS{MAE4p_W~B7L+lj-sl5HMOnOa+>-I0dQcsZzCzp{t#*<#rxam{^7)sXy%Z4OOAwNNJF>{nz20k*Z7G zmah7(WWPT}Awa%KI?e2jYMD`IY_}-SJEPD2&IA>G6yG zD2&Gy`HAJ;!N#MS2zTV?kt5j<`EMs`mUWCv`wbIyOiLX-^|hhvHG3CLm5C$98HCdB z2Y*e+(=&7d=i(*-&2R6KW%`o9z2uU$KX(tb>mCR8qNe&@TY_aT&kY|n4PUZHsL2k6 z#g{Fr>r8*oW+%wtp$ z<#e`OjBe)j4Csi-)AM|{Z}$DZ=Q(2U`E>kjCSp|ei0yhdaZS7L$rH(L2o!}V4}s_6 z#9qhT_#{+p@oni!p1jq2_PrU5@!@|^O9KQH006Vb8CnQ`F8-{5CIJ8dPyzq|5dZ)H zWN%|oa&K^RFJo_VWiEJaY@L$LZo)7SgztTd$oIr?Qc!CpL24^bMXFS)qTVcf0ju$k zY`1~8uM;4E#Hq*i%zit&ay8qjHu(kTrO`DaS&kCW+z6>RHTqc2(+8CJz_ehk(NLp3 zcry#bu^HyZ}!!l`7~TexXGiKBbx zF6+v{8uJspTD-mwtEN)-r{aK`N^r^@uwYzCQ{l@Z9Nnnw ziyZnuBryOz#62mW<@3vOj+!DbCTU)#WV$Q}y?>-+^3{*L>OTY3amc@kC%_^1^(Scl z0JCQ!gaQdl%yJ`e1ONbflk_FP1yZJcWOxRXZ~6Lg1ONbe5|e!h8I$k>Cj_hk%wUrn1RH-#jVDra%JPxv zv~ANMdB`G~uSVwI^V61Z8e9Z<6r?OCBlDGT^XvGp;WI10^Mnv#NCP<^nTZg^z_NIl zkSySBR*+miiCLKiLf)1uix(vcqMDSYmTTLdl?6068w-Q-Pi(L_rXe|IVU>|wG)792 z2116<6IyU{JdCJ(rt*I=N)nIEZ_Z%y=$qDfSiAm*l4t(z+7Q9=futl9B$5R%4VBrF zvG3~Pg&eXK)zY}9byRny5s^5kQZZxC=Y*~jA&vADC1feDe~paF(z>RR1urABBR$Ya zB+9Un&?q8#?KZ5qVA*5y$XOd`sE)VhI=1u1K5RQ(+jD#B^}H0OWX-V}uXTjSfe6OK zl06$`?SU7wNCh4NX%!VowQPqhv$9nFX0iv6R|P5K15P;sG9hWojgZNDHN>~k=D1YrNKC3(S99NmD3*;+20HIyun`R9egp|17l>!#5YixKwa&n zJ}p4P8V`$LMXrN#MRRVXB$kQVZMjsH&GANkEm%=s$zFZQgkag`E+IigN_8hcZezy8 z=0=s|?bK-e6SHLqybJ~S@zb_?vnCaz0)N`Ncuu4S4J-pGYf9j8Mmr2ED0xQ#tqcB7 z2P|Ggkv>`&M^RP3n%Y)sIZgdr3fxxUQd%{*tH3=4E**J?V<}DzuIOu6(burl*BV@* z7c2B)g#A*b$_O3vlC?b zy6sD!yD#G&(TVNtO_Gl8IF7qF?h)PgOfe5$Os>aynZsMmO_%26V*a z>3P1}H~W6y^Bl4Fd^&zM6EUiK#CAQKxTf9rP@)bTT2*%iGlYL>3$Ioi zuwH^&zbEs>*Ky=c{{<*dL;g)X1BTqUpP>2!vt}cN0tuUM`TB4K z004QD^CiFq`0>-WdXu;&R{_715GOhTF_TLtCL5-}ni?hn002+|000pH0000000000 Z004l25|fN4J^^Qw%O@iS86^M!000!{@3{Z~ diff --git a/summary_results.json b/summary_results.json index f83fbf14d..450bd8d4a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.181, + "Performance_Result": 10.2056, "Performance_Units": "Samples/s" }, { From 68c33cd03d0eedae00503aa2b87528054af922cf Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 16 Mar 2025 21:52:12 +0000 Subject: [PATCH 316/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10228 -> 10227 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 035c925ea..8597311ad 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-16 21:49:14,650 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-16 21:49:14,650 submission_checker1.py:3260 INFO] --- -[2025-03-16 21:49:14,650 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-16 21:52:06,742 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-16 21:52:06,742 submission_checker1.py:3260 INFO] --- +[2025-03-16 21:52:06,742 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-16 21:49:14,650 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-16 21:52:06,742 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-16 21:49:14,650 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-16 21:52:06,742 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-16 21:49:14,650 submission_checker1.py:3276 INFO] --- -[2025-03-16 21:49:14,650 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-16 21:49:14,650 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-16 21:49:14,650 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-16 21:49:14,650 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-16 21:49:14,650 submission_checker1.py:3297 INFO] --- -[2025-03-16 21:49:14,650 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-16 21:49:15,607 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-16 21:52:06,742 submission_checker1.py:3276 INFO] --- +[2025-03-16 21:52:06,742 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-16 21:52:06,742 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-16 21:52:06,742 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-16 21:52:06,742 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-16 21:52:06,742 submission_checker1.py:3297 INFO] --- +[2025-03-16 21:52:06,742 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-16 21:52:07,694 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70821 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71445 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418488 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26127 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 720219cd8..175df711d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -943 +944 diff --git a/summary.csv b/summary.csv index fa59d3442..9e30a0605 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70821,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71445,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418488,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26127,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 7acffb869..f50479919 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70821, + "Result":1.71445, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 300d26f89eea3b8fce7d2f82f3d7d0647ee077da..277952602e5ee174bd098d24a1358397ae795b01 100644 GIT binary patch delta 1431 zcmV;I1!(&8PxDW(a|i@G(3)$rcL)gwf3mEir#@Tu9rJ-nG=DhKze6;hH1H4&GFidG z!!*DovC&BDfGTtGuug!9SGET}z1Sktl*B5#g<6&6b@lB>#%7%UTMo zL}lK|qjl5sa&F^oD!l?K(Fo#5FAvdp(#%6NNM!{(57PjZ#8LxPmN|M@CqQMHf2T)D z%gr&gblE6@O0e#bC|tb~^A4_FYj*Ga^-w-jPtTd!HuYGh?-zJ_lwSZP+C?1c>LD6W z+Iolvp{!u+VH$vvXj}OFKdQ9Jf4S6X8gZo4hiE+M^dTDHl4&HDOe48u8p)+DbNa9Z zlFPCL9Su$&Xz7wpaN`ofdNfQb%=0TgO_t1?D!I zssK&&jX2WnLo}Xr`w$JHS;6hYG=L`20GefPAC>^nEOYy4aQi?@mvlnVe+0J=34?_! zOAQNIU9;Ke?;-Pairgee`q}E_aPcYvx47;X#h>40W{0}J}d#CS?2fA;P-)+F6o4z z34R|EhO;KK`@pPOv)$+KA@iB~ea>{=?~~4&meN+yg4QZP6TKsjH2e^aCk;PDgJ@PT z{4fomNi=|Fnc;^e05r=CKN<`_(9$KHfUN+cdMpaatm604HT!)Ye7b^(;=8*!x9hiH81^&uLBvVzx#X#h$x9=Thh%e+3U6M(YJ>!ZQz11(+B zi83uoC769^7+}fVKCqOn+3oWfmdPk1Qa$OBY%R4_TEXV6KIxsa@9Bbff@r6JNb;f` z>GUBQUpjq=28pcTfAnD*Ad-v+h%9sZuug!;GN+FQrw_DrNhiv*C6VCtpMf9`YWH&W z!o&x8+Fv*ujr{7Rk zY{MAdjaILWH0IGbjK<0LRqxq5D;Jag!89DcTn=h68~?coUcMN-aCTn|oL9dX`jxS7 zq@~|W)9RMJWWTUs9qi5f*TJWLbUmKM#v~Y}g_=#bVMME!P%eK@!uj&I^l#eNFiFDc z{imCtKMbPm5mS0wFbc!u{zG<3HZSu$_%E}G6P6BtF@9TAB>?~cPyzq|5dZ)HWN%|o za&K^RFJo_VWiEJaY@L$JYQr!PMfd#*qrELZNLqyLKwGkEpiq~V?u0aPM6CysTE~B1 z#Yvn5yy|jv@0__as9f!37yW{DLaPd8Ns1y+Omm^y3cWY$_yI-EQ^jeg6;x;dhgS7n z$qZqC+QO^V2CNt0q5x8kFjJwO_l98X*bXFhNpPqjZnc%vhtjs#Q1(q*z?;! z^%Ng~v6+bIgyd{OnjhPZ0M0OUAVGPDvn0b8uLmpLY{RHrIi>J}fm!QWoZNdSCS%|C zNnecCkUztpo0qq7)mW&*sTiQT73`LM$9!NC%^!~R?+}eA4Ln4HOjfY) zFbyzCY&6n3pvqi4tP^0e%*Ugo&E^jE^Q9f5`k_eFe4DR|!ht z^--4c3wBGiT>vHeMjYw&AsSzLeTW93tl;%w8i10FNA8yBGOrKo1fVSQ`e^X_Kuec& zqD)It31%M}23Rt;4=iPCcKbYrWirZ$R8M*&TT5+~R-7mrfs|K_V+Se|?w+h$Q0yBFmgUtP>!z%;}@S=>siY(up!{NhCOZXc!R5oIXJ0 zb8`9=6X`#R$mi0AWTo!Dst1hdJE*K+Pz%8 zF!4d2_7~2E&j-ZNz`BUL5IUFOD>`WP{P^e_|3C$@h7%-w5vJ={J-W z+c1WAqtz=Tjd?T^4}&QC5tQB*jKVOv|B#)M&C5Iw{tL5-6P6BtF8-{5CIJ8dPyzq|5dZ)HWN%|o za&K^RFJo_VWiEJaY@L$LZo)7SgztTd$oIr?Qc!CpL24^bMXFS)qTVcf0ju$kY`1~8 zuM;4E#Hq*i%zit&ay8qjHu(kTrO`DaS&kCW+z6>RHTqc2(+8CJz_ehk(NLp3cry#bu^HyZ}!!l`7~TexXGiKBbx zF6+v{8uJspTD-mwtEN)-r{aK`N^r^@uwYzCQ{l@Z9NnnwiyZnu zBryOz#62mW<@3vOj+!DbCTU)#WV$Q}y?>-+^3{*L>OTY3amc@kC%_^1^(Scl0JCKy m$^rx@{J(*-GbTL&1up)qfF_f5CsYAulgcL}1{fs(0000GGp`l^ diff --git a/summary_results.json b/summary_results.json index 450bd8d4a..e8d29174d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70821, + "Performance_Result": 1.71445, "Performance_Units": "Samples/s" }, { From a1761786ca01829da57e1aeb7d5be0f0e9312dbc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 17 Mar 2025 01:17:31 +0000 Subject: [PATCH 317/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10227 -> 10229 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8597311ad..81af1659b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-16 21:52:06,742 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-16 21:52:06,742 submission_checker1.py:3260 INFO] --- -[2025-03-16 21:52:06,742 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-17 01:17:26,685 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-17 01:17:26,685 submission_checker1.py:3260 INFO] --- +[2025-03-17 01:17:26,685 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-16 21:52:06,742 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-17 01:17:26,685 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-16 21:52:06,742 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-17 01:17:26,685 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-16 21:52:06,742 submission_checker1.py:3276 INFO] --- -[2025-03-16 21:52:06,742 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-16 21:52:06,742 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-16 21:52:06,742 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-16 21:52:06,742 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-16 21:52:06,742 submission_checker1.py:3297 INFO] --- -[2025-03-16 21:52:06,742 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-16 21:52:07,694 module.py:5251 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-17 01:17:26,685 submission_checker1.py:3276 INFO] --- +[2025-03-17 01:17:26,685 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-17 01:17:26,685 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-17 01:17:26,686 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-17 01:17:26,686 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-17 01:17:26,686 submission_checker1.py:3297 INFO] --- +[2025-03-17 01:17:26,686 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-17 01:17:27,893 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2056 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1644 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.65762 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 175df711d..2ebd3a0c9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -944 +945 diff --git a/summary.csv b/summary.csv index 9e30a0605..764bb3979 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2056,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1644,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.65762,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f50479919..22343366f 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2056, + "Result":10.1644, "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 277952602e5ee174bd098d24a1358397ae795b01..6b15afb1e0eea362d705be69f2072bf08c13c0e7 100644 GIT binary patch delta 1615 zcmV-V2C(__PxVi*%>xQzX3*Vo1ONbf5|e!h8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Ec`~Gq%&q-;bR@2ex;aYF;Xx7f777_AOraj{9F9+4U&g6O7g|P{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!` z50@khcz=@?B$rQO%CbPnTeh%x!AKBQq%5^u+xDz1NOH5b&|@F5LE|_H$uSMfjO3y= z!blnj89rYo1viKND3Q-pJ_aN4!2IU)MvuN}4f~bre-rY|H?9p4%pXWfLO~)~0Mk&J z%_;q%9$v^HTTv~Ids;`+py$qFXJw$KI^K%w*v=dKu<5jI&+Vw!^O}+s$Ev+n5grF3 z81@-`Hdy6>7lC?FPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!XgpNa^4JSW@V}T&3|MM zAg>C-;#rc%gvm~Z%mm?L#>;t@aMcv>*;A)!TY?c%wI7!1GOX+IXvJrQ%iWw?i{&f` zRa?=JT|{lF5f){mcomd|%h{9L^qO{KwfoOPn#ak){9;B#1WC%xgT56bo7??o5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iCTqHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR5dCyp0!A z%g@^Qmd*%UkP%5!ZiG}8tY*HA*5?fqr#G#?#ec)q$9x_oYGn)3`s6?)`NH5o>mv%U zH+pn!Wh{&*Wl9Y3TF6g&@_$muccgMPjp!w=Pa}+iC6p!Nwf3$~N6Iu9W;EKbf-3Xm zXG!+=!ac9I7kUR@4EMko7&7xURAyIssg4Vfu!j92Sdi<0Es~rYDT!sGc2llZtUlkU zt_3ZsE7`2isSq?<-z`ZH5vK0s*KJIxSl_6Uyd4`2e*sWS0|XQR0JCfevl)Lr>rOu< z0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!LJ!!Qtp?|llRyDiJ6brH4$ZON&D zLQ`6D6VfITwSJJ)I(hplPU0lsQ;(~e{dRUyJ=>`+`32{t(G4P5juOz^2&p#>`dH4> zN0j)$v|ydl(4akdG;8l_ZYh5^4&I!x;DQ97M3DBB+XihzuoPp@x1gBMqC-bcMjYDXeEAq5|i_Q!PtKyVBlDA2)Uk`a9I25`zxHVn#@ zQ^~NmFljxDqkHG&Xzcqw>&w9!^Ar5FczqvMO{G4ZiUVqE!6|pZf^kP}jjxM%xQ_X- zh$mi45KsH4PrRKLIrf1_VgP!KdsaTn=a=OiwMAasr+Jx@YMB#CDq2*l!^oTd3s9Yg z{F`_N9CP1(g7&jyBZUG5VrJ0Ya+B^QTLNbYlYs~tlVv6<0gIE6COiTa6_YO&7L#Qr z5R>#K8UetQ4<|YSGLuRtCL2EMPCq39002+|000pH0000000000004l26O)T4J^^Qw N$|oZR86^M!003Fz;xQb%yJ`e1ONbf5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`CC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%iWw?t7H*` zs;y|rE}}8j3d^chz6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*%U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0t*!``xOyC4ag;F1o%hIxr27bVvuT@(2Wq4uL??ArL4!1Oi3Z z-yu4L4n?s!PsDDPeWe<-@o$mtJ6_t4)Go$Yje@9=}I z&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)g5$U zrvy1SItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50nYt*0(9)QYxki1 ztasniIbka@Az8+akjjEJ%(v0zykX`H=FPYGFSz=cFQQbfY(dtX9Eh}78T?0cMBxo4 zkFKqZh4HM)h#_7}`AJV+N`Lu|RIa8Gy~NFFgi*4TvQ)e_-qq}oIdaRCz6cvuE2aviW$T5uyHu}su%%e9I%=Nt94 zpk;j}oAo6Xg65mMgai>`>P~*$#*~W9jVj68vC;S^P)h>@6aWAK2(xequ^9w0ep^(N zB_bbx%WA_g5JmU>3ZuO(KS)}H?Lb?yYM@Y;mhOZ!aYU^Ll3K@qU&TqB1ib2Ubnl$G zGpJncWf%Q|bwaBOWl4%6P)u{7+6uil>-Ygh&Qrx{rxjFa0EbrfUC9h#+QO^V2CNt0 zq5x8kFjJwO_l98X*bXFhNpPqjZnc%vhtjrx*iiOOTfljmF0lkpIrS7DfU%i~=!E2K zLYg1jjR4LtbRa=_hqENZ7q15^-E70CTsfuigMnG=S)AN^CnjUx_eo!j){sBLpPQGr zan)F;!>Jgcy5x*73)E{{m-xB}hwG^Gn{eW-0RA`xedg`7$gvN26atWA+_UmgJUurg zYgFfHzKGKz&X!G{kws3@`>(^uoBj(>o`(FJcm^DE-+qGnvt}cO0tHIUawBk)@FiOT zWs^)MMFEMEk|sO?6BUy&6&91`CMp5GlMyF60WgzHCng&)ep^%}0RR9{0ssIJ00000 e000000001hff199Cq4mXlguY01{Wm&0000w$?ZV^ diff --git a/summary_results.json b/summary_results.json index e8d29174d..6db9fcce7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2056, + "Performance_Result": 10.1644, "Performance_Units": "Samples/s" }, { From fc76a23a77ef16239215937c9db8c73eb40e4508 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 17 Mar 2025 01:22:20 +0000 Subject: [PATCH 318/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10229 -> 10228 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 81af1659b..0961cc103 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-17 01:17:26,685 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-17 01:17:26,685 submission_checker1.py:3260 INFO] --- -[2025-03-17 01:17:26,685 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-17 01:22:14,953 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-17 01:22:14,953 submission_checker1.py:3260 INFO] --- +[2025-03-17 01:22:14,953 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-17 01:17:26,685 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-17 01:22:14,953 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-17 01:17:26,685 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-17 01:22:14,953 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-17 01:17:26,685 submission_checker1.py:3276 INFO] --- -[2025-03-17 01:17:26,685 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-17 01:17:26,685 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-17 01:17:26,686 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-17 01:17:26,686 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-17 01:17:26,686 submission_checker1.py:3297 INFO] --- -[2025-03-17 01:17:26,686 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-17 01:17:27,893 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-17 01:22:14,954 submission_checker1.py:3276 INFO] --- +[2025-03-17 01:22:14,954 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-17 01:22:14,954 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-17 01:22:14,954 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-17 01:22:14,954 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-17 01:22:14,954 submission_checker1.py:3297 INFO] --- +[2025-03-17 01:22:14,954 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-17 01:22:16,138 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1644 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2154 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.65762 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2ebd3a0c9..8152c0d1c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -945 +946 diff --git a/summary.csv b/summary.csv index 764bb3979..f44429b3a 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1644,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2154,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.65762,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 22343366f..045089117 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1644, + "Result":10.2154, "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 6b15afb1e0eea362d705be69f2072bf08c13c0e7..674c556550968578d79e22e805cf3471f4514132 100644 GIT binary patch delta 1568 zcmV+*2H*MhPxMc)%>xQ^C=Fk61ONbf5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`CC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%iWw?t7H*` zs;y|rE}}8j3d^chz6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*%U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0t*!``xOyC4ag;F1o%hIxr27bVvuT@(2Wq4uL??ArL4!1Oi3Z z-yu4L4n?s!PsDDPeWe<-@o$mtJ6_t4)Go$Yje@9=}I z&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)g5$U zrvy1SItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50nYt*0(9)Q>-4%n ze%8Bh>71|?nUE~wMo4AB8s^(*bKWp>2J_}y{1;q(%okCrR<}3WyBD#rTnBPFMp+cM=Dp-h+g96G{PuZN?9si8}I6Lq{@PEPNV%gsH#YR zRAhfI-17!|p?C1ba1V@;Av51VWp=ff`nUiIYdkE26}b-BDlNE?kys{bx8+*Jn)8kN zTF|n-lFj;(3PJPDT|$D0Fm)%tZevQt=0=s|?bvAi6HrS71QY-O0JCohvKa(P*jrQ2f5_&3`EF7o^yD+o+~u)y#EnA(wuPDsH=r2TR201zBQ3kr1b zm}CTByaAl@lMRD%QgF&0 zuwYzW;_D(Fu4O)~;)ypB#8V&jiMP`t$375A3_y=@&&ucQ`K4K+y2y)pn$J@5(By;| z76m2q^xQzX3*Vo1ONbf5|e%i8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Ec`~Gq%&q-;bR@2ex;aYF;Xx7f777_AOraj{9F9+4U&g6O7g|P{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!` z50@khcz=@?B$rQO%CbPnTeh%x!AKBQq%5^u+xDz1NOH5b&|@F5LE|_H$uSMfjO3y= z!blnj89rYo1viKND3Q-pJ_aN4!2IU)MvuN}4f~bre-rY|H?9p4%pXWfLO~)~0Mk&J z%_;q%9$v^HTTv~Ids;`+py$qFXJw$KI^K%w*v=dKu<5jI&+Vw!^O}+s$Ev+n5grF3 z81@-`Hdy6>7lC?FPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!XgpNa^4JSW@V}T&3|MM zAg>C-;#rc%gvm~Z%mm?L#>;t@aMcv>*;A)!TY?c%wI7!1GOX+IXvJrQ%iWw?i{&f` zRa?=JT|{lF5f){mcomd|%h{9L^qO{KwfoOPn#ak){9;B#1WC%xgT56bo7??o5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iCTqHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR5dCyp0!A z%g@^Qmd*%UkP%5!ZiG}8tY*HA*5?fqr#G#?#ec)q$9x_oYGn)3`s6?)`NH5o>mv%U zH+pn!Wh{&*Wl9Y3TF6g&@_$muccgMPjp!w=Pa}+iC6p!Nwf3$~N6Iu9W;EKbf-3Xm zXG!+=!ac9I7kUR@4EMko7&7xURAyIssg4Vfu!j92Sdi<0Es~rYDT!sGc2llZtUlkU zt_3ZsE7`2isSq?<-z`ZH5vK0s*KJIxSl_6Uyd4`2e*sWS0|XQR0JCifvl#?F>rOwD zBqA(-rgagv18vEvfkIPSaud=f5w(7h)H-?lDo)}g;8TyQnf-QlQ9awKF8KxLrO^!{ zS&kCW+z6>R4f z6%~^(6&91_CMp5IlMp950WyrOu<0RR9{0ssIJ00000000000001hffJLA SCq4mZlglR~1{oy)0000~f6`k3 diff --git a/summary_results.json b/summary_results.json index 6db9fcce7..9a588a47c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1644, + "Performance_Result": 10.2154, "Performance_Units": "Samples/s" }, { From 4665a6a6dd5fb29558d1109bde76c95ac7d6e4b1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Mar 2025 14:25:58 +0000 Subject: [PATCH 319/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10228 -> 10231 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0961cc103..c972122d8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-17 01:22:14,953 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-17 01:22:14,953 submission_checker1.py:3260 INFO] --- -[2025-03-17 01:22:14,953 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-24 14:25:52,922 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-24 14:25:52,922 submission_checker1.py:3260 INFO] --- +[2025-03-24 14:25:52,922 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-17 01:22:14,953 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-24 14:25:52,922 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-17 01:22:14,953 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-24 14:25:52,922 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-17 01:22:14,954 submission_checker1.py:3276 INFO] --- -[2025-03-17 01:22:14,954 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-17 01:22:14,954 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-17 01:22:14,954 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-17 01:22:14,954 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-17 01:22:14,954 submission_checker1.py:3297 INFO] --- -[2025-03-17 01:22:14,954 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-17 01:22:16,138 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-24 14:25:52,922 submission_checker1.py:3276 INFO] --- +[2025-03-24 14:25:52,922 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-24 14:25:52,922 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-24 14:25:52,922 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-24 14:25:52,922 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-24 14:25:52,922 submission_checker1.py:3297 INFO] --- +[2025-03-24 14:25:52,922 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-24 14:25:53,878 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2154 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2385 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.65762 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.12259 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.441243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8152c0d1c..43c16f4eb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -946 +947 diff --git a/summary.csv b/summary.csv index f44429b3a..fa094e871 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2154,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2385,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.65762,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.12259,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.441243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 045089117..a4859d9f7 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2154, - "Accuracy":"acc: 74.400", + "Result":10.2385, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.65762, + "Result":8.12259, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 674c556550968578d79e22e805cf3471f4514132..04552984790541c03810669d4d2358981b041489 100644 GIT binary patch delta 3240 zcmZ9PX*3iJ7sqGplrT&*wlK`tvZOFI)@jVxifmcRHuf!4M8>`gGmNYivP8nzJtO;W z?9~vmWS4AFqPNa_-t(OE-gEAkd(OS*-rxVr|L)&ByjeL!&tUC%21*>IM>CWI2W^S` z4E%V#B$%;p|8DA9jpz8qz+Ta;x9H`$y>XH^#F`5NLs{AVI|Z+cX! z6p5%h*`=piE1`7>Oj^;y+?GCm*Aq*Ei+1q}x}fAiZ>sI?YWDj8i{{f9g7fLuDbBXt zutF#gl)xy@o=ziFZLiNq2~>k8#~O+sm^^yLm8w!=tl%)b5?KKfN!sTTca(gKPfF|` zYbr`*loKP1AxALK{>s!5xM<+k)$#^gn!{f29e$URr}fK-y1i~Sk83v5enIO8|L_z! z4pl+kK8WlIW9jZ7Ao+1yVZ3AvqY%&)493=a#K@4VO-`MvnN!ReB^Mco9E|%7qh%Q8 z?c2(5=teiTqlMBBb+Sev<9cDzx?=gpn&XfhYeutby;+|qi>cKp!3O2<8hqZgEC5`7 zme@s{{yz24opLOvMe6a#g(!%L(q;J)hnbpyUx@L{0{H2j`66e^F^P(46y+uqePYR! zvFHfq_oVVWQEg{~6Ml!6CI7gd?}j@ww!EY<5{&yMJj4~H{Oo#Do#Od!vXppqfb0?s z87R4Q&A2fOy$@0@=0Y18n;Gv+1(+9$nL-jTpWphRX{vf+lzKw(8Rp-Nnr-9lK+2<| z2ZQO;EohT>*B9xa__jsS7&qDJ23Id@D zO&;xTTy@g-@G`;9K27RY2j0@7XW#(<04#q}9YEaYV%T8=03gICEN1vV?Qkpi?XR-? zjaKX6t&_*fxqS7SJjorj5bY~N(OiQ}^FG&`XT7a+-c9d7&^i0g9jDM7#^a=I%}+-M zpVfor+Elh5Tu7E$MyRZ1wiQ{cp0=Cs(VDTcYQMX&k~a$LyDhbsEQBs)@v2KBj_GdqPh1 zoEpR8Mm7}b`S83V3v6fC+W}!X{K|`O{B$OyU`DFrwQ-zL2&YF(%LC z7j(9`q|!q{A1`{pPJ9Km;H7sdwsV?f4T^i0>XAOZ{H0DL4D=;mD{*hrsOv{Te1Z-* z0~h~BSN3ixwjJg=5mNG0!4rRvKW{+JKA4yfre?W=09Eyo@!8X}b zS#0M(aka96I?(XqLzHO1PYvh-+fNL1Vd7CNDj44b$B!;-K8!=<%-lqYQoWKfowO5u z=)z_mXPip&z1Yrq-kc6O6=u-K-Ei$VX*rd&ni~}FSk%NZ34@|+-&#oU`+a&>D?}vZ zA~M!}$pIU%a{BRv;eiZ)lVfmx#7)Jb7zN+%ABi9w8_H0KNeyxI?YM7x{Au$HFU zY$xfsaNEXSH=KLv`{J*o2sO3^e^&1L_wJdw2Fqn_!7Z1c$vteRZCd`#W1Vq0tHviqfzpaNIP;oa+oxdWws zsfhb;DCjupihZ#(zyp+MsTc?=m>Mg+PqESxWaWMLk2)#xk^Cp#`JNv>_eo8BUUA!d zMCnwhLjJ%o+^A?Ir{Rid;1j<|VaLPE*VeWMq$a5k*p_|;fzT1zWmYr(S<6)0mo=t9 zd>2BuG;|>G@qYc^{$o>+nZjLi>Sy`pOk z9Ocpl+z?HL{~5<6-mqXU$MXiq8w)3-$dYS|NQSPr3p!(WuBe0hIKQ4svIs z%X3ONa)(;@ZqR2%RXxbYQYR41dHukT<*~u5+O6zEJdd+cra(8Y`Gt|jZkCI+W*^rm zAX#55%n^cxjvk2)A16gC^noz&0g(SyW@QU4y$t_+eN_%Ei{!|@;-~oMcievXtU}FO z_d@sV%m!NpnA)yWE*7XXG?^O6J<`WwTDU z3YD~hk?Gre+)8i(=;Z@tjOZ_aL2Fk*SpfqC20`&h=byfOrH3#o3f^b7Lqa5q^(Dep zFPpddEh#G7IoZ0AOY`s=PJ&m=GF3m&-`*%5HW+@u_Eii#7mP4zcHIHS-IAG7Q0ZFjtgVh>V-Yc3D`O~5-Z`BqMr{`jAQCW;n@kaOJYPQKvC~6AK~6K zS(~vq#aig?=mQtN7g&^vbKP$vm)Ed2P8%gvV>vYH%gjk9Frn}8fCKE6F`f5ky$ZQL z6gvT{r15u6#{}Wtn*LrWYv>?>A$O>aSrPY#xK?kP9TF7nCxK1!D>}df{vMrm`OO>E zbK{EX+0zprCCIHY`6Cv@v2&94Hk`5RRd4`NH3Fc2d?!NkJ1Qin57A%NSx6Zwtdn^c z+ecUSREsxOjDkQdV#()5FA~9<2z`2ekanVfY^h?;xeb)%!+wxI&$EXw{_=OS(qiov zgfN+aQfsDiOAtdu8xDT_&j3Ty+1`Szit6(GVl^wY&|Vm}CQN0lPg^OkW|xy)@lM-d zhf*t4VFJS)CbhR$X&Bn4sFO^H&USEFt2XLC(TiPAg@;4A=HK9kmUCsHzvB%CE!oOZ zxY_Z72jhFbEsLH~6gsI<&;rJsV>Dpwy|xkt$LPQ@4oL5`rpKF*0)s4I8o1169qita zLXAHS9;Pe5lLEJE zPQcni=T_Q6#2T`T=WdyKbnwk4naL$9Esv@scK02cZqufTg4~jt!XGQoeP2G=_Y$pFg<)&v;JsY5A3QYIqEhz^YJwuSu+OM8JQ{NxI=q^TN+l?ZZAgCNM9qBu&6i81WOqH-VZKVSb3Ac z%r)n)XJ3<;Y3O&hxg)R9cL@0`Jmc0SHc7L}WkuDJU)m+3)FBP7$T;%x=T48!rb}z_ZIE5( zBfpc~tL2b0lh5OUZcx8;C=Fke(E~;V&;Vv^lN$sZf6t63!gxxHk?FK;(;!7iqqJC! z%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8uxLOzKp z%L5^A*~;Q2BSBP?^2~B=+q3c@EzHKkfc=RL8pml!j%irsq!5h}MzTQ2@OhG!+#C<1 zR6bMre;AC!BlDXxm^}KXH6GTk|4GR+e|K$&VEI5Y5(*N@0+@!%Y)R>N_3%Ot*@|ju z+|xR$J7z>A4ysJd=<_*ASBa2DdWsU#jMu+Lp32g?rjZ9PBeNqtq>)IJVIxVSh!nNk zu;PN|kIf@zZJ?n#-kR&!&Kvu%?R0I=?Wx!E6q=DW$7;OR5grF37!MhJHdyU}m$OI( z9sy}pDw1m14ry-Xnf%RV4N2eB>154kgv;HWTdQOdgsQD*$u6QX)e6h1RlW+!!esH}w!OC9TJQdgkQQ;eGQXG+ z5kZ=9lg|da0Tq*q2ONLv^twQP*1K=%oUj#{kSyayNM*qq=G$m<-Y|0p^X6Op7hHYJ z7g4HKwjgUx4n$h44F01zqVNWjN7q)y!gy9?#1OBg{G=x@rF=&!SJQ}I;^s8MC|OEb zDqb7!>U5;af^klx{W_?sNPkphe=pqg2794*@WpTsjFBNT-#}PpcD0xKxBv-jJS>A1 zxenMWEx3`9SSD(>^Nspi(6YXg&H9oGLG#UBLV}1ebtk`WV@k#5MwR64*l7F{ zP)h>@6aWAK2mk;8KL9(>nrrR~000n^eF#>6d2vi@?cKeFK6gvKqP<`5rW4~fh~jZL z+i$3**)W3HARLZo*ZYl&^LIO5!-$jqY}lWKvtYmRJ%}5>9{gwTTNwQj-vmKoqzPv6 ze&Z%d=DV#{Jh%y_{kR#9nT(@vO1Rup3o> z$q|l5<3aEy94w~6ELkOqf=NF~hacaJ=W*j;Z#YhmW%U?QFxqb%s@@EAg z90%Xx-0wzK%GWUbBm3}ixZf}{X$!RPmcuxSjA1b9FDA)p`0Yb5zP?G)(`IKOgK!cr ze;U(qdM4f28%cer@yTZsda)?sB+e^%2bU~2bqTaVeBI_x*EEF9w#@8ugyU? zZA~ZX&uO~SNq?IM(dcSC8wF9iUJI_0AWp90#naNp>S_?qM&s+oFO6Z4^v9F9 z@oKL%BGdJkI!a0xtwB0}wBPMwR?iEHgON?dWJAE5-O zBN9m+kx1%@L|mco-4g0ZxcKA!u_U~M;l2nz&WDS$)WvX1e4n3Oi6n^;U4BN1GKJ>E ztr5xVo2&jHyPw%fZ(mlacCJ$(=iies8r)pn>Za2)cLocktqLfA)=^3Y6ffKyifymy zd%n#{^f-wgk|-{q*C=rXW=ALi5{X2TNFSvTFQ)o`Se-DXulv07jOK)pZY+Ks33YMZiN6|+V1w^k=;tGh4Py#3tiA0e| zB#J2UMMv-b5)?^)xV-5p3GesWN=3ZqxoCjCUse zWrjP?`Pu8zI?Ahn;HjTO&~G}n>u64(GC6|FcEO-P<2BK^f{i1h0ScK$V#qX-L!$9T zp7$FKBatPLD3&GYYz}TNfH2=B?&s4gAZiNmM-apkSW7@Bn+OWvea^e}n&7|G7 zouWCD?~WcS+;p^ENhaBt^dy<6K{AO3$Rrvd6E#SGX1k0`k^smoNzmCGd>~V{bV(=5 z)RIgY)}vv7CK2JB+5ShNsh+f&t)s?Db7nh-W>%Y3NG9LSKUBC$ZhMkU)VRJR6E#RC z(Eyo517wnoeIU~+Bav0)eb3Qy$^`$u_yqO`GEjT`&ol2c0zuf$vVob#H78=q_!-- zh{*zjjzaH;BON+K<4KDS(IAr*Ogc;hOcEQ6^o6N1pAPFp8gJ}G}xV}IxUl64;z2FcTQE_zN4D1vaF(~K3n!3^MOe;e>l>=Lo}W=@DL3$ zS;4}?G{7XW(MaonDs%C$PJqcWACHnYn`3C{vQf%iONi=`D1syr;jLxOmYppm|B9{4 zS_-U0W!}l7b<^{5ZsTn#y#gxH2;xXD57Bti%tJItWd%DA(*TvkQUiZfmN|M@CqQMH zr$OFKdO^}5TXL_T(cw*MF9wRf@r6JNb;hSaS}~p!oz70 zT?a?OWD*;LFg;zX$W80&Es&>b_j2{Z#0PoWUpO0%vo}2SC#%;xCga()5&yY*aio*I zIMU3L4MwMnNnj-3=fQp>xSOZnP*!Zi7~YLmuZ%S2(Kw98$@f+7**lXU6DdKzuwfnS z&HLBEr+#!jp2fx_7^Q`pO}Al0tCvtNe^0{s^0)MF+Sf2i!s-2|o1i}oqU=+8TQCa4 zHvBT9T=TCmn=s8A0cE$X|HTgr`tH)kxkAi*aQq&?-fLc0(w#n|&5 zDCV>1&{5nNr&x%k+kaxq_&3`EF7o^yD+o-m!0-W>+L4G(NWn*>{c-I85FA4b3Uu(8 zWCUNl0i5!a4TEy!R5J7yCaq_2bnm?!ja}De-E6SN`~-ikUf+jRQ>hQ9;(+Q>aLOI9 zU|e0|>mnYmWj?Ipi8m6&Qy=w-x6>lWJ`hO^K#y_H%IEC)r6gIRy2y)pn$J@5(By;` z1ts(KVdPE!1*lF#{!Kgsj=66?LH)C6BZUG4b0`g8lkp`&2|Li5YwijF01%TrCRYK9 zlawYR8&alyWO)Pt04Nmz02BZK00000000000DyrMlhGzY4Zj%x0C;RKb98xZWpk5Y zClr%2CmI4V9FtHS6_aEq5FAO^U5O(^b0I28vk^lez diff --git a/summary_results.json b/summary_results.json index 9a588a47c..1b4d81470 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2154, + "Performance_Result": 10.2385, "Performance_Units": "Samples/s" }, { @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.65762, + "Performance_Result": 8.12259, "Performance_Units": "Samples/s" }, { From 30b2d3e7f13f8c6eda798571c325f4d294090785 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Mar 2025 16:56:38 +0000 Subject: [PATCH 320/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10231 -> 10232 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index c972122d8..1921daf61 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-24 14:25:52,922 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-24 14:25:52,922 submission_checker1.py:3260 INFO] --- -[2025-03-24 14:25:52,922 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-24 16:56:33,512 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-24 16:56:33,512 submission_checker1.py:3260 INFO] --- +[2025-03-24 16:56:33,512 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-24 14:25:52,922 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-24 16:56:33,512 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-24 14:25:52,922 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-24 16:56:33,512 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-24 14:25:52,922 submission_checker1.py:3276 INFO] --- -[2025-03-24 14:25:52,922 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-24 14:25:52,922 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-24 14:25:52,922 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-24 14:25:52,922 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-24 14:25:52,922 submission_checker1.py:3297 INFO] --- -[2025-03-24 14:25:52,922 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-24 14:25:53,878 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-24 16:56:33,512 submission_checker1.py:3276 INFO] --- +[2025-03-24 16:56:33,512 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-24 16:56:33,512 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-24 16:56:33,512 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-24 16:56:33,512 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-24 16:56:33,512 submission_checker1.py:3297 INFO] --- +[2025-03-24 16:56:33,512 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-24 16:56:34,581 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71445 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418488 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420429 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26127 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2385 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2321 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.12259 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.77158 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.441243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 43c16f4eb..a3147617e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -947 +948 diff --git a/summary.csv b/summary.csv index fa094e871..b55de74da 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71445,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418488,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420429,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26127,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2385,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2321,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.12259,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.77158,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.441243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a4859d9f7..2a6873d00 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418488, + "Result":0.420429, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2385, - "Accuracy":"acc: 75.600", + "Result":10.2321, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.12259, + "Result":8.77158, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 04552984790541c03810669d4d2358981b041489..eaf78c7ff57413e2b419b3257258118418b42940 100644 GIT binary patch delta 4658 zcmYkAc{r4B`^E>uSdv{BTf^AGh>;}*nKB|v*$HXJQudv1V;Hhjwi#hUWEokq3{gbc zx3Y~TgtAm4Lds6RsrUDNkN5fKezxPfKj(Em=W*Qkvtzz%Uh|HLJ*qcQI1mT`MDh}& zScQojOd#Bh?}9czR?mg!BG$7c#wj`t;tDU>83kj^LLOV7L~CS{)M}{%^Qn3IpoTEg zEoX(iQ>fhM+e*&cDF-etqn+Up=2As)wi)#tC*iJQ;gp+eRQ}_0cc>pBy@irDgoU%K z<8oXQe*et3-;9N^@x;T^He+7$L)PsDxw51=!*RrAX`;ECO>s!roT;gJ*o`H5rq34)~7t|xGD? z>CfyA)6dW@t34N^IyrzL1@JgnJQ zJZ`g6xbvifRg#rMhhdXL6W{g(M1{VcoG!4LObd$jf8%7p?bUP_HVkGJyXOl!qHtUUfx4TRt^}tA6$R-}G$vYG z%r$@iZ4sQGP6Z)%hG%*6*6OkA8+p$Ld#t~FD`(QdRG3=B^)$s9kqN*WW>ZN|B_II zfxSW3?X~N9DH+BpruOwN%+~HdJ&d<^+I6cNPep=UPVBr95>nk#w{o|QC`;i;tBF6- zu}k|_U^n~#%$9^H482xmeG%Jgd7UHE)|O(QuXft5snwAn3Ax)ubZI<$(+yj6`A4WX z{E*qggbBzG0B{3X0RVs=z<-9vcY~dNO~kU=;3AAoZ?Da^+(m?$v{bgZtdsc`&m}Gd zXwLXOB~Pi~rzAyFJSGp4?{9^jckW0e9(%vfkE?Tiy{VfBel33JvF<2z#4`^xa4TkW z-;7$6G5*~A%tq%08~Nyk@L;TcB%1#cHHr}RpvrMmhkd^TGaNiRN~YD5k2Wi7`)k0~ zxIUhgON#~@GNOv)Hbr}A65d$PI6-3m!IywVS3ga&RLn0SOOzN&zNe^zQtiWXYymr?NRa#q(N2%@}r(^wj zVZYVMi$9%K;?J(luW3wkF3yB!e?TWC<8n6rh0wB1H%R#+@o-h5bg zZY1iOJ-KLY^-RL$|kojEojRBG*~%^^I5P?j)50Kj`OBUV~_tm_5r~TX&kSI zW1yMz_l6}r>XlC)G<_G`XY*%!tDqRqXx+1;sj~;|KpnZ-z39zk{ZZ|^+LqftM z&86|BTu4Y&K`IhrhD|_q?u0$y#VHN?W}!N#j*!v|tJ{gl+}R-{Wbiv%dSczdU6g24 zo7N#dNHRtVrkU~L2j0-waqJ)-g>3OlLXI7rgtlt#tgl0tPAz~&V_#fCWX6SkPn|Wl z=%G9@>6X>rmA)A%B;}DiBxgz1Us}5Hu_%`7b^a^ewWw9G*G|gu3&HBaI9)SeL6@9Z z|LI3&qqtJqW!WPyab!`&rZ+#57OC{&GG6oLy;Clwi<*x6owkjkRBxyyx8f=LXWmNV z`(wHeJw^t($~P7!B9-du`tGP_4~4O8b(v}A%wD{2Pf#X@PLx*|Xo8cg!LQ-*2BQ+@ z3*8kh6N#piE-P~K*Xvnm{KJ{Q1s;6!R#$%R;09OnacKHD`Ks;{2b2LfY2DDnW+h|f zKcy_h@A*nb#I_~?3D-YOMV%mB3PizMd$*#BZb(yryj>Jv-n%x=2g9S#Jh^LwQQQ+! zH-6YJdSp5vc(wZkHgwYNWOJyACK#xg!+PaTR{2^}O-ef?;whQzXDlLG#ox1=l+vc$ zzq|W^O^)l2EIv^~W-@Vhh20_2nM?AzuEy|a=m;S5Y0O?g(GxoYFdAcwPazzz_55c- z#71n5I46-0%GYK}s`#iCNXn5>Q+YmRWiDBlB2w`->sw6OFSRe^FWSsusrUS}#1uYs z)ruL*+;XxdB#B59J)=zv8B{G}(JQK|#*C76(gq$u8!2XAkbY%N|>1!{Ons-J1N2KV?eJ7Rb@C#B&`pEQ{GSej2AB{KpbA(_yHEPdWt z>BhVO5|m9lCzcD{pQ%hkk{k%~9g*-8!_tskwVB^?@NuF|UC5Z7*P!>8;kUQy*!z~6 z_Y&A~8ht+IbiT=~KXs%!+yYPrx37&HN>?{17A{QQxLL@6=RK8^0L5ohbrA{KXR zQTO!#bk%3xktQZ`y-5AREBdeIH*+B5FFPE6 zbl;&qfP7Th#((~_-XL0R+1hP%&i$%mrb{O}#=z*wPDlDlvBJ&biwT}-ee5z~WCoce z&P)64RwVO;5J<8%hE1EjQv0t*bxKcw%_ljT6p~OPt7cYtnRyxgU6aA8PL;L%D49m; z@*UEZjDEirS^YiJ+4H3ee)E>NE4KDFBiHhk4pR2sNI%jOrOlrfF&7C>WFb|$e!3yz zN_g*C9HRg5Ww@Q}8#F1^$T;WS+4|rqx#cP-@)wHVfRMN`L2(}#6eQrfCNN6OHifF| zOCwFR2(8A=a}S*v9(n=;in9_tb-X^G7AdCS^AfzGFT0g7C=7_|R`M0#C~v7n#mk?C z9Gbi{^r3cPDE=zvqzI60wR&^daim;+S?L#3pPWcV>JEpFZa*xVcMXyf{EItT)<+i^ zYhk0Zt-`5q46ll#yq)Rb1!Q2pi!*PFd(sGIiA4ZKg!Bn9=C_MXDfmZM5UAy$Mt+@Z zMME$9&tr4Rp&81%_Z4tHfy(NSE{W29(q(I76IouMMf#vrlrlYhWdpeg#h4|EqJdE? z739!V%=h6%+@B0k=@}5Qne_e@apG-0CB)t&-Fl){dwsiaqSnSY=4CQjL7xvRl@feF z$DBnVV?AyB4^%h{=mB#l%_Kkp@PQ8qkq?MJp5eIRZEJ?@uBQjCc3hyB+}``A_YT=N zUo9`}p!oh&!{Q7xuAwA^;n7#ofa2LdZQMC6f~>>OCU^m^6f(dERuPH}1@`VcQ=)hD z`Bo8h#-=}vOy@}RAoq8Vju3kNpNZ#Sx8*6Ll_ZVBuc6mouHnH~W|j5%@}yFJ;{Grm z{bk(vFXQ}Q#zTb2;a^5N;<{nG{{%WzB3u1-!@;308gcFQ5NYrLJ8Dn8I<4R^?7-|< z7}l8{FG2rMKP7l_zhjS=ATz*|ScVnV4r_pP-JgtNKmF}tjex#Y>O}^U^&z6k{WWgr zCi=L#74NW80W8_rmowR8ZiY@PNAuGA_j8{+8P-B)v#{CN|6_BvRG3LHX)+;%(IPVf z3lGN@&|_&yhL!>AxHy>xP0dQt8v+upl#jj8$DUm z@{)vCoYjY;YHXp*Xrv9u=EQ;54S z7ooot6u8gNN7GB{{j#z(5|MA-prCO*!hEW~Wy4Y(^Y#Kb?J~bG>xXQ_O)VK00r7t|xrxRD@EJ{PeQs8yySlkHBjaR5|C!%Ja_4{X< zMuxzQN_<$+_!+6>2>8jOfFIcLf3^qvyNYotoC&b3SckXWIAwemj0KiEdy47HI4nL2 zvkLa2?TpqpVPI2S-gcmWwvIzK$|SC*deoj}yX5u6o$4brvI5N*$S3-@9?Bo(ZL6CS z{3CzUWWI96^2r@!%n3edS9$q6EsyYSK!@v?qb`{*n}KX5mz1 zlD~cVW5A{JC(g)47%@EKI5r0RKitFq~?;qk%A!1G)RrizB?V-nj3@#Pl&#qi## z<8K>-sPPhC6Wb#j%6xtycY!OjlCYwN-Okb^YDD2!V&-w>qm*pzW2Xo|$Q_r@EkvF$ zosE2jTH?l2UM`$^OV(;S?3>PS22F>gGW}zD4O^|$M#=gTpZX{s zHYGKNPH0~k5F8I7d*JH$(0fg5Ci%^ z$7qfk{rDQF?WlZ7V=Iy+xO0gX(K@4X5H-IgTDx;cJL(+Y+49KFP~Dst=T>wN=1z3)Zs`60iXK4xAtPA#QP_|b*hC0hBg#S) zalF^!*7O}Iw+ei(Nf4P#o=cIu5pgO&xq1$H%;E21LJFN{*yKa zZF_=$F>Z5J;&O9GJjJhxDYw)s2^!vyD=Xb)eee#2^pN(D8a;d;3hh=QMSIEqq02o)u}XGHG(Fi9ZxRc4%qks_4ETX5Ix*7x0L)FL0hA zw>MtYbTY|vxy$Fryitny%IEjxq8av05qEU%f^){u!XBmtdFAovEls6`I}L-Eg;Zh2 zXJ=6<=v{LopN@8tNHad6BB$JKdB#W6Df{VPNK0W|>scNb&@00`0dMgJbrF&;!S z8PETOjMyw=%h@3e0EBzPf`TKwF+oIES$?9LtO%pJDA7VxfQXjm7X8zEC<6ds!3Y5G n{9A`UM#l(3h-V>ue_kSnNpW-h`z$)<7I9EkibWLq=g$8DLj|fy delta 4631 zcmY*dcQhN^-w%RRYXwDYLekbQMOvezHZ8SBYentxP*f3n6jceaM?+DgRBWZRirAx8 z)oy9bS~X()^m)(kJ@0$}xZiuuJ?Ea!`kr&|J-mN#zp9^#&dT93)Efu@1cI?>5GVFG z6)oD*vp}9TWbSd;13n99u(%c|5tC3vY9WdBd(_@%reGKF}3f66+&5|7-~uS)G5sSt<{An?rOHNk%r=@M*eS@ljTbcrR)b+BHy9G z0*QO9!VcFe(TNE?BTYrg)Dl7jA;lq0Xiru05M1!t#;x)On<@L~8v_dgUl^uLExK8@@O19g7JC~~1T!q~$C4F=k3QK_o3ftAt5ZsUPAb)1q8 zyrt>)mj;Fc4WlPwX*DxGa#a=9%cY^kH%G?qOiuGUBJh{(hL~I2JFGMUJYy^1w*yjm z61PI(Pmz4cBg>8;cDoxr;=Cx+rDlF@o`ncFIzD|ZR_#s?FXU%^?jxPwJxNTUk^^8^ zs6!GnD7^&+0_whd2&8nJ;H2EH6IcNOKb&3$Qfi~)R2b?7Oc5Q_V>yCM570BNK+4kR z5h^NAJ()ozn>>^hV=Wr|e6bUsVcEr8fLi4d0iFl7mnbQYc!ii=ZT(&33NlG%4cuaK z5I|`BvVmBy=RfwybsyioAul<`<9oN_oxlpsLTF<+e_KK`na%Hx()q!r!>yKv#&4C= zKyh*;8=F?GTZ|~7+UU#?T}}JSl=|8db-%q~kAAc$aKWy<45i-aLU*|McA`$q0Bl&# zhg+2{Un4p6OI%88R;V{23WAI+hcVVj`%<>6#$^F) zAwkN_e|Bx0zH~49opUM>Ltb_dt$t)&%#`(3=#T!sxfe{HDXkYqHz|oXwTdGo>63gl zY2WRDfG-%rRE7nZ;0Pe)y@@#iL?pw#`iPLJ7<>fP!>&yfpa!|gSUQ1R19<(udJ0)(SJ&2AcaQxg{$PRGJ%YED8owC_-&pj?fRE6 zypC{_@Eei{E9n7QY-QU9~y*2>N|< zgiJ*-aOzL#yIwcFy-@U|g^U}}Y!qa{6nzI}h?;#4;&LD0avZRk4T?VxFH1VQgLgq) z9$EgOG8BaR&DYNyC6|7usZRQ8C*itqbb#0onBuc*=pVNY8#C4Sz;eaR>IQ};hTGo) zOpAq#Aqh9HZcN-YRyZ|CK5g+D2?aPFI6FtX6de`oEMqpq8Wk=E?X#70G(?Gc#j{3W-NuQ@e)MWdk znfquKyzVs5dRk?FoZg10XYW1vd31uj^M6+h9SyvK5UUOw2t~-}M{cAf)U{vjIAld%;PD z7wGw^-44=+F~-2t_prW)+x-E&atwc>m-fs|l9;IrPCH+JQ#{tYojBe;TfNom{ab7x z_*S)HYxDx&6eC{@nf@lWPKE5Dkbv#xpeZ=B!(>O#DFK((vY?9YepjVIuux!Ysq7dH( zT9dZ0Haw#i`%T;nSs^3JlkuNC)oa!Tat{ce>KiUOxwyOI$HJ}yo#PgA*|TyS&OB37 z&5BN2wH|~H*Do1Y6w>I~s~4mvhP1l~oGHr@Ba$>ehRJlp%q6NL{gjbM@9y;Q& z9n&}~aNNgax76w7pLGIZ;GcNagxz(6FGmG0;x*XPP%kR9#U8z--nKA$LHF3`xMcCf z;0yKjA88gi`!d5UlR&2>|LgA-CXj-y2SS3(uyy)mkj)%8u2$@s1~k0*04W%7tO8x6 zJJy6Qe)W$<2BEv)=;6h6zc^&}%zdQbfJc&M$JD7Vba6eGDNermacsu|d-i7ud0Oz~ zPPkg!4GH;_ntLstvB%n5Y&-vLS)ewOYe#KbVANE=SD`%6Zz^9^IO%6f$ zi2Kq-F;f3@9wmTLbVz+(8YRTxZ!&RubhB%AyZ5IZT;Eg;J+bt|al2LlR9r!Vo<7ff z=R0zLmmgxC_@sbed&0Hd7~DZL{&0xk>6#jWwKmmeIo^m1w`uHlLAkyiFWwwRDA6tY zUSg>qcg@JrTP|x4YQ32*;rD53UEHD%<11B5FtMipH02aDbNEK;3g#>TEca-mR^6fL z%uqv%gqPK1jj1~+wy}Cr=KXdIT;P0daOaMG4q@@KFa`Qe?MBuCn)Qi626rc_Ip08K zC)3I5E|GZ{kfbXV<^a)x4j&2*k`tq)h+s{4FWC1Mt(=*vPMR-XSAkL0EGd#${4&qn z<;d$7)U<6kbl29TuZ^3g{Z0#ifqX-gv7ST@2byEh;(B@8%2A-r4V9dtCs}^XxRXZm z>rX4|b!f5o$iy{GQQ~f38^#HsgQpNRpt|`}UF|*e5}+3YpR= zB1EKES0r5FrfIv+4{14DM;jNy+gue#9$AwNg$b&MYsG_lgHP!;h1lkT5JnAo_O~&* zUmM;*S!6Xuh5;E!DhE6b@hi#!15|O;3LjXW$CfZ$K}W2Viq?L=u0Uk_y+uuU2`Rhd zV$JC0W9eM)UhFjRgIpLxSI_5eiqWcrD{6_jywb%}NXOB1Bk3_y;zC#`Kw9UoEPUN5 zV%8&Z(zVcs(fdvu`C3T%Cw1orPNlF4$F-98BiT~}uQSFR*)YU|eMUBqwCUUnSnni= zNHjf0OGf1{K8JX|?wX!%=%vs;3^1p^omLukp|omuimf6z+DAky(Whu14fxA8^JYbb z!l-m=*7Vm&5kgxGe#p#*e8DxK=>Ku$JD>bNWT`T$FO&?kW{O>X4+mfsA^^H%^N4Hc zKgf`5BBH0Pqp+pFuuk-2ERnM8r7C-@PzwUNq(xvD=Et(FBXp^B!Drvc$B%K9Mo%?z} zbSL9DK;L+_yCCyjbveFR$x@wn7agWm6DB`GRFlcA*cp-l!^5G>3l*sT13%YV7YId~X>FDl1txN9L zTPUv&gBLYT8HWQ##?@prQPlQ~qkz;7D=H)0v*%_o6;wvEhSqMsRE_Tx8Xl%Cx%~=m z)dJ>XeD}!p&-Tgw1Eq)lgBjoI4$O+S*c+awJO&6uG`LAy0+9QVpjevM4kWh@1 z)g@F$Z;UVi-W#Mum!LE>BAmmdL6IJ;$$_9tpQh!emYoiM)syFWEuaXO=aYJHBCodr zxBiVTt06Xib%J;oj>b5l?~wKcAoCb!Xqv82qXUHaM%?GP3W({g2J+DB^Er_)`d&-m zPk}PJ`f0jAS2JtE*ou@Sx@R0EcF)ATYxTo4%(Ba(k$U-7W+^~#FHwGE<3dVbG7~-C zynW-dOz1_lGEp9UIKYso5826!tnxQhL%@2*OA%3`I`rtv=GKI%l`VnYjHciL-`NeU zxa0IoPxcdK>x#=m$dy>34W)}5G&2tKXR@s;=~6j?>GRDU4{R?1`lXu(m*=T6Z4Lx- z=EsF2FLGZ+JW-N#9bZ25liBja7FCPrgAB5q=Ns;-NLioVZ1Q7zct=yb2^Do5XzYw>u z1DUuHli~33>nbcQLN8@bm=a<1%E$;O6Uh@v`g|>!P{)jg3twbCdG#RC{liLc^6K^I0cdR1t5{t;S(3w zw553)E%Ch{l*(B`tNYdp+SjQ5?Q}6M;q+i94jbZkrK*&Qsf}F@91$*Ls7Ok76RS10 zXkV^SBIOXaNa6JnPQ{^`J3Tyqjed*>j-`n!pWm~E@M98aV@1&yZtI73OBW~66f@TN zYQcJHgp=p@UfZfbeU1UigGACp-NJLIU|!jv#!9f#ROxr$pNgm(2dkf*nXG9w+8w}L z5t545W!-vftHG~|T@L=)U+tiM_U*Ae*Gq!G;vTGwT~TCds+Re@F)*qS+oC$shoM(5 zj@Pn*&aJeE2%#IYisv4fxP9iBO*D~6l35;>Pv|5b7;jBY-(bg{waaj~S~0jtah*4U z9<_3O@bY_nCEN5P$8qX=&%!rzI-{$&76-n`DuzGHr)%>Mm7916?d-t`BdYyH(~|hH zHF#Pc)~od-$dXP$Ehp-w z@r-bt?F}W`Ub_fvMq@ah(zcY~$6@kd=Ov{D`$-;y*@tMs;2O!7_|g#B+KR9LWFFVE zHdM5|sd4c#4SP;!h1g-}GR%fp4Np|zD{bFa$~LG>Z~ zTjliwdgIsgn$E8XrxOb~$)!3oxOc}D4j+~838=KwSxB3CRBaL~lkvtU%5{q7vOH2x zPa1b-k%z5)&tXEF0rSencCZBlVcs*BK#3{OJcyV`t5)9L*r%#WETg*NnK-$xhb%cw zyLfnnPCbLnZ0nrD6S@*f+wp%Z+fm%l+4jzYOpyPNm1LnQrXx?73+`!x4w;_isjd!> z;(qPxHQDFvdZ}?jot%*)TeN7suF;#m6f27&6#brC$*5yYTgwkjt+GF3%^EN5AAHgA zBg=4<#2`JNEdAqZjS?b5)R#1|(ly;dM>MQ^=jn8slKOKFQwyr4HZ5{cZxQW;q?Z<8e z-tyLK-XgLPJwU8caK^Be#g0c^-w&ZiZrko8?c5Jz&;PV=CTuUD|9g`MVfRGX2n(EQ zAfVeiIuZ*P6h$u@MsBEBR=ea6rQkLrWwekLv=|E*`tcDnEUdhVd*=3&%`2ri+9~yq zHaSD3>gE_ch;`Je#5!?SuI!NF-oV2z&Sh+0e}NdpvYKCTeyWKwEu%M+XuC&I$ZcDs z{(OtRee%7c<|=0X!QJmJH{LS7nXy%ylAECz7~SL5`JIA(c;A~bp^wR2Io-Kc4!Jy* z_kzOZ`T&Ta(o&}vB@yq{fJ1y_fX&j)Cw1k|1!P_2B!_AU063R&Q<4@*&j_k|xyz{R ztKO}X4wpGFX@#P#$H}IrPx7;)PqG3i6B;X(@9xqPT!bPissH_r!R`v*<@(>}j}ZXC zaq%Gf?+6WZ#NHQi`kyAleh@LG|HKIZ1h`4~_&#$(`Cx5ftXLTsgi?kZYrxHhMZuW3 m{%S8w0RTLp1OS--d(K4-4P(Q;f+ Date: Mon, 24 Mar 2025 17:03:03 +0000 Subject: [PATCH 321/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10232 -> 10230 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 1921daf61..9e3c50408 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-24 16:56:33,512 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-24 16:56:33,512 submission_checker1.py:3260 INFO] --- -[2025-03-24 16:56:33,512 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-24 17:02:58,854 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-24 17:02:58,854 submission_checker1.py:3260 INFO] --- +[2025-03-24 17:02:58,854 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-24 16:56:33,512 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-24 17:02:58,854 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-24 16:56:33,512 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-24 17:02:58,854 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-24 16:56:33,512 submission_checker1.py:3276 INFO] --- -[2025-03-24 16:56:33,512 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-24 16:56:33,512 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-24 16:56:33,512 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-24 16:56:33,512 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-24 16:56:33,512 submission_checker1.py:3297 INFO] --- -[2025-03-24 16:56:33,512 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-24 16:56:34,581 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-24 17:02:58,854 submission_checker1.py:3276 INFO] --- +[2025-03-24 17:02:58,854 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-24 17:02:58,854 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-24 17:02:58,854 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-24 17:02:58,855 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-24 17:02:58,855 submission_checker1.py:3297 INFO] --- +[2025-03-24 17:02:58,855 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-24 17:02:59,892 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,14 +26,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71445 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420429 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26127 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2321 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0642 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.77158 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a3147617e..a0394c20d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -948 +949 diff --git a/summary.csv b/summary.csv index b55de74da..48452d293 100644 --- a/summary.csv +++ b/summary.csv @@ -4,14 +4,14 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71445,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420429,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26127,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2321,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0642,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.77158,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2a6873d00..7c62a4f2a 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26127, + "Result":6.00489, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2321, - "Accuracy":"acc: 75.200", + "Result":10.0642, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index eaf78c7ff57413e2b419b3257258118418b42940..06105a760ecd56d52c263e029460c29773e4e451 100644 GIT binary patch delta 2678 zcmV-+3W@djPxeo+%>xRv(ek=+1ONbf5|e)j8I$k>CIyHli8Hp78w47EvebAYj3+c7 zm`>9+4U&g6O7g|P{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!` z50@khc#{?+mrr8KvOvgNwy=1?NDx(|EVW$Q_N**Oa{h~$;qu;hYfkM$#GWuT@y-iqtk&Kvu%>9lRn?Work^O}+s$Ev+n5grF3 z81@-`Hdy6>7qdtO9sy}05zBJk3~6R%sr=1k4N2eB@o2?ogv;HWTZ`o^2vu9rkX=M=su31tqj(jRh0EEK+w_`t zW3~IwLYl|P!u(=JLx~{=TNw-ENtqHuycY74p1c(D z9jRPRBYKJJ(+Hzr31x|Rt-Y($kunX28IAUPj~2b1DSQ)^|%1M1-k3 z`E?spD%LluByYz?!(UKK0|XQR000O8002J#y`(pf?~{56TYt$wYwg{=g+6yny+(V# z-c2XQZ4kxdaJJu2O|xMHvq3l<&#w0y7w7MGyoM1c{n@ZT31`86<9iS{em(fl-nTIN zBfbfO#7Gm&;{C==lFWBot$1(~O#5*&oCmY?$S91a{UrSwUAN+S6!eG7py{NgP19*j z`{P+-m0>rkl0qXKjmCrEO*mLggITgl5(Sffk`6z<8PDU!!QOD39?R-6qF}V&I8?hw zziFq@I@nwGeK`)k#ktQ$R?62f{3HAFla>iz9QXJ>3Dz;KRjqmoRZ_y_VV8 zWWE~A=R1S>VkexG%amfy5IrNli>>^6o0%wmV|dO+!x`;`EYTTx)^SW-?JDDU<7~C zTV~Vp3JmDlv~)=) zxN!+#JsKvbOD2LftghMW^B9;vW$Ut*BCD)pb>8ihtz)g+0&^QpRe&b?MjYw(AsSD* zeTW9ptl;)x8bFh10L?PD4@&@Ombrg@G`M}BrAs;?XoA~^guz0VrG|y9uG#GK_mKHa z-9BeJ@AgR-vu($>i)Kw-fF?Re9BKC<8c*7Nhz8NDVE17fK$B{J9zmEpL543biCj?FK`;ai4HJRN9X3d)I zK7S9H&(!a8rt^NEbk?+#wu%40W`}DKP&;D zS!VdrVEBQSF6jhp1rXI^Q9yrY6~B+J+3)ihng6S=pqBb7K`FdG%2Iy8Zi%)FphVw@ zBfUOE<4dm((IAu+ygp0=P?GV;-4b2q^5@*AX-O)<>_fuZKame)W4;`Z3Ql+RfD?<+*9Dvk#{(j;?&@ZzJ?Wj z4NHBk!4-P3LN8Y6#R|Pxp%*Lk5`|u(&`T70i9#=t^g_6Xb^~nMnU*HO4p|08@znS7 zEZdf%v46xVNKH?bs$C3S?R+h_^Ps}SG89SuVZU#v3I#_>gPiTZ4)2UqUFx=U)o&&H z{V56o3RcjP!E%FjPkxOQt>XW&?a8X?h^gbxmG>sX_hsHY8F9x0|EUr?36CoA@K@7& z$$tCPXfg3k`lPQGhJBrUFp~0j*=JVXhuJ<5$A8bS^w;Q)U+PC;JZ?;nU+hO=Jg&%3 zEcXsJ9@Rv+BR`KE$%e>(J5jT&V_e#An5bh~>gcJj4PCFYdW5u zp$j+{HwkEddyg#Bmjv!5m#qD{d!SwSIItHr)%V&GEPHuw_^4_4l08CAb|@^qY*Afj zdVe-ML6)!EzVx~KGVT$b*xueG>FAE*xO?Ls(QVHZ^WeqgdU-jy7KkH#B#$nqv*luR zGp}brM@*ic=evEg@Ao~=5qr<4<7YDwqpC-2*RzRh+I>%+NOnV@C`5S(JQpYSI_AbF zp<;_~OIPybt=_Zm&0vfV{{v7<0|XQR0F&}27qj6RLkI<&ynhQOlPMw}e@??N5JmTV zMdW?rB&9s8B$WVlMI|JHir889w6%yIvJGkf9ydwTR?3Ry_})2lXXIkBmre8w)(Ndj zlqD&OKrzjQ+Lq{}Ud9h7a-J$q8?B&32XJUn-4x6arY*c#ZNPc~E(##!2s0(xd2a~D zj_p8Fmjs6j;znCZeJJe~e;dla=@xLFrgvC^r<{6Qg^ut?trJPH9wANQoJXU{M7B@$J6n$P2O7U#Eh zc2Clr%pbpwBd_|;KsgNgH}M4Mb6JPJOBZLA4v(fUpag*~UK^wiKH;?ZM000nF z000;O0000000000004l2W|L1QMFESGl_nw^;313~c?19eC=~zz6aWAK0000000000 zfPod0(k4L-z!?Alcx*3oba`xLbCY2w6q7V38UivLlTjQMlVv9m8=Sm<3nl>o08jz| k01*HH00000000000DyrLlaMDq0cexYCnE+LB>(^b0DqnbBAMPQQE3>)bzH=f2M4K40ga=UsE&aBdh85KA76R*Dt` zff8kaWDzBRBcKjxP|rHl2*3qlq_MsWYLHtfc=0i&fdnlbRr%J>H@W<}K@#(W+F<40 zf0dMq>nYcKGJdU=aRS+ym+ykyaSi3`Bz{eZKddsS+FTYoP*!Ge5S ztvc&7t2L+^Jo1cXS-+7fA^!xu{|O3pf|kXyv!Iy|ayDsS-YKskw99KqNjc;`ft71a zxak%hv*1zptNK$(#3{I1u~|NaLbhrG(JKyKJG6Gx^g;g7)=C0wz2(w;oxYlkaDbZn z!e;4c|8YCS9&1s2Z>65g41W->bh{?6 zjcwApGJp*?IXij&@cd7?2kQR)jiL2!lqaboR6f}hZv8p+YqK3h1sp=Y6kr@dMcp5> z+NL)N=X)4yv6V*rjt?HAJ`?=tH!z%F-w`Udu~5To(Q%bJeqXh4{@~qj%fbD6B1K)s zZSuo-sXMKe`?@k&ClSwvZML;NrFDax&oRSTxv#&QiR0y&>6`sInIgfxzzd;Ha3F9= z7)F|k@q*_@>6nGhc$pyk$vJkkNQ8frFJ|Rz3pS$Ih!Miq*@o9)zcgpw`A^4g9uTl- zPJcuix0UsdSe<1Jc`7;&Q54yjqVm;G$f6~?&!MoF4yVd(6ca4)7X4w1r_1sRaQcq; z79WUP;A#+VZN1mDzD*%f*k||pLq)}d9xIrWfgTApdE)sERQ?gq{BJirM|0q!Spk)C zC+gg+iQTTJ#S1+=n(*a^_IY-8d(rl)rFQ~8Z3oWy5pUZ6hzm#aATAC9g0dhGNMz@j zfg&fR&vAeR5ib(XF>G;)k&cabVBBT+EYYDe2AS+AqsfTd^tbxtw|kZIf+o1R*(>pf zy&q=++u?7@Db2n;%chy|9u;2Dl9wW1dRoxJuSzs^ zU)F}1OiRwG^IAS4w)Hq*G-hm!KG#C$FW+q(YJj^vwUo}YnX%xgE2HR-PovEACEzICO3p(6xW#MV|yB9X@LlgQu1eXK+C+efj z-N1^DAGqgMKP=rnr@>ESXz~KBhE>=JX52tPN{lRR9O78{oAw4Z$u zpo<~F>%cQHTd)EkEAB`NQL(yNyBscSHJ7N@s@IzNCXfDhsb*`5bxg_L;x zcM2b$Y(_>Eb}F9BT5496nwNaj1-3N0vA!Z7-`@^0hnRdD* zHqkZ1rC3zzrzVtzZ5mUU&{+k2*G_3rkpP%T=p&f>MEK8y-OtLKV(|`F+{PPCme!t* zH@cro?Z~BTn?s3+ycn)juErGBDlOPOlB+Lq1BXrtESbzeg3!b7NLbiA?C2P>ObY#I z@o2B_zTxAKEq9(=+q^iKL_b%4s4SjamHoDLW)ebbt&Rm>omGyin#u@160*JzX4Ood z$_N2DRf5sO3s@BR20n28t#Xn%bOF1=*!i1Lz@TJWIB?`DA8R)B1u)PH(`p*KTVlwY zjXTk?NQOI+rgY7rWr)1h-;De}jGX^xoc_be!@_uf5tokkMvnW(sTO0xTDd$^f9#2# z7bVZR#4$YcI7^!_N`M4c#`j&Js#Y#near}!8WE$a(!uB~p;6Q!UW4FH-ydz$KK{|L zLC*Z-Wor_+#EpmT4D5Mv#(%rMn@pIm6|ClLX*z5|x{xg06qz0UR=1>b^%y&BF7BKy zeF<~i03zNYreq=`c9%D||d{2DaSs3Mg^a_@s zXzS|MbCu;DYV5@K5_{+V;Wz=)Ui;7#qPr&{ z{?XUZC!!BaYCEEiUU{Xu#--kIiL62ozTN)Dd2aW*`kaFY-@R$Nyi)3hFGNQPW$3y7 zX~OQM_O|Ne^}AQ|icPjpB}Awt`Fvpj((y+)<=Tw^=eCR_3BxE_Xmie$hfY^HZ-xH* zX5y(ZLW@*H4E#(iUKAij@p$&6-0vIJ$+8181FIYTzw;$tU-XjijH?sn@Q;>q{S~H0d+N1Qs`rSDeHB<*iD9xnal~wqPbU7=#9~W!UYN%XLV1Q zMtDN<>Bp4{KA4Y=St{)~beo{;59gC(Eo)%`xJzm-j^``lFk7B_Gq$qE&T_Zzxx3s{ zQ6``zfsJdD%-yQ>67~7$fxN(@ec*uZ77=5 z9E_msg8W(LfMuiw{ExAUfk09xK_JB)tNj}(sX>5~x-~c#*pU|gXXbX6KTSCh=(j2V zjg(XcU{u{g;;|A4bRk^td`v=kz~+cfRR910 diff --git a/summary_results.json b/summary_results.json index 24db06e49..75067f756 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2321, + "Performance_Result": 10.0642, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.26127, + "Performance_Result": 6.00489, "Performance_Units": "Samples/s" }, { From 5fe6969921c64244e5f61a7ee9c929533b9d495f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Mar 2025 17:05:38 +0000 Subject: [PATCH 322/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10230 -> 10230 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9e3c50408..74234c916 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-24 17:02:58,854 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-24 17:02:58,854 submission_checker1.py:3260 INFO] --- -[2025-03-24 17:02:58,854 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-24 17:05:34,080 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-24 17:05:34,080 submission_checker1.py:3260 INFO] --- +[2025-03-24 17:05:34,080 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-24 17:02:58,854 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-24 17:05:34,080 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-24 17:02:58,854 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-24 17:05:34,080 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-24 17:02:58,854 submission_checker1.py:3276 INFO] --- -[2025-03-24 17:02:58,854 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-24 17:02:58,854 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-24 17:02:58,854 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-24 17:02:58,855 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-24 17:02:58,855 submission_checker1.py:3297 INFO] --- -[2025-03-24 17:02:58,855 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-24 17:02:59,892 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-24 17:05:34,080 submission_checker1.py:3276 INFO] --- +[2025-03-24 17:05:34,080 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-24 17:05:34,080 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-24 17:05:34,080 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-24 17:05:34,080 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-24 17:05:34,080 submission_checker1.py:3297 INFO] --- +[2025-03-24 17:05:34,080 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-24 17:05:35,059 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index a0394c20d..e2cab550c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -949 +950 diff --git a/summary.xlsx b/summary.xlsx index 06105a760ecd56d52c263e029460c29773e4e451..18f9d186def39e823dc32e269ec6ed45141dd1bd 100644 GIT binary patch delta 313 zcmV-90mlCJPxeo+AtD5me1lJuB_b<-JX%0jl1hNOq7o8CRO~E!np(sU*@m=#kCU`% zD|N+keD9pOGjh4u%QpH2>x5Pn%90dCpqS=DH5K|;FXKlPIZqX*tyWN>130v(?@DF} z(-z*WHekH~7X^@VgqaHMyf*}6$95p8OM*iMajUJQK9sh>hO%$k0M66&9!v0llv7V} z4~)%3L`NiN6Vm+Hb^vgOp#=%bJDepMK6^b_>25X*%7s%3e=u;S delta 313 zcmV-90mlCJPxeo+AtD5vynhRmB_b<-B&9s8B$WVlMI|JHir889w6%yIvJGkf9ydwT zR?3Ry_})2lXXIkBmre8w)(NdjlqD&OKrzjQ+Lq{}Ud9h7a-J$q8?B&32XJUn-4x6a zrY*c#ZNPc~E(##!2s0(xd2a~Dj_p8Fmjs6j;znCZeJJe~8_K@v7I2=XcUXddr<{6< zkHFZBL^L2d8Qg^ut?trJPH9w zANQoJXU{M75>JPJMBMSrs LoV Date: Mon, 24 Mar 2025 17:23:16 +0000 Subject: [PATCH 323/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 10 +++++----- summary.xlsx | Bin 10230 -> 10229 bytes summary_results.json | 10 +++++----- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 74234c916..ff8b25a20 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-24 17:05:34,080 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-24 17:05:34,080 submission_checker1.py:3260 INFO] --- -[2025-03-24 17:05:34,080 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-24 17:23:11,590 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-24 17:23:11,590 submission_checker1.py:3260 INFO] --- +[2025-03-24 17:23:11,590 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-24 17:05:34,080 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-24 17:23:11,590 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-24 17:05:34,080 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-24 17:23:11,590 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-24 17:05:34,080 submission_checker1.py:3276 INFO] --- -[2025-03-24 17:05:34,080 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-24 17:05:34,080 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-24 17:05:34,080 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-24 17:05:34,080 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-24 17:05:34,080 submission_checker1.py:3297 INFO] --- -[2025-03-24 17:05:34,080 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-24 17:05:35,059 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-24 17:23:11,590 submission_checker1.py:3276 INFO] --- +[2025-03-24 17:23:11,590 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-24 17:23:11,590 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-24 17:23:11,590 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-24 17:23:11,590 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-24 17:23:11,590 submission_checker1.py:3297 INFO] --- +[2025-03-24 17:23:11,590 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-24 17:23:12,566 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,22 +24,22 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71445 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72734 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420429 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 21.0461 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.401752 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0642 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.77158 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.32275 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.441243 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.435102 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8814 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.40996 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e2cab550c..56f639a08 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -950 +951 diff --git a/summary.csv b/summary.csv index 48452d293..a42f865c2 100644 --- a/summary.csv +++ b/summary.csv @@ -2,22 +2,22 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71445,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72734,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420429,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,21.0461,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.401752,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0642,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.77158,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.32275,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.441243,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.435102,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8814,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.40996,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7c62a4f2a..afd54eae0 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71445, + "Result":1.72734, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0461, + "Result":18.6542, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.401752, + "Result":0.426851, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.77158, + "Result":8.32275, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.441243, + "Result":0.435102, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 18f9d186def39e823dc32e269ec6ed45141dd1bd..9039dbb40f494117b22d51790c5007a69d326d96 100644 GIT binary patch delta 3132 zcmZXX`8(9@9>>SNgvm&j?7o(=gp}p6n=p)F5?M<2Z6qQgjO7VaBr_UK)=ai&GjYvFB$ge4d~wpKdHf~u(p2PQx`E<6eK)!_&JvNS&f{Q++&R&N7spf zla0d*PrY~JSD4+b9g(g^yy3PV&~+dL2rO$o_ST&oh)rL`xrq-sZ#9U`X2((u#~-xa zG^<*Y&f6wkH(L(K&BGgy?#ZDB-j6zKUIdbj%9nw4c@HfQ+vvm|~R9YG+nme0MViI!wn~K;Ny} z(jJ5!9lDwvO2ta)r=Y%i!XmyOjvuhzRzB6>62avBSQ`99^L@fw4(cW*yDa(1gp@9KhP%)D<%TnY*ShG8vK;L({INM; zA>XnGv-ll-!+Hn|BHOV4yKX+Y~+>nHuCRbcqzFJ~5(5&FDSy$gMVqN0D z!gJEnMG$RULCoY&nz^RO^o zkvqke*dcga{28k)11FPVaRNzu@Nk+gr-;S|M-JIRrq|dMyDFbq={~YTiVU?2d-ql= zQo8RB3pBfaDn2k4UWM;{;JK5xK^iN%oq2_+)y^=vU@UO)gfV(U(ice(d+*&?fHYcv z>UY(+ho1sUhs(x$iNXNNj?5rEElIrxVxjl}5_W+@%hALtE36Q%12%C?|GTH;Av)hF zU^n;&aHVAig^|(YDTz};DNv+;>XK2&i$iE&mtXCZvMG<#hxw=ad7f$bZKzo zr(eHQ#{5r8z-8a4*#7^RCJbVRTFyCGK+d_BM=WZKrEjU|9RC~r1blc?8@-QN^1g{@ z{IZn>a*pudgp7dE`<_`DjFA!;Bh5|(d-dW~7g)oMrCxLug5dK!%xcTNyv5@JyCL$6 zimxHK8aHa?e&IxA%M0R1Hk!q2K1u|&!c?={i>;Aw2!!i6E-UWwqVH|HZU#OVUCAoW zRxIwffJJ0Z;B6B^l9QMoLxGp!3NH{F2+MCYl&*;Cs3SLkn_9R&Md+Ug|*;;SOpWYP@PNp7T^_LGQj^g@9XS zABFgQc3#-IAu#jS5u`8*xG#lX*ca%>(8;>FdV|pmQm*2l~)CSR$Arl_{uf07tXfLei9;DTnU9 zVFh^;2F>Fs9L@~?G(`{z%w=lM6iSaH7Wg!RFYRuquni8chrU|R%iqLLR5dY|q+ig< z61A3n7`DeKoofJPxvG)&73&XV!DZ`*y%Calqgzh+0%Gd|i^?@BSI||)n@j3hz+&8Z z%TYCZdhGKy6X=GDqV6+`j=6W}UW<-OwJo2_>;wxiWPbC8n3VUpAjv{wO{Kb+C+r^q zQ6o~{ayta{)H_I$H%RigGR~)8eo4E(dE{N@E?thgK;PH*GO(*1Bn&jhP&)&s)|LyM z?1Tz11N`QbziHurXygB(t+YeX3|x3V<0Wafu9)h!zO>?GEWI0UH^yzWVq$fJ7--VF#9o*tz2cuKX7GQ7;JNztKMG=mw7wQ8!F;=|Sq9s)guviRY5V;q?oIai6T ztA9!x3+x*JJigXG+wo2$pT@x%)A@5-^qhsJ8HMcg%lql*=Coq_IQb}ZG5;NgX3^Lb zc1nfYj=QK4&JWbD(GIo(`Qb%kHHBjfimY^*T$rF+jwc>Db*>Uozvy~q^G83GN{h&r zw%o7ND>JXZaVcD8R-|Y#mL5#mp)Q(ga(c79W@O9s(z{g)&5De1RA5*hGfy1*W7*po zjkQ#xe^hJ1*2lP=b|FoDS69X%&7Mb5Wha~i%BvFkU1F$z`OXZvGG!o`GQK|ki@IQ{ zNoEM=g*de?v}+mxBH-gr^PBFupbUm_Uc?_Snf_KXtqZ2@aTfENOfvbnp^_&SM=+?t zmHhMpdUU0I{X*~EUWb*D|y8@H%#H~z2^kEQ>h3cT#E&HF!8`FSjz_etYSP#z3J3TR|)?< zVu)}WZnb=xRC`5L{vgp`JGx$M*yF-sD1TG5dc^D!Sy=mZtoo*s=Zv+5RdZJHh|1j_ zgE#KtOa1h*){@vT{k+cifN;^hr`EOZ(DE^)s-ePc^zP|{>|f{M*x;z6W%(cAL`6D& z`ebUGTKy3=7u%202Ol0IJrKb2vFQzTILYDUWhBu7?P|v_ZXa+H->DexnT+#Q$(6Hn z-}e83u)slwqom4i?{7cO?SQqEtD0{Q5dKQ+?R2f7oOM*FdSQ!yuE^Y}DHP!HEZs-M zu&_Jwh4zi=1n#Xb9}CH+3DR(v2-0NS7W35;M!oz#XR0=)kEe=H#Xnc&;uGvhtHJKs zPO4l$XT}Vve9^OBeamX2vvmU8G7g7^L2q4dv3t`neTz_58^IZ9$gHSvmYZ@liJxA=gjN2CLkT z6+|@gx8&~?3^s`i>1;Bkn}BYm+`VM*;Cq%2M&hSl|0=C2Wn)VIu2iIP+#5n%~yH)ZO delta 3133 zcmZXX`#;ltAIIlHWhJg5t~oQC2umXtLoF<3ReQ=ZDW4VVzJLw@Zdwb~tAR z3NMWM+nMUaJ^_)=PoeL% zY5iqrw!3{Ynx)aVc(5d(Zpy-=*`Y1GVi^6(Z=Gzpx3#Nr$lj~>CGwl@)+f@FDdeH> z-E}LB4hvdlAac<#f|VoLba*U5R>m_<+ES>7t;dUD%-ryme{}rb{ng|5atsDFJ^hQ1{&hdtvfkj2@O$az zf0fTbQ}D2v1REU)=^vBwG|;+Hp7%IC0jrakOn^};=PA*BtX-n^oh_n9zrN)%0#pc3AhuC*CBvhM=HQYz?mmA# zJC=Xf^+$XX%ZNPl(62eXp02-(alOi`p7ASrQKNdcRLi9hNjlGE2n57dX#c_y>eq7! z*1(67t^+t__ao;=N3~d4-7mAwLOZmCISA71=$Z4_^7kh31qH^w)Hool@4I5=`v^1R zKYY)jj3RxTriWVkDr57|q-w4`&9LT+ZUHDBt5c~9!KvqE7=GdU9aWc623l;yvdBM` z-}HQqCNY{+qG(f}^9}Q02TS^X=sssXG1EyR$Bt?eMTd?Em-`Gx2jjtW-!4MzsYMU{ zCgN_*4+h+=^OOL(&?hQWXRa0BK=o>#F748@)yZ>&(6Dqz z7#;iQ3Kn9FJ0PU0Y>D27i ze)7eF{$@r=>N~R&{u0AKp8y@#$hAz3XB)*s(@MZVU&@^P@Ntv`Cf(&pl6Uxy?md8@V9UCH>Dp7lL49&m?f zfw78WSPL9_s+NRB*=qA0_b}~a98s;$KBtrh!q`%fF1FO|fp*!<=OcQz_1yen8k+$&bWLj2cm;N<5DG#* zgj`T{`m1i+^$kCMoZ(&49S>v7?Y9oVp;GmRL5%yCzmiJtdefy+gK?B>`j$-3zb?eA zQfoD2^A+?I+MSsXr!8ZcMesw&KLj2GEr62^eb`_4{&MBop@s4Vv~)twwF{Q&27h)v zR>#4A^*=?YL1Fn}sm>)5rfyW?qNyqFCoK-lE;M%ava>*}BX1h)opYs~@4@hm%4N-g zH$)U4hb&Kx+M1p9jm*07dUsUlr-|WXSBtXi4S6ut{NG^;W{&V-5ECm&EZG6@A`0_x z5D-&O23<1I)28Rye@y9f%B$$oHAXxP>8V$zo?K_y2Lu0hazx`sw6O?3%Y?D({;usC>Js zUGa#v%^*?zt>?Ls$&izNjI*7@%XSWTMICv2)k{8>9ZwI-<$>Yqgp0PRxutWL=_tAl_v0x)Zl4Zk3 z^BZR*ZiGpZ$_mrNtitEDp=E5!?ey@}6Xz-=4Eofps}zda{ru9hxAQ8#O%VmQdfo|+ ziZy3wNjpzHGHXuwD_Q?eWyOMXWKr?~zG%0Vb+1V(WLYOWZcRQ55owL3x_Tkkk+QY` z4pno7-9O#|0Tl#)(2!o&2ous>GD<-QFt9!9(*3iN*_2y&jN{v(21P6P(xJxQBjgdk z5dJ6do_DY3{C`DdS&Rsep?;6sl$VX>q3BNKdH$wZb0-Te$=*Nlf_*62J1XSr;&6y3 zB$+^KD7?Z6PBE*h5iEXIxoH#po((W`9##d#PcG+I2UY+1Q>-f9GZnsRuO?g<&$5Y} zv17PudOZ+on;!-mFl~@?0);Ue$*3pf;vZWTMj21P(teb2FYT%rx%63-EA9ppie6dA zDRUxw8m@=nPp87s4r<{dG41u=v;fh6Xw5%p07F1F->Lars##7b)nKe$<h&4fOe5t3sWhm$hk{@Q7-jws&vX{T#cHp}UR2&f?nShP7@feZx3QhCyVD zeda}hGUA+VIXl|^mX^Y=Zsj6bX3UA5Ap^ITF@PL%=*3EG`R9~7abtA__1E?RbzTy; z-l>zy-u$R}LmP_aP~N16gYYF1sJ=Zm2PBGpLN(qxqxdf^eJx(7tr61LQQne)(R+K-8U{T4(!T@)mJ9#PgNaBVh9jYq%8_nV|e_ zsPOfEEqY6s1Dc%CUFf1&Vy9zhLB6>o12!w6)5pS8DQL0#g{AF6Nu$#vG}A~N%z_-I zBF|%CZ0BNQHWd|wHne98i=T&+?hxt?4xdgU2zt4z(NF;UvfT~I?CjQZCcHF=;Ws7k zo~wM#_2(i#+Wo$+JeW4SBrCVgTU^WLCH@xK3DlfX6B13bQ5+(5Sa=Si!V@LcPVM?I zFElAo8b_ujl#UweW+kEamXbb^M=!1q_I8y4NZE9gLaJ|nH(*RDG}&RonoyQp7&S>t z30_jghYLj`bxpx0f}Q47RBqCD^u~*vnjX%pn^L89t{kCxX4o2}_w53VWnyy=;J2pX zATBFy6H?oO^1)Lh;}ka)xacg~ot?y7Xa^)X*A*ChrSmUQ3GjTljwayJfs{Wkx^#Sw zJY(f+>qS42+EfC89QRl{aPgLJr<@?I4#>%`3pb4Kin+AHYCIIv_^Qu*Q~LcCFQs0( z;_>2}^GD5;HoZ+|9TY7IN)^>!u6HFZ>h7QZ zxm**RvibDj`<{Xx?eFm4K|!2A-RD33_dmcv!!0EKJ-q~kheTL@KSFm0R1YLR{{#qQ B;{gBw diff --git a/summary_results.json b/summary_results.json index 75067f756..2da14e6a4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.77158, + "Performance_Result": 8.32275, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0461, + "Performance_Result": 18.6542, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.401752, + "Performance_Result": 0.426851, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71445, + "Performance_Result": 1.72734, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.441243, + "Performance_Result": 0.435102, "Performance_Units": "Samples/s" }, { From f5dcc90cfe030873016c37001cb8b6f2e0c5efd7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Mar 2025 17:24:20 +0000 Subject: [PATCH 324/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10229 -> 10234 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index ff8b25a20..ae0f294cc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-24 17:23:11,590 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-24 17:23:11,590 submission_checker1.py:3260 INFO] --- -[2025-03-24 17:23:11,590 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-24 17:24:15,288 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-24 17:24:15,288 submission_checker1.py:3260 INFO] --- +[2025-03-24 17:24:15,288 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-24 17:23:11,590 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-24 17:24:15,288 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-24 17:23:11,590 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-24 17:24:15,288 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-24 17:23:11,590 submission_checker1.py:3276 INFO] --- -[2025-03-24 17:23:11,590 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-24 17:23:11,590 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-24 17:23:11,590 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-24 17:23:11,590 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-24 17:23:11,590 submission_checker1.py:3297 INFO] --- -[2025-03-24 17:23:11,590 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-24 17:23:12,566 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-24 17:24:15,288 submission_checker1.py:3276 INFO] --- +[2025-03-24 17:24:15,288 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-24 17:24:15,289 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-24 17:24:15,289 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-24 17:24:15,289 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-24 17:24:15,289 submission_checker1.py:3297 INFO] --- +[2025-03-24 17:24:15,289 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-24 17:24:16,381 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0642 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.98397 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.32275 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 56f639a08..be1c55a35 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -951 +952 diff --git a/summary.csv b/summary.csv index a42f865c2..f9f82982e 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0642,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.98397,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.32275,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index afd54eae0..5f2d3bf61 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0642, - "Accuracy":"acc: 75.800", + "Result":9.98397, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 9039dbb40f494117b22d51790c5007a69d326d96..fccb023d2acabb95af5d512c3152cbd834e16ec6 100644 GIT binary patch delta 2140 zcmV-i2&4D)Px?==%>xQ~2Y*{{;y4t3-(O)xeQ2eDT!0V| zFq+bP>5Z~m?Gp-#1JNXLCU$!H_hTo(l;cxov@eym4>WPU@7&_gKJnn|YnqT}TJV@< zL&I)b2BBHN!Z=$FjlXAS4bLE4_*v*DETco?jdJ7b=&!*GE57p;rGiKU86O%eA@aUy z@?b?%pMN)5PBZx=Vnyl;d0Q+^o)^>)D^i-6j%B%K>c^Q;Tj&>mVuM9d9MBUMlqtJHQ~8jBMnmJ9-5)=BhB+EkuK$VY3x9V_gui&C2@M1dWdRJL zGFveAT|K;%LzbeN8uzS<>b@W$jr=kZQ}%K}fGp^EuDY7NY1Wq6Hvgp}F40aJS0 zx_@+VmyYJa+VwEi5p58Ij_B+nW;(*%B6gx|;Ze3Uz73+(5nT}FZxOS7h@B|gc$968 z?|>+Ega@MhEn=o4dU%u_G|F!sy_VpWA0rS@?=b=a$(|q(FndoC2$)=F2n5Woa|Gh< zLM{*pc>9+K1eE>?f%JAE*E+-j)tw*^P=DPU1Olpii$Fki?+^&6?mYql)qOx9ABfk* zC!DJr0JT5XAuUj~F#-Yko*)pA?yN<)Rs|)Jzlkn}5n4 zKwjmvh~{w?(L#1IbS@|tb6zgen5(9M&!0O@%M=BrRr_J8F2lT;jMw~va=DvxbGe%P zfodxnvWuuqHG;fs<7i+;$maI&89B** zj>^!ge;>p(t>d!m4Y}Fsde*yDZ+}O0U0ifsU36d?9_x@6T;&l66deMAqC+51bO;2B zuDeHc2px)U5B;hOMF$5ax&uvgkaMeZK-PDNRZ!mh?*CA9pOMoai0-kwBRb1&dv5Q8 ztk$bBzK5a*cN%8tGaSgW}Xa5sI;Kg&Nx)?3Y8%_p`s871_hb#)4>CCX|S`Tz`Jt)7MWZS-ALa=mmx1xSX3w0;Ib0fyY=0=rddw?|h2~bM|1hZ-gxeNqf-w(I5 zCl#aue>%E&PNW76ECVTPO5kxuI}9r*c}D@Q3;s_BEM7yAK3W(@QB}X1+E!{gP5oO6 z+*aUHS~a+soKTR)y~&)I}a*MEJKmh zANKo(s!(vGG|1Wh>+sG<)unDrSN&G9-=CropkM_p87wzg_vF_|(JKBQ+n%hNj+i?B zTzPLId|&3hlM#13@SiHNlklh#4}Ueim+ZGsjTRH%q)+;4Vc6Hn2O}wemwjgCeVFY7 ze{uZ$N`H;+_@#an#^c8H_{Dw{#^Z|o#B%Ro<55k7JM#0$k!*x2_hKV|+ zrH-EZ+R*iyy^E&G#1Z2RLh1K|zoz5q8M=UTag%`NxA(|0eM#V6a>?4Cy9e5Jj{|#A zQ+=;3!Lpa`+nc^9I^L&I({}2F{*mR zc0HT8rrr1CiR5_*H02@iT%6eJm>ZvjiY>k^UCEQTde6Q$gE2n*4^T@31QY-Ov&R`) z2nBu2IHV?%CL$hx%}&EG5QOhMMdW+pBy9m%Nh$&AiAqQmQE{{EZE6w!$Tp;Td)%Z= zTPY`w>zVy_cI9GukWKUp)(NdjlqD&OKrzjQs!Q~-S;dbia-J$q8?B&39XPbC?h0lI z(-z*WHekH~7X^@VgqafUy*C77$MztpOM*iMai^`MK9sh9#)h(QS_970^d3v_lv7Xf z5g411h)zh(Mx^<%O%LD%9T?J-x-*+p2g99aAGvJZJV^S z-Wu{}c)Nam?^lh5I-ZIFstV2svp~JJRe`UIaJYs#zYZtf3E)p%&?nwbiwu3hqY!`$ zanH)3oR{?61OeRGEkCT!nA{<}e54U*)001Zz000yK0000000000004l2 z85NT;6&91`CMp5MlMyF60XCCNCng(x%s8Yb0RR9{0ssIJ00000000000001hfftjF SCq4melguY01|B5<0001vl=Be) delta 2138 zcmV-g2&MP>PxVi*%>xdz(ek=+1ONbf5&!@g0Fw+R8h_nb+iv1I6n#Hm5k-AyrGZ?4 zkRo6-rT5YsWlnXTP)HnzCW$k))63tFoj?b+cbRHlDxDWdoVE5XUiOarUthA+cp{7^ zG#{8w(>4u~hcrs^#lZYNJ#F}=!9|cqK}vHnFkcBbzYhQDKU4ODF9{KbG?4RwxfG)4 zSr!kMB!3HdlNKbGPh!flK*(FRuz0~p5LKiswOrfwtSm@!v$oJI0?xy4aIMB3S^_ zP?^ms{h=OS$RS%%Esc9xMRmuFh{QpeiV1x_Cx6LeDWs8(qJ%W%)vu8yvb3&gWWme8 zY)KDEB$mpsu}q?f7v~ADrsMqtFk`>3Qy;cz( z2O=2u8GSZb<$)K0dQnc#wPlq;HF~dxRczL3V6q-CWFD%RPs3K){j5(+z;yAL)=5sM-jDfP9Y;2*~#Yfq;BZ5eS$~XKM(u5S=4*FpVyD&@b1|*6aBd z92n~k+Mrg~kaL4@z+6EPV8-434m%#6|95vX8DHF7?N@CAmwmz_5zBJk3~6R%sek;< zWDg*(3c})9lE;L}PKL|`;bO+id6sb16!6(or)gV)5mL1umg+LB>+xvCXN1e$oLh_K zEC^Lw(U4t4ZK@F#Wutf%l!eRLliT!~c4M{s&qA8V$-?|%MnnWj%FTnm6(gJ5{b%GP z`#B;*tNz}JYf{JM)EjcM)%ER-Rex_wbX{C@U0rlw8XoD87F^{K2oxOxfuchoP;>|c zimtmubO;@aZU_Cc3q=P9#<~MdbdYnSb3oR&h*ePDyYBx`bf1ya2SoSK-4dPcw0-aJ zovqHtRp;xf0}aBF4rzgfc7#Bn>JSK29Rh)>Lm*&hKJ#~|4xxi=FLux`eSfGrI55^7 zv|*Gvg7hwBu1o#>0eL#Rye?x$dRG_!f zgVge~Hom1Z!WLvi(v%w^l?AJrZ=>~j!^G)L>u>SjaP={tM~Pb5g0wz45J|o;_|N)? z!t0G5U0WFo<4KtkL%bI9lYgGP6!IOZTumc-iR;q{qhJYTiFmEOtJ9G(4Tc$w_N$=E zJo#CY{k?F{>+OZ!!570lFb0Oqd<~V^RbHy&0wkztvi|R@?>vJju&DM8I5=4ZlJNb1RQ!3Ussw8j6M#EoFO9KQH0JCriw+sZ}A&eWd zD;1;ye+ouZGL^tGkg}!(9%r<}u!53z6wtch|8&6OH5BQig>e*B^{c6ErIypwzoo!! z1umskgS!gcQ{d8(cQ}^f)ZmJ~h82AcOMR`u6?(BkFIMQq3cXmN7c2Ab7*%ZzcQvDGC7!R?w2ca)WhGevK5Z;{UPj$*SpyspHR; z_a?&kW!^g(amNGysS-O0k1Fx-SJQjRe*4sDG4W0Mq^}l+eVu$TlJa-iXI9>a***}* zf6uS<*XWL4>PKNbZcL9~>_=fduEKaU*AhRA3;Jwi=(C@j8gQC(+xe>OWomap5s^tt;o?h&2X-rgkX=#JyKd*dF_ zZO;_*;Kk&6c{#Zjh$DR@k1nUP3ZuO(Kl(!04zwk!1`2g)=uSwJMAUj9sdfDK zm7T;%z^g7t_s*F+gUZETw$U$GC$y?imZT^G#WWXcTcM9;89$=Pd8#;VwSo%u;LxJJ zDVZTmTX?hDfb{}g6hO)mW-7Gv-Vlr(+kvDm2@Vy+jkc2dP}(hjHk5tSE#N#&@390= zIrS6|z}QSgbVPDCAbKa4mIy6;8Ynz@Pe{&%7NMIrIUK zLI84zds04&=a*(BiRwJf@8YzG^V=qSAbCNu{A(C_)qe)crYVs0kdo) zgaQe((ek=+1ONbflldiA0cMj?CPe{@lb0qU8{i>~8+il(04Nmz02BZK0000000000 z0DyrNlhh_b0l(^b0J=-?EC2ui diff --git a/summary_results.json b/summary_results.json index 2da14e6a4..f8596a701 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 76.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0642, + "Performance_Result": 9.98397, "Performance_Units": "Samples/s" }, { From eaec86f819614e969562e40bfab7c50e150b2154 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 24 Mar 2025 17:33:25 +0000 Subject: [PATCH 325/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10234 -> 10233 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ae0f294cc..bfcebaab6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-24 17:24:15,288 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-24 17:24:15,288 submission_checker1.py:3260 INFO] --- -[2025-03-24 17:24:15,288 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-24 17:33:20,247 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-24 17:33:20,247 submission_checker1.py:3260 INFO] --- +[2025-03-24 17:33:20,247 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-24 17:24:15,288 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-24 17:33:20,247 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-24 17:24:15,288 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-24 17:33:20,247 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-24 17:24:15,288 submission_checker1.py:3276 INFO] --- -[2025-03-24 17:24:15,288 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-24 17:24:15,289 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-24 17:24:15,289 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-24 17:24:15,289 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-24 17:24:15,289 submission_checker1.py:3297 INFO] --- -[2025-03-24 17:24:15,289 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-24 17:24:16,381 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-24 17:33:20,247 submission_checker1.py:3276 INFO] --- +[2025-03-24 17:33:20,247 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-24 17:33:20,248 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-24 17:33:20,248 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-24 17:33:20,248 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-24 17:33:20,248 submission_checker1.py:3297 INFO] --- +[2025-03-24 17:33:20,248 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-24 17:33:21,512 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index be1c55a35..a8ed109da 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -952 +953 diff --git a/summary.xlsx b/summary.xlsx index fccb023d2acabb95af5d512c3152cbd834e16ec6..6caf84e1765a3af90ebaec68b1ff640923999b5b 100644 GIT binary patch delta 339 zcmV-Z0j&P|Px();lPV$}f6HpaFc3xe{R*SKEkF7~*bcNMs|E^nY3NQ! zlSI^dAgOiy_m!Q*Nx-WvNB7Q|JA=x_UbfLMSSPfqP?n@90>v~JYFnX?W*I-C$a$(b zZMA|5_2AH=zA2d@Oj~%f+JN-}Togdc5oRj1^WG4Q9ovDVE(s15#ErI+`cT>}e>RkT z(=FgUP4BS;PdW7z55U+=M07-QHX+TAZASoS7+R2^yu(?N;j`C+m2S3SR4$xS_};** z^(0R2y%UqM>$;>XMr+8Q;q~hEeOxsbYB&`GRF|9)W`TNb>k?lU;czW=eicr<5x}4N zpwGM=7di9+k3s-)ho*4iD delta 340 zcmV-a0jvJ`Px?==B_aiV%s8YblPV$}f6Y$AFc5_AJVoSt;v{VWSxG7Z>WNB76j5=r z>}_fh|Hw9^d3)TXO)M^tnLbC2-6nctTtf102c+2a)g-@?Y%byW5@O&sY`-G1#zdXq&}3kf5wKg zZ(0M+)ASxo@RUDl-U;AO zUC<}qPKykEz@rd=3~|rO=j{1qCbL3Sp5}` Date: Mon, 24 Mar 2025 21:21:00 +0000 Subject: [PATCH 326/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10233 -> 10231 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index bfcebaab6..6cd52b50b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-24 17:33:20,247 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-24 17:33:20,247 submission_checker1.py:3260 INFO] --- -[2025-03-24 17:33:20,247 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-24 21:20:55,246 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-24 21:20:55,246 submission_checker1.py:3260 INFO] --- +[2025-03-24 21:20:55,246 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-24 17:33:20,247 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-24 21:20:55,246 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-24 17:33:20,247 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-24 21:20:55,246 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-24 17:33:20,247 submission_checker1.py:3276 INFO] --- -[2025-03-24 17:33:20,247 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-24 17:33:20,248 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-24 17:33:20,248 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-24 17:33:20,248 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-24 17:33:20,248 submission_checker1.py:3297 INFO] --- -[2025-03-24 17:33:20,248 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-24 17:33:21,512 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-24 21:20:55,246 submission_checker1.py:3276 INFO] --- +[2025-03-24 21:20:55,246 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-24 21:20:55,246 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-24 21:20:55,246 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-24 21:20:55,246 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-24 21:20:55,246 submission_checker1.py:3297 INFO] --- +[2025-03-24 21:20:55,246 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-24 21:20:56,353 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,15 +25,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72734 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420429 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417908 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.85051 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.47558 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.98397 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1773 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.32275 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a8ed109da..8a8022aed 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -953 +954 diff --git a/summary.csv b/summary.csv index f9f82982e..7e29f61bf 100644 --- a/summary.csv +++ b/summary.csv @@ -3,15 +3,15 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72734,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420429,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417908,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.85051,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.47558,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.98397,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1773,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.32275,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5f2d3bf61..64d58cab8 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420429, + "Result":0.417908, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.85051, + "Result":4.47558, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.98397, - "Accuracy":"acc: 76.200", + "Result":10.1773, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 6caf84e1765a3af90ebaec68b1ff640923999b5b..389508eb106673387decbffe3399be4192d16015 100644 GIT binary patch delta 3981 zcmZ9Pc{CJW*vDtakR{7Z{A}6BE+mTiF}4ZA*ov~0kZoj=D55blNcLq0SxbnrmNolU z82c_HO=TIf4k4*`bk6U6&w1}Z_nvd_`JC@_zt27Qc@8Z1EME^WaoJH_y!^lbKrj>l z5CD*^W%$UOOlHH2Mkmh#TrkFE2K5k&J5hHv4j9m4Ff!U#`;>tgh>|u z4>JYV%wFJ7Z4c+`sY8gM{axvyIN6RM)7WHMHFiw=qDg+WU3`D*Qqh@t^|!1say?%9 zMvIEn#i^|uou0id^avgo6DDS{jA)*Pw#uONB<;^w2Rz9D5iQ@xB;h;V`$bwx--f*1 ze-}GjwJm<8zpI0A6wN8rjWJ2U*ew+B#6yfK^?EiRkkUS$=FYc4Ts{zpdnZNDI+w2f zvrt2HviuKJGAy{ik_LQ(e2$d2@yRZDZnj=J8!o}d_oMi9&D>|Pc_g8u86etnGANW z6;yQ-El#)mIyfABDRL_6gh@uL@auxQ3s02S@uL&kA7({8;CURb!;og|yKBZlzENdz z{|2Rq#BUJf?jgmH2afL^^1CYbDvP>b{n{vQDnbpH!^fttMj2}NiV7{%J#{kM>`mlh zeYpoF65J9Ytm)0NtYCEcWiZHNN)WVFE3w21ZaU%sgG}+!dR%o9SK;rNCvqfttmbav z!Jy)ha6P?yz7W=z>mnfFM6;!vXv26=`Z-?8~x7y~~Ifsqku}S~< zHK?_wLm>K@espKA_dSdE{|IVPLKAw1S6aEhWNP?n4Mcui-@zTu71v2OAn?i!Ey{3J z&cpzd6Xn~i;218V-iQQ`92^X~I5jT;kKnpk7altOI4_*(@*GhH-cvuKCBEwTPC;(t z`X9Vp!%z>68@V?;56xK%z-?@XmkoB?F0B?badsqg(Fl+3hw0_6@UYs5fQZG0;?AB3 zdf-d0=ixpo_lH$*FX5W1c*dQ%6oIuaz)vW_>Y{8v{B!WEq`}*sFVCC8jdB0-eh=}F z^Ij|K=DE>BCg9*vASmE_?bt>j;a6$k{HT_TqMC$#SA0@$m_~oy$YPVr;>pE2kV2I0#Md(~tO3XmK0%i8i z$iA$V855Wn006K8*Z=@sfCoapW|NF#b-+^$!yIpIte@GbcU<9HI|{y#BT%Q$o7g!W zYN#;~3F&8{-qCLS+s?VbCVw$`UGF?f&t z+piF)QFvB~<6J=I;-K^MYFprYxGDY>;me~e!#8UhxS=|&Xi?b_L+E?v9Uqt{YZV>J& z2{#ll)$^vF)rqq%ycP?MZ!*@|T78e55sTOwENJb&v!taPa^1oB0UwX{u0Lz=K&=I2 z=K@|cw)dN=@+@;ah%|-#%5Itv#S-Ku#0DEPk&N!*fAGmyJr0$oXJDU}Tc!239qQ0f zaLT>+0Z&IT42`(=tN&6T{;{EKJ?QC)Z0;&omPJ+8MH;i<`}T=4?_k=y{m!7tOD@ca zp(!Ruptr`EsalS8VLtqN=*vy5My#2V^;bC5Y`aPHHB6|D zpA&Xy^8UgzF241p4+*eS8s&-5yts({HnDUf?H95v1 zncVVND0rVI$tWlwLz)eq70&I}o2E=dW;l@KJA>6v4#Om~&&~dkQ=cGTsk%3AhZ*!) zT0&RQKBOs&lQ&ky6h11~>7Zo1uiYR;3&F@<1auxyjq;Cf5+?QkGb551MA*qCG)ZU) zz4`~x5BDxr_jGXd#}~d38B)$X$MH~(J)XH%g@TOtG*z<>cY|^!9mnmxxs`a`NCpLP zOM@-@a3PZ99T^^eU?#p4jRmP@oMUaPex{M`YWb}@qcp$C6g?A$>f4U>=mV7`81P3) zHN%l#QB_a5NJdY8V#~^-8UqYZd8mzZV+7i%-9w`@#%v?cr#^)+HjuVgpUC zp2>L$ANW^3HMWB0_SWvEd=mP0QQv`DmR#!mxFumb<7beC?Wdmn%<}vG{0HaBjz*$% zBC49GIZ829e_FlG!>MN9-ruR-20J@$a*C)xr@yu!P}GbQNs(EMjHtBkJu-`0NtO$P zL8xVT_kqQnGq67~=KT)rrAYTrWBK>S=r)j4Ct#`ZLStt`l*aZr zKI-(U5TNl=2|If{@v@uLZLMIfE;*JnwHWE9|*ro>DIqL#4Pmk{RL84*xo@bxHzah_(PM0zJsUCe?V7iGyyhs45p=*$9?m-yb&^?O z@-?;iZ-oKQ z{7ak0ri6lLBT+uLrkgqtwm1VhB=$tjh~G! z*KrIZm!hPO5RAlXoTN7DY{{{^<-#koG&DO}6JK-_F zxJEp8#?bhNe<3~ena^Z3{l@R}nOE?@%w-Y30_w5N`w~LPsxazf48S}OsNQ!{R|%MF z@3BAPyia62ORx_!&%{k@JrBOt#PaD4xZS~ASuP^Spv9SPrO^HzZ`cViTl_(M;WKdiS@&zS zkzn!)D$(uMtFlq{^S-;NjclAH|1TAlHC`T+Q@vEqF}jF>PUg#Ippm34K(f1q*d!Z| z{o>BZ=bdAr$c$PTG#jCQGRnK(HXjKf^zlkZHzj8DG_FY<83c z_BVBp>n8H1j8N4VK|@wBRo-TiEH9~upT(F$YMHQx3G3sP(s9*Rt++xX3RF-yaVh9xWI&XKBE z>VP-XB*z_VfV^?K{78F8w8>U*-GXjFpiuiBPsmX8%DD%#|Ho$7pnga0`VX6n2bKK wgUlxj2da~;WMzbpZIA{609GIX0P^2K#!Mjck&|T+Kn!_UR*FSf`grSq0Qjp^G5`Po delta 4019 zcmZXXXHZk!)_{|MR6`*2CILcGsu-ob^hA0`lrAWtgFpZwlu#^y)C58=(nN|VLAs$? z04av5fGD6y6O<|h;YMfPJKvq}{Ml#D-t(Nb*R%G_+WY1Q=1+&{Si--=*@e&p0C%8- zXeNHb4?1Stf(lpkr|YhlqdY5O9b?$p74t?TR#F-qzH_o9(ox`nlA+jx{hLxtZaufyKn%%ae0+B+(Fi~h<6+!W0jYa zif(mZeUoeHu*{3Q5r5fI=p83d=Gh^XUwKKQg4?f1?r*bv;2v8|>x1+-J>+8kM-i>? zd#VCgo}47Eu;Ol;-suc0X9m-Q(ZJKCN)C#9P8uDVNMdQ8z98FoV-R-1CVFN0bs5h4L!(AG6mK1~&d7)>@V5iHtu@8ghh~ zDAS>q9wr^AdxHl(&|xWIcQQY;1Ff+$ZNUM!?9`?1>0QMM4nOwz)a7+CceS36VxgtJ z^Oj8rGPrb{!-@UtEe=vtLZS3bSf*dfW{h;m9mdilxf_+@*sx4t#K0}#7sSt5BI4{8 zo998m{vsH=VuU#dQ;wqt6aC;54E@4}t~@(a7&V)TKJ%C*b|+MHO6?_Zq7cSz@lg;* z52}uMpsgLU$<9=@p#cI;L}|lB5~f~B71RgUPRdy^M{q$ z-9+28mt6#$HXdI0EApG%_EAx!2si4~`NO{P%*V9G!Mc*$Pq;%7ZNK)9gVkC)UTD@% z)5}xupk=RqE#-eYZo+wxKX3-8?fkVdVJVP@q8q9dEHBNAah4y=si}+^DgMXwE!U|E z8>(xyd~Zi?^3+jWP-{L0)0#((^_CH6cu-|}F|Nb+%dI%>ZtvrD#K+1B+s)(c7_?;v zg3Kq%9bFKx=;@-@x3u?zU$s9vb>Lk!PUPjqy2XYyaO;Ov>}0#mE&X7elVh-xQ?QG` znYDBt$FCerT*9E(QY6eGn2A2=NxOb9lW>whw z;Fay5U}%Z!6Vw;GSn}Ua@yWleJ4$*_p1;sMScAi-+d^ zUKQKJcwdG6clGL3YAU!tNppQfuiC7}C<~`y-}8hQ*R%y$x|jO)LA=ll6BsQX7iuTd zj|-zzI^b}}ZP5C~2t3j?uHYu_;HgT-I})o~bI$Y*;u4 zNsQ;#^V=b2R+7ik=y_M}E4i<bG2*XZF6m{tMZ_Qu znJtXz=(zx30DuVq0B8Y{lUZvv3E@okxZL}v8z<*JIxJ(F$7>q**FFZ{m}Rh3XY-(f zClG>qih_4~f39OY0|Sr!-}vwajuwNdgaDp(Win&n;#B;}dK7D!)Dxuyq^XLK%xl1IYpwewQ@(3lI=F2S z56fJk*XZ^VOYN;ZPe+}84hkU6L_F3_zz|OPB{o7$xp%%FY9Rd<8hnXS*;dy$JGj(D z6{2gKcBlVQYO$R+?jT?peywXeX501ZV3!a>xB3fMnMzBWjj2K*AC7Ot{#w*#z; zjCNtAM}F%J*-!QWvyZ$x^1;vHMxww}hAsZvZj1y4Kt$2mmNK{xq};n<2n7%&nAMCn zWQCA_e|WGA>A@mQ%yLcWG44;|zV)H~dj`A4-K{lZaf52!wMv*LjFpq-#rZ{RT(kP; z$Opky0a|ZD;ki|TPKJ`+=_aK>r9_546d3>bZ6BODW)M$bIMmIkgr?DT`!ZcMpbR56 zcHXTvL3o+ZpIF_6FJ28FZ#(mTscF=I#*9+Xzi$ZP_Sc$z0w?%dV(}*9uv`?K8x)+L zD1d{Lob)6O0d7S2vM-xv`dNpza!$0jVdd#HilR0lAz;||r0m@lbNiu5%bOLdq{8A0 zFDh?3w8fZ^>v2^9?X!D-qKx`KQNkULv@31?A5)h`%up=2VCY?Nq3(oEajojvXn~g> zZYY-++R;ND1c@Z+-i-ZjtYo>E`#%vz9^K5&prtgImeTxqh}!^BeuX|%2Uh*26kxJ+ z8l?E$16=u@^B_=iO}ZXRXmS~CI4b?n&{bXb?$m-oJsTx#v=)izb34_}HU=QPWEU5{ zQ9byqS1p5B*g!9gJ5?#{xq>&#{6Msb3%q|1n1zDZg-TVMZJQZ=UGP!km!Gs{ffHs{ zg2;oCMqib8IZGldHH)mXl((uXYd+3iCJ%ifX1s}Xf%?HxgSXgIr0UdB@g`Rlw-v#P zhNmx%qDlsi)N46i8U`t4Nco~YuXa_x?R9#LLU z!h%`y4D@czwHu4w6Fx(lMV)5`JXSYUt!MVmyxsW?x}CPgT7p_k5dW=#S~QG@P+*zd zvYIAz7EoM@%tKks!BctwI(I}rmcnRH^S>YsAVHXs`alddrmV!h9b$E`Eq7{Ud<*kr ztEhN~_@S`_^eF9;YPNu>cxvz=ZPSICC^|=_lrMNMc(#dn>p0p>gs2^4M=UApUZIn7 z8fEr(q@B5Y{VVuH7jTVF)1f-UlKF?W+6r9Xxi#Cu+f=tTTbbsrSv_m+l0-6xep^(+ zYl6Q>sSZM>B`SyE9b7=0H2A$23dlqIi-7$_{wQP0`@ARW`ZOK=*j2Q~=Mwcu!%fq= zc_eqZJ!h{NSdNdD|!g!*XSA&!Gu0lPm3+nO+Ba?l^q^s zD@C(Br5``_*D~53Z={H$zEkYNw??^~bs)`rJ18u4#-La)=uJ$K%tO zGcdypnDHg+pV1X0f=m+*hT3(n^dhw3{1A4#r5)EoKnBe?*zB*DOfO@x?iFNjjKR_l zP^_3R_9(~T#1N%g_%!m zd|jPLglsi-23!*I*u*ZIUsb@Gye9dGA7ge|45cFOmghnRiI}U_@ z31>ol75(8@@O~{vu=q}r99hn(pU{jQAuGq9*F1v!>6!=|k^-d9{75GHX6^wbK}t*p zr*)|No~==$hJhu-YuU_zfEvfb9)}$5XBECQ?5dI1-yhEXCw}0Sa}!0uR;sbuf>w;bv1V>@Lh=;93u6B7wU@9bp&|IU(_{ev1g-!8e6-&4-zO&4 zn_wn+8JI@MloS*EtA=(}mgWWkxc*&28)IVm2^2{k@JoIGAk0?*6CCb~#t>YjxCj@d z1VERB2o^%z1UD&mp+9aUfB=962ms*t9{{fy;k+2 Date: Wed, 26 Mar 2025 17:14:25 +0000 Subject: [PATCH 327/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10231 -> 10228 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 6cd52b50b..5b2d37d03 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-24 21:20:55,246 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-24 21:20:55,246 submission_checker1.py:3260 INFO] --- -[2025-03-24 21:20:55,246 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-26 17:14:20,192 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-26 17:14:20,192 submission_checker1.py:3260 INFO] --- +[2025-03-26 17:14:20,192 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-24 21:20:55,246 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-26 17:14:20,192 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-24 21:20:55,246 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-26 17:14:20,192 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-24 21:20:55,246 submission_checker1.py:3276 INFO] --- -[2025-03-24 21:20:55,246 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-24 21:20:55,246 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-24 21:20:55,246 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-24 21:20:55,246 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-24 21:20:55,246 submission_checker1.py:3297 INFO] --- -[2025-03-24 21:20:55,246 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-24 21:20:56,353 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-26 17:14:20,192 submission_checker1.py:3276 INFO] --- +[2025-03-26 17:14:20,192 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-26 17:14:20,192 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-26 17:14:20,192 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-26 17:14:20,192 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-26 17:14:20,192 submission_checker1.py:3297 INFO] --- +[2025-03-26 17:14:20,192 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-26 17:14:21,235 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,8 +24,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72734 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417908 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72364 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421261 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.47558 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1773 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1254 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.32275 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.435102 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8814 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.3315 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.40996 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 8a8022aed..903645975 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -954 +955 diff --git a/summary.csv b/summary.csv index 7e29f61bf..84020e87e 100644 --- a/summary.csv +++ b/summary.csv @@ -2,8 +2,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72734,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417908,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72364,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421261,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.47558,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1773,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1254,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.32275,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.435102,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8814,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.3315,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.40996,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 64d58cab8..0d5c79955 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72734, + "Result":1.72364, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417908, + "Result":0.421261, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1773, - "Accuracy":"acc: 75.200", + "Result":10.1254, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8814, + "Result":23.3315, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 389508eb106673387decbffe3399be4192d16015..c6ea091c63c3fa340dc60fac29f6e33c5a7e0a18 100644 GIT binary patch delta 3536 zcmYjU2{e@L`yPyC5~Go&i7;l6B@>fuS!yt3oAe2jCE2o-Wn>@W3n9kXcY{zQMr2D& zq!=1LyCOo94+(=2{pf$bbN=r+&w0;zmg~8%>%Q)Do_pJ3)8Yk<132Kn8bqOUjBpU< znJs#eqpf!t`7eZWQWfV7F|M7^#VL<~&&U9-jYZrqk(&|yBG{+ajgnsG{Aj!C;|HZ0 zlFaxrL7nv)XYCFuk?yYw2|>_TCnTR z-AHEHeXM_-Mq{NN)8C-kUyhDP;$=I0Z<};48fJW58Ta@lXnBT7@Ilr{Y^|FWFS--aI z_e1G3%iE+7N2?P?S?OtRvA$6!D+}Pui zu94OXYcit6^!Q{3*St!V%BufjqIUD{1p;bl-J7d?w-}YwIW11rkXOVVCt>pK-y?{x zTGJF@>p_Sy^xR#z)`D6{vl*rs-CQX!-xhq`Z?|&pJGL87bgsCTc*9e_O*n(xC6b}@ z^i}wEzrT;|mJ2V8SN-!OxVJEg;BOur{t@NF_>5h)9RL6J;40sN@Ya{TvMK?%Szu?0| zwNpjkW=Tm5r=)y8olC3VEH$TJ{lhRcrQOm*m1f2-ek7< zP)7yBM0OtqCU??;)p*E(?ZVS={ru^-iM7Li`AQLUdAL?zwKO@N%*1NL#!Sy}&!X!2 zQc3eHq~GoK%!w(N*9_WHSo^NCaz1U9+^uLVVio2!Vy2wW*F&9Dei=zf4SYv>#r$4; zUUGOVOWwppq()z}VL?}8zQrc_YE7mn{;I-?oOwZCRRB@bucqBFLE-tPU^)PghE+*J zjKzuZ3Y+aXJ#mmoN1Lv2Y!h-?il5o=F( z-b_XNu^nBG)Y1o~V&G|L7!eLhfYTKuiLqB9yj?g^et0M++*v*Kn=!;VDce%_P~gh} zi~C`iYH~UR?hvL$avmOfo@^$2f3oo+CD$A~s*OIuM0Iv^9xVMFF}`;h$5 z*%$)I6<$b#8n;Am z`XL#Rl+{B#MpXNTYt@4F`w45d-wwzl$kNfRH)>58LBj)Kqfd8q7z{~~-lyLZhjMv9P7LJG2q$yD8$X+n z_f)*#uGX|RH@mncz6*gG&BHc4Uq!&zpv#3)#*VU*F|aliRMDPDUH| z{oHWt6MOdfvPisiyC!;QY^&E1(4k0#77F5ZToWQ~a$1#hjpE<1Hfssa{(TM%0z-lp>`$^i193&48zcNys zhxVNXDX2?*Y);&rC}ZbJq73_AUzctb07wRp&mj0NR(u?CD^1_a5*U%f0*zr^xV}d; zEiKG}P+oS%Nt;I>d@NFYNAt`wwiQSFN+3#7j{~;^l4X4^p<}LRBAG}6pQUmC1wB_{ zQyiZCx^Qz?I%?*~Q^}A@pddJ*Yeb=LefIlHXNf^a@PaCS`(~wmSc)qv@^P!tu7)Mq z?=A7*40*k*GtNWc14LNj37cF(Plly%bGQwARDJJK>6>pHM;7us88s0XTT&X;9PtKg zS@(&%>ep^&w7cgnq51k$VbNyhL_?ccMT(k67Sv4oaUkn3!~z{O$4!;wuZMM(Wh&kL zkAb;Frk=lTdU1It;}53q&C51%58pG=5}f#+DKxUQxfU%ZTB^%>RhDhqSD5scRa+cy zzdhcejhiL`VgNIhHpyJ&?t>9rmE}Fwb zcp}1vU60bcO3~Pfk{OS{$iBjZj!f>Y(gE(z721o4OsC2EI9OkEsj>~2`cayPUm&Oj z_@eNt+-8A~)rZ!upb9tl1o+6dU29ZSlnfn^YY-`s3HYe^bI;?)p59^MT>c2`0E4Ku zq=^Wp(dzNljqTK3(|sM0(5W>NEtkp6>PUQ&5hD|7Bz9*<3Tgq3jOU^&-rBhZ)gyTT zXfP`HD!@bdR3u6?^2m~GI073QZ*wx$wr8zWfw`w`xcK|KLS@{Nv$-+G(P>1bMxDIL;%r z7h-N%amI{#(6>-5Oj$2VoLxx#%4>w)AGCdDrewE0&hTidQ!L%jQ7^_TvgSmT47=L-J6+ z9>o2d?(MJ~vgt@k63k}f)Xi^^4yK$HDrM(-bWIgu1)>a#(AlIk?XS!>tJjU4zo-Ke z!iD2n(q!*$6-M0Ux*fhFar8tct;#?tv!|}`09$| z8lC~b8zA7(djBehP1?N$PdscOq~zVeomI%34_oRlASRCJtI?HhWEy z8|3X!aoytHkf-WC4bc$s&XT|IR>&*ajXykJvMqDIl8Z{0?oanvYnKw$YUDPyhlt6? z<9rZXB@J5M1i8H@T5m#f&(#4fG`3C_Mb+iRd+7PANzqb#Gy(3S(A45Ks%_dA%2-T> zZ)1tox%_K8r4MYE<;+)O1W9#1Q)Tvnqp7;oqP(mo-w~V2;VRMMr?W{&-haGsLOb<5 zabA@hdH3dO00+UxS^ZNlFf-iR$Jj6u_pj>+TMTYN;eUnX&U7g=rPv7%xY2jz}*Ec>5!iL{Qm%=v%qg}MxmEs>Ni13UdE8$bX+P6z-v zYC`_|q8NLI$x=*+H!2sIu8>(+C;i9Ex`NVTGRtK(txZqmnh2uJM8K$6i{xJ0u6Uaw6?8EqZYC zQTR#mqnQC!himeJR_%!?T^OaFY9hJbfy_+qt)|F)KFX5GH!tzh8o~3#RNT3GqUR4M zv4|#@mLxpiD(;FAv@<+J7~2^fCYhHOus3||b&oVHODi+4zcD?$te*~=!|NUgnpmWs zz58}%_j!?JUwiXsJc*A}SjV+7!n|wELuStp2p$Q(Lx1fd??(&k2g1UikV8J*m2pao z+1_e3BVa&BED9lOH{Dr~AfEv`3FvUBniks@YdNK1DG001yhZw-K}xR7VLu$9iW(Bf-`b0U8vs&v)a_e}Q zj#6(Fdk;P7!Ogm9Ut4_BzExnPLly?rwbArF0tBASIG@Xa! zrpQO<+tTdF)q}Obif8TO0w`O(!F_M_%@?^q_Yg9xYXuP4-wjQ&m>X`U|I?DqR5 z)falB%&akr#6Ql;AAQWCfUD9x_EAm1edDAjoH=6gQ8VgvbE~;fHSeIm(yy z*hF7r@+QunchGvTmWPxZHlQ&U-{PxVu_pN9pxjq?-6QWshStcDh-Y8>h_y0)TA9)jG$qZX?Y*VUN`XqUwN-zSvkguJvvu4_MNQjHqgJinIl^96by$5Yso5wNTGe3Ga zdt%xKlAO=9&TuL+%e8#sHup%flr#z#YCI4YVi#LuNoBB~0moKKJ=TL_3eg8}q2MD; z=q%HbK6G~IQ4IVkxh8sSC2`_{PA8jx+S(4Vxyj@&3Qh?6kuK!vxe~Jm;Md!HZ{Or*whWxoe%D z=07u*FNl_V*nXG*c42~}(0ugjFv_pv&dI@zj_IxL1xInzZJp7%dDpE_%`;o#8B21lW zHh}SR^@wM7o)NjI`DvpmxqvfT16OUt<1(<#bZv>Y?`EEue8w_d61 zTzh7whgi;6(vPk>DYUn}U*z|G7))@N7*$q7<#oUJPlfrFH0Z`&E8P`N0|df_-V1{f zys0tQb{lN8c^NsH>NPSVAIbjUAaxu*@XM&>^oiZv!Cgy*%H?9I=wQ4M% z7*Pk@2HNEkDgCm>AfhRhst=BZWA&j}_=iY1SQ>GianoKUWhp%+hNPJz)=EgDE7y9xm~C zviz0O8a!g=08;NCrL_TiC`pZ0UEJV?pf|$a{Zg&-HoR^+2Ll^!*YkaZa(%(N0St$} zAZv&w*AX;d%Ai_$oq5ll+?KJZGztDwtvl%Cs7?u7tbJdP#V?+k9;2+QSrcCWiltcT zC0e-l?R^|OM+fjD!tYTXTc&AHARVl>~~u7fmgGkz!WPi zwHS(Kfftp|kS$-V*YdMQ(om|}0QC@b64G2uXqQa39;Kc9VL9PFX$o@Q4js@zKc7sl zy4DG`Nqa%%>YwxAw9fhTonD!#2rwX9gb|kejMcd-LwXsWZH<*^eWJ+x9V&!ZnEWlS zIE=F_4;;GV-siQri2SfTmLV;G+g#s)n3Ys<43wQz;?hgblY;(^GwQKs zDu%mE4-x_gcg)$-?SScSjDBy5qTOz~uL^R#DZ_kyX)18FTX15nIgP7 zBy(EVdZlTKWCiPADYc7Dym0%2(sA(F6Y-bt>lTsTlOfj!4=(;t>yilvCkiL+9}dGh6Sm%@`>|inW~|8XgXc2 zFiG@t4r*MYg@&RSrQ*`@l^xD34VTrj456Z8r@-lTT!~Z34;p`NBa3j+ zBVC^5eL*hc+Rzw!V)4&?2QE#V8=zOdn$-{uaLcF(Rs0-!u1nAjGZ`J(tex>T>Nyp; zwsVkjh@4Wt58K>FaFly@RL8p--3=Fo>G8N9P;nD~aP|lP;p$ioskHIX8yEP$)77yq z(^X3g7VEW!8<7tptjb-R`jg8J-}H7{#}zAjl36rrr_6oo&_8wmslNCfz8E8Xs!pA$ zCRU zFTF-8cJ7_s$gu?GEb;oiA)RZc2y(&8F{EgBfKfhBv71@rI`Ea1D}`uJiJ*RYbY37f z^stkurYoc=xpaJUhco+k(T7(VN0M3+_3B+_Va-0VRTp2uXQ=Z|&1U$4nk@}dPt9h6 z-TF2KzDwS7$fIhu<7ct^qdCnTA=$j9(+V&o??=4#>h*cq@e!$eHFX7%?CklzWPs_0 z?;c_kA8O2TA|tbYg$-d>gBH#^M;B7jsSrFB4aaT)Qe9B|BaCcT?$Z2 zudc`(rV37_}hJ6x*o7qdyvEv+x~TCfXMVaZ?pDV~LNbYd$g^n}#7 z-)&&sHea_ZwacRj7$0A}-|Dwt8hz%xm*3%hv+l;M2Kx4glz_6^`Z-o*%}W7tj0y9{ zH^+SG9_{RMcg1;gw$>3>7E;@9azztW4jWh0LP8cC2Q;#xS~I(~q+I7TIGVdTo^kYT z#7FF`aOgD2?e0MUSM)~bth|sIQ|ou*0o=M3Pk0x${lR^i zF9&Aa@A0)l84C|Irsz%`*jzN;R)Yw#$PM&2R42Le#f9IX^7zYHKoDhXu5#9)Pk}^67izorFS~u;I7wCT-*? zJfepb5xny6eMGrgiL1_h4c0)E#rb2{(`kEJCMfrPMx3)|=GpK|6Bs zfhj;jFH3^h9HJu|vJFZ(yn4VkY{OEtJSc(4t diff --git a/summary_results.json b/summary_results.json index 29cda785a..358fc501d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1773, + "Performance_Result": 10.1254, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.417908, + "Performance_Result": 0.421261, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72734, + "Performance_Result": 1.72364, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8814, + "Performance_Result": 23.3315, "Performance_Units": "Samples/s" }, { From d32893edfd48b1532ff7f953ee7820bfbc94b53b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Mar 2025 17:14:53 +0000 Subject: [PATCH 328/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10228 -> 10229 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5b2d37d03..fdc535da6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-26 17:14:20,192 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-26 17:14:20,192 submission_checker1.py:3260 INFO] --- -[2025-03-26 17:14:20,192 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-26 17:14:48,171 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-26 17:14:48,171 submission_checker1.py:3260 INFO] --- +[2025-03-26 17:14:48,171 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-26 17:14:20,192 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-26 17:14:48,171 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-26 17:14:20,192 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-26 17:14:48,171 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-26 17:14:20,192 submission_checker1.py:3276 INFO] --- -[2025-03-26 17:14:20,192 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-26 17:14:20,192 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-26 17:14:20,192 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-26 17:14:20,192 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-26 17:14:20,192 submission_checker1.py:3297 INFO] --- -[2025-03-26 17:14:20,192 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-26 17:14:21,235 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-26 17:14:48,171 submission_checker1.py:3276 INFO] --- +[2025-03-26 17:14:48,172 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-26 17:14:48,172 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-26 17:14:48,172 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-26 17:14:48,172 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-26 17:14:48,172 submission_checker1.py:3297 INFO] --- +[2025-03-26 17:14:48,172 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-26 17:14:49,316 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 903645975..07eafa382 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -955 +956 diff --git a/summary.xlsx b/summary.xlsx index c6ea091c63c3fa340dc60fac29f6e33c5a7e0a18..c300d304ed25c321469f1f8f1d94976c07deff78 100644 GIT binary patch delta 365 zcmV-z0h0doPxVi*AR>RTlASju0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!K? z!!Qtp?>t51d*UQ5rLvM#0@M?gkSGFiv+QkZ5&y_GqP@)bTT2*%iGlYL>3$IoiuwH2?oi`?vb|+K;XOqh( LBL*2I00000C5)sz delta 364 zcmV-y0h9jqPxMc)AR>RdZsIT{0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!E= z!!Qs<_k2ao)j>AV zFIXqEDp8iCC<4Va7iw3c_j(;apvZZuIBm3o5*@*zRdrJ^LzsWI@M^UI>jk(dfRrQ5 zlxXk0As9Qh2T5HL94d%gZ6)=gv^#7l`=&dGPv2EL=U3AuvKf|A!m$z=!Sg6yf z7@(@)j4%t-YgMj0Z1QrR6Z6@&-EHrd79tF z=_1aTb#_m(Tau?=r;%6vXP_L0{F^ue`rOx_p!x%|XCn&)1-fqHFeQ_BCsYAulguY0 K1{fs(0002FtfW-{ From 089f399ba396c2bfb2ca8fd6baaa789bc17ca76c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Mar 2025 17:19:18 +0000 Subject: [PATCH 329/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10229 -> 10228 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index fdc535da6..3c9b928a3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-26 17:14:48,171 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-26 17:14:48,171 submission_checker1.py:3260 INFO] --- -[2025-03-26 17:14:48,171 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-26 17:19:13,283 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-26 17:19:13,283 submission_checker1.py:3260 INFO] --- +[2025-03-26 17:19:13,283 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-26 17:14:48,171 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-26 17:19:13,283 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-26 17:14:48,171 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-26 17:19:13,283 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-26 17:14:48,171 submission_checker1.py:3276 INFO] --- -[2025-03-26 17:14:48,172 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-26 17:14:48,172 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-26 17:14:48,172 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-26 17:14:48,172 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-26 17:14:48,172 submission_checker1.py:3297 INFO] --- -[2025-03-26 17:14:48,172 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-26 17:14:49,316 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-26 17:19:13,283 submission_checker1.py:3276 INFO] --- +[2025-03-26 17:19:13,283 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-26 17:19:13,283 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-26 17:19:13,283 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-26 17:19:13,283 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-26 17:19:13,283 submission_checker1.py:3297 INFO] --- +[2025-03-26 17:19:13,283 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-26 17:19:14,309 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,15 +33,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.47558 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1254 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1879 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.32275 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.435102 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436732 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.3315 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.40996 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 07eafa382..6943e4dbe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -956 +957 diff --git a/summary.csv b/summary.csv index 84020e87e..ec94a6126 100644 --- a/summary.csv +++ b/summary.csv @@ -11,15 +11,15 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.47558,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1254,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1879,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.32275,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.435102,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436732,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.3315,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.40996,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0d5c79955..b04ecdbe0 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1254, - "Accuracy":"acc: 75.000", + "Result":10.1879, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.435102, + "Result":0.436732, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.3315, + "Result":23.0922, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c300d304ed25c321469f1f8f1d94976c07deff78..c0b1f1d08b528f304a0eb21d903ed379c595bd97 100644 GIT binary patch delta 2738 zcmV;j3QhI(PxMc)%>x8;g@2!u(E~<*tO3km)62gf8RR1urABBR$YaB+9Un&?q7^dF?i=xM101^T=5nXsC|2<~p|X#y)I2UE6be>h-*) zWX-V}uXTjSfe6OKl06$`?SU7wNCh4Ne-#x;wQPqhv$9nFX0iv6R|P5K1%RzD9@CZi#f*puXv)olp%o(^xBJh?N%nI@hF1T55ZA1Uf6Hk; z$n8$wx8JRLTcYdZqU-CT1Jm$Ghjid7k3gX45C{|<0)e7KAW(Gu9il_%P;@)!mwhNY zI55>6XrhCh8=V8PzD2Bp^4|6Thobw8oc=&`5B)9C*-qE@4nO$R`MBzQU3H*AIMN{< zu+UBr2vi*cfvQ6wP<03de3{Sue;ukr=pfsR9rQ~dstyiJbq8J8DM8MS&H)R_En*e6 zMgHfiu3A>Ps9JI(txUZ5-p2y${ucp$MtXlBz`6fUfR5dE{65^;Snu}I87Ws}Lei8Q zA(I7bm~W%adBe;Z%$vRVFSy#w=MhyaTaY#<2ZH7+ga2rbD7?Ys(Y2Mae=wd^DKW%r zA-DA8rI7DPa)Z?G472VV^L zz!(`a^9@vHS9__C3y`qJ!y;Ib>!4iGoEs^LWukUlu2p4ozENKbR@7IrSzj_CShl%K zNDz@y-O1f;%$V5RsFJ)L7aNU#0#Hi>1QY-O00;m806zdi6gac)vv~+$2Y*dp=gpK} z4wYyLaipt@R)Aw`SJ<8945{)B{^!E^rCk;MCgHV>R_%IDXNi?p6 zvdHDb5&)D%J|6`}A82WlPJf7_0L(rl3>^i`?E{X2itRo~<feiw8^>DXc}>((}!q0>GUBQ;F4)1mrNtMWE#n(E^_*?1d_|51T76tA82WlPH^KA z!a6iePM1ssYgk>e)#ouVf6CTnB}JB5$LejjkJfe1+FD@lqN#GwM1S9iBi%kk<4Ly< z(IA>7+&)YLXc7&eS>*O%2>{I^w~q$5545yNCj?D!`;ahL$gQ|F+G&Jjo2eTc@Bb|0caG)vfhm&A|sW z!R|vk!L3UO>(DS@lYjYrVAibI@AIQlErbHtH; zAENQ3--l=r%@TeerU5jG2GA_>`>+InW|7}VgWm^Q+N2YLCis0w7|xo^?gO)C#de>+ zhsli0ZH?AhV3$M_26kd5p~e)mKnSeWjojULR#CKW7h9+c{98Z^V&a zAENQ4*N12j$`W25rU59)c;s%0F7o=YP5{axua5?=545yNC(5)Wm0*K5;Pio(Ht9r}wj>gqJ~Rx7WKJI-@;Nzu@`?1HMC5bz`RHw* zPqr#IU3pL0*n0KhG>C@5NidnjMlVcH*D7+;x_TevxqsTfUcEZ;L7w&(&ido*T@T&K z>P3&qcs4ZRKUc4ibh1}Snpv{m;CwL&jO6<~IIIPC^Yk0aime&_yTR(^k;Xh4htW9s zzUn=Dd*x!%J(`C7m&-vdX5&8>!OIuD7ta2Rp7ZJ#L%%W(jkNT8XHSZmpxY0k?5Fg$U=W7M z{SVnG*}T^C=)bdz6PN)7;313~d9x@LqXK^lMpH7Cz%r1srUV{mw8OB1l6Mr)y5RqG zz~VI&>7#{l6jk-Bscof})6~DEz-kedWk|Wk@P~ihIRvN+L@Ln!46plMe)@4@+{kyqOpI( zDM(FEm8x9~UG01=xAUOF#4;2~{b9dvs0sx~N`sv3zYgz=R9)(}bk%Pq`~4{j0SZ>o zlEHF=bx(ec6s_X_vF*vK>4>T0&z1Kk!uMs~I~j4u1OKTKI|+{}@$gsEd&z$L)MzpB zP5Pv-7KVMDd@z#ociCrF-iO&f5XXPduk_dGj$i6WVLWb3k6-LZVLYzLPb~KiHXhYP zxFbK09La{re>+jLtYci-Zw(BO?D_OzHCulXL^4&J3*GO z+rIR<`!enko!H*qBU{Y}d1iYubHJo=A2>peRIn2s{@j_B!UqC!u1CZ%bG5 z1&{13)r&x%k+hEK1H`@R% z@_dFB1b-%2V0Zwgb|Rt!Qt%0Be_T5P1jo>V0v$Xi8NnxS0H^%bhEX|lDj9kUSFJ~J za__vHj9u4dT{&7~eu6)jFK^?jsno-%IH0ByoN@;&7}r$zyoiTunGef&;K-<)eIhUM)~l(^b0Bk@&^8f$< delta 2740 zcmV;l3QP6$PxVi*%>x9^a_)AM(E~<*&;Vv^)62gf8f)N{f-{v~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw=GXCG!)MCA^Mnv# zNCO2QnTZhPz_NIlkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ@*pkD#=?Mq{fP}4 z$7x88X;|f?5RDN=vOvi2d6Jgg91o*ZK2!M^jKm}Jn=_a^`ldA=)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QEkVbln64H#RR2QMSDBR!;%NR(kCNu!7}6t&y1;)3Rn%_C=RprJb6n(Nrk8~d>BbZyV=sn_$G zku}F^yw(vO2O<~`8GSZb?SYrGNCh4Ne^n}yYS|8HZsnQ$&1DZDuS&w=MOws!$xenW z1mR-At7V>Y)fDi>Q?G4Xf)P@;AC~Getn2Ay&1Zzm-JDyiWD$g_t!T+EqA}G9%c@nr z3d+J{@#MC>w%uCq{)>u&D z-0t*!``xOyCAvN?y1p(tFb$7%NC&R+2n31_fk4qA5GXnX0!7!~Av%N(MYn@~*@vQo z15@3BCOXKu(K#UNTf{0T?_K|YD7w$c=?_Hr(BBfB?R0(b@Pn<+$5rR+ssjzekq+sA zg?55Kpz06^R2>3=szV@PXFl_Hf2aA1xenMWEx3`9SSD(>^Nspi(6YXg&H9oGLG#UB zLV}1ebtk`WV@k#5MwR647uab06HrS71QY-O00;m806zfRM5w&)vv~+$2Y;JQmG>;x z%b^l2A&zwQ5RE5oJw$_4mN51(4NytUH9%#Nw}*8CR2I2=lr-O*LQ9*C5~u_d4~fFn zE3xt5>a}7Y&tDJaGj;b|s9k%HW%_=OyGQvsP@-|fk^UZ{@ub0rXb{R079XYoD2c|E zP!_p-SOS2u$mgTL=mRZn(tinY6oA=>grTE=xqZMntf@f;^xm4SppY)d5 zw7eXrk2X1%8cidPbovmDC!Ib-16(qVsio(g|){ zLRg1}$?1}bU=6D)w)#8<=1sY<*_R+fTSz8OtT{Kk=nt$jUairUaXgulm zAsR%pgxiN{08OF+G>hCmECHZd4cyOZXXf`3t5&L7P7ixv(Mi{<}-Es zT{(K+HsyARQL((Xevh-L}957PjeL<49R*?m|7K(om1vpM*H zCfI#QC%AP9VI3L<(Eyr7ejk!94@&@O z78!mt7=ECoO*#Qv0Yr6J6p&fQ@1raB`#eVG|LQBKq`p#63a^i{l%KPQsqGvn(Kq5q zuMg4q((6Ms2xSSc57PjYWIS@WL>GB|SSJ8wk=I9q*9ThKq!VRYl1ec9&@jM~xqV<#NIKaoB+V>YZ*abt1V-|G9vs$!yLtKzWyRKv{@q~p@$Z{7PIl6i{Ryp-V0~{MbCNli=kf`helfZy)>*?x;fEsJd2Gu$}6RBXd?Q57M;q?BeQPAxNQT9`MTQCU2 zdGueiiW8Ut1wWj+BY3kX6{7-wic&KrQwb~sDQimLaYj1~D=2wK0j&%E zPX{bsLyIOIMPS*=yEz+E=D)=dIogF z#Y)Che@ZhWr`++`PQ?tHwedPsIRL1!sg=pkCXmz}H1MTtl7TgcI)s@W(Fb z6K|(QhCbj?2tbCoXXRt|^t@f8Do^wKIGx4$Vw)O0kZexo%dg|eoBj(>o`(FJcm@o) zZ$Clx2eW1)gaQQ4a_)AM?j=G6+eE0m?~^qqTLFucktQM>Kb*QFcmx0dC=~zz6aWAK u0000000000fPobilQ0z)ljSCi1+bExHzt#ICsYAvlglR~1{oy)0001?xId}@ diff --git a/summary_results.json b/summary_results.json index 358fc501d..e1bab85b8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1254, + "Performance_Result": 10.1879, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.3315, + "Performance_Result": 23.0922, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.435102, + "Performance_Result": 0.436732, "Performance_Units": "Samples/s" }, { From 288d28ef2f7b5b2b1b6c79a72cf150c8b5087cd1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Mar 2025 17:22:04 +0000 Subject: [PATCH 330/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 10228 -> 10224 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 3c9b928a3..035ba4fd2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-26 17:19:13,283 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-26 17:19:13,283 submission_checker1.py:3260 INFO] --- -[2025-03-26 17:19:13,283 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-26 17:21:59,723 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-26 17:21:59,723 submission_checker1.py:3260 INFO] --- +[2025-03-26 17:21:59,723 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-26 17:19:13,283 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-26 17:21:59,723 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-26 17:19:13,283 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-26 17:21:59,724 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-26 17:19:13,283 submission_checker1.py:3276 INFO] --- -[2025-03-26 17:19:13,283 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-26 17:19:13,283 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-26 17:19:13,283 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-26 17:19:13,283 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-26 17:19:13,283 submission_checker1.py:3297 INFO] --- -[2025-03-26 17:19:13,283 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-26 17:19:14,309 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-26 17:21:59,724 submission_checker1.py:3276 INFO] --- +[2025-03-26 17:21:59,724 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-26 17:21:59,724 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-26 17:21:59,724 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-26 17:21:59,724 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-26 17:21:59,724 submission_checker1.py:3297 INFO] --- +[2025-03-26 17:21:59,724 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-26 17:22:00,835 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,18 +30,18 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.47558 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.34208 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1879 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0601 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.32275 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436732 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.40996 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6943e4dbe..1c0f77afd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -957 +958 diff --git a/summary.csv b/summary.csv index ec94a6126..02fe7d2e6 100644 --- a/summary.csv +++ b/summary.csv @@ -8,18 +8,18 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.47558,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.34208,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1879,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0601,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.32275,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436732,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.40996,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b04ecdbe0..3b6f0f64f 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":4.47558, + "Result":7.34208, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1879, - "Accuracy":"acc: 75.800", + "Result":10.0601, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -579,7 +579,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0076, + "Result":20.7395, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -634,12 +634,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.40996, + "Result":0.419601, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c0b1f1d08b528f304a0eb21d903ed379c595bd97..d6d9f3c82cdfb53c0ac7609c993c2615ba706284 100644 GIT binary patch delta 4625 zcmYkAcQl+^)W$~%t{ZKTFoMA-(Ia|vql_9Q38VKCC3*=)i_GX|5G}~)HLg*DC{dzB zFA)SWTJ+wDkGsD6t?#UL)>-d*&w9@D+k2ls_Bqtu)vX#JA`eb^-|&hU00;zPLdX~~ zyF}cGVSv5M*tb?zCbW)MPzK|cWr^iuIb%k$XWpdA=!=bO`Ad=cjo#+m0I*NL^t=c~=QRbH`9IYgW zdiafq&3&p5&&i=^h}rM=be+NNg9ZA(wud6FP!ynUC-rzIqvd?+I2zSzg`V92bn4Fs zRC$JS24@Vhog4?5kJDt%3&mKzS6;(F1bq=@WY>gtAnt|M!6u>;)=OH82`D8avsln%y_rSeQgA#DU|G%}-KKw&H;0As^G# z&Q#&UR}uy|99(lhNY7<#{1{}qsM_*&9oj0HJjh5A>vsP9orZ;!oq>umSlnnU>>W=1 z4hsR|886y-iqgq#IsJ0LjtcxyqaCjT?~Lr#iQ*?-u>MkJm+d)0x7%g#(bXiJM!%jl zbzP)W3M3g(FFOo(q}E?eeJe;OLfh4v+$iiF^Ts)8bw;u zXlN8OJPs@!yLwi9uNkF!L*jde*ITLZ8Dbs?se!1U+lRpyv!(T%jdFsGErQa*6baty zq?P+*#L<*t3L|XPJkrF3DwFeU(vg%Z^${UcZ?O?X2D4#Y(mf3$Qs51fPF|iJvugxT z<515U74&3y9{7ICJ9uX|qGGW7+3H3q5k-3vavu;yxHMNgzHql6Og-`|DjRWlEw(jTWu%;Q^st=qI9 z^=qr|7<+kwN3HF4SqEs|NTkU$)`{Fhe8ur{MEMIWbIS%iTz;b8kfC}AlqjN6?bYwo z-<$S-QN;1=UaS{rYf|x<%6T69#p{J^i64JAM+il|wv4@-en!HJD{CI_zN2omXvJ9G z^WkEEyCpG&bGi|gUPxb^JSpKdxm;Nx`d7? zEA5Ga44VExD})TwO+^dNZv}{P=rrO&{D6bu9n&Y^%B#S`oCtTTLp2%l@J@BwetgF? z4{YTkPW30#oiR64s?{(2vLa-8P7*LgY*d^-%bnRjB978dj$_`Sv2Fk7CU!p(`VdQG zq!AuM{$mtLH^7Ql?f@!~RR#_^l@(kYhF-tO+$7~E#YOuiRP5%7*I{4+* zuijbNnr4p+XPY|UIQt-`6O3N!4+ccs#TEF;h*(}d67%YOW<3$kZS;oMfw@ep7InPQ zC&KLNFyDOwtBQ9j4>XJ4)s|ReVTLY9aKMkLhnl7&@{VhDbTx%uUZKStY+jZ#4el_- z*2Hga&a#M6f$EL%wRoi{)4nLasaMSt!|L5Q#^jZVQDG71RF+v<8}SVTq%St1T2euxn!g0zwzhC6|oO5)3j|6 zHbPgVFyfKt*Rxol9XWK6g+xyBeD`8%=HR${aP8Z&HIL2<gAT0hN(wEPA+jV2N1@bq`r=X znts@sC0tkkbh>=?5BT$&g%&dTS<3qkC2r*iTQKp2pt@$!~-Al-xw-7%tF|890@ zv!Gt6%6vu8%v9fstb*bpy+axmf{~ZWp9rzE$$)4myWWi)yGGE+$m>>s3Q!Q? z9!TD9(gIjE6t97idOqZ}k!Hx#YPiP<3P=cCc@Z;HGxPUiGt4i2VbCk&xUnXBUjyvF z?aualB2!vrc>c26*pTb3$CRKWUE*gh(2J*@N}_EXW9r~^RUk~XrRN|d&q}MB7}$p; zt{C`2kvcre^`6IIFoOC!o7H~FqC+YIF84rshWA%M+nF$*z2$L)Wwv>u-h-b%OB>(m z#LCAyaMGv#I`EXOqR({`S96;c*|hG>X=3FI8g!chYL0)3??lUdB;`kuR6#&(Krvwh zH&HrGG@y!KeOTYqVSx@lolk4mr-q>Z{=v$#&z3#yaXqhpys4XZ8e`gYc^Nk%(O4zU z&VKNyZTx)AYu4+1)`fbPL{2>Y_Uey#2(4HpUoVsb4HH%18UdsiD~kBgq(cZ2jT3Wg zrA`$JU7RpFWa%R1%I&`zcr^v;H3sX*Pfx;cC0S}KN$VOaYBGK2aL>C=Uu4hvsg=>&?2=wW$^QKG>vbZ){@NL+FiRdk!|36Jw-D#Ls8Bz-k=(&l)^n)O< zL0t`uJ5r1BCg_q5=y6F0_%*5Jzj9-U>Ga0}TmHUtS?P<)dXD^6!7wgo83Qy51i?6l zYGDaP^db@vo9rSmLNE@j_|j`POl#^UiEV;pP#+_@(B3RXK;c#S7rNK%@`JYY%PH2Q z_q5=tIT9u-Mm}_n3J=yM*g(1&K1h;ueyd3qd$igh8IfDiRzoD< zI*Jm>1ZoIx4-4m_k@*wbTF;#3U?rwO+T0yoAk8pa&nISs@1~$7myIEW$wTAaUX^um zGNs$=g*MB=4K_=USks_PNRIb1&_x&@)d(2F*)O9-qy>!QOXuG|s3LB&P0CY)bqqYL zxrsy-;x9dBph_Zy{c|m%{Kscp#FE87C}UVyLGjPy!{7F^hdy&!Ys*hjsnlwXfyl3z zIu@)VuO`FmG?HQnD7U(2rUM8_QQ2nU9FQowt!QwHC7w<1Qy`}}i)r)o&Be2g4?%_7 z*?-r*-1#SYCoSM4IfOFh_XCgoFK-=X)^^!I`PGr#Z=FIlumlL+7#BHyf~Ur|hkCyb z!&1f9B8|w~wv^k4_1iY(EFTvC*oiRu6Quj)@zJGesq2_-;E)U4JbpqgGfOkBl12?4 zeB-qe1EiZ5#_ zV3@!9!m;W?K{L}^X_&Z)>aU0teoYEKW3zu|R(IGA(x-z359?;R%?8Ao=La*zd?+&< znLYC7Z>dk%Ku}!d{97l0HkLp%=JEzQ*Ldj!G4&uwf%Z}^pY~@{WECX^k$Cz7sr^7S zg}i*<3dUE6?_u?LdBfJ}$UviKrW8UAn!|+U0&T{#hxL0<+Gaupx zI6-oB?*`cW@?YS+|A6EF1@^pje06yXOxQw*T(9B*<6;-oUbvhF#P(lL7(Imr7v$2} zv)VnBcc3+xk$Q^t=uEP|@m4T(MlW_M{2gsO_(7IobCHMjUiOgwZ<(zpC#292j%w;j6Ph+#xBeR|6e5 zK2Tj;x+p!_JW7dyoj8JyEd)tx)8ej8iG?f<%_T2{0td@1?q2mtijI!%6u_K(zL{53 zxfrRD9&8Jrj>knU8oOwPE5aQE-JDfx0$G>$y?}%7az^K!EIJ^66`jb(g}cckt8+$Y z!W9{~PULNiY!A$b@~#PS(X?p<#1j0k)Now%`Gkn+MjMSnpb?4w_vE(4SYL+cPxs>q z;rB0dD2+Zxu?N=($74%F#A++XJu}Yh={{Ao71TJtdJ*=T+*Ep>vRgk3WI8fgh5cw% z@=@%QlF;+tUF6M1AMW!VTE8*J4r|f{qLhU6esiMD$tH*0LbacajY58~eaq#^J~z*{ zo6#3Vq)Of?Su4@6jrmaa{^PPQi(#5k6nlyM<_X2K55Lr%5^-0P3-6OkHD^;R&ns+O zKk~V3*q?)*w)DT|=GgZCCI8HtdqIzr<&Sfq z(3E=)NPw#amUB4KUiA?}rK5PN-JI|aq@l`(R3hZLdRJ@(j} zU=`0rFyOwEWOt5kYhFUv%KK5(Y_O0XV++sCO~UPjhA3FYC@iYgihC*FbiETO-Oxe| z*>m^o`^>zP^RxIWi=~~J$%HV^dOYxR|H;v~m2S=QIn7i^h=PAU{ zWzTg@Bm48j>*Hp|*u4eB|28|guZ(aq;s$#@OALe^jBxG{rjFbRCeLdUm@0IlsUvdj zEeTUHfP2!8a?#X=B#ui5T=ePSA-@a~%F?fVef7lV%f!#_tQL%j^*s&;4*w)8sTVsp z5*SrWK5eV9tQdKkN8~T+zRm0QP=azU>z`lY#5vH!aV z9RDIp0@yjmwodC_q!J$tMl*!6EM}(+?Vs7zLKL(Wqc4(^ogSGiQ`PYBux*4aXxl{^hnga81| bf4gti&@f6&EVmqiHRc;P2MHbL?XUjc$J+_wL@&tDCWmal_ zUpN>|_2hU63}h-4734oid3LaMTKVmkNb5m3gPk^+95^GIq4QHsKzfqsqK9eHL*;0T z4084PRFPO#C^arKxhU0rCO3s#`VJ>e7;4|}NtNsyUO6Yb6!b%BO&tt{dc@#0%V$1{ zaV5cT_9b;diT$1fw)^W@;{oPPzaufuzjvk?+V)=-LYN>JatYd$DNOb5=C@bZYnaDI z8jAf)o);sE1{extfD33<R0LV!ww7^QTR5`OpW2bW zi#HNg^Vb=un7Bw^;_mX?^S$K4-Ce)utFWGi6q1*f7~^umg=KT*kkx$DC=cDAKl&4i zJCDo$t?EmEDKqavkb|%joFz{3xv_h1CY)2S{jo#O#^tG=8s@y|XlJEV1!DR)*iFRHGzv9|%i+cIT}>Tq#Q0)n0Yu7}lN5)@0LwTfjTr zm#uRWOk5=f%xfP}K3F}}(l|b8SMsc44WfK>_2d&9yVL=#iIW+kG?6B$s;h&~9KDtQ za5$S&INA~CZ}skhqt8d9M>M8i&1x(jN%5N3HR@_JpzqK1%z5;uPtKNLnz=YR<58sd zbjg5B004lBxKse@OtSc$D*yloHkis3wV$*&KHY?%LGGjmX;8M#>kD>yFmx_I^EWs4 z!#e@-Hk07Nh{*SE`<^3g2vOLb@`apm{9@IyQj~iu_vPB67273aVknQ~2oQf-C8L8m z3n^ec(QK>boahLY&`rGB?lP@E0xq!Q&(%_^Z@rpnz%ebI(^adGJ7&s$Tp)##Yj3li zrI`6A`~zJ(Y`YzS=pTpkYKo=BkkwR~u9n#f2V)-hc@t#YQX@V;xft!8)2>!;ZK zuO|*g!L%JJHDKjqdUq!Qj~B;t-fYh3VS0+WPKX?RE0X90BTTN00?o!NY`|H)Bv!(V58UO48a=I< zsgpeC)dzX_!(Z1m%qxjXxS!b7~jASE1%aX`PxsKftSF# z=EZwUy)<-CeWP7YKE|VuP>|8CLh|X-Y-?*;(xWJ_HvJOEnK{G!2Ni!6wVd7v8TP#O zMQq%LZCtjOase!Z6pfF3NuQv65psOod|B=D%e#>+W^`=m87e!D<w zJfr3C*fGV;Go4iKPj0x8r?1c3ErYq)EwSV}oqTf*lrZ#Ovv>WyCqaG-2Ob8vFYv=8 zm>0?4$0ZchsFSoI7#JTK0*ZzVi}0YsbwM=kR%~>TAaaPkY|=U$1dmIjjtfhrOc+s+fz8HP<`JZXYXtyRFok$KA(03 znBKReRu-Qivm$XD6Pu`Oxh{I24Y@NPcURuu}Ll5Zl-IRO~-B&?H-%rK_erQ>G+;Y$X0d7K|T#lBrMeA00r9mz^qaRJU(y&7?SB#B>ZX z(Hs!0%L_dmiIhD480*lPvtXF~yxIKLI|1f)4RFYho(Znib5UY?CM(@y@Dzg!u zB-|knxw@u@;aJn#`UB|y2WP2(@_AFlH8_xU%O_aqN<4X;cn>VzS%*l>0YsYwmsS;U z`?;5B4HJL^99SSO9(2k{!dBCT*Cxqdme9qLN@U!k`c#{DJKf%JPdChih5nG&Bhv%jjq{Bv<@-Xn1+(;z0o9_I(*5&`^yQ75 zjMb^Th&h{p(6!S!r8fu)!YQ$p8x9TTkA;);o)-6=No5J8Jen|kv7hlPU|_5ctub&)tx$b21O)}{-^^L*DNN=uy&i^%PsjBitr3!K=u%v$y$av{#oXfM zD69#uCSvN?zazO3+pJW6Ex)FFuAuPjJCo(~lVnwBdVn_5JDccuO0!Dl^MKz$LV2Z4 zYwFw62+`3KpIN2e!Iz+T{)EF5hIm0*Wmwp~)T&bnT2(ko`dxTU_i6_SkgTB$Cf5YJ zxMB^W3IP4ekgHte{o7;-p)uRa+N*>0Op3;RgGJNF7lrlxZ&p`jk+nUIUtKu73P*_y zHx79@7Kx@y)8lV+;OHs3>HVt;i0Q)t#b^!wK|Wgs{qsPY9?;#~x_WF)QUcXG?g&L` zq*{F0^r%l9_ZytHwB5m#dJn-CqWI^OiMrL(5A$R~XO;EHTtKw>7Awbg)BplseIi9Y zeQhmV5UBs`JW0a5yXI?E(in3V@;A3sUg^x5x@Qw|r1Qz>-gdKv@mhZL0HfA4GO7#f z55}oU>rNumDTW{%s`&05V$zu?4gWl1-%Gdc+?8*u(52cMlCZ^|A#KOYCwC7Do+!H> z)L{5@37Iny;JI><`V60f(kwX_oX=-;2pG3lxDe^WKmdh+-;hX0Qw)jjpH~52IH4D` z!WJkkiESDxaEjDs8_4Y6A;UX$`scMq_B(H$G!D&#;E%m zb`Op$%4f0xI|{2f(;Lso-NSi)M%xg%6-^Mi^@41O+=O}pR-Q)R-2YW&0O;kbOr)s% zSHIp`ND9jK__f^&sL%PQ`*rs9gOAMx!s4`#3bkrdzM8s`H+dYvce9a(YZv0;+baxk zn@0W|qA>qb==CiqHH<&^Mgaq-k1vZ z%ci!?RdkmkN93^V*~d*K1Y*^GL@6`kGObTd%xPJLwXKh}g>~R}Z0^A(>)loL2z(w9TPCOrTAt~yLrmgdrCUK~L`mrY&z zx)}2@f*DDxi>zp44yslcH(l?%(qjIaRpb!#$_6*2%M=u?E{;A_Z;0?GdorDoD$qt} zL7}nt^<-akCKXqW`Yf!iZp0bUo(&fRF6M^;<$9V;IuP?}WEv!j2z+{4;(CfvQrGtv z68-qZ$l5swC8MC0S~>Iau<=}KhcRbzejc*1p0M12Smqw~7-TwF>|6;7lxt*(;zvRf zZx-WM?R{@mI;(2w<>sz6A@RE>N`s7u!Qy4-TZ{!|T2ChM(eBeo!(46uyZJ(}Kq^I6 z!DIRFU4pOeV?A}v_E#$A@w)`uIQl)Dlk1Cq6MfIKDnE~9~4~}343Ygml0N(iX;mGd)q;^F;R3HU@844xkQWRis z6Z8ZWUyMZ<$@$!1527)v(DL%$ZO)Ffo^Z65vwR#9s*)z|xm;HIwWCxv{|zYHn^Kf* z!sN6$D$+fJzAs?uJ#cSQ>#}#o4vhpyK`8Ri8?;snf`e?tAoq+{Iz2MeTqUj03Od|E z!mSZ#)=R;B`KxDl_NXS;K3)I`V7xtlI4SwR{}=2QRPFlz&Wa`g02^_7{7<}i=|qKn zNyUME0A;`?Ut^&8zk?WTyZ|5PUlRrrzi@<@0f4`w=YQhGOD1fufEr0F_Nt&fi6mA_ dP~g86@)Ber0b!#Al}VhiUj+Fn*r0!}{1=sww^jfE diff --git a/summary_results.json b/summary_results.json index e1bab85b8..35cac1f7b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1879, + "Performance_Result": 10.0601, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 4.47558, + "Performance_Result": 7.34208, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.40996, + "Performance_Result": 0.419601, "Performance_Units": "Samples/s" }, { @@ -860,7 +860,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0076, + "Performance_Result": 20.7395, "Performance_Units": "Samples/s" } ] \ No newline at end of file From da27ce47dfad5801ebfaf4c2014740bcdd49a1cd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Mar 2025 17:22:53 +0000 Subject: [PATCH 331/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10224 -> 10225 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 035ba4fd2..3649039ff 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-26 17:21:59,723 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-26 17:21:59,723 submission_checker1.py:3260 INFO] --- -[2025-03-26 17:21:59,723 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-26 17:22:48,320 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-26 17:22:48,320 submission_checker1.py:3260 INFO] --- +[2025-03-26 17:22:48,320 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-26 17:21:59,723 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-26 17:22:48,320 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-26 17:21:59,724 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-26 17:22:48,320 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-26 17:21:59,724 submission_checker1.py:3276 INFO] --- -[2025-03-26 17:21:59,724 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-26 17:21:59,724 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-26 17:21:59,724 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-26 17:21:59,724 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-26 17:21:59,724 submission_checker1.py:3297 INFO] --- -[2025-03-26 17:21:59,724 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-26 17:22:00,835 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-26 17:22:48,321 submission_checker1.py:3276 INFO] --- +[2025-03-26 17:22:48,321 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-26 17:22:48,321 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-26 17:22:48,321 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-26 17:22:48,321 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-26 17:22:48,321 submission_checker1.py:3297 INFO] --- +[2025-03-26 17:22:48,321 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-26 17:22:49,270 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 1c0f77afd..3857e0ee9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -958 +959 diff --git a/summary.xlsx b/summary.xlsx index d6d9f3c82cdfb53c0ac7609c993c2615ba706284..ac0cde6bb329727e0e74a399f9c2bde00e1bcadb 100644 GIT binary patch delta 340 zcmV-a0jvJ-Pw`K%93lk-U|SX@lOZA=f6Gq8Fc3xed`0Aa;yj>KR+379x}p*iMO5r8 zd)ivW57~w^e~**4X)9&La(wTcxifOPILIdY1?z-X70QwnMWC4GLhUN_v0lb=6gf{7 zr;S!np$;5c)ORH_glP+JRvWNhfQtf1Il@eZ_TC$Uv15CX)Fr{8g1FUIQXfjYf5V2d zZ@L4Vr|Aq!@RUI>J~%NN+qO;GVz7q%8Q!d3--lIWq54xXKy}F(VHT*@wl48?5f0Z-=U3sxTLJv3 z3;M*{X^~?e@F)Zz$GB(Zvv_`4CNEK)r};yi7I8jXXOAS$$^HDRA9>S%0m{>me-qDu mWA58eQ2zn5Vj~L#1q5JQ7ABK&CsYArlgB3`1{Ea$0002bNSU|* delta 339 zcmV-Z0j&PWwH1?Q#FHOe!B642ZTskb%ySS`{=l=#53 zV6D+mqXT#}Z|*8?DK`$@oU!171fN8Z_LSQi?Lx2=W6yV>n9rg^M{#4EVj-4pe~T^S z-)sxGAY_IW1SVKuxCf?oB%%{i@DXW$Tsr^+$IyZT9X!sn9ACTvobrI>-g`M3yROT+a9^;;s&+_?YCAC0JL5lm7lxZ$R{HP6eR!v001noo5KJA From cd8c66c763b3aaef3ff67d14ab3a718dd3af0d5c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Mar 2025 17:30:12 +0000 Subject: [PATCH 332/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 10225 -> 10226 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 3649039ff..b65206d27 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-26 17:22:48,320 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-26 17:22:48,320 submission_checker1.py:3260 INFO] --- -[2025-03-26 17:22:48,320 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-26 17:30:07,328 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-26 17:30:07,328 submission_checker1.py:3260 INFO] --- +[2025-03-26 17:30:07,328 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-26 17:22:48,320 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-26 17:30:07,328 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-26 17:22:48,320 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-26 17:30:07,328 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-26 17:22:48,321 submission_checker1.py:3276 INFO] --- -[2025-03-26 17:22:48,321 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-26 17:22:48,321 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-26 17:22:48,321 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-26 17:22:48,321 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-26 17:22:48,321 submission_checker1.py:3297 INFO] --- -[2025-03-26 17:22:48,321 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-26 17:22:49,270 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-26 17:30:07,328 submission_checker1.py:3276 INFO] --- +[2025-03-26 17:30:07,328 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-26 17:30:07,328 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-26 17:30:07,328 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-26 17:30:07,328 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-26 17:30:07,328 submission_checker1.py:3297 INFO] --- +[2025-03-26 17:30:07,328 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-26 17:30:08,583 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72364 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.76449 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421261 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,13 +33,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.34208 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0601 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2611 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.32275 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1736 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436732 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0358 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436732 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3857e0ee9..6edead7ce 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -959 +960 diff --git a/summary.csv b/summary.csv index 02fe7d2e6..29989dc3c 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72364,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.76449,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421261,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,13 +11,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.34208,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0601,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2611,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.32275,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1736,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436732,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0358,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436732,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3b6f0f64f..878c0ffbd 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72364, + "Result":1.76449, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0601, - "Accuracy":"acc: 75.600", + "Result":10.2611, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.32275, + "Result":9.13252, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1736, + "Result":23.0358, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -544,7 +544,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index ac0cde6bb329727e0e74a399f9c2bde00e1bcadb..ac824abdfd96913ccb115e0f1e1d143a463bf913 100644 GIT binary patch delta 4965 zcmZXYcTf|~w#P%0fIvv-y#)eX0!S~?BuMC>5T%K9qJUIET7)1)sR>mPAwZBOB@j@g z7o~R*P*93=5Tq&X;&1M|Z{EE9XLn|IK6Ae3oZXr4?wIYGRSi(lTk)ms`%wb`ArO2x zkPH8liWW2DJ+H6Bw0oQ)d{?39aFoZ@G%(j3$r~pYFHaupn@fJFO-l~S zzIR9RB?mI6?f`ygkbLamP%|DZoB&f0$TA&MV#LKN818npo@i=#9TD5~QuK8U{pDmG z=#*QS&R4Z=9$;nYz%dFbzarnV5>~9)L7C9Y!5BO{G}Fwfw?_M;Ex9VV4asa(C{<@S zb5JgY^dpgnodv3O7F6hZt}yl`EE1I3k=#HoO$RX=<=9j4)FDgFOCme1CqMSCCpV)R zkJHyNEW7>6Zy3S-0!YJaI>Od~G4t#JiPL=kJ0FE?E#QfvHhO^{65={_vY*n0Yr=Y* zFcO8SNrY&HrHRT^(VE&fXqUFWR!;+v0W_nxs@j+kx0pUtwHW1=yf5PJnnzTdeY*a6 zkmtB+wbpBB^Y!Gf=Qy?xrA;U>Kdmt50av)v?WIVL$Pf$Dp*q*}Yl^5(CQD%m>3xoUoW*N|Kt|cb}7#EOWC4Kw6HSR-nk7-Ol^>?T4zDF586Rba{ZWg0$ zqrf-n1Txm-%GY;1y4<7tDpl**`>dHu$poRsxM@ofsov{5lH!*$NwE4=#m(-ub*;;2 ze%R$-&$z%~sDv`;4=N$-%a)c1Y-$M1WwU>1YKK>oE#Vyma7jVXjPTfXkEge-KJS; zT0aBn9dfCLYX^Y7!bhe|XAYU>evkidSPQcc%+paB6@pg{Couyp`6wexbp_1>fz)By zLnnNtx+{V+&-nI?1WGYXObLOsUY{j&1``Jw$0&3;Q0oaMLGwQ-8b-qT*x&VL|m*ik!j=40j)T zDxA=m8&OfS0szbaAOHZ+0(|&3Ik-*_0D$pfKpjl938Ub3NYbzEWNOZ5O&Km-F!|m7 zM?Kp2H@`ML(F`(bx*pmxYXa-`V(wb-AeB* z+}k1+LYb8#G>(R5>t+9lG}A?OB3b(9PrFTU{Xw=B!9Fa;$G-A8w4}X{;t9l4=qa~- z{XFIo>t1YM-fCfy$r#$?c){8^-BGy|C&HW9js0e#a@!bdU1JEL!AIbjw3@kw7P{MJ zeRBBw)~*ol*3PL|HtM&ai$~Nx+a83gGBtO-9lDO{os>NhIo|bcn3I)9U#{KG=unAI zaFkt>5$i!8ZR*N4HyezOxw#8reIPRDiWuU;p50o^mCdNp zVDB6m*nJ_U7+by@>(piDOXbe#B}yX^H?4^Un;VBmnslG8R9IE^fty!hr`RU(6E{R3 z|NJAZW7qLFpH|UaE)@AnGqedmHgGr{6kH?Q8!HO2lLGTTNp87B_Rcvg-gXWR>4Vn;Tup}gE==6Hk07qg1g51)+h7Ac`p#Js);$%a>uj79;1#t6 zDuL3)gL-S7bqCy{ZyH9E%_+yh@r6l9Xy8Tah7Y~z0-<&%J7>2cvu!>O(E`gIUDj(( z4Wbc4W^WL<&NykU0z~#T|M^dGi~9psn3S(W8-?{UWtQrb7>Sf$+XkGB-fz4E$I#E! zu}?-(G_335MU+oDyqUVllY1I-KcFEw2#^6Q$MF5M21=4h8Xz7aYygEA2&W<;O%RAd z1GF0Y9{XLJaO;|I3{`wCG_^C1ZeQLoO+o=S+Y;NUW>}&n!46jBct|O&-ZicktJOD-n3MmV!UklfDtD$s&J{&cl=pjo|evy%y1u^U!t)m7v zMhs>s7L_e`tIdB=PT9om?mbdTuC^PWA}U*ElIrHV`=H5CApI6U+nk@8Wy3RPb)4~d z!y71zB2sJwkd2^nAzbPicXh#0Yr7dl4gUB>m0fc|7jo&X;-|wFmq^up5mfDA`?*a+ zO5!fNdeu_JvXSbQC!(jiWF;H9!d2`o9MZUGX-X2L<^jM{(q_?n?P}E)#3pB??{00u{BoW;kAgI z9<|=G&O&%rtyGH#SxR^Y!!K1CZ3Ljl)E`!DZc*KICSfAZhDGFKZ z9}5Pu}np`hCmO{?m=SqLRN%`M-KUZg$tD^ZyL_aT+)hDSwTG8<6@haov_=(BogrDzlz!?ba{?fKDXc#w|mcO{pHzMc7sfjTJ12qbTz7x$CSdd55Tx>v8XeRST)3=JrpR|@+ zlfEzOPlb(c{f+Xu;;*Dv8#`t*(y~2k4=J8|@c9;g|Ev5+h#NtQ`W}l0n9NUJz!ALvy-66%U;mOF8tG@KiRY`ll3UFtr)c4xpfRVyPQ>a zN7<~EQ5VS`iyJN3$;_Bf)0h){J5${R2s=enu^!C5^lkz~3h|!cyUiFW`Qs{6K(8Dr z`4S2A&qG)XP5*X~@Az1hbliG_?AAEsd|Rv_0l8nOUsE@ZLesoB{8Jb-*j)S!Rb@(% zx@e+4x~8kpT>FQa5n|o8)}f#cAuLB|NajoEY!a+Ta+iklmp+2Y#zd%UiVQp!{nELj=A!l zr(%{1-Cy9#nTX#mnAH5t@O7k$(1e2#zZL7v&eA#9xz1-MiKh^N`AOIy*zGjmi3w z-p=^puGCBhp&HXEfiy8oFSTc|-wcD-|H^Cclw&nGb!>U?nx01A5Sz2v!nB=WX}?Ae zn0O$1d@4atL5~{7wfSKK%_0(6E9TFuH+xD~<4P8W(htSj4_#-~xqLrRotHq=>5?UP ze%=}j?TI)Z_qeiS@~5zpJtq8Rm@IBLXv!Tm8#|@R`~fpmxi`6zX1o^e*)1UqLs|s6Y~+HZb2${wwr#C_1f8HZKb4=(Y|u3(WBIu_sXDPT7=fo zeYM=&{L3mF%5)wPp~+HXnbcdNndPT%L-XtpRjN1?Cl03}QlXE$k6xpXa2nT49*Rpf zds^7LhhUV{buubvn)luw%WZ*Qex99$uC634G=whjk9ZBS{`%In81hKqA#1W5xS2UKJiA@WH#~4O*Uz@V3z*@w@td9X zmvz93xOc)$sm{?l%xAf?0h?ADA^CDa**40K(7X3w!fL`auTl;^1d3?;Fz|^Fw=G=l z+nM5NmB5f3Ev0h|W7^tH)(vvLsLQ~d=MJqMl0Ic=eRx%tS?@bxxprEjl1GH(_|Zyn zPFfz-#Ko>>fy#pC6RCGp5U1TUzNxwbanMV7u??`Lg0PS~qB0gYQ|2{AU2Xb5{icG` ziT@_jILiMSk0afI z_(&i(-dRkNA`YJ6!m)J%BZv7jpI7j0E delta 5003 zcmZ8lcQjnxyB`$BcTXP%B+701OQMV9GaXN zw@pHZHg?OCq7Rxe57A~vI)a4Y0cp$| z8t(()tpPLjvjg&e#|PX4(Y$Sb>QMr0qj-ETEkV4?>GKa-HZo2oYUV%*!;O%4At;^x)*9QS$Kt4u5&F(QP!_e! zA~tL#DO&UWgDBmIaSWMSY70w6ex2kyw9w}8nCt|B%}xe?%W9Ce$)&?Y)z1x6&iBYK zku_#3n9m!=4m&aK2)JV<&@04d`+U9;qRu)S#)pndS;IV$?PUWm)@7M#Z1={|kQW^R zLxOGMXvtHWc*%i?56^(acH_*%JGC6kK_ZR>3WX2YnIE8Jpn0^F;*|7@(&!#q_e^Gp^(<-#xn*VS^3MOO&f`(PxVtD z2|%HtgD8AKI20@=?8QP8rz|-t#m<5IAe~rPMadV~l>g^Vvri<4{`N zI`-sM(XzD_n?C#SZ_^pZr8czTCFKr1Lih(`6yHkpn<;wRY<3oEn`)<#TYsbC8QmL1 zTT*CgmD1ezEF3z!mwYanrF(*6y9ZZWsOQqe-KF~?es1muo=un3aibgLg&LZLWJDX9-7D0vqrn+a=7v|`)^=EVf0zF9S_uhdTLRTe@N1t?#hg_day?e!kdh}53w!xJ z-y*8qFlS+(LE*q68T7xNSr!7NC1uD2p>-Jj3C5G`(gr6Z2E0=bwWIA^=aD?4#Ze3S z_QT;b5~BVKUt+I(VmSiQZq=_qjpgw2?r%^I|Jf|biCax!!V&%!(YI4iNcpQun?}2Cs2R>%GM9CG zw;DKpiHYtI)oHvx&i^CBZFD>|`5?*p$2&dpG`CF<2o_%a=){}l8NY;N3Dm7iWj>cOhH3|TL1s6oFgzk7xO2c)t>*Pq4INI_fx6w{dw5;b6N0{-8 ztKRv%aPIdm3$s?IRaU<47 zaw_z(gxF)ZjX!xth7h5hlG60+TPB#?XUmT~*A9 zOJTzzq@KRFmy0T#A0Rm0eYoGj<~G$oU^)m%CUlW43G0c3g+&RD5YQt2arEAtPUm9Y z+fr4Aapkv}>i5lxZ|orT-cp$BofggQ>U1*jFT6u0S{Lf38=#!$efdAkj>9;ckLWmQ zdDkfu$Sr0-ku?IZRH31TM`#$Q->D*Wj^b1eIyYK}fd!xo`G)4!Ya?M9(^@dje%Ci@ z9g`Ob=-hf1ZKPDAIi_RwPDZ=16d7n@H}pxQfUs1em~xXF1~z(@tOje^(w>X*y!>nS zIEW(%B8PdgV6Ei0_IKg}=*`bEWfOo8(-JKRd*Jc)BpPH(0qbW&lgdk-Zl6sO_71!H zSH3M;@jZS1q}^Q<8Bl)I=4dSgzsxKU$lM>1#gIzQPeL%m2gsK=pt}d1aP!kH_uBnP zH`k4ZNi{sgP^c0-i7eSE4y=rg3zaf$>xel(+PUFQ3M8|)4wj;tUP?4ZF;+oM5gXm`)Gt~_Pti)ltXgwZrD0!#5##11eD_SNNmmAwFTXL5cRHCk)pQgYBOO)US52N~ zMvX?k*brAYaetqk11#bK!C4d3RxvP>+^uQiRkcsYix>YuKEIxACRdoIx?^7)Gn;%Y zuTT$ygSZ;buhsUY8gpRVhcxTnO%JTUuM;lS^KYj|cb^SWx08R_Gy)ghF&Co zBF3ZX5&>^voH|!{oazBXL;fuQWsr~?ilWW98L+4)Q4J^axW{iT!<2`pXXFOIibE|u zk0MkPZp}BreN$)qJ%bJ#s^9LaL;N<~IR1>K%P0@doOc=N@w{=L6mp=C|I7n^uI=$y ztd(m-4U(z?f{QhG?*-*qA}WEPUOccI-S>qud2pB~lkZu77|jn3%iZF6`(&j2T^R!Z z?^mrSA_8_6hhY|3X7M_AfBr0Kc=t3~A=;jsA^G>7hg=0ij;n;K>$K>)RZn&!yFfs{ zD*>c2`YE;pEBld5Fp{(a3T_37i#&4`qu0O!Dg@OAbv^87>5)_Ubhf>!Xqq3nXiJY? z8_tx6b^Ly@Ca#((%qdf4rM&S*$}zTf{Rge1rz@V*o|zeEYMqkVu?(Bb6R}V_@ecw$ zFiI?3Op#{@kXoc9>P?#pC5ktS]s8R#WFAZmX>Nnt` zI`gOh2NGU;9D*ptAh3Y$D;K8j20)(GR>OJ2(Fo=%)1~aOqf++BOEQap@kXXnAKu^a z^ZAvLI{!q+L9ika&f_He412{3=Map*6N&1C#i7?ZMd8GdDu~hx&+QPzO=4WOBBo@sgo8eS_lNl`#&mR+=}4G!-pqIc^EI zz%p6oKS3}^`xx>LS3XP3dEZ)1a^Gb*QZyZ;F0wf&l7q$Mk8L9K9e=@#O#(D|+dIJ; zAvPY5O#9zW!ivuuf`}6bM!P&e)W*n`Y_1kqFN)M#FFasRfw5q?GG$@&Z~^Ke2#))^ zEP@09iV;W^+}*1Hw%R1*p;h7SefO%btf)ZW)L{iF$3rF zBw5UH=zKzPOR$+V8LZErtmOt2Y-Zi6d9n3R;#SJ5qr@Ppq(A7p?)hKdILNMSbAaZC{@LyM+~b`kYKXPSxT(|sTWiq9JV;P zh6MTwag;qlw=V?rz$!3f4E}*^BzuMHrUtq;sjtg^)t)(+9+j0dED5FYlWB^@N)2y5 z5zDGRqoKqbwQaVG1z+WUnz9djGGI`!^30*)Oi?4<>+vA4k@{v>lAs2qfRX7xgyk)c zz0|2dq5axvUei7a)|vivac`uG;b`)1 zfFj+wd_LXJ#_$R%N)n0GSu(r6x0DJBy-PSBVS#&!CpWl7D+lND0*H247N*{RSlaZyFT}!JWN4qO#Nx!w@9k&I)l*J zEpNWtI~~^#5+bq}%T^>jv%WhCTnUbo8s51Q>~;RH;GO>nj{R4#$GOAH^J~Gx&BU11 z3O-0x^sMT0=i^t=-!I1uwc&y9bLj2ZZSN`A(>)_dYvbKJ66~(Hy`Q||)(f(`;>OqG zw_yHwjT2o(y#~w?6g_hg`PV@_LPPhaD?6CfP_&U+&{m-KOg6$+v>T(Rg@PKxUEUj1 z5X`do`zMdQ`me-D;R(Fjkxg%)Y|QDk(Sn}g7+Nms?0B(l^m@fBt^lG9KeU>LS!{5F_$K0KwIEP}Qr#bkt@F`7Oa|J!vBbzb=h;+-pCdQ}t3_h*B|+jf<)a>Hr*-t7%3I%8 zJH31!;!j~BvrE;bn*lZ%8n3{Av@HH8{^_x>!JkfwCd1r20{d34&G3U7^r*cr#ImTCAjL&8!lxR#Re>g3-X;Dz-Q2auH6g70M*zEb3fY(1)sFgDkXMNy(UcdVb zcHI2kpO*RqmiB zcTL>?j%HFM*H`B7;Vm+>X4l9=FQVl+9-==0jYYGxd2 zD?C8SE>3A)san)S^}6%j{gO2Vdccmid)J%rPTtr2s~nz=KuCwOPP^arb^FQHu#sxT z_Bq8wSeSR5*gZdPVQA5-)+E|>>EQMsOwY((K%Lv+Mf0+>)z*pExigQI-T`I2w zxYa|~<4R8dnvXhUd9K5(>*q04&w0mXMLp~DnEzoDbM($E`oBw{st#r(IT~nT%*aCo z0B92f0JK*nk&TEh3uf7j1WV|;c6WPY&$&ywleA;delpG^PJvT3pZfs#nLP1q%1w&;NSnp?5iie zv=&2QjDdD8)m`MMFZaW6wBs!|1C|Q2;3#4>={Cgc!(S|+|5*FSk8*#>4gEA`pv$h_ z*d9^o)0(@=_1Lgj&rhtBQzo=3ixIGjH^*b5EZ=Q1tF$wqz_CuZ0Wkc+*miYcD^a z?Jy=fFTQo!L2yj4X5Z!mLb^WLFIM>l842rT7M?uGV04!^wvU4j^jw^mvcDQZ6ZMY7 zqv-~Mtv3l;+LuICHeBJvH~zin!fkLXvHVYEsRsZsT^akI793)WgYzg8Md0FiIPYGc z##dMOVtfn$z;#*rpB5ZK!+;ZFV8q$*(BQr?)7<*sg+?3|uME+B9GsVj<^Mt+mxkwqfujl^@x(;3N diff --git a/summary_results.json b/summary_results.json index 35cac1f7b..8012e6f96 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0601, + "Performance_Result": 10.2611, "Performance_Units": "Samples/s" }, { @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.32275, + "Performance_Result": 9.13252, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72364, + "Performance_Result": 1.76449, "Performance_Units": "Samples/s" }, { @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1736, + "Performance_Result": 23.0358, "Performance_Units": "Samples/s" }, { @@ -725,7 +725,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", From f17e02b30c18d410f2a928606a5d413584be67fa Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Mar 2025 17:31:27 +0000 Subject: [PATCH 333/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10226 -> 10226 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index b65206d27..4097c9dac 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-26 17:30:07,328 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-26 17:30:07,328 submission_checker1.py:3260 INFO] --- -[2025-03-26 17:30:07,328 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-26 17:31:22,388 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-26 17:31:22,388 submission_checker1.py:3260 INFO] --- +[2025-03-26 17:31:22,388 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-26 17:30:07,328 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-26 17:31:22,388 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-26 17:30:07,328 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-26 17:31:22,388 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-26 17:30:07,328 submission_checker1.py:3276 INFO] --- -[2025-03-26 17:30:07,328 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-26 17:30:07,328 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-26 17:30:07,328 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-26 17:30:07,328 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-26 17:30:07,328 submission_checker1.py:3297 INFO] --- -[2025-03-26 17:30:07,328 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-26 17:30:08,583 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-26 17:31:22,388 submission_checker1.py:3276 INFO] --- +[2025-03-26 17:31:22,388 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-26 17:31:22,388 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-26 17:31:22,388 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-26 17:31:22,388 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-26 17:31:22,389 submission_checker1.py:3297 INFO] --- +[2025-03-26 17:31:22,389 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-26 17:31:23,433 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 6edead7ce..612eb51c3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -960 +961 diff --git a/summary.xlsx b/summary.xlsx index ac824abdfd96913ccb115e0f1e1d143a463bf913..dd080b8415d928225c1c308e3fd55e0e1e9133d5 100644 GIT binary patch delta 350 zcmV-k0ipi#Px4Q&9wL7rC%=6q0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!FH z!!Qs<_x%c^y)8dzT7>ODTe51P(3FT-1NoVhcoTpnc; z{epEus|saFiXu=P@xVSTGn?ZGlYL>3vX5%uwHJwSJJ)I)3{qPU0kxQ;(~e{dRUywb-dH z`32{t(KX65f)dc&2&vl|y|0()93?(5Em&tX)Mx;Y7R_D7E#-g4!K*VCT#(?C2-2Q% zTcd3VmSXJr78LVYbm%B@sz1tBx6ATYrK!+T(ACn7o`1)q@i$8{q> za10$N(81$8%kkM8z$t&TVN@=hN`}G0P3uXV+&eEPW8e2#Uyjz8KgXY|m$z}%ROzEI#c;bx&@i;{N#@lg`Lm!AF2B3$yC*`AjdR{M4Q;_05 wC1qO7*7*Z12_^Hd{m85SGf*9e{F`_J9CBZOg60piWFrd%1UWg@U6XSsf0KKnl>h($ From 97cf940ca9f412e34d6b5621b9a5442678b9726e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 26 Mar 2025 22:10:21 +0000 Subject: [PATCH 334/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10226 -> 10219 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 4097c9dac..442c1d5d7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-26 17:31:22,388 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-26 17:31:22,388 submission_checker1.py:3260 INFO] --- -[2025-03-26 17:31:22,388 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-26 22:10:16,463 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-26 22:10:16,463 submission_checker1.py:3260 INFO] --- +[2025-03-26 22:10:16,464 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-26 17:31:22,388 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-26 22:10:16,464 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-26 17:31:22,388 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-26 22:10:16,464 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-26 17:31:22,388 submission_checker1.py:3276 INFO] --- -[2025-03-26 17:31:22,388 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-26 17:31:22,388 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-26 17:31:22,388 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-26 17:31:22,388 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-26 17:31:22,389 submission_checker1.py:3297 INFO] --- -[2025-03-26 17:31:22,389 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-26 17:31:23,433 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-26 22:10:16,464 submission_checker1.py:3276 INFO] --- +[2025-03-26 22:10:16,464 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-26 22:10:16,464 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-26 22:10:16,464 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-26 22:10:16,464 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-26 22:10:16,464 submission_checker1.py:3297 INFO] --- +[2025-03-26 22:10:16,464 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-26 22:10:17,468 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,12 +25,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.76449 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421261 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420191 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.34208 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.44552 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2611 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,8 +38,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0358 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436732 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0358 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438504 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 612eb51c3..754d8e034 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -961 +962 diff --git a/summary.csv b/summary.csv index 29989dc3c..21502aa7c 100644 --- a/summary.csv +++ b/summary.csv @@ -3,12 +3,12 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.76449,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421261,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420191,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.34208,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.44552,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2611,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,8 +16,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0358,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436732,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0358,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438504,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 878c0ffbd..14ee192e9 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421261, + "Result":0.420191, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.34208, + "Result":5.44552, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -514,7 +514,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -544,12 +544,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436732, + "Result":0.438504, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index dd080b8415d928225c1c308e3fd55e0e1e9133d5..db6ee45c358f79c2f64d9a3fdb0643667a086d97 100644 GIT binary patch delta 3230 zcmZ9PXE+;N*vDh<6;y2{5ml=Mp^8>Yf|_k@wZ*Dcs#a`LwUS81s5C}B4{F4C)T*Lt z?<%Tlw$@`*ZThCq^}b);^WmHi_qqPp{rlhl^Wp57ZkSewFf))#7*|(t;miVDmUfaS!PypG>Ya-?W;-)~O5cOOMLICfb=W z7aOB!&bB+d;=M3t3wa|YBTn?Mz}%t3yuo4}&eGOmp22vVP`At7(+qMD&@_d`$6zd=g(YU;pg4ZoRV@-pwe2&^bRf%X1ZS#e=A3{e`9pf zDXdJq{2VuLY~U{@DtFq(8Smfw$x~i&#fR@a3w;cOO{Ce+&~G)bD4~a3{J7a8Qjy_e zQK#GqMS-~mYB#vn3~+A8RcaiRV6XL=x+hP{8Xs@j!4N%9UHn8i8E-|CLn_-5W2lqa z^-Nocb)u}NXgPa5=0K=VS!8?&ow1E`5*@ey^;Tpd&vE94W=fI2ZOi5r;^E#y{jEHY zP}`HI8!uWk#O;FT0?^U(B=?YI&+UV0tT+%fJ;WVTjFyTH&5V_L;pe~c|049K#QBv(!jvb z;S_lOoFaU2hizy z>$~$t(~M1Ij{xFppR;AbYGe@M}ka#Hjog zp3Ydf6&B&rm2e^m?&Ho?Ih@yXL5r5$lh~b^Bl6i1rW;rZWHEzLJw66_mHF5u?gA3M z8C6U&3&Cf;h)Mq%0hYlAJRP?Le%+;)d0pE95&c+F_I%Cdx$?3t9<_XPooJ=_Xxvf9 z@IboNJ?X~O@$|-+W?y;Cx2DEK(v+Eeu#dv*6*EDR4)dr;7l0;_=hj&Bd3H*g^$VX8mG6cUnKN;rWM+FA0vMFmdAXLUGB8c zS%NXgJq^9}0T~cnF7!xE^g;<=f6qF$um9J(2i7wj{k3d0%cip#{MrtUei*R#OnMhq ztAt4o!&nUxdIwZh(s_i2@)=pA9c$#CetWm7(c=} zuwH~LBoIIZvuNhUbGh|VS#E#CJ39awO)YJ_n{EX)+(*A_7Z}j=uGotSxmi6pSUgu- z5C^Gvww*3^?aSl3S*&sG^)*9f^LRHPMt!WZGa!6pbU@Qr}%9q)!P< zRIV`U1l*N;uR-~haoh5ym#UpXTK4L|r-S@8HIZ3a+rhoLGuOnt<%Vm$@Vm1(H!@={ zNJs*QM#UJ{)^NGmy1QZNbv=-AL#yBDh_&620&Zn@s^<&&NQuXFJjDtY#*v5)6a zopDcn>goXPA?Egt*e#^8`H~RUctU~*r~A_+Mkw;k^~L|kri~Eio^&?{NdKgx$vL1f zT5cws!EhmXD>Mdar{rbhIO=QTG-CeaJfK)F8D;M^dPT^sHV&EW6@zq%&uUY;zdG(* zfG9GtiA$#~=d3#_=5?OjQe_|}2 za{Od5)3!VuYMN=-F4Lc_dG@@~yQj@nO9`{eWpMH&1^pqLQH*>``kHlE4r2}|x52G; zpt}&Ae%A`MMoPr%I}a zVPlLTG!8b_uLpet{mIQ?Vbd|x3DpIU0noj zsaALeDBOLV1{+(a_E&ep%@NQG#Tu1n*`t-ymdE5MC)M=W{#trb-U{tM!~x%d@83d= zhh7-dr&?1ys)k*iF7U1Ww$CbRGgTY6v`eQ0Dl7R{6>|Ru+=ZsY#{T`Iddkf~w(0n{ z6B;8u3dnU26(;~jb!t(hJ<$jE`t$~Qw(G?*pq39IG z(F&&{XCYl2iYnI0WrPNx2!heZN@u#MrkYn!aSO=_3l+U*+EmHS!{dmLk)=t6Y{9cT zxZ&J4e%1^qTLu*QX;)UdICU^f$tdnY)^zbw8YqTwL@xNOnm)#ZxYR@6!}TYjJN&x$ zUfiJ}-e=eG7)&bc`FQp)jiG+Rip9|Rz?`u6j8w-)f}@vGqz0+DND37f&p$xLHN)77ZTF!>-n)o%x7g#Rj$Y6A!!? znzw%S{+oYuO3t=JlL3xZ-$ImgjfVXhIl_&hoNFAA<))?UMx0Czp@kD{qKvCgbW_o$ zn&wIJ82WN94mwHVBvVSH$%8@{$yNx@v3;6$DaFXGnL&}s>rv}&fKXS_H&Bi-uu|Op zwGV%jmqRP#p8eLts;@am62m8{9`Q=(Zax3n@OD?BA${Au@9TqECKsy+Dbr8lFBPR@ z3DJd2$>9O&!_lu=tQ~~Ea9C|WpIAi)UhZ~2iJFYK80+gUegaKnRD2O{JMdmdprc>l zTwv%Twe9D+z+hyFsJG49;aJ66cg15SwThznr~Z7A>)k~G1QgCGhd;MK*df)*|CN2T zNzw71<$?N^559DC|6&&QgCh?1n=p5Yy-23@H?|NvIIER$!DePeN6!ZU0GKIRylI>a z?-?`FKsYH$Ike3CP?Efjsu{}A=*jI|82J+&vJ%%`wP*$0`MZC#HF$N16cmN`+h_a4 zQYniEUwx)~eK65mcE4@5i%TZxrL2ivCFwRK0Oc^BJ-}Tm-(u_cE}TYzRbqu%g}Mcj9)y5`+OZu{_8bXqaD1ZDSB7%avm{7ALOYrcJ{e}DIn<; z154kHe6V0!*iGxTOTytReOD881g+~#7B{suuM0Q7F?vgGOga<1_|n$$tNg?50@H55 zB-k^-hU7T=wrlDR%-*PAK8bgmVW-D(MBwX(J4xE#%$aqQ{=w~;Yu(&dD;pX=(>6(k zJ>1&Mce{G2_rb&OS?cEkB$4E^JQz*Vm&QWU1s%mrs*m=6KMg9Cfpcfxym{yEnOXC#Z|^xXXTf64qS*f^=lgF%T?>?;qcFmk z4=10Dbr)G)xsQC?mhR=}Kb+=6{R`Ui-SbYGJx7>1pUh=Y?2nZ=4$-LEMCVqR`da6G zrh)s?*V-p~0j9N=AJmVU!`eIrTPN_0F-vp0BM043Q=+?YOI&1T^FeBQOnt(VdA!L62x>MZ~ZH=RN zs#M4P+00vIW13dg#`U<|9_`O|n-8`4>sl+jFJsz=RCnce*1RjmRMm0E%2o)8&6=cW zN7Z>1C=<86WTaYGhwAOax}GNDK`OsBXcO!9^WkTg#+J^1;hPuSS!DaMRZ89 z?(6Tk^cN1!;Z8Xh+$^2;@E{cUH_tsJ#t+87tQ{e4{=IO{v<5?EKJy*MpB7j4$$C9(WRW%tCw=(6uP9X?b9#rzjo1LD7?evL=evH(^z65=3rKU zzAVVQX;hE|*i?$YF+vVN$OZ;kfJ1Wy5VD2oaR^!T!w6){NQ@2Nyl$lM1lHOHM&b7 zi<0&B%Hb?O4L@D9*U6iyMEsW&bT1)y&IGmx7s%`x2t3D==~sKw9%4wfZ;dMI^QJ02 zWaZga;}~T5qlr=Z7)Iiqhx1Ye6In9o7pmUwwTZ*S&V)L#&KVuz&)YES$K}jcMU==j z9ru9&8EQ=1P|QuSW|_G-nD@lM-|h(7f{K4ag0d~q&#mM4@#qB)4dr*Xx8X_!Wm%g3 z%DG*a=3Cxvia`o1dgCnF`h`f@u}HB0L3N+Dytw_r_PgKrZ-7P{@i)VyXPR4W=5JR( zLb@#q2?%maq_Tb%BKe{(YcO(h~fVnE1jV;&wi|+MESf5z5M%dDLTX_ zy%^5p3-UQ^Per`=TNy*^x?|JeASyxtbzJ0=dqM>pT`mDYLdc@PAe3x85>yKUp(?;7 z;2Pq%LFV;~vf=EcG;n-NB=?58Nx~U8Y?M$R(V}hg#(M@Hh#Pi3Ec#C2VJAZG~_@1>29VCxitA1KUeHX36}MSFPICRWB#|D4D?5 zq&8ZLM9Ogl!8RSrzFPCkncYQyoaB_X!D@f^>Dlwvb~DqZN^H-WLiUVfM>xfbNQAZ% z9uh))mLK3CBU-!!C`M(iX3R~Lr1%(7qG;&qi)WWB^FzVf6X?=m8_1;K!R_Rn#JDiD z59b8|u=I2j?|VnV&c0)6PlO&rrwX(wtr%p&L!^Rb~2iB z7Z^$P&>WV>O zCeYAiwd;+(lWOP9vA6Ef-=rB}B{eX~Dnsno7F@f`yZw2VrB00bXj@f8Iq8ti`YQ^- z!Wu|u4w02ARKSJoQrh>9wEHWbQLU zan5Ypfc!xc{xB&Xi|ommQ^{c9GaGU`U2EHb<}pV`A5B)vH*Ugrf!}y{mF){O$%>0DDwR0dHNc$gY>EECK&-!t#K;(RhJb?1^F~DrchIr=Q#Ski! zpxvI=lC6|frdaRBP*k0jUk}mj%`k$emCXwBso6AUH^UpiC5_0NNeVyaiqY9uaup4Y zA6s+TR%HvjFdzxHhB6!tAVMhOSJKMe-E546-*uar9Flryz#3Oz?z zNELwtV5bor>B6aV0DO*Mw@TCsy1|xS+S!+tJvA+Q7(E-5uLI_}UerRUA?6LGeAOZ7 zCUp1r`O+BXx`}I&^x3ptm2}XaY;4(vU<~Y`E`n%g$U@%hDmM4kHrO2sK9W2=KYUNBI4gJdZdHJvGtxnq1iG0U*%*y^*G4Jew z2)7HkjniV^$T;u|GlE>Dvluf+nf|MogWOTdA<2i4b~_k7eg@PO1UKt?Va5@4!+}0D zb5@nmKI)=$31jnNzUHg(0?l;_lQiP?mJQyc$wF2iN-rD1Be9&-2hu4BcDKaghd0;^ z=$*@1jmzwZ9U2NR>VbJNrHUH)&i)_0Ue!l*y51pdY4zQ&wx~++)TwiTyU!osv{}gd zL)^+AT>T-grBLp7ZhTC5CQ2I!@7UE#B?`EOjz#!|K6oK`u=EX3_1%Vi8~XT+@o;eO z@?pvpjUTbE7Qb7JD62B(?*UWXB)zl856!nviy`Qt;}JN3Ico?ACv@=Y=L;XiT+M1+ z{Wm7BKA6|le0uA#4INwGJ!UkmuCYs%Spg6M|9 z@2c{pqu|BqO1`_r9FMTc6UM}eKRdd5!oVu-5-{1CSHB9|-0kGNU(^J-&?iqU{}HG13;y1<`@unheC z+h&I2BM5z`;VJ`L-R*P(niY-Q$Tlv0*N?$*rf&Vp4(O`OjlvY8f$@{(I=%Bo@Vc^J zf_xyG#`$Kr0r-@Fpbd7z@g*SM1tU%Pciss&Q`yFOqvl6Q>;_Qw6=fhd3Hcj^(7T=$ zcaK5x8(H;vhFOk@=##19j}2vl&mD0Ls=~yl4i31>SPU#r&Bu@g*2?Pf=OkQ<0%t8S zD*pVpbaQJVCoe+N-lof z8EVW8@zYIfZ3BzuJfY3d91(ftmdg6D@!+3l_EI-B%*gnl-%HS00iyg);^@ifCe_N{ zOVQ>_pfQxMVo^Z3Z z^llXwHFu89bT#1` z!Qwl&Xeuf42-sD)clR@m3m={VfV^ciU&=#pLw#v+i7d;?IZm%DcW-c z>e*vww9iW@xH-0C?UaGez5QsOtlp$(S>)-!T9B0}%;+j=CIzrs3P(xrXg$t$)RH0miLB; Date: Thu, 27 Mar 2025 10:35:58 +0000 Subject: [PATCH 335/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10219 -> 10217 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 442c1d5d7..e6478f9b9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-26 22:10:16,463 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-26 22:10:16,463 submission_checker1.py:3260 INFO] --- -[2025-03-26 22:10:16,464 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 10:35:52,855 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 10:35:52,855 submission_checker1.py:3260 INFO] --- +[2025-03-27 10:35:52,855 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-26 22:10:16,464 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 10:35:52,855 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-26 22:10:16,464 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 10:35:52,855 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-26 22:10:16,464 submission_checker1.py:3276 INFO] --- -[2025-03-26 22:10:16,464 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-26 22:10:16,464 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-26 22:10:16,464 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-26 22:10:16,464 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-26 22:10:16,464 submission_checker1.py:3297 INFO] --- -[2025-03-26 22:10:16,464 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-26 22:10:17,468 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 10:35:52,855 submission_checker1.py:3276 INFO] --- +[2025-03-27 10:35:52,855 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 10:35:52,855 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 10:35:52,855 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 10:35:52,855 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 10:35:52,855 submission_checker1.py:3297 INFO] --- +[2025-03-27 10:35:52,855 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 10:35:53,874 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.44552 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2611 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.919 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 754d8e034..e12ce0f86 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -962 +963 diff --git a/summary.csv b/summary.csv index 21502aa7c..0847ffd60 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.44552,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2611,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.919,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 14ee192e9..9a9d485b1 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2611, - "Accuracy":"acc: 75.800", + "Result":9.919, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index db6ee45c358f79c2f64d9a3fdb0643667a086d97..517967110b92568d6334ec60ec1ae299a018c284 100644 GIT binary patch delta 2066 zcmV+t2<`XlPw7vv%>xQk0Swx21ONbd5|e)j8I$k>7k>~CFq+bP>5Vd{I!`Dh4n&j0 z8QbaQ@5fG{Q?_@RYF;Xx7f777_AOq{wg+FIv($Jbj3+c7noiR;4U&g6O7i8<{60Hr z_@=={kVipEb22nv2sgiu{uw+`_Jgko5r#C7^P#yCqUc)|4_71$c#{?+mrr8KvOvgN zwzPP`NPiGjq%5^u+xDz1NOH5b&}Sd9LE|_H$q^09jO3y=!blnj89rYn1vf{7D3Q-p zJ_aN4(ER4~#}B?~jRuwLe-rY=-&`9aSlpA8gn~q}0H&cbTTuE#Jv^5~wxU`Z_oRyI zmKhO=gEAFU`gBH;9+F6`lwo6)L=nj=w_(Wz z%^vDU&e}jtb-Xp#v7J|Tuj#aH&+Vw!^OBM^$Ev+n5gr917!4SGGFat-7lC?FPT#d< zl|nUor-oH*)_P#N88BoXs+f+EL>EN)N5pIwVk^ov9%WnOyC6y(;e#muh?wb!9v)=} zjq+PZuO)cphX@4JdyGIpvPTF6%-&-J0w&i90s*t@6oGi#kTV1V-u^iP0j0k{AiZtK zr4DgHbteb}RQC#jfa+c&5K!G41Olpii+?~sb?*?!JL2{53FqksKZn`fK#{1OgKbO&uvt1HO4MmS)uAP6wy zZhwazkIw$PJ)Ta^uP^tjHi64NVVQ_kxoCzov$9nFX0iv6R|R45Jjr9iWG6%Bf`4!^ z=j9?xxM~Xc{IS!tEx`z>+7C;08P?Tgyyi2)<=32B%hfyxRa?=JT|{lF5f){mcoCF^ ztNEkb^qO{K{q>)RG>?;|`NfQg2$GbW2LmfcK5loPk(2D_hzzazdnc}G9hXyokejWp zZ@*dfwnW#(Mc36u2d3en4r#$v9+LqF7=OA@bZ}szJJ3W2IoCP|WPO8J1?9c%{trd> z899AGbobpY(b-Ph_j>Pq>U>;vzOFjZAROwD7FcM<2n4DQfk4$E5U4r?0=~?r{tne4 zbdc@&4*G=;RR;$qx`Q_ClpyC?=YWOe2C)j;BL8z$S1!t2lnuF&RwkZ(?|lKb|9?h+ zpOM}N1UU2G2(Z`eIdEfRz1c};ge}RKq$xK-D$7;#UPtTGhN;t^)jRRuaJ7}sqeQK1 zL0X?1h$LSc{AYbY;q}K4uC0uP@w7~dAzli(p(oFUd_yWz(}t+O5#qym3hx@StJ5?BUO)|9~GjCL4SQ1XrfS{MAE4p_W~B7L+l zj-sl5HMOnOa+>-I0dQcsZzCz zp{t#*<#rxam{^7)sXy%Z4OOAwNNJF>{nz20k*Z7Gmah7(WWPT}Awa%KI?e2jYMD`IY_}-SJEPD2&IA>G6yGD2&Gy`HAJ;!N#MS2zTV?kt5j< z`EMs`mUWCv`wbIyOiLX-^|hhvHG3CLm5C$98HCdB2Y*e+(=&7d=i(*-&2R6KW%`o9 zz2uU$KX(tb>mCR8qNe&@TY_aT&kY|n4PUZHsL2k6#g{Fr>r8*oW+%wtp$<#e`OjBe)j4Csi-)AM|{Z}$DZ z=Q(2U`E>kjCSp|ei0yhdaZS7L$rH(L2o!}V4}s_6#9qhT_#{+p@oni!p1jq2_PrU5 z@!@|^O9KQH006VT8CnPhwdw@tC6gT@D}P8^gzZ3EvTC5vl$Pv-G;u_&2a;MRe_z>g zoCLh;a&+&UxihF-9%LK+f^|Zx3S~)(B2Y|op_&SPY*z6jikzp4(^e~}&=DM3)^{Z{ zglP+JRvWNhfQtf1Il@eZ_TC$Uv15CX)Fr{8g1FOGQXfj&U_;qAZ2;$KdXFV|%73Y+ zxCh2&BBBw=*@QGdwjBVRVQ4{u@(yQ7hA&IDgn=DHtghWKnGUkvIJppd5$%n|KDCa^HS}`VX^RBZLA8 zQvnRxZv+4Uc$4ZSR{>;`LMBB4f0KwNA_PLl=&h5kCKdq+lf@=M0ko3`CprNplR_sZ w8@1{L=OqCE08jz|01*HH00000000000Dyr9lY=Kd0b7&8CnE+8B>(^b03Jru4FCWD delta 2097 zcmV-12+sHEPwP*x%>xcq9hCKQ1ONbf5&!@g0Fyx`6_fA-7k`i$?b)Ha2 z9Ec`~Gq%&q-;bR@2ex;aYF;Xx7f777_AOq{w)9+4U&g6O7g|P z{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!`50@khc#{?+mrr8K zvOvgNwy=1?NPiGjq%5^u+xDz1NOH5b&|@F5LE|_H$uSMfjO3y=!blnj89rYo1viKN zD3Q-pJ_aN4!2IU)MvuN}4f~bre-rY|H?9p4%pXWfLO~)~0Mk&J%_;q%9$v^HTTv~I zds;` z+py$qFXJw$KI^K%w*v=dKu<5jI&+Vw!^O}+s$Ev+n5grF381@-`Hdy6>7lC?F zPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@OBZ+PY98!u4-*~H z1~KS}&VM#ysw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@tT@a;?@IjP+L`-$W zAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svtkaGkA-u?vw0j0l0 zAcxzKD;?s3>W&czsO~ia0oA=hAfURp2n1C34u64w>fR%ecf{-A6VB5OfZ8AFkQS)g z2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=(>kitWR@abogK)rH zK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!XgpNa^4JSW@V}T&14TCuL{EAS(3+u$xepM z1b^XT#>;t@aMcv>*;A)!TY?c%wI7!1GOX+IXvJrQ%da`N7Ry->sgzuXX&xsF^NSe~5hN)$5BgS&eBACoBPZF<5gA(b_fA}sIxeUF zAU9iG-`-gDwnW#(Mc36u2d3eX4r#$v9+LqF7=OD^bZ}s-JJ3W2IX5~7WPOWR1?9c# z{trd>899AGbPwGv(b-Ph_YU9r)cLsTd|h>*K{(POEwIpz5C~Kq0)eVSAW(G(1bms# z{2i)8=pfsR9rQ~dstyi}bq8(ODM8MS&H)R_En*e6MgHfiuAG;-C>wGktxUZ5-unV< z|9_1DKO?;l2yp6e2+*;cuD7x6sO4vE?4>ip7Gy-ylp7(H1*@5FqxE^i#OY1zz4&jq z+RW!sqE@ybtxpa_k}nMYvp%BmdZS0zR>s13Ql`WZuZ7&wlb1riBbBRZL@#lD8etSH zp)3)vwRd$oQl`N$qtSj9RGB9~OR~Qg?o@fbz0f=OVz>v!z>t}*p)$M5OLbg;gf;9J z!Gc@|Y?0*LNJ%UcwVQISV)gk(buDO7UCCyBPKBV^`ff>rh%j|0cegR6Vtu1Z@^)-A z`~^@;0|b*BCK{7NCK$7t2z?9$;313~vl|tp0)GldQ!ZKame)W4;`Z3Ql+RfD?<+*9Dvk#{(j;?&@ZzJ?Wj4NHBk z!4-P3LN8Y6#R|Pxp%*Lk5`|u(&`T70i9#=t^g_6Xb^~nMnU*HO4p|08@znS7EZdf% zv46xVNKH?bs$C3S?R+h_^Ps}SG89SuVZU#v3I#_>gPiTZ4)2UqUFx=U)o&&H{V56o z3RcjP!E%FjPkxOQt>XW&?a8X?h^gbxmG>sX_hsHY8F9x0|EUr?36CoA@K@7&$$tCP zXfg3k`lPQGhJBrUFp~0j*=JVXhuJ<5$A8bS^w;Q)U+PC;JZ?;nU+hO=Jg&%3EcXsJ z9@Rv+BR`KE$%e>(J5jT&V_e#An5bh~>gcJj4PCFYdW5up$j+{ zHwkEddyg#Bmjv!5m#qD{d!SwSIItHr)%V&GEPHuw_^4_4l08CAb|@^qY*AfjdVe-M zL6)!EzVx~KGVT$b*xueG>FAE*xO?Ls(QVHZ^WeqgdU-jy7KkH#B#$nqv*luRGp}br zM@*ic=evEg@Ao~=5qr<4<7YDwqpC-2*RzRh+I>%+NOnV@C`5S(JQpYSI_AbFp<;_~ zOIPybt=_Zm&0vfV{{v7<0|XQR0F&b;7qikCLkI=J29J#-lNur`e`qMecAza;HBe|u zOLsz=IHJ}ANv)H=ui{q{@T$wvy>sTypmKebZS)J)39TxWB`JzPG0lZ)D)heF#1ANP zo+?gTt)N0DaA;lMmCO*PExcN7zE&vs)vC0tr}!Um6xBmn>bPyzq|5dZ)H00000 b00000fPo5=fhRrzU6a5kBL)v800000In&P^ diff --git a/summary_results.json b/summary_results.json index 7d12784bf..d52b10211 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2611, + "Performance_Result": 9.919, "Performance_Units": "Samples/s" }, { From 981b53759031603531625b5df217564c717243f1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Mar 2025 10:51:59 +0000 Subject: [PATCH 336/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10217 -> 10222 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e6478f9b9..d4eebf0d6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 10:35:52,855 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 10:35:52,855 submission_checker1.py:3260 INFO] --- -[2025-03-27 10:35:52,855 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 10:51:54,859 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 10:51:54,859 submission_checker1.py:3260 INFO] --- +[2025-03-27 10:51:54,859 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 10:35:52,855 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 10:51:54,859 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 10:35:52,855 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 10:51:54,859 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 10:35:52,855 submission_checker1.py:3276 INFO] --- -[2025-03-27 10:35:52,855 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 10:35:52,855 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 10:35:52,855 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 10:35:52,855 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 10:35:52,855 submission_checker1.py:3297 INFO] --- -[2025-03-27 10:35:52,855 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 10:35:53,874 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 10:51:54,859 submission_checker1.py:3276 INFO] --- +[2025-03-27 10:51:54,859 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 10:51:54,859 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 10:51:54,859 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 10:51:54,859 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 10:51:54,859 submission_checker1.py:3297 INFO] --- +[2025-03-27 10:51:54,859 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 10:51:55,936 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.44552 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.26277 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.919 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,8 +38,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0358 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438504 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438504 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e12ce0f86..03b7719c6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -963 +964 diff --git a/summary.csv b/summary.csv index 0847ffd60..aaf065533 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.44552,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.26277,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.919,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,8 +16,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0358,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438504,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438504,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9a9d485b1..eb0d0919b 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":5.44552, + "Result":4.26277, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0358, + "Result":23.1786, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -544,7 +544,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 517967110b92568d6334ec60ec1ae299a018c284..01ddfe878e6eefa6f986f39f9c4207384b07ad0f 100644 GIT binary patch delta 3220 zcmZXXdpHwp+{ZV^$CHibyqsd2oT`VB$YB~bmLd<4M^+9)&ZjI)!c0-KIiKec4>^s5 zCz5jx#cCR9A&KM=3#GjD&wIVsdtdi|_jP^0_xE#s@9VmMi|1F)*N6N8zRxaKSo#wH zIP3rb$O3361e7$Ne>rq=Ksu58qV9rS6z1Q);()MlPGLyCCZcEE-?z|Jpr>3&Tveoh zXUmH3F1`-#ygqpMOsJbY%jITdV9ASV`?^o>C)yW#9)lP+XN;2%nQ1N0}$vKt}u6@qgKLc;7K~t3%?R)b6+?&g|+s=*F z5J6f%I@nY(bs_l6r(#pvd!9U5yORbr3e`Vp+@_0u@(Nl%IuwPuvk$qcHS{|Cw$-

O3AieUIz1|C+M*=%`zwAr@)&%Z|v9dmpozD#6z8cvRO+*_r&E3hu{ z+T>^Q0>+IElb;1zHZ315jARB{e(@_P%yeUahx?I6``DLG<#SKT`Y3QX?Z&=}$N*-K zUvHvlD;?z2I76&1p{9B39vcxZ$BTYkksY<`U0S4v8CRBqb!&X=ZWIt-j>KGttr zu1|Wn`KE8x!L_@Nf{tEQL#x=nIihm-X$KtQ=kzrVBXE9|f?nIah3S#nq+)ui0uNKR zf7#kGdO7|Q6odAnWOV*Cn3~M|k%Q^s7^^~pBRa(?-6sslSAxK>`guVz#K|q_m`@6k zD^980(xxU8xg|X^3%`CoN!Ug0ni0iG69fJ6YwVv5KSY)s^qU5^(ZWR@IN@s_?rzAz zpGzj^8;eEcXxesJO=R_jrT@yW&@9+&i4q3|^f6K5 z<)541BSdCDAl#x0(&nx&N^sv!hBK*XRK2T*USqSv+{GUr3QB^(Z3&bkpPPqC?+_d3 z0!_G|M`Ui!T#t~^_QJMhPhPGLAkJJrsmPA4;@vJYD(q=?5>5r?)OI@9*|!lu<0aaA zgK`tSzOKH_J(ZUy9Xe)TwdNbjR7p%s`q3ZHTAQLoE?B|R4OOVG4pSbRbK3=9QG?d| ze_TV|IMf|;IJQmU_^ADnZte9LQfvr39cCG}bC5Pyx5XSrF3k%#(O$%{3B2N1^({&L z{>Ca+mS=gYrETTa$dh;GKGgy#f79*xdNm^t^;S4-8FN3vi1}zStfb7#eRDph1RX#@ zt6*UetO^wadBO+5K87_Y|Ccyp#&9xIN#dsz!l*sTB0ODdx;m2N{jQVTqhYO%nr&{; zRoB=4;&R%?uBh~rY6>4?;#jaCTJ0E6l7ePnU=R!XHluVa7MKEaeYYu@&=L)NUUi*7 z1Anl0=nFe(nG>RIjr^*}8?KXX@oACiyGdKo=`L8d&@O{Syx^0E_W#6 z4+n%*5a+n8B!ToF0_s#BY>qTL%EE}|ScYJb9RnM&rFdAQ0Hl{8@NlFHbob>rqTtxY z(O7ZzQT$iuIqyst6NurI#``Gcos%+N2H$SG7rNcIlYjg6-?l7UvZ<&iTq?8pYlv~Z zRHoxUMw25^yH(q)Z`TsfPxxQHN`X#LFQq@&jDwE>GSMXdOAMip{i#V;u@FF}pook0 zidw$8Oj(Bxoleng>H@a9@x{ zh{L^7&KXhB!SX3Hl~I17YqxNm--K**JTH-xfVt>|P_GT8#0uiL@1qo`%}CD3m(#3j zqemH$w86-Ydr|@EQ-v@Iz1kClXc8zLi!?)w0roAF2AWa^U?OqD@dwz2bv*Ymm=DnL z3Xr5w1U5Caf=2>)N$F+g>OL$u%^FYhUKq(ay&{nq!8et)pIZ0-q~6}{|NpU&e6_I9pT?q=D8Ea^h`rpm27oW!3vD^G7B={{vZUQPys|x zK$hR1IZ)4&!&OoZEks=YtTFuE3Z{CAKsy-3uR*~;mq#5kG7IR>dndwlTx%L6F^Cq{ ze00t^`t!QaYGymQX`=<-sgBa6+H3Q2fv?kI`lH<;q!FdG%~kPKHP9LKZCqyk+DXt! zEV9<4aiF&w0-#x#D8l{q9~@Xct%ak*F_NhCq{avtlhxl+Znl!VN zksTStb7{D_P)GinLnI!5Ra@0>-*N5Z^u_O|9T-=7sE-u)S=8;bcn%p=tN?0IgetXk zw(14UJgHBtZgmQ~OqNMpRR`P?6FF-G)ZK|IFTR>3c6DdNSn$QG#n7rp4^0-0nLbgc z{Vw^39M@~Jl=tE7-_Io`S?I9`2|5U8VUF)VysA4jCR(>XG#*3Q+B;(SqWy z@pfija^3f8->hxO=Im8B6sLc;MyU^zOrG1vD$*msbb@?v8HY@W_W5=cn$VcQA^QNV zbriEt+!;2hk%(&?ST5~aK43`E(Z<3b#}H)Q2bWFi=?~wmY{+H~Dy!NXFH*0=n=@Rx z0&d2P8+mhCZeXe{nEK(I(fHC?WJl9?aS+7$*-|&s3??Tg?(D(C7XYXv3r8i8qY}Y~ zW=r?6QR!Mb+Z~7T;sI5PXs^|ZFnd_EiBsvlhzLqp-df20KK2o@2PmYdZTuiipNJ>6 zIa2f3uloN~MT>ZR!kSD|y%5_+;z28j0&;mK@$|C5D#Ko}3;ZPUi_N+`+oT zxC>o)?Z!}f zlb~-`HuIf2WxrEY6Yh!(AtR=O;E%KVHEH?M|zVpCKu z;6h6t>3t_;t*fTwKS*KOYF3T`z}mU(DRR%65>HiwE}eL40W6cpS6CXYg1X)`C}5V- z-(NtzC1_TOl}}$i%Q-d8H_F;nv{|E*E}mZp%I4)Hr(s6D%61y2|9CHYRv1E$G_cW$ z{zj}HpS~`eap-|$?8zV&5#qh|j>37f6*DeVs`uf=+Qbqs##K*xDM&-8vdTbpn2$H9 zFV&&-Y{qbDS0n$1YMdzu)wv$w>aoPHmT_8K;3Mu|0(?xZNPw@BjK+`~61}4yw5NCj3S~mg(Y^4dR83HwF oW@?Bd2OQx*0N?`O{)_?m-xYUDly*za1n5qCs-`Lgfghaw4~i`dc>n+a delta 3194 zcmV-=42ARVPw7vvatH;Yr3U8ZlXnOwf1Ah_h2PJwpr||~Rbz=pfWT#Y%5knTQ5@f< z=EeQI%%4Q;WVecaHg*^?8jAdM8jd& z58g!m#Wa|ut0ZwS>8085lW{mt^rM48m>tXO(c)lus2{8Sliy57Zyp^i`@Rl?Z^_nY zEidJ36#bEZ`8YV#ja=FU?YreLf6iiU5Da^ZNqQcA`w)bqahjbrI}hnclVtg?HVw0L zY3kavcXz0_bnRO>NXL0%Z5$2;L4Kljt-napX!_ggkXk2Lg(_zqs|>Y?_8Q7E9MjH! zkMEOU9n)O(x*Xw~UfMf4h@)>>yzC*F_wrvx?VDzHTKakGm%KS2TXWrTe}3PfWp(6d z@Ajy5(7eqPuN4QTjYnPR6>i&ttvyeTS0g(oU~RB`#J z;tCb=M=i^-Q>jfWJ+|>Te-+p8Ya$W9`w5YV6ZVEk1h8+3#M+g-BND;Wdm<4$eIOEX z1$`tEaq2rnB7VP9l$he|`a~km?k6GQrYO!B2qa&v1pjAj}K!E%w+%-2%QPyWd|J`R#> z36nI}Yni=G=9~U}f4@Hr+(rhucw^rTsquDSV>A&cMAnk>dM1OVA9Fpn! zOC2R;i`F0;+VVdI2|m{OFo8M;`jX2mPnEqf6?V-lLlgx>uTjz#5S^d|P$UwGB9TZGQR0b? z-g_k|l5ly`Qxe|s1SqGn!6zNarKpVQVvI86T9jzyJ?fcPFK*LbKT7r{y=8_w*ZJA& z(mKkkfZ(~eg`n3^Epuxw+aXXH9KmI~U{Ij(nrPaBe~lBO0ScK$V#qX-L!$9Sp7$FK zBatPLD3&GYYz}TgN zbE_I|HXZ&jl(ke?fy&GEz*bn=4a>3%vE)zNj}<;$w{3|fYTAy(l02mcEQtoNBpSdH zH84MJe>4(VeeQ8E$Clv@=%M=rOf$sA16Xs9QU zSx1dk=F9cK)cToo!)-eScaMs1IUOt9#jzbpCTiMkNhWHLOrilYi3Z3-4U%b=kx3E& znI#E2n}ZKz$`+q=LdcY1T@nUQQ(0=cL2|ZLf55*N%sOhU@_=#~*tD^a=aeTaheYA(m6&&MfAw0kd*`o*@|k*iF4V56$5NhS6x~yL1yrJ6#F4fh zqH(3KhiH(>3eFy;0V;_Gs4O%0umnJ5nY~AWzXw`;(h0E?fX9b~p{0PCe85srv&#pu z{P~`OXX^C1RNGdc?4H@M+=4Z%X>cx0bd5OD>O(ZHwE7SYaLF{1OQw-re=?2a(k!$3 zumqCJvIHFytUl1{J9zt8621DfFXA)VmfC4_Zp7_i9lh2y2ZCC|pS_NpLdBl;9AEI%k5plJUs@|Jf`v`>;*`$}+Q$31%N? z@kuAjuq2h>_Mu^bCA0g$QnqHh&tq7o!<ELy=j1w@iJ z?MSN+(RkA8Lo`Ta1*;F!0Fh)oKxCQKhjju(mRWsFu=+rYe@{A5hAoK%s}BtWBAL|( zhp}bgK z8{7?7?~K&ue{mSaVfua5d;ZeO#iVyMjRr55gIdhOKNrEv7yTE`{)@iz>KDy?r5$Qn z>G#sCx@9l<2R5vugL!WheCoxcaF%G3V3-w}of}BLV?Q4{#(e(b? zIOq+6IRBR277U{(z5kM*lFi#ZkNyi#O9KQH00008vyBst4g|I81m}|;A}oJMTZHXE zTe51P(3FT-1NoVhcoTpnZ_{epEus|saFiXu=~kW+I{y$=QT7KeinJoMC7|g7OY$Nro?84_3O_ zhC#Vv<#5)1}=@|5xw{ekEAMhvyAg8!z<+FHx*{o2Vr}-jIi#UJSWGNRZ7GzOu`;j;O z7oZ%6{F`_NoO0iOg8H*xBg+B>qNN7r<&!TaTLFKQizYk)36sbsK>@Ur3MV=NCX+=c gCIq$W1m}})Cl&z*lZGcg0b7&BCnE+8B>(^b0PW`p&;S4c diff --git a/summary_results.json b/summary_results.json index d52b10211..8feed620d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 5.44552, + "Performance_Result": 4.26277, "Performance_Units": "Samples/s" }, { @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0358, + "Performance_Result": 23.1786, "Performance_Units": "Samples/s" }, { @@ -725,7 +725,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", From f3dc896c481ff45ad0d94d7236508f6c1272263e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Mar 2025 13:00:04 +0000 Subject: [PATCH 337/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10222 -> 10223 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d4eebf0d6..82daa3a79 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 10:51:54,859 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 10:51:54,859 submission_checker1.py:3260 INFO] --- -[2025-03-27 10:51:54,859 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 12:59:58,389 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 12:59:58,389 submission_checker1.py:3260 INFO] --- +[2025-03-27 12:59:58,389 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 10:51:54,859 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 12:59:58,389 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 10:51:54,859 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 12:59:58,389 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 10:51:54,859 submission_checker1.py:3276 INFO] --- -[2025-03-27 10:51:54,859 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 10:51:54,859 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 10:51:54,859 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 10:51:54,859 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 10:51:54,859 submission_checker1.py:3297 INFO] --- -[2025-03-27 10:51:54,859 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 10:51:55,936 module.py:5252 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 12:59:58,389 submission_checker1.py:3276 INFO] --- +[2025-03-27 12:59:58,389 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 12:59:58,389 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 12:59:58,389 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 12:59:58,389 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 12:59:58,389 submission_checker1.py:3297 INFO] --- +[2025-03-27 12:59:58,389 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 12:59:59,500 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.76449 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72562 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420191 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.26277 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.919 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1683 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 03b7719c6..aa5bea4ec 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -964 +965 diff --git a/summary.csv b/summary.csv index aaf065533..dcab52457 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.76449,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72562,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420191,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.26277,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.919,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1683,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index eb0d0919b..3b37b3146 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.76449, + "Result":1.72562, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.919, - "Accuracy":"acc: 74.800", + "Result":10.1683, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 01ddfe878e6eefa6f986f39f9c4207384b07ad0f..912bf8d5f8cb61fcd8ecb2942020d424e6a8f317 100644 GIT binary patch delta 4162 zcmZXXS5OmLw1yK1U0Ntg5JEzLC`dC>qEa;U4uVt(8X$C)BGmxW0#cFyhOTt!9RU$3 zO78;F5d{n)NISqm@uFwu&Yih4`(Z!*Ykl8Z4|{JKZ5dS!(sPVLBv21P06-uZ05}IA zi$e(WmNs5_O1zI3?Zb?r)-GUa9p-rQ(?VJ+?b4uUy%9H%rwllN9q(_03X%=IgVP1R z>lK9PybWG#`ZTQKf4DEE7cXJ@uy2P%tMeEa3Fv{RS8A>g zcfTE0e!48#wj06apiQp~O(Sx>dt2jAO(IAwc~~Ubst~Qx7;Y@2O2(ZPW64cVE=YBo z%}!xJDGIVZ*)ejjQYwTgJ2|9~Y*CuJWX2m&!#vjay{|nj4ldl0(*-9Ecnx89zGh7N zTeh4;lbugCXSmvT!txQk2r`2#d&)GqdTaetPYF-+QNZGu(#jReJK(wk!w*FMe)JE3#g?v8M`dUV-p`Fq8bBd_`= z3&n4RB7HGX%#lr3@Y%bPebT~?w?DUt=n8#$f+WOIzs20V)+Y>EY0SQ(_oFY7i@Ep! z6c&n089xYW{bBl1|RKt{Xo8iJ|hQWKLeTGp|pig((OC2?vTv-p7Urse#G69=5 zLQ~h}%f8WYgEB17T9g|tdg09GHlt)a%)Jsme)Cd)zOplvca+(GX8hb*k7aJ8cKbL! zmJ@~3r9a%o;@|bG`u!GA?tPrlGqT>sxtxCGf%0I~+~!`$@q7uPQLI@-y7`SXT9!T0 zN1Lf)hZz*h5vDc@<3gfAbQ)8OFmwcmM&pym)6uz4=*{NC#L+!Xqsk)RY`QKYx9ols zkj=wAG)~OH$RarFyHCj0k0<3r-R5iGO6b`;NF21#NWXBk=j&X|w}j}zN^Xi^omF-p z6?43J;x6MiA`?O=M*nr@Py|v|L^DYt8e_~q@*m}tHoID_SzO*FNie?Y49kM!$_g;|oX53x1`FYkhWMGZk zT+${*^9yoaZZK6Ep;r&KNa~wFCGay&S`4R^!_^O8RJtu^U^FK?_)!J&rf$HpArPajSyG0~P<6LeGjg#7ApX!Kc)l za+MWL!Tp%;4Ys9ytHF71_{)kbakX!7H>%vdpCyUF3z{kRjdJ(ydgtlx__?D_Sd8`P zLA(HV0Du_)0H^};QCSi+$|Gi*GvIOSphsRWhUZOXzEKvKOMp2PB8xQPH)Q-lptSEG z)o+W#Z7_Y?W6ns^>h3+%=E0K<%+%8bxM2s)tMsO87M8Ch)3;21LVTuDvfrmJD+IbI z(T*;xUiVe*P@K;X^5KeHe2SmGd2P#S@#)MI`ue5TvVFbD%=q$ZPo*|5*)eoY?W3R% z*YI;r#Hm;|frl3aD{!5JeT$o-Lc>nR>72Tq=LY8GGV?lW-pLZXrc-Re#1_dWIN0m< zLj%N#W{sIA<=JcmNH;l-Rc;)d;5{;e|Bl%U7 z6l1ms2XFt%%EiRh&WeK{_^EOem0E=0+|CD=NinrSnuz6%$1rfT{e6zMrSsa@el`xV ziq}iVW4!xsU|^Fi8#wp=o0~+Lp)@z2MVowXbj3LtX@(z~V4l9R!iSIJqewkqZQeLAp-zEG zQTB#jhEJ}(^0B&QbVRRne2$gtk@i>j($KZAFP*m3PaiqM0|M1GAKdpYUB81CI+0CI zxn&vN*T4VP!qVFJJ6>^ZyJ*>HHjz)ouF-=i=4z{TS0{UnJ{3c631-I#u>P%_hU4`MhNt!*13`58SoLM95SME|;+v0TH3#L-jw1q&3IrlL0+a+9NaPgGJ05 z>U1O`^aUni4nI&S1*w7vXB&o}LG6DcIW4E?XXo^13}$-+I!X>VzY?lD476rbaO8L$ zqtjr-pTJz8yvY-sLR=4mC3GMhR6f|wB=7jw(52pF2O$U}9?U(IHb~%2U)L#}zUV4- z#aW?oO}}#Uqs4-O_~M|d#VZ}pAhQ04S8otJSG2`P0cjZe(+VLX;}_u+1VEk*d}li4hD{F(P%8Wvh3!GYLpP2U) zexY1~y0{+IhssDrD4e6QnQ5p9O%hS-ykhtZRqJYhmNR*?f(x$L`@S?Y0|P|`Bvw?mz)-lB5q=N@&EPK0u| z^TVR?Nxfix;XI4(MS+Z1r$r| z>>*v6lp|9WXrPy%Zydyq2}!xFrSVy85(|R62U?2uyo)h}wir>5 z>zL0v_MD?4DPoz|_33Yc*^RAmn+`0Yeej?}Is#|i={|Lt0)(O9s8l+q9iX22J85Xw z&RP&__RH}#QxBVvu^^}MorfnQ+8LR8@fF-!)*+{MEugtT=`_uUy(qF?A<1?fUdpK_ z(j=ujGKSp``_tefX;>NIX6KjSUQAWY1Yb~yF#T@PhMk2eE-kWbq2U;BcTiT&w9Ex4L@iqU_`3s7+-b#XIwW%KrNgsPg11x*d?v( zs%O`>V7qBEA<}yd^AdK03POuR8CQHc((ytM@)x7=*JuX>7z9%ru^8Z#1mQ<>>vXH@m+@!&R&=T^8whpf|PqoC|jP`nGpkU1TD2y;+! zlW!w%4y5xPhnxt@=e+#1cEd~XagEh@t$`%dpMy$N8rwdF_WAi2diyW*+wg{-?N0D% zTmqCm;cwjA+kih?k4u&N`8AeZHIa1G24TDSvXVEsVO^x`TTPyu^EYF;v zx#~zgY6z9_IpT8|;i#r(qZ188#to=I22@7I#Ium&Dccoi4JLJ z4`;WVE*90NkIi-qP=;;j6 zK`AS*`){Zq&7Bb8^ONB%0spc)zMFh{GLPLK(!M7g--*u*w%^Zgt$Qd(6E^5P>0f!y zU%PFe6SuSyx4`E7TVpt2KkkLSK(wayzpbcp&uP7X=mjT7+h1QjqvD(Q$)8`4f=%L2 z&(28aE?>CO^i!E-tv_dM(Z#+K!BTXPqp98q#Y`9NHDI}-dVfW4B)c!H2JG3_M(|L9 zpdac2W3?iD`E>L%&5pI*XU$kaOk4AZ6k)g|VHOs`JR8=4n^1dk;nF;d3pvJ`6H%Y= z`_0Z}7M%HrX7i(=Ie}Yc2{j3i+tDw~i8TlfZe(GT@iDm7Yij#?Y-UmHxV0qGgpH%T zBqLm)qb~AxDZ8c2TMc0Td51y_(-wRC$7*`*b@Gz2`kag8OU~OZ-BC4O%}kj^v`eW= zgbPL4%N*x2(qc3b&X>o2er#sCAN7KfrSEE{n1D*abxT?W|6X638S5hCm1|blbcoMg z1qA~NDsBO?<9z77yEB;xVVfwA1(vW5G1scKHkgME%Q!r^;`^A#;x*4(=6*99aE~s`GK629&vSGGlf+G j4V0p2NQnRE+Dn2T$WDopPysqn-bsiu@`#;2&;kDgUE;OP delta 4159 zcmY+Hc{CJk`^RS(TN%qBOLj9uwt5(Qm|-xMRLK62Y(t4;Yb;|Qqm(gY-!hg+*0F>K z*>_<=NK#2ASwfanKRxgHz3065x&OUB*ZuvT?|sg7ZC>8ETsy{m${NTy?+*e1!XV@* zc46`sGaG*4_IqvqNPSmcqeyINjB_*(&xM?+@U;|`C)$_gpj>Ejukga!%I1LOW9z2S zY+o1od_hcJ<98*O@g_FcLfWe^=mS}VM2;n09h^o$n0;HBIM&ngIVkHgN;N^5-Bm%j zvTJlc{3U$x>i4)BcHTrp>P}G$g!~)lOw&f(PNd_o;d!rJHXxH zlfs(*1<&`bcggvzwEY`0B|a8i4=5{Aak_G3sCzNfu!-s@vR{(S$Pnfw#;FKc9|akw zt>DctGGay?KEsu+GUgsWf71tJSBvJ-#M3a(C~ofQXx<@(X15`hF>!{;_&oAb@IPv^ zb@_H7HERuiFTa-0?xdxu4LohPf?o+MtoHG{j} zDty61{At#s;+YpXo5b0qa7=c)aBWGmT_#?6V`}>1>jm*LZ7TT21aH1yuZ2liO;VMd zQy4`lWiwhX7%hlCH0ix1ctc@WUEI^@Lz|?T*m9iQ@1(4?Byn+1wv=y9CN;3I6#_fx zRrmuWvfhLpH2T&Z!~~z^U;@&<@W+HLq^g3L-u}{J0+x0nCENW5OS$f#f@Rr3Hu(Kh zG0_Yc8J;VPXF*I)!|x-J!J9mTI`8+kI9AtKkzcukF{f?(POu#(*pGks9Vd08Wppse zA!(dPJIi=g=d)n0ckbRXgv%m`&F@O3!eEQB=Jjex_6r|w;|XGq#(%Zx7FdqgoZqih zj~UT(%TBu1f=xuw>TP~`;p^E_?O5ElFAlkPd&nAoiVgT{bw%3zx$mii<2rhSQWty0 zJhyqU@XWNwKc7^)S7!5^My4;aUvrvF{TuD~l=qBhaTqTpq4KbJVK*nl{pVgG0r7}j zdf4|Mi9L;y%f_r=tam%KkVIZzP2#LN3YV?+5zDm3r{-2w`*%9*HQ~{-eMYm>Mr>uv zMO;2z*TiFxlTti#+8`z)0`l?aRvk29qUdwT#QD}MK>^pPIumF%tyyh0kH|QCmX-Fe zzqpO|g|#A)LHmnJlXI2}`?$>rMuc;4N455Zq$nfxj6HUM9rTnd201OoQ+`727=7}K zYg#HO_3I(mQ3CfhJR2LlHBNn(A^p*{zxwFJxJG-tG;O^c&XX+iVG?Z0xPQyTccJRNU>Ya-oQcH z#dni&tHdJis&Vm8`Er%ZQ5hci^KW*J!iz$;`U1Eg6iMy8MS6y$XEJI595)eiv9Oj? zuT_RxNNM(PUyT*BJ_bF7;W$BUO{+SJJ%yr;Ehd9_-58b|e(q*{pN!K#?oD8KPhT-- z2Jr&`0CoTe0H6;@ohcP#kngcOaf;F%!$bXxUM^bId?vqRe+pb8-{(-q>!++~>Snfj zS+s=`8okC2)Hh^{73=;I+5dqGq24t1=apUFZJw5Dw5&aKZCu+n+n4X_Y`UlR{CLNs zP4aayy8T|O=yLAuDP8TPZhzy3b*X|-n!oYaoB0J9E;C24o8$d6_8NIh8bUr|b8~IF zzT$}6wBDP237jnyuwDH!9#>gJdg!exH^x@##`$SOX#A?%+>I>s`*kyAV(((QGcU1O zp#=`X%gmi?9vbm8V1f-BB?ztd6(^6F2-rQ6ufET(lkWJB)#>+*<@^F2<-1q=R*I7q zYzEIa$_-eSs#op4?%%X=?x`W7BQ|Bx(w48!O7p(xfI)8BeS3%j;V*9z(ObJQm|lTh z5~jDppO<*>%knCSAYF%k3U8oBXK;lg9EW!Ge7Hoa4>At8-@OY)n)@Ip*Z*U`L@D2 zbAMOcitqTubE!Lb zdgaZ|q24z(tDaL=`s%1aR-uO6=l`v|P0Be&ix z8BEkvmc7jx31PQ^n*_`$3-M$Bfs5GH-Z4NRrRW&WY~w%-qGNC;s@M%$4}$bjLHMe% zF7ShiDR{n|^>`G|jFj6qhY#Kvj(QO71^IXPBo8kLdTD--yI$audR6qzo2RXrmIQrH zPnbZ)<8OhwwE`Kof9uST-8-&0pnrEyxIF9Ubb}}|OR`UUvKtMX1Z1F@?WwGv22v7l zm_h&^2w>rF$%EP-49hM@sSsHQp(2ueYM3%*av_%>{^BLiiWSzO zIa}awmkWOY%rkp0)HTQ<^pL=UTKNtg+_M1sWC$$V!+nbEL1i5}sU~BiEnZ1xk4dYT zGD*@TQ8({y#aPV!Ylpdtr#&GNn1hbL>$M}1P==fGZAum~AIl#5sztBVX-W?z4~6bL z61bhVPyppqtHz%nLW9#x5lHwX;G~7bU_aF6B8xn*p2%9RsIp%RLVA+gO=3FP9Op~;Zkz)=;^r_k2dW-Mmp;Cc$$jH+N0;U0tF^lo zO;vhpx-_4AS~ui zGT*es*K&c$J21X3Xp?kf>lM|Wc?9?2Zx?nFM!kLa zTP7Al3WL-Zf+BSMXI^PxslmOZD2Wr*O9U*>7$i!f42U59ZSw00DJ79=strWkKi12Q z9;!ARcG#Wngpq`U21r! zIkjboC_o(ZfT<4MTlgMCG<^FDAjc?CVus_fgU7+-^@ErU^|koN9ElQ;L>heO#~ z>Za6Awn&unZqu@CuA1&bL*T^8MeiiZi0Pw0m=17HO6I^s^(~mO5U?WlzNo$;@Ug?k zgUNUQ(m)X`JkxGhkzT?*kE&m9?FeQ6uB>~Uc4US}og38~wu%y`gm6)?qIXK>2-pap z?@}W1_3?8AAAp&Pc-Hy*qk7NyqU#6Oi@VnUct})HGKD>l#0p&>TGy+kRK4EV5y}{n zl(EuXBi({EraN}s4v(DD@m`|4aFHyzNFOihOfd`*9Sui3V2DFEqX&V63Ul)~c=0BP=Z0&wCFRac-;{hg{cDerLgfC|&>1iz;awB<+0u41)S{nO#QVDicejMx zt~E89w#T}9-}E|kX|>Bu>3Qyxr7L*bTVtP>z_yzY5T7PBAM_9HU)!8BR}ds`Od{4h zow>M!hR8$jy1zJB zeTvpvDugE)N|cc$azMwi($C!QY8Gh$xZghfK|fHGNHhP_tZrFv@Pn2^pZp4#O@6b` z68N%!AcOwTRWvDq5+-KHR&Uy%p!F$`2$%c>SwdO?uOHwP{}t&}@B8>lX2yCv!W3P_ zmgYEsckQ#x1JyGEgEsS8RLdRLJq3Tp1{+k0c$y|lU$MAb)TDNV&D@qr&U=zfw^S&f z1aK+kv?U97Hza6Og6+@0Fanl{x|JF0Y=XO9*NI^mY45I}-e46fxJwsr7|v-dGL6%B z#m%?y6zj{|K%w01q=%Ssuad*MMfP``hHMZ@sHV9}#CKfn)Z#6n^wUrHqb>x{aS-qQ zx5T;E`;k+E#cCg4Zp||GBAwOFFaqRR%PTZxMw#~J)de~fyO*^Y$MSh$GST{AROfbx zvj>A&JOSX59B!p|e>F3Nm~(M#L{sVganYEtm6cbC;w*t=_tX3FMXp1QZBSOAD?auU zH_c5Zo{8hXYdm=arpxl5>G=X&fXRcM41^1io#E$!@#F_^>9c=4zIU?k%LxDgf9CD~ uKhZH<8ro3%K Date: Thu, 27 Mar 2025 18:13:12 +0000 Subject: [PATCH 338/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 10223 -> 10226 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 82daa3a79..81e15df4e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 12:59:58,389 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 12:59:58,389 submission_checker1.py:3260 INFO] --- -[2025-03-27 12:59:58,389 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 18:13:07,020 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 18:13:07,020 submission_checker1.py:3260 INFO] --- +[2025-03-27 18:13:07,020 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 12:59:58,389 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 18:13:07,020 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 12:59:58,389 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 18:13:07,020 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 12:59:58,389 submission_checker1.py:3276 INFO] --- -[2025-03-27 12:59:58,389 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 12:59:58,389 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 12:59:58,389 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 12:59:58,389 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 12:59:58,389 submission_checker1.py:3297 INFO] --- -[2025-03-27 12:59:58,389 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 12:59:59,500 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 18:13:07,020 submission_checker1.py:3276 INFO] --- +[2025-03-27 18:13:07,020 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 18:13:07,020 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 18:13:07,020 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 18:13:07,020 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 18:13:07,020 submission_checker1.py:3297 INFO] --- +[2025-03-27 18:13:07,020 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 18:13:08,558 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.26277 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1683 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0979 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438504 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0922 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index aa5bea4ec..0b8de6347 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -965 +966 diff --git a/summary.csv b/summary.csv index dcab52457..e8dd3c6c3 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.26277,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1683,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0979,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438504,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0922,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3b37b3146..3827064b3 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1683, - "Accuracy":"acc: 75.400", + "Result":10.0979, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0922, + "Result":23.0828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 912bf8d5f8cb61fcd8ecb2942020d424e6a8f317..837774cc3a065e69ce6c5e7286f7989c65dfd27a 100644 GIT binary patch delta 4304 zcmY*dbyySL_hvLAH9$hC(E@^iq@)8kKynTQM5n|?H_PDkPgD0I+f#g(K#Pl>RzmuAg|417x z)``^8U6Pnv8vFU^i$hoAUKle1LP5osf?-~4u6&mK=Ju?MB~Du%Bi>CR;5ObhFD$49 z!|(QZx=a#x`K5Z=TSC!)Fbv!sC`c^IVlkIFhDEnhvvbEEmpFBeDF>!*cFg&@Rd6p& zF8TKF{9C-ErRPwR9NNz%>hZCrEbtPLM4lYcf*g zR`!&k=9o9S0nxz0*JDk(nO%c*`B)lhL872qE*27x5QOGL$+1;Wme0Rfit810aCw`p zGM&2qt?%kd`KGr^|7PLXUMluhhvTPF7G4?#wNK(!{`CM$5={dr-(`HuYV)cT>w1zKnP4@lzs%@`BSa+Fn z4V(m`y>B&udkB77#GZJ+IEXzyoHS2z33REiKKm`LZP~iob^1gXOo?jmeVNTJ$X9dH z-wFNY1&)B92Q!bo$Dq4W>Yba0Gc_|M0ak?d1;Y9|fnsO}66rmAJx=rO3NtMm5BYP# z9e!i)KRkpxE5_ccD@nm(DoQdVJV?UJ^s5;Zm5n;@U+hjFoJSH)UZLuOjo1zgbuoDVH!P)h$Pny2W_3jgFQo_&43B1T37 zADV~X#I^cj`T+DKsVD3obk3HGN-g2x1(u$IB?h04raQ$f>AJj{rNJ&@@`RXw7) z&yHuJ<=3+-2aN%&@~9MLf!*NE57xyNUX3P?=q*RhD<3>E6+kxD>O+`W^0$6$DdfYP zU18Aa*qC+=fU!OWB^wzT8STZ$o363vmUxXFEZqUB3)(sg4Io!hS&5vVpSzjVJ(1)k!_ zx;d6$r!vvk7`Kgo%rj(hP;sC91_uw`xNm(~sz;yXlmXnP@ZmTiB_uebhoR zeMXTJ#WOx$uja@r?@DZSY74(y8_!@~Ifo7?&c?<%$cgpRdFin=N0nO7NulKlwyV#=8>1vmQ-1p}Wi8afcaT!BN-EGF{r4bPewpB-#+u=j) z-ng^$e2I+x8h>U6*LI>7bKZLOtO(uw4-uQ7jMxR|Ro0oQ2_GUBtW|Dpd#kb8f~G- zoHSSI`mAw%w@2{8U+4(&H+$Pq&I4V=n!2sw-iB#yR54}r{diOIHVHrawtVwLwV5TM zfOIeul#S!-MRIZ?YdjeS41x*)p&%>-*gy`1eiW+U{}eu_Nif$0MNnWeg|MwrbO&;J z@#69h(@l}B*Y(7;=BsPu#1*7AEtU06vT{}gqsa+lVjgHJu&5Y-4Fe`Xgn&BnM}(Zi z5J0q$_4GblSWOV1tI!#VXZ&FbYxkDXP4Se{li%e%=qbBWgR0?Z55#ZDe#uxq-S9!N zFD7X2OwDM2Ei2BSPVM%kg^ApCAHTuLhWsSLZC>u7Ew3ps1i74|>j{x>?A!~e@m;ec}+nZE!OqX*3jig@24U(d|)>y4@b@7;$6e1%hNpAO<}3Dl&EXgo61 zV3Gz|d8kbL>I3z5mv z1FIBh_&VdITn6bjf;C~0I)LOJCd$VX{oH7S3fA2P1GtHexQNT=T)Sg`qFt=k-R#3J zSVkVg&s?2I4Qf5bljW zSVCe(d{^6;x2*O13O0yH&Cp^{%DwomvUO3-V}tLi%Z`(cIcudKGH#3bZKy7xDe%ZM z_gr)&@-}K#fZMh*EIbkw3Vo7;9R=L~F^|X~!|SN?3cD!0`#NZ!6JMXmJR)F12?e!S zbw^97szT1#t&$ShETX$yMB`jX?->Ou!vd6H1eR8;4etg0lneS@KwsW`z;#R>b%CVk zez{?27sx0MmzZXTL=jEUll49?xHx^e5)`X|a|F#_pdm$B6z>2fsun?wrUl zyoGM4?DBf$WFv8gWWAuK{-s%p)Y(IUjM?eGEIG?bmo;RZ;Lbxh8+H2X4Fwj|;gf;$xVnr5jRL}Xg0d@Ct?f|NP=Yc;_@-;~akkDJ;; zww_7SPU6GK?Fc*B&M25sfRGi`B@bRcX_K|zH{SofdDxH<`{SPKU!xnVels%wAGEA} zk=GRBVq$)PzHmE(466g@V}Sp&CLW~^`n-1MODM$t007FukBDYL&Y@s+^PD(eUyOI! zj^~^9(W^}FwB#=vz90!H1Rw(H;Mla$5hwosk+Xxf7!1>HRg?QDpF`QR6+`&8pFuk@ zo8`PVz$+Kupgh=vJD^^E1$`_uv-}w$^)U_oZpIky+`BTv0N;Kszcirqx}9`*e|ubC zf&;%bAivRM&FFaZ>Uk}#P2);~`>#j)>E=zGzY{!=xp_1CLu-i*woT;-9-f~1FSBNf z+_4T7H|eNR*{ijMTmx%T@!d0c_r_DHKWV>i8Q|Ol&QWaMCtC5rdT#X8h~z1Bry+@Y z$tCmdITbOSH5ovqkzObGj0j@SAxB4>$_bm%Nm7VA>CJlRs$P~5^S=uyp+7UCk+ivk zRk!54YjVFSbpr0FeD9Cdr78HVspYMY#m#C|H+cLbpNbnYl&&~g42+czk8xXPb4ctz z3AcIw^%Z+Sg|@WGitwcdaCVZ|J`gG$=7dyi&d)%hfE#b@jM=`V)& zVTBWIUq|}JJXM*l8%W;44)gswi6}C5&z0{SKU?4FgN)?4>YM+8SI%S+nBP98lH!t7 zv^e}R1{t`1#y6SyJ~cL3<;>dO4HWcfsR7U8cY7!6dIf%`0LX;C)sVDNV-$0Rbtc#7 zwPzu9Gw(wzZjgnN#TJ$J=H_UY#rr=u(dC=?;o}Oy`xWCJ+~h(nA{6f-QDZN zI!B$>u-@ucY0vu;fl#d9{A9un!`@n zVCbxkCut(9o@T-)jv~;+EVxYSb7j(UL|kfw^tj-#ZDh-Q_mvT!e6$0R)%Lh7rUhtTd%mq^8J^=O(q+I(xfE3({rW4ySpo%Dpogi754(Dy<2~*`ZrE0 zo`lToU8^_@QVCIDy;&5{>Z_LaG5`DF4~(rMrOLxQv$aYUV>cDm>DaA;tm<&0ZU>pc zZq!eMj`;pJ2M}Ah9?()AydE6K^MNGL8px|*o0?XYj&HxlCvJqvdw@%5aF(LzM{N}u zl%Li-y|>eAl| zdAmJ`03eNr$!mPr1f(O7S}%4P9iS1?Hcark@d%q*PrbP~YWtELrCKIKgZ9#OT6GiHHykBmeK8 zVO3b0>c7EBM#Kt$!M_#(UH#Wklw@RL>||tr$E*LX(C4gpl86?d0Z%Wg0+7b*ii-Rv aw!7#R05d*PR2^W8ZxZl1K8Fh3aYLtu^qDG4#gJ?sPnL%{Xdl!O;D8c9@ zh+Yz8)EER&^2mL^&wAgr-m|W=&WCf>|Los?@AF|_MEza;>OnH{QBGkfZj?+NRZcQs zJKo>M%Z1hR3QXhJh8F~BeF|J`S{_z@etakhj}RreA{Ch!ZRqAVt`{T*( zoL1!itofqC0{clG`z(?kyyX z#9S4m$xTZvNPaq-l|%uR(T%NO!1Jh4l{SV7gbv6S~BhFL5;yzaa%4lE=} zY6Ig3+=pz5Khh_ko3&g-VjVAbW|-QEA^BidFqT4sK4}_Tv%9eze!Z4$V(fj9&jY^# zW{iBXk>umy^{`4Hf9wIPAmT<9D)x2XSaTtULR_FuKxsr1+*gemfqX;gu4)3WO(WY_u+3er5>OJIZ@ zjVuB)x4naQf4-_1>M>d0EG47wz%USk!+k^VzTIG=e!xu{QhJ}y)=_!i4^_;N^U6is zcSJm>SW#YLg4^4!h+IKVkxCTY7-jO6{Up1r+1Y$G*BQ^~LQxq5g1=d%h6!8BForWq3N6w&yyhuWoMt7`k)xY6~{?dI6-@L2xg-@0@AN zR+`~cE;S)EQzg>xU7s1myC~eQSXi^+qtGEYpC91O6u$TxIeq`m?vusWGgFF6w_3{& z;gcD$6*X@1ZSE3dinVoq0blMQ=boUaqUkuE-QcQ3bfOMT?+b7bKQTyS)b6}CFfW;r z*HQaf0^K#8WCcXGh%|wK?hl{of-lr-jool>X2Zy}6Juy4hhj9?-o)RmPkYS=keh6Tqj2A(`z4OdmSWMll5YW?Cg&8g1!VO|}Jd(yl)di@7 zSGHdA10!ub7}}PuYufhHF$h+ESTY!6J^aWIG~BfWF&}=sj}{*~KMkNc2|o@w;H$*M zQ!H1UK0adkQsRQd?V%Zz@hJ@5y&UV(+KD%qPznLpj+cIP?mD^;&+^_b5`PnuBX3Pl z--Ac)y<4CwiY0*0{n9GE$FkQ{q3=0{DOD3Ir$W=gQMmNydp?SBbp`r?UQee6)qxc4 z&oC3clSZ-n+c=*<)-D22$gO-zDjrH^BAPFZ)U*h>GMbSMXX1`lSIaz0CV+WvPR1wE#_P{Etn&4&o>m#I zM2no03*P0%Xl_Koj5NS2v_ZS<1{xio#dj}RZ(TW3J0|$PmxsGvY3w>W!6#{AuZ^xc zCPIvnLvys4>v3+)ty?djh_?or7tElz3hxfJ3EO?)nxETfHkLbjL03=#gqRv?)38>} z-C|9DG=CJBj{D&WR011X-Mze=c0kX76bwXzF`8gB45W2Hs|jAlc7^_Ak03)oB0*nc z0KdS(fC55ah9QQ9BN_gGcz|V)0&?`!!chL#`ueAbT5Jv8?xi8E+Pi5+0pU6Vk_b*b z|L8Gj_%t>G(+kvOjRB(SWvS#O?!nW&b9EF>Rm~#9@Q$(B=FTUYKU~U!*F(N{TBR(1 zWeoLyp{nlb;Z?TrNRj(OA~ETKS!iGX;U`lw3!iPI-1=V8ivDaoo3eGID_YRmO2b|& zYm6)zMrH=2w?%WN@XB#U3u*z(*ulsfP1zo_w_L~2QmbSt)SIXg2?awb=7B^@wmTYP zjfMCtICEWP z;{7{Hk9U5cYC3c^W|I)uSS|faVentUToNh6R|wgdUNBAYKqz^|U^|tR!#@t)!YA5s zb5bIK%tNVzDAu$Mt+MHx&Z07ovW@FHRXblz7j%Ud2USepYq{y*h;A|1v{xcmH4dJ8!v6H=3kIf zPk~EbT^~$VpyUo;VAN3bj_Ss-bLXN|h2SaPDU>N-*=qzkV>M;&Ni=kWRSFfCsYk6p{#`yOVU-o1UAI+Sc}>dh+rI*H|rP z@>#yV3oB+ogEWE63l^p>)6=TNs;|xE`Jap)xZU>mu5J2 zs@U2%z96t?prz=*s~CoAixO~Ef_>AnVI1X463kH2A$tI%H!w%+*wF;{fdb-5QH=GM z_0(kq;Nb;EB$GZNlE76zi}7|5*8^y?O2*fXTrGpf0-lT$pIwY-rf0xoE15Mcf-c=P z<;^7#OI3e{hhpJ{7^@9X86%vpNmP4e%=R$kZ-9Pc{NvZ(*c)l_UzM7csZ4nO)!I-%p(h5<6&^7q%Xh(i*&;2QK#7+^V`Wkm1;vV4y3W223_z-WtS|k-*e}BS!+ICrz=AB_k+gE zkL_JTd!PLSz4s6Fr_hGpy-whzTpTZb+&^-k9s>SWJtA51>_;@cN<8MIjg#(XNfm2i z$5x)`>bk1a`dxH4vqsuwTHZY2%~gfKQ-V?`--5q|piXLgw>lL`ATa~VoCC@uV?vpb zaqNT8*~kczh()$dzFdsok?q^R|aWnvZ#6 zJCPZIHiucQ_0J>;Ji471{i|=;>-KE2W0qE97U&%Rs15rc#^l&=M5rsybJUA2)HHeGbk zV9OMNKabo>bXi#QKd`Aov!swv{8Z%_^RE+P%Oge2G&^V)0?jCmCKEq>d+mG8)PNXA zYH{@}o5Acu1SqIpA}+V|rOdmEFVE6`H?qF3_*78u?C~%pfYwTp$k1by!D%%*Rh?U9 zS5hVO{+_t$kI%HNW;s_y4(!7p<&MBvU!d=a8~zZ)KBAr)aSPVHJw68h%HRJMlJ)yh zmcy(O6qPJbQ9>v&dKZ&ZmRYsx%kv<`ECN)bvUNmnlCz}glu&m*wVa75h0iBf{;oi@ zsi^VaZKt)6GIOim!B=5&P0myrl+0zgC4J7Chzqdej{^b10z1!_R7~uJmkkAZE}UP8 zPrGMx-V*w#m3=VLQC)>&R5!X1J00jdm6)YkK^>nYU2x9t!OtPDdtwQDF@IOK6ZGmr zdv{(Mwcq!3jKDP0iMz0Kvd?SyDLspi4xC%*@LkTPOu$v$B(y*`N7&dd6-u{fT-uvt zeMN|5(0v!<@EiN7^*i>RiXUVBCJ)uG+``|FEJGc{hv|7`=xu5>it!BLpWoan-PA!1 zKN0RF`pmvj_Ot0$%4M0IO$T$&dtUc*pA%@_PPOOxkYXh+E<8o*nIHdsbk)1T>b2Y2 z;gugz`0@R~M)zOmAJ-M^cTc>}T!ic&97b)wkoDAU*~PQq-Kbf^A&epGD_xIz=8nM+ zT=rZE+F2ha0)Dk}yxvP(H zEEvJ>;{JRjO4A@rM%0`AhUPdPl*iS^y=+&^F+taY)tDiLO$MjHR`;nrrRa>J=y3}X zh#?(AMQM5{M@N15!!mj^@lR?b`Pc0VVN|>H?O$uiG&itI2C8#TA|;FuTY4gD-J7X0 ziU_xomryr~vQ`+brKd)zgB_*E&b~HNc|_z;()8WV5adwyS2811vLE!N8q+RvzIV>- znhx@|mzCAUL1Pv;iH^r!`^3q3PP&PTXp&OeA?mujtqmq2!{T+4w|__wH=WUckfTX&tz%X0MNiToK=m8)v!G>{&lKSFxJ{70+8 z#TS4&gg_YSzkeOJ1n*J&+eS$XS(1d~;)Ouh|9Md4004vy0QlQ%{dWY1FtXvq+1POi zAtqcu2NT`@ZZzWnNC5!m{~ozqgG1v@u2Bsv1 diff --git a/summary_results.json b/summary_results.json index 97da2ed97..6a49e391c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1683, + "Performance_Result": 10.0979, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0922, + "Performance_Result": 23.0828, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 09f6bf874c69ced9203151d76b8ee6dd73416a61 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Mar 2025 18:48:18 +0000 Subject: [PATCH 339/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10226 -> 10229 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 81e15df4e..9e7a79880 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 18:13:07,020 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 18:13:07,020 submission_checker1.py:3260 INFO] --- -[2025-03-27 18:13:07,020 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 18:48:13,174 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 18:48:13,174 submission_checker1.py:3260 INFO] --- +[2025-03-27 18:48:13,174 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 18:13:07,020 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 18:48:13,174 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 18:13:07,020 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 18:48:13,174 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 18:13:07,020 submission_checker1.py:3276 INFO] --- -[2025-03-27 18:13:07,020 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 18:13:07,020 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 18:13:07,020 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 18:13:07,020 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 18:13:07,020 submission_checker1.py:3297 INFO] --- -[2025-03-27 18:13:07,020 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 18:13:08,558 module.py:5253 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 18:48:13,174 submission_checker1.py:3276 INFO] --- +[2025-03-27 18:48:13,174 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 18:48:13,174 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 18:48:13,174 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 18:48:13,174 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 18:48:13,174 submission_checker1.py:3297 INFO] --- +[2025-03-27 18:48:13,174 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 18:48:14,538 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,16 +24,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72562 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70101 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420191 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.26277 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.66684 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0979 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0828 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0b8de6347..ccb8fd57a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -966 +967 diff --git a/summary.csv b/summary.csv index e8dd3c6c3..e513c990e 100644 --- a/summary.csv +++ b/summary.csv @@ -2,16 +2,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72562,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70101,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420191,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.26277,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.66684,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0979,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0828,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3827064b3..f41bc5f9e 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72562, + "Result":1.70101, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":4.26277, + "Result":7.66684, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0979, - "Accuracy":"acc: 75.000", + "Result":10.0828, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 837774cc3a065e69ce6c5e7286f7989c65dfd27a..b724b90c9b810d6762476834e75fef93daa8e124 100644 GIT binary patch delta 3759 zcmZ9Pc{r5a|Ho$rp_0j3!`RJuYBECB?E6;sWfGDp%94<*w=HY78OAmkV<{=?(1R4o zph!ll$x=jP7g-vV@6_}AUC;IXow3S>=X0O)IqT;)&({yKv46%SM&pOs z5WtxKw1z;Gj{7B}DDTQx*O)9R59P{jnh5q>NQt$nXfh&2sbS>Rr_5qq&X&-EYi=is zg-wagTc_N%GIrhFX$fbC<9bmd$8L=%at%U+sB}+{$kGy(v+*~ku9lV7-#v2I>1BFd zH?TAG@JCl(&Os?2uSgHnoxX^w{&-ods7b&}eQ*Y!ELe@6@oP{BbFhvnx#bXsXBW-uI2E_`3DIeop(A))u_x1^tOT6pq$XOMe`TsyBQHO`cc)=oGHO+ux*tHTh+$aR~pcDQKOLa`dGK22}SR)2DvFce}QY8r$G>VD6_G0 z%@10cF&8JdpPavJfAq}ExP+*J6O=taF!Wh>Mj~T>qS%rutm?GuP}GSn ztxxW0B&xCz3VoyiJAlsCS3?vUM!lB8^qot5%|7c%eH8`^8Ws3Px4Y+4KqgVH=~#|p z^WT3NPIb4I4-N8{Cn)Jy`xzPBI@(-qoRycp=TI-@{JMl#HAaEeOaMals8nI9uyn=q zOn!gSjX^H@_==7+l0IZ}9J9H7&K0jA-#(dD^r-f#-^1_=0C{nmytqJS8~J9MMEQIo zQKh_ft8v~5?hVZdHa&!*NV7v6l5|N6P%e8o1g!bJ;ITmn7u+kN zAwGIuZxrZ?ZCD}kXv+4r4y$`ewi-Qr-qN@0I-2`kksoUiZF-U)U9$TWGzLg_4w-~>A8l9aJnar zo#wR@T5;0Lb=hmG*p28a85=t^9*=&ZF2N?HHd@{3ilMhhBE!bo_K2e*{mo$P(@c!c%0!%z2&pcs$IMV`Q&+ zM2~yR5oEv~E?K0?nds+#Q7C4}mpQz95)Tc!XAJud73};SzKOPu7b27tnq8dEOEjgz zUJc|e##V1CG1?d!>Yp|aQE$K8wF;o%joRrWtVo$*MXA%7bzpZloW}fx9$m<)VPMGW`=54RrYPX zA0QuH0c!3yyd~1dOO!g=E?)>!Er?Xzcb-DDE}JRZeA&TSzLh92Wb@+V$ud*finXjs zw#0$@P^ND_ibhJ-Ayv_f|Gz${|DZfafxUgyybtM}TXAvr^DJA!IkmNg%&6pjrE z-YOVxSk33PBs06Sf5FLX6|qU)S^a!MVpU*7O1d$I(>^;bDFk9E#JrtDp%e8jiqdUY zF#7gMhDT*mxB^a{Y*|{8U^@9;bhP#O>%`;Yvu!S_TFbN6^Rw0|#!!>+uw*L!iWnKt zJ7Mu5EdRiL83K&d*f-p*0X~s-R?8gYf3B=@(3smkE+oltY9Tj2^u{KpA%antv|bP% zF8{W+y4Ut8Uejqu-Jb0JBD+P5rLc~r@Fm}CmckP$)LeU^f(T~_i#b`|`RVs!a5o6V z^d=>gRmK!VcG_Auyy9%wV*({{R~I%~?Z+HTH%jP~C&Yd43ai(Z#_yA*A88wYF{|Mn z2IW~ww|c?)Wk*cqL$XJcWV2j_WW9arcXTa^g5xyvpzq2q-F)gL<@{nHwL?2ImT9*o zfAX)!0e#1Su_#Jy#JhI~gM_Ve;}I|uQJ-yA+&)Gn9~eem+P(zS#RJ};L8*3L3y`-E zJxcjQ6sgimL()5Lf|dN{gv!L+hqWJ;zi($TEtj>0!^c+ewO5KV0vLmZ8k|;?3^m%S zfpW>lE|iDt@_e;hIR??N*cUO7o=7&+04t~MX&1%wddGcA3XY5Tqp}mBwAcC2rD2L; zLXV`aU}cR6K;fCkaGf?6r`o*l&rE`q{=t4@VU_p(V88vwt}TKJjS|J*ag$uUehhp3 zh$X1ZCy=-k#rYDO5Z`xyGe0p_CBGNIqUASQrjSKvw8jS3>aL&Uv{KGgqph)U!T;da ze&dEa<68RGy7^d8AVZudL)_a3f3DLBWCIuC@FJYvn@i=<$;@UxgB?0iT_=$9e5o8Y z%h2;-=%!GQniNJQQ}eEfll9~*@}u$(kYv8ZG8_ArScxN-HlPg~(0;?(KjcPm_HkoL ziD1IO3Y!Hzby+*}Qo&(GsZqRGzoYGiYQd$J_sEW*tsp>Ep&4%=6E>1T*@32`Suu|{ zB-HOH@IcL|Y=%NEQO-zRuk2Bi_)$*G#|nPzN1SQ>&Km^@oJ&+pAH!?2A4voFvWG_J z%8M^bO(33#Ry1r{mtc;AM!D0E}ZI8j3BYjm7C5Lu`vT z+BJq1F@^zfQwo1RnB$w?aavdtLa-bHd~Cy-SMfO?uzi=)`Q`4dV~*DA84!%{Ug;6% zCy7!~6om}(gXlBOkKW!a+<7eU_oL2A%C#F#u@MJdHP(;x+7e-|!|&_Kb(gB^&it*9 zcAV=u*x_6zd~(C{jvM)ng+O?UKH6qZ7U#zG{*8Z(mF5Ud=7Z!bl3dA-TM2f|Rv93u zaa7M&+cu|_)fCsw-V(FVcTSY5)>+!-aVYz2>XX&<^ZitD_5+IvLha5YuV*?&lDb_7 zF^bOq^(*~#*<^=3ofa*ZGw{`*wsFk>149t>E` z#_r%cJ7#drI-$IjMKR7Ta@CI8g*YsFh)4Vkk9(uZ{r)3&dh$*^ShNI&t|0r?L&oy7 zL*1!bWd5{X7HXb!qa{;c{yAq(5ij4pTFOqOW=+iZ!BZ_go|UQZ3_6mGNS3laSRs_XrzEZfI%Q@2nckPRaSTfT(-U$a3ctPIp8K%{Bf$wkq9hRqI3S za4aS`3ckb%7?JY7>Mx(zXmFzj381ds0*Zfi9D3;A56|#NkQ(6AcrzqI b~*vDHaON25Nict2DDKccplF`!25UdCa6aibH4wlL=fNTY9%bv0-vScWT z$Pf`Jdk7U!hRB{W1Z2J7d;fdy&E@jTU2>m%M{@UM(|pamtdkzXS2-F%?xmLm?C$1X z;CVde5NrXr#d9ejnPW)LD06jl@|}0ynsb7=dqA!p{p}5K0g{ou=Ef~Jr6t} zL^|&1DRZ6RqS7UYw|OAv9&D}}un$GM;lKB57HYW~inAA@J7(84kk^2y=f-}~Tz$`I z4aS*meeWqmCPlGIv5R>`L_AgunH$OTcBJfQ$aGw%*log#Y|YWsg>PK~yJbt>gzlwR zwDnY?>KC^UgPcN4qI^*PKsB4C;G|$b%dW<^E(r#zey_u?dJOtGR`Dx23l%1tPj>bA zT?wZ}G9gkL1j_QN)sp~)<-P&!ZzDpknmMdaJ)Cuf<{Kst??#qLYdwq?j9Cei_A!JT z?$|X4oOY6JQxI~wHd!ldDmd{(8hDz#5P3?^a!s?DVIV`4!vgr>4xuabf1;=7bC;93 zU|En3JWyvM43DIJl1hIm0D7z2Cn9L^mW2+yBP7Q0eC7BgQkOv2#0>;II)_`!&T4hn}-oOyU)@O^t_?_ z^1VZ5v@Les?XNrU<^yi}ywJaJ6kS=583(iV5w7qE=feyMNa@}aEW9);Jn~d<0|hSx z*xAuOOgF#D=y%0*fmx~YPP|n{2Q!7GwwSO2TTihP^oQ-yFLa9sr@@bn{Q%*{nnSao zSnF{t{0^b`f&W~38E?vSxk+P$%SYJs2+nI%$%}E_Rq=qeRBhcTzA!<)HI+99Thj&>-m<0&nY`k3lwye{<#7_EsTwca48d|f$j1c)Rno~4VS8J z6xn#yTRO1X4qmIc?r=o}S6_odaBxzVe=Vy~tUc~nBS&LnT6Dn{D0&DF2n1qfK04@v z=!4AeFI7Hb-hlv^n_9!2kM>?|_y4&SW=tn}jUb6$P@*BdO%CJqmzZ!T zBo`==fyLHzSt{KCmFAs;-f4t3X{w;j;?a=vQy7MBVtWZ*ncgbRXA`1aR8Za#P$QIV ztLs(jUrP4cD%ei!FK@beW`yXR)8ZqMQlPIK#n>ti1BAI%5flYcvN%YYmFtZf6TCHE zp3ui4VioR#;M2bB@@k5aI)niq+AAG1=?6C()zrzp{l1==WoPTP{EF}O_xehtIN$0c zJLfP`#bLr^ToC%Ja7w4BGLlZXQdr;JYc&`>)F+uCZG}>Hv55!=ry7QOxvs;*D1Loz zKvgsh)tS<{q|slbRh967OntB($9sn~k}1xuTB=AjWFsL}^dx&hM~{HFzKL!XX9?;E zfs}N9WKwoHiou{Y9xKQd?j?zg@uF-3AXi@1#MJX885;e4vA1VBQ#6*j!+hyRl8#)T z|3BA9rnk4h9J>bozwO09J?jV~aB>F};yBz5Lzi=}O`NkZOI3VTGRAo z+`+v(AOfe29~Xf+SA>Q|;DeA(xx_*6)n7lc86dKej*tXF?PX7&Yi@jPBFBJ;B?Kwj zWY-=oucd`J;I&Ii;IWBrBS^*F8N6)nuVL+{VV%R-Otcp|lAm%UA86z96{3L8M&&aW z$a!v*nFY0pn#Yk6)2wj#N=v!~#~&h=@BAdD1#i@07Dsm$vI8iYnXGT!=gZ?)k{@lw zKW0yuJ$*l~XzPdRdNjDzeTV(8AnNY z8ISein!X%R54;>t(F{}=s2Y%}yR9a#IHL>0lf7Q`s{TY2L3k^9&rGsSJ}G!i3^l!h2!vKGv-vs zEfH)P$>kMRqDaoZQG6QvrHgr#q)2EJaiDZnA9;@Ulqn^}Na>saV?^vp3z45<}B<2II zorEt(Xj?X7VB==g_K**n6r=^kU1_KvZ8nnA9_i3lG$+$sLYDtwP$>0Hr4fVw_3>=m7~2^`zPYdage_+sYJT6e1Z0ZQVr=@ zhef|YQ6StY4dAiLg?~7(1K7~VhUfY9mepKYwLNX)=|eZUg1q`G7AGp>D&n)4@-yfq zU?ykqe?(vosezG<*>c!FSXdD08PjyTXeD$3@w44tClx($ zekjNFqqZ|ZL?aBG4fC&=csxq<mK*2m8|9t0>iN87 zPyqT;U*)7(2vuCo59{|HLrfbSa2M_#IM|$ziGr@ZX}C8Czy12*tQltI0UBtj%;r3- z@$1kTw}-o+2H^$7Via<=>^f9J9c9bk2^kVOE~0O!r=R) z=U5o=*>g3o_>t{0{Lv-n;tofu?s^8L`RVfmSo>G$GUE zkfAb~I~mNNoBm$xVWkvrE*%hk<-CLyiyXa-yXp9SW37|2(vM8QC9DVLOmg;@h&r}> zCYDzlQim~a8khI&PJG9VU$IqvSJZfsK|%0Q#4Q;!w$fR5n_gPkR~qa__G#G-r!f2C zo~OKixTVC zqcVxf07-$aQ-R4#^!I_iJyRfNc{~3xPs?P>#`fpqHF~evbH#De@^Yxw+IlO|BXX~r zTeV%YC4WCWN|TZwoB4ORXEU|#DW$EMbMmCY+}4u@%H$mX`yj9Tj zbvUb*Y1k)@-rv$H@U{HMilmwFxYTgPA+fK{5luhaPYw7`fZLVa&O7gyP+7H#lR~{w zOG}qYvC9`?Yt@%IfBZq8gW97@lak2p!58{W)@D65?G%jF*Zmm1n}5wcsvl9`eLS{) zrhF?v^RXKDxkA5Y--~JQC_lD-MLDZOG;dxSuenf8JEyM0!fO{`_nstmcQZ5aF5|6$ zZTNpfMCBL$>&y(B&-#WW$P%J(!0nR~htbyliQGDQU7SGZlAc|K+l!v~#KlmR`-UY< zBwH!K;d6NgX>ePt%DwM%R+u3pG$P zbW7<1sdTnP1PE^!e7&+?SZ?vs>YSbrl^=VmfL5jO+lP~#c4dY((X0jDo8#L^sm2n$ z;H%OQ;hA^2YuB|KJTT$~UHyAHW=}5w-RN&vo1p$Gic|oA#IqknKD43hvEvXMadO@W$ s!HQ(Gl%&AF;K3k}1sw>)`QMF4Yfvx;IYvqc>`d;E5@S3iaeV220Ny&`V*mgE diff --git a/summary_results.json b/summary_results.json index 6a49e391c..5491fca08 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0979, + "Performance_Result": 10.0828, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 4.26277, + "Performance_Result": 7.66684, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72562, + "Performance_Result": 1.70101, "Performance_Units": "Samples/s" }, { From 55230be3e2669b13835ba92273f953c7d0083e9e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Mar 2025 21:41:59 +0000 Subject: [PATCH 340/719] Updated results summary [skip ci] --- README.md | 38 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 12 ++++++------ summary.json | 18 +++++++++--------- summary.xlsx | Bin 10229 -> 10227 bytes summary_results.json | 18 +++++++++--------- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 9e7a79880..6c11be8c4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 18:48:13,174 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 18:48:13,174 submission_checker1.py:3260 INFO] --- -[2025-03-27 18:48:13,174 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 21:41:54,315 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 21:41:54,315 submission_checker1.py:3260 INFO] --- +[2025-03-27 21:41:54,315 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 18:48:13,174 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 21:41:54,315 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 18:48:13,174 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 21:41:54,315 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 18:48:13,174 submission_checker1.py:3276 INFO] --- -[2025-03-27 18:48:13,174 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 18:48:13,174 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 18:48:13,174 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 18:48:13,174 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 18:48:13,174 submission_checker1.py:3297 INFO] --- -[2025-03-27 18:48:13,174 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 18:48:14,538 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 21:41:54,315 submission_checker1.py:3276 INFO] --- +[2025-03-27 21:41:54,315 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 21:41:54,315 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 21:41:54,315 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 21:41:54,315 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 21:41:54,315 submission_checker1.py:3297 INFO] --- +[2025-03-27 21:41:54,315 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 21:41:55,278 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,22 +24,22 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70101 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72668 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420191 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.66684 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.426851 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0828 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.266062 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1737 | acc: 76.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438504 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.440342 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ccb8fd57a..63e305aba 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -967 +968 diff --git a/summary.csv b/summary.csv index e513c990e..f0ca0bd93 100644 --- a/summary.csv +++ b/summary.csv @@ -2,22 +2,22 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70101,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72668,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420191,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.66684,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.426851,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0828,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.266062,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1737,acc: 76.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438504,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.440342,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f41bc5f9e..759d3ff0d 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70101, + "Result":1.72668, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -304,7 +304,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.426851, + "Result":0.266062, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0828, - "Accuracy":"acc: 75.400", + "Result":10.1737, + "Accuracy":"acc: 76.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -514,7 +514,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -544,12 +544,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438504, + "Result":0.440342, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b724b90c9b810d6762476834e75fef93daa8e124..0ce97199a2701aacd186ed172a436bb0732f9920 100644 GIT binary patch delta 4249 zcmZ9QXHZjJyM_}8y@deM3?+nKMVfRZl+cSH&4zRhNR?{nBp@XOC3K{zNDm-ILYlbL6@N!vx{1~MO+R>+2 zh5gCWohS=_2TzC$k~TU0X&I@FBt7KaV9p9;8wI#MoET{5=S?^B56|Y_HZF}oeh6P` zUKzO@c61;-@>KL=uu&3@RF9ev&@(Kqw@V&u|Nc^XNuh~0QLNWPNB^7T%D3sgJDu+R z@AqTD_Y5hi__MI!^$&Hygf#VKG|D0Ux+Ye9fKt$BrvIz3kj@SKuR%Z59PyWc^kDbL z$Wts6PmiZzil^Or2{@TmzfQY%HzfV}{AHHn8=BU~Tu-}%_H(Wh)c=$z@=aBpBm1tR zAEPjLkG16?^pHLZLVW*2Bm(BS&)x?*T`yv@U@(7TP?axQ%31oPG5 z(NL@SX$*~FRy%J^$y?=2w9L=(&+1?1_}o;B7@S90+fZFj2Ejg8`7E~?LYFt7!B;Q@%EJS;|AJJ5%f z_SH`)1@Lp5fdD^dx>GdwB`=^sYg`a&(LhH5JmwPur|n$aIE5DJ{;&v8`;i$Trg~AHM^Q{JZysgcd%cS2UyBFl2}LJQbI-H` zSWczKX3VA!nYE54kDE5aZv__WsE|Kh&8Qie_68YOPR0e&g2J;#{zA)je+bMaL4OA?OdZ2}oSrF7^Lt>vFyBDX-u&ss{w0;-QevLGHnvwjeBgfd?woJz0k8ao(NfAf1Z08Ql(NH)yaAJhQz+j$YaHPe8LPr zYz=dMmCi%Q`VrC|_ILW+B9L)~$KPZGbr6IWeCr&+sEe96hljvm7=R6a>p)!F(grFVf2SeF= zsCjdJd*}NKKM!NJ5YtZ=VP+pmK5ulr^6qk!=LS~De-@domK+GE&x_z+675hetKA5^ z_ECAEBrE_NxAfFwMpu2uZRzRkw5o=5Tg8F#RBm!*t+z_MkKDLw9r1D4q`Jqv8+JO8 zf!j}DO6% z<1^=CK}H!#^jC(H3^+1VC93#roH)gUBpQNCFLgbz6qg9MBo!wP7;xNtm3GI=G9}Ll ztbbxPfx<=j^mD^ehXCLQmIxdPwpSGO}>8Uf#1V2YxWVCyL-96Up2}Q z2cxfraUCuH=-hGl8JXi(dnucl^g`tp6H`y0$6muC!^>n6{Pb~lRls<`Hywp0?h$I8 zjH>DA95@D_6TBCwnnEl!5BGmCJ){qz>hQyT?wztqHs8hvhO>8(AR^us)AC_CgUW5k z;nm7_w&I90x9SPPBxf(5$`(=_u_M^-1YJ7ClKwLoGJPAWK<(0UW~GND6)74fH2#%! zcjA2G`}#Aa*xS?kIMMpM--c6(nbU)6%dd%&ryyZ%D9&&LgRnAyf2EJuxnyDR@ssQh zxmIb$L%kzXP?L%=it%mN;c?$H!*s5(b@vQ0Ymec1dfep{@0PZ$CvLKB5%#6CXx=jI z;daq`le`P_8!gt#hoKCml@KvoOCvh=>UnAQoZAbB;W_vpz7P$#rIR-K<@AGQ78Gy@ z7LGH7V-YaqZ+b)cN_toH?xlE2>}?>Hiy0o8j(`Y>Opb_ImW}3yoby56h?Rn{(~H9; zU+U{0tV`)dq!gu!(PM5GccD4Jd4(VgG7p9`WVig9l(Kk2C6@juWWpM}_=`&HW!(!? z)ih&E?uiaA*Zu~vvEH!BtL+Z+Wec~tc1&9a7$4{F6Q@gUGHz0I_@fzg+N-idAIQuJ zT4Y`BaBaP7QMi}w;ndMhHwK6^5xU~S(q*v4B3+)=z1j@Wx4CdIRQl` z*G_co5@P$@lMC# zgzn|&xJaIuwdQV24Ktbv&nC>)VbW~V;lJkhAbaq`q}wfxs6h!YsjGLYXv(HWayRog z*XA`O-meE#JmTyLf@i(*y@eFhB@6X=xuO+`J1Kl<$@zeKGH!u0K$0$ z*c6*GP?vB>2q24!&LJm;#ijqb5($yzkS^Y-s=zo3hvi<)ITus-jJGA)Oxk%3KlC2S zy1h_a`7X!geGa>*yWp?#5x!LnG$7P;nE zaF}O}nzfBXkd*}Ior%6jwcS<%S@UL@`>sls675w5fc2|^JGpOACn@EvBoorQT=V^< zEBpk+euB`c8e$bm$ucH@7nV{tp)`T;Cf75+f^o8Jg3tu1UwdMn#A|WSm3T|%8)9EE zV_}96)q z&=#(p1U%DA;hUnfuwNrtj{Eu3OQV#G3LzD0aQd6oUGHMvOeZd=p*1o_CMWz~1et0) z{{_9ea}8Zlpey`G@3QBnAdmT9{;JUfN`U1WtjkEBPcyC%|zHyXcK#X7I*xl zyoiAg+mM(##(1Kt7Tq@WmybpP(qI^!Kk=j`?g` zMMW}%m8+=dcQHPI|tBa^%SFWxd~kt#g1hSYmf8~sKR`xqhU1+M2` z+~xn`*8YR*Nu}=j7q=`K`HDG-LVkejNx-N0f}Y;Lcao`|P2QyMzeP+fm0JJ)*Am<4 zUx--62n}f>R*i{dT!!$VrU@n5B7LuS8*~x^j{}Xl@g~cLKSAj6H?Fu%5y55cAj*Sa5Sa}o@ZWrOf1@5cR)L!$i`-5I%BWYoKzkB^28C2j!PfPesw1hIPHP5!+Zo9Illm8Fr@SoK!4wi+!jnkCA zU?qjGS~q=C0_v0vvhC{@m&32nJzYCdSE-JVVWpRmk` z&%7Jj|ItoNWFr8n3|K69f{CIW#T1mPE$+l2&T%0BKG>sQcez=mcKV94CLM=kxMNGY zsLw%ugb(%o@FRX=JNG?WG{4q&dJ!N1U_k)@u#%ff7oYnUem;-SMtAe!T4lRO`t1& NP*jMTQ~2Wf{{W=4&0_!n delta 4270 zcmZXYcQl+^+s0=GL2!)G5+zZ_5IqP&^uZ_*Ey_fc#01eIh@(DQj3MgiMjItWq7Dv1 zBp6+yCW8Tl?%5d07o z0+{reQ(_BNvvbf6b}kOH2~8BRmn%LT=EfNL<{G1Tq7gXR>XE*ggdBC+dcVxawxT(l zC_>$VgzZ86iJjeW}665?$y%%w#I6&Q7#XZ(iXGI zt~%+s7gz>$T!2&JNz_p7sv|D3IThJ-d`$h z&bUXMTD@*ZssZ`*Ow>#mFe6bWTlPRp;AwDl%WW6Xa(rCZ@@K?9(<HMc2Rln8`9Ry&`hl8 z_mQeGg}jtm_BpU=2{%>A-of`_;Qf82ELzvecBC#zgHefC8<(*{cgR%Dj$ufsDc*L6 z7RUbx7dea0*wA#-rjQHWT!UzlWrDY$i&t69c)g-^DH?^R<3q0Ww#O_G@ctEkXyV#+ z2)Oo;IxMuuH%7f}O-YMyLwV(FfhU(m-Y<~Kd0hP(m-FjHcA(>Ob^5kxI8z!$R zy0eC$K$Vu|A^?~H6sqoW0>_Z}pa8ntC~A;;@d3HdpUZoUZHr>|(j}9aPkO9oIEKmi zJUat7h!3t@Vthu1qH}J?`{wm4MY?E}>+S#8v(k;HVOh;Usek7JW!y z_2AhXprcG}s0j*FKAK!c?;jf3;FTnsrxUZ^R5-c64!8#pSLcYUD@3Z1EfkjYO*BD0 z{|A(gjTaI|)Z#Pt`N~Vw`eE#&{3HFoh?1i8IIlA?g20_aRnw~dI(%o^_Y6xsc|lO+ zYr<}%{Sa?nZ^Ul8`Ch3|x?{=f42}0eF^H1{sUdf<>iH{BI&)qKSa~QZb|@PNh68uW?%9Dj7SVS{m$=93?EQU(viEPYh8M3k3+-#6OO=m(xvwL}6<)zTyL8&^j18!$*+3u= z1Ks(g21@N;WZb~})7@j_89@j7c@}*9YFhaN|Bdbycm^L!FA1o{Ei0;|H$5_J_QTda z8aR^Q5H1w2zRq=cqVbIAs^iHbJiFI0#$RVtb@A??l11iY*7eV+7^R89)`U&GJ@1Is zeiP4Zw#TrFQuGH;9nuwJ4CLIJz9g>WYpM+ku0gC30Rsdn z*BW~I+%+LWz1q(&Ep-;g^y#u&CWuzVvMHyc-mOiHJ)~ zzzh^udMJ?;9WYbS!MXJJua?MDjZ^gi=;!XP@U8J*)hF~T=JKSTBP4B}qap3Xg%`V* zg?ibd0fGu+(38E{!`g6yd&udz4?5E?34UAp+qcaufZS6O zx|X=Rsds5#N1>rU&0Kj?jHVk-Ws_{1(AwS>>fEr_MiuV$9Teijr&X-VG^qfY(sNI*HW4n}PRCFFp+; zazxAqL!jgQ4m;NKPHAXWn9`)^H-ElkY0gKA2NCwUwuvS@Q&X>+(v7fcj2I#IwEUf? zDpl-h9$&q9KK8y(R&!UgIP&>gFO*j5#fua2*JrkZG1=?Mg$g~k51DcnNrQv!trPQ$ z7eYQ zrJH^UF`hSh#ilx|dG!#FZ7h>7fm*~sqnhU#tiweB%mdSeZ2$gei;%^}C zLN}5rnIbNLhl6AoU?h3ktKBIQ?!!2lI=ESsuk)pRJ*1+O{F#aKE8#bf%1f(936!Z^ z*_Nh<_xu$ygA~rJW)Y3+dNL;SKa&kd`7)7IjxeS~J^6z$!}4guWKg#_>eB>I5T9B7 zc^H65!^`JES9D4qRdhn1(b`*>JS@98GLb1OvHs_?X+q|_s)@63Rq(mETW0^ubpo*V z|7^ES(;|JX*g|_w)j=h{UcYBk=2shZJTHz1!yJa_v^<4--!J*F1*q{Xf8yu5gZ7EO z0Az|g3_EbyCx5Xok}_mJ_OTKkE;UQZFK&B9^^fOln10pjC+Ym;=%V)x;(Ai^ez|sBVCNG#UTs3QJSO1_VfsR+M#_w&Im7fuA?ciFO@H z^;ga%R-Q}jg$B6>|-@%}-W()1YMivx@W(KvH8dvwxR-Yc{ zhC8k7HJVRag$Rx>$``E$A;^woB@*?_=-o_@t#WqSxU6GN%4xqn}C*F?%oNMiD>;{vt2F}QN7w7 z*dG@~)Kdb>#h$8WM=(2w&tQGR1OHLkhLAmNdu?4kyQE7A8s5MNX{Tk~Xy8?AvbL;9 z-+ia+Bl|CQ>zpcg`VV#MH+6dzl%*YYb(}um+VN<}{wOR>emOFqKI$qAh^j^Gp8RIc zEN$}dW^#o{?$yrnq4?sGhRc#Vj?AK)6&y`Ye{_|o zA!3jboB>?OhtG!{*L1I)SB^+*zbaS1>OZ&PNV0kOBsL11*1thzK#5sbO>oF$lo1%i zU+J+hy;sJ*R{vSJ#rwcpLAnmF1@|9`C;f!RNuB3B0-09zQ<@2?N2Wq@SO;4Pt2-u* z=|+svq9=-2F%!wCs-K^v`I4=JL%Ww8_kkYaVE~r&aqLH7&V7Neh_^gN)%)aj7bsH% z-d+Y5XPc2*ad)A_cb}`%+Lp7C5dWT zelj2#o-P$2Mfk<3vNR?Y6@ua6a#=ncCN*KCC{k`#y0p6+(nq6lQS;VrhT$4$=Kzn# zI-~%G#hpVqK?*Q9D+y-8!wJ&>7tPlYC?;=sGF;;l;_zyXS&e`8(hvZy#j$39X|^;j z5Bt|5*b5<46HS=fe$HVqp9dY=a}nmSrMV8OC(s?tlqfkM0blPThUt77v`J``ecGtljrd8k8fn4ao6M%2pYdID zU39P7mh8=T^A7vVI1A%~KX}lienPU5fD41C1$UKgXeW6`eqeAZs z!v7kxFukhZ+#Xfq*+fNnxlEPz?b^%Sxht2llcJ4KgE~dW)dQe!j9OH%6hB1+nV{`} zs^Q!6jJ>I57iTVKka9bRf}cE)t9Q>Pb=or>%f7X4X=N@aYwo{%5W12q`$62e(j9ew zZdHJ%qF}J{1FivbWX8FNYZwveQvGH^rylS1vDaz)w8oaGbZ7TOQDW}YD`%j`&8UWA zskT#c21$9ze}>2fcTF|q<^NslDEJ)m?+d}YkS5K)y}2CX9ykV{h!EiUCqdL8kO(^n z^t*-spAj0y#*UX_=fGnSP<-EID8qj@-r<>rmB82W`oakA|C^=%o`T_Hg;l{$_+DYb K3zvm{fBz3iO7T Date: Thu, 27 Mar 2025 21:43:18 +0000 Subject: [PATCH 341/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10227 -> 10226 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6c11be8c4..9d4b4aa19 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 21:41:54,315 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 21:41:54,315 submission_checker1.py:3260 INFO] --- -[2025-03-27 21:41:54,315 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 21:43:13,894 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 21:43:13,894 submission_checker1.py:3260 INFO] --- +[2025-03-27 21:43:13,894 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 21:41:54,315 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 21:43:13,894 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 21:41:54,315 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 21:43:13,894 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 21:41:54,315 submission_checker1.py:3276 INFO] --- -[2025-03-27 21:41:54,315 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 21:41:54,315 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 21:41:54,315 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 21:41:54,315 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 21:41:54,315 submission_checker1.py:3297 INFO] --- -[2025-03-27 21:41:54,315 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 21:41:55,278 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 21:43:13,894 submission_checker1.py:3276 INFO] --- +[2025-03-27 21:43:13,894 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 21:43:13,894 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 21:43:13,894 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 21:43:13,894 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 21:43:13,894 submission_checker1.py:3297 INFO] --- +[2025-03-27 21:43:13,894 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 21:43:14,967 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.440342 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.433713 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 63e305aba..6dee81878 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -968 +969 diff --git a/summary.csv b/summary.csv index f0ca0bd93..ab6a512a2 100644 --- a/summary.csv +++ b/summary.csv @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.440342,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.433713,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 759d3ff0d..e81a55d15 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.440342, + "Result":0.433713, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0ce97199a2701aacd186ed172a436bb0732f9920..a839d04b9d6bfa35d110e761671aa58a3978e3f1 100644 GIT binary patch delta 855 zcmV-d1E~D-Px4Q&bO;3O#{ZMEcnCuYf7z_rbXDGh*2+N>%_EL<{1A;N9X~{aXqIsN zFb$wdG=OH2aTR8XP~+(k7jNuK=PtEDFdhWBAb(8-5-m^M5rKR8n6lD0j_1 zma>$e^G7Ax&Vdq*BaSrt5REU*K1731mN5G;4M0i8BezR*k=ciJ0#Fv2eKeSTf1ssJ zI#H%2sRXwV4FfEh-3OMk72ACt!!j9UM5-q}lC7k+QY+YPuTOgC?B{M^YA1(C@}?bW z^&uKxT78HHi7a9DVHzNkj0cD;vih)2fXE`Nj|Qs`w6sYl%CsesVD+J4KqRyJ0Flqh z>XT2T|0E)xtI*_U-=W73Y^~S^p zdD>q%>yNWnJai|k_d6!z+0cmpT)jEc$=)1kX32Vk^Ti}ElJE22uom3S({Cs%wr2G2 z2CH{Q8uMrzM&soBs`u=rm5WLDXd3okE(f)kjsIK(FJJUtIQuVp&Z}PxQT@s|G}6-V zrD=7`Ua}w9u#OJq-C^*l8x6;^*q8)^w9xe2)Qo8L7Ru$%NjP8rl>Scp8YW3Nz5g@{ zy8R%^KBcz>gD_0)KV+w5^ES_;{{pj#6O|5s1DF&ZB>?~cPyzq|5dZ)HWN%|oa&K^R zFJo_VWiEJaY@L$JYQr!PMfd#*qrELZ(g(tJpeeG1Y$r|v zS#>$Och1}yR4(_jjefy8p;d*lBt;P@rnyi}h2A%-_yI-EQ^jek6;!APhnDqC$qZqC z+QO^V2CNt0q5x8kFjJwO_l98X*bXFhNpPqjZnc%vhtf9KQ1(q5zM0(8 zv6+eJfaGjOnjhPa0M0P9AVGPDvn0bOuLmpL)rL_yb4uZR16Qp_adz*Wn2lZ6C0#LE zL;ehZu3z59Rb!!sQ!zkw$r)i5sMoegF7bI04%brW*Wtul0sOHK`jxlCB2ypmC;HcpE;zu)8;SrjB&d<`Qn`cFW481ir85isSx`~>wMvt=X70tD>F h|C6&bCOrWK1DF&ZC6je0Q~_j@$|oZR79{`x000s^kJSJG delta 856 zcmV-e1E>7*PxDW(bO;1o;qRlfcnCuYf9b4gnaw=MkG68qMDvIv9X~|lNyiV-Aeto{ zKTHE?5)GhP!Dj|RsNw6sYl;46Tr4vPXZ%NTxi#fG29$oyZ81(noS3d&ux zkEJZ-=loHLwsW9FyAZ6Ckq4>Z8Hx11)XRi85_TBv^fD7!b*iv$%cs4ZRKUZ&#bh0-`npv{m;CwL&jO6<~IIIPC^Yk0aime&_ zyTR(6k;Xh4htW9szUn=DY2{+lJ(`C7m&-vdX5&8>!OIuD7ta2Rp7ZJ#QA58n4vn<* zdudwTvX|@!HmswAd3P9m>PExyEH);=AT2aKH#H+#y@hi5a}v&%Kc&CZzJ^H>PVYaB zf^I*EvQO!4!5|Ei`w!VE*}Tp3=)bdx6O|5sA3x^`CIJ8dPyzq|5dZ)HWN%|oa&K^R zFJo_VWiEJaY@L$LPQx$|gzr2>IZ2?(HDgo+=N=Ot@akK1gY7zg)Hl%rb+@wug zDJPEWnf-Ql#g8a*o+?fot)N65IJB(p3T6m@ z(-z*WHekH~7X^@VgqafUy*C77$MztpOM*iMai^`MK9sh`hO%#31J2X*9!v0)Q%~^` z7@LuZPDsv1r1`N;58w<#0}_;XI7>2o@p`b*O*Zt(l~W4e8JM)5#nF9mVl=jGo3yju z8uDj&yMBG|SB-@_o{9mg3eE_#K)tp}Re`UIaJYs#zYZtf3E)p%&?nwbiwu3hqY!`$ zanH)3MP|uCsYAtlgTF|1{Wm&0002bewm2? diff --git a/summary_results.json b/summary_results.json index a95595ad5..927e0c816 100644 --- a/summary_results.json +++ b/summary_results.json @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.440342, + "Performance_Result": 0.433713, "Performance_Units": "Samples/s" }, { From 55a57cc4663ea23e8dafbfcbfebc62da63d32a0c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Mar 2025 21:49:43 +0000 Subject: [PATCH 342/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10226 -> 10225 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 9d4b4aa19..dc444cfff 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 21:43:13,894 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 21:43:13,894 submission_checker1.py:3260 INFO] --- -[2025-03-27 21:43:13,894 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 21:49:37,687 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 21:49:37,687 submission_checker1.py:3260 INFO] --- +[2025-03-27 21:49:37,687 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 21:43:13,894 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 21:49:37,687 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 21:43:13,894 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 21:49:37,687 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 21:43:13,894 submission_checker1.py:3276 INFO] --- -[2025-03-27 21:43:13,894 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 21:43:13,894 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 21:43:13,894 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 21:43:13,894 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 21:43:13,894 submission_checker1.py:3297 INFO] --- -[2025-03-27 21:43:13,894 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 21:43:14,967 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 21:49:37,687 submission_checker1.py:3276 INFO] --- +[2025-03-27 21:49:37,687 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 21:49:37,687 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 21:49:37,687 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 21:49:37,687 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 21:49:37,687 submission_checker1.py:3297 INFO] --- +[2025-03-27 21:49:37,687 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 21:49:38,698 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,15 +25,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72668 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420191 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00489 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419826 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26123 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.66684 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.266062 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1737 | acc: 76.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1836 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6dee81878..e59798702 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -969 +970 diff --git a/summary.csv b/summary.csv index ab6a512a2..85853b22c 100644 --- a/summary.csv +++ b/summary.csv @@ -3,15 +3,15 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72668,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420191,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00489,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419826,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26123,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.66684,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.266062,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1737,acc: 76.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1836,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e81a55d15..6486d333a 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420191, + "Result":0.419826, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.00489, + "Result":6.26123, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1737, - "Accuracy":"acc: 76.800", + "Result":10.1836, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index a839d04b9d6bfa35d110e761671aa58a3978e3f1..eee6c88d2ae5ad5935b03ee1893816879a42736d 100644 GIT binary patch delta 3636 zcmZWscQD-B`(7}4cbk7Y`@M!F!Bw};@Z+L4nzGNI^VQ7q84;?AgrG#YU(qvi(jwvm=Fl= z6UwU6UY*~YNb};hp$uj#k`#sPrF!pe9#t*MNwn`pf$eoEl&)vuz}1ZBLCT<|b| zVWWn#%B0f#nkMo5nlNoaR!VW2+w99!DtSeot60pgv3vEc59Luw(aqq@+6YWyfn zNG^Cwq5Qoa$ziY0ipRM$sc{w2u-BvJu53H&`*h>rLkW859e0&~RBr@L50RkC|8+VmEsM)Q6lntS)h2)0OvH|!ku-UhLk(*xbW?C+TXx7Jg*W?HYG zRUAap`8P6D@J|+l;At=eUKFk$ivZjmfS7Bu$`aSis9aSYnyK<7&`Hc>*3J>k?Q#yU>Y+%@`yKg1neHG z?FC{=8DC-D;Q^!Yuu{Y=nMpoB``468iU?{+Xk*;HX^!K(x6Q6rD+R9oOfHT~-G^!% zc8BdWKjp?xDaeM;1IhhYt@CX7_ZW&&ML&3yBuEA`tUk|+x zZsh5hRk&-+^0_c1;@9C5bGB54|S4E!{iNd2pb;KnCXu_MvD`V^FGAP|V=a@_{iFL)$x0>Lyk80V?4 zbN9ddHh#LwcNU_U%h{;GmP{ns8%_>oF%O2=V^nzfrz)lAw}Nj%jGAko1+olB5@!wt zD$hgq^RQm_`#LIgkzKm1gG2Pxx_9YzLX&cbrZhSf-)IV4uK)SbuX&*rk;dkR3(YMZ{#CZ^1Uoi0{c0BQ+rj zFizg7q{)kZubHx&NsMV>50kN8zoy(E5-V<;7x>oW>D%R*nRR3{fgkO2w6t$?`dtH- zRnC43ax&?lLZ*L67F{!1T50d`%i-6zl`ENRAzIfuh4LH%CTY{0iKpin0p-z~g^h85 z_sQ%Aq~55S?%puMJkynR>vO6jVrIB4c?YoLo3K1+=3UJ899Koeb$V&n{SbT^SnH*| z<(A8z_F(En*loCH%2F|BUXjydG(gS=;5-9Zlbq!eCLe81nWNKIyR>Y>2wp zo_=+kbJ)b5^e6+l^g|CLjq93AwPD6JN;HYEU?3#1#{Q2Un&iR*TG5c~v5aTqD z5cA({!YpnUyGgo~#ydFV&;1D9F0OsNZr3X>X57(*kx4=Kn1>N7?pAYN9FljA5g%80 znyK40rV3$s;@1ww5sh!f=6N$ht5;}aSu9?9!(8G+=E1nqLn(Y*$o#9WYrPletx;zh zXX+u~wZ7iSpHt_Lf$LXqV=VRrxW6Pk#tP_tcIz-?>FPK9aD?FNC5?-=HPvrAL2!|F zO?|Ftx3ZqszxxqtbW47;tcp`^nXQJa*{QCn zO*g4gfhst+Dn@P!SzAn5L6P4WchQ~u7wK}9&nMr1a>bc6rtGYk$gcTp*c2J%xK)`f z#mk*j3aj(sbvNRWraI6i`mi02yE>iS*LRYs7NzJ9wT{*U-rv##!1TOSAx`ktP~ZkV zoGb^hr`N_9#H;Cy>9A2DFLCJNjP8=dSa3+ZE;JtYOG{_xoSY5y{=;`M8n0vWl)p%$ zB7A+tr3veH39>8JDbC9`H!>eec}{r36Ane@9AlJ6j!B#2;sX*93r+3O`424&TW_Xs zflCdN7HVu|V532X@|AYqqMXAuOYoesiOVmXLly9A=h{SFG%YkH+GY?@_3k3TeR zSsHlGAV~HVs9=CnALeYn++4N+rWs(ESSIv9n!MZFJl3F1Ii%csVk|&76YVe#pbM__L%9V2=Wv5h3#JWLND&kW%>i}x zPzx3KtDz1hI`D~Lz%}*CK3s%(F^6=f^Q!D+T~&MD*T_1oS#R4>+<@8@_)Lqm{EGm}zIN~7Nn_oD3x~Ln=I2JTsa7VrmO$#ti zfPk(mkTClR@S{>zn`AU zW*I-=A)4$tWoW!+vc7T`rw!H#36ARlocIeg#Pz3xGp>|E)420%`o@SD1ey4A6;(ZJ zPor1GL)x@vk^ZsB#f{XAK<@+VcTd-pAs zhfHGvslqv`1{7v2i4tX!vWL}_>W=tUorrd7>zN|fM`EC4CfYmtWOB#h#rcoYnI0YQ zsZx!6TK6q$s8BPqRd(IPP>Zy*r^+d%bK63a>Q2Ckq7`aL3Y$y+6I`I_29TytTISc& z)P1Z{@{2n*D5N~bX}4`OEn{Q)j|n(C=C=~RPgJg~8k1Vei~3}*TD5u{jx}#vJ>bt0 zr#6&Nx;4IiNB(8qDVU+as&kttAAV_tEUkd+X`oTbnp5ZMQ+2V+52ZOu*}_DNO_kJV zkg&3SH@mhYxR*H-CcWPgLKB;LDtYDP-|jQ^)l%>tuz+^`tvpFVKMLXM|;3 z$%lT!&k8s9{U9R9V9y8DxChln$3=2s69lu!*-QVV4Q^BDlj673lI=6;WrZdI&wg{$ z`(^Au8a|7*`uz4)l&J^qK!eBskOsdb4U0^xK9OMnqk1Ti99ChLqUvb;qz(o<`9})J z8G>&0$*7C~!741gZw-CWFFKB4!H-OSFU++SniEmsEqZs*MRWl(U;#T>T;hWa{>ppH zCH@DTeZKXW){sX_Sd}e!&Ot8;!ru%`?(qS?+?~5Hl`nbM(Z)&gwdmH4<3}u<{fEKc zmRGk|*A-LP@5`M8Vlk6)ki=kl#=Bo)9&08d#V=%T(~gHpEJXhPZrT7@(_Qyr*&k|2OZ{CbFiGQ7KA#>jc+|uose3 z%rraH4qh846Y!*Ai-s{ptSxmt=>2j8HFN*vv99;>!{sAexe|UYVrpBt>}jad&XBz# z07OUvbD|r)rGCH0HGecEWjjh9g{+_^SodNcl8kaFYc_oRc5<5JN^^D`S4nxQE zJ3ieF!oppRFU}HWOvQ7L?vNDc$cHEP`SpLMVE!_G!jv$CL{dW^vyPulJAu4oUXjrb zDCY=ssRgTAu2(A14`lvDH_!FnRW%-Xy3>VrpH*;q-od^5_{jJ8M6I0r>SSI#*&F>~ znp*YN27lbhbMN;th<19>uQUtBwxd-a_&B&u!*v8fT;KRp{ePX+Smz^*_gZdAEuGn? zKIU5fjc2Q^dbX_f4=s%8&V`{0dC!;hy!;D_S~Mhzoa)a&25bpy@capubRZD383@FA z88!c_@CZi$C89+Z10;(G@&1wIa)5)egFyd=(*G(v0?ZD;**O5T2pIUt1-|-!jTV4W s6hS5hphQJ@{^&vm0^KFQyy(9pmwR{wBM>jDM&4%zYXPx4Q&%>xP~sxKdL1ONbf5|e!h8I$k>Cx6fYW^B{TzaJZtPIG*k8SP7@ zeM!K4-?`z>9DDfnCC`i}!gxxHk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX z@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8uxLOzKp%L5^A*~;Q2BSBP?^2~B=+q3c@ zEzHKkfPejo4I0O3NRDY(<)jde5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASBa2D zdWsU#jMu+Lp32g?rjZ9PBeNqtq>)IJVIxVSh%*$m+pyw-=8w%IXKkRNI^LS=*v=dK zuN2eB>154kgv+lvw^qp_2vu9r zl3he&suh-1t9%udg~{T{ZF_CIwf_1qLR!S>%KTzRL&&Wykb3}$# z|9ueGtciciX+Fs9PT#lRt$JIc>*J#9>!JhG@JNSr;3|(mpy&_?6deMAqC+51bp0Ko zL+DU+JLs2vC^|SW)g5S}gPa?k1G2tFtb+31_5X*W`;46aKy(lNEz#Ld*Y^%T_|*Bh z>U>>wpg}m&Asw*LP7nxG9Rh)>Lm*If2n2kY&-{NKszc}?+lw9aOCPEZ4or0iUDzo> z&W+9i3&|~F6}CnG=c=w+R)wfqawDxwy!hV70_^@50e(h$e;~lQ|4x98-FEsNxB;`? z?WJ?VR%Al5j2j`91#6gZqs@84%o)s^z4$M<+RPVGs#dljYfcVCTC5EIqdB7R29rnE zR>m#DcvfY^5U-`&(vz1`z9W^ZX+$q^a~ffkETt?JuZ?$gI#Ol9IH%Em9aL4MlMe_V zN|wQjTnB8G7Tm~4EEBcca;;*``9^&$Xjxy$W_?M8p!w!5Awfi#x|6%xm{PI1Q6+gh zHX8o~P)h>@6aWAK2mk;8KLG5;|C8;LcnDm7(^`9XZ=uiKQZMV^*SqP&xDBFs9L^4F zs&3YdVAc!!?BlOr5| z4930SP1svZgITgl5(SfPk`6x}jpuRg=%7DNk7e~3Q7|~H9jpD5-?UR}933qCz8(kP z;;o;Jtdy@|_(%5PWB;&bX3_>|-z|r678(6u&|OTD^YGh;U_2Zp>1ngGkX|^6m;V~m zae6Lw&6sxY4r|St@on5sMpqOSQ*M)&+k^y4l#2>6&WlGCmyas09#vcyDRwHgc%>G^Yf&jGd{OD6 zipxh8SE!I*YEh1zN^M%{v5nuUxQ1U7iTK`6h(w&QH$)(iQwr2 zk%%kkBaw(x-y#z6{hgvji?iEKCK7RWKM{#IyB#7CXZJIah_ib}B;xFTArgmQa1OuV z97tfx3l1f&yx>sM6!pF#5>fA4A`$hzLy0fW{rh_f`Gxp_^dPR0j|Jha`(u+H3qOCK z=0P;L8P5hml&;r;nnZ>_rNg|oqUSo@{c50dV964zcGGzMh4{!&Fr z>7q4AhqnALXIvMnajDf3qi$(a`HC}B=W^;?q)uFGuTkQPEByo|Kpl}t>WD;AM*c1W-q;clP?S}0bG+`3`Tz}MSYHN{C3qilGntE$4+aXYy9KmI~U{Ij(nrK|X z#tG2?g-jzcWE#mK(fBqZ&-;yrk;oEA6pIqHHU~E_LQ9);qD(paL0E@`$!Cq9S!(%6 z=xnQi=YizIT$th54}y7mL6cey9ufYu{aE4Cb=#I$qQ-S3mgFgui47qEK9j2rBnQu& z>z?c6+>_P~D1X5RGG$AfbVA6KVI2|%Pg7ZHxj}NaRlvU&%qnUu^MG|du+7b+-L##Y zC#;I^jvg!AbhKSbCfS(uB$=o|GKmJrBpM(SHMo>*7LiF30GR~|TAPCpWXhH{=|q`Y zk}1PFGz`!rBAhdu|0p!olUB1;)L3fHY;Mn)Ufr>+ynmZazMFrnaFg8jB$=pjeMu&2 zkW8WhGKmJrBpdrcrd32HNdROPBxr38K9DI}+N2XgrVQ(lFq|_3(=H;DBmgoC60|l4AIOv~ zZPE$seSbhyheZLH%nt-$+RqwHu@my+Ojc1|DJJb5CM|7QejbxK1|5ao4@Ww5h{lr^ z9il-dOPF++2ACu^80iaBMLr$Yi8SgIC1@$I>Of1IbfV0rgc9sJGz_RDB5ANYQ&n1a zQYsf;)5&nL#!?`e+)kJCWj0YNzpkGEjaoA^6@BXvpI#9HXEhfwS=e+i6Tf65#Cx> zY}wgC@~_ystfatFRBk(YbhB=HI*&>(he|YpIMT~QG@dl`5DijU!p_4qKqax%0F_0K z9)H#eP+8>ZQPOgA3N39mN}v*~J0uEMuf)8AtJjL%JAXZt&(za%p>|C@mhv4l?>VfW zLnZn}9BJzz8c+Iqhz6-F;p|}=pps}@NoA3_ha~_ii|joL{5{apCY=yV0eE~!7+MOL z$p(RkACLo|qH3A+!|0GdPtXcpOhSOP$^$nK-T?kNK; zZPE!r6YM@D3|6u%HLPTH#crR!hs&o?V{*!SLoPS3p+RlLzjU$dU z`w)#U%|1kfP?j+JFbzOS#v`{&bdlMIbplWpnSC^veW0aHI#H%2sRXwV4FfEh-3OMk z72ACt!!j9UM5-q}lC7k+QY+YPuTOgC?B{M^YA1(C@}?bW^&uKxT78HHi7a9DVHzNk zj0cD;vih)2fXE`NkADWM545yNC(5)Xkzn zO?Wv-W9yBF(;ylKC&6SA8@(_+U8~4V>*_U-=W73Y^~S^pdD>q%>yNWnJai|k_d6!z z+0cmpT)jEc$=)1kX32Vk^Ti}ElJE22uom3S({Cs%wr2G227jw}MjG>I97f~h`>OZs zrIm|G_h=gSUoHo=n2rBj1TSCoUO4+Ndd{m~4E@SDG}6-VrD=7`Ua}w9u#OJq-C^*l z8x6;^*q8)^w9xe2)Qo8L7Ru$%NjP8rl>Scp8YW3Nz5g@{y8R%^KBcz>gD_0)KV+w5 z^ES_;{{m1;1G9+}l>r4u4^SF-vm+Iw0s}g_XS2x{(g7dy5NOIn;JG-l*D*If2^Cv> zTe^}bZ}pyiZw6z0_#d;o8DR(o1DF&ZC6gf{D}T}l!gio7Sv62-N=tV_nmD4?14*sp zzprd3P6AnVIl6bw+!<6Z_p*(C!8)N;g|Z|?5h$j)P)&v2H>>yoMb1;jX{!}fs0W9Z z^-akPVcNp0)ds8=;GzIhjxbZ9o%e=d?AQ(@bxCljAa1pl)Q8eG*iiOO8^C#*-eC!z za)0V69)Pi#iRggjY(|(^b08zf45&!@I diff --git a/summary_results.json b/summary_results.json index 927e0c816..5e12f43c4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.800", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1737, + "Performance_Result": 10.1836, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420191, + "Performance_Result": 0.419826, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.00489, + "Performance_Result": 6.26123, "Performance_Units": "Samples/s" }, { From 8ebd6fc2c0b668b13c43fff0274e2115cb13cba7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Mar 2025 21:53:39 +0000 Subject: [PATCH 343/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10225 -> 10224 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index dc444cfff..d382e57c8 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 21:49:37,687 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 21:49:37,687 submission_checker1.py:3260 INFO] --- -[2025-03-27 21:49:37,687 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 21:53:34,415 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 21:53:34,415 submission_checker1.py:3260 INFO] --- +[2025-03-27 21:53:34,415 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 21:49:37,687 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 21:53:34,415 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 21:49:37,687 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 21:53:34,416 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 21:49:37,687 submission_checker1.py:3276 INFO] --- -[2025-03-27 21:49:37,687 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 21:49:37,687 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 21:49:37,687 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 21:49:37,687 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 21:49:37,687 submission_checker1.py:3297 INFO] --- -[2025-03-27 21:49:37,687 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 21:49:38,698 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 21:53:34,416 submission_checker1.py:3276 INFO] --- +[2025-03-27 21:53:34,416 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 21:53:34,416 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 21:53:34,416 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 21:53:34,416 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 21:53:34,416 submission_checker1.py:3297 INFO] --- +[2025-03-27 21:53:34,416 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 21:53:35,630 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424492 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412232 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72668 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e59798702..3d78d3c5d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -970 +971 diff --git a/summary.csv b/summary.csv index 85853b22c..f87894060 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424492,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412232,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72668,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6486d333a..83c29e266 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.424492, + "Result":0.412232, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index eee6c88d2ae5ad5935b03ee1893816879a42736d..3771c64a79152ed8fcbaa445822382a77b51e4f8 100644 GIT binary patch delta 3100 zcmV+%4CC|hPw-E$a|i@R%>^=(cL+Cs(9LCg%5kn(A&zfT^W-8RE;WD{NjT2GKN4UJ zdo9U|PiEwq^O8Wb_U_(7pSz`A=-}78>BP7VqIew64r{7z){J1*3;W~Q@UV7u`EJjx z8FA8`^}CaB797^T2XXD!qyHRy3!^{cQ4l0XnqU?m)<#J(-)}VH-YA%M<9axM4`%5n zgD{$Qlk{ygY{c^@==PUE(@8^{rrnr!$Ftfh!+umIM>rUad%>Hqx0nXAWR)ZeCfy_* zemok_ z4bZ+@4&y8``oW;Pm?Y=nw-3R8csNSZ!)9e6y>JpQ|23xL^jPYeG40+R)|xfr+qj>M zvc$${-0uh3fvy?7MVy4w-&P-~b%Ir>ve&W7RGVn8u3Dz8t?c{wJ_*(_ja9G95x(gr z-J^pj{ANVU9^!d7`)1U>*6CsCWvySb=4@<@b;J36gOS#e9lhJ5=0W3sHcPx#T*#11 z*>EKruD2Q*>3BAS(tJ=F#$fJ_;T#n$OAM1p>|MzvkvO}ODDgd`w8k)Huz+GqJvK|rh_qZ#`%mAWeP;j9tJ)*`tbYN+jIY8=k(=& zaxoOAtD)gI87;ond*QS(ouvOx)0Iy8Y92&`oAGQAMCp1hxJiOIxrrBFr{g$V`ouT4 zR^4nQQ4lN_S&eKh)%fI}tmC5~PM0uoW4)Hy+ho4!&F6c)`C>1e&F-SbEJ+trdu!EA zFPsg=!`d&kevov>leqTkpfMoR^_MDtN=g^4K{~YMe>vm2SdB}qmKgPx^4vV?98R5s z)QNNLHA)!_iH*mtD{7rI#(SR zidB?S0>vw53&o~e_gv57B)XhL7fBQ+&})=90<#m80Et8*NhA_UB9TZE-2xIxxFoS4 zyp7>NqFX>Bh69PNB$3333H38dlqoc4?!Sk`DoQCq;|@4W&PNw~b|DF|DhZ`87#V4I8Q_g-6)*)ek@?PU-mRjBt zI@>DXc_8^P7iKv2gJ7PXkWn1m+^V{pPKQ4XWhE7spz?A(uoae0-L`GFm4hX}+kUL@ z?z&}3EK%dw5=(NI9;VHOVq&pxY1B7NdmyKAVFKn{e)9!@kuAjlsEh!tV6@( zM9D-jQ*Kr89J$<@CNnU9wOLnBAhU`Z%gmSSfvNR9=eq0IIaiO0Z#f+++{LkMNhWF> zN0NyeB$H@>OrilYQG;aaA~Hz=AhRGrd-LG~nX<(voe(l*Scinc-BgxZZjhX974Yu` zvx*wa++bY~Y;!SbH7#4^kjZyPj}>k@T8<=>Y)rb6Ow=HmL<3}h5)F`v8eGaYi^wDi zfXspf?ahY|WXcwwbfQcx$&_Io8U|<*5zd*-e-xVPNvqi^YAiKpHn-yp(up#g z5=yY^&@iBqh@`>pOjT*wNvT|XO((;}8cTs>aywnlm)S(Ayt;k@l+V$=-=^Uzc|Xrgh%mUbVaai!gdXb{a3b|0nzG>Hb# zEVBEs1b}9d-A9An2U>j62|*L=J|qlQvMe>MWOc=EpTCF9XKMGk(A##ObT!+wJuBzu zxmxF-iPjNY`hAGTm3|+hK{QMFeV7K&BpN_}v&iqm5&)V-exJ>U4`_nlhjfB_mk`#W zVZbIc{J^YPvEk=O*?iLOWhF(Hnl-l_Ke}00j+^uA5#7u|6Rjh*H2e^aD-AzHgJ_m8 z{4fomNi=|Fk>Q6W05ppXKN<`_(BhL$2%2E{Az?UcGQSVZniczf{vI-)sp02BZySDp zvRSj~sJsPjD+f(9kJ!@jLo}{*{16SIS;FzdG=L`20GdUPAC>^nEOPv4aQr}vPdWi# z0Yr6J6p&fQ@S`g>{5(eH|7t9#q`p#6?wWmCs-?V~AC+h;2TC-K*wXAnG@dm35Dh|E z!tBE|03{iZ+%C~YW*^oGKv`t=(O~v}ffk>1qRf`065KvC46tN&A6Uv(Z1;H#%Vdxd zsh)I8wvyUPtzfskKIxUSm%D+f?HnS>lXj%lhiE)$^&uK0vV_%#X@E#F9w4&F>cct# zB8#j(8mvCh;*(C4X-Oi%>O;eTNM`i`BA=7hC!a{~Nkl$ZqmQ zK{O0bg2^N{dSQCFR*{?5)ng#f)&BMBiHQ&Lw7+oHA7_tv=uTG8cTC2!p%MSNdUB+l zJvq|MlJy4Xi%DQ4-{-+$Ex4Pf-%wU;&FJ3^R?mzy=FvEe#>w|p@7Y5u7nAPMH0-}z z4r(zQ|G5ZWzUaNM_h0nvSHBog`jv5Lq@~|W)9RMJWG~pTjt=JCVeqLN4ac+Cm;{5g z(Dc~UjA-=~%H``MoG)Lcf78B(NfJ))-;IK9KZvq->1Dwn43qme*&*3H&GYELvxXCv z4h6RZT5lzjAtEbYa9f1!KwGkEpiq~V?1VH)M6CysTE~B1#ZH_Avg&el@0__as9Nl0 z7yW{DLaQ2ONs1y+Omm^OHTr0m@gs_yr;5`~E2z-`4lSCyiW$PRg*U4WSTDdu0i+yZ zrbd&hA|rqC0T`Q^h>l3kW~BMC-3Z_eLkAL+cQ{KjeD-><(%ozrl?$g7elT#;dJ<>% z-ig`R_kGeAqc!Bu@Ym|~eOxsb>ToIssHr$3%mVe=HWj`s!r?mV{3@JyBY-~*LBH{K zTx9A49)$p8ihEK%i|3biiJCmk%Q!9K{GrV=QWh4ZDA$LPSN&(8JP!Fc@dTK1Uw?w; q53^$<$^ryN%>^>EF(y3$1-AoQZzYp+CsYAqlgK9{1{5U#0000hU-m@+ delta 3101 zcmV+&4C3?fPw`K%a|i^r;BAkScL+CsbamODa-1tB#PMxvo?HaPr3Mfq3CH>OM*@st zuO(UW$&5U6UJ_{5-rZa1bGOtB9sGJXofx-46pzE%VNKP|ni0%;VShXu9@efd-|cxd zBTl-res>big2USPAg=v-^q+%oVf05l3WCH)6U^en+9*lp`;A828wJyDTo31e!7Tk` z5JuB(lD>_Gjd&gf-TpFYI%#OrbQ;s{cvf3w*pJHO2nU04FL)F77Smvstdc~*q?@F} zk4NKqTsu1GkJHbxdWg9JsWpxcmVIB3gKzQHYa=V=YZ(5Kz4_QbteKg# z0or%VVVp%qKNxfuljJ=7_8}O54@XIQ*z7E%7f#~kzs7W&9!p&_rro>4TC-++8~2k@ zme?4L`~4t0&^4pCh?8*o+v+2=POu79&N^0^Y7_0(m1R2G&c2WDlVBawSoOLb;hS#K zJvxZOZ$`B2A)a@$Z$|BFogS86*7_xD&c@bQH=N%$7-=2Z(Yrlr9yD%$v&3t~g$%ir z4Og<^daI$4j%PC{%?G7n4Cd|_&QW36VwgnY>`E?)#NCx#5s9}exh4{SBiSjp$;)j* z0wu~tg&60>ql(K%6<3ccu8R~qm0G+~3*xn?6cxUx^ijp-qlzn3$S<`h$4;d-t@PN& z?^K+_uZcu_?i`qu5{8@^B@}BjAw%&O4n<_O%lY(O}zLz9mmt-W~f?&DGYGiAv#;5klIz9^GbO{qT)@zx)P3D{4e7@J4FZROO>@Hf&l5{b(w^rTs z!r5Rvto>5!2T6B4iEFP88Ur$2f2n_>q;%05q(fW&mou)5)wtAZiBY$d@8wbFa_U^9 zPMmA6QR0d-{RAaI9g#@th(uCHB;pKx?-o!;!o?r&j|Jf^4EII&>2|m{N^K0c#P|8h zl}M5p(dB29C{t+8+zOE#j&8cW?0RM|y?j}v+8d@o&c7#N)EnL0>bg@m_j-Q|rL7Vu zR#8d`6tCPZ6q{b%_k5d^=y4J~BvG6|uTkO(%uY}OBoc`vkw_$oL?TJ_3P>d3lEi}W z7KQ_fUIB?14kUV#L=qz=)XykUrqG;u{~i*nD5V66*WS*g*tE23X~0HZqsfrjQ1wpWrjP?`Pu8zD#|N?;JLqrpkG%_y)~EZ5U5O!;IdsXDA0IKG_GLd zglK?5rjZyjjpUGMe39q`q$8kI>2TOjp{aE4M zb=#I$qQ-S3mgFuyU`aH9CD8zusDb%$qoG)m1b}5hf|io|38&D~CY>l#-tdF44h@qN zB@@9+xmCe)z?c6TsgXCuOb|sl)W73mkq6Wz%8X$j@Xn;)A;8M0(L?%f9 zWELc7Z9aS;Q?|58C(6{4Oc~apVSpwP;hfq0N1>^nw3@A=#!_=;b9>J8>W*#YU1ajz z{9}ciL2L8j1DUd=O*(XQFS++ zDZLyfckDan1Cwa}aHM~SXgq1)AsS?|goTG`fJtJbk=6lKMnQ3OdM!b{7FEjv3%{uNu7l@wTt%55i)Zq`jt=TYh9P>DtmM|ydP#*=0q zqCqN4*m;-+s3evept8u(!#V*fi#$C_T5e9ErOieORDyMfMB(a{n0IjXTCscQuZQxP zdU`I@uBpdTzGLRyhxK!)M8Ak5Z9PQeNna1qAeDb5oIOkfR1%FVsVp-0umnJ5k-bNO zzXw{{q!VH(0FMs|LrVcO`GBRMVwVqM`SU#m&(!I2skW^?PF*`{>jx&gzm+pNml|Co zjL<3whjpUMPB$rGhxzt5gAC^FJS(Koq!RiAoZPE$uTtZlfhROMoiC__{ zEB1f-JO<`Z*}JTy$TACAy>0i=y6#zC9-1l#O*D=;((Xevp0xWA4We1X?!z>ICeZ+z zMRp&S0MIP5`)IKHKueo+LeK=e4+(>nEK3b5SzWQ)=kFo&nc96W^tRn6UClNf-_H4Y zuGTqdqIJZPejlRoq~C{V5X};PAEp5`i3WerEb{xX1b}9d-)HmT1DfFXA)VmfC4_Zo z7_i9*f4!9j|RgJw6sYl1WhpfkT9GzncoLy&5Hd#e-D|@)bMkmw+(+k z*{s=gRo;Tu%0Uy&BaU?Z5RE4tKSYCQmT>$q4WLOhfM${7ha~_siyS{196!*~CY^w< z0HQi93dk&D_|X*`ejX$9e>E0VQeP=3cg;SQvXr0mqY`cBK#9f?N1A6N0hY|}154S8?LLoTnG7-_ z)st??R#IE373{XxC%tm^b2l)xlS3qV(vGzH5REUbK172=mazIT4G>Aj14I^CeOMOV&a25?Ju15$Jrwux|7xO9h32FXvBZ6o*e09 zPmVORWWB-pViFk1_jzzw3-0FWH8>^m%9&FN-x;`SJ&Kci z@5E$m+cs&7!5Z>s_;dOCKCBuG)t`z1s!PrYvp~JJb&1c5aJYs#zYHhd2;fg$(6785 z78(11ME00000O%vrA diff --git a/summary_results.json b/summary_results.json index 5e12f43c4..9b3549d79 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.424492, + "Performance_Result": 0.412232, "Performance_Units": "Samples/s" }, { From 3eeb50e51a28818da61fef78d099652a1487a3ac Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Mar 2025 21:58:04 +0000 Subject: [PATCH 344/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10224 -> 10226 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index d382e57c8..d2689179c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 21:53:34,415 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 21:53:34,415 submission_checker1.py:3260 INFO] --- -[2025-03-27 21:53:34,415 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 21:57:58,980 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 21:57:58,980 submission_checker1.py:3260 INFO] --- +[2025-03-27 21:57:58,980 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 21:53:34,415 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 21:57:58,980 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 21:53:34,416 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 21:57:58,980 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 21:53:34,416 submission_checker1.py:3276 INFO] --- -[2025-03-27 21:53:34,416 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 21:53:34,416 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 21:53:34,416 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 21:53:34,416 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 21:53:34,416 submission_checker1.py:3297 INFO] --- -[2025-03-27 21:53:34,416 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 21:53:35,630 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 21:57:58,980 submission_checker1.py:3276 INFO] --- +[2025-03-27 21:57:58,980 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 21:57:58,980 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 21:57:58,980 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 21:57:58,980 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 21:57:58,980 submission_checker1.py:3297 INFO] --- +[2025-03-27 21:57:58,980 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 21:58:00,021 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,15 +25,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72668 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419826 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421594 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26123 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.66684 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79516 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.266062 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1836 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2486 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3d78d3c5d..66ae0255b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -971 +972 diff --git a/summary.csv b/summary.csv index f87894060..1edd3623b 100644 --- a/summary.csv +++ b/summary.csv @@ -3,15 +3,15 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72668,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419826,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421594,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26123,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.66684,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79516,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.266062,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1836,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2486,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 83c29e266..e40f55da7 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419826, + "Result":0.421594, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.66684, + "Result":7.79516, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1836, - "Accuracy":"acc: 75.600", + "Result":10.2486, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 3771c64a79152ed8fcbaa445822382a77b51e4f8..416886c48d0c961140d6cf3b5b4c4c36eac3cf63 100644 GIT binary patch delta 4264 zcmYkAcQhN``^TfUnxQ@#RZ1e%tj4TadsCyaN9|c^RM03&Vkfa_sa1qhp@WZ6d(?_e z?M-RTXi;D7?|jbr-E;1F{RFag^7R3{#@S?fK*qpnq03s3ygy#7XH9`=zYPN3UOMPp59acyGH zuFbW#=^&cfL5rM{I}6Rc+EVj0Gev&E+Y){NcL~C$Vv5`U;_p2}*yj=X z(%Bkuj%EOLxoai4!&i%#6InEBl)Ha~VAH?eU@A0KH2I0-NNDFf$hn&-|Eomu=J?wS z?Vh{HU~iQDPh|-p9kAz0W=yY__Ga+WkNb>MaUSkpN94-yxu}5|%k{o&9LQ=vM5kyV z;()}2iDr#gy~O$w47C^^flr|bqm}Bwr(K9*tDC7_id;+X=X3SWe=Rqcy*)X=d0M>_ z;61ogHgUj~jgjo~_;;M;CN(by)AevE_|EeTf)?dXGIz7vSdS@2cc0a)vwmkLo+_Qz zC3Va8=pKU@VTwh^&v@ro{$p%EI!vm(ts+?-eBe&slBn3;bn17F?wifE!aK1x<)V(>1*L30I$!aS z0B>jE1XnW+`aR2DL&W}r-rypC$;1taj{#o-wm7wTsc!GBfo8cd$2x4fDyls0P*eSw z)T-cH{SSiLj-J%Qab4no!PL~{VQ5+M)s*VaHVHUpJED>p$$1^^To7dS)~rHyzO;%` zWLs#eLadA#Hvbg@M$woA?{7<$n0on`YR@Glb}Eq>>5@~h0RRA+%c%tD_qdq-hI>k5 zjm#XMZV0^^zMGD)pl+Vihjw~1cPu{+f|~>qTqp>(liY(bv32;q;834_6mGX-{$(^_ zq54<><=K4me0{-+dBc8+bR=`a?tAX#R2;_8rc}ekeHtry7bm5B>LZ&#=-!}? z+I7U{#?C8JEH4UZ8|I|FIBeDa_iQ3FY2-6_Fx+!I!FTzYL@$uD(61TE?F6lN zIb`<(8X&E>9&A(Kjhw3kJBO2t>ncQ5pVW`9(tcly)Hk14k@Y8G{*tbHY6#62&n z;-C4jgwJ2O+GKBArgQLPCL54gU0%N+$V0WIU7fe+_8G09UhjSXoD^lWzwxckkiF8S zedCU?!nkeT4ufJ5TlcF=b){_AUX>micdQo137K|_)d_SN6thd87?jWk9|mPI&kx)?e_5awPcYnOMawkgq119` z88IlrX_i*|lmZ{nFQ}Or-7hASWA|7JT)jdW3)HlIEZ`o8m}f?pl0+~l#Qf`a*7e`# zjnQZK&y*3&6Wv{rKaS7e;kd6VAxGpb8>B?Ym}{zdu)|=&vp$a#-%9lH37mbK6WaKKSP2v`H`bwL ztI8E&%ki0gq7aYU@J9k|z{b|fRmMv+Ukr?{fPrW*MhlEqc#wA0!pDu0!@dCvZ=!xBdqDe3JH)#a6Bbm6H1p+*XM4+Z}KqOH4 z5*4S$YpjNb$|1gA%hzgM)i+dmI>VZu+e?)fz>`rRfh>u-)0^sPT!0~!y4u!#)Ng1M zpUVlzxV~$B>RP3X7pS6$T%Ti)w6&2u8n=G0nlB+6*%$OVUwK(_SSB)&wSbH+OT)1n zhE*-tQ+q|?WZ=Uq;0e%V70Jd9u{Voq!A} zXb{-sAIv2^>yn-m#pKj`>COG_R0l2~m7$4+$0rs4_hv24|9rXaA$$-Ks8HSCUo=+h zm8Qx+&!u4gannr}t5JLsErp)j0Sa3#|7Y7DNTm{xjQ*(ExiWn(ws%2SX z1>59c^h03Y+`Bm!0N$AFj5V3yD9d&>~%tT5>d_#*$O)l1&;)Bh(^Tk81v z7ju>@FjE zl<6f{nG>S7#u31VQ1aRwx&?Y@8Gxe75_TXufb6l8_C;cP*M0`Tp~O;?)cnepa&nPB zTJ>omlYwp=M%rpb`{0uD+HXasqD>{ZMaRTqWrUaH*u zWYkddDt+JU?swUW3{}W4buahr%j@`sKv9(mFMe|XL7_t-sBB(uz33YugO4fe5I23)VY-`?n&9Oq@ymL zY4{^?OBL1v8PO8W((vzrVRdbsjvQ#g8S1ptpbEu5)W_?e}lv{~@(yl*o>v+Okdq(=bmqLhU&sK9h(o@G)EfkDcI?!7i3AX`0&n<0tY?K0~ZkU|lqjPX|_`Z%5ibj0L! zb)VTKQl^bQr0caI80yxU&I!@frjbKHnqKb5w?Ky3m;SyMb~ttbMPL&&eKIz92~67* z`N9&XZsMoPfBqB~-3PQj+#o8rL<5tsv(Wof`50%3Zj`AZ0&J!2U2I-GW1Dw4Fx5BP zG8)8)SwZUG`xE9@wZE7dfEKM=Z7^l^AJ&{{P?OVtDy#KqJ27^I2Sf$^>;&X&6j!jpck6F&Nv5oOTp*4* z+o?mQ?PRu-2@mB1pm@?f{^hA5)P*&&QIYWne_& z_hR?JKHD1Z$?7D=wONNBWt&q<-Z&jd4HjNpA;hp|l2Iz<*7H3j@Lemsf{c4`7h*yyOfKlD zv*4$3k6uV1M3b!Y>Tg48VOv#u~~EWw7G|@{<#(1SM!AjmXaUwEee#PTp_WK z1PvJ8Zg|3nC#pv$K-RDS5ju2yVOud2e+}{OzL@a_gsxHTb%wwpP+Kh8(@nai7KJrrhXp5v+Ua_x6tg zt+8Ss-G}GqHS>7|%=z||BG-kb;iQ!bt>N3hxo2KgWG7|H{k9JAY|Qxf!7!1Xb-u{(mxsgTL=QEeATlG|Wi>o=hujZ&cBX?&+&w9N#@4yq zd3e(C$Htwt`+v7zHr9{6w|~%$@j{(_V3q8X>nuKxLCcjCC%yVLJ#>`a7+=!c&azB@ zf0Go|T(Z1*y*}cJA1-3;1F2ggTzkRRzh^qHfqFVHnLO0kB)m%Gb4|v2OmcRN*c9KW zQ*7%}ANyEf3DT9o>h!aIM>nrQZ29IedS^!;o3tyF)F8dfGJC9dhsjp&ZAJ#p^SKOB zbAQ8M&Pr56`T#){(7v)6(ljG|`fUE-#ye7&+%qZGJLQP>AcdTdC9|Xzw39T2oQ>+j zCz*E>ccc|*uUUmz)nlQ)N3X(tsUC%$aQ|0j5ZbO=&`=IT$$#?WZt>nk_8YW`Ge>`y zS6>NT0-FiKwQrMcp>?z&kx6fyw7=n4S9^54cw8}Xb8hk`1SIpMmXd{pd^KTH1uTzK-V delta 4302 zcmZ8lcQ72>*IuGUw5%4Zvr3d#$r6#M*~G3DEjm$`AfiPJVzq?T!Y->OSS19(61_$j z8!f?NlOTGp(IOx3_xXkQt@#}%8Oteh8K03{fO|6a zHia5;llw}HBPxUjRsKe|OpRS}HNoCvgu zrqtKxsEH1S{kpHDsAlZjOC0alyV4)d7LV6R8vqS!`Ee`qrK`(GqRi#g##^-}eL{4` zmcy7wP>1*I9L6O9?4=CYF6`97q~DX~6B$ihkB?)BoUjmOhBtPQrh#G}8$-*3n)A;& zE%pB~!gbvgsX3C&Er_LllID>|eBLp@HIV7CZR5Ewm#t+~=B6>3#Y2}-&Ty7@3%^B! z)fzYH{uE$R(8*H%@-$ye=4G&!gz0Fs_3Q%Vlel)jwuCkJ=88|gc7#_q{*Q8z#5*+! zLM!LS^N8l5+lTgbwB5al=^i=q4;v>DFRVf%75F&XW2|WBV-}`ia4FW0Q{H6~nY;1F z78Upw4DlFqxt&0Kq!}KayrLUI?v6tf<0bm0Du)ln^bwb+*Z=?k-T6`j+$U1YuVSCm zS)tUc4mQ4PW;YgCFZ~>ET<)x6#0T(vVS4i3eJsv)?8?a;X{*A|_sD&)jLU1}&3+@+ zlY?ihh1P3*Jnt`0@RdBZSpydE{R~?U{;~UT`5zvT+3(_Mesh6J%udT<5waP6*uAF^ zx6WPhrGr$BQTW}=LUNj4Pt~MEM~IQAmJ;e(Tu^&vMQs{Q6*2`(FG>dUhTH)S&P@($ zpt3@6-yERrKP&u75A;Gt{mQ(2D%MT{d4gkWt+~yw+jMNbUL*FL*%1fw8HZXdndYSA zkK1z^)4+=}wao1SO4=hGPL6`jo)(0DG8E>f>aybqo!!pUn*4sxzdB}QD zp#=9RSm*E`9q{bZ+pB1$2Ak;41+grmo)S0EY$x(AT3Amh8FH`LBO3bk1P6mQZ|cvY z9sZtKYn};&2&hCq`{ke!aB^0K6}X~?a^-_YWNy2$K$ioh>)_!^^=M9;cXLXX$DKE8 zO~`Iw0~RSx^LRJuZ->9KOXu%$9GY>YDP+qsaWSa*$X+IkxPkAIf^$9$dh+do zOcrBz$E3{1kVKyzY~8_1j-0^geyW7FHfNcJa`imw)?9;GJfaMDZ6CKYeP;#5qx&Y} zEBe{Gf;Nv=VRjBxkuVU>n*>{lhGyk%zM))!)g8|LsT636pLt8CMrPvF>g(CU9jdC` z0>Xj94X1HHM|xuVLqSZ(@PpdaM}unxsWK%-K^<(pN58>wfq9vhk7`#JtUYplN~mIt zlMN}mBp+YtDzyAb^D-X@27i9YeRNG`vWB$ zwJx=s-9gw5`HnYF{;v7Du+OKcZ%t2W4waM?e$V*qM|WO^o}_=Su{k%1g0w2UG$HDO zqcDc_``$3f+uq%fJUcLf3fP6GS|>(vU(4+hdn5kvO9;!Dpxuu3++&=@ccAi=#Lwpf zN3xtwN(T|PZ{MO44S1%f-#29HMyWA4im>BycDz+8*m1h#ny0lX5hATqA*EXz4UTXr zgSiPH!aJ3(0t{-yI0F!0+fM6-^`+pHZ3F=g%f3I651qH+V*SZCPX$WvcKm|&L1iDh z8*lN{PiKhUko{*%I&sVNbqvbm7O&)VSEUB!IDQIt>_|uL9}NOGqGNA{u1XuNd14z8pFbevu$%WIso? zo)2C}x-Z#|7ewO>y!s`bh+@lFftQdZT%{BqE7mUR)Vf=iHO1<*WTHwvvD4ln=|x_5 z&aoz5U_{gOg1JxBol2s0Fo0eu*wFW2Y%vo8b+i?xLz>0v+HPLLD@)P0AwvWJqk&_# zF>ez29}{ABwB=mZWl4q>H+Rzpn0#5F9#jA<=`N6_+_K<$dvA=w;EbbkAWj|i< zH#LM%CJ%mXe)gee{r3Bf{2K+C9lsJ@^aR@iiTpKtn=J8?jCY_BI=G50Sw>m78)QT| zC&ubdDXT4^w0Il%cj!T93ECsH0`&uOX}0jKH+*CRoF1^*C^E2uD^@l;P4P53VaT=(qqxfnzj z5u)MuN8R{=T7nS}`S<1eIu zplODr2q<s!*pSvQ`B7g0ssStZS-Z1`~pmLMhU>%RRd_YT&W!@A{!Bzic0cscIF; z#l?Dy#T@k5RmQ(Z925Xsc4dnEKZ8d0wGfwtY zmN5qoIR`frB?5rNGN^$V)P~N5N|#}KC}^kjM81`IAcOtn#TlP`cJa)jrU!Qj!Mtj@ zsEPWiB<%yp>xq%=b7Fi>Ss3x=pSC+7p`yd*&=UA@fFUXGQ)kap^;kV+3X}~6)YptH z(Y$*rPg3#p&i#eIu2Mz$Er$9V880l}7c^z@r#MT}YC{qxM%L~^-a<}T7&Xm5t#v>E zB3!Sh{~5mL#6Naf-5hx^Ys!M!;05=giw80zx;Eu?_7`#CPaRVatVdW^9c}oS2~m_gknooL8Yn} zVJEx5h1yr{c7(#5_9=w2vERh9VViKMs%hTq8h^A;#+GMd_ZTblI}HefZg?M9G0-ls z&Lk;w?3;tY;J35=6*QW8r@Ga1jQ{cLfF)g%L4Td@s@$u8QT2h|`EOKijSOJ+X%=JQ zw%{L&?RL_2neTq+o465|elVI0#z7W_mEyYlej9I+?n-iEH-;f=tyWB~a@>FG>8x9p znk#AX{6KPZt)kAv|U*nwj+4|>K z!C5jk1*p*$-dAv-?&gp9aCfz6g8#B(y8r|B(2Hu>1pNCDg^y)ly)%w$+3L#L z3aVW_O~Zodt(13|dJMBbR>R{J_;S1Aa)mlg8Iz?h`ew6SPRTv{m)7_p1Y00lQ^sIP z1Y=D%KI9(y;P&VUctY@Vj`-VS>$i?mh7c4^iMn{X*zjX)ZpoYSc|Sg*G_zMi#cFH6 z8I5y)!dw!o{*2GONqUbU<31dhAzRh%397f#TgaQaS8Vi^C*h4x)oN7C6?o-*PwRKS zLl0X9g2aS30)DC)+l$Q@i13}c2Fj2;vO&_KE%5BUZ=Wj4vCQ`k&;B_X=s%I2qMbz@ z9;KXt$lHiN;&DCk%iFORJNprO{(F0OUIv~2uf-%`63dZ?m@CdV4spWB>Z^m`RoVZT zyQ&bx`5*-?+|3m^vPp}2&@+0&hhlM7l!EB7ndEqkX>0z4`Kad4T)oOidw?%Iwzdp+ zlo@2?S75XuXchG_z3fVre!r@N8gvrt-SM4DRrR;&R>8AQO=W_4$zDAE9@D}N>*@A< zO=(s#GGgPDUiq(o4bS`3SjD+7>@zGy_l@ob*LocMX<58wzj@?y>LzM$v>&}5sOa^e zakG!L&z+9#r8slg;%ujN&-5YK$Zh-Svi94iZ$SsmTyfhosQ>FSsyetV=uownEIdRg z004bT0O0C*+ky0O(D(N6d?w@I;qCZOz*Dz3n9r^qvMl$0|Dn%omR9q$j9gc@B)I12 zr{dNkA#i9JEykh;Wz(gbMMYTg_Swv;zfokd=E(Uc+*h*% zll3lUR9CWS7{DZ*(HO(^303{_W(lL2Oxrz*d>*^P`?oh4+b1e{VJn!Q`Wn;DlEusg zWc$0Mn`Fu_qkDXat>lM#+Z5L_)1wn$U!3xfD#>&|8Sc^oll_$t^1%mz6(eM~YbjUW zv4zNbjs^lB|Nf39P5gc_!kMS=YjA6PW&eeh0>_G{1Wi$~lJEfK{v=G1y+!KNjPlB< zL{@-ss2Zb2>$aa2a^(`=O8_z^$WS$QG1U*9afkR>NqY7)mw3aFBVT+xD#`TEn^r(!6a@;S0=FNa09I>VHQC*mV&N?thy%`tzI`0RY#|^ZFkP4Rgf8Mek9(!oC(2 z;`z54&qoX5>;S+; Date: Thu, 27 Mar 2025 22:24:46 +0000 Subject: [PATCH 345/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10226 -> 10227 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index d2689179c..6d3914848 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 21:57:58,980 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 21:57:58,980 submission_checker1.py:3260 INFO] --- -[2025-03-27 21:57:58,980 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 22:24:41,241 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 22:24:41,241 submission_checker1.py:3260 INFO] --- +[2025-03-27 22:24:41,241 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 21:57:58,980 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 22:24:41,241 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 21:57:58,980 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 22:24:41,241 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 21:57:58,980 submission_checker1.py:3276 INFO] --- -[2025-03-27 21:57:58,980 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 21:57:58,980 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 21:57:58,980 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 21:57:58,980 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 21:57:58,980 submission_checker1.py:3297 INFO] --- -[2025-03-27 21:57:58,980 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 21:58:00,021 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 22:24:41,241 submission_checker1.py:3276 INFO] --- +[2025-03-27 22:24:41,241 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 22:24:41,241 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 22:24:41,241 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 22:24:41,241 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 22:24:41,241 submission_checker1.py:3297 INFO] --- +[2025-03-27 22:24:41,241 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 22:24:42,504 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,10 +38,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.1786 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.433713 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.118 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.433713 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 66ae0255b..835c5f1dc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -972 +973 diff --git a/summary.csv b/summary.csv index 1edd3623b..b6adcd3af 100644 --- a/summary.csv +++ b/summary.csv @@ -16,10 +16,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.1786,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.433713,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.118,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.433713,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e40f55da7..6a0970757 100644 --- a/summary.json +++ b/summary.json @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1786, + "Result":23.118, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -544,7 +544,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0828, + "Result":23.1561, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 416886c48d0c961140d6cf3b5b4c4c36eac3cf63..bd15d71e09db34e06b3aed2beb76619c4703fb1a 100644 GIT binary patch delta 1416 zcmV;31$X-LPxDW(bO;G2JV6}p3IG5Qvwa9$2Y;^epZK^rRH6~YmR=sBaiy7uXpqVh zb{?hyDv6~As4Q~yuug!=B2SN!mYY*(@!2SWO0e#bC|tb~^A4_FD|YYv^-w-jPtS$g zHTATVXPbFcdO1|0U&NNS9-?ujuZL)m$`Z~VrU5F6#*tJOnR{3Qpt8u`qrl$-Ek5al zSbqw@<3qyGQou|;U@55B<%3xMd{4nMb^2VYZL5!6*S6aF1C!mql`}b)8eJo{wE7T@ zE3H0616(qVH{r4=>&H!A*@5g<(Eyr7b|01i z&@8h1Xt4W0i%&WsXoB5`guzOdrG}NPuGsDK_mKHa?LHTJ+wPODW}CKW<@|ZB);VaR zb;Oo_AEI%k--l=r%@TeerU5jGT}R$yZZs54(g~nhGvTTSNeU32FWbp_hA|! zlh}2D%p$)J>jcOw^80A;`#_6NIw53&--m?Zq{-|)Flkn7_xXFke5QV%3%%|4$tKOF zqw?0XtsFGbJ7P=257D^N@Iy3+W`7C857PjeL<8&DBEt_$0B9B&el!?)+i7estVHzNkj0cD;a{91NfXE`J zj|Qg?wD_bGWm=L*aQe_NAd)$KfXL_M^vNgEdlHe))#sz_T_T+(yc?vk^}@qx5DkNq zU^0n~UYMS)Rph31^%lr;wSRxTdST*&Jnb)>^~c#89=em&>m8HvY-q%Pu3j8zXD^O4 zvt+%&`C<|n$@h71SPSmv={J-WTQmB1gVie|jd?T?~cPyzq|5dZ)HWN%|oa&K^RFJo_VWiEJaY@L$JYQr!PMfd#*qrEL#NgoK? zfwpATK%pru-3e)uh*}RMwT}P3vYj{yWYy*9-Z^t;P`TKvCiw;DrO_2CvK%F#xe-!t zEA+lzrVl9bfoZ`SqoG0_c(kZ*N^U8CHx6E%vEYIPpG1)Ml-mmJLa-EL&v&4h&!R&| zabuifA(n27E#u#83z+2j9aa#SV1eNtnA(Ym4oJZ#r2TQt01zBQ0}6ETxX228@&<6q zUu_taGpCZFvvAdV6esuI%gNZbZPv~PYs_EZ&(+J@uxcuGe<}{BE(NFD0Sm@R)g?YJ z;^7+R!z!M5BSAcNQNQwbSY+%2k;DLWjC)i*W>3%SC8|kIZqt00lKV9wl+5XzeDxzQ z`cFW081ir85isVy`~>wMvt%R60tqKPK^*Q1000n^HzrpBij$8fJOUIIlP(n&lVm0k zlgb$o3jhEB000000DytMlMW|30WgzECng&?G+r|$0RR9{0ssIJ00000000000001h Wff196Cq4mXlgTF|1{Wm&0001YQk6>p delta 1405 zcmV-@1%mqXPx4Q&bO;H}YX9Br3IG5Rvwa9$2Y+t5I*&>>he|Yp*wV{GG_ExB5DijU z!p_4qKqax%0F_0K9@Ys^S>)+a(sFYOEj}A1Pzlx@5{0W*V&1{kYsK!JzaGkG>gl;q zyQZF&@@zBjIjomMCHh5dY3m^xSNeL02B|FJ>|q+9l4u-BWs$juB>*am>^%znJ<#Hl zPJf7{06ab<3@ruB7>^@8bXc7&eS!DNN z2>{I^yN?FD548BC6M`n#eMlIrWLauh$?A&TK7S9H&(!X7p||Zm>1wuVdsfb$=W3mU zCR#^q>GvTTSNeU32GK0x_hA}9lV||VBEJty0B9EZeKrRl&;-8^=>+#KA*@5gfPYP9 z_<>ooV#CjmviYRl%SwtYHEV7=esr_095=@#tD8A!qIJZUh99DFrQwHY5X};XAEp5` zi3ZRtGW@UvfM${5M}y%9T71$8K@$u=Bn)Rw=J$bFvtqx`-$UjzHT+!YZNpDCYc?H~ zx1eq1po!)YTRMJ-#+8mAqCqrEIDdYa2GArLK(om4!x8|RMUEd0jvr|8NhjbdfT#|O z0y4`Oessl#pU24jUyTKo)K?11U9(S1wN&0K2(*<0B^pO;Y4#x+Pnvy*2B9or_F)=; zl8i@gm*^t159DK-N4j#4w2+dJJRYyG@i8j5DgMp!s^2`KqMIt5LsmPVVwYxMOGgT zRv&2bNhiv*B#~hCpWNB71W|Fb>}_ii|Hw9^d3&6+O)M^tnLbC2!GQS-mErYy#N;lkaC2X679V=1Y^hcAgN1&Lj`fG zt)xDbc83jR-*g8!Pt$uW!Bb8>#XT@KBN3gDoQ+8HW19iM8HNTVDDQBVWccFsV5OUE z7?dlg6uvVsX+4Xh`{2ZAY}+B0jdhl2(v)FwnCIJ8dPyzq|5dZ)H0000000000fPoK_izhw-WRuD# LBL)^F00000rvQe# diff --git a/summary_results.json b/summary_results.json index 4a8e770b6..b061ff58e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0828, + "Performance_Result": 23.1561, "Performance_Units": "Samples/s" }, { @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1786, + "Performance_Result": 23.118, "Performance_Units": "Samples/s" }, { @@ -725,7 +725,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 2fa4eb0f86dcd000f704563bf0bec75fc93885c2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 27 Mar 2025 22:29:49 +0000 Subject: [PATCH 346/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10227 -> 10229 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 6d3914848..619e2b3a5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 22:24:41,241 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 22:24:41,241 submission_checker1.py:3260 INFO] --- -[2025-03-27 22:24:41,241 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-27 22:29:44,584 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-27 22:29:44,584 submission_checker1.py:3260 INFO] --- +[2025-03-27 22:29:44,584 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 22:24:41,241 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-27 22:29:44,585 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 22:24:41,241 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-27 22:29:44,585 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 22:24:41,241 submission_checker1.py:3276 INFO] --- -[2025-03-27 22:24:41,241 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 22:24:41,241 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 22:24:41,241 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 22:24:41,241 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 22:24:41,241 submission_checker1.py:3297 INFO] --- -[2025-03-27 22:24:41,241 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 22:24:42,504 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-27 22:29:44,585 submission_checker1.py:3276 INFO] --- +[2025-03-27 22:29:44,585 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-27 22:29:44,585 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-27 22:29:44,585 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-27 22:29:44,585 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-27 22:29:44,585 submission_checker1.py:3297 INFO] --- +[2025-03-27 22:29:44,585 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-27 22:29:45,549 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79516 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.266062 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2486 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1969 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 835c5f1dc..7f1114fc1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -973 +974 diff --git a/summary.csv b/summary.csv index b6adcd3af..214e30522 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79516,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.266062,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2486,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1969,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6a0970757..2f0b5db5b 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2486, - "Accuracy":"acc: 75.000", + "Result":10.1969, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index bd15d71e09db34e06b3aed2beb76619c4703fb1a..bbf22899b89060c66126a1b974cc1b075558021d 100644 GIT binary patch delta 1425 zcmV;C1#bHDPxVi*%>x9anGT7Q(F0%t9DB1f1a1Om4$ZM(Jp`Y%FS#OccXVn##+X~xZip%o(^xBJh?N%nI@hF1T55ZA1U%V|Ey z?M~mf->rIEqU++7Ne)9^@#bl@tFK%nRl2oxOxfuchoP;~trqC@CVbUWymeJDCO zFx4G@XrhCh8=V8PzD2Bp^4|6Thobw8oc=&`5B)9C*-qE@4nO$R`MBzQU3H*AIMN{< zu+UBr2vi*cfvQ6wP<03de3{Su9jZgabHY|+Lb8k- zA(aJdm~W%adBe;Z%$vRVFSy#w7g4HKwjgUx4n$h44F01zqVNWjN7q)y!gy9?#1OBg z+|rYmQobXVt7$|padR4Blq{tz6|aqVbvjaI!8oVUejQX*q(3UMzZdR#gT2r@_+q$7 z2gb;dnQx#nyV^^AT!4f%9+tt1TnB8G7Tm~4EEBcca;;*``9^&$Xjxy$W_?M8p!w!5 zAwfi#x|6%xm{PI1Q6+ghHX8o~vt|gq3kedWk|WQRpR-UI^FFZh%cY)6yi^A)FzRY_kBkp+MKUHEU;ZY?X z{%U$J*>9g3EhfH6pY+wju&fB5;8{ulC?i~547tZ2lk?-`d(XtWiQVSA2khMvPY=N4u!>+ zEvoBGf6rzo$ntgDmp*r2#yz4F+uNHY9o=ypcW>Myy6u@_9=w=bFE1z80&%2|)p`_1_r}Z7 z*!O+bmxDFtFYxF5^?g`1m3nt74ydUFr`!Px#z{35J}=_oI_ASXo_HleJZ+FwleH}azY1XPD1|0W&*``nkGp!ox{ zWg~xAItC|&)(F0%tK6tY;1a1O<-JDyiWD$g_t!T+EqA}G9%c@nr z3d+J{@#MC>w%uCq{)>3=szV@PXFl_Hs1BinY%g}uFMX&wI55>6bYZ6iIX5~7EF`yxRoE8! zpR2lRSrwvc$q#8|;>Gtq7GU?k2=Ft~`vU=g&i!`+bnLb(1qkG4z5ABV30si~$ue$) zR2HmZzKu5L4KrsjZ@$HU!PUon5v6Kn3$o_qK%~XW;6Iup3U4rZbZuoUjAvCw4DnjZ zPkQoF%6Fu4HI3*cZcZbNlBJZT;wvA&f*To$WukUlu2rl#->9zzE$b`UtS_k$G~e7M zB!~!8ck=5trc`WhR7u{BjmAF#vt|gq3uP7SW; zYgp0Ou+-NYT%i{$^kRixtk8=Uda*(;QRpQKy+omxDD)CZFNAAoH^8QyX=xJdkY!L5 zPkk@XvTZ3Ee@mQ#)bv!T+Qrb-&ew7~4=PM7Ly^=U_WOpaP;jI)$l3nu@XkoprEW`C z{Z_KypP~?;UJ_A?6*&i78Bp3Px@+M*w@JiBPoBEeP-o-nC$~`fBgJPe~s?=rG6B~51~6E({^#-;s+i8`jGj-LA3(DjAuN5#tO(>Gy-b zrsL@ux`1)yEpC;-S$i|4_-{JmzR@kfjH7f^5}9p zTP{X7^Lhq!#N_FDzS}qZe&6#PvG;sBel`;^s(QqBJ)5|u-S^~)WH$slT|?lxII-6; zH$Dj!TYOu(k|%HVo_%iyV|@4@P)h>@6aWAKv&R`(2n9JbUNa?=B_bbxYQr!PMfd#* zqrEL#NgoK?fwpATK%pru-3e)uh*}RMwT}P3vYj{yWYy*9-Z^t;P`TKvCiw;DrO_2C zvK%F#xe-!tEA+lzrVl9bfoZ`SqoG0_c(kZ*N^U7P4qlzH;DQ97M3DBB+Y0SMuoPp@ zcc7ThqC-bFK|FR*zw&liWb6Zx!~k@RdsIGVPtWTlC8|kIZqt00lKV9w zl+5XzeDxzQ`cFW081ir85isVy`~>wMvu7iO0tDx)niZ4pCB+3loVp`;les2W0lt$F zCprNzlT0Tj8#y#yGbI5408jz|01*HH00000000000DyrJlZ_`n0cDfSCnE+IB>(^b E07|2qxBvhE diff --git a/summary_results.json b/summary_results.json index b061ff58e..7b3cda5a1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2486, + "Performance_Result": 10.1969, "Performance_Units": "Samples/s" }, { From d22e6f6bc48bf338fb752cc01c0485f01a1ec851 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 06:13:36 +0000 Subject: [PATCH 347/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10229 -> 10228 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 619e2b3a5..87cdea217 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-27 22:29:44,584 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-27 22:29:44,584 submission_checker1.py:3260 INFO] --- -[2025-03-27 22:29:44,584 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 06:13:31,022 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 06:13:31,022 submission_checker1.py:3260 INFO] --- +[2025-03-28 06:13:31,022 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-27 22:29:44,585 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 06:13:31,022 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-27 22:29:44,585 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 06:13:31,022 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-27 22:29:44,585 submission_checker1.py:3276 INFO] --- -[2025-03-27 22:29:44,585 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-27 22:29:44,585 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-27 22:29:44,585 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-27 22:29:44,585 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-27 22:29:44,585 submission_checker1.py:3297 INFO] --- -[2025-03-27 22:29:44,585 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-27 22:29:45,549 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 06:13:31,022 submission_checker1.py:3276 INFO] --- +[2025-03-28 06:13:31,022 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 06:13:31,022 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 06:13:31,022 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 06:13:31,022 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 06:13:31,022 submission_checker1.py:3297 INFO] --- +[2025-03-28 06:13:31,022 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 06:13:32,175 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412232 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72668 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73944 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421594 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26123 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 7f1114fc1..37cdd416f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -974 +975 diff --git a/summary.csv b/summary.csv index 214e30522..281f5d3d5 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412232,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72668,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73944,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421594,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26123,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 2f0b5db5b..590b91680 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72668, + "Result":1.73944, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index bbf22899b89060c66126a1b974cc1b075558021d..f895676f323d4ec6aafc91c31239071f4240c2d0 100644 GIT binary patch delta 3089 zcmV+s4DR#wPxMc)bO;4yyfe{^-(o^qTkCdBb=YMxvK#H9uhBMHa(_eTPZ zVXq}w@yU!lb6yhDT6=eIq0ik?FYDmfyXnNZ4Wf7)&JJs;Zq|%o)(iXN+3>J-b@^`3 zs~K_9o%Oqua26caz6Wvb*Q5U&d<&yL;!zMJMw(z2AJ#@mGT(1B;@&8jcH??De-CEq zkwF+uyGi;n8aCp26m?BlOr4q#=YQ8*jr44S+YtJ1(R-) z4nH1^=W*@mpg&HJW%U?QFgUCotNoMTv{P#w9W4949tYpzt)Gpol&@j@x|>1ngGkX|^6m;V~mae6Lw&6sxY4r|St@on5s zMp_pd$-Xc!I>2Iq;YMo#es+@JKGSw#9uPe)Rw4Hq)-zUL3rm^aEIl?#H zqZj+bW zgak^IiwZH$i$@igk1DPnRa_S-b}F@ar540%Q7I~XQR$(iQwr2k%%kkBaw(x-y#z6{hgvj zi?iD%5^;7v5s5gv9U>8D_cM`*vwKD);_QAQ5{F-K4!__WNMOqg4kfO<;84;O^}Znz zQSVzK5%s=9i7(Fm`+Eubh4_K=Ag+;*1>vpxWA_u(scdMI@re#Tf615*#>g4xGfI>x z5IK7o_~7Wn?`Ln%+aEipFPDp4g6?UQwU6vXKgCT^_PGJBiMH@*3M zuQy-pg|pdRw3sF7e`0EHt-9%jv%z>+`=!yY%QSqnk#|odW+qT3KHLfGEBv0u9OQHcRi3YGl4a|=l4aJfq04xg< zw3OUWe>jDfHt9r}@_`?Ob!eEJD47Um%B>2XBbQs#WCo@->*@()R#9V_`EorlwSMMY z_gp9E?oshAr(=b?IJP6nM2+i8GEswM5)F_^G(aY5kW5`fCP@Hf79?nG4nB}6TiT=( zLZ%GskT7_f%2LY>lC!M>{=HyUQDd10tm}boe{Lr2rtPR4GWqW4vBFJ9+m&RJjY&_E zi5eu6Xn;(j0Wwj8OW9@-nIr*_S&*Q$Iru=PY-y8Dl&K||GORh8F#|k&eZBLSk8rPR(q6Wz%8X%KsfK0Nn4`f?nX;u#Iw54runq~sIg_NuIn(+-0JDl3OU;?q&YY99_uQKu+DOMz7fTH2%&Wi}<0VAr8xKqV1LgWZ{`(z26Mx%iq+hKn_p0?Fidx|}bw ziBkD>{RAkVse9*KZQFNL-EI0-j(bP#*mukaCei%iNdFGec+$W_G{|HL3lGx(f0M*U zBdr6f$i>4t0Va!lJWASZPNAjEMk#kKA*w^72$DpEx0V%Kc6N~bE4D5xDXmeGXvV^mT zX@E+iaV3>S<{p*+s4TMgDDd|{OPh2;ECt~4Az^4KU?v~16jbc;K`ej1r{I}7eJ<6u z)yJu8M{WIq$?o6EnVd_Ft`SFCeTc@BRv)4PE}2Gh$uyEnrjcCgBC8Ke3m~~HO3>0^ z^?{Z)ld2FSF83@i4^5SWCK^W^Y4;%-PuhKm2GJ~G_hA}9lV||VBD)Vu0B9E3eKgp8 zpruWd`w$=!j&JAud9KzuXrgt*k$xYd@uZV25k!A%{eAZ?navzB(K+HszYo!P((gkw zNM;GY57Pjd#I6Hm7WsWxCqQPA-$#Sr2U^;s6GA5VeMlHin#}G4lV-(spT7spXX^L4 z(A$2WY|?DHDsMe&<)De)5l0$+h{ltKAEH4tOBjBb2GArLSkD$2epmuPv&itH!SDku zZPI@U*a{%3!=ix9GJYRjvES!0GXGa!K_&H-f^ygEV<}7Jy@5d6IZ&c+#F1VfqVc8I zhiDMW5?&vs0Vv6MV7@H!`mjy_$|A3i2Comav`Ht*v?P^a_Mu^bC3E}0O15IR&tq67 zgN#V^q$jeK)K+Q%yKVJJZ=8Kk=dEC!93p?oi*}^bhiH81^dTA~vV_xzX@E#F9w4&F z>BBk!B8!|p8k|1R(k7iK)0RYn(}#uuk<94>L_Q~{Pd<_UlZbq-J|FGu66rSK-5`yv z7amT7Xc(LXlSyp!!t`{lA~&t8w?Lk&{p-~W6CdPhf8ne@&ff6QovdE(n2cvbBmRGL z_2Nh;dvT2&7n8t9zR!chT5vZ{zoD$yn$f=-tX>&u%%gD_jg#-I-m`aBE+*Zh zY1n_c9Mobq{&Nw$e9?R1?7!$auYNJ~E91~eOTU+<)h&C;eqh5oI+%Bd!KZFC9M587 z5)9Hp({ocZqSZ?%mp>=reEC!QJMA88m?Yu!{?jPv_Jb(=p5JmU>3ZuO(KWGTTcAza;HBe|uOLsz=B%;;>Nv)H= zul!5`S!Fr8ch1}yRIZM)iT=Pkp;d*lBt;P@rnyl23Vm$X@e)POQ^jec6;$X14z22k zk{QCZg*U4WSTDdu0i+yZra}kr4Z+y414!zU;7~!_X)CD@rQKsgf7uV+1J2WQfhBm# zsi(LD#%3m>5y{z%G{3g#0i0oIK!WlPXGw<7UJq8fn+?5k;grIk4BWI%;_N;;F&o>q zP1>TjhWr`++PuE^tHwfgr(%HWk~6|AP_J!W;>#i&uA$Cv!ijeR_|qxqH{QlYhCbj? z2tbColk!^UtIsr11OeZEAf;&qNBmn>bPyzq|5dZ)H f0000000000fPoW}jVC?>X+$~Kmxk!_3_$reK>;+V3N^^7GUGHM!2mS!xYp%_ujFifU6V@b)9 z!HJV4>mb|6mXIyUp{#=;cXaRb-1|KD{r7#I&-?p)f8Rge4f{>|im;=6&?kb;J8oV2BWZ{tOP_lYR=&Sd)=bCh*r_VCBQqmIX{kg;w8 zjgz*}_S>Ruv%wJvr*QfWUb@$)eBXn_>XPorD0EK}Y30?_(**kGiY=q0Ta9x23!got z_r)IL70xnw>Dv{0R>Ymyd@mKBceCUK+ktXw+FNUdFQ|Fa7x6?WiGHEf>kT&NW#DKSw<}9 z?iDS&&wlkDV{PhhO9kzt{0gImS}sl5bx3+9Y_vfNo$*y5e#MtaO`J03#m!Eu zSlZ`g-rs1pe7iT2{b=yci(BngQIUvS!zn?tvD#e#nf#DOc|T~~jSZ;~fE~_5)^Ulq z)olWdED zsykU4l0N!3(HayIq5KDYX2q(E&^m4r(RF(*G$+l35bB`Vj#(|9S#xL`%1>Sbwb>P_ zdj%e!sVCO%2LK z((7YVI0#^M|0xarkiH~N(EmF(2~M35PEVqQPpUTzM@?6qeOho}H<)R+ zJ9r_9|J>}DLK1V+J($EB5-P6QVAl~+vq1<#m)4D0&=e3FmB2Cgm#du@-8#%osTw{r z?O1jCU!Bo|IejebdYhMi&cvmMZbXfut>KqBUivaAiPhI`?kg-lf`%*)nO-+3TGDjE zWn^f$$A=-i4?^3+Ujwt;)z5xYFI6jVdvQlUT-Xs>Rz0E;(wZ5I2jo4n0EfdtFF2Xp zZ-_irtTa87#p1Z?J%|GBg`M}LD$BQc%&_<7EWnm2U)OZ3GETz+?Cx6#o1dHTinvJ6 znrwLGxGdzPurNi^`j(1DhfkE{MB(c1GfM#~t`1g$66I8Ni5&FA9ixY&uc3geU;jCBs1uZ~_!(i2W{^XVE-wYj1jY zTs)83#h)9dkVQM(Sy#p_2U0MyoqJo0w+FYb-?tx8xAr9fJIDy)^Nl#=0WLC}2SMZi(vkYW*Al=*<`evA zwF0I7{?Ojgp(y&ZfR8K?0=0tRDWN6!1;+d^jDoxOqNfgfgyQl6aihUg#HKXYVB~5? zZ*7!wboU}+TH}$wwYI~oKGEib0tL?F2M|gl0KrbbrE};M4m~Gc#IK8E^E)HMM;5?u z@8ou~)-<%nrK$0^_o_G5*5ksBDtdYfhN@7RW{@daBiA>JNCTox<=I6mZHIUc4wq&L zGlAR;K0U4mB7wQIj{~aC3CJ@t)QuP66nEn37DL#HelUotlXIW~8zQ;VIfLHpMWOpB z=!qJ4%sEQ*xdX&Bq;{Ueo4xx#V_NEM-FxpUMgeoDxuxaR8*G~tp;j%S2{tE(6@Byr zh#&L=YJQ3|;a<_&9q-(@70~;k=JrR7-6NQNJczY`2pAR*%p_5UAM z!Qyt1mE!Z(z9io6EQz0@ z=EL*>dJ{*z3p8yccfxQ8D6`lpBd@gSCC@KsTS4W~aHRAVOE}Xzvgl>iKLySj4Mn7B z-_$}A?b@dr3*9K1v)9#&++6|RCJp`=-;#BSWKx(tKA{~A*NwFZUqn21Ba_-Q`Q>P) z5NpKUZ2oMp9xzb$$|ViCVL>goxmgG1jXTHGjb#cyRVNhVgb5%EfY_wFD{5@abgg`} z?o~mPqci$ECX8OTmb?ANz?ml2O7@bYg%nZ}k!f9k2)QC|RYDxQ0XZ0FEQfZjm0WX3EB{5q z_5Q-Or#nkgq7X;&DjAdcO;d+aBp{A8&6+j$J2u%Ed!H^IosUQSjJAKL6Ta*D@^;DN zt0OfZcX>z?jhQ+IS-`7GHC4N!sb@-!;XIz&sdL0YaF&fy?T1UQ@??`xH%`9qz8yk& zBQ+FXnYGMCcfu=@h9sk6NN+{OG$K9Fa!*tRY+;W^nJZ?nvMRQy5Y4T5r3DrT8Q^+y zDMFsi)vs2`Mpm#*9@>-Gq6&bTfcZU4*{}hU7T!vZmFM`7)WzVqVroET9mNb{XpUNB zaaO(?lf>VtHqG+kv}iOu?mehcWQmudw#G*OI7AsL`r)S%6Q&C~|1s7IbVadtqMD+M zVsnqzFS9x!Jw^Fz!}QWquz~Zw)-56fZPoy7tK)Z7(;EAH6Y23kyBd7ZAv;jUOTN4S#PX98+4PxRL3v(NtP-YY$y>DnCfB7otDdeq8!RyWDmOxRV zc#^j^A=NAUS((dRQTV15<4!fM9_up`e_>gZy4f)%BUeWnT9PHLqL8h)GMv|`zyQrh zFuGs;yB%MEbSDcFj5xmXW$|2wq=(LKWc<(>rBn<|I6q`9W+-Z_x2vrn3MRzX%!+q^ z-;UG3XK7+J|Jnb|h&`nFAlCmBFmRE@tg7?Z91}qy%mvn0w6;H_KA3ew+2Bpxky!N2 z%T-vx`LzWwRTF20dmbYsO9E3TaX`L_SW*96<~ZI|8p4vVbgINq76x$K^oQbPVLlT@ zSr-Ti0+Pp}o3%tQvG$W0>{f!)ZUs@HTCxP>qAKYUNjBmwSv9K}Z<-q6J*x2fTWIum zYOPMF?nJT14&OrIfUaugDdoIcF~`4jcMuVh22K0qEDX_o}|A1 zjOWdTr)_17153>+B(C?3Cqd^1?;OUR~lwYZq zdreAN=R+FRhWXZ}b005iCAYXdckA35?SR)gQSksG$Ibqfm7BN2L%*Tab>u&PcU_1F zcQPp*a^Z-CB%S-N9)OerHRcW= Nbwic-PbweY{0oW&&mRB) diff --git a/summary_results.json b/summary_results.json index 7b3cda5a1..6f3050c45 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72668, + "Performance_Result": 1.73944, "Performance_Units": "Samples/s" }, { From a6416dcda656181c05948c035c90623e1640b844 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 14:37:12 +0000 Subject: [PATCH 348/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10228 -> 10231 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 87cdea217..721d85010 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 06:13:31,022 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 06:13:31,022 submission_checker1.py:3260 INFO] --- -[2025-03-28 06:13:31,022 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 14:37:07,466 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 14:37:07,467 submission_checker1.py:3260 INFO] --- +[2025-03-28 14:37:07,467 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 06:13:31,022 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 14:37:07,467 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 06:13:31,022 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 14:37:07,467 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 06:13:31,022 submission_checker1.py:3276 INFO] --- -[2025-03-28 06:13:31,022 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 06:13:31,022 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 06:13:31,022 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 06:13:31,022 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 06:13:31,022 submission_checker1.py:3297 INFO] --- -[2025-03-28 06:13:31,022 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 06:13:32,175 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 14:37:07,467 submission_checker1.py:3276 INFO] --- +[2025-03-28 14:37:07,467 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 14:37:07,467 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 14:37:07,467 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 14:37:07,467 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 14:37:07,467 submission_checker1.py:3297 INFO] --- +[2025-03-28 14:37:07,467 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 14:37:08,573 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412232 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.371057 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73944 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79516 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.266062 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1969 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2692 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 37cdd416f..df142c670 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -975 +976 diff --git a/summary.csv b/summary.csv index 281f5d3d5..9f73aa5fe 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412232,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.371057,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73944,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79516,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.266062,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1969,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2692,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 590b91680..1cc58b060 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412232, + "Result":0.371057, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1969, - "Accuracy":"acc: 75.600", + "Result":10.2692, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index f895676f323d4ec6aafc91c31239071f4240c2d0..186b62cff5d72a8eb76f1a223187fda903983e7d 100644 GIT binary patch delta 4243 zcmZ9QXHXN&+J=G9g*On2f+)pMj6gz%NGFugQB=Ahp%(+v4TJ~+0R75TfbAEhhXLk0-vvbdV-Ltzh&u&`mSd@1G7)cNrA3`6%nL#s* zs=(;afSt&;FGg|47L|H-$dcuT*gK1Eyl4OX5%qhfx}kc)ZuL0n34cJnd}9_jw+<;_ z!`ZX7eJMxxxII(5jaIs3Q+5c`}%7$@?fH-yA0mhD?0Qmc+9SgHD+_C+9K zr*{(c{AW<)>RL=`_oo~4%S8arkE#DGMn4RBp#5@*i`Du91FlF<#Mwi0BedY3+!$YI zNG~+90Imufx)9{@no&&!&LR!2Nw_h>zn4>7?`_LR_U-_BxlOh1X!1MnG_tNK^dA7| zuXG=UG6bzuFJGwFjv9V`ITg45KT;zuc4V_S&R`PHSMSDzmQnCBhA3yO@e)+j9W>jv}v+|adFH1JWX zsKTVSTgcnY{aQLUA9d*%zPf)#ATqV7sWuYcHM4fDq>ab>cDrSx7=gTWbabqzQ;Uw$|0Z^g5Y8%zeb&zZaI&ZDrFFPb zQ5)!SZN0CBTK5f!#Od)t^*UZQM{jU0{fz3#aWuHovy4eXlh~v%FJ3`<8@gkF`4#6h z_nr1`)cPGKISkYdJYQ6(Qsw*&$U@j#C=UqU^Y8KV4CsFbj2J*)6jFZA5-?56I}yU1 z$y&Ml_8oa>4}?#l_jozmysWG;R=TIIG- zKSA1(*l2?{M_G#-stTF!H$5wF%x2m^JqNQ>8o(E#)L%$Xwe>YQr(Nt^>LWXcji}x{ zkZcqP$qNoDUpj~YVUq4?`Pg2*-Mp5v()_8(MX%?TOz(SpEa$XzKXX%gO}51Rj~ZYT zO3}Q3z}-#s2rXE4JmgN;bP(k98-91yx9a>@O^1kp2mbAE6pXc!XQFq%QRQl-N9*0^U`q3MKj$;!d5Ca1DoX>eIF_xnZhhP)W5LyCU!#VLL0f&1WlmshXz zbKXXEkf;@JvC5qDJDhfQh9TSjDPt_9-I_CeYrB=I`hWYfxUK03z?dUL zIpzysGIFM36`|sph^ZChHwi5h84f)7nv9k7@wwS*Q5247-4~fP)%Vv9M#t}+xbAGuddu;?F3#-( z3qTu{4}O#=J)@j-_h*>z4~QkisSx&H3H@7kXaZwEAP=;$TOy!(6Mxsgl=97(EMg;5 zX*X(DwAG@GvC>B3+KZcA_PC(Syk=1l5$hNsC=wv~&XMX3s#8$7-( z7$dT1MhTc6((y5(EJK6S(`D^qf;@{)*xQ0DE@s+67JQEwQM*`ArEcLCtY>Q zSWAb^(h!^m7fT)F#@FgjUKJlqMgQ(Ra7Bi(G;452k$`7CIzFMVV{&qY@K`XO+cC-j z-~Vv#OCfhh-!6sL^UGtqe>^j|sKJVTX6io3;OKW(XlMlP zJL7&4Z#)7(j^^YfaN?N-C1WAnD4TIKLM#;w!h=(dz<7T)JlG-42K227Ez&0=%H51Z z+)6XL(-ONcAZ~n?q_8uI)w^>c-&*AN*aVSxobn@!$;2upZ$UJXflQMMU_?kt(evUF zWFs(0KRt|`x8uZ?2zL0knI0RePXD>kcMZuBcp2hae=aYmj$d3bJSAdoC~3Te2App2 zx|{K%`z?06zAAYgA`^-8D=KIGJG4B+c%qw)3RL=q(?~= z0zH;o|98^+99|lSsF2q2xQ{+_scQEXPk!ep@MAsLRMY41%e78JU5`437c6POdoFR) zBCPz2OaiS^IUfW7OY;)R}TOgwp07UL;=nvONncyVtb`#Z=N z{eY(kfDMS#883*<60Q+U6i3q8JOwXE)#aesWi%HNi|aVy*kG@$Y$+nE`)3EHp485- zibR~(g4q_M{Bl9XXH1K9?#qk78K!8 zGNM=+NFeD@r{caq%_o=bG5u_!*W-eb=hV&I3TK%+u(ofavCypGL7Q3nWq0TCXUJl| z(V}S6_g5oI&;xyeU?R%3KLbT(?l{#Wct)EJe!o`j_`jO?eoiEXW4RxuO-Mj@`<~?W zT3gt44O_Y9t5KilC`?fNi25o-jLM144~e>2V|lCTur$*_5h4PBMS(b{$<$llS57?JDMr>X0nN~XcG6_DQ93SB1v*eQ5_w%4yO#5lj@5N;yVFP?`X44s5i z=ppg{4}If*4YI97YFlUO(`S7FGCJG!2%1*#tWTQ`TRCCHB}ylPOY(w>KkQT3IpFLK z!qq_wT#v8Fz!5Q~H(`q~P8iY`)_<1r83Qlj@fy5r(re{BgyYW%B5zj5f>z*)90g!8 zukx$cgY81DK{9U-bfD{+u89!8cHi7^Q?`PXd7U0rkEL&Fx55vSz2Sa6(&*KK3kS)^ zAHXCnU+m(fsNgNZumomeiSrK^LFcp)&sV|P8FgVck{?jS6hq{*uFF-szUhH95V;HR3|&9>lLCpFn&!8}8*tf2geSCRxL+jeK(360XE7Im zp9;RL{Lh;KI?3VGDtF;lOqYWE0y**P6jkDggL03r6 za8ki0djX;`LB>x5F>QPs*e8;~m#tw6um(N8R4A>mQ_iUEMr<&|HL}>9o~a7n=uh#n zx%tsV%!g~1)J~R)!@U;|JeD@=tiy{_CKP=$4giFS#*6{AOy+t6U$3oGXuHS+=2|RD zutR*nNS0_2=#_38X~WRrX8RzS{-&_!Z=_`HJvJ9dL5X=2i3j>6R#e)&91Q#Uo12XO zP1|AKQ+hgyfvLbaw}Ni|RK{Wj|J@!Pt8d%Ly_^ny286 zr{E;MLrqhFC+J=Bzn{K(OOamjoGPzUHFl#Vb|Z}>YeWCIWcHPB27VzaosWu2?~?rK zG<*$}Z>zAn+i~$^=J@^n%r7DI#08;>{KdFsHC};BYRy(L&j)`ofK>7fNS(5vvI@oLZ+Q zoBx{R(E4MNv&5s5Lra?erLFs}+=n&5>2t#9%fvP0W~J@U*O+2Oot&4GKe=^VM`r8B zgFqT~*%fk~uL1X=oc$$YGC`){6dGA}s+Z39?#KDB?hTFqQ%FR4tMNy|0~Q0k^6b$1}bLastGN11t#`bMu@# zVhxhqdk&GBaSbM~2I4|vssf?v*-U*>s%(@2o8|b$MLR=(d)i%``mF=n@>vPZ_JQM% zd(LSMKXO$Z8^6gBzZlvb&hNLThHOt6X_yQPzkt5RJ;mH+jDP&yKQ&6v7F)sThgNZD zl)q+=AizlANm_%)YKepSeM2gGEl{ao7PCGtAGSw6`S=anS9AM_EKpXZCnIy@M7mkC zBz}P7TwZYQJI|C8(dQ|j^41Gxdd95BOfP8|n(J_s9h)>{vA9}Q=8_gmJ+PR5slrNL zxgKE;<7l2a)(bJNhHj`c>_e$ZJX!QiyT1pjBdAh*`3%@F^Hf zO&_g%kS6Gmd-lraorg{(y-9o#6}nKX1#$KUL~f?kCI|_Q_3>2sKzZ)v1zLIHX31+2 z3GJ!$yDVse&3-VrX zkhxN0@tB`y@|DHDx625vgXPj?Y-XhUY-H9?K)@mP&=~vp*KU3kZ`*X+=5FhsHSw?| zaMgL*$}INk`bK-IZ^B^>k5VVRE&nJE52xfOzt|t^-6Az4Q9ieDPjl!l?!+`wrWXb3 zqwfU}qUUOM+T|mW-<<+G#&X`Vj0L9v?ps}tER$)f%$$!)A;m#QFAbvNKTdV>4+T>^ zuv8w@Uj0v#oOjUai+%W?Kh4sTf2k(FQ~AX`v27{`bTWODnMv@$srDPLF9gEvWerp| zBAA0(W){QWkEPKP>|o$g$~O;G~3RuhTR6dSsg6H0NSP9nWmZH>Ij{+JJx;)z^`miCb!g)s_W{&l zf@7okD=E8F?>t`Op&J6n4XYN+A#`_d*ETg9$(;bhn zM9%%JZO;XU@iIKo`$|~1QrA$qVmHk7pT^c*h`?1nL_mn58CkzgVzasF@O^mKHV40Lqd|J{AMM&tMhk0dqey$GF>mzd9s|0(?+ DviJHx delta 4280 zcmXw-c{J4D8^>o5N(RH&8rv9SghXVi?8`9rO7?7NlqGxF#?FK?#vWm8*_S~iyJTb+ zLe>hC5Rsi9{m%EE^EvmN`}y40{k)&&{rvIV9fK`{%06nEyfhGwFhp%b1@$Lgqj9Ce zR>;3KA;akJS_~yvsoNJrlJAxTKoTR!3uG66m&V%TeF;_bPD;$~#*{Zdib)a!{X{{>~|{_>|{OPZN+faL7}vw#+{j!?T6vD+Ymd zz3HsJh^X|(^#d8m;&zZajpL=_a&wc2H|8FKWxDToXFgK>I_3ZdY5HOLMOzzA^n+ha z`+#oM*H+D}=6klAIH*>~UJ^(;_c+xpx9?d;FIVrYrd^5pNkR{u0QT!JA;q{^FsZ~z zUBcay+cr@wUkp+a=j*Lt{Gd&)a43l9&fPqv%hfF$YtO=(25;=z)zfu#$EUevOEpfS z11*EYWq6a@o?Fs6C$WF+Mv1Z3PI;8s_3dc}iet>~lN|%KBeM4vuL!89r@kXJPNf=G zH5ds!yK$ZSgvJm}O~b?n008JuA0@zp_p*-bglF`^*vW48h}nS(L!)P?;;w90FYk%h z9vPyD&JCAGuPtdh^J$p{&wt$8(V~jAtq?NXAjjVHv<8ou`}?}(3``qKtjvu!QabAy ziBF;ZpvT3oW3e`43ZZ{I1utUe1>D*Vxb5EcF47o}RM$B-{ zbQ^X)?3kvu)U2EI!VEIi=bN7#lOqMM!l~D&y)Vgqec>6K*TJW3N?wlMh?8#PH#3(c zTcyhcLFmP1G$ovf@e;D{ObQ{?6N6IVEXM8w-U8SnKqE(4DV8@ zQN(10=qIn5%s{?b;WmYWs-NES?XuH({$9)xv$2>7wVUgXv$2!o3b(|YO1IHt8F6J* zSXa3gx2wYn)g)j4&o?n&9dYB&8Mr(|E|oiUY@4VF@(ef{rZeeuaQ041X5_Y4f4GY4 zoJg_+;~GU9U|_d<9(wR&&Fb5(gp{dp8lA+L3zGdYT9A~Nm&Lc&7;&x{YqSZ4hWz}K9foS+x`F9>X z{p%T4+#H1(A1%5ph>yNK3j=Q)WfIk6NK*)~yN?nJPOgy$#`19I8a|rjuL)oH6(j(T zy8i^!G6zN3d}06zmA{`e9A?{Y762QqTfkVho7Hd<{YMA>=l3FZL$>(K@h{I|=gSW4 zcUV6cIb#SLIFKSa71Y(uwRBAfAPM#hh1%tq}D zLtz{Re&m~8?lqjXs3Kj@H9)J9SUw*53LZ^(^>o8q;RPw*Ai(qSc%LTtT$=}ew0q1r z&R~_`9l+K}0V7;X$EEyV^~z$K_5vzo@BfM*O<2{W3Q_DHxs|=6M3CB^njYkjF(s@$ z1&>?vNYgsJJ2uwPrVwRNQFVtB4=p)L+>1uRxvowR4%iF(e|&C0W!8^b=XOHw?Vr~P zR)FC@v*7YxpgJM@Gve7V!ej2w@G-PoaIVGt9$9_jo;=Cvid+9f5XEERlEhB~NTY(m zj0Oly9y;yiKXKG+?RQYEQ!A?*l7Tg7el}MjA@q-g4zyWGDM&atm3t8n;b)8_Da=|LN0R@OA0>yB& zmFkSx-j|mcKP12bVhrG##IFW&5&zEH4B-p+#-wH-)94+0$}YSXOtBbope?w*^7Fid ziIIKUp2w}LwLy!PJC?#rYepO5h4vARa&>}a^|%!sDWfn4Fk`&7v@+sRysnixLL1<~ z4nGy67%yTmna8SHaB(I6>eKWd<4=qUSW+3)ag)s%ldg0n#Kje{TCn;CDz&$^T-C0p zHI?L$5T|XBfa8V}Fu}S+;3*tE;mDE(2gaMlXV=Q_Di=6A5%d#_gwmu+)j9xruy%RU z-b>Znx13}wG~)6Xdl&y^P)2#CdK$^!8}~k5kR`jE;l@)Oh_qxlH`8z~+Vi=LxIzy-KZORr*K2!!29F?}Q z-4IafcYyrBCfNgO31B2~~Kcj?SeIvj;*o~i+0#_TJdl-!MI^JT|Y8LCYg%--RE6~1PN!n z=Y2$j%qsFBFQB~9t`PM4M( z3M*QJk}x#xc_~qtn@)=Y+ zY~@&M+%vj+6_7^XW53hiEL>7uP4^S~al?@}DF~Fb%;BC_c8a5{{KJvXa71^ZB#lMh zWWXD->K)|o9cA<@FIBN~a~lG-7!ex;UnSd}Uz;jo*yh5yG=;z`qjjSniymva9EFhA zW_!{y&~fD~TIPY#T?7+8BJc_}P1A!coq#UDTmFQVFroPy#B>ISZMH*B%YerBiomI- zz6X%iE2Bpt{(-#Ie+qwDnHW2yMU>9e9bl|>#3Y#XlEM(&T2QAQ0d9lIz-ZbbG20zm zZe;=n&Bpi)Ef3#b;R({!(^ zNuwOkx16Rs4cl2x?ZCblXezAih@aI8oBd}kRm5%}ic>$mca56h_`xTug|7GE&z*+B zsh}+9Vdr<3-EKYpk&T;S(c%s|4P+!AQAHZf8I(#p2fFb|+UoFe>6+x0)qTdEl=$4d%QE`8AkWGhzLaJ|6TcN8obimFYd~sNH#Pjww z;wT!Kq9i;Yuy=d^#_Ho#`25!K2y*#v81f)-hW%pu)rOP!%!IyAH(wqD8A!3Lslaox zVo$W5kwhT*R9JhR0H*k+8s;t1pPfK{B;vpFPs| z-?G)0wdU76dU7ws|AM8$Cg`Ja2DjzVcqOsIwx~j;URAv2^O^W!^mKChewjmyl7@hC+XZteQ`gEhas{4v_fV&i{1_x-+v7o1?;a}$1)>>Z0fU}W{y zMf2R+uFYAI@#oM>!U=Y9g%7XOVf8+aT=N1hEg*n>ZtM65`w6X0-xF$;e3@!ic z@)y;(<5q>7yAjoJ4HjfUgW#@0)i@ zx>UUj?Sg+MTx%CItZ=bf_ymKN=k&X^<9w-hM?{uoXSy^Z73YlTl_Tp3n`Sp#oX6kv zj~q(s$H?7x9XUtlxM1usJKbG!e9iEM+8RPXSYT&c_{|BB0eFJ002k`0B{!C|3+{KgfN6qVQl3B0G>KaczQi^e&9*45oRM?6XvFp zf)Wg&5Q2j+GxT2qfB=9Y6#&5U-|o{nID{2Okc6?GeMImTWM=s9U2upap Date: Fri, 28 Mar 2025 14:38:39 +0000 Subject: [PATCH 349/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 10231 -> 10226 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 721d85010..a42e77b28 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 14:37:07,466 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 14:37:07,467 submission_checker1.py:3260 INFO] --- -[2025-03-28 14:37:07,467 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 14:38:34,728 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 14:38:34,728 submission_checker1.py:3260 INFO] --- +[2025-03-28 14:38:34,728 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 14:37:07,467 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 14:38:34,728 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 14:37:07,467 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 14:38:34,728 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 14:37:07,467 submission_checker1.py:3276 INFO] --- -[2025-03-28 14:37:07,467 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 14:37:07,467 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 14:37:07,467 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 14:37:07,467 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 14:37:07,467 submission_checker1.py:3297 INFO] --- -[2025-03-28 14:37:07,467 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 14:37:08,573 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 14:38:34,728 submission_checker1.py:3276 INFO] --- +[2025-03-28 14:38:34,728 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 14:38:34,728 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 14:38:34,728 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 14:38:34,728 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 14:38:34,728 submission_checker1.py:3297 INFO] --- +[2025-03-28 14:38:34,728 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 14:38:35,750 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,13 +33,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79516 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.266062 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2692 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2125 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.118 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.433713 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.118 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436676 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index df142c670..366e0a0b3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -976 +977 diff --git a/summary.csv b/summary.csv index 9f73aa5fe..b98400387 100644 --- a/summary.csv +++ b/summary.csv @@ -11,13 +11,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79516,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.266062,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2692,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2125,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.118,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.433713,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.118,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436676,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1cc58b060..cff97ef9a 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2692, - "Accuracy":"acc: 75.200", + "Result":10.2125, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -514,7 +514,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -544,12 +544,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433713, + "Result":0.436676, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 186b62cff5d72a8eb76f1a223187fda903983e7d..4e56df3ddc9ce73bff0edcb0048d6f8534bd621e 100644 GIT binary patch delta 4403 zcmZXYcQo5w{KunWwv^OpsnH1GDMhGKt)g0c1Xa7#CaAr|CUz2nQZz=*8bOuXjaf6c zr)ss8BbPTIHDGYo@Qf9we3o7w;h@H9#p}+=s08*0Lx4 zjhoM6F%D90nQ2!jK*WGWi1W^DF*O}HIPl0W>o->~1U zfRMMC^9tHH{KQslJ6OY2ydB|%@ox7hiUi+dcj0%y_(1I@KTs2kC#{!)?GSHrH(j#Thc1x%lZ@W-Q0q~{Qp4eg9CaUk)csd?r<0}$j zK19=@&>YR6Zr~QB#Zh71nk2KzB;_FC-bzGvO!Nfv!s}~koV+~w9a~|?DL>*mxwUvL z%x+v>Hc1CP{=~6|&Hco-$5U#qEf9#xCearNi`*{tcl}gOLjT3oeIwy$Gruh7tfcqM zvT>E^NO@-=VUhQ`z+?N?@cw`Foy8A5s6doN0B?Kpq}_h)B_LR%HSRW0UxpaEx8c$e z|64<5Q&R{yAmhit0M;*)|K-CiB_wmN4C3DNbU9B?lN!hd008JI{Sm+^_HND=Hh>&|x@8MF)P_jqr7E7{#(e+A5c=Wf__b=je|M6S}r}ZD_YE3vv0Cud#s-!5&DkoaP&jiLEam@C$MqjKgkMe0vi` zSg1aF80XT)d%C({cKwt!C4yga3`jUeR?BE$euWmWlAm_eaZhyxNNA>9>2#dc9!nXm zW*sibpL;>84P|nD?Q1j3GwfV`@kP?DZ;iWn5pYg?Nz>$*t=3#8O0Mfq+BAQ zca1J``z0nP_L~B5c~MwhH#fl@5M97>9fK zdEf15uq33CL>e0~ZL=MmndU-n#_XTa3RDWn2YpZ-jO=#B`Byfk-pZ(5)Q~ZEzbieq zi=Zv~l`oyx@i$MG++U`+>q6-1W{f7-!&mUR%F?#Rd(`JSHQ#>MJbUaBu)W6v!)fD^wL#NRgT6|WcW(JA7nJeXGO+%YuoszRv* z??6?+I@8=>*7jCG%;ftO9yERK{590VIyE}(_L=}X&fuF6X!rCDUL}FBc?~7ooEN8( z^OYIRgnGt^JHeu z>)dAb;Jc?J+q?3&i?TPwS|@`R>m=V4p6ig(b$*i`#L?bcp5c!hJ2ZWT1$ua16K~b& z@n|?exw}<1&8Xq|jqlWQ&6t+$_B}G`QB}&hW4+WJC`S0u7mOrj50eA;iI%e2U!NqH zVD1zjO_pU_%5o>h*1vE*C-xgl05 zT171NlJ#NMhrX|$6889<9vT74PgaBYsDMiaKpxSpevBmoi%S{go`|0^F+t!z-ty)- zf_z^VK@PYwIfc&t1#U>8H)}=Fh-iq^v*91&&9P zOB^ig8?~ETB<8cdGj42~pEy)$mJ3$_BUa}*A}lTL9!{D!JkE#7MGW|Mw z5anU3;VWI{nTyj`BLV`Io_f6UD&KIs&vPb`mTqVq(LZq5Ze)Dlr-xuT^tt4ibYv5) zJ0+(162;YnGse6-2ooFy;8duM@krVqJxMVy)jxc|f$f1}4rQm7F~rp0&4ee)-E; zbU)1JVfEl(;Y1xG{jtzDo`*(_>yC05ty;--4H!J~f~PL6DCGP~WKI-}g#cieL3Q`& z_v&GZ)LAdy_j$s#0{(n|AnF`LsL1P7b#1iK8&_#l%)0-k?@+6=rv(9&_CK-mT zyt$d(FS#as5IQ~f_ObN)x-}Lim`P`0mvlR%rbF#T=3jg3)jEancfv|$;m_zygU3*? zY>-Cv=7>Z-Ca$n!TkDzAjC!egur{=d$fX%>>nU$Blrf`H`l%`OQrhr%i`V;xE%~x- z=@z5pAl0pDzb|w`qkpV5&zVOX$J!qwrF(4|E?*@9u%Dj_R8p7q61(nG2_*zS(3nN& zyd^pmq&4GMCT_uj>flbZfp{@_d9^ckvy5~$NOYV;`R_yDH`Ij8V3o}W0Mrz!QsESnG4*gET$jqbyX8!$T1Vdb{v#XguY?X zr1B3|yKc&u*+(Zz!`>XsGLW|;;$N(jr&;fMShG26WQMe=J^A1%w5ng=@)>8uhxaI& zh;=rfNL+WiFCSh>T5~!ilXe8crAX-A?g9Z`Kk?;b z^5t`+C<>@ueUd6++*dnZjhSF?4msn6<&}P2Rd%04hKx-hNJkYpkA^4i(`nZwPmk?f z_5>i;8l{jz`EY;{(Y+G{vqC;YDBl>vwuQ2BYz$TQm!H%Gdtyu;pWVKC zGbJr1&Xa%p{;^F#Hk`EvWqh$`&RV-!Q}iv?(R`hJiwi=C`WEXHriG`1CHzqkrN|z+ zLLOZphyQyDfo)eAy40~VAy#7*)7$&xuwPoI3Y>v+JA@o*S7=Y`dowvlyx(VB%ta6v zV;xxrK{0&ARMIHg5mDHmYG9Z>dQmlU@j{~v(~SIOUHGssuuwjWns2L;u=D3Hlmrjc zBRr=zN%~^xWZFxv-xX1#Qp<@B}uu*GS*2l;)p?322MQk@=es5%pei5b;^G z4j+EEkQD2f+6&U5*a`0Q59-A7Ym|*@2)aYgR@9YeMS@8Ba_ zbnqjQ7P0G5)}H6LPaB@zJ2@YvO3Zm|xlbc}l*<|I)Z@W=T3&BoGq$)- z?Sh^R84#Vy*$qqRVvP$~x9cgp72fS9WV#xJGZipycvXNpPjaxw|zvi1`1`J8m@hb(y5+GfiRdlz%BD(U; z<%)i`gqv)dj~qM<G6QC2OjOJU7Lz?bNxChUwq#Q2EA6S*SB_X5B?DS zN;4!5Jo1vztogQQq!E`)CCZEERm@M;PPONS=fyJD%v3Bytt9pHI(xp$k)OkEP7d5W zuh@R-Ik;U!IAF#mOMh|uX9C1Q#ls3@3zME=4J-#NN>Hp{9635qT* zbLqaDD?U!!Tm0Y?ZBH9=Z>hGab{4U7760nGZ=*zW1_-2z^~DTQHj=Pl;;3$+ZZ)ce zPki)V&#GX_d?mc7sqNSbS($NpGAH(g@?gd4;)Zot-woqp^I9%;SV=;3Ir?ZS#wo1mqRrI78-osFmYJyE^}d6)8q#o8z{ zBuez_0p+utz}?S2z?aF~lJhE#rU5nSXY~LYcLm|+EDhHtg+%I;lP&n&>Xs3ou?Gof zh@q-aDg9?Dw5kXFHfm9^V*UcnUJg2+S2WX>wkKUKn2dDE$e95E05$PZ2H-|MMDJpP zs0C1Ox~~5?&r12c_{8#ISxN$%HG1hm(soPu8| zvsH4HuM%4mc&r)vBgS^+8{y-W*++)lAn*Y2W^s{hwcQ+$3Uj(v;p>0uHR$Q$JMsn? zGU~&@F7t`Xcaf0yi#?PsPkF(Y5sJXtad4t!I^k85-ZcwXav4VpPW99)>&f=4{UPu( zaY)UA2J<0h$-Aj)$F;^;>=FM#r+j<38K0IMyTMuOn~LTviUZ`|P+D9Q^j5gSTj7Py z;a0m8j^2&oT)Uu2IrDRYX10Jl|ECoj=ON(G_$SKlmbV|Z?AIujS`ecP- zSQH+iY^|uv;$Qt<2W&-38H|iNIdWh6K+3PioiK;i5C+eQkM}*QZ%)^BbFn$2Yxa<0 z)KyHgLub8mG|tUsz#{!9|H3NSN}Y!@F6BjfvofVy=O;=GGU8?awHal&jl}V z8YLVH8lvY;W1sBv-xp-pgm+`{l{F|ChSg&R1VSs|r&rQ6RoRU2o~AFi`d!ENG>`fL z>FSX^4=Tr9RW>oout56NLNO6>J)X)yzI52amd=ORj@dM8CUjleR_e^kaxHfxcsWjW8B|5`nE+hdohK~EN=@B{{-%sb&1vpe=#VyUUoRC(zat}-CmA{ zh3*Kzcnl_Q@_<@C@qwQ_+)vadH{3zPb`K-9J6H~rwL6M-7||zJ2F8zL@taI&xsTbg zu&mj;m>F6gl6cG&Ejub@ci^(}oMyvJ<^&m<``l0h7ccJDdHNDuKnEM(pj1*g+dZF} zdGgEQ_;A@xocVo8&M=fsv{~l-d#Usr+^>O=G=sBIp4b>!%&A!H$PvN^L+QeHHw$s0oBOM_sj>Y8& z4=#<}%aW%mVj@htr^+qpD67&KX+cxud6LIK=z6vJ`+Q@`K6}0A4)7qVc12J)7I?$C z`-_ihXm&O`8U;mzY{E6sBY&;*6@j{kPjH06AI?8VX43tOn~Z6f7M_7MFZcS(hO6J4 z*`kW*@uYG7sU{?!?+CnjJmVSi!PysVV~`sUjiXzW)?tsFqS1+f@gzgRXlSxF6zxTW zhFYgsfah9$xQ0czLG6*Shbh{RJEG6n_;hYyC6DJ(>W^;~m~rh*&t@{u;J#;2=o%;G zt#QYZ<`M*b$zcM5B+O`7t~L~`ks6encWg@&2eqC%OpOjyAn7Uc*o9O1-4=3aWX*fp zaGehloD{M$9zRn`04_GUdZ&FKs6qW~tWMk);tKk)%y@_`_FP)2x<#5c0J5&3qojb8 zqdQR`k1R$8qb@11Ixd+VoFK6O&%7HeS~wC`DXi-J%!iJ*`XryJpmz$`)(F*8a=+*^ z>D5wmZXkV2l>&TXAG63seSMvR?Q?5?nOo!f)01|i2@`t?AyHW}D~ENdTZ1MPOnD22 zGf6B;Hg&>iJ~wu&$MCl!WUpal7GMTDGz68wSqF*Zg99vHL)QcwvVCYpl-6PE`;nZ{ z{;nBWf|=A#J=PS1$-Q5tG9z8r^pD6THVZ3zLL1mJrWKxeH745l92cIv9<6M%Vefs7 z%4O2%nT9z6UGh&_wavbOU~jvGc_>b*Empm_?LPqfG%P9&=f}~ zmuW3Aw`#-}Q^c|);ci)y{lCb>h~wlK$#RjnEnbN?Zpt$#`*p~+JpbL zL9-R6VwSE!lF-?>h`~L?MsDDGLgh=DT*q$f4jJ2xM_$iN2(tIoZASEKksC zClTsZP}D(q8RUF+&LbCUZB)Fbgh8nw9cMJ14`;Hm8l=OQ&qq^l&{QQX|4*DeIAtZoX0NyO}6W zKi5>tFbmRdN6V;05_3+MJ-9&q6B>IP2>+@sf0Rb4F`lq0E{5uva}?1qx9oxhe@wEM zvKbXoGi7;)?gUv)hY_&@ICE8{?o5VdP!2&MD)_!1?O*`bF@Te7OKE5UV?d3yJLa~p zNO3H(=W`$1`g!`SD-;)yO~>nvI)*>Q6AzQLR|*6%+9B9xtjMJj5xkiQj%B$}(j#*L ze@y)6%U3r+Y(5iFc?Yj}62X5GrY%q(_}e9=heSKD4=a5$JUq2uVB2=W1X*AyxpUIZ z@oVYo)3fxx0Ful#_R50!7cSQ8$J`JoK5G7jYlPlo0*OTLg%YZiiHj)e5HbHJpWI9r zN>F+MeW&9{INs^w;5^xi4soCu_6r0M3e1*Owud=Hpx)x z+)J-4O!Cgs=|sF`myGz+AorkP9CtKUabjssGRDuZic9=@_()ac=s}bpVbej#l%*-M z1yw=&4mKYv(nN%YmlO3;S^gnH8>Q^Egj%i~+5Zhlb+qLq`{z#@Q@wPATDR2O0lF(c zBhx(ky|TYl=*6%lZ@>+a?nPxIq%z(|aPKTdf*_)GngW(`V_W+@a5!jWU0q5WN%0;y z3P?sOs30>(U-*;4Uf&C&_c=qU#2d;RH}av$-vb%Qcrr(+VnYoKvLEv_34l(m!<1s0 zbn{1J0-sm=i7I4K3=7K9;6^PrU#zbqw7jec-jNCq&k371_?5awui8%SQkuT!$l5f| ziD&j{Aud+WJ|_qKT+mk3o#1>c`abe?=p*tM3E#Yu!__S-89aStt(zrKCRQ+v7#2ED zXmZ{uwKjOBh4>`m=<0_^{r$4B#9zP+PV||$w9Y1ckzEfRppAwBfH(Y!tOP#xA8e%0z-@bIE z9-vbzdZ2J7`adUGd?Hf{3B}3YXl)3dk0p;3HQg9wkWa}VOm^E{=bNwQefm3s!CE9` zbXSH{!nW$kN(fdh;|I;yX>+%G`j=L;U(LUnAf)R{S`;bn-U@!0( z#&>oUB5cC47RO5j?O~xS&Y8ER*u)I^E9+oyp3p}tO&Cz1!fv)uC1$q}3`9vcrEk?4 zCR}Hl&o?}Cv!5h%Q}NzLrH45!hh=P&p?ohcbWjODP71=AI~O|-Pd@*i<1RYboNK8Y5yd$XU3*s8mkG*cj0%-ZA8hwuRp7<`fFPd#y@xPc!h$6YENpP z@)^!;D4x&Jcb@Dj|8G6CvsqD-r?Jbt!SL`1qt2+?0^W&ON*fjG>ISM|p}7-e@Uq3` z@-3(VsjdIL#=wEEY-V?ls}jOn<#QhVRRz@m3E~=zW@wvaOM)VKR*M#@$7zMeo~!qs z;gB(6p5Mpa$R_A~i~+Z`GKz)pVsWKWdlBoq#-*2HbYt#T=_ggxFWid`HhmTn09TV#VJqA9T{*xY zX3|&cUorc}3W~CK%vN2)Z@kyY62)%4W@QFKZ9*A%%g+W?g4y@I7ifIC!Y*kxl<`R3 zXi|zl*AAF%K2`QVjUX)mz-R^luo7Fte@8^56E!B18iGOc3z59UB=PgG|M}L4cNKyx z0Ko4~^4}2=d7TAwm*qNfDI=zrjgjvE0&SRU0`eqM7`Ooc&EH$YM*;vCkpcjqf0q#F Wh)80CEuci=is=^MrM$`aJM Date: Fri, 28 Mar 2025 14:41:47 +0000 Subject: [PATCH 350/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10226 -> 10228 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index a42e77b28..fb148252d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 14:38:34,728 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 14:38:34,728 submission_checker1.py:3260 INFO] --- -[2025-03-28 14:38:34,728 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 14:41:42,046 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 14:41:42,046 submission_checker1.py:3260 INFO] --- +[2025-03-28 14:41:42,046 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 14:38:34,728 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 14:41:42,046 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 14:38:34,728 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 14:41:42,047 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 14:38:34,728 submission_checker1.py:3276 INFO] --- -[2025-03-28 14:38:34,728 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 14:38:34,728 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 14:38:34,728 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 14:38:34,728 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 14:38:34,728 submission_checker1.py:3297 INFO] --- -[2025-03-28 14:38:34,728 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 14:38:35,750 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 14:41:42,047 submission_checker1.py:3276 INFO] --- +[2025-03-28 14:41:42,047 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 14:41:42,047 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 14:41:42,047 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 14:41:42,047 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 14:41:42,047 submission_checker1.py:3297 INFO] --- +[2025-03-28 14:41:42,047 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 14:41:43,330 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,10 +30,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.79516 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.71607 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.266062 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2125 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.395672 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2022 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 366e0a0b3..e052697ff 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -977 +978 diff --git a/summary.csv b/summary.csv index b98400387..350766806 100644 --- a/summary.csv +++ b/summary.csv @@ -8,10 +8,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.79516,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.71607,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.266062,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2125,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.395672,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2022,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cff97ef9a..929d950b8 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.79516, + "Result":4.71607, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.266062, + "Result":0.395672, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2125, - "Accuracy":"acc: 75.800", + "Result":10.2022, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4e56df3ddc9ce73bff0edcb0048d6f8534bd621e..a658975c4c227edd8c34fe6b856cc5c285bbebd6 100644 GIT binary patch delta 4961 zcmY*dcQo8vv>v@g9VL1%qa*}DCL+XOMjxV=Ai5w?qQvB<1w)iEdh{+4y(DrGy%W8Y zsL{LODtUA7TlcN?&OiICwa@zY`S$s~KlVA&J<+WkB%*XMOCs?k27v;=SQI%c_JD{C zm~&r-GNK+oqgO>a6$V=b#q(Lq6yiqN?R9*z^xzzkLTKswvHqpFOeL~<-!vC1(ahWM zjJmTsR%fx7&z@C}`|-UOmf=p-8Iz?>h>+1Z>imAC_{iR@j8pSvodkG zXHET&=iF7wEAkY*_o(~duja|N!P&@J6`u)_$VFU3FvCw3N%;@IU#*|0#Wg!qUnXq> z3`YZU<<#^8+;EL2%6z7vF}<`wk?UM8s~%o+Al~PL$bCX`TvsZFiO0m(3jlh ztCR@MyOnpvZ`*jQipicmuKM$#C0$|cfpQ`M)AH%y<+3M>IWkwT5cFG9Cnp|%6Q-n8 z-AztBu*g76C^RKPNKms)73?<`??p_wctb)+P}lb!ug0gMdG@(1JS0{Xc_?K;0zkl+}3C4pNgJNg%5PJPO z?&K}0vh72sPRFqRg$(oPHF0u7^O1Na5q*Q9%}o3s=B^!A1uh@(seHOwM7!#DuZ^>N)=Zr4#u$2N0^XJu-jf@zbu|)cd&hg_dnrveL>lTXe%t$_ z=beoiBZJ|7EPyk2Yf<|LnV*Nc4L7zw5>dBdXqG5}P9Vi&x3W6B$nv+EA;WYX93@Fx__lb4E{3vyHL;= z?_(}F?G1(>y=w&~)fufkPJYk!nXu^gC9*8n#b=+a0)_GW@HH2<5lxG1H5L+0h-QcD zu4QIyqO;_HQ|&Vn!}Z?bgq{6&Z>&5dPH380_JZ$8+O4~m43r<~bKd5$p}thDSZ<#$ct~Sj!;B~5RHc#N;(^}}Qm`l~KYDNnBKa^#=A=J|( zB4z}Es6gZ(5a=-oKGcQS!TOQQ0^U9|S)LhULLG;0E?E}D9fhQH{1n|6ICaAJHWh-V zhrRC=ysW@ZlZy-Hzr7m9Z*g7)$YruLC^N>j&)8^B45rf$1lRxy>|B%Ocb9hk@AK<4 z*1Yuv4~4YP;&{rh15UC-J#0?Y6(~bG;0y!H?XwWX`elsTJUhvly9MnA&?A^JW{{^D{EHpN{KWB_|7gCE>?##yn=7T$TW9E%;E-9;d$I zZ}AZNJ8o>aI^aEog<~>y;-=SQw)B21O)~0PT&Tz4q$|JJnk&l`-&uKGvFZ%;dC@P*IQqUF(y50lLP(!aG_SHj5DTylc!T+5FVd)gyW2kF)+l>K}%-L?NT zqrCzR3)~lm@zG4P1AO$~n+3tH#z(PmV%>c-?BFCA-obi;fp_G3(4)_9bdY(SBW?_6 z`Nr%>7{>w)CNlpr4c;;Gh#&mQyM+$jt}7pr{gOtYa*ZMaY@GEH;v5yQM29ZH-N8f# zEai6G?!LKh3cFIiQVO8k=;;nQn7nqxF^reKaKavvUMRc&j*OQ{dV+z1(}$_?U-n{=H);kH(a1t2Vf! zSNpGXC};P~sBrVwx@`x3bcprW9v3^;(yrpOUZ^7FvNr{T+@xz-llJ zGdp+91`@o1dvy&)3ny(e)(U0U`rBj(Uj?*l1q~aKlqFD_SKvLuR-j^jlOkOnJ@IAS z1&wT|twu!~Y z%x^Us$l(HR6@CUojF1`>j1}{D7(ZAo-~vBjf4>4h5I}u6O5n_0l+6?Y~*; z*XCe2l$3BW5O{#U^rtmxK@_i-8w8;b`zH7c@|npLxIJy zJ+2u)O8zlpL@-R$8}U`Ly^5Ro_$2K8^3Qdni6(F8l?_C!w30X}WjpwDW$jkE=Rn3> zcGQyFVxD@gxEmgDe4S4hl9sG9HP+>+iDK&?z!xYzeYDZI0exT`7v=GQQ*6Go ziW4E^Vqu(sW)r|V1!?6GNFhV*w)mtFggiAo%C5)1h|;PHwV}LreRwXR-%7>n=6N2> ziT+%=<-l}5`D<$<*EFzhJL*0g+sBLDkL41Z_pY@>2+gFP2&9!LOQA`71lV{O?te*i zW5yuBpfnN+vk$b~J&AYRM$uAr0tVqePAoYHn7pIxAEeVq;m;1AN~*>)X6Cb`n;M^+ z``=Crl>1}yOX%w^LJGO^cY@Y9Pa2HK4on@7kUb05`V^^^00Mf%;Xh^r&y|W^IAe9l z)1=B&+FlL`l)f6Oy`lYHme-~4RaE!T@$dSTbxhidBGO5?5``zNVDSG*&#Kxb5Gy$- zhfJ=wlS$cYe()xI!%a6~g9Hg36A`4I)sPzOgk8yF@!E(&yh%YAr@dZm%H{}-1)l6j z@N+_8D~M>C5gx3d7F3H!3(Wrha#%_8-A^=!3Yp!Qsh-4o3u< z%AI$)JB~KqP-c~mZRoq1qcR#_k5AHSy#~Y_J@4LcRO4_y?~Lq(E)!P`!741PCF z&9gMROi>&NUaf{GYlcpcnavh3%zq85q@l(h3o^9w85!@K`cn;LSNobGF!s#V0dsr# zJ_|K}DaaVoq&soEDp-hcQgFucvcYP?79yNM+Ia*r(L0Lb1UaU5xDjhyFy6;uITWwC z!K7lQ$!tc|jkXQLODgNqb(a>u-M;BIrn^C#GlAn{i_cy^VB3-W*NS)*w9t<5qzhgt zY7sessdG$MaW)j7LB5t@ysTte_f?>+O`7D}KP0WOtZ%I+%cc*G^Is@BK?=1C=Rx6!p4t2vNnO(ZH@x2 zGATWcP$fvSzfx?pZ5e)UcT>?EqOm1LvQ|FTt)JZYhA_C^D+jg|$=`%0rdfd}mMHia zo|s<3Vb22Ca`n_E6O^cG#Z7I6zXw()(HSAyx}(w#(p*0aPI}Slp50xkMaqiVLipN= zm3|&4d_A{?wUWQg8ZvJMpbuaPWV`TU+L80oO%)lq9Yrzi7zkshpk#*`i|`?YF+V`b z$fN#sP0<}RaWO455ntLES!AG&PIe&QwJ`mK1xj&tMsU=vtwx#6oK_)&bVO2~LI^wk z``zX*GgWspW8Gi{fN4K*pnAxerYrE#6}t2Ex^@Hnv(ZI%hxO(wdX( z84^Oln80kZh`OKdS#MQLtM2Cmawdkj|S> zfuvMmUp5CO@!n`(vI!yNn*ce=#%rFw#ljSlP)V)5CA$c_W69zCKZG@r45)apJF$IdRrVvADWK|!1oF2a zWfLqn1#$vLNQAjM$$c-%ban>s#M|pJ>Y^lutKCEZHqDHX@sp>evzb)$G>6y9NJ{W>e6uN5~UmH~~W#`f5Sd-jg6_+!?txmy5L~pp~vdYq9 zQ=vB&{vLdR&(@o;^u7uQ*`ih0gAy<;aK9;Kr$IM{nPI6w_iq=Q>8V~aei0xA<*(4Gj1Dp@4 zVQkjlvp-JLo1%;QI_OuZo^0YmTZ&dTSsR1AUSWgQ8*n{RD8x@I&)(VWZ)CGRaYS#B z2B?ZVt<@9LhPP=>^HeR z`h_#Cd$d+soIr-W{u({>^i8mv5JjIjV4h(6_P-_dW)6 zyngiIOVI*u6>TR+9Q7nI;=9)?+luGzfnd@4^LlSV-Nmg(my1+g=Le7fck-!eXEi4$ z=7ji+WBK^RfEKfmN=_$DBWlL<(b3F!q1cjBCH?Ky{ZI0-*dSxe*u6|?UsA*Vrk%WJ zl{!v_`Qiv9ag_7jr{?*%K~bxm9{-Q>yR{)~oZoZpy^Qavq}Cs})Ddixe1QZd@(+wV zZ8Y(+Jgg@|Sb{l3BhB3ePjhSU$R?u3m|tcU(VB!50~rBPao?x4U^09UKE$tgza|Qj zrs9O_E&dYkJ?PwS)sUN$gc$3HT(y2LI~CX2Yhu7w?Ozs1kh=C{{~A8+pR05cA3uCb zrDyO;EtO=e&P_DXGOXQa{K7~6&3%KW`&Q00QX|fVE)(ibY5O delta 4936 zcmY+IcTiJXw8jI3sl1Vrh*S3$ZosnSCyp(mjV0tP{PF#-b8TL9@D zu5_gdNDYXsWUT5apN7@J4)q^BdHvx&uFM$9+AP57eV8Q$( zA%~lL6i6|IEgq7#ZXn>Pgp-D|b!iEfn>eX$-NXys%>Yk3F1X4z3M z3{hUs4apf*3cjdKOy!LO^HeqTHsPxoN800uC;DEW)WkoCbv_LRR# z(`78$@p5;LuKghV9fSdbCKaYmn?={`Z7oON`M@|i_UXN^!K*@glvJ^vi0$xtR3#k5 zop{8+Yj?K_o|w=-)>woh72?71JRDJh^jD)sVBmlqiSkd@v$ltLQ#PmKr26kNb%#B2 zuCkW%-a%W(pNs4UYgns%qk6+BdOFb$+3a`1nZpBhhuUiG(-iLqR>c^)P6b-mu?X97 zlxAwL4vhrr#Y`i}RkB*ys^8U1X21n^$0i@l%yT%%U~fDhp>KBUvQYN-KvckF{ZrW! z_rhSlN}Nh(=3T+e&+qjMa@ZTMHgT!3FGs@Q@tNC*#}E2BSijZhTB-l+Po|?NCIG|3 z?2_pzGMk|kK#fX$AhE+V3-Nv(_c{ge^TiDyu^JpHPgBorEYn3g`36j<1OMs*BrXk# zl$ZDQprvUd!1}KtRaG{Eurb0(eHuW6=By-qWM$9T-7{K z@H~o3X9+FomJ{DB4oVwv>^{u*{#a{3fOwGk-GY=hA#Gq?q38V;?HQ z115g$b!hF>jsNa<#UkAonuH(OYNcIyecw}hFy`Ogqmc8jrS-hSP}k z^2!4WU>r@j{3tgaOa@5&WO|WXCW_`sePrltEH;uv|7$p(48CDhnrqv<>mS&j$b5S`sDJsl^BMq|Q#ptjU?NOH@iJ?@v9#I`yga6sY}B_&v|RQAl5O^i=MvJ~Rc`*bx=N1X)7 z0000esII3+0O#2IIXjp@3JLhgpnXukXQc^ul864`8kkG)e%NhQsiD?KAnTh!0)27X z4r^J5=*+3Prl1=kyXk?ZlCs?$wumCi=cO2O!RBhI2S3wzh2GRgn820<&Ab z_$CUsRDJRg;nK=+vA$$}`+`0>f=gr!h&!#8P(z=E7SI!(wtr-s?t}*jt0&*=aGKK` zOCGJJA1=tBe@3PWrFMJmXFJC}{G#mUxo*~-WozMGQjd;>=d54QhG`(hXg`Ez&a z1RQ>YJaYFrIwy8f4hUaehN)_2r$rxisDA!EotZT5UE@dgZ~$9=Kb&f5oR1?u-x1Y7!P6 z_r=HdUC9d1^2HO{|7Gu#{>Lo+HXmxH39SzH^y4*FTG>^rM0U-q`1QLTcx9)^BfWIF zdjj^0R}OT$Ckx`X!QEPg0#?p9R%(kG7;}q{bi2P5#N9K%3)RAUwF;Rkj%GUc3@kdU zkV?UOP$jU|44e&2-`2v5o~m49M^WT1+(IsGlAsXxHh547{Y5^|{>2-tQao<^7E-Dy z522Lvl?H{tooA_Z%|7G-dHS`|AqRz|vY)+_fK;!M#)6csUc#K?0vG9!#RLHqB5?6- z*RA!-i{_}`iofLp>8A0$5kF5ZYR)OQ3?xqNg0YcsZPB7g5wD)J1du(I(hvukoYb$M z=d<(PSGMa%-@HT^-j}^ymSPcTnF?C|DDvjrmDYHg*01pg2r|jlSuVubvDr%u(9`Rd zP>WWtXWbFf!@Z($Rt3vtBKU!I)~s~D?~!S*vOU$skG+9l zwDz=s`g0^}Z@LNk{ZSb2C?H*l!~{$By*DxDxhe>dPD(kawhM>mD8_5c=oSSpk=wAd z@7BG&QWro^+tO9mSQiL(qROdMJ)Xf-PGq^B)=hZ=$mf(JHDmIagS)kSqz zrO+A^Tvf@wP6R5Y3rN5Xi)H~Gi+;R{_@WY z{=+ceht=@G!FQ7%UDF=(EwVo}`n2gJjn?=evZ*Gj8~KN)DlX6W;^p0WelP|C5WNBV zC`rzVPZf~cRLU|#-|dtUzc~T{=ZIxr!NeyqM!4%hUktgr4`;c5tuqUd%nFvgGFj$p z_;!t3_5VL%s!E)Me=(xZf)K$|OjGikrBCIHT=!L>Q5q~$+%{5<=m{yux=Y%&a%_*D zw}Xr0&_tcvT!7v8PudJ~l>FLgBFlA8sOAw{!6YSr5#!ue>#yY^iTL+4r55{qC>XlFO8dbpjJ ztmRO~tWwG6#!%wa;fZGN%DNrd(p~Xpqog3^otal($oWQp+o)gBjD8wxdn}H(qr5>2 z9|vH%p7KTwZ)x;xa#s;p#7eAz<)-&xG>J*pft(gF%JxSu2qSP^6+teNN@}?4y zW8>AKoMHz>ml;1_XrFZH-hI=`(eDV~+HBBiD#}eia1~2d6N*(kik}&qy2c$H|HaMz z!Hr6x`@!C{>Hq_#*&v3~p!{y=8#;9&|6rBdW>lGd%I8a^$J|N(v3K=o`sXKPGRarC=1cVHMBD%qY=La?*yp`#)CfIHwpPe zT^izz@^#A05z3-qs%d;)HEe5rnc6&{7RQfEnVk3Itk`xR7<{b%Eg#uKZ$1}-=t=YA z#43oZ&xB->jX+rCaroU%5a9I_KTc{tPN(Y=1=OrQPZ2ih`!G?Bo@8tax#SSdEBU&v z=rJEMHt9Nk0x!>bG(0IouK6)(W^C_I-sg1GSksQZPVq3l(Q8sDXF6bn|KQw}MlPQ+ zlyi)7*HY0WHijts%TE%VLw8Ibr~RR7({)))Urhb$_uDQRUU%6Nl<~!ZCTrtPP2pm! zlf@>1n-zkKT8woL)4&pm#{ZV%CrUA0BaAK)!bSh~C$N>4vQrf^8)7|HKC^pB2s_ib zkYf&<-^1mIyFq(fD$S*xu&=t(WB!PU8K%Dj4<4hOq2em-#D;wl4dt|~SzH$py)u5L zz!v7p5|fKJ(lak1{ZTgwZT38|cvBw%uk}Vi zgyv9MoY?($wDETlm8_}DaW>4JBj6K317@tp#MGzQa)kK8QNelSv9(L?xuK6uYwj1o zB~6+|$1;^yg;}AC#?6j<bZ59Ic{=%V!^2c@zM9YMHO$JxPq?xR<8)55A9n-}kO zM|Q|{PwqAg+>WyGy5g4Q&CUqmxM0R1Z)FmZE^#NY1>pD?3y9%VPxVF9+C)=66ahIL zKXz6}pkFbJG%j8Fw5{A{*erQ zDXt}kF@#z@q@tZMutrhHXrr5|+2k$ry(3Vx6?R0OArPr3ggR3E6yrzYI+vBs*G^+f z_GEv8aKJyGj;(=r-Br=lPAj#%X}J?DP=A=;(Iv{DdSTP*QA|autQMKG(@L|KU|_jb z=QHWSdwy}3iipZ+1l{fx>8l-@$WXCzRkPDqx``-R9ZnS8C{^ zQx}=s+_yKRm?bHlLW7d|$1{Mtycy*e4MF*q1gUD~`;&w@5PwjB`)RTBDf*$JrXP&I z#Z}kRF%T{xtB_tb-*VJ&F1kzqW@cdlvbmA8`YC9YW7KV!;c%&YEigd3feFDA0!bEp zkNfuAN36NxfS{a8yIvU<@UsE-EOz(r9 z4C!|}moXX?lgAnzTlU?wnd>NY^e5Q~F!GFOq``|8@6Zy{ay zvpL*!+cf82awwnew6Ci1&8d_}oq5?A^){0x#j7Q^^BdO^gl|l^m}i_eQpCEH9A>{< zc>}ndArJZPX0bLw{(y=Ie|OjP43?W)?_#j-+=3Vf4`s~%V98}r*W+(V;_>(;Bri>K zCb@r8>RZbN@R$?lz!ODG`uE-nv&XBz_SY$1n?^bV0O0nucl>XJh1+A)`JNCVFepAS z+uuIBzUP850swz(=6@qB9E2I-Qy^-_&_QL0gfQw*KDIwK;3Wb8bcg`}`u|?MUc Date: Fri, 28 Mar 2025 14:54:04 +0000 Subject: [PATCH 351/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10228 -> 10218 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index fb148252d..db90a3960 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 14:41:42,046 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 14:41:42,046 submission_checker1.py:3260 INFO] --- -[2025-03-28 14:41:42,046 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 14:53:58,771 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 14:53:58,771 submission_checker1.py:3260 INFO] --- +[2025-03-28 14:53:58,771 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 14:41:42,046 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 14:53:58,771 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 14:41:42,047 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 14:53:58,771 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 14:41:42,047 submission_checker1.py:3276 INFO] --- -[2025-03-28 14:41:42,047 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 14:41:42,047 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 14:41:42,047 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 14:41:42,047 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 14:41:42,047 submission_checker1.py:3297 INFO] --- -[2025-03-28 14:41:42,047 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 14:41:43,330 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 14:53:58,771 submission_checker1.py:3276 INFO] --- +[2025-03-28 14:53:58,771 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 14:53:58,771 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 14:53:58,771 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 14:53:58,771 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 14:53:58,771 submission_checker1.py:3297 INFO] --- +[2025-03-28 14:53:58,771 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 14:54:00,212 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.371057 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73944 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73142 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421594 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26123 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.71607 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.395672 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2022 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0662 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e052697ff..c93af1a90 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -978 +979 diff --git a/summary.csv b/summary.csv index 350766806..1ccf0693b 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.371057,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73944,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73142,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421594,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26123,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.71607,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.395672,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2022,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0662,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 929d950b8..09ad9ed43 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73944, + "Result":1.73142, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2022, - "Accuracy":"acc: 75.200", + "Result":10.0662, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index a658975c4c227edd8c34fe6b856cc5c285bbebd6..c3414fb17f35350474903a6d1ec3c089c4f9159e 100644 GIT binary patch delta 5555 zcmX|_cQhQ#*T+{EiMC3D=(~sr61}autP-NFnpmPIYV=_BUU!AnS&1G*CpuA9^cGeL zqW6*rB3^mU`MvYUlrwWa=g!>k+%t1eb+&b?hKMM3p>$5J!~lRF5RD{fLLU;5Aqjm)yd;gafMs0TaSjE>vU)d=tcpyuO|lE*;!E9Z~QZu_B1~PsCW1; zH7$=oe187~kHqnI`Dnx@Z`C?Ya6n-NwZ`#Z+c%0HEr~Xg$ME&rE2^$PTwb3!vgo$M zwVi~~Si*=%*wdqF);g+vFiEoCkxr&4B}nvxK_X6elZuNpF`Z|2oO2~TR5VxIEUcU5V#?H54mdmO=52um6 zl9`i?WU)TSmT6ixGOl~nOabD?d%89@TWRcVt5Mokryg@PAD5_)d@?>Mtnmjye4Z-o zD(7ZB{&%e&;sn`jj8;E$Xs|Aa@*!g}J5kEAI5y_BhMX6ghqy!<^5Qe%;2`TnT5?P) zFFCPRg+4K%?Fs zoQGu^ydMJ#|i`EgFxXLYv-3C`KLX_5~9b#l09gfoZ5X{3gPI8`+UjZB@4Me+qV{PZETvW#1ji%h8gms z9A~KsN!on?W2J@^}&#OT)iq#Z*`tC-<- zg0TP6zwUL-BsCD2&_BA}PQ8*L<|Z{1F~5HraJf+007f=L1e@CgrG+UIJz!)N$K=E@ zRKW^koHTsW#Dvh9B~IxuDriG!;A|u|lt_Oem`A$5X-tY^$E=5+@8J10lCOEBe~Vh> zY;*~jz2^~da2Wb=xX*B7r<91YE17C5DBLSVA#a<8+<=uZxKx<~VFx+!E~d&4eJ$iQ zDilyGjr{M-EQ^SxB&5j%Kn+ob-x$wx%bFcc7_g25)DAW)U(O(mh_iO`Ly_@IB7)&7 zZ$gjvgmNP!$2E%qnj2vs`@^i6B$W7XPYW-pW}ZZ(i(n*{R@%RBzif-rtE(n4dE?G8 zZe?@GK13TXfst*lf5O;X!_OVz^JtA%&i4UwO-8RNL-iCWQADdcpoh~tn)5L#;x=TA zd&9gpqi6`dDB!w$yPPBO?RIOZP{dnA+@17uQvRB<*2z8+nDMG5Q+dB<2i&1EKCb79 z?&n7{{L@)3lQSWyKi|EcPBS3SaM=d}Rl-VS&OKT5`NbtlK`wo&tGUlKiHPX|0024F zO(hQ)pV155rnniR1IbO0A6qZE0`^n=@PakM^nYbEXvh~ziB>sJK_$cmgl$v7FJ%Q?m zEarn6sio*1&(@6nx2O%>(LJHiLUe@q)1t?F33hdc5m&`p39iETkY^zhmM)JeZ3p<( z>9v~G=l=MJ=JeGSg#MM3Oa0jw71C+BXS|pDr!d=ebrWH7_EYjVqf3Bzb%j4=w|un_ z`(>n<2wGg>(s?$nrhk(xbl(b<6}bcj~p6S@>>SI`1?w$Z_9W)vV?L zD!wr{uDYk!e*S=L7O5~3x5%a4wdGQ#?3j&UD9!RLe=@~0SNU+zqc%G`@c@lroiLti ze!ZOOJT504)#ajETh5u|T_aNPt4=%eL)yE*ftZ?lv1yWxGZNLg#Mh_Ji5X<^c!48R zH%?_io6dIvjf?#D@Aqa!HBM9=>_eOI5pN8P>+N}xB|oDyxsYz&U?Y{4J&lU!o+X%9 zGycbdlM*4gIhJBKyw9((dEu6wvFV=nc zbh1!RuKp-1=GAYph9<8Mw8G!3nF z7`%3$^F)4fOf!A-csmtd@8Y&?uALn_8U1QcOvBvmLv9{%F*i`1H5s-YrDC3cut2yC zYy7=>^@*b;e7TKWeu3(4!DO$iX4Z&Kg$k-CieLU_w@Po;;ZZ+aJtp>S&P3jB?G5rz zEeeyk|CPH{u(-JnpUG}+nla;ZdZ+LyrXoe*2!~uy`m{9_Fw!6u&Bi;X4E#cJ$_%dP z*d+!la3#Zl(JCky2&GaPq;m6G5FmH4luX)WNQzok!1Aht9$P70#Q|+7hSJXtIREiD zj^^;`^qHw%AG1CbkLaG#LJ7uRr8E*^=~4hODqOlCU^JJ8zwX%Eb^sJ8=;BAwWrl16 ztQj!Ys*|`s;D0H7uRyDb5gHG6G5_P}km>c-vPEJ`P-NbBc;rPYe5@5noO5YLeBn7Y zX{J9*6@>diEI3FKvNGSs!)%o2rX&jBoPbfJt6ITC+xkxe3oJRRiBUaR;*Ud}l(Emo zdEW8qYX{Oyb6OtjuG*!VLPVwU+sKX+g=0(pudcgfowBVQ1(Jd-{Bqh z9PxDDXJuG&m4-J)zLWuKO*U2#(}|Jui~%qzLHup-ki8QXBsn)E3I z(TTxsHl40Q^v~AX*nZHhpsykSd`R}hMm^JjkcFLQ%2F4=BSJ)g`Bg)z|2p)KB-+rU zk{)QC93E*%sKuD> zlXChkOG!PmPS4py$skgN78G4TwlDCoeh0Zrb8fV21|RR#d*=Mv$D^j=IQPs)>KXP{ zPcnqnzz7r z=g)WY{R;zgZr8oq{FHKl@in-RGQb-3B&YtP72Q~f16JJ7_@jH@889*--x?F3-mA(heqm(qIo@EQ$SE1x&~v)<&uaFXoKDpALh{*BrBopo2Rp#%^)+E0XJpwHmf*3FV zwtG@w4{_M3Mro&Jh&C)hX1jAgT6i6V%zh=Hd(NG3dO;J<+X_($MWh~H2vY{Bivnzu zXClD>;n)@ka|`5Otf3L^+_};G@Qvn&%fIcMoCTe04MFbu;dg7YL|7#@?mnNxDbmFw zxf47tZ)giQH2YN7OJT}9bCLAv1uD`vfV* zU@+dU_iUdzK)|M?#hcn*hD<{=UUGa>S+q^`f`(Gbq-(QV6aaQeoPRh5Rb_|YQZTVT z9YdT3qr^*8(o^39vml~(kX0Zr4K?x}Aa&N_*pB%T-x2`w zb)g2FXK6vs6)toLkDE;WYin-5$A@``h}OUGkfg*~x&pO$J|V|$Pb`~4oi&VHkg`UX zGi7(dkp78_E)!%{>ukP^N$d1urjZEMKO5WO$=biVU?Q5+ELlQ4!O8x)WRZ2>opW;X z%WK36kP^nN{Not4Rbgh{C8IjKgHm&?X0?f`YxmjWFR%Y>aQU2L&$?!jV{cwuS#28i z<|lhyy~z0GYy#E%6)d*1iCAPM&21iYm5q&87lz4XBs?98f(zxE2WoenAM0Q6?!Cf* zmQVlU`8I!r@?EAb(%K0u^D?x)IJxnv4JUlkkc{V0+v z&y*wfHC9n?lipic$axh={aE`Xst`(5)T)MPbqSTfh;swgukY5Lb@=*P-4pN+ihI(^ z$9XrkvG@t@OZyZKVM8KnMGZj8YEoKdR3wjq4)Uvt-vrLpXm#ps477{gX&~=~yNucm zRR;enMJ!qH3LLF-0x(cTqh$`{J3^Z+3VV~4l-mM!oP zBcY}SOUt@kWoaf|QU*@lvl%=Op7S_#tvyYQQ=V3>{PG-Tmjj2A$1$L;k9DFS-{B08 zarb@vj&URwbN&7u{p3!_U}WP<8c*Apyz+<48iZ;c;eG=n&wks#OmKEAbg&}Prf;0; z<;Ioto4L`c>Vtcy)~foE)k>iLZA(RELC0rh91$8R9QKwbuht5p41QM`(=9M)b4Qi* z|9~3*aI1boWqbH(fXc8(vBK|xM)R~DzjM5hcw1Rvoy}Z@RRO-`>2i2ctCEAy?3{QD z@rQ1X2T7%Hm`?lGHtbuz1zW5`UlQ5g}$+H;QB?Igy5EJaF&b!j*$UD#X{J{+;RlPmp^y*iZ z?JmU>Bx;FOMo)B68Vn~jV9g>1YSejR}`DseUml}2v)#QOuRx`$rah3W)1hjvSBfJ0fa#*^)rA*NC1<&U$F7w>Y ztC?18E%z*h1p8(P-99A#SYESk#!$Q14&B*|8JIXD8hP@u<2b-*S1va$XGkTk6yhek z#F?{Y+$Nix75nm>wR;)O{P;d;0{vN0)2G=Qc?ssYqfTT^JCh|jNl@|G0uLI@#g5eO z7N&uo`9psMtjO~5!%-SR%~&81YOpKtHp8@sdN!gKnTW1+c$T;=a-(W)IRw6z0Z4X_ zh7hl@4x{emwz8FbNU)Hubv+1|zt`v>7ma><&||&ZEVvcZ$$V_*Q{xxF4v!Dt*7*XM z0^5a|8%$#TDdFOfNocG<;I@78i!XdUdO~&)Y1poqoXgDD~`Ur;rvEaT2 zriMRzL8%0HEDkXbPT;T-FUE{B+G+aaXhWHzc#vX^6N4-9+42O9zUj`E{MiqoSq&FL zmKSl?FJ9D6`fHc;j_tfpOq+KaNMM2d|ROAJbWErzDhk4XpLZxl#$Kp6<>R43mh zD@iH8vqDv$D{z}(M7+1Fy-P;|rN;h=;yA&MfX@^4NKU3;QU9j%P>hub|9t4S3U_` z@Sxy^)28`B4UHXReC?-W3yR{nBZVXcMBS&0oyCqI_QXu3ira5fg%Ef0ZMeLAZ7(72 z_hl+vY@QiDY^=IHk`TXz1TS3dZ;ow48OOeg# zEfV|a&8YCq$n@S74HMlS*?gYiOoCU~t8eBC6jnV;w`4LbIe&b8Md@EGnjaXsVW+}7 zI4B1_mr0!L?aFV45VquORm2@@CW*b_{hATqaaV162K`fBO;I1s>6JQ}g;MG^u%_&a zR_q-+b~}U*E@hd;Yzh(@n2jY^K-@l1(%KdJ(ImYaoI@|1WxRRWa6R%oqqo1~#zq-} z3mX^;Yo+1LT)1`E;JTLPQ2cnejC9fM-JE3fteZMNh&Av~LwH-5ds|p|TQRWQ6E)wY zB&24>4k=J%)AlCBjjsHv=uOBR?M`r8`#_0+pr;u}!#lyN&`W;0Dat^5`RlJg+TOXS z2?7XabP1S!P!B&m%KkLmYp}id0f)2$MdlRp?oGIS-Yx~9{ypjkMK&u3xl`XVvivYt z;rpn;%fJXJk!j|7>rz~4NicyR*FEek?N#o_W>3ql0@prrC%d(7%<@C4YY*H@`W?7@ zupUA&IBv=NAhxn8CAZ;ND4Vj8jWe-ik+59QcqowZ(+DGvriUpa=1;`O_$yh}$pPA; z^ZpR%$-8#Mvx$s)nj*TY&A<{<4|AfuUbgct!dTa_Lq@hxWm>~!@;En z04M;2007__06N?Q+b6!CqXP*|5Yx*`W(k=lUwAUdY$xT(8-kNNT^abzj*7x7^p$I~vH0-G*$ zcc_-D*=H5$>GfFizJu0TO3KgZydR=BWN$9m>Bv=J0kivpMGrOCHrslT@Cn{kBV~;0}T-rTa#@4q8rw+p0i1VtnI5lLLj@AlFccRO#)wd zfC$bxzNY>`x98a014{4cwz~_X2UTc{;`u9MsAZB(|IvN0)=)-G(^__>)4f?3Euh~g z+yLs;ESC6mHrsQkpVe7~gajV)LO94~896AwwQ>Vpj85X9xLSuu$kAB{w2S@>1?|fB zphRBYY3dYokGWAHkW$ULQ4ppjGE`LIbvm?b>?tSkwRal@vQtYcGWQi3SM?@QB+w}5 z704+%V1)u%h7m$V1+3(EJ?y=^Z3(|oxRDQ_*zW5MJ(|9Cz)($=yE>wekz{iHX=FWp z^xI;pNqiL2Z|N?7{Omk*@EBQOl_?cYjmej?ASLbXcRWTkEI%lX-Ql|R&8YMqH#%iw zSbXmg#-=*s(yjsQ>eu+^63pB?H_qGorD4yW6B%mtrO(;crM#!~q8}_ryyi_rB{}Vq z-lDOUxg6L!lPP@BF}MH7^n0TB5SBAs;`B|=xuxMnipwixm4+_MT_0ya?3NLpu15JT@|i168Isl zwn|4g>tQ_Jst^fA>2l}IXDL-TlBO=qRev%xk#20$Aqk}CIJ`)VVv2KbZrKm87j6kO zDV}F6ksog3wVq^LTHI~clf(o(DE_hqp`cty2nXzp7|OMp*cEz28-atg#6r@MaRnqofSf#&BOslU(JcQzEWnST zVqE0R@^H~?eZ9-3pb|W;ARa^0-%}CEcM$^B{?D&S~9Du|bj%cDonifkInl zMu5ws1$OOprgpopR@!;T8II=dJw#)Gcj}yvZ5jI6$1XWWu45xmeLyI646t5;)!iyQ zMPdh%?s*YbO-6*qFg&Z9JV_ayLl|O>;Pgj+5$UX9#XUU>f4}x)%W$g28+>C85-6|6 zO-|hl`BYuMTj}{dYaus!#qF7)pYKv5cS_l7W+e9)HF9GhsTE8-nA_PZXwear!i8mJ zl8{%t5!As~Uk~FQNRqr^t5EFRfb>H|^xO@On7%A#@Q8^tm>6lX)kQN5BEH`rP=5Y& zyLlTdZ4@8vAX6<8d}ON<=bnp&f(O3$R3W!fbarL}A!$b!-YQr>i*WwZ$4> zcH4n_GGSlK1sfLGq!;*qtUR!%d6M#_y_scp%VyktK5l>=eHh0gu;^WH0ppoVJL5_( zQxHYs_i-_>Q9b#bvLexJ-sV4&Z z1fiqzftT_nu1;u8!gSFJ#g13QT;;Ea>+i_FRTT8-dX+TDIQ&L5u5Y2z*X48^d8^^~ zq-FK~Kk0cT+eBP>dxg;Htxf__JGD%2?6<5GQ`R~Fo>LsGw2L}?z5VbTDfGQIqG4}S zVMgh1mRoX}!s4DEMshNPA?vUhvN1L^w>nrIM;*xc{_I^jM=seOguy2oSWj*2gUGv& zqmI4abD}rQ4EXv$l=y6RIlqt$Uvf; zhJ+uFz=N|pJ5d$iWSXi{(gMPo41H9^?^vL*^1;CLI4jUfM5 zv0b_V7FJ+H3by?TmYPEMBi62y?YG1^*Uyz!HzymlMRw zGqJ^tSz$u%jU1jw^oA9PFJO|wU|8F1(=@W8xFyzbZT{2kt5$P{8@M(5^wj*5*C=E! zit|?PUIlIBV{94RH*)GYjzH2plMReD5nza;Z3J;IEAjS2MR0qI8sXlkh!vVX$ZEP` z_UN?GRn7@ho5-Oyl(7((yMHk!%130wUe!)h9N{YWLObh{h5W}CK=m$r9R zt)Z&B0(hI1GrhVg18=cI8olx$D^Z*+_k`7|?g=Mx|8q}RJMp-0323o#Zk-KCQnF+v zwfuX%Bqtbu`#wUk6QH(r5t8iM?Vj6Pt&UHR+J*Vr3Y34E#D24|i?)=y&KWjq2TMZ| z2@as8{AgR^Aph5$S^sp_PV4wsvs;~sbGs8e7tBbUv>9}|%x;z6Kif4?XCrHtx=1Cz! z()g+9c$&((2m|kmQH#S!g}HjO%0%@aPwBF5*PCq)zpDaF#~fnJ%ktLdr%_KnDkVr* z;+@A}F;A{i-!E`A*}Sfd?klf{-L` zzGWL&E>6V@6H))QVjF3DDl$^|2fOYaRT>u2D3Vb zblpdHLNoXRs(I{h0f-+?HWU-y?~{5=$npG>k0&fws09sZ|CqHqMq0U^N8r$s69oDt zM<$Sv$T879sm6JTWn2ILR3a#~JFB5xx|)C0O&sQ^irmT&7{V;{h%cWc@?a&ycYHOT5RUq0)TDsnqBU3}-gWs*+t&G^xk@LdvJ7yt<3OMR&4 zbj%Tdsb#9F&@%ibFiHmB+M$GS;6qUe*DDh9vecXR;qemodnNW6Jcd6YA(#a}s5Fhg zfO|*T$F9%KOq&R&j2K*#XM#@Olt`VqA==SETpBh%;IpN==Dy(a5Z)E5j~y;GZr{yC~Q0+9-vEkvZ|dDC&> z=}r;v^rv1Kd6WAa{<+<8?m9Vm%ST_;<0yLlHr#w>hqD(kuIx_)wUGZhnzt0oRBc-8 z6XfOlxShAJz1ao$ew0dHQ+s6!-h0mYLl@Ax>{}f(dN73PKev(2{GJ+9_i=8sXxZ(9 z1*XQGa^z}re)?&=ahoiaR+ELRPe6~vm`3=t;CWEg=wY<}w-S@i9cQ!seC9}s9;%J` zW^2skPw3BeiaP3~E`SlV|$ni23f(so@z<8FGGgK}FOg>&11% z2kloy;(c>>dq>02sbY6Eqg%t;mD~b~JS#jQdO=BJ%*G6KjQ@^xA*UiEE)9HV?(e}7 z_+qOCP3bFpl*?a(J}Lu}Bc8OR?l)=0(o(GyY5jJ#o}K9@;N(N3!tFQVX$Ls3Z*Ei( zvA-@6#X;0P>pf?e-T(H{H(j2GR_i(%;W8(ueJIzcnDfP@`;}ncCm!dmjvl;9sJ%9I zJ0dXoUfXqOPFag~#wo?x>_%95u6_}*O`JN~T;lGi~ir%W}NKyZ)*)#DEVU2mRdelA(U zY#?nVaigCnMSk;oZBzBqJrKy>c-iR9t+l-S^m>`3=kn;;|0df1n`fi2!#^nmbkD*sLeFAn%Pm!kd!)Od^|gWC$8*VUUzTf@S{{5I@rh} z?pL;$FTTNG%YMO&YE4IjLP3}gZnV=QeX~N$5Wi(!pZ^D`gZfYg=5P6SUPg}<(;APQ z8?g59K7)djIKNLiZnv=0$u#1?EPzb>QD$!3=lS(Q5{M-D1nsMw5_03v(yV~!_;0i7 z5OEIa%!IApH#os!B+O9ltE_}<)d}ES8mKdv#@0ZCH zC?`i!fOB%gDTuXHDN9(w1UB$K>1yWnhQ=Q}P%rx*o*b;w#irc6t2#5zci4D;%ctl~ zZUx5w$`$$m0Ns57<;17s{&(qr0BI9>bN~PV diff --git a/summary_results.json b/summary_results.json index 8dbfeb66b..f6b446616 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2022, + "Performance_Result": 10.0662, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73944, + "Performance_Result": 1.73142, "Performance_Units": "Samples/s" }, { From 2f5b7beff1bcb33c25f1831afa10d2fab78a73ab Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 15:00:41 +0000 Subject: [PATCH 352/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 10218 -> 10220 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index db90a3960..0cd7552ec 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 14:53:58,771 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 14:53:58,771 submission_checker1.py:3260 INFO] --- -[2025-03-28 14:53:58,771 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 15:00:36,275 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 15:00:36,275 submission_checker1.py:3260 INFO] --- +[2025-03-28 15:00:36,275 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 14:53:58,771 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 15:00:36,275 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 14:53:58,771 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 15:00:36,275 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 14:53:58,771 submission_checker1.py:3276 INFO] --- -[2025-03-28 14:53:58,771 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 14:53:58,771 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 14:53:58,771 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 14:53:58,771 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 14:53:58,771 submission_checker1.py:3297 INFO] --- -[2025-03-28 14:53:58,771 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 14:54:00,212 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 15:00:36,275 submission_checker1.py:3276 INFO] --- +[2025-03-28 15:00:36,275 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 15:00:36,275 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 15:00:36,275 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 15:00:36,275 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 15:00:36,275 submission_checker1.py:3297 INFO] --- +[2025-03-28 15:00:36,275 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 15:00:37,298 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.71607 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.395672 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0662 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2684 | acc: 76.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.118 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436676 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419601 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index c93af1a90..da8e2a42b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -979 +980 diff --git a/summary.csv b/summary.csv index 1ccf0693b..e9e0cd937 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.71607,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.395672,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0662,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2684,acc: 76.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.118,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436676,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419601,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 09ad9ed43..d404b5838 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0662, - "Accuracy":"acc: 76.000", + "Result":10.2684, + "Accuracy":"acc: 76.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -634,12 +634,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419601, + "Result":0.417541, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c3414fb17f35350474903a6d1ec3c089c4f9159e..2964fa3c2c8e07d4d2c43e9afb6b2f3e424b666a 100644 GIT binary patch delta 5035 zcmY+IcTf{bw1*QqNHa7IJ%j`VqzFinDnde2kfJo{(tB?KL3&F9L69P$2!gcG5fJH! zQX+UE(z`ST>4*>S%zg9T&g|?TyL;yMojqq~&iQ_S>we8Aa=N3YIWlhw03ZZ}4X3$| z{Xq_=s00pMV>u}}Q93DbL!Y5N^hUiqkuP`vl2fg@I=4^6d4eq{f?112;R5@aUi+KJ z)eE;oI`?CkZ4l)0yx9rNz3=OSUZji8yPKq2C?`D5rc|HBi6n6d(&BS6N^q{E{7g#e z+niUT_wKvZ$d&TTJKAKTEu=NIF;wcYBP?cKzIT$#!boOVb9Ojd+RSJgC4bA z#9^LZ>`XIv?L`$sS)mw835LumOzrOXh38y#Y!lR7c zSOx8ct5GS*L*uQbXi9Mbf`HN}0y9GZKX}JlYX7MgTitV;yz6DW&id zmbkV1 zJapDt{1;=#wVfzQDf?A3&a&aELqlE;KAV)ddo_@+YAtyA+lHWz8rzK)6+^3GMIG#w zW!3hC4tw<)H{YyuK1fL`_DPeZovm+?_FjOS^abq$9dZg*002O9IaL7jsf5)vIskwf z8%AS}YHKPUnZd9t93OZnNP0vI@x&3jbKbm^5qNR%;UY-(PNk(~2Wj6$GuQ9w?;U6^ zWn9Cqn#I|{x(fB&hxN1|R*D2P>_u-}g8^pvh%=@6HeKmVGtcMf$E^;`nnVvwL`rN* z5D(?dg_M(mvlGz|Oxy`FF?aYUZD^T>I@((t6~uDLPPE2VVl=vnUlw<#X)Og?C@QR{ z5#1jvX|mEYy7s&LS-!pf1*}E=v7@@bt0`!cfX>{Lq5m_`5w^uUIjlPk*lt^q_zWrH zrEg6_`tU@YF(=0rL1)|p*vGHHf9^f8Q_w#usk%*f9aUq@Yry8`=oBw%`L^h`y@OXJ&E3KCl#%#Ey?PQWv<`aAL7Y=;x|s4B{Kga*z!j)P`||{Jf3H^LhF18w~mYZ($VgZFT2O zbC{9$LD^y}qV{2VuSf%*Rg$*BLXtMGv4!%RX7%TH%I?Vr+-Ptv>6hhs=*`#(19^YD zAFr`hb6W!v8Lk7JtXk$q)8x;xB+PG+qvLO_3!vjI=O7^ObA1DR@2opZ!ar@Og3Ajt zjOh6Gvus4qtdanzHn5X9q5HmET!9xOv}T1i4rH3|CFGnKGRK@yb|i+551A|M;W)ea z-5ztUcCH%2yf)DP?8oHq#xvUQMzZyT#~RcH_PW$|%gOuK1&3JUbCu}BynZbl5t`Pv z)AEQ}AA29j6H+1DSN?LJ2l`}{B$ih8CB8?S?z89e$3YRn~@4f8PC#buaPSYGOq z+XlnLRfzrFUyg0_SGK@GNX!&r^AB^Akttf?EKGm?N_F z#k6^aN63Pk1kYqd`9XBCVt!A9DRl}kyQ(sb zSo6i82p>TVRMYR|FKy1PCUt0B(Mu2Uw)`m_@h2wbhq1!x`@6_%2b#B=T6eyDYM)hi zuAoRX&M+kF6$!k8>Mb=rQkq+ZhU986Az9(0t{Kf_1eOdyf(8T$1tB451gM=1glvY^ z^1XY$po%wDg~XC4Rs(-a0o4VAk&VHuYVjr#l~G_!rgw59jJ z2gB9uP0mdmeWBPLsqVbB^DTd8wv|lPAK&KGN2@DJXLCLV&=|qAyrynmV|9851Dn=( zt4pi$6A|>;nm!2Wj)DD%A{%HO1*jKK@%B?6L-Oc2EKlg+=LqIWew#h3rKdQvS&#zh z#(Xe}??i&bQSLC>IzK<%fa}}0H?3LvaVqo+~PzMhbO`xjLU>Hv0Waz~}0istx^%-ILUUXva;|WqL3@S96qZ0B4 zQdfsF%-QZ}-E1lyq}O$yoM0KK#P><lCSr2TB=4+kb~hA<6g%d}^+QpHaF~mpIH)4v*qux@L@UX3Bi>mn?Hv9NA=W zEnTfV|E$`MdrYSe@ncoEl}wID#{vYim%EDLgBlDNq_iq*43J&cRpLg7*TtX!1EPr0 zX}2)GhmG|}_|O-O?)Aw+n@HHzh1{ugt-|vB`kc4;K0{we-_Xy$zNfmcSlaKsKIy09 z#gOu$EHYK%|HtTZmvXu@8yX1$WK+_by-Z=UAHvCQ#H0&%D?sVjbTQyH-OWFM;TtZZ zGEdFgd`mqQJy1N^@;fZg1IS2O_V~VAg4tziQVCQ8P%s~h45HLh1BL|HvqPQSWtG!~ z&8Lf5f)0TO-0YOrQYy@m&MK}Fz1Vc#r*(;Y+LF$@QX4mjpWBYGcB2OCzrBj@#yQ<_ z9^(hw*To>?oFkF8(dZ5d<7J{v9->I!;2Dmrg0a3Hb;<0eE!0(Yw>wfBf}v`=WnAI_ zFY);)lSj|tDoTkN-1d`y;>c36w|PqYK>6X(=a%Wl-Lh7aQH#hw+-IpL@1Z#=TpM%z z;QQd9c&cnsn*|%NHO9$e&>o!>CkTKgh27DfcGE8`(Nl5vt1Ley*qiGJVQ^vZ?rsUE9KE?Y0T6rDYz?mdXLL_^HSYu zNIX@!=)#r__!Y(}atPwfqysQwpF)MQ9*4fV6vvMv`*EBuQO_!YWqL$+BjAJ&r!x2O zhi;y7mJ8|o(wBy!#t7@WGjn;YZ~2(aBBFw6ENJ}uUFr9NM8wU@tQ(ZZ=vdkx*w8|= z?$MfoE0zDI+&ot2-voFfbyd{Xzt^gu|NXh}$R%;zu)2vSDZIW46^hG>D;-i}NfAh` zKVwdTZ!2p=JK<{1Bp6;Hr6V2DGnY=3PTEoiZ~beLS-NZI?gwkZkA~gaCZ=15C!qsj zr2z$3+TD+krUbqICf|ts3V+5NCvNZJxl12z9PxuS2r+YU&Hz0;P&(;cSrqu{@$%7> zTbL@#xpFazB`ko}h8s-fCe_U}oWsT$5s~o;uGC&xJYYyzZCrx4{U#io?N~~^l()zG z_Mjtp@jw3>P9FwE@RkDq#;WMvSq+7%BYI9NdXC!cGDg*5$Ki-;`Z>cpz4`_;!u~*eEhG(h5sbdtNfK{e>`S=9CevK6mFEv z6E+5_j@LWWs{YyCzn1-{qWTr)R9mfO^Po1PWN%jW)i#&V*zH9`HPhJf_~u<{8VA~H zCL$oi{?6XoNbA66w*pWo>76q8-M=mrB4Ui5O~u6mal_l>D3Phe8RayaYwmL6HNH*G!uf@uEJbPF|;~ zIxqe3y5=BSeO@q?hSX~&=9Ao9xQMj{{5VAAs5A?0{vB1s@`s~7EvtGq4u zM3+q9M45@vwQJ^%lKSIXnIhEfN9|JMFSM2*-H$% zET@F9Fg8)tq6nMRJ;5k#hAbm8^R0*4wWKb^j<aZDzLk2cF$YCLB2L?i^a|hs1m5JeMB4CAY0awi(WOA!(f~P%C5i zx}ExS5=*yr7kP2UMldkDMk6@e@aBcsS1J+hSNx#>)V#ATGtn8zNGI)t z)f|yeN2EUK**y#_r^(;c-Mpm~LwG5qy+}X(!~KF>$`JuK8?8sY>XzUR6asmx5hYK_ zPR+DXkec;MI=hR8cApG*@Ci4MV4X{5U+�*<5$0+)?A)9*fyu19HXgzs$Zp_gj6O zP5u7S%@gs9jBC*VkdSZ|$3y3zkhJ6bX4-1)pIgG>6{&2VmeLHpDUO1A(OZ;dohQen z$vSS9I_AJk<~5>@NI?jBwqOyD4`mfS&8=K3DCe9cipMIV&hPIrxCm-INCZ)c3Eg-b z#EPcl!ExdhKRDA|Wy^j?zbATG0CmwT%T5(0$_*I5)j*n>srSD9xiabdr5>kCGIsOJ zYNCDjqux}ALEM}aoAt`?8kEv-f^K6l>%`QLDcI!C?+urpz3oup>(zZuQfHwt7GF06 z`cdPW!L$fV)wa=BGbvm(W_A7%zJZcm@PV#YS5WdOi>m&Exry-p1I~FvK*wxgP15N0 zr=!6GTlts6SxJP7sg>ec_cxYDwH{YTPFAKTmC_z{-eI}lWa;J~(kDBn7c(h(5D`DR zn`rdD^l|rwi}_X|cN}vc%d$eNgLWzC5f#dCGUNSHa=W46f?*t$nA;J$}_LIy0$99yL3I z)o8r&xk{tbV;OEq|MQl;S&ie4;ZV51PSCuHu`PVTK#=FcB}9D6vj8jx>p&L#n)pyt zjb+j_ytsKbJai^OqFy{c$-Ds1{M14H7E10<+4!0CcV#TwX&9OfjE$ z!d-Cw$vS5YEdIJW+)4)xg_|;=9D11v0)5wG#&@Yvdi}&(e!#Uw81R$lF52M~^S*r< z)1VT_RKLkXeTXkTwK9L^AU?vtE5q=lPOEH?@p*5S*qcq=5l8sIo zcK5{Z+zn=HbP&HCBI~W!wmZl&xXg`n}YGvkfIuwzo-@pi?!1_ zFU6awAY8aR!`a~8pn>xKiQx#vwlcFFP98B|+*4+6SNKJNcwewr z-U7QwM{0dsEnLL;EX_|`zj71t14-e0zvb=NO$r^!!!YW^6Hx^})}}sTcH0dzl-u<` zj$-rA(&bH@{KBJQ)al=$!u-kf} z>OfR~r#%Z>yL(nCuH`yhurzY;k<{#R<6yyv>%l`>{T6ceWY-&EmOb76!*Tw1ewsWB z;=)I)ZPyRfGHHOwNA125_6(GjJOxe4gb|WD$ZYl`J>{a9QJ)L{Yr#Ym@C`8?#WE6U zw?n!=dJZI52*i+4{@WVEt_Z4e{%@zk2mrWt*}M2}L`FGc)nNC4(bzZ`g!6y@el`H$ zZ;Oc(|FW1N@I_xvJGSCX!1s9-V I7W}*QKdq^L^8f$< delta 5015 zcmY*d2QVD$^FCc9+NmM>9U`Ll-rXTY$*GAF?IddScB1!kC!Ed^qC`6c(TOfP;gsmT z3m;Pc^38m|ng7nbv$ONQ^USle&$~0b`+B>2)guIC`|32#ZiE1UKM)m6dLMO6z(Y6& zuz!tWCS*p8#`ARfYQ?4O)HzSGt3wLwOcI9Mw@Mx_i#3wQ^7cC_YivGR*_=JG>~h7b_3WyDZzbkzEyljXiUn;}&cVjd0=aJbG5u7bIgjZmk<9?m%IDf{E$-mZ{e zG2~2ra7ZHDWTS{Gfm)+hssGpynYQqNvd~Du@Cv~g-@|p5C5o22E*58@gbe&DBvy|89zttDA}vDx>nWAdcp^E zBl!teY&y&Bay_PLkG~jJIh%)48+>9*+ZCzU+p!&S2p?L^HZQlO4ln!YI3R!#o+cYA zd)7ua*kyaN-q6~xXm<2FE}7oFS)?6JO|6{ijv6Iy#-l^Uk}x5P@a0E|PO(ct+FRit z`@?PTORDhQoe^5r$U2M65JgKaueN{Rec2XmP+v=A`o@EO!p82HZG<{P5-r!QL67ylpTc`SnASP?p_bd9nI$-e5g!mp2{m+kQ`DU_Rr)ERbe!P1< zlWs(s>3RqRs)d)zUU@Ma@<~XRfn57E*7BZe6A;n@007e4qX?LoH3-`!yNysmq~?el z&Y;6I|AH{fqhgvhiRe{#DcOqBW>e8nQ^EZthwkqA>5b3wRrWnlpP3`wE>ky?8`hqB z%cS3LjvvSWj@DF8W`u{D2AVK~2Ljv2qXP=9<`ec-0l5MNtw;H!gGW*#2#jutOVU zcGR*0;o1r>G-~9livJ4e#qU=-eLE(*nZgH4EHjznYq-X|_}ZRc$Au%3Ii*=d{1T^b z*N$tss#A^?U0JqQg~&AbeAS~N&$^tPq$89S%cRM4^Xru?mkD{9=q^`{x(be5AFOEM zMZIp+`}B9g1F_f!@fo76OCpW=q}S&zNtq-GIQ|necMeqnyUus~jZ1uvMF(?YT4x$? z$FL?`221$B~VHjnkw{PvWa& zeP+S%(A(eXB-f0pN?lKXB`#N?tTE8YJ%Y^7AkJg|e5zPpzTqT0_SHY)DbZ)aDj=JV zC8`*}#p4u{&qyCNPG5gLP-qFI8fRsGk{V}Zn>uLqXAVX!ZrqtBhH(+0r^c!ANfl({ zsrWY(<)kjWiGVx^?v(x+`iA}OV?n9Z#?w$$sNwk~c}n!}HZX8}*@yS!%eVIS^kl_o zHVFAD)0GL;QnRABqH@k)s02Lk!*^8tj#YfNyF#H_27U@QDv~l$;b-v0Ma!?6H`g9b zjIoo`V{X~;^p<%L*5K;CYoVJH zHx={hKwQhh{e50O;Rh}tlqChS8?9zhaI}cO3u*jk?N<$ZOTp>8nM42Sip{+dIYnSz)MZnxtM zp>%4^H|(}g(ka)cd=b!Ya4U#vroh@G7sQF? z{Umezpk^TxEGkfMYODvq#ww&`2+ShL4flAydX0K{9J0_YyUD;Bqh87ab*36C32DSh zxyJ!$H3BOPap#n}6_OBOuN*qF`JDh=Tl9pF!)maa;!c=DDS!Qfvh-~(?F_?w*Cv}M zyUo^F=Frb@S-vQpHc9^qx1n)waE7h<1j-geJOE;1M>&S*OnujCFjeD*A^^7ZAV+5k zh2)3Yea|T)J6CT_PeHw#RG@3 z!u0+CqdzDM#SnLZm;k8#T>>C){sQR z4w_|z&&)iS9-H zet=Qt)|T=_6Qe<_jBqkTb#4||jOPw5*zw`vdoE&hg=`~=_SkO&Bvy;&0Yp=`$Y>3t$W|uo3C6;}D^6qo!;>jVJQU`O^owCV- zIVlQ=a!eio{$?r0owIplxT zu(wZzxH8^1hfQ=EJazf(>xr#A&AarKeu}x%lcKX;#^ey^^Lw0C zYXJs16_b$OS%$oDEDS%@3df<}=nL*;v1ozqFI?>v1QrM9-mQPNU6Xo*_A|PcHo_S9 zq@?{tE4gJj2&}wC{zA{z zbGpS$mRmZqW#D`hnBD9>HIt<6jo@+c5D+)hc(8NE!K8N`7){)0%QOI{YviCtAOi2a>%%E^C)KyMmkg3pfa-!1Ii{g)*NO@#vJ9(^spIKQpe z7g%0<)UdG#;+rQrp z1Z1k)=iQ1*4AHr-%Ga5rO?oSOE0k2I)kMX&{j)+x1^8YqQa3GA?9-ycTzB3_OYfu5 zxnVq-=Uj>B*Hj5StqN*kR%yrALgXP(F@Sx_Y!nzE6xX7_)S~dW*H8&{?%its>#gR$ zR(#t#y9~L~8Bt*H$L(XYMOh@b7@p4$D$^uzB_cd;ZqXKR(HzsU{+AcybXh^HTxSSzI-`kpgQtf~LHu^5Ek2ZvvLsq!2~rc= zs$y+o*Hq*xrd`|JVgN8aY2nehx&|BUj*_YE`MA~jTckv(T1HwCFk3MmNwt_V5fo7r*(utNXjb(bk3yLGNW*0gm-`2J|5 z#?P(Yh!ow6E{Lf140E<1_uG`fJd&vT?=HEy1r=EFLWHDAE8hfKb4`eeXW6*k;i$|) zr&)8d`j5vPik*f*$NSfd%nKJ&?TfeKd)rn^EW~-u<8E?s2~Z)3Y-Zw3 z*VU=vHP68-G-&1gH;#AvB8>MYZHe|?mr&CmB>BK5I9Kxz5jKnljdv$wwoT>D?es*B z0@3j!0!vhiKY*T=t{}Cn;)1juddvFueX3vA`RC>x)Ji0c&gzaHZOMBDnU9BQvcFuj z_|;#1Ffmv!k*~;-C-gH>gSN>WEH3803Z@j+J&P_@rzmOFv}$z?Q@oCM2Q_T&*Ijn_ z`Pne?2ZqFpwDNK=q&0pJ8T`^dJ!oY|BxXYiK*(v6+hkUzjDwB}sz1C5p2zBR8f*=; zi{JZ1+6!|Xa~Q}UB{5bfgfD*&Ya0icyRoY1mKQm$`S^92GH2tJ8!l-~rrM`E<~B`* z<~t7~$*Ua8u9#it8lFAkll~mDCN@w&zD%#k?hxsTkscYOmAQw~kK&|e5W8O>m|x`y zLg2@J-Q``e#65~qH#1sZ(dR5rH|>%(a_*kX^8{W0?Bq?5XMx*M>bBIGOOr12I z4*BO)FGlztM?|cLpYS{S(Kz&|^sQ9x-X*Yj#BV4~#zj$MI#N zi&HV&hDevTak`glu(aRYokqhD+&jHd-H)hM0rl@%E2|1PJuPRC)JkP{v^IUUQ5bFX zPqhinBAqT*bZP&(y2-kGtq6tvam@h5vmWJ2|A$)5GX{Jv34#)B<;C@O^OZJ*xRxg? z5hblEaNoIki59~5-RuvO%U}?__Te^60PpGMk)9rx_xAyNvGy2?6?LxEsYh!*qDDPXl9F3)q+-o2IUW)N2;V}L)MD&G2$o6i;R9UNAb{kmYR>)dS3K$-p+eEym7>lFx zk5LiS-v*PlsuiR|{5i~&KjR(FXQaJfp*Ib;yPCR;m0LE#)PU|op6I8!f9M}ijm%evk50~wK~MtA5zJH=7Fk>PHl=d>&M=GVhmfEH{bw$moT%Q85D_T%cG znu7tS?da0A(RY{JU6*!CviIHl2>z4;W$z^Mog5oa1`Gt>OgRdlcwH1#QAZNw1PUQf zedZisOHp$QRDGjW=d;_1mb(Z|YdII4B+LCtEli1{^&i1-@G>fb!=Kl)e=|DU8ea$c}# z;sdb5_jO2#LOxtBa-&!{SrJ3J-9l9Amx1c*L924Sd@!U|NHYcqR5#k^56Cp@p`44X zLnNW<;E%k%cmy70H;H;QeH)UvCfW;!(G4PL)$`GtD&ln4CW33yvDqcDldwl0#_%4P zP5FRD!{K*bVa(MpLgMft(p@?c-b>7(H7{k8r@);Bp^eip12!!j!b){c(iyM?zYYQW z4wjZQlJwehVz`jlJMiGwvH+>>u~5PdmaoWrd9AD!o|4SO8(j}06d4=g@-e7@qaNG+ zW`Uj9PNq`_U#x!+8!RDWSMLj88tf2mn*RdV+>@>@Wucv88A0PpbX=l3=Gwx-aQ`Mm zlLK&nEjM*^^~#RBRn04R6D#$2HL&gL;XnmDwy!F9{n?-Uyh*CRpUC0$F3jfT|0ck` z;0nhl`gaY6+5#&x|8Ff}1OPDJE-L=BkhgXyRc>{>2viI=C-eUrKOF$@cct;4g}kLi z^>QoY)uBjvWbn9Asyy7x|5gMK0C Date: Fri, 28 Mar 2025 20:02:39 +0000 Subject: [PATCH 353/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10220 -> 10226 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0cd7552ec..2f8287a44 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 15:00:36,275 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 15:00:36,275 submission_checker1.py:3260 INFO] --- -[2025-03-28 15:00:36,275 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 20:02:33,826 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 20:02:33,827 submission_checker1.py:3260 INFO] --- +[2025-03-28 20:02:33,827 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 15:00:36,275 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 20:02:33,827 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 15:00:36,275 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 20:02:33,827 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 15:00:36,275 submission_checker1.py:3276 INFO] --- -[2025-03-28 15:00:36,275 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 15:00:36,275 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 15:00:36,275 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 15:00:36,275 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 15:00:36,275 submission_checker1.py:3297 INFO] --- -[2025-03-28 15:00:36,275 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 15:00:37,298 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 20:02:33,827 submission_checker1.py:3276 INFO] --- +[2025-03-28 20:02:33,827 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 20:02:33,827 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 20:02:33,827 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 20:02:33,827 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 20:02:33,827 submission_checker1.py:3297 INFO] --- +[2025-03-28 20:02:33,827 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 20:02:34,891 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,14 +33,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.71607 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.395672 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2684 | acc: 76.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.98182 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.118 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436676 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7395 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0261 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index da8e2a42b..7db0976bd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -980 +981 diff --git a/summary.csv b/summary.csv index e9e0cd937..fc5090feb 100644 --- a/summary.csv +++ b/summary.csv @@ -11,14 +11,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.71607,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.395672,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2684,acc: 76.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.98182,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.118,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436676,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7395,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0261,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index d404b5838..8b56a96f4 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2684, - "Accuracy":"acc: 76.400", + "Result":9.98182, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -579,7 +579,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.7395, + "Result":21.0261, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 2964fa3c2c8e07d4d2c43e9afb6b2f3e424b666a..51a6aebf2eb4415241b3ebfaaca887e478ed7829 100644 GIT binary patch delta 4432 zcmYM2c{J4D|Ho(SDvT{_mNAwj$xaM1mNC{y_7K^!4aRO5%08B1ghaMPBs+=h*q*U3LswNT;W*DxQhe%r49xyI4Zwk)! zbdW57j`v!7EiAcihx|HFL8R(+huq$KL6Y~L6bjWZf!t>v-LA(y@#VFn3FRo27uVi< z?6>!Gy|9bjRpTfLcSR&~rO=q^4BZ)S z^d1|+K$N^LMjo>A$zyTp{Dj&OeQ8gTl+u{=#l5?C`~u_?*p3ZX9>}niG^r^$)68eyfj=Ylfv}GDoJpn2%*DA~WO)4vVa#=H!CCqP@nYcl$ z(KT`M6J}ZIu+A1i)KO?LNq}<_O3t(YX#TJzP+vGRg02fPpiI%)%4~0 zK<(8RNyo~bOt1A^LsDDfvu5&vh_LV=G-*lNel+&|KTee*--PI{5N32o0^`U@Uz6${ z>&;9YF(=&73)IR&!$wfTO~sFv&6*TCE*4L*LKAk=5^}NYi)N^CiO!&X8_d0^1WnEN z0}M2f@Hov6o9enkj>rNbkjy({)`Vsz6>XvK{}hUVOiwP-Oc0;gZGgBef-#z!=%6z7`{dIWaFM!gMfvL3s!_yHC?PS@CmdFB;Iy?OxyklQ*8{<^ci*-2a`{Wv{A zc6ze6Y_j7Mt+JCJ>dRe2toWkE!U3UoL6^sHYGQ^cSr=&rt^fz=y11(H=rO@tMntxh z^vbhZiyS9=qkb=_h~3uetWC|TR>ilZC!1hT^VC`izP{ulDZ5d@x$@}-zMpewfeAHZ z{3@3m7_5KCN8r+H?wQ#wsEGK?7%Pv>!FEAQd61p)dtoJ8`-Q0b4R3T}O>%NgRWh?- zkM@-bOqCi{OtH3*X&?hl5_60uAt*S3`Gq#&pQ8JLFV??Pj#re(jI0S_{&+(7_QFs& z@k}sjOAdJwy}tF%wz$5aY5QP)W+TEbsG~-8T!?=o{j!xiMX^c>>a#GYLY0OJwQ7!z z-vFt8JdB_}>yEEP$5OM;Y4M>9(gUgkKHNseT0Qp8_|sZ>jmuSf=*elirP%Yx?MPtL zAKmD}jCpz*<-G-l+Ud<>?Dn^O4SEJb*xua&)+We<%<@@=S5o%g%Rvp$(5I?r)@OIm z`D&m)_aw}`?9#uuF@Ohi=>Av>T85nM#ClzO;G}ll+&LlVHacn+w|2YZdUm4+Y|KAW z+KEHzg%sR_NLL@mojEE%g24=?Y&pT~XON8{Ywp zUh)zc(bfxJWd~a`GOffsl~yYIHgMPtoAKhdOz+>A3tIJ~b4fI;P8-HiYyDYCqp6aE zKY zU|H&aL;q*y_hnsluaZ_ew+F_$eW3-OdF)fg$>M|Y(C~S(=ZN>=c}!?1EWKay(^3tc zhB^1+kuPXy@KCMxK&yGZmfh67i;OEFgn-%^>61NV5SH*_#|e{(#Q~+|&{^fLeAr2E zYjs_YltaFFPx89bKk_#8;lmv_2D+M+$ul1fj(MM;UG77f$$SsRykidMklf-KH!ta) z^+8maIwKkSrru^*PneIEM2tF2EzSLWPIRWO_M|w&r zU0~yGl5drfXMrPU<+GsIvY*Ai`AQB4HWXY*+r~LwnY1}*<6q2=n$Y-~-0iE|uqphU zTrX0BYBUZW7VQR!!Vu8Op@W0I+|vC>LzW8zG(8yP)4RXycryaxyd6bZ5F8Ez zA8xsUS;NV3`n~gKw;{|Li~LdDD(N5a;`|17q1`V^uWv+ZLyG5{Vc?C(c((hSzsY2_ zUHTnrlGZa3K}7d0GI2}1;=#s=O~Rs z?0so2ZT~jjHUV*^&EaBArjT*XQr|R5`4b+x z^SOFX-WJY5Gpu8I%@)xJ4NCj6V+MMOc-qqFFueIl_CC|+GSPSR75qN+X!)u*sL5R z{qmE}MsMpOuV{_(WSS|}6l-EBRv!{Xk#3yq!x#jyJKZ_AVw>;syB{aG*3;*(;oKsM zT@$|thC9P@3eAcQU#R$d!FEqpD>XjWxS=M7bM96a{NzA?JSIa@=#=g?w~^#W3%YK5 zR8b`bW!(YxT~vy9CVHJb_YGiEH!hBcB(bk67}&+kuo^_ez!?y>L`a4{B+-E}5n`A5 z@5VxeYr?EOF)))%edpe|6CTkajy(451#RamWfnp^6U00YA|6$ZDXSPW+h^4Q61=in zsEW}dS%4%M*o?vh8$=KY;hi)(Y|=Nv47zW109LFN1L5?@tCA1GxJwLMFN^YqriLv| zClebq7?um*=-kCm0a1r-4cJ{tp^%L=(0+bpbwI$(<(Q!#4AMl}SgO=%*zyHMYyzEx z3vZ(Eq&{4%p zpUD#7^ki9vVU@(l)7SO6+B1w4c+_M1<&G1r{mN6?W)03xVpPSitu}9j zl-aT>8v>alPz&Mxkm(LUOno?QA(%?RsXo*}wDbno7zf5DL<=O$=YPf43NYc&?4s;l zhPhSWRgk&)VsbK#GZrR7^;q7jk({3~uMX7L%GXqB@1-SaAXP1nI0jIV+zNktxR__U zVKGoPk?tdmDf89vM+jr|NG46eXg|HID7r25Ma7>CPktn z&GJX2OyjIqqPeoK^?CqC;BJ-rv$Wgh>K+QVda2o~BdDlP`*~PzPJ5T#m&frJ1T%@x zA?yk@HwG!u!R0l@2!PxlUC?xo>2>D60XA_R#Q> z>BFT9y46v#;HHPK97%Fgx{Yk^IBvFn=2xm6@2=)S5axBM!@%P`qT$Z5867frS0|kc z^h=DbF<7c9=7vDH_|PMSL|2V=PqYCzlkOe+-IuQ7Wyl?cf=gUpQaBHmi|G6lMpLLc z`O~W*Vz9-ecU3p%CO&c?!jWxM5WBw(N)=~PgT)$J*8Y-Vl7Y+DxYUjG^{@eOu22C{ zZ^eHt|6q$wB8Uf!OUa$O%baRbJf>=*O4xWSAykh}TF{%`B!{Mq{P}#(6cL$SG<$erHFQVE zS7;ms%v$I3FP)Bek02cg^TyR-rYQm%$O;_qx!JECI28c0Tpa|N90dP5o#g?S?Ap}s z0U%ZnW3nHO`VPjeg@@z2L1O!o5aH?vrm8-BP41VRQ#pM&5UzgLPQNzuz-B* zQiWu~Vl1~F$4?5iF=H`<5Jfttws)2p`CmTU3YmP~TG@D<$g8Hr=<{PqZV)-W6 zu)p3*!)OrW@`DUhkHKN@=c2TDnU7=e~^54 zd*;?PF5G#SJb#BBv*%5|KdlT79o+4HsQZ_m-RtB&$$slwTDE|g+`zwg5&T-qe7Jh{ z^Sil5o4}a!1Gbcj+oR<~nc~j~iJU^5U&m?IxzTT4F}U*kKPWEt{dldcNW_hcx=v4L z(SxYLRoCbAe`xDt*`ESwm8AUiLhFS%OQ@*7wLPfs=b8Wt`7m%imSBllTk3Tfm_346 zJlgRl89Z(#6VL{57ET&gQd_u2v8HX_u#9T+vYY%_6af8#;Eiu z)zIHV*kwq%d0JKO846{luu!FVOuJS_P#BWKu#Ho?fM$ggtLqay1`&=nwO`yT2tc(w zrC`qHEIxYQPgZ_f7!$C5_>gxhO4ust?bSN4WIWAZQ@ELX2iIyXr&aEsDj-=04ESamLEnyP(nqz1&czR}Ryqpx=m ziKAlt?^_1$r>HvLzsR5p|EAz-#W;a%xPCD$ y;Cmc{xGL}lPFq}z@2~8lKmfp$3IJgJ&vME?ItGM`6Nds_a6RH8beBc{PW>O<`Z|ID delta 4485 zcmYkAcQhN^|HqSvRV$5EQ7dA#R@F?2QAJR-J+(@O+MB8(R?QkojMCCJMy*;wjG}6f zS}84x+MAX~QCdHG&U1d>`^UZaoO|!*ykDR3$LH?e*}PNvk&6C+JPYyx0|3EdTbYw#nsd~p%My>#!rjl+t$ z>sMR%q9OJ=R7!kV@sQ5<)j`iv#lK-a%u}sV@prSRHD)rdCUOeV6SLC_GTf$e)2U?@ zxEZ2$@3>bg778di*{9>IWwmtiG#W8Otd`z>%1?_z3U{yRLQ@932JLoNpN|Lq)AT16 z|KQK|B&2OOA|Jtqz*9>zrBC3icGl;fT&`vx8?Ap6VEVA&B2KZ$_?pAeN_06EDvy%1LG)GPhDG6zw&cs|?Is-ddhT*N7bTOIk#&1rDEAvS zQ+|)v_umy_K32gh0;0Pk=(^hRD%_4+5u6dhMuYEb9n&@Bf-7Ro+{c6M>bRsayd}@> zEDR0@8^?T(qt(f3<*v*pD`a9Nw@1g6zfAEuA&Ct4hc7mJbXaQ#dBwdIy%Chgle7~q z8lcIid1TcQ!g>E{pCqrN#X=LmF3(()C^qrgM%+#1K3>=oIoDS2dtVBKuIK<95spc@ zNcXH+gbu7aiR+#1K2}1_qTpj#5(# z@VZD>w9Nwojx}rZ^TmB`7kTl98&In{D!^k{OAi7b@d`pxcg{DtaESEtL3s#90I9ud z2ea9Hb#Ew(`Y(ZRUSAVwTKcfYOFQLDlA!r^@J=Q;@^s4=- zdv?y9JjIyGumZUKSU}`OX6rhuuLd-S_LG$5g<< z-w#3ln{^vUHvFm^(`0|GKK86s*(I$WOdD)@%Gqy5+K*;$+3JMTI&@&=qS;leRCy8F z$mOe1r_RZu@7fkeYSXKvbIruAb+`$ruJ!6+1!i%5W3t4|dZi@k1bj6<;d56LlBamo z?nRYzD|1QTT1vv^&XdP8c2fT`w_MtexF&;H<}Mm2|7py}$#0(&d#e)uS-pX%q+kr~ z*W9YJF6~kcy1RkyvVZ6e^{BvX004l2 z?rc&8d`l)RuMk4%ZecqLXdr_0-A4!7pr`f~LU(`kK2`R!g*v}&ZX)LoP2)L`hx?w$ zYo1Y}ys@PA?2_klg3tCp{0Wj*F0-*|nc8#Jdg1@z_cr1Mb!_d9y7lq?sw(a5ht-rI zHgG&n_*rLctr33UfIF#P;X>hbJ?~)D-9|@<7R3`EmK2i|#EYe#{v+dzbaAHm!zkN9 z=2nhhQu-F@Xrp|jf>?iYQ0y|w@S1J;&-2?;v=<&(-$1UWP(1FcXt6OeyLG#sE-Gw& zf@#zCw^Veuk%Kl!xb$5)#?zjbkWIev0sTq9R@0L7AUuzcu`v=CODwBm1%TJ&4iGL3Xn zvRUDz=5c^yR&pI_;YTR6%mG$vn<*GVx!tHuYjpTAE<>eb&odD7sbzf4bItMXz3OFm z_oRUKqtGbaIBu(XDsNeRL0<2jxphyf-h{UhcR3jb!>$F%M~#LU!n*9Xljkn8XGp}L z5rgH6c%82{z3jN4CcGQ`hXOvw_xZc8!U#j6wP~D>t#V0yh^fzh(R`y$l}T~u)mnbr zL|vn~L|r~JYg9?S#uMeDTS|dfYF!Gt<#|nfm|KORK40&|X>L|3Xu`uo^$_Fbi##~G z+!@w*?6);4T-^0lL0p{8EF9`{VqiqXHyV3yCi}kif+S&%W;9JpS{$Ir>EXL@gMy($gLJef)Rb zG5xx!d`<76Chbd%KCQ#z(>*SsKDM|QDvTjm?|RH;*7po?Rq$NWn(!H(~nuMPb{Joa6=fHP+d$lK~{0rh26{My|Tn zbiw2p!^21I(-AtGw0J$hgs&Y4o+kX+PKJPFHK!+KwVC%ZZlGVn3w-Nn?@0+RvgsompX# z&{xSihyYD^u^`kZrsW|+k>K2X%E1{cN6az&#Q%o%W`TT+)qGHxuMi%g<$rWen|+@p(wg(eG{O=Q?DtsX>TO#{Sh!qEsQ8jjO}HiMw(dPEifyC-vML<==|3{`wC0@o2o zzkgjfO-%N|Olxe%6#{mwBs#2_WJb3bm$)C>F>TuSwKFKyRfT z67vqifC&g|+ke@j$sxeLVs~c(^P;h7m%p@Dmal}a>@IA7pn`+!LgwsxOxTub&sjOy z3~*szN>}^-byj1zqO@=(yFZZ5R9M@4Lh=%u^E+XfMWv61teOBthcQdbS4Xy`XD=+z z9#IX3b`rsFKXx%;pAL@-=ZKmNhC#*!?00P!9%NX~K#@~dzCGeUlIC<${267Jo11EM z`Rmt`#!Q1)RYpf4_Ka7%zABaM8RpYw$yt8KCO?|Cx-aaMPt|om<;n%C5D(XYdPJaj zJWZu0(?AM0Qzsq)5NCkaq=n>qUy$$^jh9(sQl;A%mR2&t%3K^svtn1)mLwPUGU|Jb zkFoZY5xZpemV)hELx&o+8w>`=^5#=sB(tMBt9a1DuGSX#co>4<9Ij6U${0mq*5NW} z5I>1u-Hr4kLVx0n1|sA5Pi%DCv}uMVieKprWtPE-T`r@ct&#W3^8HcI$>cr(f+~-S z`Qh*gHc%*Qu`OSuSCa{kmQ@oT0b~`m6}Xe)^zncmah>5w_Yi)Qx*D`d-zU62*Z6Du zaN&z{FD6d3Ul-@rWWObL8T&c;Js$jbJ=tT)2DAI-sQUsRekvEiDpxM`KQJpR3YIm zs&3MqgjBu<)d{=0*IagF)~-+nn+^yL!$zyWz4LFzIxD-32*5Da(O7h>OE~&|6s|?u zY>{H0qmyS~6qx}kWv)4AcGnecE5F$ut_#Q0G+j3Ijy{X57yTj44Vph^&0BZwF_R}6#8}RvdaecOx~~HIGSSdxlMc6E4*`fAKZ^c z4`9kM@tE!K@vAd!0vVw1hpEyvY|;dle=b}JJmSw#oi+KcpM%PDomyY`RGU{9W>c8_d2vJtFfKy(O#+bzX7b2zq}D0DDj7 zqWJxQPTQB=?@z>r&XB7IG|auGLTkz&XJp0}_NlWb2`1MZLy|ZlYkDe=R~}+qQ3huoLPxZr3%p*!+3)xF@7A@FhdD#{t@cbi14E7nWNZ zz?|*OKm0_p8}U^MPB z?JNV??A&2tX&*&Yn#=Ngj7iIN3nERwN&Cm!HZsqZFhsu&To@nm$GPFu0caRsA@Dp` z#dnV@!D>#pS?#D)et@m)wyJlJ+MthaOwH*D-zr4{VXsJ+^{EIOhvuT#M?ayYtiQEdJ`|Dq*^HRUs)N98{&ETr^9 zQs4Z3Li(flxc%nC>fmMvnOvr%!=bT6!JN%boKr@bg5o`Wkl+!Dh1sobT3JQoZk1df zR^gv!nbBw33(_!=S&QZeJ7w1TjoOsTQ(vW`PWO`)c+e5i^5DY+cEr#2_x)9iuGhGX zFsa+6RzLBfcI`;yvxl*5Z#-o$58ZcwGLC9cLQSp{$Mi%qUlP1qPSXC|K86XS6`N1x zFEswhr--)_Y;VfJK(@=igdIM!!CY74JUN)SniJr6{MY;+{`bB+S}_8`uE zH@`I~R(-yvfTCx&3bAU$i+JtlhI-LBgdFmbnz`)hz>M}J6afww#|jJ3O-0l#eo}ia zCecjqvJhvfs^iMj8p1MMm99vcX|(Ntu)ZI*cwxi`8BTjEluclM0_iEojqO6HJfcG!qHD(PC*i#e&0cXW;y8ZYuI#xS$3e)Gv)I-FAPoP1KqfGYAc0~8w1_bGd1Qou0DvI~ d0J!+ym9sTG;sPN-1O>Du^oj`5K!nbZ{traAR2u*Q diff --git a/summary_results.json b/summary_results.json index 070d3a04e..ba352743b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.400", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2684, + "Performance_Result": 9.98182, "Performance_Units": "Samples/s" }, { @@ -860,7 +860,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.7395, + "Performance_Result": 21.0261, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 0723d087b88b454803975714ca77edd72ad26e20 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 20:04:06 +0000 Subject: [PATCH 354/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10226 -> 10220 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 2f8287a44..93ebd5903 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 20:02:33,826 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 20:02:33,827 submission_checker1.py:3260 INFO] --- -[2025-03-28 20:02:33,827 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 20:04:01,718 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 20:04:01,718 submission_checker1.py:3260 INFO] --- +[2025-03-28 20:04:01,718 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 20:02:33,827 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 20:04:01,718 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 20:02:33,827 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 20:04:01,718 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 20:02:33,827 submission_checker1.py:3276 INFO] --- -[2025-03-28 20:02:33,827 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 20:02:33,827 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 20:02:33,827 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 20:02:33,827 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 20:02:33,827 submission_checker1.py:3297 INFO] --- -[2025-03-28 20:02:33,827 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 20:02:34,891 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 20:04:01,718 submission_checker1.py:3276 INFO] --- +[2025-03-28 20:04:01,718 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 20:04:01,718 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 20:04:01,718 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 20:04:01,718 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 20:04:01,718 submission_checker1.py:3297 INFO] --- +[2025-03-28 20:04:01,718 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 20:04:02,699 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,8 +32,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.71607 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.395672 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.98182 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0103 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7db0976bd..acbef433b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -981 +982 diff --git a/summary.csv b/summary.csv index fc5090feb..fe9c6b0f3 100644 --- a/summary.csv +++ b/summary.csv @@ -10,8 +10,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.71607,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.395672,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.98182,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0103,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8b56a96f4..e1226f64d 100644 --- a/summary.json +++ b/summary.json @@ -339,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.395672, + "Result":0.411773, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.98182, - "Accuracy":"acc: 75.600", + "Result":10.0103, + "Accuracy":"acc: 74.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 51a6aebf2eb4415241b3ebfaaca887e478ed7829..92b725d586ca35dffd3f9ae56546cb95c8302555 100644 GIT binary patch delta 4300 zcmZWtc{CJU+#XD}!Wc{Tb!JS;BvDKZGBkq_k$tJLX6+Rc#!|MdGqPk2&B&gevW~Lv zTS8N#cqiFHXvn9|dC&Kq^W8u0z31F}p7T7v=iYmsbAK3rH?AIH;Sh|-;PV9n0Kvp* z7HPbVPmwBr7}X)dMAF(FjKH!d5)w%CR=UEFcfBz$6W=Jn#dBhyT~d@_^iRk$;Z=hd zq1cWPYV+q2RM6fIbRzIN0EWu~Zq!h2a0uA<`346sQ1%=Xiuvf4L^KL+o4&DH*p zQW$*K6@D1UCDP-LNpiRRS`13$)vM9!-FQGqpBLjPx`{H|7fR@s-O9e0ss4+kA~9Wg zgzdYC5A=$4+}Bb9bAtPrGo$-GvGl;5jjPaW09qia3tca5+ z%`{#d9t}2)o{43{WVMS{7dKwcz$4bir_|riNw^>jPu?EoZT0H5y%yvXTLHfklqR0I z9uB{cmPGH{bcdY2eYPJV;e2cHwUoa2=O{S-N#;tdj(Wd@$k)a^dxMSsWDtAFE-)e- zm(0ta*(%2lG^{iOGP}$OGjBFXFS7&R9G(O+>*M3JxErN!A-hjZ<;s9e@C$c=%%u;b zw6yN~@UoYzi8C`zwO*5wjGdv#<&Z@I^_t_-;wJSR%uM?dvY?dpKa)HekYe`mB@j*; zdF@|E5xegN4&HfQ(_3d%6{p1gwJR#6msvlDH%Cf!B)nz_1gN8qcGqy1TAP|bRLubq zj7R431dmy6cki!R2Sa#g9;qt*E}E`&_oF_e=ySl&w>!(6^X^UYZ@e|D^0JBKHEonk z|E5&FLdOlcJ{nlau_?pxJ}Rwn>q8V?nBp8LHE-kW@4{_+uo&sBVo0^u%zD6Z|yv^XDWVbL(%hV+o6nw(ncu01%+s7Lm(BolKnAk zl|R^l@!SzwW6~fv63DDOLzPB8;?`}93Y(27jAAich=3t`o5$3oR&2WE;p=usczDZ5 zFP#UyJ4yxTuKI_rZ$woL_n0rQl(KMjrE=559|uHg6?_A+-xOz#DAkd2bVY3imT*6h z@;o0fdOoxSiT_(OCo)j&GLr2aq%qd~z2JU+S&N62K%qw;kGnJN?Ji2takrg)K7%uOYHI(Ka$!c3TB4$Ux@mg+%)8e} z@s~A@ITLQ}+@4(x89R z8wW{``{31V@oFigs7j0!YpEpK&LC-E&_ilzCf&iRy(W)&l6rNok4 zXBdf3wgRu%Q(}p~J>Ta)s&#DN)QA;rmpE`;w-Gwzr2@5Y7jw95hAqOdORp;|)SEa+ zhfTu6QC7h?e!Yn>LRdnj&89l%FN*W1|M)nGUPn6IEPvHsEo$D!i%-&9)ZT!Jog;Og zvx3G&Vl}ZblGKD{zePtMRik9*9ibbFAAH107mBEL6K>+)6&P(|Y*gC(h@M6NjN*YT zW5chBs(8`;WZil?MKvzLN%7lxSfBges*Yk?8+Lrc#Z@}W7kvImg|xVB`z`vyDs}bp z2d>4rpv-w(Tw?Yn-d6}G*Wz?a&<|{cw-Gm80hXD3Ex+^h<)yJOGb>Wj>UYK-ievgh zg?ZHgZyR0iFsc=P;DR2IpgquJxJ*_~)7^z}6%I8^o}Lx=DfG~&)algR*14#k_A8eR z-dN=HrTjDND=munhpWP0JzLuy1?Re>*x2J6&p9ASfq-4DZ0;FB)*8M=yAGi})RY1HbrCB$1@V;y$!Hm)&=^ejZeRjfVc0tZ(>;5LhS0wM zj78@xjG$$P*EmvRYZ;F*Vqu+X2a*3sMJSwCO>6oQse`qdWJF`otv+$+Nd`BxO?7vD z9s1=ojcq)tNe`9&C~z%#!Pumy=DEQ;IpiMXPOy-yYt9J#21)D77pG4}Q9SMW_%|A> zVc*5uZ4uEle-(cXBX2>6>?q$4&y2?LWV*iGp}PdSsBF`dA59NdXwr{VZH_(*N%)$9e`CsqUm zDPX#^Oj}hnu}ZXEIa+&M(wO!RU)DMuYdGWZ4SvS1j+H)|ek72v;;DkDx4x~cP~l0X zO*ZEsPru}w`EWO_iKef4mBZv3gn*Puo#c&cx=MWwolN!J9AMnR2C-2>dBbvePP@K* z@;QAErP&O|f@NC{62Q)!Lvc2;gqh2?4z&X`Yl$D*%1u{o$y&qha!$(lrskHZ>ty$} zQ-b<70w(1}X`FETCN=&!`-$CV2;c9I9>d{q3*80Io;5t!elsS ztbI|p07(2f&;$a;$H}B(_;1jjBzU3`Oab+#S8nz|1N%+bHQ!l;5Wm|Sn8>Ya zKeV!z^aSQqmfk&#a$E0KSdkN?1_*vz1I!)j1Yi1a8|a;Rh@O^@Vc=dJM1Ysj~K^)K-~P5jCwh3{lj zaz2K*+MDL20cJHev+`)xzDGFwiF(pST*cx+M~lS!C6{sUxg>ch;(C&V{Jeji9Tqm5 zxeduG(N>CM?S+U)@u@VWd1a&?v&uSVl^w-{>rY3}(OK4nGn8QEQ=<@>oQrcuaOEkY zMeY4bOZQ>VSv`g@D9>v$M-Nb_bB*83sBaI&cl!l#0PjRCUIEI2}9sMLhKED-`w zG64HuVw3DmQ?MK7kT#3n!3Yg!K~#pbrjM91|Kim(EEyV{x!h>{>bgq=1N}U=<0mw$ zWYZ7}^GGo)WcouB%|&qke-wc~@hGX%oWMgvCrUuk!d!iZ^0P6%<1ey?p1rvLTw4qrhCX$_xi_Ox-mgfwUI z`0RTHM$&!<KlT;PT{l1I?m1fkPLxX8{sBsq8&XF{-L$CQ zQsgqhDr-4s6&=IL;|J6=^sT-B;%Uoww&)c4?&uF*X$r2GZQYvD**DbCaP^p*ymMxz zA--h$WC^-GR@O@cy@++-8IyV{kfUM9audvDV&$^t#1Y=F^e~wj&r|1*mFqSINJxUC z)0tg{8V-N#ie`U+if@8Uzxd!YjV~T1HO1$T8-ygEi@98)=^Dv{H-^y9K zY%4&rB0KM^!1I@!qv*w0*;Ujr$JfUkp9y?N8M3&6xeBaYcAYo?LrSlcSzrI<2)tg+ zz^5o$_SH;Q6Q)#NhyIpO%73}AY`DCNqI)!~?gez_FMHU9!>q4~(RYs>`4t+=I#&K^ z*GWW+=yr_M&SL;@OWzMYKAfY#b{;x#7a?7KjS_y{WiP_2)`c>XT(e)iFkl^Du6%b% zTyu6Yf1VoXL!3oO@#V@UZ8MHd!5^Z~Q8yt+9i%3fy&fd0}Gqwh$I zI0PTdeO~@)$4MldkZ|qyel23_SG5tKU#TjYwdyZke;wqhnOMR{{nG=tvcxxWi32Jm}=T$O+A{vt z@gO-{Ml8{5&SkZ{@!f&iZ|l~!-7=DCAek2a&D6JF zyxP&$jL9n^GUD?)e!xg&yS-z=q3Olbe!1# zYvB>jL@Z2~DUz5BlM(w<+6TuU3vdAd;7_>!uZ2gP5+EW31c^8ph&Uh&I{Ci~jYMua tB-0rpRt_fm#|9`90ARul0Py~|<#-H_;3US&p_uH6eR8s_An2c^{{iRZ=8^ya delta 4298 zcmX|_c{J2t+{b6^Ef`yB$TG%~B-x2UCXBI0vKAs+w!zp9Lv|B0jF4z7iAZ)TvXNg>xv%f%zTfZfJ)d*eOt(y{2N_S3#sLRGOaMR}HzAVk z0^t`UD~94Tug)8(b&u<2qeKZIpmpd9xO_P72ZYl-kfD^hdP^gO=A597^8iI2? z?@H&xM!IagX6IcuLVoS4A=C6bLT+q7rz(1niG&&yLGN)4ZqyRueZlrjp}fV4lDgZE z{I-9t6twfWYW+?IITTAF^t)LTjZyK?LG43JWzNLCb48}`r!tw}?7P)p z?>*dyg{pX+iQMB7P{a|^17r7W(Mn7To2 zFjaAiqvo0ErSA@uYT(2#N@+47Lb+F{qDftb7nH3KyczFM0z~_fwdYYkUa^l`ezXg? zNC|bN<(g3Trf^}JS1;KFy!z(FH)k$_B}bDbV{Z$EaFy{d?l{<4U9v0+_O$MPP{~@F z2h?49p1iN>$^J^;H6*PmA#*Yhhztu4!cgaB9R_3H|L0UL_DzK4B6(78ATW-W{584e zk^bc9Z;r%U`hhx`82A8Mw1M<+!Ms7K^$cm83zoR`I57veGG~q+k!lOtvBlnrO4QbV z|B;Oe8Xl+pVNFv{#1WM*0+M@c!j;&_uBI#U{l5Y+klDc*rcp|~{VIsxG8n6^jR|@U zVk=u$VgP=AuOm{}(#XbSN#lrlCw7;!!0(w4VEouDp-%gA4;#}DI##=t&i@gP)&B5M zeOf#H(aPE9u^xe~u~BbA8*GNJ&V7IfkFeCYV4r%$F)p3P0uIAthG@Vs6P%5 zP;H((O=~>FB`+Ow4|LY09>0NvG^va{L;^(;}@}N}xNXP{w{xc)E0=PT=P>Mrc&S zgt)}70D&0X@)0`w>ip!?I!sJ*a+vFa?e0c?Q)!UB$$L>1JBQh*+Es5%QdLSyRYeMi zQK#<3QEY_Dwzz;y_geaeO*pd2_li zoN^?bysm&ch+bKLYe%ZhZ`jzKnOu#q4{EJa9}y8;O+RPtPFJj)j`}PLDpO}-KrdNf z5>`R#pY|eHk2(^{F|mw1(>elZ!}Ng4fDboNvDS~gGyXQ0U*>m}?TbH5w~}}kxe*C$ z_^TIPkTJu`q`E!JRz0z{kK6c`r^U)f4%@z+&(#3Ek1w5Kdnx1Ky%1Cf3w@$~WOH=; zSfC2_b6d*X%Rc>!8yloIo8_;KuvN&>W~|rc`%W5HESwXwZ(yRP2+KEGuVmGGz=!=K zWu16sp387utq%?|xmDUORYZ>Bv&@myLe{kLuZJGJAG!Y9xrw!BBwgvNwktX-d-Xfu zl$W9uR=oN2SNY!N47|03r^;e^_bLImYCA$|!F&I~p4O=yoK9xqa#}TsTJFhA9!!(& z{Rtdh;WtAvGI8<(001`n&;s}c_`hBugs|CT3hVdQXYjCKq^E}JLHt%Me z5cqwc3O-IBtcR9Q{I(;}=fWd$T|x1^qMue8kFm#o7ZmgB>O9mdJun(xujJSCZlmHb zMdT3ByHl(w9&$)41j%vKbad{c%0lRr>Q@2Wn756l9&hTNK!PW2Mdd$5Th{RY)~g@e z8&zqOpA7fG@feqTFb%s`ixd9R)nrM3GNs1I>IZt-S?^#Fxe1D zt;0Rsm=>y|**4nD*UuZpKkbh{GwGM(^kSo$RA6=D> zwDdnNn`hwLDkLs`Egvt<68aWm)U9@yYtZGd1>t_vIRi=p>|M#Qrr?9&kdQD_DE0*> z95Ph+7!E;4CKz;WUe`-H7v6|LLY+6F=nI0w;gG#`HwafaEzY28=I92LLu*bjszWXP z15r}Y&_1-|dGVFiNL?strV$QVjZEOaxAuod-x8VTZAN4Fe$GGbyU5LC8=#= zfhB~fTvqamr-lHtdCT21Rd^ATs`>5-s_GCCw)weoTG0-{%hZ3<>arcO9v1ZY%ceQ# z1@cK#{oVexhw}H>m$viUBi`@~JqRW{6N^(`=GU)!f`;@Ke)e2Vb4uU&NNC#he60i- z+s#%tG~}<;QLkIXV3-%w?b48y#Pi0;%algdAlVl~H&?rw_rT&+s$-AM7{_P7zZZP#V5wZ| zQTawkDE44_GOPbER^@&7(7LcT1@)fodf#Tz(x#T{G_@8iwo z*_hRJzF1-@vN=l0<)skOmDrN9Ve=iXn?SNxW)lNREEA9nhnUlG5W@&EIlPVOCb#U> zFpOcx^;W>5wQ?YW6?IAae%SdUqvmts;Lxmc=NV$NMM8^y`l6v?Oyz>=St5oYT)lq zvoE)tn5Bm&810-qlpGeHFgI%PwNawVeyz87BW27N)VYzUY@uo>_ygV@fE@pD*hDsy zK`?x%f$He@E}!~1GAf=gWij(Bwpxgtz$4DnZj@8;T@96!C!ru$KWS+yQi}ths~E`s z8S`>ibGdX`o%zn=WG$4sgnhgIGcnH_lJQ>K$9H%qXEjpu&!59pl_ZTti3 zp07}`L7LTr-Nn-l7jJ7zVAR9PemyVx&rrTM7Ga7{0z(Pz(MBYoym>5g5`7^D3QV`b z(`ERrUHXuUH_3b{o+JNCzY|~#=}@aZdVIq|(?iKlKP_u%039{7lZ*4_bHKQ4z1UAU zEsUo;gYqa*3plA#(an|;Q0jZ#hbxD9o#zpnfs*Y)>QNBSDbtZpV)+CGi|{BJlsf21AOJ2z z=*mN7So$S9x=$f{K#aNOIc7(BEl{+FmX};NAzj$5h8}=x9=-}x=`q{5e8BzQz4oMSUu6%8fe6XIhe)aRD&5X#LkOPb zEzj*2u979FO{M&^{9ZE5_}>2l)$TpuDcQ<9IAsR2pbfpm^ihH>$Gj^#_|}M#A0r&O z2ZeDv8=y2vb`5x}kyZ6CId(aOVwFqHKzAoMfZz%f68BdAXZi;VWa4Uu6c)-gSK>NI z%)bKto2;K*&ug93_l}_7m{#P!#wk!oTOg7j-3|DtR-*F`j&S)>6_5<$$%{CHi9(T18 zCTgQ>h=oP%tBhA1Mo+<-<4!`pO%FN5YGQ3S^z1;v)>;h_QX%6KG_zh5G*n`YDJbmI9O%x2T#ZD)90E|gM#r`#z46xAV0gdcPi#dr>;VCpmn49IY4H0Zr)AFEk zcvg2wq5jWP>bq%4hrx!6v$DTKtG+P1lY+W3qJ&AC#j^?Spuy8o!lgjPB$g4G?LXB_ z(GPNF-D)?1#=rigoBWy-kK1)WU)_6FOU~=d!{7Vn$^|<+^a?Kz=7xSWMgNhc{aN>x zr!c#a0{$0?Ibo&`{9~SE^&eB4&eV^cWj#4dZbgX?B#b=6#Aa;^X`>#&LX!l!ai1dPZi9J?#Ou|^9`9l-qke8&tKnN>KocVOLj>!z>B2pX8VBH&*DOd-4zAL z)nF&+0vgH-?Qgp|EbThw12SFj3bWr8K5;tJ10mhMrrQZXF73r+Jsk8Y_*iYEcOhdH z5$xjivT~F`*>@Ll^=YIjf2CtS1Ee{TJc_ERS0cxi@5Yqn? z27U_rcchIG;q>WvV8#HI5e}1lVR|j&n2;Dz1ryGv?YmT<*l}2^^+yR~A}t&^Y%f%q z#i`}3RYu;I&vqiFpVt>xA0=^a)wTI43%dD+te9H84mRqk^%6S{*qt|!4EHb25f?}` z;U|1JZO}R#d!*;5V)~fyms<-i{I!~6Z|UggcMGi+p<(Cy+^M5C21_Y&q|ZpH>;i&c z>tW`x@gJ}RLiPP$G|x_gPG%(X>KQ$!CsUYSbnlYuGuFS%wOF1fKqeI#KmA%EzG4Q( zZ!Pz0yZJ3nGnKMJwF!a{8K4gV0TIBYTRg7FIYOh&m1ZlH_nDVV$Vz<*Snd&e$$lPt zENaeDUQlML?paswdm2g}?T{U`=$N)2a9AG^DQsn2vToW9RLc6EdEK$Ssk9^eriOig zgmn*(FEsIm?zr!}v=qTV+R$qN%^uB51&iDLo-w&tUd~IG#+)0Z*~FQzY)l+|syJ3K zOVd3z)&Sy~Uu)@Xvz~P7wa#ME45$A827&NXTvPC0%YGUF09gV6=jl!S|2@$$o@|5| zHZb9?gfuXbkR$>5r!V2P#5o{0p+`ao_@2NfsSdnK(3O-BIMG=g2mqKd003P7hoHZs WV?cyBNf^+D&?za#a!&kY>VE)|&GP^N diff --git a/summary_results.json b/summary_results.json index ba352743b..dd211fb92 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 74.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.98182, + "Performance_Result": 10.0103, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.395672, + "Performance_Result": 0.411773, "Performance_Units": "Samples/s" }, { From d5671282660b65ce66e962ebabcd3827f8a5dc33 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 20:12:34 +0000 Subject: [PATCH 355/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 12 ++++++------ summary.xlsx | Bin 10220 -> 10221 bytes summary_results.json | 12 ++++++------ 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 93ebd5903..f3088c82f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 20:04:01,718 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 20:04:01,718 submission_checker1.py:3260 INFO] --- -[2025-03-28 20:04:01,718 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 20:12:29,898 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 20:12:29,898 submission_checker1.py:3260 INFO] --- +[2025-03-28 20:12:29,898 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 20:04:01,718 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 20:12:29,898 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 20:04:01,718 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 20:12:29,898 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 20:04:01,718 submission_checker1.py:3276 INFO] --- -[2025-03-28 20:04:01,718 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 20:04:01,718 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 20:04:01,718 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 20:04:01,718 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 20:04:01,718 submission_checker1.py:3297 INFO] --- -[2025-03-28 20:04:01,718 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 20:04:02,699 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 20:12:29,898 submission_checker1.py:3276 INFO] --- +[2025-03-28 20:12:29,898 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 20:12:29,898 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 20:12:29,898 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 20:12:29,898 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 20:12:29,898 submission_checker1.py:3297 INFO] --- +[2025-03-28 20:12:29,898 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 20:12:31,047 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73142 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421594 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419316 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26123 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,13 +33,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.71607 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0103 | acc: 74.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2763 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.414194 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.118 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436676 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412967 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0958 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437965 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0261 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index acbef433b..9f0c87b5f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -982 +983 diff --git a/summary.csv b/summary.csv index fe9c6b0f3..674f3e13c 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73142,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421594,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419316,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26123,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,13 +11,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.71607,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0103,acc: 74.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2763,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.414194,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.118,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436676,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412967,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0958,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437965,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0261,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e1226f64d..6845f8e7f 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421594, + "Result":0.419316, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0103, - "Accuracy":"acc: 74.400", + "Result":10.2763, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.414194, + "Result":0.412967, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.118, + "Result":23.0958, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436676, + "Result":0.437965, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 92b725d586ca35dffd3f9ae56546cb95c8302555..f097a8b4871165c750fbcb12ad4b78de981093fa 100644 GIT binary patch delta 4020 zcmY+H2T&7Aw1z`3QiD=*X<`U16d_clhY~UlhPpK(wJVDr5UjXw8*YeH7O zE;ZE_;#0e5-=4jbcwj$J^%389fM?g=>2orOic>;MHQjSFYhN->TiBRDQTeW#GccY7 z0068fQy1`%6o_7C2LN~pVXVs7fj*DWZvXO3o{rk0-~e8p43OSau<-J%7@HD*sC-wv zmFGg%&Rpi$R72<)%w~Fs5lhodACX)pW=wDmx4n)gc~W-5l`?xxV&9kaJPf_p7dJN( zHju*{cwDjfHQA?0l(ziMf#--f5u>YyXWaQwp_Vk?9{!s5SI1GQz-Tn~v947TNA&%% z1VZxnx4iNOP3pExeMHW?*$>>ta(aE&I1X=R3AfzWSUL2IItp0&Jdef?!qtbaePnJCTZv7MZq3dz`|Bm9 z7DTo~aZqphX(oP-_7kIGyHLVXZ~Dv(4_aXMG?*u0!ErVF;@Ve{v24}(X?LFF;WqLk zWkJdcu3U)bah8QNI~F?vY0gDX?;t0WTt7a_I=eeJGpH{Z zrt;|q_h5cTxQdU<+Cug2@R*`^pjHi;_?`N(#dOv5^UMZv=cPY32QnkA;NDqyrw-3K zp7q37`NhUdnAo!?7c(2&G)#!8mul)Ghc3$xSDf$h0i}@& zvTYiQqBk6Mit9qG{hbOXimKb??&|-|pO*iLFYRPE%-a7k`|m>j#fhSk{4`Mq%JX<% zifU@y7A?lKjV%;w2&SK?g%3@o{e1!0G8`(wdl@4R^k>3=f&Q!0aEXsi*Po&MA6$Y< z^lwT@@OssngXh-6Awd5JpX1=UUlrlEch2-L3FLCJL4#XfwWSIOn`P(EsKX^zG3Ma> znTOKOamq{*Ufc{~ zq8JKoeIt0;ZGU&s=i*t(l2?OZQN^aQ!`x2XOUTo3D4;UTwYOw@VC?AEk_FPPWB{KZD-NCwt11YWu>5U7&jh35(83 zd9#muf^)G@;+|S@PPmeorSW?g9|qmcD@5eRsJ@#iaZT~Yvu99ynmeil8$d=^ie}g= zUR;%5RK3B4=6chwzuNYG4eJlfc_i0?~(V2{-# za1c2^k;tl@&L11yNeB+FZNECab8<;NUh{e5qS4 zP!ubBR86z}K8)_c+vbxiCDRvVv|SOf2+2V}`rF9g{w1$$@r)53`cXCbLoC~yfuZxy zrLXI3M^`9t68I$fS}d}PzG((zWkIehy|=q*B~FS5_4H=sL>Bd_EZ471Xg*Ai_tO-S z`_f){2LbUBHo@aSU;-+_BA-qf8STC*p^TsxCyDEahHgh%OrF4bls#-SVp@i+)9vr2 z@5T_Gv$|(=W+zIDY@W?T8 z$201A>PPMEePhU~Dm2tW64TwgPYh;&;aHb@mvq^${3;l|b51oqTP0^wbGU5UMs5p2 z*jf^k`vR_YJT=qm|CrfD)=8O^l^}otSmiUo>RJ}D&S&x>Lu}`-KS3h|`?J8R{aIzl z3`!+$D@SrCr?ONW%rvtw@lMKtU!>SK7}KS?;EBFwr)q(mDF~R}skJ3>7I;7>)O27* zG)(+@O^~Tn#XcRcx94u+23BeSWRY;$1~gx zT0wE-G=7nMeTaExP$qLGSWR%{%ylCjkue->^{gaHu3}ZsJT@r5mly+F*Mf1p1xt8W zT)KY4{sB-n!-ddw|3k|)aiQ6Mn;j1=r0l!$Jiwv~dMmf26k{^_oY?KuvN`dvmjyCN zy=8vPHCQ*)YzXUgXFJ1<9R?8K^QMpl$IDPUCA)=r8M1?G3Cm;JjcQtDn4F z&HB3d@0)j=Y_7S=$dW8Yzm}FzrF8CbrJ)!r3o~L-j^0b>K?PMY&tttK%<}1E6NI&) zbXV;bR**vh|KhH(jfnssm)yKP(=zJYzWKaKwA*WS=+g4;c*Gh~nd-J^Er-5tD7CNpnZzmR`ZI8mNT2#;nj>V;; z9p%0bE4g0UQwxqhL5UMXtA?=h>H{9m`&}@+IWg5yIdS2dfE~t`zRLp{g{@&O;jTUj zR$U<5Ehi9U*&GsOR9Zf#;4Q^&)L&Ht>BvR-dozLe1_WPof-62Of<|-#%30e0fr=o~g63k^oo&RcAVy8x=Jh z8x$3q!82BPs~Ms~Z7#K|{Jv~588)=(A!(GJtx8G~`&ydgOy;Xd@Kj;BQk^t0w0$BQ zM&;qmxBIl+3X6aY0dnOG-rm6V6pyQCtuTt8tM1BRLFTzpBfHi`Q|$wz z{o?_bxMeM%hN&ZBdxuYouih_!bxt0wZTFjw7TzB=N81XZ`)Ul=zX zlsyugeo>m8l&N*(80;qz=CagC;C*;`J5RQfu>BUuja6;TSg*B8vrptgmT;E&VlhvBn3!j=1B zTH$JZ7mGvM9%$#(5x?#);M~<2;|-q0HV69Oe&=>C3@BSesR@#_npxF8oZ%o-wsrr_ z4^pF$11pnd^k^NJAR-}$B}%eRN7gKi=)F>qZ)Ec(#M$eb`iL}P&Pn+Lw~IPPd&4g^ zytUuFkwJ~x8TtWQ6w+ry-HRES1~t}}1VsYHGWknYKt8Ytlb)sWx%7bfZ=*tI6il}f zjNVos$_5n}%`fhSvBdv|Y6YEDC(H2PUa6T-KI*H9IAb7M5!K9yJ;NauQS?0SN6gtU zgDJ7`L_o{MpqF|q$ve;v0YAVS>YdiNj+$v4(U0xc>~g<^DoVM{?%TN}jb-X~2Pyb__tPznMgcvCuIu}B#)CFc<$nH||vhN}x zHAEAVeHrpozvp?*dCqg6bD#Ua*LlCL_jO(8{;V6V8PyHZFrSak;tHSz078M7NJjpK zUo?$0$PDUrw)BGCpSG${pPLZS0G`v-x-S0k%BS3`gSlO8hv!e}?uc&R%&5$i?R}1+ zL7kV0J@rhLYU^4WYs!43kZ-|%sr%jq7`YE|VFA_uL8&+ke=S8p4{M5jI{p5Qs&GZU zQ|^Rk2V14|xG2xe!nysnE{E^J;g{xU?z;94UbQyPwqGyRkkfoCz&_w~^CeBQ-Z?*4 z5H|6XZ>#028X>WTlHq=+F1n&OS4J)oKEMbp49JDXlSrhL#%u z05G1tFu>cn$22R<000Ljf>99}^k63cey$9pd(+oD-;VaN0jt<8VCwcp8ZBQ^JJ$6b z^zzo=BVGM_>+|h#8X<`G>UPHs9M|HN1X7^NtbaLfMj>WKOeoc3nv#;V9dg})n1K21 z^ReJ@gHy+*dV*kw$g#(UE#C>}0jW+OI8$#eYW;!;UBq==v<4@}=)!cLh%O|r z1TH%Hs~D!a?edu@eDW8noG)2uobVK0lcf-Z7#2u#BeyL3rxlJRDZc{o3ksLL&NXge zi7JW7&I&)|A$?v4>)J>K0zr3t!renWHUKDps*GFQvHKB!X?L=F4*@w^OToY4r zHUs!vp|71;&IbYwk+#C7tF%?d->UCsR$Lj2G_%H)tglfHG?=C?)mYRHa1zMSlh}5t zV|Vy~2Y^s{*>MyJiU4~#kx?6Yk3b?_4eN}0$> zcdLwuSmeSg@ej=dt#KbKCrOz$9JMz~%YWT0O_LrLwXiF^5n17Oty#LstMtj0$TE^Z zqydN&29p2=zD0iKNCfQ3qO4}oA#jO@p+=yXGIqGc%fd9cgki)Jo$mdRWT5P@UzSey z%n2MsTH8y2=gyD7C5C@8p%NM>aXLbgUDu9d{t=6ol~>8YZ^dZq+D=m9b>Qv(iSS7Z z8}G4Oq(|{?x`V|&j9dlv>K0P-aN0yQGpeJ4;@RF)cA8t4~R2Hunqtg16 z*RgC-#rCNCPD}8F57>m`BCmCgkL-t|Np+i^+giDD&ZMbO*@lL`aOGzsA@tkt3{`UG zZzfqI(Q>%psmdxX6~UBR!49Q(t#MH!()+6RsRaFL#~)G`>>KIHqu5iP;{wZZ4g>j0yW8*h`+J$20V!h*c_nT)qTi7jNpvS zW#{#%NcxIB(15i9b%BzvkCTCJ%tMK`l4#`g)jKCz!LSC*XF|2`3E%qq zgF%_sPnPp~wminwB^k_84)|-_vknvc;W?P^fk1VTiJfMx35j_y)tDLtOa!5HK#6b( z-5ot0&{uRf`WIU~O`-!+A|D$l6b%PTLMHz#@M!MCQxV{*R4H|0*5|1gADbJI_joSe zj7Tj`lVVPADCy?m5H6xMKI2KCbhu4ONvWSHU@3IKqX{$M=S@1PKc%UrBTV=whIr*#HbL!HBZ$ae&ygk97 z?D9Z+U5IzOele9|B7HBM6nOS?cYTZZ6d|5c> zF05?%sPnZ5^5e4mxZiB5v>0Y1RYZC&sPT+8^K9pgtM!Km5)k83#%K6uq0SO#pC!(T zWq0kz%8}vOHs}k)P$ia8kVNj~*;6T{4;aga2a{G_!#*o3`nBYbmSYBxg&~+ZUe_-S ztv`t;fVh-(AY*`RIE^bhB|>aMS`q_70u&Lzpeqcby&1Ch<4oda@w+;F!`UF^;q0kX zs%L+Bjm^vYhznO+j9wYIM^oTMd7XRU?D9?h|JL^UpNO=38e)|_#Q^I`s(UP5AWH@yBDULpO){OD~ER^NcyV77j+%k$HX#ua@F-|%z>=N*6u%@nWFj? z!_ugc?2SRX&~8(Jh$t`~OYOczh0q$36Ybtw31iN$m{_s&ktXS!W^2*%Yhg=>zAoEKdl z3^^3^bRJ&_%WC4bb`!7K4i>G+$^XVvRKYw7Urdl(*Eq}g<}BlLo;3}98c!f=p|$(I zGZSD)@l6{2o4*-p4eBWA3RZpflXd71%5NfmizpUU%&+LLtZR_Ho7WG5y9!pk?U7Ls zo7+O<{j*ArA`nT@dzcb|#o!jtAKPa&pBZl? zNV3AtjER4ZSz!M*KKN@)Ort0MHFlDTDi=wjR;)UR2pWnk=;=LnjG$|pyu+kN-hE|0 zCdk-m4*(zC5+d)43pj?J*%noQLGC&WM4^*!|2}Mx+dir@1oSJ`rqS2_<@Gm$GM)B{ zoTI@1P&)OR-CIWPT0u&Zb#`As>zgJmy{K|pj$T|HCX^*v))`WvmTq1jCu=Jaf{r*? zf!1AQqLLcdB-^qfO0bwWi=H2dxX3Y5?y3%VwBGE4i!?O05ehcOy^V=u5qUz zZI}{XZ_!GBTT5pDEzyg^p5bC|y8I{J^%m!s?ub z*M=z5U1gC$>iO^9gZN70o*T3{!>h84<&cJN54Xy8LOx9^=)NVNl;sq{Sc=!oLTwe8 ziA%q=gY~Y+Z>t^1|MySu%6;lZdny2+A)70Pip~@l;rggSjHiSeY4h4m4+pfOH=WAp zTuct;vozjHrMBTcE?~22i=2r zSHNy;x~{m^xuRAXe?#8#`#a`#t0G>R9hW$V;!y;5XyOfdlkZ@(1LO3lUzEZ1iE+?J z@qt3A{3C~aw^>sqBvys4f?Q$RlvGreSNknk#Qd36yhH_TZJ*Vm=!=e5dc(=|*F01u zVjlbQsK&Jec2iuli`hoe%CGKcUoEQG;uzMfZmleQ>A`X9)@S$uVK59T`Salyn1u`U zs|i@-w|A)gjDLZEEQFw2u=C+!lJeUOZxuyQ@nM+mF$HWjTZhw&O@CQ82Ne>}dy zfHdftxEw^aLV{2Y`EQ`yj?kU$OXwz82z%qI2*X}6?#RJ5#!Y^NRaBYPvHo^>A6wkJ zEZNFcedLHcw0G;lY?eld<9qdDuG!ffknnt

BDWfp7m>DScJlJjY3jfiX-i|@ zf|~7;{g!uGz9;leY=^h_@0}1XF3&=<3`s_c{CV3Ssra>yawO6eF+|K%9q=p1832+xs~{KPfA za3`C(_nzgWoN6|cA>ft_K?eU}s{pMSBMF_be&4t~ajU1VR>F)2J;sVk@XCg-I1nvF z?52M+jUC5U0uL<&Y2}2c!3BegsV*EOG{RnzsbcPWYDTk(tXQYYe+C#H_QgB2-k0W2 zJrAkz2n-T!6H4)&bziS;d4GKEw@o`?zY1p>QbvmUVH(gcT<2tGPg<1_dFiZCO-gJ5 zc28XT!R1esDAVB3WzjSW9O++KJ9KM@KhS#mBx7*gg&@bs2j0)%&hc9^IHfC(7e9sM zQgQ$OilN9T*iC{YYF{ls5A!v_|C@6e?L*tZj6{yFb~ECB2N{Ac?yyE z|HFoP1>vD$!Sq11s9G>gP*ti67&sL2uZ8!A@=!5j;-DH-4w!DJBs~ZC5AOc}(87ET diff --git a/summary_results.json b/summary_results.json index dd211fb92..4e5fec7a4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.400", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0103, + "Performance_Result": 10.2763, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421594, + "Performance_Result": 0.419316, "Performance_Units": "Samples/s" }, { @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.118, + "Performance_Result": 23.0958, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436676, + "Performance_Result": 0.437965, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.414194, + "Performance_Result": 0.412967, "Performance_Units": "Samples/s" }, { From 598b0cfe598130490db31b1831c9139e140695a1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 20:15:33 +0000 Subject: [PATCH 356/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10221 -> 10218 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index f3088c82f..ca65b24fa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 20:12:29,898 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 20:12:29,898 submission_checker1.py:3260 INFO] --- -[2025-03-28 20:12:29,898 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 20:15:27,785 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 20:15:27,785 submission_checker1.py:3260 INFO] --- +[2025-03-28 20:15:27,785 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 20:12:29,898 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 20:15:27,785 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 20:12:29,898 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 20:15:27,785 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 20:12:29,898 submission_checker1.py:3276 INFO] --- -[2025-03-28 20:12:29,898 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 20:12:29,898 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 20:12:29,898 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 20:12:29,898 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 20:12:29,898 submission_checker1.py:3297 INFO] --- -[2025-03-28 20:12:29,898 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 20:12:31,047 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 20:15:27,785 submission_checker1.py:3276 INFO] --- +[2025-03-28 20:15:27,785 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 20:15:27,785 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 20:15:27,785 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 20:15:27,785 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 20:15:27,785 submission_checker1.py:3297 INFO] --- +[2025-03-28 20:15:27,785 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 20:15:28,974 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,22 +24,22 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.371057 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73142 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419316 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26123 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.71607 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2763 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3258 | acc: 76.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412967 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0958 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437965 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43633 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0261 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9f0c87b5f..4524e19e4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -983 +984 diff --git a/summary.csv b/summary.csv index 674f3e13c..5d8e4927d 100644 --- a/summary.csv +++ b/summary.csv @@ -2,22 +2,22 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.371057,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73142,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419316,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26123,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.71607,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2763,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3258,acc: 76.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412967,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0958,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437965,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43633,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0261,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6845f8e7f..fe75d6bee 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73142, + "Result":1.73317, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":4.71607, + "Result":7.29046, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2763, - "Accuracy":"acc: 75.200", + "Result":10.3258, + "Accuracy":"acc: 76.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437965, + "Result":0.43633, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f097a8b4871165c750fbcb12ad4b78de981093fa..53bd41a1d8d500972a5aa8321a10a6e97bb1a934 100644 GIT binary patch delta 2721 zcmYk8dpr|*AICQw!)esyC}A$UBt(l-#3rrdvdJ~MU&a=q<+AC5k-JiC%q6)@NXVTc zM7d0JSF5B_!%&n^}@y3u-FO%@5%s|yUH6GxlfI2;Lre5@=CC$y>@1v)#?LV#VoT7^asc71Z z{v49Gz8K9eJg4sEX;6kDByepah8Y<)t*Yr|zF4imU^&k;jUo-1%H&Xrz74VcXvt87 zg5Bvmm;*I!vh$asTLzD<5*~oMdQ!5S$oZ(|*SK(($QWHkW_ywg$Ups$H$A2r()D9O z6@};0MW{ZctwO1CX{8m%|8BtM8ER#pA&GV}}EKHuIdef85 z&TMgETRc1qh_a5xy&~<(f&o*u1h*Kj8WsD`q?}4Cg=e9*1OqHoI`nGSZW`Mf`szB< zP9_&kaY)qukhzBEz0GW1^IEsW5w_+wubc6&PJB(Zly^f?4@^9gb5-Q?=XMZ^h>H&~ z?n(1>g~SCT%#wpib3Xx`1AZF1XB((f9Hz}FQXV0LBmX_o_*m4W8Or)&Hhp_ptgtr( zoU)~MBWzDduul}M8!Cz zyb!P6ywm=rDVDMe6YJz5q0rsQ$VW}ad zbQ=7V0bbY+0=9DG@wa%rDwa3t%)fal=2i14`(0WfOOp59)Ts-C6)Uu@{4dy*n?f|H?YKU*0VqNtaZDl7A`o}H^C0x; z;`9}X+)~=ev<}XZ8ui z^x!o}ovuT8e$fg2_S-nIxnXTRchoKo0;SU}ulun;?Mg=aZby=GSuGV>F?ar1-wE5v z{KT$Z&acC_9Hc^7$8~To!!tz(VRA;bJ@Zc>G(h%ADilnGZW}0;?eSgNvx-hE^@NwH zQ-25yz0tPK1F`+D!|BicN4}Nk{LC$*NM`#eR2RLA_i>r|t}KVGjd>UE3$S1Tb*>4} z8o+!<-uG9EVMfvd^d0Fp%`sI6!NPxLO@K&!Nni}7cZxyPQdg@PEzhC|_@wb=8Y(x^ zRp*wT4~z1Y8MnRmM7+<=K1dw27^ssdI;epXREb2AQYTEJCqPd6cciZ(_C%QO>CfgY z2-lK2Bl6nfIcNP=9}YS`8Ss1IKUHzoS%?IqLr`>Kn$@jeDKE6Q>)H8I?+ZOu`NYwz+X zsoZ#qkkucT!gVhG`Vngq&PJRBl@oP9HAN1E+F4#!aW)Gm`lHIR%rfZX zr-%Av;JN9YW^Q$=wA*OxN3)Wx@T9_X(0E}i{VO2#f8P)j2FXF| z>`3rF1yx$q(+girT+YSvf>+k~!7p@a@Hn#FkeeXdiWsrjFZWEXD&I) z;LH#1oGb#}!`QsK=0R_rC^+KU_V!RN$KGe)pDs}yX>0yf(&|IyO)?GCB)p*^X z=TpN@_*UzBT{z;>h8lK=VJPK1eaCGnStxfob;YFC<4ch4-N~@U>0#O$>u1MX+EAnW z27!O3s~|NkLd)Ww^9p^lBaRYM#ZMMkwp`H3GU*zHB4V>CC8U_S46FMQH={de8+qi{ zT%{ZPqF-&4RKMG|>bb7=P<@Z+@vxuDhVZeBONtw^wb9aPQJWWOk3U|g?yw)t3peZmdDv?#qeBTGo6x-A#-; zcs|nN*vK&KrSh}<{kiLIxx_IW9UAlKZ>2M(HucvE%5GOqQWfno9pY3;-=qatykncIZZ^XS>E#Vj;czA)L%B62Y+ySe`t?^KG0Uz(l_PC zzv4FbU++$y`Fd@8%B&?$zHRR+%oYpfO{c29+POwX`Y|cjaX-M)6g*DBaHYOrPWOte zMS7xY*WQ@n1rW`md*pBkaQXuR*hgMSC9cySxA5rom{6&@IYrPnR`I%<^HrkO02rzZ zcCRtH%a)Ao%+S0yho=pYkUfi(u?%#mdzT?gdTi_#Okq4wK9u}Mwdr#fLGe+hi#MQdlGT+eYiO9${fEMBtAg*sUadzWisPMe_n%n!U8s zr}xDBhexZqhmN4dWt_vDo9IaLN^Uq=)GKTi`rrOg+rHOP>^CuPgE5=|RXIQ#Q|=TS zi)!GF8WpH%wut*O5}YumezzZm6T7L9FKXS-IT6R2@46@ur8lA|#$^&jvm_NOd~m@x zne~_VKMI}fx9#*Tl(FNBNLI|o)Ax4N#U3dGJ7_&V3A`ikS!@hi0Jp!a7B-!wzqdO6 z##g;ms%YHn#4D+B;r@~3UAQl_G+WC+(DgnnR+wHZ?i&#>B7F48V5G>TLO*?n`y<8j zkXmp{&}3Ri-VTnFNnjB4cINefF~nz9W1L{`=IckT-^;p;DJ6u+S^D7)4Xerk*AH_l z^7jk+$sLKILaQTc{KMoYD%0rq)?!BefhLo|Uj;R;h5D9D#GHd49=)8}tHx%+v|5`} zk8H^1%g4R3kOxk`Y0S&t1l78$#t8lDkc?@#vH1TQ7XyVj1B_xMAyodpDn>Oz2KYOp p8*v<1&k#rI1Cb0Y67i42WTXsGj1h}O16>$Tk*cDS@U8j=_z!+?7k7a@&r%q?BxjjK(y| z&xu@`OC82?t2hz4J56DR-*ldH-sgFr_mB5|KF{~_`Fy|c^O^Hl^kBRNf!icj=*)i5 z0En=_8e2o!+?&N6q zCtJvhr%r^k!35(EJkeW^udj4mmA#jD$jo=#Cgj@l0)8VvJmLgA<-iM>VVGfKUY-v$ zyy1&)M=>vp@7q`R2k9-JBw@>Y>u;BwZW6EeD)ORp$MmWEsl{QOK*7@m1l|t-0QL&E zHQ+ikHhx9|0NBUG@BI!Yj66NweKb~`0NbwL^7(5ZzK7&f)>*4dnbNlkLL(A&`vH`S zq>bpR$37%*Vu)=ri?k{Jdmrb}gTrS~D=7>kX|jT@LaS?0<%DHSQ4#cv$#bl?hbHqb zq^Gywep2OIvzeFk!``6qfmb#Zmw$?&@@Ao?I3Lwa{pv732YN;oBZR)1gNNvYLCRwD8b#rft%ejQh2h1tOX7@)d2dc-s43?qf}EgyLEc}S0~W4s}q$w0@EVv08Az2 z>l?&0*uNR7al{XuNsC7-zbn!)e^*q!12U$Bs-X$w9or^};>{eTq?n)p5*l?%5PwEMpXIODgdWgIZXBZKyWewEBOStSv>=?tGFxhm1?_%vV$^vxT}8#ubVd>Ks*`l{|A2rS?7p zF}GhAXTX@VaUsT)_0m%17mx_43LUL5#_==1_+FET6$FyIuD)nJtw~TBf9J&uL1JLf zYDxiL%;J|zs_1w}?gLu4f9ull^&fj!b_er^Oz@4i=Y`9RQFWCY_*}ekb5S z6YSI^h7%eqv8%;`uie=KhkZI3d!q~uSPKi98{U`wy-o+|!sAL&&BJyfJjJ`P+< zEg<6j!ln;F$_aI;4~Suj&SgMYrm7Vj=!u=0#c*Q&!8>?+<~_xImLn(rbq2dCG;>AA~?Gk)^MQy_dq*Ui4v`JnD+Cj6TfgjmJKJXRA=H&05`*%QDzPIi%ny zE1M?LkA$-tnm10=1t2m7sIe4TT=FMNWV`A=u}{2hMI}F)n^ZLR)n+>0y*h2?CX^_L?W~wP<%2?g97M-n+k9yg57qq z>jSsRinn3O8~C92^x6_A~IF2;lM}OsrCMe^<7uM2?P;2ZV1LERHZGaW_X3CT(nbF`-sGMw$*=&8fC`b z6B+GJlF!m+A%t}G-U{h@QcQ$&ZP1G zmPEbWajoptOHIhTk#5%&EMM5iv5 zE9@R8Q+3Tc_FNkhsFg`aLFlNTpWOh-A(=}|1LpP4fgDhJRUW9kPvrE3no}AN%=zV2 zG18ReQ`1>QGi%iiz3n)N%9f_3|2+H3Jl{NbCVc1SR%g4|4NjnmhAYSW#42rm?RY17 zgPp$ZM|fq)aUWL=qjmAycwYuBJ}*n|Fs8>1Nl}LlS)o!(Y4g`lK{Bt(r;^t6Hi)!orY~EU{~o+~ z^v@?ddzeWIg+P(@zXw=1nd%3g>>-T4{MEf&R2mM_k8qxdK6PBm;P4GcV7277hZsD- zJ`d%q?1=g0Rq?1-xTV|HIf)Nm6Q_O68>?EZM2ysK5HISxOV@mj@aZ373=Tp39xEBF zUbquL@5?-Jv)0ZS^Ho=(+2L`);Z=FGacTt4qJ{P#EKGhOH^g(l2j2M4x-0qwh)YTG z;Og1R(T+auyP=rlvM^V;ecqtPkIl5Fo>$8e?2+yHm3K~DYNU(zmRH?)X_?%kAQ3|q zxP{N!)R0p@Y+GaOA-y=e5V`GlBT6eFaJrei@A`?AQdli{r9xhoVBTD?(BP4!s4!mc z!4E$_Jk%wotxqVxhkn5qE`kWr0)aCjYiXT&dw=I-=jwXL?s-?x9vQzDX&&*|n5cw8>eOmx-~_AD&@a$q_f;^11&Wh9Z_%vs= z`AMpCeZ~5p3_+fRDl-RE?DvYer*Io}$OpAxgb|wV-mxUs)S%5{Bg1C#Fmk%DwNtp- zEBpbyX!(`OH{9Oa$D1mAmwqV_B=b05AzxH%#c`3TgJY29hN`$Ed5uCvx)~%Kk?7bn zUGtd|GdaOkku-8zL4(6QUR3uRfnARLND#Zb0Y}E|x9HSYyfDieHr;;9N|dxyXC$`} zz>-oLiIor1zoza_up8AF$^x_=iF;%tmcxU;QHlcmF1XxQ;b)mIm2w9=k1hFd%UJK? z_!>4a`+cGHo6`A$hG_2DxwgtYc*wV?&nj1Y@tfks)=WR` z)IC3gBza2LPW->vMK*?d15?OEsMf)sg9Qcv4j%vjb_3eKD=|r$+zzz`z95V0A%O;D oTRrH(pMnDc0M9+b==SeJ;hC5uMNZa30RzdcdfKA Date: Fri, 28 Mar 2025 20:16:31 +0000 Subject: [PATCH 357/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10218 -> 10217 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ca65b24fa..61252d002 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 20:15:27,785 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 20:15:27,785 submission_checker1.py:3260 INFO] --- -[2025-03-28 20:15:27,785 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 20:16:26,335 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 20:16:26,335 submission_checker1.py:3260 INFO] --- +[2025-03-28 20:16:26,335 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 20:15:27,785 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 20:16:26,335 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 20:15:27,785 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 20:16:26,335 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 20:15:27,785 submission_checker1.py:3276 INFO] --- -[2025-03-28 20:15:27,785 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 20:15:27,785 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 20:15:27,785 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 20:15:27,785 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 20:15:27,785 submission_checker1.py:3297 INFO] --- -[2025-03-28 20:15:27,785 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 20:15:28,974 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 20:16:26,335 submission_checker1.py:3276 INFO] --- +[2025-03-28 20:16:26,335 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 20:16:26,335 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 20:16:26,335 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 20:16:26,335 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 20:16:26,335 submission_checker1.py:3297 INFO] --- +[2025-03-28 20:16:26,335 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 20:16:27,453 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 4524e19e4..a8a1a9650 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -984 +985 diff --git a/summary.xlsx b/summary.xlsx index 53bd41a1d8d500972a5aa8321a10a6e97bb1a934..5195017d5770c55e0d7cbcffab3b0b82cd38292e 100644 GIT binary patch delta 338 zcmV-Y0j>V(Pw7vv79s^dgp3^|lN%x)f6HpaFc3xe{R*SKE!&}K5Viwt$*O@uU0S*m z(j*bJ9!P4P{C(wT63D8{(Yi6F9W4?h0lI(-vN>HekH~7X^@Vgqac@yf*}6#||K=OM*iMai^`MK9qKke+^~d zbPqUB(25ak%7s%3e==~> zI*GIU=)`Pn+cs$zy*1>|@aOjBtzR`3syh_}R27^NW`TNbs{&sZ;cyLgej84_6Tlx& zLBH`fE;951k3s-4#GRCn#nW@WBtcc4=J#>Bi1SsQrzBgF{Nbw`dDVXg%5liQi4$PR kef(^b07c=Im;e9( delta 339 zcmV-Z0j&P%PwG#w79s@**S10>lN%x)f6Z#cFc5_AeF~$yE!!c*2-|_SZX=mZX}>W7jU!nB1qs|{E$z(oP19ATzH2k#BR*s%jh>XP74LELF8sSl;ye`7=0 z58VUK)ASKb@RUjK0!k-Mx zT1RnmADx(tZQCa8qPK?p8UEV5zW1xfLUpHNfa;Pn!Yoj)ZC&E)A{?%v&TqnrcLMm+ zDd;nA!y@NC;86%b&T&WObMgGLC0(OBPxED*F5Q~_L*z$YUH4kZ8p003QKpnd=V From 1167346c53a4d6688dc4d0e91a1bede9cc30553e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 20:23:35 +0000 Subject: [PATCH 358/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10217 -> 10215 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 61252d002..e7f3a3746 100644 --- a/README.md +++ b/README.md @@ -1,46 +1,46 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 20:16:26,335 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 20:16:26,335 submission_checker1.py:3260 INFO] --- -[2025-03-28 20:16:26,335 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 20:23:30,345 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 20:23:30,345 submission_checker1.py:3260 INFO] --- +[2025-03-28 20:23:30,345 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 20:16:26,335 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 20:23:30,345 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 20:16:26,335 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 20:23:30,345 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 20:16:26,335 submission_checker1.py:3276 INFO] --- -[2025-03-28 20:16:26,335 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 20:16:26,335 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 20:16:26,335 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 20:16:26,335 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 20:16:26,335 submission_checker1.py:3297 INFO] --- -[2025-03-28 20:16:26,335 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 20:16:27,453 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 20:23:30,345 submission_checker1.py:3276 INFO] --- +[2025-03-28 20:23:30,345 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 20:23:30,345 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 20:23:30,346 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 20:23:30,346 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 20:23:30,346 submission_checker1.py:3297 INFO] --- +[2025-03-28 20:23:30,346 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 20:23:31,329 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.371057 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410981 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419316 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26123 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3258 | acc: 76.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0927 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412967 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0958 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43633 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0261 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index a8a1a9650..4c747c12d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -985 +986 diff --git a/summary.csv b/summary.csv index 5d8e4927d..24a25486f 100644 --- a/summary.csv +++ b/summary.csv @@ -1,24 +1,24 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.371057,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410981,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419316,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26123,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3258,acc: 76.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0927,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412967,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0958,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43633,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0261,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index fe75d6bee..8423e8230 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.371057, + "Result":0.410981, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26123, + "Result":6.21553, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3258, - "Accuracy":"acc: 76.400", + "Result":10.0927, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -579,7 +579,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0261, + "Result":20.9236, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5195017d5770c55e0d7cbcffab3b0b82cd38292e..e1175888834b7fe510508df71ff6ac722c07324e 100644 GIT binary patch delta 3686 zcmYk9XHXN&7KRfD9R)%Wfdwz&m%d=4Cu?lRjhg#Q~(Mc;uc ziY060FEi7!@lZq%Z$eG{Pk4womreYYmoq{|loX^#c+l>|s2uL+ly5$0RYY!2|8o{c zJW9+t)$nIJkr%u{xA z89J4~vunUtcJbn^0Gd1FqM8Fw2Y?LS5>D)OTBl28RkoHfCAKJm%3JRbraLKqpI!on zX$PU9OdsE#8HFdA&lWITcNlGK!dKC7ph=g`sI0ONVfrII94`Ic5xNQKpU`qTY|{?g z%ac9UoS(}QIp=HBkU`tGdmg<&$v<5d^fnSm#1*9#&DU;K8R?XXb1unEzKSY~-__CD z+iy|zslF6MbC+@dBPW;4&INNftKiaPhLq}#RvGKm)$q!pNNz5yD=`pRWl^CxBVY20 zN@DfOM1@o-v)s%#K{$ri>=Eh_x0%?r>t%4CPVI#}69Y;R8vp>HB~Nt#hw8w}0{#iD zD%zSq$;WJG5eAi%59icV7-&`pJ1_=IbjKUJEoSZDGDjyHgBe2C2|+eA&C`a$9bU}s z3l9UW%>qbnAd>w!-{;u4+Vb8i`b63wnX$zkJ!v?UJXp>0xiEj~F14ZX1+Qm;&XYW!@0T+k znPy#{bC6!A@@biNg7o1ma3Y!hc>jYfca`J^Iy5J~LE`0|n8;6#Yu0Od$Hi;53(F1@ z`~N5$aQK{fx)g<6XjOm1#CH@YFN?pXapbL-Na|XqiCTBU<;2aXffnXO^o+C9(J{L% zdhd_NGLwe=N*{F81V6HVFq-JI5GKIh7I58(3l?dV+g8kJKlT!C83qW!$ZBc%`W{Dwrxr`u*Yi~m7Su2MVaziwM~eM!<~5xnV3HKM3N`qg9JMskFJ% ztk~HV%2dpiWr0+T*^Cf)>*P5Rkw98y#VR!BVGudrFQ#HhM_IaU6RHAW-@s;OY`>I3 z_T7iq;MI#%abO+$haw*FK{L$Ql6{F(Owde08|%{PNmKOkjbn`<=CQ8M$j!Zz*GF`} z%|gb<^QWs0M&qB_uriC|ZH!~id~n!=yWqlqcMGv}qKrFUt$&6u2Zlj0Vru zTz&rH%xE~>=+CeUhWh%#1Qav$kL^P|$QQ*b^}(pqw{92f<6Ze~LYD}&mV5)5ur1r_ zR=4TY(ae!_obL#dAk}w=3}=7d587$BSIGXZm0*LDEZ!R}%eEI*;7*LIy?g%zx{@U9 zyEdZftWmrwWs#O9X&dG5-g}1X2zU+2u$91%Fd|`VW_258E@@{h6Ci>_utNt;z3MY}pcPH7FsiL5YevD{&UyddndQd6FFT(;{K2 zW{P2B!(Ag@VvxGFK5sCRA5|zACjNq z0gt^Upa_Q;FFt7=lm^>SV4X}O_kPY&YrAjh3HPLt_@V=JGebrXvWf`F(F1k#k9l>~ z`eL|}PN11DFS)SMT(2)%FCLy_x@n^qBUtj@j>$8o$ZRY(>-tr()0z|OIrMNqFV33S z@mi3uD5~n9Cl@X_2q2V19Qqa(SmQw9u9}f`xFfokU*_>fX`*}wn$~f#-B=vfq4-r8I+P`LxS@J?$%d@Bsi(0O?fz| z9Awm)!#z|VNVgkxCCl1qJt@kp0pW|6epWjB;bBZ-XpTI8VqbAI__vZAV+kDQQLSud zVH?siE{h!X?|C4{KxPkI^^kk$BDC5Q;nWz)oAHb>t$ z^$uJ_Jm64|4rAoRxkjauC>QYb;^5Q_ieWUynNtT0)^W)&_J_YlCN!h zDVRRDba?5u7c%tsJ4-_PPK8c(j=|F^7Nl(`nP|yJ1WIOUqq^mA2F0m%zhLD|2GFyJ{)r%GHc~l*d9=V0^}z*Z-S#WKNi8|z&}>igis+lZ8XSq-7H(MM8dvCYuTU!0z zI}Gj$DGtnIX!6?EvmzNzkx-{9R;!og29=x)nwbV0hBj`>Vch7NnT7$+5R~p5hoROK z8lLsqYy(cUV|jzwny9Ec1xWjt0I@cs19LNKW}K-XATGajtUQCn$L+x zp{Kkud8KnpH+;Cm2qZUIAJwTIe%7Y{ve18FE`MPuWQqR^lldR)qb1;P9TT>Wmg7&( z6WIKRUaa&L8-8fDP*wRJlNnOcgy_>ou;(Lgy%>vgXYK*p^A!U{x*sp*sjtFbF|C5~ z#kc>sZ~oCZ?_D>8+jMSx5Nd0gdZr4OE82laDzUCIT^MuYAQ)25tB9j|$}(@aW5O?B znqqW-ANSr}M~N&o-iVK=2{TX#&7{V_7l5#Y9JLBo=$o#U!cQxd5^DGa@MW|+lDRZ{ zdqa{NDW#G_Jt(a}Cxjngj#*i;)$+Er@`+HfIvOj#B&60geA;4z&S{z}P_}8BzRLAE zR{MDAS9@yU-n_1?!6}PVL5USck(edz+ZafVwCZB zcZJ;c4mjtFm}dbm-4Nv&CX#5Qmc#9ryN@Qvr1e@>PF|XRS)(!C6K%S}8SRKV<5w2U zCWWv*2U2Znat%Ep@v;|E04GT;m%akw-n48nB1{oW^KJH>-wxpBo?Bik`nm5%g7=!Z z61V2i=bu>otPq4W%yp9%#8v`@;UO?SbpQQUY3AsSQ2N3~iHTxC+Np@T#jhF#k`hgH z9(atcy6%0R=1_KFU+`vW=SW|ea$||@x<$rusR!~fep@*x2{O347eW(%D6ZtscD+NC!)c`sJuY+F zUl-1BgSR^JBZz^KFTC_QW;vQI=1xpD#&+OaWO&ZNq@621xsP`Q)|J7QmkY5bWLD7hFj+r04!^Ev-GOa$Ie+t@C zK!pBWtLb{pgaA{1asEcyHdRxt7r643?L~vJUKU3}9kfQoL+`^y0iQoo8Uc`PpLwo_ zeW6Em&q%fcF%(pPT^)W=P?P(w0iXu}n9TtIPO>HVcZ5f{;x&Y{fMNJpAsF{xg~%>Y zgdG4lHxK`g@CaIbtB@v;gr^c#0t(_agoU`z)qn#5022xTfaTwVDkv+RGWNc%JWSfYYA;c(QX6!;!C|g7+#k}?z z`w%MOwQr%K?ELBfd(ZE8&$;KGbMLvI^L>`{oab4vT(Ye0W#kADDSsQx1OVIx5ix9M ziQgFw7+?{#7nyD`;KuV2-C*AE}02?RGl zDcQWd2D!l-NZ)qqJo@OY77hD|Ii#>m-{&b1VifWD_+w9Lb1C&Cb0c0~;71KYUs zoa;$kfD(E@3G-ZAQ+Ze<%)OLJOGBGg9#WrV|4Cq{u)M+FiJ$CG37&AwUq9WAD}GG0%tVimw`C>)-ug3UjWv6n-nsY$a+C^)bj(S@xe z;%HBA%6Yo$ltVp|Ms2Wz#4MYp^BLuSaHZf-A$O8|u{{6FlsnXS?;>>Vdf}Z80zv6D=MjjATr=IgFT&8F^GWV8n2IitYNAh}Z zuKcy1VW9B$wAVI0;#HEr5o?@DhsQ;6fHziS(58u{8N8I38rPbeV|uO#Q!muvU^MzBF?%j=c=rXi;U!|&5U{Rqz<)pL_h_%m+)0*54=ya)=rKjew1p~w)>pt z=v@|Acm2A;(1^n>Cc1DpFT0~*beL^&b9#bqAQ*kI#h#l^&ca^wcUzgQ+Z-BE@(a~| z(@`?D-_&fpY}`MwYQ0euoF_FQqv%D&6$iW}aE|8OzX~l_Em8%O?yF?K_tZ2p>;|<$ zKvCC2Odx$#v&qI^U1@6R5eV9~cNRBp@quKl6=mBrl|=1bU}bOb+6B8l`}Cr&T`thz zgFptkg)igyHrg&Yfn%pwVD8h4kK_zd#P;Llo7-yXN$Usk#%=6TS0D_Z>fwX9FBu=s zgPMn;BzSc&VxVAV3bpzxJJFhctG2e^_K!pDPao3 z+aI+)w zD6w80Ht;kqSy8elk>s5ZynM@XAsyj0qFK&L#F0~SJU;=LWE<18h}AXoe3f0~L#taI-umL&_+8hE4ri5{rzN5MF%9mCB=)PnOeO?~bV@6#oWM5D!Q-yFl^&CiZVx_?*AD9$_wIX2|55$Ry1^ zy=<%Nr-m?0DW2~^q5HyC-VArWr_H|K>EzI$DV*RsiT@+>RSmb-n`A14JqFivk7;$a znHB17mKb)KN^*Av-4B&ANeM-*907V{{pHns-k|4uGHtG*3O%xVgtzdQFU8K8K&@wT z2`ARV8NEQ9X~$}ZWF?3Y0Q3~7uAUW4B`JsuvVX(q+hrL7vwOIikk0)=<~4jjXUrX} zZ(NY791pK}YA)BgTLKIrg1KdJnOsZo91C{IPvO3TRwnR4 z&5{ifGJ-nIPP?y-(%k)M?vHUNV1p-JmRKPjkgag4M*`0?SibImU|wq{Fm{Qk zR0bUPaV8S2d4zG^l`ypHnCXJf3SDZ!=IPD0Ob0MIxOh0xH^!_a-36L5OYnnaHBgsZ zLB@&?Ky_-4n4;n<+Rcy6*j5IWwcMb*@*d&sHwrud-oFr)m2uh$C0<^O23M}!>h8T_ zJ6DA7;6;4F9J>k)CL5@leZpk33_yhRYpH9kAV97m4h`l+gHJrgmW$jHcdsURzi^i< zmqYLE>w3X#-}fis8T1oE0tB(~q;rolI@l9w! z49-Yo^Qi51x!M@y{ zb;ep$JGPLL*yH2B{cgaiwcmdvV7}4^q2)!s^Iuk4&mR^P&qDpf(*GY;HUCM*;(*!S z`>+NNO1~%a@MY6I(`C8#XPdje{e7QXKRR%c7SGibk0$!+Ed3G}ihmV9dTgyJRCH&u zFg$%UF`U6lAQGb*@h2kIC^H*&6j+GG5n9h(;dSgmo%`N8=6E~HyAlYKz+!=Fr*g}X znQ!m3gMc>|cua28oqr62i_Msn9%o6qWIoKC5l;94NH@F&9%1Kkn%$)Tvl*MoMZ|-I zaf=oG(ypYNw@<4=m2~oVrl@*9N{6V6th6O(qg#m4pje&jd!s97q+nFy;k_Lsp=s)g zhAU-4rJ%>?>gK|3dpde|+W4Zy2)ID0w22qzww6ZFt zSAs1E4tje?8(+J(*cK2lrtWv9QEj&W`@-+JN;fruvP=AocKrFgsMx!K+a8lr+lN9M&l}hm%x?uZByHF~E`=Zn`%~rTB8PD- zUBOfeo)1aI9I%(wI-XBpVE!|(5vL^$g#Wbm8~^~19RMJB-0c6mVq#Hj#8@^VqLY*? zFqRlEC2{Ugl#Zu78GZoZWH$JB#l)T>HcJ_BzCH^8LhD8Q=hZ3!%PZQ5e3o%Fw u6AgsL Date: Fri, 28 Mar 2025 20:24:49 +0000 Subject: [PATCH 359/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10215 -> 10216 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e7f3a3746..9d696492b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 20:23:30,345 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 20:23:30,345 submission_checker1.py:3260 INFO] --- -[2025-03-28 20:23:30,345 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 20:24:44,161 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 20:24:44,161 submission_checker1.py:3260 INFO] --- +[2025-03-28 20:24:44,161 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 20:23:30,345 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 20:24:44,161 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 20:23:30,345 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 20:24:44,161 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 20:23:30,345 submission_checker1.py:3276 INFO] --- -[2025-03-28 20:23:30,345 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 20:23:30,345 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 20:23:30,346 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 20:23:30,346 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 20:23:30,346 submission_checker1.py:3297 INFO] --- -[2025-03-28 20:23:30,346 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 20:23:31,329 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 20:24:44,161 submission_checker1.py:3276 INFO] --- +[2025-03-28 20:24:44,161 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 20:24:44,161 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 20:24:44,161 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 20:24:44,162 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 20:24:44,162 submission_checker1.py:3297 INFO] --- +[2025-03-28 20:24:44,162 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 20:24:45,141 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 4c747c12d..45b0f829e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -986 +987 diff --git a/summary.xlsx b/summary.xlsx index e1175888834b7fe510508df71ff6ac722c07324e..8bbf823eb17da53a7a4a06370730e3be863cbfa5 100644 GIT binary patch delta 339 zcmV-Z0j&P#Pv}pu6e0zYdb4;XlNll&f6HpaFc3xe{R*SKE!#;;5Viwt$*O@uQ(C$c z(!>$99!P4P{CyR_l7LrTj_#c^cLo*fqimvIuuf=IqAW>K1d3@cR9&L?-6me4$a$(b zZM1?Coxq`WbyqM$n6~h0wE^n|xF~>>Bg~ZO;JqOjJ9Yp`T@oBBhRkT z(;9G|rVm(xr<{6<&%oGBL^L2dn~>(mHXVR73=K$7-r+3C@Wt!FN;likDOXM@{K>$q zbrdJ}(TT~}wr$cbI%~+E;m_^MTeoT~)cI5lP*re7m<8&!tqOcygu^w|`E5AyUI2eQ z1%2jiSfuX*9)$p;k2@+Ki>K$^B?eV_n%~FiBFYgc l+_#^g`UA6BBMSrtk$SUuC6j3n003wuocjO( delta 338 zcmV-Y0j>V%Pv=jt6e0x_s30OFlNll&f6Z#cFc5_AeF~$yE!n0e2-|_Ssg%KdoL$r-}hNxj@Fnz$Diw$w{g`}>cgoxpr#U>atACJ*Hrkrh==Q#59@g1jRf&H zM1AJ%w8*g!L=pqgW8AayQ9eDlBrDVuq_|H>nHGz-AhalHNxlvvZ~8AlbsF+-;u&zv keftTTKeJjR3j_res30OFlWHea0a=s3CnE+6B>(^b0F0QKUH||9 From 5356a07adeab32ee6df4f8969562bdab50d947ac Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 20:30:22 +0000 Subject: [PATCH 360/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10216 -> 10217 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 9d696492b..7d7e5446f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 20:24:44,161 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 20:24:44,161 submission_checker1.py:3260 INFO] --- -[2025-03-28 20:24:44,161 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 20:30:17,127 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 20:30:17,127 submission_checker1.py:3260 INFO] --- +[2025-03-28 20:30:17,127 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 20:24:44,161 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 20:30:17,127 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 20:24:44,161 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 20:30:17,127 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 20:24:44,161 submission_checker1.py:3276 INFO] --- -[2025-03-28 20:24:44,161 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 20:24:44,161 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 20:24:44,161 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 20:24:44,162 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 20:24:44,162 submission_checker1.py:3297 INFO] --- -[2025-03-28 20:24:44,162 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 20:24:45,141 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 20:30:17,127 submission_checker1.py:3276 INFO] --- +[2025-03-28 20:30:17,127 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 20:30:17,127 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 20:30:17,127 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 20:30:17,127 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 20:30:17,128 submission_checker1.py:3297 INFO] --- +[2025-03-28 20:30:17,128 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 20:30:18,087 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0927 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0789 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.13252 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412967 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0958 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43633 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 45b0f829e..60d44837b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -987 +988 diff --git a/summary.csv b/summary.csv index 24a25486f..46aaf3a25 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0927,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0789,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.13252,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412967,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0958,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43633,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8423e8230..f03a728dc 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0927, - "Accuracy":"acc: 75.400", + "Result":10.0789, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.13252, + "Result":9.63938, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 8bbf823eb17da53a7a4a06370730e3be863cbfa5..f9722bfb6b75f9bfef5db0f43b91c99fed947f92 100644 GIT binary patch delta 3878 zcmYk9cTf}D(#C@(KvWPShy)~*gpMeL-jNb|Q<{{}K{^oxK|=2+NdRdg28i?)q>2bg z@1XF4N|B-%dQ*yc;r+(%-ZOLN{IT===GooZGqc|fRt%~J&jYPZrMn5E=QV(3R0C)N z8;}jFlgwxA*SC#Zsr4M^4C>*|s?=DX-5F2!hFSrGmz@ti< z#3kcwmxi-KYG~uBRbz&)So&b(lZrw}+t^#$kf#IQL+I`A*%N{0%_p&Vmy`8rhPLhS z0yq;K54uU0HifU=_%R>NUc>xxtl^daori^tIO!teTMonDqADO<$-7JlM}bOga#G(| z6A1?rlVS!ESO5(=829PN2m|Ns z+e)1@8t-5D^|g4iPQ(aeTn|hCAy!6NaU2u{KeNSnmE+YJO6rJ+=D4mqLc%4cTY~1S zyor4EYp>TP4k@j)798Vg2c1cAw61-GVd`e8MnWPD8x}nqdykFs#YUEADM1&N<;UY# zgQnZ@td;T3kckW;v-|;>Wgo~B?>k)_TV7Ni331wl4KhYcWym+xiCyg?3nF3yMHbU@ z-!N@oFSgR-c@&>LPMgm!AXFUa`KELnCrH=Dnz&ENE6F)2>N&A#_(r#{x-pkH&vA|C zw)5NYKC{*PqImBm93=DP71DOB-0PX2j0@ze>vMyc#0HA~rS~2X)?Ed}K1*=UV+zBm)r(uvGKl){Z}uJEpGE{q0pOg^DMT-Y@?eaZ-m?3k{eRyxp=d)+nKRD57eGozSIOiO?d$`>`lt$1s5sH-^46Q|$71(wDZ911H4~y17|FV>rupNy zZ7535(99anf209B2&rUd5nD8twCatal*eeu<__i|VW}~I)zBh!nE=7pZiU^Fp`@?SP(Aopj1m{bw|xf75D(ZF!66?aHsaJacXLrd z16Z7zl?p6gE$8qmE^c^(31OM@MG_ZBK8i=ZpIT#xmu`M>8vG*8&FJ#;B2M}^0`m*> zP5InC8y|k(B~UCx=fhD9O0C5^UhTsXJ>sqO&y5YlBFk5rvB(Au#j8>E>rcNJ7{01` zsn^9P`wI(qkiEv^k~7S2L{eT{wVR8~zufwwMZp`pn)BA`M$Fd-5=eHw{LaZoqB)Wz z!tOVu^-4kIvI??+b(mTuwPG?N8y-i<4&3sWeL^lY2*J2d4r)L^ZGO0qJrl;s25SWW z5T^Gl5MHmcNwMJU0jZXQkV+}r)oAjRb!`S>1#5rbyS!;7n%ov>epEPd8NVI~nY6i% zq;_mNF;>@FAxfZP>yAZTtXN8Yi$6m0XYRZ$^ee*1_}?(*t9U)h@7tLc8Ue|Bf`a?2 zaToJIe52n!yja9D|Ml6=?ugB+F-P3T59F+J3MFHtI=>zCrlIiHMweVt`AuAhK3~AG zKf!u6wX7Zu*c|RTrQP?=I4AQbH_FKS0j89q|K&49B-JZjxjMmEJ1=m_>!oXu|Ll&Z zKCxSCH$}~tszoDS=%$EZi!gFN3~|-=so4UL#afVW+gyb9ya$Yo?%iz~`Hc zQn0n9dJuv^{%1YIF6GJ6y^7X@Zi_p-vA$C`T$xhd@IuWD{Zx_fc*aqjnHmr+Eb?v7 zPH4WJZVeF9M+BA+w$Y`JjPX9_HyMg#nBcP8u2^tMx13>@{UkUS^zPuMkTZHe5}j+4 zV#NOWb4k-P{UoI%7X)+q+>W0@6?2}K^^x-oQ}M6IKOLffPI=z5kB3g;%<)HiVTe&c zy4rbjBDHg0a*Vwe1dt8_TWYUkg>zJOvL6t2gD98VWs$o2nYuvCvMFd7*Iy16KW~!l zzXud6Evgj_P&mOgOqWP?3fHkHjnl;gdW2CU)9fLfCUv!1e0?AAx~vl~ z?ZS8&=X2bC=a1YmupB(47R!uJ=Zd#9IY{}_XNK7RF`o};T+Q3L6 z>FR$F;@z>t5p(p>DJzf!zaoL+9R-{*iPXKdi9K*|*g(Wt4{5}lPK$c3nUmq8oe_W+ zLrs;jPo_3mjX#zq+)Xu0(7Jci?a*dMR^fLW*+vY#%KY-dIg2) zlG0;2MH~7wSFmqE=^2p11udwFs*ZIF(D*;nHD)ALn zdik`rKW-!6A;wWRGBx~qk$4?z$2~hR{Fyo!YY8n=JR7*VAw-nFYetS}ChgVgCi}`^ z2lPp5OTTu4FgfMprvO+v#b&(!w#iand>5nU^y9cLLI8}Yg4FrvaR&7W9I8z2S?_=V zveW}$bOErlUeRWue{$JX;hl=@5T!x^N5=||iYCp&(KB^N1*I1KTWjo*WfL-3SEB!W zTqumVRWKXt$}k8Cg^{R`@zf*2+ecNvFlXGX3uco_Z}TWc-)94II-!^Tag2dwNC-(D zAMlQTc`H#$#=z)e{#qnt zGeWVXwEA4n<&^lD|B;F5oXL9NP)>5snJiQN|7Gt?0Y=xag3#$(N8b~ED?SZBXg$0j zSX{|O7^CDr=5x7h@=3;!==~mNbDCD+*k^I z+24a5AQeCc6p@#z9KB%Xy{4=CL6`a+Qy7bLA86L$r?o}U_2r5yEH1O-K^M*WVo}>Y z*H?DAptiD)=XdR(ocJV~IAOoTD*dn??pIH#UtO*}`yy%)xVhbzcFJ(fApker zyFxH1_ygyDQ97(C7lHEZZEg1OP&o8~V%0k4;2FC1`7ESY?#zvD)(V%mt=4QSIiCOG zG&s8G^4oX6EOy;-FR$BCo(;P7qEr(lJH-PZ6t?j8d7A;$*nZ5xJ`vFzbO?6poJczP zg5KLiIdCL*xaNe|?&T`9_}yCJ(CawqU&^~m-g=QZx0E<@(dCc&aL``jOIy}h4K2&V z7G$~Cl+Jzx9pyWE<~sF@rcodH#AL0TsEZO5ufaIer}pWBD?z$IrvI86;MjpN0SU?CY_^lAh)rYTmU*>0(KLwal^2K z_d9MfL~T62kl!paMe<~*JMY33y}m+^II$_Lq?qrN8#@D!7$rJ2720*oYP zOC=9L-wH=1`otq3BcExD>m?4&MaEdQL(rC06Rt(0RPtF;51H#z*=W3n%zacM47gFj zeV)kyT;>VY-{cGjTt;8!H{8;%X3(j}cBXH6*MY1Uw!@f#yyoZz8d3l^kx}4RRIO6Q zbwGrJt4cUM$2W^7!!rQ=gKWA;zxg|2?82XGbKm#2{PYtGuzu#hgib!d9KQ~E3^Ua$ zTsP>Xl1Q6Nl|#2RklLTA%Ukt_n0H;AZ+q?Pw9!cVe;)0J$b4pjg?9&J2YfVo~U#Gx08NFsDl5yE{O1& zQDF(uyxeSO6%GOd0Q}4Vz}aH)_k@Lm3GKWpbhWGiK%j>x=0T8$6NX^K$3%ehLC^8B u6I9rl33vGz*v>kFpaK94&jA38f7PdNSU3$Kj8Bp39-)lN9@ z8p>LPvg9ROB-D@hyT0rDo$ET+`Qx1DKKFCq&-2H#Vzz8nJHP@=?sI=e9%eBDz5}>> zlSP0cxS?dJwqTPa;z~Vs6k>=hs<(U6-?H>vZdRq~RN}=R4+EpcOLL3kn|Ipp^}gAT z<#k4~u!?07c)zyR1!tydf5tkHuIdt``&h(%Cwu4N5(ZY}t$u&(G<^#q*Wc9^c|_m_ zcY7jJJng;~^FHA-s?+b;2q95Ep64mD(zV>j2`6_*Z09IvYX73CijGzPK{1qZL0G)g zzP=KO6U1Q7jPLbEtpxqpxO!$X(aZDWsAjpAI|6p5{CfZLwg$?UkY*H`2pUNqZ!ya2 z9eG6~vMWf?C3K33$aH53R1sdFVY+rcdNFlC;vTl3P;(}Gbz)HDpmyyMc4)0^d|M!! zsM_uI_ZXijJ5-qGT!fn4TGUfIl2w$+(^`h)HF?I<=WEvCfEy2_#-wfD(4ixifsU62 z&bhp;aLe=mAh6MCS%bwy^WAI|qkL1UTwQS;xEI@3qtR%?fS$6(43lmn?v@KutSiC= z)8iQP3~U(ob;Ivp3RfLER(cOzj1WN2F7goDYo<7I61xcQbK&VLPGhf|AE74_uOlgb z62)~++}qfC4c|glxFu689m;%a2oNVLc~f*UmuQ*B+TFu&j6mhUOD?Lmj1bRR|kK;Mf7OtGe-D`FEtu+c?3xR zS*vD3+J9Wcw`cUWUSzx-7oupI%GKsEbz>xTxR!6Is9^dI+YKphYdcCH&=lb=5952Gs@J4 z9d9>W$6zM+HX}flcfl2oPbI=fZ@e;QxA-}Lr|5KS`|J*sMpTWCj`TUSxDHHCfLG(U z57;ECB(%aBjR#`7a9B!kRa2Vy)4DkmwOhWbDkGcTY$f{zD#>p@UTD{lai9oG5+`1h zOrd^(a9hOux=A&mV;ULQi{0|iNz*6zTZngsY?jV%S#*9VN?e6@qD$l)Jx|X!lnMyu zmF`${el1G8WYwe4aIqU*B2=|A(e~ZyR(lNr5&m5Y0Yy*XM4)`FZ{Vb{>Ms|Fr}Ad_ z37=P3i1!!=S;&q_9SM+MU<)r{KtVm{ z&I2`A?HASrkg?-~i=K(0v%G}TJvkyibhfaAf9dy8bL`>OL;X7cphq*%zect3Y*!X0A^4FW4iCscKYxCOH|TD^mpcSspQ=}rNGilmz79O;P`=5~ zv+p)G$dz}V?+TTpH!{LUbB6Xpw>urxb3Pd+*^_YcwB51t97lDL>FM$gF^@d@{`hwVyZ}FSkh}a%?yIWxnSe9T2MxY;H0*-mDNQ40=FIQ; z2*oK#SO5ZA{caD<6ac$*xb@qPcc$xsS!}*fLK6MU-C)!3{QUC-7=iEhV1Y#R?n0m{+H&E>U(lU27g^RBLB?JeDq=qh9Cr#H+pI&H21!c_*HsE$& z)hBM5K!Vq?;>#bbEqw>vw16^Xi6-x}T%0P7{7)n%)WyLCX`Zn7>k=9;FwX2tNXxJy z$ex+tViCQ&_?gYfV-qFSn@$&t4M1jwkNwOlp`^wRzT^xRMe6p)ol7}_WIjLZD!_iF zv3#92I>&R(9u)^K{reV=cU+0(cwW|(%QC;~kL>0~1NumI)UFqB$`@%JCzMJgd>B9} zjoSAsDzqa3Bi#(5Uy=?uaL?b(lhNtYVm}S+65<&I62Xb~vHY`!|6i&%+?~AW)p`2v zf~u4(g>e?k)q8^h<1Zeriwe&AYuYp=-JK*|f3Kbb<5xvW4g<0fEUr{G#(P6o5>VgW z;3OB8th!FAXS_;ob_ua>N3AFhK54mF!Hw^%^h9uLPgOb0*EB|uajKw? zrb7yh?0x;!F)ri;55%2D0N?6WOX`7fMD|Am*9P9QE4d|^X6raG8a^@l~VqV&c4+{vjdWsHdi=0M=J>td6cz!Zn98HoTM zPd~{P28NJ5B2U=b@MXc6((Ka;nl*mv)4Y#8tleX>xFE#pq@>3i>8W#Hu@l{ss%&o< zQj%E@B-54_tL{GJ(!_B#i@kib%=_e$oF*dXIz@3sky{bvtvGU)mKd&C$?y4I+T@*$ z(}yUb{72M&S}f=vO%<+E7}T>?%hK8*@TSb!JEM#TFS>3*vOc1pZo8>j$kbHl1HLK; ze9x`G9;B3O4bXaD6qR2%NZD_NW$Ssb%!-3;!NGX8ELrC{XE4r*&ZTeg3<=jc<_2e_Q2nPFML7W(e>%Dc3PDdGFL$<% z$q#5FVyq~&+lt)gC>4D-V$m_2Op>;aiG$C-aF7hAPvRC$q432Ex0-GKnb}`045#Jg zoo>M0|K=&(?e*1S<*Pa2F&}0s6yKHcMwDtGW2zpO$yN#j7u5B9YzF}{40lCYe~X?p zN5@Tos2C`o!tB1qWNOeUFV?ZO6vmlXHo9~Zi;4K>l?^3hyV5u(*Yrs>AI2f#7|=$H z$|Rp< zOVV}yS8aT!BkHSP%U$8Qkr5*aHum=1JC<-N0?#%q>%8X-)+c)a2B5EbOaUH3lMyNt z5d|kjukv>NTALHR?`FG+jqN|X`(Zudw7~q+&DcGZE&awc-T$|??do|I+_0vLX$#MA z)5!O0DtLE}7M@W+HiE@NXg|V^S)bnXov!gSQu1d6QmQG4!}$W3&0DFRr$1Xz_7A5wk1ZLU-U9wZo$NQ@uabDFa(L_tw;m$KgQ9q-$X#&U|L(-OV^>s? zxb(nWdM90fD%; zPNZu2#VJ!(BS0QmEz;4t$76}nFsJ%cLx?g+ehXY>80qv`zj_*W|;1 z8RoG;anj!&#+t{jlF?#R1E{HbL?#;^w!j2U%GIspht&0~6un(xk<(p4WKm$CTwN6| zyenD^vZbLBu|?R)LwtGoW8VYATb8r+U0wvEpqzI z6;W*~N6#(Zt+89|N;h8;j&=6`Bd!I{c^X0w=g(mRZWx>!p^FI8n3>$@ZD%bxfy7XH zF!>x*4APe-zfM;?&Y3M!^sAXgqe&&vQc)hgv7DBUB(7 zIcbaCY8N7vxww($GjpIwjErI Date: Fri, 28 Mar 2025 22:38:32 +0000 Subject: [PATCH 361/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10217 -> 10219 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 7d7e5446f..a62d4a298 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 20:30:17,127 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 20:30:17,127 submission_checker1.py:3260 INFO] --- -[2025-03-28 20:30:17,127 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 22:38:27,179 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 22:38:27,179 submission_checker1.py:3260 INFO] --- +[2025-03-28 22:38:27,179 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 20:30:17,127 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 22:38:27,179 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 20:30:17,127 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 22:38:27,179 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 20:30:17,127 submission_checker1.py:3276 INFO] --- -[2025-03-28 20:30:17,127 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 20:30:17,127 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 20:30:17,127 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 20:30:17,127 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 20:30:17,128 submission_checker1.py:3297 INFO] --- -[2025-03-28 20:30:17,128 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 20:30:18,087 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 22:38:27,179 submission_checker1.py:3276 INFO] --- +[2025-03-28 22:38:27,179 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 22:38:27,179 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 22:38:27,179 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 22:38:27,179 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 22:38:27,179 submission_checker1.py:3297 INFO] --- +[2025-03-28 22:38:27,179 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 22:38:28,184 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419316 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42151 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0789 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2468 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 60d44837b..6dbdf1f4e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -988 +989 diff --git a/summary.csv b/summary.csv index 46aaf3a25..47131b6e1 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419316,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42151,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0789,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2468,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f03a728dc..067dd19c6 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419316, + "Result":0.42151, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0789, - "Accuracy":"acc: 75.600", + "Result":10.2468, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index f9722bfb6b75f9bfef5db0f43b91c99fed947f92..568ff3d385f166575ca3466b59edcac57a60c8eb 100644 GIT binary patch delta 3534 zcmYk9do&Yn{Kq%uT5c1C%32$%mg`(AjFC0D#N>V{Wv;m;2{WshzHS@7iK6A22rIW5 z$)&j)LP@?<(-3l-Tm1N*^ZosPpL3q`{P8^Jbw2OU=lz`XdAL?HRu$d+qGPu6HNhYN zAQDWA7M7ze@T2(*A{d&WKtAFdy&C&=s4(7{O?AaweJ&%5FO86Sv7gpT8(sYz?#J9z zHO~AHb}dHzIF#>E$dU)4mBk6)k<)9BNoMs;H%d%C(hAn=zW-%zIeK&Kc@4bYK&8Gx zMPE%UCCpr?d{G#bC{8kBACpArgZNB`$B*gX6*sLR$9#&Xllg7ONJJ%lRvlXpGUMEG z5;5<(Lqyc~vbYi$Uf($Q@vpGx`2})W&jx)g9tra-0_o}K3n5`O3HTA&_1w~WU&kMG-*yR~Ym==! z16dDVqws=e|0X|>*z@NWU)XH*=t>mCaAvF~%&bEuD;4#_jOEoN&lx3zepL$7e}GEt9N{bOe5i{;BzsJ7a^0D~9tsB(3`k8s=g_d7iO|YTdHA)h@c3h%L+Ep|1fv!kV zW0`o#%{Y8<)o^8pT{humxl>_T+3eDqfyj`068>eUR0CIO>=_xcbsf`rbn>>&rq^J@ z5Z8x$t<1fOMVnTcli%Z=(yta>dSD;vKjD3{d}0^_HQ-4@rIcsI z(u6(6!9=Xz<;7%kkkvc|v#?AuZ;@F}GjA!mAx&A^vb3SMep;bma>lGM(4N3}^OjG4 zEMSwfPoyZiZ`>1FI0tjORdp4>4vTRnl;f(p=_pg?jRef#W~MoYId45l3EkOtGmbeH z138!A_x&n5Vsm@s59sDeg(wXJc9XK)K+_>T;iiQBxTY_KlaP-=r*wAgqAa@JNXL&-JqkAe(x(eSuVnW#DA%Fju!H)8Kb^ z8D2*LdjQmFYJc1rr)99NJhO&K7=T-Rad>%cRXgiOyx6~O*V&xVW28~JWF60%w=nC+d=YcTaeY=+`#w;jPF+?a8WtA zhp{k{4Z>bQM<_It zFg3B5{X$?@ER-U(#Pv=`*b;ia5G5%1^h%_zqq z`e>0na~RkZnuegaNFQPHM6F>Hi)XCR5u?(1IURzzVUWvojY6_|M>@`ip5|`T`_?x& zJN%C4TOF>7=9jwFUbjqp8#fV9FY`2rSwFV?D7j|+gQJ~NeDEg~!hzJcO7iEeLyS@H zjt`havdq9()DzZH%rl~=MHL9#K?nWnZW7Dvo1|H$pO``m&!no|9{8<^J3TM_1TtWi-_`(P~zzEQEMqAb#wka+_isgDQ$!xLtR2MAW*d{G8^EM7yo(&C2(qvzhQp zOwv^xvi(=&J1PNyqXUsP{F*<^Oe$d_fX+Bj09;4b@@cE`VpHnYT#RO`B;Ax# zhi?k~9^&`D>)qgW*K?$Qi$O>h zhTjV;mP(O_*&a;8f&p0qBA>I#5?&p$xnahvbk8GT!5X}yZ=G#OHU3@W!98SX(am!e z-NP43eLfNOcuq3oO6TlrHwk)Fiz*ZD1q~bW9uSK%Qt}NtF^YuJU;vGM;8!|Aimde0 z23$;*6ZJ&1K5Bm-44R{vy@Nn~q}em@gATqu{(-$&fn(K9`tcSGWyBXjHD5yDS;fI; zu}Z$F7IdHnMUVwMvt@L>G{J%jU?I)>z7h{dL2Rqn@}-Rn_wN=xM-%Itqv^aO^5C#N&Cv<^i z@oBe-9tS~meWd6+uQmu^Aackf@m!jscU9uz1$7@0rHj4Y&L23sHxE2A@+bOyJSKlR z@@9fy7II?9Q{IER=9!PSkdX^_Q0}j^P0uTP$Kd~0^hFW)S~%g5H5SwFbF1+A@?5^1 zN<$%S;!0A1kw)EPm8njDx}BS=A+W>S_nY43QG=uSTaPYge8b_-m>pdrVPBC(`rA!V z<*ybc#WWn>cQvEl!PvS)YCF_W0tc{=)r(9=eaSI?V5Lv`gG-oBMa03RCN6g-cAq(8 zzAcmeFynwjVn={gO5hNF`-q}W{DoI{NBTeR0aql&E;+s*X+{~r*?Kx2Jp_l@Qwj&E8GUPF61ZfNpf&u3sf_z}jf@Lr$U($ApjqGMmwoow@TT zA_|}6;$8=ARL(d#+V?Pz285_+)tmYItn31vq7wvbz2w{B1Li7OW+5dHUKk)r->VR4 zXYxlOCt_5;$efF@L;U^rbt=U}s`S+A##up*)u%Tv@>v@D9GL+;cKuxSm$N9m_#?vM z%7U?Yf{PFDm^_AYcP!oqaFGr~J(x8>0`nZbcx(?IY0ve2pf?gC=J> zv8Alf(JIT;W6G}G+5x>C>MY;KWF@Eb{#@cN>hi{JifrTr&r{xrMgVj}8+SDmWY?p$ zg%tZhjS^fCPvzJPljhiq*Vjmnj4RE(DOKzFW1SZgky?I%XJWz_fsxKnVqH<$dIb8- zmdyK#^x%Stc!Fd%_`YHx5JeH}LuzbQfMTzwO`IoB2s!O)MoaHEgzfjfNaXPLkd3|L zCLB$zEtj7G&AlN*{+u#zXFV_at^b3icKr+6dqV%hjQ$^NL=^J8nGWdEh*5N{9G_BG z+qvwHhOcB?xfB`w%6n?o)Ay9nh2^A?{uvY0ljRLbF=NN~GtDRfTrP#UWkPWMzT){9 zkZ$UxDCVX}_>EQW@W-BPnl)(#;5e(K4FA@^+z%BNf^uoo*jrwwO3DWL!bhiRF~O@Y zk85IIhbF+RzNA?y(tqa<#1m3kV9Q<2(DsN}7PLrW29$sd*&qaM814GnrVvg66-kKk zL$d~i5~3_EbEBX{rn1z!OSHjfSL}guI zs(`I^vg$BAuTJXH4c}$9O26ENC=aKK9zro2aI`!7O`>y05?RL^Sqo`WmdZc(quOj# z64^SkbQ-~7{Oq%FQ7CdqC>ZtVT+rfz96TZcB~Ll4x0#S9_lw6P&Zva^EKBn8qcVF{fAML-lzBdG zgg}p-G-u-PMWB`D@k^H}iCGDd%E1`kaGy!FOAcAJ3tfa8l k12_->u;v2*r2f0OyT_75Xe4C=peya2@^L{4_}25c)Zo_^5z zFcFOI8;p6*Ap)(Z4EGr92I$0FpM`HPdX$U>4a@~OXw`zsXFFRxl9AEHuBXbsm1-Oa26RnnbX zXU7(QXpDs2+JFwS#a$(;Hr2^+cGXD0<3pr>WaPB4Z}FEnTo<{I%^C;ii-}8?gm|@- zjpHPky10{f$$6#O$HhIzP7O;;yISk>$zKJWE{JH~Zj0($xpQg94+xnIu>nc?^QTEZ z4G@7>8&7!mzb?HuboJl6?Ma7*3S=V$s9)h9Mn++aQkBN$6z=m`yf^UmZuy(0W=6Ci zb^rhXrp{{sr_o+U-}9muv$R z>bjb~9<{Y2$vs1}tGJ+%hCxUb2bT=yhlPSeZ#=moUQan^Fi#Ge7Jr4ugaqN}H?eq4UG;&^=iUt{`C7-!6l&sJ3{f^<`ovO(<3;`;ZftIK0jdx6A&bfJ>~QV3F^qS6Q)&&AHUgFS5dH9(!kT zibv4_`*cehH1EKp@^`(BEdaez|LMGMi&*HdDCo#_$L zki17oSdflzi3r3muI-<*#Ufw7J=yDy-r$U<2pSs3PpEOu#%TmG#OKoFuaa14hF%(MkNJtu||-y9rhQPVfX1^u8OHY z4Il;&iDuXGbuzUA7JolLmXp#TFsBD91k#5IZ&?qEZKWi23`tDzehTEKMqo5V1BZkq z=?I(aU@s}(U+2n0b}T%}iQdS#q@k3~=C)WzyW=&SuI>-3sgjoX|p zXddugQo8oQ(M~rM&Z@ds&$>;1v|wNP=Ahfw3X$M9eG$!`+JfM@Y38pb_f}*aDVV7T zF_3HP`4y3maWQ@cg7lF<6@%?e86#teXJVE^F|3n9E?bq~Ju>WO`IJ9N%!j@`xGZ@a zwHt%VaY{Ak`~11I>50k1YY#o(92xV!{MBA^u1RHU8D#}xNXDULmJVe(W0Oo{P{5)!-7v1d zf>D;%GAn2YBvV#gJuuKHK(w0+uY6ER_@!pCIrzdYj-pg2(yNm~Hk2`safdLa8pvrN z?xY$!>L3gN?wl~R7qfb*xcyAQd99*z5wy)Wqzw<||Ozj4#)DorfnH<^HCbz3W%$Z@%CpKS$8&?fv%)cMvjVA;% zb!h*X*6l|P+-A#!0aI;L^Ij`q_BD#`c@j*(nKI?t9lk@b62GB)$IwVy-IT(`fyJTh zX{)|3Gu0pAvfT-!5gQbR$_gohotH-{H~=v)$f#X8P7-{X}ThQG}SN7(?G1fLHk{#p)lUMS@uR@sA!@8ru?&WTz94D z@V%>nO3t^>7jBUc8O_u!p~H(y3zWD- zy_(ugY1xsAx)XCcn$fSI>=Z~A;wg9)TTKNTl7JG)EoxFgGut0)=3`+2XQaKx7%dnP z0+6_}o37v6Kiut9&v_%z=kf1aqgnHAPLV%m^|nBI1ataNKEHx?Ds6e=53Tzz?VkE6 z&3)T^emjcl7Hf0n?W%Uih{f9#b(DDV$MqPS<1o{9*_|VYQm^sHmQ#Io0c$&GLO{u= z{5ND73pKa@YRkQak0UQ;>iPG|;f);KcU%}@PjngWc*@mB#xRIy^F#31AT%-iQYM(Q zy2-U_*bQ}@*4l5FBvq68;ZrcQg6uTjuWh-IhwWn1p9xRwA_PN8nsP=T!-@1GFdmJX zp0#%nz*F5|C{r->^sc~9QF}rzFY&iZT;OGr!IUF4cXi9=;kem4NGMQT732RXUWoXsxba6Z9?BH| zS8-p+Me(i-<_*ihQw~b_v4Bi&$=4+Jun`@nx0cw zrtW{Rx7Gl2eoi`6#wO)w(r@+0Q3r1h&q$P1u@lC~`Qf4-ES8^?%t$`(aZc1=3#X{T z_CcJe!HV^U{J=o9$ih`DWKf>e)**W1P=i`Mq=BLW$bh=sxtH$VP@7)smEF*D{qCu3 zB{_lmb@&-WX;giM`ZAZt+<53&8_@*hRuBL3wh)iAa(E&=R_g1v3y&cFAw!~+|KUrM z$R6PrkLh2q)Sf;ZwQ$`0_H!2vSim6wH{82SFfG8Gz=U(lhV@mVkzT!Tn(y6HJM`tz zc{dLu8oTCm6}`C2(dg-@#Rcu8Y+ulna@OOw-)?!rn)^;(x4SAY&t`6!K2mvF z1U4vT>*L!-1nF&s3-C=wH-{cF-s+ruc>D#mvw?IINO_0O4s+hgQET+4U&May#)v(55+QjZ*2Vt>p#sn9zAx~bt0w9^ z5eT{}<*Q%A&rw7JbgOyKWqAg4`y8m7z$+|?nJ)5Tt)unoEs4^ywDUhdhG2UgkvgFq zO6hy_9hPeOxGeK`aIm@jT$#cF-BRJGe4lJIWaKlrq+b5eMtY3fFbrk)auQuU3RHc1 z^&gJLnGk0gut=`3+Z0s#%TdJ2N(Y>gXI;w<0+} z7=#U~fuSf^n8YT&A6t8^k{=N5hSrQ?7Fe=z=<% zu;Y`U1yssGz|rflaH#e5!Zp)QpnUp#nhL7Dp{V1DuBt}N z2J8wztxHWjci~t6ov5eL?0e$k#D#g)p%bAAtR*mr tDgzM&El~vTY0BY10Kkj}0ATxjnz}|tF%V)z)q(be4p9+$Civ;r{{V$lfs+6L diff --git a/summary_results.json b/summary_results.json index 6c544bd17..7946205f7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0789, + "Performance_Result": 10.2468, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419316, + "Performance_Result": 0.42151, "Performance_Units": "Samples/s" }, { From b4eb5737cbe0bcd92bb141465aea64b56bd9f6cc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 22:43:17 +0000 Subject: [PATCH 362/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 10219 -> 10219 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index a62d4a298..0e32c13a0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 22:38:27,179 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 22:38:27,179 submission_checker1.py:3260 INFO] --- -[2025-03-28 22:38:27,179 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 22:43:11,667 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 22:43:11,667 submission_checker1.py:3260 INFO] --- +[2025-03-28 22:43:11,667 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 22:38:27,179 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 22:43:11,667 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 22:38:27,179 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 22:43:11,667 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 22:38:27,179 submission_checker1.py:3276 INFO] --- -[2025-03-28 22:38:27,179 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 22:38:27,179 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 22:38:27,179 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 22:38:27,179 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 22:38:27,179 submission_checker1.py:3297 INFO] --- -[2025-03-28 22:38:27,179 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 22:38:28,184 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 22:43:11,667 submission_checker1.py:3276 INFO] --- +[2025-03-28 22:43:11,667 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 22:43:11,667 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 22:43:11,667 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 22:43:11,667 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 22:43:11,667 submission_checker1.py:3297 INFO] --- +[2025-03-28 22:43:11,668 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 22:43:12,916 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,9 +31,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.6542 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2468 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 24.0593 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1242 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6dbdf1f4e..9ab4b905e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -989 +990 diff --git a/summary.csv b/summary.csv index 47131b6e1..fba6cd913 100644 --- a/summary.csv +++ b/summary.csv @@ -9,9 +9,9 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.6542,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2468,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,24.0593,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1242,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 067dd19c6..b63db6b78 100644 --- a/summary.json +++ b/summary.json @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":18.6542, + "Result":24.0593, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2468, - "Accuracy":"acc: 75.400", + "Result":10.1242, + "Accuracy":"acc: 76.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 568ff3d385f166575ca3466b59edcac57a60c8eb..97a477e067d173bbf54dd63710694e9183e05dc4 100644 GIT binary patch delta 3430 zcmYM1c{mho_s0jrjNOc#%!o0iM$}^oC2PjMOLiI|LiVLGGsu!KjAh6=F^Od7iAYGs zPIjTPL?Zi=rJwq}@AcmQ-1lFf>wLfGoa>x>#c0K-qL+c?+0Qd@UmySw3MGYu1xf1+ zAl#JKS0x_o4F`x8)~$$OizkXX$`0Jtam~ck_MAk;?V^`_y+@L~S))zbi5h%nENg zelOoJ+l`cFhD6H}w+OF!C`-5bI5QBe!8j5hBE!hJt4g)grmiCR9h%Fc5LIhCwfho< z_N}fPxC5_Lo4d+PyUgDGWS%15iWUM3syHAS6=IIhC-mHC8}y(xntgSw|MGs>LAj)Dk}9UvGZwD74s2ylXk>iUGk|)Gmlrs`-BfG zmTT_!uNIGOWu@?bC~2~Uz(8UGyMkDGs|5l-A=JdMzt$y9T`sgd+QM`6o?{hE1_LW0 z8?{-^hMGp-iUpxFz1XaVrk?8I_=A@3!G7H5mr;~RkmTrRg}1{mWtEV4`@FNa{(o6+Gv+c1TScVfx}@@s;X5V zz`&jvZXi8!Op>0idWDS;JVo>a($AbQ($f{a)fA1HB3=~v8u2|50xrg_$|a7w*1~?D$FeGLf%%5=_89OoD(r_kgDlBM6vm;OYUc;>r>C zmr1x3I3PXNKe}?jdq!Y|V=~`zjOZ8X(_*W}<=8QO%%Kd+{T&o^MbP1s?dq-j^cmPz zF-W)X44eC1mlsF9I1Hy~IZNx;+bJKSXok@}z%R}n-682dBRuO9X3yPiA|R%<@bvFj zUaoAoy?2f3p3b(3UqFE^YzB#Tn)gc}d@j#|c~c*CPmrdzvjXqb9`>teeW0}-Ip5R- z>U8x+HuH)EY8pO^d* zq`5aR4b54t!Twy&DeL=S@olAqfxR`E!=3ObFiJK5I~Umelv<)8>gcMn6;#Yo5a}s> z>Ke9K33pXyRHVk~IZ#nS34}tNV#oOo@=6M}RE=@YPraRolSGYqX zpf?}A!;Nb5*n4j;O?Fuc7oEX%001@s z7ytm=04&*j51OwI0rxZFh&^{9nFu=KZDM|t;=U@?b3^A`*Kmry;Anvp9tru9J#nO6 z@FKS+tBlfN>UYFB6LgC25r^oHfXHeRn&^oOtJBDm$y$U#7+CIco#BwV!G zLKCS2!0;j5yg5{ek~R;An9rI;2cQM|G7)lpnPtBjWJ-%GhqG}LlbQe6=qeh*G8N@{ z9yQZuO3Q*mqx8v5TvPP z-{u~q?!w3pcm6}Fe6C;Z>YPcy^9ggsX7k!R^OxadP2pInoNJnMx)(VIMM)Ap*Yp|m zp$X!dsLibkU<)SsnYK<_uVl@BA@}&(sA7&dF*Fnsj;nn+{-!!|-k~z?whp1F_|xw8 z(p|fS*^LK&#hQAtQg+u*fi|bWG>IqI^%=~e?8dj}SDl#gU1C8=DT`0d+)nd_mLLbC zd0yGaC4g>~^BxADgr3iP(fCb$bu!GltI|5eJXFz~;@6(e1V6Q+?bM1cC=5=eyAa1z zj6+o&Q~*_7k_WX3A3@fqX{zMGPsKzP6yYbZEvCkmv!KeOgWL$$?+35Vr>OA8S7|N_ zpH>Eu-%Ou*oSOTGI1=NA*P|xeUP+iwaYtk{bbfDxqA$fmYvio3`R~u=eD0Ho3UDK5 zOJ{=lDjS^3`|miOq&9WyK1Nj&aU)ZK@Uj)J?p_VEh5WbLVW(;JV%?m!!V0y1fq7i3Ix`Y!t?lW)e^%(0Pa&s*U>}jP@5MN^-;Z{Doou2g8>EC`u@n7S!C{Anzd-#;}Fk;WamPkVF=ZX@>>& z!Kih<`I@t_QcWtFb2{ax<&rA#Mn#{&IAx>rM%1X5J$#f|_4fSE;O89zU!FguC%q8b zqaw>zfx&sft!S1%c0<^@F+|9VTz~Dllu2DF$NDmrWq#NGkq}lZdUs1kZ8&iP%O&#SA`+IQ|x&X({s*0h3#UUg7*0}AS0Thh{bn)o8P9L8w!Ia z_m6ygc3-W~tLtr}r|0zpA0f@Q&j_-<^D}-b?zso=Yj?Zr9GkYQ4&}Mo`=UmNtggoo zc3;M3J{6OD< zCA7I^KXlm1z5d4c@-obz?WB7lpRal&FaGmF{0wW@Z>|3Do%jL=!I+!YznWyrye9N^ zBh=YfmKH)dOb2wB<*ct3;Vq=cDK!Ke7BaD;7$iM(KR>vs%y#j7 zh&Ejr>xmHvWv`K}VlAL;V(~U>((<07!Lbd?C(F;K)^h)rwp?etBoY#HN79h})slzR z;8?}*n2>D&|HW3o_1cLv^I1Jlqsos5$QL&>ut>V;&={ zbbr--Nt}>gcEs?u`HRW-10#K7zRH{z^idkg!|>h1=n`X}LfOuVla=*8^hlAH-mPQH zs@XgWSH2^olpspMYHxlFJ$T^+K9&12E0L^pVjJQu_RwLmi3AC{ww`yPlC)k7<;2M~ zW&Ef&O5o$3Ej0Rd&uM&&1`?OVWnk|vV9BctxCci^y10b3{i1N5j=7HQ{r7+F$4BkH zKON^x45E}bw>pzOV~^^&<+_zRijJe>l&D3CxxXd{cCwn{sa@@mIX3m>y~wxJxn;oy zyx#*7ezAV9^9okyv%N2EGOrOd8IZ&fW@3gdmwH!~@hv(jD_VL&d>9kc`n8*HBmhTs zt>(rYG_307HA>IH2IE#&4N?-%Yp#AbAk34ry;L`6}65x~Z@g z%H-cZzZ}v$rEnNNyLF~=@1at-Jon|2(DuOVPyeM(@6E@}hk#`l&aT^LkFSu0^P(P0 zHA~siNcExp%?gp1RgO6SGh?fp1g*h9DUCakBw;ZW&P$K0j^RG-1wq}++LwC>38@Vy z4l9Kmb|uash-OKpna6>FCUUc-@&}CHiU#E_RGpE?e+@D9_SmkpF8n8Tzoo{pY|!`_ zL$Iv(+66C31&*2Q%FhwNJWb+m>Xgu`dIis&{-AtvKSe|P!JTa9v*}*(QggGOtmOFv zaZ4^T&XMp1MxKm|y;=B~d+6lfKma(gfRRxt*cQX662@?K!CRiK>ya7Fr^4rBDD>Rk zOgC*jz-}}xQf^UteQa|FYbBS~aDx{#Y#w2@hG(41VsYTx@R8bb5;S^~ESk$<@^(Z1 zlJ5_Ov3 CiEeBF delta 3430 zcmYk9c{mj8*T)BgNnz|EhB1~NldU19vZw4N`#Ll-G$`4!jiGE2W+F?dd5|UR*kvh& z8HACgCP`(+7G-H{@v8TIuix+f-g3mprmh5{O`1=xh1vkjug2D%`U1gqOo zdE(#AOQKD?qh?xXhjqiZHs#IJ6m5d2YnclT{$nzC`k{F>CXAW&vFtzzSKuAtVz{E* zde+VLrLCIT)5=}zcfp>QICLPn$>2BL_2D@g@Xt7Z`wUmzWT#wC!%x}D_r>Iokn$du zWZ$3{W^p3UNFI$}v&PnHm&)o~_sq(4MVOcoxC|49g`98RK9^b+Rl25bCYU}L_!hM` zpFa`q*uI}g@Y!Ga0DifKFNO)j2%KuXS?>vTtBbSuPSlHx(^{T|*oKt|WNMe$s9zgq z-mS(7%BF1y%cD=$;L=j5w6@YrPE|Rw-1(79Flue)h!XV9cdh3wsQ1^_`<)~)We=Je z$fosPUF=ym`t8WY%@?KUp*o40kh^_&P;U?6yrlPcycj;>%G>Tn?<_;jh?)dD>_h~r zNn8yrU6Et`_3cQ6O~Pan_odt}$=c%PGmmg8E41-*Q*`O;NYYWS5rGc>*RB`C1Cy$h z&W1mfN?VOp3NeNlZ@avX67y1~sz`e~d~KI8lbVfJ!rjkVN-{b}m6n)mE_65lK}`pP z%2+^rEIM5Pl+&RI0$NmG1+rhC6lee0B+CQ=U;I7_WH-Yl>+?0sIv`(jju$||mvPfK zfb12K@%s89fdZhi6)ASM@s5i!kfg~T#mCPi0gZYzS*gp7N7&i6rRBgGtA8GaP_SD5 zTP-kJ7I|?VCE>RG#3P{4e_~x(TWwq_*r2LfmdQ06+Zrd+o!Y)D5_(SefVG0w>S$^G zP(uf*?A{d-F>S;q!pZfv``2;x+!+p>C%1Tud9`uhacJY77)4ZiKl<;w7BYJnbqFEl-rmND{se)AF zO%J}rUA+bZrtsnQM`giENFe)#$r)MXUA_y=@iFfcN%0(4r|}BN{?<_)nI)Ik|0u1x z9pIGOhWi)zjaefzf(74$qgQ{#SH11CV=h&2@V?6ATZp|E8mIqc5e%}GV#ik)$)J36 z*Tc&Ap2lNULq}Aj%aFLgo_Rh4>YPDx!(q)yc2lC;MU`!SPIDwb3cs)Smp3e3Q535S z^y4&bmxJx??=be@IPru@fEzf2AUyRHdy`31|lZonU;8=7CLFr zO42!FBRF9Ey~(wb%82^6Q}lUR4Vv7EHmt?oev}~tEomcqG^=0py8W-&S|}E=f8??` z2T&LQ02~D!20g$Lt$p4(as+6Y6L<3@9S$YJ02)UH8_sY;`X8RsT`caDBr%m0n<&+$V7|ID%|s%%^!p0bIWd>Gga`*upqH%By;t{ z&hF`7uS~Ka0Q%vVMn0X0lDXQnRV-v!dTc2}?eS?-zsILt4!BYFSFV-oka3NteVQHy zHHE@MDT8%Sl{Q2i3t>W<)h8*&yaZE(zQQOVx6DAJ3aVksE<=_vh`jv+Z}I*M(F!F7 zBq8BE-RNE!z*xitD&>48#HT)G%}m|*h=S$tkV_v!J4857AL~0VE8!3klEjq@pPlfO zKoPb)iw!J9#6ukwX?0g|51&+ZlR5s1wmc%g6^c22+0>Zk8&+Deu~KZS(pgHJ^-3$z z*Jv$JVGdwPw(f3vY*Zh=uR0ENU1__pJj<-F=5}XIq}TDL4frV<#RyT|umJ$`e5g{V%%L&V3&hWD+J_}}J9iy2CBFo9W|!$8W=*|V4SMe=j9gdJ0; z{?_RM6L$MjkTZ#94R@z#<9~pgU|t)iS5rL*efd#9)`d^f*?=;55KGCoQ|F#&QE;4X zTymoE>|)kb_{~k{x^!ku;#zK7q{_SMj%Ot&Dzh$@JTj6;Uc~zTIJ5#8mBdvJpZ)16 z@q~bh#ZjU8huPVRibo4;M?ZO^SO7L>Yl*&e+?gk5nj6k3vKtK^C|W%kURqdW-srzm zx8&?(|F%QgKS)Kh%>?Vac}R4QPU31oOHky;E~(_21eM=^t_#P1t5-5`i67^7j-nk| zaW!;IOrUHNl2HPc%D;BbaWSmlw?8lL*EouE>hNccwyiXMJcqF3%fpivEN@e0ltu?vSJhn>R=N8z(9PYHb@1~3HeI+TnN-Hz7c#z97aUHl6` zmWu3iqn=pH>sxpyvRBV=di{3s^nh6kT=Pc2*_dE*K!i(StQ#U<2SW;Zot*$7-71+) z#DIqc6CkB*h-9u2xW-;BF!p-p?8W%mzoHF`x2*h3WqnY>G#wqK5<+)QEw&)y*)~)eamy4CFVnylf(( z8+@3{o_@;QW*cc!v?U`%;l>V_*U;(JQV#+kFP^?kh&EG*zTx zP5*hhY}5lV$1#54v#@Uhiu!6TxWqFTO~cU2wu`p_K60e&{p)C14UfXa{q1NB^*}uawFs1E2BpazJy(xRau+tcs$b z9f|_muF<+5M2#0q_u2<~Cy4yd3=qex;wUh!2v1x@n|)^f#Q0a4;qm>=MM^UNT0_^( z35d^d#JrXL@Ewa@@>79>)`+;`g7XHnLBb{%^S&txHp6`Bd8#GZ+7PC9h zszy__5BM0H;5?7$PH!X5WQ zZ@LWvm#xiNzEEz{c>if-LCsvKv~7-K0<>IjnZ;}O^z$X(4D#>E*@Dap^XcsBoho#f z-UV6xmpslIjsdkRl$uPE-JV|4c_&RN^^m>RwNJ)dor95zax3AV_3XS9XRpXh@B2ll zz7H&tIHk~OTC_3#so>iZ=-X;O|Gc{ zjLl>7NtChmsOG?5zdM;aUaQ-|dsqc8+s&l!5n8t_+gB+fl zTkKE&1+&5zZ5hnPXB39W}6Zh7M4; zvmYx0RKPi+!uc(}8yjCN{wze>ZwUZ`9z5mXM25SfIdvmBwC4h~QmOZ@Qq&wmYTZ-Z`bD-(VM5tV z_*s>pd5+B2fDJIKI4XNs^*g&JOp)d5UW%R&G(0{aE}vfcyT5Y(^ku2Uz5H Date: Fri, 28 Mar 2025 22:47:15 +0000 Subject: [PATCH 363/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 10219 -> 10224 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 0e32c13a0..b4a21c045 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 22:43:11,667 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 22:43:11,667 submission_checker1.py:3260 INFO] --- -[2025-03-28 22:43:11,667 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 22:47:10,826 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 22:47:10,826 submission_checker1.py:3260 INFO] --- +[2025-03-28 22:47:10,826 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 22:43:11,667 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 22:47:10,826 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 22:43:11,667 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 22:47:10,826 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 22:43:11,667 submission_checker1.py:3276 INFO] --- -[2025-03-28 22:43:11,667 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 22:43:11,667 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 22:43:11,667 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 22:43:11,667 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 22:43:11,667 submission_checker1.py:3297 INFO] --- -[2025-03-28 22:43:11,668 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 22:43:12,916 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 22:47:10,826 submission_checker1.py:3276 INFO] --- +[2025-03-28 22:47:10,826 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 22:47:10,826 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 22:47:10,827 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 22:47:10,827 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 22:47:10,827 submission_checker1.py:3297 INFO] --- +[2025-03-28 22:47:10,827 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 22:47:11,850 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,9 +31,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 24.0593 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1242 | acc: 76.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2195 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0958 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43633 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1561 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2528 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 9ab4b905e..c1724ded9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -990 +991 diff --git a/summary.csv b/summary.csv index fba6cd913..e50066faa 100644 --- a/summary.csv +++ b/summary.csv @@ -9,9 +9,9 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,24.0593,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1242,acc: 76.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2195,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0958,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43633,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1561,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2528,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b63db6b78..f38540e34 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":24.0593, + "Result":16.3662, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1242, - "Accuracy":"acc: 76.200", + "Result":10.2195, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1561, + "Result":23.2528, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 97a477e067d173bbf54dd63710694e9183e05dc4..b6ac0f9c880f9c28b4483f33f05561da9b9265a8 100644 GIT binary patch delta 4908 zcmZXYXHXN&x5g7d2`DX~QluvVUs@DUK&gfj0t$#q2YEx4B1+LvLK6r*2_hW{NEM`o zrWEO+i9tX>KzfOEkS-U!_doZ}-2Jd;XLk0tXP!OJhqJW1>vv0g>6uA^YNj3x000I| z#DaK;>-3CRbN6fou7FAFAOk@&C$KaU6hny4U8-M_>UAUESfZ#a;5RW|9? zL9V>OP{|3BB8&BLjj-8Qy9Q5ycQ-BOpP9H)XM0h*5%?EWkf4Pm2>bq36DgP*%4givC`?_`M6$D2iG@QN)sux zZ+(1+r7NDUpH~Wow^H!(9nK$zI0P6)xY>C8ubZp{y$-?{_BE01UTMf<-iO_H8N%39 z@xajng-Lg3`uZ_OVZ#xOI?0WEr8(6%USp+Kse`w^jS1K*k=7W$f|5S)LHY$l0ar;)60PkFNh81ePD~=tM zzsnF5h>qg`CDn_881zc+G0@o$^U!Tn2`_*cJ|3TApwq?T)L5&9A1bw;8B7;pH^7cR zW}qwd4N+6`a_0c$ukzCY2kW(k1S5u9#M1Kk0Ay9FFuwtri4M3gAj%%U_GghH!VXXE zyUvalR?=Ry;kEqv#@a2zb!baM0Y1p@sa{+nyl{3Zur^qzDf+|V1@BubC%dcY>-9CY zqwmKUqz^;y|3h#dVRdqwOFrtw+OY|}*KMBGG;0l|+zFFpn6mv;WS8-DkZZl|-g{T` zU=E{d!Nlcj?`TWtUWd@`@ig-yG)HJ*u_HyAbajxqyYPMkb5}EZW45ZkYTRt?cU1gE zkGgA(NgN#NDIQxkj%}Uu-X}bYlz`aI{^drt=@fYn#on;*tK0sE=^d>0W|Nfqln5be=z}XqTJyKVx>K(mAU{I0<&&K?RZ~Eq;8ieSG$PHWPkPC&5K= z`y#r7T^G^4lpvj`SD&zQ+FTPhn~S4eCfuJ3qq(Qa37OloZ?Oq$I&;pvUr^uEw78+D zE2y@|GnVPG#-{K-r%G$a`xlntB7d&E^qsJg*=B9vUJbf>4L#47-&69-Nbr)-qnL2J zQjuYx`P(x$u7SG|D^(UnCeIoStnTP>h2W6#p|4Nv#xJs-6@ngdos)r`V%@?0lf0a<$ z+i_F6>J}m{5MI%eQkW_)nzYmW`}y@-?<_4F#CLkkGTs7gKYCId{B9pfAJ}gJ#zna3lQYX5rho#awr6*Y8Q~&s~zs z9p89Y)m19^ih|`}Jv(Ey4*MyU`$^IiiVH^GK!yv(wQtpUd-eRsIyt@TqXuT6vtFk!Uq^ z0DauoK4fwuZSn@U6aUD_2jy~B-fGin^0Y>nMOT=_$dmfPew~gAo`m@jD*Ws-pYsd( zj-$3oh-vIkU1?nF_$vzzT(v|k42%^WxiUOGXBMx9duk}${$eA4deg9NFguba(st*q zzLgW_G&!H^LPP$xVcT4Gq=a#&3|YM6&fDw7+uz#Pj4fK;;}CvpVhE8t-}pp0ni`;l z`I1?FJSby=6Zd0@9v>mGD2k7`Hz@+%I8G-aqOcWXFtCTWDhE#C1HTA|!>%koqQX}T zGW|0^7#wc>gthUh>UJ_0kD)p}{+_i=sbk;ai4hJCoR zBY1u2xa^Q=+4y=TWe>raiPmHMcQ$&92inCIk*0j!@5%PmZbkLNN?iKD>+Uv#w34u} z87#VT&us1GK)Wa5x3d02rKP+V>@05W73%D7Y2+U;rs-Ljm0x=7jLQ<|d8$yi^s9UmMyR2?=d&W(o$>E9sRXw~p|VILtZw}C zIXArH3$Mv7*E^&R5D+4G2p9+XA|ZqiMS=fZ%nHGrf!Hd< zFCxLn=%l-<*@$F~XYmF%*RCZX_-0`zYstgfB(~+diV44Vy02B!*X-F?Sr**pw!UX6J zW{`s3xOgY)t-g|BuHPo(BYs|3o*pb%@mW#uLg(C2XmU_NZykm%`TkcE`*{mCy<$O? zVdY^;ZJe>++LFnXckBvQ=bGV69YOD_P%?rZ4Q7eQ@j~uh!Xv?elE^nC@74dYVIVq&rbS;=6yPB@#B0wGBzm zSHFfk+X>+n;*{6Py^nw)0qBbmM^{#r_$%h4Ih=OR$lUWZY(HF$81ba9^0%R5p;)nf zQGVIiG$+^3$#=$|UQWLan;cSoP-E`qdzxWQ1LTwh`vUlrDSItLG1H*p{6f0_F9j?E6TJ0_AP3bAi65qb??GGyB3I)M}B_QS0{xI`Db*OrC zBtwAlrtxpcihF9gE~igsbBTeZB+1-_ZWjU!1Ps|kD&qwm$|E;*uR5KB=}&f>cg!nz zaiyqTb{Z7sy^HaRIGZdvHS~zrhOqZ2M_um%kGF{!wd`KPo3~9B^ir-=+%+jC=$^MD zw*eACb@ItA#BQ(V7)v+g5~plq-l|=zO_yHib|D!DSgHcM9GqtqAC9RA-1m_}&FvWt z#^O8tY{0HUv7VdkF=EBH5FvVr@GUqC9C==zig*)wbn6|bO?l^+?3}52F0)aWgr_H9 zk`g3R2#I+9*tG1010$u;ma~FDhcc{PS^Dz8a%{94n?>!;0!s za25k(kGz^4zB=5C&iG=jZ-U%png%L02At2p$%bnllqlQyu}roV{A|rOC+oFqk#FgH z_g<~Hto!luf&s3NLz~wxqMgVpNK*d?_1rJ%Y(9kdp^+(i>k(66mq>OTT^?IyB~q-_ z2!M4H1cxQk+3x~T>TLmnt(yzJ%ozm>3r4P~0cmv)s?xXKX{V;4VoEr)&HPW14+Kd- z>13^^6gdJa7jLmFQpAQ5s+H93r`qfUomMJ?kF@Z++by$x6hvvn;00fQ0s5`HpOz1d z(8({~O;jB^TJEM;38ehPs4o|*KwZ6!mCL9+WM?6nHm|nG1)df{il(<CY++4!D|yskdb?^|%(B-k)_s8+P3}|YEEbL3K0MX% zSOpQ{n7DojXL*B^3$({ihC=~zQAaALUUCec4EGH7ea$|U^{5FZd(`yK6k9g6pgrP8 zUA2R1CZ@<&^Q(>)rZ%K+?P;4UI+DCv@nM3bjhv})NA_NDm|!kY4tKU+^4d`;!zp=6 zJ7kK{?9@uZFzx^y?!J_sHTnwswt&pW@E!-}z1se=nP@e1h4TZ)JQjYOR2!#Z~wydx{n9-h$1H^hcan)T>h| z6GBe*6imog8w5^R+xk&@@q6!pySPM^{r1|~A|GwuEyv4RGqZ<W5%;ZrH9WIuE_|;B)o~T}2RQ z5|CkCorhY%e<6nsJ6dO_n+rBMYc!P5G!yO3{lH)|md@njHd#B2(s(Vt440DLKPq@HU!x)OV6lz4-X!ai#J25A zD^fp-3xm^;FF?*f z1&otS!bA#Gf9|r}&-$U`6kl;XJe7_wM2#nw92BD)RW*gxJ~5kJGjS_jrM$N zKJ4i_gpV;!WB2x7{pKCtKpl%mcf>4hME!{u_HlC)pW3riKt8hzc+nB|eRnbEM6YPn zAq%IM9*SRihnz)|_r0Y53Mbmm6G~lU#Oikp$a(@7reVNd_cgrT0imgWmOxhZVXLGG zF#aOt9$1+N+sS-p5tL`KF4xYdu)b`2EmcU)iAjd#&PQ-L6dH92-l8@Q-_3 z_Hq9%TDOvHEATPNLPkbxn9d_7ZuZNZXO%^?+x*V?#R$s4mS45|uj7UVMccJ~&m$L@ zt?^F83g)_p{)aURC&dlK^-`QIXkoh5s^j|}#MouSbxAklG~3#vj7s%1$_jWanBBFxwz9Dv3&YgrmV9}_+g;_-C3f5?d>@B(e05Nc9`GbA3&_-UGY!h4;;U@3WY2eYuKCK#wevk@cc*7n6t41&Rq2nm z*|6-(9@V~h!S^8M7H0fzS(*?y^e<9OfgK;01Rq5wEIva6fO-Y9boW|G+OVLoe zGk-s;i1Sc&zQ5feGXTJ23;=MS_KW`+fkE~}4VWe{j2HnE`D;013G5<}o!ACb2Yw(j uiYWo5h#F!rzCZFqfdGI39RR@bpVOx?Fo=~HBBlbgB{qwRp5=o6+4?WK=TIX6 delta 4887 zcmYkAXEYpKx5t&i=w_5CQNkFKhY-<)D1%}27QNRHooHjUU?P|zf(W8TBDxS{M33IP zAX<`%-ieY|p7*YM@A-1hm-Acu|KEF`wf5Q3U)Qf3Ag3wVzRv3pAR!3@<0F9__#JX8 z)SUOSJPY!^9Y`JNUKDK|h2giCEIJ65@8$GM^d# z5bqlw#fnjb;w3Tr(G4v5KP}i8UqEO>P^Wrs+ZTAf;L($}LOZA{H&k{RSGM`c-m4td6Q8x>* zDfw}dYZDb%!Rp#_59ijt7B4MEEl7HAWtC2T;_CCc%GDUx)I$L`m%L*6g{SM&gIs5o z#M(zgn@9lg!0bM#6fAtcLJYeI?U2U@Aam4 zJyuSW&!^2l7jSH(tzvcIO06#2wP7ks*-=SD!E}N;Zt%EJCGS~jRtScyX#xS`;c?2! z)n9;s{sm?L8FcC<8EMTrU3ADCCICRTa7jT%TJ#ylmoSI$~oqZgG9{*;YwwB0y8C5sVG&(K`^EOF;;0+LOK+{ME#02)9B0EIG2CT#NlSLsSg4bXk#5{V-sl4$#W&t zdJP0*;Um{zxu1GdT~yPmQ&Dv_lIsRy0ty0@C+ibU4HlZV{ul)2 zV#lap$HVjBSDUrS?VVR;gI`R4tiL0tZ^JTpL_Z0RQ_lOv1T?v#zEk6~cT?JbT*B}= z)=TWlHDrlAN>+T5yVl_iKu$)UN(6F+on|}DEp7C-{hsgD&FJB>+aKb@E zDn}Wk)S^=61HUHo6sG5X*cE*Rs^b&D6@LZ36)_)(;3^y9g6jELqLTi^m!_WH(;n|- zD?IFphVMOTMNMk5*!mp&ob5Gd*Y8bZU;co}Jy=B*VYcCG4>?B=*0~z&6bNBNhv%lv ztCTbk@qYJOI||e9Jwv#S?I+>3K4J$9&Fovzcf_5)dzSQzb`_eEd4|Jr#2qMpey*f}sl1oR?$<2yy@m z2?-qukc5QfKFLq3UyoPup+ITi@PJEruV1-2z8=VpqN1|&+};vBXy7`mW{4!b)O$9^ z_w>usMrH9Y*8!j_q4ItXJ)`G?7u!c)vp_=g7M~TU-C7+s!b*h-G^kSR~tw@>Hch#RSWS?E*XAdsV}QvaDQOOJC_aggtUDilA@5UoFFXmLPQ(FRgs$E%&UZ`zKZGmLo1ZVNnUNvLvjf|MK~h zgEp;C=N>v=hz;XXPO}7s_fno+^?vs`T+qoQA5n{p)yy_`!;)mF-+V4(cpDUBT2eh# z)ob$7y`tQ*xU6hr|3ry;@=mg8MIWe{2%Rs86+U-`_3?^`t!FQDvX~ZWp>uwBU##fN5yq=Sj7$O}z8ezcMWxtiObe&ZcL5GKL zplo^sv9e9LfB5|#3r@k7A3+q>q(d8Y;d!uz5~h_;AtrvOCCk4IJA<%}Z${ZFYAJRt zO`g)|oBS?W-ZSiEy|tQPndbDyD9J3$Ysu|a`O*}OU+I{cA0)Ivj|JK&Mfm0{s*!K5_qYItjkdJjI6gh^<<2Rh>|nam?m(HT43`!QpikXQzzQLy zbSkudBAhrTsPoml!sSRh(C_3U4O!+vhZ$LH- zLdT`RA%QUd+r!6tq=69Y^Q}vBriCtF{kUB3gru5Prw~L$cJ6g82pVJ)c_sGA_x`t- z?xml_bA8Q&8%2w@W^a$;Z>H{VYq1|{$kaD_PYnT1V&Ezn`t)fo+0mO43Etj>=W&hKF6 zCLGfkK$i9tQ2Ht%bIf_!y_C6ud=K&^ievBerXdXTv37<-pC0kPGc(>=<&u-=GN>6)7F4hf6uo5mT^}b#pwQ94@K?JN<}A zHHp|JnlA;XxS$XchAXl-opHWQ7`Zc;9)sqD7+=F`fk`qcsApfsGCKFNJ*O|6_zV??H@FqyHa$rik)MG~O*phV-s%ZA{%v}zrQ>{kSmN1O{%hQrn zhCamfyMi(yq!#F-7^?!%yjxoH(K zN`1pPJ_yb}m@%q;suE?!zvHJ&{)n|aqtTTYKBQ%;)YNi$g=-Z{TZN-(oT1to!| z?1pgiF-wJ4Nc@^yTP8+qYr@gl9DbI0x(wqi@pCkG=_d~VUCJveNB8sJR0zPc6 z7zVwWF_CLF`QWsAhc^Spl_>h^9&APXAI1?rykP%59daFTvOuQD-hL&Z#Ti?msntGk zv-YHrdHQo)2?LrR4hBVjct8EACU(`X3T>ekT~zY*=-{WT&D!FwUjV8EhDa2(QM>|L zT>+m9K2y{oHv!WdSgdY3P)GIhKTgGce`)M~l`yzqMhKjx+!mco^`L_FEaWmwIQMPS z57o`taLe8*%TSXrIg|W=j&w@iD=XTstmp;v@D`KaLQ|HARG(G?lwGhRn$cramRDJ- zVu_kdhGXX%G|%ii3Im3 zL^0M{B6$_(0eR8b{i_`eznuiGl`==>U3I{>LBY5n_l#_@OdwlTqhrO8ll^5Hp-=m% zNDXFUE||A$-Meo<&3G*j-379mMJ9GV59ZBRK}jN}kr|Z35KiTq?yYt(NtSvrFMTlY zU%>#~$}^0(c~A9tC2oSX335+dUTug9h z8UROLX_iD&4vXxaRRW@1v5Rg2JEXchXBa)-ZE%~sB;B-gJa2~7Kho#t!Y}nvvxN7B z-9|Ov>sB;n){E=csHfFyGGn2ZnqD3!H#i?~MkZ2@h&Y@$a4O^7Ne1}a84)DzY#fo2 z?2&&(GfB4YsI~b{I!FFR9-W;lIx88s4?S6sJ{NHta9s$id#(lz`+uySe_4EZ_o)7V zSt#fKgGH5*b~m9#Q<~wygFMA`{&}#)L{WkgY(=Z$ zs$9|pKPl;u8{l@k@5fE5+wx9QD_g7{j(j_ezIrtB`_l`-{YsK_mCoM0<~?Q`N1=3m z&KS_!oIuSw+^n{!eSMh{ig?X!Xd;x>TpDaPr}vQ6-A_ z|E|=F>gIowO8JKIZ(+Mw;^4l&hsF7Da>%IeAFBsFLZjhe?9jxIf=4Qa-o2ljyu1`o zeK}$6-yj(}js@3>-*~b%c-Sa1>{)NxRpex!@adn?x?T7ky-z@GIUiU`1m$w05tZHJl7{7MCtg-y>4$kM^(yztMXSmE#&k^ivAIr#s9DhR%O6UK9K zttOs1gA`!5$XH*)&ss=^l5Y&rE2N|sA;0OV{moB9f$ksSP)*V@+Dm;Z5nDB^k|n#Q zq3P$WS+j?7x))Y7U(LQ5SxS8@ZBeAS%^jNHbW@MM{HLe+$W-O{6sPs;YyTWLKCyj2 zoWvSdr7C9lQ;>!Ld;LQA0IVez=jkrfQvAxp1H6H8F<{Y0iVamix{2aMWXFsjn7p0s z9G)1Q@>gKIr6Zz-9p^ndi+^X}TPWE*bGg1V2%jkOMm)GMt6t2_XUemu5ake&F+X0N zf{)y~5ngQ6>nF1@FBawT}2o1X!+uzl$_tQ!-rXfq~hKV&Nr9^QkwbByReR6#g`w8=Z{y>tD!*2wHrIu+0*M8T)A;iMVm!!dhqpOL(MX=ca+W< z|7W(TY2&a20z}oEV)1YW7%%f1yhe*u0 zeCSzk$_fNyoLZWB5gcqNwOA^BO7Wv;MCw-cb-}y_kdcq)L6v3U-L#XITKlpQg97pp zNuRA--Zy0!7P70p#Q<_)n4`2g&P~-y7N?=dc^O0bYMOpd*^W2Ty-`V`D~n#V8LOuP zW=t9O(MvoTv*<*P!f(u@mmh;kfGMvjDC9$|ohg*U$z|7kr0IH}829*A`i_Nxg^m~c zdZvSHCZESjeHYuA+B-y=OFeJA&q_6J5@Wm-MX{1aW5>4ZE4uH%q5lcXm&0KAc~|gXDnyxG*ApNKCMl;vrm*XY`X!kC!@-Am*VmdC^^ZdDkFW?Naw1X-Y%IeU*Kx z8b^URGKzoysqpK9Dir^n3-3T#N!{?8P<7I1d@_`u>u-yNUj1VUvyzZp4Uzx5qM~W> qy-*d>Iy{|_JgG2VQwaKB|NVs6NU8COLQ14I_%A{Nl#GIZ@B9a=tt|Wi diff --git a/summary_results.json b/summary_results.json index e670ce125..fe9b0eede 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.200", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1242, + "Performance_Result": 10.2195, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 24.0593, + "Performance_Result": 16.3662, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1561, + "Performance_Result": 23.2528, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 6f7710ed117a6cb1ee372c65276d99cfad85437e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 22:53:37 +0000 Subject: [PATCH 364/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10224 -> 10225 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index b4a21c045..889e65259 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 22:47:10,826 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 22:47:10,826 submission_checker1.py:3260 INFO] --- -[2025-03-28 22:47:10,826 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 22:53:32,161 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 22:53:32,161 submission_checker1.py:3260 INFO] --- +[2025-03-28 22:53:32,161 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 22:47:10,826 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 22:53:32,161 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 22:47:10,826 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 22:53:32,161 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 22:47:10,826 submission_checker1.py:3276 INFO] --- -[2025-03-28 22:47:10,826 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 22:47:10,826 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 22:47:10,827 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 22:47:10,827 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 22:47:10,827 submission_checker1.py:3297 INFO] --- -[2025-03-28 22:47:10,827 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 22:47:11,850 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 22:53:32,161 submission_checker1.py:3276 INFO] --- +[2025-03-28 22:53:32,161 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 22:53:32,161 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 22:53:32,161 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 22:53:32,161 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 22:53:32,161 submission_checker1.py:3297 INFO] --- +[2025-03-28 22:53:32,161 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 22:53:33,257 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2195 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1134 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c1724ded9..7f9d7e97a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -991 +992 diff --git a/summary.csv b/summary.csv index e50066faa..0b770bf46 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2195,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1134,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f38540e34..6419eb560 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2195, - "Accuracy":"acc: 75.400", + "Result":10.1134, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index b6ac0f9c880f9c28b4483f33f05561da9b9265a8..79865da7b2d739c80a657142a05fbfde2f764e88 100644 GIT binary patch delta 1604 zcmV-K2D|z2Pw`K%%>xQYFo(W!1ONbf5|e%i8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Ec`~Gq%&q-;bR@2ex;aYF;Xx7f777_AOraj{9F9+4U&g6O7g|P{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!` z50@khcz=@?B$rQO%CbPnTeh%x!AKBQq%5^u+xDz1NOH5b&|@F5LE|_H$uSMfjO3y= z!blnj89rYo1viKND3Q-pJ_aN4!2IU)MvuN}4f~bre-rY|H?9p4%pXWfLO~)~0Mk&J z%_;q%9$v^HTTv~Ids;`+py$qFXJw$KI^K%w*v=dKu<5jI&+Vw!^O}+s$Ev+n5grF3 z81@-`Hdy6>7lC?FPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!XgpNa^4JSW@V}T&3|MM zAg>C-;#rc%gvm~Z%mm?L#>;t@aMcv>*;A)!TY?c%wI7!1GOX+IXvJrQ%iWw?i{&f` zRa?=JT|{lF5f){mcomd|%h{9L^qO{KwfoOPn#ak){9;B#1WC%xgT56bo7??o5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iCTqHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR5dCoQ+9c z%g@^Qmd*%UkP%5!ZiG}8tY*HA*5?fqr#G#?#ec)q$9x_oYGn)3`s6?)`NH5o>mv%U zH+pn!Wh{&*Wl9Y3TF6g&@_$muccgMPjp!w=Pa}+iC6p!Nwf3$~N6Iu9W;EKbf-3Xm zXG!+=!ac9I7kUR@4EMko7&7xURAyIssg4Vfu!j92Sdi<0Es~rYDT!sGc2llZtUlkU zt_3ZsE7`2isSq?<-z`ZH5vK0s*KJIxSl_6Uyd4`2e*sWS0|XQR0JCifu?z&2%M|gm zA{C?p16sOcv&k1z3Iw}H60MUUA|HRvYQr!PgztR{qq{9zCM5{lfwtt-K%p)zy$NZP zh+02LY8}6Q6(?~L$SKFw%zit&s9Nk)m;8bA(&!o$S&kCW+z6>RHTr0m=^Q0KFfCYT zG}LGSj~2~C#VzH=!J9J{T#(?C2-2Q%Tcd3VmSXJr78LVYbm%CqjZ-Yd(rtgRW&DS2 z0Fyj_#0mlvEHJzWrgkEtBU11QX@6Ze0tCmm^$OXOl-JMFEGCjV2-+mCF?Ic?19eC=~zz6aWAK00000 z00000fPoQ{$|gYpyps$kIsq<|M<*r(yGIhOlW`{&0S}XiCq4mVlgB3`1{Ea$00005 COWfH2 delta 1621 zcmV-b2CDh-Pw-E$%>xRefIwMs1ONbf5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`CC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%iWw?t7H*` zs;y|rE}}8j3d^chz6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*%U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0t*!``xOyC4ag;F1o%hIxr27bVvuT@(2Wq4uL??ArL4!1Oi3Z z-yu4L4n?s!PsDDPeWe<-@o$mtJ6_t4)Go$Yje@9=}I z&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)g5$U zrvy1SItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50nYt*0(9)Q>m0&O z&wBSQofEbq6Ov`z2&pVs!+aZU&KqXVVBUO-|AMQJ`65cy$`)kJ$$?0VmBD{BM-<** z^61*iSQyW$j2Pmzl%MqErGJ#~Nabo8(M#N%Mi?baDNDs`<6WJOR9P_2X|!JlRTb%v zitO)&d){C#^bWol?tw8fWab;F%&zuQ9~U5DjfZ8hBG&<1r3E)K63ayGwp^=NbG}hu z3tHA!vRPkJA!xq2OGpqArtaj|ZA_`y+^CYg9UF~*0#Hi>1QY-O0JCohunYw7K^fAs zBNe0q1KPS}v&t7!3Iy&zid2&!A|HRtYQr!PMfd#*qrENL=_3f+fwpATK%p)z-3e)u zh*}RMwT}P3vYj{yc-7_T-Z^t;P_a75Ci(^IgjOZWk`zUtnC3$5O7vc@<0Xomr;5`? zD=1M14y~%2f*HcJg;%Q$STDdu0i+yZrbK)14Z+y4JxJ=3;7~!_YAdM^rQLsFL)kao z0nXF(9!v0)Q%`XZjLk$uMOTYJamc@kC%~Bd z`V&-tvtuKK0tKRgKv{8<>?K?~cPyzq|5dZ)H0000000000fPoH^ Ti6=e*Vw1=xBL);D00000R4VOJ diff --git a/summary_results.json b/summary_results.json index fe9b0eede..52b1aaba9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2195, + "Performance_Result": 10.1134, "Performance_Units": "Samples/s" }, { From 00d2b770e1b5d93d109908b879d5d8c383ebeda5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 23:01:42 +0000 Subject: [PATCH 365/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10225 -> 10222 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 889e65259..538c3944e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 22:53:32,161 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 22:53:32,161 submission_checker1.py:3260 INFO] --- -[2025-03-28 22:53:32,161 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 23:01:38,017 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 23:01:38,017 submission_checker1.py:3260 INFO] --- +[2025-03-28 23:01:38,018 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 22:53:32,161 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 23:01:38,018 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 22:53:32,161 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 23:01:38,018 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 22:53:32,161 submission_checker1.py:3276 INFO] --- -[2025-03-28 22:53:32,161 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 22:53:32,161 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 22:53:32,161 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 22:53:32,161 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 22:53:32,161 submission_checker1.py:3297 INFO] --- -[2025-03-28 22:53:32,161 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 22:53:33,257 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 23:01:38,018 submission_checker1.py:3276 INFO] --- +[2025-03-28 23:01:38,018 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 23:01:38,018 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 23:01:38,018 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 23:01:38,018 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 23:01:38,018 submission_checker1.py:3297 INFO] --- +[2025-03-28 23:01:38,018 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 23:01:39,014 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1134 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1789 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7f9d7e97a..783646df5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -992 +993 diff --git a/summary.csv b/summary.csv index 0b770bf46..7e9cb40db 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1134,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1789,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6419eb560..9c766e7da 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1134, - "Accuracy":"acc: 74.600", + "Result":10.1789, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 79865da7b2d739c80a657142a05fbfde2f764e88..815c8d7db7cc88d6d5cb811ee17c157d1346b2a5 100644 GIT binary patch delta 2155 zcmV-x2$c8nPwr2!%>xSVHK0#%1ONbf5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0TZkN%wW^YzaJZtPIG*k8SP7@eM!K4-?`z>9DDfnB}fwbPvK7_RxTkegccl@LIH*!FW6$S=u746Cjr0^HWGSzIjf~3Dx~7o@FC()f zJ0|58)fZ*7lC?F&cL;0l|nUouZC4@)_P#J88BoXs+dpX&d_>NhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgcTJ@wQPqhv$9nFW`D8= zkXHpM;|0xQQp!$-ECk_V!K-CPxoQgd;;GlREm0Cuw;z`3GOX+AWX&%LmtS*kt&&9$ zs9q65?LNQZRbDvv;*=nx1L9Rh)(Lm*Ie z{T-r1=umV!=$CycIyf-Z9cZG1oEx13vc5&Eg7V(=|A(UcjGX>JbPxS4(b-Pd_YOb! z)cLsTd|h>*K{(PO9k9?&5C~Kq0)eVSAW(G(1bms#{2i)8=pfsR9rQ~ds(%g+Omzoc z*eOBIjm`lJ$t_|PwnhHus;*jAxu{xlBdtul_}<3??EV)4enxtKAi%l*PJoWxcKSZt z+F0-Q(itgNWJ1!E8zGYgYnX4N&3VJj8O)o#_%FEH%;ynRD_f8@CkKM&D}(=Njwrmr z1QY-O0JCohunYx1oVp`; zvm_Ox0)L89GbK|AECVTPO5kxuI}9r*c}D@Q3;s_BEM7yAK3W(@QB}X1+E!{gP5oO6 z+*aUHS~a+soKTR)y~&)I}a*MEJKmh zANKo(s!(vGG|1Wh>+sG<)unDrSN&G9-=CropkM_p87wzg_vF_|(JKBQ+n%hNj+i?B zTzPLId|&3hlM#13@SiHNlklh#4}Ueim+ZGsjTRH%q)+;4Vc6Hn2O}wemwjgCeVFY7 zaew^$N`H;+_@#an#^c8H_{Dw{#^Z|o#B%Ro<55k7JM#0$k!*x2_hKV|+ zrH-EZ+R*iyy^E&G#1Z2RLh1K|zoz5q8M=UTag%`NxA(|0eM#V6a>?4Cy9e5Jj{|#A zQ+=;3!Lpa`+nc^9I^L&I({}2F{*mR zc0HT8rrr1CiDWkfI$cBHxj3=cF*iO56HiXi6ZBz;~kW+tK$$=QrFKeinJoMC7|g7OY$Nro?8 z4_3OxhC#VS%0m^a6zljszocs0@G=H;WBZLA4?lquKag*#NTLEU1NG3%AhLesaA{#%Px+8c5 z001Zz000yK0000000000004l25R=O$K>@mx4JSGQER#qlCL26WfLr{J^^5p$R{HP5+wit001`0_TvBm delta 2123 zcmV-R2(xQYFo(W!1ONbf5|e%i8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Ec`~Gq%&q-;bR@2ex;aYF;Xx7f777_AOraj{9F9+4U&g6O7g|P{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!` z50@khcz=@?B$rQO%CbPnTeh%x!AKBQq%5^u+xDz1NOH5b&|@F5LE|_H$uSMfjO3y= z!blnj89rYo1viKND3Q-pJ_aN4!2IU)MvuN}4f~bre-rY|H?9p4%pXWfLO~)~0Mk&J z%_;q%9$v^HTTv~Ids;`+py$qFXJw$KI^K%w*v=dKu<5jI&+Vw!^O}+s$Ev+n5grF3 z81@-`Hdy6>7lC?FPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!XgpNa^4JSW@V}T&3|MM zAg>C-;#rc%gvm~Z%mm?L#>;t@aMcv>*;A)!TY?c%wI7!1GOX+IXvJrQ%iWw?i{&f` zRa?=JT|{lF5f){mcomd|%h{9L^qO{KwfoOPn#ak){9;B#1WC%xgT56bo7??o5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iCTqHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR5dCoQ+9c z%g@^Qmd*%UkP%5!ZiG}8tY*HA*5?fqr#G#?#ec)q$9x_oYGn)3`s6?)`NH5o>mv%U zH+pn!Wh{&*Wl9Y3TF6g&@_$muccgMPjp!w=Pa}+iC6p!Nwf3$~N6Iu9W;EKbf-3Xm zXG!+=!ac9I7kUR@4EMko7&7xURAyIssg4Vfu!j92Sdi<0Es~rYDT!sGc2llZtUlkU zt_3ZsE7`2isSq?<-z`ZH5vK0s*KJIxSl_6Uyd4`2e*sWS0|XQR0Fxso7qf&2nhXV% z%M|f>vm+Iw0)JY%V@jqHSO!wol)&SRb{JMr@{R&p7yO?NSiFWJeY7x+qN;v1wXM{0 zn)N z`!L%F;(z%0mHryt@k{+EjK_`X@r(T^jK>xEiRIqG#-o}DcjV`hBiRu7ZzpP&b&N~< z4HI=tOC3G+wV~@ZdlyZWi6h1tgwpQ^e@(~JGjsvx;wAyjZ|{+1`jWuCO6UlA}6on`cf#>4HUdP<{BvfqiZRtv$yw!X5y%~)0;eSv|0|XQR z0JFgvS_lQZM-r_ilOQ4{OThf%DSn8WmZN642ZTsW&zHXqV|6B|b1MSZ6fUXaJ8E z%|pd4<;KCAGZtKs;FAc_o^o5GZ3vcP?D-ZH^I3H0D6WlDEX2}nf3Rizhiw3pJb%Or z0uwASya%RsBBCQw@Cj*uT{i**$IyWS9Xu|w0-wDBoboptM&-h(WEd>mw4TJtz4LN1 z_I;oA`LK#7UP};9L)34)9Tz$D zfk@sz3MV=NE|WzkCI!1k60IeZa3>W350i!`J^^Es#U~>M6(s-w000%s B>HGiy diff --git a/summary_results.json b/summary_results.json index 52b1aaba9..6d27d2350 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.600", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1134, + "Performance_Result": 10.1789, "Performance_Units": "Samples/s" }, { From 1ec43323b6f1aa57e2181d19f63d011b9cc387d8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 23:06:58 +0000 Subject: [PATCH 366/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10222 -> 10227 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 538c3944e..bf6681702 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 23:01:38,017 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 23:01:38,017 submission_checker1.py:3260 INFO] --- -[2025-03-28 23:01:38,018 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 23:06:53,510 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 23:06:53,510 submission_checker1.py:3260 INFO] --- +[2025-03-28 23:06:53,510 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 23:01:38,018 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 23:06:53,510 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 23:01:38,018 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 23:06:53,510 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 23:01:38,018 submission_checker1.py:3276 INFO] --- -[2025-03-28 23:01:38,018 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 23:01:38,018 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 23:01:38,018 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 23:01:38,018 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 23:01:38,018 submission_checker1.py:3297 INFO] --- -[2025-03-28 23:01:38,018 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 23:01:39,014 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 23:06:53,510 submission_checker1.py:3276 INFO] --- +[2025-03-28 23:06:53,510 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 23:06:53,510 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 23:06:53,510 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 23:06:53,510 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 23:06:53,510 submission_checker1.py:3297 INFO] --- +[2025-03-28 23:06:53,510 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 23:06:54,559 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42151 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420539 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -38,10 +38,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412967 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0958 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43633 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43633 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2528 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0568 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 783646df5..8de1e36fe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -993 +994 diff --git a/summary.csv b/summary.csv index 7e9cb40db..f377494c2 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42151,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420539,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -16,10 +16,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412967,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0958,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43633,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43633,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2528,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0568,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9c766e7da..adb199ab7 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.42151, + "Result":0.420539, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0958, + "Result":23.0906, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -544,7 +544,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2528, + "Result":23.0568, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 815c8d7db7cc88d6d5cb811ee17c157d1346b2a5..1b05e0a2c0bba4b163bd71594dfebae38ed96d66 100644 GIT binary patch delta 2666 zcmZA3`#aNr9|!Oa<2%e@6xJ9sEYvNQ9Li}Ub7o&k^>rwxIVUQKPo*%0?Bje+DA|xx zSUGk;&X)5jL@0+Mh3cN}`-l6w-+#d4eRw`!*Y#ep{%Tzw{1=>WEVyWe9|A#v;eR;* z8p2Xu%onU@y$k-urxvG&Sc@-f5&2iq-9kQ3{0Hy33<~Vbj;xhm4*B8VQzCV-KYNn} z2JS}>Ht=2iJoNB4`nteF{TW*koJoeue$1c=S$!&iFf)lyyF7bMOW0#E(NNNoMczJ& z(wqqu58ip3w^;DNa)4^VnsxL`cGXuVg7Rp=*Dt0mhtFC>I#!q=VW2-@jiZ7*gAn-}wS2!xf+a$bxI3BNA3Y z7wLi1oAv1Tng0DJbYfZpOsYP~MD4M(N#mwQO&iZGiVcP0+7SR+f%LvDnb4Pg3oSnZ6<1 z<|K};gH2?M(rS&SSTJ^C;QE`L3x-z(TEj&n>ZK2N+Z|}loY~r)GsBVZef*+fmUNKd z>3_#m=;W~pr_1`4sgn&1yUz%U?7|eNwiM^@A2Ly#8TR5%e-{$(U<;VR#mmMIi61Ik z32W#gZ)nkE==j5VL->m@4<$SlNh8rsadZ+k5qAxSqX&}D?`?dORCa?T$cH>Z#?nN^ zmOPX26tC{*fH2KI$?n!mp8!w0s*F*;c~#@^`1}Mse~i<^m-|Gr^LT7k&b?%$nN)0qix_4CL``W_T8jWYhizb{TS1s$`3SP_r>b83gjU~ydZA<`_rnN1j24KwcZ7amfTXYIH8Y3tGJjDi;Ab*ECk8ZI`|XR0$3yAN22V617g{LEEtXLXR~U!Kd9<4{d> zFNit2$)c{$MwQIfR3U3<-kyDj;zb3mk`7{i8lH}hYCj0lT)M}fOlt?#9d-3^@vbb* zG4KHP9ZgBs9^$4fKk5nT2?<(}-#2w#$Dua&f-c>HU0aG1J8G)f2T3P!yXEt?vyI*8 zJO&M}-U{P43AWJT|D&UB6hdfl+dhpM;l*S|la z`u`?ejw6~%cBkWxcBgm!=2k0t{FI$FGM)}-WSZ5T)!ol>oBwVjjuPtzWWgi-az(LdU4|HSuR#Zyd_zU^w9qu(G+Y#linTG0kQs!lInI5w?GL=f zf5=bMtPX(NN+XT$Iz#U(kE z_2{)2-eOhPPp+sudYaFT4s3w%GS6expe;PLLHkaM<^cs8xS|a5PRXhBv(4{(Ug_mD z%5-=xt<>4DzU34JefF5#e%JE=nJphr_|AC`X+2Wp-zjU~)ewLUnNV6#(Kg?Tgd`iA zU|?35z0$@MLdNxQf`?$8E8Ho!Vt?wzeCXK{KThfCDIC@OsTIQX)m?B*(9}*v{UaY= zwap=!dL-_1>Tz>)DlXMRzK(6|fJre3;>=xP^Ab0wRq&N_Mtf86gHCf1Em8TleVljR zdV~cq3fj~K9AEft_0_K&eQ}&%FO=bhzF063?RA{GC5Iwa-kyl2LM-#4gu9>gRH1o} zZd-jWTYHJ!&VygZAbC|BQ&0{S_2-}UOcCcbs&~$P^kZ4qXH=u7x{tJZXGbcYIf-86 zNY>loF}!`M@3&l02H(tI?XT4|61-QT2U=vecvjIsFKPwNQ*^J#9?$$wPpcaDmR*i&7<@2THIAYIs2PBOi!>bqWQ zy9VZ&!A8A2t-|nDNAO4Zp1AsT>;4Z5gucYJv9-#qqfFrWrmse{#YVhV{~D2yL)aBT z&>dUWS_uYx%4seQ`&&PPrv2{(T7^f#HLcPcmG zGEd?t{aI2R^AyG^(n}J^;sMFN8!a2bOzbgc$(CVG$c>Ulb@^uyO&$?wOskr4_ zL8%C3F7yG=$|$j9A($wVXsiIrVpV_)k=vgQ&Q^Hm^(Li!x=kRHn0)jzZvf{Jau(0? zrJ&%|tU8Uy=VaW)e%=Z(D&co0$6{?PE3=mDLkrNZMMiw9i00v^+@?$PX&b|Fs!FkN;lu^v$gv6T?t$QVg{vJv@uOP4 z*B-ch4SJ<0HNsZ|qSe}agXe>EH(k*Ag$yGYL_`M7CbeAQRb)zB48~+j#`%pY`W-n< z{3Qr5ou{JhFFe?$95D!KmvY_zc&O&^x}T`fkj=u-`_UNnysLYpu8CYOP_;2`c4|L1V$YipQbiV4I>Lv;pK-(t)ah#!8C+ea%3neNa&_N=Xmu4!%;7=ao=^aMr&8 DmuKUv delta 2686 zcmZYBdpOhm8wc=Baw?~xw93$o9;f9{c!br$OioG3u^xv_4wYk0)A*SwZ2KaIA)_Jm zFvsahMGlj+=2V1Kl8Ca2r-dK={&=42`hEWWT-WQqulsYquj|8iTy(69kdU0rQ#koq z3Iy5*BuKas0Fr3S$mE2_r|^~`ZL_tNP_rYU$yzF;hR%X}g~o^SRysCfj+)nad9{!8 zgRJQhf$NKK`c6{Q63SD!^2t(ss^e2ujG`!+ipc9CH96DzSNCVM9+P5Dw5kjwyT|&; zSr3NNHk76Je>0bATleQi@VLx5Bj&`nn&!SbSk6g~yo}g;@2`jjUFH)VcMUQTa|}a* zWp(pgqawJliXK+%EiL5(Bbmb+oUpMf<4l>{1fQ(rux}%QhqPxdD{oNOzXCau8sJC3 z14)wu?O98J@4>yUTZ zQ>DE>B<(Qt^gIVIEf>CPXMx=EQSH>W0GyE;6S7jJ$Y{Bq@L?wInVqR*`6ISA$w z`gVyk!@krsTpNDPq;2jyYdrsW)dfs8;&Ae`QXs&8~x{Dow z5;GG@#a)k_;)ggedN2YC`YQOLHCY?^J4%vb3#Xygsc;Ax{!&|&nu3LF7nXs??u1`5 zp?tzXFt_p?Ez8jPC&E9j$Jy2}vLSHqxZlGW!$YBlff}E~6%)|TRsyKT{iIK#8cFI@ zTSYROkp}UJ5KQV@QB$796%&YGbM>uEbEHFfH$2IGc{1HvXY@{yttT19e-FPq`sI5^ zOc>8App{QAd8kv%_zNOYEcB505!th>J394qTIV;Qvo(0Hk(8~sLWZ`MsT%Yx6%VJ_ zA0oq{J|C_>dWl51i%9xA6GuM26~p52fJ64-S(L;)1^GAkn;}-WgdklMqV(1LZG4`R zAySA!I_mdo$%KZUoxo~#!@3)2QrNH8*($H3oLcTD7>1?0{;U_)cl0bY>(-+)=52L2 z?_(d+a3CxLtmC}HaK`#%JzNs>j3}zBacqx6>D|+2&dm5_@rwtchzffI(KNi8INLij zl9GR;>UmqNNdDO&!uf!Yp0f6}FpJUSqa1dYYy8sO`RS<4FSZ8G@7rkhL#p}c9exmL z3KoTaxH^fTpLcfX{71}utZ(7C z-8id=9$1?m2B>JiNoe%KAYPXW;Q5s(1F~2z9IC2dY|A8^MI$gEzddkVh>Qj0u*X=r zf`1T(xqF_Av;xwu3^Z;y}C zAzp&EJEa{=RBdqoKl8+vNI}vKF+p_0!cB2sH|Ia9OdHsSfrO(u*IJ@SVqa`!@(4H%AF?DH7jQD)+_q%6=j&|NImxJRs{_vj`*e@ zr@{hR>C1LU@VgMFr=EHAd^CwvEU-F&A6A1p#zv-zcEX@}_-N)m!D6|qUON+*zLas-%79&@H~TD@;hN}W z3GNFFdT)Ae+(PASv@UjL&1E=~+7s^!2~o|ASeDDstTjg`+vg(rNNFTyw=oZWC;g{+ zje;+$_hni6^hd=DW5L&*T#_tMR4eC$~% z?^!8CS1mJQWy=gXj!t&XMSe$0J7M~bt^BDiq(~9hJ?yw_qbd3cY+(kZQdB{f+%c?O z|LyJPrc1CFKC&~0k*l7SUU80R*inXI>z`+x$OG~=(D<}&KnBJ@{f3Q+&l9U8z@g_{ z-2M0$C6oHp%d$i&WE&eXnonIop)g2lu1LTt7>nvkggsrJjgu;_nw@nCDNLldxiuH@ zYitVX*o^z~HWw1MNH-V*te0SF?Q#08t#Cfe2c&ilrCVtu|(}cHp(x|vmC@0Nu?^x_K zO4?e4f>TkjTo6}>CB)SUt+0g=C%B>yWF?bxbg)}+C!NF;nob-Ezrrh+^<5U^c{>*QUG;|qyaLzN%(286($^UiWEVNyDc|T zdhs#B@aN+i_xkBso7eHYMSqP`xdp~82QA0$6}A`g*K9mYag4~P)KnPbTlq8zC-)qZ z3S)we$YNX_gP(PxTk^C`^0fGctt1f<%89nh0|os{I~oGI5=$PD`U8X3*xd2dl9?L( zOl@GHDZ!5s{SQXz|AlSm^A~3IFRV_<%9V5qoag`3qV3Y77k~I}{Mk;l`>fqnF8G0) z;=Jx0ijv9xwEt@PCy3Q4Tq`LZYT|J(OMnjKQe zB_JQu*a^NG9$N!Eq646;%VT*8ukC`go02A)f0Q2^XusV;s46<3&e*^Uq0xs!%b_Cw>sZ{)fopUe&#)=RZeMW`j0kS3W)!zdq(;GN08DeLO<5 zdet29{xvsk)AJ#U$C*^c2XJDO`CV`%ZHfQ3ZqJ+NgJD|Q%>jZXvFd0WC?9vg`uzvt zz14-8oST?c9Q3=7o_JGU#-^vJzK_7$M5rG-Q0=pOz0bsm$6LCKk9=nB&5{j1ZzIlM zM6L~Gq`r3#wkM)4RJuoBwr-5wKeHST8Ms}-o5K5a!X+zKEi;IrmT9aK-UA{^8L?{V zSK-HZdr`J2(vYr@BeT=|bdon5fU9d0a7Q|#?&!K3JK^lG(I*PvjFBCfTHw*23$GkR zydzE9q6gmWbFIx}Wh6c^D?OXRf}fJtVzQlt(1&58SIO$LRqq;yrGg?vNN&%)x^`b2 z>!=7kQc>({cGHexdujob2U!mN-S_8d_mykZ z2&tCG^us#u)jO?MDc@nfyM^2}ba~u$txbY-xr$$S Date: Fri, 28 Mar 2025 23:12:58 +0000 Subject: [PATCH 367/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10227 -> 10227 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index bf6681702..ac321fdc3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 23:06:53,510 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 23:06:53,510 submission_checker1.py:3260 INFO] --- -[2025-03-28 23:06:53,510 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 23:12:53,112 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 23:12:53,112 submission_checker1.py:3260 INFO] --- +[2025-03-28 23:12:53,113 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 23:06:53,510 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 23:12:53,113 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 23:06:53,510 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 23:12:53,113 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 23:06:53,510 submission_checker1.py:3276 INFO] --- -[2025-03-28 23:06:53,510 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 23:06:53,510 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 23:06:53,510 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 23:06:53,510 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 23:06:53,510 submission_checker1.py:3297 INFO] --- -[2025-03-28 23:06:53,510 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 23:06:54,559 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 23:12:53,113 submission_checker1.py:3276 INFO] --- +[2025-03-28 23:12:53,113 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 23:12:53,113 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 23:12:53,113 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 23:12:53,113 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 23:12:53,113 submission_checker1.py:3297 INFO] --- +[2025-03-28 23:12:53,113 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 23:12:54,105 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1789 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3595 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8de1e36fe..6e1fc58db 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -994 +995 diff --git a/summary.csv b/summary.csv index f377494c2..bba704bcf 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1789,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3595,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index adb199ab7..d33186aa0 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1789, + "Result":10.3595, "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 1b05e0a2c0bba4b163bd71594dfebae38ed96d66..dd72f3384d704ce2dad32985718fd706777e9737 100644 GIT binary patch delta 1165 zcmV;81akZHPxDW(%>x9ozOMX}(E~<*&;Vv^)62gf8f)N{f-{v~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw=GXCG!)MCA^Mnv# zNCO2QnTZhPz_NIlkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ@*pkD#=?Mq{fP}4 z$7x88X;|f?5RDN=vOvi2d6Jgg91o*ZK2!M^jKm}Jn=_a^`ldA=)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QEkVbln64H#RR2QMSDBR!;%NR(kCNu!7}6t&y1;)3Rn%_C=RprJb6n(Nrk8~d>BbZyV=sn_$G zku}F^yw(vO2O<~`8GSZb?SYrGNCh4Ne^n}yYS|8HZsnQ$&1DZDuS&w=MOws!$xenW z1mR-At7V>Y)fDi>Q?G4Xf)P@;AC~Getn2Ay&1Zzm-JDyiWD$g_t!T+EqA}G9%c@nr z3d+J{@#MC>w%uCq{)>u&D z-0t*!``xOyCAvN?y1p(tFb$7%NC&R+2n31_fk4qA5GXnX0!7!~Av%N(MYn@~*@vQo z15@3BCOXKu(K#UNTf{0T?_K|YD7w$c=?_Hr(BBfB?R0(b@Pn<+$5rR+ssjzekq+sA zg?55Kpz06^R2>3=szV@PXFl_Hf2a*QMU<+QEy$Xa1CbUhga2rbD7?Ys(Y2Mae=wd^88O6b zDL?7SODW%x%GETYm$*5NFiMtEmWtQLyE+}IvS6IkXul4sD$*Yn+20HIyun`R9egp| z17l>!%r{V(UG1elE56fUht^>A83vOg2mWkSJxmK~}e51Y=w5+dWv%aK4(0p^3 zkRT#V-N~=pm{PI1Q6+gh1U4G~1hZxcxfujHm0N0)BO)w+ND0DrpeeG1>^M#WS!Fr8ch1}yRIU!PjsCzop;d*lBt;P@rnykN3Vk%|c!?tCsp7QN z3M$ltL#z6sWQH(p;mv9T)(db^04Yb9snFhgLojx150bhhI8+d~+Dhs}X?NIA_Ct4o z^E7?L51)cgyd{SnqS+F0M0P9AVGPDvn0c3uLmpL&4y9Aa7y8O12?TF zadsb^n2lZ6C0#LEL;ehZZC>BURb!!sQ!zkw$r)i5sMoeG@nsPX*HY&<;lx`3{HYK6 zjkjr$V;}G+1R%$_C*`wvereXI&eMDmr$wAEo4gxAPHK0$E(E~<*tO3km)62gf8RR1urABBR$YaB+9Un&?q7^dF?i=xM101^T=5nXsC|2<~p|X#y)I2UE6be>h-*) zWX-V}uXTjSfe6OKl06$`?SU7wNCh4Ne-#x;wQPqhv$9nFX0iv6R|P5K1%RzD9@CZi#f*puXv)olp%o(^xBJh?N%nI@hF1T55ZA1Uf6Hk; z$n8$wx8JRLTcYdZqU-CT1Jm$Ghjid7k3gX45C{|<0)e7KAW(Gu9il_%P;@)!mwhNY zI55>6XrhCh8=V8PzD2Bp^4|6Thobw8oc=&`5B)9C*-qE@4nO$R`MBzQU3H*AIMN{< zu+UBr2vi*cfvQ6wP<03de3{Sue;ukr=pfsR9rQ~dstyiJbq8J8DM8MS&H)R_En*e6 zMgHfiu3A>Ps9JI(txUZ5-p2y${ucp$MtXlBz`6fUfR5dE`aay+Snu}I87Ws}Lei8Q zA(I7bm~W%adBe;Z%$vRVFSy#w=MhyaTaY#<2ZH7+ga2rbD7?Ys(Y2Mae=wd^DKW%r zA-DA8rI7DPa)Z?G472VV^L zz!(`a^9@vHS9__C3y`qJ!y;Ib>!4iGoEs^LWukUlu2p4ozENKbR@7IrSzj_CShl%K zNDz@y-O1f;%$V5RsFJ)L1RITi0<&faxfuiz5<^9kBO)w+nh=ETKwGkEpwN_-?1VH) zM6CysTE~B1*-4xPyy|jv@0__as9YXp6a9j9LaPd8Ns1y+Omm_375dn$;suJFr;5`? zE2vNh4lV14k{QCZg*U4WSTDdu0i+yZra}kr4Z+y414!zU;7~!_X)CD@rQKsg**Dz- z&eL>`C3wnzsi(LH#%3g#Y)CT%fTL;ehZtzX}VRb!$0Q!zkw$r)i5sMoeG@of Date: Fri, 28 Mar 2025 23:17:50 +0000 Subject: [PATCH 368/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10227 -> 10228 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ac321fdc3..fbb3eeefd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 23:12:53,112 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 23:12:53,112 submission_checker1.py:3260 INFO] --- -[2025-03-28 23:12:53,113 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 23:17:45,659 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 23:17:45,659 submission_checker1.py:3260 INFO] --- +[2025-03-28 23:17:45,659 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 23:12:53,113 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 23:17:45,659 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 23:12:53,113 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 23:17:45,659 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 23:12:53,113 submission_checker1.py:3276 INFO] --- -[2025-03-28 23:12:53,113 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 23:12:53,113 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 23:12:53,113 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 23:12:53,113 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 23:12:53,113 submission_checker1.py:3297 INFO] --- -[2025-03-28 23:12:53,113 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 23:12:54,105 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 23:17:45,659 submission_checker1.py:3276 INFO] --- +[2025-03-28 23:17:45,659 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 23:17:45,659 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 23:17:45,659 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 23:17:45,659 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 23:17:45,659 submission_checker1.py:3297 INFO] --- +[2025-03-28 23:17:45,659 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 23:17:46,615 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410981 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71517 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420539 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3595 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2932 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6e1fc58db..748632cfe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -995 +996 diff --git a/summary.csv b/summary.csv index bba704bcf..24879b3a4 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410981,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71517,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420539,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3595,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2932,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d33186aa0..f80743891 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73317, + "Result":1.71517, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3595, + "Result":10.2932, "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index dd72f3384d704ce2dad32985718fd706777e9737..02e6e9e286732aee8e44a48413cebeac380793ad 100644 GIT binary patch delta 4095 zcmVAxSe{Rl~N1ONbf5|e%i8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Ec`~Gq%&q-;bR@2ex;aYF;Xx7f777_AOraj{9F9+4U&g6O7g|P{60Nx_@=={kVipEb22br2{*qE|LH$d_Jc185r#C7^MSb(qUc!` z50@khcz=@?B$rQO%CbPnTeh%x!AKBQq%5^u+xDz1NOH5b&|@F5LE|_H$uSMfjO3y= z!blnj89rYo1viKND3Q-pJ_aN4!2IU)MvuN}4f~bre-rY|H?9p4%pXWfLO~)~0Mk&J z%_;q%9$v^HTTv~Ids;`+py$qFXJw$KI^K%w*v=dKu<5jI&+Vw!^O}+s$Ev+n5grF3 z81@-`Hdy6>7lC?FPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!XgpNa^4JSW@V}T&3|MM zAg>C-;#rc%gvm~Z%mm?L#>;t@aMcv>*;A)!TY?c%wI7!1GOX+IXvJrQ%iWw?i{&f` zRa?=JT|{lF5f){mcomd|%h{9L^qO{KwfoOPn#ak){9;B#1WC%xgT56bo7??o5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iCTqHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR5dC4>zPy z%g@^Qmd*%UkP%5!ZiG}8tY*HA*5?fqr#G#?#ec)q$9x_oYGn)3`s6?)`NH5o>mv%U zH+pn!Wh{&*Wl9Y3TF6g&@_$muccgMPjp!w=Pa}+iC6p!Nwf3$~N6Iu9W;EKbf-3Xm zXG!+=!ac9I7kUR@4EMko7&7xURAyIssg4Vfu!j92Sdi<0Es~rYDT!sGc2llZtUlkU zt_3ZsE7`2isSq?<-z`ZH5vK0s*KJIxSl_6Uyd4`2e*sWS0|XQR01N;K0000#0LnQ! zgYA=f2sQ}N&1HMaajsaCw+Ki9t&#~y0dA8?2{r*;lYj{`e_2^bFPy~7e~sxlJ(s#> zOuKi7wPwxuHtr{*EU_^f_xnM1qH9KP5hvmFx78uFPOu79_BvLXY7_0%Rm-%smHi&y zC&4!Z+Qddvp+m-;8M4Lp<+hzl_?~Iz27Dto2LQoQSm7N*`5RKB~Av zh5S*Aa_m%U(Mpdk{7uC*{F+F_?|wog;)J~+5&`U6esP`R8JaO*d-%H3J#1EtgagBT|2yfpXyPu#=WrI(~Cp!2f zV>%cke`lP}C{dpS?ZzKXy)EE*C>_x*8gelhNX9y%$a!(@Fa8G+pVW zpXNa{xEap|L6ol7f}13Wlbd+)bvll-rB8fwYt_w05(U9>k=4l7QjJgk$vQp?;&cfU zH`Z&Jy-ntu-h95-n=kgl+3YS_%#w65wYOH?fAqrHU_7k-QtJmvcRYz}uMQdmGF^YE zqNH@u8l*#8{x4@-7prlp)e@uL(x%nS`7`fu>Kvp_Tx+jU;)pB#1SLQnkx1%@L{dj2 z;tGB56i`RP#UJmF1>tQB_eA*VcDOi8K8Cl%@A=7*NRk-QbhMw_j(JZTP0AeqLdOSUO8JRHr=}CdKM?qC60jT1SNnXkw_GYM52fiPjvL& zD?pKi%bT8p@U|yFIh74Q=|C<;WlRTSlquJuL?iD}&%A1Jn|6C)yf^7CGu*lM&t8{S zQCnvS>mkF%&u4&$<2&?q2zO*D?6I@>DXdBFKFD`sH!gJ7OskWn1m z+^V{pPK-Z{XC)Pu;PY}luoa+AU3+HU6sh=g_hW_6*)2)oZ0+Gp{bs4UkDR zKqlGZ2QphlWRe6xWaZvPlR1L`OzT;LDfUEuoXIN6E5)QGm^3{%ZylRs)lujIf3c-khiF`B)*%{X zvV>iSX@E&$h>;F4Rpi)Vok+`0QG&Jt(+;%wq!VQ}C6r*?pmeGXvV^gRX@E*%t^q2GygjTF zpt8u_qon!f6k2>XN}v)G@*9wJuT%}t-STCmqR5w zM{H^EAsSaYe250AEaCBC8laMB97$!7&4(obDvOLh3Y0&S&ue3 zml|CowzT>XjVqIm5E*~1!RiAoKIsH^E+MQ#!{mI)M6ig}6?=Ui1M{csT~<+KP?n5-LwEGYZqFKW3!!&><(Eyr7b|01i&@8h1Xt4W0i%&Ws zXoB5`guzOdrG}NPuGsDK_mKHa?LHTJ+wPODW}CKW^kxubEBbXl1>24BEQe(-~*cA_aU9&-X(-}Xc(}`3_mbyR&4nBQ8u5n zds#`5rDn}-$4~b7$1`(GvbvdrCR#^qY4{-;R~mkZ2GJ~G_+c7AllXN2%_74O>jcm& zGW=*T{6LFOIw34*g5igR;jGF0J}_%m?DzS5$b6=Tp9{Th_*u@H&Ajz&D~C+?N$K^`o}IIYsjVC+(KljC zuMg39((6Ms2xSSc57PjYWIQlk7I}SGCje!U*GGfb2U>j6i85P~N-+D-Fu;4)HSSLVak<&+m(+667(up!HNhCOZXc!R5oIXJ0b8`CR6X`vP$mi*{@w=W73Y_3Fe2dD>q%>yNW{J#;6l7dOZs z?Ujp3_h=gSUoHo=n2rBj1TSCoUfBCDdiJYd4E@SDG}6-VrD=7`Ua}8tSVsr*?lAb& zjfUe{Y)pbdTBuofYDTnr9p&>N zY+mbm^k1`z6O|5sHiZicCIJ8dPyzq|5dZ)HWN%|oa&K^RFJo_VWiEJaY@L$JPQx$| zMfZF~@0hlTEq|8hBSYVo3v>wb;WXg@0__aa=F;cHu?qY zgjN;Gk`zUtnC3z?75Z2&<2j0)r;5{7E2vNp4lU}tk{QB(w1qdT4OlP0MFFH7VWvVm z?+wA&u^mY2lHgE5+-fVS52bCeq3oMBfb%q+VF{jc>M1?|V>1!a5y{zvG(WZ-0GwfH zL4xuQXGw<7UJq8fn+=0<;grJn25wqU;^f{tF&VqAOS)pPhWr`ctX|)TRb!zJr(%HW zk~6|AP_J!AUE<3k9ImC#ufmD90{BxO^c!!-MaDkhQ3ycBxF_Yacz#(gQJtsxL!1_I zK40er$sWo5?CUV{s{ag>$07eFo&aO+>rYU#V|NcxTH43NS=Lbe`yMk06aXay>_X&i zCN{v(S5eQXgN+v^+9=L0l{M7#26cWR>5D>TO(or$AG#Vt zzGL6&(%rBd4tB&avA~iD;N_;WM~SiLKLH*#q>EaFGd)ZaUX$GmQj)rsi{A4 zQowWQ(97}0OX=N1H917K|0F4r7v2&vagLJMtVn+9bm_vAm6$$B7mqg?=(*&LiGGoT z(ytFaK7P#`-%Tc}bhy=xLB!bLg1kaOsyDudCWHo<^*2$SUR_iPD2cFg9}94(5JKU^ z@)NHv_74P@M~p|ZVUij}O5awhB>;+>L!;*>rp27mnOwIAAoU(?H!nZ(iY!8EK8h2K z+6qDX=)?4X*|r6qx-HkODCTs1u~ytrbUq9Tyhz-L)H~lTCcIpkZfCUJ9Rp_1r89?y z;9?-`iS;t<%*MqQ%#6t-mY|iyUv2Nqr{*qG}IGh?-J47#V(vNrGdyjvMK4Fe;^A6^xTW zU;gYMeCx{_d(U)_v0XWJ)Trn~-J)WNRo3~C>L=n&FKZ7@`kvQ1q;KL>>#M4#N~W0= z55lcvNp6!oc+cge-+h1+uXu@J%YuPTTX>gAggo>7?dAff^!uay+wB%59=1;)=9RGc z^>c+AYq&m_@SfRJ+X5UUJio}bOEL58C}&T;RRd>t3vOqrqP}9*X6x^ZSb=*r=Nc0s z5Z&Z^yAH1Hoh!bFLK#<{^fy z_!~R(bDmJ0K?yJt&CF;pPLn`~^B7cy1y4T93}dpG4V6ZBRu5{3Z`iiUBDZcG0?3+= zool@M^Z^<)?d!v!t?jU){tm0vjeI8V79!7D$Wz}ZI&ap&?3SX8q4|2^4lY`|ez`mu zVebF=4*VCCiw6EVb9~3Fp@wFYM^r{yO$hvYT~Om@eKOOnix=;-&`#G9aG*D`Zz~NQ zFfsK1@nd|rR57j@o|683(-tk`ZXk{5se+hWmqz?jok~^fO>M6GC2+`c9ky!_m;Xwk zGnvdp6p#x$bd5&VH7^ZPU%f}ATM3F zi9^H_raY>zyLk1A~t+w{YnO>zLcDtkss&yTppn`g;Y>==duG6M&O0Z%sMP-U22U_~5^jyYS7deKhO&2E5%Q`-&Z&Z-JW2 zf3hKW&#Zkp>$#j|Cya{hFv(Ue+MD=5v%J|_LeT%cDWeZJnGk_Pni`}4(n|4?D3LvV zj*swZjfog3_gRt{X+eWScMj4r^_llPmZKLnh<9T=AQwq-5n#3vF0#6`d-W?@27M%Az?5$>c z8}&L{6hl&RqotX7jVt$1hQeWjQt^w?su4GWnU~ve#r?m%wW;V+d3@tjcjKxS47GJR zOfMryXgF~MLa4e(t7RD>dhc``+|hl+)fxl|t`Sx2Hy3G5S)l)3GDKise*UIp7H5J$+{LQXbbo9qNN3ubCNg@R!Ue@GF#T zK5pp4?x_>RNJzyuC;ZZP zDdX8zan~7}9{^msjhitrdNUu=8xE@NEk;(5{%Rf^nqiF0D5OeiGOWF5K3r^)Qk*TSP;RzSW79B7~br zt{|h4CgbWsQ+X_7Q<=D~QCHei3s*cqHilsribL|?0VeMggIJW^GR;8Vd!4}y~PnQ)n`T_XlK z60?r8?vt^VPfST=*;mjKxb9LOjf22liVx?lJ0&u^)9NR_5 z#+>xjo+?I=-OU1yxZ=k=D0eJD7%?mhCnG@TxS-{#Y_QHa1&<-ZIV!>;!pBwc4OU;> z#pV#HJW9Ssu^PGI)?Z$+YW&VqBf4LLytM(7n^CO1fR%Q;X-_%=kzGTKGa1fx324Vo z%KtwT_+Kc8#1RrB*q?+@=}+oAWI`3>z8^`QoJmqjF|4-L6sX0m{50hk=5jw~eT+Na zEJ9fPm^t!k5mp;%OagVvV+N*92Z~!%lw)N|xfsnv1y~%;xqw6P+ILay>76l3)&~CB ziRCQhoUP@eI}ZxcoVUb*=)HAtluRF5Kg>A3bOXhW!k#!gB$4y{v}Pe6e)5drfVsm| zXuRs0eo}lm^rfcqiChF6Pt~-rO7=6CJ!L=K<^BGHxp>lyNy4s^s+nwK@oUhs()XRz z0uRZU{Mz{J`!Dm{Iy&!Yp$Mwr%V__|59DiNnYz+OiT;VKiHHb57!Nl!2o@XTKv^$+ zHXHJOz^UM&YFG_$n2ACSViE8lIW6!m~b1FJPFFQd`%I4zf zwRH*Mt5X4ytag9970~P|Zo7b6+V#2X`^Wt%; ze~}0dkOH|M`{!9#u1?k#zU@56WnJ$6tmB2RmDSv?{7BPVQKBB){OnPCwI-Q* zn$~}|(=9CwkhZyedhG~xbw4PQbx?ku?kKE7p72?^uyRRn5RnE-HVA&**UYBeyrqY3 zt~X-e7(h9Yg@PT%3MaOAv)r}t^fD6Ms4sas`t_Hc7_XH?9Q9S$F%Vvl)nI4co_uCcqTgNEn;YDD8xs{ir)tdY(=oq=ROOO3!%&mPYV0xR9s!~+< z-CgOK@m{q0(6*j(y7F%@H_5tXyH#07dax;=qI3JmbCN*>j634kvq{r6(7zbR?;L!7 z{zoe&^jG6Pr)*vcKY3^)D@ewL*J4`3G}Eh%Xa~#O$sT{^3hoA=cCZ`|_)lx_Z^*9tOP`4Rfcz1{P5N2xKZ5OE!Q6(^ThG{WdRy-uD} zKS=MNY#cG+Auher99q*;Zg{v4=7w6gtQ{>AA;b%|UfOd^0}%~OVBG)WU=oKnf;X?} zwT56;_5cR>Yx1znYs?{5U+eno7++@M6sa;Us^~h*B+OMXm6?on-)q@>SOixqDxmA8 z7yQ{6E#QMK;eq)=?f$v%18IeJdxYV{XN{8J zvwL!j0~n*mgq%@@$6Z>bj-sNywXL5l)P$p5-k;@Q#b-Fr76}cm`^5Ep#H!l|8y@Ba zag!0LbASlTY%th~HdOmWS5TU6UuIF}U$i<1Fi@WJz2X7TuWa3<7(gIXMi2;c^z_4d z-7&rI<^ABlJ6`vl6d!rGr;&QxT2WvF`XA1$dIfC^M~m+p1#1=4Tq7gDi^+)0F(z!$ zVq?h{brVKwCyg4v{uv?5VxF?7zfI^2tEXiM0|+vw+O(E4NO|KC0lu2IP(I2jI})Z> zu%Cg8Kmi`}eQr!+T;67&GnW(d2Id~?p?uwWNi_&{OK|0s;kq8)Z^HD4o?pnWa!?!e z{PKv!dC(mtzQOP%_e;t4)6t-QRJpjW|1%1r)un_@ZniJNtlHaf{xOqUk<2|`SZZR_ zt#PSaXKnQUo&Y@TXdPB}G9oO_>iBdCISS;CS6ouuaf0xSo3EBoT zz_yXh;vH2o%aljK__5sn;XuecgVg_0>cp*01lsASZ~2EO;)Y{u2{KJ*>EC5?DLBap zMwWlSdGeN&uE@XVGbacnYzYDh9$Edr5)$f6F_M;MVELz=>?>o*;6(0`kz_q3b=>8D E0N#YG3;+NC diff --git a/summary_results.json b/summary_results.json index ad16af494..7a246406e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.3595, + "Performance_Result": 10.2932, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73317, + "Performance_Result": 1.71517, "Performance_Units": "Samples/s" }, { From 9fd6df81770bccc544e10aab7aa8d3f22ed62505 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 28 Mar 2025 23:19:09 +0000 Subject: [PATCH 369/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10228 -> 10227 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index fbb3eeefd..26d2b8517 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 23:17:45,659 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 23:17:45,659 submission_checker1.py:3260 INFO] --- -[2025-03-28 23:17:45,659 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-28 23:19:04,900 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-28 23:19:04,900 submission_checker1.py:3260 INFO] --- +[2025-03-28 23:19:04,900 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 23:17:45,659 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-28 23:19:04,900 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 23:17:45,659 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-28 23:19:04,900 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 23:17:45,659 submission_checker1.py:3276 INFO] --- -[2025-03-28 23:17:45,659 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 23:17:45,659 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 23:17:45,659 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 23:17:45,659 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 23:17:45,659 submission_checker1.py:3297 INFO] --- -[2025-03-28 23:17:45,659 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 23:17:46,615 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-28 23:19:04,900 submission_checker1.py:3276 INFO] --- +[2025-03-28 23:19:04,900 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-28 23:19:04,900 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-28 23:19:04,900 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-28 23:19:04,900 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-28 23:19:04,900 submission_checker1.py:3297 INFO] --- +[2025-03-28 23:19:04,900 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-28 23:19:05,895 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 748632cfe..eeb072e53 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -996 +997 diff --git a/summary.xlsx b/summary.xlsx index 02e6e9e286732aee8e44a48413cebeac380793ad..f0ec0ebbe75aa09285e1dac3073703eba8272849 100644 GIT binary patch delta 364 zcmV-y0h9jpPxDW(A0mIFWJ07R0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!FH z!!Qs<_x%c^y)8dTNrmk|Te51PP?whOgfvM+tp}1?$A4eNcANyV>T-1NoVhcoTpeT^ z{epEus|saFiXu=dT-Y85=G8a#c8V*RHz4sR`p%U3}JuT!mHH=tQX*-08)-H zQ=z^0hG6X29wc>1aHt?|wUyL|((bUK?3?ZY=V^M6C3woIr}zkr%}hiSlCv3Uer!7c zIK$9_1mzvhk_?}{9;|dX8wTaVDTVJ1+_av=*?n+gHg;W?bj4r|`7`{vd3hUFjfFa% ziUF!i&Iq$Wy|zeoi7$(AxRyG<2`AnP;E#RKZ@f*5jD5hP5P*zvPs&H}^xUjbou~OC zPK!8SHhDp^2a+znjw7%7&p#i&uBFbe!il#6_){PB8*j%&#y;Rt2tdZTC*`wvepxS3ou~Oj zoEC9DU*`qM9?AXe>oD@F{|uDJA^#?x0Aud!Pf-5>vt=U-1O+yQ3koKabthB-W|PV% LBL)~H00000|JS18 From 2e8873c899bbf3eb8680249624c6326055678e95 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 29 Mar 2025 00:43:06 +0000 Subject: [PATCH 370/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10227 -> 10229 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 26d2b8517..bb88da3f5 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-28 23:19:04,900 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-28 23:19:04,900 submission_checker1.py:3260 INFO] --- -[2025-03-28 23:19:04,900 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 00:43:01,181 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 00:43:01,182 submission_checker1.py:3260 INFO] --- +[2025-03-29 00:43:01,182 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-28 23:19:04,900 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 00:43:01,182 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-28 23:19:04,900 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 00:43:01,182 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-28 23:19:04,900 submission_checker1.py:3276 INFO] --- -[2025-03-28 23:19:04,900 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-28 23:19:04,900 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-28 23:19:04,900 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-28 23:19:04,900 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-28 23:19:04,900 submission_checker1.py:3297 INFO] --- -[2025-03-28 23:19:04,900 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-28 23:19:05,895 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 00:43:01,182 submission_checker1.py:3276 INFO] --- +[2025-03-29 00:43:01,182 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 00:43:01,182 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 00:43:01,182 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 00:43:01,182 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 00:43:01,182 submission_checker1.py:3297 INFO] --- +[2025-03-29 00:43:01,182 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 00:43:02,128 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410981 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71517 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index eeb072e53..806adbfbe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -997 +998 diff --git a/summary.csv b/summary.csv index 24879b3a4..f56983962 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410981,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71517,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f80743891..e7d5ded85 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410981, + "Result":0.39409, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index f0ec0ebbe75aa09285e1dac3073703eba8272849..2ffa2917bad570a1f4d907d25c9d8ec46cac45a3 100644 GIT binary patch delta 1605 zcmV-L2Dqw0>bRE{rI?f7`i2PV-4VoM_r z(SNwo$wM^AWC<@1(*TpiPa{19s>sg6Isqn&3_VKvZBC)ZXQPzcmk`w578i%B`iKn15^@?BdIKM`LF~)Ws%QEfzbzAe9{SV6oA=>grTE= zxqZM&4Tz@i+ z zZXd1dZp+O>Q{|wEz7boxeTc@DZXcpSG)uUBmp~=Fj&d5)UcA(6`Otj9x|V)+kfXmZ@Ycc)oj!DtemH@TIZmN&JkPMeTc@Db|0ca zG)vfhm>k`5`Gz{2eejk`MEB5>RD4S2( zysV_iQnTi^;U|0i4P*6P+Wr^!pHvEB!u1gJ_oU`!EflN$fg+W`B|2hjjvI z7WsWN_^{0;yU$}_ z{;#%zO6n`cq_p~I&(7Jy)PGhElxQ2VrPYUMJZbeI8icZh)rV;SN-`dpE{m)_tP_B; z$m*lP>H{r4=|q_=NhNrFXc%D0%s#M^t=R1I7?#N(BT_x-iEJgcm0G}VJAKj{XV29+ z`>6o#*!z$myfO>3;()KIuf6mLw9K zJ~Rx7WKJI-@;Nzu@`?1GMC5bz`Dj}Zshjeava$8%!)Xu=gOgw~iH%;Eo~~8orgil? z$aA%Sy?S%vgFNjoob|`qs~)PExyEH);=AT89aJ2fL(y^V7DISJ>>PwC&Z zuVIpe)BA6upxY0k>|1(QFbKos{!4aBHgEMj`Y%vR0|XQR0JD)3jt+mEs~0dO0RR9{ z0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!K?!!Qtp?>t51dtxU61XhwtfO?`55=B(p zEPLBp#6PkPY2F?uZPQljiQ{@^znxvVn(b7Z{DSk+=o;l2K?!JXgw&fFeJtnc14?{g zTCmn=s8J6d&6=BvTgrcpgEwa^xFEqN5u`okwnp0!EXCOKEhy%*=+IGI8>d)^rQ2Z3 z_&3`CE(p2D3IY=>FgyTLI}ylkjaX?KaIOPslFs?|c@Ocpr*D@a#@x*Hh z;;E1NmAAtp`#um!3_$mBkIHBH{IZ;*rXa;#O3Ji&ToOWWOG@&uVdO>s38)T3{!Kgr z_PH-VLGuT*Wh2T034Gla?C%Ny02z}wCRYKBlaMAn0u~jMFclV)WhM}l^d=esz>^Or zIsr11N+%{8oU0cwB>?~cPyzq|5dZ)H0000000000fPoW}izhw-XOqe&BL*2I00000 Dq$%>6 delta 1545 zcmV+k2KM>&PxDW(bqEQ{IXZ*w3IG5avwjGk27e`(G(9(O9h+m-QRo7(rB{b&Txr%J z8f3DBU59CaNn(hR4lz~a*kPSW%T7^(wgS@*wD_bGWi}<0VBDc$KqV1LtKFHZ(!7&W zx%iq+hKn_p0?Fidx|}bwiBfrW{RAkVsgLJeZ5w%1-BFgxF{Y{=BaivOB-%i1>Et0A zSASZ0hz6M~Vdh~PV3HVWq=`Tk`FU6;z+{o5M@hrYDYW=(lyd(PqBv4?4ZN@A`7DvP{5 ztP`NJ$lar)`Q{W_d^Sp;5==ZK3Rkbh#(#sW*NS~Se?64X)ZKHTcI`ba*amj6MpSKG5QmPKc!d+&&}>Ed|W( z1D1k{{XU50&-WBOQ_s((+BW^zb*(Hb=UI<7IhPt;Bet~q5REIXK12gtGL7VtX@4Y_ zOe49}MOGh{Kyq1>psm5`11&!31a~eWtV6@(e91(xh}9K)eI5hzr|ex;Qe>HhtlqZ! zXkB+(ZXTK{2Te4N*wXGpG_JJ!5DlVP!tTQ~fF{uZnniXWmH^N!vioST`#_6NIw5F+ z-G_w1N|vRDm8`DV?eq7L`AqFT7ZG~f?vt)&o3>}=7-Y51K@+W$fDsjc@*Z=ep=gp$ z0L>!5&*tC*n&9^#o#5Ujgmq{bu*nQRFl$z9`1w&bpR{{fNs*;y&27g|_V~v$b4;?j znS&-;M{H^MAsSa2euxIqEMfRz8bFiybpXvG!w>5O&@3|iXfXUhi%&WsXoBH~gyF2o z{5~*iR_yord&qpIhMx<6y>0ke&YI1<^=vDLO!SV}((pqxt~C4*4U$>H@WV7fCegrp zw#e|q5&)S+h93=vA87GOCtxdps1Az)Fw6LTbj5z3$H4qweFc@&SBgpL_0gW4vxljz z94OH@VoR?N(RkA9Lo^6w39k>+0F-1rFkKdTeOM;|Ws%oMgVzUtT71%pGFy^LF#FIj zz>>LrU?p3z+vhPXlR-wLdeRfwN@^>$fZewGq&LohsaIAW}EwEoEbW>(z(TAQ}cI!DJE}y)ZpptH@33>V1&sYX5rm>cj_m+Fv;9kF$3@ zbSJA9JtpJX(1`zBy+YE?ULk2_$$EqH#UwD2@AKfW7TnF#UnncKX7uj{tCvR_^JpAK zv{BFvy2m!4g{lQLZp)=A|HRtYQr!PMfd#*qrELZNJ)k5KwGkEpiq~V z?u0Z+M6CysTE~B1#de$ovg&el@0__as9YUn8~uWHLaPd8Ns1y+Omm@j6?$*h@e)PO zQ^jek6;!APhgS7n$qZrI!mHH=tQX*-08)-HQ=z^0hG6X29wc>1aHt?|wUyL|((Zq- zq3oOP0Ox6Xk0p4@si*h|jLl3$6OywTX?|=w064?Yf&}Fq&XNqDy&kM|HyZ}!!YPIC z4cxSz#MymtVm5YNmvqHo4f!+txp{dTR*i)^o{9mgOU?+hK)tqgi7$(AxRyG<2`AnP z;E#RKZ@f*5jD5hP5P*zvPs&H}^xP$^QJtsxB2J4qUp9F`vImkbzK$cW`p-Z)4f!|m z1Q>H)e}eiCvu7j90tw1FI)m*B000`3Iwn^Eij$HiJOLDw&n7_uzLOCrIsq_~OeZEA vqhvy)B>?~cPyzq|5dZ)H0000000000fPoQ{jVC? Date: Sat, 29 Mar 2025 00:46:08 +0000 Subject: [PATCH 371/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10229 -> 10229 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index bb88da3f5..d3c619a45 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 00:43:01,181 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 00:43:01,182 submission_checker1.py:3260 INFO] --- -[2025-03-29 00:43:01,182 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 00:46:02,490 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 00:46:02,490 submission_checker1.py:3260 INFO] --- +[2025-03-29 00:46:02,490 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 00:43:01,182 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 00:46:02,490 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 00:43:01,182 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 00:46:02,490 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 00:43:01,182 submission_checker1.py:3276 INFO] --- -[2025-03-29 00:43:01,182 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 00:43:01,182 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 00:43:01,182 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 00:43:01,182 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 00:43:01,182 submission_checker1.py:3297 INFO] --- -[2025-03-29 00:43:01,182 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 00:43:02,128 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 00:46:02,490 submission_checker1.py:3276 INFO] --- +[2025-03-29 00:46:02,490 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 00:46:02,490 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 00:46:02,490 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 00:46:02,490 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 00:46:02,490 submission_checker1.py:3297 INFO] --- +[2025-03-29 00:46:02,490 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 00:46:03,598 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,16 +24,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71517 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74879 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420539 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29046 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2932 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1048 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 806adbfbe..a6905f8ba 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -998 +999 diff --git a/summary.csv b/summary.csv index f56983962..e5089e69f 100644 --- a/summary.csv +++ b/summary.csv @@ -2,16 +2,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71517,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74879,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420539,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29046,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2932,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1048,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e7d5ded85..3eb5e5850 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71517, + "Result":1.74879, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.29046, + "Result":7.67415, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2932, - "Accuracy":"acc: 75.000", + "Result":10.1048, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 2ffa2917bad570a1f4d907d25c9d8ec46cac45a3..4ae58815c61bbfed43be983c6edf5d9b9fddee00 100644 GIT binary patch delta 4379 zcmYkAcQhN``^ST*^)VWwM#Tz=k6DdbjTLIvC~6mty;@?_XvCf+No`d%V$@cG7FD%U zHH)gEXpP#`s`b-;zu$9y_uO;u{p)q#_kGWE&wZXf!#%^Ak5u#x3NM%+0s(+v5CKEW zMcAUE!C3ngDzH7Cy@xg8vvLDT>d+?RlZY#=D-s{Q8w@W~uvSp?+mC)|=i|ZO_6yDC zTGuNM(|H>@+dMaj2s+#o)=P$(_>y;UE3dpp`5*NPW>;%1&hCz;`*7O;AF&mQLxpzJ z9_+3jR?l4*Yu}AxanPnx3Ulil$k324dZ2=YJ-Tk%%WDGNf%4JB1~71{j@mU zV=6C=8ZOVn5V>RMS))J{P;_=k!`Z;Kbn%zeV+L8R9{4FIm4*^`uIYkO`h5oMcb0R; zgDhU3#o}GgHomZQ>|l!^Y!E!PG-KK%zIOBHT=dmC_OX$s5`WXc;>$RNQj=@<2bZF% zFd%{CUu?oC(P~U`5_zPBh@+Mgsu#LBqzxh0;D%s=kJjZXn(Qa<@Ag{qxRyR4Eh;tc zb|XA**iQL{{M>s(M18E~uJ(`W!P0hj;&1Xetz)lXgN+B?HaMlJ%LZ4+n0bx|+c$Da zqrl~vh6@8j!6q>iaWvXl?L0L_B>4=Cak~nEJm1V4tVbt@_soxhuO;@_KJT9U4*IqF99B{$5QI)J!q%L z*MtNqvwe5%UAuZ01J1b=`yM~(9s1eMJfAK5Q1N5T*NtCc$J6B`VN44`vZYN@Nt!VQ ztxZ$4Lkmn`!m17nu)vgnl$sN>0!mR#nxx3bld%PnRClJaB1*l@!;1W?*6;s?ZQ7n= zU@e2aE6m7)p;^$gb#&O~R%GQskJ-{{IThnO9Mej8bU=h^{!bQKQ(j7Jxdy+z3t~60 zl<7sJr&Pd@R9LAJ=C5ZCB|t@aB^q%EDbDOO$Ict0f{Z4`Wa$Ka9X?^A z`?V5p*@%5^t!q0|Ie8@|VQn)yWX4|V7gHPO2KJf^YLTb3ziQuv?+U*|a;#>J;DlOp zfug(#=mT=S(dH$2F|@FaqoTALRo{kEukrNDNaYtSZXw(wUAyn-SE#!a;0ZsYGt#31 zvH<`9dfJOY1;CeaLyho=)*3^I1(L>RrmdI#NT0-%4)?s3u6akoz_In6+2uL1LYaH- z&VuBWD{O7sru;m#a?viQ8<1S;*oI9to8!G@6`I+1%PB!@z<8WUW>;*3F~0wRC%H+U zo;ckE9*DFIMzLs(dgISH8F{vq>03^nhR`EJL^pZj^maufxrQ%Iqwg})r3Je!ku?co zhq$G1e1wj?PAp$a_ij@FdWf&2r!F`N#(8!8$=F$+&zCCMCrs&K_fjH#w#QwB#C~{i zp5acn2x_5!mu3mlKa;(^{imyP zLRzCoDnT%FV~j2xTXn7kTaNd%-_1nAvM0*Z&=v?1|GK^-)M zHe?R}{?Eo#@!k~ioJ^1wJQ5@%@@WueOdQH>J_mzd!is_M6B`3XpFQ15((2jv6cWG% zF#h0V&@EMT1W5G)m4Fm9M#e*xf;ZXowcq`~7#OMrO|j>8R3Ubjlm??;>9y@x zlZ9Z40^HwlZje-cp?Cp;m*clo3IwKGG19%dC| z(wSoi8tH?;4-owcT$!i*nI%_HY&7zu@(m*E0`r2UPPFjh8yjX%>e4%;XX#|i%74$k zo?6WL1$@9;6m-`Lr+nkbG zTJ6n^?`PerT*Z2Mi&YYxUS5?sLafxjTfg-Ihr&zf-o3y={@@BIWQ-$G7`en=2wix8 z1aSRh2aSwVqM?#8F&#y&VPd3$$I~wLFRIWR>=li9UdYV8o-^e~wDtMYI3PgST;&t{ zDoL`@5pln2`2U!F94e`k{|XWy02v0*D|s5e-lr0Xm8ewFP9?}cOZs^3rT+1AicGUl z7VZO{>;0H4iQ?Q!-b7m&zfc%`GIct%51H(t`$rmxF;xeyD&oGionnR*(E*@1Ag(@4 zh*VMQ&=I<|2vyTqO^VEZE}*^DAh&)kbZ0@|+}KW3q;lH7VnSv&uXjZfn*NHa?7oWMsk@)OpSD_HI8dWQ;&W*6D4LgjRu!|058L7J%$LCl-_ ziqEg>BfsCPjkVOl7MFf){I%>n{mN55R`;T#qdt#((tuZ?_gENBEZgNW(R^%~ZXYFDi+%?Ap>UcQpD*;<|}zb+hB3X8kcgyQZK9@Uo2(()}! z%teS}zt_RktRqKhtS)ZM+eZCQ5FFv-?JvI+Yy7QzY0+@91yfPfE5l&fy1qZ$veEce z8E(duz06ZvJQeF|v$8M1jjWHFigg1R6i~nuzaqpb@-4sb4}aUoay&S0wzZQ>n*zI8En$+s8mFMPALYCT$&Ypl{V#A=Ym28ZaBc^GanpZr^W(UKOddu} z0tQX#$}t%xN5eM+e9gQ9BuNTXXmo}fEFNGj>mAz%Q|gY2K7PGhL@8VKU5q zyyEl5?m_C2{wd;0@YH5~t}G_!(dwi*QP- zzr-2&l4LBY0-Xyz5-cvF1}yZ>e#OJgOIE zFQmkLX>rmef-EkXj6Z$5kQBgaZnX1+((1~|b0&*7F@j;Qq@NU&KbCE%{N$Vco5a>s z`L_6#`-8jK5PBP>9i|?WTyC4;iJF3Hhq7wfCUq(E4yu%G{jE+D)3w+3j<^8tHbDA=NW6~9<%T~%v8WLZ;d{+G<0Bo9R5hGZp zvUb2|_TrniTWbC3#N0Dnx!!bo)$dAFyNafOY6rcQjD>g2Mqf3q!0be&@usCL?=}CE z=A9YjVOxI)RA?jUn~Iqubk0~9eC8f3HRV)RVljo%m;FKZ^e`)6rd+O&j=q2`4nka^wSbPxnvKJ7#3{*Vp5{ zx!3SwRW{^`L0Y^}H&@iiCJn}*XY@LnVrgE4;-k+d&gnP)ZR>Y@gGwOtt5q<~egW~f zoppqh)F30DEaSbpTcv$W(Onr5<*WLbL1$?1j{j5!BJf`K%>ve`sT>IJ^uw!xK3|1e z*0UVJZ!&G9q@WX&4~tU15C1?n+9Y`|?)|eI*EhNwO7hu1ZCg@w+&n^`c#1fh?!~PK z%RMxBz1hdw=S|BN4P(YG&A+$n`FaR3_1yMa(an1^7P8;Um9#yF`S0hqzK6?-7Kj9s zItjvp;utJ_1tblyhZKe+sD2d>$+3E`Qpe~v3x;})n*WSn2zG9NX4{v=7~y%l+QwW*{9Ajg?u6? z*Ox||bHeoSOJCNs&mviwN@k4YP|<2aFv6_>*Y-uZ3gT34ihfs9>32&=lu0N=lweG&*O34pF@K~gX$p)>f>`tgf|cX2!s%xQL+wDNthI@+Zu#0rb!x0NnB9^-V>f0ARU#T6wKeY0VFU@;+E1=R2~-Oc7& z_7$zHH%H<&NAaZl_iHEo`D2BpdDHaAZ_(i+rM36ETF;aeJx+?-)Nz`s+CEZ}_o-xC z=>HIvjgP`4sK61@xc!j#%=zCeSQ%0UY5Xw+R>4#Q`~F&$UQ|Q1f14JgX(8;5)!b1z zOv}5j;nM@&8r2m!s=ga^1F@_5x7xKhDA|?n3tf_pJQqnEF;TiK@U!)7{Xiq(qbnUL zc?-j|Hz-?42Oi|r(!Q(8ZvhxLOdk?G&*QP_y^2DL#|2rcKk12$>QqgSp+YMO_t|4a z3lrn>!==|Is}iBL^_8v;Z3C_D2nK{;N`G~YZhq8i@?7<5s8hnBfU{$6vBE;k*5_eP zQZ=#OV`RHzdOrit(o@!K2Ic|rvmLRcZ<()!TnPz8>W{o}Ouj1+Z&Aay?8^i`$ab0~iQnl@ z>3*t`q>xXOeOCxXIN zRBC%DfrAT7KysmJQF5}nE!q(OIh+rWeBt*caR~7$E{YXjJDnL=k&LEa{fGhD#d>Pi`Di$ z5AsxWJ3n~9dx6%~$+7fo2;;!OS4G{vY=Q3y@6(SM1b(&e9tcQzJjuK}WBke$6$&dU_J}=Vhu)ViWY#uj_H+bMN1c^W3nfh2kPcX zGxlp$MewwJNqI|vx=J?C*TWGOE9|IE`_r=5AaD<3)cds1TPE%sw9-HzHWZo!U4C;^ zsjt~D&E_OMTNQe3f7Karblk33_4Z02=sxXnD;t~CzKn^}y@1kq8eDb6Ilnn>3oDM# z;uMW`%=a^{G%Z(JEUCOCUM4nEK2pjkjan2(FoKJJtWJCAmYOe?;6HG4(qYI9btr($ z000eu5&!@w1A^ySzZ3i^QJ9+y>@F3}O>aMKc$VHHE=@NPJKxde`*RJ`KP>Z{inX1h zA+5~pR{K3U^&F{S_ZrJSY$7~n<(m2k^4gv*_`?-UQ=q(zWE9lMuw~_)(m+TpK9XKu z5J~e}36x`SA||S`=`|sK$#C3S4q)~r4dfCF^Yq3&b(>Z!Jd*8{uc;9PxM;hZm-IfP zbi*uawIP=Xo_Km?p12LK?Qglt`U#GQY)DDDa_-mXgvEO6hg{q+(nlW9;&R1^hFSl# zdI@c(n(0c!dUU- z;OWhs$D3S*`c@CCkjqhG=Rl|mccRv26x>KtaG4st!)&P8-o?9==V-e zM=9cM=gw(gq86TQbj>+Q#Mosxi#m}j-m|g!`!gqrX0%1oEN8LGaEq|r1n2yh%|>h4 z<3O4s6x6UhACm+Tv5`lK+?sFP2xJu=T@!57%72HoDd+^JW#{8shZD5)Xa?Oi#dyKdNC`) zktzoq(0J@OAQXE+F-ODmqaX&p2Mb~pAZU)~>$*rb{vZK!}F&Eml3#`%Mc__wDCbtSs=E49|2MEC;q zq~mi}!(FhN#fd6Xae{c4JBb_T8Xaj4RDwzBRI{03oUptu;cOA(;V{Ojt9E;(E6ynx zOC>Iaxob-St*4S=PPRv3wmCL9B-fWOWsRBovC6T|P?nUXeQ(8TmRx=7Ag2Z9(i5NK z2hT@l#C7bWIeJpe@f(L&=qMlsPGO#Z*{MG+!X5zuq+Ft!)&2p?y8FUF0a+Y;0J7ue z5PGF66e{oYI&`CVZ7MuHq;#k;kUSk@I$`d#ZpENm@lt8px@Y)(f=Tc`(fsSPct;&6 zHd!$orl8ed7g(9*d|S?`{#CbTH5X!h>hMINtPt4`FGLWce25Yj3_ZaB=K6F6Ov3}=DM zZI(Y#DR%#f5bmGA>#|Sf*N6_xyN!IDe1eASxAf+wREC)LoS^(Bh*+KbQ`r5w#8!@p)Bj8?K{eklX$VmNe$ z(krqt@^wW?b#ETj=s3kQsOQzR?g^%sc13~1VE4v1L$s9goz%(pJnx3>lf=PeA5(vXz8@TG{f~O{w`CLc!dId zMZ1cAln)R6qM%Em1yPQbjwoRHSK%ds=SmLF7$SM%{P6B(EoZ%Sn^cYu%cmxq2{mvceVTq@(_RU{CB;cve^ z5Yv47w$^W!ptNLW^O$}QodnM*y#YU;5?`MZxMq~Np5jFT_ZUX`lA*zhx)dgmM|6f3 z4%>FwfMoS`9%!@dt53^yNb=;RuUC=p{Gr!*tUj2zubgh>1sAX9(7f6D;VRnWwx$?y z=v@L|i4s5m%Z2^pU`m+3UYtNF%=@$&sN$G7q8Ty@vJ4)h@?yr$M26=TVBe!#k%l$% zpqjJOlu&24)AuHGIlN7k$&PDxHb;dvj0#*kQ}lWBy^6*oU9dXEiB?i^t9Vzz)W%+7 z2Sn@o3rM|;9Qtp*OWn}$CoXAO66utzHI4SKM;_SzPQp|SXve_naFcVMycJuX14C-Y zYq`-M!Kg-bRClr`Z@#Lu&O3D4yeeYF$S z_(_&b^lu)im!)$XcONyO$0ps!kIJ)N)mU!iMh}Bgvwu0if86JPoF{ic!uwG}j5Iy$X}?6XUzS53m{Ki-10D@fCRc6i2{CL?x4Iupq>nEg;**lSd`-tbj8=xiljp2uQ- zP?afLpy7bT7$~;j_g8En zohdpf8Mll)UpOMcoH0XA3xN$6d(=|Xop^kelK7xy_B5L*6?xOSg zTM{m}Fw~zEGCgKdF^Ni&zOVn~GBP~VF=0lZUCWW{X5uT0_M?n42E^M}agG$x-iqGHo z*;4xW1dJ4k(eBVU$(F62x6xyq&LflZq}18rtN0IInhF@YSWq{AJq?>|5{L_srZ4c?Lr(@B?e%vR21(PABvQjH z2fH!XLsKOJCBC0uQ6x=DL_Blk;&4?X9*@;GI!erdZj~2=1w3U7-qh&$3A-iK#wR{e zep^PsvN_KCRGw;Mz<{_|UuB06A16I3$em-z|Bed0CrR0{`dg4Tz?!=f$Zt^UW_Vtz z$N>Z)5&qCcwzW9h)fMs`=V-`mfR-4k^$_L2=wyXX9GH~Oc72*0p7y@Ya7`DcmN>zC zM2aXgd|W8qJM(*McUWt($W!OuxmoSv%Y4RM+e_l?Fj@2C)oHEKYrlEta>_H}(-eMN z1w7&pzQ5i~06)F6`|?^1VYdXrfRSlV{r=VH2$(IqUS^v0Xvxh2P%W?@e@%I(y3`Goa92bRH3{P+SzB z^K16gVMg*GbhuD?C*l|mzyY`ptsUp9eepCnKOA&t>h z;%&958xaW^5fU>36AqE>%LA;FPl_?lbxaPYjoUgenuTe}LzY zbZ;=&=vJnt5*~)gjcM+E^HxAz*O1*0yzJY#N(}foCrf&^xX)E{9IWu{7SoNgz)nBK z%y-4}$E(o}vOooEwWarRHPbg_m8e)y!Kg;O@T0>V^rOoUf=_t==ie7l*UN4}3CwzZ zMUnvJhhfNoW_WFEU_Jz*Zt?{dG-uhVTubM=z$5H2Znhb{66n;DZT&uD+a&WWbbss{ z#~QmTm5XQiL?#Cswn3!O4z{(do^mNcGVly#WX7QSBWAtvFwqM-|Q44xGr&0j^WN@1Z7V!oVc&0E};gN6AD_!t0eQxyeEwxTuEE z^K4ke8!Q0yPhZm&c(0pC9k5nfCM>Tm1jw^DRSq Date: Sat, 29 Mar 2025 01:02:16 +0000 Subject: [PATCH 372/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10229 -> 10228 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d3c619a45..4a9d37b02 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 00:46:02,490 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 00:46:02,490 submission_checker1.py:3260 INFO] --- -[2025-03-29 00:46:02,490 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 01:02:10,585 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 01:02:10,585 submission_checker1.py:3260 INFO] --- +[2025-03-29 01:02:10,585 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 00:46:02,490 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 01:02:10,585 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 00:46:02,490 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 01:02:10,585 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 00:46:02,490 submission_checker1.py:3276 INFO] --- -[2025-03-29 00:46:02,490 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 00:46:02,490 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 00:46:02,490 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 00:46:02,490 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 00:46:02,490 submission_checker1.py:3297 INFO] --- -[2025-03-29 00:46:02,490 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 00:46:03,598 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 01:02:10,585 submission_checker1.py:3276 INFO] --- +[2025-03-29 01:02:10,585 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 01:02:10,585 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 01:02:10,585 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 01:02:10,585 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 01:02:10,585 submission_checker1.py:3297 INFO] --- +[2025-03-29 01:02:10,585 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 01:02:11,622 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74879 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420539 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.411932 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1048 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.079 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a6905f8ba..83b33d238 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -999 +1000 diff --git a/summary.csv b/summary.csv index e5089e69f..1b28344f2 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74879,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420539,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.411932,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1048,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.079,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3eb5e5850..49d8bb601 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420539, + "Result":0.411932, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1048, - "Accuracy":"acc: 75.400", + "Result":10.079, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 4ae58815c61bbfed43be983c6edf5d9b9fddee00..8536e4cef2be68d9c1eaa2dafaab889d71619d49 100644 GIT binary patch delta 2502 zcmV;%2|4!lPxMc)%>xP)Uq_*E1ONbe5|e%i8I$k>Cx6fYW^B{TzaJZtPIG*k8SP7@ zeM!K4-?`z>9DDfnCC`i}!gxxHk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX z@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8uxLOzKp%L5^A*~;Q2BSBP?^2~B=+q3c@ zEzHKkfPejo4I0O3NRDY(<)jde5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASBa2D zdWsU#jMu+Lp32g?rjZ9PBeNqtq>)IJVIxVSh%*$m+pyw-=8w%IXKkRNI^LS=*v=dK zuN2eB>154kgv+lvw^qp_2vu9r zl3he&suh-1t9%udg~{T{ZF_CIwf_1qLR!S>%KTzRL&&Wykb3}$# z|9ueGtciciX+Fs9PT#lRt$JIc>*J#9>!JhG@JNSr;3|(mpy&_?6deMAqC+51bp0Ko zL+DU+JLs2vC^|SW)g5S}gPa?k1G2tFtb+31_5X*W`;46aKy(lNEz#Ld*Y^%T_|*Bh z>U>>wpg}m&Asw*LP7nxG9Rh)>Lm*If2n2kY&-{NKszc}?+lw9aOCPEZ4or0iUDzo> z&W+9i3&|~F6}CnG=c=w+R)wfqawDxwy!hV70_^@50e(h$e;~lQ|4x98RA3+OY^--% z>71|?nUE~wMo4A78sgh%bKEd<2J>bs{tK@5@$% z1Ai&yH8rQs;nX=uow(Lsqr?$c`Uy&aIwFzO5s9RZNW>NT-YKAtgo|I^9}B|U819Mi z)9r9^mV69viSP51BatLAqRY=HQKrzGITa!~9Nlz#+5OC3di%0UwKq(GoPSTks5iQ~ z)pfgW?)4T*w@RQ`MJXjvymGcsY`S&N^?xi*qRUBikwkF;y+(;6FgrmBkVqtwL?V$S z5{V?yEg+GEOA-sh+ZYZcx&v2}%G(B9SN(i9``4p6KYkSAZf3mp45H;cZWVaw;2q(tm+m ziprP{#wb&+MTth%y;x_H}!gz1eU1qp*?Vr6at)jdV2%dXe2zqt9=`|g1^M|vj zOb+9+UC<~Xc}+BqpyY&TfJdg0Kr)Ral4v}U=lw>*m}Chgltl^Jn}ZuHp~WYiC{xaZ z5Y{1K@~PuzmRdd>I@>DXdBFKFD}QES_Jd%aUXW27+}x_Vn@)^BjAtbkmf-VpJ+Kv^ zPTh7|DrdE-_;dGTh0ob7OJa!{$Cg-<=k+|RLWNhWF>N0NyeB$H@>OrilYQG;aaA~Hz=AhRGrdvoxCOxfa-P6(MYtV6=! zxhhL7w@S{o3i$VeSw)Ry9<{Cqwz;jenwG6{$mIK^#|pO}Ek}|`wkTamCTfsOq5(3A z2FOGWlG!XGlOzB#3lg+92Q44Slr28#M44KWDZ@H64A3MZoHLvMC^XfRR`&M^RP3n%Y)sIZgdr3fxxUQd%{*tH3=4E**J?V<}DzuIOu6(burl*BV@* z7c2B)g#A*b*5*3vlC?b zy6sD!yD#G&(TVNtO_Gl8IF7qF?h)PgOfe5$Os>aynZsMmO_%26V*a z>3P1}H~W6y^Bl4Fd^&zM6EUiK#CAQKxTf9rqfbB_xVK+$?+B zlVKtif1Yi1m;Hhd%9;kzf}jkHuu_?IgWi|({2pZ?awBF7&f3=D`2Uo2}adK~inv8wl7kxEaQ+|p+ z7cXz)s<|@5sRW?5mVyZnJX+t@_`FDm>v)KZbmEl)`M68^mAAtp`#z9a3c&VpkIF~& z^t_y-wj||kPO7|oSQ5%e$te9AMqc!vfch}x-^3$epZoF?w0}@b0|XQR000O8002J# zvvnhO0tpmfN1<>8004QD^d&tTbr?9K?+O3_8B_oO7ytkO0000000000fPrO`OeRGE zij$HiA{_Yf)3$m9001Zz000yK0000000000004l26cv*(6&91`CX5B0mrm{^lXoXn Q0cMlTCnE+JB>(^b0MLGyIRF3v delta 2508 zcmV;-2{ZQePxVi*%>xRxIH3=51ONbf5|e!h8I$k>Cx5H~%wW^YzaJZtPIG*k8SP7@ zeM!K4-?`z>K6v=`B}fwbPvK7_RxTkegccl@LIH*!FW6$S=t`Z@Q z^b{pzDX)KxjLOowrjZ3NBeNqt&`2c8u#wOxA~SjIHmtZ{*<0|58)fZ*7qdtO9sz$96-l*hhb*(QRQ_hN2as0< zDdPprV^YdahAaf(V!^9rM!9MV_~NP8wk=T-Qnw$L>N2eB>153>375M$w^qp_2vu9r zl3he&sudPht9TWx43ou^+xFUaYrXp~LYBvLWqvUuA_AIn^I&Mj$mVwc89B**j>ypJ zzYpS?HF1AA&4%3W^nLr?s<$P&J}$bxE;=v`k90@}uJQ;3iVlH5(IF5hIs^hm*WV#J zgbqcwgMQhEqJsld-GL@L$hpxuAnRMiDk$$=|9>dD&&cTyMEB6&5}oaIeedvtt41fHfeI^#A_ix>B&nW-;v7IG@_TdIgK!ilCqMD*T%a#9jVe_oUv%X4ywxOkBaQ? zg?rv$FZ2$+818{FGGyi(sLZbRQXdx}VU34Hup-w%xuQ8YQWDEV?Y3O2%I18dz80*g zuVk~nWJ0iPbC-}HBBi>MU$-%1VsoQP@^%(%H2w)tO9KQH000080000#03Vk-L9=%V zVgr9^(`x4Yns+#L4pJwswbv+d#Fc)65}=MqBy~h0sUs3`g}!$Rs3YOxm-okl@HU2f zBK&kaT%08z!&~C}{NzX^NsQ?7GfI>xG-pnQNDfCg-ClM-vzOk!tWxa_Qy}NxlQ8Oy zZf-E)6Ei<9Vb5?v%wTtKf;;t0%6Py!?pi6oIo zB#A^KNpuTHB;k_8g77wm1Bq?{i5Lzfx{^c^BPP_(C{d=+oVouV600bs1c}$~&ZO9E zX{VL56!kcY9-=59dW{lCKy-oR2A_X) zAeW*trh_rclxtC!*Lub=R@; z$mE+(#|rmzEL)O^8pn}jq6Wz%8X%KsfK1dNnYxHfk^smoNYLIKd>~V{_@om;rVQ(l zFnF%YQp>H9v#kRDyRW0^;->w#@ zOrilYQG;YQi^wDifXspf?aeL02Qp=gPdZVimSoDX4h;h|i3sP+=06He^`zBo6*ZQc zGn?CUrd@B^w$34wvw;qw1_kgz8Pa*PCl#dve`QLh5?BUO)|9~GjCL4SQ1XrfS{MAE z4p_W~B7L+lj-sl5HMOnOa+>{nz20k*Z7Gmah7(WWPT}Awa%KI?e2jckof0h0k-SJEPD2&IA>G6yGD2&Gy`HAJ;!N#MS z2zTV?kt5j<`EMs`mUWCv`wbIyOiLX-^|hhvHG3CLm5C$98HCdB2Y*e+(=&7d=i(*- z&2R6KW%`o9z2uU$KX(tb>mCR8qNe&@TY_aT&kY|n4PUZHsL2k6#g{Fr>rBsPe<#TD zb=#LdcVEUmq7&QOntp$<#e`OjBe)j4Csi- z)AM|{Z}$DZ=Q(2U`E>kjCSp|ei0yhdaZS7L$rH(L2o!}V4}s_6#9qhT_#{+p@oni! zp1jq2_PrU5@!@|^O9KQH00008v&k7)2n9qQx5FiqCL$^kCxHsAB$WX5L?tAOK-?^Q zo0DN86@QvW&E49fD1xqSV3Td1%?M;Y9}H( zAO)Y0_Q!Q2KyVBlDA2*Kr+rV^ZT2P_!ZRQSAzhwGRR%Xs3o1o60w`jxlCBKtlNNen>uagWMJ z`SiS6pr#@6aWAK2mk;8 zKeKZqcLE8uIH3=51ONbflk+7#93PiELGKCx02x#O02lxO00000000000DysJ2$O*b z8k1%wDgld=ktQM>@Ie{Uc?19eC=~zz6aWAK0000000000fPobilQ0z)ljSCi1w Date: Sat, 29 Mar 2025 02:33:55 +0000 Subject: [PATCH 373/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10228 -> 10228 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4a9d37b02..34105f71d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 01:02:10,585 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 01:02:10,585 submission_checker1.py:3260 INFO] --- -[2025-03-29 01:02:10,585 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 02:33:50,531 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 02:33:50,531 submission_checker1.py:3260 INFO] --- +[2025-03-29 02:33:50,531 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 01:02:10,585 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 02:33:50,531 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 01:02:10,585 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 02:33:50,531 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 01:02:10,585 submission_checker1.py:3276 INFO] --- -[2025-03-29 01:02:10,585 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 01:02:10,585 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 01:02:10,585 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 01:02:10,585 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 01:02:10,585 submission_checker1.py:3297 INFO] --- -[2025-03-29 01:02:10,585 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 01:02:11,622 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 02:33:50,531 submission_checker1.py:3276 INFO] --- +[2025-03-29 02:33:50,531 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 02:33:50,531 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 02:33:50,532 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 02:33:50,532 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 02:33:50,532 submission_checker1.py:3297 INFO] --- +[2025-03-29 02:33:50,532 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 02:33:51,490 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43633 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0568 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9043 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 83b33d238..dd1172404 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1000 +1001 diff --git a/summary.csv b/summary.csv index 1b28344f2..c7ce3ff11 100644 --- a/summary.csv +++ b/summary.csv @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43633,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0568,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9043,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 49d8bb601..4377215df 100644 --- a/summary.json +++ b/summary.json @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0568, + "Result":22.9043, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8536e4cef2be68d9c1eaa2dafaab889d71619d49..a5cdc5dd0ff4929efaa3662b49132d55512a7713 100644 GIT binary patch delta 2982 zcmY+GhdUek9>!y@Y9hLh6^#T{?LlJ`D~MUNbttOT9<9-|)JP~5t0b|~#)@6iB1Ww$ zL21SAu$3y|pjx%{-1Of2JomZ3Kj8EGzTfwKpXa-EdFyg@C_hh@epc8zDU4qYdlxgF z7eYBN+qD_!m*>pgRl*CB5qPt=W6GK0_5$v>*_Wad`aON_!$eDjP}F+X17p7S87sM- zKv4HW&^?DM7`hiX{l+KA;G=~4%6Io8ZZPqr^{Uz2IQm@mwl3bkUFv9Q&gsmNNLsA4 z@(4G5w_44dxEEa{vSZRwC;pLv4Og;AJJ#tlVKo9Na+Aq3HE3)<_QYCj0+HKOr=9oS z{*Ub<70u3$n^QtFac$j*9C#(nR*5Cc`nE6}CiU_0@%@D@2m3S;o`;NynhCu$}Y53-d6_uLiB+4+8duZynz# zTnVHeHP>32PLA(vMuDqtiB;anmWdp-YBuGw-DTplbi01s-e#6Z(MCr{-d%6^V2*zh zTYdE7fLDelqaD#`%6!;^4a^Foy-YuwUHjQw&H0WhVq`Obw`8vXkt zQ(XGsgqJPE^U%$E@Q@2V2;8%J^IE`bQF5sdRwBg%cvd`2X3 zj-^ZB&qIO|U&bQAi5fpIKt*O}&zaY0v;;y$ymv$ic>3g7aPU1n(L{|FZxnQ3{WLUq zmuKml&_E1#L+hN){9B>pa4Xvlaj3}8S8%ZHhIYp8{8W2;cDgGbf(Fiu?KyyEn)UAK z8RWiwr1nSt@*I|w!n&D42}Ty7$eaXvLJaUR@-TXPyY;9hbU&o&Ow#E4p&&}iDc9Au zpotvok@hl{x!e%~NbO*Tsqs_qww;=S8BnIWl3$DlQ!Yf!KXq!ktwxmR&q}T`X~^=6 z^vkbWD3Z3$Rtmn=mNh=<@ruP(|wCWc1G#8p3QY~m%XD}A$+AB0M7t02WSGwxA#o*6hj$F3k z91}?_evF1;P|)W?JGVJ7Fz3VXzt~zkgIweCZzm+v$9jS+BJ=Z|6Jd!WF3~25D)C?Y zX`sH(*yYm6ckABUBa_+Jp0&9NemRT)o3srxY0tGWp)VS3CKO`aPnKlpZ@!&2tYN*6 z8f?@|l#&}Z0(bLlu|I$PH#b}#N0xVYmPu9?E%w*}yRjwteLNMntZMGeF)UyXFoqK3**p>W ztL5c3R~%FzTV0pu^95I2h=TY;F;~-xOl8Rw+fwIbaB8cm?vKjuz=^f^&6(ns{-7mI zy$F}<#^G>~&Q2p}Yt?%6PV=v?KyO>JM2}}nfvx`cCqB*weFDRPG)tKn4>2ezf$K7`Q;r((S!yg4`TH;=4&#$UQ6clQtr5(V$#xR;;w@EM zT@moGx4lIshw3WTNAElDILoD3zS_H66{Pp(9*kTb>|!SCmu^YrP)X$NMF|rtl6zqS zxc*pf8nc5>#gnbYus6d^63jbz&C##vT1dmS*X^hK75o%`lw)+OvCP1d={m_wBiUIw zUHg|uJ}pGcT4R=(itR%WuyDE=LZ1!q_Yxt)IMN-{3+gp@^-BYMN!A$^aI#935d&Za z?bM|2r5Rq)^;LB;PA&P`|8<+f8hiSfL>Hp&zWP@}Tzogp3>tt{1P|2x0=}-I_Sl7%)?f0zZr#ofSF`)ogVxrh! zh6k0m?>Ec`XE`>v0BONn5&^x6=%ERTNa-sLb!KwaLY%ger+99vc!O}hdj3k?dA(_u z9F4q=(z>VUD~IZ}j~R;RUgS7sZ* zwG1VESJ=$v2YB8J#m}p5Vg8c{4=#J^O z&wum!@SIb2<_Slyvz3k0NS5ITdaDlN+VLmL;=#p}2|hZcS&#&lU7olCf!h!;?|H{8 zG5gwPclzM0(br3u1Lx(y#>>r50<~V{{`g|LDuEc*oJY3+hr>qJjSzS*ffnE>;7J!v zWEdJ)~HB5O5q>AaD;dB5>P0?)17d=sA*CFML>Imx#qS5z)&M57J&m^=SjWyIx7D|U zbR~)^d796Q$=(?jNFE&92rXI4n(uA0jQe*hQ@J zPCi5`vMcyyXLxQWhOL>~nFsXV%@%<5`lBW7`KR?CXLA0jq!keN&6Us5kQ4VqW@trg z67s`p^XN+8@b1Wp5rW@Sppq>)!$U{O<&w{*cmZB01_N z+^J@KNm-F(U{d^GM0DKT?|eOO`rdxA`oOEQHVJ611hh~5{%=PkSvp&1#bC8@qY1t^ zw2G8;E!$Qt4rnq3DMmpKjmYN8k7l7_N7oWdFcO8ms&~cqHvBzx{GMnaUS>02kfrNLZz_p{1{3gkH}x*XQEyC#>rbtUBP2BqmzxwDjq_)#pzuu{;|ED3638Cv*6Fe<=) zlPif*mNaUAzI;eT;`)dC@Jn&NLU-)Nt8d_vInkP(mhsy&;;}WiBGSj;9fh$kNA2w( z=9Ac)M&s&%fF<7tY4EVO6=l2JDNGcKi%1vnjSiUP(qIh5axY2_5RbXGN>$yZ3%7+V zW^_V?`>&e@hd?MfDJ~!3F2hbnzm8#PuCoW%ZWUUWx#L!dmpT<~UIe<#uRkkB*F4DuI$Kh2rxn+A$>tqe=RI%nc!P?%2GJ^Wi=>myI zhfg&)2b%8a865G^y=2~U{`am{m387sBoSnFI7LsAlun)^Ey^lzw39$`S{%wG3prWw YfA5w90I=Z%07L-)d$<@dDMilbUlZM>!2kdN delta 2966 zcmZ9Oc{CJy8^>qJGGv<~jh(S&NgCTU1{pF#wn|1wvTt)SN|pxW-ZA!>i7XRdYq==K zF6kO&$-a!NQ%Kw_6+%+))P4VW&;6a}uirV}@9*<`pYxpa+qT=bdlJsi6Cr6>vPp~N z*Cz}Qct`hz);KGq<1TG2Davc>#Yv(K2Rlr_9!I0KdXw#ZmU3yNBTcONxoa!oP2(Ey zy`2#FnGm9yEQ!I)tIXF^%Gv4s9;uJ0a&vn&6CP+u$NL^`KJ<~_U3CAbvZhu3mnK%3oPah!ljJg9ZZ+S+56e|_*F~kONlLTQC%*UM zrfc-lg|lO?XAp6pSiVXc?*b*iQw~4Y5}ePT_2;hN%w`&v zH~9vb+^*3A1^ZqNvw&{YEah1}P4lXlRXN=R0^L(PA?`^(k#R~}?ux5%WwW;9+U4nm ztVVhF6pX`LGKO=DtZY92`!~mwu(itpBbW@WZ?-S=zU5hU?cO~PT}b3_R{#Uot||J{ zY6Zjz!HQyRwmPQXunkE`VjPgIx~A4e$%bu(WE6ewtfH9jt{~<0w*s zp-Vg@YWc%mAxcsd8%%zXBwKE=C(VhJP=ifjvx5Yw-MyP$HEY9T{prbLUAd6 z1X5~#?K|ykP0B>`cmUY}r~Xb)uVV1+xRKR^W|n(Nm)O;>AWR%wkuokuE={-U68}1{ zUX-4ga(!M-uyaOogZy!9{kPEUn4>uI<`zwf0zFgw>SwpZbR%&^Bk4JO^weR$d(=T4 zroq4_o=&jqd}z?xG7-@hvO(C9z%*fsoGsLq?VpA<+6T2p4DD#z7QaH{DX36jyCqpk z=bRZR(+WyODN>+dGIT^so|0${{`s&lw3P?yX_&c!0i#oM>`h8gx#z=|zeihD72g9x z-P_L5JQ-yVoCY1o7uBMYisM%FFw(#g5C&kONKpfmQDVvkgq4-{t8t2;`v%55Z-edP z!cIxWjK;er_vHw`RJ&ATLAIPPDY!GSpuhg5GZHuI=F=R$Uf8Z(UHlZxUl4tbKPsYa zRZj-DMPfX(5nilw8&;FQa}kzaw3O%T^b~c5?Bg1K0jLY&2~Xb!)H#tx$}HqN5jYC3(=#%J<~yYcDAxL-!|1G)mK?O z`^{FE@e}bP+qqjVr+gbK_riB?{ysFNW$j6ZbZ0wL-|xgJj{ve!+|I?kK7DC_c%s38 zY#z`wW4KqW(7cY*ph-^a0u|k;=Ekwj zv?(S4b;-OvC?V>UrXmdr&=myN8}P~X+|xp?qHP$H7=rB}~em4Jy8f=2*#Nc2C z3r!lMDPW{_7F_U8nJ^J2;3VGkp`T->5 z{A0M@pQ}|yua-%1(+!VN6G-F9R_#Tl1!8`73l_>m!%S5IK7Pu$f>J>PydG`c#2l)ZA>rH0nMl%CNY;{kS)Z-jjo zP1AUQKoM=SKYrE`(m{WNjhYuFuOX_AdDQg`mz1)XkGsU^V_NKDK@Kuti%h8EQ%2S6 zHU@F&dMycyjxT#S+qk{rGglvEK*DeepZI!o+#FVe-w_pZ#CA?HVW!HTSb!`GmMjZo zkM-Jc7>y1W&_dD#MWqP^*q^EaJOlQ6LK*Q0EX4z~A8xA0RpVo2>%yf0!6xGb?B zb6hgEh_ei$7l1jjR+4amoEbi89`Tu2(N~2y^~mzNcPsJk)pRb4dh8CtX`M$IhsABA88X=J<_DDP-V`HySTg?k7amv3}w{qmkzwbBx~vghDhD5>ozIK z-C|qe&l319;yIaZax&XK5EUxMc5X6ZaVoVR0C^U7Wea01h$|JRXGX+Jlwkb!0~Te#f-HsYj2G(?sys32e&K z{-fBpgQkB>8Cbz^J=P^mcyQl#0}2EH?708{F#rJIg}Z4VhWjo2^i5otw{~Q3&;#m# zEmKD{aT8&&6zzFVjK2z+1=v9;Lz7KsHKS)-x=iYX{MoXq!DCK;lNY0WIv%*S6a+`y zKTHtm%i|qMC_0+@#4$I5IDdcZo2<7&n9?0zA($T@a9}#$#ZZM%MY@WU*?TDO1@huL zR%4bgDZ?~O67rKxUeZCkVjthJDZ75N`-Q=EZf#Ht6QXz@5rzPS8;Fen)n6pyO`b^s z9X(b|iDckd!UD}8Ve)MN^C8`+A^9BN)#?kOVK`!5YApLite(rIyU8XePU99UTXdO^ zjubU!CxAo{F6;}ey{A^B50ObJ_=PO4{buX0SxrpE17s#v;Q5!(jJ^vy+fV5wjnH?& zwv|s~oWXroy=sRLOZV-@Fpb!h_Gp>In=Vr9Q6zeWvwPt^t#26cq&UIBihH f4EUTTstO0{(yUcgr2ZW@C-Gk=F8&xTRn_mm^apRQ diff --git a/summary_results.json b/summary_results.json index 4548aaa70..3adb38570 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0568, + "Performance_Result": 22.9043, "Performance_Units": "Samples/s" }, { From de696c63a05a04bf7468e66839965afb799cd221 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 29 Mar 2025 02:38:29 +0000 Subject: [PATCH 374/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10228 -> 10230 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 34105f71d..aac4e8990 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 02:33:50,531 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 02:33:50,531 submission_checker1.py:3260 INFO] --- -[2025-03-29 02:33:50,531 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 02:38:24,642 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 02:38:24,642 submission_checker1.py:3260 INFO] --- +[2025-03-29 02:38:24,642 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 02:33:50,531 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 02:38:24,642 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 02:33:50,531 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 02:38:24,642 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 02:33:50,531 submission_checker1.py:3276 INFO] --- -[2025-03-29 02:33:50,531 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 02:33:50,531 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 02:33:50,532 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 02:33:50,532 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 02:33:50,532 submission_checker1.py:3297 INFO] --- -[2025-03-29 02:33:50,532 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 02:33:51,490 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 02:38:24,642 submission_checker1.py:3276 INFO] --- +[2025-03-29 02:38:24,643 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 02:38:24,643 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 02:38:24,643 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 02:38:24,643 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 02:38:24,643 submission_checker1.py:3297 INFO] --- +[2025-03-29 02:38:24,643 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 02:38:25,600 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74879 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.411932 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420968 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.079 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.6324 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index dd1172404..7d802a3e7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1001 +1002 diff --git a/summary.csv b/summary.csv index c7ce3ff11..5cc606e04 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74879,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.411932,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420968,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.079,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.6324,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4377215df..db502e435 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.411932, + "Result":0.420968, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.079, - "Accuracy":"acc: 75.200", + "Result":10.6324, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index a5cdc5dd0ff4929efaa3662b49132d55512a7713..350c032e455e0346258e5392cc8fdae403b842dd 100644 GIT binary patch delta 4154 zcmX|^c{r5a|Hp?JjGeKQZDwpmJhmchMhrsPw=9u;2@x7&>?6yV$@bVsWQ~jnQAC#P zONB_XWZ#V?^_{wYzvujO?(4eG^*-lyzd!H$`kVukBa^y8AOm!7Whf8?0E9ybC@>#k z2S|sW@mo;kL}@y*>!LhLG4@e7sFPwTc}&2~_(6fG{)IYmtip@&4`0*rwdfi{bG+_c z%;(Y1dwC>t=P32e(Q&NnA(ZBsv_Q7;xH2<7PMSF7;uKj}p!{fj9@o^PTn-AMbAyU+oaBvkqi^fO z1a;|cI;$LErS%QZJ)d@bc*d7cx{ybV;Kz9kVf(gH6Jp9rvF4+{S&z-+8lrhZSG|Nbd|5dJM`{20kvOF6QVDx~rOWb(j9P!@ z`%tYXszgTOPnb&!;|ne~fwcOXT?e+()E(_7XoS&&E>R&jx|BUTy;`w{OL}0?L`1BX zR(%f`G%&{jq89%oMNLIoXTm(3!3BY+=YG*pQSX*=_1QdxS*JTxAxj>PVZ;WS*{5c{0oL-jYQGb#A$Qbz=s>ZE?To+b~o`G_nuCf+|p#U;U14W z=}t}rdvp%Qv~$67XU^Rh;b$sPovL4RSnu0COA`Wtq{+k06bpYtbg+Ycu!BRe6Tc7z z)P9K_%qd8XA?k>j27^IK&s%kZ!GcMCbm#Sqx^#5!TYxu%aX}?PS|6JdEllTHw|<)j z7p9NXA;=@Mko?UC)b>t(%}|fU%6cV`xig)`6B8E_t5vkd1~#XtR_cg2xohl)ma`Pc z_{dVe8me3seO2zGV1x5BkjfQRI%zmX_A~cMVO2}uoqI%|URF=Hg>Ld3kJFhy=$LO6 zr4?m{)`?1;J+Kux_ma`{{%)r#)*YR!j}L&k);lTf`!;C!XJ2jeJeT<;@LXe=lvtS| zqrXM~YLntR&>!_U@Ax15nC`dcHR+G#T~Nv@JXl1FWo_#(dGU`L+85s74b!&K?)Khv z$WMKOlryqvaHO%gci@gZB6q6SHt~ehIWm#o^76{>E1BQ33adzE#MQ;0t6AgLi>ybI z1?aJE#1OOR7P_eRTMmqwR#x>kg$kn9E$w!A5olZs!LjL|yDq2_gS`+R#h)~$hCmP} z0KfzQ0{{R`KwE=e?l$Qm*cph^j{q>oxc!y9rvvrtU1fV<6e>SE%R7Ivl)Vfz*>GLL);h!bL>R%??!JnWz7z6 zls3s$nMb_i>^&jFlfz4L?Hoeamhbo%H&{@2Uo?=yEM&CCxr|qx75JlrAnL}%1b7nT z56b44DSCq_1T-BE!NSw^;n+J=Shzjjnr*H#KxFii2>Uw>!UV7H(iMBkC)UsTufq0M z9T%?BCnCGkGdY~OnIScKrRBr74%l_6M!lcBq9$@Y0VE-ywhJqrGS{jn}3`8Do z`HTMelwbBW_S8mMR}ndZLa}u$^aK+@P13~Pw~;AJs$t+zg3Ui@AW0ase-NF#Fu!a$*%s{b z(^>6OMIERxYaLSqc(9)8U6D6gxiCDN#--ujg)rl={BSd#bR-0jLql(y8^h|_ zPl1WWNxf5rGZj?TPXtyi`cy@sWwR|Nte0=ChdbrY7_@p`yO?0qZW3M{a5fqumgD+( zjNnS7?Su1+5Imy(Xb(c=k<=n25pD0F9W=&hYD_#|(#%la%{n2T>R&``(7k=(xU%pu zuf`;P!@-=vO_lMAD}s6}xIXe>8kb+zOaYuj>A7qlf*Frg)Dj;BP__{6ZAj{c(?<h zqYytVqloJN5u=b3X@!4()^){$wC-s~e_Wmi=G2$oRK+;3C9c&;0thNG0HZUT7^x#f zWM1FJ{!86Y+gei(WfZotKB;DU#q?e2mV(6Hr&b9bY<&>gmwCE#!UL$_ zHC=jJ2DNJ=f2c`P=p4{j7G|I%R$p~<$UI{%pR>RFQ}AW`>IzPTc+>b&aLyp1zjj?h z?!?GfZN+P*r)2HO!g1*@w%}E_aeK&eXie^Ta9nhTHCc;`m%PqZG54k-k)zB~{i80vO!AaV`hVZan zzH_N@_vfLhhFQ>X-1Gyob^ptn*FANSEEygb?mvB#;GyQZBg+4@KDIl-Gs=K?Uo1oS zvT!D#TpGQ1A}M#O^~bIYG)_mN&Oaob{DcEI&3(w)(&+Zm`5fEG%ae^OJr&+=Gb*G~ zWtA-co3Pa2Q??X|d<_K9UOr_J?+MQ`f;CU*8VhG?1=>6*D(zSWQc^(gXMdeiQ++3R ztP^|HrN&b=+GUfemg=Xy@{BfZn@32V`U3RQQgKD#2;yrZY*qvD9ofV>D)O(`7Y!qz z2ZXuEX7R|G9xw=PN~3LJZD>GBfNlny;pD9Q>L=>(lU%u|UqbSQ6*DW>{aaCElRo3S zn|TtNwkt*0A$GlKRN{L=h%ixG(eN`Wn|1{5Cm5F2JZh?ll!&^QKI|N`ZeX0w@J&5-}J7mm-#RMyaJgW(B{a8r5>D7$j>zp zF&>#aXL8I%$PHDB+}DH$)E?X?D8*w`5v5c`J)*xR#D1&p+t85nw)7#NMD#u?i!5Yp z+a%iz{X6*jusTCW+*a?xDj6-xZ*q!Cr0tXbMt1&_P$zY}MY1y`NzgZGv~T`G&tHrx zXL(F5_u(@txI3h<=j~5NUz&^wt0bi7;b=Qy-%rqYMGM==u4hX@&a$K+OBz~Jkc;5o zF{|~xYm`cqd|a);U#;=C=*RKq(Vr+$&m7nQUP4bL4Qix0=P=1bCkX|;mbdLHXf_(h zCED&EspszpvTZK1A?IGfC{}+rd`9smT`Z5{4FbOlNT=BSk)F+F@nCfR0Q2`J+X#o` zX(%!!x*7|M?5U`r_OJ(qh{d486AwMMirl@-`WJ`G?>yOeT6!rMm<3$SL~ zT3)^Kgv(ChG&E+i6_ygF$Wn5D4>cKawAbJLG(wU2Og1aVcHlkwa&(q#xNP1zkNV1_ zY<#5K1%A&}@>oNQo9q;wa&?JTOjs!IqqQ6FcBPfYJ49qAs#}#{wrweaCo1$SA52!a z8)}{LvEyfjH?*6C5mP3s4jm#)gfroKv-+^Al~FVm3Cod9MKdp&ED%N#=PIE_Gu$wf z&v+}3t1klyxi`G4i^hj?j>$8({W4N)~6T;VUS%Ub^EW@xi6=DBnLiMvA_G2{%#UYAWjxc(|W z1n4uJlPTW8VneIDJ3-!)p2;M>KzB*$-!#dnbYODWd?eGfU)0$p&xt&De&b)xK%-|6 zlehVXWjop2VXZNE(qHWCT#A8;!EyxO5g7BpBr=7wX5sgX&A!Uj#ES338D#NJvfJD> zzWY7E-xZK+^l2A&cdW)E`XbLJJpS3edHS@LJ06w)NDjXjI^}LTmoTNl)`K3dJ^1t^ z)p1SFSCbK9DO(I^wG}A8h#rLAG-6B=rwV(FSF28HL_M@Vf*`ao- z)uYnr!aqYIPa-vsy;TbeiW#r+T%q@fj7XCn&jM|UWmTUyM-(}buh#J>O_HY}(h*^P z$K{^Kc=hWyLnNi!eQr6phohCRX=m2Xv>!B|$#1def1aCzul`8={wm_T(5UY)=l*>6 zVtANJGj}2k1y8&DjI`tutWfK&rDs%7u+oYmZIji9*&~LZeRo%2FR9e?`9#8cPoqo< z^h0!tBoSd?H6Dp0)#djR#V(Ki4XySTYvxGXgHFk;zk&bP;*s9-*@8jfE{-b%I8+9W zjN8<*tM@80&C1-0RYW3c=eF3~tCIsHss< zaR^tq|Hoa3?vPoxXBC%uIbDL~p+g^&^K@s`ByBa%JP~j^RnDxqjU4Z^@mUD|02xp& z;`cIRB4KQ+u6QGN3$WrDo+kFNzXJX27@Rz4gy6uF_&g(ht^uyT8;J%j2@c_z3fqLL zgRcnCE_F)9Z8LvqDCn9(1-K}OIAjHPJ29Uq{A>;j6R=E)Q!PlMsx%w|leF3zA7Eu4 z2fWHueM2uX$1tCU8u5R9^KmljZ3$ac@_mzjz1@Q^)wlAWoj{ErUqyS2<#F;D;$_=Y zVYR2jS}M#ZS9U?XL5}F?Q&!SFSS&UDe;0JZzL@%jzfCa%0KjJs0Ps@U5ii_@% delta 4150 zcmZWsXE+;P`%Z{mL`qvTgcz;1s9jPch*7JkS*2Ets*xH+V(*nGqH3f@Yi~u3DvH{p zekfXd)M%+u>y7ule*Z83^WmIxUFSOYdG2SNbDw>^J-v#4N_tjnjofD-0KgAIz|yi4 zb|`7E7Vd9kS%c=RgY}{2Xvi&1+IW0i!CLd$&HksgdMv=pLjVWY(J!q~PQ0#HKnDA! zc7C8{Tfkhyk3p^%hkxLMaYF6BTCvG%HEyH4YN$6grty8PD}^`bB;V7<2zNUvt1sX9 zu{`n1`jcaC%WfFh21Q8)B}Rjn+NylhlN7#V-OTYSh-i^7lzi@!z4LH>Wh27xK2Nvl z>fgLK`?}h3r_qcYpV6oUwCPeFIG#nlN~wFt51;yti}{Ta;=$3ixDNi^S9j7C{uW5_ zd@VcI?74&Waf^CmD7BP zjZRYckAaNDO|+_K^^P$o(DSp-Qtx2ZVKRdvj#6TYT}bnW+5k__TC5|w|*@h1@MT6ADpy(G0B2~#a<0a zgYEd_HC7*USpCegewyX>b@zs>*f_Vha%mag3iS_MLkMqMT+=bza|Oit!IqtL^ZSOW z@@dem<1pjvc;`tb^wTBcNk7(}nYUc4eO}GHBdkX^LKO7F_G7VqmghL@PUnMiH;WJ! z!+L1yxgP)REwH4G1=7X_iE?xY z73ux`dZJR!Dwvj)nU z(s5konUXj2>CNFu-H8LHdxv93@7DvYFgePyWY<*;>4IsE$Z$nuI1DW)fH-)TXD z?u<~%d_M6%YCKbo*%0P(_gk6 z_eZAFjEkCW=oe33K-8Ou`%BC2V!X=h^{1`}+p^`SDE;c4QK?)T4;-w=%0=@C4XZ*r zsrJZ+xBUdcm0-yzNHh|U62K$NaY*tNDN5^ZBb&6!mK60vi6o>cAa1L28V9kb?34>crE(A5Fd$KfKiG~#Q(*WIHh3H#A>IQ)3CBWe zq-kVKRkUAulXT>mRZJtpdI}S=&76;(t-2QDR)8y?O46~xlguH0{>the&%BD)AIXcK ziDkVsG7TC0@~6!N*NFLCYc$|&{A*?FGktdTbX^BTs*Jx`0)Buz;rGE|=f z3sC@=WR{duD|e_=;EE>bro4j_B}!C30q(=v<*JVoRURm!r7YCq^OyUU_cPbVUL_E! z8AyjMXO-=np3?+7QWy3QocLDx!XyC>UV`f{N}|dCL#4gIY=1T&@XlkP=8Uop|ST|#5%LO6YrR*`Q@kA_)Ju`*e@$4_j{}hO9so16sbyP z?X80uVP@^6pCoYtzdoTt)2~bXtkBJSQYfsf9qdSBK9zUbUK&pLKAmd=fU{^8&Jc@6 zng}WlhZzImu~IKXx)+}}X5OmyGd#kaf&{bQ#jjYp9s3&1)GEj%_F%c8iAQY$jY?eT zMLrHadA}%XqUet$8;-~LLv&uG65FJosZg6;a;}DEYrjZM&7|RujP`#P8l ziRVw)I{?QEF(QzmMyVBtVvPEll4W^S-AkV!fR}0rD3vHru6G^l#spUk-=I5^b+J&=ue{MnC!BUcvZX})A|5S} z*4VxF34*#sE;C8lz?}U1@*m^1X*Y(%h&5k+~iwc?=EU=a7hl!&7+!_hkZx3R3)Pv=$e>E z0T~^o{M+I(95I4xpdrz!V^=0b4&8I;2;EaF@=Xu{eATy2S?$mnlnu4t)qHhfr1RHr z(kH79Gkg7+=jxw(XT$e~S2GR!ywWs`czfZbb2)B9IH3t<1x`MED6p zhmu^RC62KtWYS4yiwDEs4=zZ9EUCcbHU% zv!!n~xrB$M=TXJX!-pg5s*<$!bj4)KDRv5CR#(gdO@#pV$>SQK7ite)pTa(PnA`9>{n=K0?|nSqwVW-i!kE?5tz&qbgE*7o=~1ggSD zBhfgN1P_+CiX3fuAS^Z$!I6m=8WDUgJerCO86|r%xo?dUi4ZE(LnNOOq5{eg&`nwx z=E1C?blq%#EF;P08AM00a+qOsv^M5jP1z$Fzq0-Ehwi!oNrk83)n~FV##Mc){0E;% zWHH7c4vupQy#C3EcStRfk+s&L;6FkzGr4@k&M2HlRN1VLwI+afUDV%toi zdNWS4S)E-1*Yd(MF!c_|;(CKy_k?}~9EqI1-j~wBKl48y6<*yB6+RYezHu$g#`9eD zIgOUI`yYkBx2xkWrvs@De~s4pSBtQ{p}r=Ic75s>x%}{o{xt{bzZS-IHe>GEfme(taU>*Zzwi;u!i_Vr4_&RYmGR;oGeXT4th zmCJK^dVig5{+Z~Z;TOTC1v)GRrf*VWJx)%vu<=`m`9Z8*3tfAMofl)wNr&h5;{}8J zVRwJ;b|=FMPeyZWL2n1k!X=fpgV5H3g*Em@oR^{4jk zSX}dKnU*;Yn%m{7JP42vlD;fnI~h=nN=js#mxpcnlcxcY>ve~9 z-f+Fyb;aY^OI?RM_x`8osz0+^(1L7#`AZW7VK=ep@J%I)D(9S`l+>SLk{-xX8oap> z*1EIuHR!{-r}uVd!=1O5Ja(L?AsCfndV9?nn7)ozZU%jztI03Qn9RbMQIi{Qhgi7G zzxu%MgrOPSO*kta;bJLJa_u_d`?H)Ol}}zIjln&EmY4gVN0#&j*Xw>>K5AK*70}RMfJaHtHmN~te!xeGzN|s`v!Tp}{DNVH# ze;5Vb|9!>@f8dIo|CPXGHOy`V09+%B;(r@1*q(4-Pz@MOAbXVazm-P*zzeeh02liC z--ZjmOc)YWW^82#0DN6 Date: Sat, 29 Mar 2025 18:15:39 +0000 Subject: [PATCH 375/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10230 -> 10233 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index aac4e8990..eea31798f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 02:38:24,642 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 02:38:24,642 submission_checker1.py:3260 INFO] --- -[2025-03-29 02:38:24,642 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 18:15:34,213 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 18:15:34,213 submission_checker1.py:3260 INFO] --- +[2025-03-29 18:15:34,213 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 02:38:24,642 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 18:15:34,213 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 02:38:24,642 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 18:15:34,213 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 02:38:24,642 submission_checker1.py:3276 INFO] --- -[2025-03-29 02:38:24,643 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 02:38:24,643 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 02:38:24,643 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 02:38:24,643 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 02:38:24,643 submission_checker1.py:3297 INFO] --- -[2025-03-29 02:38:24,643 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 02:38:25,600 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 18:15:34,213 submission_checker1.py:3276 INFO] --- +[2025-03-29 18:15:34,213 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 18:15:34,213 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 18:15:34,213 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 18:15:34,213 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 18:15:34,213 submission_checker1.py:3297 INFO] --- +[2025-03-29 18:15:34,213 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 18:15:35,157 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,14 +24,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74879 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71625 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420968 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.3662 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2097 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.6324 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7d802a3e7..baccd0398 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1002 +1003 diff --git a/summary.csv b/summary.csv index 5cc606e04..85e9bc3c3 100644 --- a/summary.csv +++ b/summary.csv @@ -2,14 +2,14 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74879,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71625,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420968,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.3662,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2097,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.6324,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index db502e435..e38c72e6b 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74879, + "Result":1.71625, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.3662, + "Result":22.2097, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 350c032e455e0346258e5392cc8fdae403b842dd..558ee4b751589cb2741196395c1dc83460c06238 100644 GIT binary patch delta 3056 zcmZXWc{CJk8^#AAN*FC7jm8+tzE6`iGcm(Q6HS(sYHSf%ljR**(%8+|4QaCWsxX$Z zCA+cjAxz1>rZBR8<2~Q^$9KNxoadkWoa=sm*Y(`bInS2qAJZc5r-Dsr_^+Dg0Q)@Z^N`yl2Ir419aj z6|UqOB7HHO+L-n(T}>iov*Ey3U9-@}=F?YC=Nn`%hy7IunKQg>P4DjR=DZF!qhUVQ z_w+GBJT#>_ybMF^WQs=9G3sve$Oyy$(kL1QA@-%B8j5#>0lueoN=h#LC>*OIF(+%y^{ zN53z2h1Iz!_QOl4&-@299r5G#gf9^%&U;)I{eWxS#Oh0k_1ND`<3~3NcTOmg-%{)d z4}gSPnMw%I;f@Cqvie~<&9EjKmp2YosNmsAkUn?jA^BYF6?xF@yISvP^0$`F$EFg? z#qUL;G2aPjR#p2Q?RwbWtw@h$^T&PYSh+ouX0^RE!zSim*CA7(_}WWAwq_+#VO1n>Uh7)u1nSAlD{1`w&K2r>>>AuQf7_})h@n>_pM+e9rcVmF6 zUuT)vg_yQT!SUOWM{r>Y2Xc=zCRcZQ`N6l4xBPXlYqgjFzaBbcxFCHO{VdvD^f6b;(27LOGuCUcGx-4(iNyK|8l~rITHX254H4FD|Epels zX%d`2o)Y0uw!s%rvc395vh}SGNGcy9hiHS32^cr00jiBjMq`0z9NhhV?G1v&hYg_w zfHwkMAOS>)#lMo-s5R3S<2hGXqjNU2LTOy+d*0XYQ?h@ACxMJPDFZO-AtL!ThGP8Y zs^=Ai2Od_nuvM~~TD!4jm-yH%h5hPbW+=wZ%k%FgpNa7$xdhil-QtK_=q%qEmp3C` zOPNcSWJEG)9!!{G_lpJ^*!lzm#p4Yj1Oylb0ir0@x?rAU$kD+U482YmHa!*ZON0=G zmEQbygP=c%l2m}}d0TRGy z&AvYWj{qc4-p!A<;cF&f#$0_no1^k{qLNq?CiiY6FuKM->-W3nCu1vN>yz2lolob+ zbo_0skk2IrG@0cB8;d>z&hW`ueg{nIg%-Q&ER622G{*~ibe=vytg*xkoCpdJ>GK8m z>_)#-q5}=3BY~wF+!_|wBTeC$Tr&5iLj8tmv(_HjvH+M@`R&iYgML52c79&wuGg#; ze+#2k-uBkW5XKAlps&c*&&~MR%IIcPt(2N~Kfb*_x7y14nrfZY1XPQWD~9weu^Ca= zWC>hF2)@n)QNCs(1R#3?Sc*>^w}geAeeFdt{xi?t09Q3ONi{y^Ig91&iSKyNqrzd9 z?!9?T`S1MV&d$nTy7>`boOpd22voe3$4JmN%cG(97St6rzW;=(w$)mU0N5_gs zBvo3&DZLi9&slu5-2d?nI<)vdYdR zaw==52re;Z25)$plCWYF_Z*S}zX*PeSMypg3npPS!8q70{3Tlg0+Qk5@D9CZ!aFnN z3_mA7Y|aZNeO>UV>YPQ*3~dUf1mvn`dX}yHz<3*s!q?vP#Kg7+*aB_D%T<2~M9XAo zB0`Lle*RMAQ$+2l_UmUwENd1D*_O2RzRhOLp0)~5M^^^&V8npPI0(1~Ni7_yriM)4 zFAldvy~xQQFW<>0&P16+#79zHG@>kx5WH z*fW+mnQrE|PC6+dhA&NcRQBs)qaRD=-jsB3B&+{Nb{2FbYl$~MTNz-%o>kH|_xAHV zhn)68D2(^>vVKD^=JUArH|hM5)ewt=0Hb=9vc0?Sp7g}pd}%N7us3?*A+MHp!)0OP zlE@u+;43Z~PJ}KwU?`t@!=oUQtS$Rux$NHj_p8Bxqd~ z_^Kt*6-3cfHKGNkaP>k&bg6ACjX*#e(iOz#3Syz}3tST1DkSF>Tg_z=x&^JrjwB=n zK_uQ`c25I1(WxhtAyK}u%Zhei8xi7|^mA8{FAKE{53-MEgnP3)p8XT3x*eExH2SLb zYpj79#`%C!)MUzc_q1@@W_3MblUuv84qf(>e&vb!P%8tM1t!bN4$BH%_B-GH1iSX!xP|&g z@UYS^1fQKztJmL2+~k^zMA2wc?KVLEk<>=4;IxzXWm{L?T|S~ zTJ2L^uDg5cv6ARv%t6QsIyrR%pI)MXq9$v25zdLf1nhKjo^ggK8n5EpOz5-k zGYt+osZl~nr#J)E&}LAQ6c~gzX;c(IG6Rs)fm52&SFl8MP!$CTM4B(l`lnen@sGYN z#mACLon*Y5Jp$Iql`0;UBOU5HoJ*nDGwsCF$gGsv>uP6qEEIlRjUeYm3}9vQd)Y2n z4L*}p33WQ*h7~Qk7Zk~Ec(uVe%x6+GxcJevxIxMKH^H@5)btF5s9sKyh(oYPK_ zq;Gpc9*w)j9!!xSQh7)dU{Sn|FwIGeK=NN}!YxLWBEH;p)sody5^$5~qkK=qNRVgJu3Tmx#& z7DFOIHQ9toy-+>A#`6GxuZx1G*K-#;PtqIcX_B&(2)o(^lIevrq#US#=syv&0RT7I e0RTZ(yZ^4B7iUQ^Qrc|Jq%J85E-}fY`o975(Y(?C delta 3025 zcmY+GX*d*I8^^~^)){M4%$UViBBUsTrx9aKJRVzxC`+=-5@RdIZidlhY$HV|naMsp zva3;JsgNa08bX9IwzIUoN2^Z(uVeXhf?+qJ8Y5Eh`*%!@V%F~a)T z@Q8_$Q1S_tp3Ol25_kTdNd&YS7^0_n0jn0X;=D|hb0WLvCg+&VGFOsI3l zMzt?cs&^qc(%B(`dXt}e?GxnPgVa|wZ|=rkW26z*YiEm7sB`tZMri?^N{7pHZVHFM zoFwS!5q|1ky}l)$6JG&jU+SVA``C?**0#zKd+j@EGXkmbgqN6`Hg$?UvIS4-75CB3 zmyA0d-L23uc-`gkS#&0a-mCWzS%a|GW~s1#EHI;#K4weoFKjzI0fO3<47jNhn6Nt?s&FfhDJeMJi!hMvj1$K8b=%`h z^v{d3?h~RoO{W!~E?@bW>BnBL|4@5$w(rn8AE|$7A>MevHk|QIa~?tdSEJ$M40=5~6{D__pLY!aHk8ZKCC_}eYNiDQTIfihm=2z@{Bl5hSUdi~& z-?Uw>fbv?t0QIc-o@2=UBb-XPrBBskPYRT#_KMR3_Ri#`Z5*ap^oYb_FCut9HKD!M zEN6hYIhIkLa0yBZ*PMg`;m+-7!ho~0>Xx)~ZGkZ0Ew*%C8g)t;6cTAHjXT$N3k@4y zmxqPy2`v8*9ZuwLY@fS4KO`!Fw6Whf1_SPNAVKyU=kxdGKX-N(=3Y;OSV+!;InGiu zt;UhYro}@K^pBQ)o5K<^Sst0>5L5-4#G6M=O_a<={fgh+Z9jY&wjbJ}kUl#8KA4;- zcYT!}Jo(Uer1L4uQuREd=;%(NhNY!^ov9&h$=G0_)j7+mj;Ih}Rn)N%vf^%AoH2$ASUY3s9#UCa z#$N>oVHEQ$R?^V!dg6RR+kLOm znuC&{Fp!ZgB^8k_`i*OzI3Z%0fQIEEKsZF61p;@C2ZwOSb0NPDhB8k;f!&E{JG_Ng zPs;vL(9Ovyl1vdXvJOkGta@j^4K(9n2IM#MQl#?%>1dceR}W@+ zkI9Vf1ek$N1;reB?bZQUa@40`ghJIdZWt<5ShvU_!O=M}U;oOOqD_e|&@p#OUxvft zcUx)cTb9tM^&sVMAB&&Nh95E@!xSBZ%r}vy6$y)Hn9WIeX$p@L=@Rdt9^L{=*)%I* z9}~ZUN9JCZL;40v+^J6?>a)&yEIC!TkF!>KN(`KS>7pW8{9 z^K)OF#_8c+zdSFTsIKQPEnH2k14OK52UL^CpUuCU&XGmkY6`?5{oT#IFeM-Ph-f}% zkhlj9{80TYI30-rILje?)v+zq;3JxWBRHL<3S;5b_u2+NI>5IiH`xp3r55w{LX8nT6fsR z)CM0gBatNUj1hu2MQ{)Sh7x=eev30;Iv8Fk6>1s~_9M}HVnn4_&B02?$2s)@&m(k7 z*v+4i`%hv<{2SB3Rtyqj&0w_ptfKjH?jU<7NA^zPR0%@*%yX?lv^XB4XQILca2+56 zwDbXlNCNh45`P_|OGwAZswn)AfqYLz(1S1)Ojg*F9W3Cc>>1)C-MV6f(JMpKlP^CZ zb^B8(@0~pkxQU~la78VCi*b`v1R?+aWaf?MH@Rt=^x^9qAE}|z#xU`!RN6|t8h~Js z46r_MNmk!Q$CC#&hEDW8rnhD9XzFg@Zi{6*n3s$6J4~{Xi1I&2t9UiBmQRt!`+5FDj8x%mivLW<&}&} z>P=e8&n{Ag@#vJhXrgTF-N@WuCw8?@sMpIBOc9mpRy+KbG_TQ*YcPTH77o8P$1mzx z_=sHRY8Ldk@j1%5#5${*_~f;iV>TG-G#WutN)hWk^s27Czdl}~!1s8_kUL_`Xz@YB37U=dP2ri1$MjTEkn$%xobccW#6RP z)RZr^gWplw0ZzBsoY;YTPvT2O>1%A4;YW=2o!WwnDz4_oS!`vn-F{q};NZzGY|r1# z?P27$$o3RTcnj;y5$xo;V^rrf5~?IJc(vbriVN?JEbagM*Y!Ywg0a8SF@8JD4#Lkn zih;`}xJID4EB9iBPFM)VRL#mc?xA2AVj+>$FtEa{ty)Gc3S5i&r;A4Mj!7T6b3%Ok zo$tg%Is=vCNfOh6jT1@+oUXW43w&=JRi%gQ8WsNxm0DYr!ptuZ^1?HOFe(&cI#iS)m*Qk>xjbtPR==PkhSgLFpyaCJ1`Mj&-#QDn>YfG&nRw*DN!! zJeRHb_+))*{WyU@{QbE@=w!qw9GjVJ1 zFS}ewTu;%%pzDP2uKn2Sy^&=8hLzCp4RVWamAj(%vBRW@KWdKnVA5fit;5PAWn+Rq zY=X>&9eVw0WF4!7Qk{0L4vKkBzOES;td*am0IpF#t)yonF4#*y@@iptuZK8C z_g71F^f9Nb`-UayJkP9#g{da=#t58NMsPFHuv0{J=Ez)5-20$*n~co3zbm8?GDGZ! z%s00`*V-39<-lw+&S8B=$TC)Vt&VK?i~V;d2I8EvoBRr4*Re_alK+njO^t!K@&5Pg zZK&eLlSv?|T;P!-jH;OObP;4#4S944msM4c{X>%n0JzKx006n9|F;qnWe9nyC>~$J MdsVm~82S7BKN*y)r~m)} diff --git a/summary_results.json b/summary_results.json index 34a977893..6465baea4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.3662, + "Performance_Result": 22.2097, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74879, + "Performance_Result": 1.71625, "Performance_Units": "Samples/s" }, { From 1b7572dc34825686dee929e2777de17519788740 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 29 Mar 2025 18:20:54 +0000 Subject: [PATCH 376/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10233 -> 10232 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index eea31798f..0bb239387 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 18:15:34,213 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 18:15:34,213 submission_checker1.py:3260 INFO] --- -[2025-03-29 18:15:34,213 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 18:20:48,976 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 18:20:48,976 submission_checker1.py:3260 INFO] --- +[2025-03-29 18:20:48,976 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 18:15:34,213 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 18:20:48,976 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 18:15:34,213 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 18:20:48,976 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 18:15:34,213 submission_checker1.py:3276 INFO] --- -[2025-03-29 18:15:34,213 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 18:15:34,213 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 18:15:34,213 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 18:15:34,213 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 18:15:34,213 submission_checker1.py:3297 INFO] --- -[2025-03-29 18:15:34,213 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 18:15:35,157 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 18:20:48,976 submission_checker1.py:3276 INFO] --- +[2025-03-29 18:20:48,976 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 18:20:48,976 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 18:20:48,976 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 18:20:48,976 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 18:20:48,976 submission_checker1.py:3297 INFO] --- +[2025-03-29 18:20:48,976 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 18:20:49,929 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2097 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.6324 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0877 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index baccd0398..59c112266 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1003 +1004 diff --git a/summary.csv b/summary.csv index 85e9bc3c3..68488bcf5 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2097,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.6324,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0877,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e38c72e6b..06b2f8f02 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.6324, - "Accuracy":"acc: 75.800", + "Result":10.0877, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 558ee4b751589cb2741196395c1dc83460c06238..fbbbf2fc7116f7ba204248c6b669725bcefc6ae0 100644 GIT binary patch delta 2091 zcmV+`2-NraPxw!;%>xRg`KAAG1ONbf5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0TZkN%wW^YzaJZtPIG*k8SP7@eM!K4-?`z>9DDfnB}fwbPvK7_RxTkegccl@LIH*!FW6$S=u746Cjr0^HWGSzIjf~3Dx~7o@FC()f zJ0|58)fZ*7lC?F&cL;0l|nUouZC4@)_P#J88BoXs+dpX&d_>NhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgcTJ@wQPqhv$9nFW`D8= zkXHpM;|0xQQp!$-ECk_V!K-CPxoQgd;;GlREm0Cuw;z`3GOX+AWX&%LmtS*kt&&9$ zs9q65?LNQZRbDvv;*=nx1L9Rh)(Lm*Ie z{T-r1=umV!=$CycIyf-Z9cZG1oEx13vc5&Eg7V(=|A(UcjGX>JbPxS4(b-Pd_YOb! z)cLsTd|h>*K{(PO9k9?&5C~Kq0)eVSAW(G(1bms#{2i)8=pfsR9rQ~ds(%g+Omzoc z*eOBIjm`lJ$t_|PwnhHus;*jAxu{xlBdtul_}<3??EV)4enxtKAi%l*PJoV7px=iZ zFzel3IwR$ZOh}q?BV@8*4fAcZId7ObgL$(T{{>f@`8=X(Wed{g}oIdaRCz6cvu80avhW_nsXy1u}su%%eAU(&Nu37 z!HW7yHtS0!1j{ye2?-)nsyn&6jTsZ08GW25m;P)h>@6aWAK2(xeqwhRUM@zb_? zvnv&&0)N`Ncuu4S4J-pGYf9j8Mmr2ED0xQ#tqcB72P|Ggkv>`&M^RP3n%Y)sIZgdr z3fxxUQd%{*tH3=4E**J?V<}DzuIOu6(burl*BV@*7c2B)g#A*b$_O3vlC?by6sD!yD#G&(TVNtO_Gl8IF7qF z?h)PgOfe5$Os>aynZsMmO_%26V*a>3P1}H~W6y^Bl4Fd^&zM6EUiK z#CAQKxTf9r@0iQTEq|8hBSYVleUv?A{~EiEA-JU z;~9#ar;5{7E2vNp4lU}tk{QCZg*U4WSTDdu0i+yZrb0XK4Z+y49Z2eu;7~!_Xe+4? zrQKpf**Dz+&eQY}OYoFaPw@bZ%|t{;Bxe)S{MdE`aE74;3CcU1B^f?@Jy_{(HjK)J zQwrZ3xM@9!lY8&PWbC>w>55y?8uDj&y?T8gSB-@lPQ?J#C1-?LpkCX$#Fs@lTuYr_ zg%fWC@TWfLH{Onm9QuGqApkkVJt?2X^GmZtb)M!AaazRryvb%HPs#oKYZ!Uee+J6q zkbe_TfJ3uoBZ>kAr1_=)aFg^UTLER0P$oqIjFXloA{+Se)3$m9001Zz000yK00000 z00000004l27L(H^K>@;(6(>3YG?P##CL0*r=G7(v002+|000pH0000000000004l2 V6_b%CJ^^Zz&?h4X93=n%0082e-fRE> delta 2116 zcmV-K2)pxP|^5@QS1ONbf5|e!h8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Iz&dGqKal-;bR@2ex;aYF;Xx7f777_AOraj{9F<(!_Wo1*a?< zm`>9+4U&Z{qS<0#exIH;eAD0}$f6)&85x+bgqvT7|MZ_(@q;f35r#C7@qxJ%BJWui z50@kjcz=`SB$H2KR-}QDx5dKZc|n4xBBhDt+O}t<0nNS@>W!PBKC?Z+qHY~Yd>0|xKSsAFQj<@1Ew)4h5Y&vb*b35wwye4GDv1+eXgvWsh zhW&y)8%5=T=Ye`rPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!h(urId6t6wbDfXrhl>r zkXJb=;u+0iQpiq*%mm?L#>;t1xoQgd?5WeVEm06swI7!1GOX+IXvHrGm%BN)7Ry-> zs5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iA-qHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR5etS{wap zEkA4HTRJ7hf{aL#a3f^0U^Vk?v_5Z`IK658E&dy>KIXHCs+BEB>XQROvxUKb)<+ax zZ}jNe%2*gr%7hr=HJ6|C|1SF%~3Ga*>IzFU$YB89q>U$-%1Vtu1Z@^)-A`~^@;0|XQR0Fx~y7qf#1p$rA! zA&eV&vnds$0)GldQ!ZKame z)W4;`Z3Ql+RfD?<+*9Dvk#{(j;?&@ZzJ?Wj4NHBk!4-P3LN8Y6#R|Pxp%*Lk5`|u( z&`T70i9#=t^g_6Xb^~nMnU*HO4p|08@znS7EZdf%v46xVNKH?bs$C3S?R+h_^Ps}S zG89SuVZU#v3I#_>gPiTZ4)2UqUFx=U)o&&H{V56o3RcjP!E%FjPkxOQt>XW&?a8X? zh^gbxmG>sX_hsHY8F9x0|EUr?36CoA@K@7&$$tCPXfg3k`lPQGhJBrUFp~0j*=JVX zhuJ<5$A8bS^w;Q)U+PC;JZ?;nU+hO=Jg&%3EcXsJ9@Rv+BR`KE$%e>(J5jT&V_e#A zn5bh~>gcJj4PCFYdW5up$j+{HwkEddyg#Bmjv!5m#qD{d!SwS zIItHr)%V&GEPHuw_^4_4l08CAb|@^qY*AfjdVe-ML6)!EzVx~KGVT$b*xueG>FAE* zxO?Ls(QVHZ^WeqgdU-jy7KkH#B#$nqv*luRGp}brM@*ic=evEg@Ao~=5qr<4<7YDw zqpC-2*RzRh+I>%+NOnV@C`5S(JQpYSI_AbFp<;_~OIPybt=_Zm&0vfV{{v7<0|XQR z0F&}27qj3QLkI+YHA+*HCn78wC@NNxN`She5)wsJ>@0hlTEq|8hO~c=leCj~kW+I{^lCueEer!7cIK$9_1mzvhk_?}{9;|dX z8wTaVDTO~6xM@9!lY8&PWbC>w>55yy8uDj&vwD3WR*i+~PsIS$C1-?LpkCX$#Fs@l zTuYr_g%fWD@TWu2Z@e8B8T)`oApjZUo|Mnx`DMLCb)M$4I4$D*ah=UcHY3ITS3mNq z{|uDJA^#?x0AsUcBZ>kABJ$_Xa+C8VTLNbYlYs~tlWHa^0gjWECL$Z)A&eV&1ONai z6#xJf00000000000001hff$q0CP59u82|uyY%g diff --git a/summary_results.json b/summary_results.json index 6465baea4..1607e495d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.6324, + "Performance_Result": 10.0877, "Performance_Units": "Samples/s" }, { From 5e485b81cecd9a62662368cf455157d1824957b5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 29 Mar 2025 19:04:54 +0000 Subject: [PATCH 377/719] Updated results summary [skip ci] --- README.md | 38 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 12 ++++++------ summary.json | 14 +++++++------- summary.xlsx | Bin 10232 -> 10229 bytes summary_results.json | 14 +++++++------- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 0bb239387..0d16ab745 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 18:20:48,976 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 18:20:48,976 submission_checker1.py:3260 INFO] --- -[2025-03-29 18:20:48,976 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 19:04:48,888 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 19:04:48,888 submission_checker1.py:3260 INFO] --- +[2025-03-29 19:04:48,888 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 18:20:48,976 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 19:04:48,888 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 18:20:48,976 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 19:04:48,888 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 18:20:48,976 submission_checker1.py:3276 INFO] --- -[2025-03-29 18:20:48,976 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 18:20:48,976 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 18:20:48,976 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 18:20:48,976 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 18:20:48,976 submission_checker1.py:3297 INFO] --- -[2025-03-29 18:20:48,976 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 18:20:49,929 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 19:04:48,889 submission_checker1.py:3276 INFO] --- +[2025-03-29 19:04:48,889 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 19:04:48,889 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 19:04:48,889 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 19:04:48,889 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 19:04:48,889 submission_checker1.py:3297 INFO] --- +[2025-03-29 19:04:48,889 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 19:04:49,948 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,22 +24,22 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71625 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420968 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21553 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25985 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2097 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 20.0509 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0877 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412967 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.43633 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436332 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9043 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 59c112266..49bc2728c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1004 +1005 diff --git a/summary.csv b/summary.csv index 68488bcf5..de5954abb 100644 --- a/summary.csv +++ b/summary.csv @@ -2,22 +2,22 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71625,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420968,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21553,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25985,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2097,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,20.0509,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0877,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412967,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.43633,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436332,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9043,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 06b2f8f02..77015f1b6 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71625, + "Result":1.72716, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.21553, + "Result":6.25985, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.2097, + "Result":20.0509, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412967, + "Result":0.409623, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -514,7 +514,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -544,12 +544,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.43633, + "Result":0.436332, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fbbbf2fc7116f7ba204248c6b669725bcefc6ae0..608f295d1d6bab69d538a7067b2d1df3c5d459b5 100644 GIT binary patch delta 2614 zcmZYBc{J4P9|!OmS;Dmq3CTK(kc$SxwHwPILPk@5vKxEGaP32OvdtG6*&5=yR2UJi zC`+>MlI%B2HAvQw-_-B?&hMV{d;WdSc|D)c^SnRjJO`G$mNkKA*%$6hXl!tRKq5fI zSuE)ak1+mmYfIzss$Zqu`odJpdgpU4YKZ6{xA&iZllL4aIggg7w`zie4*iB63kQrA z?zRAT1Vtxbum#YkN>4>2I7@YwtazbD`Hq6(lSUZjd4J^6oJ?Nuau|xoeLYK0=xPh* zn~I3)Qm6p&sIO$b?BUfhKl7GloNtbkj;sM7E&5A$>%!Hyx6EU4b*2y&z>mzWBbK%r zKwm4-vB+@S6|z!9RbQ++0p{!fl;W}U(Ke9H4g-^bW_Fb5 ziP${irAB$}#~}4rYO`hGMBe!Md-!Ps%}w_g1n^ya3iTQakh!I0b7t4+aEWQ0RGfY=1%LI~)CYwOX=*DO?L;>^TU{vOsU}n;yE>#W# zRdUupxd(CM7@!<&O4g=Wn`Va4H_TH8a)WQGn^|R)u1TgaQKw`An9+<@1d`EI6DY(m zJA_)g?*LAL+Ru@81mYUCz`d0|TNpONC0Q2@p;GfEL6}HrM%apN>P-C%=W;9Ncgo`6 zGtypjeag0}M97A(6#+w?z3`3yLXey^MlRcqX=;dV1 zO*N5jsF}n=dDG!J(f5w*m$lc?6oOb>4L3Vrc~OTpPTF1^p2mWQ&I~5vl+Mt^nZNGb zy|=Wmf}NNpIv+b}%2n1d7vKLB^B5HLv0=} zCnxxONY&JCo*po(%sCIt(H+_^Kl6Z`iv`tD>GRHYI8md z0|8N3*xx;f<#Fv5EGFvrWIJ7iunyVcyqatY4YuuV=lf;7R0etkkx1YH+*E722#SxU@eRkC$`>r`m++QN!-w}EMhyZ)ez-%S+; zEX+J~0>St&6|ri1*!_*9jgyFGsgHy0?-LajrCPMG1vm1(Q?zDFU~XGhX| zr)#hOrY-h021AT_JKAG_d_vsjgzzZL%)of@Wja0eXX&Gej{a&L5_N0ga0xu z^I;E$4l)tYBPWrz2J4(R2f4|lE-`l)OD>f#Gg=xnd*xPWs-0S*DKYNt==cwFmzT>L z_iCyGsSEou3Cb~?6q{tfhI8Gp8VwKRxU78vc8y(33=*Gex3u)K%ezaps73$`*`j%xOYD}r~UtOEkC*GNL4oRVdfT|b>>Erf@HNpNA%T| z{TLk0xos$;?yzj2&oo6|#oO;f6p`i^=00eEo9)-*FO+WBIpR%|JJCkR;&Zo-6uC+; z$Y}Sh`9v59nR=`(a;!bf2-eyBg?&a)y_H*d`tuo$e-~3OuAE=h_dklvrMc581BZP+ zlqApb-4cmP>dr|ZiBt8EW|xTAdY9ulA~PoAKa--%6$is z#IC5~*FDF#BUy6B?Kn&qca^kmt{UEtpe$ESUDvFM#2_ZLATAPl<=?=skUOU1$Hufy#ef^U; z5?kn#y{C3hEg~{2PU30bQQ=btzt@G0xF{EKr5y}S$U*6!&yn48tJP4q&6%VRwRyO` zU(RZt#Me1{8=d;qgoTzVWpq0sEnYkg8=7-y$|Gi!JDq+YTuYrk7&6@2Hr$MFJ*9fI zed`*2de1=m@`*z}w(&FUs68QwsJO`;t7bWiM-_E-LxsxX^Mfkf(N$Ac3Ob0zh{H=K z5B5~d$#=p|eHlLjz3fOce@<4Xy!WIx7H?Lb(USNda5IPU%A^7aDR~kH6-zdaZl*#Y zhSxvKhrheo$upgRBxL{%o^ozogow?8rYnRdR1f?c%cnT(xn8n>VM*SKK3#UF)}cE? zhEODHap7@^{zX*%ADgD`XU7ERs#P6ZLFZXA32_t-daD|<%j-RfO0&BLECE%I%e1qD z6S>^xiEJp#F*f;^HY5wJE-gh>X}-eDf;5w_E)T%%!U?QUnaw-7t_MrIj*lMr2xZGv z2)wwW=&6-jdbH4q_nA@kOnoV`?GZu?ixkJE#ecNy0!_d?;&3GnGtHe-hA4Z3!rSru z{w!M+3TyoS6^7}N?WMfYqRT}&LraHlt>~blmHmL5ore*H1Hu>$Y1*Nk&UX1JQ+9>q zDZB{$`h8<@$@eofFX>*UfBfaN;CA3-AP24q<_AXMdR%S7AW(?UuYtr+AGbij32_ck sK?sAD080e?!jId9gF&EcOdt>+=zl9Z1_ERvw80+05JH+&81`f5-yCww`v3p{ delta 2601 zcmZXWc{mhWAIE2eqHH6QZ8XNxMK>DTBuqvOE+(2RDY}D6Lbh9$I|@x>H)G8ZVo($Y zS+istgt5K$xh=+;L4&gOj{bSy_j%91=XpNo^ZlLQ?>uMI=DSUKz%ha35~1#OK>*-1 zIqVn)PbTob9h;qTUI=(Qq6YuI=>ymFiBy#*(B40LnWe3mvDtR;K*yle#pT^pz}+j9 zK+pYED20zu%|_hV-CQs_%4}Omd~lMNn5>-9L8wNP`nF}_Yjp*wGqv*lk&Yo)(JR9~ zqys4-nRNrf`hEAoz@b6v9bM|odTC8hIXKmXE+)+X`R1DPiUze+-BF477^RCMfD=2v z)dmL+f{VH;L*i7WWXBRl59q#=ueB3|)532hM*6Ogc`B;S-IY2>-d`i92q*!Y$@Kzy zvf*H+N-LaNu@t1=raMy*8YmjgjKfWs7_dEkjR`XXfqO z=&+^sJJ3WAIwGUbQg<>{Q!Wloy%7tFZ#28~?PbTk$*wrfN=OToi;;j=6ePjseir+Y&#+IZs`P8LBIU;q-W6dz%wQBcMM z$iQy=6HPkETs0O{rF&G@;c~1E98IMhJyUAZ_QkGiK)o6trWaUqt>s(9H#baQ%gWJK zgC>O+dbM@ef^@T`@KOU7zp1y*&xN?E8E4mjt+Ma;y0$*Q+9mj$=A70J(oWE*fDSAJ zE%SluME-K5P+y9SPzMPLP(Kc;N}YDpi#;tgP7_y&@D%LFbQ zvC9hDFX9pHRNTf=!S ziN1|Wl<+ZfN`R8R(e+%c!XXYMlth$<((sN*WfTA_3B~ye8z!A~4jUqBRYUEb1mUD| zQAj;cou6pjOO-MYuI313V*0Y(GDkNtomTed`@4gRwy*xZso4C^b!cWWLW{`X4V6|P zd#@kKEQ4sFcob*~h|qwwM^+P`$kxX)wYW9+>xY)y@AuvZM>v|s7qa@s`tg*Bt34Bw z72`@5x(NA$4z3Tmg053QxE{^W@+omP>4}KR$?yo_RyqD zSBS<1Hqkr5&MZ%$Jq;t5f4}fL0Vhfa`N&a$P$Kw>bh~}B8zBfHcM)$YFezwoageK+ zGLd5S*q87Gn1Y062YbG>U|S2$eYy)jb#BaF5JH|>^snz@q2@+6#WTXFIynK=Uq7RR z%qHOMyn%$ot}s`Si$V=-Lo{A3+W;A9mA1BVUg$h(4>n|y8?$0iD(+g@J@}%4$vWv2 zrejeTE`XK;nI=K;kVabRcmpl+%dHB6BkECM@pKKR;B9KoM2~Z0y}o_EVtX1@o*Up5 z0hiPRdM8f>{ziPuU#woaRc;;80|`hZ&1Tu*)+baSS*)!ksorP=of0{BQ>|uwZ`0P$ zE?^*fGHAOh<muY%f_C=D?F%***WFW#Nd1;bN4Y zV9Sb0+aS7S+VCIM;_=l;hl4Q7)(h2pH(uTwNObvEPo=*HIYQJ9Z(=u>OMEBhTp#|l zGt~#2kAPX$hG*~(LZyvq-Cy5>0MAT)z(PJ?+&W5y_Yc3_r1K8+%8_!?s6%=(rZ*YdT+Scx#YL?ApV`)M6RT``2qnYk9)swZS3!3Reh-WL-aCW&D zv~P%v#HFq6o)^kRt-!fBCLxFiC;1~B_B}lJaH?RfY;LoTVS^8mHM@J4u!eTc1JScv zyXu}FJZ3{`QcaYd%)A%3&q(91?}R_)r>lS5cH=fb&h?V>mu)wJh(1VjB7d_;)%OB% zKM7-df+;Y=eZ{c;cEiA*t_N>w9$PG_bre+X;0F3NKi*M0YEczGKD1;4%c_EqO@c zHKa)k4cE8XhKupE`6WtvB<>=?e4|rtk#k1c2F}$as(<8gHA!(tpQ`)sa2u!_Qk2MF z^h?v-iDd4#Gk9Ksq#qv9J^zJ6R~ZTgB@c|wzqp4e@ab{x=^V=vDMG+bSU;vibwb=i z>M_Y#WBAc!g@Mt7&7}`Sk&UWmzcH}qn~()-4Ev!~CoNCxXI)5SL3R^jsE@SCKOc*7 zg)-+`-85>mikP+-?+*)1Aq+bK#$xEjb<(%5u_t=fPLWp`u$AT;!d{v(KkJU&e7De4 zzI}_ca=lqdex2U^O;a_}_=dD)p-PUwg87*2oOPE(`iZx1xLe%kqY7EID zqCOu4LfEgUhfX@Ri%dL$;SfNQV&P* z{5_wO@ZvTRCacRfwm*S1PsJXd|N73sm9GQi$RuS0pd@)f83Ak~i>bhYFtU}3vdm#M im4N_&Ee`-72Kdj4cqBoNS1|;7k^58>`DK+3cm55gWausc diff --git a/summary_results.json b/summary_results.json index 1607e495d..fadceab4a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.2097, + "Performance_Result": 20.0509, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71625, + "Performance_Result": 1.72716, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.21553, + "Performance_Result": 6.25985, "Performance_Units": "Samples/s" }, { @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -725,7 +725,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.43633, + "Performance_Result": 0.436332, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.412967, + "Performance_Result": 0.409623, "Performance_Units": "Samples/s" }, { From e0a1c57e338442bb3cd7ecef91882978b326aa07 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 29 Mar 2025 19:07:51 +0000 Subject: [PATCH 378/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10229 -> 10226 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 0d16ab745..f84bcff2d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 19:04:48,888 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 19:04:48,888 submission_checker1.py:3260 INFO] --- -[2025-03-29 19:04:48,888 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 19:07:46,168 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 19:07:46,168 submission_checker1.py:3260 INFO] --- +[2025-03-29 19:07:46,168 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 19:04:48,888 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 19:07:46,169 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 19:04:48,888 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 19:07:46,169 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 19:04:48,889 submission_checker1.py:3276 INFO] --- -[2025-03-29 19:04:48,889 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 19:04:48,889 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 19:04:48,889 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 19:04:48,889 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 19:04:48,889 submission_checker1.py:3297 INFO] --- -[2025-03-29 19:04:48,889 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 19:04:49,948 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 19:07:46,169 submission_checker1.py:3276 INFO] --- +[2025-03-29 19:07:46,169 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 19:07:46,169 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 19:07:46,169 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 19:07:46,169 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 19:07:46,169 submission_checker1.py:3297 INFO] --- +[2025-03-29 19:07:46,169 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 19:07:47,173 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,13 +33,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 20.0509 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0877 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0925 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.413865 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436332 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9043 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 49bc2728c..9540e56f9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1005 +1006 diff --git a/summary.csv b/summary.csv index de5954abb..8f5eba4ac 100644 --- a/summary.csv +++ b/summary.csv @@ -11,13 +11,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,20.0509,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0877,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0925,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.413865,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436332,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9043,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 77015f1b6..8ff5c953d 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0877, - "Accuracy":"acc: 75.200", + "Result":10.0925, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.409623, + "Result":0.413865, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436332, + "Result":0.436533, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 608f295d1d6bab69d538a7067b2d1df3c5d459b5..7f6e4db6d443235a2060f46cd5834be20e055755 100644 GIT binary patch delta 1855 zcmV-F2f+CCPx4Q&%>x8&dLVI=(E~<*&;Vv^)62gf8f)N{f-{v~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw=GXCG!)MCA^Mnv# zNCO2QnTZhPz_NIlkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ@*pkD#=?Mq{fP}4 z$7x88X;|f?5RDN=vOvi2d6Jgg91o*ZK2!M^jKm}Jn=_a^`ldA=)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QEkVbln64H#RR2QMSDBR!;%NR(kCNu!7}6t&y1;)3Rn%_C=RprJb6n(Nrk8~d>BbZyV=sn_$G zku}F^yw(vO2O<~`8GSZb?SYrGNCh4Ne^n}yYS|8HZsnQ$&1DZDuS&w=MOws!$xenW z1mR-At7V>Y)fDi>Q?G4Xf)P@;AC~Getn2Ay&1Zzm-JDyiWD$g_t!T+EqA}G9%c@nr z3d+J{@#MC>w%uCq{)>u&D z-0t*!``xOyCAvN?y1p(tFb$7%NC&R+2n31_fk4qA5GXnX0!7!~Av%N(MYn@~*@vQo z15@3BCOXKu(K#UNTf{0T?_K|YD7w$c=?_Hr(BBfB?R0(b@Pn<+$5rR+ssjzekq+sA zg?55Kpz06^R2>3=szV@PXFl_Hf2abm@!W&E;U0WFof8$w|5ktI| z@{^vtl=2;^Tumc-iJQ|1qhu*%sd#O?tJ9Gx3&uH(_UoXkBK=X3{k?F{8|;PN!570l zFh+*Vd;^u))n4l30wk>Qunbn@I$*1`;6_GbnW){CYZYtGH|lFa%lb+->q{yG%{O-m z2_nMOo&36uDHWR=Rg$-37NhY`P)h>@6aWAK2mk;8KLBhaIpyuMcnC)cf2P&UThO+0 z&_wTuEe$_J<4VI1(IA>73_nZ*Xc7&eS!DQO2>{I^!;c2T548BC6R;IPREI?YnPvPw zx?;c2V`ToXzJf~XD+Q(W`ZV1<=eo9XphVw@ExkTO<4La%(IAv1ygp0=P?GV;?Gjz& z^W|>Pe4eE2*v23U=G- zlioRduFiQoh_-WxB(L0&Rv)7Aq}7LLkjN5NAEp5!$#{UsBC8MU1c)rM`e?BFK#Na0 zQKlt{1gj4X10tE#2Z(%5R-b$#y(ba*T#Y{37DVc%yr*nzz4~w(e?-IJB$!NMqZg*9 zYZbX^UA+(TTn){Oq$VD<7yV;+seXq*!$K9R{De(QrJAjY%*_3pMLb&4^a7qg;MY z!uj%3`Zw)sm?Yu!{@W<%_Jb(B2Y|op|&M@uUGK{ikzp4(?%;O(E%JHYQsE zh?9~gA{#%Px+8c5001Zz000yK0000000000004l25|hs+K>@sz5hpqUE|W|rCL2ye t64WLE002+|000pH0000000000004l250i~2J^^Ht%qJrT79{`x000Y4UY-B| delta 1838 zcmV+}2hsTQPxVi*%>x9a`KAAp(E~<*tO3km)62gf8RR1urABBR$YaB+9Un&?q7^dF?i=xM101^T=5nXsC|2<~p|X#y)I2UE6be>h-*) zWX-V}uXTjSfe6OKl06$`?SU7wNCh4Ne-#x;wQPqhv$9nFX0iv6R|P5K1%RzD9@CZi#f*puXv)olp%o(^xBJh?N%nI@hF1T55ZA1Uf6Hk; z$n8$wx8JRLTcYdZqU-CT1Jm$Ghjid7k3gX45C{|<0)e7KAW(Gu9il_%P;@)!mwhNY zI55>6XrhCh8=V8PzD2Bp^4|6Thobw8oc=&`5B)9C*-qE@4nO$R`MBzQU3H*AIMN{< zu+UBr2vi*cfvQ6wP<03de3{Sue;ukr=pfsR9rQ~dstyiJbq8J8DM8MS&H)R_En*e6 zMgHfiu3A>Ps9JI(txUZ5-p2y${ucp$MtXlBz`6fUfR0q4--jD8>)l>DBjt)rNSbmZ zWU^ol^KG;_Zq{mC%Qklj z2_jOeJGr}!855fuRg$-37NhY`P)h>@6aWAK2mk;8KLFsDAUf`|cnC)ce`d3(bI@$% zpo!iQTN-|d#+8O2qCqrE7=D-r&?Fi_v&itn5&)V-h93=vA87GOCtxdps1Az)GRydV zbj5z3$H@F&eFc@&R|-nu^)WqL=iOymIZ&c+#FkzkqVc5HhiDMW5?&vs0Vv6MH|bRC#z3Bk=~Pte6B_xZ3`lGQ{GcHwqAWWe+{Bxa1u-=vC#|D z)3u7+w65Lz@bX3P zg}wixXTSQz(65X`O(QM+UYb_7>?QlahIMo>?+$}c-Do(T#l|EUq=lMwr)ET}*HJD% zC*gegDgB%FHB6FldjD+{bo)V+eM@f(24R@of5}eC=Cz(j{{^#%6PN)7`0>-Wdb1@J zqXGxox_C~c1`V^$7tsM8ib9l!z;kh8uVZd}5-PU%wsa*=-s(O3-VDb0@ISM>8DR(n zc@QXzlOZA>f6Gq8Fc3xed`0Aa;yenXR+379x}p*iMO5r8d)ivW57~w^e~**4X)9&L za(wTcxifOPILIdY1?z-X70QwnMWC4GLhUN_v0lcHC~}@EP8+SDLLE4?sP9T<2-6nc ztTtf102c+2a)g-*?Y%byW5@O&sY`-G1#zpbq&}2(e}@fa-*g8!Ptyl1!Bb8>#XT@K zBN3gDoQ+8HW19iM8HNTVDDQBVWccFsV5OUE7?dlg6uvVsX+4Xh`{2ZAY}+X#84wEq000000001hfxnXuCprN!lSwBg8+i~YizWd808jz|01*HH00000 c000000DyrKlZq!k0cVrRCnE+KB>(^b0GRt<;Q#;t diff --git a/summary_results.json b/summary_results.json index fadceab4a..f4a36adb0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0877, + "Performance_Result": 10.0925, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436332, + "Performance_Result": 0.436533, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.409623, + "Performance_Result": 0.413865, "Performance_Units": "Samples/s" }, { From b6bcc3cadf63cbd216b7e520a976283b5dee123f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 29 Mar 2025 19:09:14 +0000 Subject: [PATCH 379/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10226 -> 10226 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f84bcff2d..e8e7288e9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 19:07:46,168 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 19:07:46,168 submission_checker1.py:3260 INFO] --- -[2025-03-29 19:07:46,168 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 19:09:09,786 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 19:09:09,786 submission_checker1.py:3260 INFO] --- +[2025-03-29 19:09:09,786 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 19:07:46,169 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 19:09:09,786 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 19:07:46,169 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 19:09:09,786 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 19:07:46,169 submission_checker1.py:3276 INFO] --- -[2025-03-29 19:07:46,169 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 19:07:46,169 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 19:07:46,169 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 19:07:46,169 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 19:07:46,169 submission_checker1.py:3297 INFO] --- -[2025-03-29 19:07:46,169 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 19:07:47,173 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 19:09:09,786 submission_checker1.py:3276 INFO] --- +[2025-03-29 19:09:09,786 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 19:09:09,786 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 19:09:09,786 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 19:09:09,786 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 19:09:09,786 submission_checker1.py:3297 INFO] --- +[2025-03-29 19:09:09,786 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 19:09:10,871 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420968 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421799 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25985 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 9540e56f9..fb35a14c0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1006 +1007 diff --git a/summary.csv b/summary.csv index 8f5eba4ac..0a34e7a1d 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420968,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421799,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25985,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 8ff5c953d..a1ad3cf85 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420968, + "Result":0.421799, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 7f6e4db6d443235a2060f46cd5834be20e055755..82fb7d4da47d7d6d2d706821b343d79048950ee4 100644 GIT binary patch delta 3097 zcmZA3cT^L|76;%^q`43X&CR15k^rGBMCm=05R@vtNDoL6LX#E+6{3KUK%@!=11h3O zgdm8LrH5Vw=`KZtAc7z$u=>FF-g)Qj{5|vC-?=mQocUt&#ika=bHW3>-@HMJ=kes8 zaZ8MgxPON}8xOwj`mE=~;_<4dG^*Z!bfyzHXubO3u3+&r$Od!Nki#r}`MSpynlN>KP zvpC&YmRZ-{iJke(JxO@_dy84g_0f5!^f4WVqT?jXCE)S77tYSgqvKAy$w_6Ltb)$R z6U#jF(WH%GW04dMN&s-{RdKRLh&OBJ#ZKM0sw?hnLub=php!oytnPuiPj=hok;U>} zN-F-{)X+vek$=LSKPy_V*f`h#&hMud*L6Aj7Fqr#8nXLziD+5k*&~|U*i5`6w{@0S z8ddyi1`4sIcsOQ6dg(oGOmd9;`f#Di^0`CsZRe@fcx6Si8!*LW$H?yw7x+ZKV3e89 z9Uhq}={IcZicV}^k`tOg;LXht-o0tbSsOukP_J|aZvjcI+C$03?;AlD`1UO!3uANw zDhXc&A6{7BN68Irz~qF2=%$c`^%N*5GIlTnRWhrJ^4&f;@IkzUpBHh3albuRT*9h| zGN*}>TT8}3D06Y}t@f9AysE5WhO{Mc0sP%XaPGBXtf6r!ldCM#CVCUy8LehMpjBo^ z!CPPJo@{I7wvbLSOibLsua~^`P`SAfeFiB3tLT_WP%SxIN!pnYZW=)w_NQSlYuw{c zg?m0bp@|lRX>KjBq9uYbW#2xGKQ7LK@TD%xRp-`F!DWY}m?V+*RS-1%$%J}hai3<( z_oQc<-s?ALlb$aMYPcE!tPho3~K&7ncwFbW3!&9|=!PUCPs1urf5}wY7i{b>wo#gb0l$^SUR-axZT! z{{39$Rn}kkcnoI+#DjXeHp$-%47EXM(CMbbOi%z{f1!gcv2FMq{qF4GI+d=9uXapq z6Y2iG2gyvRD0cUxvp#r5+>fs2ymcYZp1ngU=wwHw`<}<(0;H(~C5{#mA37?|P7}@p z5KwYx2na2gYYCcV2ca7wzX}^s+>PXJauA+Ea@#X_ziL?JDIkM3-lVrHS}7PW)HiD= zoOPcHUPhSdXH(MQd>MGJigqEv(sE8)2#9cGiP$X=vgZ;{z=@Zyj5-L;?Q+7a>KHJ| zZtBC#)r*BU|eo&|G1 zyIdWi>5-EvSdpApP_TtHnCPb7`=kO<4YdPfH5=7#xIlRLh1qkkPU%h>kq@(Z<)zd+ zKHI8rUFCe5+l9lI2haapStrXsM`ih85Ix%otwbyULt#VMacC@?n?8r|0(w(9!{p<{ z&fo8VmBOtY4EED%1^7u0P+>;N%L@$LQq-kC`&!2WJ@v2kDTW|4a8J1_$8E}3PyJSM zdo;_kMK@Ml8X&@FvQPJ*_=#ww5p)<(_|RBAhEygXj^Fgir;^>}gWOLXCNiE-I@RyE zO>j)lIJuqcb<+0#l04#fWBLyHbPvQm@97a@d3+ef;ZNb}2^Gslg8+pmc&Ccffc{Le zvS^C6a!;O*B$x{e5*jG%$BPuLp{pm4GMx!JOCPev$6ELHHTNtX?050edzGDu1aKFg zbWB0OG9>I(>$(o0f5WYA2=CRt)Z-3ws|Vb+(UJ9M)(Y^pX_FpwYSX^r0fC}t$VD3N zy0}1$LeQly-@APh(zi_ft}Ub0?G}rO)?+fq0YUsZ1aQ)!pXfhBvNy{JzV)#KKyh4b z1><-cm%}>YP>XaS&-oVmQk*2KJ_^<=QI%~XoS|H!gIctDCNL~Rl%LVD=CA}AW+F2l z$+ZcvP&%=U}fpDPt%c1%R8cszL|f`ZaVm!k{& zNz7^%OzpdMnBGdrbXUb%cC6z-HgyWWwN(JLrkrdin@dq5sibslP7laSs*(*#C^Jvs zam+1Ek72hK`#m9VXS&4X_Mo`}T~+q6DTRlKx*kUsO^l1pIaJzJQ&OlE$)6@m)v)eO zGTmP_X**I*x05T6$?7AL33`1*DiY{i4-s{{wdb!g<#2b1bHxr={MID=n$_lJ4&37v zNF1o*rPQ!>_|?|H0XKMKDrWCRo?rbJskVe8FS4EdT^2DT%z0O=mv6o3M?B#Xc zD;2FDIg|}nuWRwOuABZjWY;=l@E-o~-A0kVK5<(vA-=Kwdg<6i|fXz>2+0 zc40T7b$_a0eEOBLoHtw=#HHklCxe~pntW-4^QIp(Vh%h$9&<%uQ}^<>kxr~5PumyM zeC%!f@|yqWP5#H*plN@*3VoMs32ThuJ%0J2>TiI3MWP4EDS7>{FfS*2|3|L4>CKAb zm{AR!6mu<5py;E-bLxCL)?;-~7L3BvuFt0j1I|&{khc#F5NzeHOMAob_pTjHlgSY0 z8z66+D`_(M;`ozy8{2z1->v^P5Kox?Oe;B46F8UJke>!kum}zQCJS~3r(|#qBYeL3 zfQ?94K)+lY5Cg!9{hoq6m2&(>uhsx4ItadQ#NyuYQ97y9+xV}dj`rSo1vqu0YW0Km zM14Mo{JweU>Q`B?O+s=8*L}oaUwyz=iJ2DRcR$KK1F|^F5nuMPoBKvbGieuyvF8nx z26MTn(}9QprPu?#n*r+G8ICpy5O;JK)v0#cscdIp5?5br`k>Sla%DOD#lBz`ti}M9 zh9cMQtMLn>J2_BduJ~jZn2~?(TZ^=}+-^xD1a)OJ#3dPfO0egNfzcAKZhuL5RiDtk z60$SgAY-q&tRLXVdYe8f1LlG;{6YU}QPcOV?mov_OU_vd`I+!_t)2VIl(!b^4lPbU z8dfc?4_%xb%J}KgVlpz|B^w;rlaRLE0Z}7<-hN*zHq$>Efrl$K1nzw1s=jLh-o{Fp zFD@NC-dq2ae;d7r1@C&RaMu@P{q*2e_1^J%cc7ZETkZYZ-%Ks!$jE1Ea7?dP9Wd0z zf_r;I?b}$^%|+Kx>r`X6hpurU<}VT?m%rdZ19!_t=3hMXAzmLncw=r%mqtx`uZA&X zej^7*Peo?n2xceqns_;+lq z-OG2?XTP6Vp$>~OHBv!CGdz#zqWf6#>S?Q_i*B{!*oU@2Ddn|hqB|3xm8KblXp~{V zgdXTGH<&aFB0An}OTs_UJ`UNr!Vy(w*~v1*Q&*)H@8u6pZewVu z7~@+9d3__kb4xg@4@elRt-JWgN!!w!hsE(+61e%4)O{e$|1^{Rzn^yvs5={hbO)-( wE-Xq?5fvl(Lj_69-vv(|_4n7hdaZ>d0MlVpC81%^fBo|c{=Z9*bPI<4H%~OhB>(^b delta 3115 zcmZA3dpOg59|!O`CkZRmXbx=%38~mUa~f^4h%y!x8RnchG*MPp_+v|Er!XrV|3?q)ZRp_^DK11skG%9JRWt7Nq*TuPT?9F-);27HeE}6 z?Af(`a$dbiyGjqVwrLht=xY@yqdv7C2|rgFE#Dh$W)*~c4l=%{&|wGEzBe*r70 z`Vwl?d}_QnDn$C`<)?Y$;63fzEL~W>KLB)eT7HXf;59UXCP7$o!*^*{CNe zga_}{j@5~aXKKjHUSZ0oY3e9BxETLcuVer1{JZI_TIEa07`rJdhW{4vSH1boExTmW zxAQ)In6#r?XIf8eWusm$#F#^-Z*WwA0^gify?K}kvJCYD*EnNW9zvR0ACCZ14OjeC zWu8qQ!PFQwAu=fJO)Bc;C{LPd*udl1|@P5Nd0r|5C$GJKRIdLru0&(SS%?L$J5)H4>xc+Dit=CY2wMNlIMluGZL_T|XiC=8%C77Q+Oi&?W~2K5Jf zM9nfEjl}0bDD<50#ZXu(`@U^-uG3=2ItId z&P4Rnk(^_z&7K+jk^CMxC)hd#T@d zv>54#CjZQeunlrhP()hftV8lx&6vat2Yrn()%}>f+E|mcgi#Kk_g9%4;+2eqJn14c zEj>sF?(*WT1m5@-1_?vekwS3&6k(b-Bm+4^g)f`}y@FVil?${ny%UP>UB!mDQxA72O?x2U-sr*1n{{E5x*^qLmQPbKnWxa{&hXt!aQtZB#Fg{}Z{$WY^bTT_X>DnNEhYEc~=iom%G}tpD)PzP5*juk??9j&$|eUT&C&g@GB- zKe$d)F{vKb`gSvhH0+=<11UrRN~TjmkaO}GXdr+gBK|HXL5k2W#|klT%XB*g6zy@q z`!qO^Y8{?Glke2@(f;vgsBvYN|NWsFzbp&2NwT^zu6~wtCw3wW8#KlpKPpeR5C~ChMjq<&_`pQE0UK+U- z>9A{3x!#lc|U&1A>v=@+fOtta!If!n=&<{Qp7&F9@AlOEzG+?|dkVhmE>_D}LX zgZ1Cu^e?Z^kk+nxSIQc2UhZe~{sY(ZTh()=PYc|;jyeF_eQ!)mQABp^x^xFC@JM*c z9hUWD-p{J9@0Q*(Q>^_-Iw!KN{J6_1AV=7BifBCn^aeVB(qUMUN|Tug?YNKC8fMN3 z17tD&JrECYF?UM5_2wsy>MB#FRl5NfowXfpd|2G3FM;RqR?+g+0UcfVHlM5nwGBSH z%10adXrK1fsvF3ZP6TwR*6~vk@f- zIkL};u}OKad~y@i$gtCX-fMjC-EYrr?cZM0Z*TPaNin5f;U+1raGj3ILyMN!GqY=P zc=nZNJ!#cHi@Q27N!llT0~BIH*a4BA-4^)qPP2WvhijKNrBk%F5y%AMo$s5vlKE(Z zSkH{fL@2-@c?$vBLiF6PO+;Rb# zmY0+yFDVg!-E=>+=8JC@EdcA;)wuh9tpTjpl~~b(8DG$81g+BY!QEkFU+5D?QIEPK z{-D$bQy>-ZGjK|Ui^W5U#Pl{GPN_sd^-Xwen`)WVS7C~F^#-wW!!+3zW>*5G3YSSq z3aK)N#3<1=WE?A6m7^~6(&(*OeudJ2<6<$c(kY^tj+NEjt~APx*4}MON~MPstM!QGZ+Z#H=J2 zTt1j+bzy^JP#!R9y&5O^b@^n8|kvB7EHW8ZdssCgOC1D&LL78_hp3yrLBTog)TK1dp{($eS?P`rRU zC35@**;}Flr%-t*Hdzp5)QqKuO(?`yx&M(q2z^lyU*CtbSHd*4-l`646k$H#Dq4Tf zJZHNob08;!A`0<3+`sjz`A@?glM+tp{%}4VQx?LgBJP=Vh5Po^g@>=rX~}a#L3&Lt_rvJdt^>BxS}5SCHdw z#-3IIaA)!|(?$n^Ipz*|v)6)P$v-S}kD?-N13q7Vo#D9cm9KbfD<@oVxuL?uY( diff --git a/summary_results.json b/summary_results.json index f4a36adb0..aeab1d48c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420968, + "Performance_Result": 0.421799, "Performance_Units": "Samples/s" }, { From a796a92ff7ae7614e0584a883576812b5918dd66 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 29 Mar 2025 19:10:39 +0000 Subject: [PATCH 380/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10226 -> 10224 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e8e7288e9..768113370 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 19:09:09,786 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 19:09:09,786 submission_checker1.py:3260 INFO] --- -[2025-03-29 19:09:09,786 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 19:10:34,868 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 19:10:34,869 submission_checker1.py:3260 INFO] --- +[2025-03-29 19:10:34,869 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 19:09:09,786 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 19:10:34,869 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 19:09:09,786 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 19:10:34,869 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 19:09:09,786 submission_checker1.py:3276 INFO] --- -[2025-03-29 19:09:09,786 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 19:09:09,786 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 19:09:09,786 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 19:09:09,786 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 19:09:09,786 submission_checker1.py:3297 INFO] --- -[2025-03-29 19:09:09,786 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 19:09:10,871 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 19:10:34,869 submission_checker1.py:3276 INFO] --- +[2025-03-29 19:10:34,869 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 19:10:34,869 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 19:10:34,869 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 19:10:34,869 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 19:10:34,869 submission_checker1.py:3297 INFO] --- +[2025-03-29 19:10:34,869 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 19:10:35,900 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421799 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418679 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25985 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index fb35a14c0..617ba1c15 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1007 +1008 diff --git a/summary.csv b/summary.csv index 0a34e7a1d..00b08025d 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421799,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418679,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25985,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index a1ad3cf85..79f5d13e2 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421799, + "Result":0.418679, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 82fb7d4da47d7d6d2d706821b343d79048950ee4..b028fb8b488e3d7e95813da5a4d5e06f44b09776 100644 GIT binary patch delta 2433 zcmZA3dpy(o9|!P_;cQ`JE+ul=%>D9Xk6p>RPPX!|TKNxM1JgYdHpm z%L}-@tH-R{c(Fr${Ev>)zF?(V{c2F|$$phwx3+KX<J9tDh`U!(+4Wj zu?!a^1FKVn(B5$z_{Zo7t-}uB3KyiJlTML-WhY#NzYg(4jxTX*v>wfaJ>7{}9*3!g zc&~kLp6(?ITl5EdQr=qL4H)Hq2->^g#_N?m->}m>RDFVVQsQYs+#ZL!vDSR?aP=np0tS56S9`I zvMDYl7iTs&)oduOzZH^n;2h>fE_GkYj#f#HDnUucy6w(4E_?zy;g9=GcjM71XtdNV zpSCi#(7)brcXY>ZJmJ?~hF|6LMqTz~^3f0+I$d`Fz`m8iAy?h-V}R0$4wwMxt92I&hj+=90q8MydG%5KqLxqV#!a{*S9Z40JgGtG z-CNISu9w*5hxmGhbDZmEGh$+m>bb;Pb&E~RbmOym=iVX$!{nEfA{Hs;DuEVWSp=)R z9wa#ZMJ8yz?d%O!sK zKv+uM3IA>6Guudj&7Z)DDMh(L9NHc={7J?l(22kR*+)2tn=Z*|>e4HNSrGkb+2f@+ z?}^gkEjvJSt{T3NQ(tD&T+M2332($l6hNs?HyWQTAvw3=-EgQgh2n&C=^m^cX16i- z-<1D^%s36ZG;>cMz_iH%xX$`#8LVN`N(F{Oqpq7R4+~ZU+L^=xALkq3PPVT`WqkEX z8VM#)jMd-*k#x;tYMsZ6LTMBo4O=Jhd&m*nA8%=xMRm=LqvZfuSY+9+k#~MsaEE;U zboQ{S=^w`-I?R!2GVjF~+~>hnc*#YsRe3_o{Dc?9+144gkxKq?H6fDSx( zE%+JTLjapW-vF4M9*Co$J3aj!hk>X(*`vQ~#kk+-imYOd6vGC!y+AmfXSTh$d|O+W zUFDP#>J=rwTwx6U@*k9!(VtC%KByZGD1_mC#OC17Xcrs;-BKDXHP6@ zcg8LB8QkB9X4&f|CAlb>2GR3mRPQQ}B<)_%zka)Y%5k2T5+H00NWL7(7jcKe@KiB3 z_Jhk4bJz(N!@N@%W1uJ1e{$I?WFS$2b9L zrh~0h-3@1Z6B4XUTLkTINotMNZ)l`b(<`FQV?-f~T@mdBH9@s0dFvjC(OU_Mk#pA+ zuib-qaHxhFY6_3abgFB3H?zFA#Ev-I#L($ACakuO=$-DN@=UBFwQEnvU;Vnjv3f*| zu4$`bo3lq~Q2@6}AUbbU6tlvAa&2I2?MKR2pf)Ht>xwa6&dC3C5366ahtFZCH{qkaZ1Lns6$7~Gr@uoR8P$I=r#x1en-Rp&)(G4rxFu6+qXA2$ZbkqRgs#BUK z8}Q0_X`A$Y+g`Lx+N8`sMRQ*#n8TQ#^^IQNOzFAEO}oB_KfmtZf7RDA&S6C80OuoG z(SO0H`9^Yf*env;{4aC43eh(uqG+6^3qa?|+NsIfiHG!wFThJfMT)>)-2N&`ce!Zu z_brIw?Yl~>*RyXDmrG~g3o{f`5v?m*uzWo}2Ak!{{=Q`{XODZ1Xu~HK4bdnPo+`)r-6L**omIU_RcCS~fw)B_zB`v}-OdPf=cH(R}p-o?~!_>jGntEocp3X>`V5YGgHd3(;gOvhhz{qOX)=Cld~Ckpf}2uPv1~HLlKdM8V{;m?0MQ{GgKZh7*%q3Uw9Jlh8ut zU!Z)-Ix=roy4pSb@BoiFn0J7=7)|P=HUw!$Mcyq$Hxu|zH^UV73f7a)`R)1kW!jph z9B8?9K5MWSk)Mo~uZmwv9ZXqU6L#H9LCbBM6=wy!cuqCVEH=AfHn*dCbbHV&Cyl7C z7B#*8`aX|nHY{URSCy4~{>a{oGsij>jjd2mwM^dD*&aj$@C(SBHsGmVkLkGgL*}?= z(>uXc6(i8fxw|W^A+Q|5j^?nz@S{JLzy97#hhLGWI^8!f)&&9po)Q3n0ssIY#D#m3 zVOy z3?SSRKA<2N{x90Dbw98)^s9a5l6)jC=VcJomv>FBiagGG29=Ht@c1WGAbGSf_1g-Jlcn7#X8dqmCsands$GUfFS15KDXJ=FgH@BhEIHRc?~usZw6bv80_Sf`NXvM^h@Cd?)GC3I@hX<i7G7|A6=Nc|G6fdA*(mw-0VrRK;xpi1qq;!EcJ8 z^7P!v>CFzCB5Q>T^}&%7?0v6X57I~dtl?Ss;ae@}Vk0ly zdTN!5GRq64-dI$3q>(yx`oz*4%G|Dzin?3ua=JlX8UEAJhoc1D~nkqpf8brdaf6;5Rl8`kPV~A z+cwWP-r=@dT=O54AARBDf2iHZD*QuIU)b-X*Ej}ZB$<78i*=E4c=>QHSR4b-A;19n z+aM!_^i*iL1b*`jmt-nthwMavDKNFJ{7$+?!5o1 z61JNjWik+LEE$DF=&~p&r0N!Ec|VcuW>BNdvmFgfQi71|o4YO!NOK{~J=S zURU*5hz9^g1p}`?7l6IqGB)?IjrfK+hh~nGHun^dl}5=H z!-BNSfGEaICz_&nPH|ei90~ptjlDctu+k9tm{SH}KxGnV-$+lIIbI2RVMCxmcmwLwa^RIj+mWhp&b0-5W|2%FuU?*z`z< z5ZVtYz{^wMbxFQ*b^_76(#Ch+QkFpCbYNf;KMumD`eCS`d))E0W&B3K8yQy+J8~m; z3F{->;;cf6^)hb$8(057ZumPlFR)rmK@HK9T+k2VA+pJjv3LUQDA3$UN@jPWV}pD= z!z=u#qYI^l@%@%mZOL4uO2Hegd%W>9O2F)@9s)18G@f=GaEJ@UUfslDfF*uYtGyGe z7q*tkp$GH?3Tg6V4JRFYerITYX*K&3=`(ISZS=jMXisJ2*u~o1R20oQ=J*<24{;Qc zoDS~Agsg=i>{t{)r+yQZ2%x}!O2+O;mJMl2C4g&shpPAv&TfiYy*{Nr zYs(PLL@)Q&>U!>-wv7mrK1=J@MS#(b;jsTaHn$Driw@DHb4l4V(O+pRbIl6pMuPWA zuL;iDoi)>uI8g0XeXeq*mkt!9spqPIh*kqfS-Sc<^v>uebJa`llaxy9O^0vs)N18p z#T6GE0{u^`_o38klCuY_lQ(u9eubRyy2j7{oRHvAH^l$t)3zC2ucpNEBCO|yVuet1 zox+(la7{esU-g`F+a0QHI#vR+27rM~e33joj|xC8E9nA8&tUgWb^=IyOaheT_v+n?!|Q zesC|oyjl1gSc^KIEC@E}l>2`Fuzp67fecnAYClk&B|+H(AG3Z$TLaBl&S=#4mYzbx VfvT)@G!7WfYC{`<;f8>%_b+v)VRHZg diff --git a/summary_results.json b/summary_results.json index aeab1d48c..26c11b4a5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421799, + "Performance_Result": 0.418679, "Performance_Units": "Samples/s" }, { From e24f0b3981f19300e0207851ad5b517be0625ed5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 29 Mar 2025 19:12:52 +0000 Subject: [PATCH 381/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10224 -> 10225 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 768113370..5087c6fce 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 19:10:34,868 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 19:10:34,869 submission_checker1.py:3260 INFO] --- -[2025-03-29 19:10:34,869 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-29 19:12:47,270 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-29 19:12:47,270 submission_checker1.py:3260 INFO] --- +[2025-03-29 19:12:47,270 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 19:10:34,869 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-29 19:12:47,270 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 19:10:34,869 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-29 19:12:47,270 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 19:10:34,869 submission_checker1.py:3276 INFO] --- -[2025-03-29 19:10:34,869 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 19:10:34,869 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 19:10:34,869 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 19:10:34,869 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 19:10:34,869 submission_checker1.py:3297 INFO] --- -[2025-03-29 19:10:34,869 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 19:10:35,900 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-29 19:12:47,270 submission_checker1.py:3276 INFO] --- +[2025-03-29 19:12:47,270 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-29 19:12:47,270 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-29 19:12:47,270 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-29 19:12:47,270 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-29 19:12:47,270 submission_checker1.py:3297 INFO] --- +[2025-03-29 19:12:47,270 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-29 19:12:48,225 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 617ba1c15..6cb386934 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1008 +1009 diff --git a/summary.xlsx b/summary.xlsx index b028fb8b488e3d7e95813da5a4d5e06f44b09776..acd4877348b1949cadceee6a1937cb7d857c4a2f 100644 GIT binary patch delta 328 zcmV-O0k{6}Pw`K%93lnLsf?K>lOZB1f7|IR!gio7Sv63oOGrH)XVr0Mb1;jX`>aC=l~8as+)ot z!nB1qs|{E$z(oP19ATzJJMRs_*s&c*>XP74LELC7sSl;yVnf+C-2%?j^d3v_f0R>C z@evrCk%$H)XCu=5*ro$;hM@ro$~&AT89sYGSm`DkI_1JCg+CaWw4TJ#y?0_Xwr!iV zv(6gwXZUmV`rfS?3w1md15_295oUpUZL0!b7U6IWb$%62yb-{k4nd!I8y4yNfJY$! z>EoW1_3ZhjUZN^b^Sd~m#reF>8s;R+$?e0}apYD187PM#|0bRQeeUZ|Q2hb3Vj~L# a1<|RDnI@BRCsYArlgB3`1{Ea$0002yd6x|U delta 327 zcmV-N0l5D0Pw-E$93llpv@n+?lOZB1e?LfRgzZ3EvTC4EmzM5?G;u_&2a;OHe_zE; zoCLD!a&+&UxihF-?PVAJf^|Zx3S~)(B2Y|oq1p<)H|zKTMb1;jX{QxbXaI*+^5 zVzh?*8UEb7yp5~ILLE-U0M#XDgjt|o+q%S;ML1kXo!^8LZw2tjA?P>Wj*Covz@rd= zOmR=jNAdLBtWlk(`65n>IA1o|8IoivDHdOckyrg^pga!wH}M3Ra$kRf`VX^XBMSrt ZMYJ%NC6jX}Q~_d>$R{HP6eR!v007BMmtX(@ From ffe3f7cd7a2f550b9aaf6361b181586514921862 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 30 Mar 2025 17:23:31 +0000 Subject: [PATCH 382/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10225 -> 10226 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5087c6fce..1d7131059 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-29 19:12:47,270 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-29 19:12:47,270 submission_checker1.py:3260 INFO] --- -[2025-03-29 19:12:47,270 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-30 17:23:25,870 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-30 17:23:25,870 submission_checker1.py:3260 INFO] --- +[2025-03-30 17:23:25,870 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-29 19:12:47,270 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-30 17:23:25,870 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-29 19:12:47,270 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-30 17:23:25,870 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-29 19:12:47,270 submission_checker1.py:3276 INFO] --- -[2025-03-29 19:12:47,270 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-29 19:12:47,270 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-29 19:12:47,270 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-29 19:12:47,270 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-29 19:12:47,270 submission_checker1.py:3297 INFO] --- -[2025-03-29 19:12:47,270 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-29 19:12:48,225 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-30 17:23:25,870 submission_checker1.py:3276 INFO] --- +[2025-03-30 17:23:25,870 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-30 17:23:25,870 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-30 17:23:25,870 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-30 17:23:25,871 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-30 17:23:25,871 submission_checker1.py:3297 INFO] --- +[2025-03-30 17:23:25,871 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-30 17:23:26,918 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 20.0509 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0925 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2359 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6cb386934..52a9f4e2b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1009 +1010 diff --git a/summary.csv b/summary.csv index 00b08025d..0f04682dd 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,20.0509,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0925,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2359,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 79f5d13e2..a49db047c 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0925, - "Accuracy":"acc: 75.000", + "Result":10.2359, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index acd4877348b1949cadceee6a1937cb7d857c4a2f..ea6b2d6c1a586edd861289594b31670062595167 100644 GIT binary patch delta 1419 zcmV;61$6rHPx4Q&%>xS8@R2)l1ONbf5|e!h8I$k>K>{3mvoi#40)MYLw^qp_2vu9r zl3he&suh-1t9%udg~{T{ZF_CIwf_1qLR!S>%KTzRL&&Wykb3}$# z|9ueGtclBMKFIA(-?!hbdRwCF9q65?LNQZRbDvv;*=nx1L9Rh)(Lm*Ie{T-r1 z=umV!=$CycIyf-Z9e-$|gPa?k1G2tFtb+31_5X*W`;46aKy(lNEz#Ld*Y^%T_|*Bh z>U>>wpg}m&Asw*LP7nxG9Rh)>Lm*If2n2kY&-@*#L+Bveiyib!AF2)xOmzoc*eOBI zjm`lJ$t_|PwnhHus;*jAg{WF`Bdtul_}<3??EV)4enxtKAP>N~|4x98-F7=YxV5p~ z?USts7k|C@FSy#w7g4HKwjgUx4n$h44F01zqVNWjN7q)y!gy9?#1OBg+|rYmQobXV zt7$|padR4Blq{tz6|aqVbvjaI!8oVUejQX*q(3UMzZdR#gT2r@_+q#R#>kMFZ=f=} z+Dm<0fP^(3mcfc#2W*uV+{j2Q6SdoNtzymjMk0MJXjxy$W_?M8p!w!5Awfi#x|6%x zm{PI1Q6+ghHX8o~vtkedWk|WQRpR-UI^FFZh%cY)6yi^A)FzRY_kBkp+MKUHEU;ZY?X{%U$J*>9g3 zEhfH6pY+wju&fB5;8{ulC?i~547tZ2lk?-`d(XtWiQVSA2khMvPY=N4u!>+EvoBGf6rzo z$ntgDmp*r2#yz4F+uNHY9o=ypcW>Myy6u@_9=w=bFE1z80&%2|XI|UEKsj)UE<3k9Im6zufvJA0{GJ* z=r`Vui%fmMqY!{haZk!;@%+-PP@SjweVi6?kv7>G1IY`LFE+!-tNt@k9*6v!cmhnh zuRlTk2eV@%gaQTE@R2)llk6p10%i!4fe0FtV@v!4JSGQFOx_oCL6LfJl-V%002+|000pH0000000000 Z004l25R-`~J^^Ht$R{HP79{`x002X_rIP>v delta 1433 zcmV;K1!ns4Pw`K%%>xQ;dLVId1ONbf5|e%i8I$k>K>|K_voi#40)O3{TdQOdgsQD* z$u6QX)e6h1RlW+!!esH}w!OC9TJQdgkQQ;eGQXG+5kZ=9^I&Mj$mVwc89B**j>ypJ zzYpS?HE}u3hTQJ-ef!<2wdD&&cTyMEB6&5}oaIeedvtt41fHf)g5$Urvy1S zItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50S?ancLH>z0uNod=~?f- zldcCBe{b<$aP=`?M5$WYf~+|?5NWY8_>bm@!W&E;U0WFo<5`svL%f#qlb*bk@*SyM zO(S}Vo6`uRWGQ8-cx}9^(~&9*#yO4l>!7M4{ZWzqy>QPP?1kRJ7sEX;MuyCM1C`m; zUh3llB&_kU3|8biV5_v?Mn+H8nBI;{F%lb+->q{yG%{O-m2_nMOo&36u zDHWR=Rg$-3qw!C(W(c|r1wWj+BY3kV6{7-wic&KrQwb~sDQimLaYj1~D=2wK0j&%E zPX{bsLyIOIMPS*=yEz+E=D)=dIogF zNXVl=jGo3yje8uDlObM^Y(tr`n;JQV{}6`T=ffqHGL0$a1C{S6;8Ynz@H95 zpLiP<>HC03Apq&)o|N_M`K4Z>Do^vfIGx4$yv`ctB+JR|!`E@-RsR_%havwao&bIB z>rYVq0kdQygaQR^dLVIdlkFv20cDd(CPe{E00000Y`2-V diff --git a/summary_results.json b/summary_results.json index 26c11b4a5..27fd2be62 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0925, + "Performance_Result": 10.2359, "Performance_Units": "Samples/s" }, { From cb401ea816025ac54212382121537f0eac4e8dbf Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 30 Mar 2025 23:13:10 +0000 Subject: [PATCH 383/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10226 -> 10225 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1d7131059..e89c349a5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-30 17:23:25,870 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-30 17:23:25,870 submission_checker1.py:3260 INFO] --- -[2025-03-30 17:23:25,870 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-30 23:13:04,934 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-30 23:13:04,934 submission_checker1.py:3260 INFO] --- +[2025-03-30 23:13:04,934 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-30 17:23:25,870 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-30 23:13:04,934 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-30 17:23:25,870 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-30 23:13:04,934 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-30 17:23:25,870 submission_checker1.py:3276 INFO] --- -[2025-03-30 17:23:25,870 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-30 17:23:25,870 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-30 17:23:25,870 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-30 17:23:25,871 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-30 17:23:25,871 submission_checker1.py:3297 INFO] --- -[2025-03-30 17:23:25,871 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-30 17:23:26,918 module.py:5256 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-30 23:13:04,934 submission_checker1.py:3276 INFO] --- +[2025-03-30 23:13:04,934 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-30 23:13:04,934 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-30 23:13:04,935 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-30 23:13:04,935 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-30 23:13:04,935 submission_checker1.py:3297 INFO] --- +[2025-03-30 23:13:04,935 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-30 23:13:06,008 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9043 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.6721 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 52a9f4e2b..d488f1cf4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1010 +1011 diff --git a/summary.csv b/summary.csv index 0f04682dd..6d4810906 100644 --- a/summary.csv +++ b/summary.csv @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9043,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.6721,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a49db047c..6c10d4e53 100644 --- a/summary.json +++ b/summary.json @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9043, + "Result":23.6721, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ea6b2d6c1a586edd861289594b31670062595167..8b14acf29231e369edb1883d647dc5d8a34cf71b 100644 GIT binary patch delta 1337 zcmV-91;+aFPw`K%bO;1a<7TL{cnD($e{R-oN9R58adW6dM~E#=Jw)S5R}ax3l_h*V zOaoLBZw*jcWbI*{0F_1N9wogur_kcFQ393V;UQ7DcqJ|#T)b9nIG7Tl#y5#+3#iqCqN4SbUfUs3aOkQd#8kVF`fBBA<@}qYt$Bf20%Q zC;+n$2}4H#bNhg!pkliZV)^qe1<%y-bE&pHKXzSPo}06AsZGwMM$?Ebojye4N~aIe z0GCW7xnvs2CDTYQb&=DDC6HVeC1`7K`ap|MI>C)g2B#iw-3_*nnVL=7P);`0zk9K z?W4i%11&!3grEs-9})&DS(X}Bvbtik&)-AlGj;o1=xw)8x|(g;o|W@7R_h!z(K%vE zyARR0((Xevh-L}957PjeL<49R*?m|7K(om1vpM*HCfI#QC%AP9VI3LShj_=p3=7--l>i>GvTTM6-n7hiL#!q5(9E z{5~uJpjqVi(ct%i7N2xN&;-8^3By^F*?nNvtk~}J_mKHa{XQ3Z+wZfSHLYgeg0_`| zCVEF~Y4{-;R~mkZ2GJ~GfB0b`IoGw7110)KZ0YqO8c%wChz6l7;q_q}fRc`s5SoJ&DNYYV^^zAW}EwJ!NC-)rZp{8U`o9WD*;_Fg;zX z$W80&eURsB|9bW6f5ZoQ+Fv;9kF$3@bSJA9JtpJX(1`zBy+YE?ULk2_$$EqH#UwD2 z@AKfW7TnF#ZzwCaX7uj{tCvR_^JpAKG=f1|s9ASvMznez21Lv43qmmvQx5ot>@8y0ken`mJWX!^KMTh0RR9{ z0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!LJ!!Qtp?|llRyDix!r3l-Bw&c`6p)M`G z32BpvT0cl?9lw1QCvg(+smImKemlFUTJBYs{DSk+=o;l2K?!JXgw)#_y|=6M0VO^# zEm&tX)Mx;Ymd#zoE#-g4!K*VCT#(?C2-2Q%TcceFmSXJr4ixiQbm%B08aVYhEch4Dj5a~ zv(~dXx%Xa9#=h^fz8tMFe~v%bFK^?jsnmy4aX?KaIOPslFs?_b@O2Rn*D)W~@x&Vm z;&F)j%-d;^V;_hl2B62dXXT@OdTv*!DM+zMNtu?UElQe~lq|juBX9aIKy@1OZ{itn v%zgU_nm@B-Bgz5E0000007-UW delta 1338 zcmV-A1;zUDPx4Q&bO;2X5oDmVcnD($f7W%+Y~?Lp-5e^>5n@YI57D^N)k8E$WeHyo z(*TvkTLV-US$kL~KxL7+M@jF^DYW=(lt3kTct{j3UWtnb7q1oDc>a1QpQ*X$LhZVH zTFS9nd5>Ye94gT`VoQGy(YVs!Lo`Tb35yTY0F^}JNGgk5J}d!HS>*FkVDy0&f1h+h z90g$ZAz|n!U~V696jW^YK`ej1rQn%belFFv=f|#V%X4!!F15+I)My&9rPGIKT

7 z8sL&?B$rGhxnvs2r7m*%umqCJq6BRXP9JFTNhi2*31J-?Cg)2gf<>&Z*y{5bm_KFf zvXUapEM)bz+b3Pfx-BhCmECHZd z6tkXWpy(LO>~af((gkwuJrp54We1X@53~JCeZ+z zMSdTa0MIP*`)Kg{K#Na0A!vf%hlJs*$?QHbYgTOc`FqHGrhcCbz3ul|&YD&;Z$aD2 zK@+_rwlw?@jVld5M1yFSe=z(o4WLOhfM${5ha~_siwr*+3_sA~lTN@^08t$l1!R`- z`{;`OK97<4zxoO)sjn22((BW7^PKD2%7GGnBewMV5RE6jK1731mhk#84M0i8BezR* zk=KWH0#FuteKdG|pv5PhD6=K01hWqf11y=_2bQuGyL}$RG8tqyAZ6Ckq4>Z8Hx11&!3 zM46T(60ANn42Wb_A0YBMS$*<}^qxfIb2a*CTM((6@}9D>_3FcE5DkNqU^0n~UYMS) zRph31^*+dRwST>Oe|6%6Jnb)>^~c$}9=em&iyo8lY-q%Pu3jN&XRnYnvt+%&`C<|n z$@h71SPSmv={J-WTQmB1gVoC;jd?T2KQCFiFDc{a>S?+Yh4buk^NH5QfS9AK59{yw>yRzq5!FmJWZiH9X!W0RR9{ z0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!FH!!Qs<_x%c^y)8dzT7>ODTe51PP?whM zgfwwPtp}1?Cx2hrPMid?>T-1NoVhcoT<&ET{epEus|saFiXu=E$h3I8Nz?Gg*U4WSTDdu0i+yZrb0XK4Z+y49Z2eu;7~!_YAdM^rERgH z?3=cL^E6#x37&H5DIS2anThC#B*SN~2P@sphEch2O5qO% zZdy;`?A|*u8~eUb`eL+({2BgQzrK&F#zGCJVu0$BGr}xTuWd+O;>#i&uA|Pc!-=;7 z_|qZiH{OnmOntzk5P(c^Ps(TU{L-vYou~PIoECABHrWHo3z9E3!^o@tGf*Cf{F`_J wOu4T=LH!4_VH9X!WlXE9j0c4ZNCnE+HB>(^b09oLH8vp Date: Sun, 30 Mar 2025 23:16:01 +0000 Subject: [PATCH 384/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10225 -> 10225 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e89c349a5..285e7649e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-30 23:13:04,934 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-30 23:13:04,934 submission_checker1.py:3260 INFO] --- -[2025-03-30 23:13:04,934 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-30 23:15:55,550 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-30 23:15:55,550 submission_checker1.py:3260 INFO] --- +[2025-03-30 23:15:55,550 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-30 23:13:04,934 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-30 23:15:55,550 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-30 23:13:04,934 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-30 23:15:55,550 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-30 23:13:04,934 submission_checker1.py:3276 INFO] --- -[2025-03-30 23:13:04,934 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-30 23:13:04,934 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-30 23:13:04,935 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-30 23:13:04,935 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-30 23:13:04,935 submission_checker1.py:3297 INFO] --- -[2025-03-30 23:13:04,935 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-30 23:13:06,008 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-30 23:15:55,550 submission_checker1.py:3276 INFO] --- +[2025-03-30 23:15:55,550 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-30 23:15:55,550 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-30 23:15:55,550 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-30 23:15:55,550 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-30 23:15:55,550 submission_checker1.py:3297 INFO] --- +[2025-03-30 23:15:55,550 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-30 23:15:56,609 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,11 +33,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 20.0509 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2359 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1099 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.413865 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412385 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d488f1cf4..2d1420d53 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1011 +1012 diff --git a/summary.csv b/summary.csv index 6d4810906..2dcc4ae88 100644 --- a/summary.csv +++ b/summary.csv @@ -11,11 +11,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,20.0509,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2359,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1099,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.413865,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412385,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6c10d4e53..5e3464fcb 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2359, - "Accuracy":"acc: 75.400", + "Result":10.1099, + "Accuracy":"acc: 74.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.413865, + "Result":0.412385, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8b14acf29231e369edb1883d647dc5d8a34cf71b..30aac7b2211d277f2e5c97f696ef61c554c33abf 100644 GIT binary patch delta 883 zcmV-(1C0FfPw`K%%>x8$*`$xN(F4{40_;PRsRzpi9Dy3m>$7H|bRC#z3Bk=~Pte6B_x zZ3`lGQ{GcHwqAWW4WeOi5==lw`maI28UrYic`92R0Yr)+-{f4q)Yes+nZm@cJq%n`iVKh#@ zuX@kkUb&cbkEUV&<#JGq+4#>z@bX3Pg}wixXTSQz(65X`BQ5=2npU^$CHugJb#yTA z4uenKXgHq5#v~Y|g_?DzW<;ykQ7%6x;e7ci{Z0ECCP_HG|7#R<`$3fbmEIN%!Z5l2 zBReIV*LoiP7f`c_6PE!5mCF?Ivm_Oy0s~sQW3$T_Qwo2+DN9);0RR9{0ssIJ0001F zZ(~q$Z*X%jV{dY0E_iKhos!FH!!Qs<_x%c^y)8d*D8hE2Em<{Cs7p(CLYgF^)&ohc zC*5P%%wo|Mnx`K4WqLIoUw8qMpIGA3IAhm(#bA{>>=6!Cck001Zz000yK0000000000 z004l25fzgz6&91?CMp5ElMN?20WOnBCng)dDN9);0RR9{0ssIJ00000000000001h Jfe(|3Cup#Fkx>8u delta 877 zcmV-z1CsplPw`K%%>xA2@R2*S(F4{40y{mEsRzpiPUB{%?6Y|Yk_msO>E=1twUq-U z`bKQ&^&uKhdVPonp)BF`VH$vvj7M&l=pwHV>ja=I^7?4-`ap|MI#FgzQVC`s8U|Q0 zw+}33D|Y)lhGjCyh*VE{BwI;srB<-pR-g3F*>kmk<~|o)dyO9(up!HNhDZ(Xc!R5tUf^GbF%v66X`vP$meSG z(Y7E`H|0HLW9!w2(;ylKC&6SA8@(_+U8~4V>*{@w=W73Y_3Fe2dD>q%>yNW{J#;6l z7dOZs?Ujp3_h=gSUoHo=n2rBj1TSCoUfBCDdiJYd4E@SDG}6-VrD=7`Ua}8tSVsr* z?lAb&jfUe{Y)pbdTBuofYDTnr9p&08aVYhEch4Dj5a~v(~dXx%Xa9 z#=h^fz8tMFe~v%bFK^?jsnmy4aX?KaIOPslFs?_b@O2Rn*D)W~@x&Vm;&F)j%-d;^ zV;_hl2B62dXXT@OdTv*!DM+zMNtu?UElQe~lq|juBX9aIKy@1OZ{itn%zgU_nm@B+ zBZLA3*YJ@$lj|iy1y18;sO*z6CR+iBla3}L8}LCH(s={`04Nmz02BZK0000000000 z0DyrKlglPS0lkwACprNylSn5f8yfR&Pb2{V08jz|01*HH00000000000DyrIlZhv2 D!%>Yh diff --git a/summary_results.json b/summary_results.json index 1e43d9d9e..5cb8317eb 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 74.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2359, + "Performance_Result": 10.1099, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.413865, + "Performance_Result": 0.412385, "Performance_Units": "Samples/s" }, { From 79c66186da210af5c5fea5a8b46c3560e3c235ec Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 30 Mar 2025 23:22:08 +0000 Subject: [PATCH 385/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10225 -> 10216 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 285e7649e..66b3b4700 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-30 23:15:55,550 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-30 23:15:55,550 submission_checker1.py:3260 INFO] --- -[2025-03-30 23:15:55,550 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-30 23:22:03,212 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-30 23:22:03,212 submission_checker1.py:3260 INFO] --- +[2025-03-30 23:22:03,212 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-30 23:15:55,550 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-30 23:22:03,212 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-30 23:15:55,550 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-30 23:22:03,212 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-30 23:15:55,550 submission_checker1.py:3276 INFO] --- -[2025-03-30 23:15:55,550 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-30 23:15:55,550 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-30 23:15:55,550 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-30 23:15:55,550 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-30 23:15:55,550 submission_checker1.py:3297 INFO] --- -[2025-03-30 23:15:55,550 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-30 23:15:56,609 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-30 23:22:03,212 submission_checker1.py:3276 INFO] --- +[2025-03-30 23:22:03,212 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-30 23:22:03,212 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-30 23:22:03,212 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-30 23:22:03,212 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-30 23:22:03,212 submission_checker1.py:3297 INFO] --- +[2025-03-30 23:22:03,213 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-30 23:22:04,228 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,14 +26,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418679 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25985 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26733 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 20.0509 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1099 | acc: 74.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1665 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2d1420d53..9951021cf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1012 +1013 diff --git a/summary.csv b/summary.csv index 2dcc4ae88..73a1d54d7 100644 --- a/summary.csv +++ b/summary.csv @@ -4,14 +4,14 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418679,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25985,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26733,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,20.0509,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1099,acc: 74.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1665,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5e3464fcb..687dbbb05 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.25985, + "Result":6.26733, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1099, - "Accuracy":"acc: 74.600", + "Result":10.1665, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 30aac7b2211d277f2e5c97f696ef61c554c33abf..4804d44d54d0b2f3c5f4de30c17978294fa07c4d 100644 GIT binary patch delta 3854 zcmZ8k2QZv%*M6|-8;jL~L|wg$5WV*nC2ELn5riNJR*T-3RimsRx>!+PE!rxpm#h*5 zAxhLYf=Is2_y6Db&-`cRo_prZJ=a`wo$K6l&WX{kQ6rX!;*h^O;~5A5LLi86GA6_^ zkqV(=5c(d-n-IQQXj9`%6H{FW8xcW?&r^(5KkB9!{os7E)za0nYJK!8DV@Q$L!u{_ zhDIgd_skwPi1`xun@MIQJQXvw*F*I=|GuvbHg@IPdD!)8bt`v=qDV)#h^!c8n!h$_ z{V5qJ5ge&J%|Xp43nJ85T;q_90c*5IhvVW<(L|=Jk#HVaO#8G9`@YS90N;`AZ#ZAa zB<3rX`o+{5>Tp1pVr zE{%RF8aO2y_Ez?R*!*38hf0u)v@EF*v^Dj3 z<9jpLLM$|ROwy_Xe=UfuX%?Kt=H1JcUi;j$zUR~( zZ}Ps0*y_0-`;4Q@F&mcVm2|FrM~ei*5L$pMHb{7zS0R{xTh63CU-KLyT}h)kVlrxS zvJ_-d$@PdK`8o69qRJzUt1`~(mm5XWAOGr#7L9#rXPA+Nr z3FtLA~zYkp8>9@1l zk5`sCr)riB8omY<{CCWU( zga-LiK{Sg%=^*cGK2yVJ>IDvm)6z{hq_v`=?6SamaPOrUzpMTCX!|0&bmKcK@RjP0 zf{7%hWH)Yl2+i}Y!ghoZ=02LU zbp8C8O24jHVbOMYf(YTGa93~zj(zu(oX6viR#XBr2mu51Q3PVf(e`_sVmgEXnqB4P z^J4;i+-HO#S*}E_+K2Tj0X^XhlZh6J9~V(p8YwQU&Gn=$?N!bB$T~mFR_B{x3TZZB ztxO#$K3g3gk90hoL;?v$oiY^;uRLli00*C!*BbdCOAMs|0AN>y;aHYSG%$Kl{Gj-MDck}FY%o)E!o2Y6L zxve8%yqTW~*Pmn0QzvqUP^Kp^hyHM)&%leChZnUVn#y^Uq1d0$hMjue!5H8{6BFbv zph--h#{YbPc$8(4sN?y9Y5%<+@7SG!R8`V=;V@bCP2Do^GL4L<$d7x}BHPCD#uzX4UB$rAbMLd8MVMY0tr^Lb1M{-OLe9n1y~>0=}eV1>Gxe!_q;wG7w*1 zj~eRPq3#{|w!pfNAI=ulF7;Cs7(EtovnHwYqh&K=r4R$ImhN12fXb+ zLSxzCxsrn4uYU678!ffhzQLFvzO>-9r1vY_zQ||HWA8{z-*?n&pqg%**;kX)b|r71 zng#yYl_5CHl8_yZ*)QwcC&KqqOggz8UPT0tP71 z;Akq~vhcCurE-W7;c0tFXtQv{oI$V_?qkUIhGcwfpN?4GfTpOv(V~FgY)1&yR{K(q zU6;g(O3`;XU!0w@ zj3(AG-E{Wtm1hjeGsjIoxa{6jc)-m7c98YdZYTKs2|A88mw-HPv0L>yerBOW@X-swLn~D) z-a;Q%ZDi`&Cxz+q2g(Hvx4z%f|8bqIV=sZEw6SfQT`lqqrxG_ji?7fCS9WRHclkssT_yWMTfQ7LU-SkCJKh!wjYyX0 z;^SaMw!M`Y9qgGOwRa(wbff~{@_Lkx`Slso(4*4Ex6i|u(E5EQJ0m^!Z?%#Qnfpw+ zjg(B1T4;dW*Vd!ErvaO1tg41JZ(y4*f5vSj+q-={_i;=IoSW+vwQ|g}WoMAzj=2 z5Z7oG#>SUc)0BH?e!CF^Z&HO~_8e5yL_8kWu*d3VvBMm!Jh#i@&3-gm(yr1Qa>Z9+ zwlplae48Y|uE+02z>fx0>O=T-I~Gg?yi-J_x@#)lyDZf^mM#Bnu<@$0OVvFHwTO|C&&0==U z7rkuO+0y7uyLPj^G#ys_u}7Zv$qV_XFG7aE(*`l-LTB*!DW4>>-U?hV&d>B)>D@T$ zA=<5k#@x&*LS%Fn{676LwDN8&x7pui5$4>#D~T8yO10U9bw9Ux+ZlLqy)U%AS*BPq zUN^Bmd}b7RP4QUvlwsJUFt82P;GprO-{i^i21S=;DWl-I<14$`sU-T41XWSvU2cjl z_0k;e5DsgVPJ(YsdC2k&2CEpC%ejk0`86}tJ1CR3fl+p~E&P`4VJLU&S>w=v6rIkE zLyyl}3SuqI{;0zq@KLgv>0YY;ybt&F4Syn_Yd(|M=a>@kBs_(#?(^?=ha(=l@l~6X z*%v$?E?hpzG5HP?{jP$@k*5iq99zu390|XkgNdI6e1FqG6HD|uR4nuK1Mk@d$b&=TCCLK9XQ(w6hM>HE&N$d=X3Uc2U+ z?nR_MZtQu#R%fgz&Z&6}{E9nWxO#tlqJkF#_UBD^SA6kgNC8DVyjB3UT@A4H@mb;O z*eq~l{rmP4QIX+ALf`VVt&MHxj$A4qSM06b#F5zxqDhIm-qSFP1BK$`BCL9HwW6>5 z8b{Gr%WnCM!X&55yZsvoX7M{DsdN{W?eB1CC28j5lRk7OlOq`%Tj)6sB>;ejgaAN; zAE}*voF4l5JbNzckB#NAD$l%m+O3MePEXJGxBU^mwB5>g&gJ`GgrVI);xAj!>uSLZ0ROB z2LsX=u(pH3+wh^jf_n^))7KGXGev$jWnN@R$Ae1-x-%>PUB3DhQ=x$wuSS=jab5sp zjJ!Q@Bss(FPei72G}AQvKVq7D>RAY)oOYEX!K^!0sgs`o!%JMOar-j?CX&C<3y25p zFYFA5x{uJ^ck0C9sXgWmN1N4EbJwQ6$_CsIw!xNS^;%WOE;Lk&;%w$81D@oc-Q%yu zmwhwT*y5oiC2?)M+vw16_nf!<+P=k^62dw*Sp-Al{>;5J^?|3pNj3qqSmgWwe+m4}n%zdt`c0Q}i${u2?A sRERMi6@q32C9f<24?>%lhvokl>i^!7A`*BN39JzqUM`Yb+<)@_0%XJ;hyVZp delta 3887 zcmZ9Pc{CL6_s7SOEsQM5WH&P?B2gq;MwYQJ!yqx1kR>tM4a%;>7<*zci8L6ytYsP5 zCdo&ZL`1fb?Bhqj^ZB0h{oZq)^ZfCg=XKuaF6Z96V7zWz(NE8MZ>%WUmjM6>=E8+B z^W%QeYXSvA$X6J=fmo7q9g-r(Y-vkPy$yFNhLcE%4XnV3ZZQd{T z9-wc*8Qi=29~JP53)k%r_UQfbgU29`!Z=4?$A55F>sc}STKBa)*p{#~OITVW&=3F8 zjrE_tkgSovz{$)b2#h3XiJ15w3KFyyO#D?AGQuRUE6I%rV%-Rja?d}fuWHeL9knsF zbreh3Ps*if_=65*sAEQ>+Z^gUpLXkihTg{HYbt#d6|5LC;$*fs=9M1vFw6|0S%U|H zrCJLjp(i+tV%jBia&KP)2FaM=v^OwUrzImZ2gx$ zKGRZUt;jti4!F&0Hd>)F;TI!LXB64h5r}ko&q*qt_=w9eYZ1qnxx;0B{lJcC(#6ue zq$dHG>t-(PN-wG5f)KCe7`BsA-H4mxmpzH52b@e>U^2wKP`hp0vi+HRkl-{0T z$K;7WJJF*n&0RD8pd8)5dn-)F4Rwd&8m1SWzdL_PR1==rh-(_pG^=bdg$Aiy1s^e( zAm|x*0002<@lgi|M1uR5SOEYqIh6S>1D4to^63i6W`a0re{RjN@dI1Hn?v$wv1OVS zu-(t+v?j^>S~GFhi#98L0r#%nlTU$F*Pi={xTLZucsj)$^2|Uy9s)N09@I*egQ0*d zE~Zg`=BF7EZtHpyy=j{wk1@KLp+9%Biqk={1GTJyLig4B68JL@MKWLVIrA`G!`79q z61itVvnai2=+0X^kmsoq^SbBhy{PgFw24FOZ+;Iyq*@cYDxvA$gH#>#6e6HhKzd1J z)`P+V>r{pa7tN^JRF%^&hh}g_X;Q+6n%1!~6e(Q1!@xx8wDN-nMMqZ$5AvHl4Od>p z^ep93dh?~1gP76E&7SW$#@(#jSU+jo)Oc`riZg+c0s)KQ+#(DE@cApyK0M-9V?=98ZbWqzqRafG^G4XAQ}p;;`b7%-$*kT9ii`c3o`ME!Y=%kRR@*t1 zLBmz^#Od@|Rh<&gIptlfV;_IZ)iayT9otg}kQ2a&{;cxn%H-lSa*b0Z!u2UgyP&dd z1CmwZYp92Xc|1E&7|+-PXIGsXy@WT*`&si=9O^ zO;5LM8gMR8&25MyWM6hJAr*nFzP8ZLL5Y`|47kN=y7I|1q}mm?~TP(8A^Kw-d(4IC9t-&UA6kB);p6?OsS+7oJg=pm_md9h%8l0s-rop}uc8vPATXy+{TY zf~t?RY6dF_AcI)6bm;u{BMS4;O(68{rp&3qPaA%(YE{)A-nPxlA^W~E94Nju94GT! z?O5PT5EYPNAl~2%ra*^vFJV%KW1-LN&O+Vp-QUwVPOJ7O2BI-^&xuT^I=gLxd?8$66z1!J_Fl(R& zxo4%YeT7~|ZSiqFi?=Wr<3&_D7+KYbCJ%nqn!keCw_EV4Gj7cG`jUs=R6s5r^OVtu zrW4%iUtH6_xUv7ot&){@B4Gc7=}OdNSR4hxjGpqXSWsU!?|xW#TEB#sJ<`ni70Pe? zS78oj_==xU?@w2b>;-|!;;*r8c8lAhU^w~l*I0LeA%RXI@jy+QE+6%Mn>xK6dG>|> z%^`+X5X(ChZY<2<^Xq?BH@Bbd8XT(e1&^7r=n`{k6^(%p6)xMU@LHV93xfQ zamNvi)xfy{`o-RX5CFHc{NW1c!t;-dD54xdxGv4wBR#uKJys_qzKPx9wJ^IQC0RVG zbInf}fKoOg_`FZI7bdJ1jUK0bFPFVAkxr5l`2Au0rH#D`BS5bhDVdS=ob-+r4}ExK;d$r|=3Nq4JArwWVzg!DMo213&)M zQqXl~DLZy!it`N(=ai9H3=n=66{cVvcA`>pI29f>a$HF%eLv~pY7%9}YAI8=xX?GZ zguPCSVa%t@uh!a!DX|kA?{gWzDygT3hieihs^3g81-&UwzIH>4ya+jlm zQ41+}aLT?R(?6N$cr^FGByntKQ2W=0_F8oF5#es@Ugr=gNlRSj&^{Ykw+!8Hi^2HI zt#Lduo^(^n>+FJ@D~eXg#=LZemyMcUQkVD^x+`^PxDSTuDK#J6GY;0-* z=M*dG`RJUhlQyr0XLRP)v@2I!_~xac=cXS&pM9}^d%R0wY9kevU5ByqvTY!yl5-^W zXV|{{@}kkpyXs2V3{~sqzmpRTkmT~!9KO5<+_TZaid8JUW_6;~G(Sj_2-+1Cf>wFp zyK=S0uI75V&)j`oEP2P+ zMqjgepdmC)h0*aYk-4|DC=zm?&8{TlAn9@PRo5k=Yc>oy4UgO&57brW4Z7&vQBwB4 zzENNf>zU4u~OHcodI+nM;s;wm1Wo^e}GV=Y;K~9;Cur5+E zcCGwDLCB-SMUVFzD-@ACpk*qcB5`P?f19#}R?g|mNFGGf56>Y@Hl6})_zqrhSig-C3#*oUN+=CitSSHWAbYQlr@p+cxZ2~sRb&XO z-SrKQZnIpl9d)#VKyrLVQmnrzi(KerZMH7pyZ9R&ZBH0N@C3)-lr>#|rrI-)4tYly zstkXG{3H4yUovmcKF?*`Od(zA)T_l;W;NKvg0knNxrag)nbt94uhf^f*)0pEb={wl z4@bW}Pbo!AroY)McWzbJ7S(8HwUM{>u2`jzQV5m@>b0sUMZwFMgNBVM_)be-h=llR z;Iz6WTH>22REXvgEIa1&0xSb-(R=ZGq@#j_1Kl#CUE1&K-Ip6@n%UXK)4-GK2<)L` zQumX^^~95#cQ<}+s=cc)n>pb7j}-AS&Rrh~k915DVxOHGV}KA;-Kr^AQViiU#K((w z@jw2!#)LKO9=_sF_hSY|*YC5I;~g`48d%o8mM ziu^J2-oATc2V&v5?zO0&_jV-YPcwhg`Zw%<57wf%pD+;lBfl2&sTL^&3W@`Z31L5Y zyp!Y9-ace4Y?e{KFWK=hUk0UUQi4DP+{}g$KGW-!&$g15Ax1!yi*jSYzy!aA%yv~G zt?hb&<&!Fi_8CcHgUL8Jth#LHjP`(er~6CZPi}zN^nR^QHped=IU|g(`KAc~v!`*<3GiI^u|^eEXR7&{cvw=Bns>$>^m zB|0KI;Yx?J3CYX++Xpd$H-!VFck$PNJHwp6&--+1bA+`Oaz+Tu=r|B7r=}+ z<&6^Sb;sX|p!9k>x0NMFY5vBeYhRjE4^B$u{DTfv5g%qPSV8roewz6bk_0 z-0{-#uSG<6^06?Ipte=07 zr;8uXRDuTwkpRvxB-H4faIF&JjJ(j3r~d=c CbTbbC diff --git a/summary_results.json b/summary_results.json index 5cb8317eb..d1f2510fe 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.600", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1099, + "Performance_Result": 10.1665, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.25985, + "Performance_Result": 6.26733, "Performance_Units": "Samples/s" }, { From 1933046f9416c77c3ce9bc539bcfd360fd7f500d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 05:16:13 +0000 Subject: [PATCH 386/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10216 -> 10210 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 66b3b4700..8ae68e337 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-30 23:22:03,212 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-30 23:22:03,212 submission_checker1.py:3260 INFO] --- -[2025-03-30 23:22:03,212 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 05:16:08,667 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 05:16:08,667 submission_checker1.py:3260 INFO] --- +[2025-03-31 05:16:08,667 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-30 23:22:03,212 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 05:16:08,667 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-30 23:22:03,212 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 05:16:08,667 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-30 23:22:03,212 submission_checker1.py:3276 INFO] --- -[2025-03-30 23:22:03,212 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-30 23:22:03,212 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-30 23:22:03,212 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-30 23:22:03,212 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-30 23:22:03,212 submission_checker1.py:3297 INFO] --- -[2025-03-30 23:22:03,213 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-30 23:22:04,228 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 05:16:08,667 submission_checker1.py:3276 INFO] --- +[2025-03-31 05:16:08,667 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 05:16:08,667 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 05:16:08,667 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 05:16:08,667 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 05:16:08,667 submission_checker1.py:3297 INFO] --- +[2025-03-31 05:16:08,667 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 05:16:09,809 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,17 +31,17 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 20.0509 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 17.495 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1665 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.412385 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411609 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.6721 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417541 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.6721 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414591 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 9951021cf..7cf69b187 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1013 +1014 diff --git a/summary.csv b/summary.csv index 73a1d54d7..f93597512 100644 --- a/summary.csv +++ b/summary.csv @@ -9,17 +9,17 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,20.0509,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,17.495,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1665,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.412385,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411609,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.6721,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417541,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.6721,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.414591,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 687dbbb05..aa240f76c 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.0509, + "Result":17.495, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412385, + "Result":0.411609, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -634,12 +634,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417541, + "Result":0.414591, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4804d44d54d0b2f3c5f4de30c17978294fa07c4d..cd09ede803b61cc9ada14c38d6cbe521b30c4a60 100644 GIT binary patch delta 3145 zcmYk9c{mhm`^U%5v5zcG7>$`BOO__PK@DbXnRM)lqU_mIma$X}!Wc%5v5jouFpa&2 z?At-ML_!=fEtX`m);o1wzu)^l|326C+~4o_{@mB|JnNU1FV%}ei#FsJRoG=tEunM<$do=5S=Y>@7m z@^ma%EA$13^sWR&INF8NZgSGx-hjs*CN)+M--&V?NhWU8EEe3SE!8s(ll?kmkJpx5 zq>e>1667={IB9$JS{7ORv8AHl%{m)S&2;1Ls#|6Vba~HNPk>83>KZVw)rV|OLgJE-?J=Qu-oVv7|9g&y9p17W4;TKx`ov8Zs4YmuJ#^zHijOt~t5bckG!B(=uC$H5#x98(CLfhLV3Y={z5#uDxO2!YUn{R4)oebJ*oO-sMb@@tp3Z&t-a^au_Ibp_wm)~-ela4gb6%!f zS4qW1fQTkv9fT2Do@XgTwj*FOsO8&G{Z*~It=8ybRBNTk%8SS=!TXiVAF;jRgzFWT zQyqLwUP$Cnx+HUuPg|m|1V8v=zf5BFP1RI+j_ljLf)u~K^MvHhU&hgCmT@@H)?L zp&+9hl91p%uC*=x(Rj|L7fZIwL;ON8YwYGJi0E!R42a#-&)!>}@945UVjgNjp6^Mmn+zta#w_Q%JP*{$QsgD2 z)|fSse4~AfYF0|+Y;x6uZ_r7zZ#~-?BWqFg{p)Jvkqt_(3I-=>7v(o?twt8?>z-3< zjLi(|C$=y@7Hh~&?&d04SV-2I=rybw>8`XnKC-MONrhY9P0!!)TUecHMHjTZb}Q`> zcR1j)h|_}*rbO{i($PKQ2Memj>GxBwEr9vET%@n<96@xnMI?vRXiDYsjE`Et#vxQB5-3@!ome+TSLhPD!R40 zC~DE$Hutp@##f;C(3F*AT{E*PHcy{Z5Ykj@U&kVb!l z;-O|%l|@*JRf+D-4@0Ou)`AaMM=!uoeSBtcLry6N(xzbS2@$#(URh4hgs+%&o+*T& zNa>N3p?~MMg}(ntcSGTrNekJW*pP+YdYC__eRMj@0DSEZG-D+|cYU zSUdXc{_~#Mz zklNM%xk;6Zdh`jcGPrW@WtKOlRfZ4l>*|nz2XPX;0ag??b!?*hM~FIx9YB*a@eAEzvzncN7l^2PIVRoj zwEL6fyirCD5Z~J5qNRX_4ZN=k`-j^3Xa|_DW)9xLWQYeM=l+Hg{cZ1MSxw31p*TE& zf|+;_l;(tjV*s8KQ11XCBrfxe%iI$&FJDZ#z^7O0#$u*AURpn~o=+8Oy{apzQ`xig z1)8-l*8Wb}WPp|Dm6{C)VZN@{Lhzt}v%KS2*3J$xkwHGeJ7-VI^Oc^EaIJ&y|;DKW}uQTHZ|OF)gWjiDSa@Dz~jLaNkUp zVNE>u07?*fNKGAp^2Pm=@GEv6O)ty*Fm`=8k!cSs03OQ4_LxcIw)>oPq(l5O+aU!L zbw~V@%gu$AH`2N37dVpT1N2ZUm?EKZxhxo2&x!*LR7*o7RY7P%pHOY#DZqvkd*Q`p z)_T~RcA9*JI2jPPEs%g%E~rw9(La-&TE}p0akM*=Y!z5B%+SnN$hkIi$^C1dbL8lB zQ==^1OPS;$Tt03mINqP`MRs<^IZ1#5-pD9Zc{^4*F;)UJluvJ9iab)F8=?}gkb=Ie3)*53#$JTZBp5&`RR#Hn z0TRbzx3 zrffeMPGU8`oZzbY=-(_KXl~oUYBvylHWk=_Xo>z6Zs%1?UKUQe*MOom9^t)bi~3uR z&b%e%oN&4+Q{DUs={#dNJS=awnu)`cM9o1l$$dmKFvUdEQYz*#_W%?mM;lo01_JU- z&A@_YU~eCy4R4Z=nO;>Mr>m8<4vq{xk#-eh4;CU0o}|u}ue{YAyWTizB6gMK z`GDoQ7KuvqCe`sMBla z-Ch$WhzFgGY`Ikid-x3}FU*gB#679i)_Wvhw_q2zO zPu7%S!;4RS_gJ_BQ6(HLsHdMIdiA<-y&!x&*hX!UJe)=j^K54&fn=sx#7%J|q&tm0~ z&cymfL_6Z(H!JJESUH(-1b)@V3QvzsF*OYdN<;o?u;OIam4wNphis%CAx`DllmcW1 zHcvoPZkmWI?6<${QpNSTX+(j3n`!uE6~OoxfN^V-4?v<;Ec01pXTtoERvgAz+ki|{ zE%0kTfRpFWxcg2#i$YXh6DQCgi4cf`s%^2!6%mIO!04YCW28eyMg(VO+nG(pQ!cj+ z$lT-VI(%^AoAn0_hO=p)vqN3Hfy3cUwKg1w9GLvk<%A>CmrL}W=!+UB#~zv;t$iFJ z*f2k!&sn}cMOK5y6QYZFk{Xu+3u7-jQO}p+v>oZ^CLWuwsDl(19q6~zYF(|`%qQ!A zhUixI`X8-9BKIlZyJm6Yp>D_5*B$yw2&7mw>uMKfR*ZYMp5Eg-Nu3pe2T+UIZT@-VRh(YZ~**`1}L2!%?{in}~utSsVobQQE^og>r?J2#xhe|ff&?UQ! z>b&EK2yEk+hav6pe(eBRmqu!O{vIfqGJL1jRr+Df{ z9fM2~Exiwx=+M;1Y6|tPTKD?*x$0m9IA8Nu&P8`1gIIce|j>5)MgO}k0@XEbcY^C;-{mqF7>Lz|}En0Cjr+A70-R3=9_7>s+to3OW-2D4<9 zBnl?oBprS{8qed}(LsNl9?R-6qF``XJ68KAziGSHI67GNeLW7o#ao|^tdy@|_(%4~ z$Npi>%%ly_zFQ9CEHe7Rpu3nP=i#>xf5CV-O48G2Wg)$A5-Y6d_-W}GO zHRId3pNz7^#%SE{2ib|P8NEfEgwx+vhtxX3Dpc9)SY@hBv{zRx)7Dn@eSDt;>zKx> z*X0P`bd&DUK@@&7qGb>9yqkS9YG3R0wDhvpFIjUow#K^Q{Jz0R>&VXD?NRffe{q{7 zUMntS$faz!k`32e4UKd>n?Y$lC=FvUcgJv!ik2mYNhJ2JyNU>9Ci&xr$cr7YLg(oU~RB`#J;tCb=OD)Q= zQ>jHOJ+|;W71!`joDB&qdBLH?kry0F znxft}L?Y^aOC+M+cPR11xqp8zAwP&8NDtx~`B)I%zCU(9L7mD5pNvm*fAC4hbTCHF zIG<6XOo7PR!@vheAAUc3d+vYioW5KxhT?QJG#n?R#n*Z-oHnMD^xtW^(n&wfgJ^Iw zo(+O1U9SZ_rNg|oqUSo@{c50dV964zcGGzMh4{!&Fr>7q4A zhqnA*&bTgC<5H_7M!lt6+sXNvcQ|zpQYWsp*C=tsm41Q}ppHl+bwnbmBNB0izIO_! zBjMu5`(r_P8^b*je!3kl&XSMeE%AMRawL)@Ms)cZCCU_38!Dm2CzCu4C`X&li$9EKB^8$7^Kw0~6`)Stc3LWDv8woE_hW^R*)2 z#pIj%#|rn#Emwkx8po4hq6Wbv8UT}M08FyK4`8;6z$6I(%z^~%&A|sSWs6TbAz;d| z4hh34lcdHev-N)fW)(G-nlf8EQ)WvkXKTA+m&_a!ihn{^f#pduQRAADOw=HmL<3|J z4UmZ%eE4G(kx3E&nFR^jn}ZKz$`+q=0$U&u)nQQpCbI@4Cb!e&beYYQ%B$-qF!@Y9Jm+fL z#G~qtc2v%GrP?v^m=8>%1;myv9-?ujjfZHE$r45$rU532nMN82RFRj5bplKlxp|Z{ z+nhp+&qgWtE+MKzq6m`2+5E z8hVJvm5v^wK`2XjdYA^FB)%GevdGrMIsqt)j6F*FZcd@aXQKot!M{VIaP3MQJh*nP z*uwMIL-|aNJr`=%+0#;v)ykvN%b^lYBewMR5REI%Jw$_4maz9Q4Nyrmj-;~4;lmOD zm48JZ9|a~KXz@uW#7+Q49}`mh9mW|7xtbMOI8@cNKWaDU?x z!a6hz*d&G^m^3T)`ur%GPujVxq{vc}=C;{KH|wTn<~)(r%^Wh(HDXJz57D^N>q9h1 zW(ltk(*T)717sF?eOLk@v&ieC!RrGpKIw#z30@x(hLa{qjgw}@UZ1~*%x7x$xzO8Y zpXH=!<=$zwa?nKMh%MbdMB_@g4}Z}hnkC#mOao{V4WL=%_F)MC%_6st2DcBi_@oo? z6hKsmMFE**%s#qev(IB>{;#HjO6n^Gr8N5Jrkmqh*H#XcXd1Dl(T8X}Y4jl)gtCOu zhiL#xG9I~EqKk|^tP_B;$mpZN=mRZ2=|q_=NhLUaXc%Bg>_D)Rt=Q@F7=M+?AOli8 z>3M7=wUt`HZrgm)`)1G8ZVr%k4v^$sJJRMuG@i8i5DfxZ!sf#?03;a?09j=7VVwYw zMK&J|HXms5Nhiv*B#>b9p>Ur?$?64<$#^z2;y+ifjRD(w=f# zqiqP5V(j@A6!Teh=qRp@Q!K>NZLnqhhiw2CggjscfeC*W7~TU@I}yIwh=5|bw;R{@2S ze@Ur{w6vBCX+cQCL7J1Ys@48002+|000pH0000000000004l22a|dy RJ^@;jxhEqA3?%>n002o_#o+(| diff --git a/summary_results.json b/summary_results.json index d1f2510fe..6b95b1fde 100644 --- a/summary_results.json +++ b/summary_results.json @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.0509, + "Performance_Result": 17.495, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.417541, + "Performance_Result": 0.414591, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.412385, + "Performance_Result": 0.411609, "Performance_Units": "Samples/s" }, { From 5e0380defc397e4719275693e681a300b34fa3b1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 07:09:55 +0000 Subject: [PATCH 387/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10210 -> 10220 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 8ae68e337..ed8a4f8b3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 05:16:08,667 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 05:16:08,667 submission_checker1.py:3260 INFO] --- -[2025-03-31 05:16:08,667 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 07:09:50,172 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 07:09:50,172 submission_checker1.py:3260 INFO] --- +[2025-03-31 07:09:50,172 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 05:16:08,667 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 07:09:50,172 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 05:16:08,667 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 07:09:50,173 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 05:16:08,667 submission_checker1.py:3276 INFO] --- -[2025-03-31 05:16:08,667 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 05:16:08,667 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 05:16:08,667 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 05:16:08,667 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 05:16:08,667 submission_checker1.py:3297 INFO] --- -[2025-03-31 05:16:08,667 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 05:16:09,809 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 07:09:50,173 submission_checker1.py:3276 INFO] --- +[2025-03-31 07:09:50,173 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 07:09:50,173 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 07:09:50,173 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 07:09:50,173 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 07:09:50,173 submission_checker1.py:3297 INFO] --- +[2025-03-31 07:09:50,173 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 07:09:51,357 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 17.495 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1665 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2629 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.6721 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.414591 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1111 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417928 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 7cf69b187..07b0c7609 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1014 +1015 diff --git a/summary.csv b/summary.csv index f93597512..a60fddfd3 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,17.495,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1665,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2629,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.6721,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.414591,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1111,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417928,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index aa240f76c..8b151bbd9 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1665, - "Accuracy":"acc: 76.000", + "Result":10.2629, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.6721, + "Result":23.1111, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -639,7 +639,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.414591, + "Result":0.417928, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index cd09ede803b61cc9ada14c38d6cbe521b30c4a60..5e244f98d12381a08b054f71020ce7b992884147 100644 GIT binary patch delta 4946 zcmZA52T)U6w*cVKi`39TiV%87q=X_RA|XfB zxr8PNh=7371Sv{0AjJpwfA_t4Z_n&GGkeZ_XYaGt%$~It%{R^KhN&2-F7=iMQUd^y zKrDiu5Brl!pYn1P;VQL1C9+nN2q%fs+c=J4oFVrsxpHl)qPemYNsA=ED8Kfmv)_sa z4n6AwC-;mX)ZTr=@x@$X!o=hE{m@^*kR9R531xi4R@v&!g?w{Hf=UVX1RVian$lX+BvjHU*p3S!Mag1HhSxGukzdrrKgV0 ze}5uaX}Bgbn<}^u6Ru=oF%6}2{v@qGl0HIli6noItNF&+x5^p^c;ZWkAhB1buMg+{ zrrbQ^p@=bxM2a9gTmP6n&X{|o!{hH)9Tv5YtK-UlBDZ8@?B2z{I_niYuM|wX=b?CZ zsYY$@Ms{V6e18F!DvMh-{*PCN)a$P0v8J4Qr584Q%00f)#Qo*+>>tJ}KKx^_&p0Dq zahNo$*LTVJd4{#U(ujBC9XcDX^~{vbABj=(u8R9CZ+N%w#44+}tqQyxs`+Il%p-C) z9pzppIz{1FPNS{@9E5B&J5|{Qzp=b?$>F1IgZ&+=D_(C}Va9Ci`0eFwHN5o$e{1M` zMta|M3JVw&H75W7pg*720U*e`&nt`o03SA*UIjTi>>f1`SYvl3BjEPlvZ#o{wa2_r zjghz4fv!wZ@}v}VkL7$UikkRwe$H_%m^dK|K0NRTEBnVw384tx1(naPi9J1dcNVUu zUG41rW;WQ@2p8gcx(&k7pqhT@I~^aaU8h@kx0V^sNsZ2uc-o6Ls7z+eS0mBLPmxE`nw@AO^WyO$BfA{Qou~jwG5VA#GTCefSr$?VzMi~yk}Xb zKkJcpSFd_gHU7x$)<{i@QCd@wAVnN*86uU0GBEp%_V&h#xSqkc>FXLgd2Jr1?O~`U zkKseLZOKx(dS!m{vHqhHU!wqa$*=pw$tz7}b&NK@!0>z@j;%?X!N#T>gz$o77Z_jv zJyi3Z`dnF52wUPpn%A_U_79JRw3#Wej(kV;0qi3#{Z+lc#@hg;58wtuSk#2J7xJ?Q zdMf1-Uy!swja&Bt+)zyLy@y2st4TNS(3~=^w8R~K7l`hVZ3O{6ZG&!rwriFPj9;g_ z{q6L&gTmlnl9{uTBNu?^mt;W`?ZEAY z1z5jg4{I4+V0yG1eZYcndts58d%Ss{B=`_?AIvM}iF+?)iPu}+y!$1-l=W?4hYtDr zBeOe51s=n|Hh%sL?AW4j2alP+IP^DJ)p@c?dnsj4LWF;HrltAg`Y~r8CiiWDGM3nFEeyJ!(OdPoot!#L0bk$Pdy5X z8P>vq=_)v;W_+_X7EP|}Gg#aNVZfqU6YS^~g9R;`iL8e^gU9#v!v(r^*kjrR6-F?b zU*&fYF+m^<$SCB9Pkey0;ObunAXMoyf5rB~X;{D=c*UclqxsbOTt zyumu0fwM}goI1sJggfOHal7?JFEm6Gt$>kID4LQAr1p@Mq5z^H7-J9yBANwiqXA(a zLhoICI2J;D3}K=JIzvU#>>S@cv!!l(jeJJ3qHVIR;(C0;JgpuyH;xb3r$k=9!D`9L z)?tJe69MZp6d8knH$=0fN_vnSf*oW=3((hH{9hukw-_ugf(Cv9v*a7^;&mTrk{aue znz&l)@EJen!u#V^+aOa|%_(83+$P(WvMV^?l76kfdIx^V@9ZIW4(YB_^Cf;7P%XNT zJsZcy{Ig#bXpGN-0MjLRUoUv^V&!5J8l?22f*;0V;Prn;dgLso+* zLr+Hdo$9i8z-X5mx@O9Widi$*N;RAdxgAse zQ?uP2DrdR$2?*PLEmQ~M@5FckU=y9Dtz#B)P|D5*5q(F}^TgX;J5b4i?94NfYQyg3 zY!XDt)8FRj<<`&00;Q`;P4>3tB7_wF!6Dp6Y{kL&x3-N>5pKJ~f89?!*36YBLDH9u zRV?G(fXq*zs(KPWPfVR55>S8}H^?i1MfZucWAq1xE4PzEjd`_g-78k$q-i+C9~Y&n zm-R!X9}ha!h&k}_1{;?A=)!%wufIxHj8#b7<+3CQGX{!0_r?JEFZETP-EhND>v`zM z{aXD(mEX`S`CRHYOvT*}l|)ehNeK4X6HdfOe(-5d;fD?956@{5a>#2oJASA46WdpE zWxLMz2vxF-cLg#(hN=Q2roYlDIo@!^Py_y3e0m|WDV=)>U$dcBOWD)EuYYMsLGr_H z<-d9^O(@SMG~KO-d1Nr(D6Z|?1?9gCyal~DY6*o3nzkRptb1CY(hp*IJQPR0HQYs~ z;+|>_j=jZ}{Jl;JfaQw^m~Pm|OE*Jea3q-l+1|3I_Ot^Bk9n%;%cU6xtjY4v!H^}B zDwZ)3jQ9}Pl*$;Wm9~$n5JkRa*?{mrN&z-<**z+{I5a^zpL&*snMw+6%)d2ao%8WI z=RogNh=Nmf1t&y;IKC2+Ka3ry+K^N_GIiGe<}=$@`aR>3^+;0rO#RMNR?`xOo?Km_ zlECsTl-Cmy5H5tLkU;lknR2&A*ipJR^uI4Hn_J~{I1kyTi-9!~1z`uF=>z0so3}Sb zj%**?gb99NqZ8!HLn22(QY^s$**R zUvP<1lDmt0ffqnJO5nNBlK^Uc5PD0>#iphF?a|AX=Uwm`QZ(1_iEoS z7_MHucI>p~M=)&xd*UQXWREBp{b z!W_yQaI1tuHFZ;0j-trkjXdT`j^!>4F&L$8XJzQN$F?F|KURO*+9N+XGTqYUiJaXg z;I1KGeU2K2vlOfe)|Y?1zz+YxzW$3bUH=C=9CJOqz$lw3FA`pakhXBsc9R^DHuPuG z)&hzsFQTr^eC{bX3TMe)6R0nrLV0PgA6(&2Xh@hkr%Y6Yj{X#dW=)|&DJ{4o5USyb z|Kz$jvVxkx%1kAfKTDU}z}1cr>Q>=%gH%fDU|y9qTgiU%t1^%Lw%HotC_XcC1@k)a zrUN3m4+|GA(HDi;A)eAEgO(^nQ>%5WIYb)zzn6A>zlaJXKNke7)>UOnc7qFx>}uL0 zcG3GUAj`qmetS8xRXp2$(NKS|+U?E89FN4hs-V+dp1!EDS;mJRfct}+kR6f1i}Bi9 zinxQnr>lh8u#-_j*qsMnJ;<7Q5pU0cC#nRuj;^D7Kwc6#;-TH8?bUVlY%a&Ezr*4u z+r`o&R9Q*`cM+3OzjgC~!A<0Ha&qrV$<@5rf1bcx7MRBK%mbLdD9I?`nLHfX;7qp|ibCSm-z&`6&s z44F1x_v;d2!X67Z+-``d`Zk87AjEJyrC>PR8E=j}uWh8T!d!C5MOiMD8{)Ah%z4(7 zj+?hl>Sw#O-n3^RJ9MQyFEyK|dY`s1|I z2nXIjwtqS8MJ5LpB&w3GYHaCHZXg8ClwH%r>aX4ZtAlPNm809Wi>fSpJ%S>?4ib@X zrEn%cMJsDkbR`Oa{NjC&Z9F6`yU(7LCL(8N++~4N&W8JsW0x1o^}#_B6*ntgba)Qx z2@{jS#u;YMw6Jx=rc{TgL{`wpKILssqc={oHcjug+bdE)^tfPXn=Ve#=(l+Vf!aD4^uF3z`4~LWm}@#~^B^RT$ng4SqpcLPqT_5*UQq}yh{kG? zah;TRWE;X70snKl?$fio6(!AA+vlZn9F^cSwJ!EI@`F(XgR!$-`)JWz`sF%@hR~Sc zFy$`E{;oD(VA?2$o|(nMB;xM_fiG5oH}he2siRxNWYWPs&1XY-sf4QOm9lxiN@sGt zKl}Tmm6?w^nRairIn0U94?-jR)W&Wle3UzgNgn-?V%>?i?_T$D+$f*H7v=B-ml#?SCCRu0!% zsycOO6a*ts_odC4Yu0@2#-{2frudzTd1UwRCAw6PrgBBr8z@??i8FRUiu0uQfl$Q+ zUoZ7e{Bv(_;N}x|3r=%{;`;_aX?_?kZeriM0y)z=Iyo{Gq{S*@DrcBAA^ht&snX(M znQH&^*~ZTiS=nC&UNVWF^ut;m%|o=;BQj=Vf&JSKA{Jo}=$Amb_c%qb#3H2ThJ z2bMid`)8p{J@)5IAS?1(N8TpUJe`|kq0IcZ&)rW`Bswu^WFEp}1@Ux?D)8jQ1ev~l zh%Arc>|5&FKkU7DrkwNy{cv5fyp^D}yVsxfAo+yIacvOXSALp=2II>!Ui_XJJt*i% z#SiqbFEJUcljGmwm)7~(9tQ_vAFmS0{VE9P7ndN?Okq3SOlT%ml;v$it$bHQ-nXR8 zf+WRhu?aWSyTw87$xyr}fy3>nZ3_u2Y*$PeYqdJ2F@73P-j|R##A2x@Go6mpOIi6?gJN$Y5|JL1vcD~#6$RyQ6dvOW?z=9G0 zU_Y8QvyLh8{6zQsaGs5KgZyW zNp@fbHD&yH)9_q_!EJJ7ZC*3j)1viLmBbOvxAHNrfva)A(a-c1M6Sp?isMn)hn0=hfEZPne>f*#~qySy*h}}c|ICD|sr~$vkM_24p>c44O40(lsgsC|o% zP6{^a$BNs7SvB9(F8;l5hrtKCYSjfMbA>5Xm9Q@9u+F$Z<52Yb)bN(1qq-Eyd1JOr z=ZV_ag4}$^dLU(h;F3^I=Hhxp2jn1aXjNMJgXkTsmWj|BF8V-B@wd5NDDQ5-BD>P+ zn-l5-VS=?-0wv8q%`5Djxc=q;>1tR50KDh__22P0)`K1!OV5wBmXM){#Udp{1^(w& z-~s?H+9Lms$FY~NuO;*;h}bg;FoiT$Q&K|U|A*^;-(|!`O6pR$U|S`{XxYRs*8dAm Cfi#E! delta 4928 zcmY*dXHZk!+6@qz^w5+dEukaDNGC`Sy-M%W6)@7JgeF~DAb|8B9Rx&LF!YXybm=7` zAkq~C?

PVTccHvI&O{hI30a$#x*Zn9{iJ-%7nm5bY|1h zDZNIV*t_))ee=J*BQqS9JoNc{C(Xv|2MC$5n5F%np_j{5Ef6g19=7A2tQb{_FP!x4 zF&Qv{I^y937cIXmkWg)Il}k37TCF82Y(5qfMP$4jf#j7PYMYSZ+_mZxvM)G6F zhjwUG&pxk$^7nj04-TVhKMg+G+N~m@>Pe&C34iGy`LO6aEtv^BVMLV*r@breB%p%2 zIO?gW|7X$A3R!8f8FpWXa-fWqEU7TKCGOD|rt`vTtcL|7#$%Yq-Fc(`48~-C)=74F zfAW%u0Qu=EfYA3xGd}3qRO3ph_EvQ5P_)w>DP=*5S+Q01+>@A```J>f8=Y(4A9utV zHN7LY@OjQT>F9jOF-G@NDq9}gEWtF;D995ND6+|?5G1fEXVmsu{U=DOoKAh%XvFAf zA<(Rx=Mhtq56j-1(j&Et67I|3^#Z9c6u2nSm|$D`B+8s~62Zo5-1HzZ+#K(4r)J2n z%hbL5RZ^dXVY|qj;B21v^jswRSLV~%EEBTV-uobsYILRCxgV>sprlk4*n3bNUud9B z1Y`gJ0Ax2)5x^hQY=OT80MMaA$s~}ze)9#-^MrW&jyydItbu){B(yx_gXd=&gh>u{ z5R0SXBzd3X)V0}#_F#(8ee?@8(#{oKz5!3#e*80kb7Q|o7ht397mo3m*rw{yfMDCP zIMja4YJPO%TK%a)+>_3`S6gdVx3B0@B00q;fsMcGCAHGe!%FDR)Vtra&h;XL#I#c= zd)yXuCpk(ScnUO>S~@9T>#;0A^9SC`70j64IW3Wp>FKs#qF9N>^+U5E)nG%h9p0To z{6yg0xn$}K{7-Y!6guLl-i3w*#dBS`rh$p>cL%$OLy?t>r%t6IRK1GtIAEtVPh5CC z!%vr=-*!!#pdnfA2g}fOhLRx%dJWSHB;_b}UZMieTg2JRw~XeJ+|TywXKU`P4qQ29 zKqS@iVTwa~zGK@W8{oWunq}Y2lA9KfSPTjpdt$Dz?{qX4=g*EpK{bzAs%^7)LZ)=F z8l;B5$KucmeJ5V0V-=ybQ&W?pmYq&x^IuqYV@@tfcxrj%f?G7kq6Uzj=)l_6RIaRd z>spf5-VdOYM^8yg&x@c*-AnAfGJ?WZlqh(f%`habwLM*k`^X{9(;tV}yblqHB1-%1!sH z0|GKUkjwbJvDDd_m1-Bq0jJquIX9#FNZu*jUPGdicI=b#5U>)}9Ky85 z*a(_r*q^Y|(+^h-J{Qm1ZYNC5e14Xf#ALUNd$y3RH`!Ufqs4axrNMNM1xk`(d~r-m zV5PjJzQnpIM4m|KMy^%6mt;CW89K2>y&3Hh;!#+GFX7b75<@)3q30K!I(Ejkf^p}L zqIqNcn0{eRB(rhwlNnvnJgR}-1<|IkRNp~V>*@C*an7k7}?@L2O# z+O_E0e%e9FxhGFnHmBR*`K`0IC4G!$7vx&uvfOFYbn$PJ;eCu3D zJrz>ewi&;2ul-+4T2>H)8cYU=-?q()UlNBsIG5y`z=L1FF|FZ$56{jo0>Z%{P3|-} zC|)&vn=uvM1BoYI7oj8t;Rr$EZ1A7pc(A%oMWG=^r X9|f?fp%yu)4wtXl;O;($ z9I;5aU{m8TWT3^V#|v@FDv=l6?5D<^#*gV^X2+dT7=YK0%gQ-FY*HBLIjiLQa%N#9 z%A&+aGM*Stn;#GK*P)x;TbKueQTr0T`CET}IJ+}$W^x`bt#W^^lhf_oxyV&yQ*(aK zl(k2wek9V#%VPHMxiU1IX9|8dN1YI<3GEm<2`h1czXPTZV}P||-Bg*MCwcSvjrF2v zXSp1Xi}9{#OBhUcQE)A&>s(C0#r`zPzQ8uskZozH3Y+yQQ8CFC!iZiwd9F~;Smd>g z5Xo#bfApu}vY}`2N3%$S>Z@J2@G$;GSGpTO8$%#$NJPJ>uF?P|2aHAnMUW6FCcWZP zU0K7jkYiE@b{1Y&9qvd0pX%_fxXsy+97N?9_61?~qiJ)O4Qm>m+Cqgng}GrOl(z!A z(9&1~J2|h1wOM353lQZ7(7_Oh8AjQE=N8i>1T=CgJ@Gjt(3!ufg;vbjJT|)9_z~K2 zzckO2@$i7SF@a@N-pz&v`P9eQUCL8qGj%vXGlkI$w(u5=dTQ_@U1ypzN0rD4M3oxP z0&cWW<<$f@F@fDYsbQ%CRtsxn9i7A%9p79 zu=O_KV249!Nw7s>7~)}V_dK5EYr0$b4FmUvPhrp884;QwKo&8Q+%h+c#(D%TxfA_N zpjQ@5>8g{?;;OTI9X=}PA^hO6rQH3p(v0>ds|IbCWKy-_%C1vLPQ{UqCa*{8O~DfJ zBtxoHu*-(TAuxIP$H>0|l1eyh=lT-LanjB>7y@{GhzYV!8%d0R#Qu>+J<2dk& z+_iF9!GB@PLMAH{#qoH`s^T>ZsV6woA$hVoL~xL35;V4wyODZV73-G(de89NujsEv zF2)k%1?(&&2B0+PMt-GWsGLAXV%?5)tGTg2oQ_xd=#E4#U#8uhzWs5IW#Ffo=2(px zo4u;MWvwRsx;RI+Drp>#uKdke%rq06C`gFB4w6U*^hzj4_}y_mu*)eeM?hX0f zYOaN;M!BKuFT|*dG@;=xX)`x9LlbLYHwJZ!oQd^~WX+$N<&Qw0K-JHjA=P5wM~RT+rM&vLYY)MSHyuU>krRW%K8$X1H(q%q zJ-Mv{?SNkV>y_26J^9lK| z6vG@!3C3(CSd0)K-0E~#}lTC*Ur6hb}Hm4Oo!-U`Vk;i zO_F1l9AM#eGGTubapOYOoMs^CDGW*(4i-Hm;4DOfYr0R%6ipr-0%TGJr=Ba z1l7hX>Qfmm8Eyq_(~(7H#Z;?Iikz-7z8IOY5e8XG zlwmA49W(S!v1n19FBnOf_~|?Uy4|8steU0C>bfC!b(TesjEQIO{A*qT8(#OGP2H3v zm09(=ejB)JfvFl<5@Y)Hv3|TbCD+S@=Rx9`Oyh~!*KaZzrgtMpV_P57`ne<&)d;a@ z5o&b3d@($3@Z0TivX@7hyCbn4L+i`{&q(Ev6@p&f7&0)kSwDn)rwktY?x3V{&%>aa zGe#?e)6K!+$!1BM$?tk|`ejBvp18`PuWIIB5$`0ZT@D+DsUP(zy?r5|g`G7L^m-*K z*->5AKx&mKT)*>U2PT;R_}hWLK9B#}lJ;=m z17-Zqs>;`(w=b=&kgQCMoPGqJDpAc+&$~QEXXnS3`)?O=GVc~>iYe;@WR%e5q;Hvj zgliK3v73%G#f!NI_}z0y*#ePou4YrCh{i>e_U3vo`qit=g^AF@FP-xAb^-EF16~YJ zPv}OQ3ZEe3K6@vcbd}9_%|AEZFJ_OW9iaajU!R>`Nr;ZhKt5nR1edeNKukWAgquRH z%ZZrkis2k0S{toiwz{4?O?fs~i?@~x*NiOpp6N$i-m;cGW*Rig^KZq}IjGt78rdzb z-@=&}GYkE6d}&+rIg#;2yt1g_HslsgwKxm@g3D5=onU_<2VJ_(WD)IrK7E!TziNVM z0N;V@>StEjASrG3LLedk#VVBcx z;zxd`Z|dk`i1I_k(vJh?T}@YF=V7#i<8?phwv+9?BQ+f4y>wFSe%!2YhxCQds3q@8 zskZVcrmzjO8y@;&mfs~K+&djG$?laUDGQk^D@56_I-879NK<<)TdXYJw|l?@%vHgmylGEs@;#D%J_z; zefd(6umHOO1q}V*YA)Dj+0Vw?dx-~&&vm@};kKRVy}>y`L}|*`^-U*iHjTH-80B`t z@YEU6xI}H&aj4l3g~FtQG1a6h7(#xPt6;~xLq07p@$or(?>dS_oQ))z;jFx^VZKpO ziY4i&8~Oj+Z?1e>c8#0=!;g1#$beOOla?qpZVu#_N3R%d^g-yYl6H}K-eSgPBG2K< z2xVi-7xa?@bT@s;v~19&pxy>}%{Zkx=gQyTu<1+Xy>sHN(uoHn_ai}{m&hs?S#Wh( zI|^DM_LlXZ+$tvS%3DGM7+W$}N$WnDKU0d-=Y|K5c)n`pGEZpOvNHJSI#FEGDz4W+ zt38Kw9Q`fzgn&8vV?$&Ykb;sUyplbBJDMRxb%A4!3eYX?TlA1LHe|~`2V#+xCg(1$C-=0X6oad#-*7XkM{zGAq zAM+9QFfQsgW Q5tyKQ`FKc3Ab%hI2Rrg8*8l(j diff --git a/summary_results.json b/summary_results.json index 6b95b1fde..528e6543e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 76.000", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1665, + "Performance_Result": 10.2629, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.6721, + "Performance_Result": 23.1111, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.414591, + "Performance_Result": 0.417928, "Performance_Units": "Samples/s" }, { From bf3bc1c7727fa2d77f89603ddd3351bf1266358c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 09:03:50 +0000 Subject: [PATCH 388/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10220 -> 10224 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ed8a4f8b3..8e27acf51 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 07:09:50,172 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 07:09:50,172 submission_checker1.py:3260 INFO] --- -[2025-03-31 07:09:50,172 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 09:03:45,257 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 09:03:45,257 submission_checker1.py:3260 INFO] --- +[2025-03-31 09:03:45,257 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 07:09:50,172 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 09:03:45,257 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 07:09:50,173 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 09:03:45,257 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 07:09:50,173 submission_checker1.py:3276 INFO] --- -[2025-03-31 07:09:50,173 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 07:09:50,173 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 07:09:50,173 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 07:09:50,173 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 07:09:50,173 submission_checker1.py:3297 INFO] --- -[2025-03-31 07:09:50,173 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 07:09:51,357 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 09:03:45,257 submission_checker1.py:3276 INFO] --- +[2025-03-31 09:03:45,257 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 09:03:45,257 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 09:03:45,257 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 09:03:45,257 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 09:03:45,257 submission_checker1.py:3297 INFO] --- +[2025-03-31 09:03:45,257 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 09:03:46,608 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 17.495 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.1384 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2629 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1111 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417928 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1711 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417928 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 07b0c7609..8618e6130 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1015 +1016 diff --git a/summary.csv b/summary.csv index a60fddfd3..720c75307 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,17.495,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.1384,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2629,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1111,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417928,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1711,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417928,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8b151bbd9..adf8c997e 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.495, + "Result":16.1384, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1111, + "Result":23.1711, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 5e244f98d12381a08b054f71020ce7b992884147..a1ff62b3d6ad9ac8e84e0b6bfff1085061be027c 100644 GIT binary patch delta 2660 zcmZXWc{J2}AIE2qwHXY#5ymouLMDbb+aS%@rX z#$Fw8UqX7cfzNw=Ea#_muuzV{JYEE0n(86}A2r2k&fY`Mic{~UNh*nK4tbJDU7 zcc?9;Gan%C%j(Wt$<4AHCR@~fCVJ6b4bB(?iqgm1>*p-T9WBC$m1eMG0GS3gkd0+} zuPmrEkYnyj$sy zsIbg9EW*!3p->U#9ntW+V1eRa^YI5A)GeJSK?-c&%TDP?c#Grw0eb<9z&%^u)-p>v zjrOpLz(TK{pT6mDQ1yJKq4?T5zdOcZdwLRiFKwAQ1CxueM7sFP4cEzJa` zs5H{Ag3ljFulXqpD^khhi5lDJ%N@V#`I=rOzIP#7Hd{q5Fcx;z(3c7`Mndx1;uYBd6XKUt%o$E@vgU?PpDDb%X_Pu3YJ!&S+_qpVT z6U(E_yhx>thsvChpx&G&2U>PavO9>jN|D|ZXG!7-AC#o7+V~iuKPnDm-XQ*7)KMX5}LV)W^Vom8ac1;f!eC{;&d>nCi z)(fTdoVl&0^`Mk5f7JTbm(p9zM8^47aVIvI_SFb6?Knk-351428!8QgQVKAdexfM| zZ~`$Qv;N{Y!~DBnamwKqlB4pjTTb-6WJjcj<1uznQ*m~?X1v1~_v948!Lplh(QP|q z=vKsJe6k_~;h0p)!D5eaCfsh>ybQ`+i9{R>lyXeq&$KXMw=RiSzS$P)j3ieXDP9sx zv@fNXBA&X6E1KYpsqLZjj%iU>no3rUv}h=!6yL9#@%hR=Rx6`#8fAYBy@V5#tR=w4tcvL6(RQs#> zGkU&esh}n7UvlOvJ^fGFO&tb=|I=MA#D$lUIOd}1A@bN$?+&W%WoNt=@k&76i#Psb z6*s$kf)vs|kKjpM6?t}%B&U^6@=_+1;f|4>Al!UpBJVjnw{h1DY=;#EPp#(ahrZ#A z_?r$An48RU1tCRoucuS~7^(w|YQClRMZ{>+shtb{wdrT80ukzQ;J}8aSChLwomYnS~Ih2#;w^;CjPoE5>FY{OA6&?v# zm6yt`3@?lHd~7yf0zMN{tXC|SS=l+2lY6iv?hveFmI>3<*4h>pGZyWoOVLZ} zADMcPbzWDIZ-mIbSELf~!_3nX4tSUs`B+NFs{2Ph_vB9C^aw|I*TVNpi6@B<`ZEhjNN)G{EytSRjn9W)4v$I#*0EH|2LyYD3QMR!=HB_v(1{cLhil`EhW7zbKqC zAsuWewx*veHdPzW*+y=ilvzIa-sm6t23<>lj)v;#%8}w4uWy6BwJRwXg@5zY;@~%XG zC8LVDK+v$$0L`OIbT`!i-XL-E){#E{3GOt(FT@=y>gvH#O@ju$4E+!fT-K3XLF8iPJ=MP(p#0$l?swS>g9(N8S7 z{!jnjTm2Lzy#1!}{wUmIGjQ38pb==%UX>;KvnDViH>2KYsE4-4%Z$OfAZD3OuIlg8 zUN_nh?)EOvLI@i1=%v96&)OLWcAFEEs=ojmgXpzJC+O{Sr+(J(-)vZ}FF*L#?%IvU zlZSL~Qf}U?`S5CTy&HTSQyvFhns(hPJO$~f8h^nP_Qs}aW8@iW?^4$y6OpcAkVhGA zc!GmjZ8pYI$p%j=@2z+>$sGq+8WX1{K8)@xH&%s;J%{qFP9u{6X$3f?*Qy02627T> zWg+NSl|6?Et5Kf=-5WEUYf?9D)7aFVp=GH> zNsIuA96pjJRbM8tqGuND;!r+HDj1a2O+A19#Ji+Zd}V$1_c2s!1#D24S8TPseRKuW z(Wb{^Fe~T1j#Y(-`8!He^4ELSU?&mH7C|mCHPjUM^~<7z;M3p2-x!q2fT*`fm{6!J zlNe_`1G4&F1PSw5vejHucX}(^#;D)%(e!tI?ziX?`~3mCCOF^}!7^{dx(%#NpG&&z z()2#RnR!LmsXN%dU91fGv?FAs?1PMiB+C$S4kYtgmL9*o5`cJXXo-ogOG@%gDeL4$ zBI`3k_7pV^tquK7Zr__1MJ+Y3d3vjy6}!3q@e@3!bORg#L@DWlg@JY@47d*9SH1*R p2aJ`KP8}JG1cN{}Tp*Ar=zr&NC=>`+)(8Izv?wd`iX)G1{sY(c@)rOA delta 2655 zcmYk;dpy%^9|!P_ISgY)LQXLoA#R2oOD$Pb4%M6@bvu_RXPU)i<`DK9cSO-p;tpFj zhujaLY(|bbC3i^vfpA{Go%EWE;0)ykc^tn$4?K&pXDIl@^`$`GQ(BsA$vo(55Ei%yAU;>=$bd z(UE=X4_`9lGj$YIUf&-6$qgHSVtQLTpLijg9Jca0P)l#-vf|I&?JqzcR2%#+pc#5d zg{VG-ZnLbYTZpoFZa!I-5DC9El^Ha4%z8Os3XqXVk#o(szIZmI|75nokJFu|KMEZ> z)`1EMZSWMpmpB171mq>{!RvsRWF!eNd6F^QFcd;{3DX^kzMj})Rb(kvDK|2HnSSW~ z$-{O~=MzYprDpDsEVbe2l$FR(Zum;~*6(()``T?}6Pr{I_R_N78LSW!uOn$l`^Zft zZJvCg$-|>a3ghlo!{+;A_^=Cv%5SNlOCXL7qo?yvlJI7kPM#JylD?(hgeUfJ7YE z%+-0&A*b51F5!%Pw@RvbsPSwG>U`Rt>|~jPcTpN_Q}pN6>-2Xe44Kq;zNwh>QCvRN z)GZbBWSccoeva-Y)D{sYX;*XAyrp6~^k=w29{1vTwxqHMvWEIXmR_rvGxyiF4idU>bWl}jc*E# zOpNM1+?+H>taRmWe$l6-O2Oc4)2Gomztx_Hzu(%j$T#3(v!)%4UC4e2SPl+z1Qn9w z=#53;Kz<6ypfG9M99^#)1JM0vQllIct=;@9X9(Oe0{EAYQPrb5KI65Z)r^xz7-XK4 zF1T_3tNXiej2Q=Gu0pvU7Yn29a{Zu!M|Q@fyT|=#kRt(h$=^QyEnL9tL>KFv!5jO# zQ&Pc&*p!HXHT7E+@dF|3bY*-`LC>TGD{p=7+)C*7@3)#~^7YzwX2cn}kbM!bn>Y*z zHTF)-$m_T-4FdYFcxP9H8|`>0@8~$!Y5Lgr!+$e(uu}zp7VEOgNpx2I#X5YzZP@YB zn$}fh!J})ZaJ#EM02@<9yRb0Z?Yi_XB=jaf8M4&VCT8rP;bfU~3ZAx5s{;a*$RNj0 z9%NlMHnE6%fT!14R^OZ{@CJf_09eN-mRpx6>SeX>frUxTL2w{#SCKv-pdrBrJE zOZpR5IXWHjl(fe^x@320mTm!LLK-HlLv>JI>OGtHx>%mqITCsBe08Nf7R4Eyi7e;= zcvYX!Mw^b_*0UiKFBl8AqtEuHmXFu1+>vp70PV=PReNx)JeLxb!$lC(D!?e}i(E&A z#yEE&UuM@UM%m;?dC#3ETl6`4mV%WGqK}6r32li2%;AI={h`1Z`%e-KARuDVFYvDsR7A9mrKzG z9mu$1D{;fN?>%rjtrYak7AXZ2;V%lOCZ|u@rp>&Hk+(kc@>Tgb`^RWhs8vtV37hc` z=R>|`jtX`B|J-}zhPkU7$z+M%90rb6=H>h z6bg34Bdo4PM%$vWf$e-s2{b?*A*$z*uS{SzU8;FCb?l>w>DKwVP?l3uVd(h1yfqBr z(++p&5}vcmE&Ye%{Eut=hg)J<7;oG}H4q)p4N=gGs>-5QjGJZng?lE5iS!{m`e6oE74Tm{{UJjn=M&uw-Du`UMkW`OH1dM74HfB|q$Av<>agGPbMU zXlRvF{On?5oR{`^?|#~|YbQL3X)geR`U5*nygTuZXc}@FteH`5TP>%->{?*7E$l{* zEyzbAs!6sOsWd-=y2h%72XU49^&a*^gz5JnFL>qM)qBi3!N>MsIE)PV8dI};?%aEF z?n6bCz_Kf%p41W_@t$nGWLPA4Iaa0C1Pmvr0V`L6I!I66Y6J&{3h6PzQ>SXBEBa=rHVuQJYAaJM&z7v~n-r!^O8DT7=0?d>?~*fFR1&^8SzU`w5Oh8b5i+fM>JFr`%4)Ozn!jw<2j_x{FN}ah_%)S1wC=c+SYIx z$0aOQtn6>Ofu9WlAo1I!U=hpMqCgAVIN?JxHD*%0f-~GckA@JNbRc1Q#_!66+2xEj z<+a(32*IjZb>Yd$`{}}u@ol;>t;yG-sIM|&8&WrG)6s7o;Mv|oHP2KPl(vo_e*>vb ztL0_CpN~VfUL9& zrYJXX5|D+m21^1wlpUA@NTDsky1)rE>Yre*p%uUqKpNT>>;t?+Yl*_Ocftq#8yFnp AZU6uP diff --git a/summary_results.json b/summary_results.json index 528e6543e..95b3cc46b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 17.495, + "Performance_Result": 16.1384, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1111, + "Performance_Result": 23.1711, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From fd5d9ba9ddbeaf196955b71f0472a9d860aec5aa Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 09:08:18 +0000 Subject: [PATCH 389/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10224 -> 10221 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 8e27acf51..e70a66853 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 09:03:45,257 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 09:03:45,257 submission_checker1.py:3260 INFO] --- -[2025-03-31 09:03:45,257 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 09:08:13,258 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 09:08:13,258 submission_checker1.py:3260 INFO] --- +[2025-03-31 09:08:13,258 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 09:03:45,257 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 09:08:13,258 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 09:03:45,257 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 09:08:13,258 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 09:03:45,257 submission_checker1.py:3276 INFO] --- -[2025-03-31 09:03:45,257 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 09:03:45,257 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 09:03:45,257 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 09:03:45,257 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 09:03:45,257 submission_checker1.py:3297 INFO] --- -[2025-03-31 09:03:45,257 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 09:03:46,608 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 09:08:13,258 submission_checker1.py:3276 INFO] --- +[2025-03-31 09:08:13,258 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 09:08:13,258 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 09:08:13,258 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 09:08:13,258 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 09:08:13,258 submission_checker1.py:3297 INFO] --- +[2025-03-31 09:08:13,258 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 09:08:14,360 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.39409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.392716 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.1384 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2629 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2378 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8618e6130..084277819 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1016 +1017 diff --git a/summary.csv b/summary.csv index 720c75307..615cf89a9 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.39409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.392716,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.1384,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2629,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2378,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index adf8c997e..2495b9401 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.39409, + "Result":0.392716, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2629, - "Accuracy":"acc: 75.200", + "Result":10.2378, + "Accuracy":"acc: 74.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index a1ff62b3d6ad9ac8e84e0b6bfff1085061be027c..6abac50f981a7dc0339e0ee4b4984a022241cfd3 100644 GIT binary patch delta 2602 zcmY+Gc{mi>8^_HUTV-D=7bC{ZWKWclB}^rQ7)xWhma&UOV-#kF>>qOD1C} zBI{^jXfEzhlBFnPDcgh}x8MCe&+k0XdCp(w^L)SW`=0Z@OP9W1s(Z@~!n|AvCo{O+ zfrdAzCqY+%q=^U4RbHZTmDPT9b;=3mf!CGxZ3p^ac`Z-VT4@tb?46VhsgM?}mwBS1 z#`z(u?ta~Uv-sVk2s-gj-{Aa9;i-I`5X9Sr@x>nm&O~LSQj4K_OPji$reHeGjIU;e zA9PzN<}Blw2wV>YJU8<3m|mRFxyD%HSRy5s+jb&`1l8+nW+3DjoV%3a>_2x&@RotT zufoXn!H;4OzvBq(rP%6sz4mhpmE3}_GKIcI$K!9EEu0tOx040NRGP}W`x>r>mkSle z25R63H3;Q;x|&Q`oJSc5aaxa071EewKPt8PsH!EvQIZls7xwp_>RvaLa$kSRzjShl z!wn$4+YSffzR^_H4djg$CK_>8qqi;0azhkL>fHrbgPtLR^ANE&SJie7))>yF!0lC> zGjw2l3Th6#y9^nd8?p4_D=MBI`c+!kq!Bc6hYBWe1SPq z<9%kT;tj0OYV6z1^8__NM>o0;;bMH@=B)ZLV`N?n*{$)!W9;=}^A&uc_KtuxiW?-s z#l^+Hw~e^s8v17D4sdb(K_&1P@sbMgcJJ1b`k&6jaWSJqRi@fC+6y9rT8^)3IGIrVwgIyqOwxg{d0Cx<3SF;+$$iOl0Ihs;VB*7vbG^#IB#;|AvW!B$1x zzQrKrjxJNmwV&pJ`b`nlqom%|5z|eJ34_A{BOiv*cgiMYlv`z#83-&YR+!N=^vP7) z%v&H`c$g~(`Poc4CD2e6VD0Gj!)EeFjMO8Xi7L3d?_1e}Ms=Lluy(%qK6`A#u2^%SvZ zIv7G>H|*i$JEgcVzq|(;_xPorwqR=qJl%J)TVI>sR;6YzS$J^u!qwMrO>JiilY2y+ z|0e92#)B!w+Ne=N9`7JT`Yg3?u~UpI{~R7HhzI*EUxa}y>o^&j7<&Etx-6#TbHa|C z)}xAvIg?-(VTi;G9A0}O6Z_;ZgdZiWE0?%0@nfQ&$XhWYxD=qB%sZ&M!l?sA`(%Ff zy}ksnT;gO{U1N*A($hXP>XiV_(@{C~>lgg~rN3LBl=Pvgv$W3_PA`=D&gy3~-^yq+ z^suPT{Wva}YA5*!bneA5AIa?}b(Dw!?g?EgCUq=vs~LWcD-=ioizQH@tDN>Hj!!?% zv5&3ZlP0SFpndX1Tybh%&#&Ht>0?p7`>^GIV7>ps5Wiu|#U0E0EKn4W2-Cyntk2QT z^qSG7)g$+ItDY8SZJ8Sj_<&h&oco;+)|8O0TpuuHxr8*f=e1wVXU{8~eq$P+tggkW zv*PMkpr-Rug`aJdhl`!MR(EMKIcnMVPiOlrUKkJprObi1er5p*`zq1S<&O+W|CXx~W znQ!K~k%q9UONN;r$VpK(=fOb-6~b&k*q)+&Am|EGJg$jZK%X)W4i7in9)DKj%73$F zt@bk8Dk`IFJD$3u8#!hgULP^+sZ}7DwlTmw1Nr_^Sf8;A|gvpr1yAeTea0j47ifqn)7h!r5?L8q1Gv58w2N-R?e~GTX!kGFSoh} zYg0`X)wW&p9U8wWaoQ8YaGGqP*h>?Bx&_@mkRv7W`C(-~$f|W4btBlPsLkWs_t$hR zvaUt!21(#J#@Pf^@IhXWym1oUiay&~aAB9{|9_&nP~d0kUGW3}k4ev$>YG#r$zmYM zsm0g4R1Cw5{wurvb1MFbgZ0WCpwr=BkAyHU5)=r}H1AL9{??dyq8YP;l+aEs* zxOO=v@_>upicl~5fwT)_q>fVSQBkYYWTN5lrTaiDrs%NB4{W?EWe_D1k!+%2^F=Ak zm49ULYP98<;UUQAvDbz0f-TnqZx&jcl&i~A@wEcokWy6juyzJ7XP1XbP^d8awk~L2 zG->9aLERqt^f0p$HIZAhRqgfC=-jch9S5AWFjwmq>9v^@`(H*h14kX%6JfuaS3V#& z+TKLM6c=wy8rfrEpKO%mb^;Jk3GkNAiM^AA+fB5uVrs9~$W-u%uH!Ky!5fl%zK*<{XY$#gL zi9BqXldRZtG;WB^N3!f4K8XX&O+f)~gV~whTUqU`(^(Bh;lj@r008n{c4`xf-HWNs`7LM6ZZ3v2_X+AA*WD5HsHE5jp)g#MUnw3!5l1hH`jcEva)EjW zqk*#e@2E0vF8~i#$bzw`yDQLBC2oC&0CM}5N6LbROxg$fWcKw)WZ(G5!4LKWj zTz5tYqgkWA940C}q6v0b?s68Gsz}-DV)pSr=J$)@SNpl~ts z3DZTE=!;qfoC=rdQ>+3sC^=!*)o&_|3!{-G(FH<<^wFOZobGW+MC?lkKZ#ej-ySY}q}w>V0t`0_MvJ1~wVYc- zq|QIly5ftf`7>m#pq@YXnf$Vm@%=6kX%^|fpybcE=LJ1utySGK%&f{%z~^D;(*@+B z&ylIu-*&AK-()o?J+~E9?)H&sqOO!@ZX2&Y01?gR3XHBo%w)`Ze3qRHQ*G>(2EAXq zGTHL4r6p}_$A&DpL8%-aSGQavl~p`RLN7nn9^3(65p_?Azh5st33M#Kq^82tZ?K7T zs}WAF=Hjl{TG>*quzBcfW3rH%(yI-$GNWhU0{{Tb7fTz! zcTF~LjRgPzHHI-4GooGkKZS_rFOOLx=^DyyZ>PH^Ag`d^)4Dt>=_` zq47rx>nI%E5XbAoGT!^D5X{nSu{!|5ZSv22Zq-N|Mz7^CZ!k9bMphJlj~FPnK{YTtB&jRss~FVZ9+a%u#x533E>MRT#M*uU@p0DC9-b<~(M zQWm>cZxKNd*H6m`&;Jeib^7}10mLBCm`^t&VP_EQu|3FEycwbOag%+VMTJ@7*qyJ2=)fBAu_xy)5gvV-+>|aqt%b zq#TneHDzk#i#{6moO;B?u4n)L%Lf<$BA7O(NHlZ-Ph8fqU)d)v?SGG6$*1fxo=yL(&L^JHeBJgC-eS-8z2v7RZIHeS& zTjtvr($_mt&H0ZmjLjm7AgLPc9^8sZ&z~4S79AQt+g~3~ z=~=lem@Oh~qtl7wfwItf3$sfHrM;f!O5M2IdG#pkF|w?D(QI@*yWJB!RiDa1ns?u+ z6k`9_IQ2I}^lPiet+A)bef7RovSa~NyBs~*5of|t3;2rl z%#umoML!+xaP^k8ZD-yL;=!-xr;aRhh^0FYH=3*8XuG@`NYcxUf9L<$qwbz>D32`e5ElTk zTv*pSUSR7z+|mA78I2u+j?4_}^N?(eu(&AN;)4=b%(kZCN{EMaBukBC7GU21BAKK! zCm%BF(5;PS_g)l{_8vv7CoTtjbrw3eGA8-Jsncx>1(m#vy&E6r7k{QUlr1>eKJU;?ibSiv}*UrY>~$ z*GD*@6PL;d7CK&er8)ZFG`NXCnZuaZ+O@)fsC!7=NDdLQeX8XGz~Wmqd$b?PMrA|6 z>5WJap?%L!<%x4n;Hq27H~*8C&>$g%dHI%Q5Aa6TQl^GmS98UCvZ1C^f2dtAZyhAz zW7t^TTVVmgV;xClfbgKk|bKRu2%uMgxIy^lD(q0(0F9AKW)Cs)0`{*;b z5~<^aVF29dDiK5Xb9y5~^e+GJf5)gu-37*ClBC3c_7|o>iXX^{`6#6eY{OiZRtG9z f45X!g#@bg}0LX!fm4*X7Fx}D;jC_!bSpo20PUZHh diff --git a/summary_results.json b/summary_results.json index 95b3cc46b..dd327efca 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 74.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2629, + "Performance_Result": 10.2378, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.39409, + "Performance_Result": 0.392716, "Performance_Units": "Samples/s" }, { From 0769a93314cf3d165cf78cbf1ceb764f1c3adda1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 13:26:20 +0000 Subject: [PATCH 390/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10221 -> 10220 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e70a66853..ee930d538 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 09:08:13,258 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 09:08:13,258 submission_checker1.py:3260 INFO] --- -[2025-03-31 09:08:13,258 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 13:26:15,689 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 13:26:15,689 submission_checker1.py:3260 INFO] --- +[2025-03-31 13:26:15,689 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 09:08:13,258 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 13:26:15,689 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 09:08:13,258 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 13:26:15,689 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 09:08:13,258 submission_checker1.py:3276 INFO] --- -[2025-03-31 09:08:13,258 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 09:08:13,258 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 09:08:13,258 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 09:08:13,258 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 09:08:13,258 submission_checker1.py:3297 INFO] --- -[2025-03-31 09:08:13,258 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 09:08:14,360 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 13:26:15,689 submission_checker1.py:3276 INFO] --- +[2025-03-31 13:26:15,689 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 13:26:15,689 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 13:26:15,689 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 13:26:15,689 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 13:26:15,689 submission_checker1.py:3297 INFO] --- +[2025-03-31 13:26:15,689 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 13:26:16,662 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1711 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0464 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417928 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 084277819..4d042afa2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1017 +1018 diff --git a/summary.csv b/summary.csv index 615cf89a9..3a7a606c2 100644 --- a/summary.csv +++ b/summary.csv @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1711,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0464,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417928,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2495b9401..d34fa02dd 100644 --- a/summary.json +++ b/summary.json @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1711, + "Result":23.0464, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 6abac50f981a7dc0339e0ee4b4984a022241cfd3..dd00ecaa47692c8945e1390b7513d89c61d73ff8 100644 GIT binary patch delta 760 zcmVybO;1A*mvu*cnD($e>Yop-hHK;LnT^5Z0YJD8dutShz6-FVeDZV zppuwtfXX6o595t5{)CNEVB8q1VCkx(MN&P2U>j6e+jV^ zfZKyRL1gne(hio19CHt`S>WeTc@DRv)4P zE}2Gh$uyEnrjcCgBC8KeAh|3`(AHq}ffk>1f;*QG)}di?zGNa;#OjK@K97O!14@&@O7TJ9? z*nObIC!G*9!R|xCU?s~^!%9|H?DqM4$b6=Dp9{Th_eoc?P200_46<71po!KITl#&7 z#+7~_qCqrE_Ztl04L zqijBD_p*{AOU;_wj-TxDkF#(QsSpL30}mo3lO7@}f6Jz(2-|_S{XZig7ebo29YdB321JF)Y}HVub1fqN_=2iu+C^` z&;TAS+Pj)t%8i3pXDqlN!6y-|ARyAUkJ*z+AI=CkO~QQR1(Scs+DV$1k9+X61~ zyuu0se-kV)d;q3)BBB#g@Cj*uTsHy)$IyWS9Xuu(!541;r~GWgs9ZUf41sg%K zdoL$r-}hNxj@Fo;;Lp{|+qh~f_2E<;P+JR5xdRrAYioR6#KU#WhgCfBMuK=8qCWF> zTIARVB8dU$G45IUD4(9!OVk#5ai8X8T9P#>7->;aQhgmp-t=F9>NMov#53TS`}PyG qf3sgB$^ry5*mvu*Ehaqy1(^d6A|#VGfhTe^CP#+9}nqCqN47<-ro zs3hhZpt8u@!#V*fi`+d*nr}{_#b=`gD#64_F^ikmSffk>1e?lw; z;PxS5XenTJAFvcu?Ds(|f4-;SnRf%|lb=pozv2e_Psph{lz6AEH4tOW1vw2GArLK(om1!x8|RMRp$z zb{}Z*NhbtNu=|iOSjn=~u#(jkyM6v1GM}m4=R$AWebUuz)ApHb#Eb{xX1b}9d-)D310Zs7xkWO>dw(4A^9b7$2B5D>nT6 zD4S2(y{x3jQnTi^<3~5^rn7JnsSpJfjeNN!lO7@}e>tWl6}AIy$*F-tQ(AHp(k2nL zevs5Ue*4N!;w0cxkE@yec6L#@IH)H11?Q#F70NS$642ZTsdp9nSTEB#N_=2iu*PVp zPzN3@>bsI#%8i3JXDqlN!6y-|AS`w%R}*z-Lo=CkO~QQR7*Scs+DVaxb8+X0>t z@_-cte Date: Mon, 31 Mar 2025 13:32:26 +0000 Subject: [PATCH 391/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10220 -> 10221 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index ee930d538..fe8c99284 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 13:26:15,689 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 13:26:15,689 submission_checker1.py:3260 INFO] --- -[2025-03-31 13:26:15,689 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 13:32:20,970 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 13:32:20,970 submission_checker1.py:3260 INFO] --- +[2025-03-31 13:32:20,970 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 13:26:15,689 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 13:32:20,970 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 13:26:15,689 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 13:32:20,970 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 13:26:15,689 submission_checker1.py:3276 INFO] --- -[2025-03-31 13:26:15,689 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 13:26:15,689 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 13:26:15,689 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 13:26:15,689 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 13:26:15,689 submission_checker1.py:3297 INFO] --- -[2025-03-31 13:26:15,689 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 13:26:16,662 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 13:32:20,970 submission_checker1.py:3276 INFO] --- +[2025-03-31 13:32:20,970 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 13:32:20,970 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 13:32:20,970 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 13:32:20,970 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 13:32:20,970 submission_checker1.py:3297 INFO] --- +[2025-03-31 13:32:20,970 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 13:32:21,960 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,14 +26,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418679 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26733 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24279 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.1384 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2378 | acc: 74.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0422 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4d042afa2..1a8c1ea28 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1018 +1019 diff --git a/summary.csv b/summary.csv index 3a7a606c2..793c31191 100644 --- a/summary.csv +++ b/summary.csv @@ -4,14 +4,14 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418679,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26733,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24279,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.1384,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2378,acc: 74.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0422,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d34fa02dd..c237c0c42 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26733, + "Result":6.24279, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2378, - "Accuracy":"acc: 74.800", + "Result":10.0422, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index dd00ecaa47692c8945e1390b7513d89c61d73ff8..07d6463175dd86c4f783fd3e14ed7838c4b4e4e8 100644 GIT binary patch delta 2415 zcmV-#36S>ePwh{z%>xSJ#-ieK1ONbf5|e%i8I$k>2Y=gc;yM(4KVK0=eQ2eDT!4@w zU^J!o(i>$?b)Ha29Iz&dGqKal-;bR@2ex;aYF;Xx7f777_AOraj{9F<(!_Wo1*a?< zm`>9+4U&Z{qS<0#exIH;eAD0}$f6)&85x+bgqvT7|MZ_(@q;f35r#C7@qxJ%BJWui z50@kjcz=`SB$H2KR-}QDx5dKZc|n4xBBhDt+O}t<0nNS@>W!PBKC?Z+qHY~Yd>0|xKSsAFQj<@1Ew)4h5Y&vb*b35wwye4GDv1+eXgvWsh zhW&y)8%5=T=Ye`rPS3Svl|nUouZC4@)@op~9x!Acs+dp1R^NJ3hF6FONSSU7Skl{@ zOMe%4>1rPAZ4VP2(FQT-h|V@*sw2EN#8#9oJj#~FcR-Xnq6?z@BVxJ>u@z+-kFu@t zT@a;?@IjP+L`-$WAs%H1jq+PZuO)cpM+gMedxSthvd0Jn%-$0O0w&if0s*t@41svt zkaGkA-u?vw0j0l0AcxzKD;?s3>W&czsDJJ?0s+;%K_H;Iw+IAO_YQ%8>fR%ecf{-A z6VB5OfZ8AFkQS)g2!Vinj}ZvS_XL4}d`}Swm`!JE2(l2JBXls0E_To_*U#4L`4t=( z>kitWR@abogK)rHK@ecZ-Te+b9-jYqcQP4Y++6KfZ335l!h(urId6t6wbDfXrhl>r zkXJb=;u+0iQpiq*%mm?L#>;t1xoQgd?5WeVEm06swI7!1GOX+IXvHrGm%BN)7Ry-> zs5vv&?i z&c{{f>#73{!jTSXfrWO2K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)*ZBA zrvy1SItMHyw}@5P7Wtp6x^iA-qHM?yX=UQY_udy^`)>sJ8R>mMfKz`%fR0pPd!y^E z}xaaluLhs;<;T{+RLuS5)%IqpH)o}q5*07%k3vwM43z~5wA+b!PA>o#UgtZ!6F-j0oizW`860|XQR01E&J0000#0O5OI zkh6FQ0|(DScUosIaVs*t{pU1%bDSMZd6j^2=tGDewTG!o{oAVe}s~j}ZIATk? z57D^N?n5+)W(m6w(*T-8185f6eOLlOv&inF!R~(pEk5alpb2(X<-mKs*Fx?;D_ z-$UjzwfkJ?ZM#ppnr+&il{ahZ95m57VoSdd(YVs@Lo|qH3BM210GdPtXcqZ>SOP$^ z$nUc`_<$z(eMl#`cL`w~8U}1K!w<}w6&rqjl+7pYURF|MsabQ|@smCN@ywiuvbvdr zCR%?-Y-#u*8dn;Ahz8LtVfbMhK$ByAZ6Ckq4>Z8Hx11&!3M46T(60ANn42Wb_A0YBMS$*<}^qxfIb2a*C zTM((6@}9D>_3p!I5DkNqU^0n~UYMS)Rph31^+L#VwST>OcjALQ?Ju15$Jxssx|7wL z9+UBGXvBZ6-XUpc?~pXJWWB-pViJEC$@h71SPSmv=`WNOTQmB1gVoz3jd?T{OThf%DSn8j&nV321JF)SDW8w9E7n zB|b1MSZ6fUXaJ8E&0WPU<;KCAGZtKs;FAc_o^o5GZ3vcP?D-ZH^I3H0D6WlDEX2}n zux0#*Z2%W}{(uz(CRl%9cn?hNL_|lV;1kmRx^4srj-dkuI(SSng3sOnPWjn}QMqs` z83qfp){{87cV14$zVEZX9IY`w!C$M__i@!!>iwxWpr#U>atACJ*Hrkjh==Q#536|M zwFL1rM1AJ%xX7UoL=pqgL)??{Sw6qCOVkv3ai8X8T9THOv@8~ALB97RulmnGbsTat z@dP;JzWxNwKeJvVgaQTP#-ieKljkKt8{vCjkmm{j02@>Q02lxO00000000000DysK zlS3v&0fdu@CL#pzK^fAMuO=1&4wJ?vK>@jw2q!uLE0aSfCL4YLC#fU>002+|000pH h0000000000004l243mT>J^^2o!zUvK5hVZs005rUVsiih delta 2402 zcmV-o37z)sPwY>y%>xQpzw38#1ONbf5|e)j8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`CC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%iWw?t7H*` zs;y|rE}}8j3d^chz6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*%U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0t*!``xOyC4ag;F1o%hIxr27bVvuT@(2Wq4uL??ArL4!1Oi3Z z-yu4L4n?s!PsDDPeWe<-@o$mtJ6_t4)Go$Yje@9=}I z&c{{f>#73{!jTT?fQ5E~K%nXn2vi*cfvQ6wU}rw_cc>1bgKRH$&@X+cI)6AY)g5$U zrvy1SItMHyw}@5P7Wtp6x@uV!qH4(xX=UQY_dXV2_rD16Gt&D50nYt*0(9)QEB8Gh zKkMDMbWYfcOh}e-Bc!rm4fAcZId7ObgL(5U{tK=?=8Gs*D_f8?CkG-eRtEpk98q|K z$)js4V_`h2GGd6=Qhw5tmw!^eBbBRZL@#l38exWeTc@DRv)4PE}2Gh$uyEnrjcCgBC8KeAh|3`(AHq} zffk>1f;*QG)}di?zGNa;#OjK@K97OB#i!w=H{nnVL=78!n60zk9K@T0--11&!3grEtA9}B#i#}Cs0nnVL=7CC-c z0zk9K@uR`<11&!31bhV$)nQRUW*Nheu7B9@^B9@`tFfSx`bt3wW*^(kTgqBFP@-|f zmS!KK@ub;@Xb{R0W*?>jD9L!_c8M-B`>;*`$|AFm2D1;e_@onMwj`C{_Mu^bCA0g$ zQnq5d&tq67gN#V^q(`!q)K+Q*yY2N!@0>k1Z>vPxIYg3o?ntW-(RkA8Lo`Ta34f~( z(*TiVJV0cT)rWNgL>5_nG+2G0#V4I8(~?Aj)rW=wk<97?L_R00Pd<^}lZbq-Mjvep zB6U;VQ#Q8VeK-xGVQ>;mCb7{A)6=zz+_bJ<2zjpduUGF*e2}O8g|q%Rd)Y&GvU<~F zGM){M_|MflB<<`Sl4h2yH#lER0)Hdb34&qeU^Mel{Z|DtEV`o++%j6)+W{a%_@x9lbRz=m~nFz*h7 zPu*xZp2fx_7^H=ob*E-TtM^eZKPTaQ`6>OI_BBkBaC-l36mEq|s6+kv*^)Igyr zExie8lZaYBNNOFweHABh67Z?V)y#f7yQp65RhRsN^U~-Bkt|0EXl{hm+XlU_m+1pa zd|+Cz&S+@R03I#cyP8|dje}QbEVv-SClRDQ<+ee)5G=*m^BpMWv*^%K+!&`=h^5^Hv$C5(18LSJSG{z7jFQk{A|OhTsf5t zgN0e^S)AN^FDGN)_gP<#)|j8*&(+J@xN0i(;Zz(@TMJIP0~U;HYkXbA!*$GuRXp)V zf_NOFKJ#{3*_?pgULpPtuC)E0ShpXOy+k~Jx5Q5I2BeH}*L^k0DLH00mJ zGvJu}_7k*!vtJ{G0tHyV>vwUJ=p{iMG}w3R=L!G-8&m)Q7ytkO0000000000fPrQR zlm8_ZlUgP(0fUo@CL#nv#^|k+uqGA(4U@+vK>@gv2`4%MDw9MfCIp!S4(^b0K Date: Mon, 31 Mar 2025 13:36:13 +0000 Subject: [PATCH 392/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10221 -> 10221 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index fe8c99284..efc9f0a3a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 13:32:20,970 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 13:32:20,970 submission_checker1.py:3260 INFO] --- -[2025-03-31 13:32:20,970 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 13:36:07,884 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 13:36:07,884 submission_checker1.py:3260 INFO] --- +[2025-03-31 13:36:07,884 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 13:32:20,970 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 13:36:07,884 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 13:32:20,970 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 13:36:07,884 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 13:32:20,970 submission_checker1.py:3276 INFO] --- -[2025-03-31 13:32:20,970 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 13:32:20,970 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 13:32:20,970 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 13:32:20,970 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 13:32:20,970 submission_checker1.py:3297 INFO] --- -[2025-03-31 13:32:20,970 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 13:32:21,960 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 13:36:07,884 submission_checker1.py:3276 INFO] --- +[2025-03-31 13:36:07,884 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 13:36:07,884 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 13:36:07,884 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 13:36:07,884 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 13:36:07,884 submission_checker1.py:3297 INFO] --- +[2025-03-31 13:36:07,884 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 13:36:08,877 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418679 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421396 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24279 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.1384 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0422 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0431 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1a8c1ea28..49efa1e2e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1019 +1020 diff --git a/summary.csv b/summary.csv index 793c31191..3d32b54f0 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418679,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421396,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24279,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.1384,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0422,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0431,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c237c0c42..c1dd033f8 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418679, + "Result":0.421396, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0422, - "Accuracy":"acc: 75.400", + "Result":10.0431, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 07d6463175dd86c4f783fd3e14ed7838c4b4e4e8..26d9cf6a37c49ff64037377f576b06a61385f114 100644 GIT binary patch delta 1909 zcmV-*2a5RZPwh{z%>xA7H1O`tz$+LL~VgrBDs_ErW=WyyA zq)uFGuTkQNEByo|Kpl}t>WD;AMc;x5V%H z$&pBs7}4ctlqgeZ&YTL79FA_fz3hHwFTH(PrP>>&K+eA>VbmMl-0HerH}`rArCTLX ztfG_>C|)^RC^p@?=Xz2WC(-33x=5n9fL^1-5tyBz1V|(jNg|O*5{X2T=oXMj!X=3X z;cW~D65Rq4F&s#AC5a?POsJnxqD-MVbN@XgR#8d`60hBzNwL||PAg|p^s}uD&;x(k z$+7xqlXI!jHDXJv57D^N>O(ZZCDTYQnMQKSG?GhQWc6VQB$q`A+8V4r(BhL$aOV=j zIy6kqmrMkUSY5H#=P@vU%HCxqMV48}>TSD^)^)e#<~)YgDhEw8j@Z)fLo}|m`w$JH zS;FqaG=L`20GdU1AC>^nEVBD(u={^Ni%&WsXoB5`guzOdrG}NPuGsDK_mKHa?LHTJ z+wPODW}CKW<;|Kp2Tin&*wXJqG_Lgf5DlVP!tcX0fF{uZnnivemH^N!^80KKKA;JH zAJPf#T|!ugh5?(*@B_1E#fG0BW%Eh9mz5M*YS!F#{A7=RJTvE^tZwF@iPnD+TN-|d z#+8O2qCqrE7=D-r&?Fi_v&itn5&)V-h93=vA87GOCj?C}{E#r5HJRTBX3dKIK7S9H z&(!d9p|=e`%URQE<}GMjIcTDJ#Fma9qH(3;hiDMZ5{@6H0W^sQ&@6KNumpf+k>f{$ z;|E%N(h2wqAgaTnfXp(6A6W|>Pe4eE2*v23U=G;lioRdZr)ajwsVLi@7$4AAENQ3)rV-1$P#~6AEp5!$#{Us zBC8MU1c)rM`e?BFK#Na0QKlt{1gj4X10tE#2Z(%5R-b$#y(ba*T#Y{37DVc%yr*nz zz58$)M8n`Dm`q}$7pA9c6}f3$y%6$T?O(6no%kS6`wM6NarUx@?qv0*$7DPk8u6d2 zcSzdVJ0#64S#NN@m;`@D@_im0)`Giv`U_>n){Oq$VDe*B z^{c6ErIypwzoo!!1umskgS!gcQ{d8(cQ}^f)ZmJ~h82AcOMR`u6?(BkFIMQq3cXmN z7c2Ab7*%ZzcQvDGC7!R?w2ca)WhGevK5Z z;{UPj$*SpyspHR;_a?&kW!^g(amNGysS-O0k1Fx-SJQjRe*4sDG4W0Mq^}l+eVu$T zlJa-iXI9>a***}*f6uS<*XWL4>PKNbZcL9~>_=fduEKaU*AhRA3;Jwi=(C@j8gQC(+xe>OWomap5s^tt;o?h&2X z-rgkX=#JyKd*dF_ZO;_*;Kk&6c{#Zjh$DR@k1nUPE0&@|i!s>^3B*TBM7|IL+fc+hsY)mn2<%wxAD#-if0(F4{41EcG$ld1>H1mSyMkh6IRVgr9tuI=RfnRhsK z4pJwswbv+d#Fc)65}=MqBy~h0sUs3`g}!$Rs3YOxkN3xd@HU2fBK&kaT%08z!&~C_ z{NzX^NsQ?7GfI>xG-pnQNDfCg-ClM-vzOk!tWxa_Qy}NxlQ8OyZf-E&hti<9Vb5?v%wTtKf;;t0%6Py!?pi6oIoB#A^KNpuTHB;k_8 zg77wm1Bq?{i5Lzfx{^c^BPP_(C{d=+oVouV600bs1c}$~&ZO9EX{VL56!o*L4A29A zI?s!)P0po8*N82xK1Ab6s}IotmrNtMWE#mO(?~9Lk=2JKkX#leXlt(DScUosIaVs*t{pU1%bDSMZd6j^2=tGDewTG!o{oAVe}s~j}ZIATk?57D^N?n5+) zW(m6w(*T-8185f6eOLlOv&inF!R`ZpEk5alpb2(X<-mKs*Fx?;D_-$UjzwfkJ? zZM#ppnr+&il{ahZ95m57VoSdd(YVs@Lo|qH3BM210GdPtXcqZ>SOP$^$nUc`_<$z( zeMl#`cL`w~8U}1K!w<}w6&rqjl+7pYURF|MsabQ|@smCN@ywiuvbvdrCR#^-Y-#u* z8dn;Ahz8LtVfbMhK$B3*eV7J_B;x@h zi>yAZ6Ckq4>Z8Hx11&!3M46T(60ANn42Wb_A0YBMS$*<}^qxfIb2a*CTM((6@}9D> z_3p!I5DkNqU^0n~UYMS)Rph31^+L#VwST>OcjALQ?Ju15$Jxssx|7wL9+UBGXvBZ6 z-XUpc?~pXJWWB-pViFjC$@h71SPSmv=`WNOTQmB1gVoz3jd?TkedWk|WQRpR-UI^FFZh%cY)6yi^A)FzRY_kBkp+MKUHEU;ZY?X{%U$J*>9g3EhfH6pY+wju&fB5;8{u zlC?i~547tZ2lk?-`d(XtWiQVSA2khMvPY=N4u!>+EvoBGf6rzo$ntgDmp*r2#yz4F z+uNHY9o=ypcW>Myy6u@_9=w=bFE1z80&%2|zEI#c;dAL@iatz z=Iyx1p$|k71JFaq?ax?J+IOM+m v1kFFQUn7J91meb`;*;kkLImM^Uyze6CVvF*K^fAMuqL_$egG$_lWr$}agC;Q diff --git a/summary_results.json b/summary_results.json index 967494b5b..6246431d1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0422, + "Performance_Result": 10.0431, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418679, + "Performance_Result": 0.421396, "Performance_Units": "Samples/s" }, { From 2c76106c1d788fc06231dbf8bce26a098469cab5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 13:39:14 +0000 Subject: [PATCH 393/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10221 -> 10222 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index efc9f0a3a..d2b26855a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 13:36:07,884 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 13:36:07,884 submission_checker1.py:3260 INFO] --- -[2025-03-31 13:36:07,884 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 13:39:09,188 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 13:39:09,188 submission_checker1.py:3260 INFO] --- +[2025-03-31 13:39:09,188 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 13:36:07,884 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 13:39:09,188 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 13:36:07,884 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 13:39:09,188 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 13:36:07,884 submission_checker1.py:3276 INFO] --- -[2025-03-31 13:36:07,884 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 13:36:07,884 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 13:36:07,884 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 13:36:07,884 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 13:36:07,884 submission_checker1.py:3297 INFO] --- -[2025-03-31 13:36:07,884 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 13:36:08,877 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 13:39:09,188 submission_checker1.py:3276 INFO] --- +[2025-03-31 13:39:09,188 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 13:39:09,188 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 13:39:09,188 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 13:39:09,188 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 13:39:09,188 submission_checker1.py:3297 INFO] --- +[2025-03-31 13:39:09,188 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 13:39:10,336 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 49efa1e2e..8463e0903 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1020 +1021 diff --git a/summary.xlsx b/summary.xlsx index 26d9cf6a37c49ff64037377f576b06a61385f114..188a9b94bdd81aefef83846658291a1a8ef8379b 100644 GIT binary patch delta 339 zcmV-Z0j&P*Pwr2!8X^T7H;$|&lOG};f6Z#cFc5_AeF~$yEjy;A5w-(u$*F-tU0QMz z(k2nLevs5Ue)}qR>?Gh*kE@yec6L#*I?5*c1?z-XCCZW%MWC4GLhVcRQLp1i6gf{7 zr;S!nq7EEdRd)q5glP+JRvWNhfQtf1Il@ed4&EDrv1132)Fr{8g1FOGQXfjYf5(Qh zZ@LG(NYe)_!Bb8>#V25FCL%f`Ih&B?$2L8HGYkz#P~PDz$?(PN!AdvV&?{F?DST&O z);fxl`{=}EY}+JPJDBMSrt8#j)uC6jL_Q~_X<#3v&L5+wit002^+l-&RT delta 338 zcmV-Y0j>V-Pwh{z8X^UqG|?p_lOG};f6HpaFc3xe{R*SKEjy+m2-|_SWYs{SDJ|Uz zX_AOq4x5P%%90dCpqS=D9ZK}E+r%prIZqX* zwN_A~GdQ%a9tvg%(-z*WHekH~7X^@Vgqac@y*C77$BrPWOM*iMaj&hUK9u%=e+^|n z^Z%9T?Je>N~{ z9mUCga$+(zO_MZvXASu?{Iz|3?^cb4YEQ)gRRw2+S)g9qs=(JpI9yGg--Z+K1@NbH z&}ZI;MfyJAQ3yc#xTEr!KfmlYB&b@Xi^n+4<2>7CImwqKU46GBZ~8AlISjd*I0E|I kx1XT;2eV)!3j_t7G|?p_lW-?g0bi5FCnE+CB>(^b0CVz`uK)l5 From 8de8ec4e2aadd54fde4d2e4dc074bca0855e4620 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 17:19:20 +0000 Subject: [PATCH 394/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10222 -> 10220 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d2b26855a..b52491bc2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 13:39:09,188 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 13:39:09,188 submission_checker1.py:3260 INFO] --- -[2025-03-31 13:39:09,188 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 17:19:15,017 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 17:19:15,017 submission_checker1.py:3260 INFO] --- +[2025-03-31 17:19:15,017 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 13:39:09,188 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 17:19:15,017 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 13:39:09,188 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 17:19:15,017 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 13:39:09,188 submission_checker1.py:3276 INFO] --- -[2025-03-31 13:39:09,188 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 13:39:09,188 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 13:39:09,188 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 13:39:09,188 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 13:39:09,188 submission_checker1.py:3297 INFO] --- -[2025-03-31 13:39:09,188 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 13:39:10,336 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 17:19:15,017 submission_checker1.py:3276 INFO] --- +[2025-03-31 17:19:15,017 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 17:19:15,017 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 17:19:15,017 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 17:19:15,017 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 17:19:15,017 submission_checker1.py:3297 INFO] --- +[2025-03-31 17:19:15,017 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 17:19:16,235 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421396 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421317 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24279 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.1384 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0431 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.461 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8463e0903..c7781419a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1021 +1022 diff --git a/summary.csv b/summary.csv index 3d32b54f0..67f5d20a2 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421396,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421317,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24279,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.1384,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0431,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.461,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c1dd033f8..7a6b78731 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421396, + "Result":0.421317, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0431, - "Accuracy":"acc: 75.800", + "Result":9.461, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 188a9b94bdd81aefef83846658291a1a8ef8379b..69f813995523b4705d7661f7f6354e7797ff5ce1 100644 GIT binary patch delta 4102 zcmZ8kc{r5szcvhpZ?>^y34@uTtcgOlG%}1L`jtX^rQLMQ?Xmj855P1uK~N$9;UjL+ zF#^_Jxk_A*zS~6@3R}5CWpp74g!tFgI;wQHXN3U=h_jEz*?q9HL0Eur%QrNWZw-?l zrrQ|$y=J;sJ?LN$(Hk%RB}gwWg<9@02-8OAmRlrrH7vf7{;p6BiIwPZxprega(ZE8 z%dYtzrG6)h-2qKUFPwpApKtsal%A~m1MsjSXsP4RchZS?jZtP0V%JQF+g&~$<7L}0 z>8`df;lJ=K{OzvjL|2RXJoW^R8y_`0HXjnwCeE?tny4Ee^2WD_?L5Dfu6p!ZQE2%6 zA6@b#AkYJ6f2b)BWrmVz)1xWwy41kEO)b{3SU1;+L6yR*&L}u*p?=rWj= z919(YA8ELeO&O?tox&(9RwkyFmx4}p5Yx@YaaD|$&O|OGb&K8ec$K3vmANw7BXC^0 z=I7D7Rxq-YnWEV4_Ia2?h*6Z6jW6u7#ahI(h~V2jjTKH$wG@L(qD|e0gY7H%9km~5Tkh2QuAICn9I*%3`oggn=s8zJj~1N)h6pugbQ?Y*LcgA#eA=5#iV* z4oG^PI0UT!-WW{lJi0CLm@Ht6kk}RsGNYI`(p1b=X}OVH{iquE7AKhqixj^MZOg+1)g?nDssy98RFV(H7@b6j^r?^5>8u| zf44%#cHfKYoP1$jgyo1TEOsHwkS-1}cNUt~GgH1`x8^GADkrTre#a&Ac-P7|q;qgw z%k8RNOn#Ku(YM^dHj}C7t=t_wvA!F2JXu(U0BY4`YU^cGdfn2+e*Gb_o?Qt@g-w5tS%b?W)I*#$rg1>5L~MkSZND{mQTv>y`@=8+ZNy61BY@ zR5sLp-*;%%nqSz)=Hud?cGJa`LOPZ&DXi4+CjpU~ua?;%CW5pPh1X&B_tbX+-?HXB zc9#w4lMQ>T0{rWl!yB;j6%|G)cooicl;`keQLURLH_466=IS)ly06Y-zuy4ayfko3 z2kQA1NUQ1|sK$uv4c zl;~jxG0aE5_sG7{-=>J56mGU#;H_v1Z160e?=bJ ziCU|)DX{RZy>0i8)v)*Mx9FG!lOuNI8-||h?Ki;3yOu?Ozm<-RjGj}1@4== zj6T5m3wKp`bM>b5nXS_e%$V+}0Djh%$8PJvg4R7mmWI||kNAD= z23}}8`S3nli@X6M-zRy45#lI1Rrv%k!4;l@n#HSbT;XqnV_-Q_dVN#pChVD{)v2fq zAEBsHRwkl>phnr(?diTTGf*#GPmJY7I-pD#yt5k_Kl5Y3Y-rNY@x(z@vY-_F^4Us6 zF^&In<)D#rk!kD8Pr{O+J)V|O%vYt#7J_OCqZFQ@6wcoWRg^RKRN%r8XiBBfeP%xQ zrn#&_2k`Rqxf+m_J~?G4d5*3OYi?B?nJQ%08GVo8Aq&E3Wi^2A8zjkZZ0hIuu zjL;pb#q^^@Cye~2>6HaW%8*UoOf7{g2vIJ^9;@|HA1i zXd>N`yPP>P;Gks71~Qx>d^Y`wzJ3i|=CmgE<+>MymJ(ZpLe`JTA z+~xlNx@kYb*oPc$(;$5rKCu)R}yE1WEYY%o$6NU8e%r}uE4hswStrW^%t$-U$m3|L7V#zT0`%`54y}B z`yyFCrVv1szYA|E`HY6#eKYri!`qi%mWNpCrsO+!d$b$;JZQhfPrLF_w~ydJ3lruU=QkRyby7(X#xq z+N?l>;7gj2>DP7^9%jY(X(|-+y?f6AL7B6lCwB zD+%B9itVfF)Tvoziwho4U9QHUw|fL#_eA&&tio^%cqxZH2YwhKo)IVY@#ypo z*w!!AmG0Z^u9`RS9zFGT$;> zVBC4Mz_;4Fdat1t3`WDLjkECqyk>^miL_Q{PhPTGe2x|kE0;?o6+Tj|DE{h`c~r$! zUEG*o?)JbuB81sSWt+9#=moz`|41pR#G#->v06*kY^jyG&LZcmk$q&F?KEzr0Hi6=7f=g~D&lgAvP$LygiO*cqD2cd zR`*#zg-7xwsbKFHamK$fB z0S*qI{^pMcEkifac6tOY(Loac+H2-+aF$<@^K1NV& zuOeqfG34ycr+Be8zNmo>M&M@q;3YrM;tUeh?X{8ObVO*ZnR7y*{$F zj&zdkWf4|nvHPg=mdyI3^_g_xiayZmEZ(v0KmJTT(5_95#5q3x94zLacTz z!%^sSx{a)?_z10cUea9uykDhFyywE+nI#-~a3{3N>*uffMP?nwXUp@&k6!2Im3C0 zLZZSzr(3feyY~Js=G;1INn{vgJfadiq?vPGdXmYNfU{zQS0w(a|Cpu3gdLh+no{Neo%Y)*?qECF!I0F0#&Y|Fe#0DARjaR85LKxHj3qr553WTT zh@wL%9!~;9EMh+_UN!8x*PoMHeeu=@c1#t}nzrRtNw3$*F@W_CRf49A($L# zfIgnh1i_S|8Mu~yTqHx(AJQq!ms~zW$uK7OSysEL#S%j{=CSVMm+!|l@$Yb+)C4~i zFuR$*ZqN!E%Kwpi)xN2^wq;RU)uuVrqLtGNj%$}$HlJdRiw}blc*7IDlPEe{yPpxZ zq~NsP1y|cVdQ0L}Jt2t-^T~T46+SIb}@q(B}9ZQBcmB>Oo_2Gc7-V07&JzOVXVnEMYgOVq^wyY zvQ^Z4r6gmgtjRNdpXc|S=bY#MbMHC#oY#GSKJU-xoX=ghT(ztnX5;!auO%1^1OOr+ z)M&6Mb)5}FmrO3Kylaz=6We)qmr<=M3juk`$}jId z9PK68pyVV?IPJSCo#jgEpH=%j?tIhXZ^owqL3@6j z(`njzt?#cBFcWf~@#}@WM{k{Pja@ZMdYq$+YOyRrX7*pRM}L8#ooV^_y!^3Vj4@I_y#d-%(CWcv zsOS!6y$(0X(4leDe#^PFa@Prail^2;-N5meq^uie5RjOgJtw3*LCKpArZwI7Al&vdX_d$;#5Obc}dwh7tl~rT<}PWhr@ulu4YSC`-=8F;WD;WrWqU ztSpo5hB8I1&0wJI4lgm5L6@V1_bh!8d^^q)J1xCPftF2!iLp_s0^t4~AoE};0M0ST z`+@8E^W{QSlTU?>o_;?vwX`QJaeRq)x(NH85)yyC!_ko6<@M)7-t$rgN8#aWqRyWj zSFQxGW=D6$-SyNCA|V>bkbg=E^&rX>?UQq&!)`YH7NjBI7q{1MZeMEg<=R~?AAhD| z&2Q5voyAbATH5gLJq^xo({6^ovFER7qMc|XPut23y}9^D>8ijfyk+3H-*oci>IH6l z==hzz-uq;*ch_+IOJTXZnM3?H*nzgJapfO+lAlfA{DgRL!{o1(WV;|VAr$8rio=Dv zh)FR6b(|Lf3rVsPX~r^EpNew0u;kr9&|y8d2?+GMmCZbq67nd-V5*sHXGL#Y zJ+cZdOdAIwcSb)$@>iZkudU}lecNZZuvEdu)0M{SL%b7~U{J*12ir2K6~;0yc>S&L zGT!2N|I^H@MVILi&Z$mGK68Bn)Yj4gAz)1G2k2g5WoxjL7tOz)&&PAFcZV+Gau5XE z6vK9?325elBIACqTFB2a`}y4eBY!NydZqrlq`2Nmy1YR8VAdrR#Yy# z^|ZZmJWY+0bn@PIWzpdMKZt=pPR|#9Z5yr4-uohToXyzlC24S3>Gb3c7RXd>A&5Iz zQi%J(4Y?C_n2Er_#)`5LtqQ-+O~xh7r4V@_`h7SWf_!WhzPY67g{ca%HKV5_lU@O= z&}={<0Dv0+1^@sT034Ru0}bI|4>rQ$yw$>WhQwnFN52~DIkNI05N;ruhGLMBHm0yS zuIM!(E7PuC>9th$F9FAPkh_av&(ERtNzFZLdjn~x^yBa5@fnJjz283MN|Q(lY;9kT z@jBfeZC^4iUTX06m4fSp_~%cIR-D#O1b2x8+^UL}0a0|)c2Ly(+`13}RUo{Zhqa<; z&tsh)^$6tT7XeW)h?#5}3X+6O{{f|-y5LEF9|l1_vqH{DphDqEu#1?oLTehPSUcoM z4~DhA#0jbSoNre5am+#)Z}aoqT`8KPOhhuo)F3ntVvvS{-9^e;Ba;+#BG!b9P+jxp zmWH=x1dARHaukNjWD{s=rSWG5GyJS+X**?jr^e^zZSAV`ywGgXRmTH-4Z2dX1{k+M zhsHU54vaAGoB7>mZ!4d8IEiz6n7=o85s(TX~>Sz^d-uwXJb=}#h9AR};!hxZx<>`J!-oQY+&*loR zrC&FsbD`I7w@7{9vU(DDUL~{Pn%(b~gSxJT6N$iD;RmKmzu6v&0KQ0ys1?b^Er zrT!ac3JDm|sS~d2j+rG{4sg5!`|rrkD%xwc)NAkAW8bv8cZ|B%23~W56GpW(EP|vU5)7#q)U(2BC$aG>pnXhO49)hc`svZSF8Z=mEp5 z@*`8{S-fWR0;a9{L7;LX=4B0`>U!I!V0doX4P7&NUkZlCqL#$b4|Di!;8jV+qyzd9 zs8PB^l+5L(-N%APrN@{lf2eNe`%vBC2;_pbb}!d%XjI2HWdt{cA;JcRHi{KB2+*qq zOnucS|8LlxeHXC*FYa1yR%=NpyNep0Kh{UjPqnw8KULk{65hp|dyjcSc0ejYXZuIg@$4%nF?neUwp6+WbkVZ%U-lQjg&WH0b70do++KNWqRYu|wuCflq!8`&lz z1z<_wAQffJwPl&TEB0vi1n(&Ny!;mlDFX`fC3Q_I*KN!)8r`HvocZR}Z|*pj;UX>3+67@D zWj}+fSG$Q6&u0J>e!<_Wg{2w$!Pc z&GX-N38YCSkhw<6hp|e1=PJ&Ka5a&{k~g%@*5K7kDwOuzGHc7u6tef7`V{utBbA`5zEE5h_`N7; zO29!ZhWt8{Buz8aKp&6E;TVO988q~-cSD#uNzyzdX>Vq80?k!_GmHLg?mr-};k$?W z1isvniS>};W}XXg?8kk-t9UzTd(&?BWTi;cZj^jY^d|5RLsFzmSu~P3MiIfkYk{#| zX$%bl9wc z>2L;oi0QCZ`bPf>1n05snh(e@@sj58`d8YZznN0!ig7mp?X^QJ@$U^Q?5o9a5Jb}$ zW~m;^Dq<6)D{odl7iEU<-9vh4~2dydKa(@-R%^u-f@|Mr9HV#w}USPsVR zkx1ziKj9W1Ts!}!~-l_Y4KD+$dM!8ERcY8mKa{g7&{xbN?qLg{}pEvVG(1wk| zd$aTR=-iP&gA@FRMg$g}jrg)Yfa*!u3euI9aDGgTD#*(Hl% zGIs5eV~j=D-sa_ftrfo<*q5JwW)y#H$15IW(V@q>R9lYH^V5eT^-}+EH^d3%umERI z+!t?<5`N4|;JSEOEwe+7QeO&`!awV-jY|&~8f~3D3hkIXE429N&J7{EUzOznC0XMW zuSz-s*iV(%#kX;Nx;#6U7Je>kZesTCOOkLj4?6m3r*K4_(P`|DUat15#Ujc-A$Oc; zqiCUsB%{-5KaHM~!r1($bF<_-dF?q&HYRp95i?n|IzrD$l(}VQ1@0JbB}%1vtEj6- zO~~xf^;J+A7rW`D_=hX6c-QXR;uuY#lR@xm>1GI!*=c0B3lMQ`-69hF1QbEYb7tX~Fe!l;a;><90)eXd?IxjQ#>V8Ya4zH%vB=Wi3g;a?FQa|CmbwicA<8)9? znfo=ykMbHGT+PQe?+T2>nYS_-F&B3IbP5WJxzCAcaeBu@rXt3(fvfQBs>7DZBHYfo zT9LDpJJS$E~=2*eP#38Q+mVNu*-eEBewq;>l$8) zUIPQgwNk3660*{SVc%|5{)Dw?u19Su=Sxcjn%Dvc%85p}Yr%qJ@&b)9lnI@f1EsKK z*zD!h%G|@SFzY&pVAO1T9_&b`-W2jl9Sg5#5h9JpmA% zOBuxf`E3KQv2H|(?A&e>9>|8_0xji&_D>t!mf8%O10N6mCnk=;*We_Vz{>f* zUv%mcTu`|CaX{dIul@@sAeTr0 diff --git a/summary_results.json b/summary_results.json index 6246431d1..252742861 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0431, + "Performance_Result": 9.461, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421396, + "Performance_Result": 0.421317, "Performance_Units": "Samples/s" }, { From 48fc7b752561c18b6b347a235b5c25635ecdbf9c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 17:25:43 +0000 Subject: [PATCH 395/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10220 -> 10219 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b52491bc2..f04960a44 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 17:19:15,017 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 17:19:15,017 submission_checker1.py:3260 INFO] --- -[2025-03-31 17:19:15,017 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 17:25:37,712 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 17:25:37,712 submission_checker1.py:3260 INFO] --- +[2025-03-31 17:25:37,712 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 17:19:15,017 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 17:25:37,712 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 17:19:15,017 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 17:25:37,712 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 17:19:15,017 submission_checker1.py:3276 INFO] --- -[2025-03-31 17:19:15,017 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 17:19:15,017 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 17:19:15,017 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 17:19:15,017 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 17:19:15,017 submission_checker1.py:3297 INFO] --- -[2025-03-31 17:19:15,017 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 17:19:16,235 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 17:25:37,712 submission_checker1.py:3276 INFO] --- +[2025-03-31 17:25:37,712 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 17:25:37,713 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 17:25:37,713 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 17:25:37,713 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 17:25:37,713 submission_checker1.py:3297 INFO] --- +[2025-03-31 17:25:37,713 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 17:25:38,882 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421317 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422216 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24279 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index c7781419a..b70608fe8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1022 +1023 diff --git a/summary.csv b/summary.csv index 67f5d20a2..61c8df751 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421317,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422216,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24279,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 7a6b78731..fe78c631f 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421317, + "Result":0.422216, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 69f813995523b4705d7661f7f6354e7797ff5ce1..74c497b6c1568e699e15479571692fea6f53ccb3 100644 GIT binary patch delta 2373 zcmZ9OdpHyNAIE2IlbAD-Bs8<3NSauQX>*z8nj|E1W-gK3<|YRqdm6{n3-E-slmXhc+&mPy^yf#CL0?dLPGKIOt<;Jkg4=;LOc_8-|(B_OLw z#odA3onM)v6ILh)4x;nuuUn9S9WPj?_tOMJj*Yh`2No+E4{_ZS1u9+ZYp`1hcM84F zH!jTjhL=ZkrLxX2+<@%|0_EE|-fKPsAIPvrMcCaqmED6E8RkH~G6O)tLUAw%4obC$ zOadXeCfLuxXtp;Rnl4GHfl;4l%B>u6VCW+QzdX%&Zs4GA{o(h=NA!<+zX_hfXKC6E zVN|5CwXPpPnNBW1JqQI$wTETtEo1G3EkG4q(;svCpzuT>A}si zinH<%6;1MB%zF``6x}PnU{8pG9}bzK8{)@#8>GjCN%)=1EvugHZ_}?ARVWAa*pZc9 zmJ=d-h|v5ajRx_*`CM|$-;@8Qii^lSXOkU3>R)Lc-I$A-B*!dTn5@r^)U|DD^fHO; zjtCg*p0x=@rVFO7t!!$~F@n-z026sgy_u|57sJqU?s~4i;23OY370n-$(;`6rjQTi zpBeR>bzMmCE1YoP29FzL;GZ}rR9*ft6bWMyZw}`X*)rW?96cxnKdQ}o0X4w^wfWlC ziBVs#k@4dn=uV9Lx-63_i~!&WZA01uhW4iUf{H@)Hg|bw+LGA$d(#5t-b^q74NYrWY|GuSSt+gKr*Z}T+fBEIu%MquNK=h1fjLIW)=-k+T^|J*rnKG<{gjWQ0Kbo3< z1qQzh73+x@=ACAgRkzklij%~$9OcNmYH>koE;#IPNF3(@Qdr?`^n^p7^3SeBn=}_2 zed-NmyZfHElIRJX|A;y}VXJv2Zb$yx!{743c(%fGrjsT+qB1WdD7PCz(&R|MDdn~W zAa`nMMhp!qlUTSo950mUNu&5!S$J<1E`H+q`PW!A_Mv7Po$iCDkTFu zHJ|r7{_=`fUCyW4jBl(*iAJL}Eq5z5oFAjy6HxB#sOL->EwMy_SYi)EQ;Q>EK4E9w}h?X z&hi~6_}@s5#c6OayU@99+Qv`X)PHDCf6~^IkIHHdN<9V7Nv#>}CyTc}#gQbsc8>2a z7|7wRS4{RgWF0iUc${7GS=1nQA=6O73j5JP03OmV(A{Z;mkfd^l$|4r z2y$Ybf*ctEc>Cd}!qu+YtMwQ32_^ZY_N%2&Gx`EUM9qT}>7_GO!KU9;>#H@8Ll*w0 zI>AFt!^;>XIg=#U2_69yU?qc8QCB^IXnxo7ooCD2rAQ@7D~%Cc(WE_soSN5PzpJsf zxOHW;4F_vElGa3@caIm!g&pynNz!63SBz%_DfK`|;)eCQ|MXn`hD$Hbe6RXvxC^!!{mWs4p7bfbN+_7+CRaMQQIqj6F9 zM{9aaw6Rbz&3Q7&yyR$m2TZNvMnTM75|+E>YGj3&N?O<7EL=6k(=I1$g{egUux2Em z3YV85tl-O-a^HY_Xic;m_*-z;j&;w*E`@Pqf(B8a|GMC%TFV~R~*lJmt zS10*GyHk-AZ$2_wSk!#vp3hRi?#{SWmcI}DI1 z816<5G2J+9YAi(ZrTk@AfWBS+{lWf3jB#wJEbS4@++C^hjl(=9*gI732gmTeg`==e z%9)H>QPE1NV^Kb5PKVkSXia1T`i-Ntt*>P8VS>O~*#na=(p{S`J50$+npGnrVui)| z*~xz*k`H@f{@zKxoj!(BGu>nHMssXEy`OEELzxmZkC|FOX9= zz`$a2mt&7`0q)% VCP%-Hu#`AY7a(+Gz`ENv{{Y|NR7C&) delta 2357 zcmY+Gdpy(oAIG;gL&tS4X^PPfqDTxr*;9gy`OBAa9q#Nmi8TOEF$o^dD7Lk}s`murVS+W>^!r5sCl?V| ziGAZ%QCCl0G=GV{-0qe}V83OW+UFCvj#1;sn4f-)?wk!;-N%@meP`4?WF`~&-Bvz- zXkwhwJNvw<_qf5JQQ|3$!c2z+QNLr^WRGpH0I+14V#QAIdYNSeI_>og6z!mYg9v94 z*hCbYP?11@o07H)TV_mXW-vz_c*Z1@`6MR0}h=Mz1UgYJko&=|${Dzj*uU6QCMMYtsNwPqsp;4f< zRzA5(6OFi(9y{)JZ9q8i!)P_>0(YeC8Sv$~FlEsNd2udWZ=!qlL2vZSR^X0PP^8wa z*`%Mq&1ZP$plB=sVq0}z2ObZI#$^{?S^O7~#&sm%Zz<03xv-mU4xQc%oPd8da} zu~+@Q7rMrAZD*-)*DlNQw``jyvN#ruk3zhL?>F;2i&uALgZMaN2PN|Kh`MjwdzQsm zzP;RZhnZE7g=!1;xQ30hX$7W5fgIk`t=3e+Yp~cSiS-Ztwp^t@XMJ9W8@Dv+GjcgC z$kpQLs3n`+h2#<-G`M003xV?7!MUrX3BcODsG(I}k>y^mYj(64h2oYT#n|#&!wb8G zVK)zBK4NVtmEY)AZ1Pj5Nf(QwSGHc0@D7;JjFwE)VM){gSayCf970KrinD~MQ?k7ROt>l3c^Ie zz_)t<0nBrTwQ0%m%Jfblow=}kI>dP_Ed9~0;xxw{Op|JB!l;WtNsQ#6XROH(GW>9A zG_y#*QE8gsdp-p&)VO{AemyqVKwy{$9aU0<7_o@Hvu($2j^}2I>U*Ot{7TByiI|Gc zu{iF}nWExptixAwyp1GupecXq2$*y5CqQQ@kt9s)d*6H*|5&)FqG&;V(^7|jjWX{d zswEmNF7b~fH0wL3vwzR=q%ZUvjD1HFVu>tiwB3Sei`F5WyRC9)OKlRj^06#V=B#NdH1Qdwh!s^cy6)m@Q-& z_*KKTl^>rp{<5&S;%?`8q5Zc%pREcrM?-#0cG;Wk5}nrfepo6odQC;qgZgH)4)1^# zTe3y^Im)h}W12hT)h*+VC;)A>QT%U!nmrXw} z%-vO-e!%J$;>4bE;kpWN-MG|_0P(Z9{ztP!=Pz z(KB{r2pz6?pGsQ|kO#HuzetC@P!_Hgs3l?gq#{db)KI{uc`9Yl4p8Q{cg;oRRm|E= zhL(pgC_^PhQ5}S7rpdQ0T+KhY&Hv&){fnz*a59mman67zQ+6al=RInt!8{BE86df9r!G`v%fTWD5?DB6->5s z*rXKXZGD*u6XQnK0uT-yy3}Z+G!(j~)n&Zp<*iT>o`87Pxb022z%{Xk_GO(tOTd23 zV;;X48q@+afjO64sm8J{cxl2jhw*pu8!C#9)HgGAco2GL`Ug|7Q?PE2yaSc?MeP4& zOB$~ws{sDN4zIwco!9jdQNmU)DZHa)s21H_CYH1RbSd?(sJ5HT6-dUzZ_*DY6BpZd zhpgx!+;hBr^+&MPuHcB2sG&zr^AZ>NhQdaP2{h4m?7EJ_=F}2ZKWCe zo&mqn-aU~YeyHHAh#_j|d7=F95MJR(QZwp6Y(VUD%8i_!?jQ8pyT7kY4YHIMg*CtS z=$_uom^ea-h$S}&@>Kt>icK!atv>v|g}nlkTqF6TK1yl>Oe%98*0@u`T7ij=ij;_S z{J7XMwV`>{YrcQlcn4=rjGw9TQwlZL{<})<o?EQ3kpBFc1~qg z({%gzX-eaR0iTPs6&nS8x1m>r?t;0lRQigYXiS2-v)8^5=zR4%34Ub0=6cdud*;)K z$?#gF0%v1yzhVE&-ld5ecR`ZcO(iHV7^TJ0);9)>G4XN(iRH9zhde`*LHQ6i(^nbw zEbZ@CPjfBhY29NJLsn6z-K+YFF<)gQg<=hTY8t%bgIrOCM(i(pI~kd4uZ8~C#qN1Z zHojdyqAN1BWuLV9{CX(KO`pgRh5vS5&nKl!a?riyv^X!n()>u7ifcuE)A5QFE*=vJmh4743)FtTot>w427%nSfk3LD|DA-y U?VKxEJMc-)8|)sK3VQ42KZ0>sTL1t6 diff --git a/summary_results.json b/summary_results.json index 252742861..2e7bd846b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421317, + "Performance_Result": 0.422216, "Performance_Units": "Samples/s" }, { From 7cf8030ac156c6df23c0f079824a7fd9a72fa127 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 19:31:13 +0000 Subject: [PATCH 396/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10219 -> 10223 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index f04960a44..d671f1a6d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 17:25:37,712 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 17:25:37,712 submission_checker1.py:3260 INFO] --- -[2025-03-31 17:25:37,712 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 19:31:08,262 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 19:31:08,262 submission_checker1.py:3260 INFO] --- +[2025-03-31 19:31:08,262 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 17:25:37,712 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 19:31:08,263 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 17:25:37,712 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 19:31:08,263 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 17:25:37,712 submission_checker1.py:3276 INFO] --- -[2025-03-31 17:25:37,712 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 17:25:37,713 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 17:25:37,713 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 17:25:37,713 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 17:25:37,713 submission_checker1.py:3297 INFO] --- -[2025-03-31 17:25:37,713 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 17:25:38,882 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 19:31:08,263 submission_checker1.py:3276 INFO] --- +[2025-03-31 19:31:08,263 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 19:31:08,263 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 19:31:08,263 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 19:31:08,263 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 19:31:08,263 submission_checker1.py:3297 INFO] --- +[2025-03-31 19:31:08,263 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 19:31:09,242 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,15 +33,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.1384 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.461 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1243 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.411609 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410599 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0464 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.054 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417928 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index b70608fe8..d7b1c440c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1023 +1024 diff --git a/summary.csv b/summary.csv index 61c8df751..13fe186f9 100644 --- a/summary.csv +++ b/summary.csv @@ -11,15 +11,15 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.1384,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.461,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1243,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.411609,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410599,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0464,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.054,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417928,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fe78c631f..a9bb08fa1 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.461, - "Accuracy":"acc: 75.600", + "Result":10.1243, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.411609, + "Result":0.410599, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0464, + "Result":23.054, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 74c497b6c1568e699e15479571692fea6f53ccb3..06e5e89a1df4de3ff059d5fc375efcb210787319 100644 GIT binary patch delta 4439 zcmZ9Qc{CK>`^RUP%%EYcNw%1=`y?L{A{C@YJ=bm%VdEMvz+~>LHK6k_Dhf(zaEkmFzJ|+MJ0EDp; zFp!IcEm|;^;=7>8j=5>iqKR=Y!r4To2s+3X9gJRbF${WUjN*wEj{Cb|tZzO!PZit{ zob72Ro_7J2TfZ-5x1V%uZ(lneBKTNDmOsmI>>4x;FRQ!L`R+tT+2@edrkSFxrW+_D zZBH-%mg)D}P1F78EA%XnWK(|PUa=EbEjgJp;fx_z0w+Av$g#IpwcEO(I;0H+Gb@y= zx1sF6ltcxP8-|_uYt+78r|-E6?R&gPy!IZ&4Y{adFMLiOe=3?fbX$c^X#3sC@~&2L zs|WP>$p)5fr(eDj%F@q|(p6Uzv;>S9XAg*-zTmU#zHDtGmH5C)Gw8$P*bcSqczTSA zNRJ~{tS~i+7$dthQI(3Qt*i8KZtHvJt;4JX&*-hL(I>_(KA}`EM!6>M3AwpECn?Uw zZ+slQbX>hw=QFffGW9bjjpIXEi#ZD)SdeS~BIcUK67B*n%*1f0-sOq9Tv*j3Gw(@H zhgv>qXTGv5qjvN8uv?Gr#DY=TzUM7QDNpsV7lPM3`M&uuv+<75U0sbGyyjRY^hUnJ zJY7nuH=`?9HC>U&NIVg8Y-X(CaJiIRTd{2mD@xi@OUEMhgE}trxquUQthK8UAxqj2 z(D;KGRn^)L5K#Xd8;C}DN{j|b-Z+m7p`-+YXy(q&(Ey9yA_ef26basisP!ZkND214 z91U=yMNQy&b0Y+F`+x}-xjrY965O%zg>7&Q8aczimcm&w1HnaxC$m7h4?xt35ep>C zz|9L%3(FHiODFzyfB4GO(B#H14j!%zrkUsFQz?Pb{_kwmU=Cfgr%X5ap8XCEmbqyE z$!7D84^1ZKy&y|pz}$HcSC`V00jx8#Ko!IL1s!cJFkH>6^{9PFRwJm|2gc*dl$(SU6n@LO9YN_DvHgyE*rr>W_!i zb3XLEKXKGTf^>QZqFXtHo>ETT7U9PVf*R##b8%j`J}ix23LsDHZ6=!e>0!~fHfUR0 zv;#LERjADp3xu7Q21nEoFh)Z_@s+PN&=B5uU+`%yqb3;K)l6%EP6;dsRGn;$H8Y-j z{r$HwIzM#`j64{gWzE~H!~EFFs~G%Xwz5%13w@u;E5E!sz;2o7-OcRbve0=jcfsM< z4|I5O4Wk-qiq(irnniy~T&m8@u}-eaBsuc3$=>>95?m#d|^EF7#Nl zT3`xj4OfuA#oCA5B-xZ(U|LOV8Le*LskO{Ez3kr5YMmw^s6X3NMajBV(`teUy>X5E zjK)}x7Q_w!zyJ^c0B{rVO=+uig#iHICWJ%Gu>?G*adK|fdNr_dTte}1&sXuPZ?q^M zp4^^O_Eb(NYw!J8$Tg*xwzjR5KrgLawA<--clIdTCpUq50Hh7(F^n@{lLZZoh2kt>%BC z(Zj+-_tkDo_?r}|P0|t(7o{VwgXd4~`q^|JJPIU(!^Ph-%O=k5>p1o zJ~`TNd(urvV#$l^EcJAYkWP;Pz9lr%U+2N_&x^Dvl4{n26yD3v7Cu139?m~@pVmI!7M_qFI*wI5sC2u>YHLVLfX6eg%IWgprs;Jfk%Fmk z);M@73LfVRjDy>x{j;&ah?D?JI}Uj#4dvJob952W%bxT1!MuheM^U=K_RMTH`%|gS zx&UF#5ikF>xtC{GdSIQ*o zKZz?)#6Qn7x^l>6gpN1!eOzuB^|Wf}y-fH);FI_iQAdSM6VX(x+8-R8TYyatZbv^B~in3JmCVS4TptnE}lB1^Bcp`s)ShoLGQDI;oC-51Lh`9r3&O%%5F136KHD% z=N-(c(?iNqhC~lC@vDxENwhO7G(|^NRd{*LRg2nR{!E))+bNOhfFcy7Ic?20@EGPWq4T<=t)=bFT<`J(3C4xB36;Qz>0Ijf2P|GIIu zu-Nr#Q2;?98lb;n87;PtjL7W~@0I8*tbd)fS4UX6Xslq)`|vNAby!er*ArJ>qMESY z$Dk~_Echd?HJGVp67OVO-8Xgw)?Gq+Qy4$R&bu)RT;sZCto+MPQ zIX782iKXcK7We&a?|6fL zT?R?brG{c{`3=&OVC%JI)wm>>bxs)7`AR;bnyB7kSQg!L_#Ag+ZTC#OB2YA;Us*TI z&+tX3TLDP4tzCoYzpZPd)D%+jvFqp5-@o(+lrMNr&5qqlDxT#LZ{ZOix$dZkWgcl7 z{j4F2a-mOP9s@i3@;E2AvvF^PFAP6l-@;@ zOI$lXQFc}_Z5(J%sHPro~ZT>^6{+BlU5AB=EQ*1P|d(V|Gjr? zv+DD9C}6TKCzNr9!V8ce&uicLb`biw|M$clV;xM<^K?7@vEJhZhzN3cYZ7OGm~jIASS zsH#qvP@^6?bqXIuW6T+fkqyB-r`Dq=8Bq!%Gec=5vDp-GMlHu{3}3sobg}TSK*5Aum6vP+wcTq4 zZELjD%MN~;C>94Qrh)~t$H1}ShKBj!f-NE}ziconGn~kE(wky$@0KHfZe;ssc$U6n z!38X-*o)(phiwbmyvrAyq~`Q>*|Gt(0A0ZB_g z|JE#!)((b`XEN$_=n>SVbMo=oti#?8%84(r-90zle(zaJOTg7NK7;nNzNP1!K{r+%=6LDyn@9YW5)yT!QH*jMh6AzY zRRuN?hVOsvm>%8O^vZ&N**z;Z8Quh&?3;#JOGDZh&+y;Uz3nzackRhIwk+Ve!|kz# zFft!U5e+uPu#gTaNh{{1`}v;Mjq?(sq1wO-#xoQ_uw5ezNJ z7n!2FtU6a>yxisAk@7q|SihyX7_%E;t_;=sh;i#-bEC$0Z9g3*O$~t z9p~RaepF`cUntu>eYUYRh?*$!)w^?QUOS&pggtjSCv{O$-r`_!3N<2r#!q?nGAAiZ z@ysUFPcYnmxrM+IthALcUPIU_VP(e3wPb#4G)m-Tn=dpv^1SK%L1$R^jP*j04;C{B@7 zl=SRqW_T~BC4to2!SeOI`r1MCThiCHi_MXN_Xv^8jR)N_7@f~{0X;MMufa1x$+Qnl zOfgkbZMB&zkCJm9U6~ddcgDY8=;NFSBDrD7Y|g)$H}&#gU-`;661TZ&ke2lQdeUq8 z?<}+X`d69l^eZzn30}DCquM*G0gBdtY03W#qeFKru7$QzdAa_dD) z2uLsR!#d%TAU}4%vt1e%^8>?B@LEcOgy3kP_30_ashe5_u#F0{)JLR@D@Gjm8ZN*; znSlR%pYOQG^ogZeQ{vh(9lWzl8w>afWt?qZABA`13y-=RDO_-yGHtvM{q0r!P3 zYbG_;5Y?t#czSOD0OQf1Wk7FbIfLxq0CRT;(CC1r+cIcQ;R?5VX#O`mi9QrppQ{#5 z5m9f=tA4bXm5!g5UVq-qo*Mv`}LS*QSQk85HUHtV`;n31fs!%m;b zJC;gsUm`L~o-L7&Wv`4zz3dT^f()!YjaW4JeXhh%ndjdQ;x+)lMeQm6J0fvz5CRUuO|TUf2gVR$g$4ho_y|?P96%;Qr?48ZnE)11 s1WFRrMT9T?IS2$005GNj09gLZpxQ_rln^bV0=!FjFCs+8hWNAeABpuzKL7v# delta 4435 zcmZ8lcQ_kf_m2pI5@HlZh*)inDmAKRj95jjqSn{0QEJs_tRhO0peRa=A~kA{8nySV zRjYPUVpOfFpZ52>-{*ba`^UZapZmF=b3W%h&pC&9f9rnP04Y^hv=;0M2mtVd;?Q7L z+%72v+RQyiit+h^Wr!}!)Cnq#0>@%w@;4ecga+KI?$eVn3CJ*RVQI1JhWy zHS>Z{ErAPli$ijL$A`Q_G5oE58ZQ$zDqY98RMc}SO=A0-*WU>(h}VLn1-cy-Ro8AT zu1)S)wmI}R9fZ?atCNz!QZaO^E#-bG@v>jhuBKQeIgDT*DYyG{?-DPMq5*Ed-_v!b zVxLQ>zq2*?97Ds_<)r?~$z(N`E|y-kT%mjS88&&A9g<@p_uzyjrk&^D&FvJ~(|ife ziINLc&uz4?Yn0WAf*6zv+5=39=ygGD_#W;m(N0G@JI#*C6y3E)B4~@W`oA4Wqm0p* zcvb&s=xEGjvubAVXnlSHg$Pdt&%N9P^+anPR8ACQ)lAt^=vv$WkArJgw#fTQkk=vpdV?)hc1cPS{-qIyE))J%)vThn zoJA@3mj{Ra^&=*uDAZG%ua)K2NWMl3?~IPiPR(%I%iyVPhUpt!+s)Pd+@p#G?)oKi z#1exAypb^EiCKH#Rht{V!kl)-%MDx_9E)KB=$9#5QOdHtoNTK#nHHM6y>WElf+J8! zux%VYIHi#v4ALrj00P=ivI6(2x!1vA?j%96*xsMl~*2)MOif`nzj@D)~`v4&5qBAjt_Q4_!R*&m|L1C;Dda4xKHx zjM_IVwnyKyEZs6)Ck}2%iH>u4DHfM-uahqZ*M)Mm#59~S`N+y$9PQZNX{@dLQaS?? zJ_|Rzj&+`9^FDG2zBrGFJ6X&H) zQL+yPMbm~TjBwKl$bf(Vb7-{@nNJ7=0f;ny6Z-yxwg#OuL}?#U7!W( zhq^Z)$fMx}XvVfz5OFuGc(BWGeXEF+rZs_fBlv|+s6y5z9oT>a7*eFnW#u4u;9EeO z9p)n9Gb|ERAcOwvGwXLCX-OH1n~0hy!zt#Iw}g79GU*dqoPu%QYX94pUK5s~q8OH?szlh2mApi>ySkk-+lc0Dx(W-grl4lWMV{v0c&G5Qgq4CDo$%c*gS{l3``ylP(BDaYc zs|o6}<>}orAMX>lp<-5#(Tu|e+!JQ@i6#nu>8<9$y)xpGLDD+N@wF& z?I|;@u!rRuP%@kwmR6oKcHLw9Tb8^|oZTUdo~YD&4#GKZq4LpVc8;58O`H@9OT|N~ zM!l1wzj)5}amkdtBxu}6N-6ep&ISHYGWj1iy`@_v=W+wQ=w2+uI8H0e678FDbX4IC zdM`b{8#s?>#0mYXnAye*-@?veL43sk@ZMR6pkZ$=@_vxynNP4(L2q~c9W0q$xA~~P@ei76OL_u+fo#x2)>l%WJOH~cG1@D-Baaz@i=2RO| zW+X$F>Menzbaa6NPz*9b9T6?Chg2gr0vS=YA2-l~VD|uE;)Ch0INB9`gHA6|)l^R* z4e@=>!+rvDjZ@9l&H&twSX;*9i|^h}Osh!>yR$P&!-N^_(ha{i7g8^KS44$p$YO!m zN#rtELnn&d&z6l6F{Td3bTT}fcNdRAB`YXBOw4MOuKffx4IIUYt;&L}IiP%RWuDs? zuA_h80dtb6G zzS9GdTA(O~A@4@8aWP1!=L^Ge2!I$hv{I4+)}1IMw-FvM*d~Lxl1V84>`7R0Aq1mE zQVc(vtm$p}TfLdlAUI85kJM_q@BjYE*@!11)%N9Rk}aOR=MoO*7_9Y~D|G-37efKL zD!3E_p6-TdO)uee5({}##ojBo0kq+*(iJChN)PTiNtmg`=B@Rw{Y-b7G#qnXjqf=+ zjAL?pZB~L{ka)%4gQUTt#1;5P0GD6Dx?PwOP*6ZB8ReHZVG!G1=C@u--HDejk9f+5 zihXLobphzV?tD|iq1tDyADyf}IPt1OI8|DX@kJ-F4|i#{1n+;FeSF+?{eSHS`6zM~ zwc!C;Ygd4x#SS$2Qw%2_d<20~M%=`X%*2>Zrh~$&i8o`tg3HD@H=A*qcNBT*P3uW( zHY+TPO|G&QleCWs|7iJxM08ehOC{;0E0L+%l)Nmyjx4$;#(^jZ9JpOY3NIpGJ9)31)_zgX;0SS#Czl=MLNd!K&Bg1Hw4QV zlV&nG=ia{&x=U;t9ms|XaaVgC(#76zy@w1lNEZDmN~4SFmr$@vsJVnRV6F>D6kqrY zNof#@i9{}LCQbQ0IW+wcw_X~#pIRRv{JEs@T^{@W25AF-X#@Pz_GEI{CB?Q&ioH-y zSU$;hOeZ?QfO$I51ejIc{Vk_usZ{QNuli(lc36;y3gO(KIDx@I+{BjaXONzW5u^bEUL*KMLte#Wbs-N+E%Gu_e6voG<=Hhg8 zso9sHDDsh;)<@QC3OGl=0K5UB32PFWilyQajWvywioe{=os^+rs2krDP+Ty=x&41BOg3xy;V8fth$Oh<0nENv7 zgSG9CLmpPxv<=0TpXAQt~1<1F_>hG?)lj6ght2ex^_oiS;yv1v?P2)6-04V1x?ZULrNRO|ek(p0 zVX?C^9Y8OiHlk8oc!rs21Oc5$ZAYSXK0FWr+p& zS;@~*W1>!A{JZlP6~Cndrj>om{m1?h%cO}t9vZ)k*#6FewNHL8DP^eze>C!; zCgl1Pi7m@9J}gGM`>%keSAV^wHTf6;52_S>g)e$8QC0leGwrm7v9`D+uhRL6aYz7_ zxy(Lomwq~%`N(7$zSO#~RH9Z%#OPZGRijDv)f+!-URdCVH5vUeN+NpSc(E4X$ziu( z?Ym>6h$-&AEP>2Zi%h#2eeq-|G+E(Bp?=lNY(hrqst>0@ib({#Q2zT7jbZjzb*K1> z^U1}Ggd)wk;Wjj5k33$U_^40eqN^sor{#Ca>k34Sb58&t7 z@%t)oXv4p#$IJQK*kMY*BQXE88|F`g2fA5*-48_v%2|DRF+uWY! zxj;-;?8e^9KPS@(W_A8kXHFW}cV#_+XPWNBo!>dxJ4W-2hQV7)ljGiDRrZ!dz*X%e z3{NL(_$ZMAt=%xXU=lQz z^So(Y+J<=I^~;6N#^5k&+y9P-b_218p~nr( z_(A{@vcA-A(KUOFFmTy(-O$MVI3Dn`ku_#-5&duZThYd90tRKf$2Q|&a2UGJxlNQ# z<>;I$r%^~={aX08f?2t9mhS81op5mvWHANSvaW@s0IvwWeivf9KCp!?E?jIiJjshDq3D=w;Ou$frEpr{v@ z;KO?-;H;oU~nvu{+)`s zt@claeKNsh%g6-dbB8Qh+7rn=P>QD|I`kA$Va*c`r2Kc3f!pD|$Np~vVE_QIT(%Vd zS-~N8U>pe_6V9AZgd_wP!w0|iH~B7SAOcJPz@HZ6KPxzd8rQ?8NK%cX=9eK6z^U@{ c{l73z_?by4agqFTB#&^N{5<3k-ak+N4;GR#82|tP diff --git a/summary_results.json b/summary_results.json index 2e7bd846b..3a848865e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.461, + "Performance_Result": 10.1243, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0464, + "Performance_Result": 23.054, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.411609, + "Performance_Result": 0.410599, "Performance_Units": "Samples/s" }, { From 636ae7f8c6b994fa728ce43ddbc5f282a86c9261 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 19:32:02 +0000 Subject: [PATCH 397/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10223 -> 10223 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d671f1a6d..d97e537ac 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 19:31:08,262 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 19:31:08,262 submission_checker1.py:3260 INFO] --- -[2025-03-31 19:31:08,262 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 19:31:57,434 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 19:31:57,434 submission_checker1.py:3260 INFO] --- +[2025-03-31 19:31:57,434 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 19:31:08,263 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 19:31:57,435 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 19:31:08,263 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 19:31:57,435 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 19:31:08,263 submission_checker1.py:3276 INFO] --- -[2025-03-31 19:31:08,263 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 19:31:08,263 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 19:31:08,263 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 19:31:08,263 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 19:31:08,263 submission_checker1.py:3297 INFO] --- -[2025-03-31 19:31:08,263 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 19:31:09,242 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 19:31:57,435 submission_checker1.py:3276 INFO] --- +[2025-03-31 19:31:57,435 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 19:31:57,435 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 19:31:57,435 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 19:31:57,435 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 19:31:57,435 submission_checker1.py:3297 INFO] --- +[2025-03-31 19:31:57,435 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 19:31:58,421 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index d7b1c440c..bec4b5cd0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1024 +1025 diff --git a/summary.xlsx b/summary.xlsx index 06e5e89a1df4de3ff059d5fc375efcb210787319..37890f319760fce5afc31f7efe3a880b87b6e175 100644 GIT binary patch delta 324 zcmV-K0lWV1Pw!8#8zKZE&_@rGA0i)r%WA_g5JmU>3ZuO(D<*vi+kv)Z)j**sE!_!e zl89OlB(;wJzOtP-31rpf=-xSVXHdD=t0wsc=cUmVB3X_S(A)^Aw-tI{FVlOJ_`tMa zjnPn{4m?`aHzl`}8wan>Sa3mtPa;Tr%58;qAy|sB=Q~i$XVIafxG_$#5KFgz#g_4J zwgo)P^INPSFu?-DJutNs5gm|%Pe}XYngJjB#6f@ z>Q~+li;R6Bk{E!FagWMJ@$|eUUZQ%I&*o`fqy<@%2U-w1zx(P(Ui6=U>M-Qr#3NwL WefbIMKeJ&Y3j_or&_@rGa3_BUx|+!V delta 324 zcmV-K0lWV1Pw!8#8zKZtbI0nFA0i)r&1%Ci5QOi23ZuI%E2bqCwgYX+sewXMT5=Q8 zCK0uMkkmSU`^rw@B;ZqztC{_Fc2T)Fs3!RZ=cUmVB3X_S(A)^AcNO|rFVi_nd|+Cz z#%QQe2OcfzyOLYVje|F5EVv-SClRDQ<+eim5G=*m^F1i$v*^%K+#07?h^5WP_LjwwQ@R(!-U%UaF@{Xy&{i!&hx)hvp2P_y@m-xDfhijM*t9atA1o6~G zed6u3$gvMZ5(Cg<+_UmoJin|Zm#CiQv->nJ(t@nXBP|Hc=bL`yP5%X`PDB1pJOhro WZ$Cl(2eV-#3j_p9bI0nFa3_DRsGRx$ From 29b042231bc237a4f33d94c62e901e14f28772dd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 19:42:49 +0000 Subject: [PATCH 398/719] Updated results summary [skip ci] --- README.md | 38 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 12 ++++++------ summary.json | 18 +++++++++--------- summary.xlsx | Bin 10223 -> 10223 bytes summary_results.json | 18 +++++++++--------- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index d97e537ac..d38936134 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 19:31:57,434 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 19:31:57,434 submission_checker1.py:3260 INFO] --- -[2025-03-31 19:31:57,434 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 19:42:44,947 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 19:42:44,947 submission_checker1.py:3260 INFO] --- +[2025-03-31 19:42:44,947 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 19:31:57,435 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 19:42:44,947 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 19:31:57,435 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 19:42:44,947 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 19:31:57,435 submission_checker1.py:3276 INFO] --- -[2025-03-31 19:31:57,435 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 19:31:57,435 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 19:31:57,435 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 19:31:57,435 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 19:31:57,435 submission_checker1.py:3297 INFO] --- -[2025-03-31 19:31:57,435 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 19:31:58,421 module.py:5257 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 19:42:44,947 submission_checker1.py:3276 INFO] --- +[2025-03-31 19:42:44,947 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 19:42:44,947 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 19:42:44,947 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 19:42:44,947 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 19:42:44,947 submission_checker1.py:3297 INFO] --- +[2025-03-31 19:42:44,948 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 19:42:45,932 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,16 +24,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.392716 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72716 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74365 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422216 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24279 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67415 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 16.1384 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.54571 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 13.9468 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1243 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3829 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.054 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417928 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2051 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.340923 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index bec4b5cd0..d90f2b1d5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1025 +1026 diff --git a/summary.csv b/summary.csv index 13fe186f9..d3a35a331 100644 --- a/summary.csv +++ b/summary.csv @@ -2,16 +2,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.392716,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72716,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74365,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422216,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24279,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67415,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,16.1384,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.54571,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,13.9468,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1243,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3829,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.054,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417928,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2051,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.340923,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a9bb08fa1..7938ed151 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72716, + "Result":1.74365, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.67415, + "Result":6.54571, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.1384, + "Result":13.9468, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1243, - "Accuracy":"acc: 75.200", + "Result":10.3829, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.054, + "Result":23.2051, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,12 +634,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417928, + "Result":0.340923, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 37890f319760fce5afc31f7efe3a880b87b6e175..bcd907acd5501f76747870ac0346e272eb6d8a40 100644 GIT binary patch delta 4819 zcmY*dcQ_kf_m5SfW~@pPk`S$xqH0rOuNt+h6g6s;qIM`XT8hLfH5z-X*rTep+KSq> ztM;l9O^uuY zwJ-j2b@KzV-p7aB8qo+n_ny7DwMyp^PMxdZ2peE9ef~l^D&IeM?~Xbsrq{LKa_?8_n747m zML5RpVr!DFWiL1z&IrelA*mB4FrT)6&%b7?U>Y5+&GR&PnL{5Zp09VudSErQ6b<5v zK49dwxmAXaj_Mh%&y6D!g5hB@gR1bJ^0+}>sL!TYajoTq^?sMhP5b6@I}%vHQyBT9IsyyMxU zw}W^+Q4rLLS(`u06TTin4qKze22OSM`4C?8+vN2KMd=<6@N!MMg~m=#3>`54h%`9J zCWaoE+=Kv<_9ah>1=(V05wDe=q4!I%I0U7@ZB5 zq{=T#@S}~)$1ds4WBYvK$WeBW`z57ZtJmg(>RxeviE1D)c}dG&9&Oo(HPzNlmQ9fg z5<;1n)T*4q`SBG77f;a@Xx)SpSOXH5hlg~Uz|+rT?8 zC~=DWzNLM8*NWFAt3-F;yRO0C&9nTnLH&OfZjRSKW{V=ek+jZQkuimuTMl zfvz=L)Y0G^C~eatV0$N|q`&jw>Utpw_18F>wV>BtujDg-(*X_GiGvFjIW6sE_h07I zd<=0C@){Hh$d^R_qnTA6sl+`=a(;MC#KRxVCmBWcjwZ`E$8K5&+lBTcS!T=*rYt{L~QcIz1|NkjVH{PWJZvEAJk|M zb9P^MaOO$RPq`Err7P0M!`YTU%I={$)vjrmj_;lxy6a`O``zeY+%DR%C|q`N_CnQe zskTa?=nXsm$+D0__tPvW!EZkO-jC}|Awpq(mQhsiPp;j@7d3tFBvUnBw4yKW@_bNg z-ya>{Cal{aFkV<8F_&LP#`l|VqFCrX9skrcH#|bMU~6gIxi|U8bUvn${bn?p_<;rq zDI)*?paouSa)4cLLx*3K0KiRbAP`BZFq4`3?N9uo`5kO6@E%&YB!#}$*9t9tlVhw@ zaBkD*E=;=~ANUf~|EhKJh_m$E_aGz0)#^Y+n(|efDnsu=>m;w*3gOUq`X1`ER7U?Zyk^thH5y= zVXfi$&a7SOQg$@^LFd}1Vh$MPweTwID)e5em*Zhpi0$0?;!m^2!-+2S6L1*XM$Uf& zGFGfSIL8$8As=0Fn66m7Bv*0q$Ud+SS5xaY?){@!-Ii;*4TQgQAx+0&W z3EjLJPEj4_fw5`7Qijah)Wpx7JylB>iWL&@?;a*jqR@m~N7HxJ`0Ygk+_>|*QInv` zac-(d3=M84m0rLvVs&OCRhM9zmzR@N71*7E|1>&Iy?R8gY^vh*$0Y_#emKRyYwE_O zC}`W7;8Z*7n4h;eL8ZQb-{D3`l~mXZ15xKYnou_{?j+R0mU?MK+f;%%Lu7N@M*P&z zWiifR5PBJ>5VgOc+rFIjrbvZpTvqw^jtx0*kgvs<5`^u)Q9qWC}>cC2E6Z4w-l6Kx(NYl86k(#8x5*J>GM7-!50Wp zN|DTVHCph6jX)^K%}WXe?ccJ4(%T(HskTj=qZQyZ%|F=!TO?w?B(kunoBOnU%;x$X zpa{p!)}Wxj1HwU%KM;Aa`1Z!siBN(`)3sv>ywyD7w{y!SAp`3q=#x^A6YG=YA=r9>Y5OoZ#swyAV!w z^!T$DEl^vxR^zcGd*{S3;&WYjqWuMoV;_lPs9*Ii*R36Ty2m=RvS|?IxUk_ z6%JB^#;JlLQJ7y$v8rFWBgy6ksK`N0#GpHDst52$xUyz`hAxi(qs0D&4BSvxodQ(W zl&1D+=(~%Eo!)`O8(ADLr*AaKNZ#WWNIp&#{sslrMMVlp`fjsis(xM8(z);bjV1F# zPng)6yr7b;Cb@My_c#s(+tx)7oH@Fsdihy3-dhtCnfB=)ut+l0wV)s!I2Rxd7;!kJ z9-lj#=fC~7wbqG6lv34#A)1d@f(!gJ&H)~)35$e-A20bd4h$(D!UyZH&yryCA zfYpxYUcC=ok42t*-xbjifY78lGwn$K^iO6IK@#pGb%eTjXIk2{u5MHdd_qO9tMYg6`+mXL12k|%{JJC}%gTzM;1qyw zuSf;BUovZd%+Zq6P4qt?w_*qLs#ny5U_!~(f+>c3pj|B{j9vgs+Op=g!BW_L*7Ptbs;Om`DgL{g<*Jk`JT@>%DT zq+T}28wKm&%`2=$lmFr9e`d`Rrv?JNC!;V;4WYH^NfcWn^}5el*op&au22ypH|?l% ze=wZ5@f7ivF=GF0Nsf)Xq}$0a8+)P|>*d+jM!CQ`DV`D6-t$|HTY)VxH=blGYBEW9 z86$=(^y4#gzEqM3I1fiwfI;fWtwCfaCL()uOY@<_tXhG&zZRmk9jp;-<1Y26KWPG0 z;IA)7)MNL1QNn0SnZvfePkBo}gB=!g_ysL{EUe+ zE4pT~vQJhRO|LSO4bqf`QyP`Eb%Yd+zX^~=%ft_SpLy{yyQy=}b5j$Xnzz>_p#b>Ec?A&K(t zCRh5k{Qj#S#g@5(ad=qw2`18#zKHM7sr0oKor#9x>@MN!CQZ&qLz4$77X9%B4{1{D z+1#>c(Bbyd4o6&I6h3q&+yS74BNBbPAPmF3V(3NHxg8lOob*s!Ah zNzK5V;)+NW!TgtVLyWE1&3CS=eJPUUT9xm_hqo0)(+>!A)Hsu`+pVHOSACJ96bMNK z4D#<0oM>b-DFPu^3?KbvNR-;ULxO*bZV$_V*zT(}BqcDKIF3zwo&5@PI5F&3j&xbd zj2xp^0S88OV8bA|`=S~*0+X)|!ol+RuANrU6>=DaIt=pn$pEHU9&=fVKND`I?M-1K z1YB?kXB5n@D!ASZOvLBTbeRo`y%X7vv_qUJ+-!21B=#U`bf(GDu@wcN z584@Xt&_I0QVsEwFDRepEwS*o8-X5I>^i&79!HZUd^e(0clh&Zqw#V*XOoS0NPIyR zPdjuxyrC!xw57mbRZF~$^d4Ga9g3%2C(Yy+CfFkh@*1fgxZ>Pw(6p>5Rk@89#JI=M z>Bxubn@uczLHdDeH3mJYUSe6(B)gV|-(F+0F7|SdfNxL78A5QoIiunBv0WfTNG=hy z+dHTWT66^*!C!G|Uw--*7<17Pk9swFC1qUiZxWO&qDV(J?gAU(U#bk*1VTL(XZ6Qw zW&^~jaaPYknmnI|u8)jVy_v2owE+5*{wXus*7lFhL!ZCKUx@onD88)l9kCWpr;a`z z7?pyjZLnhu6N~SOKh`9MoygMCJZqVeJ`byS@=W>CrQFvq`%X-Um8kaTGT#A<$92gb zF}YTk%kB@}P8<&??d&OSzs6tURvS;-2g{-qVFKqCDTXy0+=MS-o*u|;nh>2Cd-3%4 z4)~4i*GSV`GZEYrxK7h@K#>;VKRt05uYZZhHf3;IK8GmHq zJ9zTi0yn6^=o_gZr2C5-V*wl=bP3Xu8X1QF;QE%yn|^AMZabwXnkbGYD_ASgt9tvf zD6MSSi^Cw1l~gvz!u_d<&(iu(3<|R(Dw( z?%|nSwT#JK_B)m#)i~#k>Wp67ef#%hI*DAKZRyBL0lr*Xje||lowy<# z6Fs9e2-FOoni}jfhgBcf@A|Q5B}BqHSVM=m$m1~NaJi=zX0L382B;x-sUKIU@T-O|Uibizpc}^wCzI@yvgJYPQ zN`c# ze+LyHzLnJ&NVdZgU&@2!ghJ5Ttw=hhBY&l(fCXt&QbUY-P(2O=QZm@!_e(Ktqn!w= zM8{w&9r!%k-Tk*e)QPy){rDz!O12(q$kc;jRLV?VQMk)=YAm!!&M2Y7lviT-;3J86 zW&XDMQ{V{mY_XVS6M%&*);;t+<3ljPaU_7*)?}eM&U)`G_wFaWos* z{loLoFjL6grGcHNho`c|;OnCqZ-@#t`hj@)rfSbfp|{U#6D6A|5wny(b;2{}yrl$% z{}9C}0E9baZe_IZfAi&z{4L$L>So%eRNc?f&5yvJ7|HDwUQ)$(a$WMI1FW~!bsk(2 z;jLhy#AN?`J7Irw-)H;poM!+4+_;+j{#`-APk~q>9ww|gkKX_F92U=GL{x{pj*ukc U!zv?q{$F`F1QQVzHXLF99~o5u0+r-gEAqe`d~{^FH%DZ<#ai{51Y)TsK5U6CfWS>rYNX63UE?0CHh} zkx?OMy_X=Yh`aVo+6cGOC|G2wkb`_F@iVuxQQ%7xC{KbY>Q?jkz(PuaI#qK}u7}HlopeQ+EjL(TMDKJ_5#4J&Tlik%@FOdE{EvV*gY9s3*9du^KQf;*s$<|R^1u-W4( zDX2f8dGrZ?y~eTB_8fut4O?T=6pkW0;8ltMB7LOgBcp&F&|-iPzGL!Fd&?sR9F zTS%7uLB$$6ra^wF{yhyLE0S@O+#!jJSA2H8f;Oho$zj&ofuE8Rx-@bVsS#>oeU3

L4$j?Ndd&1rr~Y5a%%$;#HCXMjz z`*`lCqeZUQKNRzhP58y^h3g9VOj$^?@^T*KY4&OL3%5UE;%Fz?+{XY3w}ruNZQ%|e zzAK@yRVE-SFKHA`Q@{icBu}h)s|g44CVEp{G|*~OQT4Qu-G`?J6bGnJwj`LF%)j0J zV*)Qs8>iADj?OU`>@*^N{wk;*{$&1RtAdQ~V;a4CRNS*z^`cEi;KM7_3QYkASG9wn z*YvMqykxGthIkD@D#=drHa;#RS5|~jNrA6mzq0=>tZehQTf=$vGPpZ0bra_~9L{{m zPq;P^>X8qSnvs|}__yTMx~#&cJ?R3bW&z=J{5N2^l;u!3cl8LjsF9B~68-0SWy)_A z{V)COB}aWx(EYd$=e^(S=I12kp;E7aknFc3HB z^4zg4z+||~4!SkkQ<$&yjihh?j0>~#kvXJ)$GIDIQ`TwC^YviOz8Novz+<{I?K*)e zfDKq#(TsUOYX=Xjv_!l!wWYOw_^83E@PVLP^E;b#0U^V=zS`Lwv-)?YA|ZEfgDy!; z49Lh?Nk|w;fFvX&cS*KYewF{AAt3=_5kOg*BJa^5=dgai8cS>oP!U=2LP0p^=;LMZ zZIvop+jp}8o;vw(*Na_nK1F=fPYu}F(fVDgg^%kg!K~z{G|`;y_$JfzK_Yi@t0E14 zu9a^%#wyf_QD@vc{gRW8dryU??ZSB!{&^Jdu85!8t7`7A}bmJVbTv9t*oFj`CS9=lQ)!9v)fz@-v-4u zxeT4$-W4mBs=o;$o@qS8dFuEuiG4q8nGkF;s-rPKhFIi!upoCP%m*5qG7VjU2C z{br!@N5#34FgRn}LXz9Grpiy3g`}A&h^lmZ)sewuesXoax6*qb+0T#$LQvQj6}N9L zsHsF+E`JH`8mG=93r%6(5f_s@2K`RX!MU6H#W-iwV`fylmz?{@NUgY9`v7N zr?PLNGA-298Ax#sHphI9lagHVfFH^-aH{^sjc+Dq8QwotvTk zw;y_p>8D7a8+9w3=Rv!O5%<7LvF(D)fNdKg_Z|ItCe-4FY8vpR8Y8MTT8$Z{bo5k+ zX<vntfdE$$=M>yaV79EP>H6^ zCH7VgdZP7<2g=mPz1o~UkiKkKW3#L27=#qTw0<@=A^pOHD#1X(fm%W}qeLS>AQ*PO ze`(1$-{Wf>Tj(8+CagROHV7{)wGZN(%=CGizrA{2-LQ;^SYv$V}ceSLx8-KT=<6Fc}R_&-DZ z!#FwRq=9D|#gQutV@lEKEI0sPt`2FRd^!}jr}dJ^P4^^SB-Mmn#Lb7^yDpZ!5RNlc zX1He=AKO=kDYWwV?Yr((8TkY7qq8a}YMLQBBveDk&o`iQ(;Er7c`5rc^Py$*$iUGD zi%3Y|tb2L>M8(p`Tnd|-Yl{?0#DmAiQSapxS(X-=B{Q7^3X~(E2uc*uBe7%$yGgzw z_|izLZD{50kMYL_y5;1{eXWOv2P~vZIZBjc~m_EtP4N#SBia3P@@!m-> zeD@n(<$L#2gNShMYd;=6QIAwZ96%05*{emDUXJbsQ%4R)lkW_5P{}!?jj`190!R<* z=6*r!Pb{D0w}{3oi)kEOB=@z(UmGw;fJFi7LP#)7C+U@fj?VLh3p{8(DPeK;(8FKb z8kJSYCio6}T^5f%G`1eP;!>_mc%$k%6F334QFhu#Ts(=JUCom0(@I{}lQWHRVx~*g zlUEm=UZRq-(RNBFS9j5fMjo#%lI1pZixn`+IbZqpd%m98a{lz`T@pef$Y3b9xB!*j7F%rjG5&3=B z`z5S$w#smp_V)IGc)%7@HpjWY%1*N5&%WTI;~oLViPNTeEimks=udm*0rXv#Qp zKle0vc%nH?ccOq)G$=`BjSTBXR$q#ZcZ0dF3WA&(Vsqo&BlR3#v!{uYny>tLTtI@= z#_);IDVB{c5ls8B?F@X#EqrE>&BmHK3d3lV7kWjph7YG9Zg3KsLs z8Xy_QT0bvp%0r#0(G24xnOe)x>u8vo6u`u4<-pY8FzPDRk@t8g9F7GauZuf)^ZT+s z^df3VA>WlQ=2}PGm}vg-sb(wdnSZ|8%nHmyKKbkC4NdtP_$ecu?^X2};L~7o$tz|Y z$_Pu+K4;VDX38>##;eTv)*4$!Plr7aVd(%_Jn}jUxi(nX5NM^B<~&&+I=fRGIQR68 zvhm}lehn4`Qn(p@#+WQQr=}I-mgRdUOIM^PAL)`d8!tj4kN&0x`ldFzgf#-@EnbnV zOLs1REK(_X`|MoRN$o+)NZkChvy#eb{Ek%!0Y9w4QXt&i_!n*GU$naaMVtE1O_@XsoYtrtOmBX2)<4^ zE+?%DF;!%GbxE_Cr|KqO_VOuu0WwHbD;P5eJ)US92| zz96m6MR1OxV=)K5%#-bnyKOteK5WOn)~?_LDPVj?>w4jLTob?RbcFllq{0o_E3AA3 zotc&d4@p`1_`b^_*V#w#@k~a)4LPNA?w(A#oO3wZhd$v;?sUr!w?BH>-tK>MgYSOl z<-lqYJ7KR7y|{{=rw#j~JrZ$*er3-UuVa1ME?@0EZEze#x3%st&r4a@Hs-5>PS%r^ zgVHK&84V>^)E2`cj6VL}e{gzd$1?}~{qVBPbaaQx^!P!jjV!Qp_f%vzzf@`EbH4c?IWClW?Atne}91J>3*P;0xT{nYUA&O<0W)G9wL7B|rwq{OB)-56fq|M<<4- z{O>SaH&-IbXyC>u=|c_o~;uqX^b zdP*;zqagTF%*)f6(WAU}G`_!!Y5Cf{4PwlD{PG4@TV#MAHgc_n*ei$7TeS1=ZGsPCq?qadl~T`0{k3npLmBeUvEBXhKm@_Z>pZnU+X#fb@xAw z3JIZu%L+&#G^)!Wg5?+DM|yP1GHMeqXo}mVB}Eay*{Eix3H4VJ(sR`A=?T^h;3mwU zcMSzf)J_wt>+{M1t|o83R*L?n_)$8>CS-+S9=&SWTG&GEWyZrYPChg`B#)WAqF1s0 zqA1dm!A(^=Z0V~JZ7c*0>wg;eOwcm1{>L4YLD#WY#jQ7u>KP4Mx{+CXKFt&=0|z!$ z08EqB}qbEaz`RL%1#Om+l&O113rrGwwKaw6=Bq&{lzUhFf;C_<$2W-Pn9IPmfMQ z2&8jHV*FD2b@vYcu><2mvqsk3?Mf)DuvFHJkoRYsx^{ Date: Mon, 31 Mar 2025 19:44:20 +0000 Subject: [PATCH 399/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10223 -> 10223 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index d38936134..b46bb1c60 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 19:42:44,947 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 19:42:44,947 submission_checker1.py:3260 INFO] --- -[2025-03-31 19:42:44,947 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 19:44:15,714 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 19:44:15,714 submission_checker1.py:3260 INFO] --- +[2025-03-31 19:44:15,714 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 19:42:44,947 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 19:44:15,714 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 19:42:44,947 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 19:44:15,714 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 19:42:44,947 submission_checker1.py:3276 INFO] --- -[2025-03-31 19:42:44,947 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 19:42:44,947 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 19:42:44,947 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 19:42:44,947 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 19:42:44,947 submission_checker1.py:3297 INFO] --- -[2025-03-31 19:42:44,948 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 19:42:45,932 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 19:44:15,714 submission_checker1.py:3276 INFO] --- +[2025-03-31 19:44:15,714 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 19:44:15,714 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 19:44:15,714 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 19:44:15,714 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 19:44:15,714 submission_checker1.py:3297 INFO] --- +[2025-03-31 19:44:15,714 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 19:44:16,711 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index d90f2b1d5..327f63bda 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1026 +1027 diff --git a/summary.xlsx b/summary.xlsx index bcd907acd5501f76747870ac0346e272eb6d8a40..0df770a0118632019231cddc8f86126e6a438d9f 100644 GIT binary patch delta 325 zcmV-L0lNP0Pw!8#8X^Q#&9S(X9wHxq&2GXl5QOi2ipclGF`=lnk|4Dery^A+_`tMa zjnPn{19&vA?@DedHxAyMvEYIPpG1)Ml-mmJLa-EL&v&4h&!R&|abuifA(n1`i!I~d zYzugj=QFGzFu?-D9Wb>c5gn0&k4XFDnjRoHh6WVq;4#SvK6?W=x5Pn%90dCpqS=DH5K|;FXIOkIZqX* ztyWN>130v(?@DF}(-z*WHekH~7X^@VgqaHMyf*}6$95p8OM*iMajUJQK9sh9!G^ML z+5pbe^d3v_lv7V}4~)%3L`NiN6Vm+Hb^vgOp#=%bJDepMK6^b_>25X*%7s%3e=u;< zdJ-r1-igWBbzRaGgEi#O@MiV;KCBuG)t`z1s!PrYvp~JJb%`&FaJZH_zX~Vb3gAzN zpx<~qE;9B3k3s-4#yu&Y#q-N1dWq^h&F67i#6`Bw9?2{xv-wv)@~Zy~l*b|eCY}Ie X?(0ub{{gdMBMSrs@E4S>lW-@0$e)-7 From 7a094ccfcc5526140c57b7cee5b1c485c25ab3f2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 31 Mar 2025 19:50:08 +0000 Subject: [PATCH 400/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10223 -> 10226 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index b46bb1c60..aa471849c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 19:44:15,714 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 19:44:15,714 submission_checker1.py:3260 INFO] --- -[2025-03-31 19:44:15,714 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-03-31 19:50:03,563 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-03-31 19:50:03,563 submission_checker1.py:3260 INFO] --- +[2025-03-31 19:50:03,563 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 19:44:15,714 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-03-31 19:50:03,563 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 19:44:15,714 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-03-31 19:50:03,563 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 19:44:15,714 submission_checker1.py:3276 INFO] --- -[2025-03-31 19:44:15,714 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 19:44:15,714 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 19:44:15,714 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 19:44:15,714 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 19:44:15,714 submission_checker1.py:3297 INFO] --- -[2025-03-31 19:44:15,714 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 19:44:16,711 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-03-31 19:50:03,563 submission_checker1.py:3276 INFO] --- +[2025-03-31 19:50:03,563 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-03-31 19:50:03,563 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-03-31 19:50:03,563 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-03-31 19:50:03,563 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-03-31 19:50:03,563 submission_checker1.py:3297 INFO] --- +[2025-03-31 19:50:03,563 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-03-31 19:50:04,600 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,14 +26,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74365 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422216 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24279 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26872 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.54571 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29979 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 13.9468 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.3829 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96958 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 327f63bda..1f3d8a7a1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1027 +1028 diff --git a/summary.csv b/summary.csv index d3a35a331..bb55f1abc 100644 --- a/summary.csv +++ b/summary.csv @@ -4,14 +4,14 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74365,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422216,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24279,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26872,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.54571,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29979,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,13.9468,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.3829,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96958,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7938ed151..6778118da 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.24279, + "Result":6.26872, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.54571, + "Result":7.29979, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.3829, - "Accuracy":"acc: 75.600", + "Result":9.96958, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 0df770a0118632019231cddc8f86126e6a438d9f..bcfb146bc6fa1c5a2ff5801e18c9b6e70cfbe225 100644 GIT binary patch delta 4329 zcmVxP#DfMD?1ONbp5|e%i8I$k>2Y=gc;yM(4KVK0=eQ2eDT!0V| zFq+bP>5Vd{I!`Dh4n&j08QbaQ@5fG{Q?_@RYF;XxKG4KjYv1B!?|AU_IZKU4!gxaS zq3JYj(;#_Bqaduh=%2w8Wk2|e5Mf9IIUkxUA&S0b z@o+`5fPXh>L2~&drYsADyk$#^7mNf^MaojkwQbMJf+RO<3w`zx8#IoSkQ~vl%t$V3 zBaEbhkm2)HQgCxLh!XisaNhnAp3t$>5 zvjwF;)WdT*WGkwraZjqKZkZ90I4DywrB7!hS%0pCG}2L&kfyx)HL^sO)-{bRcpjQ9 z=^=^4N*Oj*NfeR1avPRh(Cndpg-6-a_zs9tM|44ye?-i7A-1Ay<59LX zz6+w%5k83WkBFI$=;2Xz&?vuk^jd;feuzLoy~hXyBzuHF!0bInAYgKxAP_LSP7#Q= z4LL&~;O(Cy5K#IH1k&4vTnydz!@ zpKzXT0M!0ahqOS|#s~!DdxSthzQ+gzfZz-&6*K#+y#455Q*biRXrv3a&x&oAM? zM0d~zwYq|wYlH*l3W5ML?)G=s@#yTo+vDlv{Q7dgY7@BZ6PAfsm5XLbGb>BwZ+|9x z0C`mq7SEGBCQNoRWG)C7b6zg8gsY~2&mTKY+Y*eBs{OE3mtkE^#%n$!T<+%FTCV0n zsM?B#>>_GYjj$*i#fzXUT+JWdrq{F^>)n4I(mYO<<`**}B1lqh9t^A)+1&0vBPZF< z5gA(b@13}&bzDxpAvarH-+r^|ZGVZbi;J$Siw;b~LmkqBt2_dMqC+51bO;2B4uL?? zb$5skp+nK_pkH*M=-|Lacc6(5a;|j_$odAc3d(!i{U3_%GjjTX=U>;vzOFjZAROwD7FcM<2n4DQfk4$E5U4r?0(Rz8e~0Q2I>`2X2mQi_s(*t66Wu`@ zc1n|F3N`dkX9z1eeZn%w*N+epOM}N1UU2G2(Z`edA*Kr zZ~h!0MC;90bw=2dj7geuBc!r)wT;)&`s88i^k?-~`ER)Ttk0uFt#LtGpCO1OUmE;p zeOTf3#}BTpjD_*EOo<_03V-==Po4|;wp7-p5k1HCxr9-$gtA1u)ZW!ONtp(tj7Ixa zP-UL{EXn>}xaalvLhs;<;T{-6L)KsemDyEZs)GY0tkIwdmgFj6%OvMUN@AI)-IS{q zt4}(rYe9?ZO1A9_Dg@0ocPkP^gsD6Eof}gsHaDsyuLnq@UrGd000O8002J# z57y9R>63W~T7NSS8QfZXcW;Mcq9#JCNjcpT0SYs$20MlkDz{qbygSi8D> zx98Q2IO)#%-AOnL4r||oxc2MOe-6Hd(I4?B2ofVrFpCdsqa>N{*Xwa_6imCZ8P0=Q zdSnnr({7Uf7!B+3JPNw~Wzcj|*Oukfr`_?aw#u*{m4C?*4hG|1@Fwgnrok*(C5eJb zH%W&dkH+)3c687mr^m8-j3^i!){fQw$#2@J)sGIAeP54*Z}HY=BP-==82*v{@Ueeb zvodKNwC|R~IE##aFz7BO$$9wgLognWlJvCMSx7IO#LIt;={P-?x@JtfcZan`&G#Fm^|~D4 zn{LuQI*7t=Mzri9o_DivM(t~zo|azL`Xy`5##Uc9oZmMXX&u?wyFF?g)NixIYsG~O zxs(l8vf+BGp`MOsGbqgmrC|)#?ikKd(X_=diGRe|m0S{uyDPaO5^q;>O(gzCvQut@ zm)n2@N|cKVG0uxe6_<}Ht{zof7b$isZSqQ+5U)k0sPILlk18%7Ra~J$eyK${b}F@L zrN=gYr{WrZO(f!bKOquv!rl;x0QN1BG;Ga?aZ_Y09Y{DO1%1?NBlTV8M|apeVvl7^`F z4Uvd?-x7(a_Z>=naqi#WOUN(852OcijeIN!Z`~iepP)`WbTCHFIG<6X zEP=?`!@vheAAUc3d*1%oIeocY48`ecXn#0PMvJdzFPzq=ll0$dy3$EM&4XxgGoB5C zC|$1wH%SmDH}T@@bR1_(pZMn1Hm!OR1;KKWRnOK^^-t}Sb$k@W=@KTcuh%kro6I-8 z`FyW8U+jgm*nZ>_rNg|oqUSo@{c50dV964zcG)CXj`{!&Fr>7q4Ahkv&G zU(UELR^w8uC5GA5TB{uDTuz;f)QM~DHA-A@rJtY#s3Q_d9g#@th(uhW@7)6GNVxdr z{jngth2g#kKiv)&XQ_?hP4RtxawU=^Ms)cZCCU<-Gq*w{hohTrFT0=FOK)FRsrH5` zkn`_J81+Usx7u_}Yp=IZx>*9nDt}5Tf#Q|Bg<``qeb2W!i5@4>LlVUW^cp3u!0ZGi zKq8Sy5{X2TNF>ijAgrn>mxB&r$RdMFG)kl(+(-6O;goL?Tfn5{V*8e9_T+zW_xNE`M)&3c_2y z0OeFRv`GhYDJo++7^5t?79|>ak9y`;i`%r@3*)^>cbVbNbAI-^w2JadAb9R?A?TY< z!*96$=0DD&vN(*(c0r?nX^}tqux~BGhHxEnx-2GVLb9UR7Sfa*tB$niPJzz;RfF;oYmZ*W5a-*SGk_3Qd zL4uZ&I}4}K(k7iKOFsI8unrBAQzjF^oViuObL4Vs!py+bmZ_dVW@8mKmYF%%15@j# z&!*=(Irou@Z$2F>+|RKcNhWGsSCWYuB$H@>OrilYQG;aaA~Hz=AhRGrYjf~{Oxe;V zoe(l*Scincb5)jFZk3#E74Yu`vx*waJZfDJY;#*_H*81ckh3lhNdkY`@wR;8yMHxj zaV|BwMjUDNAsSCweTW9QWE#mO(?~9vMslf(tUfG((RkACLo|qH3A+!| z0GdPtXcpOhSOP$^$nJlm!R`YsZPE!r6YM@D3|6u%HLPTH#crR!hs9O?HV8c+Ithz8Lt;rC%0K$BooV#CjmviYRl%SwtYHEV7=essgMd@E<8L^pqO&_wHqBMm=9<4MC0 z(IA>73_nZ*Xc7&eS!DQO2>{I^!;c2T545yNCj?C}{E#r5HJRTBX3dKIK7S9H&(!d9 zp|=e`%URQIf{$;|E&W zq!aKJKvaiC0hxbg3_rSJ!_Q-6{;$S@O6n^Gr8N6kcHZB5X*&l>G>$mZ>_arZH2V+@ zLRrG>!!!UT8ISx+XkBFXVVwY!MP?rjW*=y2lTMV?lvIM-hlT-`%o z^z2Ad*>qfXL@$^~opFe-e?;)##%gL8NZTd&>IO zyAP*9Gz?CH$s{&w|p@7e1s7nAPM zH0-}z4r(zQ|G5ZWzUaMh_FwdzSHBqgm2qgKrQb`_>XyA^AK0*t4(8oq@TnUO$Fta& z1cS6t%XDi-w0a-q@^cc-m!Hz#XsoKTR z)y~&)I}a*MEJKmhANKo(s!(vGG|1Wh>+sG<)unDrSN&G9-=CropkM_p87wzg_vF_| z(JKBQ+n%hNj+i?BTzPLId|&3hlM#13@SiHNlklh#4}Ueim+ZGsjTRH%q)+;4Vc6Hn z2O}wemwjgCeVFY7aew^$N`H;+_@#an#^c8H_{Dw{#^Z|o#B%Ro<55k7JM#0$k!*x2_hKV|+rH-EZ+R*iyy^E&G#1Z2RLh1K|zoz5q8M=UTag%`NxA(|0eM#V6 za>?4Cy9e5Jj{|#AQ+=;3!Lpa`+nc^ z9I^L&I({}2F{*mRc0HT8rrr1CiDWkfI$cBHxj3=cF*iO56^GXMm~(0~FRJkGNmpS%H_@>d%M<;vMM_i{3}ZJV{lV2$~6{JDC08&*xF?oY)5)urH+J7B@My2R&2JY2(kSj7`>B#6f@ z>Q~+li;R6Bk{E!FagWMJ@$|f2B%*poX7iL3X_2q<2RbK|+hF9C^@j3y!* zKb*QFcmx0dC=~zz6aWAK0000000000fPoW}$tFPoy^{+kIsq?}Mkgi(@6jf@C6jR{ X6#)>FhbKM(WRu1xBL)^F00000?7}Df delta 4392 zcmZ8lXHXMNyA3^vw9o_$fh3eyTByCh6;l@knDjel*0u%fkJ1!OSqX%qyiwFN9S ztPIQgoS$-P#B%Gq_a7y0e{vpY_w9vdR4H#P9gnBFvX~S5(&zJW!;Vv49`Br2t%&fp z9*5Ids}e~<-bK@NwSDwSPvBcdIU6RJ%SM~LBT-mP<$cS`2F{TB49c@|kr?qPeKDI+|1=VwleKMUF z{wz)VQAj?V9*!nKP^QeFYYu*_yuSO9VREd#$n&vx0d1m0vA&Sa@Md@=3d9k6LeFXY zPZcUQrhlxlFp)$MhJ`&CQHA$cCywwyefPx6>#b&Nj(bg5?Th2;Hl%8gyJelEpUl1T z|8d$}XggTLQso)m69Vk+KtEu$+Y4a|@zWb>!`Y=Mi1}4T7&uM&S=BNlY}rcFb=HPP z{PZKHqsUa>wX#;{*WFJ;3G9ze%6yw+dnSdUcsfGc;@oMj;^P`s!6WUH430er=J7;A zkQZj10ZdPM`vutSjMkdi)xj%aJgB$nJ5h=<{cJ4jb=ejgKl|fpfW>FTA;Grsw7~Qh zZXmH{q6;n5~*47QlxyFsR|2 zBKfc%A74L1eI!dTBKc>HWY9*xUps=2c;#t(nVoWj8DScyfc{X&mm|rbeEP zEP=B2JOU4XhE)vp7;Ns85>b9hq}mRC?G-AY`-2Af7)%&as>p8jT=v+znCe5Alc3j# zU|_Kn>R-*Qi-;xfOOf%x>!J+4F<#`9H9DHCV;uXa9qd-S&SV*_&RT&#MaJ+%1VdNe zgdX2(H=BNrcl(|3XC=BTaly43XoXFxBQC^e-`^j2-`;$|ctc_YIrc?P_A}-W_J!nL z^e@Y0z;3xBd_A1)_+p+LsNX&`?bY!+(ns}r=evu=ZgIM3!(uS`<@u|)!%A(HQqeol z`DUxaO5HE>pm_h4?EBv+TEYY){H%F1?iAK* z;-4=4D7jQzMZ)`occxtM9Sz^yJSRL#wPgSMwDUmvxA|gpGnh3NMX0MmL`)9=04RWe z8acqBkDQXtS2l@v)_H?_E6w)LuRf>-MN)J00jC5(qH65Ej>T#-RL;Af4JVP%`D z6j<8x6@zIvVuQRvL!lkBXY7?%ekVC$uGS|iGPgoIRp|y+J7#&*Ht?r@^Q020 zR;g^v;iI22^W~H~{AA)oT~N%TSra%r$x{b!nX(GuT@^c&dw5uTJySEe|62!yhM^k9 zYD{Zvr6=!DrVDjI+XM^1$jl|jDP)C_wj;DEHc1zQ1%Vy1|Grj5;EDPZJZF2Y=w~#W+U6eR$+*Bc(lDP{};LzhGUVC-ONK z-^Zij6ocx%3QEY@${024RFk-J_EfE)D_2Ot#u=o}BJqWerg62{gEc(dxM#wsML^}e zFw-NR3iC6Qmft^Wb741A4{usjRFH-X>dU~^Oioj7o>3~Bs(Af!iDw}@o#Q(+b>mPJ zu@F? zF3uhTTE{5F9Ixtit>?WdQ(>4^dXuG|**wP8`9klXdjWded2ha|Fc=jlSUT#(WvM%x zZr8*vmDERA(nX&y>O>*mSKmiMg9A0-gHQUTp|mnB+@SVxGAOOlhzgXp=(7@h6)&w6 zjbeOWrv+c#4T6H)yrhxPq5bDj+UI96s+}`e3UI2{<-0-ck_m08On24IecM0ebNmQY zgku)#kkB82ks! z-TfEhXZvX6((qz-ot&4pr2vznTO$h#DlTbswQPo>=yV%?<)2aMF{X3O&N3hKJd9m^ z`bE1IsIyP2`CJOzGc(5hxdEqDmW_N^ZEib>#XY`ygtrAlNZcC3qjz?21B+b8TCTE7 zJA+LAD^6Za{Lj=H(`U`fT>Ms?5NuOHC8uHe(%GFjBeGM1ca^P=_4FI&=MF(4V)Ood(vVi4<=`7ZI)vU zuS3i;trX3ztn%T6n*aGctW$|ZcSZ@wc zqy|k?1w|v#TMP-RUpS*lR`_p|fm#SbLU&b9;L&hp&EgzA4BrRIQ5F6M3DrYJ3rhJNFy*R#Sx0Hl>5Fv zLVR0ZK*>&%%qE$08iRx#=nemNbj$Sew{E__Eg&-QJ2Y&OW~gUDPCR_YPaHVza85bB z^m~Qx-rJ6PCnnKbsupyyygZT|EX#=w@B~d*G@QlqyKn0#g3E-E+)giaYGavH!`>lD zJDGFyA#f)FaWQcyqJiRvP-QtY{FGS^NM{g1;2$D%1bca?TwGGN3+UFU*G$x2bDlSw zQ#+9q+stbD@2K?)r2cNe*L>OBt>v#l#_p){^&R`Qq3r=07y>>$B9V1v#Tqy{;ABLmIMgi}TM79{xz6m8;G_m}%hkuVl^zP#ybPn78 zYAW)u%~C$x{5{FJd&^4{<o~eJ2{(zHQWwxqQ=Je2+y?sS%}qbSWCPRZlxZ`V891nvkB*vtZXW{vIeDM3m1CLM#Nu@ zAdjP9U^(QPS`YOodj{uSIZYya5M{2(_j7AHTdPOOi5SbW7dgSIuto%}&AIU7k^iRaGqY==s>LqA1Xg97{b8NVb(ZLMiM( zvD7=nxqQNSdjwuyBhv#@UYHM>mleHTZR-Uw?l*Kg^S%AmHi5Pz`$V-CJ*!?}Ro5cB zorOKvzH3wNU=9l=BT@DVy&ywKApx|{C%6|{_6InM@8hL?_2pk+^mTVK zGIa8fl!*gBh>)_}WjeBn*GW+UmCBGkAkVxR;i#`+~lldEQwVtgF;X8u#9 z1<<$hSC!F$c0fYW)objvg#V19_eZ~R8{urq*z@5@X?WHy7;TtZj=C>lsYwXCkfo-2 z(Y_#a74h-u3*{S^YCr#cq4;iVQSHxFe!~`)4e1{7h1NIg?z(Sh&WDwL9w{BX#@=8y zn=iXYs$vvj{8ttkhIPA~__hd755xggn9hQ|M0Qs<{7(LB(?T;5%p{&CEhp*a=ikZK z+ISW&HHnrR2YL_uL%QUv9HWZTW@i=t8`NJ_R#^l@)Llx0<|d$B!EH;#ao$nBm{8hlt* zbx6QoO|N7nmTD}dR$f-vw#q4S$hY4z6*hLM-tVhQ#2EaRt9@W11{U)C-Eg#sJpVl4 z&kfu6S(Y<+%Dtk;$#(7NCphDp!@|$?SuN*e@=JA961B4awE%vgACH(LTSc8;q+GKs z9BN$g#Pq~&AHMzDGh9TkFSd8*WdOauZzjTKXf9m29aBAHH1L#ko|+oq2dB6XJ7WG! zS}D=6Zszc@12U9W&$x&O!R9I#!Jz9wqTMCBt>rr!C+AK5X@`yM7pCyi!YtHIaF`Mz zMrrv`rMQpkbypgH>5e99_!)Qak>^~RthZ(N0}S2VTqYd6;C9#BbspBZ_wFfMbGo^p zAonz(TYmia(RGhn^B9+n)7x87edEUgb*{gzK5t4sJ-G1r?Zox;@oChapSYV=(?K6y zp9_%wH4k;j=4z)!&-^+3vD2aRwt9B+r2nrL=9t43)c^WZY$vlR5cPPlrk;xc0MI1_ z0BHaCRa;jZU3b@)uLNye-R%T?oSm}JgDzhXG~owAmD;(sxoI`_;Lehs(}5uACU?oDGOi1f&4N(|s7V}XWFA;j$8c%PKSq{(-uB5V zc2xHRk&kHAfx62-sQXll?`xJriF1$%zmn%}?I~2#+YEc>rVnEjRoraeKjlbudCR}H z2``TRulLgDk(pK&DYK@&~N;2jh0mj{QuJEWL z#R=DDFcb5oJX{C>Ud>J(UcIox$)Cb==8Jq28Euu=wmsDQcFS0OCsCxOYY8YionVf$ zQ|{s$0M-^3PWXQc5Qm%(SpT*Dw*Uav#{j^cKjQu;f Date: Tue, 1 Apr 2025 05:32:41 +0000 Subject: [PATCH 401/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10226 -> 10225 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index aa471849c..046e4c393 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-03-31 19:50:03,563 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-03-31 19:50:03,563 submission_checker1.py:3260 INFO] --- -[2025-03-31 19:50:03,563 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 05:32:36,282 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 05:32:36,282 submission_checker1.py:3260 INFO] --- +[2025-04-01 05:32:36,282 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-03-31 19:50:03,563 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 05:32:36,282 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-03-31 19:50:03,563 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 05:32:36,282 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-03-31 19:50:03,563 submission_checker1.py:3276 INFO] --- -[2025-03-31 19:50:03,563 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-03-31 19:50:03,563 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-03-31 19:50:03,563 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-03-31 19:50:03,563 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-03-31 19:50:03,563 submission_checker1.py:3297 INFO] --- -[2025-03-31 19:50:03,563 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-03-31 19:50:04,600 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 05:32:36,282 submission_checker1.py:3276 INFO] --- +[2025-04-01 05:32:36,282 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 05:32:36,282 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 05:32:36,282 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 05:32:36,282 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 05:32:36,282 submission_checker1.py:3297 INFO] --- +[2025-04-01 05:32:36,282 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 05:32:37,452 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74365 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422216 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26872 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.378015 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29979 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1f3d8a7a1..5208b5913 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1028 +1029 diff --git a/summary.csv b/summary.csv index bb55f1abc..58c785350 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74365,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422216,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26872,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.378015,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29979,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6778118da..c8a6bd9be 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"llama2-70b-99", "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.378015, + "Result":0.412378, "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index bcfb146bc6fa1c5a2ff5801e18c9b6e70cfbe225..0b5c26dbd867af336391d21c29a1fac30e0c4b71 100644 GIT binary patch delta 848 zcmV-W1F!t@Pw`K%%>xBN*Qn8Sv(f_?1Ap6&+v@tb=(@P*y1MAVG(6NHEx5`f5GXnX z0!4>Fpy&_?6kT_R=ny&--46Oi7m5xJOmqjD=pg4>=YXtl5UZfPx847t=sqK-4~Xu* zyCpi?Y5QL9ovqHtRp;xf0}aBV4rzgfc8oxv>JSK29Rh)>Lm*&hKJ|B~4xxi=&wqE& zFMOyvI55#2v|*0eL#RS z|BV29&7Rlm`1a<{0YbFid{t+JEy$rNPEg{jdmeeim~TAP|Ro1 zp`*AlPO%V6x5bw6Z?*+o5HiOK0uwASJOEQW5z!GT_=L1St{nk_V`xEv4j$)Oj?dlz zPWhV+qjKR?GV~U1T2JES-g`M2yROT+adJOUCGlP?t( zlVm0klgk+p3jhEB000000DytKlMW|30WOnCCng&lh*||D0RR9{0ssIJ0000000000 a0001hfe(|4Cq4mVlgTF|1{Ea$0000Ks(I)D delta 817 zcmV-11J3;MPx4Q&%>xP#DfMD?1ONbp5|e=l8ME*M`T~DjUEhAQ>TQXxi;J$Siw;b~ zLmkqBt2_dMqC+51bO;2B4uL??b$5skp+nK_pkH*M=-|Lacc6(5a;|j_$odAc3d(!i z{U3_%GjjTX=U>;vzOFjZAROwD7FcM<2n4DQfk4$E5U4r?0(Rz8 ze~0Q2I>>+ad|F3N`dkX9z1eeZn% zw*N+epOM}N1UU2G2(Z`edA*KrZ~h!0MC;90bw=2dj7geuBc!r)wT;)&`s88i^k?-~ z`ER)Ttk0uFt#LtGpCO1OUmE;peOTf3#}BTpjD>&kv`mR1UJChfPo4|;wp7-p5k1HC zxr9-$gtA1u)ZW!ONtp(tj7IxaP-UL{EXn>}xaalvLhs;<;T{-6L)KsemDyEZs)GY0 ztkIwdmgFj6%OvMUN@AI)-IS{qt4}(rYe9?ZO1A9_Dg@0ocPkP^gsD6Eof}gsHaDsy zuLlE2qhGUW2(%dl@6jf@lOZA?e@??N5QOhMMdW*8r=bX}B$WX5L?tAOK-?^Q+gijw zvJGk89w%wjR_cl4dS<_!UAbKBRg?UJ^U~-F^GXMm~(0~FRJkGNmpS%H_@>d%M<;vMM z_i{3}ZJV{lV2$~6{JDC08&*xF?oY)5)urH+J7B@My2R&2JY2(kSj7`>B#6f@>Q~+l zi;R6Bk{E!FagWMJ@$|f2B%*poX7iL3X_2q<2RbK|+la3}l0uvRJE)^D&;wCBqy^{?mIsq?}NGB#6 v@6jf@B>?~cPyzq|5dZ)H0000000000fPoN`i6=e*WRu7zBL)^F00000!U=v5 diff --git a/summary_results.json b/summary_results.json index 46de40b5f..fe6d5882e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -32,7 +32,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99", "ColorKey": "availableMLCommons", - "Performance_Result": 0.378015, + "Performance_Result": 0.412378, "Performance_Units": "Tokens/s" }, { From b4da9c4a109e237c7717abd43373d5efa9723b9d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 1 Apr 2025 05:38:48 +0000 Subject: [PATCH 402/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 8 ++++---- summary.xlsx | Bin 10225 -> 10222 bytes summary_results.json | 8 ++++---- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 046e4c393..2726f9077 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 05:32:36,282 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 05:32:36,282 submission_checker1.py:3260 INFO] --- -[2025-04-01 05:32:36,282 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 05:38:42,727 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 05:38:42,727 submission_checker1.py:3260 INFO] --- +[2025-04-01 05:38:42,727 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 05:32:36,282 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 05:38:42,727 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 05:32:36,282 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 05:38:42,727 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 05:32:36,282 submission_checker1.py:3276 INFO] --- -[2025-04-01 05:32:36,282 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 05:32:36,282 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 05:32:36,282 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 05:32:36,282 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 05:32:36,282 submission_checker1.py:3297 INFO] --- -[2025-04-01 05:32:36,282 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 05:32:37,452 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 05:38:42,727 submission_checker1.py:3276 INFO] --- +[2025-04-01 05:38:42,727 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 05:38:42,728 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 05:38:42,728 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 05:38:42,728 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 05:38:42,728 submission_checker1.py:3297 INFO] --- +[2025-04-01 05:38:42,728 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 05:38:43,693 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,8 +31,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29979 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 13.9468 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.411773 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.3043 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.39691 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96958 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5208b5913..e445c3bb2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1029 +1030 diff --git a/summary.csv b/summary.csv index 58c785350..83a8c07cb 100644 --- a/summary.csv +++ b/summary.csv @@ -9,8 +9,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29979,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,13.9468,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.411773,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.3043,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.39691,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96958,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c8a6bd9be..f0fb62a8d 100644 --- a/summary.json +++ b/summary.json @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":13.9468, + "Result":23.3043, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.411773, + "Result":0.39691, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 0b5c26dbd867af336391d21c29a1fac30e0c4b71..bf61b0eb750747f0ada9553ccf8f741c62d41bca 100644 GIT binary patch delta 3120 zcmZXWXHXN^9>o&`1gQpLA+!)eXrXM3N;43NkkCOz5M4o95b4qaA<~3U6cUswV(3bf zAR;Vi1eM;)(pOM0u)xwwkbPmlyf^di$9reaIrsNJcjm5JuUglH^6?+#?b$4x0sz3Y z2YlKm3VdJC7<_2*OW)BH=TQOn{P^EBA;J5;eXpbf2A^#=(flQ3Mq5q>tc(@@BNHi5 zXgFsp3Ny)ZmXIGc!6Rq=P;)aXSt0X89T5*!I!4m634h>st|Kn2y=t zkcX#2vp-1dpK~SiTQ`?-eqEW79j(Y&_nERbalGS-THTHP3p$U%{mtayLqndwCfS3JSbyH0#bph(p7G)+xEtavV9&lIU;7QlHJq4<`vSt zl3hR)hWa}gDD-;ZW!OyBw;bGIs$1#JF0O6-Cl)sMS4bi@Z@Lc)?Ol~hPh_yk z@i>_%rtV=(5F}bdODCgYBgUAbv_vw7(P!Rbp-ZF$(Cs|-?k;FUVEqIOb5l`X;G@&- zmsRDmE93p}c`?~Y>aVgX>1TUAGgPiX0(Z3kW)39G_RZwKH4L%3t@(>_KqDCZh&C|i z-TaG{^br1F4RwoE^n3w>YLM#V#bh9=#$s||iL~6X^s@SDOqf6`>2y@1tY)vBbZhP$ z`|z6?44?92r3b&ILy@DZXcmmA1n_t$s{lb1EENZ%Vxj3UA`gt}kH2$l z9F47hfE7Fmx`>BTPKzxalOy#0P8OkD%dm@S^Y-(!tIL`2S+Gb#n+}+s;lWnh&YeoM zvf)zU$fc8Uuq2iB%cg5Txz6AlerKuj`R#LM91laby|^re<(z{2ej|hM?c!4QZ4KWc zU#*l~__X^&{l5M2LuypKvvccK~@MoAgtV>Zz~Yhm#O$ej8Nq7XYYnrvbKys8jh%1;E{(ofYYKPRSX}*b>N#Vwz72G) zO7S(UZy9J#2O=oEHCUnU3~8ZHR2Tpu0IJKFa@2``dY*;N-WZUN3npacA+z}wx+_nS z^A$6l#5{OQ&Mz)9y{FVQo~>K2*nv%5HbdIRx`Bk! z@F*kr-873^SU4WwCIKV(2&2yImelQcK_=WV6zG7OT!+YSRmX!q7nWcNUk zcrFM7g7FUeJKAD=5dKUQ>F$sa_caf1N1o3cKQsg|lECIY**(*`L!pPitDgPxklugY znGLcS4lCgnJIBXKmxVpyOy_0RC4YeN1G;0lD|?tHwcX8g?{nosL`Pg~KPa8$1!nFr z7VPd(4t1#4GOnM(J3-2)4Z0q51}v$dqICve8l&>JmPCb+jvp&qQ6FIIt+-fnmKyS} z8ml7LQWcDOwCBki9E$EYKxTih^lKMT6N`pcEv~Vq@#QxoEf+t20-Hy>1?u0j${aT+ zKWvQR%IF*Z98^_LGI+UqzH%x7t&f)5jK<&Rc(rr9?r;3-)phJO&_L~g>Wi2IEF`3e z8C#QimyV-PW|Bom6l}Ti(2x3ENzMBDxFhk1tS4t2k~)3Nt%*+Mmim6%OW$oMk3>y# zh>4$RRdRHUn)x0vi?<90E~ssvXaxbDTwah0YG=w51=((btQZ352vxGDt$c1gJ*gEkgwE*KixTr}vpg z5B!T^P6faa;Wst~2C*L$7TjqPxwF5i!P7Uq5ma7((a`Fa{>)R$$lb>a7QMwLLy3yd z(Aar=smK5z(?ao`8@NMh2&WmGK9r!8c2h!$;N^c{#JO>dbf)O&3zxtaH>g9~Qc@+F z{5X35O1-AKYHjx=d%q@9T~&momS^X%+DXf;RGq<W;Twk1wt%Y-Tr-G`H#}zf@e%`0_R=CG5sx0j2k} z=>&;`Fhrh?L7D$S%Hr*ZfsM$F_EiqUoXa84IfX(oSB>?UmF9e$zB|+QNzS7cf2?g5 z-a~tB)>5J7JvoyP<$XphXS>H$2JD4%e#itA%_a~GSi1^fEF*3(FkCH zNf?JTXhxzMC|NzekzJU39+>@V{deSCTqrIY7w>ae*SWCaYXB(ZIrtV*o(D0 zSl8&THGcAZMPY1ssAS}lN$Wb|Jea_MkKr+B0JQQ*EV|6-RlYpt%@g&H@+er|$! zD~}zwX8AB7f;6^#?UmZ_%EbXPkdY|%M3~!0fC!-}Kp|w`kJ_SfYow(1-8L&O@meb6 zjd&^an%XsW->1zFM)pkWCqGP%#fcit4bKxj-aotbE@$mp?qSU4;1BS3DGWcsH*R469avF|Gtmw6NTw<0GvZfYpAJ zP;=n-t4JBd)SLWI1I@dEsE}Kk!R~&k`$30qB1vo4EsN3lNhj{AF;zo9!~|HzS0{%x zJl(HJQJ%qxS|^mvhD%H6Zb#$p0EkHu_UNv~%n=yZIUdy#)$AhJ?Q~TQ&-->&j}`q6 zw2aOP?q66J)cM}>lejBki nIAyr>FG!g7qm#>4 zZb!GFkTIc_BL^XO$}Qpi*7^N&-hbcc`FwBB`#kTw$C5|&)tw5UfgZw?A^@EMTB5HmNnmM&iNZE8%co9xfBoDyyfVJ&wlM+~Rcug-`l%Fk|yAWKF& z!Ta10nV<+Rl(X=DCW^G%wW(&0Fbz*8s9@ejGPgB>GfT%68@GacVo$$kkN1@iEj@bL zQ4P7%uByCa?A#)J_Rxcu#&{T=inYL|L$cex)yGj{dm!bnYspzgn%V=m`?uRkA1nH^ zmDx!F*=eMufzW*i$0Ibi8C#1r6kp!aFdhB^TVR=)m5lge0&kCBpL0Oy-exZI#dxQ~ z`7NE4TT_-ccN;@oo35rm!E~*g&l_@$YmR8G{jiTKiNKIl4M&#i`t_f!u+Hmv1*|k`kK{%6+E~+GkudyO{er7J5@s&qegXU3zy>Tr*oF(R z$RD=Q4AT*dn{2|h`$)W&h}IiQ4Y@#{+7WVPal!3SQ==~VvOuMQc}=0Gh~J8!w}|if zgXv8N8w|iCpJ+GeHm24qGVShQimWpwH&gM5F!QAP-RuL8bxnT-Kj-9r$V=EDwJFfnHs2uu2FU|EH|k! z>vEHJ(r3?IHi{aTOK0AExCUg<$iraz!!;e}#Hjh^|Px)~s>H^Ggau7uqb_xV+7Py4&y&+MZ| z%$Yg+N0k*8$v)f%#_BMfrX`&eFAPafJJqj`Z`J7id1K_+PSlX{FQFB;M%l#s2cd8LD-Fm=z zf9mL+rW+Rorb*p(R&-rMkps9@ZdnWV_{FR&%+7$}1g4{zPEaPgDg_NT!Ai;tw9NUU zo1}Ek6bvhl?4t~=;Ev`=+|VNcFwsrH|Lsh>p@(e6|1D%b6M3FI}RR z3&=nlctR;)x`A_|NR;p#;0QJ$kriK^eFhl!Hs@pIqxT;Ovex7pH)z1L;djcgaskO}NjI-yad^Qu+Q<-WdQo0($B%EDs$NukY(fj~FM_Eh9|F zACcifWWOy_vqt}(z;Vu5AK69}-0}9Mc6YA4z@;=kSWU73xorQnGmX@jF!X+55veMHP+s;gK+S2-H&n|}Re%o(;{Wd3l0^E}aTEP$W^mGonqv;XYJQnlbP z>;t8l9)ul>c3F6FliTC%;n#)p463l=)!2lZyFQ<~W<~g#v=ww?aQvxZJNnj)%TIF> z9u{c_7mvyx&2uC$W$i-jsRI*#%W!JT+^qEp9g@ZcR;YRi%hG+iQW8!AeFAOXgz>!n%%S!_hIwT{?#@XxrE+_%GPAEwu z=0brFKB#?#<a(N24+Uti1P-WA#oj&b1oK&|r>rsjoG z2Um5gIA6owTCd!qvKM@rLzjB#wOU|j#C#8;h)FB$pLabI_6b|zpXg>NXw`B_B}5qbT2NWxLK!exzMI_jHpo6kw1c4C+_P&|H*KB>#S((IR3Q!q=oAxalA+3 zpV|jR;87?|)-V6l-(L-q2M1(*gQ>kV zpx}IFSK^i47S>+HV;R93wdBEh?vp9A=KTF$mD)&=<)~BB&&=}5#dT}Wa0Dlx1r%{4LyT#kj{(d+QY)A(Xjhz zaKDqaV8}?GbjPCv~Tpjz;$TyUPj*WpPP`N!)$(`B`H75s>8*z-hf&N|k-Tm7u(?fL7 zSN`+M1CY?e318fCB(iH8uUPe;26B2StJ$HqgSoO}G820qIz0I@@KAkT`S2;+#jY>I zKpZI>IU%ylYHMFRvml;1_zP=Zgq(fuuN;14?>`MY&prFne0eosIp5pox#q^5C}L^Z zu*b;5-vcfTwh31T=lSGArTKTc$uf~yhba}@^ViOb zIx}*>;dk=6>o54{{}7$x>K^w)URAHl-78-u^u26P^)PTs=(uz%PTx19X7bpnx1pk9 zZv7GW8WoqiRtkHG^i+fgX*FZSae&t`Rgvoc_3M1pV;w1K5YawWS$7hDyZaNs z{d*pe8$IHEbXg Date: Tue, 1 Apr 2025 05:48:00 +0000 Subject: [PATCH 403/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10222 -> 10214 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 2726f9077..295f25e04 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 05:38:42,727 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 05:38:42,727 submission_checker1.py:3260 INFO] --- -[2025-04-01 05:38:42,727 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 05:47:56,072 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 05:47:56,072 submission_checker1.py:3260 INFO] --- +[2025-04-01 05:47:56,072 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 05:38:42,727 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 05:47:56,072 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 05:38:42,727 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 05:47:56,072 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 05:38:42,727 submission_checker1.py:3276 INFO] --- -[2025-04-01 05:38:42,727 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 05:38:42,728 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 05:38:42,728 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 05:38:42,728 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 05:38:42,728 submission_checker1.py:3297 INFO] --- -[2025-04-01 05:38:42,728 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 05:38:43,693 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 05:47:56,072 submission_checker1.py:3276 INFO] --- +[2025-04-01 05:47:56,072 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 05:47:56,072 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 05:47:56,072 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 05:47:56,072 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 05:47:56,072 submission_checker1.py:3297 INFO] --- +[2025-04-01 05:47:56,072 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 05:47:57,010 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,10 +33,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29979 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.3043 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.39691 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.96958 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2571 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63938 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.82202 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410599 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e445c3bb2..09de93e8e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1030 +1031 diff --git a/summary.csv b/summary.csv index 83a8c07cb..eb9f92af7 100644 --- a/summary.csv +++ b/summary.csv @@ -11,10 +11,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29979,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.3043,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.39691,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.96958,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2571,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63938,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.82202,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410599,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f0fb62a8d..5222c8486 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.96958, - "Accuracy":"acc: 75.000", + "Result":10.2571, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.63938, + "Result":8.82202, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index bf61b0eb750747f0ada9553ccf8f741c62d41bca..bc7354b1e5ec73c70b4fe7d978c2fd121d07e272 100644 GIT binary patch delta 4843 zcmZ{ocRbbK|G=+(DO@)yH(cCnWG9pnS6x|UlhH@$5;7vAthZHMEB9WmnYcza*`tf> zJwqZQ>n3}z`swrde!u_y&f{^$ALqQz^R@nXz1DOWbxZqbnAS(<+Mm&aKt51H5SWv& zLBjx8xMtpD514rrqz5;5fJ$hBqw!HW6f#A;&!tkAjq1W6$j)h`rxAVyuk+k5nR8V; z+h4QEZ>Dy3K>5YNuE0Q)Q1c6|NF1fYd4yL@BeTLZy0>xZz4(kw4LCx$(_Z!NqUh}6 z_~xUQC*2L(p%7~g8aj9q7P8P(_98J>={w+Tj=!ypz1l;==Q`0nCxB2jB5d`#J5QEx z@rw7hHwT_#nPD9c8Zi#03t5n8w!3Ah&W)FN;uIHarjfGY;pM1S#CFQ{M5Ut~S)Q@t zKbl?F0dMDUt3#ADlnL5Jo%p)jNt5EeyK$RkBEr#OYD6Jl(GHDd$-mdTye+T!0KmrH z^@)HEM~ye$P3s=6%fT^7BFYiBvTz!4)(FkaaQ4c{(z)QpH+_gF&gmHn)5$Ae`mY?9 zu6j5RtmchxC*x#096ye+@i6dTX65vkGhGcz2=dYIZ>qG7zb)%i^4i#G%*U#VQwqbA zpQ!tzf6&L^^>{dgMpEOI(yZzm34p}f@Tk(4Ngg`|(m9(!HnMZ;L-iM~;YGrVFXFhP z*8_z;(Qx#kMXTQ>8_{kF9@_^$>Up)eXG4U6$i$WKJ4)R=u!ZV0OYM#BHxO{{K5bAS z<_#M-kt_tJy;p2VOKmsKNxfOcw*;pBczTYOS_{CUSgQFRD74azzU7DL0pFg|Qs?`I zpirK!Y~b8AZfdGgvN|t3e7r>{_5Br4rOGfLw_YV9HPs;x0usCa_mIX9kxJ>8gJAd+ z)R(Pb|NKgS*7YLweUwWyeHiGDV)Hkh|5s{WY6OG)|9{u)M(TvPkCWRh0m zIMi4K?>NEY;Ifc((g)bG@|I{d TWhIZ+^7NecD`BY$==03{4(PmiUY!S?6Pz@*k zlrCJMVEUef_IyjVD8R6V<`>y_Nsz9MGWFyeH!yWKV>aij$W`CW*MCRGUUaLIZcJoj zQ%!c;wz6;QT=e?GDc=eIFk##Ov)=j?e_0*bz%lq{oq!r+7u8?R(HZ;OKzKgs; zr7uIk?A+8rq&vKN9;e)-JIi_=ipvT9!U}(tvnPe5UU^GX)}U4xyfwZ1CyaC)mHHdy z!Fnn_G@&!T&!Y0>r{6=*hGK<5UM@;3i>dvY8}eC#P@{a1Tm@Qcuf^*?*iVQ;=bjD2 z>2O$n2n}HPP4$wOR_yc4=S{cWBTZj9yN3t{zQUQ5M&j| z_bEUK)!|Z4ZlIZeLOv$;Gg{ynyQE!0jV=~iH$=-VF<*Zo<=W5&d>ZZWW6O55Jy*6k zkIHK6{;raYU~-ju>EUcE7WKsV?)gpgZe5=(gKoe(%Uw2lS-?dH5rxl=%bu&;EY?xa z7rVS9^R*--KVnB+ZEwH%mTT!HABIQg_nR(rN^dh8**);hdvh+fw7o^z6t@ysG#q?| z6KhNIvM4q#l%L5hp%Y&bohX#dgUHNG2_VD4hTdB%aydp$E=HQuG12WRRC?MpwCo@d z2+VS}sDM~Vm5;tNfj|&KAXpka*k|k8?N)3`r~`8W+xVcdul2s?{MU#+(JTSv6rHF^ z-iVIbU!^%e?fSrWI)eUxOFt}saom?!9%BN zlf`JaY3&?`+&;aFSc?vJxKSLoZAIka3>~US$$FG&jYv~ht!_M@tONThv)W#vm^Ny{ zxtA#o$TxqonmRWfN^X-$6evLIOHl+V8$X6#@QlZ>9R1iaHF?8E7t-?e!&jBV`;(RJ zL!ZjlSKKzlr;nU4*?!C|DrJ)J<1pFl1Ub~Pi+ogh=Q2acnhicRY(|;($E>iXZc6;C zoo3CCC*z4RBhT}mwpaLin?42PkY zGMAkrEq6!1Z6Py4)Tk3)DcqrxeX(fgOY@A;!*;uZqx=G6$n z_DSAMdGt&USvpS_235CwChQR5GXufq?u+BXeP%LRFD(5&C5N73nI7Duf&s!hH*A@w0*m&~G_ z#77V2-xdDIECBB9o*!Td=Hl0WiWgOOXY|j0me+V&GiI6D%J$%h zQ7ce^ANQFJn-!zc%5gM(H9aOYQe_%`u6gqE8g^r3^_d&d@4PE^W}AS(A+M2o288u< z22lh{N+vq#_VdNxz8Zz3I4IJxUXfr^p8eLOKWo{I{T5muwF47~30$TX2;|3Uuw&7r zaHy`w&a@~B7uo})qy=wO38CcUnUyIh2{qe$43Fckj+4-cb^U>lj&3PlzSbXaP$aHT z)$R>gCYb13GSUu^#c99Q6;$TQ2X)VbB_`=YsH4ZxKfrQ2;q<~77?y8Dg9Y1u;pMce zOca`kBALdeljUoI*v$QgvC<2_7_7OWg70Jl?D7^689;dvbs7Vn^||%y$131O3LR(L zr@o^6dWy$Ei=K&rHKT~+IE{rh<#XkdygVx%2kkZ0X@A5D?xd?*O%&6zQ}vOiEx$Y7M*JQ;G`+&$fb_Q!jJ^}z->B2kB-yKA>#cyczS1}#z2b-xy+wznaGgQ?kRU{LO+&ky% zGdq}WaEq(sHRS~VITIWER~(kU6U{7Pn*yj|@U$p9)!@Cw?=(qeZ9;E_4Yj3hO~YPO zC8I;1+ik;N<@gLcRz<+H`;z*^WjxELWi0x*qz_$we`vn%-V=`@wQnMma#jCIM8bFp zPQPf9ivAzaXZ^@Lw|=Knc;Fdj_G0hYS}H)9u$+!ddaSI!ELf_-vRi%HA!kWxv9w5`*Dbh`&7y)o>IV5KebL zk!LD`SII0xqD8qC&>{DT8_QD6QkoB?hA-wsI4Klfpp0DA8Z`O-IS9D;&V$sO6AGPD zkU5`=dbiTafGIjYzDmL=zs8TXsU2eQY<>g(rtf z1WTVQCcRTii?YRKt7~@jU9C9Cx{w7;RYWp9C~2(^DHxCFQv`0s4SXJco{{ywW5;8< zg-0?>t0MBt@aP?x;)-1evw>Ojmlhe6zz`^2ZJLJ&$_2#S_k~OMQIWa|d@{4%7Z!(F<9r6-YW(+p&MJp#bPEsC^SFS3^>%g7eC1jKDI#G7c^i}W=r?0`lx&%fv zB$DpB=S8L}iv`V>39DL!j4+&K8>7p^+h>u~b%@7Wu@#6`9VyW->Q2-ZW_3!RGb69= z4b`myRa2=~4GXwrhHltsSF;Y)46oji3AbacW*q?~qiO6f*7}=Lqslv1Ge51;h$~Y< z(-~dhPAUeoyAlie?6-<$o}GlzZ?U6khY$c0?<}V@SWvn;Qht@AhU9t#(_6whgp3+S`lKh zPhGAAc$B;N*!ADVx-6z+``Ijx{KMPh1K=c88SMlAMEXG_?2K)_1qw=53xG2R!2hN^ z?fs48SgBOmq;Cv%eG#k50Bh2~=k@0q)u+lmW+RS^;`SEu_9X9C;%hj`4alAivxoFS zU&C{#WU%yuV$vt2w1IZGS@n?Fe@%G6#GBtj3hb2*fdBJhX6c&rp9has1$N_u-N#o( z400A0#BGbXgoke0XqU4Ny%_#=M+R)mSbk=`IE>O30F0^G!N(X;W|+MptG%FqgPoWK zol=%LUyAX9n|2d4pq!8;#~Wc9ieVaCByNXYKA0}S4ayB1WD#~GbCA%&v%BFK6FBdd zyqW(Nb|(!z|6KOS-U5=BvXxf$VYj)(0m2+Yu#(0`TPCIz8_(x>?q2MCUI~zWtmi_M zRwYw+J0`C1)DcEkxCxt1_N_p%C6~h!$rzb(4EfWc4fJx?{>x{k=hhY}l5rds*N?nI zMnw_P$d@uKnQmMDqrQ9n-ECREGR()42_fda?SMpZf~1e+uhUDnjtSQ;2uLW(BFxF5 z?uYV>OFg=jnTir?eCWurYi9Z+E9sYokA)PtW&XFemzJF5w$hPg`Kt?wRCG7Qo~geq z=jX_z2ENuk*2;mJ(i@D$HT{V4x@@Ai6+>-)>Es=Y>Bran{uNR&r2GKc%A(Ki$w$@f zHAPL?6^_py1o<*ORM=waFi3?x92zerm00JM$kyDJG+AzABAaGh65X+RWl0*;X7|C~ zmegMsz*~aH2VDa1D~=2!zwq^>3#T1frrAy!$N*>kPpCj*;sgR7-P$RDG`NINKw96L}tAsp8FAvBJZ!N6I;} zvC5; z5J&g*VCdi!IIo(+=aJ+HL)<}Ag;n&BUIvSj`Dbr|>bNTmCHF-EziVE}YxBiZ<)XCf@Q z5_IpUsLr*I3N(n#&B7L2MABX$RF%B+8#lckVwP!etsswZUjyYF-GLQ>d?AcM7KVuT zyANYhsDIyfvLdyW|L43>2j3sTG|-*36s;eSV95FMdYP?f5bz$BzV#ZOQZ d68w)ho(XYK(GXq>DN~se+Jq4FOag!3{2%4WB-H=_ delta 4897 zcmZWt2QZvb*Iu0n*6ORXdX22;Emm2p2hl_H-lAKpMi;C^jZQ?A#0o;RkVJ1$q9^(i zM6Z!=W#;>4{`v3BojY^O^PKyfbKZH+UDw;ydpksMOJ%xp>+~_LJ33nFwJ4t_P**CJ!lNfakUnFsW{qW z=C-u#uov|ESO$@z@hRy3VKEl!@r@%`vlQ}(a%c~Oe(D9bBn+l4gYu~!#(E!aZx(jZ zI4GSZ0&O)3WY}|sI1}3)_Py2WBT!NEJGm?}?dPGfp|>A29(1m~W+x7!8uHRbUH!azkI&aVj}w*d=ch{}~Fy8UGAg{sPZ^r|X7bEwm$`5)XV*K>lHp=x8G;|7^& z6(27o>tQdZBvXWg*m6a%oC!TzO!rLSwCNvxnO|6r{wfY%t)`eT`fB0FIv4D4nyWi^ zGM^fvASPhuS3T=S_tStQYAj-mFWQ(bh`N$t?a0c)M8K#t(AD&_V+~0~9-iuYapJkO zD@C=2Lr_X{T;@z39wH>vAB9~OwHk_U+p(?Up5-8B9h*@fd=hh-_&u>cR%2%Tj56L( z| zw9AwB=JxYz(9yLCf=M2T0h9k}~*Ni`K`ysfZY1m(!U8C8t&Wd25tq zYo+tWO_AGjcYWTxc3q`%Hu32%F4qSsZ~2)&Clv`>4zVv(OgAuZFQC}Q<#k`IGu-9k z(lqpByIsvVGrz0I%{Mbb&18PO_p-Ud-%_`YQ)(`CZtJ3X6V=kZ+0wGv36-DV6lw0~ z2J*`i;;Y3e7i|JbX{`d|Hi5FV3zQdX42hJK+D}MwH#s)UHzKONfNzREHZA-u-c;m% zMHRo>-pV68=W=Pi_p)aAtMSHm6#>PkVoI0D*nni^qAeh)5i5RVl`4mglj3n;C1r86 z`$*uZSXia3G)jDowZZlkq0F=_i7-Se-uOHHdEuKDUxW1$_g*R&hvk{mMFyKoFTyh> zEK)h*?%xzZ=upA*_Swn$2;TjM#V}*17?Hm0G6AXkf{2H6L98`;Y!$jF6V&s+Pv4AO z$ZJi28On~CqBIZ6yHHbcbk?{`ne8`a)EmfTTrN*zN1d*INb^ECc!0qW5 zextUpXBp&0!mQ2**LN3Xr7iRgm9_SB?M3%T-Tj)lRAAXH=%o*L${+icY8`|`Kre{& z5d?&^AOHYBit9>%>-;nQ^;-Y{7#&6`fhx`$ILTbGyu+F5_26)oi$hd0f)Oq~+@S=r zxg9Ffm#F8snsb+mY2G|E(5rZ4!BA}b$8;;Ur_ zfUt(kgGW|o$5c@HYy(?&N^HDM>TTlW<=mP{4_|R7kvRJ#Ct9g}T|H?8c z%S{EJr(Z3WapQAo;(!L-!)JoENwjj$?**6Z#7~zP8l9DlYtUukX7xEcCQ4Z3p3gRn z+c9@M@S*771?{&L0u!WdQC{&dz~u|g+}4ah7*|k;9wh1&6^yI4=oySl7n!8la~hZk z5*hZ))9jh~1A{F83W$g0FO0yrhIg6L<1v@9nk?a6GFMT49FZbo(wWT%QK|_%TS!$% z$zl_XYYQ0%dX#^8dI~xq-zUJTSaP$If3i6Jy31GjU}lteg@zRsmT`wQIsNXiQzrid z4&M{;-(6#IOCyT~O-cc}k3<<%eOkcaRP@-gTN{He^Edd#9@Gk3R=BPLZDtxFR>`cL zj>?0?qw|9H{bf@z<$7U$Zqq~RAfhhc^vT{S!xX(8w0{_F4;IAhSv@Tgnl~ubb{aBPXBW?bXdqv2*z zlNG}^JnC()-z3KPtfizvrmq3XkmX9BpiXGMztl=Jk?y@eaA~I! zMBa_04sT%<9X4d{&Rf2`TGfKUlQ$3UYGn#QojhYTDN!0y3)TTjn5@P2YQr6h2OJwq zXz{HSPf!1a*Lk6BvExG^eiH*NGTOR%5n7SD-O%pu1TL!;q(KNpgks}{J9w-5p zp$W-=aiv2-h#(o>@W(gAk+8Z5*liM!0-P&>nr7=pU+3YED5?bGOv9)iH!ml{hTK^X z)Vg+}qQ;m8EfK8FVEI5JgF@! zh__HS-0}5VcQK|0zknp43iFuQQHgXNOT_fKYpA9w6x(yw9b{r?OQt9`O<;kyHT8b4 zHQ4lEWB2Q=Ld;xa4@k0>`yFIt8xK)}C!9e17Dh3c$xQJ%0|F2x2Q_5Qqs*jZU);mO zat_AWWBi@7^Cfaf*9U8E6>w)+&^QuT+}qwFvo|rY%gOOm5Pu(jVR>rFx1ngbKlI8j z_EM&iyAPJKttDX)Z4aVIgF}^hUDLF!V7zdEJw3$PgHka~z+$$H=6=QP?KJVATHOCu zO$P2F*KIa8kQz1Hkth#OA&-FoBg&kx!Xwk4C#5@D5`FaLe0?yAj2pZ$Ku%>>dA0HX z!*p5LFQZF)klH5U!RtYKD4<6X-@=I5YkAJ;;hSt5!jRvcQqt+LdXDvQG_)^T_oZT zh`cBGc^tq=TZ2xXyTo`|Czy3Z_k=6y4CBm)x8N2WuNUM8BKm!{_@Sss>G3L6Xu;7I zkV3-jQ%yIt6SCi}8J#UC@$0Q_ncXXX6-1;Hi8)S12Xm0t@10TcqKx>mZ@b#Y*ai4& z%P^gdPoKevNPAy7E8VOa`PWx1;rN*&6JPvl8WypFfl^b=Bn8o43yU(mH|B`Si5vyg1!N(i{9zIV181bD8t&c@Rw^=cHS zE64L6KRFGWLwH2lEZa=nSI)PK!z$Jb$axk}3KeySY$sDO!}`(<5m&T0K1 zdiDxD!35$;Vc+w&gvt)-BN~xo|72L^2VkhkR5kGYE+a~d9=B3GK0g#88ksood)=s@ zq-!O&_(CV_^x3-haJl-ocQrd)DPY_@oGgj)Q0MaIrUEnlIR?zNRRK_ z$ao2)C^7rA(+!Ql#X+u`C;JP)sIc#)v-jl<;{8#=CMvS%=fVgIAv3FYGaGxHhdC`s z(WAz>uhmi_Uz_}rgUmOIPzl4-8nZ~8h5_s)Dp16CWH#|A1gwnd-Nk8`s}@j*3n*yU zuBvQT4T2OG?>6Xjq|H?I_r1eXcSV=O`V20_Q10AQc8^J&rNJ5gF%!z%O%|WBr8sAK z>?tQ0hKX5C@L zOHMLwvnGpCjXBZ0AsQ?$XPjU?aCkiNn6T}cEm#@t3>f0-#s!TtJyST8F}$c;M;7l9 z9?ajAHqGqIUa8Q@rM&w3UsvJEvUWrL-4JAMf5YMR1Y%A`|HjLX>?Ql+k{2J5|_^6{}e14tU%ud@c{Q+#Nssh;xrHXSzV4f!H>|+AXZcZ zoDb`z4TqQ_of(0B)r!iie)SitG+3pC-VN8Iu~ZI2i-UstcgI`ZEVWQkirXcP|2?CkMDwAi+?|QlA6lWa!CB|L3w$D8;2%Rp}I;FJ8VV|>(Rw=8-2-M|DZm& zW@8M6hY6+0#v@Y7Z{ZHYFw&LcqY@48@;o3DCJ=Db`r)H4cUy=*SOc$y{92Dh*jhDR z(VS7kz_=rK*2Gmt=a0p$d6OT8=90~CIu(dSS%c&31av8CH{DFfrr%CXgDs00gpMA^ zT33%I(S^LbFJ`dGbE^$joW*|xf{VqtIZJg`ymWB^{YrDtr`1D>jnsPxfKiD2sEH$^ z*R!8TCx@qf<){R;g;moh*iSB>R_S|}L3?Mew|@_7PL_Kj9{w?@Un(pC7TFMqFbPYU zo~}-7jtO3~&lObXre@1tTLyb^g*{qtLsJLJ|1K1KhyGm&qC!cwW&LW>OQEM(D$~1g zwf#QbN5UA+vV~d{9?cTBwxWv+{7-FDqY;&OYCaiB~|R)`@i0XKg%vmits}hI_(3?$h!B{iJ^d$_Ybbf8Mf#C~Os5 zVH-=>+0n^J-IY#lmD;6VIMKdCWvyMCm4$Z0eOomSHhpC+MAfAZLx`XBtYU-P=cF#f zmkwFporcMV-=n!x719%+kk?$XaJrgcFGVP8rTXK8^tDrPT){LmnUNBCkcV*apy{~;QM&ka^%)%R>0WUe3mOY~zG#b>Jkw~L>ztd8 z0V*#~tb{mrytMj|yJwtt6?Hhi30`ATA#?VOnV173tLd?~c@)J(Ox>Upn*g?zK_^gL%eRrt#%-j|a}A=I}*HgL`h_qiBT1 z!w4$^ASHMLXYmb6wL5rGcFro1lx!PjsbY%&-Sbw8Hs3WP2^Qh`_XVHF+D?6;0ajW5 z4qhqeepm0qkb4hx$`uO|30wuc1Ohvwd~{;#l7gCFoWD)xThs*V#Z@i^Gt zgd~QVC=P67P2x)u2~-H=ltCV)GYN<|Ju$3(#_e?>rYw2;Tx$Qe^jiBL!Xb8yDR&G$ z+5g_1&}%#jEdQ(Yi~s;eoS6S@5s@yW=txp9+LTuiFBTom%f- z#)wENbSJM0#Rnz;AjnG(tiLmgBwKv diff --git a/summary_results.json b/summary_results.json index 2474df9c6..95ae4f926 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.96958, + "Performance_Result": 10.2571, "Performance_Units": "Samples/s" }, { @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.63938, + "Performance_Result": 8.82202, "Performance_Units": "Samples/s" }, { From fbc36fa0a99e00cde2fe076fb399fcfc05e26d98 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 1 Apr 2025 08:54:44 +0000 Subject: [PATCH 404/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10214 -> 10216 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 295f25e04..32e52f40e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 05:47:56,072 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 05:47:56,072 submission_checker1.py:3260 INFO] --- -[2025-04-01 05:47:56,072 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 08:54:39,644 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 08:54:39,644 submission_checker1.py:3260 INFO] --- +[2025-04-01 08:54:39,644 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 05:47:56,072 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 08:54:39,644 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 05:47:56,072 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 08:54:39,644 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 05:47:56,072 submission_checker1.py:3276 INFO] --- -[2025-04-01 05:47:56,072 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 05:47:56,072 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 05:47:56,072 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 05:47:56,072 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 05:47:56,072 submission_checker1.py:3297 INFO] --- -[2025-04-01 05:47:56,072 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 05:47:57,010 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 08:54:39,644 submission_checker1.py:3276 INFO] --- +[2025-04-01 08:54:39,644 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 08:54:39,644 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 08:54:39,644 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 08:54:39,644 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 08:54:39,644 submission_checker1.py:3297 INFO] --- +[2025-04-01 08:54:39,644 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 08:54:40,655 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2051 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.340923 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0396 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.340923 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 09de93e8e..4ac943d1c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1031 +1032 diff --git a/summary.csv b/summary.csv index eb9f92af7..ce66121c1 100644 --- a/summary.csv +++ b/summary.csv @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2051,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.340923,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0396,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.340923,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5222c8486..57e014cb0 100644 --- a/summary.json +++ b/summary.json @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2051, + "Result":23.0396, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index bc7354b1e5ec73c70b4fe7d978c2fd121d07e272..05d7cec484dde0dd49b1e9cc48453cd94aec1146 100644 GIT binary patch delta 2764 zcmZ{mc{mi@9>-^pE$dj)AQQ8wC?Q)}Miax>Mv}3V7CU1ZvV_r4OcG}7M$J3+*9eV> z8j&?3B>NH~YrT}$GSub0_dfTpd;UGo^EuzoIp5#&oO$a7>q-Lup)s&)(JVEL-0{S5`(oX|xm%;`^UQNTZMA`hN8QLteJ`hoqR%B< zG$Imp1d893iR@Ufdn|t*Ijeb$v&$EgBQcJ zV*4X)z-_MI9#6|jsbYkv=oJ^@J6NjDE6ORwz);u_YQ_(GeoiaA&Js;W)s=|Nz7Des z+%B0}!nTH_-Yo8yuBbk==4kg&(AsXNWM&=PDs}QpvSZqRu; z5{lx872#BV*C~*h&(-zEXddhDWYk|j@1a|yex{;ZO8q6sTYFYmM(YT1gN&k0Sfdns z@bT!Dkv$9O%*qpSvd*jP2`%$SXx(RYWI+z=q>E&wGdgLSm)QOZr~ys}8pZx@Geai) z{&mgx7bF_8UxzQZkYSu(!+Sg-CuJkm4f>i8M?W^n=*~u(kkt<$VF}Kt*qAl zt|IIN)yX{k_^CI5_J}BbeVH9Fnq}2bEM7E6>}iTJn!3ZZ_!+@$>2WBMHr|q0@iCC5 z8a0#aREF20DD!6|GEHhSyd&4G2iLG-#h9#@RoA*!{M4s@&i`OfMphl^aS!dgnNg5f z%6_p_bfkxO_QqCq$*{V^IEf>JD10tMc02o)?*nx0ACY$yGt_291JZt!_$8cdV+ENl8A2RVQ_b&T> ze>q6m9&0W@l&qksO6KAx2>OZCg&>d%6zgnMft}%hZ)L*;1(BhtXeb$_Xcl}J4V|X6 z930F|#skSRBG7sY3Zw)d_@Im}>ZR5FJ_agNF61E(ed{S0_Vp@G*EkU&cS#g0DaJM- zD?#*-0(3MKbV)H)nb9I4&)yWYfljX%6%!$^R|RaW++wZuQZU$qCv)q3q1@MTcR-`U zalB~an#_00`Yxf+XM3!n!YgT4xta+6hmp14y#2hb=<2$pt~*V1quFzPI;12C22e2D zv&8;=H%f3%+H=RnJ9a>(d&AQ<3#}90)U@ zQ`17ZP{^l3=BI{OWH>G97}-x&pW`;S4kn{b{ReR_!qt24ecd;`J3SNmGfr0Rf}w#bP82e@J@qz&_Z<`r zp?bzx3_!B$6D%lzj(z&1TqNXj%~pYgagoq+idtAsr)`%og;mb=i|uNUy_LMJ-{)5>eyJ!h`V?;5eiJfrO?^BTSi5`Yi5Yf z0gA=Z!9Q-qz{RI&BfF-5ls>7>DQC3V65I)gEhm`T-S!&s77e*XW&m|O2$o$Ec)<+r zopL}V^L9XGIY#bf|0gI-!_1qe;q;sDAZ4j8>&1DGh#F$1w%dG`wPd}3iZzK09!hg% z@HRog0IDZoUpmWPJPQhp`FB9TKLIN@fLM9yL(b~w#R9zaeNMNUbR;X`E`=ASRP#_q z-)RVz$PphKFUPA2>!D`N<3;))RM-VQ3vLV0kw?|{OBHiPmQg#w4?)#ZIwAB{1TjyU zb>a`6S`lIHlVtSHa*tj5(S1>WuQi$6m=R6O=wI1CC@;@F*;#cyjjrG;6*u`OYmcZ0&2M1v)Fz&f}pB_ zY~gryT!{!-5e*Wb#M!S5k095P6~lGRf<{|Ayli;?m^o{4t~(7clD@!QRhS;{>AX01 z99-}`E;rsQ#(bs(crsB)pD96q-qD|5Fgw3rF)edCl}AjQs6**g^cTed?4;}NN)F%M z7t0ef1v*5cW%LBI+WF)TN##b0bv^Yz5msdK6!T*BXVvoPojkeXWG&rHu%%K>@Qzs0 zsXPM|x3hZE7m61&(?Y3k;F6U>hEWhMjb7fg$gOIy$fJGwIP+rjm&kXh%7jl+PEp6W ze-!(=*xTMFX5VK{-r;z2)yUWH$A4tkN1gszGuFPec24<ASHGUph6y&?wj{;tedYsOkJGU*x)!X_`UfMG+3Pzet6Vg#4~3> zzN&C~-`8vFAK&T!`U)8y_}<>?bKN@dX!7{!J{APDCVEIvqop23l~z1O zj1$C}rHnqjoO&|-PbuxLYmMH!N@ISXSAC56A#($>5$AN>o0b4QT^ zp7^+dSAtQ@%tbkvftsqrEls-x{$r|3@f4@dq*!eam6wnvgm{7eT#YeDOr>@BhraSW zYoQlA4lZ^`C>*=dwWU1B@}NbdP#@ByAIRUqnQGx3?@b$a~2BC#LB{Xd@{w zgzTRf_VU4Yk8kdKRGH&Q^Z<@oVM==#zwcc3}b9tN~7t(L3dd z)GAI5-MdO(ir>u*1JVdnMAD8Z%R#A=>p#Ct1l$Q2O^tn-Kmfpo3jh!Y{BOnF@O+Xc$ViE6Z@v*rl$Cu@$4p7)F-FSRxe0 zE|EPN8dGG6B+DdQ$zHv>=brQa^?v{To%4J@=lMR*Ilm>F1)J(HK|WD>{gn^Y8-lK+ z*wKkr;*qFTT4ai7bXzX=?xb9D-|Uy_($79jL7%p2(|i%JtCzFZwx)6%{<{m_r59L`byB4P?*`39YYzZV6@|2uz<8w= zpUvYTKrG2c#%`In!q)q#S7x~$%k9s4YZsUCwT&oXr5C8eEemp!eWAr%0Jp&qmi^2;`HJ@ZA$su%mg^~!zNlH*mI?>c8Lx^_J!qYqY< z&QFbSRa zUX+r=*w41;9zP2KhD5ZBlbIR@xmSYqpw*xF5`pG!!ODKgku&1tvTZmeDRQQ;``F^o zA8iQ-r~_oA_*j2m-1^p!nth?q7xi}VH|g;<+NJSYp}o6zf!@OCAz6MCgT3|r@yW1X zUW=Rap<0rUDhg)}PLklyM~Sm_S`UhUVcD73JwI7=eC`H89L6WXIHeHaZ+AXDteh*7!)9rQc(u{SC-KiY%td0`uBS#GbCuqshB;wR zy!!TnMxID-*MvrWbXs^HwQ1{fftDQmdzOl&<%uekVck556xw^O29)It*ZK^vsvjsE zntg+?c+fQJR@5zVc~8hP#!!LsMuJ?NYSAsVH+{MwHQ|owH26s8+F@_5ZcY7 zLFAclR0svCMIRhi#+AO#Lr5tgq~C`V$A7$Xi`=iZtk<`W%_G@#mg%=NzrEHPxcdFL zWus+*quFUilI^q523tbQwc+n-So$C)oPrMFYCBJcXd0n}(k!6qXax!sNQScDa+Cxs z;P1tv&=x+Zr$O41B@lD>zO88yI{S~Xg`Y%=N_qwm>egyR^<kUdsM5c%d2dQr6Z7txHae9o7hOFU+ zpZSO6Mi4!o>2uW2Of_!}yJb1x+=O_C+u*$KTPo_y5wQcG1l1?`;<;1C&~t)%*d%^s zJcumMwh$%v9{YPbNarq^hNL@Y6tp3lVkMk!j*zwIzX*890AUYxqkYQfA%%=85_i5p zx>%UUZ7);71rFcIo)^rj$E}?3oi1AkI}afhjWHe<56+wVMFc{OPl_}VUAfL_@y4j( z=H3CVzGlc)LqQ$sz5uH?9S2P;TQv-KN39&n<>ikAd*2O=(m;mTfn4Reu21NfXifXT zOY73jQ#r6Rn)ObXZR8lqputz9tmWA9?`IZUmfs<+-%Acc$f!>B)DSQ*e`)h{G6+ib zjj^V4Y2)JYpB1z*+;R*#6ftZ))_VLS6)TvnU3sqagP)2OT;F1-L3n37&dDsrTekX% zKtpqOT@I}(wEt6UQJ=8B4BR{&qX+x*P&Vzi9GSl#Eu!w7YlbJK^CEpvaa#)?c(QAI zlpZMCTWNbcCf?@CK_`Uyd;{MqjU4fBNCsIkvzc(6Ym8|f7pAoKZs55h?kR2u8E1M-x$(yF)rRFpL z7y(-ND-A0bgdQM-tH2F9OX4Z2GKT1QYufsfridoSRdeKcX)?jE@)(DG+VZu-hc~gu zACki9uSyeuQ-(T6%b*IJr$+Xd3?uARPF!I(2iNr4$z@Mrv)JBxHY!h_JOs?2(Ot`{ z2-r@2Y*gj5dhVgjcIs{iG}4sdHzNmeq!-ywmm-(mE`CPpd{TR@2DJIh`aQp++$_AP3Fkd%QWyksHbA zUnzRA!UH$qBoqh*f`R<1PBf51ZHp&kc;?)~nQQy5O974doPu-%h6l7h`aOt;8JZLP zwhs{}4iQ;u9@`!u-|Q6xP9@N3>yF%EB|y8L0;VlFIf$2u`Q+lTMf z?iMo#EtA!BuLhoq4rc^L`V5^X2QCzlha_C~qLO+uqrr5Pj@3@oeg4-_&>`De7Z8wR z77Z4O2LJ9a?}hdKG|fES?Dqmb%;Xgll34ceo3*gQ#vdkwPHg`LxUUPsmqfqbO}h;~ zJlAZZuQ(HU8(hkzL*{?2rhQP&8{zfQ6Z!gev9S4 zbr{{lBPwUCR?@J8NW2ooZY!F-V3=D;ocm#>|g{LYe#h->q{gh{=t zkJ@w>hBp>J&sSa1&D;xC5iL{ado;y$r1!1jbG47NpoK1#{2LIIaRjb?Bf=HG#6#%@ zgeMX0OiFErmlYo?;q8j?wDg-7giji_KI%DG*8KYT*U^i8^1opDfOD(MHnzXG7|xIZ z?tj0Yr(rH!MCvV=F&7`T6NVD0J_!KaAZUbz|4r}?qvDh#sZb>lw}LbkB`rn0q$Gao jAAh+309$SVKmzc;iHYT*#weL^Ia1q|ApF9LzhC|f=`S}l diff --git a/summary_results.json b/summary_results.json index 95ae4f926..421b8ffa8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2051, + "Performance_Result": 23.0396, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From a4615674de61e99bdc22f63ed100548f2e8c7519 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 1 Apr 2025 12:14:00 +0000 Subject: [PATCH 405/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10216 -> 10217 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 32e52f40e..360f0511c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 08:54:39,644 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 08:54:39,644 submission_checker1.py:3260 INFO] --- -[2025-04-01 08:54:39,644 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 12:13:55,362 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 12:13:55,362 submission_checker1.py:3260 INFO] --- +[2025-04-01 12:13:55,362 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 08:54:39,644 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 12:13:55,362 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 08:54:39,644 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 12:13:55,362 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 08:54:39,644 submission_checker1.py:3276 INFO] --- -[2025-04-01 08:54:39,644 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 08:54:39,644 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 08:54:39,644 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 08:54:39,644 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 08:54:39,644 submission_checker1.py:3297 INFO] --- -[2025-04-01 08:54:39,644 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 08:54:40,655 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 12:13:55,362 submission_checker1.py:3276 INFO] --- +[2025-04-01 12:13:55,362 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 12:13:55,362 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 12:13:55,362 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 12:13:55,362 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 12:13:55,362 submission_checker1.py:3297 INFO] --- +[2025-04-01 12:13:55,362 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 12:13:56,348 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,15 +33,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29979 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.3043 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.39691 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2571 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2759 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.82202 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.75122 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410599 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0396 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0723 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.340923 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 4ac943d1c..0ff3e5b1b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1032 +1033 diff --git a/summary.csv b/summary.csv index ce66121c1..d32e7d581 100644 --- a/summary.csv +++ b/summary.csv @@ -11,15 +11,15 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29979,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.3043,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.39691,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2571,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2759,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.82202,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.75122,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410599,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0396,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0723,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.340923,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 57e014cb0..addc51cda 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2571, - "Accuracy":"acc: 75.600", + "Result":10.2759, + "Accuracy":"acc: 74.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.82202, + "Result":8.75122, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0396, + "Result":23.0723, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 05d7cec484dde0dd49b1e9cc48453cd94aec1146..941a75732a1945939e883a2bd2c706f3924d28be 100644 GIT binary patch delta 3137 zcmYk8c{J4PAIFCoYiStEFGFF>7;7Z!D4`K!>@wC0Sxb}=!WctYvyMHoi!5Qvh^r_x zNQ^8cTSl^6QDhtO8{KoyxzG7N=lSD%&g;CM&+~bo^I5Q1w5aa~@){wxSNuT$Kq!Pv z;1(sX0rlC%!zgD!e(c1j+EiQ*jN8_Uo_-m9?TK)fZFRVC#{Hhzo||DeTc7XmDjPU< zF81!ZT||R?y8EMxg{38^%mEr^6OWF;9T51(YU2BRbZIMfjG$fJQ!T9T&QM4CyZOLV)PI=kY%Lhc+ zmI2x#7`-((4JlX&B&@DQTNI>JzaO*Zg5HxpF~oLc1w_&tk>amp@oZ!D+Ov1wO&7DE zjcp(GUu_gG&9q+%9=u-wnVVP;uTPzT;ar{Oh;>U(JZ$5hAT)=1I}=w4itXQt0yAD^ zTU_k<^2}Ax!pED`>1Zdh<;g23q_3e2ysufn+MzyXM`Ju+^Id+&U!M{WqN@AJJzP>m zrE_=XzUo-N$h)`!(R?g$vB$dC`p0;P-D7E6kq3C_%9y^b;ch968TRFo=Eoy#(aJGl zE)S08?r_Res@sO&av0k)Tt#bWciM429Z3&5Rjpc;#^guy{fEb*vVLXyj%Hut&h!5c zfuQeI>AYx7k`sO?W}8mER{F!j5(@+g9RmOW+^l^b5F60ZI!6iNP6QF{uP=88#_D+l zfU;Mj^hw<}N^XP1kZ7QgYEoya4?Gslh(Eik3B3rzN?IwJ3{Htpv~w%zE~0Y-Hlyp0 z@sJ0g=PAQK(*{4`*am<-u`g8KG)REfH=-WQ&dlRRCIeA>uCfYM^`Ok`Wm0W@;L`Yd zetv9J!ugx$aod5DjYN1>^rb8$#5s-25?E%$7@ zOwgm3D}sNt*fpTU9hM_Ul@CdF_A?`k=8V@=BU2p=lRW18ezgzp=He&s{GH!w)J4rt z7XChK-vk#_PnPaM^QB|d^knFO+)91bI|8|Iwp13pO?#WDSK<2ugS=-dIx6M5rj}Wr zB?k9$;@Au8s43}I1>27SKTnuCn)Ttd-G5Hfuibx`oF&}@7bDlPxWQ079<@4|6MeH4p5+$h`P!qH4tmNjDvqJS0N+QZX8vq<6UgnEqE(%PW0Y%d2T0+|IJJ#(Ulf ze!tGR4IWIxdHpU97JXR+lhN*DaUzD1U4V37)s||Pu45_?w zLgQx{HM630-A`dsT%OM5hE)L5n}_-?s2O|kXo5cgGCQq+mqbKE?U8yO#I$lF%-eoe zi}_~gA+&;xh3c-v2Xomy6@D1JA1Yf|RsPKYW0A7>)q3%(Wqgo;Q%fkT7ELEh^yRU~ z!*Vz);ZP%Mhq)D3p2%KUa0+`E_^?@LWdVs`o(o27>)Vl~MQ&uU+s?WY$H4FczN)D} zH|X-`Ix=H}F7wU0l<3o$WiC;2yZl3}cO^RF5L&7-9lS@12I082zjQiMJVb* z7R+=b?%nau$yd(h7HMd4s~))KvY%1gXOf^uNges@CsDiX*Vmt``n*?PkRaU@$YfP< zQWqWVopp<;!dJw~m1o-LSOh>VaY7e5p*zTGQ0}~$O(~*leY!`LE23IKgSmIkUDt*> zaDOU@nez+xVY5bD`q8MicL}~TE9g*hGS-{F9}*-{!KN0^IjGpktOv2ElV12WHnVzW zs`;&D$ZM3^(U0yC5;^J$o6J4ghZf_1)s)e!Df3NVkYRg_Tx~H#OK!9b%E)(dOA2bY zOCZK^(iOjKxkB~G-hh7CD=-$|EwV#U+x~NDezlI(3$JgDx(6K+oBlC;#WLJvDteh! z=TeJqel3ch5^#b>uwErbqC{T}dpshCa}W;IqtMpgKmd7$F%o<+65gzg&nQGz9!q3b z?b{{!>r-b5OE{$hg_l1Gq*#;X{#u3c$L6GvK$+^U?Ilk(YoPP&Q~j|}%4QdmZ@t{h z*f(QBWiVQ82O7LBza@KLmQy*RRZPFX;&FfMzOFh@dcaPJ5X@oYa1@~eLMdXxrR+q< zmJstYPCiAT+zpeN+ZT0?D#+e7Vyiu}Z^5N_dn>d=QvKCxEp&-&YZ7q;f6hZtbTY5>RkLkg$=j+S84_mg+m(*l&uyJylv>>Y}pM6 zM)}?iZ_W0K0v{j9->Dv!LKXz&fySG1L?|{tYIx&P)p*z7$osJX9k9Hq;)S$fiA`qA z6U$p=YP3)L%WLnAN6P)McE53r(}gAcMJ^mlqKcaKTXSQ^boqUW&yQ;Il5~#jLx-hKF?&7;%}29=c!vXRRU(_NjkC?*fyXrl-MI zOzX0<$u~$^LqhMuq?0DuK7ORjIEEE6@bI*;7R_uyG6!9Fh@u9+5hXieSo{&%$SHQmS)* z2L)NFPgiT~aC|GLtHT_UA%hd#RaEs|d!-?xjP3SkF12*a(vxTNG=qhy8Dd=i@9l5t zM_Is(!+LKf1xeW_G zUxKSEtR>95*wiy;GIU%znk!z98J>0S3%BnU^oPei)?E_xe{7gckW4=wd6MxcbKui@ z-*stH9!$Ac?Ue<-5ypWy{-op@1ar|^d)I7yO`aV0ypOKx+K_wR}%xpR|A+)%QkjM+b-BiG2-@jVj-0786~@qu@JJ@8~Jq!1a7 ygtE(sll8@g$+k#-vBS4OumJ#;>;QnkzcAJrN#Y_$B6ZoE$ghynoP3DGU;hVKhSDDZ delta 3128 zcmY+Gc{J2}AIE2q?Ha~bjJ;X7g>e}Q*_B=8N;AfFCCl*C$R5$pR%1djW8WJ~VT5LI z(;boQDMZu;vsSY57D^tN*f?cmYR%hy1 zb>q$}hY88Y{Tno2Wr~WLWc#nX5n4OuLVif=cHVv1(Yy3YWGY+RU$^J}*mphRla*=O zGUr*Qaew&Bk`(m$)Fb1<1t`CS99J~WOz~3CfgH_j;Zl(PLPn&D-dT-7IRXuc&6UaH ze%^SV9n%T8eN9ZhWTKt0=peEQ}>6^YfGoM;i|+ zL7(dJaD~ii0i-3UUdNU@-$6gAJeK>{ag!4v*2#fIfla0X3D|o#Q+~WCuSmb5RGz+{ z?1XZSSh)@k`H=uJg0qaEpT|xC%Es-)m#jwa8E+Ol{ha_lMW9oY80UDLeyuS{XC9*E zt>M?$3N#P4Q#<*TfQ#X44qc(e0oq9mNTNMFky4UcGF7|!%-*70RcanHQWabNV8i14 z*7j=yzv>g=e7_5AH%Uoru0tHXuHGqo^iy(mTZ<+(WjU&{KSoBH>`4p3Kf6NLoqSq# zOl|q}Fdbb6#Y~PN5%K)LhORB^lseuZI$BLUOlmU%U$Wumk&*y`K>VC*1R908l}-zS zKv2j3{8tzwo1ee~_}=ujFVxhH=?-ZwG@jP(I~$WEP6G>OU2yNfP|Rqn=K0$)5O@kQ zS>-sA;3rIUi#uKrM6=apn_x+Co!&_WINx2%g&P%7^U*UcE}373L}J3iO)mJ~3anTn z>d75yq^3DyW4gLGo1CzE!QaB{-F?&f>&_QiL-kay!nQ+WG!!BBryJcSHvPw}Afb|PSkR6>;PfMQldye$RP!4aR3 zjZpj*sM{$PH!T%fPS&`c-Q~~?p)e}B4q0wam#=5>(U+rd5>b_6iW0c;8ok(QfD?6DZyllP~^gxU$c{aGPRXI z@;k@0hk2=8t)GI_UFbCumZaa!GOXpGY9|bgU9@CfmJp%>R3}s5jm~l^q$tNwv zL?>`gUx$WG)~beH(2JWK9Embv{iD{5#o3-zqHx**cTGteiQuw0s{|{49+yM%iL#!0 z3Odnj8icb%E^Z|p^_XGWbkBgtQZv;)W z$fHTt%!Hx-K{L!V<|Y*4fO|dEf-y$+ouzc0^b@fGIUav;M`GyCQ7fPpU~Gh?l%8N_ zJD=<^$(#sicY)u}w@WV+*wlX8s97Ef$&-EZNJlpVW(UCQg7%Ba%L)WXWS2N_4Z9nTSsF>i9uPe55@0`jZ>KIXQS_-J?&0 zunE6~;)(B4%dQA#FUZuCj2DZ$Jcaib(3kR1(-NfD6ov zNTC%b$}44@galy{z8dDliV$MPNkK9A3@B&fn^5cQCOWQWa^7M%qJPd4c|JW`yB=aR zJgxBlZPT~B_4(X#vv%R0)4b){Q9%WyG|G4FeZWT{WGT%*bPY%0MnI!DIyl2xy5UV|x85&^z= z)#rQF_E8-tD<1Jyh%k4GsT=50gczM89Pifo{*b@c|}4e-oW z*2f$D5X10bHSfBMNp)_A-NFJteV`@ z*yxN{=eKQudQ1j79R2m+gaLaHegCefEX-$dyPw(Mi5}+Dp%=x5g-S&%U1(WV)j>5Y zstwTfwc*aJxPV<i<8W%Ym~STY`>zyQ4!i#d^lc%zh&mohCjqw2fco756!qM zo(Vp7M)j6O7(+&~hzl5LxNco4hUInmnA$X*7$W6#Y3(7G^NIZeVeFsxPew8|9@5J1 zYS&f{1Z3~kOZ-vURKoPVbv62qpo`v`aJxe;%%$&BHLdDeS(Wx5rf8>yRzVi_@rlzL z?)O}2y*3izWK;Cz1ti6l|5Gn9%1-}7KVnF!yHF*6&o$q3)ByvepFLK#Smsch@VFwc zYBogPF%x_LWSP;|ZOE0!pDlfo83&)f9f9M;I9`-AMsi7LJ3N{8`S65L* zE;t;jv)^{HHAeZSfSuj6L)iGL&4Ef{d(z@+!cmpnCC^SYw-sgchtABWz=xr`e#*Y- z!HG6(hiz6X8LA>>Rb|ubDKg7@tq&mBkKLO4jSim-B)&l z_HUrv9XI2@h3oj+y0{Z|iAHY2Ay-{3p=iV`7e`kSn*NPx{0n?6}vTBLW=&8?! zw{@jP>v~3fypwhV_iMu=SDfrh4D%jvm)kV|9NHA)A58X3xc$fd-RcL*6V}j+?N1;5 zxtWm5Hb(pcGH7S16nvP^*o2D>ifd+-6kA#~q5}0!6HIiTq=X(^EnFh)=iCP822KJo zdkkmzJ+5PaXL#xi(nRL}Qy(V?{^vwRds2;2gn#ypT8X*}W>AkQ>w%F}V`Y@gQOA*B h5a<#Y2qgO7PdGIynwJ`_YzTIyHYvk-g^)+D{td2I(;ff- diff --git a/summary_results.json b/summary_results.json index 421b8ffa8..be922f152 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 74.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2571, + "Performance_Result": 10.2759, "Performance_Units": "Samples/s" }, { @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.82202, + "Performance_Result": 8.75122, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0396, + "Performance_Result": 23.0723, "Performance_Units": "Samples/s" }, { From 91ec9e6acb36bab638a3a360ae9e41963f920e33 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 1 Apr 2025 12:19:12 +0000 Subject: [PATCH 406/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10217 -> 10218 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 360f0511c..8694c8fd1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 12:13:55,362 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 12:13:55,362 submission_checker1.py:3260 INFO] --- -[2025-04-01 12:13:55,362 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 12:19:07,280 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 12:19:07,280 submission_checker1.py:3260 INFO] --- +[2025-04-01 12:19:07,280 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 12:13:55,362 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 12:19:07,280 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 12:13:55,362 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 12:19:07,280 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 12:13:55,362 submission_checker1.py:3276 INFO] --- -[2025-04-01 12:13:55,362 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 12:13:55,362 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 12:13:55,362 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 12:13:55,362 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 12:13:55,362 submission_checker1.py:3297 INFO] --- -[2025-04-01 12:13:55,362 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 12:13:56,348 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 12:19:07,280 submission_checker1.py:3276 INFO] --- +[2025-04-01 12:19:07,280 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 12:19:07,280 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 12:19:07,280 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 12:19:07,280 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 12:19:07,280 submission_checker1.py:3297 INFO] --- +[2025-04-01 12:19:07,280 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 12:19:08,350 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74365 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422216 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418194 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26872 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 0ff3e5b1b..da44b0c59 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1033 +1034 diff --git a/summary.csv b/summary.csv index d32e7d581..b58dae93a 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74365,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422216,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418194,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26872,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index addc51cda..e45b2c97e 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422216, + "Result":0.418194, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 941a75732a1945939e883a2bd2c706f3924d28be..f131a215b15ab7c160747c41a1cccd87f0625650 100644 GIT binary patch delta 2398 zcmZ{mdpHw(8^<^1FlA<>TFPnWG$mHfYZ}WO3VCF0WpbL)iqJzer($o39S9MoNuf)Q z@jg9bJT@ezJUR7Ns8~%kB9do%|9Jm;fB*ij>+}2E_xHZ9>o<37{#bpiD#S9`YGhUo z0D$@f06PF27bLu~SQVqlXzPoLnb$p9n(>?&_340zZVbPy98_%tBkX3m&u6=mD;7!> z=G`ByOs_xG|0CzkZPi0lknIto-p>Uq&WG$_1ej6#kL$27wWmeL4cLNV@79}B&pz^7 z_tp4c5@-v*FC(reRo)_1s$ozzBvcci5ByjPMBY>tQU9gf+aX$b5icb!UT@Y&^JT=m=>?R=2r_ znakVef7_THB#lSTqTHNs@Ul$RvQJlbYeGxQK|Tl=Q;yR1eapP@>XMQv`+VZaapx-d={Z_Fpjr|Bj1bMn@xHj;Uf!i{#4^q^)iI`?gU~6N zI%RP@FZ(r@z^qJBL{1rz_TBwMbWQ5}v>5@p@D1aoMtWkB3qI!jg_?P~!|tC}+1eOo#TXQc&4l-|LV3y}1YDm(P4#-NU(}xM#RdrF^jEsc`o~*V?r0sZ z+A7Jj`n|Z%uV0bM4SQF3OuL-{MjcCM!vE${c*>m!IDkV1$cruv&@4m%Q-2w*0DcM9 zF9Cgb>nM?}9iitVoul^l;DxzHM7KnKUK5Bt?rVR^(7d;-a%eT!GXpaGNL+zK3q!%V zkP!fDqMEw>-;L%2XkCD*fyT9P~AR4dxu#Zi&-et@Zv2C3cl^;#qb$f)I1Rk zvZo4&WM0ENsQ9y(S@8;9I%B_cxIP^>FfFCGqS-Tjai%KMY8GK-2TU8^T)d8!+Xzop3so@VibpV z4+Dp2i~#aor`^r3{o#xP`VeQNzLbSDhI~B^>nc&c7zBSXrR>o=uk9cG)dj`)lmEI1 zp0f8v%Si8osEPR463qAvnTF1JDW1cjAH=Of)z6F2Y|knd**a&&Z@Ya;pn@pOH@!vq zu11r5L?<6*)OraDtcfET6DsnZ0=sBUVh~#c8c+vnL*skG%Q~_K zBIduf4NcJX8eh@IZ|8?}k2t7!$K7}-mdX{I)}ZPhP7x2 zrW~B0CkE3`9S+R8$^hOPSWs*X@3~y27bYXzx|y#{q>WA<{1KYVaI%T0bhW)OAsEEJd1)eEnL`5CGv)bszG_vZzjx-wfcH4VhzE zkpK_tJ#D9W*9I`PTB!}fkCmMtH+il_mIUKRDbyVl(ML=@x9+7>wxd9#&oJ_Ho?}HD zH|L-1q+d!AJ4QTvL5HvsAgmcTNAWxkeY;7XHH_FbFF<1vJ2jxh(52u~0Lv{vPdz|y zbC2eNyar6f!2{N^4313`Lt~r5+%c#2e>6x|r%O6YguMw81@NYqTRhkQPHprlTL5rtf zuI)?uRui;!z0VFU7BBE4?XTM&6 z4c;shcSHsW5a7GZ|G~So>bZ`!F1AphS4HbLKXRi_ze`*UYP_s<69S@WIDFvjG}L4C z9~D?Z-IkNxW>U^L&PE#F165m?8~B3GM2wA$8eSG2yt963@{Tv@o<+g*rS}7+teQD| z>)5`P^4fcvma2VER4wv}FFvH~?_coMF|=j3j$(6KphHb<_cKGfl73$rP0Lj~T!2Md zy`eST7~X$9BQWD#g*$}P&0i|+^ z#;R?#q+0E#ob*Xv#D8`mM}NrIx*HH9dCgnWzajPZseCAD=iOQy<(9pD_Pw9fL3=PD z^;hsYPUy^3*;9AUJo)4uL98B(&`nL?S7M%jL?-B5kLX0Uq0suY=0Dvaoe4Ms7y{(pxrBo+6AMU4nzMkiK&wGC7|9hUZYQJJ%7bYW}6xQ)z zK^P(9Ey?_?Cai*Hd|loNQ`n>gg7cIwEe`=P&dVci-<_5C4z$$bSs@`+m>}e&Tf>ItN-~9>|o~?Z*R(Gk{ypf z?@;mUZ^DZD7^4Pv-6zYYeorNvUE8lFIt2-RvM_5@DWYwtg1&6N0SnsBJwwtv2~cM#p@Gzz^GsuagIJ zhhXQ0qZ^#zmoC5|$tOwo^}6braH-8taevIsEV_(Mg`4hsW3?*lq_T3>Vr%NcS6^-B z=O@M8I)C-N%TD-I12VhfS}+UsA8QB&D(6MQ#h|WuMNbA|vynhAH+E6YLUgBQ<4%c^ zU;mM4h4DMyVFF(SPqXE8WSS?{mV2D{s=5lpcC$?NTO9c7(dcJBZR$GnzxfZWdRp?+ zzJEVQX+|m;rfEDO$Yqdl7HAQGUxC-ZuE<9M*k}J(N^qJm;WPb9FQ?;9MF6Kc_g-UqWS+8@$6Hu zZb<*rGlkf2B_`#zl#*esW(Pqoj)Vh|k=BmM{+SfQ2XbX33ur;L+sBa2|+<{6Qt?sI#Tf ziE-Aa6=hVQR_LllioL0$zw-q!^#qIw`WedkG*M7_`ouZStdzSep7$u1HQ6@Q@_`lo zJWQ7Bgni5gKR4ppSTn&rPpyiIoK2R%7Gk|%D& zr0AB7S+$Z^Ltk%k$-ttr`%NS3`$1sG6HrSw67i?7`bE7ke&N@8nj0y$v|^HJv{BNM zRaU%7)tXW}Ejr>u)`E!cjSO7G5&ghtA-Hwt!A3J`7g0b-9lva>T&Kn%lkBCT@U_csom?cVcSU?H{e9;QHxf4@d~88v4~;I6c#D za_2vZ&`VCmLEYJQaKW|G=ShJv0-L{2>L}XlNW&)PpIQ=|%m1 zJ&>2L%a=J;-|AB}blK}5vu42hmbQ>R_L>T>S)&dN^7Whg@k-1njppz@ag0;ug!sVh z&^OKl++Gw+tmr)Ax9P zxjf&hR@1mWZW1V^)Z7Qad&T!6=sRic?Z&@d8aA?*9m1qI|5g~ef|LEXV@rd3=!K8#Y(U33%vDmluU2Kvs4V^U8h6@+QA?*K zH`0=ky017caKXotv7|tDAJj!0B;rYAOdp~OvL=0SR zq)cUt;b)4@h*jp!qeib9kwfuGv)r>0-?wC{9tn9y3B||RBZ!RttEu!;lw0*n=Nz5} zi0%~MJ7ZU`KyG37WPc24miAoOm2^?ydBx63>NRO77O>;eiF@;}fa=DH$CQ_6&ODK6 zkCZfii%|u=?XUD**0FIPnp>V`n7|&?4z#}XO5Ft?xVN`Ms>+<;c0EADteMPnULE(! zIwvg6JO^%4dX++u$>rv&N+IQzX;JDKDlsSLZ)Xj?+#En%iOogm_TwJf(;5)cCX=(t6Q&{Lyd;-8-nR}1X{jOHex&44o8ZZsa)$OU0cfoLus mgH}CE6$JzU93%h$MZo`7OzaVE9L5~z!R^3kfaFkzJAVVUkX$VQ diff --git a/summary_results.json b/summary_results.json index be922f152..36d873e62 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.422216, + "Performance_Result": 0.418194, "Performance_Units": "Samples/s" }, { From 175b288e70ba59b7eda559ac793c709cd0e4a927 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 1 Apr 2025 13:33:46 +0000 Subject: [PATCH 407/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10218 -> 10220 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8694c8fd1..3f313871e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 12:19:07,280 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 12:19:07,280 submission_checker1.py:3260 INFO] --- -[2025-04-01 12:19:07,280 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 13:33:41,091 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 13:33:41,091 submission_checker1.py:3260 INFO] --- +[2025-04-01 13:33:41,091 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 12:19:07,280 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 13:33:41,091 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 12:19:07,280 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 13:33:41,091 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 12:19:07,280 submission_checker1.py:3276 INFO] --- -[2025-04-01 12:19:07,280 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 12:19:07,280 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 12:19:07,280 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 12:19:07,280 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 12:19:07,280 submission_checker1.py:3297 INFO] --- -[2025-04-01 12:19:07,280 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 12:19:08,350 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 13:33:41,091 submission_checker1.py:3276 INFO] --- +[2025-04-01 13:33:41,091 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 13:33:41,092 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 13:33:41,092 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 13:33:41,092 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 13:33:41,092 submission_checker1.py:3297 INFO] --- +[2025-04-01 13:33:41,092 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 13:33:42,186 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2759 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.75122 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.41954 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410599 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index da44b0c59..95e145d5b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1034 +1035 diff --git a/summary.csv b/summary.csv index b58dae93a..6e62fa3f0 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2759,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.75122,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.41954,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410599,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e45b2c97e..c86f1f57e 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.75122, + "Result":9.41954, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index f131a215b15ab7c160747c41a1cccd87f0625650..7a289bdfbc063fef2f8ffdf39740aad50e59748c 100644 GIT binary patch delta 2290 zcmVyatH<1F#tm6lXnPQe}UH8yL$_L?v{Fu4t~9xPK?_iipSyXu%_x} z%?M__us@y+4{KMK@All95hvYQzdH$M!C~!t5Z8V^`p?0)F#01N1wmq@31;zOZImSQ z{YE41je==6u7~qrmL3^|(X^YSU!!3oo<~8qzYLmA8rn4N#KbvjI+q-2ZQcnlAMR%J_O_8C`nJ7m4)=eNxb~mn2ytP zscXixdv{oC){JlCelp4u8>4Z*e;;Hgx@PniaS~2{TOCsC1glVGuVa;|Hql;PwM<)E z+3)dv60Bnyt6rBQeA7+3M+Z^(&4`vg#Pe?U%cy;=)6>$+TEAq?+1MKEhV%OdBdsGl zd$&i;gT`%^c&)gQA(yh@N;X_?H8j%kYzC$IpfrrZ+#SO?Dq5BpCXv{?f09cgadstF zMB?sBu8G9kNOsC?@^YJyK#6iuA;x*}sN(Wb#nq#V>mtQYr7d1*3*xn?6cwJR^ijp- zqlzn3$RD*R$4;ddt@PN!-&9<~uZcwb?k7YdPS_hF5x~ABlGd)|9gzs0-V=%7=>w67 zE9fJUh*RGt67l<;qC|_cf9n&8IJ=*SM4a6Yk%+VVnMlOhJtGovcE1pb%`Z5cUvM@g zu;c}Y5=UNeC~1m%-w=tY_bri#df%bM6X*W@y@dQh{6KmT*T~0$@b>+&`w8k)Huz+G zqJvK|rh_qZ#`%mAWeP;j9tJ)*`tbYN+jIY8=k(=rF%+k(q2V|ge=WY&d*QS(ouvOx z)0Iy8X&yv_oAGQAMCp1hxJiOIxrrBFr{g$V`ouT4R^4nQQ4lN_S&eKh)%fI}tmC5~ zPM0uoW4)Hy+ho4!&F6c)`C>1e&F-SbEJ+trdu!EAFPsg=!`d&kevov>leqTkpfMoR z^_MD2N*Ap`I<)0~e{#lku^N|JEivjXxG-pnQNDfCg z-ClM-vzOk!tWxa_Qy}NxlQ8OyZfN@BXiHCg)P4X~dRJAEI%k(}!q)OQw-r zGL7VtX(X4r$mzopNG^*Kv^6+=pv5Ph;Kn6{b!eEJFPR7yvASZb&tqWzl&#B3iY&8` z)!S|#t?O>f&6zT_%0Ux-Berz=5REI{K173PmT>zp4WLOhfM$`~ha~_si`+gMf80LM z;*(AYn&9>!VX%^AsbM9nD>nQ5J!C#px6g&%cKew1R@3&Z93!6AIcTDD#Flm+qH(3& zhiDMZ5_TV^0W^sQ&@8h1umpf+k=>k`5`Gz{2eejk`MEB5>RD4S2( zysV_iQnTi^;U|0i@h97A0NhjbdfT#|O0y4|^e|>btexJw4 z{9la)mDE=X%3ZHdOPQ9Jx0JPVphV+{EzLee<4Lm*(IAv1%sxy5P?GV;?GjyN_F zltpGA4Q3x`@kuAjY)LA??L)%=OJ?_hrEJA^pU1FF1{sm+NsnYJsjbuscH8Td-Z^`& zc5{fdbBH7_+L2ZtqVc5Fe}`z0$P!i`rU4?!c!0M*QdM#gTUQ;z%<~f7TnEFD8MJe4hu0 zwcu`^{z6%?HKTtwSiLgRm`CF<8YkaZy=U*NTui!0)3EQZpZ9k5_rUE=E^9(^b0F{VmK>z>% delta 2288 zcmV){1he?CHcFED zexnihM!~cj*TZ=*OOFh~XxdHEuhFm(&!eE*Uj|Jl4Q-lsW7-|hYO4(UQJEa!e_$}~ z1#iOMVj9enRgx%}bdz-W@n}4cYextDae6GP$B2T#VeMG$pZuonTI1+o+4uE0_!e({ zHnLK_hT$LCmyi9!nwd!(pnbO-##vAA<34l%%K4%0hbKBwqe&OvmZD z)HP$;y*sQmYsR;6KN)3-jnTN@e-E+~T{C)%I0>h}tq!Smf>o%p*Rje}n`p1DTBfb7 z?DzOS3Dz-1pX@tzWX{Y;28n!})!Kk=Bu& zz1ySaLE|<{yjEPukW1NcB^$1{8XD<%HiOc9P#VTy?vCLc6)j5)lSu4cf5|10IJ=T7 zB5`*m*F@rNBs=9cdAUtUphUT-5aYafRB`#J;_6Ywb&+DH(iX3@1@T%`iV9Cu`l#aa zQNk5No!Z~jz|Pg?}Dg?{RDL?8+=y<9tSmG6f=M4+9?@efa(C?YaN4bNX_*7>d)?&~TiLe->Zsy>Qx?PSSs; z=}IU4G!LS|&3HBlqIA6$+$2Gq+{BBo({Y?Fed3#2t8O-uCE z`b!lhrHj@e9oq80f1Gh$tj48QON@F;xyoyC>Ksm;gVc#@?KMgqaiyQ21gIktNga_$ z>WD;Kq3@jn>PWcw*ii>p>(SRidB?Se*(oTXA8xqTlZYg;v~A9 zL>EaE7tm{zI0Ca1lmLlDB1t3?Ng|O*65Rq4Nw_4jAiRy?K%!efB8CHrt|XDfhza#G zN|Y%yXYRj;#41WDLE^Q$GbuJ(+G*u1MLmw9hbRh&UZcbj5S^d|P$UwGB9TZGQR0b? z-g^Zol5ly`KT{Ci_5>)WvcV@E$fc-^>0pd9(VO9D}ms-w}qfvcPzDaW4c42vk47q1AldLoIcv*Txv9p*wX1kG_G{|5Djq2G?GiE zkz6v3IoqXrgb#mTn)Sai!abXb{a3ZXc!rG>Hb#EOPs>1b}9d+ed@j2Y*_8 z(g{Hm+&&}>R~af((XevuC)6Q z4We1X?!z>ICeZ+zMRp&S0MIP5`)m$Apb2&#(g|)|LRg1}0h`S41G8quexDy@^GTbR zl@wWO*4#GyWRHJ5GiRejH*?TL=YNPT{XRtFO1}@$AetroK1>5>5)GhP!5 zj|RUFwD_bGf+qNVNEptV%i4B#i!w=H{nnVL=78!n60zk9K@T0--11&!31bhV$)nQRUW*NVau7B9?^B9@` ztFfSx`bt5$>-A|V)AI6`vQ`e1XdJPn*@tL6Y4#x+gtCO$hiL#xG9I~IqKnKvtP_B; z$n2xR>;o-6=|q_=NhP>_Xc%D0>^`uRt=R7K7?#N(BT_x-k!&Tkm0H1WdwtS7XV2Ac z4v}^ak>o`?(&|Grp0xT94Sy0@!s^2`KqMIt5LsmPVVwYxMOGgTRv&2bNhj%FOi3hI zeP|dE$*ev=|=|;ENFZiIW zsSzy*%D@OKm08v3eKE`LQ5GUMlDF1CjkXZbw7IH;W5RlP_11xp3PP4ZCNSY@w2sj+ zjDuK%=AlRqlf+vR&nd#qV*t-?oCjmvFo~`oAlO{pW@H?%Ui!{uFURK0#H*) z!Gs4Mt#2xPTBO6ZJj8iA@lt_&+$R0P+kTOu4`h}Cup#b2`ItOCFC}KEDM@*glk1$& z1uYqUVC3#=H}b6i2-N!_|0W&)L+ Date: Tue, 1 Apr 2025 13:35:15 +0000 Subject: [PATCH 408/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10220 -> 10223 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3f313871e..20e2b23ed 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 13:33:41,091 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 13:33:41,091 submission_checker1.py:3260 INFO] --- -[2025-04-01 13:33:41,091 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 13:35:11,002 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 13:35:11,002 submission_checker1.py:3260 INFO] --- +[2025-04-01 13:35:11,002 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 13:33:41,091 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 13:35:11,002 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 13:33:41,091 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 13:35:11,002 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 13:33:41,091 submission_checker1.py:3276 INFO] --- -[2025-04-01 13:33:41,091 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 13:33:41,092 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 13:33:41,092 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 13:33:41,092 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 13:33:41,092 submission_checker1.py:3297 INFO] --- -[2025-04-01 13:33:41,092 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 13:33:42,186 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 13:35:11,002 submission_checker1.py:3276 INFO] --- +[2025-04-01 13:35:11,002 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 13:35:11,002 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 13:35:11,002 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 13:35:11,002 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 13:35:11,003 submission_checker1.py:3297 INFO] --- +[2025-04-01 13:35:11,003 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 13:35:11,978 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29979 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.3043 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.39691 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2759 | acc: 74.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.81215 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.41954 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 95e145d5b..b633d3d9f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1035 +1036 diff --git a/summary.csv b/summary.csv index 6e62fa3f0..c8435ce8e 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29979,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.3043,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.39691,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2759,acc: 74.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.81215,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.41954,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c86f1f57e..6ab0ee283 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2759, - "Accuracy":"acc: 74.200", + "Result":9.81215, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 7a289bdfbc063fef2f8ffdf39740aad50e59748c..14e5047f2f6da76c30dfc70796eff6134c29ad92 100644 GIT binary patch delta 1629 zcmV-j2BP`wPw!8#%>xRR=+*Uf1ONbp5|exg8I$k>4}W2yKD5##T#^tPB^gcAd(&H* z-D;mA0w$mV%wW^YzaJZtPIG*k8SP7@?L!LY`_2u2_QAuiFIh^SXvq_nj|{tQ8HDBm z3zK{`GX9>QwmgGy;pd^BvYd{LSIUjAwgmZ%->xT;V&O(N&`VdSpb8m%$AIOR}U}b zkfo@m#yzd0x+_UYBfm<;j6I*zWEBf(q^BqWOMiL&OR_|k)-g!tzl@BI^pJ!iR)$HO zgdxpqw?V}P%O0CY_S!&0wcRz>w(K|dVcYInuG3Sm=QX8kj@fvvBRuwnKOUCsnUu8$ zUij)o*#pOtRSMMTy&6`rnd^bsW`M{%R572%ouT=p46hLnkTQQaU`lUqE*;#Zqj|8l zJ%7w}L>I)MBYNA2xsGt(5L;1p@F+VP-v&|Yh(3t&w}|;J#8#ADJj$-dcR-Xn!UIwM z7BSZmhj^4dG|F#1y_VpWA0ZG>?+F3{$sQvRFndoB2$)=_2n5WoGX&ypL(UNhc>5O! z1eE>~fgEl_u5^e4syjs>pt{!x1XTA1fq#JN-XaiC-8%#Vs(X(>J`k^qPdHaM0BV1v zLpq>p69fYCJw_lP-xCA^@;yZ$U^bm?Ajm>=j?lq0y4XR#+&tT?=T~rGsypa{T3tiV z4Z;C)1wnurclRsoXng+f-N|fvadWj_w+USK39CfJ)v_J1%*;~xo5>zPUKO;A7Jo?| z(NcCYbRj4g3tla=gsY~2FP?gB%M>N0b^BqeF2lT@PS*UAa=Dvxa}_WAK(!Su*+n#_ zT0v2@idVtPAYMExvKX@7X6LppGkM<7si2n31_fk4qA5GcC-4$&cWD7qc= z%RUqx9GL13G|@rMjm`mC-y&8)dGGrFL(zRkPJbY}hyIr6EW7KuhaYTp92VJM)>hLv;upWP7oLe(6Eg!GWpnpbI-C$bY%fIbb2V zMXbWM$opK?Rm&_DK|3!eGk=`E&aPGYm;9=Xd9lK|3{%jyb^W9f< zM#~kQ&@|;FV6t?LjknR}@6aWAK2mk;8Ka(dW5wn5_mkb3z zoVp`;vmOeF%%OWFxI9Qmp zp2g9<_i{9LU6*y`V2$|+{#w4i538n9_ow23no4lW9k5_rQ{n3(9Jx9LMUH(Sk{Ey<@6aWAK2mk;8KLA9tZ6kF836<#8^>hRP0ECn4C07Dz2$O&a8k1foDglO* zi6$ZiKb*QFc$2Uu6#)>F#wI}lx|0YeIsq(`LnkI1y}M`^B>?~cPyzq|5dZ)H00000 b00000fPoE@geN`$VUxorBL)*C000002C(Gx delta 1630 zcmV-k2BG=yPwY>y%>xQYC;07e1ONbf5|e=l8I$k>4}XDBA6jV=E=dTDl8mP5z3DB@ zZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>9DDfnCC`i}!gxxHk?FK;(;!7i zqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzKc$knp;B8ux zLOzKp%YOqQZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde5k|5=$nbfR zmfRcN_3%Ot z*@|ju+|xR$J7z>A4ysJd=<_*ASBa2DdWsU#jDOd^MxM&jx~7o_FC()fJ*1IHlwl)D zqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#)moDzo)jZhS z9)D&!q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbKyC6y(;e#lD zi`P~B?;0;+q1Kz~4WZxINn?i~UF)xAd`ABfk(C!D7n0JT5T zAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o)g5#}t*#;G z2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&1DZDuS&w=MSohv zgvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%da`NR>>j=Ra?=LT|{H5 z6_!=2d=-?1$>Pavdu_Y5{`xOMTEywf{9;B#1Zl?2gP|28AGiC@$Vv8dM21%XeGu2I ziOXp|$nCD)l>DCu~I~ zB+IxFQdzKu`8L{|H_V*DyxEKYf~(DZ5v6Kn3$o_qK%~XW;6Iup3U4rZbZuoUjAvCw z4DnjZEj@WD!HtZ>GEuuN*DBVWZ`9XOK434958I zKeM_SVF(0&zzyJ&A0jJ%D^5!hwgYX+sewXWT5=Q8CK0uMkkmSU`zlW2B;ZqztC{_F zc2T+5t2X%s=cUmVB3X_S(A)^Aw-x$mmgyr(d|+Cz)@Z0u4<0S*yOLYVje|F5EVv-S zClRDQ<+eh*5G=*m^BpMWv*^%K+!&`=h^5hQ9;(+Q>aLOI9U|e0|>mnYmWj?Ipi8m6& zQy=w-x6>lWJ`hO^K#y_H%4hNX(kxLu%V+m#K2LMfkb)Kkos;!p0|XQR000O8002J#M6+-sbpi=TC;07e1ONbflkO!~0%Qo20VWlbTqZ98 zfs>3TA{$y3w*Yzs001Zz000yK0000000000004l23zNwvK>@du3nw}ODU(JgCIo-L c4d9b-Cl&z;lZPih0bY~FCnE+BB>(^b0KY%)kpKVy diff --git a/summary_results.json b/summary_results.json index d1255315c..8241bd81c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 74.200", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2759, + "Performance_Result": 9.81215, "Performance_Units": "Samples/s" }, { From 7255a98f60c3c87c090999787f49b53f2ba4ae47 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 1 Apr 2025 13:46:36 +0000 Subject: [PATCH 409/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10223 -> 10220 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 20e2b23ed..12a1f6fdc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 13:35:11,002 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 13:35:11,002 submission_checker1.py:3260 INFO] --- -[2025-04-01 13:35:11,002 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 13:46:31,345 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 13:46:31,345 submission_checker1.py:3260 INFO] --- +[2025-04-01 13:46:31,345 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 13:35:11,002 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 13:46:31,345 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 13:35:11,002 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 13:46:31,345 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 13:35:11,002 submission_checker1.py:3276 INFO] --- -[2025-04-01 13:35:11,002 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 13:35:11,002 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 13:35:11,002 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 13:35:11,002 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 13:35:11,003 submission_checker1.py:3297 INFO] --- -[2025-04-01 13:35:11,003 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 13:35:11,978 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 13:46:31,345 submission_checker1.py:3276 INFO] --- +[2025-04-01 13:46:31,345 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 13:46:31,345 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 13:46:31,345 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 13:46:31,345 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 13:46:31,345 submission_checker1.py:3297 INFO] --- +[2025-04-01 13:46:31,345 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 13:46:32,362 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74365 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418194 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26872 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00696 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b633d3d9f..4fba33e90 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1036 +1037 diff --git a/summary.csv b/summary.csv index c8435ce8e..382678392 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74365,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418194,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26872,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00696,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6ab0ee283..6385b0e14 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26872, + "Result":6.00696, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 14e5047f2f6da76c30dfc70796eff6134c29ad92..67942deaa9e808fea3af2b22ff4df0f0562beacb 100644 GIT binary patch delta 2894 zcmZ{m`8(7N7l#dFiG-1*v5zl<^pKEjrKXxO_U&OTQH01=!k96kGLwBqmTc3=mYuR? z>S1h=HQDzydqhZh^*-;f@A(7nbDwja&vl)3;}zr501nnIzazz23Xa2$aK&SFjQ&(N z08Q6t|2S?4Z@$mlG~@Hg+T6e5F1W&BP@?Nu#Jhs_fMBl`BK=9lWa&>GqQ^(^ zgM~?3;RC+dFv;`1;ELT+b$#+)NEYADjrzAD1B@VLbnM9nw_(HHlAcn&t}N=vEjB|q z_x+cT?hK1{-77e`Z<-=HMQ~JJWA$zrzYYCNwj|3XKcqa|-Eh;6^|;A?;EPo(Xd#Bx zJFQx;X8z7u#mszj9NO8|*cJ4~tCyR7cs3<5loWfwc$tC{q2jAVn{Ax8!u0~qqdFPh z5@8!Y39oZI+OA@!0Ea^lgO-fGmK84d`!1qgd^+gjzDwVS z{zdmim!ezc^PZm3{)|0*czl0+I(;N-UT;PnU1w$28iyY8kr64bfN1!?f?IB%o6wpf>ZHz4W;+u93Kb>k@;H6HM z<4BAwyiu8W54UODzhTOoi%zaCb7)_=uQIVU|ARh`D--Ix8_;t*@pWWA{oPWIP$%1r z(`pjtGO~6CWf$z4jA+B|%hM2*tJ`$!(E zYWkwS*_l*uo-_m{7LQyaBJ5tRr!y}ht9NG($|UQer!qJ$kDU^7{nYY3aaZT^B?t;4 zUHdx<;=N%62k%!83&77U6U+$A zO=n%XnsRJ+?7F6b+BAd^>a3N`^C zq+`@+(`i!7crdFR*(c}=!JK#eCs;1GydEyrnw=N5=oqCkhe>do)mcjU`<&MoFQup* z*)f-AS_6ke%e@M1XG14zs-9AliqSe}U2WbN#7vgkE6a5U8mh)P&OC8@d^IOtSS5Nk z&P_Xh;ot@X>3jlse*MI7YPg8-t0s85oOx#mZ@-k&rp=6NB1RXYNf=R>43gYa7P_nP zGu$!5AiPTL$ol=2BBm3YC$`2Ehjoj51&dg%l#72l(%(*ha zcoBrAffPw>r3miNu_D|}WPnML3-b2GsIkUV!OVv=gS~%v8Z=c6j1mmM7v`*3yW-o6 z+*Mf2Q~kHWDg}9EgwMDd@np>92gUDW(jV;P?IwOHYp@q>Pjsd=`vNI>dMYHgRsbr= zuZGO|3q}E$RF3Hv*(F*N&gm{biIZn&0w*(a%KQAnXFUrZ6O@uI`>f|d%UH+MR8)cc zkV1sP2Q6%&&zBxQAjSDk?+a%d+oxk3#qF^{-=tZk4FMsGVsaJ)+`Lce#>v~_dPyqp z0P|=5Zo*U)Lc>-4$kgo(yy)(+X8=OxaL0#$Y>*?XeKWD*qk zL!^OJJ~ivc4JrH|(FFrCbm4RV5$KQQy-v^7&J;q%>YQQ9rFs6RUd-(jM_RL*Omd!{ zGKuXU0LsL##*t$QJ{ENRSMc#4!4sbND;5#gK}nxaf$cC|FVSUun=?o&QwMqATyOpVA;_HuV29}G1sC+w|;X*?b6dmg9r)6)HG^CwhR~y2d(A8J>pty2O>&@ER zmo+^2N26NvER=5trP=rhH@n3NDX4S#lkf=l$Eayqd?(1#dD?CT*h8Kw|lz6 zw9ODrT#Azm1Nf%WU7BLj31Yy}^YnIoPQ~kUmGN+Tk*ZjFfbHVE2}zgN7868le&HZa z(^EiK2PL!R0WnZ&d22UA6QmHSsH3mxW%R1aH4CgzUvrJ-v#npJRgEngtot!=ZO`DG zj*vbUYoL^*uN0yGHgd4{>+vq$Zu4@R*)$9?RIkYNXpq}r-aM*kCJVuu;M0S_)Z=*Y z-UM0Wkiwdr$t~=ntOBVx-u6sy`M{hTndo<`Td4IE!;$uyl0FK7cY_ebHGKEf%WtA( z*%N<8-;shMDxOS4xHB2jK&p|ybrnHWo7tV7$(_*S99-=Nf8;sAu%Qg$9s3Ih*M988 z?HL^C3SmdqE7H`8?v965z6h7a-t=_em4f~a4I#4iC_3#rL9bBUm^xq#9yF7i;F%Cr z;gCO;k4uhf}9)W$lxbzQ#nxKW*EqS>By6!m_jKjEDe_JRdTDnqopEnxb zp^FGNNmSlf=DvbzmaalSj30{dXEqT`!K)_Y{{ZDGWb(mPJlrgbDP!!3r5tsqNA`5j3 z`6bI+m$K@YSdQ}v97M{Kzy-%(Sjty+BF@Mv69=Qi`S$Fhb%zBJOF4D9o}TpYK2QW> z5=b_mg9%Zck_3v|GJZ;*h*6X}r(ndf@x>S;U(2HnzoCTLik1Ss-+Cm@pJ6 z-JxjX?eifSd}9+YF4`aVKIj*hOH+T;;h5ug8}g2LG{Y?ocWXO?Ug39sQQVMm*gi)U ztk}qmoScgs=L$Z$-W9SP`Rdl0h#Qz+bk!oSA@t8s?&WWe<6>-?wLL!Ck6KlGQN!M*`LBzs9?8Je)|tZ*xon0&-h=Q z0xvXg6;dQ%iUgx;e3dNsKjnvBHOFB2dz<;*g~at~2JI^tO#_qWu>~myj~+$RB~yOOh5@^l^@u(oVO$9&v4qYJfmhf(Uy_r?rn>m@y0)+;%^@ zQp);y*@CUfXV;!rS=Hx_%6QQOx7&9wdVqw2bXz!?`3Z{3IHsIO0FB@5JD)qj?jjSo zr8|^aOk4I$y^uFk?r&!X)$Y=Zlfhh+{tAuS0Ed|759<{lL<`SfuFot;s&e{jvBb#N zpyoi*{MZ$R2P}4Li4n`mJOzQ%(cq2h*k6q!?(ugdhyWY4rP_5a3x{=3GBuxV!oTe5 zO)Es9gk2wC$R&){)I%-_wD%%h>vW+ps$i8QvFxDnrd$Lo;WSwLiOJS0j8EYmqvb|#~Tj?FL z4<#Xl?0Ic364^#v-g}?t{&mm4=XpNoyZp{`&fhj0HZ}J-SVxoqMvJ5<4i{Y9$oRX+ zv(f8W5vklUv^?u46XK7%XaBA#S@G%P@S)Y{=X3G++h=d=e0#uKs?*6Iw!64fq^#A3Yjyi}cIqLmQ#D753PmX{!@s-u2Gex%x7J^Z8}lsLq9=$8tC6V)ZiJ*2RLA+h5zJZ@RQs;}M7J(g=w4 zS78X>`(`QPNY!@{0!RK7KYnJJnUE~EBuPlNorZxn8P6$*lv>;>KVI`)K{A3o$xTSE zW#pQ5jOj{(g0L;T_&x>AyxW25(3ct`2o@jrHczhZE%+*IH5ilQXgD3>HC1$XT~0ePQ6d=h2T(>(aW7@G(=_FUXv z2v!m(rM{Teyoi%;9*LZ-Q+odL*m@|_`rwWzA|Y=R&5Kx$kbi@W*L;f?x?MW2>3qPKaQIx=J8L4{AvpkW9|`AH^^n(v<}lboGP+dc8Bo)`%^4ApTVONq5R&+Q(5 zH-jRdEd5k5&$z7WYyHTK+f-xH6{}yHJ=I)sV~xvJd~uqjxRQ#&giTiEMbaJ(R)S>g zQ06#Jfoih6Kpti?L^p-B6b!lrMY$MMqvkk1T3=^^g78p+85ECzAVbfZL1&2_rw23R zF+ltUZfFw`0g{A$>6by3d?vp=76nmdidpcZ>z`hJMc*mStQGjB`B;=9BOZ|oGBpT| z2N|589wQ(q1YTM@V&iv3f^S zoXQK$zPRqf@U6C>%2cz&FHQ-@yEa|uqb+izn2~wYA-?y+ulIHi+F@%CtUnnWb`~^<$wf6`96=UeG#fgI zSBu%l?lbp;;Lb!mxSQZbVe5f{#Yuj# zmS4nko02Vw0Qw1i>Qa=rZNuJ6exs6etLcj2&w6is;3C#mG9A&}>`=mnV0T@_i zi8u9<^8WhLt2Qgs4+Jk_D$@Q03q8eAfXb)@i!mP@UoI4w?wDRk)7mpAL2n|Zd%qAZ zFOIx$kA?BglE)5BUzF$9J*%X2-3ajsIcqgpq0wijl4RMOhpPxAeG8V#EepDegxyU) zCF0rWP+^9F5B2{DC95Fsl2x3K*-urLYt`24_~IfOvNgQ_%CQk>;)L78TKU1Du%7vE$2u0%QLp5b%${nqNSah#;$rvJP+H9bNw`ohI}&Nylpsiqq>@h~p@& zNQi8ALDA;}cZ&q>fj5IMj8*AgoN3%c08$65WPtl0^pbXMg5GlqO8~322+r@+GTL0G zwbHmf)*M1SYEt=Ev8Qx#@TS`HvqJuGR$|XusMal?t(QMn4mhvt*=~J$JC5BbYZ|)y zm)`bR&_)iAC1tlgYe$3svN&a@jU})%6KyjY2)PVu|&VmHkKH@<# zTRD98FL!VuQS)wWFT|Kcl$%OW?vZCGw|eP*TJW9!pxo&}Zq zZ5Ns4WhI)6EbWr)z!eV*d!CalMvPU>yg5C2h5(s}c{ACRKXujN!0V3R*HLa8aVOWq z=n|{jriA;FjGL%fpukP^PeVAaxp*cQ9DQZDVW_*xe_~QOY2&Cmm=w*siPI96YRXN3 zX`7;#cTa>KL?^Qiz-M>e!3HPY^+8&AEC7Cj`_Za<(S@DT&v{i_H;mf?N4&6NIcjnZ zj6+>7J=>Q4M^iftxUUs8x3Mr_p%6dC=M0WX?#fIQqZq1MT!_hG`wRse)Y5;ng8 z9%3F@%QGOJA@xzulKDDulfDMJ<+bDf9FTF%M~utoA7C#UDcibc+zznt*^MhkcVY{= zYb@C=5>}9DUUKD?U(R4BR!LJ@kyY4`r^Z5mY6p)6c3(Pi<2tE6vv<6l})8&oJ!p;g; z$!r6Pnw&7w*QH0x%TCuYPWJcWuG%w3sX{P4`p{9EBaTd)eWBw>n^L;ipJ;NpxY9|$ zVRz=tR4hh>$qKMILR0~JQvD1j|H#Pkm0A%_!-1wem}5Gx2$r64e>rG5`j(78dQ zz9w50%CLn@b^$4{`7qtgYM8pQD_=<_#@@w0Eo&W#< diff --git a/summary_results.json b/summary_results.json index 8241bd81c..47883bdd8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.26872, + "Performance_Result": 6.00696, "Performance_Units": "Samples/s" }, { From a86afbf090a211fa3f163e71a5f5a3104cbace3c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 1 Apr 2025 17:04:55 +0000 Subject: [PATCH 410/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10220 -> 10217 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 12a1f6fdc..5bcc05344 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 13:46:31,345 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 13:46:31,345 submission_checker1.py:3260 INFO] --- -[2025-04-01 13:46:31,345 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-01 17:04:49,985 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-01 17:04:49,985 submission_checker1.py:3260 INFO] --- +[2025-04-01 17:04:49,985 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 13:46:31,345 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-01 17:04:49,985 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 13:46:31,345 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-01 17:04:49,985 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 13:46:31,345 submission_checker1.py:3276 INFO] --- -[2025-04-01 13:46:31,345 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 13:46:31,345 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 13:46:31,345 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 13:46:31,345 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 13:46:31,345 submission_checker1.py:3297 INFO] --- -[2025-04-01 13:46:31,345 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 13:46:32,362 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-01 17:04:49,985 submission_checker1.py:3276 INFO] --- +[2025-04-01 17:04:49,985 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-01 17:04:49,985 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-01 17:04:49,985 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-01 17:04:49,985 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-01 17:04:49,985 submission_checker1.py:3297 INFO] --- +[2025-04-01 17:04:49,985 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-01 17:04:51,177 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,20 +24,20 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.392716 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74365 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74194 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418194 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00696 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.29979 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.3043 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.39691 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.81215 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1354 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.41954 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410599 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.404029 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4fba33e90..d7f2bc9d5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1037 +1038 diff --git a/summary.csv b/summary.csv index 382678392..61464101f 100644 --- a/summary.csv +++ b/summary.csv @@ -2,20 +2,20 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.392716,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74365,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74194,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418194,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00696,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.29979,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.3043,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.39691,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.81215,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1354,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.41954,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410599,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.404029,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6385b0e14..854b0e270 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74365, + "Result":1.74194, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.29979, + "Result":7.80393, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.81215, - "Accuracy":"acc: 75.000", + "Result":10.1354, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410599, + "Result":0.404029, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 67942deaa9e808fea3af2b22ff4df0f0562beacb..087a9f73dde59bfa4cc19c4a2304e1c3e1afe1dc 100644 GIT binary patch delta 4649 zcmX|_2T;>X*M}22C?o^~0f9hh($#>JD+Gxcs(^sBOOq}|5Kt-pX#$~>ND&Y*6e&_9 z6cIy_9zfcqNt50Q9VyC-@B7_vXLrud?CjZje&_7Wo?XAUa<9CHnn}2%gZD8F2;>jL zhcIyAx2PdNh(5o$3rt#*Ar2Rtzd~A(>hY+$#|mca139`4b~W(x;PjpZrgLp*=LKmt z22R(`^r}4jxi8!sEBf`BR!q`LrTZZK4koA4G_Jd0>Alo6@)JX}SckKk#-hZ`;>fmj zvlF>~Hymn1(}i4U9IHkJs$=UpE@k2hB*#eQ{?X;w7Q}AGjkLdx^X2%4OV2br zZvao-qil{)*I`VsPO7vBvYY10)BUa6EMw8GE)#=FMaqt7A(kSY?&V!Y%?H>-4gYA^ zKrArQppiu$_?(|aFM}W=ZsjInUfCivbE4R*#>;0z7ZZ9APVU+7l%~?xM!)f#lyCUB z_ihx9?4~ElwYz>9X62(7ybR$Ak~iH5Ne%Hg_|{nEkbGOtzbwMoZP?$YnoHK6uPE)_ z+_ye|!-$b6dd%wvp7Pw9o2k;914Dm}j`IPIN`wn`eXJz+7R$TOJflj)l%KugjiUsM z`J(yJM;0xCoOTjqX+DPsb9Hbn-kC5lASP`sO8qY~ANN8{mX-DvIRVO0a7YspY@fi& zkVX<^pwTJ4PebK6!bP=REwIEu^X1nC8Y(RSi(;t}c%almJM=~nsz3RNhN>ta3KiC$oWi>%GcAhkS;LD;oI9ln*M^w7ij3=-$Y1TZ=c`H8ljfA)F^L>rpRYHhv9hYA zd+pjdw{|T0opC94zKHMWTWx^MrptLL_C!ov1odToALZrLATgysaL$CVOrAH?74>&Mgzil3pG6W*Vl#iEd?3H1 z`p5J}4p~%3R|745>r0jT;8#`>SxlYJ!%)N$f*KNl7AVnB`7Pc6B3GeG9S3&wzXl?U z!l>_0s&V>hB|ZUio;2R}jxl}do^)1QRGeZgP?U0-Xd+@2NO=hMJBtCSB1o#x9}|Em zs@+5RjTL7{um7Rc8@6geHk`u*L!qQ;b0ClVHy+1tHdBEyXQ9OjM}k^IWDfhD&JavY zaua~O%RXhZv;3WGdbzhK_VR4itG9ded9Ns#1y}BV?Fk@L-LL01>L0whWwmx+qg)Fa={U>MIP##2L zMzv+J>63Z`>kH@*8I)+5qF)!(TM`w7haw?fJ$GRnuZ+&0x_Hq zRgkiCBlrv-!0^MBSeP+Epf7P*v!I4z1n#Bxx^dOaT zFpxy52CYy;1Q{kjmWSVfXkb&%q z$ja7S|1-Xkc$VY2JyVkeR@$)U?;pRb9_fx(we>euP}aP*q^6GD?DGPdn^h}h_)j9` zZs6rnCmxEi#Exb9uuVH$X5_RA&D@Nb=Dm#MmwR6|znqSw#ScCyeAHGM@YM9taIELt zi|b?R7<3#JEH@EWes6ze;bJxnf?@3;E9O2@)tjItdB`|yyv*eVqf zIH>#iF1^9eo~V>tExR5ET?IiUgM$NI<_-2eW24+_5xXaJh!TWyK+WBrur~LUXCjZDiS8hDD!lilPPeMtKBT8|#H}!=>}QNeo$2Y}g+w)Jahi%ZQ|?`_m$@?UOeI zbPN$#W5X)cy^TU=PO>CL5l>!YTE=c6U|xO=P;8HkLWcEYd7<)o+DO=4tH)w4(f-p= zY{8*aQk4JnyB4;k-@i!Vr)sAtf9Oa@d+64|ua75;t48ua?E?v6_paxMUGr``iiJ5a zp}+CbsVE$8os5k?IeWOYx8Qw^{eAJf83ists(v_Nrt;dG+%vtwWW8e-K6F^dwi^_^ z;xF|{GgP4#%W3_7PQl{1{5`Eh7pAJ^DufYfRQ^mEkI7`%&Di;92 zyUq6_cdEJ6wNw;Cjq~vEoh;1=AiW$Yfr*(IXaS7uWtmd!86J~QZpGl2ya%p&v|hlU|U{^=Dn$nVRnJ>!EFi{;T^M>fCZ zj=H(LPbRYOD@5}V#NhsMu)8RqP#EgmRWw@A0F4z@@~2$N#(Z7S1@5Um8)wVTB-6e1 zfu|Dw#EksvkpUs;u)xWx>XU9ANp>p3CzWf{^2Di3yI8D@CUj|vBh>1%^!|rThr;WM zp5~ohsMrR%JFe8$L6U8`$ojqARY{EI-`Ve#!Pt;5! z!#kbg-kP}{VU}gxii4Lz=9?|kW=0{Q&;8XjydFO(UiDDo{Vn?{*~m1M+_m3mVyfgj z;^v{tUT3;Pe8Efw29j&i7wV4MnQnGUJ-}hP+wt}Ux=tZ(INmu}r{W{TzB;-?h%vZ3 zlxCx+iT=7n%snLmdvZAoW3wVYw@s0_4}csm$)%*Hso^%Dhq%gI%I3mC`_aVrw$Vr4c)>| z6>!F{9x!QY!wY$niJ}DWQ$?l{{%YhhG){t72_2@Bvb8MBEURfLJHU}2?WXjBZDsJP zR-eg_e?tDq(w!d;n@~btED#c0DO0|0tmJ1X&S^bZ`}kvCuM)CmkYbokShOUizJiKh~N(h^t8*^|dLvi7P6e0-+d^D@+h zk5O(e3j*8^{GUhB6-v%5+Hz;(6kLPJg6a@pjix}dVoz-4lj2K`w{frnnk5$O*CC1R z%^#~(cDvuKAqzLE_AJF#R<4mO#YtDh`=#x)A3*v)4GgOzF%FCmA%mb~^c+S!=F2Tf zu)f*+rhyIRj}W)7(AsPZ*e;9N@bN!pH*(uilLYGXe!S)mk{+ra6eT&yyUQ%q8v+@b z+Hv`;7D_=;ZOMB41hDjL4ew4QPCGxza#ip>L>pc!sns|9WABg8P@Yui@H<=YdV4|B zmeuGIm4uL^`_<>f72UT@JnMogOY5X=I+6bor`O$!8y(nCN0$1WK$!_DWE<{?-S?8C#XNU1UQF|M}{|PrQ}5*0;@8S>YbeltgH$R4j!+o7E zmv^43$UZVzKRhy!e_Q$7OfxCZvB$mn-oelI8@n8pJy($Zigwx!5a3)%iaIjNk+I=i z$#gWeGsjVoDHV#?LCI;NP)n(-gl99p;D<#JMTjSjAIv^ZE>Q$_Umpia?^RDbDLyjq z`9kY1an2h*I64MzK#vL~9gQF0^>PX5vW7SCKI z3v%lX^*8;4%pJS_d9H0HL9c~8cek#N%tD^aciZgidNUFv%#|=9-G9fCf0z+vW`EFc zbMU`>PsxQ%s30$v+xzjGl163+QU@izH61()R0tG-Vm2uZ$@I2BQZM! zUYC6V?tA7zQFK^I+kL>68;*;mkCLo7F4qh05G+ohD~445%w=;h$6?Bk`()ubv!LLP zCHwtO(SZP1Qt!}mj=Nf}2RVS;Hhcl{UJZFsFSJug-rv^$vq@6M@Q?+(cj)*2>PswS zr?TPkFu#-Mvt>P#vZp$2#CNP`wf-p)qAlS%MS4*^gXruw-$)#&cH7U{$|NDnGQ|TYu1S`8e6jIX7=w zdvG;{WY0fZ9?!BkX}eX(!`x0BO`{2`)F*EK6YIPWBQGooWy+}L0fY4snlL>|LtLnb zYE*^iihC3M;zj(4%-GejFWtM#4a6|EG%)R!8rQ&cBK%Sg74ZGDgH}Gwl+JMYRpVT& z-(?g1op>s9&eOLnre7iigDPd?2}LjDs!IOxPCu?;|5VbLSLyorK}Z0TrP2;dyJ04` zW&cPyq0F|hOzzWd8I$E!CX(qpPKiCcmsW&6ZFYa`Z5e}QVVo7iNS{ZruJYi3(5OIH zwpiA&RhGlJAreSYpe@<%``%=P<8!~+4$WLj7uW%*T~G9q$2If zl+xo8`v%oJ0;nb?^XsM_<(r*lNd)6l)#_Vja=h1kPiuFk&_C zkG*t+U6^WW%SmT=I=_^J7=s>pin^xw#%iB1v-#?1^WNI8dbcJQz^R*zMYM5+4^Zd< z-S)v7KH#NU5pa(uCCT9!*GQVjRjEFORIc&S9}x17tt>(vWO|wT<(RE2?iO^iylhRC zDq7P4dL2bOc6`TERi0Y6-6F7!k7o$+PI_H=>N$bT%=_L6H-XzMJduyQ@T?dtH9o^*>TO=;` z8BB1$H;u|G20l8RW()<3hEF`C4PoRsFK7yRvCC2Hf#~o2BNstU*O1vL`h;D{b|+8J z`_rBXwc5wxTuK*imWX=!P)Jwe+{Ya^e9GHS6{kK}O-CBvFFd*Lg)$aUSYR=gEfwE# zV1Ac55bM9&&uwrB UJw9Ai1#E$D7Ddo83;$XA4;FXX1^@s6 delta 4673 zcmX|F2T&7A*G(XFkPrl zyL3>ID#ah)%=h1!ox3x;b9VQfduR6E9ld?Mnju2+ciWREULXJ<0)j=7GGc!aQlo6& z!#<8dEt9}ZJd8Tyf#Vh#i?4IJ2zc{=gEr#>9qi0Wx}HH9j0f- z;u@PBF)?w+2@zbTk(yZ%oI{RH$~{B_Y?+Tw=af_$7Nw>e_hn9P=MT-R#*M1Cq>3El zX8kI;6;9tgWJ}pkPi`QwKHpWFCH#_PLO!wG&SYAfz85TIl=nh$q0a7*|@=CfqG!lG|LiF z&|uX+pfwkjVH#4uO9oO<@(m=E#YLsc3q$xpUX3HKb($1pd`29K3fW!a0_!izg9ttZ z#w)?V)zNygW-@@+s6rLFyw|iKf+IzB4vyvjCTbiu(%H@&eUH52Mmr^2XOue@6y__o z%-n1~$Zv<4Td53FbpbtzltUPO&xH07K{wwQzd8lDdp104BC|XtD_8BFtYfVi95lSh z{K}?eP4_W|9;L~KH}sM4;^b$buokJ0C@&5X%?WTGxXV-IeQEpQKJHVF?$lM1M@y!s zC!Oe(QUjgHteRx8xw1jSTeHr74r_%6&P~n@Q*D2^Z2XfeVno$7-sBc8$llAKscTt8 zsq5C|7D#;jy-eahl+nEZO2x(3xn%sz4S(W}Kk>$gI-vY+-|6=5F^9evDd_E;P$fJk zn7{Sr4xZDCzm@fP0Cty^C2jYPBwl$qb{P0{nafn!|NHMhDtM&2E{dZSeUq{++_u+( zW_R*ttCG_*V?}`DyfjFJKi(2%f>z4g1(O=H5&-3Yr8lW~ld{?c)`kb~C{K2VmL29% z$n*4}TJR5SI9t>zx*Gc~EhkL|1Q|@=!5TueqB)nffVMwsA-Rc5?4)jo?X)Yl^YyH4 z_Rcp>9IC$-&AinI=(0y8zd>#4ykcXU@S7X9uc$YN%q~Qme^-p)R4d(s=dw64&}TF^ z1rI9RNU)E;81{xCOpB5+(~p7|##0}C7J1izQtnDEeUQqbmR=)4QyuH;t!r3xBl}`C znAP|l-$TLH4u<`f@b-Td&mUzGN^sQ?GnWolof@*UaM;G8-*n_p2e)W>=o&J8Rjq8a zZ0O%CEci@cQCe*`|Jm+Qjgu#X(Zj-`R_t;M&5P%r1zJZwA(A&_`fx(fT>t<;dOIEx z0ggTX2xEOotx!si2aD}*V7~BnEs~BU9ll;?aL>A%&l4l>`d6TO>lyZuNOVK_fN!A7 z5C(f#v78-IzfyCmh;i&-yWU!{xOYtxAI2d*2C6@=k=9DO2ri&GQ}6u9JdN`ggU3^L zA?9?(s_`{6BL#W$&xm#SsGKu=Ud*wMye_A_G)QMzMcIfQ61jFQJ!cwBGD+f3^TK+a z|3oOnJ3c14b`KCP|AGnYv^{uokb6~pV7|U>GkbO-drt3)l&&-U9J$Iqn;L(O%ShK` z&OVFil3bnp*)c&&5G9mS1u3f%D%CWb}{UPJa zs91dSGK#v6*6N>tXj!t7j>Q6Y-|29moFziPwhDDysUgyx9Q{*>y5iWh9^!XX1C85Y$~ry!#|l?_9eh) z-FTR*nwXe&{G`Ljg7lesZ4TpBS?EOIlvs^@E7Vt>MA^0^Aue}co@v!JXQ=!)Y5x`- zjXc$sQtIJmX0Acrg3Lh@FnlK;<9O%22kSxVQ4&O4 z_SA&zEXecg{ezZ2wVuB{TJF6apB#0|ib2so+iQ25OVb@gmF#HoT}x8ucMkbUljeK1 z(=R}kvlhCetH<55M38Hl7UeF|N&KWK(G}{ADX)W$^rn6qfJ@+M&9&A8`_2-}$D6;} zmx5{;hF|&OpQjW?m$f(S7BdVJuiGA^nm^QNUDvb>^d7yJu=5*E$5RwN#lbpv)Pc%! zx2otZGKI$kz+3tq-FI1Zv%@E%91os1WoCHYXg5oZz(uhW#eII>^VBwB79h`ZZ& zaR!YED9E(1YTDbhc75+v4ti^nY=|Q05wfS~DYiJX99e_5W~nl0-enkdOqM06r%eV# zsd8yTA(~u?8jwW-h-NeNzk`+XEtI$-38Si@iQO2olYg|51tjecJJ8*NS^~;z4Q>Aj zNL$P~d=pkvjLS#!lVhw4x@o}Kc_4i#BFsjNz5+X}C>}b%FuQ!or8vOkMOt%9lKJDS1a8iZyg8 zx>kWX*kAhkKkEaj2!N=`K~C)VO?IILXp^NT?`sylq$nmR77?PI50v=Y3rm<4JM$s(|why($g9F%fPJBev)aOw{*&1B-q>tUyZ;q;t?Z#wA@Hfi6u5DZ{# z@_VIt(n^)ZqKN;6G686%NQ+HCyqW+TdMLv64lG2#B4~`v358TLMyml+2k(F^%||le zwKPZT8m0!;0#K3ALLcMd98CYuE&h+eGh=xwl2u;YU@D2HT?IJFf1&Q38Ww4Sbvre> zg?c6Y$`+wUB<4Y5Q4*ODEw{Z z)@-u@ST$)j9xLgR(Y1PmejPYwH-SG4O0$~%vL`N^75e9G_J9_-u-doAM85V)N!Dy% zi=9pVC>Yqn+#l2X&YCS>SrFdrpGI5?jbqBI>l;PkT) zZYNrwbw)J`W$P1gB$5(UVt$zz(&`wx+v0&l~YdJnxvOywmjuR(wA3?b}*x+kvB^|muV-a zY}~p51>xXWrD8T9;8c7 zl5)l_m_1EO==Ac6KT~h+Q$=B)Vf|*~DqrB$db&Tw+^aho+bp$3%V^ZC&}W=Xs`U}e zXa|5!K2S3H3MLH9d%AIkUpWh<&-HpV3hJOH!#%~$B5kHBW)6RyC0uAX;lNlG2vJdzD3!Dy%Yflo21var!mXvVo( zFJNoGWp0RbY?w3Y6XLNs64Q${WzSdUf^i8^hpv-kgjQN5LY`N&4pzwY>r-axA;?ulT-T-Hzrlw3N%goc>zOli6dQ$ zgH=1xk>$a&YU{Lf4!uhX`)l{pNc5rINx&C71!kt7x#bLx=Vqh1hlHQHx_wRqsiTgN zY%Eh@ZT`P0UiM7I+$`Fh^lQ9ei^UMd`Mzp&5?{>*bms$8}DAnG!%@QkdZY$nRAv#u5OoKvdPj?7oD*E0sShwrq z$mqAsfnyqU_4sC zLC-eZu{6F4KXL_Yu|G;7)F`R;)cq#CSbEy!LcQOwZD*#xl5h*4QjxE|A=3^B-BDLp zTBz8=^P=Fpz3aWl=RJS@+pBJm?>6fOIuY_mzxooLFjq~qQUh|mMc0v0a>Ye)Ilt#d zPcqu0i~GB2)+rzDoP~E3ukSFnhIqPTLpGbv`XrE=E7l%g=W^Rf=Dgzx0}YIj)gqmB zsauip8Ihv1+!J=_uC)RBN$+BmeLbz+Me6}Pw@q}N4Ig!Ippz7LC>z%%b4aszsx3}! ztzDIxiggT?9eaGV?ICAzUtQ)`0I^s1#*Y8zd6~=LBj4Gs{t={7b zEF%{?YUnji3>@(tNnX$3cQS+46cBZD?41C<%4&WffU%Odf8TA>$u*}-|IiKRER(9%q>rC zvYj;E#g)nC6+^N-NQCcA8=tpg(0ggr<$i0ipr6h0zq{uh5?`=mpu~A!n(e<81O-_0 zh#DAe^dK`v#ua->kuI^C&X^ zEt59L0RXTu0C4ZNa{8ZzggRiAc-4U6*hpS3=KpfI+@iPC_ga-%!=o0_{H2>Rp>qA1xv7vl0pf&biK5k+#&)=p01Ke82KL7v# diff --git a/summary_results.json b/summary_results.json index 47883bdd8..b8d58e6ef 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.81215, + "Performance_Result": 10.1354, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.29979, + "Performance_Result": 7.80393, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74365, + "Performance_Result": 1.74194, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.410599, + "Performance_Result": 0.404029, "Performance_Units": "Samples/s" }, { From c1b159c8ceb04c3eeeefe670eccc5c84408e1ae2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 2 Apr 2025 09:00:16 +0000 Subject: [PATCH 411/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10217 -> 10217 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5bcc05344..a63b5af6e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-01 17:04:49,985 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-01 17:04:49,985 submission_checker1.py:3260 INFO] --- -[2025-04-01 17:04:49,985 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-02 09:00:11,721 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-02 09:00:11,721 submission_checker1.py:3260 INFO] --- +[2025-04-02 09:00:11,721 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-01 17:04:49,985 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-02 09:00:11,721 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-01 17:04:49,985 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-02 09:00:11,721 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-01 17:04:49,985 submission_checker1.py:3276 INFO] --- -[2025-04-01 17:04:49,985 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-01 17:04:49,985 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-01 17:04:49,985 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-01 17:04:49,985 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-01 17:04:49,985 submission_checker1.py:3297 INFO] --- -[2025-04-01 17:04:49,985 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-01 17:04:51,177 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-02 09:00:11,721 submission_checker1.py:3276 INFO] --- +[2025-04-02 09:00:11,721 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-02 09:00:11,721 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-02 09:00:11,721 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-02 09:00:11,721 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-02 09:00:11,721 submission_checker1.py:3297 INFO] --- +[2025-04-02 09:00:11,721 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-02 09:00:12,677 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -43,5 +43,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0723 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.340923 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 38.9895 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d7f2bc9d5..89a413724 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1038 +1039 diff --git a/summary.csv b/summary.csv index 61464101f..997a4604b 100644 --- a/summary.csv +++ b/summary.csv @@ -21,5 +21,5 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0723,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.340923,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,38.9895,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 854b0e270..19bcb8701 100644 --- a/summary.json +++ b/summary.json @@ -669,7 +669,7 @@ "Model":"gptj-99", "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":38.9895, + "Result":48.6205, "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", diff --git a/summary.xlsx b/summary.xlsx index 087a9f73dde59bfa4cc19c4a2304e1c3e1afe1dc..c0f1f2380c8bcadb21fa5ce57830d8bc558ba870 100644 GIT binary patch delta 3060 zcmVOM*@st zuO(UW$&5U6UNX>Hdv|Z4&)rh5(ZR2G(}{5#MDaMB9oAIctQo>vt#NEI6!v58~RdNB=qa7Dj)>qaa9(G{G!Btc{XnzTarXy-_gj#`SQ29?a4s zgD{$Qlk{seY{c^@==PUE(@8^{rqh^q$Ftfh!+umIM>rUad%>Hqx0nXAWR)ZeCfy_* zemok_M^2Va9BH5`zOC?r`9++SoVEA4!*@(pN*`PuVMH{_T^*$ux4h` z258?chjA7e{b0~tOp^2P+lOF(JRBwIX|uDCUO0)D{~FVAdMvLUonRHJoOP@+)h61nE6a4Wo&6r)C&4!Z+Qd zdvp+m-;8M4Lp<+hzl_?~Iz27Dto2LQoQ7$CvM-^A7kUwfsj-5(vTIsQk zzp1!}UlWP=-A{-_oUk_rL?VEFOC*!A3KoAN67l<;qC|_c+a?lmc0Un`IJ+Gp5oh-^ zk%+Ti`qu5{8*^B@}BjAw%&O4n<_O%lY(O}zLz9mmt-W~f?&DGYGiAv#;5klIz9^GbO{qT)@zx)P3D{4e7@J4FZROO>@Hf&l5{b(w^rTs z!r5Rvto>5!2T6B4iEFP88Ur$2f2n_>q;%05q(fW&Cudw2t8uB-5~FS@Px%(7&gIm( zNS(OWUZcbnSNaJ`fI1?P)DelKj!486`ra*|j)aRp-X9CXTNv((@YC&ZahBQ`Zi(OX zlPi%VF`~=QC{d=+oVgVuIUL<|d)fWWUV8hoO0_pkft-I&!l*a8xz%;2Ztj2e7D`(s zP^_Yq5-47|TPRvy-O;AaN%T009+D_7pw}pI1!gBG0TPKsl1L5T9jzy zJ?fcXEpF3pFO2sl-DQS5&-vNw(kjX;f#A8ng`l6VOMN?MS*lEq;IdsXDA0IKG_GLd zglK?5rjZyjjpUGMe39qh){D=gi*XEwJcid`)E)AnP9 zPuFc*Vu>2pkyw(a^nfMN0G31pSfU2z$Bl+!NfH2-1qoV8?kAi=OPiCa4J&_pu9I{3 zsQ8xCvBF&(+mU3V#&so`s6jG`2FN5DAQLr6rY<6rBmgoC60|l4AIOv~ZPE!LQ-*a& z7(7j7spSUA*;WDnUNEbuvCIS3^}sealXlZ~R1TSZcl21{rlajjGRelIC&@$&l1Vf` zCeZ+ysKKRdvxrQR0LUyz(As|-d>~V{v`Ht*)RIgY)}dj5CK2JB+5AVLsh+f&t)j+K zb7pf3P2H?FO)GCT%XjmS6>gH-o+J}Bt}n?%4U$PTKqk=unPg)h$h3;cBng1bf&{J2 z!3Q#BOPh2;$dq9n5{7doNsV)+^?v|n6*ZQcGp*e@(|7#5DKp1>qR>}OVEYnG)Oe-@ z6Ez4X(Eyl417M;C!L*COBnbe_f&{J2!3Qv9OPh28TOSbBVNt*(vjYK__Os?vY=rzc zl2w#f3Q1casa=(`C!%uvISQ>Gjx^{HjVF^S4;z0{;M9SZHt9r}O$jBqb!Zq+Nkr0L zcc!Xz?4(pKzNT~GVvVIhGP#{Dr^{@fRDNAQ0m^4;-Z@v>?j2Qktz7et+Hvog4@{!> z!;$tKqVc4EhiH(=5)K}w0VauyMmh&nk%@RjZ$t}LR5!D5hRHS z?<{{ScI@mR`B&^*R#IRoDz}Y1>20Oy>b%=ZFNaF>fjH92Lo}ZB@(>MDS;Ec3G(aVB z)Bu%5h91@lP+4T@QPOd93N39mN}v*)J0uEMuf)59tJjLnJAXZt&(zX$p`@qB^mD27 zbErhSh$CG+MB_#~v}%PfCn z^|sqb>$+!oIa8)qIcTD9#F1_vqVc5LhiDMZ5^f)+0W^sQ&@6KMumpf+k=sXu+Xq_O zq!WTBxP3?%tYlegSjp;&%|3q*na|YibD_80K4#r&I=-Frf9GnQgC;sh9BKC<8c*7N zhz8LtVfSGgK$B9Z~J|gv!>n5ThLlLXrh02#F2&{ zqVc5RhiDMZ5{4h90W^sQ&@3|iumpf+k>N*!;RjmUq!aKJKvaiC0hwj|KDuJR&tqi% zuf~E(>MI51uGhyhEmP+$W$hd&(KzBrvk%eu((FSt2xSSg57PjYWIS@aL>HNTSSJ8w zk=aLs*#}zMq!VRYl1gy<&@g|%lG%M=DO<7K=P@jkK}Mu{(j(bQYAdyZ-S+yVch0`2 zy&NK)93sh!cBIvZXnbk)AsQsIgw=;>fJibPAhO8n!#V*Xi>y8xtUl1vCY>nLmPCTp zhlT->%<2O~J}0YBK9T;Dh?K6|z=m~nFz*h7Pu*xZp2fx_7^H=ob+=|jtCvtNKPTaQ`6>OI z_BBkBaC-l36mwFANB;%0e-oAt1pX7z`jZ(VAAikG!!Qtp z?>t51dtxUA1XhwtfO?`55=B(pEPI<;#6PkPY2O|vY13BfiQ{@^znxvVobObd{DSk+ z=nCZ-K?!JXgw#!iK30qL0VO^#Em&(bRA>(#&Fh1&{5nNr&x%kYk#n1{F^m^3qtO(g1`g|4EMm)PDFG-3O*t2k81~j;22s^ zpo7PGmgAE*fK&cz!=Riwl?;0eSFJ~Ja__vHj9u4d-E6SN{5f7PU*CsSQ>pt?aX@t` zIOPslFs?4~c@YoSG9Q-l#2X3XX&?0~Z-+(3J`hO^K*zX8<#YD@vL#xex*)|}N^Vn9 ztjHrJgyzLpKk}mg1XPD1|0W&*WA4jOQ2zn5TO-N>1ZBS-h_fjsJply%6VdvUYA1ho C8OOf> delta 3107 zcmYk9`#%%>`^PDVR+vUA#b)NbIS#`oQL7xMQF6LP&8eK`R5>(_Fd>GCNt$!ch7RsI z8zy(oXGsooJ|A+p@4noh$M^C0`~$D+dOok$`-k_u*}Peq?y6%#WG ziEf=-D$8AP?mXqpD7%`$4}5$tWp!sP?X1kWb)62trNMDINO17^mHlrGAC>nd#kut# zWCNVn9X*m@n}6SwT!|&RX_iD1HXSJ97sI3^)_OT&ncAeWhyIvM?z z7k9E$wHg~7#?E{XXS5*SNEO1c%Jb6mYg64%#RuaA_NTV2EMK4H4r>}IAGx|`I$Fi- zZm3vY#IGT~@4Mr2g87@SRw#iFBGuIrHS`WVwPNUPbG%_I56J0}6S`c4sYzKwGivh7 ztwzH?4+r1G_4*b(W>yA0v3fiZ<2ChMl?A$#g|7qSWWG3WiQqvPf~}bOB3I#Ur_zVX z1sdJl_{u zU5wZ~;E^nr)Cu}{t22yA@Sr>`{uD2fQZZ?yVdtfe>{)l`$=S<9#x#tHHEGG)P%cO( z4Aqd0q4)r)d+47lMkS=?QG)=M$A%9zUXSF1m0&(w&;PlBZRyX7UWB$_auD`zXMR@Y z3!lr#-@>-cWJN37X#-cuwqkNbinoRuC+^ram6FhhD>7&(W{CF^^lW{t6gjc%vQ2$UbP;juVmAJO^cliNe;f}^HTre%nOf@%5#fP~U z19Ccz{!5%k7q!23Fg)t}`yqR4hTx+joLBgM5_J(#I}kKksq!}aH>Nikv+pL34x_HS z3!>)(5uXf+sL!PH_w%Muw)@SRA1%k-VOT-ut_8usucdpP{;D+Zdf*$vqDy8!?n?{tXp2z#OCVtvk$W(#!Zl}l-jEv3~>uXmY zEyDCK7frE2s4O66i;V9vcTl>K75$?nbM5RmvRQ2!?0+qZDRQ(NpG<)q|0Rzz0EcD<#Qw*R2J zr>Ss_)kfZO>*;qTwv=X&dK{u0W;V}hOyh);c*@ZHUteG1Zz01t;AcRU$zx`ym;Vmj zs|Pk}BTX<#CR}PWmfYVGI*Q#?6YKef2sLqUeRuPJQF_Z8|G3;i!O@s_SZcn$Dv75J zCJGT!H^^0lCc-#WkK#0W!EH$h-TCl%xh8EGUq)_4pHJBI(e4DLH+TE2W;quF9jH{p zLbqZ0NRwKfz#{M7ZXZ}G?qN?dj>gk|6qo24YSQ<^yxJNqZODNWfw_7S1a#x&Y=e79 zs#P$H2!GcLnTBYTi^gxP>j#-)=feE*zVzX-YZjV|ru5{GX)`r@t*15H5%(bmNO z39Tpx;;N(DV1UrCM6Tt|23}RCS7zE0I98beph+t+o2fUb<+1B{xx&%G5 zaE{wUzb8=ZH(+n_@umRjM}j6aPGu5LJ({2BcJ75Ch~o`6V1yb5bYLAhJYfqK?03Z0 zOVLkqrVP+$&E3NfQRKTx_f8ucLkh%_=`u8*L#@-r;A-TYV61|eHag6NvNos8uWV?i z+#`}3?XF!UG~X*{)Mfdj@5S*^zUGF5#t7O-zI+&TzEsEJu6BUA{Q3L6pS{bO=8)7e zOiHhlhM9b6$vci2HTBi>LXX|J4?4wmn{gxf)i*#B+HMnNM~sHgAw;^yw}S-4b2+6+ zuWoAd-bo-u`kycn8Gf4NH=KU$)wi2+nh1#x*J-E7=+C#|DS z#e(v=*aD=+euedw9}86vHvhU0oKdUVvXh;kS7F%6GcL<_D?Tv32kidTGjJVg-~@OG z?Bz&CPZ`L+`tu5$^S1T$3f5uu1>wlVwaeTN>k^K0vJT>4Z{a)c8~SiUZO)HW(GbP{ z>RuU!i-w01yVjhVZXBC?)>bN7VZ~C+&$;P<}b_>uvpfTh#+^B2d$JVi2 z;-wV9f%lF+wK%9%^Fnm7?&}wO7S%_@6<@Dedewwfmee3LU08kc1YXmv*rA@~>&Ozn z13`Y8b>ngqGE};oBkhJKn8MKo_c?I&KF~WZlPhsFGQ7cvk`s|!R5G-*xf{P{GOl|* zaCG_MfUW5$-RGZGyJFuiG&>5htc#v9ssVOG(xjGjO-{Ok_%p3>yN&>kPH4@y*&ip< z9Cjt%{uq!7_b0esC~H5|Reo%_w7ZAZcvtz%+AtxTv7H`KFsgo`{p#w9ea{XcKSzr3o4uF8kUyqxzr(f(G#@z%`M zEQOte|JmVKl0)j^>oLXd z8_wVXfSxsOr`uuYzkR1Du&sK==|5l!oPo|NOc+S4SrZQ3SYDGS$ZtFG9hD4os7>;Emnj z8`B{V@_>4lWTVh8G!W>ozpGF zQX_)8#!eM1f&uCiEOam0rjqAe>nOw$Ga3DO{QHv7s%xwS=2=DM6cWmv4ypa2GT~znxnBT8F08yL5F^b+2?wJ`z=M1o4Jyb?J>J)w z@KzTuj=aOp2>s9A@)xzwXco)h{Z<*5V?9Qlm;K6^hg#o@>?b1cS`-{u;PviGpk@TD zluP8-oX!FVmBE(l&9;D_1+VsFKTbg-bp;Li^niFX7JF;YNs58)4>y^WOU;4ZX9JVI u&ZP%$OP20`)APT);q?>X-4^mU;KNnAtt9KjzaIg*ob(mW|Jr*bL-;=h!toOT diff --git a/summary_results.json b/summary_results.json index b8d58e6ef..e465df128 100644 --- a/summary_results.json +++ b/summary_results.json @@ -176,7 +176,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99", "ColorKey": "availableMLCommons", - "Performance_Result": 38.9895, + "Performance_Result": 48.6205, "Performance_Units": "Tokens/s" }, { From 86d55e40285f7916b8eca639aedacbe6547b3d26 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 2 Apr 2025 11:54:53 +0000 Subject: [PATCH 412/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10217 -> 10219 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a63b5af6e..5c13aaf4c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-02 09:00:11,721 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-02 09:00:11,721 submission_checker1.py:3260 INFO] --- -[2025-04-02 09:00:11,721 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-02 11:54:48,801 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-02 11:54:48,801 submission_checker1.py:3260 INFO] --- +[2025-04-02 11:54:48,801 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-02 09:00:11,721 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-02 11:54:48,802 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-02 09:00:11,721 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-02 11:54:48,802 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-02 09:00:11,721 submission_checker1.py:3276 INFO] --- -[2025-04-02 09:00:11,721 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-02 09:00:11,721 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-02 09:00:11,721 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-02 09:00:11,721 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-02 09:00:11,721 submission_checker1.py:3297 INFO] --- -[2025-04-02 09:00:11,721 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-02 09:00:12,677 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-02 11:54:48,802 submission_checker1.py:3276 INFO] --- +[2025-04-02 11:54:48,802 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-02 11:54:48,802 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-02 11:54:48,802 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-02 11:54:48,802 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-02 11:54:48,802 submission_checker1.py:3297 INFO] --- +[2025-04-02 11:54:48,802 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-02 11:54:49,776 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.392716 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74194 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73997 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418194 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00696 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,8 +36,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1354 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.41954 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.404029 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41185 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 89a413724..5e73659e9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1039 +1040 diff --git a/summary.csv b/summary.csv index 997a4604b..20f802b21 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.392716,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74194,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73997,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418194,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00696,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,8 +14,8 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1354,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.41954,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.404029,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41185,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 19bcb8701..4bc443623 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74194, + "Result":1.73997, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.41954, + "Result":9.66534, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.404029, + "Result":0.41185, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c0f1f2380c8bcadb21fa5ce57830d8bc558ba870..40301a472a49a49473d6567d7e6d9c2b3afc96e6 100644 GIT binary patch delta 3157 zcmYk8dpHw(8^_71DW?&dVrHjgl8_WxEzDtBNoH~ga|k(B3=@h8VGI#+ipTks(j(-o zC|h9;d2)<7Wg@4%tLJ*J_x=6*yRPqjf4<-Q{$0O0bGCV<-+q96ZQt2xVICeaTJU~# z%Ew-3a*t2Bm16wutDGqaSXDDj7OUOYbRKAbkc{e%F(*x>bCT{448JBH2w9>B#fgSC zX5eB+z%iX)7b*+qNxl0?jg{xKM8s}gPhDOgeIkJvBQ%`_xU@JdlEnu@b+#E)Z>e5p`>de8Yzh!-(#@pIsBt}h%H&rh z);uHS!P;Bw-#de8@l3zsTOGCJyH>XbVsB4|s`mip@_d>UoDdD9HEACpjzT;-fWlSdHopqB_ zFKYltNh zx-2|#=yj3_#ODs|I%ZN3J^J0!$bScy!J8=clWotuk$1QAHE2x;AB?Jb78xoR(NHo8JCDAH{ zr73XW_wA>-m`62iNvihi>}X8pm}p{j&33v`8%tLS=yRt@oZ5RrJHyUf3tBlP5Ct@} z^+vkI1dWSRi#AY+(Lv+S+9YRx?KVd4pm+3x#0R@N!&laKtF{627Fs`@f^#Fx)e9p| z`gUx_0-c31ee!~O+FL8ztP#IG$Jw=MitkCO=Oxd+Y9B_u86bbHJ^3_$564WwZMn%~ zA~L>vh-2o0P%p9J+OMcGcF!lZiCb-_>n+DTpm=c=8A6-~>hsU3?9$0RFfLWi$FY&l z8gY6v*E>sY+`M@rvv(=CT?H$1m3-^7*@;Y%j@IE5b)gCVowS$h^EoHM%pa-B6k}tV zGPF)Dw~>y3Dp0C5>f}6XTDhXV*iSn^^a`T9ZV>6ymJ%8c%!xOG24En?Tu&anxhq_x z^mXF#2A5UqAsMpcf7T`wrqD{9*w~pef?{L8t)4PYKBnP;WS$dtwhj4~k75^oHUs>no15xvF+3fr;%XQ--4NYxx}aefj4 z$qS@nl8m597)T(WMjlGG!C)au(Qy+#oCuTY1@ZXDo z%Q)u^|`1Q^bUkduAZ zj(Cc^+7~K5py)tZB}}@|t@VI9Bg+2-f;ZJmEe>G&_qRt<5h8?2k ztorI!9-`yMG&6{j?W*)0x9zI8&)IZX6(-KnM5X&`P;-(ik4Y}C3Z9QMX>hR$3gCGc z&hH7^ki`94P;VuoQdEQ{36pFweRZwHZ_4-9llPS7Pox{V_unhKtDLec7b+b1kjXB0 zoE_CwelzV~>Mhe*4NEWbIf8||Cz$5)s#68KFd|9CkGi0Mu=nBoi@hyE>dueMbrhX? zDn$Vi&x=*+M!{#eDB}l6hcxMd%?sQrVGv9Y9PPrZ5q5q2h)}(nX@THeQ)Uo@(HDEN^Z{@xzYl4Q`pRQhd z@vV0swAiQ5V^<-JLbw!HF!O$3DRYix`z1NuQ4jwOUU?1xcelA3P6hGPTzRr3`Y{5&Ld`1;I0d}-C(44VgpFacIl-rWoO%>_S7UrEU+o{x7rG@WvR0W@+qoL-<7 z9ihZuU`L@ubTE^~QKJ1|8tj~oDW54&CbsrLF(gBjO~GvW#{g>&X$R!|o{E}d&-m*_ zMfjAdMqBJYimRZ>Rqdgio$dM~Y~gpJVOH0?b*>ps$$H;-d-d`FAi zI~yJK)Tm#?h{JcIJ5qif%|*jawC|+}ra@Ifb4M;;g3Av8)Cf+Kc!sl|FiKCZV6--} zNYsQ1F#<|_Ay_U9vCyj(L$6Bno3B`S$ptb-jG6s`^(kJWDc|_23R0q6ZRVyGLHX5@ z8BuOw#uG)n%8w4}mP+PScQNwvCugLrgK2`|@)Y0VHQyr4>uA&sT`n_GtRI{Li`6yd zvjD~^6|1bSSMuvSC%!hk-za1q%mDaEg|$aTKFbRW3~t6>uId)5+S#OsyST5tvKYZ- zuVt}n8#A|!J$+R*pBQ+|tto=8>x4uLzH|b0!Vk23XC?oU-C(ZmQQSCYa2 zc-dUAih9_+nMAdhC^`OO>zv?>Ijf;Gzf1i9p>c7&zhUXcXy&o2B}X*+Pv11D7wdn? zSiGnaO#;-5F?pU~_*|u5g;?#=ujyLMYgyz&>CHvv0K61{g2xGTkY#nl&6*G67iUfd zY}i8nS{JGtRI}^e3A2BIj9?3LoARKWh1X#LPyc zcJ;VMHgS!70^89u<#Pp!`jx_-@Ge-cw7I=dpPtD?@C_F%5Pr&U;**Z4tRai z;ikvsL(3f5^u3a*oTM#d^egrcw`0>YvV(Ogt32mDMfRl`Vbl>wqVjTmX<1K!O(P^< zsvY7gE8d-{r$6QAyEP?AnbW0q&CYZBUx2Tio<=!4vO}VnV2~i9#R(|asHS_Pq|1yw z4@;PJ-wXQjrcEet*iM`MxJKrjKcb}mEPRsnF?!FY@ie3Fi<}#&J1BzF4pmE9=KNDB zJ=)8(Lmq!YT3Z$@Ic)&qxE?W>oZ9|tV<9Ew5q85Bv~H&=RF`HSw`aqD!fwrWX!}Ia zdWqe^-5zZX24lIvE#RGPwU~#Mp%7%GP$cQ10`o4*K5!Zca*Q$oHIuNy zD)Y`$jg9#6J~a!Twlf4gy zEx+XvI>eB^S;{zKS zVL-%&AQz?Odfv=QduJ7HSW&{CX9mTP?LFWSP-3Q;5yx?axB2ixX()2po9^^;;fH;} zYnB>P26t?yrP+*)%^W2o;Esa<*wb>Qij;?gC4(CdpRJ01Ssc84!`C29V7{)*B$yD{ zTK8%5n)Y`C9*PeCZpX;Wt}3y6Qu`7dM`^2P$f0iZqCRG@{PRnhGjeA1#6WaRXGQ4L zwY|g0PQ^WV7EkBsV0O+gz8a!Z&%Xcr`m4gP^M=tL!q4#v(^}zZ-a489LWdVY(?`JN l{us*3!*h8b503=T|4!Kb{j~cCJ>DC%W`vTU80`1We*hsi@#Fvi delta 3155 zcmYk8do&Y%AICSjjUQto$#rHUb7?6f4H;v|HRKj@8FIh%b7@0iF4Z>4{imhLuepxU z#r+z>D8JE^N<~U?4|&w{Jm+~n|9;PTo%ic~KA&^GpKa!CYNG@}er@s@?}Y#WFvhj%ZB1eLn;cHiR{QwH^IZQyvay=zp?FAroJM0uh{T@&KM?u>BlYsNZi z;>O7lN3hH2V!4-&T;|~p=ArdI)~xrJh40)_=i_h7PkoL<%p%duB=Fm*8r+`Vrj!}6 z`>vvSwoV5U!_1-jw!S8GKef~T(z_N;lpkU%r)n3*(pH0YE_JV{epw^a!cLyI!LnR@9Ym)9GGB16h^% zKo;w!Cz^bFPi8zKUNdGmKXyqpc(`ymK#I9x2Lc2;&Zcvjy=!9N#4G3v|9e0`ysX1qm7sE zNp%O?sU1zUo4=bmh?8`_f739??s^G6#C5(dr>(L%?Sce(lWUWKEw0t&$3jU-mM~hK zkQ5OLS>7(8W3yP(a7aX~p*R`+SPm*4uyLG{!AdZLy0gDZP0)(7-4LT{r%61p#g z3!YkkD4I!Zoh;H?@8m3Mx(-pJk2N+qaaTJzUIxrv`_BRvaR%hz8c~@8C=X&KkzO7> zCR1+W_7f4{e&|6&5qbOhxEd;+hoRO~sZ;&xNRZwz#RfdUD4Y!xpP(vOHB)vusi| zg3lkJ3WSicRC5>=3#Gu~`C!y=3m@al!d=PO8WQ#o0mx|!C|O)$As|=Pf$;nZ5llXp zYoF2`92Q`I$#tqRGLMeIjo}XRVQcK(3T2$N`NC+|n-3L5t=m^J;=& zTRgcAoKd)3|GoR!^IqAgIoDjm`_l^r7c@f0L$$Iu;9S47QzKiP&+Mbm)RW(|v|TIv ztJ*T@RKWSat**bR_s~`yy=<*h?e*>|fpmsJkw)dbzez}>eB3qbGWP|+GEG6QMX5)m z@7%N2w__CVDl$zviSJpxBgn=?SoFhl?JGv#y)}oz7X>;Do+{o&H@BRN(JB!oNpDGTrzyVl-WZjF`^G3Te?Lq{QR0L-Lla7a0Z_t_hLZvc{ka+; zY><*eT8z20fwf=d9RC2v3ApE}{R?^EH}>h5m-jei(=}l)_sz+#!JS z+ppnFf3CM2BrOVVkrTyOBLay2k|**Ck5_T^#^ZLLs2%T1q>oyAZ9}XkE;D!~P6+O& z%f}w@qVsn&WVORUfd8vT#$Z^ti=B z&?9&BgsK#->wFT+LCEQ2Q?2s8ebN@uws6;GOEEGp=`)EE^W$pxQTsO&$$2N|u?5y3 zmF5_Nw523YSt&`o)N00M6+V>QmNw=4@2OyfS?&Y=jpF<=i;i| zSFTS*Eaii(=o?*>bvgQSIWe7Za{wYL&j7;7S!VhIjudiw43Cu*=W=c|=hr^u*oV5F zaZJbJ#bIQ@C8T=n;-cb~^P;G;)TH2ax}PU-4*F%kRo zox@!^Mk)~#fLo?%@-IQQ=Y8kx*bz}e8X+`6F?nKCzb+LUPnAk>A&Ad5!!ik8#OQ1ze}fCElt6vcMn{9B^fmbhpfh2KL!n(cl}I_!L$7 zumm%Inl-1H(y=Ko9GtgevT} zHJwfn8-|1`Fn~H_{!wJ)b}cW-pE8Y0o&KFN1>T%wF3t2kALKHykU5MYf(u3_zC_(= z;$r-{LHeyFE5wgMC8=bJ^=2l1K66TzWJuGIG#wiYy%g?uO@KNrXSNXqD? zT~VML>sW3k<@1liWQK-yWs!Z#aA-soTy@iBO)P;xryr#$Mf z%GV#ygl#9~u^8MTw^7%9-?(l{+ArVP-@a-8^+oOc_MOG0&i?lOCTZlDZVD{$+rd2A z4|mb`XH9PkenFjYu6nW$rQz^V!|#i%>KOX1ISV|t$zC+k5%dGGz~g{iU!pp#llp0`#! zm4%kbLI<*9%U`4u5(C$iWMK7IU!(_;%u(`+_5BN7`aozF6C=aKjI!Y+s^fXslo#n~ zK+4d<0W1dbzy?F4{G??hR+}k&5roD#%s8BU=y@3}LiY}bSi$SYgvZAlGha1S&+z6!HP$D3j-Dn?%*2i zUaR*BHrorWjrS`*cB9u@=WJ_D{K(Atmi*`&4DY?UNZTxL&gO{ex9_G)*I0Y>k57K| zjMikEC~NF`7CJR8s%*EX#YAf_iKg0c3Fy+EUYK^+s3&`mSMvD-ubVH17`<5_?6=J_*f)~JW;-%5@SD9Xh)!GL>~knuM-td zil;*$rjD$WujLVe<4z0Yck%vhB_1usOO8c5DJ$$xLUV_WL4?q&6V}%- zPFbr!6^yoqOR$b30W3fKxrplaEGJbr0NJeAfe3AIpA`+&53En4cCVx> zTr%zj*ZS=gogfLH`C6AkZ08mJW21fIEH>Im2&44nMz8?sjkK$6S!&;(dGBbtB!{0t zU1i03xBf^FcCNxjL@qrk&i0t}<6f%m-O=;Zy6`&voI~ZdLxfR`8hTFDMY~#a`A>21 zBuXJ~x%&v{ZJ@%MdRV$?FGkGb52LI^8oay h69@p<@&Ewhfd36CS%8*`zyRH7od{)q5tUyr{{l;$+_3-v diff --git a/summary_results.json b/summary_results.json index e465df128..d00a76cd3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.41954, + "Performance_Result": 9.66534, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74194, + "Performance_Result": 1.73997, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.404029, + "Performance_Result": 0.41185, "Performance_Units": "Samples/s" }, { From 617eda939b6b16fc55df18b43e06ca2c26a07a80 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 2 Apr 2025 12:20:10 +0000 Subject: [PATCH 413/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 10219 -> 10216 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 5c13aaf4c..ecf10bb4a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-02 11:54:48,801 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-02 11:54:48,801 submission_checker1.py:3260 INFO] --- -[2025-04-02 11:54:48,801 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-02 12:20:05,763 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-02 12:20:05,763 submission_checker1.py:3260 INFO] --- +[2025-04-02 12:20:05,763 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-02 11:54:48,802 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-02 12:20:05,763 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-02 11:54:48,802 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-02 12:20:05,763 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-02 11:54:48,802 submission_checker1.py:3276 INFO] --- -[2025-04-02 11:54:48,802 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-02 11:54:48,802 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-02 11:54:48,802 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-02 11:54:48,802 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-02 11:54:48,802 submission_checker1.py:3297 INFO] --- -[2025-04-02 11:54:48,802 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-02 11:54:49,776 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-02 12:20:05,763 submission_checker1.py:3276 INFO] --- +[2025-04-02 12:20:05,763 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-02 12:20:05,763 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-02 12:20:05,763 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-02 12:20:05,763 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-02 12:20:05,763 submission_checker1.py:3297 INFO] --- +[2025-04-02 12:20:05,763 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-02 12:20:06,775 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,15 +33,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.3043 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.39691 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1354 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.256 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.41185 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409337 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0723 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.340923 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0723 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5e73659e9..6aaa2493d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1040 +1041 diff --git a/summary.csv b/summary.csv index 20f802b21..e4a0cdb63 100644 --- a/summary.csv +++ b/summary.csv @@ -11,15 +11,15 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.3043,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.39691,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1354,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.256,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.41185,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409337,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0723,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.340923,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0723,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4bc443623..edfae94c4 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1354, - "Accuracy":"acc: 75.400", + "Result":10.256, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.41185, + "Result":0.409337, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -634,12 +634,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.340923, + "Result":0.417733, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 40301a472a49a49473d6567d7e6d9c2b3afc96e6..a8339e68a8f5fcdeed517fa6b9c2c9c93d18b349 100644 GIT binary patch delta 3922 zcmYjUc{CJW8y@=}##XXrW{`;NR3w8LBYXB_kLEOD2`CQXrnK~&nD z{*07s6cXE8=yAo#;YPWfW8#=eBng*c@=o;GD|e`j-=p&m=~4tKo~o4KKot|0c9jxR zq@dZ|A`QvfHolF}*w&%z2ab&l-92e&pL{usmPrKCIx0?42;ZJ$%i#Hpce)2A#r0;& zzszx9TL&raW_7>U4hfIXKU}(cNfVmcj1z9SMsR#osJ9dJP~nux2th;34FCWbDN_SL z@LOeCVg>*>a1o3asQJvVlP%#a$PIj$HbdL&9r122j;;mYAWPFgqAM-YZd`cqal)JO z{@`%$fkfO!#kc2i#JTD}HxoVDgie0W*_=P&OpOt_HcU%As8-U$9!3^&?(4MIT^#EO zl|`hoesG(-Gn`7Q<{T`{pSn+fN1V+oJIHBLaPVO{E7>?pK=Zr3>;|3BhgnDdPuROy ziOc|8z`?fLjZ_a^h7-Rqsq&ABG3^fPmg{-P#p~7!zw9UWf2$nu`kZ(>6C&6;G-}y# zI3_WayZ*=Jrcp(w$=2%?iXPn!ei|2C&?%(GZ|t>P_vFKpRhUZJUY{!qjJaD{J66+Y zmg81cXUGnry z3G6sT#$vu=p0`dFE@>_(A>NwX_M{;ud?6PrQ4Re3%OOg!!x2XO-S!(#zYFk4BUo{; zwR^n~j7kgkil|>OWDGj*1-|}6DYWShEFaob%CYb^#LVxobbiISD-h#YGR#|5N4ITl zmc(pmc2qjQW8B3fxtL*{?o?uuY#!$MJy^2x`xsR8-9DEnr}!EhhSB~T5bj~I^Atvl zGTcN%xAvl8ojiLOSZA3J7y97Tz__4m;x`(aJ7WkH{p_6p>zp{z2hXp);6$sm+)wOW z;>#V7QMm}T+=8$3_k$=(Q}zPV3$@2|G{ zPG%Ym=ZypqaHmS_1?>YNN{j^ot-RA<&D`nE|o{pIo6+kAE1k zWB7I3n4<)d*;r%Ozvd z4G$bTpSFC2;DHd5zlsP>?&RhA-K*#(m?NHqA)5L;{8j`Ep2v=&J=PWV1pEkj`Lv}V zAY2dwTSd7JY61*68yIlc$IAGh!a>Kdv*I1to)`~>L2JdkC2 z3EM%s(~vPUxJGD@DK`s8b#V6_?t`C%73SF7Ygk>f^LZIiPMdT$-GsX9$^(`zyiLO! zcR%1ggMwsm63Z{j;n5RJdQnhO3>b(8V_;ykHx(Lei?!sK?e-NRNsDlHL_;mGFvrfg zqYIEe?riz}Z(5E#gmjVZ$(bzf9O?Bse|&+EE=h}<4q9bAk77(TFrtf(rk?-tUR`7U8>04WN&v=4q@W9enXiYK%H5pm4wcP9imoi=$2O@UTg{r~)@3C$G z#OV9uMv{>focevOh^9^NNu23_&tpFpsnF8yScLXMWlg>BZV5C^XN$?pzOk|}6d@+@ z_kYS*jV}8w|Fd~@Ts1f~$wyW2%5+zaCtU2IfG!rz55~Dg=@(GR8OJ)W;9i{LL(REM zBBYgJl#3ti$5?5nxCm8OGSoL^)n(&f`}Zui6nYk|#{5~Dqsgl65`QjXijZ@$On6L{4UK!`x+@%C95(#0K8YVOkToE!5J;R= zun8@X^rV8Uv_lDLb+p(?W7Tj`$&Y!3 zLq`sX*mI=D;g?zpJgX+4mFJW^8#8gIs?If6!FxPXn(EM5v3=J zJ#Ci1OK=h=_-`vdXW{dZqQmyC2dLh2-7VT#Ibt$WH{Jc*G#Rv6+B~xNRzUgHq~B&H zie0~8zXShAu)vRiN^nc-1)dWsrV>MUWAu5mLM*B6YkRs2i~5(its{n+Jgsk=B|tgB zXvR&cD?w{(B8L`hw&o`Gmn9ogZY+BdmVdpS(o;^*+lw9_?$c7J+}q(`GqL(G(4lY( zLIOO~ek_CsluFg^Nh~+2#`ER{#F)jP9wk7(uRIG0_Wf;L6Z*3%VJFKv?CNM`NeMxq zBukf&p#_=L_FP`M#BVVfmPlVF<*??!pO1Bm>J#;0M*z%to5H!r!}BTHbLdi8M!?Ri zF=u{c5TN79iQwqZ^SK;Stdoa$v%Ux4%(^qx-&8d!_S2-$y%TRBOd&jy;D(Aq6k}}V zQWml9;Ed*;WFAzr zt4C8?xSOZjpWf1W$uryucsf43`BxP(2U~&Fhs_R!C@~TWkgXjlj0X%!?U3EJ$hVRW z)h@Zkd`pP?UeUXzF6&|Lxqlem6Hy$Q$I`mKZ}mCbd{$m$hQhZMRIhMM;mfA+m#^&c zzkI*{!9JcYZNS0M`r?=SU;g=OrIjT9ClYP;^ z6bc?Cf8qm_{2Ae8Dfznv?*^TbiC5*w)dqk*QDQ1Rk-d)6jxP1!XLEmEIz&N;umSV9_V84rWYoCppKcs6}{|(>Fy=l>+3Y}DiV)@o#UlIbTp(tJ}q=Mx?VoQu*p)p@5j|b+{3Jp|UyxdS7GmA@%sMHbmVi65Aoi$ae zw+m5WD{$}y>WkHoSVl(PBu&><&Qga~{;9H%(T_+gIf*Bps)ToF1=ohPI?26YOF0<) zk_=j3>%zF=E0k64>r;ac)!EN^w9ni)d0gx0p@TZ}t`2i4x|Z7QpkVN>Dr|7pswp!d zwbkZ3OR()RR4-0lyb8!8fGKQ*s*x36{M#mvwP!6 z#)ufrl|eYWoC=4tae6KloB^7l9Ct_p@GI9)bitNi6*NgAsj9pkrs$#-SXG2X6X zD48d$=C+LSF9>rh^ks&`9uO`Q=i#ROzWBMjJ8&({#faMwDKl8>b(tT9$c-7@Gb@|y z8v62i%>O#Oq`s6EW>okO`SB|w-y(&ciPP1s&#*59Z-m8(dF{8n0*-upI_V2i%9i_! zV=$8BsqoZ`itMMEYNxiLKB5u#ms)Y0!8f+@Bx`V6r9gI+Vr#})gJB9U*S8|WqlXUT zW4#QZ%cu;b>knk+I*lLd_*f5>vT;C&1L{IrB&V8oQl-Wkv?hBU)uxcjuSCn$1-m<=rs)fX>C`(05bHC?Yf`BhyQZwp#A6d-(~K4O`TYXvL+=!E zApEaoA+}zyVx*I`y_yEWT@)2YJ?xO>h}%Rm7q&=GxNC9N(|GlkMc+2H_U9Vz=6n;G zHmVj%?lYJN&zYu{XPpEE88?lWD;?7PB#?Oe<0hx6&c1 z8`h)2kcRY&3_hPNeKlhYMPcE6UhzSfER$+}UN`#mfb_DkQPH4=12JZg>OWBS0Pw|q zs)>0|%fh<U5WaTK+$XW-Z%3Qt2vGMR#D}kAC z8C|EbKr7{o71vNt+svyrJ6@HIo|LKNH*@FuGf17c$T(k#Y<%$J(Vyd5J8(g>NSBu0hN(%#ZVYc6_OMtubOc59Y9tNCH5lczF=c3U zMn${_ilb)wpOM9_KyC{B&-9rA0Ou(-@ZS*??Zk+SX5`1+6O*Ju;S$9}|J!`HDlr}^ zCR~@8CKVCKAg)FQ!QB=YyZDa~Q~-bxH2}c*F9sz>MbqPA#nq{-aUaA%^sJD-OaB28 CbYO7+ delta 3991 zcmY+HXHZjJqlH5!C@u8RLz)PJ2#O*l8hX(nB27972uM|FhNcjjLLl^B1nFH-nslTH zQEEV?L+Bt#0KM`3-tXSIduGr6ab~Z5p0)NlGi&7*`Bv2+9UH8?SJ;Cd0PqI^0Q>;F z8l0zgo34%ytCVTU*E{TgD4ogG0O3k-ATgInnREmSx(x|B4q49zM*j&deR%}Y`P$EG zKiC(d6`jNnvZeiOq#SD%=@zfhRCrM2h+9?>ObXz;^y&*^qOoWGwL zXJjfn_k!a`sDtmc;`V2Zo}2Vm7f64poqG8uLPi zD{`Pu=#2)8QWNhMjMW*lv3oV9R)*qEE$?)-I5>+o7slQ6N+w9f2_J+7W3jZib?NB2 z0RR9qb*chH>V}jSSpWcbqklQU_*7;^Y(mJ`)cb%lL7N2sShi=)=>{>=Vo^Om)~kxv z90!>lo2yi_SlJ)mOWrylrEwx=Ok0(J587=u0yxH>DN}~q8)YdHA`B?o0-)oLjawpg z?^Hu#Gd9%uN=3w;H9SMb# z>C9|+bC-MytQ3loU4^f1&yPG68INP5%zJyZQa#e2uJ0acuh}AdZcEQn zT%wt|F_&uuI1mS&EY4g6E%O1 z@!p=$?Ss?mW8j+6bsxom99YoUArVFuMauT^!CKBmG2+i;NgW-&a@4iDMXXo1Z#6jCO#0%Q^rB>CSBbZxFPkBv zqJA9d*_re#6qFOE1@%QkN~oSZZ+XH}Eb(LR&6X>N1(nX#fSZJ(>S3kk>zIn=}H9T7+x5DuN71x-Lh;-O6p&;$={Thvp*ju3QX zF#0SL=)N`t$H}$!AQ5@TX^4bn$K6dd4DEdQ;hz0$=95Zn13`s5dd4v(ZnJAT|A1ho0=hOW`Q1CcvuE zv39yreRurZ5}W|Kc{EZOy1jXgo7GSt;8Q5Q8|;wF;6LZ7Y=>2`VOjB9Fs^pJcX!*p z7rY*(4YYl6yssQGAn77?$*O{>XlyuxTw%9NQkHC4^eOY;>#2dI7JCY5!(CqK<GH2!3r%A`?rF zqb_C*{o?qoIFa*Ml208~Z8F7&s*C!o2L+39DZEW|0U`8hp=TPb(mOE`tyL6KJq6!Z#H-7#y=udQAEO$I||6To2-|JJ7eO~tt^#3&R zuIqHEG4hgDz_{q%-T%dI)NVGhzdPe`c9WpLyfR)j0x>)!tK$w zuGUugv+NzWptx$BJdoGv0k&Ts35~oEW4;dxJWhVA_jr&}%D68hyf;SQu zeNqSU{iS4t+L7L%(z45RbN-qEanFlX?9$V0ry`4+WjCLC64{;fTR0B|*HwI)!k7ZR z0A3ZktqvC%1In6W?dLJ04&E%%DuRWiy6|E)97GGmIcsXXJ~4^ft(b4t1e86aI#ok6bBd7gr`S12kXog(?~|2RW9Sosp%cfFPLBAHVeG0 zlLN|7g0kGJ?0g?qIvzEsgjG%){xa&FTkyJP->YQbs~C-aA1Qrbnd*i)`-oT)EJj(6 z&IlAMQ6jZ_P(`oi5dU5?q?ySqa1`jt9o!WWo|_-+7ubfmUERl2eR4zyec-bH&S=^o zdp~Qkt~uja$Ia`4LYju_>b^Mm-nF1e#<%w1J{VE!LaK&WNpyyaRLDXN5@jAn{72H= z;)V$};zi$40?wMV^n3&5I^)u<$)>V`Ub!=-%^L?JO`8p*jB|HNh2%%B*y=X2kGvh- zxG5j$2yA300@BcQPQ34eOlZ|=dpGjiH|V5QR>O0E?xG;Z^QHu_aZQtL?eMpoOP73) zEmqtbUz$Era2xJFYW1!kLUCnEG}Ik)#Gc>4s-VN12vx_jtU229!H)4W5io#!^tuZ5 z(mRFDMEHc7Cv%MD+O)y~+5ixI*E-g0%RD(grr^Y8|^CW3yW$ z1|F;(zWY4=EPJucSz(X2uAIuNWUJfpmv{Rg-j4tB3OxIVcRRm*`!6qjAOpu&8;qKx zK8wvE99-3 zHqPGfp7gFyzgduZ6Y4h{=R@I$MPyQKL=!wHvaB51-E?T)t3f6RHT!s?=6|Bb$w_NN z6@#4kIr=hG)E2zGC<|Px%DCR;HSFF?O49tjVw+9I}%PQY5?bRo* z!Co!9{PF+U(#hmEWu-`dS<83b2T|In1fQS$9Qnth`O4_fte~@FpMTh17gRQ3Ywvp% zFKLiyg*@NjxWC0%dPM`g=Om;tzi|BKa6Kt0TKmumd|-8fsXit4j|IJqmD#@K#IcP3 zL8;Z*(|$$y(b26!XWvhjHSDe?noN6}vcJB>zL+<4)eA+L6`MYB(WvuhTHN&p4JBuf zwmMsNK!L9hziFTvMbVRu`xVw4FZxcC*Arc57yb6FYO!1?Gz{A)fl{OG3 zP7%n2(QquWZ9e8PzwzzeI9fBllWaDV4=*4=b+U1JZ=NaCmw)j}qcn2AEpIEVbMd$r z62M|{ZI`XbFavBcGG3KeX;V_E@b-p`@k$p)B`F-2xmU*9bx&3iBWH2kR1WV@e z&+ftcO2kp%rL8{ z0;^SxW^o_e^R8s+H)K8Rup^>(*LONu<%w0davs<8bUIXc##89YBa%eZMyjpIhZGAL z8N@iPXTi&bkwu>dix~H%pJ!Jh`-ppijgJmaTb8fdZXNlYxWa9XenxKiU-s00ztzXp z=g!Ri9LXNCJlAR2L;3|Za@}!T)yewsHQ=CGAZF(W_TM&xpM&$&jtJajW>|Z$|mG;25=EqaV+?Yi4T&Og1E3xWbN+1Bj)b74K>4L`UsXe4Rh@vq}hS%4nWcy`Wjx>_@da9ZI1wq5)(Oj+>% zcS34S{C5P0IN(*`>NLUl=WvMNKOY~pAtQMJfWLk6zauz=1^*F_Vyzbd0G_(ac>DOd zI(XwT2yQ$S0j5O=;ZZ_7_`3)W!GDsX0RV2(0sx%<-A!GCLzwZw2o;+9_*R4jBRlNx GqyGU?0eR{G diff --git a/summary_results.json b/summary_results.json index d00a76cd3..2a4c7aedd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1354, + "Performance_Result": 10.256, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.340923, + "Performance_Result": 0.417733, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.41185, + "Performance_Result": 0.409337, "Performance_Units": "Samples/s" }, { From 7f49339fe8f6e43c7eff4d33b5448e99834e9e91 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 3 Apr 2025 14:04:21 +0000 Subject: [PATCH 414/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 10216 -> 10227 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index ecf10bb4a..0a6b87abc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-02 12:20:05,763 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-02 12:20:05,763 submission_checker1.py:3260 INFO] --- -[2025-04-02 12:20:05,763 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-03 14:04:16,701 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-03 14:04:16,701 submission_checker1.py:3260 INFO] --- +[2025-04-03 14:04:16,701 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-02 12:20:05,763 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-03 14:04:16,701 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-02 12:20:05,763 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-03 14:04:16,701 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-02 12:20:05,763 submission_checker1.py:3276 INFO] --- -[2025-04-02 12:20:05,763 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-02 12:20:05,763 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-02 12:20:05,763 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-02 12:20:05,763 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-02 12:20:05,763 submission_checker1.py:3297 INFO] --- -[2025-04-02 12:20:05,763 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-02 12:20:06,775 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-03 14:04:16,701 submission_checker1.py:3276 INFO] --- +[2025-04-03 14:04:16,701 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-03 14:04:16,701 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-03 14:04:16,701 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-03 14:04:16,701 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-03 14:04:16,701 submission_checker1.py:3297 INFO] --- +[2025-04-03 14:04:16,701 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-03 14:04:17,732 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,15 +31,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.3043 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.39691 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.256 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.5935 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.291178 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0894 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409337 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0906 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.782 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0723 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6aaa2493d..e3c00fddd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1041 +1042 diff --git a/summary.csv b/summary.csv index e4a0cdb63..1329bfbc0 100644 --- a/summary.csv +++ b/summary.csv @@ -9,15 +9,15 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.3043,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.39691,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.256,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.5935,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.291178,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0894,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409337,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0906,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.782,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0723,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index edfae94c4..75349a0f0 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.3043, + "Result":22.5935, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -339,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.39691, + "Result":0.291178, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.256, - "Accuracy":"acc: 75.000", + "Result":10.0894, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0906, + "Result":22.782, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -544,7 +544,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index a8339e68a8f5fcdeed517fa6b9c2c9c93d18b349..e723cb105eeff8fe2a04b5d4eddea488dec96a0b 100644 GIT binary patch delta 3469 zcmY*c2{aUJ7oII!*+McH>li{N+n0Sr8C#a@WXoEX&oX3djBQAznPIGvN!iN2Yl;*` zqA^Jl!epr=yRp=-?|lD%&VSE&&%Nio=iYPI_kHf7)w)&P8!*q#FKew(2mla!8~`{8 zAfe=hNI${6VqT^CO(U{yL_R27#F7!ma4nd;S|SaSyAK@nB#A;qvF0=Z<(L<11g}=* zUc}l9z0>OqvOnMHecg>qWCmf?tbG-;KlU>}XFA5BQgQ?}WO6~t_zcn#_ELR4 zrcg*d*Gk&K@7dkfF9Xc;rU!L$zLiT(mllq z8n(KQdQj!#K07s-;hQ6;ddP`meEIlO^Ncn%^^)uvjai`W{0f`J((;>!$(@nv zyzg7^`F9qRPDUDsik=zc{~_wFIKTt*HJO``G!v(@+x$9lW;*r5&4x^q@zEWgbX$|_ zk^|Gy^V+j~*sKQp1=f@DvExtIWFVlgdWMpc<5wUczZFgpkn~LumpG0|LwySF*8%}U ze}iy|BOcOsdxHCOc*0Paat?^yHWZimbxM-YdeRjFs*1y-P?#0|e)d_+D(AviHq>%f zB%0kWI0=P%*2e*1#{Cfv)dn9mazmWchxoDDc^8Cr7IO>z3pe+^|u?I z^{kQPK+gTRkt%+B_%~H2-%=%{X6u_K?zO&GPI{Y53z(emdkv(5)L4{u09TsM))+0x8Y%^$l zWY;+zLrH542c(JzhZ>dQt5Yk9(Xqq7HEj8lG?<;ZSl(Bjj=D=pMWXk>Cy1^uPX%Q@ z9k##R{5agEnTc;wU7c;0I9j*o0of5*I=}%IrgVTGZE;gx!u`YX+sgOo*zofRbxSSI zJSC@|c+ub?QNJzEsrby_N!3{_U5j@E^W|@7IP{v(j6s16S2QvvQpr~--Ot`E?0gtB>n1nzAD?1KLOOp|H_pp^~J4M2m)cUZ0km{j!{igGrF(H{I;2zL{3q zQa%H9Je)mRq_SSE#!W21Pe_AYpysL*@n|diW5Hk)Z7{4vzDdBTSXMxzMuBF`O*BpGi+n9m;;D>NoCxE zTku4*-DF^_`yFxn~i#dFZ16#^tO>MBlUhC&ZE2x0Lgd`Etl!0s{}+kGv{nZ z{35uN^_L(~%2HKE6b_B>}H z<2f@mUb3CC=ZH*`%N{m14Hw{uSTG#ZPT;mKdExV5Xx zHg7fF`M9iK^{6v7c$?nwqvcbP!li04b^2M71V0S>p!)~-FjFb_IinmSbi>Bb!?sBvS;3_cKGNx^Lf z5oYjkfO_UJ1L!e>w};)`<$wj)h@Uv(Ltfomym_FXMPNPO_AiL?j@t1MD+XyP`ZS~K$M_YOpWy&OR`#y z5IPjL(NI?v-6A$R@+_wu>BC~`(vr*h5EsW#oUY^EShcX6em~>a3ukN*b*5CunL2ew zWH{}@{=!%6D?-%PELzR{y=ak&9F7oQ!j5nL98!G4$hltQ0roE^s_ICd$s7`jn0BQ4 zuRM0M=rCc_{SMcvconq2Abp)h*?uvFr8jyUcrQBkmJ-M}R4l7qw}=NX(lC+86`o|P z$$+!PuX<%qmZcV8%KeP1bFEKftzNL!S~o&Jjv>WNUo0Jz+q_+7x7kvlDxXa83jZyo z124>tfgK1pVg>SmNB&a!+a#t@Oy~g!`?2GS*`wo5Y_{*SyXJ01opi8Xy906&-Y?;C zdYUGk(1g50sk))j{N!DD!EUQy+mo(`O;`VR#Km#BX|M70*%m?F=wo%1TCd7ljka?t z4vVk2s7|HAiW}ak?v%IYg0UIrRBRUEM0bv{w;_p^I*bw74^pr1%NOsu7yC}ws$gyd-49wVfnvXJ(4yz8A$|t4-@Z0wyz^`@4k7d7`ey@sZ^VoZ9QNyKtjc(32~=6` znQ6(!=!?2q67L1uU#Is52lkK??6cS%R?aWzuolde$$8(w#>E8EgN7gQAHF(La_~~fe z7FFcd>J0YZ50f-2C)`LEG2(!>>fQ%byoKqyy!#8@dI+Cthn)U4OtO+j#|;Fhwn#bMtY2`o9`vV|K4?30v7XO- z4D0!N=dWvkVUG45E|7o{@)P)y4nG~Ju#?hcP$LdVx)NZBYlgj|QP;y&qb}!S*1MtW3p){;YnqP>%qB~2f~(BC6{5S6!Y~=bnb+IMth$?s zDKuZ!UWG@Dj*{vY4($!l?$tXbv0lCa-#k95rrK#~N+7bdRpb={Zu}n2db(%;9QEpd zo?+6Jx3?zn4%_G=lLF%W-;P6Cfa{6=Kk7IHBL7k%&WB_mcOIBQN|loq{b%+L_ab>A z0N@W|`7b5n1V}IB47s0)007s6R3ai{f_x)LHV8qIECLEbijfS(gh=)XKG8pgf&&2n fYY+gy|6lDxo`~ZoB_i~JzNF^}88%_~pR4}>X3}qu delta 3482 zcmZ8k2{hDQ*q^ZuV#Zq8nJmecB@7K&$C9<|uk6`|2qEJ?_AQw)#gNco%91rpB#Jaf zvS*hzk$ovNmXG(G_xsNIo^$WH_uO;ubI$Mgd+vFjXUk&2qNb0Y)oQy%HIM-WisT|h zL(US`>7jzLmLiV6T=K?{jI4X)#kNJczQ6_xPB8ZX$UA@_z#sth-;;V9VUmtte1RSo zH9!@=u+R9={`0BaXO(71nl#1disAf)FY{v?H#%=pUvDMA+)(t4BDtxsA02fOM4Z+f z+A&KPnR>pLUMy&W`c(pc77w`OD z*X5W<^Yfh^MwDxM>!0H&H~X-#rD1HBiZhH#;~Kg*)^1op9cm%tnHAvm@XIB5)B`uT zad8Sy!&J@JxcSUJ_)T=t6U1!Z()dRKTFpu*x__l?YzxT4tG@Sp^@&rMS^Nz2EasB^ zN^DMSl-0+M2Cs*@s!`QR*8@I9xi+3v^b{^9T73IB5M`Y-mcoq6Z5OC1X;RCPUmY6N z8lMvOMv&Ot2RPf%UCu@kK`GCrH6tDfWvpSP!}LY;_nf+-PrF~B$_snheQOmp5&9A@ z4Wtv7QuF{VsxbeLrUDn!b!sLI@@$787VDYG2_d#gK^V-cY#8Xg$IjAiG>ZL%FuXco zW1uquQgt|*#Ox4VjH8dmVU{!g40Ppp<8^exf;b`1R)y%mqisf_A}M2?Qu$8>Kn>bM zVnUV;EOg*KVK@x8cCslDhbcb#cnRhyhA;vaT>1Z5F1is|fc~^~L0xfFDD=wnDzTqT zU$8IZL_6-a(s;wQkViYKo|oF1Urts}G04*ruFGWkO>p=I{>a_$^MZ<2o4oyM;MkSW zW1b|-@WuU2g;znyD9`#kn`*RE9H(`YNcJzK%B4lmzMBcXGx<&xo}3Bg&wYC2Nf$?1 zf!^}#uUV;YJvY8Lwl&T;t{tZ1_<~!M+KHT;SMq|lTz%eA=EIN9YV^e1rwsgRhkngd z4c6#OnqEZ&K3)4pS~?y}B>LfXRC4H?Or}oJ0t~_Eia6SWLI5F+UFZt_RK{)2{ys`30knJhhdQCq1G&Ko^RXQc?eSmH&S9^s5Cfuui#QH@ zpJN%yIbwB2#bD1640Pf1%0Tij7=rT0o%vuWxjdfUX6DN2a1-3^;@cg%A?fysX#CO3 z@`~*1V&&QUI9o}V=rs>;_)$7Y8{Vc3o0|rtblwNHUVesLnt2*(KAr}cp|XSFC3PEc5J7Mb9UT@T&4#o&wyI!I#Q`dfaH8Q?!Ky%lpWQD`)M?mE* z!gPp4?z3PmV@W@p840YE%0xLQK-2NANHNO-QDjjAvx3p0HjTR_+`8mbOE|@RWn;g5 zWv3%%S6eHOGL8BM&6`n;Ici?>L<^o47Fmn9RQ|CCt|X>6=lDScACTFIDAAUacn*wp zz!kM=?1IPvz=p9@&2O-k5Fewnk|yky?`41acZQTqxPw7tDeeVB)})ojU2&QBg~bDZ z-Av;-2AhYT=&4XVSu#0>L4+vuV^bkvcqwck2ea>^K zkTuy0;mgaSdv?ptw$`raWtuXy7SW}P3$Lb3)RIlMW5+_kvd$@*V@J&d?>>$hWVLPJGUFOC*H(5c3ssA$+wh?iKwWu7?kv_nM^1*5gK} z0!5xL1@`vkDm75AaAV+&p)^>=g&CxNyifKmnj(9Vu?p53Kb0&2Qc0gi@=qgyfv*H} zNZ#BZF}Ip5uQE4@iVK%$`#L@P624F&Fx53QZZLW8lB9Nw;Vs$eAtL3su-BH5q;*-e_yI!!0t z0wLxT_c=M?OJ&b8QqkWwU}x_xC7K*l$kw*9fnG&u>3|L}`mA1PH~fyB=eQlhI2MbK>GVK%ci zml|Cn)Y(aHx46tCYX{keha1wSUsTP~MOOW(wo@>R#yveqtUpx0+o^Y_E~?E#senCW zw|_JpzOwud@10!fHSej{<9=yh<)+cDt#VzpjqzpLKXi z49z6F9sLNgOrO~8H(1{^SW9d>BHe8N`);86o*rE8z$MSNX<34n$iB4TIxECf*fMxa zD?P(l5yhsuWZsuztMJ76upnXUN6_Nu7u9ZA3Bxpt;-V=iH)+`>%1IH@`TbBY+~ktN zmeyZ|`c!Trm{B{w1s+i^$y-bZq**IV*s#dbpZ7PJ3pUg|bwN7P2>hJw(1KakMGud3 zJY!^ay*+Qz)=%T=ffMVr?VPovYI8-$Wkz|S$YhW6=Im7q{`P}oH6vsE&QDHX*mh5J zsT@e>jjGjEuv(B{ZPR~}BfZUKtdQXEqt-!w?CZ<5jPtVOvA`(w*P+kz1E%?LBiq-? zCf^N=ejE$Ygvyx7>fuL3{?L-jEpL^mP$mwS)<2>~O9M^q4s7c_7m{E_u8eYLW!3C= zzK@{>We!EA3M%vN6A_2bkwN0oH|E<2oOiU=3uS5v>(98LfNERLa+5^{ANS`Hi@$zu z<6}L{@bf?p#`_0`xI!Nc?C+ZZ8LPXcF+APhy0>@Vovdm5T_J0(->j}CXl`y%@Bu0N zP28%zi0;yZq*MgC^nStL$${T_ZRuoc7w5NAx{EvU?c{HZXIrqhg9+IArX7kB#(2&p zq-V0Qm3cBOlm4!i4W?S|Oqs4sQ z60F^cM#Nm=zE~dB6@K|qGkIp`d#a}z1L7aOxtHp-V;9wsEWA$9PAyqdLB9(yK};Ue zyCVOk%=+%L4iG@mSso_=27xT;Kp@WJB$ZYNP>7d&1UlexR-a#|A}nDI z!&=-bH{q+t*+`G>DT~#$aSY)e>gH^S!HpmyX;kyC=jLoOD{>FQ!>wM9S6td-{8>84 zOI42J8kmNZJz~XF<}AWZqFo(pKKVQw0Bbpp+>vN{00=M#eYC%ICC)+|b(e}x3>ULc ztNp2I`Qhf^lj3Go^IDkcm}~E@nm_1t!iUV?v{><39Gme%; z@$VzEzon~TZDQ)&EUWTOlHwDMmmCCV?kX60jDb1^&Nw|M!TAkS>J;I})f;aHdldC+GhIHtlIb diff --git a/summary_results.json b/summary_results.json index 2a4c7aedd..8a36a9f27 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.600", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.256, + "Performance_Result": 10.0894, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.3043, + "Performance_Result": 22.5935, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.39691, + "Performance_Result": 0.291178, "Performance_Units": "Samples/s" }, { @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0906, + "Performance_Result": 22.782, "Performance_Units": "Samples/s" }, { @@ -725,7 +725,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 5c9d2cfd88aca26b7633f9502da9edcc481efef0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 3 Apr 2025 14:11:57 +0000 Subject: [PATCH 415/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10227 -> 10225 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 0a6b87abc..a461978f9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-03 14:04:16,701 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-03 14:04:16,701 submission_checker1.py:3260 INFO] --- -[2025-04-03 14:04:16,701 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-03 14:11:52,496 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-03 14:11:52,496 submission_checker1.py:3260 INFO] --- +[2025-04-03 14:11:52,496 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-03 14:04:16,701 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-03 14:11:52,496 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-03 14:04:16,701 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-03 14:11:52,496 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-03 14:04:16,701 submission_checker1.py:3276 INFO] --- -[2025-04-03 14:04:16,701 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-03 14:04:16,701 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-03 14:04:16,701 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-03 14:04:16,701 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-03 14:04:16,701 submission_checker1.py:3297 INFO] --- -[2025-04-03 14:04:16,701 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-03 14:04:17,732 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-03 14:11:52,496 submission_checker1.py:3276 INFO] --- +[2025-04-03 14:11:52,496 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-03 14:11:52,496 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-03 14:11:52,496 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-03 14:11:52,496 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-03 14:11:52,496 submission_checker1.py:3297 INFO] --- +[2025-04-03 14:11:52,496 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-03 14:11:53,689 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,14 +26,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73997 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418194 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.00696 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26821 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.5935 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.291178 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0894 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.97432 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e3c00fddd..34d9aaeb0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1042 +1043 diff --git a/summary.csv b/summary.csv index 1329bfbc0..e60739368 100644 --- a/summary.csv +++ b/summary.csv @@ -4,14 +4,14 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73997,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418194,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.00696,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26821,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.5935,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.291178,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0894,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.97432,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 75349a0f0..f10159665 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.00696, + "Result":6.26821, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0894, - "Accuracy":"acc: 75.600", + "Result":9.97432, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index e723cb105eeff8fe2a04b5d4eddea488dec96a0b..8032cfbbc15b24346fbd6382ba6e17a686ed0e18 100644 GIT binary patch delta 4563 zcmZ8lXHXMBw+#>kR6^)dr6lxjP(cVC>0PAv5|ARj69~PRU=)xRIwHMEM-T|TSs)-C z=|~X-EHAz{-}mF~%+Aj4?4CVm@14DO_f+>-w{D!|=FD)_JzpRI5KV&)C1*l^Cm}=P zJ=Ub@!sM)}Rl*RZkrokPuvpGa$WDqxxwI}H1f+}}V8ZWg9u4Uf8ny&vyIKk4!KVgH zpDnLB9tD2?A*Gq}_9gtr%hr8b4}8$9ghb0E6gCV7yAFG0#u;whQ`b z9p!!|f^F4Er0(Vjawc{=9zR!Uh(JDYQSiQmlnLF6?sU{lVoV0=tBe?oHoH%bpdb&P z-3mLWW`U)m(^;qM#?(YyxsyvG(^pTQ7(DlZ#nW7BZQ6^`P>p-)Ab+joenRNI`}2TP z_t$tcC+dT&x;md4k52UVO+&r^1H|94g4sbR3;PjTPESE@Wt0?jE zp(|zeW5>Xhwz#bMTq4cj5I^M3nuz6iRM&xR754(i4Yuid)iK}bi^MO94Ka`BXMTd> z4IcX{WFeKu{5e~&ZtMCjlHIqkbJXzoleGBk)a_M$|0%v6zcVw`)9`qC`K}>yAS5JO zzIR_%mBYq6j{_{$p+g*0am`7fWHxflJ9ksUXo9b*}h#A z<>K2N6<#0IVmc|h+N&Heb)&f*_4-*f$<|F2;NIbRz?b9g{T}@K&c?4R?;cm$wtZSm zg2{^{C$)Qmnwy?To)AFL6EMRhT82l2J|CoV%PV z{;<0PDvfu!wmQmtJ2CX++g=q3<@+L#OJt0HvV7q#nB0hsII>EK!^Ro@BcK9Q6y-J< zFeMsVA^i|3HqF*#TMm@^EKMc^c@Y2P3&Ul>n|5#gEv(xxm5bxr{KZ#Bn`=+tPo|wP z`G|XeQvf03R;^3k2jc}K(TJ|gstI!QOmTHXk7y%IP0O*gS%HjJ-k<6}0)Hu=7W@dx zs*n^>+q(zM*wBpnTTLa{`R8X}^NV`1d?-||pF#f$mz_W0OqeArTbvGD`oa&}8k zlLSad0{{TX2~`dd5llILc$3g;3?X*}A`^2konFUPrS1y{F~?s^I>w%VTaXJeHB$+K zlFMAQke}_gxJ&vNmyT=zMewAkwT>`tSgQPbGyqH;>OyCmj5V)AmGv-5FyBhe_iFR)RP##psE-qFf+jA#k zT-m-P{7@Z}_T=UZb5zH^!>-zU`dl+cFa*DmC5Kbw$HxmKS#1aYVlgzHG zD%C?M0HY|NJG7Nn=l?c@A4S!HDAA`7>OSRnPkMa@yq{sl+UA@rWr@ZFT!rpg){vEb z(tW*GWB+aOkwEvVZ}oE|)ldWURfQKT1&=FO3ri%HaHCh55>=VhOF$n(0T0Ie$iy6z zbfr0MM(RKmQO=b@&5H_B1TD+AoaiMnL7naFU$%^AT+LcWIMGKb?1GElO|0rX^B7-f zc7=jO!i`}FGY73?Tc$fhhB^%(oAloaQ8|n-6!>DbgmK6$TQ=q!2rPN|*p^(P){l~A zHv(#oM6S7mt7U7a)7LCq>q#2@VY+z@kXAC4w~s(h9R}c;2E1~3dbih&K9T_X>ima9mV9}TnQbqtzYVREQRi{BFw8_IFhkIY z2!e76{U`@B6^Bfqypsazd~1ayyMbWE-a#^DKNGqL+?YI|>_>GU9iBsO!hwGCh26}8 z0y-4ZglNO8pTUT$wkJo%f!UTJx5AA4}%lW_%_@nu$B`m9loCEj`)2qR@jI?oGT~ zI6tgtpFQhj6|Gu0K72)`g+1-ZY_VazN<-oh={TA{Y-L0|34$Y6T{$X zUYCaVMFja9#-xos&BK~%I)ooM56%h1X~9yy%_q+Wtd4gMIs~F$K z0EUGct`K!s^8Xy)0rfF1x=#^P8XHqx^JVw+BK=aU3(gpf-$RQy<+eC){+hIS>ocXi z94|Z$Cx~t&h|YnI!L>*bG_|R}{&KNUs-S&|5e(U3_sQ-~0UN1yfB_<$)dUaFUPF?j z)En&@C)2Q(C2vPH5``pPJUYT=)Mw{|4cc!x-6PZTCGgx+g4LW6XrfHt@AVVBzkV0+ zO2z8#CyEHujUUr1KmJKq>;=|RQS?ipO9rJH?P@2NNw)4uUR!LtH%@v_G@Y;mc8oze z2-f;id>q1J=bV#SxiK0ahjamGVf4eQgT2r0xw^sl z{ddwsLFEUm95@#LlF>MX%+3iXGq53gG|mN}CczG97~brcCgMo4fHPbCFVQR)b&kGG zrG9|g$yav3iBxvzN#t2RaKoqhC&C<1nLFeE`AhKhDl9&>PhBi~Kvh&r_p^ZaYRGw9s z7;KcwR_DqBgG`jwyQL47mhzv4Q;F-G#+Rvp%DW!hba_U}T_t-#TDJC^E_#E4?CuDK zMkGsg@o_R@+A1VQKJ_#Fzneo9x{qa2 z)*`AdSv==ko(Kv^btJ6Aa7C2M400UKvA$TT@ew-d+i3kbE^0;?Q;}$4tuF*goHVh^ zw`z?Sj8ic)T3*%Xdh^osy_AuA|9lP)za@{$`%Ue%WTj84wI3|iob!!U$dlI)M#9rJ=o7Fv9MZ@S%dg-Z%i<5Qtv0iKGnmu%*BpW zhpz48Aoob+u(=nlsv&gn<7VA3vOy6tylbnVB;u;~h9g!ZivwY6`fRf}-soqY3GEWS z4tIRz@P>-XhF60G$mx4M4)kO|p(dDLqy3YCfJchx{q8qq&5jE-cE$Kk?bVp_E=8BX z`GxzPz|wvW{`4wiHQk=EZfpqO$<~psF1P>6q9!g-QgMA}S!pAvCdR@X$-+R-;YZ}I z6xH&;yVrH})BO0-hdTuvjC=X&Vv4!|DFsY9SqI2)b0vvthISqX}fbbzqb61 zzqzQtVqkXgTsQLiriJtg!;ryi|2Ax`t%`NOfi-^hW|v73qu`lcj8*ki5`A!jq9`F! z-|SK@dZ`x7X{OLYbhwa>DP3hSjdr}8yGWE-Hp13JnAEg&UsPElZ(AOOLR(Mk1_$ob zssFa^@vOK>`dGCebxk23%{O2h#&i%m()_nlH`SoWu64gI~#wE zn}>skCTq_=Z>L)CA|Knzcxa_r?-SaZVO^mM(0%p?%5B_oX{_ew1e;Td{_QZbWofq8 z+)HMeGk?dVTPEXmlKmxH|0v%z*v;Y&;ZYq8>J}1rTxF!9p;<8P z%pi4hc>0`l(6xZh8f3*V?M?P?e}DbvFO?#fq~A)XCAmyfX{Hv*EAh--wJ&!_G(v66 z>Lw5s)9f;4umJkjEF=qA?S#p51(YtATF8idN(-k+V*NLBy)ozMqT)7joqDk5XS>m# z51tV*28E~519?nregd=()RcKkR=8*Ww0Qu?Q;SGQr31AjP@-P}I+wUug?HWpOy>+d zZ=Jn-k2+aX!j_%Gq)hC?QG{6F*Vzjn@IQP#jy}a08xq&xW zeU`&fTU3pX+|k4o|NSsQpFm|<|NBHS0s!t1&h|egBGQE%9YM~FcH|KtNCvL}41}p*^cfE`Q74*;SDHu!t-;H~^8ao3pLG$DROlpLIFUVi NgqQmUJ@nuB{{TzhY3Tp} delta 4541 zcmZWtX*iT^*q$+7gKSxnv1A)#CtKM^lo?Bw>?DLJ%Ug_HV`Nv!48sg1l3m8WrARR& z+mIw%G`7mV?;r1XyzlqpyZ_wxaXim+-sgEf*L5G)b+4MOn~{1z%sWTs>i+ZqKo}eC6?*@PNXxa2}Vtfjiv%RTDXa&g60T|WDPDVnhJeXSr0{L@u*~&iQ0I+dsaWwMZ1bTBuRJ_ zFWBMlc|OGQKZ(`bg{j$bLxI|!dg)Xrm1{=Cy(@8jKl6Wdgq)XHBXq^5sX+K9|FqGl z%H6A^phL4OIlImGL=R>_rryF{l4o2v(nqkB%fF^#1**7*zBCk~tb(CT-Zb0d%*wd{ zVj0>^xQI%h_cHlZC~e*XT#hfhz1@>*5Jd=cCUD8u;^H96r0m?i`mEiToX&P1jw)oNw?c12 zFj2oIggMN|9qH*x!sFp^)bd#}cpA0JH1~@EzL*k-09*M+!{P5b8R==^r;C&Cx(ljV z>FpDG&uU~|FlN zkq@#BrKh_Z4}Uq&!N@I0hspcB%prIsK5ta-{U3d58>V;(&Lv12n-_StDd)zMw*BT< zmwWsUZ3CPaKJTB{Th;s`e5cK3ZmMKGa0tUNg@_Mqy`rfnY>r~2FJZ&LheS9?$~mE$ zR`LCMb*H8YSqTcDGk>u)k#0n1^FIfj!#aMH_DO!*XMMA_?14osExtx+b-MAw8PfhO z`W?Q7LkwVULKFDi5;Nh-OYV!f@T4j?4%*3LLstIhFm*1BAcgl26*xuQL?nltxA2_7*Rr5vDV(!Ot?A;^0_P+~d9LaAH}&9@B!6{5nKAUIN*|=eT>(REB19bL6=jkKRI+&F{!3H|Nhe$_tQQ`z zf-xDH#$9J5D3$4d_~9*WEO!ZUqGe}0+WzVj4SVqoC})U8 zs!7~Dfqixgm&iaiV3qgCy6TRlhkNRsIs_tAeS$rgykV?__OOeO0}>5l0zB6*#1@1N z+^tK3821o*-Lt(&R*jJaN=g%(OsY+Tiy6C<5=C<}? zQMT#X^xORw^M|C@`te>)-c%8YdsPfF-a88E_9*q8qBU*M^*Q3zb&Kdax>B~9oqR42 zHE5d4aIPyHdKUQzJx(YYsKQw`?hvoXC9C4;D72uaY=!DX%TUxIy<|>t_?(mPXt33| zcbM~2LF>hD!QEg9>Vd2AF&njEpjBUm?9OzCeGq^%_=%86cG8jsD%#V2S`2swFw5yI z&f~jlNZje8*M-K*vYw)QN1^uviVVaOAavlJ{o>z^P)K<$DN+;5x{L&3HcJk7yJVJ935=dLUg z!yLNq6GN@S)UEPveZ5%2Y^i?dg#04SZvP$a9gO`?Z;q_!Ah-h&+?2AC?j zqdL*?F}bg!2obH8=9Qnpl?S^oqVN3LXtWx&$=P@@Qqz*XZ{+(xUbX3(kN2hs#MU7) zfuX@2LV*q<JEguRi7vu>&0!mbMH{{v%!F1X-K~X6m_(K~}pp_f3Lkl{Zg$zVCZ5P_rfkT(Dsy&I?w)o;r=w z9TA8@b<7kx-Wy<=huQQ=or;G7RFYYAAuPK6C;8rL1F#T0_m?|qTP$wmI_x&ET zba4fDTC;NwenB-$Jv{S!7lt$PMD-rW6IjcMYIFR`o+2w8*#TPjNF9zx(V3qMjZ_JV ze0iGe)zD)4YIatTEMXLEQ^nI9Db%jS=Z-cgfoZ3f=hs=D`Q}?{7WVT`qef?B;i7;j zB{?X6Fy2dRVM7EWUKKNVl2HgHA>k_WLc0b~G&x%blC5(}G|d|++P0wI9)+0Q0E*_| z24OjfWu$?QZB9zvU#U$}+X#rGf>t&IRq-;D1n%2>?81V>RX+z?e~gyx{xG_fS3-w8 zc=W*@o!v5(t7+ddagnbt?%KiJuTQB#Thj<-qI`K#o9=fc|ItO4$6}SLw+maqy~$kdE^VOvw1rS0fn*R$ zu#`WN8)lF(dMOePK>zS|;}{HloYrB_{#R)F>Y(!^x2*NKW5+hHco(0|IM6wEa70LY z{4*979f9+zBoHsqz2@<{+QW=b*|lxzB_Fp^TPnMJsN^g=LwbZ z`%qN+v7t#|2JKBurnlFX^=`e&dKYS7p2SHnCn;SX!kx-2gy+xG{NT-amWS|`Wlc6| zxTB3lM{OiG=`z{6+yH6oaOG3^>de=GFEx0RQEQ*TjC!WGD>b%~tO|~!aTzZI&x`+M zImEm`$=J0GWDm7DK3ur_acwbNim#%>OKCqm#%Xv#tP?$OEtCmy6W-kaa{P$^$)O7M zFd+D18;so6bdU3KKNtME`Q%V!=O+ITOTfE{U{Z4bV$U{Z(@j0AJ0rQebYv!f!nfq+ zcBS9hzTKJcU$s;0-fMB0)!+0%g>|S=O=G^wZaz%tUrw@WEwFE2xa;^QS1_KVgNydP z>9$b;ryT>@@@rb_&&NUMZOqr6(mC)REjl^3^m)ZhkdRfTU7TpK?w@T| z>)am*?`PGA!&A>wON__yZ}!UF-|Jo#fwwU`DB1aumM9gedDaKIb=vl-f)`N-jcXGI zJMX%~VPZ=mKXk3#U{e-Qp`*KD3M2kG5IKo=$efL#52Okl`!&lW`Tg#$eZ^7m^v-Vj z5oCPT7=0-Dxbw-vYVzsKvzs_G(bn;b$QVqUOBEa8*!73q#iJe@@3V5DOpFD!H*0f$ ztA_KMrYDMZ@Wl))gVCm)gUW%xxoHVtkN#%|Pn}-|EoL;~E{!5*Hl5FA(fMCf2k_a1(l) z>7<|l8>YLTpF*R~1lBn{_M6>gnNOe$u1D1SZymmyQ+He54LtCXaI@M>SPWCUZ`!;} z;iC94a>q)tW9FtmI(Lrkh+Fxr`Ysr~csmrn)x!69bqf7|b@gdkEmHlN&ytb8VS9QN zCn91zs%%5G&4V; z9&pFJ@msOl?q74SsXSfE(VYEbjKx{ZEfpCHAfpHu2hvxsqJAMQhcB0)^(UQHMX%l8 zdeLU0i104o*g6rrq{=x*^X@y;+*b`3uUB#y52IZ^?_9bE=wobbXQtzlRU8A|&^${8 zO6|nd=)MsIMBnz(!>|d>**`U#09@ZKWqtHu)=KG*oKvmPCrZou1Ervw8L0I($okw) zz~-7-8PRAwCka$y+$I&=8hsy?*q0pAfZru0!6pzKDf>mYLN#TTwlD}cfJTS5^o8}p zIp`+mkg`&kGR2wpa`u(|;EwxD}{2 z|9}076##Jlq;2_kL}A=<2#Kq}MBGyeQU3pqejWhew7L0rL}9papC$Bw?{KUz4WJ?p p0h8cA{U=Z$0ANlB0G$2z#gjP-a~78f(*b(m$S^SmUg+tg{{d3Ze-Qux diff --git a/summary_results.json b/summary_results.json index 8a36a9f27..60bb13fe9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0894, + "Performance_Result": 9.97432, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.00696, + "Performance_Result": 6.26821, "Performance_Units": "Samples/s" }, { From bf8d5b46479196d58c81c8273431557adf9f3dc9 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 3 Apr 2025 14:16:36 +0000 Subject: [PATCH 416/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 10225 -> 10228 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index a461978f9..346df7d73 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-03 14:11:52,496 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-03 14:11:52,496 submission_checker1.py:3260 INFO] --- -[2025-04-03 14:11:52,496 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-03 14:16:30,921 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-03 14:16:30,921 submission_checker1.py:3260 INFO] --- +[2025-04-03 14:16:30,921 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-03 14:11:52,496 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-03 14:16:30,921 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-03 14:11:52,496 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-03 14:16:30,921 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-03 14:11:52,496 submission_checker1.py:3276 INFO] --- -[2025-04-03 14:11:52,496 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-03 14:11:52,496 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-03 14:11:52,496 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-03 14:11:52,496 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-03 14:11:52,496 submission_checker1.py:3297 INFO] --- -[2025-04-03 14:11:52,496 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-03 14:11:53,689 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-03 14:16:30,922 submission_checker1.py:3276 INFO] --- +[2025-04-03 14:16:30,922 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-03 14:16:30,922 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-03 14:16:30,922 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-03 14:16:30,922 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-03 14:16:30,922 submission_checker1.py:3297 INFO] --- +[2025-04-03 14:16:30,922 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-03 14:16:32,345 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,10 +38,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409337 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.782 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436533 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.782 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436784 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0723 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9994 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 34d9aaeb0..408adce17 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1043 +1044 diff --git a/summary.csv b/summary.csv index e60739368..0e74eaa49 100644 --- a/summary.csv +++ b/summary.csv @@ -16,10 +16,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409337,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.782,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436533,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.782,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436784,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0723,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9994,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f10159665..53b75e870 100644 --- a/summary.json +++ b/summary.json @@ -514,7 +514,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -544,12 +544,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436533, + "Result":0.436784, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0723, + "Result":22.9994, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 8032cfbbc15b24346fbd6382ba6e17a686ed0e18..198f5edc4e8ba21636f74837147937714055ff99 100644 GIT binary patch delta 2992 zcmV;h3s3a%PxMc)cL)XExRpimlY9tUf0BXL+PixTeeRZejShajn@)_|Ad1J~?69Wl zX3Yp@y|6!?4G(Kqm+$u6nh_`6S-(38XTf3Zdl1)tJ^IhVw=nu69tA;SqzPv6VQrKo z^ZiC6?u~+JH?D{CV3r;kgweE{q+g?9Bc4Y=x4#UUP8!-Y?Z&h_p4Cy~Q+`C95P+FzF`g@Z-^V9@mZz`s4IiR*w+{gTvaf+CTYC+qK5g!Lsk`aqunP z`fOySd=0}tvM-aD2_}Enn2ytPscXixdv{oC){JlCelp4u8>4Z*A7m%GX7m9_x z9a8HAt59XHW0k2k(OzA(Oj}#o@9}*StYaFhUY8?$(@nZZ2T}OVh?YIX^KSObsC})| z)6&aYzhuqX*c$7G^ZN!Pts^^ow@1x`#%-2(t+dP#(D9m;_^|&)uW2* zBE?RnEnaC0;iYrveAGIjQPNf#D^w`4RR9wTaiA4PFCqyDn*c&1d zz`iAt)~@6okqCdD-V=%7=>w67E9fJUh*RGt67l<;qC|_c>l2AMyPt?eoZSwQh_m~d zNW|GaBNB0TzYvMdFF2cDa5f~c3S`=NrE`Ji5Fj|<2YOT#5cEA z-E1UL5G)s2jchH|_~f6gmoRZ-y_VVAWWMRm=X<^RVlSM{?xMvkNf%RlYt>CJ zoDIgq+An{#evov>leqTkpfMoR^_MD2N*Ap`I<)0~a>jMB8kbrvG3qVlDzC+bck0W66IutW_^j~flek|Y2u3lg-I+)g-!7N2yYO!>SI!a9F6 zOwN-;1QX>}0nZW3tywbTQk!-41Td?pvCMS29++A`ajv_LopbM~_=eN5!aW?zmSCdB zaU__iK`@C1z$6*~6Ez5?E&`Jz05A&@v^NJIz?3aM>4bnO!#X4ko~5$XatGyXtAKwm zm{rtR=JD!!V4GV>t7+LPhfKaVdaQqN%h7TqnPgkim1Lp@$s`&elW2fU)Zj|CSwtpD z0Av;4@1QRs~CeZ+xL<3-wU3~zvRRktU0ALm*Xm1WafGJyi(g}Y7Q-*a& z7*3faHBOnW{{t|qsIknA|@OszZTYTLV` z>Q2-1a=fT&$Gc-bFp16&Tbg%>#+B|JqCqB0_;;8Fm?Rz==^Ic*79M}r2{2h?;!)CL za|$g!8>QT`gs2XQB1jTP58PQ+Y}eT#@~_yptfatFP;R?;(%VYU%k$GIHwQ{If!NZ= zLo}{5@(>L|S;ET0Gyo;B(*Tr3ZXVVNKw0GHQPOU63N1bxB|r(b9TJ6$S7O}3#cRdR zoxdK+XX@v58laMB97$!7v4YUvXZE`L( zT1IT?^C22n`h186xMUj1CDTYQnMQJ{i+nyTf#k9%L0f~*2U>r8(h2TcLRg1}$@!9q zU=6D)_W3-<sY;Q^-0&UZp+P?F}2De6HOzwwE7T@E3H06gJhPl`Y;WU zNi;xak=2JK05XfLJ{qh((BhL$2$^8@Az`qPWvO8yt1Gtp{5@noQ?Jj3-uC*Wi`k~_ zSvmh_uGTqdqHll1mS!KKai!UZXb{a3W*?>jG>Hb#EHeAB1b}9d*=KX`0ZlObkWO&t z62dw(4A>-&Aeb~OHv9Z2norugtfa_NljgSDC!I8%ttX|sU^a8WMAwKdy*@5(5)FV^_*0L&tLo|qH3AYc^0GdPtXcoDBSOP$^$nB%S?E@`7 z=>$9l5Y=H(KxP@UkFMD4^B9@`tEr%p`bt5$YxHTEUbC6AfvK$=DA6=xOQR3bc+%)Y zGzeu0qYr=60F-1ra<4=e8GTqM0A-QUM}yG^T71%pGFy^LaQe_Nz>?U3U?E$v)8{cN zlR*ZgdeY$e%-eDq_@qUt8<)p+Rg!zykSSWe2B)AE+3*nAWOJ>m$lgTF^NbgBNK39v6wgr&7DQ$O+ ztrs0mgJ>9>1d~Z@^uqLXts*zAtG7R%tNrWMixMB?X@B9YKhECl(4DMa>zIsZLnHok z^}2&7n8t9zR!chT5vZ{f1#|{n$f=-tX>sq%%gD_jg#-I-m`aAE+*Zh zY1n3exg6ADHvV%FynNAnVeh}_*{^;v^ef}gNK3z$rqwNb$v&`Q9UaWO!{Adl8jfeN zF$o4~p=RBw8PV$HlgrOZIA4BB|D=5llO&wpe;Wneeh_8f(wl-o7$)~$vQx5oiRaOO z0ke%0kPZaV<&Q9vAtEgSLz7P;cz-^}3IY=>Fnj=}b|j(`Qt%OJe_S^J1jo>U0v$Xi z8NnBC0H^$9!=PL_l?;0elh(61x_4fV#=h^fz8tJEKf#}?m$zZnRO-X2IH0ByoN@;& z7}r$zx`>DCm=CLX;*A9HxR3h8+i8(wABZFdpvSmp<)eIhUN2EoU%oMe&m|CR+iJla3}l0UDFbCP4welMN?20Wp(ECng)w<&Q8W0RR9{ m0ssIJ00000000000001hffAF6Cq4mYlgK9{1{fs(0002t;Ex;t delta 2989 zcmYM0dpHwpAIFE0J;j()IW9INB$*IHM&>Lf=bU0CX(H#w^u%bf%`@lJm?&p*R@NkE zp+^}Zi!!IjF^A|~@AF>Qd;fc1*XQ^7Ucc{kU-yRns(n?2Ab)(cQ2#O|R?r(JBVQ(~W`$A0wpTHRX+akb?PH_W zU%X4kGklg@mI{6rFS)V5dyoCt)Z3N}*}vsaj}>AX*MKRRAZ6Tjn|Jd~9}s@5wsU2O zpw0%Y@v&UH!%$>MI(kLmvZM<;Kl&x#s z>g>^#ZasDzE>nSPkd~g0)J+_Yb&&4#UVijNRzk!22*q;Ivl~v*YayRh#74kBqGp0& z=Vo=InrtnfpqffV*sYP5usbC)9LKleB;RMll9hG*t8SN4{;Z@LFtrU zhK_k#NHXff9@P@nw&E~Dxby44>-+)q9+bQ6uYRVHdk3HPf#GU0vD(NXhRU%K#!-EC ztOY@vUlA7w=2MW3+umPXYWh`uXE(IzXv)an?{DWll)SpocAJ%DH(W3hMyDL;f@zHY zNL@i%SexVo%rtMJBeik_o2L;slk5KCmM&RUkeXU%QBS=QyJkPS>L~KeI;*kTt9LP2 zdun5Dy=^kKT52F5V#t?Tlv>*Ml2a@-Ai(xn%EY-DTQ;-N9x>tXkEYM>T1&v`siROi z)>=Ux*0ZdW?ri-q+PbMH2+NVR%TN432wFl|$ztgRPk<##5VRc0l9P<`c)F6$%du?S zX75)+TZvyD3mShGmI~~C_YZX&X{--HLzJ3-K7?HRqd(i}IGH(Q+mQKraFO0{>cyoP z98Ou85eoqo(w$vmg?T{@;VmcX-7+ir5ygw9iA`- zvWiD3l3)-5jARKT+~Og?JjgC0lbyGrLmE&~W&-L8+0wV;{_Zh&k9f|RA5&(&67&q{ zD(iE$c&^4$T_}|%e__Z>oF7%`ID->RwsYWBg%D6&R*LE>>cUDJuQoncxQ*`XRKIn-9{>T`N9Yw6F`$)c9N+uvBmF|KarQE)-SAN7J8ocdtI z9cOyaSD@xNKYpeq&Q|A<3Hv?Mn-2+-R~^nUD>?kGDk@w_>A;DPaxYj6qbleJKz zDhaCHXGE_1dWRo8FUs_An3osQ5TYG31r08b;HKD8U_XFMwo6_48IEmZh@n>eTT z>PuMJZScDqcvcBk7NmSb#D>PBMG$C53E@jq+qv{UJaDO>aY*Z0s=dK+@181QP-0-I z`pXGZeQuPg!vPWqYE)~p{HHnR;TvxCJ;y04MER4riO@P(wDK4=44z$Ls!b5+f=NTg z5RK$uNfH_Y$mR#JaudP+J(2}sG#icX3?DgZ0Sry>S>K#-ip^%x%bQ$kj&`oQ{|(QE z)_}>3?uNBAiREfo4*|0LuPC(Nlt|z*v;Jjz|7M=YwR4e>2Tsap zqEo=aFTn5Mh->wCi*1%pyhLO!k+c1b9KXY>4AIIrJevMGcM*^8?lS=tZ1JY?>poz7$T+%$U(JFiDIcs$} z3v5GMXHw_nXjpno5+=PHav)dCqfF=|*+{xZy71z>Yq~lLi{8zNyTuNJZn~bol!Ow6 z5eg42tDmf2SQOd0vgq!7(ffpIgQsD$>sh({2J@LY6N0jc`)*`%=U*1`G*eY;m#8d( zCot*wny!8(1dw5VPaeLc`|SM;#ZJVkYAIdSqZPsy*7Q1gclqZgSJHU!PZnaME`zPJ zUV_^QEkRW#YiJmu*jD!7A?ROl4J$13rs%@tq{Jc9uscq=nkP@x^cgsZxX0C1 zv}l|S?EIjXWTQj;G|Vy6si@l(5hA<3;V^YfxV6eQcUGVRy3}iSF;%-Jzl-mW|2yeE z<2;rA;Loy(Pn`81={q)UNU7-Y1&U@sZhheF&q0s!JMU3b$r}Ag6dPw%%MG0a??c+~ zVIY-{e*RLF5i+NQ@nR&i<8%)xya(2T=YbKL$IG}4u$>XaqicO@2h(UfMdiI#De8#p zw_4(c(IaEg7n=_UoH=9@#?@TlYHn~f@vd(cnB4YPa{yW9UJ9e2c-OC+L(`kTqn~dF zR)RrHeH^8zQ@_@?a&U@o?hInj<9l%JVCX>Inml0MSvUtHTTLHN!g$PY9hXMc-XBl8 z4zQ&q%-$r8C&2+asS`-)3FPlJw>tqGI6AK$r_1saq@-Q$NWv6s~9~hFRH(|@G&^oK6-qy$>3eovn5$*x6Iw5-Xx6S z{MvD8V9otruBfH9JfNm)mTABPP4z%Zd;C{4<2oA3m^EVpEY~LG0c-l%xY)IXjmN;R zcV2E|+ySMz!*@S^iXQr+ll*{z*2?a%IAvr1Q7v?&IeL%X#Cq-3rn+kScuQ(vm%Z38 z{JER+@=&6{W5HZi*HvafH9nM`Lnyhm#fI+0Kn5~j({aPE9-tw1E5=0pE#u@G`*~Xi z^ay1SJa%HNwPXGJ>)Ld&bYUP(Xr!0|Q`F$GqFhvr#NA7i04lq9Zv$+WEKw>&(@Mh! zjkf@TkDl`J8${Y@<4;Wj>~^{zm^Ou~9ov#rE)T?pt+dJ#JBf Date: Thu, 3 Apr 2025 15:48:36 +0000 Subject: [PATCH 417/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10228 -> 10219 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 346df7d73..edd96deae 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-03 14:16:30,921 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-03 14:16:30,921 submission_checker1.py:3260 INFO] --- -[2025-04-03 14:16:30,921 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-03 15:48:31,028 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-03 15:48:31,028 submission_checker1.py:3260 INFO] --- +[2025-04-03 15:48:31,028 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-03 14:16:30,921 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-03 15:48:31,028 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-03 14:16:30,921 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-03 15:48:31,028 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-03 14:16:30,922 submission_checker1.py:3276 INFO] --- -[2025-04-03 14:16:30,922 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-03 14:16:30,922 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-03 14:16:30,922 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-03 14:16:30,922 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-03 14:16:30,922 submission_checker1.py:3297 INFO] --- -[2025-04-03 14:16:30,922 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-03 14:16:32,345 module.py:5255 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-03 15:48:31,028 submission_checker1.py:3276 INFO] --- +[2025-04-03 15:48:31,028 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-03 15:48:31,028 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-03 15:48:31,028 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-03 15:48:31,028 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-03 15:48:31,028 submission_checker1.py:3297 INFO] --- +[2025-04-03 15:48:31,028 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-03 15:48:32,081 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,14 +32,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.5935 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.291178 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.97432 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1069 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409337 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.782 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.436784 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.431058 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9994 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 408adce17..026c5e4b8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1044 +1045 diff --git a/summary.csv b/summary.csv index 0e74eaa49..9702cfea7 100644 --- a/summary.csv +++ b/summary.csv @@ -10,14 +10,14 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.5935,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.291178,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.97432,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1069,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409337,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.782,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.436784,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.431058,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9994,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 53b75e870..4f402ae18 100644 --- a/summary.json +++ b/summary.json @@ -339,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.291178, + "Result":0.407176, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.97432, + "Result":10.1069, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436784, + "Result":0.431058, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 198f5edc4e8ba21636f74837147937714055ff99..b266d36f797460bbb31f24ab0624dac874208407 100644 GIT binary patch delta 2835 zcmX|@2T&788-|k*N<>IOIRPO+2%yxTb_$3fMM5}0nsfze;S3y#sF4r?QM!}>f)r_j zf&vnHjYthOLE53xqy(f$6|h|NpSzve*`3+hXP*7$o%wd#a>cTupX&(ZTYFsq7yuw3 zsAN77>IN4t$vL3l%(2j^D`YFA6Aqz_;fqg>D`GdZmHPdwE#W|+A%OSw;oerHShB^< zkjEmc*r!AcJ!GnWX7J>_FFWYLIJvevX0b`^D*Ujdv1vh-LwsNBVzJVcMjc;_e7CQG z$%4Yn!r134Z+sXnTNIePDHk^~BN{eOue_6p2X zB29_WvLBcpb<%Bo)YW}`HN+7_4=DWs<2r_YduQW<;6#ic?%l9ny`V`be_(fglyJ31 zMAcKGB;9guV2EJ-U@VH)G^16lqOkV&MtXV@jF3kA&g~(;c_=WHCA==rHAZYZ6G+=RcqEwV&h#l`J=Fn>BW7r}S7nPz29-ye{?1xiNBVqr;{g?;H-d zu0^ITtG!rZd-nTKddtt&+V@EF**PS%2 z@zuk`2#o9OkH|ZYLe?G*c4KyL8a=T8L4=LHi0;Mdb+4Aq1s~`sH;_29MOj-_AR)+< zw``LS93x29XG+23wZS0cu_-BSilA|AMCe3hegv1zd$O!{cRlm0k%QS${+d_KbtGi`1t-+V7jn<1j0m z#|>Jy^OmPcblclQ_p=cn-z}Z2NPH)4@9zAORJL5=WfxU~{+-2Wk6kA!($`4vX`jM9 zUQpKII(TUu-?ckE+DWK=GyriEefdns0QrLwod!0Uom>_G!g#d z>{%G@mD-ntg{U?$a|u^x5&k*HRJdVN+6}Cwo7q0;p>0_ayt%y}N4Zp)Qfb?y^(j8A?lJApvN=p6?hg&X1(c20kx zP_NA2`uA4HXfBHFROfmT(V08wQXFg9) zRZO=GB&;tSyxRYoSIsNdQkxtPjasgd2T(J}0P_{ch*LENp;r zeSwSe8lmH?97Bz=?M0Y?nqynu8x6Y5XS%6P`e`+Gg+(E@Sl+;yM7HF03SmX7Wg^Tr zmWNbxlcNfO>*jXsG0bz9kU!+!bckZfo;P${2OdorJ#T6V1*HtWZMa!h)2vgnrg2WC zE4C!KNoRqJy7aZ;>D@?Q7xo7!QO`;WDH0cknaxXT0or-axMu^X#)4TnZypW>M9RBJ* zmvE=La0_@B!dct=%k61t68V8oiz9s#8Hh58A|UB>bZ3&Su*f|JP;O=S-v#tZE+suS zao8iz&-eO;pX!iqF%7;WZA%kI8%1fy1=VFO%KT-4o3Qh zVT+Pn)Z#xT<76H+bT7U^m@3C2sx-fZ|e-b3VQu8iL#HRG58dJjqTMA{?v6c-rOuZqc(BWTE5OyRwfFx<%%LT8O ze@P}n?2!l_e#bOX;mQV&(!nXi1@)Z+mnD3;WkWjt@0`>v(t=-hgzSOIb=wmFKGhe{FHM7D0VH4H45AhqzRQ2aP+!iVeRM zXZ};HrbtlTB_Bl zTkhNEB%E}{;Gd3`Pij5k$lkp5K@qnIz>Q01EH=PE_SgHDrUdNEV>A|+XW}}0_UzUt8c+X5 zU1Mr|YIPCx`=8j4dOo+7rRIvgE8Cwgx1Kmi4fHL1I)%XwFQ+woBF8Hp3NE~N-*_%2 z&_x?h2S-#{(bnF^`R;7?%rDAhshodGVzoTLAS~8fd?fu+IJ5b!Lu^cHVLH} z--3fLz0zjL9&oQv?9q(mW5&7Z3l~Pv2jgXP=$NeHm|+tIc_!qflO2>>or2>JuotUv z3yTGro^3Oa7W^O<{^}ZU;#>6Vyzsg)duybb^EJbX-(La;S=(^z1}VkHGLSc}-_*!Q z#s|gvI$p&4*MmGDgrSp#7HJ`zb70Gg5#~Ej-9ImqUgg%#BnEkC)ZjC#?gj|#tADM& zOI*Jke}CDc17OGPLvhS?n=GN%8lQ17%$}kMt#fY_+YHHl7>GSeq!+-#js~hQe7=9J zbS>~p8xRiEP8bf!6jAC9!E9b1i*H(K1ZjJtpo_P3O$d*L&Mo~bJ|30SFE%Z6D zU;W?+r#;vL04F$Y;@=TQ_M~EEje(KWBw1Y`oH`(D0IZ?HxS3$f`4R1ONbp5|exg8I$k>2Y=gc;yM(4KVK0=eQ2eDT!0V| zFq+bP>5Vd{I!`Dh4n&j08QbaQ@5fG{Q?_@RYF;XxKG4KjYv1B!?|AU_IZKU4!gxaS zq3JYj(;#_Bqaduh=%2w8Wk2|e5Mf9IIUkxUA&S0b z@o+`5fPXh>L2~&drYsADyk$#^7mNf^MaojkwQbMJf+RO<3w`zx8#IoSkQ~vl%t$V3 zBaEbhkm2)HQgCxLh!XisaNhnAp3t$>5 zvjwF;)WdT*WGkwraZjqKZkZ90I4DywrB7!hS%0pCG}2L&kfyx)HL^sO)-{bRcpjQ9 z=^=^4N*Oj*NfeR1avPRh(Cndpg-6-a_zs9tM|44ye?-i7A-1Ay<59LX zz6+w%5k83WkBFI$=;2Xz&?vuk^jd;feuzLoy~hXyBzuHF!0bInAYgKxAP_LSP7#Q= z4LL&~;O(Cy5K#IH1k&4vTnydz!@ zpKzXT0M!0ahqOS|#s~!DdxSthzQ+gzfZz-&6*K#+y#455Q*biRXrv3a&x&oAM? zM0d~zwYq|wYlH*l3W5ML?)G=s@#yTo+vDlv{Q7dgY7@BZ6PAfsm5XLbGb>BwZ+|9x z0C`mq7SEGBCQNoRWG)C7b6zg8gsY~2&mTKY+Y*eBs{OE3mtkE^#%n$!T<+%FTCV0n zsM?B#>>_GYjj$*i#fzXUT+JWdrq{F^>)n4I(mYO<<`**}B1lqh9t^A)+1&0vBPZF< z5gA(b@13}&bzDxpAvfEO+v@tb=zqGn=(@V-z%)G6AuYJdBM>M$1Oi2eK%nRl2ozm+ zhv*PG6x|N`MHh+=4oq|hn&=?sTIYbQZxE}Xytm!|q3Av%rw@qkzPlwl+iCk=@13pA z$5rR+ssjzep$=(*g?5ZUpz06^R2>3=szV@PXFm0Js1BinY|nSlFMOyvIDatF9kgMm z1Uc6_2P`Bvh*j7Y`Jb!0a#7}@Y{(C3W#ZZQ-WOo|Zv^-m>3u+eGyjbMd(B?A-E!^C zp96$wz4@xn2wRddNmFiwRFr!L^mKFrJnvF~mzDKY#AYb0OcB%Gxxd=eRzXFbbAXmWY?yyE-Q+(_oa*Xuk@o z%#)ub+20HIy#8M39egp|17m2&8f>64yUI&-aDaq08Wh2jTm@{I zNk?@pXi;6swtYc`pxNebMS_Shbtk`bV@k#5MwR6C0BQ6KP)h>@6b=9Y00;m806zfU zxRpimvv&wk27k4u)Ye?KOJ34`j*~pL7Bn9}v}HQ2-`00|A)Uvj$UagZwy?Rg_nXNo#i*>uE>j?1-owcaB2i zhb`?nMB_?-4$&Z!B^)|T156SRj5LL*B9jj5M7nf}5`VN6_;jGfC!Hv>DWL?f4h;h; ziAWmo&Qz6tos`PO*K{&mtg#eGCb!e&e3?y@%B$-qK>19qJLhWKyQAt()AMq?sA|W% zV?Hp6&JSCfcZkN7?j52*CQJBtmCV)(J3KWa3fMV{-~EJ{zUnvV^D( zi6Tf6M}H68SypV<*&*_;*te{tz*10dyLi&uO3%yl(*amtUU_6J<#Hl zPKciX{5>QL{RAX6`Uxtw_#l=)-%#*OZ9bQ3+vj7~we9Mh-4ShaE;U+4Z0YkM8dv&! zhz7W18p$QoNG_R1a;b}aJ}iOcvM51YgU<(Ae9{T-TS8cehROMoh+qw?EB5(3#^q1h zxPPpq$TI6#y>0bL*RgKP&6zQ^${`a?Bet~q5REIXK173LmazIT4UkDRKxUEEha~_q zi>y8xtUl1aN17H#j zfLY}AVF>`tBCn4IuMf2Nq!R)rczsA1&Y2`N&Y2Z^ef}OWpQ+jBLT{UWmUE`#=6^hv z)vX*f(KupDw-3>{((OYuh-L}557PjeL<49RxqVmyK(om0qrvS1Ek5Z4JOvQdVNpP4 z8MBYB*zEHdng6S)ppyDZLAh)6X_;QLnX`eZtsE%PG-6Am57Bti=tDFJWeKAX(*TrY zJaVr@7a4t6Cje!U(MN;P2U>j6iGMO%l1gy;&@jM~*nwanTd~vUF)EWm2Bdn@Kt5NCkG2Jnx+!gUjjb0QPJe@F7@P!? zNo@4O^mMHvH?6C;Kc1`o>(z@AALMC&;jBN--t5qwtX}JwjAuh5{&V%hNIQFBq?sk_ z4bB&nz(~H&gTq>IH&1_|tk{~-zZgAKm&q+96eoFtOeGQW&oZf#M1>JrSW#7`9f zFq0c1D}T#QLlL$EZON&DLQ`6L6Vk>JwSJJ)I(hplPU0lsQ;(~e{dRUywb-dH`32{t z(KRAjjuOz^2&vl|y|0()14?{gTCmP&sL>ugS~PbRx0D+Pug+L-L4r>rNPEg{jkY0J zim~TgP|Ro1p`*AlPO%V6*J8`~H){bGc|OMq0)G=MFnj=}b|j(`Qt%OJe_S^J1jo>U z0v$Xi8NnBC0H^$9!=PL_l?;0elh(61x_4fV#=h^fz8tJEKf#}?m$zZnRO-X2IH0By zoN@;&7}r$zx`>DCm=CLX;*A9HxR3h8+i8(wABZFdpvSmp<)eIhUN2Eo?~cPyzq|5dZ)H0000000000fPoT|gC{-#W|P7vBL)~H00000Cu20& diff --git a/summary_results.json b/summary_results.json index 4f5ba3405..ad6a8f1f5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.97432, + "Performance_Result": 10.1069, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.291178, + "Performance_Result": 0.407176, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.436784, + "Performance_Result": 0.431058, "Performance_Units": "Samples/s" }, { From 8200ed122f0cfc2ca816c30a7265d83f53596f0c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 3 Apr 2025 16:49:18 +0000 Subject: [PATCH 418/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 10219 -> 10227 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index edd96deae..f39cc43fc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-03 15:48:31,028 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-03 15:48:31,028 submission_checker1.py:3260 INFO] --- -[2025-04-03 15:48:31,028 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-03 16:49:13,350 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-03 16:49:13,350 submission_checker1.py:3260 INFO] --- +[2025-04-03 16:49:13,350 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-03 15:48:31,028 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-03 16:49:13,350 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-03 15:48:31,028 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-03 16:49:13,350 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-03 15:48:31,028 submission_checker1.py:3276 INFO] --- -[2025-04-03 15:48:31,028 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-03 15:48:31,028 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-03 15:48:31,028 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-03 15:48:31,028 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-03 15:48:31,028 submission_checker1.py:3297 INFO] --- -[2025-04-03 15:48:31,028 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-03 15:48:32,081 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-03 16:49:13,350 submission_checker1.py:3276 INFO] --- +[2025-04-03 16:49:13,350 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-03 16:49:13,350 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-03 16:49:13,350 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-03 16:49:13,350 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-03 16:49:13,350 submission_checker1.py:3297 INFO] --- +[2025-04-03 16:49:13,350 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-03 16:49:14,596 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,20 +24,20 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.392716 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73997 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72994 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418194 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26821 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.5935 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1069 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.4817 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99679 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409337 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40724 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.782 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.431058 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 026c5e4b8..0d38a9b02 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1045 +1046 diff --git a/summary.csv b/summary.csv index 9702cfea7..f4aca82a4 100644 --- a/summary.csv +++ b/summary.csv @@ -2,20 +2,20 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.392716,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73997,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72994,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418194,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26821,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.5935,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1069,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.4817,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99679,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409337,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40724,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.782,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.431058,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4f402ae18..4d6411e32 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73997, + "Result":1.72994, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.5935, + "Result":22.4817, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1069, - "Accuracy":"acc: 76.000", + "Result":9.99679, + "Accuracy":"acc: 75.600", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.409337, + "Result":0.40724, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b266d36f797460bbb31f24ab0624dac874208407..3109cf4cd18442160078ad579b00e7b97ca06c63 100644 GIT binary patch delta 4545 zcmY+IcTiJJ+r~qYfKo#5oj}A;grEXS5l|o?z4soFA`p5BO$h=D3B@2PM2rYhBoyHl zq&F#{3jw5xh#ERl{o?b^`^%kYd4T%zCEMYc&&y6ss<_6Zd(2mC|#k$f7<1 z=jSZ@OoP;EclXPZ7bS9YPt~|rQZ90!rr7POTrvsj>Q*S1+I=u>(4ox88_#pZ^ywQZ zvUv8l+L6y@=R?t0y*FZ`THUGLhUgDL1=npy#@A5Bd1FQzX|~x%H|Kfa*tK(e9S?J) z2_ANpxT9e2ah{{D+y2#%hjh`MNlLG9rI}o?t^3R`s+I}`UuyGx@{tuaCm|5A#N%b) zu;T6H{=JWv8ezV8w|=#n1$8OYaNa@ujJ2A-(377R%2q&E|NdO0r#~-TR*p zAwHlSnS4uLLTKQTd+<@)yvfV@-!<7X^L9b0?ys1!RzeS8<+BW6UT9pY==5DWm5J~v zM{@8P$1TmxdBd`DkFYu`rJvnvZPwn)_D z{TI#ZVJ$59ads!rswg|r30(mpN9F`%QW79xDM0**q&q<}IxNIy_%LBABJGX!FcJYHj@&pD$_+^rYSSL>vicqRriMa^BJ zr!9-g)Y1wI0x_2UlB1=WXx0&>(AK^_PG6Tnwc6^v+d~ zr?{HVH&=mYlp+`Z=K+&D@N0@A!nN8Jm11iGiwTWMVyzkfITa3vsU1`HBh^$tw#-(| z(90jEctNdK@%+`}Mx2LFQT-xX&a1WPufaScH^Y^X@(|uoXZg`5<%EP`QFY@_T<2a{ z9lMpucYG+wm^w)cX?z+Q+E{R$=yyq=CcV^HCFz6zib4{1o8Q?t*c@TPmU6a};AQp! z*1{*w9bb5F+12${@7M4He$~GCNByKS6s1$8V!396rE$9cne|kM9%`yfZ)!phNLVap z^=-2fNz@t<2SL>6Y4uXIioP|e>!y;5zlD<&noN=RT=D88U7+fe8qhN?+4kIO^UpIj zJt{g*3wbanH)0n~4vnGshly0Gnb-&;K&)Z<7ofEs^t4t~)OQf0r7$gx`p%mMoiIjW z&zP3v_(koBHdJETuOb$8aBt(Js;hXT_S^RQ-ebp!+;k;QFP(V(OPs^ck6Qp_lw|9$ zK8239Fe-@t^WFXEzy_ayGk-DNok5#0!*sClnAD<_YYnn4b86&h#m(=)XV2y^=>5^# z;da}K;CS;F;^=D*9A`w0xA&+?=`rt4LfA{^pz|H$QL|%fjxBPzb=vJ~r5gESQS3ws z9)-Gyusi0`XArN&cy3EvmnOXFBkzyK)8^J2zi$@ODM{5Z2$l|1Zkr1WiMeGmG=40x ziGP=&e1}7zPq%{LSoiR#5UmzuCkvc%sKN>jGI0&M^v5?r(Rj* z*wD6x23^pu3M5-A``DEkJzA)7QiPBqjbV>mms1c4HwoE%N|`HJi0j#Fr?)!vJ(;8) z>M_kcO%~)U?vi;1u^5U6O^Um1+b#R#*=um9k!2U6#10jCy|3*jyFPc$G83AeEo(wO zy_2fR>;>k|E7}gZQO%uqXHMm*180|f-ihz=yY|-`dEj@tnI3RxpG9;l1_6NMX`p6w zJIl9p-bpb5J|)oyfDbtD{P%?#o5|IzELcu3XPZ%@Ra@vv$lZd0)^TsA8;Jw5eo$qX zN+4l!xNxJ=V{P_^Y|AS0?LA&pPYoDX8Z2ZW?U!YQr%_5{=+$Mxyh-nsVoDfDqW?MA z22}D&H&GM!?5}2JN`CzYS~n}EO5Mws3*Ou+c~PxRBA>r{kNVly_lsjGk3=rbB5SFz z=_g1kR{JkbNGKWYq*R7*6@Tay%S+#l7f#RMlpP2!3k!kOvORHl(jC_* zD(*c&iWBuO56>X_$5zi1L$BqnowWaTfeu$gm9cdms2gU>mC$#h1t9FbZR1iG*BtWX zUd?qMopI`ko-vCu`pSpoiq`aBCy5K#*dx&lIH=pGn}93kvulZh5%(d|R%W!+BB!XL zwTweM+KoQDnf0S6tw-B(4?wPN&lp3GOuYK~zTRJ;w()9a$ zpI5aZ?+aT+)aI|vo2l;$?<5+)P@hLX>cA>tn;;e?yY}`DSgX{#5&ew>*^sU3I|+Rf zqK`3Go?0NbrXj19VBIi%E^Q2YbF#+$(-12L)ivn_K9`3NA@^bDZK4lRmQAXbc=pab zXAykxD_klftDhrWgg_&k_KKkt8nR0T_Dh&N#^n7Mm}pr2D+}V-%(25->3k`ia{7EoK_ZitJn(xTN|mOr+&p{SPE3plkn`HA|8p31xA3DslCnH5JrWaj75 z@6)44CQRKR8!41ru@yo4mAT3);2%(qL>F> z)I``BWP9<{M3aAEc?TKrwSGh#Gft9gHxymsP1FdR7Bmi_Wylo zcI8EF40VyOyaxk%Ow{hGIk3Sa`ux9Ur46w=9L1r#bK^zv+Dh=>aZmmU6tS1m!2!VO z#f!4Uiz2oRWKxVakFyjDVYoS<^x(sv8Ys3W!{e0D;)YX=>5?}Cat%ZB1r?}WiLWmMv`YdFH=H~+;!?-N1Q=x6JUIlL6>m6|_Zn|}2WPGM{fXXd=>$g2pj@tP zJSOs9N6M5ps7o(TlhA)0o3D-z%?tW^_eCe~bV(^NAK30g{FLz-+?Y{;a9i_bT4g2I= z!{%>$8@TgJ@%MPMa#upY|_RM$FiEZvpT~{583~5#O+eS^iiufoVpoZ|S=FPq(=r;=M`NGl+Y> zlwbRfJJIOig8Q&O)KSSWfucr1_;BnuLYL|56IoH`E87JK|`2LfI}kzLwpJ` zTw~x&p?Y&kKuxmeiANkb|2W+*KKU6r&;Y$W^KrZIZfo)EJ9aXK?c&F!q;~Bud?5b}Y?5$w+y8#e0L`T_A0}pqqF=j?`D>j=B`sURvJy zzDZ~mI_MZPjD(1M8YT$W}yhWuF(x6U3{x`!cd~<32ENLmy6cw=rv6 znJ#{Kt=FnT*TBa9W>>}5>SXA6ZJrU?${(5E!1Q6W)&|0&;6Q$mUxeh7I%hs{e!VaM z&<4pCW&P)L-LG@|XUs+Zs&0hRaZHNi)VgF3YV=kVlfIMIe@I1hnL^d}HDU3g5trL! zdfS@ra%7M=weFaHnTYGz6I?I{w9H3Tr;~mT9`xcr=Of(6W_a*Rk&B@S|8b71YhSGSlQ?NZ_z# zK$%bGv@U0a^I4m2KccngJ$b=u`{98^T&?0Gd|A9oUB!4v!BGR(`-;|AwSM<(5@XIg zs%^9Om=_8-j!aeKtK8mHsl30bVEwJ@e6wva@1^;!H=5#;%i^K&nMUspV&T-c%%8_fW@kKLEkrv zcsjNCEV~RomsfdI;rUThPh6|*yuFfbVD+!Qs%*T~iDvx`I~C!}VJD5-^SZxV24Z9+ ze?=~6TDi+Cnt?^m{Gt`6gI);8Nw*lh*qvyvuEMh2GCz|)9_T+-BmBzJcc?lb=cn&jkxb1kFk|{lvM8e0buhpdd3v z8(i{uWn>ju@0bzzb&vTQx^HYJwjpT$v}H}zopOjg@t1bD+(ZA2z7lNwA4(r*Umzpb zeF$6P+ES-;&+Kn0OaHBab;IW$CSvxR`7^c_QU4Uw|JKvu#4dhw#&Zeqg#~G>kc24e zPKP2Wg>qeQC1$Baj@#Bimgak!p+3D?F!4u#AN!x)D9qQ) z?M8|F^MXn5JYurO!EMD!@5ijoMGYN1P&(6BD`SVDS>`fcnHJn@y+u*)x}PAnv*vdjylP{*&d>s5PGqm_dctkXD8+rU^7*? z&p~h#dMqkeu5T&ZGrcN)#rV%Th>6ab@pZaKjX|2)hd`o}WC|_QzxO2A@8BCk|Nfa+ z0swr}@B6=&km$>ZO<)wjI!VjYps{Fa3Bi940B!()>woj8V?rVawo6)@rWp&AQKM17 m-jtCR{6`rY0Kk+M0AT-rb3xc-84VgQY`csk126dRqyGcTnR0Ld delta 4542 zcmZ9Q2T&7T*MO4{q$s^8AU&Z;dl68I^cFxsnpEi}NRg^ULTJ)^06~iMs`OqXJ)w6Z zARry22m%5sANbAt{WJfa+1WBX&pzku*)#X!%hT^2!>&dF z5{B_xY+4?<`}*fGbR?R)^Obs3!d9)z1e>y2Uae8gV8=$0z@lg)aU@T_qrA%cz2)_p z?-pGSxb~wEN-H%2Le{hh$~8>&tJHYuC6tRvqJm5W?*IY28y@!+$|&c-{2H_#H^k}SlAR;@<=xKgFw;fl zu9wTmZpqBiy9BX5r`BmI79x&26bwjlqus!_frux=m^!;;1u;ZLsDbk|!m^%0*p{U< zReNQ46rmeB6HcU-*1=Sn-|*lqN^oy{N_uvV#a=S^ru8UQn@hKu>MOVKk33SZl9*%m z19-fZSe4F9yZvrk-@^&A*gaioVN+*b4(35crS60)O5<1<*BY{(X&m6qZQR6Q&2l|3$bNZT>5`L}Ef*DROmqk9j9r&m;(!5 zgc$H8I^oITu4`$(hER4CY!&LAUzJU|L$F$*{NQElPi1yl9#gajJ$e-`ra@G?4Xi2K zA|H0PY=;~|1}Y>P46z)<2Bu?)=1MlDbSVa9esoE1uwSt*uU%T|{+tUbm~?O9ep#z1 zf#JPF!8vrCtPK|<{jd{HrBK65B<0M9hyl1v|8Dt;&$*0XGwNz<7nBqxi}O+=DnBF! zN0JB1jj>bmNP_&p!T$KL++YH|g+MOJ{-!YrwjI-MKAwH^OB7G@NdFdv z(&^|Tb>^-Ya{nM$J2S3qc*cN;F@%$O9C&jV94I%%PFZ|~GYFHzygeIS%(-G#x6R6T z{K(l}LSgh@j=ssHUY*~(TNyvkVQ6K#iYnhOwKE7Wg+3U^bw}+Z`7wJaswIc~QzB5t z#Xfj*)}`-cX}TBD@OGHQG=T&w;u@k)|G}y~Mtp{3-7L^Q>`$>_lOIDQcPJ4yU4hdm!$clhCm{kb3%+(T#f^%#{WvFE}}? zG$&mL>LSH9n2zhR?U2y)V<=tFdU{I;n96EfYtPk2y=Pq|d3ATz@fVGb1_78B000mJ z$N&IYfC`go_b%zxX-xpJJs9O=a@gBbmQoLIxZV32y(iL@`TX*m9h`10neAYHj$^cJ zu6}CJIGGPk)98GgICAksZ0sQ@h9(zCoTj4NFG{VL`&}vPl!=TW0U8gXhdR2EyBdYi zzxT@3lsJ<&3Jbxi#=JLnIvd?|E(_QQTkcXPjM!Xj+U8!*=^fS<6jRx~aut1*#wvTR@FmAb-S-N+mMQI3}{5L0W@O7g@Pb71R#7xe?Y{FNfMCR3JKut zl7!sKDyW|HAX>YM1tpYfFln^@eH&q!ZVD@dR%>SC4BECu`WK=-NcBIr%UPd?m*<21r#RuuL>Z0%{DvS5V9ULIbX$BtX4T zZ?!vGX%I23w2zkr*MUza;XkK|m=@+`WE55K^&xUcH7tD-H76SjAv7p4wJTnl62Y}? zZDc=$Bw#QIygT%XNW?BmTQV?)B7D1&2Y^ls1ZeIU2lLj;`)6SJaQAxh>sq3ZYx53M z2i-S#MEz&SvlKseB?^m0I~4MLhDRf!ahS*FmRKs~>ES z3TG#V=L7dOQ3hKxuo5%B$J|{#j2eNq9#ZDRZ}Cbct;&H&+_^jdb{UzG*EVUA{7U)ULKt3yy>@BrA?oP>6uf$g$YCIGF>#R7dKUNri(<_S(f zBYZoaCkW#H1CwdjEW5JL=!H6lvY|%=cE{8 zOM-ro)+ErOmLiigUR}RpCMo-~KzIMk1SMI{u|H{i1?^=vYc>2ff3}n3SLmO0v+nsO zoVO3Y?qY_J(9eC^oszt0Y4<`|EoE@vqO@K+oR9j9r?|rs)6ovKK5gFmgQB=*Iq>N(XZz<3lBaJ|8HT~OppSJ&6 z#*-MbQs1~~oWe+7-E3VpvZ%Oz|M=W&)1^YYIo0KJR`L;Q-6=De>s-Yde!|Z9^c^z% z#u&fVi4~(9+7U3sQA42taHLN|iW9Mpf@>(Yz%K>aKPYwuJpPIz4uaKK*NnvFp5=eM zmK1}#wjjr06L^Gf^G*BvM4DbPv*SD0wRKKO7G&JH((+1LjjrM*K6{;0=ur{7O*vYWyy*CYpXF{(OmT^lz4HC#SH z{j1pUSFs5x+K9Z)Vn7ifSCXH!pWo(Wj4mC?!g+)eqtn;5;N>Y8>w7NlDK1=>OaOg% z74HvrMs3C`S`NR}4=cui#AwyMN-MCpSLGNFHIvaKkW2gy zh*uV5ohAVouO5yq9R~hwdKw@S!7Hv@;^_$>egwOUrpP~qu)QzqQJn*+EhN>q^iD^S z7s-p>(h9;t#1U4AX2XQ^>8Ux^k*Pn&+sdd2io@ED7t^c`Za$k@h85nLy)`*hmrV$( z{LD7t@xjF78yitDOUUBJCo6%*^r8+OL+7E*MKZ&RNYVANhta**Q-ghc3*T+@-k9jO z4l{D6f0!|VBNVl!0HCBb$gJO$CyYV% zb1I8H{Xf@ee$w5*c8J|-AnrAE8+E{Dj}Sdo2E!MZg4@RcrY~3&w94`wS6=#d8Z%`8 z%k;jeOeEPSIc7CY24%YpCX!V+mYy-W;p+|V@kz8qtcYTB$QJ2k*&IT>awUdvw34^b zPr^8;Zi_PH2xV7zK~SKm;ccE}i@AGY$|m}Y%TGAU-WqpG=(}{`-*NGpbHO{;9>+y1 zPODV(n5#Kt87dP;(kW3WBzWnj&W|eSc-=%vv%_VBO*A#DIIvoPJAvr7O+i@3qfk$oI1y(liz?rJY|QddbHg zhZyo7p~6SqqVzF^cnsb{Z!d>AjIx(zHKH;#u>_P7oOIP34j@I$p-}zSyZ}S!Wf1{= zMUEP)fcn=)*JqDiIO{UZf6tN5KUvZ>*?plMctL6=c}U-G2K8!}uAtDrno@_!%+mv$Nsv+1- zYLB&(OU+TY%y<2u4M&x|-Qu*c-&P%N#iWF)DqTUl9pwAb`g+^3UQ=$+i{HGF0CmIh z$WPWW-WL8bv>(4-R_$V)Hp5F+Mv_mtI#2DsNHMtg5nPr~OOeF!eLr|Q@fz!YG3ofv zckh$@3aU_oOh4ho!7XRV!&%%{Ys1SNrmQZT?eoah3;4 zjhwP^cd^V*4t#PKs$<>Z9ag!q@RGcuc5$yZ`~BP!Pxo#}nA-X8r3$R0N2uVE**qj0 zzx$|qNRGPkty%TS#$I6k!V2bDZCU$FiY0!)vrDxlR0wZVJ3#&h8ZS_~GdNtx-B0et z9S@gzY0>*Am!@wq9b&%VZSLka&)K|^Zq3qq$3jT3ZwBOE5VyL#X4{-f;buQ{Ycmo% zaY`^E@)2{0eERLt`{=A8rRY)_cj-m;tSzH<>4c0Zn=|IFWi;bIcW=bfo)$G#;cH|c zFh+m>RMW~}O^g!kIjx2Q0RSBk06=v$x3+b&(eZG5;VER}=3yuJ%EdV|amcAlm@;G^ zNt)LpAf=^FSx?}CEeTZ6Gx4Aq@1?5qkDrwEy}awbv+9={5d<2e+|De)1NYc}GCyE|YQ*wYfaE||)EhKsongb-k zNo7D8+7XY8wz!xOYrVX0WEoOxpLLHRy{tv+UD9iP-N?@F|Elvv?mvxrv#r$wFd%dY zG0uKAr&fw-P=08pn7J52vrl+Xq&Ld_WjN{<5|c+6aLZi?=kVuOwRxUX$}qjVWZZ;b zxy7_km1X7`gxa5KLXbNsgAP-udn~UlHzpekfmjoTf REdw-1 Date: Sat, 5 Apr 2025 20:24:31 +0000 Subject: [PATCH 419/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10227 -> 10221 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f39cc43fc..744d08f9b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-03 16:49:13,350 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-03 16:49:13,350 submission_checker1.py:3260 INFO] --- -[2025-04-03 16:49:13,350 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-05 20:24:26,521 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-05 20:24:26,521 submission_checker1.py:3260 INFO] --- +[2025-04-05 20:24:26,521 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-03 16:49:13,350 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-05 20:24:26,521 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-03 16:49:13,350 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-05 20:24:26,521 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-03 16:49:13,350 submission_checker1.py:3276 INFO] --- -[2025-04-03 16:49:13,350 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-03 16:49:13,350 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-03 16:49:13,350 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-03 16:49:13,350 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-03 16:49:13,350 submission_checker1.py:3297 INFO] --- -[2025-04-03 16:49:13,350 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-03 16:49:14,596 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-05 20:24:26,521 submission_checker1.py:3276 INFO] --- +[2025-04-05 20:24:26,521 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-05 20:24:26,521 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-05 20:24:26,522 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-05 20:24:26,522 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-05 20:24:26,522 submission_checker1.py:3297 INFO] --- +[2025-04-05 20:24:26,522 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-05 20:24:27,510 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.4817 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.99679 | acc: 75.600 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2603 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0d38a9b02..0eb5aea3c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1046 +1047 diff --git a/summary.csv b/summary.csv index f4aca82a4..683e67975 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.4817,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.99679,acc: 75.600,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2603,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4d6411e32..a2aee6cd6 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.99679, - "Accuracy":"acc: 75.600", + "Result":10.2603, + "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 3109cf4cd18442160078ad579b00e7b97ca06c63..960a19466d51f49ab16f4ff6f7a7d5d28ecef2d1 100644 GIT binary patch delta 2159 zcmV-#2$1*lPwh{z%>xREZ-r!U1ONbf5|e=l8I$k>2Y*{{qc{|P-(P`HA6jV=E=dTD zl8mP5z3DB@ZnaMl0Ta*wW^B{TzaJZtPIG*k8SP7@eM!K4-?`z>K6v=`CC`i}!gxxH zk?FK;(;!7iqqJC!%)jTSE#EY_2#P4kXhBBiE8*tX@n6Gd%D(f25Mf9I1s|D-5aqzK zc$knp;D2pel0rU-Da!*PZ`sP?B_lyplk&`RZQHZ*AT7+s!hrpW4I0O3NRDY(<)jde z5k|5=$nbfRmfRcN_3%Ot*@|ju+|xR$J7z>A4ysJd=<_*ASAU6+MtX`8(u~)?MxM&jx~7o_FC()f zJ*1IHlwl)Dqlgr>+pyw-=8w%IXKkRNI^LS=*v=dKuNhS!J(NSVJIu%x#) zmwztq($zfJ+a6{*q6=cs5xs51Tt|3sh^;6)c$6KD?|>+EL?1-?Tf}@9Vk^on9%WbK zyC6y(;e#lDi`P=DQP1OlpigFrxaZxINn?i~UF)xAd`ABfk( zC!D7n0JT5TAstY)2?7E69wQKt?+F3{`JN&WFq_Uc5M&`bN9bT0UF@J=Zk}z{^D8(o z)g5#}t*#;G2H}9Yf*`<*yZaS(JU;*T?qoK-xVhS|+XOEAgjFh%YS|8HZsnQ$&3|PN zAg@Zo;ze4-gvm~ZECk_V!K-DSa@7>@#Z#|sTY?c%w;z`3GOX+AWX)%U%iWw?t7H*` zs;y|rE}}8j3d^chz6#31Wbx#-y|&$2@BWLB7IC^VznBpbL7H*%U}(k2=63%XImv#G z$k6J)58|3NaXHO~-0nJVr|;vU>wn{->+7Ne)9^@#bl@tFK%nRl2oxOxfuchoP;~tr zqC@CVbUWymeJDCOFx4GsqJx|poddGIMXZAI-u3^7qWg@T{y=mO{VmbiPS^JiKiKMg zTy?&#I?x~->5vXsXeS5+st$ob)gch5Is^iC<}-hX>JU1}_F@P9(ub;p1AkN9K^JyP zkaMGRz(R71ScPqo|GBEGmQ^9Dmi&-bCSH8+V*z&mivT|(y+07(+!HtZ>GEuuN*DBVWZ`9X< zmi3iv)|XTWns4qB5=4ZlJNb1RQz|w$sw8j6M&qAQO9KQH000080JC!ltqcUe*B^{c6ErIypwzoo!! z1umskgS!gcQ{d8(cQ}^f)ZmJ~h82AcOMR`u6?(BkFIMQq3cXmN7c2Ab7*%ZzcQvDGC7!R?w2ca)WhGevK5Z;{UPj$*SpyspHR; z_a?&kW!^g(amNGysS-O0k1Fx-SJQjRe*4sDG4W0Mq^}l+eVu$TlJa-iXI9>a***}* zf6uS<*XWL4>PKNbZcL9~>_=fduEKaU*AhRA3;Jwi=(C@j8gQC(+xe>OWomap5s^tt;o?h&2X-rgkX=#JyKd*dF_ zZO;_*;Kk&6c{#Zjh$DR@k1nUPEkF1?Q#F70NS$642ZT zsW%n+Xcp-sN_=2iu-0g(&>lRR*LNkilz$rsZ_Ze7L4r>rNPEg{g|;DBim~TgP|Ro1 zp`*AqPO%V6x51Y2Z?*wk5OR+d1SVKu_yA1pKtv~`-~-bBxV8rfj-dqwI(VFCIlg!U zIORthdgaQgWY}96wVuVnz4LM~c3qcslinKh=XkYzeeYLIr9PaB1FB2GDR;nvaY%KE zuZwuNmie%ZCtgbsPy48kyqy*~_JK%Z0D6pjRz4@sFU;`N+v}CgOiXZA{*c# zj2n3b001Zz000yK0000000000004l24U^3#K>@gv4<|YSDw9elCL5_z+i@iU002+| l000pH0000000000004l23zLf{J^^2o$|oZR5hVZs008HC`?ml9 delta 2110 zcmV-E2*LO5PxDW(%>xS16ksTH1ONbp5|exg8I$k>2Y=gc;yM(4KVM-*eQ2eDT!0V| zFq+bP>5Vd{I!`Dh4n&j0nb_&&@5fG{Q?_@RYF;XxKG4KjYv1B!?|AU_IZenTEqKhb zpIgU~EsVVo_8#`oDt!!rmMeir%(%jnQ}q1^a7`e*ROiXVJMsUXro#)rmAh`eu_ zJXq1x=YLI>(@Z{zSdscd-WE%f=LPk{ij*d%V_B}5`f+B|7W&0UY_KSb1A4@QGNqZQ zjTAKTg$$ps;+z|!K^V(tDj!nNXlQ)1`{M`CFh_&R^}jKF;%}~r@E7+qp@E>GEPz2& zW(&rCsE6lr$Wl~OynkE?X{4hl0ZVxGOVU`D)-g!xKM#$T z^bm()r3{l*9ELQj+y*5VEPbdS*=qwe)pplh+p=HTy{6r^T&JU6&r3qr9JBUXMR?>3 ze>5oA6DcYWJonX$vipuDs}!iwJ2k9gGuH#t%>a>ksA4{jS_AV@8D1mqA!YVvz?9xz zU4J^bOGooyZF`vNh&G5pM|8FkGacc+BDSJz;Ze3Uz73+(5nT}F9}%-%h^;8wc$968 z?|>+Ega@MhBVwi_dU%u_G|F!sy_VpWA0iM??=b=a$sQpPFnf;?2$)CuF1eE>)f%LW^mpa4&)tw*^P=DPk1OlpijX*$kZx9Hm?kxfV)xAR??}*pM zC!DJr0JT5VAuUj~F#-Yk9w88r?=b=a`JNyUFq=*{5M&`bL+D@{o$sJuY@Th_^Gi4| z(H*owt*#*F8sUJsf*`<*yZs$@G&=k5_INruzrNhB+5|59gk>yN<)Rs|)Jzlkn}5n4 zKwjmvh~{w?(L#1IbS@|tb6zgen5(9M&mTKY%M=BrRr_J8F2lT<6vM$$mVwU89B** zj>^!gfA7RKt>d!m4Y}F2omSVwMSs`DMc36u2d3en4r#$v9)Up7ArL4!1Oi2eK%nTl zJ4A=jq3CwdFS<~4aA2Z4&_oA0*E$DeeS=s9<-P6x4@LJGIekEM_uVbgS$5lVd+%&@ z92VJM*cxLv;upWP84Ye&IpY!GD2??w}1j zCCItfIbb2VL9D{I$opK?m5VYHWkY^QD-+M2`@R6%e}V6t?zjn~oo@6q6t(8MA^2nG6Is zEO1A&9~Gnmf7-fZN~RK622$3Pz~hW|7*ke zdWk|WQRpR-UI^FFZh%cY)6yi^A)FzRY_kBkp+MKUHEU;ZY?X{%U$J*>9g3EhfH6pY+wju&fB5;8{ulC?i~547tZ z2lk?-`d(XtWiQVSA2khMvPY=N4u!>+EvoBGf6rzo$ntgDmp*r2#yz4F+uNHY9o=yp zcW>Myy6u@_9=w=bFE1z80&%2|#Y) zChdH%hWr`cZeHJqRb!$0Q!zkg&KY4AsMog4@pTan*HGs-;lw)u{HY82#M^0+V;}G+ z1R%$_XXSJL{8FtYQ8`Oz_i?(2(|MIG$>Nb@i?4p(^b05Tr&SpWb4 diff --git a/summary_results.json b/summary_results.json index 56e93d596..8b036a69a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.600", + "Accuracy": "acc: 75.800", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.99679, + "Performance_Result": 10.2603, "Performance_Units": "Samples/s" }, { From a86395f9d4240737cb33d07fbbc74d747c7409c4 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 5 Apr 2025 20:28:02 +0000 Subject: [PATCH 420/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10221 -> 10223 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 744d08f9b..bfdf943e1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-05 20:24:26,521 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-05 20:24:26,521 submission_checker1.py:3260 INFO] --- -[2025-04-05 20:24:26,521 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-05 20:27:57,743 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-05 20:27:57,743 submission_checker1.py:3260 INFO] --- +[2025-04-05 20:27:57,743 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-05 20:24:26,521 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-05 20:27:57,743 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-05 20:24:26,521 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-05 20:27:57,743 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-05 20:24:26,521 submission_checker1.py:3276 INFO] --- -[2025-04-05 20:24:26,521 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-05 20:24:26,521 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-05 20:24:26,522 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-05 20:24:26,522 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-05 20:24:26,522 submission_checker1.py:3297 INFO] --- -[2025-04-05 20:24:26,522 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-05 20:24:27,510 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-05 20:27:57,743 submission_checker1.py:3276 INFO] --- +[2025-04-05 20:27:57,743 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-05 20:27:57,743 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-05 20:27:57,743 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-05 20:27:57,743 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-05 20:27:57,743 submission_checker1.py:3297 INFO] --- +[2025-04-05 20:27:57,743 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-05 20:27:58,793 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72994 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418194 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421036 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26821 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 0eb5aea3c..ff0ed70d1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1047 +1048 diff --git a/summary.csv b/summary.csv index 683e67975..1057417e1 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72994,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418194,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421036,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26821,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index a2aee6cd6..2a27be17b 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418194, + "Result":0.421036, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 960a19466d51f49ab16f4ff6f7a7d5d28ecef2d1..9fb3b686c6e1f5e0fa3f56e5828238202fd666c8 100644 GIT binary patch delta 3040 zcmZXWc{tSV8pjPHN*L5ZG58s>B?eitWMn7tVl0_zj2X(7M3@nZAz?s;4){=Tp8bAP_~_r9KI(R$gs#Q*3K@D`}6aqi0D5CPS1>sb19z;QKZLOqbDhA#0LYzH6h2smdN+xPvC0PEaRQ^9z*MQ zN^<7WohviDNM%aG2``6~<;`!Y{GVzWFEn_j_pUG2i1*_zGC$W>Kz5`=wsE%ruIZlZ zw@{$y!D6_dhwoL7H@QO5kTUHV)H<^%#~T@0xq+3zmNo z_7&Sg)qAgg;HVd$OG$`qOr@BKz6h%jX+~gtRm2 zUL8<+RiE0;fg0^1HLvChIz%j&5Wk92Uil*6s>*DaJ%O$(=FPW^xSLn1L2I)c!Mu6x zlv7;1zV%0UL*sn>-8Y5&ALc&QW<@IhK}QveQH{>+y7zq;pkwE9i`{ys18&7zHZsB@ zllqb}YDcoB4SuRyzPE8{O~jiI`AUnvR^^FD>5Ld)b!B;OD&K4IeG*HSeq;)4zMdqS zq`T6Ek}7M6UXrWj#OMBL@4rETD^wS|7{6;e1(l(8ej9lc)h0AsY@EYzM3ADK(L))z zFgrg+-UPSMZ)3$anr*(3{KfJW4WYpmFG>EoZLXvkfz3xI>?J;gSNM1G!Nx#D?iO4! zh^Y@ua5GJSiM|Q~dGphB8f%k2bP2jc5S0)H|_V=z`+JoRA7h}98ZyE$`?|t26zbhxI zg$kucTC3#5o%e6XGaiH8x#3-69J)HpmF=OCwXT+>x%U1WvS#*%S*AMIaK&4Zh@0w` zd(WlipHF^i;EmQGF}it48>x7KZJSU5#kwx2P4zm#+q<}WD8_KyO|evD&Z%HE|4u1^ zcUKnC0FUo zCt3`;708}Nyk*k6?f2=zgP-oZc-A)MRlHS6)9lks*W%0<7##LWzA#~?XTWvE7$9m% zCiU?XDld&|AL&c-U2WMuU=?k=$d60qIpvWb`9pCv1m_PBfy)65Vd*x(RMp%|fbq|? zYkIYrLwIvm~xQ~VXpV86vSItou#Al`_>L%xj3H9q&+U$~1EntH0g z@~s^*+kbnc&1F!VU5+GG>3QlnH|&}~ywukV5+W}WvA`n>DNR7cdNR1KUm1)?uo7 ze()M1Odm?;z5jOnR(!VYy-qZm`t=i0@3x-(pMcR)fSl)diy$3r#_I^?L7KwCr9F|j zZ=SMb0E9#&z9^o=HJp7LU?PKWOXl5y6qz?p!s&2jTeMuz6Bl$2UC`q8K)*n15w%%m zZ3>O_T(hBSv&6;qwYT>@;r{GK^h_oc3s2p@a1VX}uTl7IF)iD-CmR1Fm?lG}Y zh;8|SdX$Cmfb{hh*wbvXDBwY1-K#~~+uWpZ7NtG7?E@{&aWBN5R$(m7(i$v{+mlel zcvFW1UEolU3R1C&M5)KM0$kx85$ubL^_(j1=FtIE3q^YJbypD%44A)*(uo&Loi{BU zlJith^H5jCc)I#j$$6-{t&Z;8e3&*#mZ!txf0`aDfq4>5VfxUaube8*Mkpe?KHA{W zbvzrTG}};10@YTc#MP)tdey91wt;6s3#pM)Rh_l(pxXnINoy?jQz*FVJW$cX|kXu*!>0g~7f zZQH+`0>N@AZX~*IRsB_n@OYQ~n!A>b+?&^#EK{m?m(vT2!i@XRECvUU0HhdRCCRxG zn>JVTn0caX1i=5J*!>gv`59n8wz%DlTfxYEfKK{G`pT<>KPY2;iX4@&sk|>eea5BM zeqoI9>t$$8M#iysUh~+qE!gQr*ZZ1wRJSH_xfs{aiqL809#cEZ zRZcT62m}1i%&>!j;l!y1F!MiQ&p5gP5&g2(&jZzEnVzZDN)kAGZoQdt>UFciO#M;so!aZ&%`M1>cW4T)$#T~L@3fbx)yAQ zVMR=CT|Y>^WL*MB9q+-WlAiMmUxMRQH;fiTk_6znm30*@9J@zL^7|{d!qcJ*% z?xRYh`lGSorYd4p2x?rBZjk`8v~@nKe%|SDqy!abu-OW;Tloc%5{8s;rVY)Um-~60 z`_=tkwjnc5`FFTZtuW#?`g4D9*{kP8fTD)aD}HnYJ`3Wqt$ z=%bQ)j3awEowP^^Umr zy$A-9_~H9Y4cEHS^BEn+m)B>QwMl`@F%wnG0r8A;xgYrPZhxZH_PvvKE@5eEH+?l+ z>dyB%Mv)-tP2+>Lxo?VGtU`>-kSqRbc~hakXL|7r>VCr*zUQwBx7{1F4$Uu&hwHkbynM|Hf(6iI%NJt(wNCRmfIpte#$lb>YE{K zdKPuk_l`6|VwUS(<)XHgK4$WjQ}LPDZgc-1E*c%uQrme&NwoY-J{l~8UgP9b~Il@jp^9+<*ZpK(y3$s zlh-4rld5)4s3KXE)2GYP*C4`L(j=tqc6_;nV=QgP`AUaJ&&#Yzbt@X5c||>W)u$Zf z!uLB|h#iV|L{@U<1Oim4t1J9iyo)vqdrNl&>lvM{veTKhp!DAwq8w8V`KYhXmDr1f ztdN;0%tE!&FKDFN^ygL2RijTBm68Mo{I~rBr{54WUiU>S#Rf! z2Wcz$&$1;ov8?lLy$ZF;a3kO86MQ9W>z_&VNAL+d>Gb+D`xD;J>#_Cwwbl`2K1ytD z$YAzZ%G{*dSpWH%D?yz>QKVR?4(J4_1!@4QB5}!Rf#gYW8R$P&`p5`@j*}kB=z^}3 Onq;In1f*FG&c6XWoXHIU delta 3027 zcmV;^3oP{SPwh{zatID0Mw9;O3IG5PQ~&@N0FwbG6_b?+AAeXOj&D=*cX5^Xkl7ZIRyL$_L?v{Fu4t~9xPK?_iipSyXu%_x}%?M__us@y+ z4{KMK@All95hvYQzdH$M!C~!t5Z8V^`p?0)F#01N1wmq@31;zOZImSQ{YE41je==6 zu7~qrmL3^|(SNj?q+g?9Bc4Y=x4#UUP8!-Y?Z&h_p4CKbvjI+q-2ZQcnlAMR%J_O_8C`nJ7m4Aiw!b!aR*O-pebE#{_w0n10 zYu1c!<9;&A5*wp&zaL~Lx@PniaS~2{TOCsC1glVGuVa;|Hql;PwM<)E+3)dv60Bny zt6rBQeA7+3M+Z^(&4`vg#Pe?U%cy;=)6>$+TEAq?+1MKEhV%OdBdsGld$&i;gT`%^ zc&)gQA%B;$;Yv1KZ#6X1@oWaA`Jgn6!Q36gIVxI~7$%X}yOK*HadstFMB?sBu8G9k zNOsC?@^YJyK#6iuA;x*}sN(Wb#nq#V>mtQYr7d1*3*xn?6cwJR^ijp-qlzn3$RD*R z$4;ddt@PN!-&9<~uZcwb?k7YdPS_hF5x~ABl7H5&+&`w8k)Huz+GqJvK|rh_qZ z#((*Y5@iZR&K?FnIQsDW+1qpfW9RhcaxoOAtD)gI87;ond*QS(ouvOx)0Iy8X&yv_ zoAGQAMCp1hxJiOIxrrBFr{g$V`ouT4R^4nQQ4lN_S&eKh)%fI}tmC5~PM0uoW4)Hy z+ho4!&F6c)`C>1e&F-SbEJ+trdu!EAFMpg3#>3h#wSJIv$CJ4B>YyxG-pnQNDfCg-ClM-vwxS~ zzN}L14O1ZJ-;*%vjc#sr-L9K^y@k@P5-3(tN(mINoGlbvZr#?V#YuEIi7t{TE}+*a zaRg>3C;<|QM3P7(l0+hrB)SD8l5k04L3kU(fkd}}L<|QKT}dK|5fkcXlqgeZ&fI?w ziB*(Rg2ZcgXHsmowA0F2ih3MH4}Vb<5WPl;BOp3K37|+M5=A1BD5As@9liGoP$c2< zrl%mh?Fmp$WrI&TkV{b+)4>>J%C#ub$a~Z?uUg!u-Ch{)O}fhrcdq@j*QHgIR|3Iv zZwo=MZrfHfXIZLDj^MIgFeuP?O*D>R%0&mRdd%I@>DXc_8^P7iKv2gJ7OskWn1m+^V{pPKQ4X zWhE7spz?A(uoae0-8GwA6U8o;{Av5K!l&z&C9y<}V@oW_Q+mLXXaGy10W47i^W#QC zu_Orq%Yp=LCHE6fp~WYiD1TEv@Pn`p4U-ck6TwWmRl#%Qa%-B*z|>}4J%P+BYAiEf zt_P;p&z$S7W9QsGD!%1(tZ)~{vL%_QaU4k|YLHB#0WygO$V3g2sf)-Y34qLk1ntei z2Qp=gPdXuF%CHUzgQux1wcH>%+bZDS3uYBHmU+Os9@yq)(rQ|^%6}n~?~WcT+;p@Y zNhaBtbS0UnK{AO3$Rrvd6E#R?vxrQR0LUyz(B2$;AXB#Zq!VRoNu~_z&@e!gh;Ytq z{-e-TPg>1ZQDdn&v$=()*7cUFayIi7-_1W(xJhogl1$V%o+J}BNG8z$nM4C*l8t>J zvsFYUNdROPBxr9AK7WuYTYS<9AybBRNEptUBsI>Nt^WfstEjQmoY~r$Gh3#koSX-+ zD#v}I&`w}^l1$XNrX&+JNG8z$nM4C*q6Q!TSVd%#1VCm%g7)U%1DUeLC!N6F2SjyP z6oAS6KmexoticpJAwSM!73Gy;(%N0edfHKWOy(GL6nZ~w>0!_z8dq9$hz6M~VbWn5 zV3OEiq%TYr`E*z((x_9Epsm2F11&!3M43$qCD?Un7*I(>(qMO{s`uBEQe(-~*cA_aU9&-X(-}Xc(}`3_mby zR&4nBQ8u5nds#`5rDn}-$4@$II$IA*cgbw#kcrL_Tl#&7#+7~_qCpxmOZa`52FN5D zAhXEt!x8|QMSdR*ejjM@Nt2!t8GpC^KG~$H%q_RhE}E?zG|@X^OT!P*xYF=LG>B#i z!w=H{nnVL=78!n60zk9K@T0--11&!31Z)Km)nQRUW*NVauGsJM7@7a8ub`6pNF0oSSP?_k<~|o)mjHyeA0I97f~h`>OZsm6eN0_h=gSUoHo=n2rBj z1TSCoUfBCDReJWTUkv@qI5g7I@1<#V%U-e%Y*>NY~JO0^k1`t6P6AIsZ!f< zC6gZ_D_(%MvXUUR6{jLqYO12%EPDZ~@sDh`fw!*-5J1YQ$Mwv9JG*i@->EkF1?Q#F z70NS$642ZTsW%n+Xcp-sN_=2iu-0g(&>lRR*LNkilp62n+ zofbLvfkva0DP}2|9j0W~6oeL2T0DFmM&9&afa)~l-^4TEnEUn< z)PJ*KBgz5=B1V(`>60)fTLFWUj3zt*4U@?xK>@gv3nw}ODw9ShCIzWd+i@k6aVHf4 V3zLT@J^^2o#wQ~N5hVZs001cAwG;pV diff --git a/summary_results.json b/summary_results.json index 8b036a69a..b75e13792 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418194, + "Performance_Result": 0.421036, "Performance_Units": "Samples/s" }, { From 659a46d264bebe46d1653336201ec1e77aa52376 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 6 Apr 2025 22:21:35 +0000 Subject: [PATCH 421/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10223 -> 10222 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index bfdf943e1..7900ef3ee 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-05 20:27:57,743 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-05 20:27:57,743 submission_checker1.py:3260 INFO] --- -[2025-04-05 20:27:57,743 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-06 22:21:30,545 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-06 22:21:30,546 submission_checker1.py:3260 INFO] --- +[2025-04-06 22:21:30,546 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-05 20:27:57,743 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-06 22:21:30,546 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-05 20:27:57,743 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-06 22:21:30,546 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-05 20:27:57,743 submission_checker1.py:3276 INFO] --- -[2025-04-05 20:27:57,743 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-05 20:27:57,743 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-05 20:27:57,743 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-05 20:27:57,743 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-05 20:27:57,743 submission_checker1.py:3297 INFO] --- -[2025-04-05 20:27:57,743 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-05 20:27:58,793 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-06 22:21:30,546 submission_checker1.py:3276 INFO] --- +[2025-04-06 22:21:30,546 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-06 22:21:30,546 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-06 22:21:30,546 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-06 22:21:30,546 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-06 22:21:30,546 submission_checker1.py:3297 INFO] --- +[2025-04-06 22:21:30,546 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-06 22:21:31,650 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,14 +26,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72994 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421036 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26821 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27905 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.4817 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2603 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1593 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ff0ed70d1..d0378c4c1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1048 +1049 diff --git a/summary.csv b/summary.csv index 1057417e1..d7d3d3eb4 100644 --- a/summary.csv +++ b/summary.csv @@ -4,14 +4,14 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72994,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421036,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26821,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27905,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.4817,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2603,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1593,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2a27be17b..5a5231786 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26821, + "Result":6.27905, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2603, + "Result":10.1593, "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9fb3b686c6e1f5e0fa3f56e5828238202fd666c8..ba8893ac1c8b3544a523eaf20db0fe77f0604382 100644 GIT binary patch delta 2339 zcmV+;3EcMYPwr2!%>xd^Axa`}1ONbf5&!@g0FwbG8YbOXTW_N{6n@`dVWB>>(j;7x z5E>;JP1AeRTbkW!pCSS#paINalN$sZe@n>|EqTK7kzuzjgU~!+VUn*##^3YPmS+$y z{5g^Bd7Bk9mro*AX1(SQI4zJ!V0b(Ofh}N}BpYhR@@q;Kq0uCi0oee}|Mb z8X4d0!Q|01%<-^x{ZB%l`MYZ({N)2pX&`7Q3t$kH*^;sE>fwbPvJ}a{hCtkH$T;fYM(gki%`rl@4)0 zb*BgfRQDQzfa=~L5K!G)1Olpihd@Ae?-9rc;&t%}=jsMP?T>Uw2UKl>KtR672n6JN zf#{(7dz;en`fK#{0a_Cbq8Hgt82))K{#NpAP6wy?tXuuTDAk0nOQ1-GuZ>ktAdu%BFQ6K%1(wZ1m$AEt7VpO z)fDi>Q?G5AqNKELKTOqSnAg+EnqN{bcXMv8;)NfmwxT7wh{jYae<-R}@hVsu#EU1V z?Y6DfdiP%hERT|v@x=(K@RO7q2SYPLHn;oF$Vv8dREAdneGu2IiOX&_Fpy&_?6deMAqU-Mv9YTkq+d;qVL(##3sqR1%9pv2T z9FX-bVilD4uKzz2f8A%~^arAQ=x>S6vb&yp_`z1^;i~g=)qw`#NQZR5LOVeqP<03d zst$ob)gchDGoN`oREN+(wii3-mmX9d9GL13y0BA%oEx137Lr@UDr}3q&sANuta4Gc z0{eb}I-a7%>R@?3!c7XiMci++(e=S#ZLerF!fXRY2%(v0z zykTY!=FPYGFSz=c&%;EmY<}9D9Ec=e5&ok&qHqV3N5@jef@oHyl!(_te$vyILcSxF zt3kq-s5y-wMVYWN5wDGRbvja|{y1adejQYmCqF8>zZdMegT25#_(GfmG9ohb4OC`V zd#R5Lkg&$XQ^H@-YrkA2IVUNNWTIAEu2p4ozENKbR@7IrSzj_CShl%~sUOl(-N~=p zh%vFbQ6+ghHX8o~P)h>@6aWAK2mk;8KL8Z}J=y6B000kE000;OlL00bvy}+r2Y-&2 z=X=#A=Tf6<#FkbcqH(3whiHIHrjcASjpUMPB$v9#>cbL9E{hVhHCTP1r9(Qwol6Mo z(l9w+G7&6db;Vwv$H4q4dzY0IS!N-tvv!|kA?vo>oGDYQ95m55VoSRZ(YVs?Lo|qH z3A+!|0GdPtXcpOhSOP$^$nK-T?tcR<9nuLw6YM@D3|6u%HLPTH#crR!hs!54@&@O7WsX)2OrP` zzYpmI_bwr0}<3?vmNeA%7E{BewMW z5REJSK173Lmhk&94UkDRKxUEOha~_qi~K$s{65gqA)OF1!S6%DaMEOUADA>Nw)^}& zU_Mj7&xOwVebPx&nHjgvE}E?zG|@X^OT!P*xYF=LG>B#i!w=H{nnVL=78!n60zk9K z@T0--11%lW3D^oCs>`B)%zrX|A6>EE=P@$>S6@LT^_7AWygoW-vqW1tOrmYXmR294 z@ubyqEnUN@n(fg>1!U zpU0?-hbfTiNe^TzsjbxdHS6?A?wdVV=gpFK4w2+NJJRSwG@dm25PuC4S;FYUG(aR7 z4-i>o^kJO5xv8X-Oi%=tIMRNM`f_BA=7dC!a{~Nkl$Zn~$~yks4mo z)5yHKV+Ua{E%`{nmcz?c}`bPYH;UCugyG8N~rN!2a!QF8E#zzA)l2$;4eRJ&(Hr@ndf{j? zkBq55ObRvYPR$6{@1R_Lj)TSOQ}Q?MYY@l5?Ec%>?+yGg{SlVj6%2zQzWM1@0V>1!afaGjKnjhPA0M0NpAVGPDvn0b8uLmpLY(uA9Ii>I?1GCmqoZLqz zCS%*SNxSH*A%BLyHm~p9sCJ+Gy>}C6jR{Q~_X<#wQ~N J5+wit006#6V)p<5 delta 2317 zcmV+o3G(*tPw!8#%>xREZ-r!U1ONbf5|e!h8I$k>4;+C|A6jV=E=dTDl8mP5z3DB@ zZnaMl0Ta*wW^9ui1RHf)N{f-{v~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw z=GXCG!)MCA^Mnv#NCO2QnTZhPz_NIlkUZdRT9QIOi7CqiA#d5r;w2+NRFm?|a&6nQ z@*pkD#=?O8i47XZX-JM~SmmS;jS)t&K*;cUl9t>Y52I8*Q~7@wjKm}Jn=_a^`ldA= z)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QE zkVbln64H#RR2QMSDBR!;%NR(kCNu!7qwcD`bg65CSBWG=(p*r50>)6g4 z`>^eFZO`qg*Ykgxku}F^yw(vO2O<~`8GSZb?SYqpdQr~6wPlq;HF~dxRczLJV73`B zWFD%RPvg$edQyhhhzCfSzZOtA?$Xsf*xMdvI-(0=&=I|D#9T*sZ-}iZJ9v~G zjqiXcbwnRT`CG(%7h)^QE*@o95eS&QCkO;gu2Td8X4e@4@wOr72n4+S3j_j6e~Ca2w;@+L#0AxzA`npBYXkzS zdxJngb#D;}sO}vC0oA=nARma=!zY}l8vwOG(jgsCwFv?N`5q$>knafs0r{RH5HOq0 zHV|YXI!Ax#U>aTQpkHpDZPxQEI55>6bV04IA?F6+fVqMoz>K^56?Qy6|M%`>HodsH z+OOLLF8hR4Dw1m14ry-Xnf%RV4N2eB>154kgv;HWTdQOdgsQD*$u6QX)e3*hs#U%U%EDyvc0=-nl*7b&4%3WI&P=$1qU-CT1Jm$G zhjid7k3gX45C{|<0)e7KAW(Gu9il_%P;@)!mwhNYI55>6XrhCh8=V8PzD2Bp^4|6T zhoXP`jGX>JbPxS4(b-Pd_YObU>U>;vzOFjZAROtC4p?X>2n4DQfk4$E5U4r?0(Ryz ze~0Q2I>`272mR8As)GYl-9Z<2N|1A-bHGA!i&%wik^i}>tCm$Es+RnaRwiD2?_&XW z|BC=WBfUQm;M{*FK*w&op4|cRv)+A6=Y)T)$b@7WH$o~4)-c~joAZX5GnhBu;=kbP zW4?$|wXy|Sb8;ZkVrB3j%@Kt+m^`|+G8V?ODkFw?E#)UYc`4;PQn{K&^b$9x5k|>U z%2M&#cvq(*RThkM8tvCXRYm%vBKv#co;TPFy@M}?dti(VnfV4Pv#Y(-#|21O<6%h| ztjKl1R%yYFjKnfgyDir$)|_wD*MgSym2B3RR0x`H?h+D2gsD6EbsJMEHaDsyZ^uUC zpHNE!1QY-O00;m806zc{P&sVsvv>#r2!FP#bM{BH$+^_%8nLC-hiF`B^&uMIl4&HD zOe48u8p)+Dvih(DlFOn5Z4Fi*Xz7qnaOV=jx-?AAmrMkUSY5H#=P@vU%HCxqMV48} z>a5)-S;)FAH)qP!DhEw8j@Z)fLo}|m`w$JHS;FqaG=L`20GdU1AC>^nEVBD(uz&kN zONVqq&;+{=34@g^OARYoU9sEe?;-P<+I=o`*6x$6W}CKW<^1otTIZmN))8C!eTc@D zejlPiG)wq>m(Ve_lNo+s)~wj@^P_A& zY4@^{B1_GhS;tQ@YdV>SrMqM{bAQN0=ZG!+K1Ab6zYozMnI-%_Oao*R4Uk#n_hAWu z%p$*!2EPxqbVw(JOz``VFq|}*-3KPkitRps517x??{lHEexG#GRA$Dlvx{ac2Tk;j z*wXMrG_ExK5DlVP!tld1fF{uZnni{mmH^N!GW=*T{6I^GbON>li0ZN^Ab+!r-$z&M z_j!!W|J7GeNqwcD1h0?I*(}jk4wGmbv8C0AXgq24AsS?|gw=;>fJrhQxmlu%tUjz0 zV6w>SqrvI}EgjN{GFuW#@cPg&ppuz=U?E$v+2=7T<6#P$e$6_4lKW=Q z)p@g|okJvf&yFJDmC9$UpI?)5z!t$>~~0Zd=zc zfjn3H*X#EsKFE{)g85*QzTlxZUBBHip3Fx^^ym7$k#_ptNHa~=AAg=Nr@j$?U-*YL z|89}|LTRxzV{kWIzcJESgp(kg#NXGwr?0GBPJ2hQVDNG^sO5a}=hA=qqW{9)f6=#J z{bJ}>#-WjvelJa`TlJEDV8c2(SoB8zr(QUk%p+s!50gU8x>Ga4^*bn6pW|S$`jq@l z`x?Y?FuVUY_Im?AOcK8(cLl>Bi0{9or)2vs&!hhWvxF0t4h1}L&S)l+A0i%q&2GXl z5QOi2ipclGPH0+WB|&N{PDQFzs-oU3djYHQk8HPrx33c*fW)cC^~`=dyK*(#sW$lq z=cUm#$}@rz(A)^AH#PcL&eJJMd|+Cz)@Z2F9z2>gcNMpk8wYRBSa3mtPa;Tr%59Cd zAy|sB=UY(BXVIafxHe9)5KFgz!ItrFwgFra@_-ctCRkv&2c~u;q9ao95ov#1I{*a7 z(1HRTJkGNmpS=N`@;4g><-)0C*ju=1J&B`x=jCYZx-RP`gEi*Q@oMq5ci~fPM%*RmUGk;q$pEz zpOSJ}5L!Iaa{ASeyy`y#)p5wbi6_7z_w^@evtA>H0tJX~g=BA&=_Nr05>Pp8>XR=f n6#`@klYs~tvtT9*0R=p8&S)l+a3@p&VUxutBL)*C00000d#h$q diff --git a/summary_results.json b/summary_results.json index b75e13792..650e17afe 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2603, + "Performance_Result": 10.1593, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.26821, + "Performance_Result": 6.27905, "Performance_Units": "Samples/s" }, { From 8e333c60e045633bdced334bae6149213051414a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 7 Apr 2025 17:47:37 +0000 Subject: [PATCH 422/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 8 ++++---- summary.xlsx | Bin 10222 -> 10227 bytes summary_results.json | 8 ++++---- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 7900ef3ee..3e1425f82 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-06 22:21:30,545 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-06 22:21:30,546 submission_checker1.py:3260 INFO] --- -[2025-04-06 22:21:30,546 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-07 17:47:31,793 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-07 17:47:31,793 submission_checker1.py:3260 INFO] --- +[2025-04-07 17:47:31,793 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-06 22:21:30,546 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-07 17:47:31,793 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-06 22:21:30,546 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-07 17:47:31,793 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-06 22:21:30,546 submission_checker1.py:3276 INFO] --- -[2025-04-06 22:21:30,546 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-06 22:21:30,546 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-06 22:21:30,546 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-06 22:21:30,546 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-06 22:21:30,546 submission_checker1.py:3297 INFO] --- -[2025-04-06 22:21:30,546 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-06 22:21:31,650 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-07 17:47:31,793 submission_checker1.py:3276 INFO] --- +[2025-04-07 17:47:31,793 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-07 17:47:31,793 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-07 17:47:31,793 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-07 17:47:31,793 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-07 17:47:31,793 submission_checker1.py:3297 INFO] --- +[2025-04-07 17:47:31,793 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-07 17:47:32,838 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.392716 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72994 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73405 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421036 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27905 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,12 +36,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1593 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.66534 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.79155 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40724 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.782 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.431058 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438428 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9994 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8152 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d0378c4c1..bc51cd774 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1049 +1050 diff --git a/summary.csv b/summary.csv index d7d3d3eb4..38f07b29d 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.392716,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72994,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73405,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421036,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27905,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,12 +14,12 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1593,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.66534,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.79155,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40724,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.782,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.431058,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438428,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9994,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8152,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5a5231786..aa6402637 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72994, + "Result":1.73405, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.66534, + "Result":9.79155, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431058, + "Result":0.438428, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9994, + "Result":22.8152, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ba8893ac1c8b3544a523eaf20db0fe77f0604382..6ec4889024a21ca08c37804ac48a6ceda15956a2 100644 GIT binary patch delta 3137 zcmYk8c{~&TAIB|2mUD>Kv6#O@<> zN97cMyR}(3Ovmu15)sqeWCnJV2-ogNu<@E@v*Lh{M<@9%pMxF`Yhu4|(6D;67_~!j zyw;quTtk)Yjh3D3My6a>*B5VtE}!58dvAY1a75zjmI>#!Ha?q)O|^VZy)F-Aq$9Im z#??`n-JAQ98;n84{DwolF-~C~GAHOmH0F+yVyba5f0bL`Sq07P zkd+dBe|`nIs9oHw<)q4pCK?mtpsBCc8mDw#SbVmaNs&MUcRx2hkDN6UV4zlCM_a$@PLn+MlS|_@boOyuQ>bQaarJ!2krrcqQFyS-?WrW6adXpUuc@RrV_2MaBX@&5%1N%MreT`= zbqBFEywLDrQ=bVh##q9hee^)}{gocA8xH0w2m8EE=SY)UmAd(QoWcT6hC7(isQCl| z_C6bU9Py=cHplXLvPa1T^6+DE@eDOZg^L`;R1F=d{W)A&gO2UunKv_;^|14aB+5Gm zN$`s=YdCiMmu+I;;u+roQmXbZ>z9VVaxB|6qmChFVjGpgfJGNLow`Jtlx0lM;nH{C z$ICDpqVZq`X2TCIJ2jM9Rr^oZQ7?y40OEL`1vpj|chi?Fv_1@f*{`yxm?}v!D z`7ZMLt7-wWB5};((oMH3qjd8i$u!{0)D*e>aGP|ISU_@w)^7hP=(D_hYtI;) zS19av#3ihfiZ`9pr$#A0I=kpT9aAXXl-G#cG`nVfK}XR%;3))>!Wx*l+^A^1e~9>V z6YV-zl=qQA>Pa zGr{6=#-o?njaDCXCnH{ryP0ygZ5PCdp{9(>kE9^N z`mXec1x3LFkRo%J9hO05;z+}=FN5Mw^)w194Z?&-*`g}z#?%7ZGNYow`6-r&5DWN2 z6>z}q7onh1m3K3DH~)!VF}f2D{}pUDK*~@@dt2YOlsak5 zKYz`PN54Dyf@`}ZX$Si>#nPQmrqM$9mAT=Kw+Ac0Y7P{VxcnoIw!90CKU(~f7<`1G z!&KAB7X$}-q0|6ih6U3S!L)#`H_)WaZ|)Gt}&S zdMD`84Eqe*=(fwD-nLijYlk}7^0<_OIJz1r{(NDZtjxW9;AsSy0Z+G-r_3-?Cc{P4 zm{)HM*~v_P1FDtOG@}l-|5cha=a!`Z!8yZc7B|m6t4$lC9ZuRr@;&dEjO~ms3~F?0 z-kN^$_;&u;P*YE*kK53fPG^vD0pmJ8@@dbJkdvnakD>oo8M1`2Pk@Q&`|WE{g>?5* zH9&A@0dT3OSu(4ChGm_h{)w=AAx-zk@Tvh>8!C0rPhrJ$IwD)aKzECvYje0=O^Qad z=Ubs(L0|EhAo9wMCx>2xPM3Vi>8Vh^q)6VG*misubghf&Sn#S^HEZ?;&W(t^sX8E( zReFmZSopw^3B2h^h^}Xv>?!G=q5la8-6lB3-RV$gG2D75%BGJ5J~4Fna`WX?-aqQ4 zV3@@|GA3-dP|_DNRvq+3MPVue>(G#QX`Fd#7?XmKGqgkw2(c}MJqkoShL3wNfk(Xv z(ceB!3$trpqVCD)dpgF&HV4^fIny#Zf>f48cnOd$Q(sr%o(3JX15U5Wq;)D&xGX#sOE&u<8wr$?*SX8fW-2-d8z=Q-?7&$ZCnG+ zLI~;a0oT%v#h~pO+W5KHH1$?2LgLdYCgfB1=x;!_{jj5N@0F!)w)POky0_3V^SwF6 zN2|(Iz&mI6A08zpmCl9h6L-OQFhY3H%B@r5f-1Tx0t>KOpuatOVM0S;qizYfbf|Yt z!dH?Wb*b_aZmHB18N|hcTjczgIfapasN_wDSu+tpf?jdd0+zM+wPFFt z!~-}Igl7HVrFUV*)?JwqWeSi8Z*l2L=rJ*|+)>EQ&Xjjd==8I(3A->?rcLhE$)rhT z`iNKmRUYSaj=G=>DXNLBj-%crU91FXBtFM+P;O{b$woWbYThT{V2QbV| zc&;>A=*-@Ik@A?x{6CFF1~f!kDmjC)`C?ik3R{sfU3tsu*NgAxk?Z_^Rxi9+{W@Cp zX7-$KaPsTHc>%EYr;Oy)4m6WooPJI}dDd3ZDbjIQ2i=nrT`yWLUHiRAZsJL7m6RlF zO;xTL?r?T-9^HHunMt*&r zKj^7Xoj>I>MrB=oM#jI&jDi*r(B$ut*`oaj$WdPVaw}LU=Xg|JPqP@hdD$G>+(_QN z(68&p+854AXNbJcga{_f|1z6Dzt#_O5 z5SU;9y5UU%8$I^^h(v$?*G5*JtP*JQ+m;8b`0XG~iq#^kPRGvyuZYNuxKoaL?Q2S{ ze7(G;#Qt6WD4+S0L=ZopS1}g9s89W%%cz%iaTbVXIS51}$&Lci*@*Y)xvEdw=l^Xd zE*`?g|GU!0nWv-2;_d)4yBCG`j3@NqQz>i-eb@-=P`iVz%fpR@`XN-K|F?+=r@>xj zI`etJNys1Y2>}>RRuX{Y5$^%;#3Vdd4p!6txv=@OuwY~@*jWg05*{rP?{NYO`+8(w z}#Fp?_=ou$&> zam{+qZ}i{ePOfDqd7G>31kpD{i6-GOMQqgt$zDa)-Vp^TA8rw8I;J(foXxh)6 zJV@;>3{XmDEtgO7?w-{_@(Zc2=}!&}Uo_eYl6@XA5bnO*^m+VIl-|1L`+iGu)7|0` z?eGqKmHWYi-OpO5?TjHQ-qm{IqSOao{D=Dn=KV6d`iPhdQ6zdvq$Gre;7tBT3+_Q! zXB24}BJHqeQv0hniy zEP<2&-lCtc?C*A?qR8-xS}PZIRLTN-``Wb+fz>S6^*y8F6&e|;>Ip~ndT#q_t`BIv zKPi92Mxo@M!|4Y&Ol|2h$!qt}-bt>JTcePoNLpYl8|2q$47~JDJ?tSl+FT@{u%ZZ` zQBQw)r^f%yuHK<_0C^?D&Sn*$l$#!JZP|OG=tnhQXj;Nt44e^)A?ru1hE)w(8A)_z z+JTykb*Z?-!thURK8suUIqw+5*}j#qjh}cfWKRf2LHNvFj;z^DH31;n!wk){AJs~t%IG_pV_EZ{KfI12fHZ?B>h W05YsQC_KQE^%A8bs;DX?c>e=QuFEC> delta 3193 zcmZXXc{tQ-8^?#Fkj9Lq7crKfv6sfkQU)2!*u|ldHDj`7Ng~S(Du%?&Xk_nq_V>^*0TV9UrYqpH-Rp+1gdf(^I=a0{Ge?QOt+}HK2TP<5v+&;-8bf`Nt#|r|1 zX^|(jiBAPuYm38!Ap+@Q`j3FKGvhEGHIWEaR-zSQDr21*Gca5i!56hmzn6SEx<1Py z`MTaCc;jA0;XI*hb!Oj9EhFW$uS5FE)@UZAxqj!FCjaz-`%)dOKU$alS7WWzo~+nz zv>(7T-Q(_mwl!(rSav>TA9udw1RpW?tSI*cvFwbzE7L~S@qTnb1#hN>_%L#*rA#wp za6oj6iO`On2<+V+=i2|)nb*1i{I0ju%v#M~!$bNXpXR1bU2{xqY zJ&h$R3*iVJ}4vAoX8KkqUn z8nPRaj{ClyaMs&C4Ud7kOJy3qEwwBu$>-$cHN4vDXF+Fmc89`gLPI&Y*dq2$QI%$8 zyWNP_%NLG?<>edOKTw;R7n1M1EQNfSYp&0aQ~N=rawRbwi zFWG{*6BS44qvY0)T zRVAYb@=8&5K|4j0yu#m2RNE_Ui;a|ZYgRRd2UmS%APu`blmx-8hZwGk0P-Ki*UI50 zK3b z(`T}~&TEN7uYHd-B+H)HN<$je2PB~TcIg;&{fhM@`S#&~%asGEXer$U-zg7+u-$_% z`&@UF#C7nIj5sSqVbX=*7UIq$s3$MFTap`fnZ3F@G_u~^x;)p>e?!64-Y_3~*(JJs zI}Yb@(c&OSPB~}l^G9&JCT-^vn6{Zo6x_9m6jW{KhTFW`Am6%G{%$D2aNJe3N^H)t zWUcsi6&bv*fcuC}?s6mqWbG6-TwB~c(^2n+YT!#>*TR`E`P*sk6WvhRk9->XxT8n2 z%=3p!<62V92VF}P6mhTEtWWlbEYZQ{drm&}O+~e@5Lw!N+SxiMiUkLUebcliP4x_T ztW5x_jtr#_LVj~){L=A0O5j@S?on28rY3uUT5 zzXF*2o#moepEpD_yX87iyk*RjnzRxmu+^$r`|j3YSN*}&IeQ+VQiTGpL=lhRa4}5$<5AEky_kK7&_wfS0!eS@8RMd zZ{crG^jAkaq-Oj5N7|hRFL5c+WZ(FlImL@~5loi-q9rZ%EY%x$XfCS_h*?dAH}tDP zi8xO34V&1lt$&Lcd(EXw&>iO&0{|FhY0_a6Seq_0DQ}8t?zZ(`w zlJjwk9`#Y60~~I+)Mw>W(H|9Gg<#}~?P=gWsWLO>6qpY_hS%Q~UmWBHTUEn9i-2B}=9lEqpncm_8M5R;;Lk<9y~GTtSaQ$cMi zm``6NBc*I>j_CKzMF-^ESCLN&=;FZ549Wt-DT53Fj@HC4LVVJA3(%hZJxV@l|?u;&? zQ06?gbV$iZMdRj0buS<1fVWCF)m_)d_B*_-UPZETgKxP)8cyUU80xsZIN%MZ0>F2qFk;&@baSn2=euGr?VaTYo5hR_ordF%SBHCwi&sN;;n3!c6T9jwd^wUeZ2 zb4xt<3cgO%2}x*qA|k;sfSWxHdCL71xcIL*K3gM{!~hvNFC^!sTrsczGj?a;edwYN z@>CHZ<6Uvd_BY2+s8YHsjTQK|@v4;Qc(?t!r;d%%%NKc|Z|ZkevP;XNO!_63!ox>^ zOIs4_F<2z=OK*cA@&$5LfFFVT_>qbH6R7OL5En>F!D|(kB$f;;g?cf|)@!c9rrp^r zVV}$M5wJhri)2u=04JGs!SrtOM=sNA^I?2^~|=bmCj`V%1>LAta$G!HMmP&1mX zXj^&~*`syCtPb4srgzy8nLyyF1Ahi(qK|or3`Jez)UR$^&Tm})RiuI7A__QVU!|f^ zhQHH%E3YBG^QO=C+fTSUtEx=@;oK+d4KqGDaYVSIiQn9|w3q`sDuKI4Wn$Z1OrPcj z>XK;yn}NJUcA{!dz&>#`fakcj&q*J>Q<4SYM+~m|2J^3!s+~W^I!qH)G<*pXdaSZJ zyd-^@I@BpGm?K-Wu`7_QypKZ1;pwZpYQK7)KpBsh10j<3P-1(iPrcGP0*J;HnBm93 zUVJF*DN?y+cTd+1W!v!AgKl>0bNeNOvj=Y7Z;qfV0A;2;_5B>|YI5n!;TQ{yB8o z{jds+3x-URDPb$-fnPHrueIn=ykhD+7K~ReF7IP&ENozxvveJ1x~QS2Da0w=OTDv( z$Xp~_fFFwcILk!p1EOF79Dc(+2ShP4hVdK2ehp7=tW4wV5f?+);E>6q^~mR8RD?+jw{`LbeNuBZhs*W*hHCO~3~B>Wseg6*Lb#Y8(sPZ^3v#EMo(q zMAz-y4^_e1OA=kNjUD~Z=_#v5N{*CUH>^_#I{_!nl~ zdc=3~?oD_hQXUoSne%EaSazKpTzTdc^*FA1-uAoZlCOYhYvba<$d?uVJ+*!MsPL&c zsSk(LT zY63>v`@H$}sw3WpkeXxutdf{3&W=@ry#VC3B*rIaFG{$9N^)?1yqu$cms_|r-;d#1 zVd?c7Lj`NYmFRu0?*kgPsS}N+tlI?*dmWe4i7_mlf8q8}jp#qo<#TL+FVF_0X!2CC zr!m4%Ep!n?eFtdS3EUkvoj3H}{$jmlu}xY>I^W>Q4W3s-9rI9!^pK|fu4KYSLO|VW zNGXHx_-cT8=H&_HF+aoSPTB>l^2e$NY_dm9bjDgHc+&135itJ)=3%bcthcPG}>e)}8VJ945q>idrs;JeTV_uTk ztK~cpZu$L@O4U;Ml*Qq3g?Q;h_+t`kdr7|N095TM+s^r4-vceU1IK+@D*Q6XX<8TD qfa5I Date: Mon, 7 Apr 2025 17:51:38 +0000 Subject: [PATCH 423/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10227 -> 10227 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3e1425f82..c30821169 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-07 17:47:31,793 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-07 17:47:31,793 submission_checker1.py:3260 INFO] --- -[2025-04-07 17:47:31,793 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-07 17:51:32,585 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-07 17:51:32,585 submission_checker1.py:3260 INFO] --- +[2025-04-07 17:51:32,585 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-07 17:47:31,793 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-07 17:51:32,585 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-07 17:47:31,793 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-07 17:51:32,585 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-07 17:47:31,793 submission_checker1.py:3276 INFO] --- -[2025-04-07 17:47:31,793 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-07 17:47:31,793 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-07 17:47:31,793 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-07 17:47:31,793 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-07 17:47:31,793 submission_checker1.py:3297 INFO] --- -[2025-04-07 17:47:31,793 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-07 17:47:32,838 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-07 17:51:32,585 submission_checker1.py:3276 INFO] --- +[2025-04-07 17:51:32,585 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-07 17:51:32,585 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-07 17:51:32,585 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-07 17:51:32,585 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-07 17:51:32,585 submission_checker1.py:3297 INFO] --- +[2025-04-07 17:51:32,585 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-07 17:51:33,645 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.4817 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1593 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0233 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.79155 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bc51cd774..f1f0ca518 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1050 +1051 diff --git a/summary.csv b/summary.csv index 38f07b29d..7399c0836 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.4817,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1593,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0233,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.79155,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index aa6402637..3ecf2b81f 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1593, + "Result":10.0233, "Accuracy":"acc: 75.800", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 6ec4889024a21ca08c37804ac48a6ceda15956a2..55a2217e60ff22e0cc3c7b3707fb98dafb5a270e 100644 GIT binary patch delta 1484 zcmV;-1vC2dPxDW(%>x8nE*sL5(E~0Sflwb>X%a3;2#u1Ars=)uEzNGVPZ0qVtdkf7 zAAgM}QgX`jk?FK;(;#`sBATy8=HK(vmTwwd1bGysEGHxLm2mUx_^;tJE5GxE5Mf9I zIUkvc5XHcfwbPvK7_RxTkegccl@LIH*!FW6$S=t`Z@Q^b{pzDX)KxjLOow zrjZ3NBeNqt&`2c8u#wOxB6;mLthiv=WAn&a8)&GGx8^#w^Ts}GJ6+pzd+PPPrhjD3 zu^O*+gvWsh#>0|58)fZ*7lC?F&cL;0l|nUouZC4@)_P#J88BoXs+dpX&d_>NhS!J( zNSVJIu%x#)moDzo)jZhS9%eeC3u4d_y=}x?M|f|DttdNqlpT%lfGBlDA4K_E#C#WG zE6Oe&Wmn_7AW9wKgD8KCnCpl`Jb%g_8s)d1UQ6)Gj}Qo`_XL4}WRDRDn7t`P~B?;0;+q1KtOeG5eTU69RdN> zy+oTnQAwLj7!9Z*J#9>!JhG@JNSr;3|(mpy&_?6deMA zqC+51bp0KoL+DU+JLs2vC^|SW)g5S}gPa?k1G2tFtb+31_5X*W`+tm_{y=mO{Vmbi zPS^JiKiKMgTy?&#I?x~->5vXsXeS5+st$ob)gch5Is^iC<}-hX>JU1}_F@P9(ub;p z15@2W7j{aJbE9*>LUM~(g>8}lxvHy{RW7QQ{E${AUVQIk0e1h306!zWKM>&De zO)t&shjTsZ08GW25m;vtWWH8 z1W~cG>}hKeKV%!y{yom4X)ATba(wTcxifOHJjf>c1?z-XCCZW%MWC4GLhVZQQLo}h z6gf{7r;S!nq9Zu8tnLbC2!GQS-mErYy#N;lkaC2X679V=1Y^hcAgN1&Lj`fGt)xDb zc83jR-*g9fmZlF_f~TB%iaTIzCL$VwCXyEL3+Y2B<1HBg_Kz+DTRgzAVDw8tVKy zoOmmMKOKX9<854I=mQ>w0Az@JQa3y6o>TE&g8OignZsb+}87Rjg m|0bRQL+9{$0T5rbSHn}9J!zX delta 1484 zcmV;-1vC2dPxDW(%>x9)Axa{X(E~0SVWB>>(j;7x5E>;JP1AeRTbkW!pCSS#ppzH` zAAiXcEqTK7kzuzjgU~!+VUn*##^3YPmS+$y{5g^Bd7Bk9mro*AX1(SQI4z zJ!V0b(Ofh}N}BpYhR@@q;Kq0uCi0oehkukb8X4d0!Q|01%<-^x{ZB%l`MYZ({N)2p zX&`7Q3t$kH*^;sE>fwbPvJ}a{hCtkH$T;fYM(gki%`rl@4)0b*BgfRQDQzfa=~L5K!G)1Olpihd@Ae z?-9rc;&t%}=jsMP?T>Uw2UKl>KtR672n6JNfuuTDAk0 znOQ1-GuZ>ktAdu%BFQ6K%1(wZ1m$AEt7VpO)fDi>Q?G5AqNKELKTOqSnAg+EnqN{b zcXMv8;)NfmwxT7wh{jYaD1WL}@hVsu#EU1V?Y6DfdiP%hERT|v@x=(K@RO7q2SYPL zHn;oF$Vv8dREAdneGu2IiOX&_Fpy&_? z6deMAqU-Mv9YTkq+d;qVL(##3sqR1%9pv2T9FX-bVilD4uKzz2-G67~^arAQ=x>S6 zvb&yp_`z1^;i~g=)qw`#NQZR5LOVeqP<03dst$ob)gchDGoN`oREN+(wii3-mmX9d z9GL13y0BA%oEx137Lr@UDr}3q&sANuta4Gc0{eb}I-a7%> zR@?3!c7XiMci++(Eq_;ZLerF!fXRY2%(v0zykTY!=FPYGFSz=c&%;EmY<}9D9Ec=e z5&ok&qHqV3N5@jef@oHyl!(_te$vyILcSxFt3kq-s5y-wMVYWN5wDGRbvja|{y1ad zejQYmCqF8>zZdMegT25#_(GfmG9ohb4OC`Vd#R5Lkg&$X!Zu&gYrkA2IVUNNWTIAE zu2p4ozENKbR@7IrSzj_CShl%~sUOl(-N~=ph%vFbQ6+ghHX8o~vtWNB7 z1W|Fb>}_fh|Hw9^eS6%rO;oQ!0OT0=tbER%Uz#PV^EAJY(|Md0O;(V3L1y{aVdPE!1t?EL m{!Kgsj=66?LH!4_Wg~AmRv@60bSHm0sKxUD diff --git a/summary_results.json b/summary_results.json index e7a2e5968..547608887 100644 --- a/summary_results.json +++ b/summary_results.json @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1593, + "Performance_Result": 10.0233, "Performance_Units": "Samples/s" }, { From 15a9cec14e05df255beaa19a2ea849d6219f1f03 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 8 Apr 2025 21:15:05 +0000 Subject: [PATCH 424/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10227 -> 10224 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c30821169..f1884989c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-07 17:51:32,585 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-07 17:51:32,585 submission_checker1.py:3260 INFO] --- -[2025-04-07 17:51:32,585 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-08 21:15:00,575 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-08 21:15:00,575 submission_checker1.py:3260 INFO] --- +[2025-04-08 21:15:00,575 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-07 17:51:32,585 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-08 21:15:00,575 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-07 17:51:32,585 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-08 21:15:00,575 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-07 17:51:32,585 submission_checker1.py:3276 INFO] --- -[2025-04-07 17:51:32,585 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-07 17:51:32,585 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-07 17:51:32,585 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-07 17:51:32,585 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-07 17:51:32,585 submission_checker1.py:3297 INFO] --- -[2025-04-07 17:51:32,585 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-07 17:51:33,645 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-08 21:15:00,575 submission_checker1.py:3276 INFO] --- +[2025-04-08 21:15:00,575 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-08 21:15:00,575 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-08 21:15:00,575 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-08 21:15:00,575 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-08 21:15:00,575 submission_checker1.py:3297 INFO] --- +[2025-04-08 21:15:00,575 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-08 21:15:01,592 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.392716 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73405 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71659 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421036 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27905 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.782 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438428 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8152 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.996 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index f1f0ca518..bcbb007ab 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1051 +1052 diff --git a/summary.csv b/summary.csv index 7399c0836..4da4dfb24 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.392716,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73405,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71659,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421036,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27905,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.782,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438428,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8152,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.996,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3ecf2b81f..b1e1027fe 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73405, + "Result":1.71659, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8152, + "Result":22.996, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 55a2217e60ff22e0cc3c7b3707fb98dafb5a270e..ba9031f7c647acc264486cbed8ebf09bf3e6639c 100644 GIT binary patch delta 2716 zcmZ9Odpr~BAICR}MogoU=04kEE-}~Rh#F?*vZP!)Xk)HnI1z?*u;j8$%B6@Q$32(I zD2iNin`=tsQkq;Uu|&s@&Ohh%d;Wa>cz-|7=kxi#UeB7{XS*880npBXSJ)>J06>}* zec(J*2YifN*}{1^xDZr!f%k5l!)t#eR(J*4FYf<$(AW)+G10BLiO)3=;hRANmB&Jc z?yPfI!AGEDPlQ93$BTBL(V|7CW(g87iyV(5@?#cO+S8ZOvs1|15pz)nU~eA7OxlKH z^+Q)iZ#GgYZ0ltKkA2T(I0(;~a}K=ec}n#ROBs6bMaw&zSFU)P^8*}YKP!kSeo8KN zA!PYu!71Bx(z-N3%b-HG{3mO=-rXN6i`sf3CpEm{^Jfv(CQZzNx6Y#-)A$SqWwsrHbCd?+Q z{oJQwD$BqwCS%T8!e&KcR;BDMpLNKurthB#Jjlm<=S4g+ur^72*uJ{i&A>9Hx$k{4 zkq+ekJP`&oEvTtwDb`!FCC0h<_3;wZ{-+4fk>hVC7n|Jm`B9?9t}6Bwg_%?c@}i+D zvT?Gh(k6xVF%;qsBepY;+2=|Mocv^cDZ ziGe7?M+T6DlEJ*kJ93b6Wbr=w_)0(fRe*2lO`Q`_P;)RrN}|PruB>2W1}eb9Am&O8 zWMR9cYztq+9yar}gj%Z5ybQ9pCDH7RGOP*5hTEh`Jq<{1*VPf7YM059w-=>XZJJ&l z{Ilw3-FUHM<^_Q^%4Jcr>Xj=!4|=zDq^YJjcDgImQx+mL;OOl4*fDhXDNm2~s`{bU?uY+g(-&AgTEEm>jG9LL ze*V!)Av?Sn*6{2T^i9a@W&od5w zXeMuB+wxx7-TIu$!WU;L-V{-r%1`PS-UobAds0oU2U8(RnO6_u#@fBq}!^Y zOEtD^615a*<{IxS>=Hdh*Up9Dv{nYS3Yf|4(G|R-@03-UTeJ=0d54U3JTde)_x@WG zn6l=kSb@xekxoigW2*7rcOky-U+IpfF*>6?Ab|>vVP7Ru)hdlKG<=rUnwFTB)j{1M zOu_YS;|fWS2VLAE_RI$oFS0X<5Tbmmm91NqAY5{Uc! zIRoj|3^@wcxTx0nWqTcO;y@Yr>vcH4K64Je)~L|}RplCZSyRE>M?>$c^s#anWAZrF%r)8FZ`4FjgbmBD5RLw058W<_Ut`G6%hu$uk+8W!VPLL7o|* z@%^{(`~02UYeyGOYEgT}s3aBhEa%R}z=w8CY2mIx{h4Ks;yE9W*A~A?BD?W66lE}V z*WM)g0&EI_=#pG`=;yLAdwn z1!}IgrH>hs3n}ec5%RkpzTNe(4-z3?0z}h=22l^TYW7j%ycutOD4&5gHCq|B!Jqat z9(+ZNmA|d4wz0L1%(Z*nT*2--Tlb-H260FCPNh#Vhp0#C!ptSBFPmWIa2zmKAzNk1 z1pjMAor!$U-JU9O(uFwS`n}e}U&UahZF72;1zFx5LHSRGdwu-)j|bmG-WmP$FHZ zOMLF(H!*U`Yt)9r=g$6d0@=1$w|<1-4zGm5xw^j+xw4l?7ZFqn9tN<8#-VkXqlPg~ zh*lzF*pfEmNno*idKac*tHL95yOM=Sy^7upb zmQ5th31Pj8B3k61($R|+&%dDb;B5-(BNe~tD!a7Y2;DRUExfejP1jd@GOrD9T`nqk z11?;0jk=(9;Q9PN))6>8&2{4+nsv#NS>jZ7{YxhmAOK)5000~U006|xm+V6?2U9dJ zT@EE_TnP-g!|d~Wt|f78HAbX}qu%X{lV}tS>@JHx?cfq6IRrn{5SuY#6uG6EHV4aa zNH5R+84*Fyo#yFn?U&Eg)h*m_c1+r{SUXpyrTWO`L6Tyqa-_f52pJ6Q%jMt<;M9tw z3zucT-&Y`lhys#T7H_0wpBp5rljW|@YgxIuR!8ECr#(75^rrzDpvYDkrCY|K(SQ#@ zutMO8j^waYPnLmaJ;-J>I&e5)woX4`e8&4swVOf1^{c}Am57kA)dZPT_wmDT*$+#$ zrKlQzO#N`Aki<(NaXB~WdZNg|;^VmIyyMQX`$>vB@VkJ{$?^PxO~E>na*x1&evk(6 z3&2EH2HY4Z!RmvX0XZx%!U(9&!XeOm+ANvZ5z{tr0Q B`J?~< delta 2694 zcmYk8dpy(s7st06vRvoBWXx>tNps0HQJD!Na;v6>Ot7tG|+Os@MNV<8GoEDOBr2MGNL4v@SDDW{J!sh-;c-hoYy(8a~@~jeZjqkwn=j6Sc?8E6a-QOqBr?4 z0T$%$@VFpgigtII}pL zrpcXx;Up#4oK{Y=jF9ofRNWcTJ^eI0CJt7V z=8TGPLf+Jdolbmw4qC4L_T81G=W+Aqm!gn==+=Qe9YjmOyX8NV&L~{U)VCcjS$fz| zw?t=|eHMblXYK2+j6ddk&Psb}ewqMGrc$>!G$TQSj{t144?{z zOIvaDoor){u5pnlI3xt43x=_sIL;`J6EYL^Kn%q>g*)nSeoME{#Y7xW(1!E&KofQf1 z*lFWYV!E7D7Y3MKa^pWBx@C1un5}I$%2PfQQ%pbjXOSmur@04>r~}k(DNATU@FnY! z)lp%;euu%uH}~%*79Bii6YTeK1<~qv0L?l*YD`wlc!-AY^Ek5``$T)dS(CdLhQk)L zEyR=rxz<5o9mSB@?k4G+-gkgomfk0aO~MS5UxQzF;Ek9FnX4z1=55|Y=PKHC{qJ%D&`vSLz*8!LN?3_`ao`?d13tKdaTrnY>_2 z#9?D~`an757rAFjuK98xvB3^;4IJyuW&Z|5tT_0xF170cY+}#Ytv5T*+&2pfA%@jg z73^Y$D4FGOcaCiI5|+J)7^$Uw(N>y>w(_}Gcx;s8I%t`OQZREy_kp-h8%f3D?Sp$r z9LTN^hqxaf-)!X8wPLQynFjl^5}IhLh;l(Eau3yAJ#o8CcrU5X&97U7&M zL(SI+yMZ`#n7TWM4MzdwSl42(X}hw0 z#;86LK!Hp*!){whAlkBw>^>)C=(Sj(q&~TF;GepN*TA{ngT7%s;mp}CuF(L&t*68{ z`@JK9Y*=F>;stvBhewD{XY!GzxDA^1nhsI4`HQZjqBas$AjMXO&?lrGP0&E(FfrLS z6A@aM{LN)MWwz>=?QFRXn#Q++9@1?1J}sAO&~4=u@0wF1G?6yJNp0`fDos|m<||M} znR+h0KEziSl?&;`U8M=I6l<(hISNU+W9{Xgcg|B^<;Xz$soH0rn%ptBD+|Hq1pVr& zLePgDi(h#aC$!UVS$F#sMD3bYz0f6r?h`+X$H?~G>+f*Hwvi-v$PR#jrykn-A-CFI zv{-@Bm(^`u;8eG3t)V4Z%U9%vva{ln>!O#m;7;`k z0u}|dzZi_YJY#1FF7hKS^%0lWTeiCw@{g=!e6}H8+Q9En?dwl-S_|A)4CC(&0?eUq z^Aj&Rex*O~|5U^tmJj&NNP3nXgD7^u;(s!7#d}fkUG;5q%oZ3ZZ%<5NcasFRY0lBA z=@EXDu-AkLsKpS6`7h@eO2X|ZOLZJ1zB})AS|NRyKE^Avf53G_TkQ+~EXwYnh%4T1 z_;x7>t|=OTE@Y#qj3nh>4D8Q;%V;XXHlOMzokiqg4H}mJ*b?o$8eb%i|5zpgrLUR& zRGGX=b!?yYQGCm~cjg_jNgb&&=0(&^Gb*rrxmAl%}~syrqHz=eNr{!Xt%T2xkyI22T@2#4|&5R(+< zTfTkI;l!|+#x4Bu zt(2~k)2gY!T*Y|(m4n7;K?&t0^S=7vQS()r{KM$JsGzyVsnP0~?cWXF_d4Snui#3D zjH24@YJ#?I?fR$Xjh6*HEwpyKq&VeTh~TDL-}H&B&KEe#BQY+_GKMsO;*GCiMfZhf zM~urHR6HnmY%Sv|r#rwleRXsFC50RrG1_w!5e;Y|HJF`lEhch%mZJAeGTvRkrs#*$ z2Tv!vI70}v^)r;?^7f=H{j0V;B@}~*)-%y_y!$)O)Y`P6e~Sqdd~`ZEihGK3KklA& z`A(>Qw@-1WyHZ)Ik4Le#TJ;SBqh{6%3@W0>KAbNb@t`j*;V8x~*rh&?uJ zehZ)eW5qjf5Np3RKIZ+2K+@DkjYJSBDazAs;nNj#Wa2G9tNiDm89d|ZaEFg-?F-e^ zN$)i^-RD|s=mL=A7ooey3b_8LcE^VQ+=^!CgWx0}6>S5C0Uc-u@FPG9VRl j{?--@0(oozf#gB|f3X+kfFz6^I2d@2(H2+HS^xPTN#W}* diff --git a/summary_results.json b/summary_results.json index 547608887..24baa0d3a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73405, + "Performance_Result": 1.71659, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8152, + "Performance_Result": 22.996, "Performance_Units": "Samples/s" }, { From e581de20d435ec266e6f234a38d7fd718658f4f0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 03:46:07 +0000 Subject: [PATCH 425/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10224 -> 10223 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f1884989c..94fe214b0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-08 21:15:00,575 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-08 21:15:00,575 submission_checker1.py:3260 INFO] --- -[2025-04-08 21:15:00,575 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 03:46:02,364 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 03:46:02,364 submission_checker1.py:3260 INFO] --- +[2025-04-09 03:46:02,364 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-08 21:15:00,575 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 03:46:02,364 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-08 21:15:00,575 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 03:46:02,364 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-08 21:15:00,575 submission_checker1.py:3276 INFO] --- -[2025-04-08 21:15:00,575 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-08 21:15:00,575 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-08 21:15:00,575 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-08 21:15:00,575 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-08 21:15:00,575 submission_checker1.py:3297 INFO] --- -[2025-04-08 21:15:00,575 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-08 21:15:01,592 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 03:46:02,364 submission_checker1.py:3276 INFO] --- +[2025-04-09 03:46:02,364 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 03:46:02,364 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 03:46:02,365 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 03:46:02,365 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 03:46:02,365 submission_checker1.py:3297 INFO] --- +[2025-04-09 03:46:02,365 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 03:46:03,339 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.4817 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.0233 | acc: 75.800 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.031 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.79155 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bcbb007ab..b96bfec6c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1052 +1053 diff --git a/summary.csv b/summary.csv index 4da4dfb24..292f60be7 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.4817,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.0233,acc: 75.800,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.031,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.79155,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b1e1027fe..39d6d515e 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.0233, - "Accuracy":"acc: 75.800", + "Result":10.031, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index ba9031f7c647acc264486cbed8ebf09bf3e6639c..5a6d6a33d0d93dc6d7c9590380e867ae62441d62 100644 GIT binary patch delta 1126 zcmV-s1eyErPw!8#%>xRu?8_N&1ONbe5|e!h8ME*Mv;%+e-G0=l&wBSMosn`yCL~R{ z5i(h?hWIwx95>9I!Mynt{{>gy@_9tn!WN{>!GWOp%HThm6AEuId30@MER1JWN(}K@ z$S->GQpi`NQZo= zB_xPQsqW;*ZOoY1+^CYgof?gQ0<&cZxeNtAoVp`;vm+Iw0)L89GbK|AECVTPO5kxu zI}9r*c}D@Q3;s_BEM7yAK3W(@QB}X1+E!{gP5oO6+*aUHS~a+soKTR)y~&)I}a*MEJKmhANKo(s!(vGG|1Wh>+sG<)unDr zSN&G9-=CropkM_p87wzg_vF_|(JKBQ+n%hNj+i?BTzPLId|&3hlM#13@SiHNlklh# z4}Ueim+ZGsjTRH%q)+;4Vc6Hn2O}wemwjgCeVFY7aew^$N`H;+_@#an#^c8H_{Dw{ z#^Z|o#B%Ro<55k7JM#0$k!*x2_hKV|+rH-EZ+R*iyy^E&G#1Z2RLh1K| zzoz5q8M=UTag%`NxA(|0eM#V6a>?4Cy9e5Jj{|#AQ+=;3!Lpa`+nc^9I^L&I({}2F{*mRc0HT8rrr1CiDWkfI$cBHxj3=c zF*iO56RHTqc2(+8CJz_ehk(NLp3crF6+v{8uRCPwRn9WR!ybuPsIT>mEe>+ zV8OVi!k0xnT+4h|#1pS2h^KwjZ@e8BIrM=@VgP!Gds05j=a=Oq95n?g?o%>J$>Wlg zbTXx+`07Vq^`C+2ION~N6X1~h`V%yNvtlEJ0tvG0%NcM4004QD?lZYoi0b!HJCnE+EB>(^b0P}Vxy8r+H delta 1138 zcmV-&1daRePw-E$%>xcxE*sKt1ONbf5&!@g0FwbG8ng8SuLFP3eK+qL`C0G2r882l z$b_URH$o;0)-c~joAZX5GnhBu;=kbPV?K|lTG@iMIXMtCUm5&Ib41|{CXcSIjD_*6 zN{Jy}3;9V;UJCh+RIa8Gy~NFFgi(}~l~lYo-qq4lwO~bkC7bmn z6M|)%yMzQ0Db=0)x{VnVn;TVZKame)W4;`Z3Ql+RfD?< z+*9Dvk#{(j;?&@ZzJ?Wj4NHBk!4-P3LN8Y6#R|Pxp%*Lk5`|u(&`T70i9#=t^g_6X zb^~nMnU*HO4p|08@znS7EZdf%v46xVNKH?bs$C3S?R+h_^Ps}SG89SuVZU#v3I#_> zgPiTZ4)2UqUFx=U)o&&H{V56o3RcjP!E%FjPkxOQt>XW&?a8X?h^gbxmG>sX_hsHY z8F9x0|EUr?36CoA@K@7&$$tCPXfg3k`lPQGhJBrUFp~0j*=JVXhuJ<5$A8bS^w;Q) zU+PC;JZ?;nU+hO=Jg&%3EcXsJ9@Rv+BR`KE$%e>(J5jT&V_e#An5bh~>gcJj4PCF< zyJ)IR95K!ylzu<>YdW5up$j+{HwkEddyg#Bmjv!5m#qD{d!SwSIItHr)%V&GEPHuw z_^4_4l08CAb|@^qY*AfjdVe-ML6)!EzVx~KGVT$b*xueG>FAE*xO?Ls(QVHZ^Weqg zdU-jy7KkH#B#$nqv*luRGp}brM@*ic=evEg@Ao~=5qr<4<7YDwqpC-2*RzRh+I>%+ zNOnV@C`5S(JQpYSI_AbFp<;_~OIPybt=_Zm&0vfV{{v7<0|XQR0JFgvS_lNBw98nN zA0i-sYQr!PgztR{qq{B1X-W{b18vEvfkIPSdK1zn5w(7h)H-?lDvskMkW-JVnf-Ql zQ8nM`Hv0u1lr=S?1wk1YVWl#g8ojR;`3z+tawB(^b E0Mil~Z~y=R diff --git a/summary_results.json b/summary_results.json index 24baa0d3a..515d3ea72 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.800", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.0233, + "Performance_Result": 10.031, "Performance_Units": "Samples/s" }, { From 3b3590a824f1b14326b1701a8248f517387b4acc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 09:10:11 +0000 Subject: [PATCH 426/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10223 -> 10230 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 94fe214b0..d1b1236bf 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 03:46:02,364 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 03:46:02,364 submission_checker1.py:3260 INFO] --- -[2025-04-09 03:46:02,364 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 09:10:06,227 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 09:10:06,227 submission_checker1.py:3260 INFO] --- +[2025-04-09 09:10:06,227 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 03:46:02,364 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 09:10:06,227 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 03:46:02,364 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 09:10:06,227 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 03:46:02,364 submission_checker1.py:3276 INFO] --- -[2025-04-09 03:46:02,364 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 03:46:02,364 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 03:46:02,365 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 03:46:02,365 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 03:46:02,365 submission_checker1.py:3297 INFO] --- -[2025-04-09 03:46:02,365 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 03:46:03,339 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 09:10:06,227 submission_checker1.py:3276 INFO] --- +[2025-04-09 09:10:06,227 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 09:10:06,227 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 09:10:06,227 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 09:10:06,227 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 09:10:06,227 submission_checker1.py:3297 INFO] --- +[2025-04-09 09:10:06,227 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 09:10:07,249 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.392716 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.297575 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71659 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.4817 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.031 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.53549 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.79155 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b96bfec6c..628625b6b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1053 +1054 diff --git a/summary.csv b/summary.csv index 292f60be7..e85d85637 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.392716,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.297575,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71659,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.4817,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.031,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.53549,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.79155,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 39d6d515e..54b4c4cb7 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.392716, + "Result":0.297575, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.031, - "Accuracy":"acc: 75.000", + "Result":9.53549, + "Accuracy":"acc: 75.200", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 5a6d6a33d0d93dc6d7c9590380e867ae62441d62..4fb2c4373b39e542cbc5a31e0b3d1e21a8867f9a 100644 GIT binary patch delta 2734 zcmV;f3Q_g%Pxeo+%>xcRD9R#q1ONbp5&!@g0FwbG8h_nbTW{hx6n@`dVMTptrGZ?4 z5D+k$(tGKRvRmyF3W)>JBylEodinQbC%}~BQ)aX;m9`HwalY@|;?F+u;OlFekY`%( zm}Nu5ZdwMRS-`?LTMmuCXJ-x1AYAxa=qD_rL*tEdI->WEKQyl)DJ6CnwXAdxn}CenNeHl7k^@dMNu5k6Bd*y z%|vaapouSJ_on5^P3q*>)QD7j$iQ~k(Z8>p$ayXM-Ky=C{BcH45Dj(R|cphh3mu!_xG4@@@$MCPH2`8;Y3%x7hIjd+BV*}DN# zdVkxxba0oB=E2(aFx3%l5QC2B>>_45!rdZvqHN(&wluyCqSO&x5an+Xvweu2DBF0H zZH@1MD0PGfqWmplrXzZIlpQq6Zymjs;FTXE5K!+i0s+aMAP_KnPZ0>1TxSRb%&v0; z;_gB&5D0ktmk0!u{tAKgb|Kd~!~xZvAb${0-5Ue~s(Xt-Ky~jB2&nEo0s+;1Kp-E8 z*TpBCs~Z5dKh_~FP_;1v0r{RF5RmUF0s;A+ArLT|&NmQbA-X{5U>aTSpK^96?Qbb`1k&FI=Q^PKCIdVF8hRKELP>B8L-q$6My-e z${s*o<+OC>L{HF4CB*rhw0%J59?J1*KK{VX7{}yqS#G{DN}1n{#uy zn)`ujD;lzks7*D3ylmuef)&AP{_Hf}rqx*Q{_}ukQM@$17$FsYoN(i4U`EL1_V5`w z$$pN?(5inQ#5Jwsvg-}G*|wcl*MGxB*TqHG)kO!U;js>B!BrlCK+z!(C^`fJMTbD3 z=(>AEhtQ$u_Rz1oP;_u$qC3z;2RXMo2V{MRSOw+1@BR-(_Zd0;f#@E)JEF7fw&(Uf z*y=o7b)K#|&>$S^kQP{I#|Q+f4uL?`ArPoK1Oj&Eb8nC85IV^Aau5B=gMX@n0~6gr z8+J;NbE|W}LUM;#g>8}dxvDD{WhTmo{E${AUOo3?0k;2(06!zWKM>%|dndqNv(xId zz25c@2+@4^Rh`meNyjuvI0=|6U2S7KTAw^j?f$I(D*pvnpY>T7t2NF~>N5lpXG_9= z)Q1&rfBfWF%2*Ih%Y+i~mVe8Sd-|Hox23W+NcbAn=Mp3@VphcBt@f_YNy@|@r7S$G zf-1B4M@bJ4f&;gI5V%KQh;u}SMAl#fmDyEZs)GY0tkEF%m-NOjmT|^OLL-@|)s(AN zQJ-{F*MjBMm2BG=ObC{4?pD+fX`$}qcW%U(*xaa+Y!8q|KLJon0}uoh000080000# z05UM-+3d4;2&4uG!=&~c%gMXLlOqpff9@#T%CT#z9V3tVz$Ds0Z0Y178dq9*hz6M~ zVdh~PV3HVWq=`Tk`FU6;z+{o5M@hrYDYW=(lyd(PqB=t*WJ@nj?>KBy7Y3WMB|7p{XInE zN`nv4AeAL7K1>5t5{)CNEOPm<1VCkx&qsmL2U>j632_vF*@uLoqky@6z)?`K-3PJ! z`IdrbYWcZT+nyi0t{pGW0IN;Tf2Bs#h%KEyMB_@Q577XZOe48u8p$QoNG^4e(}yLH zToxs0YjFBNi%&YijY|mY&@ee)G7+p{b;VYn$H4q4TbGp-S!Nxpx7|MJI@WEuIa8)q zIcTD9#FlO!qH(3$hiDMZ5^f)+0W^sQ&@6KMumpf+k=sXu+Xq^F(g{Hmf80JK3|6u% zHLPTH#b%$shs*yu>^>UoKG5QmP6(M` z_aR|8X)?DDOqvzDef}OWpQ+vFLT}rBvPsizX*UPWRt}nI9kHd~hiF{s_aPcYvxMJ= zX#h>40W^#JJ}d#CS>*T8;P-(RpL7DQ0*LCcC?K}k&nY414MAwKdojye4Nv99dAd@AWK1>5llJUsR5?$o5df0FS4kwrcq)(H?<!y+G2;lJy4Xi%DQ4-{-+$e=WG1r@v5EY|ZH34OXv? zH0IGbjK<0LRqxrmD;JaQ(KPJ8Tn=h68~?coUcTtPu=ii|>{q`S`jv5Lq@~|W)9RMJ zWFOeDjt=JCVeqLN4ac+Cm;{5gP_ypTjA->T%H`)IoG(A6f78B(NfJ))zm0-!KZvq# z>0QAf43qmW2-zvwywvmPzq5uDm;nX&@zb_?vm+Iw0tecOK434958IKeM_SVF&~RP4%FYA0jJ%4z#FPNs!u#Q;{l_ zs;D>1UV_#5N4DFLx33c*fW)cC^~`=dyK=GEt0wsc=cUml$}@rz(A)^A>k@sem+2!) zd|+Cz#%L%}2Oce|yMkNFje|F5EVv-SClRDQ<+en-5G=*m^BpMWv*^%K+#07?h^4Es zW&E4ffG32^u!6vU1PcuJz|@XJbVLe1BJGcB27urg8c?8v$9b0Hvp0ZK{$|6VTsV~s zorRm$lQ_EfUXI4LZL@YdSY!SiZ&t7G!>Xy&{i!&hst}xV2P_y@75K7 z5DNeR00000004l2i<68dA{+Se)3$m9001Zz000yK0000000000004l26_d#(K>@*& o3nw}OGm}OqCIkab^`Mh*Cl&z|lZPih0cexPCnE+LB>(^b00IsloB#j- delta 2758 zcmV;%3OV)mPw!8#%>xRu?8_N&1ONbe5|e=l8I$k>34fzF6n@`dflwb>X%a3;2#u1A zrs=)uEzNGVPZ0qVtO3km)62gf8mrIu^ko|OeOHyaCs@=t8AIHn;vW?_|)Tr@^X zk_JMC&l6g3b3BZwe5Ud-N)nIEZ_Z%y=$qDfSiAm*l4t(z+7Q9=futl9B$5R%4VBrF zvG3~Pg&eXK)zY}9byRny5s^5kQZZxC=Y*~jA%BhZ6eVOSuYZk<%F?=~kp(X!vm-sw zNF>Uzkh-*)WX-V}uXTjSfe6OK zl06$`?SU78dQr~6wPlq;HF~dxRczLJV73`BWFD%RPvg$edQyhhhzCfSzZOtA z?tjwNJlNYFW;&t^V$c!2ZNywhcyEZUC_8wR9gXjRD0M_1MEP69d>3LX$}S#dSL3@N zN*&>YD1VEX>xe@<${rf!x1L^0@XC)62&nf2fq-O>5eS&QCkO;gu2Td8X4e@4@wOr7 z2n4+S3j_j6e~Ca2w;@+L#0AxzA`npBYkvd+s(XV#Ky_~s2&nEI0s+;%M<5@F*TW~A zryBsZKhhx`P_+pH0r?&y5RmT)0s;A+A`mc}&NdKaAv#CsU>aTQpkHpDZPxQEI55>6 zbV04IA?F6+fVqMoz>K^56?Qy6|M%`>HodsH+OOLLF8hQP6-l*hhb*(QRQ_hN2Y--P z1u5eN&0|u^PKGQ5;bOt7Wk$Jb3i#rw*S0ND5>mGxmg+LB>*-|8FA0~sIk#5HA_!Gm z(UM(6W2zMvRjYUvtPGRIliT*%c5A)+FG7~bbY*@qBO(Hta`Rwl#mMG%{~0;SevZh{ z>c0=-nl*7b&4%3WI&P=$1qJQh_q65?LNQZRbDvv;*=nx1L9Rh)(Lm*Ie{T-r1 z=umV!=$CycIyf-Z9cZG1oEx13vc5&Eg7V(=|A(UcjGX>JbPxS4(b-Pd_YObU>U>;v zzOFjZAROtC4p?X>2n4DQfk4$E5U4r?0(Ryze~0Q2I>`272mR8As)GYl-G4zBc1npWMJ6Ooxe+p1uZH+G+8j5`oWZ>L6#oTR-|~4x)xs8}&B1}7`O4rwniC3dFnM%s zWh{(mRZ0x;TF5VY@>0lGq<>O1jp!wA4kL`Bq^zXkwehYFN2)X!XDr&UgQ{}+qayoz z;hs0x3%!FchI?R)44L=_DzmG-)TaeVSmR+4tjKjxu4vAUl*BSoyDgWhvN_(UuLUdW zE7_|rnGh`7+$AK4NU84R$8F4**xaa+yqy}2e*#cT0|XQR000O800;m-0RPWobnCNx z2&4uFyN>m=qw<)OC=X+Qo)T&0*frITk;i;s5^W&1bn+05E3G_4gG`n%^DqrCNeng8 zM4*cNJggI7vdGb+q~YciT6{K2xqk^!9TG*5BqF@Etk}G>gXCYaiCIa3rKsHY^rZKd zo|iXKx;apyBgB@b9-?ujtA}V1$`ZaFrU594w+5grvi7h}0LmhNbB~hVn^S1<*(d=@ z@bHi*T)Yw&4=!FSw(X{&xP7`_q3GbH1oDDy&NjhIATkG57D^N;6pS>WeJN9 z(*Tu3<47utTs|xTP+8>jQDF3e7N2xN90g$ZAz|n!U~V696jW^YK`ej1rQn%belFFv z=f|#V$ICOoYLjz+snIlIOQ#RfxYFrEG{7a(NG_R1a>+E3OI_sjVF@IcMG4v(oIcRv zlTL8s62dw(OwN}~1Z!AbvDN1>Fn`L{WhF(HS;y*aw@ffk>1LeK<%w+{(}l`KmQD_LE!+2`*e z^O?GRF7&qBCtb}pZ7=V^kJdS8qI1NSb|0c~rQL^U5X};HAEp5`i3ZRtviq5-5)F`9WcOhSfXpJhj|RIBwD_bGLMGUKNEl9<%Z0YwQ8dv&#hz8Lt;rC%0K$BhtsExN zHDXJr57Bti=|eQgWC^Da(*ToXJaV%{7dd@cC%|Np(?^5T2U>j6i85OfO0fFSFrboo zePAJ5vDfD@Dw9D5qGo|`*M+Brm$7w$-(57Bti=R-6|WC@=S z(*TiwWIRA*kN3-0FWFO(HqGx~Rf)vF_oc{C2Aaq@lD zd-m?i#iV;Q4f`*bgIdhSe=dTTFM2QR{TDs^)h~vAWgHr5>G#sKx@9lf2R5vugL!ut zeCkHS@hmnb!5}TvtUEO$TD^>N`8f&a%TMXww69^3gwy+PqoCUlqU>9GS1<^}5gu28vQMB~uBr(ihPI9Xee@;JG-l*D*If2^Cv> zTe^}bZ}pyiZw6z0_#d;u8DR(n!CCG~lO`f7e@;SCYb8NyD^5kKRH~xhEPDZ~@sDh` zfw!*{Ab`ZF$Mwv9JG*i<+o?AB1?Q#FHOe!B642ZTsW&zHSkBW2l=#53V6D+mqdj;u zYwjv;DK`$@oU!171fN8Z_LSQiZ9}jWW6!ssn9rg^M{#YOVj-4pgDvCVYy-F;WQr98 ze(^b0I;_PvH$=8 diff --git a/summary_results.json b/summary_results.json index 515d3ea72..43cd5c72d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.200", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.031, + "Performance_Result": 9.53549, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.392716, + "Performance_Result": 0.297575, "Performance_Units": "Samples/s" }, { From 0ed942e576451d77f428709b3ca0ce8a081e4efa Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 10:19:05 +0000 Subject: [PATCH 427/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10230 -> 10231 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index d1b1236bf..c47a34eb6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 09:10:06,227 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 09:10:06,227 submission_checker1.py:3260 INFO] --- -[2025-04-09 09:10:06,227 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 10:18:59,913 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 10:18:59,913 submission_checker1.py:3260 INFO] --- +[2025-04-09 10:18:59,913 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 09:10:06,227 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 10:18:59,913 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 09:10:06,227 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 10:18:59,913 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 09:10:06,227 submission_checker1.py:3276 INFO] --- -[2025-04-09 09:10:06,227 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 09:10:06,227 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 09:10:06,227 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 09:10:06,227 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 09:10:06,227 submission_checker1.py:3297 INFO] --- -[2025-04-09 09:10:06,227 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 09:10:07,249 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 10:18:59,913 submission_checker1.py:3276 INFO] --- +[2025-04-09 10:18:59,913 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 10:18:59,913 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 10:18:59,914 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 10:18:59,914 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 10:18:59,914 submission_checker1.py:3297 INFO] --- +[2025-04-09 10:18:59,914 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 10:19:01,000 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,8 +38,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.79155 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40724 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.782 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438428 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.996 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 628625b6b..ca883787c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1054 +1055 diff --git a/summary.csv b/summary.csv index e85d85637..6ad8901c9 100644 --- a/summary.csv +++ b/summary.csv @@ -16,8 +16,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.79155,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40724,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.782,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438428,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.996,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 54b4c4cb7..c8ebf595e 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.782, + "Result":23.0541, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438428, + "Result":0.438227, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4fb2c4373b39e542cbc5a31e0b3d1e21a8867f9a..47d7bfd0b31349507d3c0f8db862a47f880e8e97 100644 GIT binary patch delta 980 zcmV;_11tRYPxnu-cL)jM-4e#_3IG5Zvw#Tm2!9{lteaLV$M2(?IcTDD#Fl;^qH(3) zhiDMZ5`G`10W^sQ&@A%%umpf+k>5vy-v?TJ(g{Hm{5~WMXH91JfmyR+yU*W4<}>yC zT<(Eyr7h98yy&@3|iXfXUh zi+@i#0b2n?byyUTS;p_9EB5<5M&|$OE2yNtQc!}|XX`a0yG&X+OrmYXmR294@uby< zXpqSgRv)GTCdqi@Ziz0k`mj!b$s((d2CEOW_@onMwj`9`^`T)vB{TcLO15IN&tp_3 zgA7RZq$jeK)K+Q%yY2KzZ=5|>yE#PKIe$cwH||KI57Bti=tDF}WC^1W(*TiVJV0cT z(T8;cL>3u+G#GuL#V4I8(~?Aj(T9csk<91=L_Q~@Pd<^}lZbq-HXm&ZA~n3Or?K_s z!)Xu=gOgw~iH%;Eo~~8orgil?$aA%Sy?S%vgFNjoob|`qs~)PExyEH);= zAT89aJ2fL(y^V7DISJ>>PwC&ZuN`5Mgwy+PqoCUlqU>9GS1<^}>mkB|b1MSYtGlr~{9d)lI=I z<;KCQGZtKs;FAc_o^o5FT?m$9?D-B9^I3H0C~l2YEX2~)*ncwq&1%3iLhiAGzyu2n zAAqSHiRg$Fd_>wG*Yp6vF*Kk+2aoeC$7gQq#8ldoM?0+qPLd z@2xR^jz8BgZ~dyN)Q3}XKvf|)u6O32q?I}sE$MaO*{dH+}EF=`UA6QBgz5^ z;@uL)?Fs+@8IwCER{@NZlO{X?7L(8>K@Gwg004MwFLQKxY-Mwkp(YfQGA9}WG#ryp z92JwLCJ-Br|JHvc0RR9{0ssIJ00000000000001hffbXECq4mblg%e11{);+0000C CB*_r~ delta 998 zcmVmF|++%pntvBet~r5REJC zK173LmazLU4UkDRKxUEMha~_qi|jrc>^{)qlTHYkVD}+mIB7Ds4@{aByM6v1FrTU2 z=R$AWeX>c@ZD}_L%~lSYXdSVo--l>i>GvTTM6-n7hiL#!q5(9E{5~uJpjqVi(ct%i z7Jr{~0aZvvvy9zGS8VrrjLiSlR!~WOrJw|>&(>Q+cA2ztm_*lzEuB6@<4LCv z(IAs0oIXqgOp@`)%@SSY^kJOO;eTO6K)}g>1!MpU0?7 z1{sj*Ne^Tzsjbxdb=&BZ-Zy)0?ks8N5PwNtxFdZ&MB_=H578izC44?i14NSX0FgyL zAJz#FS>*H4;PZhNpLC*3OA-k_9~uTkGM^6+`J8+{`9yk8BJ#Pqe6%fy)bO^R#@34u zr$ICfPJ+oKHhN)tx>k{!*45h}&(;3*>cxo<^0dEj)*okYdgxA8uX#+yv!N0Hxqo_r zq@BG$(#(?e2Iq@OU?kt?!C@`9o2S1}R&34a-wjr;jx^@cIE=>0_f_xNyDJxy?$I>t zzg!M#F&qE62wuMEy|DLR^z2u^82XiQXr!gzOVjF>y<{KQu#OJq-C^*l8x6;^*q8)^ zv{19|)Qo8LGRo!WB%CilrGL}Dh8#%}PVc{sf^I*EvTx~K!5|Ei`!Crk*}T;A=)bd! z6PFGI15NdylP4k|e{RAs5QOi2ipclG4z#FPNs!u#Q;{l_s;D>1UV_#5N4DFLx33c* zfW)cC^~`=dyK=GEt0wsc=cUml$}@rz(A)^A>k@sem+2!)d|+Cz#%L%}2Oce|yMkNF zje|F5EVv-SClRDQ<+en-5G=*m^BpMWv*^%K+#07?h^4Ese`Wle)qp32%&>yM1PcuJ zz|@XJbVLe1BJGcB27urg8c?8v$9b0Hvp0ZK{$|6VTsV~sorRm$lQ_EfUXI4LZL@Yd zSY!SiZ&t7G!>Xy&{i!&hst}xV2P_y@75K78 zFyz_n3IG5YlRPF@0gIEACOiQZlhGzY4Z#@z0C;RKb98xZWpk5YClr%2CmI4X9FtHS z6_aEq5F7(d^`IpI002+|000pH0000000000004l26e5!`A`_EjClHf&BM=J!00000 U0001hfoPM?CnE+LB>(^b0JL)1W&i*H diff --git a/summary_results.json b/summary_results.json index 43cd5c72d..092d022b4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.782, + "Performance_Result": 23.0541, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.438428, + "Performance_Result": 0.438227, "Performance_Units": "Samples/s" }, { From caed3d2fe7f04e174efbdcfdb09caf985e4ae7de Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 10:20:53 +0000 Subject: [PATCH 428/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10231 -> 10225 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c47a34eb6..56a6131ec 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 10:18:59,913 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 10:18:59,913 submission_checker1.py:3260 INFO] --- -[2025-04-09 10:18:59,913 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 10:20:48,471 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 10:20:48,471 submission_checker1.py:3260 INFO] --- +[2025-04-09 10:20:48,471 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 10:18:59,913 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 10:20:48,471 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 10:18:59,913 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 10:20:48,471 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 10:18:59,913 submission_checker1.py:3276 INFO] --- -[2025-04-09 10:18:59,913 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 10:18:59,913 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 10:18:59,914 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 10:18:59,914 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 10:18:59,914 submission_checker1.py:3297 INFO] --- -[2025-04-09 10:18:59,914 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 10:19:01,000 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 10:20:48,471 submission_checker1.py:3276 INFO] --- +[2025-04-09 10:20:48,471 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 10:20:48,471 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 10:20:48,471 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 10:20:48,471 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 10:20:48,471 submission_checker1.py:3297 INFO] --- +[2025-04-09 10:20:48,471 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 10:20:49,539 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.4817 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 9.53549 | acc: 75.200 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2077 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.79155 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ca883787c..d890176cb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1055 +1056 diff --git a/summary.csv b/summary.csv index 6ad8901c9..3c9cc19ef 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.4817,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,9.53549,acc: 75.200,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2077,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.79155,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c8ebf595e..ab4357ebc 100644 --- a/summary.json +++ b/summary.json @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":9.53549, - "Accuracy":"acc: 75.200", + "Result":10.2077, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 47d7bfd0b31349507d3c0f8db862a47f880e8e97..adcf70a0eccd3c828afc98a2e21318c7aabc640a 100644 GIT binary patch delta 2177 zcmV-{2!8kXPw`K%%>xSd2^W-a1ONbf5|e=l8I$k>34fzF6n@`dflwb>X%a3;2#u1A zrs=)uEzNGVPZ0qV&;Vv^)62gf8f)N{f-{ zv~ANMMM$HxSdGlT=cg^-G`I+gD9C6*M&>Kw=GXCG!)MCA^Mnv#NCO2QnTZhPz_NIl zkUZdRT7Qy4K8Y#I10iqO%Hky>K~$6S%yMnpv+^J<%*Mii{fP}4$7x88X;|f?5RDN= zvOvi2d6Jgg91o*ZK2!M^jKm}Jn=_a^`ldA=)~^3a$uoa38+;LJrxAYH8flI;uNnL?jNXOw8!>IZ0QEkbg#ciW1U{*S|)d%F?=~kq0j$vm-sE zkw}zbBT1u(6t&y1;)3Rn%_C=RprJb6n(Nrk8~d>BbZyV=sn_$Gku}F^yw(vO2O<~` z8GSZb?SYqpdQr~6wPlq;HF~dxRczLJV73`BWFD%RPvg$edQyhhhzCfSzZOtA z?tjwNJlNYFW;&t^V$c!2ZNywhcyEZUC_8wR9gXjRD0M_1MEP69d>3LX$}S#dSL3@N zN*&>YD1VEX>xe@<${rf!x1L^0@XC)62&nf2fq-O>5eS&QCkO;gu2Td8X4e@4@wOr7 z2n4+S3j_j6e~Ca2w;@+L#0AxzA`npBYkvd+s(XV#Ky_~s2&nEI0s+;%M<5@F*TW~A zryBsZKhhx`P_+pH0r?&y5RmT)0s;A+A`mc}&NdKaAv#CsU>aTQpkHpDZPxQEI55>6 zbV04IA?F6+fVqMoz>K^56?Qy6|M%`>HodsH+OOLLF8hR4Dw1m14ry-Xnf%RV4}T!9 zO2XnrTEv9OPKGQ5;bOt7Wu9`?6!67UuWeg`5mL7wmg+LB>*-|8XN1eIIk#5HA_!Gm z(UM(6W2zOFRjYgzl!eLS$!&XWyS4uMFG5B{_KMnnW@#?6DF6(b+F`_ITp_H#sr zR{wnv*Q|-lX+Fs9uH$z4J}$aGE`PedE;=v`k90@}uJQ;3iVlH5(IF5hIs^hm*WV#J zgbqcwgMQhEqJsld-GL@L$hpxuAnRMiDk$$=|9>dD&&cTyMEB6&5}oaIeedvtPo0me z&ev538iXSq(g6$Y1c5-+ArPoK1OiouK){#z%-^9pgbuR3*g?Paq3YnkRDXBSg`E=Q z+~^#zklZ3xVO!*XuIj2~Rfwu3H`2<)i|>6b!0vw$;Af=w2Lhb??*!=BZP)I@4Vd+A zFP#&%A`_Bj+z6>GSi^i9ZO$8J&S2i`#ec!oX1<70wXy|Sb8;ZkVrB3j%@Kt+m^`|+ zG8V?ODkFw?E#;P;yp-}CsefEeBYKIO(+HzvDP^g6ZM>_~ktz$uIgR$~psFJMQIY+< zaL*g;h2Fsz!#yxYhRl2emD$x^>f-_=tnsi6R^&QhtF+)oMq-($-Ii+=YtA?7YeCEU zN;d0DDg@0pcL@n1!qlDI-NuxP&5bI_+p*F3Cs0cR1QY-O00;m80JC)nuM7q7K^f9{ zvnds&0)J&nrV>~NQr48f8Vn+i=nHXujO_gRG3(XBB?*@ z_YGB{;7DnZv;EiMosp_b-IlKUtz^GHMIk`J3R*H)Zm{mjuaTlv{6DrmSv4Iob^N*V z-bDDm%zGyz?s(uoRbnUMQ6(P!YI-l(Z=V`1Cca6Z^wq+!uagf(QvNRc%*y*P+Xv$K z`G1xE8r|_r{V0sbjp^}={V0sb75Rzf-oeJBnh1B~=aD1X5czK>YL<13OZyEIbxcbg zJ@vJr>ot29O_hlw#uwLfog z?U`a8yqH`sFDKUmaiovr(dBfuT#Rn!^$h5U$;>2FZ-1sC^Z1HXBN}jycd-lB@6aWAK2$S+OLkI)`2tT2dCn6w!Zo)7SgztTd$oIrfpjEMwAhi{zB2_9?QE!&LfYta%w%d@m zuM;4E#Hq*i%zit&ayj3tCiw;DrO_42GlCM(+z6?+75Z2$(npl|z_ehE(NLidJet>c zCAX9t2XD?;a6y7kB1n76ZH0CrSc;mho@41zZsFfE5HL zSYWsZrgkKvBU11YX@6WZ00hU-fC3#n&a)h!y#bu^HyZ}!!l`8FEZnr7#L>O?ax}JW zo3)d{8uRCPy?lKiR!ybuPsIV%rQno1V8OV$#Fs@pT*G`=#uINOh^H>N$!wJqS`a#&ef1-+`p-ah9P)4C32?}L{R!$nvuPuQ z0txmB7nE-V004TE_9a&VWRp%NMFEGClqMn@@Ie{Uc?19eC=~zz6aWAK0000000000 zfPoQ{(I!C+ycqxhcx*3oba`xLbCX~v6q7S28UijHlTaKLlVm3l903SFp(OzT08jz| z01*HH00000000000DyrHB9k#96O&{o5R-Q!5DNeR00000004l2W0TG&BL)>E00000 D)*uAU delta 2138 zcmV-g2&MP&Pxnu-%>xQND9R#q1ONbp5|exg8I$k>34h`^6n@`dVMTptrGZ?45D+k$ z(tGKRvRmyF3W)>JBylEodinQbC%}~BQ)aX;m9`HwalY@|;?F+u;OlFekY`%(m}Nu5 zZdwMRS-`?LTMmuCXJ-x1AYAxa=qD_rL*tEdI->WEKQyl)DJ6CnwXAdxn}CenNeHl7k^@dMNu5k6Bd*y%|vaa zpouSJ_on5^P3q*>)QD7j$iQ~k(Z8>p$ayXM-Ky=C{BcH45Dj(R|cphh3mu!_xG4@@@$MCPH2`8;Y3%x7hIjd+BV*}DN#dfU2m zaDSJM=E2(aFx3%l5QC2B>>_45!rdZvqHN(&wluyCqSO&x5an+Xvweu2DBF0HZH@1M zD0PGfqWmplrXzZIlpQq6Zymjs;FTXE5K!+i0s+aMAP_KnPZ0>1TxSRb%&v0;;_gB& z5D0ktmk0!u{tAKgb|Kd~!~xZvAP`X98-D}>s(Xt-Ky~jB2&nEo0s+;1Kp-E8*TpBC zs~Z5dKh_~FP_;1v0r{RF5RmUF0s;A+ArLT|&NmQbA-X{5U>aTSpK^96?Qbb`1k&FI=Q^PKCIdVF8hRKELP>B8L-q$6ZxCU9)Cby z<+OC>L{HF4CB*rhw0%J59?J1*KK{VX7{}yqS#G{DN}1n{#uyn)`uj zD;lzks7*D3ylmuef)&AP{_Hf}rqx*Q{_}ukQM@$17$FsYoN(i4U`EL1_V5`w$$pN? z(5inQ#5Jwsvg-}G*|wcl*TY5E#eYTD)kO!U;js>B!BrlCK+z!(C^`fJMTbD3=(>AE zhtQ$u_Rz1oP;_u$qC3z;2RXMo2V{MRSOw+1@BR-(_Zd0;f#@E)JEF7fw&(Uf*y=o7 zb)K#|&>$S^kQP{I#|Q+f4uL?`ArPoK1Oj&Eb8nC85IV^Aau5B=gQ|lA6Mx-78+J;N zbE|W}LUM;#g>8}dxvDD{WhTmo{E${AUOo3?0k;2(06!zWKM>%|dndqNv(xIdz25c@ z2+@4^Rh`meNyjuvI0=|6U2S7KTAw^j?f$I(D*pvnpY>T7t2NF~>N5lpXG_9=)Q1&r zfBfWF%2*Ih%Y+i~mdlTO`hS|sx23W+NcbAn=Mp3@VphcBt@f_YNy@|@r7S$Gf-1B4 zM@bJ4f&;gI5V%KQh;u}SMAl#fmDyEZs)GY0tkEF%m-NOjmT|^OLL-@|)s(ANQJ-{F z*MjBMm2BG=ObC{4?pD+fX`$}qcW%U(*xaa+Y!8q|KLJon0|XQR0JCcdvuP7SW;Ygp0Ou+-NYT%i{$^kRixtk8=Uda*(;QRpQKy+omx zDD)CZFNAAoH^8QyX=xJdkY!L5Pkk@XvTZ3EOMje#)bv!T+Qrb-&ew7~4=PM7Ly^=U z_WOpaP;jI)$l3nu@XkoprEW`C{Z_KypP~?;UJ_A?6*&i78Bp3Px@+M*w@JiBPoBEeP-o-nC$~` z{D1sPe~s?=rG6B~51~6E({^#-;s+i8`jG zj-LA3(DjAuN5#tO(>Gy-brsL@ux`1)yEpC; z-S$i|4_-{JmzR@kfjH7f^5}9pTP{X7^Lhq!#N_FDzS}qZe&6#PvG;sBel`;^s(QqB zJ)5|u-S^~)WH$tgLX?NVb8%v?V{UvBDz^BxbR|#T>OK434958IKTt~p1QY-O0JFpy zSqKD<|JHw#AtE4uPQx$|gzr2>qfbB_xVK+$?*WTEsuH4QbvUH)+#W z%8BE8X1|?XxmfO1ll+47(&!T989@nXZiLiziQYG>^Z_M4FfCYPG?b_VkCxR2!vtuKK0tq}Q z$|7_G004xO@FiCQX$X^m2pW@PCMp4pla3}L8~E|lwt55r04Nmz02BZK0000000000 z0DyrOlglPS0m73FCprN%lSn5f8;<|he(^b0ARrRF8}}l diff --git a/summary_results.json b/summary_results.json index 092d022b4..f4d587084 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.200", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 9.53549, + "Performance_Result": 10.2077, "Performance_Units": "Samples/s" }, { From 1099acd93e82e8806f02d185bd01e3e49a5d8954 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 10:24:04 +0000 Subject: [PATCH 429/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10225 -> 10226 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 56a6131ec..e4d73d642 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 10:20:48,471 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 10:20:48,471 submission_checker1.py:3260 INFO] --- -[2025-04-09 10:20:48,471 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 10:23:58,727 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 10:23:58,727 submission_checker1.py:3260 INFO] --- +[2025-04-09 10:23:58,727 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 10:20:48,471 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 10:23:58,727 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 10:20:48,471 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 10:23:58,727 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 10:20:48,471 submission_checker1.py:3276 INFO] --- -[2025-04-09 10:20:48,471 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 10:20:48,471 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 10:20:48,471 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 10:20:48,471 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 10:20:48,471 submission_checker1.py:3297 INFO] --- -[2025-04-09 10:20:48,471 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 10:20:49,539 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 10:23:58,727 submission_checker1.py:3276 INFO] --- +[2025-04-09 10:23:58,727 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 10:23:58,727 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 10:23:58,727 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 10:23:58,727 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 10:23:58,727 submission_checker1.py:3297 INFO] --- +[2025-04-09 10:23:58,727 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 10:23:59,788 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71659 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421036 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27905 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.10717 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d890176cb..c500a5fe3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1056 +1057 diff --git a/summary.csv b/summary.csv index 3c9cc19ef..7613478b2 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71659,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421036,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27905,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.10717,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ab4357ebc..9eed4e75f 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.27905, + "Result":6.10717, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index adcf70a0eccd3c828afc98a2e21318c7aabc640a..90d08e625e2f6b3804388768ec61574794fb88c1 100644 GIT binary patch delta 1137 zcmV-%1djXhPx4Q&atH*=a~4mtb_fRuf3jyioJ(zTE;X7)Z0YnN8do}fhz7W18p$Qo zNG_R1a;b})J}iOcvM51YgVP6Ee9{SSTtZlfhROMoiC_(@E4KPP2If!Mx~!zgGV55q z?e*&w+&&uIe?HLS zlTHYl;PxS5u##n|VI`|8Hv9ZNWIj{3&xPK0`=qPcrtRfD_|ZBCO>~af((XevuC)6Q z4We1X?!z>ICeZ+zMRp&S0MIP5`)m$Apb2&#(g|)|LRg1}0h`S41G8quexDy@^GTbR zl@wWO*4#Gy=w{utS~-3n-OND~f1M+?^!pHvEB!u1gJ_oU`!EflNi=|Fk>7_U05psI zJ{tT!(BhL$2%6ycAz?UcGP@7Vnibo9{vI-)so&>9Z~J|+S<}@z?_RT&gC=@MY-#u* z8dn;Ahz8LtVfbMhK$BO(ZhWC^Pe(*ToXJaV^07g>E+C%|Np z)klNX2U>j6i85OfO7Qy7Frbo|ePAV9vDxP_Dw9D5qOFgT<3AU{%NM;D_Wp~W{puG( zzcLPuwDfyvTHUgj>;oIt(ZRes3_f+E;dmAslVFe*YSx{a5v|@vx%`}j^W~@XZ`#)| zNy6#GmJS6m4Vw8TlOiH3e^RPqB|&N{ zPDQFzs-oU3djYHQk8HPrx33c*fW)cC^~`=dyK*(#sW$lq=cUm#$}@rz(A)^AH#PcL z&eKPf_`tMat5ci~fmd`KCIcf@0Oj2^6lIb!h8MG+rWct;Qyy`y# z)p5wbi6_7z_w^@e{s6OMBgz5<%X1b_voR(;0R=G)n)xP^b0<^*WRu7zBL)^F00000 DV(loR delta 1136 zcmV-$1dsdjPw`K%atH+C-4e#Lb_fRue;qH+0IN;TrAE_;EuB6@<4UIw(EyiBBe`T6 z$tBZBE_IR9hb53)7A0tFaQZ-tPddSkO9<=GFgaf`5v*Z##a5rk!2Bs&mz5M*W*w`y z-9G6$)@`{tQ>IoqXrgb#mTn)Sai!abXb{a3ZXc!rG>Hb#EOPs>1b}9d+ed@je+OE8 z(g{Hm+&&}>RD_=2lEre{;l^ejlQ7rQe5W5X};PAEp5`i3ZRt^82s^fM${3 zM}yx7T71$8K@f1@k*`#eVG z|LQBKq`p#6g4bv3H6puAS~*OjZN!#VAENQ3)rV-1$r4r{rU53&c;s%0F0%TtPJqcG ztB(e&548BC6J@p}l;HKDVL&A_`@l-JVzbXObK-+M?Ju15$JwhMx|7v=9+UBGXvBZ6-XLjbZ;&*zWWB-pe_|3C$@h71SPSmv z=`WNOTQmB1gVnnujd?T z<-)0C=q%i{p2X3;_i{A0ZJV`|!5Z`Dc)fgmA68AJ?oY)5)urH+J7B@My2O`7JY2(k zSjH1?B#5Uj>NnnwiyZnuBryOz#62mWljoP!0@Vd6?o%>N$!wJq7+MfIoqhEqulmnG zbsX|<;t6obefb$^r!9-4e#LGA2C%1px>@p(T@aCsYArlgTF|1{Ea$0001d C1|u&3 diff --git a/summary_results.json b/summary_results.json index f4d587084..c00e071f7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.27905, + "Performance_Result": 6.10717, "Performance_Units": "Samples/s" }, { From da253cd49988ffc4234b301cbd65b39b4964350a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 13:58:03 +0000 Subject: [PATCH 430/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10226 -> 10223 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index e4d73d642..70e38f446 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 10:23:58,727 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 10:23:58,727 submission_checker1.py:3260 INFO] --- -[2025-04-09 10:23:58,727 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 13:57:57,244 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 13:57:57,244 submission_checker1.py:3260 INFO] --- +[2025-04-09 13:57:57,244 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 10:23:58,727 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 13:57:57,244 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 10:23:58,727 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 13:57:57,244 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 10:23:58,727 submission_checker1.py:3276 INFO] --- -[2025-04-09 10:23:58,727 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 10:23:58,727 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 10:23:58,727 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 10:23:58,727 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 10:23:58,727 submission_checker1.py:3297 INFO] --- -[2025-04-09 10:23:58,727 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 10:23:59,788 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 13:57:57,244 submission_checker1.py:3276 INFO] --- +[2025-04-09 13:57:57,244 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 13:57:57,244 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 13:57:57,244 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 13:57:57,244 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 13:57:57,244 submission_checker1.py:3297 INFO] --- +[2025-04-09 13:57:57,244 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 13:57:58,809 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,9 +31,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.4817 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.2077 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1716 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.79155 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c500a5fe3..9a9755e49 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1057 +1058 diff --git a/summary.csv b/summary.csv index 7613478b2..1ef39875e 100644 --- a/summary.csv +++ b/summary.csv @@ -9,9 +9,9 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.4817,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.2077,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1716,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.79155,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9eed4e75f..f4c2b4d25 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.4817, + "Result":23.486, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.2077, - "Accuracy":"acc: 75.400", + "Result":10.1716, + "Accuracy":"acc: 75.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 90d08e625e2f6b3804388768ec61574794fb88c1..4b55cb1f3402a6b08f0786c7dd7afa8dbff3caeb 100644 GIT binary patch delta 2769 zcmX|@c{CJW8^>qHSi{)UAlsNBBa=0wtjRKjWS7v;*niek#*i|hFvgZhjRv8@6j4JA zvSb-0S@Vl5VJNaMFTHQ?JU5|IMM zQH*75!gr)7#xpkIx-;2$Mf6EjzZ8x_h|iPAXRE3=DZlw^?Drm#vUN9a*B~6Sqrx0K zH?YR9|K&%tTSWbHbgzu$Y#P544#5R3<&QJymlY^MgF%r~>IC8s0d(7{7w1kNu{55| zZq)E_q(xd-Vc0vP&}66Efuz0K(y}aj+0v}-GzYkQ^x|b;Eby%dakdj`E93@%u2gnyHKEQk3)OCA zPGR-7mkzmncC2!~bY2!$UyCsl5G-O%4g^|0{W4OL{#Huj9sy*7VFQZ*006H2tq<_* zapIff0RY5lH@NoL38UpTBYD%OlY-}@l}A)++_>R>lGZm9p310_db77;l^AFkZ?R=MwD`m4Jz%!|Cnvv-fCbC@Dj#jn9lfukC33qS z)JeutS~V5!ml&!ga8+Ci?RRlHkr)ECyVSglhdiz4S&LtmV>aQP! zO>P@KJBgm!q1B%yb%g=S@1H%5j6OBRMxFsxQb|Od+w7z)6kByGo9G*B{;imF6l9N` z_RI&6O%jeO>}VDDexuoTEMV^ZNK?EC6Z|*dyJak23bvPY`hx`z(&T$J)pV|I`n!tz zsH7rOzzbsy*~kB6tik2Y?F#t_$RISku-S;|+;Bnwyh$~t=5t+2FV_Py018e~W+KPk z6B9WPzYu*BwR;_2@U&*yXd(B8OGilVHQj7?{>&MPN-F(;a8VZer0N!9pU{F50?Rak zOJ@N}kX0LsOB)8s)@a!T(9&iaN<#!igNXASi`32<8Su=8FkU4M;vT74o@uwrU zJ{TizcWEfc1vMI^9ILFgb?U6QT3hS|>+5o^=7ChPis5qwE3JcJuO1jb$v1{w`!_{n zQ{Wx}vET3bGg+uj6rX^3E87{dBX}o-0hYuxjb|P_I#Er<943y(i3_yf)ian3Q`?*r zo;{t|Htn~}td=>?7`_g5Rn^PmV4_ZP!^vM}ZqLrTm;|{v2FIUr+!`o5jL>WQxZC7R z$fM6b)^(;&ojB5yaCUq82YY(tD#%=&fcY$urwWfFMHg@qUyTPn+%Abpv9XFCq$=_l;)5zY}wy z;?n@s?eSsd6|dXw<-N(G*DH-wZD$mC=w}{hE3OEctKRa*X}>GV_w^N;PxH1BvBspBjX5+;OP- zT0jwa4enW1l2+LPgr8pw@$JCh$6xj znlN^wZUIy9Ms?ySGjU$9kSKTYDyLm5Cs*++F(U0xps$eF9ibb_HTOfMy zz)vBQZo3v=s;Ec_km&H;s+-qysHv=t*IA zVjqi8`FQ>T3)s`Q!jDnn~VR>nua-0lDS0GqPcbERigk7}SKZB%*A&%te{3laOu59wi1L>zLcukbNsl zjV#UBhbzQLp)4V?%?$d~{qB9f^Zd?p{`$So`+LuOp7X9)u31+0^73s8gVIS%UKbwe z(3&%RK0L$)!#GwnL&j}oa&+tM(p^jO9xhIgoUQ{5g%^}Iv-eAf=n2*c;u z*&AIX1(j{!_S3POw=r>7e}s#VJWuTQMAMhbrvo|YQ;qxe>J&;%BqhK^uzXF3@4jf1 z0aIQaiQ?nAIPyUr6(f4FAv$6#o*d14g%t%SqUbM~I&zDS?TW~i>pzId#(w&|7-nnW zgGAm^K={hHXv@5m(!LQ}L2wN8#31muCB7(wVR`Xq`*^nT2DHcQo)1O)pw%s-hOgI) z7N%%6K?AXQ5;GHX(v^=UAXi--XNctsCGNHfB{1#5uJ)vraAn#u@zB%_%9bEcE|I&! zH#h1%vfL(;wc(?hCs_}XOmm+QqAJ_9g&p_qJMPivGWoTG*gEX-D#5z0n&195L2eM| z{!O;`P^@;AZezXrk|Jz=~VW5do1(>w~#S1S)*ok6*j)=n82;j*8x zGH*%U>eF7hW!TLz)mk)Dlu zK=g5!>XM`?n28i1~BGB7C}CU>QNn<9Ds>A}TEYPn~y_}nv&KLlN_TKz%Qsi`}2 zqu$(DM<)em`%vs+Q59`XVIR)o|KKnp=)J1@bOqtao_l;ty{$w+b@wDoRMd6xgoR2Z zfTR-*FkiHfKDmjB$Ym&YDRofl+9;dS-HfKTo~i|8K-B19=>?5)jYV-0b^ABvZ5lL% z)9ohlS&D?Om6s^)|3Ml$MtKT4gbgODKant1TOLp=OiQ2)FPqzXPGYG}VHSwC4zO92 z8{ypbzQmTTrEF;Ies&18>fTpl%3hxcP=4a%=*BtI92xKd} zes%@Bq#eam5)Gu-ckK`~x71=|Y+LWi+HJ3N!Vd{DO5wg77888SzUuA`LB?yGWK9~6 znbpgsdF+ako~rNa!e@iThvJSoAqkxs^d zVobRVm@?X&*GA$2l&VZ%SMnLSr}i61F@K67?0-0NEIy{FIPzsU&FXR$OR#E(oq6vD zo=tpdGoCNoR9<~UFW1bQa7r`B$ZLLme`r*Ag1`$mFatJZaw^-1KuyX;D@V;#@9P^q z^b69*>AYQh9D|*f=ps_8RL{{B_-r}W_?$vlU$kpk+Qg1j`E>VYtJifIN`^!01GwT3 zlvrp?^vz5^u9`?8RwL3QbvzzM1ZX5ppurR9fls7MLOD~L@~)L>?o?M;sgf3VN6!s? zrJ+A|GLTCt9ecVotGCTW30ed^935QVH$g2|T!d`@X1Prx63ztVTnv=W1PmxwbG<~- zg~EZdgTg-7e^LZK=~>~e@#u&skaho#eeyWtus#0Wgx1cS2dmbBs1qT}>M@kegVwMA zq`3Qw;s3L(0bb_mjxh#hMLp>UDUTG@Pl0GbJmNN&1>1`NnP z{Fe)djiZPEcD$p6={L?#$7hGTz#OZ_zMNJh846HB1|`Q#@k(`(tUwM$yDz_#B@J1U zhA@r?QJBQ|o4TI+_pMMJ!;JTgG*Oq4tESKGdl8b9j0PwF8amw*3u&FZ&=zUpu?~b3 zkeG-9lNBrmeeFp~J%1|Ld1PobbXE=X zH$i=z>94k8EXIOf3L{uc6GP|IfSjj9P|2M;9gLi8nQpG&P_l@qRPbKUGtly>6 zrjrAYUcJ2bkjDYEQy^;B90v)nReMM-iO{So`-_mnX*m3%jP_?O-tSse=m96xn&?~W zT(A>!q>^0WO0Cd*abET6T*rYHyCRU%Ixg0kJYaS>B=Nke)f_C%S!iUyC(`2F(4fMo ze0L!-pW~eGK5nf+%sk6aou^vYB^8zCRm=v;*kswoL8$s)w?u4Z~j=To`Ns7NK-?&#O+*=wFSeP`$8+9YQCRZl1ax*R;EZ;L~G zw1Ua(;X_o%d<(!T@K@M^_2D;_6(lhu>%CLlo-VH1xZo$^HalYv%vv?uMJB&}Fu$7g z^ORyI&Q7*=7Uu{B&3s9Pj)}8<5%|nI$!1&;$vfs|C-gSziWW6PK^7T_(9UBqgDZkW zi?>5(0)SsW!GXQLE2-|BG+N6{TAhB7SnZ;W;3oMg$Jz0oyK29Ptft8C)yB^lqOl#1 zRZ14kiTxf3`r4iG$LOHnJN3wt#%tAk-ZZ~?ICQh+Sn}#e;{Sfp8rm@jA->ZhIc$;)M3&g=^F~cPW+nVSeXISF zbqNk;h*!FFvBai?JgFQCEL~0UYKUs6kd%vTr@r0ji8$9p&>W{~vH8FJ$<*vRb5CMm zKTT+Ba_stsjMKMiwJv4M%f))Z8n>1eeNJkNPUcok-Qz33Jlaej2YX+rJp4z0Lt$1w z`GT>(cP>so(}x(XGX24e_l4XHP)W$VlIAC7zGQ+irc6Nx_g(}7gdP_0^P_@uZ|I;y zdC$)JY8~u~y~3z$@EZz|Fy89xW{%#u{w^b0^K<_yewBlMo@2@?xX+7MY&{u7yXS8a ztC|%2B9mR2fN(Ywx8y$i5Pa?wVU9Po9k4FGMG4F7SA}=$;15`vTN4V>=G76k$ay!; za7y!nKJTzU{dFROCcm|TxVZ`DaB&9U?K{hQMahZwM6d6v+qy|66@O#1WtvDIS3ax{*2&P=O<`pDq6dT9QG` diff --git a/summary_results.json b/summary_results.json index c00e071f7..86a7213a0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 75.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.2077, + "Performance_Result": 10.1716, "Performance_Units": "Samples/s" }, { @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.4817, + "Performance_Result": 23.486, "Performance_Units": "Samples/s" }, { From 0d847ca38287f6227a01abb441a706a5f416b79c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 22:24:35 +0000 Subject: [PATCH 431/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10223 -> 10225 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 70e38f446..22cafc9b3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 13:57:57,244 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 13:57:57,244 submission_checker1.py:3260 INFO] --- -[2025-04-09 13:57:57,244 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 22:24:30,254 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 22:24:30,255 submission_checker1.py:3260 INFO] --- +[2025-04-09 22:24:30,255 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 13:57:57,244 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 22:24:30,255 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 13:57:57,244 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 22:24:30,255 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 13:57:57,244 submission_checker1.py:3276 INFO] --- -[2025-04-09 13:57:57,244 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 13:57:57,244 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 13:57:57,244 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 13:57:57,244 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 13:57:57,244 submission_checker1.py:3297 INFO] --- -[2025-04-09 13:57:57,244 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 13:57:58,809 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 22:24:30,255 submission_checker1.py:3276 INFO] --- +[2025-04-09 22:24:30,255 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 22:24:30,255 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 22:24:30,255 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 22:24:30,255 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 22:24:30,255 submission_checker1.py:3297 INFO] --- +[2025-04-09 22:24:30,255 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 22:24:31,443 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,12 +31,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.407176 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.390445 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1716 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.79155 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.48276 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40724 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9a9755e49..22bcad02e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1058 +1059 diff --git a/summary.csv b/summary.csv index 1ef39875e..4a93ef3f3 100644 --- a/summary.csv +++ b/summary.csv @@ -9,12 +9,12 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.407176,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.390445,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1716,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.79155,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.48276,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40724,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f4c2b4d25..7834df2ec 100644 --- a/summary.json +++ b/summary.json @@ -304,7 +304,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.407176, + "Result":0.390445, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.79155, + "Result":9.48276, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 4b55cb1f3402a6b08f0786c7dd7afa8dbff3caeb..78dd8f4a5f090bdd0df10ba26296ea6449d92765 100644 GIT binary patch delta 3174 zcmZXXX*?8a8^*^P$~FoqMq_3uYm$>~P-7X!Foz>c3lc*iYqpG#29s@UlO@wm-3tO@`Ck~W7Q zK;(d(BBt|6;sOH4^ZarYVQ)9QJo9dFza{gE$nn2p@0xHDoGYRB8$ASF|IOr?(TS$8 zChTrHo@jiJm`vGnlw zXfg+jGmer6qDfJ*7YDg1dsUidsqBbC(Ou(q>Y2|SxDYk-q!WK(CoBd*h0bz$CWdvi z6VK676B@Z)RGqvL+taKid0Ha=2iJehmoD629jK^NM6m`boN{3<&d1c0#m- zX^;4DT^*RzJWZYB%qPTPcHKqk;>F)T6FhcTt47ODPj?--ra?80=OXmG(E)u+%CnH1 ze`>XhM)_@D_S(O@VC_|K&{%^qp^xuwJOY;Akt(~DDHlFu(P(lEv)6~u(0jY(&FU+8 zR53I(_}+o$+BZHXwGz9v&ns6UrxR9Z(ih!@^U4UWs82qZS@XqA^M=o5jlqrkykvI1 z2BDoH-Jx|s$?*iyH0NdyG^M^NRS~=q1pSPd^?~TjYlbvfnC2rIN<`+GgKd1-C9}(z z&cGCx;@=aldm1*2rx*Mskq-Z}KHSFli2$1{K0j7E{3>0BzLy(+Z|`yne(fN}^eunb zv1<^{u{wkcv0C61P8lpZJ$>GcdZoz=Eb6{1mV&3y&jEdd^u^+?G`S&HJn=B{m?v6}5YE(Eo}tepXi?zBRIm^GcWy;%m0mYIAT4>A>=m15h8Of~8U=^N(u z#%i8^u{1*@Fr5fFzVJc>i8F=r^s#Uv{6_?f)qGIx{~hPoAQ?Y2^65biLHzbg%Y%t5 z^dPO6X(oT5AzaYj7p%!w5YQsdfEea5-X>HI`Q|7;n$31>^43gM;LAuTH?Ga_4EKCd zK36D<&Q$Zg(~>bxcWq_%ErzwQT~u@WRtq|nkP_Bm_eLyKa|F9OCRE=Wa2KEnMqU_YE$R+wdNY_o|#ug^yVT1IK3!;9gK8AWWK-@pk8YED{n++ZEh^I$R;lN z0`(x68wQn_S%D8|D^3V74w7ODi9>*q+XAMLFR5>%*2QDF;%;%oNeV$iQxQNp*l@oB z=GCX1ke@QZG6gbs+~=u}MPpRzy?InT++@JyBoE?94S%Sn0E|N%s>Lg87~xFT4meLD z>zkS9`Fz-%Y1ZSPVA9>>k|WF1=N;0pU-gzrx0Stzyj2Oiu*tizI{obNmB9I~Pw6}ph^&MBio=k3rNnN+{yQG5D&KqZQnsUYmrBNuhW+Zl7QfXhS0kRSm@4&bg|?+)0p%QQXSVhvMj$WD9M~tMKh(&eBq_ViYV&%fBpYv3s7_CMb^6``puzWD~x{ zwEGf~__wJ#X09<6H7rX)aJd2nljEcy))Fa5Ab`Zf?~olW?An{5$&5)p-=PgTkylnT z>{s^nepw_=HS3RIy9LZjs8crGtjR+s#N58sGPL}`{y-3vcRfk?>`v5~0Z zncr*e5ilgcRUCr#!U@5X&pS>OiQV-?zB;k;NzH(WfAI1CKZT!cEFJo6F?UL}C=E|u zdB57t*6I&CIX$XxTkq%{21E_lw{nYJ=7S9YvJhNOsnQXEp$nWu2te&Pkg9nM)Rv~~ zHOS9l9UEvWi6%iVqe-JbIWi9^J$tD^#i(g&d2{FeL_s6m%gQzulvJ{TLc%zZ5BQ1;J`0szYG1gF~H?t-tzfNWn_nL_1BB1?!k@zCqHq@J_IiC4w& zauwQ$Ib@~KfGiQp&_;2gfcgXtd?~m?Xa%7i6c41H3=Yaa^2E*hDvrNZT8h^Om=Z9N zTY8~I=X`oXCDXOu?#}sSly~uawtBWg`t8q_k(;E6;EyAvo9P22@6}A0taNH$`2#&fv`xl?JvBiqq{i!_n(nV1KtQ??P7}lf2Vp&kX#7h+GaPb4kmtiUuGpyf zNjQGK|KUkp>v#R+iP|4m0`E3nSftr{5dBLBF3p-23J(MiE?v`TgASRhkc|C$G*ZoP zx^6r3Lo$u{9i#~Gqb-|hg~kM8vvO2Pe4|y3F2((Koc>A6eQy@8L`fW>yS)gIUcuSl z`j}SwhtPBE8Q-Gur&wJN);TEzC1(7o2f(6$LnGk>T!|y!-jy}r#Cm*`{d|5m5Rg~V zK10g3rqQLcfBM?OrOW#_mb@sIjnBQbBc`H375<+Pb9lK)_@Oa%hsHdIGTtC4RdOpKCmy4SjJz|us>iGs&yJvfUmyOYUK@G7-rjXtGyl6?@qZahu z>is3RL&CGZMUzK_yT1r0|3~=kzl6I3PP_jS27!RF6qx^hTe*ETeR1vF7HQ&<%69Ff zeD-C%imlbl6=yXDbY@LE4rhoPT)(D)cjNC69s)c^aACzyN7!*1P`cL(+t;~{R%lG% zDD{ya)FVJQtEU$)s{g!x+Yq{w9_9CWbwrxEu^y-6?svMqcvphrJ)OLav~*Q0tr+GE zqBDt8SN8K*e)s@CJVcd|Vhk$SAP+pnCU*mkL9aN#J;9OP;KIW^;+53DW7WPJm}dtR zpd5kx#vmzPgnFT{2s?)WXCuB_f;Sr`!l`oj;q}_QuF(YHVndA+=%^luW~g&$qisqy zZghk$+duNp&T?m*@Ftb^U=-x;6S{=9tqehRQC^DvYzTc^L~Jn_{E)iCyO@AL>xbqux8e2+2{^ zC)0*nW064g8cvF5GD768KKUU}v>Iv^*NyI3Am^F<cW;-9g|UB&Q5jgeHMM)U8!ESU0$?EZ@%_w zlZaC1*p{@y0%hqFh9OyOhbPldv5pejxB!utCseZRK2$bCPnH5!AX#K&x1EM&~^>Xt3x(|qIFh>qNkpR=>01|s( ph8)er*uP`-l|RKHOnfR2=eR@cl9%H>1qB?w{{Zy>{*eFx delta 3205 zcmZ9PX*?8q7stn*br`Ztwiq*029qu3+V`h0F{JLy)(?b(Ow54!FfU4WLwDHNCxgWGU>?rImM6?JM0a zg4B5FVWHs1VUy|Jll=Sdr@z$3#_WbMtED1F3%|FMuZl}gKIe&8n=1JwoxopWxPTFc zSmb+)%S~FK)#t)B7v91j#J<0V*8$>Ih{lq3?dUx%u=c_qV$u5p#Ve%`?Z(1v+uyr| z=J*(%B`Zn`^w8hgz4EY4cB{7ragoDFLWZ&#OA%|!#fJ9x{J%?LkUCW|m51cH#>>Id zGKl@>vOIGO)l3VQ{wd^v;{LZfysL)B6^_P@T!zlw^c%=!RcPNIJ8PzRq--W>tI^HP zN{!MzfX`Ub(ZAn@vu%${ZNd%hXs^Rc5OwFk+xy0`)L@Hn5!n1z^Tg>U`&vHdA1If+ zAkno6=YhtioI}|;SSPBi?o*6z*ZJAf>(Rou=h6eGP4zZ#9qBVuD18-r)pwgoN=9{E zm`+DHauikH5O;lCFA)C*N0_?9FBN&_)DtY@yNju!3=`*=542`T4$hvVlu-Q%J|F^J zxfKEmxD;s)S%0#aZ}lt#TmA;B*38SB4VO6SOOd#DR#nc~Q~z;`>V?&dGYdIS<-P7& zW9A9gM^%HzEhe`QFn1$YF9xx!@2MQvz0f+yx9Z-wVFFqB^921gh;7wF;R?By?<77{ zL8N2ujCG?qicwL1@EOMoi z$||-6iD}p7K!l2I8Ed_W^|?8lUb${yDNjU30^&sYoLqZRvAut?Lk|qK1Mx#iDh5!P za7>xe;;PTPm~KNoFks({Ge z8)rY6x>W#4BNxPfjX`BL&>Rw?f+vP7KwO=X#EIU?3mFbS`%nga3(>ry=O=^n=yo3q z-X=VsDp32;HNC3pK1hl_*VODXyVW)Atygt;;WH09PLB{=BLwD9zl?u`ranNxIbDGq?t`kfwadsbnNWwEC20cEJ*|z?Zlo=4jdV zLT=+8*LSSKK}vJ?#%>?ciX_SU5ReUb3LK{J5v4Q3+UTn{aBB2KH`>gp zGNa)Rn9umB**BlsE}!4VU*o5FC^=S9vq5k#T@U!vH*M8+cgf3<|zELbQSq%oNbk{8o^8QunPd+9@1_FgAju5TuP{22zBM9|t@_Y5gS&s)W-x2~+FC zrLRJ+l;>!ixh8E6#E1#gEeMM8C}ZAYD+tJ3fe5E|i{fSI+kB3Y`EOX z7J$iiqX+(x#8g;NPn}oqBZn5}lS-=a}<_l6Xm`^;tyc(uZr_ z%0`84!P?Db>6a0^QQLmEnbRsZz69{gTvyW4_ar42AlHn;mCB9l&q(n_g8;c)yffD8 z2+F?=6SjDKWWdmJTj(v_sLESQy&uROg5vngFNa(2T};?qah<=B>2ITsd9OpV9hc8D zR>veCX{neOXP3^f5g^S|1o2_boLs`TH0vUnPdt^f0W z>Xjt)LOD^E-5I;&QwSiNr^qTDYL|~JG+_GIY|=hu!EZrC<_rB_ye*W0{JNt*h_~iL zn16Umv;5q)^g+dIRvOL{_O)>U9{G=(YM$`t^#G0mvb)_m9L067yTSne#AZwM=bF;y=Dfk&d|WPAM2YtW)c%}XY0`Pbe1Bl7?F?Y@^#&hn``8vWRC<21Om z{!hk{p*1xEl4A~)%LP;*@b$a!FS|xZCviH3{mWh!Qd11s0F>lA8@n%`YdDAbVk={K zKj`&0zbWS*RjE;r?1pI;%run@1GD$mB|sM7zO7;_7Tl*aW~CZKWTh%)d5S9q1|#>5 ziJgfyCq9oLb$?eEsq%?=VPEv}?}IbH-Omwo!^u=O4;ieWCPVS9x=v*-x`EWt?p&N4 zHG(#s8Ynyj!pW7fX%e_c5bFE2oJlz1v}N)%k9)!>ud(Q$0Yt#19-&o*jd|f!)R8(E z5cH#UzgGZ23I!gNeVVY`LA!t%~ln-D8F1UDL68W=+2Ej zHk|#(F!q0j_5T{qS|-o_*Kqa7Fu?BPkDG1{zRk=z(~lZW`<~08?GD<(ZSG66!A~Pc z%r??g1PzZe#4~^_`Fte-LtBnOkYKSJcE^VoFr>h^$ISpX?v#qwo;13cYR;DDe_+t& z+rjk1{;YX^%leZtmr{J<)3*xs1Eo0($c#nEmG7s)4(^F*+<6F}?>=B}Qo7}O?y^E9 z@H6KFMEzc1?LQmA;A~8Iz2v~$7u6epRaj;1eY~E9zj|-dMTct&9_Ub-c}U;}$x{?H=uKim5l?EgEGfkAQn$O_;#m#9ANnp| z>Ul`ay`g4T>G6Y<=*>W(9>k?~mZQQnrJ25dAc0Ynp*ANC>cjy2ei8ZV^MmZ4cKcu` zT8I8io3HIqzW+Hbjw=~yyWW8<$8KI=mi%IGNE6Kw!oQAw9S3Ct036u?01*HHfDQL^ zj10dTrREnN>8~0e8d69auxB8J&9@WG7ZVpsX^vHp96*e@66B82D&qQ#dpB@8MH7E- zGjPV*txkIGw}AC@N=;_{og1B~ht4)Y0|2hpPqaY|POuA)7DR zP7V?L=|fg#9Ha)3D>p(W*mM|i<(e3jUv$bW?y#fElp) Okb7Z@+@eay#s2_&yx;`@ diff --git a/summary_results.json b/summary_results.json index 86a7213a0..2a0d6600a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.79155, + "Performance_Result": 9.48276, "Performance_Units": "Samples/s" }, { @@ -365,7 +365,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -401,7 +401,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.407176, + "Performance_Result": 0.390445, "Performance_Units": "Samples/s" }, { From c500adb5297df290de057691b6453d485f6d08ad Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 23:08:30 +0000 Subject: [PATCH 432/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10225 -> 10228 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 22cafc9b3..8200dd317 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 22:24:30,254 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 22:24:30,255 submission_checker1.py:3260 INFO] --- -[2025-04-09 22:24:30,255 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 23:08:26,014 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 23:08:26,014 submission_checker1.py:3260 INFO] --- +[2025-04-09 23:08:26,014 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 22:24:30,255 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 23:08:26,014 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 22:24:30,255 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 23:08:26,014 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 22:24:30,255 submission_checker1.py:3276 INFO] --- -[2025-04-09 22:24:30,255 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 22:24:30,255 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 22:24:30,255 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 22:24:30,255 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 22:24:30,255 submission_checker1.py:3297 INFO] --- -[2025-04-09 22:24:30,255 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 22:24:31,443 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 23:08:26,014 submission_checker1.py:3276 INFO] --- +[2025-04-09 23:08:26,014 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 23:08:26,015 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 23:08:26,015 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 23:08:26,015 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 23:08:26,015 submission_checker1.py:3297 INFO] --- +[2025-04-09 23:08:26,015 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 23:08:27,002 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -32,8 +32,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.390445 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1716 | acc: 75.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.342409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1445 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.48276 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.996 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1762 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 22bcad02e..bb5b99233 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1059 +1060 diff --git a/summary.csv b/summary.csv index 4a93ef3f3..049544bd0 100644 --- a/summary.csv +++ b/summary.csv @@ -10,8 +10,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.390445,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1716,acc: 75.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.342409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1445,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.48276,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.996,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1762,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7834df2ec..83c3c3b59 100644 --- a/summary.json +++ b/summary.json @@ -339,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.390445, + "Result":0.342409, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1716, - "Accuracy":"acc: 75.000", + "Result":10.1445, + "Accuracy":"acc: 75.400", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.996, + "Result":23.1762, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 78dd8f4a5f090bdd0df10ba26296ea6449d92765..2edeceb7f8c2d1fec6d58b2fcf6a8401755a99ac 100644 GIT binary patch delta 4287 zcmZ9Q2T)T@yT?N>8cG1Amk>y#OA$e7f=B>Cs&r|g_o}o2Q6bVyf+$TO^di0YCJ=g& zPCyW8iU@*8k@rTw_q%uI?wp-7vvc;pzx|(mcJ|qQ{X_lA5ej;rw1=8LlmI{o2p2}n zf%{IOM$R2tBSYy$j;v5Sp@&?R2X@Op1)VBOG?Sio2}LPjQh*js z`2j8E3x+7w2>~XUA|?6V*%blBrwn%+P!aPncoc;ZF-jPzIM6hq!2iXvR|K|a{SOk> zJT|b+sBt{L0?OPC2;cjT($7pP9i26y;e0B{Gf8%7moiFqT7apDq-PSN+WGoqcqNA< zTHQ9M)^V7#^Rd%7c>Gx=XnkpmyYl4{*woH)9a+9pf-s3K5t5%A?2X$C7wi0nWd7)! zcT7W`fjrxz+?w;#A?}i$nD70ZRIn437BKHicsUp?H{39>(Q3DGd1J)p{Hm4bMjXsU z)bgbqlm;wrO8w6Sx4~l+A=h2h!R~gcw0-QXq1QG+=h_Npa9S4sx=|rPN;}0c<;84Y ziy}{PwQa_fO9w-V$Rr=z{K}={)-IbZp0Mjn6gCcy1Hjeew9l?J{fY_CI|^M$6N`G^7IA7MqcD7$~?r8 zNps2zd%+Im+P_WT{2MVIFgae}Dqld{FRLG{ctMw#xez)UE)S34 zOX~a47!)whSJ+=25)WhNo=Kd(Ab2m7%Ox_Tgg;6`dGF6S3yOYrXTQMw{$7PNlaX2X6Wi+!?Zx1UM0QBPes#IHbWk)(Q842&Iyl!@QOHC?P*93=txb1>mazf9(xWwUi zjjPYd+ZLf>r3 zLTh9YWx&kN$Cog3=g~DMRL<(;IyOEq-*u`OY(dd65EC1+0zwxZ^Jw4p{kF2AGa%W^ zm`CHA5GF?Jy~5X$m92+B>Gweue?c4+*#(_5#$W~os_S3vKB96NJF?o8j#Yy69w8;J zs{1x`awg&?*4*2!>TylMPmg7-xAU?#)MOhu#;Dbj%V(q0A+fmh$NPbbi8cB9VgByZ zBbp$pEHyn{slo%nSL#RjzxaPJ+=G*%!A{vyPs09hs02d2_e4n)k(-#S@uDw3)kN$q+c_{ci;FeW}5MPhw+S&QK z76cx@bs(did`;F_@0k|EJ$a%{kV?(#<@PeeU7!*F<_b<~C&9}rkXW7`ifnr}`3?j# zHPEJGt;i5(we%v+6ugHAD4->O%2boA9^<}RE?aeyr?PCW%yKWJr3i4a$hzvd;O_P zv9{j`CGH+jUyc-`7Yf;I*Lmy=RGtNW;BXo!l?qm$aKAUm3U_%wv>-Q z{FEx$vh|2^>6S))5-I0RXI0&sUsHstG!h(i#0Wn^I7tQF#OW&vI1W}a z(4jU<#Ojtr6c$B3u3U0rlS_H>p_zAXqkhuOB52t#We>+Ix)f1M5}U^dkYv^V$U!*y zS>zg%J+ENkAV4bB9}gHE2d`uW;C00b$w)gMPHGnr^JwY_k~M7yUOG=Am2&{0QVRg_RT5c=6Z4GQPgL+w85t|tnLSXWU2;*BMA5s%?X(j0plP5 z=}Vwmc^bZfWO3zf${T9@B|Uh6uG3|5sdvu6f;rHMuW%`>f)zKzfLrhvgYHqw^cUxj zQ`+m4GE6j1ma5ARIMOvROaXAcjDkS>l7Y$|MNDEQp3;-*Esx7S96IQs4eM%9?}^j~ zWA$2OOpy7((0sprxy;K~-NmS|gxvszM@}sH2Ro;950s6##+nv>?iV%ZdWd?zUE1G& z!>EhD)SIHoi}%gP#kjmAfNXdR$b`{Znjy5tn9P_!a@{K&A<2h}8F%8wG_N&C3Y4hh zX|WnlgHzZ-gM(h4i~f`ny;jNkYBA;QVZEV3{U|bLawG@DBTz4U#FQXvt)zi6PWgT$ z1$+Q+l~YAv>PV8tS5iZh510Q+(ium`#AvMUylQ;xb7WDKuvrmvkeU`M@u9q>FrT|H zwIOi(^8<7rv*l@MY~L$tKD?TUj#=0n>LLi|Jwx;L13S9NL2yOJhH`~62(f;IzIMvvP1OMWBC6yrS)WpG)4wHp-D>vDMxyfyH+46u>CZG>N z7OPW$v64<$H@vD4ZbnW(7(~}o7yi{;YJB=92!05$bb(DDdK^8uJe*Y>Xb#69u2hFC z?&by(J)0Da3QQGT@ZP;CUPwNYm5VVhOd}vKz5+5?H1)Wc-&rMPq!VWG9%_-sB5aaQ zjb%w0a@n+6Os>o^@V!%3N6fJE&d;XtMX#&?xy9PHR;Y@_W;S}1*>WMAH0vPJtYvlP z!rxGjLpW7y2EO%x0BLuFvPhvM+x1Odv8zpo_RF}GE$$@Tb_Q8CxA!C0L)rDX8VlAL zNn3o*M=VP>746^4O(!Ax_ohVrAzb-O81Y}igZ~h&+(WJWOL$HqbiVI&9A#c(-!itM zz9DmTX0_$np?|}@*jYqC|E|O*;j)jzv;XXNH9|R*HH<(Ck<>*R(`vSsaNb;L z4oB4zs1g$MF0bD3(wt|he#obH%V5;LOzj!<1&=X zSoo_4wsX6-DM<0Dk(vSlWhl#Dj&;=I<3CVP2tPm{zSUUwKRR)-WGY?_R7!X!&!>lS zVb7q%!95PU4+GwT<=>Ugs=Y1!{bf3xg>N0{id-#I8MQeY&n#R1@h09`UGT#Hh#Gz? z6qT1M^aVJ271ycNK8EVC@R8T=$SIpko9N&wTjp^c&h=$a!0nYT*SvlxAxzA5okKQp zf3UxC;m_O}HW6yK*ReHU5pn%c^wYSOPRHwlY4K2ka-|&)&u~l6MCYtE5!qbe!~n-tBKi4e1%rxZiw*1Ia!NBR zJ_qp{zcG6bE>`(^3^d94sO9>y=J)JsCZspxa41yp+vATaCic)(10miEw-CvB?<`Jn;dXe|;dFOp1&&eE z@IvBjc<@YWk!B6~>*Uo1CvjirH!QwCVS7LB&ztQ8{du{oFE@=gczqKiIL~zA4RuQi zjMq5>vIOet@F*SC<$RF~y?Xx@TCk7f>Es>_@_zr6OaR&DnlRaj_Z|juitTJ$$JVI? zGuD3LrTK|3IJL2iMo5kU`Q(6)s_zvM7@oaoXW*)aF{RP5;s_h_T1M=eG9S*6GlnZ5;9YtH^)T zoSGgE3tHsnm+dTJG63KKIRL;+%6lBW9Ul03`}j*bc>5tF9(#I_mXBAr6w}ka@D~He zy?T>iHz74QQN6n*#wGY=u~4$o{UnbD0q;jF{E@xI{YS$Qw_5z<7K?A3P;KT<$PLOw zgT|L=i{<4`%|@rLX@}WbRL-~*88GsWq_w=T97HCd?*CmkU6RgWywgs=t>6xUWKL-2aP&xqwWWn8kcXs#9_syGm^Ub;EzH{!(ckhADu1}Q`h^)3icjgggPbo#cfZ$>NCsmSH)Gt(8OkDIqgEdd`#uiqK8rb@h>O7D=o4;@ z(Qal$Z&!d?L_!NnN|kBOVsCw_+laOQVO{F=-lN~^NJwyK1^0+;`Xkw-?u7Q)|J&IUY zlZ2EB002;4zKQ@c>!sQ)Y5;&0<3lNla&?=`bWRuIZQHfC%`_uz!;;hUP<=l+QYDVG zEQ1*Dc17NI*^5~io2vJ|=DU^X^@yT*PK&R@p1ytQjk}S)8_t>(_k04>9}@bdxaY0+ z^IkM&t7JYS7`ISPkVo4!-#TAgFl9Ywhz{hE7$U_Tm&<6xo%rQ49I3Qaa*Vg4JjFGm zuYW^MX$^sLEqO9km8+Ytr)slLNo90Y%4LokvJrA6Wxusp%v_rbZfuuIh7^PK#PPiN zouxsKTjNo*zn2b-45Jyyf?B6OPbr>g$$jYvSo&|RaQ{g^g2c@{aF_zwx&K#h6aQj44h617dQ0U-C|!9JdSv3tK7hxkn@ctLE&SKkAc%W>Ju#U%0$sfe!{Kk zc8c34STt_>HqgmkkumIE{VN!E-xdqE(=plF&b7V((yE39-No)}v=A-5Z4PdR z!zKt&Z(A<@cyXq=IVr{*4N{|9WIr*YpQ}@JS5(gE4v}HYT3iAq(Xt&eS1$x<$tc9OW0Rb%4hG@=RLyXM_^ z@Bd=FKJCsdrKG#(IMCfGVA6U$y? z2J1Pxx)g6}&(3U$#8+u#8$A+$=r%Z4>sB_@2JRy$PpB&)j?8Fvc zDJHpM5|3P#-$*eRaTs zQWVJ))44t|h#J6xf|ZTH4z}lQ=NMi@8t3&zGbf zsTlbRawBnJ-&wx=Mz+K3oitPZ4$;K3+SAY9)_zure1^T>RKrBju%SdCZ<)0}V6~un z`922}mYnRo1#sR#S(oC#wa%5X;FN%(-a0ShlqVxb)~jZ8@X{=WaryDC+GqoxeZ0{> zeo?>FB=wf+;@~4(DKHWQkR~RK9jwn1s}2ODkWrbW1<@h86J_whF#@en@b%2%iV^4H z<=4gjD6!;A$W=Ymrk6$9v_`#+oTsL7la5!(o6|veND3li2!qHW?*g-PVUYgtk%h-q z#xP!W0D>8ew5L^w5ip&~V|;0=o_~FFKwJrfe$)5*Pws$$j!Cbfo>h@7uJ&ENL;g{k zY`-_t{FtI)jj4k-5I$1VM9L^l$vX&0hLKpraryy<1yRHpFhKkouu_Hs^fg|@eu#=l zKg3OqRXYVNt(`JlrzuM>;^dPS2z0X%WyE#@y{!VqXNQ*Z6>Mx(lJ7DPQ@MDvtF zVN2>|w1Zq2$P5%t0tYHaN}-FmTWL38Q1@_PC6kAH_LWN3`VXj@IN8Y!fU&Mq8AXDP zasmkvWq3r5k(EFU+@YZ7L^6#(!F*iDe=lXqqi?h-T6Of<%7m8Tb#bQglUAt&zGb*#|)yA({-TKvAnzash9M^{WKLCAsqAHK* z-}9Zz!-5PF-yDn6=&DP7%B$#HZUq7o9?5{n6hKHDj5hiMpg{qAordhkPm++p@L2`) zO8;A?`}*Jeu~SuN_uO9A39dC8+F)FY2JbAY<V43|bHZuDqX3Weoz73mGS} z8&=f4DC?hpxOzwW)NI`zr&EVYwTJr61%XOk24G8Qo>_%UWvVZgNrlWPz)6rmnr!1` zf+ZE8m%F-ub%7*xfp9xxVeE<`YkAjC1#vsW*GDIMGFg?L#_F+H>t6fjXooSa>F&>` z>(-YHZ#w18UNJoS%W(EzhClz4;nAQ4>gc~Hx`TirI9`|2uO-Hxr`NV`@25=piXK$W z@})`3m+o&#m)?{blv`A5y}HDmhf-*3s#dyTKq`y`$;{?wM687C+{ta(A-Src3F<4? z`~Q&i13mjZza}Ah@zh)ya+nb4Jicj_vNdK3b5E&sNrp{dx*m0zux9aJE0$a)43FFG zoCQ8Z99m`oFpmsDG=_hLj<-?gZ`pj<>aSr<2uOI{Zk_eL5Khh(v!Sko5GpDiA$Fg} zW9IIiW)hsyuFhz%*i5VnC}$Tt7>bPP1ge1YiMYEw{5!dGFa3xXmi&zr|E;8%=9&XH zMFFW)KTCadSOrz4!kGMnfwFq?!@cdef zrASR$!Mv{F>)!Qw8pEoGzw_ zZ60n4Q3q;oqu9x2{pepRV&9SliNm*0o!VWi*i6+6vj6>0i%Vh*Cn8f!ujBE+@!rTK zMuk7V1JS1vCK0gHSa4F2yDr4Pfhfp`dJ|yuU z0M*5U?T~lB=4aU00Ds23)Me7~73;6C7hp$erU#B37!*yl4~+JWJIm1t!i68kj(`ZK zAs=<$`Vjoe(M1hSa?d^BQzDA!_GfMYBJFw>=VbFYRROaug zfy30si2Uw0hNbHd@JB(-`Ac}VI)4`@jQ?u&QKzJ@`oeSPuBohgiYd2f5^p_y-!hSw zilnuW=+qFgNuCiz=(oil=25qN)GHh#;-v1EW>&q}(yhU;U%#~CqPFfu)!*G>m?gj^ z=$^wrB_(0(0`3i|?yWmRO+{7i?t78Dwk_j5o2Kuc`pxg({CwmC^^;+g_~_N*Dwv4>Nro$*p{jLI1h}*TH5?iGrHfTR?~a z09|4LfZ?+DK{#6JIy=5`5wmi1ektl<|2iYCSG!f5K6v~7qh;?Gd?J)3-~_-S3<3^U zni2M%Hfd9=ps|^|#b+~YxDmGGW!;?dtk&%36Ufh^RexFxcY7H8UxdLQ)km2eGL3m-$C{7Th>dV1mZiG5D~Rowc3-VAGj5gI%F zqUZSTG_fdUZ7Ve~qpTq8Lp+UXK4O*Z+J=hl%7r@bQq} IfLxvb2hR}!JOBUy diff --git a/summary_results.json b/summary_results.json index 2a0d6600a..4fbe413d3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.000", + "Accuracy": "acc: 75.400", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1716, + "Performance_Result": 10.1445, "Performance_Units": "Samples/s" }, { @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.390445, + "Performance_Result": 0.342409, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.996, + "Performance_Result": 23.1762, "Performance_Units": "Samples/s" }, { From b0964812856ed98a446f5df985dae955e3fb8d8d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 9 Apr 2025 23:11:44 +0000 Subject: [PATCH 433/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10228 -> 10220 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 8200dd317..77189a846 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 23:08:26,014 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 23:08:26,014 submission_checker1.py:3260 INFO] --- -[2025-04-09 23:08:26,014 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-09 23:11:39,201 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-09 23:11:39,201 submission_checker1.py:3260 INFO] --- +[2025-04-09 23:11:39,201 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 23:08:26,014 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-09 23:11:39,201 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 23:08:26,014 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-09 23:11:39,201 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 23:08:26,014 submission_checker1.py:3276 INFO] --- -[2025-04-09 23:08:26,014 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 23:08:26,015 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 23:08:26,015 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 23:08:26,015 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 23:08:26,015 submission_checker1.py:3297 INFO] --- -[2025-04-09 23:08:26,015 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 23:08:27,002 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-09 23:11:39,201 submission_checker1.py:3276 INFO] --- +[2025-04-09 23:11:39,201 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-09 23:11:39,201 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-09 23:11:39,201 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-09 23:11:39,201 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-09 23:11:39,201 submission_checker1.py:3297 INFO] --- +[2025-04-09 23:11:39,201 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-09 23:11:40,148 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,16 +24,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.297575 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71659 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421036 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73032 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420469 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.10717 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.80393 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.29349 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.342409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1445 | acc: 75.400 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.48276 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bb5b99233..52b9c41da 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1060 +1061 diff --git a/summary.csv b/summary.csv index 049544bd0..5a54230a7 100644 --- a/summary.csv +++ b/summary.csv @@ -2,16 +2,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.297575,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71659,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421036,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73032,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420469,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.10717,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.80393,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.29349,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.342409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1445,acc: 75.400,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.48276,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 83c3c3b59..4b2f839bc 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71659, + "Result":1.73032, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421036, + "Result":0.420469, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.80393, + "Result":8.29349, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -369,8 +369,8 @@ "Model":"rgat", "MlperfModel":"rgat", "Scenario":"Offline", - "Result":10.1445, - "Accuracy":"acc: 75.400", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, diff --git a/summary.xlsx b/summary.xlsx index 2edeceb7f8c2d1fec6d58b2fcf6a8401755a99ac..1eab4672967aa07f956cc1e6651c717aafec9c24 100644 GIT binary patch delta 4944 zcmZ9Q1yEGq+sBsJKy;{b>}?Ksn(uWB?gChOCs@$3oZcQLkagMWPl&x$YM)-RsVzQ zj165tnv~d3gcO(A4?IB$P7n_jzi1^M?6f&vuS3@@7$5$MPN0P}3ALqAP{@9Q{IIm| z>HqHen?YEW+s7j_ zno}2~GSfSLrXH>3;=?Qtw-ZPW?Kuf?i%El?Pq2~~isNE?e*Xs=zC$kFe{M~1I~rab zjH9<8OXsPY@Gj?FyPR9JVW?B$@5bU1#0%D9#Lol+4s*Z5S6H6EqX9DPXc_Y5c@ik+ z^FC*ZaMJnH()Ua7G6Fhi{Uv|HbqYU9zwE8-B=TWIjUmHN<#!4zBpmVOWig@0rGpWy z=4w|eTIQ-(ar))?6zFb@Lx@gQQk8y-=;FjS_#@sLQt8G+`$gif!v0_84VxtAa)-;& zE6nul^0uu(gL`lI&mdTolU|1SylzaDFN|H;**~j~I}g=HiEKU)JFCrl?GH*j1JQi= zp4J?KOJTL68L-fs6orZ98JPP#a>=Q!hJ#BD000Q@whX{2)pE+-N#OEGE8y1^YF>piTbWCl(A#1DnLfZ12K~CP7;$r&8HuI3k$C;8G7$)j){ODlrE)*R4-SaOGP-gG2d=3o6z4<#D8EB z9L24^tQ1v+T?OP&Tqt$aJecZ&dJCw<-|w`a)fj!0W5u4KEL-1p|C1);tZ;g7jYP(T z0mFHYkXUDj<@~+H5Oj}l3aA9ABe2b}eY7^>#XJ>9dcAgPXb?|<7u+@bc~<61L!!2K zq@(I^7jpD?@%pVzo*zk+Qr$nC&oPDmWn*KbL&j}3n3-wD z-LM}w`0Qou62A4ynBZQh6T+*kA%QiyYDHDl)I~&i^!P1)-c^=xOvgM^mlzM9$$dB& zX*K{#Y-on@vK@PZrW96PfD#*`-i;dIEQQ8G^0gK(eaKbe>W)s2N1}CsZuHAHvMsb9 zQJr)jrHdQ<^dX_dS9!D>(?#R?ADQPa)1w?OMG_+pZ$rS{M7~gYAog^{P?~bk*gCHOKucnzaJBz}&2ocexN_B`g zes3~M*K$}ilLa8J(1@8io7Qd2rZ4)+njeWdK=tscK~hguo}{=;!`W4sZZ-Y{vN(@pd8nn*QJ zoGo#jA~H7)+)Z=6_&6&rB-&!}(Y;O+`h)1Bv3*wvVwT=H^6&?oHxVd^8v4eilRu1X zq+t_-K3s73ow@yD?sHS6TKiNz2np5dEPUG3IN?#?d~imqY68+OMI}*knKznn8JjPR z| zNkO$^m&B!K|Fs257WpDc_d~a%i=FOUqZmk0(% zaUClvAKqe9t2DH1a}&B;VJGR$40AB`=N0}5nb3wYwFsgGa$J(udR)^DJs`o$tvUUd*aVfQE2+;ChHXKsi5AQYaP|3bXf1 z+`LulO0XdU!WD^((}ExHWanv!>*OQxSxe=w3kSkzliv8lTRZIHlT219K&i zu^na)P#bSoXdKbzQ^9VpVv$vY4#bA)**m~BlejZ~${L1;4KTnXEC=U#mc^_aMJ#Mh z-&B`p0BmVQ5UJ0maP)TVzIlA}a64e5S@ID#Xh?e{L8(68P>xF_LdY=!NKwGCMB#Ug zD_X*E0k*OqCciJ^q*UD*!)iHLNlFx4P|R05uO>Ffv%GZgkEHi-6O?)Sd;X~NtmdLB z;tD3aoM;nOC;_Q>e;h<+Gl&q~R!B95ad(Le z-yc?>&`iZZsZ)1U9A|jm@QJ(@t?wVloI3xrX%)Zfd)-G|Kj@_87Rfl3Y>L8f1Trqe z>;u)R&}62@il$H)Zu{&1CQbfJYWjz?ivgb7%P@8=vs@L$sD?>F_}zC*^?sTv#!MZW zk3E-cgX;YU^uGbY`DO|nDgYZ=puH2RbON`@Y%bL_R!|{nG`2u0FJXI4rbvpl%z9sH zDXK+ReO5yK%~I_`%gxY3Xj+^cK?J+MxWbxB4(SLB98@ZQo@FRA*;Ni#bSqr`pjgtb^6MF{Jj#d(ULx$*Rj_{l3Oz{_1poiyi}aayk1sL7mU zPMWlZ56R6Ax0l@BVr2}e3MmM80I05(U~xsil`5qQtm>c6$vyl|Z9Mxn2>`l$52k)% z9vMtPS3zUq^~(p!m09$8Q*K|}mv5uVglkWCds90ypF-KkH=-B*of##|qc-iEf?o(^ ze5q@+eq==yILH3oAatWQS=e&I2RLs7yA4}Q=!;(m;*4msUP~iixT`KnzBZC+V78tJDk)A zcZUVUEGQ{Ttvq@}Gy58Ta^E06QXAPamDHz88&63M4W$a!4zf~7vB$YzlgDzVODhE1 zAY88nND@?ppW7ymg@XV>(MV}Vr1bD8M>@Gs$F_W@CF_uGmz=@Nz0)h-e!qN!%zJ33 zGZngOs77D?yL085ud-B5>~W`m^w4*D_K2f9y(|^sK-JClNC0-0Bi;2QtLZ^NuJgJ(?sE zFtRxvu{;g>Tj_`pN~E;#{X(E8i($XP-l_NfL7Rl*!i>{rb+B30GK+dA>20U#LkE5W zFed@{q?l>@gq6{v$u9ywh;QY@iV*;dAqeDiyQ5d!TT^hAPW+i4PJU?iz9xCy zGa2)1!Sd`Y_!V<(mp#(Y0(zE#u5lCGX4dHXHMEgMTYa1vy}S{@{#1 zwj0R$M(|P**I{WnxOD_z;>s+iS(0nN_TIbG=m8R|L~l`PEXg*>KBIB`LzdGQ7)gbF z@zn#DnYtIeJYvm}Yr+HBBulh1EOue;C^5_+wfKFwPBGZn69=~$hePLvMtqF?q*6tX8Q3nvK#?$p2KM_zD^l=2 zYiN|4x8O(GZ_z2gb3am#?|v8xZ+J!SVH=fI%FC#Vt=t~!Gw@CO(*9MP^P7AJYdlTr zhKXMG!J>W>2$hl^sCQzcvL9L{2khUml9hk*M!SS1Of`wc-pa^vBPT-tveJ-hfku-( zqNsme(QqA7B}8g_^m%~vMYn9Z54UR5q%MzhEWc=LNq+6?*>dZgxfYF;&;qoagV)Tg zXbWy$7Ylbnu>n}C?Mo}lm+R#Fp_UfA=kmPTfR~8e+V+zCx_5b~sR{G}Ee(qYmXmyN z-BY*rH$#&%n1vqtOcuJ`3>5)6Er6ITqJW^BZZA*`3%6;*nk;)R?Qm`P%33_*@prqI zV?&VYIsN9kN@uF2+l|=~|IFz&Nh*u?l5gMp^pcKfgc$JsfJO|vMC-Tb&$Q3D=^bP< zg_HMEtw&a-z>2UD!AVdNnj>HVQy56UB`44T^t%9uwjvw+=&|Z*gR{AY6MHSv>~@ZL zUT0a?c;8hk=!V!-{Diho7wOr6s<2YD=+d>ATOmdpX4CPWT8EmK4oB1YM9T5&Y=MZ; z3fakEA68@8W~_tRR7Bnitx?G9tMQ8{$t8W%XCMPuL+fL)8T7u{z8|RmM`drfD7DJ1 zRhvs8F`lwg*N6Q!(!&^iy`4Ds2^Y}KEq4?^)nGKL!#dW(A|RH!Z1wl&{ed@I5k+g? zK3;HiUc6qEV1V@D{4S!DAd2TXK6*LkI}mU)ZZCN3ah_X25r&iQCjdKnKVxUG7(OFS z-uJEIbZRTkVh5^hCF!glZ?SiGmiKG%U66`0KUHX8mx+Hkz@&5JiCUU{GcsBU4c-Mprx zbqZlQGxV-YxjBq~=2guQX(%Y+aq;e#ulbz)r0$#v4$|)}dZkcQeM@OTvjq<`7ngbV zrnNL{<`#Mj{wIA?*pR&V^_2~qrW7(4Tg?5fsDZHyoNq#9?I->(_oOmoGB64;#nKST zCDx2>!&b>eWb~^mrmht@qu@jQIO>an#?Ld=GEW&}jytN+4Az9uD*neKAS?ht8~d&& zzw6L#TwZCrxwyXLf92x#`iYm*+jQ8FW|siD;)S2$y8o&qK!;Za^xRB`D3XO-Vko1I z_E^4-b$@9sUu9O+=<~xR3A=+B(l?8*)Kw?fQl4c`Hk9h#OS9bmspBl^?Dpi zX!N%Lq>p#gU<{qmn$VXeB!>L$ideZ1t@7n04oNzuKQn=MV9VOF^Y*+&WO)nL@-t5t zu)mI-GOHaW>1iEP(5FR4-~;WbMt@G%5x9nB;}H!?AVCb$zEXzUwhw$ZI=Mq-=u>Lr zR@WCL1R0vk)tXTCVLusunYOi@mGsInY4rdmKz1QUS>|)#rtW0LXQpcD7U5JbxJP~R z!*+{dikklIIpl%J)wn_B~*f8ACt2AS!R$M{fWye;j@j_ zU$|$Fs%_aru<`zmuHc6t*@yp`bLayA^ml&!cLfI75W<598R51ZJXjI%L=LtG|M~yZ z004gmCI7C#AWHaG4q2>vI60>{7B^gtljFhv57+;EOAL?Ul*TfL4{@^NlY{=e|1a)j BO*a4l delta 4935 zcmY+I2QXY;`^Q%=Q5VtMT6T5ONwg5rLe%I%^ct*)x@uT8c7-SjmWbYaZ^4SPh`N?U z^hAUZQR4UCT^2b7t;6^F5#Ed+#}Op69F4u~FRw3H9UL2ilKG0f0~-Dx8uX z^_@h6m@BMNhSZlBUaQnsKkWSLinvt-6&)-gU|KAp{1=aRRrhjzy< zowT(`J%%S@i&zABTQ8%qSmRfYqr1}@8@ZH#jwLb0M+?D~N86{B_`cW-i9iqR{((a~ zrbc(^wN9~Xz*qaQh=cF3Mz7K@S42}7xTmlL4}#Iq2z-IvA%C`#*c0# zrxe8b@QVY|oy7os{65i{)xpmhB}Y+deyics=VLK)?ZAz(9dRZn}NejcB z%G~7*4*9e0y>t~KGrTNIYu8S@`t5hP!*8sR*gtX}9ap%YW4HfCRYHA$gZ{n4of?uB zy=%VC;4H+lPrF6^R(NsG^`@#?=f<9r+t~2H?B{$S13U)rV^F@hQikl}=1heX-`6K0 zWd;BMl!RLi@Z^oTjzQgUnjoGH+*BEJPD11xfOYG0xwSnuAa)@sXT~BB$Tp1`oYZ$}zJQ=n{I^J#Ne7{BEjxiq@0h==L1euwjJ0-~EThtIUT8c)dujVILOvb0&ha-J)=``PRJO?RWZIuIf8W&nB7%1f(yv@vzZax5pnxw4 zgE7RfrMNF?-a2qyOIcn}RuS*6I?rr-VxcL(J>_~omtJp>Z5)3izZmU;a0DW{McctZw+8`tz~5>%a&@{=o!_nqNw!i^ zWkOk5obp-IMI?C5ZB;7TBuu`VEw8_5;j>THGya6oBRo~DXuBPMX_6f9)y#8RFJ1i7 z@SWUcuFlYDwrUF|Mr%>oqBMs%9iMULbUami8GCik$@r zt$e%;h|%rzTE^mKOye1unfZ$Y=psoE36hj0m2 zP5%xKjx^Ntx>q-=A?K{_`KhekZb`wGhHM-A6uCxb%|c8bI1!ci^e{*{t?`Xfc%awZ zgf@_@KOkdnblxP@=nA8(O1%!FVp5p*%IUthmA4w+vydnGb71MKg!MQQcB8q~0)ITP zrpY+=8H4i_~PeF-|51X5E4kJL?K~vQdbC+2#C-^>VT12pbT&a85sFk*Gc0k=U}u} zYm^oZ1<+0xgkWUabIXL9x=*ga=@I6cCNV=^!EPqrElsmyIm{|)=maQrqJ8lYBg3mg z()(Z_0+gY{Y`l(ur>%yOL6MI`W=$DZf09DqHuOWdN8gmC>^h`KZ(3)%Z>a3$JP`Mt z^_5IH6~enbySZ?+^kqh5Of>pqdux2*1Hbm=`iTp*q{cD>y3Y-ESSVk}J=wIvjX3wH zyQLZG(y3si`>l-a-`ayOovFZ?cc_;0VmZZ%-a%eLO($a+;Pd>BN9CX0vaD6PlyM>- zL;B97L|q+z#yY&Ve{RgRvQp8JXPBav;wH$F_3=l5N*zm~Azm)e#&lRB>$3f#eqj4+ zmSDYB>LYzI%#YAMq&q+#O(bSaBD1NjSr4WHypJLE5e&X=_@4CDf8#j74m#; zfF_bgkdJE7BFmFU!BkP>F{Jww{S?wJ$wtcj&SQ0SRI%F?V$CZeN`z3>uDPEumoqSq=CfJun9!lU!S-&-DylARveAk45y(W7i8{ zXajLfrXx28xjT??GG_wLoV%x6xkLb!bCYkyewa9pl-UkRU&oz~PzS#ke-MM<7(qFp z$@{?^0F*nxpka{Sp(Ux0V8eewrT&6C`~e*lMwJW;H^NWgeJ%vdTxpvlUGMkf{zYY6 zoJqGIMsMm$8-LD48hqf((IRmM(mqFUfM-QOC@?_!8n8)$f_F4iTy>XJUW2b<5DhSJ zr6HE8cB3s_1fKZ|RYGf-QS)@Dss_J4y)aDR>g+VhkW z2_Xu!Ln+NNUL^z(8(e1(%{<2C-%g&=7H)-0@>gi0DUn)n!C5R}!B3uFivE-m6|Q4` zfy;V*++w2CGFdb;Q3T}XZ;?G=NENkH(TX+8`hFrsdtbLpPTeJ=nILI)Jtr*lc=fL& zm03(eg4UZD*e5mPs`-banPHWKa=6%F4qs@6(;_wS{I4@d5 zMBg&}C3zW`;A;?NM!{vHIC~m(v2=CLbWLj#W+l&W4W<=UFpWinm|T@}&^y=*U5F9)U1$C@b<0 z;o4uq$bSis{zJHSH+Jp6gqH+Dw|lOq(N>L4ol|R?TQVmXwmWd2UL$$04(n~aS4#j^ z>qnwEx0=>xy%G9pzFa{rH3JfNAZ?-5#xF-IzyxR{oqXi4h2L4!%@Xbq<+#wObvO-9 zJ#xOP9gfl46D8ZMS$gZDY*Wj)c2nBqBLD3X2Ao&Iq1jX?ADz3W!QZbxDiKwMCywqtMS^ z+m|Q!XD0cPUH*5K309&E308vi6$w@*fUXNeJsxD<=yK%vY7A zNpv>kVJQC;1XoF|R^T;^b!W>bMd^AU4;;g)K?>EC3mUI0 ze}9?FW8&TLfPbn|o3uZ{zN%XN@iN6tQ{XD_gk1M#SZqm-;1}9SR#cx(&s6N7^w}a0RUl(m_;i@#LGR8|t!dTMXvaA_US#J%=c5eiVq|8;BO;n zQQ-X5e`E;eb`=Z}wCRx(%$=8qlEXs{r~>(Td_H(;h*sE9~L8QmSQj%=)dH z`;Pdp{G&0qQW{S@4!-@g{SlhzpBpcWkx|@NA^sAVlC3Dz{o-E|W;jH}{vzYnye{A~Q6CAux zo{?OqrwD)gBH=j=jVc$LTpbB3k#keZ5BbRKclpizN_8Tid}MXq^$=URq7$dMVcFFq zq)={a&;uWv_EKbH9#n6gX8F6%6^XQ~4#XPUq*FL3#T^U<`fJSII43^0z6%G{?Nm`e zx@ztY$b^KPaq|c^c)l3U*Ibg%_i@k1n&%JMt{#snmiQc7uf)T)ta2#&EG5n)uQJ(U z0YIp5Hmj+}At?Qqk+q&i_hfr`;!Sc#w^GWnijrvXV;bAC?8}t6ba~e=rLMU`79X{u zPjOFmlvqMUeFw@ahuS*0HsKz*$)r|lVjg{WXgvkB?k53B#i1hAPnk2Qd9pasHwQc@ z8Ci0=XpY1P2~t0es{B$>lH#a&wU+J!>)Y?D4>#unKQv_-;LN>Y*{xJv-Ydm^ zd@5I+bwf~OVDOE8$Y_6u7cd3Oq;7bBZ7yQug!7{bpl3C>E*ZN&af&%{Qpy|8PHwDR z+ALZ1sjxe(_hptz9*W!E!MM3Yn+x*}4px%xvu?s+S7 zTXo-?CeOzYqQj_dm5=B~O!7Hwrx)tbwT|Vra&5OIEq2~fcUu;*iXJ<~+oQ4i%%OA?x8EErMwH6gCIqn-l zm+eQZTEBY6!yp3Ro_d`GYo*SsEMbT@UTuEGvk6lRN zDAs?Sy}07QAL_$@LsLdlcMp^Q^ektH&DH+l;#|t$jqMD9C5AIUh-X$%iv9&HQ;>l^ zx5`O#(HFTe)_X4y0>kWaGY1s#dn2{ehP!1Ly7+h5;uT*Erd%PTM?tu3xN7jjVHGqJ4wnb!#`|*x`>hCc z?B_V~biK5dl!Po0`qGVI-iJQBdat|Y!hUwMryPEQ|Ns1(zj2t|gpzFg z%Wi=XiW|fOpYR=!Vu(A4NE$sIGMoW<3Tk+Y8s4cet3a=cg%MRArhB*Y`#oghiyA5) zIT??<*%=^*E0;eb+kP`GHzpGU#I8`5E6AN&PR{-~bYoISypeRM**7Nlb47 zrx}HRB{r$ma06lCgta;xNP&a(3 zWG`;NmSApLe)@JjDp&9E7~N z)X|xXkkZ|I8c7M6D{nXatuiwWJZ_vVr+4kCk<9sz_97sC{HG@n_bWi7y+9l>+23Ln zYFki)>wjeoGXQ{%P}umlMMb+%qM|4{Q1(J1Li+;S C;5cXi diff --git a/summary_results.json b/summary_results.json index 4fbe413d3..bcfc4e7ac 100644 --- a/summary_results.json +++ b/summary_results.json @@ -82,7 +82,7 @@ "UsedModel": "rgat", "Model": "rgat", "Scenario": "Offline", - "Accuracy": "acc: 75.400", + "Accuracy": "acc: 76.000", "Nodes": 1, "Processor": "AMD EPYC 7763 64-Core Processor", "host_processors_per_node": 1, @@ -104,7 +104,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "datacenter/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat", "ColorKey": "availableMLCommons", - "Performance_Result": 10.1445, + "Performance_Result": 10.1918, "Performance_Units": "Samples/s" }, { @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421036, + "Performance_Result": 0.420469, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.80393, + "Performance_Result": 8.29349, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71659, + "Performance_Result": 1.73032, "Performance_Units": "Samples/s" }, { From 71e75c9836390a5b6188a16925c36279a7aa91ab Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 10 Apr 2025 10:58:43 +0000 Subject: [PATCH 434/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10220 -> 10219 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 77189a846..9a43535c8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-09 23:11:39,201 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-09 23:11:39,201 submission_checker1.py:3260 INFO] --- -[2025-04-09 23:11:39,201 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-10 10:58:38,854 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-10 10:58:38,854 submission_checker1.py:3260 INFO] --- +[2025-04-10 10:58:38,854 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-09 23:11:39,201 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-10 10:58:38,854 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-09 23:11:39,201 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-10 10:58:38,854 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-09 23:11:39,201 submission_checker1.py:3276 INFO] --- -[2025-04-09 23:11:39,201 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-09 23:11:39,201 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-09 23:11:39,201 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-09 23:11:39,201 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-09 23:11:39,201 submission_checker1.py:3297 INFO] --- -[2025-04-09 23:11:39,201 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-09 23:11:40,148 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-10 10:58:38,854 submission_checker1.py:3276 INFO] --- +[2025-04-10 10:58:38,854 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-10 10:58:38,854 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-10 10:58:38,854 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-10 10:58:38,855 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-10 10:58:38,855 submission_checker1.py:3297 INFO] --- +[2025-04-10 10:58:38,855 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-10 10:58:39,856 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73032 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420469 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418802 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.10717 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,8 +36,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.48276 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.40724 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410968 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 52b9c41da..36804484b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1061 +1062 diff --git a/summary.csv b/summary.csv index 5a54230a7..872b7e91e 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73032,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420469,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418802,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.10717,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,8 +14,8 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.48276,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.40724,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410968,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4b2f839bc..4e9051346 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420469, + "Result":0.418802, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.48276, + "Result":9.63051, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.40724, + "Result":0.410968, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 1eab4672967aa07f956cc1e6651c717aafec9c24..3b8883f30a994f564c347e89d9f6b0f6699ebd02 100644 GIT binary patch delta 2407 zcmYM0dpHyNAICSBa87J?4)xn{S?uOEqb0-)Bg=A0MTJb%D3@e8WDRqhOU9U7)6S5J zjU}m6F1cm7)seY$Po$zU&E@#5e$VeY-+#Z)^M2m%&-eBIJl}cG51zspRkhbeO}FQN z0RSMPbE*M~jQGkr8ycqevCCXnvlNPGxHaPGrSL?P#lPSNrx6VrhL$6NG`RoTmw)`g z(;d-AQ!hnkWQ(43N4qCpy5!G-vor(FyRt0P-VM})`(D1UnRxW+gmh%0_`uaR$}ex$ z&Ouz;hQ;*_yJ%+wEnX7iEO9#XZvC9wbzUI^9(ZmS zCwO$Ep-A-ml1&z#hlVBOLVO9aDG)-w8$63>;7w$i+a`S8Qsvh1cCV-7pO3awmG`Gx z%@P^>6eONep8B^i#o>6N%DkZ|{Uzn=e#;?|^8(8p7%7q_by4UHfx z&^{ha@ZZ}gUoO8s!99$}+fEq2h<_l%8Xac@Ntr>FfoyM`$;qp?iyvn@Fk`UoRyh$pW3kL=rVpmna z>FzMPaf5{FFZBEecHO!2O%92O4UN=V&}p13HQ$bDKIH2O5M@Chk;^|M*DZ-=ZZl?V zZQ}B?qU?1|CdG}qYRGVZ-xL;9i9-dDc|fc;Gic5LOHu%cV2&rxEC9*l@-$# z|F@3n%^%pLrs!r zw)AF(NL)e+lnBQiQ~rMULb0T&darG=E;w^ehz4+4Qvl@ELrf#7^ZD{VIU5i8ZJm5+ zz=ZhD{ei|=bZnCKNUbxj5;q4{w>;EwwG;OMZqex$chNZJgV2K?P>XhSPo}6Gmb(kJ zLKFruB$p`!geD0!U2=8g7H41U_fn7a>>XT%zBgxYmz#@N`((6rCG@4bj2ZukvOo7v zMU;wlKzS02#fu;^Pq`}}qsLq1=VnmZeGpf@8X%HY;tEyWpI@9oQy^=%#uwry%AMTH za`$}-aN3_i)P%E|RU%DIq#x(OGT&1{2O0ZZ93>-^;d4rpy5A9 zhMjS$p=zF>Q9xmrucDXsc!D1a5Srh(oAg{Pr)(Sz+_UhRjrP9_Q2}8BO>Qo%EqUR6 zjt!XE_AkiCW%zDdEkD6u_}xt}Xs#;SXfa1H#X-j|&aKoP@5PhT0S zPYvUEWL5xj@bJwamEb!hYR1wP7}w#s=K^^hUmQNM@cK#7?A7uO-|D@Ot6h6C4I}Wx z*Q7GgFrb)h*ct-uLyWp%X!+s{1lwO5K@Ahi2y&=WCb;0gFH>Uf!43R6=dvR3d8cJQ zQZo~yrUBc7QM>3j8qP7e#oLdC7daynTyegc+yNol zHguddy-Lo(wvd!jIr~oB>h!eh4bNu3F0*Cidi<7zi8&7Dc-e)hnfW|w1rsTe%Pq;} z66A8jrZ1N$P=dTtVgrF(j;b9`w^Gp`QWAh>8A@@OZStlj?G08tVI_#Wc=FFe9eT4wk(bFpkhe(y`ceK= z^1_OodYqhkoQZU0%{~P0BN|MbwptD0)YQ_ZD{6|ENZZ{yVasxV49QP}l;!^$yz|rG zI34%wXI`KWkVy0UoqeeL<6kCcSM6NiZOj>8;osZm^~siQ8oF|X_|YkBhQMmj+%2D6 zcd>^)&k^lzO~1IQ3jR933~{@6w$m1f&MwD8%ke)~;b_7Bl&2Z#KyDXE$z+_7Rgcf} zF4=4ed&;ToIk@jgo$~z6K~!6m)V@N6&ryWOk6pMS1LvB{ADl_x3T0Sz0I6G%pgo=_ zr!gd7yqhWhUz;QlH=%3@3n;kTu9B4Kfw`0jTSoz9{`Po@CSk57wr=Omz<6tQ9wiz= zMm7%rIy%~%HF<;4Fs&HhAQkuu$cgOQ^)%5x*5`Vhdr+%hq^)7d57-=;Zz?!vc4GtI|@Q&iPDP3F4JJpihBKQrK)K%BB zG;n^`ofwL%oq{9%xy+p> zOmD&3QJQ@Fw>O<8z@`tMeKkYB6TKhup4?jNmsx$1Aq-tB+mVDywF^>CR@2`4DY;IU z^*;0WM5080OastvaifZ$lK0iL=R_2sc1@h_*cxs8AE|caVb53Zr{is)cN!ku4iT?&n{!n`rroZ$@RF8`W9agY4N$*u< z;ovJ-D6*NbluE8KYV|zR)UQ_Yn7a7=Y2MvA2gRqSd+)PQ54J5Yp{{HIM1h9=3jh9t zI2K4>U^<6|v{L|UaWGoioGGL+u$iNZvIU|z&M4#`KL-p10K60c08PNZCn;H#lZwIv P12~;1!%ZN>kDLDjfLvNN delta 2403 zcmYk8dpHyNAICSjCbzL-Ww~rK?3A1dqZPJnrn#Li?$lzAhWzNbOq*1!A={iGgwZHY z$t9OKKTL5Za?35`h$OdYDwKZsJ-_GtzW;un=l#6i@9+EbJfAtId8hJlc?DXEr~Tq? z06?uWR^EFTBb(3r6g)oNUq{HT9q=h^1)p74aA0EeIQ?p@qEtdFczyh*{M74g-|>Tr zEuNZ-?2mm5VVC(Mnjx&EPdsA|?1-S_CDKLMol^(avpg^s5j*_-g-^^zg+sNuV|)(7 zsinlczV20IameE8UI$OObD1C=g!HNNM0)s)^#~cAM2Q6p%0(W^A4Fc32{1qcK`b1X z0OR8HIItKQ7&nY~-X=;>6pwqv#O;v-9V6V}K#u*j52e-3-j6E{2(Y;}@NQyPX2q5pT|aTTvO+@f z161`W$?~<)#2dwJ&FJoW=+HZ1Z^#IF+=0Kug&L32tGtPb;d}`!tHj2XCEE^z=%}03 za%p=xL=YfLM!|!TouEPy^N-HrvOKtkA`E&@H%^V5)ydGv`aldGHu@E1cJXLUdw-`S zL$R5%B{0ZYSNExp{t8TnSnu|2DV|DI715xT3^;iZbn}W7 z7qTD?j*V08$Xhaq5?;+CRnxv+{veJx-Swli7ct{nc%hYZ^q^q5s55cv>xCRI)08Q0 z5GFlFapAvxIYUPP{{Y4faI zi=S?XtH}1u*?GxYY&%_eCQq@Ewo2Srd8)1EU!U!(x@^>G`l*s_xyCIyD*xk_Ts{%a z4B!lI$)VW|*sb1EuMH+LBCc6ws16kPHzAHlHy61-9-Oi7q4y44gyZ>l&^4XddUtc0 zaob#Bwk1Un+{*FUwmq0gO?v_IIVc2f?-Xifyv zXDh}{sA=B);ThY|?6?YIUms?P+$XkC&(%+@+E7hl7%c1nmT&wIaBK%~9t)Y@@yomq z5MWTd+fi>n_kf?qLG?c|BEi47`WhmC%f<3%LmE#|GKf}*M0AQp?|VD+pXhZ}B~$m7 z_abx?OzZZjaS#6F!2tymcZ#IPPZGbg==|%T94vZI3G9l_IY#@Lc}eshF(qxeZaL=&(~$Mc2z zgvG&NMqQA1qwI>;g72g8%&$658;XJ7FMqz3xJX&O+D5V74_VGDHMhqMXX|&O+(X0u z%#=R45(hmnaP4XIy3)mG1Ie3Xeyd%C3(yo1HHYrIdbhDE{KzcylxVYKI)88FavpnP znmw)<_yhkkdX-(^tCe`tW4#gmN60YgbG*{r`{RD&+Om1|Jz>`D6ru@1(QGcHGwDod zf!9@X%i4Jmu09Er^fBGdX<1D(o0TLB z3smp@HuOX;>}G+Cg4iktu7!;~c5XIcf6R1?;f>>(nboP2Q{z@gw4OdB)eCBFe$(3Y z`gg->>2q6cr)Pjy@WAWt`981fTQl$$UZh;#MDN;k6j(%c{b+ GC;tJL* Date: Thu, 10 Apr 2025 10:59:59 +0000 Subject: [PATCH 435/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10219 -> 10218 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9a43535c8..3797a8f50 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-10 10:58:38,854 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-10 10:58:38,854 submission_checker1.py:3260 INFO] --- -[2025-04-10 10:58:38,854 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-10 10:59:54,848 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-10 10:59:54,848 submission_checker1.py:3260 INFO] --- +[2025-04-10 10:59:54,848 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-10 10:58:38,854 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-10 10:59:54,848 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-10 10:58:38,854 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-10 10:59:54,849 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-10 10:58:38,854 submission_checker1.py:3276 INFO] --- -[2025-04-10 10:58:38,854 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-10 10:58:38,854 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-10 10:58:38,854 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-10 10:58:38,855 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-10 10:58:38,855 submission_checker1.py:3297 INFO] --- -[2025-04-10 10:58:38,855 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-10 10:58:39,856 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-10 10:59:54,849 submission_checker1.py:3276 INFO] --- +[2025-04-10 10:59:54,849 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-10 10:59:54,849 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-10 10:59:54,849 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-10 10:59:54,849 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-10 10:59:54,849 submission_checker1.py:3297 INFO] --- +[2025-04-10 10:59:54,849 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-10 10:59:55,838 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 36804484b..69ed05c04 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1062 +1063 diff --git a/summary.xlsx b/summary.xlsx index 3b8883f30a994f564c347e89d9f6b0f6699ebd02..14a2d8614e49c5087b432ad931c1f5d0f25af896 100644 GIT binary patch delta 338 zcmV-Y0j>V)PwG#w79s`Jj0>M7lN%x*e@??N5QOhMMdW*8rzHriB$WX5L?tAOK-?^Q z+gijwvJGk89w%wjR_cl4dS<_!UAbKBRg?UJ^U~-F^GXMm~(0~FRJkGNmpS%H_@>d%M<;vMM_i{3}ZJV|8!5Z`D_;dC0HmsUT-JgmBs!PEscff*ib&1c5c({i7u!<+%NDz-* z)UUi978(0MBryOT;~tfd`P1`yB#G)7nH4FyO>?r&2`wIIQGE3yFZxeFbr|w*;t??B kzWfCBAG2K}3j_t#j0>M7lWiwd0bG;9CnE+9B>(^b09+27&Hw-a delta 339 zcmV-Z0j&P&PwP*x79s^NBaK8RlN%x*e{RAs5QOi2ipclG4k;D2k|4Dery^AKnJ}@m< zYcy174<60yyOLYVje|F5EVv-SClRDQ<+eiG5G=*m^DQXmv*^%K+!&`=h^1?=e`Wle zHGn6CJYWTZ2^JXcfvFvd=!g`2MA{$M4gkS1w4gu-kMk_YXKw(f{LO|zxo|2O_7-kh zPvYp_c{v)puFJY&u*Up3UN2wYhgDOl`%`g1btyRI4p=a*F7ag%57#mumhr?J3F2uV z^&4-;MGk!+k{Ey<;+~Yx;`wE@B|!CrOr|NhPjj-$37yVp@%Yt`yy`y#)p5wbi6_7z l_w^^J{{XXGBMSrtFC&daCX;L@Q~_O+!6zdI4 Date: Sun, 13 Apr 2025 21:52:01 +0000 Subject: [PATCH 436/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10218 -> 10220 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3797a8f50..00eb3e97b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-10 10:59:54,848 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-10 10:59:54,848 submission_checker1.py:3260 INFO] --- -[2025-04-10 10:59:54,848 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-13 21:51:56,202 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-13 21:51:56,202 submission_checker1.py:3260 INFO] --- +[2025-04-13 21:51:56,202 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-10 10:59:54,848 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-13 21:51:56,202 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-10 10:59:54,849 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-13 21:51:56,202 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-10 10:59:54,849 submission_checker1.py:3276 INFO] --- -[2025-04-10 10:59:54,849 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-10 10:59:54,849 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-10 10:59:54,849 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-10 10:59:54,849 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-10 10:59:54,849 submission_checker1.py:3297 INFO] --- -[2025-04-10 10:59:54,849 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-10 10:59:55,838 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-13 21:51:56,202 submission_checker1.py:3276 INFO] --- +[2025-04-13 21:51:56,202 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-13 21:51:56,202 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-13 21:51:56,202 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-13 21:51:56,202 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-13 21:51:56,202 submission_checker1.py:3297 INFO] --- +[2025-04-13 21:51:56,202 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-13 21:51:57,197 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,7 +37,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410968 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409454 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 69ed05c04..373aef1ee 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1063 +1064 diff --git a/summary.csv b/summary.csv index 872b7e91e..c67f70128 100644 --- a/summary.csv +++ b/summary.csv @@ -15,7 +15,7 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410968,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409454,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4e9051346..9409215a2 100644 --- a/summary.json +++ b/summary.json @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410968, + "Result":0.409454, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 14a2d8614e49c5087b432ad931c1f5d0f25af896..56461e997982a367a504feb549584ecbfd708457 100644 GIT binary patch delta 796 zcmV+%1LOSaPwY>yatH-Pnw)y>vv&xR34hbKw*HIOE|hi-l;|39q|=9JeChNd8icZh z(}!sQN-`d~S)z-aKCBafvdHP9!RZ4nZPJM{ElDL3M7=wUt`HZu@-F`)1$MUJj5>4v^$sJJRMuG`_U?5DfxZ!sf#?03;a?09j=7VSk+f zkVQ5h4K^QWX_HQrX-go%=0n2(Nb&|B0P;DxeDZ54l zhQUcNnZ!mfOi$M;a?`qc59GPpzh1pE@j;&U7tZ?Q>>Ur?$?64<$#^z2;y+ifj&!nD zN19o(-r#&O35?|XJUFZcck}cc%72Qj8U4G#>ZOs!JQ|15IQhQnJ$q~AV$waDhW(ez zK`mzEKNrEv7rhtG{)?XT>K8-5G7gQj^m}Pq-LjYL0~^-S!Mr;RK6RtvcorLzV2~DS z*4>&BtzJX9{G5dI<)`#l+Sf2i!s-1_qoCUlqU@*ireF|;$^8%6DcQWv^8)Cfd@&OQ(dBib2`V^(gO%$Bkz!2+$$r9^H+bm*u{cTxpE5bdJT)#vpBntme0nn>!L0j ztv-K(KX)&0TBts$evy28dIpD)9BiFRmrl?fi=O9N1&;^@X?7AX6Q%zz4{u zxM$@fdwL|Ux2Q_vbQQ+;VUpEplCS((e+?~f>MuZXYVvR588GF({RGvsTO-Q?1w@*h zdhU}eCR+iBlZPfe0TPqMCP4wWlLjX`0V$I~Cng)6ni&}-0RR9{0ssIJ0000000000 a0001hfeMp?Cq4mQlffq=1`s6x0002q0e6M~ delta 798 zcmV+(1L6GaPwG#watH}?AdDo7#`4EjSZ9YVUK$fuiFbx1n#sfeW*?d?h0DokW z%}0aH2U^;s6J^>GNU-_PFaVOg!3ThRPA;E(ApIu+`CL6d+7UqNrgW({wqAKS4WeOi z5=4UPEE)vF_&?A4KG zmaI28UrYic`92R0Yr)+-{f4q)Ykx-nZm@c3q%n`iVKh#@uX@kkTDh2XkEUV&<#JGq z+4#>z@bX3Pg|q*n=e+vG(65X`BQ5=2npU^$CHugJb#yTA4uenKXgHq5#v~Y|g_?D@ zW<;ykP%b|w;e7ci{gw7LOpNW93mfo%}&EG5QOhMMdW*8rzHriB$WX5L?tAOK-?^Q+gijwvJGk89w%wjR_cl4 zdS<_!UAbKBRg?UJ^U~-F^GXMm~(0~FRJkGNmpS%H_@>d%M<;vMM_i{3}ZJV|8!5Z`D z_;dC0HmsUT-JgmBs!PEscff*ib&1c5c({i7u!<+%NDz-*)UUi978(0MBryOT;~tfd z`P1_xdWq^8nH4FyO>?r&2`wIIQGE3yFZxeFbr|w*;t??BzWfCBvt1+00tMrhvbyY( zEhbw5hLedVJOU6ElP(n&liemN0kx9|CprNqlS3ya8`X>ppCthR08jz|01*HH00000 c000000DyrAlY}Qe0bG;ACnE+9B>(^b02M=Y-~a#s diff --git a/summary_results.json b/summary_results.json index 8e9aedb17..0017b0a24 100644 --- a/summary_results.json +++ b/summary_results.json @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.410968, + "Performance_Result": 0.409454, "Performance_Units": "Samples/s" }, { From e82e74592597c1211143deced26913efdc8a08b6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 13 Apr 2025 22:03:41 +0000 Subject: [PATCH 437/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10220 -> 10217 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 00eb3e97b..cba7f5960 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-13 21:51:56,202 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-13 21:51:56,202 submission_checker1.py:3260 INFO] --- -[2025-04-13 21:51:56,202 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-13 22:03:35,846 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-13 22:03:35,846 submission_checker1.py:3260 INFO] --- +[2025-04-13 22:03:35,846 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-13 21:51:56,202 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-13 22:03:35,846 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-13 21:51:56,202 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-13 22:03:35,846 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-13 21:51:56,202 submission_checker1.py:3276 INFO] --- -[2025-04-13 21:51:56,202 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-13 21:51:56,202 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-13 21:51:56,202 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-13 21:51:56,202 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-13 21:51:56,202 submission_checker1.py:3297 INFO] --- -[2025-04-13 21:51:56,202 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-13 21:51:57,197 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-13 22:03:35,846 submission_checker1.py:3276 INFO] --- +[2025-04-13 22:03:35,846 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-13 22:03:35,846 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-13 22:03:35,846 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-13 22:03:35,846 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-13 22:03:35,846 submission_checker1.py:3297 INFO] --- +[2025-04-13 22:03:35,846 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-13 22:03:36,890 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,7 +37,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.409454 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.4168 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 373aef1ee..f7b4fb992 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1064 +1065 diff --git a/summary.csv b/summary.csv index c67f70128..5693e508b 100644 --- a/summary.csv +++ b/summary.csv @@ -15,7 +15,7 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.409454,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.4168,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9409215a2..4fbab3ce5 100644 --- a/summary.json +++ b/summary.json @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.409454, + "Result":0.4168, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 56461e997982a367a504feb549584ecbfd708457..f1b86472a8ffa8839bf959874b4097d85731657f 100644 GIT binary patch delta 792 zcmV+z1LyqgPw7vva|j4a!BguB000oPdkBsRf3?NaE|PW*lIR(6q|b+FeChKc8bq>$ z&xdIMNirU}RicZ0KCBZ!vdHJ7!RG@lZPJM{Ey*MpeP|d!N!&oNj;+|}^B9xKAj45T z>1k{wwUt`FZo7QayJp|hc|bZjK$17@NRtoI_|oJvxyUr z4g?v0&n1%_A}oK}P=xJ3Te51PP?whMgfvM+tp}1?$A4ehPMid?>T-1NoVhcoTpnZ_ z{epEus|saFiXu=`e;`1BZ{1-iqlprs8A0ME$h3I8N#%MH>(X;FTh0sq#R+U zLVNEG!Pv1qNa~W{P(j>kE2$5q-C;x7H{Aiw({zC)c*=jNr+5IyW+tK|lCv3Uer!7e zIK$9_1mzvhk_?}{9;|dX8%E{ADTVJ1+_av=*?n+gHg;W?bj4^5`7`{retjQTjfEOc z#Q@bMXM|axUfa6Fmqj>SOPybb6K@6Zr#|R6-j0h*eZZp-fJ|{u%4hNX(yUOOr};yi z-p5(dk0q>5R)+`R{@2S zi6%S(4;7Ox6&91-CMp56lL#j|0Vb0}Cng&if6pZ)0RR9{0ssIJ00000000000001h Wfd`X>Cq4mNlfx$?1`Qya|jKZoO3M7=wUt`HZu@-F`)1$MUJj5>4v^$sJJRMuG`_U?5DfxZ!sf#? z03;a?09j=7VSk+fkVQ5h4K^QWX_HQrX-go%=0n2(Nb&|B0P;DxeDZ54lhQUcNnZ!mfOi$M;a?`qc59GPpzh1pE@j;&U7tZ?Q>>Ur?$?64< z$#^z2;y+ifj&!nDN19o(-r#&O35?|XJUFZcck}cc%72Qj8U4G#>ZOs!JQ|15IQhQn zJ$q~AV$waDhW(ezK`mzEKNrEv7rhtG{)?XT>K8-5G7gQj^m}Pq-LjYL0~^-S!Mr;R zK6RtvcorLzV2~DS*4>&BtzJX9{G5dI<)`#l+Sf2i!s-1_qoCUlqU@*ireF|;$^8%6 zDcQWv^9Ja@P)h>@lVK(rv!fGo4g{T=85xrqA}oKJP>k(BTe51PP?whOgfwwPtp}1? z$A4ehPMie1Drh6!J7?|;DmF*a2ESk|SF%J&6r%tnRgBA~MDO)Bd_aM9L^9GU2_@>m zqD^&IP@Pj{;MFJ%#&NKL2T7Y#U7~|?I>*@30|;Ux?~q>HDWSJPlk=r?Hqr*XW- z0vvx~#1T9ILp>AG3CZY;)IX*j0gPg3L4b4?CsBegUI#|l#ez|}atiKx4U5*ZIJ=LQ z&&ICnqAnY)K7WEgcQ0?_s3DibswkkUU^J%&h*PF2@b$$nt|iv({EGJ+*kkYYg}2in zQys9t2gs+mXXPV%dak#qO5=1D#`j^8)oB=#ul!km4J~i#FF@^9i9Fy+4e1l1q2 zTO-Q?2%4OF?g{_^5|b__R{@BVh9*1#5|hIwK>@du1t&TIDU(1aCL5ia85t!3002+| l000pH0000000000004l23X_2+J^@~nz$YUH5G4Qr002_VbZY Date: Sun, 13 Apr 2025 23:41:20 +0000 Subject: [PATCH 438/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10217 -> 10220 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index cba7f5960..b948991c8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-13 22:03:35,846 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-13 22:03:35,846 submission_checker1.py:3260 INFO] --- -[2025-04-13 22:03:35,846 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-13 23:41:14,963 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-13 23:41:14,963 submission_checker1.py:3260 INFO] --- +[2025-04-13 23:41:14,963 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-13 22:03:35,846 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-13 23:41:14,963 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-13 22:03:35,846 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-13 23:41:14,963 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-13 22:03:35,846 submission_checker1.py:3276 INFO] --- -[2025-04-13 22:03:35,846 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-13 22:03:35,846 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-13 22:03:35,846 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-13 22:03:35,846 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-13 22:03:35,846 submission_checker1.py:3297 INFO] --- -[2025-04-13 22:03:35,846 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-13 22:03:36,890 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-13 23:41:14,963 submission_checker1.py:3276 INFO] --- +[2025-04-13 23:41:14,963 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-13 23:41:14,963 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-13 23:41:14,963 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-13 23:41:14,963 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-13 23:41:14,963 submission_checker1.py:3297 INFO] --- +[2025-04-13 23:41:14,963 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-13 23:41:15,987 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -40,7 +40,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.4168 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1762 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index f7b4fb992..00146f256 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1065 +1066 diff --git a/summary.csv b/summary.csv index 5693e508b..6a97c2528 100644 --- a/summary.csv +++ b/summary.csv @@ -18,7 +18,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.4168,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1762,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 4fbab3ce5..a1dd8bd1c 100644 --- a/summary.json +++ b/summary.json @@ -579,7 +579,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9236, + "Result":21.0819, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f1b86472a8ffa8839bf959874b4097d85731657f..8d5a391071eaf987e10c0722b5d1d2f35fe1222f 100644 GIT binary patch delta 434 zcmV;j0ZsntPwY>yatIEzBRE>_3IG5QQ~&@N0Fyx`8nc=R6AB8q9X{EL+*9&NPqVZV zVGjiP9egvB86qEl&1%Ci5QOi23ZuI%%T7uWwgYX+sewXWT6z=GCK0uMkkmSU`zlW2 zB#={&tC{_Fc2Tw1sV?~i=cUm#B3X_S(A)^AH#K^1m+1pad|+Cz&Sl3XC#3yx-3SmILk9|U@R(!-pS=N`@;4hs<-)0C7%behp2W$$^KvrweV_H^ zXpQ*^{#?DhjjN_o?@z@6HI?9$J7B@Mroxv+JY2_oSj7{sC5Xo%>NnnwiyZnuBryOz z#62k=<_lPV@#0f>`^COiQWlfx!K0k@L{CprNslRzgX8~Gi4GbI5408jz|01*HH00000 c000000DyrClYu8b0bY~9CnE+BB>(^b00E7-NB{r; delta 423 zcmV;Y0a*U*Pw7vvatH-POTkm?vv&wK3I`nlq;5)&db76@VGjfuf6pb893mfo%WA_g z5JmU>3ZuO(KiW`)?Lb?yYM@Y;mh6NyNkpv&l3K@qU)fHa1hVRKbnl$GGpJl1WE=g0 zbwaBOWl4%6P)u{7b`|<)R`DZ>oTrM@Rx7Ab4-PHsyOJ5gw1qdT4OlP0MFFH7VWvWR z?+wA&u{}uYlHgE5+-fVS52f9IVMEzB-2u+ibb%#!%BiP#0LEq}q9c;C8EJlOI|4Yv z(1HZz9nO*rpS>QebT=DD<-#e2?+x6vp2XRGaAG!gU6*vlXbt%@{I!04A6Jcq8cxLk z)g@k!zUvK4J7~o007c@u>Sx6 diff --git a/summary_results.json b/summary_results.json index 6fdd88e27..c452d6635 100644 --- a/summary_results.json +++ b/summary_results.json @@ -860,7 +860,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9236, + "Performance_Result": 21.0819, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 8dc2be7eacfcb7e6fcedc57964888a816d1eab48 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 14 Apr 2025 08:58:43 +0000 Subject: [PATCH 439/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10220 -> 10221 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b948991c8..bfcd78932 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-13 23:41:14,963 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-13 23:41:14,963 submission_checker1.py:3260 INFO] --- -[2025-04-13 23:41:14,963 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-14 08:58:38,880 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-14 08:58:38,880 submission_checker1.py:3260 INFO] --- +[2025-04-14 08:58:38,880 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-13 23:41:14,963 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-14 08:58:38,880 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-13 23:41:14,963 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-14 08:58:38,880 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-13 23:41:14,963 submission_checker1.py:3276 INFO] --- -[2025-04-13 23:41:14,963 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-13 23:41:14,963 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-13 23:41:14,963 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-13 23:41:14,963 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-13 23:41:14,963 submission_checker1.py:3297 INFO] --- -[2025-04-13 23:41:14,963 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-13 23:41:15,987 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-14 08:58:38,880 submission_checker1.py:3276 INFO] --- +[2025-04-14 08:58:38,880 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-14 08:58:38,880 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-14 08:58:38,880 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-14 08:58:38,880 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-14 08:58:38,880 submission_checker1.py:3297 INFO] --- +[2025-04-14 08:58:38,880 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-14 08:58:39,918 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.29349 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.2593 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.342409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1762 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1592 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 00146f256..5ab2c45b3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1066 +1067 diff --git a/summary.csv b/summary.csv index 6a97c2528..07f20a33d 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.29349,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.2593,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.342409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1762,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1592,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a1dd8bd1c..358f7e995 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.29349, + "Result":8.2593, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1762, + "Result":23.1592, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8d5a391071eaf987e10c0722b5d1d2f35fe1222f..96ca2f7a6256e55b5de73c494bacfb690afbdca4 100644 GIT binary patch delta 1942 zcmYk7i$Bx*AIE1zl3`1ig)KHrZfTL*Lc}se5+c+%Rw=oK@%xU^lItvExu!K7IxZVJ zNYP6rzQ=}P=rOE0jp9Hb!xa9h`uk6p59z2j-1fIwxmUf;*v&yZF zQlz`Zj<8b>9N)kFfd2{p3zSd}Nd+6SFne2g#i#PFna_2xs!CtE5ys64H~w8e`D&Z5 zlAB>8`9LGY%yt%E4CzNQ&;@1$NdnX$4NI#~=Ry{2n<5`Vc?Z5lmh4_q&_x^BB2~Dw zJCu%{DXh?mH^=0pRtt$QJg;N22)E1ln?^vsc8=eu3uTSv9sO>&fo4BE$r*cFyHc*I zkh$4{GP6=4%^ZWfG(|s%qYXW8Sc`!GD-nD+b#+BVYR zfOYn1@>FNtwTX8SI?(Y)AbJloF>Iob@$r(*#Cz%hKLOf@%-C30OxKdPw547Em^B-Q z@>cj;r$B-9N?IiN!okhhN4&`b8^`=~Xo=J|B&Dt6_=%ONnfhxo@BI|!Pl`l7<-XT5 zpZPSh8t(`N7gCCp?7hhkIvF3;O09JqUy$>p%Te&lZeGNd0C8#6Q!~Mt3OpWO>x~l# zhk)L{4bFy7N-1B7SL7Gp(E_45QN@n+OJDDP>W#Sc4yYW^6u3PNcth7ml&-S*v|x%O zhwfvPK9@~GS)k?KU2l7dJM^-Eka`V!7rJ5&jU@sjD&ki#_k1H7rCUV`G=sEan9>SH!#$ zJeCqf9~I-UsjqP2T?$Sdr9~u~=PHgo324zgA=S$EsvQ;B_s0#41fuPjkFkw?IH1YP zB;aK0oNLh)=_S8`sZ}BSGXhz379Cm(; zI_3OWnH>Lzno)7Zn84;vetaE%aL{bod9c?SR;ZD|+ZOvR{qU>V{@ZWzhumZ>&+GX$`r?GX+-0E!vQP!t1EfZ=duYLNq_tyksk)>d8VTfJqz0<%o@+OgK&BFK zaKaBy?LnqhVh`=UMAn?h5;PaS_H9m5ct*Bc@M)!g0#ba-?lWeST~;f4-A+=Jr!xGF zQqr&H75#E)juR<@c8Avk&9QznREs5p0#VnPoLUoOctVx2dMg-s1N`#wsF1jvj^z@Y zTQ{#k5BGeH3H6lw{JYTPk@A(3Tlc8LM>W!8u#l&B*W!oCUwV5x*kr8ywm~k{`)wzn zpO|Y9Yf!Lr$WkzDaQ`2HDsYnU>j1AQ&|pl)xaui2=HB7BPv_cKb&WM!k$OW_FHdQB zH&eqlO=Sgq7hyG$R~?X@G`L-S&bkp{JLV$PYenv7Y$NI}*2Uxs`T+^<16USh&xdL^ zB~6B^Is$NRH&jeowy>Kc&#+0ZHQDaqAy`h9Hux@gG32JwpWVjn7;*vI=}LuG#sX_6 zDr!C~Cd+9}(#MYiUR;p+VyF7gnZ?`~Uy| delta 1941 zcmV;G2Wt53PwY>yatH~uBRE>_3IG5QvwH~n1b_R@yeGbjKXyM>_?X?cC6=gh9f>7* zTn|_h4PZ$$fF){RqTFaGmLvgSS&*QmfvNSg=ep-QId_qYZ#^9=+|98a2_|Y>SAvNe1e0h0OrilW zQGbJA>LM^n0sylhL2GmH0ZiG_CY=y4Wmt!V!BbV1TJEKsZ58nE1+$78%RFdZ4{UQ& zX*X?0<&epDNskq7KH9D%lWb6Wl1$VfnM4C*5)F`v8eGXXi^wDifXspfth2&c^EKZ;ECq_u1nHGh_xGMigu>So>Yavt_5zNdezaI4(*B$udh zeaR(ikV~QgE{O)XB-{JIrB%cwNdR0HBxr38K5!{p+N2Z0r3~wkFq|<-YMe2x{{t?o zsIkOrilYQG;aKMP!l$KxRRL)_>;U z1DUd=O*(-c5QyrqC;*eWf&fhWS%WF|Lw=meD#|Oxq`kwW_Fdb}yT8n_=_qu7IMSy> zG@dl-5DhX}!m7hGz$7ulNOzbja_g{8q+O>dK}&&Q2U^;s6J<6flwjJSVL&AjNhjW! zs?xZVQn~n=PKJv$mIBG-cDkG|vww+F`E~sSD4(f^=Ui=@cvRhOn#~*!kJ>Tum=8>% z1;mjq9-{H2jfZHE$r45$rU532nMN82RFRj5bplKlxp|Z{+nhp6n~hTLT|!icL=hy3 zwFho2EB5W|5cyYZU{+FKDJZw?Jn3Dfa-BRVy&Ndf4dO^c57Bti(L*!{Wq%1z57PjY z#8(4Q7TJ1OCje!Uu}4YY%_+3B*(d=@@b8c)T)Ywo4=!FSw($J*P(D**&xP7`_E^ew zw@}!n($Aq1O(Tx<_7IIH%{@eeRF<&!Fbz;iG_Itw$l=2h0F^}^9|a~KXlav9h@Ak8 zJ|qn71SB=u2`cvbAeKMhPk-=C{XUm!+wkMmmFcRS|5v3=&ZS1jh$C%2MB_=D577XZ zOe48u8p$QoNG^4e&4(qBToxs0X|VY~OPh3p+m;a4pWXbXk8$}^_AM(Z zvdlVGZ##Xou6vf3Gh=F%LneAg9O?8S8c#ZXhz7|l;q+k|Ad_f-%zq-M4@&@K7CC)1 zIDMd{O*$cDg42hD!9td$hJ~!I*y;24koiolJ{Nl1>XR;Jn~rbi?2l-jgC^QW9O?BT z8c%wChz8Lt;q_q}K$B*L$ z34qKZua5?=545yNCxlG!`j9Z3G)ZclG%NP{{5@noQ?t*7-ZuMWlcuM2-kqkEgC;sh z9BKC<8c*7Nhz8LtVfSGgK$BGL5PM6!g>hiL#wG9I~AqKkY! ztP?=8$mgTM=L0Qm(up!H$s`zkXc$0A+(597t=Q=E7?a5$!%;oyX>296m0G@TyL{5S zX5Z6!Ksq@xGBYAQ}cI!DJE}y)Zpp ztH@33>MfAxYX5rm!o&x8+Fv;9kFz&CbSJCVJ0|1V(0_>kT)jBb$zB|3X32Vk^Ti}E zlJE22uom3S({Cs%wr2G22CG*_8uMrzM&soBs`u=jm5WLDXd3okE(f)kjsIK(FJJUt zIQuVp&Z}Px{mM8r($epxX?4q9vJY%nM+funF!|AW+Yl_p*z+wY=CkO~QCu6RScs+DV1LW_H`@R%^86kv2u!fR@E(}jiHMF!!6&5s zaoq?I976{RbnuvD1fRVDoboptM&-h(WEd>mw4TJtz4LN1_I;oA_3IG5Qvoa<) b0R{OTd^07JZzog%UX#QpBL)y900000HTjI> diff --git a/summary_results.json b/summary_results.json index c452d6635..21d3e904c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.29349, + "Performance_Result": 8.2593, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1762, + "Performance_Result": 23.1592, "Performance_Units": "Samples/s" }, { From cb258cd6a99a906ea56124db3083670e25c3bd3d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 14 Apr 2025 09:04:48 +0000 Subject: [PATCH 440/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 12 ++++++------ summary.xlsx | Bin 10221 -> 10216 bytes summary_results.json | 12 ++++++------ 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index bfcd78932..15da85672 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-14 08:58:38,880 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-14 08:58:38,880 submission_checker1.py:3260 INFO] --- -[2025-04-14 08:58:38,880 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-14 09:04:42,828 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-14 09:04:42,828 submission_checker1.py:3260 INFO] --- +[2025-04-14 09:04:42,828 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-14 08:58:38,880 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-14 09:04:42,828 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-14 08:58:38,880 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-14 09:04:42,828 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-14 08:58:38,880 submission_checker1.py:3276 INFO] --- -[2025-04-14 08:58:38,880 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-14 08:58:38,880 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-14 08:58:38,880 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-14 08:58:38,880 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-14 08:58:38,880 submission_checker1.py:3297 INFO] --- -[2025-04-14 08:58:38,880 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-14 08:58:39,918 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-14 09:04:42,828 submission_checker1.py:3276 INFO] --- +[2025-04-14 09:04:42,828 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-14 09:04:42,828 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-14 09:04:42,828 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-14 09:04:42,828 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-14 09:04:42,829 submission_checker1.py:3297 INFO] --- +[2025-04-14 09:04:42,829 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-14 09:04:43,889 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,13 +24,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.297575 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73032 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73086 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418802 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.10717 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21557 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.2593 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.34472 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.342409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1592 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417733 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1592 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413841 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5ab2c45b3..b9650fd8e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1067 +1068 diff --git a/summary.csv b/summary.csv index 07f20a33d..20056ef8d 100644 --- a/summary.csv +++ b/summary.csv @@ -2,13 +2,13 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.297575,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73032,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73086,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418802,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.10717,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21557,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.2593,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.34472,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.342409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1592,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417733,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1592,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413841,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 358f7e995..86fd24182 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73032, + "Result":1.73086, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.10717, + "Result":6.21557, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.2593, + "Result":7.34472, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -634,12 +634,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417733, + "Result":0.413841, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 96ca2f7a6256e55b5de73c494bacfb690afbdca4..50b055ab5e83eee29ceafe6dd5d5318bf4c98dae 100644 GIT binary patch delta 1964 zcmYk7iC5D39)^F4qEtwOVIvl~fVdBF(I!OHdnq9?Gn@u@NJ^+M(&&zgVP@k}rh%Gk zYHro7Tt`Ps$1=+$Ma`ws%vRp=xJW+Bqxii(bj9u{N5hz`un$^?dQmSy8N$XBvhI=qxF(*p|!24 z#+^VK1;n`x{<*Bp5KmLpFV}ZG9Xxl5{-(<~%Y@;KKf#RtRVCfzscE?)!H{9cvlM$B zh%YF7IjWOZ<5BfEGuke0c4CkDfLWDYv$6-jpjhOvgd5>}Tg9q*i7enD&e7n&O?8N8 z#$}jyONz}zl)$ush~5@JmmmY^pT0)4nw_$+_V<+Uy?c#()ki6$ks>Pe)A3G?pn=7)g)PWL-4~&7t*~Mo3;VO)b-dG zA{N|I|N8m)?eSPXiE^2qf|KK*I2z$0r>v>90S-tH1fno)HbouEcIyW*6d`qoql6hY z^t|l&`*y7x;zsaEjCx9BP=C90@F%l=S--)2P!45Zj1kn1@{#5Q>(oyYV@P&%&=EM) z!GrUO4FOsV{Szd084NB@dU*Sxx!FV$@5gWrn3b=2<=ooh5i-MHcIB;biFt3rBLv1? z-(}WG_&_}i>-?o_;9msl2}l_16_88;x$Eo~F;#QQnm*>$4{tUBDb69ipW1` z>)7bK=b3zL6>{sY`2E;XzTU;mnq!q&pTFL#Xs+%a|NdNlTQrx%`)q(5lJL+s=|7Fij(NEtc}lS!t7IB;m9$FO*T4T*SZSd;)Rf>kJYA4M0eB;uG& zW!)u-JqX+I4HQ26ZqL|1&w7P-HJ*7jKb^V$-Gkr`N`H(|?Iy2S$xFvPR`R%TXpYD5 zqO!v_!(!R!5GZ|U(I9K69TO$weqTb{=7{4Rn_5)35wa_U9<&<;#%=uagg|kxi|wop zV_$F+rSEUZv~S2i6_rS(VQ~Vu_%FjbrC}z4b}Ex#zO=Gt$00MyWmg7mRyoqbBH>_G zVe(~TqSG$kvT~wt!(+gm7VKp<(m~2qiamlk16-Xq;e^-WIR!366^2-4bMlEE&!_Xu zFRYRdj((jqyOUpc^vDb`+~T*!S5}e>T9R?ybNRApu4qDe5lUHNe9N`UF zv{03*qYx3kh&F+4g_tz8JQx&Y604`#TZ+&NNBQG*IX~XR+QmP}uRPk?$G_A}btLTvg@OMA1TJ1J>(C*dg)+?@aBS07+>WLZ{M?=j|*kI*q92`P8b*y{G+( zeN(^ayRL2xSU9*~7BUd{E> zjXvnPQcHLc_{MqZ67yQWh>x*AWzQMX!u3T%rx_Cv1s#lue(a>k1JH0pFJ z`o9YsMQ$C_8g5IBP-mQG!|+D*RMHvW(U1kd@)+$zEp-lC0l`Rj8=8ub7eO@dqmzL} z3I-kLFklhZ@A=HW49a_t!t!1nn~eP!dh(AfpX&5k_l&j3&t2ol5nV?JH!!L@16f&VN1&fL7;Z4Q@i zcdmMpGJm#Gg=93WWiHmFm(&Yj^l1u3BY#M{YvcC|fadjFb#>w|nF0IBA)p^eify*F zq%oRHssufuJFv^`t-soFAGPfEhx`JQGX=Ww3^;!+%$*!pG}fbP{SW*j7h2lr2r`azi<7iDc<;Jl_+VPF-cVmA+5@nT!;Zv4)UBhGUA7Ee!8(7@B0s_+^-^X%dHy zZ7fI2v6MA+><958&YX$LHWlieGkUN0I_LTKxvu;A-1qnXKF=Rc@11JGcgmh<$Ao^^ z1%W8k(|3NB0xq1NJcsBs$oX2aRonh>sZqbeCS%b~w`VA>Bs-XKx>Q2fMRINa)QJkT zETcC`N+0uC#qr()2XmLQ_+=S?btD!?(MsAMQOOY4EsRET{VV!g;-IBwPNk6zez(_?(o!Og6hC}jp$a?2975sWR zrovBc-x+_Sb>_$-CwHyNc18bNK#IjeBkpH$4)z8vOdq&>7@5$D8K06dc%)@OhR)QX zNS7m)6Ya+bv4x5Tfu5}7*>tk&+jQDD%`mT%84rRe_QaHimrrbuo9$6fDS}SsQcF7z zW4Prc<5cNxF(bmnuI&8(_<(-_{su`bhowP{xcHr|{9=>5ix%@;+^W(iu9OK2%BBA{ zOg-7^Q_Ib?mb|GQYHstETnrmPftUhwiX;VYm`-3-Xw}0OZCavk!THKxqDr=h6m-T~ zwn!Z*?FMHP<_arx6D;sKY1Kk%vu6N4i*n^*AFyGZuao0D=EPX#ddIvPX=K@sOqGql ztX(cwSGc{=iZ-`Wqs<*aI<>^yl;-NqzV%1?WyjEXfR&0@tS7FO%I4aVtNZbM{YJ^PY{kJN&5vYy12Rc!|^&EVcc`!9&Y4a}5_|Ui&I692SXu z%FhSfe&Ex@ZMrHHIHwjV*?QA&c7oFyr6gVZW_q4}FA*Bj~n3aA{!3S95|KW7_gmag#n zSiubYGPaL##(Xvno!ipA&;hqINQ3hU&GFj_w}sm86nDzZ7p_$0AN9hZxU2aqev6Z% z>=mS5*_1)_9<%K0;~r}0J%F&(&Wzj?yDlW!FsYodO(Xd%G6Ap(Vh^z}5aYB0BC>$^ z>vpA#+k_|D{SNfGc369yU0UCucgL4`6w5sg6_NMWJ9ITge6r=gA|XPxGuOBEo{IH? zIm=?b5;B1i#2ypN5z?Lz#k&-oG{%ZdvdC2&z31PGJtWn}^Qs*a*bc-Gjs{{*a_$nE zfPP|&*OAbJU5oeg&8Vf9pR+G)+a6;DXT5}9&@7YKuB_hbW{s2>bj+4HCgUw$C2MR< zg55sHyJ!=~yDIdAKQ)YtE5-%ZSMw9<$U{TsA03B!Nr*!2O#YUb@BMeanyvryO?~HE zA|jW>b^&Z5qXJTDuD7dO@9n1nmB$Gh+G{|Rikqu1Ugvrtkuxzz+e@~PA+M(&9Tc1( zJX_yTl-KfV7jzI9=>50f=nkADYL3dHyyn*iX4VqLPlpYcLxrak3A;tO}Q(^d~4rRtJW5YbHV@3Fi6TZj593ww%1ZHVtC^}foe#a@be(QCD3qO#deU-cOm~8)mYCJ}0cOnqTdoojkNve2UbhX*2F5)Ne!W2Ddcp&eX-`3I+fP zXM0I0&<~5OFGYmIcD$)}RlTL{Vh3VGbfdY86M8}U~1l@*D;5a zX>!#bsx=oFX;3?}c1p^pY1O3K0D9IirtMx*^N5G5=IO^BH|nmoNFK8vqU!-|XB;hG zSQ?okl+@V;4t8q+`4X-BZ%CtcK8A6qUzS9E4<3&j>FjmH$Nw->CPF)YU?RZU&`Dmu z%;oOl+1H}J$(l=gZh)?Ixq+GXOQ>jK6!T24eHQudt9e1pLB{cD(l0B~KGY>ijT8tq zo$g9peJJSPHBukhcylPWpkMU}jkIS+kLI&Y_StVglpKR?99fat7vT#eZ7S4zYxhF` z`DsCY>{*GMU<&q_1RQL~lH{BALm)9Bh7pmmA%PK~3r-nC Date: Mon, 14 Apr 2025 09:05:16 +0000 Subject: [PATCH 441/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10216 -> 10216 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 15da85672..054156c6c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-14 09:04:42,828 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-14 09:04:42,828 submission_checker1.py:3260 INFO] --- -[2025-04-14 09:04:42,828 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-14 09:05:11,537 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-14 09:05:11,537 submission_checker1.py:3260 INFO] --- +[2025-04-14 09:05:11,537 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-14 09:04:42,828 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-14 09:05:11,537 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-14 09:04:42,828 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-14 09:05:11,537 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-14 09:04:42,828 submission_checker1.py:3276 INFO] --- -[2025-04-14 09:04:42,828 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-14 09:04:42,828 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-14 09:04:42,828 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-14 09:04:42,828 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-14 09:04:42,829 submission_checker1.py:3297 INFO] --- -[2025-04-14 09:04:42,829 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-14 09:04:43,889 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-14 09:05:11,537 submission_checker1.py:3276 INFO] --- +[2025-04-14 09:05:11,537 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-14 09:05:11,537 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-14 09:05:11,537 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-14 09:05:11,537 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-14 09:05:11,537 submission_checker1.py:3297 INFO] --- +[2025-04-14 09:05:11,537 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-14 09:05:12,509 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index b9650fd8e..48c6edb20 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1068 +1069 diff --git a/summary.xlsx b/summary.xlsx index 50b055ab5e83eee29ceafe6dd5d5318bf4c98dae..a58c6e6105e41bfacbbca54afdec1798216a09c3 100644 GIT binary patch delta 323 zcmV-J0lfa`Pv}pu6e0v_T#y=*7$P8lZo)7SgztTd$oIrffU32UAhi{zB2_9?QE!&L zfYta%w%d@muM;4E#Hq*i%zit&ayj3tCiw;DrO_42GlCM(+z6?+75Z2$(g&3Iz_ehE z(NLidJet>cCAX9t2XD?;a6y7kB1n76ZH0CrSc;mho@4 z1zZqvj}-(aSYWsZrgkKvBU11YX@6WZ00hU-fC3#n&a)h!y#bu^HyZ}!!l`8FEZnr7 z#L>O?ax}JWo3)d{8uRCPy?lKiR!ybuPsIV%rQno1V8OV$#Fs@pT*G`=#uINOh^H>< zH{Onm9Qr^cF#tWpJt?1)=aqfbB_xVK+$?+B zTEsuH4QbyVCu!4G>WSleX1|?Xxti}(oBV?F(&!rH89@nXZiLjE8ojR;={-t(U|O)& zXsFR1JeoH*6}OZd2d~apa6y7kB1n76ZH=}eSc_4*uQm+InN!KIw{X>Z z6esu2%gNYvUDg$YHRjLp=kn!kST&WpKNSbmRDx6PfCb~43ZED8a4qv;8Be^HARhNo zzw&liWb6Zx!~k@RdsIG(r{~oq0yQ%-yG==%=H-e!P*T#e`07Vq^q+w0Fy!CFBVf#Z V`3agovsxny1O#MZ`3aM0Cx4fAnMVKs From 9b272173f7efb314462560fecfea4b7e98dc31d3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 14 Apr 2025 18:47:10 +0000 Subject: [PATCH 442/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10216 -> 10223 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 054156c6c..a0f0da015 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-14 09:05:11,537 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-14 09:05:11,537 submission_checker1.py:3260 INFO] --- -[2025-04-14 09:05:11,537 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-14 18:47:05,159 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-14 18:47:05,159 submission_checker1.py:3260 INFO] --- +[2025-04-14 18:47:05,159 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-14 09:05:11,537 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-14 18:47:05,159 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-14 09:05:11,537 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-14 18:47:05,159 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-14 09:05:11,537 submission_checker1.py:3276 INFO] --- -[2025-04-14 09:05:11,537 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-14 09:05:11,537 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-14 09:05:11,537 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-14 09:05:11,537 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-14 09:05:11,537 submission_checker1.py:3297 INFO] --- -[2025-04-14 09:05:11,537 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-14 09:05:12,509 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-14 18:47:05,159 submission_checker1.py:3276 INFO] --- +[2025-04-14 18:47:05,159 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-14 18:47:05,159 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-14 18:47:05,159 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-14 18:47:05,159 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-14 18:47:05,159 submission_checker1.py:3297 INFO] --- +[2025-04-14 18:47:05,159 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-14 18:47:06,364 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.297575 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73086 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74612 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418802 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21557 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1592 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413841 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2354 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413841 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 48c6edb20..7d3a042f4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1069 +1070 diff --git a/summary.csv b/summary.csv index 20056ef8d..43f59e8d6 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.297575,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73086,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74612,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418802,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21557,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1592,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413841,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2354,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413841,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 86fd24182..29f8d96d2 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73086, + "Result":1.74612, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1592, + "Result":23.2354, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index a58c6e6105e41bfacbbca54afdec1798216a09c3..b5b9a36d6ad68d28ceda8033762612908b2c2e32 100644 GIT binary patch delta 3211 zcmXw+dpr}|AIFEyEgF+2kK0^kR1;=$zqMgw6NyR`F@{iXNorC*jF8R!QrR%&DYD4D z+|Bjp5{({mPvn-n;K@0LD zf~bNeB z-w-ni<$$5F`=Z*kO0)d{{l~NpDSpPmb!u+Rgb8qZF$ZSsANB`QP<3rRknCY#+gBJM zHF)s%ea7FK4T-UALi(qwKf$r)VRV@`H*wX;jdvdvD{RYz9eWL&vV0}J>!uM6Mtd8Q zS7>f>gU*{ZvXeRY2TVUDGzD6|T~d6y%?h;q?wj{C-F$dY#jm$**jYDsTvy6Rj>D-n zCaqKiuv+|DvqbbWM30wGBFQ8FwQgtqnL77yK$AcCt_ZWZ?1mOz=~ku6JZ37u=5$=k zSh@>@@;>E8+SuG|rQ!p~EdA``A4~SFyNvh*E6EIE{3~nqt1m!n&KKf^uU*(Hp89Fu z3OYHH;hgSKaz4i?f;4^mdz0~bDj@aY#Y6d_aN8A*S5 z4p|hd71Xx-7(x`r*plV>O$>Hc_t|5E%kAT9U9;EKY%iimrf0m?D)a@JuNGjx?q!c!qa z67PonvhoOXC5@2{+L;<*O`N6E&fU}ytqPWtt7sF46#4f~Az4d=dJqoLz52bv(zE8^ zycKSNu7i!O{K<54@h-Xb;vcbVPj%c$#eg%`}$ntJj{cL-KFXH@&AVG&q* zRn+gzGv)GuSQrGSl!Ap&Fx0PUPq0lY6qQLi5dla&1fnI2-GEVGmiXdl_H=x{{@Q^t zY}no!1ktE}g?r=gGjj4${GO}@)c%-Qy(Q%oS;63}&~q-&LMcU!-f~>7e%rZ5G9aMO`SQ_q)SBgHNeepwZ3sU!*my0j!#Z!QnoQ_kL-Uy$= zqIVymFI0G0Npd2>;6hc7ham(%RP`Dtp(6ms!rG zE4p=-$qPMXzC=`fw$W!xuC9PsNzx@oBv>lhaff;Is`S z^>CGHI15eG9NgE) zfl85v(MVBvIuFwRq30;klL|EZ*E#FA^8-1txNmRN0JQ380Dk}S!xLNR`?+j`&XSgb z`qqLi*D>{)_lU9uwU8*i0VW#t6!lF^l#IOG__7)G0jAZAy`6b7c(Kg3z?F$Gw|d|% z;9OH6L9A3sbsNohH&YSEsg6@#4p!05SMOxo#w6&Z^Rj2cYH~GLl=HiUUqH2IFJk8} z%o2=VE;uKhjF74&PH90iF-PK*U5zl)I2mxi0!{L<5uOhZ(N08(6(cpki@;1WBKU`z zPo;iL`e~WM&$W+=ez3Ni`);)0S{N`NCigCXL-;vH zH%?j9n2WXIVpAoJjPZPSkZYo5m%SF<^8l&DHhGnMy6yiRAToezjqlO)n`_pxMpfaj zI88rB(7X6VV{hLvj>n5i@AHzzk2a4KV+P}t>woitxx6&VB`)s*B=OV>MDR`-(2SVX zZuNluPd%=y0vKQ+v>^{JF)TW}-Xw0eZ(G91;eB6qr_BA_BBx5%Ra&?hl+4;K15&%n zfpVf}pri$=?aeVrQv?d%n}GrPU22b#bp3mmcokUP3yj7EK5lY^6eV&^U?P8%BXkg+p*^$_+0&{OkR}l3`J28-ote>jh~}cp7uLYK2VUQyZ<$8@NGhXc4%!B zJ|t0ZV38Z3jXg0@>gC48W|SHCE@lz0Lel{qh#z~TRq~!0_rjd*0G5J(P}yHLkb%|RKzN?1m{;zjlAPho3+>n@{FabBQWu5LDlkCe{g>YA~-ml=)A z|LA54E4*x{M|GbJfV#5A=CzQA7cZKtZ#qT?jV*Y*kUcks9R*?O-YqF6Q2H9VurHP* z#uhe#7V*H?;WJK@(?530YyoWKvt<5f;D4UGw=kO9M!q{hXL`q$r2Lwws!L#9Rfe8a*L0 z4>gTlOF9UP-_)NleYb-)n_2|!j9$}|Nn+_5ZK#%v7{aGH72=205w{9x^?C!{R8dQ2 z0?rLKG2eJ!xhjV_Vdr$CS!z7dsq`^wu3ztQ8+)7VxWX|!b)0t4h>~q^IqIt=b672S zwLz-xX4o~6+>7s}_R`)278Fj{41PvrXE=DOT%$`^@j`aui%Gj`zPvEr2gX`bchbKKBMu&Xh*}&h7WiQ000Pb zrzwEu3m0S2C4!EEz>z6O;Iu!Lcfyc zR?HVWJnPyk;y&6`@{#|q<6r22^GTWx&+E5i+?GDc!$NU4ixrmNrVYgo43j_EZ4vxF zuu%2w5Tn^4he%+X7mUus=ER{)D;Icj-AQKA>qotDF&bCFPBrY9tJOi)lZJe>VU}JE z*!FQ-r9Ulf{0jO&gdA{fe9zG%KZ$ugsxw!+31L_2pEbRxB>OtQH?Wy0qnhgx!!z0YN>7Vr5?6eDIMyWCEGpS%bGO%!EIOIUyD`7T0DuHVM?QA$6I0;^ma|c&-S*enpY>*7jGuKzg`*M|p7gqO zX*-kh_nWG1ttQ8}*CRn?K9J`g8R`+kc1>1dPCM+lbd&aP0bA^+k>$g~Lw&BzME3Z5 z$d~)y_C(do)luQ~R_wc-r1YER4asU5)qh(XxCfr1Acxkki5BtlkO?hQN81dwb=?lq zv5bp7@RWv^G#%*r4fq&lCJ=_2GYD(6!{%WcpUTX>x_K^`_jG2-se_#2U2-=5&nxDy zdwqmb8|+rXPgQaI0NC~Wu}`< zZ{cc_$2Z5B6}g6#q&PFV6WtN=;9Qd!>^9monT0HwoROeb13m;-(WDNAh;C6rj8Rq_D<0n6BYmjG5Za{22^Id0tK$%M@}E|HN{si^BMk?fNZ8i+aOpm3 zfH&SZq}{7y&TVYA$n*T`Q88)vv8j^9r4adyWkJhzy=D!_pJpL{AfwfXvC3JNf;i;s z?r%{A1Z=f1ttVf&oWgFAN*f%~$Uf$5e^+Kyjj-`#);HZ1jW(Lp=7zoDA+#=gZQb?C z@kn-3nwomnl+hn=n&1lur*prBm{fxE0*owN1Nun?Gf5wk!05~MBr5!hyOlRic}yrB zBZLP@c~K#m$4)~Yz;OV)Jj_q4u`1|*9%+cGKSwte|1wBH82W%@ZL{1&jIDI*=lc?D zNTVZiZ_4sJ_2m|Bd0pB6+PKna-+!@i#?7JYz+a))tnN@DV`*ObqAf58fbIvdeW#k) zByLLsywSxDPAEkHoa<_En!>h6_*vG@r4^-zYx%l~{0MLPg#@~e2~AF&o3NX7G9+$X z9P+tGd7MLMw(a=uhHUgN;bZR$ zJUZ5@#pESHolX4p+$<7{ir>!5XEnLhzhcCiJXl>+(V5b&H|gQ`;2A=Ne4_b9+vE!z z@7Y2J`GbRTAV7wQgzMwGK=wNdH*R~se6Q-W?Kiu<~A_|b+JEV98>zEDM~PP;9Xnj zi#j{wXDbHmDe%{a)UF6Gke=$p>+3R!nj~Y)UE6fT20{vfL%twZclWh{0CbB0WeS{a zQ1WYkZ>HfGGWL);4Ij?!toHucFP%c^5WNf+Nxu^^*-xK7uRS?1skR=HhdCXmE@gt5 z#aRJ4s;O{e6I&rikd&DV@e4r$&3X?B`Fnm$6=3O+M!S3M2+R?x9n+_G8l# z?d&sfZ-Tsqf41ln1|;k&Y1?p+GuS{N=jPDVzmeEW65+sMKpIBqoV;NKUNEn^d$!=s zr(aD-RLMniVq>0UM;BV6%sV_(+BX*VByvhWE}SJPMlqT$)MbC8Q)E;q)x1h!T3^1f zo|J*gFN>f_P^%Hrwm$UbY9ctCU`E+PEX6(O+S-*+@ zZ^$R|4J(pFDHZ=wT;nS;khm)hq|Vmv4-+EVnK^K8+F*}JFD@ZZXBVg4)iAPB^55nU zaUwEKl%q@7CP2*i51I{q3A6$D$&W`O-$&*Z*%=8q3;f!3vP`ABf#K&&4nV)2_A1XCR|5bF=X=_~V8^2jLzv1CE4JTC&d z6Y&e^R1D7&E&xgi9iu9Wk5+*dBb9yhj}Xs1(=37{|IXZ715fs%Y&;fX6UK z^UQV~1UDLuBEHwPX}YC^Tca2Kud&vGAec5TEYsGzK%uzgD!9@AB1>e$ec7|ZBXdX9 zVNcR`KX~hY>k=6BI#bwkBT`vu^v=t$U8zeQqX|Dh zcy0D#eUua1{1}u7&&?bbGvwr=vR&KHzPUVk=6Y^CcP@Tf()S;Wfw0Z^C!R-QEnRn6 z$nt=3o2}bYUp{+H9~I4Q;shDTC*lxT$x~kf*l`z5JaLb*Z{=+`@9HdH&4m5^{bz~O z;If$0mh&wegm~MhpRr$4uX)UeefM;<5fVNG006}K$CQ~NcUM~EihD8yPFH~{Q`iBm z2$`7G$Oqlq?e@{nYH6yq;1cM0?ek87>8$9H9jm&@7uT;NE}T0S!)xm)3)K@p?mIsXK3Kyb?K%Po#|>jr|3$PgN8P?8gV0-*JFq!+S~h zZm8iOR4t$!-}-W%6f~T)uadE;r*x#R2iA$W%Y-C>g*EVA4PwPNVEO82*JS-E?*9S^ zcMV1h9yRV~D&?{S7#pV*Mdw;eiJx_+YI52+tnCnF9qDl`y#?}Z;U+60QsshO(TYQd zz(^4{73I~^(A1t`VeH-(f$x+HgvFH{UphVgFkWfb40aM=(r%)aOJB%#Z&$96ex^4_l8I|`Ga-YY2b`-4D7Xt)W)&=WL_1*O4s u1X!NV)zB7T(j_&G1mJW_O^yF_8>|Ty5T(azq6J*(oto;R2jTqo0sjZ;5AKBk diff --git a/summary_results.json b/summary_results.json index 84a650ed2..0d10f767f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73086, + "Performance_Result": 1.74612, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1592, + "Performance_Result": 23.2354, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From c73426d512cd9da3ea411679870fca95c14e1df8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 15 Apr 2025 03:28:18 +0000 Subject: [PATCH 443/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10223 -> 10222 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a0f0da015..d3427b191 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-14 18:47:05,159 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-14 18:47:05,159 submission_checker1.py:3260 INFO] --- -[2025-04-14 18:47:05,159 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 03:28:13,779 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 03:28:13,779 submission_checker1.py:3260 INFO] --- +[2025-04-15 03:28:13,779 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-14 18:47:05,159 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 03:28:13,779 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-14 18:47:05,159 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 03:28:13,779 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-14 18:47:05,159 submission_checker1.py:3276 INFO] --- -[2025-04-14 18:47:05,159 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-14 18:47:05,159 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-14 18:47:05,159 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-14 18:47:05,159 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-14 18:47:05,159 submission_checker1.py:3297 INFO] --- -[2025-04-14 18:47:05,159 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-14 18:47:06,364 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 03:28:13,779 submission_checker1.py:3276 INFO] --- +[2025-04-15 03:28:13,779 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 03:28:13,779 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 03:28:13,779 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 03:28:13,779 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 03:28:13,779 submission_checker1.py:3297 INFO] --- +[2025-04-15 03:28:13,779 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 03:28:14,733 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.34472 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.77649 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.342409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7d3a042f4..7e1796776 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1070 +1071 diff --git a/summary.csv b/summary.csv index 43f59e8d6..72877ca95 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.34472,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.77649,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.342409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 29f8d96d2..81d323e44 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.34472, + "Result":7.77649, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index b5b9a36d6ad68d28ceda8033762612908b2c2e32..7f8eecf34d26da6bf1387c1eb8d666d5e7e75f71 100644 GIT binary patch delta 1949 zcmV;O2V(f|Pwr2!atIDK3T_|p3IG5QQ~&@N0FwbG6|gh?uQB= zv)i`B5;d+Pu_TY{0ZXC*EQtoNL=8-o8x6&hBmgW+5_FW@SU853F6l&>^2r~B^=O!! zF^LGK%v=G_5zE|snQ^Jjrg{RHb<|j8;#?0*t)D$NJ=ZC?i&T8;=}_Trj_pV=QRBK2 zOw=HlL<3+F4S;`%8U$09fk_eom?a51n}ZKv%9bwagn%i-dL#^4UkDRKqhK%CEG3|lOzB# zOA>T82Or3kEnU)yGPNXAhV^I|pot=!GTZ+sGS!pTvUPvdSZT^^=g8FUrsWkp>`{DA z|4`vpx$Q|VQRDiOOVl8jL<3wB4RA@e_kl~Rj7yRLxGYJ~*&KY}QnqwSCxlBG)+1p! zW0KT3V_N?QT-H%zr5V$bk6|sd=q9tkgQCz>VEd9x)Oe;O6E#RC(Eyo517xBG$+XMJ zBng1bk_3O9&A|sUWlNWI0y`iO)nicrCUXS=nD(;cipbV(=5Y)dG? zv_r#yN+Oa@yc1QYaVMp6@imT$uc*Ol4hG@Xz8+1 z%Dqd7>X9gdB(e6ujb+Wgoh>5&iVe(K3akWW-p-TWRqA$b*Kr$4uK-GPgE-RALo}Xr z^bmgyLRrDn!!!UT@zns7WwsvH2|!t9>`~Hpa||tAHcEgJ{5vEH7q7&@gNxUiEj)ic zl+V=IbEdYPJ(g;FR?$OPzko_KjX2WVLo}W=_Ye(IS;5}JG(aWMxRS~;hYw2tRF-*s z6qtOVrAs;?b^x!m36GnJAJfndRA_MxlN`jAQL?!j&%AEjVGNxM1y2jaQZL}kV$_u zKxUcKha~_q%bY$MoIcRfC7lp5!RbT7U?Iy=!$MZq?DYA2$b6<&pEI4e`lO55w&U9c z`y*Nxpoz8-M|ypT#*5-5)F`9=JjC-fXp(lj|Q&~v~)=)giP@IkT9GyNot%lYxes5J!C#pv(K5%n|-oL z)6=@>PSYwt6P+WDwEGZ^C+$8&gJ@Q;`!EflNi=|Fncasa05r?&J{s&k(9(Y;oq(wT zqIxU}$gJY_(KWk$9wYOAbrsZ7UnMBFoj#Vz@wAPkU4SHdMjYw$AsS!$e24~-tl;xu z8bFeaM{bqqGM^9Y1duHA`DpO@Kuec&qD)IN2}U0p22c_=5UgWsHu^lqWHQQdR8M*u zTT5+~malo2PkPtvd%6fnrvQIQ@}?bW@*x^untX@`fvjNiVHyCEj0b=$Gx@Mi0LU_v zj|P(uv~)=)%CserVDh1103>;T4*>a`JU+!h`cDG#xjKAw-rJe&s6 zb#N3+Cb2OH)6=zz+_bLV0(q)-FIO*2e2}O8g|p!}d&5J2vUjD}!k-LGS})@0J~}ZP+qO;G zS#J&bGyJuAeeYL|h3Zbl096HNgjt|o+p55~ML1kTo!^8L?*;IuQ_v^g&Wj9vz@rd= z3~?{Y=j{1qCA&sdp5}`0_Azgzlj&Xko*1Vy! zl4t-+q5&*X0~6&&L$M?Y0LzjD9VIswj-jPXI#H&4@&{o(8YX8T82OqeUEnU(H;Zlb6NEptTBsI>M*8c&Qb<|jC z#u_yqOxq<*p`&ol2_CtQ0$vVob#H78&r1o9gExNxfu<0mtfH=~pLo}W= z>JSYwS;4BqG{7V=!$^0SDs$_wPNZF@EI~(sVFy~eq!VSfC6r*=pjaoAbMq)^wmF8DE*quXyM(A7i6Tf6YY*I5 z*6iEaBJ!`;z^tXfN>JwQJn3DfZs&F#x1sb3phP!_BMm)7<4H#kf6*Y66+Asm15grQ z4M16D>tUS$lx4;qC4D!?(9&h21SrA3L!xl;N*p}6c&*vO^VdW9OpQHfYTMakskUbo zJ%sfOs6^9^)2aR1%FVsVsB&umnJ5na4+g$p>1xq!VH%0HY5H zLpuRUjdp^Xy*`NLf6w<5JX62Vxyl=UoTgT`UGV>^w8^>D=ooRN&4*|_Y4af(;F4)1 zmrNtMWE#n(F0=Wt1d_|L1RV`FA86^4PH@{2!g@4J&X+_4Ygk>g&F3*Lf6BgPEk#yY z$LhS(N9(3%7&8v z11(+B2_X}lJ|qklvMe<$WOdC>pTCF9XKM91(|M~;x|nS{zFn|CqICh9Xd7{)*N134 z>GdHRM6-g|hiL#!q5(9^ygn=epjqbi*&KX86TCj86WqCkupSKqHi;n!Ce50?K0k`) zleR8vDYDX}f0;M?=yubzi*7S@yMRn|jX2WlLo}ZB`VbA0S;6bWG(aZN0GVZ8AC>^f zEc5zk@cKYYmvlnN1g{SX!%35*#!0hgug~8@<})?>oawyTCz~`qt&8q7tpYUBIpRpW z57Bti?n5+)W(B(s(*T-818A1neOLlOv&`R*5e2`LIp^ z$uggh2A>bKbV(=5v?P;Y^r2w@C2<46I<{t`&tps`qYOv&q^Gg9)K+Qvns@o6cg?=1 zi-2?re}E)!+L0z7qVc84hiDMU3ML<>0U*hE0LU_v59GKG}-gQ}RhqEBC^~X%JlpN5Nzg8-p-CU8~4V z>*_6#r)u|d^}@sldD>q%8;-L#JoG24*E=TTf7!JW|G9c`q?5fk(#(<#MyHEOU?kt? z!G0sSo2TDUR&2u<-i=nTj5OxaIE=>0_f_xNJ1ZBH{=qaHzFZD!F&qE62wuJzyl{43 z44hZL82Xj5Z=|K)OVjF>y<{KQunzX-{p;XUKe`^zVq+4F(n8Iq+c2WlODLD0lW@NL zB9#70`x+)mIKBVrCg=}?DEleBB^ZTaa{ohiN;WU^JoqoOiW86y1rL0Y3nr5vA|8Ls zPQx$|MfZF~l``B_x7C>@0hlTEq|8hBSYVo2F?iWyNxQ@0__aa=t#u zCi(^IgjNO0k`zUtnC3#&1$wVG@d`!GQ^jec6%?oght}m?&J1DN!mHH=tQX*-08)-H zQ=q-~hG6X29wc>1aHt^ew3XC{($;_2Q1(r0z_T>H#}Yi{)Kh!}#%3g<0m<2jG(WcK z0i0oIK!WlPXGw-HUJq8f$%bCJa!TPl1C!RXIJyr`jK;QYlXl))L;ehZZeQN|Rb!!! zr(%H0oHN2KP_J#7 Date: Tue, 15 Apr 2025 10:15:33 +0000 Subject: [PATCH 444/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10222 -> 10220 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index d3427b191..4bbb5b700 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 03:28:13,779 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 03:28:13,779 submission_checker1.py:3260 INFO] --- -[2025-04-15 03:28:13,779 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 10:15:28,376 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 10:15:28,376 submission_checker1.py:3260 INFO] --- +[2025-04-15 10:15:28,376 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 03:28:13,779 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 10:15:28,376 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 03:28:13,779 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 10:15:28,376 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 03:28:13,779 submission_checker1.py:3276 INFO] --- -[2025-04-15 03:28:13,779 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 03:28:13,779 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 03:28:13,779 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 03:28:13,779 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 03:28:13,779 submission_checker1.py:3297 INFO] --- -[2025-04-15 03:28:13,779 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 03:28:14,733 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 10:15:28,376 submission_checker1.py:3276 INFO] --- +[2025-04-15 10:15:28,376 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 10:15:28,376 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 10:15:28,376 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 10:15:28,376 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 10:15:28,376 submission_checker1.py:3297 INFO] --- +[2025-04-15 10:15:28,376 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 10:15:29,634 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.297575 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410951 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74612 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -31,8 +31,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.77649 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.486 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.342409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.6879 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.342409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7e1796776..e255a8555 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1071 +1072 diff --git a/summary.csv b/summary.csv index 72877ca95..e7a017caf 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.297575,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410951,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74612,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -9,8 +9,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.77649,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.486,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.342409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.6879,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.342409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 81d323e44..438be23aa 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.297575, + "Result":0.410951, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.486, + "Result":22.6879, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 7f8eecf34d26da6bf1387c1eb8d666d5e7e75f71..c7541783ce6e1a1e39554f3ccf5c17a33db9a8e1 100644 GIT binary patch delta 1739 zcmV;+1~mEZPwY>yatH~@`ndt_3IG5RvwH~41%Hstb{Ux@0gzdeptCvnK&EWzl1`MV zC7Cj;N5cS36ycQF{zs9ip0t*&qsB^8W;;iw)=kHAi`KGyQ~yxmUb*c_Fj3?B5=_(} zm_!3$5)FV!_V)oys|-w%0KhCs(AgY(08_SfNhbtM8P+3VIAxO5IAvP@2VmAwW2Gt6 zl7G)(EwhNs0uzctSAp$IGEw81l1$VfnM4C*5)F`v8hrR;myt;l0GTBTI-7$JWXhH< z=>)bwAgafr08C~J0x<1o4W`%-`Ee%eD6bNe_7;9SGE%}a>tktl*B@%F%-WzEK&Eh7Jl9n4w^tORA=&!gK--Og=0 zZbRu6K#6t`M>=|l#*>yFqCqGtn0lB7pd`kB8i2CQ*TXsiD9fBZN*ZsDp{2`42~dK8 zheYAxl~{Oi@mjNo=dXwInL2yU)V8(9Qf<#FTE6-PRHAFdk>(zv@ua(lXpqVZ{vM_Q zDv8FGRF+wMSOTE3%;clMhnw2U~?=R-7}^!X4CaLF{1OQw-rGL7U?m-&2H0?B1rf{q5C543bi zC%A73VLciq=Sw1jHLR}L=kpksKV{>xmLjXHV|Cu@qjl4>atq9DGF1VYXc}>()rV+2 zY4sr*B(s9mhiQOJq5(3?tUfFOkXdGb_0eGUftD`mgpdhV9})%&S(X|Wvbtug&)-Al zGxhqM>Acq`UCg!}-!9l6(YgRl^o=;u>_ar3H2V+@qFKT0!!&><(EyrdW*?RS&@40i zYz{u431%PC3GQ4%SdWGQo5T?WlV;6kpC3i@Nn4k-6j^D~%)5PbyJ^}*x0$+sT|g$< zMjUDOAsSDbeTW9htYG$G8X%KsfXp(p4@&@KmYIDtn0=t7OFAKBg4u_J;iO4Y40W{0(J}d#C zS!VaqVE2KRF6jhJ1rXI^Q9x#Y6}OMB+3oWfng6S+pqBb7LAmYpu~d$yZ6xgiB+)bC zNS_bU_|oS?G>BvcpAXXjl4Lw`t3;Rid{`%dWSP%LgU<(Ax}+0jT9QdH`p__dlDL6j z9b2=}=P@ReQHG;>($m;lYOAz-&AWWkyJp|hML;?QK$2JONRtoI_|oKmLo^6v1(Ofc z0FY!n0A!iThjju#mYIAsn0%n6OFB`eErA4+4-Eq#$qRe{$mitoDF)Ji5|Gc;;iK~o zpKL|$Dfy(Qm3!miG>EQ)qhK4aeCl9{Q8j z`yG?I97f~h z`>OZsrIm|G|6m#pUoHo=n2rBj1TS9L<65{$wy zx&I+MC7ZW-9{d-xiW86y1fY#my^|gyAb(E7Fc5_AJVoSt;y5J;tR$5H^+Y8kia^{f zdz)IsKe7#J-ySz<(^l$<<9cSlon5(_Z&jE4g7ebo8j&nV321JF)NPI4SBvx>B|b1M zSZ6fUXa^q6o12PT%8i3pXDqlN!6y-|AWn-DC;*z*l2=CkO~QCu6RScs)-v43U! zo3((8Jio&V0uwASya%RsBBBFQ@Cj*uTsHs&$IyWS9Xuu(!6$D3r~K81K{<0O8Fm(~ zT94x7-g-G1`@YZm*J^@~n#U~>M5G4Qr007VRHy8i_ delta 1758 zcmV<41|j+EPwr2!atH}F3T_|p3IG5QvwH~41%GfQ+b$!MBmgo?5_C2PAIOv~UDAm% zwIox9^=KHNi6Wdb+y5vs)sxn;b<|jC%53Mz)a|C_6+G-wd{6&S;a0irNiI?2`jSi3 zAeTe~ToMg%Nw)WaORJ1ak^s0YNzmCGeBe^HbV(=e`;tu5c%~#1HAp7W0GUJsWTFPiw9CjO34qL!1f9*n2Qp<# zmvjOnoUfJtJ8k?t^6=GI}INV`s1f*Otj!w$4`Nhiu|ODMs#L&JbdB9czL6IG{i zlU@%Se-F>8%A0sp)3qE`;NekQCLZ&FNwk1C(#1nGp0x204Ki85$ipT$uc*Ol4hG@Xz8+1%Dqd7>X9gdB(e6ujb+Wgoh>5&iVe(K3akWW-p-TWRqA$b z*Kr$4uK-GPgE-RALo}Xr^bid~S;5o8Gyo;>f7Jk#WwsvH2|!t9>`~Hpa||tAHcEgJ z{5vEH7q7&@gNxUiEj)icl+V=IbEdYPJ(g;FR?$OPzko_KjX2WVLo}W=_Ye(IS;5}J zG(aWMxRS~;hYw2tRF-*s6qtOVrAs;?b^x!m36GnJAJfndRA_MxlN`jAQL?!j&%AE zjVGNxM1y2jaQZL}kV!N^W|`B6B>*zZf1ExVoIcRfC7lp5!RbT7U?Iy=!$MZq?DYA2 z$b6<&pEI4e`lO55w&U9c`y*Nxpoz8-M|ypT#*5-5)F`9=JjC-fXp(lj|Q&~v~)=)giP@IkT9GyNot%l zYxes5J!C#pv(K5%n|-oL)6=@>PSYwt6P+WDwEGZ^C+$8&gJ@Q;`!EflNi=|Fncasa z05r?&J{s&k(9$KHfT;kYdMpaaf2`v6(KWk$9wYOAbrsZ7UnMBFoj#Vz@wAPkU4SHd zMjYw$AsS!$e24~-tl;xu8bFeaM{bqqGM^9Y1duHA`DpO@Kuec&qD)IN2}U0p22c_= z5UgWsHu^lqWHQQdR8M*uTT5+~malo2PkPtvd%6fnrvOOurX6YWAsSzre|(4rfvjNi zVHyCEj0b=$Gx@Mi0LU_vj|P(uv~)=)%CserVDh1103>;T4*>a`JU+!h`cDG#xjKAw z-rJe&s6b#N3+Cb2OH)6=zz+_bLV0(q)-FIO*2e2}O8g|p!}d&5J2 zvU;~M>^SyeTtl7TgcI)t z@TXJIC*IDB41K_(5P%GEFUsfa`DME#Mpd5Xi#VOf*%Ri!dC0$s z7r>DF{u5MxvtA?00tq$>ZXfUp000n^F(y|5ij#>ZJOUIIlP(n&liemN0lAY1CprNu zlS3ya8>-h}5G4Ts08jz|01*HH00000000000DyrElY}Qe0brBECnE+DB>(^b0478= A8~^|S diff --git a/summary_results.json b/summary_results.json index c1bd91854..5ad6ec80e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -365,7 +365,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.486, + "Performance_Result": 22.6879, "Performance_Units": "Samples/s" }, { @@ -401,7 +401,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.297575, + "Performance_Result": 0.410951, "Performance_Units": "Samples/s" }, { From ea62a1d47a36ba55c81dbd2c17fc473d53a730d1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 15 Apr 2025 10:17:18 +0000 Subject: [PATCH 445/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10220 -> 10220 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4bbb5b700..90efb8f2f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 10:15:28,376 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 10:15:28,376 submission_checker1.py:3260 INFO] --- -[2025-04-15 10:15:28,376 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 10:17:13,243 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 10:17:13,243 submission_checker1.py:3260 INFO] --- +[2025-04-15 10:17:13,243 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 10:15:28,376 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 10:17:13,243 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 10:15:28,376 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 10:17:13,243 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 10:15:28,376 submission_checker1.py:3276 INFO] --- -[2025-04-15 10:15:28,376 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 10:15:28,376 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 10:15:28,376 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 10:15:28,376 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 10:15:28,376 submission_checker1.py:3297 INFO] --- -[2025-04-15 10:15:28,376 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 10:15:29,634 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 10:17:13,244 submission_checker1.py:3276 INFO] --- +[2025-04-15 10:17:13,244 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 10:17:13,244 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 10:17:13,244 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 10:17:13,244 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 10:17:13,244 submission_checker1.py:3297 INFO] --- +[2025-04-15 10:17:13,244 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 10:17:14,217 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410951 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74612 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72939 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418802 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21557 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index e255a8555..bba0bc80a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1072 +1073 diff --git a/summary.csv b/summary.csv index e7a017caf..10c358b94 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410951,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74612,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72939,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418802,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21557,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 438be23aa..83c287e5f 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74612, + "Result":1.72939, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c7541783ce6e1a1e39554f3ccf5c17a33db9a8e1..9d225a8b267b92c54343dcfd06c1ff742d563312 100644 GIT binary patch delta 1377 zcmV-n1)losPwY>yatH+5i4*a&b_fdxf7O1wXrA11@t6-xq6fs0HXfq!q>qPakjV;8 z9;N{%iJL|`2vnJwhjju>mf3lfblV(5OP7sOZeBuEk3|oYX zU?nK?ejeR!>UM71aT`jn07|rjIMUHWG@i8d5Dh|E!PLVv03|Wj0F-6E9@Ys!e_7`2 zQPOyG3@u$YN`Mj!JR}Miuf)QGi`SYxJbyiu&(zs-rnap;mTG%e(el+Vpb}jpjx_fW zjVIkbM1xdT@b@qcP)Rhdq_WK7!x8|MWhNg5E+1&=l1_-90GvJ~4E+QoHTnr^Hv1r! zKi^RBObtKhD)0Dlnp)X*!GEmMe&4Tr!R1l4&HDy3FUp z5=bt~5_B~9e4wREI>CKQ2IbRYHtYLM{KA*?9{3#okwG>%p9jo(JAFZ37m0MtL zlc@^GMAL{Ptv*EKNvjXhAej}cK1>5-5)F`9X7ynSfXp(hj|Qs`v~)=)e}qi1`j9YK z$g0-9+_;$f_AFT_}MBj)b%|1lqNwW{pAet4-K1>5> z5)GhPX7*tT0L?P9&*tC*nqc-Jo#4(Tg!O0`ut^+2Flp9o_W4mXpR{#ZOOcf(&Ai)3 zx0|M2bepN$1!SUa#F1tnf1>fE*@tM5%nD{7rU5dE2FNTk`>+H+W|`SXgV_gKx}+0A zCYXIl7*3ibHBOo}n|=NsGM}m2=S=6_KG~${X5@*sQ~*&u76oKhar@|+-9C?zfBC<<3TmmZ5|rCc zA4}zU+D6hYKoUJ8j`aBujW2yZM1x3H@cA$eAW6m}w@P%G&xdsaNS670H28d=rAsg*0D7keI8>n8D%)CCq0d=rM61T*SyOoy=(S8T?C|403>km@X1!>o{~>`TDdnKPJ`$=I0`0{*cgQA=~_i@T34@uJXO1wt2ZV- z$kYD9*>Ie_;-NoTz27kz&#sO5&()hFo$Sq#W|nL)I$cZxe);DxjMV&J^`#n7*eeIqUX zUYb_7>?QlahIOzv?_UR>`qA}x78{dbloo0>-G&jZ-a@(joP_h`r}S6a*Dy)K>HSYP zL4O!T*-z;$55Xu5llvdCQ?hxR=fQsgvxpOr4g`sopT(0LA}fEmrGTs?l>qfbB_x7C z+$?*WTEsuH4QbyVH)+#W%8BE8X1|?XxmfO0oBV?F(&!SAEJq1wZiLiLiQel~`hXH2 zm=>%x8cMVWkCxR>V|vpm1U3IcxfL*vToj6V}61^*Dr7Vs;Sh6Q*l65Avom@STL?C@MRGX*D@d0@x&Vm z;&C7KiMQh-Lm!AF2B1USlkzctdahTfn&q=an%|~mQIi>*l6y+#Ux$%b{b!&$4*56n j1Q>E(e}d`{vtA?00tDNM6Y;YwCOrWJiI$(mlWiw|tbmRM delta 1377 zcmV-n1)losPwY>yatH*;`ndtKb_fdxf3+M{VB%3*E*|rNN%Vj?(#AtHp7ik$4Ki85 z$-^|jByrP72Z1Ux^RP~U$uc{Sl5U%0Xz8+1%FRoN>X9gdB=PpZon_6&oh>5&iXF^a z3akWW-p`}kP2J9IJ8nbi6+nr05Jx(Ch{lta9-=`gE0}ti2B0Ly8i2CQ*TXsie<;hG zJxUsHj-jQ?MhQ@Yfrmul;+0r3jQ9Z0V;{cl~k5ld{_dYvdrY8z~uuiUD65h6M)l)grT2+q((nM&1N6O z^5+{0o~hyIT;&}Z5hjvvLc} zZ8B8>nP?huq}7LLJZbeI8YHuV)rV<-OrilY%d9>u0gzc{_0eGUftD`me}s?;Rv!`u z3t5&L7P7i#tIyv<<}>yBoawyRCtb|89p5h4AJMu1P4ta8((FSto;3Rq4We1W?87vG zCeZ+zWo93i0MINm`)m$Apb2Il(h2TdLRgQ60h`1T1e0dXW}hEL^GREmwG>%t(#*Sk zbh~NVMYox{T|g${=?UPNKp4LTonpOdt=p1pR-G^vAY4;%-M6-h3 zhiL#!q5(9^>^>|3pjl@3(O~z1mM-Z8Oa&0tV^Kh66}OMB+3oWff0_TQtDu(pDnYsJ z^s!Wqr)?zd0wmEh;z*wl(fHEmLo|qF1)mSo0Fq=pa;rp_`FvO>fMl7^M}yA?TDqhY zWm=L+F#6CifRea@U>#et(dRKHlTn7FdeYO_T57Abe9gOj(z|Bg(?vi!1wfKl?MRak z(fHEjLo^6v1(Ofce*lnVJOE^w$%l0UK$e+&G?;v#rAsD<1lj)%zWj@$A}&|6IK}(#hT&X=cd=qtnGCe=w5o^I*Rb+|AQ(C@Z#M z4DUv(cSaiXXdFi4MfMZ&q+96eoB9(eGQW&oZkO* z6ZD5cl>L<64-$;RFuDIBJ0+X9c^>>1vxpOr4g{c$Q@xWMA}fD5B?zn}l>qfbB_xVK z+$?*WTEsuH4QbyVH)+#W>WSleX1|?XxtecPm;8eB(&!qIEJq1wZiLirjow#_^d2QX zFfCYTG}LGZ9?hGZid)K!gI8xPxFEqN5u`okwnm!}EXCOK4JhWb=+IGI8>d)^rE9Te z{F}9ai#)%>3IcxmEe>+V8OVi!skUiT*rJ^#uKk4 zh{s*jue===8T&vaF#sLo9+i*T)AMS9nj$aCG`~$rxgr@kB_%B$zV;(8`cFW081ir8 j5isVy`~=M(vtA?00tCtWxdF2*COrWJpp8?#lWiw|cD$Z_ diff --git a/summary_results.json b/summary_results.json index 5ad6ec80e..fe8ff9233 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74612, + "Performance_Result": 1.72939, "Performance_Units": "Samples/s" }, { From eed22dd2272718279a36624631221ef4326491e7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 15 Apr 2025 10:22:39 +0000 Subject: [PATCH 446/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10220 -> 10218 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 90efb8f2f..70f5040b3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 10:17:13,243 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 10:17:13,243 submission_checker1.py:3260 INFO] --- -[2025-04-15 10:17:13,243 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 10:22:34,292 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 10:22:34,292 submission_checker1.py:3260 INFO] --- +[2025-04-15 10:22:34,292 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 10:17:13,243 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 10:22:34,292 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 10:17:13,243 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 10:22:34,292 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 10:17:13,244 submission_checker1.py:3276 INFO] --- -[2025-04-15 10:17:13,244 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 10:17:13,244 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 10:17:13,244 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 10:17:13,244 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 10:17:13,244 submission_checker1.py:3297 INFO] --- -[2025-04-15 10:17:13,244 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 10:17:14,217 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 10:22:34,292 submission_checker1.py:3276 INFO] --- +[2025-04-15 10:22:34,292 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 10:22:34,292 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 10:22:34,292 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 10:22:34,292 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 10:22:34,292 submission_checker1.py:3297 INFO] --- +[2025-04-15 10:22:34,292 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 10:22:35,289 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72939 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418802 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21557 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index bba0bc80a..cf635ffb9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1073 +1074 diff --git a/summary.csv b/summary.csv index 10c358b94..f9bfd6f27 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72939,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418802,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21557,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 83c287e5f..cfc2634c5 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418802, + "Result":0.421964, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9d225a8b267b92c54343dcfd06c1ff742d563312..87e2ff4beb564fa3d98ef44d8304f9f4690bcdc7 100644 GIT binary patch delta 3197 zcmYM1c{~&TAIE3pNNiT9rHBoYEbY5o4Vh!^D=cz^95p%1J(Vwpw3+6bqag|vll#b# zLQ_U=lH==4&N0#Vr+&Z39-+OE z7c3+2p0jNLOmck%$E|fHX$G#7CI?39!v!N(s9}11%@d|_9eD7&X}`Pn*1?RMJPfBH z$xpFyZ%VrFg*!3hDJ!L4vtt;Oui2O5Z#AFZpPO_zzAy6lfz$;S4`b&w$|!{sl`pcb z-%@#Uuq_l}{P;+#*N7=gGT&7uhhU&n-Fzg|3_60$?x@tr`C@y5ov)(S+Tt=MI1$tM z4w()wfmtFJWEWPadqYkSCJXOQZ`#{F7CH2=ZKQlei({(srlYr|;>VKzs?x;nZTG?m zp*F1wNi-XD>!vI|nmrzH+>_i3;vIhnQv)?e@^`dZrsSYB1Q-6Mgm9`a5Ka}%Sq&z6 zZ?C)_C_VADW8Xa$j?$lw()wT)#GF@}g3*3esTT|g*goyTy;rct)A#EtObkYcx7Q*e zrEbs?r!<*}KGQk_KFb|uJoR$>hA*3099h=a$LhY;?9Ln>f-c2w{N$A>lhFvTHef#N zz*9rYYDj0&D!v(^9Pj)^K(f|u^A>XQkV!3Lr`psM&N&>R7}BnH!pSxDDT>l-cj1GW zsXH)@SyX7PsbL2f7EsFY5$!=bE|?q_j#%sCM~AA z1cl+ImOoBPi)=N)A(lThQg^1tnw!%|&he6lpeZQF9z0Q}byrI#yDJuTB6oh0lCgyM#b!6`mj~_!)E-ah`|{b3mL%r9)aW;oZpLaZS}>B^M}p`r z%n%emJ*ZJ^45mXHYfpUD=TB3LoXT>j_eG`1^HUQ`^{c2p5k9%4)A>?nX$XI}M(XgW zd(#4QF1(R*Lz%{0p}#wa6}Jw*^~F?~Ce+b3qWmU`Kni+KsbMeXU6AVAN>ea05--zH zubkCVo368eWL!o)9&D`mwOlp$d{K|yV*4956#lY?Wo`lN7aiwXX3iz3Yp~D7+c{J$93X@#kvPCn_Q| zbOcC-uoR3m83rN3dKIO}F~$(VR)^!lu&_h08>*yvECidBZlQG$@Rh)S_QMTJ>8TKy zQ@94@Mjz|t6UMPEwFER>UMlPn#7;Yi@BpGsHiRW$q%AQ-W%aNh;(3Oxv!>=cA>HD6 zw1b^3XrI^fFyV7cJv&G*E}wan=&XeA^HomTmL2zq)%e62?q(@;De6qtHtXd%-^+Z4 z-_6pJ(DMioOZyJgUsGz9h1zKcTtP<5^ckMc&O*@?8xpcv3p)JY@uCi_nwL zEZJ;%YqN*ofite_Pp3Vp*U)J7QH9CSw;aUTn=b5!E;&vlONp_u7j!~w84dSwp^tgVQ(;^#I8t8W zOh@b4%1W$USE~6hsZ!lG!b}ntgYfVbU6m*w%T&I2{teE~9Kc9`cAL;Wm3nQZkcJms{c-3wQq$`?=yssB_a(f+EUEJz_OYlh0X^~(M!9VfmSWN`m4V@Q zhX|2~P#6P;k;MYs#b91|VKnKi!$`qV40FDD`_a#)hRA`$rx~1^<>vQM^X{B`P0cX4nJl39wB#)to>9DJf|d0l9k5i0ydN+^=)Lm94={n~ zj}Lgtm@RxN9>!TgKnE%C7gtMwEaMPYI-9|3eLH?nm2x%JE&&@Jhc(V<%C1ggt}Vz{ zMys-lb&)wii(nxYTq~iZ4fz)4-)0z|u28Y`+N{92NM6VIz8l}Q-dUJ zj9AULa!wJovJB?fExXuSxtx`+-q)$s})bz>nl2(HL$=l}9rv z-mo+inc99mv?NtK0GFCq*fPhhgl7q?I1sMb$HjJXQsP~NN)*`~&5l{~;hNHd4=PLc zjrgtJx)q#*;K%DKSA{dYp#BA4;=DAyx6T~&Gv-U2cl7T-2)ZHlg^i}tti(9*n^Y~Zhb?JEZrtFO|gCz-M@1-e22B?=1!dliv z#@`fR9K@K3;r0^Ch_nZ4Gd?Uc)OMm}(<2S#Y61mZPeFNaLkz$snW9;dQOB5a`0KV- z?$5D-qu=whg8O>vnf*)&m0r{zJL%sfSBL1{Kxf~qaV?*BXXuGP&p=7{-iHz@{+?Ay3AdQD2(|JF`pa;TM+8}^T zy}Ob^cO|{NDQ0hN0G6l34!KkPf6gb2qaTww&2@(rp1(SlT5l$4j9;rvEXaNc3)7ca z*f;}q-i?UoC7=7Y;R@AGNiy(#GEb1s1ugNA&aHF2Ryo=ScSJWaRuRybgz{QIE`dwk z6j8gH9quY0{A|p_pbl7kA@$Yh`P|Ky!`nfpNoK;NnX4MC8C_%(k^>e3;#z0gkO0^T z5CG@C?wSoC>B&k8$^Ma<>TN95GNaoDFkA;p0@l?+Bi8S4=JL!C0)uzv@p0x=(%Y#K ztiGYJ>2JvR*cL1zqg`Lw#C%X8psy}WXr^|g*{SiunobT|yk~cMSrjzhxwxO)8G=Gi zY1!RL4yw3hkcq39n2fj}kTmmCQ?>Nay^YP2T?u6;_Ul^~v&kKr(70YzPSfJ$aPQe0 ztjn3P`)S=eN;jvSwZvPx#T&(YekJB?EQ;$ks|Y`;+zj`=HeQjo%zbj8x1~Yj2s$|b zTZq_6iVI21XJ9;2q-^3!?H(;=W4371Ze$RFZC%(m&(9kL(U*%hsV?By&VA7E+2ID} z#(Xm){wPV9#~01*M3In|By{_{RDvaK7AwX6*=`o_o27+|Q@Zkw5O zH2t779WOgWh}F8Jd}sPaxka4y1?}L@6Ru*#gH`9(f<=S}E$1$MYj@z<^XK3FIO5%$Swy>Sk;TTyaz2KPp>#XsE=?02EIDjr%sIj>Ioysp z))-PG=avu>$~oi^_rv`>uh;Wj|6Z@x`}(}UpYK1{x2yA4%L9b?N2ro3i;O>nu23S) zzzRk{{EwvsKIzydrbip>{AyRHVYs93(xUoWo#QvDh20RHOzB#KqY16t2Zq|+Q4(DO z6Tj;ZfpOv?b7JM?ZmA4ur z2-69mzNFnu9{D<5bLt^5oqPH5k9k7dR^FpnW2qG4qY@K!mm>ZZ2dhUS*R8hyocu*- z<3IBy#esRF_*%M62x;n%Go=&57{%tFN{SM&c`!}LbsV5ptKTz*h8I$PVifOg>q93+ zwz1I3fj?p}WJ-r>@8oy9?;JEL0nL)bCw0 z#?9qwW20++6}mck75f(r0(su2d(LYYzMCGX!)Cr1a>?UJ*=|digdr7a0}|Ay@mP-3 z_L%yU_{iAnV-VAp-=r5~m#4@tbWNQ_YdJr+I+L+<>E3CtBvnh2`6n4#V{_d%v5Fm5 z=Gq1^*=L4=94eOKOx;SGfuw%>z#szU~^xD z9Xy^Y_k<4%1#pTqEI`H3XOk1LEy`5o2{}jCaJrs0^ zV8Rb*G?n1~y5&85#_rMf$tw`TQSl}t>RE~c^0IKwKJSVmO*NNuT(0S-sReXmwV)8% zICa<}o@kg7q>z$((Ak?LU&4UhYI|E*N{u%~y!|)E9-=jK=GQ#YTW;D~VR@iOVpb=! z#yfrQtC%r?mC;6Q2qTc+mQmZK5>HBiKfl(Z8yC>@fjEuT#&MD~UG1Sqg&M`cEAV$i z<*FbY>`hX?LgYF&l{~x2%4a4dm3w?a~RfYCnR#6LQ2ihqzV;uIq~#Jy!}LG`4@E;gz%=`8L%TxI*H*+vQavy z;nOc-t;pZ!EGGQo7qD=`R6T$nP)Nlbas|L)R2e?Yl!rI~l&UO@Opg$AeX~!G3_IN# z0u{}ARxuEuF%7kS_40TZ(&zbV&K*>Fw^{yZjhwZ<%D6Jhit=KM)Ii5q7|h`iga{7+ zoQTuXSMf>g3O>Pt@+8{Rb1E-x>KELcqnjrcDYFpIjaxxxM<j4u1=}4iRqTpYEskxT#mhbn!|)x@`M!!4yH5V$t`FS(rly38^yV%^NAS zePz~vlsyiV`D*kPo^|Ac@ScZx;R{~if)C*y+0MHTU2|1d{qaD4)bCrc--y4gNg2qbDnhrf*H9~~YE2w`*`7Vnss&Qk z90DZ~o8^au_3gjT6Z-B3imE$bFbM-h^`Iih%Q!6&vBPO7JqjwRXMh`6P*Z|p`rNna zU8M|a zd{FIeVV?^-3d~F67Rz|>rI|BJoBnAI++a54d7sM6wLBt+hKB5W_e7JzObcjEh`29w zZ)i$=*GwybJ@*y}(%K~FtEB;{v zDM`;tc69Y)6p6gX^v5WbpfOXpVzFKYmeg}J{-15I_gr@q4(mb`CkPaa9h4S{FW*jt z+r6$BC{G;Jb`0HMQ-zi6~lcdXWCk^15C|24$ z4XSNUNz+dMPr{M?grjKg(>ZVFph9n{9O^p>+?TlbZu7TFs522|7@1ZjKm0gSb zrKAyqA>iPDN_k}kBA|}(d)WBz%%+W^6KGp6>qUHu%7CO1bBO8m-H%3n=kx)TQvBMM zc$`Kk8WVxFS>A$!6>#lJwI-AxR?q$>Vwqdy|5idMP-gMviGdHPQe}mP{?e8WmA`>F zLDYQ)L&cTk*O}{*M3N!$s*gqKpw14 zww&b9$fe~@n=e`Pt04|`I`ZZbvq)VL z6~+hY$hkR(D}D~PWe@eN8&ZmORM4;c_gc8WXEaSdxn$oou6d%TG;XtJFkLBgzMJOB zya`7-<9U^~&|(|1>`}els0w%4_GQ1S5U;123GHrL=FMdxJuVJ)scT{d6U);97O`Bn z&7LdEt8JhERk`E-xpBMgec9tsCG}mGG@IIa<(*eicgl5^t_x;w_paYPCd+V5I_7g> zG)kgm>~h_&jL09;g=@AShvB%Ex!tQe1{7%w_v)NFTUC^?1F5jAxR*Y|rtnloNhTiQ zqcXGG7&3|wO1F6{Ts(3~@5-0^lP~haolkHdxc(=#`{@pxOBG=6ad{o@ew#Pc zsSwYXna*`~i&p8s!hiJ50yH&aVUl0ufEjh{yY+S$07*UeTwBJZN-%*rXvS8bZf6&t zKjxJ9nIY7d72ueAONi+{zjf^B5Bs25RlmoUYMef8$_K<=x6G?by_c;bypn2gGN>RJ znyYkeS;Cv*G;)Zq<9#@!AF@uZol-P}#6Sgd3_mee zH7OybP}@J#rYowY?zXO JxbnfxzX7jt`hx%f diff --git a/summary_results.json b/summary_results.json index fe8ff9233..b37743924 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418802, + "Performance_Result": 0.421964, "Performance_Units": "Samples/s" }, { From fdf11ad93c7d6b023f37d624792bfa3e8be55c77 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 15 Apr 2025 13:08:47 +0000 Subject: [PATCH 447/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10218 -> 10220 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 70f5040b3..b44133454 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 10:22:34,292 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 10:22:34,292 submission_checker1.py:3260 INFO] --- -[2025-04-15 10:22:34,292 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 13:08:41,540 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 13:08:41,540 submission_checker1.py:3260 INFO] --- +[2025-04-15 13:08:41,540 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 10:22:34,292 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 13:08:41,540 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 10:22:34,292 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 13:08:41,540 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 10:22:34,292 submission_checker1.py:3276 INFO] --- -[2025-04-15 10:22:34,292 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 10:22:34,292 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 10:22:34,292 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 10:22:34,292 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 10:22:34,292 submission_checker1.py:3297 INFO] --- -[2025-04-15 10:22:34,292 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 10:22:35,289 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 13:08:41,540 submission_checker1.py:3276 INFO] --- +[2025-04-15 13:08:41,540 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 13:08:41,540 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 13:08:41,540 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 13:08:41,541 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 13:08:41,541 submission_checker1.py:3297 INFO] --- +[2025-04-15 13:08:41,541 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 13:08:42,694 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,13 +31,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.77649 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.6879 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.342409 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.6879 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.372039 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.4168 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407871 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cf635ffb9..695597613 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1074 +1075 diff --git a/summary.csv b/summary.csv index f9bfd6f27..8be42e68e 100644 --- a/summary.csv +++ b/summary.csv @@ -9,13 +9,13 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.77649,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.6879,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.342409,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.6879,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.372039,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.4168,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407871,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cfc2634c5..2764e908e 100644 --- a/summary.json +++ b/summary.json @@ -304,7 +304,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.342409, + "Result":0.372039, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.4168, + "Result":0.407871, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 87e2ff4beb564fa3d98ef44d8304f9f4690bcdc7..ff635ed43574dcac15418435b3f22e5372c2424c 100644 GIT binary patch delta 1775 zcmVyatH~(P%-iD3IG5TvwH~31%F)0HjBt434qLk1g*`%2Qp<#n{=W~ zEyk~1QRu`FTq3& zf=M(0CeZ+xWPcyPw2Hta2>{H31g*`%2QXzzn{-0JlwlnbhEpa!ee*k6`HI|w( zt$*Dq)754(kIWnsib7X`?MpII3(=H;DBmgoC60|l4AIOv~ zZPE#Bfk0G;MFE)176f40&l*gzA@bu)R#9FlChZ+2EoJ(4-ZD1Fr=!pU;z*+o(Rk9S zLo~=_39k;*0F%TGBkf_T$gaaWk$#<`1b;0BjvZ)elTMV`lu&|ehlT-_L?o?vXR1o) zPDsN8liTTXzRV^{<=6ESpnRq#o^!SB;!$;1`?>Svj*G{9U=lqbjFiG4r(m|k#%si|UV6w>0qomvB6k6JBlydVDqBf$$b`VE8dWgo8mL8%(C`*`nm68;{h0e>oq#+6hSS$tRmpt8v1qrl|@Ep5^X@e_d4hlHV@ zfTTu0LB(bt#Pa7G3ZALq=TdDuew?~iww?1ItF+0v)My!Tq|b+FJn8cx8sL&?B$rGh zxnvs2r7rUMumqCJq694sJ|AdllTL8o62dw(OwN}?1Z!AbvCro*E`Q3#Wq&0_mRZN@ zZL5#gbzu}h$F2&MB_=T578i*C9FP717s2nkXdB)VF`fDBCC%Es}Hoa zNhgF%u=GM}l}=R$9LebU8j)A8+`=RR8JpozW_N1A_a-iol6Mo&@f<=ID%l(tk~@Hqi8;9 z>#~v}OHG>FZXeyOn|9u9rf%ktiMA0(nth1IlV%^HK{892eV7KwBpM*I$n3)s0GUN* z9}Q+7Xlav92$^8^Az?UalGHe9R&4h9d&qpIZl4Ri?e@tgO;78*JAX|p2TgR2IMVJz zG@i8k5DlVP!tTQ~fF{uZnniXWmH^N!vioST`#?*ZbONRVi0ZH?AhV3yM_26jd5p~e z)m2bQeWjq>b^2JQ>$!RJrJVyMx<(x7^dTBwI(>)+p)BF_VH$vvj7M&k=pv^N>ja=I za{6d+`anyYbfQd4Qhy0n9~uT&5Q(d?5WN0r^}#K6=~ZldZ`;C7<;)w%&O-4S%9xa1u-=vC#|D)3u7+ zw60zRd9L=aSMN-Gkf;5Hv;H`H$wPOtdc$Kfo(+xo&(*slo$TF_W|pisIA2TxBl$iL z4r{^PJpG2UVrxeKZm@c5q%n`iVKh#@uX@j3Te+BYkEUV&<#JGq+4#>z@bX3Pg|q*n z=e+vG(65X`BTOy*UYb_7>?QlahIMo>?+$}c-Do(T#l|EUq=lMww`N4E_fRfBC*geg zDgBl9HB6FldjHcX==Oss`zgIC7=&SR|3h|4Ht+L1`Y*GF6OaxB@K`{ElNur)f6Z#c zFc5_AeF~$yE!#;-5Viwt$*F-tQ(AHp(#8?Bevs5UdHc$C;w0cxkE@yec6L#*I?5*c z1?z-XCCZW%MWC4GLe(Yu*skLxikzp4(?%;O(Fq({RSyL-glP+JRvWNhfQtf1Il@ed z4&EDrv1132)Fr{8g1FaKQXfiNe`7=0H?0BZY5IsIc*?1#xC6#!B%(8tvk_^2Y|{fc z!_a^PjD}!k-LGS})@0J~}ZP+qO;GS#J&bGyJuAeeYL|h3Zbl z096HNgjt|o+p55~ML1kTo!^8L?*;IuQ_v^g&Wj9vz@rd=3~?{Y=j{1qCA&sdp5}`< zoyXZ?o6ShNB=daNjlAo>0_Azgzlj&Xko*1@du1}8cJDU(4bCL8crK!ha$002+|000pH0000000000004l23X_5- RJ^@~n!6zdI5G4Qr004~KM(qFq delta 1778 zcmYM#dpy(o9|!R7Mq|o0WKQEWG@DD8H9M%BkxPu_9u|p%GQTtB)^Tm>_-!uHm@v28 zid>E|2_ct`EYaEAZpBeAvIMv4`D{mBmx%>RWqRxmg4)%(ZZrwmu=TVDF4 zFKu{C-EqEQvOfGj)aFNwitANlYnaT^z-TDeA3-Su;o~7Kn9WV#;RVSZ)AL_5X3qrg zh-`cgf$X0{sQgC)zsJ$|j8Bi#xfWNFq2>kUEo&Ow(nSgx>(Ab?2=S0kGg;Eslf+I8 zH0I51$;$3lDT1$cb_m!_718fy4`6xwK$&Yih~uwuTLzp9X)X!mNEX?e+noFCPWR%h zW=)KNu+)^87cz&CB~aVS+leihz;?^XOzrCV$2P^6OEk|~Gj4fHUECd6Pb2NaNP(BJ zh8$Vv9k)za&vB^#nq|XG{3#vGpSsbIbAlJ|AP(J@)V%*F{*ycXd6?69|4rR%d1HZ# znQ%&>)XNmT!-b2*cUgXHzP!!hhahZx_PNv2PAObgD42pA4o?di%`vggPI3tFFmYm; z$zkH3Ni8sr>gX+q5MmdRr?Z2nmgcinLIA_*5wGLvv8 zBu%B#Fd391^?l|fbh>d6H7)-^>nDk0bRl%X&BT?({h#jYhWiY?%C;ese!4k#dCX#7g(%SfXvy16QDjPt zy8`9yq?lzU;fLf#yMh_UyiO7C)YlknX0#=)E*R?dBMS1Xyaj@irI;Ok^I$iQe#zOv z=Mg!0^m9>mXm8K+?!ImnfGY@9S)JWI>{t%O0F&X4brvlxkv>)_l#7oH_r4@GA}{=%Sk8e3?W2eQm> zpk;5M{b(GU7AL^|N8~=A^uW!TgmKc{6!D7&8SO_^2h*P0psnePHHpQ!zhffKsmv~` zp)PMk#X(YzPAq$&j8pGe1l*fZzYm|6WFK7;`+X4`?|h>|iZubsIEO|-?Bw+k72Y~} z8-^V|gwE?uwh<^dGW|h%)#&m2D}M~Hha6_xz}Qn4&IqSWiOobYLKaMIn`$Qlm_u-Y z>a*mP3$V?xXj$xDn`ty_)z&G~cEEB8j0ToYg-0#jTDdPd6&(6?hR(9BRbNkw67~*7 zOimEvn2c63p8N9L35xBYc2I9agzQxP$P16AlZ$8b)|7fSCPj+yasJ$PGCxd@_|eGa zT5?FWzC{+b`rU`9lh8X;n}!Az`+i$qQGT6Jd1(82>s)S1hark3Fc3G-{TxY~{zdq6 z7IQnJ>+G>BlSWFdT}n+#J>L@ZmgkgAUl3qPH7iz;w2R}_86pYg0YPiynFFNIqKPo1 zGRKpR^dA_{lB;}YQoofGyF6X8;xaObC%4UR+ZN@I!V5(uE9ssHCVv|~tTo)&y*%@R z7pK66Nd}N4JUA-qXhxTRJ04-azAaPKbj&E#A@;MeTeW|_T}JvsEHTimQY!W0E5_wN z$vKjZb3q}0=hW+#x_|an+P)oXQZD05PEfa_1Q(3RH0jTO1)29a{z)rQbn*jO&a0RB z2fRkw%KIS|N*@XVhh)5CLGC)!Q#1s_gp#k9ss=p0*T)jpGKTEd&bzgQXT9@=<^|1Z zkpOR9IRTVE97{B+RR-mF%^2Taf?XvuPk6wc8-|(o^+8q>QWCrYo Date: Tue, 15 Apr 2025 14:43:51 +0000 Subject: [PATCH 448/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10220 -> 10220 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b44133454..a59d85c6f 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 13:08:41,540 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 13:08:41,540 submission_checker1.py:3260 INFO] --- -[2025-04-15 13:08:41,540 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 14:43:46,221 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 14:43:46,221 submission_checker1.py:3260 INFO] --- +[2025-04-15 14:43:46,221 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 13:08:41,540 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 14:43:46,221 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 13:08:41,540 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 14:43:46,222 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 13:08:41,540 submission_checker1.py:3276 INFO] --- -[2025-04-15 13:08:41,540 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 13:08:41,540 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 13:08:41,540 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 13:08:41,541 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 13:08:41,541 submission_checker1.py:3297 INFO] --- -[2025-04-15 13:08:41,541 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 13:08:42,694 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 14:43:46,222 submission_checker1.py:3276 INFO] --- +[2025-04-15 14:43:46,222 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 14:43:46,222 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 14:43:46,222 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 14:43:46,222 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 14:43:46,222 submission_checker1.py:3297 INFO] --- +[2025-04-15 14:43:46,222 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 14:43:47,459 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410951 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.427762 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72939 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 695597613..bb953c4b7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1075 +1076 diff --git a/summary.csv b/summary.csv index 8be42e68e..ff80cc773 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410951,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.427762,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72939,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2764e908e..3ac90f616 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410951, + "Result":0.427762, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index ff635ed43574dcac15418435b3f22e5372c2424c..aae620c1054579e40d309038319854c4b56a722f 100644 GIT binary patch delta 1230 zcmV;<1Tp*UPwY>yatH+J9$4bDb_k*d2JO0z&S5gM9uEuwe|rk+=TM2R5l5PPh{lud z9-=`iOZa=32B;(&S5jGI@nH#o$|94G0+$c8v`Ht#PXJCI5{7;Pk{bO46`Oq!%b#y3 zc&3J*OSSFzaq3#xcFupS(kACpqh-XAJ|Cj-q|b+FfJ>&4Tr!R1l4&HDy2$6l5=btK z60|h%Z^O9<=GFgafm5v*Z##Xg_Mxcn&_mz5M*W*w`ytv*`UJ!@-$xl5+X zArnm_jM1y3Ou=+3!kV!N^W|7s0B>*yutUem7KG4!8oe(m?>O;a{A^VHzNlXn@QjvkyxEWEPoyG?;y$rA;~^WP;g; zgyEz~e^TS5S+UvY?;-PB#iyARU< znnVL=7TJAR0zk9K?xVr(11)XR3785Xs>7my%rb5tU9sEeF*5&GS3xE9m4b5D>0_C$ z=jP3qb`F&28gZo4hiH81^dTCAvV_xzX#h$xe;&D6qKlk9tP_B;$myfO=>sio(up!H zNhMf)Xc%Bg{6Mght=Q`G7?sH&15!Qdd2A)Mm0G}V`+U;-X5Z6Z4vnqPqrralzi6H*m~#TG>C@5NidnjMlVcH*D7+;x_S}hx!S*8y)*Ga zp7s~c`s3^+58cV?4UfrqHZOFgH!OIuD7ta2Rp7ZJ#L%%W(jkNT8 zX&93mfo%WA_g5JmU>3ZuO(J8?o0 zwgYX+s)0gNTCx+;BoVb9NNSz@edT8o$g0cHy>sTypkj5Bb@U6?39U+$B`JzPG0lZK zl;~r-jvrCvJXM_5T0x1<;LxgiD3~EkTX?hDfb{}g6hO)mW=eGQ-Vlr(JA$Mx2@Vy+ zy|$A2P}&23Hk5tS1K?SjF0lkpIrS8`z}QSgG$1*fkmkqM9e^_oHAqn2;VjAU#p}UJ zcekNauAEZ%vw^$TQJmZ-CnjUlG)Xh>tRa7fzc#P$-Kw!r?Wq``s^E+;3)E{{75KUc zhpVacn{eX20RD6i`kl97k-iUj6atVw?x=jupI^2mYgEnB*&4nzJ;907gq+fPva0kd8s$^r!G9$4bDEG9hx1grqsDU)p{eyatH*!P%-hdb_k*d24(to-ZD0`9uEuwe_Ou#IaH!+#F6G6qVc4= zhiH(>68;{h0V;{cl~fj4d{_dYvdHA4z~uuiZPE$x6M)l)grT2+q((nM#bzJG^5+{0 zo~hyIQf)haoVr%Fo%0{7w8^>DXc=*&&xdF{>GL5P;F4)1mrNtMWE#n(F7o-X1d_|5 z1T76dA82Wle@<}U62dw(OwN}?1Z!AbvCro*E`Q3#WhF(HS;y*atB=-o&)QmG?vkl; z$VAhKBdtC}<4LOz(IA;6tUgQwWD*UKS!DHL34qKZtB(e&545yNCxlF}`j9YK$gf0})W#*=0rqCqrEn0=TA&?Fi_ zv&ihj5&)V-W}nT$2Q2~eTc@BW*?$KGE10!mR&4h9d&qpIZl4Ri?e@tgO;78*J54JGO>~Yp((Xevp0xWA4We1X?!z>I zCeZ+zMRp&S0MIP5`)IKHKueo+0;U3p>aZvvvy9tESM2tAjLiSlRZvNNrJ&q(`dFsx zxq0)YodYGhMjYw%AsSygeTW93EaCKF8i10Fe@AYX=pv^N>ja=Ia{6d+`anyYbfQd4 zQVCWc8U|PrKM<^BE4KPPMrAU{fK*R<9$QImr53Q;KA-fy+4r=U1EiA!Bzf76wD}N? zFKs?VgFu$B`7jLtNyY;}7TJ7QCjexT%}0aH2U^;s6J^>GNU-_PFaVOg!UuqSPA;E( ze<1xQ0r^}#K6=~ZldZ`;C7<;)w%&O-4WeOi5=z@bX3Pg|q*n=e+vG(65X`BQ5=2 znpU^$CHugJb#yTA4uenKXgHq5#v~Y|g_?D@W<;y^P%b|w;e7ci{gw7LOpx5P%%90dCpqS=D z)g}7auHz+&oTrM@Mk^@M2^?Bg4+S%XX$x;w8?au0ivmbF!c2({-W!6kV+WAbCBdPB zxYt%vA4*$)V?)_DtpVq0`iLcX%BiQg1IA_~qBD}S5ovyG(*roe(0~Nx9nO*rU%eiz zbdwFea^sZ3pA1Y|FXHGvIx!mCwoTeuZw>h~{Iz*~?^lh5>Q2P~RRw2+S)g9qs=&8J zI9x-W--HwI1@Na+&?nx`iwu3hqY!`$aWBf}?D=ISyGB)>=8HI;$Jt_=%}BZ=^L*Eh syz9RL<$1`zi5I|-`~DMDf3sdA$^rzxP%-hdEG9hx1n^iugp+M2f7+T}OaK4? diff --git a/summary_results.json b/summary_results.json index dd610c36b..bee66193e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.410951, + "Performance_Result": 0.427762, "Performance_Units": "Samples/s" }, { From b458c8828add0d612c3e332aad99388ac394bb40 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 15 Apr 2025 16:15:16 +0000 Subject: [PATCH 449/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10220 -> 10220 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a59d85c6f..e1226e04b 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 14:43:46,221 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 14:43:46,221 submission_checker1.py:3260 INFO] --- -[2025-04-15 14:43:46,221 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 16:15:10,779 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 16:15:10,779 submission_checker1.py:3260 INFO] --- +[2025-04-15 16:15:10,779 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 14:43:46,221 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 16:15:10,779 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 14:43:46,222 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 16:15:10,779 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 14:43:46,222 submission_checker1.py:3276 INFO] --- -[2025-04-15 14:43:46,222 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 14:43:46,222 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 14:43:46,222 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 14:43:46,222 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 14:43:46,222 submission_checker1.py:3297 INFO] --- -[2025-04-15 14:43:46,222 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 14:43:47,459 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 16:15:10,779 submission_checker1.py:3276 INFO] --- +[2025-04-15 16:15:10,779 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 16:15:10,779 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 16:15:10,779 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 16:15:10,779 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 16:15:10,779 submission_checker1.py:3297 INFO] --- +[2025-04-15 16:15:10,779 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 16:15:11,868 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.427762 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.433525 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72939 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bb953c4b7..6d0c1073d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1076 +1077 diff --git a/summary.csv b/summary.csv index ff80cc773..a96a61aff 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.427762,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.433525,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72939,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3ac90f616..1ee8853ea 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.427762, + "Result":0.433525, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index aae620c1054579e40d309038319854c4b56a722f..4bf90af1fc861b4b2b5cb9d323de5396019a437a 100644 GIT binary patch delta 1501 zcmV<31tR+FPwY>yatH*+Ok@GGb_k&cf30TI)_FIWIX)eQ77#}ob%@53P935_CQEpA zm`~Hqa|$hOHcEgJ3_K(X7q7&^e}jwHiak7kJ(SPX*>j^!!$r8(YTVzB8v}808|#4d=$8RpruVZA$|gI`j9a66Oh#CC#cx$ zgINB2L%}mO{9LMS$B$Fj%C>X0c;y6#z9 z3(Q?IRSubG8gZo6hiE)$^&uK0vxL=$X@E?k0WyoMJ}d!{S!DImVD*8PHtB?r305Bx z1`Ao18Wys;Vyn;JL*_H}fBIbLZLd%EEY|Vuoaa7T=b(wc5l5PRh{ls_a-iol6Mo&@f<=ID%l(tk~@Hqi8;9>#~v}OHG>F zZXeyOn|9u9rf%ktiMA0(nth1IlV%^HK{892eV7KwBpM*I$n3)se*l?9W*-e^A82Wl zP6(M`_90<7X_C}9X;y6Z`FqHGrf#1Lz3ukNCQVQ4ygN-R2TgR2IMVJzG@i8k5DlVP z!tTQ~fF{uZnniXWmH^N!vioST`#?*ZbONRVi0ZH?AhV3yM_26jd5p~e)m2bQeWjq> zb^2JQ>$!RJrJVyMf4W8->GUBQUpjq=2B9qB^kEu+l8i@gmgpj<59?QlahIMo>?+$}c-Do(T#l|EUq=lMww`N4E_fRfBC*gegDgBl9HB6Fl zdjHcX==Oss`zgIC7=&SR|3h|4Ht+L1`Y%vR0|XQRvyKyr4g_e8z!#GoA|HRvYQr!P zgztR{qq{B3sY4OA18vEvfkIPSdK1#d5w(7h)H-?lDo)}g;8TyQnf-QlQMK5qF8KxL zrO`DaS&kCW+z6@L8ojTV=>tl9U|O)wXsFR1JX$n&6}OZd2d~apa6y7kB1n76ZH=}e zScf+4vwk*MV}61^S1)hFs;Sh6Q*l5| zB{<~{STL@s@O2Rn*D)Vf@x&Vm;&C7KiMP`t$375A3_y=@&&tQ_>3Jo+L`{(wWt!io zq+F9ZB_$=~>oD@B{{mE}A^#?x0mt08pP=~zvtA?00tClQWC61*COrWJXpO)ZlWiw| D>I=w` delta 1501 zcmV<31tR+FPwY>yatH+J9$4bDb_k&ce=Y60j?Q5+$ETyv0^&%c4$*khsY5i#WC^bh z(*Tpi4I}Mgs>rUxI+1>zq694kjvZ)elTMV`lu&|ehlT-_L?o?vXR1o)PDsN8liTTXzRV^{<=6ESpnRq#o^!SB;!$;1`?>Svj*G{9U=lqbjYg4KPXEG}1w!ip)H$6JWB)&ZDH;<`i1mY?N~I5~4aJiXch6J#c4Pv2kaI$iHF- zvyuW!LAmYc(apMU+BwcT<>f$$b`VE8dWgo8mL8%(C`*`nm*r93t`SF? zdx*x9?jE8+Dogl#m=u&BB$v9# z=fe_6E{hVhH28d=rA<1)eM<=I&@ee)5)rInb;UlP$GH3{8<&+7S!Nxpx2--}*F9@% zfw@bj${`a?BaXEC5RE6TK173LmazIT4UkDRKxUEEha~_qi>y8xtUl1vCY=y6!RkZ8 zU?Iy=!$MY9Z1wqj$b6<=f1eAz?e$3)vrWghbDsNXor5O&MjUDOAsSDbeTW9pEMfLx z8bFh10L>z^4@&@O7MXoE2OrP`vk&P6cP=5UL&Jbg;s}CCvtqN)kD~dct;0PG^X@dQ95m56;z+v>(RkACLo|qH z3A+!|0GdPtXcpOhSOP$^$nK-T?gK4t(g~OfAgaTnfXp&(A6>EA=P@$>S64wL^_7Bh z*Xd)KuIJ{>mv#=6f9M)3M7=wUt`HZu@-F`)1$MUJj5>4v^$! zJJRMuG`_U?5DfxZ!sf#?03;a?09j=7VVwYwMK&J|HXmqdf0Is>X-go%=0n2(Nb(9F z0P;DxeDZJ5*{cs4ZRKUeRLbh39xnpv{m;CwL&jO6<~IIIPC^Yk0a zime&_yTR(Mf04#K8i&z1`M&Bsdu`=n(mk4n{g=x@EoS3C7s1OHy%)~@i=Ol97el`? z4vn<*dudwTvX|@w8`jamygLj&b)(^U78{dbkQQpz-I@`t-b1HSZmpxY0k?5FgmU=W7M{SVnG*}Tv5=)X`)0|XQRvyKyr4g{{*6bLP&VVs(;r^b6Jr ztxA+7DT+Wb&4oIY=wrK%A5r8yRh-sZL5a@b(5iYUm?2DCc(dAo^#WWJK*|wjN_6zz z5R4r=f}}194i&_`wvzf#+5>+!lzr0!;8~h3u>?;!^%S?j*i1w;AUT_m=Ev3@fHMp= zNKoG4EXnZ2>%mHQx1m$6oKpC+fxFgGoZKfTCS%hyNi*-PA%BLyHm~p9s?!U$!M{RL#=aB2M!- zTWqtO Date: Tue, 15 Apr 2025 16:22:08 +0000 Subject: [PATCH 450/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10220 -> 10224 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e1226e04b..ba0dbbc22 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 16:15:10,779 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 16:15:10,779 submission_checker1.py:3260 INFO] --- -[2025-04-15 16:15:10,779 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 16:22:03,096 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 16:22:03,096 submission_checker1.py:3260 INFO] --- +[2025-04-15 16:22:03,096 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 16:15:10,779 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 16:22:03,096 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 16:15:10,779 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 16:22:03,097 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 16:15:10,779 submission_checker1.py:3276 INFO] --- -[2025-04-15 16:15:10,779 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 16:15:10,779 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 16:15:10,779 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 16:15:10,779 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 16:15:10,779 submission_checker1.py:3297 INFO] --- -[2025-04-15 16:15:10,779 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 16:15:11,868 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 16:22:03,097 submission_checker1.py:3276 INFO] --- +[2025-04-15 16:22:03,097 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 16:22:03,097 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 16:22:03,097 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 16:22:03,097 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 16:22:03,097 submission_checker1.py:3297 INFO] --- +[2025-04-15 16:22:03,097 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 16:22:04,145 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72939 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.21557 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2693 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,8 +38,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407871 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.0541 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2354 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413841 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6d0c1073d..19d504922 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1077 +1078 diff --git a/summary.csv b/summary.csv index a96a61aff..5543e8670 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72939,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.21557,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2693,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,8 +16,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407871,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.0541,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2354,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413841,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1ee8853ea..d5899f56c 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.21557, + "Result":6.2693, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0541, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -544,7 +544,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 4bf90af1fc861b4b2b5cb9d323de5396019a437a..1ca073ede0e72f00cb33029bee35a8f2b4fcf894 100644 GIT binary patch delta 1218 zcmV;z1U>ugPw-E$atH~g%vaj-3IG5SvwH~r2Y=4i{@w0>m@_$+8XY5!wD}N?Cv84N z16VSR#FA+wmP{kD)I~NQmOx@zl%S=-<^wHl(g|)`LRg1}$?1}aU^fEOPp2 zaDVzhOPh2;$ONYk34?VkOAYH-U9r>W?;-PaN185Sfj=aO%XegSb6F{@b>$5rdfF^i-NGG^)31J-?25b^T5KNjC zdwqTs%_r?#R#IfCNpsumlRf*f{G7W?-G9s>6I~;Y^!gBuC%ryIgJhQQ`Y;WUNvt|R zW|7y2bpm7-d3`i^eW0aHIw53&*N24Rq)Afaq*<}o=kFo&nVNkr^tRb2n>0PGbKc;g ztsFGbIO0gR57Bti?L#z(W(l_s(*T-81MAr$w+~AIXcoDBG`M}BrA;~kPXR=ASbr3d zS;p+6D>nN)M&|!&DyXEsQc&(1eJs=U+`ReH&Vdq5BaSrs5REU5K1731mN5D-4M0i8 z1M_8((T8;cP!<_|G#GuLrA<0frX{Harwq_$EE z*ln9ndf)7O+RFjb$pMnQYe%|#h=0bHE+3*nAWOJ>m5913*3}lTSX7{*!=wt`;A?ZSl$0FHWUZdzCGfjn3H*Q-}1KFHJl!dZWuz2l)fS-s#f8PA4B{O9V`k$+D1>PRz7 z)*GBJCV`QBp9hDv;BKCNLs_vkqklJ8y)@F8N8>OWC*N1SXK$@sOu9$Yu>W#7sKspj z=OTFdqW8ktf6;SZ{bJ}>#-WjxelJa{TlSKDV8c2(n0JT4r*1SH&thW|4AMf)x?3}% z)oUo1pObLD{FMGm`x+)mI2yhGX%uw(L6rTJ-V_YNFuDIBJ0+Xfc^>^2vxpOr4g}g3 zFeZ~6A|HRtYQr!PMfd#*qrENLX+jaU18vEwfkItcx)ahQ5w#vjYMuOj6~B^zS6zM1@0V>1!a zfaGjKnjhPA0M0NpAVGPDvn0b8uLmpLY(uA9Ii>I?1GCmqoZLqzCS%*SNxSH*A%BKH zw=ZwqsiLuk*;8{tHkJL;g)10e$Y5yCM5v?08jz|01*HH g00000000000DyrGlY}Qe0b-NGCnE+FB>(^b05Q}qxBvhE delta 1192 zcmV;Z1Xug;PwY>yatH~>Ok@G>3IG5TvwH~r2Y<@8bN*wMHaV9XEhCQf`4EjKeLh43 zTr!R1l4&HDOe49}MLr*vKyq1>pryg*11)XR3GQ1$Scitm`I3lW4XZ2m`8>wuPuaMv zq{uSsSiNoa(Yo$gTMNuxGF1+lXc}>()rV+2Y4sr*B(sFohiQOJq5(3CtUfFOkXdB) z(SKm|ftEJugpdhV9})%&S(X|Wvbtic&)-AlGxhph=xwi0_AJ)%?VRU6TIZmNz7a>7 zeTc@BW*?$KG)tI$m(DS@lQ@E4(yZ9* z^P^}!Y3s6*B1=u0+ioA-tebY;ZKiJKkbjA`5l5PRh{ls%2QnD+f(< zjyTfpLo}YW`w$JHS;FqaG=L`20GdU1AC>^nEVBD(u=_wun{)!E0*LCcC?KUI@P)U8Ipxkx(Sf=Z_dGn>610}jf9O?8S8ecknhz6l7;q+k|fRc;mCb7{A z)6=zz+_bJ<1bMFZuUGF(e2}O8g|q%Rd&xt0vUb34&qeU^ zMel{P|Dxx-`o++%j6)+W{a%_@x9lbRz=m~nFz*h7Pu*xZp2fx_7^H=ob+=|jtM^bY zKPTaQ`6>OC_BBkBaC-mKC>H4UgDCqcy(t)kVRHXNc1kwy^E~=5vy2mv4g_e8z!#Gs zA|HRvYQr!PgztR{qq{B3sY4OA18vEvfkIPSdK1#d5w(7h)H-?lDo)}g;8TyQnf-Ql zQMK5qF8KxLrO`DaS&kCW+z6@L8ojTV=>tl9U|O)wXsFR1JX$n&6}OZd2d~apa6y7k zB1n76ZH=}eScf+4vwk*MV}61^S1)hF zs;Sh6Q*l5|B{<~{STL@s@O2Rn*D)Vf@x&Vm;&C7KiMP`t$375A3_y=@&&tQ_>3Jo+ zL`{(wWt!ioq+F9ZB_$=~>oD@B{{mE}A^#?x0mt08pP=~zvtlF40tv@VWC899000w{ zH6~X9h?9*bJOL7u$|gYpx04JfIsqw@M<*r(XpO)ZlW`{&0Sc3dCq4mQlgB3`1`s6x G0000*U^65D diff --git a/summary_results.json b/summary_results.json index 40a3b948e..6ef1528c1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.21557, + "Performance_Result": 6.2693, "Performance_Units": "Samples/s" }, { @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0541, + "Performance_Result": 23.2491, "Performance_Units": "Samples/s" }, { @@ -725,7 +725,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 63009f48f89549e8f8f0a26f43b9568e8c46eb33 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 15 Apr 2025 16:23:49 +0000 Subject: [PATCH 451/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10224 -> 10222 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ba0dbbc22..fd3296dae 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 16:22:03,096 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 16:22:03,096 submission_checker1.py:3260 INFO] --- -[2025-04-15 16:22:03,096 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 16:23:44,928 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 16:23:44,928 submission_checker1.py:3260 INFO] --- +[2025-04-15 16:23:44,928 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 16:22:03,096 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 16:23:44,928 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 16:22:03,097 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 16:23:44,928 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 16:22:03,097 submission_checker1.py:3276 INFO] --- -[2025-04-15 16:22:03,097 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 16:22:03,097 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 16:22:03,097 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 16:22:03,097 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 16:22:03,097 submission_checker1.py:3297 INFO] --- -[2025-04-15 16:22:03,097 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 16:22:04,145 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 16:23:44,928 submission_checker1.py:3276 INFO] --- +[2025-04-15 16:23:44,928 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 16:23:44,928 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 16:23:44,928 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 16:23:44,928 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 16:23:44,928 submission_checker1.py:3297 INFO] --- +[2025-04-15 16:23:44,928 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 16:23:46,053 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,9 +30,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.77649 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.6879 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.372039 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.73443 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.9769 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.372039 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 19d504922..41495a483 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1078 +1079 diff --git a/summary.csv b/summary.csv index 5543e8670..1482e8974 100644 --- a/summary.csv +++ b/summary.csv @@ -8,9 +8,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.77649,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.6879,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.372039,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.73443,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.9769,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.372039,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d5899f56c..4c5d9bdde 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.77649, + "Result":4.73443, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.6879, + "Result":18.9769, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 1ca073ede0e72f00cb33029bee35a8f2b4fcf894..fbbb93e547a615eb49661cc8462422802894936b 100644 GIT binary patch delta 2649 zcmV-f3a0h&Pwr2!atH<76atg*lXnPOf94?rt+jXe7W&*R^(r0wdN-XIw?Pz-!`Wd? z)y!?{d)ADgKuHO@W3_+sn|5lAql0DN*W=(@ zy!F}0O8FXwe`H^ik_jdOuamh6I2`ZseG;r=8mnHHBYe|Mxvsye<^|DmAi$a<<%W++MGm>ljtFd;sSb&5?5e$f)XH+NF<3w zB1t3?NupOkA_;VHOQHcRQ3Dg@MnkbA2>{E21T7^u z7EYn1O*&DgeDVij9U3NQOd^6QbE|;oh~?INnQ^Jjx_Sbbe^u02X5w5AOs$_i*FD$C zxr*-kGZjS9pFj3>W5=_(}m_!3$5)FWf8U#}pfk_eom<0)1n}ZKv%9b|ign%i- zIwTC9seb>pm#pIj%#|rn#ZBK%U8rPR#q6Wbv8UT}M08FyK4`5nFV3Gs?W2AM43`@w6xhM<>n6Rlr-L)LQ9*C5}*VF z4~fFXE3xq4;XEl|?2W1uh?GX_HQfp8%XbBn6kc0WygO$SiXDumnJ6k<&+m(+67Gq!U6WIDJSMtYcYfSjXy$oj!jLna|Yf zbD_7bKG~yK$G3AH`)HkmCfY_E>GdHRPkMcb2GK0x^C)g2+_>%K56H&k|Ik@n%icd?Aedy=iFuLW)7L? z8gZo8hiE+M^&uK0vxL`&X@E>()d4b#ygsZGAhXEpqrvL~Ep5^XArrhlBn&4_k{Tz? zioHI651G%@>~o>F%|6+r>1mzwe+Cb2<)Dei5l6axh{ls{AEH4tOSpZQ2GArLSkD%@ zeOLlOv&ikE!R-SrZPE#N3LvV(qJYdYW*=R#+2=7b|5sB%CH0kpa@Xi%nXc#N&6jo# zlxP}pq|t|Hd};I{8icZh(T8aON-`dpFN=&mtP_B;$mpZN=mRZn(up!He@P`ceP|e9 zN$fzdlC9Y3^B9%MAOli8>2Yi&wUt`HZrgm)`)1$MUJj5>4v^$sJJRJtG`@8C5DfxZ z!sWv>03;a?09oYnVVwYwMJ^u=E+1%VlTMUrOCZ7JL&E?_@&+FO@;RA&@`3c91mttI z_~>nmPqrralzi6H*m~vRe>8}O!AUTg#6~YnPuD7P)4F;OIIL!OIuD7ta2Rp7ZJ#L%%W(MUAxddudwTvX|@w z8`jamygLj&b)(^U78{dbkQQpz-I@`tUPHP3oP_h`r}R(S*Dy)K>HW7+(Cr6N_AR|B z7=&SR|0T186POMJ0FJ4qlOQ52e|b=Z?Lb?yYM@Y;mh6NyNkpv&l3FK!U&XH^kX4tX zd*{rZLB;AQo9Gv;6IzuhOHvepVwwxJFVRQ6jvrCvJXM@FT0x0U;LxgiD3~EkTX?hD zfb{}g6hO)mW=eGM-Vlr(JAkAv2@Vy+owkztP})5nnNiG z<#N_WlvBD?a!5i%(mmbRFZbsUcs`!**X#X$JU&Y{%Qg?L?Aue`=QOqggza;ouVhV+ zj6I1IrY$idOa)t~tk9iQv6nMJS1AsmY+nJk#~Zo9#H5C*-mplI{$yaO`dv{1d-}n; zLGr~`*) z<&mcpHDuLnE@=0oKbc|hVl{BR9}p3Wmnf6^_~zbl`3>3eA4RDbe;m(F{tA=`A;Ir} zZXt8<89-dv4BQG(ghS|tx--p`bjupXp-@ZJcXc|UNAC`p&cA3_ha+nracyk*hQ(7# zGb|9HM8(=y8-d_ZgxmJY4n=ILJ?Kort-z!-_HbvAWz?-(ZYhWqarbDm6wSXr_0@=V zzyDl5-n(>uojaa)y0XPf=)-m-)U1WmUsH<<4Zh!GJEjtQ7FI^jC4B3dH0G?#XJ}Kc zS$~t(vWS<#EoLdI7@QF#%L0)@RLn*|QxF_g`RoKssBpXMp#zABJ&3b9S)WNz(sib- zVF@wcIQY}ANQ-J#E)?Mri383Kacgg|CBN&Y;#o)JBhsNxgpfEWA=?6xN>sEXrfBFz zd_8#EqWz;4-Nq=q_u%cqoG*J+M8x<6!j`4$CJHpPjy5&E-`9ywis7O`{+>PF?oz{hb#Pr4OCM2loV|g=#MON?5^^$?_^<(Y3 z+xN*ZmHfT5ApmxqQJ{I#)%${bnEr`N>b)nvXTp|lj28riv41#X*ZMq ze};&mqly3mq$v!o*A|rP%2lUwApx<3Q%R;^W}lRR=hz^eT&J)yLw2JgJ1`vch9pI(hiaG^I1)NEaCt;oL-1sMybhORf-SCJ z6B99ZZXfQzf{@2XK)I&RWJe$b=MRf^-0)x_b*lEs80da|gwVnJFWTeNMpF&+=?1xj z!*7G{T&*!+T!?U-a-bL{oeW9W;ZE>yAsb>M_Rj6^Iff0TqXQ#5lS_1_mM#M>qm)s< zDVC0=;)(ne&GZ*J{s;zHsczU*TaH0E>zsSLtnCy3ZfSw8#+0PQ?zx;~Pe_$&* zU(nWWWyW0scSRxbp=y>Q3l#Udq1mqrtNAXpqic(Pck`^u5 zkN)E3XzzWG6g<8N82qf;E__0s>{F0%SISu1;?2!N$9prDw=g1Eq-s5aZ}wcA9veM(WF8UkD~kzfk;!+`~*B2u)$~T05MUuQgSWS zZ`~swigN}d;I8-S`h|Nr7##=-c=P$Mxj7fpKo_U4vHDIwN2{Dr_^zRyc4vBF%gh~| zbIbIx!-I)t+q3hpv#rrBU;0%%5^=5=GmdA z#|%qqyBmm=9z1&vx}R&@-|VWzdVTRo-Yp>L*>>*E$)9i(NvGwvzhuZyqugJ)Q}8653K!8v(!j-0;&E-ho2F@en&+UvR6-C9`oCS)QdG)n6LFr#8BVAzQ8{s z9ApPlHfJhx+9FLbR~JD!%?O=SVqs$^=S}=K-q@~NZN#i%-Mt{iSLpM~c>Z%*;+&Wd z0mcP#;~}5{3|8LgS`r)4aqzyN>STb)T{qpbWi&r~#IA6{9F00R-bi}#xmtmqXxOB2 zX|l4;cGlq-;dI+!FR6;rMwR7I@dKl_vrX?`;K1A8B-iqryt1E(hD-8g;g>3+)QXE2 zL@HOXXFBI-h3X3driWGvOVjSJcHP_&wmo37V42(1TJ2TakZ|D|~eEd@RRB%<-)v%pKV(q2grIE>Vi diff --git a/summary_results.json b/summary_results.json index 6ef1528c1..3e9831999 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.77649, + "Performance_Result": 4.73443, "Performance_Units": "Samples/s" }, { @@ -365,7 +365,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.6879, + "Performance_Result": 18.9769, "Performance_Units": "Samples/s" }, { @@ -401,7 +401,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 96b7f93f454c1bf8da6143eedb49169838705ae3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 15 Apr 2025 16:27:10 +0000 Subject: [PATCH 452/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10222 -> 10225 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index fd3296dae..1ac01ffea 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,30 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 16:23:44,928 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 16:23:44,928 submission_checker1.py:3260 INFO] --- -[2025-04-15 16:23:44,928 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-15 16:27:05,149 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-15 16:27:05,150 submission_checker1.py:3260 INFO] --- +[2025-04-15 16:27:05,150 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 16:23:44,928 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-15 16:27:05,150 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 16:23:44,928 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-15 16:27:05,150 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 16:23:44,928 submission_checker1.py:3276 INFO] --- -[2025-04-15 16:23:44,928 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 16:23:44,928 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 16:23:44,928 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 16:23:44,928 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 16:23:44,928 submission_checker1.py:3297 INFO] --- -[2025-04-15 16:23:44,928 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 16:23:46,053 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-15 16:27:05,150 submission_checker1.py:3276 INFO] --- +[2025-04-15 16:27:05,150 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 +[2025-04-15 16:27:05,150 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-15 16:27:05,150 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-15 16:27:05,150 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-15 16:27:05,150 submission_checker1.py:3297 INFO] --- +[2025-04-15 16:27:05,150 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK +[2025-04-15 16:27:06,125 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.433525 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.387848 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72939 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73678 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2693 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -40,7 +40,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407871 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0819 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 16.1275 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2354 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413841 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 41495a483..5b1466985 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1079 +1080 diff --git a/summary.csv b/summary.csv index 1482e8974..1de991bd7 100644 --- a/summary.csv +++ b/summary.csv @@ -1,8 +1,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.433525,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.387848,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72939,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73678,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2693,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -18,7 +18,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407871,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0819,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,16.1275,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2354,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413841,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 4c5d9bdde..40819e09e 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.433525, + "Result":0.387848, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72939, + "Result":1.73678, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -579,7 +579,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0819, + "Result":16.1275, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fbbb93e547a615eb49661cc8462422802894936b..fa28a2ce73392c8721621b5b1538c39f207fa211 100644 GIT binary patch delta 2489 zcmV;q2}btrPw`K%atH+x8LM>ilXnPOfAWyv*4n#!3w`dEdPN7n-c2XQZ4kxdaCTTz zb+cv!vtHOA&xVJ!tIKzLUd@P;?yTRPgtOqV_C1JezaIVP;9D5|5s!i(G13IH_^>ug zlKFn45%)&Hv>VsMc`!?l48myIP0}BuVI!VLLASpQnob(pG@Zt@JD$~68TO+xLOH_0 zVB8DdguTTym?f(uQ84Ky>G0#xcple|4*KKtSXPe_1%t!dvD!cRO*^&5(ZRCs>v8Zc z-ui50rF;#;Ke8W_k_jdOx0AUEI2`x!eG;r=8mnHHBYe|Mx$`% zt(F*dODo^8ICUDc7Q?vE+@5UoCCZZZC}YCf#L*JJ0#q%hD>!D?#A7zeS*5cbcYc`kOx- zMP+gpm+b;Z@h!h58dpGaLNtIQ(?}kfMgmDRzR2@_qhUz01d_?31g*`%4VBQ+CY>l# zPJQR6xiOY*oLup}D5l4t-+)WAf!(NHW&0>H8$K}*Sv zg;Qv0lTMT=pZq~shla@+lZar-+$!KXV!1V6W?X8suATs9E)_MFnK;)2Q|o8XbDUJZZY|${;|Tn za@&(&qQ>=qC77r|Fo_1hBpLvd?C%4ZRuPya0f1SMptU*p0H$ndlTHYjGOROrilYQG*YE>>@Hr z0wA*>L2GmHflS%bCY`_*2t;*Q6oAQWK>()xticq28zMi>WEJI=V$$Aa(sRA7{p4LH zb9_1qEg+6G>JW`5ojOE=OqTHKFbyzC+%VD}ri$!3tP|@eqwCeLO^iOqOu+FbyzC+%(cbpo+{qtP^0e$j+mr+vXHn+H90^ z^Ae&uB#IzOyghJdS+Q|vhseKT2eXm_OF_Bq=h4l&ZrVA{I_2d+iFOc2I(mr4la?N$ zK`2X@dYA^FB*q$mvdGuNIsqt)oIOe!Z%(0qrOieOP=bMnMB(CQfP0z}E3hU=kiLMbxntO=GlkOg(K`Kl5dzc2OBpO#zS!D5H34qEXlaB(I z545yNC&W(xP9GA6egcvj{R9=8eGto^Zzy=ChM!Bd?f7x(+VOMz1lr_SYIKY^(&j^d zG@i8i5Dj3-G!jdukytW~#8Ma8d{_dBWl@5b2AdDGv`HtpZ3$r=8YZVpB7!BXuGr@D z7?wX}-?EY-%PeE{w$n%Jx@UQLWU3r8(KF&mrw`G1(&L-d^=}*MC%+h z(Kg~puMg39((6Msh-L||57Pje#Hu6jFgF^CCg}vwEb{tn4nCj>ULVp4Zd^iGhlT;0 z#1I6NX2o8gA4T&?JC~IdS!&YUHv44Hek?!dE>kyi$VAtOBfUOE<4La%(IAaN z17s4b4v<;o^5plJUTNS!DEKodA?YMjs7EA82WlPLyd$D#7VP!vITS2ZEJs#ZI5cs7wYKkm^Z~ zV=Jkx)B<+f=9Atx`<~8m6lf<0Nb;^7>GB~OU%GsV27xT$@?jbPl8gs9fGl$PuucHT zBA1T_mk+eGNhiv*C6M6qp zA|8LuPQx$|gzr2>Ir3G0@Dgo+=N=Ot@akK1gY7zg)Hl%rb+@wugDJPEWnf-Ql z#g8a*o+?fot)N65IJB(p3T6n?7T&BjV7&kr z1(0%tnG)^2Hw0tH_8_TCf&$ul(v7yhO%#31J2WQfhBm#si*h|jLk?yCnRSh z()`$_2XKa=0SU@GoFy5)cs*F@CL4O?$|;5K3`|E00000 D;XRg| delta 2519 zcmYM0dpy(Y1IIUprbdl}Rx)fuhsY7-)|AV}+zy?VTjY`{o!pYe`7(-;>(^~AiQ!Z( zHFGIl(1^Jtmk6oE%=P4c_v^Phzt`*c{QJCK@8@&-<5{%*$M#9!9+8jILf`@zy2lBa zR1~hL30;@j`lzvbRDOr5moHamBAv_`^00p2m2;qz>H!!YIx6ARr^kwgp*XFXAl3PmGptIS+)gzAcu1u0nu{dAN!>_$50j!tY%u4T z8z$4P0S9&Rb*d@)iIh;ovI^}v3-2?5G0{y=WnaK=B~Y}i{vZ~)_geRkwK5kCE9K=^ zTBlj%N0h}yV{1HXfTi3Ueu>Oz@6UtdUz}g84z*csqV51*7r`_*4*D2;20wZD+hPQ= z+!IyimZcotXI*b0X1~!Dmub|zd~K}*M2JUAF=cok9`=`J92wcjjlZ&SA~o(a$P~vy z8$hnO1@ts{V6Qn80LgoUf$xEDLRhFQCynDXnSZ?U4484T$xrB~UztB)=@L*5YC{i_f)5qaMI1`UZ$C1B{LN<<8jSacq*1kb|A#MEgWCaj1Pny;gKKunt4 z@xs5%>Dsp^JGa+U$YyF-!i%*VxR%lq*IzHI%>+qDnA+o4P}s`IDHt|VDU~2iH!X@o zUJPFOq=`?B>Bt4{(H1f_@uxBn{=HEy@$K0XZ&V#B&Fnq7hh!U^hD%4?j!M)&8yH~E|@lKAf3zQ4*fh3WX!@N*%OwWc= z_jckPMJ|$nOF&lZis?C?vC0iDBAr5s+k{%QSJ8pw1nqHm3PP ztcsxO_QFy;^Rp}r9MVC9X<+qaYzj-P_3cZ;m&82eQMk(_C7N20;H@~`KsY1dVOKp> zC>uD}PxkrhB1x(!Fd8%(&=xkGAcO)W`j)FPJMiG&a zY!SFqZnUI(d#3JmOr#n|AG7b?qpJQuO`gCBpqVW|PMXY8-E(s-pHmEu#*$uHQOmC{ z_uRlTyrO$SuOd+{289NFBL!@L(WRtK6hgDnCRc(MqDVu%xO`OyD{XIb>>X&1>T5Oj zX0nol5ktm0q3ZJ#c$O5woXIX@hB^6#=t*7`m`ULVd0YmH!Cb_ zT*ERRERu*gzZ6mtRGTi^W{NQ0VI{O-;E|m)SVdQpn6|f|LLq1L{`VgC5FA?sn!Uyr zb+{V0a}pGcBc6CGmRL4a9r6j`lVodADAJs2E?=}*>=5gl!bQ5tmqKy$L*~OA$5hwY z5T#vxGdSQRe%RpM57TV5LfD`LLii}1;I#PGG2?B1dV9-Yu(nG%OPYYI>*Iwptkc^c z&#K@zN!EsZzmXQ^Oyyb-pDQn$yxEAM;tx~C6;ZZZH{(PZ>SGfgDEG9dl;UXoFN`p< zN~2e5z6G(aORc(kNR>Bm^#yoab!q&qL@LlC;e!>)qy>(2Gjl^mtAp}F z+B>3<^XR1DmHmk-8b+qkc9~kMS_nMx?g@x@>f_iDAv>81!sPDT*q$~`amky~ft^Nj&XMtA>AP+?t#@eq^TS{HO=+c!TN zH*9`*dR5R1H6l*5M}2J71BD04ArtbG=4X2_`4Aw-%&kDME1jm;t|;|FkfVxq8cL-r{^h?S=L90VjE9P+Df!EQsH8vdKK;!B zi~5MTH{dmeA`=~}rxDqr?=UEGx6HE$vJjr#HqUID|B1UPj()@`itN<(QE+7q@sDeN zKks0IUCp2c4o-P8D*2OLSvMroeFf@zVtvCYD~hO?s+gWw3dBlT*cO8I5l1y3GdzZ`P}Cwkzyl} zGT~~_%R8&{DtL20Lg>P49!Cd)IVcIaFtOy34@ogQrhxc&KIs%IbQA9^2SQj9Re&s= zVuUY6trWl}f@(IV{1a?zaqH>f1AQN;yfLs5NrcLR;!>B;w!?#3#Mjfib@uP(#q8HK zUaHB(cm#gO8enq=!;b9AvhPZzNQgj^9+qN3MWrJ^haX34zuEe(IP;~(NSIyBJl?s= z@4j7D=3I;p(X>J=%}E8g*zy>YaxRN`i{F`V(lVo(J0)8#<&>dn^}BS`rJTJrnmraT zlN9VH4A151I+@UIzPx)xRn1l;Ip1UtN3IIFNaNcN_*e!neA<6k-XOC07+m6)=KR{U zExTbOKsP<2&*+PxbrzZScOEsqyYU0bR0n-3H)XO_S3dH$eF%cxr!jTzLd$HEWoTWY zLuKCY8(`G)0sFj<7AiQ;ky@g`Y`G$Er$P7ex6wx|JHEql{JExs9x@LGYmY939FQ2a z Date: Wed, 16 Apr 2025 01:45:45 +0000 Subject: [PATCH 453/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10225 -> 10222 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 1ac01ffea..ad4abd088 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-15 16:27:05,149 submission_checker1.py:3253 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-15 16:27:05,150 submission_checker1.py:3260 INFO] --- -[2025-04-15 16:27:05,150 submission_checker1.py:3261 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 01:45:39,860 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 01:45:39,860 submission_checker1.py:3266 INFO] --- +[2025-04-16 01:45:39,860 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-15 16:27:05,150 submission_checker1.py:3266 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 01:45:39,860 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-15 16:27:05,150 submission_checker1.py:3271 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 01:45:39,860 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-15 16:27:05,150 submission_checker1.py:3276 INFO] --- -[2025-04-15 16:27:05,150 submission_checker1.py:3278 INFO] Systems=17, Power Systems=0 -[2025-04-15 16:27:05,150 submission_checker1.py:3282 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-15 16:27:05,150 submission_checker1.py:3287 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-15 16:27:05,150 submission_checker1.py:3292 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-15 16:27:05,150 submission_checker1.py:3297 INFO] --- -[2025-04-15 16:27:05,150 submission_checker1.py:3302 INFO] SUMMARY: submission looks OK -[2025-04-15 16:27:06,125 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 01:45:39,860 submission_checker1.py:3282 INFO] --- +[2025-04-16 01:45:39,860 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 01:45:39,861 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 01:45:39,861 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 01:45:39,861 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 01:45:39,861 submission_checker1.py:3303 INFO] --- +[2025-04-16 01:45:39,861 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 01:45:40,836 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,15 +31,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.73443 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 18.9769 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2422 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.372039 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.407871 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.438227 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410188 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 16.1275 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2354 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413841 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5b1466985..e74c832b2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1080 +1081 diff --git a/summary.csv b/summary.csv index 1de991bd7..6c879224e 100644 --- a/summary.csv +++ b/summary.csv @@ -9,15 +9,15 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.73443,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,18.9769,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2422,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.372039,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.407871,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.438227,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410188,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,16.1275,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2354,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413841,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 40819e09e..671d37848 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":18.9769, + "Result":22.2422, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -489,7 +489,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.407871, + "Result":0.410188, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -514,7 +514,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -544,12 +544,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438227, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fa28a2ce73392c8721621b5b1538c39f207fa211..20565004a5d9b1b04c6c85a043debb19aed39a78 100644 GIT binary patch delta 1722 zcmV;r21WVtPwr2!atH-D+4O1fvv&yp1%E}R)^%-dZG!HC$v5?n74DVWo&*y$t}nqv z4T4ED04C7@m}Gw+z_g0MBnbe_f&{J2!3Qv9OPh2;z?5Mf5{6SINsUvc^?v|n6*ZQc zGOgVy)754(@8M652}Pl+!1g7XsPRlmCTfsOq5(3A2FOGWKK!wZ$Rr7X%z^~1&40lM zGG$AfbOKu-5Y=H(04B2q0hso)22*T^{5X?Ulvj#LdzVSi^|tntcbUxb=_s^-IMS#? zG@f+o5DhX}!mGnHz$9_QNPCznvg@!;q+h2fK}&&S2U^;s6J<6fl;GN-VL&AjNh{u& zs?xcWQn~n=PKJv$mIBG-cDkG|vww+F`E~sSD4(f`=Ui>OcvRhOI&L15J1!pcfl2g$ zIMT*LG@kVF5DhX}!pXxlz$9_gNC$x`GV`!bfXN~|kCJYiQ)p?kQOeCri0Y6if+X?w zz@25q#+@A^|B4;VN(w9m<+h(kH|x4-=Q!(>mjfl*K^*DmAsSCwdWZ(0EPr9@VH$vv z7;6B^B3}>d1fVQ(_9$t*Ifa%s8zn#q1|AZHi&tXd!NqIE9-hA*%4h2Axlp^-9!oVn zEAJ_+pF<_OMjUDGAsSD*dx!?9EaC5A8laMBTuEh-#fK#TDvL}$3S2(W(k7h{KLI#> zNErGFNNV&GRBZM^EPuYC;D4DKelFFv{9(J|skn-9@=(&j@nfF;vN zESW}P$uts6U1alN2_%+930fL#KG4!8o#3`5gmq|`oGyt7maw{Fo6lod{*-;oN{TGA zjMdvtAFb=2<>is7a>zu_h$EdoMB_=P578i*C7eD?17s2nkXhvPVSfpL%p#|c2B#0S zv`Hs~OmO;;Fj&X3)Ub}#6+3aN185QrpjqVgVF>`uBCpTp-~*cA^&y?$#wCPxXc(|b3_&nyR_yipQ8b^l zb6H7|r6$d7vrqQy$A9v3?lN^VhfH*hIMVAwG@kVO5Dk)9!t28{Kqk=unMGb7mH@~s z^7?4-`anyYbVA4kuMY{sNt2|;NwZ?F&)-AlGd25M=xwu4Hfg$kGiR?vTRCW=am0~s zAENQ3+lOcn%@S@OrU5jG2GA^W`>+InW|7-RgWCsM+N2Zk6n{WeheZLIWz0UhVzbX< zWd5(Ff=cQu1?8^M$4YNvygVrF94OH=;z*+p(fHEnLo^6w38N3w0F-1ra<4=e8GTqM z0A-QUM}yG^TH2%&Wm=L-aQe_Nz>?U3U?p3z)8{cNlR*ZgdeY$fZewFr1#Cf zr}Kbxa)2c7+JBKQAENQ4%ZF$X$Pz9erU4+ycmT*Emk;X%fGl$PXmI&JOPh3}Oj`m8 zE*}~OK$1830Fckg(wh0ALMC&;jBN--to|#tX}Y#jAuh5{(p1z>PRPhb)=ak>kZBqlfX#6 z&x6BSa5qoCp{&@N(Z3t4UK(l4qj4CGlkcnEv$s|*Cf%cH*nhbk)M7UNa}m6J(R<NS+h&q+96 zeoB9(eI5;yB%I#=Gzz-?Aj*D9ZwdxsnB4!6os!M#Jdgehvy2mv4h0Bh{T?NgAtD}s z%WA_g5JmU>3ZuO(*?CZe?Lb?yYM{`Rmh6NyaYU^Ll3FK!U&TqB1ib2Ubnl$GGpJnb zRh#^R^U~-F$je|F5EVv-SClRDQ z<+eh*5G=*m^BpMWv*^%K+#07?h^1?Pux0$4HGm63=2$^sf(3?;z|;;zbVdq3AnlK9 zdw}2=T2P>a$9b0Ht2cmCezc)iZk$SngN0G+MI79FF9&1Sby+v*tucR&zgDmB{i>(^b07x?|xBvhE delta 1714 zcmV;j22J_yPw`K%atH+x8LM>ivv&yp1%EwddUfA*@@_Htrv9i==V4}wLC77r| zFo_1hBpLvd?C%4ZRuPya0f1SMptU*p0H$ndlTHYjGOROrilYQG*YE>>@Hr0wA*>L2GmHfqzWd z(k7k276?RjSQLQCY(W5~{j9+h8zMi>WEJI=V$$Aa(sRA7{p4LHb9_1qEg+6G>JW`5 zojOE=OqTHKFbyzC+%VD}ri$!3tP| z@eqwCeLO^iOqOu+FbyzC+%(cbpo+{qtP^0e$j+mr+vXHn+H90^^Ae&uB#IzOyghJd zS+Q|vhseKT2eXm_OF_Bq=h4l&ZrVA{I_2d+iFOc2I(mr4la?N$K`2X@dViP(pd`i` zfU?Nf!#V*di<~`58gEXarOieOP=bMnMB(CQfP0z}E z3hU=kiLMbxntO=GlkOg(K`Kl5dzc2OBpO#zS!D5H34qEXlaB(I545yNC&W(xP9GA6 zegcvj{R9=8eGto^Zzy=ChJT++we9$E>e}&h`~=$MSZZ{PIMU`rG@i8i5Dj3-G!jdu zkytW~#8Ma8d{_dBWl@5b2AdDGv`HtpZ3$r=8YZVpB7!BXuGr@D7?wX}-?EY-%PeE{ zw$n%Jx@UQLWU3r8(KF&mrw`G1(&7&8v11)XR z2_X}lJ|qm*u`D&LV|B$&pTCF9XKMAi(A!p@bS>L-d^=}*MC%+h(Kg~puMg39((6Ms zh-L||57Pje#Hu6jFgF^CCg}vwEb{tn4nCj>ULVp4Zd^iGhlT;0#1I6NX2o8gA4T&? zJC~IdS!&YUHv44Het#@K=Ppw>bI3&3h$Fo|MB_=X578i*CA>aN17s4b4v<;o^5plJUTNS!DEK zodA?YMjs7EA82WlPLyd$D#7VP!vITS2ZEJs#ZI5cs7wYKkm^Z~V=Jkx)B<+f=9Atx z`<~8m6lf<0NPqIK9qIBR8eh76hz5Zy;qqY`0FsOcfGl$PuucHTBA1T_mk+eGNhiv* zC6M6qpC@5NidnjMlVcH z*D7+;x_S@fx!S*8y)yAZp7s~c`s3^!58cV?1&_&iHh(nYKUc4gbh1}Rnpv{m;CwL& zjO6<~IIIPC^Yk0aime&_yTR(Ek;Xh4htW9szUn=DYvp3nJ(`C7m&-vdX5&8>!OIuD z7ta2Rp7ZJ#L%%W(jkNT8XHSZmpxY0k?5FgmU=W7M{SVnG*}Ts4=)bdy6OaxC%h?}tCX*i` z9)HbF!!Qtp?>t51d*URe1zAZd0qTiLNEA_Vv+QkZ5&y_Gq(C3woIr}zkr%}7KiBxfVi z{Me=kaE74)3CcU1B^kbWJy_``8+zr+DTVJ0Oj^(4=sq|x8r!x_+F5T6`7^v-zrOdY z#zGxW#Q;?WXM|axUfZg`*F`v7L!DoT6Ym7@r!MFdZ>L3uKHyOZK!&(y<#YD@vL;!f zDo^uyoZiRTe3LCm{y@_C*Ky=c{{<*dL;g)X1BTqUpP>2!vtT320tFEnt90^{FD6?7 zjFXEdJOUOKlQ0z)liwyP0lSk5CprNwlSL;c1lZGcg0b`TJCnE+G IB>(^b056g&ZvX%Q diff --git a/summary_results.json b/summary_results.json index f3ae39edf..a805f4c28 100644 --- a/summary_results.json +++ b/summary_results.json @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 18.9769, + "Performance_Result": 22.2422, "Performance_Units": "Samples/s" }, { @@ -689,7 +689,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -725,7 +725,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.438227, + "Performance_Result": 0.437938, "Performance_Units": "Samples/s" }, { @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.407871, + "Performance_Result": 0.410188, "Performance_Units": "Samples/s" }, { From c11518c93cb5d416dbba4c759f772a6e19fe3989 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 01:52:57 +0000 Subject: [PATCH 454/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10222 -> 10226 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ad4abd088..58b034b79 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 01:45:39,860 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 01:45:39,860 submission_checker1.py:3266 INFO] --- -[2025-04-16 01:45:39,860 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 01:52:52,790 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 01:52:52,790 submission_checker1.py:3266 INFO] --- +[2025-04-16 01:52:52,790 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 01:45:39,860 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 01:52:52,790 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 01:45:39,860 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 01:52:52,790 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 01:45:39,860 submission_checker1.py:3282 INFO] --- -[2025-04-16 01:45:39,860 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 01:45:39,861 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 01:45:39,861 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 01:45:39,861 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 01:45:39,861 submission_checker1.py:3303 INFO] --- -[2025-04-16 01:45:39,861 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 01:45:40,836 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 01:52:52,790 submission_checker1.py:3282 INFO] --- +[2025-04-16 01:52:52,790 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 01:52:52,791 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 01:52:52,791 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 01:52:52,791 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 01:52:52,791 submission_checker1.py:3303 INFO] --- +[2025-04-16 01:52:52,791 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 01:52:53,813 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,11 +26,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73678 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.2693 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.73443 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.19947 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2422 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.372039 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e74c832b2..6d5177e44 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1081 +1082 diff --git a/summary.csv b/summary.csv index 6c879224e..a1126e0c7 100644 --- a/summary.csv +++ b/summary.csv @@ -4,11 +4,11 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73678,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.2693,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.73443,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.19947,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2422,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.372039,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 671d37848..29c0251fa 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.2693, + "Result":6.11037, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":4.73443, + "Result":8.19947, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 20565004a5d9b1b04c6c85a043debb19aed39a78..5646841aa0d83c374581e6a61248e15525012bff 100644 GIT binary patch delta 3090 zcmYk8dpy(q`^SelADcpQYBm-sIp-{!nawE?b4W4U9CALS_{fAQ)i$|vM+Z6Hj+q=I za7ujh5WU)SSteREuMtd0;8R^U_@b3{NOX~tzC z5B`9!(=BkuTOl{#`N5Ro+Is~SHtja1w~{Xc2levwYRSf#tSZIUrZvXJMj0s;nz`lwBYO3s_qx{aVt#P zoi%;2-#v+26Bgw;Ikh~Kv9HhPqHHYWqGTEKraw0h%SoaEO#H+btpZtxxrN!Ts9Yti zw{FpNb32|HaOKb@!#R6GE^}vB#bikTaeX=??BA(n&8H64BCZ3*ZaIO{U*Fsz7*7ri zbZ=s+Z<5R7y>EWEH(XLuM1C+-#cxOrHowhr-oY2A75?=!vG;H06a8`V#Z7S=JUWJR z@z&phnNwP903UwX`WT`8PH@9aAgSQvTQR$JE3zD=(IhqUjC-A&8qKJgj)jKD!p|b- zzS>n%J7#TTUWKeh+`DZ_jc`8Pb%|Xvx8~eA_Bdk&+UZazQ#G0#cT@v)?YKt5negqB zIgtZv1u_d0rK3f6l&0bwI`x14ZQF6l;xHtv2B8)%Ao7*VRN4op^)$>6D0KJHzIT&dh>>D$MEhcn8fq<3W+aH-fqj5 zRj~8A^x{#$k>vzySpG~s61tp_205p&v$+Xb6J_%)S$U}{i@x;SoSk-bDz15C-=%K6 z1@kioqW3Y#8K(#;iQOV{MI zcv>n7%RYZ*A(K+C{Z95n@zjSo#M2Y+AdBk^O&aV({_DMFhjJSaKu;=6M_mTq?)Le_ z@6^~mHN;)b1sq?N7&bMIMHgTE_Cv9#OB5=w)`^TX(2mDOw2;KkOrOsl2q5M z)5#q{p~Ta{_{pac+4<%GdKkT*A6ex*Cz9^q#EanaXtuBn1RI53-R0enX?V7DL@0Ed zH_PSikeXsoVabgp-y1p$`2G6X&%)Vdg&~t)ud!zen)|{QPg-2|z@sj~g-kc=g*Yn? zBenLxdfMX>+`wFocN=10Ehs;nAEu7vGt$YNr;%Z|ksq&{0;`7dsW7Zcj+WifZLIZXCC#l@AT_#G|`jaRMJRUty8QoQL1l5R40RS&(n32b%~wEE-!cq zx#CkPSjRhBra4<5ai5~`Yhu^^gI~;4wtIe8G%R~Q+{_oFmjoXMWtUu~Mm&DxP6sCu ztdg_oheV5kA(mPm!o_@>QY|6Q&{f;_BCFB0EwUpbsI{+QG{D=8r%xkLOEuH8`FALd)*n&jVTRo?u)8JU-ok2uQn-8`ZK^;UQarizdK*j_)Rzpd=>JR7_f@vR zQcBX1c4L^>{eNmkBaSZwmn8wm@4Ak%%ed9DfRn z;h!(i3zJu2kdkntDmnd79SpAuQ^hK1eSt5hrqJHlxhUKJ6<3%~&pE{9QEzHZKbrAJ04^K`sJ4Azx?&EIv}-y{-zj!u;y%=)k5`*!8# zbQ~Sjqh-te#BIx>U^2tOR}`FYtR>RsU#9{ZNgVgh5T_{ZCf^j(3wu#0kJNf%U%;y9JFvL8-5686OsdoL=#uP6oL zWuhlm0)vt}Oup?~nQY*j zNx#w};)2<=ACV=V6?a7=vs8~|i z+JXm@nV6i>Z=Sv|Xt>|hrG2n)GUlhz56-+^q!|C?yjNiD4I|c~wcdTswv>2SuO1y+ zMDyY<;c3}z;EFqMFTpuETi}6YEjI!Up~B}C<<%Ck&}9Ac%8A8wxxXw3A8Y8)*X4dc z>c-G3O~jC++OfT7PWtDQJ7{jo`RszRG}7%ph?{Z|j}BF!N2}&vu4Mpi3{4sPV|hx= zJ;7E^%ikl=y=OV2!W@vy85~`Sd{p0JzH|YoTiS{l_?13>l9V6XK?~U9y-g!Y^+N)< zJUyzwke1F)H8?4N_Lj&^|KGkFhx6VKz*C}0a@80GD{Zn{@W!^Y-h|Vz2o(nOEVxhCF&BHNKdX{WCtx~PjLE=V9fsu*8ER!8cmu0 z)3Ejd%QI;xo*cqE^QH^+gMR+Msw$K1mNoeoDo48QLN7csMSN924C~L?7E1!dTdXB( zJt2Z{vE&FS%ilDDiIhE=;I0lL&_>Xdkv~8Jo^DtD?~ryIee0?(-1EuU;t36gl|Vr} zYkk5lQ1fZh^u~6o2xGT9$tjeFayfZ6el^iG}xm+*< z6U=$Z39En_R+PzF-7ni+88eYqo+kqz$}I+b)`s%RXTIOPL9|x=75U?vt@`EIf*kmY z_<$^<&9-%r(vA-^bR2YY3-13sEAAGWsrPBv=tdXNyXm_6Hb_&J@oiY|%X2S@z!Qgl zHwyW*er^fdILFC7`&{ndJK>~)!ddLda(0Wq@3#%AN`x(3CjoG4+4RN zxR04Fogx9A%o`kFC_y!WDe|d(b$lVc|6D!gjc<}>!S+2>vJK4pnXrJ)1wUJkh+55U z)GcvWCP?olN1rN#5HJ}~CtT#C4S0N&A)7BFw>V?OMiVhCtbjio%deY47OiqSRPB>= zo!9o{OB_{V=1}qzKf&9h@E4Dg8%EqtE88`9r`Civ3Fa0riD172SuT#@f;ak+NL4yh zn4B0q!e`VzlmuQ-9H5Jxd!|$##T0LeSjc+$K&%fxI5%%@eB$6#o&6*#RD4_aXJf*p zm9v@GmK{1luKdIlkB2@}|Fpe98F^Xf&8AAPg1>jW3=wC#P^sqcorNq@8;A>jY>ro` z)1BgN9f#GA!LG$}2R%OdH*PK4%y7;nMJTL=&K=aAZ`}otG0&;K=KaqNud8;J=O*L2 znklc`0fzPg8OE%d22Tq^MBSL@I0L1w_P>dSs>|?5F;dmddHflj>Z$@VaPGuG{|9>M ByH5ZB delta 3102 zcmYk8c{CJk8^(tuL&9j$RA|f$F_tv;w_$2%7HfL#W#1AZdevYiRCZ=YmW+K%3^I~6 zTXtf`k}M^Xtw@;SMc?Q<-#OoN&hyXn$M3%G>%Q)Do*yPFCKdhyK+X3Xp5Fxl01%BR zV100Oyq+jTdoN%?N;ur=F>b_`%5#*-yn=q!boi+;cpRD0QKgePVsVm_r>fQ3;xH-n zA)>JznF=d`nyDI^Qqks=d3WK?(MklC;P$mEM})_pkr&q5eOL+oCgCYS#AW8mdS>;Ll0Fl2Oy zudeimcX%fUkr*|JXHzOa;nnSKt0CEIxA+UTvXQYZlV{qr))P-q&ufCO z`8^ceDmXqjRy{_ZUpuXka!u&OB#EHar%b;AQm;fc#$HIi4u*f z2}6ORG&dC?Vjn-A!&hn6dqc$Be~MEgnGm4$glUdi#K{ zx?$RMrbp-q%*br@6hv(EH4J37s+06n6d_e14j(s(J%#}7o}Q9Gd=PQ&htJKO%X6465=gTD0GIJ?53hc zyzDMgB)6qMNL?T|pz-)56iuIe6H`9qM^_A)Nw=-{QK!lYB*&B*R43mIzL`}znB1dtS$0v-D6#9I22rXHzw|LgyFV=yPLO(6 z+&S!tl)s-hItqT?A%YAVS^bPMoxnTguT00~A2=N=NYY#DnYzUKu@tZFZJgVQ+epOt z@iZDlDX8e6fr(fs6$7C_K~Yeaf;1(901|4ol{f;ue-P@VnYe%h;bK!wb-Dgty7Oh1 zh%L=c0zq#Ob!bjQ?6N1!qnjkHXjHD8^!+%HrCvZ7NRNVrMq?mmm?&lK`>Q9iv8{7P zCg`Bv6WMg`%og>O^j!@PIVYWam;3C+=eQVqMeiXW<=CI_Dc46j16%KV+44OK=;_)f z{cQVxo)&s@{QMEO|c0Mw$$^#ge1!hVI|WHcyY+rU8MEUJV$ILAGJG| zkI~;E{A7>~&yX`W4i_Cg>-ZyQ);-wmopT5_;rE<>AffMeKc_Y#6^yK98x|Fvu zy0PxCW69I&{ue7YonEe}E0&~%!$@C4OTVA5_a-3BX0<_tA2pH{P;fKV13(yV#?-P* z5jfuFN@+BqJMN)5$-|RooOCX02W|EEQTIt&q(jf7{Ir%+t(v`?-A!g$Ccy2OT5_7^ zy94IS?gN1Xf&MFyV;}EpThydG>wpZ0Ys&b<)db)yKq}^dLoR>Iu)YJ858#Fl8~rkU zAhoiQ-8m+C^2YOacIh5`Y7 zq_BP452?5k(gQ9vCpn%0Ju-*m;>-&-Nq;(js>+x;H#FjQVfB+qJWxNUq?rRtd$wkX zgWpUv%;nivi4jKayol+70?B>W(F*Y3aeVTmb4+8p?+;xVyb-Qa)oieeUTiR@WAlITT}!)K`|GLkqZqYh-tNzL>*&hNsH)odl{+y19oZ z)2yeT5V@2n7Y1G9G-tTBcVwJ_{Ej};hZjT~oU6{dqOn}(Ij^~#lyU$zjy}D(a29N0 zb3c+l2jRHn2qse{yZ}8Ady!?nY=l2Y&a*-5VY2!um>!T{67(>}(&!s@xwtjn-EgVUZXTwWb9lD`=7Y1I*&P4%$Dxl-C6!vOLzhA2Th z)c%@?>hfrYU?sMJ*qa;T*KTpp2-54E=`?Vt(n$4eXR%l3z?!%R&AFI$eg>N^!U|%4 zxr`*khOme<1E+2zh;ZFy-H`-@ChG%jz_kKRfgY)Jr6_^qiaMwC0cWedq>x^G zG=dpBGGQ#mSk9cAqZIphVFn^)GEkUVT#*P%It@lg86I!}2`5^A-f-l`08*5|UnEk+ z4Qhm;?{=b!RcGaJMf}Jyosd0A)vV%?`AfHLL)b5G4Xu46(Z7i??hIlkB4rx)dF5Q* zQ&BqLm_ia5wIw{~% z7hkyuunkAc@4zvg$_GAP{&GR>U%Le^P2ICD^n!ai##YNC*Y6F*M=EzMf8w!+Xg|nD zs*nd6#XpGf|F?pV>_(Zm5eh`Sb6yXUFh26<(63s<-lg_P;J#CkObGp{wH*C9ceZUarh8 zkP=s>baxP-3zEW(A{pVo<>9-v0TAN#k%22|6DQcWa|{BY*+XV3hpS%PiSl9#IgN`s_*iRI2;-!`fg* z>5cEHvF?{;_U>#hVP%Pd`Dw6a;kPGf&Dhri;cYg)8YWFSrDIQ7O{Yq4fIa*2{3PON zt0f;u%vTml@Tok{F;ea6VJG#yJ7NvNEaIoHO$%D(q31UgW(To^rj)`FMPfIq!tuvpAkLh-i+`5jXy zel-Ib(4APS`S_e>+c>DgA^%pOci z{u+Vw!!Qm}ZDdLI?KnDL!;*i%D!ul?GrO;Ur|*QVvlj&CPNIPxJ`ux44^gAN8)7h& zilUVl^4bDGQgP`q)wU-tT*DnvBi$k^mVa@s`cl0pqJ3=rjNBLI!l2nCxX3E9z^lS_ zCtZy!bm@x~+JO{y@KHkZ@DWUafgnoyTY?vnJS|~knjPMGFL|~^gKQpJm1F5c$3F^x zXtu0>dXlJo$oXBMwZ&9}*Pmg(jFzdyVN+>$4nmH_{Y=P0cYj^?SQ9g)@MQJ~>u;ivY^gz*3Xrn~@v b7}w_i9rA<2v}jorj{~h+R)!x0+kgKLmuJ#Z diff --git a/summary_results.json b/summary_results.json index a805f4c28..3e9fa2f61 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 4.73443, + "Performance_Result": 8.19947, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.2693, + "Performance_Result": 6.11037, "Performance_Units": "Samples/s" }, { From bfb00bd24bd4f214ede99666af6050021962f9d3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 01:58:39 +0000 Subject: [PATCH 455/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10226 -> 10223 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 58b034b79..f88efc5c1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 01:52:52,790 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 01:52:52,790 submission_checker1.py:3266 INFO] --- -[2025-04-16 01:52:52,790 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 01:58:34,128 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 01:58:34,128 submission_checker1.py:3266 INFO] --- +[2025-04-16 01:58:34,128 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 01:52:52,790 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 01:58:34,128 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 01:52:52,790 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 01:58:34,128 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 01:52:52,790 submission_checker1.py:3282 INFO] --- -[2025-04-16 01:52:52,790 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 01:52:52,791 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 01:52:52,791 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 01:52:52,791 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 01:52:52,791 submission_checker1.py:3303 INFO] --- -[2025-04-16 01:52:52,791 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 01:52:53,813 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 01:58:34,128 submission_checker1.py:3282 INFO] --- +[2025-04-16 01:58:34,128 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 01:58:34,128 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 01:58:34,128 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 01:58:34,128 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 01:58:34,128 submission_checker1.py:3303 INFO] --- +[2025-04-16 01:58:34,128 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 01:58:35,114 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,15 +24,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.387848 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73678 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7355 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.19947 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2422 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.372039 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2422 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.409159 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 16.1275 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2354 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413841 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6d5177e44..15f05f40f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1082 +1083 diff --git a/summary.csv b/summary.csv index a1126e0c7..274fd92eb 100644 --- a/summary.csv +++ b/summary.csv @@ -2,15 +2,15 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.387848,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73678,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7355,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.19947,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2422,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.372039,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2422,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.409159,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,16.1275,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2354,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413841,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 29c0251fa..6a6644147 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73678, + "Result":1.7355, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -304,7 +304,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.372039, + "Result":0.409159, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2354, + "Result":23.2526, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5646841aa0d83c374581e6a61248e15525012bff..32f2c5042f99546839c8313568e139ccfe26c0fb 100644 GIT binary patch delta 2614 zcmZ{mX*d*W8^>qt)G$UIO`^tZGM1w`60&40!&oLY)-XlQC?phv#?rAQX3SW_v1CcM z>~g3O$Ck>zMMc?@u~WRG_j<4M>3u#tU!LpwU%%&f-}iGp9Lr_PXFftgPS=-KL;wH? zJ5Y#3|9X=5&FJ5OW;^3b>!*(Ypqxg-^{a80>sxh)Li zsQ^bU%~?j@WCiUZT3Ti>ZeXv&Yqao>IMJkFXGXZ!%HVZ*#V zcSluD3jYoS39j(Y={w5H9cp&ie91n3NE0~8wmsxaf0^!mxwtKlWcIK=FLBv6T{GEw z2IlwKJ}?$y<2Z*x1Lvb~7mVR7O_;dNY+!AOCPaD9pN#*-q z5xmh(z-VbvIE{sa$RqpTqKWzM(gXHoAw}pszUZNep4{Q8svBQ0hQV<;475m;)8p5Y z;EXIjD{dSt-sA}Jb4=XCEZyv{l9sM&kkkeQ`7do>c-`$osH?b1AMH}RTkYQ$xH`> zKDJ|7#`*DjkXRK?!z4wTg$Z@7K967DJtFLOQ(vBAO^t|-JY^GJNt&$~T_C4Ua?68PcS!vK@XMpI_t*e_?|747K zI9!Q_4#nHD{#g$}yay!Vcx`h8ke#s+SMU%(62Gv$sV64=yjhxW4qsYY^zOONb_6gQ>7O@Der#$TsFmCH?TCYl53!Q#8crwrX4e@ zdV3|5Fw%?6KwF!sx!FcW03V^76)LwGFza(vB?rUUy!`I3QmOVs)z#I)^@os?{ko(& z8_z%pp}+109~Te#9*~0Lbz%Js2K1f!Kcwkuc=vR*l6_uP?#=rX=+q@bYSNsd*77II zIAO!=qE{P;)F&$hJkp(cKo~91jsvl%zr}N{rgE*O$;q`RD0%b>-wf5e7-6YSi z8GQ>Sl%M2G5lBQQ1?18Px8fO^!*0jlgs)q^N9^@vsQo9QmYRN10;!l!17xTRtB#y| zjvzVWXN#iumRToc<-IEB<%UM9r9n(@Exl9CVnyXjE)y=zjpGX)gUnGatx&U2*V|f` zdy+?Rg+J==@g%(Mf9+FRb6NZ8H}$24>pEIU*;OFEDnz5V)d2ff)e0B{vukc|!G^$p z?=#^IJ;mWXvjB9!&M;Wl(yS@eN@8cDS=2leO+ODW-*^@4*7_pj_=)`opl(wUOzU(_ zhVhw&(Voi6Q!0X{zTq0Z@3^79YhciMyXL+YEC8`G07x*bksTD)vvXY}_TKdoQSwkX z4T1d7f{P55zHSP?G4w!5n|_XqIqZ{tC*xd_JB>79V5=KxijCFi9plDfH^HC_cJ$_n z7OWoPeI2V47Aa;1xF{76bR;DxE6-f>9j0oP1hb!ldNRBaZo3zx+jM$;*YT-G z3&Yyd(&F?#mRaUKc}D+4$$RMFrtoQE?*YjK`RA_C4n&`cvS0k0NCeYS3PGVh+t%Zb zLfBXmcrcrM`vn|k+cN*aLxVU_Pypp(nUq+YKw=Qo2f=4aRB&|Dtpa>x;bKbUrCB;?LsJ;kngK<~E^3=QrBe zJX`r%T3lxBxg0;yN0ejUxkHf@?YZ@nP+aBxp-3vgJO_x4p*QGYfLV6a>mxt5LXiH! zCa5}ZVe)aPz)gq&nD9t4F!f2?(goK-ywBJo``nN6AL(O+58WpvF3ZQQ?SNxZ zjXJn+*QD<|svssFt9LDaC=vlsi)qk-Ht4+LP7_$IX~CfRRxiBJySI3F;TPbW1HeVL z`6GJiWcSF+eiD~Z#yfZHfNy@RNooM=E&;l^w2*?3A zXH$bXp=+6Z(>~Q(v&Ul?);6PwvYBmzigd98ee7Y2unxF7jZCW}#itDp1fF<1u(v*+ z7%j%BdUnMtqaV=;9cO8GfiZry0n7pXvLaq zYs&&f$wtssij2|J^zPHm#gfDr{3Zpu;jAoJof5n6$fx4G=A^N!LhC7VKK!%OLIc~| zyPQY$>vF0PyJ>4IxcW_Xt3Nhk%I+p1OyB;A-Q62TRW#5Sjt`_KIkQ)n>f8bsF4&wj z(yx`njnuB?yMQA)ekLz6Z#vBctT|W4NiaqE5}e9H*f1F8SOEC`lAc3_d)B*zq;L1J z-nbG$mOYB@+!~j`rov?C^N_}6^%W7r9_v=>W66YBKGBk*$grc$Re#zPfoxS?83J=< z$oV+IWl+;_Ij_k)>!X$a_zlHEvAmMo>$P}y?Pq*p1)%x3aeW) z2S_JV^ujoz*YOc$PGY3`jtGnA-lm@{1&}4-8ubi|hix?;>^p-lb_RQg>7LFL^_Q+@ z_Y_FeaBT-0ir!JiHdmwGX!z!_0 z8)h-%T{E6{)ULSw=ANHyz!06kvx`ZW5Y)@u-IKUTuXL5~;`#S;cLr$#jA1j7+CWKm yH&P#1&lW{#0x@hn3VH0n83G6ZSn&V=;(&iwNGOQ?0Hp&YvpZ1o0uqRWlm7s~ZN!KG delta 2638 zcmYk8c{tSDAIE1H%Vcc`DaK`HOt?48gk;SyhOtMsq!^4P`;y9xC0pFgeC6_smdTnm zgODxCl4Tf6k}Z*>2t!<{-{?N~dG7h+{Q3I4KIgnY=Q(TE8`jlf{6g&NV)h{rNbDR4 zBn1M%s3X7~eqHiL=A8GJ=@#UT)$AA(zP1@l#jZfnm+ylko$MnR-VlcS1e}?aTvyf? zb^Rd4I1__xXKFU`41?H|4!FQ>+WFm|dn&1t)_j}Ct`eO`O)b>~%8 zi?*YQ!?kw{3`{#sb#KRCAs@x$V`@n{8KfxPvI_OjIHGZAV#4zV$!6%aTO?HrM?Bqw zWyyNi{b*gcQbvZ|y&UDNXDTiz?BDJ8b3yCNs5FkRueGZC`P1kX2KCOyDZ!sTu{!_& zMS^F6M$WFxn1!l=g#^~QmyO&M-~t`4-(~?B0Uhuf;3N=EwjONw!|}ac8Par`^(som zD!;EZjH(k-(PEmaT*(&80!3iuOBG;v8ET3$yT#Q|1}fRsY#@>NRCP-7UGdv@^XNLY zDcI5`piD)m(fgQa+oGYo#v}5lk57kRd$Zr~92K_ zcSYKi)3eAhw0ol~+Qof(xM%>+)=rhBq5pXB8Xdypt%wH0Qn55M1Pv=sMa1wRXuhSZdMu+yW7oq;G{SBku7xcjWWF>LaRkgdGO|NBr?}4fdzQcd3zDka)2*|Fe*pR9gtVYbICE} zjQlqjxpQj@Ep=>*vb|n$jJYwkiT2)`aFe1KmP>t_w1s8v!0+mHmHz(zV&Rg6)>3kL zaMjfIf#;3?cp~=pKt&D^6dvRuf$$A%y$-Ukuf;1u2uH_%yQEE|jcOBJe)4%Z+Ip6r z7$HyQt6vDz+?4&XkYgj>bSU5=EmfMC^nWs_ILk-jH_KO0)y4X5`PCBod!U<6AL@}!q%>>YHX?35%gj`RjU z>9Ui{owE9Clc;H{)K+;Pw5V}vJ+CZK^7i9rmfZuMq1`V%^;J`4MRAkSoLc&_2eq`X z{;DOtEN5flEjtcXb@p1aWd(*E`aMw7yzwg3Rt9F0Nk+`bHZaO38W_e47oQ|JttX#?mWtj}hA~Uo& zN%MXLK|gOdrU$Q^tLQdFS4?znzBb+^)LlR3nwGL zYj3d^Fkyl`lM8NswdvZdC1cDz_O7Tq2H5qoYQIWSEU zxhYn|hk2!|MjjYW|1ymH*RbXv!D(E=^sk0hj~JdyHUZ}nr}UdHKV^D){iv#p zH~*x}vsgLOZ60*FSr`4~IC}Whyjiix@D6L)7{gD57K@C4vb=Od8Cc2FH=GnfMCu4G zVT8jYa~?z&ntAo#L#pkxoj4Dq>yxo1H6_4^w;+bKIbrUnTt}JS-Yxhx&|17n2@j

s!71YnTYGTu@E8G_N7;{0=CS*}Djc z6#PWU-7o9LtwmqYYhqP?zkMNtToEN}mEbFt2LbQ_><)Htg}hcpnWWLZvi+5@Gh*-s z68T-Z!JzxbP=5K$*PL`8V}&1KTdQV@*P;q?kt;%jl0dszD>I?fC0NUvY2z3$@Nrhi zF(?D`VOTr8tM{A3+8cjmNnmvt^SRAU*iY@)&qjXt){ia0+ZWk+=i8)g|GpFATTnP_ zG4d(9#S1=B{ipDU=}TWrj|p`%CUPJNwKj~^@pRuU*;k88h`f^qg>9_XBs|P&jV#SG z9V<#SDAeLfP$jIAdxU#G+_;DP$-}rMmMILG%x4Y)aCsFn>FnWp?!ey1ac3{MG>Neg zhwEg`{PMrwSj6Q)nm>hxujMsrlsLhOV*BwCXUkwje44xsA?)EcGNwvvojWbBIBmpK z89fX=?s7hgMjH9<{$ zdBw|zEWT!u?X5%j(g(0u1sVfNiVYay*6tjlKo(^NX@VD?P`2)ym58qe8$aBRxd#gJU)re58U5AZhvi}?Pi=mn Date: Wed, 16 Apr 2025 02:03:34 +0000 Subject: [PATCH 456/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10223 -> 10225 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f88efc5c1..e2066488b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 01:58:34,128 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 01:58:34,128 submission_checker1.py:3266 INFO] --- -[2025-04-16 01:58:34,128 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 02:03:28,924 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 02:03:28,924 submission_checker1.py:3266 INFO] --- +[2025-04-16 02:03:28,924 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 01:58:34,128 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 02:03:28,924 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 01:58:34,128 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 02:03:28,924 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 01:58:34,128 submission_checker1.py:3282 INFO] --- -[2025-04-16 01:58:34,128 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 01:58:34,128 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 01:58:34,128 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 01:58:34,128 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 01:58:34,128 submission_checker1.py:3303 INFO] --- -[2025-04-16 01:58:34,128 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 01:58:35,114 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 02:03:28,925 submission_checker1.py:3282 INFO] --- +[2025-04-16 02:03:28,925 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 02:03:28,925 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 02:03:28,925 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 02:03:28,925 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 02:03:28,925 submission_checker1.py:3303 INFO] --- +[2025-04-16 02:03:28,925 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 02:03:30,301 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.387848 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7355 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73476 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 15f05f40f..ca298e35d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1083 +1084 diff --git a/summary.csv b/summary.csv index 274fd92eb..66b769691 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.387848,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7355,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73476,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 6a6644147..476be0278 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7355, + "Result":1.73476, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 32f2c5042f99546839c8313568e139ccfe26c0fb..330990f1ca54cc6efdbe2b65e8c258f134644169 100644 GIT binary patch delta 2650 zcmZ|Rc{r5q9tZGeWXV1eGMcf?B6-C$A;$LF8U``ZpuE(Osg$+F(lC@QVIEtgWn}5K z)+oH$h9V3#la#%(WUa%DMAVr&=Q{tL=a2i(@AbKV_xHZ9=ZovIYi-zeIWg{r5?=}c z6psMFE&zm}mB5_sc-l(l`@p5KW>oCw+$cMV)^TS{XE5~nWXJ_FDUuZ^%JO}u{_P37-6?V}fz$Pp)f!(0eBpIDXHE{AI zF#UMWyboQ)%lK+IT+lUl(aYR^r;Oa0uG2rKjF$#r4iddBHC^obOAD;5oRtGIKVMw= z27;0($OPCZT=%jjBn=_5(#zZCtdD7{OSk*1O@MG|Q^*&PEFD2}?QhWcoOG{1*^lvh zF6y}y^*#vCG7YV2wolnxEl|t`B5k!EXe$ulS;>0^%|2FeNtMO3|aRrtgQ_humtnY}M{+)7{FYE;rXxO*%jdT5K|8}<6~oD{4JVmd%5vHjc+2$ z&V620cJx6Vu4HD@6m)zWeRRBi$GD|^1c6Bkf}x{-_pMH-SRh6RqL5)2`Uod&&@$S~+wL-IJASQxyI{fPb)7SfpHy5N&m|j%mE7;5~d_F`$PEZ8t>{ zEh08K?!B}whf_XcczF|b^m7@`6)5MmG&VJ%(8ptqcX>H!x$=IyLX2;<#1oN$ioFx{ zVYd@>f4tiwe>izzB$v$Wj6~*6g_`>jaB(OukXs&~5>|YV%n*$Wu%_oS)TGN^@z6y& zB#ZTgS~*#8KY9l{yV0X9{8`wU7EZ%J;Bj*Zho*a4Z+~dDMCstm+Y7fBrbg{_TUH|~ zLyWsFAoI&Zk4vIYLy1g?L5g^rtt|aPS{qUl;Lt^@`ts!cdUB2C665xCm?C9v|NUqn zY~S?Ria6T7e2)S0jP*j4Utyu+Q>vM660yZ8sxpMny@0w&N#KDLrg%3JrU3@U5eMGW z9G`gEq6q*+85tDZ5~D$SQb)k_3^0hQgZKR7(5WFkXMYsGIO6Kv>*;p#!5&tlUFl$C zaBwjFGrik*HACrh&d6<1#RJv**g_rw2cWq?K82>(`{C_HkN=sjVCcuV8~xWCIsu2KpD29Fhs2at?7d^y%N{}r@{yk z9}Vame>VWl5`*M_Wr>r{CU2U69@$4z3DFl-7aGS0udL{U2QA=Nm7xx>$mCAY1L{!$@u%Jb8p{J%@H$w1V5f2h%@1J+Td;r zmZ7p(ulDW^TpKd<^&yFPT`a%WC9Ye`EQ4{bkQ78^x%KopCC_Klbm++CuWdyE&pV6Y zT0c|a97icVvFRsw9E=x+`}r*0eG-m!2_`(QFtknpmHpGJeV_vepbQ6rY`Yf3prn=8 z)S?^jQn=I~!A6d83Nm=4luFIB_Jp*zspfc^dG^bJq40Ybs?2jwM|qBu$d*Zt5$Ptp zQDI!f8dTPu=G8vYfwMw!8yG!WY4T3M5#^gPI~!u|mN=VgUJ(h|$TScrW)93^%HJ&} zkblJ3u=gL>Joa~RR|d(|)n$Lz;k(ZLCHB3^+OZb4lY~3+Tp*9A-AI9Tqh24ECmq!d`{7AhpOH^Qo~G8MM%Kl9B%2J&!o1p67jH>#Cy%2Svj3 zh&eXs0{h+6RZWa=t1s! zQN9kvtjD4AV-H~(!X^FTF!8er4p3r&Qeo_soqd?#SD3c=AK)6-Z!KPU<3)*LznX6qs9K*< z{cefB>*mjjio1*rN9#SeEasfA=Sh?lZA1NIzk5!g3H!@bIJ|p$9|ZoeIL38{bGfrx zW%3K6clE+HfP!A-F`jcEsY2 zpk-(+0+Hz=mpK(+#+$LhwbZRcWktN&XM|-$s>%ai6uy5EkZEV6E^DM7L4sZJW~ z8JiB^(2A5guyU|5W+h&5M|3KD_1i2wBZG8wIOpfXiifn-{thZsHUad*rlkAkBdR^7 zN+VabUW7lUHAkKLn1El}cWbq4$O-We_(lx@E}sp0Myno&&r!bIz_BRJ?DZ20ySmpd zhmfT|&&;Xzq*keJJ!Q{VWp(0VNnDflmU;8&;OP@Q^MY$zdA-(u1WsD2uzOXSRr-IV z-R94$*c@)f%B9x{qN(0vkMb6`sSR@3O=dC{k;NY_DXW2gnaZaJ#|m~-joUPC-b&<8 zmkEvzzcnDV&uzID-yM->E|v*O{h-&nx8#R3hnimUXIoimikWiT)@v(4gq8-a&!xiv zn)n58Hx+#X6E9VX%mG$yQOH!QQN8F9?`M{eWT_Jf4E43=e9B~0U~AFw#)2T;f0>thq*c&fGD^&%1UosGN(y7Z@8W)n}_o)bx`>tm&>=hoXCe)1PRP zQy+fvIm|i5y;s~|eR8M2)E}wq_IR?dcNQo%w8H{ctneE_0_IyjMv#f>#^pio> z@p6byUN>CF0hjd(*@Uo}BiO}1jy*;^?{4OYVJ{rdi=6gLUk~4GjJd4l>0I*J^*Q7~ z&iiaLKXyZD`zJg6vECSRr(z8%xhr;{rU|aB`rQg?2*4*1IT6bo9o@tZMJuaJ7ouwQ~OB4uhGNP0>DemcMq7h~)o%20e0qD1nAb q%HVsnE~E*R#+X9%L3<4P_jJ!=R3P$T62<~@670liiz}f3;rDNXh}w++ delta 2634 zcmZ{mX*d*W8^>qt)G&=mbA-lhvL5C*vddV8F-*!HCX`XgRty?T$C5B+tl?O)X3M_3 zR7@OODmx=8%AV|7@lL(hdtL9B_xbRA{9nKOci;DOJ?rLc=Ffcj_{AM>uC5CJ01!qn zpB44XdCu39{|Yuen3CVVaPB+h&up&K`1DI}`qZxW-G3Nx!ge@GKr zYVY5J3`8&|ccm(SP_}X+y-8a}ngS88jD`C@F(=WuFNg7lQ@?Q!c{vEpx6A!-0~-!` z(eKAc#Rp1ozdurTS0}p<`9p`k23py>jVJc%&++j_`X~kp?}!dLr7$mylr@_y^)$PWVMg25j$4X{cRI(#R^LX8y4IE_W|@oyABYy5=%8Cg{3Os%u;740V&O@~@K`Jiw$99eXhMD>S|G;p$tw|X zb}p9%I}Vm)e46+vE@AqR8%@6xG(@!oPj7Iei)-@C)M=)~vqSA-ZoTzr54oV(Pu8Za zI@XEVHr&08j{|=9cWUY$ZkT`rzj~iq7|SjqXC8ggNyt~e7}s|f_(ZL*Dz;06mIVfV z=*G~Eijs68@roO*Gwxz^xp3DSwBD`F*(eitGJ}vTQ>@3gOjFu_yRKp?PnDF+_7wT1 z`5`3zw2IQ9o|3s@H(Jok>*{;c`A*5MTDpMgx1eT;aRG~}%8;4fAi<}5F1s#&B~EzS zUyp^3CE3vb*$zXz1EgX(ZSr}Lz41}kaS%W%H@}^Umt58*(@g(-{QjI!hdKs7kdEcD z{9*-sbE>amUwQU&X23R=61A+Py8l(PM7o_$aAj~M!|zP#-sn(u$9r*Bl5>7n03y9a zPbrq?B^(M6{)4Ybfr9}DD4l@Rsp@El$=FX82y+yjWET_owD)pfTXJm$$5ETJ711Sw zw_-Wm)9b=&(Q1jf^1i}ib;oNXR*EYfWMWQ_g~A!`VfcKYGBJ`WJ8|Pw1{_GUrRCP| zuZQC&29eolOH*Yxo9HOuV|0g1?S3N6Qj08aZxCNt)c-{^!)~m;zMjA31af9r$EwNN zD;Raq=|4gNLl^!JX_hk1BTKpBm{W;e`EW9oyo%3AU6j>W`)HoP zZ;)5^atD!7vW~|gJ!n4(V|lu65DD9+iIqyWOuT>=OF*>aNR4>2Eg=wQ7-4n zH&A@_`Hfk;712=!x%%F%e4b)|*z+&^mp%W}cDmx^;qy>)^?(?j47raZB&ajJiBzD4 z7numKK~V>*ER%B!UsiLnLt~XQAtrZL-zw$OV+usp@K+b7aHS3*W+-MS)HK}HSL511 z+61=rd&_-}Q|}_shGRt(EFQv6|Mm(TO&}#H$Um*m0O$FzB)^wPTuz z(M7y#qW8k0g-iigM1w+3r-$SgnyH_^%b|2YQ>qiD4ZEXC#+M^)jzVw`+sp(g@^418gs|+3;IM z!_y=8af<|%Voz;9te8#dDta*#qZay$(lufz`a%1p)WWF0$}7hgH+O0K_zLYa6xI?$ z;YMacPNPbi8vGDKv&kJ9ZPS`pC$D`V` zv5~H+-wu>OG#p0vM$%X`0-zk%sts+`e#_n_uyXsde#f0bc&X1|`S|iLz}G(kR~TlG zsg*PR6EB9X*o5Lf1+1ThJHH57|0SII58%$b4?E$hs*7Bj+~5%sQTN8#+QiE7l})jX zhZvqx#Xh1r~l$A;o$wmc^&TV9?Vi7hXT{>aEHbpiH7XqhWlPWixH z>ELHesQGx)%lrgMD31+916=MZd-p)o&`r5J?20)>!XD>YDEf@0nzQwMV%J}SMi8XU zUDFu+tdDu_zYhj+*mito4&X;}oUTz>s!*{W<|HAa2d+vXQJSohGDk;(rQVDjZ7-$7 z3T@QAxHBqwb1iUzV5Jy{@2br^b=*t|C}6ay5BAfydFJDA&hVM}c9MKkX7Nq5Y{QMM zHJ-9GL+GZvxZ&*F;nUrfij+9qt~+$cNrAUMJ^t8%OVMe|QSDHXGEnAp@<%U04Kp~n zR!9!$bF2}%ZDYi{`BiCeI6i9D_BK93&#uJw-YvsA3h48OA7mgcXHbXi#Dwz~@6H(N zHA-S98n=p^!BIUw(pG4<9p{6#oND8RX@Z=|j#UgPm=v|ogQ)~I*!>~!q)na`1b(op zYhUA$`!+fC>jR7rwt|;#hoU-lCS-H9F8TVeWi_jm*hsnw|A+{?<0bkB3%#jZ zvZX?W71ZjymCVtwP0Y3H{D1o{$!K%Bs6gIv_e{iKbQ%@`StNJehz-G;xTq>eA*+_2 zC_=N3$wzY@WJRP}OCsS>SA!?x?x?N)(MhV8({#&Km%M@E>tRnLR667=sywfLV8O&I z^9H<`vAZ05voZ_9-f0F0+8>B~x(7{>8@~rrrV*YNDaHFR!n{wzRN-0M^6uM50TMxD zRPNqB8daEAH|Ow3_zJbwMY@;ce|~_fNNZpmBLS%i6k+rs^?)r55K0{=$1p}Ae~I-$ WodJRvu_$dIiP4Rc<`G5!*yrDEtktyu diff --git a/summary_results.json b/summary_results.json index 439b15406..6c443f008 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.7355, + "Performance_Result": 1.73476, "Performance_Units": "Samples/s" }, { From 2187c71913cbbb561220471cbc786048cede7699 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 02:06:03 +0000 Subject: [PATCH 457/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 12 ++++++------ summary.xlsx | Bin 10225 -> 10221 bytes summary_results.json | 12 ++++++------ 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index e2066488b..64287dca8 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,30 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 02:03:28,924 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 02:03:28,924 submission_checker1.py:3266 INFO] --- -[2025-04-16 02:03:28,924 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 02:05:58,778 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 02:05:58,778 submission_checker1.py:3266 INFO] --- +[2025-04-16 02:05:58,779 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 02:03:28,924 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 02:05:58,779 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 02:03:28,924 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 02:05:58,779 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 02:03:28,925 submission_checker1.py:3282 INFO] --- -[2025-04-16 02:03:28,925 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 02:03:28,925 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 02:03:28,925 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 02:03:28,925 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 02:03:28,925 submission_checker1.py:3303 INFO] --- -[2025-04-16 02:03:28,925 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 02:03:30,301 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 02:05:58,779 submission_checker1.py:3282 INFO] --- +[2025-04-16 02:05:58,779 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 02:05:58,779 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 02:05:58,779 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 02:05:58,779 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 02:05:58,779 submission_checker1.py:3303 INFO] --- +[2025-04-16 02:05:58,779 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 02:05:59,877 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.387848 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424071 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73476 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73914 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -32,7 +32,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.19947 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2422 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.409159 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 16.1275 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413841 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421143 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index ca298e35d..fd1f63c24 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1084 +1085 diff --git a/summary.csv b/summary.csv index 66b769691..2cf159440 100644 --- a/summary.csv +++ b/summary.csv @@ -1,8 +1,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.387848,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424071,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73476,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73914,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -10,7 +10,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.19947,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2422,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.409159,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,16.1275,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413841,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421143,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 476be0278..776dd2449 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.387848, + "Result":0.424071, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73476, + "Result":1.73914, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,7 +339,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.409159, + "Result":0.384271, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -634,12 +634,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.413841, + "Result":0.421143, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 330990f1ca54cc6efdbe2b65e8c258f134644169..8f82e2b15922ed33f8626966f6c24d91918ab624 100644 GIT binary patch delta 3201 zcmZ{nc{J4R`^U#VJlO{2JGPOTu?^$V*coInV@v8mD7$B@HP#yHV=KlIW;FI^G$cX^ zWhp&K*~3gKBxRe%Qb`d~zp3;4>wC`k{_{Tfxz6i)zu(t&&b@JL{a8~pSV%9JCGvrm z0QLfwRnyAuC2Z5bba<}>D}5&$Rmitl%jP|v@^_vbEtMW2`vc|*$}eu~&aGcCP;=>Q zxs(8Tmil;l8+Lo2{DqNr*8hvWxmao+US?$e@w5hEc`w)QwSo{k%14sZgW3;~vSUY4 zf5?j~Y?zC6?gx*@vc_w;n3~y*hWkTJD%XHPX$j$lGoO%aIyF70QwT~b!HAHdlJ|78 zJwA3^<>p-QSNY?5#tlmK`?R^%6XA+VsJ%zZB6IVNV|MOCFZI5w?Y#^9*%T7O5+=^+ zWRMR2XWwu9v=epn(@ch+xgvt=-_nwQ#+Xv{6Uru$QE{Wi@gF6^M@u&%UmvqO6%Ytu zK5VD0S-s48bJ(Sqd$#`0bJHHJlP0U4!MB`SeLd)0qM4b>oO+LG&E1talU}3Q>cm*7 z%X2xzDI4>TA%EMghgT@m&%D@0gg<-w!qcdH&PYB~X=>tuMVK-ss--Xd$>jqbh2kE~ z@5uGn3$FfsL}OIshW~-(5Ax5B`%eWF@^>0N}F~E5(21AX|@dxM;_%z z4B1|m=RB#(8n&17$kV!=Eo*ko>!I5p^Y?0H&zv%E@eUwFb0gAJR z1({WQ6nx*rx)jB@lR?2L&K@}|ZNL3Tf${g$m~TQHD}L`Y=QK!5LK)PW+HLWIO!6Fr zQoF6>kVIOWn{ymM_eoZZkaCky;Ip$pf~9rE&=te~Fy6nV<#7FS%1PKNTnT?cOGb+1 zRqXdmTU!tJ@5by#cgSb4C&$kLJu3@HO1J z{B>XU9X6?4=i+>s_k$=yss@;z-C%d09+niwXjrI%xfJP<0=wu_GeM8IqnyO9od9(C z=m+K#6yeZ`#PCUHbh-G`$J6M0m-1u#Y3*Apm4Cw6UyHOH91b;Fo3;W=Kck<<-Gvr; z>HN7ucYhQ}TJdqI$bL=Ck93f8JY2ucO`l}wz^^^ddsEBN+rOZ#_W4==xa z1h$-&IAmX!`;KnRVE9nr6zS8K>?rz|mq!|<2H(A}nHm1@_ZDlW=&xH{{$h*Y6Crk8 z6Qhl{&EjJ2w7E`er3A5h;;dcEZ(;AA0qfN}nN0)&uL>}tsq_-M@Xej>hsSqOojNi?GcuvxdS zwa9)o$-*)=VR9AhYwYj^E~5P>DmT6 zi3ApEzGT+kUChn))rn+Bp>wv8v%y!)#&)KLSz1Hd)^9uhwXO6`D6NUyE&FMlHMEG* zoe|2JNBxV0dRRpt#U!h-@yaE3`~=LSfz3-*{`h7=>L63FakNN5>-r66)8jSe!m156 zy^A*eV<_NWQc~>wzq&V*H~rm4y4OT{%)4M!#_b({#i1(>0rF!GTBsh*64_7Ns;l$! zMQx5r?R?&Bo*Xu$Es}x=^RW*gD8Ce-;{-e*;(iopv4#;e3qb&R9)uG$JMNwn(}`qJ z^p|Ls;MH+GlR${HQ;D~bxjm|Fc{s;0^2MYK(A~g%sxQMK`-SbVb$sY>d3L&b-uqa; zSVByMeJTa|@|qu0xCage(TE_Y=dj{VuoD&3jlz#lC+&c0B0^~{;9`AVGjHp(9Cfzc zp4}Jwww-#)HaA$g^|r|Uj=I*e`&H2ciw~;%#f(%UJ*Wgj4GRJ?EZ>RAr%w?czyad7 zgUW1!sDf4@)~PR+heh;*9m`^wPCA2y{z_0`A`^UbWZ5#y>1*}92ss0KseYMvP}u30 zFJ2(u>lP=S`04rXOvgGw@A$$lV@gIgxEig=#;jT!%s;U!QsVlj zdg~u!y5Xe9_{3tZ=G7*bYTuiHhP7i#pokZ*Vz42i@a^<as% z%av>cI{)Bed}lejh2pjow=bx{sH1RvA|ISTIUBjF)!$bY?_<6!Kzq+es?N%kqeCx3GDz*JTB z#(pjXmUer_(_a)`WGm_gO~Q_| zkQHwqA=2ZB#pP%U_(fA^0AnoB=SN}gutOGrYPmW&Wd z(p`mVt?%f0u~>Kj4u!Q0tneU9Y~5AF-G4)ZOZ>j7`ZwhV*ZHf~Yv14fZV`k%acAuE zo2}jI?5UcC8Ixtt#+Y&3T&CJ67PmlXk(_{)B1W)I0z`;-<*A@8e^JorVW4G<7kW(3 znvH2sh2yU&^`jZ5`mq{bt!-8jv7xKY)WeR>r87~CwRd?-oHgqE5Q!3^0;zh6XLRV6 zDiq$5KE*To*F0s9U{QSeyvh@PTPZx7)AL0`R3KMqNJ1_xwqh1%8DJ*iyvbu2E8NN?KH6R#v-l0yn+GY}w5_8Jzt_FKf{IuO$o zB{qOs*qV@ga8X%p6Nr;L>E^4ckx+qkC(A4>S%-40ez|PQ)33;dr=NzSF;Blr_(B07 z(3oA0ZII!mhFjIc75KfFbO->#^CD5+kMP~9iQ4#TWmEpzAWXo`f_otXaqpT}chp`I zYcFJueZk7q#EM$9jxUp(_{Li1g)vC9j2FiIAahVq!DvPtztzF1B+RR%%HIoCCq=MK zQd|HwsGS5F;4FR~6k!37{ISV}sswYa%jr;$AO0SJs}&h+7yuPk?+Jn^bm>|6)DJkX zuGr>t!hVv!KI;>?Fzsu+n%7cf9;;q&WR6^z8qfUc(`CjQe4`W^GDuF_d;-^_e%yT9 zbnx{k`$(aj_mSO%v`KHZ+zS|qD$?hRlZji?LreQbT04W$QzC{9RcRLz^h`N)V0>S$E2o>cnBi&+EPeJ(`TaD;W3wu|kpled^}P>Fo? z8+j3JDQ$OTV99-T{a;5enLUU$n5i?|5#rpKFc>eo2o zT4iw_6aGOfN)s@WcvF7&ZK@yxm%mjw1C6w4k_{Ydt1KR4+SrpK%L7#DQE560a}fgV zu%IkCL8PZ&yGTtuoT=vaNjenpF#`$qkEil0nLle$WOVVBe>PGOUh1t2dga|7YC)+= z3nUhUj@ zGu`O-0tJ1bFH+mpNiP5R(qo>>R%@JyW-PN2ni`s+JKrYr9CKl(yNWdDgV2VN)_iR=adlHZO&VSusqo)|^ECf`l2gQJ6m=}0 PFKs|wO;{T7>+=5qkG}qx delta 3145 zcmZ|Rc{tSj9tZFtl6@lEFeBqPGm<3PVr<7)X2ufA8G{Zd+f-sK$vO;WDPhKvEoJWz z3gOuItTmHluY|JI9W!Xe>8gA0pZEL!_x-%SpXdE~p6`Yo%dR%?2tPk#SBgy!Jwl{x zroH#xm}^5seaQ+p7wDL`LG}BA`#uKTAUKB9dGpm>npAvwJE5U+I5gtYXd?Yf)w_cD zy7#r)hKW}@VEdomyU6YzO{E?`!{n>`R;z=faifcm?pSoy%fIaj)4+Gg#WU*jN{lUf zEYPXc>l8Vy0uM!3whK$|`h)Xfc(=1SgTz28eaMSBUO9m)4mjqkwY_>v~jjVl~PPD4K7?z*nOZB>qtN zzOy5-Da<;wV(O}_DBSI`^KQ=WeEG9SHvK65Z@^`JS!JL6mn!yUCX4GH@8gQaI||zj z{VD7D?XK##7lO656ey_TzPI70^L5JToUf$6nONh|INyM?4@9XdZZ-V6STIVLwXhbf z;DayxwmeatmjVb;m*AD@Pf8($)2Ov?Dv7TV*0Y2fuWg+i|8O?pn@_Tt;W*vjT*BuAIfJwET}Y` znU(m&rk_j)&J+#*eC@0JW-0hNg}LB}sr`{l;oqO(8g#8=^C@=S6}o>nzX>V7{$)!V z=Ylw2RYb{j!LfjWT-g$jJw^;KfEh7P+&KXBI6V1aAN6uP?uj_3n?1bV9>#>DrZk;M)n^%j- z`~V7bNKZ3mF`WdYVLskLf7OND2Jpvo)Nrq+Bby&d5T{Edy@p8OZhgSoD0pwgfigT` z>EptW7S~fm_KA!{gGXKqYbM9#xm?^~hOrV@RVCdLk(a6kp7V}Zo_yCB_#jU8pUHi~ zhs!r+vIs@}Awbq5+0Y%U6@y@avMOSe0*fCJXnZkVrqnE&ym0v|CbB@q(TE#YD<(=m zi#*zu(R{Hps5kIx`$*Hcx0@jkWkmIg+F5e7K-t902Wt=37H7>>+qZ(N0(1v%0QW1% zZi2`wVCy0tjike!7*XouJDq?aXoSl5WHejceJD$3B|hdvk3Jys;IC)nfv1HmU-csM}fTkBdFPPWmIQWf+ zJqTIH&vSpf?y%#v#63Ow#f;yRu@RwiYt8c$(VOb@b4Jix(qKzSNMb+T7FuMA#NQ0L zfA~IdU6v&7Xnq4aK}q7Q9Ygq@2v|2c2rU{Zio;k!%`c4rD)!O-82o3WZL*=RK?_a9O0ixv#zt+d#-=^4Wv5S+1Z{QIRDUQtkhyS zQ8~&8^ATGp&H!awD>r*W1`)5V)I-u=CLq#Wr4hcQe{m^4M@jvj1wLp{U^@UO^sFXF z8KsiBjymcmPdppZck*|Ms;`cinBWb{w$3y|=2|F_wv-@t#Hhr>gK8dI9>2;t0{>VLrUWp7$7E*JK-6psC-N;nuxYL19fm|~86S(sW zIQKu``hS2s4ELSbgVpA<=Koo76K}c3)auC%wrtLi%F2hdU7YEOe4`bcyV(Mz1&0uS zTPUn>lpufT9}Mr2+6RsYgc3u$Yk58EGrU}Ujqjma;I41L?yq)R;~fN@6d{KR62{;v z&y&x0(H35sFJdm(g~MF1ex=g2Ge(7+_NkpU(f`>^VZi)A5O@E1W1l(ul6g*f!a>HMvMDLm|N`Wtx| zopsH(mr^|sUGe;SPpi(et&97ogzVr>SO7uz%kqlcQ1TPG{g$rPC+YoYNCHET+rDZT z;kWcB(=hk8_kQ-U=_&7z#xh;QGHo(r|J-@NUX?NH&=gCl=R}Z*bI-EZ56Mq3x>^lH zj6#Y(+?1B5yQfKCnV8Ex`efd$c`rYny;RQmefq5iwtHpYuK3}Mc+q+}r_3FEdvITT zN@=R~CHr$nU7}=~7(chRDh5G^!WAjDJsRNnuW0kdh(A%WLRCN}XbXb?l1*mSB4&ua z#?M4a?_em>Yp2zukKx`O1#Zo`UoYhCQQ6GTkR>@&LEr1~<9U#_8tD~7T%Pj_c$P3hr5duu9c5thRA-|O8N`&O6pIpw5_%I$7H|f|u1>+mLC|NB3=RB@p z%^o3MNi~hxLyiVV89XPx#n&5GLqXKbzNWEauusnS?B+qX+)B~7z~u{?>!(~BV10va z>_E*MZrLGA?kU`$z2?X;c_*9FFLr%A+L`Y&4BWe#%DO+9qg{t13H_2ah{S=Y@tRhs zwA^<(&tg9V^GNdj@BO~#UoC?g)-u~ POz%Z0AC^LZ4$i*;E@|Is diff --git a/summary_results.json b/summary_results.json index 6c443f008..68f024b6f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -428,7 +428,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.409159, + "Performance_Result": 0.384271, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.387848, + "Performance_Result": 0.424071, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73476, + "Performance_Result": 1.73914, "Performance_Units": "Samples/s" }, { @@ -509,7 +509,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.413841, + "Performance_Result": 0.421143, "Performance_Units": "Samples/s" }, { From 5bcf424987551563015cc4df1361c47b9b54c405 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 02:11:07 +0000 Subject: [PATCH 458/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10221 -> 10222 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 64287dca8..d88e646c1 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,31 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 02:05:58,778 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 02:05:58,778 submission_checker1.py:3266 INFO] --- -[2025-04-16 02:05:58,779 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 02:11:02,792 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 02:11:02,792 submission_checker1.py:3266 INFO] --- +[2025-04-16 02:11:02,792 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 02:05:58,779 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 02:11:02,792 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 02:05:58,779 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 02:11:02,792 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 02:05:58,779 submission_checker1.py:3282 INFO] --- -[2025-04-16 02:05:58,779 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 02:05:58,779 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 02:05:58,779 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 02:05:58,779 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 02:05:58,779 submission_checker1.py:3303 INFO] --- -[2025-04-16 02:05:58,779 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 02:05:59,877 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 02:11:02,792 submission_checker1.py:3282 INFO] --- +[2025-04-16 02:11:02,792 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 02:11:02,792 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 02:11:02,792 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 02:11:02,792 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 02:11:02,792 submission_checker1.py:3303 INFO] --- +[2025-04-16 02:11:02,792 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 02:11:03,837 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424071 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73914 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421964 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418992 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -42,6 +42,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 16.1275 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421143 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.4162 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index fd1f63c24..75864bc50 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1085 +1086 diff --git a/summary.csv b/summary.csv index 2cf159440..1445d44ea 100644 --- a/summary.csv +++ b/summary.csv @@ -1,9 +1,9 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424071,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73914,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421964,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418992,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -20,6 +20,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,16.1275,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421143,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.4162,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 776dd2449..f732d9d3b 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.424071, + "Result":0.424658, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421964, + "Result":0.418992, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -639,7 +639,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421143, + "Result":0.4162, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8f82e2b15922ed33f8626966f6c24d91918ab624..382fb406b5da3dc237be6202600640499a5da67c 100644 GIT binary patch delta 3170 zcmYk9c{mhm8^*_2iU?zrCLw0A6w^eOkHImPVMfWxE<<+3aBRmmwqoov!ysFT$W}zs z$u7GXTSMWrG33x7#!}yOuJ4cU{rkP{>v`|reLvUru3!D<>XV@3f})mGn@u4AK%5qN z+>n%r7m+goWo&I^@=2w>qPzDYF0)4ljp7TM!*=Q0FC14T*N-AJi=>_#f-`%VF-97r z_rym8{wC_aTnKO~?3n)sF)7^k>iGdJEl{pcjLMk>jB()0Pw{)0OFPeF#U#wwsCbV2xyL-Z0 z_xM6$LZ5mbtOja@WFc6a9}B+e`b&Qm>1T!6>zRPlzMoC<-CfaKS9drTIoz(+-}PK; zmpNe0f!rgx>qkldqd9_`1FpaPjI@-$C6%_HiVcFq9Ehbg1W>o1FzpBA3VVk;Na-vs zJ$jof?jYqNK(9F}J-wV7usxM4V zgodKquf~Vq{HE7}5(D*!rbsNJ!cDNL_x_Sbq9fMn=2 zmWM@-FZjLpB+sT?J`<#RR@J@7gH&Idli_t)ar9V)tMHfaQkGV(Ot(DDhuaoonA=mSAiwltG2EB zIOQoPejlJY&~MdTFR+R5{-`ZEyXhw{zI>Ey*2@!V1O>i*j&ZwVu^>)fVj1gEju)fF z$gS~c5Ls*c9QWFW6wE>Ty(%TS{D5rMJFzZK)@myuW2E0p$cvlqo}g$Q#{aR$zia?> zyh&6H{LG(3ZJ`|sog)cuyd#C_8=f1!r+TJjWr>!?a!o4?M3-aoffQO&T;@siVGR5G zvxBE$he0o-QYNRzceGyEN8LHu?c=9|{aIY8QnWp(n`~nHk*S!>*l{4{@C61*s!&CPm=auYNtRq9e{j0qWa_e(G7U^mOoR7uk8uk99iUrU8^4MA!3 z4{$Wb%3z!Ome`f!uvmGD8JLVIO9r_FZ!f4BQWA#?im96!Y6jYL5obo6Q_4`D%xgvK zdSk0wgJ(~7+?=aCGL+iD?GFow6>l}=R)l2R#&ufjn?6_!o~yjGU!X5->=AS-cO7UF zNGG{Sq^rnj%R@RTKF~~@LL3z0^6Afx2_)1WIOuL4KlWu^Yo4QL;dEm>)Btzs?wE@MG_5L>|EtdOaUpLW(R5t%?mw796t)V(KpIZ=JnDjLZq!lTF01}DCssrH6#bd}i6cP&rG(io$wm$RbHC9;YoF@!DkrC`s zqCI@AG`X)2Xv%=7;PYp`XS3|*F5X9-z$pqj^S7uCV$!|L|9`?P+{-i^6)OADCPDrvqe(lnD=dj*!wH}zcSeQ1%}x%0~=DqBbUY+ zJwJ|}qLBJddS2wWiAQZLPpntAq#o7|slAXKU7ogDcYW@|~Z%rSxdUD$}JC8et;s zv(7amBJy7|{`qBwckCc4LDT|(quWP;!=oPjBIOU<~gj^&JC2dBS^fK5HhM-(%*WhG*52^+j~gXY?;W`6%^+<-t?zc3k_g~lNHj@ zn2*?U(FwVH$uu;M_g7l&J-1y2V@FWsnF8ga&w{atL>zKI3u*Vdaq3Chw8q(J(hnI_ zN%iD1Zn+4}y56$B7aFHwvj$Y3(D-0hBRUbzST#~l_T;Y-odn#+@OsnjqwRnOO(UyK z95xUB$$pk4aS_CAsqkB9j9i|&U(5cX;{B_0t+f>c<}}ZRHJaZ3)5G{^i}$K>5_pA- zXZu_zb!be2yUmk*gZJ6((M)kwWq{Xk5l%)aYuk9sV80&w5MQog)?Wd&EbWnjAg=# zj+PLd_Esv9^|ynuzk_W2b=+^(4nYzOxOj6+)|Ael2g;To zpi9=7HTg%Op;j96tSblnve_CMW&DO`q+Bad1(4_q%N&KF-#(eJXCPG*GQj&9VL3#c zooK*<&Zd>&0BC>@?v&ngN#|amnB%xEc+i#F+H!L%&%C3Hl#v?H&f^3^^+Duwb%UY+ zU;O3DB(A6MuwtU-c^`I;#|udCR`U0q4&ZC(@iuC3(r!+PJXSu_q}3W`DDt{~EybVV zXdRQ%ajIeC=y;r6aK8ddNYJ_a+WQ2n$_#a&-N+VraYgN??-3Z3w34$ zz8PBn#PQIEn&PoTNrms0gWI66@@&fBDw19K6FBwg{7-l>Pyf5;B@Z~fO)d&U!2kOa zpy0MVDYO*0E)Xn9Q;lfCJ`c( zOX(ox9=23S%5BZ9qzI|sI*;EU-+#XEzu%AN<9U0%9*@`hv9)83G2#+Np=_~rNf1b$ zmLzTm^xhJc)s*Pl*>)1l!8gE9Zw%z1W0%y^O7ACb)4#O)Er+UnCz+Kiwc07P%bHuS5~2_y45mD+vUY|F_=WfkP!V^y)) zx%yE@ufbPF-!=E%2mfpg3uB8CX7w|O2miAlICjR3y7_53BM_|&=i!^1DQC^eg+Czx zhe$%kj}#?*l#U!J*@${`%<*(kFlv1}ZPoTw&RY$S9^PNI4KFOabx&EW_=MhZZwc_G z^9WW}YO`A1mR0wb<1Kp3W-627WUtQV5GL)>AH)81T#GDMrJsGd3y*yM?4^%c*{qpT zxXR@CL+c1tRCIH1qml@DGs{SV{AJ6aAeFsF=&VPT5}^P4lS$Xv&K zzr3~eX#Za9ZcMvU7H4AYd|A%HfR)bkQ$-$Z;5N@iYab=U>>jybD$b1SJopA?SN5hi zyMaS2)4w!V>i00(l&URG&#rTPK#xd{VARdCpdN)r#NbZ)f8x3EE`#8 zK1JdVolJ_Ha5pTIdiG?>@cv~=Tpz7%Yq{cgDCcXTu8Yf|dOORO#h)?H;_pEUef57| zrh7dOCNBGXlxM#nN3NN;_!Dn2Duk;w!8nzC=jBYS|ILpxg3Hcr-xl}pK_6Xx|5)5+ zM*5IbP40WTIfLO(24G~lGnXC7##fhy>tzStf2f)sTK{v4Jze<6old;u!uKSwW9Rrt z{avg0*oM~QQ@U4z6mJ#SZVb*G9sj(1-89OB*@yc=!4d^J9m#q|R(28;XBZW$kq-lt zVdHwx{B&3FZ#{lW(lC+;EXbVl0SCtA6dkwW`+GEYX+H^Yu45K}VP})9Xh9tI-5bpc zUoEmQOl>GB7wl~tmkPGccZOwQHBMm3hL)s{hbo+V7Tu58T^Kr4QO1vqXKUaP>LwOk zp!14ZeQzN*J3v2*6K$BYjhG3&W;MDqImFf-)U$in{*OaNKw?Q%)Nbid^Q^%Iq`|aE z&K&X|1OV~2jX8=+(d0O*7CQ>!P>=gJFSGE@O~TXxrf~g8;bGmIx7;mHRF#RU*V*?h z*!PVh?){X9jvdr7s|A;p%KLjX^y67Uixr?MgJ=@NX&zJPL z7S(fkGkJ=bu+}ItJi^~82&ei=h>jKV0ZRn}WJr;=86*pi1Nj_;5p>$`otM;)Vv~&* zX*S|3V@4LiU^%y9KM}MOvUO=F$2IEZgnU;W^O>ggL=;8y!TsGc8$)er|;e#rci?s>B6 zzEvoRp0HUptSp~()ceYstMQ5XVk%#3F;*x9EJYzhU>pMt8VqFo8@RFoRf-ln^qb#?cYi~ zu4S9h1qT)qI?4>2$(}p$`@-6c8Zy^E>fwb`Gf}&`eZ8y%fAo?N?E@e{tjx_Ipn3D^ zkdUTyT1I#Zj!A^m9KuxTpdw_`rZ(?^#Z|f~PuSE@{f<3ToZ34rlt@=RoS2aLT`$2% zPp46#VEVG^OkJea=;)nC%G%@7bIwjO;}_U$dz5&n#7P+rr~oT;{NN7BM=+(Sf09+X zv83>ed^-358U{!nYUu4r%YKz*jRvd>Mqfsrh5%WEFI@@kCX4pIh7=3KBKLx zGVOF-zaL3_P5pCEz@Qrr2sIG{H3Gj-?;|%|i9-VXboPyR`0>?ck2)UbFyLd z>f5c|%IwLi`Du$KpZeG_>};my2nIWkYnB;@mcWNGZbAgGRK@9#ExZJ1M5B3>UwTyc zs=Z}*x$CbX4WyZ;2D0m3uWqsknDCV*s)nn3$#gVh^?lwVca{1fOuCqGm{>W<&pCWc z9Rh1kpXBHKYn-xyVGL4$!bctt+`7V#Ih`UArUt%7L*R00VNb&t|6~mRxxCdr#z#BH zj@z%dQGVmV8MKox*Sy}cC!4FahQX#0=#6_uQWZEP zO7c?RfcCzfNpNb>W*qFp%G_E*Z5d;<$T_y%gbn)3~;fX(Lv zLiO2Ym^yiWZFsD1p3uLGNgoIIxkOM3#C^AIq}IP)-c-6V02T7I=3P7ozkkEGD|#=P zy%#pixoB%?VM{Gsb1sz`|HfY8moY@5lwU?l2pSZUKavqIXnSxn89=>GuK0Js>R%Bq zm5~yH4(KI=`ne0A2gKL_m@+ys&q_pNJkErB|G;|(uaswSpb$~5?hx?dP9tmo+J5(y zWd{m3;wS0r^IoxwQvv2HdCi6BIIUVUG-7^oEc2&-rxknPtx9;<04Z(rDa?raar0f{ z!8aqEBL#|nM}XbLv0n_7V-0?iKMNm!Kal;D5*We!ZhFEXMjO!p&`kzZ2yoo zcf#=6D!qRvtuE_9XiHq zXlcMx(>Z$#N4r)v?Bp??dpGm|ZBMPqzUd-^I}QjmXUS8fMRIAe2YvHsNKjr*LtP3K^~gYThsS-__Ec*7?bH5(;X4+ zt#OmVBdOrAH{vyBCc8e*IbsFhx>^G3$B(nDFQ6jVb)$96q~Dg^dzUKAz*4pfrXf-G zjS9h|trbP1OnWC{R9TQZJvt50pPxM^)CLX7QWQq`1h$D)CBT@Po}c8x0e>ryaNk&} zpbGkVvofPopzO2RVbR5&nvmCiZQ<5rRuXrSu-aj;f=Dx28r_K4*u~V9xXduJgd%ji zbrQQ$iSAbovoEyKcN#Ob&brDT+pC_vU}X^VK`6f$^hI{NGTGzr06IMOg+itF$r#2G z2d<;76Tb6pBJa_%jnD+K{~xkem-_ZsEy52_bD-v+;C~$iM-biunY3&KS`e&AGg3TA rn?>jfw9+KCECqCE4qAx+_xHS(qJSjrnifVNfc8{NQ&bKP;=lg`LCy2< diff --git a/summary_results.json b/summary_results.json index 68f024b6f..7fdfd4d15 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421964, + "Performance_Result": 0.418992, "Performance_Units": "Samples/s" }, { @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.424071, + "Performance_Result": 0.424658, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421143, + "Performance_Result": 0.4162, "Performance_Units": "Samples/s" }, { From 5e2c78e89ac069e7c824e2d5d2c70b861000611a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 18:23:52 +0000 Subject: [PATCH 459/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10222 -> 10225 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index d88e646c1..cad57e336 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 02:11:02,792 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 02:11:02,792 submission_checker1.py:3266 INFO] --- -[2025-04-16 02:11:02,792 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 18:23:46,427 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 18:23:46,427 submission_checker1.py:3266 INFO] --- +[2025-04-16 18:23:46,427 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 02:11:02,792 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 18:23:46,427 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 02:11:02,792 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 18:23:46,427 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 02:11:02,792 submission_checker1.py:3282 INFO] --- -[2025-04-16 02:11:02,792 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 02:11:02,792 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 02:11:02,792 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 02:11:02,792 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 02:11:02,792 submission_checker1.py:3303 INFO] --- -[2025-04-16 02:11:02,792 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 02:11:03,837 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 18:23:46,427 submission_checker1.py:3282 INFO] --- +[2025-04-16 18:23:46,427 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 18:23:46,427 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 18:23:46,427 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 18:23:46,427 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 18:23:46,427 submission_checker1.py:3303 INFO] --- +[2025-04-16 18:23:46,427 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 18:23:47,836 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,8 +31,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.19947 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 22.2422 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 75864bc50..669ca49ff 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1086 +1087 diff --git a/summary.csv b/summary.csv index 1445d44ea..4db56c236 100644 --- a/summary.csv +++ b/summary.csv @@ -9,8 +9,8 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.19947,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,22.2422,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f732d9d3b..606059955 100644 --- a/summary.json +++ b/summary.json @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.2422, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 382fb406b5da3dc237be6202600640499a5da67c..b6984edcb887640504d980a347ef55ee2818552d 100644 GIT binary patch delta 1583 zcmV+~2GIHLPw`K%atH;DDFTJ^vv&y61%Hstb{Ux@0gzdeptJe%flS%bC7mc!OEP6x zkA?x7D8ebT{f{D3J!vglM~#)H%yxdt^qa0}7d-7zd{h5W;a<7zNib34`VvgkAecl0 zU=j_0N%r>vOsfn`k^sOgNzmDR`2ePD>5@(em@=$K!f?tYsd37*{tv*cqsB^8rhm0P zWqOu&i>Ay16N*Atf$d8&QRA7COw=HmL<3|J4UmZ%eE4IRkx3E&nI#E2n=c>8lr3G- z32cEtRF6dgn9LRgVA{_bOtB&I<4o33UL_{&Eha5(Ik~BBo5=#7jzSBFBaJ#l<4LCu z(IAr*ygEz+OcFPYw1=rOyAJC_`ZRUQ5_A+ecA%w8I#Fg@LJ6)N8U|Dnk+kBSs5+fH zDV2+_>14QAV=0hKZl}xnGMgyVRi2Z253+y2?W+6&CebtENNW$#c+%TLG{|HHcMsD5 zlSBhdmKl6l0${St;-kRh11(+B32_sE&xeGen}DQ7H$lx-AB6Jf+XPUZ3pIkL4HaTk3WJnP?hu zq}7LLJZbeI8YHuV)rV<-OrilY%d9>u0gzc{_0eGUftD`mgpdhV9}CP{ydlV;6U zpTCF9XX^Di(|NB?Hfg$kyI}iLTLoyMZ^V&iAENQ3*@tKl%?f58rU5jG2GA@s`>+In zW|`SXgV_gKx}+1Z6hKsuMFE*rygs^Sug_y-{;!^bTI#C=<+jhqN^fGkA}H+wDA6Bzb`k0QsCeKE;1P`cDG#xjKCG zmcz$sD(%Yu*44_r@o*YM*TGRRnZ(8*Ob^#8a?`qc4dkiXy=h6F z$?E-%$#`~c#DA{d9O-0ljx@7mgVE_?5*W$%d9dFI?&j$k%8G3m!@JSyosq^o8i&z1 z`M&Bsduioj(m$An!K8-5GWLzM^m}Pq-LjYL0~^-C z-n@SueCkKn<5_G>f>Bzi*>oF5w0aBW@^cc-m!HyKXIEv2%OR07l!m5?X` zakK1gY7zg)Hl%rb+@wugDJPEWnf-Ql#Y)Che@ZhWr`++`PQ?tHwedPsIRL1!sg=pkCXmz}H1MTtl7TgcI)s z@W(Fb6K|(QhCbj?2tbCoXXRt|^t@dqqbg7H`#7D)*o`(FJ zcm@o)Z$Clx2eV)!$^r$ADFTJ^lP@M)0gRK2COiTb6_YR(7L(s5DgnEb3MV=NEt5qj hCIy7n_E091a3>W34wHr_J^^Es#U~>M6(s-w004G4=NbS2 delta 1591 zcmV-72FUsGPwr2!atH+vPLfvevv&y61%GfQ+b$!MBmgo?5_C3SK9DI}x}+0jYDuOH z>(MYk6Gb>>w*OINswb^w>!`8Pl-bUasdZCZx?l@L@lE|hg?r_;C&5IG>q{_EgJ2R3 zfJrm}CfVNyFs(8$Ndf?~Btd8M+?I9XZdV7clnXKUMVH#kP zXn@HwgAYppOqN-E6nK1~rAs;?ZUXT6kT7%;kksfVsM+d+Q2u;7!85h{oU6Rw$7!Y~ zr(Iwt&?e_nqhZ96E+3-tq|1kBfJ>&4Tr!R1l4&HDy3FOn5=bt~5_B}Ue4wREI)A}k zO9<=HFgaTi5v*W!%`TtExcn*GmbDaFWfiOQMjx%4o|Ri(Zj-4B$VAJCBaJ>p<4L0r z(IA->j6O^QWD*UKS!VQM34qKpqmKro543biCxlEe`j9YK#B#es}IuvnnVL=mRWsR0zk9O>a+Rs z0Zp*_kWO&l62f{k4A>-oAec03w)*@inoruetfk0GlV;xQlRf&e{DOT;-7X*#O(Tx9 z`VfsLtv*D9WLB{HFb$ANG(cvV)rTbjGRv$!8mvCh(j}b`GQsLY!f?_gsef_Otl8@G z_mKHay*_6;@Ab(hP1kQ1Y+q`t08R9bIMVDxG@dm35DlVP!R*5{fF{uZnq_7mmH^N! zGy7;T`#?*VbOM$Fi0ZK@AhU|sN7wB2d5p~e)l*PQeU+fx_W4-pO^jCrrCk6edPW@S z^C22v`h186p{(HZVH$vvjDJV|4`^NH^I@F;lx0324L%=e>5@*AX-O)<=tIK*OX3EC zm2AyMpU0?7Mj4RmNzY?zsjboiHt+ID@0)#37Xj%M07+i8BTYU;<4cnd(IAi&Og>Bl zK$7tQkYy$x)(HSvX7bTs@`08v=|q{f1QJX>Gz@?wFYo~%pOeR@7=K9rNkBeVhmYQJ z_&80aUHRX-TDdnKPJ`$=I0`0{*cgQA;aWv*T34@uJXO1wt2ZV-$kYD9*>Ie_;-NoT zz27kz&#sO5&()hFo$Sq#W|nL)I$cZxBl$iL_8Y<7JUv5Mu?=H*H(I?j(wIl%Fd8S{ zSG{L1tz1m{2h(u)a%4HE#ccfNB6#^?@WR=BF>qe}V(3@KzLA!GFHNgk_L6;I!#dcT z_pgIb{pfl;i;YPzN((icZo`OHZ=qa%PQv-}Q~E3IYnUYA^!}%tpg#qs6OaxCiYp;xtuIq|!(pyu0ioX`G@BON|GKW(MKuskD6CQZ9zNzqakq+1L5EtphYX$OY zpY)Np(;~+{kXZ`Aj&aY*=j8ciIV49-Ny>+u+~;(X@}~a+)Tbf;CY}Mu z+_#^g`2(|KBgz5=4^EO+@RKqoTLFrbjwU<-6qCy)K>@jw4JSGQE0ahkCL4+?A#x=F p002+|000pH0000000000004l243mi`J^^5p$R{HP5+wit000_S;p_ka diff --git a/summary_results.json b/summary_results.json index 7fdfd4d15..544c2ee7d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -365,7 +365,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -392,7 +392,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.2422, + "Performance_Result": 19.7056, "Performance_Units": "Samples/s" }, { @@ -401,7 +401,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 35fc4a0bac76321a1a52630788d8eb8ee27dbe62 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 18:38:56 +0000 Subject: [PATCH 460/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10225 -> 10217 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index cad57e336..c5137030b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 18:23:46,427 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 18:23:46,427 submission_checker1.py:3266 INFO] --- -[2025-04-16 18:23:46,427 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 18:38:51,252 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 18:38:51,252 submission_checker1.py:3266 INFO] --- +[2025-04-16 18:38:51,252 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 18:23:46,427 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 18:38:51,252 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 18:23:46,427 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 18:38:51,252 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 18:23:46,427 submission_checker1.py:3282 INFO] --- -[2025-04-16 18:23:46,427 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 18:23:46,427 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 18:23:46,427 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 18:23:46,427 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 18:23:46,427 submission_checker1.py:3303 INFO] --- -[2025-04-16 18:23:46,427 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 18:23:47,836 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 18:38:51,252 submission_checker1.py:3282 INFO] --- +[2025-04-16 18:38:51,252 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 18:38:51,252 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 18:38:51,252 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 18:38:51,253 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 18:38:51,253 submission_checker1.py:3303 INFO] --- +[2025-04-16 18:38:51,253 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 18:38:52,602 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,8 +24,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73914 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418992 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74527 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421449 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -40,7 +40,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410188 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 16.1275 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.4162 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 669ca49ff..d8d144396 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1087 +1088 diff --git a/summary.csv b/summary.csv index 4db56c236..b1a4aa60e 100644 --- a/summary.csv +++ b/summary.csv @@ -2,8 +2,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73914,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418992,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74527,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421449,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -18,7 +18,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410188,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,16.1275,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.4162,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 606059955..4f6a85b9c 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73914, + "Result":1.74527, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418992, + "Result":0.421449, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -579,7 +579,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.1275, + "Result":20.7859, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b6984edcb887640504d980a347ef55ee2818552d..ed1d115b597caa3562d65da22c90671d46350725 100644 GIT binary patch delta 3181 zcmXw+dpy(MAICR$71k7zjQJRH$#hO#K#T!^MJBK4B<1vcUSU2 zWcd47J0=`y9`&-0jDa-mn;Tg(SJ$T6WTVze;YPe2)0T3*{*a!z8@C;7Z&9yuQC&a6 zMiP^n9}R^>xV}jyuGf4iPN2>-Z0jUn>yX_0Hgj2Q?{H?E)IT&X>W>Buvm91*`QaVo z&ZgtzU16%`9mf(zn+hv*jk}Ds@tJV}_+x6tYE3xJbXa zI8}3d48@&hN&DJY&d{zqabmM?ImA)N_{br_Ykj_d=FhB+iJR~awb%D`wuG)wv-398 z5B=$n{_#<_hD>e(e+n7$C^9nE6$=(6&TIeLmKGB-!GwQYRjLrRnqBzPiw`c6GnRw&pBwexDs^Ng$u%5>ZHoM&Cvq|)W_-{1itME7(1EV-VDer z;7y-YVmJj#8>)DB<@r6G_Qhi><|YoAG4*j`@z1mpZ(ekGB8JF5+nQqz4-ApLgq~e9 zt0Q!OF}*$LzY$cJrAG*|k?sv++@IaB>3LtCvI6h1suZaoiwjjgX>{k*$*7BgtoySE zj&4+lEPQ-0R+T3?8EVy|{;R;WJ4{y^8u)S=LI7L_i~xsG2gYIjyioDEi2laMR#84o{Iq)|oO{EmB0!|1+= z$KE(yA`mC29_kO?t4ztE z8C}Z~^k-QQKUJ}xyYNWXc#1YSSoqw0%B+mxP+cE0z0 zii2+Y^vA;9W2qN+zP9t}P6>*dJV^gaI$K)mOaKUnU80Q%$UBR#Yas()m#QX*)_t~V zlXvYNws{M;hV3`RHT85;gpls?R67CI2z zW=v32)zatBG6izb@N57|07ez1vJ=dq>=}y*0^!`im9tq(7$_z!|GaKFnrv`$aWBHO zhLQ^fTqCrISKiU;@~B6)&ZVFz@>1dH&`Wy3ad@a+wkeQ;hTEVCr!>RAiOMD;iAHSk_Z9^0wgh%vBht+V^Q^ljr9961KOY~nsF z;)ewC-qH}o^0rp-kPp%x8~5gXNKPn}=H$tlu$L@Vg{$t{@$MzASvC@%KR4vJ;$vg%!SHR{Juo8i%0WDyWdt9I= z2LLG@gf``Vb~t;7q_K=9Zw$Z^ZejEBMgI`^s$SP}l*o`SGrfB&HCE{S?)EmGWWr4y z)#m7J=N)IcZz>lUwbhv_DGQJ zlO~KDm&Gx5FaKW5t0YAz_*~}x713Ek=|u`TjdM9&NMgH&yoc|Ztbi3MOgcdNJOZGIdSvuJeg+cgg+vX6&0U|br zO+(tm3*lGcNDPM*RZ9OFbAglGSZS#ls|-n|H^_sC6LBDmUB`F@4RmzT06QK%6>VJ= zmC8QX9!`2A`>M;|a?F~R3yy};9vI@)`n~w;1>b>)-2yUe9uNELx-_$+iz!^n+!-gZ zIHFbVMDF>xYe|XVJTy=n>hsX-o0ZwZ>^03}DjysU0L01Vp!U~@!Qi~=QqaO_bw<%6 zf6>f`t)yCJN$>j8(AT_(QO;S@<3j9WeTMtBF@&~s*p)YsZdU{$=M*EbuQ|IGvY z)9`aOxR``K3=mp)*Oe4C)>k2{ORMho7u*%(TlTb{JC5HjDYXkbcb;;hJ3B*?!l+(F(#BQ;7sxxCYJ!=2qtL|80iUc`sOb?ESHc}$(fXqip})iBOCM0b z8LT-)C^d7BsSu;B1&K??0xO!6u-eOO(&Ao?39U)E7z_oBNIisN`#E|sY-Sj-?8qiO z<$w9j@YP#Oy$j`qWmg^v8L*gZTq9%aHy=FNLs{wQoO@HmT|#@3w-9Fh_TCUTP9xgROsW@+~>SWi5uspv;Oa%l749DTm-czI~Z1!*Tp#ov>Fv_;szwy*GwoR6!*XF4E=3kpEeq$JvXUKshPI~*Ew{e%cm9ADA9)MD5%FEFx)kw9;i8h`q=N9qaJ zpelqvwNv! zqCJeFk2H6P%IE0^#HFbH^e*<3VT17k_OSiv@R8lOq&?2!a>mZ$?ESV_5&e->uB8Xt z*ffOY`1M6UH(y0w%T2c6DqB!(_pjjU{{^r96-4f0Rd@Bw3xhdG9Q)yxqNa_TTw2`< zOMCt2z3q8SRAt%cEtbQwW!hcd%3@C0aAS#OJX2cS?Pp{%&poBDOb_vxoHUc)LRgo1 zrbjU-*9Zj@K>gLr#bHi%E*W-#%x8Jx^89*?OZ)Cy@EpscV@5m`AJmR<+@4$@PM_H= z8Rn!7fOB5>niNDP_ufUYyOuJ~kId(Z0p`F`sTQajVNhfLe0k^m{#_H0{&;>MKN2ej z!nn?G>Y?N^tx;LOi7}CNg0Qt!j^*%wYpNe_^>*PP0`Ww5Bq7~3U$V|_zUtPNc>m32 zd|SBZY`pf`siLjE2}Q~0#P_==hzvh$H@HDmJhacEjP|`r`7j z!Iau#d##=H;_O~+@ubnScvd@ICjz%{g{D)GxOaC*U-{~su4w0wXq)Kp@6-||UDU7x zDVW~08G&<}YP`G3dHlm@=L@Yvs9WV{T&Ui>^l4lb z^J;m|y1Z;sh_YI}S>+B%9M}_@ke+z)hPl*n-kg)`2nYn?V{ct5SvX#hTP=O~l`K&j z4#Nk$QiUXZk4ov^=?;{aZ_JTuK3Wa8J87pFKx&J5|HI_Tfxof5kCrOXAr|Cj8idgt1BSs2UL*0Se0(sZkgA4F+W{zA@}kUg+&CJT2m!HACd1 zl>}Wbci|&lS!D?xVLApc$p@s_Ds|KRALG4$75alk!%$rZIE0l{Be|+>0~D~rkK`uZ zx;1c|CssdJP?DH7lX@n`_IH3`z4N)eL%at=YJCEOzqd)GyNr8JRkZZ};=(Jtec83M zeUMo_8&hC*WAH?SyR+@r6}V)bH%e)aa0c4_PMRJvooHkgC6yxh@QgVYY&~T4Y884PQK<`!OzjU9T$=%%+R>M<0Wd=F^_bosoE`#q7GZ5Nf zA>t51AKXL~kkbSsh^BIg5tKMD}SCxL4UXmnoe5UG@aJ8Kb|#K8Fr&8Il|FsJP6)|gT*wMC95P+FzF}h z@Z+2DJZ>E94aezsSv^J+jP@IcYWL_j?KD~kd&|Br$HBKa_u0rw`5K0QWIucy?l;U# z+5+vn7*0`P*;3DlW?uTa{Y8QVZg>s1y~xsPs|A`J;*pRLGNBmSd|@n^t;g z<2e=A@M|Iw=lzIC!~uIlBm&sCL}G1A-Vur5>3=9u?-(Uo z9NjLFh@<<7NW{_Y5s5gupNT{q-4h}aNB0YnIQ)Wh_yy-c0$W~iC~@Tlhmy9a_YIMV zdfyU>sP`R8d~xjG-%H3Z#1EtgagBT|3Gdv0cRxWL%Z4r)pXlI|jOk&FoN+#*M41AS zlYfVSj}AWke)9IT`>}WYa=93a)78-RIJsGTZ4Sa|YdT5)o~A3E^wT_uMpxt6D2USa zT5y#FadH(ezD~z+w)BawZtbSoN}?cGF0xwLTB`M_d$f*^f;e5m#I5yOW@nT6YA~Pg z4CafSa5lS(7PBN>Ozo^ySA%di8ecblX@3lZq(7d-jaPfE5t**P)KOBpXbsY#E&nMP z*V$@ZYPG~@TFUctSK$_QE~m~#>cqA78YQl{(vMIA)DelKjz}bRL?W)x_ihPwBwYOR z{#X*;!Ej%MALqlxQR-s2CC=w3S0YJbM3*9c|j2M30l`A&KGwdW{lSV0MHO zAdyHUi9{kvBoaxYS3)8Qmn4>icQ71C^h!v?a3Il>B$600p?*e*GKJ>E`}dGoM=2FZ zy!5sv#kQqgt6(YWa}<3C4a7f=m;f%B9TZGiA17^5?^%m-Y-Fsgv*#Srk4z(hBpP4jdB4#xBv}Hz%K03)2@I2srm=rTGhe0q;FUTkku5MM+OXtNO#_BDtHd6_+$4&g^$^7TVjbC*O6F~$Mt|E(Eyf216ZO4Cd!S5Vo4GJ zmL&-~N^UG1Lra%*qD=YZ4}ZdXG)&HzLlEBYD!%n}sBkyOb|jdnaa{=}Y7k7K0WgUMz(fs#sms752>{HJ1f9*74`9lc zF6o4TDZ_ds44$g8)N(K7Bv-({7tA_ptn#3BJ+RG9rQNn2RX`@+C4W6sxcO+il1#Ee z=}9tCgJcp7kV!N^CTft(b{Ux@0gzdeptJe%flS%bC7mc!OEP6xkA?x7D8ebT{f{D3 zJ!vglM~#)H%yxdt^qa0}7d-7zd{h5W;a<7zNib34`VvgkAecl0U=j_0N%r>vOsfn` zk^sOgNzmDR`2ePD>3@<=2$(XgN5XK*B&l)AwEhpktfR(CQ>L{&WqOu&i>Ay16N*At zf$d8&QRA7COw=HmL<3|J4UmZ%eE4IRkx3E&nI#E2n=c>8lr3G-32cEtRF6dgn9LRg zVA{_bOtB&I<4o33UL_{&Eha5(Ik~BBo5=#7jzSBFBaJ#l<9|t~4$&Z!6}&o3156S( zjI@WTGP@4zMEZ5g5_A+ecA%w8I#Fg@LJ6)N8U|Dnk+kBSs5+fHDV2+_>14QAV=0hK zZl}xnGMgyVRi1tVl+V<}bE@(#9@TW)e$L*w&E%Gg$9!NCJs^&>@eqwCeLO^iOjdC6 zFbyzCG_GW_%zw*t(e0+Ti~Ow0D}WO1AdYnO5RE4-Jw$_0RxtH24M0gW0A-o4ha~_g%bYz*8gGuF zrAsEnU(HaT9>ghlHVFfs=d?9DijlAC^FJS(c!q!Q}%jUD65eT0&TlhRNBIh+qY)Yj*iO z#^q1hwydSdDyvwXH~MJZ^sLUIH{Xc}>()rV+2Y4sr*B(s9mhiQOJ zq5(3?tUfFOkXdH+(SKm|ftD`mgpdhV9}CP|HxX3bWgzlY3c>h(F(d9P14X}W&9 zVEa;A1!$sg#F1tnqVc5JhiDMZ3T7Xs0W^sQ&@40iumpf+nb}8!*#}y>q!X|dKva)K z0hv|2KDuVF&tqi%ubzTh>Z=6hw$H~(Z(_V6DD46$(KF&mpMMY0_|oS?Gzet{pAXXj zlw>^ee?aRppAYKBzb`k0QsCeKE*)#PXh9}I(+n&!^deV?aKew)ylo`a2iC{ z!BH@o#Ks^@57#Pk)4F;MzL7w&(&W7Xc6%YN%>iv$%cy?{Xf3Dsf>11z? zG_z!b(dl9m7|Hi}u-^#o=II&AiftIfyV2^Mk;Xh4hkwyH`M&Bsduioj(m$An!POe(S!_&#QCg_k zbQ?yrdJE<9a}v&%pVD7xU&AB`r}sbI1pQ$UWk01i1*0%b?tjP*$>wdI2mb|7O9KQH z000080F!4X6tklfat;NA*7i^)lNur)e@??N5QOhMMdW+pBrT<~l2iiJ6P1uC0&%nK zZE6w!$Tp;Td)%Z=TPY`w>zVy_cI9GqkWKUp)(NdjlqD&OKrzjQs!Q~~UB^olIZqX* zjaE>i4jfulcLg(qX$!Ab8?au0ivmbF!c2+w-W!6kV|$R)CBdPBxYJfrA4*$ee?!?f ztpVq0y1)`V<@py1t&TIEt5beCL4s-_E07P p002+|000pH0000000000004l24wHc=J^^Esz$YUH6(s-w007S+@YVnT diff --git a/summary_results.json b/summary_results.json index 544c2ee7d..e722ddf56 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418992, + "Performance_Result": 0.421449, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73914, + "Performance_Result": 1.74527, "Performance_Units": "Samples/s" }, { @@ -860,7 +860,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.1275, + "Performance_Result": 20.7859, "Performance_Units": "Samples/s" } ] \ No newline at end of file From e94b325b245415f875e3695aadc0001d9927766b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 18:57:06 +0000 Subject: [PATCH 461/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10217 -> 10218 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c5137030b..aa90634c3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 18:38:51,252 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 18:38:51,252 submission_checker1.py:3266 INFO] --- -[2025-04-16 18:38:51,252 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 18:57:01,508 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 18:57:01,508 submission_checker1.py:3266 INFO] --- +[2025-04-16 18:57:01,508 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 18:38:51,252 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 18:57:01,508 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 18:38:51,252 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 18:57:01,508 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 18:38:51,252 submission_checker1.py:3282 INFO] --- -[2025-04-16 18:38:51,252 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 18:38:51,252 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 18:38:51,252 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 18:38:51,253 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 18:38:51,253 submission_checker1.py:3303 INFO] --- -[2025-04-16 18:38:51,253 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 18:38:52,602 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 18:57:01,508 submission_checker1.py:3282 INFO] --- +[2025-04-16 18:57:01,508 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 18:57:01,508 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 18:57:01,508 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 18:57:01,508 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 18:57:01,508 submission_checker1.py:3303 INFO] --- +[2025-04-16 18:57:01,508 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 18:57:02,677 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74527 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7152 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421449 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index d8d144396..c6ce0c4fd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1088 +1089 diff --git a/summary.csv b/summary.csv index b1a4aa60e..cbef8ceb6 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74527,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7152,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421449,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 4f6a85b9c..d9dfbbd4e 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74527, + "Result":1.7152, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ed1d115b597caa3562d65da22c90671d46350725..2cf937022be3083a675682ea0cc063b758b59079 100644 GIT binary patch delta 1424 zcmV;B1#kN4PwG#watID41^F`T3IG5QQ~&@N0Fyx`8nc=R>jr;nYvX?x$sH4q`G6!^ zKpg4fAsSEGc!&m(EMeqf8bFe0T#00nmxm<)B#YcUN}6p>p`}eafxSzJ>X0ZxB(e6u zjb+8YogE)Ot*Oi96kyxKG4!8oe(nt*nCJBnh8j1 zG!s(DScTM`khV0FbNpU1fTDZ7@H6j^2!tG9hV zTGu^mYk9d#rph4`9V3qP`4EjKeLh5kWR~#xFb$ANG(cvN&xa)dGK+jZ8hk#`(k7h{ zGQsCV!eD0YKm$FUAw{x7aTIZmNrV&RveTc@BP9LH{ zG)p*rmBABLnng~Z&6f{og42g|g4>o5)}dj*Cb0v-q*<}k=SR_e(!OOS zMV6X0x2-$` z3Mzl8uN0KKHXkd!iShEFv~!?D%ZMXwK1Ab7n-9?-lqGCFOaoAo@yP!Gt&40vtP_B; z$mXNL<^wHl(up!HNhSDvXc%Bg%s{Y`t=Q-D7?sH&15!Qdd2A)Mm0G}Vn|#vyX5Z6! zKsq@YgYGSZkw<1iX0-&eh7@2p%*x<}Kn|8hB~#ccfNB6#_t_rlqK(Q{t? zV(3@Kp^=t;FHNgk_L6;I!#X;ccZb2JZZsUvVq+2v(n8Z?Q!}E~ODLD0lW@NLl>SQl z8YW3Nz5i(xbo)V+{gmDm48kzE{~-)JB%7Cc9{m?kO9KRxVI~^0qZ4ut1p=7)J0_DF zA|HQF!!Qtp?>t51d*URerLvM#0@M?gkO%^Cv+Qka5&y_GqP=)GRYOB6Xz6{n3>P@*F^w5sk3W(d<3UadA@y#N;l zkaC2X679V=1Y^hcAgN1&Lj`fGt)xDbc87lrW#4oMI8W0Bmf$I;p5hZQHX{-BNX|y2 z`LRt0;0!|p5|np1OEP@%da%+>Hgw9BQwo1HFlimc(S2}YG`4M2!vs)v|0tqGs`7-MY000oP eE+#ku1p=7)J0_EBCsYAklfWk<1`Z_v0002s+K!3< delta 1406 zcmV-^1%dkNPw7vvatH|!|EZ$u3IG5RvwH{%2Y;5W^Pa};xOmJ5CeZ`pNE;8)c+$s1 zG{|HLClAvAlSJc6CX38GECDcCWam-RZF34OZPE#BUP4rdL=hy3w+HSlD>m-z5cyZ^ zU{+FKDJZx7Ji1xecAlSAc{xy`9mJ829-{H2rH5z`$`Ym?rU5942B0kR^{@m0Ws$Q- zNq^(bDYUdnCkQ1Nct{j3UWtVV7q1n2c>a1QpQ*FwLhV|6EVTvHE|Y!^ljs?7q_u}= zJn8Kr8f3DByN79jNumKJiwr(20Weu)@loLMftEJugt!U7=R?BKO+Zqko1kK=4?_9# z?F7%%?sKWO{XR}TJvq&s1*|qXml_Qtj(>Fd5RE5YK12gtGL7VtX(X3SBe~Q?E+3Xa za#@t1rNQL`Ep5^X?pi`vhla`7l89ght1EW-JjUfu*|w~t$TF)~y>0Z-y6#z9%gbFd zRSubG8F8f1hiE)$^dTA~vxL!yX@E?k0Wyn>J}d!{S!DFlVDy2OHtB?r2}U0h27k*~ zmKv6^x?-cx-$Ujzb^2WBZKqGVlx;e`onwvFItNX3jX2WkLo}YW`VbAGS;FeWG=L`2 z0GdTsAC>^nEVBA+zI;FvtUja@+_!|V4h;h~i600i&5Er)KZ@p)HZChEvecxx?e)nX z{aAj^zNK#Fkcp-dM_PS|#*_ar3H2V+@qFKW1 z!!&><(Eyr7W*?RS&@3|hXfXRgOPh28mI8?CuqYt2jMqn3?Dct!%>UI>P=85%rJ&sP z`B>>qjF$(codYF$MjYw$AsS!$e2502EaCHE8i10FNB$3JUF7p&odA?YJ|7J}A82Wl zPLyd$D#7SO!vIU-27;Ar#YUgUs7wYKkm^a#V=Jkx)B<+f<&)kw`<~7N(#ZjmylO|9 ze2B)ECLf|fAWN8hm^~c#O9=em&`yG?!OIuD7ta2Rp7ZJ#L%%W(jkNT8 zXy)}C6gK=AAfGbFc5_AeTvBU#BnGn zvXUUR6{jLqDpf0Pmc4-0_(!(ez}we^5J2M8<9cSlon5(_?^T=pg7ebo8j&nV321JF z)Y}?;tQKjB5+9frtTh^HbO4X$%|pd4<;KCAGZtKs;FAc_o^o5GT?m$9?D-B9^I3H0 zC~k~XEX2}nv43U!n{5FXc|OAm0uwASd<3R;B%%Q+_=vPWuI&MWV`xEv4jz+?;EOka zQ~qv4uUt8m3p5mUB&&l)4Y9xW0A}^+C{+N>4ij;Ix(rK|i zj=br=0M#($-^4TEl>7D*G=H;PBgz5^5dW#7>k0q>5wkBQH~|GTjn2X)lWZqc0b7&7 MCnE+8B>(^b0Jm17Gynhq diff --git a/summary_results.json b/summary_results.json index e722ddf56..e891c9012 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74527, + "Performance_Result": 1.7152, "Performance_Units": "Samples/s" }, { From 9105383e73b72cd78a6589f369a8c3d85040f0a3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 19:02:05 +0000 Subject: [PATCH 462/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10218 -> 10218 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index aa90634c3..a3f479ada 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 18:57:01,508 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 18:57:01,508 submission_checker1.py:3266 INFO] --- -[2025-04-16 18:57:01,508 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 19:02:00,039 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 19:02:00,039 submission_checker1.py:3266 INFO] --- +[2025-04-16 19:02:00,039 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 18:57:01,508 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 19:02:00,039 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 18:57:01,508 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 19:02:00,039 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 18:57:01,508 submission_checker1.py:3282 INFO] --- -[2025-04-16 18:57:01,508 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 18:57:01,508 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 18:57:01,508 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 18:57:01,508 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 18:57:01,508 submission_checker1.py:3303 INFO] --- -[2025-04-16 18:57:01,508 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 18:57:02,677 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 19:02:00,039 submission_checker1.py:3282 INFO] --- +[2025-04-16 19:02:00,039 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 19:02:00,039 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 19:02:00,039 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 19:02:00,039 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 19:02:00,039 submission_checker1.py:3303 INFO] --- +[2025-04-16 19:02:00,039 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 19:02:01,334 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7152 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421449 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422377 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index c6ce0c4fd..336df5690 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1089 +1090 diff --git a/summary.csv b/summary.csv index cbef8ceb6..3e70d154f 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7152,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421449,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422377,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index d9dfbbd4e..b8f060852 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421449, + "Result":0.422377, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 2cf937022be3083a675682ea0cc063b758b59079..fcda487a8ec1d3529ac0f0734de5b5cfddb8aec0 100644 GIT binary patch delta 2351 zcmYM0i$BwQAHaWOt~J(5YL#hi#zD>OP_ElCwb>fFI3|f0TgPR(IGu9NLlXtr2lX8AKrSM4}<(!sHGZ$>nPfU0I@ccbi>*TVtUjF-W zh0j1-zIsJ*m1LT?f6_MW+AXOxe(haRk=F^^KkW)Dai zDN9@35#Af1jeB{PrVd2CTPrwsWQMt@)p%Ru!Wu8s2UStA=>z#1FFMmL_p~YZng~~v zU?a`^#k=2E)%K(!tB!r&xb3{)Bd#TH?CTVhZ6v>HI{S7I*;C=TUuO4r7vi!Ax93u4 z3@CqhX9cg#DE$T$GSq?R?<>k6z|wZ99t^TJ@vSa|RGSljnUg1au-F26rbRJPcQPHj z*z?JjYK=hTMHtOG4aH_}u;pU^B4Ryvb(kAOMjn1g9eaXktKsWYXMYo$ywvxqD32YK ziX#qvt@^8~xqYh6x{8!GNw$)VoO{G)0aOvl%3E{2e6L#-N)PCa2V0}Lr%K|+lI1UAbE5BdHJsDPKHTpcI_z&qD-${o>~J2Ch%X(Aot*If z5P5l;jP)LAfYJI*vq_qPFi0E^4nAlsxM^U_bvn$vD-V#q@&SDdz?R&pdYbTJ{*j&i z)fidKA%mRQYii#Qiu`F0oJ6}2BOyWf6g+$1(Pm;oPNm{RL@+XFLx+z`YhI z(teDB0|FL6t8Ow3^m7m7gMt52v8&Xd9<4B8i3iOnX$Onuxj^W5VA(qhSxs%OJyVhG zwV^>j{1~4H@6HSnX-iOWQvoZE)<4vO_;)gm4|G_OdtS#b5{klCCKk8`EOM~Ye-u(@ z3dsI=5y(w}WFW$-Q&b){>=`VZ-TCUeGA8HgrwQ}gxZQns-0(L~E1?h^qq(F7{|FV` zLO!4UL-k}Oz2I1}IgxrHSjX>m!Fr}0I&UNPn~I+ zG#CWS7@Y~yi<2Oeu4r5`tgWSBT4{hC0+tx6!H0K&jU0c6N`*6KXNw|*)O!&N(aH%4 zTT!o9D-rjP8%GyNl2}e`u4+4ch|fk{qwNrfz$Jw{af_8;ey0d*4KYKk-j^eTH9O#G z&$38hOW&hak};c|TxmHI?ViPWqKpt;d2y!uwO_zT{@s%ur1s<;zq_nh3Rf(1(6KHw z%@7zQoiRGMV)+y)yrM1qZh4V5yO@skZL99DOyxc|`YU$X1Sh=PJN8r5mf!md!)2eQ zw}zFz*uxUBTQleg(We4g5)0Y|qa%tyc%F8T)ybtwa9n7HENo7$d3{K|J#VNEeen!y zQzd1JfMxqksTaNR!rCYXC8N-+E26~n^LV@JRGsx1+a#Jgb$8#yYE(;52%phB(Zc;_ zo!se&td&qtF0nXEgw^Idz{Q4oEfhgP8>QxWardB`!uH;h|M&{cj@|HwZP>zrBP)Ut ztr-EUf&i7dE^XaYTy|`g=u3_(&K7|`zm#LR)C<8J>oK&uaNo20!B>0w9^&+csaaB)5qaM(6 zLBO?rVYblH5X;YtPJ5ZSj<`df=S+@JA__NFzA+Q3AHG^RZIVtmNFUwr*f-*af2jTf zg=jFEM_TXzYO^8082uE)%>&ZiO%M>1@59byQ%xU_xU~Re)v^ho+QW!dT~G_bqo;oS z!U{7c4{TKqWnp50ztpYX2nL5NR9DqhjMJGD)+ zaCqjGqouLnV=sQH$=~3F&6aOs-GGPVjhxTN)Hi-^+O%{vnpIFZNR=tG8bc5-D=!-; z-UZFx-2!Q^d6^APsO?d|>`!IQ!=&;t)Qw%Pg-v}A6DctI3>U8T?hh!_wa{KG0ZaSH z?J$rim^U@|ebcJ>bA7sjR(-ww7+L^FT5ykJ{?so;T?OVzXjBT}t=0KAeyy(65cZ%6 zhAs1v@+dB;p`C`=vUdsy3+%4^6R%p z-_P8lQ}N7~LyY%ClFv&c{*2k~n^HCA==C(L_jZ204ZR9Vz5gWZQ&gRnKNQ!l&(&CO zg?$S;=m_gwXYnPu*0p>B6d%kYTre;hV_fcj56~ASi2-8H51S1(nw4RqPtP=NO^bLF zI$e6d?dzUp|HcRNB}E5*Egp_j9^go~RR;PW9qvS$ln1$#4)g3`ts~o&>!aG;6J4N` zwS=U4j6C?W`ayuG!Su;dk4RbUL-;do-ARfc| E4?smza{vGU delta 2369 zcmYM0iC2pezbW9LL$qjL# zEK1XCrUrA#+%f~onNnx0G*fG4)W)JRKXlIdzWWC}=lR|H{LZ;^-gDl~@fzw_btQhj z^{hrHj5#=1aVOBLs_+Rb_MMGKKwLwOQ0+1XNj2e6<}Q!}YX6#m&QX5)IK9$h`b*x6 z4vl>iI!5kSieB*TYH9*lAWJVW^8gE<_xhnkci_dF#<8J4qnG7lW#2bRL$#*9q#+MT zWy6x|j!6kOZu>mnksW5xz#}aTPPobz=PueZyqKas>U%ah4kA6kvcfsKXtF6_33)pQ zM6y7cl_}>O4XNGjzeNX3RR_cD*l(yv>W}3<`|8L7*M!$AX_RXs9uf>nbL568r{Bn|~EAaD02vvxbA{$(VjbWeMFyCe@82`dNB+s=}9G51H( zH-_^pe-&MBnJOv_wq__!?f&rtb|Uh;R#_bK%GDj+ z7P?sbDw^*{Bsz@km`v$Z;w+;=n9I2#)j@1u`PUmff_m+UbPHkixZ~UlT9^ZDB zELa>spG%4Emm(iD*v`*shWy|gE?^}%<8Eax2Yg@#n9uAwxF8XS z=iRk$$8?d4<~?w}xgkh+7L@>CPw3JE)v#0`#1M>Obfg1^+S{n+YV>bZ!~A`?w1J;y z8SStBC9Uh4=m8$#_bY5{{P1(nF<<+L#Xxy@UPW=c)d0AaR}JdhP_QyUK!!z#HVm9J zJvq4P;GTZF9bmH3KP3;a{==*{n+96d33@|3qAbHe6b zrP#UPKXYiSajV5pnpD=YiUc)tQ=6~(OWO=sc*23j%^x?#P^Bo9z$PJ z>U3nu@+)qH{s_$*ItqY$AC7q!+SGr&$AObsB~r6izi1M3oa@DV#^m|r#O9xel#)S7 za&(i&jF-ps%UFkRc8wm@0=Z+e@f{DbgNdc$8en>l%|hi>hEZ|T-4n5|m_`F8`o+pD z(2rz1`*It~`+4$(4V;~!Q=uqQNlXAMf>V}n27ra*P9B#SUbSofQrd+%wuLL$vfNn| z+1pWNV!B!YXFl}M+SxnRUgf@b{`sTU)*ZVydL-uB$sfBBsuSPq!d;JbD|!fknGE3V zZ~vAL2zTO_EoSf9uVtoItG24c>b_+a_xI%UTVdKG!3dS)a=_)l=;DUTUs&>yP^2~H zV&bxHk-5Z~oJ;59KH%UuDxouib5CW5Ei@)FwRRHMRz!CXfgj(V~U=f&7>(S|cX_Uo=&694x!jKh=Yz z>?%aI5z5nrs(HISX&Up7arXX5y=x&!z1W(*pT1b$<(Fkj#5=wJ{2ha+#7 zlxh9xjR$J9rHuE~vYVcLw-YK`{APr#Mm=|jj0+FAUTBdEfo7LczTq*5BDMC<_8@^v z`@L$0%MdX+iu}Z!F(zMkv^KO8+XCyVKh~m{ahIjbJ0i{Y^Dhx@ONh_v&_g57+Mf&? z<7InZQp5;VRkLmcN|dG;p;%uhL;H-Qs!sc7>$+xPYt3)duOxz-%sop`=EDV5L?%Ko z@?4r^yMc(Nroxugyi+d_HDGzRx3up zVWtD0(}4b&h=qZTawd{J@^rGUD$q0#QT-30Jt)}@E($%3q+?rT+KQfA1>vYVdH7O&*Bezi;MaXWi6RT_@3h8KMtd0L36dfUs1;$Pcq7fgp@lthG1Y}bU7 zv)i#2n>yy+qf)%Q-2D{}W&XzzY4l&$0_b<#70s*h_LqB4ewo>xtQ39de#Vp~1g(U< zIr5;E!_VC9dDVZpBqOB9!J+(YzEhX@6v22TEF8H>?y3ksv`6%mW>HqdJ>Btz_toPw z3QM($7##q2)?PEO+a6OXf?kq2Xr9 zVdxY6Qnm&(uJtV=R&TZ6wmu_moIv%rG`~16SUGcMIxLCp`CfZeQ{urhvn(KeH=Osp z^o=yAkG9#~+dy3OgO3Pf*Vgo>R+NUqFi2t>A_a`p+41KD$am7cYTO!dZa&rtg5ciAx Date: Wed, 16 Apr 2025 21:10:41 +0000 Subject: [PATCH 463/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10218 -> 10218 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a3f479ada..466aacefb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 19:02:00,039 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 19:02:00,039 submission_checker1.py:3266 INFO] --- -[2025-04-16 19:02:00,039 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 21:10:36,111 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 21:10:36,111 submission_checker1.py:3266 INFO] --- +[2025-04-16 21:10:36,111 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 19:02:00,039 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 21:10:36,111 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 19:02:00,039 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 21:10:36,111 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 19:02:00,039 submission_checker1.py:3282 INFO] --- -[2025-04-16 19:02:00,039 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 19:02:00,039 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 19:02:00,039 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 19:02:00,039 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 19:02:00,039 submission_checker1.py:3303 INFO] --- -[2025-04-16 19:02:00,039 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 19:02:01,334 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 21:10:36,111 submission_checker1.py:3282 INFO] --- +[2025-04-16 21:10:36,111 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 21:10:36,111 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 21:10:36,111 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 21:10:36,111 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 21:10:36,112 submission_checker1.py:3303 INFO] --- +[2025-04-16 21:10:36,112 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 21:10:37,307 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7152 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73161 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422377 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 336df5690..97938e6e8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1090 +1091 diff --git a/summary.csv b/summary.csv index 3e70d154f..59cfb938b 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7152,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73161,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422377,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index b8f060852..8d63ac347 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7152, + "Result":1.73161, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fcda487a8ec1d3529ac0f0734de5b5cfddb8aec0..9cb16fd50629e45d93f91428acd843d1cbac7795 100644 GIT binary patch delta 1392 zcmV-$1&{jbPwG#watH}d#YBbd3IG5RvwH{%2Y+r;<=$HExOmJ5CeZ`pNE;8)c+$s1 zG{|HLClAvAlSJc6CX38GECDcCWam-RZF34OZPE#BUP4rdL=hy3w+HSlD>m-z5cyZ^ zU{+FKDJZx7Ji1xecAlSAc{xy`9mJ829-{H2rH5z`$`Ym?rU5942B0kR^{@m0Ws$Q- zNq^(bDYUdnCkQ1Nct{j3UWtVV7q1n2c>a1QpQ*FwLhV|6EVTvHE|Y!^ljs?7q_u}= zJn8Kr8f3DByN79jNumKJiwr(20Weu)@loLMftEJugt!U7=R?BKO+Zqko1kK=4?_9# z?F7%%?sKWO{XR}TJvq%BJApPiml_Qtj(>Fd5RE5YK12gtGL7VtX(X3SBe~Q?E+3Xa za#@t1rNQL`Ep5^X?pi`vhla`7l89ght1EW-JjUfu*|w~t$TF)~y>0Z-y6#z9%gbFd zRSubG8F8f1hiE)$^dTA~vxL!yX@E?k0Wyn>J}d!{S!DFlVDy2OHtB?r2}U0h27k*~ zmKv6^x?-cx-$Ujzb^2WBZKqGVlx;e`onwvFItNX3jX2WkLo}YW`VbAGS;FeWG=L`2 z0GdTsAC>^nEVBA+zI;FvtUja@+_!|V4h;h~i600i&5Er)KZ@p)HZChEvecxx?e)nX z{aAj^^H|-?Arnm_jM1O;1mazIT4UkDRKxUEEha~_qi>y8xtUl1vCY=y6 z!RkZ8aMC2Hanh{V>ht%I`Aoe&7kb<4lTDhg-^|&*)K(6f=o@jQ*@tL6Y4#x+M6-n1 zhiL#!q5(9E%swmupjl-0(O~w0mNw}GECmqNVNpP48LyA7*z5Bcng6S&pnsD3NI|oYij5yNgLo~kh`4A04S;FVTGyo+TkNh9dy2$6lIsqt)d_Ed{KG4!8 zohZ|iRD#imh5?qu4FoIMij6*xQJD-fAk~we$5v8XsRit|%O|~W_C1{kq>}?AdDV_I z`4EjSO+G|}K$bB1Fbx1n#(x7q7MXllCjexT$w!092U^;s6J^>GNHF=(FaVOgzz2YQ zP9C3pApIu+`CJ`7+Svut_25}gW9yBF(;ylKC&6SA8@(_+T&u`U>*_U-=W73Y^~S^p zdD>q%>yNWnJai|k_d6!z+0cmpT)jEc$=)1kX32Vk^Ti}ElJE22uzwcZ&C@fK6v~h3FpgC>94e}VUmQ?`=3TZ zw;x2=Pw7p;APkfH9|^KUvU!{5(SHH6h7*tu1Ye>O{gWCZAb)DZFc5_AeF~$yEz53A z5w-(u$*F-tQ(AHp(k2nLevs5UdHX6(;w0cxkE@yec6L!c+o>-31?Q#F4I){N642ZT zsW%P!SkBW&l=#53V4cy>pgnjrYwv1qDK`$@oU!171fN8Z_LSQOZ9}jWW6!ssn9rg^ zM{#YOVj-4pgMTgK-)sZ8$ny#-2u!fR@Bx_Gk%&%6!AGS1aoqqA976{RbnuvD1Yf)X zobrI>-g!A1`@YZma9^;;s&+_?YIV4AIkrxkXexH(RSrAHcT2`yW y$eaEPP@RVSn|KBsbKib~_7AgMBgz5^PQ^rp>vADJ|Kw}5J$Rrh{lsP z9-=`cOBi{W29P8gS0Y*Dbm_*BnBb_}& z<4J1|(IAs0%sos4OcD(+S>*3w34qBWhmQh_545yNC&WwuHXjm(W&)BL%>)%YeGtl@ z?B(v4xCyk$xzy+vaet)AhiE)$@*x`Fl4&HDOe48u8p)+DGWoCs zlFOn5Ee$3gXlav9aMKdPIy6kqmP7TREo)^*R?T3+sw zsdC6f$A}|+K1Ab5pAXR>nI(KaOao*R4Uk#n^I-{q%p#wU2A>bKv`Hs~Oz`=TFn?Ia zvedAQ)fM}E{vI-)snO>`ZySBmrEJsj?Hp&U);VaRX~dCEAENQ3(}!pf%@R%@rU5jG z2GA^W`mh9mW|7lp^W_7Y;PfG#;I<`%b!Zr{N$fx{X;$p?`B5~Vv~O8Sk)rthfMU0IMV4uG@f+&5PuDlS;FbVG(aZN0GUNjAC>^fEOPp2aQZ+?n{-0R z1g8%P!%35*#!0hcr_bL*<}D_qHV;HULT_Iq}PXN5X}-^ zAEp5`i3ZRt^7^m@fM$``M}yY~TH2%&a1=mPheZLIWvo8BVyn+%Wd5&~f`3ZtD+T4Q z&Bsb_V!S*k?Hnl4GU7;^57GG2=0h|HWeJ-P(*TrYJn}z4>mr*E>ja=IviWGR`9MpX zbfQd4QVBjE8U|PrGZ3s~EB5(3MrAU{fK*R<9$QImr53Q;CZF`a+4poFkWLPe*9yodA$U9v=-JA82WlPLyd&Ai?89!vIL~{vH7G zIaz%2f%KmQ0_f_xNJ1ZBH?$I>tzg!M#F&qE62wuMEy>Rwl^qg0}82XiQXr!gz zOVjF>y<{KQu#OJq-C^*l8x6;^*q8)^w9xd})Qo8L63XT0B%CilrN7d?hDj1m?|&Kv z-F^^dKczPXgD_0)e+kGA$>wFANB;%0h7*tu1h}|2=aU*DAb(E7Fc5_AJVoStVkd

{XNeg7ebo3gsC=321JF z)Y}TZub1gPN_=2iu*PVp&;dMJ)Hfx!lp6=H&RB3kf=?nyd&+Hvb|F}bvFAHb%xBS| zqqs3nu@FnQ#ebIZZ?*+o5HiOK0uwAS+yPTN644PU_=vPWuIT}SV`xBu4j$)Oj?dlz zPWj1(Ub%2884eaEttWAG@4Xz2ZQEw;thdJeIsRO|y!ET5Qg^4~fa+3k${ny^TwUVJ zA|9?`KCI%2Hxk6-A?g!v$3=!d5J?O`hqx!@WA^mCUL>NrAjMruZqt0e&L1c#C?Q|n y$gBP{P#uT-n|J~YxvxJ#{RgvLBgz5^=4?dj>xHspMYbSqb*p^fP diff --git a/summary_results.json b/summary_results.json index eca2161c0..9e3396a78 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.7152, + "Performance_Result": 1.73161, "Performance_Units": "Samples/s" }, { From 971022db91f846ffeac535c2be9c33738828f078 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 16 Apr 2025 21:12:14 +0000 Subject: [PATCH 464/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10218 -> 10217 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 466aacefb..b331a1db7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 21:10:36,111 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 21:10:36,111 submission_checker1.py:3266 INFO] --- -[2025-04-16 21:10:36,111 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-16 21:12:09,198 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-16 21:12:09,198 submission_checker1.py:3266 INFO] --- +[2025-04-16 21:12:09,198 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 21:10:36,111 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-16 21:12:09,198 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 21:10:36,111 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-16 21:12:09,198 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 21:10:36,111 submission_checker1.py:3282 INFO] --- -[2025-04-16 21:10:36,111 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 21:10:36,111 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 21:10:36,111 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 21:10:36,111 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 21:10:36,112 submission_checker1.py:3303 INFO] --- -[2025-04-16 21:10:36,112 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 21:10:37,307 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-16 21:12:09,198 submission_checker1.py:3282 INFO] --- +[2025-04-16 21:12:09,198 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-16 21:12:09,198 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-16 21:12:09,198 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-16 21:12:09,198 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-16 21:12:09,199 submission_checker1.py:3303 INFO] --- +[2025-04-16 21:12:09,199 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-16 21:12:10,230 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73161 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422377 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41991 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 97938e6e8..9c79b4bab 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1091 +1092 diff --git a/summary.csv b/summary.csv index 59cfb938b..e7aefea15 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73161,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422377,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41991,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 8d63ac347..80f3d5795 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422377, + "Result":0.41991, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9cb16fd50629e45d93f91428acd843d1cbac7795..5c49d860717e7b6333899b5dfc814612d237d96b 100644 GIT binary patch delta 3027 zcmYk8c{J2*8^_1KWt(IRBW5g-nGlloF_=NJ8*6%uWeS5lWgl5*tnovZ(SjyhS%xg7 zk|A3Qqm-hu4~EDjYp;6VbKdvhH6pZmJL-|uzKy?Bmst|6S4@0w)L-6ehyNCb%C zbtQZ;xx}21U|en;9n}BH3ei6qlB6P+M(r-CDLtiF#CpCTsbf&*<@My__lp+fh=9En zD48p*W!2PkhqYkL^ZEHgPNWzoDOaheC#}UkZ)o#qR+}zAWui^)RgzmYR>XoHk{6$q zniVO>HL( zFXYgD-3XZ~k}~hI-|qM0$E#0e3l;uzJ|_vk_AWqiXgBb2&El7Uxn?Z_PVDfzJ`Z%CMo%-i$SwC9HT0gs5hL-~imNfB27#aaNn_~(QsJrs53pS<7gNRH z*A)8_T`RMrLlZjWiswz{S|r+s0)i))^hm8;0gQJIKMUQrXrfOjt%e*1s|HOz6Htc(5E7H}R`T+NQ-+T9Tn2g4(c_+{AdJ>Yn zd00V|<3}orvjI<{jTbyt1b5M66hsZ;=sj1W0X2MVR)f9SfMBYA`qE@y+Hz0EBPJfUmjX8S#anG6da+P_%%qtjrG)cU2LIMc?@hGNqaT> zrk%H}ZPdGhc0^*E&kCfkko!iIl$?#CacxhYNXSmk_&fm-cQ@5kbI@oqQWz~J@#a!~8$0!IK8S==KtsW3g{Q`n{95hZ z;BjWk+GORUq0sr$#xc(B=9^X`kt=x7g|RXUu5^D6VrgGY?(u6+uqV%sA*cCKwnXkw ztaQI39VJidm57?eYac=vnrOP^-)=eGb_Z-VUJ3Raz==H9huRoj-W_rnyk|)W@Q{s^ z3pKJymnzzmFS;jz730xuty#JIeM)%R4J~Z_Evh}_J-=Q3&B;^lr*lk?5EkR>zNtP6 zwpO>F)(2a!H_J6ZrJv;mf~-fMA=`)T)Y0tEn&eHb`{Pm^RdB$8A&^~DOj>Hc6hQPD z__X-!GA;bllcBaze=kF}q2m(U82>3xU7Pz5!!g6L{`L%~*m(x#t3U8XP@a$^>@$$3 zQ=Ock+N}>gBv;ck8m=}QRLk+p8&SL9TRdO4lo}jzoJnEM7~q^^fep;p$qJ^GX8(Sr zaz9cq53OyYGz==PGSw_K$L4XG_?sraoBP61+*En?xILv1JJ28`$f0eY?CAd;Vor%3 zaetU@R9K=}`q7bc{_5E?^Uk=OshI5-{H+7Q=MqWMV>f)txx1j!ARrKA)6^;K<8JPg z#|io`wdgnXn9l)|uUi$U>{X?28-u8SfcGa<>D1;@UB9_VJFyagE_93_6D#jn*o=lX zaIxFVigWpBx{4&=j4vU2Qu}hW+zyOBiCabBaDyL87OxZ!H8NBp67)vynQ7nDUKA13 zbbDUYt^EvovKw`^=y>>2gH@&1Jrz^yYZrL_SgZrf_Zh!)!2~9XR-P=b8EFF?+JBbV zre>TfWbdE3(m{rcP)V9zvkYaEEgM6F`HLTEK$5T)0T%4Q7v%kyQyNkfN^0d2k0O*@=pvK42Q0sa+4hjXXnClr=<7qpgL9u}=;-j0(^uGo!!=GxTgWihk4c~Fh$V8j9jpY5j?yT$2(QlYG*Z7d z({vzK(%>JlKD8ttqyV78M7s$5F0RY#QJI6cnu6WX%7b-(vmOpxE_DP1pMYQOcRGzN z_oL-nK+4xDhtmSD4}mQp6gEu~cb~fEcEdrqa)&LGhW{}CQj&~ANviw0UBvV*re_%u zgo|H~GA`d~Ei+;`JY@X5l6}f$Nk)74)D)^#_+aQoGi{JlVJptxHchTVkP zw@5wxGeHRxhk~`?`7X#xa|eNqMQlorc*Mnvy;KhNbQ$S)0OlT!DaUO1&hFXY4Vk-6 zqKI0$#}Ru07f89OrZpz6h`SFeEq}o327`*tT_A!kkl#zmTkOL+c0mu9H>!to31V%< z^uU+9l9M6!E(3f%iY=u2MbkV?hh#hy;0ZP z2}JCepi709!K@o(J;j|^5Y%IF>qV6fx777T^lM)=hDSBm^20&pPW%tUGM-9+0~$ro z*4`4yf}b@)CYcl>wh)3K^f-r!cyTNS2Eyj{TPF4Y8~t%G3bFMD0m^&)lqVt9R_+#1 z2|i3nNu$e4b1QdY!ACn0vJaeRC=o32zRN9y&gctMNe1`fWY&uhYH}UG+a+0GEhvyi zc;GF=z{0b&so2i0t-ryN{hLvt7=e{JhSY$*ht~Ghq_=8{Sy9?T6=Caf&M`k;4fIsT zXbbMZZ)CX-^%67^Z@{DBk>mz{y##P1^u{JH9ue1Rd!qaXP@>u)d&tzv!L8+rq)-mn?CkLto& z^N;e>zruu?CZ1D3#S8?eTsekGZ=644Hxw{@r=snoZKH^FYY(At4gZkKb7o6Mjn}5( zStySU6NR*_{GvFr-$vo#&A!9Nr5|adnfQ7VWV_X$>fIiCTuMk>>ArGU#A>HZR`5rE zW?gIVF6V*fliSXY)sGy*O#V^Tn-#(!9%!uji3*P)ATDdPqkL0qA}mttw1OFg2<5(Q z_6b-&858XdSG4z<>G3N0Ou;_U;DvF$ggoiUb=?cNtuM5tyT&OQex8uHP4!S5WdFa9 zH(be;Ed@wZGGG%1XiBDR%>cKuKARF?s;qSMx2`ZY5Xg=l1QG@P?<8E~0TPvs*gSwP MWkqg5*zcSF0AehfJ^%m! delta 3045 zcmVuglKFn45%)&Hv>VsMc`!@A8HCZao1{NR!$v%hf^L5qG@UfGX*!K* zcRZ`DGVDiXa({$_!MGQ^344oaFiTcRqF~ZZ(&5LW@jR{_9rVZPcUe6~6bue)$7=uN zH|^9KM+eKkugAf+ci+ZQI^;kjeq<7AUn`Cqqm5YaQfToE45Cr z3RTWJR+(xO?bnrMI@-?8$M;FFj%loVU5@ZgH|ZW7MBz6hTJ{jnyV;ph`&y@mrI)pS z$(pmVHP#L1_YFo`M|SjXkD3RK+br=~aUnx4Wy6(hxZY}Lq~qBPO7lT!7=yVxhI3R{ zwiqUnIDfm6OCoW1C09h^?MkkR#NSAE%5Cy;n~*?>a#111dGV;?@=?Xrql)Vy#ZILb zuhfEgEhU~2bqTaVeBItq_xBR=3-JT#L0lsr z3&LCX-`!78r?R0<#wR-XBx5=lBWIk?C{dpS?YAf9#yTTrP&_ zCx4^G*Lp9UHl~yG?`gWyNk7elXmB&04T30LuLU>=GLy8jU)9T>SF>SPHuW<0N`WqPT!wqr?@MouC9rBoav?kw_AWM3U$gkVwKMi3Q;;3S z37|+M5=A1BD5As{9liGpP$c2}6>c<&_}t+}|S5uRBfCHvP>Xj-oO-i_3Naqd3d2iN+O>oDdD* z$TX5irjbArjW6=N-)I<;EP-UQC_!uU<%UXVX_HQrDW^dQ>yR+{(D5@%Eq@;$ooyBH zJm7qo6f-dUK`>7*$S4kOZdKh&=fxk!vyuu+@Oil&*a}duu6*ChxouSZvHP*Y$LzK( zu|$pQNG!?Ydccxs0863)EKvg!5;mSZ3l}4@|9}J=Zh2&c^EKZ;EC zq_u1nHI|w(o7+>SUw2JA=V?@r;+y)%3irxwPlAaW*Oy?T2Eim60F!6{OtQZZU|L0B zk^}%|L4wxi%Lg!JOPh2;z?5Mf5{6SINsUvc^?v|n6*ZQcGOgVy)3dajH)ZCSP!zh8 z(+(kjeEC48Y-y8DU<(AIIxGsnWVRpx(|*=qiVcw;XR?a&N-=5gFllMa*_!HhnauI& zD71h$(x^i;o^#$CwU#BQROMzntTH2%&Wi}<0;M$>K zKqV1LE8dx^(z%mTx%iq+hKn_p0?Fidx|}Z%vxzcY<>@Ct`Akhb=W5%pryg(11)XR3GP}#Scitm*^-E0e+8>6cKJNU zO(Z1wE7SYqFKV~f5S9@CeZ+zMOGh{0MIP5`fR>@KohJ!q!Zk?gs=_`12%~t z2qw*ntv)}B=94xqD=D(nq`B?&$sYY!e$Mk)-OM2qO(Tx9`VfsLtv*D9WR|e{Fb$AN zG(cvN)rTbjGK;J}8mvCh(k7h{GQsLY!f?_gsd3V**y{85koio#P(BxW+v}4}ny%l> z*}l|P4w~p2airOYXgq25AsR%pgxQB_08OF+G>gnWECHZdWcJZu_JNi*=>#kV5Y=H( zKxP@QkFMD3^B9@`tEZrn`jY|@BYzo>{2$P|$mhd40Vs=nJ{o*J(9$NIDASTug3*VD z0hYuK1S{E!jXsZ2nG7-@)svpbR#IE31?;xVC%td>J)H-nlLI7q)s8gz5RETQK172+ zmN5A+4FE~T13(s;d{`#{WRb~7gUJV4+N2X@+7d`G`Oq)`lDxnNfP79KpOd5#6Mv0| z(;ylKC&6SA8@(_+T&u`U>*_U-=W73Y^~S^pdD>q%>yNWnJai|k_d6!z+0cmpT)jEc z$=)1kX32Vk^Ti}ElJE22uom3S(=(J6TQmB1gVj4Djd?Tv~h3FpgC>94e}VUmQ?`=3TZw;x2=Pw7p;APkfHAF@NTd7J0ae*sWS z0|b*{CK|J&6LJm&U!oHIlNll)e`>=p5QOi23ZuI%%Wh2(wgYX+sewXMT5=Q8CK0uM zkkmSP`zlW2B;ZqztC{_Fc2PassV?~i=cUmNB3X_S(A)^AHx2q&&eKPf_`tMaozc*s zJ$N*0?`m!-HxAyMvEYIPpG1)Ml-mYvL$DNM&$pnM&!R&|ac!JpA(n1~e=XzRYy-H+ z^9m~nOt8T40hrp6h)zhsN2L96-2f0ALk9|U@R(!-U%UaF@{@7@M&*JFb zc{v*UzR&t{u*UoZe=T0$hgDOl52xaQ+FEeR9k5_rTjT2@9Jx9L zMUH(Sk{Ey<5K3}dR;$CvoBj(>ore6Icm^DE-+qGj zvsxp|0trsVM1|}M000q_E+$t2hLeUSJOL1s!zMuiwUY%WIsqq>Kqn>}U!oHIB>?~c nPyzq|5dZ)H0000000000fPn~;fhRrzT$8{jBL)s700000hMub# diff --git a/summary_results.json b/summary_results.json index 9e3396a78..802b6a9c7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.422377, + "Performance_Result": 0.41991, "Performance_Units": "Samples/s" }, { From 9cb619fc63e18573e425278d703d3aaf0e9705a8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 17 Apr 2025 20:02:05 +0000 Subject: [PATCH 465/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10217 -> 10216 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b331a1db7..e46be650f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-16 21:12:09,198 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-16 21:12:09,198 submission_checker1.py:3266 INFO] --- -[2025-04-16 21:12:09,198 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-17 20:01:59,730 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-17 20:01:59,730 submission_checker1.py:3266 INFO] --- +[2025-04-17 20:01:59,730 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-16 21:12:09,198 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-17 20:01:59,730 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-16 21:12:09,198 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-17 20:01:59,730 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-16 21:12:09,198 submission_checker1.py:3282 INFO] --- -[2025-04-16 21:12:09,198 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-16 21:12:09,198 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-16 21:12:09,198 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-16 21:12:09,198 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-16 21:12:09,199 submission_checker1.py:3303 INFO] --- -[2025-04-16 21:12:09,199 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-16 21:12:10,230 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-17 20:01:59,730 submission_checker1.py:3282 INFO] --- +[2025-04-17 20:01:59,730 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-17 20:01:59,730 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-17 20:01:59,730 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-17 20:01:59,730 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-17 20:01:59,730 submission_checker1.py:3303 INFO] --- +[2025-04-17 20:01:59,731 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-17 20:02:00,765 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73161 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74871 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41991 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 9c79b4bab..113c4d24f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1092 +1093 diff --git a/summary.csv b/summary.csv index e7aefea15..7062975ad 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73161,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74871,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41991,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 80f3d5795..a44a8507d 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73161, + "Result":1.74871, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5c49d860717e7b6333899b5dfc814612d237d96b..4606c5e47419bfc5a8060f80c9dd72cf2cd0f9a0 100644 GIT binary patch delta 1418 zcmV;51$FxAPv}puatIDCA_pex3IG5QQ~&@N0Fyx`8nc=R>jrk(kJ1!pcfl2g$ zIMT*LG@kVF5DhX}!pXxlz$DSQlF1@74@&?{7TI}}blaRlNt;vxo0kyPAyEWL;_ZPu z%ZiOVJ4F5!JD8PpuoRTrejeSdYdg=*s=OR1(GKEBM-S0>($YgT2xSRV57PjYL<3M3 z`FdCafU?NhqojZF<`hcWq!NS@3_K(X7q7&^gNxUSJv@Isl+V=JbD?&vJ(k)6YL`hr zhe`B|IMUifG@kVK5DhX}!rj9(z$DQClSKv}mH?P6viK-TSr)*nR(#bNbSiNoZ z(Yo$gTg%H`GF1+lXc=*&(T8X}Y4jl)B(sFkhiQOJq5(3Cj6N&@kXdB((O~p}k~XP? zkO@W~5(aO(Z1wE7SY zqFKV~!!&><(Eyr7Rv(rC&@8h0Yz{u4305Cc3GQ1$ScirIo5T+UlV-(MpC3i@NgJ1y zbh6Z>x$X7I9{pH(e^jZPIb@<~#F17XqVc5FhiHG0%o0`~rU5dE2FNV3`mh8*W|7rL zgVhI0+N2UfCRlw)7*3ibHBOopTYdf>GM}l}=R$9LeX>c@^_w}{m)gof6MZ9&H2V;Z zC(S-YgJ_m8`!EflNi=|Fk=chO05prtJ{rtEP|_xqfTaMUIxGsvEaUaj6?=UiBlCat z6jXoGzfw@{`h2YPCdSKy($0YrJtL0v`4EjSeLh5kP?qrdFbzOS+9Ur0S{M0zSS0{u zk5ZlJ zGz@?w@9zO1pOeQYA4va6Kt5N8k9Kx}bUk?1)7X09;WUVb!AUTg#6~YnhietNDP6q< z@?7m-uU?q=AW!QHXZ>;ZiihrG^?Ju-JR2JEpQ{%~I@yaO%`91OaK4xXM)G|g9M*q= zyLozsvd-3w{@q~p%1C1#jl*c1d|%a`y|Z#L=^jnP{>x>f7PIl6i{Ryp-V0~{MbCNl zi=kf`heq1PExyEH);=Anj;6HZ>z!y@Yc4ISJ>>PwB6; zuVIpe)BA6upxY0k>|1(MFbKos{!0utB%7Cc9{m?kO9KRxVI~^0qZ4ut1u8g>#w3#( zA|8LsYQr!PMfd#*qrENJ=?h^y(3Y$kC^V&^J0VRRQR{)E*2&*jwi73TthyZCJ7?|; zD(5@ZCcogkG`d20Mo5_#wiwJ=^B4*8UJPt;DV5QtROJK0>eEpwG$B?kb+N0 z`{UXHAUK8=6zJe_p5^%D4d9f&+At_*P9?+M!d2^0oZLGvCu7%jSvMQ3F@KKN%h&f| z)l};KR2)!U3QoBL7L2P)d|t%EwakZQJn=??c-lw(%G+U)u@6KN1JE(1TG>6CbKCfJplzO YIF807lWQkb0a}y5CnE+7B>(^b0Qh*Ay8r+H delta 1405 zcmV-@1%mqMPw7vvatH*07;L4pb_fp#e{NId-dgUsc+3YT(F5X08xPTV(#JzI$Ycp8 z57PjXMB_>(i_APM0Weu)=TXvaa|$JGQVDEcLR5!D5hRJX2ktB@Hty^Y`B&^`rB4>}1f5w|r zC~1>Q5K1uckSJWd5(^J5UMu$S{Pj>iQ)kbG+O_srY73}cCjA^H(KF&mYY)+Q(%VBc z$Ycq357PjXL<39~8GKj*V6w>Kqrl?>C2dj(aT9>ghlHVhz7W18p$QoNG_R1a;b}4J}iOcvM51I zgUbg>+N2WPwS=$^4U@Aa5y1*pSM2h6jLV<0ZCObt%dBGcw$VrHx@T=IFL%jQIb@<` z#F0iHqVc5BhiH(@5=I}U0WygO$SgAYumnJ6ke=K8JYFNhV zij6*h51G%@>2smCoj&POw(0nGjx|>695m52;z+9x(RkA8Lo|qH39Ap&0GdPtXck$0 zSOP$^$m+8>_<$x@eMlv^ZwX-?8U}05z3uhMCQa9G=4@YTD+f*VjX2WmLo}W=`w$JHS;FkYG=L`2 z0GdT+AC>^nEHe9OF#AABn^Xdp0*LCcC?KQlxayS z!RSN7088Qqf|YE=MxV#1Oa>W{>PgRIE9qOQ1?;xVC%td>J)H-nlLI7q(~dOx5RETQ zK172+mN5A+4FE~ne*-`knS59!0A!KLM}x@+O4_6nW!e%*F!|6h0Fu1F2Y`G|9-n+5 z{U-tWTpd2z*#*+|;8{;&>xGBYAQ}cI!DJE}y)YfFRph30^%lr;wST>OVd8^4tuLJQ z$Jr|$x|7xG9h32FXvBZ6UL5IUFOD>`WWB-pViFk1_jzzwe+%yB=^4s8TQmB1gVie| zjd?T{XNeg7ebo3Xv>F z321JF)Y}TZub1gPN_=2iu*PVp&;dMJ)Hfx!lp6=H&RB3kf=?nyd&+Hvb|F}bvFAHb z%xBS|qqs3nu@FnQf5n#ZZ?*+o8gKy@iN Date: Thu, 17 Apr 2025 20:09:06 +0000 Subject: [PATCH 466/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10216 -> 10218 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index e46be650f..a2c2e3b12 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-17 20:01:59,730 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-17 20:01:59,730 submission_checker1.py:3266 INFO] --- -[2025-04-17 20:01:59,730 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-17 20:09:01,299 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-17 20:09:01,299 submission_checker1.py:3266 INFO] --- +[2025-04-17 20:09:01,299 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-17 20:01:59,730 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-17 20:09:01,299 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-17 20:01:59,730 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-17 20:09:01,299 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-17 20:01:59,730 submission_checker1.py:3282 INFO] --- -[2025-04-17 20:01:59,730 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-17 20:01:59,730 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-17 20:01:59,730 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-17 20:01:59,730 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-17 20:01:59,730 submission_checker1.py:3303 INFO] --- -[2025-04-17 20:01:59,731 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-17 20:02:00,765 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-17 20:09:01,299 submission_checker1.py:3282 INFO] --- +[2025-04-17 20:09:01,299 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-17 20:09:01,299 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-17 20:09:01,299 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-17 20:09:01,299 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-17 20:09:01,299 submission_checker1.py:3303 INFO] --- +[2025-04-17 20:09:01,299 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-17 20:09:02,282 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,8 +24,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74871 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41991 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70869 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417473 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 113c4d24f..cf23797bc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1093 +1094 diff --git a/summary.csv b/summary.csv index 7062975ad..264c701c1 100644 --- a/summary.csv +++ b/summary.csv @@ -2,8 +2,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74871,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41991,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70869,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417473,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index a44a8507d..9ed59e1ad 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74871, + "Result":1.70869, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.41991, + "Result":0.417473, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4606c5e47419bfc5a8060f80c9dd72cf2cd0f9a0..ef2d6f2995b42995f5ab4873f80375a37b6da926 100644 GIT binary patch delta 1798 zcmV+h2l@EuPwG#watH~I?De`TXQcBuR{@@-s@5DKuwp zg-8xZH{D)#KeLzKzN}L14O1ZJ-;*%vjc#sr-Km>ukL8m z=95(nG#d%^GfI>xG-uwwhr}wnQi8;5Z)Z|$TH3X8mXo9mBQ5DE2ygiUlv7#ICKbq~ zsEp}gj56g~6g8H-aq+9AZQAXH@!q7n%y8#9KYLk|^bAf0bo|Ux%g2*Q4Jm)FeBa7> z4yyQL_hW^R*=<{5i5l0DSdz!}fF;oYmP7+sq6Q|)jfP@L5&)J330g{SESy3~n^dAq z`Q#76Iy6kqm_!6q=2i!uBbHn9WyYm8>*@($R?){Y6X&{NYW?iF?zv9RU8Le$Psa*( zb8JU~i5l0HV4?=WBpLvdXaIjq)F7C;2uzXyz${46+8lfUQk{+O8y%Y*2cVOw=HmL<3|J4UmZ% zB(qsWCP@Hf79?nG4nB}6OWLFoWok*L4C~M^KodncWj6m&WU42vWvhSaW2q^#xjkk2 zb=S0Wp7todsei0+uiW+|n5c1m2_|X~OrilWi3Y$V`}+W50glT~!D z6qEK2la{uet*LI8$sC`KLJNo^jXFf*Nv96cAd@A$I!psh5;u&rhp8gF4y#1^b&3+S z6gYOEq)jSOW>Z25t{oZ%R1%T2;+?4~ojWO&i?8WqxL9K;kW7DWr_1>=n<&#&o_+$9 z&(y?ouC`q~DqX;OPTnTTj*G{9U=lqbjFiAA7WU|Q2!x8|K zMRpz~-8QFC(k7L_<|RaRNEAVmczfW^vSQ=T4v~Mw4rV1CECuDZpGP#_9$t*IfarosRW?}0}qM9#VfJ! z;NrDn56@o@*OiEItZ6K2Xvol@K=p_(DSc zTM`khV0FbVpU1fTDchEnbh6AUR&N`9w61%Wm-F`_TIG<5mJvr9eTc@BMjxUQ2$^8?Az`qLWvO8qt1CA8{5@noQ>V{`-gf$= zOWCI5+c{5tw9Y{jT_cXP`VfsLtv*D9XqK@0Fb$wdG=OH2)rTbjG>fc0n}ZK%g4Ksq zg8P;b)}dj*Ch-Hoq*<}m=SR_e(#B;aoh&tJZhHuQvPVCbpR;e1aS{0fUbT~K5-C@5NidnjMlVcAC*^4+sRXDiDj`t>VrSXY)*^n$Hl+D`oTN=#sVkP_d*{rZ zk<0l`HOVhHFO9BHo)MIQ=0-@psnGjsk=~=k2c`vUjD`wz;L*IkD!HZHICyo&f(sIS z5<%KiZY#76!BUJp--2R3iw+&dwQ-7tSh@|ie~f>#4d8;1TdW{3!2-iQFtrmA?U8~{ zNc-cO0U$Vr1{CPvah~P)=ndeMzt}J+Cr%|pXW^ptAWrU`my@w=+pL`p)|fxXpUaoG zVbxUX{!|=LT?$UQ0~U;{OMF_y!!^u@Wjyg(f_Utre&KDu$k+!Wi2>*s_n>^to}O0= zBvcoqxJk)%n%}JoLdgRq`By*ktp5m9`yu}(9spzR^G{I!0kc{o$^r?F?DeCOiQUlfouJ4Ye5n0C;RKb98xZWpk5JClr$;CmI1KlRqaW8wr_sc_je= o08jz|01*HH00000000000DyrAlYl2a0bG;6CnE+9B>(^b0P>zK%>V!Z delta 1781 zcmYL~c{JPU8po4Zg9x$o3M1AK7mY~E2t}<~?5ZdlQ$>n)s;UdqkXmYQ^Gjo`4pQ3E zsiHHD)=QCj`^CAn#(PO3N%qISr7OLVa-!&CL<$05_<8Z^c~~_{j;s4?G)JyLnwE2t&RR(7C9{sSW9h(V z==-TU2?;0o#xWDP!W=@G+}b%}LlyqTYtDR1HUSOnCwWBQs^VKVrD ztf1Hplt=y5;c)t|`A66lxXOv3`jeM@2n)254{5u9D*e`%$%vY`J{%l4>-R4Eh6!vLHf%*d`YsCA$BXc9Bh+E!_Idj7J%W`$DXNoF4M zH7ll8USNVW0OB(!qtiUp^BWW<9n@>-Jt)#kHwa?45lx%rAOYWL%TaO1)iP+#3D`e! zdP{7>Jhn(Sw2xRCt$(~ioLWBcWXU`w!+yBI4PT8HtHE$#-4A*AE)yG`_oXt6 zs-wipU`S=Dn$R#nmiR%EN$AfnA`slaX%Wh9r!;G_Ca&pEHYE^Vzk2vW&wNPrgSQua z=<@I|-3_mJ8X5#20kPa={Iz$khC(u0ddKTHYxA0W@>vKKM-!z3kNMr#3&!%D9O+K1 z*Q4VXYEmCLu|<$Gr9|gk;%1@!uxQWk^S)`MLFtT?ddmq0Bck&*t*;^ zS%7Df>u*JfjY{EU1Fj`$aCmw>Q#z)ym#hq%+<3EzME7QfQ0sZR{>*wq;)#%90j@E$ zBWZJ}eI@Kfe}5$Im}+^v&UwyIZoDIH(-3LWnl+S5%b?VP>9~TzBbMNsVKJLS)XhC+ z)=j7<3L)Pg%0{xEwEs}|`kmXn?Zw@Q#iRzmc2<(Z?{`iW2^|!CQi|+{#nRSoyK1o7 zux>FApCKLn3Xc$J>p{zw-9*(RKSM)MfzXu`N2PlTuH?gIf!IGn?xH+4h`Ob>Q(8tj zT#4c*f&SOfEA(I>S$zjG)dnOXu#k@*fx8GD`}gnhH*#JQvE_WvgOuC^EiNXqLvB4n za`JIh*|ve-vU1$l^Plr`S7JV0{WHc<7rAz~@qm}Zcmc4A&{OU6KBZs8i?2#QDXi{j z1&|tU;4C0Y!x&uJ%Ftk`%30VK_m}{>Mkx9j9#+j`>#I^-ht>F!ax`I(-Gns9->V@Q z`sf8LXL8;M-W~r{!z{C@U2l>Qx1wUPKoFukr(fXPk6!Wghg9eqUJR-!5gPsEo(HEW zzd^#ZMAyAGcI#)^$urac(NVVcUM0%X{J4(>56hBmKBp_^?^rsdXxjB)w0~GaCen5O zj)hG#Zl*OEU|&LbPS~d!>fhaD_<8>|BLW-P=V)s#PM@!y+n$9@vpPjs^Eh8q-c;%~uv{*g+)Tp8`cMsj)-pZ9bi_q5}hC*!in z>ucXc^sOgwcZAHK%}T&dl~ZSvw()&y@AIY12|PViD~riMtP4wCEINaQ(Y>8{UR`B> zt!s?ZTO*A7Apd+#z(MqJa5j*Ob_Bx#KH3%B2FRN^fYE@f8CrM$VZk7fKLi9)2mNz0 VG8F)Z*%5F!@Y2j!Ud43p=D!_5GHn0= diff --git a/summary_results.json b/summary_results.json index 03cb9ad5a..b99781f6b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.41991, + "Performance_Result": 0.417473, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74871, + "Performance_Result": 1.70869, "Performance_Units": "Samples/s" }, { From 59d4a91ba30de59de4419f4600b0fc1a95f5a8cb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 21 Apr 2025 23:49:36 +0000 Subject: [PATCH 467/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10218 -> 10217 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a2c2e3b12..88a656ff0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-17 20:09:01,299 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-17 20:09:01,299 submission_checker1.py:3266 INFO] --- -[2025-04-17 20:09:01,299 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-21 23:49:30,668 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 +[2025-04-21 23:49:30,668 submission_checker1.py:3266 INFO] --- +[2025-04-21 23:49:30,668 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-17 20:09:01,299 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-21 23:49:30,668 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 -[2025-04-17 20:09:01,299 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-21 23:49:30,669 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-17 20:09:01,299 submission_checker1.py:3282 INFO] --- -[2025-04-17 20:09:01,299 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-17 20:09:01,299 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-17 20:09:01,299 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-17 20:09:01,299 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-17 20:09:01,299 submission_checker1.py:3303 INFO] --- -[2025-04-17 20:09:01,299 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-17 20:09:02,282 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-21 23:49:30,669 submission_checker1.py:3282 INFO] --- +[2025-04-21 23:49:30,669 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 +[2025-04-21 23:49:30,669 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-21 23:49:30,669 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 +[2025-04-21 23:49:30,669 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-21 23:49:30,669 submission_checker1.py:3303 INFO] --- +[2025-04-21 23:49:30,669 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-21 23:49:31,769 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,8 +24,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70869 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417473 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74115 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421168 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index cf23797bc..fc15416f4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1094 +1095 diff --git a/summary.csv b/summary.csv index 264c701c1..30c5344fb 100644 --- a/summary.csv +++ b/summary.csv @@ -2,8 +2,8 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70869,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417473,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74115,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421168,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 9ed59e1ad..4b4f87c22 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70869, + "Result":1.74115, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417473, + "Result":0.421168, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ef2d6f2995b42995f5ab4873f80375a37b6da926..7a1e97e281df93d9174bdf4ac5ba4db24370c5fd 100644 GIT binary patch delta 2823 zcmYM0c{J4j8pmglCCe~`ChM4mvZX<`$^5KiETQa*Vl1U1vV;+ZQHuE@BkPQamXm!5gUH3WFeWozQah?qKbc!ZW5FruvU! zuaS;oaxk@iy2*Z_`ehXwYi8a-q`Pr#4H7T;Zr}G)Cph3~9!x9d$8I`HzfxNINB455 zvw^9YfY6n0|KAJR>%%ZJp0}@_b+^@rtuj*6H#7x)_e5`xohheMYr*58Z#}b3OwULL zO8^V{yPNWo5Hozl*qU;og!SCwOjmHW48i?W@l;cX3xgPWYzwf>7?VsU{o0e$d#_2a zPXuo4Ost%!v#vUPzF*gY>My!Jn(nPT-rwK7g|B+(S03U1@SBzPvW&FxM{Na{P2qv2 z4^-Pc37Wc?-THCuQ5H|~gfhz|P~J$@yCeN-!?eF|;rs^y zi|;r;$+$+n_?zdPo=c)qPm`J`1bAR5($RQs-J+8GddB?Ln}Cg=Cn<*HAY1vaa8~Ku zhHd8vJ!uu*XEldV&s-!wKQ9_O3VG%Zn*b;~_yfXihFzWHmmo(Wmpj8t!hnEM5KEdaVl3ZR?+?0A#J=cTAMYu8Z6~Q z$lLz(cjc30#NpUQcv)&i5iD;Uy}GTKIE1vEbb98!sYSgW@tu9)`l~U1A4Ze!NdmKT zc-kdoh~FoURkcZ|kVQ{tVtG1xVgIeQ%YJY>n=J3+dTSbg(MEzsAAwDW3(bzjBHFZ zM?g)|wrTXOXmAq3ZTg4U-TFxIo2OTO0igxEb6vserqm#nBp6}3VS9wBwZ7RNcAe=? zZ1|p?*Q{JYdkN*r{pHHF6H?+gUfja&usw(J3H+eyA?Cx@9BkkqQABBADXQs44+d&Z zW0~Y`9TPO7AX6;Gk-h=1CeQMz1Te`Oix^%{X(&ooH=6i#ah}eo;K=34h5(niiD!5} zEgw4C+16L_uiMKnw>!cd>^%9fYNMR4;FXc|udX@+xFe*s=<;tTn*s>Rw)47B!bXEk z6#~{u@dXivTS>-^gJ+fT9IZ8SI6`moJ$qk2z&cGSZx`p35Z`+gNc4C!d3;vqQlsWY zqMJXXG8aTVqM1QGJ<4gn>N5~B5EAqSeq<&>*P-FbWj(0na6=`Rs3tFJ5R_@m=|SUe z8#eKvaDj-p*uOu$R)paWzRUDt`BAGi`a)eW*WtoW5QBPj!29&;;N?>-vezEXApl zxk;@E7zkMV%9-gPl8FGv{D%tuLlsuvXWspohPRtf^V^e+aXtWAUI~5kLwWs^6;%f% zcCV6S;;|R4>lBh&iqjx-_aI<$qMF;^x>ex-0;ItYt<}1SO;K_kFBBc)xKk>0pF^Fh z!rNM#rYyT-hVc863R} zw_aqcF~}P3i_2(3wxyN+lK(^}eWkMK+_X`iw8SHJDmdH(2{;^?IdJ6ObU({WX1y7Xj7u;7VeHfP*XzhaoscWpckq zzzi8ZD;-c!mWe}N-(!ex>_|c|U@0O=GRero)Hh>4&}H8ae5fDplhPQ(%$b)#2E&*u zMi_Ggi9T;A{S#SU= zT;A!+UMB=B=96EEm}MQv5dX+z{!eD@j|`C{D@c?**t%9bkgc@%A+HS9&gZ0N?4wk6 zOT^*hPoIgUHDP5>G?Q5y`L#FmjRz7HqOivEggnTgTpBVGYsq<*!s_`AuDBnGEfGhbNY6tQD_Y*qUEwiu3+-^z{nGYoVC7GVn(_4>5G;e3T!My=t|p^zJZZ* zwI-dU?bLjVnQ1n!4nxm4Hv6bmxLLk18>;#pq*K~?b$1CK#-?qxe;^Isys&ravtw65 zYDTo0RhjE1CCZ~iS2r^>p{?W^Mrz>FWvBr2uc6EKr)VRtijVVTfg7)O)Bdse1w-La zH~;n%3;u;m+e@1k6KGd&{4KA?Ac;N};>t{eaR4y*anhXrVU81cq@lYl9~myJ_cpg! zoMrS}zHeq85ioAyUM)4F=kw)(o9b3ga&((+!TEbN2on*-;>NR9_olVWwq;EE?K}O- zj+`syNyzwV)hpmR-d;AsT`n<81KB-Pw=c-qiuYI>6GDXGZ-LyK^ z8lE-n$w$f!T2YdTsiu-Y$1Gem&61sX2U4wh*g#`PmK~(P}{LF442%vJ}08S z_-dEyr!2!w;SI-s57rYXSMWU`0i_Ry0IyLva4o=%)&-*g92#}(V3bHO2y~VM1QG%L X|Ha(l0bOl^BkX|5(Ti}i9ZP)o1e#pU_? zHE$ETzsK(-7@a4ke%bKS4r~4_-^Z8p$$^sG)Ko-PXH30K+Q8=F#0G7_;>iZN!APfI zyqF1-m==~8ofs&`Gr6bksq5v*^p|BW1v)qr-X-^a;ivFL`l$sSBBuA1~Mm?b4`^J#F(4Ez4isR|i zUR-s{v{yRu31P_jlZf7+UyJ9Ea!G0a?|!HjNm)+MTDj|5cH536cx^5{9xa#o*!3Hq z4%af738S3t;cdFMe#j4ONpU`E`<=VYdbZp>qeN>c%*Cm=aPdcIXaV>4tgg~gg3w2Q zU}?xuGVC;iAiVM{H&3R%DnM7YrF`AOfnHxxeNyLz8ge)cUw9|vt)AzHYt*qgA(@-X zLY4S7mV+VG=XJ47j<)%^!vm9A^um`l#;2(r;~A7oU~2R6Mg=hG;?480J(C*dBx0E; z0ahAvv5&odS+BGq)ZUNvXu8R)nrRom`8Fc(L~xwomGyO-&e_7aclzwOYK!uG<{P-L7ZzN+Kl@#iP|xd@eB%XV}>Y*>yaT~KOQm+d}{rKg81CBBpjDO zq~w})%!es}Pc@**ROv}I^z=`m!Ov23E`b|CZTCAeq>h;9FkX;i(d(Nh zQe*d?pQEaE?fq`~ch_jwz*}srr#s^3=M%QKo&r3U&;3innhd;8GM28d z5$e1Cd9(KE++uPbGqb_yI65Sn_WqZUVrg++i>`()$~MJWLG$L>`LjpZIh?)aPf)#ei5&+&Qcfe z`i@7=ef1v@zdAaSP zupMOXBY^ytNF)YKmgI)CL=8I#NS4h5<_9b(O|}so*GV3>H*gd6H)#x%>5!=;54y@` zS}@kqno|i(MW>p<;*?g5jhBCON@5zGEhzJowm8%6oZcV~_B?o$u;7}YK8usoq&@Rn zDSO~Eu^6>7Rn$D-J9p-6h!f6u(_AcYi6lNhmRIbRyZ2Sz)FzwWNoWkWq0f$?X9Ou` zma)9Pc-dYk(+ooGk_wt4X&k`N4Ah;{@714axDPg;C;}7uyu@DW!mud+-2vPFZc`8E zqk(e1C~TZe)^AAGLn*u@pZ3$zr3Y)%qBBkyQH!5Jjl_3?)>W0EANzttN`JX;x<@6C z`#4{v%8sQvrggKz;X@!An#+;FOF{G{MqkB%K{Or#oH@a02k&TE9O2U_313Aj28b*oFvHMx2J{?WM9XgTOoH0^BFEqLeF4buqr1sSdk z-Zq#l2=D}9-$JsQ__1lAR#me%6GuWpcC?y8@KGdw0%rxQ+n@Uk`EG56O}OFhIMcno0N^_!h?lVsIstHCo}>B!XJWp`iP)X zlr8=wTcz4C_OKsl>1hF*nnj{ z@*@i%yv48#3I~$P+X~y+0To~1W>a1NOAY@^rKDCJ;Ul!CmpHjZdAx&Xm&`A~xb? z147jDaAa-N^4i^q-PHjuQ2eVTfejIW_Jm$hLd@(vQjlaM6bnt%MbBU!iVQ&j_%mHB z7Z$9YP#068&@57h-VTTZ*GcN!zE^Z!#parwkwE29Szdc^YTa~RiAvr-${F!h3-|^{ zcjXi;@nQd#dae@lLVg%$`s=qHw<@2w=&88023t-cdXwV2LtVi}`jDi$6|qD`vK~4T zOY>bt3L`NZmH3*Tfp#zmFupAZpVWG2J&ed`{R&41m`NI@1%`YYthqWfD3VI)6uAaJ z3Ji;Ca;0gN0D@v>+<#624q{dmwq2Wx1wVTxXPm*}<*ospCkwh8c9q}LGW=@>R2)c$ z$#s<>w9K{5W4PHADoxx}mK@s!n8-5>)h(pSS-d?kSx9ZqJli7M*hF5~L|(}YFz3&9 zT|egbIDt^&CR27?YkOB8kFtCCI%?|nc9F$+@yz=(R^+-tLkSlOG^uWz-Btw(9qCDX zyp0sThtUYXmO2sx2WiBYqjDofJtIb9>?3w)Vx#xnS`jN;ZADJiBXg!hp~GDSc%G`J z7cqofV|tcWP)jPwJ%eX>b)-7TGggX5Vmwm@zz*`of2XZlv;COcfDJZ{9O3)(Sd9>l zWHV`sWe!Z!GUuQ(c~4Hgpk!>$h{UF0n5^70?=vU*3?<0WB#j-m;H$ra$esTQj{g-L zK*xi!Y$l<}eV0~!P`j%Lt z8ktSArv2PPQiKgK6VQE5oYYpRBOF9XeQg@~`mY}Wmu8OKeuHdHYq;wQcdj0qJEZ~` z@)iXTue~$hcJStCUR=d2=aQ}FI2OKuFqiA0=TYbx3qW)Tk^;w~uv`Xd8=w?pguH_Q z$O*qXp82d1 z8{LLMW_OsVVPmczY*cn>;a$D7c%^D8>lei8yO76kzm2$RMl0vrTxp?Ar;H&B>W_SQY-u^BE`8JLP551w(BG7CBzp~BdpAJW^yEueyM=7 Date: Tue, 22 Apr 2025 07:14:47 +0000 Subject: [PATCH 468/719] Updated results summary [skip ci] --- README.md | 57 ++++++++++++++++++++-------------------- dbversion | 2 +- summary.csv | 3 ++- summary.json | 60 ++++++++++++++++++++++++++++++++----------- summary.xlsx | Bin 10217 -> 10301 bytes summary_results.json | 42 +++++++++++++++++++++++++++--- 6 files changed, 116 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 88a656ff0..1fea89759 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-21 23:49:30,668 submission_checker1.py:3259 INFO] Results=24, NoResults=0, Power Results=0 -[2025-04-21 23:49:30,668 submission_checker1.py:3266 INFO] --- -[2025-04-21 23:49:30,668 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 07:14:42,031 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 +[2025-04-22 07:14:42,031 submission_checker1.py:3266 INFO] --- +[2025-04-22 07:14:42,031 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-21 23:49:30,668 submission_checker1.py:3272 INFO] Open Results=24, Open Power Results=0 +[2025-04-22 07:14:42,031 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 -[2025-04-21 23:49:30,669 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 07:14:42,031 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-21 23:49:30,669 submission_checker1.py:3282 INFO] --- -[2025-04-21 23:49:30,669 submission_checker1.py:3284 INFO] Systems=17, Power Systems=0 -[2025-04-21 23:49:30,669 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-21 23:49:30,669 submission_checker1.py:3293 INFO] Open Systems=17, Open Power Systems=0 -[2025-04-21 23:49:30,669 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-21 23:49:30,669 submission_checker1.py:3303 INFO] --- -[2025-04-21 23:49:30,669 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-21 23:49:31,769 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 07:14:42,031 submission_checker1.py:3282 INFO] --- +[2025-04-22 07:14:42,031 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 +[2025-04-22 07:14:42,031 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 07:14:42,031 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 +[2025-04-22 07:14:42,031 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 07:14:42,031 submission_checker1.py:3303 INFO] --- +[2025-04-22 07:14:42,031 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 07:14:43,192 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,18 +30,19 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.19947 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410188 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.4162 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.442878 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.19947 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410188 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418085 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index fc15416f4..ba851d843 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1095 +1096 diff --git a/summary.csv b/summary.csv index 30c5344fb..da8cdb62b 100644 --- a/summary.csv +++ b/summary.csv @@ -8,6 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.442878,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.19947,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -20,6 +21,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.4162,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418085,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4b4f87c22..fa21ddac1 100644 --- a/summary.json +++ b/summary.json @@ -270,6 +270,36 @@ "weight_data_types":"fp32" }, "9":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.442878, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -299,7 +329,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -639,7 +669,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.4162, + "Result":0.418085, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -659,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +719,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 7a1e97e281df93d9174bdf4ac5ba4db24370c5fd..627b9ada15300982b25d780bf375a32d8f40a9fa 100644 GIT binary patch delta 4505 zcmZvgX*d+#+s9`HSt3mKr9m@@u{1_a6k`0wSjtky_OoRfvPQ}}N@47bM)qZF6;F7&p7Z8@anALf z8XN#11R!0P79nM@2@>?PdRXK;);)h+ZD3=|6w!)684Pzd57u``+}3}wT3cpLJH$9VCAN6Q!ez540GG$h3mMfeyd{Lgw}nv30Rra2L$^1FC?Wp+p>!_uV z-3yjm%Q~pHga#vOyUP#BLwpweuc|a<$>LY!#f1sTr7^Bq20m$Wo*2)d+4Ib^zIE98 zffX?dv-?|U&Q8wnwd|BLE2kV5R~up&ySqCYXK)wGohxE_o0p54C=sYVmyXzTDV9o_ zD(-fROO;!j!%BgE8g;`jr}o!=oXc(=@#9TQ_Q(#Q=pwQR4r!xY^-K*(X!#dM7w_!dx0;#Dzg;(*@M=woKj=t; zwb63B#Lp zzoK#u_^^pSi-6Jumw-aW0-ioE8ziR7ZNew`Sv2(xsNha2AO`){o1TC^LBfJ)(O6UK>*F z_tvDewYQO+qtC}*o9)|1wYF3`tv5Z4Ue>6*Db z;?>Qkw*G;om#!D`KgIu_MU8{1z7J6L@cxvODJDAglEeK$!4f=QoJ|1I)KkFce{Muv z2tbEr^&A7zQP|X3SftRcxJKYC1yv+y3r3&+sD_@!)}NtCkoXVO44DkU-3jvPaOzU! zjqn}>JpnXwriQujtn(OHUwP%TvPBnSH9_G;8NHEYa^)z)C~}jZ4C>kPs(!AtNkmxB zEx2kmP&9H>YTdq@7>$fwRbd!<2?zFLq-iWOdNSO!hl9O3$PZ2|1Ql*NfP`i;B<}`V zfb6ULX_UgLd1fonAp44~wnbVbkkm2Bbk_OfWVQnIC`O^*ilYoYS6qat&m2=k)XG(a z>m#3_28VQzIsU(SxJ)hEM%s~DFdE>Q^YA$`pcJmME$h~F<7`#fAJmTgPNElbk?>CP?? zGT;MTWgoA+e(Qmn?W6{ZSFT^v!k-=is%Llo=_Nvym0%w6 z6E9{gG`fov04cbRl>Zv~{M2Wth!(Yfr5y~&a1O{l1`7XpBFg`9a2n5Wk%O`u_$E)D zC3WHlh88kB@2sF}o`Jl4sIg0j^eKIPH?#~w&`>ja?@q?NqJz|PVR?5v<#aTY6L~b$ z0wi%POlezTI==Q#2ubV#5O5?(8XHe?`52Jf64fvMdp~Q@sO6@H+uN{r_NUFPS?~o9 zI+7zH1?xrAQFxw@9FdS>Ad-0MGhF4Mk~!QZeHs%n&3cRs5g|^L77Ei7DzNNaqujl8 zaDd7fupC%!iOyENQw~9)W@??ZUCbKAhJ(P6AsL? zoZq2++c^OSlmLnTRV^z)jhV%aEbRow^dAjxm>GmV%i|+MoaHLm^h*Hx8yn(oyOJVy zR$Aj9kc z%lU2Ex9t;?1%@|4>|qd*Q(ES0`#-Zc_l+eEU4ad+%puEOObbD4LUpz4rh{0asVCjj z53__0Wz}u^KHOj2Fe-O-KXTtCn9?zU!*)pVxD8z8R#zXcvY4_EpM1~|((sDZGXABX z^gGIR(-&Vjl2 z%Gn?w`w)eo5y}hzydz2T!wEC;{0LpxBYm#4TOjKtGyN(?n|gCOOl5i%Cmq>iVIB?( zEI=vzrn%j(9~TZ`6tV|BN$Cs>7;Aq!s*}VLDCD|GpPX23nE%Z)i#+X?;_5RN3>x0Q zba1Ct2;+FS>qanlmY_^z+hbO}3-nD8!hyJfD6%=Qe$8c`s-^P>Kxn0RNE9y!U5KZi zr z2eM$_>-39zpgk)LL8|g+8ONQ!wN3X@Wde7_bD-7k&$_a7MlWSu8${*5LJ=Dp7l^)k zjGwh{PA~LR5!~TbHXC*a7cQOxLXAxom5K*vbkD8l)}%fT%J$pkdBFMplOb>FZ7dV^ zo6s9$rCnimh-d$>(MP0FSt}8=TECujPj16}{D}#U=zIT4FM*B}WUmgh7z~dU8Pbo= zSQ zx)s6`S4H!jlwRwl+vX!1y*x#2BL*Fk`1Eee7s}Yp+tIFlw>;b~dBxjTzgpzfC_?F3 zTIId>Z%LvU!9a8lCUGC=sS{Cmxw_lEXL$GHL??5u1nc5!tg`k1;L7z-+_w0Lcenv? zCupioQM>-~l*~x%oCm^A@zZOUG*VFgi-3vlx4KtjwRJWwK*-#9BLQS3c#wO> zqA^k`v_?6h;ANOe%5OP7wJrE;z!yZ+Rxt5Zydx)~>{nM32d=HgIS8v{8 zGp>A0<|HRtY=6h;)3pCW7zt$=3D4CuUN=7A4TEoLe27**_m`r-1if0?acE}-f$cA! zE$r-o&|O1om6sO3lafcg_ zMhRYK$A<+c!s%(uE6}vE!y<=#&-O8@peoDJFc=;j7Iiu99`OMH05$-C^h5eAbx>G`jhTGqY>Y>BPG7Y06%{7f-BO~hxS_Q-{x-+hVOw7*Vzxh}t4G9qxqgYJWFb={XF@KN z1$80?-EjrmLE@Tu9|B*ZJZm0<#oO|4-#1g%d0w~XUJG1gc_SO1dcHpuzuF`zq+Z90 z{TD1I7j5kc`<-8-?m>{Plf9vDT1 zw!2oa%1?Ocl-Qt()-X)eVUw(JRvGC#j5_SQpJkGSNX^Si-V-mYj7!WeYTa%Vb`U5X zsgqlC<~=iF{NwU;hc)nEU1%%4-XXb}`yn4FNn@on1fG+#s0w|#fozGVJ7@j4A9S-C zW;t(Hl8ifa%@d0}$V7qm#z%7V_L!*F=X*gM|Mx{gQj#_Y4v-?Hr9}VtdFKZJPD1|A z3V&otYL~taViO^qlDPs5B5BD;i=KoI0|EfXAOL{(KPQiC_#5u{1cy`(2N;K9(p z{_KUhv0o#Cyz%j8p0TLt=E`CJmybtcP~WPSb6+>E)Ez0sxOa13Y^_+aUoa+z^WL2x zZ#=1!(MdZEEn+;@>S?+)LkN_BcO!4U9&gGnexOCrQcMo_MBbFjm1=U8PjvNHtgM#Y z({}QC9nsUq_KxahysHG<6o$2%(3@BtEILrE;o|x=xb1JDq{U1_=RW9mxgonh#i30x z+*Lc+(;Bebn3{elO>;REdh+=}6$aBpGUq>RpQEYufZdl3wW0X?h>x9K8-@(}yeCr3 zroXniJm8zd4Y#>hy4XpuY(xa#Iz}00d}dEPzu;B)C|%SVgL-y0zjLQqzmEFRsJto0 zjcI=--AR6Kbae0-R+r>j6=ak2Lr->_n@eL=mf!M-VXSi&V|Z>^7G3hDIh2^ySTFsF zcI%i{%ToKrFVEyGio|)5152bgpSqY6A^mCFlABBM`YeskPgPg;h<1gj=O*SY>;kFx zJJaAGZ-0oH#@fD4O>EyX^u>FRL!ayvl~^A`zJb7l^0h<5{^_E)T~NP%32W_Cc!20_ z%}DXvAtv6ZY{AMgRa@k?tf3-U(3$W0zVrKszDPKwHl9<743AjRUf6Rcn5p8@*P?%ZmmN z%Nh#x1_b3#z>f0lNPg#>F(JcJm5n|7m53tx?tGk*7jiGB%UUdI%~x8AQ?RIiCP+L- zrWEz-yKBb;?7>Kc%d>lVwBbTFwbx`}41%%;YrDRj&hY$`FQ00%6T#Hso1n_n>JpCp zIl;aGw$AEl7GDZ@cs-EBm+jACvpfywP*)p0PhqiRsWZ_VG<;w2-mO=mK>r zXOd?JZVwrX5XXgBl+4{KRT=FTw4b`QxUkb&b}(H-4J+Q@_k#A<+GU?2>X3I*awfv`4kJ@327RTZq63Mh&!HV>TI7en>W zJsq^Ll=D(|bl+_qVb!(f)_cM-qWhL|aKro}7mu!mhg6~-6S0z;Qk3>wKmi{4P(;P%dQoFFX&yxmJ<8!a;sNEC?Q1vt zdq%3?zTAFyLI^N5vZqoOpCPj%*_lrNZY4LS%7WYrt@?4d(*rJIxGoQbAGUGVfjRJL zE&%uxUZZ{dp$Lo=-@vO1CniQ3yC7Xr`e@<&->T-(QN)|57`yiiw^!sG+NEs~)^3e8 zc>n~9bOz@B44Lt+^O)b5pU*cC%W{ysXN& z4nU}efBG^^A>jni_Ep29ud-Y>cx2(${>G=}5x$%4!)-T_p9OwObt8Hu$i*hEbajDc zh>~gBsY_>?ZiDJJOwuso36M4gCZ!_amhu6?hyjzNSfms*pB}!_ot+TK6*7iPI$a5q zWC`^OH=ZVgjFntPtuzx$N3egTxi{^(q=>w2pSY;0d$%1Jrf*fB#El|BD_*PJlVG zhIIbT6YRh}2&PN^BC2cmh1yxRU^%S?<=KG6wYD!F&)bqL6l27sAAQev6bv_P*vw)cE7?ITtdk#Bf znu|5@Nh&590uYGbQ`G73&@U{}lYcb=kJ(8P3I`t?;d7n(2XeVUmvLMlbf`tHZMNiV z%kIvN{%pK0PNCBjaerd`O26qB1Ff1x?-W4_02v3MHM_Vo00kmGM?$Xe?Ky`|s5MSXpa!pdYiP-@NB{WQ{DhaZfI337d187_$c|@h3GMF(eA&lr$oAGoBqw z&`Q9h@J~{E{_2S+Q!6Teyy`!LEXheQl)Zxr>gLO2^7J27A!JwX3T7@uY`SH6u%b+` z4C5&HQDDoc$i6RCj0byAJRI#31}{yumQ378a|OTY7}_KNpzj$)2z3y>>T;jR=yaW>0^ez8nfJ)z80y+y4c7)1Wul z`n;;%61i0nTf?Yr84@#qGUdUlO2N%TGOvFGbEq^9?GS)~EKSq=kvH@%;nUH+;pZ6) zp9<~q0taM0#TNC`TiWBn69Y~V&5SImR-E!|5zp}0Z0pnrr{uWQ8nlvrJYZr+T?&ez zs-&L;U^K{{uv&zelk#4Hi+Xm*t|ko?9}}`l8B8Onm&PGD5?8H^q{oa+?LII3Lo41s z8_C&+du$fOb}?#a{Sid`70mpf;ND+B1P?8O$K`5{;Ee!=T&&lgWE`A*jl(G?r=E5P zQQxkpWkz=@^W4G39vl`lJulE0i{TGZ)>wz<(~t9_k&s|zU9yL12Sb-6)$zf~RZL+G zLt8~XTYvmwB}HKn_7E3#JJ^CsU0U_$uRqz-iM(V+O51@?AFXP!GD$~=s_be(Zsaw=b z7a?WMarY)Nwbvy$1eeWl&f?Y9x*e|ClXaIqa^?L`e{X>TPO-XroN2DqD#> z&|Cx7CBMJ8nCBi!e^>u!OsOVa8BW^{=N0kbEWsPx@@22-7#|quz%CLTmLQ1};$zEb zE0=uV-#z`_&gq(z13dt#2&Z+(s!1IOe=n>ne;zv5sM}+-MeLTQX{8)6_4?>cEEuCO z(jPfrC||7H$a`ja8kAu)h>=a|W}$dNtuGi!8b7)tv}!wlCtB0e zWO>yPRQblDSI)$PFq>yQ!C)p#Dj#w%-QooM);@nZS~Ed1w%OmxeDg&L{A0 zI_p?`=4Ej@^VDwmaCDm^gC8QUpz02X4yiUiGKe_#@WjP2$tb)=zF8Dwa#Cl`w8)~*A5$^3p=rM1 z-5^Pa_)$-!d(@{|6C|wD@yY16XQH@FS7m9l(_D>1(fm8T)yOiOnrp!9oXk7&5(2Mq z%3D*YLHCC)>`S4;FW(Ie_+P9nz=;9UYU|s}>KlPIk=9noTdXX+ek3015iPPlc(>tC zvm=ZB?D@QGyLm7PH3NX0YDO7F4clHAoP@k%(}BKlKKuLn?x}-(p6C~6v&mua#(9&@ zmO4+S<%`X^@sRw_-HJ@M{)$iig9hlv^IK6UBg_3~ z2CvSkt>q6{2aR%Gv|(%QHEjt-w)3mhIP*fbJHH$vA6I>hX99g`A+0~_2Kr=}VHft@Uggx+ zw6zG~``vWk6HJVEl6f4smhHY9nwcH3ioBza?X^GQc0y%%DH=4%{r@|bK)vKoIFVz#H0pO81Ak?>a8TXC5$Z`-^} zF*PUN@sx|OisHD-c|D2wxU9W?w()^1N5Vl5@*$jzoEZQBP+si^0KVMET-v07MlKFr zO7cuJ2Pw(}#EaA?NHAy~2WxJGtSM5H-hn|>Yz(PlK@1ARd8Moe>MeXD%j;l|IUSpN z&SeGXZwc07#|??0J@SQ*A{)S(j6$XD5A`CKWGhd&HAjv6T`O5k$|<5V{?Q}S*w6J= zPG7HLTfqX1k|c}^bC09!_CC{r{b0|^c|Ducr=llhJ|iC<>D|HyW-ZxM<@xLgbBEPq zV!wuMJk`qLnNI?YN;L7R21Yl634*m0cNV9k4CS8dtiB{OtJd~@@jW*)(Pjo>bJtS+ zUm9eM?b5&_$HFM%yDEOfoA#{bGc95VfsAxBMq4wheO4sD>*>$ZTL~azo*zQwyv7Sf zhlYwuUZ&vk(aGO3`fT2L9*5x~+|bkzT7K~}1C>DbL$5VPiOsIx|1El#AS($4YFbd4 Q#0teM#DAR@^5@Qf04k$DCIA2c diff --git a/summary_results.json b/summary_results.json index 2762ba73c..9cebea959 100644 --- a/summary_results.json +++ b/summary_results.json @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.4162, + "Performance_Result": 0.418085, "Performance_Units": "Samples/s" }, { @@ -762,6 +762,42 @@ "Category": "open", "Suite": "edge", "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 76.951", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline", + "Software": "onnxruntime v1.21.1", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.442878, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0023", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -792,7 +828,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0023", + "ID": "5.0-0024", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -828,7 +864,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0024", + "ID": "5.0-0025", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From fc54993948dde9d8567e226051cfc51d95f5820e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 22 Apr 2025 10:15:03 +0000 Subject: [PATCH 469/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10301 -> 10302 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1fea89759..71d3c5f8f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 07:14:42,031 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 -[2025-04-22 07:14:42,031 submission_checker1.py:3266 INFO] --- -[2025-04-22 07:14:42,031 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 10:14:58,125 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 +[2025-04-22 10:14:58,125 submission_checker1.py:3266 INFO] --- +[2025-04-22 10:14:58,125 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 07:14:42,031 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 +[2025-04-22 10:14:58,125 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 -[2025-04-22 07:14:42,031 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 10:14:58,125 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 07:14:42,031 submission_checker1.py:3282 INFO] --- -[2025-04-22 07:14:42,031 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 -[2025-04-22 07:14:42,031 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 07:14:42,031 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 -[2025-04-22 07:14:42,031 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 07:14:42,031 submission_checker1.py:3303 INFO] --- -[2025-04-22 07:14:42,031 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 07:14:43,192 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 10:14:58,125 submission_checker1.py:3282 INFO] --- +[2025-04-22 10:14:58,125 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 +[2025-04-22 10:14:58,125 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 10:14:58,125 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 +[2025-04-22 10:14:58,125 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 10:14:58,125 submission_checker1.py:3303 INFO] --- +[2025-04-22 10:14:58,125 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 10:14:59,207 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -24,7 +24,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74115 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421168 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index ba851d843..54ba9c111 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1096 +1097 diff --git a/summary.csv b/summary.csv index da8cdb62b..9d6a42010 100644 --- a/summary.csv +++ b/summary.csv @@ -2,7 +2,7 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfMo MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74115,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421168,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index fa21ddac1..88affed40 100644 --- a/summary.json +++ b/summary.json @@ -99,7 +99,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74115, + "Result":1.73523, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 627b9ada15300982b25d780bf375a32d8f40a9fa..5cf72ec0ae703707069a95dcf079e4164f6ecf63 100644 GIT binary patch delta 2915 zcmV-p3!L=5P`*&GatH;6Y1{xdlXnPS2DH}R-CO9B+Xz;FjtrxC(o3_i@l7k4#bIx- z2%3yr+BBWkq&J#2mKpZrDmkLzaMTaqM*aCDoTkepaX9X!+3=Iw(JW~k9SlaM+b|(uSVgwWaG1um-01={>Z<492_>xT-pNdyTve0V`C5wd-HL6 z7Jd5=j&5#$)9kd_c}PDRCyRfL$tXLQx?xOu_lJ#k!}vBDq_=ru<90L{g!zeX82x#Y zMw8!{htw*;GE_OMSY@hpwBJ;g>1aFuJ-&~_RZMHy>tcj&dui|JAdbEn@uG)h*2{kx zwXbz{TKakGm%KS2TWi&D_RwHtb>wI7?x=mxy2}%PuN3DpAtK2p(w+#uDC>IrCoaav}E}m3e zKB>4WQ*2dg@k%X-*P>EX_@dG$6&Ft`E>R(W)Uq5~mD;q@*Eas9;u?NKB;t2JCK7SN z-V%v_0QNT`v9=}eh(z%8o=5~wABaR;K_7`koca!th~Mu7C0d-_fJnsI{X`_<>~@Jn zoZZhvBF^q9k%+VVg-9HJ!8!bbb0C2&FF2IAqTXYav_-vdiA2=<88WL z$RETHqz7@0d@Kp?JREzNpiX2%fE{FWueg4HWKb7_FxY$vmZ@blX3R%BwOlapJrh^ydF)5VVo`4!s|3l((7dYbuvow zwNG+=XE)7O8i(OxmDS1@Q>{v{s9my>;g6{%p3_pUwB8>GVFHPt$CF zHMO@=UH7BuaCFo7r7;N8-e{aOULCZCWWN4VM@iYLHOz*#_+P=e&X?m-%QZ&R(n>iV z$Ij*0xrm*(Ox~cx6-*tY1h69#i5-zh?1)5Mq3_)i>`1uy>;17Lyo2G&6u-#HX1w_8 z6kxn1!auo^NfIN5{Fw|(&cvr~jZEHu++O$k`5nz(b_26awRe*dIs2YQasT%EPB)#V zx!0d7ZB<~gj#4VHcI($<0yKFqPT$Gpu`mr9is$LBoc`tkw_GYM55@G zph&_ciY4J43`Z2b5)?5WQ1m2^0ubMwxwNvt6(+i za~6GMQBd>-C9Xi^7$v|Wkw_MaM6!qyU*7)sB`lI~xf(4A@Av|g6WI`82e=xQL0uLk zc|(mQZ(jU*ahvq|QL;DgEi&AD&d*+%)=^#s1kd~p1pTJdHf_^i|HoNWCWmp+E@%|L z@*ARY#dLH`G{7U%NFbR;5=k_FzMLM{8pb3`AfYTv&{-c`V+kz*=|q`gzWM}VT@ofA zJbq@W<&&M$jRJN9&c|6X2XhdHv+RP5lkoaZHN9+N{9!z6sjvc{7ps9S0ri^7_pO3w zp^86xf35IQyKPG>QRBK2OY+Dbup}D5l4t-+)WA%+)=(@-0>H8)K}X4d&4m+a2}mc( zY)dRM6q)K-YuP$#tTbh|H>XU$ z>6&)I;~vE~_FpU9FSk7jCfRj2C77r|Fo_1hBpLvd?C}GbRvDNGBmsa~lAyCbldcX- zf5AsGWlKOhfjtn2>ar*RlUaiROnaxn6gwh6&SV|sRbtZKV$#xha~_k%X~fxj6Tp3kWPrB0K7gV3>^g| zH987vw)-HKKi^WYqn4iwwdwhBno61N0!x85IhPtuBaU?X5RE6DK12gtGL7VtX(X3S zBe~RNP9K&)a#@z3qrvF|Edl8ScYiJ+tV_e>B9@3?5vyx<`aH$uPuaSxrN}A^S-olY z(YonbUcv39Rt03DZN!mgAENQ3*@tM5%nD{7rU5dE2FNTk`>+H+W|`SXgV_gK0@4W~ z6U;s&3>LC1H7sOx&1Rp!hs+&y`<&}dw@^nEVKKp4?dy^b|2CS?p;DymxjTlNjyP-X3ch=A4T(7o0qi| zS!vST^!wybe=NUX?^3r5$VB6aBkev!<4L;@(IA->>^@8bWD*UKS!VZP34qKpyN?FD z53~fN6GA4~eMlHinj|$&ntwIhef}OYchv85t~dQY`K0Ok?Sd^#Z55!2-VsL{eu&1G zh99CqG%FZ>mk&Yjt@ulO3Xb{Z`jvuB0G>Hb#EPr$Sumpf+nd3); z;|E#-(g`?zAgarvfXph6A6>KK=P5G(Tgy)^^;LpWSbmhLydo%_0w~GLcckTqXnbk; zAsU3Tg5`&407^0*PeNH{`C**^lx3D54VE8h2}mc(bS0Hw`JrKOlS$s_161yk%n8#*2YUo zCt-XO9*5&`V)UczbS)#-t;;vHo~iw-lFJHl$j;1$8^5^p9 zu1@}PS2Iu6AD+#}p^<){g@=vsewO`0d9e*+a6eqW*43EBqbMGw-N>Jbd}0|H9dS(RW_`V(3@Kp^=sTAkC^<^pbyI!#X;c^=`sXz4&G{O^k6k z%nHrUO~Z(nFDZ&$d`_d;;#0QB_!^~YG(X;FTh0sq#R+ULjE2$5q-G5_4**Dz-&eQY}OYoFa zPw@#Dn~8`2-?-s*2jg3J#?9IpN zS@i8gIJ&uiO|!#h=OO)QoGkt|CZp_F>V`4t-5)mE4ddHrklyBrjoZ;+5atKEVf5!o z8clv*{-jn3mZ8d7#VS*+qy46`Oh?=K`S?B#S23++uZt1B?WMh=gE;zT#ETx1SuZ~` zYG3Q@u=MlRFL`r5w$`fQ?4iNP>d24Y-BJ6Xb(bf9UMbFH$c1dUlnqxK4XtcE>p^Kg zC=FvUx5sdX3dT%tmr)Uq5~mD;q@*EXJ0aSgvA5^>&-i9{T* zw?ra;fc=d~tZm6VA`v{jClbNa2O<$y&_^N>hrUB3;{2VUM2n*v5Q#XtpNK>p-7b-c zqx+di#L+z^5^;3D5Q)PtIEP)O(DQwy5_lk%)SKBN9>XJCyh$%liij z`Gfd@^dPR0k0s%qhu_?N?Z=LzNKb!6KXY;*iI=zqQ(==Ou zP3^5z*ZpWZ9Njd2X$-=&HyS66R|l;jnXkXpQBt;Q4YQ#w{#P)r^X0hIa*ffnv{H`8 zv2!_gE@CGxlQ$@F1yjc;0qlrGVn-wrJ0cNR=zF&WI}$GbdVeel?_jtx#Thx-j2C~M z0*tpr_$OB~Nn*s1Ka*j}nfTPLk;$8X+v{FGzoXg9ZeW(F_HHsFXW!E(?%!VD>88^( z_xf|CtqLsGQA!0CFWn6mEwAZl)8;6897PXN6c^ANl(+(-W0U}jL?Tfn5{V*_NEE#i z6iK*5u_U~M;fSJFf+EHPik?J~#E41zGZ~g#@t%7B92Dy)r2>jq-qy6(wzO-16|6>m z&Z3Vj3X0yK#1)7fqXbwa63HTwNET7z%iABnghdiASED829bbTQA{zqi09T_jsLO&R zZ>X{4&5K_zZj)X=O7_ORMTUFN`PnPeI?Ahn;F-UHpx<=brfvG`|2T`v5?(VeeV-gWenHvS{B90)=^`XiE}kDwR-;C^jxRlK2q`Rr`HO1 zbZke0i5l0FV4?=WBpLvdXaG#qAeg!gOp*Y=EJ@H=|M&=|Yzasw1WXy$C1LPjm8F*3 z9H$!v{ByyqqsA)FT2}*qTi;dMZQD@=Wb&QT*9tcvZC8>>HYt5cCTfsOq5(3A2FOGW zlG!dJlOzB#OA>U}KR%KvTLRLFGA&7_4C~S`m@-j>Q)c^ricIyawQLz%5m)o8MlkB>i5=_(}m_!3$5)FV!_V@uzs|-vCk^sOgNzhsUldcX- ze~*u3%9en10(&43)n!otCbI?snD$PCDRxADoXI-MtHh+e#iXSz$1d;;r~{D7pN|5g53~fN6XGZUuMY`BM*&HV zj)I!)K8WSdw-oHC<>x|edVZXyQf9lrQlL%FrAE_;Bb`1(<4LCv(EyiBBe`T6$tBZB zE_IpHhb53)mL=$DaQZ+?Ksv#le@h7K(lEJ*B_deF>YANCPjUHEwk~TavdThMZ<>9y zZhF?necv{js(?(ijX2WmLo}W=`w$J1S;6eXG(aZN0GVZGAC>^fEHnFPF#AADKsq60 zg4u_J!9td$hJ~!I+3fT8kh!C7pL4zG_Q@8rZO69@?6F!Gpoz{AN7{Xef5ww`AEH4t zE7*OQ2GArLK(oy5!x8|RWp5>F7IS+m{eN6~!N=4CBK zR+=<7{XY5AAImS;yVUIhGSN8VNV^Zwc+&1eG)QIzyARUCP|Hxe`d{gpTCF99rgR1>rKB;K54psyI>1bTLoyMcf^r~AENQ4 z;fH7t%?gGerU5jG2GA@s{ICRoW|`qfgW(5S0@8^x9f>9wen=RiS;g?9Yc~A+IGWE{ z&+e+@M{hfREK7TC5t?=Zn&=*Jq~nKZeChZh8bq^#IlTemfepn{}WtrtigXITW0@8^xT}dTaerOonWRf@f0F}Gs_#srL z!yHKUtgVt-YO8c_f4S-R(Pq>4bP+(&lXKMdy`BKwIdDdSv9gOlbTpN{^Vhc93BUpV_O`p&Cg4E@SDG_ukkq*--~Uh)rYSVsr5-c9(a7vGGgi7^g` zS)tjnX&CYHDMhi1&uKJUe99IXU!ydQCJ*0k!`>i_^KaSx`Y?*phcEddS-)`i=s&a2 z6PgYLmJ1U|lWrm(f6HpaFc3xe{R*SKE!#;S2-|_SWYs{SE-l>&X_AOq4v~JYFDE7dL2KY$a$(bZM1?C9l@bhbyF}y zn6~h0wE^n|xF~>>Bg~X&@4X=yJGKW&T@oBBh+AzX^`W#oe{3lGraQoSn%-dvo^t9b zJ^^Dh5z&C;Y(knJ+jIcVFf<@Rd55zk!)LDtE8T2Er(8It@J9o))=`|?2PY81$L9VUfNMcoYJV zKJKV|ES{e0B{iz@G{240Wt``AdQY+?S>|7-kyrg^pd5z$n>Ygc+}EF=`UA7DBgz5= z_}GDjHIsiPTLI6L+a^2#S(6wiK>-MpS|>UIdXtqWCIprX6G)T4Cl&!)lh!9b0k4x4 MC?f`8B>(^b05gbzyZ`_I diff --git a/summary_results.json b/summary_results.json index 9cebea959..2082f7010 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74115, + "Performance_Result": 1.73523, "Performance_Units": "Samples/s" }, { From 81224ad896081dd1702951909e86a08eed447746 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 22 Apr 2025 10:47:46 +0000 Subject: [PATCH 470/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10302 -> 10302 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 71d3c5f8f..283237ece 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 10:14:58,125 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 -[2025-04-22 10:14:58,125 submission_checker1.py:3266 INFO] --- -[2025-04-22 10:14:58,125 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 10:47:41,110 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 +[2025-04-22 10:47:41,110 submission_checker1.py:3266 INFO] --- +[2025-04-22 10:47:41,110 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 10:14:58,125 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 +[2025-04-22 10:47:41,110 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 -[2025-04-22 10:14:58,125 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 10:47:41,110 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 10:14:58,125 submission_checker1.py:3282 INFO] --- -[2025-04-22 10:14:58,125 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 -[2025-04-22 10:14:58,125 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 10:14:58,125 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 -[2025-04-22 10:14:58,125 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 10:14:58,125 submission_checker1.py:3303 INFO] --- -[2025-04-22 10:14:58,125 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 10:14:59,207 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 10:47:41,110 submission_checker1.py:3282 INFO] --- +[2025-04-22 10:47:41,110 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 +[2025-04-22 10:47:41,110 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 10:47:41,111 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 +[2025-04-22 10:47:41,111 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 10:47:41,111 submission_checker1.py:3303 INFO] --- +[2025-04-22 10:47:41,111 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 10:47:42,109 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421168 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 54ba9c111..e2bb11dae 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1097 +1098 diff --git a/summary.csv b/summary.csv index 9d6a42010..d2ae005bb 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421168,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 88affed40..e6f2be6e4 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421168, + "Result":0.419768, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5cf72ec0ae703707069a95dcf079e4164f6ecf63..5295c70470959f0b463eff2b803f155d331bc58c 100644 GIT binary patch delta 2402 zcmV-o37z)7P`*&GatH*$E=XW%F*$vDx)!t1;;IKy-`}K#@o!ibNt&Boc|DSArr5mnfEmcQ71L^h!{~ctFvUD3Ta4 zX@4fek}KX*@1KKW9i>!2@ygqp7TcC~t%B94&sp@5MM2RUl(+(sW0U}kL?T%v63HS; ze0lrhm#|2}&0!->qp7nxVOk~?>Rqv zWm-ph6%ai0HxTrjPTRCifBhe4QJEaZMZ2I;{K{{L#ud}iG0^~zOe2A08c8J4_;Pw& zYZ#L(frPRwL1%q%jU}`Mq!VR|`RWsdbxD|f@c5afmQQw0HwxGdI3H)le;mv~7|yZ_ zGETzlJJs~EiSdW=tfj&Vd|s>uwgl8`D&MyXo`ov@=>4_ANA0#Pu|$pQN-W7Ed%%)t z0863)EKvh9Y-_%sOhUGI6d3rdH3Nf194`6x>HDzWwxC;f{{&NH9_3dJ;_3Aecl0U=j_0i5dh` zmw`zV0GK5SI_ravV9J(&bV9(CVOEoe-wqz0^65lqQ;X}6rw>gi3Z3d8Xyxj`0&RrBa`APOlA!NFzuZNQ|yTRIFoghSBXh`i%Cmcj$PmxPz7Ecg)R_BnstcA zlWrZNK_)Bsb(jX2B#s?dGFfKXVVwYzWu_ekt{rFzNGHm)e1{K)W#lm*nM4#5vPbI-ZjcK2AyGrgi662AaT z^o=;u-$OK>H24q=Vp+lB!!&>;(EyfZE+3Wvuq^ZWe<(2eKubV6A&vs@`j9Ym6p+;D zD5%-)gINB2OTms>elFCe=f`O(Wwr|}1={3XYBY^F(&SOUpqS%QuRrw_CQq!Zk^gs?6RlZ#j)f<>&Z+3E8Xmp^6evX&yNEM)bj*+=W9 zXL$v;f0tSnkcqYtN1Ar329qZ71Ob{g+kJi%&1Y?1)>34p zNpsWhlRy2j{DQqp-7X*#jU$e<`w)#M?LI_$eNGFtt^HCVEF4Y4{-;UmAXh ze+JR4VEADgK$BX54BVE|+WzmMKIzYhh{e-@CtYW30XHju6dk6~LIFD0FX z@lALfj>n17kFwLXj9j-a-_&}h_OF&NHGPz4{YBHkD1QY@Z@hd3YdV_V7|EZ@m%BRo z%U#VpS$}voABRTzeHI=z!uwhFe+%WsHjKgjaQRwSV-}C1c$9u$_MX32c0TSMO`^fe z#h~WX(Vz41<%|9cXa7atdG(8-Um1r+R{Dc9t8URt{(%ka=wQ~n2|xAXo6$5e#^Eq4 zG&?s9BVN8JcJVolW{XeRBI9e6rqSf#+ilnzgmL~YyI&thQTp&DKPBrI1nwUF2eZ%< znhpexk7kvVZXzFl%WA_g5JmU>3ZuO(+ese?+kv)Z)j**xE!_!el89OlB(+ZdzKUN- zz^g7t_s*F+gNoHbHqkFwC$uV2mZT^G#WWXcSEBcN9Y3JRd8#;Vw1N^H!J$=kQ!qoA zw(x4T0qX_0D1ekB%#>*Fy&)Jowg*XF5*#XsTWuxvp|m@HY$*GtJHUCG-eC!za_T8Q z0b?@}(SYP^LYg1jbO6pUG$28FhqENZXRik<-E2drTsWohM+39gQJmZdCnjUtwn@9_ ztRa7fKQ}LL-Kw!rr&BRNRlyly7O2;@D)40y4%blUH{rxv0sQe8^qIF|k-iUj6atVw z?x=h$o}TL^HLCJ7zm3ynoac3xlI1;FW?!d~SN&(89ESXxI0E|I*Po#J1GBIr$^rz+ UqQe}sekMHu1dfkpm6N|Ge;3+o1^@s6 delta 2402 zcmV-o37z)7P`*&GatH*5Y1{y_b_iwzf6_`h9>>n**tv+ExJ=%l#1%{(qXe)c5{Vs= zNbHD2T%qsX66{F0`0M?#B)o&+$`rrI$!5Ix>l9$TCBi?sl1UOHhWwcfOU}fnZjDUd z++O$k`5nz(b_26awRe*dIs2YQasT%EPB)#Vx!0d7ZB<~gj#4VHct3h@!ZF-k`)45FMigP$UwGB9TZGiA194m7qw%C5k2C9Slbly%H2L9#HfoiX=u% z+Mmg=-C9Xi^7$v|Wkw_MaM6!qy zU*7)sB`lI~xf(4A@Av|g6WI`8e+Rf4l|fw=BzZ%PC2wB*dU2cd`cbks?kzIhd(O{Z znbuKW1q9Fh4FvtB(>86>U;oEhR3?XU(Jp8dzw#TRam93WOf;VHOVq$jxz-86J=VG zOc~atVK8N)2&c^U{}h?(S!>xkYOFM6wl}9tzv-HG!Q&pqH}+pE+%LC12`1TfHzk;; zK`@C1z$6*~lkD*Wm{u8>Bmsa~lAyCb_z0$K2}madOc~ZCVK`-y)Hr2Y{|8{!QDdbk z)7qXgJxjYqQ)Yn?e?_6Q!1g7XsPUu~g=mmWq5(3A2FOGWKK!xE$Rr7X%#sA1^}$Cn zWlKOhfjtn2>ar*RlUaiROnaxn6gwh6&SV|sRbtZKV$#xbh z-pQz(f6XSt`3g&cWO_GQ%$NB@nXU5lGoaj2AJ3WEH1epX+qO4+dfQBH8F|b{Cea1r zNGA``c+$#4G{|HHGY``MlSBhdmic*D0${St(W9i_<^);-(h2TfLRgoE0h7eu0~2M< z-kmKZ|B5Zle_9HxMCGQXN4J~WE;6+$uYgK4ggDaFLo}Xr^$-nGS;5!CG(aWM0F`Cd z9+m*8EHn2gX}vjtmVk6Zs00rW3B%shR=>&H!A*@TojGKmJrEHnGC1VCn)*++xf2U-Hs2_X~A zJ|qklvMe<$WOdDEpTCF99d-Mh>rJ;$wwP@@zFlCC)w%#pbdEUE?n5-5wEGYZqFKT2 z!!&><(Eyrdb|01i&@8k2tPeh-33eaS3GQ7&e^{4>!K6t%L4an>cAp%t z(%kg>Kz1l&QQTD4hZ*$;)@7<%ei|Y55@< zgtCI=hiL#xG9FJtS!VfRodA?&mLCn4A7}|kC(3jsm0fBAh_ zCjexb-$#Sr2U-Hsi84J2B=~)37ywzp@1u9l??Zv~p9SQuT79&;4W#SAW7yWlOGzhT zd=nmr<8fm2qwI7oBiF6VH?^Lr{j23mO&{f1f6;U>%3s0K8!unMnvSM7M)K$K<*rWt za#u4?)*qhD$DxsapM{5w@P3y4e?ob&4P$UWT)x)Tn8l+g9;M%xz31UW;9KVaX8Eh z&CX53h?g&lU3^ZX+2T{S$oLwiX*7BGb{qBvVVr-x5P%%90dCpqS=D?Mw7guj3_(oTrM@Mk^>$2M(>Ohk_Zx zw1qdT4OlP0MFFH7VWva}?+wA&u>(lzlHgE5+-WPR52f9IV?)_D-2=|k^bt$&lv7Xf z2^gD+h(;u56Vm+HrU!6_p#cfXJDepMzIZ)Y>Fzf4%9T?J-x;`TJ&Tk3=)`1f+cs%u zy*1>|@Ym+`yU1gws46%k%mVe=Rt3H;!r>a~{3e`uCxAb7LBI1hE;951k3s-4 z#62sYv*(v3dX1_)%@=VxkMq3FQj*QdV!1tyyy?FH Date: Tue, 22 Apr 2025 11:44:46 +0000 Subject: [PATCH 471/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10302 -> 10302 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 283237ece..66eeae1a7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 10:47:41,110 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 -[2025-04-22 10:47:41,110 submission_checker1.py:3266 INFO] --- -[2025-04-22 10:47:41,110 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 11:44:40,935 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 +[2025-04-22 11:44:40,935 submission_checker1.py:3266 INFO] --- +[2025-04-22 11:44:40,936 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 10:47:41,110 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 +[2025-04-22 11:44:40,936 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 -[2025-04-22 10:47:41,110 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 11:44:40,936 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 10:47:41,110 submission_checker1.py:3282 INFO] --- -[2025-04-22 10:47:41,110 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 -[2025-04-22 10:47:41,110 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 10:47:41,111 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 -[2025-04-22 10:47:41,111 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 10:47:41,111 submission_checker1.py:3303 INFO] --- -[2025-04-22 10:47:41,111 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 10:47:42,109 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 11:44:40,936 submission_checker1.py:3282 INFO] --- +[2025-04-22 11:44:40,936 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 +[2025-04-22 11:44:40,936 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 11:44:40,936 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 +[2025-04-22 11:44:40,936 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 11:44:40,936 submission_checker1.py:3303 INFO] --- +[2025-04-22 11:44:40,936 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 11:44:41,998 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.442878 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.436766 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.19947 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e2bb11dae..7d58aa7a8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1098 +1099 diff --git a/summary.csv b/summary.csv index d2ae005bb..6c163426f 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.442878,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.436766,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.19947,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e6f2be6e4..baed7eadb 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.442878, + "Result":0.436766, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5295c70470959f0b463eff2b803f155d331bc58c..d096fee38463204b97a5f003dcd4cb8d1650ed8a 100644 GIT binary patch delta 811 zcmV+`1JwM!P`*&GatH+3t3xrfb_kFOf3zjgbX=!k&qUh=Xrg<>k&Yjt@ulO3Xb{Z` zjvuB0G>Hb#EOY#@1b}9l<41$z2U-Hs2{?Wrs>`B)%qor_U9;ooDKh_C%TF!!Rf1Aj zew3-aqWRJ(fRem?M_PV}#+Q~KqCqGtSbmrWpd{n*B$Q>AAJz##S!VgsVEKWTe}HtN zOjl9~mLD1hH<{#(K0xIzIerM0=`aUUJ!`9^mf9-aTW9pe-g$w;c++~Cq_TYPS-MW-MV~J>zUfWTE5iuQJ(b| zO$Vd=6)e5+@)fMf|qXHS=Ws;n{o~8tM00c-RQrKk+%$}M`J&jx=QNrvK4puHuTh#tlZS7&VQ&z| z`M2zTeHcaQ!EM71?Q#FH6mG#642ZTsW&zH zXcy@tN_=2iu+C_x(EuLJo4blz%8i3JXDqlN!6y-|AW+Yl_p*z+wY=CkO~QCu6R zScs+De_+e_H`@R%^85iS2u!fR@E(}jiHMF!!6&5saoq?I976{RbnuvD1fRVDobopt zM&-h(WEd>mw4TJtz4LN1_I;oAv(Xy!6a2M&eIHj%rQV;418OS4DR;nvaZQCUi+H$> z`LK*9UP};9L)34)9Tz$DfkI} psE$MaO*{b(xvxJ#^9QrABgz5<*{ee_vwkK$0R%%~X3mqpCx1<6jDY|E delta 811 zcmV+`1JwM!P`*&GatH*>9p#+QyCqCqq( zIDVK0&?Fi_v&`|s5&)WIjvozHfk(M8#@ulU5Xb{Q@mLH}8D9Ly{31yk(hjju_mRWu@Sbm@-e;}PG z)0I?$<%fpBO(uDx4^X*Fjvqp0I?RDo&)O=frM61lf}G{Rgwq6PgYLj*n)QlWrm(f6HpaFc3xe{R*SKE!#;S2-|_S zWYs{SE-l>&X_AOq4v~JYFDE7 zdL2KY$a$(bZM1?C9l@bhbyF}yn6~h0wE^n|xF~>>Bg~X&@4X=yJGKW&T@oBBh+AzX z^`W#oe{3lGraQoSn%-dvo^t9bJ^^Dh5z&C;Y(knJ+jIcVFf<@Rd55zk!)LDtE8T2E zr(8It@J9o))=`|?2PY81$L9VUfNMcoYJVKJKV|ES{e0B{iz@G{240Wt``AmXhT?S!Q3Skyrg^ ppd5z$n>Ygc+}EF=`UA7DBgz5<%A&&@vwkK$0R)bZW|fn_Cx6p0hI0S_ diff --git a/summary_results.json b/summary_results.json index 852d84f5c..25d32b1fd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -788,7 +788,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.442878, + "Performance_Result": 0.436766, "Performance_Units": "Samples/s" }, { From 7d542177f009ff6ab599b70c20455297cc7e0ae0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 22 Apr 2025 11:55:09 +0000 Subject: [PATCH 472/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10302 -> 10299 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 66eeae1a7..da2e9fe65 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 11:44:40,935 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 -[2025-04-22 11:44:40,935 submission_checker1.py:3266 INFO] --- -[2025-04-22 11:44:40,936 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 11:55:04,586 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 +[2025-04-22 11:55:04,586 submission_checker1.py:3266 INFO] --- +[2025-04-22 11:55:04,586 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 11:44:40,936 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 +[2025-04-22 11:55:04,586 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 -[2025-04-22 11:44:40,936 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 11:55:04,586 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 11:44:40,936 submission_checker1.py:3282 INFO] --- -[2025-04-22 11:44:40,936 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 -[2025-04-22 11:44:40,936 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 11:44:40,936 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 -[2025-04-22 11:44:40,936 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 11:44:40,936 submission_checker1.py:3303 INFO] --- -[2025-04-22 11:44:40,936 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 11:44:41,998 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 11:55:04,586 submission_checker1.py:3282 INFO] --- +[2025-04-22 11:55:04,586 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 +[2025-04-22 11:55:04,586 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 11:55:04,586 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 +[2025-04-22 11:55:04,586 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 11:55:04,586 submission_checker1.py:3303 INFO] --- +[2025-04-22 11:55:04,586 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 11:55:05,548 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.436766 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.19947 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.73283 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -43,6 +43,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418085 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 24 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 7d58aa7a8..d878b321a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1099 +1100 diff --git a/summary.csv b/summary.csv index 6c163426f..5beebc664 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.436766,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.19947,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.73283,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -21,6 +21,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418085,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index baed7eadb..6d6d9bbae 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.19947, + "Result":7.73283, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -669,7 +669,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418085, + "Result":0.418257, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d096fee38463204b97a5f003dcd4cb8d1650ed8a..c195f031c405e05f0d2bf39ad0a69fbac0e21d36 100644 GIT binary patch delta 2964 zcmV;F3v2YgP`gmDatH;)A^74nlXnPPf5}4vt+jXe7W(K$y`qC(?C%vQ@&BAGRWEjPhUYfm)Z(7ML4ts+|&}7`wrs=dMz0tI>%&;FY=wL9)j^*_jaX36|yjJ_iziYSAIyzYNeKiWd zB^$4eyp*p|^hf^Y>#2XXYxh!;I1vtItmsC})o)6&mdzvRvN*jlTGvxf#F zt0O;qcSr4m)?J=>r8t)%7qa0}He78qw6gK62c`L-G>pOA9>W;FGS+-3(nye zoC67LdBLH?74;sYq%G=wOC+M+--txi`wk_($nyR{LcS0`kRHS}^06em^Kk58f;y26 z0d|nhz2f?jL0t@zQ_g3SSAH>^K8}2N^x@y9znujiyC*LftDz)Ye-7P@(%bphW8h1Y4Aq}R#(>tvMVYoFx$&Tg8mG!DbXDyx+*rdpqZ z<5hedCfOP$X{{DBd+W^C{n>1YV-Tjj(Ku`g5+jEE znG8$L#HVhJOy1mH_xkxSn!W4?lQj&IfBEyrFJX~{%hhN}c*hr@oXCa%JHXYb4C=BV z$s1}c`Qyc}7q>~TA0>O^-Xg=j=ltxIX&vQNK=91pK+tbGZPPaW^&ifnGC7Qkc0r@~ zl;04IE2g7kq5&S6Mgqw+l1QTQ<@C7LFeX_731wM=&iddQOK1s5C(0D_)h7t+f08hH zq2+uCav zV99U2zgBpw-L@r`sBvA1CAnn}SP~6jNi={ZYG9^ZYbcf^0bp5@prhpG!U?nlq!VSf zC6+R*OT*-hNklMZZWOSKSZ>Uhe>s=hY^rB~Sx1dkCeGEs)aw3o({r7I-;s)MKfPAC zqhmV~Ow_oZ1QRs~CeZ+xL<3-=2Eo*2V3Gs?W=Vq1`rsp&vLzs$5HMv}mxRHMRhC+A zbDVAz@XrOajvA}nYh4X&{j1V$+m0$Alkb$iR=5FayOK<@N$E>6QG;X>e+`gHG(aY5 zkj!=&nIr*_S(2c$KKMwcYzasw%CsbzGOSC(V9G=hPMPihDKgcw*0OcfSZT^^Z%&zh z(>3ja+dYbJ?7vpHUv7I6OtR~4N-$A_U=j_0Ni+Z^+2aQ=tuio40syllL1%sN5lq<< zkWL7gGOSC&aLOd9amuv*e-FT{qsB^8rnNm~dX{#Jrpy8(ib7|B?MpII<4G$D(IA;b z17s2nkck?+`D2%nNfH2=B?&s~gO6m&mVk5udms?iWl;bovjzc}_D+K-c0_)h$vVob z#H78&q@^v#F7OPf0L&S;5T1G{7X$0Fz~Y9+m)@EOYcI z>9;w7mVk7EyO$8wf2Co-B(e9vL|L6gH%@V^)L-kNi;xZnYD){04mGOJxW?{PM{?qoe(O)!$ZPw`N~{8KxNG~ zp1+>T9X0oyt8I6Wr95pH{5KBe7hs9L5l8xah{lr!AEH4lJ}X#!mHb#EVKKt1b}9l-DiF95lyiBkWO%a?-IhgGz=z9;t2vYYqtCRD4NgOysV|jN|WZM z-zUHOWBCPpm%3d*CK^W^Y4;%-PuhKm2Fa{o_hA|!lW2gG#PeP1kQ1Y+-7v08R9cIMVP#G`=){{16SI zS;6qbG=L`20GefnAC>^nEHnIQF#JGEKsr&TBhduI4+%pws~CQC&4!;JNAo%B*IiM9*SME8gz9X~|lOUDn3F$KuN~qNhr%KKdcjgvdr?M!SVww0qI1UuA~wyKQs(}WRfTP0F}Gs_#srL!yHKU ztgVt-YOD0ya?|gl&8F|^A|MO=KAJq1NBVt;#+QB{qCp@l_E)Da#yWB+T8}y_24#aYvZA$lQ6ys zkHhgeG5S$uSv6Q528T@5|ov=gQ8{*6bLP&Va(R?Z^b6JrttylyDT+Wb&4t=m=wq{rA5r8yRh%|jL4{7>(6YWOnITMD zc(dAo^#WWJK*|wjDs=GP5R4r=fTS)74i&_mwvzf#+C6_Zlzr1Z;5%mHQv!PcmoKpCcft%JzoZUw!W@Fp7NxSH+ zA%BLy*01mVsHKLh1BzFyy}e1oa=Yup`O>1;iov z;xv^Cng2_IsUIClfWkx0a}yRCq4nKlNBf< K245ur0002|pSe>2 delta 2983 zcmV;Y3t05KP`*&GatH<4t3xq1lXnPPf0>62wAS9;Tj+DQ)T?yx>-}VG+=X#6il&DR z)ifJMIPFJ+(e&o9ae47>&ubV-+M5n~<7gTlHok{R9#T1N+qzOP2% zw`Ak9k(cr{ivGyId>kA$%v{<6?YqS=PGe&b4tw))dKP{A5RPtc)9kd_c}PDRCyRfL z$tXLQx?xOu_lJ#k!}vBDq_=ruf8%yE7=-zWZW#S}l17u?mxt6U!7@}it5{{Kb+q49 zmg#6a|2@8s!&OXc+3RA2Z+mI)=pc^18S$crWY)`n8MUuzBMaA6skHaQ4t( zWOd|c@9wC5(7MYLuN3DpTt2C|DpPD#YVk@fh}WW0RQRIO zClwb@DlSnWf7G%ZTb0_h($_Zrrs5iYLnPvNKPD1!!rl^z0QNT`v9=}eh(z%8o=5~w zABaR;K_7`koca!th~Mu7e?Z;3?I`x}vndf%bM7g^pvNXQ?=52OcijeIN#?>roPn4nH% zLx3G*bFa95WKb7_;7!E*PqSzqUrQLo=?+kHMO@=UH7BuaCFo7r7;N8-e{aO zULCZCWWN4VM@iYL1~trvw)kI@(F-y$ulL82@D7G6Q~V+)oAKhWQ-JZ72>;|tCP|DK z@@FzEITN3{H8Oc~d)@2jcQkw14U;qsl7D&o&0!->qp7nxVOk~?>RqvWm-ph6%ai0HxTrjPTRCifBhe4QJEaZMZ2I; z{K{{L#ud}iG0^~zOe2A08c8J4_;Pw&YZ#L(frPRwL1%q%jU}`Mq!VR|`RWsdb$>~i zeDL_0rIt^2PB#kJ4LBcX#T?8*7|yZ_GETzlJJs~EiSdW=tfj&Vd|s>uwgl8`D&MyX zo`ov@=>4_ANA0#Pu|$pQN-W7Ed%%)t0863)EKvh9HDzWwxC z;f{{&NH9_3dJ;_3Aecl0U=j_0i5dh`mw`zV0GK5SI_ravV9J(&bV9(CVOg zi3Z3d8Xyxj`0&RrBa`APOlA!NFzuZNQ|yTRIFogh zSBXh`i%Cmcj$PmxPz7Ecg)R_BnstcAlWrZNK_)Bsb(jX2B#s?dGJjcS*1{K)W#lm*nM4#5vPbI-ZjcK2AyGrgi662AaT^o=;u-$OK>H24q=Vn12I;=?q6CD8zuWiB6< z0I)3c`6w{@KubV6A&vs@`j9Ym6p+;DD5%-)gINB2OTms>elFCe=f`O(Wwr|}1(SUc zP=9#^x0hNKkcqYtN1A34p zNpsWhlRy2j{DQqp-7X*#jU$e<`w)#M?LI_$eNGFtt^HCVEF4Y4{-;Uw;~Y zhz8NDVEADgK$BKz1l&QR;`O+zXlDvFJT7HPemzE!* zK`1L&ewYTJB;)ZUlx3D5)(JpaX8F-z`GJ;zbfQdGQVEtH8U{C+EohZ|jK!V?gh5?Wj{62c;{5}*&|5-rps?|rk+d#S=Jcey;yp(hj z#y8<{I36cPKgv$mGIHIzd{gV0+P_-9)bvrF^%qSCqx=;tz47uDtm$ZaVk6-_gQ$@2!HQq*)NnA+b{HZ; z7lWElM}N-4moNG+oc$Ml=hZKUeq|gQS?Lebthz-n`3E+vqk~!RCj8WkZ${I^7>C2G z(CpkajClE?*v02Gnk_zMi;S;Pnnsg{Z?|D@5XSkp?0$V1Md`zr{0Ee*U$}epAG6LA znhpg+VP?)HlWig%f6Z#cFc5_AeF~$yEz5CB5w-(u$*F-tU0QMz(k2nLevs5Ue)}p; z;v|q$kE@yec6L!U->EM71?Q#FH6mG#642ZTsW&zHXcy@tN_=2iu+C_x(EuLJo4blz z%8i3JXDqlN!6y-|AW+Yl_p*z+wY=CkO~QCu6RScs+De_+e_H`@R%^85iS2u!fR z@E(}jiHMF!!6&5saoq?I976{RbnuvD1fRVDoboptM&-h(WEd>mw4TJtz4LN1_I;oA zv(Xy!6a2M&eIHj%rQV;418OS4DR;nvaZQCUi+H$>`LK*9UP};9L)34)9Tz$DfkI}sE$MaO*{b(xvxJ#^9Qr7Bgz5= z*{ee_Hj{iNTLI9M*d{yyT9XwgK>-PqS0_3Fdy|qUCL2RxX3ix6002+|000pH00000 d00000004l2T$9r$J^`?k5-1}EVkH0o005A`lRp3e diff --git a/summary_results.json b/summary_results.json index 25d32b1fd..9d2a1092f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.19947, + "Performance_Result": 7.73283, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418085, + "Performance_Result": 0.418257, "Performance_Units": "Samples/s" }, { From 4cd11add7e8e3cefce1c5011050c906978648824 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 22 Apr 2025 11:57:00 +0000 Subject: [PATCH 473/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10299 -> 10300 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index da2e9fe65..fcd5a09e3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 11:55:04,586 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 -[2025-04-22 11:55:04,586 submission_checker1.py:3266 INFO] --- -[2025-04-22 11:55:04,586 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 11:56:55,894 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 +[2025-04-22 11:56:55,894 submission_checker1.py:3266 INFO] --- +[2025-04-22 11:56:55,895 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 11:55:04,586 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 +[2025-04-22 11:56:55,895 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 -[2025-04-22 11:55:04,586 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 11:56:55,895 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 11:55:04,586 submission_checker1.py:3282 INFO] --- -[2025-04-22 11:55:04,586 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 -[2025-04-22 11:55:04,586 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 11:55:04,586 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 -[2025-04-22 11:55:04,586 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 11:55:04,586 submission_checker1.py:3303 INFO] --- -[2025-04-22 11:55:04,586 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 11:55:05,548 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 11:56:55,895 submission_checker1.py:3282 INFO] --- +[2025-04-22 11:56:55,895 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 +[2025-04-22 11:56:55,895 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 11:56:55,895 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 +[2025-04-22 11:56:55,895 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 11:56:55,895 submission_checker1.py:3303 INFO] --- +[2025-04-22 11:56:55,895 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 11:56:56,851 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.410188 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402837 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d878b321a..4f1e6aa1b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1100 +1101 diff --git a/summary.csv b/summary.csv index 5beebc664..29af7974f 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.410188,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402837,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6d6d9bbae..a6c245628 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410188, + "Result":0.402837, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c195f031c405e05f0d2bf39ad0a69fbac0e21d36..d5b0952b42755cacf88bfed842dc3814f29fcc25 100644 GIT binary patch delta 735 zcmV<50wDdnP`psEatH)7F;W_{b_m@Gf0)`Uyff<*KuI3HBP~Bf<4emA(IAu+EI&*G zP?GU@63Q~m593$vKsr&TE2#v_4-JDKndFH+K;Okh^O2(e5^ot_QbaTN@80orLjCcpQ$$ ziP4X;)3uCTw=SR5dZzZTmJc<3lxO`#)4?cz1WRwcd<1Jcn%)@6pUa24I{Cw0%{*Cu zcs3u0M*4jg9yY@JS@sF##Wsw=fBkUzSXW~fkD_>#eqZ*UKUa1>?j22{!OO*<=F`!i z^YG=1{tIXSMc;Y#i=kf`helTVgEXsd(M$e<4eRJ&*1HKm_2Qe+G%?2EFe@}WHw`0R zJ}7qaI*n$FSJ@)tYm}zZ zELXBXNfe_1Bvp*dxfd@&OQC*;; zb2`J=(jy3BBkz!2JSZcG^H)=2P3SkN0jF`i!2%qAVZ;&K14BI%(S&4lM(Q8a3;;$k zG$24ai<2n97q0^&>|(*7TsZ}IorXp0S)AP`%V%TTwo$tptUiB&Kld+h!>A#b{i-OS zGG{cS28dIp%<=WbFRmfh?fr@m9N1&$^@X=-kg*O}-~(i1+_UnrdU~#Qs7&K@9me-z znpQ~|lC3xXeDy7F>MuYsHTgI33>b6YeuDB3v#le_0t7QLQW~>+COrWK#ApQcC6m1; RQ~|D&5-1}EU?l(m002+uS~36t delta 734 zcmV<40wMjpP`gmDatH*(A^766b_m@Ge<)LVMf0Uo03~_&j3F$ zKuN~qNhr%KKdcjgvdr?M!SVww0qI1UuA~wyKQs(}WRfTP0F}Gs_#srL!yHKUtgVt- zYOD0ya?|gl&8F|^A|MO=KAJq1NBVt;#+QB{qCp@l_E)Da#yWB+T8}y_24#aYvZA$lQ6yskHhge zG5S$N%RZsJ*oHBuSv6Q528T@5|ov=gQ8{*6bLP&Va(R?Z^b6Jr zttylyDT+Wb&4t=m=wq{rA5r8yRh%|jL4{7>(6YWOnITMDc(dAo^#WWJK*|wjDs=GP z5R4r=fTS)74i&_mwvzf#+C4UuebYVQJWU_41W!4C^%Qr&*vv#UA~~Cp=EpWYfHMpY zNKoG4EXnZM>%mHQv!PcmoKpCcft%JzoZUw!W@Fp7NxSH+A%BLy*01mVs3y8% zn=B(2MM2VH+l{>HKLh1BzFyy}e1oa=Yt|Q6<1jHfu;(^b0Fu3Qpa1{> diff --git a/summary_results.json b/summary_results.json index 9d2a1092f..7ca5a0297 100644 --- a/summary_results.json +++ b/summary_results.json @@ -824,7 +824,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.410188, + "Performance_Result": 0.402837, "Performance_Units": "Samples/s" }, { From 69e2ca5ccc27a4431187840430dcbb3438fe892d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 22 Apr 2025 12:52:34 +0000 Subject: [PATCH 474/719] Updated results summary [skip ci] --- README.md | 45 ++++++++++++++++--------------- dbversion | 2 +- summary.csv | 3 ++- summary.json | 46 ++++++++++++++++++++++++++------ summary.xlsx | Bin 10300 -> 10379 bytes summary_results.json | 62 ++++++++++++++++++++++++++++++++++--------- 6 files changed, 113 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index fcd5a09e3..8a99ef2bc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 11:56:55,894 submission_checker1.py:3259 INFO] Results=25, NoResults=0, Power Results=0 -[2025-04-22 11:56:55,894 submission_checker1.py:3266 INFO] --- -[2025-04-22 11:56:55,895 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 12:52:29,463 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 +[2025-04-22 12:52:29,463 submission_checker1.py:3266 INFO] --- +[2025-04-22 12:52:29,463 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 11:56:55,895 submission_checker1.py:3272 INFO] Open Results=25, Open Power Results=0 +[2025-04-22 12:52:29,463 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 -[2025-04-22 11:56:55,895 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 12:52:29,463 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 11:56:55,895 submission_checker1.py:3282 INFO] --- -[2025-04-22 11:56:55,895 submission_checker1.py:3284 INFO] Systems=18, Power Systems=0 -[2025-04-22 11:56:55,895 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 11:56:55,895 submission_checker1.py:3293 INFO] Open Systems=18, Open Power Systems=0 -[2025-04-22 11:56:55,895 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 11:56:55,895 submission_checker1.py:3303 INFO] --- -[2025-04-22 11:56:55,895 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 11:56:56,851 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 12:52:29,463 submission_checker1.py:3282 INFO] --- +[2025-04-22 12:52:29,463 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 +[2025-04-22 12:52:29,463 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 12:52:29,463 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 +[2025-04-22 12:52:29,463 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 12:52:29,463 submission_checker1.py:3303 INFO] --- +[2025-04-22 12:52:29,463 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 12:52:30,442 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.436766 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.73283 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,11 +38,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402837 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.5402 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402837 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 4f1e6aa1b..134390252 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1101 +1102 diff --git a/summary.csv b/summary.csv index 29af7974f..c7e8aef02 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.436766,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.73283,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,6 +16,7 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.5402,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402837,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a6c245628..2336d91d8 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436766, + "Result":0.440114, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -510,6 +510,36 @@ "weight_data_types":"fp32" }, "17":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":16.5402, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -719,7 +749,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index d5b0952b42755cacf88bfed842dc3814f29fcc25..0a55694432c5e3a03b5713cf924e063a54ef6aac 100644 GIT binary patch delta 3801 zcmZ{ndpy(s7sp5Ln#&APn#+cYNJ}c$TDjjMnaL&5)X1fBS#55OS>`h4R@7Xgyi))`BLt;On%evkADCBKL5TykMlmybDpn1-tTNkwxt0ALg$sWM{&Xcz*Q9G zfq*=I)NaH?s(E~)#HMgL!5&!EKcK>EA-?=boFL|QD_gaN+IXsJD)Gko-`gn|k9jOP z>Vua<7Upow=-G|?uBHYxFu972ZANhJXtcl9W999dVa>bhPgZHa{Q_!BT(9AVmhP&X z)>3dmV9O5=cd%*S8zAi43n=hz`j z|I?lIAK!G|kc}Swt&7&Zb(GzGQLPWts*(kE1c z7~V9<{95#I)w{#$BK+IGo>ya6qnB5w8^_))%(-u>Be#pD@*c#|3|2H+b{>wl=2`aO zacQC~-GN^d&(0mplV-lLo>*Kz-g!UsY4j@1R%^E~E_%4$zDE0}{`{Q!wQcSB7eOJ{ zfA|h9L~oG?D#z4JbCw_^%W17f#s0h0vV8Yq+77{6hFreFT6zdR?vX)qnNfN$Xj#jT z^4#atg`0{Oj(ajv zFmGLMqrF&2pxKhqkpq!sX9&Xfj#USg6Lm6^%oY7l!>%GGeawo9O|GWepL~`APzmqB zQ?&DsSK8^Z?KdT_b{{Qn2OmsvdUsiPGNk}4SzoY;isCIhF(yqSSYs&n&<5hVkD0z0@$0NoqCl3uI8i5(OhnB8(oT$F1x0{h>%NB2 z>kC;R;!v%eqBwTru}RaIo(iZe$UuU~u-1vr_7j1YP6CZh3Eu*n_|f7MFVRoL2$b&Y6T?p0^WD=lu{v`TT7vk+_Wl+b zX}znD$BQ2dvU2IZdcm~nP>o!E5WOVM1@22W9`5$9>TlDxWCZ)2e6s(84yyFgY45b$ z({1x}!D-)aXMdX>^F7cB7h!B2eU+|}pYfuRsiiMM!2l`nR8soWEBNWmvAZ>WnO*m3 zZQm5KEPFAhD(Uv4tF$(^;8(l(8;pu`hhV=l>!!mx2D9f>kDw|E!#&fxl8@)wuRBj) zk7JfvjNj8GKW#4y6hG%*UU!jrEbPpAK{sP4=$vBWkT|iaOTCdCMH(ODfcoq|4fJ2m zE@~n9aD7BghGo?x;Y~qGga?-hu$i8~HNwkuM)D4W244Lx%v3Wwec%EY;8$nojr_tw}QK&~qLeD2S+-XaXi8 zXx~$wnS6l7?%iw*t^GUT{fWoOhY$k-?*m zalf8igdoA-J(R9prDOmJp@@Wn+7!(m_)jb|HQdD+mKV@u^U0BV`6+}pPZ59hp%f!I z5VSO&@$KH+J{5?w)>KavF0Ig4P73P$`;pKeR~ zwcV+QWCYVkt6-@G{(5b&YHv|z5YW#Qj(-okU6t5gDRYwFP6oKbdt<=XBc(9#jAmenaLq)uRbyUmTehKG#->fn zeQ0`t|3g#dTL~64E-fNoD*~8Q_^GvtC#oZ9|Brt@s%Uwd8)CV%JRPH>8auC0o$&g( zIND2t$9 zAt;AjdW@#tK$mFiKR<%v{!CJ0CcDc?5h%*Be!ZBToF2Gq#go@LAI&IUw;%Sd9xQg3 z{<3mnt8eFm1SDf53rf)qO|`Bg;`!R3vhoykn8orlNNdIaX>}sk^fD{gdk2z$G>%JA zw!9tKyQ(%NWtqWyA1txMTcyE7#JzxK6d1vYd_7QELy1!{`PT@_Cno?IFqzM<;;hl1 zsA21qW^fMoqlsLi;uH7}{DBM&WH+8O%RJLQmu&b`g3oKciz86_`!^mmrKZG>a90?PbCideekgEx{2Wbp`T~=ZH9!hkA%e8wbC2O`stWFFT%ULrupz zs`T1lr9Sg9Tj>gG7RxP)k zMp%G(r=Ik5O>GP2tvX6lPi;vk4RI$%X-5yiGK~~v%2OoO^ey&TfPiZ5?aumYT$;$~ zl@ocwGdxb*y@EBu^7taghXm8>zrrOko#ZSabXSCAq|WO=3mE;L;{^1SuI{pI2s48x z==9EBfM-?w5(8dFHqJBPhA<`|#S;gi0t(blveUk9hEv1to-g3(6BT4k9Sh`ZT(0yQ z?;f|FqMpUqr_Bvye@Xdyr1tAVQF3FM{dYP4Tvjw%cXm?&FZ({p8%niMD!U;|%sU#$ zF69BWm-G1W5+GUx~0v= z>$Xf;FG==azLD~%$SH`e`p;sKqDIzb-W#-_rbIG?;{c~qjss-Ko&$WXYI`jjdZka? ztXv{mXtEQm1k&9$72Hb`JPX|wY8~EdSWH))#r5a2T5g@Gzvkw8^))w~d3Mr-)7WP3 z(gk>o7jKJbA7C#`)H67|Bex;FV=?Fb;y!h*Sq`_@0B7mm4w~$O44{EQ>@?{CZzUM% z-Rm60{uYE+Yk2SQtcY8SdG2pPlm8Y>N9?=7Dd@;8|F>RLf*=6`B2c^}8SyWN7grh- z0#R}XxB8YmDxDjyEDG2S<9X4vd;bN^g?%D znTmEx7YR`8Ow~I!?7)b3I?YzinZUk8^P=1~ZNK^G^)6n3*TeB_F(3%pxX3XYO78_E za{3z(7UO7R`$uV@U48a4_`_NF{+Ob#o3^N#&F6?;8j1m^PW?|K(B8GLmoR}p>Ka*^ z^}zTKOjTy6ZB?yQ;;Vt?_GFEY(Ya=9boMtSo|$XVF`m>fr#_bSL`3C{)AHctPP^6b z-=0d=_<@7Y9haU{aoL3J&`r%wvW-ho@fw7WbXwzfU+{ru#_Be0gZ)o8-FCz6(3)#E z|EbLVyBUv4T-RLA)Ru&HWsbwwE{zP9_d@cJ_f}k%MKO;2*uI}W!Jp&qL{kt4A4L}R zV8*{Yy$eoVVF$`u&SaNY?hdM2<}9(^)RvBjO0d`{^Fcn_WPMy)P}{#xhxm87=v@+N zTtYDwVXYB7002UY;wSlTzqr~Jsa;efn&dXWM%R%t;Kn~sH3?O4zEJC~{lnUv;zm-7 zqy_(^giw9Vm@$*!T;)QuAIAyB>kZTqyp8*HnOMcmit!-(#0aH4qptg|6wymBeSuZU zvJ0^7omn9+A%~u%#%1xjEOzJ-r}kZsk;E%~;&v@9DI3Q>#CwkM%lNnOH`>DPgAOAn z6Yu!%-qaj5{%2=hz5n`T{4D*Qll1z%Wz%cR(X12ZJAIh!7}gT19d$Q)L7m@S7h@qh zeocfTFC%v_>dOyifk(tVnsoXgc9pT$%k810y24vX)ro!RmopWlAi9oRIsGP~Jq(G;~HKC*_H%(y>J;ja$#S1a+z z$2ll@-5RGx{L&Tn9mL*m@rgX4&laWg%eo8j+&#u#r(hHz_)aT^rdZU1?!|A|$)-Gk zB4`R=4FQA38h%m3W;!()lmorT536|hIs8jxg?cqSy;ki!Z=cDISn2&PB#9)0dcFu0 zW(w7nq|WM@J#cd>9Y7cK9M*N}KMi}6e2~AS%hAs$!dlS;`^o1~m4ahz*$m32<0kWY z&f61KWn$)oc(?Y=)3*T>-3Epbw}hJJmwk5U#F<=2Hlr#Gisb+i9%!R*k;Br+aO*LE z)z&LuWWcn&*1WoN<>BVGx|IN}``1$M(Hmt~q&EZCmbHsA%to_g_q{Z4JQDOi!v7wz zn-u((x>XvhGG-zczg^%Oa!yTaJ>29v;Os}eqY(Ow8Q2==AE?R-yunfsDIL`%2orVP zX(j!Zu#qFMGlO{^S8Es;?eNo!l|FI*?-MgZ*@622nDSX!;ox7P0RS-I?-v$Mqe!V3 yaXC;NRnBs?QYb3Q2lu89;{pI|xB&q1zrO;)!V4+=D*9YI6bV%&z7p8p(SHEv$}y$@ delta 3726 zcmYjUc{CL6`yCUaOc-0)MrJIb3=Kt=sm3z4!p9yFvTq-T-wFNGW zO$=k>Kkg7;aWZUUpKhzr7*q#d_6Jf&NrXlR`m}b0F zht%cSUdl~iMNQBS|K4})=7Q+E>r#$$mlAQt2zAd=!Z^ImO;a$Da3waWCm|@*j;uTQ zCE$~g&d^3BUEM&iRrGNAs}1j;)X5kJcP9)xf#NWyZ0NFIVLsRjl4>qG*VpD2=ybLN zXP4cafN~c;Sgx)4KrN1@a(m7h3`&#L+IEdkQ zoxT|3Cnyx{JN7(zi1?ir>l@(z{C-Ja|IIz>RjUIp1MTsKVUXisqo!xIP-h)gthtXz zsIHJC)_W^}{Cxn2$UbOMD>7%tzwEZ^kTI<&JS0>Z=+Jh*(Zhw`+_Ubn6M%-%^h0${ z`fc^a38=$}c1M)b@CxG4;DAl*Jf?4V%(^Uw7|RMRgKCBb>8wV7w#^MLzM64fplZfM z`HmN&x{6+YvUoQikw}^pC7+X#v*Ez$=6F$9QtN4H`r`Cj2@BNhTM3(m+K>h>-F#F- zDc77D#>9_aH2V|XVU~5T%p}3W{dyDsvw}8(XBw~SgH8P6Pwl?MFO0n&#t@#h)Y32W zjcy2ubLZ_Vq{i|5!s{|uwsObUTny2HMV0KJ#Dn*cu-Q|4y0F>dpg2@mgp@ROc7+us z-oGX#&h1jC51UvwNx6Z8_3R`(4s3A{25o zuwlVlGt(ulFIXUrQt|?2FJU%krV@(!6HkD6C z-7GK|W^Lzc?*%gR0E4Yq(kNCbbt>cRq1;_skfzb7q0`jya`S=zoMI`|~{ z*#(VMTwgGKmDe8AB+;hXkcI|BV-B)7^!wXOyps&ROxo@2?aF62ubk{v9Qk6XJ--qv z*l|c(zBUz*y=aGdQ*HLBLsjq9huPvrQo(|fh}VR$R&wF@^G|1i4;j^w0f}H_sxIs) zO57L*jxVr?5@*kl#y<{%3mb3g-EI#?QgPC0XBFHqk{EKD>=)DZyy$x(#s47b-L!0e z8a}WhYNT|)YAaRiI#npW9+>Kw4oE7LRX zcrFXZl)@lP)cCDS@I3BNH7x6xuzfd*+bC=G&jrO1;^(Rd!@8P!3I^BWp6#WakM=c6p2OvB**7Y!YdTvXdr+HyY3Rc!EG=OPBCK3WPrB$ zcEeuzbypv8ekd>O)H5a2s5?Kz2PP1DfzYQnTpq15*v+A3Pf^9PP$!OA#tSy>I%cGz z!GJ6YcGczRWT$R?WB@Q!jBCW%7$8YxWy<_zgo(Zoh>FOesCJjN7d3Yj6<_SSw82pd ze2cvy5C`p3Ma`k>&QQg0dxu3miT3*QN=u9(?T++_F<~ z9eeRr`A`$iSmjT)Vaz5U9!S~puF90(j90W-SaLx&6l&yNu5p_;HmvhnE@*sbgV7e< zOhpGgx;jX?iO8+~$puohBK49HA7E4f&N)I956G1B-;!RcQ+dQ7AJv$_^|{#L1MJ<- z2b2{t$mKMp_13vCDbMuRCLScgF)x9h;6PUDO+_^^gres+i=irhH1`O zHL(%^nVw#CJ!>DM{NoT{WJu(jZN9tRchr#1eagA5x!od4J2_=uLCM8xUIGVK&MXMn z$G33n?1jd4;#eUC*X3j_!!lVbVS*p3dKbuG0Nz;x!lDg19*}lpfE@%Jg9WAuim)ir z!nu0#d@hU`_Y6Fl*gCjsSlk=Plrb@X#~qP$b#RFxH(-wo1M+1|oIWA-j$+chNrp|2pqbnd1D);e45g@;>H3 z-D$0q_U(?(`|zp`Pa1s{7+U_B#kIh{Jv9(gKrfn#b>-?i6$l|PA>vs-$r1M}nIrFK zO&w>`uyQ}e$afvw_8!3@X*kOcM0nKhA4q2pOf;1jf7HV*5~ihE_Z@dxLNngmA|(ZN zgxz+|3*MRtk7K2j+wV9Cis4)_k-dtIocaJK{CGHGEc~x0iljA$C4-E#50;OwZSJHr z6h`FvH!tl_qzp%W1Uwzw%K(O3*sFsZHxVs}11?a36={Qvh-6HAwsQ;w5BMZ^`M?!^ zfYe8-8WW&YW-=%HO+?w*hyccp0@2x2Ssu52aNw$+zf^C8A?3!`p8kZ|iGzx0qUQ9w zIH_XUe8QRrCtSgxe5=LZ4(xzF#ja%5uD^cx9w7h8&|=Cm6V`lxziYtM>8K8$XZ@p$ zxw!Hmb3(T1@-i$4&MXhPj6m z6b;FmkJmshTvCZ(d$t5WX`=2?R)n9BIjo)B+IJ-`g}!%XviZ70oqwk`xX#eLwa*0t zT9MYt2mtIHC%|fP$KeGa#W?~3iujwu?k!l#*PWLR04J8#Q50zTyZz@z zcFpbC=TwA3=L}pHaqS8SYNsI`NLNj|pI|XAKIf56pw7=sd5sA@_B*i{Xur*^C zq4$|{Qhkq;c$PPl>C|u`MkAaV0BGdHz2|G<;FmMw|6@aPO|W0tB1Z9R0x^T9X9Opb z-nsc(XEWP4o#XBB*X$*Fnm&!;DG^h;X?7||A)scrW^yt56i?4>*nY`jOQAKd@CDGn zwfpX2aFYk8D~ox9<;T}={lj*aD=kO4|?bnnEP@S9xTLI@%#aF5xxsmF0O{YI^G^)+!|L{ys`L`R63WV~*dppVYafygOt0^@Nk-B=P6FhYuy*1eO}i1&DO+ zIAoin(<-hUg>7H-`5})~P};ph=Q!&>9TD}DlXwX22LAsqAyv{G*w2HzAa!2&zbpd) zynz2s>=_Ia1N|h-DH!CG6bliFgCIH|z5CGu*?>!*$ku?q~t- Date: Tue, 22 Apr 2025 12:56:21 +0000 Subject: [PATCH 475/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10379 -> 10379 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8a99ef2bc..1f869c0d3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 12:52:29,463 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 -[2025-04-22 12:52:29,463 submission_checker1.py:3266 INFO] --- -[2025-04-22 12:52:29,463 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 12:56:16,312 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 +[2025-04-22 12:56:16,312 submission_checker1.py:3266 INFO] --- +[2025-04-22 12:56:16,313 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 12:52:29,463 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 +[2025-04-22 12:56:16,313 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 -[2025-04-22 12:52:29,463 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 12:56:16,313 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 12:52:29,463 submission_checker1.py:3282 INFO] --- -[2025-04-22 12:52:29,463 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 -[2025-04-22 12:52:29,463 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 12:52:29,463 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 -[2025-04-22 12:52:29,463 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 12:52:29,463 submission_checker1.py:3303 INFO] --- -[2025-04-22 12:52:29,463 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 12:52:30,442 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 12:56:16,313 submission_checker1.py:3282 INFO] --- +[2025-04-22 12:56:16,313 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 +[2025-04-22 12:56:16,313 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 12:56:16,313 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 +[2025-04-22 12:56:16,313 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 12:56:16,313 submission_checker1.py:3303 INFO] --- +[2025-04-22 12:56:16,313 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 12:56:17,367 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.5402 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.402837 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 134390252..21a801b48 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1102 +1103 diff --git a/summary.csv b/summary.csv index c7e8aef02..6f06c2f35 100644 --- a/summary.csv +++ b/summary.csv @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.5402,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.402837,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2336d91d8..1820a69b6 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.402837, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0a55694432c5e3a03b5713cf924e063a54ef6aac..d1b6db831fe10f5e7b47157e0a72e3982acea499 100644 GIT binary patch delta 732 zcmV<20wev4QHxQqatH)GMp2iub_gj7e{7|Spe(TbDDs*fY55@ zl9_1?NoASmhxGzfmU(^@cz&QIpu_Yfmf-neVR)5E-tPlfncHOfAy}q^oJjL!&q}t{ z@T1lZKgwwMo+?_w78rgMd0US({1A;V4L?MKL{>2TFbxpNtTaGmnd67`0z{TMe|{7= zexM~_!<6r*JcY0h4Fe)8IDXXDIesXS{)>p*RjSxDnf2dvEYKjBmrEa5PG^ zUX<;wMdYe=@iOHLbN71j-r*;C)?YN~5A(Mqbw`W0B&Wm4t(N??c>l7Kzkk`tll2A{ zvr(v}->2ce9zIO7Ba|1bYyF48f8x!{+B6sP&3&hD$8^X4~Az0vlytn^1|R^7ap`~w@-!QQlc8=iLK+u@vm6wg4g_H( z#kP~XA|QWi!!Qtp?|llRyDi&kLJ_tDZON&DLQ`6L6Vk>JwSJJ)I)3}gcH$)9Q;(~e z{dRUyu|CQs`UUHRRwc@k6h)wz=0epadf#p02NXF^6{n3>P@)bTT32@kGlXdiuT~qd zUVw`NNIAkxi4NWyg0W)exHW>diXHSj!<5#uJs=Ve~ULSYtwib#l!UbqWApO&$ChYU>x;d&j&S|4F8&i zuV3|EIlHfV&YRyf^+wy*veF-=S#|SX@(*lS2Yb`*ZFt&^Z-OV@vm6wg4g_>7 z{PvT(A|QWG!!Qtp?>t51d*Z~U2&^QP0QE#AB#J=XEPI<;#6PkPY2O|vY13BfiQ{@^ zznxvVTrNPEg{g|;DBim~TgP|Ro1p`*AlPO%V6*I<9k_%~|+lRUq}3IY=>Fx&%EI}yT(us>$-VP(GIm{;b+f@5^B4GY z_3}2Xno8ZDiUX=k!6|pZf^l_;&x?4tmie%XC*DX9kNc=!c{?mJ_JK%Z06NA!Dj&0_ z=k+8K)g&kLG`~#=Sr>%P2_+9-{m6^{6Hpz7{F`_LjJYpALH!4_{Ugc(1VSb@-LuOk OJplxCEByA83@Cq31#qGO diff --git a/summary_results.json b/summary_results.json index 30125c554..9de4d1f32 100644 --- a/summary_results.json +++ b/summary_results.json @@ -860,7 +860,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.402837, + "Performance_Result": 0.405224, "Performance_Units": "Samples/s" }, { From b320fc9aac11e878820dd0abcd3227b7c274efc1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 22 Apr 2025 14:35:19 +0000 Subject: [PATCH 476/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10379 -> 10378 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1f869c0d3..42c29028d 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,27 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 12:56:16,312 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 -[2025-04-22 12:56:16,312 submission_checker1.py:3266 INFO] --- -[2025-04-22 12:56:16,313 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 14:35:13,191 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 +[2025-04-22 14:35:13,191 submission_checker1.py:3266 INFO] --- +[2025-04-22 14:35:13,191 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 12:56:16,313 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 +[2025-04-22 14:35:13,191 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 -[2025-04-22 12:56:16,313 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 14:35:13,191 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 12:56:16,313 submission_checker1.py:3282 INFO] --- -[2025-04-22 12:56:16,313 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 -[2025-04-22 12:56:16,313 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 12:56:16,313 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 -[2025-04-22 12:56:16,313 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 12:56:16,313 submission_checker1.py:3303 INFO] --- -[2025-04-22 12:56:16,313 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 12:56:17,367 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 14:35:13,191 submission_checker1.py:3282 INFO] --- +[2025-04-22 14:35:13,191 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 +[2025-04-22 14:35:13,192 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 14:35:13,192 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 +[2025-04-22 14:35:13,192 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 14:35:13,192 submission_checker1.py:3303 INFO] --- +[2025-04-22 14:35:13,192 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 14:35:14,417 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.424658 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 21a801b48..0141c45d7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1103 +1104 diff --git a/summary.csv b/summary.csv index 6f06c2f35..0e093a048 100644 --- a/summary.csv +++ b/summary.csv @@ -1,5 +1,5 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.424658,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1820a69b6..d712b4a6c 100644 --- a/summary.json +++ b/summary.json @@ -9,7 +9,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.424658, + "Result":0.457477, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index d1b6db831fe10f5e7b47157e0a72e3982acea499..13923080af6e4006f66a8021a9cf704b085e96bc 100644 GIT binary patch delta 1619 zcmY+EdpOez7{@o+Vl20!HqMV}h%LiH9j8rn7IT}jHY=Ax)153HX!K5iRqjPMJWe}hXQZRF3d6>k(;uDl{`)@9=lOo$@Ar8xP|_$3v09MwT*&*=qIj(t zC}*f%W<0d3557B7u89ubX1a4*D#Cw0@jSG&?(Jkaf@`>`)xH$tG~3`;QB#j`AITlN zWW?0oVO{yo%ckwIxc&H@2-NOCx=3(fX456z17r*&XF`>*HFV4<@+71(1SAs{p4!;U ziq9NA&bVWnNE=;)OO@!8=1-k}B7g#A9Cs@_TGB=6#q|hUyYOBTO(U$acYMFM$y4oH zniWQICPW(loQ2@gw-t6ng=W7XcPyG2pACl9dCGntAl*iEZ;ZWii4D$Un#WYU;4|H# zr!5c_jl5xgG}pTfY}=#UiU9}EqZAHp@)cO{L~?+3C19A+vwKXJCp*;FTKs@h+>K=| zCOHgVC)~4pv-4z|26de&`JBXnw1ur!m;sc5F$KTw>%tq7E3;{}jvabr^P3oQKQ&Bo zM#zohr2txgXW?+lhL>Rq$|fXRzo+<7tKEi=^0((TO%o@bfV z<4PR7#0$@jDvY%JHgcFvKj(1# z)ShVuWC|T{5?shMi>Z+EnMuoTJw-ezKN{p)2FCZ85-{Krb-0xOW#K6(;DP$oEJDT@ z`L&4gK`~c0>5IqiSJ}UG8Hj`RueTq~76N7Rt1>=QpVJ2u0+nEVDzqOXQZ{HVTPB^6 zt_;?AuIk{#x&jVx=G~y zLk9N_?rrv&YyYDqq?r@1@G6=%@tqedv0iTmMzLKJ$78RdIatYq$-kyToxeo1v^_q& z`ryQDbkhyK-it{s+2aqvOW_R(UmKm%`4l&w2lk22LRD@^mfjP}(sWXQsO$3$x?(

B>y{J6wG-AilV1P$Lj2}u zgsOqK#SM%FPWp2^g22A7b6f%+J*-v-53+|%; zPR~IxPJjEX3Rd2NCFBGqe=j3O^;|V?3{~%n7#Ugbt@((QN(@8#i_No>TQ6 zvk0V$$J%xc7U`9S3h+5mS_o@7HYi__cDX9;1=X&)-@r8ZRXo}$Ia=oi710f;Lz{o9ixa`m#>#Tr0eEjPDJB-UeDSxvgXp~uG>=7H&e0G_O3?n& z`JE0#R@b_4N@%otVc;kaXyN&SCx0`|o?s=X^ik=Q+>ia8Y=}iS-ET zzVKqcGHd;!wZNmd(S}Xa3+ubG&)a*sinrflUL~HRF)xI1jxBnl7zIva#bGk}*(hTF zp-5RpI?L3DY_|IT!vO=nXncaprNsJP^Q==0ckleSbiYtZkioj&@TuiwQ-NkvE6e$IPp#De2C13yJZY_-SEY^kU2L?gZT_4#2r@Z)-{4N;_?`lV!|9gE5_AuQ#uPpt$#He*X)tTWHajq)E?rc+$?_BI>W}UMsa;bC zj1GeblWts8#AJaUPYnt;a@j#dQf1b!*TlF=<62P3;|*fmZK!X7^&`?+L`I5S_m#?6 z!1e|upRboEm|eLIlH@wBYdhyYnXPZ}=MUbV>^kb#A^0L8KJpF>(E3k=Sdy9hjX^+u z3Ze@$04Zg{Q)PbqC>TTe*id!f0QZ8YfdPY=ZdGZ@od{Hb8kU3`tE`;H7$Wv@0#I|aj!qzl%L z%JACiu^L^+)&REBy~wft>y=CDiN4-Hfvx+D-YXq1Xo3Y(+8}LH@X9k&VBa=PXNqn% z@|VcpU)PtJ^t<=}qFq!e-URbFfBjqx!kvfIqs?cX%Q7cNba*YWPv_zZjO=dPNFJS{ zuHM`?uvC-wYV{M=Kq_c{_3_I^4P|wxUloY0E{vc5qUj8AQ7wQd9kJzTD=y);7HOLs zPSAH$<87NacWy&fyVnSBk6jBqy}K@VPeja_5Bne5srZsXfBwHcnG^W&h{AgLN@iww zV?fWgWyEYAhtP8!ySJi5zvn68%vk!wI;`IThnlOLuuAktF_Bq_oq0T)pE%A9M{-24 z5WffP0`#c!`NqU`l0ky{*7??FoP#)4tROLHc=sLXG_UP}J#}g@;yFBVr=1qrJ)8&7 zTQ);(a#I=S=h0t<3du(>oVH zytN$R!#J$o%-vY~tYwGclpXq6*z{f9uV Date: Tue, 22 Apr 2025 14:44:52 +0000 Subject: [PATCH 477/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10378 -> 10381 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 42c29028d..52a5fc27c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 14:35:13,191 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 -[2025-04-22 14:35:13,191 submission_checker1.py:3266 INFO] --- -[2025-04-22 14:35:13,191 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 14:44:47,852 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 +[2025-04-22 14:44:47,852 submission_checker1.py:3266 INFO] --- +[2025-04-22 14:44:47,852 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 14:35:13,191 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 +[2025-04-22 14:44:47,852 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 -[2025-04-22 14:35:13,191 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 14:44:47,852 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 14:35:13,191 submission_checker1.py:3282 INFO] --- -[2025-04-22 14:35:13,191 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 -[2025-04-22 14:35:13,192 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 14:35:13,192 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 -[2025-04-22 14:35:13,192 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 14:35:13,192 submission_checker1.py:3303 INFO] --- -[2025-04-22 14:35:13,192 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 14:35:14,417 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 14:44:47,852 submission_checker1.py:3282 INFO] --- +[2025-04-22 14:44:47,853 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 +[2025-04-22 14:44:47,853 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 14:44:47,853 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 +[2025-04-22 14:44:47,853 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 14:44:47,853 submission_checker1.py:3303 INFO] --- +[2025-04-22 14:44:47,853 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 14:44:49,023 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,12 +38,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.5402 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 25 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 0141c45d7..cca9df410 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1104 +1105 diff --git a/summary.csv b/summary.csv index 0e093a048..cb6c56762 100644 --- a/summary.csv +++ b/summary.csv @@ -16,12 +16,12 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.5402,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7219,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9793,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d712b4a6c..99df372f8 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.5402, + "Result":16.7219, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2526, + "Result":22.9793, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -694,7 +694,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 13923080af6e4006f66a8021a9cf704b085e96bc..fa3b25bff25a2d1b7c648ff890a845b9db924498 100644 GIT binary patch delta 3267 zcmYk9dpr~R|HtPZnlVC)Cbpq4?JTz(+sD=lObn-hX^z3Gsw7{{xb!@h9#yX#l{1 zarb~d70fl+2zGJ6yvZ}`{kj60q6b5pTlHCs3m%-FD&M}iO-}1lLwFFM^pRTyTROiv zNwO5@*Zq|uiVkDqmv9vk~FB0gOT;$|L#wZ=iuMqz)`ukSU^nek@NC9 z#8OPh;~5^=&*<|YsB<(IlU!g3`Z*is0^<0zez~rToVlT}leXtOGBz0R{6%yrim%`0 z`U)!ZhE1Tzg>5Ow_^k9TRu0|4AtCVfcU7zUr&BBkn@6L<>ho@lMeh92vn?J4Zc%si zF}-gF(&3)yU01&rXN3I&Ru^gj-CCS0#z0dqa&#K?LKX7&3%=Yc2Q2McyiU~^t=h9PX z_G+4f^K0elcUSqeKc^E#wf7e|csCmd_$ z&p0&Auw!3dKQ!nb8*iz5wMcB$@=~mn%cafY$u)pJ!BILo0Kv^fc?G=3r@Y0PuB9I28*c6UEfH? z{WG*A8)H;=9jo58Y)p@4Z>8Z>!W)gZBLdX!=^~AiYjRRGk}DrftJ0$>0&3xgh-IkX zsj$%GM3?9+9B2|^tS9M%IeIrE`@Gv3nrdpKk$eHF@V?S>gggE{Yb)))2jA}E#?%$h{RJE9Sx78 z@C`rlx0aV8d#$JRGfFF1{A(}nKvZ<6ru16}^zi|;sY4F#4f8eyr|ajvHCfh2Cx>92 zmoJ{S`ecJ-9gM8r^r2v^>o|m*cSAR1T0(bfA177V-AD!A+@;z!)->oeyXy@o1KWT6 z{c-AZt;%Voa(Sg;r{o%8fifT%t6r`Fj!~uK7DvE~xSC%}Vb0*M&Awb&rM*_lTO&o{ zTX;jy#VaWMwSN-3Tgs+YD9Zya(@Uc^)hT1Wi*APcahFb4AEt9xJQX&UZ!Oz#Je5z^ zrxg`T=?oAosbY$yBc~K~ODD8Avsx6DKS2Zqr5-`;h(tl{Sgbk$298mu;}q(_3iujq zIqYrxM0C!T2=!)>*i0<_pJ@Enqk~o@cHQ+Z5=|zSMYPGP>XW*!b!HsJUJ|m{|Ivtx z$f?%yYvNHKfQfV*EDo#gfQ>=kayHw9>s+hxq#o$>!-ozlJn6syHGEC@#1M^wENzEkKQTR(-;fBHo6PUFb#niQk zI&(2?y-N7?{jP#LCpFp7NbX|nBgXgHuiha;{l7S&0~bhQ=ExKgC)sZXBg{2NG3T|Y zl_-RnclFIyxRy0yt9RYmat zgk0nE+RN_^{A&$dXNkviNe#3P6Hl*VRFIy8haS|z&~#9{<_VfF)Bh|c-Q=}KD>i8Q-?IpqODMsS0u7}l68fan&{-VG8t(OUj9!b z&)2&IG7IV0H9&C9!?UZ*JU4-ZWzw1a)X>{|J^$DQ3^Ux4Ufr%;utDPxy?`{Vh+9&8 zxElAkhP=Ei>R)KfK2)ql;0umqAn@^wd>+-@L-o{U7W!2up39~tJE1)lKNGBfD1C0g z%Ad+iBIk(r!XV}hGQg%^jTL;D)$u@t{IB4%eZjaBz5D42XtBP1m3t&|gWK;;=i;9r zZ0?5j1wu+tahaWpzk;m;9~kx3oIz&>F3SRw92)C5g24P%-m`~LTQYd_7M+0asl3%p*+&m>IF}x zf-U<)`UcOS54ek62XO((g{xtb?rF7ALFzU^?vh{f(PJ6rR^}!rfwq50>K#>#WW(ia zliFBMP1&VGZbnYmv=gmYKCbgoSz7GVRDGVXR(1bK0I8(zZb#&Bgl%$4^r${CgU zm17WFNNBWpS}}2z1R*93c^?$i>5SqD|HV8UX6?x_zQ?4=T>K)fR?PVzYdN`j5ANN` z%mgC#WD+cN#V&vFADz#=#JRnruCpVo(`zw_dkN|d>03C1j&>KT7TlcKah@t-&omCV zN7CHWWQ)%$e;W%YIVw&Lr#j;bUsk`Ai)%{!RP0ogU(+z>PLt<9;?!J15 z-kejJ=L@b#36B=9af7tNxj4PFvu3LV_e?0${F(zZ`_=n35nZ8qQC*Dez%de$Pbz9$ z%k_G_t;WQRu3xl@u`|U`y++rk&>f2#esU=uLMh~57QCC{LPIj*wov zDMm-g*TI8NAo2yyff3EZ2Nufo?VG@I#nEskhd*zb z=1H|+H=;Fu$6%-M-Jo8zv_qyCYmxowahTcOu63$xmXvHlAqgMr!qg#kKI?Fqu|f&ib( zqZQ5VqaIE?clcL#TZ@JCSM0ywmWK}*UC(s{y7R@Fh20M&#w^QG&k5pdPRWGkSfEaX zgr|zPz#wYuj&%+gkmk;WiZP*NNu1^b>$2@c{S5R=`|wKw+dW{~@HajoJrnwJX7g2q zp}bZdzy_UQ)VF>H-SA^u7MSDExXwY-1VhQ%Zw8=8hrO*;w!QxU`Q>ah}!0iDOSp^=-}nP=LsH*YCf`DbPy!Xsk2^-gx0i= zAh@7LlCq*T`7KgD#f$YFDlDV5$d%_o!b`ZReDn}pw_LEBZilCQBfUc$A@R2zhwR$r| z;qlp+fw&58RkU-J-qnIo((+a0SJQ1+_huYohqRmde3`tOIUQ(J`zh(2q7N~6%~1cy zor5(8%GhvSx`WL(P zd#S^(O*=+|{0U@Nd`VMWwQ)D5pf_+=+i8m$--m`^b_11yw@4X#{e4vy3V8{b=>j+2 z<;DAZ0Jf4smW*i02N;I&Mjc9Zxp7}#p=oSG8@2+ex}`vn9{vwk>b|kTPsJpsBOben zZtU=_Jg=M5X27+ii~#D!h*fMSp|TOF-2Oy|0B+I(;15kzX!qW&2M%hLJ;t4Ngg?Xn zZeLlnEcGJb#lg3byhbA#gYRmd0WYiqeb3J+J5c}@-qWgmad+KC{dT>mkYdRl1aKVv z7+MB6r|c)1MSGSRL;eG0qQOh>lmO%^JIbm(ZNevy#|_?-3oa-ARgw`=ca@XF)9G5y zone=HN7!rS6bihgy3lg!huI^xzRd!x{Wt1>O99i6r<9$_+D;D8>?|KCTW8*xPl!qC(-6?)Ea)3gw>XXI;|2(>XLG!aMklZ6Wb0DlPs0CIr;TeN#G U8R}YQLVFC7mWFsae1GSE0I{Vd&j0`b delta 3272 zcmZ{n`6CqS!^Ms4yRl`vqcMXl5fdsIGR4>_%aAQgBTL2-S!P^h3>nL0Fk~5f$u^}3 z#mK&^a9wj_!gW)c$S&`6zdyX+_lNiS13u3==Q-zxC(|L_p(&h?zv3bH-wA>o94@3d zK381Y!iAl?UPyR5RPou)0X?R20gusrpmNYYrxU4FyhZ&P7XGTy%L6;hiqt~BB4MK? zZF&-p;tKwL4LS_kaF;wT{PVugV}E{?!7glm_4Go3-rB-_X6RH}^&(_mH*E=HFxu7k zV$@F#zwpxtZFF0m9!6n)4Wnx7uJ<{GBXG&xO)^~>u{4Y!QU=wUmpBcSd^a#I2bj-Z;#bO~&da z$aE;yZ^WajWnpUA9lkocH%l2#&FJURdZPSS+z#*4&Xt6cn7UoC*%^O&UhqrLAano| zC2YwuJ$*8{`FyJ2b=TIDDmjKlxLhkR#sKPqSPHRhAoO}6^@l=O;qKXQB$ug|A8ZcO z(uZ$Jxs0nf3`@%Axpg=SF6GrqN_Ew4q24aj(lxFRB)gG)D;$fQW6{jBQmfzG!Q%Z# zX%@XAapn-t*&W2q5KCh*!pAk`9KI4`ae`qok$_m=*Uw2~MkfKG+o3oU$jyxsAYt~E zB3L|b=Yd5p5eF9kfi)p$k5y-7 z0ly2fB-TuemHVNKU;0C`B;FB%`I=5li^VNn=ZVs>usweZ(umkU4E&UwYm?2vg+7+v#akR!vPgp3?d~O`m`fkwJh8`29S1iY#VbljG_g`m>7)(EXy$mBf*m($8wA-Jhq% zQesJaE)uAm8hjnMYa!}I{vSBa>~KiWpqim`STFQcO)8s%*1vumT) z{!ZS_L`U@|T+DJqGy0yexvakVBvUOmjcWKigW4;VqEOCd4*?RuITn%x1oiK{GK(H) z`mdYKB|CwVXMeNbf}}&PTK{p&&o%sM?AN0UY9Aadxg{GrGXtCbPtT~kX6_eX0ih%% ze>oXDR({BlfdHc*z(KHOJZACxq_&SZ-Np=!BU&xZcPT9+^ttMJ+Z6Il6@idOopF0# zB;zplo_utiLyO)&V|qq0W8{W>~py+{86dfx0&fyXnwRdm2r*lbGFXC&#VVkemWK?j3aU;3Cp?1(tAsu`=@jS@ z_*6!pbGGITKbO_xy(TNX$?f;S!55cT7ja%E;+|mpVw-JuRpnr*i6Uj!zW*+yuojbm zRJomPO$F!@cm@!{xphMWy<90Hxk6vw_o?doTbaZGbp2iNDcz$^WH;kVLobbwGf)~) z?#Z8bJVb({`~zPE_iLJv;98eptW%X=EMX2zS)k23B?{sM8Yo;wkVDX%aH%Tyu0BAM z{m_m2#^tBoIseE|o_y$|D4j}DNe`O}KB}02c|9wBTG3}o3J6M=+LCYZk@<7;?B4W| znFNS3UjiW+#pc^}5_kq7a!Mq0qV@MO(7?0*N9kfE(vevicm&EunP=vy*aSu3rnWSf zrEMtOcO@l`xLdU+5t%g*O06l3z%v1nGm$u#R_&3hqPNIFfy(~zB-0K(L{8jZvV^gH z-(RdGfQw;Ya3?9+sBw0dRmf@2QrM%^kf)cf%tL5zUiea#dvp2}sjSKfJUV+sfT38; z&N7bo>nM3H+Jf_PKw&bjpG#2m8CRuDj0L}gqBM7e3XS!q7*&(CdQa!A@-XODxcj9` z)jB_B89^l-GY;_mN8kp6k*s*VNl-vaPIZ&@`QY zxKbp6=C@F-UPP*Ozoqwtwfccw98omVt?hF~Lbt3mxpUW!VvN0TD-!;9xvM=nmdTX3 zGb2&aS#x3Tt?IF^OHw)nhM4(&Tv{KA2Shbh(Qu6~;3?Tl;SZ<#*e=aw4Cj3zm7LbQl!cDZEW%;o9A9PG?yKtNRRb^9xwEKq80=&@8-qlf65u>Kxa{y8WV7O3j*96VBed zzmpRqlTtyyHm7sg^v-+N;ha2HY`FC6Ze3NcI*tf>{&g247qETsGR=yXX!VR}IK?u9 zl|&wj35TP)S#U7vqzI`T^*haVXyi*=SS z9pIviuBBx2r-Fbyf+(__bW0HTVUJ(v6Q+5{1#CT;InTWv^XmZ6};L{+&<&H$Cg|+_%jNbk~53y;V-0%4k%- z9d@VxyI?T)kh+pYC+@$1e*Xfqkua100CoQt$XDpSd&7ToE@WyFHABE*3 zuPLcc#m~`lL(U!6M$xl$<+^IpotwB*L01?bl!f64rWIXuQYlAD2yT4>p{pS*hFaNW zQ*+p@;U==FvpUNMr|nMFo6!>o1lWy>B7!U{kHO#J~8HT)(2y_0M znnEV#iDfbk|vxSzWlHUBs}6~jtK`9 z7d-HofqI;EHTx-q3-x5AI{B9GUps3OH7M&NT|3iyh7~0)GUK>vu-JPMaL78jRrq2P zTx0yvfzQMiW4>Gw6U%@$I6SJ^)3uesl-!|lpC&^zy7qGnj%3P=ME49ZT*cvkVUzYn zX~)V#od5Onm{M^8$dJ@kjX2AtNN=R1Nmx~NfD@@s^#Y)u^hs6aKk3x1JKD diff --git a/summary_results.json b/summary_results.json index c1a60ed18..85bb49b24 100644 --- a/summary_results.json +++ b/summary_results.json @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.5402, + "Performance_Result": 16.7219, "Performance_Units": "Samples/s" }, { @@ -545,7 +545,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2526, + "Performance_Result": 22.9793, "Performance_Units": "Samples/s" }, { @@ -581,7 +581,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 1336cdbee4ccc266a03b71f8042e9c9f58e193eb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 22 Apr 2025 14:49:36 +0000 Subject: [PATCH 478/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10381 -> 10383 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 52a5fc27c..2d3265342 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 14:44:47,852 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 -[2025-04-22 14:44:47,852 submission_checker1.py:3266 INFO] --- -[2025-04-22 14:44:47,852 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 14:49:30,705 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 +[2025-04-22 14:49:30,705 submission_checker1.py:3266 INFO] --- +[2025-04-22 14:49:30,705 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 14:44:47,852 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 +[2025-04-22 14:49:30,705 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 -[2025-04-22 14:44:47,852 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 14:49:30,705 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 14:44:47,852 submission_checker1.py:3282 INFO] --- -[2025-04-22 14:44:47,853 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 -[2025-04-22 14:44:47,853 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 14:44:47,853 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 -[2025-04-22 14:44:47,853 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 14:44:47,853 submission_checker1.py:3303 INFO] --- -[2025-04-22 14:44:47,853 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 14:44:49,023 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 14:49:30,705 submission_checker1.py:3282 INFO] --- +[2025-04-22 14:49:30,705 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 +[2025-04-22 14:49:30,706 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 14:49:30,706 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 +[2025-04-22 14:49:30,706 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 14:49:30,706 submission_checker1.py:3303 INFO] --- +[2025-04-22 14:49:30,706 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 14:49:31,686 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,12 +26,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.11037 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24672 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.73283 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cca9df410..9437b2fe7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1105 +1106 diff --git a/summary.csv b/summary.csv index cb6c56762..d9667fc48 100644 --- a/summary.csv +++ b/summary.csv @@ -4,12 +4,12 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.11037,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24672,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.73283,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.00743,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 99df372f8..9b71aff1f 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.11037, + "Result":6.24672, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -309,7 +309,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.73283, + "Result":8.00743, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index fa3b25bff25a2d1b7c648ff890a845b9db924498..d7741b238931cd12732dcddbcc4fc5eb5a59830a 100644 GIT binary patch delta 3264 zcmYk9c{~&TAID8{)Lip*Eupo!)kui3#UvToH+O_A*Vhq}P#eCk(JUKtMy&|BFD>aPQny`q# zEe_Vz>~`$rUY{5FzJCCh_WbFR1$@^2##a;QcwD|QADK12v|y7ea~I)g+Q0OC@}v>< zw$q`s+|Dx3;pnc|JAz!WOPZHmq)A2TWG?GY%?Ktm_;YQh2HjJ;JrPL1H*Qv0y6ST- z{k=;KQV>ke=zwR<*6L<`O90mU6H=u5oZ*Yn%k)p)Qs(_e6m6*>ruSC+#O;i4JYoAT zcU4M_6I$QC!Fw!8?>|K}hLmY7?}kd?O$YV=3Q=@=_N`}+;ZJbV3h)$pB{SQqO|HUg zf9t-)GTQilO}nAdw36vIcdF4Nx%2R!_pOT#N{%OjWj24%N*?@k4#I}3a1Fl?D{n?8K&E1>bn_VVh3KhC&jB;1%geSgTTo+r$I!z^J= z56bjCV5HSwx%DD9wFcjvxn6!gm0z69Z*Ogk-&_*359``Rtp2%b6ywklqdaoEaj>`Z zeI+<~`tLvC+^&H@@%%DxH4FjbuL(;d)8A@Q$cOaB#5{* zL5@Rbl?9_A;Sa7t2lKujN#f1c2N9=OB1v(|-_kC(75Iq|xAx5~2=ulQe)ipoYY<`g z8bj#xVzw|*yM7K0>0WY&_-`GZTRGkx!ByQbWicIpGtr_SL}63s*;M)`L50+`srJQqZ*q&c@smg;azrUs@z6So7jw!(QYO}sb}@vgI5Y2D0EXM$zsuG@!O6Y7!ktj-v< zEA$QR!@(N{2+UoV(UXaw{jM8{pycFY`&Ms)$NHh@!v)4ohuNE?zB0^Y5F+t0 zqdZVp*T1~ch^QQX!GO=%7z)ybBn=A_+qzZXCPpPRHIhwR#w2{f)6+;hviJwihc#`2 zq4QVdOdd+CNlMbP40|TP=?#tbtj#sujK_=IZb@Eqnx_MH^hnd6Cb}BEwE10MXq`*3 zig-uRxl+(Pol^gvK675`*07Wj`PsKq@zb242_PGxfG7|u=`u7P4Z%V|M7m8RgfCTr z7FDArh26Mf(RL5jM^&ImsCnijxwKc!*Zo|0Vr#4@YSFN%nfH#C>+sU~{CuF;102?3 zLkw0@dqB3}Q~?ojY+?d?awLXvQ6fJxcGOrh1CcsA}OA~r*y|yi9 z?-z%Ef3og*qqBW6_OnJk{-y*Io~@_`Fw@H9Ol zZ(dIoXLV9&)Mp<+lH%3z&_6DnDr==jGr; zA%dY)k8lhfppN|;SE#PehF3xj!-7_bm=QKS93we+%4H=jC6_iPI``Mnz1O&3NcmDm zMDdc^ciQ)~qaJGyn1&)Xr+sYbL~zd|4?1@r6pW&JaG3o=kj*Go*J$@uquWuQHtK9e z<$3dYoJ~YuiwUxoFwanEF{%7PHelZGYpG5IE1P8$|1e?t)y6okY<&2-yB90xn=_UT zrFyq8_WkG=)rs66p4ek|T;~K|aXl9wE{Vz)NQk^Zl?-+6E@JkrBhqfJuYE`#w+<#N zF6vePRd+PTDzJ6Z6>*U`EA%1Rg8rT=AVa^XpnUIs8ut@4Y1veP;jLdJ;#sO}+N+t% zKgVUy*-!Mm2fDJd&Q_|L{xQXJaFLb%pi8cG1Y{b43q@x45K_0%=@StJ;;6b@&79+b z6LYSbyl`2c%9lOV*%St1-1ef*@HqeVvrXkCuxdTDD_IWu?R z3?aVq2@UeH2siM0PX2m(D9xsfVpSN9xkJ|OUpLgq-q;i3r`Ww+Z`DAt_bdIDdbSLW zp=u#guwRSe!uS zWc=DS5d%3}{AG5%tp0s{7}M$VV=v#VhddClZ~a#HSh8cd-2XFh`jtW7?+ku5w(EBW z9{qSe)+j62RJJ+8zuOCqbu(%LNp#c{x%1yL~TsX46E|q>*%z8oFQM#yz zLBdD>hIzSrmiB9I;T>C5^XR-YSt9JZ@n(yMvwDVGmt^nQ9xk z5F7p3Q7&XikT8DBm&%HDP8SG(bftbz@qRgDTznm$+_$1<^(s=eY)@frR@#$2cqeK96T&L9C{@k3K=W z3VzhfLx1y>cU1-Rl3TYl5yy<5p%;usSoY`=dxH2`>o@S=Qt@S3JjH$AhYi(WPS&_ryHi52n(lUe8VON_W%}ON&4P|9Na3R0<}M9QzYBXjm&5eXIZQ6zZ@T0OCdo8SL{H!)<|gf5YsS$x zR!#9&td~FNZwXd{|Hu*GQWhSfP#YmKgf8<|xKPwG!su(*G*p$h2%5t20}X>lqQ+Ii z#_ZdW75tu`q_s?#)*Zwzp>9o&9qVjQ;zaf%gA87RZkfbGEHHsj@4`Y$hgS9CvtGnA zFDWZvc?60JGb1HiUParyp_>Jh)DP3MWx1UQooP{wUIupB zU!DG{udOCIR&6$F*|IBtQ4E_V%sO{%ojvyCCpC$kVtnT8Q^U%5s| zl=da}HIWcGzjB3^=KRgC$M5&~`}26d-p|+L^Zw(LfQ!df`W+ENOg(jt5Cs728~^}W z0QEWi81?B9Ba+uiFw1Z&$jJ`%D&KhE^BQP|5~R@9p+jF@a%1+_`V7WxGdotA!vi^l z_pDmz%BAfI98!3Ty++762>({mcX$t<}$QIcU4f?hc(Nf~mK-1P15q4anr_7w&km3x~G zL^YhaB)NyLKy^QvV-tP#KaPNUCUQ~9g?gYL^I=XPrgz6DcTMfNn^Jpe2R`GIBk_)( z_*bGhI=#*>6vSRJa3K<5J5plaYlF)*qjxdd5ZLCM+I5|CDP|*W6H#GJ`8OxYd*8LJ z$|gv_ojo1Yz^mbOmaHPPK97>{{m-v8C?EMqFd#%`~qU3#^-6i4}3WvRyX4Vb^+0eKy6O=A3&u{b~qt z=8k&#mpZgk^Omxbgvzo!xcVz3+OuXtZmTR*uzCsjGQ=HaY7B|3yl zE!yHU3-vu47Mh&s6rF_uO+yT{guPKG?`33PaycI%pGwx3EJPGNF!k#+P`(QrT(DAB zkPF@3;_Spt)%$ORG(*g)=lV%2I!))((%YUkR#Xjyo!Y0*oC zZgFyY6xwsu`keU(OKtj5a^1Ff2+E?7iOYF2dQ+@Dbg$t_Qnk&^R3PcrzEw+ovqqb% z*03zF>-*pDXFfK_os+4Olo@kKuIJ?{1A@>>Rm$KPc?xEE9K4LF|G5(82>#sW!xEP{ z=m>eOFO7cDESF-jDS zR1;VVTaT`SzK;JAopUWxsf{l-6HWOi8oTrOsCl_fUz3witD#wO#B^=l8O@g(bM^wI zxGcuM%5jl7b*jFtY|=w8o`QkKp_S~=G1|8sjkjSMH%Rp!M|ym*p<^<68SUfJ!F%iJeTcg&zlVEr@{8TpW=$Pq}~}q zEsR5qNY2G`)@hryye}X07T!Ih!cZWymSZ1Nzs-O43LYK$$qXI7Ob{^9PT_M9|EfDq z+prgK+=yC>f*X6)-RgjOI$HT*Pk!P{LGw9+fj%*i)YI1>oiMnfBh7`F?nnBU8Lf z2&a@ohY;d4Ja5GeiJLhNwj5HVqxpznV0H|Igl5=~;iJkI zkmiV*YeHH5$(lSXtrTKsrI;uaEBU9s$IE>jk*;muJB)MA$1-Y-J+^@(m7*t5o zai<3kE9KH+eFkd|2*hTW-yJT-KZRS~3mXi8lq2FYd!&B`Sp+=PAFMx*%nVo+2PWCI zG%~p@`mViYj3Rc#uqN#q{-kfI_%HWq5#M>!lf^#}a60aY6QJ(2!0q5nA!{$YeSPZK z*TplBir)um*1aw8yj>siSVeUJJe>+Q8wwsAIgdQzD&P)c0g{W>!-QSa8lnP~ECXGI zKNTP+Gfd1)49@_q{t(tWDNSa;BpZ@C>CcQ96{9Zt4mZ>iE!N&|a!80QRmM3THm_>k z(6~RLyzyQ)d92!Iz%Az5TWH96xy7|p5GzP%v|w5pew_fpCyja?<>vH6v3dWZ9S^hc zU>b0z7qk8(s#wN+CvG;qeIMr4L(2rh55y8oH3hD|^P5=Ay~4b+r=+pRt1)0YjVT2U z1P?BqM@G8})PYDW0ImBHpDoQG+*UinHEr%ae;=rZKOi*%_AwkItA$U~J0e|q_8e(H zT%&GiXS3B&?_jn$z-m;$+}71c>)5M{a*G`9h#}$8g7q$t4j2ogm3F~+9p{>P!pThB=Z;>Bb4XF<_xlXdCONK#f>M0 zc+xj-A&O{p2%fEuT!`|BFwf!9>M}xU@c7udad3DJ*Et}vjrYh>rH*YYSfcEPO%E+= zp?+7bH_|?~tUAeNDKpCpwq*C`ZKHgVcJx-X^6x1047Lw6pqO^d2xY-{xKeiW90GMx zLV(owbxJY~&pGu>9MQ6I{vG~Q*Jb7(LEKeJ?Y_m{Ewb!O_}~}Xh^L}!Mf%a3zgXsn zEc$<0hnsZB5(e}jx)ON1xc%ixBBf1iqaJ<9v-@Sct4i1QlODS4l8^PD*vz#Dx^e{C zcwLVr#;i(QyvS{@EhSSS#}sioI6PIb9STuobZ;`jfHYT{f&fi{D2!2gXi>SFsFQ&# zwGF?*l|2BajeX_d(lZrG=eA!&>Pf0LQvD>GBD&Qx6>Jy_3kHY-2bPs4+a~gIYWPuyVkK2k3smguqKouH11^wcs(Lz= ztB~yy1ZkiacRHfVWU{Nf*|~4o+*cr9pk2j30up65T5OON>bu6_EKmzVeilvF9y>jH zwJX8CZdWz;%e@k!fCHX`n+^MYaN|6(xn@@!2<8^a-CmbtT?y)p-OB3`?f>iAD|G|- zv`~EbIl;t5fYg=n$*>Cq8zf0k#=ic*eTGxYg+MpbZ+eTJj9R}HEcN6<%y3*at2WxP zR_l6UC}H)w_GhDAXy0}me2=i7nYT(@&zud=Z}^b(M%o)6w4tYS;%@clsT;Z?wcmG; z<0H%{j~~Ff*01&lgv#KpYTfuKoRP&F?T|jFeWMeBAYvQtfF1f5qvKnJ-Op{CwWo}o zl7-hCOKb0%FwN6@IJ%AA8tmXLIj~Z1>H-j>3Nj{VCLd0?RXLpsUfgxh|i3nP3d-}9EJF>oU(!=JP@ZXVl%t7*}e z?Ru(#1GlOIu*YVq)duc00Y_9TpI|Q7!=9smx2-8&6)EvAIr&M~T%UKKe2uV$fIvKmwB2t$cotP<7i z59q;Zv2>` zW4Q$F`i(T~R7lff%Vek0chiFfA2)cTOxZ#V9%v`8@c%y|)FJpa9tc%K#fT@5>Y-xF nV@oYnG34o_&a1$GH7l+v%~MKMQ8ng0P^P-8DhXD>0NnE*%QiEX diff --git a/summary_results.json b/summary_results.json index 85bb49b24..9c71555d3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.73283, + "Performance_Result": 8.00743, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.11037, + "Performance_Result": 6.24672, "Performance_Units": "Samples/s" }, { From e21dfec764232be6bace11d45b1b4ffc6a70d2a7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 22 Apr 2025 16:53:26 +0000 Subject: [PATCH 479/719] Updated results summary [skip ci] --- README.md | 75 +++---- dbversion | 2 +- summary.csv | 45 ++-- summary.json | 488 +++++++++++++++++++++++-------------------- summary.xlsx | Bin 10383 -> 10483 bytes summary_results.json | 62 ++++-- 6 files changed, 370 insertions(+), 302 deletions(-) diff --git a/README.md b/README.md index 2d3265342..a782e2149 100644 --- a/README.md +++ b/README.md @@ -1,49 +1,50 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 14:49:30,705 submission_checker1.py:3259 INFO] Results=26, NoResults=0, Power Results=0 -[2025-04-22 14:49:30,705 submission_checker1.py:3266 INFO] --- -[2025-04-22 14:49:30,705 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 16:53:21,231 submission_checker1.py:3259 INFO] Results=27, NoResults=0, Power Results=0 +[2025-04-22 16:53:21,231 submission_checker1.py:3266 INFO] --- +[2025-04-22 16:53:21,231 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 14:49:30,705 submission_checker1.py:3272 INFO] Open Results=26, Open Power Results=0 +[2025-04-22 16:53:21,231 submission_checker1.py:3272 INFO] Open Results=27, Open Power Results=0 -[2025-04-22 14:49:30,705 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 16:53:21,231 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 14:49:30,705 submission_checker1.py:3282 INFO] --- -[2025-04-22 14:49:30,705 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 -[2025-04-22 14:49:30,706 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 14:49:30,706 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 -[2025-04-22 14:49:30,706 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 14:49:30,706 submission_checker1.py:3303 INFO] --- -[2025-04-22 14:49:30,706 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 14:49:31,686 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 16:53:21,231 submission_checker1.py:3282 INFO] --- +[2025-04-22 16:53:21,231 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 +[2025-04-22 16:53:21,231 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 16:53:21,231 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 +[2025-04-22 16:53:21,231 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 16:53:21,231 submission_checker1.py:3303 INFO] --- +[2025-04-22 16:53:21,231 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK +[2025-04-22 16:53:22,322 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24672 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.385386 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 22 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24672 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 9437b2fe7..967ba3225 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1106 +1107 diff --git a/summary.csv b/summary.csv index d9667fc48..941da8916 100644 --- a/summary.csv +++ b/summary.csv @@ -1,27 +1,28 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24672,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.00743,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7219,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.00743,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9793,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.385386,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7219,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9793,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24672,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9b71aff1f..292f12b9d 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.353561, + "Result":0.436959, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -94,21 +94,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73523, - "Accuracy":"acc: 80.000", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":0.419768, - "Accuracy":"mAP: 49.593", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.24672, + "Result":8.00743, "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -183,60 +183,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.405224, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "8":{ @@ -245,20 +245,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":0.418257, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -274,21 +274,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.440114, - "Accuracy":"mAP: 76.951", + "Result":22.9793, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -304,22 +304,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":8.00743, - "Accuracy":"F1: 80.0", + "Result":0.440114, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.384271, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -393,13 +393,13 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", + "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":10.1918, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -407,8 +407,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.8483, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -454,22 +454,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -515,20 +515,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":16.7219, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -544,22 +544,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":19.7056, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -574,22 +574,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":0.385386, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -604,22 +604,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":16.7219, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -664,20 +664,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.9793, - "Accuracy":"acc: 76.000", + "Result":0.419768, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -693,51 +693,51 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.418257, - "Accuracy":"mAP: 76.951", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", + "SystemType":"datacenter", "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -754,22 +754,52 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":1.73523, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "26":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", + "Scenario":"Offline", + "Result":6.24672, + "Accuracy":"F1: 80.0", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index d7741b238931cd12732dcddbcc4fc5eb5a59830a..4faffe9fd13f3f657c128e5170330fbf473b428d 100644 GIT binary patch delta 4619 zcmZvgc|6qbyT%7Y*}@q6GBpe%W64rVmNL~a6p8HnUdR?>nU4^Lti#BJ7)uM;vyWse z6=Ce_7fE9a*|K%?J?Hg1=a2LJ`@Ej(dEd`<-S;0)lF=`ta^I7uf`w~8POyVOOtv79 zAc&YJ#Rn+K2YKF~&$9Fz`^f)z`fHh?5f1aZ^R()Z@0A+qYQ%Q4&O~qHg|2JbBLg6Z zh!=HeUcMpVeE5bWhefO?h6CmN3lO!70>m9i9zokW=PT|uS={> zKA!WkN-CO>)KH3bne$3LQfu^Vr?S`ThK5)y(U9cjDM=@9h2E06O^2 ze#?0Mxx(m`fo#~lxxllfGl6J$^NtRjM|xQ!<)j@TkvdfLjVa}1jwXcg51lt9R`y3M zru~&SFJbp6(TLJM${CB!n&gG^xYp`O&MxOb2x^V4Xn4=6E8J(7d0~yc=MlK3=^w+{ zxD;_pfy`pqElK!iZ!~=IFdSkFSe<~u5Z?r5rTZi6t|zL@tbA}x;vJ~Cf3lqEGgLL} zW9})IE}4b2Z|RCT8VR8f*68nC-({b;kv%sYKWLaGALA9%HFff$Yx5ObNKke$;@|K-A~qOV|%q| z{PU{Y!+w#c&abn_(>~H`CCOW>UaKaPub!z>>kBOqW+jv~2+F4ZBEO~0q*1+Xod_~m z^ct5DRG?Qqt!{Qh3(cB~k%U}>rywB&bRR7j(~5}ud$~kh4$C#*$qA?i0Z!l{Zp#Rf ztekUjf5f51$U<=3@Kk5@$bFewlXBs$i}9M3NMvvfLwY7%6n{0f!vJN=JIsr}9(mk#u6L-O(4m@`R}GQeQcYoQK{B zC|X=gaCYe%Y$+L}{b*RY#2lE;4+w1yweG0bNOYkST=kTUBApFb@X%9%U|4d_D_*6-_%?9C-Jt>;kSfWX|6yURW zESXXx+lNk&wyomfr0A-IOZ9=sFR!RPyQI>!a~G|IvCf0focKUN}0AxC=OQx=uCU%i5_$&PX6C;e`hYCwz9H_r$I zw=BY9Sw>Oz+xEhCL?N$mgY20Y7tr~D92oYJZ(<(eG&TGgh;j+V*6TDSu3hEe_f^@A z&HMe1aeZO*bN0`Zxs}e1aG!BUJ{@+U|$tcc-k=1ynooy8`t z5&?H$5aDLo|IF_=N+Ly@<I_!MWr5^WPc%nptIaP<^SLxsJ5V?sSsv zU4>j%B*T|k>q1i8@w#C8(5VU#OKtYG+4mRzS(B8a?!%y2UW zP+an37J5O;H+?fnK1Q=22T@`oS zq%SWvL3K9@e0T{s&J97`S_pjp& zv;lQeI@HXCvv@;1cNs_y`Xq1GcD6ViyP2%kzjy3b^$|CxuX*cUn@S6!uMUFE-ikqh zPyszK<{p?ED@Gdl(H(gldnRvjEa%i^lZsy_Dh_78H|go29D9s$b4kW`Si+_0!`ck%XWhe}E@^AE01p?Jufv|udx}w zW$Q(=6C@I}&~w;h%F|@T6D@rvHwb6$E9IkdaQLxke#;J%90MI3ro}yEf80J0KW=?8 zz_*WY7U&ib0?x`*4iscwIC|;Qkh8@es`W|!H1DBA;^zMcxcx8Sa1!J2{{Yq<1KuQm z5dsrkI!d2E*ZEEw!^_{9+9n#1OQ%sII_QhcZmHS0f}b&Nq4fPT!sx1~pE2$r0}8;P z5>NvpfqDn(@z6SaF5+}{?$U|lnaa_Jn%gttYs5Fhx-&{#YX`UU+qX64l;4f^OOcst zVAmf0OsP4}df`fp8+SLP<~Zvr1eU)tW{ld#u6ssydyh!r{)|Pi*u`?>>8dC68z%Sn zaEV|33~m|gh85;QOJOwA*TBh7jKfLr{awuPUvIZ`R8Bgx4j+4)_GOF7TPdIvHpFiD z%I^^Q;7}Xe5-n3(c==KJDbC17{YnPz31=@NJ;pARLka#SgDN{}`Z?*7B|go3N5iB# zLazJ(S@|rf`rz1LxnqOTr1AqrPHywgjeLL7u`2hgssm&>aPY5e;3JK{vP^KS8Gc~M z9FI&nijSQztxejvK=9WH;vWCFvaWH;wRXl{O}OK_g5IrRCEv-0iQR*VF{;ff&ynP< z{dlQE`;+-=B2EOGWM8yp?pJjH6BU>RcHI=v0%E#N)o8KT@KTao!W^p*0OOCc^7W!H&=F=9v z>V-l_3s{50Q#)`E$J!c3b&{EJg04%nsg>^f?_1n+D(vp5t`EjSz(e~=2aYX*HyrMD z-VE8v5|EC59}$9)q5Z-#D%le@lnSj5EK4~})3kLSa<$OfPZ!YzWfH2y**4D3z0mj! zL7DLXv3&0qZ|ATb&BV`jA~Yy7CYggnQ&dp3oLPXusrt#U^Y~}PJbp?2C8XXg$Jm;| zk(iPTM%tpc8k5_mC>Dr>_*|i#+XL@e+<=$=C}DFvs;By%KpVzEz;ej5Z-SsHY^xbu=)fo%H~g#;PUxFCv@C%t6zWzP##fP1 zY(YpZt&WPboco-x@X~!>$@6YSpXZd+S}V;aPo?R;r-)5NzkM#p}e;X^C$QiI20I&ws*dFEWlrvv_(tCB0R!To|skmEHwkGz&C{ z;0b2@$XaPN8EEy6K_2|i5-HL;%B%F9K`H;|QKx3BEbAV5sVy(#m9f^~=RrLR!!@h8 zf?*Zs#k4sU)Pk!m3Hk_)E-dGRan1ti)4VEbsC%}TGBC6KS~MN6TCw-2i0 z)qB1$PT4(S(w;@6?54wGo08smzTk?h3Z_30Y*ch!267Yf*oM-Jg$5I@Dx_>)t&4sr z;R%dUNK)_l%~`Ik4`}^Dji*trbjV=3jng-*Ijq07THyI3*oe(}$V9IFFr5GywcEnq z?(d}ITniRS_E?!;rj^srtGRRSnB{~}u=3TeGBRcU;6ja2 z7>X1%agd%#K^%Rp9nlRkB9!(AsNWPn!js`}8UD@;C@ z@ya0yt80_Xr=P?m&}a5XuLvf|ZT#@jaEPJX%^4X)sq4+QWDIlv_LlY_1#na~Y6u5> zsTA`(c$>1T8X{njPLSvnis~;@2Mj*YlsEZ$It&=e(-V2kbSz_FDt72-qP({TuwNOC zCr1FDtoTWG67{l{&P!ahWJnYleA_H^l+v)idH-R=UbEMJf76Ci8E^K;RL?Dj#8tJ# zM&&Zz$!+~}y+F2=oPyL{MxO#1SO0U_asv5FIE{AohWVhJpB6>)mN^Tt)CJmlRi;_# zD$n@V{aJF&e3!(0y&7rw+@u!t6%UQ-jlB{ke6Xa^82RqKAnXWSWg%>T5)3Z(+#AQ)%Sw;j-Zau`l!_1f&52YCX~4FlA{*Sa=pIJlv{Umlr_>OH*96qofPj%F(RvX@8HVlWV)9OcelW-3x{6CTfq zx!9`zcza4DqQWsGc>wV~FQSfSdK;!U4%q38Uho7hxHq&zF3%4bhcb4$z$pw}qfsha zH7g8R_HI`qHcq+fMl0BGnKq0@+tmB&%_On7yvbLq=enuiS-O;3`s2+RAX@8!g7@L; z3d0x1=d=U5gzWiW4%f)92k@L4zP)f^rd^%kU|n!Cv(`Sv54MH`pI)fT-!QtY;f)x6 z*+UEEd9mY@<6VU?hf02JzA53or=We2$ zQPObH{|sn;5a@Vn|7V3i0`U_)q_r8EiN(^F7!}2c3(|=Hes*poKO>I>5sB1ba3;DS LC0GQc{vP`;L}9c? delta 4582 zcmZ9Qc{G&&-^Rz1E$hfOgfxRfAzStsU)!jevPZVbmL)_PnL)Nm3}%chQBx$@cOiRX zY-5V-B55Lf_T`!S{?2)R&vXBKf6nW?uj_qXpL3sk1A#-71+sDYOpINPItc){-NZj+ zlf+Ek7&j5@oZc>lzgbVVudLW_VRB2V<-R#=b6_60jJ()V4SdfQ>5Oe;#K^Jw>KM`5V@U+c0|NM0J$I|eByctaq=`|S6_;XGNK-yySBH#JHN=9cK0Do$ zcQb#mh8?UTtb+F*A3bzXWcx<3oEQLdWKC4}R$w>Cy^E%Vri_3dk~U0*Ybb?Awe z8@<~!JlOl85=5ANbX}3fEr?U7pv*@JCKaFtNr%q*nU%zMj1wIAvCXMIgz?$=8p+6W zJw`FY3+WyL=>z9Wa;zNE fSevl{FA%{43{vm=^`Izh>U{jE+=WUmiXy%Ktr@csQ zep44a4aDwD;^Lv%W%r>{5#iUs!};HsQrL2If!G-ue@eXEkEbRbZ~XbOJBL@{q>fQ8 z`q9Z7QhXk@hTz$y96qc@!va*Qf7wGSVCTfb>dF2{=9}K>jtmZ>1qS)6KXC`eeSMhrxmV~$A9co4L?X_#Ce}2Q+}VriKfht&uQ^Gx@X(?l z*Leq|y=pSY>CPgPSH{s0N#)bVrv$J}3 zIH4}aC$$|sD9leYY11dd>mnk=Y{P*`P)1`@18r+vEA!bBi+hUqg4)@@J)OkasmZ=3 zZw)TD*XozjtRg!#&EB-krZs#Z&8v#t{vu|Cd->zc<5@;j0?>vxFdC?zVgh~)l|q1l zSdvYY6nnZXF}hYs46$hj?|7h3!ONx!D|zLnxO7+l#MEyuKDRYKCjf8UQp@io<~h7| zR{a?y7>-21Hw7Ujbw@ZjqdX9cq$3mPt4kysH%A+Bs&l=#d;#mfA4T9F0jhy`LAVM)C0S;}2f7fp=VlKQC=Q^w7X z$B7Tvdhc3(e^?yx>-mP)t={gXxUVV=Qgh(OdUerTYN|(yxz(1o41?tP7QtxuJdu*T z90)-#R!?QEyusJp05OM3g|F|3vq3(Oau%7zzfGzBb+L}Cn_b`b z58|Z#a>sMPc%L@%p+AOnMH$QT5e$;V`|L7hSc_yljY$5N3jIsf&*xlBR?Jp0_0Co) zK4wy0EGQp(<>h-dGbe9?WLnws)RjbS*@9#itP!C6s`1hkKqi#w&#YDCL1c)S{=cf( zE&ix>z|vu5bQmHUpMS_>xB*$jjB0upsd$Yo;`6C@)9f}m3tC#;Na`Iag5spLS8;Kk z$-NvG#Pb=Hu{hS8A)-jA38R5~MtT?XC7vqgS3+M(q6@eaqAue_P|p2DRLX|-Q`e34 zu8axm`?zyUIyHCdPsCaUwNJZgU%`Io?IOTQAMo4~q${#=4<0^cc@9m)teDG^eRPZX zy-MZG2i5Yp7MSfBhp=7`fM%;3^idTuKBid?FVQl>`=r}Pf##7&lwQt2`YtqMGV+a( zetn)=?#ZCZ1vj+`Vw3xuN#u~6Gtf~g>E0f1qD-+a^byQjabJ;3v71vpHoBsBwH?JE zOP1HmSNWE$Z>}GAE)m1RJM7}Z23b390oJ$&4Y1%bapTX4jB+mZ%e#oeuMKEEBAS8t z0}}9x7U!u=+dBj5sEAsL^8%Q=7ct0_8}*8WS}J-E+@khb?HM&;EkI6vFsD#iFNFEz zmrNafj2`jkPsTmhlJSVZxOcCYq98@e8tEM2;G}!U^bsd9dQc_@<-~9F>ZQZZ=I#_I zZ}XT-mdl$DT00*_5+m+KeV1UP2SD+-?342mLNyD+OU_+BdXv|2ep#Y=`6vL?6X;;o zl4sWf$ut(D?VbT$L!skY>WgM}T|hIbo)Ki_0UmRJ1JMqB$`<_`BKhojIiO8bp3}M< z%KX;W3R5}1aQOn}apiNO)Y~HD=XVP-H@i_po3d1^!U)(soW{_Gp-Rr?0WVjoUF$}> ziatGH8MkvJh=7msl?O%Kx+&~Li6Qng{PKEWPQeO!Re9tH$4*T5g!KS!Z1ccYp{w?! znyKYmfQ8DxtoMIdq`#~PIt!Ee7dA+T$(9|^ccOX4jUoDPjRfTN3f;*AIMS|^#}x%5 z>r9mq9lgS|Vvj>cx5x^PkI?P-vpRavIr9M?ie!d>wjPWL3FJz)`@ZYK9?`2z#xaur zWR%}P+7@j>ucg($yyMtj-`@uX76Z7?frr+ZRxJ*KV}$g7%yar<9_6ojf12L+*F4Yv zGH;%>O%u#XLDop>SN<997ej*#)8RE--*A9NpLu%Q!Bky(aFRzj6+y z7h7z}M>vr$VRUQsti>OpbfP3d07e%ISJq0L=(fCFA~AdtZufl;F|$Vx3pX|0Hfk|0 z=BuN0=m^imZ9hCM&N+iSP^vHeSDMe;dE?@nXaZ$b$Eqqyq3l3*{kymqhMpKFd%9?B zkuyCpk`<5VjnnMVwior08S}=G^Yl=w!FEe}Zm=y5DoR#F%^UkZn~ww1n&UtE2Jg%I z<4r~$v6lB$-)Fnb-Y*qekwm9@92((|6fths?^iu= z3)IY>@a}d>xUZzOi)nb8sBPzeS4lL+elHxfy!f?UWj|)!HMsa#(Y7(hF6%?Pta$#k zP}4$bWKQBc*13XXly1Q&a%V*T;bGx+&uIAPps9oWH|k^WiD^f&eT7-usTk*W%`aIY z{_Rx1hg#X-Fty=GkA(y_c*ht5A+ge9JLP;75zbRh9wG$!_ z$6v9dqEkCVL&nAN^y?$^K-A8bou11 z1dngFdL}q8{f=c4);o6Mc5^LdxtIh zR4j^Gu#79UYsrD<*R3a_5jNO>;LAN!6(RNtwCzwt0_c-j{PR-~HhZ|G{Tf(GKwDk{ zyUh0b4Ac_2uKcl5_LgYarJX_K{(0AnHq}6gqZhkC?T2*H0R+`v8RKhHPVt?RUhSdY zdZE(j=PP>a(U42>DHAKXS29lDohUlr><+hIzvUy>uH+#bK`*!PU*p8gMdu-s z!3@5Jk80E_y1cuGcl#zgnR6vs7fVbP4F&+0uY}udpC0j#FlX37O}Ab!s8yeq8A+J; zQFgk}U*evQN7cRwnZ$IxGtyp2zjJe2+x2iArac)_{+T8+4prx~ul%%s#-Lf@IzyH_ z*Xy?y<&Q$d41}*G%eA?hrjF}zb4uoiRG-}VxwV_*oA=^Y6j)ufePBG(j@)ezd{T^Yi+oxzR#BX{)C2;^ z`o1ytBSsHVS>zTy@rzR67PaI#=g&p*A8qg~go!Ej)4waxH+Ju6=H$FMsUf7!?uib= z$y3vrm%-^phj}h9eA~w=g;Y-*4+9Tsgyh3QLu26^oTgiBs_d`Ib}Ed5=V+r#%y#en z_J;_onh^eHU#9v2%RiFl>a=bsQ+x+bxoviSLxo|Lac@-a0m z?D|bF?OzHX2E1{DwlnQTh||VdM54dKFw=Q{w8TRO=q>9Ll} Date: Tue, 22 Apr 2025 17:01:06 +0000 Subject: [PATCH 480/719] Updated results summary [skip ci] --- README.md | 78 +++---- dbversion | 2 +- summary.csv | 44 ++-- summary.json | 496 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10483 -> 10483 bytes summary_results.json | 2 +- 6 files changed, 311 insertions(+), 311 deletions(-) diff --git a/README.md b/README.md index a782e2149..e00879780 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,50 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 16:53:21,231 submission_checker1.py:3259 INFO] Results=27, NoResults=0, Power Results=0 -[2025-04-22 16:53:21,231 submission_checker1.py:3266 INFO] --- -[2025-04-22 16:53:21,231 submission_checker1.py:3267 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-22 17:01:01,043 submission_checker1.py:3301 INFO] Results=27, NoResults=0, Power Results=0 +[2025-04-22 17:01:01,043 submission_checker1.py:3308 INFO] --- +[2025-04-22 17:01:01,044 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 16:53:21,231 submission_checker1.py:3272 INFO] Open Results=27, Open Power Results=0 +[2025-04-22 17:01:01,044 submission_checker1.py:3314 INFO] Open Results=27, Open Power Results=0 -[2025-04-22 16:53:21,231 submission_checker1.py:3277 INFO] Network Results=0, Network Power Results=0 +[2025-04-22 17:01:01,044 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 16:53:21,231 submission_checker1.py:3282 INFO] --- -[2025-04-22 16:53:21,231 submission_checker1.py:3284 INFO] Systems=19, Power Systems=0 -[2025-04-22 16:53:21,231 submission_checker1.py:3288 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 16:53:21,231 submission_checker1.py:3293 INFO] Open Systems=19, Open Power Systems=0 -[2025-04-22 16:53:21,231 submission_checker1.py:3298 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 16:53:21,231 submission_checker1.py:3303 INFO] --- -[2025-04-22 16:53:21,231 submission_checker1.py:3308 INFO] SUMMARY: submission looks OK -[2025-04-22 16:53:22,322 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-22 17:01:01,044 submission_checker1.py:3324 INFO] --- +[2025-04-22 17:01:01,044 submission_checker1.py:3326 INFO] Systems=19, Power Systems=0 +[2025-04-22 17:01:01,044 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-22 17:01:01,044 submission_checker1.py:3335 INFO] Open Systems=19, Open Power Systems=0 +[2025-04-22 17:01:01,044 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-22 17:01:01,044 submission_checker1.py:3345 INFO] --- +[2025-04-22 17:01:01,044 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-22 17:01:02,095 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.385386 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.24672 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.385386 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 967ba3225..9707e8768 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1107 +1108 diff --git a/summary.csv b/summary.csv index 941da8916..f01397806 100644 --- a/summary.csv +++ b/summary.csv @@ -1,28 +1,28 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.00743,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9793,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.08642,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.00743,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.385386,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7219,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.385386,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.24672,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9793,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 292f12b9d..47b4acc3f 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436959, + "Result":0.353561, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -94,21 +94,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, - "Accuracy":"acc: 76.000", + "Result":1.73523, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.419768, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.00743, + "Result":6.08642, "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -183,60 +183,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "8":{ @@ -245,20 +245,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.418257, - "Accuracy":"mAP: 76.951", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -274,21 +274,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.9793, - "Accuracy":"acc: 76.000", + "Result":0.440114, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -304,22 +304,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.440114, - "Accuracy":"mAP: 76.951", + "Result":8.00743, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.384271, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -393,13 +393,13 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", + "SystemType":"datacenter", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":23.2491, + "Result":10.1918, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -407,8 +407,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -454,22 +454,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -514,21 +514,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":16.7219, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -544,21 +544,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":0.385386, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -574,22 +574,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.385386, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -604,22 +604,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.7219, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -634,21 +634,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.7859, - "Accuracy":"acc: 76.000", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -664,21 +664,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.419768, - "Accuracy":"mAP: 49.593", + "Result":20.7859, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -693,60 +693,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":22.9793, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.418257, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "25":{ @@ -754,29 +754,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":1.73523, - "Accuracy":"acc: 80.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "26":{ @@ -784,22 +784,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":6.24672, - "Accuracy":"F1: 80.0", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 4faffe9fd13f3f657c128e5170330fbf473b428d..cf68b25bd05ca3dd9bea4bf2f61faf35e32916ed 100644 GIT binary patch delta 3366 zcmZ{n`#%%QH{}Mv39_>EmEsUY zlk|FN_Wk|Plj1ts@zQ9-P%B74Lo!;eFVzZqdv*C-`^&m}49OoK)qG1Caj}mc)E%Bp zHTfLg>IUD#OIjR$c7-?31-oG5rtio)73`KJ%_9nU57UDUKQi|gkkP+)e$8a6^8?f{ z8tDw#duXlMz`8{?R)gM+8w`=O7+2`jAny`)d%xD79;OuQI&fop)JoSP{KsT5vNYjF zn_RiiA&75$o1N?!fJL?^QCXTn3}%HI^4{ZH)W*Ij=iIo>2Ho{&<=mLck*HOB)?8}xNeVa6+9+h^j_to~V@j;a( z&1G)i>T7B?mu3+T-L<4thPXLFtYz@5<9hi&33l&cSR79G1+sHqj+r>8tq)6Foqo(pW7-AEO* zy78lO@w;R9X_e(XCz5-~^|YHmCObB7xO8r&pkg+YQ8=*WbTs&68`~JNTDE*5i@#7G zKv3yz9~*%vNLQbT{C+GRKc7C_@wt6bI9U)A%K8r^hSY){`LmHKH%}* z5gNjClbRN+7gMs%P@bxNi?xD)X~8J)Hr%7qUEQe2i=R2oZqYn)-$aVfL+j8eJ# z%z`_P53lZr$TGwV6K#ji_o=#Fjs_lVEsW!a<1R5W8+u?1M0_%2JdyychBL-R@vd&b=2!~yPfoq@L6W5CO>iBM^1Ns zy$9V0X|GX8X@V1Q88Tz{i;36}?+8;+`2p zgExbsSnC;HO3ew5Wgp&Q2nDg`9p$)tVXw#$wx8(3BAz#0UuRrbsg7J(c7V^7y$o#) z(X|fOOI4>Rs~6u>zSFgWO92>UnL<)gU>gX4Kygk43*;%2{X+_ zQq@4_ErB%Yps%-!O%%7~tsaKLWMd|iTr&EL)dh$Q{Ihb+mC*MUd2i6KKs}e1=E0pw z{=dGK`pU1^xo&-=X@1q-s1Ksg{}4+a9nniqi%2M8l0KbGSAgai;V7JE^Nj(0Z~&U3 z8Ga{2zv|ipGqT}rN756j{##xNrR+@J#Q+sVmPWsk*wA0n{k@}biGUTMQ zk4p1APVwn7jK?uwdNz4$NrI~4QSX4$fYDm`oaO{I<@=YdV3C-=gB$@i?-7NloXf3z zC<3pzBGBNJwnktTPCLii!hm0`Z$xvs0`Km#?Ry~uasz+YQQD#i#l7QQ6(0zJ>)!WD zQEB#ViXZ*WuTm?mir1^M&{!AStNykE&ZL#U;cfZ!IsgszbHNe`KuESvyak0*%O=U= zg_f2k=Q%3G$UAn6U@@~eekV2@t01?O6bu{+<@t$=?yD^6QOUezf5Q2bhf^R; zM{s2Mg5@V5Q-3TJLN33v;;%z7gcEp2AV32N?G&#@Va)di=}}nUh$ATGs%<^9R?41o zzOw-)tb{rB`RNSXTGFy|G#y;XizHvcA?+d+xRrqBO1Ri@O{3QY}}tYy0ew0IaEBYNR5 zs!0D27!#RmVhMtmFY`7S1^6zW1KKx4rtpqdE-bkL4YKt-IR>>Zhz)Z1kY?@+Z|xYQ zSw(Zg+sBMLLOytuoq43Q7QXir27fcL>K^bT=0nv)MB2?-Sz{23fC(o|kI{|bRkc6F zMQR+|*E`{D(2E~A9(&?JWX@^W&QF~5SXP*@nm(Q9Be(d3r%gO`XP<+t{W!YN&^A)2 zrKn#``=^Wa5N~eT$|K8!qK_d$<*3dSH4GB9WHu!>dN%iT4ASDn4Xw@ebI)Nr?CM)j zS#>bO-iPxAM_roHC{1Q@cQn9-Cu=O0(J+6V9|NsVcRt5-m%3`bD}t2`1mLRRO=VG> zxz68Mdb;0NK_&Fsmjx@a5@xSB%;34xW2|HNU*aNH9osp&;2er!)5-##`2s0>T-Ky_ z@xs-dhG|yG=hF?uCq3QxJ$WuXw4uuQB=xm+irg?y$gLhYOvv$z%CL_w9n&KTZb0h8 zIZvjmVhvCy9JJEAh`taU36;TaNPyZzV{UEw#?LF?Al6I z7659HdN^A=T`3)jN^-fW!PXE7LBrcT^S6%_)oC%vwInN-#LeXJ#1;_$T8)N7H{fLD z?J%AI#XMve)7Cu8?q-cj@K3aTO%+}8rN=Cf2PL74)B>VxKiy3Bc{m{00xIJ?u0Cvs zRn2XH^UB{aN!Z|XiWw6)ClP>z2usxr0J()`$bm>?@_^bN%cZ<&QwcjzA2_TM`RDTi zF8Jux+vZDRqmg4wBY27z&yv_U2N`w3ha`h@;4+{NFiP4OCyolm9g92;1>p+_6Sp91 zAI7yv4NA%2{La^mJGhXdKSx>Tg7*zw9%C563wK%KA{57VMkhQTI?6%ztOt=e zMqptq>}z-enw{t=fA3<35s9O7{b4-kpbuI1if^1}y1 zrYv_dXPYvfm)q3)4nUE)e%JL6EKE8_=-EQmc(u=9O8C=g~LN>A7;gsNzeW zUs6BZaO_fOH7U0MJD delta 3366 zcmZvfc{tSDAIB|Y-^adW7>1Ft#1z?YMg~JkWXqC0WfzM13Sr33$XH@9WnZ&KU8E?& zShHQ+(%8wCt>4`4@A>`p`};i4>%7i+zdz@B&JXh+=CuLLpob#OA2J>=s{!A?)oW*| zXY^S2Ci_|}_ZjF-j?&yo^~(cv!A^*M+q?2QIq^9eaiHe$xmmmKfz2gZbH{tmS}Gge zHd;?*jxSdDj6d}oNj&QATgR@i>~tsJQe7<3q=Xm_OV89quq}P5Vfr<7EHZEwzuBdH zVFZ*B`sP8lGAo{zSFc*ztQ=coSa75FI_fZ+q`@(?TJqz{M9VR1W zcMr)9Pd=ryOTFE@_T_{N8#-$fgo{h7wa;hVa7pdXZy@fnZSK3@{J}|pqj9zhHdHepy6>5;`7zBm6pRuai&mf*8Mo z9eV~k&$JJg){>hp58b{5n?Nnrz(l`Z3Trh*JK+M%C40lRD;IZc-i>}byffx!M2T$CM&Zn>I_Tv&ha zlEOP$cMo8$rT@rxG4DemCu5)7a^~Mz$_3g<;eK0`Hy&^DnGExnCKAW4=PAbd#P!Xd zm~;pxi7uHR9jM=4DS1v>xaX&u$KGYr=EUZku59MB_%nN3d2n23d!XaCnrI(_U9h6^ zYSXj3d7!w8LmQzN-p^Xfn(||&N(A=U=2wn) zPn9X0hW_gV4la7%RzF*2W>(vF2YDs}ti8c?V%e|kT}JKjn;PQp2ft3X#^<+8&K33W z-x`rLrqP6^P4eJ+lF>c(2fLY5L=a=YtW^XKv;B)0rPA&C-Ou4WA9m*|J>Cj@D9Jm~|yF6O3?Z zy)0noFl*lG*dhvlC}zNA&L=penc1?qql;oGL`#DWplNV04mC_JM0ddw{@GkAp@7lg zDHESI4vOQ+*q0N=+q)Hn{SxP^fR}=Rgo(M{#>snf&6c$ydzTY+>f!K*ao}4s2r(ka zS{mF3O+yQqFDAY=5RLB>V`AR6{TDBc2EM|y-i^cK|Hy9V#oHpeYy$7 zcd=TD2ljcB)|^O-!ZC6iZ~&NGZw?qAFZY#;!e8*=V}9%kwCnG(V)|=2*`g^K@lI!- z_-m75zR7%MH+LJa5>&P|NOE%@9(!9kM*iNmticeHB>)KTjdvYrHc9oNl01x6%%few z9H~Y~J(;_ym{KWTUTzgCdo}u+Yd=16pU#zvW{F;|M7#~&WdQ#pDl)A}ei)S`>)61E~AE@DObo3hr5~*L`+Nx11BYviQ8nw&=Z$!!Bby_aKPtLI%Y?} z!RTL4;J0YlGM8&pkn0_~DS>KQYB>av=6R1(pv0koQ z1HtZL%C}JI^{aa_yf2EX2$&2I9$?3VR;ue0#Oe4$LlHr7>LWe+$bz74ZVsYN*ZXej z!2|ng2pURU12mp1Ygz(_VTIIA&=8y`RJ;2s7F?KI%eIGD0mAg%A9a@^bWA_{JT#F1 z!cm;^vqz>4B}Unm6uD*_5zjbzue=!=1p-s-CRAe;!!0mc~A3`#4)z@7Au+{Pvnp7{g7Xb59xxLfkG{jq^siOe{%MQHh2z}bU48yDJ4h$5 zriBk9bFlO-6UZ6J!aLqlDhx2`&|S8gJ8>DCCFM7~_Ps(8)Wea$hHXH6Z^k}^_y^oX zDDoL&p)02-tY|bR(prh8bdbUIF$c3Pod(7W6h$qpKHMbqab?1Qo+#$*oKK-KE8>aU z99PNB*^pB@L{?wjO;1_>61I3xEeah}zE zgT(}+g({tF?pps@o?xm6XyUW@tlc>(cEk(UfYi`_MeFYK6_F;psp=y~)U+DUxH$rB zx{kV$Z(+kNV3Yj4I2a8gco4!c2=Qb=%YGk-rq-U`SKNeSPRp|H2UFe2!nfDP#sI=) z&?0Op#p0I2drJoUayrG~D~sqQuHnOxLw)22H2rE(9-Q~n@f~rjz;IGZHKjU7y1~b6h^p(Hfmc8BgP@%>XHjR+r<&aL^J>^bP ziqJ(xyW+P_m48nsY4D&MOatU-OsCLIb2WlWc(2GzV$nj1t~nQ(1?u^yu~^O=yOrbC z>tXZP+MeCl=)AqfYVnG72*rkzO43CwnNTey2(YKR7g@N;O^`j*@YG*Iq+Y0vB!U5fWMG4+hssC@dPc_WZ zm`I_U_^)6KRnUy^Nf?yj{;vA@bNz4lFIdH!bNd;lgz9<3q(160gJ*hvSjlFbXE^0p zQUuiyvl-__V@jk&1~)1)S{GN;Z>#Wo?0Gs9ZKWCB(KdOo; z-BuQGd{>tbdc6`5w3*kc`qKWKtLSM|@RbTyH=_jTbxdFpfwjYTVw*l8??nRHO8(^# zkC^RP2uwA`i><~=wALXqkq&*i3-e@uPBf~{B%nq{xH z9mq>0uPBVR1n(xKG$w;N?V<;&w0(=fsU=wS^u{+LLXe-JE9U zv_Qxf93-*aQoLiXrR_WMa*!O(^Wwn2z_&pdX2U1_xzkL__ee?aGW$<8e^6fFJT~fx YtHDjWpYHcxs6ka$oA%!u3xWgx0crzeng9R* diff --git a/summary_results.json b/summary_results.json index 7e17609ed..3fb90bdd4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.24672, + "Performance_Result": 6.08642, "Performance_Units": "Samples/s" }, { From 80257f15ecf4cb7278f60b655f59e7df84a8096d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 05:05:22 +0000 Subject: [PATCH 481/719] Updated results summary [skip ci] --- README.md | 82 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 8 +-- summary.json | 118 ++++++++++++++++++++++++++++++++----------- summary.xlsx | Bin 10483 -> 10663 bytes summary_results.json | 100 +++++++++++++++++++++++++++++++----- 6 files changed, 223 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index e00879780..c0ed5d92d 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-22 17:01:01,043 submission_checker1.py:3301 INFO] Results=27, NoResults=0, Power Results=0 -[2025-04-22 17:01:01,043 submission_checker1.py:3308 INFO] --- -[2025-04-22 17:01:01,044 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 05:05:17,173 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 05:05:17,174 submission_checker1.py:3308 INFO] --- +[2025-04-23 05:05:17,174 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-22 17:01:01,044 submission_checker1.py:3314 INFO] Open Results=27, Open Power Results=0 +[2025-04-23 05:05:17,174 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-22 17:01:01,044 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 05:05:17,174 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-22 17:01:01,044 submission_checker1.py:3324 INFO] --- -[2025-04-22 17:01:01,044 submission_checker1.py:3326 INFO] Systems=19, Power Systems=0 -[2025-04-22 17:01:01,044 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-22 17:01:01,044 submission_checker1.py:3335 INFO] Open Systems=19, Open Power Systems=0 -[2025-04-22 17:01:01,044 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-22 17:01:01,044 submission_checker1.py:3345 INFO] --- -[2025-04-22 17:01:01,044 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-22 17:01:02,095 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 05:05:17,174 submission_checker1.py:3324 INFO] --- +[2025-04-23 05:05:17,174 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 05:05:17,174 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 05:05:17,174 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 05:05:17,174 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 05:05:17,174 submission_checker1.py:3345 INFO] --- +[2025-04-23 05:05:17,174 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 05:05:18,159 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.385386 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74147 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.314499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 9707e8768..af6316f1e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1108 +1109 diff --git a/summary.csv b/summary.csv index f01397806..9cd44909b 100644 --- a/summary.csv +++ b/summary.csv @@ -1,14 +1,16 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74147,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.08642,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.00743,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -17,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7219,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.385386,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.314499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 47b4acc3f..1cc1064bd 100644 --- a/summary.json +++ b/summary.json @@ -1,5 +1,35 @@ { "0":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.425121, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "1":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -29,7 +59,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "1":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -59,7 +89,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -89,7 +119,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -99,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73523, + "Result":1.74147, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -119,7 +149,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,7 +179,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -179,7 +209,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -209,7 +239,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -239,7 +269,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,12 +299,42 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":23.0024, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "11":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -299,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +479,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +599,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -549,7 +609,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.385386, + "Result":0.314499, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -569,7 +629,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +749,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -719,7 +779,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -749,7 +809,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -779,7 +839,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "26":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index cf68b25bd05ca3dd9bea4bf2f61faf35e32916ed..2918bcf37aff0ad96876472afa50fb2a4d964a57 100644 GIT binary patch delta 4920 zcmZ9Qc{J4D|HsEpsj&+oV~j~7HTHeYlAXe2jiDKhUAFMH6_PB&$WHd9>|3^cA`Oux zM2YN$H1>(%H+|3VobU4f^SwHH*@^^WZ8TKSBwJW zHcV4dp)<$)WkZg`Cw|+CsNNZxKk=;4pe zpgg&cCXW}7>zvgRZ>T7Fs9E-}c7LgyR#QgpQG6}}CV}CeT$bcCD8SX-!IuZfy@bp> z-WZ&?c9l2ruF6j-@#)*!FifS zW%sq|ZW{4&I_GQ&8Fv6Or0=7>dX~;!rRVc2==Sb>;NmQN{-Y@VAPJ@hIBBU>l*;Y9 z{T$0w^CKy8*L+`vLh=s$IM~HE7~+|o05SG^>f5P0^rkiXI=^?bbb7C>or{mNoD*Ru z9oswUBs2)5#3{7fbFklQ(Ss~(bcwP11V|a^DUDd)(WpAgxEl* za$1V*$=h9_7szWHdw>+s`Q+Q~)o?qH*s)M`f9YY*jQl&%=$Sl|ZS^v;gZL>{NJ@8V zaT@Nhq}+U{lrX-bWFMaOY}|PV?=aUUS(Pif5Y#l}&f8uQAP*?beUe&`nd>dy9P(Vj zeA(V}Qd8&ukfBG!?Yq|HxRL{YV(_nppbevkrDQ+93->4_sK<%_m{1DCQHxS(nwRZj zD5mN({ptE_h@E>U5u+%v+Zo}v>h~*j$)w|Ro=Yz;AerYv-zv&Z6pYvZrLWVWs@Q(OV zsrPw5W7kO0fj)sk%%SY@&Q2OBX@*?qmIV=dTyy~46B{s`k>iGiD50wbZR0x^z<2!6 ztlC9_+wAzlhGar>QCJyTnctU38*Og3dr`D2NH)iX-UiA}gv9IfTz`QW9j_VziB@HJ zg|Y7CnicqpfOV>w+Jw_@s9HehJUjcIiq3V%?9s|^f>U=fV~$O~#eSds?8FL=SAo9Y zZUyM>e%{6nHvO{+@U{j#@bTRlO)vlPdm+Q_z zUN{_zm_)uk6Vo-vic7?($qokY7R7B}0Z1RY9D)Pw(9Ro(TKGs*+ zU)XM>HG$&z@2tUtTfWmi)2ZeSu4_8$s(J@DWy%r7;7?sEZf*D{>RRPXLI`c~gAa8B z$k%+&XY7(a8M>^uyDUgY<{!DfE%>O$!`x^kM|)Xc7$75HL{Qdq5sW)$RyBhLFg~Jn z?`WPbYg@lJL{r8Um6X;~Vl~;=5el?PM*>n=e?=#zhTb%9N!x$fyTy`yyz)c((E-%n zu|7WY4FTuOI1S}Bhi}A)t2I){`i9sc&mv-7<)C_8c^8$*M#2zpX04WKd@#N(&zi z@AojtEK|8hVFv1jF28lf%Onn#?f?;Pf_~MFIHjXOw|PuAZW=+d2P)k zSv($uVfizLlqT}F9<3Ck;`N3)cj||A zP(x(#*akVx1k_ef?^ir4IfWv?pFkmH``nlG+S?(MYJ|HigmOZV1uY~~>vznm_+jSzD5?v0Nyj=|06`AvtGmLcCJ}NS-gzE$PL>*oZM>^bo#?R99#k zLbMRd=W1YWm2GrsG9hN{lZIY2Grb-}$>_(LCMqeNCJjnI73Mz#84Q?`@#!!A>$kg%WlLyuXv-Qgs8Q$^dXd~ad zGVH~=Mi{1�apRSXq!wR&PEv>^4z{+De)pJOd%ctc|Ei8BBvRrve+|)Yq$-aM_CF zw)g5_v8novpFc}R082)BQjo=wNT!)Ppbm|;vlnYHZqf-R37dMfU^3yTF;TfCObN>_>lH z=ZIw6!-`oB8FL|0@h+8ITDU&rX!RvZEo%X_XHYfno&#mUmi3~y>8PP(`59z3S>#%a zu`;6?8~E7rf(#X|AsATQkqOKd_GoG3Bz%<^bzsR{Y6vdgq00GF<%reu)RZS1AT>%2 z{>o*-7WIH=(!caXIG+kzJWg=qUeuey9ft}d!MDky3p-&r^+rRrIlT7*wR9Jzd?ru~f6SYM$R>4S7P7~EAGC4pl zzrIk_jI?@l>xnwK5{ps}%sd!Hkiuoi3V`sBzuY>h++?l#{ocp%n5cd20q^PR7nHTEru?nwmBIn_JMYz>HLZtkHlQGYdanZAS@iQ(ioxH8 z!6mITB%cSwAG;LYSTH2VqSM4Sg*j*?P`VaGWa>?&OKr|Gu%p;LNS%!ANL|n0{8Eoy z%+l4gJxu1;52lD=B*UF8%o2sOK_eYUegCAlu;?(6Npj}*m zzAkF)SVTmpK8pMnKt`asclzZr|895;ee&7qgqoCdmf?_K-hq%t*5vBnb?W;)nd)4` z-v;_Z<-2~aRbfyH$P?cf*d^q`FXg$g6NEGnh=!`$uMcdF?gc)!t5{50ubB#pTcwCK zd~ZEOjGUk;u9v2R0><@E?SP@Uz|fQS<|pqMi>j8PfG65XXkwh*KJ{$YUSb@@3rdD)NLB+XMba0@H1k_L2kp!@vPS%P?Q|8oe5LV2GV6>Jv!v{i^o^t zcL_WT^6k?X#0&yX4SjAlJ5qf2)y)j)=<)Z(`5V%>LTJwaAqX07B*L7RwpG5CP?zlS;SOuC0Be}($cI?4|@?a%eHmcOvn)dx_vM4y{o@8 z{YJ$+hwx{u@wF`>#L)^HB~-ZNlb0L7+^oc0e=TVN7d+F}t}1_gej&=0-Ef=LMA`I> zvg03MSAkdGWatpPsu{P?D-(POy1b|$1zf@tZa9=0z3*NgTe`u5SO9|{#zyX_SDEz+ z>jl?pkUleYfC*t zcr$BNS|76(C&5;n25S)s81G7;XA>rLy$AhmAotq3#Pyo1Ou<<*4yYABa46jN`{U5BsA8ccolG=^x;*^s{V z6WcD~VOv-C;2Z4s!1%>N*$xvAeuwBWXEQGN?JFhX`LO&^)g=_==&dw>k@xp68{N$gS>%}ed*N#**tGq+#jv)gM#eWUE6R1-L2@@PV$aA2laytoKM z7W&Tt@v$SDLoK6w{&GUbZsQGAq~-EFk0175=mRUZPW(5W=E`RC^D2@All*N}Ib)sA z7K)itwvfB)kn~#D32!C9k=qtC2q`Af2O|*#XNkyr1!!Xy$goYq-SZ`w%LT3?C)mhk zEPT8_ZlUi*rKzz?Nn?&NB~Z4ts3*N~WZs0 zS88-kjK5Q-CI$Tt0=?$UxRDFBMi&XyLN1mZdS~FG!IXo zU-wW0?)JcAp0WZNTl@IN>y3h!5M+)RKWiP;ZnLEii1bg=X{#fH0WmgVLYG^7Nb>w2CtCu~y!z2m_ zRbhiYbz6^|@;eht2bMS%Ss}}H>op1A`}qExS@jjC0gOGj%%{$9McNWe+yQI~W)>3o z;YK!3gBSDk)}wNVW+8;iM9*MdoLv74XU78L5G*3neRNqnL{3d}7Ni{M+x9T*Yr%7+ zlgak@1pBGIW+8caK zFLK}SD}6>D;G<8Z(AW@y%h6BDk=@lnui!JmR*P*4-f79|-Kn4anjH;{hHspzH+1pe zWd^<;XCuUDKp-<(5QrTF0-?QJ%zVAP{iIyHd|j^wdEjzW?G4{v<9fVB;TAja7#prg zYo<%LS7gh9e|xlqJ1Zv)9es8BZP!m=BF_1khXIRrc872iD*)Ue2) zk3F+sVrvlAI-@F=7B;Rr|1?<{(PL?%+9!BTPMSSYf7q>27i7EPtTgo6)mk zQ0tEAoA^XFwjr5-098|7AwSFG^ftV4@*STvC?&|a8)EN=% zkRtBlcR#Pxa17+;$Qe_iyCn)%{al85(Yz$a2K_X?AD;5RXg4~}L)dgu?C>z|1HA9y z*0dw1+|_-Wyj|nRFFC0@&brNP4mSOlWK3y;Cv8)+w7(5Pn_|w?8{UNSGvOP= zKl;gm7$euz`e+a_HcqKoOK1RHGL!}!O? zqwO1yFdZAQY9SjjY61_&g9TVy&($Hzg^O2`>9UmAaC78q$+2Q9v6`LlHdt_n=ZX5c zofCUFwan>j0;~?T+K~CREH<2cgM*qt|8EC@k4MxFI}H8NRD{Mkz4_SW>$&+7ze05( z0wfPLfnR6gY`Pd)Fj#1jYaS~+9Q8>(QLFvV%g@(^i)<$5Y-pozIm;)jm^0!H`bB!g ztnLPbjyC5;umfRt4qn!OF5@bGBX53nIuYM+lx~mAtb#7=ZlLPFqL6)&jsdrcSEFF& z<wOZ`s_>8}vF26o^ ztr^Mg^KU2b-X!X#_{vmQKAhg|Aba{w9TrH_;b$EAh*SIPotTT5f(PNw3JP z*SM2sj+wDBLxTzl@qQ6`hlE+y1VL!3YA(f%40X011RE+>+Se;lsr3E}EuxCG3E@qp z(kGg0rXiE@uPPqgDA8_Tto&?1Fjdl1AaRuQ2PmOm@DQVJyedeyWv7)pP#;16?B1ZM zA^^o$g6cryY(rIuY3Xm3vD-g4J(xm%yPL4-f0f5(siod+p;g1tun2*a*=3~MN@Y^@ zbd>2NdIqHzi=%BEx11ysA&;V;nsDo6t5NMZl!m)O5ORU`v7${*jp0N8axQ_#m(tjm zWU0%Uf4=pp7;#7{s-#3J4ycesAj2;GUAaZqb((Mpw*cZB5-svcSPb$p^EbL#1JF2(id( z$wf9-1%aa(gn*Q<$;8?>Frv5XxrfAi22H>r_Y7(dtGJAEMLO!m8HLg8Kk_HeHQqQi zI$D`&B&0$6{KAHJiHYtUch98{c2_lo$z1forr&EZAbNO3Q!~8GZE$n3ing&BW`yo7e_3i5c&%>Ld2qf5K|_c=B$zo zyP+h2E5UZ9gjwmMh}ngQ!V{@h=A78*N@$$tXw|uuYqPD&C8IrB4Cyk7*5J9%909z0 zm!@9T)oK1F$f!ObmS(7AZovwqoT6X{=z~i#_W>LU6M>AIJIe98!4!zLVbyleN!!Bf zuVhz!_fEwTwcnO)T#rL0OTYQWKd$0c2cO~4zPRyWvMQpq>X?P8!lVtm)`4h+-u@1F zZ5ItDQjCl5oN^B*`!GrRfpQ-$=OmPJHLQ+#>(1T3qFG(oE(>Elo%j z#GFyJ*CQB5D|8l7;x1U(=OPD3)4N5Hb5r=AmlR+qqobTkKLoxBh-a6mQ$$2>&A#AD6@>( zS<%^1OD5@8@;FBIv~(~%(vC&h~bq-223NIK|AOP2~kQ!eK)&wzgWwk5D@Ptmh z6NI1^W&9q_#j<+;B6B%;tyEMJO8kYLZ0+?B#>eI2G9{LVebAg1ev&D$q$1idkXdgZ zsfn@UD6vY@ZxssCbagtru*(!1OH)^3d1eKg!#ncf>@xOZgaG2COe8Q9IXIv4VDL4q zl{p&?cIGYt)EM~Q{wK5bQAb~3b*gb!=G*P=?A_*xYP4Z%f#@8Y1f&LW&XORUfK-dZ zJeE3_V)8^GTJ18nE);K(LXfLbx|qm~7~jZdFx_f}lyMguYl*cFwX0A%vh&bDJK6AY zg{a3V%BZ50%=y7~irO|eqLa`J&fUu2h`~B0^#Ys0g%p>iFTQhDPNi(0v91xZPHP%6 zOn>v5D-y!URnZHgVwfPhA(1h?5_@NuqQ(tj^WGY_0nvcx?|LZ_L-(w;7np|vh7VN{ zvG&vp%p(Az+7(9v57wAcxw7{lul3-{ML|)8A}>vmcVhEl(ILv5gf(;ZVYzs)&LS1V z`5AKOMScbu77-g7CKrU*|0euDfR+CRxT+Dc`fotY1)wl`nK2SoX+JhFaMMTI4ap_y zanb`te|F(oyP3O2iRkISJA9yu$l5()VahdWJLo`!Lx%uFyBaV73m6plIfHqgT}0?# zDr3ygo}##OO}h5-oc+#1kVHw70r%o{4cvODj$+HiFqBAL1HSLKkz8}Rd+RZdEPcS5 zi_N?2Mi(;WRN?j+WQ_>&f3m#_yYZfs8{R8g2sW%F|5SkgF&fjA*fWfL_$ka+`<(!c z0fUUgt}O_fUqlCW8J$!P%tdtc;A6~h>C+Z{m^0wi%HQ2e3{E3L3!%q?{<&XSJszkvjB^-}^^EB`=1p!VB(UL#t7pX+*O zjdyT}ZsdxrJY)BQ@oEg2Y^q*rmd=}<&D$_WQf$v&bi)4dmHOg`SWQU3Lln*nGq&VpL5Da@%v!*yz7^?J} zU#GoowJT?NWGH)QZm;-UEcn21Hf$%paK`b()buzH@xgsadq09{)zV|Mf$2fe(7GeY zWx?_lCo$`FG^fOY`IC?b?Gy6`L1yPdI(X0R!x8gAUMl^4gN8Ru9a4d(H}43oo~<&U zQ|f77p<)LB0CX1*G6e=>t%P#~Q2?!Pg08=y0s#6sQ))Oxu+5s?wd_`+)V6U%3{118 zLN~R1s`F0s2O)>QXJ7JVGTN9xCR&vWyiI@Qjr?-2;|ZC_rwt%`x86dO!LKCo^qGKt@aw zBcl=@w@f)T7lm!riyhgp*Ik*_Z3^Y0Kim)~M=jhkQLOJ^sv9x0iH8zvy^P$<>I(6d zt1zv3#>q|eIX~Q9Q`mH{N;9WJir*i`=A|+seN2TtHq{U)Unv3FoYysZrwRuiFGOXKVr6@oQg=(P(bt^-jXaFp1*&6 z3tW{N8IOSv&Sr|CqO|xWnxxvu^yZFJhNH$i?;GF%E@RNHxJvF)skjE$7-Jl zA5tk8`=cXSgDC%|ufRb1){jIOa9dYbbYOYF`h-RNb+O|2P`9QmA7-u9frTGmkk@+N4 z^{I&bL(rn@L0X_g{x5=+w#d%oirMTcmK;lJ2_A89`R|@GV(!wZTn)d848Qc#5KN$l zK**EjrW98H5E!4KweXK*q4R;-aa4dF?gS|{tGz#EWD<`<=Ib6iTaAzP&|(Eeuq1_wU-7ydb~`xX_vZB4jTbQqw6wpY9qtz*cR3;O71ceLhP)5lnjJcOMI zPZ=v46hTjWEBM_`tCJ69g1OLXIKgh7XTxQ8u;|Ytw|6=EyU>)x+3#MwDLpA`ZUNe@SrVtp+M~-l<;-$l5S1Tr$i0 z8Qb)8WUDg6xb!#c)Sl}+v1X}9aH&q7FeW{Xe8)P{41Y=3g9a1d!56*yjO%hXN2AeyBgc19TTvvt_-8uAIqD1I!AAR`tj>sN;NSC|*=5a3hG#smLg*i87Qp8Bs;=I7 zCAScX@_*m{3u0Oh=uW=ci|Cs PkQA(l01X%Pa_K(+6mQyu diff --git a/summary_results.json b/summary_results.json index 3fb90bdd4..32d215708 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.385386, + "Performance_Result": 0.314499, "Performance_Units": "Samples/s" }, { @@ -545,6 +545,42 @@ "Availability": "available", "Category": "open", "Suite": "edge", + "System": "gh_macos-latest_x86", + "Platform": "gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 76.951", + "Nodes": 1, + "Processor": "undefined", + "host_processors_per_node": 1, + "host_processor_core_count": 3, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline", + "Software": "pytorch v2.7.0", + "operating_system": " (darwin-23.6.0)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.425121, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0017", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", "System": "gh_ubuntu-latest", "Platform": "gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "UsedModel": "resnet50", @@ -572,11 +608,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73523, + "Performance_Result": 1.74147, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0017", + "ID": "5.0-0018", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -612,7 +648,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0018", + "ID": "5.0-0019", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -648,7 +684,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0019", + "ID": "5.0-0020", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -684,7 +720,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0020", + "ID": "5.0-0021", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -720,7 +756,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0021", + "ID": "5.0-0022", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -756,7 +792,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0022", + "ID": "5.0-0023", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -792,7 +828,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0023", + "ID": "5.0-0024", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -828,12 +864,48 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0024", + "ID": "5.0-0025", "Submitter": "MLCommons", "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline", + "Software": "onnxruntime v1.21.1", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 23.0024, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0026", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -864,7 +936,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0025", + "ID": "5.0-0027", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -900,7 +972,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0026", + "ID": "5.0-0028", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -936,7 +1008,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0027", + "ID": "5.0-0029", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From 440cf438d88dcc430e2389c1a1762df83019c07f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 05:21:32 +0000 Subject: [PATCH 482/719] Updated results summary [skip ci] --- README.md | 88 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 +- summary.xlsx | Bin 10663 -> 10663 bytes summary_results.json | 4 +- 6 files changed, 50 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index c0ed5d92d..eb3de19a2 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 05:05:17,173 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 05:05:17,174 submission_checker1.py:3308 INFO] --- -[2025-04-23 05:05:17,174 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 05:21:27,622 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 05:21:27,622 submission_checker1.py:3308 INFO] --- +[2025-04-23 05:21:27,622 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 05:05:17,174 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 05:21:27,622 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 05:05:17,174 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 05:21:27,622 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 05:05:17,174 submission_checker1.py:3324 INFO] --- -[2025-04-23 05:05:17,174 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 05:05:17,174 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 05:05:17,174 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 05:05:17,174 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 05:05:17,174 submission_checker1.py:3345 INFO] --- -[2025-04-23 05:05:17,174 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 05:05:18,159 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 05:21:27,622 submission_checker1.py:3324 INFO] --- +[2025-04-23 05:21:27,622 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 05:21:27,622 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 05:21:27,622 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 05:21:27,622 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 05:21:27,622 submission_checker1.py:3345 INFO] --- +[2025-04-23 05:21:27,622 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 05:21:28,606 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74147 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.314499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73772 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.314499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index af6316f1e..0a75ff761 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1109 +1110 diff --git a/summary.csv b/summary.csv index 9cd44909b..2ea375f4f 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74147,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73772,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.08642,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 1cc1064bd..cb36fc457 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74147, + "Result":1.73772, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -139,7 +139,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 2918bcf37aff0ad96876472afa50fb2a4d964a57..cb0c2096200802158d9e009dfb943d1af728121a 100644 GIT binary patch delta 1615 zcmV-V2C(_3Q>Rm~atH;f;K>_%vv&v&2!Hl{&7ss5hfp58 zyB?6%t^vWbO`NX|jJ<{?Q=+z<-g5 z4im5h`G=Ij4wf|yJ6K(_{PXu?d7$RcJH2cE6#u#_&~C|FORY<|M2Eza>JQ0;QvD$r z-2l=WyC>|!fOKe}e==P@q-5+kpcNmkm)?pi$sm~(tRI#EGD!x=G)X_K0U*=l{Ah4~;7CL|A!LH{L&|U?TgCa&HGexl--paY zm45WT^yAoG=$72Q)NToyD35qj`XQN6NwI{ICW9O_TDYLHU6r z5$ObVJSgkYGFUV#C_lPp<>y<`e3Fk>%Ooo;n!C=Awp-c>OKz2DuY^qU**wzuA(>D* zKO}=>R&ah;2FN5CAk(D$uzv=COq21W!T5nA5$S}G3C0g8!$q@-@uO=te!dTxhpPN& z@%cWsAKE3K@1skUA5A{KM=C!g6H4WWWDv~?$`8u`noQV%E3+o$hxGz%nv5R}#t$5c z*f7DygR&kigH@AwL9lApto(dCn@`!$9;o> zCY0U}$sn2)ydRbUG?}#nXqvnq)(fC%(tb2(KX4>s!$dSe`ypkxY*x{Jbj{k&_d)Ye z}XBy4{HF>G`T+-+#fg+ zkxsyqg{mH_0y3HT1Ak;5lKJyDh_bcJuM(8uK9rWL%N9(#ME>av`lVhNeDKx_QBad!pa*J zNUA5@$gE{+6!a$e2y|hcj9~%`CUy45@qbzCukbe*&S;72a89=g%`D2^R zAJz*XX)=FoFn{1k#D*z9k-Q6KJz54xR`}Q-`{2wUie&gCBo9>jxw6$hl70ZUN!mMK zk1>rGH}QEqnWT+DlAo@1<@V_MJ43G2@yGSoYdpmB{*u{nTznR2f3p5EkZe4=X{3Ly zzh=ZcKL1@?MSo+1(baMiH?ptu__P^+ndi??G1+Eg_+_;If|162F-{ia?CZMs;u}tu zlm6K>8UDB$)N(fda~c2m!{7(+_=kb_)6Wh4Q{%LePyIfc&u-OA@dF#y*~z?r6Lpae6`>9ViwtE2-6l`tu|o202c+2a)g-@9lSRLW5*64sY`-G1#zdXq&}2( zkADqi-*gW+Ptyl1!Bb8>#T_skn z6#iu3rgai$_tA;j*tTubE_!RopW)B#%Ui!{EL3+Y2B<1HBg_Kz+ExX=EW+U$>ijmG zcqf2Ao`QbkZCqsN10IC{WQaQ{AB(5wdL)CYJk6JJdLQSDI$e@HBl+s98+p}#2Fh{B zzljrI$bJ0@sz0+CCCUN?so=>Qdz0uVTLBf5L?}D~pOb+oK@D>o004MwFLQKxY-Mwk z5GfRs<0u*eRm~atH-8E>~N6vv&v&2!Byd$y%u`4y8t=#FHiv$%N7bA{l^^Wh9g= zBcUW2Wl2|7n@k|qi-gkDKu3cJ1VoY2hp1P}1gl`F zcRe7jTcID?C0MEwEYTwIr2RuOp|pQU2C=MQ|F8^T$=n^l(q#XzUI0sz{G&nsfqx?r z9VTE2@((G49V}}acCfl;`RDJ)@<7d>cY4?SDgJd=pxv_NQkQUv4v8n#ACd{B`a?3v zWd-$zWq?bP@g(MgU#a57hbj{MwV_g0vMqVwGthAHewSMxQY~TcCJ6WfMOw>m_Y5kB) zD6JooK{6{?KP&@ek_?b(l73hNK&Hw0(ct{Rk%)9c$OPwyl;K9Uiu0puc7J}p51EH5 z{pfw^$FaT8ExCKC-4ZlW9`U5~Lo%V1en4SxWcCgVqg@dHO9(g`6Gj2}{li)Iz$N7ropd>=9oRr%54 z^L=bTv`aqUN0%r+ntXhZRDMV%l*$juAet4FAC>_$nXm&_W=+Zu>jl^}89y3~A2<@R zVSpR%DnQ1RzV?}|V9s%cp}cbWImEPwHSH2F9m>HUyQ zD7_z&K{P9PKP&@iGHVCWGi zKU&B?j zoq#6`RXtV(WHR#y$bUQ}^XG36WowyVB`CvvC@oi)EtqzR{9~g<;!F95WRxZC9})s8 zSw>RHGLlLBDE|m3XncU2h5Nu}cgR_5xl{YGo zR8P8*S|yOa@yBX~fi8J_X_tsUHYy~(6n{uYS`KT@^1kOe?>VGc9c3N;f^|Zx5~T@25h$j)Q2P?SZ`bhy zikzp4(^@MiQ40>Os+)qDoM{WMRvWNhfQtf1xty629lSR=#*Q68QkMjW3gS*%NqxB5 ze?2yoebYVQjF3Al!Bb8>#T_sx%7QHp(Px0sG<*gqz7OGnn15_29<;(*0+ExWVzl6or)cH+V z@lF7LY=eH~Z5(8%10IC{`4D$fJ{C{UCEGQsGLkK0avNuhEm`K_r1xK4%ZvIGP>xOh zO`HHj?#oY5{Q Date: Wed, 23 Apr 2025 05:26:05 +0000 Subject: [PATCH 483/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 14 +++++++------- summary.xlsx | Bin 10663 -> 10662 bytes summary_results.json | 14 +++++++------- 6 files changed, 38 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index eb3de19a2..f73665dec 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 05:21:27,622 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 05:21:27,622 submission_checker1.py:3308 INFO] --- -[2025-04-23 05:21:27,622 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 05:26:00,225 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 05:26:00,225 submission_checker1.py:3308 INFO] --- +[2025-04-23 05:26:00,225 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 05:21:27,622 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 05:26:00,225 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 05:21:27,622 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 05:26:00,225 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 05:21:27,622 submission_checker1.py:3324 INFO] --- -[2025-04-23 05:21:27,622 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 05:21:27,622 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 05:21:27,622 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 05:21:27,622 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 05:21:27,622 submission_checker1.py:3345 INFO] --- -[2025-04-23 05:21:27,622 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 05:21:28,606 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 05:26:00,225 submission_checker1.py:3324 INFO] --- +[2025-04-23 05:26:00,225 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 05:26:00,225 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 05:26:00,225 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 05:26:00,225 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 05:26:00,225 submission_checker1.py:3345 INFO] --- +[2025-04-23 05:26:00,225 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 05:26:01,251 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,20 +33,20 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.00743 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07639 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7219 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.314499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.314499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9793 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.7529 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 0a75ff761..5f2f16bff 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1110 +1111 diff --git a/summary.csv b/summary.csv index 2ea375f4f..8137697fe 100644 --- a/summary.csv +++ b/summary.csv @@ -11,20 +11,20 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.00743,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.07639,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7219,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.314499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.314499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9793,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.7529,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cb36fc457..ad5bdcc9c 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.00743, + "Result":8.07639, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.7219, + "Result":21.6529, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9793, + "Result":23.7529, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -769,7 +769,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -799,7 +799,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index cb0c2096200802158d9e009dfb943d1af728121a..c2f75e12dad8097bef6233ee8c72c7e8f91e1db4 100644 GIT binary patch delta 2393 zcmY+Gdpy(oAICRrZmYQ_m$OZ|j9jut9Y-cpHmp`6l8Ux$ZQXSt`I1UH#oC7Ot0{DO@0s_Gc&*&jxz>rYfFw!@4%jwDcFZCuN9Xg62TM_2Eo%UWXA|=Cemus!9fddQL zfUkMARe@@P*{QWBrza1F6HO=`Iy-M?SO)ND5DFKH_aMFkUOV6;4O2>sE_BT`>f%SQ zU79_|XcwTI(hMsrnF$iju{f>8-9L-6stQsM4n^1hIazMvrF89yGmr<=KckthNp+RJ zKnTdV1_u$OV8qv5z%1yexk3sqF7tk?i|*$KuIT;!$yEOZ0m?OvZc2Q_9$3VBz2Tj# zInnU|=oT>LZUr&o+Ci=%;quThQ2MpE&YFZ5Ejb#NzY{KYxOhYwzZk>zr3Xi~+N^Rd__DM> zIq0(^H(Jkm`jpva+nZ*mNe&!1QiRDjw?XAV@p*RIjC?$5K&z06OEA0Za|A4h$=^G} z!R_)Q_dH_vI(wV??=Yb`| z^O<{O+I6?W4!*oo0y4WP{{N~YcT zvjE95R%C{55bSR)^Ul%ms(q;P@f!mcfX&pG|LjmjT(pn4O_h;n(`i~08V({hvQZOb z*}#>0D$ssd9^^&hHUIkZ)J!dyUSOO8; zo%fEi<2Pxn;k|$7Bx_}-J8P9ko01*^NpJu1Oy3q|6LfV&hT{U*ijqYl4q`7RVIw#B zlX30bi5N<#lr?%6IkUZ^m9LPPH$VV&7)Qj?9z6HQ-KKh zBRk7Iu}mBgp`B7fR!e=qr`TU26r1wtU=!N?a_E#am&a>TwHp=gwRBcqta((pJdvb6{uP`0JFFuVBi0JS{)Gmjg)=i}+kCc91j>U6nNt@INee=1btwEm zAEyOQ2(|=F5q=d3Oc8^NC}bW>2Bnsgm6Ad=n4$P{-YEVo!_p<+oR5wcDrWv*_=;NZ z(Ye{nI@gc49%4GW)P>!CYJw9vDoEoX*M#~fm-2sHiYDL(SW6as_`hJwK`@I>=a#H0 z>35|s4OOAI(_A*+pUlWjs>pKNnv7SzLY1pQujuIiLvZ5uE&jlivX%WeW_2t#7vqG&T`5 zmJ$9nIRgHQ*1NQjsYs4+2i2-1g$tnCC8iRcsdWBFLF7i2#FIrB((NQ>12r7LhSjGW zh!X~ABqPML#cCq1h!2q9(n`!?HKLk~tG?q|N2rAQ^^9m3M}QBc0B`7)Wfh5)25q!4 zB)~}^jlCSgwD;nk2T?Dos5`6NqE=6tC|fuE{vua_6k&tG2-=%E>+to zz#YM`#6*7F2YgOj17Nuyc^n_=#CUtSiZl-$bYRMpoLca^2Vaa&)3{!6a&mod_S-v? zhYjkswW9m>R7@MaR0Z#EIy$XixFDKeXnBWYZ2Yd9dwS z*Kp5%HT@=m*i0+9`Y~5B&)R4;)4VS7itzsA``_iBk(=k^dT8H=4^YL@?=R`z888CCahwak4XS-nj z*Wj^xw+8nQhjQtnZyKB0w~q){ESe9W6nX@4KWlPjeD?FNqn`n4u(l2ljkOZ4F)vyQ zgT6;xvFAxwqJYi=O{-1@M;+)W7^FOhh*uQ_M(YEZBIt<}@}xf9w!NqcuzcQNitP(C~Hr_5{l z*1Gq*B93f0w`B$CO#8vBf7(V|elQ+I?TC{U-Ff7(`GHYz9}p}4zLo{GIN$N$T!W4w z0-aL+OU9@3W=R19=)NV8MvwSHKi#YumsX3zkTNx`sc_e?*@!cSyk6nD`cu0zTb{{< z&ul6^fK0*;_rpyU<*(C2kNQ#U*NrZXSY1{A^@K69&jTGk{lZ#bb`(9EQQNbk)z^rO z@XI>-vOQ4w?TP*F+ohOh7yG}MTdPD@mK_gj>x&toEW++?CTb`?+L?8S2^(f~g_id1 zTL1iXc%L12Kuvi8SRd#OUMhCOlB3oT$Z59w@%RKb{W#Ca$&5FQwbB~?g$G5M^UR!# ktbg_!3V~3yAP{57KPM&i1&`q53ti4jc3KHwnH;*p{nKmeP*xToZ4G}iGw;1bQ%6h!Lu+@h~RfyfRSJ-5bAgg^?RLIp3oM| z_;qD?5@HV*@;a)*MGphYeWO*Uvv&=+Gl5 zi}UB%44D||&>71mT#{GM0<1Xe6rqIax-5^IU_4|haH=q~O?pXKMOWa2KhY(Z$uSgB z@a-6G!UVf0@i?~;!URC9OjKrc7_xx2JyiDLUym<=?@EIegQG4_{pHoo&CCqvV|8K! zPkT(Emv0x>5hmR*sI*1+G_ZR+L6}3a5GuFdI}$>m1{FeY>aC~bW7>iY{Y8$6x4hEp zI&$sn`FGcwND0Rl?p&Va9j$!XkR{Y30{hKXm0!H^IdfTlx(Z>mMr0_G3az4t=qvpo zLGDsTB|k-L-_}6`tWL;C%t02$G!xP*Fe=;EZ^rJp7nWW=+UKVHy8pxdw#h3rr6cw_ zyxt^OKt`MT&F9gJWSeGSa>(Vtb}dIL)cB;<4Z%x(5HH*Xy!@qg0i-{7Z(LbsN9M0lWcY2p8{Xu2Z$wM&{B zV_7Bi&e2LiQbKu!hc4UB#Hw0jVHJh_IDk*PNb&k7*z!PcQ2$I=8#OBFzFA;g>d;8g z-LXftqN*|v<#k8Tgy;I9fb-uAS-sbXx8#LZgubDe=F`36p~Z$CQas$BUmU{uRb1&* zN5vR_^!17dkMZ099%Eg7X0BZLR0i>fvp@ys9 zP@m5zP8(dcEeRPXuR9k3>(0Myj*b@twC$`a%Ibg(u-0Cjzu8o@L7m6z*v$XY$c+}& zpqX8aNiTob=hm9IYPQzo)So1c45$yd?~~WXorX!zg|65l-iq1ZI`QrH(j|BRSKVm| z!BOAzfxX0eYHY8T6T$He)g8wrT1pgG^}pZ92OjK-l$+J1HDo+rRz-wQ?1_CrYd#k+ z4D55ixl$B}ZA5WAe{f=uaoR^XhRN~bY?^j^73JH2%?jx64BHFIn`apZzE%YAP*hUeK1ZnVMaqlJ>*l$7LSdLQzMwvGc zWeR>p89jl0Uof_#<4LX<>709zo8~D>7!_(V-N1S7N8VUUsY-oZmN_C3UTy9HmKpps zvLN%1!nt*BuRv}B^A?aBp`=lpA!tF38kENmW)m1>aAZ1t2%gKfvG!A1&K&?$ja@4cC^2Lokx8c0kBp$y`z9q2PXZ6m&R#lb88!zK5!1T2)e3hJ_9V`H{@Z+^=@q%Wh$8 z(!mAM!nLKC#z!7kT3RImT_`Ja_^}IQbOkORSS>Q%dbmUL6T(A$K^;@t176rB9uF{g z026*)zc`SK!AD|mgsxxs(im!aq6c+>zJx}@4t#qc9wBkEA@?ZbZpR8;E?MkSrMs6@ zUh?FW-8!pWnEK}CeQQ|tbpCmwd)MqLN0EFaa|;6yui}fC?#=O)z^#XIU2decC?L`| zCU;=uG<=u%_Uw`b0hGM&3vp4z#Vo~2R@uc*GF^hUOn&;CGF#lb5iL0{5HfJ)nHR3U8vOjDOHma1?k1K;c?4QT!% zSY4by#202i%-BT>_=!fyh*0s;XFR)2pRq0FSuN-vqTGej*7q_$bETcjKv^Q{( zSjz?%^Xr>+E^aH zu4=NBC*a++@`2WrchAa?XYr_W2BZ4*{yck!3mz`L=f|&1wZ7_dGTReuUUMltT*wd&zGO~aHmEUMrB76!g`Zd) zC{T>uJvsP8Mq+o_>%PzO$cG8I1F4R>Hyn4{^+4yx!*$wlk6v3&YlwfOZDWzIv9l*n z9-&WZBhN$V`ECxVi#6{v(7Cnly;n|Upx$3}jpDf5si*6#(nkPY*{SY_1*Xy6Su6Ur zi_@% diff --git a/summary_results.json b/summary_results.json index 5402fa7c5..c2d7f3616 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.00743, + "Performance_Result": 8.07639, "Performance_Units": "Samples/s" }, { @@ -437,7 +437,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -464,7 +464,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.7219, + "Performance_Result": 21.6529, "Performance_Units": "Samples/s" }, { @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest_x86", + "System": "gh_macos-latest x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -631,7 +631,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9793, + "Performance_Result": 23.7529, "Performance_Units": "Samples/s" }, { @@ -667,7 +667,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, From a36c03339c72b3ae2b16c0a104bdb0b11345826c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 05:35:10 +0000 Subject: [PATCH 484/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 10 +++++----- summary.xlsx | Bin 10662 -> 10667 bytes summary_results.json | 10 +++++----- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index f73665dec..f365a5023 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 05:26:00,225 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 05:26:00,225 submission_checker1.py:3308 INFO] --- -[2025-04-23 05:26:00,225 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 05:35:04,733 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 05:35:04,733 submission_checker1.py:3308 INFO] --- +[2025-04-23 05:35:04,733 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 05:26:00,225 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 05:35:04,733 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 05:26:00,225 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 05:35:04,733 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 05:26:00,225 submission_checker1.py:3324 INFO] --- -[2025-04-23 05:26:00,225 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 05:26:00,225 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 05:26:00,225 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 05:26:00,225 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 05:26:00,225 submission_checker1.py:3345 INFO] --- -[2025-04-23 05:26:00,225 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 05:26:01,251 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 05:35:04,733 submission_checker1.py:3324 INFO] --- +[2025-04-23 05:35:04,733 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 05:35:04,733 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 05:35:04,733 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 05:35:04,733 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 05:35:04,733 submission_checker1.py:3345 INFO] --- +[2025-04-23 05:35:04,733 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 05:35:05,809 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73772 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419768 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419822 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.7529 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5f2f16bff..ecdb865e1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1111 +1112 diff --git a/summary.csv b/summary.csv index 8137697fe..76bf22e80 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73772,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419768,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419822,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.08642,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.7529,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ad5bdcc9c..5c48a29f0 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419768, + "Result":0.419822, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -169,7 +169,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.7529, + "Result":23.163, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -769,7 +769,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -799,7 +799,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index c2f75e12dad8097bef6233ee8c72c7e8f91e1db4..f15771dab42f349e2a97e775f6a57d4b7832f550 100644 GIT binary patch delta 3653 zcmYk9cQhMp|Hfkltvy;|#E4m{4W(5?YmcZ=vs$Bx(uUTk+OcPABt-4mmQpjQ)$vfo zN>RI|b~Q%v7$M>p{k`YB@BQz6&gWd$_5EJ=IrnR;3ac6)7S{BiWWO*r06-ZV%p!@N z*9XOJ?%S4?IYO*2m)&`gx&H0BjPIvNZ;iNb_nHN`RQa_TS&980omemF&mI|%y~l79 zHuT24W~%Qf27G_xC_qZkQISb3v3s(MFX1>99ape*Be=Z^a%KO!Nk&RQVT zg+RZlLy*)vr$MUk_O*M|wD0;B4-q2he(cDYXZFS9>v@O(p#h-L^?SNIYiZtXx$MJh z<4bTEdxetE?srvk{P7+wd6Ul2$DTn~?~ScWil)eP{L<5K94HQKSw#rKnLYdu2}W&pHnm-YH4Zk;{r>Q|XF08)y;Yf1>YMqj4c>;8+C+9R zPyX?)F2afsHwsl?vh3Z?=Thd*@HR?vV0FK{E@~zOeCc84*-A=Ff$*$J#H>1mw4C!R z?DV@Vp1kv;Bw;^AT1zAQz%E0!|J$;1L(#pYELRCGQE=MX^#xL;cd!4rXuPi z4>+{m%wM78>UQpgnMuq?w~KMptyM!@&r2#9>HhgQQJ3*nwUKUHBCro%r*FoW$*oe2z20MQ*1mC^5ayuUx3!DV zO2$nlE4G-GK-$f03f!=eSqfzQnc4JOX`iga<9cJJ+#ys_@*Hv*_AsWpd$KOGEtIklk11!(43-OuyE&#mY^4FaT`TjzsRoYXkhr22;RW3OuF4vOM{ z#U6*-$mkGUe{ud1NGYyq{8`I-<3P!ru}3Q~32yodm;`RqJm>jw{o%-*?4ht8*n&@~ za7AFNGigp(GF&3Tkk3fB*ieE;?<2jQ^(KuabPq0TpJUP^UY<{|?YO2NuE&$6fQd(NOZM_fwx5xA zM(l3CyQLSZ##QFn6Px-%Y`>(g>{jC9^t+~)wM$2th4w$)qFi!?;mjn_4+k4g?P{K` zL081u&g`MZ*ruwG>TWeh;5Qao!rf0>e_Ckt5skXe$N8B-wIO|m5PN;75ww?2q?vfGk zQzu&$u9b@ky9;o(`S+{!wrB?vJ{rXrn9#tC40VvUh4VxreIL`uu`JJIXJN|+8iVMw z;r+FP^Ua!<8rs=@>}fVF{wSS)R+KN-R=eV z2GEJ3Ev?Mpujz{7LQF&H$WTk*c{}^@?c(F58AfWmyXxg`^>Ig<=HO?mcF7gS6N^oz z1#)e?s{t4q6Nlhc?vW+{av%&S%u!plH zIRgJ6M-B~Y%k%Lip03FaIQS#L6lz=9%rq8!sSDB@IX%|T5T+l0W`^k7J zG5LQaLNkC9es3obnt&XBO}}THS~YU$J2zDmZ+i!skG(&`kU zYYE!CVPB9gjx*h&Ln{dRhy1?3s>#~Nup2g0UVL!>(F3QC@~(pE&IvaYE{uXPTd$6E zpz%odqc|kM#BIC>4JdlcD5jdoH(a6qOWr)ZPzn(zcao>i*?Wj*n{mZ(QFMT#HLUo< zbWbbWI~SR-t3Ocf*ny@Rd3TbGd0?4D>Xcfb9kp8RiK*2^W^Y7}ipX4%86IPL4x{8^7KFWLR%=3Ne*c`k+_D_y8HY)_1!`JHvf6c-Ty&U%Kp@tIlS^O1N|;f1rO9ja0wfgC5>S%q0}=w^lK z9zIe}U69j}_w?({DP*f%MYIXtz|rG4)f6rEjoyhWa_VZNt?JTX~W17(x zno-o)6SN(00wOpuWK;e(H{;`>Lh5O5bntU;4=c~~Am%w!0}bcV#Mz$&Rp>_tbD*9q z+#pl&rA@-o{UY6Yqy0EUy4(_&hhE-Z-;w~u7zb(le8In6q&1@Rv=L0(w8v)c(o(m? zeqrUUzC|}&w6pRfkz9&oX`6R)Y0&VCe*I;e2b_=i+rafA2Q7>1udF4gc4esnmK1S8 zr14H%G#3GcUZ%|#^Rypa(MxeD0bs0`Q<#?j15W)51`t36hxbKt$vSCl`O%9J#xPr) zyu>Q`Nlt0~ex5+RdmwL~!H9<|+rpZkr~I>bIZ9`mExW;Lv*lY{#IB^35+f>IVbtof zkoGWv(q%~f9?}#a971hX-5HEss6daZK!>Z7e&P8;?GZ$Ctx4SFL+DIMMSQKD+58gZ zxJ6L&!~r|%_e3?x09M8M@Ja!tIWdglau5|v>G*oQ`atIdkqesqwplW{Hde9-tLad` zZqMdbJmVawoF&tx0(!tZFbMCn^ZM`xaqJ6H8iQR^T(HW0y|qSdns&dGcpgW7Y||&u z?HzPt5@zf_yYI6HGdE#i#EQy->M!#baWIB|uzQUilT<)gnQ%8UcoNDZZdy6`@D59J zH-x2rQx80nH=Bq2v`VSSI`;Rs(3~#6NM>4jHN@Vl9Le+bJ4?A5`=Q<7h(Y?nC%kjs z(APrE0F_6Z5545nIA(Hde?~GE=%((aPXj%5YWVIGn+jhNi**dVJ#$^fab8i13$rMO z_hO557g$WuHJ|>_y_`ymH#7WfPxET^xG93Jvr2~=6SJ{oB*J( z?`&A)9%sFPR4@Ji{OU+DwsZnm9a#;!Vr;nV4R(Ja03ZOR~&|(T84^% zKq%^+J*ohGK<>fPuM+pimn#emiMr2@zdJ_mS145~4w_v!2<7uIfMAnvP=I>12AR5F zU(1*zRr*-$7pbKx(R&&`%EzOqZgsXA@gHs{Cs2n;j+I4MRkQX}#&3rN(K*Z9sWYE_ zbDROAh0_%0a5_gWzuT>c8^k*5%t`*%8Zgi<{}{;4?;g>~gjbEseK*S5=s%B4^k%ER zBIqxAvrA$#LrmP7cgEikO-MeSEP6lp_Q0j88?~?dP9f#ug;rt-rB7SSeaG5WvewGO zH|G{_XBmC6doaOy>xqs(X=-vE?btR`Kk{Z6eav}%CqMzPyA|Ol;mA%w021}JvROv| zr~vc_3!PsNACivuF#VyYF?^22zG+%l)6MFimQsr*P>Q*}wr%X1FWs#byHgdUBl6GN znik#>s{@WHicT=GN93cuGSvDRgU|vzc-~>^`nA;DcV9hhj;7_0G1;3Yu>1sJKQtyh&1B;erSiTf5VKM{MPnDjbIxN(@IN-;f{9 zEbh+tu>f1!N{f3wT~fr?0;rYo-eyZjPyam8=LO*Pi(L}pFCNdmBL7NMN4uM&fr31v zjjKQUo}hzPj!Kq{&mdH?eg%oaG!`J_J??-S%p1d^U;IPYU8QA87v-`;p9U027 z6^^SrHnvP~Op!#0QW%IY)HWzM7-vw@PI29y5HgE_)0Dxp_2mqsxm_2#baV3>UN?2Ci3VNQs_ zr&T$wY*t;k410EA9nb?lKKizHDB{xfn7(jKutASs`tys?1bUVo3zN^jtg%Z{)H^je zF|2uc^4Pmgli2SacaZs9Xs=IzSvYtkPILXMa)z5sem6r(w3vDdyU(nOsXz&Ef@h@WBk9jU zJl9)pB*4s)v4}?ql`{w!+0)%DImJd||I`5;O9?RekrM#b6cjt@pqf@!MQ{pagvfI4 z*cV;VmbB-{Ea=9*dv!OTa7->x%7SpleoaBO?zZ96D*{dk7)^zYlhS{w)7xM88Khd6 z?tfQg{h57spfVI&Bvb=WJ$c*#!SMUo5~s?jzrW6ZWD!bPBlfvfE~_nbe|#hz=r06~ zpQ{FY3;`_P&dPrmfv2au0u(N^K@yqH5q8KZnZBbmP_m9{GC`JxplRBm)~?BWp6!!D zq~%GOj0jb^|C{`9T}5vr_(hRr$K=A>tcPL>n^7I~yhk8PBz9fp0(Gi`jMwu&kA7N{cZx_PS@x1{ zmGVAyAHD`u#pg3D#U_XX*-4Kj<|Ew^R$*J5fe`4N?hz1$bv-jIS8{aGT^B&jKfaRM z5@c|=vPNWVUwFynSPt{ni!80$w-aGv_%4BqFUhi4DglJ2@S*fAK7Ub1ISWVTz4*Ha z*k}|&P4UFsBX*Q2YDEhd@s#v(c}jLK|0(KIoEdeU;rjmFe1Vt|TeshP-xBtH9YNe6 zN_V(fq|6H0piTp2PNMb7NSQ+fT~OwO-v+^lQY=*lVrv(N{w`D0jrJ0^sP?7rY8ri| zhE&|_dfQh@jAKVxY-u3WBd)c_ODICHOIwWT?bz(;K{c zN*V5}Nytk^R{hn!16ph0dMN1EA=f7A_zo{QE=-vWM4moSu@H*}24Xd>OX$d?=rdFvSy&LrcVa8SjY%oYzHc3T(sXV+$yVuaIOT-)K~0Pf_$+c!@HL8rseokn5xAOjQ%CI2|Zl*~EHHRgezb}YN;D_u_tn)P6 z=w{8h!{H}m_zTB|J%tPZO^e`9jnU>Wa23Z@hL8L*jB#*Lu#N{*$QnC5FLHM+kRY{A zoTl?Bb1P&AWo)ptdYm!$1I z-{`PJ$I$)Qxzcwlv>U~A#ZJ>P2yyG}7{Xtj`!=ARptP?l96e}h3p zUpj9OI0;rGWZgVTNWcjknQO%vMsG%{J;daQr+EEd24-nq_-LIE22tQfmR^;Y&l68c zm-9Rf4^PK#Oq*)&ucAdAJKmUEO1)`_w@8Imcy zO03m?h-g|{p%*1ps{8m%n;%I-r{>(tk77ug1`9XCG9NV_h#?#)b1DW6LdJ|T(ubnUrO$xN`90}Lx|V$NN?m6gx zR;eH>ShYN@=G<@-{SGtn>prk6YoqTF4DdaR3)kW%#g*l-EzBPhAgeK6AeuHXop_y> zWWC578s}{zCrTA2pT|*P@pc4JR>^t%3a}7u0BBOisY#~xb2A<1w{cs1qw1ukaIIF< z<K+)ecHCU+7z&|H+zS+bZuy*)TWv9Rjw>&j{}#Js=xn#; z-A~w@xMtruy!pwCGXaf;*G|9%zn#V$Svwuye+s?6O+E&d_551P zEA+{hZ0u@Kzpu`K+u5?Mqc+op&quFLI8wZ5Yuu`lYD}DTi+@PDsJ=C2ft$`u@vUE; zS^&yrB5_`Zq6QAbpD06%lsJMeW{L6o`U#W3-{R#;_3(ifA^ZEZUy;=5yF;gg+^ zkTaD-1&7#>ncQB(4ugAQpGWr;jL3Z~NuL8-hr3@a?IFJIRW!Ybx+PhaA}>_Z<8RuR#D17J)|=93ao$ zmk7tw;e-1@jrzgaW#HvvF0ksbP|N4L;d&3e*gws^H2Z) diff --git a/summary_results.json b/summary_results.json index c2d7f3616..25d7365fa 100644 --- a/summary_results.json +++ b/summary_results.json @@ -127,7 +127,7 @@ "a#": "", "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419768, + "Performance_Result": 0.419822, "Performance_Units": "Samples/s" }, { @@ -631,7 +631,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.7529, + "Performance_Result": 23.163, "Performance_Units": "Samples/s" }, { @@ -667,7 +667,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, From 0f5881d1b4f21f745a38dc79e45120ad3b0fd95c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 05:49:23 +0000 Subject: [PATCH 485/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10667 -> 10665 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index f365a5023..4930d73d0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 05:35:04,733 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 05:35:04,733 submission_checker1.py:3308 INFO] --- -[2025-04-23 05:35:04,733 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 05:49:18,592 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 05:49:18,592 submission_checker1.py:3308 INFO] --- +[2025-04-23 05:49:18,592 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 05:35:04,733 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 05:49:18,592 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 05:35:04,733 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 05:49:18,592 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 05:35:04,733 submission_checker1.py:3324 INFO] --- -[2025-04-23 05:35:04,733 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 05:35:04,733 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 05:35:04,733 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 05:35:04,733 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 05:35:04,733 submission_checker1.py:3345 INFO] --- -[2025-04-23 05:35:04,733 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 05:35:05,809 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 05:49:18,592 submission_checker1.py:3324 INFO] --- +[2025-04-23 05:49:18,592 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 05:49:18,592 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 05:49:18,592 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 05:49:18,592 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 05:49:18,592 submission_checker1.py:3345 INFO] --- +[2025-04-23 05:49:18,592 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 05:49:19,586 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73772 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419822 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index ecdb865e1..09e35abcf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1112 +1113 diff --git a/summary.csv b/summary.csv index 76bf22e80..3be5bd97e 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73772,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419822,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.08642,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 5c48a29f0..9078f0c24 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419822, + "Result":0.417637, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -169,7 +169,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index f15771dab42f349e2a97e775f6a57d4b7832f550..a872cfa359c1451780299042fcbc42bcc71f7b69 100644 GIT binary patch delta 3633 zcmY+Hc{J2-*v4n-1VhPjf$G5!mEq5+OgzQ6=%wVmR98;;F~J!hRK0Aphdb>xOsc1YR*e)d@62G zXua-Ol#GwW#YgxX%@#5VjiJde@MB)KA4=V&a(HpGp}|4p^uLV7%inzo4uJZq70w}I zoGx0_bEB6LMAdYlC;#|*d$?2VCGe)GXF@&;S~sygqn@)xwnHR((4-15%4;69^0=&E ztsCw8v+q2$ynMQ;<77tY@b$XDrndve2+;&iN*tyl`)jrDRHtOlzv_sc<+acp^*+mI z^YltL-2@{YEk9jb;;)v8cVoKR4x9V8PXU%8k^TaBksc8N=6tNUI<0`C4*cj}wkp_}HKX+=|Wv>mP33uE-n|Z$f)^ipt-A9xfL*8CABCO5gJym+^rUi4Zg2hRUBz!X_y2`cEYq6OIdwZ( zp&`d=HYmQFZ)ka{@onSKFM{w>*N);1oAx`y`6=5{P!w!5F#?txC^nW=)>GtTz-nf3 z@`PLVx;VNCP^HZFx0~3y1r#33EoR+x{3dWB|K)N)>~%JA>MOv84$CEnN=+i3|t3f?B6Hv+u4FtiKPKz4pk2G8 z6-Qpxa*oV=L3Z1~yG++Bl)O@y3;odJeX_AQ_=48-AfPm-I^9lK>+%2TIYHfY)kTVa zOE)3;Zec64v9#ddp7;iFSattSAuT9K*khjp^JCl{5Km^`tJ5k>32>T^@GLn=lkZSQw#5wY zb9Sc!(!Z60S30J-_=dITj;!tCOl_nZBFvvjkKoN)jNt+P<$I6k6nZW*j=4___MRy!PIMo5VETEM4OPJCTH+xFz zp=2ydz|`g{g{3MhHB!rfYk zf7=r&z7->v)n$E$#1v9VVt@mgD%!Ew%XW;lFFLhSFG!fUL6@#a&97(=aZ@`eS>X5B zXM$W0T0}-g`y;$*jYPzG+UB+gH&9;JSjKij@fnLH`UEMgYb^FCcksLwi zc@^%aG!X;ACX&N(P3_}PtHc%XCGaC{RrM;54A_D^dgMC??aVKNd$fMqV=0qIE&LP> z>R6`b?q4Y0aenK;9kBW_$BH3INx!&x&u#ZVgA$Jtu{{e#pIYx$pG!({8E!{*Rl5FidkF8=jcts zX}{A!r8U0ejot2}R0W`J;dA2L_D=5Erh9)l_|bP*e@{E8*hkb2cGH0>PKU+rU60&8y6`li{RX>C=p64%wI0O2gFv4LS_qFHxg z;}>_`3w=Y_lTl{$(@))EWZ1=VM+j{Oh!}VF=x8hm&BT=nPl&Dsd1hLOv!8>UinUjY zfsg7Cj;5mrR28%CabKL~=`m1?7;K)qB}Y8_DZJ{}jy=Tp^_e+SQZ$Ceiz&_gu-PXQr5h{=3_Cx*>>=+@PTaB>)^C-YN|$#((AwmRGUd1Zat zi|dz{KQ_;&$Ap}d&F6RKv)a{#oj}b-W0JbKyCB`5REtLNIrKQh5Byd&18Ekfc`}fR z8RgZ)gS}|;%pTFfSy4I$P>Nr?Ie^JS2KVk4ZFN9R8XI?qbklAgNs~}G1B*9K6B~dkrdzQLe-7DQeqd#j6Z5cx50hiBazf%&{u+u zx>_;VxRqGzx%azPYDDNL8 zs-^Rf%mM)bCF1MqK(2T+q}GT8kg1&K7)|8HIV*})1It=kQ}tlO6Z0!59WO?bg=`NL zcS%w1B~qUCIxb$RQZ*J?3T_=(*k|riD}NJo;7CK2^{^CnrmL#3Ep;lR^^q$ry}Rmz zMdfq(l8_J3ax8m}ZEf5K^N)t%?f78`>B+#Z1R%0d6}9;V(JD;>rNft_)j-!Xv!T4b z%mz_1OY@fwHxphPwNc;`94S6eRpO(1XJFnWtd{*NxF;7FRrRXWRv$htm6H;Qf~O2X zNM|wx3D7h!!jz*^Rg>A@88U(HX&pI(GSrg5L{Sa80;QacX#Hw($m!^B2KOR?f6W{7 zqb^HnlTCyEM>3aQgbvUoGOV?j&^_$}Bkei9Ged(}qk;p3##0K~c9xRiGWQKPoP7PSv-zhQ-m&V{A3&~wfoi;QH5m4zY3JM`sI})RcYVE zz35B`2}}rw@5-}!B5rV&eano84)aua@kgq1A({LAg-ry~6;^Z94*i@n4P|EKo*p<2wqu{^JVNPDG+g{0kn z*ZtPy7~T{8Y%9^sym9E*?N@U`PILAc+yiO?fm1%$XyYE$?fd-;Dh~*I28jH-0E%O* zbcRuJ7>X4IYQ;EohaU|M{*y zLt0|=+c>Ap0oU1pD`$#x?n+gs)n6U^G9LaTafqAMnYuNg$OrgdK}DQpBr(lJGtiI0 zux~Ik3?QBRndpZAn1Tr;dSx4($(LzpVX9)vrl-gmhgn--^|1;di<`wK=~yrS>ayEI z?Sd!aUh5GuUsYWu+ka6oICT!w$cEw6iGmd+?s^*{bY!}ia=dTP-2KaK3OH|{v@51p zMOiIOX4EtM`PO&nA%21hyvEnMO(#2n=H7X-I~|*YH0Z!&CCsMx?>ZTYZgglk#8;QE z{D>=8bFkd?AY*psy{)^)cKWdNdB|gE(>#>phZXX|KdS_?vF_%cr*7j_r4ER%y@$*O z13kH>G}}hDGVE z%7>3TwnN6A)c}e3LNB>{9d{zuv+Gsv)K7M+@3OBvu@mgV2^i(Q{HgCJ_wPsP9H)S$ zm0;0(JL`N_jhfPRGj}eskjr~@*H566>`8Z?WN@!FU0}OK3_eqA9%aCOyWl-RHzgQ| zVn1aQD>z%dfh0DyuE0Kf~t z0;Ry{<-EH-3%cN#tpn?_GJB}y)v~(}GdI4yko5T+;iVe;?S3;ahthcshBJizPY!3U z=*}G*j(@;#;x~21J!h-$%K3kP`7cW@c?YF*ZL=5!i(h>!H>Pw~R}s=xf+w%w_U0*l zVxLgZHAi)@@nV4Q^buJ6y~7~YXXnO!YU+1gwAmx1U_W+r+#~yP(#<@iKmP#8;O2ep z-St$jwp`Yc^@(MKq^)ep7uS1=Ies{|mb@uP*b|Sy>-Wdk#DtP1JAUb?+7A>5w5%cd z5KL}GSM>;dx!958a?27$yDJMcpe=pi173rUB}%Bz+cIiAfABh4g4-Y>wCreF}JZY zV>Y9uw|p~g$>f2`sq))dT_h?_kNYZ> zR3-oUDlC|rC%Z`K@>aW5$Gb#$p)#(5zA4wyw%62l1Kv2;IRE?Om!6f>g7#JgVyRE& z^EN~qR(uQ9!8G;9tGWm)NZ2e?giEn@J6=ehKg-=H#)j4Y?!2TH@Bg)jiEBGCH5tma zE*iaN7u<5ex3JUqsz}oAkCOO_eM$ss3*(3Jpc~6SJH}ITS(^x2eOOXeQ$x z$%c2y{}>CZj6P&jf4gvvoU7fr8)_oD5Y;Zs30$v+I$so%H_-m`Z-O?%?P>$_keVvvG%>eh0#^?VNXYcg@e6tfZQDY3Tk+aidUpi=L)KWE!G zfe*FQ?%Uo&swZVkCCRmzlt9}}tO{JP&^a=6;8ioQ)|?yDd% zeIGB4ni-e6iQ79#7no=r@G2jWDDkNHJa^1*`g3scdq7W~f2~?~oFr8?{{h1XF;2W- zy#A)XcH1q7fb)~7-rK9kXoOC{#lCdu-}3v!uMSZhM33#LeN{ZPoKx$gc&zvY)~H%j z;)!Q9QwLdYz-*t*Wpr#9p}Vy37%U&#H1WJ;qj8{Q-q5WTl!!2X4N8Qxsa^2+wDD+k zUg}6d2V%w}U%1M<-I+KqAQmPXug_zkU92z4rSpkS$NaKcrPP))`bP0D>%zFbuwr>L z=r!lHAR{)T&6&#-G<(7+FabjOr_Ob1y56X<1o-f(#sx+l!qtU%>y8_`VLDu?vY0p| zr&uqKSo>KSN95kl``bDp${c0(JuxXSg%3*V%5Ent&Ae}VRl9tgS!nyyCDJKZ0Kr5Q z`gpkM(5~#^9C!^a+;(^{1I?AHksOVyv$TOat1d54F{W z8NhmZ1ly;hdDC%B@hD67A3q{*UMUhsmh>d5sYwyBuznJH#yl&;2LQ158rs0t_SwwX`xphSKCj_!)=OP$A}^i#E0sJH?dcSq5smtJ2kOl?i*A=Ah?m zHc1tRlS@s;1=4NYYyKD-GXu%3&?A8d3_i9&SKprAvsf#>HL znqJ{-D7`C&r7)f^;?2Ei8g=OZsKCEe4utKvByl29%TI<=iAnz>5ta^`^nEvpR0HIkSMz<2W>>F~zI#h4;f`0J>G)ezjk_~*CSn!R zwPMkTqq#X=x3*rJH8AkZcPhG&3$0hgyI>d|iwt)AlfqJL+cEFNr-3zqg}J16X$1ik zD~yF0W!O(s(&ly-GLPw#SOaGcNJQsvMXwZr)!&sJ+VnOWZRPU_Kz73sJlgisXi~}x zc{vD1Q|Yn4x-7jrYyz~If2Ua;$K+UVcwlDU)f|?Q@`6qjcxTV{d?6kuq*3owy~?Q= zZS`m4j!+qK*Cc>Z#-qEwoh#FxBz~v((t_a}O`bG4g~i-fi{!*th;dWaK&9`~v1d!VB|8${g3v1>ov+pz;pH8QS5Nz;Hb(UfWB@H;d0s_1HM^9hMq^Ai09 z!;@Gf-P%bCiv}LXWtfp_ig^KhEmBcIhA1UO)xC?X0#TUkZUw#?>5r^azeF!(z+83w zD`u~N4vUV>pXgyAI+h=7Ks*}8W>u_9&C^Zex#BM^LojDb$yzXd6e}qf6=wKm6OeAX z!_LoGR_Tpa`~V+9bB%N`$b6xQplZTV_c}mU%Ij1zh34$xuE|y+oCIInX|UKCG$(Rh zHWTrBP11seE28y5&6Q5n8kVO9u>8(CL$Z?~Aj5K&u=#~a_RFzIWZ|WAryZ)2C4%jz z+Rq3uXP}#9XL@*uJ#~Q&$6hmUwx&_7HWg7uI6Zr`8zse9_&sd_EJcId>2p_aQ=HCvO2CbJ_mV@j>Ku@K4sG<~nhzq01oK;9pfqm~8H>1(e zy@UgiUDG_sJ>O9GKGs}JPUf_{j(kBAlG=kzI=#n_j0?p9~u;3x=f!JQ&roW7Ev z3W?ze$1!)BI6tL=x*|2xtyROVAo$cfx9lM0tVCCIhATWfI41d9sJ=wXG&^26`x*|C z0aVPw(Z+RyyYtY-JsNhTXc+adaSt;PW7FpAY`c$!S?|Ka>q?miAAnL67A zs_Rac`Ur(KXbWvn_wlD_8_*<_Z*thG{I73@l;J|^sc%5Ug?C4lXM2zf?C6vL{RP6@ zPrMTBlbtC@M+%{rDfh}M{`f(W_JYAdEHX`c8Nx*;W2IDy5mNs5n zic)RLQvA)yB77*r-B@%K2Ohja%NO&kA5zgvb}9j2ELV~lSN%JZ`&)?_BpKUhphA7XKZ*vg3 z5?4zMsKCO=wH1Di5j?p|pZYzxDK03O+N!jR9*kM67*m9eR44wzoe!}^5=_;nGOixM zW`ir@YHdswmZ6juKD84&?3nLUr9?e=75k%W1?1+0P_oluWDvPyh_d!j>jaq#p8B>` zGPOQlvIMW`P`PQ#;#oZF7@&|P*`)}6$UQKK=(F+s_!dd=0n3lWZ^$iLtG=Zn}F!#>))5ywu6384obbzI?y z+6p;a(keGqWyIM??}0B5zhzygdis9xIPXPjf+@KB+ezvAPqN1oEkY_qwwKO-0lv{UxeW Date: Wed, 23 Apr 2025 05:59:37 +0000 Subject: [PATCH 486/719] Updated results summary [skip ci] --- README.md | 88 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 8 ++-- summary.json | 12 +++--- summary.xlsx | Bin 10665 -> 10665 bytes summary_results.json | 12 +++--- 6 files changed, 61 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 4930d73d0..9e7aeff63 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 05:49:18,592 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 05:49:18,592 submission_checker1.py:3308 INFO] --- -[2025-04-23 05:49:18,592 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 05:59:32,469 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 05:59:32,469 submission_checker1.py:3308 INFO] --- +[2025-04-23 05:59:32,469 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 05:49:18,592 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 05:59:32,469 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 05:49:18,592 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 05:59:32,469 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 05:49:18,592 submission_checker1.py:3324 INFO] --- -[2025-04-23 05:49:18,592 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 05:49:18,592 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 05:49:18,592 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 05:49:18,592 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 05:49:18,592 submission_checker1.py:3345 INFO] --- -[2025-04-23 05:49:18,592 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 05:49:19,586 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 05:59:32,469 submission_checker1.py:3324 INFO] --- +[2025-04-23 05:59:32,469 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 05:59:32,469 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 05:59:32,469 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 05:59:32,469 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 05:59:32,469 submission_checker1.py:3345 INFO] --- +[2025-04-23 05:59:32,469 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 05:59:33,492 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73772 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07639 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.314499 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7859 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07639 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 09e35abcf..2b4ff5662 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1113 +1114 diff --git a/summary.csv b/summary.csv index 3be5bd97e..a522abbb7 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73772,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.08642,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -18,12 +18,12 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.314499,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7859,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0292,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 9078f0c24..45ce31aa1 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73772, + "Result":1.75529, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -139,7 +139,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -574,7 +574,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.314499, + "Result":0.38615, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -729,7 +729,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.7859, + "Result":21.0292, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a872cfa359c1451780299042fcbc42bcc71f7b69..c9246c77ad7a8338c53170d4dfcd598aac9c9c30 100644 GIT binary patch delta 3514 zcmYLMc|6qL-yIa`hsm0$7&ODk775wL*ast$ge1O0GgO(qQTA={8$HkKdF~(g&vVatpZB@%d&_LfY^wZ@@trBRXb#~4fmGZi&}F*5HhJurE})fBT-rSY##-{JvrvoM`rxLlA=5Se;?? zJrJO^M`g@5nh5@eVnPV2%l&D?j%BTO^LkApa_xqQYdg5fy+R^514scsD@$%lZ@5?7 zdn3f6{ok!i#MsWg$A)cU8xe*_s#-{~wa`GcR79yAOo|YMk(z|8MH{v`jZ5+`Z#msE zj1N|q;=lX*KHBhWqmBUo=91}DY_Z>kF{ZwdBHUDg0=`pyXQooyx@=~Z@(UvHM+ zum0wYHmul$8>a#AC8pgbWJP__LnVqpdaIlku;kM?UZ99IXp=@5&viW!7oqCRD*Uw5 zZBl}|>uLt?jnkZ8JW+3ohuU=(s3O5*3ALj$rF4nRx>7Q;7avBa0$H8${p$p^xRcvN zh31|~)ke!lc_mXXHJC+10p7Gbx8pT#iq$gO@z2H*fgbhsc#UqbGCugb;VUIN#MU`AsPt|7p{?{`d_JB4z8=3Rc1lzj*JPAJ2N!Q3@%meV% zpx3#N9H1%=Nh>;ySo76O0lI!Nxt_hslkqJgy~^xgQ)8!-3}FEq!K?#%3((bFAbz2~XXd9(a*T%^qFY&nKY&O>2p;5AeT7 z7QA$j1&@EfxSr!3sIY*my2>bf=PG7i_HL33h%u7E1v30D#j%+v>j||B&RsGNS;lH~ z9?LhC5-<`-MUY|*mCxs!cERJKSNJGr;~oHPMipE_97L%ye#db~6@@_WMxK2g1id=# zI3kQ^qt0lwzuCs`4DY3Q$r=PAD2d-$`*e>oyF4itn`#J)7qzB{{D9X?lm*jfClBt=Z{>aKCNY@pR~1Ij2VuUIO$) z>$FGxp8M`XOReqCblNsfSZjCnqNPvRXv#n&NNzBa{Mk$x#EqDA)5{>ij1k#3@Y&#Y z@RW!JjyQ~1alL%9E2zlgW@relfii?p3=r`ZCFuT2IGWX}kvn^tYIMqZyzE+vPtrIK zPC>3hZ*I4{jUM~MoaU@Uc|d*90w8S&O=JG>j12DSbbS))JeuGK*)m5 z5ssfLD4Y)FIoT&rZ9NUAE2gdEGQVC#AFJ!mIeJDaLo3)FRZ{0L&yAy3cbvn61v-9G*>sN_$g*bo`#D%~f5`6Q7A*_(UMgl++ED0;-#17?x_0&uEldyzWR#L9Zqk= zNSnoknVNFsUw$g@q;2pMDk&bW4;RmH%ibob@b?F>gb1{U0WywrH&rBB< ziM!Spa-Fje`;0dqeP^3DKMKux2ttxu(TJLZ(@;Zy%Im~1-$)V@y zOP1Q72>~p%T_?=lw?F{pqMYuR>b1P6nFn`}zdtu`PT>NEni-uc^6v#g1ukbHVS=XX zG1wEd<11hqh+$g?XT zXG%ZMM4P|I-dA@L!^R6%9CzH|zAFBVtLVOw#Fv$^fs>CIlYrHoHB;@fgA@zII+VnQ zus7DHstA~otSLg+SLg$mLQ@3m6xCs5pFls}Oc~r2RKb%ElYH~5(4!96$Y5&ydg~x^ z{iu19+ySn!`2I$H=s8>N^tlwmz@B8LRRa)`t$eo z%neJquSNhn23->%T0=BPMWy}T_HLQB)$iGyZ|h-kK)*2V^>lhM9e(2pY7)I1gVmSa zPvP;iM=@O>6C`X1v6#B+A_k&JFCI2KC4VmkWtGk@tuO4>3Bea&U>CfGW zwm0mXJoX3*a(~V^eJFPN0Pi}Rn-*p&oi+e~sS=Nbi=Y%XMKtFd)l0| zA1)P{pTy;+j+e7q~U(tbNIf!5O2S?-pa&R5gelT3g9j=RgU#*#Y|kxUraX6e;r z>BV024ZF_y5eAVMEMivig^f3llC1|7q?vKp%&H*&Ww2mFy}&N zI;)%d6dKp2DEY+sSebMviMx2kKqc1w=uC{;-wa|pzzd*0Rc_>|C@7BjQ35fAvQMVG zFR@2$^Sq7MF2M4EQxY5T+n~#Luwrs|E-N(ex51Bs<4A{)lH|o0t-OTlWOw3n zA=PBVbc*Ex3yLWIS|?{uxLU~{*wwYz|^QKyMOb%@i))+S$W=QHD*)A z(33eA4)csL6s&+=i5_(qUN9N*+zWxKg1_wbf7rWQ;WXos*DbKgLRw2*Vga$54uY+X z*llIqHPm-RdeMFr9}8}$5Vdz`0DG4F|3tYBDzTcdtsm_G~ zUrJ6CAS#VPSpi4`OpiUqLeOxU>A9a#(@FU3X;$i3;|JBrx&4>dcG*X#L$QGwa7m3% z3@(|2W3oP1CCYNwjNKJH%Kf5WZN7z;3v)J1&lEdJ?GHaq*}B4%gt(zsW1>z_kN;o= z8X(zzf{tSMtE@q2^7cQG+Z+n~*(FS&mBTp$k+A=60Q`!)03b-;$tl`aKWDqXrH94Q z>^0N8QNgov$|cK_(z?kq2i5g) zs8jOIkaDWu05V&;6>3Lz=;;6yr=Wl+b3*Lk{GYI$`U2q2W3`pyag)_A&y8inZVhT` zWp4URdINVa$7UB7#n3he=5X9;uY|Lu# z&Zp74snuYD5POR95_%hvo2xE&br#b$=I=pkc|=AZ29C!1d8n*xgDX&_JbrVefn=P*+3vGb`Xdk O^#3dTK_?j{i~BEmTDiFZ delta 3514 zcmY+H`#;lv*vB{LQ!Sz#%EF9tSVnTngqXt+C5QC&)#f;d5(@bg6++k?LOEoS$T@Pz zp{yAp=6uMx$f->ZbC`R5f4J|*{r&@9*Y&!d*Xw#b-bHprcICJE4hL-KZG{PeKN9`b?0L3B2?;Z25~XRrrZ^+3Y`6;oFNV zce6Ente(v7Rk-OVT+-9=)wiSkY#w_%qOa??v486XU=lGXE{z3;Z}1ah@@oNs*)cMc1j`o%Jk%Q3yRx!S!<+;r*v4X0bB zHKz0Gw{t>64p**B-?ZuijrV-GfPxrQ&+PV1`9L_1?+;I9TdNl9(Vt(vaB3$1UFw`$9(A;$Y z)t8|kNY@rCoBj!t&E5(W5ms2{j^%R&9_u`za&GQ#vd z7gyuBDyifctp*nnVUJo<*_q`gdEr~Z2s7gs6O-t+S%g+%^OT-vODaMuetSkjj}oKu zV>i;cEu1Z@HkmGdIStTamaVGK`L_*`1nIK(ZYr*gqk@|aUD`Q?rE(YYl;T1iU_&oz zNiIxAQ@A*DVR-{U03V-z|Dzgl-8Z@~9Cb&%+b=!GDGEE9ZN)$pGH)mxcu6BnhGs_; zZ_Me|Z0p3)mUUbrGM~}h)`>SJ>r{$gs?6Md-{pP0p(yyg4lvn|n9-PQBdv7#{qP)P zY`E&9B)_JcQt_L(iWfL$@Gnndy)>+$>Yf}cON$~4h<*Y zrVnjN8QtRrI#cC;A%d64lYGL1x-nlexAP)Pm!`;z=tTdzWnxFE~d9C6USlgD1>&U?RL@T6igkVoeCd&8yspWG`E6DAw zTnEaPGr8Mp!_LQiO?R!R!5VYQPugGQE@+#C)4jDQeC23a!X=SpWjABTSB2N|z$W&H z$sO@ux_JikhSD((klUy;aiJu1(?n>TglQANzr^h~3WY89lA67zEvEJd!e}}L*TA2O z19*W?`7|?8cLsCI2 zV^r?cK>S)C$q6rAy0N@y=$M#7El%`zq%kDP^W9vBIeVg0D+~g~DeHUkb?Dh8-2njx znVtoHhkGK%cfVO;Sh6?Vo7q4?o@H)q0UiQnwGE|QN7WxT+MpO0N7M^Fs`v}^l_!M3b7c@-&$Fx{?Ldk(4$Maz2DyA0=P@(hXam2 zE@bJeYE;WHsdVpr(YDJQ4*~z>581|S`7}>Ao?e)Q0TBzrd_AJ^o*y&Fk?RF`=2hMQ z4aVGqxqUp9Sy^J*F~yWDC5mCk{Uf*K`zYDad==z1hmEIiDP}A?vRjoL6Gn={nMOL^W#@ z^%S!~I_Y~du%sG4+R*7f3@}t`=RPIQY;EV9ZVX!G5y9N%{GR;(TRnL+vxM7#_RY{^ zcZxHUfo==EsDWcoeY(;t^9zo~L>W{))3$fge7TQP52*g92A}3|THqY#`4qexJU~9{ z`9)1Lo?)1TE3rtUrNGCH_#T9pbXvee!(Hv9fmjEGE%`6RwARX4nd_PpfOwYmNZ+6u z#k{ki;j_E``JMsX@ksN%laJkE6u6}c{G?VRWQ;p^bTkfxVdKh#Cq&nPJTon&xz9jO zu;pUl!+Ip4bWERyYF3cIg-O9KqjTZ?O*1!@D5pP$RsP&|fZ$)9nlYnBV>!G4W|B~) zW9$0A?Dpz31Gd`ccD@~ZfFZ8tli~x}%=-jmxNHV)UVQ+kg@U>=KW3YL6Lo5>wY!&F z+M_$ac6sqb(`aTxfaRsgC#Y(bN z?1B<5swzE4DRW*Y#3L#+Vo@dFmcF@t_Aa%EG(!9M>nm*r<#1CSl?AP-6B#WJU76{f zmG3Ppo+=lIyoZ+IxU=nQ;@(?)Fb-=Y4nnAp`hXo7iMh(ijnBvyc`7I!z8HNDbUpJ0 z^iVguLA1ic?8U#E39pG<&-V$Ake+2Agc;r$*w+cmrGN5+lma3vUzXSz!bjz@QzFpt zls*XcRE8J{ng&Li@w97bv->+mBkg(GL`^^}Kv2Y= zruEt38w+XUjs1T7nG4VE_AzBLY;@SrJ?#S`9C*I5j|N+#oCky<((_xl*(4UZeD(g~ zcF<4~N7cK}{&emQCp^2>2~$$Z4C_do=(BRTU1%I8k0n6nk~`QwZJ9C>u&PMLXQTl; zG8yn$woPp2Bj_RE(|dEeFSx>`n`U?UGK`QX&hvuVn5(#zS?H5vRZ@& z^Q*Mi9veae8^YhZ%AB6ae|Ss3X2wGY1sUo! zU%~6+jKXri_oRN`h+tf$RR;SZ*YX`-?RzpL0DaqCwp;K2+}^u2ja82sl{k9fRBCY7B!N2sd^hZbt2Kn#yWgGAr8u z%UxZDyv*>|QC@`uzSI6!P8DU}k*i9pyE^iDH0*ogfB>fpW3x|H7{Fgaho5F8vBSkQ zGK|4;uSWwo5cSNDL|-JpHcX=KRkZH0`LY8o&QML+@RT^^IBf?kN2vpfglS@up3UOV z4*Om74tN~ywHB`MMdQYJ+fO={pvhwvQ9qbEmcN7$sI#R&hbD{YM|yTG+&|xEse6{@zK6?38IiGB3DfDlJI8Ia=*_(6HOH-Zq^h+da6wSx6MLaTdz+-5PcNk97jgM1Nz~Q@`OVf(*es zcqpv1Fyku-zb{qQYO$tF>pgE zW=>dUrO%|`o7d)-0UHZWE~1R0WR-?k-eK-Hws)?>iKB8#fRph%SSl#!k~}QZU|BhA z)Ts?J@}wF_Bo=rn1(9!uuf3>Kzg;)psky_w^vGVUgCKe-_xTS)U!_0aOJ;aQwXMYp z-`U&bacWhUteLy>(T?4Cr|J3;beucs_M;4emB#a2Hz~oViYy|HxNqgZBk89E#~s2= z-VxNYriCfD+5uJ<$QRCP9u-bNcf(1{iWSh5kddHBN{E&PGJG?%yl(<8o+4N%%5t45 z*nnBBemqL7SgI!}@%XgoO!sVbj+J{WUjfeWyiwJb`^vE~Iw;?y{At?h*PhOcyK)au z(vcCfzSrl5dG!Rq?P{xLOvUW%EU~xi8ab3jVb2JQHmGAfkd*R4tSn$8ZPzUG{GQ{4F?D$ Q4En$K;NchI#nYbu1Fap!4FCWD diff --git a/summary_results.json b/summary_results.json index 39934fbf7..5c316da5c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -437,7 +437,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.314499, + "Performance_Result": 0.38615, "Performance_Units": "Samples/s" }, { @@ -595,7 +595,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73772, + "Performance_Result": 1.75529, "Performance_Units": "Samples/s" }, { @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.7859, + "Performance_Result": 21.0292, "Performance_Units": "Samples/s" } ] \ No newline at end of file From d7b3857e06edbb7ff2c1fd2fa2af9446130bd844 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 06:05:30 +0000 Subject: [PATCH 487/719] Updated results summary [skip ci] --- README.md | 88 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 548 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10665 -> 10668 bytes summary_results.json | 4 +- 6 files changed, 345 insertions(+), 345 deletions(-) diff --git a/README.md b/README.md index 9e7aeff63..6de0343ae 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 05:59:32,469 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 05:59:32,469 submission_checker1.py:3308 INFO] --- -[2025-04-23 05:59:32,469 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 06:05:24,845 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 06:05:24,845 submission_checker1.py:3308 INFO] --- +[2025-04-23 06:05:24,845 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 05:59:32,469 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 06:05:24,845 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 05:59:32,469 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 06:05:24,846 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 05:59:32,469 submission_checker1.py:3324 INFO] --- -[2025-04-23 05:59:32,469 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 05:59:32,469 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 05:59:32,469 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 05:59:32,469 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 05:59:32,469 submission_checker1.py:3345 INFO] --- -[2025-04-23 05:59:32,469 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 05:59:33,492 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 06:05:24,846 submission_checker1.py:3324 INFO] --- +[2025-04-23 06:05:24,846 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 06:05:24,846 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 06:05:24,846 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 06:05:24,846 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 06:05:24,846 submission_checker1.py:3345 INFO] --- +[2025-04-23 06:05:24,846 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 06:05:25,855 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.08642 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07639 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07639 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31404 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 2b4ff5662..1b6d75c84 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1114 +1115 diff --git a/summary.csv b/summary.csv index a522abbb7..4954db42f 100644 --- a/summary.csv +++ b/summary.csv @@ -1,30 +1,30 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.07639,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.08642,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.07639,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0292,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31404,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 45ce31aa1..a2e15ca39 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.425121, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":1.75529, - "Accuracy":"acc: 80.000", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.417637, - "Accuracy":"mAP: 49.593", + "Result":8.07639, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.08642, - "Accuracy":"F1: 80.0", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -213,60 +213,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.405224, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.418257, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "9":{ @@ -275,11 +275,11 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":23.163, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -287,8 +287,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -306,18 +306,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.0024, - "Accuracy":"acc: 76.000", + "Result":0.440114, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -336,18 +336,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.440114, - "Accuracy":"mAP: 76.951", + "Result":23.0024, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.07639, - "Accuracy":"F1: 80.0", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":23.2491, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -484,22 +484,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -514,22 +514,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":9.63051, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -545,20 +545,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":9.63051, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.6529, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -587,8 +587,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":21.6529, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":21.0292, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -677,8 +677,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.425121, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -724,21 +724,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.0292, - "Accuracy":"acc: 76.000", + "Result":0.417637, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -753,60 +753,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":23.163, - "Accuracy":"acc: 76.000", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.418257, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "27":{ @@ -814,29 +814,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":1.75529, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,22 +844,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":6.31404, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index c9246c77ad7a8338c53170d4dfcd598aac9c9c30..409b4aef5cd7717d31ce3dc21b5504404160c554 100644 GIT binary patch delta 3519 zcmYM1X*kqf+{b6K7ZJ%V`!a*Fj)_Rtu?)%9LQ=9tw#Xj-wrqtle@4chY$wQ}M~hPk|@TThG=zHa)d=WA=97gwuUKbF-E-ddNHeBnjan+zd5s!Bu_#0H;plJ6&iRQb0m zH*MTSGi(Y5KG7Jf4R;y_E{?bZg{Ttn-9SdPQpq$uYd253e7Gvtd;$~my0~kOppo8{ z_!$Ybq#Lwl!nGTH!WwlVDeA@WMFTM+I_&0xm%A1{c$`#_F#y^{6@-6nDtcsQ7i+F9 z)_ED*MV!*suJ-3Du%8R?&8-akY-AOxeDxe-h8B9#-*>#gZd5Z`!-DI!LLdE}`)f%V zC*C!4GYQ|KNtMp#jmBo366gC9cSENJdq#llsr%u#H_Q|_|b#&yf_zQJv5E+RkPi8eM91Ap9i%0D;d&~uS!m?ZD#-m?rg_M$mHDu-) zD^&81xsa5kbu<#Td1d^+)60e}Cr-adNnhFao@SP(@ju4!Kda7a$<+<191xGb@cte^ z*GWt5W)`Bz5}OGq_Lpr~i`xckSQNWWodxRmIvz#V{K=P=`nJ_dcAECKJ@v#^fQ9Vz zx8>~Artlqk*_Fx0{mDx3*w#v*{PdbaVPDBz)sXK!3gTvZH11+|@`=S5BU!hWHf*Zn zs?{S!tFMf@70uZ|@~8(lO+L~cP+a{s71n0r(vys_wtX}Rn#L$&?|*b2s*8@gssAB5 zdULxiukFTzz-TfTmi# zjt5D%_uevArGMsYWKU59;e2s4f4`^4CR<6N`?P948DSI8L04?fZ=Dv|HCn*MFi|Y&f7Xou<&KV+Ju5 z%vQvvqOwu)>8(&xdsRDRPWUTvyLCQAxAUbxt&XEJzHEQP`cQy-6P0_bMz1m}FB#Ix z+Zt8IC6+@iGs~g3H4IO1nC|D@X641P@|3Zu9>?T*1?5@}^Qse~7(?B*Q+({FLhudnUNs*UxWU1}orv>w&JD6FSEC||BjrPjEl+M(}b9Pv>DTsT)Oi7-RGF{BG(5eC+TrjemrA}sa4900ElR6 z?|p+4Synr8gueA;KlJ*&$%}ql3q!9YuI9d^Od(>beivDEgMMyZjj)2pMp|gjFY=v7nhc&RgYPA;JMAxTAM!C39>*y!KOjF^+tt|zl3is$$$L4 zzovI2&LQ|ebJ`ZYL@9dlD-D3*XZh%vC?U;MI0b^2XY+PTE*;Y-qeMZFeCI>Lwe?Gt zoQ<^(KT+!2uHbF79y$|HP_)N}Ty%7=^d}PA{I?~RbNvfoSJI99+j-;*9@8W!a4(EC z7P^n=af9rS)hvB>(5%c01`=%Wt1884nguyPq#ji%mT(A%&$?&YB=ny*T<) zJR#Oq;}8H3|D3W3@y7c}!CVTDc=J#czaKNaTyhzj|6u`&@EwVglyPw6m^wNa|4e^1 zr;*FEo2GjsE&ShI!Bcj^L6s{eh}QKO3(o3V*WH)Awv<}2(jgD#pk$mSNH4Fz-8-;I zcjmm}B8VVkMfKx6`_pYgm0J{UMV5{aoT~>`nIR+xQr+e&UdxxL5_0Z5KZS13lo2Hp z5_UTgaWK$#whgd_@11YGQ)+>`}q^%L%jYsC)R)Ar8dmKl<1r#GaqJQaUU5D*y9XQAk1 zW{gJkViRh$ei`^3=c%kte8IWHbev-21 zK2Bkmg7zc4Um1I)!o8LWm-cPM+Nd38nDtGH`k0Ya=FaiJrt6D2kxZw~%H6M^nIPhku@;=v z17S-k(GpDvUqtK|#r7&RohrR+Xw_52qk7PuOL)|g1Q(*$%#Vq&=-&8E!w`HN>+won!=a zBJwdz=In=YE`>dArNBbD>f&`XFBRXK;N<$PW~mxs`kUFgN*CF`U!Gzjzzrc;K&GH- z_xReu?WLXmGXa(oH33Cz2Z+(L{q>_))zW}5r|g0W(z0kcax+ciAx@hbi7bL=+$x-O zDd=@soWF1U;uEdA|LPq*GLbgT@Wujvf4IJ39KwVk+I-}JfYxi|^ZI)(&XKn}X`k<$ zJTp}H@Fz($#>pegw)KmbJFxEA%&^bya??r61J@V5)_0w)uMBsLDN8b&ia*`aXWSEJ z(mQ{1ZP8Q8c7_M|m<}_fY!#+H{&~Q8%fD;|BV!b|yFweBeFYRfp9|$d4YoD8Er)If zv{rdA@10Z|G0^Gn`uD8$%osDUk?ML}<#%Va#BG5olbRuq}7KisC4l{aZzd?jEt3b^&rwSA!?NgrM% zw6h=Km5QsDNc)*|JQ_7FHOdF+&ToA4W% zU9GFcR`FU5Ak*NgT43k1>Q5;SL59|_vvWrs)n8Qq4n4*{sv&iSMi%3l4wBkPYaEvC z))zqY8fFCz^qMo)n_)MeR7t;AhHd%Sy`U6gMOD0Rretd~MM`z~7}_-6v-7P?_I@d% z7_9dY$H7kvTu^X^ih^c}deCw(QPBt*4!&2^h5o`S!NvBA3WGu*wuc~)W03zJaq+#N Sk&-@C5{y%l;~a$Vulx^!F~fiW delta 3501 zcmYLMc{tQv8y*zt!(=UmvCJ^a5^2c3#$ZH}h9ushnXD-p3_nyDyBTE3mOYa!r4TCX z@Y*vNS+cxjEBiM1#`k^K^_@S?Kj%E>-1q(5&vTtau`aPL_c?V^q}05DK=wO@05Y|z zHHZTu<>FXs@FXq4Iq|okkAA=<9tzW<9^Ki^>0ac-S!^npV> z?e4bs=i+{2<@2!|tI?S9SFKgFra?xAhnHrF`% zW<*;99eeOd1hDW6_7pJNO@aafjh}_H5N{>v;Y*!^Dhg*~oirB|Wkvv-wOwD$hMKOm zx-NNBb}H22U?eqeegCHFg@MDD1BGMm1Tk>c=F2TJscIf(Jq_dcvDM#vwZF%_o$+)7JurI|o_#;;o zZ6fL)XU}C+(_oN+l%p~5==WIWh|F@`>uJq)Q`hV8cSn(CCB z@^d`tvr-b^x8hYdk*k0)XqJXwpYM1rCPdYmlmBI_U9Sjn)m9HYnCNtT_gb|+8e-d) ztD*xQi>Vr&Eux91RTq&NJ-A>R6-aN3?%Tv8qR#FT~PcXt@XI;8*JO{4x%z zyGjG@@P|k%8qh>$=w!Y(#xxzijf2g$(bTf(3jl0A;A7THJBYGf+?rM$#%%qXpSF)o zmV1xVRCJ?Ij}q&T#K@UAL#Q8AGAs8Tf$eGjUB>1?tL6At9~a_Y^jN^Es{ZrMIJ|Bh z(10O*kCd1o;iv~Q{qGW~8KaAbIe6sbq{c9`pYIbg|FxrZSoGKB%}h^!`9*B`Eqcip zXHm0~FH;;qgpmx^pYC%lip5w-SD=M|{@QiIDn_mCRE~)xpAlaooD^xObUDYQ0~Qs& z#*-oy70O~n;SgXWN|x~2PuMHV`+EX)T!&h4T1C=ncre#iwb5u_gSC$c_qAw=N;o_q zj@ME1LbnpD94Qi$Xb6oyXO(j9mkl-~aj^5HcKDtJ+_ENtoA-iVma&Ufya4s3y_eE- z`;ekj{TI~!GpJVjhOHKX^K;{#yItGbMP0n$itf2;%~7AX-usXuE3`ce-whrFP79f1iNo+U=No4`0`kr81ra)H zQ-vYsg38lVk z)Un3bkNLtufJyh1CpTmIMC>xfT)7tIh#{KF*XnHf+vcM?}Qa(vDR;lZCl z;kZB0d#n&!MkUNSul=KFoh2>LrQE~$)JN1lq@^&RpI9At^)?-`Jmzu5eK*D4=cn=N z_D;4eVetny6iA=p#v8vT3LBC!AQXv3MaiJ|i8;1X41_WOXD-Kru^lOqv0>PbN-^0c z5Au`P(ZY2g#8uRd^4>#zf7Epn+vq+jS8-F|A-gAhq}hDZ6ruF&KlqftNL%A3l@r{Y z$(tbe^8xejUWEJB)rRf@B~wpwDaM`mN1M-h0vRhhFxWzkv6?hDcIbCtgm|Lj) zmmcgePG6~+%nGKS=%17jHmnM3YEll7K1NAyH3$T$prqqjlR4~;h2%tl{|C-BQ@>|h zf8CNB^5NaiaA;A7!ii;BZ5mtpd<4FKRU%E-R2%vXd%Kr#hrJi`hC2rb+zi5kLkg)| zR{bqCAt{kDtYR#;Htz@~P$^>KRjzO6c$Ht+{8_dXD_&T)*bTH)FZ;a%8_zs#T5M?3 zOs+jfw=E%@Um3RAq^t|8@(eL0Z!*zcL`}t$?7C0BXVU!@PP>Ok3;?bdA8Uuoe z+UP@xJ`p&l%QPmg11Ri!Q7I<>%r+Hz;wVmUP>xkzXSV43Y`EE1%u`hdQA{*H<+S}C z=Pj`}9QjX;#0S^L`p>?kPgy?PFwrbIN-&3SLP#ud^mRa=s?29fvPu!ad_W#K<(a^l zx2O#3^z!xLOqIYL0TiwrsKlqGJhy6KE0v+K^x016&PlWSOGntcf~Q+GK^JW}ljjrg z{Ra|hmbE(2XSZ&919#YqG5_#b;(UN%xNZURpfBf8*UYep^H$iNLB}Mhvo@TKQieX* z-7nDuEPu~sj&BA>0eynlk2A>yG}ztOdQ-^N2#mh$VFH&AT94r*JW0Y3hy~PrCs9y} z^zyOUQ*sUx^emH^BFM7Yr1)~bS?XrxInmkpBl)ch(dI6?-t7P{f9a2i!?yy5Z?F!j ztfXKQ>7;%zRs5x3J|u-j0m=SJ<(du!V4LjEXJ9RZb6}Fsa^g5#wCva;hEI3Im!Wt4 zV>%m~+e%#$lX)uZy5nhYzi@U~R9bMR!Q=7$yG*|7Oum@g-oba++o8hZ1Nn?{o)B~L zYTnnK(3D6n{q_^n5lKJws-sr(+z`PH;IxQ0jQOD1wu<^*`MQnilpNwh` z%|JQQ_2g`X%YSMR)e2q&^{Q~{oPQ3%65GY$(|YLml&^(oy`8?R|I%T;Ajs~_JfLv{ zKduBXr9Ayc@-uyJt1rE)DTZS(b4z~2C&FS#j^n%F5}}bDQxWe9MDp%ZnkU9CMIt9FyvmO}${ zYy*b&seY(Vxp%89wSNxM)1&en5bzfCmry zz8yTGILx|bckyjA12z&@m;p?;2xF?k_M4bZg8GF>%RemVVb=9tp5DQ+a;)ic|EO!b9Uob@>`dnY|0PRV-R5--Wxxb706IO`=OOxMARlPGycqcf{eXw{%vqHAJW+JkP2 zt`M2fc(4+4S7Ey{Hu$T0uFZ=|s1Yx`u*MSgVqCR!N6e_(x&>D%c&nHrB4rar<#M8b zD;;E~oSl3Px*2!bIyZAW{QlA~57ZawT`cbTA^CemKLQxG+(KRW;-5b5#F^!@Agc}f z>k4llFLmU(icYmC=#>*zFKmtH0W!Sj9{6_cNVr~&x9QJ7wkjG~t!CT8ytQnRRCbOc z0fg%eDS9=%sj!T&R?_vQ3OflE4^=Cyy72N0-JBcx*=*0kr~vQ%J^CoI0*n`6O;B1v z?!vRORAHBH&EXm%2S(Kt5x>IZ#7wLRqSxMZ||*<__J97_vh0f8)8K_Fhx|3^q@8`)4+ Ro8 Date: Wed, 23 Apr 2025 06:17:34 +0000 Subject: [PATCH 488/719] Updated results summary [skip ci] --- README.md | 82 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 548 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10668 -> 10667 bytes summary_results.json | 10 +- 6 files changed, 345 insertions(+), 345 deletions(-) diff --git a/README.md b/README.md index 6de0343ae..4ba3325f4 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 06:05:24,845 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 06:05:24,845 submission_checker1.py:3308 INFO] --- -[2025-04-23 06:05:24,845 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 06:17:28,519 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 06:17:28,519 submission_checker1.py:3308 INFO] --- +[2025-04-23 06:17:28,519 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 06:05:24,845 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 06:17:28,519 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 06:05:24,846 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 06:17:28,519 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 06:05:24,846 submission_checker1.py:3324 INFO] --- -[2025-04-23 06:05:24,846 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 06:05:24,846 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 06:05:24,846 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 06:05:24,846 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 06:05:24,846 submission_checker1.py:3345 INFO] --- -[2025-04-23 06:05:24,846 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 06:05:25,855 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 06:17:28,519 submission_checker1.py:3324 INFO] --- +[2025-04-23 06:17:28,519 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 06:17:28,520 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 06:17:28,520 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 06:17:28,520 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 06:17:28,520 submission_checker1.py:3345 INFO] --- +[2025-04-23 06:17:28,520 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 06:17:29,564 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07639 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418257 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31404 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07639 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31404 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420956 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 1b6d75c84..42ba96426 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1115 +1116 diff --git a/summary.csv b/summary.csv index 4954db42f..8c8cf4748 100644 --- a/summary.csv +++ b/summary.csv @@ -1,30 +1,30 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.07639,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418257,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31404,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.07639,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0292,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31404,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420956,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a2e15ca39..2fcccfa14 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.425121, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.8483, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":1.75529, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":8.07639, - "Accuracy":"F1: 80.0", + "Result":0.417637, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":6.31404, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -213,60 +213,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.418257, - "Accuracy":"mAP: 76.951", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "9":{ @@ -275,11 +275,11 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.163, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -287,8 +287,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -306,18 +306,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.440114, - "Accuracy":"mAP: 76.951", + "Result":23.0024, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -336,18 +336,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.0024, - "Accuracy":"acc: 76.000", + "Result":0.440114, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":8.07639, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":19.7056, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -484,22 +484,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -514,22 +514,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":9.63051, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -545,20 +545,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":9.63051, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":21.6529, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -587,8 +587,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.6529, - "Accuracy":"acc: 76.000", + "Result":0.405224, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0292, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -677,8 +677,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.425121, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -724,21 +724,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.417637, - "Accuracy":"mAP: 49.593", + "Result":21.0292, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -753,60 +753,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":23.163, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.420956, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "27":{ @@ -814,29 +814,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":1.75529, - "Accuracy":"acc: 80.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,22 +844,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":6.31404, - "Accuracy":"F1: 80.0", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 409b4aef5cd7717d31ce3dc21b5504404160c554..2127725478b28d89e72487cdf390c250adbc01b6 100644 GIT binary patch delta 3290 zcmYjUdpOg5AKskH9#B@`y* zl$=kmC!^$$yiS$mFz3yocU%uV)%e7jqtVF=gPNJFQzEq|}%>Di?< zWNd$%x>dKOQT-^dGLd@1;cWVAN{ylaYv#BgS_wdu2FmoX-P*V}^>OSg$p&+z@>sEa zD#v$tAZhy5pZ7cKs%ORniBIAKO|u~R^4uGG55LY7GRbII!O#kGq*I-el^Kg1^5flA z?DnP&(+cC>1FtW#Ljo#PHYXAM4B31BQ}#UBc3`TZP9amj)O0xVXZI@l_vApd^w>vo(ZpXvDTInmM*}5xV{(7ts4p!mwr`|vvTcrpj#)CP89Mxe zfzH}~TaEWK8D#1F<1#j7;6h_m8IUMvzW+ zz!j(^)PUEJ<$3a*KnL}E0wRMwz6Pda}$CWT0k?{{A1N5^; z3((oDNF$fcTW^{oCSxd!=-2Qd2ldSD4sj@VT?39s4BD(EU0w??FVt37>i z0ZyO=qPZN_pj%P%r6168TN=SA`Lk85r}d4HL9ruCNFh7mow zeLKUt1-2G$EmGFNh-Sls@ao}DFCo+e?-JA}6jmawTinMqWw;yep4Jb84G=PaFRbCL zKi7spW!C52c3+zG=-_n&D0k!FKa&oI(#AKY4@zmkm?Dx9;Ll9Dqz9n}dMeJk<_E6Xnc_5_WBX}>hOlc@E>hfsxtue0YjJ`}GOinMZ@z1`lOQNDL zADO%0SswS~f+yGN>AO5&^0ld7wX#ii{&$U-iG3@(^GSy75;-_^s0}nxn-OizG04Gn zX~sk@{hp)|^A>DVA}$+3S1*!r8*@V)3-YgB`DLa(P|n(~eQ#yG%4UQEdpc#PctcK@ zbpG5PoUZ_`45!Wz!0BGICSEJJ6b80+K7NIN_(yFRU_NcBAK@z!r1ID%#1V1DzrHnA z<4S|bs*r&aWOB;e*kygOb$#atn*!@D28~}l7ka`wsA}4^aUOd%zrrwXHlQ&-kd@7h zJzjYGEXo|wjYAM^QFv4r6jnd{;PB&0ar%>@sQLLeSV}Ln;`~%=mI`YXpIXP?NTMEl zdHymWCwJp;Am(SRQChyXE6Jgg40RtXO3L<&8-sWsJ6dA?eRHlwKibQ_+rum=tn+09 z)+w-ll&YDM!kvC>$*?$q7kh_L03AH_#%2-j>0Hg6@y)6(6r}SQu87M7MrKVO1-pe|srXqL4b2q5+GVuOfuOhhITC)i8Irl?MDsjpNtqO+>XEIWHw{%)~ zhZ2^=>NGelkUTsp6xj(%OHuPSMxu$*p@q4-zKf&D_8qnsXfiRIHC~0UY4hvnVa?fb zdd|c8ncIFq|DD@UeMfZ59c6vQ{8fiqVi)Z6Pu}-sK+5o_cStA7zflR-uQ69azKVCh z&->FnovI>yWs4?Oa!MS!Jx5v&_@~cfV|$(|Qv}MN`1KzvOD*lWMICyO0|id!9-hF5 zMUiNf#RNzmv2pyWf;m&bvJgL+D{vCYJRQ~r2=5swk{mc--2_4o<+tFU!wVr zIg#bV*JRC8q#>~T#v!vcNO8>1`ijru6oi!Q=ciktV7HnnFaE}PrqY{BuM z=lqiW9VBPz1R{H2St7MbXCpvc`oe0^Q{&)s#{TUytiU^6$8(_#0qHsLGc+A*=VP=Nqk>ak?pI)rSm~yY3%d-IY+PusSVrhdU)T(K zx>gTAU2C)H1Mkiev0uTPQg#FsKME-79__&D3cH}HZFe@s$xFbXcHe0OYVSo)kcE9m z;gMgdw$fevN(kU_m2qIpMC>A4qLR&Nf5*k}`UkBMW=XjmHYMr%Yrf3KE{DD`)~hA-baE z`T??}q~QVGUpJkqk~E!9lBpaX*R|^A{!eCW)s^;dtf?)`F6mAYN1mF(R7LnwGwPjY zCny;?8ASzMPK$v5)+AjSK{~+V?uPQ;5I2hrj-Dho{%>CLVBjm=iaw(%?=A1qC0N5x z)mc^o{YSLcqg{erD-;g!5}k}9hn``8QG_M|!wxhLqe$CR8*yea1gD@vD&D}{8lagU}nx@ z_2EtLqzz0O%`<)_1cyuzz^SGkqQR5E=+sgT>{;t{8eD&W8Ko}-{T(A{ND2^67bl0f ziq#?$!#zKh46XA7$NSA3vvafG?Bq+ceLq#|x(CQSv*vTAglPpUg4nU{;lwy4s@r9d zoK6e%fwb5N6!R1Cf;ME2UMN_l#8`#b8O9)XUM--dBy= zyS$w4A#d((+!+-wyo3~ct+5(w*{SsHEbh5heIl`Y_<6?FIkUNTAJMnJ2L(}*1F`x# z2-wVaLTp9BhLvSBNt7!HGG1TJypOaa5R7N?{8fnM$@<`f;oUm4z0tfVzOWIMyUuKu)!kP> z+PliUSFjHqYL*fm7zu47<>~c~Gc@ z&1c=<5!V)M#cz}`6$5;~a~FN`u2(~*d$r3?^1O=susKsvMQ;4YZ`6B- z&eqAi%a*dp4pfqx*$h3c2AY}7Vy*tw+53bfFusjp5Z7P&2$83<3V`ggWfT_$5ZDa5^rtlG@ k(w@Iy5a^r`2qXjg-xC_v!L&Yt0mGR0k7$U!NA7<44>uh(1poj5 delta 3334 zcmYk9X*iT^8^>p|Z&^wnS;jIm$k?WdNZH0RJhD}`NLeCVWDmEkEIk-=Te4+jc@l3J zYb8r242hu#d6caXvd>`j4)1#$?|Xl^zTC(0zkcU={>~3qrA?JhO&}*1-gWXJzzcz( z!TX$Opcw29kqgLUIYkaWUKlkeE#*@lBQJD$eYJ`8k4LLf4OL;|Ng+Ej4I-;e$J@aM z0gJD*IVyEtGs+G~Ff4lfic;lR(tKwaCU>epU8e;IpVbh7jk3K!&RLYI9{KhIsjTltn(_N zt8Pj|qr#UX-+Io^JF6_@lb%^H%Ju?%W-a)%ulIPq)#%wsH4~28a(%RWt}i8}?RnRY zjU>DaCzU!|wwhWriyZGy+zp-@>=^+vrtXK{-p1`IuZHD^$(jAr_mH3T&9%6=u>ZD( zLcd(u&Eq2n;xAOG0b~r0Jel4iC+M%U9*f;)^cV|qg=D;$8;^Xc9z-Mus!7k&SE%GY z;}bAR^GFP0``Y+7#bw<$rxf2|rLOIIPBTi9`5)u?pI4;6$8Bf3i^ufDhGY-krOx423dI$paozuk29dvdvBSb)Iakj zyr(dXbg?^0?!ZC2oAa)*4B>)M#+Qtwu!pVdyq_}y*G6~548leFvJi_ZDO=P22?9$O zrlHH>1e~?(!$`Ao#13dyZG`mqtLEt3NAz25b9)SbqM94cPJ5qLNo*PoeKpruRr1Yr z;WFE)B|B5LQ#xU6r!D}|eV?*Btu{4NEX#_D8%8(}{VKcDslvb#r zwXzi^GqhIRYLid?)7QSPnK#D}mlt z(>=vzco=t^85hHhLlKhP1Z8`LWZ!V}s*)n;L*15Be5}A35t6i6a^`-og+7@mBXErv@_-t>-G~u5IGng}vW}5v(v{Q`f zniSeC)9-K=Ea@*S!5t35OHfp5rBjj>?mpfwFXG3F-ud+p!Rd{%4J5LWo zuiu-z?6b2tR4ZYdRYRFVN0t97H0cI?Zgn*w@*nG|15zrDOT zab=~8L*3o1$0)=9!ZxwWqR(Z5EZ~>Ft&?KDReHi#!n>;8cl@32S~#MvYy1(FA6@x1krIHp!iiGX1EE(V2aTp}ts>Z@}__P{@V zNRlz;2K9cud9d{-xf`$Z_)dn7_q+QyD;>iAsj^GmClO&spsYg*39p?~iS zov{)QC|dyx(5;(MChQeeF8ejSmXsg`%y(;93MIcuJuLbGN_i;_uk`<>_*1n52623R70 zC#K3o1=(;ov9Sx(C-~&87kO~V4o_@$r>z~uO;aV46}MmnuF?YoGz8ISB5!Y`k3;to;;PgabbL;7 zl~u&QVBh0#(AP3om}lCb1xR{J(DP|Hw-%S~Zw2KBE(HgTCa6Vj(lrBgEGF87oqA+9krFA5s$8BKB;CIyWd5`~tH!Sr7lwco zi^#iOTrcj_cyk=FlAbD*EH?=wSY>AdCY;aQL~+cOIJ1y2o92~23sX-a4}H`dWv3r; z(JOkGdpwK|e~gem|9+f9?l-3b;9nqH@nqA;L&>``@L|JF3#?L@_G)&n%t_|p^D}&y z^}e*IJ8mN}L4d+N&))|jTzwCla6726g5W^9YdoDrNjkrL|K$2n3#IJ882~@y&jRv< z4iK+xx$wPL*;I!WrQduJ|B!@t_3SLQN8LaZ7-S18ljzL!zY^`4Cr8vMdpF>(4$bQ`1YQH;LzQ0W^E@@V~Tq|HUdJE{I zX?R0L65iLwwX^c@!eS~Ul7A$ejKq$^M)@EeYC%L@j`R&ww}?OF?|rvU;&sg|-upb@ zNZt8Yq0VVtt*dpd;#F!unvSiCf9LaxkBK$`y5@-UbI0veUsU`G7UUmQgI!xAi*Zc{ zz_yZ`hGn`h3E+5j(*p*2jp>^$&hV$@QtwcR9WSdFlmddNlE;n23=M_|QHzhRvBrCT zzLmk+537?v^d9@ax5_U$Tc{{#BCicC0UyZgK|{fAc`fJycuWB)cGOuY1Y&s%0uhA# Y?~94;1q~D~K_$UB1zGk% Date: Wed, 23 Apr 2025 06:24:56 +0000 Subject: [PATCH 489/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10667 -> 10665 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4ba3325f4..33bf9e5de 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 06:17:28,519 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 06:17:28,519 submission_checker1.py:3308 INFO] --- -[2025-04-23 06:17:28,519 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 06:24:50,131 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 06:24:50,131 submission_checker1.py:3308 INFO] --- +[2025-04-23 06:24:50,131 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 06:17:28,519 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 06:24:50,131 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 06:17:28,519 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 06:24:50,131 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 06:17:28,519 submission_checker1.py:3324 INFO] --- -[2025-04-23 06:17:28,519 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 06:17:28,520 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 06:17:28,520 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 06:17:28,520 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 06:17:28,520 submission_checker1.py:3345 INFO] --- -[2025-04-23 06:17:28,520 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 06:17:29,564 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 06:24:50,131 submission_checker1.py:3324 INFO] --- +[2025-04-23 06:24:50,131 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 06:24:50,131 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 06:24:50,131 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 06:24:50,131 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 06:24:50,131 submission_checker1.py:3345 INFO] --- +[2025-04-23 06:24:50,131 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 06:24:51,561 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,13 +27,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31404 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26878 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07639 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.85635 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 42ba96426..822fe6349 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1116 +1117 diff --git a/summary.csv b/summary.csv index 8c8cf4748..c1b94cefb 100644 --- a/summary.csv +++ b/summary.csv @@ -5,13 +5,13 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31404,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26878,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.07639,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.85635,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2fcccfa14..6cbbed5dc 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.31404, + "Result":6.26878, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -199,7 +199,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -369,7 +369,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.07639, + "Result":7.85635, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 2127725478b28d89e72487cdf390c250adbc01b6..f47bb55fa8517a9fafaa7018e1871ec78514fcb5 100644 GIT binary patch delta 3275 zcmYM1c{~&TAIG;`kt`FwsOA`>h}e%DnK5jbsK}9|Ig%r}Qkl8Rk#iGaax5YhCPxt! zVrkCIP3BH4H@SY}_kBEmpFckTd>)VIR8d{`9Rg$c-IKR3t{EF`+=?JueWi#ugxtLWGSY0-gtL513NfC9a{5L zN!QT0a6-9wMKH0MaA<1&agifjvmm0L81?o8yZC;$9vDnU=k3!ROFCZm`x3KivCMQnvg{#zrSSvWAr! zYFDdjHM8v>4*CR)cC;`E$)gYNxvgQg&a#3sgJ7m3t&c&}FjbTDe)B{n`uNcOmaHJ^h3073 zuANTvVIN~@qBb~rC^MK*LLCE#WEm%1&6W z9F@O$A!o+)I3~XD?+DSI<~NwX$3m7RA+q+JZ01$*L#X35F5XI=&_3VzV@=h+b$v(`zFA|@e!Q-1yc&%CzG0UaBZeY5 z1F%FO{G36@4>`CkD6}9qYGL6k58TDH#Rv&^VNc~Vmo=lJ_7=g`=2N{WU5jBenK2i5 zv*v1A!hH3sNbGLvtlA?MO6$3^kXoHmeEN?%b_jMxZ?as2{!He@{cV zRKZ?fJvMl-jK>9U$2}}F2a+@U%Kyk+5M+#67D8x72qkp zzly;(E>}i+9xr09>{s>1VB<0G%r-CqM#8~_3kxKpk2dL;_9sCM6dx5u=qbFpf2+w*+$;!i5 zB)V9H%l?JFg%i+U(|&z`Zbxooe0%Ugy->DN+v;T=Cn#^j;wQc%AOhJ zpxz}an8Uj#BVO5Z+Tp*M7Zod|h0|8ND|K6@U^c?DaQw5u(3Q7#0@uCuJc!-$GJPt>fK<^@*kF6}_yl(#%iUEfg z>wa+EbORA+^#*OlE@4)MKMdE62bHY)rq*(=$Xev-Gfh^boDUW4Ch_arB6pm`LW#}* z6Vhgb7C;`$OypkVvC~&Q90qQLT;guN%J|O-#{fFBeH1hqW`|(;cmAb;O06x{cJmps zHhBo9$^1TRCyD>2Epp3A>=T=1N?K*J2#VOQc6N>xk$R0xE=_jB-wD<@*L|1*Abau4TkQ z7(gF)a0vV`q617sT6~S5>MBL3d2)>z_bP7de4TW>Io?BUF`4emTciZEvNi}VJpJap z>4gc^T^myAdg2IZgtN6uhz)lOR7|oPx?L&|YDQvFKCg$RvJ26|IU1ERhOCTbQ9Rc0 zb)fX9)7vxnv6r}-u*-pC%?e1NCc&$G`(9&F&d2C{U)Ww5P>x#}!ECP__bxnqM)$z0 z>A(wH9!^-GcrfsF!gSdCpbAB5SHb(~tv;n;UXu}6r#Elz`Yt z{7D#9y#ma-(@8+O+@Nv9Jcp8$SY#4!B}kzGTOK=4@u)oTdbM?HF-Esiw9lmkZW;{1 z>XsV+)n$8)9(dL5dwpjlLtunFY#m5~s}DY_EA9_rM;rdR@xaZ2ghrNnFn(_ZYiC~6 z>^N1p>X-k7ak0|F&U#E~3J*7Z!G?Y^Hrb!->;bfEA8OVQgk!td#16P7LY#a+6vX<3 zpvAm1t;i6E^5tcGht8E{<0H#2al(&E2zYhV?`%?samGGrnZ2BJz9}{-6UqUzBWIw;UCJ2%YgA zBJurSaMJod7eM2KY~mo~O@n0q=aCa(CWxznmVar=ZnNQWa^*mm*-I%o9jo|5~_Yz)^HnB3Be#=eL3+xBnz&CA) zgc9rS!sHMK)JytItMtRjr6zR=it;%9YAjr&6;n%o;iy5G9(AW?2(-B_&NQ#SuwJHR z7tb_99wVxo5&Xm1Ay>R-vc+GXPw4G+-{MsV!EOXjTo z7SkGIjSqp_v&UCrs+^MjAb;dJC8w)J zC;is8sj?1~(-)Tw$!+7omDn_BHp#GZ&S+%SG0gX$r8`V6pEmp+?<++4N!f3Q{{3|P zhS>u}NN9N-U=Hb#JQCvvZT;5O4RXaz zfz4!4cIl%)0j*|A8eZuh3z^os)L-c1+GCA^wdXUpepGY6^SU*fhp>Gu6jW;a(o{89 z&nCS&llOBXpm9s1?#a{YMDlU_vl+Fd`cwY3^zVKsB>-L)DAQBr-p;wLkFC0#VlYRj zh!x2vvwVjKk|$sP{gByMH~l@(^;vwNakfOh9OtIq;~&$7bRr5`Ftkh`VXBj|vtkiL ze%$+t-QJX8N@3hbp!Pyl$lXenO*VWZQ}%)Xge_OL1DI%TRLIgVGaind?Os70O?h$- zrBYf5OkddIl2LVa_^O7U9@|I`%Jt7sD!y{l(LkxHC8fV`)R&SS+c(n4Zl5EdVit|+ zhvZ+sEc8W>yrLGhpS`|vr$Lc<+6^I%~#Ye;LZp^@!3o~09XObOi?8UHBq zu71vF0V;Kek5?a8SdO%8^%&EX;jDXjT0RaofXm!k zSjAd?Z3uzLtj)XcY?{>Q;57pXXMO*_Z2MDEr*BT~m(+mLg~TJkU+GlwOMvoc&MZy0 z>_gp*>b-I1sWI+FW4Rb*aU-{&YPv`XU9-p8UPKakjQG znT)!4XzsjcMclLVo*av!cTWL!t+8L7vQp5>kTgi{@2xmb0G6(muc7H!Eg z$i;SP#zZb2P1cBc3$`j1l?|b)7t6Shxg&Lh{2P}4m}n1FF!pLcSYE5K8eu_;CQd2d zJSa#w=dcUs%ZDq+s?+#zI#(@SuNM3j2DWxSc6meoS3?+JI%%#S;VTrRl4KR)2*2## z)E298xmjpMz(7fYJ>h-YW$ky{+V-z1612Mn^!DIma($>EzOit{*^MR_gGjNPjN$)x_t5@da}er}g%@864^a z`#KfjC5@+nQ8 zmOQ@!!7qw5YOq=*p5l<92qq{!Rn7Y}0_7?dT9~)vyC{li+i7ivBD&@~+2Zl#+*12)&yiMX|BQJ|Y)_FgiLd;bU%zg7T3Js3dFW9t1UQo? zKaQiDN}=~T5#_>HWnnX5{IZ2H{jro8a5~uGw13AmBk*3=u{_8H8T(&oINNqCijdH? zw_U0ml=M$M_%MDzY7YDyMa9_o81CL@!O3qAE76B6bke>Hx(t-ApYNz#g6l|~-wY~R zZGxR?u-f#2b>|A%F5`?z+X=D9KEr5cI!8J&I)X09I_vFCQQ{&nsNHwQ03i2X@C2FJ zb`~D`lV&Z|wNWi`=?YDH%Shxxm00x#Bg^qG_oKq}=~3U1gdY3y!`{sb4Nk$f1v2atfECFlDD2Vf#%jA-``&p~&D_2M#Uk*~J9?d%UI@*s zx0_GO-@&Y#;(HOZRJCx~Pgm*ZA*d+!&M)w*iiHj( zZXykc#Bk3qr9*36{yTd+*TiAwr;S`$j_;Rh9gn**&n-7t6M~e2Wq#}!53nMvQdM5L zL@tv7HSLG<%FprDF6Fmy`D*d|SwcUf_$}|+#%1XlVa7yzEcAfth>+v3}n&bQ6i`8eRhpGpd+Sm~{>0dOt}0-&OI(`!n5^a5QXfJ|EzB&g6sBqMoQ%pvb+i zTX;LXoE{)0WL}vwnd|ToetR@1h?EkD z(LW4_PV*9CD+|^w%%cgy9Da~-`f8?qgl)cH9G!dZAy-gWEu(NtVsZC^Rb9Vp0J9HI z0@}0|dV;aT^v|xNHW==tjz_{ReQb^R<>(W@<75L?5Go7tTl;C%K`)a!foc&b_<#FY^5 z@OIGa+A^_`Ym?Z1)sUJ(U!L(=-ZH{F_qPn`sPNuofAT?d0}k-kKl~P5SdYqE>;LGo zkLSd$PSoY0)64u#Y_CRvplkDYg zbD1UX=Y^BaOCUsCjJkteKwgdJ#`6@8U`%FFOuzLJZ5CMB?Yx2T$q1784QHkDOQDYi zu!isLsCU09g;!iNO8BJNk{AAl+x})a*>hy%GZLdk;D2lM zZ4DL*-i*ux@98qi)KowRo48CPlul2+KlBJApIV*W%4)iI6;t?AIOyP3cK%DB5?WO_nZ634-TAdlzEpUX_gbHbZ~kz^o$y4nw`y9ndLr1 zTzY!geyQP9(~~9FtbcKL?H4; Date: Wed, 23 Apr 2025 06:30:12 +0000 Subject: [PATCH 490/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 12 ++++++------ summary.xlsx | Bin 10665 -> 10656 bytes summary_results.json | 12 ++++++------ 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 33bf9e5de..c79da6cdb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 06:24:50,131 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 06:24:50,131 submission_checker1.py:3308 INFO] --- -[2025-04-23 06:24:50,131 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 06:30:07,206 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 06:30:07,207 submission_checker1.py:3308 INFO] --- +[2025-04-23 06:30:07,207 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 06:24:50,131 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 06:30:07,207 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 06:24:50,131 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 06:30:07,207 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 06:24:50,131 submission_checker1.py:3324 INFO] --- -[2025-04-23 06:24:50,131 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 06:24:50,131 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 06:24:50,131 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 06:24:50,131 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 06:24:50,131 submission_checker1.py:3345 INFO] --- -[2025-04-23 06:24:50,131 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 06:24:51,561 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 06:30:07,207 submission_checker1.py:3324 INFO] --- +[2025-04-23 06:30:07,207 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 06:30:07,207 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 06:30:07,207 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 06:30:07,207 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 06:30:07,207 submission_checker1.py:3345 INFO] --- +[2025-04-23 06:30:07,207 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 06:30:08,293 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,15 +31,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.440114 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.85635 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.63051 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.62032 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 822fe6349..79e6287b6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1117 +1118 diff --git a/summary.csv b/summary.csv index c1b94cefb..086f2fedd 100644 --- a/summary.csv +++ b/summary.csv @@ -9,15 +9,15 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.440114,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437182,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.85635,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.63051,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.62032,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6cbbed5dc..4c731e2a3 100644 --- a/summary.json +++ b/summary.json @@ -304,7 +304,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -319,7 +319,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.440114, + "Result":0.437182, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -349,7 +349,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -549,7 +549,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.63051, + "Result":8.62032, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index f47bb55fa8517a9fafaa7018e1871ec78514fcb5..9da32e268a7f32aa09df32969089a90996086f57 100644 GIT binary patch delta 3642 zcmZ9Pc{J4T+s8*3)mPanTN=yEm}(GBqGb6-V=NgdOSY6XOGz>bAB-3Y!&qmU2+<-j zL$*_eCV&oP^BXLjLvl?KYZ@Dbuk#Y^tc_&2;b_g0xG`9g7pc#x}f^y(k-!AehpP zOL=$Rrhyca0tpE`|2!|2P8!z$^3&o=2La$8)=B`Y6-%ft_q1VZx#M`X0ae2{S z^v><;bLLj)^{kAQ_RB{YZk|O_a@4gLw@WgfR3IZouaj-tX?N7tv-__9*o~=WMBC^{ zx?fkt$#bL7QAXk#Z5;2lUe?{w67Tj1ONJyx-YQ&+AKx$LZFLkLjoVCEgH9} zkPSomadp~R`zW*iu!Biy%J2r?sz<2B?aqcMv>3!R`cwLc4HG8zeWQu0$dhz!GVZTC zFu@C-B;B69bthMuI&)e)!(ve?al=!NF;olIUK$BxdVxS zOUJPh03@6f<+VPo6^xZ4*&F`_%i{S zvL*f~5l1cjw1jU%kRy?hw(uS*|9w*wP-r=#__sGWl4B5L8Y!%jcs6pWZR^(B$)3kv zgaMu>A{De(Q3m(EJ~U=0t1PreDL|;7wil+}TXj-K8{^;3W629qK23x3OBQ}a6Weag z_LxUiKLa2)$8@gX)SrlN#3@y^O`s|-r)K1`iU&A3{Q*ARJ}xmD4&Ll^HO%?UrP)>mW!wXgA!5NllvH`R((mK@Q zOU||1@8XV6G5a)hihKVW#m$Yd%X$Jyon9x${ESji8Gh3W0TYd@M-{5p>9{zNZ*lt2 z0H}Pqg2>|j;3?@sg{g`5HEH1DLxI+0rCF5+bB&G>d*_%F`{vn+m@5jALBOp}eSpXaWKZK7=XP@5GMQ-UIYiPh}ETLT}2N zL(kltdBnZ+7gHGz2tDyPdAa8a1pmpU)5Ejco+fiZGe(YvKm^8#IDV9BQ{971BseGN zLt`Y&KS<16a}Q1EZ7tP~WM4*L%ZUd>r1gHGUCiSTF-mRksCK6;IMxM*QC0;ofvB1H zf$O3;U}dy8ZsuAj@cYDY*xiz&=g6mDc^w>oV~<5VXHG-K2)vq#lHZ6fPy$i`1C?}% ziF5G2_|#wsAJN%!(QWhF5wv^s0AfkhOuQv9$Hc)esI?-+3a%Ugzbk>&&X3|J)@wYU zR$&-OIgY({C8|V^J#|q!onJGx`8Dia?H@|moF+{ATI2f)K*s`?JmL>aOWQZOqL-+5 z3InczN`k~rmtFF?c*vxWH`L?Di|k42<9~MVtGms;uh^s@8Oj$k)y8*Fpkq*bJv7&g8-|dJoJ&wE%WtVjNu5;*fd5`FpvP%imUmTX7S2C1>i4~Htqxvo z&Ndm}LieY7-GtwTQquI)G2j8H*QsW$ zt{TyryBvSh;(`a-w)yBuB}&^bnxuL#z*H(9&R(Z(x$VD`RvD&G8h&u+zfir!$oT@g z|67&3Q&pBs%SX4|9faDmXoK52x;vQH)tQ^Et{r_LgEX6oEJV^j!&q8$dQGy|P4M6j zpL3`wyrafNw!eWJd;s-lCwEP*7r+gErDo<|oA{JkTC|*<`UoIFF&pgaQ*AbC&8@6= z)jr$a3rk5mQS{)@tnBli2ZnIs=s|`o`S2H+6dqK%6t6pZQf7!Z(_<;%&|a+$(IRtU z)4MoBNo7ior=9v-53Wj_95uW0D!8b5pVqYpa_(k7iy$5PCmTqvLJYnuz`UHKK0SQ6 zp4%2NFOVte+r>v9lxbXiBDjseSl{_jp+M%F<~N}x-@@E9J zjNxhXZ13uZ4;Fg+k`V;ll_j}!-Rs&={8h`}LDt|`&IJds9`}OXg>+wH*37Rom4~Jf zV&zR2m6KKsC2ul59;=}Nu0?t0QnpEjTG+v9>QHI$(9no~!D^A9aiM-p1F33$EI6+Cxm*p%`4gQx7j{36k~5c1aKx0`n(4)geS3u7@#Di!V71~ zeXv%-<6p_@gF5-VjGWi_IPiwRb$PYDLpW^TEteYTzS?ULdpd_6H97WTvRAG4#MN2* zHS~GY68e|zNx13I*?C)M{Qb)}3TPsqk?${x`3Zh$Nqj*3Q!zsJ9dj zx&O7U{g&M}Ox;0=lwTY_aweqM2RxtYrNC5{6YCW*lhT2T8L^_AiC^|N3AKXKp}fxI z2uL3f(PJqXwXWSEnxMBg{o5u{ZTPe0V(|U1RqbnX%h<<|Pk;)iSVrZ3t1_ny=ANZ7 zffScPvTIb}gz?=oHus^|u;LiMSn}xe;EEluJ^+kp=(fIjY zUk1CrtWnWod@-&&VA&fE&x<`bZvyl8e|GK?B0e}M9lkZ3uoQw!F6ZbZ_Yu3N54R=?|8$|^PXiIevnPmG%_?GTT8H0wIZ-dnH2Xw` zDL?d^NwU^CxF1?o9AbEBkUPb4`n2$!DTbbx)$N)DwhGNb7}6RmbdBrqyvas6Yj}htZ{zumGSc(08bqRYV#iHb3YI7bC zyWXk=L_T4t=Z~j$vOV?lH*P*dHVyW#{AgTDLn?03J=8e9{@B*R^>!ByO9&77$bx`J z&2((pigP_Jc_EOmZlzK$*TZD`1{og#v4G7O$|AA^zKFi%| z%gJ}HrEL)XGKh{^mR9Q`L%QZTP~vp8?U_*|_3_)jY>o0#hs#Y>3-qd^?^J;}om7=O ztT(TF>R)SQ=88VMCxDdbzq>dzE1oWRTG^~oGcU+rCloWbg0o(xJ GIr$%BfC2FU delta 3620 zcmYk9c{tSH+s9|JOQuHnT52p~sfd^#Su%D8BPx3;&DgS)Eo#Qc_O)f55!td#L=?ta zM51hsHDf0`iOIh648P~Pp6C2={(N1pbD#Hp-RET8F1TId$;!rv*37_hfIzau=d8kj zjn@#BKa93j{NUBSa{2!_NoNzAeI=fxGVnC{aAx3VgoRx50wtlwtF$V{e_qh=6-9nS z5x!d@6}+@6&M(A@vp0WDC(D#MPhGVZC;9~p-H|RFjF@foL!^wSDqjX$sG61hKO;iBLoU^UYYYJ%I<|SXb+8BqBH{3Eg{=V(k?{zRc zZ!WIpWk@A;KPbPNiXI*!k8HZirYI`j5R@rgKNr_bv|AUs_`#uDb<~?2ofm>g3-DC2 zf>X+f-S8Ue`!_LsF*nmXf3ED%Dj!vL8dEVqx^^bd*4Eb8r_mDos=O|=-thi#8vu~A>f2-_k@kOLS8OrzmZwHLRw&zfJT1+R=^jX@| zk?M?SUDE(PMS(^$!uPUqMa2R)mb6gir^$ruHs8*4YJYLzl=o)16v8SiSLCX7FS>bQy8yGrv*^OGrH7xgToNR1+n3fM+{prt4E87G^Tt2tDx_1>ugcwc)gCYus2JS1#xBD~5M zcb-*S3ZCIDB>AZwk?&G^|6KPeN83WCcKkeh_dhB{^)z5p5rDFDTx-~Y%>~1@*!QLi z6&ptevm5U6G8IbLa^*I!C4w|%YQ(=K!;gHUIYQ#lTD6(=v7QJ)1)4CD5aAbAx5_b) z;JMPjC7=cP^R~Yf`z;f-G3gO{c2Q*#yyQhrD775P*Xo_u zCL<*pm2^Qm&;&_Bgm!{9JgN??bL*-pBJpXdpee9^M)Ki}ta-!psF;D>F@kfe*KqC; zO`|k6_m27sslG~j=)#hZK`aUEcuk#$N#~g7NaqF8c~XQ3>^adksOV>Q4q5WEot|cs z@iUBlysA-pJ4&tF4-Mwv4BF&qZGt_V2q#H(cNf}5n4+G?Tc$pn zUv?h}Vv`mY(7E?#LBlFDv9&PH!>WIk|C_xXI6P>svEvm+}JfFzkl^Y|5YXOWgy_-Am!`gQmkl0Q?yL+c>+f;N)9C63VQXbVbqg1 z=K=X5$VdxfD>ja)4or%V5s@j~%qqIsKK%iuS>ndS^El!G&*qPGBe#m5TwO(&7&C>= zrKX>oa^CG*BXWqH8k5fRb){3)Q1p*&>zqhFZ6X$gCV&yww7PzZBdj2SKwftE^73~U zguP*lt_H%Mev}3#b81Dc6JTZerJk3fOtqXzUG#leW04%YRQ)a#eVDW$|1V8j`8AeH zzD|0uiGz%i`$88^o5tV6_9~RNJ_9M7x#9g(QLW^Xjiy2rZn%`i9&gP&EFSiFxctb} z^_@h9)1oCN*RA)edm5n8UY1Tcjf|>qJ~EoHnTwU_hI@Q@3Qv8dfj}{L!THp=YG0;Q7)XkU{o;Xp7`t;qc%s6}QdA5IkSlo~W&UM_(o;P^&JZuq+Ue~(2upJLX z%ZR}U8i*@SYEleNa(&EQhyt~87WH22p5(&hU3Ei!U2q)e-EI+k3)UPGZ&gNJV@>&b zdD~YKa%aQ;ZS4(7mDv6jgIblDWRu8u5m$U*gq2*HmYlvl(^R;9pn=0E=%x5#_$KI1 zY7T1JUQ!5$an_@8L}z0wzr!t?S<{IEptC z+JF(jl2*E!t#VLeB?|#{o5l9bDx^FVdv<*_tEjUr5Csh`RQu$x;|OW3 z*Xk&=4>HUDsk5awENwP0yP17Q%p^y1(qJPTd#d0to5eILqX=Qw00$QnuW?%r*Ml$e`r9_CWTEOU^P2201V zgLsaIRvLSDd|&7=L*fP$#;$NkmgH%B{4M4jsYQP6!LO${8Qbee65rcL zni*NydIyXo`5t^l)(*|}T%CaSowK-%gmdZ7gag?4cJ?+%J17~>a5ps?Mg>21h6W&5 zkzG)NhROF3vYK>=yerd$Ucc17>i3!Z52yObEe3M~IV&WecG|W*%b+n^N`7p0&$e)i zIzJK~f;E>7uw-t5^9l9B_e(efjfgZ-!&Xody$~&o0I{IP?+v3z8`02oqGn6f*C=tJ{@KH+q}^2Nx|qM=mHUEv zv(m-dd_sB_k1%{gXMWK$Kmv5=!U?qJ7-}>KMxcA>$S#Bul0W?fc@F(JK%dV&%`6>e zTeh}_?^3<9rgwVnEr$EwV!VRk4?3w!FZH-;YvI^<3i~V_lm=&zuqJNP6N31^xapHk z`W(&F#d`w+e`xgt95NeLCIK5}!EyuPJ)Yi!Va~~~kP6E1*YuXtwYqg6ESyE3JTIk0 z3*K-MRPx+BfdNUVA-x#{MozqD97VQHMuA;Y=r36pSo)r5V(pXIH_v(mQYwnxxA9TO zdIO5ge=QP{lYJw%FeJ@M6fGQ^DE18o)r4P(KuR&_lL0m`L3}Md=9GTw517D@{{_dN zfI+S7upJC6eMc*S<8|nCkO9)d*L1g4#&IDyx=e#?yR8{*RbCs}L0UjKb4>kN9=fw^ ztvC+#%#B65*O^QSzmDPU)FbZqB3XJV{lFXI4q9}_YwerJGW|t7)TSa3TWsD7!yi-jul4&s+}oaOVq%M# zy?&*KtMRc*|R!Z2kfNJ(!>5Hp-bvEt|gxaH`L`;mlvVTMIuKg>bQR#1^dUdZzcC!gun?ac5L!D21m-aRD2JfbNRh{{HdjyP;hjs@}yK1Mx zhSxzIvIrp!|L`(+qeU^xl~Jr#@yPYuzY+{Y>Wk~2;*6ueSGo~>sWw`N9Dv)s@jWw4 z_NUv|)Xf_B?0E9~D=+_bJIC2?GN(Av^c=juaDF|q$}YhR_E(ZoXs()f#%pt*EM`l( zd~;1Fy<-YjiB5)R5_KvUb;maD2YLRp>M_amxC4L0_6}JF6>B^7|GsuJ5*A<{;tfev ta2D~Iqz2fVSSzUvo+WNcivPE=!k2i!&BW`M)W8D7r Date: Wed, 23 Apr 2025 07:30:40 +0000 Subject: [PATCH 491/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10656 -> 10656 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c79da6cdb..d76d04bad 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 06:30:07,206 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 06:30:07,207 submission_checker1.py:3308 INFO] --- -[2025-04-23 06:30:07,207 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 07:30:35,459 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 07:30:35,459 submission_checker1.py:3308 INFO] --- +[2025-04-23 07:30:35,459 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 06:30:07,207 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 07:30:35,459 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 06:30:07,207 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 07:30:35,459 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 06:30:07,207 submission_checker1.py:3324 INFO] --- -[2025-04-23 06:30:07,207 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 06:30:07,207 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 06:30:07,207 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 06:30:07,207 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 06:30:07,207 submission_checker1.py:3345 INFO] --- -[2025-04-23 06:30:07,207 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 06:30:08,293 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 07:30:35,459 submission_checker1.py:3324 INFO] --- +[2025-04-23 07:30:35,459 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 07:30:35,459 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 07:30:35,459 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 07:30:35,459 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 07:30:35,459 submission_checker1.py:3345 INFO] --- +[2025-04-23 07:30:35,459 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 07:30:36,594 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.85635 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.50193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 79e6287b6..487285274 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1118 +1119 diff --git a/summary.csv b/summary.csv index 086f2fedd..7d0073a49 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437182,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.85635,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.50193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4c731e2a3..1d6ac6e9c 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.85635, + "Result":7.50193, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 9da32e268a7f32aa09df32969089a90996086f57..9de856de2e2cbe9b93cebed343fd39865b7332ea 100644 GIT binary patch delta 941 zcmZ1wydZc(5eLibs%3_o%QysC>({MyTF&BAclGbgL#yAc(>{2_M`1nxB%UTuBL=mT z3^QgNG%b=|l(0#T!N-|jR`tD3jDzhbPn)3;c8LpK+`C znY#{)Jo9#MM{z34*=?LaQx`x?Wdxfl;2U{OIdwYCS)f8{av2}@puB*@(Ro9oL(Rfw3D0!jOQw3WOS5e5bDwqoT6JaLjmIqWv~~+@ zQU7*>zdmD;=p)a~?I}zvzfIs-6zTXbQPN>Dr&+@?&kqZ28D^;+clTs|&J?9&#C3x? z;A(QHgIaNA!y!F%fjyKExZi`R4y>dZ|mxzmv=FbT}3cB)> z9JRD7>~xqzjIOh-FxhFhg?+)LCBHQ^A45{nvI_C~Q&mfNHm1b!Cq!<(5iINd-!npP z#hcH5Z*SY>zI|sE{yU;gHDZzLBhULlCu;tgz>^f|_%l)Rz$8u6Pw}=IlfNEzn<%zx zWpf|Xd-f=$vKgP`)4@UbKvs?O;QqJ()7SQ={eS9}?tB9&!5aHT#YHwZ@c7 zR~+4&|1j$cO0wkIXuRHNlrX8~!K(gsk^gQwe|jJADQjkesMgnYzZcJ4_Q~st{Mya8 z1wIO~{I=bEWAg=NO-9M;D50ox``RXo|5aaL#q-nrBgc*B`~I+;mESC+dWs2>B_F8U MGJ>^CR@AHq09;|WhX4Qo delta 941 zcmV;e15*5;Q=n6@atH)?1)+Acb_fv#e}U^cB^y=6uev`|c$M9CC775AzHGeC3ACNYIE_q8Teiqn*83=GljQvTu-8jnLtZ4F@tE544_FefF@=Lf128a zChGyvH1yEfKKOv9Y>C(?h^8#-kutcf%DR@9>MwUD@b5*ljyYDj(z-dY?R!esaXnSS zCcjboOyT`U*OzQ!CU7L1m_as42G}GSV3XY4MK&E1o2&<5)6hd_```ncvL#}p$eWE_ zDC^NOST-?)%ck>>VpBb7M_b1ne=9AU&hD}qHhtSIx!j}po&9GDZhXC*a**^Hd zrfiAWC{n=bLRpWN!Lo@VTsGZ*7@JSo($+D@O3S9Z!)87=&($TCm@2V$6zU+J)OJWF zklqf-Aet4-9hLz!nHmFVn&chY1E6X0cN7Rb&=Rpx5KVA+NEt4hq-$I@>(qF@7n+Ca z^{9QZ#|b=Db|Gj_?6Duve?%+9lV%Ue1XArG8AP*!+`}?}CKGG`O_Sh5djK>|j*pTC z&IPnYWE8LmpsL5JfK2A{0GTz*JdcnmT!L?h@<0Wjcgo*`vhDh9zr?mvK?#*8h7N4Nz%veP|CrrAhZuQpUM}mWYiaP4O<2fAvTiV#&N7z_Mns zr#hDYldwEc+2@_ww|&~$-F2?^u?$PFM2p0ez7NR+QurYm#Il0&!!m#+Q)=YDdQc|o zhm9gFbW;Z%1>O&|MC>r-KhEevS&x>%J4_OSf4!h;)_<0t@?Gk3!=+#{yPK{yvc(Qo zg-!hgDi78EQM>w&T59@k>6=S!b0{@>B%aiNNG6o}56J+OEF+<083`rJh=uNzHrYR{ z1IeVRgN_FI2U;RB2-Kjvxns0q2$#v!75bhUH3;jO|`Sk?L(=v z+Ys>*1e3ES^|R0;DFGQ7EeR#}U&F|Y{u59ghWwj&1nhHPeuCx?vl1oB0t9#kp?0(2 PCp`fKld~rElOZX8s*S|f diff --git a/summary_results.json b/summary_results.json index fe8f23b49..95f8c8154 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.85635, + "Performance_Result": 7.50193, "Performance_Units": "Samples/s" }, { From 23032c9ae0ff5cdebeba613cf5c1fb097e1128f7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 07:34:24 +0000 Subject: [PATCH 492/719] Updated results summary [skip ci] --- README.md | 82 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 550 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10656 -> 10656 bytes summary_results.json | 8 +- 6 files changed, 345 insertions(+), 345 deletions(-) diff --git a/README.md b/README.md index d76d04bad..a2d87642a 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 07:30:35,459 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 07:30:35,459 submission_checker1.py:3308 INFO] --- -[2025-04-23 07:30:35,459 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 07:34:18,849 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 07:34:18,849 submission_checker1.py:3308 INFO] --- +[2025-04-23 07:34:18,849 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 07:30:35,459 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 07:34:18,849 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 07:30:35,459 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 07:34:18,849 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 07:30:35,459 submission_checker1.py:3324 INFO] --- -[2025-04-23 07:30:35,459 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 07:30:35,459 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 07:30:35,459 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 07:30:35,459 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 07:30:35,459 submission_checker1.py:3345 INFO] --- -[2025-04-23 07:30:35,459 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 07:30:36,594 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 07:34:18,849 submission_checker1.py:3324 INFO] --- +[2025-04-23 07:34:18,849 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 07:34:18,849 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 07:34:18,849 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 07:34:18,849 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 07:34:18,849 submission_checker1.py:3345 INFO] --- +[2025-04-23 07:34:18,849 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 07:34:20,035 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26878 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.50193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.62032 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.50193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415382 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.62032 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420956 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 487285274..b1e1a0c00 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1119 +1120 diff --git a/summary.csv b/summary.csv index 7d0073a49..231865b8b 100644 --- a/summary.csv +++ b/summary.csv @@ -1,30 +1,30 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.50193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26878,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415382,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437182,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.50193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.62032,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0292,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420956,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1d6ac6e9c..4751f515c 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.425121, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":1.75529, - "Accuracy":"acc: 80.000", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.417637, - "Accuracy":"mAP: 49.593", + "Result":7.50193, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.26878, - "Accuracy":"F1: 80.0", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -213,60 +213,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.405224, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.415382, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "9":{ @@ -274,12 +274,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":23.163, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -287,8 +287,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -306,18 +306,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.0024, - "Accuracy":"acc: 76.000", + "Result":0.437182, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, @@ -336,18 +336,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437182, - "Accuracy":"mAP: 76.951", + "Result":23.0024, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.50193, - "Accuracy":"F1: 80.0", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":23.2491, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -484,22 +484,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -514,22 +514,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":8.62032, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -545,20 +545,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":8.62032, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.6529, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -587,8 +587,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":21.6529, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":21.0292, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -677,8 +677,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.425121, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -724,21 +724,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.0292, - "Accuracy":"acc: 76.000", + "Result":0.417637, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -753,60 +753,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":23.163, - "Accuracy":"acc: 76.000", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.420956, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "27":{ @@ -814,29 +814,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":1.75529, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,22 +844,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":6.26957, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 9de856de2e2cbe9b93cebed343fd39865b7332ea..3ca778276826d6afb1a8bb56e9b4dc9e92778836 100644 GIT binary patch delta 1249 zcmV<71RndKQ=n6@atH+G*dWcbb_l5mfB08it`n4~vRap5iT;QuwI7lRrS?NIh-C%s zhh+duk^wAD)(`6durx_O+Xo-81nGwif}$*x^+*|rvaD+mWp^fUAXxt8rfe-!tYXUQ zUFXMZYArCgiBOMJgNMUOemEfl0hyjC_gL%T#^iMX)=CT2f(FC_|YKze?Ut_ z20KX^B9+ZiH%pwBrEM>ca0ydn?XA$+sL*{$V7F-lg1CpgwpsS z86>lU@xwAeCdmMqCgF#50A!kc9}T__v_xbOLMHe=qzpH*ReT>^v+whL$UIcxNAC+i zZ6{we%P#!5C1|2J;z{9$WI`$YlN}O;f8oxjVeF$>;`?awF+I}vA(>G6J|u%^R`7jT z2GArKK+~lAunquClkKCy_JNj&41yval=WyCESkg#f7Vw++p4)v&8w)vd{jd%IO_TPcLHmK0hztUXEL8Pa6_ClKA0YFPq@TaRk*#HZm7omwp|o9Hwq80V z-j9O@i7&k$l2Nu4e@Ff^l!*fjL4;w{dY3iWkfcXP05j#xzImlfo z>(Mf~mn9+C%Q^>V{se{l}Z`=LmN zPeSrQm7gnTSNSO}X6aCFU0XX}f-#L3H}QEqnWWYr$xqk1a@)H8W{@j&{Bivy8V~V& zyks^U7oYgqpRB*~BOA|dtn|(Ptve=%83`e)N*_~YuJmb3An%lOA120wVmKMcH|ezx>a)~S_G z{XUw{ZZ%5r13Rp@6aWAK2mk;8KLF=)vmF($4g{4=wM~=I zBOiauYQr!PMfd#*qrELZ(wDFuXiHWN6q=IKoscGpsP#Zn>-g_0+liAvR$Y$noild^ zmGix9qF=C1XjP#sNl^rfX)e^ZLLaL|{D30osp7QJ3M$lrL-YEkWQH(p;mv9T)(db^ z04Yb9snE`QLojx12a>uZI8+cf+Dhs}X}5paQ1(r?fb%rH!xB8@)KlC8V>1!a0m<2f zG(WZ(0GwfHK!WlPXGw-nUJq8fs||y4=9I#B2CiC<;^f{tF&W#oP1<6xhWr`+T)w^! ztHwh0r(%HWk~6|AP_J!W;`1ULuA$B^!-+Qn_){13D{qHI#y;Rt2tdZTM`c|+zpNz| zsLs>;Hcn@8UaZo4Qp`y9_|=cR=syAFVaUIUN5Gi-@)Oj5vl1oB0tDvRAkDMjCp`fK Ll})uxlOZX8DgI&` delta 1249 zcmV<71RndKQ=n6@atH+LdZsb6b_l5mf7rm(f5em856Og5`ym;`vV!)* zGJqw?0G1}}hjjp0nxvoYgAZ7O^g{+gQ5MR2qzpt^)-{N-I}q=5-w34@uc!YGNDv{NCvsAp!~25a7i-2rOEhV9RQam;YWk;e*-NM z83biKDC^NO*uz#3ess;k&tqKvB{p6yldQCl-8Fu+ZU*h3Y$Mw)ArsXRPZ~cY6H4QU zWRT1X#t+K?nIr>bnuH(L0g!3(eKhz!&=QeB2$|sfkTTrJR`Gpw&A!j~A@fj$AH6U9 zv>h*W%P#!5C1|2J;z{9$WI`$YlN}O;f1y+MPP4@K(d1)#r0+vAq4a%72GOkG`>+h4 zNiu+@N%vtL0GcM-M}zGHEfE<6MLa0$(K1*xi4z2iX3e_Kx1;%#{p^8CKUaEJ`pFkf z+upguyiaC{^P|b9_DJW4WJ2lukPMPp!TDhsAd_T(Oq24%Ish_F#*YT$2U;RBe+VHH zj2}{li)Iz$N7ropd>=9o)%(%H`*C~~lzhU7F7bXe`4AuJ{g6y3y&sZ6G%I*NECXng z44`S!epm;9rb+wJp#4BgLGOTP};68TQ8jw z@5e!d#FyR=$tYWjKO_WHvW%pXe`O?<^nm2&;W;Mrhm9h!G(Memvcd=ce>ex{{ZJ&s zCn0&D%FmUvtNaufvver8uC1Lf!I;L2oA^AQOj2u*7 z(fVsdtodS`EXLW__2|X-f0!&M{j+H@{Bd(t7p zejm+ew;HARfgRS_$-I9Pcl(Q*@hr6_@hG3D-OS&BFV@6aWAK2mk;8KLF=)vmF($4g~u>v)Gf+ zBOiawPQx$|gzr2>qfbB_xVK+$?*WTEsuH4QbyVCvDSK>WSleX1|?X zxtecPm;8eB(&!rH89@nXZiLirjow#_^Z_M4FfCYTG}LGZ9?hGZid)K!gI8xPxFEqN z5u`okwnm!}EXCOK4JhWb=+IGI8>d)^rE7n&W&E49fD1xqSV3Td1%?M;Y9}H(AO)Y0 z_Q!Q2KyVBlDA2*zEJAc;dAL@wkinmAAtp`#um!3_$mBkIF~+^t>fn zpr#2 diff --git a/summary_results.json b/summary_results.json index 95f8c8154..2c384efa1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -631,7 +631,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -667,7 +667,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420956, + "Performance_Result": 0.415382, "Performance_Units": "Samples/s" }, { @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.26878, + "Performance_Result": 6.26957, "Performance_Units": "Samples/s" }, { From a5e9c2b33bd6303e3b540090fc90499e4e0fc6db Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 07:39:04 +0000 Subject: [PATCH 493/719] Updated results summary [skip ci] --- README.md | 82 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 550 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10656 -> 10656 bytes summary_results.json | 4 +- 6 files changed, 343 insertions(+), 343 deletions(-) diff --git a/README.md b/README.md index a2d87642a..6919c2165 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 07:34:18,849 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 07:34:18,849 submission_checker1.py:3308 INFO] --- -[2025-04-23 07:34:18,849 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 07:38:59,245 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 07:38:59,245 submission_checker1.py:3308 INFO] --- +[2025-04-23 07:38:59,245 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 07:34:18,849 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 07:38:59,245 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 07:34:18,849 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 07:38:59,245 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 07:34:18,849 submission_checker1.py:3324 INFO] --- -[2025-04-23 07:34:18,849 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 07:34:18,849 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 07:34:18,849 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 07:34:18,849 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 07:34:18,849 submission_checker1.py:3345 INFO] --- -[2025-04-23 07:34:18,849 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 07:34:20,035 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 07:38:59,245 submission_checker1.py:3324 INFO] --- +[2025-04-23 07:38:59,245 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 07:38:59,245 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 07:38:59,245 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 07:38:59,245 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 07:38:59,245 submission_checker1.py:3345 INFO] --- +[2025-04-23 07:38:59,245 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 07:39:00,241 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.50193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415382 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.62032 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41736 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.50193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.62032 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417637 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415382 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index b1e1a0c00..1afd47a21 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1120 +1121 diff --git a/summary.csv b/summary.csv index 231865b8b..6ff773185 100644 --- a/summary.csv +++ b/summary.csv @@ -1,30 +1,30 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.50193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415382,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437182,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.62032,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41736,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437182,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.50193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.62032,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0292,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417637,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415382,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4751f515c..3af014c14 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.425121, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.8483, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":1.75529, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":7.50193, - "Accuracy":"F1: 80.0", + "Result":0.41736, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":6.26957, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -213,60 +213,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.415382, - "Accuracy":"mAP: 76.951", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "9":{ @@ -274,12 +274,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.163, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -287,8 +287,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -306,18 +306,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437182, - "Accuracy":"mAP: 76.951", + "Result":23.0024, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, @@ -336,18 +336,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.0024, - "Accuracy":"acc: 76.000", + "Result":0.437182, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":7.50193, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":19.7056, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -484,22 +484,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -514,22 +514,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":8.62032, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -545,20 +545,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":8.62032, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":21.6529, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -587,8 +587,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.6529, - "Accuracy":"acc: 76.000", + "Result":0.405224, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0292, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -677,8 +677,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.425121, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -724,21 +724,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.417637, - "Accuracy":"mAP: 49.593", + "Result":21.0292, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -753,60 +753,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":23.163, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.415382, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "27":{ @@ -814,29 +814,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":1.75529, - "Accuracy":"acc: 80.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,22 +844,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":6.26957, - "Accuracy":"F1: 80.0", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 3ca778276826d6afb1a8bb56e9b4dc9e92778836..b227249e579c8b6b90e66e539bbfe01b6156030d 100644 GIT binary patch delta 2764 zcmY+Gi$Bx*AICS-T$)^xa*7$-np)D%LTk?@0HUt@D}0sNg%JwLYciriZl zL`_~y{v0u-0DG}m8D{<0Gkf;C1Xv%to-Gy$P8zUHtGZTfr*3>jF+t@&zmFmnf~5PM zwc>ilEI$9ro@+l<81k1~QmBhSE(yAIVt`G?7v=x6FL6ex`xzH;+w5L+s-xqgU$E`4Cpz+j2 z_mGbtFWy8a!_Muq*V^E)7Uv<0i;=~l*$EVNgJvy*T4fbuVZ_4gMslChhFx2QFy?84 zrtt#w_jLV*3uAlEew-e6_T2kNhg6T4id=D5Jt;TwXK!}rzFK89-*^kP5+2?$7@W}x z^M1Th`DO6ZEsESFy%9KQhw!?5Iu)J>&!e}uKRTaKMu8Al(nbdohX63PCnP)N(!NI2jg2$`1ne)D)NW#mD zR#t9drj4`k>`bK!`^~>6AVq*|IEZ%o8o!W46Y2s;c-7nEajLXtUP8)MTC-5sFz(UA zqQG#0H0Ye%s_%N}#^M}M&$9<1I$drE`t;KMHqBl}NVEufR?bnFs7>KeVn}SDiKx|w z(y^~h6ag?%u?k^Us#=#Cp3pZ*s_E*E7wYbbqi)7Op_{LwOx|-YSA@2dy%m1@VU*+7 zQT3TEwo2;jX68>as<#ucA|FZR|C4?&$(5yIWvQ}I{#vN=JwtKCeaKDH+xvx==UfpTg;Dh3l{6 z7;$H5+UT5ZFQ^?Wtg;2}p7T1vMHTd?7WA*dGu}S>iG>>ej=3rW#6T8Bu5dgJa?i{5 zlb^N?Cu?6!zUXtV6(AcK?HKzUB(q{eyL>#8#r+~AK7BR6PsRO^kDp^-3*mMldqx}25ZJg~70wVzr zBZ`!pC`PJjUga$_idioZ^OsSoM<>G{EL)*^O8VxUmoIJMk`u2c>_kA>O81BH7y^^J zp^Zh2ar*iZkb&%C%>j;Phd>e3Tk9Gf(VSSb7RGBC1EK@9A|IalhEo!QGNOA6qkA3{ zfA$#(fQn1yoht~I-xmR3W1qb;Qmz^o?u(ij8x_CO2u% z_4O;PsfWYn8#g{=o?SV&VAv-xGWAwQmjCrFz~ID>Q|{gG4)dqUwHK0-f^pQ1puItF%O zt9j41?-%1D2~r&=-#wd)>;%1`%!-bqzKM%@?n>N(v@qt86<8MeAr2l#K1ODGg86Xm zn70&Z=_Y^^XFkt%IhL0IvE$26vAA_}8KEjY9zNysGZK6_zq8GvHgH9AH9;JDRm1(* zmGY{^`i=Y}9xUZ~2w zA57VB%8Q?W;ucer++DnC$CkrLU1lwPUZz_F?0u~_m)isWKjgsKT4+ZLVu?zP`7`$ ztdv!7Bkb+GtlHmX)l!ZbGd#gfaHh6i2Yk77Q;Z8|BCk%eUT%|opUldGl}QtMfpgK% zRI^ySmPez%JQyEJKKmgaS|v}?bhI>FPIE!6H5|KJQPyx88^ZUWj`mcD$tOzM`Aae* z!M0meWlIR6)vEk7wZu!Za148QXy83YO10!;rB=pHTE0YMq z_NtTkY3s?Bg$w6ry*U6ws!Rp+fDlx3PRV@)|VOx-+;`h<3DA0wpoRhtAhL#~DHsev|n2&zaHDhWfeBAP<2KS3KxAKv=)A>Qe z(xo?gQvwMd)65%CRRU(94GJmRM1+;zLU`B>(XwA20Zq~g* zB81CFdX4nAKw(2E%JBD`3uihpRl80wJEf<#*eVICkB#MG#QI~T^Eki6ipHdx&_n0!#{T6UcN-@ zG#3X3j#wDyPYocgQ+IOS{RUt7cHHB_iLR57Ai)h>$#eW5><%;aX3Mp~fz7xj&a5ee zGWa1SLs~60QlFS#zH)x+ZIy!_Vj`V-a%9TiK#FO*n|oTQ5;#w-kZP$1KHIi4`O;jd zRm9YOq?ou$&xSC)b>0P;NQ1Rk=j;0wCgxeOl8!c>RAl~my0Fi2=0ov8L&hmq=w-}A zU&CkVz*$S@;{8O+=A8kiGp7B?vl4a@<28HX`_3xAXeS{$t9SMCb_r|h-h8XSXC7Si zlWBNJG`HmQTB=a4vgP@v4Vov(zmS}E11R%Qy|J2uB+6kZdcYE;xtoZv@o`m3)K{|* z-^tgS!dJpt`v&vO_w%~9(Pd!@qs~0nJzfZRP)FrglS%QtUE;7C`Xhw{ z74MaIy^F&qFXmIO8K3|1J^<9Y@efykp<`y@o6bnQ@n!l(_EWgoQ8m+#PT%qV+v!Zy zQoc&RC268m{_ufW&578utOS~UkO{0c^}GV$+_vLs{F-_%_7_f9T*l9Udj72gg2~$0 zN1b_Q)mIL=yeb)EmuxG7F|Xk~_Fk-iSVmrQ+h-Fh()QCT=&CfD3RmCMb!5cjOT%{H za+&I4M%ATsJaiHzug7}gIGh(2;ky^%di%iw`q0I}8qWuk`#09g(I QPzU_`#l+Tf%nWJ&1^ntB&j0`b delta 2764 zcmY+Gi$4?k8^<>*w=*V`mTE3D!y=;+vbmiZmPxsE%&jeR?V=)KZinf@Y`N~7+(jgr zh>BcFlH`(TkxLSCZ%i|taelAYdHueBz~}q^JkRHOzTaG@T&K#5l2Ts>R60Yz005kG zO)?ln3$aZ%%n6xP4;_24vRA!<&k0N3F_LTr4~47Msqa{nj+{`gRG%2@T2;2ZD$$X4 zU|R3W%nQ$W$rUy^-}k1ztbqO3tF;d( zH*oNS`AfY4A8~O-J+soNe*H9QRMbMKYoG+dfyULv>5m}A>G@&6V{LRj5Aft}pd#KX z-$qT|T0RviF0P2Lv+5=eH(j*SKbTABMf<2Y!VmdOT@zh8$JLFaL9So6FA}>Bi+e`t z_6ukvRqc~GdPB7ei40HP;rVpWTYWJUlTJ>xg5TKfShIWT6g4{aeMhZX!q4MKVjY7f zvqRZ@4o7iJQEadM3$vQbzE|ipwc4vZfH1+acr*af4#>=GI&@Jv->Ygrz}^zFPh%eC zj3pesn@5GB*t?r4y=%CKrFW_e?w?6cODFh- z4uaVYv+3@0j_YqX4E{XwW(b>)4}Mbn|D>HYQXJeOb=I-vq8;b1soA z*D3u~wgZrDe+y_xSdyIuzB61ik>0j{GZl0M&B(M6oI&V0q|3hSZ~FPSPD@^RU&Fxo zwmaHS(aKb7jbX2H0LG-5 zYwLuWp+G0mAH+)x>&&Vpa(fB|NUiaIV^kcrnOg8evIoS=jWgi1&+ZrAb$`kC<>Rwl zv!K~IV;6yIf`)6p^%(xM2vKXD0rqh-U-O1OXKEc<@;ruQ@mhXQ-Bn*2mn=3faa!$4 z<`T?ztAr!_5K>Kz9yH>i8BB-6n(pPg(Gv%=h6paogFX^v&Bak-X^H;7N=QQ7p~%`F zokcZAZ|^xK?=X#lE7%GX$pG8*RASZ5cqXOQV%3)Y9o8R1?&&kf#jS%5?0{|yGDXfj zM0+vgNA9ImxNQttF3l5{{Ik4vYV#@t1%DSW+4jNi+vZyRNo!?IL)zm{2vwo+Rk1%? z&;X5gz-`2G;$;9ayA>ug2C23p%ZX{wF@@lo2ju|+4b#~#avEW;L64$_r${q*`7Z7i z3b#wDPG@)}ZO}Cz!I)A-E0$bM@Rc@O~1jpv1Zi~i; zUR53-N27>_Q^j>-qB!t?Q@PLUY0_j{fWO`)yNXrn-FUl`IMdQ~hCMP3#x&Q}L1IuK zcP99}A3Dla;&~CC->tHawX+g8heBS5SA$n7cgDj=oQ37x8MLHZE*)^4*)yTHVE5VP zCP?n98oO6kms?pcMQ(fes!%v5cCiJGrHOV(7GhLnmz*5|lWEwpYk}aK6@rgcor)02 z%`Zp3GCetpoBs_Hr0Hqcp}YQAgPLzNh96BNTWmI5lu7<(CdmK-Yre%8U`VJ>5g!sY z8!dS+Y;y@!Qk~yLUh1sxs?A@6D?LcZJD)aOjey-P<$AZ%BIRe(7mOTa+88Qi>*jF0n2 z-`@59Uf?z+pf2ej>seLJB8ueVsLl(Ku3KxYw@^?iE8?3iXj+L$r-NhF_HV74Y+9>S ztTpHC&?L7z;&s#twkSvEnU3(wjpCsBPu|0toYuAOFHYAAmo3qGm!fRM+sjARE5Qsr z8e5EC^WG;a-8e38aWiu945^n8trly}^~HjK!%+>8ti)*{byR57p#3c9#F&t}q-xe9 z5P^kbw!otP1*7#A3b)8s3d{wYWMbD*CwZtWPhMVkmeA>(G1 zD8&|Z@>YuRSFq8A?E>!t0249a2K?V3hqP@yG125dA?p`p-wN_<5*#d+uxU9uzAkH8 zx@V_|!K6P-JY)55DIHn7YMNEpuhQ=Dq5dMbe&Z}d=kodmp~fBvZ8730`S4z)9R6TV zdE=70a#wKqk1k$DQdrw$W5APdfd}hPANSD_7^gTWFJ@fb0Pkc4xwhyP@IKI8mtPpU z-Q+x&>Ao?q?{@sBZ!eiYKAoG+SN> zat`b2FT~u6(vfku!~TRZM>EGKS&cx?^2SUuFC_hq3#LwH#)(O3|FKcaeT=!O^y1v{ zmOUEyE5Gb6*w%P1TwK9$z&jgdx2F*v5j0ru)L}ILmV!H2d@N5Fzew)C*_WOw4Ju} zs~w;bR|vJy%ZATd(CANh6YI4OAFpzpXh0(K{JZ@Ac!=!thToz@2y@- z(h)JflhO~JbBWe3%|;w0u`eIvC?wOy@?G$fx4r9!kNaNmDDSy;;*HG;|9#5SZy4Z& z_p8Pv-1AJql0qMiBd_sV?7#0PYeLfnD8o92GzDg`!ja~{0M=8aF>r{ribN=HRTc;U T5XAt19f1E%bW8)wNQ?RpnqMFf diff --git a/summary_results.json b/summary_results.json index 2c384efa1..23cf69ce4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -127,7 +127,7 @@ "a#": "", "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.417637, + "Performance_Result": 0.41736, "Performance_Units": "Samples/s" }, { From f2583949cda26aed0a3577071618ca54752a4d93 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 08:00:09 +0000 Subject: [PATCH 494/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10656 -> 10655 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6919c2165..af8c4535f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 07:38:59,245 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 07:38:59,245 submission_checker1.py:3308 INFO] --- -[2025-04-23 07:38:59,245 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 08:00:03,368 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 08:00:03,368 submission_checker1.py:3308 INFO] --- +[2025-04-23 08:00:03,368 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 07:38:59,245 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 08:00:03,368 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 07:38:59,245 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 08:00:03,368 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 07:38:59,245 submission_checker1.py:3324 INFO] --- -[2025-04-23 07:38:59,245 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 07:38:59,245 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 07:38:59,245 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 07:38:59,245 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 07:38:59,245 submission_checker1.py:3345 INFO] --- -[2025-04-23 07:38:59,245 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 07:39:00,241 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 08:00:03,368 submission_checker1.py:3324 INFO] --- +[2025-04-23 08:00:03,368 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 08:00:03,368 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 08:00:03,368 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 08:00:03,368 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 08:00:03,369 submission_checker1.py:3345 INFO] --- +[2025-04-23 08:00:03,369 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 08:00:04,665 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0292 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0074 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415382 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 1afd47a21..aa309ccc9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1121 +1122 diff --git a/summary.csv b/summary.csv index 6ff773185..9d66f4e1f 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0292,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0074,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415382,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 3af014c14..44f2e7d06 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0292, + "Result":21.0074, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b227249e579c8b6b90e66e539bbfe01b6156030d..5606050d3d89c458575e42e715b54d0f83cbce37 100644 GIT binary patch delta 368 zcmV-$0gwKmQ=e0?atH)F68YY>5Viwt$*F-tQ(Afx(k2nLevs5Ue)}qp<0Oz%kE@yec6L#<*sC`A1?Q#F zHOe!B642ZTskb$HUoX=+N_=2iu-0g(Q4bz1n!Acy%8i3pXDqlN!6y-|AWyAUkJ z*z+AI=CkO~QQR1(Scs+De`3q{H`@X(2zkH?0uwASd;q3)BBBu~_=L1St{nh^V`xEv z4j$)Oj?dlzPWhV+gL2_iGV~U1T2JES-g`M2yROT+a9^;;rkMikxB)vpUL5lm7%+jJ< zlQ|`XlG)c`I}sKz1xCY}Jt+}EF=`Lhuv%K`*D68YY<;3qu+1vJ!1Z6%W;DO3Rw OlXod22E!%*0002Me5(2Y delta 383 zcmV-_0f7FWQ=n6@atIFgjCX)?3;+NlW&i*f0F$vN6|KufveV|IJtLDOvbM3lCBu6A%BKH zS1)hFs Date: Wed, 23 Apr 2025 11:18:12 +0000 Subject: [PATCH 495/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10655 -> 10652 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index af8c4535f..d5e9a55d3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 08:00:03,368 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 08:00:03,368 submission_checker1.py:3308 INFO] --- -[2025-04-23 08:00:03,368 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 11:18:07,060 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 11:18:07,060 submission_checker1.py:3308 INFO] --- +[2025-04-23 11:18:07,060 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 08:00:03,368 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 11:18:07,060 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 08:00:03,368 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 11:18:07,060 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 08:00:03,368 submission_checker1.py:3324 INFO] --- -[2025-04-23 08:00:03,368 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 08:00:03,368 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 08:00:03,368 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 08:00:03,368 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 08:00:03,369 submission_checker1.py:3345 INFO] --- -[2025-04-23 08:00:03,369 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 08:00:04,665 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 11:18:07,060 submission_checker1.py:3324 INFO] --- +[2025-04-23 11:18:07,060 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 11:18:07,060 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 11:18:07,060 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 11:18:07,060 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 11:18:07,060 submission_checker1.py:3345 INFO] --- +[2025-04-23 11:18:07,060 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 11:18:08,135 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0074 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9646 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415382 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index aa309ccc9..b065a3fac 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1122 +1123 diff --git a/summary.csv b/summary.csv index 9d66f4e1f..98e283c7c 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0074,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9646,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415382,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 44f2e7d06..c5b342600 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0074, + "Result":20.9646, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5606050d3d89c458575e42e715b54d0f83cbce37..c9e6b3a34f36c58198a89276208d6321ebb7774c 100644 GIT binary patch delta 626 zcmV-&0*(EjQ=C($t1M~Nq)N4mD|?!7lT}>i5xncB@f}AJ}1?oy_|;aksy?8P8H{ z5|8qU+Rgk8_+tINB&(mZWWM^zZCIa@EK8>Mzum_DVZ11Qa~jqtNwWK2ic_-vewVZV z0kaGhzYYXSRMT#g&m$jy%WA_g5JmU>3ZuO(%jqKs+kv)Z)j**sE!_!el89OlB(;wJ zzOtP-31rpf=-xSVXHdD=t0wsc=cUmVB3X_S(A)^Aw-tI{FVi_nd|+Cz#%QQe2Ocfz zo040~je}QbEVv-SClRDQ<+eh*5G=*m^BpMWv*^%K+!&`=h^5<;Z6pVO^86kv2u!fR za1TuFL_`Oq;1kmRxMlzdj-desI(SSnf=}K6PWh`1gL39nGISQMT94x7-g`M2+qTWx z*OZp)CCUN? zbf8VSZ=rzyC^yV*^~AtCIm`U({7U>DHZ{`lQ$_o0S=RP MDI*5JCIA2c0P`L=9{>OV delta 641 zcmV-{0)G9RQ=e0?atH-G68YY8vv&x!3xDh?KgGo?+n1|kYv(&Krt#t?K947p)EXrD z=~`EATi0IWt^*4TGp<=QPYxrff{uU8yz8EKqarSjRdhsL<5heM+({ncn|)8~2CtqWH~eSfeD#?tdvx$@cqQ&i)6p2^GH%1T@r0 zZIjF+AAiki!!Qtp?|llRyDiyHOAxjLZON&DLQ`6L6VfITwSJJ)I)3{qj^iYdQ;(~e z{dRUywb-jR`32{t(KX65f)dc&2&uO1&{5nNr&x%k+kaxq_&3`EE(m$R3IY=>Fnj=}b|Rt? zDfoo6Kdv1Bf@5ewfes$$S&q-%08aUv4TEywR5J7yZdy;`O+kwLl+4niT$4E^gp%3UVdPc+8K}k~|0bRQ$K2PSp!ox{4kgM01v?V?-g1-O zCtCpwlQ}3n0+tq&uND@QjVLMsYLmDqIsx31^C%`8G}K6KB>?~cPyzq|5dZ)H00000 b00000fPuY}G$}p-5tDN%BL>4J00000fdez= diff --git a/summary_results.json b/summary_results.json index 113b8e3a1..4f9223c83 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0074, + "Performance_Result": 20.9646, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 91482fa99400698d9c82763a7203293bfd1dd74e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 11:30:18 +0000 Subject: [PATCH 496/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 10 +++++----- summary.xlsx | Bin 10652 -> 10655 bytes summary_results.json | 10 +++++----- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index d5e9a55d3..a4a48a201 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 11:18:07,060 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 11:18:07,060 submission_checker1.py:3308 INFO] --- -[2025-04-23 11:18:07,060 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 11:30:13,330 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 11:30:13,330 submission_checker1.py:3308 INFO] --- +[2025-04-23 11:30:13,330 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 11:18:07,060 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 11:30:13,330 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 11:18:07,060 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 11:30:13,330 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 11:18:07,060 submission_checker1.py:3324 INFO] --- -[2025-04-23 11:18:07,060 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 11:18:07,060 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 11:18:07,060 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 11:18:07,060 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 11:18:07,060 submission_checker1.py:3345 INFO] --- -[2025-04-23 11:18:07,060 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 11:18:08,135 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 11:30:13,330 submission_checker1.py:3324 INFO] --- +[2025-04-23 11:30:13,330 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 11:30:13,330 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 11:30:13,330 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 11:30:13,330 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 11:30:13,330 submission_checker1.py:3345 INFO] --- +[2025-04-23 11:30:13,330 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 11:30:14,544 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,8 +31,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0024 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.50193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b065a3fac..4707d3c68 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1123 +1124 diff --git a/summary.csv b/summary.csv index 98e283c7c..0db659dbf 100644 --- a/summary.csv +++ b/summary.csv @@ -9,8 +9,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0024,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437182,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437182,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.50193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c5b342600..f46dc6aa5 100644 --- a/summary.json +++ b/summary.json @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0024, + "Result":23.1815, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -319,7 +319,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -334,7 +334,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -349,7 +349,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index c9e6b3a34f36c58198a89276208d6321ebb7774c..c82564b92624c4630c91f3e06df35515faa2185a 100644 GIT binary patch delta 3632 zcmYk9c{r498^&jlC7v zmz9x|*gG=~n{x+a;K;Yf=lBZt*SvIVf6QIbwec=L^07hO2`LVvFu#9oV9p-&l@win zf(Ep{+Y->xliMfDQ?Gqsg|6wK_cNxuZh0>Pr``{bbOmPjI zK`CifbY{p2(TFh0NtF?mJXUL~d7!RKj#9AW^MIFC0{PDZ?@p>p;;V^~jw7!4Cemz5 z>=^1i8b+?_^h*kxR+AJ9uMCfU#Sc6fn|binGpNd%@-ZectGeX%){hSb#4-0FrX15I z`)$q(~qa!t#Lz&}lzWLP+M>bq5j|o3^SuVkC;pOx_i~#+oW%`>= z--fgFLz(I#1G7so8Og0sAxOmdVIfF6%R*-5pvO`Of^3n2bd2~5K?aJnVKP5<1#sBz zRSsF#sb&Zcd-s*4%-r-!2KIHt41xVoGdG!PZp*-SlS3qdcYhDYE0aS`Fgs-<(;%jm zTZeGi4K~-aMNPdp0Jyc()-;gbhOMR@oiktSvYUR>wi27%lzs$~Uv%QJa!Qm7Y^&f^ z>9w6-tuC z5gtb5)cH+SjBDI|W$+4_1nned&t|+!CE2`(w66OYEW( zXC-9gWN>E3%uR~TWQ01~<75&ZL5#dMG6}?>2SJd51K#Yg5gl;!|l0=kTi%#h6yYMCWEG8>pIP)ztg{xF^BstFt zBmvJ?28Rj^y+V1NR}zev2389Nm^kC?45_+@unbJ5s&3TFBkIR}1%)m;3Kp)U zx=B2SdzSPlBi$7k8Hm4WOjU;K)!f%p==SRdJ2lYb1_JYuk5F(?-{_cfl3^=6_H6dc z${LQGRc&Fn#AR)Fw|h2xx%vk0jom9FI;tAF+$_BX)S~#k9zjMq!ZrMJ08dQHCH@}^FHNYS?3ibSElvp zQGWbt*4z7+_N&To{x#EoOAjaK&7nF(q+DqqKVecOkHN^G@y)yQ04J%A z?M8}JS83Y&<(U%ON<|wR-zM6xJLa-z#PIf{f69C@AN4HBnx(&d_FgcmsTCu9HePmu z%3o)WXlFC?7y8{XBkHxZD3I0>y|6}Y{Tf9#8LbD=)wPGc^jx{=a`!{%VJ!QAXYz$U z43`^E0!Jo3Yf;LAo8dXY-DALw-+Ax0`aT;Jn@@)aTWY8$`Vjbw{>b&_B?bOV>%XOj z&|&WLjT0=CL4BI0)1(%CkM2D<7CX_1E-qFGJ3M7qhs$fv3h?BffQfk!F>%&k2im=) zK2hZu;Bk##gA>!+?PUSDaRp$wZTU}KI~sA)Q(B*a6Zy6}!(+8w7WA@aig&Blz(+VC zS1IGpu}P^%4R_4s(kBIIQX~=Tscf)Do*1VOY3kG@XOZu4#If!3gF?0TN3HE5&BTK8 zOZ}&`7!8rFqNLgyUPPqPn+PglH>5S+YbaEKhyJCj(qvK!b& zg*qGZWhr1I9oD4t3|#eLiI<#yM!Rr7#k@FT=X*QGN6_Gj`Ifyy*PFO(Y^ux!=IP80 zGcixvTgmo1@7i)eH=9MQLE23$6;8g2nVkF_{$jJ1-?GLeg-UGwqj6hWj9Qnl#+C?o z^jknh*~u}IQ)S>`jy$IhO$#=d1d!Hdyu%(>n)ZDWk>l$?UA5C>p!#fg6vN-lI-JS- zZ0*c5F~{}%U0EzgRcuUK{O{1Qc+Bvqp=shirz%8@D-fx`s6}`XlD<5{zd*AljMrA}*8I>OB1#)C_DTxN z)+|P$RTf1i_o&2gg0v)u18K3Ox`bT{0tBalGUe8j!$Ec=D)h)4TQlY;2ONtk z)QBWO-Rpm@pU44lsm^^e%7r%~QlVA(hUXQ|_cU7en1oY_U2MC<>^8yb6POf)0ec|x zr~6eH4bb^uiJ$~RrR~NnUq3X%D8Dgb!%whMpWslG2$_s$30Z} zWv_<6h!${aM_OibVXwwM&YJk6#BeJ<8*43W#m>kOVq6nrR;dga{yKcqtPYU0<&B$R6|Dy`aL=A5EBps{lT>dRtxv!C5fE#+L43TFU; zzre?Oj-8J|4IJtu73zcQk&>brUB+Jq=UMH65rXD>JK`!6u4B%8y&+Hwfd28k%@}LR zJSP_=pbKl&0L(5IPzoF-%z^%Ig+ zRT$in+m({PjeKO~;Gez_@u}%Hy@}p=CSAT#^G9kfniD?%=5v^W z(}N>*_Mr8AB5eX(*AN^y;(J*r(CsY-SxKcupH!N zZ2G8f?4FX%94y@t|2*&D&K-NrnTpdphnAiUY099}ga)U0E3`FteMFaKEuuYKYd-iq z9Rf-A=Qlra>rGA6zNzwBszV-9s<+s9k$zOAb9`M@;cNXrgZ5X%%U!Tf=zvGxwl*Df zdTk%9%xhKC{!=n{5{FGQ>yR%!xb>O$sDv@kUtvFUF delta 3619 zcmY+Hi$BwU|HkKh$T^2p%wd=sYTp&*Fhb_Aa;Q*XfBt~${k-1S^?H3C?;N`Fh9+j~L)fbv}c z;23~ZttN$oYGV9+=d=BS#@mlSZOOTv=eC=dU24>L>7G12yUoj`N}+ON@IsCr_rH22 z#Y{h#vzSQ$cg#Pxd_K`G*DP52XfY^dQcjbm4V-E%;ptb(8!<`oq%kk!b1P4!`gy7M z^%^hug<}rZ!JZCY!#ToxB$RZ{`Q~=7+2@pltne#1XuRH=nNIw)QD&)l5g2|-p@OwqISs5?v3Bo#!nk- zArkkzZ!X)|n{B_$+3dd}P4)1pKCYO}h{btamh;KZNi`bSy*1!;@|F+ne#n&T{06vh zaw0uwu;KK@Nzf#f$RJNg`EJ+KK6fQ}JhaXLCO>trVkArp9`&<7sY<}?B{4uVt7Zm? zh$`R?LTUOuS=%}K>c}IJa2K7I?f;i zbKp6{r8-CQUUI)zq?x_=mrRb`+Hvd^rI}E;ns~qmmqa{Z%U;53)y##6|F$Sa;D6yl z!ee-v6vYFUbU{ilT8=#P40xa{A?{IUsWkI*3t^HqQ3q1$`$vUzj8jgG_rbX0#y57fhXPo}> z#1}ux5hLC}E{Qe`>E~ zGiObCHIGnn86B|$o0#)4GCT1a&3!Yl#w*Wrie_s@_X|e;_NV!yV|85oaFm;${9g^l z^=WU$>hx-#_41))UKrmB*H7zD;N6(Ffo4WHLOdLsvoDSpRPs@MVa;CqUyrpr;`Gc< zz_V*Eqgq)N@#^#RVI6qQ(1At#^2BcamoU4z#`AnTl^bQFDAcj9-()_`kia3g0-h*cqH3~*mgl5Rw! z)ppwm>BMln1iKjapPh-7`FPO0VtyHWhV8&aO0aro;8#_2BXGd(R=UiXLZ8qN4ayQ1hSN*wBx_5ad&)u#nM6RK8>Ydo}kLSE_$1D z_4Q=NFz#%VpqR@OU9rkkW3wRMlwL)rH%A0N1?^>_I0FkPk!_z6>&>BUQq3CUE;_mSg-dz8Y9Eqc9HNfpKNfRL|X=ws;^F~CYGv>VS zL9&3Of7M_1Hb-P6~%!)SMGf-=$sYD|v)g;%8lGC*9Ocb?{!B0b4C zc>vN_G9$OXm zAJyDLRl%3)-C+en&yj=YG&5HYxDGK3(UhhZUtYhJ=Ng$Rr(!PEy%$!=OcQWob0!}0 zne&dKSVB&`fkYBxF}KIL*q)!OwHs?PgzLju@7h(GrD>wS;J2*sJzCvLD%8e#URg^S$^o&oMB z(aFq7m$Lr#^(s z6WYrA875#*!)HjF+Xj1mb279vxy$N5*DdR*5qdn zKU};>%#RlntuikD@Jz(zswhR}#_^%`Z}3_X#hf+=&UM2dB3N=3E*eM6FBMLP;A&{s zTnM}Q*9DSG5;3++#dPaea8Q)>4QreOZ!SnSauCI;5&>9{w)2hlAEn}jF6H95*P@eK zw^Blc>GsZ)?G1);Xzp;oZj%pF@&Ml*@>t^fjb#bQ0C25f%NUyX1&iRVF3)9F3Avg> zBMYg!ZgPNB)kTlJZ(OMg%tRyWWOmWjxolEHgkSz-&n9HID>WI?F7jx z&_MfDB9T_4)%;e*AG@90Q|wk=u8mpE_t}bQ7EO=Fo!b8qFtz6P29aQ*tVNUo! z%}Taz&mJmO2c8k`oe+$}B2<(6M5zktQdz)M4yZYACrlk)&ZR&@?R%cV9#&2a+PbZX zuJ^og78~5#-p^3nuxqGZ5qakIuAb08O9j2*HODp# zr_hHtWulx)XA_oJKV$F0ShndJ95~ZWJ)#DN0wr#Z zjf95d#_=sZ8y+QTz8{ZcWh+`t+cLD;sdus>oD*EaXx0|v!@+U*HSOrZ|ftGuud57-51`z!)yYT zKYRQUPSy&QP7+CgJyEJjsR+C{>S8nAP?)e zqh+c)JPU;bQc8~T5}&{Fm)1#z-PWRAun2d(S=H`YV$CP7v6M-?)fE?I?*XY-XjKz0 z(4Vip)F=k|G`b-N{!aPv+2-r9Vh4O79e1aDTbs_QxxO!dn%}DG{e4lZ6%!M_f^SC$ zS@m}T{PB4lJQp!m(8pIN0atd)grfq#UxNlqSKdsRnVWogL6%1I$qL(rCJbC=mXeJ|Ry-9XG-$grc{Y~FqA_d6Ar zmkph{y1F#djEPd&$h@cTVKenYpPpW!IjF9f2h{MO>weQ{cZZJJOt*J%<2E w53I*QB87sDI6O#=;ENn%q%E-Oe{NPmU7DkTq^)kiAwqhfuFU&h6~O-f4cis(e*gdg diff --git a/summary_results.json b/summary_results.json index 4f9223c83..302766cc3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -869,7 +869,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -883,7 +883,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline", "Software": "onnxruntime v1.21.1", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0024, + "Performance_Result": 23.1815, "Performance_Units": "Samples/s" }, { @@ -905,7 +905,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -919,7 +919,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline", "Software": "onnxruntime v1.21.1", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, From da8cfc2464ad21958d1d135810210cdd972a3825 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 11:33:46 +0000 Subject: [PATCH 497/719] Updated results summary [skip ci] --- README.md | 82 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 556 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10655 -> 10648 bytes summary_results.json | 14 +- 6 files changed, 351 insertions(+), 351 deletions(-) diff --git a/README.md b/README.md index a4a48a201..f96b1e7e5 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 11:30:13,330 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 11:30:13,330 submission_checker1.py:3308 INFO] --- -[2025-04-23 11:30:13,330 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 11:33:41,015 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 11:33:41,015 submission_checker1.py:3308 INFO] --- +[2025-04-23 11:33:41,015 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 11:30:13,330 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 11:33:41,015 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 11:30:13,330 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 11:33:41,015 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 11:30:13,330 submission_checker1.py:3324 INFO] --- -[2025-04-23 11:30:13,330 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 11:30:13,330 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 11:30:13,330 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 11:30:13,330 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 11:30:13,330 submission_checker1.py:3345 INFO] --- -[2025-04-23 11:30:13,330 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 11:30:14,544 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 11:33:41,015 submission_checker1.py:3324 INFO] --- +[2025-04-23 11:33:41,015 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 11:33:41,015 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 11:33:41,015 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 11:33:41,015 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 11:33:41,015 submission_checker1.py:3345 INFO] --- +[2025-04-23 11:33:41,015 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 11:33:42,007 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75529 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41736 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437182 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.50193 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.62032 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.3525 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9646 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.415382 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9646 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41736 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74623 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 4707d3c68..706146535 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1124 +1125 diff --git a/summary.csv b/summary.csv index 0db659dbf..db67d4be3 100644 --- a/summary.csv +++ b/summary.csv @@ -1,30 +1,30 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.18883,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75529,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41736,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437891,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437182,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.50193,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.3525,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.62032,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9646,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.415382,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41736,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74623,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f46dc6aa5..cb249cad6 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.425121, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":1.75529, - "Accuracy":"acc: 80.000", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.41736, - "Accuracy":"mAP: 49.593", + "Result":7.18883, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.26957, - "Accuracy":"F1: 80.0", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -213,60 +213,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.405224, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.418237, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "9":{ @@ -274,12 +274,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":23.163, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -287,8 +287,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -304,20 +304,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.1815, - "Accuracy":"acc: 76.000", + "Result":0.437891, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -334,20 +334,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437182, - "Accuracy":"mAP: 76.951", + "Result":23.1815, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.50193, - "Accuracy":"F1: 80.0", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":23.2491, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -484,22 +484,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -514,22 +514,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":7.3525, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -545,20 +545,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":8.62032, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.6529, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -587,8 +587,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":21.6529, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":20.9646, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -677,8 +677,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.425121, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -724,22 +724,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9646, - "Accuracy":"acc: 76.000", + "Result":0.41736, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -753,60 +753,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":23.163, - "Accuracy":"acc: 76.000", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.415382, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "27":{ @@ -814,29 +814,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":1.74623, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,22 +844,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":6.26957, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index c82564b92624c4630c91f3e06df35515faa2185a..c4a16f77a0e62c610c66ab4433f3a9a5efea2099 100644 GIT binary patch delta 2834 zcmYk8dpr|*AICR$lG##m&|GF4aU4r@n5Kx$$n6wPDfe7PH}{9lrPGYuw&fm*NbYm% zxYj{#xur~2lv}Z+xo#NG>Umzz>-ppN=kN1<|3073_x1V}IA%H4+>($HKVc#tAPoSh z`2Ya&09q~LFzvpCndq7C;W{5_A0(J0LXbTQ3twF77!S_7ivLhk!q+W5Sn3{q3{=N@ zOBhGf15A?rZ+4$Dou_4g5rJK(Qfn4|hlWp&GB5HJa z`x+u?uCBp3Ic;dC+oUL^VMuqN(3fB7`|HB^r2gd(hR!^r#PYDPg}nN2i7AF8N*H6V zjlfk=JpE0-ew9Z+JmwYazj`rM(0Ld>S(OE+gzuzp!Dd!0Pp4sPpggSFCj&;)WwI_a zR)+5UE4&LmZITnd?>p03L`f~_nzCX=Jq~=r^mv>l(QO5_X(j1%fQMGkpWLWza>%S& z&raGZ3zx{g%2e&iwm&qPd7|HO?FzH&l5GAX%wUNP7Ev`g=8LD4^QQyAt;II+Tb}d}5`Pi@p zM-5Zk&2-80VJ@zY1?L?h7p!j&1x{8Hz&+)j7t2rCG&GO$N{HZ6o+>;r@cao&R(`ek z8gJCC9c|@Pr<+8Y*B(f9w9I|8p^}~a(lEI|vGWu=72!8qN*hlgOmw`hE#95@<+I}@ zezn(?mGLzPwwXm;WmKt%ae?P&_0XBxO(peqZh3eL_<4#sEEJ|zs}AM>qURN#sL#(- zZwVeNK5$=WNjH?;4ZS15X3gw<-G<;C)bB>&OEj8jcv6Q2?0Qi3PW+6*p#7eR&xg08 zHZpU&0U6q2P0X<7#gofLFB}#_MdjGeR-$qkt}(JPzN9oTvq=7ioUpk)Iws$<>nP~% z?R$Vkn3_EdOi`m?<%ES3i!VR&MXtF0{zzWlt}D`1FAYikrFN}NV~pC$dZ_o#KG9xESK|@bIzE%Di+E?<8)JJaxKud*0_dvHO7-cS@!D1?*(-k; z%Wm8BywzYmWc(gq9twQ;ETDj9t+H_QJSs+Y085f_v53Q>`j05LP2QDHtQYTlR7M`! ze5sQ>0;#p1JAigYveQw##Mw*Z4N3KG}$W0b_gkcCTH*>XvwP)+hwbstLn$D`*o(u1^XZr>}R&mjd^ZwV?v_abhD~(q|lVN4!h`Id_j4u@s>rbRDpw z=qTOW#6e8=DWhusz(%I=0q)i#m!4lp<*h@_*%Onpa;vzAVPmeLbBSl|JSoFsTnUCDXQ-c9+2d6 zmvD$lqj6_HR&Z$3V9g>csIS~4v7>t5`V(tDNM#guy}3yvC+o})PMCt}A+Hl6gN>3l zv)>5GWH!2bwHMm6VdDbu(11UnQR&iiM|*%k-(_ z9Bx(40P1JU3;X80QJxeaFv8@%82<=f*5arYEM5;*fyoK1P)+erJYC@DXhydjo71n1CF32YD0Vn zk}d$Vt>=NLfY69zs*X=Yre)@1w@p^)9pc`s4Hl7dx+)H$H45Ss&zhlP=}}m*vWh}p zjkI0gABu&qi?>IP#aPjni;9zO0|jB8i=6q{%Py*)NsRa4726PPKEgesuxi-2L3gLV zp{4xtvZq3Z8wEl^gg^y|D`q5Ag16|bfD!NWrkAu&I7H9iV5UUKZ;)mmb43e`r73V0fOggv}^dirB< zkT|F4;K9}5V^k73-DyCm+^#I_19VyczLVHvtysumi0xAh?2y$4*-N%xY6&y|pu>di za$Z5lfu?>Cl70ea1d@J6g08ZchC0>GsQScx8gPWPM?`|&Y)6!aRTj)l-w>VDjJQ%3 zBAM|eHQWKWqnd5!kAcNIUmPJii`Bpsp1=0u*T?|ywDljo!>18@RgnDx2k5cLfo3uL z&}%^PxcRF={o{--t&)nV5Mpb|=Gmrucv=|P>U_&9cjht#Wv|RB=9r-#>JzbIeHCuI zHPUG~lxYc$>#hvQhEH3@uL$da74D=-wu3Y~;|YhSEty|;qM#_y1@7q1Kents!$C;7 zX$uVmaR3U@JRruT?jBC(15oFLWnI_S=$vDuAUf2;^=PcvXvN2!8tFnDDy9UtC6Ld7 zvn=D6f5?lRLnGOvs0n?_A(334!RFznA|8G1YpfNYv+KM1(Bw5w zV1tps2Cs`Z1UB@-6ja8FcWY!~tZAI07u;ASHZo5p>;qpIMI=aBEjSdlW)|6trJ zrk=+VmGV7NqzNPhFKCbWtN(PgR3+?Th2_AUOU9j#+H9pfp8B0u``F^h*Q~6DipqZQ z_&XZ$tB?E_DkAedWn;F`bOz5Rd(m|oppdh$Lx@pxv*w<5#)q$=3`oZhzh`H*cL?n{IFoHjsDKLDb%le$Z2z z%pE{?kQ2KNDn4b7ZsW}>cpiD$FW)gYzH~Q{va}KWIzQCz&68#Ggx$}mJr4N=Xo>?8 zUMVfm;nPC==kXn>3+0!R9d@6WgoL72rn^4u&G?}4l{j?&QGdfV#K)u!Cbfb%MCC0Z zn%twU7~0uW`=N?kWX>b*pr3OQtnV91^Vhyjw(HQIxE)?tL=FG|NC*xF@cCAq(&1_u zktKx0K;5wVaiFlqgY-H+Y)VM-e;--e9Ks1GN<$;jz*JfU@(j?E)`BzwzNKv<;Xg0Z ZR5XtO8)=4`r-4$mP)&8QR=7YM@NeV$VM+i1 delta 2870 zcmZ{mX;c#08iqm5;e;Bfm6_szgwrv}tekKv!8xs*j@2>rIO|DGG8Ly1vK+t>HBHUR zaKh43LrqN4$}tmh$f3-TJXj7m(51WXUF-h2d#$~H>>tm5zIT0luU+6)>Q;YMQp(_z ztJPHy0H8^}A?dRRh-=$FN1l@WfM-vZW3wEF-{WhRPPXeucTv{ci;aqJtz9>L6PBJ) z!{$XCJP;Grbx3zaw@^T*V?vER^0mX=o`gKBktwPUy)mJul37Q_xs7;bwoqo%6UI;` zp0LuI?x57DX?+#NXlisqGwAl%%nfgae0x_+9`hOE;o&@|ytaI4 zD7LGo6_D4dsk`2Nu;mm0t6r%FEsNSgIgyWWvoaTbVzqGGD*&bvV~N zl4qbWFuMp-SJ{Y!fMdq-dm-Qsft@;cx7Qv5HhrlM?i>k*fCtJ>z|_BO$q)%WDI$B@M`-i76tQc|!J6sOgnPTw1y1S6&w@6`4*o^9)O z4&g=bf5?h%*)ddp*gbpXB|Wm}nvOSv_qE`?wv$^;SQP#7B}QNZ0pndX@@A-Mk<%HM zU|67^Nn$PI`+ZOKug;GS!K2O4{uMmO`Nh{q-)+rfCN`HUkE<)Khezvbp_*RvM=$); zDg4m7);7A8H58&Vqt*Ftahy*m!PT1e5cCDw_pXT5Yk-N$MD#M+#?U;Hcj#U^K0 zSZwE2zx1v2saT$b+N-T5oM|iFhBQi0n;``F!8W*UCp6>ND}kK!}UWH>kLPV7DvJEXRd2pDr|Tk((Q39zCwwWt(_| zakfH9(>hy|?c3`K%06v56nSTJjunvszMO3XONXg@G6=M8be)yx9qL8;s=;WCRZ`#0 z5W=|R>PhYN*h(~gA~wCSDUJ^r+dKIbg&wZJ;NN%UcWgh&tcupH6G?TSl}UZV9ULmQ z^ox{q=O$b6#?6luS~x5eV-v0O@Y$-gJFskQj-GMsGiihUfwGV@Zn7oZyWWTIYx-97 z>X`Ztez4NzaZyE%^1&{T4FwxO*tg1A%ez4=I*(&4golhx`70;n zyi=A=_;}xRk}B9K-#&Kp0L#ZcnOJCXQ3a+bM#cghNWh~8DK{33Y&x-KctF7|6=OSk zLj+I{taSRKJJ-rHGmrU_lzO>Qc$Ql7i$=Ho*r3+O_Y(b;gQ+yXFnE>GhkHdyE4gjA z&h6AZ{Oh-ww||=v;r=3eLkv(k2AK$F0w@J)0YP#Us)WVJp!Huj=0zzqVKJ%{8r8P% zW3{%hlE>{rdi#O5c;hjTPgWd%U-pNb+WFY(R7ZjN(y^Q2s1`a_?pTrrivglxkR3v1 z(ZX9F?1(`FGZtVZZ5GvZ!n|I~Tixh(TwTLnKQk|JHvCouJ4$dir20rn0LSCXgU}Jy zlu=hzc&ldq4SD>PpM|eB`ll?)G1JlE_J#&20VGfv2I-#&_m`xE{vX!}Hq2+fi6uZ; zG-e#XFk!^rX8RA0C9s-2%gYf_dna9J#KMl;5MOZ?Oz|=qo9Os?pu4R1^)MOe#-^mIV&S z$ZwhqLK@EQb9BL>6~l@ugC~tRPjU1El(g!#NAYz%Ql{sxsRcMMH$Vq%bQolRS*DmP zS548TF?q5}7`|Kxz>)mX`Ub$zF(Z>C{%+4x8?1c2L|JtYzBRR}t4G)d;1mt>C7uI6QM05Pa#8h{FW%%Ib`N)P1UL|3?M5R43}9 z%W)2>-lvJ(SjY4be%4}{j$X{idsjj}xjy$WdTBCliE8Z#LeASU?N zJ-n_G=dN+vM#jd_!j;&{6>H@EAMt4MA?nBNdQcfNM47LqKC!S}e>TyP{H?-rBPoyI zDCZyyb_h9soG35W>i?EL(A>z+5#=U%q16%8SQiUSba3&;Vkusw&g!eCByz zp~E&1DT~?WE9tVl#@wa)BA|Bc_a;tbf<;Vz0ZPUgMmNNx&#+wayJ~7>BtKZLuEyg^ zIg&}$m8Uh2oAoWsT28aZuelPY6}l?tCxYx)$& zSR)C`{s+I1^7EaD8}nsWbM`A~!Z*Gf3bBIe5QL^^niP;EY^J*+Xv!O?#GC(cGlg8d zZ9jnd8^~yr?C#t8A{hjdOB_WHGpq=Im_7Tc4F&p_gh?^7a5nMKN|MAuHyhF9PYVWtoi-D zb6D3_4%QR-@+a5mo9z5)UeQSX84PLar|YC9I~;dv8|q>&DkcvA03?Nzd=LR?)KIuD zBX$WXLTv2S&;g2`(rj#8hfRz8&o_4Jh&vEOK8`R2=8>-=ETw~#003Nweq_}35Wh$= zP)nXXiZ~=PsZ8c6E0FhS?Nj Date: Wed, 23 Apr 2025 11:39:16 +0000 Subject: [PATCH 498/719] Updated results summary [skip ci] --- README.md | 88 +++---- dbversion | 2 +- summary.csv | 50 ++-- summary.json | 552 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10648 -> 10652 bytes summary_results.json | 6 +- 6 files changed, 349 insertions(+), 349 deletions(-) diff --git a/README.md b/README.md index f96b1e7e5..334cc9b45 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 11:33:41,015 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 11:33:41,015 submission_checker1.py:3308 INFO] --- -[2025-04-23 11:33:41,015 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 11:39:10,635 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 11:39:10,635 submission_checker1.py:3308 INFO] --- +[2025-04-23 11:39:10,635 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 11:33:41,015 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 11:39:10,635 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 11:33:41,015 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 11:39:10,635 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 11:33:41,015 submission_checker1.py:3324 INFO] --- -[2025-04-23 11:33:41,015 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 11:33:41,015 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 11:33:41,015 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 11:33:41,015 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 11:33:41,015 submission_checker1.py:3345 INFO] --- -[2025-04-23 11:33:41,015 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 11:33:42,007 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 11:39:10,635 submission_checker1.py:3324 INFO] --- +[2025-04-23 11:39:10,635 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 11:39:10,635 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 11:39:10,635 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 11:39:10,635 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 11:39:10,635 submission_checker1.py:3345 INFO] --- +[2025-04-23 11:39:10,636 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 11:39:11,624 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.3525 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9646 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41736 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74623 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74623 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42098 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.3525 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 706146535..64975abef 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1125 +1126 diff --git a/summary.csv b/summary.csv index db67d4be3..c8fb5fecf 100644 --- a/summary.csv +++ b/summary.csv @@ -1,30 +1,30 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.18883,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437891,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.3525,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9646,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41736,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74623,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42098,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437891,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.18883,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.3525,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cb249cad6..aa3f16479 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.425121, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.8483, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":1.74623, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":7.18883, - "Accuracy":"F1: 80.0", + "Result":0.42098, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":6.26957, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -213,60 +213,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.418237, - "Accuracy":"mAP: 76.951", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "9":{ @@ -274,12 +274,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.163, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -287,8 +287,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -306,18 +306,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437891, - "Accuracy":"mAP: 76.951", + "Result":23.1815, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -336,18 +336,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.1815, - "Accuracy":"acc: 76.000", + "Result":0.437891, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":7.18883, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":19.7056, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -484,22 +484,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -514,22 +514,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":7.3525, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -545,20 +545,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":7.3525, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":21.6529, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -587,8 +587,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.6529, - "Accuracy":"acc: 76.000", + "Result":0.405224, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9646, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -677,8 +677,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.425121, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -724,22 +724,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.41736, - "Accuracy":"mAP: 49.593", + "Result":21.076, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -753,60 +753,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":23.163, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.418237, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "27":{ @@ -814,29 +814,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":1.74623, - "Accuracy":"acc: 80.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,22 +844,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":6.26957, - "Accuracy":"F1: 80.0", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index c4a16f77a0e62c610c66ab4433f3a9a5efea2099..5edfd201c249cb4594e8edac829d25a62bc0173b 100644 GIT binary patch delta 3323 zcmZ9Pc{mj6+s9`tgN_VB5ympaF}7*4luV9gMwU3T@0!WTP_&rBjN-(kW0;YhvCFO@ z2`QZ+%R!8#$(FrBV-m9e==WaN`(E$!?{i(>`}?`S_x)Vglje}+P!lRB?yVRuVkL&WI*~|d18ilIW0YkbGUrdjLsQFvVz1<|h z6~Z_7z&4ge-GWu1_C{^Fq{oDLBhRVV5$U!SO@yQb8au%B%<@CYK2YZkqb9?k2*Un% zHJnSpF#YH@4I@R@YiSRdNv++_im;U;6}||`awXjrsQpnFl8IfNang_*YiAKZTF-M~ zBup}_co(*RJA`#{A-)0WA!&0;GicN+@FS#8W>G%jxZ@VuqJXPFOcG9E5mGA8xiV?Q z6bSL&xtx-}YiUykq>41k^Egz=-~L~J?u4PvxcQ$de}+t$>HE^#=cN1OlI3?4w>xtx zK=KR5 zP+?7ufJWOGsUydNoMlzXgl$sN26UQ(GEPL-KsM1zQ+f=2&)7>N_eGM^v{acPE#V06 zUOzJybqr!dewALoX+3IJ-Da&T_9)#rgRqaA=2e}gUHf|Fda+jam($1T4qO?ct{x7VQ zrkB>yW?ADeph~^pRLF6)+b79{9sULG@hkzkhVT5bV%|p43nxOD5#EV$X`E`P()_l# za+lPp{h`Q*JLVBK4@I@9`VYVN?q1o@82k$#_W}4wtda3F);zT5uGQ8PZBBQrG9>%e z9%ruFxj%EFjZ(gBmqFsh2Dh`9zdPJaw037%?o9@?jC7JR-o^K?5%eA%-%QYK>77P2 z*{0HqXX-xe6pn-ivx2el23|q6Zf(J3Lx1<`uf-em=<5aq)@B-hT#q=P)<*%Y&e_8= zBVA$st}WSTKNeOAbm9>$0Xt(S+FeOTmM4+fbrzV`y~+gjInwB`0kUpr|6;=8_*TQ< zJz76L@SL#09piVss3m6LvY|x;*O@D;93@9aFQLu+P)gJMTV48gsIE3|&H=4RiiZ23 z_tJ1YQc!I8McXQSO@&%$A-tI)o8n=swJVd<{PNW|@S`vUYV%~;ZD*9eVd84FE91nN z*W{Ptkva+qWK}dOX;*ROcS*aQPn_HjA{jaLo(X<$?1e?mxjgq0#JrnN^y>1cQRB># z%%G2;O7_CSYSd72eqM9*T%4cJ1+hh7+l74eHD=ewLD8{#Ptd-+#A4b-ehDNfygZ*Z zOl-pF%Y;gqV(jgH$SSuG&SYKTbAgI=Dv{Arx}WK_-FCU<8G$%1I$Zr|1!orHx=tI(wOk+3r9S&eCZ2iI<44WCWoO`i>q zG6lgbah=UG8~#VCpEXtS9Xox$`79bPsM-lggYx?Lfl@ny;vRI~1Lzz*e=E7U`UX%_ z?4-Gu)AAp#l$6dQYK%|iSo#~q2QO>wq)3=+O{F+?=(xS!t4 zF7ZEdeZC>b>_9#R=LhPCN?)L105<=MBAyG`4D;p&e9o(lTmjm!#h=-W+PDrYV{h0$ zxvi-^=>5(5(`5XiWN7y7n?9p2lB;I!mBrW)Gk>JNZK70ZvzgYP(0aPjc&ad|#uENU z_2r9r{8?2!SMZzSyx89LRslV%71JW=g-ny(Z~?Xk?2 zG2}I)vByjs@QwRAdUQrn<&}gdDK=rTJrTh<<(C`J5wf^?dJ>r;=imoV)pG-J<76A6ghyzT0JiQlvei8hCK)Zb?`rJ%YEK zXZc{<1{?XTexQs>cN4AEoh-d)^Hc{^7`5N|o!+yN|nHGuz9t)FgZe8Jc4~xkE#Xnc zog2P#fDF}fRP;AqO)+UcMUVZZf#B+KZ_g($`z{UaMmN7IM=d=>(0(yV2z^uaU8WDgEZMmRz z{Sfhil6>+SP8ycM6nbD@d*E!eKcb7cexZo2@k(P?BFy>${Me;b17t=7j&kcTfZl@u z(n*q82LZq>K^+M7g8&p*m>rzXuK+~htv?m94LB!V;;o+cz754yyH^m+X2?76#f@5( z35qIxO8M0x;D#wY?kYbg77MVU0ueSJ4*=gI0+2FwzW{SP{&1u?0@JOxfN-$o*5*lB zw%fe3P}dE}#nGX27Np_5Sj$ubQbY#Y@`H%BVE?(JsF-^&%THj+rG)}eq)4A`Iub8I zE-H;q;GGB8nJwwel_-QA%v$%XS4xbJ_M&XorgBWP{r$pw13~|~%{yU#TU@e4-Z?FN zt25K8gt--Br@$>*Glh4hd;|+AQYGsSfT3(4kgCdw{hP#t&S-`RG#_~V;M;Xcj&u01 z0;!V5EAI>qII&97#zLLEj>M`g&$*I@w7G6D47VJ?+m7%WV=izpRJUj-(tYu5(Bf90 zN!1|t-I_kmh`BmNW#90KnD8et_F5=-qo!vA$dO6tv>biEYB|5Q_474Da^=#Zk#P{y zn>4?9?DD_QeUH9=JUprB*K>RZqubk3h0<=J@@Ti-c9PZ%gmhx(lia?} z3UP})fOIW!GG5)r88;27wSkvvdTr--GaE*|#~%Bao*|<)&6cn3?S((vbD|FQxx&47 zXZ0#BS*ED4wkLyU_H?`?UK<__Oop;|MYP-0*~F%9_CCPQikFWjswPvbESV?5_pT|` zpVpzQ;9~oaj%W5ckUtf~a{5Z1xp(z- zw&%-?<;$5n4^{_0%Ot^2Lb%|V!AzM**uodyixEC)yLjwP8cY^tP+ou8q g0PJ`G05QP-o~ZkcG#zzgpa|`{x-#gE>fz0Q0G*FZSO5S3 delta 3298 zcmYk9dpOhm8^Tjl@3KP%CR6=(o<08_@dJbRr%MD_)gEwGp z1fp%EZ!35B6RuXB^nnnqrH4MQGFKA4YX+?yZUHhqxuZF`+a{Cst-Q$o_ifRn1XiIVr#Y)HL<3@H{wirY$~v(=!og0n1&5mame-z zuUSn)BP2GabV>>0|L{+k!s}mG#UYN{B-73@R)Q&-9k$Nk9*h*Z#tiF-(frl^;HmD- z7LIJ$wxc+N9vTaAkB8%dKIux0ajd>I?D z5}DugNmb{oAAi)ic=l&OyWL_SuLRx6f>*+jaUM|@Syb$oRv>v#g4@&<9rpZI=W)>| z51xYJU<$S{NUTCEMuM9=uJBsECt}6rT)w2_rOsf$NGl0JI#IRWqBKhCsq6nb6M0h> zR}x0+ak=XlFgbZQ%{N~DwwCEJ-it$8w0QH5b7o-|#go-ZF>VmWzGI3lLWfUNf|>7o zF3kw?svf7v-&A}z_ey;!S%K+G*YKEJTf{kO-Wz4hVTo|oWzid4D-{Nr)T(A7vFm@G z7u&G{dd8HdUQo_OmIi`fH2CC_E#8lZ60jQnAlQzNpEPl|2+OeD^Ul5T19^XCw zDZtW~ax%=yL^S7Tf~ap_YD|K{jQ6rJo|xKR7c2i+I|q=+TEZ$%>QSA%rUC-%2Pzk* z?(~-G$F)`LTh2}`+>slB-EFK_%1pm-U<@W@bmR=rK%Jl!l{?St$BXKgRj}13PM^iB z@KilE4VOnQ<>t+#(es z{YA}>0yy!Lu=f12hu1`FOx?s%R`bew`ogf{5ssk&18YMkNkDY6V?P)9z1Vp`x0UjG zmhrdGF@Q}#h<@C8T>Pj!mo{^ccT(l?^^%8zsXvH8cDwQ!m%I&OkxmZ71Sh^qnADpO zZmdcX5RSZYkOnS^@J*8sWjpqft=j z>fh^nnl8(^weXF$Bvs7c8)UzdiR zjdn9u*-H}6$6sK$N*R&eKvv|H4m}Xcs*%X|h5w2IQsD2POVk4QN-u|DPBR;a>hqbf z;}xkF9VSuRA<*KIDc9az*rb{~IfRS25sHDQw2Jo1WgHEGrh<`JzIUiagEDTpQMX;s z%FsL|g-AdQ*}f(3-r(g>#gVHU+Lm9x^bo3H(ABfrkt5nMVP4(2Ju+_}T)4tvUXas!(If05j!jt83V6A|evc4Q{Vdxm_xT7+y zCD><)cBbuluD#Izz~m&V$4L}$bGXNe^c_q}ZnSW(QY6fhyjsxC2v3a>Hh#x57o9R& zgzsXEoRv15iGFXgNvQcvkTFE)52XefW`;Z=ZV)(x(;Pz1B5()~))nPtSjKHMTFCMo zCL9okDIB`2@4)MZT;zX;hAn=-d(X`We?tk;5sL70TmdBUyHL0@jiu|E#Q3l)FXfI~ zjgAVlNKO8O^0bg-jUmW9zl8)Q5aKw_K=Z$cdd5zm7K-H_y(ly9pU0<0eO0H+WHS|` znr*`igFj5A*Ory{L8za|gk=xOA0*{8>)6e?JR=&89_^N*Q419}hN*dSMh-&}1e4$^ z(Le$hCbJpJ-~34pRTUVs`f(e`JXLW z@ZI~4WUofIi%zeKa<3HUYOu%;y%%t8#BZfn6YTch6g>9OWY*YKi53(>}^@U zkK78)-CXcYU<$IYw6+pUi|kfxy1-|jeN8#<(%w+?Htiz%T2F{m_mK*vm}2=Qo$17Q znH?jT7D&HSUnO^89oJ-Jm3(~L%K#@CPf?0WRmw>oBg|jkmwJ_G!nQ(#MgFO%7E&7I z`o9xoR@o8EOFoA{a}ke`D~=r_Cm>Y8x5y0$J@6QL9RZgq TWR!{lScv>UMUk%=e(>`jPJ~>Q diff --git a/summary_results.json b/summary_results.json index 6d2a6c76f..dfaa4da42 100644 --- a/summary_results.json +++ b/summary_results.json @@ -127,7 +127,7 @@ "a#": "", "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.41736, + "Performance_Result": 0.42098, "Performance_Units": "Samples/s" }, { @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9646, + "Performance_Result": 21.076, "Performance_Units": "Samples/s" } ] \ No newline at end of file From d57080ef3a0f6e0b2d3fe302b3803001b1714a12 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 11:48:48 +0000 Subject: [PATCH 499/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10652 -> 10653 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 334cc9b45..a5628826e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 11:39:10,635 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 11:39:10,635 submission_checker1.py:3308 INFO] --- -[2025-04-23 11:39:10,635 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 11:48:43,121 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 11:48:43,121 submission_checker1.py:3308 INFO] --- +[2025-04-23 11:48:43,121 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 11:39:10,635 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 11:48:43,121 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 11:39:10,635 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 11:48:43,121 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 11:39:10,635 submission_checker1.py:3324 INFO] --- -[2025-04-23 11:39:10,635 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 11:39:10,635 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 11:39:10,635 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 11:39:10,635 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 11:39:10,635 submission_checker1.py:3345 INFO] --- -[2025-04-23 11:39:10,636 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 11:39:11,624 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 11:48:43,121 submission_checker1.py:3324 INFO] --- +[2025-04-23 11:48:43,121 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 11:48:43,121 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 11:48:43,121 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 11:48:43,121 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 11:48:43,121 submission_checker1.py:3345 INFO] --- +[2025-04-23 11:48:43,121 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 11:48:44,189 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74623 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74019 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42098 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 64975abef..eb8e83187 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1126 +1127 diff --git a/summary.csv b/summary.csv index c8fb5fecf..0c18334d5 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74623,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74019,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42098,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index aa3f16479..2a9651e12 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74623, + "Result":1.74019, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5edfd201c249cb4594e8edac829d25a62bc0173b..df0f19f82a07808096eec295d57debe338ed14ad 100644 GIT binary patch delta 1585 zcmV-12G04MQ=L<=atH+{W{0eBvv&v!2!FOJd2^|44y8tq#FP3D$%In>AsK*@Wh9g= zBcWs&vCzHJCi{nVAel6E(9t0OKubghK^YIqdaMkXBq6x!qidFb_K^HLjJ#SVScOWx z>;7n`sk{>PN2?MjQ6cf9`a?3IRDVbYp{$_(una&+G61E?{9zpclqT^l(JNx@Pg`?}ze0t)F*#*ZRpfuqx0_$@@yJORz+L#FN?&$%In-AsNK7 zg7(8QfF;QQmL}_mbpTkJq@V4B4_JcqLk2-n7Rq|03`AMhHHfl16F3ko|8i5dmMK;- zW%aJ}<2AJxx@BBCC0wF9;z{L)WPd`b{E!TCSwZ<>8Q_v+fJ>9{!#V&iO~Q``;RjkG zG6>3eP}ZYmu!pT6{OFp6pU1fTOKiMaCRu48yKDSt-3;1g4!drbkcsMuCygJH38nEv zGDv0x#NEvQqtN1>;X5Z)gkbilo!jIk; ze%emHXqH|0aZAudam16t56Og5_#qiYvx4x$GJqz@0GcM>hjjpGnsgrxx(~EOWDr#G zpsYvBV9~6g`{cA(MOpkMw;=CX~Jp$sm~( zd>@tpGD!x=H0eI910d66`+sP#eV`>GgAg*o_911sXjZX(bj`NU_aXC8bssH0#mDhO zr{rUNbcyey$*1&4--l#E>HCljqFKTBVHrS^WB^T*?!!6&G)=aT2HOW(A~Fbycu>}( zWw2-xCkPhJnsuLVNAoHB*#nh+uJo?-Q`|8L%1k_6;{0gxp*_<1A%B@rIzJ?XWL9u~ zSO&->86eZ7{ICvyOq21W!T5oehzvr=1mlO4;i6f^_|Y{RKi`MUL-l^N@O~UWDBaJN zct4tahL7}qNG6ot56K{!6}%so0W?Vl&@^d3tOG#Pr2S~nexM~HgMcCnRXtV(WHRXo z$UG$J=WlRiYnfjqD1XC!C~a5o7+L!tCSBtFIB1ag()%G9WlQmggn&wxkyNsbq>>(x z{5U+vWd5*GB$lQQIu4jW&=Rr3l%Im!g|Z$klY3bbg1xMBaORH?@PfdTYnfY> zEv&OA{n*Vg&?Rp!of7NEL4m}V)(^=jTWUWfgh*Dl&cySY- z$CF8F4U+t9tqZrU>#qg5QpX?H-=Ogj&xcE9!*TI>pZ&@D`#!Sq?8Zv}Tz`{@cYOZ4 zwu{aNqpRg4wtup(^Z2w8f0^e`P%+nrHT*JKe}{-QUyPH*IQzODz4#K7<)nW$O@=?N z4r)0Y|GA8R{9*8ecl^V^`{`#(|74w7`ONR5`RrDs6hE-TIy;&7Z{lu$aWkH!)+8R~ z6SbT9`|rj2TS-(v!?1AAieg!!Qs<_x%c^y)D05g0LNEOI8gO>eAAkkS30(^*~bV`0uMY ziIYH9U5@UZGj|46%bo0_U$9PSRii9PQ3Q%u^e_jVO8P~<#Soc3BljYe>2+1ypk z5T-4>T5Z630WJz4Qru8Jw?wu2}aTtbV zC?;#jpW)B-%iFYSEY$u~3{X>XMwkWawQVYVS%ky&)cJKd@kRiD9D{!2?YPLH4|o&; zkVD*)@=-iJw?~cPyzq| j5dZ)H0000000000fPuS{G$}p-50i5#BL=}H000002WRgjD(V9 zB$O;87P?p3WdE=ZB$K8NIvV63Xo<)mDC0p{kCg$FBm`G|bj|Y59+H2DkypzEt5B(T z-5>2Vl~)F(DuEIe5>Ki>Boj*Yhhz}S3hEEb0F)#HP@2pi)&W3i5`Q#^KhP49L4OF8 zApVdt*uJu^Vf(6U7JvSJC=b;7d8c=+pL_$W0_~K%uhhB(OY}!Psr`^lD77DwK`bk1 zKP&@Sk_=#JvVK?xfTcawTQ+uIX#-&rjC8{HyRDMV%lz++($sm^%lpmG>E=dNsG#Njv1K`pm{Adtvjp5sE&Bj_#v548b2h1 zWL7YKSO&->86eXn{ICvyOq1`U!S{idhzvr=1mB00;YPNK@1tw>eZCKwhkq*k=zZa* z?c|GQiB+K85;Rd9@ucuWGNBZHNCwfYApEclph+@-rpfnV9RQjp-A9A&11%951XVmJ z>(Me;G%M&nx@O(yThV-yi&x7eD=nIPzK_$??XcuNiS|m!B%i<|eIJqurSC&BNM;4! zhh>0Fk^wSJx)19B$TZnL8h>mbXo<)mgiNq~NEt4gRcs$!v+eVJ$UIctM~hGKas1FJ z`4}Hv;`?awDLvBnA(>G6J|u%^R`7jT2GArKK+~lAunquClkKCy_JNj&41yval=WyC zESkg#fG5 zk0zhtBfTGz38nW#GKgjc?}udoO_BjLP1+CZ0MImPKN_?jXo<)mpvXd1k5vJgO!@&b z4@vs@8ywkM=2r>IaDN|4+toWp*8Yb{mv}!88YI5-en>{yQv4wypps=Il`JEvqz5EF z4$m=}KWr3r-ryzHstVhe_UY3MlFY6qf`6Gn9QGuj-(rwII=2m43 z>+DHCb~6lg$(u{3#QJeiAn~R3Lo&*i+7AgKk`=TcmH{N$0e=CKChv!h0!W&?9|yc2 zXo=Wi%1<5dLRpWN0g@Fy@W(kg?}s88J_*SKRer9VUFD~^nB~h4ENJa~1I9F7+{EYc zWRhBgBtKj0!fosNYeBBm@yGQyXgtL8;gZ>KTzuYVf3p6*k8C`7PxL;g73> zTF%CQF5@4682sQJ|1j`=`q|PyS*KP$^ZRH%yVWSg5A3kcPUiiaxZ7XcjAyAeiAVWF z?PmV|d$ImjlGV>yGGG1VHmpxcmL=2s-)`gnFkTeDIS>tNlqA{xFU2X@e!I)r{{XWJ z6~7Jy$Gewjlg%R^f6Y$AFc5_AJVoSt;y3{WR+379dZH2%MO54@d)r#XKe7#J-X14y z(^l$<<9cSlon5(_?Npomg7ebo8j&nV321JF)SDW8Ea&M1N_=2iu-0g(Q4b!?nwyGS z%8i3JXDqlN!6y-|AW+Yl_p*z+wY=CkO~QCu6RScs+De_+e_H`@R%^86kv2u!fR z@BmEhL_`Oq;1kmRxOM~xj-dqwI(SSnf=}K6PWh`1qjKg{GV~U%T94x7-g!A0yROT+ zaatACJ*Hrkth=*&L4~ux>wFL3hNBzp%VUc|wh$IG} z`?yEtvwVJ8CC*V(lNOV(78a9^C@KMGles850ojxEC?*8QyO(E^9w`<9x|202 RJ^>DsbSWbSz$O3y000Is>JI<_ diff --git a/summary_results.json b/summary_results.json index dfaa4da42..a5130e1a5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74623, + "Performance_Result": 1.74019, "Performance_Units": "Samples/s" }, { From b4636a6460912fbc08baa7a8df50cc6229dfe803 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 12:11:31 +0000 Subject: [PATCH 500/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10653 -> 10652 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a5628826e..b6c1392a3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 11:48:43,121 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 11:48:43,121 submission_checker1.py:3308 INFO] --- -[2025-04-23 11:48:43,121 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 12:11:26,577 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 12:11:26,577 submission_checker1.py:3308 INFO] --- +[2025-04-23 12:11:26,577 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 11:48:43,121 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 12:11:26,577 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 11:48:43,121 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 12:11:26,577 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 11:48:43,121 submission_checker1.py:3324 INFO] --- -[2025-04-23 11:48:43,121 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 11:48:43,121 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 11:48:43,121 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 11:48:43,121 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 11:48:43,121 submission_checker1.py:3345 INFO] --- -[2025-04-23 11:48:43,121 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 11:48:44,189 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 12:11:26,577 submission_checker1.py:3324 INFO] --- +[2025-04-23 12:11:26,577 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 12:11:26,577 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 12:11:26,577 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 12:11:26,577 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 12:11:26,578 submission_checker1.py:3345 INFO] --- +[2025-04-23 12:11:26,578 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 12:11:27,536 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -47,6 +47,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418237 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index eb8e83187..3ff08f728 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1127 +1128 diff --git a/summary.csv b/summary.csv index 0c18334d5..1297c2dc2 100644 --- a/summary.csv +++ b/summary.csv @@ -25,6 +25,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418237,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2a9651e12..c7911a8fc 100644 --- a/summary.json +++ b/summary.json @@ -789,7 +789,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418237, + "Result":0.420623, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index df0f19f82a07808096eec295d57debe338ed14ad..388a82afe24175bc2b89a33012a9220b6e05f451 100644 GIT binary patch delta 1325 zcmV+|1=9MRQ=C(6!X>IBo>YEF zCX~t#$sm^%lpmG>E=dNsG#Njv1K`pm{Adtvjp5sE&Bj_#v548b2h1WL7YKSO&->86eXn{ICvyOq1`U z!S{idhzvr=1mAy$l;K9UitnRq_IHCljl3Bs`VHtlQlVpHQlkUSh05VOsj|STZ zS|TzCArovLQih9W728MGZ2Np4G7nYv(c)8l96xkQKE_9v_&%C^N{{q?NG6oN56K{! z6?`9-0W?Vl&@|~jtOG#PWcz5aeV`>GgP@29Wj$I3izab`V9~5u_xW}-pR%7lQ0eDN z?@B+#9g}~c%*4|r&W|P^+9RDGk_n~rLo!Ha1?PulfJ~ABGEK@4>j20!89y3~A83il zAcRaXen=TEnpKP+U9<7?eaJji??(&o$MJ*G{cMT%qseFZNbiSaLh1dG45C@V`(YVC zlVku*llH?p05nb7j|S}rS|TzCD6&x1V^u&VlYW1I%tMlX{su?3mibkJGTevKcJ+>t zwf|w#CEkyN28l1dACghF6n{tvsAL&QCCf-E=>f@)!*fjL4;w{dY3iWkfcXP05j#xz zDac(Y>(Mf~mn9+C%Q^>V{s9pVP}ZYmfMkUa{BaJ> z`=LmNPeSrQm7gnTSNSO}X8H013tBthfH931H}QEqnWWYr$`a z)~S`x{63n`ZZ%5r13RpKa3Z}Z%)G+B}sPwOL0oJ-|lkuKeGxIzYYXNdMBxq%_A#+%jpYYJJ6P_ z8YncSp*tZ>5>e}cq}K7@SGE%;fvmb5-8*OQ3@Yb))g-^*yfnH(B+F3(nj0bYwn87P zMf!*mAD9-bF&Zk=fk*TDrsS4#}6D%;?15-N@(E%y=gtR}d832M~Xh4Au9+QmVlQ)1<{%XUZ zoH>;YorSB`qd2+uUQWigZL@YZSYv*I*UQ)UVbxUX{!|=LT?$UQ0~U;{OMG6$!!^u@ zWjyglf_Unpe&y}3$k+!Wi2>*s_o#f%o?liAR2O-1m*%%=F44BBN5yKa||iRy?ajUSQ;rSU^DNM;4&hh>0Fk^wSJ!Vl{J$Tayr z8hjsUiO3*?Oz?e3e;ICMtN1>;X5Z)gka?)WkKPx4+D^V`mRlY9b?^nFMsl)ewiAej|>AC>_!e@O<&H0eI910d66`)IIz zpd})M5Hi8`A!WE|RD*KO}=>R&ah;2FN5CAk(D$unvGslkua$_<@#) z3_{2RtqFKfG(KQ=C--paY^?tPQejGn2-OrYIKbm}okMw>>CY0U}$sn2)ydRbU zG)V@~G-*Gq13=TH{b^1V=WlRiYnfjqD8qdyZCCFY zS^FO*UE=*XXps2Q`ym-+OYw({;*LbmZlCm4wygC60yUS zpMu1Rv- zWSv_1%{g=`Kd{3(JDK-y;%vBdT$8Ej%`6wmjs6j;znCZ zeJE{*4Q1c71DvPn1D4<^f2W?}Juo&i5gn17%}DcO`w74qh8`p+?{Jo6`0VvyrMuZM zDHl#D{Al2&^(4;jofETh7=~mhCTqx_;m`HU+q7yd)c#ZqP*ZV6m<8&!Z7O_Ogv0gJ z`E@w)MgV^tgMQ=fxX7UocoYJVL)??{Q9M1jE7as^ejlf0oEL4D7?E;8%HnH3@~Zy~ zl*b|eCY}I?+}EF=`Lhir%K`-_W{0eBliep<0Sc2jC_Dm`7L%_Q7L$!ADgkJdxF|XS z+LQArCL0Ocx6&m6002+|000pH0000000000004l2yOT62J^>Gtb15SR!6pCz000NB Bgs}hs diff --git a/summary_results.json b/summary_results.json index a5130e1a5..49d7711b9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418237, + "Performance_Result": 0.420623, "Performance_Units": "Samples/s" }, { From 750d7c97913864132bf50d550f370a650c080f62 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 12:27:12 +0000 Subject: [PATCH 501/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10652 -> 10654 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b6c1392a3..aec70d8bd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 12:11:26,577 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 12:11:26,577 submission_checker1.py:3308 INFO] --- -[2025-04-23 12:11:26,577 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 12:27:07,854 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 12:27:07,854 submission_checker1.py:3308 INFO] --- +[2025-04-23 12:27:07,854 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 12:11:26,577 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 12:27:07,854 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 12:11:26,577 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 12:27:07,854 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 12:11:26,577 submission_checker1.py:3324 INFO] --- -[2025-04-23 12:11:26,577 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 12:11:26,577 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 12:11:26,577 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 12:11:26,577 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 12:11:26,578 submission_checker1.py:3345 INFO] --- -[2025-04-23 12:11:26,578 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 12:11:27,536 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 12:27:07,854 submission_checker1.py:3324 INFO] --- +[2025-04-23 12:27:07,854 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 12:27:07,855 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 12:27:07,855 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 12:27:07,855 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 12:27:07,855 submission_checker1.py:3345 INFO] --- +[2025-04-23 12:27:07,855 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 12:27:08,909 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -39,7 +39,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.3525 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.52106 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3ff08f728..6f2c5fa6d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1128 +1129 diff --git a/summary.csv b/summary.csv index 1297c2dc2..ab76d713a 100644 --- a/summary.csv +++ b/summary.csv @@ -17,7 +17,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.3525,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.52106,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c7911a8fc..c0bd9d256 100644 --- a/summary.json +++ b/summary.json @@ -549,7 +549,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.3525, + "Result":8.52106, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 388a82afe24175bc2b89a33012a9220b6e05f451..e30787f1d270df36a8fb78dd6704a7d3b7d9e0bc 100644 GIT binary patch delta 1407 zcmV-_1%UdTQ=U_>atI00;G!6D3;+NkvwH}c1b<=BbhWZe7*sZgaMdm-6wmE*k_p6t zoRbW2$TAW{mXRcqjIu>XueKq^BJDxKXzHP}eQ=8-v_xzad4;D7Wj#_RFFC$tUCRp- zmpcAOM zMSphJm0)5fke3HZ2Eim50Fz_@Ow8aKxeZY;Sq}iFp@)u=Hx@3SC1Rt<>#AKS>(Mf~ zR+11bmOB$TL@;+&%7RR7H`No+tYeN3n?nj`~gk_@1U8G@!Zp~-pxGz~p;wto*kpeb7-HVUFC%X*{?uBx)G<(2x&oeBJV z(X3;RRW7t{4s83L(sf)^Frn9?jhE3mg%Mada z@*Dfl6y7g)1BoWN>yw@iLNG){QRo)9p=1*?fh*a>46;cwz$VE6o6Mhq%RsJ)P1Xaj zY3QM|eei)z*%Gl)q<@nT4rwk&j}v&R>_X6<*keDSiB^aw z%^s2oq}oF=h-L-3hh+dwCfER)Cc%gH0BD*VA0-W(3uuYRC}0mjRgZsF0h!F@0Wxcr zc^)BCxCGx0<$(%5@07m>W!v@Jeu+P>f)XlG5b>nsLo$K1d`JeVtl;^u3{c5j8=%tU z`p_PLN|WxRq>OU`EfE_ya|Vl6gIVWzAwwbu9fSVR@jk&pWkm`?R&Y>s;+) z8J1v)7KtZ)ACd{A@Ix{(h-C%khh+durqsxP^q@@E4;w{F=%x-j3cMd^iP&Mvf1A;T zvK}picbFsu|9C;wtdn368U+2LcJ-5f5H}PX56XJ1445P#xay;8mVfq;{5zDB$q-fn zBeNh8KLG(plVB1%4`X~BKXghy$48g=KALY(F*`2#HxJ52d8leiOC0b`R8 z6eTuafiaC2H}QEqnWWYr$(PsElbRGp0hW{D6e9v(@3RIK!VUyFT23pI%p)s* z+vy_++kv)Z)j**xE!_!el89OlB(;wJzOtP-33%1z=-xSVXHc;^$R_#)>x5P%%90dC zpqS=D?Mn1suj3_(oTrM@Mk^>$2M(>On}Qj_w1roz4OlP0MFFH7VWvcT?+wA&u{}uY zlHgE5+-fVS52f81G=S<-#e2?+naZPvYc0I58R9woTf_U=8^*{JD8~8&-{l>QBW0RRw2+S)g9q zs=${;I9x-W--Huy1@Ol%=reD}MaDkhQ3ycBxF_Xf@$_7;QI)6pU7X&=`J&Dla*{ug zbotefyy`y#<#EWri6_9A`}z}9f3prH$^r?|;G!6D3;+Nkli?><0Sl8kC_Dm{7L%_Q z7L$!ADgkMexF|XS+mrJsCL20hPAeq=002+|000pH0000000000004l2ypuF3J^>Ju Nb15SR!X^L!0042We8m6& delta 1414 zcmV;11$p|OQ=C(p|gGP0e?-|60uPbOWpGiIbuBN{U+zrc--~7) zbF6Zmb#q|bx0J5qda8s?ew*}}!n=>IFWJOQ;7B$xgKUxvut_q&Cb_qZY&s@3Sr5Rb zp@+`)!3Q>FOTJE_VZoCb{X8pbkQR#6(f37Pz5g6ElG;*~AR8Nix7D$pD*7pMlFju8B?71F&i6 zp|gGPflb*Gu~DRc(}l7gErVqfL%3|Z|1dV6vZbwKj+K^8cZbb?o8$ zJSpvvOdzctl0h^pcsncuXfiPd&@`z#vX;r4&{ORJ@1si2W8v!+kS~Yu7VOO z(GT&Y<3lomlzd2k2C1x|`LGO7$y6Jl(xm#(9)L=d?W3fNa{(<88%28JT`233GQ^T; zJ%DA+UQcx_{U>30psvq5wJ-a$wY#fa?PD31V2KimCv_i^38e2sGKggbx_ zfAgSB(hnO&I_RbjItsKOXo=Wi%72>Cg|Z$kgLjxD1pjs+LDg*kEI;MD)a8at!DMzf zU2SBG9jpqQ`UzAXs{EsN?UR5IHxwBU%6hB}m?R;%>Z5CxfA*04JB*Xd5LN*jvmy~c z0RbnIViG$KQ+ymhbV@$PN0<0MntV!=tP)KDACvkLMgeD&K@&y+VUvgx92Sqmb4=zB z8%1Ji>Y(F*`2#HxJ52d0le!aE0aKF_6eTv_fH931H}QEqnWWYr$(PrZlbjSr z0hN>E6e9xP?z0FL!VUyQdMBxq&Lb;-%jpYYJJ6P_8YncSp*tZ>5>e}cq}K7@SGE%; zfvmb5-8*OQ3@Yb))g-^*yfnH(B+F3(nj0bYwn87PMf!*mAD9-bF&Zk=fk*TDrsS4# z}6D%;? z15-N@(E%y=gtR}d832M~Xh4Au9+QmVlQ)1<{%XUZoH>;YorSB`qd2+uUQWigZL@YZ zSYv*I*UQ)UVbxUX{!|=LT?$UQ0~U;{OMG6$!!^u@Wjyglf_Unpe&y}3$k+!Wi2>*s z_o#f%o?liAR2O-1m*%%=FG z`aH*P3;+NjljA2>0SS{kC_Dm_7L&0S7L$)CDgkGcx+ppU*^~7sCIm%#C#jPkDHZ{` UlQt Date: Wed, 23 Apr 2025 12:29:45 +0000 Subject: [PATCH 502/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10654 -> 10656 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index aec70d8bd..ba4bb2202 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 12:27:07,854 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 12:27:07,854 submission_checker1.py:3308 INFO] --- -[2025-04-23 12:27:07,854 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 12:29:40,130 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 12:29:40,130 submission_checker1.py:3308 INFO] --- +[2025-04-23 12:29:40,130 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 12:27:07,854 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 12:29:40,130 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 12:27:07,854 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 12:29:40,130 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 12:27:07,854 submission_checker1.py:3324 INFO] --- -[2025-04-23 12:27:07,854 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 12:27:07,855 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 12:27:07,855 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 12:27:07,855 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 12:27:07,855 submission_checker1.py:3345 INFO] --- -[2025-04-23 12:27:07,855 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 12:27:08,909 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 12:29:40,130 submission_checker1.py:3324 INFO] --- +[2025-04-23 12:29:40,130 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 12:29:40,130 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 12:29:40,130 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 12:29:40,130 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 12:29:40,130 submission_checker1.py:3345 INFO] --- +[2025-04-23 12:29:40,130 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 12:29:41,141 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.52106 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.38615 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.389516 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6f2c5fa6d..5e8d8532e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1129 +1130 diff --git a/summary.csv b/summary.csv index ab76d713a..221917133 100644 --- a/summary.csv +++ b/summary.csv @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.52106,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.38615,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.389516,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c0bd9d256..f45afb712 100644 --- a/summary.json +++ b/summary.json @@ -609,7 +609,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.38615, + "Result":0.389516, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index e30787f1d270df36a8fb78dd6704a7d3b7d9e0bc..5a7eafdbafd8b6e3fa1b15adbdc9c03a989c4149 100644 GIT binary patch delta 3558 zcmYk9dpOhY|Hp@%O07*Pg^8JS7>%692Qf7%$vHVp!&D^7>Frb_AE(V(p|DEQGFDLz zAHELk$oTR@LnBxFZ4r$&y2b z8~}H<4ce=t&Ld{dbqqCEis&>*!Rm5?#bik0*gV&F9g~v>4<>6+n#8^f7lf?Xyq$%(5 zIZvI=TgtyYRGB^AYn?(RyuEcz~EwawhaCUnJ%s*;?LvEp$^GvsQS_GlO>#{Hm{KvU<%Iwm#Zg&*4 zw<%VqGdLMBg3j4o!ZQmM-)i@w^1V;aRAwn6{?h0*7hQy5!|Gx`+F--yI=&WbhW}!Q zD@iqUVN8E4PQhiYz8`=>>Nu7%^E)l)p%DE~GLZUSUnnHGK%2W64iHTncY>oJs+~7Y zWtek87}LkYQt@%Ltqjw~flqLVe-I!GNk;$F=R~d)@YxyuRA(lHE=kBHWi4`ZS9a1( z8^c`PO$V40wN+gywQ+A4hndF9pRGqe)Xqmds7g5q(KFKc%T8>KYEgFf7N&N2ES!CN zqL9|V`)PYkDWn&u-t@1Ov|QQg?$D`riSRi@^7hqb>^}Jw`egh0{g208l5Z5=OI`fV zoK>-+%?7f)b*&OQE{23pq2{mB}|Zd ztCSpVAhRhTV{+(aA$BGtrTyG^?&q#Vx5sPEe$8ZL`;m$Z2q$rJndBv;*&LOU*#5dCcGIIF2}OhUF!CLXX5!V{Mv)DZIqL(kcOpUfvZ zNj#MU+zVTj^<5NZQ<3)>EY(@ZYgzAqWY@VFv8hsGMu*1l<)Punyu-qu=NQ+(BQ!HA z+o`;ni`s&8f!TI)(8~fGDtIPI@YDRt3w?GOZ7@#aT-*tC^{uaNuXhsM~ z{h&c$TA14yucHj=(Z{mE2jx<#WM4VqdiM{2W|YakJ2#3|TD}I^81QuH{<6n0en?~u zI)~mAsIgm?xXjAT%To+H)4T@H$r7$G71fO6=I6lq@a_)_USjd(QXs6!-fh76){MBs zXnbqAlw!=Ql2YIM#hic}yt~)5X0trf54_Ih8!~(zDA{i#M_BI1Alx)2+4I zJutEvNg>34SBRsi%%%cJXB4LYJ%PRp?k@F6O3&z6=ah?ARJzg$87~y}5A$lL5!?~C z<#Mx(|L}=lC$lwgi?^J4%k_%ldZlX{QWc%JURQ5bS@#vB82F6N`bT%$;+zHzXw4DVMJo?qEHQwKTD&8A zrgZEN0=I#3+o?P#>_hN8&vz}1p{0_}v2JMUB_mj5iPS4zt?YI_d>>&&Kbn#vT=S%r z?+g$&q6j3n(r4f8K%<=b<*qzC#l*A`J$YZUQ$#L17_YjC7myezqhieZQv*;sV$>XS za_Y|N?eF9wX0*RdT@0~C%o7u30M$SnK}milxWaT!eKJ?`kHe)S1L>dRo0HvwO zKJ^qAmTQZ9u#k?LMwjw2ieAPf32X^E${>Yi?pgPF;OgMQ^*ERdK--}aCq~C6z9H+C zuFSZw&GoR8>F?2w=|=H~;^l{7B;n#M6QmH1e}h|;C5cA?I|H5t6=-FnPqf4E@h#H( zEY$%Eae}KuKQ;W@nOro0P7^HI`Pf@M*frzgoOin6a_04T-gx;>dr=$Q(sBHY+^F*; z9u@2$7?=f7wH(kxCt=imvqX_-ib~p=g?LtZsjoUN7;#(Lo-|~fFAgz|b(9}K z-xSWlC*Aw9c`Dob1R5TLzp5T#FVGDaH|m9NQxAc#X=`|Kl^oz` z7#D1DEGGUNi z)yVFd@CYqh;VKLKqwMGZDZ{Y;C>uAAVf|0pXRb0c+9Ot>FDjx2oK2kB!H@0eL|9kl ztFnLyE%p)E^bWU4?tmC;Dc|EZx%Xyyf8jP6Nx^LO6zuFZ$zvu~5>mRq#q$d|<-Wz2 zuF;<@eaKo%>xWy`gJmNcUtrjt%3&D?<1IIFBa-A{Y47ET?+TGcj4l7VEv`aI%JvV| z>AiZT9mYg~$1y7r3ZyY|ioW&aRO^6k#G?dF9G6eWuK_5|9r4}GhM$6q zo5ZM%-mVPJC5CS0a`veC%E0S$iV@d|cY~0Nqm9EHrkDQt*BJd)c^~bSsnLYB4N_@D zCy6~H|IRJkvRoy5WH;#K29fn_v%}lI5YAWt6s;KRTP~d(LE3?%wTJedm{wRKcgjiw zRt@Th%{Ic!Cc>F;RiRRNI_R#M96Z`+P9=2Eu@OA*umVWVwR2Kb&KM7!YJZZN@XZ+2 zqh>E(p>!#uNP)LW>zI$-vyBkR(ZZJdB|3f;(NM|yr?z)()J?MJOFor6v;6NBBGY8s z)9{O*P94{y3u0WL5=wDSesXJ1EA$9}!WKoZ?tadQn{ZyhjG`frhbE*aa)R(z4s0k8 z)Tyo|ZG4|dgE!oGBd9z~FnR8*Q@mu(N9()v_=li8!efYzss6Y06z~wOt#)g;9&YFA@c#bMpI`8sk9JtDmG}Ss%f{vHcn*^F6!dvB v#mH~P#K_4CN<6;gDg|AhVe*;+;=gubiefx9WEA&KQ1V?xS%F3bh`as+-|Wsc delta 3551 zcmYk9`#%%z|Hda7YAl4Da@vN7tc55ev^mV-EyqyU9C9ehDTk397NLzfGl%ppisq10 zba0yUA?I>T#qypxpM6)KU%vMraNpPKd0p3iKW>U;o@K>-F7APu!ZUs#06>Nm%4LW5 zQrW636?B*~s6Qsx9F}QX))4$Gi8Su1cWF6Run$PPcctO7PgwBLs=SA-=SV)!9tkUy zuh!J&IiFs6loMtugfD*Qmtz;gS-D>2myKJSw^o$++BOt1Ve*5igvX{Tl^iG?1^5ja z{YH;O zSq*$yMsjsL>cvy{rS1J7)Fpd&xze}Dr1`$Z-ac#9G&93h6tgRP*7LJ+;rWgx2Z^?Y zPoix_hxdt&(EMBN^VyS1NFQGdI2yLN)I9PaC*Sy6MahB63!WFw;b*~>lZAL!df*lD zwpN0#Z8RraGZ%uCWT)J$Y#fbB58(&_v^iMIlIb?V^?b$R+!)LW83~U6eJrUkX zI^WbghiEWO&o7y;8rv@(4e}lG#U*Gs`Bd7s`o0=|+N-{ipwXkQ>giRPeR*Ot>}p0| zGH`9d0-hair{r$el&d*W{Dza}jcD@R|9Ym)4u8c^44GSX9gqEZ_&P~pVRA$RSv7oQ zltllwTl?i9X~5e{3#1XK-QJ7(-Qu-;`Fa@BhAA!^DM3L0M(ewvWac<_JJromovpVR ztZ8}4iZ{R9`aP0>o0VNh+2knr>$quH<2cpj!>)!tdeiDR0?TBRr*r3!e zr>vFRQO|tEJ54W^j8-KBjh{~nnwPnr6ExqSVNk|+v+`;jlUzPnfKC}Ok3CG{yVZE6 zSM``YF<$?fKI92R#sUPAC*CI%7BogL#Jk+q;iI#)>J*~yv~+H<4*go=2s}D_nx0|A zaT*B>EiD`xiD;PEKSW3c95hbFTIfum`+U0^BcoQA-U5D@?g3oz-P5v}8~xL^kc zloZ4z5!Hp1DNvW94*DxcdHMjn&YE~iiHv8RrGFvFy7Q5VkNU7cf0=8N6}h`7>Sa&$ zPq&#+C_`GXd1KvV_M7-9$-@QK&yVSr%7USFq;?{ zQEfNsDyeMnj>Sv|c1_(`72E4OJ*5)FPJq?Xr^p^IzyXMe4hsaP`xIglhOh%ivN-f^ ze&j^pj9aejB5-!esrXr{VSO8}@&F&{CgERJh0%nB0!t)o)#g!<`IR9IVV*F4H_3Zg*oPb`fcm2@?wS4OH z*RDxvWK2+8Pnd6BsWC*=N@iKUO81KAa*BymwFFTYp%{a?mjb%u%= z@+(1qY?8{Ntfasl)C@;!y+=HmVJZl)72RUbDOn)+)$eQld1J0f3c)&FuIDU~yLdtm z9`D9c7l#92U}JzR*vGg{fJc_NB$OxTqz9ox{x(?0D%pxHvwMLJ?nMqcD{WbOe42M5 z>V!6)k=o3Oo7|X;ARym7;y-6`idp@m10(#e5?;*{jglUOG3QQv_P}3;K^c(aN5Q(oWk*WL zg@!K+vBn>I?^%L2kMG}J0F&TG9c2mOGRFIGJZ2qPN-4 z^NBpx;hqC4L@z7SMQh6I-IzZaajQD7^Mf9a?%Ey5Dx0M{zcjAU$G4Cs{;1oN7HOeA z+WZWUgwgoO+df*?1L|4`b+YQCZ#Qx&p317`DW#+PmNjGrYgOFF zo3;5%)x!1A(acO9= zX9?Tdu}0*7pBskLj1qq2G3F5gT1s_hh41;{G}H=uIu3hnaW4f#wEMW_EFwk1vK0OI zSrY)q)FcBZIqhZOT+#yGlK;dexCu_8M!omKB;jDIt${h$FmdDqBkgun`nGmfm`U<_ zI15-43s{z57z-G}5ZI4LENYjr>kMJ{JSxKf5{v&!ECJ^C==MMvOIaWypPGNweDS9d z!wA_bL85fG3_*L7VfhoXxvXG6U(6;UZ0lY;a5`m=5S**&v$8%5;a+W0MOJ zXc6Tst3*l1O6FR4$CVol)JaA=GkWmAo3K@cr^Rif8Gi;mNC6evecW^Y-<3t$WDSVG zOe6!B{|+cmH6M1h(t(j8BD?j75r;rj4SC1v5&Uc1VBkj5v+5(yLFiO+G?1Y3UXo~0 za~hB4`WGu=2<<|jTV$j4efBr`!K#~!=MrT;a)*^$Et4J94pxKltcM^SDm~Vv$?1N` zC4KjX)XTgFVwL1NtMN~op@YfaKRpS+axAl|Y#68$ zSxnn=E&>o?jCM8#OKM=os%v|L`A88i5*q{F_v7u|D?rDZU0c;j%U5)_*-=n-+gW8l zX|HuFO7*F*I-VJFC9`-$iq&?!YRVBhaRP!!AztHr8`3rlW~=a)6F$cpv( zmaX?~*bsu-e>QtH{PMm~Z*S_NxTl$HCdNz;IXo`t?_>K}rD!8P-gd23!!EJ1oIa3n zSJlpR*L?`PJ-}6WtzIIlh?HebVxeUkIy$az# zY`sJ4XN^}th6it zx_GzUpiO(mf=j2ZR&K$Sx$aO7u}FKdkf>NDt+8**r;&@|tXuq%C>a|G-Ks2(dMiG!(Si PPLYBXWPzVx0M`0Hb12Lb diff --git a/summary_results.json b/summary_results.json index 5d5dcfcb8..8468fa553 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.38615, + "Performance_Result": 0.389516, "Performance_Units": "Samples/s" }, { From 5840972f0e92ec0deeda750899f8c3ed5929e32e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 12:43:55 +0000 Subject: [PATCH 503/719] Updated results summary [skip ci] --- README.md | 88 +++++++++++++++++++++---------------------- dbversion | 2 +- summary.csv | 10 ++--- summary.json | 16 ++++---- summary.xlsx | Bin 10656 -> 10663 bytes summary_results.json | 16 ++++---- 6 files changed, 66 insertions(+), 66 deletions(-) diff --git a/README.md b/README.md index ba4bb2202..9a49803cb 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 12:29:40,130 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 12:29:40,130 submission_checker1.py:3308 INFO] --- -[2025-04-23 12:29:40,130 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 12:43:50,558 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 12:43:50,558 submission_checker1.py:3308 INFO] --- +[2025-04-23 12:43:50,558 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 12:29:40,130 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 12:43:50,558 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 12:29:40,130 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 12:43:50,558 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 12:29:40,130 submission_checker1.py:3324 INFO] --- -[2025-04-23 12:29:40,130 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 12:29:40,130 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 12:29:40,130 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 12:29:40,130 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 12:29:40,130 submission_checker1.py:3345 INFO] --- -[2025-04-23 12:29:40,130 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 12:29:41,141 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 12:43:50,558 submission_checker1.py:3324 INFO] --- +[2025-04-23 12:43:50,558 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 12:43:50,558 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 12:43:50,558 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 12:43:50,558 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 12:43:50,558 submission_checker1.py:3345 INFO] --- +[2025-04-23 12:43:50,558 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 12:43:51,536 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` -| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | -|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74019 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42098 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.52106 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.6529 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.389516 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.163 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | +|---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74019 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421087 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.52106 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.389516 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9278 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5e8d8532e..7fdcfb8c0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1130 +1131 diff --git a/summary.csv b/summary.csv index 221917133..9cdffb9e0 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74019,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42098,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421087,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -18,13 +18,13 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.52106,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.6529,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.389516,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.389516,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.163,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9278,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f45afb712..6527c7c51 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.42098, + "Result":0.421087, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -169,7 +169,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -574,12 +574,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.6529, + "Result":21.0349, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -604,7 +604,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -754,12 +754,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.163, + "Result":22.9278, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -784,7 +784,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", diff --git a/summary.xlsx b/summary.xlsx index 5a7eafdbafd8b6e3fa1b15adbdc9c03a989c4149..baa91bf4492975072d2da9ca85e2e6325f3f4c76 100644 GIT binary patch delta 2981 zcmYk8i$Bx<9>=%Hb-9I-*f2wGVJ;Qsmgcf@DH*xM2vG>@C;1Aw7TQL+3#^=dB zC$G_5J;rCe3%JlSh~ry?{o}2?@Odoam6=<6LXu43{vy@XNN4z9-t%G)<}X^5!BX$g z?6&(>q(tM48|%fePzf{oyKmkX4U{f#M~&|N$?cQNjVHJI4Bc-%X{)wxXvYUGhD(+X zg}-DWWlG;pDK*lro1&tT_cYT9 zLvwVmBlM;D{$5155!#o$h?$sfJ>0!9VKuxyQ>-g5xOzFbdfNn4D7D{`Qtl6DRfKM zMlx<>_Sl!-Ijg;%l(70;so}OIyHcSn8OagTzP?(zJ;Wdd{6JaOZVa(p8x{p6(n-XL(8;@ma+@l_i`W4=h60imVJPeX^i3aGa zpSK%2_~uM~q21@_@2wCbyrCzxU(3d+w+0_~F7RXJJ8DVu;~;C?OR|qd9iLJj7l}(OrC4vOK}r%AX@_XuvXl~Nj-}uoDt|=g6vUi zT6Hj3T~zwGyYsvmHa)B386n=Y@^kiUx!AgD2T5E&JtaTJSLRB274DLdZIFhsKG&G) zKYJ+a*G|0`r`?~36vK3KyF5sivf#*L2w?N;0`fZ$`91J*l!;@l zMWDEaW7Y~Ly)QVNupK@K zwS)VBXP<6mMzvn)yk^ z8ZH?4Ocflej1d0;%kPgsBQi#XI$${xoibTqWcC3rznn>#VXi{|CA>JF;h$27V1lpy zWt;84O52x{_+#bK^~yGWv$L5&{%f7Gn>R51W`>m23|rV2S%Rsa>LhBc`~+Y z6Lw63WS5+fq@)fd@E1>+YomR3_a^$pSR+P2nTVmpRnY02D5zpko}eEm5(kL|PK!u# zS+;XvwTpV!%DiU)6hEEo+4WJ%^)ICrU)*K_pmZv?O+VUDg~frq4JDYWsT!cvxSX|1 zEw_f^wEayDM3wEMWI-Pz7amP5E55BL(~oo8YAD7?KO@1W2^XR7 zHjhhHFBiz&;6E)}wndI$LqGl=sXg>hI3-A^#PUuH5!e#$i2txHCly2Rfs@n}is0F9 z3;ugb^q8E7_)92pX*_IR7x~D0AKK!oWL3G8t@xg7e$QDcFhXn#sXWc%cWQE27)H71 zn>Ug(7Q89eREyLW;_4sKEjg){s#tcjc8uZ`JjjN~ZngxgVV2T)m^*?oitVUk2oOOn zzxb)?M-HRl6_n1U3yjXVEth_39?sXOT&mIdDkdX#stg6+YrT?)%b*MV5T=9f!Y6Fh zFjK;_ASyB$H6jhAaxW%AVyo0R2SKX6s{CSx4;~Q=bKYXq2C6%jd(nTmEwo(>&q_8t=m7j6U)c{oeIPRV!K!W|NqMdnPmsoA|!q}Ync}(Sm-$m9yA}S?{ zp>EZ`m-pWVlJA_Q!&T>T_=EmgPBC-< zoZ%lDjWj^@h}|*e%N}QCXn$^Z3W*Nu>VCf_w`^AeZPWzVHTR=>)#_I#vI>)zl9Wm~ zJu3NauASt&dnVCm=VoBKIivOZ+o~Ql^Me4K4_* zfA8lj(RmLqB`RLd^m`g_yL80x15+;B?kEZ#hjR*!cIIh^OTqezSyZ7ITjCzg$)|FlXdNy=ajoxS^b5izi0)VOZ zP69XrBx5H5lZXFHKn@rC+x%T{Z@E1CdrnEgg85WY8Or4mDh*2ddBLV1iZO0K@wXcy zT|3>#fpgA6C&&3he{BGHZ^!x9gqvluk9TQjuW{jI32V#xepSyppNf&b8k1@t1Nh={ zb~;JJ9yl0=j-qU3+^QHYx>|pS}ONSDy`?GY2 zwJJFGHSF~GXXw|-+XC>9#L?DrpG#+IsonXj-ulfZcN<=WS=T^$cZ+!hpy1@d8Fq$< zJho}`yp%~*%G$)e1}LB>0P|mpKKLtoXPZ&i8g{lB?X=zlogQz1rr_4$aN*O;y3mNs zf^S|~n%i$4KU+51-*+`+Z$TT;Dqs(R6<8toykb_9x6YvrIygpN6nUR$GMC0>f#l+nDf%~ice;*}6LOJ`{OL0nLX+j<2x=$c^eKf* z(v&_Y)ki&|2VFkm6?NkM1)n-eQ{7`yccG8GvAwxEDji$teJ!J3v8gEbMW2%c)Oy4d zFdxzn0!{lzrYl~3M=5>5+v+R-K!;Gx?;jm_3}n#U6U)tz=_Wut8f44Es`tc&GDvTr zdpccCgoHx#?%i@5RCLpFHs68(m)u7a&UhDXIBJ=McSxO=d^A|Au^28YH0Us`|Fz{9 zn6kp#ab T8mJh9rHFVH1s*yA#QFXMbQzgi delta 3038 zcmYk8dpuMBAICRyEty-gGQ>8Mxiq&DHebYOwUV!U7}GFaB)RUO3lg!piwLVEEz`Ov zmu}1@bRkJ{DPQC=_vQAR9^c>N_d9=n&gbzu&(HJqetaJ1wo|baJ5*9i*Rad)oHPie z$&8lt5&dTC*}xf)v=jmAlOFJ7hpcAblD<8ypQK-YmcC8y0DkXZE=mh*`>Oq?3H7ss zv4S%rrJSCP7pn_?-&gBJ50YU72AlCV3V%iyuHKG(uIzRFWNAdk z^yi@o%wf(%m>|&Xa7z1;sMt}H$-lWt!=FsLppjc z3RbBG-l|AWZ_b<0>>qu1ONrxA>1_vx3*L65`rcS*V%G3K5Q1%#ehrN09;*vUowN2> za|!jF?rtjYF1Qxp(dXOnaBhFNC0R=~xMS~7O8wPQW$-V18?v2-`~i76Xs1hEyxB3m z^{D#2nHSiey|>oMkAV}bR!QtS-KmZ${S1ULx%u%3dn7b;tS4!teeyg&GB4fValKOm@k=hJi4(ae#3UcQ zgiBsz-OKUsrRh<|QwkCjC6K)DEr4_UmYIV358 z7X>odW)YSd<2xwgu3>WZdDWN8)vCJ8V5C$d`Ke*NwdlE%<(TG=;Z7FdcC(+Z6y8^r z+%KNdr^M@AW>t>ybBc?5c#=MH8u){Ny7Mi1cXYyQhb+4tyTpQtv3$?rM6@P zNvR3cmTFb)i}xxjgCon=Lr;oqoG_Zm^~>6NzW~x_c{>c=kOjF? zDm+v`I*|c3A&7rQ=uc@Bq$Lx(A?(0^N;oh0@x)&Fhet<(Q-zAUu&yS1FSWAfu9KZC z0_{ASRCII;-c41*Ok(?Vqbqpcp8UPGz|C_sJ96)HgX9(#6TJgn+(Fs0nbV z(R^p`x8*n%E%`G#iG>kr_cS5&JfU$npq-U|X<6Cri|f^}nn&JTTJ|0EmVOb_Od1(= zM#yRPhY#k%6d0a?H5)$^YBdpR<(gWuwLOGd=TAQ)0o|n(HjONqLtIjO`_qYAbH452 z`|7bNc3wyOx%0^gmO~l*vPqKGeK_Xk7SQjSRfSaA1C^! zw{Hmtpsk>SHa6IVcm+BM{%l0Bbl9_RN2h$oQM_7Qh3Y*Kqr5fctM z^v@U>y{RR2Yf_Ss3elj}{Pv=8*WX8gm$Uegn-_w-mjZ(?xIFvPgvB(<8_OTrf)g?h zC;H2_Zh0p}%x=4zF&#xhO(v#l=!Q8-z{Q53H8#HteiRr!BMBAGC^9d7@84PxuyInL zJ;2g4_1uEu)Q$yddC+{FKYu(@JWEGRa!~Y7D9J6>F>{MQlk}lmi2dCXPcVK*dIRxSefk)Z}J$ zQ3nIS9!B$a>tV+NpNtD|04`IaV(s~d+AkgBUYH3bLh zhdEX)QEy*z46`#ITTMTNE>vS!>FMmlVIZCiY2ZxYF}kZnLpg7`%V!nuXd`a53)&J@ zBfZGpeoc5*{+x0SkeueyiqM(#yO5=Jzrd>Y5V>bk9@_=ym(5yZOODg+#d(#*t8D3X z0v6BWE&IYjd8k3Vi!%^iIoPFish3dgSE;n^gLXZb)fhD_^wVABWK0s_RkrH=^0mc&RgyNcw z#w>S+{VCznq0U%)m>!Vt943sLnu~2VKR#fW(!oEw+Nmv2SMwTLq0a8Nc)uhB`)#=i zHWT|VSlP-Su*uTK#Xn%Ef5Bc$CD?KT`bYb^Vp`~n`0|@NFLc%vALNQwj9BD7doAc1 z@ry2-6DkY)qwM>pGGNmgLGZ`fu-!%ezs}wYo!N4(@=JqF=ylM%#PK!i;MzVq>Df(P zKHW&5s5Z7HY*IKMRz^Az!X`JbtiboeBI8(um9FBo%_70JN)=I+TbljfL(?Pe-**gr zZ|=s{JCLrdkiJxX+grVeJN)c5uyVFjoxeF`E!jg*DR34>^R7RPfV5B-#7Cs@R((tD zF+_NniUClJBqwU)B%W@iSh%B>su7~mZlmAZ)OL@gZb5uvo_JeHIpxe{zbl&eGkt~6 zF@GB7P^l&%ChzTzLzL3)uBFRrxLK|E!te~Zta{)q{WChQjJI+YXjlCo~tTvfgG@a`gLPTQ5?AqCtDbHSAEKir#|wf=NnPi}g7^+L#w!fgj#S_w3Y_b5M5 za~<0;T}-Y5wgoR!YSxwHg6ovEQqC@(w}|2!qgn|Yw0vp66aF#OWc?g$WBJv0iRtCU z_@xgW+3P&BN3(eYc5}V=4{FnW63>TYrw3m5uMY*7JKiUlw`c{K&W#SFEG#i9>F*eV zajnO`u@2Rkyl)%fyOt99x0c%jT}u(XDM0%$Z*fJ^>)pBp$q^#TR73CNV7HgZi0&Li z&oFy5)q%sW3}kJWVr)lZhY-4wm540Rd0QkR0sjjV{l)z?wD(F4aJ|6ALt7_%IC`|L zFeBxYl}VSrt5(gPquFI>kvB#=f?RGbMZpJ3np(+f{x@H#qR??|OR%4zTvyx( z?XP^bzA(@`10!{-$V3j})yW4EI{h^3+n)Q%XY3%H?xQ!pN@$_{`nZJJe=2taS8%5K zr~4}&l5RWg#XC2`e3kC>aQDnmz5 Date: Wed, 23 Apr 2025 13:59:48 +0000 Subject: [PATCH 504/719] Updated results summary [skip ci] --- README.md | 84 +++---- dbversion | 2 +- summary.csv | 50 ++-- summary.json | 552 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10663 -> 10660 bytes summary_results.json | 8 +- 6 files changed, 348 insertions(+), 348 deletions(-) diff --git a/README.md b/README.md index 9a49803cb..1cfcb9bbd 100644 --- a/README.md +++ b/README.md @@ -1,52 +1,52 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 12:43:50,558 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 12:43:50,558 submission_checker1.py:3308 INFO] --- -[2025-04-23 12:43:50,558 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 13:59:43,381 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 13:59:43,381 submission_checker1.py:3308 INFO] --- +[2025-04-23 13:59:43,381 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 12:43:50,558 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 13:59:43,381 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 12:43:50,558 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 13:59:43,381 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 12:43:50,558 submission_checker1.py:3324 INFO] --- -[2025-04-23 12:43:50,558 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 12:43:50,558 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 12:43:50,558 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 12:43:50,558 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 12:43:50,558 submission_checker1.py:3345 INFO] --- -[2025-04-23 12:43:50,558 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 12:43:51,536 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 13:59:43,381 submission_checker1.py:3324 INFO] --- +[2025-04-23 13:59:43,381 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 13:59:43,381 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 13:59:43,381 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 13:59:43,381 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 13:59:43,381 submission_checker1.py:3345 INFO] --- +[2025-04-23 13:59:43,381 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 13:59:44,614 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74019 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421087 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.52106 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.389516 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9278 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9278 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.41581 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.410481 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421087 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74019 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 7fdcfb8c0..4880c9fba 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1131 +1132 diff --git a/summary.csv b/summary.csv index 9cdffb9e0..9390cddb0 100644 --- a/summary.csv +++ b/summary.csv @@ -1,30 +1,30 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74019,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421087,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437891,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.18883,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.52106,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.389516,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.18883,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9278,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437891,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.41581,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.410481,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9278,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421087,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74019,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6527c7c51..4d4475b5f 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.425121, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -123,22 +123,22 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":1.74019, - "Accuracy":"acc: 80.000", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.421087, - "Accuracy":"mAP: 49.593", + "Result":7.18883, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.26957, - "Accuracy":"F1: 80.0", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -213,60 +213,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.405224, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.420623, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "9":{ @@ -275,11 +275,11 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":22.9278, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -287,8 +287,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -306,18 +306,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.1815, - "Accuracy":"acc: 76.000", + "Result":0.437891, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -336,18 +336,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437891, - "Accuracy":"mAP: 76.951", + "Result":23.1815, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.18883, - "Accuracy":"F1: 80.0", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":23.2491, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -484,22 +484,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -514,22 +514,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":9.41581, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -545,20 +545,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":8.52106, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -575,11 +575,11 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0349, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -587,8 +587,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", + "SystemName":"gh_macos-latest_x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.389516, + "Result":0.410481, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":21.0349, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":21.076, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -677,8 +677,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.425121, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -724,22 +724,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.076, - "Accuracy":"acc: 76.000", + "Result":0.421087, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -753,60 +753,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":22.9278, - "Accuracy":"acc: 76.000", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.420623, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "27":{ @@ -814,29 +814,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":1.74019, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,22 +844,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":6.26957, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index baa91bf4492975072d2da9ca85e2e6325f3f4c76..12d33133db01e2d8453d660ebdcc203ef92f275f 100644 GIT binary patch delta 3518 zcmYM1c{tQv8^^~I@zB^cWsGHpsYn=7JO)i>CPt)Wmu2j0n?%2t*c4MP$m64^r$ zEtZC4%Q8u_uk+fn42E|+*YjTQ`SYA}-RFGo@8`bHbqXx=EX%GRJYakyL?DRwtl8TY4jYm`*LL0B)%&`D~TFTS2`S zrW0z>_L2l;RlgxepC7Q!BlEI}(pP{joH1=#E+R#U(i=i~t!vXnj-bFI!gQY%Jgy{< z>Hx**NpE7Y1&=-EHg|%tTDBg_Mdj$k@y^+AomT2kF5*|P%k7z?UY}3qs(xv(hqg?9 zI?_Vhy-spKWIMHvXAY^L{roL7a7wdtjr}*DW}6I`<+IfagbSSS%7QD0ask)*+j=rB z&2j$L(Zmcy@5lhSkib8GY0}U8$?NJqXUjK!7wei=mv4y$c-zifT*YqYX0LUcNzpxg z%8$rrZ6tUYLw&NcDTdu!cHQa<1c&dIz$e!mIkJ1F$d(Jf=|76yzrgXMt zkT6a=5{*LpAzt3c8fOZk}+TWjRR;#Q1n~_A_g%)Hfc0g3h*!8jvhmFSnLYA5BGHw z%qDhxC@OqF%|P(GSuG~*r>-Gt)AeJ-Bi-`{< zYr`AQlxd|$SROF^of=7Be22CSNgKX}NKY4-U>lMQPvQAKz#$-U&01ssa}uWgJds{A zIkXAgzofgw$}J`y_nk~#^Q79Jz#o(X_RmsC*F8Zr%&*Rz6Y%U;Z#`}@HTB{9)YJ{l zrT3(Pz#BiE)?rs${Fu*~4P){FUCM@9st(PGPVbUUd;4PZ2x{>Evm zNBfe5%J3Q=ltP2088=ORD`X=H=fGLaXs~Y^CHAqxi0^`tO%ml}MV!37b`~I!IfvK% zadyhui*fUIT1VOJkUyO%+dRSm@|T2foKE&6)eH}b5x&Z0zE6sCt zq5g0ZzZseD8#joI%~!$9r)bz~;|FSetM|ZN#jc4@C>`s&6_BboPSw7YTBP74_vut$ z7o|W%X1bv^o^KEixkeU11s?sPRZb5BX$${-vl>kP3k)~k84hauioW8)M1D8aD`X~{ z%Dsn@3{fZ2byavgHMID;P;}iYeuLhLMWjf`bA$^`F5~x7^1p}o7cPgZp}>9f7Dfy7 z(ypH1i|gb_@KNqT6!{rg6}Wrcp^=$3bNe_lM@(4sqOCq$9M=)!mfR+o3vLHJF|Fo0 zPN)Dgxn81J`WJra{9|+8@n8E;ol~I>@>uu#u3v8L0mLZ$-rMz-+wU&g)z$U%=~J8q zC@)h&E^W$F4UVAU(biUh5!QTVZf1{nJiu*v>@thIhyHWV0h+;16eSj#zcT=FK+bn- zgv8;EmO7;`ckFhqST+zG{A(pl-Gh7k%&G@Jy(%lrHkizrLwgVGSr8iiibz+}eoUEr zOW^fp_mWMKY#50PX=7|e*ATlvvZ(&!@`k?WOv51w(3_|kbeCB7=Tv>!)HQ2}0p2KZ z1Upc?w*aWKf^ng5Ov?l*VFSC}?;vx~2jMW2EB6q#@Ml#3PAPRAOqZibF2E>UD%3h2 z>x>1Mw%|1{5`g;hPefi7!(Dk|Sv1k%^C7DNmAxIdwd_mQs@Y-&x>>e={bt;gf?EP< zGsz*B*z#2KSDZ}PTbTpae6?_hlZ%%I09f-!zQFJ7gM)}gyNRR{bBeKT1{9)0v`U?3 z6wk*MwHl%i)kV_IYB3D$Xn7~c?s_{`7h#{PKZa+T z0@RmDAw651E823d=w?;MG3yH)BG(INmmP{UC9l#xvN%2o8GYfMj!7Lgu8v=&`P`0Z z08?X*vuEzk7BD9g+!^m-HN1pWtllv;$Bh@TeMWI;4x^kh@wy(tQJRlubwA%RFv0K{ zLcwieRYHJNhhSS$i4BWoh02~7@6(`PNSJ!II*z)fs}fp-xr1;cb&jTd1Jm#Ku;d}| z%6qGyH4K#KAga=2%q@^x#v_<;3%Cm7CSs&?7@a31-Eq7nsf4k~(sz@6wL*KaGQOE( zdE28J!|I-(h2^cB)5y6Tf#d5ge!f#C^c=4d3FOl^ouQx%rBQ735|64VgqNtdkD)ax zPKr0dy`!bGhQ>-|FGPi*uZ9h5)1?WLFG&EU(4-1R(0_~dM#IG~a9l|KD)@s_E>t*= zTShUgpDWpm6oJBV07)@CNR?xAY7te0L%(yMo!(9keL0sTHJ3Eb|6-dV2CQMp;FXj) zjzA6|5KQI0tYo zfX;IzqU5Q=WI-);vK_4I$qWa#pd5jpP1DP6B->~>+HU@n-~3j)VafO9*0l{!y{6Ss z5+l$)e8vN)*zHg9)vGJ-K@W^5k3qBdN|?EoA7{MJ#Jyga*VFfFa++d^U;g^WS-A00 z{|MZrLwejz{aZr`R;{7U?aXX8fjpX26m9#h1vohl0}k)}hVGqOir8ZGps&~zSn02B zd+OKqA)CPqW#7!E*qOCMPCp-c6l%p`Ss14K?$(|1U4K$vr;UaadqTVTBA$Zi-5&8D z-#z6h(RfDKCl$_Mt2G~EM6|Xu_CV$)JiKHW@<^qIWX>Vp)h$#Dc|cJPF#CK;GKkXvQyBfoIp}j_HG3HATBc>5Tg*boEitQ|B3j zdn<=L@DDp3!B+1wg`=;JrhZx!{#o^`iT?hEi^lrh>_5%dgjt&Pjtvdqq81~lSu;pW z(&N6(bH}0lPig0M?6Fx+55)s`mkRE71=K^OA={Byc@Sj$S!(APi}(g^KzKG)siQhx zZ%sTv{hL&=bM{gF9GYh`=u2aq@`v>8EpZf32|=`R|NDWOQMTg}C1a3!T$$u3q@i%Y zQ4lD^SMA2lP+#H=vdA$B@;LGc_b8MkD@m3(CL*(+XapAsWX=r&iGuz=BBMSjl1CNA b_cxK15h8#6Zz(cLiX3rFk?)Jr{>*;?&tbmz delta 3474 zcmYM1cRbbKAIGoAyxDw1T;sZt6|Pa?63MzUN=7KIam~!zSMssGW}$nLl@StAxKgsS zyD}oOg-cetWL)m$cloXJ?|D34@8@~F-mgDSk!6Wxr9boOX5<&;5FCct9&l3F#lPme zvnDRz)}!F~l4n)kp=cfzJZ^@pzt@4+Q99S}V=4rhZq{HGw949cYdeHyc9cAM>o>kf zTtl4C!*xKGz0bK>O}ENwdQZBJQ9gV#VtP-i$y3GstwHb=kW54)2!Q=$%~!JcFOJ*S zh764-YDcqny6~MJW9P4>0b1BWca1nlXcQ5i>#}OfPb$hI#^y~YEurVvQ0jfYGKx1#iM0d7oh1;=yb(JxsSIs%7 z+dWI8q`gH|!i{XZR4~@?PD)+0{#lCzR1+6wdQ zf8|`UHKZhE3&><8I(vExzWF6o9xBo?R(~bH)E0BCE4%%Q&8CBDXv!G~M$nc_^5QVs ziu!sVymP2{=iUooJ)sTC<%`(N)uT|mnYdYYBp`;+N&TIdtG|~o4@7VebwsQ8VUl6f zh^)Qe?sJ9GRcb@3d1#@PcNx;KtBOO{c{U)((3a>1OXQ=qFT=&kj~3@1T@Y#OH`JTl zSb~b1erMz6YB@9#U#E5(b8~4ligUFNd2(|l=c~~!9|5s??Nm$zmt3EZp7`8au%TYg zlt_FW{vchieXJ46?o=%hC` zx6u5tKgF$m6va#u{o7e;V{#_A`Ko<)Oq_83*#eow5NqgY&if)q^6#cl_07St)gy<) z5WdD`z@GA9oV}PF_R}?IN*z*G}@t{&peQvce~@snoF@-4t(Vmp#=xSi7{^8X5-AP)Np(tr46q z5jNL{2Vp^)2Cju9&^^DkX?>%Lh*i8ls3~GJpSi z{q~%I*r)y454!bDDC}^Y*bKG}yV_IiD1lRZKT6dhVHc%<)A(O2KVj$n4^@o$qkxIk zD^q{6b_Va32G*ZS3^FM`l?b5-^H1mu4b>9N!RiTgFQi1Y`xWOBnc6PtgRXHumBPm9 zJ=KFIM9^KckT-{?Vk*y>gTYcxn)zo9cnzOe?@!Z)9@G+MXQG^hElb1ay6*bg1dfmU zKJ&V++vgM~%bJku-2PW`Ou%DMA^;V)fpYbT>!-@;hwusBMQv{uCl@sFu-=qM>9*^q z9GKe-OI+I|j@b3q5Lcg7I?TSL>R?5X`>#cs7^WQ?)he7r_(U66F{0ospC&cAduC(j zKHaL%H=BIlZwTXL@;6m#5hECI+UR2qls@wdH;%`gO{`&JDJ+fc+v$``0tUFN%oikc z*ZhQ`-Ro6=1-jKqgqai;7X63#xjpC{0jCjL2o9Cj5Om zIWD1V4%X{eZIaV_-eIHf0#%clEn<;%cnf}%M}28-xT|nLc{R$1-Naj7OO1X;=I=A5 zdk=5)^EN%758+1kGT6R|7to{WBe8UFMcsj67~F;<{AI!MYnb3J#_jq#a2K28)K-^U zBkvk+0(Hs}fI?PKZ`J3tkuMS6j+P&jR!g~6$wI|6$)lh90G6higyK=|hV_85q4Zu@ z4CX1as~xsV_Z|LcU~GD?<)v5rcd5-UoivNw1!S|&esUQiU#}|7ttqT($ttLMe5 zXPK_MZ6F|@UF3O3b>fK0eMGU#h~WAV7z2-wE7++T-{JkoKSIbzRgvcdjZ^f54_w*9 zAb593rK%?cqGRQ>7+z+LE?pZS6=F zL7W&S#u&Gvg4OKTQHgTptl9f4H$}=03xlbUZ+|9gFZ>-;>djuP|Fj)<$k7x1^+-%0 z9P0v2m6a%fX4-F{&lIbYX@gjbOQVt)sZ6%wAsJu0qN+t?=%oyq-yHnR+6MA8VMpxJ zZ_`<c&jfE*2&ZS0`dWr40onzHQF@g5#NNSr>xMTnZWQ?P>>C?#~Hc-H0M^aAmpY_tupxVqPlE_8AcW^fm!1vyQ@4I!Bg^LJlkA1Y>yFFX^6e7 zBQ(TJC_O5x2r@#Kg7y!Ct0M-upXxAY&Qem8CcCZt!UDha_3w&pnH57C6{t1OLkDH+ zcjnXc<2U2Z7t_|L_>Znqoa^c3SeKJdBl~DC9e_(C1XCgI@!g`Y=5=h6f%MiFHag@% z6&b`#!XhrT!4_tN9-Isa%X#S&E9sl&{x;fV^OD9_vRJ0sWdt+|WpzKynz0)yAUae; zsbmi~!I27E_af5cz*oB&=3`TTys+um8&nZO%*6KU-U_SPlZA9~Jg=6)OMXs>s5RVEHLOYaOUarW$M8Yj*ZS!jP3~pd zX%w56@`GV;@`A|!5>Prx0EGpooFt(A@_z}4p(6k2e!f3gE>8WKRh+k>yI4?$uziI{ zf~5We+>D1INUiSc|6LBwFDJ{91*P@6m*#vvS_yz>oH*Zt{ijUi^(m#yU3!!Vc6Uq7 zz4~46Tkg~!TB)_KF-7In?GvJ!V^~LH{5h*xr@)O+RDD%9cQ&R?Pm=i z0*z`QOsBaWJrK}BuVrcqr?^e)!94-(>V)0-j0T7*t0;Oi?A-6LlQJV7Hk_2%z-n&* zvNYQONkHvEp&U2Kb^gI=d2>$b3P+z_zuVG03k*H*K+4>d%vSFl(v?GWj9OX zBh+(Sa_>eVdCjVj7g5=E`AX)l@0<_dr<_|n0u>?hyvru9ZglL|8(JaUdP28x6O~$$ zHt8lCK>d@0wg={vMR>V67PZ^VTY@?eR&YC_$8+&DvolbHXUVu>2~ohK?IP#3T`Fr~ zqZ|Tj5xuT$SM8BMkedE08t$i8!GyIv4>*p+KWAuI@eka{sZ)M+M;IkUjSRZ_n$^WH zLEOwFpdgF2(RY+YpO90MFk~h3YJlsU?Kh01ax|Llt(8!vLwMpTgG--G{@IxpimY;e zm@+KcS`hhR$jSm@G;tTu9aHlLEulk_B_Dh)E&0II=_>w08H;B@hk0EAskaS8G7yFP zuwcgqk@7(NnJ9l^>LknSKK`e_ldXGX2ZX!3&=w z%Ua#rwEt9%`d7XN2U*RJA;*6vEOMU%&oBA;)PMd?#PfD^yf_^x4Z2)hn3N8kKdwdU sGTj31Knf;wY^o$32xN8&1mXhy|Aa?$;54MQ=mc;Nr6m{%u;ZJ50d@PMCIA2c diff --git a/summary_results.json b/summary_results.json index f095ba7ef..d045aa61f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.52106, + "Performance_Result": 9.41581, "Performance_Units": "Samples/s" }, { @@ -437,7 +437,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -473,7 +473,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_macos-latest x86", + "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.389516, + "Performance_Result": 0.410481, "Performance_Units": "Samples/s" }, { From 8e513816d9b2af85823b62bb1a8b2745ad7a7ad8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 16:20:14 +0000 Subject: [PATCH 505/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10660 -> 10659 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 1cfcb9bbd..744378916 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 13:59:43,381 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 13:59:43,381 submission_checker1.py:3308 INFO] --- -[2025-04-23 13:59:43,381 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 16:20:08,643 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 16:20:08,643 submission_checker1.py:3308 INFO] --- +[2025-04-23 16:20:08,644 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 13:59:43,381 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 16:20:08,644 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 13:59:43,381 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 16:20:08,644 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 13:59:43,381 submission_checker1.py:3324 INFO] --- -[2025-04-23 13:59:43,381 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 13:59:43,381 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 13:59:43,381 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 13:59:43,381 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 13:59:43,381 submission_checker1.py:3345 INFO] --- -[2025-04-23 13:59:43,381 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 13:59:44,614 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 16:20:08,644 submission_checker1.py:3324 INFO] --- +[2025-04-23 16:20:08,644 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 16:20:08,644 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 16:20:08,644 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 16:20:08,644 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 16:20:08,644 submission_checker1.py:3345 INFO] --- +[2025-04-23 16:20:08,644 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 16:20:09,709 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,8 +29,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9278 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0899 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4880c9fba..66bc4e1e2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1132 +1133 diff --git a/summary.csv b/summary.csv index 9390cddb0..5e862c9e1 100644 --- a/summary.csv +++ b/summary.csv @@ -7,8 +7,8 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.18883,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9278,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0899,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437891,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4d4475b5f..fc06064c7 100644 --- a/summary.json +++ b/summary.json @@ -259,7 +259,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -279,7 +279,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9278, + "Result":23.0899, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -289,7 +289,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 12d33133db01e2d8453d660ebdcc203ef92f275f..53959789dfa83545008fe2788136df9652aeb404 100644 GIT binary patch delta 1527 zcmVFxKO}=#RL0ONM!7jFf^rLH*ejelUFER3JnPjD% z?5_2rbu(y}dF;AfLMG}Xp0s{QCV!OH56K{z6|5hY0WwJj$TUemtOFp^4#(x%?i>F z%K(}r18AC@AJze&X;OYPC_m5=kwH+$gR&kigGIA~@}p~3e!dmWC;51_On_$ zNe0j~89%H8K+`1rXb^s&C4VA=po|A)Jz55fCeeam(X83{`F1p)vY$Or@8?SIdO!K1 zsRC8<&oXt1_M^!s_ekxBWJ0O^kPMPpLHl7DAd_T(Oq2D)Ish_F(vJq|2U;RB2q6=s zA5w;kW)fA2JVB{n0}Gar~fkKU<>yX!1cmQvD&BP^v#9gMVmNP=8nk&?Ff^ z(`5dz4ggJ)`J=)7ftH920+uXP^;i{<$-EyR^N_rszd@6&Wqy^Q4ELe5UA-e^?SG_n ziTdN9L*h&Ihh&s3-5(MHDp^KS$ug2kdO%V#a7^wG8%1Ji>Y(F*`vWZzJ52dm$XzJw z(K5N0B_Y_$ItSN%K(y9ydTHp{jgC0Nt5^EfcFC}5j#xz+2dU(>(MemvciY{I0xtbP$a`A zA$g$6&y};Q{1i8{eEFdTt)1_|n8u5n_&lCWQfrXpXKP)!ZGT;VImned{7 z(fXT2todS`EXLW__2|V{nJg#$vuQH?adlA3+4#?8{NoRUAH3rq2HsCUTly#K)XHam zAI)dC8m0Jw9YWUG$-I9Pcl(Q*@hr6_@hG3D-OOKqFV^2pvidno=BuCFhV?1QvSfPy z+ilz*#*5-Nr(unfB)k8mI3?TfcsctYvk?`)4u5P$$|EHK002+|000pH003lfV^DH$ zaC0wXZ*pZWcx`N*lFd%TFc5_AJVoStVkdzLtR$5H^+Y8kia^{fd)r#XKe7#J-ySDv z(^l$<<9cSlon5(_?^K)og7ebo8s!;5321JF)SDW;uNLV&N_=2iu-0g((H=aSH#Zfx zlz$rsug+L-L4r>rNPEg{jkY0Jim~TgP|Ro1p`*AqPO%V6x51Y2Z?*wk5HiCG0uwAS z+yhfP5zzrD_=L1St{nh^V`xEv4j$)Oj!)hIPWh`1gL39nGVCo}wI0REz4LN1c3qcs zeVyd4%9`#>Zy z03G8Vm5=i2d9^@IL5jPS+@?jj%4f77lstU(BQN?-Ky?`MZ{iU!=Dz#{%^$NCCCUN? zDoE!Sc9Z2NTLBQ0K`1-{nv;GgK>==)z$iKa;FJ3(CL3%<$|EHK002+|000pH00000 d00000004l2!IL~GJ^>YzdnqFZ$0h&(005=v)Ykw2 delta 1528 zcmV0U{atH-Rb9f1Nvv&wz2!Fa6>K%oAA4^q&C0Zn&w0}q@l=ctFAeI&E zAC>_unY#m6n(QAo3SeoHe>BKH&=S$Z1S~=RA!V?GWnIG#R@W^5{QX!SsQL3w@0vfw z9@h4BiIt~y376=QcvAf#nNX@fB!gU5P=8nkxFi{0VrepeSOvjp5sE>Hk`hOvrP+C7EgJf2)epm*`BpD#nB>k`sfJ~F~qrv%s zmWT{O$OPwyl;K9Uiu0puc7DDOnTIO<=zZy@?c|GQ*}WgP1WlAjJSqKxv82Y{wY`O%>KKubghK^+gudbA7{%?iqou37o{Ry3dFlU^TRSgCdmMqCgq290A!ks9}UJ2v_xbO zLM9kLqzo6$D#nkl+4%WBWFD&WqlNP0_@PsFXSPK7(d46gr1C>Dp;UfI2GOjb{ICq5 zNiu+@$@pO%0GcM@M}zPKEq@Ui1Z6xZ>(Me;G>H}ji)PKn&$pxbl>O|1dOufs*ZavA zO%#1AxkqY0Boj*Qhh&h<3fd3L0GT8MWSXoW)&Y=dl72KuKhP49K?s>3 z{g5(TG^W>!ckK+fW``HrpN0Sfok?If0gi`$>8Gl5xg8IWUfF{WR znkMsybpU9Z%pVQr541#N5U^ySs>iB;Oy>OnnTO>4{0*9HE%U1cWw;Ne?dlyVYyTsq zOVl3+9TH!vKP01U>Hd%qP{}fqN|upS(gTu`fn#!i*eDW9QwJRf+#hI(*kQ`gLheFY zkCw^3ED6D0);T!$M}IhZqXJ3wq_;6^nOl`Dth1;6*v&A|B`+?W679!9fy9^E56LK7 zdOswDNLKKESO$=+;{7-#?}v>7NSeGK2fQC>iP&Mv&mQkWS&xKTzu$ff3p78k8C`IV$B!hWHHXZu17Dv%49j|pG}kDkE?@P&c=T(;~#$*{NNq`Fz|l*+0s8*r&d1m z`)EG9)hNXe>_f25PUiiaxZ7XcjAyAeiAVWF?PmV^d$Im*lGV>yGGG1VHmpxcmL=2s z-)`gnFkTeDISp%+B-#Bh#VOf-$IIFO0J9JkzYc%%`ZUHR0RR9{0ssIJ0001FZ(~q$ zZ*X%jV{dY0E_iKhos!E=!!Qs<_k2ah7)fD@TYyyZ@e8B8T)`o zApjZUo|Mnx`DL|0O`hhnI4$G6SY-v7J(9BgI*h#PKLh1)$iImvz?l2`6EuIb6(!07 z1x9mt33rp^CtCp#lRzju0h^P3C_w>llfNiB0pXMSC?*^8`ZUHR0RR9{0ssIJ00000 e000000001hfx?qJDLw%flX@v52FNA=000072;n3E diff --git a/summary_results.json b/summary_results.json index d045aa61f..2bd6d9426 100644 --- a/summary_results.json +++ b/summary_results.json @@ -631,7 +631,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9278, + "Performance_Result": 23.0899, "Performance_Units": "Samples/s" }, { @@ -667,7 +667,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, From 7e5d0b9c9300996112f335f24d97f1ae97df5d58 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 16:28:01 +0000 Subject: [PATCH 506/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10659 -> 10658 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 744378916..f0c62658c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 16:20:08,643 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 16:20:08,643 submission_checker1.py:3308 INFO] --- -[2025-04-23 16:20:08,644 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 16:27:55,826 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 16:27:55,826 submission_checker1.py:3308 INFO] --- +[2025-04-23 16:27:55,826 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 16:20:08,644 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 16:27:55,826 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 16:20:08,644 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 16:27:55,827 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 16:20:08,644 submission_checker1.py:3324 INFO] --- -[2025-04-23 16:20:08,644 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 16:20:08,644 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 16:20:08,644 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 16:20:08,644 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 16:20:08,644 submission_checker1.py:3345 INFO] --- -[2025-04-23 16:20:08,644 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 16:20:09,709 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 16:27:55,827 submission_checker1.py:3324 INFO] --- +[2025-04-23 16:27:55,827 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 16:27:55,827 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 16:27:55,827 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 16:27:55,827 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 16:27:55,827 submission_checker1.py:3345 INFO] --- +[2025-04-23 16:27:55,827 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 16:27:56,971 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.18883 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.07986 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -49,4 +49,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74019 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.26957 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29935 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 66bc4e1e2..a788b7b3b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1133 +1134 diff --git a/summary.csv b/summary.csv index 5e862c9e1..87692571b 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.18883,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.07986,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -27,4 +27,4 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74019,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.26957,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29935,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fc06064c7..44f8ce954 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.18883, + "Result":5.07986, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -849,7 +849,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.26957, + "Result":6.29935, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -859,7 +859,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 53959789dfa83545008fe2788136df9652aeb404..7ea480bb8870e4c10dc725c9ca2b86376d4425a9 100644 GIT binary patch delta 3337 zcmZ9Pdpy&9AI7(oNsdK1pW0^5wVYx(M47`J5>2C`k<(JjA>l{lcr!Lho6OnW9fggK zqT8WRE95N1fRA|qPG!A}~93cb) za5skfSp2r#SyJd-<^dS~ za3Xm`4{gX*WL#wOZY&-5;AW4uUH6*;6hCzLv=MSY5^+^GugN=Y1sr?uj?T?>sm)uX zbNgE{S`YOmc-P$YA2Jw(+f@MoDyZ_d@rAQ9hh;O`5;jgDv5u1-_49a1r%eyD?Uitp zll&F+=IgPz8l@mfqNDt$FN>Uo%s>Xy(fu-@@Q&8=kXGfRK_jt<6Zegpdy)VzYg|^$ z3rgG6-*Rmwyik%CjCS$eBz0K3G5D$r5)Gf3ZS1?5f6@MHb@{elg=mF$jFLEgxcD+V z9Hr3KdiN?e!IuK-8S6KCgn_WnOc4U_Rn_-;yVen$UbSv7hV2gtJkNGHcPzT=)8-ZR zbQZl1n~}xbYCTbsS;y4tECl?wtNeAp_YLWLel@~!Owt}lM10R@%%^1-Jinxewgzo!r83qJ>)BfWTuHOu?ex^v8g*whE9Lo$;(p2Xu|205 z&tFQWraT^oNnQ`T4OM$GkJ~SKZk=S^(z~5ujpD^EZwN3G#5_51%_(~GZ*Tlz?9;%P ztz191Uy>b@D(Rb^4(!aDS@G1~z^zgs?ebQ)rg`;SinAvTt%L9QBtFre_0Ar-F-nS^ zYTjD%{$4`w-)Z2kYDV?E+PEI3V7I*d`J?eGuh>8_Qb3SN`;O_yPbM6v<;bHil|G$b zR5Pz>?f1y{>z`TbxtV`3nYOaYBn(vrOTK%2+5NFGrtW1IryPittl+4^2?WPuHp2x^ zA*|mHlqxhe6ul|s2(s`LGDWl9%$^XDdi5xO}pZ@ zpW-hp^evxZTb-s?-BZj74ltW$+RV2k%pE@+y?A9&;c$~;DtaDm9crUCA-UOMbjhZp zm9}7%v?P{04fNx$FrB8deq8{To{`v?`IGN?aPLxv6YuKq_N!bh3Ww@NrcaZLh{UNyP*kSm zMU)Kl=q>~)#X8#>ng&&K5fMR^P{{Fq9;d&VcMKd6djtYtza6 zyuZeUWsmYgu(OdI$|SC@F4}h@^a?l`Jk=bF1s(rm@)j@4vvhIwVW1!YnRAoUJwPfPyOLNx^K=T#b?#R&t@B1OPf^VU+ATl20@L^o(g{aFB4{=a= zQF10}LEF>9zU6ebUS)VurA;U6g6+v_k9qBJiMJzz8vBrSU(Y&O&@&C(2a!exT&ag> z)cA#asN}QUhKeh~W6bL(QsI8Pbh=tZifxf`@OBU-Pt`y00HqRQ(ev2~xN>_^fKm(d zDN&)+R@!uo`?~RGdBo$-vU5S7jlF#D+jM-yuAwT zaK9`J32bO9H~?LkSEFX8V)_KC1O6$ZH!Q|u27h@(&ER{~g&Z(EnmAm(O!RR*F=LH; zC#OZAGUw}M4P$^|9n-F*h&8@y^N1*v!Xm8KTOT-<@{1EH;O~=EIA~bZ9tF}DJ#n+% z$gEHUr)DTztkD@&D)J1|ZE%%3wC?aNPkew1?xYM6Y!_76o&)~1C}DBgiya?A%VFTiI%-eEe#qmWPgc# zbBK~WyNXaws0P}+*ed^w4Fw3g+d@>PjU6mJ4Bt~~Z1x;d-c;fG zzx;Nta5T)zb&WiQ;q+HCvrwFZx7)xMr@|%YRpmPN&Y$ID^=|ESd%_KbK^nAR1wg}d zc;fA#9PPtJc-g(Mb_6b}W{9;@?E=jy7c-GtQT9*$N56~+@jc-#=uDD$Jzo;k2dw3B zO-o;&2M%KQ*f#|2_*Vo#yk&F$+HKE3qXKra91wHLBuj0r0ds8uEHVO8tv_fUEY;Kv z-SN)?3Sq@PyR|KOs=n9m_E>O%{W&75`=O4!ChV8xoPgdla zZPf?Pf0;uQJl&lEtl)E~>_wjdB!Gp)mDzHE!O@L4Sc*AbBFC2>@D7VyU#|r1SM)f6 zw@Z4@e1klEq;c*GQRWw1lZ#x7SpgM^_FrsJkGA=Mb$mcwEq^|sCrRX5Ky54*(o|-< z*`=~=VQ(#HsVd$P#Sn31(0+T>d1H!hlddS6sO*r>l_rn@jOGm*Gd@ z4&oCJ4#08Or?Dn`Yv0>ZzZWb|>W7(X-+F9)8-^!+`j*WUZ%^#zsLDd=cb1!x7W`s7 zfKMfdf$)?+lttUb8P<4-lW-aHQW2w%nr(cpjo(~LC3lx5H@-2|eSVa96m$KTTuz+f z@OPle-+{lob1YWyGgQ{12hXh*0|6NF&l1ax_}mncJ4mHpF$|_y@4JJc`%O*{cSU#H zTC-@NX*IC7S=4gjvx7kvupuIxHgcG6{@$`RDb-=#Ul5F9XHxzYB^8#YT! z5KGnTRVKOA-vrP?f8u3rVV!8rnZ#23c;1MzVOmS~7t43xb-th>hU4-6cj<5C!c>)w zJ53s&n?ih57O*=%7<`nIzdbsd5uPvRrfhuq~6O_d|N4A$T+ufLuR!7c|QuH%; zG!@L9M)DSu--?eeiVG}M&~&!RVS!Uxjl}PdaT#a0P32+X$c3?%ue=FAlzo*gs^d_g z(em_I`f3EZGL}i=%&I^0jBnSk;hg^0J4Y_th?{Y|}uWlqE_#vs&vH6I^srBente6RuzlA%= zF79RV{{5%U{?NWXbiPG=Fo^*W$%X-1VXdLj3;8dOg&K@?8gA}kC%PMSkfcr(=6yd& z2wJH*-$F!$7HB%+stQj-uKACSxP6cS%5`3Q5SpUEy22_`LdOh-ZiNW7WzJcgxxYJE zc;Yn9HU=?wP`=~M%)x5~<*?jwjf#Q^B^hdJiX)l{FXhzCfyPCw_U+G!Ft$fVuSHdN ze+`z&*;gUOyEI-NCZ)bUn%X`P=G85oV8HtHGm5imRw=iymzD+&)MN_LEgI@=F?aRS zuRYH-X%tZ(6&iO+E*y(A)d~F}pmZ0+l8HELmOg#IcbCB}%BP7HGiJJ*JBb_rq2p;R z%WX#%;bg_bD1hIM^wSDC0(5Cu$FBck5-K@{~FET$pQGq{_qTGXIt0L z*@ftB)K$qnf04ly)^5cWI#f5u=46tGgDnl0@5=Q8S&^pAXTxx@yK7Lad+;Fj&^Fm$ zo@(^6BX}ymZudUUL7wpNS=4CE&sFJC0aSLTlTUEv9h*Q|HlqYbHw`4p8jE{#uczmyp z{LPq9U69p~X7l)7(84C~wvFNCAnmGJMCMfQ{D)pQ{X9qXn$2o=@wZ^;o0H|b?_PQ- zcg(z2>Y(v&kh~$(tDRHDBRYt%aAz1wYkr}1@aBCgxlK%gKn!$o)r{QtE5)4kr8CA98yB1Ap5sG z`@N2OhOi@JCa%q_LqCj<6-Ey{KVdSiInE+&Fuo;+ZZg?@U0Fd1PDS9{yMFYItZxTp z!Y&=vBF1ie=4@zAuGpAmqv&8RO8uJ=p;=MvE_PBxYnpt0!7rlXWXb$#1*%KbA%wQn>I*74&De)rdr+R> zg^WO1-QSWRrp>JgYh1oRmS**Kc$t)D+X zfLed=Dp!$b8P4g>U%E1$53(^cN!TGwbYHD0Khu%{zBE#1Faa9-t8QfG@vLfLV+EPl z%Er}@cMY->Xp5_%vU+REp16|s;9Dgp5tz>4S}wX~Wwc_fOFrtwm%xVfG1saiB@qh> zQAH8H1A~EQqU)Er@u#TZL$lp9f1_=N`v(RN9#fC@29bubIQC|f{$F+cSOUoiZxVd( zUDA%HvP;*qCsBB8)o$L=p=M50B}a=kJlG{+OEI8E&uQGA^i9Z=y2|q~?G%l)Q76Hr zb4GL0m=;&Lz$H}E=K4;D(|Fq&50}xp+{~HOvq<5Dt)|@igQZz1W%&}43K=5XwOVUM zgx72*PeOW}`3D2XKJ{Y=4%564iVpDE(Cz2*hRd|J3-h=3%P$t#?w3bw!gU&xD;|gC z(+)?&nfK~XW@LOjs*$KpM9HG9%2Ddl?XMEmkL5#^y5R1aP#{pYR>etNGyS2p z&L0shAJ!iu0Y6uewV9Q!=Bs}6XF4Wf%g^i@X=*|GR`UDL3*ZPDRRX+M0z7%N6(db5 zAS36>pTNT&Ex38VzgJR(^Zqj|St~V-ca`U$a&S!?`O{(CEl9NjGnkP5(1y(aCspsx z#-$a^V!@UDqBXCTMc)K^Hr<&NKVEFfysjXdKKJ`9Yu}{#2Q<*wUr|iG8zqgl9xkU{ z8jK;ZNZn=#v^E~^`*zi4Ow$xt~)-Z(nJcsf%;>@E>I3d{v9`Up**%ADgu znov`V@K54vkk#Va>VUg)OkC?(x?us{Wk2$3ZX#>tIl?72e=G)4P#`wFgCrqOp@FY7 z6#+7*nbzT!vi5@_iNW8?Xw&fjDD+9!I#1vGbTXGiz!rL)K#R)nouys?KM{beXe?e3 z!w%|q9=DsFef4p6Hv06>7Sd35dh5n7`)?y>pKABNwJ(Jf6(9}Jz&=eybD7ioFku#C z4gjkZ|2-6pGk0mU3YE2Tp>R-&qizA--JYC$xXk88xku;Gs?p%Fqo02pK3Zg68OBpu zoe5j^-?p=!>J7iO|B5owPYxxC)6uY3T5-1(hHu)3GE@>VmY|a5WKd`qCGDR2MCg)* zM;4`xK~(cHDj~=gFQDOHOlIAKhi~4>@1f6+gtL}a+9!qy6+aI}pUK9Oo{f!2W8SM4 zX^LP;;-y5zq`;9Btwo_egI@yOHxnxep~TP7vpkM4?=tiw(zJ3p8*Z=KqD(>> zoG5_nh+tucz&-;Oy#9bWdwK;b8CweRrKuKu{h9s0%MDhp;`9tapB9}hmRL39c+f|J zas_Wl(Hpwh4~r$DEq?YYxc2aS*IZs=yu+EY_JOzApBx%Lzy60_Nj0A-TR>bN+I7aX zhEb;mgL;{%eR5G>qA39G0Ha$z9agi``_f6 zspN9p70uBWkrOsU)w@gj+}o(Q27*2PASHf?-|xVdcn#CJC`Q5N#18P)3Z1?nG?U%k z=md8l-T#NgwO|I}Gc@rLn%}RwYi?%+lA-V8rslRuP+|_j-M#C0>l=P?rL(>8mJq&d zg8K*?;`U$=;#F=riRPsGTU7G1#cy~ZcYGATlKgSRM=YL?GJvAB8bsLsDne1^XCs{` zd)*Q75a2PEKJ1g&*WEfwc@mI*G1Q}FaVHkD{AV~z7rdMsN!&`)ZUB?S9{sk2y$jgC zoEng5M-n=T83Zi?m5Mn(sOSMg4Tk@gL*tlD|ZmYQZ1CL22+G!Xy7{mP_>9Edzy!h#Fs+UlOAZM*@P3xpj()cz4f4eRD4h}zzJ#P;XYB>)&Qm)6F+LrD`Crs%kG+~u-aFTkqn*)OQA zNb-{ad|s*eMp?agi$=gQ;o(i<)>NtGq@5sYxw>OuY2o7@))!bXl5M>C1z4p0573eG zzQGHnQ;FLG&EWQ2k9Yqzt|=d)s`k0tvED+mwn33{gV zX!DNp;4te2LFV>$M)9gngVrjP`uEuTs|3Bl2@GTFtlexCb@VRh7>nT_zdI5(3hv05 zJoDJ+9!ax#_mMGVWF_|OHn>hgu2q5%YTcy)5dr|5gaH64007`0;pcQCA~IUfFXBeP z$=I;a5>k)Dn^Tf0E2ihC6MVJr9IV#N18}S$n(5}FIte3}-y#@?f+tk8gV}Bi#Hske z))Mcgq7~O-Uefx&ccmFcq!>0leIa}Pd%k)9)-~B44MQ6^)J1@2RLx64kh21b%ibIdNruDvs%_~z;1cx$IM*2$BpJS z;o^?CVIKW(Y!@=j`eY>N?~tj1q;>xBvF3%cyyCXGranMqYO50ON zl8k_w3KfxYP^3^e`6bju=qs5E)jIP3dJ81~cO~9wBbyvE6Otpx9n%1IYVB?Q7r9hK Ai~s-t diff --git a/summary_results.json b/summary_results.json index 2bd6d9426..99117ec07 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.18883, + "Performance_Result": 5.07986, "Performance_Units": "Samples/s" }, { @@ -703,7 +703,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline", "Software": "deepsparse v1.8.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.26957, + "Performance_Result": 6.29935, "Performance_Units": "Samples/s" }, { From 52c79f64cf469450df68da90a2050131d7185cd1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 16:33:44 +0000 Subject: [PATCH 507/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10658 -> 10657 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index f0c62658c..db1d331c2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 16:27:55,826 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 16:27:55,826 submission_checker1.py:3308 INFO] --- -[2025-04-23 16:27:55,826 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 16:33:39,334 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 16:33:39,334 submission_checker1.py:3308 INFO] --- +[2025-04-23 16:33:39,334 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 16:27:55,826 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 16:33:39,334 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 16:27:55,827 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 16:33:39,334 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 16:27:55,827 submission_checker1.py:3324 INFO] --- -[2025-04-23 16:27:55,827 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 16:27:55,827 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 16:27:55,827 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 16:27:55,827 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 16:27:55,827 submission_checker1.py:3345 INFO] --- -[2025-04-23 16:27:55,827 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 16:27:56,971 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 16:33:39,334 submission_checker1.py:3324 INFO] --- +[2025-04-23 16:33:39,334 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 16:33:39,334 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 16:33:39,334 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 16:33:39,334 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 16:33:39,334 submission_checker1.py:3345 INFO] --- +[2025-04-23 16:33:39,334 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 16:33:40,578 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,13 +26,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.07986 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.32211 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0899 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.437891 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.435272 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -44,7 +44,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.410481 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.425121 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421087 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index a788b7b3b..e601c44f5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1134 +1135 diff --git a/summary.csv b/summary.csv index 87692571b..30ed5d853 100644 --- a/summary.csv +++ b/summary.csv @@ -4,13 +4,13 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.07986,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.32211,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0899,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.437891,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.435272,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -22,7 +22,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.410481,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.425121,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421087,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 44f8ce954..980e6caa4 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":5.07986, + "Result":8.32211, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437891, + "Result":0.435272, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -319,7 +319,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -349,7 +349,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -699,7 +699,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.425121, + "Result":0.402824, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 7ea480bb8870e4c10dc725c9ca2b86376d4425a9..6f8fbcce9152b1c3dcacb656c062d1604f2b4209 100644 GIT binary patch delta 3363 zcmZ9P`#TeS+{Y)(A(C^BLpJ1;LrbU;v0+i%IhMpQEF=@dtKib_C@wJ_jtMNl3PH;FkT1_6Ee~% z{j`PbUgWV;R8V2sbU9Fdw4fb*t486?@2?lg#$0h<#7Rw0OH< z?XSH>trUOXoYZhkr3IUFwW=vRJ%z|Xo1I&ED&7aA1sOM8z+l4<*HwL;&_iUwU80u+ z8TzpkJ(EQ}%*S4pz?IYo=R4yb@=!P42IrxE&)iU#8|xg59$lTmB=UKIvuo6v@AK;1|;)8pHn_#kg*G%511`Koenl2o&xYROqKc7r&<&(tW?>aq3<$?z%zYA3$d6uuw zVf->**Zt#R{|;l@)bV>QR3y;PW%-(;<@O6Qv+t?|&Bwo1TCrd&$;U#@zkp0K9oTUl zaFccQ9}Eheygm0@wRd7XC*CI|S`|HLvz$Z@i7;UAFSKCQa6z}k; zTKwpFAumBYx1hP`cxxg{$nuh5ZeK;FRulNX|KzdUjsT4|L_h0} z-FKC<@2P(s9PL|qm?`_ci5XnMgcLJ|dQMo8^vNL$ryoq;iA)i8-*YzZ;tjG?A{dFJ zkEJIxTy|#xf5IELfA4qNO|-7>VUt=`nwZntm*be2wYab-nsSA(*++0Z_&d?M#R3v| zw|e*SH2BN41Ag!Ee+H1atFwFlBn@Ew_XqD6&#e@7}KuErh9JkB^-D?A(RAi;#EV;h+gdKlIYxJt4CjY})BBp$qq*M79+ zxQblPzQxU3{aKE;&D*u{(X_ysJhKKrNwLI*%?mWHX_Hq?{T&OvhZ8|vMlKF_X`W|K>sTRrY|nXO&!52@h3_t+ejT{T2`00#OFJt z({;vre*V+9`S}pN|2`1DrzN&*ZP-kWUU;K2&~8%*&doN}M*#bkrA$QixR993L?*yj zI^stF2y0?rXC5GCZck#urN(ZcZqi+uSw^DUTQ8iuWGlze=Sr6wZZRmdN!2J^ z*?;?%-?IYlK!TL>vQ41~;IOjPZDMm%Qh*@@>MZoBtSG*Bvy}&mvTnCX4}^vlRaw$+ zGulsT<@Lf6k<1b~j9f_7+W^BRy7%w~rhYaob$azD%zpc6qYPZ1Hu?*Lhj^bv5;*?Y zwhNe2|DP)E|6H~5O8niRqlY|yhPR@7N8%U)BO}_bP-{1_C-)x7SV46}8wkk174noh z0|){&G%o1|!P(qluZw`Zr}BoQJfPtWO<|cU{028wmj`s6pgM8%vRxt1-)gQShVxEn z#{CDw8VA1&Q1j$7wP9jQ3eUfV!68HwAqeh&qnE#hf0WFPUt+2%Zq7RTE;gw|dZ+Gx zg7_&ucZh^T((Ve;6!OG>%9A*tIUbw=xdNaO&KfiUo8b_oRR~rpj=2Q*>*5V=U)>y@ z{wl`&LoTGEDAp*G$8WpY{@_yPo5@Y<`$pm!cKD|WDw-eyU&*|gE*!-@5RXH}SV2Ys znGi<&Ccq&V3j$;EkNe_>Rj-S$UUp8+P$b#>{*sH~Q%wOwE z*GFZoo+;r~_Sxp8hi19}FdeSyY;hT(+e5ll1!chk#P~`p9;X2Jd0dntvcGmcrpT*v zsv>xQ?f%GRk}pCIPpDOs_-?dbUmbysphrxdo|c|z710#29l^2Ht#*&pH;>fEIW&-) zLNeSn`ZSpIEXWuC!{Zo9;V?~ixRvt8gQwD;ggc1V5c;8Rn^0fOs#zZfb+E=PI zB_bzc`|1yl8Sj-w=U-yuvXp16HIVZ{%YYhvJdC|2HC$`)0sebpWM(#_Q_|E06{tul;PGJ^0s2`caY483Wl+l}I0iATj)6F9VxLu_J$ zJ_06P!WQgSP@Oe(P9?Q!)x~ZU`;(&OXV$fK*8Ot4N(+9zVHSt#`SPIDy_@1}zF8Wh z_tR`tUxNsyy6R3a4Nfl{K_CrHh<4&JDWkCl{@pXNHU}zt#r`(y2|DfC!5TPc1e=Wi z+blDBrC`#@N4Pkfv(a!Vt=zwoK{u&uzoWX19p)^L&vmV}71*{9TT;zg1GB{uCF^Fo zMGnz$;6PR3UX4H@LN}(2w(TJY$c0QuC% zM-ec`i0`{JNu>BoN+fYxLFA-+&w*8Onhbg!_@2eFtNAzveEd%}y>wT3x{Z}r7Xtge{BzvFs%=rK7{@GD z_*MMQOrhDdHG69^G{-?A$IdugT&L+0tRhUV>s}19`6VQCbM{+|9S~E`c!ERFCn#McG znkxC__}}h0?V(8|W#@wRd?k78DN~I`@rpPc#f*VElcvv?dpsj3HyxH5fJaxuFYJP- zC&XJ$;Bs6%hQS;FfZZ_wKo|f3cm;ae-3tr~(e@0yhtdki1QZbHer;z@UfJ5`FaDqj z`)U`rV&Y2iE40fZ{fg5^n7`&BxOCy%JHqn0C`yfFlG%?9yPggifWIf6*N3IIC4%c3U zFA@H=Gad6&uMjF7_b_9eALr;DypIQm_f(W`^Co|>F&8CJrO?pJ#$-ORe*aa)k3x7Yaaq0EhEPYj%wUdGxNQNzZDii$wrbhJ#HY1agqdpa%j7bi*?Xg_j7^}KgP|k|V*P3qjrGVA= zDe#t*)73kDCKm)*=hwJSg}CElcZ8^7;GSbgUmxnU%0&+DqkAPPt2uCp5+ACa26jN$ HmkanGJjq}` delta 3380 zcmYk9`#;nD8^{v1{pJ1#UXSPd`MR#_@pu=zmbli23Gbdeh=)-kgkb>w+Q@)x z;^O+V&&QeqE(Lu)cS?;k?=Xb~!;@L^Mq8a0W<&M@&d>+UiO{{r5*K3|s-s4dBPQr; zQ`-aPzoN7!_deh*`x(_OOr17z@UIl>r^|VuCV{czOF{hh^uep9j&T%X7MBWD)H^;d?ca6p!Y{O_h z)Scp8aYsI+(}_200s&NT^-aU`ZgWRv(%Ta@P9v}m)40Y30>p6>hu&U^Ksm}?Qfs*y zhp$rzhL9ZOK7Co@EM^4J84e!4fc%HTzK4Z1J{pB&CTZ%vK}%mE;AM%=jD4BkKJ&M1 zdnqrB?4??G;lng_6y6+i*%@ICoBPo`aQ(pryRWqs+q#uvmEN%m61340UseQ4zP;`4 zWo&|9zG~mZpg|P|%yO9_2HmS^?DuwUAUeKo+g=Kn3=KNRaz1+^rsvbzPd<8%lEM1tfi$ z6QWkmDMNmIKVH~!5H|8O7nTyWm9-9?TC_rETGuLatX010*FX16(jB}do{iOkwW20W z@}qVFrdmou<*$`g%iWI8Y%Ec?$1{^(tQ?VqtWWGYLx1rKl9K#n zR26bH{3cZ8=>lF7a(10;+1kIIZi(W>vo{183F4ld_?Bedh4zn>ean#BK6jqNe|Tq&T^lFI&9rPSdw(yb z4em5?S2d&iUT<6tm$zlJKX)6x_KFJPUymn-n1)w9oC+WgQpr(<5s3~btVRo-f!Um%i*Hf3 z{xsdR8F&}T)r$IMRVgpM-5|YCuEmf=QVX1}qzoj73^n)Hm2rn;XI=5z&j{xi2iPtw zi!-#Edq=WE0@1S!tA*Bt`IBd2mM$&HAANTu#d^WoGR#V43bNU0aM7x>t#HvGaalZP z78t~Hi4!%40fBodiH4H5^{PD^)R} ziP#eu-0T%5FLp9bxn5oUPo6A44^B=d@bl87og1a++ymmX`nxL8iJK1UPA@aXAadpf zR({T;109w4(BpkMpDz3zeT%vVwuwg0J&WFui~?4N=@~Qb@xL_3ZUM(a8t=)eGm7d@ z-gy#XUi>1okYcX{3v)ocQ|_|fk2^!QMsyxJ(lnANokj=mEv-sothH-re3q_GkBBA#z2>_f!- z$Ek`!s?^e{XRl0MHvrAoEZbwprT!TN+X%gf%77>Ykp0I=g_mN^$8Zv$isGaU@*>>R z%&zrJmTq-KakW(!^SsTeTHFGBa{v3WVU2x=hOcgpX0!}Fk70y?lq>acA@$bcJyen# zx2f`y$OPl6Neaw=mv&F9Xo(FX9@YWM&s7cxlFF|JoArIR0508}7R;|#^(j@#udlZ1 zob+?&zom-Df6LB!Bn#_%@9a4PsZn(0^&^8?#-ElAYzsAFc6wG-skwXQJ7E5q7_xxA zjnEKuaY2Qek%AcztPT9PnAWs(3p4!N5jCCfs0T5mfBep91)JpKYBFbu{~)VHq%sy7 zW%Of#QElU%<;XRDYRi}yG@nUaZ?u#;k^GwzDj49CSTw9(+z|~zikVz*G(Z<=;8pZR zN;JBn%S4}Jdi5?-=XV5uW0n(XP0d6h;{1IgRaGbT)XR5bzY#tEgGksV_C}PMKSVGA zQ4{KnJb(Y+2u<~3Nim4yIW$ztx;u%G z*(p*Y0d#@x**6MkV(KO41s)+eXxkCP!@tL`OmqBhy@G}XJa0%4I}ozw_MsP$%mD{& zy>Z=o<%oa%y0;&lntSK}B>v1IxD^yExNatRX=q%f=buLT1{G+JU zjC(658RN=h{iTz=8X$vegDP*#?tARNbBP0Jm{#Z*c#6PiujhV1@$%m8Lth+=mYr4= z8(2G!Dkd7;J7{*qYwCKb(1I1f8kWtI=m2HIj}{YT_NsQM>j(O;GKZVK(Mz=&g0#7bZb=L4v=ZD zKCehNhieU)!UeI2NN_E3*fiw8yI$x{KqgS6TGF>0Zq8HoyK>vlt7WgpAbf|rja7KW zVLL45o3_aetVJw5jMWL>d;j0eD3@*JJg3Ke5uP4S09NP)ROW(DAOgUG!B={n>u}HS09X@$*W6zXoXWg724gpZf*ld=BD07NbqiyCxO8mM{Y=@7R5@LRD?^ z0UP*$I$8mIKu@yhmB9KqEcjiy&1R3%Hotozb7hInXu7CF-N7mebsMzMx!Zmd3fd{mWCf}XNrB@s>qZfA;;aql<1Wt1ocAl{y`(qaTcU>GE!S@1a( zgs`&I14Wla@USL>^mLfRK@$`nuavHr7dB4cy*x~@f4&=&ZH3bK%~j6lLTXv_xdO_uJ{TYPEQf{57&`DT>9N2ousEY4#xvo$eix4|Hqd4y?{N93^#_~cw!0MR3(r< zeam7}ZVjxu%hp;9sDHhPWT#K z@wJT5D`p4fyOfq8wSxO|>CfIN;l5b_4-(jM>f~>fOqA2iZ0E3hTOK>LhEXC#kdUta|Y6B(hz!87Fs+kb(EP}U`^j>0oNkVY3vQT@Q5*{?8 z)lB;S1fTB0eOD14fmobq{mPs2N7>cbpgIo+8L($3(pDoW)o~0m=ZD&J?^xSff3bgUVlXYjx@@q^9c01I^^7A61!Iqd?0ARrKk5bEm`7JB8H zu5W0VpH37p zsL+0cBfg6k&=FCZ5k8?eawAl@J!9U?<^Jv@k*PCyn^^VvLvo#O=MG&ds8G$B)Tk_& zQjn&mBs*9$U}c=TdC;V&#XiaGNJBeB%vyA9@7ECN?0uC7co!!t!VjpekEe7Dg?sfL zOweO~`W4ODL|4o1>n}_d2-0K-)6AM0Z7_Fr)2_V8F=`f78yB8*Oe&g)GS&|JDX4H4 z#FUP7L#NH&@86|&BmdLX3YwAT?m_0>`q=p_j_JN5qwZ+I!^jJK7#l1s;t0~DR6BS5 f`3up$shJ4u`6HHcRKrc+03}M}s8BnMZx{4G%Y1_Q diff --git a/summary_results.json b/summary_results.json index 99117ec07..d564aba8d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 5.07986, + "Performance_Result": 8.32211, "Performance_Units": "Samples/s" }, { @@ -572,7 +572,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.425121, + "Performance_Result": 0.402824, "Performance_Units": "Samples/s" }, { @@ -883,7 +883,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline", "Software": "onnxruntime v1.21.1", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -919,7 +919,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline", "Software": "onnxruntime v1.21.1", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -932,7 +932,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.437891, + "Performance_Result": 0.435272, "Performance_Units": "Samples/s" }, { From 21f5119f45b97fead2853e1e15c299182b5ea8b5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 16:36:02 +0000 Subject: [PATCH 508/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10657 -> 10658 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index db1d331c2..39f655d74 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 16:33:39,334 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 16:33:39,334 submission_checker1.py:3308 INFO] --- -[2025-04-23 16:33:39,334 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 16:35:56,962 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 16:35:56,962 submission_checker1.py:3308 INFO] --- +[2025-04-23 16:35:56,962 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 16:33:39,334 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 16:35:56,962 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 16:33:39,334 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 16:35:56,962 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 16:33:39,334 submission_checker1.py:3324 INFO] --- -[2025-04-23 16:33:39,334 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 16:33:39,334 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 16:33:39,334 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 16:33:39,334 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 16:33:39,334 submission_checker1.py:3345 INFO] --- -[2025-04-23 16:33:39,334 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 16:33:40,578 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 16:35:56,962 submission_checker1.py:3324 INFO] --- +[2025-04-23 16:35:56,962 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 16:35:56,962 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 16:35:56,962 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 16:35:56,962 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 16:35:56,962 submission_checker1.py:3345 INFO] --- +[2025-04-23 16:35:56,962 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 16:35:58,180 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index e601c44f5..61280f5be 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1135 +1136 diff --git a/summary.xlsx b/summary.xlsx index 6f8fbcce9152b1c3dcacb656c062d1604f2b4209..0646be62e5de8a952555ffd41a7de7807ca27ab2 100644 GIT binary patch delta 365 zcmV-z0h0cqQ=(I_%_D!B57dh$0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!E= z!!Qs<_k2a`C3woIr?>~kW+I{^lCueEer!7c zIK$9_1mzvhk_?}{9;|dX8wTaVDTO~6xM@9!lY8&PWbC>w>59P`@@IIndVL>OjfLt@ z#Q@bMXM|axUfW5!#Fs@lTuYr_g%fWD@TWu2Z@e8B8T)`oApjZUo|Mnx`DMLCb)M$4 zIK7YaVx7%NF(b2wuYTlJ{~0KcL;g)X0mj_dpP>E&vlAr?1O=K8)QcvQBPmn?6q9%< LBL>DM00000a&w}z delta 364 zcmV-y0h9isQ=wC^%_D!AT4C5F0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!FH z!!Qs<_x%c^y)8e|P=xJ3Te51PP?whOgfvM+tp}1?$A4eNcANyV>T-1NoVhcoTpeT^ z{epEus|saFiXu=dT-Y81B#rdiqlprs8A0Mt?Ijy8Nz?Gg;%Q$STDdu0i+yZ zrb2t~4Z+y4JxJ=3;7~!_YAdM^rQKme**Dz*&eL>>C3woIr}zkr%}hiSlCv3Uer!7c zIK$9_1mzvhk_?}{9;|dX8wTaVDTVJ1+_av=*?n+gHg;W?bj4r|`7`{vd3hUFjfFa% ziUF!i&Iq$Wy|zeoi7$(AxRyG<2`AnP;E#RKZ@f*5jD5hP5P*zvPs&H}^xUjbou~PI zoG#+LXtE_K3bIJQjw7%7&p!BDO3RylX)p4 K2E`@-0002A$fL{v From e4182854b3cf7dccd60b3d426669e22fb258cb1f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 17:29:25 +0000 Subject: [PATCH 509/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10658 -> 10653 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 39f655d74..1f6777429 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 16:35:56,962 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 16:35:56,962 submission_checker1.py:3308 INFO] --- -[2025-04-23 16:35:56,962 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 17:29:19,711 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 17:29:19,711 submission_checker1.py:3308 INFO] --- +[2025-04-23 17:29:19,711 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 16:35:56,962 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 17:29:19,711 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 16:35:56,962 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 17:29:19,711 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 16:35:56,962 submission_checker1.py:3324 INFO] --- -[2025-04-23 16:35:56,962 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 16:35:56,962 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 16:35:56,962 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 16:35:56,962 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 16:35:56,962 submission_checker1.py:3345 INFO] --- -[2025-04-23 16:35:56,962 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 16:35:58,180 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 17:29:19,711 submission_checker1.py:3324 INFO] --- +[2025-04-23 17:29:19,711 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 17:29:19,711 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 17:29:19,711 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 17:29:19,711 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 17:29:19,711 submission_checker1.py:3345 INFO] --- +[2025-04-23 17:29:19,711 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 17:29:20,893 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.32211 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.86363 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421087 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421364 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74019 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 61280f5be..3393c6ddc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1136 +1137 diff --git a/summary.csv b/summary.csv index 30ed5d853..a06820da0 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.32211,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.86363,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421087,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421364,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74019,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 980e6caa4..b7a05bd31 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.32211, + "Result":6.86363, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421087, + "Result":0.421364, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0646be62e5de8a952555ffd41a7de7807ca27ab2..9f44cd0b471d7399fb58dccd18effd9d6ef032bf 100644 GIT binary patch delta 3515 zcmYM1c{r4N8^_7gQPx4I>|;!5A;-RE!eFG7Y!fk~Q5+(>JeC|IWtm~DSt61Y1}WLH zj(v;BzK!h0)?mgv&UxSKeg1u}>+^i>@BRJ#uIqVi`O31y>kqc%%&X&%@qT|?0vfK+ zdb=mse~13_+@@PRXDcy%U?FPna$LBJbk{!dK7i|-8CV!rbEkXl zX5o8CGjZE0u|&g(1_f>R373&WplaOwviX_KysY`Ctch5%>(&ZNty^4vVZgd6 zcVOYFPcXf@IBk`P|NcX~b(8#rOg27a{yMehGYCYG_R`WH4%-v~j@8(BWOM-LRN z`Gq{J8s&aN4Ow)_7{8=j!A>gIel>UWY9Z*=VL`xGVPxEcT>pgiA2Z7e=EUVER96l2 zn64XvPsx??k~3t>}Vh7swRr#Op8@TE*i=I z^j7oJA$J?CH8t;^m#+Oy^pEA6iiLswz*3*3`5s5sQn-F3cx{poUgms_m~RK|2vzUQ zTn4T^+llqRR$Yt24PqrHagDfT(i;b9yy~Z6n*Jtk0Sd4Y*OnF`vR?E*P0vL9T-F?? z*benM7M5FvHEZPG3Tx(}bH|wXQp)GO1XyC7LZQ!3FRB{`KUU0eX0-tGMN35+3&3Ho zg@rA^^PuYn0Vg~7n(ITQhFG45!4d|sz?=1p>I$DdsyXS}&>_X7Q_v93UBdOBeYKWl zqFm}Rl4Ve6U@V`5Jr`M+`3xLZW-Nuf{sLnQ$|@_24Jx#ZqdK20AdhS)C>t7V;Vs8KC;4iuF5ddg3N(=OJFrCH?m86e8@ z`z5GA&#E_$qbV!{&*|Bw1cDI~z_=G>Fr$x^k}XoJ`rj~1ImD-)G1BEQH-Uhdg41a2 zTab12S~RjllRsILUyTzgKGb-k?f&_mEmN<4V_laz&qYn#Zvu#?>7b&Esvsk95_Y~LfncOpwqfZQ6 zjr1T_Wy1Kch7mAM?0sMnJq8}p$*Ww!ff!=Rc3or)v%!oQ0hQk51rRnv(r8b+B%g~O z-yPP>rY2)tM7`g875I2N{v1yHuo{Ryu%R3Nx*EBz`=u7eU|t+);w*zJ)k8>3>2u>q z)PPDUDUeBpOZ!PBk50)2J1Ldt!NgKVM7Hc5kiR`g%8zcky^~$VET=fXK@UtIRT>E% zpJhG;2>=g{QojMJo#T41;S;`o;ItHdWjK2;Lzn+5E28xoWSO~FdAoN+=XYqnTl5bo zHeX>mLCIbPK3wIdadZ+hq}XcGkx#61V~^eKAasF*uIJcK{f$F(vVLHAwjKL{S=^Mu z*#Bbr3{kP+sb|F6z1wh71$`Xr33H+Lz;aH2fZ&`i`$cd35!_Cwj8(U?RT&pmiK)}{ zO{Er?ycdnsz{*yL;yCg?87N_0*u10AOo;pzrZh-E)GiYy?NF5M1kZ6RVEpIBZvEW> z`E@qAOXJVX^~i;sKy4GOHdWAee~pwm;;dZ~ZHtxtF2+FpG*D`DVXKZphay~nA57_x zMh43*4J-%IS5a9WU1=d%032fbpUCiwN-yqDh07x~Uze>a1KGO}yJ zMQewmyIgMgD9~i@2V__FXaW@<57Y?UPomk1zJU~ON^jRMzFUX|$~Qg5dNoW1bgMI@j`x zUb&S<-nnn88`f^{9T@y@lxth-T!_1yJUV0XPynv=>nPp~N)yKc%upM{Ya}I*3GOCg z+o^(d#P|*rA^JplQMVV=*O;gymbharq6fd`IcYTf{%A#!>Sr8KVODaA5I9Wl^X)kG zO-bPn2LY4d-%poJwHVa}M_5~x0M_g!s5Nd!*D~M-H1rLe-Y0RSXA0xtFJYxdB{pyq ze7H2Nk;KC-V9z-qd!a|O<`G@s2-iV*5}j1w!;!og?|+NNz`bMO?5!*qxGJ!SgOjTg zV11H-J3KC>3|1G#n~2VNZ`7_wKqDg*s;`EzG*B*oLO)7%611~g57fP98kK{5J?C2e z%yiI5tB}T+xk4PLe@PyrtH5H94EfvfYM-sQU zgp+~jgu5MJ`Bju(v3Rjv7=L1MiaAH>+c^LJeTJYdLl9+E5@OBX1{RSR0MkmiL(TA8 zxee4X0+#by8!I^~aoVu*`Bv^D-;i3K)F?UH;?rNPr8T`5gA#rbvhim?Fkpq?`wjF_ zFGvQ3>zPjI<{>@nr%H-|iQ!v~N_vcQlnR-T@?k=yy%6VWVYD5^dkrG2o5v~CCD#)7 zVdOVe2PO87k{d~JKA7&7by~@_d3}hX+|<5BWeCbK1Tj_?48h}*J%Bna;bubMH*)oV z9}9*nG1d!UL=@6sr^BxkU1rHY`-0m;cJ*qq_nk> zyG||fbT9#FU=~bTr}z+aHeh-x)3D-m(iBEAo2sU7xhxARN1Guw1LK$Q#Aij zVeV4m4kmT11IN39UXAbup5ayH;&0ywJVyiwpYM!(8x0zb`+kZANtx-t)ED9=U>wZo z%ZswSJ%ejl6x%xE;vvaR#JN*K7=_Lw9h5d4{0m;$JO+zM*i0J$D-l~EP~0>=Udv*h z+&*z9`0tF{?+XCLHxtE87MC)_VK6kDdq*QJ^tYBVd ztAd6mN_Qq=$TgC}nw2bYBx_ME%SPENU<8}uOGu09}b!igi(3s}0a7FwDbje&Wo zNO#4#2QobnJ{zC?wb2GQojV+nwB}^+F+=Um#z_{TV!T@Qo_S|Wr90lY&(46(34Cyv z*uUeP^Tkq8?@oir{WEWeE9Ev^`M8HI$ivf3=a>&SdACn{%t{%EZ$Mck494J8Yc1_z z`P=iZA^Ziyk89r&HHC4y@mjPc!^|D=Ft76zj~P~BuPh*ZiH9g4(|+Q=KUuS~cFd>o zx==0VBz!PbpV<{(3)Nu$iKjv#=l)5OnTg5b1QQb<)Bm5)u&;QH3p&g^_<#%2>%7E7oRv0)K%NGdrl77B^+mgHCto3k__5`|3;QBqE` za`>ni;p?1J%VC(~HGRI+{rPx3p0C^Om1~o4Q*lRt*XnU2IDm9dKplAQ zvhh5(#IXLnuiRj6Gwxc2{F~)}&XDzaqP~h9x$p_Ivz-ER4gT%q-O_7PHEr&#u|_Qw zf7g&uyHBN&kae-FJ~$(Af`!MjT^F9+NY@}W(Vn**7!elyw6Wvsq)WpUk1kQRAqm*_J$8wOfA9zw^Vr1HheeGzt?O!p%!&j z9IK*+AH3#00Pd2WlS`Dmyn!|;Sd5lrt5PJgZE*h~sf2Zx8~Ju7}fC5`F;WtmBD zV@->nc+ReG-80p`;;B+viAo&rn(6Ga(@e9(ESWCSb4GEW)$`TbKU|e<1tvd9w-)T& zAv#0I*W1Q(2H`0D?MnzWbY`~kN5E6E`A|jCwnp)xVwW&EA?jeh>s*k&OlwQrZHI^M z*^u6mKA{pU|D5$C{#M%S>hCT#Z}FD(Ez666BEGk-%w4iKUC$@8x-Lr4JiRNW6>`@S zJWXW1bIIw3JsVCvS7n^MfnLABksFiCu+HJ3th@ibQ`H?-9;Ol3D5DR3)@ysaJCZy@ zt+K(X(XJJ1Nuviu@%A#%xZur{HRUm;>A7Td1(=Oi97R%2JI7e|Jvf}2sSa)S{}8IR zxYN@dV=4;9#=Xy~W?KwemA6=E3dd)i%MRZ48smmekZ#So`WC4JxihCE$u|CnP%6TU z4P;WRj*AA`7__MxSGA_3fieEG5^v07S&lKjYUqnLe!6iuE_Rx^PwC3jp;+T)#sV=` zr~aν-duTr4#dqc}FdW1=Y6_#gf7ru*FwR5f%Sua8OTx5X+=ZAKb5699;?=19iT zOVv8;^53T>G{(FEL{u$$_-KdTTGCLtl_W*sCO^#NS4X~_1WkKdZN~oEaE6fSIo;6h z5&HG={L}9f!1jaBwMK-79#aP=i>KtXs*6ZljlG0&(ymsLWWn5`kC?`itP>%(1s9a7 zj>5L#D%qHF=Fg&`4hjE;0gt!wL$;;Ia(!nd{j+^<{__LyIDNNjfgN&^3_dhPFK|O{ zQXD!d1iip2jI$7t;YH|OuhV=?-3cZl!*y{_+9S7}WNkWXUi*i8mF}j=^wqQcOIXkX z#*dD}X6ahwyVFYd#%_it3SZxH)Nd2;GgTz&iKP5YiLE_20Tf%@Jq-+VX*`Fd&RfXC4H9_+=5E$?(V zUrpb=cQe`x1x?m7LA(-knP$8asC7Mf^~0iKd`7`hKQV6OOIHoLG{+JWMrD*D6ywlh z7~_1jqDaff2*u+WFzF6GheQ}3pID`wm7sEbAz%Uz^QF1a!Uo;{FbyjcOKc4mT`Owf zy7+|}^XdWxPfcS9-V1xYQV#DUhbMtCro;-nC!}0^#CO#3Jz>W6B{OPY+Cj#VO5urc zTL~6Ag;2+mIE80#K*}U*aX@mRDQV|bjK+gCdnR@+^BS)}`6pR{&mK;755@$66FEk; zw31Qu`I@Xc3GFBfr3Y@Q7>5s=SL9tDAAIwng}gA~L?ArS}m< z?l@^pW`y!)GB=K%yI+PcRcOwIYvN?$G#gTf9-q_uF1Q3)%&9DMj~1^bG@hx@&NS@p zxsvvP#(ame3C@siK%%E=O>P^BS^VIR!2Nn&Frjx)vRkC` z(!`VR>ZvTRtgEUR0g1g|DQ4=ax*HZ%=gjW+`k~o&!hCA#g*{j_s; zraQ4R^hDR!@*Ue`N`K;pWcpXNU(D5BV^C-#u&|;(+opXPJ~yF$$~l>aNDN=UveXSy zLw$meE)?k~^s(r9Oy_D7KhoX2*(Ajm8T7o&lzxNJd_+B`6BUnT70Tje@0L~h=+@Ic z`d9EZ6G2I1%z2azupV7^46Q{Q{Ed%;dK|$Lx%PCM1r4eH-}PTr^_)WQtH{A!{=MPN z=pLZ}LvUa~!wG4ARqWB72gl5inxeHt>=*eQWo{p$U-4heg-#exwSifMj)W33b--16#X2;C3U<#`f z_FgmfDxn@p+aIB~6rS0Ja!|L0X!1GY^K$82NFd9F+b3HPJiuLnBN9?={nYayiUkqp zK$p+j@p>V$_`Ay(Q}=nW3ZewPH2z!b4K_RH(%y`$ng{5Kr&`5D$Ev7-iTewuj1Xvd z-kuou2s7v)C=JSpSq0f<6TlGs(}P}d{V;2B=6T1YRE2blc`mp&GW3<}3~E1}`)wF7 zToN`^jGw#F<}%WTfH#R~i&!2&DT|`VW899=4|EFALFr#gQg-JJv#P+z_;1}<*#NOv z6ldbksk^!b2i?zJ;nUE>2<)&UUomza!#iq9=!Y^QY4f+=4bUW<9K?|<(UK^{NDVP%_5(L?jAl~p1$9%$(lKU8o&)0Iz+!X(IlcKVmZJuH{1Mc zzWUdEhzpKg6*|USp+!SUPk_DRe!1*Rm)}iB^fyu7xbT;H1L?p{I9%x4*8SppU&&+F z?=Xftsno6)>Z2mEB9^cAc&r$=Wcq?b-PkHgX zt9sN&S$s3q$89HDz+Quih6q&$6r9_QD42{RhCA>Z6w%l{|JgDyH|A4ti(EG9@KbVX zVfSe3p+;i97^Q`;d|2rXYUoxAB^?$)15b-?$3U(nY3Isu0Y>AR>xr>n^gOZ0M-z!#~_=!csi$`mi~m2jQRF!#{&sz3=Cp6On|IcA2<&qh3~!&o4!#(-F`j2MX3l9xs@f1(WZwx#6KfiR zXcl5%jqtPhEp+U{P26nWc*G`WPCW~@WBcB9gQ&}ZDD=~H`KNaTRD5maOP;MDJpi@4 zrIbg3t5x87DZY0TEiY{Hw{>92>(-kJQtsx=n5ZMP1H+v95id7@+a|UvE`czlweg6x zf5D9LzhDE(rl}bQ>a3`Gze~P8%w7-#R?Q z!CyUVq+hH*kp01+H2|XbHWP9d;cuafF!IZxFV_K$A0t@G#u;yJcz zdH)m}b^A8RN=j`;m@0sz#-F8jjj`R|8wWm{qUyX)TE3Yp{S~9BM2#sApSZj$1xqs8 z!Rm9h{_G>??Qm0tARYVu`^p+tapF2m(uL{o%akBhN=T3rVJEnJNOdq>u3^#&4EoH)1 diff --git a/summary_results.json b/summary_results.json index d564aba8d..e0abb69ab 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421087, + "Performance_Result": 0.421364, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.32211, + "Performance_Result": 6.86363, "Performance_Units": "Samples/s" }, { From a297af0b8786cc0abdaaebf46aaba3aa2e5b31b5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 17:42:11 +0000 Subject: [PATCH 510/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10653 -> 10654 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 1f6777429..9ce2da151 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 17:29:19,711 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 17:29:19,711 submission_checker1.py:3308 INFO] --- -[2025-04-23 17:29:19,711 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 17:42:06,327 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 17:42:06,327 submission_checker1.py:3308 INFO] --- +[2025-04-23 17:42:06,327 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 17:29:19,711 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 17:42:06,327 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 17:29:19,711 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 17:42:06,327 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 17:29:19,711 submission_checker1.py:3324 INFO] --- -[2025-04-23 17:29:19,711 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 17:29:19,711 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 17:29:19,711 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 17:29:19,711 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 17:29:19,711 submission_checker1.py:3345 INFO] --- -[2025-04-23 17:29:19,711 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 17:29:20,893 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 17:42:06,328 submission_checker1.py:3324 INFO] --- +[2025-04-23 17:42:06,328 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 17:42:06,328 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 17:42:06,328 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 17:42:06,328 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 17:42:06,328 submission_checker1.py:3345 INFO] --- +[2025-04-23 17:42:06,328 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 17:42:07,311 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,11 +26,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.86363 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.94993 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0899 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1817 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.435272 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -48,5 +48,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421364 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74019 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7344 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29935 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 3393c6ddc..91d0f8a3f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1137 +1138 diff --git a/summary.csv b/summary.csv index a06820da0..92d138b97 100644 --- a/summary.csv +++ b/summary.csv @@ -4,11 +4,11 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.86363,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.94993,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0899,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1817,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.435272,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -26,5 +26,5 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421364,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74019,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7344,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29935,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b7a05bd31..b79fcf90b 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.86363, + "Result":7.94993, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -279,7 +279,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0899, + "Result":23.1817, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -819,7 +819,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74019, + "Result":1.7344, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -829,7 +829,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 9f44cd0b471d7399fb58dccd18effd9d6ef032bf..a0d608d01020cb6f1cbce374242c043349efddcd 100644 GIT binary patch delta 1579 zcmV+`2GseTQ=U_>atH};y%Aw>3;+NovwH{$1qJ<9=!BuOi4FJyf9;eosV&Z=Mvugk z`VYy3QvV?tV3K7dlPn{dBpGE%EmoWKA2y0)($qmmgZ%?75j{+)<3U-El>w6^1h;*3 z&Gye8l7EMjSIY#eP^ow2A8of(ph|YIT9sgl5{W0}ACd{B{6jK`Wd-?%WdKX2?f{l1 z`G<`HSeo1)4ek%Le?;^!0ZVXyNEvKkS=X?E)it|6e?OK7D*n9FyW&sr54kK)mq>Y9 zmvD&+i6_k;k_n~xLo&!^1@nhxfJ>6`C6*@fhjjp0n!KOwgAZ7O_d^CjR~E{8qzqhH z)-|}YI}=Of7TDlgwpyU8RW8p^}{m2 zCCLDnCh3QD09=}!9}UhAv_xbO^zopqN6TOrTfzC!H9J3#aru`Bd9_Tk(oS|)`q8=- zI6>Kp>6DO(_J}8?ACd{B^g}X8W(DboWq?eQ0WwX_59&7l~_NT zd~lDne||_Nl-3W)Aej}cAC>_!Ne0L?Nk6OuAk*ahXmEa@B_e|mGQs&FWw>Zoaej2o z&d>KD^H9wnEzBR=)jM46qgi79X!1Ee()=NrP?|p^gJ@PTe^>_4BpE={B>u1t08Nwl zqe1+EmWT`jk}Op9SQU`Tv>zbzkhGt_!IG_Ie}0vq4ELe5TwSK)*(K_ajSh(~)gO{k zmUMqe2&iNkNhQljD(L}9$-p+bKWr3r-M8tX3H4k{6eDiS}coK;ld7hh&r`y&n=nBrAA7ECWbZ zfAM~7llQ|$0VGY{j}6`rv_$MMOrDnD2DuJTjd z%<|=D7PNQ124fm8ZsPNJGD#bQBtKj0!fosNyFsqh@yGSoXgtL8;gZ>KTzuwdf3p74 zk8C`?F-{ia z?CW~;;+ssClm6K>8UDCBsO4<@=Q95Bhrtit@ec#pWSMd;s@0iQTEq|8 zhO~c=lcs4ab;WXg@0__aa=F;cHu?qYgjN;Gk`zUtnC3!lEA-JU<3|)ZPZg)FR#2e> zIJBtmN@fVt7T&BjV7&kr1(0%tnF{T^Hw0tHb|9%sf-7@LWRMkHqw()`$V0C0w(1qsSKoFy4Pdp%g`ZZ-_cg;NTDFmTg)5-0cG ziOJY?UD6eUHRR9mdiDA~tQrf|pNavhOU?+hK)tqgi7$(AxRyG<3MbwO;7^C3-*_7r zIragMLI84%ds04&=a*)Q>O9Tw<8&71MUxsmkXcUB`By*ks{ag>|Y>rYVs z0kaP!$^r>*y%Aw>3;+Nolj0{=0Sc2kC_Dm`7L&0S7L$%BDgkJdxhOgT+LQDsCIw@w dYsn^)A1M_9yOT93J^>JubSWbS!X^L!002=U?|1+J delta 1578 zcmYL}jX%?Q0LM2E)g%)s>%qv(j7u%2F0wI~B&7%1(t2RBg;%8`q5VX?M&-uuHxreI zMv)%IkV|59$&gGPM;$Q_l_d?0+2(%Tb+7yW0q@uQ{ri5tzq~!y_lTkwptAGU|8>$B zfv^`REI=6=)&;%m&J(9Cn$`nCOTm5FJp%#9miW4Vn9JBs&|*O(fxg8{N;2C9?h;If zR7vNK2)7daSN-Dhp4@IGd&>8);!n{q6ko3seLh`x<`nB@PH#O^wOwQ+i0{(LVq1)! zkemWh0#;^Rht4J^poff0p#_Nm+B^|+%;Ye;0{D}VRP3vhS`lfwJ*kksNzW~ zDfOt~2W}X`A4IzNx1mQ7?Jf@F<$NqFt;xDX?n&nfBsb`uGqN8Y$$b}xp~Oe~@pQKA z?J15z5c|aJJ|)493BK&lY(q=Wcg)yeaN^{Nrz8qqipaqaabplnAontMQ6sw2n`!EW zw-Bt=OC_`ua+w4OWLi!C2OSxj&7RD*pdtvg=;-m7BH?qWL&R0rZzGK~}cJR_oqrKgm3(P+G zKs(2P%pO>6_v}AvU;k4Z{ic>%gk99}-E8Z>W*xkHboD=GYT*TSXl*>V?5vC5YplOV zlMg%#^KDaHrs)f#kLVUeE>547+luW(7Qh2>xdIP2)3X7*aNDX(kCda*XIg=(i4Cj@ zpsMx08Nc6yFDPqCbg%W+mdk9u6sxJEW^fFRLS{MHmNGHEcAKpoFn{7z-1ODuE-~%p z5-r@h>ZZnNa1C6V^R@*vRfK&?IR%`DHV`xR?u=g{7W@|2{iG+Mhxdn(K0`Usu|~#; zT(UJ0p4;V$A{Esum1+1HzE?m>?Bzu%Iq-`plFpV(X8#2%zI+2?tm*w!K#?-RA%Oi* zhI_!!NONx>=i*iRV@-SV*EZGR?eb^-iB>!D?9hzC@CZgt6a)CFDQlCiMrW1Ijq^A_ zTJh^8v{tdRvrnCw9=;)c^5v8&E$;b9eMH&&$h8mm?*In2e2uZnca=L&Xi&uAo=D^R z%bk6=jw{ZM(vWLs8N7puS<>hZ@>Tp(?7sO`*V5+>G?H~|rzLK>Bk#16r>JwUH%1Mv zujdVmgVT3w8caqN%-;I6TJ5tJaROKT7eao?AI*mV=gvO=3iVHZ!)5tzkIMIeVbV$6 zgEuwg%jV0T9gh{S+XgCh*#rE@dijt75F1(4n;7;v5a~Fm&6Wt@hS+wc1|^O}o5cnp z;oYw~S96+Ni?j*X!h=g41@eTE*6W#IcY5KXjj`8yr_6YxaIPUf=5s3V85^{C2K5%djHzS;_>^$BSXnStXAIms)T<5*Equ6V9VJkLIkL2*tC~2@UTo48Fw)ik`wFDy zi|ORT^Q-|n)=pvhUUJx%W>}zn`3^f>*V7Sv704G&T!X$+>$qiJr})Fd3?aY zGCChf{)iTv+BNI_|7(Zj_WO_vAvc1%Ue;1bv~(%NBM^~n=pkVf@(naaz<>Xw)(#fP UN06(72NDg%IM~jU;1Tfo8xEKOKL7v# diff --git a/summary_results.json b/summary_results.json index e0abb69ab..919829866 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.86363, + "Performance_Result": 7.94993, "Performance_Units": "Samples/s" }, { @@ -595,7 +595,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74019, + "Performance_Result": 1.7344, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0899, + "Performance_Result": 23.1817, "Performance_Units": "Samples/s" }, { From b1e29b28a5a51350127216727ff7b134f6693f5e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 23 Apr 2025 17:45:06 +0000 Subject: [PATCH 511/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10654 -> 10654 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 9ce2da151..7b40b2a19 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 17:42:06,327 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 17:42:06,327 submission_checker1.py:3308 INFO] --- -[2025-04-23 17:42:06,327 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-23 17:45:01,339 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-23 17:45:01,339 submission_checker1.py:3308 INFO] --- +[2025-04-23 17:45:01,339 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 17:42:06,327 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-23 17:45:01,339 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 17:42:06,327 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-23 17:45:01,339 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 17:42:06,328 submission_checker1.py:3324 INFO] --- -[2025-04-23 17:42:06,328 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 17:42:06,328 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 17:42:06,328 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 17:42:06,328 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 17:42:06,328 submission_checker1.py:3345 INFO] --- -[2025-04-23 17:42:06,328 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 17:42:07,311 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-23 17:45:01,339 submission_checker1.py:3324 INFO] --- +[2025-04-23 17:45:01,339 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-23 17:45:01,339 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-23 17:45:01,339 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-23 17:45:01,339 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-23 17:45:01,340 submission_checker1.py:3345 INFO] --- +[2025-04-23 17:45:01,340 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-23 17:45:02,647 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 91d0f8a3f..43cff8aaa 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1138 +1139 diff --git a/summary.xlsx b/summary.xlsx index a0d608d01020cb6f1cbce374242c043349efddcd..62b6025c1b41095b2c4013438ac1eb4a13dc65df 100644 GIT binary patch delta 325 zcmV-L0lNO4Q=U_>$s+_r2-nq<%_ARw%}&EG5QOhMMdW+pBy9m%Nh$&AiAqQmQE{{E zZE6w!$Tp;Td)%Z=TPY`w>zVy_cI9GukWKUp)(NdjlqD&OKrzjQs!Q~-S;dbia-J$q z8?B&39XPbC?h0lI(-z*WHekH~7X^@VgqafUy*C77$MztpOM*iMai^`MK9sh9#)h(Q zS_970^d3v_lv7Xf5g411h)zh(Mx^<%O%LD%9T?J-x-*+ zp2g99aAGvJZJV^S-Wu{}c)Nam?^lh5I-ZIFstV2svp~JJRe`UIaJYs#zYZtf3E)p% z&?nwbiwu3hqY!`$anH)$s+_~t82-V%_ARw%TB{E5JmTVMdW?rJW4@Ul1hNOq7o88RO~E! z+FHa9*@m=#kCUcpD|N+keD9pOGjh4u%QpH2>x5Pn%90dCpqS=DZ7cNAEaOKMIZqX* ztyWN>130v(?@DF}(-z*WHekH~7X^@VgqaHMyf*}6$95p8OM*iMaiguIK9qKUiw$Mp zbPG67(>a#lDW{&|9vGX6h(;u56Vm+Hb^vgOp#=%bJDepMK6^b_>25X*%7s%3e=u;< zdJ-r1-igWBbzRaGgEi#O@Ot(7KCBuG)t`z1s!PrYvp~JJb%`&FaJZH_zX~Vb2;fhL zpx<~K7diF;k3s-)jC)c(i|3anW{K)N&F|xM7UxBiJ&;*W()m|E@~Zy~l;e Date: Thu, 24 Apr 2025 00:03:04 +0000 Subject: [PATCH 512/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10654 -> 10654 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7b40b2a19..2ae8723a4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-23 17:45:01,339 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-23 17:45:01,339 submission_checker1.py:3308 INFO] --- -[2025-04-23 17:45:01,339 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 00:02:59,335 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-24 00:02:59,335 submission_checker1.py:3308 INFO] --- +[2025-04-24 00:02:59,335 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-23 17:45:01,339 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 00:02:59,335 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-23 17:45:01,339 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 00:02:59,335 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-23 17:45:01,339 submission_checker1.py:3324 INFO] --- -[2025-04-23 17:45:01,339 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-23 17:45:01,339 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-23 17:45:01,339 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-23 17:45:01,339 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-23 17:45:01,340 submission_checker1.py:3345 INFO] --- -[2025-04-23 17:45:01,340 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-23 17:45:02,647 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 00:02:59,335 submission_checker1.py:3324 INFO] --- +[2025-04-24 00:02:59,335 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-24 00:02:59,335 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 00:02:59,335 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-24 00:02:59,335 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 00:02:59,335 submission_checker1.py:3345 INFO] --- +[2025-04-24 00:02:59,335 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 00:03:00,544 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1817 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.251 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.435272 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 43cff8aaa..f36e36764 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1139 +1140 diff --git a/summary.csv b/summary.csv index 92d138b97..1497aa681 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1817,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.251,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.435272,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b79fcf90b..b4973e847 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1817, + "Result":23.251, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 62b6025c1b41095b2c4013438ac1eb4a13dc65df..63c1af2de464e166dfcd31d2109d9fb47385c451 100644 GIT binary patch delta 1468 zcmV;t1w;CtQ=U_>atH~#1dj-D3;+NnvwH|w2!FI&^0HE^5-QOl@ud4hGNE*TNCv5_ z;Qp`-P|3_4pwi_2uu*_Yllr4U{ehN<9wtx;>JKS{{VVGl_OH5T_2=)W@<6?xcY4?R zDgGUo<>?Y9PwNsc(ID}p_(L+G6n{ttxvU`muncfXGQPypE0 zKO}=(R*-&J2Dl^{;L_y$unvGrlk%fM`GJ;*41ziyl=WyCY+@@YKe}e+=P@q-5+ARY zNmkm(?m9nOw*n_9TQQvyGEpA!r1L{Ep?`FKNCwHQ;QX)*kV!H?rb+o>9RQgo<41$> z11%95gpdiw4=KZqY!%~2*KGWJA2JVB`O*8zk7IkGTjCXHw**a8M?9(gkW46*ACf^d zD=0rK189;AplLFGSO#NEt4gReT>^v+whL$UIcxM+@P{_QRde!`Mf&MEKFKZ^L@xXRPjd(@yB-c4p;kVmWV%^e2kA2 ze@G^j;t$Cnnia$!mH{+L2GBHlKdb{l)8zeV@P42rB7=Y<3spT<1!OYo2gp1m>*sHf zWNVpUB`CvvC@oi)*?4w|`D3F(;!E>~WRxY<9})s8Sw>RHGLlMqKvFV)uubX@8%1Ji z>Y!tT`U5QyJ52dW$XzJw(K5N0B_Y_$+6Slp2qSM)AgP}8HfAkztFnc)_pBeQ6$ZNG z#id=M{n#jw_)_~J8D&ZDhlCKx3f>RP0FqU_AKT>puu%X>llNnT_X8~vJ52e><6S80 z(K0}?!bkqt2j~4zB*Q0vA$g$6&y~Ha{1i8{eEE?D?VWGIn8u5n_&lCW(#9ak&(^wd z+q(X0kSlfkas4eC5Al4sWHuZZANkpztiSUk8_#YU>7VOw6Y-AEf7e#g*pVVf#$V?76I9H#*%*Eqt-nX4F<*?6#W?%A9=-S?ljWp;Hcf_qKdugHIUE1EjDP%L z@Pl{!!@&FL=Z5~NaoWgdejm+ew;HARfgRS_$-I9Pcl(Q*@hojj;!!?PtChe0UaY^F zWc72F%vV3T&BmuB%aZB+Z?|!O7%z(7>}F$>B-#Bh#VOf-!^_$K08mQ<1QY-O00;m8 z06zfdasuOB0RRBAAQh<&1*b;>#wC-^BOiZi!!Qtp?|llRyDiyCOA)pMZON&DLS0&N z6VfITwSJJ)I)3{qPU0lsQ;(~e{dRUyx!9{V`32{t(G|)wf)dc&2&uOf`e>HvBT9T= zTCmn=s8A0cE$X|HTgr`tH)kxkAi*aQq&?-fLc0(w#n|&5DCV>1&{5nNr&x%k+hTvq z_&3`EE(m$R3IY=>Fnj=}b|j(`Qt%OJe_T5N1jo>V0v$Zgvm9T%0i5!a4TEy!R5J7y zCaq_2bnm?!ja}De-E6SN{5k$wy}l2trcxhH#R1i&;FLRH!MM7_*F`*B%Y0bH6K^Dl zr#|WvZ>L3$eISw;fF9$XmCxDpOS2?IbwP^zl+4p&-Vj2`j1sawjJ)Z;0M%*8zlmqS zG575!sQ&=75GBe23A+T32yqMm040;-CszRrlRGFp0+kk%uof1Rk0>euX_LArIsx00 W^(ZCatH};y%Aw>3;+NovwH|w2!B+dN_MbXm0*byi6`YBk_n~!Lo$eE z1^I_%086Is0G1~Chm8VQn%o}^?hmv?^e_QSaDPY{Y+zZ}uz}SzyFY(FmIo^Sywkhl zPw@}AEKiq6d0Llni3*7)%^#8prTIfL$Ylldhh>0ElJO;$Ch>=L09cy5pY4MWSc3OM z27f_U7Rq|03|v{(HMp`n6F3ko|8iNjmMK<|W%aK194#`O!5yKaX+wmk4>a zOtR8Wc31k*x)nG<*^23ukcsw)C#4^f36c{fe{N)}C_lPp<>&j5d8p2h-gkZ++Y8+i zu|T^eXrepfN#}=TLh1aF45C@V`C%DAlVku*lk&ql05na;j|SrhS|TzC+IUdbqh+vY zRxo~a&Bo8SqWL5pua-$xS~T~RAG@WUutY1+UJ04xV|b+ULo%UMen!gYW|_5gCM#3BnI4!$q@-@S|%Me!dTxhid$2Vf@&BxbuM+`)HOJ zKbm|}k2HQrCX~hx$sn2)j31T(G)V@~GzmYf13=T{`)Kfepd})Mpo<4(Jz55fCb5EG z(X3hc`F1p)vY$Or?dMAGYCr4uOv>&7f0bB2ntX7Nw0=k?l-3W)Aej}cAC>_!Ne0L? zNk6OuAk*ahXmEa@B_e|mGQs&FWw>Zoaej2o&d>KD^H9wnEzBR=)jM46qgi79X!1Ee z()=NrP?|p^gJ@PTe^>_4BpE={B>u1t08Nwlqe1+EmWT`jk}Op9SQU`Tv>zbze~`4F zzrm8NWqy^Q4ELe5TwSK)*(K_ajSh(~)gO{kmUMqe2&iNkNhQljD(L}9$-p+bKWr3< zrKy9C4ek%LMC>r-M8tX3H4 zk{6eDiS}coK;ld7hh&r`y&n=neOrDnD2DuJTjd%<|=D7PNQ124fm8ZsPNJGD#bQBtKj0 z!fosNyFsqh@yGSoXgtL8;gZ>KTzuwdf3p74k8C`?F-{ia?CW~;;+ssClm6K>8UDCBsO4<@=Q95B zhrtit@ec#pWSMd;sjZut51d*UQ*0a-~Z0qTiLNEA_Vv+QkZ5&y_GqHuTDsQwrZ1n6#e7(S2}YG`4MuZwWFhC06vC*BF*PhHR_-cE}QeZZp-fDCcZ%IEC) zWwRwhRi5UHIGxA&Y?D2Z`GTbR*Ky=c{{<*dL;g)X1BTqUpP>2!vk)c90ts)u5n*r) z001VF<0n@E3X?l1JOY#!ld%>SlaDAW0cexDC^`Y!ll3Sj1w;tf)h3f5DHQ>`lQt=6 DWxBoJ diff --git a/summary_results.json b/summary_results.json index 919829866..b4cbdfc27 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1817, + "Performance_Result": 23.251, "Performance_Units": "Samples/s" }, { From d84f713464d2200ff3e41a3b4c30495d32d3a1de Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 00:08:45 +0000 Subject: [PATCH 513/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10654 -> 10653 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2ae8723a4..fc80adea5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 00:02:59,335 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-24 00:02:59,335 submission_checker1.py:3308 INFO] --- -[2025-04-24 00:02:59,335 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 00:08:40,097 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-24 00:08:40,097 submission_checker1.py:3308 INFO] --- +[2025-04-24 00:08:40,097 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 00:02:59,335 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 00:08:40,098 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-24 00:02:59,335 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 00:08:40,098 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 00:02:59,335 submission_checker1.py:3324 INFO] --- -[2025-04-24 00:02:59,335 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-24 00:02:59,335 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 00:02:59,335 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-24 00:02:59,335 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 00:02:59,335 submission_checker1.py:3345 INFO] --- -[2025-04-24 00:02:59,335 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 00:03:00,544 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 00:08:40,098 submission_checker1.py:3324 INFO] --- +[2025-04-24 00:08:40,098 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-24 00:08:40,098 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 00:08:40,098 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-24 00:08:40,098 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 00:08:40,098 submission_checker1.py:3345 INFO] --- +[2025-04-24 00:08:40,098 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 00:08:41,077 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,8 +45,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421364 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419341 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7344 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29935 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34456 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index f36e36764..e60c25e12 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1140 +1141 diff --git a/summary.csv b/summary.csv index 1497aa681..59e77632b 100644 --- a/summary.csv +++ b/summary.csv @@ -23,8 +23,8 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421364,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419341,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7344,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29935,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34456,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b4973e847..47b76c9ec 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421364, + "Result":0.419341, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -849,7 +849,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.29935, + "Result":6.34456, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 63c1af2de464e166dfcd31d2109d9fb47385c451..78e2101beddaee01d40fe955fe732c3d319550f6 100644 GIT binary patch delta 3407 zcmXY!i$BwE+{Q^LztvEYiWM`%C|V*hE5$H5h0#xB6GBvSXzfcXW)8D4*>dy zMM^Y6%&9r$)WIRA#2B9SJkR|HyzlFCUDxM+z3vSAbo;VfyTpf8g4zS=A-hxo*W0|l zLyzZIU${SUt2}wiTPxkD%8<-GQg-zD?H{w6U{Q~g;-*P~G@wEC?ZNuh-imP_72bU= zkFx42**NhsLM?EqE%j$^ko@uPAYDWMB1W%Y6C162q%**p1sQ2LE5(dQ&U8NB#OP8k zv93;3^4q=u*2ZcSRQI5M$DuGllKs0fj73%^IDPC@G|j2jYKPFLedUxu>rRIBb@;4n9cq!rB8nF9>`dF{>L7f;k_oe3C9h<;yF z^SQm*J=LGYs%ud7*5g?0J)>#g$H&)k+q3v2VgSNX zC3l$=)LGzJy4y-LQGW2u391|#j#VC)`Tf|mDQ}({Q?spoBKt#cyt(O> zVQq3u@t}%KQ1B=%rlvL(s{H)dP3ZWdNpwgYMt&&GB5`(}Qo=tKy_tDDN-UzYI z^OkXN%PWAuX;ooKe({vTxwGaY2U`tStB2=p&PrY#JVvT%AES|7oW=01zf0O01f zb=Ikw`4g4;g53U)fHvRj6n!^EHt&N^(QtgnX<&t--wxLCxp$js!23fqdeWIFu{>p~ z%es$4T*r}~p5f&_m*n^%>wUI{jx^(dvltdCz0wT-eycP=b*iOLKa)2Ns4jE1iZ8vacQF-0g;&L<27UFWKHABQ1bQ_n^W((4f70re#nRM!mQzYLTTokz=n*^uD zm?oJ*V)eX@cxJcC0#&w{T-((FhRy$2%Y#+;}*gqF`nW?4A)1;6b22yK)<*$1J~Y_Fm_nVK|AtD-M)=cLG4MYSEl72i%c& zm+P`@hN?R;FzI~;Y6@Se1ZWIG1p`$ekQ3be&&OThMLq25Ie-1$gq)c-c>unonxUP% zVxC59U(}j$sn-p9#g2F%vE+e5L>y^eJPVj>cdxBmzkEQn6nG(S;5@{&M{mNaB)B&p zSs&!7x-U5mXQ~rZoQ8u{pFG(;ITHnqa;JeJFuEczufW8i$M7d5k|NyC=gl}Fa@tv{ z*K$6}JJ#9s7Ej~(7OG|8S`HQarLo9`?Uc^ax2qNz^q+g+ zz?(%;>=)T`TWKt=Pld2hp&``i00?yg$C<$;<*k~fEV6&5eQniDO8RyF;k?fM_1izz zY1mIYtML0yeuQyESWq!PT1ZF|cCZOHBRYWm{rStt+S`1;zw@^@3+MM%@M9f(=g-|T zD=*BrlU?c=pHhXr7cvNPt)BuIuZMEh3^#;Wk6#k9*rlpEm98dGHbyr>h-N-H=6wc+ z#DOF7|BLSU(=3T3ST~d@k1}B=oO0yprEmJ546M9b<rOHJ)K zo$PlF7O2X4g4V-G3_um!>AR5&y3`xj=qnH71#dkh|i3(XP!TI*n_=R=b4%Y6gBZ|3!*~p29Kqggd0Mmyd6)3I7@uS`+~B3 zXd&K0d7M$(qYEbgE3R?u7@@|W_B?2?#D0SB=B-rJpOZV_X^=OcF`H)8C&nsx6lymo zB-I*0qX0k0vU|=F1Ng^#%6xQ*-r%?Sm?c}g##a%Hn7AG5A`UCg2eLSLB@?z6>$?o{ zk=4SSgHplRvNeChp|*Y^_28ri{6E<}GM8M8q5I6+qw&;MscdkYNQzaJs0OwIJSSR! z3E7tiInU5ZU?2Jt(?1?klrQ#XAd#{L9kQd&z7ChY&AN5hXb<* zvS%7xX?llgqFR8;1$DdtN^a;kOf!2k8!}tPR!Ad1|D9JuHTt%3Rjo7nRIg;C$1#i1 z_;c!8RM0IPTHvAZm4+vh^EewGiXsfyoN&{{7+ITCv`xxiW5q5rg7*~7cI?=@v#nt$ zrIbLvdk|F=HmzEEv0^~%eHf2pkd`LQ^)HQNn^4t^a-D&W|&~E;* zG@COhXcXQtkmM}!5vp*Mn=2^W8(~FX%xPSUWW<2ZecU+_ojiuFie1dPp+~9*r$rYD ze%zmGDYtv`S!+?M()9tXNF~F8B+Q}}%sI*w-pUX~VkABz6;LH&CS0h%Tbh)msvMOS zC(OY`g5@PXX4a1o5H*>XDi(cl_&n_G&b4w^-tIOxx~r^?XaGH476S|*$@_P*yxQwK zt8@6t%d?R#D5voLbK2fTV z9SI2+wk!#~D=kYv)g3|>!%?+fQ{OkJVFlxfa^s0}(~8gkbRlE_B>x~pn35IzgV0!{ zW{l8RZ-6G;&n2O;!u{On`nvhoSWz>%qVN)_kN7??HO#!PbMz>uucF?<=JAG&%X4k^ z;;ig)%f&}cso%iDA{L>bI~^+Ga%lu;%AIp|x@FjD@Zk&Be5SKJqt?Z=WLWtE=;YrQ zF70d^P5LQeF3KKwm~>i$>ToIiFq;9qyln7}cdsw8S4!;PrO9sXE|TAVb7J8pHLfCe z_58UQy!d=HL4>0*lIXug8h}USayH5vH-w2{?MX|a@_!pZEm=rH{b>wC?dL3D%#{k3 zZWHD;P#-9``u+d#3`Za6#LJ&v3_F{a8;_=4Ee#&43f}BJ>NLGJvF?fll*aji96x<+ z<0Uo<2Yffv%NRy>qxwf-C=qaJe)-IdacHq3(HG?HBiW{%ygl@UX*<*gWH>v0}a&jp`w99|U0^ z9jAoWF!*J$Y9r>-iT8VMzK*9H*)W0Xh!`~+sYBS$zOTrh5?B9aZP@eO ipAqHJSKzS!vkIavCt63>Q#Td`(}UGjBpP8N!t*}{W}cw{ delta 3375 zcmXY!_aoK&AIEJeIXPAo$2g9osGMA~$vjp@DJx{(%QYj?`H;$ytaFTvjI3NEIrdEE zu}8)sl5pY}9a+b5j_>K-@B0sUzFx1#^ZESY{l+@Sx+?G#dox?&sQ_%~DQUnrXtqb> zMSHGO$2HpM@aH_o*2g}f;F7S8rwzj<#E6}9=aj6q`MG2$2Zk)KzBT$)2;90bmUFeD z^~*@5&5xZcOQ*AscidIIzjVh1XAhcZ{p0?}1+q^PcP}n$Vm0RAYT{E5si7mB3xFA1 znpmB_;Y$zP&&7`5mlAiC5^x574yu+-g(Icl}gN%$S!t+6fxHqL|}OG8ZHl=B~u#E+i1We{Hzk z7=~P4ow(OkFtK|5VFbOoB6Aamot%O6l8M1YqJcu{E#!_Nc_8bX_o<0XZ+-jP97M7C zTztUnm$z<7rsI>qg^2*vm4`TmO+dwWcBqF)zSbWyPH^e$VZ#K~3+k0-}H)c-syCnDA3aDdLN_7qJ(o0UrVxcPiYY z_WD2P>J^AVN-m(5HDV8`$sj0r!%!1r)DXX6HoZDgo6jm;9LE)qy)9GuCupdqf z0jI-tC6_*j8tJJ6Lg&YpHVmMW!+)pibrCj`(Knj?kb2qkCSYFA17$Gzkex_AGaSw1 z5Z2++B{Z_AV&J3jRR}7%O1&J5rnGgTtGn>P!(0h~*9j}prN7f-vCs%?pq@7+o#j>2 zR=9araIea!QpFG1zO&O7abstUKso{L)58T?*Bp-)tk@u5+2=bZgazYMoWCfbYe*OA}1-V za{V+-hm2W;1MIl1^&c>qK!E;-iD2{`No{_|ZfeZAw8lEBTea9KsvAV-i8CMFsaXl+ zXH9qz34eL!hl)W&fI_w_n+23NRx;LzAF22D{CNxL3Rqt^jB|ju`(vc!6zj_|0+kf#^Pw;eY4Q_W~U+eo#>T&hX{(Kt=uQfMrs8)&0d{nhr+H?5Iqv zjcviHRk^}7TtK|#bf`GDeL^5>xA1Mj82`yDaQ>0aAPiSmR5q5|zYhV{OG}1gURCQE zzOR$$k=)ezrTrQ6M zG6_uXnGXF&VA*6z+0}M{`?5a;73BQaHJZ(jXZemSuyzvN^qXVIbgJ_MSkeI}lA{AE z%!GAgG96qLX*_Fr%@)K|oF9{b*_LzAHt8}*R<>!jm@04&l&*kE+>;B1`JJ&`*Q4YKUEI}3#77vODQkJ&(QQKM z13sKd>8wU@9Z0iE)?QNSWd!a6<9s`g0oV(FLd}V#&p$GoUZmHqMeY*|j!K1 z{bPQ>^CaVw^5DGoTjWAWC^Yl6jv|s{6e_IEcb(0w_Zf`DJgT@mO3*xvEcS_=fuoD% zNXZBXCFFF2kLn4h*OWr9@jx-I-G?J#e;{oLETo_Bu=oOFHpo`V*t75Veo$5)L=?y0 ztc@@wThK=av*G;?P9=AfF2Ep}#ayUAyoNi?jQd;l8ohHSqCc%JLB+;SXbp!@?n+5SR?`=M+1G21&%=YXI3W?!p<0XC{|7?11ju}l()&+ zg($sr8)r<3rZJ{yn3jImu}w^6#zNWZp9Bx>kBDJ_80_5vwla&-&=rVpeU1H2wN)8| zgao;es*A`E27Z%Rr2i%{D4I9EOs}$JkT@#Pp+%*k#b6d(>mb=^+GT(wvj4uxV~@C9 zvZLLa8yY@Mm8FZOudIP(Zjr!O#>B~mhYUG>hFoL0I77}KiD>H@U#3;@FqvTYag6$y zjH3F)-}T?LO-zZpG|D`%-23LM^H#jYqdxuWurRD;=Kr$m&QU z!@^h#8tq_{1fcG5G(d$H$Cr<*cp^=(I|Vuv#)8~WRmD>Fyo7xoT!w2kd8`l>uAG1& z@6(Vrvwd5tT6Xm;GH`o^H(-RN7`gD*?|T_W;r}%%35n1yMp9fvnKEGC4Wr2ne8SI? zxOzaco$)>b-wmgIjw2ul&>hMcwLj6->T$mPl7Lq|o7rtc_0r=cQ+O%_qoc@}D#z-D z;m-3$pzd)-79$VG#E-R2^f+{cLBGo)ICT0`Vx{dN#YQpL?fWDGZydLpLw2pN;Y*vx z+JhCCuVCZBNZ_7gEI(VE$57iyeE}tfwh?8gbvJza2rpzN@UHr?w)A)+EO$<-2!i1% z+SNrEd9bfBs$hy?MwktQNKAc-vRWc35JnNipP5>qjD)Xb-bsNt;&y&L=~KT=HatV_ z4cX(lOZ7@aYh^~fN@SP>OIZTp#WMxEAZg5sNPuHhF-)tRReNHY=JgYmCoLHrDRt(K zh7G$Lg2dUU4kZ!&lHC5vaFT|1K7q^MO(#4GiKVy}+*~sG72}p7JqZD`T(UK=!Xq%o z;o4S+DzkOa1^ox!>v^uxNB@N%*~ci<7@{#7xS=iu7xuJ{d- zx~6ZB(<+7U_FygHJW9Bt-LO!>=@qv72DM47^-vPKQV1xSz0j)U`BUY>N#iTv*yU$* z=yB&q>KgF{y)9Z<|ddYkVx@etPmO%>%v22_vQ=+gFi^c@{uRjHk zemPZIFL30QwkBm9;cQnm<@L@2BqqP!jIr;Jk5=}SYY^{{;Vslysk@223zY$&Z~3f# z@_GXfYD{dsqfc2x)rWBxom!Z)cd%!p16OV%b_HP-o&mwztrz1Rr`_j$s(Ox9=yt7b zpt52sh31~D7r**ozJgCKjDcu=6s~s_ng!b!Y>n? z-b0<2)Re)A2MgzJ;$MNQHD7^S*4mjSz5Ci@JwGZOY|6Z2FaSySv;6;)vmj&7d3x>@FPkA1X0fLFN|hAF`Uv=di)DQXHRPOpNzG_`8hl diff --git a/summary_results.json b/summary_results.json index b4cbdfc27..69e8e4001 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421364, + "Performance_Result": 0.419341, "Performance_Units": "Samples/s" }, { @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.29935, + "Performance_Result": 6.34456, "Performance_Units": "Samples/s" }, { From 5e46a9a91f37fbba20cad9889aaf415872f877ff Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 00:16:25 +0000 Subject: [PATCH 514/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10653 -> 10653 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index fc80adea5..ea121b03c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 00:08:40,097 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-24 00:08:40,097 submission_checker1.py:3308 INFO] --- -[2025-04-24 00:08:40,097 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 00:16:20,554 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-24 00:16:20,554 submission_checker1.py:3308 INFO] --- +[2025-04-24 00:16:20,554 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 00:08:40,098 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 00:16:20,554 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-24 00:08:40,098 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 00:16:20,554 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 00:08:40,098 submission_checker1.py:3324 INFO] --- -[2025-04-24 00:08:40,098 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-24 00:08:40,098 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 00:08:40,098 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-24 00:08:40,098 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 00:08:40,098 submission_checker1.py:3345 INFO] --- -[2025-04-24 00:08:40,098 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 00:08:41,077 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 00:16:20,554 submission_checker1.py:3324 INFO] --- +[2025-04-24 00:16:20,554 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-24 00:16:20,554 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 00:16:20,554 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-24 00:16:20,554 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 00:16:20,554 submission_checker1.py:3345 INFO] --- +[2025-04-24 00:16:20,554 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 00:16:21,623 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,8 +29,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.94993 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420623 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.251 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418152 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.251 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.435272 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e60c25e12..b4fd007c7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1141 +1142 diff --git a/summary.csv b/summary.csv index 59e77632b..56f9878df 100644 --- a/summary.csv +++ b/summary.csv @@ -7,8 +7,8 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.94993,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420623,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.251,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418152,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.251,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.435272,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 47b76c9ec..e3a7b3c48 100644 --- a/summary.json +++ b/summary.json @@ -244,12 +244,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420623, + "Result":0.418152, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -274,7 +274,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", diff --git a/summary.xlsx b/summary.xlsx index 78e2101beddaee01d40fe955fe732c3d319550f6..ae5e832d728e07073e397e0f1a927484e15e95f9 100644 GIT binary patch delta 3510 zcmY+{i$Bx*{{V0jMrte^5_6eLHIbI%ge{k{#Y9ep5>A^-u3?GT$EDMVHa3@-P$S1l zZY%d|(VF`u*IbHFG0c62zViM39>2%$4|u;G&)4n!e!QRBJ+&(i0Et~f*T?%y2neXr z?t@eSpTObwM;_K^IXCKVcXfQscB;Ma9Sq3}X^g7qu$&27l9bf9J0~rs$y>MB7yqTw z@0H9Yh3-t9SG6OZg;)JUe)}$-v9*Lh>p9XK7nIRvP5&2vMuF>{Ai-kNd#6e3I+Q3k zwf0S>D?n|V=$#%g@YxAi%c6C%CnziWWab56tPyVPGv+q5RWWKUTF!l)RYnalYK6+H z$9bY!0jH7Iy*$-<@U`>3nCsV51~S)J%;_r9gfX%dhZZV5}2nrz#l%t?eAx}y4BieeMiWocNGTR?eAUE z+}JDYt7f-*8>{{dp*!O9eF%e>$y1)?_mFkg6Ikqh;~zgj&MVBFA+{=S#J=DHdS-hf z&6edjlZUr?y>k`u8>@c|abxs%#8QZ~`@(!1(sNu!5%7a7;Mn(v7EScSMm#-cSWIKB zcJRe_r|kJG|G&jpG9--Z81bud>;^v(41>&BAgSnz$8*?$>E1Wbgw$WeiQUOqP{P&# z2=15eCJVd2{9YW&$`@L7Y?D6x?8$dV>^emQ>0*k^e}P(-Ms0;^EjqLVvIqx6GY0r z+(LC?u1;O*&RrdhdB*5Cx@l6!fKvb2=vzawjzRp!guD?eMthncW!@CF8}`(As+?_# z0k3HO^jDn;FTDr%5^GK+1kD(G~FZ)%ofG;TQ4dg-V2a;^1FuhGr;LKf4u=0mVImmr%rM5x+A zDCRX!Yu45UUcYL1y@`99-gok@9mhAQF5kJyvo03%i9XuaT{bh@7)C@q(V=783} z{%Bp|eC*aB7^B@7TIxs-9tiPb)=Uc>&p3rW$-8tko+SiS>{<`Ir0v9PC z2(FrXQ^07v?3uSOj$oC$FAln;DEl;f%CmJ!OXO&?fUUGn_KY02Iqu6*wFuQ%v^2*2 zIa(Fc+z_F9ELH1Bi=jPPOGHGvM9D@1_9*89Jlv1%$`1d8UtBOLA?N>j4&dG@E{6}3 z*Pc>29v>Ve63?4-2kv&U&z~&il4uT#gQ+?+#gsOoi%f;$3vSk}p86PU8rkxVJH({Jh1dX zIQsTVb+(OBc_&6gQZ&F?4`@(nMN2wdprZ9!Ad0oa59Q)si;6MwNgZpA+T<=siS1YQ z%#%s0=(L*YjTC}*k=2cPtIxXGVfNqy%A`9+ozB!aL|vWTD`jU$y?m>qn|);KI$h{` z^!IX8O$iP}mv?2|UA(BYx^VyD(~&huv>r%;F-$PbDCz0j*#!rNR z^EYbq+{I~&2UURZMM3P)E?5%l(65H~cSt5^>-0JAYRBV#oeRh%1iE}$Pb%bu;`t6l z#!A{-KBZ7+sL9I3QvX!6H&vn-qt~EOSsmv+tfB7!Zg}zRR}n<@@s>>&5`R~yw&`c! zi@pB>rDFNe<`vgIPO3It{p2Il%7%mL z^e19B*cxsLSgN2tMr07T@8AuQ0IYYZQUulv{FD<7_HIduc?_ruXY4l0Lmp0-dyDVg zCYA7#w0(9Hf9+jm=q(Eb&*?2+O13u?CrDnpZ3sJz_9NjW+r_63wF;zLJr+b@mkx0R zyAV7>@%>63v4dJM8U{u=5~ z(iRvwj|2CHLD+7IS^rMMINy z`6qolYbq~{UNz!>>`B}FiN0in*f;o z;POuuc^4`kMuV%R;p9s?YI`*JB!Y4xg3#5i{PUcpxrm7ZxFxheA_1mS@q^)&0<6Vs z879>@YS;^3texHQsLmaw1}8fvpNkcb)$D^(#d0^z^gwvwRT5s{B2`ceI}M%{$VSAX zjx<4c3#+{Uq9lNppS$TWPz>DfI}?WjP)VY$uA4Oe+(8ZN?$w)pO>fF8DvA~G)OsHz82IUEMP{7_-&cpas1Xq|_|DgC?1SpOT! zS~CU8n~rz+ilu9j?_J(dNwX10!J<8`N)qiwx?u9>dcX3E4+&st^9+;q$dnj~sN+g| zyOJ&w1a)H0FTe6YvYTJxsmZeK#^WqsPfd9B=45gL1{rO!iG&@#I8MSPw29Y&TLsc! zLb&uGXun{BXK6TM5ZMJuGy$c#@PP!q8;F;K$SSu_GyH^*f2t_5RTK`%FzaCn#VvN8 z!Dg+XM|U&K=O;)!cQ9FZl|M(r?TR?N86vQy6p^953?JRwyJtKg>LA(nbe}9gi{)ed zAt3^>aHVjWrjy>>JOt6ROoC#OZ!C_Zc4bq0vKBD#KeYHg8ia@dq5ex#-rAFaP*rYU z_GH5p*oa>Gy;hcZDs+) z9g*{8DZDFDdLyKs076=vdn#}brGErgw?af;aJamsqLRJ42gz`?{2Wmj3f!rmhVqt< z&CQP{ADjliHlNd-no$mZR$$Tk^=8Tg7acEX22U-l#x^o1w8y8-ar!F$fY$;Bb@hZq z5SwmU-LzR^tIW+^HPdUM4ui#1sCyF(8xjCP!hyNG9Y&k zwLQ{>I%BQCYg=IO=B_fNQN{1h8m6uLP51(LPG{ym#u&9|-hQ*RJ|4HaPAwpIPeYT^#OxwPD*S2!xr9Mx1CbwIr<#ZIJYXp=} z7q9iZ`d#Q^n2Td^zsEl|U?uI@YU=ft$I;i^;T6h_no>E&Q)Q;rlJMriWks!N=C>}} zkt2EbF}YsF_$~UGhy$jw_9iY-fbc_b!$-1tkf}IQl|%6ojrc6*Xp>QMlZ-)Qzeb zl2;md_TI#+{oFx|@CnZA4<%r8h)~z95$4jV`=Mh6Hckl>NcH8{Py9g7>sn$j`gKU7D?$p8QV delta 3553 zcmYM1`#;lv*vBU!zNwP^QpER2^o6`S|@Q=v(cld!pAo;|H zM@|Dy_t^a>o-J;?aejWY;@o);)ik|Ioha6+QkmEHe$J~vxLnTh7$x{ofO`3l5_KDW z_;p%X~T)HVe)OLg5OGe%C8)M@iRUC8ah6O)IRSwLCnqCvVwzcpF0)&32z&{rU#dWN*Vo(Kl`7=J2)>G;*j2Rc&2%`76@^v5!y9-oBB z+d~`#__Hi#NY$tt$1w^%KG>TN+0ahfoOGP{^=}45Z0+`AQqL%1j8xx7tYp4%W zova_hv9o(2|eZJ-ZT0)Clh zybH>RZdtk~EVQws9ToY{`eUQRrS+6Z1~k}2_VFuwm~2o8Ms`Bz&oiUO+(mL^^?};i ztfszLw9$HFW?_)c9V_(Rg&8pY-w z8OpkT;9wmV&0-YIZUDRX87qv^+H^LmMiwo_%TK$Ch|1l^TR})u>8sQ~ zD$vms$T?`*hyBIrH}>81(WdyKNlRKrgm3e=g%z7SWb+}G$^Mupv7ZUhT1ee%K^LOM zx1v<*zD^)(EmAV_CM&dda|Qx^+r4g+v~Ef<*iD{=Be7pKfi;p=2SnZT;XRr*d!3;7 zyfdD6b=pdU{s@b>jU~Rg$j*7q$Mi&ZtrDPx9T5aY-c9 z*j@0n6v7^$=GPCFhyy~h#et1d>^1qsoU7n1s=3*3g%o0}`RMS?4ZbEDTK2Bwmztt% z`is`B0R$H8`)2We`yH%tcy5mckJy}pDUTRxQwMR;u$@J1w*qTLb7d(?8+58qkoZ1p zUjo->gkM@TEQ`yv zHCT2ltx{#jt*22YJzBBgge!*Kf8%uf#R)lcKqQzfwlT825&h;8cTDk3{WbO8z)N9m zPrV}tkF4k_lR|RP`XeDENn?G0e~!nO4Tb&c&Z(arH~YTVcbj#QYR~nl$dB*SSpY?^ zF=TezVo!a#UYlt-T-9j;7ZlM}ko-=@!y*xKCNN2SRNPIU%Q08j;ZM4IFW-JRDQ4^u zHK@2EpRSg)hE64PEUV5r)@k^^V}yPVU2#DnLQgd>>jG%Cp3QBGw~q;CyY7fPSQj3( zvAgg}Ue?{G77}cE=N0>r3?)J`FcKjD{5k*BTsSP;nF0nUlp9;A8>L&;p9f+EhyDQ9c2P7`HbiFfai#)CiyB8bto-`Zl`xG0*$&jyjq-JMxh9rb~SCVeK226{O$KDshWV zuEabH90EJlO#{^T!`YiUI~=UbZ(d={N@cBFcOy80AVVh*Uo#Jotk*JU(PKqV=Wqp2I2F<*91@;;$#F) zuwCFQhe-WWkZ>?>m|6k75Z&8_k-9G$iQ+Il+kmug4pcS#)pOY41A<}S`~d~&;QRUPJC(pXc~|Gfdq**cr=}lN z4m@geM<}n79g{7ijzuYqKq>sWJC`-kc%F?2e5k4WZ%{IPBO>;YEv*-NbQV%!bf0D* zpO4A`e|-ZK-_fd{rS)ahXS8vz5l4UhyQqL_@M>qtTcr2P-^enYFjKaJ zN15$V+$U z88;q4>=IOH*UFc@u(({zz0S=tCpLfk&HZK5R#?U-x!AuNmN^>Sc_uj;%qd(Nm`{*7 zP%Ed9CkPy?tujx{fu%=iNnhG^0w?9->Z4X&+i~N6##`1ur*(cFG-eoWc29+e6smJaQ;NR*hKuotkmoEZiEnk+>(HDdSR~E#8HV%4FMXAtxEsx7W2lu^;xw_X%+SyE4_I)=l&lMH344$T8(P8<#Id;&>J# z@F1r0XuQt~aZoWd2Vm}$HSBOYg0Ug41d0Fc9JzQY0kz%`j1pxonV_ZfSMK2#l~GM3 zY@O&oY@M;k>apUQ%fY&-IkAS6TO|SGl>vKwGWN5ZliN;6>4_~rrni=w{t5$y1!jyd zq>mtbPy=J|38{>O*CknuhM!*AQ769*1QXaX^URwtoA@?Q_o z3t7GbZ#Oycrh$PQ7%%OU2k&BlU6-Zsz}?RucYe)0+b{5Ex7rh-9rjfxayP>;bMv^- z`l#Byrj8)$1^BY!t^B^W4B$-vn0fiOw%L!OjA8dl8w#HJXTNHw8WkhnboT*6-IOxF zpEAKm@va}Ao(C8L$Lo(BZjI?}Ozedrj&j^2 zjBFEMl)Crefz{i=q6=F*Gr8U8+s-`}?ixN)P<1}!))|(zDZ$yXwBIA&3L>patBrPQ ziw(EGji{DsP!P(|o36ZCD~M)!6Wkl>Scm|XoA(cE6j4N8F@ANymTo*@NQQ@2W^6-Lp5@sE1%zK Date: Thu, 24 Apr 2025 11:02:10 +0000 Subject: [PATCH 515/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10653 -> 10652 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index ea121b03c..37c455e06 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 00:16:20,554 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-24 00:16:20,554 submission_checker1.py:3308 INFO] --- -[2025-04-24 00:16:20,554 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 11:02:05,517 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-24 11:02:05,517 submission_checker1.py:3308 INFO] --- +[2025-04-24 11:02:05,517 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 00:16:20,554 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 11:02:05,517 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-24 00:16:20,554 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 11:02:05,517 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 00:16:20,554 submission_checker1.py:3324 INFO] --- -[2025-04-24 00:16:20,554 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-24 00:16:20,554 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 00:16:20,554 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-24 00:16:20,554 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 00:16:20,554 submission_checker1.py:3345 INFO] --- -[2025-04-24 00:16:20,554 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 00:16:21,623 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 11:02:05,517 submission_checker1.py:3324 INFO] --- +[2025-04-24 11:02:05,517 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-24 11:02:05,517 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 11:02:05,517 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-24 11:02:05,517 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 11:02:05,517 submission_checker1.py:3345 INFO] --- +[2025-04-24 11:02:05,517 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 11:02:06,575 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,8 +29,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.94993 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418152 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.251 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416065 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2966 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.435272 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b4fd007c7..e29bf83e3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1142 +1143 diff --git a/summary.csv b/summary.csv index 56f9878df..e4f4ed387 100644 --- a/summary.csv +++ b/summary.csv @@ -7,8 +7,8 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.94993,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418152,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.251,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416065,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2966,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.435272,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e3a7b3c48..c7c348883 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418152, + "Result":0.416065, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.251, + "Result":23.2966, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ae5e832d728e07073e397e0f1a927484e15e95f9..4e7b8ee24e151e38f7d15ef15d92aec109f411fa 100644 GIT binary patch delta 1176 zcmV;J1ZVr5Q=C(u3Am1D0U@kU`Lt zg@3XhDFaiMbq%KM&IAqw%fDQet!0W;R9U?%{dlc>3tJ`@XuE_VBoj*Ihm!*nB7fY-RuO)5&BD+3A@fj;AH8q;+I541#N5OndNtVhdW z(X8P6=$d_>Z$kW46jACf^dEBHPv189;AplQ;5SO=9o)%(%H`>|cU!__{TCEkxF zpW-9EACd{B_d_y>W(DtuWdKc*0W?k859#}R$RMD|LRF7d0hvtt0e>&Hfc#Fy3&$tX)|KYt{INLJ8(SO$<}2Lwo(ydO3SAZhY`Z18@dC1Qsu zKX|+gWj$I3NLKj7AN%0EABtr7BqR@1`MI)pm7n5bmM_JE_Rg1JOyk8(d>&6GX=9M& zXKP)!ZC!se$dx+&xc(B2hj>0*G8>MIPyFmp)?fLNjb}HF^w0H|iFn87zkh41=xi{$ zT2A6d_H`bgHsdez{0S=N+H4HJjMiTx(wHyC$zq&+U5{RTkI8bKa3Z}Z+5dWN|NmUm*SLczu@KUf3pe|zYYWy z0~2<$%_ArQ2J)Pef|A9v`y}212{9PBS#Jyg03?&*CszRplR79o0+SY#uof1RjwmVt qXOp=oIsw^}^e83-6$2A?lO8D+0lJelDLw%XlXNK~2EZl&00025TP?i+ delta 1210 zcmYL}e^8PK7{}jNjnJ_uP13x4F;LLl+%a;cjaPF`$XH~T4L8d!D6_Ks(dkz4Dzy+b zq4xzGmOqMEwHhh*!>MUqH9~E9H7zLfinO~@@fRVGYsg)<=kEFE`Q!Wfe(%2boZ48k zu^|udyPn-qv7Z0{l%|7tJ|5{u3zg~01gW2t)(fG&T4xdiFPvkO2!XXINaqd_-vaea zDZKa&lqJ1K_E|e0VFwRy;XiIT*t@(OMqtI}w4-?y=lnimWlT!PS zh#jFY6OCk5;1x&(MssitOg?ZgmKzn#tH?BFbY`@rFk`W&+I1~4hW043}??T?o z2`K;8^tNC!CMaHqjopp|7~FOlZeVWuQU*RO;1HC0EW{MmF4LLp4wfMF_c4wA7s&8# z_4PYZJvqMB^va^t=`c0V52hZ;N}DhG9@j$+PI&an)i4j>VxyWwdx@%)AdU)E*s_A7 zs{;$5jHc1aKeAZHyBCG15+U9Hdccsgix%DLv?%2q){$fr&iIje@d>MC$J5s?`A_FtWTY0^h$O!5Sfgx*bW~$B zOtJgj2`{+hgfsj^`?lZ&OyG7j;tt5<>SZ{SCt!405wA}pY{>Et$CK_?@2%UHJQ?XS zc^?qzrgW27O!eKnCVxhX0^+i+CSzR-A`v$g54Y@i;h~ykB(rQJyJPqoNBT}4$Q^xh=W4QC9V zYyNdGrLq3T0Ji*kN=yp@ort%P+QNtUo$VpaDs%S&(Ah1tFm z2`nZqWhQ?nW9jp8ht(Vv7gC{9Va+EF*BH*-pYB?Ph|B(r4Y?hov0X1^4wdKXgLZCx ztXZ^myZ+SFgWrr}h@m&}vz->WYFq{rL}QgmP!T-ytOs8HU5KOGs&NdbX%@l_x8vf~ z_I)pNklZh8wZ}QDI%(NgC+q{|HamJr^f&5wL%VL8RtWfv8y`kuM~Rj-<0t#~RG`$> z584&sPpyS3T@WwD+h-vF0C@Mp!0S9OO9lu4f+lj{`FTFF&6u;Pn?eV1_|pzC$3arpJM1Y?2-kGAj`7*5i2rzyden;}n;) TV$4VuNW}6ZDY#n^v-kf1cET*K diff --git a/summary_results.json b/summary_results.json index 47eaba0f5..af13409f4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.251, + "Performance_Result": 23.2966, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418152, + "Performance_Result": 0.416065, "Performance_Units": "Samples/s" }, { From ba0051c8f03890240ae4260570e03b1b0f337387 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 11:05:33 +0000 Subject: [PATCH 516/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10652 -> 10651 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 37c455e06..0c4cb82e0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 11:02:05,517 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-24 11:02:05,517 submission_checker1.py:3308 INFO] --- -[2025-04-24 11:02:05,517 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 11:05:28,070 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-24 11:05:28,070 submission_checker1.py:3308 INFO] --- +[2025-04-24 11:05:28,070 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 11:02:05,517 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 11:05:28,070 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-24 11:02:05,517 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 11:05:28,071 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 11:02:05,517 submission_checker1.py:3324 INFO] --- -[2025-04-24 11:02:05,517 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-24 11:02:05,517 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 11:02:05,517 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-24 11:02:05,517 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 11:02:05,517 submission_checker1.py:3345 INFO] --- -[2025-04-24 11:02:05,517 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 11:02:06,575 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 11:05:28,071 submission_checker1.py:3324 INFO] --- +[2025-04-24 11:05:28,071 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-24 11:05:28,071 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 11:05:28,071 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-24 11:05:28,071 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 11:05:28,071 submission_checker1.py:3345 INFO] --- +[2025-04-24 11:05:28,071 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 11:05:29,119 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416065 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2966 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.435272 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e29bf83e3..fdab865fb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1143 +1144 diff --git a/summary.csv b/summary.csv index e4f4ed387..bd2a3c4f9 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416065,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2966,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.435272,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c7c348883..2de610b4c 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.435272, + "Result":0.430777, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4e7b8ee24e151e38f7d15ef15d92aec109f411fa..0ab7778cb228f12fdcb8d1763170761a483c4868 100644 GIT binary patch delta 409 zcmV;K0cQT3Q=3z;atH+?LR1oNvv&x;3I#2{Ovlr+JQIu%1QM69F_X+AD}UQbN)WaK zZON&DLS0&N6VfITwSJJ)I)3{qw&Ns_Q;(~e{dRUyu{z2o`UUHRRwc@k6h)wz=0fdD z^ii+lC5oJ(X;FTh0sq#R+ULj zE2$5q-D5-9H{Aoy)ASKb@PCw3Pw@#Dn~8`tof%~$U66daJ&nBS zzX0Vp|^C%`85|^+sB>?~cPyzq|5dZ)H0000000000fPuM_G$}p-4U=;zBL=@F00000 DmvXH< delta 410 zcmV;L0cHN1Q=C(tl9U|O)&XsA#R9xdy;l3U7+gI8xPxFEqN5u`okwnDoQEXCOK9Vq6r=+IHz z8mCx@rE9Qd{F^m^3qlrHL12G^1%?m6)Q&`SLJB@2?T>2*fZ!NfP@sdyd6wggH-J-q zvSCoJoJxk?!ld;qj_$peqp|C{teXwim_Ns#>zB7-)l}-ksW_mz6r6GgEErdp__~OP zYnc!0c;c-D@z_Uw;_bA^u@6KN1JGmKv+^-}dfu#1U6A5FCG)hHZ}J(Ol7f=O*J0#M z{{^T{L;g)X1CF_GKSBKmvkfK60tGP`xLI$L+$UQB36nS|JOYyzldcvPlZ+@T0cVr9 zC^`YzlkzAg8x;c+b|nD-08jz|01*HH00000000000DytIlQStk0S=RLDI*5JCIA2c E0RLXIJ^%m! diff --git a/summary_results.json b/summary_results.json index af13409f4..a6e552b68 100644 --- a/summary_results.json +++ b/summary_results.json @@ -932,7 +932,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.435272, + "Performance_Result": 0.430777, "Performance_Units": "Samples/s" }, { From e840a7d804a2d998186c6493534a5abaadae8526 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 12:41:39 +0000 Subject: [PATCH 517/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10651 -> 10654 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 0c4cb82e0..e11258ae4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 11:05:28,070 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-24 11:05:28,070 submission_checker1.py:3308 INFO] --- -[2025-04-24 11:05:28,070 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 12:41:34,021 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-24 12:41:34,021 submission_checker1.py:3308 INFO] --- +[2025-04-24 12:41:34,021 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 11:05:28,070 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 12:41:34,021 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-24 11:05:28,071 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 12:41:34,021 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 11:05:28,071 submission_checker1.py:3324 INFO] --- -[2025-04-24 11:05:28,071 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-24 11:05:28,071 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 11:05:28,071 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-24 11:05:28,071 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 11:05:28,071 submission_checker1.py:3345 INFO] --- -[2025-04-24 11:05:28,071 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 11:05:29,119 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 12:41:34,021 submission_checker1.py:3324 INFO] --- +[2025-04-24 12:41:34,021 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-24 12:41:34,021 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 12:41:34,021 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-24 12:41:34,021 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 12:41:34,021 submission_checker1.py:3345 INFO] --- +[2025-04-24 12:41:34,022 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 12:41:35,037 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,11 +26,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.94993 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.24147 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416065 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2966 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416065 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1937 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fdab865fb..23ca55f5b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1144 +1145 diff --git a/summary.csv b/summary.csv index bd2a3c4f9..c316b1c1a 100644 --- a/summary.csv +++ b/summary.csv @@ -4,11 +4,11 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.94993,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.24147,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416065,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2966,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416065,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1937,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2de610b4c..5da339058 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.94993, + "Result":7.24147, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -244,7 +244,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -274,12 +274,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2966, + "Result":23.1937, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0ab7778cb228f12fdcb8d1763170761a483c4868..9b698f5b23bc22fbc73e454e132822ad659634f2 100644 GIT binary patch delta 2349 zcmZvedpy%^AIE>fa5t^yxO%dd8Hvgqk}#)YMozU%Drq!5x{FHPsr{U#oZ62$Rd(l8 z&%MO34kD*wjze-xMWe`ZIV{FKUa#l5|9P%|uh;8yUEk09^SNGEmd8VnqF{LikMmc` zg5UsvD+!fXlm$z3?lYQLcTQgg^QU#o#~t?>C^d{nVk3PR@Gv4C!1-TF7>LnHXUhSd zd|P|o!HsHsWU<@V=&n5}bF>Wa-9&lSrUwfD+L?ZEx1V)B*K16H_Q9ZDmozd`^?qdN z^6u#h#zOL)PoX#Rr$Z9QiADxWv^bpqW@m+2} z0mlQfu+o8y^74lK3%<}C5V-Zmw?Onw9}liwpzl>hwkN^8RERqkl5dPRa1LL^cyEP4kUmeNX@YH;rk+ zeczhqJh}?Bn2mS_^j?`yYpH6d_i`vhk91Nt4!CmSGOAJoePO*=ln+D=bH?$j{C0HL z;l`!F9;2W9UdzR*{J^*#9rvp5Hk2f0H75?^ZSoOR&vNPQkhtmClwV%!KXa{qkKyXk zxUw8qxtUz9Ek68bhy$#h?K4>fXYt;vTXkZm_!@vS^d`sZb2#&MLIf?M*0%9dFZSdx zhf?rPSiv_x`plkKTp#Q>_3%fA#RuuE$6)kz_X$jf)hi2ZWFWaH)DzZ2jZrKLz~7`H zs`81mZtm4D!^2btNk7si5X^L+5-S%o2-x2oi5 zeX)!#;@GNN`AX7x5yo|tTP=_-V+5>{qd1ltMr7V|;>1%K)D*2U&kG%3j}C(< zIYn_^a>~zrd5Lm<7G#(cW3%^syKz4u)L~`&kyH9gY@58nRIv2HYI`v$q{u7PCk`2> zEkH9A@|JD!xBX;Av3>w4e-dEgM6n4QJTnK-ijxR%O;L(;33NaO`cS%y!n=&i2_S(f zr@QjHvwX-UsG`24u^Ii8^ujK>4jMu5qB@o_sIQy|Q7yhoi?@g*L5sC!fS&VU+kCa0 zUiA_tlC@isb9Y#w-uPDUUP(3Oncz#k6HUs0HE?bmUz+g};kZuv{1AmSTGI6n@E0b- z)r)|Q^d=Y4Jz+1fsLvf(anYtQ5+uy27D-|RkFeB`R2}uhceg3wmH+6d!Jbtl<{6Vn z?&D2561<*oIA@>Iuid!|qcpL_#c|EU@3GY6=HEIHO+W@#nwohZ{Xr@@z@2WSL*OIt zS<7dzmXq-V4P-a^l@_qEZ2=>6eSof*S~lgXz0&dx4gLiJ7TdnbM^b_1ZZct9u~eHU%gJyC#aeU*wL?-uCul}u9Ef+WNrHWm^V*4*$0$Xi-DB?ht9-HV)?2=b z#v{5G+iMn!#g|5<4d&bR4CqFpT#bO`WG9^jG{f3xw>m*jn#O>$0+4i1xv+fRLi^_> zE0P_|S`E>7qZF>{>?>RSh*B~woS<3y5kgsgZK;D@^gE%raNTl~6TV{FY&B@n@m~Ve zbpj1o)Q-+|0#5%UVACKo8k_Rv5#_C@-S{BkQ*5)XF<80}3~bt~G-qpXAD6wWRyMt`xapnfaT{OkPMp7T2br zqX(in8v0Nu1sPGnzIACb$#wRyEMtuL+~D6CCsOs<6$N6?r~CL?7xzpcC+ZJ}*QNC# zcsisT1Z8aZy`||?>`s)W z0#EuFz3|6LLBWf30jZc(Av9L+ydI>KT@Mla)Ul#oFSXxKbbnF|&PyZ|zBxRMd5B1B zju{>@BrHL5X>{svud}B3viR!G!5Pt1pRU=q_(CT10=#$YL485XVv{)-ap$GUw%WZp zb82x$BX@&1>gP^(=cs+6a_;|Z%jZqkcUkM78z!X9BoW~B=|oWdIzc?ZCm0{uCTgC_ z5&ad^OKhtN+>c154-ZWY^{|KRl@GY6 z6g(xn{;6qN%ARucm+w>^WSjU@0BW+y!BH8%O;qfcIa@!!;|VFkRyL^MX|81ow>pM% zu@7kN{3%76P60h!f)Md7c6OSK5`_SI;6cE&mLDM_*xt&3pMI^uzEYwX87xStK`SG&Mk#7m{AR^-&ZVE|<}7p~(r+V=ixjF0FG>6xON!S9RToOub1DPRA-u$xIiV z<)0gTcj(Bis+`SX_lF*)w!IZL`A3hLw@eL}-WK{o0`=U6hDSZ~niqfVwK}j#w0{O} zDyS68%%MvZwHi14^P7P_Yaf@|2$|}bZ%9>z3RP91cpY7-0O+}nnN&YCtApFN=BSnw f0Jv@dfGxm3D=hpqWVp*xYBLnPOA}UuTif{?pfhmn delta 2329 zcmZ{m`#;nDAI7&ZhsHFeq}#|WXIffri{%i*mO?@l+8k=`+%l=~LAqy!%{G%m(e2QI zWTtY=A(bh|N)Cyes4$1fX`}C|@8PHK`ww_MuGjOruGiyz(=FSrEL={0r;R)O4=4zP z7C;g#B2sO5PykW@T z0_^T`-R&TSv!5z&+DXQ9g99L4NVNlO5{UQ;P&RugPwbJ(3;BGDA2SOk_Q82al!;zF z+T<{=bgGfb+mq9T;h}3khK3hX&-o23aY}_3$s(`V{DsUW5u?KyV|>-7szFgF`?|c302Rn2vcMQeNH1xFq7NjTNGaEd1BkY; z8^X&lN-H-hWv5JzG3Q&bnql^~Ci3ew8gGST6e?M~f>Mp8r~>Dw(DN?oo>&E{vdd{K zQU!mOMpAB57*~1;$|9tI&4^!>gy2rI6|91^MliisHwk6FzW|y+?A~jfO+CGTXk(l0 zw~L(3mke(&Obc%Iow6*WdjmN-oh>;V zgXt~0d9VXV8AricgmJLhnMW5itle6qhYpR!Z?~G^)VmZ;Vq@vnr3rBTF6Fs`F&da>s5*XIH-u)4Tp^x~TXDg?HtszcVhr)%Gp$ zru2Kh_Y*+KF}-WlUSQ8?X0O5oiLT$Z)cjavB->CslcdILuj@vMXjIKKGiqm-=Jx~6 zb`WcISnI_SC=;nwIgx*c3oIsyv7B0Ol&6eS-Q;?3O`+|BkB8_UI_`hljyNvSNH$;A znNU}+#x;MnJQhj9VD+g5NS`>)fXvH!?*!0tx8Sc3u8@IaJ$or9scqvOLoQ?Sr*Sh2 zrHgJp!nqRkjKidag63Sok}dky!Y>MI5(<(~7D^EO2I8V7gG9qpT)#D@btmUaxrUyU z8<(nBG}SJxB>l49C;dM8>QQ&EI)k;S`_UVZqf~=y5}hL)_J0vG34rbm1k2jkz#L~N z<->x8g}BCg$~F`Vmb?7K|J}9%is5ZZM)>mQ=C@@4^weW7kp`ip^XZ8D$KzEv_^Ez9 zEtSDdF8Vm6iqja4#C)aD3VR94L`spek`X;%W(YjiOb*uxPqUWG)D$z(?^`@J#YVI> zbu4KC^M_Lqb!WxqkG59pc*jRNciNq6WstX2(yMdn)Ok0F;ILheibjXGJ3v zpTH{YNXij&c_j`n{klqqMW3@`a?c)|cSN5sVwY3|Ps`&Xow5FO_q;kdT0on{|06L$ zPt1g}ysVe~)il|pI0Ab{Vgj02%>W>S5*N^NS}G9;*T1d>ZW(lVS0ai=pkZ&9x|)`F zYhj;8O&7Zi=VHuOax}h(m;~Jz2{1E*Wh+CeAEzOtJH}Hsu1K)hS_}4jOqz5Uh8~dpmImF>kusYLP{m=XJ7 zlb}ws5Zv|(grvZlc`Y_S3Y^KDJJ);fFqd7$ zT54p+d0Jwe6F1GhJh_0S!4x0ZKTxnj`0! zWTgRoRxJXfpc(QtmNFZgrxQTa!$s9@BwKnz@E*&}2+?pY;wotwiG;bYRF$7k&m9Ti znmkKbY|ro`j6NeqYq+37PBA|Qo$;*<^#%5{q+6_4uESGkQC2=^_0(%F;whJ&lPhT{ zLHkk`I=*Glqh5TfKU4gfVp{EeXvBVz3TWhFMos3$^7`Zb6li^N&q?j>>n6_y0XZ?g zj1$oeL3lS#at#s3WFO|FeD`^dTQ^;9#IE}fuPvOj{r<*0^y8j-#yJ0YimSLmX>yA6 zp`Ko|_N*<6i$Trl=H7YRIs};XoFtU5{_Z?poclg-)Qv?JEUZ5ssKh02eRehy#^3G4 znp)2o;ZW*_$EE=O#x3Lt2U$LU6Ny9;K!4Cf-JJ}h<#$KKjeu6c8*!q2l!ChG4;3?a_eS=c%uNO8s&vF1a zmpS=u+4LlPWROa?xVK0)?5|s3`BrR>j@pwbOV2OUn1$AKH-#a!$}^VN*rLzPB+39L zB&crh$Bx-|!`gx2odI8L^ z0il~M4RZOPGA^dv+uyeJPYcHUg&X8;NMB}jm1RopmwM2c`NyH=J$@}QBKdnbzOqj1 zhW~wK*rU6Sf#uj(-TfO_s%)OBDx0CJ4-RBk>so+&+23?gzlb9ARKZnjjGh%(kxkXp KhSZ}#lJ_q;)kzTm diff --git a/summary_results.json b/summary_results.json index a6e552b68..733f3bb3b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.94993, + "Performance_Result": 7.24147, "Performance_Units": "Samples/s" }, { @@ -617,7 +617,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2966, + "Performance_Result": 23.1937, "Performance_Units": "Samples/s" }, { @@ -653,7 +653,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 2063196818a9052d2c433a304fded2770fd62e68 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 12:47:07 +0000 Subject: [PATCH 518/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10654 -> 10658 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index e11258ae4..adb7278e6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 12:41:34,021 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-24 12:41:34,021 submission_checker1.py:3308 INFO] --- -[2025-04-24 12:41:34,021 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 12:47:01,943 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-24 12:47:01,943 submission_checker1.py:3308 INFO] --- +[2025-04-24 12:47:01,943 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 12:41:34,021 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 12:47:01,943 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-24 12:41:34,021 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 12:47:01,943 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 12:41:34,021 submission_checker1.py:3324 INFO] --- -[2025-04-24 12:41:34,021 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-24 12:41:34,021 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 12:41:34,021 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-24 12:41:34,021 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 12:41:34,021 submission_checker1.py:3345 INFO] --- -[2025-04-24 12:41:34,022 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 12:41:35,037 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 12:47:01,943 submission_checker1.py:3324 INFO] --- +[2025-04-24 12:47:01,943 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-24 12:47:01,943 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 12:47:01,943 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-24 12:47:01,943 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 12:47:01,943 submission_checker1.py:3345 INFO] --- +[2025-04-24 12:47:01,943 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 12:47:02,990 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,14 +38,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.41581 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.410481 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.394555 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419341 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417538 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7344 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 23ca55f5b..de439d056 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1145 +1146 diff --git a/summary.csv b/summary.csv index c316b1c1a..307b3041b 100644 --- a/summary.csv +++ b/summary.csv @@ -16,14 +16,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.41581,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.410481,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.394555,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419341,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417538,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7344,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5da339058..9800312b2 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.41581, + "Result":8.15108, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -609,7 +609,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410481, + "Result":0.394555, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419341, + "Result":0.417538, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 9b698f5b23bc22fbc73e454e132822ad659634f2..0771bff5f087fadec2b40acf3598fe46565994b3 100644 GIT binary patch delta 3075 zcmV+e4E*z+Q=(I_atH-l(VU!hlXnPQe~G=^&#qQ*v5UA2C%=4|P8xUdA{{5Q)23=! z%|<*MB*XFS=Ct|Y{mY}E*+{egY}lV9v-q_6HBOtqoc-tIbF%m&y^Z6nk#{glPn)+{ zHa~8+)4^>#?We6|9?$YyM#*B@&+=avH|=!3i2K7;(R9+*mgTjl{qd~1?%{Y*f2EIP zG#U@$7s+5bjc3`qlSMq~XZhi$x8r%*JUbbV^V^E~G#2sbwE0XOpZ{*FX8Y`9HTK7G z{5jqExlv5zQR9Xiv_=Xb zh~ViJi3pxvlZbc(y&(~Cf9g9VBJRHnjA(IoBN7p3w@V`8?Dj}RoZYu1BF^q5iHNiN z8;N-Q0q5}toCh7a@&ShtU)+P|7_r5CUyz8H@2?~x=KB&O$`U94)qRFMBfcg>hzH3V zLwV=^w)++8LN-L~0L2JhS=7TKx#qkj{mO^;<-?JW&R+lb^4F{Ae@*Y=$JKTy&G$n$ zNqnsp~;98;x(8 zKR1VQ)*nyO=1(W>e-T-*f2(6k`L;FA4{h~d$#K0~AD3EhGFp!ET|401`J6i+xf2hz z=NR$D)8aWsfIAYA+>wanjzq*G^p$VGj+Be1?>C0>4wfrR)L-n@i)Vxg>m6~2c74et zsS!iIC5I)~lS{uwCU0)9`-9>o%~AdeX5H1%O-|(eYnChqlT8dH2M?g<81cn}lXwhH zGtqF45nz!-B#R^>S;UC4#7M6~1B|R*zVtEFcS3>6g=~n}2y#Cvi+ZF;zTA3?8ArZ; zllcrBf1st@-6!Zh29?DjT(t`d#eMsnWCC#@=OhChvWx_gWh99tqb$+Ut8Iv}NPCbl zntJGLZ`|StEfE_umExx$MgIlxk%&dJJkyE zCGnfFtYwB3xO~4kur-)|OFPObxeTiKRrhBKf3LE;t^^Y^fxJFQG6*Kg0GK2LU}6T> z$Zd#%$$9`V4Lx*}d}rYTS|T=zys+AZvK}pyYb6Q6V!1PcLj-eYr7XzQR!cnr%{t~- zWy#zenA%)?ZUw$q@)}a{_nw|9d^^YWB$}8Bv_um#h$hJZnj`~gVuqlpO=z+n08K*= zf1T}(4`|Akh>e11%Ca6QgUhO{Yk8^ua%TenUNq~NW0fndn*-avrgUxBQzdNjH%gx= zeE-q)C7YNDY{@2OkWG>SHc1B9B=7Dbo3@Ef)&sC<=%KT{@qta*60uR_n~hy4>(Me; zHZg?Dru~m%Q$1-%TgMzLEt~f4vKh8~f6Fbo+@ttA`_B}F9W#*qMv33;dAfD8ANG6cp4#^;z z70ex$0W_H!18AD$9ohq+Y4Udz2t3ddu~86BaCk@=E}NulTsG^}c)k~!hwAmHeX++5 z9oH?p60~RbSPyKX7UD^@hhzfjfA)|JvRT3IVHseP88*PC$?%~)0GlSsM@a?e0$L(6 z3g`n+)niqFCX;yp&6;hVN6-{9!M9U+poY&o$|lP>F_!CoLb638dyj zGDu|w(T8P#N+#O?l_uGT_5f6xd>;Wume>QunW9dH$ z%LBE2-l=`v$FX#<@ekDXLO;gN6X+PCJDixFQ}URpXH}~o4VX^DVWUermLN7v4vG(OFx0i zL)CxOuK%N2zU`DSsV&Z=e@2hQlll+Igi`+@8DNrSB$F&7nIsuyNi9~J^dB~gWYW|@ zM}z$XEfGCTspCOekCg$FBm|dzbj|k99+H2DlUK_Gt5B(TXEc+-QU1I!b@<~0? z_#v548b2h1XjU+OSO(A}89>t{{ICuHO_T4V!S{idhzx=*9+dTH87!K_3W7zmX5r`C z(R|8&_CU3tE4{1z6z`b?s_fcqiS?t&2lq(phh#!&{g4cjS;6{Y86cBnfJ~G0!#V&m zP0o)7=LdgUA~Fae6PzDXhKptu=SSD<{CpoW57qq9!u+uJbN#2;vh$RHreLRF7d0hvtu0WuFs`}rFz*;?jT z3CeIEO3T${I-Xsk{@Cb{_)`5L8D&ZLhlGGimXUu{vW%pX9*~p_Y?J%LMv+*WI_TKo z{y>av`p@0NeK3`_QAP7LdhEyNUA5jjakdws%&BHJ?+P8g@G=4acP%m zKQ;;^zSMq5Mp@GPAt6Mvg7?ERfMgZ#$2NICY!pD!+Q?^qAI)dC8m0Jw9oE^&ynhpS`-_|LENx8UQ9edcY zS3kMU#-}99lIi_#w{d?MFN)vnW@D5j+5IoYDcOF-%h~?`P)h>@6aWAK2mk;8KLE2C z6|oKkPP7-Alg=X_f6HpaFc3xe{R*SKE!#;S2-|_SWYs{SE-l>&X_AOq4v~JYFDE7dL2KY$a$(bZM1?C9l@bhbyF}y zn6~h0wE^n|xF~>>Bg~X&@4X=yJGKW&T@oBBh+AzX^`W#oe{3lGraQoSn%-dvo^t9b zJ^^Dh5z&C;Y(knJ+jIcVFf<@Rd55zk!)LDtE8T2Er(8It@J9o))=`|?2PY81$L9VUfNMcoYJV zKJKV|ES{e0B{iz@G{240Wt=bTEGNr*lICBhkyrg^pd5z$n>Ygc+}EF=`UA5NCCUN? zTG5=Gb(7yGTLBM~J19H?nHH0=78a9_C@KMMle#E60pFALC?*6>v=^F_A1M|Az>_v9 RJ^>VybtxkT#wGv&007Mc`1Sw* delta 3091 zcmZ9Oc|6qp7RN{SC$blh5N0MR*(EU}#Eh{E82cnUH5ya)u}`+^_PY0vd%v&O`RDxcKIi*6pVxVv5|=`k+Hes;ms@vUh2f(` zynz?Z5jy z65ULjT{f6(3wxhMJNMcHGdf;^&aoR91QeNJn6r^2-38!$@M!l(4p)Z^Xqns|&%(C7 zG4AjF=v&}VU)x-_yv1yN=Nd!w^s5Sv9KM{z4t#wVMsZ--+sB&C&WiY~k@iODYP@mx zYHI{%msQ|Q^W;?eIW_FL862a1o;2{DIA^dDBYSgcu?rSBC8rc3y@aO6jx1j^iW?6M z;Lu2B8r>4MZMyE;v``>LlqQEos!d|PnWb-SKav1T&)dLoh&K=C?T2QVub%TmDl$aF zo-QfdHvtCRul-aTm+K)FM~bTW*FC#rWu8C&QJTJ)4Tbqyz^W=>8~(Ec1+WP2dAXXe z9I#s9W`wlV$$7za5nlizbrZgVvl4KuL0B!XJ+Y#m{w|)L2DnVAN^y6t`C4U=6d$D- zldnlj-F$7M#+s)sELd5fgR_4 zpI-;Gf#wImTl%CwZ>r#9&2ag zM&=pU&ofpQKJ`N*%@F>jT>Dkdj84zbY2@h6k21ha?eIoOq}{2h#*e(`FZQm6doO*a zm47B?5=t$`oS$AOir}b6v)VU~t5dc}ceA9vWLbfu!D?^SrMSP{XDimKvst~Hik30L zj6BG!cGO(EPqN6I5;r;OJ{?>oWl4cxAwzi7duwRO?V9Z*mR7g(-Vy(a!G3#(Z##bx z&frmE5Mk|Njy$(L10OBspg$OzCFT3b3Y-B}^Pr~HQxP>5{F&8%P}X%nMO!3vrUa!8 zi-f~9GwzikC=>TG6l^E}eoO2)pfg2$cai?t)#LbwE97D))7QO7sHAYXgIzuzq3eV=QBMO>kwg{10HFn;Mc3*Ug<*p&WZ+n)}?8pW@cJG&9EX)FE`^ha~@2^-m6%S%9=wO=*k47L;WS~aeM|yp+2hE zMHgZp+6?q0RoMg>6ISWV)sj*a?9cTxyDXG{#bapY@-DkaV6XWX^s47M;l}%pKSp0( zZ7gD+vn@K+wEz7l&b~{a5iO(N0i}b8VE!N+ncPtkd>y|I3i|!t_aK!xUzbL!AlVpF z-C;j7Pay_0HFTt5-#O?XCl>BE4=<(3~79@&J}`gB!q zqX|Hs=r}TPpr$ROg52iMji#N|-*-)aRq7!J z${>6E`%?2S352wVi2;dimYty^;A_+M9+e+goBK>%Yiy8pv~b&{!ubNT&-}!{Q|?|~ zP%SWjX9i9R0^Dk2-GzqHsp7SPdT{{~EfvTW=gX~c;|Xb*hkk-w*WKlc>1bthl0T6n zWsqXGyx^+*_t#Yi-nQ#I%5?sOiKy-0R+dyeL4q!Dc~w=RtUmF3o1QrY9~^IspSO$! zY1&h=aYj9Y4F}_>dJz4*+NM(y1>FD<42fogb!%selGzEPu%%Cw#c6kw;jrP&Ix$qO z)k7-&!vS$%OI3kdpSaql7Y~WJ^$ib~lmp)PeE+4Fmz{JXKqanMoP~pBYI;_eQ_r7(5oSr{wU5mQ1xpx$9&e(C}=( zzYY~u>zU`9E}edot%4IR-?7w7^AqG@{XnRSMGyhX!#+IZ{$yN5eDN2N4;2a^6oPy` zhH4WA4asl-P7v1IYhsdFj$zF-{7I=(e61vBDM`LVZpHW;(s1CzR^4tWhhKN2|BNoi zfLEp*xRYb2@lXYK(crW^{0x5pPOLak8sjb;UqRpO|FUh4vJ$#@OrrB%gV<5~>w>K% z9;D^fg#efzJhphWH*azf^B@+Qph-6Jy1pB*(rPyFQ?4_lfod$1QF^4iS@m-b!m^~p>BY}4% zQEwbO5FS~Obu<5OxRD8%;tC+FJqC7FRf}Xo(du^!Ti&DHlq7qgvKtOaqi$k)1m6?o-oRX11ey1&rrgM^%2kY1o! zCs6Z8CFmNcdvI_v!fpBByE)aY-C^j`!Hf`3np6~Q$e%w&Ys%ir`FDm?~Fq3fTn^;XcAX@eKFZcfAVx!G_r%yJeWe;2+x-Ul~y zP~>$jmGb7@N@d}-Z6+5AHbYpqfiV5T@`p&%&Xc!Zt!z$ZtY5`fV|sAB*^{qFwsO%P zdd>R_MvD^Dm$<}@6#Za9U9ghpuJ2nqK)dp9`f8LsyE*yslEeL$QpxzItirtB_Em;|%B*Sk(sV=G>UBK3tfQl3ctbtpr?&a|BfN85 zF=IAmS72R*Bn#<0^q=E>`lJ`%VLV*JlrIk-r*V-l5KqxC;TyyAG@y$AulIrEzg~Po V6W-($j876Db4p$41N7g?e*xgD2f6?N diff --git a/summary_results.json b/summary_results.json index 733f3bb3b..77c4a8431 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419341, + "Performance_Result": 0.417538, "Performance_Units": "Samples/s" }, { @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.41581, + "Performance_Result": 8.15108, "Performance_Units": "Samples/s" }, { @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.410481, + "Performance_Result": 0.394555, "Performance_Units": "Samples/s" }, { From 222d86980f36195a8f1d09b495ecca1f58b6519f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 12:52:39 +0000 Subject: [PATCH 519/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 10658 -> 10657 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index adb7278e6..1316384a1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 12:47:01,943 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-24 12:47:01,943 submission_checker1.py:3308 INFO] --- -[2025-04-24 12:47:01,943 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 12:52:33,902 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 +[2025-04-24 12:52:33,902 submission_checker1.py:3308 INFO] --- +[2025-04-24 12:52:33,902 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 12:47:01,943 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 12:52:33,902 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 -[2025-04-24 12:47:01,943 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 12:52:33,902 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 12:47:01,943 submission_checker1.py:3324 INFO] --- -[2025-04-24 12:47:01,943 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-24 12:47:01,943 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 12:47:01,943 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-24 12:47:01,943 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 12:47:01,943 submission_checker1.py:3345 INFO] --- -[2025-04-24 12:47:01,943 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 12:47:02,990 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 12:52:33,902 submission_checker1.py:3324 INFO] --- +[2025-04-24 12:52:33,902 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 +[2025-04-24 12:52:33,902 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 12:52:33,902 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 +[2025-04-24 12:52:33,902 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 12:52:33,902 submission_checker1.py:3345 INFO] --- +[2025-04-24 12:52:33,902 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 12:52:34,981 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,8 +29,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.24147 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416065 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1937 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418319 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1937 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -43,9 +43,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.394555 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.076 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417538 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419295 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7344 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index de439d056..8966a6064 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1146 +1147 diff --git a/summary.csv b/summary.csv index 307b3041b..86ec2f208 100644 --- a/summary.csv +++ b/summary.csv @@ -7,8 +7,8 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.24147,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416065,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1937,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418319,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1937,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -21,9 +21,9 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.394555,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.076,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0048,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417538,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419295,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7344,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9800312b2..efc3a6610 100644 --- a/summary.json +++ b/summary.json @@ -244,12 +244,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.416065, + "Result":0.418319, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -274,7 +274,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", + "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", @@ -669,7 +669,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.076, + "Result":21.0048, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -679,7 +679,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417538, + "Result":0.419295, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0771bff5f087fadec2b40acf3598fe46565994b3..620e9059a6ace823d64d063d7cc0d20fc4bd4235 100644 GIT binary patch delta 2767 zcmYM0dpy(Y1IITj!G0~_BCWO>cwsPXcBzn(v8Kjz-iUJ9lMhVlLVU({GKt{<=*`#@x5IGf?d{E;hap3 z@A~2VuHO$|n@?-BZf+wp1`|>58BZ8|_8s57;Qod;YgW^YFIZrE?d12b7|oFNKkSv_ zNBE|1XFOlOI$2TY65C}`c09IAdR;2TsecBjbBpuqIEIipG7X7Hll3)IeqktrWhC0% zv_U7M_Z&kYu#AwT;-Z%L&xw8~EoJ%8F3Vy<&{M8}smdH}@DbfmQ{dmS+pb_C8p$dC zNqw)yOS_)eCF7x>aOhd0zMABOGE|9Z!y@cq9JztDzcHt6)Y0?(vpv!}T2rCiy9cP$2s>KZe1MbXVv`qp=`~Hq2{<;sn)jbop0;@`~DG%>MatN)e)Y(Z=>Vh z`8#{4K6Yy^bRKPZNyNy&L!59OC=Nl!!>CD@{nBpc+Zh#BDlQ^Ye_|oA5(!k1;^WT=ruD(#MyTl-qr}K=r5LaWn}n@!~$pR4DfVVkr`Dsg50hrxLWb!Wua z;PX+IF7-CS3N|jxd4h^wZs{3sUuiz6D*e{)Rh3e`y1turzi(={Q@s^<5aLY#op_g z>OS4vNvYhf*)*Sk^$NK&_#&w>;@JeU$hV*Ah+>b<(NijxV1;2!8S!1wAxI7ez{hsL zgeL~JzuK0Nknxd4Z0e@eI;BoT7F{j>lRms7@<;#0x2%0ukpcAi4ona#-!+neNk1pe zR?k)Zp~e*Aoq`4DAdthbnm|TWhU*2P>3d3I%pq=vNZYBnY-}N<$JEy??Vn1ytcTg|=#8jD{OQL4WLB5r?(sG} zXuS;;9LcOW_K$N-!)zrlcg6ZS`EoPYHT=(Ii+i6(oW#N5F^8Zzf?35zx&w9|$$(-! zY;kNHgUFQSkM29m0^H$pL>KsO$r7Z_=aN#rgHC_Ac-GgnE8Ha^o{138drR0xw8{r=>8cRV9-J=eT-?NqGgix(r>X?| z?NhdB3ivc(sJ9`v`vH?k8t$3Yw(x8aIgJNLGzYX$@^592YFnV??LH@v)1Lzpz0w>= zHX26Gxq*BQO%If#8Hri+%iXjR%T`}a$Bp)oWND#&k6#KUT6zay=Th zIU3>30#}7zq#_wsM#61%wmK+W!5tn`#umK#uwrR%BWkrndedvAK8ZGZHBYcaGAclS z`Ul6hOZk%j%O7Z{lV>(w8QzBiqr~V6Gy*2$u`uJ=ir0`#D2^!FW3*&m1ydg=U7_|z z3lknMgR>H#D2-IotLV_arY)>!q^^0-vjyws@dtOplBP=76LW<3n-;8WRZp;4Ju<`* zoD^DM`hnqo?QIbj9o@v~YhJXj{JdaioTrG+9_E+>C9 z>wra0R4_eslkCM7S>FH2v@ABs$g5NX+P`F-i(6!dLczMX$Xb!QEy$4>PNh44#9Jt_TG@0`Bw5FG-d!=BU5ZqnO)o8I@4R`R zLdz^f2#V^weK4PR8e{ZSi6s3lB;y#<2q1`+lU0Zs9D?>YC|mkzt>bs;T}S)^zZd$) zFAHxNQ->w1o_`;LEDh8UXBd8WT?}>JRlOpA&Nx69xKzpd>!F|uS*a-6G~RW`$8T*2 zo4&|hMyXJUKNGIWuqB3t0b%@zfe=B#6I&?S@f}_Aov>j)3_io@my4?K90*&YV9_6$ zC$PSWD7ZErt8H;&p?b!}`#DQBi5xlMWl%OfhB2pWf^~+3nSqDZqxIG8JIC#I1KvOi z&~0so)X$cD)`h1nX!564d%A0LZI>sOM{HU?&Axx=7HX{QqpNvw?}`ciXzp~@RhqOS zc-Hv3uPuK{Ht_k5Ph`UvZPj@FwjT1qZv~FvM|x$v=ZkJY?-LSGUj49p!{8Tf`B4!N zlxI9s@g6_RyKk#SJDCs`j+=V_^zHiC!2mP+UEr0Bqk}=8^*eU(&tp8+@-$}uIeM;k zT8qshM{JCo9YuDhjO}mcDQghJH|#Ik)xZB0vS3jwt=KNLjRg(=1LW@A2ecnguGvks zCrK*gaJjeBF=^%VmdE*vQ<^q;nn)5<34g)L7!7#~yK#AlwX zKjm7TcYLw4qiSBqX|h!w{CjGpRNwYYPr=8R^PG{!Af3UFt_WwBi?+A6e}8yZJId$U zld!?1umjU12%s8gJ{#^iJ_I+Dfc6+pdMsh{^Y`x?91mMwy!X=&n delta 2768 zcmYM0dpr~BAICRyo8#Cq>(qo9+f1g5w2Bx}w#GRwMRai;QxlaEVJ>AP*GbGhN-n2n zE=9*>o&tvl4(u!PU>8Al_BBNS z08@EWo(7^XoiqM4yq;~0s8xZ~Ozc=}c{_Uec!V)DWjH1FOQL17_V;`KO^Ipo#q$Sa zmXy-<$i1#J@8W%K$xo|aW?T>Bp-Pl38qf@MKa<>e9OHlKsW5sJ-DS6-;QgUbVDrZp z(ZzI}?6dx$y0Pl=Lax1Uu~hsu>xWYD2$A~lIr(FBU;Y?YR;>kK2ZQ8q9UncSrX8Ub zM}(0q9}u;mElm+xy7#fF9mb~dxE!*EDE+^QiQJS}r+5P_N!6g7SwA%7D0-F$2G$O-teEI{sM{=?x9 zI+G@{_gau=tA9~w2@_^}NnO^QK!MaaccoJ(s5n$1P9P}ri>)BuOy0{1nWXs-kIL&S z?>u&Xbn2*cQX21KqQ7(hQb+UHYiT7Ay@RpL23*WtFR`I=Gypahket~B2(=?UHX(}I z!O_r-=9iXV3@+KY;v=03g z+oS^VB-!_<`8lnunmC(xbP(HjvY@hlG16uLT4DDkE8!e=T3~P*zwZ|RkvPf2!Nk|< zOq-gXe%8ty2VN7(*RLHeTr(DS!y4S@WT7@oH7~RU&59351`%7d8Az%T#?ToO@u-0D zdZks)kV0-gZf;1Ccpu@A>RKsp*9m5oiH99s@Ht}2{(vgFYYi>^IaXmz>R=kp0}~(t zHU9???#2Wg2Ef}7R7kwyC0=U|nzD7zNW8)V>TKT?QrI*P`TI7jh}Ln3nyPD^6+N7( zIkxkW`S+CZyj>O7VP7?Kd#@8Q$sd(?nyhV|>ghnkOjQt-^+kOUl<&I~1d~7V23kS4 z`^_fc1`)nHtLWlc{B-Y)hV3lI*X74y6>UoD{rWGURX!`HA+!g-kCLhxUQ$yl$%-6j z?wq>>4~}no#9JsZ0c2r#RwS7TNzn}yCPB7+o~I%(%wRwl%P-gJ4a|W1%po{GrrUUI z-frr7!{__;*Pv;(aWSzv`iN98y=dIRfJl)MFenixFhhWQm;uHL;3O*)f%eP(g&OxB zYrk!Nt8J0#)-yodMf}ZD&nW_!52_CPSQ7T3ts;>9nLY0bPj`z`jnla;Ni15lFo>nd zOfV<_@_27M;ifHSmXV-!AkaZhU1k(gWjV8V%+)8bpxlc835}tHxp{jYv&4C0#i3bs z!vVS(opX=Ma%dH|Pq~u`ksTI&^xh|%g)R-VX?sibUl$!uYG(d`@NU^}BH?bS6ciJu zor!`|NcxUh-x2sbUOm0$R?%XRzOspR$!Bxtqf;2i&wxV2fldb5+>s4>+L2a__0A!}HkSmT%kJZ4q zf#)E)(ZAxRRq^aP@70O{Epc2SGCE)H7m5kpUiOJ3@9aPhSWP~iUW+@!hn2N+LY#z) zF6)>YYyCIZ!S|_2H+p^`Gi_Z6sOYnf?C{gF&r$GCJt#3-DT2)WR>)nK;3-C#YNRbw z;WHk3hX{4v6BGM(Z$gnz(@BnlytxwumuFHwR%t^zZ*`F>v&w&-AfQxEOfkIZ$(;&4 z5I!Ii1@dBLBi;d%eeJ^yv4l6!6m$8^d{MH&aJ!>jcwkpY_bT$6Z4RP-UUaBtdmY*% z@;dpAWl%Ho_10b2k#Nh(sBf=+P9bKDaScFa-9`M`Dyg^EZ&i!=l4`pN%H2ehtb{!O zW8X+d=k=N8Rn$4UF5x=a*gY{gkoz2(X;hIdZ2y5n|#GHZ6 z%Ou$mIZ2PpBUNr8>|^4+w~bi!<3ynd14!SRV6-rlM!*TNh;tqy>JrJMXH6nZRU(Y) zxK|?V4GM1d7A0aU0c@D_h1}JDgzeTyqf3n8Sj`Gi$11%?CWZu|9pNix)p>gLWn$QT zrgZ7dAT4gWkSDiFoR)z3inqYHKmHH){U5N`NwO3F1IzviHeuC2^Ak)YcqyvTZ?tZI ziylwHe_aj`DVHBbMsaCpgpj!=*I$S(Dke z3%4V82VeHRPbJ_LPz}A|#N*CALL}0X6SXQStJk*qHheIF1~cpVB7Xl#{4@CbjjdqC zS)ool# z*1IOUEIvS2g)D|X_rj}f&s!hJD{uU+y|eq0>QvtJQVX~M@V^>~Mf+UuGFuJ_>on@uFwC(cI^HIR@I>jt;hmeccj+Sch3FvB zt0v*pM9fCX<-vBJa735SGu4M{k?QXTcz@GXf`su#S&KM7>rp#XyRv(Y8|TX#SzgOm z(>t9AwOy^nD1Pdc*#FsO`i?z!{xjq4nT8q=?`hE;2yfvJYyv8kwlykA|53G_#T01k z(T7L~`Ju=fuRgPa-TEKEzNxV;5LERa8x40{Nlj(DtyW)W8b{yhGsg?|oYtw@WtU!r zm1;5Ee!=O%YKS@^zdgA0kY804RGnLJ>YBq|i;S9u3)R4BndjJuH0{nb>U=$NhjEP@ z=^RvTSL_)-^m0L!F%2txWZV?Y`sfbMrC<4~YYOz-uPli8LOaSr3P9G>;GHgtEOUd3djoYEa zznMDjTAbn}_Q_X^5oH~c`Z3&}m;a*@isnoG`|D(m>o@@+OjGO;U?%e>)*R@^tj8Jw p|HE9vVs>uHL<0c;TPXlQ8Sw9lh-_jWmRt|UyrHWt+ltxR`7culE-(N9 diff --git a/summary_results.json b/summary_results.json index 77c4a8431..e0e2164de 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.417538, + "Performance_Result": 0.419295, "Performance_Units": "Samples/s" }, { @@ -617,7 +617,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -653,7 +653,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest x86", + "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "UsedModel": "retinanet", "Model": "retinanet", @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.416065, + "Performance_Result": 0.418319, "Performance_Units": "Samples/s" }, { @@ -1027,7 +1027,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline", "Software": "tf v2.19.0", - "operating_system": "Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -1040,7 +1040,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.076, + "Performance_Result": 21.0048, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 757d6d9e556c62fa4496ef61b01208d05a6d62cc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 13:04:22 +0000 Subject: [PATCH 520/719] Updated results summary [skip ci] --- README.md | 53 +++++++++++++++++++++--------------------- dbversion | 2 +- summary.csv | 3 ++- summary.json | 54 +++++++++++++++++++++++++++++++++---------- summary.xlsx | Bin 10657 -> 10736 bytes summary_results.json | 40 ++++++++++++++++++++++++++++++-- 6 files changed, 110 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 1316384a1..766141755 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 12:52:33,902 submission_checker1.py:3301 INFO] Results=29, NoResults=0, Power Results=0 -[2025-04-24 12:52:33,902 submission_checker1.py:3308 INFO] --- -[2025-04-24 12:52:33,902 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 13:04:16,959 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-24 13:04:16,959 submission_checker1.py:3308 INFO] --- +[2025-04-24 13:04:16,959 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 12:52:33,902 submission_checker1.py:3314 INFO] Open Results=29, Open Power Results=0 +[2025-04-24 13:04:16,959 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 12:52:33,902 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 13:04:16,959 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 12:52:33,902 submission_checker1.py:3324 INFO] --- -[2025-04-24 12:52:33,902 submission_checker1.py:3326 INFO] Systems=20, Power Systems=0 -[2025-04-24 12:52:33,902 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 12:52:33,902 submission_checker1.py:3335 INFO] Open Systems=20, Open Power Systems=0 -[2025-04-24 12:52:33,902 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 12:52:33,902 submission_checker1.py:3345 INFO] --- -[2025-04-24 12:52:33,902 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 12:52:34,981 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 13:04:16,959 submission_checker1.py:3324 INFO] --- +[2025-04-24 13:04:16,959 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-24 13:04:16,959 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 13:04:16,959 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-24 13:04:16,959 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 13:04:16,959 submission_checker1.py:3345 INFO] --- +[2025-04-24 13:04:16,959 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 13:04:18,046 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,7 +26,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.24147 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.69804 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418319 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,15 +38,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.394555 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419295 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7344 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34456 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.414526 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.394555 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419295 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7344 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34456 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 8966a6064..2a95aacb4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1147 +1148 diff --git a/summary.csv b/summary.csv index 86ec2f208..ab3eed1d6 100644 --- a/summary.csv +++ b/summary.csv @@ -4,7 +4,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.24147,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.69804,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418319,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,6 +16,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.414526,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index efc3a6610..2f08b074c 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.24147, + "Result":6.69804, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -510,6 +510,36 @@ "weight_data_types":"fp32" }, "17":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.414526, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -719,7 +749,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -749,7 +779,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -779,7 +809,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "26":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +839,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "27":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,7 +869,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 620e9059a6ace823d64d063d7cc0d20fc4bd4235..f5c146a1abd464209fcef74a3a30b528e4b6ee4c 100644 GIT binary patch delta 4783 zcmYjVc{tQv*dANdJDLVr!)uJSu_Ppf2{DW;l{FG$CzNdAhlC;9j3xUPQkJnKd&)Xw z-N^Xk)|EH1BWslVU1QOZDi_h?v%f^$SW3 zb+XeyC0k~m=5m?y65>7llq`OO83x*r&*;ZF;-<2_$h z!%Qc!cJC0Ys+=yEK(=CT>B6qM`;2>Tc3BrqHeS6*Mg2-lQ4kuZ9(&IQVwee-LOnDo zORl_Wv0btP`)T+M$)i0ETuB2O{63L5D}{O&K4D}1B0mC@oE_aI(K4XI$E zi|nuTJXWP~nj0~SNT*e`qCAw%(Z=uSg5m%3EySod#ftFPx~ zBXEXsovsl+voZoakINp1yaT{76LYc4b`Qt~?c>3k!uNE%H)pb^#mG$s3kLJ$hyXFZ z*}h1K3M)BCjf8ShlVE*BpsaX3JKcSL6b|)N--*|v9``(9J53Q5)P;Y!K}mJnUHUu} z#@5#v@nVIf!@enC&Xa8$sJ!S@s>A>%mBv zUCT*a^G*Hd;()%-E-5&YsvV>qdCm|w8M!lX>~Wwn_RKxHhfyTQ+G4C7*09nnblfYp zywv_l$ElL`VDoM zHqTb96iq}uPc6L8-0JDcy%~R!EK)i67pH{$R8XCuYY3T~a@}XG$o*QOonbN2(ZZmRWsnT}Bntx_d zYv4VGXy^4LxWvk)Pnj2y;wwoo?>3Pn9L~<&A3klFQ>SMHR*Io35EtlN;ok17^NxpV zBx%88;4ou26l-Uzk(8{@@dsIlgMlH&9DsE?M@B=5QzxhSlY~RDd|57ng7A`Mwk|W;A!2Ns_Bt z2qxSXaGP^qHz_E8@6_e^pkua)f4y8wnfbsT~wqFxYqspmfbRXY%F@MY}0`596 zWg#8BSd&4_*S3|&I7_@K_MN+<~^m|71k~T+=dsoLigPHSr3j4I_=|Qxt_Fx27D)`}p_9KhfWQ zB~-{vcc8!ODqec(jsp`=a&4jkeq7VF2JY1Yp$3|eY;qjLy*D*F4mKycaor*zWE zLmgn`=%svOAV{9|nor#owLA$cnBtiNiN3&>;CBWi3J)>|e$!<RqtB`Z zF1CBHYu0--{+3nDWqtUv+&(U)&cH9Ke|{5x337qHK?QFJ;Rfo62!ScZ^)dfeu8gSf z`>%qly3g`YN;R?7I#J9a?ggClsJBrV`ZY=-xZ2{wkNZ5f_*aOYV&Yc*HBhD~Q3u7e zLT?F^JG07QG;V(d)}V(HY{9`!IPcATM&zYH8DWt-f_3jeAj2JfDpfYG|nj zXV>kI!X);d0SX{}pp*_*q@+PLgiQBJ{zy&lvl7?P9~Fx;Cn_W~5$RJ5<`eFe+ub7o zTPm~bujV_y8WZf-G}2lMc~Wh$yjZbkg1FN~+nRFG4ooXi4j_F;@*$f(p3;7UPSRjD zfqt9W5~gTyv?+PwT6>2>HvBVHtV2I-aK}CbKwCek=RiC9jgF)0ls>*D6y<6x<$qUq zo&0SqHRr4MVvN(tq)=xbbsUAcUiv!`EK8Kbp;D93)z7s|GB*S#!n8)lkUCGP3{p@Y zMh{i-^(r|HT*TMc2^AK!Y2lqsqa9ZZ+wuF!)+Yc$i)OOXVxepQipA68v0tNgl>l)Y4(0S>}`x(cvCS=J^EFuYD8Ws zpB)d0!NZ_PfawBD>fGIwatG|4+ZHVskNwN3ELNhN&P@^eK02yv)Dwv869nmtN+$?+ zabPbOlMr*}wj$!;ur2iu0+J8X8(0R0RGoh=XkD|t%bhl-)&0gHDdlW+Q*XjVdE2zH z`2o>KuSd(T1K$fFMwlE4@xW`sciQDNM2P?>U34MpF$j|pA;|g>QmchIcU_c4{UsPu zeJA21-qvn#kLv9OolQRVc=EFc)AuK&knH_XLtb?_yWT<+=3M`ou9Ny_z!{uRB23Wb zx$FeXAF2(pB_W48pZ>A1K~4B(p&KQ*m9t-r=>HZ`JyP5tMxBx(M~F!0PyoFixZp?O zzvbW3l+$%&`WEF7ipJ5&8MH#yK|S(oYP>#z&e)!RBhW*^x;zgu?)aS0`tg_6oK7rc zr|Nh}`e<^bvEyKP@loK*O-;G{yqDECGF4Yzuu|>tplk(@Z#hJoTjcqz!fwSa(jyxo7MYcK)5A+_UY7_1)-cS{e4xXbGXl^*|LdZ2kd8pE`C($*+NA&n6@jbIEy;SW!S z9UTfd2TaK?=^Xjxmv#4^L)kIY;h3dq?up>sh{Z5obu&uE_N}{^x(is$lNFLn@%Fr^ zT7nq9_LA;Slbdyehc)0Zn4(N0(0nta{MBsa_~K&|=E;T=bDj7&rz>%LmHY3g{@p8p z^P5$7*+$Eo^EW6C0=bF{gS%9yBqzxo45%ke%x#Xokm(}YWd&O#*aUqTdN`tBbR?hV6m>%)kr zmL*a~O$4(|{rY@<_bCsyF_c?aa+X|UEEmYrzTM?grA(fg&Ei8sWN*g2z&IccpClyj zw}NS_m%GBIED58t*H*@{F4e0!%Ml-{3!V|+4%zI|v~n}iH?DeizJQ^`;AM8JRHtRQ zRN%jonc=wI!({#B`_R5<}%&s8Jk=q ztyEY>^yliA!?C*`PK>a$S6nijY!*$2mK{4xnpQ?HZYZOwYxArRi>WvNanuvbz&H24 z`Y6jI!GWz0C9(t+cmYgdEC?mH`DEZn#f4N03!pH8&7$VhtpS%Ag{*6d^1^-c`VGx% zN!}TL!xz1|M$$_j^B8yO5rO@Z4sLDvLWOTTb}*@&w9~(lt`e`zMA9JMea>Z{IFQq0 za2#X2IM;U(bv>V!I>QlitmXeyY>^)Ca`gPhTn0O&mWC*Sb5F zXFM7)QNUOVy%T4AEk;-SAeI???VEh@X=r+LbT15Lwmb_~lb>#lnd%!yXrr#&xY1F* z-Ko5_R~wIhBdJ(TBt384@-cH}m9c7yOv@7oaQL_}Am-H$$WHJnu8Mv4@g0! zAsxvHx|qs5r@eYbThg}A^;=Lwg|AVkP{cOlAq_FNb601CwsZjI3(&wwM=kx$W7-*^~{(Dg;*DkeOKR{J+D`I2+zd=XXM$q&G=U z!>23&tvmZ;QGEsGlLf;?S5qUv$RA=!R50& zrFHxDo{eYDPgfe+qg^^BO)fY3Yse}wh|fk2K}guY8ni|%%pskSK)DDQc#m*YS^ZSK1XY| za})Z~{i)!mp>}O~{>3(#x0)TzhUK3r*;x*m^EMnS@E8oVnFUHez=cou^Lv#UvDE|k z-fGH9Bv_F(V9SLPEjo>|e~lPo zEy{f%Bg_e|;@d!bs;&m)yuWPsVTct039Vds)7g2b_ixT#yN~~+bm`^xbqf{0?L0CQ za+Q?I{~7re+p>ZmuHhl6feGu((lCnD}uwgx5BJnkOV((P;9jV3+0jtb+RW8E--UkH9TJuXKGY%_JZY|D@>OoT>hyv zJmYy=D1jJa9iQIP_VaQ zZ}>|DN|Nq}; zMa-1$HeO8}#`oXU2Z1<1|EL4A%H&;7rF2^*K{C|5j_Y5{qHR( YG>jFFSC+U&XNUKf5I)5r_UG<@04I4b!~g&Q delta 4745 zcmYkAc{r3&|HemT5ZU*gF_y@ZEyk8TTe3t**%~5C_VC!U#xTRkzGaInL-w^2VG*fDSSlGz4UUv#q%w*3Fk?u~IkdK^+(;LR58LqfsQm_8peIg%H zf7oy&XPn)YbGEYh409;hk+e{i3Is?Qm}ECu%d0(Y8td_ZZI!)s;`y>?ql0yr`m$%w zs<7qT8F%u*V#Ig!8+z$o+1YMY%l3SrX$itfmV*}0ihX!9>G5PMN*lKJ_-MYYX(4C| zrp%taKb>$S%d)%LaAe4xfAO@^CM@r7E{9|Gvul1>7GmA6wr_W=fp_f zi}h}SQ!khTH?h^oFEtChgC6L7`6dx|2+H@s(@|;YOUYCD(lg02qqfOI7>`Snc3;x; zt`{4taO)(S_?UR+d1t4-hmuu>n5S8#_`A1PZ3gOKTP%dW@A~Z(t~I)L-1CWH-_(?? zA^Dt%Un$nw?PqSRTWy>+0MrTJ<`S0i6I0nSl_xPestEk%Z0@A+R$I}65k6NuNRm^v zKL(*pvvnHS2rfC&a8)FKa=}(DHQ$Vv(*HP(Xcwjj)b=8@BG3AD87P}lJJzN`TIGEU zKKe@YC-2V|#}g1xRan*`{PyPH`1}pOf(MD24gyR!LS=W)M?Qi9!+zzj<9$C2?UcqI zU;j~tmp;wW`gj&*EP#*h;G-wmtbyP+bsr$(ey%acQ{+8B#RZae@Y8#*D?IjOEk}S~+h#VP3x_ueAy4rLeOe{~*w^+%9xB z5KCD4kct?3a6#DnUb$+tgYWz5FEU8bR9M*bKdO+^E8SXyTm)q8`0eE{?>hLZJ4H?cP&nz{CSnqGVCZo$H!dvv#wLqog?W0DyAzF_E zd#rbd4R6MMPnrA4RDP>I+aW`_h?#is^s|?Tc-LJn!EWw&y=^@$A6)@F{qcapL*0Si zF#>V3U;vjcv@Yqt(R<@o^g(j7W?QEday*WI4*3DOfi1BYL@6|VKhxMnE{gz{>$lfe zp~61rat16>(;LdOd-uPMJgjIii|ADdww<1czCbDBGRGTl$uU^7ZjXKZx3S| zlD;H@jMGZhf+Ro$OsT0wkUsH6h3!#GaSjjF0s0M+UiBh9w~wO)G#7fN$&UPjhiT*0 z?aNGRf-vUOH^l_RItDZIwNnM~ z0o3;gW|-p`&5zTgv(`cC$qX=XcTB~wB{qO5iz{M> z{(O>19%iUBT@=c@o_3&@*}i;;_LHbFwB0uRDOC_+MSF!W)*JSP7nvyA*=hEc&z_-+ z9e3ZwDgsByCbfxLYicOYLE;J$?EQdJ;Fnn2#%(ES#{P{cNYq0lb-2C^2^M=LWbYDooH4n!Jyyc=g6u5wJaKz|;L zs2J>BvHba5sD_^z*jdYb-}KDZq$oa9-9T143XY^M(3c5^x=>_IMMF18hGl=FXs$8? z8S9b(&|f>+^~pCaTSi-N)^0ty{oP;1;yn~7{+;=rz-woZiO=jtyE~9iLf9XooWhOv zwy$GcvnQNXK-n=oO4FRY6KN9koZ6j}6E8445>d{Kn$7N07b5asQT`2npX)|3t~=MW zAK{v2kT{iuuKEkZdCujhd?OIgHf09D=rdmyg!C_6JbPrR<1N$y2+k3_>h2k$Zq}&f z&ZK3AKG0*op8fiXt$gkUsqDrp#}enUNqc!x-4kb=OMGI{o$FqjSg6aTdzg53r@wqi z68TJZ(u@dO#_THaL!yh1pD&PRV~8&5k(6C=R=H1(gT{DG`d&P8SEX?_u<7HvCXu~; zHM@5oB|*ytR7i)H>i;NI0cAk}-H2{d;@&Qgkj2KP&13XiKRMSMC#^g$3@U$sm|}gt zIqW@)xu)^l1#!>~b>q)7f362jc}k1r%Vyr>K@&M1k-BG5^G=c0x*|i;%)N-U6etc? z1zD(AUWUtLf`_XnmxIWR_!&c#)4^_s?PljZKzj0qa@eGeum)v1JT7o_k3Uo0O%)Pl zZb=tvMPBZNp}FIV429FGmtYB|qn}#B(Q!!v7Q27b$nJps$kUB+sV`GPpQCDFccxcz zr@n`B8Gx0qo~WW9QKHLJ1MeOZ7vxZbL|IvSg;|l;IvFRO)`Oi#!zM)TYAW~*1EzfE z>INULnw|UUunq(_P(N@$(w!bfzL~tR4jX8yz|mwxVmF_*PGMPWte6{rGnT{5&vdtr z)!r;k8&jgD@P=-`^ue}<68+BCNa~gq;qrO#6wjZ0;^C#VpX8N98{n0+bET~}Wi*iTh z854?~UIU=yqCf*JVm3IY%5qHbz2n{cM=Xto;=}%w8LP>GqenyqC8C1uKW~~L>_10e zpAr=eP`kxFhkq3!<$2N$;Oy=W*VxorT^3fPuU{~5*P!(+r~0?W)b2N3daR3@79JDj zXPl)s5mSRO-d>7zO8HDRK#K!qLI6^yte3psNK#xG!elAbh8a>dC^Gp@Q)FiPE$PF2 z2j71o@bBMA;_dJ}RcvH!bd#4E=&e&Wpw!4#$kO?yv_ zX(n?tFPcO~pS)l8NVSTTZ|wC6d^nUC(kMC^4FbP&Gu(0{)OWD;A66)@gld1^AI-MK zni9YGonq|DZ@sx#r(0_$Dbz*QJ%mT>>L@3W5o-H`5=hpl5kWo;zAqp{krjC7Td{L3 zoI#j^1sKfo%lFjX7d@FIO%JR7UH#jv-I<}Bfzc#GQ7tRP9pU?zZ0H|Z&i|2#s{JK% zI)zJx{g+IgqtRCV-(Sjb>vA>o$4h#?<_u&a-~ zKC8V75b^#LYn=S8WWik1cRD9WAK^@VVHVrwK=+K=@{>$IHO4zBA3j#0|OOtV|5qD;%#og7D)W03FDQ_qyhSRb;JG$Z|t1* zf2z;cIbJ!Q)@onapL=KFE6-*pA?z)9qJ%KX-pvRKqh&^U$j8|0Z0yjxer8>8k{;@; z)|Kq+ce=TjZ-kmhR7Tw&`UG5G;04%De{x?){R%s7DFobdWGFtjP>)E z*_rz|dyaR38E~Qk(_~D>-IgS~SvNeP)@M0F`(I6-*6)g9F;2l3(;hRtpAzv3ZIf(5 z`u-ON-nSd)e);@St2bwUMM8Cq=B6kj1A$IhQGZyj0qk>kqbrzFD#jOYX<*Vri=t=g zb`BfSIs8DE4$Vc*Az|H8S2)b==sy&bwXA)ZZuyjmzd1yt{;t?*)aEu*u z9jY=5hpc`XjkRN}Pv2R0Z4>jbsjKt*0sS>F##JcSZtTJFAZpa%4wHz*wG!ccX#R-G zqV?IyhZ}GOukCcFc8;Qv!yS@6*moV^>36n8A-+(Ox0+GklrbNsEZ(BB+iC=TB5fm8 z5*ecNNP;8XX~EU3?%@WMWE1}JvBO(~O(O@s@;pW^WA&B0OXYbUYj)FQ0TnyvzFQCH z%4YNPDiGWVUmF$1ScfZxLMCTBGQT$j(rcZ+dMVj6+rV)GuXo9UWW>=9ViAvl0w*I% zfg!8JM^{VWq6N_33APtMLFX% zMGr3{d%>(uowUxz9=YIn9m$bZA&w5M;tcrjyGak;j2OrTSLjH7T`l40FfYz<`!~4y z0!}&jsm--mCoB4i6AMf4GoWj*-rZs#hNY}P*2XTrLArrk(BOcL^4hJo($ZI|n*^G# z<@0*hulBdh)^bzd*t{%$Jz;|lQ@JD<9?4p@DD@mNanD4f|Y-q<$c{JxUXvb2Pfs>NbYxYOCnYcV3h~Xyg%xeeJ>3#mV;-IoKB%1-ptzxQ#4n z2Ft5z%z~67d|IA|&J?^-I{(%dpJ?~}ppi#G#6JY5y}Gd)?0>Tr9am{f3C>_$nDq6! zQDG0Wb#LqB4QzANWJBJsIB(i48_Oo~)mp*m7LQc?MAhSuV6FdZneP zbs@)}HQMVN4ryPg)wgm!qPUD9@s|@q@j=9ZuDc1f+;9U+8m6q#(cBbha`}P!-HpSY zO2v3o_&vMi?c5syt#8P?n38h z_oOvnHU4sYN3wCbStLA7@bfr)rGuYMxsg=gj)6-o&c;LNps+zgJ}q>N#VfCrU=>*g z3|-yDWw#&f3QZ*Cw`Agf71H?a@p*IVJ7(TGuP(2@^AHQMi~is@e&naPYY76lB31YO?=Mb( zVN@jj(%aTYB6)cV*TxrJJLAS=hHyq`c9gbp!rq3|Ug^c9v@-q+XdX%5OjCZr5ox`H z(7>ht{~{zWVo7opl_Vm-`M+il0x^O9_k~BiLJf#$khr6$MU^i7;6~|*3UU3jngj&8 fa|r~Z|KD6tctn>VDqd8Dgb~#zDoEZ7{rm4fU!4n4 diff --git a/summary_results.json b/summary_results.json index e0e2164de..ef211f883 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.24147, + "Performance_Result": 6.69804, "Performance_Units": "Samples/s" }, { @@ -977,6 +977,42 @@ "Availability": "available", "Category": "open", "Suite": "edge", + "System": "gh_ubuntu-latest_x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "UsedModel": "retinanet", + "Model": "retinanet", + "Scenario": "Offline", + "Accuracy": "mAP: 76.951", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline", + "Software": "pytorch v2.7.0", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet", + "ColorKey": "availableMLCommons", + "Performance_Result": 0.414526, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0029", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "UsedModel": "resnet50", @@ -1008,7 +1044,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0029", + "ID": "5.0-0030", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From d6aa8505b8787b736ca2473cb7f0a1e37c2fcdb7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 13:14:41 +0000 Subject: [PATCH 521/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10736 -> 10735 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 766141755..3b210d50e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 13:04:16,959 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-24 13:04:16,959 submission_checker1.py:3308 INFO] --- -[2025-04-24 13:04:16,959 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 13:14:36,225 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-24 13:14:36,225 submission_checker1.py:3308 INFO] --- +[2025-04-24 13:14:36,225 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 13:04:16,959 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-24 13:14:36,225 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 13:04:16,959 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 13:14:36,225 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 13:04:16,959 submission_checker1.py:3324 INFO] --- -[2025-04-24 13:04:16,959 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-24 13:04:16,959 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 13:04:16,959 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-24 13:04:16,959 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 13:04:16,959 submission_checker1.py:3345 INFO] --- -[2025-04-24 13:04:16,959 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 13:04:18,046 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 13:14:36,225 submission_checker1.py:3324 INFO] --- +[2025-04-24 13:14:36,225 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-24 13:14:36,225 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 13:14:36,225 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-24 13:14:36,225 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 13:14:36,225 submission_checker1.py:3345 INFO] --- +[2025-04-24 13:14:36,225 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 13:14:37,350 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.69804 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418319 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1937 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419295 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418086 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7344 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2a95aacb4..d84bb2950 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1148 +1149 diff --git a/summary.csv b/summary.csv index ab3eed1d6..2b991971e 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.69804,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418319,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1937,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0048,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419295,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418086,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7344,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2f08b074c..49e480718 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418319, + "Result":0.413064, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419295, + "Result":0.418086, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f5c146a1abd464209fcef74a3a30b528e4b6ee4c..be9feb5a8df8515e97288afda6c305ca429dbb68 100644 GIT binary patch delta 2865 zcmV-13(oZLQ}0u-atH)&@p2uLb_ikw1do@HliUbZ0g;n32|oh;K55?-9e-~UZ)M3l+3;RAeAsDd=kwW4O7lr+n1i)Hhbv4tu9zl? zc>9reB;xN!-jhhMANfEc;Wo0jT$?Y~h7OciE+)i0-#(gn_h{n%qlpiu#NJ9BzETJ3 zwOA=8Lb1|E6Ym~PyvKxmQcZpAt<4Yub|f?A`X3rM8xxVfe|f^ZbTyD=ypj&9Niv?h@<<4M8wg(BoT3R ze;08P#Cl(1L|LMn-|jQyFXAgQgm{s> zHk5bnAG_b7E@VSQCVwdL?8>4Z7Rf#54e3|DFfSiwK015#-^*XGqSw8PA6KVCX?{L* zGtO?8A6tWD+MZ7G|4#FBo&2YHyck`NXQOzLpRvW)S)69q>GI=roE9fP>Ghr4vf9}q zj#sBy?c!{z-HpyS^@}*oPh!&c=4|F@+w=8cK0g}Fmq*EL_J4V?oMrjx)X^q&JxFGw z@lEsR<}l9s<4M~5>7+d(8}{#YEGa*2jq|Ck{#P=ux9fSS^+`s{QGpedaOZRGeB@3% zozF4ii@$m27y<4`L~=(Wk~oI6dGV;_40hwP~Qm!Di^XLA|t^0 zs4VJ{BKbz?4Q3qq{w1szxM_coq(_tfs)x^k_pQ%O>sVd|2(Q8&2!ob#l@)LXmBoJ{ zT(t`d#dG_dWCD2^f@FY0mXRQ`j3kj{lqL7YZHTc*dyp`idgyE)+~No=5gSEboasVY zkCe%4j&E4ka7|%n0fzzQ!+ux*IgI0Z{*qjz@%5c*1^Jfv)mYZD!U|lz+e~Z?rq}Yq zVCT}sKA8Nf`!j`C*sVu@ZPVG`HUmfdW!q+ni=t32a6`!^W&%&L zi5X;*WPnYQ0XCUH1DAnZ6Pv6DVAIe;XZzp-o3bThqeyk83uQf4hTDH8hH%?-|6y!C z<&?ILHCEa--5oZIzdLrB`KL;(9fdlGC$$}t38c3}GKgjcbBARBP3G(4N_2J+O&dh$qz^ zk_n{SLo&!_1-pl3fK7j*vLKr#!-w_&Y?>q=B^8_tXo<)upbtP*k5vJhOy&VJYqohF zK~u;CUr*(M8a{88zelCx`i@^>kgK4CN;E_~Y59;$AT=M7K`JYVJ}d)N5~~HNG|4`+ z2cXjA`zUGSTtG|2Mv;-#c1JTT z!4fqRPYORI6H4QUWDv^=&JW7~mdx7mC6*@bhm8VMn!Fzc;t#Y$Y?|`lRdk`O$I9?6 zCJT8I%bNY4<;VOqb-CeEFqz#=S7)-tDXa=x`UzAXs{W&P{U6oxZKs4uZE+?wdL*9I ze@G^j`VYwflPrHDnPeHsB+E!7waNZr9Y`ci9dtCvKhP49LD0s7vK}h~BuNOa`skYF zpFJY~4kfRa1y+Gl@47$Q&j04vB`2_2l~9Qei6`A3k_n~zLo!HZ1^0(#fJ%}9DoyGS z>j0=UnLirLA83ilAcRUVe@GdezOt_2^i|hv{`~z^9;grY^H%R_KgB=evMsM{v(%F! z5hDS?lSUCIe-yS%EYNldm*|do()l5oP&z*(gIrc{epm*$BpKk+r2McBfJ>9{qrv!r zmWT|3HXfAqSQ(zgRxo~a&Bo7TTz(TBua-qtI+5K~eza}{&W^&mPo`5sCb}b@RDMV% zl*$juAej}EAC>_!Ne0L?89%H8Ak!rLXb^s&B_e|mA2LDsA!T?XTSfTMH48uAhs;AY ze)PWaJr3%!v>Bll=Vm%V9B}$EbW6+f4;3G zTg&>YU};N&aD@0GB5D z#|HTaS|T=0`O)HCDC@B@c#la!@E+4XIQd72vZDe?^(4KZmbF!RYiaL#KiX=AfiAh| zW0z=uKQw1^>^`qWaHUQBmHarEkEAz`5)RUIvb3xmXo-V zeVNCn&G_>?e};<1HXFmwqxJXuH0FzOvKVJy)}t3+FtnWX&!)-n$JL~kv+-Zc_{Sdx zKX}JK47{IyZs?yHr;U8+_tAWHt5J#{*tE`0=KY(v+h5#_XK7;+kMf0DEx*}VtiLfS zX!UcJ%vV3T&Bn(h%aZB+Z?|!O7%z(7>}F$>B-#Bh#Ua^#^U~S>0JBsT#|{Nzq+B~C zlOQB2e?QVtgzZ3EvTC5vl$P#alY7Q z83oA}q*#8PM&9&afN~h}Z{i3z=f3>}^&hihCCUN>Zt-#*voI(<0R>{DTstL`aw${+ PVUx!xBL)*E00000Wlnqa delta 2880 zcmV-G3%~U5Q}9!;atIFMAznhd3;+PVX#fBi0FwbI6_b?+Lj-`wYax@|2vq@=lQIcE z0)AVQa|uCz-R2f0gJhDf{@a+2^ZQaa8`J*h)27{Qd>Rk4+oI#f?RYqhi#xj67%bB) znf|f9No_hOj2~E5$@8R{Ch--J^;3n2-;usgJ#tx^$(_TzpK$bND%lh{ye$ zM8q9`_JTwNu)mUsvmbd$B7&#iNJQ}TibTXS=rxImyS_sr;_~C^tyNP*cv3$_H>f}cbcDd@}K7MVst&8jp9XqVvDb{IL)rp z<;UqbEe=2F^_|I=nm;#(an>JC(&kSm?Gf3qf3IUn`LQ+5r?&cE$-LgK=cU$% zjFzK99lD%5pL6FUcjDoEjuBt{%{#{ka7QANI}(xHk%)MP{^lF7BjsW#uMOoLELU<{ zyx6T5e~u%pcSL#DmrRlxG37U8T5=~qxb$me^5*usKPX<(9ObWI)?FRl}`v-qkX63I2v*mLu64rjL$I=h-sW-1Xv^y$s&nJ7BQkM`2r|3z{u+5 zdDKwf2?Z(_vLPZPz_^+=I?q4Wkbj(q(R)(hOUKS4sA7C;W;c%DBc7ioNbr&>Y2C4M!Q zwXCoLm+v+cTZ8GfyfE0gbg>U6zv})>;Z=6mm0)5f&=O2?f3+UKBpCpcWB^Re;2yaR zQ7~B#0H&dbj*{;zTtG|2Mv<39yHM6+Ww2M05Nwt^3phkDcXrBxOl`H)6VR+jBU-^w8Pf z_<*KtiP$KJf2J(!kutcf%DR>p*)Mk%@b5*ljx|=f(z==0_BEwzyPhgxlfO~=OyT>F zt}ofdOyEd1F@tQ946sQuz$STj7umE;Y_cAJO+ycz?Trs?%9eFjTtfusGh zZL`EhQ79LFtmVqFKS* zVHrS^xjTTSN#3D70GcL$M}fctEfE_9(FBKwl;O6?EFRc4>(qF@7n+Ca^{9QZ#||CW zExQu5XZBbRY@!z8NwtS$0_paU46<3l?qL~Vf0L*z$fn8gp*;YbCdo%h1?K`v9tZJtNa6f(irQ+c3<&s*j1QR%q8LV&=Rpxq~zU&vK}czEScB?Sk`RzRL9bP5|#&Q zfBU>u`?`-~=|GpgqZyW9i5iI~g&&d$rSU^Dh-C%mhh+duX6^VAOOy7)Mgb~K-j4$D z2U;RFP5JLCx=_|*W%v@4g*=I6&Hm5wV}49sZnzXoW_Q!oNwzq`s<5S>K;@z8KWf+i zQ7zwgN|@9ZXHug_;z|97WJ0O`kPI-%e=?FumXS=dj6_nK>>t*FMAFnjM}zzWEfE<6 zZ9FLJu`)oCgy5==u37%sBl7Q1@@iRN6)5$t`=jmrZ;oAZfYqvmN_0p(>Hd&RDBT~D zK`JY_KP&@Ok_=F3Qh!(nK&8q2(O~{SOGE}CRD$_K%Ha6Qx`yMcuG#$g`>8xo5bfu! z-qn7Jf5sKqs^m(n){`L-BLT*fMG+@|2wNr=XuE_!Vk1WWDr6gCI~;I z3=gtZgdbhA@bi7hJXGUH?;Agk?S*dHVb+rh64narfl5DDdRO`>-jnJQ#DBXD99byq zkut!Nbq!eB2dDmgTS>N-^;N;rw)U}9j$8H-&31|UW1~glOZA6jlqKCC5<)C1xIZid zSh5KPl1r2P!$tuvP4bTo@(;8`Y?|_;#k)|}V`cCflZ4u2b07;Yh zv%T>FNf3Y7H04M6ccHAu$`DCn2LX}?F8*wg44;JLff_$o_O9_$Tx7N!iDY}{+i9lp z;wCNbjZumf(LWr=H>?Fg6nr zjY!TWr1`N;58w<#0}_;XI7>2o_Ij|=%{KJPg;NUO8JM-6#L0bdVluXEo3xAG8uDlO zbMx}nuNn(=JQV{}6`T=ffqHGL0$a1C{S6HdGpz#qGy&%BL`41K_(5P%GEPs+#Q z>A7B`Do^v Date: Thu, 24 Apr 2025 13:22:17 +0000 Subject: [PATCH 522/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10735 -> 10739 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 3b210d50e..5c865875c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 13:14:36,225 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-24 13:14:36,225 submission_checker1.py:3308 INFO] --- -[2025-04-24 13:14:36,225 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 13:22:12,001 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-24 13:22:12,001 submission_checker1.py:3308 INFO] --- +[2025-04-24 13:22:12,001 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 13:14:36,225 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-24 13:22:12,001 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 13:14:36,225 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 13:22:12,001 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 13:14:36,225 submission_checker1.py:3324 INFO] --- -[2025-04-24 13:14:36,225 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-24 13:14:36,225 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 13:14:36,225 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-24 13:14:36,225 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 13:14:36,225 submission_checker1.py:3345 INFO] --- -[2025-04-24 13:14:36,225 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 13:14:37,350 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 13:22:12,001 submission_checker1.py:3324 INFO] --- +[2025-04-24 13:22:12,001 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-24 13:22:12,001 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 13:22:12,001 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-24 13:22:12,001 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 13:22:12,001 submission_checker1.py:3345 INFO] --- +[2025-04-24 13:22:12,001 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 13:22:13,050 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,8 +31,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1937 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.1815 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.7139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -46,8 +46,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418086 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420737 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7344 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70404 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34456 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d84bb2950..765f5bf7f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1149 +1150 diff --git a/summary.csv b/summary.csv index 2b991971e..5a06ef16b 100644 --- a/summary.csv +++ b/summary.csv @@ -9,8 +9,8 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1937,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.1815,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.7139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -24,8 +24,8 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0048,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418086,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420737,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7344,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70404,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34456,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 49e480718..a5ab77aa5 100644 --- a/summary.json +++ b/summary.json @@ -304,7 +304,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1815, + "Result":22.7139, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418086, + "Result":0.420737, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -849,7 +849,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7344, + "Result":1.70404, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index be9feb5a8df8515e97288afda6c305ca429dbb68..840be1b0249b84610708fe8229743cf17756cef4 100644 GIT binary patch delta 3420 zcmX|^dpy(a|Hn6HQe$qkl4XWD)!ptzobmz`- z+|I;J7?D$QNC(H_c9=197~kplc>F$pTz_8I`}KM~U)S~cEZKAHOYcb>NNn&s(h6yK|@Pog(943_a$+__>6)Y$(>rfGnSIT*}ABI zcwlFC;MKTK*da*y*vrYbg$JkNpBmJT9Ml=mN!=-HB>0|je6HnZ|3>GZq65yIWrhn1 ziS5q`^#$G~c$e~WNvv;?m1HK$rh6NS|MpQ}14Sjt&<927ky5(cIsj4gf z?$UV1h{i(QOJLG^qDbeV#^EnrPap>S7s8Ck2;Rn;`|p=>XZ;^bw#yG7W&N-#31-{; zj!>v1R~2rK{Cu9vxXqa!?R^C8dKaBCJBF8AQns!KFzjafuyaAnl{u3J1?_icsLPvP z6XUtD>nRk--vx;c$+ZFbAFhU3y}Qpl9UIfme__^o1mH%QiIl#j#+1V!5)e^;eKj{* zUX3$r{o}{U*qFLTQ^W+jt|>-g{f-%8;>!XxraCwjiC7%h-&-u9vg?aW$O*Q_i}oyk z<$!OU8cOHG*vl_&v?|Pp;}C8Qf_)vAEaOp@0i3b$C)?-3&OMQ_rG9#{(!=*%HyC{E z69aUJsAakk2U`#|v-K+cPK}xAmiKtilAX|v`Ae&{!R*PoQ}mhHobTYmaR^Uq34BIWhDD&CGxd^wXCFQ5OiwCCLf1c7kfc?3aF~%GnK_pyhFATbyb}KlS zW6ej{a7?X6zr47aP-)}}nc?I%hdrdfzX@b1b`cryzY_Ec=5gr5&-~J``sBMqaNGVF zTa5+Tky4=K24SP@WY1rfo35<-FU7O@L*#fyuI|@kBmH4L8Ht}`$(+*y)%r_Hs!c~? z%@@th?%@#=u+4P8rUd%a{CYilaOqW+FWar^&+XF*rC(&HU0l(GTdnl36mM1r@U6cj ziIqM!zmxrdaG(A*iWnRiK$APYOI3T?ys(yW=b`SrSE<8jL6h4N%YvqfP=I_+!|UPb zz0+}E{0Woa-sSR?R2zr&dQJ$Ts4-?qX2>`s;)1j^ih3{hWllY5I>zmqiR?W1oykk{ zTdfVdPpm96tB;WYNL-*!zEB(-h-&cd3#wS6E8$Ih++uX%$n4YSg-$%X9}jwpL=cg( zEW~-bnk(qB*g;~0Vd9}VgB+9TvwCLkBBv5NUn*--kJGm2z0w~HN0d2ALb*U=?|6jox!10qQt$XH~q zPR;Qagtzx-`w{HELO-y0Z;Vg}ETahs>EjAv=qcP`dKt=@|`MG=aJ zgj0~h*QnBwv9QmoRVq^I^8mOBv)X%MCFAkFUk6gX1P2yMb1k4z-S<4fWQmy`8zQBk zh!G-`7Fbx;$*4(98NC{IN3`+E%At$w^~_AoFoX6rWLk=(PptY)8EYySLlEn2mB!hp z_qrQ_(w-=v89NN`rR&Nb7VV>Z7+n;5eu?;L1+g{VBec`tr8rd9rgg-B#Y$LZSeI_i=Ukq_)`#C3uP=&0tye1RwEJ46UlP!f>MT%Nii+16 zbqo;_`6`Fp!)+7Ccv!Zc$6-8(iS+N4gxWUWfeoxldBVxdz^GmZ-}evy3fD?sMm0)m zT1cPnOP$96dzw+l|DRFzetm{4YlG_4m5NBAhK(RX7aDqz0bptgCG#L#%`Nw6+P7tjKyd0i;jx<;hd!*I<=8bS|8tx>oHJSLaV3iTUU~kJ z_25~<0Zog; zPvGVlfs)m~;t7Xmg`5O>>GgGXL#iGzYI32VuC(sVImb=ge^*R=v#M^nuHV%;@`D+S zX@TloP-W*Wo1$|t5$0lLh2OSIrLVCHkGQ&d_eTt)O!8S<7mn_jShl{>+;usp*a-+g zbai7=G|CD`c8L-16AQZbKaXPlcc=CYO9AgFU%4vx+R%ZIU)aR{J?(6hdNW)1K>qop zR@x4P72Ut90*j=2An7`)Iqoo4p59Mvp)f12fRrG%CGLbkOUymkOQ;Dc+3W##Dd(4y zr@nrl`p9h3a+q`3z>jUfGXN&=`^!s`$*80U7k5-r?K-hYlEY;R;3)PQ3Cr%E+9^E{ zX+hh}#ql38Vx-T$7Fi?Hn>UgJ_s?E&Dc|I*+ds_y^1-=boD>I7Mf{s*Uff(vb8?%UC=kx%#cr;Y^TaedaCH?vG34nSz8M%QqJNh*`3lQ)eq?cJx2;(O?v?!0{eSk zjKH94l5G&p7Lt$rB>t1nb{HHDxwO%@ z)tPuBG{<>D-m6N)+5vA^uuZW-CBygNMqveTQ>C-L(GiZ~k*^56zN6a$7{zcf$lisc z-!1iTVlk1o;e^R@pZv(|ut=W)GTAq(D&!a)4@R|BXkk)`0&LXjZ4lPcRQY%~1B@n! zbzmd~u^t_m+VWZs&sIZ_WYEp)DjwC|m)YR~-HjjCRXBE!;kw4GcCKmNRF>sD^gEsg znM?Y2F9Mzl%&a_WSrCExK%fvyFw^y1P`ZF>(vm=8x;jw(3z6#EP;H5z?zG2QC7v8j z@loNA5yh*;T-w{g)nu2TOx>~w%*HomZ@xfk(J!fvds5M!f*upGKEIg^Ao*UD2tkbp z%rLl3!EnBplOh7T`#-GXICdZ5x`Xhsm9BV0mzO(U0p?)^JCt~4-j?ZE2~R+Do$LX! zp*~VABJc(7r>n2wNt=7_i(rVa=gPE9XtCJH8I@QpK+ltE(f|;xZJ{6Xz9 z4vOX7$Tgj?WRu;V5fvepg&lL694-HfEp5drpUFJJ|=Qo{0 ziwD`{HO0?{tsFLW#Q$p8AwPh*b0*Wru+BRVD0=&&-N1O))cKqzsqf=Gs6G=o&f!(S zfXvxMyRzY%H+T;#cH*+y!cN#IO#asuvN~b#ghAU8T(3LTX@mFlc5cIVc`e_>mw4ny z>jNM5^X(@ql8*C@(@O4~v|tOT4~?+u0lDC#s z)2JHZrMMH(4Z90^i;&54>~xuNpLi;=Za2T$veLCNfQv16kZ(UA@_Ojddm0=G2fA;y z{4Nu{6fTupr|V3xl&uu@WfeR+>*91zu3ugDp9sp}nTRbdn+fDM`@}b?{Ggx&gD*9n@zaB?_T$C-sJhapI zagd5*T-mIF-FwfzoZRXYHf1uFRlqw2+pkj8YGoU&En!vKvlmNy^bDDrNvZ5kJ*S%jaG z7k79bWxp6HIynb*Rb>0?a<0qU)+Zg!+8(pJ4nAE6v(IzScwAW88^iR_S$&LN>DMNLW+Q%`HF?l^cqHjv_=FO07K6TSp-DY1 z^sj3;Li9#~3v8_V(VN&YOaG+3ua`$U1LKyHb>`NoSX-?ke!SJ0Sut`Z>7Lclch}Wz zjm``vU#+e7DKk5#g_{X;lJPH}3vRvYWD#4~eO7$WG~QBt&oJ*e%-F28hu(Tl=GYxz zzD{oAgwlLtO<=OTX&M@Q2dsGxEKj-QZkm>UX4j4}HJHUNQPDeiIUNawQ?^>k8>(=< z@TL%D{wlpM8{zf7&Va~luS}OM~@KQ zrQ%LlUWoW&Gb|;3U@uEUQ|mpq1ok^klY4q*pTAYKkZBwRU$5M@qJBkyP31*Z|xA*+pkiItdJ zsRoRjlfIzuBa(XZs;69YrcOGgq_eLk2jF|J;XEz_6M$p&Bs}1syQAS7RF{JJ9 zx%9QK_;0L_(k^uaR=zumdKW&rPjojT@AIGVj@)+STy%_~WSO2ZV1K;{pb-F8RG`7( z*jv90(H{sW$bkFt>c*BsPA{YAiT5-QQEogcZT{q54%X;{h3>hQ=sx==T|BG*%46+3 zr)HI>zFFCn$L)<14nJ-|<}`M{TV1mHIkT^q=5q;J0b$%XLtrJkz{;kIN5pV`Cuz$7 zAC;|cB24XS(7wCScJyV8vWtV)#nz-hna8)Ut>!zCg`zIRK`UQgZg=%X>x>F^KswSc zZ1eLnB>TS9J~q@l6-NacVkP>(p`mHGfd=R_un)1+#X+vUFY=}I=`0v}?EQ4JROSUv z=4TCYli6o)ZOkn5yq8gfyA?Vh<6JjJcbPrM@7OqU+o)~OG#~5#`n3YCZ9fAn>O+;u zBU6-Piah^Iss7moSHBxR*c3%f<2?)gdN{*|q82=_0dGoDzK#Ahb76HN)+~xLfkueh zsxr$|G&9;oF}rk|drz17W!?|bY!>7a6BAOj-Rp!y`~Wom+CcE8hc9w0y(amnL%-pm z8rd-E-SHb)++xd8wcEyt+SO`Qse%{g@I-I%pad@fZhRKNfQBcm0DKsHHE|KS=UYB1 z?o+oFs#S%#50u)Ca$SAYpY$?U8r6>nyS(^g*_F%jJW)Zpkf{AtT_`qS4sY=mf1ThJ zh0AB{&xY0u#e5lH5M+1Gb{`74MJG?r^nA*BAj4`Lh`3r4u5M=6BIag27OjE|$arT_ zIJ%zfb}98~UYsQYZ9?RxTJ1jyqM98zn23al+ABO2e~e%wpKr_S{>9TFj=GqhX*ium z4HY6Okf|hGl5=;e;@GL3*~H@2OJ)`wW~PzJg`yX4SR8?UP&le!Eloy&6)i^-2s>M) zh4%!2-qt@4p+jd9?upnl-hjkMwi;$E44)6)isfHD==N}hR)*l=STi3H^6ICpLDnG7 z@iMDXw^Z7BUCLhl$p-U=0Q7aiuQ(3t7YIFi6P){njDe=F%dr78m{t}+nGJ5mfUDUX z9%>jcPA!;>XhC%;q?vU!#j?~7ih{9{&(EaI2wsWZLhSQm$DB)r5+u=0=Y0)B;T)EU}5^TS#+Jy9mcP(6JaubkW9KsF!+ zw>Ncqx{wsr%IT*ZjME0slb-vFcO-wTwJx$d39Z%2wI>N3{3OTTpS3$3LCMDJ^mA7%Tfm@?TDP|_{1syG7XK;4Alz$kT%8a@3t|eX68ygythypG%KnV=!Jvy( zvx|`}CicS?NpcI7f-Ze-Z+C42dG+(JpqmAA=I}>Lkr;{3r9XHTGI>~dR1tR7Lrpk) z0~WC}qI&YmVmK3>=}P!_1VvFS+q=ECvzN(DU*w2pke)3)TeABOm$Ii&iqGVLaL5`9 zq5F!K6A6UWwtrKd1r{YR(uIAd65%9-(!YU|%yQ?a!oQOdPnA%$-lG$|)bNs_ls!W! zeq1l(T>W_xLKf!I9Y(O@2E+IrXz;8f1hJ?dG*m;fc=t=Vv z0M1(gU?dO({2f$Zz3HcDub0+^W5405o%)B#zQLFX;zsISR>8XYNmEEG=77^_TQ;0( zIZ!~}sE`F`6v-MJmI%wLtd~fJr3;s|JG)i_HWO<{t}6pOx2`{NCpz=+2b%SGiQcc# zg38aD^M&i?@K4lPIVlbL?(f1>z&LRt>_Y5@2NRG9rPECqY= zuX+#i7}p|^#T3CVgWu-StIvmDB@^dvqWZ7gj@ExQ>6z(+GE@QAec9@|o2Q|HX2q=5 z!=u32JNFY6$bC9d1z5gt@kfCZQNpFb0ggR|R^gp?NN-|A4XT-Dh;#%e2U>=Hm9XMH6!Tokl~|`cCHZDi1N96v$G# z?Dmbh*UjL0&QfO)`)~8t0(#ZFTkBdgNCFksjFNftkmNu4P^I`@#MJ8Nh>)k}?tuE9 zcbpv;WWOA6`+oPb5z$j2W#RS(y>}dZ>AS-|%R%%uN7SqKre<&}bxIKMB9N?f=+5vT zZ_jivM{uZ7gF{;E=4=A0@VlF3$(IUTjzfR|fQ<+MAP)cl2mzip*8{Ev9r6sg?xi2* z=lg)#dZpfA=c!eJH1n+zE={N%I zX{kCUz#&?@jv26)CZ}r*gwZT@p{oDsxzP51zo_UPv?N_5(1Z3#S4%=sd+YFj05Ttf AJpcdz diff --git a/summary_results.json b/summary_results.json index b117b3a23..7ae25ebce 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418086, + "Performance_Result": 0.420737, "Performance_Units": "Samples/s" }, { @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.7344, + "Performance_Result": 1.70404, "Performance_Units": "Samples/s" }, { @@ -869,7 +869,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "resnet50", "Model": "resnet", @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1815, + "Performance_Result": 22.7139, "Performance_Units": "Samples/s" }, { @@ -905,7 +905,7 @@ "Availability": "available", "Category": "open", "Suite": "edge", - "System": "gh_ubuntu-latest_x86", + "System": "gh_ubuntu-latest x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "UsedModel": "retinanet", "Model": "retinanet", From 553d67e08bb4627f4d0cbfbdd72f7fc7005d31f2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 13:27:56 +0000 Subject: [PATCH 523/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10739 -> 10739 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5c865875c..c141c7917 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 13:22:12,001 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-24 13:22:12,001 submission_checker1.py:3308 INFO] --- -[2025-04-24 13:22:12,001 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 13:27:51,391 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-24 13:27:51,391 submission_checker1.py:3308 INFO] --- +[2025-04-24 13:27:51,391 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 13:22:12,001 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-24 13:27:51,391 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 13:22:12,001 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 13:27:51,391 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 13:22:12,001 submission_checker1.py:3324 INFO] --- -[2025-04-24 13:22:12,001 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-24 13:22:12,001 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 13:22:12,001 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-24 13:22:12,001 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 13:22:12,001 submission_checker1.py:3345 INFO] --- -[2025-04-24 13:22:12,001 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 13:22:13,050 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 13:27:51,391 submission_checker1.py:3324 INFO] --- +[2025-04-24 13:27:51,391 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-24 13:27:51,391 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 13:27:51,391 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-24 13:27:51,391 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 13:27:51,391 submission_checker1.py:3345 INFO] --- +[2025-04-24 13:27:51,391 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 13:27:52,687 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,11 +38,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.414526 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.410549 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.394555 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 765f5bf7f..691cbe1f9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1150 +1151 diff --git a/summary.csv b/summary.csv index 5a06ef16b..10c3dafc3 100644 --- a/summary.csv +++ b/summary.csv @@ -16,11 +16,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.414526,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.410549,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.394555,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0048,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a5ab77aa5..e316cb1a4 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.414526, + "Result":0.410549, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -639,7 +639,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.394555, + "Result":0.290026, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 840be1b0249b84610708fe8229743cf17756cef4..ea81d0225472fd92a51bb65fec4a743e67b8eb76 100644 GIT binary patch delta 1960 zcmZ9NX;f3!9>s4+k-=yn;t&v`mV`lY$V(}uNElRdiD3|tK_gRw2%!x12|OW}NinER z7`E=Gg>%aGJ?{gNs7raGr z+L~Lkv)(pK;>owK*VD#qX1xxc1 z%qD%000+}8l+H{BPo`Uneg@<|ig2=|iP1BNN|M~c4H~lD-h;X4B^-NoNo-=_VwE?D z^c!F;pXlg?Fnq0u&%gV8zRwMva=h7GhUEuw3^`_(aM?QLb1uXyOwEZ*CO{3F`Gldu z%k4)Imb786#++8b9vhl`*7CmRKq&JQ2EJs^VRkCJ=R?`Y=sz?X91Dx@>jxWb+p0w6 zMUK#T$uv3QxysA>b)1ZZjEXh?Cd*W?2vG<*O0*QnvHH4%fz`o3qY&yrRj@IIKr=p1 z8YK5%Za8^%q>D}VA_<`OzR#2SGew6?Rh!Lf`I;w7j?c=-LtU!LoV9fQ8n?>BoJa6H zTAFcOxf?4KrzzTrj(91MOIXSLHvpZ;{|mQi8eMmoq34XRBnwepwbhZfEz|k^MJ+`# z30e1sLaK}1Rftr%&?Oej6H1v2S|E6r#;3RCvD3NuJ8Tf=aV9$yxHlnuP2%C*KbnpE$U$g~RoI zwr8&~Bfbz#UwGyi7|)_#EiafE&WAP6yo7Cg&%4at+vO1Z`>r`CBv82uOK%w@_?oZD=FSL3OMR zlhhgqNv)696VgtH{hUOR16GQpLj@k+^_WyH|5zShU!*9nsC^HK7g(I7GO5-I<(*pU zxwUnvt*h%&zxzI=Lb@7U2ILc_T+axHTip)O@Ia`3Nk7-!qPjbtliOg3ugIoh<^{Xx=gvf*D}+%J|o`*3{_`Ntpy zf`kSYawGc}0Rz4qSxk}Jvwo(tfIgxe48QTIcWpP83Cc4wQr!4APKd7wvzey34WLrp z1WR=jSUjj09};gA{v~TE_;-e)I<&h|-XLIEzq^xG9yJqs+v?ZFvSI2N&gc*(P)0T_ zvpT2Z>4o^WDub~PvuO}Tz=(pC>hkWRt#UW5s*AwNpYMCzw9fS79ysI8k&5J)Rc;K^ zRWo}UOmni1yfZ)4JNyk)*RPAD$Gzy-rw_5tQ<5x{ylY~i+{O6{Lvum>EPyt<9eagJ zkrx2MtCHq91n3D3lE!bz<4dYLWP0F;79v~gt`cJLP(>MIsdi-F<+}^sr^6a{au^pU zHbzv;jSw8TW^mm>%6wBK{kJGO6xp3?rxYpE3NjNM&Jc}$y6Q1H<1rA}0bf7+?e)dl zrR{489TQOaok`Y#`vLAFN0*rQvEBz5$} z^+MzVLOaeE4(NIUS&!nN9f~)`sskU{tG|^%5x>~Xm%nyh=&w0y!4B%P`#x_adgKQv z_FsbX>!7`Cdqlk=EcBkTx46ICpBT6{W@Chdt*8|%Ew7%e1^;f&mK0p~%4V#z7{p7L zpWW9~rwiVj6|;2Q3lP95;%ASFGfHRD{0jsf{r4;@?8rDGRfo z-sP=bxG)})$fGT89MGw8%fVY@IUel1R7G99=l_^%9w)GLH>xQFefxrd9AW{H8@d;Vpmr0T&}EyDi+KMn< z{Bj9BIN`84p^K7EHEX9_qfVzXj*e1h%r>^2FTekOpU30-*XN)2>-BuTKVMb2D!et- z!nEjavUj&6!@_6=jMgY_vA@q(KXv$!w|~f=(5U7IJHB=hki4*?d)#4(h*HxrJkqz_ z{O}-J>(co4kYCW=@1Qx5pEH{*pR;iA>L`g^Wl`>r(PX;8hi1>*Sl;8d^+m=q;&q6r zbHp{LajJ~RGHKNEtqw$y=8qt&1#}NeJyl(^2U3c8Lk6gur%a-^d0i;EbDK7@oqf*P zHVYyvmsd{Hb)gUD{F16Vj>jzi;J)>}I0N5{a1YzAteXwM)!;c4v$m$U%dJ+2rA=F7 zjwTFnMqT{srAuLs2EU+(*IkXwefE8sO;#~S#rz#@O=HF=&eNhs+4{@*(vK%^c}q8v z>2^2gZGi?tgq}R3BmO+e2q`;O?XjW$yW)pJ16*1#V01#}3fSlp_w6+-q_ocKZ(dSJrPx&g=y6zXb8`aID&EB}Ty3%rsZZRx4?Q{OWs zI@MzMnd(2EG46iQPwb$x8UG$T))>^fm-|pS4vu$V*$MejoUiG{22=6~E_E!kD@2K)II0=&1iu!<>Nq&D6hE6&?Yxa^jq6!-Vs|D) zDs$50-Om=CRJ5}w4`1nHb-?Y3s-O>ucVr=ay7<8g0*w=LuOxpKM1UDQWg-PyV%a zqc*QbrLRQA#hE|HM-dah@Eb(y}j*4Wm4UP_O1> zg#_j`Lwp(~yV1Ze6xS^U_*=x8F7nur5JBGdW~v@+zXG#B=IU13UxSZom~M0P69VUn zxqmsGCj#6Z44KX?4EZTMq3c%g6G_k?sLdXTm|H}2t-Gs{Z%vcp1c9RB2ytQEfR_}k zj9L$F!umOQ``(WyVf$BIYh^KEB6W|H_+~`d%+5L3+TNVwUaAn~1WK6OjHB<1HQ*W` z+0-Rofwjz14T>fL68-1UCo1MqUY02d13=<)U?pZr)y_Z9g0^w(%g}2)aWmXOpaARR z#*YX^SRIdIl)ezDTeP*!kzc+S$b8e!Yhr5y#)FhRR*AD5h44pXuYmng#A~4f%mVL{ z38j2!Wh*H8y58%qMESjS>^+e`QV>K7J|P8Ih~u&}Xf%9C+a=DpS~ZT7Oj6LVd}3%{ zU$Wn=JVPj}yF?PaVSeaK`k-TEwG1hF3-*5y9;%drv6MsEo00*s!Mie%a66$6XukLU zv5&7YfUz%*+drI&*e&){Y1)xM?_2ZF8eZPw;IsAma7BBur{1^xmy_QFDq$5PW70c-ZN}2Y3Z;8e z@OW4^?Z)Kp%@-Fgn>BNO7igz(+ zkro77^92L~u>e0JWGdA}AxW_Nbpy~avgxTa6at}{Kp-~Y9VZo-nJjd}je=v3PQjLokc{*X6k$t`3Y{WiSlMwvc6 zJ*;c=hL_fKs=nXOO}xdqy36K3GIqY-eUMzDx!-=cxt~_!K!_g7aPBApHk&8^PER4_ zhi;=}^(CE6ccNctoARMOjq9(e?ffdF%{r*#H<2>=rfKlpG8AhO+!16?tj|G#gr^3{`@XCu7m8vhSMm86pZ diff --git a/summary_results.json b/summary_results.json index 7ae25ebce..7f6419095 100644 --- a/summary_results.json +++ b/summary_results.json @@ -500,7 +500,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.394555, + "Performance_Result": 0.290026, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.414526, + "Performance_Result": 0.410549, "Performance_Units": "Samples/s" }, { From 33db327e15973ef89e806a46b0352de03bfcb623 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 14:39:10 +0000 Subject: [PATCH 524/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10739 -> 10739 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c141c7917..4598e51b1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 13:27:51,391 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-24 13:27:51,391 submission_checker1.py:3308 INFO] --- -[2025-04-24 13:27:51,391 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 14:39:04,876 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-24 14:39:04,876 submission_checker1.py:3308 INFO] --- +[2025-04-24 14:39:04,876 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 13:27:51,391 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-24 14:39:04,876 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 13:27:51,391 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 14:39:04,876 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 13:27:51,391 submission_checker1.py:3324 INFO] --- -[2025-04-24 13:27:51,391 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-24 13:27:51,391 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 13:27:51,391 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-24 13:27:51,391 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 13:27:51,391 submission_checker1.py:3345 INFO] --- -[2025-04-24 13:27:51,391 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 13:27:52,687 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 14:39:04,876 submission_checker1.py:3324 INFO] --- +[2025-04-24 14:39:04,876 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-24 14:39:04,876 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 14:39:04,877 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-24 14:39:04,877 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 14:39:04,877 submission_checker1.py:3345 INFO] --- +[2025-04-24 14:39:04,877 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 14:39:05,885 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.402824 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420737 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 691cbe1f9..d98057a11 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1151 +1152 diff --git a/summary.csv b/summary.csv index 10c3dafc3..0ba1c3c0b 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0048,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.402824,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420737,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e316cb1a4..5962449b7 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.402824, + "Result":0.438894, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index ea81d0225472fd92a51bb65fec4a743e67b8eb76..356b768665601f874777b546babc1770976a9366 100644 GIT binary patch delta 1775 zcmVatHB$hS%Kg*B#F?G4&QZSj_ zO;;z`;s~q4mVN@2hpPXmUH?b5e9I}*47A0e)F_g8(*GfuQ2IY415mPzgpy?>lz%KE znbaozhjk#CGE5pMU%Wt8B|F+bnenmS~W8Qv4yAP>Me!gIHD&e^>^v zBpJZcFOVEDEAgIbhS&x)~D$BYCRd#0q2ZH4{cV%l?Vii|b?^-`z z%XXYHvq0M=T%tVUN$H1VLMi=_402gP`e7O1l4O8Olk>wm04`0+j|SxjT7M!k2nvI|DL*}6>KYCyJacnPi zOS}T@mY|91h$odFk_n~qLw_=eW(DPkWdKc*0W?j<59$$RH@=L0ONL z;ig$Z_|Y{BKVOUHlWe?N7FlW2+%tacmUhAt8Bcp9WRg$ek;V_ngwpsS86>lU@xwAe zCdmMqCgF#50A!kc9}T__v_xbOLMHe=qzpIBD!z}d+4uQAWFD&UqkqLG`PhDFm;7^S zT_XHw@xX3k zP3G(XnkMUqjRI(zq<6HIzEpomMp@GRAt8jbg8RcV04K@7k=CUCunvGtllr4U{ehN<3<8=gRP|UDpvlA^ zK=Y8qpTEJAtz~_cs0{b1v|O!A-p;g3+#ee)5?{JMB%>@T|9_AWP{}fqN|upSvW%qC zHpxG%14*T+gN_aI541$=ZUapg%6g;>uw-2WmiEEPKi}4qtz~^xu(YjxES2S!e$;!3 z{9~g>;!F95WRxZC9}+?=E7(6Q16Z;N$&;*Y(tp?}z@$4cYm!bx2@}M?YUCNAJ$)R^AOL+ zOJ>7y@oBvM$@=SfvhnPuk^Z&*q95=0{10svoef4;%SqhGzRcs(X8d`cKSIS~n~mY; z(fTWX8uP_CS&XwU>(Ptv7+OyHXVYZ(<7!gN+4!$z{NoRUAH3rq2HsCUH}p@9(?-7Z z`)Izp)ki4B4{TazC-eSI-0d%J#C* z5P%%wo|Mnx`DL|0O`hhnI4$G6TxBIG9!Wa;8b)6ApMml?-;E?atH+zl8WKIvv&xj2Y=~6m%XDImSBk*i6@00k_n~pLo$eE1?Pul z083`=_!3K#_QOU2Dox&x0`Ui0A~sF=?<%@b)?;P(5|f2IiDk|H&+=n_OkHlc6ijA! z)743~IKry1rJq3Mq3SQIN0F*2vp=22eC4b9E zCbdcbVI4>&O&xSJ*gwz`kwH+$gR&kg113obF8k=3?Vmj){|+awmIYR!Qt!$?+Rp#z zl>LJdtxBjwiNurg56Og5{vjEpvV#1>GC(EC0F@^9hjjo{n$#Z+>JPL;WDr6ns6V6( z&abR%IKS$e)t|qg$^-R&-s)ZNCx1V|D%_u zNd~Yqc|WWJz|y4sY;SzP60{#O2&%GB)+1%0%CfFOmEBpufnfQ~UD;ZeSjCmqyVj4_ zvK^<)EYNldmne^TQu-m8P)a`}gIrdSepm*$BpKk+fJ>9|qe1zBmVby0f;t|Q z^;j7mVk;;=x@P6)F)qJ}k5|hgD;;Eaogb}RfwSYV?vv@1kcsk$C!HUX38nKxGDv0x z=Z9s0Op*aIP0A1J0LU~MKN^f5Xo<)mgiJ7gNEsevs~A7JX5;7kka?)ekKR{)9NP=s z60bnJC1|2L;z{L)WJ0O@kbexKSwZ<>893eP}XB* zxM@}pess;k&)1^)Bpa`mMONB0_lzIArJb-u#?xL2ndDP=r13*Cp)`I-2Fa{o{ICp= zNismDN%&zM0GTG=M}zMJEfE=nkO{sIDZ@>(itnRq_I^K6Orqsa&NNb84WLTUYw45C@V`e7MB zlQ}zprpfwYqX3#F>3>Iq^aCvsnqidFaz7Lv*s{Ux9{@9jZX5{G- z^+%&a;!E|1WRxY{9}+@1E4V)_18|ZI9BEDJ59|DiOO)FO3T${LY`gX{@7@d_|p9$8D&ZNhkt~CN|upSvW%pXWh9lh zN&aCSNGeSobZn4+pe15=8)&jn)+1$rCF>fnv=2`H`L>>HE$gd-rETqFsVukjquxv8 z9~(UqU&=ouqbzCvkPu>7!Tw-Z!o@8y4{=-HAE=~H64f+qXL~NS!)5W_`)?;Px zB9nyRMW%gl`hSmbWk&^)>Pdn@Eo-at+S1-Nf3(#K16^|2$1V|nY;;I`DgKa*vZVP# zLWpDq^M_>sNj4!s(xm>dQ244@!}>vk0+C~F-Y>eYkyt2ZC!tB&y_mOLZhj>0- zG8>MIPvh-R)?deyjb}HF^sn_7{dmXce`u@dY%sc7PU1%PWgee4WNB76j5=r>}_fh|Hw9^d3)TX zO)M^tnLbC z2!GQS-mErYy#N;lkaC2X679V=1Y^hcAgN1&Lj`fCt)xDbw#J6CZ(0M+)ASxo@RU Date: Thu, 24 Apr 2025 15:09:07 +0000 Subject: [PATCH 525/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 12 ++++++------ summary.xlsx | Bin 10739 -> 10741 bytes summary_results.json | 12 ++++++------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 4598e51b1..b76db4b0e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 14:39:04,876 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-24 14:39:04,876 submission_checker1.py:3308 INFO] --- -[2025-04-24 14:39:04,876 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 15:09:01,802 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-24 15:09:01,802 submission_checker1.py:3308 INFO] --- +[2025-04-24 15:09:01,802 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 14:39:04,876 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-24 15:09:01,802 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 14:39:04,876 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 15:09:01,802 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 14:39:04,876 submission_checker1.py:3324 INFO] --- -[2025-04-24 14:39:04,876 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-24 14:39:04,876 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 14:39:04,877 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-24 14:39:04,877 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 14:39:04,877 submission_checker1.py:3345 INFO] --- -[2025-04-24 14:39:04,877 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 14:39:05,885 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 15:09:01,802 submission_checker1.py:3324 INFO] --- +[2025-04-24 15:09:01,802 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-24 15:09:01,802 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 15:09:01,802 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-24 15:09:01,802 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 15:09:01,803 submission_checker1.py:3345 INFO] --- +[2025-04-24 15:09:01,803 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 15:09:02,870 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,8 +29,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.69804 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1937 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0719 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.7139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -42,8 +42,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 21.0349 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 20.4138 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420737 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d98057a11..7784c898e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1152 +1153 diff --git a/summary.csv b/summary.csv index 0ba1c3c0b..0a04484e7 100644 --- a/summary.csv +++ b/summary.csv @@ -7,8 +7,8 @@ MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x8 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.69804,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1937,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0719,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.7139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -20,8 +20,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,21.0349,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,20.4138,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0048,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420737,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5962449b7..a9d1666d0 100644 --- a/summary.json +++ b/summary.json @@ -244,7 +244,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -274,12 +274,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1937, + "Result":23.0719, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -634,7 +634,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -664,12 +664,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", + "SystemName":"gh_macos-latest x86", "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0349, + "Result":20.4138, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 356b768665601f874777b546babc1770976a9366..08a4a2f0082db721f5d906ab73e936f33c2981c4 100644 GIT binary patch delta 3488 zcmZ9Pc{J4R-^XVNEpEFO(pZKlBFj{w5yF_pRxuOFGBSR*L5Q(_Eq5W=W|*??N=9L9 zS&|TA2u;WcWl4w{ax;eUn>^<^=Q+>y=XIU)y567n=l!|Pxz?_&U#s%pf1tT&Bk+wd z2=tZ|wjT}1Xioad?Xw;+HqblS7M^D%6UlsXO+ z&AeMCoPU&fXu?~V>_xu1i-woLNn7 zfYo8z(|2xq#`-^x3_Q{w(Qi|aEn43wdiBIHjNx~0YOrV2)~U5whf@;a5ZHof$x!}@ z?FbwD<@&TRcfj+aTK~yXSCJ9!@fTXa*G9K2==8GrWLkV1B*a>=J#v;m;h*uINq_un zCs}51ZmjTDtjPb9)l-`lj5dp7D;`ec8XB?Q$Hrpr1Zxr)$RuWqn= z{Ni_J;y^j!FZfAuR!cr9zR^X~!W6_gop7F$_86*2*h(_pXZ_N`v?a!wclaBSlMo-6 z5-_W6Y!Xk1uNYSrzk4qXs`z3>RzloigJ{}0ypwBcz>QhkJoG+R#EBi#5wDq-7x3tl zTPu%C*>I2$Uy4W&_ek)GY#vAj#BZl5bSBQ$6}ANIe%cYary#4SeL2C?;qZEu#mWU= zi?-GKA#AO+p|OfxXKtu;>k4pdyHQ`HlpXT@PR_&`c)cL4)-ZP#uv(*9aT`Kfj^0Y~ zEGDGof0|xWx1ufuvhV6!C3aYcM$Fe!g6iXEhR;^q$|++`d=J0Y7SGrgBh{aKnpun= z7>#9b7BGz zUdg?t{jy;xq)<4J-!`56t5C6Tu=FvzQ)jqAAgq{P{?O!aK#~%WrXuh13YYM5S6& zZ>f-gvO^w*b}BzFymr0SVR%1R;J9GAD905%tCc*@M*63{8x4LgJ?l8_;EtgmJnhH! z(W%yZ;!Hmjo9lRl6w6_9#7b0tU7upMkDIaLR^NJc}vM<2%sd~>xd!!+_Uw;JcLH~U9D&FT(1K-r? z5Yh5YPwN4x$efu_2~^*{dQ(ZAcoiBb3(Ajkdr(l0!quDrp-e<`mt#e7z2x|$7#z@B zb3!e4{5j?C5YmpnCwSLccM0vDaj`Nx0Y7!EseY+k0>&09ZAxwK1;e!jtth#Z05&h&B`(P3%pi?rS=;PN>tm5q5KFIybAlKqw!R$ zr1`MIkY~;*%p+leIE;sJE>kCrA_TKN#42H$Xh#QzSqP3%TsEqNs}R=@Z~yMZZVsM^ zLXl>ROJV}R8i4pQf#2X~i`A!$NHl|}JN>W$CEUTxFH7YHHXZYos~gV(ou*jGz>Cu9 zHVF^nPijDj2THz~Xj^y)48?gwS-RE9XQ+(D{R_ID6%J9x=N!WEnqarOVvI>iS0G>g z0apJT7WUkI0e*naUl%d?JYu4|I#yntW2&RxE}u70S4Wb)m?;3IE*Vx9epK0wOHwSk0*cA~MuNvT!YE&tLJ0!mcOhCtMoAiF!M{jx{NqK?cvN_cTHf!@h<)j{9 z-{7Y4(!2SivQv@CG-5ptYh2Dv752wm-f=zlg@~OM56Uh?6GcEs(k3IlVlXkTK847o%2jyrt7>)CYm!=WRe~3GKM9v;D3Q1b; z&-?K{dyBoJv%$zk6DS^a^y2x4|f%nN>P1a@C zj8ztu@@gY3RMJO8Y4VxU6bKotjS%e7MZhWi@{HMSoL^*tvSO^5b~{_agOFM|z5P8a zH~*)}w?a#J=ociNNYT0x~Se1!MQzQ zi-f&|d)eDW2KM(3j25w>keB2u_DtQ497uC%Zk1^_!}&nw?8YyPd0w3h+o{4tOgr24 zm^=v^b(OTrqiLhwr3lXK(G(;fLF^u6-;tg%ANAf2hXC55H`qhd!R$|w=(3pwi8*u9 z?O~-1UNTEpO|;|h2+YNKfKVzw-sECK-Rn0Bbr$=PrFR;Bd*ItO zky@csT&IOwtdl^+!s+H829DD^xYdfVk_{8>!rze=fV)NV|J&?76740 z{}x~7M3duDl0*BJM(OvTpQzx|&D`S0ogi+loqJMKM|mlc@%2&$ zx#AwJJ2NO@$GzBW8p(;n0_n=@5DC6h7&AMXHqnxtcAcrhiMuCX&Krg5b}moWEtr@` zA@9WkAxHFFydXI{Yk1Vy%@GFylHt_u5T;R8zt=F^=~T*x}hm zKed&D|G*oAH>HT(>dU)-YP`mcWbHkwEi|YCf#d~1a=1#~W46&Hp+pxOI`er;Lv3(B z1Yh^o=<(-&tC5`$SviT@58=>abuVtJm?EZqo0k(JYJaM3Iow;y=tBwD8xn3L@eu+; zu{qn#f3gw}WyOMM{0;!j;(@jCs!?wK>9A3@nvafSvT050nzSB+a91+Fn}lEAT&H@+Q_UF zxNzjJWZ)3SzW_~lUw@A?1iW`HvES#dme@Z(BOdl_bXr9{uOrIWNk3J* zaq5PxFS#{pc+U8p`^y`ox6m3N zgwi+P)Vd@M*%XQxWQ_(ZH>3lyLcZ_OewSmc6)nPg@%Mui&=0HUFWU5Z4iy$Rp0%nL zLpF4gS-$lGPGbHk()(1hb(maL(#ES;4ut z<=F4)pKJ3vhgvuMb?fojb)^YIk}r;c#+ zJ4s(?{L?dw=O&XAk8c@4&w>m)4b}2V^N)x5h6U8_sAZOQ1!fGWhLn_SE}A>9=SMG| zpPlNI{F2JqlKe=;D%1J?&oAn{sss2YDOJ?~JWJ|Vy$pUs5>?X&Ym+S1RAm3Z=a>L*ME1CR12(dRK2yk3o&wB z6&A{XM2_iUYkF;;T%)13to+AyeJ2im>h58a=6BB7y1(5^)T^Ss7h22oXAdMcr{Jm!d`iqOwf`~2OlM`F zDAw_f*X{+3YAVPC76y`EgJb)?#j)M|X7Tl3ZW}ARn%|fo$rwqGD zJ%H$m&xRWe;d~6##BY{zr>@@z#G0jh;FA6rPJ#(As7TgI#x| z+8XbrP7Rswn^(YB1L%bhwEhbP^^U&{YYu2Mny5OgdykFe#H^$yL0%Lj)+AR2(L}zESl#*PTn}?4rs61}Yx~>rLy;VitEPUb5Or2Hd|HL zll7m>y-K#i)~3%d0ad{Z%JG*FXH8}d%GokXdD}=q6 z^C3Kx_Q^BrU>kuBeV(9QFpbrci}z<>bfRwbL9M&l)+)0$k^`kc$yMB1*@=$lmFupo z>e1q<+`g!X^qj-rjtq2%cVr|E$3!ij{VI<-vz`;cn92M^X*IM+I`I{$LbPv;DxzGV{%my8t9M5S1$!3yrKH zeGaOar^%Zeceq8PVxtyLnh73@zxfap2Zs^hk}Q}R4dDv9FY+&;Mn6&Zx8763Nh56& zcVXSc)@%heiY9IJa?_&{*N?aEq;o}ax6r>zZmr6Pd|M|vH*?etL!{>n|E-|CEQ~mw zkJ=0TcyY-_RbglfzOSWC0xl$9{!sXl2RsIzgL7TSbWL6y++IHhLWaoy0pW0&WNX* z@6j6iK@q)_Bis)Yjw4xW)va??QdXaYog;C@K%Y=5T<9_d91$Zkto&MO4{}&TI{ zU0BJ?R14Q@UWPML#fULTPYFCl0F4vrY5-&H(!1PGgBTAKP7TRHyJ&|c<#v4rXdb6+ zMN-ZazH(q2lO2LvHQs$?P3n%$I{Sud^8!d;?GQwc&Y3K=@a@mbi1bu3c^1+$iCCO{(coE>p$O*JWBxl(BAsw8c*cRO%R%s$GC-$J%>U+4NoYcte$j><0+Ek+QZ=nDcJ4 zs=ZkKg4rZ1#hODDELtN~4<+QS6)f6?X6aRk8um=gt4f4CLAd28GQR&w;uB136dLl( zknLPCTC}NAc%{kx*;U?p@6;(l zK9v%ew01Ut@3Ux+a!+dAlMQJDB{y8ds^iNcvjX|(HG?yQx(B=X5uQY3aT%FaE)M;p+fwk<}r+8=)_QYq9{I@JnKw z#tae108S#MaGAR9iOtfzAWJHj?@9s7E^>=vek&h8?T&g| zmB{%ZF9FfWNu6L1UxJ&oqr<#2#-%Oju@=*C2yro6e+>64%H2q9GYR}30lR)h<~zkb z!jigJe+qO2-U-6norx-bcw#5U{%vS|3)GJVKI+-1z0< zv*wN2Pt`HET-R<8H3KZG1!w6`Vtz-fBtt1;xg29aE826{WRwSJC*1fZ$TX-6EryZ3 z$Cs)_QFR=@Z_o|W% zoEf=LuVu*3UK;X8R3ONm5e^lvg;bchOBf+n4W5B@+SO|liw9QL4BQo7{K)|tRo42+ z-YLABK9^uTZw~gf?J8e_J$wng9VEz3B3`Wg0Lgu6}YRlU?n25eaBy?E0Xou>u@eb4J9j5FV5Cb_K z6))(`cm_blyZPvdjA*0<<{9{PyLxwGF=2hpz+2(PdQR^8)in>Z%p>Oj^eK3)Jz!x* z=-yOPy2p&%sMi9iu<9Q5J=-eX8-b^;=U_MQ(4)bE43zz{lk^i-1JrOasUhqA2WpYW z?c{v$N=r++iu#Dhz)xsBHsD5;iO?${G42`|^j+y0uw+EpfR8cN4@p zd{{GnLihvxAptIRza&KGeqdp~_3hL<4aoc6sZ9Y1@9l0vx?+kEpr$|({R6x!G+mgjI`G&TqaYyq# zNatNH48CJ0RytpKVtHhvA2F5~zc)s+5lT5nM*|kS9~EBSDg`UDkir&>Ua0mSMSjHi z@1UaCZ%BF`550Yaw(KMW!|@aiIdxyM#3?_hN%}TZQ>~+cCc&-`X}q`&_RtN^~|`de+}q@TuNptxt!PFU;gU+IqE>4V_=* zK}gaW;%1I*xZ#%CPvbqFywFUHQKMUW{Ew;6>l$R)#YSDiL#F2R=eYTQozJd&^|-H& zU$bo+SS|73jYIB2tH+@tqt+OmCF}Sev)ZK%@7k&_=BkYlqO+IE;ud~*R@1MES#d1J ze9fqc{*8$0bCs=FKWWK+nKkS=ewE_ETH1O&^ak_rV55H!^Vk{6%vQ?Skpc2r_rNxY z=`7v6S6KYqKZR5T9166Ztv@PZx)N?WaeJ&fw(g9rn2n>oQFe4}hKKcqxK*X0B=-D` zZf#O-u~Wi)#_?dYVs_Yo+55{GT`$ZCl>7PuO84fAMzPy3 zHG;1UeoX%&Qh4xtHpn*Ns9jFR%7e=@{d?hoxa;|feig~RQQdw(+0&VHKY=NqTZxE} z)`I6Dpe9dPwlak&=0hgx3F_8&MG8+I>Y#wLY7`0sbmYt?KdRlI7ay+3Y+%&{yJ-bo zN(pH67GFDF?NVEt__E1J{JC2MBlh@5luZfA?nfI;zBso-vpz;gaMcdFB Date: Thu, 24 Apr 2025 15:10:54 +0000 Subject: [PATCH 526/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10741 -> 10740 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b76db4b0e..48532fcef 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 15:09:01,802 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-24 15:09:01,802 submission_checker1.py:3308 INFO] --- -[2025-04-24 15:09:01,802 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 15:10:49,508 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-24 15:10:49,508 submission_checker1.py:3308 INFO] --- +[2025-04-24 15:10:49,508 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 15:09:01,802 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-24 15:10:49,508 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 15:09:01,802 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 15:10:49,508 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 15:09:01,802 submission_checker1.py:3324 INFO] --- -[2025-04-24 15:09:01,802 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-24 15:09:01,802 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 15:09:01,802 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-24 15:09:01,802 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 15:09:01,803 submission_checker1.py:3345 INFO] --- -[2025-04-24 15:09:01,803 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 15:09:02,870 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 15:10:49,508 submission_checker1.py:3324 INFO] --- +[2025-04-24 15:10:49,508 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-24 15:10:49,508 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 15:10:49,508 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-24 15:10:49,508 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 15:10:49,508 submission_checker1.py:3345 INFO] --- +[2025-04-24 15:10:49,508 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 15:10:50,595 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.410549 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.412498 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7784c898e..ff7ff4d07 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1153 +1154 diff --git a/summary.csv b/summary.csv index 0a04484e7..10505aff7 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.410549,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.412498,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a9d1666d0..d7dbee113 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410549, + "Result":0.412498, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 08a4a2f0082db721f5d906ab73e936f33c2981c4..4725cee87c45e83ea375576772b7d3fe6bd3065f 100644 GIT binary patch delta 772 zcmV+f1N;2-Q}k1?atH;@JhUvovv&ww3x8=REP0P&kc_gV`9ng8WCinwWdKPwAwbfk{;*L1Nq>|2 zvwiRZNl<^-H0AH}??PFRl_8SE4+10)T>aS~89oWg19g6`>|N)lxXo-i63O#zFp zj?e$lR?*pDbhVtsjqJ-jK5fRI=YRP#R4lgH7=9kDzvQPeUyPH*IQz05z4(@)<)nW$ zO@=?NCbgW6|60aB{xJB#JN{wd{q%D~|I|2b+EFSzlpp3#m#t@ zHYV{XU#Qjcn~lZ#YlBvQ&yxA-Z*H^kG0C!IdjHdH+#kk^;wQV=7$r$|{|Q5JNw#0V zboM{9S{26*1VlR79g`&_AZlvEFc5_AeF~$yE!l}n5w-(u$*F-tQ(Afx(k2nLevs5U ze)}p;;w0cxkE@yec6L#<*sC`A1?Q#FHOe!B642ZTskb$HUoX=Kl=#53V6D+mqaHk3 zGZy06oS%D<89`=k*db1u4pu%+q4N&P$pT7Fv?8 z!^oTd3s9Yg{F`_N9CP1(g60piXC=x41LH~i4h)q-ee(^4p^;j9a$s{3olW8BE{v&MJQGuj-l3-BF z+N!*_w0F%PZMDKcm)!QTOT-@=9TH!PKP00pY5tH9B3Z%wVHrS@O$d-QsXuHKK!4Jt z{%jw7KoZm+Hck2a{JT)rV`YdW@q+-#16O}GNQO^B@<5%RD|^@ZDQ+`cjzqG(^A$DI zcySY-$CF9g7$o`ST32pc*WcT7rH((WzvAX0o{yKzhU4P*@%AU{FXYL_vztcx*ZQk| zyyNpfv{iIA7+ozVaU=UOk58NN=YM(r3>AxQHin-^>o56f%opQiG0wiMM=!o*XgTSh zO_Sk|t4S?qgnu_iy5Ee{nOO zrHx5E$`@+2{AOdZ{@S3`-?L=C`kUKqd`z+|ncn|&8~2CtqWH;fHbzO3-3tFuT$1hA zFP;4lvso3#4g^OX-QANUBp_;T!Y~ko?|q8M_ry+ssrNPEg{g_Eu%BY(IDrgkKvBU11YX@6WZ00hU-fC3#n&a)h! zy#bu^HyZ}!!l`8FEZnr7#L>O?ax}JWo3)d{8uRCPy?lKiR!ybuPsIV%rQno1V8OV$ z#Fs@pT*G`=#uINOh^H>^Ru zIsr11ODQHBM;+bWB>?~cPyzq|5dZ)H0000000000fPoW}j43_=XOqh*BL*2K00000 D7J_@# diff --git a/summary_results.json b/summary_results.json index 423bd3c44..159d12b3c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.410549, + "Performance_Result": 0.412498, "Performance_Units": "Samples/s" }, { From b510c699e8aa057359cef6b8e84437e8822e91a3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 24 Apr 2025 15:17:33 +0000 Subject: [PATCH 527/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10740 -> 10743 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 48532fcef..93c54b302 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 15:10:49,508 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-24 15:10:49,508 submission_checker1.py:3308 INFO] --- -[2025-04-24 15:10:49,508 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-24 15:17:28,793 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-24 15:17:28,793 submission_checker1.py:3308 INFO] --- +[2025-04-24 15:17:28,793 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 15:10:49,508 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-24 15:17:28,793 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 15:10:49,508 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-24 15:17:28,793 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 15:10:49,508 submission_checker1.py:3324 INFO] --- -[2025-04-24 15:10:49,508 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-24 15:10:49,508 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 15:10:49,508 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-24 15:10:49,508 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 15:10:49,508 submission_checker1.py:3345 INFO] --- -[2025-04-24 15:10:49,508 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 15:10:50,595 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-24 15:17:28,793 submission_checker1.py:3324 INFO] --- +[2025-04-24 15:17:28,793 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-24 15:17:28,793 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-24 15:17:28,793 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-24 15:17:28,793 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-24 15:17:28,793 submission_checker1.py:3345 INFO] --- +[2025-04-24 15:17:28,793 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-24 15:17:29,774 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -44,10 +44,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 20.4138 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0048 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0259 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420737 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70404 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34456 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32648 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index ff7ff4d07..17edc7ae7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1154 +1155 diff --git a/summary.csv b/summary.csv index 10505aff7..957e696f9 100644 --- a/summary.csv +++ b/summary.csv @@ -22,10 +22,10 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,20.4138,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0048,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0259,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420737,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70404,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34456,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32648,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d7dbee113..cf2738e0b 100644 --- a/summary.json +++ b/summary.json @@ -699,7 +699,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0048, + "Result":21.0259, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -879,7 +879,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.34456, + "Result":6.32648, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4725cee87c45e83ea375576772b7d3fe6bd3065f..a73dc37ff937d0d8bbe27d64551456ddc6a22745 100644 GIT binary patch delta 3683 zcmZ9Pc{~$-{KqGE5s6$eG;*t1ZsyG9C@O3u*U%ifk6d%*&e$?nB6D;&=ZGjoF>)V8 zMLF_yEMF5^Y<{cn@1NiA^XKRBc|708>;3wC9-p7r_pdhubFsH{R}S^yBe-NS{vmVk z#8SFSU3xCi$3~{goL@!zMF>=*6WN1G@<(XP(o%j^@$v#!s9RVzk=~F6N2N!Qa(x$p z==Rd-y03$(6GB?Fm7UOuwoKtQt-=xwyCENq#aSuQY;Vo#;r*>!lI+8mi*~(x8KAU3kiK((P{ge~iF7|Gv z{9=|leFfBEVel%WU-JbrZtinAD#vPKy7c~<7Bv6mY{4G1RDpo&H|8 zn-Bl-@@TMXLjCUlE#0!_>in$Jx|TH6s^h27Pi=P61G>m_-SocX*`;9!>MCHc# zDBGLER_Tj{w*8K))fi8kX2vauu!nDeSlthCF`8aRQ$&PUY4cV_wr2VZ(sGGkq?)+F zCZu_1+mEx7RtCxHnFIC;$(?oJ!JBr{2XqCt_s?y?K_9U(Qn`WRv&D78)xIX27FNQi z-5&-3tvjP3+l=u(&g;bCVM)UD@OK|H1i~E zg^c@awTtD-nDOC&eNn2~qEax|w#;3B`K`~;@GfKLfs`~Jo5k5YGks;8?}+$BeT>^J zgRGd)z4fmAu3KjH(ZbNWRulAV;7d{O7em-uBe4I~^R&{*n0ovX1S?G3*!B}tqe?i% zl)OcUm#9Lo^rIVT@cOOKgz1qpL7j76Ety|#bcmN=0@j6tiUQmx_PjJhnklrX3#CBb z??XhC#z7}ygcPG2)(m&%!_D4~nLKRQ&T3*r;xsTiXzV~DJzC=W;EUQIgXU=$*`=rs z$$ZBEuw!;>Oa}_Sqa9%oeE!kMDzoaYyQ?BBKVYW8<}x^4fm8Kk3o)$W6*sL`S{`>M ztzaxvGle^L7fYA8HY9B4cV+Pvmw9NsqJ*2=s&ui!&h}S7_%dEV%xSs>VGib++POPw zNRr6sTMKK152S1k%gU1D$}<=9jw`?3k3fVb5~@}5i6Vi5tfsblnWGmMGd-8io{N`B zhY7+>%V9DCZ@c4V6!PW826YkH@*Et3jTlKA9#BTb<%e`l6e74kO!P3?rp7~2$mWoL zQzf&N7+a0f6$?INg0j`SKYCFh&F!HF8v=BS%tW4H%17gnhjb6Duf$E+EO4_c3X`N| z%;V`#l-F}*XqVS`z0T4YvNghOXiQFpIiB$>bu?kWcE2jjs{UBA!?%;kMwXcN!Q)y< zgNyuibL{sP+hebQ6FPZ}z}?bAFn;7^91J{&+gQ}-LsaO)M6C5gTlww;B@@yl%x*ng zGe%+`kbScQ5FY}q-Mo*bvaIw*ApqK0OLrN#n-$x;B|hvl+^=cG47m6B&DWoJ{CKNA zP~W(&PP{cJIDB@Xe$Qdn*#VQ7UTA1}fjk`rz@q?GIF<_+lYEyoOnXrJfPxfB9gBHn zTm0XE0u*MzydFomaRcIq)mQ$W&q#$6;FjrYR}Gu8we68$P1fthnWMGc`xf&iig8OD-*PS2lvvReBTui!+<0*rK?Tk&UpqR z^TJfrNG?qLLZ+8QNKJ5Mn0f;x+TugFzTF$SF5BN$-{|Ha^8MW90zt(EMjCLAk04Q3 z{O2!b8vdt?YdBpe$4tBrzOojRWZlhkUvPKW%y@XV`MGoDTOR=5Rz*!@w=z?8)S!Iv zRQID1oPyi0m~ZPIj0(MH`S;N(S=#2u8 zO3%qJZt4qlx`V16#!l`O`6(m4r<5EksaFy>BZyLgt}^Rwa0KTVgsj`lX*jvOCC?vO z#;n8$((_H4SqC_ilQ67v>Cx7`?~kMOeOxxjAA9C;hxv?dHcEapRq4a~@R8XjFcfJ~ zR8!)YUU0XZE%M(V>EwSO>jR4A$mVeN)nkq_!GdF)WzYJb0j%YA07aKR>0!$hP>J=` zR72j<7lRALiJ;hiwQEckPbfR~lnsMy2{yM`iQxt|%PE+i!eNe; z{CyzKNLfh>s?3T2`}XG}ikgApf@KrSba{RZ6WIIUsJ{G!r@nnkz(wQ$9x=0dqzwK3 zX=)|KvIiBe=}bXos?;2J`ptbq@)?5|P6tj?RjRGj$Ihf)Bf>C?@S{|e@!C-ik0BW> zU_r>W!M(5BZ%(;Ze8`N#UHLRAn}g2jD5GKZ?{*neCV+)N;)cOXFS<9D++{NO+%wUx zVX9*f+v-H9w_x~|*~=oz%@*MYOemTObw~Cv+(f%^?Yi-mWQLnLi?THp%R&1K>=Rf9 zVy0guk#H7 z@q}DVE|OoLm=-6U$g15RI*iI98B~M>bVy5rPu1}LC)PL6Bz7#e%#ql zC%%w03-53j$zvn0{gy7Yv4Db-ZrkU?AvnfBqRinwY&Z$9##4z#v;;!hS++1e`He>K zO^A{_leDtxJ063>It+o7REnZUuwN#wAj+4}^~qLLZM_K^lcvL@q3i>hv;mN))MN#d z7H@^$CieVFz%X&iIo$8~6nf2?GPa4n8gZ}r3R9mQ-rhzUv;qlhkuObY6Rj)PNoEq!WP!sMV-v4Gn}lT-yUOX;NEKr zF*^9;X8ZBD(BZk!eWM+l6`;xra7ZnL8k&Ac9GBUyJTzfm{Cj-d(hr3;nH()RM{Bbj zjoEv@qGcd2^>6Q`2952ob}U6!bQTC7oK6$Bnsxl{kUoeBEVUS#ez_c8h3L+|_89Dqi$eu$tQ9~RrR}SPUbW=XQ0#0YX74e`aP`?7|&f(u{Jq@&-1R%3I zU^HMUSBKNa)*x7knMiIKREq!!}4D#o}9fsB_y{O&TqO@P@2hO+z z?eLFc^WUc|E$&1f;CF?osHIu!b97(29dv%12AosW5zne4;M;Oz3;U~wf9>dDQZmRI z;=QB9-TXty_sk%Yh_}m(x!P;b+uCcT*7$GO=SC{tOI?I*ER^SF+`rivkw_^0^BDBf z6>-|s7Lm4TZ>|vM+N+~mxF4{12NR_JsPaBsZ}7ZN%b}M?BV(>?dK3wEp)YHQ1M$k1 zRr_6E-qGDU<>Q))@7v-~`(1+7sDoWg?<3YGq)`7!42_YeB*y{(*c<}@P5}S_RDg#~ za6n*)nnyser)s3XUkPsLO78`Jv;7#e<=91#|6K%Jk!3x@0>+l`5xECxNduC2(G1D+ z{*(6W87sf$wL`p@-P`g0KTphX*IGP~Rmd>BIDF5?;^(dU#b`n8t3uVa_BQ2`XC3V| z+0~OV=o9t!;tjXrGmpWv-&m!$&Wk5DdN}1w$@V->Y#X<~2DJF(YBH#fqvQN^k7@OeC9%T#V}dD}+wfs1rMJ>0t)Tv*RONO@^;pVRi|lnSpHtovxGT=efROW5?<{CC<$5 zvtW$@By98z1Fx&2yT|SB7O5?UO0MqcIOS|+FY~$d4rDS;+eoGy7DuvehPcEft~Ytg z4jud7&)_9_Cl)7suDl-03VuZ1h@}V5ub{)Cj<-~h6aWAHX14tAicb*4XDJx4pz%Wr L=QzaA|JnQ>+V2k- delta 3671 zcmYM1c{tSF`^RU{-De!uVaJ^$X<^*Zl!pZEK|&e?JJ-pHdNn01AR%l2%mrT>?if{c;+*oUH;NGB^@P)Ss7Lx$-*`0Qxq$s6f zc2E-P)3>A!*o>GM=qhzaKe3hw-;kG5YNS0U( zXya}ZUC}zfvw*p@;LxzE{VHvqm<5DuAZ_wn(0cHjEwrvcgSE;UUuo*5g*n-Kg1RYl z^z>TD>m)|^<)M(_5yE0~C3A!La{VjTt~|q6M?z@sDB1IDn*Vqk{(`}>Km8YNa>F3- zNufYL-KRHFA{84l5nYi^W!Hr^#)LIUFuKpMve)-t*{Lr2;P%n`|3qRZfTk3TWP`dB zt<_cM!N#-B0$(R`rPIdlH$94azDu?M4$-#Ps42BHYJXai##*9POa(+s2sKWY%=fKf z&pb>$JQW~7xJIyIJHyNLq=?9zma5By3CBdWMdv4BsY~+|==7(i(b5-JcT1jbQ>~AX zEBm~4P4s^rr9RXj(eG3R5=yprN?ty8iJ}FC&U}0~YUlc@U7JxB;}p_?=*Ut0>DwJO z{*PB>aUs?JoXVh5xtHJw>%>8zwqnFt&CuGj_cN6}o#->e;Aw`D6axi!!K=D4h*yQD76HyItR6|LDc9 z+*E26_Aj^+WTWFLKB?7H-O>!iIF)>ck@W~FkKIc%J7805Y1R?%&ME$q5`j=>c+8tx z+aSqF)su>l+o4fV`4{VwA`qt?oLSFjW}%q@D}HPD@P`CJS9*MRlKPV;!4Jp0UvaR+ z9VZdUN=!1uFB!NQ+fK~{ARbwAJ*f-L#T~)yF{WUsoTR+g`DA~mBinVB>t}CSc5dE} zplfUmjaReXS&`ze*4_47^##l65#O&CO#KZf@sgVi3l{+Etww8BOGL*pw+#PMY}V7U zxfNCG#+4BIZGG$1Zkx!MB@!`=lr;ZYv)ZkoVq)rhG~m#gMB5j}lTKIWmIhKs6X^RG z4G(F>#_^@zJA}pls#4UC$jtZV^ZDPXgI)#O{rDz24H0Fgw>q=pY#$qtwpCTba?q-6 zS&69HmMdMvP+E{Aou|P|3gL6h(diJ6{rvEB$ZWb9G#o1bQUS7Yh&g!L(_*04-d%Fw zobwA{frPDhQD>b^ubYM}x_M|JuJ>fmWL_f|&JNdRk@F{4SVdT5aCT<`Iy^K4FM5`Z zQ%vnz+b+Brp|*mlyEIYt##;(m^=2*#NW7qgiI})^HU(^x-DV%gFgSQEJj@m#vNP;KtZKRbd5j{FzIfu#mzK7#5(|tv?-*K4vQ0@Sn zO?Vb+vIn*T9@F{taolM4O+`TRnDVC2+rP87osZwlKlV0|#7 zD0-L5<~0y427)G`Zu6w&eVHP3zr?A){0}VB@iPu@upf`UD zYDcE;n?rh2D6xK+w7g!ID>EwP_h_nEtv*cbhwPJ2VV1DWX+g4Vp7?ty0$3-vog0(4 zAT_~VbZs+SjNjm+NWGmfDhO-IImi3CyMc|?X-iSX5zpK+CXPmhV9aS0b zRb`lIt9FU#ir5^2Lq#vnC;AFg4t)@!fC%p=HWUf4wZt^8OL-kKJaa-EyX9@NtG{!I zIbU6@j%-2+x(qlagRU9+@~gc@?i9*NX?%{q`Yj=rD<|lIF#GA;o&`;A2b`-!#5P9uR2(#t{%sIrgh|kh z#VC=zs!0#GjA&M^J4A>aWetTPT0uD`ZcB})OdFy=F1wF4d z0AtE=>psVbNs$3Ffe<7()K`<_yWzMGwKa&x0Y(Abpgi~Z?)Qh_D_ngC8l$T z?L=RbDpsbz9rSso*YR1L?*b$&zt|Zk2twj_8)Np-lU>DPl?t<`ByqWhxwHCFt{eUjG*@{1!);4r(BSckz#~AJ{*6x#nma8!|XR z9W9|lWoii*94B;k3S`>L3+v4KY3>Je7k2)!T;kA{_cH}>=q|d~aap`?oE3hPL(@X_ zW$?~(Xn+>KoajRt`v>_~#pf+X1NNh3w1h6xhvvfRW3kQ^^UETO7I=@(3OO8Smv5SC zC0*rNO!NbxV5@Hm(HxXHSnt0mJyhLB5wU+DXcX!$yqc$b3hqJu<{;R_car70?6#o< zR4<=uUo~)8v*qM^8sb^qT5*2j%znLPc!1dK za2b1y6D)v}EES*=dvqX@C@}4d6Yd~430botq}6A}_J#a#{NC?GAZ&p#oyDJ9KYtGfIb4qr z7gWfVIWBwL=pdOpH1@@++frrgV8B^Gg#7OZr%zV%y{Kqu8`w3U%!Nlc++GH{0k^0* z`m8!uN`4~N$X!b^J$0sk@+q-1Q%RC~Q)1a;rFL(F^IHDsI5h4V(p=D!wn{ND3haBj zMAyEgN*X^FD_zsMP^szh5Z`--(IEY5mr%S|2I!$4z+b?c3)0wE`rs^$-L5O=SYETt zU%{sAPiF!bkewOds$J6T_wS=tR(+>CliQiSgIIzdY(v@O+2_stwBEfN36202NB?o_ zX5)q?xxqJ*O$+=Y)0aA@)INSp5fR2Vy^SDwbcXy3Z*&pw7vwS1AD2?ZD?z1!!Ry@` z!uNj094c*B8&5_;NZf(ss=Fo*S0Y7+VUXt7%uo8UjFZ-L&{>C!>Re`6m|gQ&wH}>k zseDLxKs#RK;}`y_7Trh>bsAsx?wcmhnjza#X@e^|;o>dXl6|&r+NxcQV8ez??mIK~`&)W=dtV3d5|{Fc5!w zF%OSEw!r`FYb^JfMI}kz#epi{?`l?8sB_<^qs~gSzo4{@d#2HY_g_XvNIIg*jS5n@ zub=Ev34arR8+E@n728Un*QF>;TMB2yd{UaWPCdaehH8Qg`wUOXG*&J_w?%Y$R0gu& z)4X4QK<0^F!PR!Z%V!<7Eqt(_kL5iPJQExLtr4S0;rjm%-x*~WFdCn&Yyh6e4=JDj zt5XsL!Um`ZheQPU1>-$c#PHfG5@1cdwF*q~PnW?UkS!Mo1OfeT#oZObXQ&u~ui`(b LDDWOr`d#@CLwG4_ diff --git a/summary_results.json b/summary_results.json index 159d12b3c..77d648463 100644 --- a/summary_results.json +++ b/summary_results.json @@ -716,7 +716,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.34456, + "Performance_Result": 6.32648, "Performance_Units": "Samples/s" }, { @@ -1076,7 +1076,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0048, + "Performance_Result": 21.0259, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 009a474403b47559b9d213f5234dbbd04190ec3d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 25 Apr 2025 05:18:52 +0000 Subject: [PATCH 528/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10743 -> 10741 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 93c54b302..6b3b703ec 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-24 15:17:28,793 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-24 15:17:28,793 submission_checker1.py:3308 INFO] --- -[2025-04-24 15:17:28,793 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-25 05:18:46,682 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-25 05:18:46,682 submission_checker1.py:3308 INFO] --- +[2025-04-25 05:18:46,682 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-24 15:17:28,793 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-25 05:18:46,682 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-24 15:17:28,793 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-25 05:18:46,682 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-24 15:17:28,793 submission_checker1.py:3324 INFO] --- -[2025-04-24 15:17:28,793 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-24 15:17:28,793 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-24 15:17:28,793 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-24 15:17:28,793 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-24 15:17:28,793 submission_checker1.py:3345 INFO] --- -[2025-04-24 15:17:28,793 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-24 15:17:29,774 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-25 05:18:46,683 submission_checker1.py:3324 INFO] --- +[2025-04-25 05:18:46,683 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-25 05:18:46,683 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-25 05:18:46,683 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-25 05:18:46,683 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-25 05:18:46,683 submission_checker1.py:3345 INFO] --- +[2025-04-25 05:18:46,683 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-25 05:18:48,163 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0719 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1738 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.7139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 17edc7ae7..5fcff9ffb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1155 +1156 diff --git a/summary.csv b/summary.csv index 957e696f9..869ee61e4 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0719,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1738,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.7139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cf2738e0b..9c1b7c41c 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0719, + "Result":23.1738, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a73dc37ff937d0d8bbe27d64551456ddc6a22745..bcaaf5da3be5c1554d0d646195dd2d02a95ce62a 100644 GIT binary patch delta 3664 zcmYM1dpMK-|Hp@#4*JX?`XH9tL^eLw9Ew@8vdNr6mWnbmOb0n@w^SH0Hmz8NoH{7X zu|$aou~-dLDW}RIIb{x;?{xjH>v#WqU)SUPe7~O0=j*=ig`@LFOD}F%uK;=*@{$$4 z0SsGLT1{(|IELz&8YhV%7Xfc{V}Mi+#0o=7wo{98mP0}l$C-W z=f#T&sANZu*I>nBdmrFn1y9EE`^`HwfJpiBxr#`R@9a>Rh1RELK22yhI*kCx8HhqL zVm0&v-{E_54fcd8aW|DpIu$L1^0LOYQ@6P-;Sc9c@1UhCu=x?}Cl;h87LDzQt<&Q1 zNBdgO-@&>U7nLR27ImB*6Rkiz&HB&oeq4sVG35BOlb3QvAIci$4c_W$CjpMeYy& zl$y|3Q}e!~**nv(b&fG_6TokH?i$HC?)#7$+Ut@!LaC0hs<$5`5~DF=0r5slq}2&> zdQ1SqeOLZ0ZTj5@Q13d0YzaGE?xGY+ON zZf${vX$Ti%s#x7IE*hiWle}-6R8kEJ2|K^ghPCLT<@5!Z@`SBb8onlF0@*X#UYRQg zXD4qo=2bIvzhJT=R{aOB>X%i!B$Rx(5`}b6qB|AObv4!6s!SOJI@zvAq0t7RKkc%g ziS8uz{0>o8;w|r^CvN@|hg(=mkKVIE5GQS}j{JF0s5h+-X z!!<<#aHX;@xBhiz13yNqmlYPbO{=OOe1Siu*=Di8_&nvbjjOX$B_krt*BO#Ld~Y^> z&cN&TEy!Nz-MXczk&d^9hmx3Qf=oWVIxB%}1Cf zZCj)x>pKV0wGOwSMMD+lEBQU)!R-NOlg&?Ql?&fc9(_*f1kC1>%{w5*l<1JGeZoI6 znAaJ)VpuDCG~`#zu!H4Yp7CHZFxzP?`zbwfI28vAhwVI-`*IOHfq&-F z7HAVG-wxdTq-GR0{s!BkB(S*uV*H3MA?5arCJOmX<;0Q+Dl#PcGj!`YO6pAbf8$S6 zgD>vVXsQN2q1{G;Ql| z-gg*uNI6}PnTSir!IG}X9C9&qMrVYVZg*Bt&^zOb$@v$*m5@@ibnb1%Fnt^)TFCKe zaoLk2(&_D2quMSzv|d0DBjVrQkI=~WfhqFMTQ-#9cQU0FlEW{Z=5Z2HIc*$27x zDw|vl$u_vdv9qin3fk*b|r*u?TCw^wWW`vw@yid=3#r8h~?t-ZwV39$@cZ%rBjYNT54&_H~`Q_g-#6IOR zXGJ1Ou*>(n=gm;$IWFM!3-NUz*-tC5_+nA$-m=MPyPhzN+iTs`oVm=`Jm^U;u9TS` zbOzi9Q}k#B1;GmxuOdAeA%^(d>J#bF~c`sVX)#RN<B@6FUDi$ea2eX*!%fX1}d1s#u|#sPL`or=+nr zrY*e+NHuX6nts+YogI3fH^H^2fD2L&YBjNh|Fo{_?$=SrmB&46M8)ccgoB8z7dFu# zG*OZ&dqw{EPSZ{IH~s_BZ=|wFmZem|!|50!H00yB^YR(SKgh@S4~|CrG3H*)pc|wz%wjw-um5xl#(ijLC34ZKZ<_eW*Qzc20^ zja*06QHjCaln)TSaFb}?OdYqN(SwWPl=pNgPUnx_Y1!LZ8@`;*y(Ke;}>bx+Oa5P24!K)F-&qrz0$J<6#*D zafSh2T=R+8q&y6D0q^z@nB4IW?U>;WFh;IqVb9G!T`YeNs0K#*_7@m_Q`S;;e1wIm z*v%)9)7m$ULE1sLV7nNl{R(fGmp~p_P1%!IAs%re8?ZG=tt~2FflLyZyyaSKHdjQv ze1`OFSAN{fjp|vRK3?_b4p-bQCC)232CNY${}P3NiE<~hilG9T^AsixjhEgQNVj%+ zdv&ciM}yY?wCzENEt^NojAsH!gIkNe z#ia^q1k`~-!qRCiP^Rv8+pDv3NG0`aX&XDGvAi0j`rWo?VkHKOg19YpEq60~qH{fG z7S2|9#_6{0x|czdy3q{*T6bir=0n*sXae{>dLOn}?f^f3#ovi0P2|;_)HNgGIezO$ zYgBbO{EZ>A9>|1d7C;X0)m&s4o5rJ)Gmw`hGSZ3myc z;{V@Ynj)$}q~lvdaW|%2>S*RI=*&9ZgaSwjn=zSC5bk#*14@**GF;a{ZBkH{YQ~zZ z%u=~QC=QH#Gkp*-2J}b!nhLstm(%-1lDlfheRP{7v@Clfse- z?rac!A+I(2SNfN=h%EoD^1@n`%fqBqUXVrvO%yT!@Z{f&;8ZH0F7f?-s*}>2iesQ? zh?}I#MN+nc+U|9>kbhN3kaX=>J#YjE zmT^{l6ji+GJ$Ve06Nw+=4S2T+xb=3f@yo8qp9G^&QBNE>(Rw1gR&s;PUxDylNBQK$4d#f*J@t zQgF9)7*lxi%xdqCbkSM|d9+WPG zGE@YMaPYS--^yHU3-ogHWC`=Ih8%XKTjV*)R2EhkaQ%v*4Hvgk)}=FDAhMBu6-OZ( zb(pH}q<+Bqu7L=wCF%5F#4(<9wZg2$r}-HG@8#yTsNj{c8`3P#&!4?Y7QtYU4~x&Z z;diqKP0QmMM*S9E&U;g~nN3GpX6C0@GrgXMj#L4m;yTQ+F;U}^L9q7VVnl0@xuHb+ z9Ke+vycc9yOUVbGyc}z>uv)e8JaqPI*T;*>W?Z9LQH45i?>o<_tk=&+6j>o*W;e$e zJE#p!v`5Vidd9O^l0LM$q$cabMN>(j`l_h$tNfHG;mQJ^w?%w(s3P4+Y?W0KY+^Sp zXTuTnwY~sU!k3dd(+H~lpmV*`;}J8>n*?!zDeEr^Ql+7jXnx&^=lV$PyUfa{dzG%jk`B44oA{)kL0nZurx$&SaSP`}^p~6E73Bv4}n32YbwSy_206y05QCy#984 zY(@7@gh4D;JB~I)Jwg55Jx>`DD?`sxW{B0LXT9a6CI}P}WE65KGRQZC z<*mP!wO3z5#+c=xudBH>W?dN&$VnCiQUd+|6MIFO#n88sIm_zQ-vw6Rz4qpRNr3yp delta 3676 zcmZ9PdpOho`^P78C?b(l42_&+)||~T+fbClMsgY&lk*{`3CW?1Eptlb_^BM{kfIR9 z$oWuIltVraOD441_^!Uc-*x@2-}}$kACLQaKcDygdR_0G%X^m_LU|9jbX5*^lcIPP z@PT2oZzNMYOWeE9vPOp|OWj_?;-ZABu}OzRit~mUOLDTfszfE>3-nC_mv~Ruyo>4s zNSTqlaO|^^sk+Yt8snn6jOFd{@wQB{Ro#MOsNVXL<-KLYDAsz|M^YV}4mg5OYtU|*(6Jg9suTBsAwJ%}|NoGw;dK)?oXm&|3Y zV7E6gy2C)#uM?==@`JrnpUM~e^H z@y{gRESG8Bv0^QQ+L6XDGWv9$q7!C6mSM7O#-~c|uIj?_p3mg(!b+et-^GdO{7)yP zb*}gIcwV{xudjE5O%vu<-!HkARS&nv9k#WU$yR-wav#0f)ez{exK&m4!g$R;Me`T> zU;~pA=db2u1z%w;6xjEb-;!~E6q^eBSjoHyRg_)(SduJ77- zfpOTDo@o7-6_EUcy)fb9XEqL}^LpSFj{CvrL<_EqojGZrS4Gp)oJRwMk5|7mmtSDC zccq>5(~&bGq3ZiWqqs|tgh7@`x5wE!td!Iq1F@}c<~IQWFoGfbO0fUm9KB% zL^SEru5rb$v600ZunT?IMh2pO^CNj`_+&`OtZz%^Cx>>aVtmk=SV&=z*Z8ilPFORI z5p%W#DDZW#90UE`ff}a78ALQA+ysd;*WxCa&-Csz?MD-#czrCPKZzA9eR<$%ZHRI6 zl)J)WOuI~;OAy#4t2M42gV@%KG7dfSV0eYyb?1#0agJ|ri%=_hqJi-7`qB9~&d7=@ zHp?v!JCc|2NR3Rf_ML^2Mc#F3`?(zjQpI_GhJXb53ZDi`vY_MHi*JG%Pa#$egZv0* zD;>R@ZA~;q{NvU6Rq{JZ7Poa}@e#FYt=uDOFZZHQ;Ys9b^}KTNU=dCWd&A5Tt%XdV z#Z#vf<rCCLE;RF)Dqw@9Odj2%BHqvHI1mJS9L+7}_gjJ2!r zRuQ#h3T>!owwA|NV+XJi z5g7q$fmYx$DtiyS^D08H? zYY#bRu9C31TJY+FUfr3oUKYo%X+MzOby`<>9;(gF!|4 zrcea;dys?&>EkC$XyA#W8Xk{{QFFiFpE-*t3SL$@Px;#IroDaIa9+H5w!eX|t768p zTG>)vptMgu+BgQ;HT0R)mcj24k-4)`oau_sXy1@*rTG6NHH*)7nzI!9@583}?;N%> z!j0v!g_Ihrf#4aC+ahI^2`okTol;dV8s4W!=+0`oLance+zP!Zg@kIIh$h;;l`R5s z;)whWR4z%6iF1wA7ywk@Pm0D3%C7Q}B9Df{AJ+7-X}v-kMc-g|hN^Mcb$q^erO(7C zPwn|Sg8_|pGgmK~(xj>3BU-kN>HZ?eZWx@&LmtvU_h9qZ*M~7i{_Y!N4}Eg^BK$`-8fD&FsP~fm1*u%)G&u=OQ_`m% zaF?Pz`un$Z>i366fFeaa)e3QV<&cYfsK_W!>Eph~09(avK;gL$hJ;dOOj3O{-9(_| z>A-yXcu0Jo<`~Vv&P+j4{y}pbTrs;)kSd!iY5^p3sH9INITT`mvz&HVdi=fS5RE?q zbBIx512Gl+jTkRM4|LPaWohdJ1WM)~+O`vI-8f5{%X3B=Kh(fwJqb5dW@g(c-31a& z)l_w1YMdx=KwloJuo)OBQaZlGQWCdTJ$7&)Z|X`%Z`sOgOZHQ3h=lgp{r z-Iz!nHyS2Wz2=DPFFpsE$NS}xEZ`Jfz1l{5^yKYJ!EKD$>OnTY2^B1iB z-`4Fl<5MpcG1*o4Djy|hakF?_&DN0S$+g+r}@AWpCreI$@X1B zt1G!4xgSvKq<|{3TtNJ0Q?YES7rJ}jQ=*Ib%pkFny6#k4T6T%aDE;Kv94t&uzi*1F_jY>h^gU^ zY80&ix;6dS7SEgS?SJA!%#Eb9PZOqVyJi|ud(?1bzeRj6XQS8qoEFopMl0>gsM7E7`zHo*ySiR+jry0GWID=WXVZ~QY(n9EBz_2K9zicZ0G zzCtBJ^rc^N1$Ib043vD$DLVnhJqnUw4>z;n7{Hc9FE?e}B&VI?iZE1KZve6TuuifF+<=>@=1T zJ~TVBXS!{-3{+nR4r+$c!_%4NWAa;-Omp@Nu`9?YcVBda4bnw$w>B#>*xus`mw*EF zzuwCZm^l(0ISOr991uYSiy>t*)`!AIX}nu_9mKg_ZY47uyT@C1 z8y#l(tmnLE6?O@CWzsciK%zZ)5)bEXtSLQc;P*gV9yHqi5;Qmxj2*}0M*q9OC7N@$ zC^D+aIt3uEATTX?0*_BoH&DrTxG2o;U*MX%N8YVtl-D|m{|g3L_X4G)h21k?pmH_r zezuT2x|``KVL}w(&4xUJZ73yHAXtAM70^F0Tt0o9`S<=$18pS(sGQDtC_MF|8Qr9~ zL^bvuLs>&Rpe=r8j*(G_?PyCMS)_sj`H9T%T2=hh+VxG^vrB!zY4?zAp%Fsf+tkH{ z?dad69Wgp)amMyED}dz)o7-XlXI1p2?$nVB8C0lL&q(=B9ux94 zGlU}U=RR$v`O@c_-fD?0=?meBsk+}{CwYqi<4^dWg%^*8u|AH%&RvjVOm3DlHk_=K zEfYL?^bHF3f)=jhL$n`M-bEM=obhjA`g%9+&z4S&puz5}C8!jTq-I^U*ZJuU%dF3y<_cnz}bWr?K_5N?B}W~asU8!hX8=%0000Jsia14*X&7!?uSWjV@{WV$PXUoGdh8Q6>LnVLiE6+>S(|lU->dl zViO-W0cUVXw`VkoYgu}j$Y0VSQxonib$#iw;Z0+bbibcr zEIPDK^60ty861y~4D&Xbyl6!@-WF~` Date: Fri, 25 Apr 2025 17:27:07 +0000 Subject: [PATCH 529/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10741 -> 10736 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6b3b703ec..63d6bc74f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-25 05:18:46,682 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-25 05:18:46,682 submission_checker1.py:3308 INFO] --- -[2025-04-25 05:18:46,682 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-04-25 17:27:01,935 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-04-25 17:27:01,935 submission_checker1.py:3308 INFO] --- +[2025-04-25 17:27:01,935 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-25 05:18:46,682 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-04-25 17:27:01,935 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-25 05:18:46,682 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-04-25 17:27:01,935 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-25 05:18:46,683 submission_checker1.py:3324 INFO] --- -[2025-04-25 05:18:46,683 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-25 05:18:46,683 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-25 05:18:46,683 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-25 05:18:46,683 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-25 05:18:46,683 submission_checker1.py:3345 INFO] --- -[2025-04-25 05:18:46,683 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-25 05:18:48,163 module.py:5259 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-04-25 17:27:01,935 submission_checker1.py:3324 INFO] --- +[2025-04-25 17:27:01,935 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-04-25 17:27:01,935 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-04-25 17:27:01,935 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-04-25 17:27:01,936 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-04-25 17:27:01,936 submission_checker1.py:3345 INFO] --- +[2025-04-25 17:27:01,936 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-04-25 17:27:02,939 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -44,7 +44,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 20.4138 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0259 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420737 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 5fcff9ffb..31ba99e92 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1156 +1157 diff --git a/summary.csv b/summary.csv index 869ee61e4..d7433237d 100644 --- a/summary.csv +++ b/summary.csv @@ -22,7 +22,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,20.4138,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0259,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420737,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 9c1b7c41c..f646bc0b3 100644 --- a/summary.json +++ b/summary.json @@ -699,7 +699,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0259, + "Result":20.988, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index bcaaf5da3be5c1554d0d646195dd2d02a95ce62a..0af5a5a12f6fbc932a0a8210a68392a3b120798a 100644 GIT binary patch delta 666 zcmV;L0%iU6Q}9!;atH~%ZL#yy(QpX?G-*EE~&xcE9!*TKFc>9y}_wi)o*-az;YyC|>-tqY#+A2C5jINfG zxRHIC$EVHs^E`ioinTTy!_TAjclHTlFaeo*uir?&JW0WM>{V&BO*?#-d+5Z5uSrx|)e>55< zK_vkI08jz|01*HH0Az1tP;zf@b1!3Wa%C=fZET&A%TB{E5JmTVMdW?rB&8HtNh$&A zib_ZnQL(e^X=)KaWE;}{J#JoYrLI_x@0~MuMlO~I*+jo!ozSX8S(2g%6w_R&x zgVEJ;5;w9h^Z2wGf1c;hP_fu%WB7Tr{*s@@d@)WIC zL<6^_?To_GQI!pHtrAOMe&>6Y>bj5yZ@!QB-^iFI{P29RTak$ z1-k_!9ww6^Bp!dwPQx$|gzr2>IZK1M~R07l!m5>MmakK1gY7zg)Hl%%f+$2p~ zsV9!>nf-Qll8;ackaCY*RHfIl9Be&cOiWatAPg#cuT zds04TPtPXx8dZ6kFXD6_=ZiXBkZeij4`0WTSN&(89EbdycmfQ$uRlTc2eV=&$^r=t z0LXK{3;+PalQk$;0a=rcC_Dk Date: Mon, 5 May 2025 07:51:26 +0000 Subject: [PATCH 530/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10736 -> 10732 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 63d6bc74f..6b7f6cf6c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-04-25 17:27:01,935 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-04-25 17:27:01,935 submission_checker1.py:3308 INFO] --- -[2025-04-25 17:27:01,935 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-05 07:51:21,149 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-05 07:51:21,149 submission_checker1.py:3308 INFO] --- +[2025-05-05 07:51:21,149 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-04-25 17:27:01,935 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-05 07:51:21,149 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-04-25 17:27:01,935 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-05 07:51:21,149 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-04-25 17:27:01,935 submission_checker1.py:3324 INFO] --- -[2025-04-25 17:27:01,935 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-04-25 17:27:01,935 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-04-25 17:27:01,935 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-04-25 17:27:01,936 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-04-25 17:27:01,936 submission_checker1.py:3345 INFO] --- -[2025-04-25 17:27:01,936 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-04-25 17:27:02,939 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-05 07:51:21,149 submission_checker1.py:3324 INFO] --- +[2025-05-05 07:51:21,149 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-05 07:51:21,149 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-05 07:51:21,149 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-05 07:51:21,149 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-05 07:51:21,149 submission_checker1.py:3345 INFO] --- +[2025-05-05 07:51:21,149 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-05 07:51:22,192 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,9 +30,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1738 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9773 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.7139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.998 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 31ba99e92..ba3343d50 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1157 +1158 diff --git a/summary.csv b/summary.csv index d7433237d..3ed0951a1 100644 --- a/summary.csv +++ b/summary.csv @@ -8,9 +8,9 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1738,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9773,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.7139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.998,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f646bc0b3..0f647fd75 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1738, + "Result":22.9773, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -339,7 +339,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.7139, + "Result":22.998, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0af5a5a12f6fbc932a0a8210a68392a3b120798a..96546ca1eb67df2d2a8ef4f93b8495df04dbf515 100644 GIT binary patch delta 3560 zcmZ9Pc{J2-`^QJN2xBRd7|WP!iD{DJYto1rOO_&pEMsj@DI?p*QucL5BU^S-NMlKM zNz)i3OT?6=v1O|-LKDWL&hMPxbDsOp`#$HouJ`+To%cESzgL#8)Z979XY(dIyHNlH z>L7+3lmmPN81Ih7wCCUG)Y$Fs`&i)E8iK=%mU~BW-_FbIW6j9P;>seFL@%%XNaZ@( z6)X*j;(8DX z2{#rW3McI~HF}q?U-|;6PADTbG^oA!XC>8>eh*AZR0)F?4+IIfqj_lFy0nozH6{j0 zi_M=%nfG2_#%lLLROg0l-xd$eoxT^!j;+dBsU;4-L?%s}FZ6v#c2$88XFf19KXnZO zW!46t)52aKdpBF8nPjv(3vD*mYfYe>ePtKU_)>&TGaVdmPhQYQW;TsI-bChWt$0ii z1w9@Ymp2h(%{pR6TV_J*-P5CWXzu@fnti`N~{ITIYPMNY+cU>291GZXL)~dJY z^WfBPyS-&>dZ>?@jS+f&mziZrv7avo+%4+ZP6!FNoBqeDze^os=_b_T@gDhgKN3>T z#=j!Zeg3ryrrxHL{cXM(scxO{F(sKpk?>Hj_B%^!Ve5|G&NK<{g%{N zPBp#!CW_bCR6@)(*B@fm5`tJ|54`2Q;_6=}lwZNQVc&aLtnl$daXda1HsJ3J$K~A*LkS?ldS+zkrT_@@dup&MG;Tr8o_|eygfZ`YT-zH+_nCjq9WyO zD0wEUxg$>8^Q^qc?Z9TJe4k&c!P~L0{rhDnLY#JUJ28%OwjZ+Xqz4IX(le8Nziaxa_Lau!-L`;ImlWITDQbuI68p??~liqEgs6aL) zabP1zD?x&laRBk*G~StLYt5 zn?o!-BnGtQooAF(^HOn$HgEQqJbofA*`=J$0@wge=B@KE{J3L;eLeZ#)_w{hrLsh6!WEG9bdC}wsi zjt8&KYrhFL%D{(Fp%o&ia8!)qwJ4KzLm1%i_hLJmd6I6)7V;P$9x%H+vtvS1<9YQ7 z-!xlaS=?#mU49_(o@wcU#2>rT;&}zL-n3aIp5xsh%fla-0ZG4H$FwO23()eSmqgQ* z2yq6Q8U^Sc#e|qOA(BEuB*&#n-ou>n@)aVUdxDarFSi|e_GAt(JvQb(jN}TT6|DeX zlgeW$F}dBsc8ATU1CqF0kIo;JlUS>&FWfU5(NuJew>= z6x`Rcr=>>DI9wUc>TtQhs*w@DAT%BHq#Y{Z9U4|{uipj@KSQeTtvQhSLyPlz^g?}$ z#^~$2FTt?(MYicz>%dQsoLyo4qNaE1WjdgeAR-oI+BAYL-6OskL@3_=M0bGHGkSD<`dKJc)?719P;(!s{v zk(RiesU!AltH_RW{J8_*JuZOTell3^X#S{%>Yt%^C1`J8Low$u-*GC;Bh8G!p@rcY zh2rt)W}n}}mN-TcKl{h@esz@L;zuoKeBY>NY}iVw=Nx_GKEyV9S@EQnB zKPGU!ZI=H?Kqy4O&WhRwaHuuqiilG7j zSgbIee@=)7%I+QKK$xuwt#I@sSUUV%GG6;Kh0SgSq4!1hG9%qyc>e%8LM>Osu8HcTorE#afOF^fji1DYnE1>uOmot z$GLxUJY9ULa)C(IjqspreNgZM__L;#Ds(Nn=Bw7Wp5AqsTs66?b0oteJ|^zONeF?j zWDW&KyK@i3xbc@N@P%dVx&Ov=p;!#h=M7ag~amTbOl2lA1gahcSib-nk}OqEt31bsEt zQKTBDEga8VgBU|DD@BxI*aNbA7fjy9#M?&gpn9BnX-zTGmDP4^*7i$Xi}m@ci8Fvr zrt8TUxYr*Q9g0v)%tkEsVg)-*;EtXCu8ZUpK)}w32j&#o5`;h|#9y^x|8kfJ1Ml@M zNPk@Av?JXUBro@I331W<{b(cbZk2}0iH-0UQq@1@GqCsulSYHEw?qb*`oL;iPI87E zwb=#lP^5w~VN0?1Kn~d@5WzukoxzN((u}@c>Z7mf4N*UfZ<4}grq^##H)JtH)R@J*>>LpT%3H5X$DoS2OEGK~WVI zUVUkBPjIDx1f9Ppwc3uwvPb04Ob^4U#))%J*4f)06+xVIple_boUG3WX22Rf%6;iaTOWGv6b%?4fNL0NT)QkS!-yKyZgXQd6kF zN@7ZV34kpx!T1WVwA9R>vAj9W#e0HF1ZuF3oTSZ8Dss{$qsl$px~8rwOzlx%j}|#F ze#foYkZHn-#bzk3KtwP8ADuQY0UU9+E9HPg_A*3j`L9M`mXe4stfcfz{R89c3^CoF zT87BcQuv|6#GNSnIh)z4(pFWr*YhJhO#)QW9~8rfZ!;&rR3(!HB=iB=Mi{W=NyBm; zfSS#Lb@*ky;lO^j%ks^ErOI}+_x{GPJm#nDg|!ZPz98W%F-_NEcD`ps6obKr%pdY8 zTLqXfz0?!LI!*LiZN2)JhU)hEd1Gn77U-m-bde5lsgy+X=@s*l8x&%)=h@_+FE%80 zgDCTW*Yj^UjVAlcOv6ognS8D359>R8Dsv2+05Wyt#_G`HQ}3v*xnXvY)?X=wvn7C* zSq!q8r#?4%&;A{ft?{cRj!0>C#`3f=kQTiT3#B-EMji zNei~Bl>c;iEhoAAw@I`O(vupxLHvvhH6^QMP3#3qZN;t@{i6G68Da?IdsexGmCRFl zyWm>uBNQR-rz?h)P@)_Z0{C=k2w&RT&5ZJKhpqwjppc!THhvo~ih>z&t> z`rKf~=yn!lo!+Ux9|kc@ezGQLW%*>j`nDgy?suO%u=8paTEoRY4 z_d1G&Ba&{L_sla{r7}*%D(36%xtryeA@AV|_ZwEOI}PVY-tk%UXYy&x)jBGEIPhP; zu`^27IIM`tO1d0hh&@XB9F0T)?|4&5ZA!3}e9)}Z= NqO8CpeB#f`e*jkI$w2@B delta 3563 zcmYk9dpy(c`^P6m2Q`Nz#G;XEBWt9XC5sJnDrKFVMuv&x6z%O4MvP4>RL*o*4s%W? z5g`^bj3}X;K0-2w&3F3e_q+eSACK2{U)S^Xy6?w5efIa+(i>a1y(X7S)=7at^^CBs zYQUuc9#<)LcB#bq!PSamdY4f?V?{ver5xEjWTa5AP;lNB6Rux5F4`zfhX%Ss1@gXg*#|p!5&s5>u?YdI05ae+oRXTz3XACiB{xisp{i!B6@Q zCdW0@)O>Dla!&VX`9oVY_3EyFc`h{bqWe>FV4r>R7^ynM_>I*F9v_C9^orG4A*@dm zQ^UPr=MUsBQ06{8aeJ}Q1|YO90y-IqwUx(rURc_h6t422r_Qk4h8KLMIkWK+VcuH* z)%~52ASK>%com~7++Jzid4}_SyIk^-LY(>#WmJXAUrk+t%1)abr(kQ6rs2KgZIwCV z8qCCPx}0j7<`60)WZiQl`cPT5eO$?x$XoF92~=CwpU%cQbGcbvKrQRsSxDHCzzK`2 zTK>a?-roVTGMv>D)c75LqcEFG-_qmDA)z?+sG?yk=J%fjOyiMXdenH!VKte#wJXk; z#OKPX@ii~6$jV%(Hdh~+e4{4gAE<-FT>d(YYy6>OKzsYm=-0b@ItF+7BNHc@A?iCh%*#dR1y!VNPE0^egODHTv|wwyB#=j^zlOM zpCe8W?t_m(9=%?f9V@xeIW_cw_WXx!Mpm3}Q~%7on|F%UKsQb}OrmgGfQ;tE&czV@ z1chZP{7uIQvd;QGr07S5!CHQAuy32!wM2ufD&@Qnq-S4~I(8giOf+Z*>yp9(G8j2K zaxtViR?1Ucc#v zm$<9!WLg-?JF?sk$vcM6&X)e#47a(YZLa%GOCmluWKWh`qryPb2{X_c*;FlhJSG(b zO^6gbW3O$8ObagEXD214<%-JQi`|V&s#&@Iv0{`uh2Sq`J2u-N&gQH24eTKU=Bw5% z{>V{S?8n>?r7Sn7bhknC)>5oGT_g!}#SrFq+ImK$_OvZL$$2BU{hT(@6k`^#@B}uQ zs=jPCvD0JUvQUGiP(uPz^355-XK|d!r5BvSKv>D9m=54t% zQhVDP5&3GT3F4hv8&;ZNMhCEnR{H9QexqYP3cDANx>*JG^^al3C#hfWxS7K;ZyiN7 zA7^>j6qS}oG(^n1pkWdFn&ylEoOTy~#kw*o+;Yte=1p+fqCHI3tdi*V4s|IFm~w?! zQF;aG@9UWotYb^+iSC5FpJK(1Qo{s2j?;&TLhXl_?ld(LfVwBK{Sm2Ia7t zTb)awYJITaJ$~`d|4pZDN#}VzJ(x{sR(kG}o=JV$-v0Caw=j)aHTm`FiS+lhfXg@mo2lC#uK;)-2E=y3xr>1JN*ay zHk|08;?24N6a^kDn+db%4MI7**Idv3lm4Cqx#GkYX{P$PO7ue&of*Uk{QQJAo0Qo{ zM1eA6RZk8`Li*C+GRkKtR(*Hasz(~ga;7z8-ny2455u}5kX6=IMVj@^bZM&|qz|AH z0QS1ocgbP59&W!7e`Jx zx)qSS`-k<#%~#zAJgAAew%UHCEt(#iY}iyq;>MBaV*2T)#C=|B>Jz~43UlH*{?<$C zyIVGzrA}HYwBBM#RpX_;S4{{T`oddNtCIE3^Yp)}=r8lpH^vP@OHIq z?g9qX6fxydwGD^}&46GKp7F{w41~lBlQ%_5UR2lLp1W-q_)r6xK`<&MbDz$I>mb2B zQ+BJav|aG$R!`1`dC(T$X?m$WJ{N}7Oz{P`?9Y^+R%MFmQ(L359zT$R@We|=^zgKE ze%jd@>Uj~yAP3a>EuM&U$ijqCtDLt1z zW{QaX5{{L4%G$qDDrJjf?aA+gu6|aHuwcyQ?+Qs%92%sAn6$QEYDaC$6T`!-1j)*Y0_GXlq;cr*+uakonZK;5$MVnl-h|D zm-0@nMKa^@XNBvBU)plr-1&PLcZ{g+?LyAH8^!91Z_REikKpbeT$RE5XQc&Mr^a$7 z(O$jlcJSdFtw?dQzm{GC^zk16tC=PQw}8@M2Xcst+ca9HZ`pWjRkp0V;xYVgPg$}b4s@PwJ?_fn03{?UZU+pRVc*qNjQcPka<1e@+ zrc0=%*vO6&h&XA8I9W0T4Q666{5(x!d4m(aQFat!FLBV3tM|}3uzj90OdE^LKwn=h zUM_zr?;YwsP@w%?Rz=q485$~Qu^2~8Y1=*tZUfy1pa*EB15zL8H$jdWjaf6%V8sL`jn{_c%s)xq zh9p${as~^+74sv}DM+lS-Bc6XkI@@ZIqS1!!n_wIFlrh*t~>8VAQmvBri&Nz0+09w z!ML{(j1Mwe`xRqSwsnX+7wsee)4X?jEgXUXJFIlBcG29zaxTp;F?PxxJ{+B$yfMJo zOph&N=Dn=7aCnBz{M?2jK7g}ac8Rym@}|+uy%d~~;UWtKqNNe_d^PtP(s^CV&GshF zlG=jpJqTRLq|L+?1bbMfLGZ$Jv~v^@yjGEiGvw1COfh77sd%slz7?#zr4OlyR=NxE zr1l}}oBp+Og%M&>>dh5ZMrGQZ_^kSbA5akdA#3eLVaYUm!H2q(*OK*L>d*$cRW)EE zVoMtlD-RMyY)OO;iRaNIX8xhW$W=h>V+TCsXGEO{!XZeogRs+HShj{ZXmkG~S_v-e z)QX1Y!AR>V((eiNLKnpd2Jsu#n+`ZizAMBDJNK`jvcyP;+3k6Di?#hTaT1&zik;*P zJGXM#Z!FHmuAT!f7IQTq*LWI#X-P^{of!z-j`;<^Zhb;j#A&o)4-?GnNtx3SZe)Jf z7n5d*Rnf76`cI{auD3(tFhXxc!EL|4oN7IHZ-SFNjO-1uEO=BpiYmO~y59FImA_$4 zcbp(DM!IeLipT&=hd+>k{B=>PEEz(#XrVY9kjX4uTOAPvwG0u$z@7fB*==kfFz;jM zZNe==>$91a4x!gcvl(cf*PTdhQ#NL;tW#~SfNv^#pT@!4)ade`L_DDVF(4dcMKnAp zev%`)@t`IGm+mxx^LEFE6|l1Cttm!+i`Sxwd?*zBWx1AJiLsaMA6kwDGDxp#sGJ`y zP_h(#&FjUE>K9WbGqv{35bm(wf;1=2(xbtD?q|pOh`3pusR6?b`aswxCF?bC*p6Jisd(DZPr{ILBc=(4?*z&u%C!bk9 z=hVh;hDe}8n$c3PvK%PBzp|8ylETN9I>Q5>+a*ByDmtvk1(`JiPJ?%^k=$fy^LH-7 z-}XDQ-s*WS1~5NeK-ac6HACCz6N2m2{=iixt-HG~KY59PWIC+HtN(mDQqj5H;^yW!X3v#+DHq=R z%pM7r!na(Iee%(>xv!UEou~U$p@BHRC;R`axc4Qli`xg-{>6L03(YWg-uC#$Q%?1+l$Xv}|NO`y>2K%O0px3Ud8q(yK|d_}J0W;YOLyKg$UkoK64lL#C^B zLCk?c)iM$rW;AJ;h}AK+9y%!oXP6$+RQ@+qF%Zah69^;&`o9$sxr-4aYR#3=aOi-< J?t>dU{{!l?-$?)f diff --git a/summary_results.json b/summary_results.json index 0797a3bc2..73df6fca5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1738, + "Performance_Result": 22.9773, "Performance_Units": "Samples/s" }, { @@ -896,7 +896,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.7139, + "Performance_Result": 22.998, "Performance_Units": "Samples/s" }, { From 659d987dd9924266ac6f38487924f38b7691e64e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 5 May 2025 07:59:52 +0000 Subject: [PATCH 531/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10732 -> 10733 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6b7f6cf6c..b7d008439 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-05 07:51:21,149 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-05 07:51:21,149 submission_checker1.py:3308 INFO] --- -[2025-05-05 07:51:21,149 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-05 07:59:47,167 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-05 07:59:47,167 submission_checker1.py:3308 INFO] --- +[2025-05-05 07:59:47,167 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-05 07:51:21,149 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-05 07:59:47,167 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-05 07:51:21,149 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-05 07:59:47,167 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-05 07:51:21,149 submission_checker1.py:3324 INFO] --- -[2025-05-05 07:51:21,149 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-05 07:51:21,149 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-05 07:51:21,149 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-05 07:51:21,149 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-05 07:51:21,149 submission_checker1.py:3345 INFO] --- -[2025-05-05 07:51:21,149 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-05 07:51:22,192 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-05 07:59:47,167 submission_checker1.py:3324 INFO] --- +[2025-05-05 07:59:47,167 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-05 07:59:47,167 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-05 07:59:47,167 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-05 07:59:47,167 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-05 07:59:47,167 submission_checker1.py:3345 INFO] --- +[2025-05-05 07:59:47,167 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-05 07:59:48,225 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 20.4138 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420737 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70404 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ba3343d50..1b66bc15f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1158 +1159 diff --git a/summary.csv b/summary.csv index 3ed0951a1..2ecafd245 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,20.4138,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420737,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70404,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0f647fd75..f813946a8 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420737, + "Result":0.420573, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 96546ca1eb67df2d2a8ef4f93b8495df04dbf515..a40fcf0fba6084da8e24c0255a03b3aa7b5abbb3 100644 GIT binary patch delta 1992 zcmV;(2RHcaQ|(i*atH)NpSAFlb_iVy5_`Fy)vn;mF47f~-U#0TpR>LRUjctF;wy3p z@gjL`DDT`qcE3Vh$cBgvP~zE@MLjH%Yt9?euY6%%K8$>H_UgZvzg|VJdlx^hwnJ&Y zAG#T5x66;MK{9PmC;6Y#e6N%LG>;df>+x(9FY+B*e4WK!Q8$b$WQ9aGA;t#LlI)n6s!db=K%T5mF1j<#Ii=G^(5J0H0d zPv>)t_~MiI3nfd>afJ1bDDV1`Nm3(*{Durmt^}8UjZEI$UiSyZTbiT%9n89`qnn(_ z`IjtN3~sOQbjxd5M}wu(P6Zb0m`Vi}-}^f(Izh|JVdS%C3}*uZ404mu4}O2E=t5bK zmEl`V7V;#PHTyrykNGxrx#3bUncYoSJK16jtHPFk0+olV|EOL6N40#*DbozJ#i7(F zl6cbpA(>G6KO_TCvW$e1Wh9g&v+F&rq0}b-haE*iY3iV(LH~i4hz%2bJSgk2GJukV z;JS~lS^wE%^6!xHYMEdaEcJh`{iAg&^vnLoh*l+7qDA6K`-fygY5$N6Vp+ldVHv=Z zxjTTR$^KzS0W3}Oj|TY%S|T<~#1iBmQU*I%)-~*4binm=##uKBaN6l-~9 z%cU;i5*-pxsy`$XO7(|ikjo0{56b|TBqO)5+YH5$bO2bI#Gmbh4_JSK_(KkYvMiMK zNEyhotZR^EcP4NkSblR`ww5VYv1RqH_v5u}?KvekNwi(UCF&!d)P6`Nl-du;AeR-i zAC>_wNd~wySwE};;L;@hXpnxOB_anwArH!WtPFRt6{H_sv-I;Am*2$5t7VdvcCx$H zPrj24oSy(g*`iOrgtsjyJrS(HHNM;4=hh>0Fk^wSJ(hutZ$TT@W8k`?!iO4|+ znc)18GTg{kaej2o&d>KD^H8N9y)XSZwimi3pFgAB5;RdB@uc)aGNF`yNCwfYApNim zph+@-rpftX9RQjp%#o2Fa}8{ICp=NismDN%>(N0GTG^M}zSLEfF~gArp)r zQih9W72`+OZ2Wv5G7nYx(c-gwY(Fg1@pOsuqsd40NacrQLaF?a45C><`C%DAlL}3Ww>e*F9=r6nw6igXY(l=+5;7TuJo?>Q`|HO z%g8Lzel+>y9;yA1OenP?lB{$@V z)vRLu=$fsc??dLHx<6XFKbBvn}XBy z59fn zv=2`H`L=(eY%TMvf~9TkW2v-Lwvn|<^8|Bw)3S;78c8NjlN{bQT# zA9fVr(q#YGVE;f%#D*z9Uc3usJyr(qF-ZvCW7-F2{|H%jR3NFIQf?OpLl zTdgqAWk{BIKQ<~PzVv=bMp;t)At6Mvg80KSfFyq#5FlwXf7np~Nt5}reeeNEFn`!E z<;VGVp{&Qs5J{p30g?x9{%nv8pM>OrDnD1YmPkq>+dE%SGmRHF@p(L%q>Vw6pRIM_ zwsrljJy+`Z!}<$u9^(1olG$)vd>U_mvi>@rY&^SZq<^iy=*K%g|3h0vXM@qzauPSP zFY|x+v>AV%=TA^E*Jfk*d9?nDpT>MKP8Q?r%lhcWcML5j{j+H@{Bbp?-RdaC4{TUxC-eSI-0d%J#x5PX%90dCpqS=D?F;n2UB~w*a-J$qE3KeFCva$0-sH>>rY*c$ZNPc~ zE(##!2r~sbcy9>CjvYW!mjs6j;!ayheYn~^e>RkT(>>r>n%-dvo^t9bZh)~FiKs_% zHX_ZBty%zQ7%GsUyu(?N;j`C+m2R@2RW6)T_>+N2>mZKqqZ6aCuIr?px7Lt9!=Ia% zw|3N6sAg3RP?>W^m<8&!EpvQ%35%1+|F&Do|)vUni3%de*8Rs9(#`zHS;4uCHA^(QF*0JB~t$^ryKpSAF_EGRtz a1-9e$OC^(TDO3Splfx+^1`#I!00027^xXOY delta 1991 zcmV;&2RQicQ|wc)atH)g!kU_sb_iVyQhT|d)vn<3F49$#-U#0Tzq7sxUjcs?@fA6Q zc#*s|ly~kQyI-L$WJ5#-DDmvdq8=8>HRlcKSH3VWA4Wbpd-dPTU$3Iqy^9}L+o3ey z58aHj+vUgBAepwOll{Xl7x|7YzRuz_yH1xMr{lEP{G``+Zp&(C zi#T3wv)aXOs@;vwH}#7+%{N;yX?wGqIokGoJ($mr2J_`nGMjx~EN5B1ojTg2t_R6% zG`?y6+#JSPe>_Q>Kb^EkWWoNujw$8a);J&9>OUppdb=K%T5mF1j<$TeWJ&WmcRq3_ zp3dhO@x_z(3nfeEIKp~Ily`l}B&iWYenW;OSAt8wMka4=uls}IEzME>4rble(M?X| z{7aTB2DjIDy5+U3qrp;Xrvi(0Or-*g@BJMXouK7u%jL6Y3}*uZbUTyK4}O1F(S@=e zE5o;#EaXWnYxaMZAMJ!yGP|3ucCy75R)sD71S$_z|53aCk81grQ>Gbci$keV zB=MyGLo%WCe@F(PWElx1%Sb3mX4iXKL#a*v4?Bv4($qmmgZ=|85gR7>cu>}3WdJ1! z!F3;9v;MQknWSc3dR%3ue}x`rLBu37&1`>{Mw^XIMJHGfu@VlA(1 zxzr_GqC?_I^@n6ass4}*a#=zBVHx0(WaJiho1s{e4ggD&__KZR0ZV@nf5<^lmW8q& zDFa!Sbq%uY&IAqw%WrPW)-uH^wyfUue!P~gJ*VU*iMC6)M191Q+7HQuQu`qpxXp!T$-dG4bl&^MC2eS_of0xpAMt;r^+PhDw0=ki$*f@gundq%GC-zD`e7XanI`8)gYyF|5jhAU z6PzDXh8x)`&X2Cy`T0I%9;)=C_oW}l_CmMh^JlbMf+or%o|Jw_CX~_-$sn2)q#u?6 zG)V@~G&w)413=TH{Af^qpd}&)K^+gudaMi=%?iqou37o{S~P#3_!Ne0L?DL-H2J6=sr-;kD3u?QK{P8UKP&@iGGPa< z%$k%Rb`)UKWc+_Hd(6vZR0fLqZ5=1^I_%08WyD9j(dz zVI2UQCih2!`vWZzIS6>NP}O5qfF?740L?=(fBpthwwC!-qB7j4(sH#fxtnR1$Uin( zB)*h?NJd%G{vjctl4T^7EF-C88A+vWl7CnSl1ftt9UJ5yXo=Wq15XypdZY}nWL*Q6 z_QA>p@}*f8bCi+7={$I9S6CJDiNO#9&MA0f+*3MAE&{DNBMR^_dwy(|7` zs}%;i49ODj$3}(3m);M_C`*bzB!oy-5Pw((kYs-Y0whi54?7AVX)=Gd4?Z9X<_{aD z{5bzEl=WB{B1!ZhK=Q!NpAC}XlaM@6<>$)Q5=lv9d*=&krt#t?K947pv@uBXv$ZbV zwywXm=Sm%aSbxFILp(oRG8>MIPvh-R)?deyjb}HF^sn_7{dmXce`u@dY%sc7PU1%P zWgdT@HsjCp{0S=N+H4FzkJexD)0i*D$zq&+Ss%Ulj-lnGe>P2qKduI~oQ?ll#y|cr z_`y5=Vc`Arb3^~sIBn!JzmMj#TOFnNfeq{IWZu7tyZyz@c$PLM@hG3D)$*H-#rjKw zRzGLSeD#ytY=p5JmU>3ZuO(+aYZcwgYX+s)0gXTCx+;BoVb9NNOGbedR}-1hOhUI2xBIl{%w9yJm)PX~*>aJjhFm2(@Y6I2_ za8Up$N0=$m!Fxk6cI*I>x+FMM5O>;2>ciFUf3czLo9+SUX?l+(c*?1#xCh2&CZY++ z*^D$lwiy7NVQ4^t@(yQ7hRRL zBVMB_PxED*_NPuC$TA~&w(VP9)t`ZKYVvR52{7iq{sh$@vtK330t8mVnwqmMC_MoM ZsO?6qB$IC`Q~_R-#3>^N5GMct0090b)D!>! diff --git a/summary_results.json b/summary_results.json index 73df6fca5..3ec4fbf06 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420737, + "Performance_Result": 0.420573, "Performance_Units": "Samples/s" }, { From 65dc05bd91bf0fc70213213f7df5f3eb7d784df7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 5 May 2025 08:11:14 +0000 Subject: [PATCH 532/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10733 -> 10735 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index b7d008439..7a9e81793 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-05 07:59:47,167 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-05 07:59:47,167 submission_checker1.py:3308 INFO] --- -[2025-05-05 07:59:47,167 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-05 08:11:09,363 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-05 08:11:09,363 submission_checker1.py:3308 INFO] --- +[2025-05-05 08:11:09,363 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-05 07:59:47,167 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-05 08:11:09,363 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-05 07:59:47,167 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-05 08:11:09,363 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-05 07:59:47,167 submission_checker1.py:3324 INFO] --- -[2025-05-05 07:59:47,167 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-05 07:59:47,167 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-05 07:59:47,167 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-05 07:59:47,167 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-05 07:59:47,167 submission_checker1.py:3345 INFO] --- -[2025-05-05 07:59:47,167 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-05 07:59:48,225 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-05 08:11:09,363 submission_checker1.py:3324 INFO] --- +[2025-05-05 08:11:09,363 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-05 08:11:09,363 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-05 08:11:09,363 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-05 08:11:09,363 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-05 08:11:09,363 submission_checker1.py:3345 INFO] --- +[2025-05-05 08:11:09,364 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-05 08:11:10,351 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -30,7 +30,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9773 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.998 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -49,5 +49,5 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70404 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32648 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 1b66bc15f..d94081a67 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1159 +1160 diff --git a/summary.csv b/summary.csv index 2ecafd245..2139a32c1 100644 --- a/summary.csv +++ b/summary.csv @@ -8,7 +8,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9773,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.998,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -27,5 +27,5 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70404,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32648,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f813946a8..42819b77f 100644 --- a/summary.json +++ b/summary.json @@ -279,7 +279,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9773, + "Result":23.0648, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -849,7 +849,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70404, + "Result":1.74317, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -879,7 +879,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.32648, + "Result":6.27691, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a40fcf0fba6084da8e24c0255a03b3aa7b5abbb3..fc5c0d02f5662720b6ecc3a704836b4201d5a6ae 100644 GIT binary patch delta 1573 zcmV+=2HN@UQ}0u-atH-rCpc5Pvv&vw2wKNhC2OU&IFuSi5>NU+Boj*ihhzXsmXT1h zjD(V8cD<)Hl-lI~u%k#QO&xSJ=s(aBv0;Lb2W35022hd^T=&s6>py!;{vA?YEfcJQ zrQWrFwB52ir{toKlh+VRehs2ZW56Og5{UI6TvV!`4AE=|@C>j1bkNk1B-A83ile?d^lgR&kg!(D6z z=||Tr{XE9yH!<>RnPjD%?5_2rbt`a!lK=axof0xpAMvF1Lo%VXen_ zCdwn8lzvDil+q8$e;}F_q#u?6G)V@~G&w)413=TH{Af^qpd}&)K^+gudaMi=%?iqo zu37o{S~Q>Jj20!89y3~A83ilK?s>({E#wSG^-dtx@P0&`;d94%8wSGf8}HQVVRDnOOzi?KB`A5 zKO_@M<%eVt%?ip7%K(~8*numvCgq161=uthKN^f5Xo=V`!N!BK9xKCDlXyX}YSyg$ zd_9{_+0Y)S_;aOq#h?7GW*C-{S)%=D^2t3?`yrW7YCj}{WLD6ASO&;s(hiVm(tg-c zfJ~G1qrv)tf0l?16UhYYhm_%}S;hL%HCsR5hs;BDf3$ReEWb?2(_KBpKMzn%p1O0kCOue>Av1&=Qe@fF}!8Jyr#1GV=$}JS6kyZxCf` znO`L;!+k0(SL>3ynRbc%W1~glOZkUnlqKyS5&|k&e@0TtGLlM`kyP3y`G<8NsWf%a zu|fWUmWZ7;@MNK^N6G+8)-_;hADsO2ZAIBy=2rzv+uFxcX{T%>YnRABHfkiklz&J@ zSHUz5vZVM!LWpDq@rPvqNj4xr z(q#UyqX3d7^Jn|u1Cn6=uwlxN^Y21gkCh>kL=OTa58V9OAQ?Uh$pck>u52xllti|7 zzMy6rFK*)Vcrr;FgCsv&>%wj8`dfRh)bWS)e;3?5#Ph=?v*Eb-G~WJX{dGLqcy`lB z|5|_1k9U0jhqj8&2BWLxByMD1=J9DW{yfj0pkl7g#_;oK{S`lr`C^dcYS3kMU#>XVflIi_#w{d?MFN)vnW@D5j+5IoY zDcOGc(%Jt2vr-ku4h1bso1r9=9wZ)r&1%Ci5QOi23ZuI%IiV#LwgYX+sewXWT5=Q8 zCK0uMkkmSU`^rw@B;Zp)7ing{on2Hc52{Ii!Fg$PiSmq~1T;57>RpLG>Q%Zxi4RN* z)));X>cFFAbysjpxpDC3j0G1Y_#}d~r`(okAA+SAd%g$7d=?!#id*9p3-RiIcGxoh z&31riggjscfe98E?t!TtiRgqBd_>wG*9-u`F*Kk+2aoeC#}{t^r~G8WpjscJ#2QNos+qPLdAFMHdj=$Eg@5896)cvYBpsEm@atACJR~7jB5*ODnAJ%cjTM6Q+ zi~7XdX^>+bh$IG}$GB(ZbN>7!Qm;@oBeVOI^rt2Zn&*@}Zu*ut^%tNzHTgI33^?Y# z{RGt?vtK330tH|vI8(cmE+|_8Rg;P+JOSU6$0$JoyORkiIsq+{L@6c(ElZoBB$IC` X6#)*Dg(*G(VUxruBL)*E00000-LU&d delta 1584 zcmV-02G9BLQ|(i*atIDXpSAF~3;+PYX#fBi0FwbI6|G6KO_TCvW$e1Wh9g&v+F&rq0}b-haE*iY3iV(LH~i4hz%2bJSgk2GJukV;JS~l zS^wE%^6!xHYMEdaEcLGaqjf9v%l^lRlh_bSe`U+1F5wa#5>Ki>Boj*Yhh&h;3hEEb z0GA{qx3Jp`#gcRYSenG2?Sl_kg7`xYg0d`>^+*}WvaD;6Wp^fUAXt8LTeg-dRp45IwCY0I_$sm^%v>%oME=dNsG+95a1K`pm{b-PWpd}&) ze?cJ+%6hB}cd-?uA6>Kb^B9-k#K@~lMS4pY$xlKkcs+;C#@fn38nQz zGDv0x>xX54Op*aIP0|nR0LU~sKN_4LXo<){2$|sgkTTrJR&joG&CbvFA@fkBAH6UA zIJOtMC7(Z|-4ZlW9`U5~Lo%V1en}3Ww>ZoP=0jH%FoxL`6M5&mPuAxH20hzyQQ75M90%!37O<`c%<_~GNE*SNCwHQ z;QX)*kV!H?rb+o>9RQgo<41$>11%9b2q6=UA5w;kW)89J8)&zr2Md>0GlS`M}zSLEfE_g*mzLZ zV`aE%5-$i=&6<^;uV?cq8`=XEf3Ebd_*2|83CqYV(S9`fLhh&r`unmXv%Apby1#7-M{vQX9|Wq>8?8nCnvPX76}qHHbmtAeF%?PIC5Q?`+{ zOXMFLH4>qX%;L>FO*kJ!aOT>mLKVG~GWj$5~ z?=eXT-ecMaXa5LUc2pp#f1czQ)H1g!Z!PUz@kd*&FwkX4mUurlDkQ%2en>`HQv4wy zM6!bT!!m#*8xSCAGJn`n07;YivwiRZNicudFy+VjccHAu$`DDS2LX}?ZvJeL44;JL zfhs>&ww6drBHKG(P&17eH}QEqnWT+DlAo=0;kI@Ctvy%j_`~`Oe{LS)`Qeh;a9n&E zZ-27>I-YDiyJ@6BhsO4<@*E0U`hrtit@ec#pWW&x z#Sd&)XD9RiP2BA-Ic~UW6~_(*w&V3nC6gf}9)HVD!!Qs<_k2aZXjR_i%n+t6yjpF*dI2s9Ams=%1v+?d2*!>bKvI_khYI3O zTSM3r3u^EY|M{+hI&5x~G0B0B~kf6N7S(4$i*MpUA zvY=HioKpCcfl2Eij_#uqqp_~*q@K6dkUztpo0qqC)L5uyRSZy>b4Hj2>a{I%e0d3r ztElsvu;QHn{&))d#9Kc|R|h-_0Z12jP(J2Q&n4S6Drf0z5vR@Brc1JTAh*k}rsY-r z87TWE|0WKAF8B2(DE|PnVI|4}1w)^;@VJvOC|dzilZ+@l0o;?xC_w?ZlM5+20V Date: Tue, 6 May 2025 12:50:12 +0000 Subject: [PATCH 533/719] Updated results summary [skip ci] --- README.md | 86 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 598 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10735 -> 10735 bytes summary_results.json | 2 +- 6 files changed, 368 insertions(+), 368 deletions(-) diff --git a/README.md b/README.md index 7a9e81793..7a7b5fe7a 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,53 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-05 08:11:09,363 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-05 08:11:09,363 submission_checker1.py:3308 INFO] --- -[2025-05-05 08:11:09,363 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-06 12:50:06,352 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-06 12:50:06,352 submission_checker1.py:3308 INFO] --- +[2025-05-06 12:50:06,352 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-05 08:11:09,363 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-06 12:50:06,352 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-05 08:11:09,363 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-06 12:50:06,352 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-05 08:11:09,363 submission_checker1.py:3324 INFO] --- -[2025-05-05 08:11:09,363 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-05 08:11:09,363 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-05 08:11:09,363 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-05 08:11:09,363 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-05 08:11:09,363 submission_checker1.py:3345 INFO] --- -[2025-05-05 08:11:09,364 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-05 08:11:10,351 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-06 12:50:06,352 submission_checker1.py:3324 INFO] --- +[2025-05-06 12:50:06,352 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-06 12:50:06,352 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-06 12:50:06,352 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-06 12:50:06,352 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-06 12:50:06,352 submission_checker1.py:3345 INFO] --- +[2025-05-06 12:50:06,352 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-06 12:50:07,755 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.69804 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.998 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.412498 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 20.4138 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420573 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 20.4138 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.412498 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421672 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.998 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.69804 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index d94081a67..afe5b3bb1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1160 +1161 diff --git a/summary.csv b/summary.csv index 2139a32c1..afa3a818b 100644 --- a/summary.csv +++ b/summary.csv @@ -1,31 +1,31 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.69804,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.998,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,20.4138,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.412498,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421672,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,20.4138,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420573,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.998,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.69804,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 42819b77f..f46b23e89 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436959, + "Result":0.353561, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.8483, - "Accuracy":"acc: 76.000", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -123,23 +123,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":1.74317, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.69804, + "Result":6.27691, "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -184,21 +184,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":20.4138, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -214,22 +214,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.290026, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -244,12 +244,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.413064, + "Result":0.412498, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -257,8 +257,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, @@ -274,20 +274,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.0648, - "Accuracy":"acc: 76.000", + "Result":0.421672, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430777, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -317,9 +317,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.998, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -424,29 +424,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "15":{ @@ -454,52 +454,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "16":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -509,17 +479,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412498, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -527,9 +497,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -539,27 +509,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.15108, + "Result":22.998, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -569,27 +539,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.384271, + "Result":0.430777, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -599,57 +569,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "21":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.290026, - "Accuracy":"mAP: 76.951", - "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -659,27 +599,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":20.4138, + "Result":10.1918, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -689,17 +629,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.988, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -707,9 +647,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -719,27 +659,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438894, + "Result":0.436959, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -749,27 +689,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.420573, - "Accuracy":"mAP: 49.593", + "Result":6.69804, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -779,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +749,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "27":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,26 +779,86 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "28":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74317, - "Accuracy":"acc: 80.000", + "Result":8.15108, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "27":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":23.0648, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "28":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.413064, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, @@ -874,21 +874,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":6.27691, - "Accuracy":"F1: 80.0", + "Result":20.988, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, diff --git a/summary.xlsx b/summary.xlsx index fc5c0d02f5662720b6ecc3a704836b4201d5a6ae..68fe8f9327f5b29b7e751cf335f83639fe11fad0 100644 GIT binary patch delta 3298 zcmV<83?1|DQ}0u-atIEd*eiCr3;+PYX#fBi0FwbI6@M}=cx`Ouom+F-)*6QI_pe}> zUbHhcXf2&RwmXT_G{J(n&Y`ofJOqTz6c9ziN&fwo0Au*xSIdegjcE5xx4n43br3vR z3-N4o^2_Jxq;VH7(s437ZK{^lY{aucG91rtPMh!Fy*vt+8@uF>mH65Rr*Lqqwyeqkqnm8c$Td@S;UimmQO#u9naI|*~xI6KUS=# zv4}^f&1dTP{100<+h-@Mu|JICPwCFjjbbSulYivT;+NOM)23BK+mL;^n#Sd#F^ot3 zzHryF0T_+IL0Ao5WjL@=i9qmkl3w8ru1M zwv*C)QX1x9?a$!~6OJpUNh02U@C;k%eA2cC6iePgDJ7MQire9fqE@giiuFH^ncOByGIl6F(IE+Qy+UPb?Hin)vh`734 z5)oIoM={d)UE!O*jM8tZ3C4UjI z-j^6rmgwfU`waOP@f8_DyhvUf$~*Uu-S1ErvLPZ9lz4V!Q4fpcp7VzED_@wG4>KQ~ zz54IvuUFCQ-o=lr<4~HPhi=B%?eb%5kWAatN&eqye%8r{Xl7x{@TzRuz_ zyH1xMr{lCZ{G``+Zp&(Ci#T2#vwzyfX{z0g&NuaoIL!|+X?t^;IokGoJ($mr2J_`n zGMjx~EN5AMoI2X1t_R6%G`?y6+#JSPe>_Q>Kb^EkWW)ZwjwR*C);OQq>VHb+^>#fk zwLWCD9IZTGbMAc3osZm!r}H^ReDQDIIYxjx5|P}Ih~$n$#4GeS-+&z{7k^86Z7A@ZtfRqFX{Q26g=~n(2yh;iMLkj^ z-zdGoj3eK_g!KY9?GKXlXwqNx@Hz0l_1UzJl$A38Sfp&i27Aj?faZQRK~;E|m31nY`!t zhII}16m}MH7*Ia!hXs(sIG*P($weAp->FuRZ;4-xWi2bLz~#Hm#MWSXEiVjqZe8qy z$?v*9Q+Su%btRaX3A6;0+*YjzFi8f$BpCn`Gq^`?LljKb1AuAhp`+v{3m4E5u~FnL z(JqwrSQ+eMuDDa(4K3~sBkuH{Yk%bf-Md(o_8jaBZnZYH*UPwCpOr%Kr5ACx{*`1zyj zOExhRIFe1wAe$ruY?2JHNj}|0HffMy+Qth8-9``c#VXuoXR zEOAj3$^~vH*~CoXNj5QqY?2JHNix7D6KH@<*Tg340oXM3(Ahrtz@}`8*eFt+=|WkL zf0g03i6PuJ-G3OHPdU=ovBpZRxo#1 z2GC^g4xnk0cW4iQrpez?An-s-#703h!QmlgxNS0v2e!>RHJfGhAKC*@Y4UxPv~ez=C1RsU$-4_>JyM2PGO-7+tl8|Tj-~%3e=HBw z_Ia!JbsxtLg0Sop&9DSZ)JQxj{E$p2jUSRhEGsxaECX0FYsZ&ZnzSD_3Q%eCeiVp5 z&=Rp}%Ku+Q7s`694Buk1kSDRM+5cI7%#W$d4VQw+>~6X`$reXg6}I#fs615tNA3DQ zs^vSjD%mTw#i7(Fl6cbpA(>G6e?KGxP_m4Kl4T^6B(v*1t)bK=|A&ntp)_^S(V+i8 zOT?xLJ|2|ySQ$V`LU7+l*R22SG5L2$d9^IC3YL1;{?T^J@|=>JK3bJvi57_`?H`f} zrTs%Ph-C%)hh+du=I#KNCi{nt0$7^l9}V&kv_x#0h$YBBqzq25tZO*If9jg$pT8f= z12uo%>Rt0^bt~5LO3tuamvD&=i6_+`k_n~yLo&!^1@(tzfJ>5*N7!wKVo5pxEKTCi z_Q3}%LHr?upezezJyHg;EbAI%*_{O(2$tX6maS!pRcu+k>-~5w?R#O#=a=9oRr=BU z(vM?%pG?}micV4;uy8G#Nh{j2~!;*fhb$gR&kg!(EejL9lDqto(dEn@>5= z9;omaBEir;~{jEh1eu@MNK^N6G+8)-_;hADsO2ZAIBy)>j2f+uFxc zX{YQUlh_je5s6bLZ$uFp7 zZIdGuCKe$;(q#UyQ2KT>LeYrxYoFtp^Jx7QKaKffoGixKm-XnycML5j{j+H@{Bbp@vSfPy+ilz*#*5-NyV)2eNp}BBaY?oXU%qtqKeJU8#|{Nmt6fng zlOZH3f7u~v5w-(u$*F-tQ(Afx(#8?Bevs5Ue)}p;;w0cxK^JLeznxuFF88WUe!+QZ zbcOPapae8GLh7bM@0(TnfD#{=7OXWID%68V%lfY5mU83Z)fo#eNbpGnX-~PW&@Kc^ zG4^~1iuo)$bQHJ7DHh_@HP|x#%^JW3Aq%V^e=xxU!v|n$M@mx3n@AQER#klCIwZiT~Q^IaVZr64U>l{Xi^DccmMzZ delta 3298 zcmV<83?1|DQ}0u-atID!Cpc5P3;+PYX#fBi0FwbI6@M}=cx`Ouoo#d4#u|pdpI^Z+ z{m{^~=;l7GdY>1`Zmjl6?ddfL3rviWhloepl}X+Lcx^LUm&GD;THewP2bxM`>JMcf~* zil&pcwk)qb?T=^8bq~jjDt#oQ(RdKQNCwMkJj>RdEaFK&%ZH!dj^}Cf>|{93A1mh5 zSj3~#<}-DC{)erb?X#2Bu|JICPwCFjjbbVvlYivT;+NOM)23BK+mL;^8ph?KF^ot3 z6r)5yI{$4Z}V{303uI?Ke`8>a zYzL+JpfrrZ+8@IeCLC7`lSI7z$U748_apB~B-oF9Adzqz*_*D-r)xt8N=z3MVw`Uu zO}u+F@&3`o2UB8irVgK}1NB3^e%caJ9CV?sWuraty&>e87$bMZM9ui@t; zBA)ki5)mit1&Ih?e|T`-h{s=W9)H1k(19yoa2WB$({qjyTg>+biHQ0BN`E3^ zzArJNEYZzx_Zjjp;wy3p@gjL`DDT`qcE3Vh$cBgvP~zE@MLjH%Yt9?euY6%%K8$>H z_UgZvzg|VJdlx^hwnJ&YAG#T5x66;MK{9PmC;6Y#e6N%LG>;df>+x(9FY+B*e4WK< zcAYLiPRD7n`AM(u+?LhO7ID1VW`DJd-Bi09op0(Fahh*p()MOIbF}UGdN7|K4d%!Q8$b$WQ9aGA;t#LlI)n6s!db=K% zT5mF1j<#Ii=G^(5J0H0dPv>)t_~PHZbBq9YBqF&Z5y>5ih*#)uz5zQ@E`O%-+ECuX zawV6=i`{zh&vAtHjwtW?l1Wk{hWv&MORfZ$evM4t++Ozw#ao)A{2k1?tD~Eo$oZEn zSqyHk?{v#+Sx1AV(oO{y>zGOf7T^0jEIL8U%VFdw1{}ozQ54VObBy?6**M1tphzMT zMG}!Hl88hxFrY}vC5ndf4wH=x4u5C=4H=f)aa{)g9u(`CN(B@@1bfS(?P%XA*^Y*s z#SmE(Pvdip1Y#KH7y%YZM6yUCl0}RtOTGaL4KT8LxgRyucS3>6g=~n(5nw+mi+ZF; zzEOIE8ArZ<3F`%J+8-q8(WJlX;d9`9>%D0m)2jgCRk#CT&{D3l0?weaIDdqzc0r+d zZl9A(ATL9Z3~kQ`ninVLme zM7vPdV`Z>bk`OGGI}q_o7+H9IISu-3)B|p3=2lPnEFAZmG$ zC7YND9LXkTkWG>SHc1B9BzJd_P20pK>jBs_^w8Np_`s%YiP%x(ZNe^;^;j7$n;61n z)BZ=Xsh+f>tz(XrmQ8zi*$i90<(6FTQT)#Se=~(Q%-ukuNp8El5>3n?nj`~gk_@1U z8G@!`LX-6XXc~IxY#)3;Q?^9xD2S#k>ya{CHc8jGY&!o3Xx1^uO3S9RzibAM_RE&d z5*J0GT;PV1P0R$IWD_&UCdmMsBm-D9rfK5XWo$Z4UY|5619Yv}$T`23Z ze==M)F@(#e`wwIDDO=h)=2&Ujba&Xyx9g6w%dXItSUU=J5Kn45Bojz)hhz}V3g!;W z0GiC*0W?kW4($QZH2FIU1RiLK*ijHoaCk@=E}P8afn~E!jpuuzd8l5G+829tXjx@K zo!T>dtOqtx3-P4dLo$JMdq@V^tYG)Be+;lmR2IB@VlsSa55T5L@=;R3xqz0490l|N zsOqsQK$FQlfM(4$&m(9Gnc(ZGJW#{ut@8J%bX?!@OAK-qlu(I=h$k%{k_n{dLo!HZ z1<{9PfJ$PuAeAQBhxPzentUH6ZJY~eiP%x3KO_@Mj9%K(NU+Boj*ie}`lMN|upOvW$e1WOlu$HI&-q|FEM-C`}!7H0VFj z60u=|j|XKvRt8X#5M1}sHS0fnO#U5GUM&->f~DTIf3)4QJg4NMk5(mEqDA6K`-fyg zY5$N6Vp+ldVHv=ZxjTTR$^KzS0W3}Oj|TY%S|T<~#1iBmQU*I%)-~*4e|63B&)<*b zfto*W^{)A|x)f`9C3{${OSnXb#FOd|$%In-~5w?R#O#{Y!0^aEbbe zC$%4v38nT!GRS2G?T2N6I!lrPE=|@C>j1bkNk1B-A83ilK~TtpvK}kLU2FyEN7pR< zJjUfWG4g7eWTl<#uJxmJD{z96|NE?y$Pp+YIS3&WoF7t#8`&z(kFMGI`95SGs`R7x zr60%kLbv4elMoUrDF;Cv56XJ13>VD`%8#yD`T1HjpXB4!GRaDd=AQFox3p8bo%Kq{ zB%hOG5+r{?2$^8~kTP5}s~A7JX5;7kka?)ej~1WhWBXy5j;BkMA5A{0M=C!g6H4WW zWDv~?$`8u`noQV%E3+o$haCmjG#Nh{j2~!;*f7DygR&kg!&Q@bL9lApto(dEn@`!$ z9;olh_je5s6bLZ$uFp7 zZj&PvCKez-(q#UyqX3d7^Jn|u1Cn6=uwlxNlV21r8b4d>!fosNTYIk5@rU&n+&skd z!zHufxcD@arxYoFuFb~q^Jx7QKaKffoGixKm-W$$?-*K6`e)N*_~U9&%h~v^W&Gn0 zgCD%(9|qn}KR5JGjnhUx^ZRH%yVX&OAK0+YPUiiaxZ7XcjAv}F$>B-#Bh#VOeae)-bb{{XX86~_(*ElZoB zB$FT{Dt|elB^9;^J5jGzQGH$v)Ni9YI8x zr5Jm@2gQ6A9Xg6z;}i?=>UP*N{>^rPXM{Xp1%H7F78veEG gK>@py3n@AQEt5tmCIu}^o1r9=aVZr64wHu|XtUu}761SM diff --git a/summary_results.json b/summary_results.json index 0959685bf..20aaaef8b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420573, + "Performance_Result": 0.421672, "Performance_Units": "Samples/s" }, { From 0c10e21ef5d32d9bfabbdb21157eb6edbcf1993c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 6 May 2025 14:28:10 +0000 Subject: [PATCH 534/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10735 -> 10738 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 7a7b5fe7a..8936c1dbf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-06 12:50:06,352 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-06 12:50:06,352 submission_checker1.py:3308 INFO] --- -[2025-05-06 12:50:06,352 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-06 14:28:04,819 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-06 14:28:04,819 submission_checker1.py:3308 INFO] --- +[2025-05-06 14:28:04,819 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-06 12:50:06,352 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-06 14:28:04,819 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-06 12:50:06,352 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-06 14:28:04,819 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-06 12:50:06,352 submission_checker1.py:3324 INFO] --- -[2025-05-06 12:50:06,352 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-06 12:50:06,352 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-06 12:50:06,352 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-06 12:50:06,352 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-06 12:50:06,352 submission_checker1.py:3345 INFO] --- -[2025-05-06 12:50:06,352 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-06 12:50:07,755 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-06 14:28:04,819 submission_checker1.py:3324 INFO] --- +[2025-05-06 14:28:04,819 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-06 14:28:04,819 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-06 14:28:04,819 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-06 14:28:04,819 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-06 14:28:04,819 submission_checker1.py:3345 INFO] --- +[2025-05-06 14:28:04,820 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-06 14:28:06,158 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,9 +25,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74317 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71468 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 20.4138 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7593 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.412498 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421672 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -50,4 +50,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7149 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index afe5b3bb1..378d4b14e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1161 +1162 diff --git a/summary.csv b/summary.csv index afa3a818b..d0d743dea 100644 --- a/summary.csv +++ b/summary.csv @@ -3,9 +3,9 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74317,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71468,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,20.4138,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7593,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.412498,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421672,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -28,4 +28,4 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7149,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f46b23e89..0f283f3ad 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74317, + "Result":1.71468, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.4138, + "Result":16.7593, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.988, + "Result":20.7149, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -889,7 +889,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 68fe8f9327f5b29b7e751cf335f83639fe11fad0..420ec6106b061d8d668132120225e9e05bc83cd8 100644 GIT binary patch delta 3627 zcmY+{c|6qH{|E3HqLL+pBr=wvu|!Q1vQ3toz3dES$)E`#*$N-jwKu~|5waI4S;kU! z?$~BzjVU8>8C+$__@c3l-(0`t@%VlIJdelceV(7!Iq!2GpEWzCUG@EAf=P)K^HzQk zs1pkUodyxXa4`U(77^e-pBE59?>UpuUf|w&VSjjVtngOrBmZzn88)6LI4^6E#gLWp zFO7vmt~1y0M^2MNrd?F>k;QuMwvQSM#w%x-YNHNXtoe-xqaRbnm$b6;G#vWxYRpc^ zNZ@=m%LcaB+@yI1uH|g{lG(8A?y>>(VEW$tGU{9b0q8DR3Xt-B+VU9vklr#tVg)zn z3ErE-q5>2)A~I13^HZN=G7p+QV#_xE_y(;^DJ3>Es8;E%;C%-|g?dw!qhTc@5hvSC zdHb=t{j@yQSu<3BV!?OZB6edLqcsRsS@`VSRq}b^QfL%A@qO+}EpfaCl{R-{X>jQ2 zZDlACU<}P>eIb7?bugYxk8V8EGhci$&18Qb)@-idn$oNJ0UbmCR*Xrv935|ex}=55 zYMOYlg(}ck@tOM^@nY)KS`>!+i?s)>3eq$sb6JJpTxFN&RJH-Nw5A(bud(WLhU3N)}U;a^fc%x9s=nK-RbA z*UVsF@li4vL6Q@>l>=pdMn}yp#YJwv41`*>JVLIsN4k#M-3}}jEGXyRbonGIQuJa8 zC`k^-D}MJf)Gsg9-}3n}nynwQenqV6??sqg{&omN_}ns2@-YuULD(0?Bp4sL(Vmes zvm}R)-KKuASaoC|a9e^YzAkpsD;qG@4cOR zI;gpssx_c_5&Nh%msoT{1Zd0CroXB@O7TawVcFlLdeDHbSuQHC&IHrRZpA6hPtvrY z^dBgNxrfctSuTMH7o1~$)s=Al#X*zRuFotf`L^G=;W*(LoG~mMCf6u0yw0~jpy^>k z?Y4K5r0O_VGDs0MuIg+l6X&%gW~F*I?DFltz`kmRdC=0AJ_azb5(-$iyuL!6?yP9m zZlWZ=1So{=nXR4cA6ha)kU)^>8SNS$s#qf~cn$!VFCS#%Um*j-g} z@ibV}%(&Q84)O;i1J27UYSw5xV*S1IzVXEig-M&Sw`FW#muHvTWyu7c%Kn+@$G6Yg zmB-E|dIX~D&H^hs3X>!0wWsA$XR%SZjH~=}Qt};g$)=~wj7dqt-W1KNMc`|W>f0p= zJJ;dXjp+iySGblYl9~R=ba3dO81b(u)#o32fDT@AXhjriCwD zhTnzi;MgHye=0F(Tn#Qdv=Tx%&hS1)01Kk8#rGiy#sI-4^QQO**&ewSgTLyBHbE>u9dU^_tE6$ifN^^e_+yhLZpwd<5`RA>j^Z!U}$xIV8UOzAsnev4uf%bb9&w%!gb#p`U7~ghq)M^qW!l zc-owp(zff5ZVZWj9}Lz@R`lU=El!p&CSpKV1TgZV(eRie5-k*4T9i1r)p-=@ZPA5` zze6QSD?RRiTj3X(vCpY~;D-G9!t1#T@IE&#VA`&!;b%ESgY|UOmv0T#)P@nc6emB? z#ZNLrbSP3lxr|jc5;pfMMh}#mAgw)l6#Dhqd10ND7OSs+lB8A0-XcaH{pkmAjd5aF z9fvwf!mg?1diH<#M(Ec|nVtYje_429`xR$30jVi54J|$4t>?(8mJl@Y;K0*g-`m2qS4!o4?&>OOsXj`x&SF@#CQS5) zO#WDysbczfK#(GnBlhWRCK#FB%zY<^ahA?qh-HS09@8D>)RW1X`IU~isgd2y-S3lk zW@y`R!@5;hueDG%OO^g7zrB43CK4Y7S%^|ezA)zOc-*VU$a^I*fy7SnxPCY zf(QuTaM6c}m<}Xj(y4sKLj9oUU3U$Xtw=&E;5SH?zBye>oC7583c&!Jk8`}jU0A+o zjH%+Y9wCZMrr1kpCRjDQe$GPGiY}zXXN_gXh$f*pFL+W9s)`RI3cQH!BRbQT4^&YL zlVkG<7M++d&08y&)bka5t~>2~iu7U4OM>gebBH zOnDnC3fa~CL1u{iCz=PWRn3mZzBb#N(ElB)kc48$!21o+_-Apd{Y|{+Fkh~t@$;G= zy#2Coi@7?kJw&-#?7BJMKd_xtxLK?}YqpYzF)ZIt=f96O*!7f{Az)%Hh?`_>kebyf zo)=g;>LPdlq$snnqL&&Z?rN|za@hif4`x}VwYVsvvFDoo_U}yby)0yGy$&<)Gmh>e z(u62aMt5W-V72j^WW-fYn`J95C5D19IWM6CgFtAGxn+lK~_X&i2;j~ zoz#&~UHH!$#7(3MoKtFk(QhP*fOqPLshrPg9>h1e(v_wLA1Yu7Lw{!s{_A2Lx;FmT zHGk+zg9`NgkLv=*rIzE62EA3gz7gH03)s(_qz*A__J`G)5w691cbGPMwk*z?Dj%|} z|7BtR61a+#?!?d!#a_>qNXmEl2iqB8r-@b+x-drB#K^2)s$0yK`z|AncdGwKz;(||G!UpcYE z(#J@@-7R^&>&5am&sPfukmj(q2Y~4{v@Kg!_DOC09vJ~TN9n-P80>06PM_|UbnYJp zri*^_qel5EF&h=TAe)T>9>ZPx_mpiUR_(BjE?M6O>MR&g$@36*DhgJjiUkhRPg+wD z1oO-cds5n~Q4I>Jg4^tM`D2^_lfzoBi#x|@I9pz4QYF}8cZN|As6wS7+LfK@H2q9- z9dAi=voX8IMt=SEo)Lmd*b+J}n<=*ZZv(&kifW*+Jx#!BR$F+1t;26*&UH6i)4aTY zNfV#U=#@qXZ2|anZxbHE53?A9^C9n;w9)TkK;1YW_pJm>^0rrr;jG14BIcPwHxyzB zQ%ORpa+{LCV1nXC;#r<_WHyR(qNg9(NEEP(c?ld|LK(1!CuJgM7KV5!zsyXf-!MbG ze@4Tid8U&_zq1l@uq_{xhvq1-0Di*WgZC;Q-UC0YRvK-4p|t6sjgv$8tWODZvvU!< z#Lew>%-n>P{J!7573%vs3rbhkmQ1V6B!0|@&h9N6_`RVEB&1L@B*{Z1o8f!Nh16iu zXZN?=v(cp+r@S((U%Co_)_2cuZ~X zQyT(iyIr&lv%6y9d%#_5FV-t~MA@~@od|`g0Fzk5lMUPfl=3*kTQ2uSMiqtYW75a8 zV|JCpmSO332I5$s_Pks66*y9IOs95{Lox~WhcLXf(RvpWJ@zvqS#Gg>wr+&ad@RoB z-H#%VgbM+G58UOZod~i4DVe-b8tHswbRldRL}-F(*`ExjoBtq>uhtYT=dOYmu}vKi zKzDx3E{dziN3Qr1q4qd7#{V<77I`@K$o$zs0>68Ubj5@46A$O9N`J2Lf2+sMGW4}C zN=!Cb(b0hdtV?_Kkxh+~ULM?q8Jbfk_D zVhoHo1R=W4J|CLcuxP#JkiW2@CE#o0D}jf(6CSkb)wh(lgXSA4?aA^UebKwe-=bmGr?FVvmwM K@2PVj&huX?W&{KP delta 3628 zcmYk9dpy%${Kv;!BHGs_qB1lhTUc|cunKL%+@+B(F)|^NOD_9}zA<;3CX*0fx=`+v z3SZY*?&W@|v2KWwavQ(Zt9DSh5ltk*vs_~Sz&$tS{y8|Js6BU9q@$y-oLH-Eb>rapmpxj%xfcY zMk;E)?+y=#Ot#5&XV+}7#sF&*wPf_rgki5Da8tDUN6pP#@739`(}yQJhREX5n7pxKeI%Ih+SWHyk=#+eZup1w;~Z2k|?%iKL*+wF)~ZKfJ(lj12jr2 z^s7bwi}m|SBg-MuQhi$|DT#Z3#+eCKG*J_)p*L}=aiud@v*jO2W^Gzu^eBmzGb&Qt zZ8s0IMFTGFcH}-gc1W zxd*FverUPh%Z8kQ=Dprr;yiaAn4fM>e>!W+$WI9D7+YNNx|(4%F@)QhA(4lf0HZT` zU@byFNq&QZc+<;9HCkswOJ{2gw~I%@16h7ok`4bpTs_`SdNP&TyVqbX*{}zqONt6% z&}l!D$;`yLG(NB&1~ajTHRUaD252mKyr537 z+XG*(e>VIt5|b4k!-PGSav+~{9~OuZk#ThDD<(%~ZY zBjGPOwPi&u_M}q5KHkZ!S zpeCB>>J(tR)spV4?4ut^x+P#?+C5U$hXGMGHkuJX{2P zXEaZy-MQcX7eUr$HY7#JQ}Ocw0Xdvm6;_pMo}yv-MnGIrrz-NMlHR-UbGN$x@lWj9 z6Qp$4+}IvAqw%^W<;YCa?KMGzMF{nW@1XnI^2y~lYhyKO9H-djW0SXZd^Md30#|Ga zhyHVFEU-1I@58BQ)T>}c42cS2sw}KDfICmP?ml_vKyr}xr)}6H z5}_>-7U;GR&s=(mc8JKH3RSyzPpn3WHd!vJgA-|nDS%{7G++WvWvzz6v7|4BK+)XM zz4CuHL=5mZyr{5n7bcX;=@r~3^)@fBO4-2`u_lC?zvy)r5pqJe_Ql4ec=Yo*<6?Hc zv9ZH?aN0Z$r7JTkjeo1mfPsCf5(N~6xDx$BW+VtDX?&v&Vp5>#s;V#kP?Hr|ED`v$ z>KOH|(+H~x<$WV5=AZ!CVKO?_Tq z6!oKj%w)Z}#Gf*%VW0Q)93lPg$2yA2r2Z6eAlK&P;BaM!#Y9PCs}K27c$QmOLS5Uq z-J|#j`w5ktGm*e)SiJ8Ao$!ky{lxG43P{v&Ur1~TbH+}2`HW@T`Ln2&k1G5#B9ley zdQRy*So-HvVWhF2`@%q>_g%42zrKY^rBO3P8_f^G6dGJqQ6N^wO||mv*%6k9WN05v zqd@w2YC2WCSGdNEDBCYu@TBtz2(P{hV(3j6r59+L5aXVguAGs~ukW5I8Vru<)Yk2C zvOTA!sQREL8mpecfH0Ns?l*(d1&&eP$2l^H&~dOWX(cABCir-PnkQ`&0s~}vDugWG zVU0AmGpsaHq13Z*EkiUMd=YuIGy5W=2C6U4UqO~lMDtsX)V=F&F5ewlCNbk#m2tGP zY;2c6ew+T7C7B(^TnK!5iYbS=xWDd)%p)U59%gY9{e*M6?%0@CZF}K64Qj;Tu1b#W zCK?8|txCB~^@e4^s^v>)TVlnD6^K*w==|gA)2l6{`)^;Mi#cCp-70+ z=D^lqx>r=8>&mYyHLh{$-A5i|k+Fb!)*!!282gL)U4p`i^rKFq%fM%bino1j$pvbv z@dat3-o_X1HdyBgDF1UPaq$xR%ut;;!SrIuMx z{ElO~BN?ln)&0j+)^Ari72t3mzSKeC(f^eiMYVMNA?8MlI9qXbm~YIOAJ=}RN9!T~ zhPUT$m-`%`XA&Ord{?rgihuHfz54^K;5RU4@nPkmVqS$^tQ8GdD{9%iMNh=bFoot5 zvOYc|v~LdRxuRzh-kOF2H8%iwlNEDF?R)xpN?Z1K2vftd8x6Z1bRh${DDnZOU^u{- zc*-;VpVO~bLqPuP43d4hB8ZY!YJEbMdobF zjRazr+9Cjz7Cc5cZ_h+hE$Eq!QK&q-!?)$8zf3GF!*qizhxI&2(EHn~ z0p?>a!_LB%6XTuy7j{AUYIYY|-w*aC3&5Ty4DP6?v{l9NT9L8d`?osv$I$+=9|noL zqaJ^9rMdem;!k|1y{Lb^>a2nE?f_C%3y;p&8#A>Zc?UR6yNoxo)9YL=7qBwmT(Z9- zHu@_sBc-Z%-Sk`{QJBlm7fxU0%ph^6!VZL^ZB)lB3*dBsQ4}YCb?NursX{e_2*Mzf z+hfWKCSpvm6Zx4xA~VaZF8ob_kD_)Udo==yI~WgZa+&`xV1Cmz3V8O8o3JK7Y1Fp* zkvbN1XI|+43y&^?VIj}&Y`zJ2x#Ewy4et~CsPsj4A zFK<6Ob;I*MykjhT?Mr*O1mY;KnuOdA)_ctH-yT8ka1&Y}(_9F6sWkuKW@!iMi2Fti z%6-Z&i!G{OUQI*a+*SYnQ{G^fFn+JCD7x?h>zcmYytbqZ@4njVdz-*&t}e5R7`ZzU zFbVBQW*an)9ZfGPl_=&D|-ru=T$myw@9ZeOc1&pbXr0T!{%*VIRIk-g1 z40cG>k~7|&B5xhDX2U)>oRuf}H08ND9{=?OPj0zMy7A9SN`Vl{;M}*8b4$5So3YZ+ zdvaxZipSo?V_2Kd=IGaFKfG7idmOur6C&BXDDOWmWU2czS~O_`e(Hv7eTSwT`@P;M za;HbJSy*LnnKGc$BG@YG;fGN@r6LT?{MlIxJ`*H?(CBZOHDYnw4>oI}2|60i)@r8L zE)Pb^VSY`s21Y~OhI>S=zRjQQTVqY=_#Ioj>~zT1P2BngST5zMWpoTB`DzYizJXmX z#2v-NuoqEXmHW)3!Tny7)hoJDk<(?iZW*2rs1b6xuM`SG&M|o-g&aIj_>nb!2X6#U zdvOhYmM2 Date: Tue, 6 May 2025 15:19:10 +0000 Subject: [PATCH 535/719] Updated results summary [skip ci] --- README.md | 86 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 596 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10738 -> 10735 bytes summary_results.json | 6 +- 6 files changed, 369 insertions(+), 369 deletions(-) diff --git a/README.md b/README.md index 8936c1dbf..68e7b91e0 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,53 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-06 14:28:04,819 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-06 14:28:04,819 submission_checker1.py:3308 INFO] --- -[2025-05-06 14:28:04,819 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-06 15:19:04,825 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-06 15:19:04,825 submission_checker1.py:3308 INFO] --- +[2025-05-06 15:19:04,825 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-06 14:28:04,819 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-06 15:19:04,825 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-06 14:28:04,819 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-06 15:19:04,826 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-06 14:28:04,819 submission_checker1.py:3324 INFO] --- -[2025-05-06 14:28:04,819 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-06 14:28:04,819 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-06 14:28:04,819 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-06 14:28:04,819 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-06 14:28:04,819 submission_checker1.py:3345 INFO] --- -[2025-05-06 14:28:04,820 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-06 14:28:06,158 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-06 15:19:04,826 submission_checker1.py:3324 INFO] --- +[2025-05-06 15:19:04,826 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-06 15:19:04,826 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-06 15:19:04,826 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-06 15:19:04,826 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-06 15:19:04,826 submission_checker1.py:3345 INFO] --- +[2025-05-06 15:19:04,826 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-06 15:19:05,836 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71468 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7593 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.412498 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421672 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.998 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.430777 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.69804 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7149 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.69804 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.998 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.412498 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7593 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7149 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421672 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71468 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 378d4b14e..1bb44860b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1162 +1163 diff --git a/summary.csv b/summary.csv index d0d743dea..d2bcf1e74 100644 --- a/summary.csv +++ b/summary.csv @@ -1,31 +1,31 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.69804,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.998,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71468,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7593,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.412498,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421672,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.998,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.430777,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.69804,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7593,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7149,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421672,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7149,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71468,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0f283f3ad..16698b670 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.353561, + "Result":0.436959, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.438894, - "Accuracy":"mAP: 76.951", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -123,23 +123,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":1.71468, - "Accuracy":"acc: 80.000", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.27691, + "Result":6.69804, "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -184,21 +184,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":16.7593, - "Accuracy":"acc: 76.000", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -214,22 +214,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.290026, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -244,12 +244,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412498, + "Result":0.413064, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -257,8 +257,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, @@ -274,20 +274,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.421672, - "Accuracy":"mAP: 49.593", + "Result":23.0648, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, @@ -305,11 +305,11 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.431769, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -317,9 +317,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":22.998, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -420,36 +420,6 @@ "weight_data_types":"fp32" }, "14":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", - "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", - "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", - "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Tokens\/s", - "weight_data_types":"fp32" - }, - "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,27 +449,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.353561, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -509,27 +479,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.998, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -539,17 +509,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430777, + "Result":0.412498, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -557,8 +527,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, @@ -569,27 +539,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":8.15108, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -599,27 +569,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -629,27 +599,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -659,27 +629,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436959, + "Result":0.290026, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -689,26 +659,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":6.69804, - "Accuracy":"F1: 80.0", + "Result":16.7593, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -719,87 +689,87 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":20.7149, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":8.15108, - "Accuracy":"acc: 76.000", + "Result":0.421672, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -809,34 +779,64 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, + "26":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", + "Scenario":"Offline", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "number_of_nodes":1, + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Tokens\/s", + "weight_data_types":"fp32" + }, "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":23.0648, - "Accuracy":"acc: 76.000", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,21 +844,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":1.71468, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, @@ -874,22 +874,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":20.7149, - "Accuracy":"acc: 76.000", + "Result":6.27691, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 420ec6106b061d8d668132120225e9e05bc83cd8..5f8ac33901b860ce0679673458977385b406f053 100644 GIT binary patch delta 949 zcmV;m14{hzQ}0u-atH~XjB}s53;+PXvwH|Z3V$RSK+|OWunquClklTK_<@#)90X-N zDC@B@Tr`Om1dC?P#?RNI`IPxX3kOp*aGP0|nR0KhakKN_4LXo<){2$ai+7 zlX*XY<{^1Me}g7l%ls-)8SYbQxmuUp%d|_>9~&JKU#dSOqb%wEkPuMGGLlM`kyNsb zq|!F2Kdb{urKy9C4eAfHMC`PIB@1OeQhx?mvaSJ3`{2}{Z>!1HGQTQV+SWdnN;_ry zSi401u~8!NrTRlM%98F62_cph+#i+!EUUObw#ofrM*%KP?vD-b541#VnDWELyHM6+ zW$+f0gy1cveQ@rNP-RC2lIlrzK`nEu^3KxU)qb?q3Ikn+WQp}-qd?+I>xX2NC4aRa z5<(;^Xg@3iNU{L|k|ytm9R-jyc|Y3+ACLs^hYeGHn12__daMkQBytcSdEnm92FdVA zNFJ#0b7kv&Bz*$s}zIlKgb7E4QudFYUQf#~;?;Z}Sk(kC)7b z-Zi;&Qrx{s})q}2oA04yOJ5gw1roz4OlP0 zMFFH7VWvWR?+wA&u{}uYlHgE5+-WPR4_Dh@L)kZN0Ox7C#1cGz<ymCUSVR5{e{Nph zhEZdo`c*MNb;%iF7O2;@F7fpxEUu-_Z^DXq0{G)G=reEUK~8nRqY!|c;*QG4;^}$2 zMs=R%MVyAS+-3zB$sR~reDy7F>MuZfZt`#92sq`w{RH(Nvt%X80tuarbDz2l006&} zHz-#DRFjV=JOSR5%qT$tx|0qmIsq(`Nhu~9_T}3XB>?~cPyzq|5dZ)H0000000000 XfPoE@iYYz;VUx)zBL)*E00000Pai+7lbJt&<{_Cse}gDn%ls-)8SYbQxmuUp&9qD89~&(aU&=ouqbzCv zkPuMGGLlM`kyNsbq|!FYKdb{urKy9C4e}4PMC`PICkuaNJyHf(vaSJ3`{3lCZ!5~y zGQTQV+SWdnN;_p6S-V93u~8%OrTjxO%98dE2_cph>>ri^EUVZ*w#oisM*%KP_Kywr z541#VnDXPryHM6+W$+%8gy21$s}zIlKgb7E4QudukE=~ z#~;?;aPtt)kC)7bn)a!VOBIl{%w9yJm)PX~*>Y-qU zFm2(@Y6I2_a8Up$N0=$m!Fxk6cI*I>x+FMM5O>;2>ciFUv7zjn?g8g%y1)`Vf92Fu zd;-R1BBC>rvk7T_Y|{fc!_a^P Date: Tue, 6 May 2025 15:23:30 +0000 Subject: [PATCH 536/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10735 -> 10737 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 68e7b91e0..ee4ac0731 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-06 15:19:04,825 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-06 15:19:04,825 submission_checker1.py:3308 INFO] --- -[2025-05-06 15:19:04,825 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-06 15:23:25,332 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-06 15:23:25,332 submission_checker1.py:3308 INFO] --- +[2025-05-06 15:23:25,332 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-06 15:19:04,825 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-06 15:23:25,332 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-06 15:19:04,826 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-06 15:23:25,332 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-06 15:19:04,826 submission_checker1.py:3324 INFO] --- -[2025-05-06 15:19:04,826 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-06 15:19:04,826 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-06 15:19:04,826 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-06 15:19:04,826 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-06 15:19:04,826 submission_checker1.py:3345 INFO] --- -[2025-05-06 15:19:04,826 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-06 15:19:05,836 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-06 15:23:25,332 submission_checker1.py:3324 INFO] --- +[2025-05-06 15:23:25,332 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-06 15:23:25,332 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-06 15:23:25,332 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-06 15:23:25,332 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-06 15:23:25,332 submission_checker1.py:3345 INFO] --- +[2025-05-06 15:23:25,332 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-06 15:23:26,410 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7593 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7149 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421672 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417812 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71468 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1bb44860b..0fabf9ba0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1163 +1164 diff --git a/summary.csv b/summary.csv index d2bcf1e74..e774832f6 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7593,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7149,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421672,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417812,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71468,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 16698b670..695d62713 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421672, + "Result":0.417812, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5f8ac33901b860ce0679673458977385b406f053..c66130d3279396edb2367b333aad0b2593049833 100644 GIT binary patch delta 2829 zcmY+G`#%%jAICS!C?6}`+%lI+K@}iB_i}? zTPq34byjlEwMmpqa!an;zEh8W_?|!Db)N6{c|XrNbB+s+6#?Q>^$DuAuO$J1Iv)T) z20#PC_c7AM?M2CpQ8{%Lw};mr7(lA_*nSI4TpjcObnH~Pf%LuMd$Hq5W{ny@^||2( z5eFP^`oZVitEil@rYkk8feIcqs%_P^1Zq#La-DJmRinPpO_iqq;v>&=A^j_y3}XD4 zo>V&&z=v7wj~bdW7+G}f>vS(+Iz}`dFLH=zTH#A2obDhmGA9D|iY9r6!{aY596NQV z()WSaHaqCD zkghCfP5AdbeCD3s&G!rc%*Z`Me77Fe$-RRElXognt{HxCqb4A0YaPp#+@$N|IIiBf z2u?_LM0x#B*9$V=&%Npg(cFT_Mf2*_GnP8Kdq@&$Rjy!ns$@7;51{6K^*_Tg@QgAL zpt}QO&NkS2;P-}B`N9n=^>7+k@%@|}2?do50}Y~S5zBR2g4CzmqHguo+x=1?d7*0G z>IL>~i>5gvBgY#CEz8iX3<=*jHSb+ER3H*7*3l?w>X_X@Fal)7C>|dIK{`^kb{p)- zM{sUEmeek#KbtuDOBla-yR*zI$@v-ciWn=@c53k16U+W_ealCP*MiA)s?ED?NrXNB zwyLoK*-%kmYISvMk)J1R8Q@EnyEW+xx#RB0GW!l+M0p%&yHzH6-$wE?-A7Q+?_@jJ zDP*{|fjKStMCVB=iGS(V_u zW{&C07P8vek1wA?#Z|fLAD56zkl1gu;cIq-t7$@_V*v+F6n$JA z>CiT1G!EJqN8NB3QqDBFWdMyLx#)!8#5xE+q;h?zAtZ2k5u0nTG;bn|izJaF8!&BG z2MGJYn%O~@47oesW0r6A)bv@rbw~Slrr9_=7i{93>3mv+q2188^1+Xx3I^hD=>}OI66wYFZaAzeXWGf3p;0)Ot6?~? z3PNAf;Yy?r`a9gDkiB8A#24npAOv;szdKmlL{PT%#FH5U#%6}U<9&JU?X-cB>(6Og ztO|ACsY7-*i`SC}m&s0?Gf8L01f&E^i9K}Tu>hl&_mgZPmMioP#<757y+7V|#gy3a z!3OcsT|K7eQG8LY-STRyH|+Ynhya*GeGue4gU@Ig=PFP1>eZ{`rq6Gf4{31%+bW!3 zxP0=-G3*9~aKcBBBI${*=X=P$p;j4$7qa;t=0HuG#bmKHkgE#Hx>0%@;vi;#>pSw@ zJ?Q=72X692L>AhA`PrJ|=A(10iL~z?&$T~GAC$IziiYm7T&56GI6J1n9Ke03>WhjY z$tLP`04}TX(QFhL7rBa&ill39wY8N@FT0#V3{ktPgbNEi%SuPMR1%k;t$caToVyEK zIsE+mTiCSf!=5k;wbVf{Tj~BjCL|rGjTL)|#G}hZN4uarQc74ZOa)}B4Bk>Fq;lV5 zVZlF_WbShtTT6-5v|xI5Uvgjh8wtxRZkL`!Tlbh7`tVY$;pM`U*V7>5(3+wIs$_}} zH4FzbDPpg6j&7P*nMs{!-S&-=`uP+qeyQMca9AxklcLU>44rDJtm)8N+mxM6&y8D* zJyZyxN)-OY=*4<(pP~=}TX%*{J9q}rfkdF?QX!sXu-g6a z53SvJZcXpXubSB;ld21Ix@ODHoKRa+P@LKng5~@MJNlQ);Sbkpr~6;7FG8+-eA8;@ z5qUJ1@($x1#$45e41s6q8Vp!lvKpjUHEj!P@i*4QU##ZeSQ2Qt=s#G^TUg|VlQfcP z1*y+>^NvxL?Z5}c;+4R{keWxEIo>kq5qM7?X|dj@Yh5_nuENoFcc~DL_8X`ypc~4o zkOElKmI~_EBGO|OqQfQH!H9+o!L%sV{u~Y<^xgttZq^>OKljzUi{Z~5evvFE_X%M@{ zy4s9a*{QrxxQ@|lqy7L(0Mj&8O9--9p$_VPwDpGLDNb3(C;M}^d^;_VquceV62oCL zaurPQY2F$u%U5fuG_ufd)=T&nv&hNMt$HBxFljg8+!km~@l$pQHQ)&~pvP}`We-Dw z_l|ar^ZQ5t*o!{*dMDAyMa*9pEB?uI5cG{N?i}o-RPvB;phxe8`kdzM`C0R`d(}TxDffqJ!uzine8z;U%2sD~rSuDU+%(wBpe1bOB8jQYt-$KvD%?%%vK zn27$cbg?+YmqFIuG;}35*m~8<97?$Mix~QIOd}?p#vee+@?XmJ1fckrWd*AZqoBD} zO2kU{hktmEUzqo7dB#~Et^}oyfly@(+D(Dlk6#D$zu*Tm8Vl~^uT}Y=_Rq{oy7C^t zmU?ZkFcx&#biysckl!$*Iblq{dO!w-4Hj5kvt)5=M6drsGo1FdNNtnbF)2fXKsCwA zdE!G6tnnU`JZ-HS2_IUlE85~x&g)QOOR)4jA~yla;I?b4_n8lvk==7To|@sQ`Qm%K z&lQKp8Bg16#;SZP8DnFdSROL`2x&F&Q|J209La7fZ|%#T`$asS>Aga%=+ z7AyT=ru9%tN4zit;1SXG?(279H^^-zPhN^Pd#mIF*#ptYI@T2Mfd|PXa#89cHsf;-P*_>yElH|~bLSKcdUT@&YeS&#k2A6e@2 z%P`GndF}oU@gX9s-H6G5{Tr0N9Urw9H)+3SJ`uRE>ELgd_GC@(bOXM3fGRV-lX|$W zI$eMHK(0d8u#E!n+whA=x$7W?2dqtG`|p1&O$UA!=t4_`8v;jZY`7`#HEk#2I8d8* e3ZeF=kyjD>fnXX1VGQ)5H6v8T_8kHU*Z%<2lT^z9 delta 2831 zcmY+Gdpy(oAICQ$3Cpe8ahc0F!%#~px0#g9tTPcqiELOZ(m@>gj%w1y*fhCRSUNc_ zF(j9YNH)11DMAiI?yR4?@mtP0kKg0>`S+^oSKc9J*S(oBa8L5YNuRN|Q zjgSeJ1l%4yKN4Xx*6J|R@aKxU?(5}@h(yJ~M0f9LA0#ADvaQ^(x^Asb0(Ljfo_kA?!ZWwc1|N4bUEMXnjlo#rmrw}2q-d4u!6_&xF!_Sp6Sd!y{AIEQD++SHgF=l^jgRr-7@pYt8=C?ksG5 zE45XS87MATAlL1v6!$JPKq6jdwAIlgDX)+nC6ONXp@h$@gg)p!9eoLqdhIUMJURc{ zFKsFVt(UD6^!+8gauT5SEHa3|1e@(Y+TNva<^iES&5mzftlMrz!8atv1@gI?2G^Va zCc_}w=u%!s}C)urmiWS-KYm3y55va{%_*JR}bvAR7Ca$Fp$R-XsT z5X_ixsTrTi)}Rc!CSdVpJM43cv7+Ve8p7wBTuq~Tit^Ey7kW1lhd#2x%fzgmTXu}x5m#VCo|^Xp=2OKnF?57%P|$U&s+>?3j$(rQ^3&Pn&cp*}1!jiBES$|y=8 zbW0XfzlpDLE|$=vRKmfAJ>5yMb`#6dLwbJQ^u1t1!F#<|H!{wcncsxTD4?gxGK(sVP?51}7t3j-56%HesMCki-bpt5 zKm4g}$?iMxm1eo=;94>kAA0U`m9J$OZvq}4(%$ax=${VS@brqky+>Qm#>FY_X-RV z*-G+?zj)ffm9Z`HWFUzzn*(VE-ETOLF(hyy)8J=jD>jb3hxhy-#PX`e2R=zBp-Q$F z7xYZ!CFX9zk&v#%$o$1H*n5Sz;;BS`Qk>p-_#>4VtbR%(gr~(;&4jVR$h7idJd)50 zv6LlJg)u6(&|*Db{^s+XEeC`5do-~~-BX(Y5{)o!O+3%qtWVU*HVbR- z$lA2UH_&(3Xr?TXptd->kbZ|x1)Ed*S0b>HSS6$2#cV)BfF50wjib_U^D>m1BREc=$ca>=%p_>?*Yc9^^+=}hVR_a(o` ztkE{FS3w(s-$)cMZ=R;xq>@R8_WYC8qo{0G04rTtkhM`P7lK2D?yPPhB5~Vkxw!Qn zn`N5bEwn%j(QN0zNQp%@R{_!<;Q?HE&Yr=E*O5r*mwE zaMST*yF5{nT&M?T%@;bzB-8M$g?bB+389lAJ-*#jCv=yJFstA_KV**=X0_uW9?iJC zLbzMwsqaDkxjIM`*e{!zC0NW2cWko1*1+n9@NRUjYD3|raSQbT%2u4oH2v}TFc3LC z6DHpZDYF-7-Eazt!9p*Q^ADq|{Kv-Dk|LjVG6qs()W7WPXSO}=xU?f3!MJ*IRqUOk z*ty0vw-)u-7h42vgKhW?S7u<;APWvn# zHGh(Ujba6Hh*vmu>swq%k7pAlhPJFoG;LHn5)E9}_iA1@bcW8jP%AxC`7f9lA^R6e#Pl)ZBDvQpUqvNOkRUullmIpIj|CwPOW*F&^FA2UzB66|5|8NQ8;NjU3v07tov*5 z_g!s_)DQIGq57+9tc8ks@^BA9R}>_&Mi0!dz^XW%;`mK)=m9 zq*mVsa2Cwh%|`EB=I#wR+wEZb`dr|V@y7(#UyOW=alr5!_B;bE7c-X#?7fN>^|hl9 zo={uqn&qraxj7ib_|#iirY!{zT>>s2c$9k&XT|?H{c(oBU&%&m*2gvv)LD_>kQ$7I z#6Mkn)88oatwF?;X54dTl9%47{7zo$#|LbJB}?5@-z#ie9QmTm(uN~}x@^@Kk{5Y- zv34$=QOcc-isdm$T^5mR{1a2~K^M#(s{fmZXIzUExc6e}EV57C7yFu zLEOLi9PKf`2pXeSH7HpDBV8_il70siJD|Lt8)oPi8lX(yKunxaApl1=^@g}r^~7N+ zLmmHONc$IZ4p%}>I5iecGB52X9o>)vE_n~oo&B;`Pwlr2<3W9QKl}8;WPU*pS(Muv z{PuQ)#Paa|ZwG?MFek!lIlE|?Ly7$NL*tRv$HN|t9?PVc_W4WgWCroW4 zFA0N(WDbwK7HwOAmHO%NCI0@nvQP*wuq!JOVFeywH6t+KDwe!~1sKV~8tAM2Fj5Z; f0ud!ZAVtvs7A;num0(~4_GL90=t?X9{A1_8>G4&l diff --git a/summary_results.json b/summary_results.json index 6a89fd663..90366fa52 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421672, + "Performance_Result": 0.417812, "Performance_Units": "Samples/s" }, { From 32df907de3c872123ba69d633f9c5a3ff4bf70fd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 6 May 2025 21:49:14 +0000 Subject: [PATCH 537/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10737 -> 10743 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index ee4ac0731..75878ebed 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-06 15:23:25,332 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-06 15:23:25,332 submission_checker1.py:3308 INFO] --- -[2025-05-06 15:23:25,332 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-06 21:49:09,111 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-06 21:49:09,111 submission_checker1.py:3308 INFO] --- +[2025-05-06 21:49:09,111 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-06 15:23:25,332 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-06 21:49:09,111 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-06 15:23:25,332 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-06 21:49:09,111 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-06 15:23:25,332 submission_checker1.py:3324 INFO] --- -[2025-05-06 15:23:25,332 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-06 15:23:25,332 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-06 15:23:25,332 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-06 15:23:25,332 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-06 15:23:25,332 submission_checker1.py:3345 INFO] --- -[2025-05-06 15:23:25,332 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-06 15:23:26,410 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-06 21:49:09,111 submission_checker1.py:3324 INFO] --- +[2025-05-06 21:49:09,111 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-06 21:49:09,111 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-06 21:49:09,111 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-06 21:49:09,111 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-06 21:49:09,111 submission_checker1.py:3345 INFO] --- +[2025-05-06 21:49:09,111 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-06 21:49:10,102 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,13 +26,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.69804 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68023 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 22.998 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0501 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0fabf9ba0..169ec5606 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1164 +1165 diff --git a/summary.csv b/summary.csv index e774832f6..ef40695f5 100644 --- a/summary.csv +++ b/summary.csv @@ -4,13 +4,13 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.69804,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68023,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,22.998,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0501,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 695d62713..44933bdcf 100644 --- a/summary.json +++ b/summary.json @@ -159,7 +159,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.69804, + "Result":7.68023, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -304,7 +304,7 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", @@ -334,12 +334,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", + "SystemName":"gh_ubuntu-latest x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.998, + "Result":23.0501, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c66130d3279396edb2367b333aad0b2593049833..613c933d7cfdb629d538dc8e0452ba6d97d2b7c6 100644 GIT binary patch delta 2455 zcmYM0X;f3!8ir4Zj0K|%w<1CqLQF~m1ygL9B10uHqy|zTfLf766ABe5Ac}G@R3g|S zCJ+>43`!AD2sok$LV*Mst4KgE6a>UDPnIF%9=NOb?zPUJ{o`5BdcW`8>&($V(JP}g zGy!IXS%WqJyr_=XP*(+mjvb#0^ImRv@Bc0z`&2spMAz%(wm>fVZZG^g3VxigotR>H z#Nwfu7FrIgCoFb(&v(2~t?`(|%=6iwOV5>S_a4&a{t)c|YDO*c!;SN6eYA&IxXN_?F-1oH{52Z_g$_HX;!u;b;8+r@Cbt?ZpOHhQ|fWL@K%q$hn(B1$*I@oDYT77iKR~Wx7=&Qq1koSk8VBE;&LLtqcxxVNrrw z_E_O_rl-Xd_s{Rg)$aKI#1!V0t5fMzx|iuQW4bfHi&bB+F-bIi&)~Fx&^Zg8F zM_hsB2~Sr-!(V2sM`3nX)P?O1U**kQ8Y?{K9Qwm#cOffNGbXeXoUAhG^T0O=LJ^W} zJyT|e%xdntMkqz^fZ6F3$!sAMAzI=|RvlLfG>_8nQZKmEatENI7l7=q2Ar-IV42)| zCG!V#@*heEMLm%hT5N1veQ9o%yUp`z&XdsT*$9bgrhx}i2>T-CHTU~$CbAz+k zsEM@R6a=sF&p>NbtX;)CfjAyeEu$M#lw^jE{nmn3`OPXCy>w? z%@9y%a1#RJZrmTC;HdowM=gk$pnsXzrCSv&Ro*MwO7OI8K&yb!JE_DG#|B=FCQ}-i zvHOosF|XjuwSrbPzc;zFA&OOi$k+3kBbk2_`-f?v`edgGWx5wef#kz50SDbAcDjf7 zvZcIUTBtzTm}Xu`i0SGsP#&N(C2P8vWGpcZ50Kp=-g&+$S5DfloFSdP?^UTKJojd! zJGO29rX9{#Q^>v+*Rx@EgVs+(B{BReL55l-b9(!6ju`Co1}K7cP~^ICQFvKLuWH*0 zFQZ|*Fh<9bakP4U;Zw-jg0aq^XFqP}EB9uFpS6rlH+@m2Jd@HPArTT^iJd-Qp z@Kf=@5(On3%!)`^y}b86lCODd+1;*{r8>o9B~th>Yich-4m=`|iMxxD?37klT5Hl~ zcaDasPI$h-DV&-|_uI~g-eBI!oxHmwim{5|@&~f|v1WQpI}ZmEQF@2uJZAb!tzJYM za03NrW)~xSVdHWR#7F}(gEJ2!vEeXQ^h@!SuqkzqE`6c_Md zttGq2X_!30W4d7#>DomBuB*X@2R0uK!9t@0K%h#ms*S2cOS$9KTtRt@G@`q_03v1P z-}`of>~ah*D6e5pI}rPS;*>-?$}`i+E<8_%O|S7Sua5tEfz0fTEitwNW4p1%d&gzq%Yt#=n0Pil|{hR)l5LqDxeXE<+y5Ec9i$A&B z6)q-_)Vj-u6qVWql5%6H#|j9Tjh^yMTHXQ{Y4#(qjjZa_Wzmnl33Z5U-WN(XR*4z3 zioX|Nu4LDc&SRD7zlw_$?rs{VRM&Vc;5+_BdM`>nOz-OT{aTcXaL-MU~ zG00HoAEK)p8%;v1(Z_*%|7)|tmCU1X;c|nFIM|y}!AkJ0tf{zw%Zz=&|8zYzMG+$M z9Hy-Cm>HH!NL@Fa)dwOD&hP`f!7#x_W17m5>_w!};wJ9P(eW#13ZaP(QPG+WIbLf+ zi^$-%G%eI?!M3EpF~8Agx1d(A%M8?>XiIeGS`PA>vW5{73?-Y0`i128CDgHWCR_zY z{Rq7brW_ye0m8*p^CQCO_PUeOdb73Wmn&DA*v zSKq9bB|IHjAlCRs+wVGx-y3PMW_R^)&fNIf4Q&HKxP5ZIr`%?R^YZf%rfo-E`PF0R z+q$B9h&kv+5b2*3*wGB)N08vI^&w;Bm$M10j|;)joA&digPseriZq|~`sA2cyP3hJ zcN@beohU|vxQxRD;{NvL6N>w;W%ZTIy$d=(=rse@{S%zmW(fnFg`n zNtWeV9UJO23VfSB000`$M+C;BR7Q@%0DxH?Zcw>hRd>Pr8??Y2Wd(BFEyC3M`E>ZD zCxk_GG1WX&VOkoqzMPrcM6NI4TKBw6aIF2EVZzi?aAVxcbXnw7bDG!Bg|U{azx&Tw zq~0t^{qvv|##?q9{8v3f7Ngw$`G}pJGR;394 z+Qww@=_l};B@=rnvtS1Oa3tc_c9O?l)G3Re^}drpL~Ol?$( zy)>ik71%la!R2e)-2)G({j%daEBl@XJg-BDn%$PlP9)yp?t6iw-3#{}l2aH*+;e~6 z8Jt#~EwTQfce=ziEv85L70>- z0QEW(L#77~(-g4k5KF1b|9_qY_85OykRT1?1RE8!VO(Ji0v)UaY@dLHMH_9!38hO@ d0f3!Q-v6A$i~52TtP6}IXvJEn8SUM?{|{kRivs`v delta 2442 zcmYM$c{JN;7YFd9hKQP`qno88mLw8e%cSYNwS>kDu{8FEF1ExZZB(b$YKd)xnwLz1 zT1FA24DCxoI%u^@5jtwGP(zEBs@kfAEE9VA&hzi@oO?d^KKJ{a-@3=LN6i&wa7(&= z^8-}?;1TbdGDwl`dtUwh*~SqN+b@D%zGT=m?|9T~9|eqP@P+uZRO9jH07UTR|GrE? z=2n9M1HCwVfyM4ay=t|kolcx{$oX1xq7h#CV0R(-uwJf(5!tq!@B0o+88+)Mbb1r) ze<5tS;lRdu!8M1ci&j=1(H0$Ru>4#wEe#R8*M$W{Qx^M(^Em9GO1uScG~Yo84r& zJo_qxwC}r4gaA+gQ=qY$nmWrv0@-T-nr6%_4SL<*gKzg9xq=Be)D=^$decSq6_+Y2 z9q}a4NBiVf$L{inO|%q~i7QT~>&UpDWO+}t-dxh@gWVrL_kZe+bik}c5CcGeZ57Uk zSQn$t=tb_g^T=6q&@e5yeeo=&P^N&M&kZUg4+#A*3iey%?%=d^j;rZqoB$ELi@;6O zAhAjD^3E}NHVU*x1cvSdd0bw?==szF!jPTXGW3S>OtaRA@^b1=$O95attiVRclnEz@m^C~UfZZ!MN+i;m~Ng;j0G%-;(auVsMJgPu&0>HilRW`%LNl|I!iY4 za*340R$SL*A?YB*xFF(!<;3p4aBDGx%_F6_Zo_@=25-@YK~^>(Tj@A7Lgz z(>gT!k$u_JW`$VWd3R3aNZB=(u48ZSj=qRJY@z&wYmpxS&Z+n<+I>Se&s7r( zOCow-jw347kcKmk)S{`__eh&E!IqnjR9-LzC3|#ZXM)X7NxFnUAD)aS&JFdrsb#9j z7LJG4vrYIlhO{{Y*Q*tqnWJlTx@Y(4jMJ}WlyqFB8*KTu3};^Qk?x>WOws_uyB(1D z;1}0?ag{Doh{cWM{y|&&B&mYQH(H2|XxHH&G5`*4iGZHtNqHU96S^})<}HTAcjvb3 z$4t7zx@tV(M9vSd@mmbiNve#g8tBs^4bXVZYOshe6G#K>fyOQ?^h_lY$X5^J+@>}S zbyu<=4j+5(AMw}9^NGxvgnVr1+C7nnxacf@Ci{Irz3D5pZ`BBYU}5_l*BB&bx5_(6 zH{d2r|9;Jw>Qh!YfSBJ_w2%ZLCT`&NByx?n$J)uQzQ`m;Auj6>Nh9aI;+U6>_~3YaOUvc$6&`xei!Z}I}h4hQYB1h|E7GkedVj|?_DEb?Z+?; zzyM5Avj^bJUM+1ACFG#cSIsmOnHuZ{(~IbRJgo*$(S5>z6;Yt9GID))gq)Ec*KDaSFHo+5gnCUs-48 zR7D6LIyc!#YWsAT%QkW5U91lfFfiL#zmlYZm&d^{guU5%g6WxmVs@l>J8!47h+NlU zeR4c}UbChSa!MlN=hIA9s}jpX7J@44JouUQPJ>V_xSF4QZA*>kq|CkY7)azX;NI8@ zDtJkUM(&^NpO%hHeoim;?1SB8D{rMxGraQOfuo?eQf045Po2tJq(g(|_YD_~7tSr% zpSgRqm+X*3#9~xn&2@1OKl-?D>FU~X6F;?@$HTPt>QGnWM%(~+_Btw$nQ&<6>Psaljgoj_N?p635;-+y{37P(NkcqgwtyA8;6RT6-7RgHPa(FGX7ygr` zCA@j{`slCN=c_+ec2*is={L;9!Y;FTbZ^rD7r zqjeIrxWPzRA9((?#AB-NH-f}E|M{hetk)n+HIIE&hWPO5(D5ms(cHGu>l{%76?1TY zQPo%SJA8GBaEZ5UCg74{WMd&Oq3!9fk2eX`;rK|I)6b6liHG5zu)H$|I`$}Ps?4gx z*#Hi7 zj{sp2_{$YKAL_CP8CBy!I3Z+Qp=Tic7kD(itMbmD3hlvq9H2m$jDEzQ1O6UBu}NIn z^AkSh$a#o&qcHJ&!xhI@1f}4+rQbJXo^o#Ihp;32&fuh~^bo&^Ew`YOMlUlPuNIAf zgB36MhD&R3<9j7Np{+Nfc5MuxX1@z<)p>hj(oNZ>=V$wwuw}7(sB88ek@=}spCKVj zeR?q z0Dx%Pk+ATqG;$c*AGM$T9ZC~u%07icXzffF2mrY50sy`N{CnaPAnZ((H86 Date: Wed, 7 May 2025 22:41:26 +0000 Subject: [PATCH 538/719] Updated results summary [skip ci] --- README.md | 86 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 602 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10743 -> 10741 bytes summary_results.json | 8 +- 6 files changed, 373 insertions(+), 373 deletions(-) diff --git a/README.md b/README.md index 75878ebed..497075d36 100644 --- a/README.md +++ b/README.md @@ -1,53 +1,53 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-06 21:49:09,111 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-06 21:49:09,111 submission_checker1.py:3308 INFO] --- -[2025-05-06 21:49:09,111 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-07 22:41:20,725 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-07 22:41:20,725 submission_checker1.py:3308 INFO] --- +[2025-05-07 22:41:20,725 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-06 21:49:09,111 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-07 22:41:20,725 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-06 21:49:09,111 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-07 22:41:20,725 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-06 21:49:09,111 submission_checker1.py:3324 INFO] --- -[2025-05-06 21:49:09,111 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-06 21:49:09,111 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-06 21:49:09,111 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-06 21:49:09,111 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-06 21:49:09,111 submission_checker1.py:3345 INFO] --- -[2025-05-06 21:49:09,111 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-06 21:49:10,102 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-07 22:41:20,725 submission_checker1.py:3324 INFO] --- +[2025-05-07 22:41:20,725 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-07 22:41:20,725 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-07 22:41:20,725 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-07 22:41:20,726 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-07 22:41:20,726 submission_checker1.py:3345 INFO] --- +[2025-05-07 22:41:20,726 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-07 22:41:21,788 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68023 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0501 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.412498 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.15108 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 16.7593 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7149 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417812 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71468 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71468 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417812 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0501 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68023 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7149 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 169ec5606..0a7f95a2c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1165 +1166 diff --git a/summary.csv b/summary.csv index ef40695f5..a63f1d448 100644 --- a/summary.csv +++ b/summary.csv @@ -1,31 +1,31 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68023,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0501,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.412498,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.15108,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,16.7593,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7149,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71468,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417812,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0501,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68023,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71468,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7149,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 44933bdcf..a42d19135 100644 --- a/summary.json +++ b/summary.json @@ -4,29 +4,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "1":{ @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.436959, + "Result":0.353561, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.8483, - "Accuracy":"acc: 76.000", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -123,23 +123,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":1.71468, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.68023, + "Result":6.27691, "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -184,21 +184,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":15.6951, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -214,22 +214,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.290026, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -244,12 +244,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.413064, + "Result":0.413749, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -257,9 +257,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -274,20 +274,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.0648, - "Accuracy":"acc: 76.000", + "Result":0.417812, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, @@ -304,12 +304,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431769, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -317,9 +317,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0501, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -364,22 +364,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -424,29 +424,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "15":{ @@ -454,52 +454,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "16":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -509,17 +479,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412498, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -527,9 +497,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -539,27 +509,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.15108, + "Result":23.0501, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -569,27 +539,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.384271, + "Result":0.431769, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -599,57 +569,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "21":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.290026, - "Accuracy":"mAP: 76.951", - "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -659,27 +599,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":16.7593, + "Result":10.1918, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -689,17 +629,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.7149, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -707,9 +647,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -719,27 +659,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438894, + "Result":0.436959, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -749,27 +689,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.417812, - "Accuracy":"mAP: 49.593", + "Result":7.68023, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -779,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +749,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "27":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,26 +779,56 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "28":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71468, - "Accuracy":"acc: 80.000", + "Result":8.53781, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "27":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":23.0648, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, @@ -869,26 +839,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.27691, - "Accuracy":"F1: 80.0", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", "notes":null, "compliance":1, @@ -898,5 +868,35 @@ "has_power":false, "Units":"Samples\/s", "weight_data_types":"fp32" + }, + "29":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":20.7149, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" } } \ No newline at end of file diff --git a/summary.xlsx b/summary.xlsx index 613c933d7cfdb629d538dc8e0452ba6d97d2b7c6..b41533251347eadd8496844a86c2ba37750af1b9 100644 GIT binary patch delta 3665 zcmYk9c{~&T|G+0nlv~QRp;?ZxER7SAAh&iDO(v${mo-Md+-@U8SXFrd{`8|1)M&b$fYNQZefnkG55^$mjU>8xu1(NLFcGN6SCXATz>u_Ue*OxPQkn1nb`agt)ViqSno`rdS z$uu!_uV-ah5$vZ+@Wsw=8dS3rs@@I=WCjZ)g#lXzYcEx$Fy9%`=9=$gRtykf=>mDx zQp~LoM08J>rPODa`Q#s>V#%s8D1{N<3GTSf+O!m~=ucz4G}HHSKP?8PzP^3Vtw>Jb z)9Ba`g6Qc!+1zwtk(8^>Sra;YBSY2P;^Qp8{Ay|@@QQ&n@bY=rtwB2H0@)UMHJiEd z14{v-BAq+Raj|0SQHF;r8%eQL>3b-JsFE9Cg^*CR0_()D7{lfli>a|8`yFQsS_YF$ z4RgP94!OTGS77~YP!#bCg8iNf2D6q==F*(+0Pl{qos=;STmF^n+AOj9z(B#ZfrEXr zXr8EKj=#G)tA1r)p#ee=h7q-5Ryw}_6|CBuK6XEjHv)!-~MSG=Gi9oM;8{ z8LBnVXFdZ7NSa8$PQmVQrUg+-W8xjw&R_BUk{5Pmd+Ox?Y@|nKV$sF+rmW+%VMhaw zUWX3|EOT&TT%3WW2dPxr(XOjhZg!8lcJtXkt9DFw{b+Su*k~E5M~j7JBv~f{;=u}5 zV39b5I8$+M?)W0d0uQA{mkR}_PFdocL-jRMm55S)>!ZY5blCx;rA33PnoEy~ zmEl@1q?DS+Pe>QPd*U~-lUeCB2B{Ax0u%2ufDK1RQbuz6HR}0=onjn>kKok?IyT6; z1~|W?ne$PXmxuMvWa&G_$VrZ+;=r-c(2$B;zcB-M(6_t}?Ql)*h1vlw#ExV3iMb`qeHFKPwuqbG?fT)L_#MOTguHpk zqbrBJyb>tKAdR#87C&r)@B@!sTp?Ye7XEL6AHk9U5*A?bWlXAu&+05j2ta;LNG1kL zbmS|#nxthKc0Mm>w7FD(_SIdg=Dm^KmpMJ@-- z?M*yhiCS1t$f=G)_j6Zz&OE2yTY82b6s;<39eOxz*^k4$_jWy+iQWOVkVAWut2tDA z#QZ$&ocC6S1Kz*JWLamYQ+(uXFn1iJW`4m;QzP1gcsw5kkAS!x%}(ayQ1kzf3j9Nr z%chi@x7`^8-yEl~cR{~nS5?dDSfEI1Ghq%0b4T$<{*lsV9L13TQKTQ1a`}3`iXI9; z2wb|05E`K>J17K+<6rzKH7ukfy!iI7b-NAk0tpuCEdhIoy`Fj;Pr)=T+>&%5l|n-{nz5T*zz8&H zqOd3`;E0;Y1^yXs%RZb<1cY=C(4Ot{jJHc1-n3y|Qxem8e{#{1{k~y5^VEcj(sNM( zk*oH4U@_Bn0yedczd(oqc-(y3Km|<^n&&Fj+|n~+YPj18=QY4c{7w@V(|^8|;vn`-42plR=D-jy`$h>J+F7<|Zu~>c z0W;Nqh^~oTy^REmnay)D&){crGM@?NcFZmZ^>f|vB!+96BD+M>qVrREaW`#aYQORK za01@G7s~*-PV~X)W;%mDb0%}PV}})iwLxaBh7a(~q<$g#!=7z{53a6;~(#hq|dNndQ4Hu8hp^z?LU&Fh|&k;maOL&v@oJM+4I`LdN# zLauGgXIYvd!Ou03 z0jy!)Wb;hI}BK=vM|@#;ZD z15MQsB$V3&?#0`sd5;5(AftA=^Y?!9;yj|(uW;Ut@`AmY?6-M&(Nhg=Nc%Qq+eX`n zS4JyRN=?+c4GG2akpzmm>J@Mt=&*^bXf={6=`9beY=`9<2jTcM&=QMNc=bDfZ9>+BZ*>P*SB;4NgWhTI?TJGp`_ zWCxz!^IEe(j{IPQm?{_NgkboSOnAU1uS%$NAu?VPa5ChdzYE}=$uCI@2ni~WXMOTtgKyT8*lTjrMTr+1eJ z%jRuN4HSAZPtiVw6>QabBBsh-3)t`HDpDt}`cjuK&?#8|jUn`~w0QF>ORIM96a*Et zVeV&!aBjT2f}}c}pb7FwU}t6&NoT-7JmdDKv%*Fjd;Epqc^I=&TzfvXPRzwWa&h=Z8@E%!?a;ZuY~FT=@C=Lru7wFwA*QV18h6?DBF_c7yl?E5bxkE zdEn;o(dgOeUp$d~5~uV4);glyBFdYwZ;-FpTv2~{=tl|yNr_`rtqALIO8yP?T(g*t ze~Q+?$qV(Fg;hR7`CcLx?B^aZY2YOkJ11@h&L?Y?mcf@0LevJoODiE siVIHqmoh~9Pnx*^0Lw!FfC%8fBO+3al%lN1tMtT1laY z6#?HM#s`V`<|3EYGt9o8FU41zB78z2m4sN{fWq9K-Dx>lpNeP%Q}WwRdMWZV$XkIXSPLSEy-E_tu;okrhw#)T->>S-B>|+ijY^ z<=Mu9<#kkcs|7IjXBN??iijOW^S)A^4}iuv+#N<^H+eUpq44OfDFWJ8aU(1TtzmKE zb5sud-5WyX#)UCxZE^+a)hp%a`b+7a-NE3_l+$-%W&L5tn$7v>1U>p-q4FdKO;0HL zo;F9=Sj21hK$T`cJGPg7o;@2J$x5gxSgI#|eSuD$x-{R@`@roqlr-HtnLE_>8K|%~ z8O^x+R^r1T}q` zZzrU+ZgsX@zw+qE4fi^$di-AJp4`iM*Q-w(t!v0%Ug`KCJJ%`BdQcbmT17>Z!B6jd zYI`{LB0e+HTlun?`5bdH&!*F9t`hLTQdrkeC*5!QNz^V&O6=+-*N2AQD59(+r)ed< zCQldogsDjyETSkIYd-u>c8gJ|tqf?JC|t@xkYv)x{HP0A+fi7-z=qQ!Wi$OHM+$1Y zD{mU}TUebGaeL$kwQh_+t+D#s`R&~NDvlOa@oYJLIxbT3aK0=lG#&ok9WXMesxa8T z@pB;0AZYEJ=<|Q(VDgW5f*``D7I~8ny8?>Bp6GR=$)QWlnTZqgqDhGkeK#S(l~iN- z$=!yQ#1o!c^1=Zj4KVqhTj?h4BX>Fbi`R&W+6~R5#7-A~8K5=1FGk75Uu>=8b5NSF z^QRr&q{Lm8g}fV8O}$hZ=&*%Q-Cg{nBiMlO=-xP(Lrq} z<0o2i>P~}9u9KgJQ`&l5VRK!?*zO};=v$FXb3v*9PWpX6XBw>w$sEf}oK7=oSK8DX zkCsNBv^6i8aHhnR+IIkO0W zzkd1W5BRA-mZIf}xX}gOZ|hEj{jOCMyO<803fq{D9oEs5i+$^qnZV;bX>QSoWY4v8 zrjajv9=RXF!i1CLlgy7}OiIPXu&^1(1>gD<#x{BMhfr3se9Vk{+!B?OD>-`H>Zw7gpcI%q4kk>KH_cKU2i@lr!!?bi zOZ?O=IX`2hX)=Ad4w`*I-xfBGXlhMW7;nC}#jUjjCT@BSxNNPSTcvJ&r=-o>N3Wh9 zi#Y>$AsujBo>-jJd;5lBtlsNh3_K%O&YVvhFD@WYH-}(Mae-II;>A)Z`~qcF2|e4b zDrtR?7iAw)&R#_$JnSB4lINGVl&!_34mPET(pl=OK*8l_(gUBz{HVwcHW- z0#^%av(IL|=RGgQWD&7W^g7vfr&@5(0utL1=*gXMc(%s^mr+%dbB8-W_|uCn;#={D zj4P41c&QHSGPY|GmwJzX?+MUPg5ThFE=>|QA>l#RL@t!Nao-m>3I~p^DB;`&Xyr$F zShlCddUcUyka6^<)i=X4nH+icHPpgG_iUw_f*9X%yLYb^svw%X4+cDstf1df@giqq z6h_(w$rz0mI1N-Vy{t+m6b$3_K?U(LI-~s1;Vczl-Q-5=;rnD6r8W-{<2ODGHbV0& zDTKm79Uv1{R12=tUyl(1x_Sd z&D)>lp>G>chtEsoTFY>p`A6!jF>ryQ?wQBb+Ik1Hl!4c<5i|$i@4Q+-6_*G4QZ-?vzqeDOg@6 z&%cb+Qf@f*m)jEJi=kfCQihdmlQhu~rSwo&na4{bl)n!})1~mA|2(~dI2Kl{=w@b! zYW+vf@V@i76|t4U3?)kY@*Um@&?lm4J^rt1Ni#b zZ*3eGZE&m0Nwn1xj)icV8HRU`+etp~cprc9)6SohI$W}(rj((Ym zw`|3SXkA^xr>ImPao!X7OXex7EQAUDx}H6i{V$p^iS<5?(VJ&ySRt5DA{PQ@YDCk3 zSP)0H9uH!@N$j>X`4UucUw<+Cyk&d5fPqZbmZSK2jFJC0v!QDCqWW&AdH74Kx+6Jv zfFV|Y2z~vLhO)gtj#p;b081_p%Rs~8952@aj(m0a<)f|yGB9idvLx-78M0z>5`{!w z9NJCDYCW10ETrprwSIqN$347ww%57oChIu+RIYEhVuWLjaAMJd*8w za7_!$FqNf<>olgNZ?1pRUgD{jP@=$-ZDWm(9=uBl9?_OWE>-J6@B-)!3WJs?hk7M_ zG{k<4J7~gHAKrPhA!VlChZ!yPb?-d* z8J~^}g6W_<7!N+b96HFAK?saOX`}lfnQ_N7T6Z}?%dDB)|B+uD4Vu8NnvKQc^ zcH7bbFl|uuD%G3B%qQTDkjxChQk>!5b@B5DPXKA7O$UUs?n>vI;par`*q629>dr3z z{_0|mQg+dE;KByR;F=1ln&P#rgJ#?!RNmHI$j-nRB$(+OaLxVZ>iG{>)^9G2D3sWL zxc=pEd0keIy50SucSmjmn+;Vlpm%N?H!ODfB(0ax%(l#DI9OHtzp-`?ump7dJqFA7 z>&iIc2d2#8)_7EeCE>$i*gKn>ZxzWrDK zbXi3Cj<8&_!a5N)3i}U7AC-YZkZ95juA` zk&ON7ZXXS|13=xzwg8LI?p4mNeW}J+4P8>9{4u6#=XS;O-K#vb*)p$hN*HLQjDrbkh zC+<89Nn*N}nN3>$O2B6+Zb2bN%1S@b$~=yj`vD&kt1XTDO8zEPa`Qm$%*^ssYt zU4y^O_l{5@PntGF|NI;lAX3yx!Z5^OyYo&Cx15K@M}wZEp{tuGXXPG4)$)WoJLcB5 zPc5iT(>~M}nC=bjJ&S6b+CWq6Z)=KO1yoK2DE!h$zf`a`7tRtCCHZQ`7nv=n?Giqn zoWiU3P^;6gMbi5M2hauZHmF0KQ_lxLWeN(2?N3Ov&#Xr6y)6N}A8D>vjhk(Z)TG(& zx5NjBXsnF4e`8JF(6f{zg=Jkrp|!d{xG`TEJ}aunQ}<}Tng(!Oj@1g#@ZgtdTbRxM z4K2ws7;PCqUZia=zzb9kMcq`Y|g<+aF`6x?cH|f*u9mzN6H#hMhV6WKKu-# z9nbxDs7-jewY)b|wkmjkJG-zQ(^46$I`AnHokF^Ra2~php z+hI*1`B7!^XHzAF4gXP4a_XJ@VW6!pG5X4yG2$w;d$tkROa+&HFxq( z#)U2kqJF%qm8w5P{3;F8f?LXtMN3UR&h8!AfLk^h**~5I2T1fZ_M8oDmYtt`<`-@U zaCO;{0FmBk_vO(`SSDA!r(zfP|9|#05zbuBBqBnOYlhT|Fyv|_2_tp5&XTN Date: Wed, 7 May 2025 22:43:18 +0000 Subject: [PATCH 539/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10741 -> 10740 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 497075d36..b27bf45dd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-07 22:41:20,725 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-07 22:41:20,725 submission_checker1.py:3308 INFO] --- -[2025-05-07 22:41:20,725 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-07 22:43:13,316 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-07 22:43:13,316 submission_checker1.py:3308 INFO] --- +[2025-05-07 22:43:13,316 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-07 22:41:20,725 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-07 22:43:13,316 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-07 22:41:20,725 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-07 22:43:13,316 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-07 22:41:20,725 submission_checker1.py:3324 INFO] --- -[2025-05-07 22:41:20,725 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-07 22:41:20,725 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-07 22:41:20,725 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-07 22:41:20,726 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-07 22:41:20,726 submission_checker1.py:3345 INFO] --- -[2025-05-07 22:41:20,726 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-07 22:41:21,788 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-07 22:43:13,316 submission_checker1.py:3324 INFO] --- +[2025-05-07 22:43:13,316 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-07 22:43:13,316 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-07 22:43:13,316 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-07 22:43:13,316 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-07 22:43:13,316 submission_checker1.py:3345 INFO] --- +[2025-05-07 22:43:13,316 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-07 22:43:14,306 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,4 +50,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.7149 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9539 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 0a7f95a2c..785c0c101 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1166 +1167 diff --git a/summary.csv b/summary.csv index a63f1d448..11f016beb 100644 --- a/summary.csv +++ b/summary.csv @@ -28,4 +28,4 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.7149,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9539,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a42d19135..93475e88b 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.7149, + "Result":20.9539, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b41533251347eadd8496844a86c2ba37750af1b9..27e23e07e36394c7dacdd699a420641f3678dba4 100644 GIT binary patch delta 586 zcmV-Q0=511Q}k1?atH*u0iXM`b_fRye_9~v+H#d_Z+%70B%WW#r}224Hu_0^x>l8& z*46j+T&lxQtFO4ZkLSlrrh`%OdAz;x>I-?Y(e%2J{>aooth z&En%`{B@Q;L&ap9jltL9>Pvnav-v2QkFsy8qZi*Yv>5kJCduIGa!`xu=&wcmUi3-- ziFf#<@BQ*?L;un^Zsb$Hi{`Ui9;Nt!4eR7+*1L|+d-LnjG;NIIVLnl-7S{26*1?-GV1SFFt zBp!cBYr`-Qh41|pMt56Yx*==_+LBWPg{Cz0CZvfYYAr}=o&5VM-jYC01zn`~=9@Qz z%Eex`(Jxphw5m{+q$mQ#G#9F=(8qciKcdKasyJ=6f(jkLp+$XDGDDcQ@Mg6E>jk(d zfRrQ5RA}eDAs9Qh14&&H94d%gZ6)>LY8!uSDEp=j;5<$5u>?;!^%Nh0v6+czL~=GG z&5vz+0B0Clkf6N7S(4$C*MpVrYC*4@Ii>Ii16Qq+IJ@^w%*L+klCJ2jA%BK9tJn8_ z)L5wFsu-ZU@c{cH7AKi=W#AKEH9>kls%QGICyi$f(sIS z5<%KiZY#74!BUJp-+^L2iw+&djd6;Fcy)hUY#IM%Tfjx0-(v-V2^JVW08={>(Ge;5 zh_pYh=>dXcXh4Au9+QmVvp0ZKezKrfE}Tk+&cdYiB#!RAm!q+5+pL}S)|j8*_44(- zA2pTwuqqCyE(NFD0Sm^}CBD4G#Wl=_WnA$_f_UnpKJj)OWT*p?!~k@Nds04U&o3pb z1*(g@xJ&bR9#%y`ZwW2P*P-QA{TZl^P5wE&vt}jA0tAENr0KIXC_MoM Z7G&t?C6jh3Q~_s`%PAuU87BY$001|ZDFpxk diff --git a/summary_results.json b/summary_results.json index db47a0781..2d9e4d7ec 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1076,7 +1076,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.7149, + "Performance_Result": 20.9539, "Performance_Units": "Samples/s" } ] \ No newline at end of file From aa3ac9b40bef69c8e2fe6103fdb8b180f7910981 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 8 May 2025 05:00:56 +0000 Subject: [PATCH 540/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10740 -> 10740 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index b27bf45dd..dfcbe2c2f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-07 22:43:13,316 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-07 22:43:13,316 submission_checker1.py:3308 INFO] --- -[2025-05-07 22:43:13,316 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-08 05:00:51,412 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-08 05:00:51,412 submission_checker1.py:3308 INFO] --- +[2025-05-08 05:00:51,412 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-07 22:43:13,316 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-08 05:00:51,412 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-07 22:43:13,316 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-08 05:00:51,412 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-07 22:43:13,316 submission_checker1.py:3324 INFO] --- -[2025-05-07 22:43:13,316 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-07 22:43:13,316 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-07 22:43:13,316 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-07 22:43:13,316 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-07 22:43:13,316 submission_checker1.py:3345 INFO] --- -[2025-05-07 22:43:13,316 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-07 22:43:14,306 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-08 05:00:51,412 submission_checker1.py:3324 INFO] --- +[2025-05-08 05:00:51,412 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-08 05:00:51,412 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-08 05:00:51,412 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-08 05:00:51,412 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-08 05:00:51,413 submission_checker1.py:3345 INFO] --- +[2025-05-08 05:00:51,413 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-08 05:00:52,406 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,8 +38,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0501 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 785c0c101..ddbefde5c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1167 +1168 diff --git a/summary.csv b/summary.csv index 11f016beb..cd1a05aad 100644 --- a/summary.csv +++ b/summary.csv @@ -16,8 +16,8 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0501,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 93475e88b..5cfdcbb27 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0501, + "Result":23.0274, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -529,7 +529,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -559,7 +559,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 27e23e07e36394c7dacdd699a420641f3678dba4..49704957b69dbf12b8906e986478d36a0058a61e 100644 GIT binary patch delta 988 zcmV<210($OQ}k1?atH*HGh8{db_jk7f3#nAakj+!(d46hr1wKIq4a)82GOkG{jdz6 z$*dhf)8zfIqX3#F?MH+511%97CZY-24=KZCvx@ekYu0{#4x0Na|7a=yDBml4uUR7h zXtYRtDgTg+vZVb(LI`IC`-f!!PLhEwtx5i29RQmq`A38N11%9b2#B&!)n!$HeBdN4a z_7CeoQfca-V}tzzEfG6yAj(2nmy`jPtZTs1-aGr}$C|RW%&!WTwzZ3;vi;HzdoQtn zZ1hNcY5$OnvZVe)LWpGr{fA`$f6FTRk8RR_*inE>lm26a{sS!$8>akx@pCBavNCv+ zNkZ@@)80G%N7%B%0!j5K!JwA8Re5h|Z<{~bYK4I=x$I+?h(9(uB)$}XNJd%G{2?Jk zvV!@;GJqr-5Flw%f7np~Nt61sdGH=dP=DAk<>&dILs^%VA(F%o0wnibfBjh_89oZh zJ#~IAZ7q;=ZMjOex4xog63?&W(|9~i8~r3dU8~AX>*{-ZF4f_u)mPly$MfSQ)4{0t zJl@`T^@TjyXnNg9|5|<3k9TU#5;V__kQ`cp?_%{H}a|9Mf2G$ zk5c@=hIMi@>s`m^z4`TMnl{GqFrTQ^@|%tM>T83RKWE8o`IFmhd`_|~ncV$$6ZZ!3 zy!g#-Hik)(-ThLWlFipIo%|0_O9KQH000080000#0OxW7<6Qv&vu_o$4g_(2FB6j` zBp-jxYQr!PgztR{qq{BH!6gXWfwtt-K%p)zy$NZPh+02LY8}6Q6(?~L$SI(UG_&8% zE~*wg)g^!6yfnH-c}7qInj0bYrbZv_GM%Hu2c`wAebVLe1 zA?>g0Mu6ZLI#8g4$9b0Hvp0ZK{$|0bTsV~sgN2*clQ_9|UQWio@3Ve3T4Vkkf305M z$5B(M_p9Q7no4lW9k5_rQ{l@?TwKR|Sj82uC5WdX>NnnwgBg;E?HUyQD7_z&K{P9PKP&@i zGHVCWG>t*Fq|($u#|HZcS|WDZK$L~DE-3>nS=WH2y?6G{k2Pg$nO_wwZEF`xW&5Qc_FiKD z*yxe?(*7YCWl8;qgb>RL`VY$hf0kABAKRq=u%iH%CjG|-{RdhiHca{X;^$D-Wo7Ut zlZ4<+roDIikFaHj1(NDff&kc_gV`9ng8 zWCinwWdKPwAVAWj{;;C}k|yl8&*46j+T&lxQtFO4ZkLSlrrh`%O zdAz;x>I-?Y(e%2J{>aooth&En%`{B@Q;L&ap9jltL9>Pvna zv-v2QkFsy8qZi*Yv>5kJe2gqu>FBRT{PaoxiFf#<@BQ*?L;un^Zsb$Hi{`Ui z9;Nt!4eR7+*1L|+d-LnjG;NIIVLnl-jk(d zfRrQ5RA}eDAs9Qh14&&H94d%gZ6)>LY8!uSDEp=j;5<$5u>?;!^%Nh0v6+czL~=GG z&5vz+0B0Clkf6N7S(4$C*MpVrYC*4@Ii>Ii16Qq+IJ@^w%*L+klCJ2jA%BK9tJn8_ z)L5wFsu-ZU Date: Fri, 9 May 2025 05:48:38 +0000 Subject: [PATCH 541/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10740 -> 10744 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index dfcbe2c2f..5ac76eaea 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-08 05:00:51,412 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-08 05:00:51,412 submission_checker1.py:3308 INFO] --- -[2025-05-08 05:00:51,412 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-09 05:48:33,079 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 +[2025-05-09 05:48:33,079 submission_checker1.py:3308 INFO] --- +[2025-05-09 05:48:33,079 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-08 05:00:51,412 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-09 05:48:33,080 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 -[2025-05-08 05:00:51,412 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-09 05:48:33,080 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-08 05:00:51,412 submission_checker1.py:3324 INFO] --- -[2025-05-08 05:00:51,412 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-08 05:00:51,412 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-08 05:00:51,412 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-08 05:00:51,412 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-08 05:00:51,413 submission_checker1.py:3345 INFO] --- -[2025-05-08 05:00:51,413 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-08 05:00:52,406 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-09 05:48:33,080 submission_checker1.py:3324 INFO] --- +[2025-05-09 05:48:33,080 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 +[2025-05-09 05:48:33,080 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-09 05:48:33,080 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 +[2025-05-09 05:48:33,080 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-09 05:48:33,080 submission_checker1.py:3345 INFO] --- +[2025-05-09 05:48:33,080 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-09 05:48:34,074 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71468 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74186 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -44,7 +44,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 20 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68023 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32683 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ddbefde5c..34684091a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1168 +1169 diff --git a/summary.csv b/summary.csv index cd1a05aad..b0920d3fe 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71468,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74186,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -22,7 +22,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68023,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32683,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5cfdcbb27..c6d653d70 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71468, + "Result":1.74186, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -139,7 +139,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -699,7 +699,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.68023, + "Result":6.32683, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 49704957b69dbf12b8906e986478d36a0058a61e..f1f2d0856a9ff7dc52a524f4a65b0066650cbd01 100644 GIT binary patch delta 3523 zcmYLMc{r47A0C9rt}JCz8WSC)F_utcD`VfH>`OCtbyBj=Yb(YQX38?QG)x>qGsuz< zVocVMu~lO`IF2kCyKnS;*Y&-BKJRti*L~l=`~E%GvthGkQx$leGme50{)i7gt^`@JURhkr=r;JAnfnt}RQUCK5lLS@ z-M@HvbqDwP(DXh-gR&+-T6i!GrFjDJZ|ugJu@{@Fvp+L@4!V@HEyC;%mi8K6^v*rF zU!_1ub{wm18YxmIsM%eU>Mh$y=Zhy|_GA_=Ls#LBL9o=$CIx9sm`ZU`ExJ6trR-33l}xDLK4c;j)Y06){m$W)J=e(G%Cm=;+t^$F;hINjbh z<&o_^F6Ag70`6H2T6|hqYaT&8FEE*`f;Lrl^1pVEl;kJpyt=#O372T^z}~yF ze}f>4N7xCL262~J&%W_{@kVXf-KHkMqOoIrH9)G;&!LQ{WVrPz^OyOvIjXqd2W>sg zrBi*$nnRih#r;>moClne-_!v2>fZZRt%aWSTG?#B>N_b0t`uECQ@QCst<)mMe7$F< z>3qiPx;_%IOCUA*f>Qpy7sRKyExR!?Kh zWf-9dS>XpCUD6Ujo53PJD>q$m3Evk^VUOp#;1fbPo6ZYASk!{Z~YSoisy2J>()5qg5*f5;r2XPT?@KC)OF5nKt~_ z!=|l$SlE|q;SCc4#oL&_!=6tnTvy+v7O6_kALh-3=k2|=*OqokA+?3%uZ%Lv6wE1Q zk5)rr#z{REVL?l3%XVt1)3a)2w*QbE<_8uUFlS$HL!kk041-m;bMh;9D0E$*Rs%`; zQn}s%r@gb`UK+ARWD6L6O>sRFtFe^w$!9u~IFhC?y0VyM{UIP_y-@M+HmrhK_Mmha zN~GK2=T2w=?`~|@U(SZ8dUV~VR?Qgy`QvNX%XO4s+rSG&HomN#hD^6;=JjwCQBT6pOYxU~Qh`0W^9>**p!dMwB z@}!Z$D+vip$gIc>{ZMY2a7Rm&%m_zfv_bk%0_89R;4}HRPALWuObjwDW=|}L$wLqL z_%%C|!TeW~&XeWxKg~<NfK7I5gaE^yPDsAmC;NhlN4lH$sU*aLo81BOVC=Ayq6_i z>LVIxd}*ouuFnXXmU$k@rGb*KJ?-LPJM$r8L!Ki1{e6C{8lpf)5j9yR$ zFLRpnWbOU6tMcSF-T8J5He1I;nK~Q>!uy~=rZ_g`E3tkvP=i*~HBi>mvzo?J_8PGn z3E&$pBG=-{N`J)G;NslgP7qqI$JEzYi=liQOV37of*+QXQ5D>z5%GeY=#>}m<8SRv zeNlPx?xB;#D{OX4y0-_{07Oy;j{=!}7J}E{-wKEW-^rJj99&S%EaJ`LJ(_cOW{sD^ z+;W)DE2hKHyY?93M)t?u)QZdWcGLb0e-AD?MA8k9iZTlf7NxpIh zcOUVaf@&Y}YeS3b*ZFH6%Bc+#A6Ex`emh8B@V}hn{^i{yj&?2a8yz71d|H172?wYF z!M9&wB@K>9;n?6;^!xpD`dO+8EY&&OD+A9Jg|&EJ${mVS$B)k zl%__1u7KrC#T;3sM7{bJe%YKX9&IYrGmQnLX2dm$V&|_~AdIrWa{dcOKLSg<<<(nQ zPV4@Y-Y%q+1N@6f?&e#99N8aOTy}Yq2 z;s1oIS)Eud8qs8i-r8qa(N*2kN zbnsN|D9;|XJ`Qi*?C-m!6E4m={*9RN%U8N=E%jti%j8!_AfaglLF%~|rwpVs z*e+p*V{Q>p(qkN!K5LQBs~_~fl=U!7$1$%y4>FSUpthfFA$w9@Yi1k}t0rv5=bOfc zfygm)p}1ucg*Fids|%||KlNdSa14^`ZTa`TDn2)AIoQ?1XAm=L6=jTP5_R^XGZOf1 z4jL05$}N#$5lK|N{C2YEA*1J9#cG33sirpZh8Y)R3#N(Y86owu^2-@M2&Y@v!d~C z3erFstXV0TZ3=4fy7}M$c-L=4nT&IXX|T~&a)mp0nO_EQfB_VkLwli>?|KRtu7Ou; zZvb1*^N=}IU*d? zV;^UCR*cJ{UR`GxEybhseD>4$aQ3Y`?vm$Ntu)1Nv}uB1X1%Fg^V|+HfkJvv4MY*2 zI@vZ?+L^Mx{;+G}_JZMxF(RPz#3Zh-VBj8JROBY`n9$(dkoU{D3sTT_A@68B>-sRH zC+B=TI*PT6U}oKdOQ0^eB;s_kv^Ta(_jtD3UK-2^UM7KP@SL8XPphx&P8BPdo!%<6 zd5o?#*dCZ6h83-`qI8QDrGfI7tSG&t#)0c#e%41(GQ+PDTeps)Bs#x)3 zWu86VeN4)B7XNa7A|f-)aZmmp!Mph%hCh}`q#81jD=fhaEWxkrTX)@Ad(2)dibUOY zH)NG^ER}1SrC80>K(t!g`YFxx)Ux5%^8Gz2IMeoE!1}=I!|hEMiowBvclws~!Ffrv z$!>g{Q19=k0J#tjM+f{K%Q`i)wN$R{-_{AC3;&}11u`QYfC#%q2YdV^f5r7O19Y89 z@v^To*+WxyGd1=!_VThc&Z>3@KprnaNgA;teaZf&^+MJ^(j(DDgG0KVY`ZM9$74Fr zpOBU0ITY-_Mioh$kBLrj&f|f zA>CQTY6d71y5GsR|6AntHiLERpwT~}__*(8Buem3Y85)36Srs;qpSF^Y;9_gt_D<3 z@^Lr^(mUma0jWpG2>FAwWODJgzFq=~b3GmnGEzIIGgx%M0@lVse|fEczlM8xQ03m; zw~jd4;QA1(^fBbW^*fZCu!xhm@WnkRO^0&}nt?C;10rwqK=NHXU$2JbI^fe!$^rq! zz?%W9pH`I%N+vhXj70pZ(WMnoP>|X^YQUWWyn+cdxOuX@Nmu(a=HclPWg~KWadj5)x*4( zhGO-Tb}CR$l>>hC_;QS#_|pm`Z90c(H95P#d+}G&|E0 zO1SRElOMP!e;ssJMPz`7N>6|zYQdmfcW>R8H32W9r?HZ^ty7hs^Kf*ciBNUAW zrg`$GiX~iyMEQB(se0L0ru)#HlD5AjUN+1mE1wwhbIjb@5fS@pJ%8K1!;x*TRB)$# z+$ZfW_@~^lbDecwvy>1F)c&$n-s^1z^R`f@*D8UCi#Jw-yhCjOwr(3dAleso_up{~ zOO|V%vfb?eduXZ3IkGw7)8(|;=frU0V)$9POKczULh_nyaJ-2;^vrK>*+3vmb`VGy a^#3m^`V>A{{yLj4o+f{ROX}j$pZ@@foyyGs delta 3552 zcmYk9dpr|t8^e(_w&B@pZEQ_zn|ZA{jTdiY&fhqlm{IYz@K$J-%PlFP#v3B zVt{(tninv9y{cIJcF=oRU=H(3c#zl;?Q_w`!8zH&`e&^p(p( zjm(UU?iObbHS{iJdPXG+q~pl+(H{3UmP#h~(>v1S-9Y<0EJ&;5*b&{VR(i*sK-Lb2 zZESf1znMFOkiDH!AL|`4RL`IcVsF+xTe~xErQxqk4I>%DY^MlrM6O;_KruCZNCuh& znW(CQzA>ChA$K?eCrooSOlJ@xr1 z{lmo6*^scU_}AGSLg8eEamu*HOkY=u$EjQbyK6jCCw7ADX#O=lsz$aI+fuKIi*IPr zt+Q>cPqpmVL1I`#1x2~ZpC_AA7%Ik@wS!Mr8bLZ7&k3ei`mm&&PJl(MscX)6&%!1+ z6}3d4RjmB!?)b7k4&xzi8%t(UeF~%V3>hMH;L6B7Khb0J&tU9ho7TTP4lf4AYM7 ziL{e$b?2mh6PHLm9cQI962Rt9*rQe>&wFkHP6HZyY`j$wD%wPpADJ8+UC}5C(O8-GLtuM@Y_IehE>=`UG+xQ_tk=hKvuq>Div;Wk@OG}@E49|JGDao5M! zncorXKW2jTGp4rl=j=nN8$+`v32Uevu0CU3Z}T!W%&1-(5x;0KTG4V1nd5nFW_%fy zaV$G`Wwy^B$?*fxnf_Pn1`w59g`Y!q4K6JsY*}uW^PY#b4R5U8rpE611*Y2;-3@sq&3aqaL4xoS#>@rg}8niyXPQ2sNVPG>9n<$$&(#l08@~K`8+) z$U9yfv9!{-z)_P+KJ*u-xPJN3an2)kmOfPMnx(WQO9zDblr zkfzx^+i#Ac{ZHIIAl>4&L6x9pELaLaKmlyq$E0h7?9U*90CIkOD*pbl&OBug%g34K zUHN6qHuu+HMJH%Aq&&?%;B|&T7R#s9G7m|YeFe>$dsw>KGJSRIf%3fK(Wdnk+ix7% zeu9?=q+FTw)Ktq#up2C8B5`&m`HR%l^H*znYC!{2hA({U+y*hkQI#n2!d~H=jkczQE*{aYkPYVd!;6$#{rstv zU6p|7<1hFAo=**5c#aqpuPFTZ_0hCl9~%A6&*NYwVq3n29PuT!ibr+U=5Www{Wgg% zK{b|(`rBQSBWLdOV-p}XoS)WOnz3GZkvuC{G{o~DDOHF^&HIN6`b||JQOa!E?hb;l zPf%udKlgQXF?f&^(1tS8WEoyjA8#qE$ zbx{hHEdA}`lW&v+MOUo$F!p$M+F(zgCf%Azu50kCRnxe~?1ol^O3o@g{}Kv^q>mhC zD3Fe}%Mtmsvg-J~NNhP4Pu8>ub;2RB~%F}D# zyD4pt^eS(yIj#E@92>W)j7RO8SxXn8Am9JIDdkQmfrhPPX{e2EP&9(TE-Z=(7E}{E zFZ`X~t{3eX4IzXBI!OM{4|Gey-mg2<6vuTvn4GtpdDl3Rd7OPpC0|@r?6R{FSOVS= zk4kS7E&wtCPiQ`-rw|k%hp$9y)5w~od;6C_IAdG3t+q&{0_6d8M^XnJaGe=cGf+Ydtro=jlP0u^qYV5np#y z_b#h{U9w&)ch0<!5$9p02Y@2RyQbbhfST?o z9xz27`;}J|vhUnSii^a61Qh#D&4nrXYJd{)b$ij7wf--BHkhUUO?*}CvX3!X!kWXg ziYd(ES(ORqcO{j{WBd5-dgC!#a9&~`$L6IUKwrNaS2rNs!wYyT9LJRRVE4ip*7}3~ zbCz>g$M!3O>O!qQnn%$HeL%*eo-IWwFg0fN1Iz@%0AxXC;zIx^QjWZ6C-AK~Tj7!I zT?||*0QE}OxYl!edOE%4O;6g$6O8QFL&Ny4+)n{QB*JlVb8M9hOLe$2LK%hc`u4c< z7kBeFclp2E$=}>$dC}JYaL@3#>LkZx`Rmlh<=~G)*Cs8Kx|S+-yVdJpE(JSYOZJ9W zHWjn%P8ow^y{_6}oCLd3|2piNbxzgzhe|VLA|U2o0>FZGU1HJ}6q}nY@C53w|ma+btK#W0c96d}im9H4yIXSJ&GK95-A7=Fk5PPQ@2=dXwA zH9u};;*7QRz>ShvxCOFjM^Vw760`XM77dm;qTqdn2L>2e5w>{eH^|BkJ8))ldNY@I z$30QR(}4TH>#n=V6H-RP48stNy1!cbJ^vc(s_Mo9Pw}od6Q9EjF3AuwB$ctlaISMy zZ`ys{`?V^ubL#`MMK=I7>bV$rc2AF)o9{vCXdRooyQ*xQG8mcBZaG-p`zh&;lymZf zlLXYlz4Ik4_g4CqFk3}jnto)@I%cynAL}ivvs5~Iad}WJ?aEGbTu6}C+;#eN!{(|N zNxOCNOm0cNPr0{U=X&$=fjyJkLII<3+vPd-zy*d`ml=q#Ux){twG3}fO9s6 z`^Ys#9X%N+{!ukANsBt#ss8eqey=iitX`|uBHY4mA4x`Wm24r;fjujv9Uu$LCC5pN zj(sW!FM0Q3NH;9FHRh0bq_`=|A;9;o==&?lhNCeaMoEX|S}Fq*{|XDi%J@lfZ^|?a zD5g1?y2l8_tv7$UWPZ8kcG&ZLU*f9!L?rgF>^_2S z&3(5^t&Q_kjJETwJz4B)w^6#iL+Sdf@o19u-8Q*$cUOmROdD5MdZbg!(KpJtV#QgV zksFBnoFGqXND$_VkFVl<5|^3MUT2++>z?~m&W|P6^vn;#8~?FmV_3U z>Z9DY?>|4U^C~WU&V(l_27GKnkBX^ivkU+b=BE*SFWe6uOt`5kNzhW2=7SL~t3qUc jXPFNGu-gX!hyngLq8~~S(p8Q4JPBQ@Cj^h5{I&9L3iar~ diff --git a/summary_results.json b/summary_results.json index 991eb9f46..e4b44540d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.68023, + "Performance_Result": 6.32683, "Performance_Units": "Samples/s" }, { @@ -595,7 +595,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71468, + "Performance_Result": 1.74186, "Performance_Units": "Samples/s" }, { From 1bdcc073369d9de43bdee9cc5df2907560ed91dd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 10 May 2025 23:55:13 +0000 Subject: [PATCH 542/719] Updated results summary [skip ci] --- README.md | 63 +++++++++++++++++++++--------------------- dbversion | 2 +- summary.csv | 3 +- summary.json | 64 +++++++++++++++++++++++++++++++------------ summary.xlsx | Bin 10744 -> 10833 bytes summary_results.json | 44 ++++++++++++++++++++++++++--- 6 files changed, 122 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index 5ac76eaea..01f897b7b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-09 05:48:33,079 submission_checker1.py:3301 INFO] Results=30, NoResults=0, Power Results=0 -[2025-05-09 05:48:33,079 submission_checker1.py:3308 INFO] --- -[2025-05-09 05:48:33,079 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-10 23:55:08,050 submission_checker1.py:3301 INFO] Results=31, NoResults=0, Power Results=0 +[2025-05-10 23:55:08,050 submission_checker1.py:3308 INFO] --- +[2025-05-10 23:55:08,050 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-09 05:48:33,080 submission_checker1.py:3314 INFO] Open Results=30, Open Power Results=0 +[2025-05-10 23:55:08,050 submission_checker1.py:3314 INFO] Open Results=31, Open Power Results=0 -[2025-05-09 05:48:33,080 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-10 23:55:08,050 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-09 05:48:33,080 submission_checker1.py:3324 INFO] --- -[2025-05-09 05:48:33,080 submission_checker1.py:3326 INFO] Systems=21, Power Systems=0 -[2025-05-09 05:48:33,080 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-09 05:48:33,080 submission_checker1.py:3335 INFO] Open Systems=21, Open Power Systems=0 -[2025-05-09 05:48:33,080 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-09 05:48:33,080 submission_checker1.py:3345 INFO] --- -[2025-05-09 05:48:33,080 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-09 05:48:34,074 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-10 23:55:08,050 submission_checker1.py:3324 INFO] --- +[2025-05-10 23:55:08,050 submission_checker1.py:3326 INFO] Systems=22, Power Systems=0 +[2025-05-10 23:55:08,050 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-10 23:55:08,050 submission_checker1.py:3335 INFO] Open Systems=22, Open Power Systems=0 +[2025-05-10 23:55:08,050 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-10 23:55:08,050 submission_checker1.py:3345 INFO] --- +[2025-05-10 23:55:08,050 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-10 23:55:09,086 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74186 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7158 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -34,20 +34,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32683 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9539 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 13 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1249 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32683 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9539 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 34684091a..98a603ee2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1169 +1170 diff --git a/summary.csv b/summary.csv index b0920d3fe..a2aac85ef 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74186,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7158,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -12,6 +12,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1249,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c6d653d70..3d537720b 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74186, + "Result":1.7158, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -390,6 +390,36 @@ "weight_data_types":"fp32" }, "13":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":23.1249, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +509,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -719,7 +749,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -749,7 +779,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "25":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -779,7 +809,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "26":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +839,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "27":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,7 +869,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -869,7 +899,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index f1f2d0856a9ff7dc52a524f4a65b0066650cbd01..aceeb5a795ac8c2bed10dc032e596ff936060f35 100644 GIT binary patch delta 5050 zcmYkAby(Bg`^Sg0(kWfr04c%IF_78_5kzUhG0F`C1nCb5N_~;B!H^KpA*e$dB$SYn zPLYvP0s|IQli$bXN^u2^ZRt-jt^{B;&tt zNS|sR2r8Xe6q96p(IP$g`R@UleeA%*I{Kc7(xxr<1DzUj z+9d@=)N|jq4EI-0VRMDwwV%MAHb5We3hmo8v~K3^E0x4{Sjr_=OB|w9limY~?H2|A zq!YF!PYx~!y1EonJp8xt20F+ar@j^)l&X>5_R7=s8Xt8b*N&ELt&N)dx}t2Pqs!_- z(H{8r;r`XEk%H)eH-73>T{aUx@?L2X$tmMb$9Ejdi-wrLNsX>ruc5qNi8KtcjbO@bEa*_LcA+Mw2@YQ3O`sfC@ZfPr_f(Imbjiu# zPIBU}fQ_}y9~!CXr5EJ;As>&11Y3u99^j83WJrA^m4DWx-C%JSF71U;@f9habQZhZZ1vubEU zCh0W(;t!0~PWE4tcsSR*!y=UvRFs3FtC8aD6p2UtL>Qr#1WKhA=d3EI_Ro20ZRrS(oSUqK#}T5Dz?T;FnPZCAuolsd=2K!G=*5g5wxp?pExHSs z<0u}de>~R^Zlcn#=LDFkG)Z(LI2=sy2$e1_p^0waxU$b7p!1fx58kWaxgSHsJ(c7O zWAo6pfx7yKwbYGDZdaE^9!GB7f%ilSn+)oNA{6n@)(C(^onpSw3=bq zmoolmo$KO0`KJ|cL)T9YE{mSMYC3@M!I9k!&k~7L0YN?QB~Lz)d6v&eH#}fy*ZGjI zFAtJWqK!*gJXkLlN>o;`l9B7MzC;SUJ5YVV$=;D1TEDtNPYG4xKSKca0W zH015Cb>WUQZ&dpR0;U}8jc3V2szgFPnKH*?per7dvGEc1#C5LqC{IJW53ya;^O-YK^KeK`o2`zQm{eN_c(Q&NNRMlc+WSmYCQf(` zY{?fcEQ+?xP=L3;Sp9*HbZ`9oDVKPtCx<1T28Lu9Xk{^|aKB90j`S7`S-x}kgWTGqVyy%$P>PS!oZ|32TKd)WNOc{ySp(AfZDmv=ixvMzrkZQ{F=I>+CxZbmY?A@R& zBadd_p_RoG_Ej1R45?>!LBnf+3vaD27Wr0`d7;NU2t5LS$Uy-j_@SOz{RS)6aN)+XQ+F6^6 z8PgVa(v%}6@*`pdDDCj=>FbxLB*&Z!AM)xBOP1yP@=cOk(+=1>>1<9V1l$0;2bdr) zl>rujoGy--`wjtPGI%6)A=Vx0LJu-+I-<)(iKlhZL%TAVF|6#{-WSMW$U%X&x@oa!&S5Id1#I9u!BSy)p5N@p^}^ zPd}^KwA5fAbGmnO&Cr2FhQ1>l9PNCgG{T;t1}yLm7y=)cvPGH`ev_K_qlh?`M|z^E zf9wTvP;wgZI?tsgJw+e^XV)z$_Z*!#lsu}?c&(sp^$K$a6=(AoXGX=LZ{Vo7&tL)Z z;e{hA&4RE+GWdsf`7_-j^=cYaCv1^0HzFcfGY)?oExRlgy4Oct+uFP|4#*mCPjT95 z{Z)O5s(SA~s%L^}y)#qAMrrbVR3c#KsYi!57-;{iM2ls`qfHV`JdBOu=sUYZ^+~n`nMT8Mhh2#N z8;k-MTodzVroa|spXX3rLTqk#n5_V|^*Z6a-$vNAMt+iM^Ap*wR^Pwi0vu3=AY8a4 zwRmZWINq9A8Hr#TLko05yAf&*CVDfKmRuaG`Y(Nc~NTr1+8p*^~PEA&j z^c5LX8{N+*Z#;aj_mu<=o($fQ`xVfMDW_JgPecIxg0^Y?v7t&A{(zlX0?)fm^%^+C zo)(dLM76HC&lZ6F1>`}{!>O)O_o;c;kpXez7qF(g3#cI&*f0CJR*6mUA+)L|Arp^F zk~2*b$;+@z3w*nHpW@d#zlpk&EAP$)?D-u3x%O}q`yoh8cqun+Dz~UnUQVRHSg;dt zT`igQW4D51+G+*G&u+t6;UVM{kA3An z5D^9)lyQ3fWh*nO?LhA14S3;Z*dIy4Fg7fH7?v}A5-9Q~c2M$Ep}vNsapf-1snoec z9=vkGSmDh3jOJ{zgUq)IIuJ-vfN;Vm2MEmA#Ju6b@u=2b#aVdd$lN+NCo(gNzqGJn z#|Cb(=K>2|vU-zEJXo%*b1DdR9rZaQ>eD%=qrEQf9RTY6q?1tb#b@c~&g3uc$oqI> zaOb6v@!*$5zh9l&B`dSFj{fg?ps0q@6$`>+b{K@^JAH4AM@G8r?A(l>NBPTqFbZH# z-GV8XdDMEE%@xsp`p~uK)3HNmp4N7H<3Xtb&Ntvdmv1-p1CVA4Tgmt^a|z={+jluL zMWrum&xOx_l&&!KI`iF^$jdg5So)(Q$BF$u9f7MK9 zzu)1V-@{|zkOoerW0U3o0fE!blHxT7ROGua3kqZI)yJF0aHqVVgZ9*!^#J5Vh%4o+*!Se~b)S5U(v8qtIz1I0A5?Zd?@gc&FQAm{ zsyHAn0?eH6+m|piUxouc9Iuz9SWZK?(+X=ZV}ho}VILTbZ{c_l*X8o1a|?2tb$crg zHmv%arkZNg@te1qOb8pa-+h=_{F8$0k3EV(hA~-2JI&vP@UQ6vAf$ z(xKg>8xv)5RvJ5#eugP;U6MU=BCCY7N(ocR?DO)nBXnDToEWOF=3GO~950SXH6+(J z>bQ$(iT=2tE5&GdE|S;$SWI#|OOm(LY~kh{0@+ z<8KPS^RBSIJ5dazKxY{;VvRQ^bAQV_rN5+Umrcv8D9*QKdF$l`fotD-eDCWB&rIxb z!;27AVOeJX>cfPOmuv#e{qp5PSs+2korsE*3UJh~AGN2xvAb3YL7(Mphs+>8DS z1(L-o2;tAxY47VqEa@11jabD#E}ME*J)`4vOQ-rXJe;7g-M?!yN18#`D;$boeRr|+ zLaw7mh(WX?+Z<6QNP93a12tN9_=BeHwc+MP(adMU2{9mxk}KbGkiF zgp7ISwR3&ubLqg%D)GqCoHMmWv!Q z8{ub3^$L`B2-8GV^}>k$3t5Ga%szxTM>yiPNMks!^nI?+vK?P~BDCvY=WN)fA4UBh zT4q=jFktq+6El*<5MIQ$!UEB|s9^5}gL#9JzpoqiDlE1)XIXa4`6nV!rIns Ypvb7Z1e~lMO$(u4R*E4&`fu0&0jPM3c>n+a delta 4962 zcmZ8lc{r5uza0?~*~%`ZA<7nG31LF^y~w^ac2cr$uPwVUBm0(NkhK}fk}brT?CaRd zI+i5+&K>>keeQkk`{(C*pYuK6^Ev19dER&6>48C&`xUBiOc4DyY7pp~BO>4mKcK7U z+x0XaI{jEOb-koL-K#<-(``-)vvm!%@_L>ZX9q-<8%{Jw-)*c;|1sxy)+Lgz<7a%fc2fVocj>iT zl|V+E=|oNAXn|yggyAEO-qM{U>PTeJ3HORH?5SI3i=$Fq= zl{0t{@`kh}o)o3}i{;Mm%Oysv4L-Z-G<0llx^h%Ho0KAjR1We}ceURwVVRwqWFJef z-RO&x9+s9DJk8%VA25q6{|vmW{pwt`<$K+JW3T<8(=-!jCF6!ImID7*Pa%)b6wZUreP-ve{asMk$^ z$mVQodnW~Bu4Z(0dd;Ecq;6%0<^%eebh)&HzgBa1coz^(t1s+PLfOEI7;~<{CR>m; z_nfah5B=+J7Ku@yJ#PlbjVBiHs9!5LqOaswAfkLOH{Payy($Cd&S@5lb9n7_hk@RH zKTD|{zYdR*YH(9i%9_4zn=YX(~@%c5YFHXhNO9s!fvFn;Ce};??cBpoKQX z*m>i@si@!3BmerTYlVkF%YMnz0&R6rY3 z>apo7svg^;8L@RO z<$j$&j_X*gJkUu$`bGJ$PB*sLW%bgvz0p~L zIJocm(yF0kBPy`pl5e`A#fHF`AzfStjCYUeAe)HjzfKM)2He|YB z`e^QG)$3|X`7A8D;6(;yMPCXWeMd_$_b6#hh*HvU6o5HTq1O0QD-;UQ#&~KLUW&;L z&9)tIbZ$0BbJ9GFy^ZF}`A%S=Y!}gfa7h3j1xr!_$Fk_v1dv{5vx4oq%R3csgkO3u zhNnxq(-QQQ$x|&U0J*~?=6ZO>Q^s7YR+(EsF+YQjtPI&l?xQ?4D=&qo&N78Vhdj3$ z`R#S_0LM4mhCE%_FMVpfiK(|$$)#Ydl-xK~2DK-6ck`)mme^_vN@Pi7?@}wxyMBgG z$gWs1g6=4f?N-LT>8QAw!FWbXkb-aXO(unFb3Q4}V1i?SXK4ns2^=Ju^7%*ArBtp_ zQqzzXanAM2+LUQ0ACJY}8sW{2LZj0aG(@l?fFB3}18KrZL>`1WFYqe0!XAOrB5q2n z&lpREr9^QmTQEF|L<`*y`wS1a`ZSf%A{ScsS)U2!)KGFW#E$b#85&kWff{Ab%?#Oi z|26XI$;^=0+b?g-bn>IqTap}X$Opiz3d9q%e$YWaQ)-Qp<~c5Z!1(^Xsf8zA=2ixP zF4O4#g}%3DVgYL=-ldtjM|$b`qWYSjRpx$j#k`;Gv2hS`C;i)TLWMBCU9&&M#fBUY zX0<}Vg0#y{bKbd>TZr zxCm!j4}F=2;~vHQ!~>|m5kMWgDg^85ZImC)s&v5Tqtw>oD{N__M*@h)7aob?z)~Tn8_m)^SG{sh9@(PQWnc=mq-MX zP$M6QU81R4$rvYsW&Q^Ye*qTr)V}va8Ls;k9%!c%%KRfA*VF0Z_z!Vt+CpL)#awUe zkn&uH%9BpVNe0ZQ>J|y3>!Dh@h682%m?nW}ZCzU@>4zN3P?6nqh#uFY(|YBj$|{uk zV=lxNqa9DM|4>(HkuY(?-@{?j_Hp*a1OMOt5_(rR3nmmrLp=Od$?;`Y`&HCcYEJ>C zul_co0S>e+D77s;{nm%XrgL20At_!(-%~(fWrOW{B>G4lsM3EOmrNP75gNC=i(MLm zCWRk#J=1;6!^WLc9iYmam_duZ_5OwuR|?6!=#kK;88EI1QeDTbz~t)Jy?Hz~DoODD zhsmC5tTx#8n=NQNOp$cT8M121U=koSPK*RjXw!wSGYGUX2%k_EvycqS<5Q@EKILVCE;A#Mk6yG|xEoEd&|MLJMbV zfa;;@#&STXDz@-mD}x||&V)ReQT>?MD;(7xEe@1I?3IAl9<2~%nHly6JtwaPz^uOpVW(Bsf;A93Nazt!TMn|f zvKPuubr>cFdGrkSDVm+p-Kk7bGn~M_G?h1z4D(9wteE71<;%^ftwq8fJDw&|!;M>y ztXXdnQ>lsAX_E$tYV{_N6DUl$-RU0lO34>cOQN4;ahYSbx8HP48Lg;nsLQ){UY&*m zeYpcKDKq$p%#Mi6)QQX-h{rxB?gefF+;7dW@=PWDBEw-rTI?Jg4{ZG(vePE@O#b`8`8VU=f# z<8p}Zf?i3tI!OdX%^;;LM6`JVA+fHqJAo;8asJ?gftPKK(&4}Y(yw5P=*Fi+Hx88L z5#5-F4d<*Cb=JRdqZVQl*?Mr{Mn;$a5EGAPCM&h;>F(Rd9H!B%=fudT`k9{azszuG z{ypasy^qAI?5l1NC0ij%mT%m8Yz+{nqp?gd3VUp=Ld^9<0{J>oGzm>5#%eD82%O!G zb(OKQ(-RKJ{2)#(g&6O>z65bw=XA>( z0-XNL8jA;oTzeKa=Tgf_2ZR5(of5g>sE#g@@N9>xvw)Z_P%7_fC&}r*J02d+5dhs$ zTid_9i11%pD2(?}LX~agW%#OIC{*xG>DJ63UIM6|rY1Fa$9MA60~{Wz0sLo)ap=NB z#m7;w%iEE*AT^0w3WEh_L}WPcuKo6}NASF}D(mjPZTX8Gpa_oUB2)cScZ7K&8gL!% zzj|UOZE|Zx+WozYYoKxuI0vxoEZX$RGC2jyo0oQLt*!d|3ysVcjTlW$b#Bk|47C;? z$j;38?7o%7JNUU40_9Dsoiehgij&Irr?ZPT9H#g$#sO1N&AnUp#c3{WYyq4VO$(hF zJu29M z(F`@3U*{)H+#^m-yF*|s8r)d&CI0tA`drzD-Z+nQoOE9cX}PfpQ_&-`@|$_ePro~U zl3t&l$zq4nifG5A!EB*A{t3yuP1I-0V&6l?O{#`w9uY>O9ns4vG^ihz{!Tr%*df+NM)7(ga4 z7-N1I^g2$9XFW=U+`&OK>b|Zdz0U9+xc{1YFull=Rqwkx7T7InXHkYA#b>u{xCL`k zDbJx!{KZB*skECL-E64IN3Ws=2aef#aU!*UB9RR?lBvO^Q(r~&Tz)0FLS&< zYSe$=fLQBvLZNEn$Z4pO*>IlTQ21y*Sqc5Kc)dq4s>=JZ6x5GqAt*}v^siCf+-gQk zGt_brd{fLgZbU#?P4dZ;_VUdR>AySG?=(yANuaTlaZP_cj2vi$O&cOo^Z2OYmr9)8 z?))D5Emn)P9XLCkWgn-Umf>d=Ro?d~+ zo^jg2k$G;`WG#%rI>GC0{o8GVuWkp^P2AifsjWMcXvz>q&Esu?ydE~Pd}$DDbY_QA zP|WCz2~l7AeXCmsGmGj<+Znh=uqiJrN#UR%UB_j%0S;v5B`R8;BM6`}9P3 zbh+cdixY4Qwn~RDx7{X7lk5k`jRB=+yTg&V{O4o&1BDOYXH->qJf;3fu;q~#)(fk8 z;S}?`E$F9iA=|qwC7#K^@)_2Wn%!Erda4`vN}U&)jve>(#p-<&L=r7kwAPLQ7a6pP zv;?VfJg7o^yCm|cI8rPe$9U3$rSdRyW1>3ab`JgN^4ZhkL4M5&X*_iY{XRA9E zXVvO$G4d072@l2haK0+jWvB%Z-RM0+1hoW>Z=Xe#t3#zAw9+4nPR;%lEiB^`6yCZ= zpYoe>9F_W=p_|EEZoz(t@mF|1O<``D@Es=VPwc-n-DH=2i66c;t^G6tw8540r5zmy z>bG)s+B|$7qzf&ztr^RBZ|1FEhAvO0a85vfMK=-<0~ppxIqIr43L)-2!Le%#{4I4- zhIuSqvL&03SnBn=a_z*|bEb{su`FCC6c`Q;>nlKLY7aFp2?(Tn2?U}Cfj}^~7rO3l zuigm1aC5g6@^N*^hU1*tMCg?FPpOcNg7O(~s>zTV*oeXhewt}Y$GIpITJYz0=S?vP zDN3Jb{ZHz>d#6~R3hklegIc9OJbQ}U6c<%8lZD-fR9cCH=kriXvVv6!3bm)Kj`I6Y8sNBNF4fds^%*z8P{>lsWFJ z{BocdVhjV6SbP*jzJ?Q&W$geB-MVJ479ngwYgmsBh1R*N&-#-hMln+L``U5eE7sr@ zEM>QW2+7j1ee2=p#cdy1^6D4jM6M1ytC^-A95FDB=@X2sJ4{JVN@$PTCmj=CaQ@{Z zyVY51zliY(G8Wd$E;T5#=hVeeLB0KdYeZ|sPd{0~VWh+=+T_W$3J|9$%g UVufFh#0kN7?+!V~-HZBv0nL7nBLDyZ diff --git a/summary_results.json b/summary_results.json index e4b44540d..5ba85da65 100644 --- a/summary_results.json +++ b/summary_results.json @@ -608,7 +608,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74186, + "Performance_Result": 1.7158, "Performance_Units": "Samples/s" }, { @@ -941,6 +941,42 @@ "Availability": "available", "Category": "open", "Suite": "edge", + "System": "gh_ubuntu-latest x86", + "Platform": "gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "AMD EPYC 7763 64-Core Processor", + "host_processors_per_node": 1, + "host_processor_core_count": 2, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline", + "Software": "onnxruntime v1.22.0", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 23.1249, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0028", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", "System": "gh_ubuntu-latest_x86", "Platform": "gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "UsedModel": "retinanet", @@ -972,7 +1008,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0028", + "ID": "5.0-0029", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1008,7 +1044,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0029", + "ID": "5.0-0030", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1044,7 +1080,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0030", + "ID": "5.0-0031", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From 7b93b3733466cd6214fc0b16a9574298908f3693 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 10 May 2025 23:58:57 +0000 Subject: [PATCH 543/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10833 -> 10831 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 01f897b7b..330dd0535 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-10 23:55:08,050 submission_checker1.py:3301 INFO] Results=31, NoResults=0, Power Results=0 -[2025-05-10 23:55:08,050 submission_checker1.py:3308 INFO] --- -[2025-05-10 23:55:08,050 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-10 23:58:52,243 submission_checker1.py:3301 INFO] Results=31, NoResults=0, Power Results=0 +[2025-05-10 23:58:52,243 submission_checker1.py:3308 INFO] --- +[2025-05-10 23:58:52,243 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-10 23:55:08,050 submission_checker1.py:3314 INFO] Open Results=31, Open Power Results=0 +[2025-05-10 23:58:52,243 submission_checker1.py:3314 INFO] Open Results=31, Open Power Results=0 -[2025-05-10 23:55:08,050 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-10 23:58:52,243 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-10 23:55:08,050 submission_checker1.py:3324 INFO] --- -[2025-05-10 23:55:08,050 submission_checker1.py:3326 INFO] Systems=22, Power Systems=0 -[2025-05-10 23:55:08,050 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-10 23:55:08,050 submission_checker1.py:3335 INFO] Open Systems=22, Open Power Systems=0 -[2025-05-10 23:55:08,050 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-10 23:55:08,050 submission_checker1.py:3345 INFO] --- -[2025-05-10 23:55:08,050 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-10 23:55:09,086 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-10 23:58:52,243 submission_checker1.py:3324 INFO] --- +[2025-05-10 23:58:52,243 submission_checker1.py:3326 INFO] Systems=22, Power Systems=0 +[2025-05-10 23:58:52,243 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-10 23:58:52,243 submission_checker1.py:3335 INFO] Open Systems=22, Open Power Systems=0 +[2025-05-10 23:58:52,243 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-10 23:58:52,243 submission_checker1.py:3345 INFO] --- +[2025-05-10 23:58:52,243 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-10 23:58:53,522 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,12 +25,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7158 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71494 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417812 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421368 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32683 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 98a603ee2..75fccd696 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1170 +1171 diff --git a/summary.csv b/summary.csv index a2aac85ef..67ba85fde 100644 --- a/summary.csv +++ b/summary.csv @@ -3,12 +3,12 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7158,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71494,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417812,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421368,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32683,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3d537720b..6514f4049 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7158, + "Result":1.71494, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -279,7 +279,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417812, + "Result":0.421368, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -289,7 +289,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -729,7 +729,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.32683, + "Result":7.44519, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index aceeb5a795ac8c2bed10dc032e596ff936060f35..3dde2ee8b1c3bdc16f0f98cd85363a353306a0e2 100644 GIT binary patch delta 3769 zcmYk9dpOg5+{ZUUi8zl z)dnCv?)do?Jvrt6$?ON1&MSILDv`Zwzbe@)&9RZMhfGVy{d$h8^lw(O^{=HjWN>EZ zJT&*t)g?^kCkLECC#DZ#kQeb4{kxC#=Cj@dxWiVfPEqZIJ_gHq)_%zknHU`D+KU@y zQl>ws)b*BC{rc&R%}T`SocJ?2YX+Q+$r*Y_4$@n8Wv}%w^s0HjIw5j}?fL1+vDLu9 zQNlY>D}xdL(h%RW6WjgpU8>j4D4fDlHrrJT4bs=~a=ct7GIWIDUi0T2B&_aX8@gs_Q}-Bd@N!%HLa&SXkt5YZa~geW zX$0)pTc%vT`Ll|Go0MVOXvDO`EC1N-$!#6%V3|El{r+Zc>Yr-~Gwp+|gbT*H3o;vV zx&yimYCCWK-0r>g<$ax3Wu>Rv)B?N!81IY;8K_s)#j1=mNMa>FY52{uuRUF}rD-WM zfUVoaLQxcgN|=K>Kqc=0VuVo-g<`IVpJ zEp5S48J00GpXm~^LJV&!)$ zGm~3Ivrxsbok)`xW1kO6O6b)=B|i3_mXt_*JqT4)4~aHu*$Ie}KooeHC{Fx(MJ85O z-|%nM z__@YHqIMPQ6;VWib`Mn0>^e|BBKOM@X@ZN8%zuO#_@r1hTdT0sG&$`x(@eJC3)ybF zyjC5&9vF1Oe08Fyi|Jv9lfGwQ2Z$s^Wt9W`~(A+ zayU9;jn(RjS;C)eW8Af?(LsOcru9GG(Yv;avK+={+!ICWJFnj_jRbb_K1r52-)Cao zDUcjdCY=$gDVI<{ZO|6*VIGg~pCQxMlkneu|8lSi#mBGG5qtKUg&TQ6^sfrEX*btk z@NC!YDEsrvz#{!-qQ=R*oTK1nQxo5-ilmE@$=}%gi>6hZQmkR?@sGaB=gUf$p7=OD z{gY6M>JDj#(j{($0;p`|J;R1S9yr#ot&%~FSA8d@x?SvU%iWqb>8iiqUJHZ-(VVah z-8l`GfwGK4{#{iW2hQBX;<*C|D<}1nwR$<2QfImTj$ClGVa;9{S z`5s2$u3N7OD}@twBPCq8da+PNObfB3wfTw0T7dyB9?V0EFSlX1a}ll6S#-Ivmt#kE4>*r!h7{D{Hzip_6-Sz}H#xz@b4 z9S!=U5SaW906!Ry))u|x6#K6(rYX;rJf_WFDI?~=e#hNGQBI74h9=6&OuQ^Mps z(h&g};Oh>($$sdzeMC%JBgKVezW}x*fzO!n1C1@h%gO2DeB`w-mt#-n;Owp@*)kJ9 zY5NFOMs>dMy18UTr>_Ti```@o8YYSUp7I1leVkw1L8*H6+_7PERESVGb>Z#AZ+-on zoBjz4W?E;ZbR^MV!4$is?{ZJW067Y<%o8 z(o-t^tpKjS2X+6^q+EK5@*7$p35{X8x>kDLIaq<-hJ4C?(i2mjplvsdTw3xxhi6lq zSiXr9zZVT|ZbnrdH`Qt%S$<_Abq*lH+ClqX??kuJ=?*?G_)v@G53bF~|Lkct!lc~7 z(1R_oRgW+h;#TvK?3y~yYJ`R# zqerdSU;3B)s{wygSkky?yF69qiyQ^4l~%*+MxByl^1g9r1xO#%+iwI2?Hy<;>y`dx zBHXj|TKueiW>q0JaabnU&kf^h{BrngaQ;Q)^+PKuIH_~te!-^vL-{TnMcn*s4-+<8 zaGOOZlhEWTfc7@x5kEpgmTwK5`>W)XA-P>I>|Q6bTi;D6?MEAZ^hWb+9V;N$@7DJf z3t?ZG6ziBMq6z{+63m)GX_~t7^+vmfncw*!{|q)1{U^+pN`NSPzBuO=GY1cEkt{oD z6Xl>;EKrm`v`A0L+LKq?7+4P~^nVO#6OHIm{ivx1e3aCdv=TCaN|{eaOw-DHAHIyf;5! z0eG^2Z*WO-?kyr)+*|ypeQl8?>Zf_(W(Tnwz1j-~7cF zbDZ1F2xUj;t1`e8ma;C^E>DqJgiye0X*YQtsEZJ!kc&GB(E4n^1?A2|k>5+w)cIIa z@bf^P2L;#-<)*+1Dy-)Jpoy{71HWt7UYYBG~F_*96mY69fCUxE>w39&vU>p$`0?p|bM*(48XD zFiY|(?ISy!LKHUsd|);@<)=~kKdUqk|Ikp8wCz(nbGPT)itBn1Y!%or!boab!peLf zIxx?r-YL|_rM^Ff)E5vY__e!Ds1}@?U9yq&e=wTSr4cNehh1o>bf*A2aNk*W)e^_8 zChKM2WWK#IjD6zdJSWK#{t`km(mbtWpvH?M2`6!790Uxr+c$e%IEEG`{0g{;6;y|J zfBqfL95p_}88uUX3NUaaP!gA-{fADMDt9EpAYYZ9?I*@UX%g@8O-H+e(6GhE)1|QK z=y)DHH4Y-u`oAq{-ET?rZE~Pdkq0xM!_nc6bI&q;^HFBSr;G`-)KY4lj?=`Bo3trm zekUoe7?27dQ;c9b!&9*1GMJ2krlglSY6bQ zE>TBB#+Nol@6Cmj0H+%15Wn0pdqLBO7KZFry5D@XMXISGau`~l0)&}f!SwMcD!7nc zq3LQ$LpWJBp&w~xn9m!NO^-|vkrXd8!HE}r%Y4UpH&x!gEdC9-RQD(>w4fEs*m}44 ze7&3}<@a++-s5&3rn*lUzRZj;4@&0#?*S8I?s^a;a@PY1dz&2%5Lp=50G>$bh%atW>VeFkW=Gch!O%pi;{TjKL;mEX` z<~(>fVle#R?{M`XG?P2Rw%*ZDcAtd%-aZvz7iCPu@$GOyo57eM5^Rr)#gxi{9OppX zV!mlj2e|C-9155u!+`{M?__9`enuvZ+Ap=0^N|V=0qyI|jjQMh_D#aE#5Q+Y##E}u z%_{fXA#~>7x}37DU3yL6C9I}g+&^NB<#_w=lxOkT_VH_lUpKsiqH{7n+ywlX5Ug6O zkJ9#~*u?ae3QVY&Ospr8|9e#4!yga6%+PR`I_e9rXw>uI&n~~TBz>HmetWpT*Ex1` zg|@5o``_!|gIP`}1NRf>Sy|t&tbRpYRSk9N*S$dfQ!ueQ&Uv7PK0X=vTw~w?yR((> z=OvbrQHm?)w0k1!p%{v#7jU#DAviJ+%b_@r{zbx7G)Sl7R>#~__IOL*%k+pI=!`8L zA%{aD0OLXP8S%>-J3_QNgn_qUlmBZ3M%&uW+#P6vjR)&`p*o;z zRMr-;Q4+h`sGE5+z65$%0-jxE?HPXwnHLzYI7^=>>lb>U6^3HWWZsE7Vo&gd(f`dfsmP{vd_h0fzmulcSlQE7`+BO z)GoKZzc#L4v#Sq{P4YGBnAkqU%HHEi$x#T9>JGcRJYvV!Z delta 3768 zcmX|^cU05K8io_;E*$~sErgEB#?T2!ODM7wr6__iC~N{MNR#pdK|;9{69@tdXeers z4pNk|U@$akqM<59K$?J{kPvRXd(NG|=FH4_=KH>P-kCG+-G<$&gLwt9UZSaD0wB

77>P@jXMc>v@ol{cUmG!r$O8uiXMF3 z|7l<%MkIQJyRBn*bGD2~b*x8Hx7M1TdtIy(8Q|W!WzuRC6wCZR*Kre0s7J``scmVq z2Qmp?H`R^|52R1A94g7`ff%`d!u;3J{6V3~b<8a}M+OSeDzvRzBRRi*xHr1?@$CiU zyi-qEw=TdQ*EjezuLl4o*vcq_%Wf+w^6Aus+-4;jW2OzM5^< z?XXL2g;Zr}IsKv+U8C5VS(W&Fu{Rl%KT=3AkVCGG^k^ z6ZGgt;z+cM^jF{Y!7sw4}4|lo|OZ+SW6x|ZAme7 z;pV1ebd&zp)=amWEBZ#MT-%7?7`Dd6ffHT&6KXb9@GF@wvONFbljn0w_k*!chMP%J z5%3_nY1Rp4b-8*8q8xXaU}e<$KoX)HaTle0uiPD?EN@Z8o;=V^NoH*Ft9V_W&a&$M zV5%l^J2VQST)815D&qcACncG}$gsN1iVNB27`!hiyC2t;tX6rbVS}rgVPnS`JSoY$ zZ5_$A-CZk-fk{m3XzL_lA;QO`?G0c(@Rjjc9*&pBsp_#*`wVX#{65n1;C@fvbib`y z6^pT#nf5(&b8YLJK{jUT3FY?PH_Q>~?$NzFgv0Rs$Dy@FUx(3m3=7krT8E+2bVt=u zr-+NWD+V+n1Z^>Ww02@eIrU;sMObY9-$nTHf&=Sp4E|xs8^ZxvTvVHE{}RBHu)nQX zIMAj;s)HGym&3W_CPcucea-?{L2^B!5Hg@Z&-Kz_73Rw3#~qhd zOB)K=r2V^O7~VKDWJxJ}u5*uHr7)tR5Y5@7B0`f(L2e+e&`VO)DvNVY@35AUl6^T- zLxZWibB)5Q#&d$EKOw{tz{X);Adkf4I2@-upix%)Sc-sMNS0}WXL#tX91+8#LTXy) z?qYgSv(Bn-bw4ZH&9^3_)XjH`?Cx7L|LSs}p{#Wgd08cO4*Y{(rS0Kc4I7ZP=mT{^ zveg4CNNO9{+C|qHmK|Or>-$=Ll7oF3WqTYrI9Iei>JiDr1QhV0OMmu|eq+>#O%^9+7OU>y`nOceSyg}@9u(5I-1^jr5|)Vk(8C8 zEDHi<&&5|+1Z8kjhH=2dO1=T$CU1PFc5X<|JBGzeC;LVOW=Q_ zosGKt;`_R6PmVvDx`~9HN$@A|7o+szpx1fxClaA6;9=O0q3M|jd>M{!MLn)ed-Z`{ zgeQ;_`Z6Y^j3rs@*&|JPJ<^i*Y0t3qnsE?4lGN^e;P!=)nW0AS{?8^7ekK7&;IBsM zbwDG-CF%8!ID&kZqovUFdU`mnmtCJZJGh@`X3m$!6s0NnI&h-kn}ynh(YQyzWPRG? z^`UL;@`XkDx3hHM#dm>_o?L&zt7-2P0D4{6kpPCFxQDv=E$XhFBmRisf3T z9L44nxtUz_Z@{`s#TN#dY(Gh188awS8y(##a#L2Q9p=&!zfbG8_-btM- z8x7_t=N}No+CCASvkSyyqF2_l_@8=qPF~F7K%(J?J>Spbck#6NaaF3A;Gu&oMeVWU%Yn|?qGXo%z0Mh> zo~q$|iO*IdIjn)TvGAxD*H%n<`%D{RW+>j%<<}UoLmEJ*o_eR~SyA`n2#6SR9*+_3 zJ3h#ck<@>{2O&Y%@#)HE@3``KbIl3WC*tzAeXJf?{ubHQLE<=sbe4hWJ4HVe+Wf`k zbl!}EyBeLzOSwmi1?4ba96ft_MD{Awd|lkas9M$@EV0S#&N&dGak?BSN%<1|AVh7I zJP07AdbBuZA2OH@cJW(vwgtuf+jH@m|>*K0AppMzZA2U(xxa;#{e2 zFJdCdGpL;3{=z*~J4j_VfP`jk3(*uw;tL8Suy&pshe!T`0{pmdKrjGnLJWzH1RNuM zW@YIabktUU0a37F04rAeryDb}Dh-%(VhJ_l`g9Ec`P=&+RZU1G>^JT!8<#8I&&KJZ zY}!t=4*aE!;)yGzG3aaqsYW0$!gu_hRM_@*SQ^`m={Jgi9{tk-)2 z1KtbR&8Q5A@#XqwHjNx8<(YdSAPJW*R>yj9H$kL6k3bHq1>@|Azz(@{Kc1Auf6q)l zd)Gs%5Upi^Xt}<$WTp$GX1Vt%YdyxKjbx7Nw7Ztote)q~W8++Y!82Zq3(#m0E>n3th`4J$Iu?UCF{ zn0wzXdvA?aC^MJy?-%)40$Rnd@9(bgep7Y9_R3OeI}`z*n+msh516YaZ_h za{Rj5p;ZT+$EP2|_DY(EEZ|wIl@pE_Kk3;f)!bs7eeMq>cjoECyqp0wz`O|=J#n=Q+P8}sFPDt3ZZ#A0Dfwjh* zjgC>9vc*N%nvb<@n#N_`ZgI78qrvsMhJzETf^2Atn%r+@HkZN*jrXI(9%d6{6#q|9a$CttFom>EH#_inv|JO;%pSj&E_ zRC8CG#2^S`-=2P*aw(l~)KL~14BPavUOIgH zA%X;h4y#{!_F+3e{q2F)n~R9@t>|6iC@wl>6jmrb!wQqz#SbeV>9jVH4bI;LXs2j< zl;QI%o;r&7kRWI4qtneq0HOd4{M~h0IB_%PIJs4aPF3ZdjBi{RGk>C?^M)^gnCyi> z{w>!YBd&H^^qoe}hyGaO)@OyAZn@0(osnhkMJWqD|DeRN0`8a!i4}fTv(q{rH()S- zQ0CWl%K@E5YG=Wp5|^_+6M07pgC~mvc}eKCeTfr0o_ChjE7M)IyGH_nYCk4E5G0vs zg64|nlQ=oU;c6^0ETZu%vA(@h27K?@vU$9q{E}zJ`x(&r{Q?1U@E12deu*G2r@4ih zNJA+`+xvsydMeY9<<-`fNhx#Lbb1SJX!x9M?AzJYNX@l2x!e=&K^3b~O)gjZrazOB zCVOBGIRi57Hq|Iv-06e-9Y1mJ#09$-Fg424k_*O^j?% Date: Sun, 11 May 2025 12:14:45 +0000 Subject: [PATCH 544/719] Updated results summary [skip ci] --- README.md | 79 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 7 ++-- summary.json | 88 +++++++++++++++++++++++++++++-------------- summary.xlsx | Bin 10831 -> 10931 bytes summary_results.json | 76 +++++++++++++++++++++++++++---------- 6 files changed, 160 insertions(+), 92 deletions(-) diff --git a/README.md b/README.md index 330dd0535..e39355f75 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-10 23:58:52,243 submission_checker1.py:3301 INFO] Results=31, NoResults=0, Power Results=0 -[2025-05-10 23:58:52,243 submission_checker1.py:3308 INFO] --- -[2025-05-10 23:58:52,243 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-11 12:14:39,749 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-11 12:14:39,749 submission_checker1.py:3308 INFO] --- +[2025-05-11 12:14:39,749 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-10 23:58:52,243 submission_checker1.py:3314 INFO] Open Results=31, Open Power Results=0 +[2025-05-11 12:14:39,749 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-10 23:58:52,243 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-11 12:14:39,749 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-10 23:58:52,243 submission_checker1.py:3324 INFO] --- -[2025-05-10 23:58:52,243 submission_checker1.py:3326 INFO] Systems=22, Power Systems=0 -[2025-05-10 23:58:52,243 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-10 23:58:52,243 submission_checker1.py:3335 INFO] Open Systems=22, Open Power Systems=0 -[2025-05-10 23:58:52,243 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-10 23:58:52,243 submission_checker1.py:3345 INFO] --- -[2025-05-10 23:58:52,243 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-10 23:58:53,522 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-11 12:14:39,749 submission_checker1.py:3324 INFO] --- +[2025-05-11 12:14:39,749 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-11 12:14:39,749 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-11 12:14:39,749 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-11 12:14:39,749 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-11 12:14:39,749 submission_checker1.py:3345 INFO] --- +[2025-05-11 12:14:39,749 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-11 12:14:40,743 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -26,29 +26,30 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71494 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421368 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1249 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0648 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9539 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 5 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.3567 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419011 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1249 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2154 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9539 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 75fccd696..15c1c7c63 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1171 +1172 diff --git a/summary.csv b/summary.csv index 67ba85fde..c7f6c793f 100644 --- a/summary.csv +++ b/summary.csv @@ -4,11 +4,12 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71494,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.3567,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421368,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419011,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -27,6 +28,6 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0648,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2154,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9539,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6514f4049..08e5a0100 100644 --- a/summary.json +++ b/summary.json @@ -150,6 +150,36 @@ "weight_data_types":"fp32" }, "5":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":21.3567, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -179,7 +209,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -209,7 +239,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -239,7 +269,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -269,7 +299,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -279,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421368, + "Result":0.419011, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -299,7 +329,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -329,7 +359,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -359,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,7 +449,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,7 +479,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -479,7 +509,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -509,7 +539,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -539,7 +569,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -569,7 +599,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -599,7 +629,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -629,7 +659,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -659,7 +689,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -689,7 +719,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -719,7 +749,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -749,7 +779,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -779,7 +809,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "26":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -809,7 +839,7 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "27":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -839,7 +869,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -849,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0648, + "Result":23.2154, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -859,7 +889,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -869,7 +899,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -889,7 +919,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -899,7 +929,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "31":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", diff --git a/summary.xlsx b/summary.xlsx index 3dde2ee8b1c3bdc16f0f98cd85363a353306a0e2..f4d390d5e5eeab244e4c6e09bba388ac3af61cb4 100644 GIT binary patch delta 5131 zcmYLNc{tQx`yShf$d*0pcgQk|Y3zzJWNGYQO!j>+*6iEIz9dUCjIHcTiZC=pwk#PU zOUN#gq!L4#Y`^J!uj}`oKh7WLx}N(y&vW1Rd9LfcvuOUeG4R|4gPFYNKCB?nu|EjJ z2f{f?aRaa`VLq6R$9DJTbp?~(tvK|`Uq^d*jX&?YPE=2uigGMy3&D;>ei9SKRL4r4 zyG%LE1-a}lw$9wYoPmzSei%9Vma@8c#eeS7$zC|jcX5C`AXOl&{O^dD(we6x|AT7f zH=~C;_QLd|gW}(jjGY0njI1JuH63yu?o24!;4=U~eb-fdT!rrNz8PJBKNcb`q@Vg| zY{}gvL5tm6S#XrfJol9`zxnhzjUTL*I=BEE8~kkXzV6iL_LRQx+#A@B{_nFKASWA; z;_7Xq&%14T?o$qnneOSINfVAaOZbKwg{+h}qaI&>?uRaT5{)x0T>!p;5}u?6L5kT8 zNq{6mQT7ABLCR(k#f}0MGt$Fd*x4dS0wM~cB80MheW$(=UWV&y@?W()Q9o}Izv2<}josMPX2EcM@%S1mh2kcXnfeRO=nX2U3O_?Dk6@L&p8HuIHNY%E^0UCq9Sj?+Y-g zL-Z1yn}jFnI)U%P4gXAw-`kZZeJi*}VR~S2B3XxVmowN#6V{Y7|+qTTYoW zjnq^6w(`~5ECiF$Tw8xCTQEBvVeb_^gjSVlVb)9xZ6m9O-Uz%bjCrTPnds+L{t)q5$t%Kt6 zHF9|seEt{d5T+Wj#V(L+k1G#A_aWk@5RNqQG`)*-boVQ5%h6K6j-zThAD>BogqA|K z6jq?gvs+?n(iz3u+s!zjhBu=bVCx+tqSfa^4KgdlJQNT-3_5>{VVslJjn(0N2*swl za{M#Q$u??LMx$$XwIsPy;eF_>xM3=FAI|M*$fUPd>fIoRrSspu0WfVd_)czLPEQE% zo9^4}Ihyg90@UzUG_OY*K~Cd=KAKXhg)-^dZC<<=?;rhACDE@vj5L7th>hs6qja%) zFoUk3Jug9L=%g^@LA9{jf?%u4Eknd@j-x}>oM8QesL@@y?YrqEy1YpY-xzb)j+`!h z63z6O(+=!PZP>Ap9U}HO3=;6%maBV9?#|xwC;0U?;OVD9W0H!LPF#xB}6aex@dl~u$ndE=9c&c!nt`N~;}Be^=jgXi%&zbGkfrEy-`Qtr)>|Elzf#pQT(M7Ar{t z73PZpXfLk9N=C%gxVA^+I$a?_qPzlBNT@R$UUmC-=7oYO zEgTGwH6Le8dN8j5LD3{hcUogY8ow*fH5Q4bQ6hD83|0F$C*L_|=I z7D0NXy^f_7$?}otc~19 z)f^v;;`c4$d+jrIlF~%U`Nv1dyROBjeZXhRC%nui**)z=g$d(sB~cp|w;!Sho~U#D z(a2ItS`^Zng6uR3vmS(YZk>p#86HaTEPo8$wN zAGN#pszP!_-W&*ZKNxOIG_cUukUq)2BULYgy(uFbrU|99fHi=hz-u*I7)vK1XHtzo zsf2Sy&sDe|e>1EA@@m%K6rj`WJAQk-lnzh$^eGN~Vhj*mp zM6onblrG>*f^iyRP4`+W>^*S|{kjjk#hkyER@|hG$EPCAPcKYRt>syuF-XXklMFBO zCdT4Ln$5ok_EkB;I*aNq5GYKRNpdF#7Bhr)nakF8@K}oxDP;Z(O{}8W zn^LlTT2ML@ShKF76cWrX$WRJr(h{H-MFVMH;8il`-RvxrO)S}CgGOn`FW`Wbt2QIF zvQDTLpk--tSgy}3&oNCP?ecdKEK}8qHZ0YR;V8uoWavH(!b={dWYAg{ek%E%!DbzQ ztX;$OcOPD*A8AkK!Z^{KnL!Vyk0&}lXW^EV?fLUUnDWrsVUx#H+6y%y34pDoJYWrO zgYY{zdvXHSjImXuqlwiMEkgwOW|q)!dnJoT$c$pxw|4g(#-`)X;*oNOb>f8W@?YyO zT6U5ob{xEF5)JonYt;Uw=mnLcE3#-RMYdQLe@{Y;HDgbe{+|ia|KeHk3{y>L+|LHIy-l3?@2WaG*b*nqRCu>?UmI4I6Qc(v)rp=ovSjPS zw5mxICm~RQ^}vuXU7yO z%4#~Kf5utN-Q_`fPC?Z)DI=OjTLDA_W#UQZZ!?M*znlJMN+SCx^BCI*s1QQ;7ifm2 z)Dsp1=hGbLY#K>M$L~^II7@Zmb5!##IeL82d}GsqH@75ecE9hn!?QUzpt+wifzJ9p z`5(Bgl*H^&=1D4C(Ve-tS$E_ln;^p~Tvb!xkqG*KN;#5LNN`-KL;aydsFVa!ZFsP0 z#yeetS!ro;__c>gqg0rR?N(8=VZ+$3Y)oQB-0wy3G)e_+beO{TB1+7So-hEOPt9<` zlNKm)Jk5qJsyxfrY5TyrECL^)|yP`aVV%&TYLVUBHAG zAY=C@Z^aNUDj3b_;H3qrfNs$|WG4WqUOH!fN1W>?E~AA?nx+dNBo6GF0H&w1ORo{~ zSQI&Tq(^>!~SY%&97^h zhTB_ME)cj+kFz>A1x(c5KD0321H4c+Rb=mpvk6OCmLO6(foL8NDTH+G1A4kBGmy9# zf)Eeths8q70dJ5#j;3Fi27&}g)Y@_S;3B zR(}@X;o9-@qc1KjhO1<7eYbqVm@x9<`6BVT=I#$Op%`mGWl>v0@M^&YU5W!?Cu3H` zEibcLIb)z?n~g-iU<}M!E~9|3mzUmT?2kZi0KS7I9&^`ot`g_?qJ*Do3O6VeWnn2& zCj{Z^XHi`nZJ{uIu;z2g7mqSeZ3i_^$Ip7zkIQyW_U~z*y;t#dHSErEe`b?`1$wK4 z1L50C{hOVXQFJ@{!MYRC)h}BN>A9!vmZ)7Sl7Z&?S7Gc zS@R)VC(0CwxfuVCB!}i_-1#(2Ua=$a4tfx%6?cah3Q&s0?^MNBJ|QK=yULIeOg56s zyFPNN+NUuaTCfLhH?19u%~a(MCOm^%?}T>q8Zoi9ck@Vz}>$Yhw4C;f44C9tfy zyb|JS4CL&=Cz_m8A`I5*=;B`u{|TKGz*HE?5Q|=(e5~xV5wAD)qg*QO z+I->;Sp@FV9#k%^pPoqM(@o&f5x?RH2bDd`XhoIk*pbp4#@$Dq;4PNEE%+--CH+ON}wYy!9<>?Hn- z0>PTbQ?`(yHBVjID_$nDo(MzJc#HG*4P@;RmGL!m*OD{+1twEd?-VnH^9-Di_6Ca- zVQh3T8&5x%SIWwXtYu0#th8+ojMbXDqX<9A@&UP_*Skye^?eKXHn zFhZCZ6Fw;1V(0oILRTpn3_6dG&_sN((vflOd-5@;3#v4Z*FLw-@2_LcLb5Oe@TSt< z2E*$QTKGPvsEH(-lSJX2lJx&>f}Q! z73^BeAa^vY^dAB2pLqeRFFMSfdS$%Q`{GIVmE-B$5ly$QT7DT|?zArBmECtvvakE^ zVMFb>x6FPa)?VNWx;d_40cgTY_A(bR=n*Z_jf4-0!YRjijG>NPz zMIj|owfa|V96$2HBVpxuwj7)RMna>hUo$GbKJjvHs&RDsLU(0W9pIbr_WGl5t$s^j zF2Kma1;xlgVHe>;P*QrOZ8m!`jQzX&^%1;FyO9T`d|851>!I5f^2K*Qm2{E*fq#pZ zX50A?c}FaBoN(x<9`erDY26f=c(?HF!-O`;Jx!w)`x#i!T-1Tq$lIz7r$xo~IXAzG zT3A58N<|qw=~zkN0dS>DkTldsHK0U-3wFM1^?PxX&Tyi=`FBIddfCC^R zz^GTBJbU6!G;>9V+=#vb2a(Q)R2z58{B8u-1{&RtixgwiHM-w0({e%Oqd?Umka8}wpB$cm ztvZPQ&3HVmkcK3GFRve{=Ge8b)K)c@r87x?8+B=kTDZ$>FiKglPC>J`WIM0bY1-mz zU#&`M`^86rj8HzSyx;AqDc%Ka+hJ?<^t&H!N3VTv8ERrfaMF#tEH>{cL`Ez_>P+lD z4Tz8ON{b1anZRnu$J_FI)n~L8HDYIy#WZggTT4n$n|mZiXqo@_KZnbevZQ6?!F`t! zyY&C_fFA_92>RcOjeE$8Gv(#MX-Uh_#NfQ8)o6a=s--3W+xc;69-0Zio`KoF#*OKC>O zXwcCTehKM#$M^SM*Bj^0bIx@?_jAYRxvr<&wEt0wH!b~-@~S{qMi9u(9UVw31Xw+v z?Bseg*I%5O2L+ot8dYsdPd$vfS!169?)%z@jt(iwCulZB86qiSO_HvMwMS^?Bk8+8?@!INTjx80YvME6WG*k48InI)iI!8pN< zcdh?;i+Zc1>asI;*O=MX%5P=7OsZAlF50X1#KTdpyEx~WBBa5i16DV2pv(&&)9#2_ z8?Kbbgyse?U#<~Aqc|^HM@)Y>=*JjYV=WD~W>S}kFg*CCF+1{w2kw$T4ThUoOin;6; z4tY>9I?%UVo|3qP9=bu}_L7PFgQvADjBU&JnxIbWTudN1!T#2El-3Uz+_^^4u85Ie ztkl{d(!a6k*4D4t){X%741=hTNy>paOe1ckCI;aSIs1^i)2RoMbpF6+?9sSIzP{Pf zNZauv=-Z5h$m%&mT`lX9Ww3C_X_!{pOy?ywR%ip5_51L3HrBYzF|e?7P=r?N>C3mQ zQbo_Sgo(e?(UH})Hf*eB`;1YM1;-ez*3qE*!U90E{|VW$I!#ev^OIKYu``ZqmF9n* zv$3v`U|V0=*=t!4e*wGOikfG_jl!CdG+l0*{>yunQ~bXT6t&=?Y=zG)M%sjHmg}#c z{+M5Uw$y?)I}bYU(B7>L*z@z})7#k@7!|9t4RrmB)(w#BJHOM_uY71&?svDI4;HZn zRV{5xwn%eIUM1q;iChsGyW8#V78^)`4!q;zIwjb*e%$bjQvmv4N7G=^Hp7WYQN?c0 zxjgI)=@oDAd2T7%6(jhWiI9&EmgA2t!Zs<;coD9|3@@Q^d-2HeKfldQf{`&h`BLX* za>e@v{`sRd90J7!2<>|xDkA5X8Es-(;1)ndIZR%)w-xo&RVM`c4q+ZvlIbh$O zG@AYHEpn@(9N2i}WflH6wpz14s0*CW`Y>2CSL9r+>8~4n`}a}Vm`wT*-$K9rV;g?! zMXkO@=dOAn$RB5Ai&tKe*;W498DMk) zXmrA{j*PCv1$|yn;`Qdzgc}>}GF*EdcoxQLPr{1^3tO~CmbE@iPR=5`XCjl))1Wx= zQsZ@Z3Nmt>b{i@YU7Jmt-eljNusm2|`xDX-j1qVsfEQ?_n(e=vN+_+TXR!^pVxg8A z+XJc&#-A8j@cls4Wg1Oa_3J=bfHgy1Y9%-ioiJX9P4}LKgJVDqoh+9HGYFR-(YMsj z7(2i=XxtjwpH-;ez2Fw>VLoOVKKo3{UNy6RiusZdjTaOzh}w~FtT+BDG{**9##=r% ze~o8ei~8EUYZW{H1;tIn35=fR|1g`d0n9?9QkpULD6=&(0~FZ}UFzmmhRuXj7Ao{^ zh&}JC6|uv`H8~9w984!Ue^U8O0NttC*$xN==?o#YgcLFqC~cQ0t|$_%^nb8AS!Cv$zUYy7*Yw~Fx2CTs%U+El zA*O;UkSK-{-Psv!MR)epY#R9RfTr{FdH&Qgkxw{36wHEvKvcUs&{n~Y1>1684MbJO zDmL%YvEPh@bb&6oI=szX{V)Jl5yHL| zP)@z_XCP6Em7i)L5n*HUb_%k}TLoLT3vuFiE39x=uA6fD>C%LplJqj>=%#umHqmPE zlSDJ@UwU%Q5O$Xk*jOuM$1Mb=$^5Z5<|VQl6B$d?S4037$w|K|qpnUX&H$HFnQsq~ z_N_OSTi4)TD<$KDACN5d<@ha>KH{qbiJxWa62fx!@MFm2Q|c&OunYVIichZFFF9zT z7l{DDJpFWdo~8B>U%Qt-h*>tvswuXOo8%1eaj`&Xe4LaGD7>R+d}%8Y&VKW?Pk;{f zM4|mYP(l*$5m(@TEBv5DARdNZz-5O%rCU6$9ir{pTV0m(J1O|q^Oi2iY{kKQ%(m#H5h>i(R27FlmwZG_#5&F8z?B^ zo_5nEYDMgID_ezt$y;-|Qks&&iS_*0oO2;Sa(`sczu5PMUK%hKeiK`% zONF61vz0)_Od0g^g57%;(@+shaC!xx14-Ot?h?Xqt_xzwBvVW&%(3f&iuCrbC_uhb zm1LAFIY#I_UQX$IS_pui0SoCT9qIPov8>diP|C_v(`Pf5R^OJ0fhGdaDQV)R87 z9A^F42r#Fv2J;+Ru0u3S!TcZ$Wa}d87@@OxbtBB1GHG!N3(O0~vULLa9wk2G8JlZD=rU??rGF?rA<;6(aC8!(yYeoyKJail9 z_Bc40qnGB;?}Axw&BlpKUCzg$`9d=d$RVG=>PftT*hfK>y4-aoRY?jsiXol^V@{)% z+jTH#f1;+ZMST?>3T$b1dkz;moUCADGVC;$8YNUM12J(M5xx`G%SF25AcCXUk}ee6 z)|88MqFRhVfMAgI=IiB<#fTVkB=)@^WBY$n(teSW78|smdWjpM@Z>~^)aHTB{DbFN zRc#qW82d(YgOU~T)P+MQaP>6)eOdN_rbed|K($5c3k)&>N`-9wi&!tnIQuVRGsI6d zn_<6KVm7X5;`mSjxmj?E%Q?jb7O!_XMV;b|uWjX=YRYw^j+p7Lrg83);--{hM&XIw zCB?1TbqJ7OAxT#1+HH78C8A7PDlDe_N5uI`P#GZ6oG<8;H)BSp;{oWa9k%EDtVZZ3 z>!>MCf*TVg3ZmuciHA0D;uuN1hSj!y#7J^9x(fG&LgO zRHr1Pbs+>03UW?zMF)E#c;$ptjdA(Km-ZJk2SbM9JP{1&Kh01;{R)epHrwc^q(T)* zz3kc}?B9nG4^0Dk<2qb%84Jm7rZuiMLHUF?$|n^^XZdx0cWveP-@jwVZ^oF-0hpwi zT(g+n;?aFi|A@~S-%Nc7g0_QqR3D@c8+(X9)+t+uc}v zPo|L^DZBa60QG%-F`H|6&@TF53wL(y&p*a<0ozuZs;+#uwsYq0?Tkv@7Z0`{R=$n> zTSVNMJ$Wh*yE56LRn`E(=bOX6}48m0d@peask&aBJ|FNkjo2D?+@s^ z@=@32&AsV=?u2-uw833BUrqXdD%{CAH;a>HY9+k?=qsQk^S}W8hFM;OVTr0Y9FdYN z{);&0jVSw+4R!#S6Sw7MOA$3rdc;DCANAb_t{r=n`bll)k9|ti$Q}k(I6Xg0*4{W%e6DU6?n$mfE($z2tW$d@StUe&0yM;{#%bt9za+&Pr zOh}uNOx8^sgaoFZXY$<4Vei$k1V4ImnVIvQmUXiq?OkJ_I#i94E6vonNA;E&(>7S9 zX-G7x&@x+gxdO@CC!m`aubtLvJ-geg1WR4{R()x;2Q9}e?pU+XbYSJcpotwodHTyH zaP@DCoa^d|=Z)P?X7=iE$?oA$w@3mfQZk8i{}xfD_=StQJs>BE*Wl%I^Q?=yAI_do zMTGR!>h%Bxc=hz$8lCvjmb7r|kxb(u5h-mtlWUR^I6t-odcmc*%e6mTwUV{q+t8kJ zM7`X;ljjutSi(WZ>>G=S4G$e_Rr|8r+~}|`!qT)f0X7TSP5NA;;pR6pDPK>m{Un}4 z$Mkx(Xhv?vca`4qqd$iI%`?q4$*8TTwD-_|D1HL|MU2x$vYq<$yHpi)jN{Xt0&ZrN z4?H%d@2yZ))4t-m2ZB-@P>u^QSj&u)CzI1=ZU{4nOr;j%)+Pewuzk+XW@x-|!_-h+ zCF{Dw-09-v>&CdcCa9yJvcPW%m1{JbmqIx8PX&c{Q-wK85Tl;8=G=Gkh%7(+$f2xU z`f@;0BH~du7PYN(jlLxmV{NQf5PFUaX>$`B>WCyRmVL2uVZQ!zvg4t>CA`9IjGeWXl18r5L#-tf8T_N zn}gR$9rGjI_mNar><7NG0+%lae6oEm zV8-*tn27nCp6dN=Fq0b%4FHYJ^?1rncv<1;A{{7I)V|-g#62qlli6Y$zm9MYneSq< zO%Eyw+GA$5e-kY-?v&Zi*2!hlo$ZWfA7IR6*J+BdTUNj9&@soOP&4qt#=}`4JZs)c z8pMrTJ6v3`6e`-i{>t-qdXIBhs;dgvJ8R=8_~!(Mf4uR}yN2cYU)Q5~rUZ6N=MS0o zBs=yN6WVH&FvTAPR0_7NFDngwg)(D)tJH;wpk-&znahLup3nll(dAPjWFU|cIS9lG z0)gNjjz-=dp3h|*J-iXpe(uPRDE!lw8!X!Ur}Vf6DOFFpAHJDV779i#dipOluu7i@ z6uMvMn~I6x2jsSh+qEzIrgU~NZ&g3bB zmNqInOPT#R$m1QS%eO6{O~}W2UBmV7hvoW)V7Bqz>fOZS8{4_(@j%(C)!y2>XrX{RFE|O?(fSg1$-j%Bqa}o?pNn36_%B%yvibl3 diff --git a/summary_results.json b/summary_results.json index 5ceb80b89..dd33db0c7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421368, + "Performance_Result": 0.419011, "Performance_Units": "Samples/s" }, { @@ -509,6 +509,42 @@ "Availability": "available", "Category": "open", "Suite": "edge", + "System": "gh_macos-latest x86", + "Platform": "gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "UsedModel": "resnet50", + "Model": "resnet", + "Scenario": "Offline", + "Accuracy": "acc: 76.000", + "Nodes": 1, + "Processor": "undefined", + "host_processors_per_node": 1, + "host_processor_core_count": 3, + "Accelerator": "", + "a#": "", + "Location": "open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline", + "Software": "onnxruntime v1.22.0", + "operating_system": " (darwin-23.6.0)", + "Notes": "", + "compliance": 1, + "errors": 0, + "version": "v5.0", + "inferred": 0, + "has_power": false, + "weight_data_types": "fp32", + "p#": 1, + "Details": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", + "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", + "ColorKey": "availableMLCommons", + "Performance_Result": 21.3567, + "Performance_Units": "Samples/s" + }, + { + "ID": "5.0-0016", + "Submitter": "MLCommons", + "Availability": "available", + "Category": "open", + "Suite": "edge", "System": "gh_macos-latest_x86", "Platform": "gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "UsedModel": "retinanet", @@ -540,7 +576,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0016", + "ID": "5.0-0017", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -576,7 +612,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0017", + "ID": "5.0-0018", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -612,7 +648,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0018", + "ID": "5.0-0019", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -631,7 +667,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -644,11 +680,11 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0648, + "Performance_Result": 23.2154, "Performance_Units": "Samples/s" }, { - "ID": "5.0-0019", + "ID": "5.0-0020", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -667,7 +703,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -684,7 +720,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0020", + "ID": "5.0-0021", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -720,7 +756,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0021", + "ID": "5.0-0022", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -756,7 +792,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0022", + "ID": "5.0-0023", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -792,7 +828,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0023", + "ID": "5.0-0024", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -828,7 +864,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0024", + "ID": "5.0-0025", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -864,7 +900,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0025", + "ID": "5.0-0026", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -900,7 +936,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0026", + "ID": "5.0-0027", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -936,7 +972,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0027", + "ID": "5.0-0028", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -972,7 +1008,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0028", + "ID": "5.0-0029", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1008,7 +1044,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0029", + "ID": "5.0-0030", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1044,7 +1080,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0030", + "ID": "5.0-0031", "Submitter": "MLCommons", "Availability": "available", "Category": "open", @@ -1080,7 +1116,7 @@ "Performance_Units": "Samples/s" }, { - "ID": "5.0-0031", + "ID": "5.0-0032", "Submitter": "MLCommons", "Availability": "available", "Category": "open", From a36de88c0c77a2ed6a14e660fa4045dc2f5bd617 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 11 May 2025 12:20:56 +0000 Subject: [PATCH 545/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10931 -> 10930 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e39355f75..b2c707d5f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-11 12:14:39,749 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-11 12:14:39,749 submission_checker1.py:3308 INFO] --- -[2025-05-11 12:14:39,749 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-11 12:20:50,500 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-11 12:20:50,500 submission_checker1.py:3308 INFO] --- +[2025-05-11 12:20:50,500 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-11 12:14:39,749 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-11 12:20:50,500 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-11 12:14:39,749 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-11 12:20:50,500 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-11 12:14:39,749 submission_checker1.py:3324 INFO] --- -[2025-05-11 12:14:39,749 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-11 12:14:39,749 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-11 12:14:39,749 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-11 12:14:39,749 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-11 12:14:39,749 submission_checker1.py:3345 INFO] --- -[2025-05-11 12:14:39,749 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-11 12:14:40,743 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-11 12:20:50,500 submission_checker1.py:3324 INFO] --- +[2025-05-11 12:20:50,500 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-11 12:20:50,500 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-11 12:20:50,500 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-11 12:20:50,501 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-11 12:20:50,501 submission_checker1.py:3345 INFO] --- +[2025-05-11 12:20:50,501 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-11 12:20:51,561 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,7 +31,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419011 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419137 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 15c1c7c63..6e64a9148 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1172 +1173 diff --git a/summary.csv b/summary.csv index c7f6c793f..55cc1a749 100644 --- a/summary.csv +++ b/summary.csv @@ -9,7 +9,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419011,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419137,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 08e5a0100..195041b23 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419011, + "Result":0.419137, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f4d390d5e5eeab244e4c6e09bba388ac3af61cb4..473d4d0c770ed6e2cd99d61d751ef61273d72012 100644 GIT binary patch delta 2963 zcmYM0dpy(oAICSFJ6V*DTqd@KTxv!gr74OvLvGnhE?H6TkrXykMy}t@l;u8D9G8)# zC?RGIjU#e763U4jq~kK{H~oH($M5sU=g;To{rS9JulM^?=f-iX373Hucvo1~U6Toe zP+YHZnKM_v#Y{OI8Opzafb}2hkO{Tb0S+r?jb_oVe{Wt=9LfqmHm`0SEzQdwP3cKl znd}TpTuE;PKDYP*12p}Y$bNzEr>1|d_$|%+XnNmUAkgsZ^&a%p9QCv5X<>6;P;SPS zpFPjB_+pZM&7j7k)7hYAb=;)p3)IN*EXCONz=WO@t@t{~veQnx_uOyb9XOj}J00Z{ z%2|3+Qaf_XNLZ6C}^-|%>2$99sAMat02 z-l9r;_tcg25aN515Ywb*=#8Zm#tbQ4)s_xB!qWnlPsGHQla4hY(Dr+Qi7R*)7 z-kN=?(dmXX*=8VmykUB~n)AF0s$v?i+lr_jgxo_wya5Gjvf8Nb(>-#iS#T@nd*A7$ zyxXS!$zSwZpf1cvqS{E7mo?;(#J|T6dx=ECYjSZe3q(Xtp(hxajzwrfiEG02|I zQYICgsvvyvqn{A}!P++l)%C%shFFK$4^?NbFWvkEVR4YfB~XAs93+z)gM4M45;ou+Wj6%L4e4g?AI#?` z8`K(Nct+9BY!{<`6zIpajVdB?r-Guqij&oB=zgHBQV&@7McDfTS}=mOQr>);l7Bao zmM-Dn%`w(*3!+D)jg&3Jt?GX9^2UNpL+ia7C_&6dJ-KO#gN#65lnjD(@0dAGY6zj` zO_#;ID!bxKUo(4ER+QB9`C=H3;Nvt=VMch1#n4cQpA(=RJ=69w~o)cooa#@f)Y zP^XER1p3Fe8&>87AOn!=QLDEJ-0>(h-gK5|oWvvyI<`7;RUVj4J|nH?b9J5uUotjTsOn+jkZ#%d?E6 zR=GCbE!Bh;wbc%&b~=|hOycBeVjMV2!JCU##xt`#fK=e6=E^CSp|*L7P;tA9w5Fmm zOnmX&r6^jDI9j}^1I*ij2e|T{b&~bMdwI?|>c*1v*)hM!Oul1{LEvn4IfwbD+wZM^ z7lYZPmhe+Wu1JbgF>VqE**Xc7rEr|s3i4H>0{6{G#{$?&4 zzH-osoCFa2iYoQmJ>lV)VUntcEU$R8WS;3Btj27D;s@-G1pLgj*PDj}#|-xkVGSW3 z=nAPC%rG`^$14HiXJAOoE9EJ;1z~GC6z zn^@ZP!Ck{wh`u|sJa;{TSC__`7)JkeK8!q9jGe^IizRv_mMENfOf1m^K=GsE=__KO zF+q?aHTw(L#Bmab@OldJBJ6?h6=s~)rzKVw=ZPUlfNSvI^dFOZ@Y;kt56m(%lb2|F zOW}ZQ>33_i-9^dSBr??=0HX;#N?$=;7=7$+2HZcnOFPltDb7w!%Wv3gIsR54zq|L} zJy#v`3if$M#8H-Z`8D?$t<9UvXBXl`unb0=*p`A{u4HeyQrvzpzhf&QR{dSz;@TPsQ^4*XftT0 zh(v$)&^QmNK7`s9sAe08Kwm6rU5%$OX`bdbDyeYiJI}=_jN<2}6)fDE$v42BX?iG} zdR_)s6_NPHEM!l{$(!vtJzKa_VqBnmrWiNm*S+n9`4u2e?38lu+Pev%)&HF`dN5fH zx^a;&=zZU|#t_HwF*c{^p~!EetaOu;a|!4$p{PcohTE>` z-Om=)oTd`!YbCA1cqYw3WeB>Q#Q6Az*dcsp#=`!U#E)H})PJWi7P60w)#HV5^%Kfo z>sUmIp*|=J_DY-=2WD0%M;82`0SEFeu{%W2j0l8L3z5G4&)?mQOsY`8=jeeSO@Q%Bi@MGdTLsF~5@9?!RdIpLx7G2tDsd3s5>O6Gu}ex( zhNNRoSUbJzv(sQHIW+T+e?C*@EZ$x$P1C)^9W3HfT8Fd#V$YU`ug*RrJnUR-t?hhw%5hQFjLVH-nXKdTK=1f z<uJL=jJLM4IV)e*J}(<}Hq&wF)iAS1(-Gf4qeY8JBBk+d-dH1gt|Kfep>Mz zj6AyE$m0@bz#o*s4)O}}ZaFjYO5IYE~Zz2hKHZ5neA|uxrNs{pOaTSuIjk$Vc?qrEg%kf1* zwe3etIZEY=QVF@ra{bo#`}^bf{^R}U^YwZ?kJtOnJk2`Y7`00TH&WQZirIAu3N%Zg z)2_CxSX@0CrKe7QX)2dNy5ld>vR{^?mg%_Qh_U{4UI6>%*h>GAUH(_*+#qE~bZ4X8CPtW=8J1I4&nC*8dO&kH_q3v?5%?FyI8)XsF= z1BKqK@~#NcUnE*oC@H!0#@U+Y=~GmiNL@O^e1C#scb8b7CCv@oC*3{34zYT36_-<~ z6>N&xD|+JhEMnsZ_DLt?ZX+le0kZqKa?<^0%E9(-CS^A|OkZ6+p3DV`$r)q2maPr5 z+NvCnuEb_3I~)bX=R_m;b9z~+moC2UnSHV`5{>z< z_uSn0VZjlZx*d;0KGEVi$`I@+j!0Z*^m3@GKZ36gzg44d`BwF@`Ee(VgY3pni~K9t z;)MPsO5F<$a-Hfg7F%){AoXr$>G?rn|Fgjo;m zVylkHPyOwIZDE^5?a3JGU-Ju2DAUvsiX{Dat3kU;a>Vg`Xa(tWIjds{j31X9as}ZD zBRYXC;gp6;Z=<)NA^+mB+QVTvKX?GUUrzBvx%cD^3jRML$pP|*UI+&+g+s~6WY}0V zo{a2*jiFWt$KS6%sUc^6)-Mlmr(Gc^N0cf(4Ikc0ZxZD14_6CgG>5nKg%5{cD?fK5 zDDzRJi`Y^X+YXt$8si8XVh|S^CFz_AHQQnMVxzhYp4wGuJDE(&$bp0aDLr=F$hUBx zX*Qwu*-FRY#F>W<^EYiQn;FRRQ}abOBhb=&*JIu0wTfNZY~ZP6&&GH!*cjaK4<-qj z;3fT{2D|L))@_kW%gEMN7`UuI$48EtWiCPA2K8IMJj1Lji>`X-{&Ln$;Y8WD*_`<} z9hFuI+#m+uOT916N#g@+TFMw!%6yNIy+aX`*?8)bT|5Yd=jT#z1Io>qfISD}>Gyf} ziKFn9=RT~U_E}lv_1Tp*mxa2e%hfRj8qe3%x^8hB({LVGYlE%4lll)dD8~$qV$lew z2l6589;y!AEJo+Uck~+px{iP46BCpzp)g;IRu$G8^&;pRssnt?bf6m(>GGGML=DUEPE$?JWNJal#UX&RT0q)$^49}HvA*d$Mc4Z zr%Y0GKV(pf3Z$Ey<3Wu42+ctAJ3iOA$E}cxKVNSSwSUT`zB_VWxiCh&FeERQ zzbVwc1PxF1_B;r9!CK(TzWyXRz)LKtnzJ!9xn=8wfgQ_Le*=wx=_F(#N(l`N$~EzBgrqD9Q#dM2VFQ7+wDRZRg!{)?oMJ&)!su$x zQ{VdWKvu6{C?t2C{}0@k^t8Nw3BC}n^yGN*Xb_$+ttvW+vOufc(fB{7Z0MS%5x;3(wNMh3zd#jPCMxV z`8uMSau9QZ{ZnCHPMK*X4V#JElJ|($)}`j`hR8Hy$R(3ac7#G2C!3l^oNoRJaCQXk zm>)v!0;ij^R9x7X41InTesDr*s{SeG8)^%$M0^3?mSGaQq5jH5YQT*Askc?&CMgh} z3JXYui3rsdv5vX-m1)=-&a@I?stB>d^sN-Q$W4QB)EGR2ulU`+JoVESl({fKSVB91`d=noOmFP1T3#(7B&du ze8)O2)URK#f?!8!?E&rI|Bj%>Cd!vRn_j9Ia%S=#HB7J@&`S$$YKdNe`GlRdstHgm zWiRk$E@qEv1QzDh9Ler0o0I0ONI3(ep6>}jZ0o+~*{eLn(THbd!Q(dhChT#g1oZ~A z`a{#wT#7({i>Yq2lhFCGjLpSI?x|6f*{S3W5xYKR5S-!Zndf9I0m3c&fQ&(&AJP_b3pMS>cB%h(M0I6NY~exYDeCxFqsQaxMO7qZ>&}AN z!tSYS+de;5B8uQ&wvGsN#uFp|=J>M2_Kr!3swD=P>QHCkMgSHk%Z7$?>XMJUpN*5} zKa+iwnD@#ual`P`6nxS>x4w8QCdRdOqTWm(I$J%YpiRh;8=TlxS1QqQ(en!wmq0w* zI1+NtMOxi>uAOp@o1JEVbwZ`qE2BEBF>tf= za8j0a*K-MoJb*Zu0IG?BSIE14mn8w-Rv?k*gDDU-YhC4T#fzZY&43=dw?%9AQLDA1dl&qC m;&B;$8_I&~av}fzwU_m6!2e?mW#|Fcp=KKHhmGoigy+BCWP#=Y diff --git a/summary_results.json b/summary_results.json index dd33db0c7..cf8ec39bc 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419011, + "Performance_Result": 0.419137, "Performance_Units": "Samples/s" }, { From 22c48f4ee819c36b6a4b07fcfa38fcfe0e876bea Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 13 May 2025 16:07:30 +0000 Subject: [PATCH 546/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10930 -> 10930 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b2c707d5f..12ee0ae60 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-11 12:20:50,500 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-11 12:20:50,500 submission_checker1.py:3308 INFO] --- -[2025-05-11 12:20:50,500 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-13 16:07:24,861 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-13 16:07:24,861 submission_checker1.py:3308 INFO] --- +[2025-05-13 16:07:24,861 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-11 12:20:50,500 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-13 16:07:24,861 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-11 12:20:50,500 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-13 16:07:24,861 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-11 12:20:50,500 submission_checker1.py:3324 INFO] --- -[2025-05-11 12:20:50,500 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-11 12:20:50,500 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-11 12:20:50,500 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-11 12:20:50,501 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-11 12:20:50,501 submission_checker1.py:3345 INFO] --- -[2025-05-11 12:20:50,501 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-11 12:20:51,561 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-13 16:07:24,861 submission_checker1.py:3324 INFO] --- +[2025-05-13 16:07:24,861 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-13 16:07:24,861 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-13 16:07:24,861 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-13 16:07:24,861 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-13 16:07:24,861 submission_checker1.py:3345 INFO] --- +[2025-05-13 16:07:24,861 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-13 16:07:26,095 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2154 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9539 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 6e64a9148..e7ff19bd8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1173 +1174 diff --git a/summary.csv b/summary.csv index 55cc1a749..9e6e02543 100644 --- a/summary.csv +++ b/summary.csv @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2154,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9539,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 195041b23..f5428b8a9 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2154, + "Result":23.1481, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 473d4d0c770ed6e2cd99d61d751ef61273d72012..933a0b037a7a56d56d38f0f08a1717664672b897 100644 GIT binary patch delta 1605 zcmV-L2DMgc5M^3V3h2P{GUq0K?3$kA5mi+dY_Dawse~3p)KO_@M>4#(x%?i>F%K(}r18AC@AJze&X;OYPC_m5= zkwMUug|aRy!%dTTL9l7oto(c}nom+?YguHaO|$6yXuGAI(&KE2@}p55@ul)ZGRl(9 z4}S?EniZTMmH{+L2GBGqKdb{l(`5W;Fn*vVVwVlk1mlO4;jT%##$B^!xuv zI6u%5(P>Im7RtJ;4EIf92Eo2rv-I=zaDP7KNV})%&!sM^KiR%%X}|2{*%I?dqe9|K z^M_=VCDk7iLN+U?KP&@mGI0mkG^syq6kyY2{%A0Npe3Ty1UA9^A!WF6Rxy8c&F0Vd zVRK*oA1(bK<$EPhH|-Mt$3~IFm;MjQC`$@JBm{7>jD(YAB%EwQ0HM!PJdGe z9UBB7Xo=V~!DENAE-jNACkesEY44N&^LL5twJfsA#%UKdpxigmrGNRoL<6$XC-J2Q zL^8^f9uNs3n-x4DmH{?NMn2)RO&SpE0MImPKsIPV&=QeBz+;E1E~}E;Cc8sm+qC!2 z0{XThdoAm$vSZps2`Jk!Em!N3N#aZYhh&r`1t1baBr6C& zECWa~wFZzh89;0lK+a6B-gx~{ zcIjw((^&ku{*XNH@bq_W<(>71m&?~cPyzq|5dZ)HWN%|oa&K^RFJo_VWiEJa zY@L$JYQr!PMfd#*qrELZa9V`zKwGkEpwN_-?u0aPM6CysTE~B1#Yvn5yy|jv@0__a zs9Nr27yW{DLaQ2ONs1y+Omm^y8oh5;@dJvSr;5`~E2z-`4lSFziW$Oxw1roz4OlP0 zMFFH7VWvhq?+wA&u^mY2lHgE5+-fVS52bCfq3oNsfb%q6UM1?|V>1!a3CY=n zG(WZ*0i0pzK!WlPXGw-HUJq8f*@jWMa!TO`1GCn%IJx&uOvb+NlfD?OA%BKH*Dr75 zso5|#^~v6lk+HB0S1## zDLes?lZ+`r0ezFsDLMh~lL;y&8(fNUO(g*U08jz|01*HH00000000000DytelS(RR DN}c-E delta 1605 zcmV-L2DcCvv&xt2u@aR8J4;POVmg_Y5$N+DD5ATK`blSKP&@S zGOGr#G}%9F6u{CX|7>r3z!KyiI!(Y5J0)DAKjKO4hh##j{g4cDSwZ_@8Q_v+fJ>A0!#V&iP127B=?7XOG6=e| zP}XH-c#0)v5S(IbmVO@N@|(!AwJfsINw#SHXx$2&!Y^g*lIfO^iSmdir5}0Fk^wSJ&JXJV$TTTG8k8SsiO3*?e@sw*NEsev*@pP=25# zB7>kQ3uRqahMOkwf?(6ES^4=|G@qo(*0RV-n`Y7Z$zT3)N{_Q8%8y2M#Fxqs$tX)Y ze?KIIXjX83SO(A}89>vd{ICuHO_TAX!T5oeh+Q^B6O12HhPx)|8h6c_ji2v>=AJ4) zSGuVD~eQGiX8`J=)7ftH9)6W9dvhm_&QS;hR(HJd-* zhs}NUf3);}l<$>1-Ly;m9~(syU-~~Jqbw-^kr2SiG7?Ufk#MpJ0i3o;0Aiy^e>hDY zbZiiSpe16{1dkocy0lDgoFoJrr@c@9&)+4o*Rsee8>d~=fU=EK>C(UaUZMfn=#%(T z10oq^Ne_sGkj)An5X%6YBqN`2+9nN%bpU9ZG$0!^AZUrmAmFh>RhL!CZIj(0ux;9V zX90a%k-e7nRoOA^q6C!fn3k(`f5~yyF0p`Y^htba0g;Tdqy|JnNM!{Lh-H9Ek^w4B z77*(Is5DtX+Z!LK1Ph2=Ht^V?tV_xe%PJO7VFCM&<&#AATGm$yOOSy4(#^6c0jb~t zOY-p}QUW3wWl0N&gb>RL77)t-mdvk#gRDsdVxs_;CJo324G3ByHck0ae;4Oa)}>`| zkgf3Nf9yTifYM=(qaonMRokG5K2pi5qqwVe_q(IoMu|3fm$k^&G3A(9mY zAeI3nnOXx#nhYQ|3Lt4RfNU^;pe16{lpif~4rN_h21r&gfb2atfHp{mPeO8E#UI-d zNV>M%CEJA$cbmlXoA@*ye~%Z9ev)0Tb>+5o{fT#%>hQz*!{Hv{*?7rxFv|aueQ&(} zD7$nty=g4|Tz^QOcX;}{w(`#U!^`D3Zlqsk@o_W$Jj?E(e6h{O;PY_(@pz5de3Z;b z>6i8B`RDU3$GwwDGWc;dspWL^=Q95BhyD-V;SYW9r=J`8r^az3PFwnYG+W(jl>7@e zt&^i!?aM^Y5~s zcf%w}@4v|}$@YT_PyPb41{T;3e}X?wY$O2y08jz|01*HH0Az1tP;zf@b1!3Wa%C=f zZET&A&1%Ci5QOi23ZuI%%fY1x+kv*^)IgywExie8lZaYBNNOFweHA-#63D5?)y#f7 zyQo_3RhRsN^U~-Vkt|0EXl{hm+Zw&MtMmaSJ}@ma&)xt|`I`-+a^X}m3>I!$PvY#}dpR5XzR&u_XpQ*^{#?Jj zjjN_oA5O&qHI?9$J7B@MM5e-*MLb-`d|1a5ZzPDvA?i2Yj*CovAd(n>PH|7l$KvU^ zU7@DPi!#m2l#rGbw8&{mz78X=`p-ah9P)4C2{7fp{<9$_%>o6VV5>cClk+HB0SA*$ zDLes@lZ+`r0e+LtDLMi0lL;y&8-hPhY$O2y08jz|01*HH00000000000DytflS(RR DQpo+) diff --git a/summary_results.json b/summary_results.json index cf8ec39bc..246b548e7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2154, + "Performance_Result": 23.1481, "Performance_Units": "Samples/s" }, { From 8b60b1cb1c59ab1432b1140fc499120915f9649e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 13 May 2025 19:45:14 +0000 Subject: [PATCH 547/719] Updated results summary [skip ci] --- README.md | 36 ++++++++++++++++++------------------ dbversion | 2 +- summary.csv | 10 +++++----- summary.json | 12 ++++++------ summary.xlsx | Bin 10930 -> 10938 bytes summary_results.json | 12 ++++++------ 6 files changed, 36 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 12ee0ae60..ed588c15e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-13 16:07:24,861 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-13 16:07:24,861 submission_checker1.py:3308 INFO] --- -[2025-05-13 16:07:24,861 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-13 19:45:09,195 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-13 19:45:09,195 submission_checker1.py:3308 INFO] --- +[2025-05-13 19:45:09,195 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-13 16:07:24,861 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-13 19:45:09,195 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-13 16:07:24,861 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-13 19:45:09,195 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-13 16:07:24,861 submission_checker1.py:3324 INFO] --- -[2025-05-13 16:07:24,861 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-13 16:07:24,861 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-13 16:07:24,861 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-13 16:07:24,861 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-13 16:07:24,861 submission_checker1.py:3345 INFO] --- -[2025-05-13 16:07:24,861 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-13 16:07:26,095 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-13 19:45:09,195 submission_checker1.py:3324 INFO] --- +[2025-05-13 19:45:09,195 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-13 19:45:09,195 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-13 19:45:09,195 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-13 19:45:09,195 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-13 19:45:09,195 submission_checker1.py:3345 INFO] --- +[2025-05-13 19:45:09,195 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-13 19:45:10,278 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,17 +25,17 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71494 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.3567 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72287 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.3946 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419137 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421396 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1249 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -52,4 +52,4 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9539 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 31 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1107 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index e7ff19bd8..1c509f5be 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1174 +1175 diff --git a/summary.csv b/summary.csv index 9e6e02543..5e7e5aa5d 100644 --- a/summary.csv +++ b/summary.csv @@ -3,17 +3,17 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71494,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.3567,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72287,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.3946,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419137,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421396,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1249,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -30,4 +30,4 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9539,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1107,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f5428b8a9..737a6d842 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71494, + "Result":1.72287, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -159,7 +159,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.3567, + "Result":20.3946, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419137, + "Result":0.421396, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1249, + "Result":22.5761, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -939,7 +939,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9539, + "Result":21.1107, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -949,7 +949,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 933a0b037a7a56d56d38f0f08a1717664672b897..819f3846b3fcb5af8e7c3b1b481f6f51ba3f9084 100644 GIT binary patch delta 2279 zcmVvv&w@1AkiCVF`CW=gvp&#DncQMglR7Q;Yz2BqF&Z z5y>5ih)3Tm-+&z{7u3Enly|UP$*t(DSTBAYM_BKOD|zlqCP|H$@>?=3EjF$TzeXl+ zZm)a&{3*>r_6%m-)xk|h@Ztb_hiX{Q2XU_gLI2iX< zJ$w$lZ+$kcV|f)IybKEv1})e23wzZLgUaF%uG$5K;=X-OGJ$yHo{|i3$TAW{mXRcq zjFK;%yxN8si?jy`qp63^_Qoxa&=S#6#QuE_WnEGxUvqrRy4K>S(M4ea`+ouD!+w|p zIf&y~_K=(};_Exr3bHNntFf$Qg%!Abx0%=)Ouywgt}1zPQ1MsYpDBEm-E}3HmDV7fTk?#k}`N%m33`f5>5RdK(mfDR(Yj$GqLS+ zO4oM1vhyr|qx6}=_a9wfvWc0%lWbxJ*(4cYlVpHR^6oCOX`9$2J%1pZh8{ZG8z0z| zEfF0hkZj7bE-S-r6GOOd+W#mv)ss%Nb*!<{wrLl)&9LQLZpq6%iodh}OyL{mZXnSl zZ@c>vP0S#gBm-!Y44{b_f~I3alk|XS8hYq#Z+t*gwnTIkKvR}=Nf~aNq-)$Zo&Nxu zb*!<{w(0C{n}MVKvVU!}#6?jk7r3Eh6Ei^|*~AR8Nix7D$pD*7paC{r6PwJh0X7Xi zbhbA>uqj(2I!Y+nlx1C3hTA5FaNBhMVQfC-NL$AmD{Y%@aocoTUg(ygSz_%d)ImI{ z?T}0$y&aN4G%J`pECXmVcL&fk$vdnEMAPK&C=hs{C8DDMnt$N%kTTphN!Pe-)~WG) zFEsbn>ruO6j}9%XOsG>kW{>s2CTby`RC`D!lx`2nAe$BJ9+m+%iOPa(nhYP-1F~t7 ze3Vph&Y&e?qezAG9Ll=147N>T{J^$Zv(EFCZ0aW+Y4_Cgxl~2bM`vf+uuLvcK?$0u ziFi`=A(>FRK7S;GXjZU&SO(B!#tpo&X)=Cj4}hjg`cYEJIfIsni~?p4RCQSu?3!f$ zxNFvI_dG(T@O-`=$~`rHuGFsbv3QMR&mBf?M56Ohm`XL#lvV!-+GC(Es zYJf_U`a^pFDoyf_0{aJAA~p*ALIad_Nf}~Ex<)Mb&3^$Z#nOKgmV4>|U8!&vOFIZc zhhwQNj-^JO#FG{f$%N7ZA{oGvWh9m?Be5hIC4W2Wl{Q&GY!vw$BBl;H8YCcSiP$vd z?+~3sS(lc{V=N2VVv-lSW(#QfF*~I$H(UxP)4R#)Ae*0HRoK#BPh~But8#+XMFU8+ ze653$y?;`x5-8Cm@uUGnGNCkpNCu&-U;wcUK*`J+Eul0SKx`C%(j)+B5P+a1qSKVB zER=Ow8A4e>0Ma!JKs!wS9lmTW3#^1?(f-kPOF3o6tkxx1qDJCL`-fygY5$N6Vp+ld zVHv=ZSv7#A$^K!Z0G1~CXM5uVmLUJoX#$oY{~VAqIKi^6;RIV)z`qpBeKmjduKD9+ z_l;L}hPAVG5pV*3Y;CSz_7@H85)H^kpTw6M5XmS@dO#$EY*z4qSO(Z68To|MHfcbt z13=TH0okAdK}$pi0goN3x~xiWo9qsOZPVU63+UU5?6s_~%8qFlC7^7_v|O!AjP-(J&SO-9*$pYFZ-uOTzSU~KufyWMIT~dZv zRlH_Kx_YTB?Z38p(Fy!`&wF{3bq)$Kyq# zpJbP7UAb*tf8yPxI{dKyaJYwfHeNCvjPk!^-y5$#$}SyEZyJj~*B_GS9iINKt-Q1T z@Nzkh8|jx>eB6vb&$4?cUu?56_&i*HJYHiqA0_iq`ei+O{`q{%aqnc341QcqYB?Qa z{ke>P{GtDYclbl!`|0O~{;6@?$d-N|%~rP>CI5m=>*Q$GyNS1UV_#5N4DF*+t-97fW)cC^~`=dyK=GEt0wsc=cUml$}@rz(A)^A z>k@sem+2!)d|+Cz#%L(f0X$k%cLleUe;WsH&RB3kf=?nyd&+Hzb|F}bvFAHb%xBS| zqqsFru@FmFW6StAs{v04dB6$+6D%-%1g3T*q8=&uh_pYh=>URbXh4Au9_Lw(&)xt| z`I`-$a^X}m94y?l4&vzEdpR20w$0jUXN~!Dyji`zcdMpSA5X;rRfXV`J7B@MNvgn? zMLb-?d|1U3ZzYJQL)34)^^2VPKqN5$J;fcA&*}5adWotDnaol$OY`YEpVRvpCG)T2 z$gBP{Q1wIpO&kEH+}EF=`UA5hCdvW@D?Rt3bCdHZTLBG|PboYBmy?VsK>>u5&M7(p z_LB)JCL3Y|&k-g8002+|000pH0000000000004l2*ON*rJ^?I~i7F!o-X{P6001&p BF^m8J delta 2246 zcmV;%2s!tEXMIWPk&o3 zM={_i28g0~06oV@AkSPF0Tf9@qDUeVMG}!H1_l&KxkS-W-obK2F)*Nr^?+g^Q6x2D z(|${)rNy!?f`1Q+bu6U89=>fqs^w3fAorN=KiRdW41XGrESsCn=Bm|pfVFCLH zW?`qy$<$U$Jps)+)>viB+<#0=ZC-qC1-@7E98&T3o}MXuJID1TnwSY(i6&+cO_BjL zNe0lw3_(+y&?G$|nuZ=a+Z!Lylr0e*1<;gbT~Y=wtFo?bOQNa&18CN<#wxG0ZYH*U zPU+gNS9YG|ZHffz)@5b5ZDI(wP5U3krh3wewvIJc+BWUtwi&j3%Po1iNAY*|pDBF9+zlj} z_ zvyL@Z+BTitZ8LDRUw^i3mbfSi3AKC+;X_9`FRC3OsB_gAM*#lKwRt38z znLqBDHQPOpkSRQ$uZMC^jh`#EtNb{w@Azd@h9y*@O5#cBhh#!&{g4b&S;6~Z8K9DR zH9)0F{h>Vol_vQ|f&Bw55gP@5p#jReqzthnT_cwJ=6?W{V(C8#%RO~~u2i^-r5yyJ z!?DyB$5NwC;zD^>?kj+o9Ds1Vmr?QsSRXM@xq5-5@ zzH&;TRDUH#{P0vVs7lYZicZnEX3@*;*D@3Cp7WqwSV*%8Xg9ORz+Z#FO?9$%NAWAsNK7g8joX zfF-kP085ko!$tutP4dt7#s@4x{-M(ZEJ6MuWg2jTWnIGwwy=PIDVFj2O+ zX+Sn;K+qDALBL~&sxGUN+a|k1VB56!&I0qZMjRf3m@(_iRU-*X*?b;8vP`@T-rP#F4f_O^@qbf#Ix~|>0p%qCHvlZ{ZV%5XnNCF{JH*+Jn!)IcWvdJ^@o?saokA1 z%;Mu_{CSq$L-}Hxjlt*P`s48$v-v2QkJ2ye(euyeTaJ4tlVtGYYEsMT=+9+f{NoS( zAH2gK`rc1JH}p@9<3_gh`)Ibh)hPKFY+5Hrv))a7-kaZyri;cn9%c)*T7I)JUw`1< z>gzO_t-f-bjgLv1CX@T`ZsXn{p6B0XKktS~lHPxlUy|(y7oPkDvkey54h39_aZM$Y z>Leb2%WA_g5JmU>3ZuO(KX6)v?Lb?yYM{`RmhOZ!aYU^Ll3K@qU&TqB1ib2Ubnl$G zGpJhbWf%Q|bwaBeWl4%6P)u{7+8VuYR`COhoTrM@PAjO<01hpiyNVgYw1roz4OlP0 zMFFH7VWvhq?+wA&u^mY2lHgE5+-fVS52bB?v7zjnwt(|AU0?~Ga_T8Q0An)|(Fw`f zgfu_48v&eQ=s<$<4rfV*FJ2E;y4i+NxpGS32LrR#vpBi;PE5wW?~}e5ts#GgKi4mB z Date: Tue, 13 May 2025 19:46:45 +0000 Subject: [PATCH 548/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10938 -> 10937 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ed588c15e..babe64df5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-13 19:45:09,195 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-13 19:45:09,195 submission_checker1.py:3308 INFO] --- -[2025-05-13 19:45:09,195 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-13 19:46:40,349 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-13 19:46:40,349 submission_checker1.py:3308 INFO] --- +[2025-05-13 19:46:40,350 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-13 19:45:09,195 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-13 19:46:40,350 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-13 19:45:09,195 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-13 19:46:40,350 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-13 19:45:09,195 submission_checker1.py:3324 INFO] --- -[2025-05-13 19:45:09,195 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-13 19:45:09,195 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-13 19:45:09,195 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-13 19:45:09,195 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-13 19:45:09,195 submission_checker1.py:3345 INFO] --- -[2025-05-13 19:45:09,195 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-13 19:45:10,278 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-13 19:46:40,350 submission_checker1.py:3324 INFO] --- +[2025-05-13 19:46:40,350 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-13 19:46:40,350 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-13 19:46:40,350 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-13 19:46:40,350 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-13 19:46:40,350 submission_checker1.py:3345 INFO] --- +[2025-05-13 19:46:40,350 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-13 19:46:41,460 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 1c509f5be..5ddd971df 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1175 +1176 diff --git a/summary.xlsx b/summary.xlsx index 819f3846b3fcb5af8e7c3b1b481f6f51ba3f9084..0980cb8bcf76fbacfdba78f8ffeba550a8bbd604 100644 GIT binary patch delta 339 zcmV-Z0j&PIRk>BL+ zPg9HdA={AV?{U*KZKbSOj_;i_cSg?F2iZiwV4cvaKv|Na2o%#?sJcM!)h512k@Hk> z+Gqs@>cF9Od6zRon6~h0wE^n|xF~>>Bg_RkT z(;DzBO;=cgr<{6wr$eRduzy_;m_^MTfb^7)bUgdP?>W^m<8&!EpvQbgu^w|`E5AyP5^)G zf1-LN%Q%}?*#lXu$RhnZj=br=0Oc^`-^4TE ll>7D*lz+1fFc5_AeTvBU#153ISV@rDic^s)m8z&W z%U*)j_(!(ez}we^B!I-J$Mwv9JG*kR*sCV_1?Q#FCCW2`642ZTsp}Gbte5E{N_=2i zu*PU8(E&VKRCfiplp6&|p2zkH?0uwASd<3R;B%&TE_=vPWuIT`RV`xBu4j$)Oj?dlzPWhV+opRw+G8`=2 zv<~9v-g`M3+qTWxX=jc3bG%u-zIUspQXfyn0ab Date: Tue, 13 May 2025 19:54:10 +0000 Subject: [PATCH 549/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10937 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index babe64df5..a925f3e23 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-13 19:46:40,349 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-13 19:46:40,349 submission_checker1.py:3308 INFO] --- -[2025-05-13 19:46:40,350 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-13 19:54:05,172 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-13 19:54:05,172 submission_checker1.py:3308 INFO] --- +[2025-05-13 19:54:05,172 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-13 19:46:40,350 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-13 19:54:05,172 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-13 19:46:40,350 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-13 19:54:05,172 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-13 19:46:40,350 submission_checker1.py:3324 INFO] --- -[2025-05-13 19:46:40,350 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-13 19:46:40,350 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-13 19:46:40,350 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-13 19:46:40,350 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-13 19:46:40,350 submission_checker1.py:3345 INFO] --- -[2025-05-13 19:46:40,350 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-13 19:46:41,460 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-13 19:54:05,172 submission_checker1.py:3324 INFO] --- +[2025-05-13 19:54:05,172 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-13 19:54:05,172 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-13 19:54:05,172 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-13 19:54:05,172 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-13 19:54:05,172 submission_checker1.py:3345 INFO] --- +[2025-05-13 19:54:05,172 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-13 19:54:06,200 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -31,11 +31,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421396 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421307 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5761 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.033 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5ddd971df..bacbcf473 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1176 +1177 diff --git a/summary.csv b/summary.csv index 5e7e5aa5d..63cb437db 100644 --- a/summary.csv +++ b/summary.csv @@ -9,11 +9,11 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421396,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421307,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5761,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.033,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 737a6d842..b6abf5f7f 100644 --- a/summary.json +++ b/summary.json @@ -309,7 +309,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421396, + "Result":0.421307, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -429,7 +429,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.5761, + "Result":23.033, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0980cb8bcf76fbacfdba78f8ffeba550a8bbd604..39516fc874e45b8ea22babba3fc3104a8d45e560 100644 GIT binary patch delta 3030 zcmYLLdpOhm|K8>}(HN;Yv|(Fl^0a1^Vni{T93oS4Xf#ES4l2dwd@A8HrnYj*;VHhc zkwZm^(#UCwIi{p6JVMUmJN>Tf_kI8I{^xyN_jTW|*L~mDyT-N3wd|7o=GG&tS{Ic- zpp{?{Xd8&7VxUpZk#~?{x(V3dF8>?$)#0Eh>kL}CYu_9BU>jZF06c9VE#dmF+8HHL z+9k_LO{-A3mW+XuA7-1cR5DE}`$pv!CZ_GvDT)^|WUTwP@|$ z-JaS5H0!pye8CIm&G4d=r&~fk= z$rjaqY8zOMY@KJT3qK4`)0dT1BiC4WIu6xevflkyPQqM-FUkpR;@icrFD!V&-xbA# zC57M$AW6um2SJ@zE2h%5Y^>UR6L1Pi8(Ls|m)oFQXk98=MWA~m9kZQNdPwSa{%Lci zn`i!+ih*S8=~ZU(RKA8DR}Uelg?*b=-*YEy4c@(JAiw9By4=Xnl^pOd!j4G%8xUa68auC{EmV z(|%24N4k-UwoFI5pQp2|aWVf%kbT~xbt!r`@2z2J#RY6-xP!Rw$%(63qkpHHyDY9f z9JeR#R=o~T-lMo^sX#!nj65`Wm>8tW&6lGwyE?4S-W-wQ@-cZ2r2z`H+t05$$XC9K zvVwe5cte<2dB?=9k2v|I5>Xk}Via0qGktA7+aRoAKnczH8g$JoFA-tQqJcK3?zZZ< za+D5iSEg9O9!w^~=*{eeI}my^f0sc+5bH{Ef8i|BqWllugTbIZ!4+PWK}>eFzQQPE zZ+f6FRvyj0Ye^)_iqHse7L@R+@VqZ;$>ddGZhYJPTREbiy|oP#!{wK>Gza%LDqrCK zOS;`!q!lpkex@L(%jg34Ld&K>Ji)$1JJvK|$5U+pnqOswPw1R9FY9JqpgUqGqz|E6KN!BI5b( z&^mzfJ6~@y>xOyeWBui^iKCiqfsCQPrK*X_vB&0URlA;e8dJGdp%T^t;=Hep+)Jdq zg-wDx2xO9-;r_E7@HD4a6OoU16-Rlp-XpKIawfBE-l=7^_gK{wy=~9vb7g4tTw8^! zy2WCS=h=NBi{ojB0)D3|I#cUcd=NQ6sEI`Z`FE}vC6Zu=pNJUeIkqd5qmaoV| zd+M=QzJW5BzG4!z#VjP^$?@u21=`ZN4ZIKPP0kM;zK}O1NZ`O}irxY|+@7804%`M_ zYR}nmaVTQas*;L}oVF5NSt6V>b1k7w60Mf6yV=zO)Z8V*5)aZs zQd^Ie83u|=i}>u1uKVkMbfljrysd8q3Lwsx)7v$&r!ny9qs7$SA@H44$VKA z?Z{C?5fv-~;1L9Qiw<~afSs#)^Rp7g+!7Q>%#q2_#;$Mns$|Q!z2?)`>CX%9BHBBaf(>W>5XpCW3{FmIewi)@_X-kNxd<0ZX&a;q<;M_9L&J(=kM%^XX)?!2ON z&+!}y9ev8JgW)b)%|s)Hg|y#gnCB2d5n~0Du5^)N$o2pl_ z{6M2yC076=50Xa}r0AzT^*QYshL(5m|JH6GSvx?rT=qerShR?vtl0)r5T4SUB0DPQ zp^KN-QB6lW$H`IWd%}iMC(CDM!`}_KeE)8@^#L<`RfcMO#h$TZe-lFoNR4_0NB~;! z>rZgR8^MtTo6-gw2Xzn(NE+!#dg412z4;I|Gvjk1`2|WuIPb=&Q*Yhzi2oGF7Zy^l zFe(Ox4&sZI)=QoL+*x#9(n0utllaA~L=LFyxgl!~d2b|XuM~|Ae zY7Xu}dyYJzmQT-ycMQ0UO4wy&hxPnwcsQ{m)JacS6#jt|$H%0$DhMTT+@rmb zJ^mCSOjBl`VEBE_2_Vw|>kH{8Ull zH9r%Wxb)CThdEuaWQA`Ae!!qQ+$|rFY%uX&wNxPI1Cppv7Jz3G?vggV2jP0JrQ!$n z72hR2^Pl~`QdU#Cmcm|3nW+VexK|U`&pfpKV!s|wOUKBd;E;XPBd;Z)aUkkMbP=Hz zfojmAN6~kbAb-*_%-d)QK{6}Gd!%B4DOg{6ZawdzPW2n9xTWD(cd-v+EbPOmrzV>w z8XKViC-#R3TlvnC957EYhm4`#l{lYWKq~+cj|4vWOlgf{h!wHH~ zY~>OpnlUK&A>2+|3!IcYzE-x4huxHr3^~QYp)_$QdZLIhy2JDSpK9D+`Pyrpw=e*) zi{B6YATJ*?VxBL)5@z&+)ORj6a#tv&`>I{4by{QQGEBm9nS_q9b^FP5mrbSlE1@J! zAkvS>CxiHwAc<~@3xofDe|ks&0!6h1@fS{k`cMdSs33ffF`+$W^E5pwMK55hd4CYR z{qHYb*AJd1ER(O+bZ7K;&&dsWK5l@Gx5wQ~V4fYpbk*yxTg}%_FS90uoxcisi_3?@I(_ z@7Cmt2j8ood%Eg>rf%&;?Z=(~{K@c9q9&3>pkNn14Zf|lJlOuSg)yYWLf5WiU!wcl zjSentDpyh$ZI<@-*dfajqj5|f`4V&nDC95(9c3|ryV>W_2C9y&0kR4?Lb+x`HC9W4jwJ0HOc27Rr1_YOi%=Zm%3!v@TvtS{TQaqlHy z(P#GvS4CGr!Ew1V(B=d8F$nWeGk5V5qGIzs7X{{D!5?p^on078>=lK03P+<&zPA7T zzC0S_ZleB}v_vH*_IbBb+iyn>CAsTF3el;dw`eVwAF%o!`@by!pp7;CkCFl=Q`A%#z1p5DfZ`PXr z3GgPCo54YF28(52365fYG%yAKVksM9H8+Nd0fRt}5D-WO^uH4k`H|&hXbv`Dr5ftV J_V3tu`7aBMkf8to delta 3063 zcmYLLc{r3^AAZKjk`YaoYz=y58VuFMlvKu&Z4fG%# z09fXQO3~ra8R?KiqixpzHb?(3wi4TY^Jo$f0&3T6-^|xYh`0W#_m!ccqx`;ZH#?;4 z6#wyej@!-%b!)e&c)hFtk}$S6GiKcJ`uc=YnR+YCMZYJZE6X}>&E^n3pSv3!tcE5@ znQxY!_J-6BS8}3PU8u!kH-d46VmGE%#BMmY&r}QtKnn0=F9tSIvOO?KeCEyd&BRno zFcH*t1~(^oc@k6EBn8=F4V<+C9G>VVhm}AccG*t&@sx7%ByK``VLsRGm5Xlff|j69 ziEh2*$gkQOpkzUDcA8b+9(g4t+e_RH+|1WbuujQP+j`+wcL-&}7ORw=r%d}LGnZ(?g{rp4h)*BU)()kbK+U8#B zC$}RzoSRp}H+g%wU`_{ywbu3w6n7jx16zU(LroDGaO3zv@?6T@odXjl-5a_3(~rtI z-q#hY!=By|sM^;L?<>{4>hCmt;&YBp*zM23=!^_$b!p7i%54px3c#WO_HVvztC4ly zWh*CDloyxGoWk8TBUxHcqcgai))vnvpq2n9BO|chC1c|4SX6a#T-vv%{H3n@h>zEK zNH`Fv_wxmFG$pZqe%8MKqjBkuj%NFBc80s7eVK?{TRa2fBbE6f1~UfhFr01g`?A0& zq9uICGKz2$O#Bt{?O?h)(qRQfPKa&;bM%N{e$ zzGej17T9}j}x5M_ay**rGKVo7peGFPVLpwBV z+~>HsNjB z_zn%~{Ci=|vBMqK=gxg7vo~-ulg=zjxZ?6l>yA_HAut;9bu9~blW3>Blzm=K#Ygzc zF^X(|JG=0WqtoJ(#C+tA1(OvgGbyXp7kX&(2h59phShCrIcOA7`o45#u)1zAeK{a* zBdNslb~CF}jvv$cTT7*Y8#UjAC!&St8FsAS7NLNy!^|zQCCQW8gpJGQaJEPfFWXux z-27XZF7!_)giH$^2Rt5JQYM*5na7=dpr2f5mEFuzk{7{=X{$6ay}>lx_Hs}K4VjGJ zCXk6Vg+(>(qD}i%K~3KTf#L9!e6|HSJ&k`BC)4M}RIc@r@m<_)QfRRE)J9C#Gyb?{ zR*vU!LFefs-ujOo1w?fphTBwzJ&tC+aCrnFu^r%a0n8GgXUMRT$m4hd|nBrIoS;8G?F2j*sp5T(e!0FG2R`k9hXVQ-Y+>lXt(=T z1GY2{sNN3T4>r2;2gf(l9&<| z6)$yd(y0gS{jHo=+IY6tDAY85c&~Eb1+tf7s)uqV>PcbvPTa?x*+_#9)O5j|1wQY6 zgpEWECwHX`aS0OL_d@>fGa5iTX3RQz7OuZpMcjjoo#5gR{Da&hh&63Trkb_0udnBX1M7-e8MTMA9HyUn#KrppnNsM$`@Zb^ ztT4NCkd`o$@2oLhIAeitVYbOhYh^0lMWw+EZ6(?bbxb*MVi#rEcUJ@?vQD~ehxlL| zyT8=7_N52UXK-{Wx2Cv7#7@_Av2-l%dc7;J)h9ui-?44FP`D1)DT2#VED*sF?r`Bx z-ewk{2pb!y0TG7#MEp9acE^=kYB`8`2QP`nIx=ro`0J7=D4rFMzbiC&XxdjtrPfal z_aTMW`R337Nf<|Aqq|W^bAn!pDGpA<`%4N8ADGe3CF9LrwgTTN=;CaSilYYCEJGUs9(a)|xORVX_j(ZFxymHp->ZOHa=!%T6L?Ut8rg zr>n|~Ag>X@7@F%(hMR-{R9TGaB2bCz^(%n&<-$Y{jRfqL#=}~UonO}Erbv3%7ZaK9 z9}_1lIZs-WsmkbF+N+9Fk4eeA%LyAO`9}swGV!A08fhZUb@hCSI0$}>iEyHF8E%rq zLcZ9CFAGGp8*!2w`D*>OjCFG@XKa)agy3>OZZEga&MiuOC_Td7&RNW*5uP?QXk8#~ zDH23UURolsr8$td;hyNL1-zg zuu0wa7r0gorBxo^U$3-+qF4S+On;8OV1l-}Z#zI*TNh=6h$W~eMa02v@RbrNe*%Hn zNdfvi0W(r_k&fxfkw1}}B1(C zOCLYyt^N!!PF5RBCx?AXGJVGl#h#||b}t+_1s(Y7ECGcUI{)@Qcx`nttkr*t!bC)z zayW0Su~L%+E$6L8hb&lpT^bvWU47vH>2+EJ>aFWjRrFev!^Dtp&$EPgp~5WVj-ZaM zubOCiuB{gdBNK9;+{3dmyzUczyy@MxU6g>e&-71f&BEnplcS$^*>)~noFQ>nzWvRr z2Y50{f=%L8BdV3GQ4k01jJ#cd%q=a6v6PR`_C9j-#)8(zl#wa_E<<`D$1Qm(dryc} zTd!oFWxZ#1`$Hm`d0$GQ>r_cCajF|rXPqiM=KsNW!-nyOyx^C&f zGT7VSDELmj(AsKJC28w{RM(9+p`(B0RTvT2;a#dfaSkWkqKfZA*Aj0v2$C!fHI3C$ zM_v&QGWH;3ZJfUJ(ArfPai`#-3heH*f3tH9~kBl(`591k-l%*5r8J64owqq0psv(*MT90c~01yoO_RC zb1HisPrzhZ z$8-+B(pjlGX0kEr0C0u2Gcf2X%{!2#sjI*e>Zpp%ZDYx!6j`9Iy!v{D(J%mT5(9wE k!2cT~vKM4cgIm@YvG!u*W&StCh}2^x>u#6$ykmXjKSVj2djJ3c diff --git a/summary_results.json b/summary_results.json index 24356e4ea..baaf3312f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421396, + "Performance_Result": 0.421307, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.5761, + "Performance_Result": 23.033, "Performance_Units": "Samples/s" }, { From ea0e598f35ea58e6178977c380ab2e040fa7aa7f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 13 May 2025 20:01:10 +0000 Subject: [PATCH 550/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index a925f3e23..eac13471e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-13 19:54:05,172 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-13 19:54:05,172 submission_checker1.py:3308 INFO] --- -[2025-05-13 19:54:05,172 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-13 20:01:04,923 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-13 20:01:04,923 submission_checker1.py:3308 INFO] --- +[2025-05-13 20:01:04,923 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-13 19:54:05,172 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-13 20:01:04,923 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-13 19:54:05,172 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-13 20:01:04,923 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-13 19:54:05,172 submission_checker1.py:3324 INFO] --- -[2025-05-13 19:54:05,172 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-13 19:54:05,172 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-13 19:54:05,172 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-13 19:54:05,172 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-13 19:54:05,172 submission_checker1.py:3345 INFO] --- -[2025-05-13 19:54:05,172 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-13 19:54:06,200 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-13 20:01:04,923 submission_checker1.py:3324 INFO] --- +[2025-05-13 20:01:04,923 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-13 20:01:04,923 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-13 20:01:04,923 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-13 20:01:04,923 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-13 20:01:04,923 submission_checker1.py:3345 INFO] --- +[2025-05-13 20:01:04,923 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-13 20:01:06,154 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72287 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.3946 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.27691 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index bacbcf473..38ca1c199 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1177 +1178 diff --git a/summary.csv b/summary.csv index 63cb437db..4a494b215 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72287,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.3946,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.27691,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b6abf5f7f..223182e47 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.27691, + "Result":6.28538, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -199,7 +199,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1012-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 39516fc874e45b8ea22babba3fc3104a8d45e560..f18832c4f8dec1d785f0abdf8c30766d884b6d08 100644 GIT binary patch delta 1538 zcmV+d2L1WBRku~JatH+&O5xgavv&we34eiG_!YukE}arC(I4@o_CqqE)P6_?xvZf5 zuncfXGQg$D`e7XamnP{)gY*L}5g7zsSt#qWGCai+GYC$xHA_E_arsST*;*D^=_FgU zeza}{PT`lbcFA;0$V7R>lhO~#gi`t;86>lU^usbhCdmMqCg+EB0A!k!9}UV6w0}fo z5JDy>Kcox~vh1(&j5xu?$0l`c9z`3tgM=$8EUm-b4~B!7rUNZOMgFK59d>k zw0o-lTzE|>e(=PFUY!pd+>Hm<7vZMe+ zLI5YrNH|$W!pSBCaM~sTh<}YD;WTy7u|WWWmWWLgJa#DS(lWVmk`Qd1_CEPPf0xK! z%Ob06oOV$I$~R8C2+CHeFb=>U<8vZMq=LWpGr35aC?OQzSrIo4zW zu~C3alLcgh1q3Y-n}4SKq>FPX>(Vkf$5#03KlYwmKW6(i$*`mF4wwp z+q(Y1yGwQWVg1=~5AkfgWI7n-f5^T!UVoBZI-1@z7JsfkBhNcL{assmXZ_*javV3( zFSGc#8GoK-_fWprW@GSqxc+p!#%w-H=A-n>di4C``Ih6}$s`&4xSG^*I{I@N|M)}y z2k-EQzW3A54O9J7$JNY! zJG-b}>{XZig7ebo2IUz+321JF)Y}HVub1fqN_=2iu+C^`&;TAS+Pj)t%8i3pXDqlN z!6y-|ARyAUkJ*z+AI=CkO~QQR1(Scs+De`3q{H`@X(2)V}!0uwASd;q3)CZZ!! z@EK`;TsHy)$IyWS9X!sn9G|@boboptM&-h(WEd>mw4TJ-z4vl9_I;oA`LK#7-bfISL)34)9T%DUKqN5$o#LL9kMikx zBfUgzL5eCRRhpOUf>4rEQhgmpUiF`W>Nw=z#1ml3ef!2#Em}WX zw*sf|OIf>Qx+P?yJmN{|hh#!2{g4cjSwZ??86cBnfJ~F~!#V&mP0Ei3p*L>h0TeI@>eaPHX=jTcnouB*#Sub=;e)~&%C1{dA#3Q92k_n~sLo$eF z1?h)n08NqsG)>MA>j2O+DL)#NA83ilAZW@$S(laJrb)aY*feWae!dpXC#kZvEV9z3 zS#*B1-O^6!akfPH(Ws92Qu!enWq(QMhlCK#3eFG90GcENXquEC)&ZbtGJZ4|KhP4f z%Z6xz@k7dR*Cbuzu35A3^L^0VQ|0GM7nPrU*Yv}}U#r<=v&8w)Xpi{P`5_r)N$H1# zkj)Cx56b|XOxc0gW=+x$8wKDrIX@blA83i_G^HvFWnET=`zA4iVBf4+`hWR)IG=K) z-Bb1FQWw>qY~QrBU-t5BiTR^ZA@Qa8Lo&*e>JJGan-$a_mH{@IxC3mO)E_nquxT=X zG?+io647Y_n_&KsGTb<;m_NE^^XL1pxv&0@mi~|Oy^^P!c8ULEqe$XQ|A%ChB?TZ7 z0ytSl!pSlcPBtNc(>4h}Y=0C9r>TRE4FV9fL~NSiu|rvxmdTBigka;e_sReHyF~U{ z7FlKEw2K;0zH!!AZUr$G=JqMU7SN%mzKdfw!&ZkvG?2pN{2a;>Ph-fEo-ZEd@brf+G>S? zE_qGXc1n;$k;Iq!56LJ?`adLuNLKKFSO$<}W(^=|5`fq!fTT$PvOxfXmWWMLezMFt zlyzwtAXz~GviDp7+8`M|3CVr+e(W9ZN41oKk0WUpKHF^)&wp>?(|9~yH2O()xz?52 z*7XP8U8=(m>(7RJh-c#^)4?eJL-xJ#`jhO^(e$RV_;dXkdEVja@7l^c>kluN-NwB^JkP(&e%=j}B)$J8za-nwEj;-PvkVs44h3LZCk7>x z=_DS1%TB{E5JmTVMdW?rq#+2bB$WVlMI|JHK8>^m%9&FNe=u;>dK72(-ig`RbzRcsgEi#O@aOvF zZCEuHsy`J2RF|9)W`TNb>k^+A;czW=ejQG{5x^f0LBH}gEi(22k3s-4#yu(@`O|YG zvqJSET@-Oz#97{C52UyyY4O#Myy!mxklO!q-8(>-|1| Date: Wed, 14 May 2025 15:37:39 +0000 Subject: [PATCH 551/719] Updated results summary [skip ci] --- README.md | 88 +++---- dbversion | 2 +- summary.csv | 50 ++-- summary.json | 600 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10935 -> 10936 bytes summary_results.json | 2 +- 6 files changed, 371 insertions(+), 371 deletions(-) diff --git a/README.md b/README.md index eac13471e..e4e8f49c7 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-13 20:01:04,923 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-13 20:01:04,923 submission_checker1.py:3308 INFO] --- -[2025-05-13 20:01:04,923 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-14 15:37:33,764 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-14 15:37:33,764 submission_checker1.py:3308 INFO] --- +[2025-05-14 15:37:33,764 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-13 20:01:04,923 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-14 15:37:33,764 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-13 20:01:04,923 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-14 15:37:33,764 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-13 20:01:04,923 submission_checker1.py:3324 INFO] --- -[2025-05-13 20:01:04,923 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-13 20:01:04,923 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-13 20:01:04,923 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-13 20:01:04,923 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-13 20:01:04,923 submission_checker1.py:3345 INFO] --- -[2025-05-13 20:01:04,923 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-13 20:01:06,154 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-14 15:37:33,764 submission_checker1.py:3324 INFO] --- +[2025-05-14 15:37:33,764 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-14 15:37:33,764 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-14 15:37:33,764 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-14 15:37:33,764 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-14 15:37:33,764 submission_checker1.py:3345 INFO] --- +[2025-05-14 15:37:33,764 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-14 15:37:34,808 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72287 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.3946 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421307 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.033 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 27 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1107 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72287 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.4194 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421307 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1107 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.033 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 38ca1c199..8c8b8bca0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1178 +1179 diff --git a/summary.csv b/summary.csv index 4a494b215..18ceafb2e 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72287,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.3946,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421307,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.033,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72287,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.4194,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421307,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1107,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.033,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 223182e47..84042f9b1 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.457477, + "Result":0.431769, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":23.0274, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.353561, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -94,22 +94,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.438894, - "Accuracy":"mAP: 76.951", + "Result":23.2491, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -124,22 +124,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":1.72287, - "Accuracy":"acc: 80.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.3946, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -214,22 +214,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.6951, - "Accuracy":"acc: 76.000", + "Result":1.72287, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -245,20 +245,20 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.290026, - "Accuracy":"mAP: 76.951", + "Result":20.4194, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -273,60 +273,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.421307, - "Accuracy":"mAP: 49.593", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "11":{ @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.405224, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -364,29 +364,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "13":{ @@ -394,22 +394,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":0.421307, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -423,13 +423,13 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":23.033, + "Result":10.1918, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -437,9 +437,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -454,22 +454,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":7.44519, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -484,29 +484,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "17":{ @@ -514,12 +514,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":23.1481, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -527,9 +527,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -544,22 +544,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.353561, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -574,22 +574,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0274, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -604,12 +604,12 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.431769, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -617,9 +617,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -663,23 +663,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.290026, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":15.6951, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -724,21 +724,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -754,22 +754,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.44519, - "Accuracy":"F1: 80.0", + "Result":21.1107, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -783,60 +783,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":23.033, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "28":{ @@ -844,21 +844,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":8.53781, - "Accuracy":"acc: 76.000", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -874,22 +874,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1481, + "Result":8.53781, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -904,22 +904,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.413064, + "Result":0.384271, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -934,22 +934,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.1107, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index f18832c4f8dec1d785f0abdf8c30766d884b6d08..ee1b6ccf10cf379234c2d2ecedd235a6081ee8aa 100644 GIT binary patch delta 1894 zcmV-s2buV{Rk&5KatH<9G;q#wvv&wn2MB1Z<*3jpK{J!m51fBxo29lmmKt>uPg+1E z6G{t+WB^N+kyx^f#FAu`{Ozb$+GGK-QRHukm^$cakbs~iV$+nrLv#*hU0NoOu`FbZ zNnYrhEuiJc?3B9Pa4DEf?wGK-5O07zuM3cmm z1`x@F(f}eEgtC8v0mL!@B{OTZgwkXHu~7g@lK`Ya0D_i?PE)F~P}XH-2xSEUNY^X? z?J)Uw__DPuuo9L<`$yX?<&+t-T9;sn8i^@pP=25#B7+bzLHQwNc#vg( z%?A#$H7h^ghs-^7ey()U`N?09^+LDgx4*Pkf+m0YLp)OYA(>E0KO}=_R*-&J2GArK zK-1*>unquClk%fM`GJ;*41%UClyzAdZkogkf=#n#<>zbBe3B|#%OWdnnnmYF+b!*c zWiS7fC_ft25nn1lB%>_p{E!f$S;6^X894$`n%?i>F%K)29 z*@4$)P0|k=1>iI}KN_4LXo=`Fr78<$T~>zsCNYCx->g~s`Fc2?a-`i;_2*I-)t_wN zw6tIL@@$FuqfsI8rTIfL%98322_c&m)E|GA0XCVq18kbqA2tfGX)=E_m_N`G(P;vk zVE&LY+&HV4Ke}e~=lig^ul|pg{*Us#lBb(?iT`7xNa9QXhh&r`1t1avI9W!*$ubg7 zHX(r1HVHs%6bYxPgN_XX5VS;Wn&7cRS(lc{jgy36(%K~!kc~QtFD)RFQI^zzNC?rapaHQAph+_F`KE2MfLI4WrpW@b!2*JohztT6 zJ5+U9mE1Mi9Rj}AOxAoX-Sznb6(=Iwd*@kJkT9=$=?Gg#dMxDf$5)jEKOIkoA zgj80rfLI2oBpIO6BmuDwfJ&1Dw7q}vfl82o*kuEa9m=|-46&>t0TmXo?^r&`W3Od> zm9PW{$S?hrl%fNqf(IOb0Qg@G=4P1b*QN{~d6 z#FzRH$tX+uKO}@mR`7pV29RWC4IpU}fY>O2q)7m>K>&i5h)q*|vdlS@b!izOSwR4@ z_gnzlAQ?Uh$$j;H>>ck%wUmO7BWV{t+ieohZ{pKVNp`u`mD|?!2i{$(!w>7v zhI@!-<0aF7J;q?XgspUe2iANoIdhd=bapMGxW zpBl%FZ0YyWY;~(q@-Nu5PL5{1oA|spzZp#zjd4887HYNpW@EnoyuB#Z*J(0aedRVA zACojqCiman#=Sv2&%eul-VKu^z5gb^B-_s|JoyW=3>Mf91t51d*Y;N09i>Y0qTiLNCZ)Fv+Qka5&y_GqKvI_khYI3GTS?;!^%Nh0u^EZzgyd{QnjhQt0M0P9 zAVGPDvn0b8uLmpLWJ9l9Ii>Ii1C!RXIJ);vjK;3(k}mJ9A%BL~tJn8_)mW(GsTiQT z;s{&WcX#M^0+p$~Wz0+1o@S^3PLUz#N)s%Pn}h|?m@ z=1o?R{DI{2uj9y@{tHl^hWwj&1`N4xKSBKmvneLZ0tMbQaL#d)_$XTe3zJkSJOP!H zktsn9f*b$klO!q-8_Dnz1ttLi08jz|01*HH g00000000000DytjlTRu>0VRku~JatH+&O5xgavv&wn2MFlwOdFQT1uB!$51fCY!?DyB$5NwC;zD^>?kj+o9Ds1Vmr?QsSRXM@xq5-5@zScp>Ua3_HlxUK8 z(f}fvP#QoagHV4~Fo0MFpk!u^mQb1uAT|m>X%c`m2td#h(P>Im7RtJ;456$b0O^_q zpdBXv4qvvG1y;hcX#Z%trJOQjR_hWhQ6urB{X;ULw0}qjv8-VKunb_ytQx@5WdE>H z085knv%T>FOOSu)GyzMHe@GdeU|H92f-NlIUy9|vnm>Pf*Zgs^`^GCf!`dZWqCw(G z^M_zchE z#PXYq?6oYhipZ{u+K<=Lfm`?$!d)(%5-!mn@uc=cGNIIdNCvsAp#886a7i-2rOEnX z9RQam=|_Kq^aCvs83bKfDC@E^JjD_-2u`szOFxfs`AuZmS{7O9BwMt8v~C4X;g_;@ z$#hG|M0v!M(htdmQu-koB(s9_!!kf7$pD!q=ZAFwWSW#84ayI+L}U;`CMZ9o3=gvG zulc}1wr1t$`;fV(&d-%DIzRafvR>$x{PvgjO3;5Ke~3p)KO_@M>4#(x%?i>F%K(}r z18AC@AJze&X;OYPC_m5=kwMUug|aRy!%dTTL9l7oto(c}nom+?YguHaO|$6yXuGAI zu5s2zBGSGMp;t*At7Y5g8F~MGQcJicYsZk`ol&6HcjS_2J;76B05cA z6U-k{h8t%U^GDZg{(K)c_tpQ=(*IGuSMqezF7bbC6iIyP|B#HbqyR)h04K{xI9W!* z$tDDF+9m;rjUwSRbE$geYVcJCpDBCbCSL>3~tX(1j*{GBFQUW3wWl0N& zgpkS#77)t-l_UdHnj|3B0Z?g@fVO`(K2Qk~5W8%ku|rvxlp&T?B%s0q_8rS7dF-{U zuM(Ev0Qse#l2UYlRPcZ$`ScO#0FjKcqy$7lh-C!{h-Cmvrq{qZ)?@*(QGiR61!RK- z1T7Jpru?Lfb13W5GC0Rp`0GFRo?AfaFh^27N&l&3ZIzC%Mg2!xtuW9fugQPfP6?7I zlK4{pAsJ;!|A&MS$qN1t%K(zhtN|oV0uUPokTeNEHV8n_60vE@PnJ1{vMwzHBr6C& z_MQts8zjRgA-S*KkGDf>Y`kPT80CM+zBgWfl3jl~n%*=Pf380x&pSN*U0Zo){o&Q0v-r3ff1YLc zP`=n^WAJ&n{&c*?Y(7fnqx8#q^!(%bmgC;ZBpLj;n$&VS`g0lo_(T5(@9>Ad_tVb} z{Zr$(kuCi`nyqd%O8y0#*2&SVcN3rY<~O71qA`w#*+Q+B-)zj+pSLZy`Z`T!tFPQ< z<71Mh$>jdK+qgG~=lOTp&%0rgr1#(Cmt_07g(rUjP)h>@lQ1b7vndw34h3s6;c6t4 z=p-M1YQr!PgztR{qq{BH#-#|`fwtt-K%pruy$NZPh+02LY8}6Q6+3Yf$f?KG%zit& zs9x+tl9U|O)wXlT#?9xd9tnp?_^gI8xPxFEqN z5u`okwn4iPEXCOK9Vq6r=+IHz7^hf>rQ2eE%lJ3j0xk%-#|i=yEHHckrgkQxBU11g zX@6Wd0tCm3O{) zL~TKeDkW8#m+OL1l2cNB9Y$XDpMmN)-V9I^{3EDriDkjPT1sF=<+H#ZmC|dyx zlT|4^0+tt(u@@GTsVOQAgB$<=cx*3oba`xLbCV$|6qEER8Ugi_4=N@bYck<#Bmn>b nPyzq|5dZ)H0000000000fPvPNP%1tFDU**XBL>?i000009$S3x diff --git a/summary_results.json b/summary_results.json index a07fc5d52..04cb961e3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.3946, + "Performance_Result": 20.4194, "Performance_Units": "Samples/s" }, { From a2a0bf25ce6feb39816e4f5686b5ad2ce915583f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 14 May 2025 15:42:32 +0000 Subject: [PATCH 552/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10936 -> 10936 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e4e8f49c7..d9b170df1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-14 15:37:33,764 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-14 15:37:33,764 submission_checker1.py:3308 INFO] --- -[2025-05-14 15:37:33,764 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-14 15:42:27,053 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-14 15:42:27,053 submission_checker1.py:3308 INFO] --- +[2025-05-14 15:42:27,053 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-14 15:37:33,764 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-14 15:42:27,053 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-14 15:37:33,764 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-14 15:42:27,053 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-14 15:37:33,764 submission_checker1.py:3324 INFO] --- -[2025-05-14 15:37:33,764 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-14 15:37:33,764 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-14 15:37:33,764 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-14 15:37:33,764 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-14 15:37:33,764 submission_checker1.py:3345 INFO] --- -[2025-05-14 15:37:33,764 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-14 15:37:34,808 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-14 15:42:27,053 submission_checker1.py:3324 INFO] --- +[2025-05-14 15:42:27,053 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-14 15:42:27,054 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-14 15:42:27,054 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-14 15:42:27,054 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-14 15:42:27,054 submission_checker1.py:3345 INFO] --- +[2025-05-14 15:42:27,054 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-14 15:42:28,157 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72287 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72942 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.4194 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 8c8b8bca0..0eec1ae5f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1179 +1180 diff --git a/summary.csv b/summary.csv index 18ceafb2e..1f2a9ce34 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72287,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72942,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.4194,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 84042f9b1..953057899 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72287, + "Result":1.72942, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -229,7 +229,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index ee1b6ccf10cf379234c2d2ecedd235a6081ee8aa..f8208e93d2cb91b22847bc56e1316fe284102d78 100644 GIT binary patch delta 1689 zcmV;K24?xVRk&5KatH*-Hv<{7b_ihze|6~S5-3#(lxUK8(f}fvP#QoagHTp5fLI2g zWM++)P?`)NHVQy#5`Z)aK+qD=X-ZWV%DSu!p{yVP>6!(g9VY({U$&M7R>HDq|7g3V zoHAop>k=$cBk`pDLo%VXe@F(gtYH7J3}DHu8o<(I|FBU2OOyPwz3~A{kbmeje*sI7 ze@GdeU|H92f-NlIUy9|vnm>Bi{Bg4T#w$C++9h0~LE=gChh#!&{*VlES;72a8Q_v+ ze2Jw={9zpcmL~5#{OD#S$|JPO&vhKaX+wO=Q_x7Fp>eTeN<(ZUs)^m$G)rbW6xYdBl^_56Og5`XL!4 zvx4-)GC(HD0GTG|hjjpCnv@?6$`7$x{PvgjO3);Kh(}64Boj*Mhhz}V3epeD0GcENXqucK)&ZbtQhqcjKhP49 zLC}y$a189;AplMQme^>{ArpfrxVEjN!#4a163C0g8!(Eefjk{*e#?SXbb5E6@ zD_vB6@?FypOP^|%I6oTg5nnn#B%>@T{g4o{SwZ??8DNtsJMh}9N%~=<0GuZ0M}zYN zEfJljRAr&8%gS)yBxVron>9;6Uk~R~j5s2zBGSG zMp;t*At7Y5g8IWUz$O!SfK8M7!$tu%P3DgV^9NcYI!#~`%pX#Q8)p^sN7rord>=OV z)&J4b|53hI@^sTK@qcU-Nqp)5kc_gV07OCnC(B4USw_OiCIoQWCIN_zBH=W3(6K=P zf|iI)6Fhb(>(Vm0e{qr!Y@GH!`9FV`$X?4Lt8AQhQ3J|1PP^nqTD!yovQa1Tr3FMX z%90uo2_c#lG$57%G)YE2-?U8@5bFTQG+974SU}JckwHLXhpH~ClDj6mLtxjm_f7)( zwjO&e>#MS1+C>K_+b}Ix>yp!~T_OS5sFV0o0wNh@NehUCe~`)w77)t-l_UdHnj|3B z0Z?g@fVMY2Pze$cyKJDbLs^%UA(mAnpuz(79m^+q?6s_~5|-cq`K6zdQgnb+@PH-x z^bzR*k&LpW1VloJWd#X{WdKX2*T6Z}WC5{JfJ>7FWP=3+EfJfh{G^L>DC^QPILB7_ z>p%9MTR`bBe@9Y1N&l&3ZIzC%Mg2!xtuW9fugTg@36dz1_)`BN8D&ZThlCKx3jPnv z0FunC0VGWV5E})MGzma92td#hv1!UrmN|#AE-eEjD+oaLo(n)5B*P~mxv$=jz2p6; zmQwI>B<;dyyG`QxO?(=U$BRZk$u8Hra@)H8z`ILzfB0ej*>Df>Y`kPT80CM+zBgWf zl3hBQ-ZU0}u0JEsJ3ReeTX|>w;pK81H_|V&__!H=o@Mt?zSw4C@OilYbiBrFK1$}J z^vinm{NwqS4t!_0+{so)X z$(cxHpLB`FGjR zyJ3=~_uu4~Wc#^=Cw~D@O9KRxFew_dDHgg81nyY7JCo-mD}N+y09i>Y0qTiLNEA_V zv+Qka5&y_GqK~k3lhYI3WTS%mGl*)S+qPAPn6 zVA6UPNB6;r(b%?a(#{5J$e-cO>h*nCH5RHr6$4b4oDpV$dTr|xUl-wU4RwAMPP`Ss zpSqw=yqy*~_5qJV0CJ3bRz7FXFY6_$^E5Bww1~6$IvOj;JSX|XS3mNm{{ob!A^#?x j0mt08pP>E&vnnRa0tCo60~xdUC_MoL?pV7!lQ}AX$wMOa delta 1689 zcmV;K24?xVRk&5KatH+8G;q$db_ihzf3*%u_DZcvphT0zlLipUgwg;a8HBQe0mL!@ zB{OTZgwkXHu~7g@lK`Ya0D_i?PE)F~P}XH-2xSEUNY^X??J)Uw__DPuuo9L<`$yX? z<&+t-T9;sn8i^4AE=|@C>j1bkNk1B-A83ilAn3|M zS(laJDVCT)aEh&2`gx4YZz9XqvdBs&*`oEMbt`ZRzm&C0rdvWL$|IhXen=*i(htcX znH8iTmH{$J2FNryKdb{F)1>@pP=25#B7+bzLHQwNc#vg(%?A#$H7h^ge}~LHb$+gN z(fP?=ko7{hE0KO}=_R*-&J2GArKK-1*>unquClk%fM`GJ;* z41%UClyzAdZkogkf=#n#<>zbBe3B|#%OWdnnnmYF+b!*cWiS7fC_ft25nn1lB%>_p z{E!f$S;6^X89B6isjO)!2)8Sa{-Yuq(!Hh#VjntQ7J zT4$`n%?i>F%K)29*@4$)P0|k=1>iI}KN_4L zXo=`Fr78<$T~>zsCNYCx->g~s`Fc2?a-`i;_2*I-)t_wNw6tILfAVaJ`J+)G@um4g zGRl(b4+$Zg71SS=0XCVq18kbqA2tfGX)=E_m_N`G(P;vkVE&LY+&HV4Ke}e~=lig^ zul|pg{*Us#lBb(?iT`7xNa9QXhh&r`1t1avI9W!*$ubg7HX(r1HVHs%6bYxPgN_XX z5VS;Wn&7cRS(lc{e~pubVB@s+$^ZGgMD|)1S!Lt2iyBbAaoQy>(%K~!kc~QtFD)RF zQI^zzNC?rapaHQAph+_F`KE2MfLI4WrpW@b!2*JohztT6J5+U9mE1Mi9Rj}AO zxAoX-Sznb6(=Iwd*@kJkT9=$=?Gg#dMxDf$5)jEKOIkoAe}q(4uz*+ws3aMn(j)<~ z4uDFN1hl>Jfl82o*kuEa9m=|-46&>t0TmXo?^r&`W3Od>m9PW{$S?hrl%fNqf(IsxsN%~JMYpZm8E$TnoYK4I=c}>=KN{~d6#FzRH$tX+uKO}@mR`7pV z29RWC4IpU}fY>O2q)7m>K>&i5h)q*|vdlS@b!izOSwR4@_gnzlAQ?Uh$$j;H>>ck% zwUmO7BWV{t+ieohZ{pKVNp`u`mD|?!2i{$(f5Q*!&xU)5XX7Q)!6^Sj_Pz1? zlkC#b^ro@+bNv~4-r?!*+R8iY4=7J;q?XgspUe2iANoIdhd=bapMGxWpBl%FZ0YyWY;~(q@-Nu5 zPL5{1LYw%!H@_K87maZ|%ob|3{AOdm{=B`_*J(0aedRVAACojqCiman#=Sv2&%eul z-VKu^z5gb^B-_s|JoyVyO9KRxFew_dDHgg81j+Cc1(W9_D}SVE09i>Y0qTiLNCZ)F zv+Qka5&y_GqKvI_khYI3GTSIi z1C!RXIJ);vjK;3(k}mJ9A%BL~tJn8_)mW(GsTiQT;s z{&WcX#M^0+p$~Wz0+1o@S^3PLUz#PVXX&hn(<091O&V5^{DI{2uj9y@{tHl^hWwj& j1`N4xKSBKmvnnRa0tDVPaL%*%C_MoL$?y^dlQ}AX9&#`~ diff --git a/summary_results.json b/summary_results.json index 04cb961e3..0b70741b8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -631,7 +631,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72287, + "Performance_Result": 1.72942, "Performance_Units": "Samples/s" }, { From a31db814c3043f56eda76c6038e47efca6bfdb82 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 15 May 2025 14:02:38 +0000 Subject: [PATCH 553/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10936 -> 10933 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d9b170df1..aaee0a97b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-14 15:42:27,053 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-14 15:42:27,053 submission_checker1.py:3308 INFO] --- -[2025-05-14 15:42:27,053 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-15 14:02:32,684 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-15 14:02:32,684 submission_checker1.py:3308 INFO] --- +[2025-05-15 14:02:32,684 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-14 15:42:27,053 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-15 14:02:32,684 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-14 15:42:27,053 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-15 14:02:32,684 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-14 15:42:27,053 submission_checker1.py:3324 INFO] --- -[2025-05-14 15:42:27,053 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-14 15:42:27,054 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-14 15:42:27,054 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-14 15:42:27,054 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-14 15:42:27,054 submission_checker1.py:3345 INFO] --- -[2025-05-14 15:42:27,054 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-14 15:42:28,157 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-15 14:02:32,684 submission_checker1.py:3324 INFO] --- +[2025-05-15 14:02:32,684 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-15 14:02:32,684 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-15 14:02:32,684 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-15 14:02:32,684 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-15 14:02:32,685 submission_checker1.py:3345 INFO] --- +[2025-05-15 14:02:32,685 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-15 14:02:33,689 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1107 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.033 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9554 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0eec1ae5f..69c992d12 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1180 +1181 diff --git a/summary.csv b/summary.csv index 1f2a9ce34..6d46a724e 100644 --- a/summary.csv +++ b/summary.csv @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1107,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.033,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9554,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 953057899..a0df607b4 100644 --- a/summary.json +++ b/summary.json @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.033, + "Result":22.9554, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -799,7 +799,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index f8208e93d2cb91b22847bc56e1316fe284102d78..a44a3a491969c9fd31974d9234600d1697df9f9b 100644 GIT binary patch delta 999 zcmVP-(J&SO-9*$pYHm z_OUvLOTj9_D*n6%4 zrNbOa^(6nNmbFznzZU%;ZMDKcm%J!zJAWleqDkUQ|A%ChB?TZ7LL@5)Kr91DGPMSf zG#Nl_6hP8s0NG#wK}*D@DL-1~9Ll=143Ml~0NHzP0Bw*ApM>PTia+*__@i1%!RL{* z3m@(_iRU-*X*?b;8vP`@T-rP#F4f_O^@qbf#Ix~|>0p%qCHvlZ{ZV%5Xn%Us zSp2#EkUa13^mlFLo%M&8%W>RDzs%y}X8d`U-9!0en~lNe;riq88ngK*nUB&h>(TSi z=Ua|@CzE9G<7!gN>FCd8{NoS(AH2gK`rc1JH}p@9<3_gh`)Ibh)hPKFY+5Hrv))a7 z-kaZyri;cn9%c)*T7I)JUw`1<>MQFsnXSHZn~jf2nkJL`?{4GXAfD&nWk2tRNs``w zlV6hU2N$0F1yD-^1d}i+8nY=Dx()^YR8PPpljtNKf6HpaFc3xe{R*SKEkBx4gzZ3E zvTC4EmzL~=G)Y9Q2a;MRe_zF~B#>2?qkHGfok8XDAlv8{tP@&QC`(clfnu5qwX4ua zvx*;4(e>RkT(;eVEO&_oXPdW7zpMbHMiD*P}HY3fCZF>M`7+R2^yu(?N;j`C+mF{Lk zuUt5#@J9nTt&=#r4^GU+uIrMn=&d1thQHRY@BON=P^VKdKy}F(VHT*@wl48y5f0Z< z=hxxHTLJv(81x%&<03;J@F)ZzL)=OEES_JQBo(UjG+)H&BF+{~c2Clr6vgH=@~Zy~ zl;e@H0Sc2R zA(|C5AeI3%Nk%^3v`rQe>j20!SwJ>eK+qDAK|o`NsxGUNyC%CsVAr(wP6GP29(yh8 ztFmF*MF%L`FfCW>lGChRA_3W`llW2sA{k{#3y6e}$_f?`%K()m15}zMAl3m;X_A1p zH-A1*2@(*yY@o42S(lU{mQ^I6!UFak%O`p4wXClamf!&SrJs^gbbwUwfF=3#5$OPt zjIyKzL_&yV1qp~{086IVz&X}r0kKhlOOpj;g9QXF5u2v`q>FPX>(Vkf$5#03KlYwm zKh+QvV?tWl8^sgb>LJ{twFllFY0D zBuxSk8wHRw2|zXoK+qDgY06KQIft??EdwMg2tf9p3qTtr!zUrRuilTnW6(i$*`mF4wwp+q(Y1yGwQWVg1=~5AkfgWI7n-f5^T!UVoBZI)9qp zG!}oZKO@gOJpEl;d1w9M<#HT1(l4|4xEX(*W%p3N*k)t!dAR;`yvA%kO6H^V%X;+u zt51d*UQ*09i>Y0qTiLNEA_V zv+Qka5&y_GqK~k3lhYI3WTS%mGl*)S+qPAPn6 zVA6UPNB6;r(b%?a(#{5J$e-cO>h*nCH5RHr6$4b4oDpV$dTr|xUl-wU4RwAMPP`Ss zpSqw=yqy*~_5qJV0CJ3bRz7FXFDC0Hs`E52;E&vnM9X0tv`B0~v7*008@w`Y2Zc3zJeQJOP!Hk10U`f|JoHIs)__lMf#i zlVm9n8}3-UJ0<}D08jz|01*HH00000000000DytjlT9i<0V

DkBEmCjbBd02*(? A!vFvP diff --git a/summary_results.json b/summary_results.json index 0b70741b8..cc619f0d2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -991,7 +991,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline", "Software": "onnxruntime v1.22.0", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.033, + "Performance_Result": 22.9554, "Performance_Units": "Samples/s" }, { From e1ef3e86519170529449dbdd5460293f82882ec8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 15 May 2025 14:11:26 +0000 Subject: [PATCH 554/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10933 -> 10934 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index aaee0a97b..47e468217 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-15 14:02:32,684 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-15 14:02:32,684 submission_checker1.py:3308 INFO] --- -[2025-05-15 14:02:32,684 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-15 14:11:21,097 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-15 14:11:21,097 submission_checker1.py:3308 INFO] --- +[2025-05-15 14:11:21,097 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-15 14:02:32,684 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-15 14:11:21,097 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-15 14:02:32,684 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-15 14:11:21,097 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-15 14:02:32,684 submission_checker1.py:3324 INFO] --- -[2025-05-15 14:02:32,684 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-15 14:02:32,684 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-15 14:02:32,684 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-15 14:02:32,684 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-15 14:02:32,685 submission_checker1.py:3345 INFO] --- -[2025-05-15 14:02:32,685 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-15 14:02:33,689 module.py:5266 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-15 14:11:21,097 submission_checker1.py:3324 INFO] --- +[2025-05-15 14:11:21,097 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-15 14:11:21,097 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-15 14:11:21,097 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-15 14:11:21,097 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-15 14:11:21,097 submission_checker1.py:3345 INFO] --- +[2025-05-15 14:11:21,097 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-15 14:11:22,084 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,13 +28,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72942 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72618 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.4194 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421307 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.410375 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 69c992d12..7fe7c5186 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1181 +1182 diff --git a/summary.csv b/summary.csv index 6d46a724e..443358f77 100644 --- a/summary.csv +++ b/summary.csv @@ -6,13 +6,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72942,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72618,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.4194,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421307,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.410375,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a0df607b4..857922279 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72942, + "Result":1.72618, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421307, + "Result":0.410375, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -409,7 +409,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index a44a3a491969c9fd31974d9234600d1697df9f9b..208736aa98276a2ee27a8a20834c29310670162b 100644 GIT binary patch delta 2863 zcmXYzdpy(a8^^cCB65sE8M8Sbo|Y(2We%~8Ig=?*(#T(duQsx z;d>g?U8GUBD(xsEVZBXF-olehi&hPpRUX)7oBBqhk<70}^vR$el61~VtHH~+#XeX? z-IDjqu|=h$kfhF3PN&VPXAmuCh*xtiO!`ZaOKF6*8h|uQJeI{RH`b%~AA9HDozs8z z#r1^3wT<2neq*PQhY%q#eU*uG_>S)cS0 zyS+EW=XL^2*&pX*rZh4yh-@OBrPeKa81yuE)^&4omAoca{t5IC{cS6EvgO`XmsQu< zD?s&H`Of&&l7lUwO?ypTgdgWoIUC(uuFd63B{&dccMYQ7{T70X`f_q+r_N!=`Hk zqIcgrswruobj!%@xmQRk7Yp~jpbEB98rGRw)B~aHChC(es(^Z}?DWmRv_};#JC`e) zZ7>OIQ8@S@+i#&>y0mxf09JrntXG#KGTJM!6A8JgnLC0KftNPi@KKcSWHYIzd-dtS z*r_`E`CG>=n%I~!r}_NjLRjJ58_{m_nguRTj-!$&UiC5F@L`m`6ek!H>n+<;jbCwf zdww)2H#GyP^yaeeEFUv$mbMHV4eYb%KgFpniLCs`y?@3{=_I1)^wl!=#6Xus?GR+K zM;uO@`B1hO&|oQ15V=s(f=jS43hOpv1XdbMk7s&ByeUN^D`+B^r>~cIt9F6q0_*VT z@vLmUVze65#80)0_TKo)>qS1`rz7N|L9GEwHEb$GZ43iJU&S)B7DV zEYft0H0Y95;v$m26o(y9Xb9$xYg(jmjExwt zyp=g!D3i!O<$;y+Q3T(MEqffRuQL9&GP8VsufD3inCs(pYFg}mIy^gMeZ-(myq3*OFl z)a^sit&9!wEe&C3=(kN8Ow%mcunzvzR?t#B7*kT|D;dyKxccw2s>h;y8^0xa$6wBQ znv3?57hQ4vvFu|JKF>MNc7K81&?+8^2eplW+VYf|5V-)<7!PkTC4u!O_^7R*?0@SS zNCo(M-K_EUhd+x$NySyQshxX&qMYrU;(>2i|Q#yJ%%$T%Wn^`!;Gl*rA3Pw(_WVkB4` z7hym^OR$28gjpZ!PD$O#Rs21QQ98RRSh)zyqDe9L>85-6W{`({Z0%2NNg*@QRR{%ZGXor$1_cwa^N+&AcuUG28rXACIA~wFfO*f52|{9_ zuv0oQt`W3v_I)fjC(8Nn5(i%ofD-U|S17lS$z5N*@mUhh9wsS_r($Pc2)*5 z&14Hfcg@{H!%0k#RU~zIxFe!uh|?$v(%_2vvs%W&OyYmrlK5*@e6-D-6cRH8?*N^j zGm8qMP!jo0hQK%kD;lwWYa}`{udutvuB0lPKN^qGC04LL-z95~8CD593(D z5_40D?(sui;L{tdvY%|QxsHhGKU^A_uT@(-^>c^=G!tgtmjA4H;I;1)eQyA@MhtI? zkNy1f*T_I@ty9V3(9{43zFG*OYr{Z%;aJidYSJ3u$e2#s^Zv9U<>f-@O41rJ{kuzd z)88|)$4)K3dM6KfI4HT2v_!_SEu*O7@y?p%=(eCW#$3b$gU1hMo!eSHQV>Ff%r`lp zzuTcY$#pw#E@twJy{M|Ir_-rs z@RF3-tXHA0KUR=W5Zc-H{gx$RM;D45>Cdz75>< z5r45l^dGN^sk8OLMf>}%n=fAri$BCNH&&gD&NmtgS?}wP-AahLUnd|pZ*Zw*C$xiE zjtlLr88J!G#X#Oxo0>NM`cGm!z8P^QdS!3pF6Chv@%uTm*U*d^lseEC^71#i`or@l3@6jf6*>hc{-YRHb22kdgT9ZZK3ROHU$n|Yha|9&CR|e!cejT< z$I;2kW6p>0C|# delta 2862 zcmXYzdpy&9AI3Lxwlarw6NY9+=#E()rA=gJEoa#zhc=Cnx z{GOk@5*MK3g0Kkai6xenyyBZC0E(#*#w8M|@PypZuE^R2wkF>8T|lz2d}`hq1Wb|d zmIk_o&?6(1TJ`MN`r6#eqmZ=X(?@nhpLwuwT_Vf;BLJ&b8o%PM7D2cjGr6c{o)Pm) zxmRJRxX(#VW^A}DsWbk^qt4O@<4TF^+5X^{`4a=N{*?p%&ScG(9?$cghaPtlkg&`l zHzs%|**@Wo8PksGvQ)x4IS4y*dsW9Dy(0WL{N$AqMb9d=|3zm(Ik9d0T*i5FA4P2KVMxKX6Eg!9CnNJF&a;oW{rpulOT-*dxy&SW=Kyb%HqkIcf)a_VQ(6-bAw!%xt=}c`H$yPCUK3$RC#!1eQpttc&4Vyw)m(mAI zm*KdYf9K_X3N;U}^%Vf2oF^#x35kR3kN~7Cf_KLTLy+!AsQary1|m&<1#yvAl2+pAT#=TPgkr7TB~_dXpsDS8a}YVB@%XKkMc%@V z$@;6-!6)as3TRn!5y7wLsm`WCn>f+H@6%Au%(V z^6%7Z?Lt)l49~MAXLA|_gfw_R=3MWUdM0VJc<+4P)g#$usCD|oi3dVM8*`M6iiMI> z*%5?_V_y;OdYF=lb?bG`$omm4Ov6_8RB4riN8~W~rj$a5fNz|i3n+&KcxpBi^J)e2S0#`!V zEU*S;A=mKnn-CMFJyNj+;I?PQ&}xLwUe3lO2*?VU!OFE;N5oaQKe<(+B}wCz2rM}o z5z0;!oLHekG8IJR-1g$9H8ZJf(fosx15|Le%lP@^J66+lMqIvrKo9$c?WLVG9||(F z+L(HaaC(|+l0Wf!j{R&X-0;rv1?0oTkk|wdehgY@n?Yv2%x&Yr)=c}eF}1JgOqT&NC49v7q>hnB?&mAir*J=EHsl|u{4bwD#* zseCQu)=pmmN6PahpT0$ZUXo*W#&cHL*k$ix@usa=K`r1QU0v?CRR#5HE!Sx1rfyZa zfQT$n&EJb)A_JI4Z{H5;K#m`|FFe7-c*Hk#qMnPr+9$fe-)Px)sA)j|RH4lm!t%DK zzZlSqyesbSP^(^;qc=HnP)?A%& zb?TdnTLYPW-SOT(cfIs6%~dw-f-e@Yiq*Z3b&!5qp1)ZIK*I@U4JEDX;)&VF8GPdR ze<%8oQL}ur!>^RhWlvOF5V4sT)=1Qk6gBb;!n-!MHcaZoXf{8x^%C)o-IZomx0p`X zbzOBtO~UskGaRxNbQd|56ak`g+3+20V$!Lwqev9C7^=vht6Xca5d=<5Z7`nRf7h7> zm|9`i6!@p7Iil6 z5IPgOJ;_TZuC(5t~i-6w0+I8aDZeTjF`(+m_BBHh(x}-P`Bm9=qhzqe#3L z%(293%q#g?VG%{fhMNo+pKP=4FRls0L*E9<3G4nvpd92BK5M$Ai*^^pHyXFyj{5c9tJ5)Lm>|;+$k)jqkeIbei z&DOWjQ9f$4kB&2j0Pakva}YnyO9ofcDlstp6Q}Qmo8!m776;n(rk7`RoV-GRl<_?T zzmH$<=#?Njo*jGK(M7_w5yBZC!xu~1nYA(1U~;;gUB)^bu<>1Ahzyw}-E0pwC225r zXu!n9$XzNjeJ5VM?ISwgqn%s*(G}p5CQZT$V(UH!sPnLTp|%MRt$T^f{tI>PkiJ79 z;R|tV8yP9FO2j@{u}TNCI%x)49T%dC>!yM}`hjeS&83D|%A{u7vsx&l+z<3IIt&!b zfZn@4UZ09x&G=bFacrErU8?~9ibIQAUEzGPT?^o) zH%^VNk1cA{tfB5^cJH6-iktAin&9ihPS}Irozlnb z1>ZO5f@dF4Yr8`z2Kbs`Qp_;#b6V-aTP>E_feLk$*qQ^rz{v1Fv*^2z>M!!rGYMQI zYhxj7Y*;66>N5?yaFf6s23Wq#d70gLBB>gAwf>z_zucV~5UeSLHqaj2^gHa>k4^ca zzfG`Q*X#2*Z+>vTb2ex0I+|ejNQOx<0DNCnA{ZXoMbT3oxi%9^I)l@@%T*{ID59%s zd;nLW?x7H0lL~hkKkShspxE5V7!mU8_ubV!?tQstAZ|tClq#-GDbT3)$1p5SYcR3c zRBq?4=0^hNy7Pb*fMH+j(Hc2Sf>%j8{bxn>UR?Fv6uR9-%-*+|q0ykTzflIVr zqcwerSziNfD^FRLe8WQyK4MuuS3J?tQZdV{n0}-S+5m2=@%b;j C&_ZAU diff --git a/summary_results.json b/summary_results.json index cc619f0d2..f7de6bfa9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -127,7 +127,7 @@ "a#": "", "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421307, + "Performance_Result": 0.410375, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72942, + "Performance_Result": 1.72618, "Performance_Units": "Samples/s" }, { From 0b3edbcba9e64d1d3dc2d4eed600d39bec31251a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 15 May 2025 14:22:50 +0000 Subject: [PATCH 555/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10934 -> 10935 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 47e468217..7d34337c9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-15 14:11:21,097 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-15 14:11:21,097 submission_checker1.py:3308 INFO] --- -[2025-05-15 14:11:21,097 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-15 14:22:45,512 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-15 14:22:45,512 submission_checker1.py:3308 INFO] --- +[2025-05-15 14:22:45,512 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-15 14:11:21,097 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-15 14:22:45,512 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-15 14:11:21,097 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-15 14:22:45,512 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-15 14:11:21,097 submission_checker1.py:3324 INFO] --- -[2025-05-15 14:11:21,097 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-15 14:11:21,097 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-15 14:11:21,097 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-15 14:11:21,097 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-15 14:11:21,097 submission_checker1.py:3345 INFO] --- -[2025-05-15 14:11:21,097 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-15 14:11:22,084 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-15 14:22:45,512 submission_checker1.py:3324 INFO] --- +[2025-05-15 14:22:45,512 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-15 14:22:45,512 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-15 14:22:45,512 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-15 14:22:45,512 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-15 14:22:45,512 submission_checker1.py:3345 INFO] --- +[2025-05-15 14:22:45,512 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-15 14:22:46,505 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -37,8 +37,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.410375 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1481 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0731 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7fe7c5186..10f1cd98c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1182 +1183 diff --git a/summary.csv b/summary.csv index 443358f77..3ebde87db 100644 --- a/summary.csv +++ b/summary.csv @@ -15,8 +15,8 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.410375,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1481,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0731,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 857922279..bba3408ff 100644 --- a/summary.json +++ b/summary.json @@ -499,7 +499,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1481, + "Result":23.0731, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -529,7 +529,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 208736aa98276a2ee27a8a20834c29310670162b..dd0b784beccbab18955037bec233d0b0ab3d52f4 100644 GIT binary patch delta 3283 zcmYk9c{tQ>+s0=QB@7aetZ59&784~|vJNIAC6a_0&1lFP3f~D&Mv~0eQe#Q>tz>D) zQptodM3kMf|A?V%jTp~(pZ7hE_x|(x<2pa*b)WZj95?N9?d8{j`}c*yye0#{(EYAJ zZ!2?WG44l)=49aMt(jHqXkEfwf`iUEztXgCM0#J8&cD#NA7?CoZ!+9ZO|%%!5V<>r zJ))rho}MtPrLuc=E2VSSd9*cPxZU8X=HrZ7g27p0t<=s)Sg&gh#l^5oUK+8A(&~!J z{9Yqf9X={k{tz)mqp4S%n@=q43W3D|tH&eZEfd8G@2$6(C`0r>R_9cJR#)NsI=L>% zE-EZRr7xC4>qqx2_cnt)(p4 zF3>)a?GyW{Jl$A#0rPXQYHhXu`|sHZza@0bOr+T{SF5OX)`i8q4Y%=MhyZ2VamItq zstL0fFS5FBo>l8oDew^KXUPJZCnxGWsqo1q%kgCLdzo;1rDnfr-uT#&_RoR7@2<%h zU_9SG6eir3iS3oG z&QEif8Xr6d$zkL6DD!KtrQk}WomBJ8v36T=F~cS$vF5&8;$js!`fw#cGcwMcz8d;K zOyiZGxzgl9Hb}0kcM})8vZ)bIF1u&SEpaTBY#Kr^A<8NQiWoA0in zBL&k{$Zq89eyT|S0%Wj;lFFufl+MD&9`%mQv^g0+t5KO+d=D;`gF6Jt?zn-xdF||* zDzrZaiBj|Hp6mdftJ87MKdontR1MD?KEBdcekY3R^W3T^V=KI|UF2lK#w;@E^%A?@ zI74pjCNlOeY$ZM3_COGdO+_(Mq47qO{--zb~JA9R=2>-1$NK27(og(^_ z?WT7A^f{;X#t2&@iyL~!e7kxk)F@EpI?$yawwj3F0q&9XLc+`7)@U$lOk zvyu3sF=ocb&*J8%yT1lNSs}T7-+N`T%b<~vHdG6|E#!!_xbsl#MzT3P0j}giCs)yJ zS1;p9$zPeZ9=C$oxXQjz&dnq5{FUvf&$k;7ieT3FPu|W;_{g)t z+Yz^;Ef6T$R=ucv#+^P8T6!4kI(fy*g(-N#pY5kxdFp8tGvM0gz#z=823EJkI4;!f z0K%KcDjLP==Y0ymd2djJNb%C6$7+uXBIcpZ$h97a-{8M>@$vttzT*`9-T8T32jn4@-fzJ5lvJl{`*!%F_#wC|RP%AC3@j_%}pWqwGP zj9@^ha9+IUgPamHo+2-bHrsOp!*`^SAM@Zlsx;LsV@68LZ;ep!3%d|%NO@&PR(}_r zwyfqhvflBzdON1~rl2tp9LeAk+gHU)V?itlO@Zn$iW=GreLcZ767k60e~1boUPKM3 zy(h7Y-F@~E);VU9i!!>isZ4!pibV1Ez@<$N>GGl5V!8glaQzJ#{UH?+x&BCn>OUzu zqk5=;I$wibDDtPXSAlQe(3l>t)nD9*cYH!FM$_MnD;{m~|HPk}@Dl40de7W>|0^DG z!;xkloC08~)R^_g^ceGiYq3z=2lQ-^TbhHoiF88z(~2{6#F`!RW^`! z=aanLRi|f|qC1`PKkuWzo;NJncw(nICQmWd15ON{Q|Bd$r|ugb#0KXbE3JlWyH;{8(1vS>H3Qz5jAx|D4N zv`b`n5173zYVA%Pz8obx5V}oqXJI@Jq!v@cE^ZxthPi4CkGbTu66MTajJ+<5AzTFT zFp~ll$Kl4V2^aFdJD$Xigqb>OfIRc=j7fp>mi@;-d%2twhgqHk0n;Jx5{<_NmRWRA zQj`(f%cCUyfr5jlsUNF^ODh9sfrV5(7ZaicTSlo+#NY^A5g$K@oLLslGIL2!V|b|_ z9P*?{)q5YrvrnEa(A2*si0ErzEGT9?zj1l6*WDhcQC1ch{mp{c`c1^rVOHDn`4XEP zbC3@5?}!RHvhJ;#*j_B8(g9)vLh?-^(P`uhj-^!?#eN)aKW-n?dO#hr zq|FZfQ}ssuwRoDvU#k%fX00u&4Ll*H426LXVv0N)qhOHOAaKjb|H(@iHroixDp?IG zNce0YW!8UijPjk+d;I`)T46G|7siO?&Q99AI(n@|hqU*g5OZ-M5Ii+SLAV1}V~!O? z;$D<9JN*0L*F}lcQzwkw6K26nLsunT2z}}X#>0LD74?Q4$J}lFC98;%?I(|_@>k8y zlfC!N#ep5-r=YHsLoxuXi<+;XC-{L77==7-+#~*lU~^frgB#{ev?qV_JD2LU)pdWP z8S93kh3*2)Qe$=-Vpyx;oU11Xf5|$6z}f4%Wp=*Zd*6B-j)X?AAQ_1DxI`WzwNpWO zmWvro{eXCCoG+yR`S@(otD$YS5s_84n^&-#7f~_21QfbW?hdL_{`Br1mByi<!be28GXz9e z249piOQ&g!Y|%lzrKW1qyO-_PnOmicPwzPJblkCHeye}fA%3vk@;N=7n?6-r9}{@& zC8r(|H_LG8oL(%nQiNm!?yHGhxo`8DG>GF(lH=}bp~`Yk=wZp%%g#n9Yy~bd^)=st zcC8(1N-kf1w17HeAj$dw8JhwqYbc7JS3luwqR#8A^PXR%3L#8R~C`%vH8_<(%bi|RQwggLKhUP$h;7C2EwZS&zkH^l8KkOH=5o_?TnO#cc?MBY= zhG1k9L7j6AKxK!9Z8gw-f8J_g&YbqkclZ7kpvy62X1|E2>}vh7>c6udmN*oV(rGoH z4y^yf-dxpaYXVU^e*#XG%mlB^>2Q9fn=(z4MeD#bF1gaeis875X6PCT3vJsF?JSh7omZ{1Mi12_v;0GK{4+M*VlJA!#l8YAJD z*d-I!DxVxodfH+vGQj+$0Oe{M;Qk4f$=A3L7`&YMR=3z$4lA=09df=*9EWd`jHE{_*mO9w`udijm1HEcnHeB zSdmqe`Zcas4;aL3pk+RUQ#C93?9Z%ADBQ1ZgV5n1G6<*FXqLU#9#h@Zh~(#d1q&C8o{sq{-!GX%ETb0ow5u zUonw^i*x!D^s~_I64(}fBs|$sVMUcsv8YaJCTL^(qH={pY7txMf>Z#lxcUquNXOkT z>DcmCg-N*l@7t66{5v9d50n~Lt0Fph{`a4;sp<+n2s)}6LQ}yfYNo=`vJglZQ7b4o zoQMkoJ=7$?-)cvA*5pB$!eJ1nCMvTxYGo({g5iNc#JG3xe}{Dc8+hTk5mXIKJbsLS IL}jo3AI8l@y#N3J delta 3329 zcmV+c4gT`CRkl^IatH-F$fU1ulXnPPf0EeC{p^R}@-A@M9R2cnGH%?(^TjBc9ye9X zYBu6&KN*asH^TU(abp7ci3=DLT&e|eQY zlHqXFk6$GHrUqJxR++rU)+vni{{DEV3ggKuctAOhsVul>hSb;+iA8>j#guT z7{#9!g|8d=Qa&chANeebm0oJKiL&WXU_(@Lo22C^WRQ`D`bp`J^<=!P=d}e2D|P8gpSgISipTJC5)t?NDT#;+_JTwNu)mUsvm1Fy zB7&z^BqDfvO(NnE^oB&lf2Hq`h`9gGFrvlPjYvdX-E$HVSGP+d;_AL75pi`dNJL!S z-$=yc4>*rM;5_KSl@BOv#G8AE1B2TdR}UM$Y?prvVE6x=X36STQla7QANI}(xHk%)Noz48s%k#a%p8$)>q%az=U&WiQo$CDil9R;7#9AwXv zKMXJd50h{VQ#3(#iVJdetv zE-8}FJlUyeL`M<(_c@exNtt}j@h$6Gi=Rdpg$3*fln?u14&)$?XW2t?zKE~yR4d4~#IMG( zmK9du^4(@)YcTzm7A;AK_TwQWf>^?v}(I@Va_mDbI~w$CYD+x5!Mv;2+HXA0kcbbZMt zW&%&Li5X;*WPnYQ0XE6IyU3<(Vw3cMY#Ms#Y;Tij4k>>x_bC3({xgMdn7e^Qlf3Qj zOEfWqXp#(|Niu*YW(b;&2~E-iqG{-%v%T>FP1zFBQ28p12G}(8(AnPj zz@}`8=qP`oWK))PSs8Ad7{YDS{fDvnlp}2&Ypk?wy2WkNZF!+vhGvPiqfiI&q_#sc zf%JAr2GOiw?ywA?$=n@4(ao@&T}a1(lXdKiSYy5X3aX!SF)*}bfn!=&*xGVMIW78ccD{)W>A7AY9gLg zeMlyht`Er|niXsxmH{-GaRYB`nv5UX1E6V=ew0*l&Y&eCqk!21Rb5sEyC#`G?wU2* zJ&%8oDLkLAhjLGipDVSi{5Y=f_$9yGp~4a>Q6=%D^g}YCw0=kisjT4runbViyc(d= zr2fzzfJ&46qrm=wmWYi4zt8|>T~dZvlCBZUeRF_HvGkvW<(@h~S1R1a(hh>M%~D$& zON}~-CoLe738e)@GJqw^NGw@KVo5Se{&s)VD{ZoX*eLQhL`)rYG)O?u60vE@-yu4O zvMw!?$5FaHBoj&lh-47T3I-6%0F=zE(Gp6N0mMcDC`|&81_20KB05c}%0eGm zmz5!u6$Bt%vjDWi<`e7MBlVku*lk>wm05na?j|SxjS|TzCnzB&VWo5W&5-$ig z&6<^;uSN4os%$Nbth8wsogZztv=f%S{8OU*XjDghsr-*0LLk#?vnA$_Muo(e<`2mzOR7I4gltw&e^>_CWa196X;OdKD8Q!4{Lx_kKubiY z32cJ-L&|XDtYZG?n$4f@!{)yFKU(@f%J)j1e{R|({*R3!i7)*hl2MivfJg}7WElx3 z%SbrcgaA(4Bml8dB%G!WIyMME&=Rp}g2xVJU0NnLP7;ER)7~fl=kF5PYguHKjngh_ zK-thMxVr&8W71SOL{;ggltyufLI3DBpLaH(>7^9tOG#PqygEW0YOVd ze+B`M9jdylN^YC%4uNgc-a8BE+luV9tgp(BX%{7+Y{#@*t;?t^v4Cv!NqlJmk&LpW z21G(gWd#k0Wq?YO0V+)v5bFS_G+98~8y~0y3y57d@YtcOOUe+-Di%;-0sD^SlSKAf z)>jEjkbwNsk4Y&?Kq`2^l6?G#lz>P^e_7H3A|b@Gf(67ffF<*5;2>+#fY>O&rAY&_ zK?8!8h)q*|)WtcJb!izKWGnpnAA8R=pmdlcsh;Hj)Uvip=hveDqpemL=#m#@ZKniD zG)a8v|B#HbqyR)hh-3u;h-Cmtrq%$GCIg6#0!W$+AR7!IXo=V~~gIux2@|>yt`C~ zAJ!iZ_YlvoAKva zb`Rx?Z8ipE(Kc8Qv@ysBp-jxYQr!PgztR{qq{B3sY?;I18vEvfkIPSaud?V5w(7h)H-?lDo)}g;8TyQ znf-QlQ8nMGF8KxLrO`DaS&kCW+z6@L8hxx5=_5*fU|O)wXsFR1JeoIm6}OZd2XD?; za6y7kB1n76ZH=}eScaw-}27ACD{adhvz9F2Y7XZ>uj#{2|- zEnnY Date: Fri, 16 May 2025 10:33:22 +0000 Subject: [PATCH 556/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10935 -> 10936 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7d34337c9..eb9a6ffa3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-15 14:22:45,512 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-15 14:22:45,512 submission_checker1.py:3308 INFO] --- -[2025-05-15 14:22:45,512 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-16 10:33:17,436 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-16 10:33:17,436 submission_checker1.py:3308 INFO] --- +[2025-05-16 10:33:17,436 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-15 14:22:45,512 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-16 10:33:17,436 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-15 14:22:45,512 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-16 10:33:17,436 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-15 14:22:45,512 submission_checker1.py:3324 INFO] --- -[2025-05-15 14:22:45,512 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-15 14:22:45,512 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-15 14:22:45,512 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-15 14:22:45,512 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-15 14:22:45,512 submission_checker1.py:3345 INFO] --- -[2025-05-15 14:22:45,512 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-15 14:22:46,505 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-16 10:33:17,437 submission_checker1.py:3324 INFO] --- +[2025-05-16 10:33:17,437 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-16 10:33:17,437 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-16 10:33:17,437 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-16 10:33:17,437 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-16 10:33:17,437 submission_checker1.py:3345 INFO] --- +[2025-05-16 10:33:17,437 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-16 10:33:18,427 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9554 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.53781 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.19844 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 10f1cd98c..faa29ef7e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1183 +1184 diff --git a/summary.csv b/summary.csv index 3ebde87db..55f5559cd 100644 --- a/summary.csv +++ b/summary.csv @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9554,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.53781,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.19844,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bba3408ff..a7da9bf17 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.53781, + "Result":9.19844, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index dd0b784beccbab18955037bec233d0b0ab3d52f4..8eff7e511d91cf512edfcacefe52495bbe0acef2 100644 GIT binary patch delta 3872 zcmYLMc|4SB*q%Y8(l8|ZSjKur;!H#|)QCwVOR_IHgBfQyr0l%5LS>s_WTvu~oX8*& zl~cw#5i=!gvSp7UvW{=``@Y}z{_*_ryzhJ6_jTR(bw9u7r9-^~{rbTJfj`DL*U4cA z^?{z;wovt8Dg+_^#hMk7VJwi;B_V459>a}M{m;qxS4HhOMrH8P3(j@awo-ld-Fmt7 z*iGYy$R9=Kdg*>`*w59}G`xOF^29(_@ak@3u25`ER>|%F9hK_W8@y_jjRXugM|(@d zITqJt%?T>=UKBZlf7mx-`vcVzhwy`!0T+%j)hFZLc*~1?U(|`f!H#ILwd7|-vn3VU z%z(?=6V4$c&0|FuVH`YC0j)Fs?z&7D>!$i$JnM4JtoqA|2$r#-OnOb*${JF#ccy;x zlKdlXU>J3gT>QBo*4kTIvOIm^ZFWZRjs>@wHWtk8EDmUY>xP~1?v=Vw%)L$mJX-TT zEGB3}$VQ^IhhB<$$6Qm|?CC~I!@&J@6wF}JcY+;$|Lb>ogTS`tYuTPNzJJj#vBI~u zx0GsUz1;sD##U5+sx!iurBC^*183h}D#Bu;Oy>M=>ueGBCh=l`KjNwid^MK);pzE` z8m3;mE5M|!zN$44$^9jgMwACOKR2GK$`}lbXrnQ55E1IKUzxWdw{C~(=P#BZ^RsUo zIyHvWGtg~fY`9pM!Ad&%^+YY^6@}IqoMho#JU8Vdh_OLU5+?=_XZ@BT`&X;8%1S zF|$d`ldQyW`6|`f2>i+zS+en~R)ULXTRvbGDM)shOl!|N-u=+h{J}!Kau49l`)oW3 zTc4OaosF9=qy*)y#Vs?D#RHQpE?jS#(g9T&HeYiqnF$*5w<{N|{VY!^7ap9x;{1{GlaoHruT>Ut6xLzAUQoezP%unz5B$ zPnv#&BLSaMG@De}EbobiWq;ht_w@!;`zT8ex8&95E|D6)I zBD}_uw_(wYtPLa5v^Q6SBvVEB9n$$Yw+RqxF{0RkTOi!-I(*oRFPetwf)83i&Jq$P zZ7Dh)xKb&wBnDk=sZ27zr(yLP>3Kj<3iBpfcD>bRR760FS2&zX`|4A~T$b`?_k^~u z(aNrsjZP4ot7v^xZPx*kAM?vKA3N{lap0g%B43i3)q&Wt)PZX@&T{V^8VhWUftMdK zt;Z{e?MdLH&KDBe^+;%v?Mh$Avw)N;%c#tg)X14Y*zov4Z6$g0h0$r7J{6ypM-#4n zwUU6WyM!biW^;BS*)X$NgX*W#VGea4Eu-WIq(Xmn0shjmX~+ylI6*ckw_33z=1B2z$bMRqbBvy@8Bki_irmST#T&_668f}A$xUJ zEBul8hLq%0gFtw}-XRvU4f6*5%hU^;+hm>*^JuAMU0u_ZrB|v;O2_sVX7k=>gvdD- zrAm)x;k^ZW;nJpL90>bC6j>{bW$_mgW(pf)mciHoD9aXvb>@))f?RWM2Dn18yD-z5 z^ix+~u^aOG*G(S(b>_L&H8Nncb!lumBVB!fac}9OzHTCsbcBx4`%i82#_@p{SRzRw z6MG&qvX}@Vc2Y9a$F1~zoHKI9>Qa@4^%nsN+8joG(5TOeF!bW?kZ;clqI5DeV?2KW zWxo;sYQhTH(e?v)gckoZMXduwh4bURZx;;c7Gfezks>BHUP8_mXu6&=I#i&^j4V0w z63aRjNPD;kqTVd8V$}@{jOg#CuXomUL64gkx_uKMVLxoV#fksf&cmX?`H6b*g8erx z5%rn``;k9;M$T^h_k<`ss}Ya0r>r?i2hyb82e3DjtANdHl>qs_%U=W-wFIyUuj1`* z`=-_w+Xw|et~;;y_GhT2U~jqC5BiaUj$wIIHhhV$Ac`e76`OvG=Mm<^Cp4e$3>t#F zPt-Y=wywVE8Fs9+SiECmL@(DXv|hY#(#KDGdaK@UQ7O-~!33U2yh@L966}X-9j+$m zMLR*-fhU+1Tf29LiImJVEMf?g~h688kNWoy^54|$Kz7jk9I>DU4l z$F2XnkNC^@+>_<#w+fvmr(&OfdcMLA241H-t}x}bGzy^(P{)fXxVYtfGA^_8&{qf( zl=I;kSPMG`Sp+{tcAyR|!0^HY8q0@8_Fa&oIxAB(Y#{Vh!fFZQ^rMXb#^VHcM8K%d z5TdrF>eEwlJ&R8{&(y^f&hJ!xk3sT*Q{csu|5&_XsU1#9QwGj6KAAZt3IGDyO0*|C z@O@UN9L`c%seKlW|Uh1vuX{iVgLMqW&RM0yGYJ(6k52B3yKk?2Ifw zQdq7Cnx!5nkXPF~3V7C04qSa-jo-sJ`b30w>^fCg(a~v)oOMWt0E??6&8SY;ZPtrZ zt?>t0k$0OffDzc0kL&M*5N6y$wg7Kc@(D`tac>r`qsIZ7lx=_RjQeUS*~B1lK)Ub5 z7*qu6NZnU;m8UB6(087yx$p@L0y_?w2houc07~XHPu32_nX2>78QJk|-3@{n?q3#m z6)B@`xJsW?2Xv_Wr`Tc2x_sk;)R5*FbXUyLe;;u1H{jrJKz=uBIjj#HL?Fc?Em1vC zl37pAE#Bm4wnykDev@#m`MD~UJBp@M_+;f+SDVjU%u_}+ckKF3 ziC6tZ+||rjC&6~Ov?6g?3hjGXZ94fT2v5z1iF86JOCX-S6Zp11^%z1UAA5 zH5fQ+Z41*#?N?0*s2yjk#r#yFA?Q{y{3JwF!RnH75ZbH0`P6<{R;`@%6as*bVK z`!qA0wZqE}z^o-FWp*8!qanHhP&mtnQV+h;G*sHRIb4vEk>vfX&M1{c*5GrFhX<>Dm=zj#7fTI4LB=#q> zFM{BMF6Z>I%l(Dr7R+pg}c}l>K5L6=ji%`yKYJ>bygSLq`DFM2Q|S8080!am{)| zS#ucBT`UsFUc65!zA2rfDw&yfHdS3)kz`Bx6lZy`th@kRb;JgvXZpB<5Ac=;<)|IT zZ-3*@@d8&d1r^R@uUSfaJGby@A-qPbqDk!@%q^{J!q7~|ty7xj!j_v5CsHLW3o6Ph z2X7Q4IbEgm+)2m9ipb%xGNQ5v>X8IfeLT;d)r8K- zh@{5fF3A5|$j#4sykooACk}Ujzx`_nKRyw+42S=6*r^I*z&_iVU+J@~JM8&GKTXo0 zmw?QJrYVVSFH*LpZak~zov*Dp^ORgJ$c@!1C z)mXREEm+K)g8P?ve(w$bbIbVSqr&K_uGZyM{O($K+NQUhPU~#$+RWy!Rc-ZMGQoH! z_*B^xac5d@iadYvD(PHY~Se%yOX**P8BGkmaYNajst?kxYsXfGFfH~wj_g0TxS z?AXg}S)s~qry#S0%gUBfodIceN=|pGrd@45`1KYR*XvqWiCTRDyqOyYe}j)loNYVA zCyqE74SEs^FA}#~79#*|1|Wg9o_IcaV>Rh_$NvG9?%(sg)tqh!^GI)f zyHxX5u3Q)OF8^8X%WI#n%r%#tE_ecca9s!?_2tS&LrU`eeaN%$@iYNWea04Jh>-Oc zECnuD&{pp^RU94Zp{iqRKiPul^3%+##8@*)r0vS)b{hAv&BNO}v~WSWkkQzL?~ocz zwNAeOy`gSsxPp(6uV@;BDda3oGm%6&5Gb6e6LKSh=o3O#)RG{tYRd4fDUd~B$H;Ik yQMrAoRly*T4Ic<32KxUeF1`;==D@}GKO!5bi5~vHS8?%LiXGn&zmC0pSh%af6i8B0baYnGmD zAw)_TLxk*9)`UUUk>wq~_jerc``&+Cf85t`oY&|2e$VrJA9syiwOy(2;X{F7w+SC= z;9*Um;Zl`OOmB!wT1^1u^Mk2Vvc!Cn;$a2WW-`caWwg5gkwP5)K~D=~cOm+FyV`{B z#qH@;>}XBwY^=4$C9k5Cuf*owFpa-ZcjMESzP-~uOp7xfP8YmCiItGK(AXS1qpq;8 zvz^qjj~Q+88E(^lsurJKP0-dMR*UV91ob!ql@tfvPHA!IDpI{O?Afq~QS`JMjY=vC!V$j2WqWg5-48Ax5!Kt{);k9uc*W6jCV6P?tYCdCIZXDc^p za6)=l-;cSas$SxAsaGv~XC5I``2nvf3)WKzFomcWm%8vuclhA~<11WkCeA>2LA4c_ z!~LG~*R6evs$k5_$nRq7F}u}g#c=~R7bL0Uvy=H_#1M(2JtOs#EsOXwaZ87Z4D?!KhGTp_QuGUna`a;5MQ>V^lWA()hsZOd@9Lpo>Q%Ne~ z+%NR}LiyTi|F_>WAzq89kJF)sryT*)uyy8@g`5qi@fE0#yu~>E;b!>{!xt|yI&bSJ zbt>fD66|M6sh#~%<4S`}ESiibk{cz1t>l`#rnq9FM%q66dN$mU)JD6$eJZqCe;>ZW z{L=i#^q7ogOh&SHNS#Pe9IknJeqvqAXhr+7=cp1q}{K}+~ zYigZDgs*L?#*m927_eXMnaD{4O~E^dW(|1%32D03EGD)gdb$d;bVNjWo?W*2zO$>5 z!^Gdqzw`|EOhQl(h3*8qDJ}t{h=4}J4EjT|C0fo$Ew&Z|*?d-=>#V9F20kZ~S--dc z>2>OtbmmY(@npB04tVTw&&YJEJtD1AVRE67x==#hC7kGr9>|$HlX+XAYD+s#Y;N6^ z<)d+ZD*E}S^^B47;W^#-Ypo>#VKk5DrUmKS!S!u|XJa>J;C`ixtXcq(esb+LBND5< ztLuEFcj2OcDbi_^I_G__cjv7Zxpy32kKeo3Crvr}r&>ED>7?T%(W7`jxnthrlKLAx zY^4-#sDbfzRVz@>SK%hH>#PWw@XVFF=Cm;VWQ-Zp%&30*CW~jm>}}RY+>83iX>%{* z+n?^Q4ERK?$JbD18?XSSkF3-v-`nFlS-X4(`+H`icA)E9j}&&vrQhk~DAwq6LoU4@ zco16Pc$1<1Q$0CAxfIiCqg=W*u3q|+Th+=@UXe%Y6UbUrJ!|$azNxe$r%7@#W}--= zY%@8LPwz=`N~wwx?K+rvMIIi}GHJ8`2%iG|2HG?UAlEmrODq{Kc(^{u-C>lx@QU#O&< zW`u4+d$yVqWl4A3wIy-M`NRG2^tEmJgv+So`m)w?!}?HV{InfuSp%m3SLKZn^#wZ} z0p(c38V@niN5Jfc7M*({`GL8{`%?@vAqNwRnTGkecJP~{XHO8oA||%rjKxStsTcCu zIpp0xQth?5mg9aQjN1N*yE(BRI5)W3qj%5n`UxDGKE)56<7k4f zbzA?Y{(TM~^HZ^b+cw4DL3|+@!kv!0#qsB``>lcwdY`?;C5}$*ehiK=YH8`|rB}`Q zO(hsyxOLmAOEm1+DCjM2Y)9i83KBwpB*=lJv`!7_}t=)hK}1~nn ziEI%_`LKD4#;7K8pvF^sp94ORam(}U9U9Z*GW~-aagT}3Myq)nvI!&f|BEn9pNm)v z)l75d`X55{e-X~@eH;Wr+7qkd&)W}cj!jg}Rn>Q^8dfw9TfdA9vKcv*rhi`t5{b7z z6NusI#xe!72$Vn!n4C}lW-Ys{Z((H00dM+o3RmMv`NwCZ#CDQ(6VEIj$O-r)ee$|} z8oD5$LwY_O)puF9aN~)k;+QnWKvQM#5{!!|l6+_gF0NO?iv@ZiEXBg_&kQV@O%nNM zwUYb^;>7gkt#Ho@_l#MUp*}0^9%kXJU*V(dgwL<)x|IyhDHXFQ4BH-<3e0C~DWOFs{@ZS}#|>SUJ6ai%MQ*I>xKP&Gc!B%_&nLWmyK!zm~J zo`Qp2r3>{Ka=x(qhL#`CDTK6M( z)>)lAwF@`+p}lqVU$W`XUF-&XoUL%G#l^l6UyZrU-h?b3XSOa~E;P?FqBc``4h{p! ziAR-3a5QUxuyjOxLH%B(@=R8Xz~C?bV?#kQy%-H*c^#vS4?8-Rfp&#(fwfbuu6 zQjrwnKc*w9jA{#JtA|sByayz+5-@`%ZGh>rDaRLNQaE5~c3c?Rf{Z4=3xNVuf%RPY zeuFch2dh%#G1$DsmD0Zt?O2qJURyO?q9Lwqj0WLuW9p(T;AXmnJ+}FsG86+fm8S?8 z*A2>J^)h%2qJ-;x&(tn}Uk=NXmy6756~sB&@JQ>-OXvnTRcuCL!u%57rO4Y05BXdF zS-Gz5y;$y6V*5dU`*dbs7@KXF&6aw+lFg=uu0z0LtA2U0pRK|S`;UxKzU}qg zK zUIpPCJ43xXbp3<_p%k*rRxdJ&FS*Lt898JhSjOo37oe;m-E0Imb?sS<{?L6Be`4 z*d+Np8IjAHDrRgq0HOwtm)S|nWn256;2Q?Ni0BsiLNK>G(asLCO%z+d28W8Z)pc*Z zX|uY5UtM`>MF>_abXdLalB4S${;F9B+g~=_w&CSA_e1N^I1&oV1f@gQqvJSKM&_S2{|=*aWV8au&2iE6l43L zi;LMjNv~AzjZxxy)xMC3Ysip7jzl&auh9V;n9OFQB;LK~N#$wQ2EkDhPmW3eLI-Xv z556d36irppGld2<7vEKo8eX;CWNa5LJPoksY!9$xe64-lE^?&JaD$Rk(|JxY$_yg`G@O2TFTFCeZt zJv#`9CWEREh<&%Ew>(LLevq%!o#)s4?dNu`uK|@``MI2YZTkbvpRnT~bq~yj|a_pQkK-NUcLnuuvfq39Ydv2)f!pt?x+fUQ3;Y|JJSDX>affu^y}a*YrLm^lts0 za@{@nI|6L)27t^A4BD=%`u%zPBV+oaSFW@BiqE+{T}I}Mkh0E}@2lRs>p^itAxS`o z>0IjiPuAwDYU?{HQe(^KeBt!HwONh5mFPY2_z!UCQo(b{?SdAc(~jGraK|x^q#ZEv zx`JuNI4Wd3{oCXG*p`dZ?nmBS534xq4Ez}X)kV;7+@*VJ>Ol!KOD0eo_aHJTb3bV} z^nxX2Z7+IE6!aB+)*Ko!SMd=cbc*HwK_{Rxo)7#<(b2pMng^5?u$9SKH9 zE$Ta#dt{+gQx>A&K1Q#2Dc4&8=TFFI-1WbF?=3xhd+sIXBvx`a!vAuy2oCv34sGf8 zf+iB$H@_up{Mq&sq4;>xOWtVvuZB$mw}wBYf8{9=`<@N5i9chT{ciK2%fg^E#2@MX zQkGen+!x)W2@Ik)P?GP1X=-KMR$A+#GT~+7F+U$q5shdvBZ8pMXDVe4f}<-SAFsXU zRT;y6%o@FzxBM;~Id3dA#;aiBHsFuEAX#z>c-W}tiD~7_^lH!2db2p zJ%t4aY-TU~Xx8D_DFkmfj|3;$$}B5#%NEp#P5W)^*vP-OPA*`{T@mxCdRd`G_tS9p zN;ti={aQa*`uE+5L*DJ7`+P-+3T0?J=l}o0vJ@RTj!^BCbUBi#6eWYB5mF#f5K-Ol zUN8~oM?Ix1N?lWu;9QfY3d Date: Fri, 16 May 2025 11:06:02 +0000 Subject: [PATCH 557/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10936 -> 10939 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index eb9a6ffa3..2825c49ae 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-16 10:33:17,436 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-16 10:33:17,436 submission_checker1.py:3308 INFO] --- -[2025-05-16 10:33:17,436 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-16 11:05:57,837 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-16 11:05:57,837 submission_checker1.py:3308 INFO] --- +[2025-05-16 11:05:57,837 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-16 10:33:17,436 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-16 11:05:57,837 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-16 10:33:17,436 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-16 11:05:57,837 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-16 10:33:17,437 submission_checker1.py:3324 INFO] --- -[2025-05-16 10:33:17,437 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-16 10:33:17,437 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-16 10:33:17,437 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-16 10:33:17,437 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-16 10:33:17,437 submission_checker1.py:3345 INFO] --- -[2025-05-16 10:33:17,437 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-16 10:33:18,427 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-16 11:05:57,837 submission_checker1.py:3324 INFO] --- +[2025-05-16 11:05:57,837 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-16 11:05:57,837 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-16 11:05:57,837 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-16 11:05:57,837 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-16 11:05:57,837 submission_checker1.py:3345 INFO] --- +[2025-05-16 11:05:57,837 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-16 11:05:58,895 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1107 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9554 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0345 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.19844 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index faa29ef7e..73a426e80 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1184 +1185 diff --git a/summary.csv b/summary.csv index 55f5559cd..6752dd473 100644 --- a/summary.csv +++ b/summary.csv @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1107,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9554,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0345,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.19844,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a7da9bf17..eda0d7c33 100644 --- a/summary.json +++ b/summary.json @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9554, + "Result":23.0345, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8eff7e511d91cf512edfcacefe52495bbe0acef2..83955e997e39676df7b65d149b7081c287bd60f6 100644 GIT binary patch delta 3581 zcmY+Hc{G%N`^U#ND2&LyGsf~Y-y+F_46@9SofxUa&|r{#%`K^@gt@I_iR`7U*+%3c zYxcsR%@)}jTZW))$UUSNs1vl|ODZrLT*i79fUaaPf!&sxE z`h$_Oww$62YU#wt-lE39-`QXW*DR)Hzd=>v)`ZOkv9Y5cb0wb%QKGQ1X`p4UYb{Lk z$>zz=1g><9s3maYYmtUTg;^T8+&+Ek3On*swe#Bi@9qptUAbqa$+dxBMyR!C;>cOc z#P?otxGzH+BEe${zX7vm~%i_Y4O3v^tAvS-Cc4 z@@}<6s8iQRIuA_vr7kDrriJX8YFkylH&s0qpQcL+`;DKyh?vjy`ssG7nCIH^Ira@( zs&KT0>})hj{*-_F_2hWSt^?ols_}JV{I_L+1dhDxV0;+sTbWb-%K#ZJyWV#03En^W z65lC@-#LbzpVm_18=L(whR&Z)9$#JH^QW;4F2FQ}v~JI{@W|ygR+S1@zmK?)9PhYM z36~u|;FIg)gT*=g7TM&do)G!yej!G`>2cQi z5EZn!|3|3p`UAfR;1=rj(@U5T$!3w#hW?~i`Z_+A3Y507)e7*=`6R)@6)NF0vWEX^ zP_8j1B3A`v(T$1T!@SA_@n&nx?m$Q9^T#vK;o6<>wE@pXm+7q?7hRO_ zhYWTqVK=-wrYl?+gjBn+v}KK9IUWo?u@)hm&JUPl}1>|YOuuA-arcNwgY z7fgMPZoQA7&Vk}w5^T>##c@NRF4tKm9ng=<_PivnRG3K+V_1WhB^o_ZG}BSP!3Gt^ zyp0mw?XvpH&8mE>Mg}L>vsI?@Gews6VGik`wMheibIAZtDL1XK;{UcM=UdG*3}L#2 zl4fj)OgvcQ*d8^aA<=rX7<8ECbxYZmzNx-5{P|1c`%_K^T8}2%h6*Z7`sTf#`e7%W z=<3aiDlVw}s7Y9$^h0a)AxYw6cyp^Q5;e`&Cfe_G#qn-`Xm-M@_HC!!r-i21#qH?! zzi*wm;T{Tzb0R$GPg%hhCzuP7D+Z$mL3rFwnt7m@S;WpdHhjsUH&k#iU%sd?jPEjb z+|9kpfyhfpzt6TRmw7)|O~Ljma)(uY#nCMd8F)_hO*pk_A=0-u5dHpZWJJlYHSz8= zylYn|oRq1rj)4xtMfCX3v6}lBfZH*6(TF8qKieoh+xobm(|We)J7|%XioWK0WJ1Xi zEFaq~`m$xTIcswFU&U_Pua$3tZPxpjzaJz$RiJb}*)Y{my+#;Q#G!FUH zt@-_ne;!NTK|@C>Tn?(Z@_V0%g}II4=);l_b8x$`h}R#|!@xqH;b%+s?{B{ssJXYBU;=JPzxXP7`R6N@r5ea$vC| zeUzGFvGODWrzcUvd^OhJghcZh6V7Wkdfv6kkH<z%W6Qo|%0_maYQ0g9^?K}9 zZmr;7%x0)|w=M<^t-^^U$oHdFFvh)Kfu|A0=7PLpMn@0g%kRC0a&kBsXqeNq4gVqE z6z(^u&z!&in!@Z9{lTKVpF1PAciXpFt7lQcQJ`*n!WP^A^F@wJ@Cd{PX-U36F&A23 zCvWu}Gi%Bck7M?Pq;MHv+aazZF~qF%P6nQ8F_VgrVhm(d8lrE8E>*I}0*Z|0$JN9_ zio6A5LNe*vQ4}XZN6!A|hlOkD3l#avZm(;x@?P-QbpXSPU4|9u0;>!wRI=<@97(HE z8Yf&BR@7h6b>ZBEx11tB|c(dA>VWU$-&o-aTo-nX#1AXmh|2^!i6|IQk#GZ=<-wDxMfKM(6eS@1`en z#lBWa-EkDjb4?IoEuAD4KkII!BWegp;BiF|5$S|-YM@(u&!`FM4KnR94DU=Eh4r%9 z?8}J2Yh$-N)btR&kSxTHhrtjWDOHNQ4^|h4<267iqKdd+HLI1hclz}n3qKB$n@R_V z(*D*$(N5O*eoLkAW`vjx!-04cQxu#Zz5EczJ8)tY)(;`US@5JBsd1)!PrxQxU6C);^*YdTRi!ulOH;>l+*86Sshzrm&zjN1aA;UW zKBGTtGs9K7r`T&)kjt1B#7eL2{k2Udq~rB8_ThOSV;GX7EV9SA?41B3mwS5@a_TWG zt`ZOug1ZOG%~(M-j<8j$Pj2=B)3{I$LiX+#tlrlv49z?CA(zRDOzzqx3;1gRId9BV zq{Z&=Svu(*12d)Y+FQOa@)=35nuAAR25tbB`YsYv`%Lv1xIy-3sW_hOD|nivxlTE zt#RVE=V%OJI3`~nHMZ4U&L8=MBix~N$Z}Z782ZQ=K<7n1I!Wl%jl%WE3h#94?=6>jBi(Mk!pf5#o$%?1VLq7 zs`CbZm);{6y4qi!6Le1g26Bw%1zat>!;$W~ zN|QXjg$-I#^i8x?=^S0Fa;ImnL4?9}yH7Cl11e{uSW80S&(GQXI^_vCYeQJOhhv!~ zbPyu#4t|hQI<6ysMl+4;?S0IOk@#o!GbU~m!PpC86{@KAXDb;EH$U0^oGaJNFX77x zU{?{Iu-mJTf9RbBTM3(e#Y&wtxih=g(o+sIe8X57W!pe5uC~MK(bWzLz zFMNG8fEp|wJ^`@3*h&WY&&1fSA6O34nQE00ea!#=M@Cb;$^--Lm2{Z0K%$a9_XBYV zB#dw-Feseh9tg@Pp9Z&;M49&_!BbKKpu94#*s;6vOc00_GX!!9@}CoPmjYTT>oOt1 NBxO11SB2v@{|8j}~kN9T9W@A>2T<9W`x?)$p#>$*SZ)9769OuupPK=9A87}Mj40eXD?le9SimS;f-5I2# z(gJ%zSFLlAz&Y+{Pg&%$#SK|=yvm#}Sx*1&-dk}4!Rkq0aYJ_Q%Z3#Hto!3FFAD-t zCxeGrvEpkf&x>bDE4A4{c3Tr}H%FSsih)b8WgJogeQNyu4Vg|(n0h*nV^=ey{%RtM zV|Y#`v!-oj4Jp|(UC*_Vf3h1KL0KS`^bNpTd&)|erY<(+W`%BB>^4)!LVG((g4&xr zF%y10QWr~hZxFp(3%o5Rs9%wd1Y2+2RCU(uyNsE$jpT;GhwCVq{zAY+Z{)*oz`yeP z!EMb~bA6@*{-WD(A~&}-m1<{vz5Y9lsbqYvGr(13{&@fZ`LybbeniDTs%U5B@_K-q&)_@9H9cUgb!Q{CtqLvG1Q#^ zWw+dUIMr8@7Lld)4Mo}Ty);jI+3Yfu5UqXXUYZH*R}Zt%LSj)zso=5`?EHza=}QIY zHXhykvUC$F{%K1yE%oqvB3gj{j%XREzd}R{*i~7gmwr*u%oEe1X{px0sH*t#_T#JQ z^yjj~)cS@gxblnOEHtz4wzcy3tamo=)4QBhwok}O;Qj0F1QnkUsb%I}qn1?d{;fnb zGa678um1Kx%7GlI^rqoc_B(MOk`mMXn^Z^R@2rtU`#^>gTq4)<5xZDHl4UfdJ>!!8 zv8DN=g?iNhDJAT56Cck*KZMbA}Ei;7^u#Ymg#6iS@ZOH zrlePy1JPTZoz)vkgvAl(&X@Vu_L?89%ye7WM)ny)?DW^tHCRV@4%@t2nW^gE)~`Vwngvu> z8{9uxS3;yKGZ}hKNcq)uj{;Sq3U@7q;|{NbgZ_FJv? z-&$^{zACQqd%H15k`<9ZFv^j`E|QEyPJ`c&f<>Ipcw@_?z>-#IhNUvm{Jw_u8>G(x zK`E=Zv9jx}_M@T#T0EJsD($P~h}j%vZr6l%fWgWRXpC3-p-Havb%TS?x0ess#dU&Vjr$%IFL ztt24pbySiLvp1ekIhWn6K?yv?GKYGNR*(yV(tI-fwNKFg`O(#S8i@vxupk>E__R(^%%mRrJ={p~VX{KQr8q{2Ag^Kz$+v@{ za3bj~@s4i|0^tjLkC@Lrmp^o^LcP$fP3AcvpPE+D**QgCe66~ubZl>aCjUd$O*xn1 zH0jYCoS$G1T-tur#Qju2}&Y589<2?{1j8@I58ypr{sXN+7f%A(5H1~x=p{rEd_gMzCY(dRJod zJ%OB^$%jj9HtPua3iq0*b1Q3IecL_kQf0Al&&YsI(=D=HcxcqmPc6S&@3^3pZ_;1{ zPa*(U=rOK>18}Xw47_fvE2RCI)ry1T`*TU;>K3@Rj@|x?hc?x#7{(3**w;^!h*-~tiLWj|rxEG&atn`My$#hv^%WG*AL7ky4 zmr!sC%efS6cE_P_5H=|9<8!bUW)`vlE=RIZhvs28;X#e1!=n2JPgR|fsUFrBDnDwy zXyy8|g8$ajL@z|p=&7%W+Lr3i<)nHJpK`v5y9u1%m2sa*^oNt-C6j*x7H>Ifhf_0@ z0W;=jQ_h(oJXs&WC#(>damXBqj*LRdyy3~pQkqbE>mnE01%9ODj+v5o zt{aDBSGsYz#@L?~t2dvfciF|6^@~0tj_z`Tg(;J;@f;3MJRYPpx+d`#PrUtjt4*bZ zfp8Qc7be;Pd3gsbhV(C@wRME_!o9@^lrtw#KI#9GE)Tj(y5V~f2B@Ln8C(1Ctui#X zr#v&#k*g%_Z#9*2N38Qu7mxA&vY_Ph7#H#!kCfsuX0VfSiOak!e&J>Dgh11f zryo<7j^`D`e)*Nic|}p}%DNZ>FiBkymWDNt>m9-$t^?SPBg51)Rw%;Jcphh5NL-<6 zX>aa-5xKlI@sr9FNxWSXFf%nb7Tb{%Zr4{}?R6Pg$-=yJ36H5`ZugXDM{>6JyIV`S zlihh}me(ECW*Dr;i;e7o0c1W(<|A*uGspzYX~0rC7 z%=(IHWbRcvz?Vz+%yc`5d}6}jM+lWMthBH?RqP);RX|bi-ksjhre6fX2i-5|VU`Ap zXcp|rR;vs2(4no#y=^b$80~Z#BXr1IRKvsL`&m`lGOuFr;4~@HkzbkD8zgnVHw7^^ z2%y4MMni_wL{s;tfmD=5D(<`Qd$pN!mNP4eWTZq78j1-7-}cCPOI~vx)LAGN$X$3y zE(w#)Qf^qwFd8cxTe?0a8fBUza{J13S5*+^9dAmA- z3F~t-zua$Gci88rUWTN84<4Bh&oF)D^&;#?UD!@W#Dm$T=1umP!uVk1GU063p6b>v zcIU8j_)KJYujQ^MU_}Z!c3kR7QEYW*>(VN2XDu>=>nC@rb>_j^H1{{D;MfUs2*F+5 zVs#gD>zmK;{Xna~pc#z2?%Gv@uC#X{f3Hm;pH%HmB4uF{$c(bzsF901S6{XR9lK4k zEuqnSW^q<^KAw>)$7>*x14jgug-~?T92^0t;l=CtgIE;_7o6OqB{;3(wQ<>>4WhQR zY{%i>owA)2L{S%2|kZ|GkKACEF_JH#ge zAWp}Eo`u7UB^;N;@qnj3NT97dfluC0P5S-uzd>c&*ycjdb9o{$he`c4XB#5CGh3S$ zYntR}I;i&r&mX+H+IM-jx#Dc$Gw7omLI|lZmp2+xQ|2B*o=1*n2rSoUZ8E>&bN+&* z!UYT4>H~k2#729ca&Ve#L39Rc=2uzSO0kK=t*YjB>h59t$I08&NI|)qqj8D Date: Fri, 16 May 2025 12:06:25 +0000 Subject: [PATCH 558/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10939 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2825c49ae..3ba90f5bd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-16 11:05:57,837 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-16 11:05:57,837 submission_checker1.py:3308 INFO] --- -[2025-05-16 11:05:57,837 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-16 12:06:19,986 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-16 12:06:19,987 submission_checker1.py:3308 INFO] --- +[2025-05-16 12:06:19,987 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-16 11:05:57,837 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-16 12:06:19,987 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-16 11:05:57,837 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-16 12:06:19,987 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-16 11:05:57,837 submission_checker1.py:3324 INFO] --- -[2025-05-16 11:05:57,837 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-16 11:05:57,837 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-16 11:05:57,837 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-16 11:05:57,837 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-16 11:05:57,837 submission_checker1.py:3345 INFO] --- -[2025-05-16 11:05:57,837 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-16 11:05:58,895 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-16 12:06:19,987 submission_checker1.py:3324 INFO] --- +[2025-05-16 12:06:19,987 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-16 12:06:19,987 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-16 12:06:19,987 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-16 12:06:19,987 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-16 12:06:19,987 submission_checker1.py:3345 INFO] --- +[2025-05-16 12:06:19,987 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-16 12:06:21,030 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72618 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75204 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.4194 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 73a426e80..3fe2180a0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1185 +1186 diff --git a/summary.csv b/summary.csv index 6752dd473..058ebecb2 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72618,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75204,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.4194,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index eda0d7c33..860bea9c2 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72618, + "Result":1.75204, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 83955e997e39676df7b65d149b7081c287bd60f6..f6c997359dfd4b36a354c9b5f80a59b9909a8bf4 100644 GIT binary patch delta 3414 zcmYjUc|6nqAK#p*$=OAkYhxj$g_tYCFt-?rgdB6v@O6BIGUb@f6{YzSiAtMvsOXCv zOB$j@hpTet9Ao%Rzu)8W`+oj;|M7V|p6}=NdcPmeR9cuLGJPoSP1 zX|$p6jJC(KsNBoYP9>``AUAW272))x)?cYubaW(^5H*~zJvVXX8%{Y8(4Mgy#o~v# zlCW+s+pU;o#075du(`wM4!t5+d-o9XHnd@=^SgF2n{p17t3grMbR zwmEV?Krb+u5joVi6+7U$8vg8FaIXbQCQF!kBew3AIUWC{Q?Yot54YYq8(WIB%f2=? z(~w}0brz`?YUq?+)iiY5e`7r06gxd)8G+yqcN8w3$lmS%;!dJM-Q(&-S%o!$p6W6! z0sUUKy^OWK>41c&692;PPHUkBW$Y&Xm54VO&>RicN^@h#n#QeeHEhUD*=VQvrbuQ) zVf$MYpZ|Ob+YHKm9$UouHNLW~ne+U5o)HHq z^*U8IQ$u8zh^kp=$k)v1`Nzr=gY$4Y+mbYIM4w^J%OeGc(v+B4s# z+#CtGEL!!0Emy5aH7hUs*GbGNRR;U;``hi$mq_{x-tpNIUN&RWO3iW^mUEV7krs-h z%+;=w-WFYgPr@J*=ha;!)%4T`ZU8{GY3*;KTHG8Oj?OAt^w84K(0;h*kvTjp^wCBt z7tw9PhWLN^_y%J>&b5}&6nZ)s^J1c*Hua6;q^;ooX*Vmu{ompR_vz&*ufb<)g{2cS6sskG686rX z1<@nh@%FWs5c}-8-+$w75G>+vlg9#@u@KT(6J}KA+NcTPx`cPGneq)~MQkgP)@r)s z;+>q^Z>23h%HOpS!gp3zQ0<#hdh1VQdWef6nSy3-`E!lyGUHahOz>$Nh^ifLM5 z)n`w*B+qN98Xv58{(fMt2#^!XZ&UDmvS)W&;>IoCgoO?R*rD$vKV_o25?c8X%u6rs ziRg*_fnt*8PGVnXvpf!$!79p$n;VC_{;IHf{bQ{l`axTh`GKo~Gs1ad8yEMEAHV*C zWg3}lHW+h%;m@8>w^}$S^yK4HMHg!NBAh#Kq+V~D+Oip`naNfH-n3?Z%k?I^ zPJ?a8;Nxha=2pmD?vS`3WjWbf=C5BzIq&~DRG)^Iwo5@U{;Gwrtt3;rX?4ko0`xkk zAHG2~cT1D~<$`NzexdM6ZB4_c2|XHdd#l&t?RunN-yVA=iFvn{Z68tYxXvGMBKz8ca3Y+zf|5g=`rSduf zD*irC;^{>XJ_{MBUP#bHu6~?$X!Ob1t6pPs&q?1uXQde)r0~!=-V#m<73-8Q**9~f z003)ebfw^mANb=kWJbm~Xk|)g@lT>LCpnA8vC4mi~SzA2+2U7r5HL^xXL@ z)8gmAd3*NTO0N`nkm5t}Zr9%S)89JBz8ycxkChAYm5e&(kJl9TeBBYBX@<;g65 z7lU(xjqsaBbH%5I401>a)E(eadZb;}9knMlT>RljWBCcsOa)=ie_M@OO{V+u1#Xfc zj@UOF7Y|X%ai@&$qBqM7K>2Wv#g$Ed4Fy?2!w(=o1IKG_UUNpnOboP^Jd zN{ZK~;ffbeQYyqx0zJ%ghw||LR*;Gm zF_saK4-lC|-aWde+FKzu!Ubk&PeG~O|LMPWR80&kN zf@7{TUrfa8hIW01^&u3bCL~;~jFhD;d1;c~?d}z0fNm=D`xf7V4uDC4j!8&WqYsi~ zlU+BdT#^>5gVyweUzk;#e+T$&`lD3O$o0i9?zm>hiz`lxlbW*yRP>g3PF@nPY`uTV zR z^q%5XSJvp8aJPX=(nGuhr!xklshW5nJIW2Hw?IvDNtv{YWz#bq{(_UdJTR3r&B1i| zWO^1~I8U?ad!BvxV#oZev?FRQiIgIfx-g!72?n?9PWR4LuSZqin6ley$Y6H?7fLlw z0LY`trAWZi9P7BLpx{eSS?A>emx@WoP}X^M(Z>A95n2*amFLJGKDeq|vC!?Hz6)vojzShNUjn{GU#^K=@}kn3$zFf{gy~ z!~~R5K*C6&2&&UeE#P?sSsZtZZOAbK3Og<^6l-v|2oC&RI<}4TuD@bf>da)^G8$D4 z&gEeWo9*ZoBl7f;bM16PC_=4zAS6bHBr3xjDPQsVPi6Pp?~gZh&WV50o_$46e*J(4 zxqg49h~U82MX+HoUySPp7rGg1uPc|Qvs1 z;hZAadQ3KU#(qG_kLb5=&pqB{)V_#TjMtd1NrPD$Uzi-MkcP=m7#Cz)9pE0?PW zf^)@vuy6Q$4mFcGr@~360(n#ay|)6)qJmAVJ1P-PQ=@-QyS=o1Til#iKV<-Wy{=qm zyG0-f>v4A}n7Pe7r#2tmd4oo%5)Di_fwcA}ubJsxn5oR=KHnmvR_B*vLMCpDnin$A z{gzGd%E$bd<4V7na(qg%Mt(Barx#?F+H@~u_GwNtmwx~Buw7T_i(MNFa%8a9di<8h z#8RQdEl{C3J0qQCyR0$uhUJZbh8G`IIbt|>6%&i4Ee!sv*Kp=#^d-qG+EsUrM=ZD6 zrtq+pD48Btqto4Q+jTKxlbc~r9e{%xi{EI#Z}W&-U$@C#HkY5+^2Y}e40hVg#mGK9t}kw6O3-N-WN_RB=S_6W&&MEl2gN&1cVJdZe2xL1~X zf9;+)WFis}n%3XAI}vwoQpqa8+r92>z`YL=DE9#|*wZ{&;js5^;U+0|ik9(RAy?T7 zZr30GjJJPxnORo;?3C?eaqFio1q;vyeot{)^KL#sO8xjS*hycKnJk@P_StK28T94^ zl4>lRv!vdBAPiK+^)-Fuc{(3rv4=81*L@*&A^UAZ2%RwCacS!?@WAWahKo~mqf>-& zG(vY`uiNp5#}=X`#0HN;2NqgAL`EkMu4FvDnCk>3DhpAoo?V`ZVsmNOvKR7gDA*Fx z3%7VMg)FGi5u>wMxy|R?LTb#o+>BVARk^oKSBEqB{`dajs(OI;Qrr+Ia1kXRVFJEN v8ARxTS1FQ5;j%kM!oVPqJs$`p0s8-!eB~3x>8K%CokBjUB=l8f=g)rt7S?K=i-_PUm`n;a+&-?ZMJU(siOm{}OqEJVP&h@$;>;C6M)ez*(kWG zm9b?ys}=F}T<%!J2XgYG9v}0UiIGbiKb1PpUb_%~9{nKe1U6cuO_|=@n|nhNZXOG zHF9M(n@y>C>fRN>f%kseO+`H_De%%X_EPufmON`+J{qGDuBQ>)fAI4SQn9aC%l_SL z^>oZPjJ5g3**M5NrnPKMhtlsGwNdm>sK9>Jl~56Np$bfUekSu{BLgfx~wWUJg?mYUpLY06Ovyx z6R{`B+Sxp%#E(^Q{=iL82295pwmV%X9kVA%G^5g|oT$I(`+NC}suL^TVz5A;tWpP% z{y7t;wA~@`+?479eh%G!)kKVGd~5J~7Nqz&2)Ad?vF__e8U*@2up-vF9-~7(c~jrs z+~iMj$Uba)qRLvA(?=OG!}Impg#A6scM)H6b@(qQkO8Jke0wLOi$mL5j@im|SQclK+G9+%COa=x zQ9MTF{BqZ{&Y7$Zy#LVhe7BdCX~sxL|D9Uf7hi(1F5upJ2~67b5qNZ2;s`WCE6v%Y zUxR!D*7n2|i5`{jQ0etL?Rlv;rYQY>=Ze?utO~ocYHkAeALn=d=^vve1;7IYSz?sO zUQj$)It6PCf(Km89=oQ0t5QO2XyAkgJ4W$UnQmo8to$FiVLyL{2U#wddR2UW|E;UH zjrCl;k*i`RbDnGM}37KBK4K2fUtK56cX)7FA`@PD~ z?1AAqN{92tALEWbLCwkf zY>)ECW#^AKrE8C&7JxlPU$J`cOou&5qm4I40|u55^e>Yd&p&bjp!&N^M0Wv^-2DDn zIy^wmfeha9%96Hh@cXyXfB6h8stg)Knf(OskXMB?G%!Op)EDwYigXWTShYTtf)(B3 zC9&ZFIYKQI7BWH*kzvf<8X+XhFi10ZnC-kv4qSPTvk1EyF$dHckD(4;8LeK1`YjvS z{?}q;#%T3WxF}t^Pd>=@j_hZ_U!oIj(c;oU#R_)hwkN`{CR%EgBrKz4TNR zsjV(PAeNSvH{OJ8>k4Ub%$!cR)B5mi_Pf0ak$9ZH)$Ml&6KW+;&cB3Jy*puiDKY^A z;nw1RFr)zn)=iNxL|A7C>qq%gQJES6L29d>G%TKW8m{nXgvd`wH1*M2&@hJV^(~XgQC2g>dEbE z*K006fJjMtS)m;GW|T^~L%=EK)wdHdcieQH%CVpAM6VM>NT6J4D_keYS2=}Tc-YH| zWSlah530g~I9eb}dko#c9j8B6XMCkT6;v6dm=b+UV3x@BQuLJSE#InGpiXjiA9e?x zN!1O6J!k}kY->U`>fU)F8=}x%)RQ)^iQ0u1vNat!j+X+Kpts?8sU@hvWd+2XiX-Ey zxWK+{%wyShk%qoK|0JyU6(d@zfSA+55oi}$(9p2UKBs>JJRvPGSdB9U5rHZ{d z_-V_+9s{w=R(}y+DQzA-d^mQ#EnrciPB5jDfV8jGsh=xfn`mlW9P#fSag-{U$!>Ak z6zT;6{|F8w|0DQ`L)c=-rq~Pb(sJVxHAg!20U~eJQ@O-9U0IAiLaQq6ZV|xigON9C zBgsfFv{$phalD{DnjLA^V;#rJ=RnzFE*sj)u!hu?XT}zAHmDFjofZWm(DF28UO*eK z5ag`j`shk&X#M;{{!`1QfJtC_GpCjQsz3jLDVqOFedwn=!CFDI{QwNOZfl2zDPU&P z2y%V9IM7}Y4JJyY6>AQI%SbNCCi=q0AX^UPU9H#n^_c@D){;RQt`#IAX{TKLgK&f@ zhlUveky&MC$?Zk!zR0$TVo_c|+4}aTXxJ6v>5D?(oiQ~8R0yxW`z^HWs z;BVAVy@giUD}PFOw-tz*?Hsv!oaNRzWjrO_>}fT2>s$q>2)=B4nJ*0gZ8$?}3Tp5y zvmt|D5rhJsK#MlN{v^)+)bB^ewUG+e(Wneq$qKOJ4Wy%;Lh*+cBleFI6hw}OP(Uo3 z+>12RnRh#F+*2*3VV1W_1PaHFVkJ?I38jM44tv5w|6F&JBl9kJQk8yY{nr&{FqLS5 z+JKe3P7!KyM0Iv}C0N2~%`V4V5&!Z<$S@-VH3D}F8+@;OVI445Wb zM^}sNOch37vaS{nu9L>I1u92-XeL^yT^C85l;DAsGF`$@o@P{-tp71Dhy-^Ru-*M3 z0+7*HpTzv7$45V^z0cu4n)yk*w|Bkm%Sjy5H5EqtS23-Hd1o1?z+8hsSPGbB1BN+3 za%3QKjaPzcOBMdFgzt1Oh`kGv(>@IufUrMdG2ia3`z>Ba|5cT7w>CFe>G)WP3No~O zh-i#exqOuAGxs87ZUazQJn&{bZ8h!LF;)ZbGxuWgPuY6QBo#9U}BlWf@By&O7 z;hM={!L;!PFRc^$Z5%54gI=n-LGKFN@4b;^;{84_Pw)%sz7?e+?kC)I(I~X5q4Hq5 z4cF>E^=&X>ShJ;h!!&dH-oxU63BTfM-pV#}^e+0QB0q-P&vaI!|M>W&fUwzlRSy6+ z$SRJ=iixz3#ZbP&eg%3vwIxR{HGO$)amN7JE=8hT_d;U=)jN{*>V?UFujzjCGTDrI z*Gt2#NRYbBnNi6_*h&#Yue@fev|l(qE24Y+H>78f$?Llk+0?!h&ykZ|o%aqY`s93~ zNODTTy(^L=slM}kjXle_$Qk|6OjiVpv%v5d6fJ<1;*NLk5&;wtza@%QN5jewxqRPE z0({IsV%-C&BI@RF Date: Fri, 16 May 2025 12:10:34 +0000 Subject: [PATCH 559/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10938 -> 10939 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3ba90f5bd..8e2649544 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-16 12:06:19,986 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-16 12:06:19,987 submission_checker1.py:3308 INFO] --- -[2025-05-16 12:06:19,987 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-16 12:10:29,158 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-16 12:10:29,158 submission_checker1.py:3308 INFO] --- +[2025-05-16 12:10:29,158 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-16 12:06:19,987 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-16 12:10:29,158 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-16 12:06:19,987 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-16 12:10:29,158 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-16 12:06:19,987 submission_checker1.py:3324 INFO] --- -[2025-05-16 12:06:19,987 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-16 12:06:19,987 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-16 12:06:19,987 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-16 12:06:19,987 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-16 12:06:19,987 submission_checker1.py:3345 INFO] --- -[2025-05-16 12:06:19,987 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-16 12:06:21,030 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-16 12:10:29,158 submission_checker1.py:3324 INFO] --- +[2025-05-16 12:10:29,159 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-16 12:10:29,159 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-16 12:10:29,159 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-16 12:10:29,159 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-16 12:10:29,159 submission_checker1.py:3345 INFO] --- +[2025-05-16 12:10:29,159 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-16 12:10:30,257 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1107 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0806 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0345 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3fe2180a0..03183b0af 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1186 +1187 diff --git a/summary.csv b/summary.csv index 058ebecb2..bb186a6d7 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1107,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0806,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0345,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 860bea9c2..8cb820363 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.1107, + "Result":21.0806, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f6c997359dfd4b36a354c9b5f80a59b9909a8bf4..c5720373e37a566580bda75ce06542cdb47e0441 100644 GIT binary patch delta 622 zcmV-!0+Ic?Rl8NNatH+iuce@Mvv&x(41de=E%|2)+J!H7n?{S9=ro#4lExs;F4wwp z+q(Y7yGwQWVg2QB5Akfgcs3m8f62Z-S$~yXI-cD$l0Vm9lII?t{;sXOv%%J@=LP);=+@^0J9Dj*bW5rp0nJO z>Le?F%ORl^wgYX+sewXMT6z=G#u2rCkkmSU`zlW2B;ZqztC{_Fc2T+9t2X%s=cUmV zB3X_S(A)^An+m;eR_Q%Td|+Cz)@Z0u4<0S+yOLYVje}QbEVv-SClRDQ<+eh*5G=*m z^BpMWv*^%K+#07?h^1?=W&E2pfQvj|U(FrN|h_pYh9RPx3XhDGv z9+QmVi#LE%ezIXuuAEAS-om8yEROEIm!q-kx~!WG)|j8*&-KgOuxcvx;Zz(@T?$UQ z0~U;{OMG3#!?nzZbv*G_f_UtsKJj*1*_?pgVmJw0z$s4ntip62tEEHo13Uh-c%)v*9@ZOZNTA`m5~H@$9CN{JH*;JooVQcWvdJ4MvyCNz_Qc z%%kIG^m(4$L-}Hxjp66f`s?u;^Tjw`jMFdc(ev-;TTc2X(|GvfYEsMD_|IkZ<6aMg zAKb$q2JTNkH}p@9<3_gh`)Ibh)hPK7Y+5Hr^ZreA-e25|XGvocjk1MWEw9;FtiNz? z^>Z4}S3fz;#>Y5KE%M*8fA2fiavv$^7WBwd}tzX~!Ra2?EQ*l6bDLCa0 zSTL?G@pTao*DxQ}@x)sR;^`RmiML^qQy+*V2B4?7XXSJD{IXf0x*)|oCG#|2Z1Nd` zk_9cucQ^8;{{mFQkh_Uzz$y3bC#e4cvn?jd0tNs1b9r=={3u%i4wG0ZJOP-Kl_^01 zg_GAQIsx~S5-KJe^ZeL|B>?~cPyzq|5dZ)H0000000000fPvVPQz||IER&KdBL?0l H00000c9Am> diff --git a/summary_results.json b/summary_results.json index 9400fe16b..fdc6759b5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.1107, + "Performance_Result": 21.0806, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 9bd4086f7eb25bc44ed118be39c33537b27c32c1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 17 May 2025 00:25:46 +0000 Subject: [PATCH 560/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10939 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 8e2649544..c75fa5048 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-16 12:10:29,158 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-16 12:10:29,158 submission_checker1.py:3308 INFO] --- -[2025-05-16 12:10:29,158 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 00:25:41,533 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 00:25:41,533 submission_checker1.py:3308 INFO] --- +[2025-05-17 00:25:41,533 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-16 12:10:29,158 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 00:25:41,533 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-16 12:10:29,158 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 00:25:41,533 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-16 12:10:29,158 submission_checker1.py:3324 INFO] --- -[2025-05-16 12:10:29,159 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-16 12:10:29,159 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-16 12:10:29,159 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-16 12:10:29,159 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-16 12:10:29,159 submission_checker1.py:3345 INFO] --- -[2025-05-16 12:10:29,159 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-16 12:10:30,257 module.py:5271 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 00:25:41,533 submission_checker1.py:3324 INFO] --- +[2025-05-17 00:25:41,533 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 00:25:41,533 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 00:25:41,533 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 00:25:41,533 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 00:25:41,533 submission_checker1.py:3345 INFO] --- +[2025-05-17 00:25:41,533 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 00:25:42,562 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75204 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74249 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.4194 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 03183b0af..c3e05c83c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1187 +1188 diff --git a/summary.csv b/summary.csv index bb186a6d7..02e130339 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75204,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74249,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.4194,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 8cb820363..0f3e8dd84 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.75204, + "Result":1.74249, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c5720373e37a566580bda75ce06542cdb47e0441..f91e7c72c1ecc5ce103866166495db5fcee60edb 100644 GIT binary patch delta 3864 zcmYLMdo&a7{~tnfTO*QcqirrtwANH;L@~3F$R$LkEt768Nw!d?g=`F4B8)`tx#U`r zSaQGL3XzZyxeVX=oZmUW=a1))=Q+>oyk76u`~5oSsjz)#n}2(!=qP$>&t0bfPBf+3 zphPdCBj`$EnIGfp{c#ltIfJUcQ*HHM9N_BwU}^UgwMgRq_I}>RO!&8^BRAcmf>-sG zbbpVoy2n~Xb6yjUE6*1vOP%ZBj7_ZjYs-dsUX5AK2pv9=_^F{;dp>oQ*SHmR8n%d0 zD82O@T_}@5ADi%If0Y@SJ-dZ|@mGQJ#zVof$mwxoP&=;3by9b>o_z41Saxb-*&8jM zmt|lY>CVu{@0I4@#+xU8HE1(7Z-Xp5eAN~M2607G+BZy#&MuA3)uMg~D*ZYuXV^Uv*OJyCTg@E)NMFT`I+? zj{d5K?Y`x!y?r!dRyQVlA{z5e|7UHS?1`rjTC}m_xr3tQG0wFBVyUv(HY9x=nm!_H5OxEFl9s z4AyV??sxQ5I5d(H<17_eo=?)kaX^Ev`zT(ntWwY0Or zD-M@O%C)kSzK#X>ez`uGnYDcH?z^ny2Ko%Ji0E_vnE1epymd-ZpAn@*-z=k}W$_29 zIm$+y)9H&zlrhz9PUeK3n2r2IpRhEYRq4y8WM9Sh6&(MXQL^URSee_scwyT39M76H zTk%{k`Of4a|Izo$`wu-#_@r@;st1Z0jG>K3nbf7t#rf1m#Y$cu+1sRNpF3`M*DgnK zYrm=e9PRME-#9o08=0`+gw)bP=StR`gZtdbyT8HLXf|4w@*L`Vk3Tm)gJXYLFYWPv z_v39&=;j4drZC+X!4;;@Z3xrPw=NXhhPc;SO99W$0|KLB3}2M%2&*$_CNU<^Cs0*u zDJgZv?TFNum7zaq?pF+VPYcx>rkY)^%h+I9;OGSB9b1%x^3(R_qVkv>dIPDm#1V~? zQIDV310Ce$4I0cK2R$(UUxBSU!L_$>aTA_;jq0fu2 z5Wg6zt6#>nJom5iVL6Ir*t}$f%%>__*<_J9r)2!Jy^H8h?T^A?Zn#HA-=q56tQ_`# zu&qe38l;rAVBx^%K^umUjKjH*&q#&|!+L?syfgq51PR0qwba@-Tvor^QT5c5-kO$| zIk={AuP7gf9!I{{U2FT=V8hjX@PwhvXaLIqI!RwyJO4I>jN@NNcsaO<9vSh5mxy)v zoQ%pgciDfDZ~O1sE?0uTvjp5z*fa#|@Tf)%TUDgoqkTVVnC+WW(8MZFOz5v%8Qc|=H-0jG zbk4xQX2maR8jCtAR|dj=fv}+>M21vKgi=Tb`Xx5d5G8GJw?y}JiiVTwsa+`=ym#po zDHl6}^8m}lW=p&^=W3*_3u(xLy#BoXt+QoZe&dMxgN#2LrakdwclaMDC&37f73+m6 zU13&M(8>L9G*?=p%be}Sf1e8;l!)v-ZPzSuhS;joUFQv7jwU$7$9*q}P#HuIQT9?emM7c03qF|-WH<>9 zSc>lmM3%kYE?=PKbv>w@nNPR}uzA;n3=hEmG9-k;FUU*_wV_t^cI`MwjQFWeg`dz* zmny}#AHTuzy{mWmdJ7A3b>?v9iPN3pb z3n3i%$M7X8h0KP6VSAn*BS!S9U-+&cUU)<+)SI~%lkeS~vls;9tYJ<24VHqA*joQ> zEB94oRTU`3rbMu5YPMhj|tt)K$yZz9rG-7o?THnp!{P*?nwJZ=kX|2s?MF86~o zViux4fOuP9(D{lP&@QZ+Xys@K6|paTTE`$jS4R z%h13W-<2UaJHn$25FT`_%TKme%8!Pb*mr|BH4-f5;Mt+%;1F9k9K|5+ZBA~Jgq@-^ zq-)`T^Ky*k8}k$sW*t)u*cjFhP*n;m3=yhB0$|6yio@DH00q_nb0#3&2oS=eows>i zXw^+1>?DW^ne`w5vpTE(D}|J|hjhpS1PNLVXs@FhU&e z#UMoNk=bJTs6Gr?zP25@MfM4*1&xDG;kWk-Vrur?$Dd!3lw65-FVCBZ9zITaK2ts4 z^flj4BA)mxpq;OhXvM}rLoeIuQ0&FZph~XpBoM`3oXTxnZ=-zMFvOXN8ATsxdY3Mx z@7H}iVCJ3YoOl6$LiN5DE=RR+I(2#M?cHw*r_Yq}xHcbPYlmKRZrju^uql2|vB0Jy zSg|sc113l0TRU`v_$)XDrW%@ZSd!pu_c+tuUF^L^iRaYh(IYhx2q&KE;Uo`wcXaog zT8$ueuXqbeX_!Tu#mE-jX0Rq<_7 z`NWH+L0SA~PYcr*jn4>reR8|k8G=b{lAQ*$qLeWEo`MClLoi0YUr`~Hc`0}s!jLR4k(m`5B`pjrHX*aW z6&O3jvl=6rVOPeA|21x{R+ub1DmW0vis}2Ts&Dl_syS1`^NbMlNV6QpNz(+729RyN z0fq<>GUI8`2NDMb=@v>MwNuHIR_b=UP#a`yo$X+yc)#%464G3f=wI)%%Z}Zb2o1$_ zkX6Z5$J}mMgF{7~aABL;JXh32Ab1~8LM3hw3L=OJtk^4*gJF(HO1p=Uw@*+C2m$fy ztGz17G7V!yw!Psa(HT9KC8*EK!4-xu5?;X7z`5!|S>J?k3zVfp7lc)n2v|v47?8J1 z0muWezT`a&&B2Pervc*O(x1hocRH$~@vF9qlUCha!p&!I{!CJ440@>TZc*uhj4c~p zzh;sveKWem(Jck%68km@1Z{)PbGNo#5f+eS?Aak82~BhqF=Iq^nOvzauKM}WQyY7OgG z-W@m4y`($r{Q9Hpw08J?+O;uQ$9wfH+;Y|YTJ}tP+uw|Wl{*`-d&A$S=K7oTre(F` z$I4=Y6YCqAR~8W!;S2R><`s`r;Ty6GMI_N!~fL`eX26jZ=vuutCZo`z%5)bw#4;vz=EO`PGm=soIImLOK6Y2N*V|e|G%7yBds|1C0YLnZTe=3-L0wXZ-LHtT zK+l>a=W@1>Z9G3@vHj=1A4*0N#2aCq-jTzX=zoP^r**XKuc11L;s={KoF`6Ak)fBa zA<7`{)pn;F{3toc0V3*q7p2uFihj0QUslevr*XXsCN8^;xKPE^MuDo4Tm=FM9B@6)U2K@h`KI~=MXr2~AFd3SM K#D>+jZ~g~YZce!X delta 3864 zcmZvfc{r5q+s9{&rG;$iDUxkw5b~sH^k{4m24fFn$r3`7Ee4T$!jqA0%-9+w9wHH% zkwT%6WwH!KLdm}G24TG8dEeuBe}DY$KdwLS>o~sm_j{h7>%5M;=34Ewil9S0T;~0utlYK4XEUYonI&#C{M`7tGVd!{d(c8 z;d6@-G}y}y>vP%+c>BD^5G9_($4m&Tcx*rq9&VGRuJ_`8w=IO_Ak32=jLla?Y9?Jp zX!vPa#}(BNKJnTa_rA!A3tU%J+#PDoTs@z>-wK3ZK>0a@R|*VdmiRa+i#K|IaWU%w z7+bw_-jR2*y)rx6On6tI*u6LBC2#gysY~>-r^X70agpMJy@T(@J}rb@N?biUjN{Kh zi<<*3KMRnOr6%dM#n&>v+w!S=edV+;zS)wAt}ON`H+1e;zN)?O^t8&Tc~Y})tjCY8 zrBfl#L-1JM@HzkRJ6dm8g}+wo@={;Q0rhu7XN&Pkd)@+yboVOK4Cto$+bs-V$m&&2CeLEs@ z?IMD)cAKYGRYz~B#o99@^ujj*cGZsLtSK_qBU&W>4)#lm9 znu9XDT8d*Zvv-BM492o2yE@#`SS^%-?Ow|WedPSlS$iuyE{RtBl5M9}Ub6A-%Wo2M zR)N7f-T9=YdX02>#Wf;X*u`}0xLku|ywReOevqN;D08#@f~#RW&uf4F31?-yASDf@ zz62myuXKs16uyXtqk*KXRYx@y6=d1bas#-(U-?eVu40ESi{ESb>j#X%_^ye#D({>A z(3**=(wGlcGiE#|=Io7mPW+7EIdM6P@zwV5LoGw5;qhk3^j`*%;gcT^L~Top-G zAQSbO>bXYku!j#;&t=FxtOH*y{5SxHYPae<6let+nTiJOk2b~mt$BT50= zeivR{%(`CE8)(y(5c9BpmTmg0WIkE$LsR0RT&$6+&o&5akrG8Wh@fiL8~C#o4n0FnodouP`1oL?-Szq)55#9!(LLuJn4|0n!-sCev7`=lyg5K>Jq+~Ad zcG?+5ASUw~w!aF-{IU;5%-P>aw{uVz9n>lc#H9HVci!LzUVp?FOU*d z-%FBp5lUH?l_B10p?uwlw#%h_`n}^z3PNGI940{2k!2DmFS`Hj?Y`obNL9q<*E!2t z_mpGGC8p1<392LWh4XY=fUN(N8-1_e%Nt4l}ir4sPhcHm!}Czhi@C+ z--D>gNbqQNgFH2@9@vx1gcnpEf~HKnrz+qiq!GN+no(R+=Y0CZoF31zR*0IZBz?P7i6 zQ`TkY^>TAo8P9-dPv^$NRTTt4F=NMX518^HTy2E_BmD|JY5s<5r6*6PW# z7UxMYA+JC);45(;f>$w15N4QIqs}>WKNCKqFIe<4X`#)CVY;g=tM1$66pC?FxWG% zw55IdxXEYfOU-@Cd#Fzn>ZzW$upZASIOcC=%|z%CsrK)%UO-VsbVAt9SQ{#8#A!b1 z^N}7w2IvtK;z4~Z)dwc|SVbWew7W&fx`!U+?vm6{t+e_{2;TrT{pph4WR8ZmUCs3F zxN2+7ee2a3)rE{)G+QVoEsBF?&%bCE9H5Ej?MD2iT?*0ZE!g$8h~jV=ldJe|GqM5r zO0;WlDRz6?0r9k;y^q#vobAQjX5vjkqiLCdahg=pEKxpPifow?9A$KfSvs;?!Jo?+ z7lr6;H%Sl`K4uuR!oji@M}(i$F5nNv^jzeCE}{LCuS4@%R9Ud)c3UV$RpH(X?C7p{ zr6Fo&my}2=Sl7GK>cuk?=Lrxnh4T%!=3`GCC^g=rWa(D; zbLtnV`Plrgp2gzdcZw5c&DmA)toG|UD(3;jS!fOdFfzbe?a9b^(4&8I`T-XWC15DO zIkX@#lPSDwd=6JMFs`DXrK}!E+!R|g?8mVqrI4Fn_nAHWfMt&3YH;tl6##b}jhfj% zFbC4V0+MGKUBViCY=d(Le20UHNwU8XA>FTaLD3l`j3^4PFvnD@4kQcV9<#KzwSmmm z>kQcv++(~Yq+Q*tcG+$^bU>L&z^$WE#o$zqB%l64UO}QK0YgqTzjT|z+oSy_dI&9l%X`uCV!WztQRo|T;XR#D_mZRe`^9_~-jb^wZ^~fSj9B`LZp!;9aSM(1R2%bEz!&TIZ0#0vpJ}VK9MYCR!xwR*5&uz3 zq6ZI{=m|m>_JPQ72r-Gvh{`W5!Oxiq1BeO0j`3Izo00)0i}6cGbP41NkUvmJLv|}I zKZF4r#;VHp&7P)*67q&v^`*8~mLus_wBglxI(O#}$8;NFEHPfr1D zb=p`U-B}}N7(v$VcOg%m$zhkhd`JH<9rRbc_)KPa@KB;w7d=A31xG)>i*f)ouLC#g zPPQZj{`SI-nT1LNzKwqh#V*}Dl3%OBkjWFP zg4}K^zJJBq;jU=UC*jvoq%Rs#J7>*Cp(EBQC6D&-c+2Xc667||)H9}-EH+uBd+1Pc z_L2AgwlkW>k**FEecsP41kb3<)sh|ClVa>{D!-NfATL~?IZ<_`R!F(|+nS`pBz?Nu za!9<$p7`um1>m`xeUXrWa&hIkfB3{nEs*tP5$0jVEn=m1nItIshPUqhV`nIdBMIsR z>nNI=McHHH2R}X*Y$zu$zR%2mHCI!bEzp Date: Sat, 17 May 2025 03:15:58 +0000 Subject: [PATCH 561/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10938 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index c75fa5048..129d21a9c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 00:25:41,533 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 00:25:41,533 submission_checker1.py:3308 INFO] --- -[2025-05-17 00:25:41,533 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 03:15:53,225 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 03:15:53,225 submission_checker1.py:3308 INFO] --- +[2025-05-17 03:15:53,225 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 00:25:41,533 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 03:15:53,225 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 00:25:41,533 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 03:15:53,225 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 00:25:41,533 submission_checker1.py:3324 INFO] --- -[2025-05-17 00:25:41,533 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 00:25:41,533 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 00:25:41,533 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 00:25:41,533 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 00:25:41,533 submission_checker1.py:3345 INFO] --- -[2025-05-17 00:25:41,533 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 00:25:42,562 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 03:15:53,225 submission_checker1.py:3324 INFO] --- +[2025-05-17 03:15:53,226 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 03:15:53,226 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 03:15:53,226 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 03:15:53,226 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 03:15:53,226 submission_checker1.py:3345 INFO] --- +[2025-05-17 03:15:53,226 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 03:15:54,316 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0806 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0345 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.8766 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.19844 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c3e05c83c..ccdf6eb32 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1188 +1189 diff --git a/summary.csv b/summary.csv index 02e130339..a2840e68d 100644 --- a/summary.csv +++ b/summary.csv @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0806,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0345,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.8766,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.19844,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0f3e8dd84..8f405d2f4 100644 --- a/summary.json +++ b/summary.json @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0345, + "Result":22.8766, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f91e7c72c1ecc5ce103866166495db5fcee60edb..4dff8c9744eaca87942ebb29ff952bffe8a35997 100644 GIT binary patch delta 3632 zcmXAsc{J4R8^#Bb%D%p|j2L6dmNW<<>ljm(k|Ap|vW!>Cmf}lPmMqO=>|3__nHYNr z$yjEPwK9@r1|`Y9jP*Bq|9Sp+&UrrPzOU=P?sJ}o8x1!|_qdM)(r26SLENqYF^DrY zSWtXmKs8s6(cz@)@*FqkZ(Bq3kjdG=w|a)&AA5B0XU4D}t28(s(0@{HXzLwQ)>#A4 zW=;&0p!YlJu*4zLFRcOoWABkeG`Gg4pA#MU@mDE5bgzvrwe)MDUK{hf#Z76wjB(c+ z8WMFim(T?6W}J$9x&1Ap@uYz@6yx-XMU?SwF84rY2M-+9dpz|bybPyNsP@zCA; zw6Ch-uaz?fdVMppVVbIQa*Xz0EeRBPozw1prmY(n_>w&QqK(%UAlcklf?(W_m3p`mde#QWjb_4&=g;O|w*~852%NLbt>4Ou`?_>r zNU(25AvQ*Mf{5s^y8A75b2Omq%>W0?Wr;O&0;bT6SDtF?NvK1!Q43O zJL}b*I)pU+ceAXZe~{|o)r2{jp{kYCGO7FXLyD@_^DR0#$DJ&;8fGZ%-pt~rEh3)F zBi;)*f4|e^LjF1};I87wGn2j3`RC^A3p9ZjY!;AD7;EwP>5NsvTB*q_-EF!J|Mj3{ zYh68bNN>m5zvi?9dM7{AI<#?jA2`Ig7L94y^>1pk@3*}we7lzF|(zM%}u+W&LlY9S%OfbED zVOHwl(W*bcCZRI-He`{rd*9}DzOOiZF$CUJ+eWru zDz=Gi-wcm5k@I{os%UtAQl2a3RS272r@7STt$)}=;Kr}arG-p{UdVHCiNNcP2du_h z?-`$0%Zpi$aF?=%Ty!4@M?5WRuU2^s@W%yO7O}^HA7^6*f~tYVmsm5SPwp{F$K!Ht zLj+VSZJ&eUA<`&_AVu2rrL1#C`4x`mh0e{q4{yI-jAl<^IaMLkwf9=-7fzb_iJ|Oq zin%XK6n_UV&sE~B}b~sM8la>xe$e z`;tx~P!ibcGVz&qa+FpMMwP;pZf>@HE9EHEBH8WWVISEYoEiDHbo<0Lvl4TDNe zX?1{KA=!&TNlC6^cGN6O-*h)&@1w3%u#g116JFXQ$Lc*RB0?C9Ckhmi{l}|)jvR)0phf3Bo=oj8EM3y zy%5EZr4nM!$R=Z{l`7}jFv3Rfu%id^u+R9Q>cF0g!AdXj)Lznz_E zvJXHRe}QwqRWc(B_9`(|WD{$7@pt)X;R}XRG%`=+VR;l#rez)*%t<)2r<@J;AB=Pe zdudgv(A_}fNk#HZRnG0VjXa_fY7@2X9AMw+m2!=W$Z)ejDLUkZf zo_1=%_FVnwKYV#!xL|w!m}1I=wbYUU%OG~o$8AQ`yEKRsqY_opvD(x>cC*}UAx`g7 zHBmkP>Oz{{7mm!=PeN=K&Zb*6=s}{ecBF^)Tq6+06O~@-kL>xois38PHm%xGgqYJN zidn!9t1m?u5>|+Dl+c?Im{X3n)%QX)mgMvmDaKvCRQKm&B|j`mNW)rqx$u{fdVWBd zIoANub%(^0M|pm7%XJ97+kXax#IF?;;_h=P3u+%v)3f@Fxk2*AEi?Xys`|dlo&Ir^ z87jXyyj5@0NKImFC%UZ6@QD(vVijZMiU2U~SkbzXx;ch(@fjw{8)?hQGLI@IgsZ1E9+j|egb-mbL zSjsC);nl24DeUfMFD3-e2lfw0u^A%BPZe5>0{Ada4$wNa3MF*B zq4UoS<-D0rY-+>UU4=BuMQCXPwmIsC8v@Wvsp%=B^4W>=N_}2A?X;1C|0W=%S^#

-`Gm!UwTZYzsR7@u@kS*7@mq*_OUU+OKqq97RfEgw+? z-#-k4Y?ppO+TmHq6GS5VjI@eDmsH#Dx%7gYeD<|y_ZcY*ZVN&x15(R{OqS#tfH_w^ z(RH81a~H1OOdf>nllVXg{ANMxP9!1b)DgX)Lv@x$P+uo2<};eWM51X9P?VRPV|94-znoX65U56UgDm=$E*(Di41GmmFPR62 zGTe5Pkiw%QEb*&tmq6-BY7*DXQ3vUJ0#~5&0IOTg%HJ-lkqOvdYTxGB``5B_&p8Ez zubilK{z-7!S+?kh90(-63^#|q9OS1+Wr!An34c?np{-BkFc>P_^|T7lla)SdXe&l# zB}yDg=WKn2s#;mC@w^OAd5giNi1of&Em$A~eXQ)(m!&3|29i^${1kb_C4C4G?P+(H z)o?#V;t<7>5Q$$dXx(BpoDDM;eh#*IS_++$yxE(Pkh>T)o1>_Iv*GpbbZIe|(EM4+ zKU`x}ra2(o+!STAefBK6>hW3|EA_TPlcPlVwwoE77V=XP*AG^+mn`2l4tFy)6{KV`ucSb= zF*a?*PW*?J%;B1Q@ph^2VB}#LTT$C7_&VgEgiwEDv_)(W>|68EfR)o35h?y45raWA zh~^6VyD0qdBBD}&BQzzXwxNldl;AI5XO&KWnLYLRq|D_Xs12SOyf2I^_u2e6 z#zl`+Yjtpu%NMi0#lE^zn^&Qf$i7*og{;lbUJAkf{X`JBh7I$5$3%^`aB za~>zTW~d5J^d-}3a@r|jciq*4oKgdruDr|(yEWlqV7r7Tzpk}|KFi*TM&&twYao9$ z(Yl4mc6lbecx0*McK@C64wI{49|{B_vgkP}eY;FdmCTiOX(thX2QsY<+6%W!mY&#< n@!M84(B@-mF7s&3H18c*uQ~_qX8-#utlE~>WBX4#9Yx^(_CE`j delta 3629 zcmX9>c{mgPA9sY@b3{^Y*oK-&GZh*+XBHAULS!()64s?Yq#(iv9uO-5-fhE)u4Nb}w53yU~S?^Xe;&_|t#6vr<}0 zKPk0&m+e>Tk@rskC~v4fMn} z*H6z~@bTX*UxAOv8)oGrUr(8m>N+9<6@iqQC}x~`=yZCl;mDh(WwfToiu1&<%GryR zCRZvW4b@9L;U;lKOZ(^Son3Ui{|z z6N6LkiT2c>@m;&|w;i7*y?$i)f*N-jk7)r#z|%VNw1oJKNJ=DyQdgVJCavTCKgIRM#+X` z+$4Zy{R1$vQzCy}>NPM{-w;a=zN@4(_C*-!-5O(6-7G=NPAyhJMDF`2A3jYOHB1X% zj6zOo{;Y`=zW923s_vb0gT&HP9a-l}7OwRw`n3XD6W}m*_iHsVw*}y=5K@oH@8C zPuX&L`_GS@=u?p7x)$iKMH8BS?CVU*?eULxP^JRqQ}u(kZEj7JDP<*opZ53p=02a1 zx%K$bm&~MQ(lVgX&HLH~o{@LCdxb+fgknvj^q!85S$KP-1rNQ~2bmybs7`24qK95w ziD)9tnHsLh4CN8B?x2S9)xM_}?>e?r3I4|cnRl9Xk?pb`3 zqCv2RX!K|_aVAo)A$2v@yCyP*-+khEtA=&XjMXEnY{`R@hSsaZvyXqFp{R)Xbvsoh zB}5K?wE@)IEBDDireeF!mH@$ZtgjYhFtd!NecLM;@cT0RIXiU!Ixd5Qfl*O&_c^w-z|BBQn6m8ozjkcoQHdf-=JS#tpa1@eRNl=l^(vqb0H+9FHr zjoK+j?hWbxs3vF<*8a%BY4Ig%V=i&z5si_Q6}Jh+OOY?%HUezK#R*!?MxfI!8OPZr zP$`|6M|-XqR~)Z#rzLKy~me>L{YE zj$=#YTivw;O6DhO00+${JHM4$mJmN&u|3Kcm^*VRZE96Z%VOI%atQ@LFIFmw`39na zIo-$t$>CBV>4^8J0ByLS)}vzeE6Iv>vX}Xi72Cd~2{>3`?MJA376&|?*>@r=9dKhN z_`SE?p9%J+v3V^M3QyC2|1%tj!#hKNgX!@)2o(1qSn4*VqMU>uh9XeHU(?@*ecx*0 zL158(hP+^UL3Uf~y^#ih7kvx)3bUCe=6T8^yU266(ena+hibR}AZluL8ELWovP{|t z-ZJv5N>A(p5!%9b*sAl(MoHupS7!{?{`$)i=)#E-%sevK z^*_MNZq@h^*nM;HRxU4vp#~Fx5#oF(o-B}uQmuiCxdmee8lW2arVCt>`7Blh= za{;vT%?Y)!(Y}qb!!heZOYB|n9Sy!Cr`^JTDg;9>YNiR4V7k@3=sxb66a4K@Hv@We z22*w0&WJ7cMh@N4$_uw8%J-cn^D;(sRlmA&m4{)2F)0NgI>=h?h1*-&mT{J~IjFg)a@Tg>4mlUTj`^kc zqZZTi#w%x#kxLpuEnylrtgk{og5Sji89`(EgYi|pU%7?j-P!2z7 zO3Aa3=H-|#qxTf!O;({OD8lzEB14CPAjjKq#&qo{gnMyG-VV3QZ{EZrm&^y zu!T^G*DO93nDt|^N3mS&MguUm6_p*o?Sjv+JW;Ctp5cg6TM62Ma#jRLz>C*(gAefD zfirl$gbSG-b2NQS&GLUD`E+L4#iyxw?SrePIiO;xxEQWhRB8{ zpXJBeTfNM%cIK{CEOuR(KYy+|9A?)hdp6O9)F0LVsYWqS!6VM3W z?LWawaiYpc5$&|tq+IgdI+c;pw?jrqO2vfTYB>VDLEB{_rc8-8#n)o2QR zaM&9cNR;EEjX_xoxUsPYMZgUOic1bmzLJJaEm|CDizW&-UPCWT_U>%MQ2lcZj zsxQ%u;XL7d-yQgbKlyq1h--=Tx>h zsNh#_!K1da2+WS9D& z(m)Cv1N~b))>NsZ*+FRAuat(O0uInYsA9)k{_B#3LK1^QN6@A1rzneH5aC+A3gG#w zrvA(mXL(3)HaM1TQx5`ur9BFJLbq)|76{@dPOIa3tQNzk16->U?aOH1mT}CGjqaZ( zW~Fp^-MpUndPa=jB>iEBTv1W7z6 Date: Sat, 17 May 2025 14:03:58 +0000 Subject: [PATCH 562/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 8 ++++---- summary.xlsx | Bin 10938 -> 10934 bytes summary_results.json | 8 ++++---- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 129d21a9c..ad89a181e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 03:15:53,225 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 03:15:53,225 submission_checker1.py:3308 INFO] --- -[2025-05-17 03:15:53,225 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 14:03:53,366 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 14:03:53,366 submission_checker1.py:3308 INFO] --- +[2025-05-17 14:03:53,366 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 03:15:53,225 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 14:03:53,366 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 03:15:53,225 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 14:03:53,366 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 03:15:53,225 submission_checker1.py:3324 INFO] --- -[2025-05-17 03:15:53,226 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 03:15:53,226 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 03:15:53,226 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 03:15:53,226 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 03:15:53,226 submission_checker1.py:3345 INFO] --- -[2025-05-17 03:15:53,226 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 03:15:54,316 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 14:03:53,366 submission_checker1.py:3324 INFO] --- +[2025-05-17 14:03:53,367 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 14:03:53,367 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 14:03:53,367 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 14:03:53,367 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 14:03:53,367 submission_checker1.py:3345 INFO] --- +[2025-05-17 14:03:53,367 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 14:03:54,359 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74249 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.4194 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0731 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2224 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -46,10 +46,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0806 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8658 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.8766 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.19844 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62328 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index ccdf6eb32..0b2a2c0c5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1189 +1190 diff --git a/summary.csv b/summary.csv index a2840e68d..863333c81 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74249,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.4194,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0731,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2224,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -24,10 +24,10 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0806,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8658,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.8766,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.19844,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62328,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8f405d2f4..9544b510a 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.4194, + "Result":20.7698, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0731, + "Result":23.2224, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0806, + "Result":20.8658, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.19844, + "Result":9.62328, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 4dff8c9744eaca87942ebb29ff952bffe8a35997..c238ab7861eb3bb54c78807e985a8b4559a800ad 100644 GIT binary patch delta 3795 zcmZXXdpMK-|Hn5&2xC5Oo)vUp<kaH3uq~(}7{#O6|zSs4=ulu^M>%Q;ne%_DQ^M1ac*L7E)=bSISE+zp%@3ruk z0)a#W=#gU90PN%=jx9`7iA5Pq!-(=->P6hat0=n5r!$a)XtA&TM(HD*?ca|)t zNyZmMZ^CB`Pz%NvgN^RB*6QX*hnr?IJLmecXnJ?#8O15L`OYsexfC519zJ?) zQiZx!llBV%`)+NPYFkD9uG>G=UhxQmWZV@u>?*O5x>#rVF#U+C4fg zR+v^*h;r*7Kt#23k|s9n+KC$BV>Wb z9vzr0rWPf;AQo;3AMXyewHg7g{PSUa)*1nS{zLFo$uLqjAow#ap{iQ`_a#5X_`UGf)2D@lUKXA zMe5E=S^_lqM2#KqJzeqr1vC5erZqTgCU<+jo47F&y^SWppX^ypzz4$8^ptvM zctOC>=FF3$2&GlOU|lUN=MBG?v7>Xp|D*RM@8GwTBW>)c%)sfuACZ$MF#~xX2$`P4 zEn@yQaGEJBZ6qz>#&YeV)Ii#G^LfFnyV^5G5<59_E%zqG=2QZZT~S;_vFzDsI~KF$ zP4Jigx04<7!7p4f%p-4s{Klu%CRJSOgD5U7Y-e@-WaVo{@OxLy`Ezb%Da>NF-s@b=Uz`bnTf1pG8YmI|lZ{sn;~7CJwsX!6{SJpSjOQ_bNx z!&FV0VwqPxvZvdfDBoDWtHp$I2HXN|?Bq~PheD616zsiXXtf6rIo4$eOV~B2WgT-2 zmZnQfAj$4)gF!nkJOW#vG04ADYuFoVakK5dPr`uMB1Zdeb)l`<(v7<(Q0>`4}%Dq(DOF&=9RU((O{Ng#QO&!1kYjS{c=6tM8fB{zHJ`Quo9xs|e zQlKK*ZBck@t~_x1lCMS5@u9fp?q16)ffJ%9DAaG7?i$#5w!Rkqg;^chX_)}{iWBS0x&i>}=SoO_ zhMPOeACWZ~908(}KqqM6{f2QhUr|<#h@v)ES^SN4#+OPF~vo+?+LvcsNhu=)%vD}_GmgrSSqYEE_iRaZ6<4kNr1er_dq zAOXb%Wg{RI_d5@Z2^5;D48anb+mQ^Vv@sG=geU@StvGo{Vm}A@p?3#tJm=_<(eKVzcmTO&~fqB8AAF zl)B;in5tkUut6YfY-ssD=)@mR;D11yeCa{S!>lDWTzSethaq(CC*K49FI_pl3`Z@A zgOEe%KJF+nL{66ZA3Oy&da-c>>#95bKDvSRD86&*%`PZCpG_rb9v++78NoHx4!?C_?AJ zAB^u~B+jY+PDBi908=PgN$5U!Ehkvw=;z%XQ3oS?pN|(rI*PI9f`LP0&E9 z(ZrcwEsfrKigd-3V+imH^&C`x{7Q^Zc90s9{NMnEI-q%G3L}V@UYswz3c!~7_UW!h zB6HU9HbX}bM(cm%EEYUt@NS!nXA5@Cq#I|}_ufC{ghj+r9DYVSioPfPka}I_bseBU zV_5i|(`P!5#4!?*gi@gAf{A~B&n9hGUG4kM$5^oTN7t#g9SsJa_deg8sH&UD+{})b zuzO-v+)Q7Ga&L9->{TcP*s!mQQbDQaVWCU_TaD@gM8NtxqSB6kP0`{{R zXd=$%FMbW2C2gXJP!~*Y{MAn`*OHjY;Ckrp9(1S4S8I9hrYsm@nfiUs5~%VT?xOm| z2X5yFx?SzbdL<>-V?Pdyrh4;J^f$iJM`YB2)1c#YP@y>}mXUDs?D6tbePps2Sp;m` zuM3*r&kxNiuh1=pF@{&~ah9e#s(oJcA{`%FkK;GuNfur^sgmx*#_h`jO5(v7CU^UC z0f>R(3l=?up}zCpOGt1`H9FkLe*Q(crXmK=Vr+PCe{)%CBc>QJz%$UN2(4RmYsZiC5M7&Bqk*8SB^r0ze8)@}bvt0iYm#{0|Q|{{Wu-19*rq zF4D*%mM<<QblSR4bSHtE;stF-6=O;8b+|6%MmUXj*$qUP_mIz&|v-tZLI z*WbiX-#4H4FE%s$;uiN8!ES?W(8b8YD*g>~&Bh`Mbc?rBW!OaXt;>Rbp_(?Z@LPhg z;A}LycqgqME+)ey&dTVIqkgQ#^<))Gr0?c*&xBpt8G#NP29OSNmqZJ>FDdd9O3qUj! zSK~)NFtE8qk#}gGHDHg_;r4>EY}4A>B_JLIF)8ozOXL8@tzx) zlS*5Ndgqch!rW{*PygeW_>Zm4N1K-xXftl-HVGS}H&z~n3{t{-Ys;sWtLS0PpWaC?5OQ5ZS0i!h__BLl=-QzYvxAzyBl$Qb6$q| zN%0SCz6uMQyO4a_Brg&CuSBO zemrSf4qyrT1uYCXot~~`4!%k6(=q`s(Pgz!O23UofNBvo4xci+Gb!) OdYbk=(IJiBcm5B!zAI1w delta 3821 zcmYk9c{o&m|HfyCRF*;b(lTO%WF%(xigvpTa?+?3wK9s{mj)|RfT!G+Hv1GN3i?&xM;nUkJx;jegV<`D^{(~KJt}0uyw15RvlJFKdbe2kB2bs(wK?G({9B}8 zeen5wpnM*q1ICfxo$MKZ?K9brz`B<`ymcx-kmV6@_JRiKVmZ6N$3HU%p`$Ua$Zq@f zHVL3B>7_L=)0YjQIO0uda49^?8TZmZ zc-TI9(nkj{Y<|o}qwhoxeH3tC9vGwd>vM+E%af_;D~mTQj``-~kfX*nT*k5qubO;b zzX|ZhE_L>feT|PCYVOh-gOy?Fddzt@1aMbLySi!tzrTo+ul=}XeW(Cn(AsTy+zFE=O z8oGqHJ7y0>y1u#+UE&~r+R?kWX@*C zpDQm;Ga*mF*02vaJE_Ep%q4)lTxveW@|b8PecpY$zM>T|Xs~&!=9n{f^I4W%MB~=* zFGbQSHhVjiI2&A<=e2M*%pg;k9bS7qv3xM>iJJx)s#sv&%)wm!RHS}F%Qvf-CP1yeRqfv7F>|peO8oqXk@aT%wqq@~%}#0N z$E`$r%G$xsc=ku39u~D#tKS1e5`t}tcoLxZa&Y}2)pJkrmZl#7&p6eC3Axu{5RFQQ z$Dl-*90n#zm$P`P;Fei_p7(K~TeHxe>o4Z+^Q3Z(O^svPoeSf(z-&Vh_7YT=yk2(;+ z!WfHtomxaQq8+d}Kri)=tFXG0qL~@-6Av9_38;x0Bsjh8eMUhsCeC!!QT)_)WLyu! zsZd)(%A>O>vAVf94LV1zx!K{Btc%!NnJ#B9rm8gEB*m;o!D7%{7aua-e$z)+Q1vAlj2Pz ze2t8UdqL1dur`vvk0xj`c_G0azXk&8HN>_84iN~>oT`dvlj?1D1? z0;hkgXT=n3SK_K@=5|U_ub*MXPn*axX@Zq|pGOB}+2n%}uHsqUQ`<`A zt_F%=8d~ss<@8SL@LdMEmaE$t@s(9MUHhg_&9sUJjM%-14YU7pBEvXB4;Jg~s2S?S z--rDpobN*nbrOjyrhl`Ooj2wN#N&Oh)s%6Q33FvvVoKVVn)*hslv~aw7@Vu7Xg#|y zn{M!lH|xcNF#FjP8MX}uuvok!^^Oz&Fiho8rH|HKCy~x#N>>swD zif~i_FUHGoeS}rIg2J)+ko;{duVEYkwnd(biVE+eG^>s-gLVx>zVc`lbSZNJ%C{;~ zZT4Gf{sF(z3{i_l3`%nPlf`=}p=cMTx;6xEu>Y=34uL#XBag70e20+wKHw)dN@}u1 zuqwvFlddI(BHAr}u`92p^LSc5`35-^+}AG)HbK!ID!(-i62`eY!|N(kv56*bDnB#T z@+Uj+X$_+{mD6qJ;H63U=Ge;~D8L}Krn`(G>?kE9`*HD@>uM_L3q%$u06m3+tztp@ z)I^UJU&1}3bTJ@|gFGo^7|@8v;@mvWFXNhmYT_!P)`c99c9rn4VIOM9O5bNU#YSls zN;FYDIc%iL>!el6Pt|4`{Z@)DLhkIXi9)AQQ2O4zP%wl_@f&WDgwJcV-FEMRHu>%7 zG6UJEvmUcz>OjBjLLrBnhBjcuUr)KVLlwM<)M};;z;>v@AQWk>pk*_L9Cvh|LCBtQ z^TU|W;}tWRO;FlmR^sm(n~H?avcwU>=YbfP^@HEl(=HrH`$-71-{g*ruMsTvs$*Z2 z6aQS`&^%eoPLBoue_ZiNW_?Icu32Vu+cW8rPgyGEi{9`Ke^vo?l$B+T#2hz zr}DS^QcM!Qhta!sBJg5P-bvS>sKrB-Za>Mcn+w)`uw4k@X_OWG>3|4bHWQF2gp&U* zu=ZIa3a{~C;g+{;RK9}TeiH{t26x${&@8@|=a{O+r5bNTWa=v%F;%kX`BK3wIplq1 zmyrU4U=d79V~Eg|Q0I(b_q`o&as~Inr1wZ%2hpU3f|hlz;2eaR_+zO3!&3OP%$1(Z zq`bMa8g45&7nzC@Tw${lX(j~vx}{6h)eXYJB<{gC;J;_k57CNAv;svDYg zFkYhvO0*+i0;~j>lzeW$R66b0wZXz2Y5^z<4qvj2TePus@bcC z93vpD)2$0us+t{Q$IB=m*l?qV+WZz1>a@tCrO80JE=3r_?&BTjqxYIbn)7fF;bZ*Q zcQ7-byHIArJjeAe1!GdLBADVlUBykIl6`d&Our2GR6t@--P)L$;8X9MhQBc@cIYot za*yfDWnOaZ<1$~sr1J-`;0obrYZ6~|n0v%VUx#NM27%usRx$NfDBP zRGZ<`7agVkS9VWc&G+sO-zftZwH`&Tz;;W>^_NH9N^V2_Yu+2DU42@UBL?e-EfnpV zYB7jBkOL=(68mJyU|5DTuT`d?Y{+5b0W64l4fMB20@MQciiF?FQ8W6tflf5S>fy!n zx82*IcPzK;)J{@?2^U7l_o;tLVNv3F?d*l;{TC|-YsyAz%jUvDUkV#h0)*Efy~$}d z`wO<*E9a6v*Hxk{2eaB)y({R&H?4!;YEnZ#LEY3lZpm+JB@(h0eqdGwCrSPY{=7#s zU)X27mh7~kMed(0d*^!Vj;>vR)5kkL=935e^SahMpc_UmVULWKoARRm{h24zk%a%X z){hS6~OGgj+a(deDMz%J6?Df%d*`7cgfs(Y*@U^#J z+nY@Edx5T`!Z#VoY}mL1p)_MVB*d;>zZ0>+H*?=nfH^$n{FGRj8fJA4ra#MS^IJhd<`4u_W!?&Di%vN|0(lJHu z+p2uPhKBK3VUEs%pzdVz_P!Npl@gr6^Z$Q7x0D>g!XyV}eQ-J{McG8?!BG$>46hLw z6pqIQl01|TlNOa_c-D@SM4%ERoHFFdp4|#y5XhDX1QG@P-xVD*OuC?A2v#K}smSw> IDej&87XnC8H~;_u diff --git a/summary_results.json b/summary_results.json index efbc91f4f..b203d7c33 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.19844, + "Performance_Result": 9.62328, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.4194, + "Performance_Result": 20.7698, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0731, + "Performance_Result": 23.2224, "Performance_Units": "Samples/s" }, { @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0806, + "Performance_Result": 20.8658, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 5a5a6673a47a12a006bfe9dee4c951c4124b739e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 17 May 2025 14:05:17 +0000 Subject: [PATCH 563/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10934 -> 10934 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ad89a181e..3fb7bbcdc 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 14:03:53,366 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 14:03:53,366 submission_checker1.py:3308 INFO] --- -[2025-05-17 14:03:53,366 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 14:05:12,739 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 14:05:12,739 submission_checker1.py:3308 INFO] --- +[2025-05-17 14:05:12,739 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 14:03:53,366 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 14:05:12,739 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 14:03:53,366 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 14:05:12,739 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 14:03:53,366 submission_checker1.py:3324 INFO] --- -[2025-05-17 14:03:53,367 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 14:03:53,367 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 14:03:53,367 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 14:03:53,367 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 14:03:53,367 submission_checker1.py:3345 INFO] --- -[2025-05-17 14:03:53,367 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 14:03:54,359 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 14:05:12,739 submission_checker1.py:3324 INFO] --- +[2025-05-17 14:05:12,739 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 14:05:12,739 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 14:05:12,740 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 14:05:12,740 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 14:05:12,740 submission_checker1.py:3345 INFO] --- +[2025-05-17 14:05:12,740 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 14:05:13,787 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 0b2a2c0c5..f37c021df 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1190 +1191 diff --git a/summary.xlsx b/summary.xlsx index c238ab7861eb3bb54c78807e985a8b4559a800ad..dda3b531cb115ba3022bcf58342aa0e3ebb921e0 100644 GIT binary patch delta 313 zcmV-90mlBeRkl^I;v@tlwyGnO<|HbAVmqOLtR$5H^+Y8kia^{fd)r#XKe7#J-ySz@ z(^l$<<9cSlon5(}ZgiLZf)C1?2GN3`42-Z+nN@?{7qk2UWg&7Sd1nnYXbS;N+nZWA zCai~7ZyorkAY=(-0u!!5>lhuwIEXc99*X2JNnBd5c}%5WVaLTcUjZ%&xx*TNB9}aJ zyaVQLB%%XSiV^94d^Z53z|esP69c9N#V2nBufx@bK{<136}JwqT94xB-UKxo`@S#w zYOtpK6o1ZN-iB3kWp<|$fZAFLCOq(HeOu%6A|0;dACJO`v LB(|y}lQ$}VQQVG* delta 313 zcmV-90mlBeRkl^I;v@u}v+x0v<|HbA;-sN~tR$5H^+Y8kfB2Y|op>`#Duh;Pdikzp4(?%;OQ3npKs+)ot z!nB1~s|{E$z(oP19ATzJd+!aw*s(oG>XP74LELI9sSl;yVMEzB-2q;t=^d7T;3=n` z;v+CN6A_I_&L*V!u}u%)3_}AFly^8wGJN)Wu+q&o^vZ=(3f~!+wVuSueQ;tjwr!iV zd2bE*GyJ)EdFxk=g*u*!0jdhl2(v)FwpD>Ii*UGxI==}g-U{H4UC?LV#zlrc;86%b zhPWr?BY%3X*Qi>gi)EZH Date: Sat, 17 May 2025 14:56:41 +0000 Subject: [PATCH 564/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10934 -> 10933 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3fb7bbcdc..3c18d54c0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 14:05:12,739 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 14:05:12,739 submission_checker1.py:3308 INFO] --- -[2025-05-17 14:05:12,739 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 14:56:36,547 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 14:56:36,547 submission_checker1.py:3308 INFO] --- +[2025-05-17 14:56:36,547 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 14:05:12,739 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 14:56:36,547 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 14:05:12,739 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 14:56:36,547 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 14:05:12,739 submission_checker1.py:3324 INFO] --- -[2025-05-17 14:05:12,739 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 14:05:12,739 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 14:05:12,740 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 14:05:12,740 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 14:05:12,740 submission_checker1.py:3345 INFO] --- -[2025-05-17 14:05:12,740 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 14:05:13,787 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 14:56:36,547 submission_checker1.py:3324 INFO] --- +[2025-05-17 14:56:36,547 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 14:56:36,547 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 14:56:36,548 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 14:56:36,548 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 14:56:36,548 submission_checker1.py:3345 INFO] --- +[2025-05-17 14:56:36,548 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 14:56:37,860 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74249 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74624 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index f37c021df..8cdd21dbc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1191 +1192 diff --git a/summary.csv b/summary.csv index 863333c81..95c5573d7 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74249,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74624,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 9544b510a..393fc3108 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74249, + "Result":1.74624, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index dda3b531cb115ba3022bcf58342aa0e3ebb921e0..a02405095c285c0bc5ac7c696f00f3f6d224fd58 100644 GIT binary patch delta 1771 zcmVfO1oAvv&w#2!E%`pj0JLqDkUO1BhfoX#kN7LRrB8Vi|yv znKfEMX)=J=C;+8N0MZ};K}$rZDWzE`>#{P0vVs7lYZicZnEX4;*;*D@3CnH!N89=H z=29$m36`jlc+&nMnNZq4B!gI1uzy$vuw+&ZU}>^{*eHOdN&eZ~_<$wIKXjUaC4b02 zqzq25tZO*IZY|(nisinVKYG{vaq|1dE925G;SvoJPntg@6H4=kWRS}W<`2sNmn7p$ zEKTAM>j1Dcc|RJwA83ilAn36}S(lW7$IiM2k6qX7{UDa#Tx73hiB&{)y{-LtEgiUa z$qTXCDd7_R5l?DABoj*Qhh&h;3V+%U%K(=o16-P{AJzeIX_9_4NI%dLkwMUvg|aRy z!&59VgWwcfv-I;Am)}H|t!0swPO{t9kJhcg+4`ldT{7JgGEpA!r1V2Fp_G0|2Fa`- z{jdy>NismD$@yU&0GTG`M}zVMEfE=nkO|5UDZ_&-`@24HkgZwy`95UssekixrMI1* z;sseRbW48wOM4|~l0V5Kr5}E(mUMnd z2+^$I{ICq5Niu+@N%>(N0Dqb$<41$>11%A|Y=|ZpKcozIP0}^)nl&3g-v`Y-RerAY zw(?W#ntoXNRI|kS(P)qO()l47Wl8CWgpkb&(hti3n@rh(*Je%94;uyGG&w&SoF8b3 z=rpA&3uRqahWjQlgJ9pRS^D{UIG=K)-Bb1FQg5q2`MzmszwG7N5`XhYqe9|K^M_=V zCDk7iLN+U?KP&@mGI0mkG^syq6kyY2{%A0Npe3Ty1UA9^A!WF6Rxy8c&F0VdVRK*o zA1(bK<$EPhH|-Mt$3~IFm;MjQC`$@JBm{7>jD(YAB%EwQ0HM!PE!XR8w4O| ziP$v3V~4UXEt4B334g)HY44N&^LL5twJfsA#%XVBK*g)HzE|=)CU%JiWTQ{wOAUx* zlqEeN5<)gBct9)zY?6$8!fBf{Al3n(Y0`jf(14&NB7=a(4pm)NCAUp>hrqUJ@0|to zZAJE4)>mc6w6`Uoe8;rhQVYl~v4Cv!NqlJmk&LpW21G(gWq$>+)}>`|kgf3N zf9yTifU;qMqBZUC*344;JLzKTD#vx{Wl zxf03t*2la};`vQ{8jr_mqo3rLYgM^vU48c5r8@ku`hS?Xhj>0-G98SHziHnauRhc+ z8%=K->7T2Q%kvITf7e#gS$}xB7{`t5%Pc-_#-C^TJya~V*%*8tu0ANQF`JK)`6&Cc z8ol_WzQwqAGD!wME+@5^j{aQ4KmO4F!8`n+@BQ?1L;ut`Zsbe9kLIgej#7NVrgd^O z>)piXy*l~LXqq<0@i1Sg)$*H-`Rb$hmS1PdZ26VjY*} zsyEACg4Os(w%d@muM;4E#Hq*i%zit&atl9 zU|O)oXedz!9xbc8f?LXsgI8xPxFEqN5u`okwnV!SEXCOK9Vq6r=+IHz8mCx@rK_=j zW&E4ffG2rA#|i=yEHK;yQ#%sT5h?hHv_GyH0D@y^K!FY(lZ@cAH-JfiavvxXIV}61^*Dr6ws;SidsW_mj5S(%cEErc6__BzHYnTt~c;c-D z@z_QE#@lg`Lm!AF2B3$yC*@=M^t@Rmp=y#(W@$c4$znt9>1*DLes^laMJv4SyT}0C;RKb98xZWpk5Z zDHN0PDH;Ot9+M9r6_aHt5F7PPM1dv&002+|000pH0000000000004l2(vwaqJ^?3_ NjVdDs*(U%1002tRKZ^hW delta 1759 zcmV<51|a#hRkl^IatH+jHX%ZAvv&w#2!Ag7l!?EU9I6OY}!Psr`^lD77DwL4PhQXg@3iT#^iMX|jG;2f(FC`q3c$KubghL01;a zx~vROvBV65Q*6!B&tqJE6Ir&FMOHe=Zd*TEw*qJDm$G)rbW6xYdBl^_56Og5`XL!4 zvx4-)GC(HD0GTG|hjjpCnv@?6$`7xv82Y{wY`O%>KKubgh zK~omWx~vR0P2vT?rdhM{^R;L`NtLZ-k(D;hZRe+W`Ns)MbUa<6{Ag51e5w49jIyNj zLqdpV1?Pul08NqsG)>A6>wf^yG#Nh{j2~!;*kwaB!T2F%xNDNGao4Qb`1w9)?y2%~ zrMH!zV%PM;(x;jw&W}cW#Fx$w$tX)oKO}@~R*-&J2H0fE4!kyNl784I0H?|M(ct_* zOGKwBRaq$OvNGH^i5Ud@X3f&i*TeagBki85KbLx2{mJ)DOZ#On&wrMfKN=MhUz$H8 zqb#ZZkPxz2LH%JFV3UbEz@|z4VWR+>Ci6#w`2#HxohGme<_{^ujkAjRqiZ&Qz7L!G z>i=lz|0v%pdAezr_&+v^B);^2NJd#w03soPlVv2FEF_p0g({0S-}Hh8DNuSU(*RsAUJEpxY0p&ZU<(67Nc8LXKqfg>X3y5TtB{d)tLVqeNXh19jRFVu(X|jM= z2SBCC0@~d8KqXi}?6QH!4rN_ZhFDgyfVLK}?^r%bWUpm?m9PW}$S?iClx+z}1rJz~ z4i5Ky6yjHs}%;i@T0Fe+PSwR3| z89{DWCa7r-g5(Jjb!*FB==SPv7KEc z1J9L6wzod!Z4%FK;?sCMP824xiq87O%f&cuWM5|SaWnor%kQCLvCYQd^KkV+d5zh8l*~uj zm(}RSC-p7Hy^~2Y_;ER@#dP%NBL4A*{tw>a4}I^apBwt8#&IKG`h7HC-Ex%T3pTBj zqgn4JK0xoyZ${I!F^-4%LamnHY|K|5y|?^2OJ>Wj+-Borl4Z%{{=3__H;CuOclpn| zVUlF`-xQZ*^AUz8e*sWS0|b*TDH^jT7P<}vB(|y}C6nbOAAe56Fc5_AJVoStVmqOL ztR$5H^+Y8kia^{fd)r#XKe7#J-ySz@(^l$<<9cSlon5(}ZgiLZf)C1?2GN3`42-Z+ znN@?{7qk2UWg&7Sd1nnYXbS;N+nZWACai~7ZyorkAY=(-0u!!5>lhuwIEXc99*X2J zNnBd5c}%5WVSmTPH(voR3Aw`>B9}aJyaVQLB%%XSiV^94d^Z53z|esP69c9N#V2nB zufx@bK{<136}JwqT94xB-UKxo`@S#wYOtpK6o1ZN-iB3kWp<|$fZAFLCOq(HeOu%6 zA|0;dA>l2(J49s z^OFoJCL1KSsv{)<002+|000pH0000000000004l2)RRprJ^?6`j4C4r+9v=2008}$ BE;axF diff --git a/summary_results.json b/summary_results.json index b203d7c33..59682ffb9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74249, + "Performance_Result": 1.74624, "Performance_Units": "Samples/s" }, { From 13655979c3781971976c1668b39595b9302f03ea Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 17 May 2025 15:14:30 +0000 Subject: [PATCH 565/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10933 -> 10933 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 3c18d54c0..e46823419 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 14:56:36,547 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 14:56:36,547 submission_checker1.py:3308 INFO] --- -[2025-05-17 14:56:36,547 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 15:14:24,989 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 15:14:24,989 submission_checker1.py:3308 INFO] --- +[2025-05-17 15:14:24,989 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 14:56:36,547 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 15:14:24,989 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 14:56:36,547 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 15:14:24,989 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 14:56:36,547 submission_checker1.py:3324 INFO] --- -[2025-05-17 14:56:36,547 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 14:56:36,547 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 14:56:36,548 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 14:56:36,548 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 14:56:36,548 submission_checker1.py:3345 INFO] --- -[2025-05-17 14:56:36,548 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 14:56:37,860 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 15:14:24,989 submission_checker1.py:3324 INFO] --- +[2025-05-17 15:14:24,989 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 15:14:24,989 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 15:14:24,989 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 15:14:24,989 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 15:14:24,989 submission_checker1.py:3345 INFO] --- +[2025-05-17 15:14:24,989 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 15:14:26,104 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74624 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72259 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 8cdd21dbc..9f7893cf9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1192 +1193 diff --git a/summary.csv b/summary.csv index 95c5573d7..fa8af0676 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74624,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72259,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 393fc3108..15037a9e3 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74624, + "Result":1.72259, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a02405095c285c0bc5ac7c696f00f3f6d224fd58..04662f3959c22e26b2066f0a6e5ffe64c3a9b9de 100644 GIT binary patch delta 1692 zcmV;N24nfPRkc;HatH;RUFXJcvv&w!2nJfaB?gdIliUzWe`Z~_SLzZhQ6urB{X;UL zw0}qjv8-VKunb_ytQx@5WdE>H085knv$^pBOOSu)GyzMHe@GdeU|H92g56rczZA=T zHGlN3`QzmGjaSB{UBV?AB%U;XNG6o#56K{x70e%&0WL|#mspy_AJze2Y4UzFct6k* zkwMU7hq5jye*=%5bqyZ7uG#xREWf$PUds}zi0pb>`|(;jaP5*8VzpDkCHf_wNd~wySwE};;L;@hXpnxOB_e~MD+^^^R)(ioVg|t}wr1()F)qJ} zEL+PWE1hJwtskvhfwT2XS-WJqC1j#J;z{X;WI`$Ze~=84SwZ??86cBnfJ~F~!#V&m zP0Ei3wI{ICuHO_TDYLHU7}hzx?JER=Ow8E%@y3xZ9vX65H= z(R`9Be_P8UD{Y$F&QJ03j}w;Yc)CRS(Ws92Qu!enWl86Ugb>XN&JW7~nj`~gnv@^b z0ibCzel!?A&=RrBhG>HEL&|X1BwgdKS+nu;ebC%f<>yLoD?i1q>4&9HHA|cyjrNEy zogb1>mXv--2-&P4{jdzM$&?*t=GYIz0 znx&twhw~{%+C5c&F7>whlkc0B_RC(LEir#IDkQ!%e@I4IQvD$zWV3?$!!p1o6L)}3 zllsF(0X9wMj|TGxS|U14U=z$AQidC674t{eZ2o*7Huu&4(bE4>zE|>e(=PFUY!pd+ zf9d~_jIyKvL_z>3%Sbp`M#9M^1aR6W0f>zv;WTy7u|WWWmWWLgJa#DS(lWVmk`Qd1 z_CEPPf0xK!%Ob06oc6W`RJ=;-dnLbPVwY$@Hu@yK)PP7vS<(X{A!M_H2gEYKCdtSr zoVG~=VjTdQCJo324G3ByG6;C=P}OBse{$PocL;2o_TE`Q-&SO=Wqnn4OnX}b%6Cl5 zEwzB`5(~&ipTw6I5XmS@YCt4}R94V{SO%yh8KBZ+0kIB%N|Ob&x$%KYuz=WQ1CJfb zx}*%TtYQIeEnwfVe3Hms%lax|2@;TB`hh9i5|9cWup}QuA|)V_QI@oTNC>g4e_#Qz z3}DIp8aT+BG$1w#aB0$jY|wz9C1TT*A9`^PWnEeZ2iXdL{>R>P4JaEHNUA6KKeeo_ z()o4U|It<}40OqhvbIx#B$_0?^nXZ3SyBKZAw;r*0K_tYBvWeuNs|G@Mgb&E29ON~ z5VS;Wn({+t&Y`SJ%K*s=29Ukye+JMR$?!=??yLA?JG)2*o-2`TZ+*<$B%a^Ir}224 zHu_0^xmJ~%*41aO<|a(e$Q~{<-?NJn!)IcWo7& z^@o>>aooth%;Mu_{CSq&L&aj7jlt*P>Vxtcv-v2QkFqbT(Th*&Ta0@rf0JbJ<8o4q z>FCcz{NoS(AH2gK`rc1JH}p@9<3_&p`)IzpXVflF9vdw{dR}&x`N!pLfF~$?m@?F3IL23{U<7P)h>@ z6aWAK2mk;8KLF=)0^?l)lRYU4vnCe04h8!(_-rMUcC4j6X zl>qfbB_xVK+$?+BTEsuH4QbvUCvDSK>WSleX1|?Xxti}(oBV?F(&!qIEJq1wZiLjE z8ojR;=>tl9U|O)&XsA&S9?hGZid)K!gI8xPxFEqN5u`okwnp0!EXCOKEhy%*=+IGI z8>d)^rQ2YC%lJ3j050?~cPyzq|5dZ)H0000000000fPvGKO)6-?KPOcH delta 1688 zcmV;J250%TRkc;HatH;>fO1oAvv&w!2nCL(%b-+~-4IHD=29$m36`jlc+&nMnNZq4 zB!gI1uzy$vuw+&ZU}>^{*eHOdN&eZ~_<$wIKXjUaCCERd3{J4DYdFDfE#O~@<-VFf zde{7M^83asNismD$@yU&0GTG` zM}zVMEfE=nkO|5UDZ_&-`@24HkgZwy`95Ussq=HCx1FEj1z9h2OMd%HdnIU+KglDd zACd{B^g}X;W(DboWdKc*0W?j{59E(mUMnd2+^$I{ICq5Niu+@N%>(N z0GcM_M}zSLEfKqHh$a|6qzrdW(lzdyH5)(Q2hBZIey;Sk@>A@Zepvccv&8w)Xpi{P z`5_r)N$H1#kj)Cx56b|XOxc0gW=+x$8wKEhG&w&SoF8b3=rpA&3uRqahWjQlgJ9pR zS^D{UIG=K)-Bb1FQg5q2`MzmszwG7N67xr+LgGvFhh&r`)gKZ!xF@JQ;=Fj(Gb6@=*E&U(mdnHdd?GpdTMv=sS zm;MjQC`$@JBm{7>jD(YAB%EwQ0HM!PE!XR8w4O|iP$v3V~4UXEt4B33Bkr` z@00)YcZuw^EV9bRX>V&l#jCWwSMob1c8LaLqfg>X4TxlvB|RV#LN+USKr91nl8k)9 zX`3`4)&Zbt(tvExfS@HJgMh~lRb5tpCAUp>hrqUJ@0|toZAJE4)>mc6w6`Uoe8;rh zQVYl~v4Cv!NqlJmk&LpW21G(gWd#k0Wq?YO0V+)v5bFS_G+98K8y~0y3y57d@YtcO zOUe+-Di+Yz0`?utCyDH}tgjN5AOZQMADFT&0jb~tOY%V^QUW3wWl0N&gb>Sr3KkH{ z0G7>+)}>`|kgf3Nf9yTifU;qMqa zonN>8A8ob5K$pBIYda-KqDkUQ|A%ChB?TZ7LL@5)Kr91DGPMSfG#Nl_6hP8s0NG#w zK}*D@DL-W99Ll=143Ml~0NHzgZUC*344;JLzKTD#vx{Wlxf03t*2la};`vQ{8jr_m zqo3rLYgM^vU48c5r8@ku`k1(fcs^b-9gK>KKGZH7O>Y|MpR13{^A1mc*H+P4 ze|WhV$BpdEEIw|=pJ(|!R4lgH7)piXz4^^(nl{GqFkh(E z@|%tM>ZA9TUuVf|`IXyjd`z+|ncRPO8}|nBy!bBvc{faw?EahLl59T0@Z>L0O9KQH z000080000#0OxW7<6Qx>9Tvn61@%otfhLpYBp-in!Y~ko?|q8M_rx(2sH`MNZN;fb zl}gpBH_Kjv)%ZuY+mN@f6Ci-ZsmJxqemlEzvD~XB`32{t(Ip~TjuOz^2&wB5y>C|O z14?{gTCm1wC{YI1&{5nPr&x%k ztFeD&{F~K)CwV@{3IY=>Fx&%EI}*_mDfo!AKdu=7f@5evfes#%jNr33fK&cv!=PL^ zl?vZKSA{evnM9X0tL%}a#L=T_9$Bc2$NDNJOPrEk10U`f0NNEIsx&M3@RoY i^-V;9CIJ8dPyzq|5dZ)H0000000000fPvDJO)6+3kR68r diff --git a/summary_results.json b/summary_results.json index 59682ffb9..593308b87 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74624, + "Performance_Result": 1.72259, "Performance_Units": "Samples/s" }, { From 8fa28a84d323eeb73bdbb069ee2c70c7d243192b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 17 May 2025 20:46:34 +0000 Subject: [PATCH 566/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10933 -> 10936 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e46823419..ba0788757 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 15:14:24,989 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 15:14:24,989 submission_checker1.py:3308 INFO] --- -[2025-05-17 15:14:24,989 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 20:46:29,278 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 20:46:29,278 submission_checker1.py:3308 INFO] --- +[2025-05-17 20:46:29,278 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 15:14:24,989 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 20:46:29,278 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 15:14:24,989 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 20:46:29,278 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 15:14:24,989 submission_checker1.py:3324 INFO] --- -[2025-05-17 15:14:24,989 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 15:14:24,989 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 15:14:24,989 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 15:14:24,989 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 15:14:24,989 submission_checker1.py:3345 INFO] --- -[2025-05-17 15:14:24,989 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 15:14:26,104 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 20:46:29,278 submission_checker1.py:3324 INFO] --- +[2025-05-17 20:46:29,278 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 20:46:29,278 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 20:46:29,278 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 20:46:29,278 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 20:46:29,278 submission_checker1.py:3345 INFO] --- +[2025-05-17 20:46:29,278 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 20:46:30,253 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2224 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1167 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9f7893cf9..9f6bb6283 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1193 +1194 diff --git a/summary.csv b/summary.csv index fa8af0676..ea5627c70 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2224,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1167,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 15037a9e3..696abd5a8 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2224, + "Result":23.1167, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 04662f3959c22e26b2066f0a6e5ffe64c3a9b9de..262d98d6b22039150a5dcfb1efdb03f3e92f2050 100644 GIT binary patch delta 1199 zcmV;g1W^06Rk&5KatH;!|K|{Kvv&xt2!Ac5ypkiV)+Jb?M&e2Phh#!&|Bwt~S;78c z8NiZRHGrkb{$Zm4mL~aUbK?V+Apg*50+t~EkTN*IvaaC-yS0FSDVF6`C6*@fhjjp0n!Fzk-Vd}yWDxY&p?|DP z%D`i1U4zH2YxaH+%Wp2S*RsSaBD>z!e!P|rT)X6jSnZT>iT;QuwI7lRrS?NI$YllX zhh>0Ek^wGF)(`6dxHL&W8l)d+iO3-6%0gL}mEkFtm_cxgty%hcjLUB#%hs~UN+;QE z>qqNW;B5U;)-IWD37II5cvAWynSW49KO}=>R*-&J2FN5CAk*ahunvGslk%fM`GJ;* z3_{2R<%g8vL6-epA2`U?to(c*GWXQ^xzgLtPw|4R7nZ*E@pP=25#B7>kQ3uRqahMOkwf?(6ES^4=|G=HC@%GR>T zN}J}k^P}yScES=JPnRe^8r2bBDnBHnEb07^5TaSZ`C%DAlVku*lk&ql05na;j|Srh zS|WDY5KS_&nX&_~&6=biHh&7hX>xuvI6u%5(P>Im7RtJ;4EIf92Eo2rv-I=za6aWo zyQk{UrQTM5@_p0Ne%Z^jCFYMtg~XTU56LJ?sy`%zY*tWzSO(Z+;tsHBQh(Sez^2Lk z(O~{SOGKv$Y=Zei%5dYXV*cow&7beX=DzwrTKYfA_e!2_+9m#vjejDEFZ~~qQI-^d zNC@C$83`xLNI2Pq08ZN^0I^XdoTd&sHV8n_60vE5#|~v(S|&G65`vA>-Y5U(?-JQ- zS!9)s)85vAidSiUujF@3>=F&gMxVr&8W71SOL{;ggltyufLI3DBpLaH(>7^9tOG#P zqygEW0YOVd1_6&9sw29rN^YC%4uNgc-a8BE+luV9tgp(BX>UtF`HpG1T9+JW?Xy4> zp%4Y@B2Y|o zp_&rC*Q@v*Mb1;jX{!~Ks0W9Z)lI<+VcNp0)ds8=;GzIhjxbZAt@nmt?AR70bxClj zAa1mk)Q8eG*iiOO8^C#*F0ceoIrS6|z}QSgbU<=8A7`@V$Yn)}uJNcTP;kuIrL+Hd;ge41ca)-o{m9p@vg2KvlsRVHT*@wkq&> z5f0Z<=hxxH8v*>W5Bin2!y@}W;86%b_HmEO$L#63UZE;a^Ld=k>o3(J49s^pgxKCL8PJ1ZE}y002+|000pH0000000000004l2)ssysJ^?C| Nj4C4r+$R74006#CKIi}d delta 1222 zcmV;%1UdV-Rkc;HatH;RUFXJcvv&xt2!HwWW?i;d>JltbBk`pDLo%VXe@F(gtYH7J z3}DHu8o<(I|FBU2OOyPwx$yx@kbmej0ZWj7NEw`9S=Vra-CDrE6w7@zfAp^T%Mz=I?0Q@K@me}??UEN_wNt_+`Xipyen=*i+7HPfmld=h zmH{qF2Dmg?Kdb}b(j@(8kba;gB7>kS3uRqahNoC!2Ei$|X6ffKF29K^TgxIVon*JI zAFW$~v-L|^yJWg0WTHIcN$H1VLVqd!kPMPpLHc1CAd_T(Oq27&Ish_F%8v%+2U;RB z2q6=cA5w+~S@w5*;2>MG^7DPj+*9Z0N^d(q#S5}tSo+$JSAr(_lRQ%TA(>E0KO}=_ zR*-&J2GArKK-1*>unquClk%fM`GJ;*41%UClyzAdZkogkf=#n#<>zbBe1DQETgxIV zZJOK8Px11P6PDjIrJt{d^C?H# zJym}$^|tzx@0*tP%U+%>F@H2FB)&9%NJd#w{UIS_vx54=GQcJicYsZk`ol&6HcjS_ z2J;76B05cA6U-k{h8t%U^GDZg{(K)c_tpQ=(*IGuSMqezF7bbC6n{y4>Hm<7vZMe+ zLI5YrNH|$W!pSBCaM~sTh>arQG&i5h)okbb|~x8GP!Y*5Nw?GKKVa?m&jhr zBCBki_O=F8yh`hPCBI{0muNsX`Xs*8fJjDJ(gPwPWV3qfbB_xVK+$?+BTEsuH4QbvUCvDSK>WSleX1|?Xxti}( zoBV?F(&!qIEJq1wZiLjE8ojR;=>tl9U|O)&XsA&S9?hGZid)K!gI8xPxFEqN5u`ok zwnp0!EXCOKEhy%*=+IGI8>d)^rQ2Z3_&3`CF7o^iD+qr~u)y#DOzlKO2c+N=(*C%1 z1PG3y1qC{IOfrH`-T+Sds|}-a=2SBD7Oq;4;^f|WIT^dI%evWUjrj@wT)w=GtEN&9 zr{aK`N^r^@uwYzM;qxLMu4O(fDkBEjCjbBd0A=tu`v3p{ diff --git a/summary_results.json b/summary_results.json index 593308b87..56fd0ec51 100644 --- a/summary_results.json +++ b/summary_results.json @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2224, + "Performance_Result": 23.1167, "Performance_Units": "Samples/s" }, { From 7f3cb134e982ce358e9c72812477918ccecb23ca Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 17 May 2025 20:54:13 +0000 Subject: [PATCH 567/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10936 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ba0788757..93cb07817 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 20:46:29,278 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 20:46:29,278 submission_checker1.py:3308 INFO] --- -[2025-05-17 20:46:29,278 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 20:54:08,147 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 20:54:08,148 submission_checker1.py:3308 INFO] --- +[2025-05-17 20:54:08,148 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 20:46:29,278 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 20:54:08,148 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 20:46:29,278 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 20:54:08,148 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 20:46:29,278 submission_checker1.py:3324 INFO] --- -[2025-05-17 20:46:29,278 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 20:46:29,278 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 20:46:29,278 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 20:46:29,278 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 20:46:29,278 submission_checker1.py:3345 INFO] --- -[2025-05-17 20:46:29,278 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 20:46:30,253 module.py:5269 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 20:54:08,148 submission_checker1.py:3324 INFO] --- +[2025-05-17 20:54:08,148 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 20:54:08,148 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 20:54:08,148 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 20:54:08,148 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 20:54:08,148 submission_checker1.py:3345 INFO] --- +[2025-05-17 20:54:08,148 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 20:54:09,309 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72259 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73501 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 9f6bb6283..1e3b283bd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1194 +1195 diff --git a/summary.csv b/summary.csv index ea5627c70..fe17b0ee4 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72259,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73501,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 696abd5a8..322bd4219 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72259, + "Result":1.73501, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 262d98d6b22039150a5dcfb1efdb03f3e92f2050..6c861a71869b6ce94f939c99ae0376e480e93840 100644 GIT binary patch delta 1784 zcmV^{*eHOdN&eZ~_<$wIKYw(Z zfF;O3qzq25tZO*IZY|(nisinVKYG{vaq|1dD?7v5C0wFG;z{#|WI}2FkPLEJ!Tezv z;F4r~iKR*WVI2ULChtds_X8~v83a9cDC?3k@Yq?`;IZqPy&uH#n~UtVEU}8nuD7)x zucZUmE_oqVJ0)DAKjKO4hh##j{eO@Qa#=z9VHx0(WPnSP^}{*K1je`&7-P4Xvsr1V2Fp_G0|2GOh_{jdz6Niu+@$@yU&0GcM{M}zVM zEfE<6O<5@GvNGH>i5CQ$X3fga*P{6(RkoHzR@yYTogZztv=f%SDqEubXjDghsr-

j2O+89y3~A83i#WkWQ<_#tJuYm%;U*R0w2`95gw zsq%BBx0Rn_*Yv~Er> znm;6?EUEsG5VBc8{b3nklZiXPrb+!_qX3&G^GAdE11%AqCa?+S4=KZqvx@nnYc_wr z51aey|7hv|DBml2x@njAKQ@XazVv@cMp;q-A|Zg2Wh9&|BjIEd0yu4x0K`U-aGE;k z*dPEwOT?xL9y^qEX@8mAI7tXLPJ5sHpTA3FuVs-{Hcoq611esn^}UkcF|kWDARB!W zUur-kqb%tGkr1+3!2@C$V3TCz6HeQt0kIAMO_K&>g9ZdG5g7zLcBtyID!Fa4I|Q~( zd+#iuZ!5Cbvc4)iroAlzd=QD0fJjDJ(gGqO#Ik|~#4>;-^K0NBYtn$&D8Qvj1F}H_f|iI)Q-0{hIh1v2 z860FQ{P`by&wn+bY*-+vp5*`3vbIX+*KPktTdgqAB`?a_P6?7|lK9g9AsJ;!0f>YU z$qE7x%K(y0tpOxW1`rzskTe-UHW)zA60vE@51BcKvMwzHBr6y|_MRI+Yb3)bA-S*O zkL~Ot8F;Qlvc2^&Zf`df!_(ijRdm)LUM|LQBl|LokDKx5S$+={i)}UrpNFdt%4^K# zqhvnHzN|(sKB;dp?ww4M!H>&HEvBPC7x9lj^ndUUf9QKZ{oK$$HI5tk((j}B>XxGv zU$AMN97N4}H}QFIelwb;jd4887izWqW@Enk=)L9FSu$IG0|b*WDH^jW7P<}uEi9(tljkHLe{RAs5QOi2ipclG zPH0+WB|&N{PDQFzs-oU3djYHQk8HPrx33c*fW)cC^~`=dyK*(#sW$lq=cUm#$}@rz z(A)^AH#PcL&eJJMd|+Cz)@Z2F9z2>gcNMpk8wYRBSa3mtPa;Tr%59CdAy|sB=UY(B zXVIafxHe9)5KFhgf0prYwgFra@_-ctCRkv&2c~u;q9ao95ov#1I{*a7(1HRTJkGNm zpS=N`@;4g><-)0C*ju=1J&B`x=jCYZx-RP`gEi*Q@oMq5ci~fPM%+uC3Dmiq$pETrupNtAhf)v zWct;Qyy`y#)p5wbi6_7z_w^@e{s6NoCdvW@+nts~bCdZfTLBG|RVh3Imlu<<7Z#JL zDJl(w8~^}#Y%g delta 1753 zcmV;~1}6EsRk&5KatH;!|K|{Kvv&w!2!C3;B?gdIB~YSC;zMirAYwNAOJy2M5igGSt#qWGK8{%0HkXcfOeStJIvWy7FY?(ZTm;t zEv3AYBdpdXSfWPaN&AOnLTUey3}RWq{$Uxwl36u?rOEzbqX3pB`Db(E1C}8F(0^$H zmLUI-GC0AquHgi`wSa#qmiub{=w0*2$?qGl>JWd-erWq?bP0WM9}59fJ~F}qe1zBmWT{O$OPqwl;J^^{aqh8$kwdr?Ur`J5*<&MC_ft25nn1lB%>_p z{E!f$S;6^X89B6isjO)!2)8Sa{-Yuq(!Hh#VjntQ7J zTvuhx~vTMO=1SYzFD*M^Yw5(d&R#R)6w+)6#y~%YU;a=8r~&#Fyp| z$tX*zKO}@~R#1Of2H0fc4zOubf7mF%rpf%#VE#Z$M5hUCg84(raO13E{^**`pYOxw zzWP5}`ajC|N}g`oCH{|%B8e~kACgg)6o5zw;A9yIC(B4U*@OU2+av(7Q6!wE4mvgn zK+qDgX@bWNWnEe(H-AnNf{oMOC;#W~64`56WR;E6-qwJMS808(g4U;(iVV9ER%ILMkbAT|ncY0`jf(14&NV$+l#dT|bAU0MbQ z*$RLD$KG=dD1RFkNUA6KKeeo_()o4U|It<}40OqhvbIx#B$_0?^nXZ3SyBKZAw;r* z0K_tYBvWeuNs|G@Mgb&E29ON~5VS;Wn({+t&Y`SJ%K*s=29Uky2GAPG@JUGStN3F( zyGRC}E0Jt(eazb=p5Mf$@pzmz`bmDdR+XF9)o0&bs(-@|tB;9$i09)a)4{0toA$l& z>O<|a(e$Q~{<-?NJn!)IcWo7&^@o>>aooth%;Mu_{CSq&L&aj7jlt*P>Vxtcv-v2Q zkFqbT(Th*&Ta0@rlVtGYa#D-w=+8y`;}88Gyu%;*-cLU_^iPfBM!xj>Xui7TD8(0S zS|>-d-abuy-kaZyrfFjw5A%guEx*~AuReNj`E{1emS4He#>XVflF9vdw{dR}&x`N! zpLfF~$?m@?F3IL23{U<7vkey54g~Av1ZI=zBp`oI!!Qtp?>t51d*URe1*{~M0QE#A zB!WQPEPI<;#6PkPY2F?uZPQljiQ{@^znxvVSngyS{epEus}f~NiXu=BtJMaq7vQ1*QjRcFqOJFaVC>iyBy~w}s32~%mDGpQ zHrRhq_DvhWd73V;1W!5j6c51IOhj}*ayB8&k8MW)XBb+LpuEFblHrrrgO%=T!>F7& zrSQFhtJb49xpz)X#;)sv< z#2W$pu@Cx{x5Fa)KHyOZK=yHu%E#>Kxn3oqDo^uyoX+FyuFg|(yCC_)*D&&;{{)nW zA^#>G0sGvSpP>2!vn(dc0tLSR=MZs|`zTuh3zJtVJOP!Hlqo?0f|J%MIsx>P5h^Ad v>*WMyCIJ8dPyzq|5dZ)H0000000000fPvMMQYt_j00000o?BT> diff --git a/summary_results.json b/summary_results.json index 56fd0ec51..d3d8b0297 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72259, + "Performance_Result": 1.73501, "Performance_Units": "Samples/s" }, { From cfd0363341af8691b93cb4363e17dced50c675a7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 17 May 2025 20:59:47 +0000 Subject: [PATCH 568/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10938 -> 10938 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 93cb07817..2a255ffde 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 20:54:08,147 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 20:54:08,148 submission_checker1.py:3308 INFO] --- -[2025-05-17 20:54:08,148 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 20:59:42,128 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 20:59:42,129 submission_checker1.py:3308 INFO] --- +[2025-05-17 20:59:42,129 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 20:54:08,148 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 20:59:42,129 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 20:54:08,148 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 20:59:42,129 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 20:54:08,148 submission_checker1.py:3324 INFO] --- -[2025-05-17 20:54:08,148 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 20:54:08,148 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 20:54:08,148 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 20:54:08,148 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 20:54:08,148 submission_checker1.py:3345 INFO] --- -[2025-05-17 20:54:08,148 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 20:54:09,309 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 20:59:42,129 submission_checker1.py:3324 INFO] --- +[2025-05-17 20:59:42,129 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 20:59:42,129 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 20:59:42,129 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 20:59:42,129 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 20:59:42,129 submission_checker1.py:3345 INFO] --- +[2025-05-17 20:59:42,129 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 20:59:43,089 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28538 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29747 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73501 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 1e3b283bd..40bd097c1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1195 +1196 diff --git a/summary.csv b/summary.csv index fe17b0ee4..ad08d7b14 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28538,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29747,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73501,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 322bd4219..2a897f02e 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.28538, + "Result":6.29747, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -199,7 +199,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 6c861a71869b6ce94f939c99ae0376e480e93840..443ef954081259984380fe02f64bb856e7c8efcd 100644 GIT binary patch delta 1458 zcmV;j1x@<8Rk~HMatH*1#Boj*Mhh&h<3epeD0GT8MWSX2G)&Y=dQhqcjKhP49e?bVD zp!|?BJjk-Y>jMYbnw6jLL*|}3KUaF&`6*tI^+LDgx4*Pkf+qQsJW~20nNUhUB!g&H zkbYPO&?Ff^)8zcH4ggJ)@}oidftH92f~G8#by*p1n#2o&O|xd@=WEe?k}6xvA}ej0 z+s==+TiOZBUX?9Tel)5hzEplle@0o-`5_@hvx4)(GJqz@0GcM{hjjpGnv5R}#t*ba z?6M)6VEm9W+%-wpxNFvI{CpoY_f+}0(%Z^Uv1|IFUE=)c66Z&wJ>pB}hh&r`r5_SP zHY-R!ECXyZWd~lHHAz2g6oAv@{Ah4~pe3Tyl&UP0by*qio5T!)eY0lif9LDre9Dn_ zPt~7Gy{-P_`=+J+vX^H|%pZ*ki7(9`l2Mjae@F<~tf2m|46w<>9bnU>{;*MiO_TYf z!Tf=ih)xsO1oMZK;l^3T{LwX=Ki`MVef58|^naA^l|0?FOZ*=jMG{~7KP00pDFBfW zz{xTaPL`2yvIzm4wn+eDf1^k^O&xS>5P+a1V$%eV9m=}2Om3Vc1RJNlPyWx}C9>DD z$SNDBy{!QiuhRNn$?urhB^r>8K8Y_iAd*p*^ngeR*{t9Ju?(lbK?V*U;(kq1|B<@hjaF8`= zKx`D?(xd^|paDTkf5fIKKlI`p%DS`+4zd;g{Exlo8c;SYkW^3de`;AHm<7vZMe+LWpDq0f=P)Nv75Sk|qO)jRHuT3?Lf}AZUr$H06iP zoI_ccmI0C#3?O^Y4WKoW;ggWuSMkSoc99G`S0dTo`k1##e>}g5Pvh}8ZS<4;a;+*i zt*g(zyHtlCRv#1h5YNX;rh`%OH|=}l)rZ<;qv=f}{d4tkdEVja@7gLl>kls%#~=DX zc!xjqy`O$=Q|O->$BlgH_tAWH%TbCi*tAZLX1$yEyf?oYP1D9W9_9<)B?*)1 zBp-jvPQx$|MfZF~l``B_x81*je_pwTK_G4Qc-#H%Zf0%8KRq-Z^t; zTB#!RA6Qi+h+oYX!){sBLpR3pR zZq-<*+s^E+;3)E{{75K6UhijV zx)nHEzm&C0rdvWL$|IhXen=*i(htcXnH8iTmH{$J2FNryKdb{F)1>@pP=25#er88jK%k ziP&XBG{N{GWw>jSu5s6_+4%WBXzr=#{Q3H;EYp`)1A3f6v#$`IIB= zo~l2WdRzU;_f1RtWiQW`m_Hg75?`7>B%>^;{*Vx|Swa0_8DNu%JHVz%{b8d3nHjF-D|xzUm-s(6iX^`De@I4IQUD?$ zfRkk;oGc^ZWD^27ZIb}Re@2mTnmXv%AOJy2#HI-zJCt>4ncO%@2sTc8pZuS{OJuKQ zkySQMds_o4UZwTDlHW10OEe%GeG*@4KqR9q=>d@tvRT0cVi{nQWaJZ0+oS=p4ggJ) z24sT<1T7I61Uz=A>ar@iZL&KAwoQBQETC^Ive&Y{Dm$jVEdk{_f2QSXU2>eYODrH8 zeG*?yk3WvWf+? zwSaxc@<}3lE$gd<6riIjjyMp@DVA|b@Gf(67ffF<*5;2>+# zfY>O&rAY&_K?8!8e~3*}e(1$Hlyzwt9Aqo}`5$}FHK1%*AgP|@|J1U!O6S*Y|3_P` zFwiA0%Gyo|l4z3n(*GeDWk~^ugb>LJ0uajpl1!}uBuxeo8wHRw89+7|K+qDgY03|o zIft??EdwMg7(n)(8$fF$!zUrRui}sG>>?R>u0*oE^)YXge|UZqpT^^H+UO_w--?deA)*oIj#&IM2 zGK-I!@#k574;71THU^)Es}IU+%;uwHKFYqVMlU|8Z!zwjOp?Km%SkP!qdyn%k3aN( z@D6|Idq4f$Q_w#(jvM*X@1yzZmZKD3uxXtf&3ZTSd2fC*nx>6$Jj@qrwftsdzWV6B z<=0s8y}M_OD6Z<-NwB^JTJb>f8Gs~B)k8nxFnm8Fg*DSvkey54g@VMrs0$6 zBp-jxZo)7SgztTd$oIrfXj)_?L24^bMXFS)qTVcf0ju$kY`1~8uM;4E#Hq*i%zit& zay8qjHu(kTrO`FYGlCM(+z6>RHTqc2(F6$OTY3amc@kC%_^1^(Scl0JAJ6$^r!2ot8wi`Y1gC M1T8G4;gdTme_hwIL;wH) diff --git a/summary_results.json b/summary_results.json index d3d8b0297..e121bf1bd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -739,7 +739,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline", "Software": "deepsparse v1.8.0", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.28538, + "Performance_Result": 6.29747, "Performance_Units": "Samples/s" }, { From 27487bb67051a122e4d042a17e82d1674b31525f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 17 May 2025 21:40:44 +0000 Subject: [PATCH 569/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10938 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2a255ffde..c1f6f75f3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 20:59:42,128 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 20:59:42,129 submission_checker1.py:3308 INFO] --- -[2025-05-17 20:59:42,129 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 21:40:39,226 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 21:40:39,226 submission_checker1.py:3308 INFO] --- +[2025-05-17 21:40:39,226 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 20:59:42,129 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 21:40:39,226 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 20:59:42,129 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 21:40:39,226 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 20:59:42,129 submission_checker1.py:3324 INFO] --- -[2025-05-17 20:59:42,129 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 20:59:42,129 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 20:59:42,129 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 20:59:42,129 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 20:59:42,129 submission_checker1.py:3345 INFO] --- -[2025-05-17 20:59:42,129 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 20:59:43,089 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 21:40:39,226 submission_checker1.py:3324 INFO] --- +[2025-05-17 21:40:39,226 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 21:40:39,226 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 21:40:39,226 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 21:40:39,226 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 21:40:39,226 submission_checker1.py:3345 INFO] --- +[2025-05-17 21:40:39,226 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 21:40:40,186 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29747 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73501 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71949 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 40bd097c1..34a5f9bca 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1196 +1197 diff --git a/summary.csv b/summary.csv index ad08d7b14..78f7f7d57 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29747,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73501,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71949,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 2a897f02e..6c759ab2b 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73501, + "Result":1.71949, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 443ef954081259984380fe02f64bb856e7c8efcd..e821fbb986365feb5dd578e8374d3bb1477aa96d 100644 GIT binary patch delta 1467 zcmYL}do+}39LL{x7?(D;Q8wDlsBs%(YG-Gu)|we+#n_2-h@_m%v_v>5mtAYzwlSS7 z=GC|stxIc>@y;ZSbm7FX$*qYbAzEY$MKk8IBY*7k&-2G~p3nJw&+m7>t^O_kg7bRF zi*b>b=YW%(#2C%H`qhkn#+n6o6ZX9(2tJ<)qeQ3r-}jbYz{Qx zd~(!WN|;0$pnQtSSX256hnd^EW!9({D8Np8;~7&%3xI%>M7C%{hjY6z|1`P>738ON z^|k8cggkqz#PBChFdJB0e_H(AdS#;HF8Sfnl)3y^t}4s+Hkzkjwdzh}ce9X-99F6~ z4N3$?I_oD!4jn&o89LUak+4Ctc}{+1)FJyXm&0=N1kv1iXq;`bh~z z(fUPpIf;I9t0G@I8Jocj$V~~<>BgF4hF&WKhN(Y7%N0~v8b52RTetQMCx4R4SB9YX zcT4LO$!IolZCKVaIM(O0lwS8`E2}}7XdTWYOZHT42!3!UW}QLKaaaI@2a*_l8oz#$ zJ=r4vvtRh8U$s{oW`qs$HSS?k_+=)HBc_V$R zlI_E2KIl}`tQA!}jCR^0h#;~_h2(zJKm?*BRFzA7vA?#E+3Qgr@Yl1X1+BNgF@^N@^~mAq7FM*r4BjiQ=+!%->k{_-2h=MqD2+I+E^C8 zgq;m#hE9vtBx;$MwtCzwh&3&`^KJQ*|M3Y2*)CHtO$#2dU8>TdY2+-8H{@I}8aXcy z*GUn>!gymI5frCzN>WFe{o!>g~OhYp?urJj^C16 zcLj+%5ucIB4WtEI!@Rnkq+#N`bp9j-QR=kH2btix=;U$~9uq&RVZsD^cYJLiCHRYx z1bi75eg?PG1Jku%7o>#tG-+3+>*6K4d7k=Nn$(UMlpEe5kb>6=B;M08gl!D6vJoeW zNtm4^OgMLhG{ZSrx)J*UwdrA*x^eA<7Q5#Xk~imT@VBLU>gCm8fX1yD$*=`wvl6@@keLB$JDQYKD{_nC>=p)z6ish1-?e}u z4`MQkw~kYd^bvSzZ|K;uVDyR;_l@&Xgp1|P>=fRqTmCcE7e9_B6tL&tcg%-PSx(6k zB^UPLRMo%6w8?$NQKY*OmF!!tgeTv*C_{pO_(?fAbNVZ1c(3qNvpM_v%$jfG$eFPB zNzL-|wntH}^x2BHy=Om6L2|R{mlCYGqPcYV^g(%?iTabfTKC<1PW#%>69eVZ>E^t> zO4s6s=mYd6WQkNVP)ANMEnAy)u84TPqpKtNMnG?j(`;?d%ixr2*20-%(ZZ##B{1{6 zdi>mrot-OFV#S;BeO~Z-%w=y_rq==81B3=9|S)_CODg&8{cd|0PsZsfC=#bNllYNemGaq0m{Se K(HXbhc=-=$nZ3^d delta 1489 zcmV;?1upuzRk~HMatIE}!zo5{4FCWJaR2}q0Fx0Z8nc=RJqUlcYn8n4qg4r%Xp(r+ z03w-C8bBn2P*yO2SO%bEW{s9mnhYQ|3P5QRfHVj|&=S#UN@*6#x~vSLtRMjCngyU8 zCjSm|ww47}!gAaG(RNEIujB};bqSWJk$BSnA(>FxKO}=#RBi{BiR8#w$C++9h0~LE=gC zhh#!&{*VlES;72a8Q_v+e2Jw={9zpcmL~5t{b=0^oULEV+9lI1 zArs{hPf9-|6H4iaWRT1X(hti3nIr>bnw%fj0g!1@el#dQ&=QeB2$`V#kTN{TvcKyC z2icmHpYMM|=AJq~S9;s|DPEBELbv3%zqD6^Ci#;*Qu-m8P)a`}gJ@Qeepm+3BpE={ z08Nwfqe1zBmWT|3rYw|oSs8Ac#0!E=vu5SzYtej?DqG7UD{Y$F&X2ZR+6l{E zl`T=ITRQb8m+saR|Yx<#G;{50m=SQPG;!Ed;WRxYP9}+?~D@Z>q18g#7 z2VR>sNk429fYapsXmEa@C8E=msw|XsSsCt|#0-Lcvu5e%>*0LLk#1{^a|n zrTu@hmuE}NAB_r$FU=p4QI=GHNC?@ip#HE7u*t+7VAG`juu*_bllh~;{DGE;P7~M! z^M{n-##zPu(KVYt--pe8^?$VVf0XZ)Jl(WQ{2v=d5?}g1B%>@T0Fe;D$ubg7mXUC> z2?3n8NdRJ_NH|R$bZiiSpe16{1dkocy0m{xZk!|p8>hWb{?Fefve&Z6DjTQ0tpOFU z()wP>@0i#n8jy`Xi7z!El2MlQfJg}0tl$B$46sQu@(HJH(tub8fTl?UvOxoamWT`j z9y?TZS(V&2*&PDgroDF-(6<%YYgu2F9n;>Hfbt#Fazy zfJg|bte^q03{XijK&8n7VjTdLCJShD;{%mo0kO*l9y^qENf}~U#RA$|z`kSoB$2(A z^;NsoAQe1dNj``~Nz2|=#P&O=(R8R7MYFS&Q^Xs<%qpemL=#m#@ZKniD zG)a8v|B#HbqyR)hh-3u;h-Cmtrq%$GCIg6#0!W$+AR7!IXo=V~<%i6iLs^%W0g@FA zAbZaZpf!@=laSn3@yB*{kqkUnBH7;hn72thzll%d@i=Ytll*e6DmSgG&%S@VREHl{ z9~1Ww&&NxqgHiD}?R(?ZhuUSM=}ja3bM7K3q!!cBpNsg%ANoIdhd=bapMGxW zpBl%FeChYme09rFiZ9r-PL4!oy_@*FH@_K8)5bU+<_oo2ezP%Oee~Y)>nxcqzjB+6 zk4cs#ll$*( Date: Sat, 17 May 2025 21:49:11 +0000 Subject: [PATCH 570/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10937 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c1f6f75f3..a1a7ec6b9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 21:40:39,226 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 21:40:39,226 submission_checker1.py:3308 INFO] --- -[2025-05-17 21:40:39,226 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-17 21:49:06,160 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-17 21:49:06,160 submission_checker1.py:3308 INFO] --- +[2025-05-17 21:49:06,160 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 21:40:39,226 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-17 21:49:06,160 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 21:40:39,226 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-17 21:49:06,160 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 21:40:39,226 submission_checker1.py:3324 INFO] --- -[2025-05-17 21:40:39,226 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 21:40:39,226 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 21:40:39,226 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 21:40:39,226 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 21:40:39,226 submission_checker1.py:3345 INFO] --- -[2025-05-17 21:40:39,226 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 21:40:40,186 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-17 21:49:06,160 submission_checker1.py:3324 INFO] --- +[2025-05-17 21:49:06,160 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-17 21:49:06,160 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-17 21:49:06,160 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-17 21:49:06,160 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-17 21:49:06,160 submission_checker1.py:3345 INFO] --- +[2025-05-17 21:49:06,160 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-17 21:49:07,154 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,14 +27,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29747 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.33913 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71949 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.410375 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419566 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 34a5f9bca..5a9780876 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1197 +1198 diff --git a/summary.csv b/summary.csv index 78f7f7d57..6c8103c66 100644 --- a/summary.csv +++ b/summary.csv @@ -5,14 +5,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29747,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.33913,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71949,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.410375,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419566,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6c759ab2b..d87b87b54 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.29747, + "Result":6.33913, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410375, + "Result":0.419566, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index e821fbb986365feb5dd578e8374d3bb1477aa96d..7230264366ad2a31923de0929d4d526923aefd9c 100644 GIT binary patch delta 2907 zcmZ{mdpy(a|HnTwhR9*$cDR|tHiuhiY0<$J!q}uajuhGyLs8th$%YDdIkd59%2_HR zb10I?sd1A-PH|I2iQ6&f;XD2F`|Ef8aXlW_<9a+^*Yo{;zuxc1Rqg!J`9+ABl)kL5 z>ec-K00{;FMSv-*t6a_zvjx-iuD^lKMRq<)9!U@#zIIR8N4ZcDG-qQOVy($I4wX%= zbFBj%CmKAtc~NKHtD51_Jrwq4J@|L;H(DiMs|_E(kNC7*;Z;-&zB1q|Kd7yKGq-ZL z&{%ZiaJsXr=uy-A<%T5G{Mh-iY1RN&YEz=%(LVkDvebS0b!#u(uE($n!3WuSOkYU? zt*SYtTv79GOzt;D@ut{Wd98ZQ1RDjMB{yD{k`Oml)CFigWuIvrRs3;02Rn&JTLuV+ ztEHXxB*L@8Yn$(BWxc(%B6M_mh56HK#A9{AbfIo#s3K|7nZAG;i81=;rG>}Vmnn>W z9@iwPJ**+`6+4TGS66E+go7=)_beGS3Y-bg`+8TW6s=)}`0tdD-;bQ~>R>Rk!t%O& z88gd*#7^b8!p68ImFv~l%PQ&H^~>Iu^t*mOzUgP^?sU&X^^RegJ zRpuYBm?jmq)qgF0i*a24>-CJS(IGh#D1NEtBprtWkUStgz2V;wxm+)XnhXtZA+3rS zyJ!;2iccE1Gop~#X$>Vf^A~|Z2fbhoBcBdSpep-_XRzf-M_h(Z49GdouyZ`3=hIGP zIPpE7T-rA_7<@7T=_}-#o1kRLBm)En$oP;&*GHr=PCEW^PF&|yi>NWh&e9rWEYMDE zkSH#u4l^$Q*abMD#bs?YYRODOV5VnH)yz{Udc))K2l+a==}iXVAK^^0+~M~E+6*{2 zfUy>SpD^5noCXaU|LhtXS+9Q_knmk6n(W95B`F1ED838q-LWovnt46<`NhPvDndlW zu%t)bB&&8IZ#{x95DlG!KnY8bTEp%p$y^+YE)eFVYh0!s`+q>cRN-n`0(&>4vq`PW ze*|W9&Q&Ryd@>wH+ZD^-=0vO5q7TUl3ozAx;Yq&WIVpSf9x3DDRlIt;EG}J}0B3L! zxp}u31X4eR@+#0r>?L6h)-vUoR3V7Fr)oOrN=8t-1npmxu^nIawokmah#EsYzrg|( z?<*1)M*#@FivS0i7`H;d%)zUP=t=K60xK7wM!YkOoK}9>aqM>H{OzeaWix-*#qJzR z+S5?~*2OZlLCaH3OolY?R?m`#61hC)TO+C-dEY&Sz?^CP3yUVyBr{N*N)?s?^19bD zEodY*tDy=v0vkkA6(Z^wgzbE-|1y;^Y=4i9ZkP78ciF-n*}?>Qs}s80DR`adFvcn4 zKWSX;i}G@ogVc>Br9=n|(_cMDX7a~jJ&Dzp;w?8k8u@nmy9 z(0xfS9tvv5y6?ug$9Su~Vw<6_j{ckPDeEYzx*sYdh@7+ZGNCi4*slHSAsXv;z#*Q5 zXSqnldU_V|H>!aMbLd}kc*;Th7kUBHrG;EppR;Mhk3g_ZW#j?Y_EMn>!XXzojsx$# z1>Bc3|3yw`JUHTR?uVie>Hq~ephq1LB%@HbfTT0SMcv}Y0Qpn}dRRLD_&-DL3U1=< zfm_zY2McN$TD0PRgj=rFIBqLWpeM+L52<0tTdAHwIzlht?Re(_Eab+Xn^@|*U4BA> zJq2abz%BVA{JN4((`9Y>?|k=MZspCZl+H?0*;f-F3ECYnhIqk-1?oLTWI2vPw__|K zSjfe=P{55-3)?>`FcMkj=b#||w)%^(OP;ZHQFl$0n1);` zjCeO@*LZU>&68fAk_?UClGa}MOg zSRi8@q|4Y8Qw$dPiGSH4DjT60o2@HB@ua!j$v`f~c=Z;OHa6Uiv+~>ex$kW=rmB|n z{$qu`gQ}ik>tK}oWH0Dp;CcGf{1lm7Qp1L~5_WGKF_#2*#)Ot+$bu7v0&K()yC+N7T3t2{lY^(;DSp# zIQcO2zIwlqC}R^$7qr{Y)>)ygrdV=~-~(x>BbEPLUb$*EY4f`9$A^MZ7uwRz-kzwT z_Hx8jveoS-_LMYF$L<*hek;Jmf#wDoB#%qI@FUU&b?P7U_}yMRSb=B617bET+OAF% z`|Nz@bY#spPSw#iG47j~wl&+#pihXM3_TWZF4bNRQ6>o zdJ|svk#1u}>H%wN2u6lzlVD`<-fKUs!hwBM0U8In|2<^C*j{yS%sTW@oI{4me~JZU ztd5efV=m-U%1!4me1wNk+n@OMzM8KlRJPkM&5AuuqChjB8v|)VVG%q*UrWS?0!|$L z?2bffrx=kY0~LW}KY|iZIUsSF3ffAVWLmDe_q&SQrO^33()b}3o#wPaM{KA=bwT&< zaq~qi&EqK|67dwkZfc@3}m2_eTq1mhy(F~95XHJzCrl*xOS^<~UzCKWsM z0!uu}+kHw2OzSKi-bt=<_s+IEpIZ3{3h#z57VPd7T1ZA*gK9;Nr&+&N<&a-kVEJaihha4vxyF>`SgxyVPF zCALmfbe~yjuY5LAQDU|=M^ue+yNMsMH~}+!AZ67&RxzLG3iD?tK0ccG^Fv?jWO!WR zTWZnwVodG1dlOcx6b0DPi4Wy`H9dvl07)}xR~RjIAicwy7nypKlz2XpIOa2Q`~f@` z#%8+Xt!W11FxvOs&#F6JYf+okIW+&@k1iFASugh;wAQWj>OE%4SqU0n9_9NQ6I3Q{ zq#J(p-{@$GTL&LmpEdAzS zCj=H|9uoZ9^b~5OgoVt`HZ#A&VYCkWMkAS9^CwP!jTna)h!n%qfE7ywJO(?WACckS zgyHV@8&%T|>~;B(F!7h~+ee<&ZgXUn?U+#WA}9s-x26OBS`G*LW}s+Y?R2MOvtQAH z)F+~GtUf}6r1z(HMSYOs-#+I8#3Uof+~;d|yk-X#;GwvyPt=1d(muxZplO2+s|1z$ zaE`$X;WMTy^3wH#c+2b95VS@|a-cPENzpweyS9(@SX3FRxRrb<~1Ob495CF&m Z|F`JTADL(M%s{%#6g@4-u=d`~{{X5^S(N|) delta 2912 zcmZvedpwhE1IIUKD@{{+rr2nkLqoJCwaj5THl!0tylk2r@;HPZ+z!>I$8k5q5ItCV zNX$Z%c&KbbA%`T7V@WEMQx5OcAMZc!{kcB(KiB8;yS~5c`u(o^t~veibVZPybZD@z zT2NJ(oWCR>;zwl$zYqQ#Ic`ttzvYEd>^}5VF3?JYeFTyAHjNej?eUyKe_D{)%x;Sa z*|v>*E-%l6MEmBRBYUu-&9t2xK@~u z8zp2%Y%HwU*F55czI4}`IptBATztx7_VJUae<=w_lKvS5p!ZfPkX)#K(SV{XeXP>} z!IB-&wv0K3NW-&PwC#IE2zdM1+fYj<{rnq`4SK@NV@271tbVyUlFpxXMb;+s;eR5n zOaS?5>42_j&8eo^nolFa|3*z$`KY#)LPqGd(*Mc2%&cB$ZU=HL>A&yEoQbpgU7xI( zZA!Y~U?an#Armk_n>EhHQn>5H1%LBI+c)toT|E{TgU4i!GZH>%d1{iGd_8UOzDWb! z&386<%F`G6BldlBhjW23Ys_UmHTTZ!_q;;@n>7-DVh_+8E}ebv}kJsVKM zw6j+R(09I1X^>PO;@@`?>o-?(JYPFfv+FxHCrh3r#CfMu*-)a5Oh4w!yhcuS0<}w$ z@AKhy)x+6X>Hy-^2?-$&Kg~CMOEkDP$XiSCJb84A;`&@=pMTE6LN&5FvQ0N)#Ae7y8hO5+U^cOlcbnNU`An8bx1(6eV{`arZKA-z z7R7*^&*a4RJ9~V@zKi|4hu#r~IJ})_Yw&S9p)}<+_)KF97V+btErzDp1AZ~KpcLf~hz?aHRRS(|PJ;+ry(F0)YJ>E& zsk6i{w#(=|?E(;-KQ2kSl?3jxJ0fS!^eWL!F1E~T0hJZSY7!dAMm8P8!SAiaAhCeM z#|)y2so##}dJW#T!zviG*g0Y1vQnOfOD0F?1>m;~c(QlZd2eCN8jy?ide6gfRrSI- zv}>-%>HaQv2fA))X;9qjL7CJ`%R2ZU6At%m3n*XJr2bTx+(3;;D0;;xysS)^B%9iQnqq)g119iBQ0pN3QrBFHL_P>h6D z^oL5Z|Mb}BsvUJQDQ#DBU4@85l+m8c&Yfim-*#Yv| zS}DImU@Hy*?M$jT1g#h(x_f-JVq45{aIVm7IhMoPYTha}!Au`91{twy2@ z7kic;b`5qo%*K`Ns-xdu3&%3pWZ~Z8&6Iw#Fl2d4Pci8~D@pATbG{^KlU34xaxz4K zxl3lo2+!sbx}V^CCb~hnU9*J$0MGB@J)Bu()EDPfP@Z>VMJbk{Etw*?P|NNx8FP(s zCU7&O7^G=;kr)p`1!oJJmc)2&M2b=(%IQWuZb~Jrotu>ZC4Taj4*Bd=*JbT1RI

z7v(m($`DvZPOYXU@{7J%0OsNniSc5nyibk3Rk$f;l0+ak7p2ev7f?dDP8rw5S7vh* zm}ORRxfj0{c{JC-xa)JtN^nWd>-@-JfxvxX&M#rP@Pzto@v?=Xi2V^Rub#8B2gQLB zvuh>&xIHK$tis0-AY_qWB3}7Oy@$xarG7+gM!5liIr-NKXj){`Ce}Z5snRB=gNDr&t2+U zBz|g2Z)&QDH8XY+#F{!bGIt3sre%@^T{NhFKx6>|kG?V>mK1*(EbyhyrmkE1!|?eS zejmTV);UgUJTuw@5LHVR#mVA@vT4@xEU{}{G{PLe&)F0&>1_B!W*Z-;z)F%h&n2LD z6MmJOc_DYxc9U84KlvM#i{~XpH>BAgZPp!heM>{4hZX?AYnY_l++Srg>F5Mpg8mwc zeS)(i##%0|3=OH=iWc^$nEtA=4G5(H@lYwZedZuYw@2p+IddFOGXwW>UXx>6O1{ zn~v4KX=>1Z{SD_s;e}^#t)N>v_Pg&DP-FucCx&*sP~_=8a`A7x3e|(5C*%%b-$Y!N zdzL(7>Xa$PgpFFd+C*v10Lr^X4O7a2w|HUBad&dB-b`uiPoxmg#=FxdpA%JM4s2Dh zc6>L27k(X>Zn8BRc&WY1UVF8vvFJM{`SW9?!b<tR_E3~G&9EGoO$sgL`PI32<591Bsq zxEOjXZ7}_X+LPqH_g6Yijkhiv%}RX;b_e1 Date: Sun, 18 May 2025 13:44:43 +0000 Subject: [PATCH 571/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a1a7ec6b9..2e1192b7f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-17 21:49:06,160 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-17 21:49:06,160 submission_checker1.py:3308 INFO] --- -[2025-05-17 21:49:06,160 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 13:44:38,715 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 13:44:38,715 submission_checker1.py:3308 INFO] --- +[2025-05-18 13:44:38,715 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-17 21:49:06,160 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 13:44:38,715 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-17 21:49:06,160 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 13:44:38,715 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-17 21:49:06,160 submission_checker1.py:3324 INFO] --- -[2025-05-17 21:49:06,160 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-17 21:49:06,160 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-17 21:49:06,160 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-17 21:49:06,160 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-17 21:49:06,160 submission_checker1.py:3345 INFO] --- -[2025-05-17 21:49:06,160 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-17 21:49:07,154 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 13:44:38,715 submission_checker1.py:3324 INFO] --- +[2025-05-18 13:44:38,715 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 13:44:38,715 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 13:44:38,715 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 13:44:38,715 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 13:44:38,715 submission_checker1.py:3345 INFO] --- +[2025-05-18 13:44:38,715 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 13:44:39,942 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.33913 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71949 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73089 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.8766 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.62328 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96013 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5a9780876..ad9544af5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1198 +1199 diff --git a/summary.csv b/summary.csv index 6c8103c66..404ae696b 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.33913,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71949,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73089,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.8766,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.62328,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96013,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index d87b87b54..f8c2106f1 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71949, + "Result":1.73089, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.62328, + "Result":8.96013, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 7230264366ad2a31923de0929d4d526923aefd9c..fdd62687f775827ecb15afb0796dedf76431db03 100644 GIT binary patch delta 2673 zcmZXWdpy(oAICRly3jN_wNyj1R<>vj71<~;vvN6PI!f9)mMtl%gZ7lfo=kK4-pRdR3{eC}Rug9m#tIA7!eg(vB)hm;- zuoamQAn@v)g3w2*6Ukm5z|2q!^V+b1^l+og9(k zBM2X1oqfw)iCeqbacpqd?Tsm8>faYtq97b+rymRFPy8I&g;-6{4zX=P@}>#3wVn@* zK5Y7Xqs-UaCrWlnnfO5O%dw<^`*;hLa-tnDsC_^F81C8%NrLd7o`w1IbhuFkD=Ii`2>zQJ8-Zx{@55KoVld2Dr;lSS3n)@WCD#cO?;=+q4_ ztM)IAujd#e-hC0K?dWyD`j5AsS%>|0@rgp<$xr3-Q^)R0`%#=NqD8U?&dwO$_XiA_A56JxeX33}U`?1u;jM3d!Ck{TY zrR(pG+Sc8uBO&AK&{9z5;FaU_4y#gfILS(Pf2ftUop}2(lYK5m-DwOz@3>xc+_~QJ zC3n{SZMOrJKBo%U9q->;HDm-=1E%Kln8htiyRQb{eiC(%{}enyA?%6?5{Y#yE4kCnQp&ZRM^vnLl^{pIaX331ryd{OoAJ5CbegTSGYv|aVrRJ-EA!fs8wJX~M&+j~>dlE;TpAVn>dQUpAsq@l2@5Hp*$(8At^vfPg#<$$&h^Uxf zy)C#t(0lYaT{k9=8wev6kTpH}!1k2JLs2K#T*=Pu2e9PM1VwE(slaFM$g}fD*g0nF z#siqz_9N%870*jw-{W7FYna?}^Ch7)eI(Z#c??trDi0W(AlJyzs}pXR07n~N+dIWT zAh@-e!g==H3mByw|HpNP@iEsahfWq4XE#*%`niBoPY<)EZK40EFIshMPFW;kT=Y6D z4(|nd;67AF!!@0Y;h0yb$$aoZd@h0O49yk!gGzNs($FGCYn;*Lo}ASof`y`3uVz1Dv6yBL5|^ zj=xm2u})5361#&a{Q2bwQBr^r!t(2k$nvLko{N6xTlcp?%N0vtrbIO`ID6x0SX>3= z(ZQHT{t!$WE1x$;3`(K_RRg}xRQe50;eTt`2f|X{CqpN+eNFHy!pD}YrLVNCr2bT= zWdt!nVZFXPh!(Ie)h{@Rc)X>u;lcWyVb!57j}8Kj{2sNW$M+wSUwTshsmN$Stdl~j=TaWfR4P0A zc+vEQf~wou{aQ5h(3a!hCtYMHfamms%+ViY%9qJRX$A_ikZ-^_Ob>uVz!243+RM_e zN_LtxQ2Ceb#mqON`AD4Lh{q3e)C;|;S78*JlZi=%lr-dxj|t}KNS=E}D93rlD^cIP z_(xR;$;Lwww* zdKCyHf0L_>5e)KZd~8bFs%3kbuH8|cr>Sz7p;jMZFL7fQGB9z3P_Se5a+7^W?~3vN zAfnBw!uMpK&wuCnW!NLQn5e2{GXEY^bi=2e@*Mgp)A95)(|?h02P?CCM69}dLbSUG zMMRc?e+PXr&;nRy4`Nk6Z_EH5g?R4*LfeMgXHQ-@v4F%aAQcg&N31`=mtGu7ZM?Y% zlVT{VNGQlT$@N%11C_;c02Q*~+%63Ku&4HBLx@peVi%$QRJ>#-j&HbaWqlKKg1YJrOR+9d&%k80}Gj{d2`RjN+fphEQwwNli@V6}lHgO@U!VYl`!FH;j`wt@8Zjv*-s z)v(RD`eA?ar!f=6<&uMvnlHP8`IkBR*-Zl>JtNyx3SY0YzDKjKTq;^a6xpH2jqfDi zbk)x55AK<5Duz|G9O}qCG&lTlVCrc@$A=29&^O5AJDWnbe#ifroH+>$(>Snof3mHh z+404&=AhQ0Ji1+Tt>}=sT~Y9^@!PJv`s5ZePMAkrKbcIN0Y0w*Mk0s{Z)HTwBtw;a zB2V=(BhTQ3n0b3sTe$V)AHkg60^yAi+t&4GD&|k0s+}9DQFPMq$2bZrLnI$dLC=18 z*HmL~YhK#|yw~NUU{98Wk5HM(7PhL@ZWK)j-+cSVGR!ZJ*Ffs|huZiL%@Jhd}@2?_MI_RRKDe4+&3>zWvZG?5l5rE{(N~Rg(`T59+tm@t8tKYU*&1orP zz1KDK3>d|6dAM-j#aQ>!~LI;}( z-TJQ5&CVZreQx$|&zKd#3fO=+cP!J2J58v00k>7W_KXBIokrc&=asExa#%K?Jq@yG zh+^|gp3aG3+LMLu-xZ&4n>bJ}1z@iRNa^iiQ*Qwk!g#p}; zyxq^RZV%ah{YsNfB?T`ZHY8SQd^?4lNv!HHd12S*zJin#(%$SeIrG$%mb>rB^dDI- z2Uw-jRrt)*{{NH~Q^F;Uv_j)0^d0bvw^i9Wi)9#ei^k7y;V1;4Tk)) delta 2673 zcmZXWdpy(oAIG=M5H%s^;4rk! zWiFK*S%ldkm)zn|L_~yg9e!W+=kK4-Bj%DDF}X{pH~H{`IOxN`Z~#j;BFdfkH08Kd@J8s>}u+qSOdRr|`Q zD;FezL?+Eg+G6;zUX{h0vj-CjTk1ZPydVHh%YQzfvNPSMY^Fq+uRg*iVIde1BrQ$& zUx;$HH%Cj6Vu{d151%%R=EWwAkWCr1)D&Sc$)c1Myw@8k9DMtmJhrmi@ffi@VZUqt z;a+8@snjgbYqKeb)14(=56)~K?mK@Z4&%4YEjv!lic5!3IZ)g_%l0=I90?Jd4`ju( zjW^2R7-TCOC=P@t3+0Q7m;)UDPn!^aXi-@Uvv?|wmEl!gHT6&lzv7wyTBTNba+O2< z7nVU+{;e65O-D%uahAgGPzM??lhA(L*Y^Iw<+}W!xC#AhbSGXYO)WV6=a;Om4V$uu z85gr3pN>zdqDDjvD0tS60lbG&V|A^Lxw z@2g1FjjXN}L?*3C{a03c+f0?3*;|tVJh@2a2LGCd9e$rO4Aj*AuBT8ca#r{5+FvH3 zXn1$ETb{WvDwQrkX9GFcIaFE?gYlH*EBl1HglrslN~i$m?yQ;&KARpamj4fnBTF=I z_{QoPa>XL zoyftqsa047sTf?y0Q1EYR*?pgeR!s7L@kH9p1bFBhB_fN&~v53RlM{v6V{puqpDaR zHds%h=s!ksj^cj3OfZu2$1cH#`f$$(6*9!G+)aIhkN1v}t*oI=ZR7lj;W=_-tK|)}bGGvcX+4H+F}azf;7AifeAO~JYItHE zXRg2~l-C~Knx9mP?<7G^e5CSPvGdcv@-$&t^<{5V zn;diOM@aIqbc+D!9R+U@R*B5J7P8$GZ`iv`#n z?+tiqN6+1QOj`}VEk3IYhkv4yKE|Leuk>KsPgM=a}FqeSD-yt#+ts9+bAkXk}y_t@g zi}LO&qOGiW;O^(PY{{jMS1sgx=7qgf(ma-`Pf+jPb;^){$}yUhBq?UogeyL3#I0K6 z&Vzb5YGWdN6x0jrSEQOo+N#a)bD3by@sYI%e#9fmiAmO2LErxMcH{Gr7vmOkIsmP? zyah{a3#Styr<49&4}Vu6Rnp4OMdG*D1%yQ6%n0pD9RYSOsx8$;3QJU9xUiO1{&nHo zIrA~w=LMf$=MA|6^H;k%Bm2eW=0)6^* z`65Y&_XZKno?KAWie(FMZnVgEV}jp$f3|+Sc9r0k#_^6r}7p>R$fNLwH$YyHZ&|RMSp|mj~v6y;4+s*O1=k$62L* zCXJo~UV7NK+yd4Gggwkueh<*NzS*^XxT z4%w~Oy*%yE&2R}BpkI;=N?#nJ5rsVU!5ZZ7r85;_m7XX3SLg@63uVrX$GA z6-2=sutc;7><9VSP>3_%D6=&l;T$F1V5|Wmd(ezn#%_6kCUh-fjBB;%0ra@Zk(1bx zP6VZ&$9De0Mz3fq89?vc7UW7>S;R7=N_R3#UiJ!P-gBm$d(N zL4T?fYu4<#yg*pYU=sUFiH=7^n-3YRl(v$Ajl?PspG@+}_zJfM>yEa%1L8k0Xv~O235_@55nQ&WIQ}*`z-?=&~smm*i6WdVV|9GK4=wpr-jYhzIB z-F4_7|6B0L!jQxdN7Wd;oM!SSaHX~J%D0XPaY;G7O47o@n`3A^%FK)B@e$FNPhLsg ztkJ-b5Mi6t^CUdd=kokd4rD19*Tg`*Twk?%Cv2Pf@mJh&6cT8)uQye^K706&4-q4% zJn14-3gnv=8WlwxG>S;~Xdno71Pp2EvASG8#f|>y_u_A_XYMm}jrFKdi$Wy^>6Weo z7U-hWvxHjoL&VK?oNIn6T|DK3Ke|bzAWs97ya}u8{(K?>XiL>>8AH|pDHSp zbnbz6a7D_Sm`*&<=deiCs0-&Cm%@%6{HB7a>!Vm*B!=L1S`%3|kTXAfBxTmr(>U+B zz0oN%f%OmW9nAak&$Ysy9J6Pa>?w`KQ=DMDhvkL>P<(2r(dBg3O{aZC7onPu>Vw`Y z!!O~xcJ?}c?0w&$0hN>?R#TqT>#mu Date: Sun, 18 May 2025 13:51:55 +0000 Subject: [PATCH 572/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10935 -> 10934 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2e1192b7f..36890e712 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 13:44:38,715 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 13:44:38,715 submission_checker1.py:3308 INFO] --- -[2025-05-18 13:44:38,715 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 13:51:49,813 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 13:51:49,813 submission_checker1.py:3308 INFO] --- +[2025-05-18 13:51:49,813 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 13:44:38,715 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 13:51:49,813 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 13:44:38,715 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 13:51:49,813 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 13:44:38,715 submission_checker1.py:3324 INFO] --- -[2025-05-18 13:44:38,715 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 13:44:38,715 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 13:44:38,715 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 13:44:38,715 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 13:44:38,715 submission_checker1.py:3345 INFO] --- -[2025-05-18 13:44:38,715 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 13:44:39,942 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 13:51:49,813 submission_checker1.py:3324 INFO] --- +[2025-05-18 13:51:49,813 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 13:51:49,813 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 13:51:49,813 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 13:51:49,813 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 13:51:49,813 submission_checker1.py:3345 INFO] --- +[2025-05-18 13:51:49,813 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 13:51:50,871 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419566 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418149 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ad9544af5..59a3ec59b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1199 +1200 diff --git a/summary.csv b/summary.csv index 404ae696b..a8b2bfb22 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419566,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418149,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f8c2106f1..334ab3fdf 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419566, + "Result":0.418149, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fdd62687f775827ecb15afb0796dedf76431db03..0fa32c96e1259241adbc8e970a797b90e8323d97 100644 GIT binary patch delta 2989 zcmZ{mdpwi<|HsXiC>lcdoHmTcCRuCl95NA`Svj|nLUTt<5pLyhQ52mV#>6^Ip%ik8 zMB>}T9HMc@(4BH7a!Ag6Fa3W1ey_*pasBzap0DfudcPi@N|y?k;$TUbp-N$FS(xN` zC}8iyt!%HFPcfPGjJT6RvKv&;s#8lOPm`^JZ8X1qTx>S-v}xY<^1Sr08tz0e*0`@| zkNF*~g1`CcriHUw z^PJl!3n07PDYn6w;~3kpxS@3OSjSxXFc7Q&z2hB&xv{NrPlDSu^#hddHoCTQ3}b(YEd$7@SUj-t zamY{*oK-(>IM_65;Ps}Cc6l9b_QxW}EOJ-P`6y!HnM;bO8tF=uv|ILBUA4fCL4w}R z!B+u$Om9Ri9}>X68AjUTN$3_bnqiHn;B9LY=FaW*b{49mIeF^FToohFBUCFJIg{l> zX`z(N$c0(`Nw=F1se9aa_5g5GWY~N0HVf7Ex$8rww_p5l#x@x>zK|&AD7vD4e!lFD zwOZ#4zIrBYx3RK%ApS*VbrEd$fmp42^ou^|T{P4)RXQdS(XVf$u|sDZQcw8V;V@@* z?SOCM7^W8H#0g~}zGtCfeIuX5f628Lnw6oP-1Qe=gh~3D^tOv;hQP~c#B0T+Q-U%| z!@>oD%B+045Dnx+Q_!Hf)ue`Fd zt$5fyP_Wbd#@ohCyRYt}fvAt@^L4B8v{qCW8tR)W%}YdF$vg8uI0Lyk{{6)Jn_^j^ z1tr3Y@$Z4$*vIRoETD{U>+lruDx10cfCWm#moDAn6Nqu`av?;I-Q3o42vNo?glKfQ zSl3 zTc_Yh!F>540=WG3SXS9zC8JnT*KuRvbpGtc{EBoW>t&>Yv~34?ti1SaAzTnAT+sj! zR%KTf-{E6)E}kE_O=GWGMu-t;D@m!r9}wkMOoa=Pyk{-XJiF2JjUHuxwU5f-<7ej= z?5FcrFXn~~BO4dg-VSw}I@6D6+iE}P0?0aP8*l+CT~Qjvf#_v=L7lVEs9vbOpD;!t zoYWy-;xKSRg;zjrmZb_0dRLt%0`|I`4R&$AlggNFd^Nq-Le^N7$_m0AoIpP)j{)VB zWCdazoK^`Fk*F=eEJrxLC>Mx(KW_S>Xc*e@5d{IA=jj66G1cbR<#y_47)?s-nvR3+ zq?<|0x}hZ_j{7q|(>U8j-(b)G!`Zm3=g;s+-(B!8F~+&2vXI9Y1BB0Dwgif)pIC|= zMh2B?fx-Sk!a?ovW!6*-O+r>>8;>AGQih?xg(Bndxq)ohI7aO>K$h}1z z>s7_t^sUZKI}sfB&ubX9i)`w*2@Y%u92gGUV~&H;@y{iS2_o|Vg*qX66I`J>`5~k+ z4SWMgM~S`kPY3ULRw~UHGz6b+L%#9f_;WdGF#W#Ql>EMrsM(xlOMEr~AWKvgcy1I+ zU!dS?`9kzY@h&kcd+~AoYAh!~`ii;!D)4S!qa_W;hs2S%Ob?K}++6PV_0+fUJX9^V2s+ z*v?yUex3bc(dD0o$LNm+$K{7@9vgq74qs=cY2Qc6A0c4>1o5`2!p{-Ji^&r#0Utd< zKK?$HM^f+My6-&LA-AjlYJ1b&AVC~>CPeZpe*b;ne$Ud5KvW&89xm7^coTb5sE!zVq_pj}nJrP9^ zpLgPv1pvMIg!9tH_-){-dGERS*nO8AK84vPTe0~WV3A>{J4om_GPv(_iDc$E8I&JB zrUw&~BM>XC7rVkqpgBc1wf-($6r*0Obg^H|O$%{YN)V+81f1O?sJp10I#rl-iOZko zE)mp|I<3tG2HnqsJ72GSucn4Bc`XlDF2}1J|$c$07 z2OC9&kIFV?f(`Geh%y+F{kP29=OUic1Ii4SOixDHpE^TCYR^0T@;P{UmJ)R{=m)rY z(I=0cGgMozN&wcfH?RJ^Wcg)veDvBzeo%i?Ml`bRM5Su((iPg|kngAJ#P*1JGww$M z>(-!Gt^M<1BbE7oY<}p!{g5h}!&Zux?lKwps?BNQw9L0lvmP7sfp^iOb)yDtQDod# z!D)Lc6vj5tRW8FQz23FWIsVCYZT2Wy9cVlEZoh2YkEk7wtM#$(wI%yvD~7sA*}D2~ zr2NagHr0_wo`(qDDj`#QFT?+GzkTFc)Ehijb$O1}y z)~>T4?Q?5#SV_C$x*5pb-jsx6J&XNSweqZw!ehXXBVeX=riGha|Fs7G4XR`<{4ZHhZT(fE6J qd(4^XzT+KhWf+|n@&CJoFJjCf|Nr+6={Z9%>^pi|61{uHz~{d$n0U(o delta 2982 zcmZ`*dpy%^AKo0M92%n{mD8-14b70KO%cO#ddTu9hpjAID5;R&910yAHuN;)OtQqB zQV}`yI)&+R4iA=Nj!6@Fr|13mz5lrXyzcAzey{8L+!aohPQ}-Dh}lVYsh0-s@D&9D zs$ugXAJgK7Vk8EyKNR!am#+w(w=}zEq0Tgh$;H*2s{tE3Xg|B-g^ z-|tbaQ8ZDd^8~K%+2Y$=R@V1Qdt={|w^h~is}J(A5(2%yoX$xY8YPtKkT44)7e=O& zd%3&TrE=3GQ9UJzlBk-sN|%7}O0MDo+oK<-8+fKQ%uu+l_uZ$t0j~sc-Oo_5 z<6O5z*N>W$}f0I@f<_fl^#_@&f$o_ZxI?p@1gmvDWjd>xv z>KZ%#9HT3kUFcA#8J>R)vuP8H6=S`<3Uy~nGYA+E;*wwGP$crN90{@R&eImnYlVJv`(3F9u3y)Kvp zfeJW#9mlwIz=yNUo(=L$x%{~3YtZvZ;;-asy}Ju$uWQKidSQq98>G1eOf8}rlrnMW z0=e6ugb;)W3}j9O7#N-`KJ2P~+)A}Sk>cijZxC_MZ?J63p7#7un)j@WS1!GKc-Wvm z*|$nlX9M}`;FjK9-?TSjW~{g11#9fl2ydoz1_8rBT-w2o{XaZk(V0d5g8{!py?aWU ztOoN+-_Rkk-&-qyc9@oV=B3elOk$f@n2_gxw)w$DX_zEPUjUJY>4x-SzjgKx{iw~O z#Y`bH$&Sf4NJ>{z6yMXkH={}fi5hgF?lSu8!NI)2$8}JX|NEq15F6B&|#$=X-SVH}TVnWUya`(h~@xgc^Yd$;Y;dKaFPT9OA zRp_r=N--U_CQ;GOZhZQZ?1-DTe3ReQkvrfVTK4yHwJy?B4{0iYSI*W72bZ^+jRMHB z?UKXLRuH?bI?T|3I1l{_?!->xtXhmNcfu0vYnQ^FYZr#QvD#E_wzE&9Tl`0%siXIF zRsMWO(m)chzxU<_^Wut~$Ic{V(k+KAwI_D#IC#V#&bka2$TRXi89OSI;SI&-5=0$F z!KT&?XTmPh8C>ndr%{C7XnsvUfX{VVw|#%jmY$)td)|{OVLIi9DsL?L^o(_jC!+qu z?mQkY>cV~4z@)&F;n@=lglZl_D*BN+aIWE#sYQgC7+NkRZ-bt79XZRx+n zm`p6g5|YVr2bDR*5AH&1 zMg5Dt+JRL8RxG|7 zR-JHwY4O{d(^1>bF6zxrRp|j|P#r?CNSx4?rk#m*x!PLwMAj|SVrma(BLvO1 zb@)w=czsaN6=-ej5fhtlov89?CR!&+g=wE05Nf%ji!-`W{JSdh2;217h;hIoVoAvF z4J{g>h|7__N#yCzv-Op6BNFW=oI;DfRV+Ksu|vHdAH%(F5OjIR{oo1IlqiJhsL1$# z*`?U@hxJ|}@}SD%qgW%{zEl=T5vlDW`-wOj;XcA4tqX#VrC-bW?6^AV;9H0j z)KIDK=J@@v6$>Ey)R^^36gg4zZxz@n1FdH`l(2jJCHk-hIGTccH~E*hF0k!AF@w4d zHOec0=c7PC`_Qd4`&h@9{++iBdsOza4|pb!$=kdo=Q6f z%BRB^eMs0@M~MT{-T@yfEU~R43jF~G(aa)?GzCQTI!m`l* z^n`M2tg7-I*u^V&cs6PpoFL03)rop4xwFU<@@EHgz$LptL>7b`0TBjJNz5eh*$nqc z6`gA=RzVD2KxO!9aeKKnM~w5T(O|xD9{-BlwbXKJ)Ne>(q_a^4LP!UaDImnmV3HYD zEb)fppd-XRonM+Hj?{^GK9(^FM9wlX4cz01`AFLXOKby^0_?qqs1fAF@p7Y&Y%lx>z2k!=Mr zZC-%kM2M`ic)YL|3>5L5uo7wu^oh~-Fh${7iumtOmGM6C`a^0}FUR7(&Z#TiDl+3% zuO9Vf-3paYZyNU>m^vijpLe=?{#^QzVG9 zGwYKxsKt_i=7pF;(+ z&o@B(?_q(slnSaArSV$GLTZ)XwW^_3gLU!QNax)xOa=3dzrLK(-yJl<856&snt#9B zS?ie{bQgN|4O8iT+1+!6GA=r`=HQ(D9(kvX7i8-CN)A}gFh}i7hpuQ9Ky&Fpapqi# z0b>PQ{Q+je|Ku0~YPtl^l4q96QA26Qpkwu&;bQy)ADPk@-yY-yuw){x@e|*#K@cx4 z`v)wfDL!)R)pR@Ra{8AvP;_zlo|9MRu6$3%3{&qt%I+oIZERM3e^Ktom-R7~B0s!T zzQt5_tA98-YQF;?>)+90vGnb&CMol{?aJ-ckK?qGW=Tv+!q`8vy32W6B8mBGTZpIN p4?~sd8NW&oAzDTL{e6E&o&f)Ee8-Ob0Y1!nbVLI(cnBms{{>B8jAsA< diff --git a/summary_results.json b/summary_results.json index d3066585f..0545990c4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419566, + "Performance_Result": 0.418149, "Performance_Units": "Samples/s" }, { From 9a308cb8ec373157104feb9a47c6a1fe4d3f56b1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 18 May 2025 14:25:32 +0000 Subject: [PATCH 573/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10934 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 36890e712..1520e9797 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 13:51:49,813 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 13:51:49,813 submission_checker1.py:3308 INFO] --- -[2025-05-18 13:51:49,813 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 14:25:26,853 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 14:25:26,853 submission_checker1.py:3308 INFO] --- +[2025-05-18 14:25:26,853 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 13:51:49,813 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 14:25:26,853 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 13:51:49,813 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 14:25:26,853 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 13:51:49,813 submission_checker1.py:3324 INFO] --- -[2025-05-18 13:51:49,813 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 13:51:49,813 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 13:51:49,813 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 13:51:49,813 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 13:51:49,813 submission_checker1.py:3345 INFO] --- -[2025-05-18 13:51:49,813 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 13:51:50,871 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 14:25:26,853 submission_checker1.py:3324 INFO] --- +[2025-05-18 14:25:26,853 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 14:25:26,853 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 14:25:26,853 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 14:25:26,853 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 14:25:26,853 submission_checker1.py:3345 INFO] --- +[2025-05-18 14:25:26,853 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 14:25:28,030 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.33913 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.33125 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73089 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 59a3ec59b..44acfebd5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1200 +1201 diff --git a/summary.csv b/summary.csv index a8b2bfb22..c2aff8ece 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.33913,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.33125,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73089,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 334ab3fdf..a9ca2e919 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.33913, + "Result":6.33125, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0fa32c96e1259241adbc8e970a797b90e8323d97..cafd4c25494844116f85797942db2c1ea54458d8 100644 GIT binary patch delta 3651 zcmYM12{csi8^>qJk~KpKSsF8A_*=q+pJd69!DRa-L!vPw6A>+TSCOTxb2Ty&WlJKA zEm?mNLrRp0Mp?4&q+#Me{m%cK|2^-y=iK+6`@G-h`97cbJ-7Dk%d-_hf_z}xdS?(A zENCMrsKVa55Aj+By9JK`T9(X>`Vv=XhJ@V?(-YJ5mwz*`wWj2wcZd>TIFk%gB1Snd zEqcpC&hm_t&YoAGTqi|&-m}5LpH(!wQ1SJ`M!*Cv;DfdID&`gW!CRFk%+AfIfeUYc z1O`;7(UV5LR5kuA(5LHtK6&I_=|;Lpe0~%MKC7j&ig6FtN$muhFxk7~H!*G)yA=NX zh1!Ud$)QF@{>D!6#VsRe7?DBMvOBNQyVdB3uU(mI+PK}WmQY;-f7j4z(c#Qe%K5i) zF(E^4DFZ!|1M?x*uVs5@_V(IrGeau3>$GLapB?q(K-Tm`(O16+S<(KH#)&tJKF1Q) zg}?f3WZuJMahic<+!aHYOtk=~QuUfLF}LnYqs1Rvotc4+C6pJZ-FsIov=(!Y^n!ND z?8~to&ojq#=j`Vfsy5aK$JTyC_%GsLPeqz5yPb~OTspIm|I?kdiV8&9u$XaMRo~4Y zKhEjBbV9XPrSPKI;L;II?eFzOI%0g$l9is&A{S~0Z}SHxc@twty50p{X+AG!Xit3k z;J|ug7-n_pL;KEYNk!Yloco3mg<>ClD;;fPiG`^s)=7Fu>i*DT`t}f^#OB=g+m4;T zjpA~_g!(Z%9QJ&LGy)#|`#$#J*Df2Fgi#Y*qHVwzCXt$_kAQ1N#$(%m2ggZhl>1}h z<8zO=9{~;SFo|qT}!wE&>G5tPMZmoN3>pSmzMr1rCjT>EDV#F=f-Uh z^B`ggBlLs|FYv&8K# z?Cp(z?wP@FEp&bg^hwW&_qUu~T=Oc;wfzEsfj^fKC+0~Wz2oeaFpT`0^@a9B9>uHh zrVHvOXDsh=Rv7io*W8>>hp-eR>UJ+>|}Vs`yP z(`<`#r+X-L`C>7>lKChd`4oS;iM~MK7rA8V-rSTg=2aK>oI9yiUcL|US2kOyq3&hC zbm3rp5-p$SY;EYy!xM@xCB<(a7R}a-|a%8g_Up6gmt`}njhIJp@I47Vllg`kh z=q!(27@MVit`gg}Da;X4AFRv}Qg7Y@gw+krdIo4c4k8A3p-<2Xrth_3UAoei5DQp3 znwo%3$0F|Df?7CgI$$z_D~>pbh@gol^A1}{;gW}{$EXxa<@0;r#$t%_HYL&H@t&95 zs^u3?AUi*#F%HA;-G1^XmzV`9Y4@6S7??Zp*mTNP z59_xtRN=mbyCWhuw6P;eIkzQtlShAc6YOZKubcaR?OWtLgCgHk3i(cUi9Y6Y=TYJH z`roMoPt{Wk_nwFs!T|_x^g;6hKXRJ+3@btD+vy`@H7{C*oeqOARpFxBF7|fsOv+o>Y-&oDKx#eYI{h4*j4&VBbKW z`Dy0X;dgn+C;7i%wuC`4xx$xvz%Tm_--7akJ12EjoZK`v_hIJuyLjF2y{4;O6ADg< zYAFXqm^#&BuS=Oa(Qe>m8e6*?YBdszcE$4uowB8kKN_-zeuBrvw4jS1Squ)`8}S}m z)2N%gqu4Pv0Tiv2UDHqVq3VB&OhkDfz*C_Z9DiTCh}nSlV_cXiM)c(MT7Ba@RX4by zP@ZZ}LxD&gzW=y?_B#O5YdGja8M%J5uDA6=)Nn0~*2WpFEs%<${3()`csE0X5*R~q(?>E#9$lzAoJuf^AK zpEb0u_w=c+220&N`ba9o3YEq$&QziwJfc8@VzN4Tedrj4LEa}6olvQ#Mt!gk4iJ1r zsFjR=c7gAx-n>*;vow9@@~@}v9jD`IXNv|)&9ZkjLB#ktuSB`<_mP%F8!IV6fRA`|5pK`H@!elBw1gy{ z6!E`r5DqTT0n%>0@8AO4=v2|MsAh%klQ)3!kL8?+`sRsD4n1DT<|($M9o!aY#`fvY#NWapetNYh;x}YgX=dD*I=o>)yuEe(GS3A;g@;c zORh@WoDe(}?^U9aQfQge4jwwdjm86)X{C}808L%F5+Q>G?B?$45lsE0*stJ4VurE^ zC7;A0%v6A6jyWNd#`IDb9wt(y8@z>m*xDxwHTBO6p$1+t=ajN@JMym4jL8QwQ%W>RJJn0l=FHM94-Ogs4FPf-ERQJKlA+w9jmtw`oCm<{;rwIM_u&r0QA7I1~@2!<{E;rv@AqrA=Q z^YL(#geiULVP69AV7j|>jp7UPNC)qG&CE0VGiOpm9Cx_ZU*=kW-nox!eLX_vTR-V{ zg;=x=xRGzReT$YLa=TnT;|4+I5iwC_U-1yVFc;{o=b);9S^QweCXz)YH}`+-B!SJkTrD&uw-(f(s7tP;n5NM{$h8 zpiTN<>1KTycm80A2Sdy%D1*;A5gK$mN3t)0RETHHaC}}B=2T!i2aw&VmN(inyTzGk z8$%OBtdH~6KfR8SF&!x2RER+`o&1Z6iZsj6Vx{8X1W-)1~H8jo6nWTQ6Y zlX?8;ZxuyLmbEg$Bvjs2EN_woz(@*`eM&K9|k?+u};zy3f& z#z7NeA!lgmSF4g@LA>C%YCO*YIg5WNb9r7dkbN2wKsG9bYjYtvXaaF`$Y3#Y4cOI8 zZ=zW^4;w5#;m>HXo)1U1co8#wO!c-A%^#z!ED8O*@S`NgZGm>3Rur>KL&+zKrY^e&J*=@&SCWea zU>>ffhk~uzg+Oeb}>x;?1-y#FrRKLML57&uHr)UJszvS{;^H`$&$NW=@p*# z5!v?mcKv{}qoTE&7hpcDUk06~Y)2zQx=V_G2`2SgVO0>9l>0eDMFLmj9z*wcpLmFr zX%0E*GK-Q6o9b4*vn*ok`hI-I*g#u$q!#~$5FpCdZFFyG-YR`_O5Uyma+dE>EFtg0 z$g|eZ76#|db3EcDm-np{YtBdp6kEg@e{&EtjjO*jb?qmoihy{mX15;t@ZE%8C36X9 ztiqo795{*J<3rn-@8XSk<3{Up^R^i(s2-mG{nu@w-Jl|%ld2Jv2BxW+iN+j(KtlYq zu3ZoHBVPkAs!D-tstbY#A_RS$dGI!OP0n`vK>mYxu5%f?)!IN*Hdp_YhMw}BcLf&9(SD|0^ue> zASWQ8u!)7qDJm_X{{g4k++1fQoi3P z0uhr$NtjkM)psKsWXVLE%>adUq{pY%1&ZfL8@SG^2}?Idnja`P)ns)|_{MZ<g$J3J21oY$a#hmcINFIrb1$Y%2dfa7rd=ry+eCA+l(69!N8Jm$Zl;oaua< zdG;nnOZH9oc?YtaZ`*IF}&2qhmc&FeLZquLCS4u5?n#tV|RAEn)VRg(H7>wRcZ z&*es|HqV_iNr}7rnTFIuk^1r+L$0RO;RlTf;nseXh_FtRlJFW&gn|47{iccG7~p-P zZ(`mEA^&=r{Zr{EpeRfV`;n3O?)~wDcR`4WwZo6wVb^6te|8*}Hi{y-TWNZv{P_SE zuKez5YFM_dXq2fa%kyUW)RgkXHB7u?rsJQQ! zULXF5u{!mKr5@9C|9fap%w^dG`kqecsgA(oFz0QxnLT+uVrK>|UQ7Xw?oIw<`YFXR$Fm1OF0HKpNSbdwC8awG|YGR?)5f#26s6 zP&tEN{jT@{VV`}@s9Jkb>U!!o($B8BMLtZ|GoKVyXpu=WSvD~YHkP00+wRbEGwwL_ zHi&Op27M_Qr9Jxkxb8!RvYXT_ooz+1(8!#jmO~bo8ocYO0BY3@HpQ%E&cFfg>iPMf zO3J}oiujrR(S@_)>rUVFf&7|_ac$aU+qkxErg#cq*}Gmd9W2WE&?6R^e7HB{KG%BJ zU17a+Lqsw{-UcC@h4V~77_c1b0@aR=b{7}gQ9Wep0~u{jeK=* zKBs(d6`4ThBvv(L@}2*2o#aTy85-AwW|SF557B9YAQ)jxF=ihU*Zq!p*wJMXC?_TklGym?My8EWC;WgK>Q*jEdh&Q90j}V-08GNxx5@Tvw zaz~Wrpsm==UYqrE_fGjF{YHM^x=0QqNG#?exJGBbacl{9m7lAIUvyUc!-ZXN^gNu^ z9db_dZ45ZguMQ0a#ROoSPlskeEdh0CH|Q-O57gClS4=wx(vIpTH@CCQe>u+v!ut?G zExRHXUGQ84r%wi7UXsM^V$TcUW~`57#Bcw=D7}ldhM}T*?1fDw0x5;p#&Azqkle7c zN_y?wVjv~1V81G$nrdeAQsP}c@z^PS_%M~P$`sS0M1xSwi=a4{W0bBMD31SJC=Sp% zU!meGuWg1L&h&C|xHqbF@A_yc%iaI{w25=OBr1Tiyj43_+t{mSP)+?|gS;1R`%^AQ zPf`bQ$H(D%xFdHjQCu+3mmcmYm{?4kvdLT5W4i#ww$8|Ty$i^AO&Tq`_?vOXXSwuO z!3{B(jFUr-(~BF@u!T4aQ9~US29RcsRD)&c<51z}(VYGwC5MKw_21rbj&QP-?*2}&P-Kij5kQPt> zD<@%CeS7;OCRQmV=;s3qaMvIbg0=XYni2d(qQ;OYdOa%dRp%?WK>)GQ?oxfVXI2<< zcWvX3^>WE>NYRaHSo#%H#Ia4kAlxX#5BWt``}Qv@?sT$@>{y&eT})SC*iX= z*w}m8SuZ7(p%PX(W2=!fZb3dgJi%4^KV)Wv>NSE(*0HkhxLCW+y7D|Kg#L=l3~Q?6 z3rROe^20Oqk*E`TXijTPwGkd^!7U_pC=bi4C&>lC*&)UjVUHe|Z|xuy8`{m$D*yrs zXPi4ZQ10~sET4v=%uB)XU*B`#2TXK)S!~0W*oL7|L%L{AM@%Vq1$J21U$ljeV6mgw zA~FTjwm5j&aw|K2JN?+LPJC6pE!edi)^v^e4xui4$jLtVS|P&#lMi446Ylyl zH)aLjbyLg-DjmVB5P;yp?Sj_bIC3K2ZP~Nzah3>QEW5Tk6xRJ+(1X$a@0JtU@n7QL z=t$~d)zbBayD2UwpRgfeYlT~E$bpyx#Jv6g5L=39f8Avu*-CT9cQ!fIgeACK2*`yh zj82=#DIUwp(*RyKmBg?h4m*hx<`4}5`#$4G9_QURuZuU&G}cksXR9q@tHq#9*lLLx zd}+SHF^-4zicJ}Pd3)@rbITELuN!o%!aJF-1*M7C9AZ8Q=0wi#U7ib=eG#tz?dW*b zU`(S3miBBa{UDjP|Ch|ogJjYmxk|unLKws+HxtI+A;`|95Y{Wdth3{KKjMmzf9`3N zu4~E+Xm>!)w}0$LOwBmm(((D^oXE6sf%XJ`I)Rv}_yi_$0gL*FP4A#8d~MnEVvA`z zeQs=e)RR)rc|QrNd~^XrA2eL;{?xOrA7YjD4AoS=C)!X98&S+SW(JJTTYU&}8c~nD zRZHFLdzrRF-;qL#k`=@p&@LJu4EVNATx0m8i0vJ;A8tP zm`gQdBYJ4;63ZCRGG<(%^;rgzI+S0A@0|K%dZB_evOC4vbCHi!ODU@&jf_fKuc?-=WbHjna(NCN z{tfQfa>}#G2__zplNDl%*BHZ34l9vjsr}7HfF~WNj7E{f<518_&2DvZ&fT# z>%29uxKM*X84-Jc=oHcldtzcPTl#6r z?AIp(d-<$8H+Kwv?9R>JW0sJ{K4-_kx-Zq?rHgiMTP#fZd}~PRiQJ&{VFT_@x__`P zoz{}t_-eeO*8C-7cpX(Yyz_`iz|^yram)M@x7J*k8-b4y!~0roio;Ryzi6)3cup=* zQ$?~`Rs6lcA-lUT5){EX1Pt)|s38=;5`CngK@Iguk!Linc4|N`Uq!8n*B4B5(lzmu zX^s|CCW)-7w14f2Iot5kY0;jt7q>zGEt!rs=Jg4T9~hO4F3X;iRy?nv=6DS|@bYk~ zc(*~D+VrPZ$|>>7X+J?8`qNObb}E38yR^2BAQAMs06(B5(1z->wCf| z9`lNpRp_IU^h8UOR^!6`uqB9)w87CMp{tLGT&|7K$Lk>fo#1IZ%%-?fxi=|vrAhjM z%b=HwrlV)f?PiBJD<5B#3?y76=~*cLGcDk#@w#L+OjvLb19MoTO7TvVr2eIj`lfrr z5)s^a?eDy&tHWam>IyceI=#JevsH55nCjy%bnfkHtQWI5>|V&-Jk}HIxjbssyH~BM z)X8z^zvCSE7wHT=0y?PZKr=y5MIRamepS(iZh%6nN>cxADii{-=72ziA^(5H#tncr Ss^_6dFill~dqnx*%l`smo51`4 diff --git a/summary_results.json b/summary_results.json index 0545990c4..cb9583801 100644 --- a/summary_results.json +++ b/summary_results.json @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.33913, + "Performance_Result": 6.33125, "Performance_Units": "Samples/s" }, { From a49fdc8cf95e75a54794abf972ec462842d871df Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 18 May 2025 15:33:51 +0000 Subject: [PATCH 574/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10937 -> 10939 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1520e9797..0eb85dd28 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 14:25:26,853 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 14:25:26,853 submission_checker1.py:3308 INFO] --- -[2025-05-18 14:25:26,853 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 15:33:46,470 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 15:33:46,470 submission_checker1.py:3308 INFO] --- +[2025-05-18 15:33:46,470 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 14:25:26,853 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 15:33:46,470 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 14:25:26,853 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 15:33:46,470 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 14:25:26,853 submission_checker1.py:3324 INFO] --- -[2025-05-18 14:25:26,853 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 14:25:26,853 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 14:25:26,853 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 14:25:26,853 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 14:25:26,853 submission_checker1.py:3345 INFO] --- -[2025-05-18 14:25:26,853 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 14:25:28,030 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 15:33:46,470 submission_checker1.py:3324 INFO] --- +[2025-05-18 15:33:46,470 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 15:33:46,470 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 15:33:46,470 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 15:33:46,470 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 15:33:46,470 submission_checker1.py:3345 INFO] --- +[2025-05-18 15:33:46,470 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 15:33:47,535 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.33125 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73089 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 20.7698 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.5179 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 44acfebd5..87577dc7f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1201 +1202 diff --git a/summary.csv b/summary.csv index c2aff8ece..aea80647c 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.33125,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73089,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,20.7698,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.5179,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a9ca2e919..3eda05e41 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.7698, + "Result":21.5179, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index cafd4c25494844116f85797942db2c1ea54458d8..b8ad9fae7f37a6111544b18199ff4d98fc951d5f 100644 GIT binary patch delta 2012 zcmYL~dpy%^AIE<)hMLogk|*0;Hiu1zJ))bKLuG3>8`YQ+YDB4|@_0JzH=~do+J+oz z4x1hsF^8TK=}wrXn|X@qh#Qq6hYqV9?w|Xg=lbLN=X$+9ulMJ3eXq}TcW>+7`a~mB zhNUp$v8K&6sKDq$r zQgjY}BEI+~dIVk;rvL?lQ%}pV>nb_p@V(&-gH~J-X{LC~Ij`J2$1B3y>4~&gLVPn` zx3zBeJNCh3`R^#FAg4Kz){j`e2$B)il1QKdNtYpQ$`p360b9;4J^~bVz_|q(*?JE~ zfavhxOQF0v3;Bh{lwA#r!nBDSBib`dJ1 zD<=~uJ7sV9XTm{q8;yh2m? zsUY}(%z`nEA_T3U6KS9%nxG^aB$2v$WTZ^82JBF-o zH0q-NwrWk@Zw}39p3eq+EQtW$&qF4}N!g)>9b(Fl0gT3*x*g#YV%2+3f}}NcBDuTc z`xJj1+Rt9WY$@2maFnO&JTcnvK*<&gDSjnq@K1GY0=o=bF(|l25S(FJ&y+v`UTBPYKLt*Yp6|VyM0+Nr zhVKET7X}qO_X%eoO}76ah5aD)e%p29_y|^;o4qb`qXx|jDp<)O8@**&8{5mt zTa{@*;0rsTAqcc9tul3YHS+F9HNO!&X=JGm?UM#p>*j6M?XSMsP{#Kj~)fICCCUY!AEbNO4=lL zfK|VRQRjW^g7wx}M^lz(q?yZ?S((d$f+x3AQY94l!AhVIXVQZ(St_|IN1U!lWNC9-!lKHk2L#R6QFa4%b7Gwz1gQr>@+|csNV}pi zIJp>!ab1RinjwA@D08Ko#lqG`9*u?qh>Bey-U!wUYMm$EpQ0^dp-k-^8{k-`E zKbZ}#5O$5vRSeLi^5(jiEp?=yGr?=wq2O;X{kGX`EHd4xdebzUJ%m0|m{aAJ{a84i zD_=0BfkTYfOfdQGKf1FQ9{HpZM>; zh!g~=4%b|oqut*{()Q~fdZ^21HH)YGbbJ4|^*D?AAUCFFUp5bXSal}RW%^a;RGX?V z_Fn9j@u(Jh&4ioH!WaM9+V()c@&o>k^7}PW9PzGh#&yPI?BCk(uM5Yo*}d1esjy0R z+NdmoPKE39H$&fxlhr*_{R%@MlK;*QzeZF}XHuMCA2N55^@C0}emr*6Lf7Lex^S+P z?~JttNAllQ9(cTb<*zi$o7oI{#rmyHl9^>OMuS$bC9aWez$$~h#Iz#Rr#3hJL)~97 z)l{yZ!O5Chk(VGT-6Z8eNpF*_6SG&etc<**kX0d!xFy7veY6b z{sxtH*_ z*?-H8c-2#)T~5DoNnMhRK;At;^7&%eh{yHn8T|bV^M5$(gPHOfj@w|@__?PXGV_ delta 1989 zcmYLKeLRzEAHQcd%WF}2#;J|&%?O=TNNV2OrZp*!dy<;1HtDJKLWz3^Q76J~&1q7~ z+tG`#bVSEXtwJTwlt^c(C2y4qWivdt^ZfB#pX+n|bA3L)>-+Ni{;v9vJ0X>^1RT;l z_9#{!M+i26i*86qPkPUjp_`TwAFQ=GAH=1DDZCZ_GOWW2Tfh(R@=+w0h>O7q+-7xSEzj6hl zHET^(&e{-*tCKMb7ca`H7%>ac_e%gA*jCxFo*98_>uRWr#^xSj5_!zJ0LLFs@eU7^ zu(>oL8XHl5z%p7sfw+@r@FYSawx|NX} zfz!2XvB@v|v5cA%y@o7f@(os@AEH3;b8i8*#MFPF1C&*JZ>1{vK-Ct2g#g9gK(h4I z=B+GwP?tbJ5a2M3myW=vpEYs6%WE7f&87ZR8+)Bf?n;(j@1lfV33$hxO6U8CX0wfX zQ1i?#lmN6w441Ud>?#7JRL!0QHC9TV?GWjqXSR!bb9KdA{qV4+#)Xd)*TCZy8a4k% z<}XLnY65HhQWVICIaPHw#P`+7n)@zIl-5b}KxFd12jIw7W#LB+xVA2Y;~#^%D1yE! zc|~*7%QXq~Q~hq1J$oVEqMQiJ8q!Tc?*lCS%sFrKyu87z_~0qj`e@Yp?V%m0^|c_e zPsyLIG!+LT<3--uU!|vvlPer^k3htWywgPevTj+)rAM_oYITg$nd^roPFy5m<`Hne+(zTz?HkLr%+Fe94!WVBs#!!SV?6xd?|33(AV2fN*aGC-1Jub12 z94`Mhme}%o>%aiZvfz>q2dy~#c*oRNRtE@6HnQlT1q&@X|EZzU>@gq%u|nAeQWvIB z6r8l0M(;4s@$SeyS?#?1J{yVmt`v1$*PU>Zae@VVA$|#4KByDcmlRa`J?f;iWd$5j z02Hb;v62sVwu-Xv&9+ z4)kB@{?NUzFD1RAv}ZXVOj`YFS;OHZMgW9{$A&LY9!KqJ&!`p$Fz3ATtkC$iS zd6YIHaZyzNhT9g_)DWcsQIHcM_sl_Z<2ZKfZ|2#CODjQ=pDl$f*&@9qW)P~b@5EFY z?`+jsxwptxlU)rDh?L@(4sjuc*O^O#j~_G@L~!v2J%;eiBh?`j+L8~9t9BZC&Gj2f zP0iq}@Y*7L-Ftti*WTNlZ62wmqaf{pocXqpH^3u$Ny^uVi(3I+5g?&rTkzPqVaC?j ztQwfpQq0F;TU3Xz!W8}_2>uAsRTV^l*k|V$w~a_+uBQjwzKUSar7Nta!TRkZZTm*F z@=@m(Z__6pDb_3GUnN63b7$F8s`%;4M~8&RJHvE$W@o1r{&wG6EHKXy# zJN0heG3npF8eV>8^T1ne$~P;yhIT$nxnkr$W1Qjevk7(x|BE_0`0>!-Gl5QRHch&F zpQKfB=x*bjOf@Iq_V|?WcHIk_#xjO$w^6QQpHSt$$tXSZ(0%D{qZ)3Kd}Sm&t{NU$ zef~y6Y<^}IQ$|ImM*po=kuMIs5zt4|m=YC?L`ptw{MSVtPHwum+f?5Osj5SU!R1z9 zNf5+zzzm;eF&1t)DF3)GdckaqYw@v!*T@RWQuu1%cop=(XyIDUqO#1;pev?7aYN&9 zt5QWq`$MfOcZkpEG1lJObo#r>I=Lz;A^SO{`Eqh?XJ{~q6&~f=<8}ixAkvB{@onX0 zlZ1??3_m+?&oZU1yVT%7+GWhVwsjXM#D@6wVMA2w Date: Sun, 18 May 2025 18:23:23 +0000 Subject: [PATCH 575/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10939 -> 10939 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0eb85dd28..66d2056c5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 15:33:46,470 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 15:33:46,470 submission_checker1.py:3308 INFO] --- -[2025-05-18 15:33:46,470 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 18:23:17,551 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 18:23:17,551 submission_checker1.py:3308 INFO] --- +[2025-05-18 18:23:17,551 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 15:33:46,470 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 18:23:17,551 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 15:33:46,470 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 18:23:17,551 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 15:33:46,470 submission_checker1.py:3324 INFO] --- -[2025-05-18 15:33:46,470 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 15:33:46,470 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 15:33:46,470 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 15:33:46,470 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 15:33:46,470 submission_checker1.py:3345 INFO] --- -[2025-05-18 15:33:46,470 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 15:33:47,535 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 18:23:17,551 submission_checker1.py:3324 INFO] --- +[2025-05-18 18:23:17,551 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 18:23:17,551 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 18:23:17,551 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 18:23:17,551 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 18:23:17,551 submission_checker1.py:3345 INFO] --- +[2025-05-18 18:23:17,551 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 18:23:18,907 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.33125 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73089 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73896 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.5179 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 87577dc7f..c12432697 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1202 +1203 diff --git a/summary.csv b/summary.csv index aea80647c..f2e433337 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.33125,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73089,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73896,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.5179,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 3eda05e41..2d16ebe1c 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73089, + "Result":1.73896, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b8ad9fae7f37a6111544b18199ff4d98fc951d5f..c76e68c9e180452272480088bb7862f097d910b3 100644 GIT binary patch delta 1742 zcmYL~dpO(o9>;$nLWoPV)((~vdXl(BMyBbiPJeM(CGx0N%Ul{&5`?%OtFxK>;<9Q= zTNO=Ky`ZSIPh6H@Q&vnS5=UJ-bgj0oHO@5BSrKufd(N}ZKcD|z&+GGff4tf@%rh$UQ9 z3$G>B=-5lK^qew_(eMAcTsW#9H)PYn1}DpZ(@Uljl)!buqTY~IQI0iNy)*PBIDxe& zOXfg2&lZ)xY@|QvW6I)Z7q_a~>s5L?7jUC-spzY& zDA_=~KvOgFcXgd+7dNWM-Pl*F$5F0J4VwllYOxg4ZcZl8^J8!rJ;DhLwN^clp#BJo zMf5z3kiF6To_=JW)=~fKP#x($IQ5Xjfx1@ar_h>!8s8a)rifLJe~d-@Qe3|QiH2Nq zhVi85J&?OMW{jmvbcd?&cgEn99<2lW`;0ITC~I(G=#R7iI4-fm6 zZ)!G~O{~P}Px@hCcs_hi%!>&;VGz<|D4Hvv)x!PtfmmB19Yp(x_97VZ`WZvx2q8j# z6h;cNEj72bYVqe*toeQTNdHI%Uf4&y9NL-8X@p>Py&uqj$L-XMT0_0Mthcc#tv^!5 zolePz@w7Yi8OF|TC`NiooU4gml16StlzQ4wxqGRjEMO`I@L|aBI)ki*4^$6YP^FR> zU@9jSNGU~C7S27q2)4?bvqEO9ckJ%tIaKZEMLw9UG& zzc2bXC7O&WS?l6Fw`&5&NtmEC+hTA6D_+Fh)-^VC%5<+Y4_&rf5=VUbv;-WI<~W_< z$~@kBpN5V{$5F6&&52F+Kfln?a|be3S!F2R>7b-TnC$xtNY=>T6usQ)uz@tn!t#~rsN7ow+2c=BWoi%l z7%OwjKe}nz3x_ptGT8% z9c}}JpGfaC}-M1r&=Ojqtc#|(78v*Pi8c!xW_xxIQytj*JmzCv6j z1|Snt3&(FDawXQBuh7`9ttB7X+IsQ>H0#w1qRiPBgEnR)+G{$T6WpFMOo1~z*1t5n zxIy&F+B?zbY#qBqnS{u?UJAe18eV&keC^C+h-z92VoxOi=eCuQeHO1}Yd5duO z#s)QekL831R-~3EBWE}lHZOfQyT9m&6&*#j%m`K`Y-pvCxT?1P;)sf&kdOx2C&guD zzvh5z_jgdI6b{+%1@rxWTMi!98Oq!UDuX{iA)(eO3T7z`clL%|7hZGrft?plID5g~ d3eheEyLVfK0l;ws0N4ZkRs6gu;YpXo{{dTGDeV9N delta 1742 zcmV;<1~K`&Rl8NNatH-h6`zQ7vv&w$2!FPMk`t^}B~YSC;zMirAYwNAOJy2M5igGSt#qWGK8{%0HkXcfOeStJIvWy7FY?(qWz=o zmQr5H5mxIGEKwuzr2RuOp|pQU2C=MQ|F8^T$*dZ{(q#XzQ2?<-VFfde{7Mvirs>JHy%~T%tkZN%MzfLTUbx402h){9zg3 zl4N{|rAho@9RQXl??;3811%951U+^r>yk3?*jd-$vFn<>AH?#Ti|n;5v5Ls9i`tLZ zvhCu5XO~N-giG{CJgNPVOenPtRN&3+s{Xk1Z20>RA z%DSu!PqD-df>UhG($8aDeiK=?mPJ-N$ri02ty_Up_@%5}GTjm~Q6BN6^g}YClzvDC z$*ds#undq%GC-!u`C%OZnI`2&gYp9{5gCM#3Ca&C!-FjQyFPG`ty%f`K7VBHsq=HC zi_TB}f~*(1Wv~5sC1{dA$s?s7k_n~sLo$eF1?h)n08NqsG)>MA>j2O+DL)#NA83il zAZW@$S(laJrb)aY*feWae!dpXC#kZvEV9z3S#*B1-O^52qT}fjy$a189;AplMQmSbqnArpfrxVEjN!#4a163C0g8!(Eefjk{*e#?SXbb5E6@ zD_vB6@?Fyp3xBs}m(3FAN25LBOXr7VlqIDf5<)gBNIxtCY%*mBUYj*ZKWr3$)8zbU zaDJdAqSKVBER=Ow8Sb0J41#^LX6fha;e5)Gc2CuxOI=idvVGIiet+4^vnA$_Muo(e z<`2mzOR7I4gltw&e^>_CWa196X;OdKD8Q!4{Lx_kKubiY32cJ-L&|XDtYZG?n$4f@ z!{)yFKU(@f%J)j1ZrUaOkBuUUFZ~~qQI-^dNC@C$83`xLNI2Pq08ZN^0I^XdoTd&s zHV8n_60vE5#|~v(T7M=tP7;ER)7~fl=kF5PYguHKjngh_K>4e*zE|=)CU%JiWTQ{w zOAUx*lqEeN5<)gBct9)zY?6$8!fBf{Al3n(Y0`jf(14&NB7=a(4pm)NCAUp>hrqUJ z@0|toZAJE4)>mc6w2Kl@wqshZ)+NVTyTk&r(I@ex1w=B+l7AWy2_cmgG$57%DoF;Y zG+98b1EA7m0c~%5pb{(~cGWZ z-~mhWK_pTFA{k{#3y6de%L*0{%K(?tfApepr7@+(SGYFPRQT`QNnf zjn^M)myV`4jm4kqkIVB8Pk+}|-dTTmxg5uh^vf(hZpNQy**%mmw%Hhb9Cm?Y`_H~A&meuUx4U$YMu*bW7v2LNCulk6lPe@??N5QOhMMdW+pq@hq*Nh$&A ziAqQWQE{{EZEF$#$Tp;Xdz_?ATPY`w>zVy_cI9GukWKUp)(NdjlqD&OKrzjQ+Lh>| zUd0O(IZqX*jaE>iBRI6I?h0lI(-z*WHekH~7X^@VgqafUy*C77$MztpOM*iMajUJQ zK9qKce+^~dbO(5rrVm(xr<{6C8C=gA=2%ZQG>Hduzy_;m!K>yo0B{z_m5xQ$eaEP zP@abTn|KBcxo>x6*C{#y_mdJT kCL5v$0AMBo002+|000pH0000000000004l2*ppK#Xi94 Date: Sun, 18 May 2025 18:32:25 +0000 Subject: [PATCH 576/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10939 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 66d2056c5..1e6ecba5d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 18:23:17,551 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 18:23:17,551 submission_checker1.py:3308 INFO] --- -[2025-05-18 18:23:17,551 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 18:32:20,807 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 18:32:20,807 submission_checker1.py:3308 INFO] --- +[2025-05-18 18:32:20,807 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 18:23:17,551 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 18:32:20,807 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 18:23:17,551 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 18:32:20,807 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 18:23:17,551 submission_checker1.py:3324 INFO] --- -[2025-05-18 18:23:17,551 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 18:23:17,551 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 18:23:17,551 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 18:23:17,551 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 18:23:17,551 submission_checker1.py:3345 INFO] --- -[2025-05-18 18:23:17,551 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 18:23:18,907 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 18:32:20,807 submission_checker1.py:3324 INFO] --- +[2025-05-18 18:32:20,807 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 18:32:20,807 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 18:32:20,807 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 18:32:20,807 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 18:32:20,807 submission_checker1.py:3345 INFO] --- +[2025-05-18 18:32:20,807 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 18:32:21,798 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.33125 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25756 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73896 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.5179 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index c12432697..f8d65c316 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1203 +1204 diff --git a/summary.csv b/summary.csv index f2e433337..0c97db765 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.33125,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25756,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73896,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.5179,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 2d16ebe1c..fb5e18bb3 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.33125, + "Result":6.25756, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index c76e68c9e180452272480088bb7862f097d910b3..947b6630fe3b24fab360df02f86933f38c2734bf 100644 GIT binary patch delta 956 zcmV;t14I0~Rk~HMatH;or6ZYivv&wf2?k5Mewp^8v&j)B0)Kh`#4gc*Z1hQdsR5CU zvZMz@Lda$X4~S)eO_GsMIBk;##5w>pO&X958W6NZWDxM!p{mQO`l{@hc2NS#c1+9Fy5u-(msmhH`Xs)zfJjDJQUf9(q_Tnr#4^80g(`5S-}Eg8NibHHE@tMX+Ufg;L@Z4*`NVIOT?xrKlI`p%DS`+4zd;g z{Exlo8c;gSkyKCee`;Aq9=LXOQ$?!=??yLA?JG)2* zo-2`T7e3}~63=hq(|9~yH2O()xz?52*7axKU8=(m>yL?hh-c#^)4?eJoA$l&`a|u~ z(e$RV_F?UgJL?ZGm*cpRewoF`&G_>yyNB|{HXDP_!}SN{HD>csG9RU1 z)}!a2)VCb>PA19V$JL~k)6t*H_{SgmKX`{f^u3>cZs?yH$Bk_1_t9*1t5NbV*tAZL zX1$yEyf?oYO&5)EJj@nqwftsdzW(UF)z@h3$qUv*bW4y8Re;y>m)3HAteagfwtt-K%p)zy$NZPh+02LY8}6Q z6(?~L@F~aD%zit&s9f$%x8Y5_#wiwJ>9*K1{=>F_i#&hC3IY>< zEHHckrgkKv6H@RIX@6Zi00hU-f&v{pCK(}>T)l}-ksW_mz6r6GgEErdp__~OPYnc!0c;bx&@zh6s;_bA^u@6KN z1JGmKv+_B6erZ;yF7jfY=JS*+8nU1lvw{}n`!MpR{{mE}A$JqcfMf34Pf-7}EGEkW z1+t|hnRJuX3y5TtB{d)tLMkh0Kr91Pk_=F3vVd3z zK&8n7+TQp;C0Ktz?6QH!4rN_ZhFDgyfC>xPcPyVIve&Y{N?3vfKt*!2_1$ zgGi(VL^8^f77z&`mK7`@mH{l8UjqkOlLo{_0WM7%kPR9Tv_x#0@M! z&;Qtat^uXP97**g|EHFG{U2?$!a$e2C~G?oy0i?CtY84ydu{-2kPM%MP31hj=z#G98TaziHnauRqi- z9Zhc6cl2+>AfZvU@0BY_l==JY0WJUSl>NCG%1G zWj%WSNqx(4?_`n;eq2pzIUW7EjDP&0|ATk zXx6)l&wKNm(R9%m$HQ!)R?BZT=If8%TYa4-v(+P4ZnN<*Nz-I<|J`le8^rVcyX@!P zFiFz;Z}LmB{RqR8zW}oi7T691gC>pQlj_Hv$C5(18LSJSG{zXKw(f{LO|@xo|2O1`9W>CvkG`yqt`E z-)DU}T4R2Kzm~7>U%%(s{agB$02tUPk=-2>rc@BvnwXc z0tHAQ!0&aF`zTuh50h6ZJOP=Llqo?0hLhGQIsy2T5h^AdgC>pQB>?~cPyzq|5dZ)H f0000000000fPvYQQYt Date: Sun, 18 May 2025 20:04:54 +0000 Subject: [PATCH 577/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10938 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1e6ecba5d..2ba78ff76 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 18:32:20,807 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 18:32:20,807 submission_checker1.py:3308 INFO] --- -[2025-05-18 18:32:20,807 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 20:04:48,754 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 20:04:48,754 submission_checker1.py:3308 INFO] --- +[2025-05-18 20:04:48,754 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 18:32:20,807 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 20:04:48,754 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 18:32:20,807 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 20:04:48,754 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 18:32:20,807 submission_checker1.py:3324 INFO] --- -[2025-05-18 18:32:20,807 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 18:32:20,807 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 18:32:20,807 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 18:32:20,807 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 18:32:20,807 submission_checker1.py:3345 INFO] --- -[2025-05-18 18:32:20,807 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 18:32:21,798 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 20:04:48,754 submission_checker1.py:3324 INFO] --- +[2025-05-18 20:04:48,755 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 20:04:48,755 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 20:04:48,755 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 20:04:48,755 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 20:04:48,755 submission_checker1.py:3345 INFO] --- +[2025-05-18 20:04:48,755 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 20:04:49,740 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25756 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73896 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.5179 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index f8d65c316..b0536459d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1204 +1205 diff --git a/summary.csv b/summary.csv index 0c97db765..7883dc9b0 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25756,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73896,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.5179,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index fb5e18bb3..c50b6148b 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73896, + "Result":1.74523, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 947b6630fe3b24fab360df02f86933f38c2734bf..8b6da59f721055789cb954d9622bc322ef2012ca 100644 GIT binary patch delta 1741 zcmV;;1~U1&Rk~HMatH;nURbDevv&w$2!C-4zaX{?r7D3EO%hKUKqM1N1Bhf0$_fS$ z%K((jtkDumlL5p=0VquZkOlz=S|U14Da}Gzmz5!u6$Bt%vjDWiJ{xBoj*Whh&h;3g!>X z0GA}=ODs*|59A0!#V&iP127B=?7XOG6=e| zP}XH-c#0)v5S(IbmVO@N@|(!AwJfsINw#SHXx$2&pzH~zTS6wvBc7CgNG6oh56K{z z6{H`Q0WwJj$TT@WtOFp^r2J@5exM~HgAg)7`5|R^kY#_@2M)3|D?i_d%zr&~ey()U z`N?09^+LDg-AnD2ph^BDkCc8$CX~_-$sn2)q#u?6G)V@~G&w)413=TH{Af^qpd})M zpeYMwT~>yhCh>w`)2vzf`C2req{`N^$V!`L(fQGKOFLnSj;BkMAC2mWFO?sXQI>Rm zNC?ra;QX)*ph+@-rb+o>9e)6tCgVqg@dGUpyKIOi7(b*8cTLhY?wU0lKi>zmXv--2-&P4{jdzM$&?*jIrJt{d^C?H#Jym}$by5Av_DxIsWq&WvmY6>p6%t>X zKP00pss4}VHsePi95iiN&R7?0GlTBM}zqTEfJk2unFc5DZ`Dkiut2!Hh;bk zoBQhjXzBkb-z#~#X_xpvHi{&^^nXZ3SyBKZA%K%*B%CZG;bap6IBk;v#72>DnmXv% zAOJy2#HI-zJCt>4nSb0kNeDJhd!PKDze{AVWsy}jPP?c9<*(BEUdj6>c8LaLqfg>X z4TxlvB|RV#LN+USKr91nl8k)9X`3`4)&Zbt(tvExfS@HJgMh~lRb5skw@r43z_w}c zodxu5MfO_OS7pbvixN<_V_L4(CC6F2!~(L>C-J2PL^8^f8h;Q8A(a(0AeI3tNd~Ai zSwO4Rg$3+8mQNDdYgu0>+)}>`| zkgf3Nf9yTifPd0qj-+~$|5MA_DxF`8{*Sg=VW3N1l(n4_B+(@CrT;@R%8~*Q2_cdd z1R$0HB$-+RNSX{FHVPnVGJtF_fS@H})07`Fa}H%)S_Vi~Fo5hmH-I)shEGCrU&SBW z*+nw&T#016@G)es%kvITf7e#tS$}xB9LJ6H%Pc-_#-C@|J(MrD*%*8tu0JTRF`JK) z`6&Ie9zFl0zU8=gGD!wMt|qmdj{aQ6KmO4F!8`n+@BQ?1L;ut`Ze&Zpk7lb|jgo)C zrgd^O>ptDY=e_yOXu4>O<6*W?tK~Nv^YusXt-el^+3G8|+4z{GX)?M0?l$fX;(7jE z_VaF-Bm(n4Zo)7SgztTd$oIrffQnj4klKn< zkt&s{Rd1HPfYta%w%d@muM;4E#Hq*i%zit&a=F;6Ciw;DrO_42GlCM(+z6?+6?$JU z(>Y3fU|O)oXsA#J9xdv-l3U7+gI8xPxFEqN5u`okwnDoQEXCOK9Vq6r=+IHz7^hf> zrQ2eE%lJ3j0xk%-#|i=yEHK;yQ#%sT5h?hHv_GyH0D@y^K!FY(=UI-=-T+Sdn+=0< z;Z!no7H(Qk;^^LcIU3ux&DzOejrnu@xq5jUR!ybuPsIV%rQno1V8OV$#Fs@pT*G`= z#S?ENh{rDKH{Onm9Qr^cF#tWpJt-fPr|0!064eDMW+|Da`FvdvN~U!B@YRpJ>OTY3 zamc@kC%_^1^(UzR0JAJ6$^r$kURbDellv%J0S%K^DLes}lawhz0fdv*DLMi6lMyN= j8|`5$fhGX}08jz|01*HH00000000000DytllTs>ZJ+>@5 delta 1741 zcmYL~c~sH~9>9Mn;vGtAtLBm|;Dux?T4TPzC-+Q*hliJsp0< z--_Y|H}t~wz+L~zwJSWAPszY#*ctE3)vtj05cL>3+UX47ICC&czd{OxmjQV=m|qG2 zcQ*a8GLKnWE0>wb479SJ(t)=ZRdX(vV(1sJ_G_`lj z)>6z*-V_nJSV1&EgEmFSV0eF3_fm&0-gB!I?1O8PEh9FW5h~JesN;eT1?$mU6QOE} zbk_QpX``6J*bGF5T^CxTw(3pnWay~V7+hNJ0=59Rkait7Tl^et{0>>~6KS%jEUl%< z+9EE6rbwQ~EOAWaR|)*re6i#HWAL%o`6Pp#QnVg$*HA|eGR1&VOv8o|ioJhZXwtDqOoo?LE(X?I&g1afAE zx=iucg$nf!u2+M5U|z1lxfSuXf9WbNN4pucv957QhsixC1?zaQg{-O8XxTp;(cVHA zEULf69v$;*uKL_vL8u|WX<@Ni!RfI#D79~y*Dy_;7MX_~K%>0eoF0+!+H7K+u98$u zW}98;i`!f3I-m_{~;A(U>>r)fU3(VuF@VAa9Y>K8on#HZ=Ldedu8&tIQwN#O;QHt01RlN&A;j(g z$=@p5*ve#Q0q2#aHS(I#2I;Jc(rrd4;I=&J43n?bOy5s9sU=JJhkT%E^l(h!S&G{l zrlp?icZ_}t)ZTAf&QC^&i4?39$j2F`Uk8GU@w}N#WKgn>m|%T#`otiXk3Xr(*pmoJrkEGweQdj|stQP^f5 zddp#NcrZ_zxYB{VIJ{TZRByD4Z@#sWB^Z}cPxSZwm$r1KuqLfPR9UL2qr!T!}uezjS-r<*~Kbv<}1zV$pNVWB3n3) z=s`&5seZGMS0auy`yqaE%3<$0`sp&N`|5DtO^-0r8a1JDFn?(9%cb|BV2ublm6eg0 zerW{T*KWW4U`e>J<~`G^Sgd@u^8IGqt#=vhkAsfA3umz)3$y2FeTi6eaQ%33`ouGmX3`Ea6<4D6x`c zvL}}n?CXzDstzdY1OCM%3m>dJKh$dJzb7-!fQ;19K5f7 z)Rol!A%i5+1e^Vkq=##h6KNPDx4MA_@svbryX3nt^3ba zynDRap$-R&@W}=xNZb-NaPN75Vgs$+c$eEiWdi2s-KXYFSzzc9050iC*0S-yS+?)F z3RG=Gig@3BlW!!vOP!+c{XOx3Z^Y~C^h4IQ2g&W{r(Q2#90>X>sy@tXGJAMU7yp-U z3s;)b(_)H{Q#Xe*;S2q~Ny9VsozIXyDD9qz{|Z#!Byw11FpmhT@Si@Z^YB)Do*>y` zblpW&xvdpcWqV_HH#cH-DDcF#wg8WlY5nnopmqBY*e>W>2M^eND9^zYb{86SaEC2J brj9u4uU&-!zzHn?Fa`b$?j0p`$}#GH Date: Sun, 18 May 2025 20:37:52 +0000 Subject: [PATCH 578/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10938 -> 10940 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2ba78ff76..72256bb88 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 20:04:48,754 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 20:04:48,754 submission_checker1.py:3308 INFO] --- -[2025-05-18 20:04:48,754 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 20:37:47,295 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 20:37:47,295 submission_checker1.py:3308 INFO] --- +[2025-05-18 20:37:47,295 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 20:04:48,754 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 20:37:47,295 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 20:04:48,754 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 20:37:47,295 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 20:04:48,754 submission_checker1.py:3324 INFO] --- -[2025-05-18 20:04:48,755 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 20:04:48,755 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 20:04:48,755 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 20:04:48,755 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 20:04:48,755 submission_checker1.py:3345 INFO] --- -[2025-05-18 20:04:48,755 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 20:04:49,740 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 20:37:47,296 submission_checker1.py:3324 INFO] --- +[2025-05-18 20:37:47,296 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 20:37:47,296 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 20:37:47,296 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 20:37:47,296 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 20:37:47,296 submission_checker1.py:3345 INFO] --- +[2025-05-18 20:37:47,296 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 20:37:48,258 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25756 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74523 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70913 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.5179 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1167 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b0536459d..e944bea9a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1205 +1206 diff --git a/summary.csv b/summary.csv index 7883dc9b0..ae93ecaf8 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25756,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74523,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70913,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.5179,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1167,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c50b6148b..92a583919 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74523, + "Result":1.70913, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1167, + "Result":22.9236, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8b6da59f721055789cb954d9622bc322ef2012ca..dd80d4a507ed386d22dd76d540829e67d8bb6edf 100644 GIT binary patch delta 1756 zcmYL~dsNbQAIE=)*Sw`udRUpb0L{QQCTiwXP@Lc}Th2tYi{gc(PC3z4>JQ0o)JGo0iwKCF#I!pf<$RCq3v%mNB5$c#Q97ReQs5TnQ}0$1(=R8avL z3Lu39*6glgxV)4QawI&UG8w^5yQbbhtcXHHnhlj}pf6ZfZ87h}f^ke_dC)9%gRkf5 z@-1Q+ecBVqULk|Xt&(a_Q~)BZD)GV`T!aF3qbvgvN|#UY>XS6g7)D3-_Sg+&62a95 z)f`iqcR1}*TS$B0#Q|v1TACwMJ0a&KJXJrT7b9J>4T0ABy#XBksla%}BG@Y- zGcs+gXyHXeqW75^pp?&}SwlCYD0Ao}8w zh|#P_<90N1TE^5gS#d+4)e6#jwkX$v*MKYJFpxK6FCz_W7*1PRJVsty@t$r&V#n?Dhz?`=EG!>S;ke{e}3aHD9X{|6jE{|Cpb{-AeS zJYdeFw;SNu?u{V10;#BI$Hi=-5U5x;v_z<&L>O}sT1wV<7~a2yE1&!Zo&B(QPcD$JvmK* zJDcVHcvpAoCQk3@#BAZM>f^v3N}FzDK&$;hFfaN8f$@?)4{!P}_sNXnqM2^$FiP&3 zEXwNM`JiS8OBt@|ctm;cJn4yF28+9VqGPp5;lUX!ZT?kB<$3^lFu~|OKb}j+z6P#i z-}Cbf*y3SF%SF9ms<;<>&9BnJ^MUK4ph2Nl`LbWv>ME*q6&a25&#Pl*b#fQsnoQ?} zjHkkKbAV7$!5PWb9Eff@-AXsux}9=|^`b29B4CKOxsLr#zbsW?e99c!N$$0r`3}^& z6?BOx5el#6a^u1t+RF==l~Ot8*RS8PB8M?In!n;lR!3Cvj>KhK^TD9f zfnvl~sY?iW(jT1@9x=*|H2xWlER!)|%~o6*WYpUAHk;POmB+&c4)k#w)|R@&RQz;s$ZWljF@m~KHsap%snXJ6wcPqz6EH#Jb+6(X+ zUvd3uB28;_eO&s`zb^fwQsts&#q;vde&j32^`d9&;R(mC-`BnA3AK$=33GA6YU#Z8 z_GG(f?vg59F>?Eauu}PKzL>dTbIBQmwOAtLR)*Ib>yJav(T?CyX#b~MDas7WeCDi= zOMM->2;fjK=pX!F;VwLm8n9MeY6S%lmbHg-Hzi}`M#twV9X^BDb)%O#5%%|mc9gPf zi{xe3)o#XJbGv;9>}&EO!hYK2)mpbgOE4Wn9If{_-UVzk%1_BD*|OZ_X&86!Z`3a# zya1!Kwe^+0g=rw)GX7briPwQK6I|b+&@b*feCqL$jb-oyY^AE1Zar-g0Jkkn_X;6o z{C&Pj_JsXck$H`sHfg$=W9O{wyDXLec1nimBF>MzSz2GHKSo6Sy5AliUVOtQA^ZHP zm$t_cGiNM0Sq`C{_Pve*QqNltT*?dh&hLSFyYz71A8MAce{T8rIq}GBDkAX;{iZNB z^!3uQCxf-x&%>+&wDcXYnyHD>GA+CXhrMs`zh4O2aEXRlLx*?!z)GOP-6Ys0XneOf pYz?w;#ctmmD;5R-VFmzT1N{GSvlYJ{xBoj*Whh&h;3g!>X z0GA}=ODs*|59A0!#V&iP127B=?7XOG6=e| zP}XH-c#0)v5S(IbmVO@N@|(!AwJfsINw#SHXx$2&pzH~zTS6wvBc7CgNG6oh56K{z z6{H`Q0WwJj$TT@WtOFp^r2J@5exM~HgAg)7`5|R^kY#_@2M)3|D?i_d%zr&~ey()U z`N?09^+LDg-AnD2ph^BDkCc8$CX~_-$sn2)q#u?6G)V@~G&w)413=TH{Af^qpd})M zpeYMwT~>yhCh>w`)2vzf`C2req{`N^$V!`L(fQGKOFLnSj;BkMAC2mWFO?sXQI>Rm zNC?ra;QX)*ph+@-rb+o>9e)6tCgVqg@dGUpyKIOi7(b*8cTLhY?wU0lKi>zmXv--2-&P4{jdzM$&?*jIrJt{d^C?H#Jym}$by5Av_DxIsWq&WvmY6>p6%t>X zKP00pss4}VHsePi95iiN&R7?0GlTBM}zqTEfJk2unFc5DZ`Dkiut2!Hh;bk zoBQhjXzBkb-z#~#X_xpvHi{&^^nXZ3SyBKZA%K%*B%CZG;bap6IBk;v#72>DnmXv% zAOJy2#HI-zJCt>4nSb0kNeDJhd!PKDze{AVWsy}jPP?c9<*(BEUdj6>c8LaLqfg>X z4TxlvB|RV#LN+USKr91nl8k)9X`3`4)&Zbt(tvExfS@HJgMh~lRb5skw@r43z_w}c zodxu5MfO_OS7pbvixN<_V_L4(CC6F2!~(L>C-J2PL^8^f8h;Q8A(a(0AeI3tNd~Ai zSwO4Rg$3+8mQNDdYgu0>+)}>`| zkgf3Nf9yTifPd0qj-+~$|5MA_DxF`8{*Sg=VW3N1l(n4_B+(@CrT;@R%8~*Q2_cdd z1R$0HB$-+RNSX{FHVPnVGJtF_fS@H})07`Fa}H%)S_Vi~Fo5hmH-I)shEGCrU&SBW z*+nw&T#016@G)es%kvITf7e#tS$}xB9LJ6H%Pc-_#-C@|J(MrD*%*8tu0JTRF`JK) z`6&Ie9zFl0zU8=gGD!wMt|qmdj{aQ6KmO4F!8`n+@BQ?1L;ut`Ze&Zpk7lb|jgo)C zrgd^O>ptDY=e_yOXu4>O<6*W?tK~Nv^YusXt-el^+3G8|+4z{GX)?M0?l$fX;(7jE z_VaF-BY3fU|O)oXsA#J9xdv-l3U7+gI8xPxFEqN5u`okwnDoQEXCOK9Vq6r=+IHz7^hf> zrQ3gE%lJ3j0xk%-#|i=yEHK;yQ#%sT5h?hHv_GyH0D@y^K!FY(=UI-=-T+Sdn+=0< z;Z!no7H(Qk;^^LcIU3ux&DzOejrnu@xq5jUR!ybuPsIV%rQno1V8OV$#Fs@pT*G`= z#S?ENh{rDKH{Onm9Qr^cF#tWpJt-fPr{^a164eDMW+|Da`FvdvN~U!B@YRpJ>OTY3 zamc@kC%_^1^(UzR0JAP8$^r$kURbDell>@L0S%K`DLes}la?t#0fdv-DLMi6lM^Z? w8|`5$fhGX}08jz|01*HH00000000000DytllT<1`0W6b~DkBEoCjbBd0Gb{za{vGU diff --git a/summary_results.json b/summary_results.json index b2ecb68c2..ac7be3892 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74523, + "Performance_Result": 1.70913, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1167, + "Performance_Result": 22.9236, "Performance_Units": "Samples/s" }, { From c99fdbc12856796942a27ccbffb7608fd44b4852 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 18 May 2025 20:53:55 +0000 Subject: [PATCH 579/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10940 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 72256bb88..a1d552801 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 20:37:47,295 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 20:37:47,295 submission_checker1.py:3308 INFO] --- -[2025-05-18 20:37:47,295 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 20:53:50,572 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 20:53:50,572 submission_checker1.py:3308 INFO] --- +[2025-05-18 20:53:50,572 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 20:37:47,295 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 20:53:50,572 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 20:37:47,295 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 20:53:50,572 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 20:37:47,296 submission_checker1.py:3324 INFO] --- -[2025-05-18 20:37:47,296 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 20:37:47,296 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 20:37:47,296 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 20:37:47,296 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 20:37:47,296 submission_checker1.py:3345 INFO] --- -[2025-05-18 20:37:47,296 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 20:37:48,258 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 20:53:50,572 submission_checker1.py:3324 INFO] --- +[2025-05-18 20:53:50,572 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 20:53:50,572 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 20:53:50,572 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 20:53:50,572 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 20:53:50,572 submission_checker1.py:3345 INFO] --- +[2025-05-18 20:53:50,572 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 20:53:51,550 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -47,9 +47,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8658 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.8766 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0979 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.96013 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.47472 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index e944bea9a..de0cde04c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1206 +1207 diff --git a/summary.csv b/summary.csv index ae93ecaf8..37c9e57d9 100644 --- a/summary.csv +++ b/summary.csv @@ -25,9 +25,9 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8658,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.8766,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0979,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.96013,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.47472,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 92a583919..eeb8ab828 100644 --- a/summary.json +++ b/summary.json @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8766, + "Result":23.0979, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.96013, + "Result":9.47472, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index dd80d4a507ed386d22dd76d540829e67d8bb6edf..8d48f179059cdc90a6b3ad4d15a1c2403176637a 100644 GIT binary patch delta 3831 zcmYk9dpMJS+{ZV^h~`jAa%`4khqZwaDD&-?rSysztSbgOf#2;Lz%52^IIDGdOu z2T>z;xDoJ_0p`YB{21?W5c|LeFC2P(`2Es*txOdIH)a5w<8P*TzsR7ib7jF(MWpkC z#ym>A=_WLZoPC*Ny_7-SQjW5lBvJ^PiPHKH>G?FVv#R{HzvTjMclKKi}e}4 zy_biReG?}c-Ck)UwDRI(agWDg0Zm#CJA@FjV;lH^kad zy5w5*wcry;iQzsnAUyMCL}p}!GWq)T-npt*;RrLuBko&pNWt8NSzn)?waDR`>?DT* z!s&?Y(BXPzX|bEpkF%3DgHs{Y(SCyxlsMOUA}}Dmu=dLpq!F!nWPS4wG}++9{I92_ zip|r7P1J&LUdge`Nfi%%@$fdv0XZJD=&!36zdro@yve`5@rECU+g|>4gcQYUW*UD5 zzrZkz**`tuky@~tYCi5zA^!>NZ?C*}?m05#>#3};EkDj3?aFHB_;OZEgueSdl4IF&H~sY)`GsTZ3$8Az z#=5s>?DAfP<=t!hc}q@qWlfuuu=97Ul}N=KNe7hdt8;Zo4bE({rtHnzN46i0+S12ChGerblj)Xpkm3D5oOik2 zbOmnIX$|it0r&g@^1c?*M8+*VpBB5y}cccE~F8!*du1>8ck7(R4DgqDvw3&msnlo35Hc|B? z7qy1v<_vJSFwZCbi4$!Mzf{#|h5pd%%z)x0Lr>-ELB9X9D3zS+vr~&2#&o^G?B0)W zYp1xIxR9XPT1x|#i7B0K}4+Wm}l9I@GTpIjl{H$UK$bQM^S^SghwF16y>T5ZY4KJ@mb7s2Z(8!@YfJ)P9e7VH*r#@n16)3E zFvjH86%j|UEJ=f8yBlGZzi*!-ctXzFGF+S~`@V&!(IZ?qX)Qa8_Qfocb-Ru&4K-wr#y-G_fLNJR-``KecdXxkLCe;d@xX+0e=GQDKeUuWx zhs`BDZkCsH{e-ck+oVi6p;?;Nyv(+X9}RhYZi3ogu*jX>~?+ZN(_HAm5gcW4R89OO|fIwSq9CxT#6fmTqcAbTlqD7IX;699$o)5QL)(%}x)g0oi5h`!3#%6N8MU+8vD2^~?r}qqxS*k$ zEhpTwLpgrj^FD=vy@#)7Ut!h_Q!?=C%|nKkT?FDWtK@ob)uE)m)y{0T z4n%gO)+V2DAB2x^YG-O2yYwxq>4Q#B?}RxIsb`u;7=!QlxgQNBoEPiD{Mhx#pAvc= zlu*PNbyE9jE{FsW_V$}Pwmy*Z;^t79%Q)ZD??-8WLCAX*RD#Rk$%ImW4^l z?-YBj5f#d5xbDCyJz<<&x+oi_Xj&$Y=>_w(O2!_ZM$&CkC8q@W@>j%V)BnT-R{Q2m z>9Sc4CwqRBOc8FC+{<_PeXGqan!R@Dv_#fZQke5E^-RYG(`|tjQQP`HNHaEj9|8-6`CqYPuq}3=Ag+MFl4y>qf0v2HB+vg`y z$N+d30MZ~p+dBXeec7BMSM1|5W`|6Nq)^HWG+^)wZ{a8Q8rn-!FY&j-?z2h5+xLz!5oSZ2 zJnjl`%UN&b%X;_gNiucBz>}@mcleu}w46hh8Cc$KDISx;l$-=H0U6+Nq#lX`;sPpQ z;b!|ARsBfqYSWbh2hw8NJ~fDuJ^JT1)M%rZFy&a1n^lMhffe0I3)z6qpE1_FlyAmC z%*SIaN;XoYg3+hft|*RCFhnHvSL;c@VN?%UY({WQ&>am`j(u2c*gZ{OnTOujXwfR; zyzKU>mwHt_h+4l5%prD*L`SYygC-{5goBt7@U#Uy3i|Zw6-70{4%t(e1Ux9JgDf^7 zPS^&j%crS2$BCnV_eYMYU99FUMYRvROikJC&BvHL$Kmo;DrUV8!o#n*7I^ zz*kqqBBtW+%5+M85*U~T7WT_ZQ|$vj5(FN$&#X(Xk{Rb0yJ*=F)DQL;e}S%ET#D)% zc9{^UOV5k!Uv7POxs;8qrt`b~$bWkN(q`iCrgTd72x!^%?3jLER1Zi=bPQ%>w*R5v zJf>lVIy<+!Yz=bkju&CH486*aFw6^nGUD=U%6adXJltqa&E?-b9Eo4xq`p|lw}f_3 zWRVrCYiQL z<~31@vqMN*J_WWMZfajLs#t5X2s8<9I;;#auE~Yloez~!a&F71x5#}Gz zN^di6_8qPy;+7GS{=n4xRZqmMENro&(g|3i9R;X^Qjp;Sp2E=PLrSVoMRq25z<&-r zE<<+6Fm)6huK?GQC#V96edi>QDa7?2GD6$VTqmrB61kp`Mzpm-3wlDGUKJ}!hY~fH zu8pZ(>md$kwG<+Rt4<5Y5@-o1muS}ezx{Xsqr1*m#eY8W=${7%OLD(&v8rnAruy@@ z3ZA>~o2Kl=1{RATV2H?>trUSsfR#l8MieM$1Vm9EfC3RtND{Kml2TKW`mcG;FduF^ z38d_iQp10iek4u#Mok$$x7ZdEk}73#PJ?|ovVMpF&K;!}FNQVxEmP-$pX=5==Q=dL zc8yo?lL=(tiuhQUfGZ=Xwo+el6V*LNFRN|@hK7+YLNxb?-NEmAAeIm2RN&t zlI~A<%cCJUaY_T&&{f7@&~Db!zixow_}j9MxCD!$%b85DTug*~j>~t;GT8TFZ)&pv z53fv{$MSYvMZ|sf1i3e4OH+ftrM&}*UBBSKnfq#dOlhszVb!}TZe4Nn!kcD?(hn9% z#oolMd&l4DIhF^yd}C@Hcv0}Fyp?1i^ru&5S?NZb^{%Lv0vX}~PHx&~4!_Xj;A4ox z67($7@M2B4T}Nyhd&Q9VY)dMwO+8fJfoS=I>(I4ctq)-e{r`VcA#HcyPAXd41ei{x zX`cqhQU|n80+*@sIuND5&H@1dM8B_&%XLl2QgIAf{dDccN&QONpuEHQ-2 z`ozOD$kKyB_NDAZ_VFEE*YEfJoU@6Y?X?z)=|H!Fiqoi@6v#1X;+ z0PKN;@KZOz(U`AyR`2Vx5<{b?^WTYI2!S)o*(zp~t@;ud$7+6?1ByGm@oGR5xU(tg zt09bfsQo>GfC?Ww6i^{GzdBs|Zm)Li8F--sC9~xIK`UUq=*M#9w>w)8XAlqjEPOWL zAH6fWRCE5|lRUjfKKxrJaZ?pR?`+0g z(F$x1!(lUAL=itQVe0abL^3z;|v5f2W$n90@<@{f6 z;~TI*S<7+C^BwY(@tZfVyYK5BEUp?dNG^v)(1BM{#!dr@o}Y34H-jpGihV8~Gs-g2)^ zMC6yCOs&%zmjM3~gWDE%*TcklI%<7U&p1-L31%Gku0b_5boJrKi8no7#wIZN6fL5u zn5;3W&g+hi#f<-YZ0uKleh;?SPZFU? z`Sgvy^5Lz$c6aA~bqpPiNwzA^S)igwV#T}~B}A&5K$GDX*6b|BErsp%?ti>F>(vf(7malf<9A_KcT9tV$C2ls+1R4sJ z7LRxv&hoh6w(Y^cw&SfqvUPdY9*w(!qW2nQDrVoSWJ1+FK0la*IO~l2WaLQdEoyNJ}cdZdy(UI-wORhdA{USQ=q?VXvL75imc+gxJVM31M_m0QS zm-YfOB}AJ~vF=k0v9Q zs!?LpQeev4347W1S$1J?{ZZ1uUy6w{f4i;Aug)z%&&GAEU!(vjiZUr)5DJtczbE~< z)IZJ+@if_q6!skrJ{*hYRN@yiU7lUA9hCLn{I8>~%ugv&*-cQmQsRl;^2@8F7_Dq? zKFXeSQNf0toU;NZCA#()GqHWt$h(}8ejb&VAaK%$VzAk;B&`w zV-acq%;%4~vFGv`1stpcy~fumJLh_GW%KfP1+1J0y}c_zKl~YV)-$$cnT%)GuE^QD zDs8jEmz+G)x*uIpeTRX35Zb*%sYbL8|VE38O= zBf=Ke_dKJThv5eHwI;!W2KMP82uMTA=MS|?@kMmLT<4DSr492RGKYWB3jIE2E8HVA zd+%FK>7;-aDzrAMNrZ-K3>wc}iBM~SVRA%hHApmGJQ-=s_j)CP5B)hkE>XZ-!`qpd zHdPxhHKwrw3T7=LuIp~;i=@>$ zdlbrl4Ih3yUo32eK;Skv=0p)^l;~(F+p8sf;O#foXSt(Gj-Mu+uWxsgdTe)WP3^4H74QPgx$K zNn;WtEV-zDFg5iNV+~2?V`o@yx;e6agk|cllGLB-B_&^cne!GGWb-ZO(z{E?S`}-% zvd!~v{t-~HQ`jh=8>_w90?pY=%G`k}Zd(fXh1`B96WAw?wIHw;w6H{{$0aZ~j{x3o zH=aA;EBL7q)ln8zTRE|F*&_zG6vo>7d82!$oF2UJEU}Z@C}1;DdsBloXRq{v#LH66 z6%l5X2|`#QQuxdTQ+h>QNlYR;=P4Iy4dmXa38S+Ly*JNMjcPMypHd$~~kHxS=$0noEBp^o@h8Z60D8!4l_XgaT5yjhtHn z4+s{YsqSKgQo|0*1BD2nUlK61G%3DV=q$fPR*NM^0|kD<*vA=%I$bmuVJA*%-Vxy% ziuaw{CMju9)+x_y8gYPQ@Q#zYJ)FT-`aMVs-dP=6NTJ`$2hw4C_n*$Nq)Q(6qL>-y zWHqf;a>W#@t5Zv}AT>fA+^X1{@!Cna_4xTa%X*WTHo`m~CAR-i3NlNM-utB1B1_p7 zuq5n$(gqZf<4Npm94^vor^1#`vSqTWx}w!{Nd2jP+?=Ke&Cl6GC0( zpwF6<_&w>#jRUDn+Xwp4nA?uMkxuOOP{?Gz*HI-e){?N5Z@3?o9>*PBp_m%t49WJ0 zhp-k8lL~V>>JMRLw9<>RV7!uxJ`JJqM>JV3BY-ngp26E^@GKZSFUMpCuM-N99GE?1 z@T>@%`G(-$3DBsMbq<$0Wte{CCZW|iSaujLp|{V-W=OdT;) z?>C5M*ui%kPVDC7Ce)8A*F*gt(-Z?E;Uh9l!6!6^qm);5^cG_}VD^tE-#HVuKAACT zA$yY87Z~%QjV$?WbdqIFh%QiaSko^(vPrzBSQNG6aP9*XcrqT+rQP43YFr!Ib53dvECumKFvZ0J*6AsKkYC0Ukf*@O8YDBLW&S}e=6c2e ztV;Y@uqTYwQs^Bt1C<)qT8Y>M>B=dM>86e}t(83X)MksNr?P7~!?Nd-6ZGO$)FttD z>04OyQ?#;4rmB+w-{dUz@5efEV2cneWW$rT=4KYxaR_lv8`GePF^?8Ch6?xA2{D%6 z-88WQP}#zOmWX=qLKMBfBEZ#YmeWQQK0}eWd|&{Lb#Uy(Ig&N#1B zStUsUBgIQj28t|BYR&AtWEc-4Y;Kd_lHdinq;LE%d=7W#m9aB1z16VxpPb{Wo!cxH zxU2X@1l>l-uQWJPXPFf}c6MD&4M9O98M#5;83RTnA$pvY2q6%CUxd=~NEW{W{?pNboZdr*pHJ#| zVWv+}9{Lb8`hmXu(K2NB-jQk`-v+wDOZ(g$IMAed2mH%Tn=}p=1}pTHzv$>5xw&HQ zA5F~#iE8$%RPWY9z-;dw+ooY3&vi^aJ-6Xsw4TL${uVWC>yWUYJ~K{%LJsRmUxT#w zA3=_HjgL9j)^ScvI|E;Mj_haZQm(Q5)gqzMrOzX;*mqF8j5K~#`p9af{Pe)wZ_-M~-z$5|;bt$Ah zD17bJ`zt~3bvt4EY)i3@LZ9)1rnwpOMk@+iP0sCHf>&*XiW3}d-^ggyl&x9#oEi|m zQ?B9O2#ByH2gPQdUTM^1_3Zd1Sz;M&%obQ$UZ9uy7zgrrPXu#l*$i+fw%S`?YZtq$ zah9LQya!gWSB2!8L=8Lsv37j9Ia z{j$EWKUaQNUHESuAr=SXD}JA-F#jciyTVg{@PtH)SvLr^NF*6HZ)hk6EwYF|17-z7 z9rXSzUg(CN{trX?G94^@{{=D|?_s^VaQ9nhDgBBquQ44d%2Yfyl#@%ZmWNT9|L+F% zNZyf&kKk}omnoBwa?z0IxhMcUL909p#-P0)5hN7^2|q8MXWkVjoRJVBKo!o2{{G)g r0AR}u0Q|uJ9@r=vl&}Fkd-4!L6LyB{|J}kyDH9SEWY{O=PDcI*bBj}n diff --git a/summary_results.json b/summary_results.json index ac7be3892..27c5e2a9c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 8.96013, + "Performance_Result": 9.47472, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8766, + "Performance_Result": 23.0979, "Performance_Units": "Samples/s" }, { From 43e7ce938640e1144a666f375ed93e2767dd3d31 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 18 May 2025 22:24:06 +0000 Subject: [PATCH 580/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10935 -> 10936 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a1d552801..d61eb409e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 20:53:50,572 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 20:53:50,572 submission_checker1.py:3308 INFO] --- -[2025-05-18 20:53:50,572 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-18 22:24:00,894 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-18 22:24:00,894 submission_checker1.py:3308 INFO] --- +[2025-05-18 22:24:00,894 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 20:53:50,572 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-18 22:24:00,895 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 20:53:50,572 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-18 22:24:00,895 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 20:53:50,572 submission_checker1.py:3324 INFO] --- -[2025-05-18 20:53:50,572 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 20:53:50,572 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 20:53:50,572 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 20:53:50,572 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 20:53:50,572 submission_checker1.py:3345 INFO] --- -[2025-05-18 20:53:50,572 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 20:53:51,550 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-18 22:24:00,895 submission_checker1.py:3324 INFO] --- +[2025-05-18 22:24:00,895 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-18 22:24:00,895 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-18 22:24:00,895 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-18 22:24:00,895 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-18 22:24:00,895 submission_checker1.py:3345 INFO] --- +[2025-05-18 22:24:00,895 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-18 22:24:01,974 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418149 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42018 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index de0cde04c..42d797944 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1207 +1208 diff --git a/summary.csv b/summary.csv index 37c9e57d9..bd2309430 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418149,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42018,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index eeb8ab828..5d7012c77 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418149, + "Result":0.42018, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8d48f179059cdc90a6b3ad4d15a1c2403176637a..cfd804fb206e2651020909426411ce1d8e99fcf4 100644 GIT binary patch delta 1971 zcmV;k2Tb_4Rk&5KatH~1kQqpF4FCZ9vwH|@1Ap2o{5aad&gaJ;z8OW^sxU zz>Y*Db|fOPBN6f7d*vIbBjp0xH-_>KmMggvofYfFPvZ#d9dRYkeTgKg5mSCkrlrNc zb>Y{D`kw6}~Faju&h(wV@B#I;=Q49y;TpN1MgcOP3u@*1qd(00)%19@f}}t29?DjT(t`d#eMsnWCHQXJtZ07kYyx@ zEF(!I86{sid9@8O7HJO>MpF-+?TuR;p(UcDi2eH<%DSXXzU26pb*;ruql>}<_B{j2 zhy5@IauCO}>>)W{#MgJK6=YlDS7TYr3M+8=ZZokpn10J~TvhVgpyDsOKU4T3yX#6Y zF%vivO!8uro(&lRSd+L7D-X5jR^WRj&mk3m>*<-oca!oBFE^?x>)N&?n)*M0W*uv+ z@3)F-Sz(QD}pB z(%K=JKx#WAgJ@O|cUT6{Waxa14~V8o-%;T2KubhN0W?A3A!WF2lCE*vtkdH8 zUTE&C)}wa49vxa%nNO2>4=@1+ld2DQ0dA8b5G?^dlTZ*m7dh&cHd#Py6!{AxrVct9 zBp_&s*fiy@v!f8G0RhL8OcEXf@9>js5|MvHyX5a|=@REhqdnqF=Z9pJC8Zw{LN+T% zKP&@mGGzx|n>9&4Y!ragF4X=e9Dn_Pt~7G zT~vRvebdr@*~_yf=8r~&#Fyp|$tX*zKO}@~R#1Of2H0fc4zOubf7mF%rpf%#VE%tV zOGKv$Y=Zei%5dYXV*cow&7beX=DzwrTKYfA_e!2_+9m#vjUtIJ{U4H1mK1~@I;WTQ{wOAUx*lqG*XAQD10D|kRG18kCve8Op)G$7UiplQ;8Y|wz9 zB_e}>#|~9pRwcJhc89>WY44o{^le4^qiE z64`56UnML-0`f~gC8a0^(PtHb{m~LULclAKTeQGVol9WV`U$Zj*R^6Q9Q8@uJaBvdguu+_tVi@a|F_ zepr7t+(SGYFPRQT`Cqc{jn|)KmyV`4jm4kq&&cx*Pk+}|-dTTmxg3ATjr7YbK5oXJ zXW2cJFSgkjd>*bp9j`H)kCOQ){jwfC|9HOTxOXy120yMQwVaOrT*g2C(Eq_Z{Gspt z^m9Z1)HrTrOTUk1t6Pndf5E18ay0AR#OJ;F&1kx4jN@UpP^;xP8}s$&?XA8}liBJk zx7qlZq-ip_|L!*K4dN4d{$2L-ZkQzL{Wtj~*?w-}$zQVz7T691U$){uljkHKf6Z#c zFc5_AeF~$yE!k;G5Viwt$*F-tU0Qk*(k2nLevs5UdHX81<0Oz%kE@yec6L#@+^aVE z1?Q#F70NS$642ZTskaq+Z&v97B|b1MSZg#?=l~uq>${R$%8i3pXDqlN!6y-|AS zyAUkJ*z+AI=CkO~QQR1(Scs+De`3q{H`@X(2zkH?0uwASd<3R;BBBu~_=L1SuI&MW zV`xEv4j$)Oj?dlzPWhV+y>j7HG8`=2w4TJtz4vl5c3qcsv)&r>=lFB|^471KN_{*P z2UM4WQ|^ETbFGAG4?DC1!={f)w+V z%+q|)6a_8rDapT%Bd_|;Ks65eH}M1*a$kRf`VX@yCdvW{eUKSQat#0g`;+`AR{;!@ zR4F_GmXnbwK@Ec(004MwFLQKxY-MwkASx7-^C=nv^&XQD9~F}%Di9oBw&Fh}0RR9{ z0ssIJ00000000000001hfz~9G@gx(IBq|V-Hzp7Z0000000000fPpHLjw&Mt+$R74 F004}sgI@pu delta 1952 zcmV;R2VeNORku~JatI0SCb4624FCZAvwH|@1Ai(|Wt+Fpx$}`b@nCz7kw8r26eGYL ziAe59L~=(W;?eiYH(*D~1+{Ms!80>+Nr=|9ZRXe;(Ncqq7$?{ zZGX8O#ekz2Ad2Dv^c*9BJab_LP$UtFB8f;8NkpO;7*HhT5=BFK2g?z~zdUz6uR6l6v_x?2Vzm6ADz$WPd}%Mv&)GS=1#(@|nk5%sBG-OIR;(lU_er z9E^Lb9zF-&w?3QJvAhZpUWNq-!UyeL`M<(_c@exNtt}j@h$6Gi=Rdpg$3*flsgam zVGiUVj%V3Ja=wVK?^G+uw#2W-vX&KA;PTyOVrww{mgBgptpUu5sI}Q{(ww zXzr`mqjtp}9a>hIP?LKPFaZjatPghqa+4$wEdfH4QV={BJL;7-SwL(Q`5Pjp4muhn zAZUr$H0AHJr4XnA0oRjG5*`8%@sn;6k$=O&U#r<=v&8w)Xpi{P`5_r)N$H1#kj)Cx z56b|XOxc0gW=+x$8wKDrIX@blA83i_G^HvFWnET=`zA4iVBf4+`uTb|pK_$#Q}yRk z7uBC^-?X$}_VR3r`J+)G@um4gGRl(b4+$Zg71SS=0XCVq18kbqA2tfGX)=E_n14Ue z647Y_n_&KsGTb<;m_NE^^XL1pxv&0@mi~|Oy^^P!c8ULEqe$XQ|A%ChB?TZ70ytSl z!pSlcPBtNc(>4h}Y!nHnse_IU0uZ!BY?|P)Ls^%W$&HhQVB@s+$^ZGgMD|)1S!Lt2 ziyBbAafW3gyIrCI+31t_QUf9xWq(Nzh=h>M3LX&40GlKupK#hH4TyCBXqq%28#Ewj ziO3+}u|rjtRmp9W-661T+IwdKeOr;emi1NHG3}xRleEf)%fJjDJ(gGqO#Ik|~#4>;-^K0NBYtn$&D8Qvj z1F}H_f|iI)Q-0LNIh1v2860FQ{P`by&o!WQm?Np4~gIux2@|>yt`C~ zAJ!iZ_YlvQ@Pw5xc5BIl{%w9^V|G=M|P=AmMSFm2(@Y6I2_a8Up$N0_P6 z&U-^Jc5DZdx+FMM5I5RN>O*O_f7np=L$`qQG=0PpJmu6=JOX1g6VVaL*^D&5wwnN) zVdy}D@(yQ7hR5VzP$(8U9+ozE7*hLXD?l zfSQUk!Yoj)ZByaPA{?%x&acCXHv;(65cC^w$3+f(z@rd=9O9mo&*J%|C0(H=PxCTP z%Q#!Kc}mKHl-c(<@~Zy~l*b`=6HkCc?(0v``~$NpCdvW{?Iy8faSZ?f{FD7CR{;x? zRVh3Hm6MVwK@EZ&004MwFLQKxY-MwkAu1G;^eGww^d6HD9~F}&Di9k_HN1i)0RR9{ m0ssIJ00000000000001hfz^{xDn0=zlaDGR2HPh90001-BapKI diff --git a/summary_results.json b/summary_results.json index 27c5e2a9c..5a0617291 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418149, + "Performance_Result": 0.42018, "Performance_Units": "Samples/s" }, { From 8785706a29e8a1b9c9a9f8bc44f5b1f87acfcadc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 19 May 2025 00:14:35 +0000 Subject: [PATCH 581/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10936 -> 10935 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d61eb409e..083a9c55a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-18 22:24:00,894 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-18 22:24:00,894 submission_checker1.py:3308 INFO] --- -[2025-05-18 22:24:00,894 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-19 00:14:30,119 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-19 00:14:30,119 submission_checker1.py:3308 INFO] --- +[2025-05-19 00:14:30,119 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-18 22:24:00,895 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-19 00:14:30,119 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-18 22:24:00,895 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-19 00:14:30,119 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-18 22:24:00,895 submission_checker1.py:3324 INFO] --- -[2025-05-18 22:24:00,895 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-18 22:24:00,895 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-18 22:24:00,895 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-18 22:24:00,895 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-18 22:24:00,895 submission_checker1.py:3345 INFO] --- -[2025-05-18 22:24:00,895 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-18 22:24:01,974 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-19 00:14:30,119 submission_checker1.py:3324 INFO] --- +[2025-05-19 00:14:30,119 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-19 00:14:30,120 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-19 00:14:30,120 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-19 00:14:30,120 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-19 00:14:30,120 submission_checker1.py:3345 INFO] --- +[2025-05-19 00:14:30,120 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-19 00:14:31,236 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25756 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70913 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.5179 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 42d797944..f72207f95 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1208 +1209 diff --git a/summary.csv b/summary.csv index bd2309430..2507590bf 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25756,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70913,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73124,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.5179,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 5d7012c77..4fedc3508 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70913, + "Result":1.73124, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index cfd804fb206e2651020909426411ce1d8e99fcf4..1bc33f795b28b58d34d382713c5aee9dd295b769 100644 GIT binary patch delta 1770 zcmYL~dr*^S7RJ9IAV?rF(6oRMlyC_Kp)nw%3<(OBK)JXI%hj+LkW}O%*9!85i%YY+ z6v&s023Mm%Q`{6oNU2a6WkDc8K!O*7Etj$g2pS>im=NNAbZ4jMpZAY5^PA^6=REV? zjr}QBdcneM|FyQ}ypCdvGz&0kndBL3$<5@HsDOP}V;rdXs+c%mR=l~0kx|Q!5h7ju zcTt%b7TJRO7&FcckHspxCCHl+7hqXvCUSEBXhZMho5lTGmidtpEj)H!aPMVPp(X$O zh7y%=@70(o3ce~nP721GOBr$2sZ1F->B(xL8P(GtSZ+g<0#La9G$W&=l5+vQSnOM$ zhZpE0xSn&y0QIGL-g!Zhr69AuL}h?kLYS2t=L@s$;_c~aObi&mlhvZw9G~5yMueCL z*EuP>n~{M<&-_YOyE8O9+YK-Gm3l`=_Q_Umy`TS4z9L{!rU-MGt=-XIQSi3muWK0f z;P9>~)G#2x??@I(aL*9AOinpDjPe-M5d4^TS$1H5-$~-`t$l_+@# z1AWxp6Sj(yt=8!t1b1dEEN<3=UEmEqy$e3T;6m^M} zusSb_u5G!dNHL>UxsBExFYOnMq7-s29H)HJ6A&wOk!!O5ou&Bh(^9FnoT9I;Y#+lJM-tRhx?Q6VpemmR|gA^+(Aa<+sdF-9%Y~^5hiv zMYS8YzBY*MHH2}Dx!VZ-1!K%kY=Bj z@X*AIm^#y5#u++2toQnhC8nOPClEp{!^}|eoM>DFW?Nt)ED(+wEKtfQx8B2ubU-_r z-6b=r{+V!e+nLJf?tmKE0q?4+tjm;-XEa^rT$_9l{}eLZNPHHzY)ISy-HqQF7+3Q>S5zdbtP;V{ zbRp!yq%sU=)5|zHdVo{07><8&d90|hU)%VB>|FP0Z8;~Ut#F~UaI2zN@ak%}GdpF7t_^?kz&0ol#;jJ3Rj#=zk>H_O^b z24=OnTgxT2>N4)5FO~)GmUM$!@4+63M*5KF*Sb)!u}Ctm)fiFJbEr!i@eN14cp%R; zNq>o)^y^n#gO&deP*Z}&Jt$B!l5>0_^voVq$K^!t`|2xqC^h_xV=QMw1RCcGAYQ{7U*QXbJBEM)mokt2Ke4|0zuq zw^&Hb_)jByy%z?0|8)?XTt5q0sIFf~x^>=ypSG&Q+G!IC zmOQinagx}t@|y%n#u0}H2ampyRz5JbLgHAhPVA*`?8-u>aL?ZZy`RfP)SY!&;^5gF zKx|+fyOSKiLzY@W&!B+r;?v&_w5`apr<2YY_keBmX?^U;R+RWR*6pLYN+)S*a_sEz zKIgXdAjv%Y5T<`V=#_2GVCY|i&@?2q=B~%W8RJIs9oHB?*N)I|EYlccdsLfr*n?iv z@aL_wqQf@2Lu;m=U7&-NuQ~|s!zTaxejuG^62c0KB?KaP5Qh+KUAYGUO0sP8sZOJLJAK-Cta1U;+?XR<8MZP>oo|73`Xj};iz{8nF2Kh`NFup&*xUv(7y8?(E zA>6&pAca5YNHdHpBvtq?UHo|c4 zpRaYdj0;46dLL8S_Pf7EnjwQrh#0FtO&+6AX)qc=jTM)VH)9zU`$s5~dZ3H0&{RnCTaC3vix`}*L>fI0AEXe##y;|PS&30$ z=eE$6+cSpIM`VkdOt51xT&)iLq)fVIMDzHbI&`n^m~qLQxq0&6xQdTBUl9Wc`iS%T z2Uqe@?{*^615S8aQY5WAsG;0oxt*_RNZeM2I@#1%P>0YsM_DRxdbO6SqE5@>DwaX#D;vk-kb)ybrvb z{1Uq^;5+aN3yos{IyRk^^SR~X<{DCSa?`CbCtSD!VJ+iuQ`rh|^& zhmc%;?~v1mdJd^dqA`m2z3$)oYMT}m3S#HA$`j|xjFwIwT&DMb&1`Qtj|`EYpx5?e+i5Lr z(E4ttg-W9Uyh3P)0F8GQ*?6HP5LXK97cZJ`jX`-219I}ajB+Fn%~rLZ$g6jEA>pc{ ze9sz9?~?>?D#l=Oe$YiK5nJgDxjqb`E>7+agF5J^yEQ)-p)}csPy&+Q>#}odCXNwefJ@+Rt;krBu>J-{;8; z>H|T1^%1@hd{}il#cS$MpEmV9?bC$jgday^DEq+~8#A^=$uW`WdSpN6W^n!oI!fA6(_5n8I*D^+g?<9eQW5nxHjOS9aSd{>?>p z4Z^QC*_`<zROzz);z&(DU+37qk zo9GH1+K6O!N&++Y+iZy_=7!I5j7Hxg zW40}va?`}_48D5-ZEq3!V1eL{U&iFGf-*T-r;@^8Sx~+}eW?ecT z6zr~Bedha_xz>BZaC=G&$X%5=J1U|7|9?a)&PO1oq6nA$5Q*r5i=SD}4gg4Fd8B@y z&SIvD(5}`Zjf)-hoxKQwvJqii5j!@E#zFuf1PTC_z(0$V-H#D1V5~MTiM$;VTmL@d QWV?zATzA6;oi=y=2W(?J9RL6T diff --git a/summary_results.json b/summary_results.json index 5a0617291..b06aacbb9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70913, + "Performance_Result": 1.73124, "Performance_Units": "Samples/s" }, { From f9c778b23afe73f1ce8b0665d956f8cae083de6c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 19 May 2025 00:16:17 +0000 Subject: [PATCH 582/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10935 -> 10935 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 083a9c55a..39d738199 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-19 00:14:30,119 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-19 00:14:30,119 submission_checker1.py:3308 INFO] --- -[2025-05-19 00:14:30,119 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-19 00:16:12,269 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-19 00:16:12,269 submission_checker1.py:3308 INFO] --- +[2025-05-19 00:16:12,270 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-19 00:14:30,119 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-19 00:16:12,270 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-19 00:14:30,119 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-19 00:16:12,270 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-19 00:14:30,119 submission_checker1.py:3324 INFO] --- -[2025-05-19 00:14:30,119 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-19 00:14:30,120 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-19 00:14:30,120 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-19 00:14:30,120 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-19 00:14:30,120 submission_checker1.py:3345 INFO] --- -[2025-05-19 00:14:30,120 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-19 00:14:31,236 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-19 00:16:12,270 submission_checker1.py:3324 INFO] --- +[2025-05-19 00:16:12,270 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-19 00:16:12,270 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-19 00:16:12,270 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-19 00:16:12,270 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-19 00:16:12,270 submission_checker1.py:3345 INFO] --- +[2025-05-19 00:16:12,270 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-19 00:16:13,289 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42018 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422032 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f72207f95..9c2de7660 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1209 +1210 diff --git a/summary.csv b/summary.csv index 2507590bf..d78d68514 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42018,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422032,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 4fedc3508..f49b93642 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.42018, + "Result":0.422032, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 1bc33f795b28b58d34d382713c5aee9dd295b769..fe046950ff61b5914aeb2f9a114d9a6fbc2c61de 100644 GIT binary patch delta 1774 zcmVV z{2H0OxxMc7^QSZi*)y1RR|hv4k+UypGVkAB-|3duvJU!7rJV{a*0GcdEWY;(EIL8U z(|?xBQ4BbW0iq}#K+iD}$TJs407Vj!D3XXokwhelfdNHQE>Sd;cd#5$3=Ak@J)js! z6iJQPwBM3xX|b$};NOE{9ZRWz;)h^oTeKbRJ0-`_kh2&fi-Mx(7zxCK>=YxwB8f;A zNkp=U5oL*o=Bv;EBdM29!`>L`JE1`3Og1(|Yy^28l|@}rB%gV_#f&4Lzl8MyH|h10 z#lg6@>fv+Xee1Jn9m}f#;bmBWFl;%#tpU zu5sI}Q{(wwXzr`mqjtp}9a>hIP?LKPFaZjatPghqa+4$wEdfH4QV={BJL;7-SwL(Q z`5Pjp4muhnAZUr$H0AHJr4Xk90k4xy5*`8{;*)I>l7EH2Rvuhx~vTMO=1SYzFD*M^Yw5( zd&Pvsz2GjX=%Ug<=GPRN25aGOY?_hlqJ<45<)gBs6Q+NY%*~N*fgm>Y!qPA zWd3L{e}AAQqSFL6!Tcd*xN%l7e{{{}&-Y<-U;Q5~{U7CfB~Lf)6930Wk;Iq&56LJ? z3P2%B(I@ex21GK-l7Ai$2_c&mJRp_#MS3+C>Q{+c7Oym&ti{i3Mb% zPvT1ph-8!{H6Ri~Dl2F}ECW=M3{YvZfLI4WrO5)?-uOTzSU~KufyWMIT~dZvReEA8ob5 zK$pBIYda-KqDkUQ|A%ChB?TZ7LL@5)Kz}R)NHVnskTe-UY!pD!WB}P<06|N{rYS#K z<{Zkpv<#4}U;x>BZUAkN44;JLzKTD#vx{Wlxf02C;lte~@%$z}jmP6fqn~7#YhAf* zU4P=;r8@ku{&2X5cs5=#9gOn7WZxUFKgup0O>Y{DKi40U=N+E@uC2VY{_t`+j(;2J zmsxz=j6ct^dnjLQvoZKQTz@=XV>TZp^HKU`J$nB6e9Lj~WReViTuo{@9sRkCfBd2U zgLn8t-}~w3hW@E>+{l)GAI(;`8YTaNP3z=n*1L(%d-I#obkP{c!)&2e%WpR3>kr&p zeVrz=)mLt_@i9r$WOD!AZQL8g^Ar5L?C0GuNz(go@=LP);KGx?0J94g*bW5$4*8pt z=OiG1YQr!PgztR{qq{B1A)ytv18vEvfkIPSdK1zn5w(7h)H-?lDo)}g;8TyQnf-Ql zQ8nM`Hv0u1lr=S?1wk1YVWl#g8ojR;`8~=)m2ga053k-j@KHg? z637H5T#dFdI)-r&ThKfd$zhVXwqEm?O25H>j*D-;0bCL?!x|!&JaT*h=58RO6H3)3M1Ej#vf(8=BMUV@^PQ^k+;(#$3Bo*3c!wW&&tQ->3Ovz zKut-?X-=j&eOM8~=#0_w>oD@B{{qyfA^#?x0mt08pP=~zvneLZ0tpBR=KgUF008{6 Q{3tj91pf~Co0B*ye>wL%fB*mh delta 1774 zcmVJ;z8OW^sxU zz>Y*Db|fOPBN6f7d*vIbBjp0xH-_>KmMggvofYfFPvZ#d9dRYkeTgKg5mSCkrlrNc zb>Y{D`kw6}~Faju&h(wV@B#I;=Q49y;TpN1MgcOP3u@*1qd(00)%19@f}}t29p{M9sxO%ISm~FC zQ}`mg>q;;&6F3q~@?w*p4H*GglerBm54Go3;Cm&{Ar*h?>6yZJlk*KPH>xV@+O{N` z`aghX9c!%eLhEK?+vk+7?Ru(&P5wUVGlg$Iy1rx+Gl3`B#0;`YGQcLu0Gs5^lUWWs z3}gS9!uQMFK%zxa14~V8o-%;T2KubhN0W?A3A!WF2 zlCE*vtkdH8UTE&C)}wa49vxa%nNO2?4=@1+ldBJR0dA8c5G?^dlTi>n7dh&cHd#Py z6!{AxrVct9Bp_&s*fiy@v!oEG0RfMbOcEXf-{F&N5|V#*$=})1CC-mVd&HN{56LJ? zN#{Q3H;EYp`)1A3&)38G zlq2n)sy~;ysQzU8rltL|muE}NAB_r$FU=p4QI=GHNC?@ip#HE7u*t+7VAG`juu*_b zllh~;{DFU#h)xsO1oMZK;l^3T{LwX=Ki`MVef58|^naA^l|0?FOZ*=jMG{~7KP00p zDFBfWz{xTaPL`2yvIzm4wn+eDqewVS9dv9EfS@H}(*%zl%DS{nZk!|p8>hWb{?Fef zve&Z6DjTO=)PVAhGb|I??Gg>hMxVr&8W71SOL~7mB!p~M@PJqb*d!VGgwr-@K&%5m z)1(2}paDTkLld*cI@U;(kq1|B< zEMR}%v3!!qUd#F_VF?nDU-~I2MF~g+4_J~HJ#sf3(#K z16}f>tnHK_i6)6J{U4H1mK1Nm+J7t`m^C4;@Nn~bTG>Ql6`Ny{v^9}G`(po{#<`Xo_BcqySDPq`oqiRIBtKW zUuN-fGyXiw?xB3K&BoyKaQ*3cjoEyZ%tz^$_2~J>^DW1{lSwl8aW$#sboA#k{_%(Y z58mMqeeb8A8~UflaU)y$eKcF$YLxs7Hm#GRS??x3@6B&U(?w$(53_|@Ex*~AuRm{Z z^>v!eR$sZz#>XU0lga&ew{dR}&+`)RvY&UuBuVeT$uG(Fa|=)Y0<#Mi*bW4Km}R1q z=OiG1PQx$|gzr2>qfbB_x7C+$?*WTEsuH4QbyVH)+#W%8BE8X1|?X zxmfO0oBV?F(&!T989@nXZiLiLiQel~dXEwxm=>%x8cMVWkCxRfL*vToj6WBwd}u3z5z zRa2=Cr{aLBLU76*uwYzO;L9Q&u4O)~E(e}d`{vneLZ0tqXeuB34d008^5 Q{3tj91b&!hqLVl(f57iXssI20 diff --git a/summary_results.json b/summary_results.json index b06aacbb9..5ced957f6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.42018, + "Performance_Result": 0.422032, "Performance_Units": "Samples/s" }, { From eb1e5a58bad5679ecd3a73a4c718dc023972fd3f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 19 May 2025 00:33:29 +0000 Subject: [PATCH 583/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10936 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 39d738199..d60a12fed 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-19 00:16:12,269 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-19 00:16:12,269 submission_checker1.py:3308 INFO] --- -[2025-05-19 00:16:12,270 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-19 00:33:24,138 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-19 00:33:24,138 submission_checker1.py:3308 INFO] --- +[2025-05-19 00:33:24,138 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-19 00:16:12,270 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-19 00:33:24,138 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-19 00:16:12,270 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-19 00:33:24,138 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-19 00:16:12,270 submission_checker1.py:3324 INFO] --- -[2025-05-19 00:16:12,270 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-19 00:16:12,270 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-19 00:16:12,270 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-19 00:16:12,270 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-19 00:16:12,270 submission_checker1.py:3345 INFO] --- -[2025-05-19 00:16:12,270 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-19 00:16:13,289 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-19 00:33:24,138 submission_checker1.py:3324 INFO] --- +[2025-05-19 00:33:24,138 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-19 00:33:24,138 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-19 00:33:24,138 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-19 00:33:24,138 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-19 00:33:24,139 submission_checker1.py:3345 INFO] --- +[2025-05-19 00:33:24,139 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-19 00:33:25,363 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,8 +28,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25756 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73124 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.5179 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74351 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9c2de7660..44ba2fc68 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1210 +1211 diff --git a/summary.csv b/summary.csv index d78d68514..6513c7efe 100644 --- a/summary.csv +++ b/summary.csv @@ -6,8 +6,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25756,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73124,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.5179,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74351,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f49b93642..6326e2a9e 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73124, + "Result":1.74351, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.5179, + "Result":21.4359, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index fe046950ff61b5914aeb2f9a114d9a6fbc2c61de..a279ad54aa61469a9489284963d502bf0f5d05db 100644 GIT binary patch delta 2062 zcmYL~eK^y5AIE?5uqY2pr6byd_JBs+JF-sE2sLJ&l5B{w$&e&RI%!+1Bju_6n3~hw z)|%|ppaIz&hZ35(N7NFXmF{>$6T#G~rAth(AY#ca?>Huh5gUrkf^)y<(|4kpLZ> zD=ZS8f?KaO(SCuvJt4E|-_fc%*;gi?IwSBHQYJa%oQ+fk-3cCTIeD&Sn{w1(gUCD0 zDveN#66zMuJCmqv?deoD-~GdDAQwNMx&ZizM^M@ehP*15==F;i;1-{Rq$Xi}M!;0Q zrr)LFWi0V)5A2cekc}t-Auwq14K$-=>`Eh8qaitM7ZM-WP9-w-9d#qQs}p**9+&a^ zEoR6r9-HHBsg9jS_c?8UV|Jy=Y^=eS0AtxR?6&w0?lv~ zRuuD|{JV|me`r*z^fEj%)CgIqj09*raH(-0j7XpPbD?<^(MetKN()*s?AT;>aHDQb1)par z`W~tN7(Cn3I%QfusTY+On+8uKbgbm6bS6D#YtKTt(u4|}2(L)}83+*nj?#Z?SmnY( z%=H(P(b^iqjs>T*NTU|Gi1rgUvOZapm3f^_B;_!zj(cr^S9smXrVFJgT7bIm;ZotX zWs;=|y;wI=Lvic3Bb$Ld{P_NqIZdc|tzd@5CmG59nbD6rp7(dqC|M_!be5cH%oxW8 zR&ixu^t<(3%cwk$2Lg|r{q~ki*&Y^9{Z6Eg(CDhC2#Bl>wIhMF7yCf<{nhPAm zO6JWtw|zmktHp+9tSEnOz2UX96RM!`o3r2(^rs8#>BEubTJ)LJ#G*NfF9+i5Dnmnj z5;qdRq*V#=iSeWRQ)d3*qyNJnP#x9ZMBo`XUqL#FJdxH(ToCSLA%ZVKA)9~vYtH|M?2!?ow3is|?%ARii?42@{<3zYr`h%jE4smoX-6H`ql1;3>KiD@zxEbe(rqbPa^5>mMB6gd{8l z(6U~ek58QR?k#4=4P!C=J3pVB;i+w|Jch!P@BDldo^#^2f{^C$G&rl!>pe~XA}a^y zfHvA|w>8m`s2MFxPx^zCZ;S4QNjTm7=uW-_XP&}uF*q3Pk9$B*NNuR$OkdZYJMOx` z>^;P8sk&_BT@%KX_TpL?yt*b0SrpzF<>oCpj=t`fBW4sm5vDx>&jPWKN`uN>PGK+o@ zE+1%rV|6{`vO24Vc|<}xoKt<>2-}5K6fZkNkQxxAaDx^EiM__uq>A;>cB|1G50phNqL}jC%njh}9CWrf-FD4jpE5)?ozaJ4=Vg|VGoIb}WY{`u;Kl3nf6L${ z)|$0s%(WBu#5CVHCU?l$RwMr*2`Qx@%m0<1B~)50$&e1qN%f--Pm|o>6e+&?w@h_V zGezCG{B*afV0vlpl=E`W&KMTW<4IO{X<&ws_q62aINMKuHH|%1c0@diNFP2@<6SyR zHl3YcHFdAv6O>PEeIQ@FTErpME>GD_>Aydt4qljzxr!ZMA}eunZtPHNNaY&iegm!0 zTGQ9y7#7X*Y}}3zOxwGToIOe?&?&>_0`o37>?OxR;@JUa6Gf)ad5a_*f8{@sHvCKY zlk35?;%UEpRq|~2$4#RqPP!2>f}Z?MkIRk69o&;bnH9Yew|xwe9r^i}0^2fD4jm&t zL_a00EpiyG**Ab7KI@;gw)jx_vC}8W^j5&d;(<_J`yzP&u+gb)T&=N=xZNH&uzPJ6 z%yLmsMUifXC5J^goi{*^F*3d00-nlJ{ld5pexx~LDKqQ+$lEhNnbb+!AH`KQtA0`k z)&HJRTHpfrW9^1CLA~fj0(5yL9%HBg9PU|yj3Pa6rJ(23Va1Jf}fj>2j%*~n^ zKU0l$q&r<=xQBDp73KmF&N}AFuU9rM5hT0xlw^fY|5&|Bs{>VOiEq>R?|Td`S_H!M zKtD@+m7<`x1^|iC+fK!wiDsVy(N^ohS<6it^GFbZ+5lp$5N0b$<6r>btpNaQ kf&YK}^SxM5fnB$92qfbXYyP*zKW_zQT5W+35?22F2Ni+CB>(^b delta 2042 zcmYL~dpy(oAIEoL?w3}SiQ#N>X@m!>B%4c4vqLU%CW%^ZB%d-VJR?(Nx#e zoK+@^(=|gifKb-hso4#Sy6H(*zhzXb(L z_T=z>%ZJ4o96B9tv?!n{TjEEU%kExPpRzj4rTbi2bJOu{Nou>H8-yfYzo{_?A^RSn><6+wH=3tn`ZY;v63=8Io`0=0W6 z1CYGT50E1@Nn0sXz49K;{MNWo9^)epEA0J?w#quA&_jcqj1h%~qV&;@vFFkSW1% z$BCF6YFGn6MKsXD^r9Ie02sAl3(3l{H(%M<$HY{@#$Kr#&`=w{yW80K)6Rz}QBTnK zq!k7eQ1VWI2&Rmplcvn ztY7K!Y>s<3w_4Cz{dAz%!Hu}0BRcw?cQ3Q%KGH>ko+{iJqYa$TcN`nU3BKd#|KVI> z%WM0kNyC_ou!8lx^X zMCO+0VVaWe2$W%O466~k*DPaOlEa*U)F-n8%g0Vdt)uPN(eE;`2Y3C2RQ355_VSem z5*vy(38R)I%0JJI#r)3nf>b9U*`qedfDjG9%vA0Bh3SsT6*<^*O|7-MA2B|1Rf_by z#U{K+`iQaBwJd8|x8EnN-3KljvR)D*d_cG6h8u3(EkL(qUysQ~f0W?@WYC2{BGAFB zQRvwflY7hIHs>X--8=ea_K;>~_#ec=#cXZXiR%rfQIc64>~guOuv$v?w~)ryqMH_yRTT4qIDiG#y`x5xo#~ z_))XfX85e1{DNS?-EiK_^pBNl8v>0%fjlETC=g!~1=k4YB6@wH4*sYDJ(OXi4^tqo zaCll?3sCweEFvZcRHK=7P+`w@@)wM%n%5NS)@i(sPH1{S;@`80--|tPgFHo1ZB(KT zip1>D1VtWjZbB5en1C-2Nk)T=MMFF(y;Ziv!?W9^ZH>AGwb78V%#)CW0^QQ{GaVO+ z&EiY6iB_Z}ZL;SzZfOnYak+1Le+5uVax(}=oz3#)owT>6 z^+c(InSUbc67dC^GiUaPvhL4&V&$ilJy%TU`Qcfw@`(eYOq(dC*y*c#B;bBDhJeN@ zg1*J>`$f5u)t~&4#?fU(`k&Ziw%(c#td*;B(!YAYU7er1psY!L^_a&(Q=Wv~MPXIh z{tGkok@mcSOvSaK)S+#CJIH_ zbFUrAveiU6MYKNgfZGWX>l%;htEu!?u5sUEEPxX|j)vzyp4F~sx0hI!8#TidBsJ%} zLJt43S%%xx#432^c4pNsU)Uz4sye#J)%AwDr^&WsLg#C!`}M{v z`=nkozVZ>&%^`D`U{`u^(oa-_`ZsQ$f;=B1AFJUaFE=~6T}U>J7mA9DETv&&qlUuk0GkMr?*xsXyGBs8-)Y_pXoJ z({wRYqhdt7yCvOI?@`^t#gnp#5Y|N^e`n(QtybsU5G|5FR(-R1Mq@|Kr_+V{FXV>& z`QCVSd2;FJ!n?y(eI5nj+Fe(Y2c&&7^JkhcB!9A$5m2?~EOpa^_xVHe1xI<63ihnp zQib~i+V|>iY1s-~Y=@J={{4=*D>e~O9d4+tJG7X~vGvr;-VT9eFn69xJ;RJW#kI8C z!u@QEf_*mUZbBPzv37>08%blK5J(^l0@(!lZ?Vrk=LXt&KyA5K?S4SKwB9)R2P<-} AWB>pF diff --git a/summary_results.json b/summary_results.json index 5ced957f6..9c671e600 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.5179, + "Performance_Result": 21.4359, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73124, + "Performance_Result": 1.74351, "Performance_Units": "Samples/s" }, { From 55ccc36c747ff3119a8c98a390776029a1998b97 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 19 May 2025 10:39:22 +0000 Subject: [PATCH 584/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10936 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d60a12fed..edd397306 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-19 00:33:24,138 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-19 00:33:24,138 submission_checker1.py:3308 INFO] --- -[2025-05-19 00:33:24,138 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-19 10:39:16,482 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-19 10:39:16,482 submission_checker1.py:3308 INFO] --- +[2025-05-19 10:39:16,482 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-19 00:33:24,138 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-19 10:39:16,482 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-19 00:33:24,138 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-19 10:39:16,482 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-19 00:33:24,138 submission_checker1.py:3324 INFO] --- -[2025-05-19 00:33:24,138 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-19 00:33:24,138 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-19 00:33:24,138 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-19 00:33:24,138 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-19 00:33:24,139 submission_checker1.py:3345 INFO] --- -[2025-05-19 00:33:24,139 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-19 00:33:25,363 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-19 10:39:16,482 submission_checker1.py:3324 INFO] --- +[2025-05-19 10:39:16,482 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-19 10:39:16,482 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-19 10:39:16,482 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-19 10:39:16,482 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-19 10:39:16,482 submission_checker1.py:3345 INFO] --- +[2025-05-19 10:39:16,482 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-19 10:39:17,545 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9236 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.7442 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 44ba2fc68..1f8248cea 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1211 +1212 diff --git a/summary.csv b/summary.csv index 6513c7efe..e1333c2b1 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9236,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.7442,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6326e2a9e..96382f80e 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9236, + "Result":22.7442, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a279ad54aa61469a9489284963d502bf0f5d05db..52e7f4d33a8110c65f31a448d36fa7013df8243d 100644 GIT binary patch delta 1644 zcmV-y29x=?Rk>BLatH)`y9*Vwb_lWvf9yL>i7~5n36`jlc+&nMnNZq4B!gI1uzy$v zuw+&ZU}>^{*eHOdN&ea1_<$wIKXjUaCCERd3{J4DYdFCc7Vs~{a$n6Ky=(qB*?r@c zonh?~F3}+Ir1?WKp)`L;2Dz+Y{;&*iNix2~(j@+{4ggD&_oKo4ftH92f*w1Re|1S2 zc%oM zE=dNsG+95a1K`pm{b-PWpd})MpeqYyT~>ytSYigjDYjoA}gI_i`Gwe zk`0`~FJ6Va*@`xv;ACd{BfAm8#NM;4;hh>0Fk^wSJ&JXJV$TTTG8k8SsiO3*? zOi+GE86ITWU-N;3Y|YBg_aSpnou4aRbbj&|WWBKL-AnD2ph^A^kCc8$CX~_-$sn2) zq#u?6G)V@~G&w)413=TH{Af^qpd})MpeYMwT~>yhCh>w`)2vzf`C2ref27LRvdBuC zX3_c4c1t^9*{iZ8%8y2M#Fxqs$tX)YKO}@`R&ah;2GArKK+~lBunquClkua$_<@#) zT{c7$j2}{lyC&%xcg>oOpYMa_o+>|Ax~Tl*yQUu&{#wm0n_bpTYFETHX;4^)B$#4a0n>`>MvWr$@J3#hPw zeaG@iB6}_CtAr&;Kz`}Rq!c9}6+B=`K7K?>KqR9qX#tTCe_~m|0%94!lKC}okTq#Q zY!u+qqygEW0YOW|rYS$_;vCAlvz2_QGI?Rz&Px60iSzD#^YtjGFRx1p2 z$&0eKQ-UO#B);^2NJd#w03snovVs7_GJqsgYXC`;0mMcDBuxg84F(XjL~NS!qh-#a ztV_!P$qELLf4%1h&<4ryNl5Oi_+vY}NCuuOk!%+}+-(xiZ{pKVNp`u`mD|?! zC*EDE!w>5ZhkJ-;<0aFx%$>7J;q?XgspUe2iANoId zhd=bapMGxWpBl%FZ0YyWY;~(q@-Nu5PL5{1oA|spzZp#zjd4887HYNpW@Enoz`fPi zX);@V0|XQR000O8002J# z=W+t$vmO@44h3>I*$*a@=_DV2Zo)7SgztTd$oIrC5pA)OAhi{zB2_9?QE!&L1gr6n zY`1~8uM;4E#Hq*i%zit&ayj3rHu(kTrO_24S&kCW+z6?g3Vo~==_5*fU|O)&XsFO0 zJet>cCAX9t2XD?;a6y7kB1n76ZH2ZWScO; zax`{bmvzNpjrj>)FJIq>Ra2?^Q*l6bDLCa0STL?G@nsPY*D@cL@x&Vm;%OiC8*j%& z4t*e!7=Rw)o|Mnx`DL{wK=mY_Ow)XtlG%#nw3t!y@YRpJ>OTY3amc@kC%_^1^(UzR q0JAD4$^ry@y9*Vw_$WOA1#&mp4vyRk&5KatIF5NRGF14FCZAZ~y=p0Fx0Z8nc=RjtGAoYP;k$AFWHUM2*Cg z_7BN~(*7YC#Il0@!!m#+vuXfKll{X+0W3}O&-TU#EJ6OE(*!I*{vl;>f@NL93AV6+ ze<_yxYX0b5^T)~V8?WpPYnO0|28k!lACd{B`9m_uWd-wxWq?bP@ga z4c-s5L}U>3*r9){OUl4wXI+EGu50#w5X)~ave&Z2Dk8fsYCm2}yMCGWqn#2i(I4@o z_CqqE)P6_?xvZf5uncfXGQg$D`e7XamnP{)gY*L}5g7zsSt#qWGCai+GYC$xHA_E_ zarsST*;*D^=_FgUezKEn;1qr-YnM#7giMr2JSqKq17wm6kZE#$ zSO-9+N%_&B{6I@Y1|ejE@gryObbRQ;!<`1+)beg~>m_MWpH_j^NkFMGL`95s!tN){=|D$}bY!tT00b=&nX4TxlvB|RV#LN+USKr91n zl8k)9X`3`4)&Zbt(tvExfS@HJgMh~lRb78pCAUp>hrqUJ@0|toZAJE4)>mc6w2Kl@ zwqshZ)+NVTyTk&r(I@ex1w=B+k{S>RA(a(0AeI3tNd~AiSwO4Rg$3+8mQNDdYgu0dB$@30R zf7e#tS$}xB9LJ6H%Pc-_#-C@|J(MrD*%*8tu0I~HF`JK)`6&Ie9zFkjzU6O z<6*W?tK~Nv^YsVrt-el^+3G8|+4z{GX)?M0?l$fX;(7jE_VaF-B@6aWAK2mk;8KLF=)0^^fEDGakJ7P<}vV7VLBC6nnSAAe56Fc5_AJVoStVke;p ztR$5H^+Y8kia^{fd)r#XKe7#J-X14y(^l$<<9cSlon5(_?^K)og7ebo8s!;5321JF z)SDW;uNLV&N_=2iu-0g(Q4b!?o12PT%8i3pXDqlN!6y-|AW+Yl_p*z+wY=CkO~ zQCu6RScs+DV1LW_H`@R%2)V-w0uwASJOEQW5zzrD_=L1St{nk_V`xEv4j$)Oj!)hI zPWh`1qjKg{GV~U%T94x7-g!A0yROT+a?&BVnkMikxwIo1IL5f*QW@-MgB7~MDEpESt zkr(|ZpgIisH}MGA=f3;|%^$NXCdvW?(MXQBv-v1J0R>>W8`UL~J1SHGDwB{ZBL>_j H00000lV=nm diff --git a/summary_results.json b/summary_results.json index 9c671e600..3d0d22a87 100644 --- a/summary_results.json +++ b/summary_results.json @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9236, + "Performance_Result": 22.7442, "Performance_Units": "Samples/s" }, { From 079f05a57019ae031461bb7d435c56462ec4b7bf Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 19 May 2025 10:44:51 +0000 Subject: [PATCH 585/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10937 -> 10936 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index edd397306..9f20efc5b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-19 10:39:16,482 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-19 10:39:16,482 submission_checker1.py:3308 INFO] --- -[2025-05-19 10:39:16,482 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-19 10:44:46,644 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-19 10:44:46,644 submission_checker1.py:3308 INFO] --- +[2025-05-19 10:44:46,644 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-19 10:39:16,482 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-19 10:44:46,644 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-19 10:39:16,482 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-19 10:44:46,644 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-19 10:39:16,482 submission_checker1.py:3324 INFO] --- -[2025-05-19 10:39:16,482 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-19 10:39:16,482 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-19 10:39:16,482 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-19 10:39:16,482 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-19 10:39:16,482 submission_checker1.py:3345 INFO] --- -[2025-05-19 10:39:16,482 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-19 10:39:17,545 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-19 10:44:46,644 submission_checker1.py:3324 INFO] --- +[2025-05-19 10:44:46,644 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-19 10:44:46,644 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-19 10:44:46,644 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-19 10:44:46,644 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-19 10:44:46,644 submission_checker1.py:3345 INFO] --- +[2025-05-19 10:44:46,644 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-19 10:44:47,749 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422032 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419649 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1f8248cea..4c11389fb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1212 +1213 diff --git a/summary.csv b/summary.csv index e1333c2b1..e538c0593 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422032,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419649,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 96382f80e..89ca93498 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422032, + "Result":0.419649, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 52e7f4d33a8110c65f31a448d36fa7013df8243d..2bf8d3c217c34fd0e08168263e149ebd74472309 100644 GIT binary patch delta 3050 zcmYM0dpy&98^?b$$zhI_Lbb)V97eUID2*s)txyhA-Kec5_E1sIze#0uW40MaISlbM zVxsPJpj0-w6K&aF2|$bMmVfVo>Qqc4LcPjhY(PG*KS zr13P)_OFxf_N|nAhkJ|G64FdWmzpy7%wxF42M#V+=J556-d*)K;qJWm?os>_?=vb! zrRM9G0j$aSISl#pV(-}Q_tSndN3i_vk%Z@Ve9wgE^YID$H`Akf!1J~#Fqdr(tDnP1 zY!4_!eSWH1;A*V5UoXRy!*brXwZK)1Nx3$y#WzVwfbAEpTn$e)F4xVSJBGKkwH}7X*;Jg0 zT7!Cs(9pSN`SMb1tHDwIq_9pa)7A467dUE7Ci6Th(}|yf690+Lpz>lz<()Q7zQy*g z>&W)ubvyra?h2okW;{1}!E}s+Ul4pcCacvaq*|$KbZMJ{$hl|hn;qA;iIe29y8iv; z-fTv)e=5z=Lewx+-`XS#_-Xm_oR{Rnf#?1aVCgrni|;Z6PGZB*+49b4fJ+4qa$sAx zCC12YoEy>hsO)PS?c5&MgsE8SZg`#*AQf9#BtGI?38=y00+5Zkah`2X&)M?L1(cVD zw?lGrlgq)@>unVc^KLht%$GiNY`+>heE+8Ns_h9DZ<-KAL_8BK0R-8b=x0|6idQ;! zymmSh^9e2xV~T`;KrA9j2WVchS_aYjI|#Je$$W=0WvxwhZDuEWzaLf0Riw6`QSS%C z<0RPAN{7YC22?JUN#?+8B=z;prHqr@S!M`m7(Ega8XIt=&N-atAU11xAVSbcK^c@K02_tY=?$fB4Z*8j?I>T1Y*@kDwU*CwJXbM#o}O z^#ad?u_0J=m+{~^qk|Ducr4^3p-%iy_XS0gvs&t{vHN-*qSYPetQsT6SXQOQ*pOtF7=5MIM+VNdB$y4y&)2()G zKH~EH@Z$GJ+J-&X?9U$`PCHdko6J|n(2KBx#Lgt@zR{m-5K%;FoC`E?;2P}w_M%ks zqtG4@U`Jhum|=vUFUrrMb|auyxs4@J*;4fn+AToltJ^Sh@)zw1SednipgxYICwG~C)6jH>0%2iB z4XgWht^~_gla`6TU(4a)B~MS_CQzyy(n{)f@eL#N4*NMkavCfuK;mckivMilPX=zA&@wgOk|X zi6QZqy#h&~eP*qwQmW>?Ruk1ZU9&1gq5%^8G98k!$(uD%9JLadGhUUp0?wiVZl`5* zwGs(%QV8&{FxNyt*;4mL=}}sts70zHRaatHQh=Vo>B(=jU`hZ6cYW^gTygiMJiSxi zV|q9bld0lGCw$>u&{d^c6tG-{2#|;j8``PMRUQC|tN7iVBsMr=4I_bU(A6L>fbuTv zvL$Z7KV|5#fDrvoaZdc}g~je?J(%M~j)TM*D660)HsU;`Q4we`z@f$_mkoZmDaaY=80hcKOytc{fl!CU}Hwn$EpGUW=B>ENk<7lH@SN)|um=4QLtA=#{O@0(g{*`(=lza{vP8H-KWiWP-XS}s3 z;k90dqiRlpf{owPEPnDCzOq}8nLp|D`KBF6@OdzzkZ);@4(zVa>alR-DmHC2`wdtE z0a{zQLVUmHJLo*b$1Ei-yeUxp!*}E#zMbU36^*5on(3KZ`d(sY1uY}frnP8BIKjX8 zRO~||@sO!sSX(lY91nhYysPRP(SQOPMrc_;N>Kd-Sn!ie;uKWC`Vba&9ibR;iQK;9 zt6+x_p?_toy;po?xW|Iz5}ANg)Cx}6kn)hJbF3{le#`M7xo#0^u?kTo;cbXy$k2;W zxQDni!o~$?xDOrquh?BiqumeH+pap-Dk#anlQd)I(qr!hT19v)O6JI(=Dyt|>4xELq|aj8=B09Ek~35upX=?D*~VktWieRD0*@;YM|Q5E6Yp5;J)8f#psDS1B*MB z@reK3t9#o-kuq;0n}0h(a!zJd%?z0(#s-S*EPPI$`szs}Oo&bXv^y-2-92(uTOsbAB=(H4>YtXiq1axjC3eHgIAUEFKfG z+^%H2sY!f!MH8NhnF$_11-CQ1Ord^;iyh&Y`7H90__xn(Abe9N6$)~1zP+eAdxl zn|Vow{KM+yIg>^F_4M4SVai4Aq3aAAmHec!^q0Y}J`1UP-JXbC+6eifwqI`?zSBXi`zdAE?8OEN4t{-7pBj@Gc@xZ9d}xFvMlX}&iFL^1k<)A4 zNpNNEHs~8^uKz$wd2L#(KX(L;1X~V0+p4pFK6bx6ZV=6g@+F$XzNEOhu1@Hp%-yYJ zA;p?OOB*)_b$ClthJJjoy>Lc;GKw8Bt07aXaN}XjQXICmUhT;oiK|JO^pNlI%9Hsy zCi{iaDeqUm(kZok=*MV?hFPhjhE%s>dSXIVU!+YqdS)pnsSkP|BgUCdfLcb<$D>+d zH%ssNUcQwVow7S1>5q#rx%0ElA!_|}=%CtG z`Qs(!PGpS*R;sk(pFcVGr|B`62G`qc4=j_*HFJbr=60Id!DhKS=2)ZOWyQh(z+DLd abb|!+bByoXa*hQR&W**NBL&y{~YhgzcG z4W)I#+@%XUPjQVNjX$w2fb9(yo)>U9{f$E$t7AdMFH$m!jQuP^qaNrqf2XZO?XRz)3a7 zl$MUo!NRUz1vwqI@enaqHl_u-{w z>AzNX3Y(aO=Fdxv*uPsn%5J=r`6){grv2KzDN2Q3H*K)*B(E>NN`n|McJ#ErNWmgf za&AxU-rGr{3&(xt#!ANiX+nE1nLnL!*=?g{r%ISRyPt|j0LB8~_U)GaQ97kTG(8o< zA&4$~*xxRNk$!8;bvMaBKd6i9M)V{3Q%Z zX@5+;?E#iqfT6vM{y;7%(D$$R#(zb==d62?t4!{OPWhcKkLa;FS9D6$@PU?pS)Fob zz2K#7`yi$iKNLqoOobjJ2^?~IS4e(Uk4$73l0pTWdB?ty${K_QR-_T zqvS;B*Xq8qxK=&gW4;BR598YPrmniZ`NOgDXEwofP>*&HWzoOiQkJZmCqIDHI#9ET zNa=w|Oe;t39CYP5=O|3<%Z&}wA{X(Nu13^_+?}u(c;jK;{k3u;y7GFd$Iobf?I3&6 zBUu&cr8dj_atLuhCJ_e~*95Jw6iWm*brMeZ5LdhO05|%48Iu+b-nC6jzFn{43){SZ zuVD2;U(LYRi>ZWB$(8-Nm(Q*quMGUer(8(nTT4DEYcM=B%ri{Np|maK3x^OmlKi(c z64mANTOiMPA?+t{q;MQ6{}aN&lGT^4flZ}`@MtlaeqW1Zk6bTbhgmMZ0W}>SwPM7n zq$=|d2kdmsw9hcvr7u;J$0De35zf-zIT(*{S&MsP5y7VRo6gx{3A!;z3J%kyS{o)D zHn2})TkWFr{WRIFFzc8O&91rAaV~-98r2C8;^}V>qR=f>tn%mggvWf3ev+%c0nR=v z8Q4-kehvD!O9__Fqk~lhsoj8~Rw_Fg#6tk^J>i_xZX!_Y1-Q@w=0PBtk?!tY#rEta zkv5a$LAX9kU}m?362HWLY{^n(IIWmm?w)pV&#H;2*ETLEybT>wxUn-8-B~r^b}gEk->qAtPIf@K;cIpSi68Uso|;j$ z66?=0B~q)TrGlT-1g)_YBk_o4UN>xwr3|=I=!<17D_0rmiphEoxv_Yygz1{_@BOhI zLy%xehttnv3_mAgpVJIKJgm>UCyC?z625G7?@bcUIQJbypr5b7figf=cyJ@qA6+9m z(hd6+nw?YvH%uw9*t#)_5B-9{EoSv0lM^;p&oe9oM1~f3i8p#Z;%G{DURsl8D6~mN zcE()#Z|CQw&WFfPN}VUrKnS8BDrlWiL_`?Em;E zMaHwqK|MA863H=SF;m#fZgVLL5lh5zt`4ULs@ZU1mUApGUkDMHy4s^*siD4+Nz!4I zhJ{p0+CVq@Y+1{aw10sptlCZJ;lG|5Pin_^6lNBGOZj}y9DTH@Vs-rK1E}WOu@0*u z`0Fg^%e?&88)`06HL@=yTPoL26e>SMh0~JyO@W63iFjmHI@}M#^23 z3}^ST+st(y2EL)rW$u|5^;?RmI*CHggm9kG2qbNIFRxjyWE75p*!m9zeQj zQ?iw`RXA9p%J=^JRidi#dcjwfe$))Nj@tZp+X>Nf)%5K+FBxGRLG-q1+%7!d4})5U zHFKyqXaB>ph6YS_GMIJ%x?rRwhkLcAu z&_D@DnvM_5&|k)+uPsMWnXR$5n#G;a_Ox%$IpQ%#IqM%#{v%wxF7~xhj1k;m*Lx?! z3SrIjlnqG0&;ZL$26zZ6%sU*w(?L9E5X=pntpt{qO)jL&m62ZGuW=Ot0tIk3cLLl6 z7JSF!zE&@tc6f>d-?PXeZ$I!MLbdb!SWklE3FI<9?n!Un``!hUp}?wFkSR#o<@7Tj z;XN&;>#je!v&&`E-QxKNwM)^z3@#3hPJVkQbeMuz=YMR>Nh)aNcP#y|e3~}biqc8x ze6_&;xHp9I!wUf|KiOM0QQXGCCm*JZ|Op4mRva@(GH!LeQQ?Bmst)6HuyxNrNy5CMswP&!aL(#?2r zc<^8DLC4PLqNE`uI*hw+{2cZkV&y2=>gg&gr%tyq)2cJmY*LnsvK@fXCty%Y$GMlg zRZ_ntUT<6E*=&obrRhJ`cxdQj?&Ecy!Ii6?%_28kZTLq`pUSJ`FSI8)JdN5nT#1s% zPn3%_4E=AZc751fS)`Cyqd95hawX^(t2eZwOV6>eFCN~ItF!6+Yrn{Y$sUHd%fg6j z^#*>|YbOFc+s}yamo?hB*Q&VoQG!HEw&8%s+^U|%W&-O)B!n=7!~V$}G3s7xi~}~` zx-TzppFlt)cMbvWk|L#)sL>;c>0KW<_jqFNk0@kZV18_CP^M(d3G0)?X+GYSE8Z6{ zv1WFkl>K%;vYI7rgY+Swj9H$}ma#DCg3fuqlX9XyZ3k6v>|So?n1p2f7_$H?CiGk> zqzzmj=iK(1`~EiT*%!3z+Y4>aGfZ9>EH0Q-tOIrC#zL9jzizt3GytSX_c3#nVQi<@ zZr7w&nwcx5>;!-WvUPOKMKURxZeXrXpEuJ6Ezlw6#yT6lHU Date: Mon, 19 May 2025 11:36:44 +0000 Subject: [PATCH 586/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10936 -> 10937 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 9f20efc5b..772b60387 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-19 10:44:46,644 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-19 10:44:46,644 submission_checker1.py:3308 INFO] --- -[2025-05-19 10:44:46,644 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-19 11:36:38,999 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-19 11:36:38,999 submission_checker1.py:3308 INFO] --- +[2025-05-19 11:36:38,999 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-19 10:44:46,644 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-19 11:36:38,999 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-19 10:44:46,644 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-19 11:36:38,999 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-19 10:44:46,644 submission_checker1.py:3324 INFO] --- -[2025-05-19 10:44:46,644 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-19 10:44:46,644 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-19 10:44:46,644 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-19 10:44:46,644 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-19 10:44:46,644 submission_checker1.py:3345 INFO] --- -[2025-05-19 10:44:46,644 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-19 10:44:47,749 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-19 11:36:38,999 submission_checker1.py:3324 INFO] --- +[2025-05-19 11:36:38,999 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-19 11:36:38,999 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-19 11:36:39,000 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-19 11:36:39,000 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-19 11:36:39,000 submission_checker1.py:3345 INFO] --- +[2025-05-19 11:36:39,000 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-19 11:36:40,043 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,14 +27,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25756 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74351 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419649 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420549 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8658 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0389 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0979 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4c11389fb..efffd3bb9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1213 +1214 diff --git a/summary.csv b/summary.csv index e538c0593..dad98588b 100644 --- a/summary.csv +++ b/summary.csv @@ -5,14 +5,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25756,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74351,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419649,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420549,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8658,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0389,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0979,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 89ca93498..e5276eca0 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.25756, + "Result":6.31175, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419649, + "Result":0.420549, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8658, + "Result":21.0389, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 2bf8d3c217c34fd0e08168263e149ebd74472309..1c33be8778050a1e485e5060dac16c1ac895f374 100644 GIT binary patch delta 2922 zcmYM0dpy(a8^<>@r^hLwgc*$-CRz(6HaTQ-KIFHFQf+cr5|VQGj#fxc8*{2h2_>s$ zG>0BaO`1bLMrjTmNRlFluw;I#=a1k0$NkU!dVOB+>$*PI=XEz7YCgn{k%O99HDP1x z&&i#JK!#(w8Pk6y0$NvfI;gj{!mo^jmvpwL?vFht?Ty%!X^`ovj&rKmy44#ospjl> zUbcHv#}lf-Ysi&Ur~JVTk2UiWre=4GoY>CzbzOmiaf)0#*LkGB%`ZkrX>XQ3$JqEn zPCj=33(s9rNO}e--wBbm)rXl7)~29jNxvXMf6czxwpqddB|viIx#Ugx5C;vN7fL!0 z_t-3m+z1xYj3XGL&egs+u{}?w2vbNFvQhr*uD|%Mw+j84rHymi}wSb=qJw7s3aR1wd zyEG56%k=I`bC{Mk(&pW%JA^z%4i5E1LRa>2ESNN4RfoHPwNkQ+<67IZO%5CE!^4FI zU51S#Js5&*UGpg)ZcjG9%C{Z0axl!`ex@QbqfqNSdtQeLS zJD_ngeG;(EyQMukX-3oFe)LkG@a-zKnv(h1m|BgTp`%8va6?U}_HBUC{|XAntRM&1 z$RnYdSry(pqaFE1p=nr7rWT1oqI%H5RwY;nrNzjq7)=pu2WorH^-#A zg{9c&q4 z-vgUT^%y*`zh*hl-FOT}cd#;hV~2!L)C!?Ow3a z6>spbYA2i+ApPotfjk&Xw<_teGUl7KpgpiVUpS_3_V9n58Da@%&(qvLh?49I5+i;E z)dWv=0%wvtEku*b8cgp@)65-JdUW}!AC4$jGITDTOvQhC1u8W9k-iM_Fa4+oT~=vu zqNuN4k};MHgh!9dCd`&W{I67PAdoAA;1=HxJDKsSH04g;!#Zx+`!aW!!zF4^CluAN z+Z9Q4IgUf&#qon?r#RF%#tYU??z(Ykz!lr4Ru|s&PJdrIXQw%%lc34zKv=}}YW6Lh zh{p#(xzQg^Seby%+X%G)=069FnOZz`mt|nJNx1Dl_JTgB&EtUvqHpd#b_L= zjVC~e*~%a{fU$C`eJz!lp_m4MuJ^*S#HG6~M4A?I(MkwU!4 zkjsk4H#q@3#;^*X=DVh^^^udd{0#NPkPj*{nn^zk3M#GCQxZtVL{g0--(+=neOGtu zf!lPvMy~IFK53m#jwF(*7?l2>d{0iQ-)a>L2u6*mp?qNTl}i%Ff)rO7J`29iR?Nep zu+@CgFkl9L4<8UcfHCJ zijjDz`*K*?ykl4N5F&WuKCym&F}ZKlZ*mI1rLa6{XzAJ0qqRbEJx|mhmb)tWqc)vR z%j*Rc1QNM2cVX&KsE0sB8~Pr-7qjh}URdU5bd~8}zte2ZXxGJ%cP!#h(FL3Gn94D~ zA5*?tUX@ecH8dPuUZ7-(Zu#iXS4l?PMgO8AuV1DysK=D#PZNYvI;dx*zz#7)WfC|pg-hUoH? zk5cor(x(ns$zv(%^0#X)<|s+2O17}{eVrq*h{?}p&_X5}R`?sDM^4L<-jOF7vbK4* zow=N%Rc5#?FWWxTVh^0|!(5`e$_ZKT!F7sk9BM)S_9+3CAWN;r0T7M@M6w!+QZQTp zv&6ztE1|B`NOkKu&AvtrVI=Wrq3*y=JXV+!-W*t@bMl+oUoTwBc}1G4d}Fr=EQ01A z&_^Z6+(BsF5Rkxi%2Yf9+R!?mif2>bkRiFcT`iT0b*$rH4NdkO!5&9?Ku#`XOUu8o zST8Q9kgiOV1(5vCTvz!FNy+ALwF5{q0u~7&pdQO8bBvTSvZaDb(Qs`vJS_7Kdh%fK zwcj>LGbB&>0fYSFZ0OSxD@lBnBj;^sXNd6ypG4`NfMz*tRjI_kU+Wy zs7fI7ND)f#aFir_sFi#Wr~P*#cmRGW4LyHS7fx&kaaKk_!*DHod4946@KtyI;6tQs z-XqeZ?;oNV2`QV)pHIwJ^m}o-AKoA3-p9<(`s$_-0Gm6e6Phcqt(*Tk+<+41^>D|J)0-aRM#Z{BL! zBBhn)8Z zbte&?Z$6F4xa~MN_m5K6wVznTE|6V^omu(hVmBfpwba1EDBK>$r#Q=*-)^{eFL(Rt zpBC6XiX!WIMVraCb?M%E6Qgdaf4{#Bk?~=$6627`eyO4@jJhqFj2aVDC7KQhlt8hF zjXO^v$1?Oy)fr1B+rY~V6;mVaKdlCXK;BXy&?eCT4>fg&;caRUHe=+PZkHL`@%!a} DR6S`- delta 2923 zcmYM0dpwi<8^`Y%a+q@|R9nn)7}ZEoo)N{Y70O|%2elQkABu7qw?k!kFdM@thdKVD zn5ZWTr5JgHtdXJ+LXMFdes?{8{O*6gf84Lv=k>m>&vkvi-@jcRxl~<|QFONWJ`|(? z0OCLZ*aomw^fha^GRNU?gQa%%qEz>ZdI+6+(7E?3D}K7%td;)JM(BOEI(=x=cpD^L3;)$t-r zM-OUVQMaKuIWBG2i3w%iJ}>ExRuO0?*sJd=#LmB_Z_)HZe_QYsY1u^X-dvUo3T`oy zQC&QK88LCxB=uod(CA`j)t|$dCsF)^+g4mH>52^jk!F<_!nU9q5c0~mj9RbLJ2g(L z#|QQq>up|{xyq7l(OF5SMB8xEQ2f8pX_R#O$Lh80a?^_?w3 zuOCLOK`w%;$6k1bfF*M-*WPA$oyP>Cvqt;E0X79V&VudT7Z)*VX4{OmL}lIDY2om+ zCQ#Z)Wygz5uXet%VcZke4X+w3)(hE;olZB;9=Ke-JD=RDdDlNXCjl()OzNs^T5-5z zW3c|A*W_lv_@g_vY9=v;-7Sbh0xlXO0JvGZd@ieTi#FNzzP7m(F#_lEbqjd_mtR3_ zAEvqpYiR`2Zy?ZVA#@+h5O%iIb?Kjb_wBT7jySpJlH3p&94WwDlsL&x(4ep>(Ighk zOi*9nRzf?^UWx|&HNz+T1L$6->THA4t@!#Kk1G%lNr;biz8gHz>5~@S!s?d}?@G=M znmy+P{H?oZMQ{C|bqF)7KE{^aF!PmYx-ZU$g~V)I6Dok@{7; zod6P9s7rlBW8DjS7rYMl`B$%|o+h2L+))R1s%|}6aCZ0>i}Xq|hN5QD1{IuxhWq1U zq(#9be_W%q$aLv-|A+5QCH@H$dIgkIoDj0ce{!}xqxG)ESFfh)(02G$JfJ;(LF=W3 z6`Z+sF`<^{xM{wq&IT8=bj@D6yTa>h6(V0SxWlR!JzKR?nZ$17Sq;ZPBO&1v3qH4L z4}6C{H&RC~eYyDL^`P5*yO~5wiw70Bf)g`YmO^+=+09VrnO)hAk1f%0{;pLK?$RQ( zfjr+AM|W3hDtVliQfk~~#Qj!RNPH>I#$6BY;9}foVXC0r#+>P^(igocOf|ePz75Ca@ zUAHQr`W9-``~{#q-Xp^*0h6#>0mYInhS}m|s8VnS6>zvH^pPzV0Oxr?ItFGR2S^&J z+%7pyEvV>dS8kV+VBRAG1Kta>b5uXFmo7GG`Q&m@|Mgte3$9bDSSOwNqBR>_!2{4< zy1K$^qYB|AsL*Wcp=?w+0YtX=*KiV<;Djj*gbsU+Trcw5z*r;fuxH}fQ!YOItvHDP z^QGaT=L5QD3#~s979p>E*O`bb|c{%&cA93fxFVL(=lvuXLFc zk>3tiPjR0hTNZMXrfad)iEa0i22$Vl-1971^C`VST(Ub{zQ#PRSBqp>8ykW?cr5%Z z3lZqck8l|gOhkvb_W-XJQFJ|HnxA7Z4 z!#DMFGxBC_KHmW?@NSPM#G`xK!oBD_4$L4QAth9FzYN_$DkiRsAXFN{4INR|^P zILcn8yJA=jEh7{bkP_5CE{GYii<^fUSntokY{M5Ju9JH9f93Ai!Vhigv~-EO7wohu zxK6@j#kJgXX2e`%@-k!3?O$@~kX(NSYOM-UB}m^9N)w{jAahT!7kSOA&~l$R^}MtY=gBpRt2{q5@TpSH&o3K3lN)+NhGCC= zGr!}_646hWK>K54XFz*bG@FXf#p-;ds7V~8v4NIjX@Oje2@wjgCbpEw?mUzp;+s8q z5G2T$oruU2=QR>3_o-)U}IXBCNitZg|Pk!_IGEvwg#1;#JIuil-sG?0fnPZ zUf=*mfe`}ua7|HKrYN6;Hq>ynjZ3DBfX05mtBN)daS0ZAChZC5n@j(`nL^3*nJmuT z5J8a8xH{T@i9q*-rTxV(lD0i!PX(nE7}RJ77L*g}%2Pm@DcS0s7P3rIo^j4>PE-Kh zFv=YU>W<#9pDAN(+hq@-n=FwTE$&Gr=sQ>$gf_A0Vn%v2RC14m_O2FwtW*mwT6fWJ z80FU!Ez*PL8P2zc$8s2?Q&DeTm_s#dY^NBCSCJ82`}2ddbjrayqZ@Ddbq?ci3paR` zuG=P9TdA~CNrEYXKDoh!C;{6e42l#CN(4fQ5MW{8jARLWE&`>6fa(^Rj@fnO+K@9j zFxD0XDxymvUxD?J%^Y-_Z@qRc4RP2_yWg1p=pFjbP)q52XoJrZd-_VH;ls+QlMVH! zunsJNos^^#Z;=d_+V9WnCFuD_T!NxCpU{=;UW;d$>+ZN_yW@=xQ|Ae}W_Qfkip zIQg2=SQ5=lIxl`Iwbk#{XCCDbhsFxKE__~q`PbXWS~duDn4UtD>ko>jiLw|9mgm=6oEh%(_CYmS6w8NIo@+WC^c787yzelMl& zM?1~z4~#GO_}9kz^H}V$jc9E8A!f@@A}T&zP+yPiDTbCm|wj#dOnO9 zvLr8*t#nA&)lanA*LiawVls3%-VCUy)&6u+G4M{wL-*Ly-0;LhUjE0lEws(G(muBe zEu&2k`|TY+e8dXo`lp&54)nh^+FWm4Wd;XzUP*i#aFu$BefIJ47qLZds9`j|&N?sjkTGrvmU;xhwOHySYf8s;JLO!$ zxO+YYt6Wmb!RjuWXAeb&=-mI&Wfiu4A>fnj-qEM)l6^G>m<|c4-;ZMU54|%mdA6(m zVOR#6t#1vBW%ub@z?Rs`1{kg1ZG{1VqXYn`0RR7@WBS-m27kZ|*h~W*>A~H<{`?Q~ Cp<{Lc diff --git a/summary_results.json b/summary_results.json index 55aeb0c0c..a27e80c60 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419649, + "Performance_Result": 0.420549, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.25756, + "Performance_Result": 6.31175, "Performance_Units": "Samples/s" }, { @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.8658, + "Performance_Result": 21.0389, "Performance_Units": "Samples/s" } ] \ No newline at end of file From f979ac6a79933f21dcd2e77c68d44f2785a23e94 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 19 May 2025 17:56:02 +0000 Subject: [PATCH 587/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10937 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 772b60387..748ab9b6e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-19 11:36:38,999 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-19 11:36:38,999 submission_checker1.py:3308 INFO] --- -[2025-05-19 11:36:38,999 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-19 17:55:57,696 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-19 17:55:57,696 submission_checker1.py:3308 INFO] --- +[2025-05-19 17:55:57,696 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-19 11:36:38,999 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-19 17:55:57,696 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-19 11:36:38,999 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-19 17:55:57,696 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-19 11:36:38,999 submission_checker1.py:3324 INFO] --- -[2025-05-19 11:36:38,999 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-19 11:36:38,999 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-19 11:36:39,000 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-19 11:36:39,000 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-19 11:36:39,000 submission_checker1.py:3345 INFO] --- -[2025-05-19 11:36:39,000 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-19 11:36:40,043 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-19 17:55:57,696 submission_checker1.py:3324 INFO] --- +[2025-05-19 17:55:57,696 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-19 17:55:57,696 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-19 17:55:57,696 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-19 17:55:57,696 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-19 17:55:57,696 submission_checker1.py:3345 INFO] --- +[2025-05-19 17:55:57,696 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-19 17:55:58,682 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0979 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.47472 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.82399 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index efffd3bb9..b2c0b55ff 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1214 +1215 diff --git a/summary.csv b/summary.csv index dad98588b..7008da88e 100644 --- a/summary.csv +++ b/summary.csv @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0979,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.47472,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.82399,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e5276eca0..fe98db89e 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.47472, + "Result":8.82399, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 1c33be8778050a1e485e5060dac16c1ac895f374..aa1d0e49e2d439f015b1dad728ac44fbfd224554 100644 GIT binary patch delta 2729 zcmYM0dpy(oAIHCIZn-o@b;fZ?Gjppfw^TM!HZ!?Irq&Ugs8x<~FW(&!>ZfL7Y9#kg zqRfQoh$xMeTOtm*%dN|Co6DT9e!s`#_xa=VczphOJznq6`}ux-KF>U!d6WdpZHeNf zdj=~3z-kZx`~q-Q4Ag~uxo{~k)-!9SWF(-_c*|fR%>_H#JC0UOaF{g^0rAE=ZRseI zt_Uk`7d<+kARtDu_wXYpm>2toewa5_6rD4PR;Q_LE75kbnU3Gb zjc29Dwl;y4`mjdnZp;s-xA;K@_&_U6x)>?h?|8+Qnd$9Oo!Ad2!OyJMYBZ z!|A)9FqxM!oAUBYHMz9c^l8NRXIqy}i7pXe4Mas2e4F|8BR9Yo6Qsw#V{1l|X+@(5 zs(Ym<=MQn`SOMyh6Cm^A6@q)Y{beNUdS292Rc?}4AVFkN8`U2@Z?C%By|6F; zTy>`5Y3>WxWv9WF+mCnmG&p@GS(wH$If^ADTrDOdY+QZX_LYP}lh0Smq?4qVUCX*xL%TZ#YN1{V z$ZRUzWmJD&)voe<)%qTb`@wm|eo3k2lnUrK4MBbr_CoWLJpYH|l!k!`L3B`BPTx{#iCuFqnsGrVRye`Rw1Pny6Xp67w- ze5fgGjPy}`s(iHYAX?S2BA(7}kq1K&%|N<6!`6aw9We##F(cVH8d{xcMG9Q=UPf$T z(7a`OF{aTzy+zbKLQiV7FHF>hf;80pZU4+C@pY@d$`SSh|x@S z{PDBL#lN2%_)d;~W*4V-EXQ$(^d+8tC~%Ff=tiwuIgQHVM3@uuNo&~$fmDpx;g2=A ziHe4FbT+IBYKQ<$+sRuQ15$mz;5RoP3}HP!R(UgDQ!2Y&(7CPAE!TO7q>uoq!IBky zL^vdwm+Zy)59bA_Iq#TmtYdLwRY}ENR#OS7TxQSY{|HNXmt90|9x>&`F-=7ZG@*3a z>YaOTX&SqvxnKUKhp=0YmKZX>Jb6&c`{1#!ZZZd!=N%myV-hjw@Cb=*Lx0bM+2gX! zSVWoZA|F7YND#eMH_0|b5twSMb~F&E%z_50Bc;$7ObMZ)4pD7^c5?;7ekX}$4bw!S z=o+ID2SeIWk0A*|>^zwxTb!PI4Mm%b}XH@g4%A3#j2NuP z)#{0skYw&sy)|G=>Ax-WRn}VPwDR$$v6+Dm4=Egli%1!G;rG?k8aC5P-tPyTs@9} z5dn8G~ouVmVz2RZ7HYdu6Bji;yR-P1-MGBdKmL$GrcA*pCDSyq8)sm*7V zKyKKjk+bB(;F0Ht_uCXj86?-X-W<~`r$sAb_^`N z+R+}}(=5~*yL}|CAq{FVmRyKOUt@Z@fL=1)^r0;!P_f zn}UXmd?ge7miu4p$7+`G-plx=MaRqwpY$ZRlXMezuW55!d1@@8+-Ev!vwkzts^=Qu z+=~VTuIP+{E!Gxt1Sl&4V;VQ20%Pfnj7jR~*Iyt1iiBXHrzUmmI!h40MlCa#ya0m5 zGP7M$^vY^_6Z_!}8GRJR31+=hc5^?U}4E)9Rve+eECOJ72x7d=)5VGY_JN~a04oSB#L)s!JI z<%;_4#+yg*qFo=Wf_tkZ4b|k3rPFJJL5T0n=XC2|MnO@HIF9AcHAl{^{v+lM_lwY| z)k=P6u=VmK)cTU$y6o3)@imJb@0Hfvhbxm!)@=C27t9?c(&=5IwxTfA`-@|ixnFWj zZ^ic^=<815um3v70--=lnZ@yg+mt3!DnSzaVV*yIgIqLiyZI30LS1y9## zmmGYT8)y{rD)`FRjAwg;1qa?>mZhhzdFnNB)DCCe89OxNIOqCgNcu*`gBu-3jIeDg zzj}LyN$zbJO6UR`!f|*2V)90t6UYs>P7G@~(r=iu(x0~iac}mw9zZ_JbT(!;!H)7#6W21-9 zpQKN~wc-GJLX?}$`1{?Md3g`vWnN>T)|uNO3xppTBR8L181Yy~VHV=@7|*{P#0 zYcQb^rfHM(&;KQN8G8bz$n`Wh0ON7F2DY$R?gs;F*gRL+5To()USR;>vIziGfd4Pn LA6;Bm!+?JQdJ6;) delta 2737 zcmYM0dpy(oAIHBlQ~J53=t!8(EHSCI5V475bEo_^ai})AjD+Om@*P&laoZTCjuJ{% z%`le^r6$eg{20YtN+d~<%fZO}R=+=fpFci-z8=r_`~7;q9`DA}ji3 z2`$kJA=>9qB3;BE6}pzg+VvL4o$OC;Utc+zcNt$4-#u?KMc9AVJ(i>(9+;f;GLT0d5DWxz1IV9A-yr6eB&9-4Dk`{H7qSqCzP{5mLJeBw8 z>(vKz7pcSa;cIi)rWW#!!%6$Z0wo?E{Y=JN{9)9O+-Fvezlt+cwvOhT+i>@v*=tFF zi}NUc-q(kb*CU2&H$A8i_T37aA)YS}>pKu$e73WGgaH=B?F27u7Ckrb8pW03-^Q#M zunTQ5c!eGr>yE?nF70&WG^7`^)FXboN~@rze=?+1pk^585i|TiLzv|z1oJ;Z@u(T} zHx{Xm4B5E+<^h7zbs2Ae{y!iIfd7wLQ ze`ga%soq8DJ%3vIVlciVm!EaSd&q2boIlO~tH;EyMlj@YKPA?4SicB&)e4ak{FHk1pWCT)JDvh*B`wt_ADTz5khKGNN#J*+2x0=uF~lG4#}p@=XM4r61Dd`ThVaMvbo0?s;a9Q>JD5J`=c14Q7v~ z8FV_LNDk-lXaXpW={FAJ(FP3{%7N}w-v6GIugXx!36Y2u`5&(7JtW+=X9}1KYC^gQ z*sDe7B|`*gz~FFbEnP$gsyzS#6DYFnD+LB|96$kI4@$1A^6F4r zb=HaXeQ}~MR<7xXsxB+C?&rper20kfLs&!#(VGhr6}V(?<$z!wIA};K$4h^h)8G|0 zn*HN1Q*z~ULM>V~HNTm*Mf!K^HHS(q#0eAmx6Xc5KhO)GFO$edg#jHE8aZHql&chh zN4E$>2q{+uaAL09ZCy`dr70!rTFKHyEA|T~^xNR&?zgJvB;6;LeOSa3DIbU+V)edHDtUnSl@}K+dFI27K9Nddec%n8<3;lslZxj&D1i zy5JUVZ%`Y@UQU|lQov9WxtvMuT@$+UlDt;SIezF-b+m`>eA$wWF)z_kLCAq`aFufK zXk3L*a#jZ@LQbAcS%AvCM9)#|P~HC`1+^Y(6Or?fB$3#VXDb(@$}|3UlO}U^v6|y5 zo7FKwz42Vp556^hE&H0}r59+LEHc>0DH&f7$j;MK?)V9a`3V@&hxkp6%06O@_0Rao zJSFzgBn9`OEU`o3!TV5gr}_4h-uqdro)Ud63Vte;uGYQnZ`DqKjerKpl-@vY2cr`) zzhGq^>bxG5Ja5+#F@W%!cton1Urgv3@tT|>?8+~RA6Rj$(J;)C)@ggef=r*N-$?M;=NEc*h~FNv>LyM3#+u z{WImc>vakBU2U!Z@&Ywoa@RwDzFaoq4#sCyMg3xpem#~fe;S}vcE6t4Hf}H)#1zU1 zdxPbQ13+mGdc>lt5Ze^EIJM4gd@RjoNwXK48TX1~(l^iFbsTrvOeNRLwaD(a_}15pD3~#~>vkpDVBU z%5}fgTZK7_Rz|N0I&5&B)E7b)^`0&T{x)~;H``RLvSq{MPY&l_(f;_WF5*X z=Lz}AxA)=9*u)(rFDK?pd);`Qb&p2)kFfKzo;ry{&|<{yn6_z=ii5Ct;%tV*aD%67 zy&2y!*>DAoSHM0(*4@mK`o6?Udp29GzTcVKSc&vq=Tk7*(@qz???tg5BZI5314U5^lU&5nS5qUd@2l zY24d;+UT@#IIEfVdi5lK_}TCPT@5$hKXEni=`FuT_T1^>@5zgzPrGwbCl!Oj>HWjI z6F8dFzY^jDe7J9d?)O5DvCUOA>hrX>T>P6DVVmr>*DHR;{INW%fYyeJ=L08> z`F6$=U+%bwNV{j(Klhh%#?3Vx;t*inW^G(Dx!7qClEmI?iVU{F3#lizn%t|s`7ry} zi)*I1BT5qUc_oX7(xF1je diff --git a/summary_results.json b/summary_results.json index a27e80c60..f9dff4160 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.47472, + "Performance_Result": 8.82399, "Performance_Units": "Samples/s" }, { From 27245510a44494246f0ebfac3486af6e641adcf9 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 20 May 2025 07:20:20 +0000 Subject: [PATCH 588/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10937 -> 10935 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 748ab9b6e..ae361e2e7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-19 17:55:57,696 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-19 17:55:57,696 submission_checker1.py:3308 INFO] --- -[2025-05-19 17:55:57,696 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-20 07:20:15,523 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-20 07:20:15,523 submission_checker1.py:3308 INFO] --- +[2025-05-20 07:20:15,523 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-19 17:55:57,696 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-20 07:20:15,523 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-19 17:55:57,696 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-20 07:20:15,523 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-19 17:55:57,696 submission_checker1.py:3324 INFO] --- -[2025-05-19 17:55:57,696 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-19 17:55:57,696 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-19 17:55:57,696 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-19 17:55:57,696 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-19 17:55:57,696 submission_checker1.py:3345 INFO] --- -[2025-05-19 17:55:57,696 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-19 17:55:58,682 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-20 07:20:15,523 submission_checker1.py:3324 INFO] --- +[2025-05-20 07:20:15,523 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-20 07:20:15,523 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-20 07:20:15,523 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-20 07:20:15,523 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-20 07:20:15,523 submission_checker1.py:3345 INFO] --- +[2025-05-20 07:20:15,523 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-20 07:20:16,664 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.7442 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0109 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b2c0b55ff..cc7bb991e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1215 +1216 diff --git a/summary.csv b/summary.csv index 7008da88e..2009d054b 100644 --- a/summary.csv +++ b/summary.csv @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.7442,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0109,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fe98db89e..bf46a8706 100644 --- a/summary.json +++ b/summary.json @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.7442, + "Result":23.0109, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index aa1d0e49e2d439f015b1dad728ac44fbfd224554..d2279e6664e679f340c7c96343f8af2cfb81be40 100644 GIT binary patch delta 1708 zcmV;d22=UDRku~JatIC^To&nZ4FCZAZ~y=p0Fx0Z8nc=Ri3oqucFQWh_OXklF2NEt z5>MJcBoj*ehhz}T3ic1n0G7bqy!j z!UF!KSnjL&qj$|8C%bRFvNNn*!X+9co-}_*CY0t6$sm^%%paBkE=k6hSenEi)&XE? z@_saUKhP49LC}9=hq5jy1CO0`4IaC$+515(zq!a>%Mz=I?7FD^cr9Bgzr>c+P6?Oj zk9bo1A(>EWKO}=(R?vP}2Dl^{;L>FMunvGrlk}rO`hk{+41%sKlyzAdo??j^1gF@V zrJu*R{3fz&EsLylk}X<4TDJnH@Jm^{WV$6}qCDbB>4$%0LMi=_43b$v`e7L$lVpHQ zlk>wm05VO=j|SxjS|TzCArq7zQicav_SbyiAX~HY^L@zNQ|ISO7oDH{1z9gFd-qa% zC1{dA#3Q92k_n~sLo$eF1?h)n08NqsG)>MA>j2O+DL)#NA83ilAZW@$S(laJrb)aY z*feWae!hPe%_pg{wJfsIrdf1;wB6E9SoW%HiSnaS9r2~|Lo&*e&JPJ8niZTMmH{+L z2GBGqKdb{l(`5W;Fn*vVVwVlk1mlO4;jT%##$B^!^>9ArNV})%&!sM^Kl$4x+An)~w#59=sF3*5{2>`-N%e<>kj)C} z56b|XOxyuBP3jLD1=uv1KN`#*Xo=`FflV-fNEvRNRm>k zO}l@@|FKad@umMmGRl$y5D5XCEFP-(J&SO-9*$pYHm_&_CC zKOUvLOTj9_D*n6%4rNbOa z^(6nNmbFznzZU%;ZMDKcm%J!zJ0(b>N#aZYhh&r`1t1baBr6C&ECWa~wFZzh89;0l zK+PTia+*__@i2u9mrj>UHEXf zNj$%aPvh};(dZ}Hcc~6PtUny?A)bwwOb4UyNTaN7I|e;?MPm z@6aWAK2mk;8KLC?EDG;+N7P<}v=#06nC6nnS9)Hbh!!Qtp?|llRyDixv zX%V&qZON&DLQ`6L6Vk>JwSJJ)I)3{qPU0lsQ;(~e{dRUyx!kKZ`32{t(G|)wf)dc& z2&tP2y>C|O14?{gTCmn=s8A0cE$h3ITgr`tS7$7^Ai*aQq&?-fLc0(w#n|&5DCV>1 z&{5nPr&x%kYk#n1{F^m^3qlrHL12Ofh7Z8hjzn}q3O*w3k81~j;22s^po7PGmg9>z zfKz_5VNkA|N`~ITr1dP0?!A|zvFp05n+?{OKgXZzm$zZnRO-X2IH0-|oN@;&7+070 zx`>BsnGfrD;;jVn*hhWh?X<|T4@43J&|}=Q@-cgQ-X*M1U65j)l6hK?4Y{WUq50zL zF!HAV0#v6V|0bRR$K1D{p#B51D<;YU1shxz>2Z_#C|dywlU6A_0hN=JDM0~(lhr9Y z0rZm)DkdA~jJd2O0RR9{0ssIJ00000000000001hfz^{yDn0=zlaMMS2HPh90001U Cx*)y) delta 1732 zcmV;#20Qt;Rk>BLatH-`lABg?vv&xr2!F|*H2aRDOR&@>SfWPaN&AOnLTUey3}RWq z{$Uxwl36u?rOEzbqX3pB`Dc6M1C}8F&}jmeApejuIKi^6;RIV)z`qpBeKmjduKD9+ z_l;L}hP6w$M1#bW<`2n)()=MA>E*s|Iw;S&83Pij9T6H4ueWRS}W z+7HVBmm~vRnyeqz0dQ%Oel$ow&=QeB(3OR{Zzky$a189;AplMQmSOf30Sh%@XHFqdnqF=Z9pJ zC8Zw{LN+T%KP&@mGGzx|n}0P)KWr3$)8zbUaDJdAqSKVBER=Ow8Sb0J41#^LX6fha z;e5)Gc2CuxOI=idvVGIie%Z^jCFYMtg~XTU56LJ?sy`%zY*tWzSO(Z+;tsHBQh(Se zz^2Lk(O~{SOGKv$Y=Zei%5dYXV*cow&7beX=DzwrTKYfA_e!2_+J7bfkBuUUFZ~~q zQI-^dNC@C$83`xLNI2Pq08ZN^0I^XdoTd&sHV8n_60vE5#|~v(S|&G65`vA>-Y5U( z?-JQ-S!9)s(=KX2`NkRgWo(vcKsNd$zSMw7Mp@DWA|Yh6f(OJhz$VGaC!DrP17aNj znkEg%1`P;WA~Fbg?0-9g3LdZ|A3q``Ad*p*w10p|2(heS0kI5V$^05P z$eJ`DHVSZQ(tvExfS@H})07`|aSml&S_TK%3V;5`-g6Bo9p*@?C;30MtgX`dwdnt7 zs}%;i@T0Fe+PSwR3|89y)}>{DWPb$%$lh}UXoF<Q0v-r3ff1YLcP`=n^WAJ&n{&>8`Y(7fnqx8#q^nd*G`Ih6}$s`&4xSG^*I{I@N z|M)}y2k-EQzW3A54gFK&xREXWKANp=HA?;ko7TzEtalTi_vSaF>7p@?huK1{mfvj5 z*B`jI`Z`T!tFPQ<<71Mh$>jdK+qgG~=lOTp&%0rgr1#(Cmt_0Fg(rUjP)h>@6aWAK z2mk;8KeHGX%MJxu?Y%H2ljtNKf6Gq8Fc3xed`0Aa;-poeSV<}Y>WWH81W~cG>}hKe zKV%!y{yom4X)ATba(wTcxifOHJjf>c1?z-XCCZW%MWC4GLhVZQQLo|!ikzp4(?%;O z(GeV4R(Az6glP+JRvWNhfQtf1Il@ed_TC$Uv15CX)Fr{8g1FUIQXfjYf5V2dZ@L3K zOVfKS!Bb8>#T_s<6A_I_&L*V!u}u%)3_}AFly^8wGJN)Wu+rUZ=#>kn6#i)7ru8IF z?t>GPv2EL=o%hy|Kf{~#>wCXyEL3+Y2B<1HBg_Kz+ExX=EW+U$>ijyKcq@QE9fN-3 zZCqsN10IC{WQcoGKIhLbCiMzcvviinX&z^fb@o8=oaBqIZsb+}87Rjg|0bRQL+ Date: Tue, 20 May 2025 12:15:25 +0000 Subject: [PATCH 589/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10935 -> 10934 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ae361e2e7..c3aaab5b6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-20 07:20:15,523 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-20 07:20:15,523 submission_checker1.py:3308 INFO] --- -[2025-05-20 07:20:15,523 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-20 12:15:20,329 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-20 12:15:20,329 submission_checker1.py:3308 INFO] --- +[2025-05-20 12:15:20,329 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-20 07:20:15,523 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-20 12:15:20,329 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-20 07:20:15,523 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-20 12:15:20,329 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-20 07:20:15,523 submission_checker1.py:3324 INFO] --- -[2025-05-20 07:20:15,523 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-20 07:20:15,523 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-20 07:20:15,523 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-20 07:20:15,523 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-20 07:20:15,523 submission_checker1.py:3345 INFO] --- -[2025-05-20 07:20:15,523 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-20 07:20:16,664 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-20 12:15:20,329 submission_checker1.py:3324 INFO] --- +[2025-05-20 12:15:20,329 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-20 12:15:20,329 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-20 12:15:20,329 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-20 12:15:20,329 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-20 12:15:20,329 submission_checker1.py:3345 INFO] --- +[2025-05-20 12:15:20,329 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-20 12:15:21,370 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74351 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72916 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index cc7bb991e..ab6c70d81 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1216 +1217 diff --git a/summary.csv b/summary.csv index 2009d054b..8135cd6a3 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74351,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72916,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index bf46a8706..70d690cc5 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74351, + "Result":1.72916, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d2279e6664e679f340c7c96343f8af2cfb81be40..4cd321e2b2abd9132c50045f3cc0b1916155c9a4 100644 GIT binary patch delta 1694 zcmV;P24VTPRkl^IatH*A9x)fQb_ibxf4&Y?$&*X1N}xoO#FGXP$%N7XA{m6Tf&s)b z03|bPw1m=R0I^X3N|OMjK>&i5h)z?gvQXA#We8;j0Z7*@0PQgOclffkEU*%mMf*qF zEvxw2$1aw-1WVLNJZb-sOepOil0hsh*gq@-STd^yur%2}Y!txKB>!x0e83Xqe;+zc zz!KyiQU)hj)-{}93k&#{V!5y8kKQ$Zob0~w%FeKM372S)c+&hKnNXTPB!gU5Fn?GE zxFi{0Vrde8SO!SAK zwQQyQ5?fX~C0wFE;z{j?WJ0O^e~=7vSwZ_@8Q_v+fJ>A0!#V&iP127B=?7XOG6=e| zP}XH-c#0)v5S(IbmVO@N@|(!AwJfsINw#SHXx$2&!Y^g*lIfO^iSmdir5}0Fk^wSJ&JXJV$TTTG8k8SsiO3*?Oi+GE86ITWU-N;3Y|YBgfA=ACPo1AD zU37l(7i7J#?A=T4m7q!f5Ra68NG6oh56K{!6{H`Q0W?Vl&@?$etOG#Pr2J@5exM~H zgPtl<2x44_FefTl_Le_t=GYIz0nx&twhw~{%+C5c&E_G4;$=^27e}37^vnA$_Muo(e z<`2mzOR7I4gltw&e^>_CWa196X;OdKD8Q!4{Lx_kKubiY32cJ-L&|XDtYZG?n$4f@ z!{)yFKU(@f%J)j1ZrUaOkBuUUFZ~~qQI-^dNC@C$83`xLNI2Pq08ZN^0I^XdoTd&s zHV8n_60vE5#|~v(e_AFtP7;ER)7~fl=kF5PYguHKjngh_K>5ZQ`ekgEXh1glB)-&u zNJd%G10o@0vw{c2GQcLu$S0h(NdsaX0GcKZ$Oa7vS|TzCcfO%~Ah#s@0F0%Dg9Ja#DSk}|}yiUm|yz`kSoB$2(A^;Ndi4DB`Ih6}$s`&4xSG^*I{I@N|M)}y2k-EQzW3A54gFK&xREXWKANp=HA?;k zo7TzEK&*EYpZDfBqv@hCj)&Pot(M7))J{Y+A_bq2_Q$mYKyVB#DA2*F~csxdZ=51W$ z)CVGo0q802S@~EzJ#W{jE=aLV$ucd-HZN$t7NiCF>PO!6Uw~>H@^9i9aLRrA3F<$y oDJIGS1dARq7qj;$Jpl#EwmO3(lQ}9>0VtD>DkBEkCjbBd0G328;Q#;t delta 1695 zcmV;Q24MNNRku~JatH()To&oGb_ibxe|~m?EqUQbs}d;DB=MvHL^7c?fJg?RtY84K z3_!`u8ZDtT89;0lfYKxYX%K**C8E=msw|XsSs6lEK>*S<3qU(e{vEz-EeouKWzqi8 zcFQWh_OXklF2NEt5>MJcBoj*ehhz}T3ic1n0G7%Mz=I?7FD^ zcr9Bgzr>c+P6?Ojk9bo1A(>EWe?KIHTvpJ2SO&Nx8Q{`n{jd&zOOy1YLHdD~hzx?R zER=Ow8J=Q^83d=;nx&t|xcnxvY%PndbdoJvKU%i}r|?TzyJWg0WTHIcN$H1VLMi=_ z43b$v`e7L$lVpHQlk>wm05VO=j|SxjS|TzCArq7zQicav_SbyiAX~HYfAf9F+*9Z0 zN*A4<`~_JrEPMA-dnIU+Kg1)YACd{B^g}X;W(DboWdKc*0W?j{59^>9ArNV})%&!sM^Kl$4xf7&m5dA7v-(WsF4 z()=MAWl8migpkb&>JQ5Rn@rpRHcjde8wJ=jnLirLA83i_G=WVpe@Gc_oK?&pU9OUvLO zTj9_D*n6%4f2G45N%bWEreEA8ob5K$pBIYda-KqDkUQ|A%ChB?TZ7LL@5) zKr91DGPMSfG#Nl_6hP8s0NG#wK}*D@DL-1~9Ll=143Ml~0NHzP0Bw*ApM>PTia+*_ z_@i2u9mrj>UHEXfNj$%aPvh};(dZ}He|M=4Kde6-?jfFymrMtv{4d$} z#_Nx=OGne2#^TTQhva#Or@w0}@2o$(T#n;L`eha$H{;K<>>kP&+iVOz57!@$*O<*m z$$XT4S&yE7KHqZOJDDVdA6JuFPDg((;~#(M|KJ_|(D#1&xuJh*95=G1-$%36twzbe zVADD|K$`V#;`848W;9(i#_=#)sMYeDjrsZm_f}u0$!ztN+iZMH(lnXee|H=A2Jt-q zF8g^mOp^5eoBWb&Ke+JZFS834*bW8gjJd2OljtNYe`yi618vEvfkIPSdK1#d5w(7h z)H;6qDo)}g;8TyQnf-QlQMuf!Hu(kTrO_42GlCM(+z6?g3cYVu=>tl9U|O)&XsA#R z9xdy;l3U7+gI8xPxFEqN5u`okwnDoQEXCOK9Vq6r=+IHz8mCx@rE9Qd{F^m^3qlrH zL12Ofe})gh)Q&`SLJB@2?T>2*fZ!NfP@sdyd6wggH-J-qvSCoJoJxk?!ld;qj_$pe zqp|C{teXwim_Ns#>zB7-)l}-ksW_mz6r6GgEErdp__~OPYnc!0c;c-D@z_Uw;_bA^ zu@6KN1JGmKv+^-}dfu#1U65j)l6hK?4Y{WU7oqv$>oD@B{{mE}A^#?x0mt08pP>E& pvnVFY0t6de7U{G0C_MoM=#06nC6hQRQ~@cIjVdDs+a~}3005fGDV_iT diff --git a/summary_results.json b/summary_results.json index bee4138ff..caa1464e1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74351, + "Performance_Result": 1.72916, "Performance_Units": "Samples/s" }, { From 31d235f04fb15d862a96fe022810e274c0b060e8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 20 May 2025 14:34:48 +0000 Subject: [PATCH 590/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10934 -> 10932 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c3aaab5b6..88fd9bd6a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-20 12:15:20,329 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-20 12:15:20,329 submission_checker1.py:3308 INFO] --- -[2025-05-20 12:15:20,329 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-20 14:34:43,122 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-20 14:34:43,122 submission_checker1.py:3308 INFO] --- +[2025-05-20 14:34:43,122 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-20 12:15:20,329 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-20 14:34:43,122 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-20 12:15:20,329 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-20 14:34:43,122 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-20 12:15:20,329 submission_checker1.py:3324 INFO] --- -[2025-05-20 12:15:20,329 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-20 12:15:20,329 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-20 12:15:20,329 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-20 12:15:20,329 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-20 12:15:20,329 submission_checker1.py:3345 INFO] --- -[2025-05-20 12:15:20,329 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-20 12:15:21,370 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-20 14:34:43,122 submission_checker1.py:3324 INFO] --- +[2025-05-20 14:34:43,122 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-20 14:34:43,122 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-20 14:34:43,122 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-20 14:34:43,122 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-20 14:34:43,122 submission_checker1.py:3345 INFO] --- +[2025-05-20 14:34:43,122 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-20 14:34:44,254 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,13 +28,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72916 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.64643 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420549 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ab6c70d81..5cc3b964e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1217 +1218 diff --git a/summary.csv b/summary.csv index 8135cd6a3..9b18c2a43 100644 --- a/summary.csv +++ b/summary.csv @@ -6,13 +6,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72916,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.64643,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420549,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 70d690cc5..ac70d5a7d 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72916, + "Result":1.64643, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420549, + "Result":0.430926, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4cd321e2b2abd9132c50045f3cc0b1916155c9a4..b3b5f0af58c4c775be944909d0c23c5e09024159 100644 GIT binary patch delta 2117 zcmV-L2)g&SRkT&GatH-u9jO0qvv&w@1An#^mT~8E?tJ74pVAy;&tTSF9o%F@&c39{ynlOrr(0gjI_NKzb}Fz~$5JY=_}(wD=mafKTYoM` zG2kc$h@yA^J;z8O&s-P*6iGy)NFowN5|Jnd1{6uTMA1;*!E!_~FrbL_fMOs~BsF5w zeoLmM#j-Afe-DavETsa9AA+52(RQ@&lx&M3XE8(;1x3#>5{L)cDMo-r5|J#Dh-48X z$`TLFSD^t$QZJu|y)o2xLV?PeY&MA42=Y8Ci@KypKJ$2s8Am>U3F`%J((5OSgK=-w z!{@;J)@Rc?mRAA7%dh}p&e0?-bV ze}^wy%K|H5S+sw&-Li_Wee7bXORz+Z#FO?9$%NAWAsNK7g8joXfF-kP085ko!$tut zP4dt7#s@4x{(qs<1S~=RA!TraWnIGwwy=PIDVF6`C6*@fhjjp0n!Fzk-Vd}yWDxY&p{z^Fz+-1!gU7CG_I?n{ zZ!WUevcxJPyDn-!UdvX>FR^8{Q^FkS3uRqahNoC!2Ei$|X6ffKF29K^TgxIVon(vFkJhcg3Cf;ex+P?y zJmN{|hh#!2{g4cjSwZ??86cBnfJ~F~!#V&mP0Ei34#+iO_BjLP0kPN z0MIlkKN^%DXo<)mXv#uamzCkBNxUG~G;3CVz81|Vsj{^!veKqmbbhqm(oR^Sf30Sh%@XHFqdnqF=Z9pJC8Zw{LN+T%KP&@mGGzx| zn>9&4Y!ragF4X=e9Dn_Pt~7GT~vSaw@tKP z_J8thiTR^ZA@Qa8Lo&*e>JJGan-$a_mH{@IxC3mO)E_nquxT=XG?+io647Y_n_&Ks zGTb<;m_NE^^XL1pxv&0@mi~|Oy^^P!c8ULEqe$XQ|A%ChB?TZ70ytSl!pSlcPBtNc z(>4h}Y!nHnse_IU0uZ!BY?|P)Ls^%W$$yQLgka;e_sReHyF~U{7FlKEw2K;0zHx?r z8Ji^NQI_<8NC?@i-~q7=ut_rV38!t+fLI5Brbz>`K?8!8hztTAJ5+U9 zmE1Pj9Rk~?y>}MSw-woISznbM(=JLt*^X(sT9+JW?Gg*fMxVr&77)oOOKLzQgnv|4 z(12J5s3aMn(qsX#4uDFN1+=~Kfl9D|*kuEc9m=|-46&?Y0TmXo?^r%bWUpm?m9PW} z$S?hvl%fQrf(IO2q{#rX!2p7mh)q*|w9Gk_b!izOS-}9Z_uK&5 zAQ?Uh$$b@n>>cq(wJbZ3yJWlY;ck<7eiNU@mJj zJR2{W4o3N3vhR)8A7z)0rZMOU|_?V<= zGP(cmHtr4LdH!AY^KO_V>HRnPCE0#(;mKdK3l`W81=+eSt|gP`Bp!duYQr!PMfd#* zqrEM^Qi`x0XiHWN6zbB_oscGpsP#Zn>-g`h*ol)sR$Y$noild^Rm;8XqF=C1XjP*u zNl^rfX)e^ZM(^z^en64)RB_sA1vMJLp=EPdF+-TP@M^UI>jk(dfRrQ5)M)3uAs9Qh z14&&H94d$#Z6)=gv|E2{DEp>czM1?|V>1)c5y{z%G(WZ*0i0pzK!WlP zXGw<7UJq8fn+>CK;grG;25wqU;_Ti#F&q27Px@lChWr`+T)(`HtHwedPQ?H<6=#H5 zpkCXi!k0xjTt}T>hZAoE@W&zOH{OnmOntzk5P(c^Ps&H}^xP$`P?M+mB2E`^p0-&@ zijtJY*J0#U{~0KcL;g)X0jAv7pP=~zvnVFY0tI6osQ+%0_b6Kd2$NGOJOPrEkSRe8 ze;fb+cx*3oba`xLbCY2y6qE8P8UpbilMo*jlVvFo9ND@pt|b8g08jz|01*HH00000 v000000DytgB$M$Z6O&~r5R*105DNeR00000004l2CX(f00000S9Z|% delta 2104 zcmV-82*>xdRkl^IatH;B9x)ejvv&w@1Ap3b%QkPHbLS&>;=%SDBY~L4DMo-h5|P}I zh~$n$#G~((Z@`X}3u@mO$~#!Dk*dj_-a>fk0La`q)n=Kb62JKgeH)dUz6uR6l6v_x?2Vzm6ADz$WHv*@Mv&)GS=1#(@|nk5%sBG-OIR;(lU_er9E^Lb z9zF-&w?3QJvAhZpUWNq-gI1vJu;gf}vljvw06(0e`*@Rmqb}txBLolf;t- z5Xpqn03sQLvVsA`G5{qrYqW&YWB{>I07{bpq(K0JmWWPMsT@ygDyb_thgka*Jk zA(>E`KO}=(Rxp292Dl^{Ut(zze^>{ArOErz;Qc^LLx{b=0^oWd_4#;2Op*aIP0kPN0LU~cKN^%DXo<)mgiKI=NEsev*@pP=25#B7>kQ3uRqahMOkwf?(6ES^4=|G@qo(*0RV-n`Y7Z(RNEaVTq2X zOOzjt>WD9uACgg)bbd$((X8P7uneF{GJvK@`F~*@0GcM_M}zSLEfKqHh$a|6qzrdW z(lzdyH5)(Q2hBZIey(&;`N?-pKP>#Unq4+aoF9$$h%cQVl2MkFen<$}tRVfc46wk{a08W$hqrv%smWWPMs^;0e_JY zQdvO*Vi};4WPnPO1;jc4Doqy9_QnS)!2)8J4Lo)z>yk3WvWf*%Sirtx`6Q9Omi1M_ z5+oqM^kY(r5|9cWup}QpA|)V_QI@oTNC>g4U;(iVV9ER%ILMkbAT|ncY0`jf(14&N zV$+l#b#V@5U0MbQ*$RLD$KG=dD1RO1NUA6KKeeo_()qRM|7fcf2D;=$S=%W=5=|0c z`adM2EGYny5F%MY0Ad+HlBqR-q{#qcqX3d71IPvg2wEaGP5IF>=TO$AWq@P_1IXTU z189R}_#`CvRs6Ac#2?kN>_G04?ZStW6(i$*`mF4wwp+q(Y5yMIe{_+kCw za1Zfpykt5U<$uY(H(q~~T{@cHG!}oZKP1mPJpEl;d1w9M<#HT1(l4|4xEX(*W%p3N z*k)t!dAR;~yvA%kO6H^V%X;+u^ZAzJ-pM2x{J5Ibayt5R8UOf0{|E2zhrajI&kg-k z2sIF^-4XLamnHY|PgmxVQQ`O=hdF+-Bor zlBUVz{=3__H;Cu?ciGRoVUncx-{hBM`@w}Le*v=y7T6Ad%CbPyzq|5dZ)H zWN%|oa&K^RFJo_VWiEJaY@L$LYQr!PgztR{qq{BHA*Bf0fwtt-K%pruy$NaKh+02L zYMs1&Wyf(6@Tteu%zit&s9YUXoBV?F(&!5189@nXZiLiLh2FR8^Z_M4FfCYXG*sva z9h^1?=W&E2pfD1zI zv4X$^3k>(b)J{Y+A_bq2_Q$mYKyVB#DA2**-?pgU*JUwsMs4hscOvy4W$Tlx%zN7{D>PO!6Uw~>H@^9i9aLRrA3F<$y zCML=P1&bar7jcvIC|dywlTj%=0hN=EDM0~(lh7$T0rZm#DkdAswmO3(0RR9{0ssIJ i00000000000001hfz^{tDn0=ylZz@N2HGb8000288rxa` diff --git a/summary_results.json b/summary_results.json index caa1464e1..25f91f15c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420549, + "Performance_Result": 0.430926, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72916, + "Performance_Result": 1.64643, "Performance_Units": "Samples/s" }, { From ff0057ea445dc09e68df1b88ccb7a72adf06a0a5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 20 May 2025 14:38:19 +0000 Subject: [PATCH 591/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10932 -> 10932 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 88fd9bd6a..ab7e9990a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-20 14:34:43,122 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-20 14:34:43,122 submission_checker1.py:3308 INFO] --- -[2025-05-20 14:34:43,122 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-20 14:38:13,853 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-20 14:38:13,853 submission_checker1.py:3308 INFO] --- +[2025-05-20 14:38:13,853 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-20 14:34:43,122 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-20 14:38:13,853 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-20 14:34:43,122 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-20 14:38:13,853 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-20 14:34:43,122 submission_checker1.py:3324 INFO] --- -[2025-05-20 14:34:43,122 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-20 14:34:43,122 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-20 14:34:43,122 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-20 14:34:43,122 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-20 14:34:43,122 submission_checker1.py:3345 INFO] --- -[2025-05-20 14:34:43,122 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-20 14:34:44,254 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-20 14:38:13,853 submission_checker1.py:3324 INFO] --- +[2025-05-20 14:38:13,853 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-20 14:38:13,853 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-20 14:38:13,853 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-20 14:38:13,853 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-20 14:38:13,854 submission_checker1.py:3345 INFO] --- +[2025-05-20 14:38:13,854 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-20 14:38:15,210 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 5cc3b964e..9e3410c09 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1218 +1219 diff --git a/summary.xlsx b/summary.xlsx index b3b5f0af58c4c775be944909d0c23c5e09024159..373d16b08d4066590ae3218ae9296c30b0e5d948 100644 GIT binary patch delta 350 zcmV-k0ipi1RkT&G;3R*H4LF1)0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!LJ z!!Qtp?|llRyDi&EN)WaKZON&DLQ`6D6Vk>JwSJJ)I(hrbcH$)9Q;(~e{dRUyu{z2o z`UUHRRwc@k6h)wz=0epa`q-}HC5oJ3vX5%uwH;gW+b9BlCu$Mer(eN zIK$9@1mzvhk_=zH9;|ee4ZU*Xl)|43OjTtl7TgcI)t@TXJIC*IDB41K_(5P%GEFUsfa`DMFCRi5UH wI9xd?6I6e*CMF961dI(hgp)NYf2q2n%m4rY delta 350 zcmV-k0ipi1RkT&G;3R+9x-G6H0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!FH z!!Qs<_x%c^y)C~|im)AMOI8gO>eAAkkS2+!^*~bV`0uOOiIYH9U5@UZGj|46%f0NP zU$9PSRii9PQ3Q%jk(dfRrQ5 z)M)3uAs9Qh14&&H94d$#Z6)=gv|DT_`=(pKd79p137&H5DLw#WGZWDf$=QrFKeihI zoMGrdg7OY$Nrul}4_3OH4Wn}5l)?`NZdy;`?A|*u8~eUb`eL+({2BgSzr2mB#zGxV z#Q-%GXM|axUfW2f!k0xjTt}T>hZAoE@W&zOH{OnmOntzk5P(c^Ps&H}^xUpclc)J2 wP8V^WwpmGvl9a{QVdPc+87PlK{!Kgqrrg(`p!ox{CMF961lhVRu9Gz?f3s4ekN^Mx From a2e25dfdb6fea77ee22e5200b7e032f9c89ee20b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 07:34:49 +0000 Subject: [PATCH 592/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10932 -> 10931 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ab7e9990a..347924b71 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-20 14:38:13,853 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-20 14:38:13,853 submission_checker1.py:3308 INFO] --- -[2025-05-20 14:38:13,853 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 07:34:43,983 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 07:34:43,983 submission_checker1.py:3308 INFO] --- +[2025-05-21 07:34:43,983 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-20 14:38:13,853 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 07:34:43,983 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-20 14:38:13,853 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 07:34:43,983 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-20 14:38:13,853 submission_checker1.py:3324 INFO] --- -[2025-05-20 14:38:13,853 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-20 14:38:13,853 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-20 14:38:13,853 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-20 14:38:13,853 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-20 14:38:13,854 submission_checker1.py:3345 INFO] --- -[2025-05-20 14:38:13,854 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-20 14:38:15,210 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 07:34:43,983 submission_checker1.py:3324 INFO] --- +[2025-05-21 07:34:43,983 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 07:34:43,983 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 07:34:43,983 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 07:34:43,983 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 07:34:43,983 submission_checker1.py:3345 INFO] --- +[2025-05-21 07:34:43,984 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 07:34:45,011 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0389 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8508 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0979 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9e3410c09..151861262 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1219 +1220 diff --git a/summary.csv b/summary.csv index 9b18c2a43..1ed1ff86c 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0389,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8508,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0979,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ac70d5a7d..e04a136ed 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0389, + "Result":20.8508, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 373d16b08d4066590ae3218ae9296c30b0e5d948..3b47b604464f80b8bac17c30e687adbb0855af58 100644 GIT binary patch delta 661 zcmV;G0&4xVRkKyFatH+gCY0Z8vv&xv41a8A7sw;pK81 zH_|V&__!H=o@Mt?zSw4C@OilYc)Z4JK1$}J^vinm{PX#iAd_tVb}{Zr$(kuCi`nyqd%O8y0#*2&SVcN3rY<~O71qA`w#*+Q+B-)zj+ zAGo*rI!$J)uiR$iW0I!H(cxHpLB`FGjRyJ3=~_uu4~Wc$H|Cw~F62o~55f7}LT zwIu-n08jz|01*HH0Az1tP;zf@b1!3Wa%C=fZET&A%TB{E5JmTVMdW?ryb8!lQVCF3 zR6?SNik)RoTZ{N1+mPn(and$zrLI_x@0~MuMy}?2*+#!$ozSXAS(2g%6w_R&ZH+!w zi}(>m&Qrx{s}QebT=DD z<-#e2?+x6vp2W$$cVaSjU6*vlXbt%@yk5S(kE_N)4X0vν^REKsj)M^oX;A{?%z z&M(7>Hv;%mAM_h<$3+f(z@rd=9O9mo&*J%IwLndt=Ce4R#d)?$A4pM>viurGUiF`W z@;Kz*#1r6<`}z|!vn3|W0tEmjl;3TW^(b2b2a{1LJOPoDjwwL_ev{BCIsx#L3o0fX v+y-T}B>?~cPyzq|5dZ)H0000000000fPvAIOe#JBC6kLPBL>$e00000Hv2== delta 662 zcmV;H0%`rTRkT&GatH-u9jO0qvv&xv41eq$@kg~RJCM6%yYS&|lX!j;pT^_yqR~&X z%eAiDwyr<%?ou6oSbsR&Lp&QVnGQzzU$XCw*B@ndB$@30Rf7e#tS$}xB z9LJ6H%Pc-_#-C@|J(MrD*%*8tu0I~HF`JK)`6&Ie9zFkjzU8=gGD!wMt|qmdj$-~? z#y|eh|G_)_q3`|lb3^~sIBsN1zmI0CTaA)`!KQU`H0#~O=e_yOXu4>O<6*W?tK~Nv z^YsVrt-el^+3G8|+4z{GX)?M0?l$fX;(7jE_VaF-BjD}!k-LGS})@0J~}ZP+qO;GS#J&bGyJuAeeYL|h3Zbl096HNgjt|o+efOvw?#Ny zL!IA*6YmA^r&G`;-p-2*eZZp-fDCaj%IEC)WxGaIp5}` z0_Azgzlj&Xko*1&DLes^lZ`1s0e_RvDLMi1lL{&( w8;lJ&ge3t008jz|01*HH00000000000DytglS?W-0Vb1*DkBEiCjbBd0M4U3*8l(j diff --git a/summary_results.json b/summary_results.json index 25f91f15c..2e7e7c7d5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0389, + "Performance_Result": 20.8508, "Performance_Units": "Samples/s" } ] \ No newline at end of file From f53f077656f071e58d3fe147c87bed3b91a519a3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 10:12:34 +0000 Subject: [PATCH 593/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10931 -> 10933 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 347924b71..23cbfc2cb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 07:34:43,983 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 07:34:43,983 submission_checker1.py:3308 INFO] --- -[2025-05-21 07:34:43,983 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 10:12:29,228 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 10:12:29,228 submission_checker1.py:3308 INFO] --- +[2025-05-21 10:12:29,228 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 07:34:43,983 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 10:12:29,228 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 07:34:43,983 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 10:12:29,229 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 07:34:43,983 submission_checker1.py:3324 INFO] --- -[2025-05-21 07:34:43,983 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 07:34:43,983 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 07:34:43,983 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 07:34:43,983 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 07:34:43,983 submission_checker1.py:3345 INFO] --- -[2025-05-21 07:34:43,984 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 07:34:45,011 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 10:12:29,229 submission_checker1.py:3324 INFO] --- +[2025-05-21 10:12:29,229 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 10:12:29,229 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 10:12:29,229 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 10:12:29,229 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 10:12:29,229 submission_checker1.py:3345 INFO] --- +[2025-05-21 10:12:29,229 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 10:12:30,437 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.64643 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70876 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 151861262..3acca38af 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1220 +1221 diff --git a/summary.csv b/summary.csv index 1ed1ff86c..25fbce7b6 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.64643,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70876,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e04a136ed..8728afc0a 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.64643, + "Result":1.70876, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 3b47b604464f80b8bac17c30e687adbb0855af58..b690e0b7ba17132eb4f35614da24bab098374d46 100644 GIT binary patch delta 1779 zcmVJltbBk`pDLo%VXe@F(gtYH7J3}DHu8o<(I|FBU2OOyPwz3~A{kbi&Z zGyzMHe@GdeU|H92f-NlIUy9|vnm>Bi{Bg4T#w$C++9h0~LE=gChh#!&{*VlES;72a z8Q_v+e2Jw={9zpcmL~50Ek^wGF)(`6dxHL&W8l)d+iO3-6 z%0gL}mEkFtm_cxgty%hcjLUB#%hs~UN+;Q(^`mtwa0wI{ICuHO_TDYLHU7} zhzx?JER=Ow8E%@y3xZ9vX65H=(R`9BTgxIVZJI^rN82s!ge5wjE>V6osw2Ksen>`H z()l4FM6-hP!!m#-$pD%r<$s5D0BD+w9}UJ2v_$N(A(~+PkTTpgN!Pe*)@=NIA2jz= z`MJ_XCi6#w`2#HxohGme<_{^ujkAjRqiZ&Q zz7L!G>i=lz|0v%pdAezr_&+v^B);^2NJd#w03soPlVv2FEFRhL!CZIj(0ux;9V zX90a%k-e7nRoOA^q6C!fn3k(`$#K>$v4Cv!NqlJmk&LpW27g3CNM!{Lh-H9Ek^w4B z77*(Is5DtX+Z!LK1Ph2=Ht^V?tV_xe%PJO7VFCM&<&#AATGm$yOOSy4(vL|gN<9Yh?IaxMp@DVA|b@Gf(67ffF<*5;2>+#fY>O&rAY&_K?8!8h)q*|)WtcJb!izK zWGnpnAA8R=pnr6jBdMO`|J1U!O6S+2|D&x|80eB0Wo@SfNi<1(>Hm<7vZMe+LWpDq z0f=P)Nv75Sk|qO)jRHuT3?Lf}AZUr$H04LjoI_ccmI0C#3?O^Y4WJE@;ggWuSMkSo zc99HROCs4We7M^rp5Mf$@p!yw^posztt+>!>rcG9RDXvb)*lY{5YNU-rh`%bm+X7v z^+(yIqv=g!@#p$O^1Q>--?f!@)*oIj$8jV5GK-I!@#k4~59Nz(HU^)E>yO84%;uwH zK1#o=N6$Z>Z#nLrOp?Kmt4S@Vqd%ANk3aN(@D6|Idq4f$&_6Yf8`;wDquJ_KqvT(( zX`LL+dOkPtd2fC*nl2jSc$h8JYWdB^eEorYtFO~!w))C#Ha;e4noRD$yN!E;c%FZk z{k$6{NqYZHeo3|;TzK*qvj-N~4u5$~Nqi*%002+|000pH003lfV^DH$aC0wXZ*pZW zcx`N*lFe$vFc5_AeF~$yEz2RL6}AIy$*F-tQ(AHp(#8?Bevs5Ue)}p;;w0cxkE@ye zc6L#@*sC`A1?Q#F6(U)V642ZTshbLYte5EnN_=2iu-0g(P!Aq0>bsI#%72Z6H)kxk zAi*aQq&?-fLc0(w#n|&5DCV>1&{5nPr&x%kYp`Yfn>B!oJio^Z0uwASd;q3)B%%{i z@DXW$Tsr^+$IyZT9Xuu(!541;r~G8YpjscJ#doM?0*L7Jp8>}%u!C$M_ z_hHpk>cgoxpt=;CatACJS4fxmx`>BsnGdUY;;jVn)JJ{d?X<|T4@43J&|}=Q@;Q5c zSuast>i1&nY?q@{e0?-bVe}^wy%K|H5S+sw& z-Li_Wee7bXORz+Z#FO?9$%NAWAsNK7g8joXfF-kP085ko!$tutP4dt7#s@4x{(qs< z1S~=RA!TraWnIGwwy=PIDVF6`C6*@fhjjp0n!Fzk-Vd}yWDxY&p{z^Fz+-1!gU7CG_I?n{Z!WUevcxJPyDn-! zUdvX>FR^8{Q^FkS z3uRqahNoC!2Ei$|X6ffKF29K^TgxIVon(vFkJhcg3Cf;ex+P?yJmN{|hh#!2{g4cj zSwZ??86cBnfJ~F~!#V&mP0Ei34#+iO_BjLP0kPN0MIlkKN^%DXo<)m zXv#uamzCkBNxUG~G;3CVz81|Vsj{^!veKqmbbhqm(oR^Sf30Sh%@XHFqdnqF=Z9pJC8Zw{LN+T%KP&@mGGzx|n>9&4Y!ragF4X=e9Dn_Pt~7GT~vSaw@tKP_J8thiTR^ZA@Qa8 zLo&*e>JJGan-$a_mH{@IxC3mO)E_nquxT=XG?+io647Y_n_&KsGTb<;m_NE^^XL1p zxv&0@mi~|Oy^^P!c8ULEqe$XQ|A%ChB?TZ70ytSl!pSlcPBtNc(>4h}Y!nHnse_IU z0uZ!BY?|P)Ls^%W$$yQLgka;e_sReHyF~U{7FlKEw2K;0zHx?r8Ji^N zQI_<8NC?@i-~q7=ut_rV38!t+fLI5Brbz>`K?8!8hztTAJ5+U9mE1Pj9Rk~?y>}MS zw-woISznbM(=JLt*^X(sT9+JW?Gg*fMxVr&77)oOOKLzQgnv|4(12J5s3aMn(qsX# z4uDFN1+=~Kfl9D|*kuEc9m=|-46&?Y0TmXo?^r%bWUpm?m9PW}$S?hvl%fQrf(IO2q{#rX!2p7mh)q*|w9Gk_b!izOS-}9Z_uK&5AQ?Uh$$b@nY-bnA zz_lcj?ZStW6(i$*`mF4wwp+q(Y5yGwQWVSoMMa1Zfpykt5U<$uY(H(q~~ zT{@cHG!}oZKP1mPJpEl;d1w9M<#HT1(l4|4xEX(*W%p3N*k)t!dAR;~yvA%kO6H^V z%X;+u^ZAzJ-pM2x{J5Ibayt5R8UOf0{|E2zhrajI&kg-kGH>2sIF^-4XLamnHY|PgmxVQQ`O=hdF+-BorlBUVz{=3__H;Cu?ciGRo zVUncx-{hBM`@w}Le*v=z7T691+y-T}ljS5If6Gq8Fc3xed`0Aa;=BsTN>T|>S5!ix zh>D$MPg{%lA={AV?{U&LZKbYQj_;i_cSf$}d)Y?6V4cvaMp=@g2o%#?sBMitR*U!% zMb1;jX{!~~s0WAU&0WO|VcNo*)ds8=;GzIhjxbZBo%e=d?AQ(@bxCljAa1mk)Q8e; zf3czLn{EN;X?l+(c*?1#cmT#`BBCRbvk7T_Y&!xt!_a~R zHv;%mAM_h<$3+f(z@rd=9O9mo&*J%ICAC0Jp60VSoyB>!N*_p3lCu07Mqc%wf$})y z-^3H(ko)=*G=H-vCdvW@049{*ZIkvWTLA}?QYkzEk&}-pK>>b~(J49s@RJNGCL7!a tWwj*%002+|000pH0000000000004l2(UVOoJ^>|@j4C4r*Czk~003+#HY5N5 diff --git a/summary_results.json b/summary_results.json index 2e7e7c7d5..660ee3728 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.64643, + "Performance_Result": 1.70876, "Performance_Units": "Samples/s" }, { From fb36e72312b52f782dbc3e7382d6a5b67252663b Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 10:19:29 +0000 Subject: [PATCH 594/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10933 -> 10933 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 23cbfc2cb..1b6fa10f8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 10:12:29,228 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 10:12:29,228 submission_checker1.py:3308 INFO] --- -[2025-05-21 10:12:29,228 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 10:19:24,625 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 10:19:24,625 submission_checker1.py:3308 INFO] --- +[2025-05-21 10:19:24,626 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 10:12:29,228 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 10:19:24,626 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 10:12:29,229 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 10:19:24,626 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 10:12:29,229 submission_checker1.py:3324 INFO] --- -[2025-05-21 10:12:29,229 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 10:12:29,229 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 10:12:29,229 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 10:12:29,229 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 10:12:29,229 submission_checker1.py:3345 INFO] --- -[2025-05-21 10:12:29,229 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 10:12:30,437 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 10:19:24,626 submission_checker1.py:3324 INFO] --- +[2025-05-21 10:19:24,626 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 10:19:24,626 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 10:19:24,626 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 10:19:24,626 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 10:19:24,626 submission_checker1.py:3345 INFO] --- +[2025-05-21 10:19:24,626 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 10:19:25,674 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70876 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73856 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 3acca38af..9d63033fa 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1221 +1222 diff --git a/summary.csv b/summary.csv index 25fbce7b6..7aaeaefd3 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70876,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73856,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 8728afc0a..e48f515df 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70876, + "Result":1.73856, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b690e0b7ba17132eb4f35614da24bab098374d46..4275d8e2c8e71fc3ac63090d13ff0e46ff6cc79a 100644 GIT binary patch delta 1590 zcmV-62FdxgRkc;HatH(g)-!dpb_ieye;t&)U!qkBlxUK8(f}fvP#QoagHTp5fLI2g zWM++)P?`)NHVQy#5`Z)aK+qD=X-ZWV%DSu!p{yVP>6!(g9VY({U$&M7R>HDq|7g2q z6<_<<#Zs4Gi5iI~?H`f}rTs%Ph-C%)hh+duX4L?eCi{nt0$7^lpY4qgSc3dRf2Rpp zg8W0u-~`LMh7)XI0sm4g_tpH-yXKFR-8Wv@8P+c05)Behnm;5HO7n+gkjo0@56b|T zB;!jgP2vyh0I)Q9KN`FrXo<)m=&?gtmz06W&bkJVUDxdWAeP@;WUpn3RYZ1O)PB5{ zt(0G4%W9{DOY}!Psr`^lD77Dwe?cxQXg@3iT#^iMX|jG;2f(FC`q3c$KubghL01;a zx~vROvBV65Q*6!B&tqJE6Ir&FMOHe=7OfwxTY*#frL0{t-4ZfU9`U5~Lo%V1entu1UJaU9)E6=lh_!r^?Tj zE-F9yuIYz`zgDx$W{LBo(H`-o^FuPqlF|_&nX&_~&6=biHVVLLa(*;8 zKhP4JJ+Q*fg0x8q6PPiRd(eO)!5*8E%|a%pYB|`SX3) z+*kicOaDjtUdhu$J7+C>d0-#A0RjLi}a$VQ*Uml_bs zC`)=kB!p~M@PJqb*d!VGgwr-@K&%5m)1(2}paDTkLX3y5TtB{d)te?lrNXh19jRFVu(X|jM= z2SBCC0@~jAKqXi}?6QH!4rN_ZhFDgyfC>xPcPyVIve&Y{N?3vfKt*!2_1$ z<42?fL^8^f77z&`mK7`@mH{l8UjqkOlLo{_0WM7%kPR9Tv_x#0@}n-!p{z^G;2>M! z&;Qtat^uXPe;i5mB>$(DwN*O57X2S>wZcG`yeMlsB}k%4;!FRBWRxWZAQD0(D+oX= z14uHp29PuvKx`C1(qsVHU;sf&#HJ}fTIL+ey0i?CtY84ydu{-2kPM%M6cl2+>AfZvU@0BY_l==JY0V~USl>NCG%1G zWj%WS`FzW9?_`n;eq2pzIUW7EjDP&0|ATk zXx6)lJQZpZ9k5_rUE=E^9<1`x{t zl+3Kr5=xT+#6|%qO#+Yx0SH?<-VFfde{7Mvirs>JHy%~T%tkZN%MzfLTUbx402h){9zg3 zl4N{|rAho@9RQXl??;3811%951U+^r>yk3?*jd-$vFn<>AH?#Ti|n;5v5Ls9i`tLZ zvX$~nY+3D;aEbniC$%4v38nT!e=^8r1?`7rfJ>4AE=|@C>j1bkNk1B-A83ilAn3|M zS(laJDVCT)aEh&2`gx4YZz9XqvdBs&*`oEMbt`ZRzm&C0rdvWL$|IhXen=*i(htcX znH8iTmH{$J2FNryKdb{F)1>@pP=25#B7+bzLHQwNc#vg(%?A#$H7h^ge}~LHb$+gN z(fP?=koCf{cQ3V9f+qPxJW~20nNUhUB!g&HkbYPO&?Ff^)8zcH4ggJ)@}oidftH92 zf~G8#by*p1n#2o&O|xd@=WEe?k}6xvA}ej0MdwG`E$xIQI-V|3el)5hzEpllMp@GN zAt6Mwg7d>NfF{WRnkMCke{}$8nv5R}#t*ba?6M)6VEm9W+%-wpxNFvI{CpoY_f+}0 z(naMb-!=WP@YiZ~*(`B>G}p6%t>X zKP00pss4}VHsePi95iiN&R7?0GlTBM}zqTEfJk2unFc5DZ`Dkiut2!Hh;bk zoBQhjXzBkb-z#~#X_xpvHi{&^^nXZ3SyBKZA%K%*B%CZG;bap6IBk;v#72>DnmXv% zAOJy2#HI-zJCt>4f0^7kNeDJhd!PKDze{AVWsy}jPP?c9M3LX&40GlKupK#hH4TyCBXqq%28#EwjiO3+}u|rjtRmp9W-661T+IwdK zeOr;emi1NHG3}xRlj0=USwPzxAE*Qih+Q`D*rBXT$`H#c7EoaU`;O(4MD|+NR|!jyfc(;rNhwM|DtN$> zeEf)%fJjDJ(gGqO#Ik|~#4>;-^K0NBYtn$&D8Qvj1F}H_f|iI)Q-0LNIh1v2860FQ z{P`by&o!WQf0!ewp5*`3vbIX+*P{QUtyUQ5k{4xdrvynfNqp)5kc_gV07OEFWCa0; zWdKR0)&P~gIux2@|>yt`C~e;?K#4)+kx#!IGyQT~_gd*k&- z*`=fDO=I!r`a|-(!_(ijm3P)3UM|ORBmFXqkDKx5S#}TQi)}UrpNH#@$7{^yqhvlx zzpO{kKc8QKr+x)hvp2P_y@m-xDfhijP+ zt9atA1o6~Ied6u3$gvMZ5(Cg<+$*#4IeUItFHv3O#XQaDX+hQ`r=+086)o&m?)x1XT?1G6V4$^rzwio^!9^(Z|71bIzKe3LdRf4O${2LJ#7 diff --git a/summary_results.json b/summary_results.json index 660ee3728..edfc71e93 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70876, + "Performance_Result": 1.73856, "Performance_Units": "Samples/s" }, { From a1f849575853c59397eb24aba50a8cb6f5d9edba Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 11:10:52 +0000 Subject: [PATCH 595/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10933 -> 10935 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 1b6fa10f8..e31af95a3 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 10:19:24,625 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 10:19:24,625 submission_checker1.py:3308 INFO] --- -[2025-05-21 10:19:24,626 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 11:10:47,421 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 11:10:47,421 submission_checker1.py:3308 INFO] --- +[2025-05-21 11:10:47,421 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 10:19:24,626 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 11:10:47,421 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 10:19:24,626 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 11:10:47,421 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 10:19:24,626 submission_checker1.py:3324 INFO] --- -[2025-05-21 10:19:24,626 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 10:19:24,626 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 10:19:24,626 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 10:19:24,626 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 10:19:24,626 submission_checker1.py:3345 INFO] --- -[2025-05-21 10:19:24,626 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 10:19:25,674 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 11:10:47,421 submission_checker1.py:3324 INFO] --- +[2025-05-21 11:10:47,421 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 11:10:47,421 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 11:10:47,421 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 11:10:47,421 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 11:10:47,421 submission_checker1.py:3345 INFO] --- +[2025-05-21 11:10:47,421 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 11:10:48,511 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,15 +28,15 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73856 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70122 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.430926 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.410575 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.44519 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.08844 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0109 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9d63033fa..d97ddee79 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1222 +1223 diff --git a/summary.csv b/summary.csv index 7aaeaefd3..6d6afc281 100644 --- a/summary.csv +++ b/summary.csv @@ -6,15 +6,15 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73856,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70122,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.430926,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.410575,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.44519,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.08844,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0109,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e48f515df..d01bc33e7 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73856, + "Result":1.70122, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.430926, + "Result":0.410575, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.44519, + "Result":8.08844, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 4275d8e2c8e71fc3ac63090d13ff0e46ff6cc79a..d71ae9767b11d265417ee41ed9f4864fe6ebeb70 100644 GIT binary patch delta 1569 zcmV++2HyF#Rku~JatH-^(FhE2vv&w@0|m-*eV22S>I-{+B`5|Q#Q;$h51{853FMgz zBY+}_NEAs#qDUeV#lV0fDVHc3$~#z&CLz#TlMfc@V@ogw2tLffbcRb zKp3spJSMi+$z><5&85Bp&bL6;|N#-DYCZp zCZ;xjFFv;d-z#|zsrY+O&lJ9$<9ZTJ%ml7P6Elb=$pD%p188D~ps7u0k{%FELl2$p zjSpzbmWYl5Xv(rKDT9|)S=Y8D(bWF|H0xMnl~-Cf6WczgbZysDC2aCHN}nlw|Izg& zo0tha$tGryO_BjNNe0*?@9rX-wuw#B1F~s<=%KT{@qta*646lt$)+spvNGH@F@)Qu z{f}Z(J?TVS#~Lebn|5*A3|qeCmb~1f_&fX06ux2Z1`D&|CDx8Y9mJE`4#@=4+aVc5vx2$9GJqy? zcK}V3yu*4xG)?}F0)YowB03772@Vf`DZ_1(bdB3)of^;gLUUie9=+LstggUij z_E-;Wq88#wwTEOv>GqHevRT3IVHsePs4U2)$?#!4Ae$!1M@a?e3|b;Kic~nyp{z^G zVA~|d4{Vz?>pWk{rhd|qc27N@OH~wovY$JS?Uuc<8I+)jnusS=ACd{B>q9bsh-L-b zhh+dwX57FVnkGp2gcF!YZ3eV^3q1;pB z=SuA=KaT4=e#tL)sIY`eR7pH3{g6y3tsjy>Dl2$DECWaqDL&VfUM}q_eEfJfh{2iimDC^QPd5mQtTTJpo z*K7eTKW3-Y<%UbaWO_GQ9c1$ptO{HD>#3||byZHVx@Z8YmTwjB^<5~F+YsgjRM{^- zYqQ1?8xsU}8W50^YBX-W{b18vEvfkIPSdK1zn5w(7h)H;6q%8uhCkW-JVnf-Ql zQMo**Ciw;DrO_24S&kCW+z6?66?)&S(gjLE&vnM9X0tI=|2n=zP_9$Bc z3zJeQJOP!Hk10U`f|JoHIsx>P3@RoYb{Y_nB>?~cPyzq|5dZ)H0000000000fPvMM TO)5SCDU*yUBL>?i0000027b|z delta 1608 zcmV-O2DkaQRkc;HatH+h)-!c)vv&w@0|d4emT{Bo3wwVqM={_i28g0~06oV@AkSPF z0Tf9@qDUeVMG}!H1_l&KxkS-W-obK2F)*Nr^?+g^Q6x2D(|${)rNy!?f`1Q+bu6U< ziXVcVZP9kL@04tdA!ji}76nDmF%pOe*(pYVMG}!Ll89sxBgzsF%~zoTMp7@IhP^S= zcS3>6nQVWE*a-4GDvP?LNIvs;iy22ge+laaZqn-~i-U1*)x+n&`_^aEI+j-f!ppD# zVbBV+9hSTXp)3yJs$Ebh?%U@i6NpFdDainbEF(c=8A&3^DEZRKt8Iv}NPCblntJGL zZ`|StEfF0>?BC~5)+J@~HOIHCYb|~nT@)6uA5ecj?1wp!gE*dL56Sr=zP?kfAlnkZ z8p~Q%Sb@uTn~AN#^jnVOsxmP7tM1PfzRK>p5=_hljs%muS`T2741h^8048Q|kKBeR zn4|{;)6hdl$#)jcpe3TC_!3N6)@5a|SCSBHmW2iEBbbGqGAC18E%gL6>sVuzEpsz5 zwRwN>xfS?c$#Y1>-+Owd@a-JelW1Zla3z|UK{QDQ&?Ff^6Eg%&Z9Ft}ofd zOyEg2F@tQ946sQuz$STj7umE;Y?2<3O+$YVo$ZYeY|561juJ>VWm%V%;kJn(+&1lh z6r1WvC)zsJSZUj|i`!<{@-4UIpeb7-ItrjE%etftw@uPDZkx`30L?nqSZUjIcDK#I(SF&sS>k`9 zD3lA_P_l`cAdqZg2H7MTV3TBkO(xI)o34pX=GOq5h8{ZG8z0z|EfF0hlx)hfE-S-r z6GOOdy8kdXpK_$FV~v%zO}Dshx-Bns%g`*bb`596DmZ7*60uRF!g&s5U0Md) zCNX|s+pJmV`ARnRla91|>iJx%qUe+T+;MET?ERCV1WnXLJgNGSOekF+l0knoE7(3P z186ei2Hw~-89%fKK+`1sD5>O}K}$qN0ka3Hx~vLzO)`JnHEXte9wAeBK3@;zo*F+_ zYFGJjT;K6aez`-1B~+qH;z{X;WI}2EkPK2;!TVtupptntK&46jp*;YVCizE!{R1r# z8wGx$0m{0h46!6#BbNK-0F{4Y=|2g}J#~PtRJeHv14?{gTCmn=s8A0cE$X|HTgr`tH)kxkAi*aQ zq&?-fLc0(w#n|&5DCV>1&{5nNr&x%k+hWW3H`@X(^86kv2uy#l!0-W>+L4G(NWn*> z{c-I85FA4b3Uu(8WCUNl0i5!a4TEy!R5J7yCaq_2bnm?!ja}De-E6SN`~-ikUf+jR zQ>hQ9;(+Q>aLOI9U|e0|>mnYmWj?Ipi8m6&Qy=w-x6>lWJ`hO^K#y_H%IEC)rCFl7 z$cuTJ&(nf5B&QhUkrwmyVdPE!1*lF#{!Kgsj=66?LH!4_DJIGS1p(GGb#IgSC|dyu zlT;}@0h5!FDM1Z@8~^}#Y%g?~c zPyzq|5dZ)H0000000000fPvE_lkp@IlO!q-lQ$+13jhEB000000Dyrfla4AQ2H7V7 G0000iR@4>% diff --git a/summary_results.json b/summary_results.json index edfc71e93..be74e1a87 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.430926, + "Performance_Result": 0.410575, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.44519, + "Performance_Result": 8.08844, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73856, + "Performance_Result": 1.70122, "Performance_Units": "Samples/s" }, { From e8f10b1921d307e8972aa5a25739cb31d407e372 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 11:11:59 +0000 Subject: [PATCH 596/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10935 -> 10935 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e31af95a3..bf350c091 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 11:10:47,421 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 11:10:47,421 submission_checker1.py:3308 INFO] --- -[2025-05-21 11:10:47,421 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 11:11:53,619 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 11:11:53,619 submission_checker1.py:3308 INFO] --- +[2025-05-21 11:11:53,619 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 11:10:47,421 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 11:11:53,619 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 11:10:47,421 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 11:11:53,619 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 11:10:47,421 submission_checker1.py:3324 INFO] --- -[2025-05-21 11:10:47,421 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 11:10:47,421 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 11:10:47,421 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 11:10:47,421 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 11:10:47,421 submission_checker1.py:3345 INFO] --- -[2025-05-21 11:10:47,421 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 11:10:48,511 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 11:11:53,619 submission_checker1.py:3324 INFO] --- +[2025-05-21 11:11:53,619 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 11:11:53,619 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 11:11:53,619 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 11:11:53,619 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 11:11:53,619 submission_checker1.py:3345 INFO] --- +[2025-05-21 11:11:53,619 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 11:11:55,218 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index d97ddee79..d6e9296e2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1223 +1224 diff --git a/summary.xlsx b/summary.xlsx index d71ae9767b11d265417ee41ed9f4864fe6ebeb70..31abd3fb02ef3c1c33b309a0ba1b68ea21ca23ae 100644 GIT binary patch delta 311 zcmV-70m%NhRku~J<0J$w#JLTV=OiqDsY@}o18vEvfkIPSaud?V5w(7h)H-?lDz@V! z;8Q^tX=cBjT~sa)vWhg{#w7j_oK!_omRyF)g@;Ivp~JJb&0PpVR0>WejQf46TqL2L7#aW2N~*sMMuY!Hu*R43>b3XeuDZBvneJE1OzU` Jxeb#zDt~A%jza(d delta 311 zcmV-70m%NhRku~J<0J%j8W50^=OiqDX-W{b18vEvfkIPSdK1zn5w(7h)H;6q%8uhC zkW-JVnf-QlQMo**Ciw;DrO_24S&kCW+z6?66?)&S(gjLE&vneJE1O#>( J5Rj8NDu0>6ll1@q From 0c28a4dd71aded3a4d83466c35e83bb4f1c805b9 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 14:47:52 +0000 Subject: [PATCH 597/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10933 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index bf350c091..0f81f1dbe 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 11:11:53,619 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 11:11:53,619 submission_checker1.py:3308 INFO] --- -[2025-05-21 11:11:53,619 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 14:47:47,901 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 14:47:47,901 submission_checker1.py:3308 INFO] --- +[2025-05-21 14:47:47,901 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 11:11:53,619 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 14:47:47,901 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 11:11:53,619 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 14:47:47,901 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 11:11:53,619 submission_checker1.py:3324 INFO] --- -[2025-05-21 11:11:53,619 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 11:11:53,619 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 11:11:53,619 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 11:11:53,619 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 11:11:53,619 submission_checker1.py:3345 INFO] --- -[2025-05-21 11:11:53,619 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 11:11:55,218 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 14:47:47,901 submission_checker1.py:3324 INFO] --- +[2025-05-21 14:47:47,901 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 14:47:47,901 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 14:47:47,901 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 14:47:47,901 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 14:47:47,901 submission_checker1.py:3345 INFO] --- +[2025-05-21 14:47:47,902 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 14:47:48,904 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,8 +27,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.31175 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70122 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71572 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index d6e9296e2..eb0b863df 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1224 +1225 diff --git a/summary.csv b/summary.csv index 6d6afc281..0238c2ef0 100644 --- a/summary.csv +++ b/summary.csv @@ -5,8 +5,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.31175,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70122,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71572,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index d01bc33e7..3d15e79d3 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.31175, + "Result":6.29378, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70122, + "Result":1.71572, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 31abd3fb02ef3c1c33b309a0ba1b68ea21ca23ae..4c5e7f3a2eec4d0bce023e7800a033421bb0ed2c 100644 GIT binary patch delta 1791 zcmV&i5h)z?gvQXA#We8;j0Z7*@0PQgOclffkEU*%mMf*qF zEvxw2$1aw-1WVLNJZb-sOepOil0hsh*gq@-STd^yur%2}Y!txKB>!x0e83XqAAdSc zz!KyiQU)hj)-{}93k&#{V!5y8kKQ$Zob0~w%FeKM372S)c+&hKnNXTPB!gU5Fn?GE zxFi{0Vrde8SO!SAK zwRC9vK?#>m376=PcvAZznNVs!B!7ckR?vP}2Dl^{;L>FMunvGrlk}rO`hk{+41%sK zlyzAdo??j^1gF@VrJu*R{3fz&EsLylk}X<4TDJnH@Jm^{WV$6}qCDbB>4#)ODgBTP zl379eVHqHkWPnVQ^TRp-GEK^l2IU7@A~Fae6Op*L>h0TeI@>eSgT@Q|ISO z7oDH{1z9gFd-qa%C1{dA#3Q92k_n~sLo$eF1?h)n08NqsG)>MA>j2O+DL)#NA83il zAZW@$S(laJrb)aY*feWae!dpXC#kZvEV9z3S#*B1-O^52qT}fjy$a189;AplMQmSbqnArpfrxVEjN!#4a163C0g8!(Eefjk{*e#?SXbb5E6@ zD_vB6@?Fyp?XstuCC-mVd&HN{56LJ?N#{Q3H;EYp`)1A3&)38Glq2n)sy~;ysQ%<{n`po6<$u`{^GBmX;!E>~ zWRxY<9}+?~E2uv#18g#J2iP>JKWr3W(`5c=Fn^#WqSFL6!Tcd*xN%l7e{{{}&-Y<- zU;Q5~{U7CfB~Lf)6930Wk;Iq&56LJ?3P2WHcK=h8+{UAYCt5T zEa?G}5VBdp17aCqlVs!*PTQmbu?_%DlLlmi1_Uh;83a6bsOqvRxoxsL1h!3k?<}Bi zE3((JzA8JWU6g>b9n*5PE;-KHB^Hp4K8Y_aAd*p*)PP6`sei1X0kI5FNismC$pT^> z0F@>SXnW%Wm0$s}%LX1hlyylNVp+ukDlA~%v3!!qUd#F_VF?nDU-~gAMF~g+4_K0q zACVFe$tX)&KqQ1%Ro(JM$<)O91pXFS}niXn6E!@Z}oMW%vN8y&Bn(hO_Ry}ceinG5YO}PvY&Uu zBuVeT$uG(Fg9}go0<#Mi*bW6?48`asljtNLe@??N5QOhMMdW+pBu$~Rl2iiJ6P1t% z0&%nKZEF$#$Tp;Xdz>^)Td60G>zVy_cI9%pmu>V5)(NdDlqD&OKrzjQ+E(bjS;Y$! zIZqX*tyWN>130v-?@DF}(-vN>HekH~7X^@VgqaHMyf*}6$95p8OM*iMaiguIK9qKg ze+^~dbPG67(|atzQ%*g_Juo&C5sgUBCZzeX?Ev5mLkkj=cQ{KjeD-><(%ozrlnbX6 z{$Svy^(0R2y%UqM>$;?y4c3r9!=LMyw_(*-sQy$8P+f9Hm<8&!txJ4agu}Je`E@w) zMgV_21pUU_xX7^&coYJVW89PSF?)J$C03};(|jJM^El6%tRTe$DHdP-$gBP{P>w_X zO*{dPxvxJ#{RgutCdvW@{n3I+Zj<;ZTLB1@R4F_Gl9Q1sK@EQ#004MwFLQKxY-Mwk zASx7-^C=nv@g9>99~F}%Di9oC48`as0RR9{0ssIJ00000000000001hfzl+C@gx(I hBq|V-Hzp7Z0000000000fPp8Ijw&Mt*(U%1003CVHWdH> delta 1761 zcmV<71|IpfRku~JatH-^(FhE2vv&w!2!C%C@AX|MRSA@6l6cYpBAHMcKqP}uRxp5A z2B2hSjh0ZF3?McNKxqJ{xBoj*Whh&h;3g!>X z0GA}=ODs*|59{ETAFW$~Q~0HNismD$@yU&0GTG`M}zVMEfE=nkO|5UDZ_&-`)fXMkgZwy`F}oS?y2*0 zrHjr_{(`I*mc4tay%IFZAL5bH56Og5`XL!avx4-)GJqz@0GcM}hjjpGnv@?6$`7^>9ArNV})%&!sM^Kl$4x+J7&5dA7v-(WsF4 z()=MAWl8migpkb&>JQ5Rn@rpRHcjde8wJ=jnLirLA83i_G=WVpe@Gc_oK?&pU9OUvLO zTj9_D*n6%4rGLX5N%bWEreEA8ob5K$pBIYda-KqDkUQ|A%ChB?TZ7LL@5) zKr91DGPMSfG#Nl_6hP8s0NG#wK}*D@DL-1~9Ll=143Ml~0NHzP0Bw*ApM>PTia)lq zi)7$h63KSq!`&wF{3bq)$Kyq#pJbP7UAb*tf8yPxI)D7I{&2X5cs5=#9gOn7WZxUF zKgup0O>Y{DKi40U=N+E@uC2VY{_t`+jvMKhS$y1#KhLszC|_)|G59=Oe>`4eHXkMP zQTk;)dj9!*%W>~yk_>)aO=>wE{ke>P{GtDYclbl!`|0O~{;6@?$d-N|%~rP>CI5m= z>*Q$GyF!W2d-I#obkP{c!)&2e%WpR3>kr&peVrz=)mLt_@i9r$WOD!AZQL8g^ZdK) z=iM+#()(}nOS1jo!jr!MP)h>@lPxJ4vnCe04h1g6xeXwMFDW{&|6EHRt5sgUBCZzeXZ4clKLkkj=cbFsu zU%Vcybh8D$a^;l59}Ub}&*J1hI58Q!u1mT_Zw>hg{#w7j_oK!_omRyF)g@;Ivp~JJ zb&0PpVR0>WejQf46TqL2L7#aW2N~*sMMuY!Hu*R43>b3XeuDZBvnM9X0tI=|2n=zP_9$Bc3zJeQJOP!Hk10U`f|JoH zIsx>P3@RoYF2uPFB>?~cPyzq|5dZ)H0000000000fPvMMO)5SCDU*yUBL>?i00000 D#eyyr diff --git a/summary_results.json b/summary_results.json index be74e1a87..e0e6c7129 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70122, + "Performance_Result": 1.71572, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.31175, + "Performance_Result": 6.29378, "Performance_Units": "Samples/s" }, { From 1a848df481c069aa51697aed793117dceadc27ae Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 15:30:00 +0000 Subject: [PATCH 598/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10933 -> 10934 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0f81f1dbe..11a3a2baa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 14:47:47,901 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 14:47:47,901 submission_checker1.py:3308 INFO] --- -[2025-05-21 14:47:47,901 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 15:29:54,967 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 15:29:54,967 submission_checker1.py:3308 INFO] --- +[2025-05-21 15:29:54,967 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 14:47:47,901 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 15:29:54,967 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 14:47:47,901 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 15:29:54,967 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 14:47:47,901 submission_checker1.py:3324 INFO] --- -[2025-05-21 14:47:47,901 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 14:47:47,901 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 14:47:47,901 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 14:47:47,901 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 14:47:47,901 submission_checker1.py:3345 INFO] --- -[2025-05-21 14:47:47,902 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 14:47:48,904 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 15:29:54,967 submission_checker1.py:3324 INFO] --- +[2025-05-21 15:29:54,967 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 15:29:54,967 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 15:29:54,967 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 15:29:54,967 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 15:29:54,967 submission_checker1.py:3345 INFO] --- +[2025-05-21 15:29:54,968 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 15:29:55,988 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71572 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71648 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index eb0b863df..f1f06a035 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1225 +1226 diff --git a/summary.csv b/summary.csv index 0238c2ef0..657f073c2 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71572,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71648,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 3d15e79d3..e1c2fff68 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71572, + "Result":1.71648, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 4c5e7f3a2eec4d0bce023e7800a033421bb0ed2c..13a39888203ee9674180b7b3f6f4bb8ccc49f4b1 100644 GIT binary patch delta 1530 zcmVtRN&3+s{Xk1Z20>RA%DSu!PqD-df>UhG($8aDeiK=? zmPJ-N$ri02ty_Up_@%5}GTjm~Q6BN6^g}YClzvDC$*ds#undq%GC-!u`C%OZnI`2& zgYp9{5gCM#e+kMDDZ_&-`)fXMkgZwy`95Ussq=HCi_TB}f~*&oy?d#>5;Vyl;*ru1 z$%In+AsIxog7m{OfF{WRnkMImbpU9ZlphVs541#N5Hw|>tjo%9(pB} zhh&r`r5_SPHY-R!ECXyZWd~lHHAz2g6oAv@{Ah4~pe3Tyl&UP0by*qio5T!)eY0li z=j-8of69?|Pt~7GT~vSaw@tKP_VR3r`J+)G@um4gGRl(b4+$Zg71SS=0XCVq18kbq zA2tfGX)=E_m_N`G(P;vkVE&LY+&HV4Ke}e~=lig^ul|pg{*Us#lBb(?iT`7xNa9QX zhh&r`1t1avI9W!*$ubg7HX(r1HVHs%6bYxPe}j$<0uZ!BY?|P)Ls^%W$&HhQVB@s+ z$^ZGgMD|)1S!Lt2iyBbAafW^wn-_3<4fIRCQUE+&0-A0^6p&cNWmM71?W9UzHuxE=oYzj%m4CmmFv9 ze-aDGMxVr&77)oOOKLzQgj810fLI2oBpIO6WC5`bfJ&1Ew7v0xO0a;~Wdn~L%DSWs zv8-YN6&A4XSUyQ)uVsCeumlOnFa4O5q6DOZ2Q10Qk4OoKWRxW>AQD0>D_B4*16VS@ z1`e_&4Ty~bT$(f>8#EwjiP$vdM_rsle_5B7!9ljdpZ~Gdi4DB`Ih6}$s`&4xSG^*I{I@N z|M)}y2k-EQzW3A54gFK&xREXWP(GTiZZ%5&1)J8%(X4k9pZDfBqv@hCj)&Pot(MZ=1d}Z( z8nY%Ax()@uM&2?dljS5Ie`>=p5QOi23ZuI%+rgz3wgYX+sewXWT5=Q8CK0uMkkmSU z`^t9WB#={&tC{_Fc2T)J$Ts=~>x5Pn%90dCpqS=D?JD%qtl|e0IZqX*tyWN>9voWM zcO^4~X$x;w8?au0ivmbF!c2ws-W!6kV|$R)CBdPBxYbrtA4l3kW~BMC?FiruLkkj=cQ{KjeD-><(%ozrl?$g7zBh2wdJ<>%!HL<} zbzRaeMr+8Q;ji`U`?zW>)Nm>Ws4h7p%mVe=)+N3y!r@x#{5qU?D}X=sLBH{KTx9A4 z9)$p8ihEK%7tb%vBns7enip|e#Cg_a1<4;tQEY~hSN&(8JP!Fc@dTK1Uw?x753?sG z$^r!tJ0A^jllCZE0Sc2+DLes`laDDu0fCdzDLMi3lME^*8^1>0G9>{308jz|01*HH g00000000000DytilT9i<0VtD9~qe1$CmWT|3t}K*wSs9*Ui5Uc^*qWuE$GH3^vTQAj ztaOqsT0dI10;ljxS-WJqC1j#J;z{X;WI`$ZkPMPpLHc1CAd_T(Oq27&Ish_F%8v%+ z2U;RB2qAwHlpj)t2U+&leBdBkv-0zO$lO!s=SmlypZo<`FD!faQhOz6l0U>Fr5}jD(YAB%EwQ0HM!PE&se9UBB7Xo=V~!DENAE-jNACkesEY44N& z^LL5twJfsA#%UKdpnT&D{W3O7G$0#&5?^XSB%>_p0g({0S-}Hh8DNuSU(*RsAUJEmQffU+IaaI07;VpWP+5o{fT#%>hQz*!{Hv{*?7rxFv|aueQ&(}D7$nty=g4|Tz^QOcX;}{w(`#U!^`D3 zZlqsk@o_W$Jj?E(e6h{O;PY_(@pz5de3Z;b>6i8B`RDU3$GwwDGWc;dspWL^=Q95B zhyD-V;SYW9r=J`8r^az3Tl!3WG+W(jl>7@et&^i!?aM^Y5~scf%w}@4v|}$@YT_PyPb43KrN71z-%t z=q8irBp-iH!!Qtp?>t51d*UQbp|X-x0@M?gkO%^Cv+Qka5&y_GqzB7-)mW(hR18pEaz>a1>b0#)d|8CUwbc1_IPpdRe>?>J#@o2au@86@0+3_elkzcp zdTu3FsLs=T9;fp-&zh_t#RDl8U;W6d{xeXHL;g)X0gkz^KSBKmvnVFY0tNlif=O Date: Wed, 21 May 2025 17:15:03 +0000 Subject: [PATCH 599/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10934 -> 10933 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 11a3a2baa..5cf1c40a2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 15:29:54,967 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 15:29:54,967 submission_checker1.py:3308 INFO] --- -[2025-05-21 15:29:54,967 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 17:14:58,308 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 17:14:58,308 submission_checker1.py:3308 INFO] --- +[2025-05-21 17:14:58,308 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 15:29:54,967 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 17:14:58,308 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 15:29:54,967 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 17:14:58,308 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 15:29:54,967 submission_checker1.py:3324 INFO] --- -[2025-05-21 15:29:54,967 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 15:29:54,967 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 15:29:54,967 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 15:29:54,967 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 15:29:54,967 submission_checker1.py:3345 INFO] --- -[2025-05-21 15:29:54,968 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 15:29:55,988 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 17:14:58,308 submission_checker1.py:3324 INFO] --- +[2025-05-21 17:14:58,308 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 17:14:58,308 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 17:14:58,308 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 17:14:58,308 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 17:14:58,309 submission_checker1.py:3345 INFO] --- +[2025-05-21 17:14:58,309 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 17:14:59,278 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71648 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71755 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index f1f06a035..4f7203748 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1226 +1227 diff --git a/summary.csv b/summary.csv index 657f073c2..02b07a912 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71648,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71755,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index e1c2fff68..18caae6ee 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71648, + "Result":1.71755, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 13a39888203ee9674180b7b3f6f4bb8ccc49f4b1..28793c47f18963f1e6bda674044268472be24867 100644 GIT binary patch delta 1771 zcmVM-hc8>p0xMxzw12eS zvWl;L>|&`)utbf-llBkEgwp;Y8N{-J{lhYVC9`S(OOyS>Mgc5M^3V3h2P{GUp?}i^ zEJ6MuWpILJUBd~suz-Ilmiub{=w0*2$?hAk>kaEbniC$%4v38nT!GJnWr1?`7rfJ>4AE=|@C>j1bkNk1B-A83ilAn3|M zS(laJDVCT)aEh&2`gx4YZz9XqvdBs&*`oEMbt`ZRzm&C0rdvWL$|IhXen=*i(htcX znH8iTmH{$J2FNryKdb{F)1>@pP=25#B7+bzLHQwNc#vg(%?A#$H7h^ghkwjHb$+gN z(fP?=koCf{cQ3V9f+qPxJW~20nNUhUB!g&HkbYPO&?Ff^)8zcH4ggJ)@}oidftH92 zf~G8#by*p1n#2o&O|xd@=WEe?k}6xvA}ej0MdwG`E$xIQI-V|3el)5hzEpllMp@GN zAt6Mwg7d>NfF{WRnkMCkb$QDZ*iT2B0o_{Sde>5s2zBGSG zMp;t*At7Y5g8IWUz$O!SfK8M7!$tu%P3DgV^9NcYI!#~`%pX#Q8)p^sN7rord>=OV z)&J4b|53hI@^sTK@qcU-Nqp)5kc_gV07OCnC(B4USw_OiCIoQWCIN_zBH=W3(6K=P zf|iI)6Fhb(>(Vm0aetB!Y@GH!`9FV`$X?4Lt8AQhQ3J|1&d@JovqS^3(I@ex21GK- zk{%EVA)6IEAeI3(Nk%^5v`rci>j2O+X+Sn;K+qDALBL~&sxGUN+a|k1VB56!&I0$5fF=3( z5h($YjIyK!L_&yV1q+B}088fAz(Lle0kKhlOOpm(Vkf$X59C zKlYw$KLJ0uajp zl1!}uBuxeo8wHRw89+7|K+qDgY08h5Ift??EdwMg7(n)(8$cT*!zUrRui}sG>>?Ss zmPE2$_;9yLJim!gyNTa zN7I|e;?MPmght z^={(x-a7ndG+i{t@i1Gc)$*H-`T7I*R$r&dZ1t7fYRHTqc2(y#bu^HyZ}!!l`7~ zTexXGiKBbxF6$mEe>+V8OVi!k0xnT+4h|#1pS2 zh^KwjZ@e8BIrM=@VgP!Gds03p&o9d*Ickc$DAT-53$i4Sl-$#D`qhuT>OTY3amc@k zC%_^1^(Scl0JA72$^r$q?H*DLes^laMJv4SyT}0C;RKb98xZWpk5Z zDHN0PDH;Ot9+M9r6_aHt5F3B7O6ev6002+|000pH0000000000004l2(vwaqJ^?3_ NjVdDs*(U%10091jI@$mL delta 1759 zcmV<51|a#hRkl^IatH+wJ0A^jvv&w#2!Fi7?-cApsY;+klf;t-5Xpqn03sQLvVsA` zG5{qrYqW&YWB{>I07{bpq(K0JmWWPMsT@ygDyb_thgka*JkA(>E`KO}=(Rxp29 z2Dl^{Ut(zze^>{ArOErz;Qc^LL{ETAFW$~Q~0HNismD$@yU&0GTG`M}zVMEfE=nkO|5UDZ_&-`)fXMkgZwy`F}oS?y2*0 zrHjr_{(`I*mc4tay%IFZAL5bH56Og5`XL!avx4-)GJqz@0GcM}hjjpGnv@?6$`79&4Y!ragF4X=e9Dn_Pt~7GT~vSaw@tKP_J8thiTR^ZA@Qa8 zLo&*e>JJGan-$a_mH{@IxC3mO)E_nquxT=XG?+io647Y_n_&KsGTb<;m_NE^^XL1p zxv&0@mi~|Oy^^P!c8ULEqe$XQ|A%ChB?TZ70ytSl!pSlcPBtNc(>4h}Y!nHnse_IU z0uZ!BY?|P)Ls^%W$$yQLgka;e_sReHyF~U{7FlKEw2K;0zHx?r8Ji^N zQI_<8NC?@i-~q7=ut_rV38!t+fLI5Brbz>`K?8!8hztTAJ5+U9mE1Pj9Rk~?y>}MS zw-woISznbM(=JLt*^X(sT9+JW?Gg*fMxVr&77)oOOKLzQgnv|4(12J5s3aMn(qsX# z4uDFN1+=~Kfl9D|*kuEc9m=|-46&?Y0TmXo?^r%bWUpm?m9PW}$S?hvl%fQrf(IO2q{#rX!2p7mh)q*|w9Gk_b!izOS-}9Z_uK&5AQ?Uh$$b@nY-bnA zz_lcj?ZStW6(i$*`mF4wwp+q(Y5yGwQWVSoMMa1Zfpykt5U<$uY(H(q~~ zT{@cHG!}oZKP1mPJpEl;d1w9M<#HT1(l4|4xEX(*W%p3N*k)t!dAR;~yvA%kO6H^V z%X;+u^ZAzJ-pM2x{J5Ibayt5R8UOf0{|E2zhrajI&kg-k2sIF^-4XLamnHY|PgmxVQQ`O=hdF+-BorlBUVz{=3__H;Cu?ciGRo zVUncx-{hBM`@w}Le*sWS0|b*TDH^jT7P<}vzee6NC6nbOAAf4YFc5_AeF~$yE!)AR z6}AIy$*F-tU0QMz(k2nLevs5Ue*4OH;v|q$kE@yec6L#@Jjgcs1?z-X70QwnMWC4G zLhUN_(X8SJ6gf{7r>#~{p&lGs)^{Z{glP+JRvWNhfQtf1Il@eZ_TC$Uv15CX)Fr{8 zg1FUIQXfjY!+(adZ@L4Vr|CVG;3=n`;sF?&nTU=^&Ss?fvF!-p3_}YNly^8wGJN)W zu+rUZ7?lgB6uvib(|QtT_rZzT*mYgfEk-)HBEYxr+2BijyKcq@QE^+CV!c3foY10IC{WQu!IJ{Qj~%_Iudd72k-TEuzQWCh6|Nl|Qu zkyrg^pga!wH}M3Ra$kRf`VX@wCdvW@5IY|YaFg~ZTLB7_QYkzEl#`DsK>>l2(J49s z^OFoJCL6y--ZCWt002+|000pH0000000000004l2)RRprJ^?6`j4C4r+9v=2008AS BIJ^J= diff --git a/summary_results.json b/summary_results.json index 993fd7bc5..e6c067113 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71648, + "Performance_Result": 1.71755, "Performance_Units": "Samples/s" }, { From 33a099b127cca61405d61f52e241140d23d60c92 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 17:29:16 +0000 Subject: [PATCH 600/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10933 -> 10932 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 5cf1c40a2..e31d33139 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 17:14:58,308 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 17:14:58,308 submission_checker1.py:3308 INFO] --- -[2025-05-21 17:14:58,308 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 17:29:11,223 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 17:29:11,223 submission_checker1.py:3308 INFO] --- +[2025-05-21 17:29:11,223 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 17:14:58,308 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 17:29:11,223 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 17:14:58,308 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 17:29:11,223 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 17:14:58,308 submission_checker1.py:3324 INFO] --- -[2025-05-21 17:14:58,308 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 17:14:58,308 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 17:14:58,308 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 17:14:58,308 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 17:14:58,309 submission_checker1.py:3345 INFO] --- -[2025-05-21 17:14:58,309 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 17:14:59,278 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 17:29:11,223 submission_checker1.py:3324 INFO] --- +[2025-05-21 17:29:11,223 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 17:29:11,223 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 17:29:11,223 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 17:29:11,223 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 17:29:11,223 submission_checker1.py:3345 INFO] --- +[2025-05-21 17:29:11,223 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 17:29:12,224 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8508 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0979 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0754 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.82399 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4f7203748..fe1b06087 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1227 +1228 diff --git a/summary.csv b/summary.csv index 02b07a912..1b4d58df4 100644 --- a/summary.csv +++ b/summary.csv @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8508,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0979,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0754,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.82399,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 18caae6ee..b6da68eb8 100644 --- a/summary.json +++ b/summary.json @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0979, + "Result":23.0754, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 28793c47f18963f1e6bda674044268472be24867..29527dd648a8dfef0aa133bfc601bfca3167a2d3 100644 GIT binary patch delta 946 zcmV;j15Nz3RkT&GatH*#uEUnIb_j9{e@@>m**NVI4ai2H#FrWn$tX*DKqQ20R`7sW z2G}GS`GnIpX+W$4K+~iF*`NVIOGE|%j~%MItV(X1><)o#)80D^=-Z0ywXCnoj%gPq zplrvqT&>HfEU|!W^htba0g;Tdqy|JnNM!{Lh-H9Ek^w4B77*(Is5DtX+Z!LKe*_DN zT{iI8p{z^F5X&kSP+S|lyzwt9Aqo}`5$}FHK25uBdMO` z|J1U!O6S+2|D&x|80eB0Wo@Sfe@Qe+eChv?jIyKvL_&yU1p$a<07<6S0FovHh>ZeB znhYQt3?OKU*fixw%bY`5mzDvN6$~JI&kdjrlHrq(+*k3(c6N~rTuUO^E_}G#B%a^I zr}22aX!Mipa;+=3t?N&`yHtlC)*lY{5YNU-rh`%bm+X7v^+(yIqv=g!fAQz~L-M@C z)8Dm~ch(oAKvab`Rx?Z8ipT z`32{t(G?V0v$Xi8NnxS0H^%bhCw-VDjD_`u3C@caLOI9U|dLD;`1UNu4O)~;)ypB#N$5dSKbbbjC~-I7=Vs(kIKjF>3O|Gb&(hI zG@qvhS(7_jJWxWu`jHp?C!jhE`8V+h7;|5Kg8C1$Cnm}Q1i-Gtmb3LJJplz(#MS3+C>Q{ z+c7Oy>yqQFU19;*=#%);0wNh@NezgEkje@g5X%6SBm-2MEFjhaP-(J&wl_Xde+d>4 zyKLaGLs^%UA(mAvpuz(79m^+)?6s_~5|$tV`K2F|Qj~yH@PH-x_z@`qk&LpW1w=xK zWd#d}WdKX&*T6y6qye!}fJ>7GWP=6-EfJfh{HTj_DC^QPILKD`^FQ{UYe4BRM^Zh> z|EXndmCmn4|3_P`FwiA0%Gyo|f0AgD_|pF&8D&WUh=dTy3IY(z0Fq3t0VGWZ5E})M zG#NlP7(mbxv1!VWmN|#AE-eEjD;Plbo*O_LB*P~mxv%1n?d&2MxRyk+UHEXfNj$%a zPvh};(dZ}Hcc~6PtUny?A)bwwOb4UyNTaN7I|ef8x*ehva#O zr@w0}@2o$(T#n;L`eha$H{;K<>>kP&+iVOz57!@$*O<*m$$XT4S&yE7KHqZOJDDVd zA6JuFPDg((;~#(M|KJ_|(D#1&xuJh*95=G1-$%36twzbeVADD|n)Pnt^WOYsG+i{t z@i1Gc)$*H-`T7I*R$r&dBy9DS+iZMH(lnXee|H=A2Jt-qF8g^mOp^5eoBWb&Ke+JZ zFS7_1*baYxu}bMC0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!LN!Y~ko?|q8M z_r!5%T4W_bYAa4fs#L0?-Yk0otMQL)w}H2>6Ci-ZsmJxqemlEzHQT8+`32{t(KRAj zjuOz^2&p$U`dH4>DN1}`TCmn=sL>ugnl*P7x0HVy2XD?;a6y7kB1n76ZH=}eSc{c-I85FA4b3Uu(8 zWCWkR0i5zT8wTaVsbtt&xM@9!qkHG&XzaQ!>n4LW<|lZyczqvMO{MNn#Q`;y;FLRH z!MI7L!k0xnT+4h|#1pS2h^KwjZ@e8BIrM=@VgP!Gds03p&o9e4YKpul)4WUzvLug` z+|zRU)sMXDKLgcq$iImvz#;ebCusfvvnD3W0tC119@De*C_MoMf3ZsGCX+TQQ~@WG Niz*`q*(U%100423yu| Date: Wed, 21 May 2025 18:05:09 +0000 Subject: [PATCH 601/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10932 -> 10931 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e31d33139..1ea9d2231 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 17:29:11,223 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 17:29:11,223 submission_checker1.py:3308 INFO] --- -[2025-05-21 17:29:11,223 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 18:05:04,004 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 18:05:04,004 submission_checker1.py:3308 INFO] --- +[2025-05-21 18:05:04,004 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 17:29:11,223 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 18:05:04,004 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 17:29:11,223 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 18:05:04,004 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 17:29:11,223 submission_checker1.py:3324 INFO] --- -[2025-05-21 17:29:11,223 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 17:29:11,223 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 17:29:11,223 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 17:29:11,223 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 17:29:11,223 submission_checker1.py:3345 INFO] --- -[2025-05-21 17:29:11,223 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 17:29:12,224 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 18:05:04,004 submission_checker1.py:3324 INFO] --- +[2025-05-21 18:05:04,004 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 18:05:04,004 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 18:05:04,004 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 18:05:04,004 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 18:05:04,004 submission_checker1.py:3345 INFO] --- +[2025-05-21 18:05:04,004 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 18:05:04,977 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.410575 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41817 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.08844 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fe1b06087..f954c7ae0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1228 +1229 diff --git a/summary.csv b/summary.csv index 1b4d58df4..f6204fff9 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.410575,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41817,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.08844,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b6da68eb8..ba405f7e1 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.410575, + "Result":0.41817, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 29527dd648a8dfef0aa133bfc601bfca3167a2d3..a664d63900fefbb790aaec7e4cd49e75b2d19fd0 100644 GIT binary patch delta 3057 zcmZ8jdpy%^AKok@k(oouS=*c@B}PtjN@f-!tb-gjSx64aA^Q!LQ_40b(?L$9JYo)s zoT4-oL(8Fr5>k0Wv7FxZJb%3J=e_^9|G59SKG*m9Uf1=xYmQbQt&9-g%Fp*Ag-U@y ztJU%1!4N>#6=}}yty+{Z|KS~fjc(mDDYu)ZlIHl;(cb@%=49J)mx=44F43AW$te}I zSA>?C9zWyG3mLxoq|@zHIPS>q(lwSdJ|ki7*&`Wl{^*jiIDd`_KCsF_ zEpB}KdFyPf{#-$e$4D8?Uj<)An~ZOv70T9FgaS{LpVNwVs;5tamy~xWT15D5aYe|a zX{I^K;p_`{?sP>=!4KNSign98Z>4GSA?bs`$W);R1)1Dk+IW_6%*7fA+?tgCqRGAyxS}*;qjUT z*mXE$b-ED;kp~yjI*<35F3jS+4bCxrutepn@eft4#Ks<5vH4X`C15^RM8%Um3%*r; zLW+~a#l5~I*@l0x(#o^8h;YF{O-YJdHMV_>GRlD19*7W}Y1iOL;6(D9D1cl`Ky>x}3yYvqkMpYu%O zMDwWgS5;>{aS80c3vHha8nEswnvHW#d-utj!kriVEonGI5Q+=Baih&LLaES~p$c=h zlU6}~@HS0mrCyuF?jW=HROT0*}P{LB2Ty!rscKf1o=aG2t{iq2}J0D8P#qj^g{|MGk2?-et-$XaGo28 zK;$FIe?QQj8>vIEe`9HZD#7IowsV30qtn?H-{KxzB5YXA?!O#{ZD=OR?oZSj?vUi* z3|obK^FO^hNgBXZ;o__@fRtraorzVY&T&mMC-vcM@=4Dtqo^zAMr$_RLLO+{3Yzl{ zC|m+xk$mO|a3wc*E~8kVC8F(Lf+Ay|7RJoE;Yxm}JdKQzS7- zGy(&R$-_cI8c>)#A4tfqxS5c!i@xr4_EFic9G&ila>>U8$%(qzjrO67UG%zZ#-1aX z_mg$Abx-^B?W*lZ+=7y0`bRYKtk0Mul7l>97rdc;q%YeGC=nODrBh3BAKb?0t?~AU zz(zI`fCjlflu;2rT0Sr-@Co|B4NP3k2Rd~6K=yfxVn}|+N@8er;H_!B;XxO({_kbe zk!9(HF25o>tKM@=Fv)NuH`yP>U+fI;MkP1`P+qz3YNccW_o`A%zyN8jPZdO9E)^9s zBBA;+ilRNL_rGv-hm5&v;cv@_zsFv3o)BEL%#J^^cB1U$RA+cXr@=nIqu#d22Z-C* zR7X@cUz)3)BmY#51=h{;f_Sh8)kYw$6b2wSgzbY@xs5L?7L?<=VEBGx|0iMFT>%}| z2D9>=yV#7=)bY#-|%plH!0;W4_>#H-sXrhBZ*AR|pxGm%s`B0dm!eLv~ z{l63x*@z^uQVH%G61+d7=UKeqWa*<~?*`Y=SZdYoUn1H|d<~oO%eQj`TRku9umAMm za~gEDyiXd@Ji5J_j~6Ni0Y^oUY{YFNnvgma##PUjmsDqgweyHY$ZQlOHB-cuy+vJ0 z1Xg1o#XgdAnX^aFPFqjhx*-r8l)`Ok&;5&>m{W-!!H%3IK7VT!T{VdGE_9g0t)`G{ zDVtf6{=~KoKgCuC(U?}k-S8^{u}}hGBS2W{n|%}!;SFRNnznH&Cbk^+Zf%k4PA~?R zDg+-%%b8x$S8DO~_Q@U0&(hpW{c9SEpEyZ^cEeUAcLk1TuC$wh(1fw;(0Q?Anug+W zUtNW19IvXbXbX629FI+%GTAX4O!SwhpWIQa_Nc^eRNARN%BkKS=wOZr@s6zigZHZ) z_yY*aMa`w02c2LSt4Z}s*VvO_QRKVj-Ccjn;d|6gpYze{7B8ku=;UA`se~EA`^{^R zH@$qSYXmu-Cm-`urbne#G)ZV z3`t>fsNQ0+EH*8LQHnQ7*&rMPvWS@Kbfhmr*leVJRR%B`QJb=otl(*;Gr7@%1? zfAz~|(Cnu62ff$^O7034la6R!0{(CVX=>1zWW-JM?g^5th-oMjB#g^Bw%N>(I!M?P zrCU|a|77KzDNY=7I3BpOcBchTdPJrl#T;(D51vxGH_r;W8Jz|t6}e2}rBX@RNK19N zEk-ZKTG%{{s~4Y9@{QE_-~viNEu@cDfYeN;b#6a;(i8cZ5S`LmN14ARx)gfw*D<~q zxw9q%?phactb$MWV0l|(ej}P|Xn#0DG&PDjNze{MYe49}9Mrbd^B@m4y5L);v`3O? z#~!7afBc+Emfv>2l9tf9fYXfEpL=)->1a)zy;m%URF}8##aSiTJ6Q|06BY$b7&H-o zVbBD|=dz3`)**7^{l(zn0`Q-Sb+EIO%3TdLlG{1*8*A5_rgyoT_0T0Z%@%zJ-0P{viXa$lUvcqaL6y6&)IS^yvWAqyr{e+)0%Ks@gcJe{H z^jzBG(yEonb;a1RiP`y)F7r8QgWQkxbZXx7&Q~jo z=*Ese?YOZqPt<(H&YO~_9QM=$&*lGbu%4uMz>ApLTgg^A;M*YjF@_spfX)+%n^1y zVKD4zW`^>rHDVvg@`a_2TJ^o@pKxPQov!R`3jNXI0VEQ+d@6Pg>+q48^RKlo%aGo8}Zbr5t{9D5H#R4n+u?ZOG(B-{9;Df5y% z2((faxyerwXbXKcQqNh9L9*S4 zhGKpl5E)(SaZX@9Zj=$O%W-RQ*5&;AY{VId7#s&YP}D(ElpVRVc@GVd zv=&xpsy3l5f^{sbSO9UjM zDzMr&ZJ+t>P)YtRGUG#rn30agjc|d367dMj!Pv1Obk0}ZOJBGfd8=f%`GES<12%X& z`z^m0&CQH5KCa$*o~ z?b^8Z(uX&y)pnsO=i_t)FUp}JN9DNpTn;T7QpGv>jqLmAejd$R!>^o6xr8nOOpFda zmH47}RiW%x8 z64W-Xp-z033$eEr)!Drc#!R|h$Z&Nx-8yinG&G?1B+7rz)G=QtYQ1v-m2+Fxl8^RI zBe1|GIO#s*ztf1E+9ZNdk_-5Z->a&hjV1Iecb<^om*`J&jR#HkCJdCUrh1(`IAQ0} zqNW#+6MlIB>d`QoSvOep%i_0~rsuS@GGVUvrBNpjT8J~IZBV@pr<$WNI7NENy~Gcq zW*H1cEISxOA!en2t;M})4(M7^EFk{D`!HRrdf=1MfC+#LESbYa6&5A12*e%=%iHC?mD5Y`wHu`=!mbGZUp z1wc`!DS#cCGT@!7*1l*Ha=w0djh<+{gWHy1UUNa@rs!X|eF#^RNVb^}r9rvG*VCS= zB=!US))M}V=RwUJG~XU4H3oG@m?WP%(0;ERFCYydn-%?Y_D@=#=a<9?nc>F|=Hp#d zqPh7Nj31fxm8$6lN!fHQ7Z^+S{-iBnO!t-=NTh%W-h++mIq2a;5r~>y3`1qy-A7Go z)h3CoU9wzWKf!%Ved+x^ynPQ@u6=Cs-r&jMF~=XeR*XrD7{1Y!2b=`QQ#ebV)wHA9 zk6|pVu3m(dzT_$63h4)!GOKEt0wHo;n*$V}yowr)FxFy2;F-mKyx=dpfC6XzRV~lw zr3xJxa#t#=<2Ch}^3`uEyC1M4gc685PVUx7NjL5-9*cs^L>7$%+Sd!wo9QVCz?RpI zwU38D47TKytY{R5Xio}_uim1-J&w-V^!kGhth`IRM{nR?ldqW)HOTW@%jGxDNO-I1`gt>O{Z0cjz?sY-*vA`Of5)?-{+az1dp#eIXzkS!n z&(62-r`0Q_pnh3FNzzGL?>DSkvNdj{NX;iwXn58vDib}M7zT37u7k;cR;;!oEB)*f zd*9Z0;o{l&mM4Cq7$rfVA-9*y(?z^A^tC#NPppNd2Ti00C9To5YBumhfAD?)e*nRT zgdW97&@&9qO05_s_FZ$l&4!qj@l_}DHZ~~vF>(3oEkPhc(8FzY)aox>C<@aHl6#e2 zgD>&0LAUM6OOJ6trwbMNns>Gp!!p6fIO$f5nN_)@W>@hF)hVU`g@{)·r5or|Q zy^I2Pzn;~X#CHe7Vt^;1olE@lX$x<-SBz?0x^RQdI%7D0ojMg><0{fB%sXm z3{S*h9`0#472|M(C|rWW#aW!+$fCF_(4%7gsM)swn0&KXz$q>~o3xTM$ ze&yPt4#UWY#Gw^xwSCmYO4==M@lRWuR&S{Zv3P`%sYbsl^h9Ec zL{Ro!nEVGZT4=U&#&xctUQ;ytHkK3EqrVz;@a}Ol;dI$@NZHNr$Jc6;M1pg9273Vv zpo?t40Aha5t8h5EDsH*SxDmbe3XQ;C77vRF;ajTo{6<$0NL$~*oNuUK+%|9D>%SHS zGg!@`Vah2mTx_h&&yI%;%not zStAqtPtVZY#MQ03l+b13V_hz3-Bk?kF92ilcGBQfjfXGF9WUifDZ~bJeYV&k_srO@ba^h}Hw@ zum_OM^s^>2f96?9pu6446a1So%C-}^_qfX9(`YNgRGf~ZwQfWsD{6(FPTTNS0s$8X zO^PQD+PvRCntnoBpLXMYoFo@3xpCu@G2L&)Z?BK)F0$Ur5-hTPe?jQ&J!N?#>)imi z)^V)2aJ}g9>7C=O9h87l2oCc zVpb-YwGFBn*)Cro&-%(v8+4s*2@hvN&)^aLhEdN3ka=HtrLz%r)Sv9B;KzDTA5R{y zdwD7m)&o;mRB8vB@g?ym*0QD|hZpSo?-xxFF3gTC+s@TBb$zMHxY7eVXV`Li&#Ljw z)STHdVz}by%>WbW{KU!B7Qc7jipYl?UzOXpTjd9uP3InEy-t0EGbk;x%9>5J`U%V^ z0OOHXtAia@`x)x}!hehUzn?A|InVLQ-Sz&O>Fm|On0?uXCThbGw+*^|m)nG9@xTA`i$~ty4$iuB{VdaJ8^A5pz4LWVZJy04^hPU=$bx za*_anwtzq&a^MB0;J}~|qYHt-6vK!B|6G>9x9KmX!;7nOMfC;(c=V=$*jn;O>w7u| zf6Mv|(cKl1HOJN(xVZ;p$sb3@KbwVIbb8)bX0jwTcE58Qg2+(r7~E8rBUfLMHF3}} zisa7ep+4x?ZRy@^ih7z3sLND*a0|l59Y;7swgp_Tg}WtJj(Iq}yV!ZJ=-J+5l}h+t zlHixAA85zbn48-mXgAU`nUZisp``PStPgO;0t9L4h?CH;M615m3Ivs|e7u1*3(-{- zZGYmPIyCb6s+sD`CxRFKq?9GuwKMf~@{dZK7fXj|-<^ZnG&rO)_GgFY5xbvI4wVlj znJV*!4}Ew2tZ?&Iv?gmOjd3}(tz~NBYQ0?;bH_X;zf_9Iws5y}vT0am?pW0-Fo!Og z&TiTnay}+*xeAFENc{I-&iIq6`24 diff --git a/summary_results.json b/summary_results.json index 344588238..9b9f87fbc 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.410575, + "Performance_Result": 0.41817, "Performance_Units": "Samples/s" }, { From 414079600d44497ed3e3a161f7946c2693cec9fb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 18:26:22 +0000 Subject: [PATCH 602/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10931 -> 10929 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1ea9d2231..93c59e6f2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 18:05:04,004 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 18:05:04,004 submission_checker1.py:3308 INFO] --- -[2025-05-21 18:05:04,004 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 18:26:16,404 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 18:26:16,404 submission_checker1.py:3308 INFO] --- +[2025-05-21 18:26:16,404 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 18:05:04,004 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 18:26:16,404 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 18:05:04,004 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 18:26:16,404 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 18:05:04,004 submission_checker1.py:3324 INFO] --- -[2025-05-21 18:05:04,004 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 18:05:04,004 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 18:05:04,004 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 18:05:04,004 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 18:05:04,004 submission_checker1.py:3345 INFO] --- -[2025-05-21 18:05:04,004 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 18:05:04,977 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 18:26:16,404 submission_checker1.py:3324 INFO] --- +[2025-05-21 18:26:16,404 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 18:26:16,404 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 18:26:16,404 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 18:26:16,404 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 18:26:16,404 submission_checker1.py:3345 INFO] --- +[2025-05-21 18:26:16,404 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 18:26:17,884 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41817 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.08844 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.76569 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0109 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f954c7ae0..f70eccf23 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1229 +1230 diff --git a/summary.csv b/summary.csv index f6204fff9..f4ddb2fb5 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41817,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.08844,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.76569,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0109,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ba405f7e1..0290cf377 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.08844, + "Result":7.76569, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index a664d63900fefbb790aaec7e4cd49e75b2d19fd0..1fa1fc3da71a26c8a8990cfa541db9eb3a8bad50 100644 GIT binary patch delta 2632 zcmZ9Odpy(oAICRiau;z94ns5dOKVa6Ts9Q5mTQg?iH*r6xuhcBS<3xjbE&!HUPRbX zaSGWlMuib+MUsOfb==a{=(qg-`8|H0KR$na{&+oJpXcZGdOtp_Q>~|3V)jUc+3&Xw zXC>^BkpLp(>Soq#CXFHV?!}!Gvss_+8~3tLymz+XWNm^F9MDZTBrpiNH4vFj#a=;M z$l1gdBn8jP`syCdH191S`FleX^`v@32tGZwRKjb+SY-Pv4_7qnk3-&5NGoN}lF4#E z3^Y`VA#A!6Dn0Y!f=vb@MC|vf3^hkO5rn4!VAKJJk#PhHtB_t-nfCJQGf5N)=ja4R zkqE^Fz4sa#!+A9F-#jnWmDA$lIQs9n1Afmi+HE!~33WFA8Y*=i=o=<%0tv zyCrZ)1kEZTAJ1bWmcUb}jghJ8Z#{Jp8H)xfM9-plf?8C`uTLY#cAQ%4%j2q=fZ&|M z4t&?c$Q!0JF}Q%w?!ugHkx%mkNCx8aW5Ofr-hW_dGxgttanNcI@I5y%Cnm7kx zF!idcK@4UiZZMh$_n+%n5=xFV?&^gm@MM2mDUK}BGs?7o&?Bc;a#Ko-g9ypP)4-Nk z=xgGRY|9ekHSvlZ&=R&-$$Gq+f}c_vb+_l#nJhDrZ!Hg`^6I`T1g{%9y*WDJ6UK|n z;??oi>>tR!PzrOZl>2}UPa}%eujI*A5ioDhiu~38i&DpVch*Lmfw=0F(hV`)t0^%e zem$*VwgcPwERuCHYay}nQL}IEo6Re6WCF>y?9GA1MoCM+U1qiBjRUeWF3}k})fl+d zBHM!XQYDTNzqM%P;prc%YpAhM)4dwU)JqrKCTdA7kxFxt9WNO1<;_G-yCvtYJLSW* zMwzx#!uJj=(h=Wn@bMW+4heQwx&&{Q|yi{v* zB<{0F1S%Sa8UiQb>;3+$kgkQj$mD*#~-;7$D~s<^hlu(6!hd_rM=$9Q%06OelqEX6RsoolKv9as*xLBXE5h zY2KSd2Bs(JtxC$KYTPiiMA}|sQ9Y7tiYlLWz)d|9G_$fu{veY`0f}fLu!Iq)oSt0Wa;iE}3BflYC<4DG_1t*^AN~+$`ax?d2RKxx! zUdhhK30%I8jwtHuKcmh8F%a`2eq=^AOzxS8-zTOHBT$%!e0vBx`7nGULYM?YZF%t@ zagWXG12c!VN6Zo%2p@#xHm9T|VO}<~X2ZdLAVgwGL=Rv0sVAI;SW3$gi)NI<F)imOOtzD3N?ng|Ki3Ol>$CoL-Ra2qIuuZDdH_Hf!JnW2kv)%X z9UUTzD(FM@Ae0j*c)CE86-)eIxlmwTQZ&l(fy&Qe9yc^XRo0y*BX*KtmRqlWjB{#xbQc0y}uMi6JOov4?|HI$4w-&6EO{c3?q4)y{|UnsE_|lnT9|G4Wi@ z!Opf5QKfKkjXRHBcZO&5L`fMi(zxS4PFc$tOTby4>~e%KNA*{0p^ivkkfx<5rBk9} ztz9G`kXi+eNO7S{u#bX_@Tw!CVi@&`1r^I# zUo=>zA=%7K2`|8>=SeUhp8vRL^VCx%xOVr3r$?qzp0rjA{I;#Go;J4Na>*M5*D3K!Rup$N&_w$5+=9&*va_2QGyu z0U4N%g^;fizro>Kr-ZF#{&2`7&ACJ7_KIc3g4Jr6wU)X1HoWWf@?hk$0P@A`)oj{k zXrB$6wMttGEcl`KefrWi_Tm!x%9xMv(N4DYmwz@ABInCC(~5_H3vZpfpMRJO5ne8O z=fDpt$?08~6^yN_v-k$La=GTqqwBc~et#tD(~1Ago%0x4S{x<`=M=VKfA+xy)2?<_ z8?HHX$2&H@lRG+J8SsCJ@YQ!>P2-cMOJ=g`vYs8*3Xy5|O=cZ9158iRj-im6|&W)5tzFDULKVDsK&dgdDyHmFz)1@xQ~ z?&n1g@Fr6g(I3X=u7q@#`9C`Sn-^V3UFR-J=bJc5k)u+_yQHr+7BBs#XKP{V6O8h@ z3&~a8bl{lI1J8O(4yvBcdr#lPEsyc5yt=0-=o4RS-#Q^~(I6_v(}c}kX(0(qRzSa8 zw(XM^Q#P?k60f@muL2u>OK-EgzXYZRlhG;iTJ-GqSymL!ohXOq(?<$r%dU4#unu=B z1pM)=Ba(HI>ld6%?e=c|vfWTW-tTz<9dCGeUf##zzLVK%{9c7g3z+vR+fV94eU_l? z!P$IISUMb9apz&^eC(Q#8BovZKXj%2=TW$YRp?*VHatT@*Gm}ge zu;7RxvH$+~=oQ7}c?yuBTU?7Ni%|e%n2ws!yl8G$i#PozQeJYE+bv;)TH3~#q&o!@~;V+vcrO*5;T&`1sEV&C!A2VmevEwXHDFzsQ(8{ zlOZHtAR1kYVSagNHYKk{3%`?iq1(bsXB?+Og9pa5%Knadm_%E4nA~&aG^w_Mp|dA` zhpbggM8!9${N}!Y6~O2vKBmTyi5u)AtF0Z%O^+Dcx}s!R%mCl31K6u)1}ay)*bjHy z;f785hZIHh?mU%yyyMAWW=`xTZpez@`FzJ)az1tF!aPQJi$b76yH#Fw_N<|`C1EGQ z62e@!Y-oX3%)T~Z!Yjs2W#R{k)HgK|wku%5zk0H_M99yJ%z@Br*YmqA&g>UsLlelN zYe)vb;L$R{L;(rHuEi1s++z^-mY50l>B~Xh)lQLFo+8tZ+ET42G_8^9$>o;*%k3G} zH?4f-#J8i>lhx0N~ID zX|kKs&tL*&#E4lFky&YM=`-tr$`y>nd;tOF!KLm0x_hEi}k>mY| znU1&nLqz%r(U1wV?@sUk_^U0+uXaND-tv!_$R&ew5E-!Hy6_@W@f9x9H`m9F*CA>ukvRL{^mzL zM{M)RF(GYvtsQ7aLKomBmFe(&db8pF$C0_|lQbWD|BlnhSW#c@G?78ZsN zT7?ri4KQKsL*>!QeC`Y|E#!pd*uTb|X5@_Ql<~zpG^FYZm8s~dW0Ao6^as|t#~p<49do?Pq_f{5#sXxPKEg- zsJHIakN&aFDNJ4N`UmB_xV9OpajeDEqa=(AnKOC6a1#clYZpXyh;wo!tHM(?1zZK1 zTCfTpE-G-&z;sHzUw!Wn@1=e<(5 z4v=^qG`)Tt+6@(dp}Ks2v=Fkd&IMYA$zX4G7>Gc0ogVFtwmU}nPK|mh%^r|`)fftR z*aDmCx*D5uVhq#UY_j6^t^S|yz+XL*x#GtQ=a-O|hDIi*8y;6>Z3 zg;`u(^2`gOeqvY4*ZX5up|f6i!=gZG!MI`9ldbayu;Ssi*M(o10r3?*eSbkRj`V9+z zB<=%I-w+_su=DTUzSOIy*mX5_aE3Mj==v3GBYL3A!cMWiLS3(a;~ICXs$^gUjpCkc z*#%FUj$2nyXCW(bgI7Z$#3k{L-iKp!da&A6ob*0h&Z61PF?70j+9SbIRFqd$|0AM8 zeY{9w)xIYI(LJbH^2918(6V>0>*b)s<)^N-{j=LlK2}V0dky~QzJE&5Krpvs!FoV#r{<-mb!s}eU6(<{QO5b&rVB}xq_Sw{6>Ru`pvE(&;dp6hQxCtKKMDkQ-?i9-w(y^Mb?#d)XIWV73Y;tV=n3Y(uy2z zUx%DDYc{>ulyH`I#qvgU4PQ}yeaDasJT0#8^t03(TNlab-5~py_NImv_s73(M_8$w zwr6FrGJc=ki@{*#1CulSmQLhtNmk}Yt8G0W$W8cIVT|oi`}bi7%$OVjYXApK?Z6qp z4O0r31H3Y|1%ClF&9M5bnPR{okf$05qz(Gt$BUN$R5J%K21qg64t=v@_0N9+bLz`L diff --git a/summary_results.json b/summary_results.json index 9b9f87fbc..c7d6f2c8e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.08844, + "Performance_Result": 7.76569, "Performance_Units": "Samples/s" }, { From 01ea05130f40076cdd07aa9626cfc29e2f5dfc4a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 18:44:16 +0000 Subject: [PATCH 603/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10929 -> 10928 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 93c59e6f2..1be939f36 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 18:26:16,404 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 18:26:16,404 submission_checker1.py:3308 INFO] --- -[2025-05-21 18:26:16,404 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 18:44:10,867 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 18:44:10,867 submission_checker1.py:3308 INFO] --- +[2025-05-21 18:44:10,867 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 18:26:16,404 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 18:44:10,867 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 18:26:16,404 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 18:44:10,868 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 18:26:16,404 submission_checker1.py:3324 INFO] --- -[2025-05-21 18:26:16,404 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 18:26:16,404 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 18:26:16,404 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 18:26:16,404 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 18:26:16,404 submission_checker1.py:3345 INFO] --- -[2025-05-21 18:26:16,404 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 18:26:17,884 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 18:44:10,868 submission_checker1.py:3324 INFO] --- +[2025-05-21 18:44:10,868 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 18:44:10,868 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 18:44:10,868 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 18:44:10,868 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 18:44:10,868 submission_checker1.py:3345 INFO] --- +[2025-05-21 18:44:10,868 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 18:44:11,955 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71755 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74056 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index f70eccf23..53d108b9f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1230 +1231 diff --git a/summary.csv b/summary.csv index f4ddb2fb5..36bc91900 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71755,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74056,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 0290cf377..f651d9160 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71755, + "Result":1.74056, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 1fa1fc3da71a26c8a8990cfa541db9eb3a8bad50..87fc00eb43949fd55e4b1a2e15057ab311abfaa9 100644 GIT binary patch delta 1620 zcmV-a2CMn8Rj^gCatH;>d5imMvv&wx2!HRCzF(qM36yA(c+vnOnNS))B!f^^Fo0MF zpk!u^mQb1uAT|m>X%c`m2td#h(P>Im7RtJ;456$b0O^_qpdBXv4qvvG1y;hcX#Z%t zWffoh*u_$pV2K)uC+#1S38no*GKggb`-f!!OJ>yomL~g$jRIJjMVFCYAEcey?(Yxl4lifF7*%{U@;SvoJPntg@6H4=kWRS}W<`2sN zmn7p$EKTAM>j1Dcc|RJwA83ilAn36}S(lW7$IiM2k6qX7{UDa#Tx73hiB&{)UDSTO zmJV${DB;p6;S&83Pij9T6H4ueWPgy$3fd3L0GA{KT$-#O)&X#7l72KuKhP49LC}?j zvMwvbQ!Fuq;1pZ4^z#^(-$a(JWs#LmvPJ7h>sH_tekp60Ot*wglt(-%{g6y3r5}<( zGAl?wECXbc43KGZepm-Urb+qHp!`5fLik^k zqVto#AnS!??_O%J1WodHc%<}0GNF`yNCwfYApNimph+@-rpftX9RQjp#{Q3G>I1kn`X_*&)1^)BvrPSMONB0i_VX>TiOXrbUa<6{Ag51e5w49jIyNj zLqdpV1?Pul08NqsG)>A6>wf^yG#Nh{j2~!;*kwaB!T2F%xNDNGao4Qb`1w9)?y2%~ zrHjf>zH9oSUG`M7#QD)^kNDF0AsJ;!>4$`n%?i>F%K)29*@4$)P0|k=1>iI}KN_4L zXo=`Fr78<$T~>zsCNYCx->g~s`Fc2?a-`i;_2*I-)t~%r6YZD1Jbzna{%BN4d};oW zjIyNqLqf=A1@(tzfK4Xu0GlTDhm8Vkn#>;!<`1+)beg~>m_MWpH_j^NkFMGL`95s! ztN){=|D*h}KQ>~Q_&+v^B);^2NJd#w03soPlVv2FEFhrqUJ@0|to zZAJE4)>mc6w2Kl@wqshZ)@4+dSU@)VB)+tONJd#w10o@$vVVdG#48GR=B_I_%U`alG zL`py;qbz9wkq}~8!2)6#z>@hjaF8`=Kx`D?(xd^|paDTk#HJ}f>Eay9y0i=qvK9XP zkGcsyP-`bl=V)|K1V^#|Tvs>2WK&wqw{h-c#^)4?eJOZL6-`jhO^ z(e$RV_;dXkdEVja@7l^c>kluN-NwB^JkP(&e%=j} zB)$J8za-nwEj;-PvjrB|4g?^1#&?t9BrF1e3X=~dcz^zg6$B<&VE713?MOreQt%OJ ze_Yc81jo>T0v$Xi8NnBC0H^%jhF-aHDj7Npcdch}bRWDNjcwaz?WDKH`~-ikUf=sw zQ>l-q;(+Q>aLOI9U|e0|>mnYmVLq(liMJBOQy29+Z^I&|J`hO^Ku>Yc%IDe0?-bVe}^wy%K|H5S+sw& z-Li_Wee7bXORz+Z#FO?9$%NAWAsNK7g8joXfF-kP085ko!$tutP4dt7#s@4x{(qs< z1S~=RA!TraWnIGwwy=PIDVF6`C6*@fhjjp0n!Fzk-Vd}yWDxY&p{z^Fz+-1!gU7CG_I?n{Z!WUevcxJPyDn-! zUQ36zACz$ElyHguh$polk_n~uLw_>JWd-erWq?bP0WM9}59_!Ne0L?IX|oeAk(D$Xi$EjB_e|mGC}zvWq6Qff6NCCvNbC|-+zb9J#~Jr zbkX_AUy${}vUe}FSAr(_J3LbQA(>E0KO}=_R*-&J2GArKK-1*>unquClk%fM`GJ;* z41%UClyzAdZkogkf=#n#<>zbBe3B|#%OWdnnnmYF+b!*cB|4riQGPV4BfeCANJd%G z`5_@hvx4)(GJqz@0GcM{hktbdXqt>44aN_&MC`I5nqd5pGTb#u*SKreZ2Wv5H1|~b zxza`DC*L*w&@OwbS>pU?v`2jD{E&>Yr1V2V$YurUhh>0GrtH9LvnJ_>jRJ6*oF5I& z541#dno^a8vMwvbeUq3$uy58Z{d_%~PdU==srqxNi|SAQwu$!3UVok~F@H2FB)&9% zNJd#w{UIS_vx54=GQcJicYsZk`ol&6HcjS_2J;76B05cA6U-k{h8t%U^GDZg{(K)c z_tpQ=(*IF@*&iFROZ*=jMG{~7KP00pDFBfWz{xTaPL`2yvIzm4wn+eDqewVS9dv9E zfS@H}(*%zl%DS{nZhxF41RJNlPyWx}C9>DD$SNDBUDSZ`jnj8aHcq=l1G3R4@udbt zGRl%35D6ih6+9r80X9iSKH;=Y8W8IM&@^d4HfTW55|Kf`V~46PtCHI$yF*~xwD-;e z`nDo_E$geYW7& z04hxu(Dud$D!~F`mkm62DC?3k#IlM7R9L{iWBDYJy_WS=!V)APzw}d5iV~0t9R>P4JaMvNPns)`9HO+tKS|6$ZNGMOoV^K@v?8U-~~Jqbw-^kq{zTK>%VI zK$597fTYO)Vxs_(CIiR@0|;6oHck1-GUrg%rDcF*1p~<5a|39DWcVZ`_f`C{on0gY z*OExK3!m*aiRU-*X*?b;8vP`@T-q!lF4f_O^?zr>J;bx|lIdWS|0Vn0c>PIs z>1cY>Sp2#Ej6Cn~^mlFLo%M&8%W>RDzs%y}X8d`U-9!0en~lNe;ri3@8ngK*nUB&h z>(TR%=Ua|@CzE9G<7!gN>FCd8{NoS(AH2gK`rc1JH}p@9<3_gh`)Ibh)hPKFY+5Hr zv))a7-aDJ$jHZjmI38vTwOW3&F<*b)-sfiavv$&3V}63aRe3Q&6IsxvJ2P!56SZ%F Date: Wed, 21 May 2025 18:45:56 +0000 Subject: [PATCH 604/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10928 -> 10928 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1be939f36..9f5a74e71 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 18:44:10,867 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 18:44:10,867 submission_checker1.py:3308 INFO] --- -[2025-05-21 18:44:10,867 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 18:45:51,755 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 18:45:51,755 submission_checker1.py:3308 INFO] --- +[2025-05-21 18:45:51,755 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 18:44:10,867 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 18:45:51,755 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 18:44:10,868 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 18:45:51,755 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 18:44:10,868 submission_checker1.py:3324 INFO] --- -[2025-05-21 18:44:10,868 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 18:44:10,868 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 18:44:10,868 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 18:44:10,868 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 18:44:10,868 submission_checker1.py:3345 INFO] --- -[2025-05-21 18:44:10,868 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 18:44:11,955 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 18:45:51,755 submission_checker1.py:3324 INFO] --- +[2025-05-21 18:45:51,755 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 18:45:51,756 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 18:45:51,756 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 18:45:51,756 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 18:45:51,756 submission_checker1.py:3345 INFO] --- +[2025-05-21 18:45:51,756 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 18:45:52,748 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 53d108b9f..9ee8e0229 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1231 +1232 diff --git a/summary.xlsx b/summary.xlsx index 87fc00eb43949fd55e4b1a2e15057ab311abfaa9..48f9f995ddb387ea0ef374acc98db30f9ab809c3 100644 GIT binary patch delta 323 zcmV-J0lfaORj^gC+$01M1n&=%;3Ob_YQr!PgztR{qq{BJsY?*H18vEvfkItcdK1zn z5w(7h)H-?l%69A|kW-JVnf-QlQMo$EHu?qYgjN;Gk`zUtnC3$5D)ip0<0Xomr;5{7 zE2z*B99q?PB{PI+3$IoiuwH-7@L`hPDsvXr1`P!0N@Nm3lfxfI7>2o_Ij|=-E0_?3#SzRXyB%G z5@+|piP_k7UD7QEYsjDB&&|u*uxcz+e<}v3E;%F20`=O~CB7`e;ackaCY*RHfIl9C ze&g-5$k+!w3IWI%cTzqUPtVOH8r6B47jas|dDdi0a$k@l|LR9x^`C+AH00mJ2{7iq V{si?Ovmz!71OyQT?+=qPDu1vtoYw#V delta 323 zcmV-J0lfaORj^gC+$01bdB%5>;3Ob_Zo)7SgztTd$oIr?fC^bjklKn+il3(*Mty2;?(1MX1|?Xxm+Anll+47(&!42EJq1wZiLjk3Vp1X=^Q0KFfCYP zG*qYqj~4Yq$t~r^!J9J{T#(?C2-2Q%TcLdjmSXJr9u)Igbm%BZ6#{2|-tzO^zRa2>tr{aL>QgF&0uwYzW;_D(Fu3 Date: Wed, 21 May 2025 18:49:30 +0000 Subject: [PATCH 605/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10928 -> 10932 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 9f5a74e71..d8347f6ed 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 18:45:51,755 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 18:45:51,755 submission_checker1.py:3308 INFO] --- -[2025-05-21 18:45:51,755 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 18:49:25,401 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 18:49:25,401 submission_checker1.py:3308 INFO] --- +[2025-05-21 18:49:25,401 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 18:45:51,755 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 18:49:25,401 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 18:45:51,755 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 18:49:25,401 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 18:45:51,755 submission_checker1.py:3324 INFO] --- -[2025-05-21 18:45:51,755 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 18:45:51,756 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 18:45:51,756 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 18:45:51,756 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 18:45:51,756 submission_checker1.py:3345 INFO] --- -[2025-05-21 18:45:51,756 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 18:45:52,748 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 18:49:25,401 submission_checker1.py:3324 INFO] --- +[2025-05-21 18:49:25,401 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 18:49:25,401 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 18:49:25,401 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 18:49:25,401 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 18:49:25,401 submission_checker1.py:3345 INFO] --- +[2025-05-21 18:49:25,401 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 18:49:26,457 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41817 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420588 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.76569 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9ee8e0229..939b0cbc6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1232 +1233 diff --git a/summary.csv b/summary.csv index 36bc91900..9cf25ca2f 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41817,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420588,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.76569,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index f651d9160..6c8cfb843 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.41817, + "Result":0.420588, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 48f9f995ddb387ea0ef374acc98db30f9ab809c3..8e50e907b3ac0824e4000377912fd8229181c03f 100644 GIT binary patch delta 3033 zcmYLLdpMK*|G#HpLbQ%ZleQU+BK8!rVHh)^$VhqEYBI-iC?5Ah4jr{=YAWQ^K{cBs zdQd5Cq8!p3Dj|m`g&fDe)9?CS&;7^epZmJr@7L#bx;}ibaxZ=qOoeP7aVu0E0Hnfr zSO^sCib`f3A3FXcVZhB=ax)02*|zH?EW$~jZ3VyFb2&Y3>fZ@<$>k{9?>hD;l^d`0 zoNYe)yRRWK<@d!(_D~It-JWh(i)w2P?63M6LYo+zuKMscw{;WkZ9s>=ZV!#vTvI4| z3da9a;&-Uga(Mfd7$d$n?q`n7o zb%uThX)P3bw(#IB24*~9_pOZ^-Rn;gOfxZT?XuuMJETE zU=!9x=c)=xHj{DZw47`}l~Kj8)=`6V$&umjUkuePbR2E7~ZXMTQbq<7wW2@v{4-J5V=KyRaKQZ?NGI%yA@aXep6v|l)W(u48s4gAmdschZ(5UiPQ?< z&{&j-A0~p}90_udUQQ~w*|J*)A<2>xgvw-!1t!~Ggs!o#4DFI(aPK*8<&R03(GlOuqX%>)`Jcnm<^Trv^6_z^*Za-1kPr4Xc3U( zng_x8ugQ7y5Q0 z1eYOBV)E1w9Z*3qJ0WX-L~|Z0uz+-Ca+A2KT&;o^$$`DcYUP zO+T-2L}Pz6b=??`wp1C?!JM(SEB={Kvst{yBJn}{gNgpin*MBQRN6}NZQ_-BPOF9} zrFDLbR-qti=7kvnHF=EX!kM>^2aG5amlPJ1ef4oGrFPIPxgdegRgT!1CWsOEvK54L zF5`) zveZWEF)}3PU&l3X*i*B)q9CklXCQk+)e%+3oT*ij$zKOoqT8xP-}G{F1H2@yhxgKr z@7;?^dhY-wJxVB>RL&{THX+W6=;KBh4ZGj|i zy4$|H0{0F&9Y$A#kUn66?>pvVX9bV2d^p!(!KvH$-fUNc3Abtmxt(MkEX#Dod`GZt zvD+kgEY!#Bo)X`@6VF^**NT+Mtyu>jKv+_iPft+Ez^B9FdVs3xY6qJ@^;MFN?qWP3?hTve!M z9n)bEH&%6IT)9j-s2^`kF4Y%|-kR=zbb_&fs+E@BfSa@+eicsJo8`ABl9Zm5<}LC; zwFtT8i$wE(>jK&65vSxSsPSs875-GrsKCtnC448$t;~XqzZ5w~p>_JMr;Z@~$`i}O z6y3gL3t~hk$ghbf|4(>Nx$sC~(v)0yd4R5=hti?}8ZQf>F4ia~(lqVR7<{3chG6XB zT%B#jk*`AwnNLcZ-(CStEb()yfSZ-XXzsCl@?&jmt9z8%li1 zF}glmtRcdXE%C?{hId4w-0DazLR+DfzY3smeieYvW>0g|{c*%K=MkQ%)#wrJ*e2eE z-0|eWn@0EpRkx)B)y0S|`d$n2b;O0~^zAngY!7Z$A_>-6s<+5jri`*39F5=ken0 z3qf0BGGsvoq&~7Njh2N{)PpPRqB=eonZey~TUc#46?t4#O70|IR9Da}Co*}xWq-1A zfs+!@Up@nhWOLQTX}PlW{_=C$?lqC~VWL;T*$e`19$C}QA~{l9q54~B!X)MT<(y1!?t zk9QesrK%(rn~M~D5Y)K%B&`_8PIPtgQ1E;P^gXAR<`Nns$7yV}#DkP$yLA_zoCl@(vgFuV`_Bs_L#a!5B70tQ z6A?|`kF`;0HXpoQ-bgb9C{I=9iH zWFJF&bm}Li)nh{+cR05$ginwKi@nb|wSaKNddVuqN$21D(VQP5EI+BT&D#^<;ui$kcm8!BG88<|LdA*il+41;H<5@2$fH|ZrIKRhh}3+DirDHp zN~z-Z@w3EacujbTL#j=$2aJ$fdGzb?X{msU$~W3OdGdO5zjZfF zHA+UBnEMbpK#?h8_S!E|tf8qrk8|_pp+bGMSmD3tH|ICz07RKXF|>wU=A1ROg(Py` r7+OMpa+HnGI)A8z0D!v!0IUZ7`&g%2I3yz*2%3{-v{gxBu>9wLlZ=t& delta 3063 zcmZ9Odpy(MAICp)nR_xS*Ie3`%XINwy2*yg*36w;qGHn)x+ucR=2DDgHcd_CPD(W+ z6(O~9zsB6YbLWN1$~8wOubfo^ zfR!?;Y>*_)*a>!++g>`Sa(Kp*dMgdxGOD(NuEW6oz+(LE3`XnzH0^h?YmzQMulQ95 zaXfVAAroP8)&~{g;n8CcRvE>4p2}B2A~VYuyHl>Ha$SV)`hxmY%CxZV^IzrTw0sr>>VY3Yyp65n~a0TFMEk z!Z2W9)o_?xb#*7`SBR}mq)fBQ`!{q0p=44FCapX1$cpJ*&erBi*|j5v!qRkk!)V#? zaNEJo8rM)wrGv@(d_%)1aV-`N4tpgKtVCiO2AZty%!we^$Sdt(MJ*wIq~z47%Dc4@ zX*PnR%z9_9ZkX*Xqu!_G!0aU6bN2CtW-EOd!*Myl3hEoWH@ty-&n?a^%-3_AA8 zS4Mi><3}YygDmp*b$}aIQre{OB$E-6 zsCpT4!*nPV@AuPLl(H(ex%-MF10LD?DTC|}?u2TfH>|d7M(U_?^JOp}J2s$8a`~{_ zJb=qD!8ctPg!+zEOp0VWO8(V5b7fFbdpaW|6J|_HvVQSK36^6c-v<8C`%|b3V0>#tV+d44t8iGhli&8IUq_7E@1;YWQ!SDAS)M8`A%(d+ z@G8kbG3Qr$&BMPI_NHN)RCDDG)!@^hbr-~+9#Lo2w^a70 z{eJx#!d)W}C5R&z1ywyZmq27T%YRdD1~Q;ycm!coc?#emBT<^Kb;*ndy@^sAaTsIj z+bY4SzP&$I^*BQ(h|fq2cTBi%cl2w{rVy0jNSDzFWL3rS?1tX-zj^~3dNioy32uq5 z5OjgUibi5l9gsAuj@#GyyG-7>BKF1&tNE18k&VK`4nX}2Btwo`Uf9>Z$UMnM_& zFXHXf?JSH|nf8VSx-2U*6D$fLNn|N9DWTDxGU!6wQfIaZjmLoXiLtXbP5FK+hqXe_w1L~oU+n2j?N2WRQD z4#0(fwTqh-2S-DS;AOaxLL_6GRAyjnQ+sdj64bqdW9u=1Om0i9@EMT9U$*eg6_MTS zt`>`!H~INlXD%FN0W5FZP!7FD_Y< ze-_VPDoAc}tO;7q@%`;qCRpkur!ICa>1&X1lL_(6R{1V5y-fT5nBG7_m$H`hF!CTu z^Pc!QCXfZj3gWqYV+Fo9?$l6;*YZ--fvnK{Pp%8&F57ytL?Nov8Pyg({%d46>{SYF zZNuNPjdeqy^g;rOPJsUeGT{s2W(nrD=qi0vF2`KvzJcFC{FnI}PO=4sfYUM_%vKAla1-b3sdA*xCo2w?+)qiGkohbH>;04uzp zP;UF-^f1cZzYpMK%^j#&qAXf1Ufe2a?&c0v4?$nGihgl|?}anK#5~7Qyiy`D9d<+y zVr^~`fkuc2_sy$sDVaz;At;;DMGx+vXTuEQ==Gc2eLY|=2;m8DDk#71OV5Yc{`M9g zC)JlHL!2rCyq^k5ZX8dv`7*SooW9{v(m+3-9}5q<*D`?)WkDmu#a^#!*%@c>X~KskFL)7!qUY{s~VA?M<#y_G##E&-kmY_GA;V< z`})?!IYd>$?0a*qxb}AoMPvH{=bX)S3r-62#&z3YX#a75HxAeLn@`r z5>WHiS<&YvvNY>cf+zoU28TU%p55J7?rw&E#_O#J{JWB)U3U( z?&+nN`CR*IuBFRH65xM^K7FXRvS_buj{dB~hE}h`YoUYfrWQ*IPIyTvHo{1=47Rm$ z6HP_xmof|l08SDBpaK8@@qB<&==lqmb_bjfCGCm`3BJ#2J^OCY=J;RN{4pj+%)MphsyyTj-ZXqv5H f4A=U9=3?ysE_zHG+uqm`WWY`~-U063xpDX(+uoqv diff --git a/summary_results.json b/summary_results.json index 5f7df1aa0..cfb947515 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.41817, + "Performance_Result": 0.420588, "Performance_Units": "Samples/s" }, { From dbf90fe86ac8d2ad866f051d6f54ba8b8e2cb49c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 19:01:02 +0000 Subject: [PATCH 606/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10932 -> 10931 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index d8347f6ed..195e2e98f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 18:49:25,401 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 18:49:25,401 submission_checker1.py:3308 INFO] --- -[2025-05-21 18:49:25,401 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 19:00:57,366 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 19:00:57,366 submission_checker1.py:3308 INFO] --- +[2025-05-21 19:00:57,366 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 18:49:25,401 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 19:00:57,367 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 18:49:25,401 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 19:00:57,367 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 18:49:25,401 submission_checker1.py:3324 INFO] --- -[2025-05-21 18:49:25,401 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 18:49:25,401 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 18:49:25,401 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 18:49:25,401 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 18:49:25,401 submission_checker1.py:3345 INFO] --- -[2025-05-21 18:49:25,401 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 18:49:26,457 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 19:00:57,367 submission_checker1.py:3324 INFO] --- +[2025-05-21 19:00:57,367 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 19:00:57,367 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 19:00:57,367 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 19:00:57,367 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 19:00:57,367 submission_checker1.py:3345 INFO] --- +[2025-05-21 19:00:57,367 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 19:00:58,349 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74056 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72726 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 939b0cbc6..81c545efe 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1233 +1234 diff --git a/summary.csv b/summary.csv index 9cf25ca2f..cb019410f 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74056,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72726,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 6c8cfb843..ef02fa55b 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74056, + "Result":1.72726, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8e50e907b3ac0824e4000377912fd8229181c03f..5e787773986137d1660f4b9a26454d66660e84ef 100644 GIT binary patch delta 1737 zcmV;)1~&P$RkKyFatH*wlx=Ubb_ieyf9;n&xzwrzN;FA4X#kN-C=DQzK`1L2Kr91L zGP6cYC`|?s8wH>=2|yYIAZUr`G^HvFWnET=P*xCtbj0E zlJO;$Ch>=L09cy59}V6Qv_xbO^w^=SOUl4wXI+EGu50#w5X)~ave&Z2Dk8fsYCm2} zhqhnx%g@>=;S&83Pij9T6H4uee`Ju$3fd3L0GA{KT$-#O)&X#7l72KuKhP49LC}?j zvMwvbQ!Fuq;1pZ4^z#^(-$a(JWs#LmvPJ7h>sH_tekp60Ot*wglt(-%{g6y3r5}<( zGAl?wECXbc43KGZepm-Urb+qHp!`5fLik^k zqVto#AnS!??_O%J1WodXc%<}0GNF`yNCwfYApNimph+@-rpftX9RQjp#{Q3G>I1kn`X_*&)1^)BvrPSMONB0i_VX>TiOXrbUa<6{Ag51e5w49jIyNj zLqdpV1?Pul08NqsG)>A6f9n9yG#Nh{j2~!;*kwaB!T2F%xNDNGao4Qb`1w9)?y2%~ zrHjf>zH9oSUG`M7#QD)^kNDF0AsJ;!>4$`n%?i>F%K)29*@4$)P0|k=1>iI}KN_4L zXo=`Fr78<$T~>zsCNYCx->g~s`Fc2?a-`i;_2*I-)t~%r6YZD1e>_`a{%BN4d};oW zjIyNqLqf=A1@(tzfK4Xu0GlTDhm8Vkn#>;!<`1+)beg~>m_MWpH_j^NkFMGL`95s! ztN){=|D*h}KQ>~Q_&+v^B);^2NJd#w03soPlVv2FEFh19Hcoq={GY!|WUpnBRW?q$r~&00r|*_*oOX!@WTQ{wOAUx* zlqEeN5<)gBct9)zY?6$8!fBf{Al3n(Y0`jf(14&NB7=a(4pm)NCAUp>hrqUJ@0|to zZAJE4)>mc6w2Kl@wqshZ)@4+dSU@)VB)+tONJd#w10o@$f3kuG#4BpoLB_I_%U`alH zL`py;qbz9wkq}~8!2)6#z>@hjaF8`=Kx`D?(xd^|paDTk#HJ}f>f#*Ay0i=qvK9XP zkGcsyP-`bl=V)|K1V^(Wq4s>2WKe-DRyh-c#^)4?eJOZL6-`lIa9 z(e$RV_;dXsdEVja@7l^c>kluN-NwB^JkP(&e%=j} zB)$J8za-laE@lPoD3vn3X~4u2&9kjf=lyb7xSw*sCV_1?Q#F70NS$642ZTskaq+UoX>pl=#53V2#mGp$rNPEg{g?1rWim~TAP|Ro1p`*AlPO%V6x5bw6Z?*+o5ORwZ1SVKuxCf?o zBBBFQ@Cj*uTr&U!$IyTR9X!sn9G|=aobp#22Ib7DWaunhwI0REz4vl5wr!iWv%wnk z=lFB=@;0oRO5LA|1FB2GDR;nvaY%KE&x?4thWW6HC*DX9k6qNSyd4%9`#>Zy03G8V zm5OZq4CdvW?yOeEj fv-BuE0R<%ikjf>KHY!vBC6kLPBL>$e00000o*Ox3 delta 1730 zcmV;z20i(+RkT&GatH)VG;pG`b_ieyf2Z{Q60J(0M3cmm1`x@F(f}eEgtCGG#4-RS zGi$Vj(qsU!Q2wC1)@5Y~Wd#99*DL_-F!^`*vb8L*5|%~#N82r{ z_}a%VmbwH>)JQyO|By^5?H`gsEGyVQECX0Fs|K(%**|O)z|tiDY;SzP667B`e@(y= zBC_kE_T#m5 zX!|9<{H&c4F3}(Hr1nEHq11jze+Id%p#886a7i-2rOEnX9RQam=|_X~11%951YKDu z>#{OD#S$|JPO&vhKaX+wO=Q_x7Fp>eTeN<(ZUs)^m$G)rbW6xYdBl^_56Og5`XL!4 zvx4-)GC(HD0GTG|hjjpCnv@?6$`7^ce+~dmlkua$_<@#)T{c7$j2}{lyC&%xcg>oOpYMa_o+>|A zx~Tl*yQUx7WluFroF9$$h%cQVl2MkFen<$}tRVfc46wk{a08W$hqrv%s zmWWPMsf3qd#k4A;Wm*x-2 zC`+n8B!p~MP=8nk*ks}kuxV0%*eJlJ$^6k^{yOUvLOTj9_D z*n6%4rNbOafAu8)reEA8ob5K$pBIYda-KqDkUQ|A%ChB?TZ7LL@5)Kr91D zGPMSfG#Nl_6hP8s0NG#wK}*D@DL-1~9Ll=143Ml~0NHzP0Bw*ApM>PTia)lqi)7$h z63KSq!`&wF{3bq)$Kyq#pJbP7UAb*tf8yPxI{dKye{i^mcs5=#9gOn7WZxUFKgup0 zO>Y{DKi40U=N+E@uC2VY{_t`+jvMKhS$y1#KhLszC|_)|G59=Oe>`4eHXkMPQTk;) zdj9!*%W>~yk_>)aO=>wE{ke>P{GtDYclbl!`|0O~{;6@?$d-N|%~rP>CI5m=>*Q$G zyNSkr&peVrz=)mLt_@i9r$WOD!AZQL8g^ZdK)=iM+# z()(}nOS1jo!jr!Mvj-N~4uA7W^;9MS002+|000pH003lfV^DH$aC0wXZ*pZWcx`N* zlFd%TFc5_AJVoSt;v}W0SV<}Y>WNB76j5=r>}_fh|Hw9^d3)TXOI>J~%NN+qO;GS#J&bGrV2DzW1xf zLLE=V096HNgjt|o+exax*F`v7L!DoT6Ym7@r!MFdZ>L3uKHyOZK!&(y<#YD@vRR=j zPxE=4&f`4WWD9ctNb>pDapX<^1t?EL{!KgshTONGp!x%|B__%O1WPn Date: Wed, 21 May 2025 21:11:22 +0000 Subject: [PATCH 607/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10931 -> 10932 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 195e2e98f..492d2739e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 19:00:57,366 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 19:00:57,366 submission_checker1.py:3308 INFO] --- -[2025-05-21 19:00:57,366 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 21:11:16,382 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 21:11:16,382 submission_checker1.py:3308 INFO] --- +[2025-05-21 21:11:16,382 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 19:00:57,367 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 21:11:16,382 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 19:00:57,367 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 21:11:16,382 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 19:00:57,367 submission_checker1.py:3324 INFO] --- -[2025-05-21 19:00:57,367 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 19:00:57,367 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 19:00:57,367 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 19:00:57,367 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 19:00:57,367 submission_checker1.py:3345 INFO] --- -[2025-05-21 19:00:57,367 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 19:00:58,349 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 21:11:16,382 submission_checker1.py:3324 INFO] --- +[2025-05-21 21:11:16,382 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 21:11:16,382 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 21:11:16,382 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 21:11:16,382 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 21:11:16,382 submission_checker1.py:3345 INFO] --- +[2025-05-21 21:11:16,382 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 21:11:17,581 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72726 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4359 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.1249 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8508 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.0754 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1717 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.82399 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 81c545efe..d729899c3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1234 +1235 diff --git a/summary.csv b/summary.csv index cb019410f..b2d1d4eb4 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72726,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4359,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.1249,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8508,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.0754,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1717,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.82399,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ef02fa55b..0bb36adb4 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.4359, + "Result":16.1249, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0754, + "Result":23.1717, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5e787773986137d1660f4b9a26454d66660e84ef..d89ecf137fbd4cd86fb7d91e0093ea0c2e290746 100644 GIT binary patch delta 3789 zcmZu!X*kqt|DUn8SjMP?>@$PxlZJ$`Gh|Qpl7@yx)`a4xQbw}PFdSQUIiU>MGek5T zyKF;?tz#dambwalg1BiArg-ZnYv%XXARU{ z?A@Xb`Y!X&ez&*VZcI^`zj5X;1=#Qoo2psIiBeP?hwGixpA5dBW)}hJjMA~e?FCe? zauz9_X9h#rsZ$fXIcBFJI(+bLwqzQAUxa^{-aOm65~lfV{rEJVhl~-i2CjWBLWz}I zq&1Z}kSDKjD}AhTTbcXSm5!+?^Lb_J-ut&Pdgb{Ur5Wpl_r5XMPyK7ccV65Hb><3R z@(&Nzt=lU3y55qXQXvCSBksy>+3qT@g^}us@HK4vy@|i`2L}2Us&i5oNJFaJx9;+# zXMET6%!JLXntVzj)QN^$OCQ{xV~^vYf9MPHYjG3Rvhi%lfNV5Js54dvJ9bS4B`(F~ zq=jyqU$lAk-ds(9kfxg%_6s+ogP6mWU}|o^J@xe1i3*Ewd`k&Y=+X>J$Hno9N0! zLrkc6qi|_mPf~@xwx6}^1a)}1oD_LJ=~Uqo1Ajt^BIp{DV}gmuQAAsHVWPG%m227L zk5)pqHCoEe9`%;z9vL_CFew@!Iw_h_9~Ok2I| zQI%MH*@N-c(W7d%Zi>ewv7p~}Q^1%3ibxc=mfN7pCPyR~r5M4U!lZ3CdF6;faRt9y z6AJwG1eU@TFeFnv0AE~zeK z@xwP2{kN@_yN@(4WNbR>+~1n^k3?162tO&=%ySH~qjZg~PrC)_o`FERLNT9Qv@A@;W+_qx!Y|0@n9sac;D?*sXGk*KI@zN=3JBkEai+&b(*cf1)L4wjOHv z*LI?j;5h<<2Fir7<2;|_m09A5;t)$i{^^wjeq1*>Igt(5O%a!WlAIyM_$~)HeqF=-x%~aZK?W=(Z8`~NZ~>+{coV{ex#5Dy7xEqKg4VNLPP5;sfBMTe$^eu zwsJz653qvgT2v-FcqkZTE2~!4r93c7Ofc)OS6~#cPDq`G?rV8h11)pOiBvD0D^KW% zCjs=3gX?(Rc{9;}AzB5}LtJiuNzHfmT#*~mcPm>-ex^dY$S(#iB02Mm3M#(5^uqq4 zchb*Rgs)WP<$Gp`WKt}@_z`TO+2f5f-4Ub;eqq40L8-VWCVW0UFD{dy;2=RfTC`+{ zBzSYuW4%C@-czljS%}%#Ef6lVNCKXgeGYe&Cg-uG=aA&kR*6|iYVz(4?U%tG>;JZt zKV?7r&3xr7TC8g+3{IWOe+WHm3lA*bG(RskdqVGr~@EYPMHgjX%a!jgjK3X zvn1n!&?Hk&@<@`wa(_4%!c!6KgakI5Ob$t2HddREOuMwl6mbPZIfWy7jox5fEREWp zUg7m1?X}`0dh)jO`!U_nsu8$Tfys>dZY&G>VKR@WbKAjn;Z~ya`Dmb+c5uBAL`E=g z-kW77OCApRu(9@Jq-GnX@+*g{Q!DjEPvm)EvywSXTNSNa=DC#~5XTj#(EHmdrXli! z7w1N-SF{C@0V5+Av3EdDnavWspZIHHu#nAh7XR+n8#mn_byeQlih}g`D^tcO9cQJR zlzq{+Ex2nPf%TsLM*Da09hu6JC>GmEewt+Z$r=eVSQ*1rukMU4Xa6z;+s7qG<-uW( z@)VB$&SI&PCs&(%t@q+LHDzj?qHB+|LJl^5G-*_cYQJ4v?vBiXB5lS0bv~8lJgr=k z<$N**saHZ{UKa!XftWXaw9c+1B^(1JAMNIsZ-1n?drLPUxx$&r)WxorZ3Gv`_gNo2 zEEm=EvPMor)oBtlDMS1;4!a#GVWaAYo9+LOAL1K}FfKDyg17r0!s=f6fEuVPXrdL_aYUXx zAG;_g?&uQQ_Mpd}p2Y`c(SoC$EO_p$QD!B*v;UWVuyxvV1^3HQ>+ZwHmno&0@V+FI zdmW@cewv(xIT{v&zmgo~#5IoX=4(dXB1VnNfl6u2!7r?8W~K5D&T_wR8;v+&j$QBG zq5iRUfSD?@lKUq*z467toPQSX)$;rVbi=gJU9fwu6Fh6NrOMDbDOHN>pqTfLZA+|2 zHg5S%`70<{5*%JEL9$DYPCblRXc4%|Hrj&3 zg?+@up(K@6viAar50+aRL~4Gxvb}-c%DttxC}(P?Lpv<<@=NP`WqfGrTG2WNSLvS2 zfx-s=pO#X@lN>DKus>Yb%`$l;QO6{GkkNS6=sY-fI8{O^nDy$qaG70Ejb7H#h(?)l zSsMghdoR&c9bdHO1f_+3ZJ#a~oIz+M00>f%8lk%9P32T=C7Q8nCl=)dXigbwi>l=E zb}_F})~rV*ywVeIY9!3{|BRh$h%KewQdq;QHo93AnCy6oi)j@0Z!C*<3(@gB=%i7J zq_GT@PKIZ8lyg!pp!+#!2>nNAyM!ajrLPlEE|dUH?|l(JPz?Zf8u9_(Ga6Qu;?Jhl zA1h2~RF7e;3z@(C3pNgBytOSvNVrs`5#R9MPS5=Z_g=ml4r|%z_Ce`(`iNMsQJWHW z+HomB`hK(M`@o1p%P35(gj?9UP&N0daXz5L;cqrL7zDCm1AzoUAQ0C7hE0%vK(NXU z{~#~rFh6__vDdd<6%w-+d91KWsYl9BpqAZ_R`$Ta-X%zI2qEw$DtSaJe78Gs7WTzn zU&zOsn&W<1neR_bR9m=D$YV2dA&0t#!4LPxx}!at0&rbbC8rzem%kqoSNhRJbOpK- z9%?Mv$JKmEjtf{eAc! DZOSTX delta 3788 zcmZ8kXH*l|)=lV$7{r2rw3vhdL4-k2h!kmo5JXf2kus1VfzUAN2%)|>>s#M?>)jvs$64#{bMD^f?7MEwwd!k?A!5SsGyMbo zsdvTnh>aI&4HNsLJ#y;rP`aZ2QBfw8lHg*hyp1e?*Xl&wP?BnjPZVW@y}L~Q%sCx2 zdUBT&p%XCC#%+V=DC%Eh`|3^Ojb$H|>N7hxR@@YYI{yZL!-zHn>!hURxldbgv#I+E zF*aX`h=7@M{#XTD8D{6WZoWe8c=Db1aU> zc5$H%KUuTv|N4&H8j)p0`4Tjc<__7Ng*+);virJPpY7f6K9TC3{FT+~kvU1Lc&?pB zW_I3M{IPa-9AA4W)zOQ1qOIWgg9>?`gAt6S#M2KHVPZ7WYE`yF&ZZJ=nKf@2HwQuabJ zVAM~;`ehi=C5Ksm->j0_!Qav6Y0oEj_BM4=_0N7=e^D;mIA790EsEfjX}hOX{;|%9 zvQ+TRccsNnZMjT6?tayPum2F}gIywiteBcij2UQT8%{x9V_AlyKiu@9;kq@|Jc4Qo zuUNOEWh}oH3Ojk-mTLZC@}r^j>fKW-){v2uXyuF``BCTBo!Ne7qBaPbBc4yLr@^K! zkx2wBnCWvc}^I@LN#r)w`Cdn~7dP&!P{rjOF!rwA9Wm?GXugdI~f_l!Y~J^m$D6 z%s>k)Z1PfRYcGa7_f^?0s%Z5Yum5rsN^kkoqLafNLP~Ynt38dJ;lW5Z-f3>(k8O z`|GLol(|MMg*cF))&%YE^PGA2!w<{(`t#jzcR}AtY`4ddtHK13GOa3VMnbyQF= zsrcqIANY#L`DZ6i*fSzcwA0}fxhnj>DV=3nnuwxp1~dm?8>pqL*2i${u1#u@$}Mg4 z{W*RJ$~&^LioB(m|`>VT1O--HZ3y`NEBeaggfC?}|}I z-0);sDLs@O2F?u2Wk{(A4CcfhrayjY7hTD zPMm`rNwE5^#sZKLr%bW3)?gN#*T*XNPpCpgrJu=-oF@cMOp9qARt6b4#>%-KJzl9~Z?sH4pB{hi2@b^Kfg2mIyiy*QZjxi)aCExIzMGf$s%6ckaV`?) z!MQ$Jpcb7q83z615Al=Yu!uK`%@A1tk%|S_c6?TRD|z{hHAt+qkeo_cK|D4^n_I0y z|DeBQw|i7TR7MaI5+hb;Bn)wOrL{K}#eIgV{#JMS>h0;voetIb_!(hJZQ59~{W5<3 zwQ4WJ_Op$ycDxVe;8SZI63kmHckDiF0obd%)-yV_Nh&2I&cjKih$V_nxN{aU%-=;C zv#VrMoqEho|w~2C3uRALS#LtSQ3Ww{WKk) zHv+k!ocB=CU|e2lB$Lk%G@Z~+KdsT~f5fcbe{3JFH%PUs342-9SBsaa>#Ur031G1= zBsEp<+kTP%8lc<@34FgTjUjd@=RyR3)tV?9SL$_Mm`$c-R$(363+eb~V(88=` z1M-UfCiL?60lEJIE-VM+-7DIq`&Tn8F2iN&3$aIjGgTgC9Hd+8G{abcJlm&0J=Y_7 z9Th-5$qTn@^^GdkT)+l4@X<`1tK-9Q*u$Xl+xx#0jAl{tE=)NM|Ga?U@!KwL^Rz-| z&u{PnQ)JnB0Uj`JoE06YB#am0E9I zc3_>4rzJcR@CIECb3V45i`}dD8zO8VFP8p@Zd7c)a5nb!z-!J-edA2_P6+t``h|HG zgW3mT-S68{ku9djZM5o`K)Jpo8|s!>6hIxrA*Kmzi{1e19oV@jn8_#2)B2qyVxtqN z-quHyrbR6RT+4!F(1v0c$zElesU?>A4C-+yKAZqpwwZ#1&=9ApAhJ-R;bNAavkBoS zPaa%)$UqLF?2|a6JG&y4Z|0m$V|gJYMm;Gq@4O{^c-n>~r}b|hfc3v;Eh^^~5IwGq z^toTcX_l9V#0^>qqTWP*%f9e-;7=hNU27yBkYNj#RHm51HBOiy1nqUIEzxj{h?KnW zQ(bWrIU#rQH1GUVZRW6SHT6v;apqAYb!}aHQKiC z-n=W9$@(r|v3$@kJ19{kQMFTvB3{BX)A@)K=8|zFG-VY^gRPRo08Y6dlq9D>HRf2U zwUNS?==Nr>TdwiU*M2`^rPzU|>BDq}I{5u*Ut>2kxt8e8VRw(IeUaWhe96}sa>V8v z8Jo^NFa}}+vLS*ub6+YA0fX?ASY4EHk93ApoO-ikKy-#;c z(B>jU6>J>Gto?^FdoLGs66@y}+X%{uTHLw0@Qh1sQeJw1TXFNP5W;ZC;l*oaI)$0B zD3RWm%Da{S`a~&<_qzfh7dC4Z&k@u-EGvX@hA+`|CUg^Jgz~&Q+q;LK@gUb~X%>!S zy2)K>wJu}g=-WuwXFEPut)k0z(jU=@#ay$C3EskUWE=oP0qSDfK}&!hU0GwfM@sx2 z!kOu|Y^vbu{V1=4hhL6v@{D}GmvKT%lG`se`fU}u{=8bv;Pv{XZT9h0WO3FDz+53d(21s|5n+CX~%nfdg;8yizJ>a2q~VtvOeb^Ggz?W zf8<+s1JcEF&*v&C8o$S{uXxMs;zw`cb-uzF*mpo4Y@M7UK*;S-lKje7_;zkH><{DO zgZf?SU(;TX?(mEV-^=%lO7@F(=*D`U*_+0CNsz`hBFfU)0RUDZQE`yRh*Mv%M+DCK zQvnRij38pJ!Tg9az!c3W6<0LO48!M4oU<0wvx_R&DQH4tD{vXt^z7ch0M5!kU!f5m z-$@j~7G0V~OD145p%;{;FY0N8^9O@Kc@+H@K2KM92>2A=MpLmw&xx3AketFDgkfOpY};N^`0FJ``mVyTB|y3ROOa z^sqLHHMcQ=6d~}y3@K0N76DJbRq)ORrJu#~9pZCeF>v%L@YjXaj2EYgd>^bic!+*% zZzru`PR`z-Bjf5N4$;@yrG)F(yWi`maqu=PnO=J(4wCy{Oq$c<01O=xpFAhtT*?vj#7#HP|;jLciqViI~eOT zbFi4!-&?@mWGZbLhm(!h8VX_q<_qMQDFn>IMlT_({4z&tq{c4QyCHA$gMO1NMkXZ~ z*>T+W&5pNM<6d*a;4ogtC1zGnUWuxUfo*)6{JX&v+1H$ad#sr!xe{Bz>i!P)t#^P#uw-;R{Sc*1ify~o;ri<9n| z5G$)Y7U{0lACd1F8yT9-+b7p>@W3Nnqm1V%-*`#-p&;*?Z1XK3GX ztGBSIUfZrp3SHMmTMeugz9|)wd`t-Dr(mqnMR8+Ru$lT#P%)&Ftt0yU&3#r3{+3 z7p>gZ3P_7*{`tf><|PHB(?LwMWeW_=75; g4wL_1zWJ2@yGZwXskZ7yKqxg?{glYC>aXB`06gI@S^xk5 diff --git a/summary_results.json b/summary_results.json index 9c95b5264..ff72052b0 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.4359, + "Performance_Result": 16.1249, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.0754, + "Performance_Result": 23.1717, "Performance_Units": "Samples/s" }, { From c1886d4e479f85904812afd97fd9020a682463cb Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 21:40:49 +0000 Subject: [PATCH 608/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10932 -> 10932 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 492d2739e..8de47c4f8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 21:11:16,382 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 21:11:16,382 submission_checker1.py:3308 INFO] --- -[2025-05-21 21:11:16,382 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 21:40:43,734 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 21:40:43,734 submission_checker1.py:3308 INFO] --- +[2025-05-21 21:40:43,734 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 21:11:16,382 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 21:40:43,734 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 21:11:16,382 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 21:40:43,735 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 21:11:16,382 submission_checker1.py:3324 INFO] --- -[2025-05-21 21:11:16,382 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 21:11:16,382 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 21:11:16,382 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 21:11:16,382 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 21:11:16,382 submission_checker1.py:3345 INFO] --- -[2025-05-21 21:11:16,382 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 21:11:17,581 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 21:40:43,735 submission_checker1.py:3324 INFO] --- +[2025-05-21 21:40:43,735 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 21:40:43,735 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 21:40:43,735 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 21:40:43,735 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 21:40:43,735 submission_checker1.py:3345 INFO] --- +[2025-05-21 21:40:43,735 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 21:40:44,976 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72726 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73133 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.1249 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,9 +36,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420588 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.76569 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.72811 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0109 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.045 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d729899c3..3961fb45d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1235 +1236 diff --git a/summary.csv b/summary.csv index b2d1d4eb4..88ac7dd29 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72726,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73133,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.1249,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,9 +14,9 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420588,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.76569,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.72811,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0109,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.045,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0bb36adb4..3fda2f876 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72726, + "Result":1.73133, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.76569, + "Result":7.72811, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0109, + "Result":23.045, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d89ecf137fbd4cd86fb7d91e0093ea0c2e290746..ad637199745ae928559478c2c8ed1fdec7cc4692 100644 GIT binary patch delta 3885 zcmZ9Pc{J4f|Ho&tM3JpqNwPGCVW>3OFC)YZ6Y1J=EoFwm$Wqq9XEfc6B+OVsmh9`z z(w$sI$ykQ0BRffi#*(c}7~iSyIp6a;zt10^Ki=nj9`EPt^?W^F=X{>qJ-4e45)u|s zSrqga1%Y;GL?KO}>36z*94pd2qdtV%L!4ERClr%Zgp@hoQ$ZeU6EB7yC@0{F)RC^e zrP%3pjeyZpd+XtvwUx^LyG?N!bFeW`Ga)g{ zZN_XPowhFy!4YD@}$th-E`Mg%2?#XhSc;p%Xnc452 zGMVVa;ymgDURQqo@%1HXG{k|5oVCni7Pmh6}$cg7!3DyP15m})V^|GRZ zfzeO161YLhP}+xK?J~?kw(XpsPihhU{S8aqw35kP?zZM5?TgD>r4`am3q_5zg7EdS z^KJ>%kGIx|mt}pjUDBebemGA(>8)z?u7Bl^$FhNrXH%1LQG-og`csgdW%$JkNEiZ>g1 zSjSb%hboo^;;xfk9oCFttWZsw_Wku9V9)Qn{$K`y(!sAJIbT~EZx+-WZu#TA;_bC_ z8y}hMeYf2R>BhyJy?VVP>a`nv-71YP(S$7;im2Jc&Bl6VFI6}h$hjsKw}jCp}~{Z#V!4qjrq^=SBM2``5e|&qJ`GdhcAw{A%ujwvMR1x#_;KnRI*p_DbaJ6@!r+4~RHRr&TDxT#>3TnLLq93EOO37ad6sGG3N32|w7C zI`N?A!PZQB;JvL!_0;($EEO1}z+Nk{25x?8`09&Y|NO0CxHos;6t>rW$VGMnYueLP z$b1D5qe`4zyYv>$riQ3iJGR@ZR&P(8tzO_)KTA$7I1zA^&-AJRiFbRFm&QZT1Bl{P zh(w%XoW)T-z6XVW=HpdX-F5R%oUma;AkQbMP*2u)|GLuYc=K!EV5?r5Ex1*zxrdI| z9}7i4Dp2q`Z+c9?toOVm<=`mnGRaIb8BV2OCC+q8igi0Y1z)y3n|r_U{9x#%m`Vs(?qIypEoCP%TU=~|Wixa8}TUDxd$W4c1K z;;NcAotusvIy9FSJoZhTlMBU zI>$tnJH@*2m;Zcyj;Yjz7`1+eNIpuF1Z9FUA1mN}q%sQ+nzH&#yojIpGHaFc3qYB* zm6n~rG|q15A3qdg3jlTN4LQ!G4s|o=y)c*Ytxh_IdwKD~U#mt9O;{dLd6Xeii_(0< zBWBG`x?ovaHK+)rmgw(y=-P(WFlR+QwESoa&c*)zIP`wN`1Ad%gmUF(%&oS`q$ zU-hY*)G@}cmXgD4#}zV=5&94^-u_&uqhKF)D_PTwBO-abR)9e_`;3b2yZ%I}vlc%Z1T52P|!}$o3 zsRE-2X~W8cSbzoLsFY1VF}7q}PChiRn-c6TO~_(w$N1HGX3wjQ4qEFCtdz~)E_;xl zxPBd-F1(?tVk~u0!n)iLB847~#foP!in}hfcXFZKnaerm9Y=F|hfV%1Y3ofLvx`184214d-BxUF2&a}(!)$*jWY}Cc zf{_2P{~GNmSc-KOwehwE@M6d!6!)t8K9VOg#tl!Z)qt76QNxtECS7Ea;}7e%LDsT) zYQL@A^gd|35CH$DwPvuC7B+A|kw-&9_#A5=GN?F68l+&>-@I`EdYF=+2$(Pijm^pY z0Kd9KR`ITE{%sZZfM)nZoPx8}`lVq4Sl!68WN_}jWivKH+esF?Ays+E`3JkK5*KWD zrMdwRDogXPI3WDerd(2`AF@^mHr0HJAXx}Lt#k*~h#;BoPO3jeNFxbHNdpj&w>T+J zDW8>64o?YolD_0*3>vt27^!c?m2CnD>W3@u;rhA#AzeY}S-C39Qqy<_ZC#9c&z7qo zU08c}w?z}FMDrn`X}c|#!K#>*-fwLORh##oviNy#1zZ3L=h6bESCar(>W(UR!DX$T zxgzsyNys-nAnrh%a;qFwxM zUKo)cUM}RezkP?I#k>4hYKChWs;6ISTsLyHjHSD+zw8}@eU#jiu;w`=g=WWMle!KK z@vKjW@co%tC^rNS@D7Sb=os;QH?~Rri?=vhOyCmaX>sSejg1rO9~h}sxa8C$wPuUv zZ2M_`T<_UQNll<%cCB#3MDyvLXbVAR<@9c~ND@jDS&ZV`kOgHxKbe!)_|zq&? zX#8eQD(qU#2}Im6fIrK#yRCOPp98r@PcyS0L&SF`(w)bIEeQrLH+S&}^GNUAq=&VI z>n4{do&s~RI1n%02;QKxL462eqm%dK@NpwJ6NK?LFNnUpAP!eL@zmj0%GP-5+N_&= ze>`>h_D7v5r(Y#;+m@psjYG}H-b?B})Whpt7VvH)X4n(cBPJs-jer@QOyS`c5Jj-V zKlj)URD$gOp>mYNiO|UnDxAEwF>iakJ9pPbYB{~pz*&)tM_5F{xIg7_OfGeJ3T*HU z#(Z7|WkEM$6AlnE8FHdCJb&+GzJXGV3PrSgRA&_g>1x4I~NZ{G*VzEp9 z{>Vb|w(b5;ar?3NKv&X|18N`MqhW-mpJTXUGG=eRzvOw4_jy)iLGepALq)itPY{I> zo#m(jfUA+%$m`sw&T_s$irb3{8^KhVQhz?*M5;n>#J}^qw!d~GwX-h#>YA>qTVy|BQ5i2*8EBal$ zx6QJce3cu!MEta+JX%z+i1ApP|8ZgcRSWCGKdGTZa;_TffoJwK;}f&i=AA;s=HmUd z1ao8A*6qIUJ{MpO0rcxi^IoGIf8CXAD{6CM1xmf7z$kq!(P$I+dIVS?8tqN=8C`rN zGdi$ZF#5&5VAieLJNs03g!WpvfAob+jgz0k9;*-faJt(f_ri%4&sbRck4`@=L#TJA z)ynIa2}E*O4v@b0N(PFG*vCYp=()~e>&I;@`S_25Kp>$yiDSJ-03mxxdk7c=vN-?( zN$`%b{vI|#{sFhodiV$7G{XD{+0-tdmUH4}-}ep`G^&rI&=Pb$KW2H%rEAw4qy``o z|D3k)$=;2c^OF2p-qqiD`*Tm;%_wT{=HVgXDs+kx>;dw8_g{J`3sEoP?rUgJd#qpuIEVc+M%p^?=TT&*geUsW=9KtDS^c|(8p20@V>HsKKWxVMv{Is2 zeA5P))f#?l-L0c9-i%n8BII?ml0LT&$&xOJT(q#6`Uq!!ZDlv3wa5El4p(5iqi-sM zk1Bs>#xX|_m;tz+Z%6LO-sLXm7e4Smh^m66syb)h9oQPPHy3Eqc>B@k%dXawu(Kqi z9J1kt7JT44IPnr_pj|dKv6Z!Pi(GEd?GV`TV1tnEh0UPOtyjo~vjA}CjgHrGXYzWZ zc7)4~>Y&ih1OMm$mZRbT7N%W=8GvcDM=)bBiPi_x1FzE#t3qY|f6IAY|Mie!STrkD G{C@x!GH#Rr delta 3861 zcmZu!X*ksD`=2qE)L@KC$UZa3J~>DjJ41G|L`i69gh&Wql`@VnW*82UT}~)N_6!jX z$1Y+xr)(u#gD}?L^gGx8x_&SI&x_~9^Wwfg%l)~p>#23Fb$)&47;m@R_wd_%AkaSf z{xLK#nAsgJ7m_WYBs6X_9GR-i8Q*sjX8I0$7=vtc)!nXqGlP|#%tLJ5!R$#lw4CtJ zG^a)9*@)g7k`d{ynJo=)>oe03>+E1IFEp?Yr+Xo^35(%F{HN!gHQyRoNXu;6v;DPi zjx9T~jmd?zX?NAEQgZ73bujpPa(kf0iX2cGA<;*@V#f5Iw5`ptCG2)M4}}en zQReH)4ww~|K}&4p@at^pG(K8XV1&^!+qDv*{dE1rG#*O9irNC# z{wdZJudvE!E_bF(UglB$SnaVgx80qItu6O|W$xMc!xXdf{H*efZSwnoc%Q!q)TpjA% zGZ&J)l$e_lzGHdO?$vusbwNUges;t*ZblC^mmBc?#?2Cb&&BgRYmUq_v4D-tOe_W| z7}9=~NC0iQz!#Uzu966!7f&S~%ezV@L~y>977STzM#!wv&OgP4+%<*^x@HN5o zMrddX!kTzYpZN8p1rK3C`sW3-g>{1GApEj<4X?}2RKJh9MkKhdy+X*0?ZIVx-~#b3 z+al{i%-=+eD&~_5TJ8e?OUitv$%wL&2O#D`4^|O+R zkndjPCW+3Pp`B7zXy}=e117@%Fl1BhmtOSr3fGJ^2l=G?whyg3_W1?l367ONH}y1Q2+xtA>(^S9VsljbdNo8EjM#hlgd z+)Zg9ea-VBjRFJ(8ggXVd$#FEfY18ERuksK^Wn2T172gE;!{2*{j_4Hckum~vMaZH zbLCsPM2kbWmO_`vm6v1nGjcOe7B!Wqr3F3EzjQu8` zJn2qi;jh+&!XN{|C9vtWLW_XT(gVsRt0Z5~og)e0G--w zw=#rG`+j)o%gf~y+4jz-|D<`5T$aF>NMJPp%iY!qT}@-axX7DvUZPRo&awL6iPXl2 zowR+Z&>K|&-Lix~?4!Ppi{Y$!?YQ6*@UkQ?&QJVS1=aU9s#H@|zkVOu57%Jbw;edy znmgM7xBhD<#YE^l0YwL8BOo~MC;8>pIFba++E`$EC0PL1Lm{Sc;CiSM3Xd|K-GK2r zoa_ngrIwa(_ast;*!o zC0Y+dsQ%vyn%^=XSWmF^iKdq-EmfnZG}Yr7r3b0rj(UqGSGq+N19QSd7*O}T`Sg~l zqMeI-Y?#+qci`6RbQ?`;9rO~A&fBh+n>`5+`L!hM-4s7SBOtl--?qAH59WhDnW4Q zdpv_0^otcdCL?oe^00_}WN{HF`|}SXn=f{}A#kjm8Lg_Z{c*{@**leA>E)#tju-t> zf3~3lq^q3nTcC)y1?W;arzRks&3U#^?vNu@>Cr6`v1YL+$7-qWQzchl3JvmgN1Sw!;S$VloG9 z5X8>F6IW7|U~f!loMrW9Ui^HXj)BX$Pw$7f4)QyyM^AS<6xqyJo*?*e_=D4+UUW3v zijwBa&J|7kr)7?kl;-dIe1eWVo_ugvZ9IU_UcX1^FQ_o@ol~bg#;em$c-C;$?q}sX zWQwV9kr$`K?h^ZKU!TJ*zAB?qK004$|6T|#cF)`D$@2(s=0Ij6sx;!*0&~OYQt79l zQWUWKA8tj+bQFY;%wdgLMeIEcBuazb- zlegX8kL!n5k0MvYZ0H}@kfo*?c}}KTB>;2Vil@$H zy?3C=@f~tkwrVt%&32MKPMN1_B`IJPEO&>RyCc^GkkBgW<<5s&f}PyCwmGFNZz zc7-~>#jolrv_z#Vy)ufqK1pM#W6CtgostSqbS@kX*h~EDd^+2CdW96*c_J2VP)cWA z7YF@;nm2#64pEjCi6`>*2q<(sQrf$vA55%tW3lvoR?9a+OOpC+4)dGrSA-R z?HgEqP!TKS?PA08WUmSKq<0N$8;06uJXiE|ie2{{Ik8MF%SQI6n%(at_Y2VFtt>H! zc>HBztSk2?AE1Z7Me`OZ_LDrQjLsVR%r0k9rr_)*|7(?LC5mwF`SlGCjCZioWmoe4 z#AG(TIGhX0!M$3ZpMY_<{`&j}GDaq*0E+pY$5>ImD>}>{T(s zsoB0S?b{G?Sse)LTNv2B9(wISQu#n~J_9(Y7HsZ3s=z#q zUb!4tn|}@`J23GhqlV+$;A8Gan$|yl4NLD_7uM)?$mHJK(Y~k5edVMIYRIH1pFBf5 zqs%hht=R-Z9g<@fbDOX$O2Bxfb57~9lc9+<)BjRyEecx#4QlusXyOb80XWI9?MA|% z5-&*%RI{-cqjtNWW4Df7a_J@Y#-u5)JVRaCe2Fw$;pq*Y_(qNx=8zHb#qts2i9N?f zVZERX1jH#@Noovi=TD8R)6%!&Jq>Hdf37xmOHG%iI4kA9PXVnp!eIzjLSSuKpW8IBc$tL@{g$lN}UxH*-jb4vGL zB!$W^4WYF^T;AEhY~|fDSd=$+(4!xf`vzw8zcM{Czf##Yj#TZL%!MO{{zpn_(kV_h zal{|)5Q}W6l;&|Me{>vPEsni$N75ygL-_$;J$G)4%W84UdRlRs7TmT*cdoscY_3Tv zUUP-h!@qP)mk!OKbdpizVs%1I@0+Trx+)BF)j>Sg70{kC))iCDS8MXHDKy*ll@Qk|8rWEt=n-b%p_tS$$<*;I)h?#@Ga%M2 zH4o7L=gu(ZkFE|$7qWYQ7obuk37pygEODq74D2=*00C#Us3rfH*LY%A*01eO=xM5AMHwH4@Rf+vBgP-{miAyGCnH-tEAp0htFaV(&wu zj;v#`b&?(t>%w+c5YCez5a<}FM(A`uKUb;q4-O=G0U-hO1$3yv;@4u17d0#QN(Tzo zK?3RJ4~!h$?+6W}1mDCGM|C3idQxT)pB)W_{rzaUo=z(Ke_~_XBmKi3TTqHPHMESr zdp_15>)jMY>Z>Wc-q3LRc1%L~do$@uPx2$Jn@SB*?Q(*}dh>NgdoZQ Date: Wed, 21 May 2025 21:41:15 +0000 Subject: [PATCH 609/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10932 -> 10933 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 8de47c4f8..73b037caa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 21:40:43,734 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 21:40:43,734 submission_checker1.py:3308 INFO] --- -[2025-05-21 21:40:43,734 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 21:41:09,681 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 21:41:09,681 submission_checker1.py:3308 INFO] --- +[2025-05-21 21:41:09,681 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 21:40:43,734 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 21:41:09,681 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 21:40:43,735 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 21:41:09,681 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 21:40:43,735 submission_checker1.py:3324 INFO] --- -[2025-05-21 21:40:43,735 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 21:40:43,735 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 21:40:43,735 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 21:40:43,735 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 21:40:43,735 submission_checker1.py:3345 INFO] --- -[2025-05-21 21:40:43,735 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 21:40:44,976 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 21:41:09,682 submission_checker1.py:3324 INFO] --- +[2025-05-21 21:41:09,682 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 21:41:09,682 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 21:41:09,682 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 21:41:09,682 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 21:41:09,682 submission_checker1.py:3345 INFO] --- +[2025-05-21 21:41:09,682 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 21:41:11,141 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 3961fb45d..7b6369b10 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1236 +1237 diff --git a/summary.xlsx b/summary.xlsx index ad637199745ae928559478c2c8ed1fdec7cc4692..49c3c798a415dfc95ace5a40e0cd43675a9fda27 100644 GIT binary patch delta 339 zcmV-Z0j&PCRkc;H;Uop^j?29zljbBIf6Z#cFc5_AeF~$yEz4<25w-(u$*F-tU0QMz z(k2nLevs5UdHc#v;w0cxkE@yec6L#@*sC`A1?Q#F6(U)V642ZTskasSXqM?CN_=2i zu-0g(&;dMJ)ORJflp6I>-g`M3yROT+*9^;;s&)M@!C9_0zkr(qcpQiVERkT&G;UonJRuWbuljbBIf6HpaFc3xe{R*SKE!#;-5w-(u$*O@uQ(C$c z(j*bJ9!P4P{C(wT67Z_a(Y=V{&fw6xx+|C=Oj~%h+JN-}Togdc5oSts^xhDR9Xo=gE(s15#J#qX`cT>fe>RkT z(*xi(m)*XN|3^hnl-r+3C@Wt!FN;likDOXM@{Mo>) zbrdJ}$%)C>G)>YhI%~+E;m_^MTeoT~RC_80s46%k%mVe=Rt3H;!r^M_{5G6;FMvOu zgFf>%EYkM@k3s;_#~qcA#nba{B!j9v%~x@{it}ujXJna@ Date: Wed, 21 May 2025 22:07:18 +0000 Subject: [PATCH 610/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10933 -> 10935 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 73b037caa..a30e85aaa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 21:41:09,681 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 21:41:09,681 submission_checker1.py:3308 INFO] --- -[2025-05-21 21:41:09,681 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 22:07:13,240 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 22:07:13,240 submission_checker1.py:3308 INFO] --- +[2025-05-21 22:07:13,240 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 21:41:09,681 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 22:07:13,240 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 21:41:09,681 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 22:07:13,240 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 21:41:09,682 submission_checker1.py:3324 INFO] --- -[2025-05-21 21:41:09,682 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 21:41:09,682 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 21:41:09,682 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 21:41:09,682 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 21:41:09,682 submission_checker1.py:3345 INFO] --- -[2025-05-21 21:41:09,682 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 21:41:11,141 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 22:07:13,240 submission_checker1.py:3324 INFO] --- +[2025-05-21 22:07:13,240 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 22:07:13,240 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 22:07:13,241 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 22:07:13,241 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 22:07:13,241 submission_checker1.py:3345 INFO] --- +[2025-05-21 22:07:13,241 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 22:07:14,398 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,13 +28,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73133 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7395 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.1249 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420588 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420745 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.72811 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8508 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1717 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7b6369b10..98537919c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1237 +1238 diff --git a/summary.csv b/summary.csv index 88ac7dd29..267c14c43 100644 --- a/summary.csv +++ b/summary.csv @@ -6,13 +6,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73133,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7395,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.1249,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420588,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420745,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.72811,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8508,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1717,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3fda2f876..667b2f324 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73133, + "Result":1.7395, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420588, + "Result":0.420745, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8508, + "Result":20.8526, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 49c3c798a415dfc95ace5a40e0cd43675a9fda27..f47b4e76e1f21545b687bf00d024107cff97eca5 100644 GIT binary patch delta 3771 zcmYjUdpOhW|DRJLnNt`!e;kw2Gm_>INy891B_$!URYDG73q9y@+-5YCD4kH|7)m*m zS!$-1S6UId5o{>?! zaoW9QUF%I|X>D%QVaKe(hEQ`WWCQL43d+(&SH`IJkbFDjf5gpL-AnJVg(FuRhQE#rc#EuYDf+%HFSM$v4qHg)@ik-V28)4b7RzAA??on=~7P>VqT6gWtmeFDit`V!f7}$QBvGj;a{W4io zjGx5O4vU6GOXe3$n*|Vov4vAZxoHi`u`ZgtYrlljg_&JnZld0uS2lMIcvB$tyCn|F zUg+)EvX@nJN-xg4weE1-^;M^Xv2$b8gt8;xdiC8KndZH}Bs?|5T4dnxy4#1J;IuIoJPbA06T z=8i(^g}<-IU9qalvW4GLZGt>)pk!B`fT3J;8LfTu6}SYGZ27V|Hd5{&N!B;A_!K;@ z*wD)P6FhMpUbR?6`0lK5rq(>T?swnWEf{R@+o4k6cn84h37zz3{$BOBL|Z9-&KG4P z*J7)So-o2^{c%mI9W}k{OCy&wfqz3gvKfvPc#IcJBK{4a3Rxa=;sBF=!QEnc^>k)>1A?ZM%P9@ z^n%2-SBsGpl#gig6I!vjA-x(eZupKPZg^{(@Ul`yE+MNlTkoDIH6mku`SV?A>)eyT zJwWpeAe~-%-8vWVD7SaQ&(K~jLFk@0SZ`Ct&P~gd^(Um^T-y$sQ{w%}&7frc!!Sdh zm9=KB{izpIqp1E*Qt6Z;yEA!SPV2p9`e}LiNm810P5 zg=AOx)EnQ8^yD~bRs@#asl~wuVr&Vy>ZA-HHxM5`^Xt25hxcQP(~1a-$cCfW!cz!2 zS(dG#zVteIt4t&DJ^f%kwYB#BU6{JvhzM%fZM$Gk(Ym+?Js|;yT}Pu_f3{saXHhY{&P>qx5>^a z!r~k4uNwQ%i!=Az8Y1?4m8L_6b3+1l^#TxUTp$SXSqfSwhM)Fl1VbBYtZgG|U`NmNJI=W_s-l`! zAF9yMQ!?S>Or}`eYpvOF8daXmI@AkXM>pS=L3~yx0zx5JiIQ|kQt+n~of&~%=n8dk zXdP3I6r0h*lpFHP{?ioxqY-tBClo@j{^-CAz7ow`IX-ZJY3g`h?(WSyvZqar$G5P2 ztU^EtiIsRw7FP1_Gp;6Y^B1aZ^zCt>LX!65GGQz~?aM)psDRxKh$%Jn$4U3B|5BWh zYd}1m*yRi%YSXi9R66noB+2VErUv9>P12k8BN&~CXb=L|(jI3GMewo&?}(nWF&8fp zlLYIxqi~U#%-~|BuN<3*i*Hry4y;YTCv~To*Ii!o1|v)5#~kWX0|jT+awJGT9~Lh0 zOZd#NtDL-a$pWojs6)1gi%dzJvoXDQ*6iz}3VVt_6Zi;57Mje#E|SX=X}PytEFL;~ z$R*&JZcR=FUWsDFXX%Y@s~Y#Mn=FCk5YN{M-n*!QANRIYhaO1`0b|PHn`hj}$M_D) z5DW&NUG?5KPYmQWhS8B6m`(Xo{5#cd3zNjEZ&k~E=-uVspgf!-q-4;e29D?s-2 zTxwuWxA+j40?PaRQosN)3uXyCgOaVp=hZ^Q7xCr-N zcrFzAB+1VZH4HZ2?}!RFG?@jeYB8sEa^$7RAcCP*bs!{No2<9NJoBoSEErJMH7sQu zMg5Le7dYYARj!Xto!;LaSL>vD*{4e$*FrV%wWee<^rPO4x*fhhgzyq&Yzx0kBJko2 z!hHPlWGjr6tY1tpPNWBwc!=Tju#IjY&HL8k)`W864^@C2Nwi-*LBz9^q_-2&9F)g3$fs+cEj>Br!1<#W zFGs@7=L7c=qy&4+#QF)Dd#MbRr?Yy6@1?vF9yL^uKj$1VYb8~UZlCjh--_=}SWR51 z;`EUhlpE%S!}4#W^U@ByRl|xs=GY$T3KCgN4Z=8L*#JR@l^O-|CRV7S_tL?&DS=Y8 z{e0oO-v?arOYW=+t51}oT(7GBLFzD2YBqAPy5=ZaKb3!jp+73alK!tqGklRK;xxWU z`GC-QVv!nM;6|uhB2vfA>!x+8VNxMx=aemVYbe51-7d49d-uB-u^b&k=_8 zw;kw4_npyV_n&1ox^3{iWMBm(2YR|DdhhB2AnJXA+pi@tPPnDA6YF;fITAKq!lSA9 z(GvvjK{c~3y5V-b+7ZsT^*1!Tab5H=`raZe$EeOXNoV6P*#Dx1{EJ5AKSKYa@w?}L z(2^b);#a*P*NEs2@gzh|@aR-nZ#-7dm!f$9MYJ2YXC53Ba|nd_X>Nze*~9d=b23Pe z*|s)m!6M_SK_CQH0E%b>&w;4W4NIl?Kdub2g?`{@1T}_XtHOU&FgP2QAq&r!BA)!y z=JGBq?vCHu*_w{^-s!9u+PVfL2tJXTlG7vJ4W^($g--o7XM1|Mz) z`b2L}kcH>pTRQ9{|KSRsjg2_7hc_P&y0fR;6gnSn`^mEM-okvxxFK?^fgzkr?oXSp#n-XC(@)D)02A@>F;BL#=pJyMI<1S{LMh0d0QCG&(1Y82{t&%4sg zf^Loj8vSOh1QMEOl=+rm=gBZ*l?vlgMcwY&Z54h20*q(47~G1i)?qQSx_`B;N45tU zg&Wj(jT*SEE0EyJzq9JWFlLns;aLTQvM6#m-KZLaT!F)I_HsJ9um5$V6vaT^^^(9= z)&vf^c8=~P=9q(Np@HH04%Xf1_t^M#99|AU=OEk@sh!90t09p6fd{tPYPY|dltkjS zoxR*WDctk#BvtHhG1i8JAYCYNK26-iwjCoKv<_r0RXW}N?9Za z3b8g-w5H0CLc76)W2q$7gF?UoMeDT(%GSz2vaZIPS7X_O3_Du%NVa_K9yOLJI#d2= zZz%LWka?oWuPSPyK8hO~oj<$OzF9V^I>v}$Bh9|-WF??UJoV;CSaT?VuCmhnY60W= z`Nm#J?wzO_0**(9s`46% z-)@@nA|TxL_7PFeQtF+pzOQHZa^1%pa}RPYiIvw(9C#vybaHPCTDF*@aCF>gmIg@Ish+-Cyqz#{ zMP}&*rT1;b3wC~VZq9=cei172JIp0#n$U32{O<>oB4GrwRSMf7tg(!Y#QUo;&k z&~WmNsTshm3Z9pyDa$m>WdhDj_fDqagpU@{X0z)iT3%eX! zm1}t4?{pHXeylKO@)KPJ^OuClY1g4qL-JDR+y{rFOsc+@v;H>YeMOX?uR01 zhUQ^i4@Y11%-wFdYIydx!LDZpW?G*GR|h;r-Wj!xUY*_6DT+CSu_{is{B3GT;|OHi zfM}mo^1gQdrrZKDU%-0u!<#aGBMYa11a(HhkF1uNR?x)ZU&9^J)Twt|;qPiSx;ov0 z|L=pjsplmifpgP`2@r7E`X>aEalQKH0;@O)10CgmD=Pp3xe9_n(xCqvm^*YF!oX5M OAD3aEwTr3y=j1;E>{4O? delta 3790 zcmYk9c{J4D|Ho&t7unj7Z5m@&G+E+f2{ANT%ATdnFf_7MwvqRULbA=+h3pAsso|q2 z88pT|!dOa{&{(pS3FABUJ?H!T-9PRh_n!NBJ+=h{TW@FV#|4302L-CK&CT2~98 zpWIsy)3`O!zR|9bEv02g_0^pAF%WxPrq$W=Yt>DPv!?^HjO1w!)JROub)Pof$RzGd z-m~}ysN9;-_CrZtI^Nkc4$qS$jVG}D^cGbCXNpR`R|7whS(54XrPqHffv&eAU~ftc zbaBZT$=|$4`_>fO7M8A0_!=;j><-(H5fiU+6RnDk!gE6=8YKm|10xG_6Ins2 zFyfaHtqLR$!*AIy(M)~9}Yg;2p>)i6zvr4g+h0e6@}sWETF9wY1avepLMAmpg6{O9C-bH;kA=!{(Hr>=$*|A= z{HrlS>W>uqCS4n>8l+CPf7uG$jj}RY~{?Iu}2#`Jl9Pb%-$_xUx5E?H$Pm)6}%nrE;8Y;b>psvu!;b0b0r zSwAz>3cEXLRn|6u+?f9+a~W5>R>Y*);LJ6bzI=DI4aO$cS5&i06$M*0c{PonxfhFv8mP$s=CC6u=S?q0cz z8}GWWw8plpsrJC)}r)w5CRZmB!7M}<>!eL^ghi6Am@m6~bdIW{oK!xKW@#aT3 zI3AT;E5azOdgv6LIAKi=H&jniAc)uc{0-@Jy!9Qx(|$g~7Thk_+DpRdj)hnx70Y_7 zn;hdZ?Nb+tpyOsA!zoiXsIRt8&T+C5C`d6qtjQO24>Tb6m=qobUy#|HL$}s<5Q#(wBMbVL9~r zxz2{Fo2rrGu1Aj5NL%YK#p5<-K7gAY zvOUwIt4>Whx|tB;QLm!nA>FF7l%|tC=362E)TvXv5}TbHU(u7$SN;B%MZU2T>oe`_ z1ONHPd=t^LQOW~k{vw119!3G>WXWQDMRQ7cOlbYa-nbc#oI1IpVo*+9m1P$&g|-`h z?T3JG0id4q!;W+5!#!kDpQ7vdRu>7$y101u!Ky)13yMvYr<3{X5E}2<#N63QR}@XN z77=c!De%u7l9qmrq6_4_g$z9n?P~vU9QH6^{MEh<_PKl>hT-U0>PJ(%m3y7qFhR(~ z9Ra-evp;BcPZ|~>J_iH8V)1>~ET^`&m#fU-CQi@=tsXY3R3I|psor?|^)34$| z0W>I6pcM%Pw7WiH*j(~$m;yyO2z!`ibqAs>&_OLd>cv% zMxy70t)A;cMJ+~RQ9?ZjNrF^RPIqIJmL{@8a2ebWM;KTr>3H{xq}se+ih8bAdEb@T zEgjO6vz%}C@koB(i1EMW9ewFzc9ADXfRJ5+`-=5-zVv5=P}|?K+1Abm(CDl7KO!Bu zpP}3YtbJ?&jG$pDf@R~eZ^#xI<&MGEsVN$(AV%WmT67Fc9e-PW2(prVs(f_ih7XU? zLV(Ka_S&Im#L&S55H`&a%Hde|m`p%JGoZz@{$|aCutRZ)5Wtu`WMmf232>?k=a%hC z7Ts203~GcuM$5WfTE8%Y1*;l(l@HB5v}{F%YdJ}xHbkoqy8LETRH5(MU9M@ufGRTl zD-ZC!v3@S1FaTXE0h?$P!tv(Zg;gG)T7E-Hx43E{T+9%A1V0D``3T{k$`#Slo~xwY zbrQSaWCR*KcgRrp5=*iLz^Wdqia`&s27;+U7PHdTmd{KQ$i#I)N{lT_R;;A%{%)Iw zp&Zc{Z$aE`y9ic7uJrxv;8AMbE2MF8c_l3Xj0wInK?x0w7P6bs(9_#B zy5q&o;YcB>|4Qvh9a%~0jheH8n`JD?ef>?}nBrHFEnzFRL!uUpSX2u2;4s_zOen{- zoD%6_aDY!xBwX8o?Yoh6`UCdhXfsw3j-C>7so&T*k@vzRYxq+T8ir*E=5Hf+W$dW}O>SS%)muR~C=Em~#A zQ$HZ3n!e4{N3B$>-!4t)vSNi-9dDBJck_M8iChl}{U2@MFYPIN2K|?o^FJD=sgnYu zPGbTde+=Nv_3G*98!2K!uaGiK?Z@B=UCAVuF+NMIp6iWW4BRZjXE)_>UCFxfg*Y#+ zxmYxa-Q93@p))}Ja6SX^7->xWDB2iqw9RhBo9sp$sd8eg!>m-SvDI~0wcPw&==$?& zy$N$bA$i-DDJzCX%*V#0^c?JE=PnoM9*r9DLiP$ua81D#4aC#fxCMBr;-NotYzHPB z?eV3G&SZvb7X+0|UfGzpJ>FBW>nggO*{tURVPW9r5sIuo@|eaKK6-I&unk6jTL$IA zHewSGU~|aQ0@G}N@8rCP#Tk_FYxOEkC%S%MO#AAG()^^D$UTagRHELjhiqJw6jvZLC|_XYu$ zT5>Zxb9C+HBCa&QCtVJ59aXM3!1QOtZQ@VSQ)l8o$~>(#w#&4m_z@>%Yy8G=^5A^EC>boJ#TjHq;Y-cp5DniRgpitKU7Dum>((H zFRr#fu|PPE_ABQ^z!4FP`+t%KGPw|k;sVE^L5Q+p5CO&!mw3QDhhIj5UxsZed`_PF z#Mmh_157x|FN5n7cq%~nN=%t>SnCW9PUXL538AY+kj9}v{ugrjTIgHMCUGgKTDhT` zA^7&q)mZ7_0)bybeI1r%(Kf8uCEU!GJiWAd5$U-$|NHFv+cw&l*Xbd{(r#*;*>#XEmBD5w~B2>$Z4Kmk~4K{%v zhk*s0!QRA`euHyK67<2B5%pCnYb-VHvU#gn z$zI?xNa&>Tp@VmRWT80S--4g469uA!=B{d-RJr8bH@)x~rz8I61+|Ipk;vp@IXAZm zR+icRDxb!Gw+-l$tns+!I?aGcy+G?%O~-4AkZH2tMg7Z7lkUk7PT-|x^3(h18%6xqJ>@Sml#rHc5xk0Pu5K3Lx5t_gopbvfc;RE5f=C{J-zzu7U%Yk9bv4 z4@@McDjEskPl7<9*weS~gkjOQi84w@h#QLH2iB#C!qP{HP$eOWzxf7(K-LF9AYst| XEqoM>Xr-hNmM11Do#Lj;|GD`Okkd`C diff --git a/summary_results.json b/summary_results.json index 87e799175..e92a20875 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420588, + "Performance_Result": 0.420745, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73133, + "Performance_Result": 1.7395, "Performance_Units": "Samples/s" }, { @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.8508, + "Performance_Result": 20.8526, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 3131a9238039f3a6bed007db625cb5b51e6d7110 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 21 May 2025 22:08:39 +0000 Subject: [PATCH 611/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10935 -> 10935 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index a30e85aaa..4dfe1b293 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 22:07:13,240 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 22:07:13,240 submission_checker1.py:3308 INFO] --- -[2025-05-21 22:07:13,240 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-21 22:08:34,300 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-21 22:08:34,300 submission_checker1.py:3308 INFO] --- +[2025-05-21 22:08:34,300 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 22:07:13,240 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-21 22:08:34,300 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 22:07:13,240 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-21 22:08:34,300 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 22:07:13,240 submission_checker1.py:3324 INFO] --- -[2025-05-21 22:07:13,240 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 22:07:13,240 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 22:07:13,241 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 22:07:13,241 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 22:07:13,241 submission_checker1.py:3345 INFO] --- -[2025-05-21 22:07:13,241 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 22:07:14,398 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-21 22:08:34,300 submission_checker1.py:3324 INFO] --- +[2025-05-21 22:08:34,300 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-21 22:08:34,300 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-21 22:08:34,300 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-21 22:08:34,300 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-21 22:08:34,300 submission_checker1.py:3345 INFO] --- +[2025-05-21 22:08:34,300 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-21 22:08:35,336 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 98537919c..4b5d879e5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1238 +1239 diff --git a/summary.xlsx b/summary.xlsx index f47b4e76e1f21545b687bf00d024107cff97eca5..99aa99e4bed08526b4a20009ba070df485da439c 100644 GIT binary patch delta 310 zcmV-60m=TiRku~J{BszPO=p-zEX-W{b18vEwfkItcx)ahQ5w#vjYMuOj{*6bLP&VVttfN^b6JrtxA+7DT+Wb&4t>R=)K;=D-=0T6{n3>P@)q!w65+7W(d<3 zUadA@y#N;lkaC2X5*@rZ1Y^ezAgN1&Lj`fCt)xDbc8?8Z-*gW+Ptyl1!Bb9uJ;fa` zHZu{8NX}-Y`LRt8;0!|p5|np1OEP@+da%;nZ0MB>rxgBV;HGsFXZO*G+1R#i(k^;y z$e-cQ?aN!gYAjTDDh8-3I3vsg_1abizAVDw8tVKuoOmaIKc0eq<854I=mQ>w0Az?e zDIbfc=X!&xJk6JJx{ULz&U2C%uE^s4s~dUMe+J5N$iImbV90&_393J{DJBa91nwj{ Idy_aSf3*ORbN~PV delta 310 zcmV-60m=TiRku~J{*6bLP&VVs(;r^b6JrtxA+7DT+Wb&4oIY=wrK%A5r8yRh-sZL5a@b(5kvCm?2DC zc(dAo^#WWJK*|wjN_6zz5R4r=f}}194i&_`wvzf#+5%mGl+t4XjPAUA^z^ru?C-=#T$=Eba(kwb_ z$e-b_&Fg!&YAjTHDh8-3I3vsg_1abizAnPyYU=zZoOmyQKb?a<^ENEf_W_SW0Mf@D zmCwcV%XW>bJk6JJx{UK|o984KeIVIl*N(jDzX0Vh*(C5DW1l1q2DJBa91X7ur Inv*yxe_a8ZGXMYp From 00b4f4b445c9a12141f58474b98fbe900ae72053 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 08:53:10 +0000 Subject: [PATCH 612/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10935 -> 10931 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4dfe1b293..6f9457e5f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-21 22:08:34,300 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-21 22:08:34,300 submission_checker1.py:3308 INFO] --- -[2025-05-21 22:08:34,300 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 08:53:05,034 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 08:53:05,034 submission_checker1.py:3308 INFO] --- +[2025-05-22 08:53:05,034 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-21 22:08:34,300 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 08:53:05,034 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-21 22:08:34,300 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 08:53:05,034 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-21 22:08:34,300 submission_checker1.py:3324 INFO] --- -[2025-05-21 22:08:34,300 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-21 22:08:34,300 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-21 22:08:34,300 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-21 22:08:34,300 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-21 22:08:34,300 submission_checker1.py:3345 INFO] --- -[2025-05-21 22:08:34,300 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-21 22:08:35,336 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 08:53:05,034 submission_checker1.py:3324 INFO] --- +[2025-05-22 08:53:05,034 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 08:53:05,034 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 08:53:05,034 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 08:53:05,034 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 08:53:05,034 submission_checker1.py:3345 INFO] --- +[2025-05-22 08:53:05,034 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 08:53:06,229 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420745 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420846 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.72811 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4b5d879e5..39ccce404 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1239 +1240 diff --git a/summary.csv b/summary.csv index 267c14c43..186dde388 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420745,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420846,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.72811,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 667b2f324..c7bf1f3eb 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420745, + "Result":0.420846, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 99aa99e4bed08526b4a20009ba070df485da439c..8bad905b13db80d99a46989ec50ab90b52abd753 100644 GIT binary patch delta 3710 zcmYM1c{mi>8^=eob**E|)}Wa&mZ9bfMTXl{43Z@oLRUkELA2P7t&qlO26K@uTV2W+ zLdlk~j3xUvMA;cNp_2VK_1x$Aoj=cc-p}(n=RM!^Jg4SLAYbeg)hu61kf0u+ z;cT^G;$ZmojM{tT_hEA?%J||an4sF$W*Xqe+GNehLp7pT7}s zpIZsAZRdTx(~7Ex|F$@tUh-jg|0g0@|J3L2&r0PQmWt{r1tIItPr4Cl|9oE$yP)Kq z?UEijwe39hr1wQVuJ*0J7nV)vD4+U}7%|k)WjF-(;CSMl?ox|sxc*3!K%TbR;n`+-yWbCMVyp`+JpBep0*AX%~01qaEq?iSg6jd zG(aQ*?&|!Snl@w1iW!iQAB~op(!>lN{hWE1vok z8KYRiigf)4>+cDBzMFb!w}#i{aMoaz}^sx?P1m1YbQ27G1&)hyP>=VXt~uytJvN0a8;nO=`Ukusw} zeqG)r8wRdQm8>n?zY)i>{ZiQ(rl1z|4IbVZya>8GejU-x5O}cJwZ`ozWuF7@4DUHo~)G@a*@5s+BsS!oR$2etGYF%@~CQ3 z%u_Cn5xXjNyormGF-Or8Bs>!17fSm8$DdQJTc(b#80MSITk4_kheM8~op-SX=Z3s( zeRMpxG4A^z{pIhJ*5=xpl~1=nMzAY!N?oOZPacj@C%pJrq86^Pmu2E_4Qs=sHb39y zB?~bq>LN;4tM&Z?;?fh6Rex}9!`NiRR_3hp$SWBnQTxarV`4DSj~2xaQ^j6-{9L$% z+w^mX6yHQQq6b{FtZBZkhcR+}8}24>U-rwr=DG4WNV}gJb!_yBladCMY!P`eCCD_! zAeIZCu{hy@p;FpJRHYq0hCzB&qzRnmjCVD+uUS7$%xFrZ?1{*psjOe)u^I>V+W@RE zBM^_TbPYDE8l1U}jPg|SDO*RgJ>4f3Q(t}#^_YMX3ch8$+XT6|(YfUm$|cdGgfkNg zps`fAgc2kL-$~cjZ1I&mSL@3+zvs~D)p$=3AH=AHeRU1uLQRvuV>Qdsa) z?g#tV?EtO1l{3M-US#3CM7fya8-G#0MS01GF5&&dP3)>scce`0((jtf`DGRQOzLGH z05X5`mRJ3^jsvfy32X=M9$u+ z);Ff9JFA{S!^TrRTpS)uKp)(ixO4C~-gw^3xm{k4iRwz-h%iTdh5U+TX8BMElZ?1XWbUD&H|kXU#Zz z51AX`cph8A;L2nkEzAXXCIVKz03{J^@_@u&F-V8oRJCh z`lDk$7RFFugSQUzZ z71#C1FYD_3DsJ&s*?=)pwrkWjuW#ewgTWTX;`_Jy-$)zWcVn_3gK!neVF_yuBv=wX zACFDxKGG@H1;_yNVly9sMtD+iz(`#|WFPRcsbGfkZVJMvgWfpLD=?mUKbLcEAx9p0 z^^Mo=oKAaazVrH2(GPSVnDF4D2H`)Pc+YR#uay4A<^3PmN4fti@KzK@QyOkWYd0L8 zX$}1CXI1I7>s42jO+&Q}Lf(-s1Td)Y#F#J(c_cW_%i%<%BYz)Q&SVh0OA`fHQ8)$7 zo6+eBE_x`*aFFeTI_R=|P!Az9B9{}pkaL8q2iu*iD+aMVtWQ}y0ujF?=nz3^oApOS1u7qO27(|K#E>**&CR9`W#~^#|$*{^-d}%Po)JP`}7J zD^1Y8YqQIoKGh#N-c<>iB3+1PP`UkuSv=(G;T^;Yj{IX-0!x&J0}{qyO~uTe{*QYm z7N`;-e9Y##&Rer(GkZ&!^|_pug$5SViG}RkMCbUiAU{Y%`9z)1zublhW`ZBFazbid zW(MPNfJ+mF2Q(Y-lVZ^`dQ03^RN!J-MLk@cp!%19F~q?swVg%I0Qah3(};uGXp`}_ z=i!7as@*JzObYQ#q7FAlTx8Vm?ebIz)65a;LM6WRC`W;)V&)-w^RFNYACfV&zl%wm z)EieiG*Mh11njFB<|5tFlm@lWn2aamUOQ10Z+-^;(QzS{L6s5b?$_G{9B1#)7v;$R z3@n#7faR$WjY3oSs_NIG(F9u5meIu8F#=l5c#@e=0yY%_R|fv2rRiv*#p{S@fE9feFC64LN3%EQv zhoPo7n^wOA9aL;y<3-1xCa;+9Tf==y??l44ku2}~(b^$4BlplL1Qdlq-T&2g5P?5qpSwL(5!jE?yak{j|Yu_L^j2H+boJ>ZT5p!!)Ct=j29Iw zTiqb6Ep8uQf7>!R{VFYJ1mvpS9&mgQkwnT`Tg2QGn@{r9~a*?oIX~PbDji4{!>`R~!oF+-SIL#hy_8 zwGjWvKQp!EXM_KN4|#9(w1w{lgq4;LV!_qhoxXa;P+X?vm&Q6`SZr_(fx7or0csw4 zU=?Xz4RQ`%m#q|*858AeMAe;wP?o{R2tD2{C&c485l0H@;Nxu&!HMYVxgnE$b$GoH zj*{SnH1k$!jF46f0CuRi4|hiz);-JN*i(0-HwKpY=jBX=y!_&ZviKv)WR`^?#wXy8 zH+6Nr0HlS_=X&7tZOvJK%3MqNND&;!37p9D6-;+u-I9^oz}#7ay(HRcF$R1IG>_eQ zUw~|!kvm_skj&93UOPjHehKxTaaD~k6?kH9ESW@$9$UmD_f7#WZDolR1b(td2+R06 zSua!5jn z%8LF}a+2b<3HT)7EaR+t?)*j<#58->woQX&rU$ z3){MHpSMyS+Ii@|FX2y^GmikpUfqc2G3BwksYs+E01%AVxqT-D?{S-=tRY3&P?taS u^%zA0BuxQph%5X)YaRgL@*w~~0`UJ472Qv<)-dLQP?9x{^N*_iTKONGqz|@tCmK_~-+Oke8-Z*xch5g|Z@ugwQSC(|bWvs;&_RnZlS3wyzmr?n7 zbmtp!dS`rWK6ck{Eu8QC$`(n7@)I$8!aHrcW8mL8#(x+O4$79?28*kS*k zDTB*ZTj#Vtbs|hS^J;&p^PqoazRR_2ckinD)F~THmcif3v_J`(xS5d~vxjP~wX zh%4Bn6z1I7u{vk}qR&cKG&#qR7{T$o8`iuD}V8BIx?hVv9`+du|}$* zL2s^QCIhxz#8%cXUc)8Nt{hD{QgjWU9KqG8BpS5R4pH9d`YY8pD9lVu)P-4aq4yI# zE4igD!198}l)#^@Gh#uzzz>TcJt_UbuRvjyqL!DH(sg}Ne`hAUZoYshQ}@JF2E;&# zE==&2IO8k(5>HxQeD^n z35dFBRGV&Qc}KntT-iLBQDb6&u~VmYjjdPXV@_vCmaZ|-651qj_t3(NmQjUJBinbD zu{)Nvn+1eVwo;cH^nDtCj9u9?Fqryyq6BiT7eXKM`{p(IxQA&H99FIrmcZt>=eU-@+P;a<3|!x+;f> zyr0l394^1~!EB>9fqwR4v~HSXDj*!9+^Ql3CYxRC|gUjC*zP&je_djzKIyA+O6}&A!QrOf z7jH>~qt3HiQ$F7QCp!t12i*eU?Ot0hQ^lImIa5K9M~@ii@cBYAAWzj9H@yTleBSaT z$}I!mE2OMX*lblDRyvB^T*>Qh4msvhk^-K|^7ZB!g@BFmNC4u!2&|D0K>?QdEk9}R zWN;v8VwB4Ogd0@_uP7`nkxp+xO{!r_f97>F73Azl?9(}T8)0c|+oY_*c(%5ZTwgAF zfFyss2P1CQf|)qffPcc}Hk%@BGHVfT;#C@6IH3{rgFVG6T;*j%A^wbLVbK|@I zkZ=G3-_a9g3`4NeIq&gaHPz>T%qI-e#P(qDp=pyog)+lT(~!SAOC;y58k3NmE8CfZB)2yk*TG)+DrkQN`TOs_alxNubjD#=eYk z^jiAYw9KNZb1Fs=9={Yrk|{OQ=* zkF{H4K^^HXTZ3|18c)QW#9XY5AmW$Sl8~7L{L`R8K+gN;99oD~5S^n6MmFMKSMcSd zX?zzDvZtz`l2un^>b{)et&*_DZd$r-JOzf9$9qCCGa&tA)|epZ=~YN=L);~`Oi2+k zfB;pgLxNLO$r`(pmtQoHIlW8!XGCzBlLjGT5i z9?YO=hPBQ)oXnd>xbV{Uxt_-pSW#O3Zk{>f)jBf9&lYJWDX7Oze0XOKT#ExhInS8i z7?NP)EE4C+^9bpkMzA4rDZ=?r`fXuhJ4Pd76hA9R$$_Y9!My6(!)4+@u^i-3*t3bi z?Oa0AIP`5t2z584Rec`%LQ9l4;km zGP-Te+q9G$NYk7Xql^AG?-h35gZvbB-nkH<2eCkb$`R=25DU{_YsTVCMzZh!87VU; zBYsxIy?*P5bneWBoElVCjxfo5$zqK#qq%QI)ycEv%d1LnaoA&nv) z0V|Fn_g@Of+2FsIn(W*oWD1)0bIm1X&zTSov=#LFsnGpsg;UIrJFOJ+1$FFMs!#!r zsnh5lueSS_fdlq`INyJ9BkYIbUmSZ9{)dY%f@c461>Yw2^2a0Uedd<@N275X?t{w5 zF+}qPi%IP%J}abwr}Dn9goT0Ter77EjA3S)?8Ci~gaRON956&TXbms|+clJl{?lM6 zGuRuZ5~{@?R~z&hTTNr`8v1TL{T@>Ou-opHf7CtC?JM=YJEP0#5tJPzFv@Z4&;j{sjHEP9G_gZLXqsK&P~vN~ z2%|E=Gat03GiqsjyN2kE=Leg3xUw+?16W7Z z?rKkO4q6pHg2JiHMhYeWu`46OwhK`y4xCUENDc$|5}}H+P(@!pvZ7a=nNx-n8H1lv zdWo61tIiSQ&PM)nqUq90fZ6m%gwil_5LKrx4*k>8z>gz~?U(0*=_M4mP%|E!`vC2m~fb-5*wWCDyQWFj16NBYiY|MCSNLmuB} zDBK-B{WvsRnvuijh6y25&i}S@rP!9iO$&s8WLgwdW8^_ty za6k||BC_0>Ft9OP+IU2aVl;8@AXKb`Tk!yZXK7cj=t`JS~EO*86an9)l&Z6x%}$+WoK) zY^{;<@Ogs+Pxr{138Ra_Vy7u5ek`|uV~TDv?O%=i+}Me1Ucr0 zmw;!HVe+gN;>Tb(D+GM7(=*G<{GN1gZ){lTz;U4^OZ&i(SabWd6^lEcN*?XNiWf4a z&Mdf59$hF!KXD%PhfA(yOuEMuzE%D4$;Gywp=&!95p_6)`KWO5;ePx2(v(K}*j~Vv zHR)iy8{Y+}=an26#%@6L??N~L0DDdVKokG~AOmjL2L}X(oVXDXjM9m~pbK$>w|Y*B zm@^OfOFOkE{j5ZqfEe<#d#0XO+r%jl(MI`8w+i=T7H^96KN}qF4Egrqv3E>1;->(W zuf{4{Rr7(d?$G^-+~v;c-s2Ky?T%KaIop?r>owGF@cSGY!%AMOfqQiVq_{JK6Z6F6 zI?iz2cqgp*)H>dbJE*dz%rN^-h7f8d6r^W2#tVJpXnVd-?j-y#(X>k+e9Cs~5TIE> zGoV;@xSsW?=)MsK*%<#m6pSJdn#k`TGE5@;Oa~hm}iF3C|3Ma)a7Z ze^h@6!PWN9v!2UFe!}t$1od~Nh`~Tx+C;smjcaMs<)-vSK#6Qqd;h9wbv-uw!APr{# diff --git a/summary_results.json b/summary_results.json index e92a20875..12405c31b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420745, + "Performance_Result": 0.420846, "Performance_Units": "Samples/s" }, { From d44e7e3b662913dafe538ee740f8d882b98a72ee Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 09:47:00 +0000 Subject: [PATCH 613/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10931 -> 10934 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6f9457e5f..0c85a5878 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 08:53:05,034 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 08:53:05,034 submission_checker1.py:3308 INFO] --- -[2025-05-22 08:53:05,034 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 09:46:55,069 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 09:46:55,069 submission_checker1.py:3308 INFO] --- +[2025-05-22 09:46:55,069 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 08:53:05,034 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 09:46:55,069 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 08:53:05,034 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 09:46:55,069 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 08:53:05,034 submission_checker1.py:3324 INFO] --- -[2025-05-22 08:53:05,034 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 08:53:05,034 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 08:53:05,034 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 08:53:05,034 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 08:53:05,034 submission_checker1.py:3345 INFO] --- -[2025-05-22 08:53:05,034 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 08:53:06,229 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 09:46:55,069 submission_checker1.py:3324 INFO] --- +[2025-05-22 09:46:55,069 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 09:46:55,069 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 09:46:55,069 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 09:46:55,069 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 09:46:55,069 submission_checker1.py:3345 INFO] --- +[2025-05-22 09:46:55,069 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 09:46:56,115 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420846 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.72811 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07116 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.045 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 39ccce404..183f97e61 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1240 +1241 diff --git a/summary.csv b/summary.csv index 186dde388..e47443184 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420846,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.72811,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.07116,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.045,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c7bf1f3eb..1d44fb28c 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.72811, + "Result":8.07116, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 8bad905b13db80d99a46989ec50ab90b52abd753..49ac032d3d2c6fe69daf71fe83cee2b53a9f4d94 100644 GIT binary patch delta 3638 zcmZ9PdpHwpAICRma-34m@@xzftu;>)6HknmLnR{=6LXeBIkYKy9CF;)G;%&9Qbuw< zM3(VBV_TT8KR-{I2lEq<_`Qem3$=s$ZUyN@ZLZ3{FZ3ZivKpU=U()B z@I-ThYGYN}r#U>KNBg-&DE4rjUG$=F$=~-a5k`xjoS--SD$t=JZ(Wk>EB%oo9Gy>K zE3as&FcJ1a@T9IrST-T_BCW8OoZkvvq9B%((LXY~vQa(kl`W^)b*LR@P-yR@ z6IU31m1Gd-J1v};VI$#f^lS0H%*pLeC52y7^WWgCAG!)(rtYy^1D=rx@N3i~q)I-K zZMS;QWoSQqswyMFBHu9BK5(i=UWCuzKQ|-cP`Q`H*eJ~ysf)Z>vNigy+d5U{iP<78w!{PB7)hPI*u)AzqN{Ll+yTM47J&F zpZk&@yOW;$;8jV|i?q7H!v5P079U)8FT}>|AEX+0@P-*GbB#NpJ3c2gfF+2n!x|t7 zy+#!Yra-8X^4!n^Q{yS3+o{31)hl4-^6#8mrIQc||DYLCOikU%-$%aSIVOK*m!h9S z;6L5`a#zCr?^>L7e)h9TNupxf32j`q1G$n4Q4o0Z{oI1;oF_cWwnx?|WcjdqUw-u} z(iryy6dQmY`Ji08+yFTsdZg4oOlNH_{5;YkG{#Q{_7iR^L=Um`0?&2(oK&%p$&NF%nx!v#=O2cA@;F&s@+%~; zt~|R7Pl!QJt_^=`Em~eX^a>ux))XbJ{MZgZ+q*SC(yANu#iFPkGKkq;UQ+4{&G(;R zue`hjmi^@$1XDAp?qrQnjx?T+{l~-6BZxeI3_<)H+_>v6cenA6@`YNh=iu|s-M);f zs_~jLY?v(Ba8yv46`A*E6a3*VS=oLQla(M9);6}}LHjaH-z!J2m7 z+Xg$Z`QEyPb&##DSHqz!I^_v8Knlx#J3M=ea{S?p(+_b;`0M z6m^!E3?<1_;F}@CL(bTftsxr~)-bFI&@oO<7nRuNB{x=sR$Vy`^; z2qzhnthNftw=%?CQhq7jMf@E2JXO;wVW+GcH{RTzTV~$9;F;))9kXL2>c;I_2q@ND zMJbf%+(U{1co`r5sMyZ12|_qec~|z9i@Gr?Sf_X;c9Md0z|lCsj0Ko=(mB$B~{=VNE2=LBSvESuzGx4||rCqxlLv$dMP~o~J#BUWvwC#vddL=V4p- z?@FV{wK|(p#f*d8OPneXOwbpWh3%B#LVxO8{>Sy^TTc>ez8qV*ye&X{-nPpMh3sKatWg&>Pn1F)poQ8X^OTZm7*2axvZEl?c;5?=?tMBq_EdP?rG zwD*F<#T+3_YCd$mG&s- zLl-->&Ne_T{nK}qo=7kMCpYh!6ECH`D=Y67wDGM+y;_{S8|#{LQ;-ZLJ+uZWKerS@ zfg`Wlh)3A+rQ@6vC|7AbN=PAX{1y?hYcK#B#GAQEb=OIi2|q9U=jbc#AX^@MBX>Z@%$(QvG5N zV^kXzNrHttM(~fWyc^N35#}T&grc$@9R#qYl_`pha{Pf4|MB^EX0*7uIYjoCdsY*~ zdwkDSjnO;QFV{~~2n`fbZv*J;klHM0UBbeQ_*;EWTa$Z)hQxXRR515#TBhPC&<}4- zKtMT9NHH8peaZ=hb|V<`$v@)um0R1Vs58cCuU%8pMCp@#Nu#e@KJP7-Il#Yw;8ya? zf-NI>?hS$|AEPXX{Z=>`4M!C*cJImqo~z87L~e3xN=Qc~3$}@=wMWACOZAC<$*-}y zEPdSf5@tZ*`-JSdMO&>v^Xd8#M?u)gN(}f3V2q1fDPqW|rwT;g6(tK7{h4+y)nDD`?!cv%lv77l;a?HX_BX~Ed zQ%vwQ?L@pH1xQ4o`34Yz24&o+;e~zo%|{$-P(=oLW};JP%Q@5UP5Fi8M}jb*eG-3 zVzs}p&^0=IN+X|kClg*Z)#*h;w0dS#sc&K#yB z8w7H)LxJ;QCBPJMI@XZeS^5qcw;*|K5(lghFL&8Ovk$F3G0qR7I)Bq>^)LXeY z&xZv=*GTXVdK_=nyNE|!TWFau&9%}fh$)J&(pBhXj#7WvLJAj-O^qOwSpWGPFI%OOtc9Zy zdbQc|nD>#1{cCOF07R>M;+i1IowPf2bXwXK4UCBrX9#g|B7t=EH4u#96PM*pMo6QO zr%(f+3Ve4``h{rv8BK1m#wpaVELoH%xLx&h?VGuz^>513HJc{KChQfrkg>};OuU{# zaRPzlRDN@@DCj5S0d;k=ebs1AVUG2X5vZ+xnD@xHfw*T{?}cc@7ZA(!Vf`2N4jkYZYYcfZi zb@u`!j&_ZX_%}AAn|6l#MGmgd*Q972=+oZ#TrgwuW{ffh%N|H(?=0!$<0npawTI{0 z^!Or|V5MTy_JkLZ%EengmCUmYI`00_7w;+Aj_tX6_VO2PG=1MU&YaEyw3C|wmHblU zqQn%foK6)W0AR@l07w7;0GzLj<$Yhj04*2a`>tmmdV9Sf^ZuU5IMmtR zrylA*DjYk-p!c)Xui3xOxy0B&8XKE{-6V(Dxja&hPww_mBI>z2|=J_dd__ea^X6*DJ2m{CSVXJXN523jhGS zcSzy9m+-vGJKO1iTkDh6!w;2-?qR8;o%<_spElI*jLGb8gus0!TQ*yu8K<T3dm1R0Ee#Id6h<9=5IB6&2b3rIHYT{71C$5n4lPe1&v~DPAG^ z*$uEla41?~Mr7-OF{5ZTCBEkH!%APW|rS6iTDL-{9c+iVay6? zCnG=UerZe>(th-F=6$vc1IUUnRB)+hb8T169zo4?g&jZMZnt2JV0p{QS`Rip?0awP z3}$%6w;xdwU8Fxxyy)K(Vus%xQnG5n++flY+dCA zu%5p&U>RnjxiYnSI zKXsKK4}Ms9$!pw~Sy^d2d~t2nuso^A+F;#?RskDN`q5d{l2UO}Au&3iO=HBYh$}X- zaX?dJdc25hLfk@0FF^4H#iDuY)lj7hT?V_mS9AF zEJuF?evC~EUz!`zWY`w!nVdxH1ypJI_{EarlN5fe-i5MK;X9eL_QS7%NTT|&^&?wy61rhaSNc4uCKLPo6aC`Ag?Jr=AivHxiOnn#O3yu&q&GR5{!POG zfwW|2;JuEg=H9Od#d=F`5wBNubu$JRS{7|Sg(>yOpTMs!WxTJxd($F6A)_(f-G#dk zENMW(0?=K;h}V4Rey)7nm%Ivm{W=GFOtjAufW{0fxvtSqv&O5ynY&1Wo2+N)28!k8 zGO?KU>T9U$1iPa_ZV2xnS>>vH@>OUanw<;o*ebTsS6X*pwTp# zh^$hIR|j2Pwb@(DsK$Hz5UM#ux>6ZfTHarUlc?^d&D;AhJN57n8!8UXKT9w8$n}AJ z>wbx%yW}##T<)p-xe1cd#kc+@Z4un1Lce(Vh8tK_83%|PSdjxwC4H!co|Bqchk(pK zz@-%*YE;2K-MfNu(Gn_AlZO@i%TR8N!B01ZH?kv2HP7YHbP%I(P>bkI=u#n%*bxq8`D zLxk!R9W^d@co0u7f~ix=V>~do3Kb5{Rmv{_WPVypflq3hj?{P$zgQ2~Kx_}OEbCm7 z1z{!T&#iRalD{`XJmj;j!%QG)K0-`6>WLNnDKdvGHi;hbTPqV)BAy>(z zz~ug>Oh;`ce-+wgudxbK@nJLFYgt(qw>`M|#`t(rR`Q_4PC|iKP?C)ChV`i$z(R>Zl{~rq z;6o@4|663E2EqVCNs}iRdNTZ{Wpb~n?JNKD=9yq``mvz$${4u-ZSwext;e9LE{@}| zITR+B%Atn2T9##Wv`kXEdg7Ko7)hhv%!`_BDB>X;nJ z)_TE?mm0fTlH^5H@nN%E>A45JI8OGQ0@rS!zxupqXWQ72))DbNa_iOhEOwiC<6iV@ zvl?SOgES??jP3s=FF8k#-fo6BDlvbGnvs4qUjV3~hm*NkJa!6jUb~g6Y>PRUSG;O+ z898T32($y1#EsF8#+v%%4p?%+pIj%GSlNTEj zu{ezci=pP@uqj=~I)pj_8DK7K<|EKBM+#0f62Xh?6@6^Tn<2NC0@rP)H_UV4dB!sD z=d2nnWJ@7$)VlA@X|#pr*>6l0{Xq4C@eeNl7bo2P2lv~ve{i|~!}XHx|MowcWNS#n z4Xf>i!!j*I|L8+kxbL~w6=l(kTL+ZhCtvboP~HnMp=MG@aICw{nMhmiUOc;74S@G( zf&esWwV-J`iX88tX)G2Fy7r(BvLqGMof(m{8ncjnjI9p3H&<5-8h5ogXZ8q0{1&f4 z1f^P%ts_l%$4YbeX#6y0tZcCNaN!=!5^#mIUeNTDO}k?CkbZU8)wRY~RD<_NcV>D{ zS=^?|<#D|ec=bTbJx=B{$xMulS4SC)wY~_yuFy5E1WX}bNYG#xu_bws@CVLO&QCLi zZ}2_yrW0&=2*pfJdeiSB=H~?JSO;R#CpE`pIVXzigG76)huFDxN|zl_zhE$ygnMI0 zk-7a<^mzNF90mm_%znIw0XWv$rk7yL{TwWr+mGca7bKwYq@$Jf>rp5gVbql?QMRMD zkiWdth2sQ}wrZgY$*}4@=&*eICKoF1eCo34fd$O7Bp?#D ziyZg3AB9&Bu^hgKN--ueC#ANFF zBIcgZe4@7|cTOBjsLiu$p#ZCoYAC(jgUs?h|1HZrwJEv86!xqDNm)-uZsFGi@n6D_ z`;)!Mb4k);{i_9I%QgjbPBdJWOn0c(dWdi2UzzGsv%ybbgC0BGt>GH`eqkkL1K6tF z4sT6;2rkq7TSFZ&EG9S`PuYJf4KWQpL`Rxdf$W1fB(9oq9U-Z!NL45UYXy%s7}uRs zA}xSHc#q~Y!m+Cn#|r9TW35WO6H!%jg9dpjuzEfmDc%lg;vw4*aYETov|Xibs4G&Z z?s@jAHDxbqv;PbCyrdzYyHD&Ol{=yoxWuode+FiI8-Z{KBrkk5>W0m?Hc|fq-c0#K z;_c7&pUCy*CA+Nb0L3>k0bii6h*oNhes3DybuZ2vAQ7wUax{t$cyfx%N=KwmMCE5} zWy33Jj_6xo459>2En<>-rT|xVr~(E4pRM8iKp#7cnYpQu^OC>b_3Zjrw(gxjuzU9| zxup2osznRO>(`&sbex-Ns#+-*b#JzX+qR08DaAjNU)GvRN|f0(0G|cuG4xb&<~KV% z6r#qX#jm6Ot%TZOFO{UGE{7pE{ypTkZj%oZvmh9KV_Ec8Kp Date: Thu, 22 May 2025 11:21:37 +0000 Subject: [PATCH 614/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10934 -> 10930 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 0c85a5878..ae51d4523 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 09:46:55,069 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 09:46:55,069 submission_checker1.py:3308 INFO] --- -[2025-05-22 09:46:55,069 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 11:21:31,949 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 11:21:31,949 submission_checker1.py:3308 INFO] --- +[2025-05-22 11:21:31,949 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 09:46:55,069 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 11:21:31,949 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 09:46:55,069 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 11:21:31,949 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 09:46:55,069 submission_checker1.py:3324 INFO] --- -[2025-05-22 09:46:55,069 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 09:46:55,069 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 09:46:55,069 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 09:46:55,069 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 09:46:55,069 submission_checker1.py:3345 INFO] --- -[2025-05-22 09:46:55,069 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 09:46:56,115 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 11:21:31,949 submission_checker1.py:3324 INFO] --- +[2025-05-22 11:21:31,949 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 11:21:31,949 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 11:21:31,949 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 11:21:31,949 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 11:21:31,949 submission_checker1.py:3345 INFO] --- +[2025-05-22 11:21:31,949 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 11:21:33,262 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,9 +36,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420846 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.07116 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.48247 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.045 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2638 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8526 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9465 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1717 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 183f97e61..bd9b50707 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1241 +1242 diff --git a/summary.csv b/summary.csv index e47443184..3687f2819 100644 --- a/summary.csv +++ b/summary.csv @@ -14,9 +14,9 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420846,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.07116,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.48247,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.045,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2638,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8526,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9465,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1717,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1d44fb28c..5cae0750a 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.07116, + "Result":7.48247, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.045, + "Result":23.2638, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8526, + "Result":20.9465, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 49ac032d3d2c6fe69daf71fe83cee2b53a9f4d94..61a623b81c94b1dee4148213206b70667f1a84c0 100644 GIT binary patch delta 3866 zcmY+HX*iVq`^IM|Te1w1bu43022Ep$Va!9eF}9E)5~hr;#S&R=MT|Uzv5ssZiHfm| zBobjr_AR0iDH6hi!EZco{?Gq@alg3lXs>9I=bcGAMEN+m zbk9?rIwh$!-5eGwfRae4(USVp&Oqt=N5+fxT(;c5{YN?bpsGG`wk9P=-9=LC`e0!2 zn7`T%eYKeGrBEG0)k9j$Q(UPLy3HiDtLybzfa3{#yPqGyHEcivpYv0iZ^3@YG`7_S z6PfZSlM+IGp_|q^A~H+889s-j7>&8A~hUOQS^S}VV-b#U6RbX@5->e^~!qqWxJPnO>U@_g}`hev2@a zcQ%RK_+`0NwCOUn3csmmcV-B9n|Wt{t zJi378d=cJqrA<_HQ&_&vapf39@F$DgcIUFY5n<;3?;rYx`{%>)u>wA!9!e{dnBZo^ zTt+O}!dmX8YC?k}Z0ivnD87t9it?X@<~8mgynmV5pYv<{Vfk#IoG$<8zXvA1bvR(M zYZPafnkh>aln9N4h1Q?_fhT*bg`C54+Y|oWsbXr2A=X-ZXLdgN5taj`JIXyAbTRna z-l9SL9@^D2F6(?)0QKD1X87m)9=@IGZ&+k*WS&|nDfp4jSn-QpfDx$OK-!;_Y$IJq zxuO6aZ+=GXp0gCLE9v=7&D!Au^ZB6Kyi6x3(kZEqtu#hAKVU)r_RRZDN%^~3?aIbc zx{#E~6!Kh>QCs;+`t`R-X~(>$Pj=}#zL;|ebf^$LkTh{zA)gNYg@csE`=+Lnu&9}=3FDSoavV!d+iKhBy#jTi$)e5XBET3sicxU z1?9USYCRVV!N#^~0B@Mb)0G&u^t0mvRf2P&b|OJ$DNM#J;;Q8KI?hq;2E`Rako-HW zTw0%VimsnCQq{;K;2L|BE83x8xpGcAhNc};9*ivv@z1AN$hf{Lg|SQ0{yvgIgvG;( z=AHwTixUDKDVs7^P#b;Y`t|Pxo#b>Lj5YTbR^Ynk^g8GjK8Ssd1|=0YOhNbiYTch*BqPokFBWMr}nUTjq+UkaP4R{gHvL0v&sz`RBgHSC`|LOO+O|Fyxm~m<9;i(HZ2V8Js$ikeS}%YW7zpK}4kTp_@o> z*CPwR+99F73Xeqlx(_{VL-X?ir+>dm&6hm^tSn}~qX%5FdK{nqHr>meqaP+_K=B0O zKM0^|x$y(+k4fJPt0R7c@#Bzw8xWo_qCmXioF3%o48iX~4XTqg99|7JWRGoL1I$N* z_3YolfucXz@3x>WmuV~_#n+u<946*Q@r=T+{Zgfoa>ihKGC8r*Iuo4Gu{4CBltQv^ z7ftnatDlfzz2Bz;CCy=qWlc~;)j%Cdl-^VM&Do!^Ed73Z9C-5kcz{;2S1SE;L_GYe z06rCrGUMoLF^XNcQ+LC((wZt{y853P>;@tBWWTq znBMcYKXSZ|F!NX`?bxI>OQB5ie-y&|Kshhs@$;jg|EDnRKMETT)HU;O6+ib#Os~V! z>-`!KYJuH-DpkS4zz>YWKNIsOzz6(_%!3_J!FO^fm1A$5HhJ?6@sxl0|~*U+N{xwTuGFpZN@3hvhF_$ zim+_=GRXJ~`+WW9(t=3Y=yqB%0v=lEQwKz4K&vnOxYb(XhJ2G#!AINXcDRq&+prs< zHJnm(bU{Vgb<49b9S$0f2kQYZ75qxl{0?HZPHAHKW3fVvxE*?q5o4{TMo?3w+q zXC6}Yv|PspM^wK;jzX*+U|89&va;{ynPFw037f^K;HUV1f+|rYjCc*yE51|aX_X<% z)Vu8;8#oeOMl1FJnT+X4kC()Fdvvf%O=M#RpI@_YbD(*>QA8VM$dpB1}+9kH;YyX&_wu1!Y!{m2p%-TqDMCeerBSwiu(+39aM zp?!av@kwD2AuIG*&`Re};Q~wH2cC~Ch1oE6z>}H=9RVkkkddg9m_A4fmMhzhnT$kt z(lGPvoa5BQ{1wCbyi=Hq4NsYKTOM!v5TWa~VG3TU&8_mehu*ETyt{gIf#u!daO1mJ z_4c8}FyepzTdz{rU#z|M^@&-H%oxf?GqOpfM=_gLt3n%iLa>Wmn*cDDj_$LIkXe(W zb2GTvIL|bNbSUI6)M+JFUwWnT7O;dlaz;Hd`=?)qX@F}}m3Ykkj$gUeg%X(f`fAU+ z-Ysl^Rbu{W7gXe$%OF4QdQXNDM*N%L#sx3aBcQbbMs%utt*>+($fG`gZi0_O+0RwNWX?_gC=ss ztG$P&+GL;rQ?Bs}xBW`qN=BoU+o0!urA-u`@=p<#c}Nr*JF*&=_O+dGp4qRFfSHc# zi5LcIhJdt{bY?v%^E*cK4yQ2-+Bk&66RPa!sKKgvO?amo#<&iAn| z2dq+`aEt364$Gvr8JDJISpJ{<=5AJ}q}Xd(zV-qWB}&)qb@ zw#JQQ4S>X&-7f>h;PC=5mQ}MdW^qQctD%OMk9J~z=i!?Laci9qDRx$CfH%RpWZkF# zpx@y%Zy*Ctm{7(-@Q}+phdC>QQUv4(SdhDzL28*sR51hOUIa>ovv+c^b2-Ur;F2#!E@F#ydn2 zsMc(6^^#bZ=!7q?KL2Qb(r8wX2_rl$lJGDOKu_PA)Xj!PLoZUCEKLB_Sq~`i{osq# ze&1H)&Z|PeTT+WwGHp^l{M?w)>^7)#j=JGLvp&VVigH1H=bhuo~ z@ZCtC@C~;ZKxIz?wo zk-D#+=B)%~?%m_>qH?4>HWE8|r39J7@#dw-Hhb4!bH-~Q#l8BZ*{Jl}wKpsr`Cm@y z{IC zNv6v1;@#EHpky{fPH_v~Ea4;hZv-3pv?Msby(_by)U7A!#Z$sWhdh^do4%K~P}hW* zufjtm9D35vJu(M*`zUAqep|HRm#CHySf3PEZ=vXVI{D#l@pOo;DsuDg2V0P4yPh(1 zcyQfY{;reL0KxU+E%S>p>gJB;oB8Bl?(`Z&NUmL<;Z~n-+CQ=ktyY3}vHkbD%2aXy zb5brSqroYZWM%B}NJ$VV#8)jKFw~bAK)IqUOj%PF2mhuVKLwZg*J&^aWX%Qw34;Fb XiMrcQu{@;@hEWnv$#aY-9{%|si90~> delta 3872 zcmZ9Pc{G%5AIE3xWF4jK^32#p(=?tCQ#@s8va}h}!icf&Wt&^*v1FZLMnlM&NDRro zL^LC0dF_$0h3pk#ct?M|=e_^9&bhC1uFw6ue&6r!d(KU}QFWuthfiQ(Dv;$V1Ojn+ zlS23|0g$U3tLotPwefd@vC8qzV-W*)R-6pEC=VZZ&V8fwki4f)Wftb*g2u0F7LP<1 zvT)ke2^pQqrr zMw{Z58!A&jG4X^>?Pume*!|_bg6G|H-kxQE1~qPMlve*MPlt-UbzY{s_(!r}R4!pp zaY0LI9bxMaPwZ&W$RY&kQS-YH>2yfQuBJoiF?t>1XHtPqd4_lxxSeZ?*!eYH4!pf=)0(Z=9ghE@7vS-wnX zzlS#vBH6J-{pB(+D|q(@ZGzsbzcY}gb=kzJfthI;WiwC7zgxL~6LH+v^^NqYRcWam z-Pq!X5AEgOuEr(|#i(O9t|IFxLh@CB)aXtKlGGPsB{g?%xqAvCo}gZUR-Evdk&W4| zOWuM)zF#s(GO3RFt}!&%`St}TjQo`UbWibR@l`oYJ%CtmNg2i#wcc7pHAtv`Fjl$w z+nM<#H)bm>>EWxQ#OJBCe)&DO>&-tnY@dsX-esj2w{ZtwP~;fCi*6fXv2=xfs(1`0=IuX8jIORdez@fcWv=sPbjW+j z`F&;Z%3KQZzHffo{^`#~1+nspyJ+K*E!d?Lm>l1ml`}KSOjmfMb*HpZ;QW5ouI%C! zq%kfW661><_@G!bUk_seF3B~I(stHnejaEM8saAWdI-1WqWbpq{mw8vj?3G!aDAC^ zn}G5TR{6k1q9CR-0X7Tf`=|_Q|T#gqO{R;G} zEzRn{6Qa>$OZ}f(3g(yg-2y&mX$lh;er$%E?%J3hXwmimVqVY+^kO#W=M=hwa=l0Q z7G9o*O8@fo*HAUAYTp?kAE-YY`j5*U7ypmW=SBrGAS)r=&lAUEc*lI>xa_40Pk>iU zjD0RE67?WriH)}!!FpDwUWPiTtlq03+V+kFG;Djz+;8}+bf!k@8T70@!;@ZFIb3~e zPwH8HO~cC1r-caK9pFsbja$4r=aMV5u<+H%JCh6KA)4^-Bl*t-GzQC41T-4A-dX#F zlM6K|xhK4iavEFehh;iIlblY)fnuSGmQb-6#Tb*LoSdUwvW~j0AkrRQ4!j;IM7q5A{Ht} zS3DGt`sN~MT;3f6p0N(V$8l7A`ONBsOeGG5&xW}~PZJKWUsqQs0bM z8UJ8Z3JzpR1RI}@sGC(X7K5Ev$oW@J*R-(%@Z!-Fll?%6X3$QlN7#`f?~RSyG#e<^ zzIBp_b1BZx1mR^}OK5?m$_0a3M(;Q#)dP`5+jSN_M;pthgCLksT&2e1XQGwpAS#s8 z+5X?`B8)RpP84G#^1Lfv#F;^kPT+86vsELGTVsc+V7&teV8oKOL+a4&A@MQ&I>66; zNPP^sV_U!`iZA>ovWb~E@iQOU#B?a98*wWz88gX4 zR?;K=#C!Mt{mbP=nSNET9pY1~Fu}oNg0_vb8z`SK#N2d0577t=h}G@j%LP}!3akzC1t7#UEA*7zax^3?T}J;Pw!r-lKh)( z7^GRr(I?c$(x%AM;#}-975hQZX^+d9dvZ8 zcDLtyhuM;EpSzY&>0eUbSGC~bneNef{OwkP`pu#Z;emLu}#wHU#pA8Vpk;|pyhq6j^*Z{wsSt|ENC72d3rY?Fi`Ewu)rkeJ#;BD`C&(%9{&s&A$dmO^f+ zLXQLT!$?<&;6@^bmG{?jrj#MY#dgQ?j@bg-{|-0IIY*m`UWme7#IuqFbFeMD_a#xJ z8l5$XLOSckd3Keo%jk3Sf;Nh9fxitb{^NL~%$2w^U5c%k-{gDtetRcKWov*{V^z12 zIQ=`}?uM(Jn#K!AKE!;{5GrnZ5QR%(00KOsouJfDWneW7M06Sa5`jkv=qotK46Cda z3t0@p9HU_iQsv`1(l3vi9iXjOiiG^~-`i;{EAYuCwxs5iq9+rP|%1g zH$}s&dD3w9@#HI1E(N53Hhu#@Y#V-t^x{pOBp9_Ng3n669(<#+vSgtL!roP6?wdJ2 zw)?#Soj#101F-8k#!U296o9+9u%9Gog>7Nom&_)fJB!Mvv+n1DQk8xlS}8Sq|1+M0 zy(Yo`ZlaY=?bW^w15{IY7g^uzRy*jc(r>>v#e#V1m=Ar&lE4)m3%=oi^CzrL)7_@UDPL8 zZ~oat=eTJ;EvdwNNE?oN-~nZgID^^&bGMvP6TA>%<)A+I`T) z=oxC=Nny`ntRW@Hc29NUY%gJ?vjR4ecLD)S*xvcjiZJ(qlSzv(@c+1*ssKRsTi$gd7>qG05IYEiUTD1 zl{-!T9*G299m_i?vxH_82_#p;qmj6clK|6yAFVP+F?VWuSRHeHY! zM&dI&wAETT*n(&Ts4NlTZM0bK%C}*m9UEw=V9llCNSFzVu+Wk3W`ozbd!_ePQhW<)b-i7dXSo(U@z5nGfeu{PB`9IQ6}C`QV2VfMVT`P1GFmB`62g%%eJqzKf|;bqgVNf&cyA$gOieuXmE6-C|!Vq9c9ETufb3Z zkEk?v5<(J%Jb^mwBqK5HTomn;CMQ(=1nO6&G|Cm)s(iBM4Koo~{;nuly>|J?sIB}4 zGG<<99j`A}7*8PHExj>Y;Qy2Ukg~Yex@g3dW9~en`)RA~=Y)FJ|2C;}Lp0#?fKmgE z9zA`Qo$WtPtf^Uh86BEd#dftMkU6_$Q9k^@VJ)!wcUpbBn}O+E?)OfYA57oh64MiO zsN`OC#(=%4Ws%l8541TXQuC{86My~4r zGZV~&!j4&^)xe7}rY1)%y(_-EyNDfpMCOGCXXIgQi_RcEZ>=9qk*1$ z?Fq{$D+*VX0Q06TI#k6yLdbEl5}Z-EAY-RS96YJFDXmEkeY>y!+! z2aki?(MyoK5>sB9{)A&!>yI^O314Vh54er{&MEfn$gXE*-O6ZW{DWU6KWRw2cx{lzn?e7N~v8LnP& diff --git a/summary_results.json b/summary_results.json index f9a5997ce..64a84dae6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 8.07116, + "Performance_Result": 7.48247, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.045, + "Performance_Result": 23.2638, "Performance_Units": "Samples/s" }, { @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.8526, + "Performance_Result": 20.9465, "Performance_Units": "Samples/s" } ] \ No newline at end of file From ac3fbac34ffd1d31a7c942fb67557e725d662cd3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 11:23:07 +0000 Subject: [PATCH 615/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10930 -> 10930 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ae51d4523..93898f960 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 11:21:31,949 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 11:21:31,949 submission_checker1.py:3308 INFO] --- -[2025-05-22 11:21:31,949 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 11:23:02,619 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 11:23:02,619 submission_checker1.py:3308 INFO] --- +[2025-05-22 11:23:02,619 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 11:21:31,949 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 11:23:02,619 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 11:21:31,949 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 11:23:02,620 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 11:21:31,949 submission_checker1.py:3324 INFO] --- -[2025-05-22 11:21:31,949 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 11:21:31,949 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 11:21:31,949 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 11:21:31,949 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 11:21:31,949 submission_checker1.py:3345 INFO] --- -[2025-05-22 11:21:31,949 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 11:21:33,262 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 11:23:02,620 submission_checker1.py:3324 INFO] --- +[2025-05-22 11:23:02,620 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 11:23:02,620 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 11:23:02,620 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 11:23:02,620 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 11:23:02,620 submission_checker1.py:3345 INFO] --- +[2025-05-22 11:23:02,620 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 11:23:03,621 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index bd9b50707..cd2bf3dd8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1242 +1243 diff --git a/summary.xlsx b/summary.xlsx index 61a623b81c94b1dee4148213206b70667f1a84c0..846a39fb3dfce91afac215f67625a495b5a1766a 100644 GIT binary patch delta 292 zcmV+<0o(qvRkBsE-y{U*X0*GL<0LBqTaynZ7=LBQaT4&U$JNY!JG-b_9#otBg7ebo z8WmZN642ZTsdqJc->lLHl=#53V6D+mqa%2 zW6$@Xn9rg^M{#SMVj-4phb`mZYzLU+`8`$;m|%h79+=vRh(@H~6Vm>;b^r*Dp#=px zcz;}E1-^I#IOS&>2Ib1BWH?%wwVuVveeiNJc3qcsp5mUBkMikxvqBBY$s)}cDIuGp qproXE`PGlS>AwKgION~NGvJi__7gOJP_rc_3Iqh_X0*GLGb(>3HIs+{ delta 292 zcmV+<0o(qvRkBsE-y{TkFITpc<0LBqOOp>J7=INzaT3U>$JNY!JG-c!?{t^_f)C1? z29csf85m)uGMfgyuNL_O%0lEu^3ED)&>jMsw|BL0Ojr-E-a7D6LC6xw1SVXAwlO+} zaS&V3JQT@clDM{B^O#D%!H$b>z5z_j@*Zo5T=K~A0hqgqh>l1pCZzlE-2jjRLkAj6 z41bsu1fRVTybd=T2Ia!3RopwcX+4RPdl%GX?EAjxtHGM`6a2Y+c^g*El{uVB0BUO~ znDD@(^=*wWi*&e-hqz29UMrA~`=sA^J1#Qzfy`0>HpV?EAJx(pg^4a!OZ( qFiKcOzYZg>`p-ap9P)4C2{7iq{siqGP_rc_3IqguFITpcGb(>=c!=Ks From 8e6e951146b08871f43a894ccecc0586a73fb5a1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 11:33:38 +0000 Subject: [PATCH 616/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10930 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 93898f960..5e5a06156 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 11:23:02,619 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 11:23:02,619 submission_checker1.py:3308 INFO] --- -[2025-05-22 11:23:02,619 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 11:33:33,539 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 11:33:33,539 submission_checker1.py:3308 INFO] --- +[2025-05-22 11:33:33,539 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 11:23:02,619 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 11:33:33,539 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 11:23:02,620 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 11:33:33,539 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 11:23:02,620 submission_checker1.py:3324 INFO] --- -[2025-05-22 11:23:02,620 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 11:23:02,620 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 11:23:02,620 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 11:23:02,620 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 11:23:02,620 submission_checker1.py:3345 INFO] --- -[2025-05-22 11:23:02,620 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 11:23:03,621 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 11:33:33,539 submission_checker1.py:3324 INFO] --- +[2025-05-22 11:33:33,539 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 11:33:33,539 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 11:33:33,539 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 11:33:33,539 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 11:33:33,539 submission_checker1.py:3345 INFO] --- +[2025-05-22 11:33:33,539 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 11:33:34,660 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,13 +28,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7395 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73636 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.1249 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420846 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419825 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.48247 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index cd2bf3dd8..afe70ec06 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1243 +1244 diff --git a/summary.csv b/summary.csv index 3687f2819..7dbf19e92 100644 --- a/summary.csv +++ b/summary.csv @@ -6,13 +6,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7395,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73636,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.1249,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420846,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419825,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.48247,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5cae0750a..32a231dc1 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7395, + "Result":1.73636, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420846, + "Result":0.419825, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 846a39fb3dfce91afac215f67625a495b5a1766a..b3c0a03fd9a7ab5f522445626992a59778b94919 100644 GIT binary patch delta 3899 zcmZ9Pc{J4R`^RSzA{r)-l83Pjk*zc#OJiS#$(Ah!F_y``MTyUlLK0@isIg`&d8imm zJX*+z8QDogTC7RK6S5oM>G_`DIp6d9{Bi$rpL4&?^?qOP`?}AE=JedDHu8{wQfJz^ z2w@Otml%IY6G*wB-=vp5eAD}3^EE+038 zAKIOr0W6BAI$?|uvGUBCTW*mbU!aG#uCx{kg;H@y0N1%1NS@TZdu%w7lF-_WJGp8$ z0(*M5g4oog`rLq>70??4NzFKwK&KAIiM&kX@4+Fg#|l+Tt&qJH#UJm_TdmFn>hy*7 zj19Y*Qm5z6#>8#6vJn#}EvoB~_o%j3Ul+|^k+LFoEjYdTUn7Q@)R6919_VSG!QFd|T{kF`YkwXbmoWffcbOjK0oX&}k+A&3dqoglV(=r-$Pne43Ne4(;l=Lh}0N z?^|m#6t}&zt=PEWjrr>%`U!2MNjFfrD>CNfYve1NQZzc=;QQ4(8r!)2nLttCs;aX* zd^It#|FOxF`X0@0cc3SKwWU#8IB!EZGgu1P8g139%Nmcp-jz?!RlR;v@>9pw-#&fh1^8XUc>5 zQEGZ@mf|NQWz!q=qh{o*N=%7oMy`*Xh)M$D=h7fO-lkh`bfl{fr59b^uNzlqnw_0cs<$lVC`!{TD|0TE4k!K7r0#94Yb^p=-M4qdigRTt$DxibZO_Pu$8ID zB=?ZZa|^kt@2?a@BVsfxxm%wt>9Y`>JmeRTZzb}u@Z&$}w+%Ye)ZXtTo6>Ip3ZiwN z@{Ze+<78g^-Je~Y_KYFoqZTGKI|=MeEX6)P%Mu|7px}Le_K=L3qE+%*wb|U%l{qSL zZ2$G8_7+$d6&N3^OIjr1im!zf7YRQqw5En&{qf-TCb1*(0mSas)^6;0dEu%0f9*Q5 zd7lC6l6|i?g(`|CSOE_56}w~`>x|6BH`O_e{E0=541kKtHn`;$8xu|xJ-<(Qo6*kx zRT$c$w&-1ZX}qeyB42u5fcibqUAUY*Tr^!S z-BK)p=WdxtX`s@w&-v1Cl6LePhJ$%ZgcEhR*9yZ!K5oZ2uL1r}U9tnX6T<0h3e+2q zLfxl=1I`$V^P2Uanabd2DVyWXpgF39G_0sR6AI~dNdlWYXcXUhd1fS*CyDtm_z>&R zivFpL8)cRkR^u`xEG_^7-+4pFVRS}&+3W9L>z5N3>v2*&)u69_cKCs?t??r$&s%ol zso|sQN@5#qAd2v_Q+|Vl#hZ|GWN7!`Iat!og_%(;x@(z%<#Dt@^mFadh!k@6eYy4J zD5b?5%`M8b`};<5WTv{{;SUfc*m}pG-B-kV>r-ds(w{D0>gjKyKYL3f|Ka>)Wzyd# z==4kSbWr7tpIxU{3{dfp)A~h=ykRVi&E|FZJ^~mamAwu}KoC_t&?#8-;C0ury z5a8+ThnXOoI_~oi)c9sPdEzE*f_DmNzOf!oZxjLQ+QT;+Fufw|(X+2qSaczpm1dhF z1C&2AiNOwQ)p40x|7F(xX5Q6ck^Bst?@TD(37=@#w-5g7WMSZ~2_^{DG7g^b zuJ`pJZk=nZc$B)d{+3_M&)U*L0G>~)>EhDpBBA+(z;Pho_WaQMy0@qa|H|f&YnMX@ zluhdClf7^w0DLvwjq z2#E_d`~Y?szNJ+4rNGF#+dM~b{ygbdphR#!eKRSf-oJ2Og*9TUJG4|aA60d?G%L~_ zl_#*u)G#`FPSoYE3-F_sW2sn?Ucn|v4~X)vIZ0a!{T9Lo4b#RP4d1nrk>$6}W&|?sTe|o7l zqny2y)KNDg-sM2OIE6Z##-)LCguEG=P-}9IE8!mDwbUd`6)eJ&WL@?bPtb{(0%?Cbo*;p4{ z#WZ%y%9SPkaZm8gEXmP`H(55q${GsXYR+CITDO_MIehwotOP9_g2* zMBI;>pnpEZ^nQZ)Z1tM0UnaYnVaLqaT_Fo?>&$iQ$`tE03y-QjZkifl1S55oK%^>< z2SF$tM@w|&Z!5Rj9+cVR29wXoMV!eU9kz9Jp=8J4-l2Y9kHn*HO=L4?U55A**c10x zj+^B*7)?M=0b8p7-_9#@J5O*k+P)$oL&V~3h^rmCAWbwgi*J_yva#kN*`p+w!ySIN z^|6~@WZdglRuj5EI?m6djujT~^15k%BfW^PnuH=4u&JCO2;-o`wycC{eezm|CZ0Pn zD2nr`F`z&S2`0>hG^Aif5Ppw--$P&+?kN&XIj4g3%9tZwVqF$@!wswIBDZ>w7wt-? z7of8iRNX17*;Fh^p%4Z&MyTj;75|=UI0vp`d|Jt;WuMsqgN?3A|*fkK<(D zqm^x&7g{z~=3qZOAx`HH_%GYP(!noERa*N!i+v@%R96f;$)a%ZYV{ zU3!qje&B4eNSLm{`25QpI`Eq*%w;kKPjH#JaGpyPRhjo-I~)z)q6*SeNQKnf{}(<{6YM1HN7kBRY`qqsu(c_w49?-&$7^bw=Er+D%@1Tf zH`ZqKz6@MvkGCfBVQ=yM)?BgsID`BC4r=OjJ8!Nc*NcJ-7nCd<3>f=Wej&Mi&&>8M z+@G|KDQGaqFA5L>|Hn!ZhTs8ikp!D3U^tpjM3OffAz`R>+;9L&!!dy@^7&Mnv>Fdw z<+$N`k))v{{MJd?mS_c#=O&&7)O=>He+KG-nfmXEg6$!`-!nA~7fT_uH-Q8MF zcynr;mZ!et{ngjl&&4%Jrn*#r=m|#gB7_kIu?5DQKe4={TMA4w*Lw7IL(PllSq3DH zCf4yp=_g3(XJ@?b0n?p2do*c-{hP05%K**HROC~|#)8a!4|62v^k%!k?v*c`wJ4MQ z5uH%1_3JGEYWGY6&|VP}iCF$@I=|Q18%IVc4l21n9$leic5d@*ID1w8y!p$ilC^of z5~u%ie@W{*glfjYtgnWObuH{^Zz-@6!@n}T?5ueXih)4&N5lt@0Oeih_!Ph(klg_g zNE8GDVZ(gwBE!O?GfS36FNm~HF{QeUbeDGn)YLwDNM61zO|j}%0N z=@Wj!<&%IH>tew{+V{xl<${*8k3E+Bj_oDK7*`AWS>2Vhvx#~%E|`F*-V`GXGa|MLB3~?4vI2Z_YlY?NpQiHZ-&V$M*^(w^m{^_|IQbtMt&u|feC$V zD5A4o#`D>4uK&kyc~k_-4S(NoJ+i?Yy?gw_9^Rv*N6hA6ea(^=Um(W{qRYn UMAWGZU}a+ZDS5t$lfO6r1D&B;a{vGU delta 3913 zcmY+Hdo)yg*vALQrHC<y#_@xiRXDdNNEY+w?h1A5${qHbq82F^YF<43&SivN2{sF-dZq zb0@d1|I*IcuBuC(tVs(!?4e|Mxi=(iC`fOExmdykd^M`W>85D=X__ZJ+N8-(@4|AO zq3d2kYhWPJGqOjKko#Rla_01gZ9=mXE+*|)7A>6q%p|96(vwaAuA0zMbWNK$r&ZU+ z{8p@rieW){b1yfrY}o9WTk__4S-?%UF^Gp>&qe?|ZiBp%;Zy9Vgt?A>jP7)@1$GYA z2F#L|W<#;{FA3Eh)6*K8J`Jv$%p>yT!Bd8cw6U4Vho1wY6!!FtG)%V6`yQq&i+>LI zm30@B!*7hS4K#MmI(Yf7Cps^lO1igKVdtki+p?}Ul>0rm@!w;oHQG$}?nsJ{*32_?mky#IJDu<@*-v7^_^Z0K^s z4b0+?Zq{!b8Mu8?UWRd0sbrsj)hYXUuhM(SF;jZ@y&7`R zE8@iMpWn_>-jm5LbrY%2fA@?`wYlJOYBVS3nrL$sv}paiGtJ+-L-u!8%eY77wch=8 zql#}Nk6LQ(oY?pnNSqH*X?x=BVuB;Q=q&CfY+^lq60?s-2GfrXtwwzY3OJG*)l(Me zyqJ8wGQY6<#zQ5~ezQZ2>ixDxl~{h4F`gK}_>vSmVbfh1-=6aHs=e8 z;App$TEVh-lY-!x1J@^7R+SFi%x=}Tjx~X$jiym}DORi}3mKQ+q@;^q;_h!Zaec;f z32~{A+g7r1U7#F__<=_PcFEaG?r0Rs;F)-nHS%)M^T{1>~NAWx+CyF%hSB`2<-h)IcvyqSeh1< zVwGlvxN}qB_$gf{O#1aoB_|09Eico;{kF1p$wM_4Gi_s5_@lP;>$P-9xWcF*sbE#ntJn@t>hkfZ=rT8pT8$X}%oi1vf>{d&GVXD+ z%=_jh8cd75P2acdzj%ZVW{|v)R}k5}Ak$boGyp-eA=%k)O|L0FAW>DI_Lfr08eF1L zTqY%D0)LoH%*uG{Y4>L(ao@hmd!&6~?f2~iU-ZdzRd^?#uKmDa^-WI)ANu*4TA;QU zSeVUu#|%E}@E|GYO@^>oaO@|e1KwJ>>~6CJ@ETlR2}_OfG`Z}c7hOy0~+Kj zfO|%0pgWAP2{*4!(RX>-Tc0zudhx_ynCa113?t-8G@n?) z(hpsRU+xgXR5dq2#dt&%K9r7dj}+P6FvsjM4XKqfE7W#Z zH{M;S-BDj6!6fwP2jzYQAROZXSHF>vFec|scg%1tapHkiy7;Igs8BWaKMIjuklbfU zgy})Z|5KR$ABFYXhYbpVYb8&;lQU`w%(}pOlwL?jmrhlf>^If?zA#&4>X;;3mA?0b za;5+#=arBzou@niy0~;*Rt($quZLH{ZRLY5E7)m3k+ibJF^o9jV^YyRMM#o8 z+(br@M`B1L6~vkK69y}^l$ywI>t`Q7_*&eM*+YKUu=_-p6@f*2EX9uRSy7X>s=hVf zjE+Sf4H{1-&=xHtAs9l>J?e>%@-^XT{87Fb zTpE@EZ(C6Yn&vz>mGLK>?XcRLsrFh=;Z%!*hr)3>LeA*tB9AcTxEj&I9>lg*B~Ym0 zRs2Q|V@}LUSIaE%XIFR-rb?#s`!#d2Lz>rRe#S(}Wx=1$G zRwSoF6;dT}n9Wi^xs#NuJqA@`8X-p zTfZ&KcJ18T#~~f{2|@wq1-4J=texg}znZv5Gd1)amLAak0uzQg+`)m_FM`>3?4AI# z&q7S%bqHfp-yxM4Kb%4h0QXILullIU5_I)u>&JSLWRJm$P1Q{9sP}Vn(rIj%M@>va zret7KKvT$xI;&_FZD1$)4K;ZlHZ>o}+wpE0^zMT24Cvj?Y!lkSdb=?4IEBCeEmy1kPjIh&d1zOo3JhWV4PqMi zaWr!nFLW3^4~fn(OCz|Gu3nR~ut}TZV-xtvM4xnx3^?ph%ponXFMX*Jg`hB3(b$J} z|8%SJ&G9T<#Rt3INohBG&_a@5UEsXyT(byvNG>?!fr(l2=#|1>=45K&6sDxl9mFAu zb!PdgPzLes`gw4d$pa==E)Vedl}?b8w54K;^_!WLlYh|}jss+J-DTJju~iN|ui$y& zvbN19Q++@;w42XfCE$OPnHTXac(O>6UJIP+l!*as#fB?LTa^-(>;`30ug_McQ!Ih@ zPq77m2PiZ2$6{jomsa9&ez*Qz95;~@-4E3C0qF}F0Kd*J>nqOS2BNVKu(c-kY^$lo z(#VMWItRY^$Osw*(zcte)2`5szYdLWgQY&)rC_o%E2BnsVg@H83w{+eb%33ccG}?N zD_;S!V%gH^q$B=WzUBrSATf4_D5S3DQ+-R6g?@u%zlVcF5tcEyB@|NkC_rlfuwA-_ zioh+wt7YdyhYaRlSI0I!ZS(Bdcs!RK<|No2X5Z0Y&olDRa{ZTCv%@?EGD{e@L|ABZ zu>UfU;*?N`GhL?;91Z7N!eAc0bB^HGg+ksd{-`oGx^2l$B@*@hNxX)wdCY8CiL8d| zUM$JZEZXV>U=Ah%?h2?E@vWej{2S`J{*_dHKX7)(drAa`3k58|nzeDW6H4v%H4>7H zy;~diUe72v8h=Q0c31)~5#39d{kyljT@FcvunEKwZ3`Fyc5Y)QXKhFtRGnx6@sc;E zmg~osupwT>5GqpWy@-&Qo4NsEosFJx1O`gUu|C2=@5h3LeB*rnK_I{(u2Q~PA_O!s zJpKt_+F_fL93~gTyT57o8Hh}IV9Qe9DuDsQ?7PIlP2)LwV$SCF7RQEr%W;c;~jgQe+f+m*7 z_!lr9nAXL^?vMJHa+B9LmRDWQm9Ya>GDZTT?0Yadqri2=HIYT^w(jq>plrnKrs2BA za7@q__Bltq&>yl82t*75souAbBPl#{q2?8W_7$N5a4t0;hQ=>M?=F6Yaz~bmjohr| z^_drGqh9awqfy*&wwLl6qRDMnVL!{;`ai^&yn5uO;+)IBwcPVnVJ|TS)-@_teqH|t zx1vWh{hv2Aq^I4E#RtM`0)89}(l(Uz%&Qu|^+$nK>T3Fk!n|drtas;HVRq*OOPj0y z_r26IgMdWTBsV}bo40Q20?j7f{GyWmDT!iTGA`4zRTl;kODPf-HV!}QBitA+WHfbA zQDmyU+{ef*`$c{WJNoA}beKn0=oBe=U7Y_iCPLALQ2N86ML<7eJl%x!w(jA(!Q)YkIkRR&$zcV8-b|)a}S%X&XLnr%Y9H-QWCM=f3IP- z3DR%G Date: Thu, 22 May 2025 11:36:40 +0000 Subject: [PATCH 617/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10935 -> 10935 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5e5a06156..1b4cf523a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 11:33:33,539 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 11:33:33,539 submission_checker1.py:3308 INFO] --- -[2025-05-22 11:33:33,539 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 11:36:34,933 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 11:36:34,933 submission_checker1.py:3308 INFO] --- +[2025-05-22 11:36:34,933 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 11:33:33,539 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 11:36:34,933 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 11:33:33,539 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 11:36:34,933 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 11:33:33,539 submission_checker1.py:3324 INFO] --- -[2025-05-22 11:33:33,539 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 11:33:33,539 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 11:33:33,539 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 11:33:33,539 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 11:33:33,539 submission_checker1.py:3345 INFO] --- -[2025-05-22 11:33:33,539 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 11:33:34,660 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 11:36:34,933 submission_checker1.py:3324 INFO] --- +[2025-05-22 11:36:34,934 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 11:36:34,934 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 11:36:34,934 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 11:36:34,934 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 11:36:34,934 submission_checker1.py:3345 INFO] --- +[2025-05-22 11:36:34,934 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 11:36:36,030 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index afe70ec06..3bdca32de 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1244 +1245 diff --git a/summary.xlsx b/summary.xlsx index b3c0a03fd9a7ab5f522445626992a59778b94919..ac51bea02945e986b5403de3944b8e840944ee9f 100644 GIT binary patch delta 311 zcmV-70m%NhRku~J<0J$d)72D{=OiqDa45odpeeG1>?BSC zS!Fr8ch1}yR4w1)c5y{z%G{3f+0GwgyK!WlPXGw<7UJq8fn+=n4;grG;25wqU;_Ti#F&q27Px@lA zhWr`+TED(etHwf&r(%GbiZjA2P_J!M;maZ%uA|Pc!-+Qn_|p*d8*j%&4t>C*5P%%w zo|Mnx`K4WL delta 311 zcmV-70m%NhRku~J<0J&7$2A6%=OiqDno@-AKwGkEpiq~V?1VINM6CysS|@*B*-o4U zvg&el@0__as9f%47yW{DLaPd8Ns1y+Omm^y3Vk%I_z^|UQ^jeg6;$W|4lV1uk{QCZ zg*U4WSTDdu0i+yZrb0XK4Z+y49Z2eu;7~!_YAdM^rERgH?3=cL^E7?H51)c5y{z%G(WZ*0i0pzK!WlPXGw<7UJq8fn+>CK;grH34BWJy#M!-fVm9`DpY+9O z4f!+twSIjcSB-@lPQ?J#C1-?LpkCX$#Fs@lTt}T>hZAoF@TWu2Z@e8Bnfic7Apn`; zo|Mnx`K4K*I#2UOoG#)#Z?cRS6a^{nH^a!Q{xeV>hy0s(0!+EDKSBKmvneJE1O%nW JH3pM7Dt|aYlCb~) From aae36b8eacddaf290bacb383ce2f3732c64991d4 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 11:47:58 +0000 Subject: [PATCH 618/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10935 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1b4cf523a..02473d073 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 11:36:34,933 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 11:36:34,933 submission_checker1.py:3308 INFO] --- -[2025-05-22 11:36:34,933 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 11:47:53,012 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 11:47:53,012 submission_checker1.py:3308 INFO] --- +[2025-05-22 11:47:53,012 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 11:36:34,933 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 11:47:53,012 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 11:36:34,933 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 11:47:53,012 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 11:36:34,933 submission_checker1.py:3324 INFO] --- -[2025-05-22 11:36:34,934 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 11:36:34,934 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 11:36:34,934 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 11:36:34,934 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 11:36:34,934 submission_checker1.py:3345 INFO] --- -[2025-05-22 11:36:34,934 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 11:36:36,030 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 11:47:53,012 submission_checker1.py:3324 INFO] --- +[2025-05-22 11:47:53,012 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 11:47:53,013 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 11:47:53,013 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 11:47:53,013 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 11:47:53,013 submission_checker1.py:3345 INFO] --- +[2025-05-22 11:47:53,013 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 11:47:54,002 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73636 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.1249 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3bdca32de..4c127d807 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1245 +1246 diff --git a/summary.csv b/summary.csv index 7dbf19e92..d24946315 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73636,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.1249,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 32a231dc1..0aa7f4b7a 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.1249, + "Result":21.2724, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index ac51bea02945e986b5403de3944b8e840944ee9f..bfd97607fcf2086249811cffd17668bb1ff86f9c 100644 GIT binary patch delta 2067 zcmYk7d05if8pnUc;=Y74&7c`-KU9#>8HSd}Odyv`!lg8qaw-MGrP9=H>K%Wi6=^vk zAm%b=x#dy`Dw$=jlS+%Gl~`&{YHpReOQH>SpZnZ<{y2Y}bDr<>zMuE~zUQuYt#`#K zb)>s|Vk)}wU+VkSAoqo+-SNnv(?2lk5sVj(hf2Qi_L>A>`82YX+@1($lZH?Oax#We zrAF-rS1e10b7*uIXIQ0LX0I#fJ2#Z|Mt2!6{-O_oKj z`zagll~T9}5*J^mdc9CHW+Q**?8n|d(WCCDA@8Wf}*5&@p*A6W(JTd#c1+L&)aWbOC0WR6{D5Y8JsHe z$kI01s!0t&bI9t{PIOm@tM>+3T)Fo3auZ-Y3z=tZ{#Ey|RD?XxBw>GL-!JTMy5smE zu!LUWnA(!Qpx2J*S~bPAND^0?NgNzh84Az^E6vU@B4egw;gK4VwtBrAq1sJZ5s9pT zW({ct2f~Y%Ls!QFW?S1o8QuM$9hMQ1j7qk8y2e%EKx$(lXH{J3{Ao~RKFy5*e6t>4 zbp}*enfi0`>*c#;4ME2Ol3H(vEhOkaK4~rKk+cxZ?pudaPS7DWsgLxqE$+|_$~)JO zT;{rx6`uAfKicw9%pjl1p$q1lZ2@aW*A>*bn*T16dacCcRS@e;3NBbbCooKuuxV%} zyG>F}lyvQpvRBysGk-SOus-s8@umhN>wR1)s#1+}nY<^OU3!m!Oc2h@%E7*Z@j~_` zkS!^lXg)+!&&*TuNOo8a1W{aV4}Ig!EWCd z(@d&LGl_kGm*9))1Toa^VP%Cbz!1#j(-db#Jnh6HqHx|9b0VunE0IwdJTe;g{NH@S z!N+D+$FrP9Eg(g!hHJE4zzrqaYqwpAk_TOu6wEs8x)_Wc<+1<!k+!9WG1TclFa>KSR8GfUWt2l#W}u3@Jz4WlCBEw zT==)(0FgRNow1i2+p+^%>&sHD?!hsT-1BZ^HwXgE81I(k8~HTR(Nt40!4nKQkAfq zfgCZ*75@Fm?=R)^E!mW}+^4_KLsfH-a8zX1Uqx49jhxW#UnpQc5f7y4H?)(`ZlbgZ zi3Ti3x(5i%hg`NZjH8e1e=`i?im@!NYVV005MfkY=f2wU7N_c!HiA)czzdEfb$@KB zV`ep-Y-02~+C7x2k{pMn=}Y_?;enVV0_L!R+nWNIc&!>JW)ceVlPLyaclzdpm~ij7 zWCudZh@T?nowQOZJ|yfY3J`K6#@!RpnB=gHm`?nrsd9Ecu|%c0&yS|k%u_X>s{q+z z3lJC^I#u2vG_DKG(6KxHZBR&3kY2tG4}%n7TAgU#y9o`N+?fm7mxAWaU@Jf@#SCYn zqF2RnQ5TmVfr+(`v?~p_w`Gx4-i-+9;)Hj8OBoVkVxcJh0Jr=c1D$I{G?1_WZL{E{u4f_4>3fNu_OL&4SkcWq@T zo|8sBUgfhmOeDN*APUY-Vtxj}MQ^3p_-p|jYLEmhMcfUks!4|2^4q}>?ROte-LkrS zYtsF}eJ?Kd6;^jqzvI!RW6hDP86TDf^SgTshk6;C6;Uguv7i1PpL#9ZVRYF#LvS-K zA#-=@)3I5*``nqowi-2eT56^{70Z6ORL)dw@R@u zG&nFZNzT3VaDr@v9U`X|FTU+}UO8ITR{uC*Aobn$y07N-;2S+_JB3!I)&{vQqLblz zLof@ z8F0_nm}#(*7_+Mk&DxV<=CL>G`e|I;ot|lUN=|V~x3iU{r<(4q((FV`Q`LE&G96!= z1}9j-#-l}!@l!gDa`8@=3I3o5tuNH7M6a+PI_l87tsnNRdrp`BS-9VQdi{^t$rqi$ zV+LJHd68Pgh3ha4*^V3>#y)3 z9m3u5OfOcD+%h6_d;FYFOtqhyj@cOd5<YGX+!AOvR=$&CHT!uKNmuGv}Ur-#^}e-g7?ZyFAZ%9|qhHs7lb& zwP?-SbE+m;&u=|9Cn3b^lK!+}r>n^x<0H2Y;7vRS_-tgS!97F=MEvYQy8T}IQ-lh5 zPNI^%toEIJjuL#=IGKLS{B{Nle1=Kx6srn7+BX(<54hKrKkd#R3P`(haXcmR_wQi@ z{}QheuL(XI%2~~Y`!buB!_BSmXvJWOtfqKWc?1Q!GiTW7rr ztg(B4T`@HFThB5?A!{>Z#8b&xUByz5j`zDo|BTr!-@vNTtzEf%)%OzGRu`OjotTu{ z$*8j8Mom?qaB(| zP5;>0nh@HFc^s{FP^ONGu7;~nwmsNWWsB~1=_IZvtWFxHu7bx??Fygt;?-3~5E?iy zXQWDO)rIB>b3J0J_(=+9R6l`d_{DVgb{{{Q zX2QHDq69k1O)D@JA@zO>Wm1;iegeD1>F?$(wap8du1n!&Sdh&-3{JN=LhR^@9N5_U zU^hEG9M?pM@jbVmIX7@-7#mVEHJ#f2o8S1j*Vk8zR&<&6ABF$^`Q~dBNsbxG>qW@d z`7Yti4%TGMwAk^8a)_8X&QF3wiG>(L1)?^SjU+`DOWPFjFYw2ZG2%Tt$f4&yfd0y0 zQzqwywIg}4gY4~7GCF}8^N=`>P{;UeA%S}pgK{J7aL_uA62Px6S}kfNFJT|1yVqqAGC%O#f`+v+EKCZ zvigv7N4c_iAm+<$jigK?ak~G0;yPDA!OBB?-DCd6Bm7caYz3DWh3Yw1uV zn$pbSe!3QY7P8I;S3pde^y!yGCXC{7gAS_HDX!W!@3FR8rGu&jCXBtJ@FOUstP()^ z5lr#b`9_OGWM$@d8klQ0qX56HD1L}&n6=gh35zCSqx9w2;cEAQoB(+&_!?IgJM{BO zAT>lwoDER!R+X$1sxAk5d=_v=rL2g*n7z<7rlFRpFxaeR=Hqqz#n#w3Q6r*J_bA&iifc3Rz=-k$h?Y~Ukl~$reSm4Jp4CpK52{k;z;)1aykf%*g_n=!= zr0#~SNC@r2K%;AaDNy|#j|RfnXJKoWudm$TonzvY zjEv3is-$Qu2s;%+-ui;5|H~37H-KtI3JzBQcsdM;*3BiEx$NEK@*E}OC~`#m z_%d@F9sG_>E^~!w7c~Ctc3fTMNRCPmzjZ-Jq-y3rqDxxoiJR$MBK0`(2Pw4 zJ5)5TbQbQsr`(igUO_gm+#PyhwAEo%X6~#xQ$JS9waey{?-1`^$<`b_L{{%wXmVZ& zA6CztcGvVd#IfnLhMW_(L$U?jrt3)wq~G5j9A9niOc9ZYJr*H1UQUU#TbFd^14GOI zb7nT6T)D8Roa6LZ^JVWRgoL7I%}vK`Y@1lMUnD5g;@`jn<)Gyxa(W!eh7{ z-?!{SQs!3;B#&*#@fOWLT(e>B@wq{xCA*37CH*?oiqVG3NxV0d8M*Iaa@TKW{xiqz0T*==G5BT3&xZ#zgAB9SA9_szNpi^6&^m+T<{Vm{d%|r;(KwI zpXBQNxv?>fJZEyfP7GB x=?Y|k{g#fv3}|dcM6abv03Z+_C Date: Thu, 22 May 2025 12:10:26 +0000 Subject: [PATCH 619/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10938 -> 10935 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 02473d073..8087c2b12 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 11:47:53,012 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 11:47:53,012 submission_checker1.py:3308 INFO] --- -[2025-05-22 11:47:53,012 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 12:10:21,787 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 12:10:21,787 submission_checker1.py:3308 INFO] --- +[2025-05-22 12:10:21,787 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 11:47:53,012 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 12:10:21,787 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 11:47:53,012 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 12:10:21,787 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 11:47:53,012 submission_checker1.py:3324 INFO] --- -[2025-05-22 11:47:53,012 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 11:47:53,013 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 11:47:53,013 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 11:47:53,013 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 11:47:53,013 submission_checker1.py:3345 INFO] --- -[2025-05-22 11:47:53,013 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 11:47:54,002 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 12:10:21,787 submission_checker1.py:3324 INFO] --- +[2025-05-22 12:10:21,787 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 12:10:21,787 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 12:10:21,787 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 12:10:21,787 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 12:10:21,787 submission_checker1.py:3345 INFO] --- +[2025-05-22 12:10:21,787 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 12:10:22,787 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419825 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422993 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.48247 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4c127d807..23ea21889 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1246 +1247 diff --git a/summary.csv b/summary.csv index d24946315..d7d326efa 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419825,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422993,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.48247,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0aa7f4b7a..fe2e21325 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419825, + "Result":0.422993, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index bfd97607fcf2086249811cffd17668bb1ff86f9c..eec66abe56516d5e9545e4b8f89b68bdfc644cca 100644 GIT binary patch delta 3588 zcmXw+c{CK>`^RT&AsL3$kj65`PUd3^*&@a?A;pZcWSz+tCA+cI2gx?W2wBIHk{GgN zOV%O#zNM(Fk=@AhoxZ={_x^G3InO!Ib6)rTJnwtX{cyYbc9{=5E3~b|#)ss`u1aXU zQl%L)6l|YX^O)Eh^j%cMy)aCQU6Q_+0&v`zt{#0R8HWoZj&~oeMt$8>crbC{Xfpue zIo-C^CY5$k?RNKl)j7NloplWp|y6#(vakR=MH%+ zYx~;J&ks>80V&$VZx4nO>|jSfU|EGLH-2u^q`C~*O~<>${-h1qC(n@I6)7b|b@aF{ z|K9eS!dIEbTRIZXw`B`FeJ4b}sST$=>1tkV9cbBvD$zeaDsiy^jy#~CZf`fhvsI)9s4A~NwZ^W z#pqKjH-C8CPbsJxxre$;E}A(!-bci%sr=e`^#;m-BvnNCgchM)L45;h(eERyslj?T$l!@FqK8Gx;1o)RL>Ha6U zxk%~qikngIhQkBpEuY1hm23~R)*11wD~Yb(vVdyJ`YxGd*ZOBa>-bG7=9DBPM%SG5 zUdn45i8D61JM$3CYgLUBn_F&_;Pv)Z{2Q7zZk-3IC+K|DTKt#-Qh&Zb_bJPvRbVAV zPt2jAi5QJpM?IH9Y4O<3xOSShjybWHlhyv+>^<_>(;QB7kM3lV054Eq#Txqb1)vT_ z#2#*3+WJOYIk7togfuK=5sqp!xaFx^{XJ3*Hlgl2BxDex_c+tZHFNb1RtsvIP}t(1 zl{r4QqHLQ#7W<=vhD(Bm3XJ)AcHb}BQnwYZn55%hhg{6`Twd4^(`=QS%ozIIT(z)r z?EbhvTOLGQ+TIGjjINj;Zif5Lm=?ASptlx&irfgIY~<00%!7;xs;jfhn46E?<0^{Z z9?PbUjRe}>59!2RwvNi}solElr|lzo(Y;Rz9F2eJz)~a5OB0SUUNbh(ILy>0eY5CIQ^W>ywV@!V9`ODm<^ zo(M5lyFsTfO%08NA^CUC!rQ4PnrAU3j?!l_!RzNRk0QuL@~M>b5BQl4%(cR@9ASyB z3eN%0U}EMl-WahMV}54lXZd&Ya8m2`m-Eh_H){>lRZ5g13YO!WB!&hZu;-eaPPJ)} zZvxvu&AnAP%}GC#c#5d2l7RrL;orNQnvlL0W(ZD_BNC#Wc*VQGpdPDS;I*5I+5a{u z4g2YbcQHCQ&c3N76Z;|GSbNJq7Gk7L;8Q+Qv+3ZE69=7j3jAO#?*9iR(E*kYY3``XKF%M|8PyP~xTZz(sej z={a_JGhSiC)!7BJQ3>Zwm5%BXH~COO8-9T@sG0U^)aArT^(zpf^=l+ZR}W;#K7TR~{p2uDwV2h5phg>oxY#I&l?lc~a^1P6;N zO9pp31Z6!k9(zx5-D&(muAiptYyM$i-dke>qc^MS6bD+l@{owIzo;)naT;)CBq+Kt zH>!8H{gkpZ%D@Py?2JuK{#J$j(C#rr2jyV@Td#Mm3Zg!#IHiiZ%m0+%6E0+;s>H`a zM1K~^fHAuhxyooPg7!ly<+MY_(SOx7Xr4KPDv?KdLPTRnotK{)%A?XsM@%qB8q)BcX%ixqK#GoRgwuQm>&<}tn)wcxdDx;6%nZi$5 zi)nD?aBzd9TRaG)M{!dwnSyXSdY*a!XL%_EWHSkb8#8ik2O%Fmy2K zvFVj;)NTF{YPRO2;`xd?9Ge6#4?c$+`CIcKkt+{NXx_aC(Ir9DezzCW_H(ZQ(&0-+ zNQu8y5u7isn2k8!E^{N&=S#5_Gh6IwpSI|MQhUSuwKwa!66O!$?RHCDF!~a&VOhzc z#_;4(5KScIY(10=lu5H%h-oU;`glaxxD=oO z2^bUs=%EaXq;F%T81wmr8Ue_669{|!;!rbq!dd{s(Vcfh`9Rluf` zIxs$yKT%FR-JH9iZUFi7YoHu@7bL`5rcBeB5`vbo(4%hf{oRuZ%^YOkGm_(q5KS{P(Sb$c zEE7X9Z2d5Z?$GzcGH!;2HYs}aaB?(PM4?==FP4-J7Uo1f(z3}r#N9FqCN+aaWn;#j zuPQh1i#pf15lg+tN+$N{+PJmi&4&fCU8aq02ltjx6%xG(*MeG8NoV-P+5gv9k~-go z(?`@)SJY={9A}Io$&ODO-No((VD459YTc+ z!FHrqe9(W~dBSoe9EMmDgu6aeUiW+nV-YH9Fg@;@m7tDMJ7t85Y=*Wzz9 z_Os=|M8-56$lT)qC)yn-NgHw)vw|?h1ZWbNz(bQlS?j`{U@XhRFl=Q`p=T~Qp#^z_ zSZkE=x~e!`hHdd(2_MuM1129C>Hbvb#BsfrEt&fvY=m@}Z9QtabZEUzs2t&Bw%N=p z`IRgZ1CLQWf-)fa8IVaOB7kx(5!$ez?w}K(+#G-;0i7a&Eb_b=g@?Gbhf;)?ah0Yo=gXv#muN{`I(R(yGr}+3L3|ruB8VqM)Nt z?meEqc2r@c`ElxM(EN_kL;+<9?YOaUaCNi3WoY(&ir*;IR-waN{0I@7kg>6VeGK{$ z>!He)9kr6s;r96_#Yx+wiBMwHr<3hLe)vAUr#<*6Jm}54A*ZT?E)P|0 zI6nRQT2ozIP^5nrA@!(U2#yRqL5Cu%pm+T@&zYf^S@;0}06PFs0TJloW=;BZ6$>M+ zz%CNH@3&myG2A=iq%_Fi5iMY!3a@e+(an{XpW(n0qpd-(_wgr<&+L#QZxoF7debx` zyGG1aXo~8JaiwNSKcs*?afyL;xqH7fLUdz5*H4e0KXZ2ZBjt98l@F@c`}gv5)^o6Z zSVU#fwB;gCM8Wx>P(WL-fUw3h&3XRQ<&T00?vN9gppUS=_40_1ET_4$!-_Lwa0Z;( z;=dBy0>i$Z7c|R>>@c+aTA>zZaqM#S-Cgw*uFR(yqsW&tMvAVEQ&rfy#kWd?a2+*8 zucOz+!$lU}y1vrtKe#cMN)! z%%6wFx33K1;Y5!ttb`E5d9;9g z26{BzM8@rF*>z!r-sou=Y2OmO66E6AiYoZSM>D{^y&ogcB(Xbobo0$g8^C#P2 zoql4)DHUYbG6I#N^y*sPj==H~j>AnN&ig(bQh+ zFQt0-gbF1jtK6Ye1|o#tJ>c&4L0AvxsufxydrI=ZJo;|AK8-uq>)$=ha5AAx&7KL3 z*lAcoj47IzS0Nu#(3am8%^EXO1NY4Ld&u7c8C?{=jz$;Ml*hnvGs=Dt0dV_}?`A$l z9z!(v=Dx)UqX|PzDf8zVNcGIen@E`UlGhX?;_<|+q_%&1>-7xxIWNyjo56^k-5rJ6 z`D?Dvzo1@FKhbn>%I!g+icQEyn?e*ST7Th2yvB~t!8A?;xS{4C2Vakg>3e4UqPknN z<0{ZiUVl@oEs(V(km4%|u!b6Rt5Qe9BiqUJOtnZwiHYubq5JV+TKP*wi2RKB^G*%H zb@Y1-5e8f&Tzl=2$=mUcXvk`~9LxefVircaxjr-6e;3l-cJJ}`AxdI+s=@>^XZxDf zkBU#&b@Pu@fDc#iPq`d&-L=o{ZSrSbKEu6n}SQv z9VuKySPd>>Sb;hh*ILm*cZPkOmqJ8L%RL=*k*M0fRtD;~tQFPRW^m$(mmp2wU1)Ya z?)~jZ;XbeYW8Ka++&4p91UHmc&=7PoUnAqvz^E~NU-s%rQ zNpM*^xJaUGqJ`KUj-(=&LY&fy$N9oJ30r!U;XsxWLA=VRNuDw2;e2eriMvgQbP3!h zwBJj^>5c|lJ)(fUPa7WPH0?V*nas^pHp81j($xsS1B{4V7euJX@h;fxl16^~yVIY- zIqr64`|>b(*7Q`8qlzpH)+3Uk<^}-o0;m5djNbXasA=I_^-9cQwU1YV-unO5y=7)f-0=u7HG}A5HWPDa(5Jw zE+7Zo?mHPNShgLF?h|XwWdzFSJYLm>pgv0Hz&UUP{(?kkY>C!W%P2#Hz@O3#3Y4#2 z(BUg!uP)+ubHKWekxc$Q;O?*ROtE+K8MpD>eM0pNcd>6#04Tq*Tee`jc^J_%jjBvK zKh;vRS)mg|o*s9_Ftn=JNG&!JU^6TKFq1Tx8CZRX_%Vg}fU%kbJKyJuxwu6q&tgA@ zxdyIMzHV@qkjcmK@&TaUwUCn`jajT3PmTmx{ZArS*YeH2DWislq?n=tx~!=tA- za;Wk}6i^Pe!n`YB4DLEbo8S9eW+n~g9X|F;aw*e~EU%sfTzFF;YN2&{YL~!1G#Sp} zOW?2UJS={#+S7x`I@?_GG+}x31Gg5|+QOU{PNr70vvG7`e{wExoQys{__^wX)tFal zonP1$|9)lTYWk?%v$(oTV=_?pC|!6Q&f!$J6IVZGS+Kx|5bh-OprrFZJL#W>Y#JEx z{+kRIsPiMd3>>TYcZ%hPTP`1_gtbXYIyq$9mWF+lTD*(;bit);GtEYNT#8_%r8=Uo z#^DRe6fz<(>1O|z*M zX5eb>=TQ}WVJJ=D%h{W>XepV0BMjVyX7a?5jyD|Hw@%(v=DTuehHausqG~aWz+1Fo ztZ9MeTuH#To1n`%(=g6Wp_+TXV%zt*{#cq$2uX2TCUaa)0*bfbYFnSc z^_SkOlV~g4^5kOLIg)rN#PNzDT-4&o0}RNTl-DxCFhZZXC0M}4F-tx-ihew%prAZ1sb5p4E z@Zi1-lunKP-KvR~uf74YCjM?U1FNCd`q+L_GunU3IL!V6GT4AvtZOSTtr8C?lXh?H z&$+I*cP>l3{=t7UDp4f;ac`xB79ydubQd4W>4r;9$bVo9rZ?~)5DszS7Q}e#(I}_WU>v}Om=$G zV%X_nC!8_8S1AZW5!Wb2th=N%{@4<=j14E^V~8Z5RfwKz;U4a?xgN0zBBRR7#2O>s zZ_8loZnA?lT*2IC8Q_-7)`E+ZyU>j{qm1UQtL1xB&7$c<;t^3f4U1A`duOG^D$Ot2 ziO|ElUOj_-!>z0gxuv}4$gKKj`(i^X_ z-h)Y)LeRt#Q5e6U@LxY3PL=G6A`Xa3@JBv3$*#3VNK4#d|FJD92gpDV>xDgAs8V_3 zN7X${Lq)Hs72Hx~H>y!Zx5PCAPnBdW%u(Gns;t%iE@EeZqE-e@7rx|d#Nh!gV7JPO z$Vt}$si2Pbvu&jE{*XtOfL_?&Oi`0KFFQ3ida~f@??@Z=EfBc$t6!d%k|1eouu3=# zDJu_16YNMecnhAYifiC1{%f6k;^#VY5Nw+V_zdEG`~Npiu+YFiapK};@&Wtp6Tt#X zk~0?16_r1igb~7pV+N7nXH40(n3U31w&TMV%lI z3W%V<(ZR*RbLD~-iZ!75-4^&v!QwHT932zE9Jn~lPdI$bey>XMZV?C2Om*E3c)Y;kOz_-Us-P!wp&+mKvxiv2$6+xL2*d*dRUbRrBgom0vMQ7(&cmgEo{O!ggzv5BnNCMf!i(`0@J!H#i99^k z@QYgX*cCdGD&R9ArESS@=t`W5@@mR;ed8pF5&az-dZ8SGwM>$=wF!P2CLi@`c$FjR zNof+pOjXH_TkyqmaxAo^=DuUOfRjeEDY)pw^%9fl6@m97Rr(eSgs-+nUv8+D2^Eh6 z*NvIlJXe3Y$iK47EM;k4%nHmKb~f{0=?q#8JrOwsaCHTC zG$krFr|s^DUIwa_6}k@n-{Wjs=_*)|cv)Ew{Fs=oY$S011PB!2s}UR$>5B~}N~?$w z*OkQ&ZAlSDVE+)|D#8+fwFZMgwue9<5q1;)f8g(q5YZ|IU^U_+6*;a6#Xs}^0|qwI AX8-^I diff --git a/summary_results.json b/summary_results.json index 4465dbc1b..ae33403d1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419825, + "Performance_Result": 0.422993, "Performance_Units": "Samples/s" }, { From 9da0ee6ebbe0e575484a6bdc95ba66d72507351a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 12:14:43 +0000 Subject: [PATCH 620/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10935 -> 10933 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 8087c2b12..737e86847 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 12:10:21,787 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 12:10:21,787 submission_checker1.py:3308 INFO] --- -[2025-05-22 12:10:21,787 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 12:14:38,301 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 12:14:38,301 submission_checker1.py:3308 INFO] --- +[2025-05-22 12:14:38,301 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 12:10:21,787 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 12:14:38,301 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 12:10:21,787 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 12:14:38,301 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 12:10:21,787 submission_checker1.py:3324 INFO] --- -[2025-05-22 12:10:21,787 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 12:10:21,787 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 12:10:21,787 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 12:10:21,787 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 12:10:21,787 submission_checker1.py:3345 INFO] --- -[2025-05-22 12:10:21,787 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 12:10:22,787 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 12:14:38,301 submission_checker1.py:3324 INFO] --- +[2025-05-22 12:14:38,301 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 12:14:38,301 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 12:14:38,301 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 12:14:38,301 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 12:14:38,301 submission_checker1.py:3345 INFO] --- +[2025-05-22 12:14:38,301 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 12:14:39,388 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73636 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70059 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,9 +36,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422993 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.48247 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67588 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2638 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2002 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 23ea21889..600fe59b5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1247 +1248 diff --git a/summary.csv b/summary.csv index d7d326efa..385365125 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73636,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70059,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,9 +14,9 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422993,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.48247,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67588,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2638,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2002,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fe2e21325..862c28dcf 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73636, + "Result":1.70059, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.48247, + "Result":7.67588, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2638, + "Result":23.2002, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index eec66abe56516d5e9545e4b8f89b68bdfc644cca..77a636909f4e57269543b443e36a62738884471a 100644 GIT binary patch delta 3768 zcmZ{ndpOhY|Hr50kT9l$L*}&2A%`q2IkZGFa!yf1Y>cMJS$Hc7<+M$sIfm%qILDmT z#AsnkIhV5}IYv(3_4)j+>-YWVd;j^muj_d~UWe!Fy6$S*O54)geB73Zb_H)rARij& zPH78J@S#CeL_eFegA;Z4MzIbEpL>Jd2}3m7>HaRN9K*^FWWv{OV>V@L>i0XK&o_l; zno9)s$ceX=kLP>Zvy3z)8E+SVOx0}c40nQ?`i=P-wo8 z1yHTST4VNj{(y|n*jTUElGG<%kL;3wWw)TAiY0<1UHx)zjh*I$p65;Jc|giy!RMa! zDKtwhjglfThNb_h){t`lY<*I)cV~J$f0!650qGsAAAdI=q!qWcf0!sh#Y$KJPNR8f zsY27obuVnG1DAMJKb1MmPyB96!B)KRC^_%Q`gIO7pLIla%p$(kGYU7-wIJ^Q+&{pc zCwT5wu87&tYeC(&u2q~soGT1_H=jH@BcJE;OhudC_4wBlIF0WICm+M0`7k~CJHqVPn zUx>|k9I$SpV_DK_q9GFV_%to(H(^WkmKxe-UXXQv2eg#ZY&&w5b4uC?^Q4%X z+FMvdhvEHyLnom=8{iL<5Us=7-VCz5@!0y8K&w zNp$gq_}RkA_^0)E0AO%i$R8N#*727I37G37kCyy|=!z=*w1;aw!I1(&5i>-YKY zOsR!2%6r!HH++4?o^3C}J$C#)!bpOf)!Ni#sjIm}&sV!9Q~s5iDOaRXSEPy}o2Q0? z({&)PMIlwwVvDVs<{Ep3NHG%$hVVeR94xFv7aWl1V7d9ua-{V(qB^JCrD5IYh)dXc z2d1wD3hN0ll1IrN9%Vj`K_^hbzITSDCoDXK=H5gg(l3iwi>r<#54BbL))A9M?Ig9Ai&L{x3DHm&eNFzsOu{-pjm@|BJ$>WQpLD)&*=jqG`N%1mWx%B7f zA5I)ktWKS6pH|mzXg1r7)H;>Aa#=5QZTMCgy2v&7piI30zwjSbr$K$@4dIS4;o<@A z`(uq^fH|r$Xd;s=;N6ab8d6DcavWJbqkiCl*rN*GeYr29f3Ce#P3VTQtr$CDLU=uE zq?2!GXKmkDLC?ACT$@~zZ^Ki;r{Z#g*ODCIJ*yd>8~xPvPj-FuH~bEbgBOT``JZp= zwf#12o+zos*sLqpjT;}+7!QP}PJ9XhQE(vO0-5WWM&z9dv|$TePbT#Ntk$Hl+?z*L zR;rca+(CyOjcg!CIhU)bI7_#v{763aWWe)J>Dqa!w|^9q9Tw?5G_@f;3-PW<2CH~ppWgow5@oyJO3!)r^`gTgkE9MGNhozOr{FgZYj_=t!#j zDTMlO#hnj39L!XmMng*d{7)>PsS!ya3uOOgb>pK+{SQ#@LAc|1RsRO$Z{|n6(lUm< zdKU;K#6cAws{Mx0BSi@jW^D}Q@{I^H1IX}vJcQUmO-g_o zqw)5MkH1yL$_}A30I@U+;1v5`L>o?kDyfAhQ0;X1{e>@k>cGcMZ$;yTS^QH%ogmu# zVJ=OxC80U49?cCM{=G25zFEZxq?T?TCtvo7n`!=@x}*D9VXGk@i@)T8JC#Mhe@y6i z)hms?5_9wujJz*z&;Zr$#$6YUk1)%j%cLRSMz2N?X#iXraWMTjVPC(Z^*7yUI!rav ziz0?8_3A5J4n?H>!JZ$`TMbpSwOV0@|0I++zyFyto01(%i#cj5%@Lh5K*hLm4@Tob zW^O~xl4;2C=rs_5mM4vP{8yBz=)88uHqo}F7(10%=f(=?lE?9PP0u|YvKgN?nODY0 z0c~?Zh^C>Z_n^vF2%l#w2B^Cg;l?~K3cqfb3Rp7hC9>IhJKeAd6UtU@^UhsrQv9B# zZV<-@5A4XrBAsEr5hi6TnNgFO+Gj2+e`#f}ZD-@kKOSnc&NUx1*&l=34FSP znyjyOG!>%)b|A+olxfCd3c=0niEYBK@N5J;GP~`s^?6`Aa}v4%j^)22?ZSOY4OGM> zk$!)xO`ogUr43M$W1vZmA;5S%0AcR6-jo%epx;}2i)uzNK`F?&Xn&9cr!9hPNCgmu z6ZXSDEo|%fb-?t!j8qT@V8u>FQ9T29XsWt{4MY!Z0ZpYIN7eB+{ zDis2mQ5JJs);Tldh&B|BleT_Phn$rNWC67JtcA#_Ol8dF+UHvnb@_+C;4My=rc1}K z#VBM*ldY(neEIqdWjGU9%1o9D^6GGt~D*TtPFK&(!4K6d+h zE+`GTd_HmmtSXyn?+!-?+E@}htkMkH^V_RN3`NZ+(Qe;-UId6n>FNDIq{Houd#mDOa5 zE5Va&(dKo}x$u$&OTfig)QuFv8T21`=En$*BpWOziz~m!%i!!SX1hXq6&CCw1)?DZ z+j#`}uAD{j1^ux6OJn2^GMt7~L>Tjx6*_a3ajd*0q18^)ltJ@*d$1#j z-4WRcTyL_pzYoVDS9o~)`GK9b5nn+?sY&CP3LBWp!XoxY<;*VO)xq`JiF5XKHunHH z?Z0YiNWTYSpd#QlH$Yel7sH4PZpM=&$n)=3$i@G&6+ldWo}`9po^BrISFG{RSL?2U zd&yGkVCwvv=sL69Rx>{dGbGkk*gfTj<7+QRU~KrA_EzbRcTIE<`kJm}!g3RCG^-HP zp(fRJQ-TR(%|V#H5DVb*rICJTzt`!8@hhFW+Bc1|ru+4h)vNl&!eK=iSc!&H+<2?j zHXWw78`?OT2cY7kF~!Kr^!Qx|Bh1dR^%lLY>tAuHAKP}PXTmOK6Uver~43%Gf+)E6N>($I00RkmgqkH6b)i#4q zJ{(`!N!ub5F*Uy?)XN19eG%IGrMESSG6O#f40@OdSAo5lFYlgpyizO5hKp3fj6E~9 z!Y19J?Jqlf8hocdWV?hVo2~5c+gD_gj8J-bMuT;KAZ4augmLu1DT{*z@s})fM71i4 zfAYKW^&Yxbgm!KK9p+BD9rXk%Rey4?YnzbF3)6cB+&%?Gd8d;D5(V+2eh@8-US5?J z8^BzzRq7b}xJbP52e-V9GBe0m0Ru2JIZ(?K6}lEu${r@pJlbDFgp1Kpl73V$9TNFcmzCmE8@F8l;kFogn9}R) oD+^2htuCAk1hU)%0*QeB|3uvVNU=~p&80v|P?qQESNMDLA2U%I&j0`b delta 3754 zcmYM1c{o)4-^XXNM7a%nUP1#c;`^`ZLl zr~>tuDm0=8f*n$-9#Xo4zKV;v=aZy)q*yyiAg8s-%Hbzcv4kMXXy^WN)KhhJImSCD)9q%w9XqHOHQ+&G1y3^9*Tw- ze+T3|XVpFMqE}D1bxkOzi&jj;9C&Ch$OAS^xeS*|J}N6W*=4%hYdmI5bLB4!1FH9} zTeOjk%_{@n-$gbBBxzB;+#4j@Blf=`GV+(MeP63eb{VjrjB|&7zA_|L@?*^0ZhJx^7^a-(;2OU)ZytA(@+yYzU@y=LruGhgq2tt4er&yX1*@OWtE18b)(e&FV@sZGtp{BZ zy_;@J3;ncv>(ld|*LB3IS}y{A321-wX(~Eou)b5{6QUT;)EN6|ry4A+TwcaC())wn|;kSh9_2)#8+>a!!=}m7mPEj{WG7m{RHkq1$Ta< zqN9#)hwF2eG*F_-L;^4p;RTim3I9Nx#EjtPzxvI2tMO4)hu!CMT83hc^lwkGKeDXE zO3WN>lrHy&Kr~*wGkFagS=}l!VSPUc?yq^aNmcha~Q=o!|JvTspMh3xw4z zWb9X6LZex3JZ<1(W5wLk0pR|yH&Y%$S=ihNzJM#6 z9c)DUPMPF4_2D+=zKdN8qOaw!2F!wtRF|h0t*$?Gk1Z>FeIT1MG8AZcH>90#!6q`h zt9s*tpO%l*S@#|#XcY0eBWKk)L8fT5(Tb72=B^!!d)cTuYdbc(HsYJHr-5rr;M%PE zhd{&I%=`9e3~;wOahgITmU`R%Qb7mY51LwRbT+c5R^LwHtTJ^&p&9AfHS>b5Dpv-%OK~;e?2HRd@<|f{-vn2u4dp8wqi6 zJjuPCLy%r|xR7(^jA?VAj#9idMYxpMAUV+QXnne|0XW)nk#-&20%`27AZU#H8OPDZ zU6u4taT)x9IXZ^ALNb6R$Wfy3g6BG*kS@z?@RjR|ng7-)4f^SYb+CgQ-0;De%-iv>snSi zC0X|*FtnfY!E(&jGCVrveS!nUatVBu0)WqBxZgMPtt3nca8Wl&4#I)+(yZO->!`Ty=ud1k3d1P8PfWPopkV%hF zefn%56!(#{SpUR7QAj2z6~RGp{5Pu*N1&V)z+Hx<|A>VUI%rXGM+hAZ1ylr(eA9ZQ zLb7l05Ujjp<&luO<`e5|5*RRWaP7h}9K| zw2Ek4KL>+2pIuhB7KR&x_L&`F=F2TH(37f4k0$c>$&Tii7W8km3Cnt< zRy#@mrSY3>^Uex46t`Yc11R=2^W|XBq(5Gti4!g&l`)X0{Ori??bf5p&RBgzu(Gpt za^jZ?%)3^P0Tv|7`oDtRD-|&Ham5K$>}{dPK4Bups!As~DY%bfX$TH?3SSA6Q`liZ zrIfkHzV%*q4VvXhV@T%EA5n1D!_LlC9*#VfxE*-wcfdTZ+_$K-zhdE3C9d+@V^3JB zR+hgK(s>K(^Qr;?V0(Z{Rt0EE@fhG3WYE-HO3Xs zl+_TpB?&pm8O+ch(cO5y9P7sIJ1`w8OzmfDKJ!=hB~U7I!4M<)M-+YR*(K8<=bI&N z6xM99b=lMgPs)c4R-p9G;IHcIH63JNb~ny`yV%7_Px2)qBQewnnK%q#iY1+_h10+? zDV7{JQU~GV9HvAHAT+Ao3oSMKd|EiG2-_im?N@QE_To21(GI93ljcX~jUU%ojq3`~ z+-c)8R(A(6nnZBIRcU!K3RuqiiX#&Z#Zt`HE9rJbDp&KBga&cP#Voq+)g$4wD4h8L zf8T{fUt)@8D zy)GQ&bl<4YZ5Rcj#FsCw)QDp6T9u^4fYg;Ay9qMCUN0-B@ZY zXN&GDd+%67;#+L~Ej?t%x&O(fpY1T^WG>qw8pGYLd_lX2Lu?>-k6;>_B`FahfYxP+ zJNw5w16@b?87i{c(~%!9`Gyuh=wmVM=AAl1{dzdKV{@@T-J+n;hKwd2C-X08P~xk?p20q3qNHOk2+sgZu}+g zT;)b7_8tL>#(uH12rGr__w!;pOzPcs?<`=;B)iF1f|`@NAxvp3+uS4f|C$@ccktJ$ zx=}($r9lNuR^R769qj)_jGk1BhH?IO>FVzpX<{Xw`;qZdJNey<1N`s&ynRNpEA}Hi z_E@xH7o1{7y8#&Sj1^@al(4%2Xcl!C$UP`ABjKnldgC4&_}Yn4&W*^#FRj)sdwnWt z?9=NC*#gQpfyC7$&8c$i9$dr#YQL+Epv$2^2?Fog<9v;laF4+YeqMnAIns^PaiS00 z1xQ(vjsu)z(T;cJS3RF2I7JHTOb&XcB^e`(b{UZZ+d&c|)|>^r)K7VfRTdKof!g2I zVM{K669m}Zz~RX52K?6(*CukorGL`k#z$~-$_v6cH6OlMWXM!|Ff=$Ta6zkbSzfYi z$$A-$eA`44Lv+B$ckE5_sN|{mSgmM9_R5*j21!Odg+*E5Qa)1*S|^Wr#8DKKggj1Q zH(?Us050w1KC6-Fi-9>>w;Yj?bC@m-rm*LESLO~cG~Rw!31z@*#09aUkl_hX0d^oZ zI*3PB79}Jr?h<_HUw`9q!U7^(RmwyiRe}z?6^I?=ck1j7k&lRQf2@7zv|7!b$bTO( zMBU4@8Mav1v)Kg7hmLXBuICi}NE8F2knD2f~zJF8Pm-LYyP!_ zP3p4GYuU=zOD457HzMJ~aQ+>Eo>pvrgxNvza?tFS;aDDh0q3*^%neqcUGdw;T5TP03ptbdh0wrD~BYF|Q3|GO(tZ7skn^q*7(;aUG@@`HC- zU9G|UVL`9n4B#uy?RI#mY9WcKS63QpVuK?5Gl1m%S`j2B@DLY@ses@1Uq4-T?9?bf zXTpce*3|C^S>Tp<9?wKr1%5~;8zn!*OQb~EKoD=^4(p%TW5ix5818hZXhd`jnW-=p z)fHolO%uLJgS%qm1MRbSeq0REiH2M~K78h{lZ)@^H$yCau+`o_7oT#SM*KpAmlsT0 z%nO9)oe2sBwFI9Mz4%09R_J)?gCKXGK(3)2xU#1B=gPY9#Xmm&n>}u;wfX895JEU2E3hohgmkjji-f$cE@cIeNuca)aeEGAi9}P@l^U z1gAh5@;qnceT>^jh+l6W!d|qBA0n@j#CNu&UL1nTq>vp){`YrsTlyB5k9r-Y38qoW zC|z&_wGVai$ktgZ1b&i=M+u1hIZXr@1TsAW0ttft|00DAQ*km{U=;PK%sK7}sXuT2 E15XPSdH?_b diff --git a/summary_results.json b/summary_results.json index ae33403d1..dc8c50231 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.48247, + "Performance_Result": 7.67588, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73636, + "Performance_Result": 1.70059, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2638, + "Performance_Result": 23.2002, "Performance_Units": "Samples/s" }, { From 19108af7f61470cdf3972df6f4c477ea2931e147 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 13:16:45 +0000 Subject: [PATCH 621/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10933 -> 10930 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 737e86847..5bce33a96 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 12:14:38,301 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 12:14:38,301 submission_checker1.py:3308 INFO] --- -[2025-05-22 12:14:38,301 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 13:16:40,192 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 13:16:40,192 submission_checker1.py:3308 INFO] --- +[2025-05-22 13:16:40,192 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 12:14:38,301 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 13:16:40,192 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 12:14:38,301 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 13:16:40,192 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 12:14:38,301 submission_checker1.py:3324 INFO] --- -[2025-05-22 12:14:38,301 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 12:14:38,301 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 12:14:38,301 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 12:14:38,301 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 12:14:38,301 submission_checker1.py:3345 INFO] --- -[2025-05-22 12:14:38,301 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 12:14:39,388 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 13:16:40,192 submission_checker1.py:3324 INFO] --- +[2025-05-22 13:16:40,192 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 13:16:40,192 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 13:16:40,192 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 13:16:40,192 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 13:16:40,192 submission_checker1.py:3345 INFO] --- +[2025-05-22 13:16:40,192 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 13:16:41,215 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422993 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.67588 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57961 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2002 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1717 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.82399 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.59831 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 600fe59b5..c738463c2 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1248 +1249 diff --git a/summary.csv b/summary.csv index 385365125..6de91b562 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422993,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.67588,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.57961,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2002,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1717,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.82399,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.59831,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 862c28dcf..cc3d6eed5 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.67588, + "Result":7.57961, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.82399, + "Result":7.59831, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 77a636909f4e57269543b443e36a62738884471a..db74ef884e8fa00a72989c736a854a862296e3be 100644 GIT binary patch delta 2713 zcmYjTdpwhUAKuI+C9CDo6q}l{jI>6FCpN9jXd$O{5Y>1`Bc41canIp#c*g&Vi%8DP&fGLVbJk)qhoml09qnr{pk!FDTs45Sz|23~`1FKcdtK`2ru3UL zUTx2%vRjWOdHO!DW`~1L9<{l9-wGyZez$yJNFZc%sqgRW1G)N@NDw4E30%)6ix+a)!;IL*2m&o}Il8x=*oAmh_w zNnjYh(r&9m6CLI=+;%VjEt z-8R+VeC9K(2nXZxb!r$dhWFcrt`n7OvAa8I(XlF+NMcrA_Jz+p+;=MQcIe3!jm&SD zN0sNh(xsWUK7)4eUrW*Td)f%+?$AF*1M|g%3lE1rniY(OM!S`!qxT8e5GEf=-bU-3 zy3&cW#}=fRG$}V=I)zrI#_&#Ip#7l~C4R!FvAqL;wD0a?1uLhf!bI`(*g$1a`o&Ur z>u`-0e9#4dNjjP>VDmsMyL>8)yBlwp?+@dGiiVxJ5a20IjucWQgf;+_w2`SZBAr-# zIB=mg=NHAo*V2jlq5A|{oovwKTGH)9N>jA|`QLC!93Pu#7NV0qty>bzk7gm$DqCK9 zU_Ve@o#{~WgAr43UjEGprPx!YKF*oG(O(wz$i$fClkZ*?^~_{4f!VY7SoM7N0qn2| z-_g!;z@7-exTsvXf{b<$gZkmr7S*q=I=Vh}%2_$SOE||tb49W*388gCh2xfr9=F|H z)79f=tN$u>n?FmBg`bKtWHxIud-ffx{tANnVfc<@%K-->j6+f(UJQ`YlzJ^{H*U$I zdK3h8c7oPvT|X~g8O0<4L$;@-u_mUTK#YSwa;(UR;v3?tp4fT(iKTi>S6}F!z9rqV z0>jrXpQDkS1K~5NfjpNty^vh|TJ`m16A zM_lNEHXV9kohO~qG80uucjg)Wf$}{<#jSH*mBxm+=H;*?mt=glaPj>$z;}=ht@Ri^ zlXhkI+mLKti5a_F*i4Jm4YWUk&#EP|FEWn5EwCyXdom?#8VV;|IWSXsBZ(WI=N)DaT}t)|WVwhnEt*}BP3UAvB2-3#m~B5@pWMZU)OtW% z1dwt^NMc_0LC@7Fme@aT1c&5hslf*8B-ybgdoo z{Y0NBpcGEsi)ibpiPFBz$-dpSXqrzgpx}~@`Pcv~1aKRNb~_o20$5tSik5HkS4&`) z5RO^Kt>?>i_C)pB#!NK@&1lswzS$IOCd{DWNCc z8F5EkvQ(oJ&LG{dX!&}=7;+guK~MS&vob|%`W?2ItR?MvgxbF1V6CkyB`ZSBWGjvs z9G(DhN9*J3!_2XPXFAU(Ml3(_lL zka(3XYLH$5evFKoNlNo3If6UOVv1UY2JRHLGr~8x8+$ka|`*$$n@DV z595#JvoumGzXPdlTA+UwG=t1Qbb`FZFlsk@kz*4NCg}2Nxuz;rIC7KG^w_U1NHzYg z+U+PXsa`OljZcMOz;NT zV&Qt7f=lN5xPt}{uwU!e1RnwhEbdpfEXo6xL4T{Xxru68Yq&~$5?Fp)&|W4@_Sr1152Vfs3J|Ve*^T94mLoPGPK}th^g-(AlUDLi*<#cU!wq z6`8@inx^_)r%w8wiB` z3UsabBvjv^W`^a!VBZRi%dOD_w4YioT@jXQmM>}%Tb^V&Yo~#RSYc6niIyxX5ym1l zBXlMB)oB?T#|p^U4P-1wTR|S9`(33S&dPh>DE+7Sd+ow8m*hqPD|8Yty4K4SV@G_$j`kPFL|5xl#FUNG zrbQWs&r?*3TNMlDlUWkiCv_sH+3~Nm*`bHsFB@{B?biEQn|o|tB4uTY_j8 z&Zt^=IqbA_qW)=fTl|ZR*E{ci$2a2;N~HB3^{Tp+1_Qcqzr0T;%>?5(BCvO@6c*ZY zJz=lkk*0NBxU~%?&LM@1ar(`Cp08i#bm8`Bg#Kh1x75GpAji9u$nenoE7-c{-oBYR zd^|PhlUKwb2X3}Ifj0T>Wtgg@F>JZ%=e@ZE3{QXvJ3ZzgPx1JfkUh6*YxsB@m0DZ#XTEBG5hdvw4E`HSeoZomN=N>XfPb1r5`q~9zQPJe;>$7D4#}lNEPrHw1a}D=Tci;N-RJ6Rx zU6;1Z+z5@ER=T+_LF>%%ts1{xkAiuNqfmt45Md8ATaZO?gvJTvgxwl5CITczU%)1; l+3-_MCbPAYT9H2PCr7fEgeda&@{1=>r^`-y- delta 2695 zcmZ9Odo_x|I3&ilO2dA=X7+w;6d)caK7DK!-j zbfScWj}vp0t<#FnEMy!; z_08vp%7%H11HOZzM;eElM)xtb%KkiM-|>~)GFeqm^adBP4HGUqEGA-6Igs>3f) zb?)y4onp^TDDy?=5&^&$RqXA0MW+tl4<}j9gMh?l(0A(kproy?$JAgnlB%A&lrBB4 zF%CxCi7~*}i&j~K z)%XVg0sVi@h*6aPMBT6qwi>xvcy9iS@cz$?D&{CI(sM(XiW_pcE$uUzV3Ew^XbMUA z7K{#qMd@h6%!aWjPOujr(_zdmUdB}r?n0pZUAz9eF4bs^kPmLob+?xOMD$jidHPm# z2G7@4BS!l`s4Z%Oe3DJmmT0_P*A4KWzJOR#zO0*p^jXSmHP#q1)Zd%1FjjUYig=~F zPUOaf+KsQ3B+Q?EUO6OC?#AkTcz$&_$X=Lw@RHepAxFJr)Cu1S2CFo%=`2dI2$_p{ z%${d4`6vW>W8O}N`ml-DYdf~ca9eUXcQa8KK3FjukIwx@+B;%DA8$_eoD(O0VpIh_ z{Z#xuyC{`^$&89nh|W0SF9oZNvgrUNc&tS~7c;?L02urd1p4}_sMsWM!GU^>+FD6^ zd#xc@0-o;KO4I0>p1C>ZJu&V25lb}an2AF-kKIgyV?EK4rE^aB^Ro-0V1DzI@lfE{X=E-V+( zYMyWE@DjMaab0_frW69n)C?5k;bdN>F9%HHG#kvj=Hifq%S&48y;c3S;M%n*O#YYH zOV4gr?CqYk;9X!^h}ZhpsMIdykhlx6jQ3ULBHjSiy7fabq~u@$ z-5MQQ;Dh!kCcSZ0UVRJ5)ZFdY3LMjAac=B7xH|;T8ZrS&Nq`gyIFen;O+l)k$xJmq zMU(`Tgl%G-&RMdJA3nIrSG~Jk7^cu(&@ zaFhj@zKYONOdQom2-nEiZaS-bR;xNMM$35Uk&?XB_tRaOAXZZ7c6$5){kU&K3fa=- zaELr(A5wzgIGUS*IYgZDVNRNHwPh-?Lb*jcxy<-Gu?USyF6vk{7pS4_jGotnAk@Y?%8cl78FWyhx=ysqj5yAI zY9EtW8Qmun2V}$~OaDw9gIkIbZ!#4+YrMW0={naW%{~6F;o zQMwKaF5x5#S0YjIr{bxx`EtO+m0$3PJE)F>@hjhVaf($8VsMA@vjB z?gt%K5&Ql-3QOK&fAg{~tj4ucw3PIe0Mk)$Y%W_ext_XxwE6id^&#IGei6m%5gh*M; zURDvB)}MuliBwF;L8$7m1}N`%8Kq@nFy(T`SVp{GaZf6{|89kGcjIFHX>rpVIAnoJXVg7%F`-5*@l-eu_GN_pF#3-?=k&R5JDU@G3c=}N`>gGC!!m3%JBY2~@q2^zWiTJYUN=^U`NEasHu z=W*xh<#tIN&l1yTa_H8;7k+yCH|4KB0p9|eboVtzy?s`6EFp_B zw{}qdo@*A`^sU)Kb6tV7-=CpoD^mftJ+E$j3(E&6yg%*wrA71!$RX5VhGgRpaby34cR5?M6` z_Da)98J-nu1(Nac3ulGW7tD9p*OqUP(_WRuz>k0JOC27n3zwRo4eKm?-Se^gP;)45 ze=HVy3v0FYsnYz2k4h8n(QCaAON6{QV-~R>DRCnFFzubP;RS$p$<`EcrF{Bq@*O)# zc4U3p?89fLI4`*EAMWa(833VOk)6${I)R-D%j$QC9yMr;Sn217kvnU02%^DtH?@Og zannp4A?LZzOl=^uTy-o)|3}^+0N|kn0PBJOF4p Date: Thu, 22 May 2025 13:17:46 +0000 Subject: [PATCH 622/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10930 -> 10930 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5bce33a96..061a97fd6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 13:16:40,192 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 13:16:40,192 submission_checker1.py:3308 INFO] --- -[2025-05-22 13:16:40,192 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 13:17:41,467 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 13:17:41,467 submission_checker1.py:3308 INFO] --- +[2025-05-22 13:17:41,467 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 13:16:40,192 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 13:17:41,467 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 13:16:40,192 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 13:17:41,467 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 13:16:40,192 submission_checker1.py:3324 INFO] --- -[2025-05-22 13:16:40,192 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 13:16:40,192 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 13:16:40,192 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 13:16:40,192 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 13:16:40,192 submission_checker1.py:3345 INFO] --- -[2025-05-22 13:16:40,192 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 13:16:41,215 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 13:17:41,467 submission_checker1.py:3324 INFO] --- +[2025-05-22 13:17:41,467 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 13:17:41,467 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 13:17:41,467 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 13:17:41,467 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 13:17:41,467 submission_checker1.py:3345 INFO] --- +[2025-05-22 13:17:41,467 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 13:17:42,494 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index c738463c2..9c99798cf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1249 +1250 diff --git a/summary.xlsx b/summary.xlsx index db74ef884e8fa00a72989c736a854a862296e3be..7009578d8485d1587ff3febcd6606cfdc1b51589 100644 GIT binary patch delta 350 zcmV-k0iph~RkBsE-Xwn@f%>N<0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!E= z!!Qs<_k2aqfL3xL>B*Q1K2P@sxhEX|lO5uA0SFJ~JcJG~-ja}CzT`^ii{tSPvU*5)5W1)sq zF+g?68DSQv*S1J5@p%yr*HY)#;lx`3{IL)EmAAtpQy=gs1Rzt~qw-NaJ#SX1&eMDm wr;9kxH(5cldvcq94I?l5Pe6GX@^9i1Fy+4d1oa=YBqj?41R#O>r;{@(f1~T4e*gdg delta 350 zcmV-k0iph~RkBsE-Xwormh)mI0RR9{0ssIJ0001FZ(~q$Z*X%jV{dY0E_iKhos!FH z!!Qs<_x%c^y)D~mLJ_tDZON*ELS0(A6VfCRwH`=no&0?jzmkAgU5@UZGj|3R>!WO< zU$9PSRiZ3OQ3Q%*}sxhA@9^;nivb)(db^04Yb9 zDbc}uLojyi0Ft^SI8+dK+Dhs}Y4_Mr_D%PI^E6#y37&H5DLw;ZGZE2%Q z&M-6}L3xL>B*Pc42P@rdL#JFhrSK;Mv({0Z+(#!SW81b#yXdSTe}+G|FK^wdu~6qz zF+f$p8DSQv*S1J1@O2Rm*HGuT;lw)u{P7g@nYUq)z7Kd50+2rLsC+D*p6d;&@-$z@ w=`zmqI$MxzMeftD^T?b23s4S2{!JVKeeT;&Q2hb3Bqj?41YMT%Vv{o}fAM{u;s5{u From cb47d8a92f59155d9650e3e52f3be0c7fcf3bc61 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 14:30:38 +0000 Subject: [PATCH 623/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10930 -> 10927 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 061a97fd6..64d94011e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 13:17:41,467 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 13:17:41,467 submission_checker1.py:3308 INFO] --- -[2025-05-22 13:17:41,467 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 14:30:33,206 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 14:30:33,206 submission_checker1.py:3308 INFO] --- +[2025-05-22 14:30:33,206 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 13:17:41,467 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 14:30:33,206 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 13:17:41,467 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 14:30:33,206 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 13:17:41,467 submission_checker1.py:3324 INFO] --- -[2025-05-22 13:17:41,467 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 13:17:41,467 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 13:17:41,467 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 13:17:41,467 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 13:17:41,467 submission_checker1.py:3345 INFO] --- -[2025-05-22 13:17:41,467 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 13:17:42,494 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 14:30:33,206 submission_checker1.py:3324 INFO] --- +[2025-05-22 14:30:33,206 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 14:30:33,206 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 14:30:33,206 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 14:30:33,206 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 14:30:33,206 submission_checker1.py:3345 INFO] --- +[2025-05-22 14:30:33,206 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 14:30:34,277 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,14 +27,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.29378 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34203 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70059 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422993 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418438 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57961 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9c99798cf..3a20daad1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1250 +1251 diff --git a/summary.csv b/summary.csv index 6de91b562..c19ae09a8 100644 --- a/summary.csv +++ b/summary.csv @@ -5,14 +5,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.29378,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34203,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70059,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422993,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418438,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.57961,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cc3d6eed5..78cde6706 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.29378, + "Result":6.34203, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422993, + "Result":0.418438, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 7009578d8485d1587ff3febcd6606cfdc1b51589..85c47489ab9689f4db74ba11d20f7407aaed375d 100644 GIT binary patch delta 3800 zcmZ9Pdpwi<|Hn7SxE(@wC1*CoaHl1qMhP=Rq|BK%ky9jR+d{F#YLjIX$|)htDUs?n zW#lvx5h0|7Y|a_K)%W-P=X?EeJs#KRkJt5ny{`B3^>|!#mnxU)h&{ri_4WCIT7$nRtuN@6r?Rg>O{#^&@Nhy^BzWn~B%S zZqd~`F5@pQpsqM8HDq0i{iL@@G?(IPl+a`pS-^-mqBa}58Zh;_kg!hF zJoP}oJ{zud{WwT z9|Idpz%N~baq~S2NIfR*ta?)EN07R*MVO7H$CeZ!?bniohMULdG)u;ZlUg$2*PB7= z&lcTfWiGBK@sD^JmS#J6?uKAbvXsXrzAeo_|Homg;F|FRAoU@>Y}#roBaO}wocxe@ zYuhpbd24S_>?Q4`u?bw%t1}Oox6S{N)aK7PsM1^RHeT6>a4Tz|PDd>k&MEj4cT;p!(BZ2XW)Dy!i37_-3rEpud?G}a)v3Uk1C~U5Y zYw5FJ%t+%GfjeW03{)7@o(5B@Per%wRAz!_-VGV&f8x9%#d=oG7GFrclU*qAPRcm? z)Lf7CA|rX>gq6UN4Cgz5Xpnl7h741QkY+uhJAqd^d|G20qqSHt#(m+i~=O72vGwJ#A8&%Uuj zP(3~A@!UZ%M)Hq+8n4v}Hv8e0^fVa^A$7LA51>&-acCPq_}%o8`HY>BWw21J^8K@( zPN4kQmi9Xu`L9w|1&+_Jl2?4j+}D1d_}RKTQkOa7g8ykamed#458&= z^gJw|Y==`-ZYl){D$?%=-NS$gpu(>Rttun6x9STP{|0=2RNEAutWjqfxvvJ7e+EU# zVt!Pdpg*hM2#l<$$Mf(UUl;_h$XJHpY@#_qBE|zL2$hFO(@nZgG?LAxYrjh5$`}+*RUOiuQvdQK5mpGuF)L{XGTX$AufR zcBml?rurgjZ*aEzU&Ge_G)8qM*@e^(%Jo%T%5Z6TzseZgYM;CCB_76KIh-Y zYaR}d2|2It2l$EfgXGQ0XaMs4UTD1-k|p>!@EhOCG7vc;+kXy#L=Ay4Kj}K`PYoa( zbqj{~R|=}&*s|mNXmeT57P3QW+Zn9O;uE;YM5d;{fCHT%k0zt>ND|P9+1%Vw65>Z* z!3Q6T`xYFU;N#q28JT5HKiu`ALZT5VG1(BfTR42)!K8nv=B@2$$;+1^xAZ0=5MGPt zwjc8NqW?q-5;S8+KU4lQChos;D#=lcL@oItT)s9fNTu^b08r@_h|a5{AkXQB`EAkP z#_u;EFGr6`C@xS!Xkh)@r<-0&f@u%L1o&g#{XHIBWEd?VSv^l3Mq?$b|6}^bAJeCq zB@i~K9p=4mAc`I8h=h&8GvZ&{nv z)Mkj+ycP{Wl3Uv1%%RAgOo1DsZt5oD1!7XthucEPkr|`@4-fUOpT6e(Qn_At?OPwc zGsUF-(weJu@Z71kO#}==OQz z-bWRPE}v1*Tj^lhF}9gEvjoe!7v*p=A5pKj^%WLf_X_iV@v(eX0fCyP<)T6oD`uZD zO!F5WO~n9^{$sBdXqwZhYk&Z1kqY>J?$H23hqlj}xeLYR%`^BcvDv>ns#gOo-aE&B zH0C@#XS<*S%Ve8knFi6gC)Fvo=iz3Dd3Wu!D^*AqI=V41h~Rm{Dv>A9GFY8x0SmON zU5zUM`xU-6fSU6eP|Gd~Vnj?R7C1q8TE0!CQhoQuqigDtRUspg`u3W^C*&BGfO?Lj zn>k3(m41go!ho{*T!mS`WNH4DmZ6S2!x*%%P8Jfc3B1Hx{G~v+$g|FZ-PEX@|GTaDr5_Vn*{>y z*=b9vk}R}!51K;+9Sv)(5ZXwQy$ZqwP%-mELaG6TZ_r|$zc8K`ZaH4qi;wIU=#wJj zQVra@1axt}O(DAh&Bb~MiccA$+A@PdUXx~~uy)ks8T2F`|NDb$uK*BaUYivVg5>nh zz7>U7>^vF};@{OUSCcTFZDLVDz+ETE`iJfQ=2;?n;<(p{{E+^ntfZ_16$X;C<))(V zc8SkQcL6A%X#@m0CCvs@Lt|lwT2ulExDM5CHNTtX#=D*uB1F%RKX|Iy6CUU)48g@R z>SZM_roAc_6(vTo7o&V5#yI^_%W=E{10{Q0=)<~-JO{~tVWD}|Qg{S;rO;yQMs}J+ z(lbcbb(DObA3QppAy!bLiuCtwwVsslya0gl)LuzK=dLyJu{Ow3&ju3J=!eoPlOk^27Rb?md1 z2Cj39D*I+}6)y%HK!Pzeihu9r%AdWQ|Gk$aAQaidH`@n23_Fy?H`}2a(0Lj~xj$I~ zZ&4kF23^3m;`VoImC@@VbXG~z!uCFcU1MKDW+@^|CF6UV!F?6dx$H=+&7S^>FWc1; z1#m-r34GI62|(4HwoYCa)RSorPSZRHE0TytvW_iMk14<=M7vMA{2q?u{DX>hU5jpr zBybD+m6waTXAC<@ENL+{+8?V%Ids9QZmL<4ijny1zz8hV7}h7-A86~czmwl&>qtM5 zH?64J74GWQQ!_TrrQdst$1mVGi32R`NLMX*mS}a8K_Oq1QwLg-=egf|h zGt>zNV%meT3<=2IvAHOcpU*+sai-0v!{00(1CyVfrkry(Hdh(sF(-cBC z6>31nFF^{zbm<@ItvaS9FZF;krX@4q>D~so8U*BzCZEDWNEu(#31z|8EFhurwq#+3 z2|x$AzmLDoI&TF0D3WGT+H8^`gMM4Mt{$KGjssVZ*JGEbfNjjN^Y4m+5#~BEuSp5p zX<2w1D-@q@H9j1|3_cv(pf9RHEc~$c>teVd-9j)aI}PcK9c;&)$hsaVuZY&&-c*V2 zJysCxs#=!!b$5{7U_19QWY^lg@}EAiLFXrXF;W0&gSD$BkAVZzQuZ>%1@?9UNOqM# z&>w{FLgmek4*p59lbC%Xm9DQR2;y_TmZxAI1ZvY#YpR+eF6;p*Z3t#BR7ywF7+l&0 zRGEkyu!1oSK0b{Yc(KK3MqJ2K{h&m2s!7i$h=9Kbw zo>e}s-T4`DBvWC8f=U>>1MB6)KXDDNJ+fiuk!a?1$w&pvJ-3EA7Pq{akRHB-CP`pD z?5`YA;x(n=wjb>#M{k^&_%%OwYo{!Htb<~QRq4Lis93y_gk*mW7-}Z;#&c=?(2zUR zzO9bsGshLUA4C`RUiFYhIJ%7^8xPPZ!=~LuqeaQ7*k#WhZs-Hh2=911cqHNKB-6WA zWdE=f&htscIv5B5I0^s&G5`Po9d^kvBJAolgG*r%82y-#;D@BHD{cSmJ-f~mFMO#7 zqdKK5n7h-j6gX0{f22UeEh|MyZdy3+5&K*r`uB*)rQD{YPdpYe2Y5->PnU~h;J4Ks z?XErgB%TN@-;g=mh+So@PZ~;7%`dkuZqN2T^?iPWwf8sT4Mf6dw}HT(d~L{qRPdLn}_V%0UyAZQ&0t7 zqc$%Xl(WVj)(^c?<-&x2mY(SgYf(CovxT=-1LREQW%W*N7NZ`SIB?JA>Q9-4(c_(p zinNW|I?8vBIx44b(_#ednyO$HJkeT^W~ada|ABI~-GPE+hr?#T4Dy}B7Qi@i&tVhb j53-n!ro#W`&5!)gg}>2Fw$(WWR3j5~G(-lpejolHn#eiC delta 3793 zcmX9>c|6nqAK#iICCi*640Gj3Yf-{Tj?o+mtB5p0&dAM2NcJt)HdBOh`?bW}zLol# zgxra_iaC;FZIzpT`}Y0(@&4oU$Mf-gJ&)J>@qXb%cdEX%Uu63cQ0adN1lkD(fuup? zx9Sq)?ETU}e@;iZT1YNbOKik;Fe<~4FS$qju=zjOy*QOOc*FJQ^)p!Iu>$1UEzGuJ z)60XddX}xR1-3E~14{6YT2`6gr5+2kETfIHJm0jrH`xtoWm*mH`HvPdYQHvZs#xc? zKHgz}b$q#@+?-Zamvvj$COxmw*8t<+NPZiDUQq*{N6PomuUK$A$Lwl@r1Vz^uT!_$ zoCd-MKT+oz%6B=m=GEz7htiBlv<$RJisNxQ&3mzhY>r+mXZ|FQa}0e8jvVS>25d&l z-1iBah%FPb0~*ittaT3fEkjvf9WQLOWF4Eke&lx+@WVTDqK@k$OF!l^)PzT;3{X@^j_Hd2s%Itba4z{k+_l2cf&v$doO5qXh8N0N0zhuHx z9~*wV;&%8YfvBaRs8Hj^wX$jM>!(@Im4Vji2(>kbZ5?(bxrwC0CbY#czd!Bo@13vB zr_7TFbp>xmh~zx{ddf2wIkRdpm6h5k8)c`o7dQ(c@{Pamy%q48OUB6K(;xZ9Vb8C*-ynOw?ul z@j1ruN>*x9OAz*)>*T?If9bC-k$ zxeEqOc3FpW#W*M7XK8b3PhZ9W!1%7nZB^N`C{rS%S?k-itclPmQhfb;SHlmnVcf0X zQmt20f}(F=s?yD{;qoo;^2Y9r=ca}Mc52Mlp=BC5?qr5!@e;>hT$?U-B`n_p8=bF( zvF*UdZ(ys~kEz49QVw+n-t|9*E^ZqdPJTRAggnDQuJriMdyN11&}5E5SEYQO!?eyaVJN@G#2 z=c|p7Bc5@VuAQNFD6AjApnN9z`qqTFMV>@Og~m)O%-Z>i&~NB2RfzFYC^}u#y`WilpE@b^T-G_{ zUq~f~pNG^*w}?egU*4ZfEebLbT>_iC7g_l+%66&eZSH%yZTyx8#Fy~a{Nk96Gvh^* z16RvuPss#JLI@7_KaVKCj?B#Wzi`a-)f?P)(y7xAf4UgOu1yBT={>(5C4qP;eBf}jw%YfA zcKAifd%?px>f-Awp#X`~uDnhmB$!dN5cCJgECMBVadudbfiE$#lExW@HR=b3BvUif zmDbpy>PuOu&0N-%PxX?RG@SznN1^HpYi&o~UlZ@Cy*H~wtYTm6?rUJYct@v7IDg?z zdV5~cef35gLT>%GcYgZWa=P+c#)HpU9we6~@Octg7c1}vUSXU7|*JG);KT8kX2YuTF^LtF3mNC-^-pElBW|yGf&a1*(?m9W!o>?bD7D zsoaRM>l1&S&uW6by|Nb-kqX3)u%G^kG(Z&BNK?ffvVmkcZfev(ZHZs566b&Fp3HNkD`q@;pv9Oowm`~t}GryfJZTZreU#e z;*Ew`Eo01~BB|-ZxFXy=`V2D3$5}T5FVIg|6?x)Iioiot%Q`JS` z^OCAU#%YeM&8t4E&qj*F+vVl(&X4TN!#~I`-NAi4>sq;%c^)w-Pcqfh95vE{_`|aK znP|8%jVB*+Q@!#F@YvMmy+zi6h0~N@K5+j$#>Sn1TJQV?P3A{CgTd9xh0w~sitN@x zEo1nvhfCpF`EVc4xmIrRK;0BbrfAJkV=ageiX|{LiiTtA1WksJyXuv*JWwkOaIN6~ zqG<=haVTKoAPJTHGDBH; z@w4xFC0Nf7gR)hB-CF})%$-%>UxU^q`-^Yf75KLU3{qGcel-tr@vqW#q}KXJaLa|a zI@AWXph{k8!zrrFZJ5ht6Qr2c!BhgshMI+^KVXmul%16dmoWmk#$h%n^`XI6zs!Uu zDgzDIePa(HE;L{D-%}$=OlwKTw)!srj!n&k%MVL-E+&8Z zbx3z_b+}NDPiE8Lj^ZPeTnsYF)maz73p63&cAma4056zO+POXOW!D6-u@W%Nkn4&r z5jG6}bTV?e#hWcyvoo(0qisV|qAwM#u^PfWV&a8z@?kmy3L&LyW~e(KI6h*W^Gss*9FMKOx$Ra6Y$7>Ps~!!*-jn>YgVvgD;jo+e~x*rkQOPKL@to}pYcC=#(VQG@{H##CKRq0x&(TU*1*V66num_5y|K6n7Tq(ljlb`hpjeI!ny~q+C&K(Mm-T)23qxS@hN+|2x zpSd;ndUUBYNS0{{m zLHpsBcXh{we%jv+fG9wmePxhTXH%zWFRH zUV95O=Ewh9C(7Z|Dg{AQC93Tz@CFy>4W30qfNF#hogZaP{wp??3??Lj`Lq@D%65Dn z@owe;lnJl>1a>pH&J-01BW)ku8(wWSQYO-F5>1$SY~j62ab^tnDeLQmRFN6kTb7nPBo zH?R;D4xPInpqEl;xs=gy^Me#C^0;j)&9VD_*LKCZE;^rVkR62a7yP!;xW7eS9^O=PV7_)Y(Ir90;jnxOPI_vbb z__qd6o$k_+=A!Cl&GxvvQ~J*~z~;SE>;NMqyEBm3SS%h=h?6TP3lV#=t8=# zL&Dkl57XR6RNRhVHro>hs@$jEIbDC6 z;&|NTb|?IrWW`8>4kt=fWaPra$TZ^$c&7%smEG8;{4>+WJY|G9hN}3xqBdR<1dWY|qLam}|*FOF~Z&C7h^($Zj@_7wI ta3-0kVG52TcWW4c*<^vEs{e_qdsGtKPqsX23`UYujw%a$QUme6{{tUdEDrzx diff --git a/summary_results.json b/summary_results.json index 71c1f2a99..dfce7afcc 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.422993, + "Performance_Result": 0.418438, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.29378, + "Performance_Result": 6.34203, "Performance_Units": "Samples/s" }, { From 1168c14e2aadf98a31994800f850969429b317b7 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 15:47:58 +0000 Subject: [PATCH 624/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10927 -> 10928 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 64d94011e..903c25707 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 14:30:33,206 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 14:30:33,206 submission_checker1.py:3308 INFO] --- -[2025-05-22 14:30:33,206 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 15:47:52,777 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 15:47:52,777 submission_checker1.py:3308 INFO] --- +[2025-05-22 15:47:52,777 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 14:30:33,206 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 15:47:52,777 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 14:30:33,206 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 15:47:52,778 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 14:30:33,206 submission_checker1.py:3324 INFO] --- -[2025-05-22 14:30:33,206 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 14:30:33,206 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 14:30:33,206 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 14:30:33,206 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 14:30:33,206 submission_checker1.py:3345 INFO] --- -[2025-05-22 14:30:33,206 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 14:30:34,277 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 15:47:52,778 submission_checker1.py:3324 INFO] --- +[2025-05-22 15:47:52,778 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 15:47:52,778 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 15:47:52,778 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 15:47:52,778 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 15:47:52,778 submission_checker1.py:3345 INFO] --- +[2025-05-22 15:47:52,778 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 15:47:53,750 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1717 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.59831 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.77776 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 3a20daad1..28c4289fa 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1251 +1252 diff --git a/summary.csv b/summary.csv index c19ae09a8..53dd3d445 100644 --- a/summary.csv +++ b/summary.csv @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1717,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.59831,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.77776,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 78cde6706..cbdc08fdf 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.59831, + "Result":7.77776, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 85c47489ab9689f4db74ba11d20f7407aaed375d..cd4382c76436f8e5d3e506810725f87f28fb2594 100644 GIT binary patch delta 3710 zcmX9>c|6nqAK$1sa;_rh(6*eZeKjQ-5!MW;u;fT1azu`b~mVYf!ULGC!F=;XIdxCf9mMdf?Q(A=q0V9q3D8b;<3zedEJS+ z(0dX(a-RKJVj)vRS(t8JljlkrZCBy``5(ZGxBib$aK!T2{wrvfd(=Yka9!MyT1Lv} zNzX9Wu?(va!p_20{*#U${CC)#T5i2+-D`WR>x$wDS)F~~*h`hY-r4e6pLUMLVDcQCbz|~K7F0~k zt#Og~EL%w*d4E5oc@Y7o2=QAJb6~HuRlPN`81cg zcC+uhi1TXMII~-y+n26wW`3h~^67=BD>G?%&Ws&f9q9nCV!5MBa`mvhhA?9S)s0J(Fnu zjxw(~@Pk#6!;eUf6*}HoWVU}v5nlu7EY~Lw5(*pbW({klG@4CRZ8tq8$8ss_>4|q< z6(&4MsSeERyjo+`e0}2-CHl{HvPsk4P$Lz-L1%2!P+SdMvc6XXDyd(Klw1}JF;VZ8=&iLNUPURnkSqroYF#<{nAoMk> zRL#^Nwr?g?-A(8In*P139aj5j90=?TyQ&z~v!x&SPn)ka%we0@^YqaQaD+=ZvU^ji zGio50eeCJXk;)Cd*xq?PN7Ca*J(Is^P7_sHMNitN?%w-+Q@WsV?OtH@o9s`XVbR!; zx!%w9`7?7n-oahjT4Gf8>I&(2J7=n|UN30OD!;KCzcTYfsUsxUe|U@i(g1+Vt=$UJ zRKrxZaQjksH8KXi+;qAb)ar3c-EW>8FcYvynzqy*e0m)&(R;F4(9cYTdK#KEniPF^ zwU#3~m~`7{fyY&Vl`jJGRXuUd+0*yU5^jK zqQ|HqT%Gy2oDchjFLe-&n@kgDE)e<<=QtUk96q1qaB=q9el#ZxR=lOwO+1@_$srp* zXYaIEP`gn}a62`7*RFvkE$Rsbi|5ZpJyi$O>SlK=z7@^C(Wq~-_-cNPS@au(WP;M+ z{4TDL^;Udz1UAI*n5aQuGU>)Qkt-^CG1~RXZjQE)P=rvudsPMb{+Sf8nx{|dd74~-Vz-=JfMP+Y%aL)^O`91neThxmgZ;n*nvt$qZvMZSrs`vAM zZJQ0GdAlNi!L!nh&{)VIT*ie~^FG#l7>UM;x8~K}FGK=2b~74uKaUwc$BqZDBQn#Cbg+;vxQyZc zfA-)%JwUR--8!pXL!%qyd~eE6RdT-O92GT+#tyvm(c1;<%-LVh^XVNLL-U*caT{@` z#6Gt^>SxSExifz(Q6zhwd|1u}oBC6csnKyN2Ri~(=Xy|sXv#AG zhGKMr6 z-Cr&F%#9oaP$rUXhDFTqEQAtHHvp`^xLq#sqH?F+5X#3^1Tz{|xFM?ReH&Fa@7ofl zHshT+ECqGV9Oz}J)p-nj|M|HnVq%-{xHt=x^0^4Cj-M4@0Ke2>Vua^m9+W}VU;9L@ z`w?!@2)1nLsII`v1GrtR{bC`1U|xfx=7N$S4vq8i*E}%`f65@6Ba@{72_`62^KdCq zhJs+Kaz`!{Gnjm?xvdit$Ah}MmB2>0rhGLZ>Zg2LHN!&jI-ymHR?~ID!7w&~s}@xH z!{%7V7(P{CXmU?cL+Vb5x);GQWj}o!O}@l>tNC_%fWJaL9Eyip$r|!X=Z_{zvR8o0?60ITBp5+PtgUZU`YZD$E-Day$Q_b54*gBXp|6W>_V&6=7qz{FSd}@~UMpE)n>t^LICw zdF(;4H~^$mAwdOglA_+bU^ok=9nMEfDme9*I{EHhme!EHo+L@+m? zINC;|9!L{Eo&Z@A;5AAxWd5(JUrWp1)L$Ps5sHcat}8v{MQ}ZstSw>=WWo*`5spK0vljgE%}_O1Y= zUaiudJ#+kM9YxcDrK0hK`1UpIwL|#++f2o-ft5|N7Es^RF)fA_z}g;gQwBw6X=Y(8?VnGvq2JAN^(JxOqln3fCot$)9gUM1O@0rm$TLryFna+q&+XkV zTZXu%qjJxMTEx{F9ch7a@7>#5I@gT?tWtQzz1}+T9>|BKZa>j-oZRwYn3339iI2)& ztQwzf{XbWH{`^O;awIz{j?XXiF-){WoM94}X_NAA*-?$g`?PeUl3V?n+a!a>R;PLv z3pWD_-+sv^O)wbFYaDM1H}`__avs+h)kd8a7gEkT*vIPzI^!L`cV-_9EnMP>0(8;= zOc%7?Y=Ver=+TQCn0qwfm2H@$O@+HUe7h3#O?lNx>pIxcamV* zz(heU(aPjrw2-0q0ilAz$aqmc35JPx=GRhVLRu;_j`qN1im}ez<-RvryCOw!n_|2@ z@r*Qe>4W+O_XVhEt7&#qnyds9P{oB9!XJ3s{v!V!Ju=(}y=YHEI=^A>3-Hi0Ui?fT z-~@$N)%5eeS@UAyKE30o_x&7+nXjN1x5azj!WnYoJ*%qc2XOPyeg9!fNibD$H+hhk zY`b`nFX1xJdWhgsG4grZUPje&`1um@pc{XJLR<9&>eqyz ziO!qH)kdI+!iptu7)Ax5M7@y88x?}Qq=8F$N!yY=*_x@NgckPm>>+!1k)I)4dy`>7 z{$g&9S0DmI!obd_@c>a(IVtm3N4S^@`s*HU(8zALt%gV-Gq$YqVgx)B{$Uj`HznAB zRFqACvItNUiVVN#1;YNF2VgiJk|+xKPfAX%TSpEZ#{w~hA&Ip!kSdSHOhE%W_i&@AM<-#x&)u&IJ6-Exn^if$O?h2T-h|o87cDcj4=%lW zr!s&0POgk^x$EHNrtYR|_ub^Sw~&5^hq8*xiIrPzKIk(@LaN1L-TS!EgOPWgNNKw@ za!73OF5VxDhF!hOkuK+#8Wh`eUb?}OHUWnL?Us$lBpxj3pPLSygd@mwcnavZ2?8Er zII0<%b)^YYCh9q%sEh5jouy2LxYs>#th%~Ll-`aAI$I>{hK*6Nwg|`>7OvGT+5n&H zxY+DjaPX3cxR};l7FDK!@vmKxsJ-^bbKn>9*YeLgEQ!_+mTH;LQomXoje2Mlis$(N zr8)CimE;wj+D54U_r8lHP>VbAxe#;ckq)L(4)pliPs0d~*bVGMD<^T=>LSs|+dq;U zmPD^sC^iq(2DqZv?q{#Y*2{8kYy@m`&~({-K_?ILUrKu{=^h^9GY8F<{^q9;6sBNV zBl!`wZdAuXYoXS_31>m576WxyY{3k|{oh&r_SlnJllr&HB1Xq)uFFA(?sWrvZJw=l z6tj=%e>nU5fO2JI8{gjlK900sst({iGz&F-a6FBoW&{qUvDEay3p9v2^8czHQ{Cl$p8T@p_WU!K!`Nhb*G< z9cE_cNX8MnF{`hNo#-6=nqhXm&BPnXnaGJ?=Fi$Mj=!5%N=Quvq*-5-tQs4p*Zb=6 zWWx~CY)L%#95LG4JNHw;n$6JM^R!NE;={9qsVl>He-O6|R1jS0IJfdIX?Dmd%C^2C zF$N+t+kBW6`hI3Anw@82#{CjUUL+RJjw;%$l*h7IHg!38*sASHALUK-kk-?P?hYTv znV<<7TIWPizOP!L-@9VUn9i&4NEe3F+lAH1x2PO{_?K{p>xFnqT>VNYq#GNVnJw2` z{6^{c@!0x7{}QCZcG5MXSA4CO`$Ve;^0J#kQ_jW2&$>%^y_p2xA-}BF{HaX85!2IT z^}?g<*?kLD7sgNiaAt9+YndaHcjtm47q4~BCRwTa+eWW1IsPo(@Eu$+i3YM{LJx`VSy^zqDAPtaz5bBVvM z!4b)miUsYvdtEi8K?-(>iSM0pCKskf+AtTVFIE;hd?rTkE`y+~Z~lGL+rO?u6yCBq zG->}!(PZE9Ogz!WOCG~unKq*v>l5Ym@W{n!$j7sI$4~2Q<336K&>KnQ)1zxQzZCXK z?9_Kz7(B)l=#?Qc`6iy_FMiWwjax%^$K`3rD2O8ks?eB*MRo30XM<yyTg>-P!Z`ly{0G}5TFT;S8$SIkw|jtikV=c{0lGp)y6vR)Bu+u|$Z&^# zPJFEoz5082n;*wGyLRHhh1GGA1L3ItY%|KsE8OP@Cd*-B+;4NI%Z;4Ek$wNXGKG@g z%_QmMZAR5#mt%Xwkiy*C3`ap}mZ!a-^yF`E9TSV*q1;{K}ffLU=6x^3wX};xC?}Lw7iR$m`2cGvnhBlj-I7fDTnt0*#g9NFUbIc7!GCtPz%0-A5YbI zR3a=eG+<>*kC*;X1b$k)CFOt|L8EIg5cY@XdjB(Od!jk6C)pvQarods)x|9LruXZ# z;hnB|kFPht{mO^2%L{q`HDC2{dYX~<`e8_r&>%?GoQMKo5%2dy8buHc{->eepVYUk z01-^7K^Fi5IRZxiqH3`|H-X4IP+YJ=Skv|AnKIg?^ZX|`1#keFoiH%L7s|E8p zQ8Q#wL=+A|02)-TSH?iTGffK|;XlG3HX$y>kBP}Ik|HSJ<997vUW9D8<i_asqxQ!40cuaGN#n&0Ps#A7(%5NOV;ahGVdK6S75Qc|U<~wDGMsXXW#&&W!!RDiIh`qlH|p+ugT^nDw|W3K~7h9S0sp(vaAf!g9XRZ&;Ug6_-i?e+DzI8AcS0^2!5DrQfi^8z-9Zwu~=>sLjI_Rq7PjJwUu+b?QCvstDXx?cRV+En}VFf&c=eFu$dMS_Kv zb^=s~|GYts&@)H{v_ZIt0ov23#TJ2sieKwN%#oxV<=m1mTFj(;k*f|@J+P%*V&I`@ zd|g9|k`7a+v8!(Q88Lyur;_LFWsU{$ds6Sw2xw3)sQ6oy?MsH*Kj~R2SS#AFg>8y} zfbIcG8AsviRSg!yAn_hr+1p{&@>4ia_v~Exa2FZ=1{G(7Jci0VSS9Mc!>Xu4Uvqb? zy*>tT)t#h{6Z|EIp9|PSU}m9!2M!wIN(2iHZ5eYNerJPv8@L8SXuq5wwg@6(epEmy zB;z}zRO=tKug4u{kNvpVKE447A~sFWyO&Qpuq9$Iq^(pJP70`i*IH-Mh#QjhRK~8d zERCAX<=sEH{xSegZvE&yzIoFV~1wk3ISb_3Ua%thae>rN@aNW$RSi+Yk zOhcrgqq9w!x20N-YC#UhPA!KaNmt4{T%wo@G=K?pLrz1h_?3oAI3doMBCtxzeglhb zWpu)QG`3Gzf_h3DI+^zV%r(r5wI;S(yAtbmX$?04972H7R7?NN<=Wr5T=+AW1S}AO z_`nl80MUdV&G{=fq{k9TdN@@EYgZbDgn3{(r2Et>sEs;QMp?@u=aAl>abQMvIXp)( z>qolYLq&p1ZY;)b-(b~OPOVrG%m7yg+YVF!kkw{vQ&#wO5444+tI0r1#NrW*Q%mGi za?nZPzBBHB$gR5lBx5n28}UuC8SLW0V=JX>D}x>aLsCSU@>i>29+khQj~5t2(bu6d z7>F@+Kx#15-d(zf*JPWBAfY!M$oe(*`jxY@c5UT9`b#G~u&YT!3=Ffk9z2J)xlJR@ zNzygo&!M2ya1Up!C$D>^C2B)mv7;KV%jlfJfL*LP+`3#OJbBxy=gLe~(C&9~U*U4+ zT*nU7YX{o1X)U@*Up%k3UC|)0qR^p`DGF&hWpG1 zr0PPG-$|f-z0`~xrINFTCe-)lm!+6tNYq2#Vg$FE<+lCYIaSM^*ZGU-#lzI8;6b_~EqIVc1QgDmC`dB_Xd$Erc*)Ru!xuyn zH;dC?5e#V5yMow;MqJXnRhY^RT~=i(*v=fY@UA2rZmyN^nsAeoo`bWqLGr3XnU^#{ z8ZT+=u&+vC3>?qDQW!tgfrOnw^ywQKEDBM%2qTt)!Fty2k z{!_%Bt#|c_0kB@rXGal22w{t{r!0$x0n=0W)3Ks_`+ETdhiV|`Zy(%;$eJ6<{I#+N zpL;rudR&(u#4C-B0y*=vVlHzeP#P*X7n zZ&^+LUYNhLTM<3pO~NSlU1*js-AYEVzJ-jmW%S=*QwAXs_hte+oGWLI0Lk5^Y^AOz=|2`ZFE#1*kt_Vwdww(qQn_{D+y}R=(DK9QzXK$6S#1Q_{~B%^Rf>D+y*>vl zkq<1hG+h>dGg_x$_k-}&*<$>%c7-JH{F2G0drBq~*-&iY+bk5kv_OXB3!6?CB(BCrS`wB>T5{@`-pa0XMxJZG%Z!3A$0)n`5$--6+i$0 diff --git a/summary_results.json b/summary_results.json index dfce7afcc..449fa672c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 7.59831, + "Performance_Result": 7.77776, "Performance_Units": "Samples/s" }, { From 6ca22ab53274aa750e107b0a0aa2170a18603080 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 16:29:17 +0000 Subject: [PATCH 625/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10928 -> 10929 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 903c25707..e2386b406 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 15:47:52,777 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 15:47:52,777 submission_checker1.py:3308 INFO] --- -[2025-05-22 15:47:52,777 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 16:29:12,580 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 16:29:12,580 submission_checker1.py:3308 INFO] --- +[2025-05-22 16:29:12,580 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 15:47:52,777 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 16:29:12,580 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 15:47:52,778 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 16:29:12,580 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 15:47:52,778 submission_checker1.py:3324 INFO] --- -[2025-05-22 15:47:52,778 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 15:47:52,778 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 15:47:52,778 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 15:47:52,778 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 15:47:52,778 submission_checker1.py:3345 INFO] --- -[2025-05-22 15:47:52,778 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 15:47:53,750 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 16:29:12,580 submission_checker1.py:3324 INFO] --- +[2025-05-22 16:29:12,580 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 16:29:12,580 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 16:29:12,580 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 16:29:12,580 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 16:29:12,580 submission_checker1.py:3345 INFO] --- +[2025-05-22 16:29:12,580 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 16:29:13,564 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34203 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70059 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.68775 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9465 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1717 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9392 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.77776 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 28c4289fa..6c4b54876 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1252 +1253 diff --git a/summary.csv b/summary.csv index 53dd3d445..fc365a86b 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34203,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70059,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.68775,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9465,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1717,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9392,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.77776,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index cbdc08fdf..2d2af58b5 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70059, + "Result":1.68775, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1717, + "Result":22.9392, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index cd4382c76436f8e5d3e506810725f87f28fb2594..f6cbc49c4e478eed413dc3f5adce0a41995cd17e 100644 GIT binary patch delta 1350 zcmV-M1-bgLRk2mDatH-bFLbe0?-bVe}^wy%K|H5xo!Vw zyQM8lmtd(&utbf-llBkEgwp;Y8N{-J{lhYVC9`S(OOyS>Mgc5M^3UeR2P{GUp?}i^ zEJ6MuWpILJUBd}>YXSdKEcey?(Yxl4lixR9*%{U@;SvoJPntg@6H4=kWRS}W<`2sN zmn7p$EKTAM>j1Dcc|RJwA83ilAn36}S(lW7$IiM2k6qX7{UDa#Tx73hiB&{)y{-Lt z#jkl;b_thG376=PcvAZznNVs!B!7ckR?vP}2Dl^{;L>FMunvGrlk}rO`hk{+41%sK zlyzAdo??j^1gF@VrJu*R{3fz&EsLyllHIm`v~C4XP)4R(LMF;1o|Jw_CX~_-$sm~( zq#u?6GD!x=G&w)410d6+{Af^qpd})M5HdmeA!T@wWq-{F4ze{XKi`MUJrH$%uJpF^ zQ@kMSg>K28hSIZN5`zJMZJ`~O_y%@~24tg8;!6#PWRxX6AQD10D|kRG18kCve8Op) zG$7UiplQ;8Y|wz9B_e}>#|~9pRwcJhc89>WY44o{^le4ErWw>g+Kpe@3{t)4GSdIll-4r)>i5K zy6yjHs}%;iO=Cp!_(ijRdm)L zUM|LQBl|LokDKx5S$+={i)}UrpNFfD$7{^yqhvnHzN|(sKA&$f?ww4M!H>&HEvBPC z7x9lj^ndUUf9QKZ{oK$$HI5tk((j}B>XxGvU$AMN9L;(+@p*54Gn%H2aXicyYPI}k zW4`*pz2(;?Su$IGIuFmxb6d55zk!xygyE8T3vs9ZUv@PmO_>sg%KdnYDi-}gyhjMk7p z!=Ia%w{g{2sKcokpt|IYFbmXcTbKB{2#4#a^P6zutpNTw1byc1w8*g!coYJVW8Aay zQ9M02C2LgYX}*ZlMV#kNwj}uj$%?PT$eaEPP@abTn|KBsbKib~`VX@rCdvW@P%m`6 zYm@OPTLA@=O({G9kCTciK>>V|%qcnn?vn>9CIt2EV!4wsDi#6HlSnE)0V9)#DkBEf ICjbBd0LG1R2LJ#7 delta 1366 zcmV-c1*!V6Rj^gCatH=t z+HPsf(j{2x5-d?8@udAjGNH78NCvU2VE?cTV9Bf+z|v&@uu%X@ll-%}@c~Pae}CvS z0ZWj7NEw`9S=Vra-CDrE6w7@zfAp^TkS z3uRqahNoC!2Ei$|X6ffKF29K^TgxIVon*JIAFW$~6O@tZmXL|^h$p2Vk_n~sLo!Ha z1?h)nfJ~ABGEL48>j20!DL)#NA83ilAcRa%en=S}WZ7TyfrD(#%Fp*9a}Q6QpDVrX z{1h+9dSU5nKeJ#Gg8_fzRK6-vX6+IU$VQ*Uml_bsC`)=kB!p~M@PJqb*d!VGgwr-@ zK&%5m)1(2}paDTkLUtF`HpG1y6o3W z>=FyeMxVr&77)oOOKLzQgj810fLI2oBpIO6WC5`bfJ&1Ew7Gxrfl9D|*kuEc9m=|- z46&?Y0c|Z{-?4m>$X?6(Dq#r{kYD;SDccf|3LdZ|A3q``Ad*p*w17wmv8-SLu?%3z z{2DmOnlvCb3UF!CfNao!pe16{lpl3*4rN_h1_#*+fBwhba}6jP7D%cm`9HO+t<##2mXL%d+Wp9Ch`0xK8?rYw9!xU z%eAW9w5~q!?ou6oSbaF$Lp&cZnGQz9U$XCwS081Uji!G$jr7mehva#Or@w2f=&V1y zT#VyJ_GK0yH{;K<{2nS6+iVOz4_6s#!A6$6y7qbNx*bW3r z7s5`H;Uph_%}&EG5QOhMMdW+pBqe~XB$WX5L?tAGK-?^Qn_9#_vJGk99ydwTR_cl4 zdS<_!UAbKDWEcH{bwaBOWl4%6P)u{7+6uiltM~y$&Qrx{rxjFa4-PHso01vAw1roz z4OlP0MFFH7VWvV`?+wA&u`NjIlHgE5+-NJQ52bB?v7zjnwt(|Ay~7eb<v+cb7D63eV_D;!5Z>s z_;da8Hmn*8bvP9RRF|9)W`TNb>k^+A;cy*wejQG{5x^h!LBH}gEi(22k3s-4#yu(@ zi>K!$W`*iJ&5JlK;yiD%g52JdV)1ntdC`9Y%4x{IiATVg`|=aif3qVd$^r%XT7DF2 zlkzBA0R)pyDLes=lZz=q0eh3pDLMh|lL#s%8%h_#P9*^V08jz|01*HH0000000000 Y0DytclSwK*0V0!#DkBEeCjbBd06BbeRsaA1 diff --git a/summary_results.json b/summary_results.json index 449fa672c..a328450c3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70059, + "Performance_Result": 1.68775, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1717, + "Performance_Result": 22.9392, "Performance_Units": "Samples/s" }, { From 9a3ba4deef33b200a3a254d78bec0512f737ac02 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 16:35:05 +0000 Subject: [PATCH 626/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10929 -> 10927 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e2386b406..baffc1acb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 16:29:12,580 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 16:29:12,580 submission_checker1.py:3308 INFO] --- -[2025-05-22 16:29:12,580 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 16:34:59,951 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 16:34:59,951 submission_checker1.py:3308 INFO] --- +[2025-05-22 16:34:59,951 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 16:29:12,580 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 16:34:59,951 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 16:29:12,580 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 16:34:59,951 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 16:29:12,580 submission_checker1.py:3324 INFO] --- -[2025-05-22 16:29:12,580 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 16:29:12,580 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 16:29:12,580 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 16:29:12,580 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 16:29:12,580 submission_checker1.py:3345 INFO] --- -[2025-05-22 16:29:12,580 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 16:29:13,564 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 16:34:59,951 submission_checker1.py:3324 INFO] --- +[2025-05-22 16:34:59,951 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 16:34:59,951 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 16:34:59,951 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 16:34:59,951 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 16:34:59,951 submission_checker1.py:3345 INFO] --- +[2025-05-22 16:34:59,951 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 16:35:00,995 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34203 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.68775 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 6c4b54876..1e590facc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1253 +1254 diff --git a/summary.csv b/summary.csv index fc365a86b..f8bfa574e 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34203,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.68775,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 2d2af58b5..0765b411f 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.68775, + "Result":1.73438, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f6cbc49c4e478eed413dc3f5adce0a41995cd17e..d728bba09ecd5c53eb09fea77ec87fe568ad7a43 100644 GIT binary patch delta 1749 zcmV;`1}gcnRj*aBatH+q5k{41vv&ww2!FRre``dm5-8Cm@uUGnGNCkpNCu&-U;wcU zK*`J+Eul0SKx`C%(j)+B5P+a1qSKVBER=Ow8A4e>0Ma!JKs!wS9lmTW3#^3Yw*8~+ zmbNTif~79O5;YP}+CL-{O8bXo5X%bo56b|S%&Gw_P4*8P1+X;9KbspLumt&sPJa`y z1o?-Q!3mai4JX*G1^i2~+*k8Q@0veOe&2XyXIQ(0OEgG4Y5tH*D9s;|K`twpKP&@W zl8i60G>Jc~1HjVc{b=xhpd})MpvMknT~Y=fJL?)ec3rdggIIoZk-e5BRuS3tw)W!{ zzvgAxC0sfsT%teXN$rPZLaF_b41aQ2LHl7D;F4s3OOy4(Ish(B(vJq|2U;RB2)eRR z)@5aQiX~Vx)nG<8JTVgnJABVQu-m8P)a`}gJf2a zepm*`BpD#nfJ~F}qe1zBmWT{O$OPqwl;J^^{WTvr$kwd=CR)PMQ8(%a5Y z@q(-ux+Q-aN_!<}l0U>Fr5}E(mUMnd z2+^$I{ICq5Niu+@N%>(N0Dqb$<41$>11%A|Y=|ZpKcozIP0}^)nl&3g-v`Y-RerAY zw(?W#ntoXNj!B90qtPDmrSn5F%97F#2_c&mq#u?6Hkq;mug#jIA2tfWX>xuvI6u%5 z(P>Im7RtJ;4EIf92Eo2rv-I=za6aWoyQk{UrQTM5@_p0Ne%Z^jC4c6RMuo(e<`2mz zOR7I4gltw&e^>_CWa196X;OdKD8Q!4{Lx_kKubiY32cJ-L&|XDtYZG?n$4f@!{)yF zKU(@f%J)j1ZrUaOkBuUUFZ~~qQI-^dNC@C$83`xLNI2Pq08ZN^0I^XdoTd&sHV8n_ z60vE5#|~v(S|&G65`Th?)7~fl=kF5PYguHKjnm%NfbzA4c39#Y*d-c}jXsGlH6W5v zmh^x~2-&RQ0kI6QNiy;Yr)|=JSO-_3<4fIRCQUE+&0-A0^6p&cNWmM z71?W9UzHux-j;y!9n*4k*{_$_B^Hp4K8Y_aAd*p*)PP6`sei1X0kI5FNismC$pT^> z0F@>SXmjHOm0$s}%LX1hlyylNVp+uk+FHQAWBDYJy_WS=!V)APzw~2Lwk04HJYY#a zend(@B%>^80g(`5S-}Eg8NibHHE@tMX+Ufg;L@Z4*`NVIOT?xrKkDKf%DS`+4zd;g z{Exlo8c;SYkbhK9@_%YsTcz{sw*RB8Rv74#7iDdy1W7bWeChv?jIyKvL_&yU1p$a< z07<6S0FovHh>ZeBnhYQt3?OKU*fixw%bY`5mzDvN6$~JI&kdk8lHrq(+*k3(b_9|R z`~l1M)`z=I;`vQ{8jr_mqo3rLYgM^vU47!+r8@ku`hRe^hj>0-G98SHzhvJVuRh8y z8%=K->7T0)$@30Rf7e#gS$}xB7{`t5%Pc-_#-C^TJya~V*%*8tu09^GF`JK)`6&Cc z8ol^@zQwqAGD!wME+@5^j{aQ4KmO4F!8`n+@BQ?1L;ut`Zsbe9kLIgej#7NVrgd^O z>)piXy*l~LXqq<0@i1Sg)$*H-`RW7rmS1PdZ26VjYrQ3gE z%lHr50xk%dV+DZ;78pJNQ#%sT2`Tu9w7;$$0D@y^L4gh)=UI*~-T+Sd$%a9>aw-{m z3zOEfIJ);fL*vTin5WBwd}tzO@URa2=Cr{aK`N^r^@uwYzM;p-wEu4O)~;)ypB z#8V&jiMP`t$375A3_y=@&&ucQ`DG=&L`^}8G9_hN6zhCWXC)=%`!MpR{{mE}A$Jqc zfMf34Ptg1Wvm++T0tE^YMwMxk@+eyY1CvfEJOPc9izz_?dXvp5IsxpH2r4EUCM%hV rB>?~cPyzq|5dZ)H0000000000fPu}ENh&@8A(MzIBL>qa00000{pT*6 delta 1747 zcmV;^1}yonRk2mDatIDkFLbxrRuF)6%>vL4 zlYfUVTgw6~VYzMpXuG8?OP64&ORz+Z#FO?9$%NAWAsNK7g8joXfF-kP085ko!$tut zP4ds?#s@4x{-J-<1S~=RA!TraWnIGwc54CuQY`n?{L#DSkCWdwUfCJeF5wal5>J{x zBoj*Whh&h;3g!>X0GA}=ODs*|59EWKO}#HTvpJ2SO&Nx8Q{`n{jd&z zOOy1YLHdD~hzx?RER=Ow8J=Q^83d=;nx&t|xcnxvY%Pndbdue+eza}{PEbasTS6wv zBc7CgNG6oh56K{z6{H`Q0WwJj$TT@WtOFp^r2J@5exM~HgAg)7`5|R^kY#_(2M)3| zD?i_d%sqc~ey;Sk^HaPa>xFK~pN7(237X^&@kr^1WI`$ZkPM<(LHc1CK$BzuO_TG( zIsi0H%8v%+2U;RB2%54`)@5b5X%a68HqDxqpRYypNvdoui>$P1ZaY84%Rf$7qT}fj zy$a189;AplMQmSOYr1V2V$YurUhh>0GrtH9L zvnJ_>jRJ6*oF5I&541#dno^a8vMwvbeUq3$uy58Z{d_%~PdU==srqxNx7D9~-?X$} z_VRyhiTR^ZA@Qa8Lo&*e>JJGan-$a_mH{@IxC3mO)E_nquxT=XG?+io647Y_n_&Ks zGTb<;m_NE^^XL1pxv&0@mi~|Oy^^P!c8ULEqe$XQ|A%ChB?TZ70ytSl!pSlcPBtNc z(>4h}Y!nHnse_IU0uZ!BY?|P)Ls^%W$&G)Lgka;e_sReHyF~U{7FlKEw6`^&d~Kl} zmiPvCi3Vh&PvT1rh-8!{Js=W7HY<2QECXzkjC{grn=~NS0ibEpfNao!pd})MfX5D1 zT~;NxO?HRCwrTI31@vu2_FC3gWyiF)C7^uAv|L^G>m_!H1!SX7;!6vNWRxW}AQFE< zDl2F}ECW=M3{YvZfLI4WrO5)?-1tBxSU~KufyWMIT~dZvR9Ar%z5E})!G-*IKXh6^s zv1!VWx;TfjE-izDY=uAnWAC{Jlns9iB-NArpIX*d>HNCw|7fcf2D;=$S=%W=5=|0c z`adM2EGYny5F%MY0Ad+HlBqR-q{#qcqX3d71IPvg2wEaGP5IF>=TO$AWq@P_1IXTU z189w8_#`CvRs69XfusX}z_Pvd;ck<7eiNU@<8j*PC;8=CRc=~WpLlnv4nKdaJ{;~L zo{yJI2czOI+4sh)kFv`~)0;;6=juc9yu;JqwN-T1A6_oTaU=UOi;tV}=UILa6^m^) z2A_wkkH>4w=A&dj%D${dFFv1dG47pAlEIJ5NiC+MKNsT-1NoVhcoTN)R`p%U3}M>BtJMaq7vQ1*QjRcFp`G`JVC>iq zBy~w}s32~&mDGpQwtv`A_Dx&Bd73V<1W!5j6d!=GnTY6w zB*Pc42P@rd!>C+2rSOA+S?gJx+?X<|T4|o&;kYn7l@=-iJHzjLS=V`u((?y);O|~TY z1Idc7!^oTd3s9bh{F`_N9CP1(g8C1$AtuTK1yC<^yla#2C|dyqlT0Z*0gsc3DM0~z plglYO0q&CqDkcQ=?P9r;Fe(-S&yz?>hAJZl)h7S|003rGGfDse diff --git a/summary_results.json b/summary_results.json index a328450c3..4e66945af 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.68775, + "Performance_Result": 1.73438, "Performance_Units": "Samples/s" }, { From fb382bfd79d1d1f093e6ee2f194571516b6cee72 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 16:48:26 +0000 Subject: [PATCH 627/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10927 -> 10929 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index baffc1acb..56a7ecc1f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 16:34:59,951 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 16:34:59,951 submission_checker1.py:3308 INFO] --- -[2025-05-22 16:34:59,951 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 16:48:21,545 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 16:48:21,545 submission_checker1.py:3308 INFO] --- +[2025-05-22 16:48:21,545 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 16:34:59,951 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 16:48:21,545 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 16:34:59,951 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 16:48:21,545 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 16:34:59,951 submission_checker1.py:3324 INFO] --- -[2025-05-22 16:34:59,951 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 16:34:59,951 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 16:34:59,951 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 16:34:59,951 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 16:34:59,951 submission_checker1.py:3345 INFO] --- -[2025-05-22 16:34:59,951 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 16:35:00,995 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 16:48:21,545 submission_checker1.py:3324 INFO] --- +[2025-05-22 16:48:21,545 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 16:48:21,545 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 16:48:21,545 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 16:48:21,545 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 16:48:21,545 submission_checker1.py:3345 INFO] --- +[2025-05-22 16:48:21,545 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 16:48:22,589 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9465 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8397 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9392 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1e590facc..adbc5cb04 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1254 +1255 diff --git a/summary.csv b/summary.csv index f8bfa574e..743029787 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9465,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8397,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9392,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0765b411f..e8f28fe00 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9465, + "Result":20.8397, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d728bba09ecd5c53eb09fea77ec87fe568ad7a43..e93394daf32bfa41b28ffeed676c6a72196cb258 100644 GIT binary patch delta 630 zcmV-+0*U>vRk2mDatIEC$g6v64FCZAZ~y=p0F%Ke6|%-k9 z@%$z}jmP7((NFTrwW{2-u0HYZQXPI+eK_1hJRdKa4o1aavhR&oA7z(~rZght^={(x-uz}X zO&jBQm@m|7`OU_B^?`fKud`&f{K{=MJ|jJLG8`@3wVuV%eeiNL zwr!iWlinKh7x-)S`rfaaO5L4`1FB2GDR;nvadnBWi+H$(`LK#7-bxTp$Ee?V8x}eB zfkm((rNlvC|K1~T(7c=@er-W>}kvIJppc;n!n|KDCa^HS}`VX@q zCdvW@gUG9UYm@LOTLA@=Oes78kCTZhK>>V|%PBem?vn;8CIqG3Y$1~{Di#6HlSe8( Q0V9)!DkBEfCjbBd09z+5{Qv*} delta 632 zcmV-;0*C#vRj*aBatH+q5k{41vv&xu41WZY4*UVj_ST2HP2%}Yd>W6(X``RymupqI zX0-G98SHzhvJVuRh8y8%=K->7T0)$@30Rf7e#gS$}xB7{`t5 z%Pc-_#-C^TJya~V*%*8tu09^GF`JK)`6&Cc8ol^@zQwqAGD!wME+@5^j{aQ4KVJUO z|G_)_q3`|lb3^~sIBw)izmMjtTaHqE!KQU`H0#~O=e_yOXqq<0@i1Sg)$*H-`RW7r zmS1PdZ26VjY1&{5nNr&x%k+hTvq_z&9xE(n=p1%U|`7(M_~I}*_eDfo!A zzpfnsf@5ewfes$$S&lE>08aVIhC#VhQ9;((e;aLOI9U|dt->mnYmWj?Ipi8m6&Qy=w-x6>lWJ`hO^K#y_H%IEC)WxXUs zO+kt>C1qL^>wHdUB_-thF!HAV0#v6VcN5QmWA58e(EJ0lBPPlM1qu;Hm1&dmC|dyo zlTIl-0gaQ3DM0~xlg%kQ0qm0qDkd8yE18KU0RR9{0ssIJ00000000000001hfz6Xi SDn0=rlZYxK2Gb`10002Wx+z2e diff --git a/summary_results.json b/summary_results.json index 4e66945af..1afe43ba3 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9465, + "Performance_Result": 20.8397, "Performance_Units": "Samples/s" } ] \ No newline at end of file From ed4502576eb839dfc597ce2802aa5af234ee9799 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 16:58:25 +0000 Subject: [PATCH 628/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10929 -> 10934 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 56a7ecc1f..e9706b546 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 16:48:21,545 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 16:48:21,545 submission_checker1.py:3308 INFO] --- -[2025-05-22 16:48:21,545 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 16:58:20,304 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 16:58:20,304 submission_checker1.py:3308 INFO] --- +[2025-05-22 16:58:20,304 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 16:48:21,545 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 16:58:20,304 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 16:48:21,545 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 16:58:20,304 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 16:48:21,545 submission_checker1.py:3324 INFO] --- -[2025-05-22 16:48:21,545 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 16:48:21,545 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 16:48:21,545 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 16:48:21,545 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 16:48:21,545 submission_checker1.py:3345 INFO] --- -[2025-05-22 16:48:21,545 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 16:48:22,589 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 16:58:20,304 submission_checker1.py:3324 INFO] --- +[2025-05-22 16:58:20,304 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 16:58:20,304 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 16:58:20,304 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 16:58:20,304 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 16:58:20,304 submission_checker1.py:3345 INFO] --- +[2025-05-22 16:58:20,304 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 16:58:21,344 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9392 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 7.77776 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.87223 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index adbc5cb04..1b0f5648d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1255 +1256 diff --git a/summary.csv b/summary.csv index 743029787..3fdd8b340 100644 --- a/summary.csv +++ b/summary.csv @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9392,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,7.77776,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.87223,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e8f28fe00..5082dae9d 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":7.77776, + "Result":9.87223, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index e93394daf32bfa41b28ffeed676c6a72196cb258..45ed02a923120e90089b7fda8e07bfae944de35d 100644 GIT binary patch delta 3838 zcmZ8kS5y<&7ES0N(xpmIBB2V#Ac&Nq1_)&sG?AtfAP7hiX-a~^P#)3}2u!FF%BYAE zihxK_1OkQ@1VWW4qJlVdBM@GE?|r@dao4)%W3PSoKKtHv8$4S)pG67re>ZNEh$O}e z!SSPo{V}H^$>O@wQ!eB2Ic9t*bQuZTR|rlbxEp1*_oSJH(3~l2+K95)S8we&=3PYb zGj~0e^#YrB{{u79qq#0nlYE@D6c=-I?#|Kr^4-NyP%s|8=E_1x&mBmC>3%Mp*lnoq zLEfC5`^ui)J~2Og&1R;kR05PG)h8k)O@s(j%;CaL7>9 zEQ1p_>Ae@x`uDiKnPQgUSZ3U_61&Fed71`!n-Thx_ARaq=9-=QVXfuPg)Cc`=?!y_ zwA!xO(!l+X*t5H7k=sz{!HYbDW&e{$c}M>Q#?3WtVwI={ml&-mSnAl{T_sic!j0Og zu^*q{bp2vtp(qPM{Tc3WI$U3U1734D(vN(>`~g#`c^==NHMu^s>j}cGg|iD7!uj8<0q_(*iAZkGf?2;6W)p=sc8us`{$f zk=zhUu-0IW$0BUz@Imrptgopcjrw(yjP%E|nv#*X z776LC?Q*|L;ULzgV9Ahbgnsn<0l|}YW<=mICA={!#3%(dq+}hx$7$JD{^|@*!Q7L| zxQU*4tXjTfQ|3ZFa%pUI z*?k({B=3_GM8W2jPFJqN9LHBN>_}6W&DX9`i7QQYG41JmB-Sp7u zu6GSpYtxDW9$tq}F}{~p;~(y*Q{K7b`gcuYRaSh+QhWW7#-n1v5>re(bymA_9QS#E}9MPGeXt?|wbNO(UoV=#+w*7H(@%;gIA-FMCvK+`X5RC>C(xcZlE3J45Z0tmk94SAqM$<=40(UX%Nk z@w>U+ps}6O>ue_J$2-&{^_7Ts=d5$+@tUcqj&4uzmCZ}@<pXJ%|mfkz@>CH+gBigCVTAQ=LIP+)t%DH+Q=`)?0W4)r42%{t}I=M(4 zdtI7bC1}GKwT`|0ffrAvf~zP1GPTZr@O^;iWOEn-wu1-IO(#*y1(WpNmJz7(EOZ{u zR;nJ6={))zM?k*)QIC;6Gr&bF2JwaxVq^1=v%QcnKqhQ^bbjGScTK497p+c=2WbbT zaHClMRTzU~-|(<3y7GyCRzVB$MQfOV-bj?A@4EXz=^;KyWbt(DoeW6)>^u>BO<)Y9 zU`fOPkZ&bm=OHkJczVr)U#bhj0-PM|fiSKH`AaQ`_P(&jEmh0L?6VhV;OAe=)+~*c@bajt|5B{wp7o3)21j)pVr0Im)lbAY^c60M(bUK%-YS!Tu)kjL zV#yzsCwVJ_H)Cal%zjM-02vYI!0opxWDdSM5!We4-v7Guw@-t;AjuOl@ld(3KjDgW zv&5xG7eLYu&(koubdebGAppMECcp}gOch_{>W46Q)IJw9Dn)g#dKJ%|uge0n5JTLR z**f$J+Y!D)%x=2<47Z@FP>!ZMA(P1v8#F)9k4Y zNAtM{_oTGl!I9P5)<$=ma1H_w>)sze6Lq6?C{~PDaYj{=9Ek^%Zq8j&ZWfO!Hi@bN zwbn&Ur7w4ySXQGFTwg`GDp%;7Si2Uyk8`74>{-} z>}am`FURqaW9eBM7V6i&Ry?-ixkt)Hdf-w&pBzkRa#rz256BWbM$dX#4&^YRx7wC| zKmFGN@{%C)fba1=5+^}B1npBGU8k>T`DmG0U9LFxh|dv*@D2v#IjbF4BUowa$Xn|2 zx*0XvIcXsUB$W6oV8WJ)d}>(XFR*IuV*wW~rn=!TN+dDdGrRcXg^+)u;s1Rv#!UPJeAt}1jB&}WmXX`HJR013hl;zo$1b~GBy!`*X@ z&h=31m-xI;-CZ(6iD&WK^=t-hUFrFnXh>m1hCqf6U6mkQ#kMvY2;kpJ3rKPzY~Vp8 z$VS>N0E*n9dVE2=DFq|3J^g?i#%}*0pXC1fkX@X_P`EyIirTBK@k0N)#SM<3!Q+q8 zkamw(;GdfH6F3J*?u1P6{|lF)GXZFX92Q>% z48WikiutNoVVYWsvm4GqK?TF~+q#<^N*^5AQ@`n2TCU?0joXyN%y&cC47V9QE8>B= z3iL47S&a*61=De`(B;JvL@R=$a45l~8U%Z+;=t0$0WV5*Cu)pj@%PlA8#lL_ zvF3<^Cjof{aC+qyvFm^#>=!`|%SLGFEO)t2>d>4E$!X#oHv0SH45a`}zI z6f6|)bNMj?tYJMr)B{>W2=$Rub<7`ZbL{%V&70MkgRY$+307auL%_p1cDH+p!{Rh8 zC%mHtC?)h_E)FFyjTsc{fQJxp(^`O98hh;hFMhR9Pqfx*gVCIEh63MyAbosf>NPuG z^IO>?i^BH5Ihg&L>X+2QAg8ygSt1#yxc<76{=hp3G3zQg&qSZmdX=v*syNTwlaNF< zrRRW~85ikkzOA(Ur9SkVgR!&4!4`{Y^u!%}(>6YTg=9A0r^2hEX1*NOg{Lmn$&OFI#_O&{ve3D1yhqyf|k`Zo|%OomSB|S z8uf1fueUH2b+I8y&MlCjgqFdlB#F`Y7gMQ zw@UL{lYIORRzVHj1EB8B0cO+`%!Fx)C-7^-`h-s90oSW7$NbjeS+d4s!LYwTSym-J z^|zK=ZXIUD7JmKb_0ORt6&5pYBlO(x5g`#zH?WiIdRLcgHYW?gM^%C}I0xpG_iiXfPE{D|sRc&bg^@ z_~Z3^!M%Ud?(Q>2jvsn_XwB0*G$%i~an_L|4u3+V#2-OSzZ=}>YpgQNuw(xAgftbZ zJL%W@{AaA<$fctwa#l_xxsS15euKg}efgtBK-YJ>$dZMlTtC7$Ik~a}em5sxJ{bT2 zAk-)(J0gbfRu1b10RaFv9sod+TgHU@yG4aZ+%)hHj|$L_3kxkJ3|)VHTEcekP=xYQ zZ&qDhX!dRk=AGpOsNTGAFd_Z2IH=i$)46#@IPTr@=LM6SL2eBuL#99ZS5q#62MIE^ z6-A$ZQ+C?!TyUn+kDZj^4Phihbkh0LJYjD?-TfymjF5b+^Celq)I!C`TYA` z)E+wBv*zEZcv5AazngwMU~yaMoGar5={>Zzj3p~yoRKa73X&c_m~Q8dNPQq6Jb1nY ztk9Wo0re$N$#l67`e&DXQFAuyqxpG5h2k0n**54n`^-Vx?fBJ;S$|%4eFO014@yB- z_|87E?|yE3&Kp^}_@}fg-#`6Mqd^O)wm;3^y9-(VY^7{A#3d|*yBE~5ja&*N%f2_B zK-VM6aLtJt7@y l3fv|NX+xC$Uv94A|5-_Q-w_eo=0GiCy0)gk%&A|+e*+0jLlFP~ delta 3846 zcmYLMc{J2r+@CRs8hgk((##kd>G6zF!VphP)>N1_60(;pS*H07dSb{n!`O<1k;oQP zWJ%d4yJS}s*)z6AmUleoeb0OUxaZvE-0%HK zzVUsc#TI7F`mSPGu91s@!}YN$wG}|}r%87b>tvc$2w`_|oAI=B+V@T=I{op;aCyx} zu68=sPEWqGa4iuUnM2xEp4C9EV6Fye#<$d>vq&Ll((^iKISr`#SN2x76h-1Qy86G( zE;IYQvgFlUc27p*@~%2)N9PgEskrDnp`{X-nB$cVMdF$}!wKl)Zdh~L*YJ0o^ixr5raT$s^c z9dWsUJy6+hlQCDCC(r8h@db!7E%WIc9{oR{4r}G(Z9Tf2{!|SU)7!O^nHM6j%%tQw zRP5Sfr2RZgt*@$HmTX z;u3Mk8F=M!pYIBk-|rGP;o=MD^b<_qQWnr(rdgHQ5ClC&@UN~y6NzO-iy8p4(vUPr zDEMeMXHYA3yvZ14yX7`HmP6T0O}PKEApU7`wSQjMwHm9YTfa{Ol&C*DNyd$Qp@zyl zgAVw{q1YO@SVP}&aWUOmq}Yl;h>`MS?|oC_QQ-EGp2@j$aOL8q)0C*f5rmj;zyvkw zZMF3N)E)Tb%7G^>u)B)>U)vAJpCkI-wED{_X1z{HtZdu?Z(OvAIG2P_6ewBx`@71d zr{+U@mYh-W%r3zEqcCTVHFgdcr}z>2o0O|&Y7jdfiB&3aD@hDtjJMU6EgMW>03$s4#!)#PlubTqizx+&lS;c4MMS zz3^$rR8a6I59$1ZjbQ)k;;a^TQWSnR?axH|PY6 zSusBLSgg)WD|LqC_zrK+zR-iqZQKb!tKyi=oPPS=@w6|WJ#Kgew7Q*EySqU4oAFyB zez(*e%)A8`?K{;ZaMwhcdPY2PG%+e@t(Gk`n0U`{k(=uQR`O_cCv(0jeoA0o!57i> zpbcIi23#bWG4h(41HOG}{ob({P~xDQce*vJp{QEtZ5!pugEngL-o}>Yn;LpRw*zYN zl4Dt1-X+JC+WMyF5*51MRq{>boFhunli)S^V_~{}H+ik$ViaYH>0!L_^CL&B;XlOx z*$*w&^q#Cd0ErGWPVA4e_@kaimts{5?CPO_fwH87D3U-R^PiEPx{LSZg-*vDEk~h3 z;t~yXTP^?MK~tlg6OG|9M=sEV-0BVrour0vFblESbrJ_Jbvhe0o=cdyK%V27TyXZPV*`KgDjK*Q{}cd}^a zfRbWWJ{%%n{+Kt>1x|tId^b}m!5(c^UNrpRHkbXn$~WRzg|YwPG9b%{kdoB)MSh~Z zxbnBVe_17Yhs?f-LzCqjSI~fIXW#EF%gS(~*|rDTlW>#o#KbF=KoQ1#WTqOJRzJ6E z{;hDK_;^F3`B&4EZH4O~WE&_I4sisW_%~ZEqQdbZ1}BB|{F8{czd^4k>qKibB)MF- z(*k{|H;dFG>A`)$qiz#TdR_AabzR_3o9h(3IEc5Hc@mCejdRpObJb$EKJ z;57=;Rn&qC#^Cw88=;05GrHk=O6iXjbu#2&-AU?V3Vhw%vu*e_3d@VV0kh`?dl0vQ zvSziCieeRJu;uKq&n&ZT0`-9LYschEikB1e7d#`?5R2y@gv&V6YTm_o4I{C5kyb`_ zWao7A0j!Iev56o&qr8aKes`o%)F)#D7j3bgE8(r+SybUYJTl@s$X0)|P=Ew(@1=dv z{xoLr96uhoiAYa1#NYw`Zn%tr#NWIYEf0~i;2z9c_t5BXGQ*4V6GdjA=c$^bSp1i_ z-a30=%)I@rJnz1tF)YO7kIRTFCFZ&HiMvK^NY`+Ka*B*I&6&*28k=SS@v25*(;OgP zvLQdce29)0na2J}*5#_K>LTbH3+OlI?C!@wVtXPpPRkY;#*Qs|_jw0rUw!0Csm!rH zu#(F((hOS*H0AFpa$G8fK5H9QFzACXmnw={(CTvxzJ}4G;&|M`KN@kiwVG_x%iQ}qTQ887|~vROQuW?iiZkMOcB4YlVYsCKB_U6W|fMGw!u zVAF6Kh@eNsDob|yq2_xv2eFhD$cSr)PM2MIL6(2+*u}0H8}Ar>PM5;s9hukf-1rf0 z&ykb(&ymh&XSWid&j}41H)RZHGTOhI8EwC_^+4&!Sra^BhD#yMgQX!|6qo{f<)@H$NHg@fEh$b%qs6+HxgFn4km=}h$X@VpZZ;#%TX!}~z|iF6ia9o*moLI3m; zjJDVik>e}n0>PC~DK96|QewYPY7nFc2&oLd;@dVW8?{?CSx(1Uk|u&&Zri_mXofM> zb>B@eM|Hrch9{1v;?R(5)W;fS7z(Zk^1k=`j@vKNGlVh_@vKF7UyX`q=4EvQufN`m zuqXK?*3uESVngf9jVa0cX_q0yH66sY=eSAyBDeAdI|J2xr>_l0APbTt=zy`*x`?*__7CG30QW4n5n-`y(>xn<0Hi3wEk?plF*2 z)7@8|shgCze6u82>|B&aGK_kG z?oTom=qdWT&6Ofvlgz+#ZYpqjK4lWI0#O%x=M{xKg3E>8x9HVcO4pLZO&sXZyG$xx zW3n$Uq?7Eo;_#H!oF2uMyj3I3c51JP;L!}pTo|3}6XlSSj&vXwhCTS+{T>N(3fC>( z<9`0KR6ExuHZ7d^8|nzJ*H+i>>i;l(mp|8)H*O2CAN2K-hFGr=)A8s2ZA_UKX+xR4 z#K9kW*;)f!bvsWAm%7^kgQ)k6w(TCrT>~@Yd_k{p4%aEBikxGB1q`~y%iP2F zsd5FWZ7Cie+c&PVH?u50wx1iepy(O63~G(nG|1yhQ~{KXJmls#1xa=U zm$A6iObWouI1!q7hOfM@yTyTX>A8OSgE(BNe(jbG6QLxj>xkDU8J{>^H5Q$4$%+kl5U%E} z&NKINUfCY`NtJf-&gaEr@Wk1@gvaH%Amg)9&V1k~7Z9E?0HPuxQL#Lx=};v(s1nzj zFh5LKPq+^>9ZZ#kDus3lJ=3Coqa}>mt+WINCPEjicclQ=(D{#J$dzrRer(HBMUNGm zRrzwXqF8@r$wq->|KQZ9kuFRlP54}sL-}&79n2*qmZ2TH*7Z5^dvIiay=URiB3#Y2 z$5ZDvT%=&yQ(dpO71g9hyrB9bcCh#_sU0S5#H0s=*c(Ls38&FBY48YG3hj!y32YDr|+IIMA@6UshggEi;QGtT{l3Vr~eW%Y89j}x&yS?ZZ)4kF!f_Rf8ci>K^ z{awRwD`nHrcJF^Tj@n0;&4gNiy4{nV^H$5UOw{ZR@R7c9hY2xf6CX+PBJCaDLKuPY z=U~;}Q5Cr2S#bC*H-j)?MQK8q#yH55^G>Mt-j7R~^6I0NnN2+pe%-?`y)!Cly#!VO? Date: Thu, 22 May 2025 17:16:04 +0000 Subject: [PATCH 629/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10934 -> 10935 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index e9706b546..36e4b0ff9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 16:58:20,304 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 16:58:20,304 submission_checker1.py:3308 INFO] --- -[2025-05-22 16:58:20,304 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 17:15:59,059 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 17:15:59,059 submission_checker1.py:3308 INFO] --- +[2025-05-22 17:15:59,059 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 16:58:20,304 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 17:15:59,059 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 16:58:20,304 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 17:15:59,059 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 16:58:20,304 submission_checker1.py:3324 INFO] --- -[2025-05-22 16:58:20,304 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 16:58:20,304 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 16:58:20,304 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 16:58:20,304 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 16:58:20,304 submission_checker1.py:3345 INFO] --- -[2025-05-22 16:58:20,304 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 16:58:21,344 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 17:15:59,059 submission_checker1.py:3324 INFO] --- +[2025-05-22 17:15:59,059 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 17:15:59,059 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 17:15:59,059 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 17:15:59,059 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 17:15:59,059 submission_checker1.py:3345 INFO] --- +[2025-05-22 17:15:59,060 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 17:16:00,127 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,8 +27,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34203 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73438 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32138 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74374 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418438 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57961 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.10039 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2002 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 1b0f5648d..6bbc2a3cd 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1256 +1257 diff --git a/summary.csv b/summary.csv index 3fdd8b340..369df0fd2 100644 --- a/summary.csv +++ b/summary.csv @@ -5,8 +5,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34203,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73438,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32138,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74374,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418438,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.57961,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.10039,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2002,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5082dae9d..8dd85540f 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.34203, + "Result":6.32138, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73438, + "Result":1.74374, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.57961, + "Result":4.10039, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 45ed02a923120e90089b7fda8e07bfae944de35d..fe7360c058633d3a0ccd2f9322e1341d53bd0864 100644 GIT binary patch delta 3500 zcmV;d4O8;ARku~JatH*YJsZZ8b_h3rzCE@(iPL;wnYeb6&U@q`AZ(_9C7?pA1IRo8#vDcP|fu zW@C}|ri0!%na0P>FY%)J%gO(aJ|*)%7PoPnHnI+;i{s{Pn$8Z}?M44Kp7a)ftz;HY zvs;GAe9}v^U*|XN#cUq;2CJgUxUDVAYfpNkX>;Af;k-&8$#6L8$1jrpauQF|btm(9 z+)K0RFK$P(Mf2onFv@Pr*VCBC!{g>Nb$I%_?KImbN2{?vjN(s=!q1I-DIb&MkNlT6 zgX5-^N86BnxthksyfKJ}z2!K6y-Ys6jz>4QX?EG%yriFu7pwm^CZp_H>Sklo`+VHA zn~hJSL3*2a+_)VL2629&n~nZ*ktUPh*Egw62kTPhZEBUJw$))vIhLp0{QmedjyE;! z^{A^EzUZaBlcRa^sWD%Tu$cAodqy22D|P8gpSgISipTJOa}p8v`zeWt3-*FU1hBu7h_f4cNg{%$S0o~MdQBqY5%h*c z#HH_$h`9gGFrvlPjYvdX-E$HVSGP+d;_AL75pi`dNJL!S-$=yc4>*rM;5_KSl@BF4=!nr6qTgH7tXpG=3No955WL7etR<3;nQqxO(& z*uT}Wr0m!lXH#4KuVh|V>v^g5A*1D}z_A0)ozJ=Rkvs8VdybJnOyd+Iz#WN5?np#( zM7`;tk2QX{7PmP|{FjqAd%k;$9e z>s~*9N^_7sgIRZVaFY=^`;sQ}{_XXhZh0;1pubewslZ|#OR2!(d%wV<6SO>Sxg5oS zqZlBH;sNv=BY`|~VFXYl5s4y+NEAs#q8J!ZB;^uCLwN_w5yilOBGvV6P-0*enYR*her6 zJ7rF$wp!{5Xx6dDDqH4eVrui^bGB)ECC?!hfA8s;!nbo=PojyLz?EoX2GJxLK$Bzu zP0SE9wFynq1EOi@p|id50ZrKw(NO?RS=J?gW$>~p>)N&?n)*M0W*uv+@=EJwV%z7G zuI+lNgiZcN=`)4zKf1nT6ElG)*~AR8Nix7D$pD+=-CbnUHnB;1KsF6MbhbA>uqj(2 zI!Ykflx1C3hTA5FaND&1QEaLwooMS=W2J4=E^eD)%eUN;mwOa{XaAYPH_Y8YqDkI= zcK0Qkm_am22GArKKoc_rO~-^L=>gF+^w8Pf_<*KtiRdVRrY!4{GTb&v*SKvu{{b}X zSYxGa)7jlN14sL1+h&Q2qEIeyL&+v)fHfoi*nG;7wvIJc+BV(dw&}LK&@Dr=#M)7)gLqQg zA(=pWJ0yc>Rxo#12GC^g4xnk0cUTXIrpez?An-s-L`MNM!QmlgxNVZIaoem@w!(wLOiMVkW47u9+E*eE7(0O18fqN1=%ziKCB0SWYZ-1 zD5>C_K}*C&kqYNIlyzwtY@5XRfo-#9o#!jr)K5Cn?y2W&7cHL z)I>a~`jAX0T_2J`G%MIXECXmV;|AW?G#Nj%2SC##{V1vAoIy)OMgg-2s=BNSc1<#W z+%;>qdmbTEcs^eb<(?WpS87)j`Egv|@k>73p~4a>Q6=%D^g}YClQ9q-F^G)~u7 zh)q*|hUgs1y0lClV_C=+lf2M1TR_W?*(r6o;ZiV}-c42q+57~n!j}GeDr;F?l@qKk z8bGS$JGNg2r7D3EO%hKUKqM1N1Bhf0$_fS$%K((jtkDumlL5p=0VquZkOlz=S|U14 zsmelGmz5!u6$Bt%vjDWi~={A8og^W$6+ubqSWJk$BSnA(>FxKO}=# zR0Fk^wSJ&JXJV$TTTG8k8Ss ziO3*?Oi+GE86ITWzvcr6*_xG~??dLEIzLys==|g_$aV6osw2Ks zen>`H()l4FM6-hP!!m#-$pD%r<%e|uXqt>44aN_&MC`I5nqd5pGTb#u*SKreZ2Wv5 zH1|~bxza`DC*L*wu=J^BiSwh;9`U8~Lo&)ulF|_&nX&_~&6=biHVVLL za(*;8KhP4;wcQZ@OAB_r$ zFOymmA_0h#eG?it@0C2=v`hRS8$}Xd`adM2EGYny5WvYY5>A$paIy&joVG~-Vxven zO&xS>5P+a1V$%eV9m=|s&J!99g?3ot8`vcpkc~cxFO&5XBM68L0v zy6od6c8LXKqfg>XlSmXPA6-(0SXQxs3JchGET1H@*RsA!Sb_xPmwru3Q36uIlaLfB zG(>Eg@~bY+p{z^G;2>M!pZ~GQCHvlZ{Z)49XnNCF{JH*;Jn!)I zcWvdJ^@o?saokA1%;Mu_{CSq$L-}Hxjlt*P`hV;38ngK*nUB&h>(TS?=Ua|@CzE9G z<7!gN>FCd8{NoS(AH2gK`rc1JH}p@9<3_gh`)Ibh)hPK7Y+5Hrv))a7-kaZyri;cn z9%c)*T7I)JUw`4=>gP0>t$uQwjgLv1CX@T$ZsXn{p69=1fA5A#lHUK4Uy|(?7oPkD zP_qmc*A4{+rH&mXljtNXf5CkS+kv)Z)j**xE!_!el89OlB(+ZdzKUZz33%1z=-xSV zXHdD^%QpH2>x5Pn%90dCpqS=DZ7cNNtm1nVIZqX*tyWN>130v-Z%Sqe(-vN>HekH~ z7X^@VgqaHMyf*}6$95p8OM*iMaiguIK9qKg4Q1bS3ph{HTP(p-e@;EcM__CwA{vmK zO-S=&+aACfh883!?{Jo6`0VvyrJHT&l?$g7{$OC%I*OBf@5E&6x-RJ!y*1>|@aOvF ztzR`3>Ub&!s4h7p%mVe=)+N3y!r@x#{5qU?BY;01fa delta 3527 zcmV;&4LI_*Rkl^IatH+PH@Og#b_h3r(3i({Cvln&EECsG(s_?O1cc2L5JkdC{(U9D zSUmTwwc<%5+McxWazFbaxV#Hon4@1lPbZDLXpxNL*>O{~tY#yc4dUT=c5~c(|L*01 z-)tmle>Ut-;#qXu{1PS2Urzpa^eJBak=#a6+Q>SXCCAO%G@T!|+sWWIn)Z`_Ry>bp z*)5}ZG3}?>kBggjGG9df;i_mlX=}@J+tdDd)?D{+xTw-cJQ|G$(TjMnoJO;B-N_=F z^wVtm$?bTaG*6C()I&MBwho`^Woo4&wXf^hSar7xE{JW7a z+4wXbrnh;=joa~X808nb*%&O7G@kyxzDaF5SeGhyQ>!estqxkMW4YSN?~gB&Xj9W( zkGh)Si+5pOs0o<#iJ$OjS$ zwvnCX+I+b-bfCm?F(Kx8^=RVVqlx#ACO()FJ1gz*m3E+BiWD-sbry(SUy2zo;z z;?j3XMBIO87}4VDh9n}c?m3BwtJ@P$9-WP%cj zb6M2GBDv?hCH-oVxp^yWcPH&eVTZ4Goo=&p=PP4O4 z_OE%g7+sHNqiB(x*rMw+O493O`Efc<^21MZedn~ScDjh7)iJA`pQhU9;pwJ+5hd9n zCTVX@GY8wAuLtvg`N3emJcwtr&x_?O&5lzCo7D9no{h#g&7YgYDD96YN%N#!%l41S)5;A!H-S^QbK9ks|rb<1J=7^7%_pFL2ZTAWjY@{Z$X2 zefL|RP3u@*1qd&L0)#=!_jKX8=MIC);t;Od1%={&zI{$IzIf!Gk_>RjG7?0VktC9g zk}sXS+J+d5vPtYeK;w#?1M)aJ$KY}0g0o^?v}(I@Va_mDbI~w$CXY z+bP{P`5UFr6u$rHc#=)b_^xCVGsq^%0GlKOY?60(kxkpgCg}m$H1yEj-uS?#YzgTo zzGPFD^;j8hn;61v)BZ=Xsh)JAtz(UqwoSXZZ3ZpRa!OwAQT(0#XA0jicYKK^dE4E8 zlW1ZF(Igo_lVku*%n&p?CNxP8h^C>3?)Jt9G-XRjM*%ctS&x+Awn@6iZL{+qK(mfD zR@yc@yW6JU(O%iMS>mE7lnb0dvWXermuzAN*(4cYlVpHRCeQ$zj)_g?*8rP_9=h8b zAJ~*FAsr=$sn2)%pH~iG?}{tXqx05)&rtx@^=&nJkS!-Q2h-8yu}24%RVLJ_9ka)JU=y_vSE@ZE6G*p*WRT4Yb`Q${n?z+nHcf^P>jBw+G)X>6 zDmZ7*60%XG!g&s5Jz56aCNX|s+pJmV`ARnRla91|>iJx%qUe+T-qBvcnB7IwFF_ME z5m%}{Boj#2hhz}V3bqf+0GiCWfj2fy#t-cQ&@@RuN-8;L&=QhS!0dsl9;<>~lguA? z&6@3=N5~YO&(}k_r^e5f+EspGI*!-zNBAd*2SD;Pj515h%vMoTD71`rzspfm|U8U!F{ z3F$PYDhp*jR)$bk5P)>e0?-bVe}^wy%K|H5S+sw&-O`q&OR&@>SfWPaO8bXo0%`w{ z3}RWq{$Uxwf09`>fThX)VWR++Ci!Q3;{%o;|IldymLUI-GC0AquHgh*Sirv&%Y8L} z^sf2S$?hAs>Ld>`XjE?en=*e z+7HPfmld=hmH{qF2Dmg?Kdb}b(j@(8kba;gB!i$U3uQf4hNoC!2Ei$|X6ffKF29K^ zTgxIVon(vFkJc@}j20!DL)#N zA7}~5e;|ZRP<}`m9%R{H^MQkG&C1XBA#+cipDSH-e)1P&-M}gN(@@$iL6iI;9x45h zOdzEnl0h^pNIxtCXp#(|X>xv82Y{wY`O%>KKubslK~omWdaMjLP2vT?rdhM{^R;L` zNtLZ-k(D;hqVto#{L=|abUa<6{Ag51JgNMUe~hxE^Fu<2W(DVmWdKc*0W?j@59zH53x={qJR&W}cW#FNes z$tX)oKO}@~R*-&J2H0fE4!kyNl784I0H?|M(ct_*OGu|FRaq$Ou`=8@i5Ud@X3f&i z9M{A7lq2n)sy~;ysQzU8rlq~ImuE}NAB_r$CzDzeA_0(-eG?it=axL(v`hRS8$}XN z`adM2EGYny5WvYY5>A$paIy&joVG~-VxvenO&xS?5P+a1WYYwX9m;x>&J!991$I#4 z8`vcpkc~cxCzJIPBM6WT0vVlSmXPA3airSXQxs z3JchGET1H@*RsA!Sb_xPm3~c1Q36u_laLfBG=yxL@~bY+p{z&C;2>M!&;Qtat^uW^ z97**g|EHFG{U2?$0$-QBC~J30kVKQjlauTeB7cxgQ+~C~Ih6Hi86a7~0J8Vo z0NNlKJPFBt6@To`E|R_-$WLLm3t#RwjTSf2X*8K6jX|7Uu65Tuq(d zg%ckH@W)HgXWoWI`aa-M2tfL{qw=wMdfsnQm8bbKPM2|>@3R$I7p_U3ezha-`maDa z4EZ;41oXM@KSA{evneLZ0tD|jxe&AWC_MoMz#VPCB$GKRQ~@ZHjw&Mt+9v=2008IK B!NLFl diff --git a/summary_results.json b/summary_results.json index 04dfafbff..58c2d469f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.57961, + "Performance_Result": 4.10039, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73438, + "Performance_Result": 1.74374, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.34203, + "Performance_Result": 6.32138, "Performance_Units": "Samples/s" }, { From 37cd1a8d4b5a236b6e8eaca982ad03a020802c68 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 17:20:18 +0000 Subject: [PATCH 630/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10937 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 36e4b0ff9..e5e01f7e9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 17:15:59,059 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 17:15:59,059 submission_checker1.py:3308 INFO] --- -[2025-05-22 17:15:59,059 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 17:20:13,242 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 17:20:13,242 submission_checker1.py:3308 INFO] --- +[2025-05-22 17:20:13,242 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 17:15:59,059 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 17:20:13,242 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 17:15:59,059 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 17:20:13,242 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 17:15:59,059 submission_checker1.py:3324 INFO] --- -[2025-05-22 17:15:59,059 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 17:15:59,059 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 17:15:59,059 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 17:15:59,059 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 17:15:59,059 submission_checker1.py:3345 INFO] --- -[2025-05-22 17:15:59,060 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 17:16:00,127 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 17:20:13,243 submission_checker1.py:3324 INFO] --- +[2025-05-22 17:20:13,243 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 17:20:13,243 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 17:20:13,243 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 17:20:13,243 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 17:20:13,243 submission_checker1.py:3345 INFO] --- +[2025-05-22 17:20:13,243 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 17:20:14,324 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32138 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74374 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7268 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8397 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9392 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.87223 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6bbc2a3cd..837dd962a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1257 +1258 diff --git a/summary.csv b/summary.csv index 369df0fd2..e6572bfe4 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32138,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74374,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7268,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8397,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9392,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.87223,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8dd85540f..f60304eed 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74374, + "Result":1.7268, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9392, + "Result":22.9804, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fe7360c058633d3a0ccd2f9322e1341d53bd0864..ae5ed6c9b0b93e83a007f7388fcac78624c2433b 100644 GIT binary patch delta 3867 zcmZvfX*iVa-^a&HNXQb(zKvyU-NrQ4G@4Pi!9-L{C46@W{EZMp3jeV^D(Q`b<^MCPNFRmBob)Dbi^ZWjm^Ej){zdK)gbFa{pXSGa# zCw`)6?<+A$s+vlAYXt@Nzj=12`N9Gj){R}nQQWt5RF&q3CVVn41WQHDQ zoM47PT|em%Q&`)Z?X@vVJhBqwzXXGA_BCZJ7$)vC0U<`{>lZ^RMcElIy*Se;OVYjlpGFlnd0vsq|tIAOETDSdC`L6AWk!Ss+FgdZV-uEMov%$yW7livs zBI!6eTj26Q0aCHpI<2PYe0t9XLH!?PF0-SnUo&u(Mc%J1Tw8yeVP015J4bh`bNmDlT|Cvh88~NrMyxHYVX|ES~L(WbMTC+M~_xg*Kp+s!jNm}qKaoAXMG}~wD>b0l)T_=qM=N&kQ zB7ie$!x0z_3IF;_$+4gnN6E<<^GlT2!AWWI-p7}yvB7&js7m-x)oorw zs>a%oRBx6PyXUwnyKsT0Q{PfXW`0TF*? z9u^yB&n&thx2_oZUauKw9D6OFgI}B&|I2D)1lPGa`P^7de&t4>9_$3&oc;a6j_$+m zA9pVKgf|fbQwPez+p@pDTTDK|aSKh9xNI{brCz5XdXGVEnN{yr6i^6`3)FiGj+_4gj#I21^I;BYuPc}-pOq5JojAl9dnl|P zsYbSx!kIp?RFgCth*X1RXv;O%I3;SY3rLkI*)3`OQ(#W&xxuKcbYk7#nFmbD5}%&K za-wT1^m`O%1FaJC`K}paW%$j$84kw@vSH?tW|Ap-lvodm-So9OM)8^3AI^F3!SMTS zNL2Gf(q8u71+-4wodU~~i@|XZ6T`M8zYF(OVq&S$IN-xN4`(YT+o&F z1!LFXxp$v)%a7fA8Dx_87Pyf^jX$jPt`G`QD39Nh;t3@~^GB_2Hcl!5Q<&1HB68oSVyMCe61*!lP9y! z#>VebSQ`>wHF~^Yc;499VJ#?e#0p_JazYZ{qC$g0h!l}rvh3Yw$UNK~EJD;cpz7$! zT$uAI)V^F8^G*REa_NGnYj3)#!;WBk@zsR$m%RIJNE?qj%M=~cN*M#%;m>|=S$w6E z+#7FeWM!HlaRNV~vToG(HF2aKO(agN=~-UP+)AS$($m_~f}@dDUi`KB@7@OS>sc$9 zfAGr%2mBO$i!1$lzW5mtL(&}RC$8*koH3w9 z2+^LC-`X$lik^f<6EBp8I|;OVG>X12^M*$d#Uh{K*3a5Ac17dCUSj8#oup@I9@}qy zS)^6;sb{~#J}NaEABiaWUIHw1`M*zeTo~FfooYwcA?rR>rU(@*pN3m{^3TV3Mp}*k z0HtXz$E<+P(MFZ|SHvpsk)+nJkkn;MS$ABux!!6kpU#)gP|5VrDUfTC+d=Ez)o4nq zy|aRK0*b7wufkK~H-$%PFfH{6=I%JSRRn4(4N1U~q_{#74uXKIsH8YgleS=FI$_7k z_gpx{JVnnOy3o65X~LOg{Ddm3MOU~}O& z5YW2)Z9fG7Vho&_VBNEXRG_lIGySP`>30pMCuYM|+!zlqY$sX7onW$kMYfKDI}-U6 zw|ylal&-mYEp!BuKDgcP4d%U*;m7LAONKzsPIa|X%y4O7sbsB-F(+R&v)4D{Z!(yy z`(@YDwbSq>Pks*1Q)>zbnyxt?69{sm0hP+4Jv>uA#J~T{E4FCHUcKfT-B{wwiV$bs zm@DfpZ_rVyF}R4i&SbrqsrXjy@JVAzy`(b2QZtz)#!$(SE>=$GGr$SFMc^>SkipK; zP2%4Xxf;;uT%A9W7CNa3)Whgr6BC98wPamqrkZZHM_gwTOTTs`BW%eUIHBS3#BNgR@k4rP6ooWh|!Ji!-rN`tJ35QaZ4nw9rQ>%nBlJU(L4|#%;xT zq-ty_OM;IP%s(;Ul4p>rkzGhGh9o6aSQK3%y1mZBb9M7c654Ol@;<#jVYwOLyJ=&V9H}t65nM*tMJ%x<|l_L@p9tRpmu~e7C;iQ zyXnAQjd`K;lG3YLbK0%9awZ}|IyVp?PA|t%I!JsUXw1h)@^QFX> z^Mp_FhJL?e(HeypFEzJp?U^Uf-^1J*Eyjij6JE-E@8TnJY?kNsZZUVWRo{V>h z==yRzb(tuu;Gw%Yjahl@i=EWAus3Gx6nXlkwY@gmv2bQ{&u6^X_PuSDY)WmQQfH*p zfZdH1Eq^wu5b8nYt9_sz6?%ZLy7%*gr20dk z)PJG6fU2Ja9pcA$5p~MpV)=nisKgs3OP}n`@Ud%>zh5vrKHM5_%XJT=1TG1)gkD*t zYQX-%8n)g?rT%-hCQ6duC z;)%Nj215I#8$Qq}>vla5WNLU9WDt#?FWX~#99WjBs{3hUicmS8l|25Hr5I@>a>UZ^ z*N`6L_m}BhlX82dfFNS3q zCm%#>u-&^8-%r+p#{L=|Xpm^1__(#_hjIl>vl;wf7t>8`N51{kb1*btCN%|SB6?o| z1PUe{z7-Hc^1MY=(UGDq!sNiKN>p)WX)07lT;Y$yd?1h=7zC04{ofNA^%F|vKo9Kx ZL_Ml0F8sfrBBOMvi8|^6{aU*#{{g5KUe^Es delta 3834 zcmYLMc{G%5+@7(o*-BBy(u{R1$rdua*1})r6Etq6OsX1PuH8J49ENAzKO(th+9-v3NFK`3#qYwW`q1Try9oYtEC0T4NXw@ z){^I!NI9^I^cCT2IBzwmiCN>}1mfM4d1vfFf%-1pu6 zx|>BTDrNtn1&oQ1Z3ug8L^@EGy6`z9@56z~b=!TTr`xhX6;>8a_Pj@lA95mlFB`9S z1w(!cyey);snVuiMaLEgU7vn8*X*cB2{l>m*2QzerP0F?YRu>=>?F_0Fop-5OvXXhB-UMS19pUk+~nD@S` z`9yM3%aRGu;%w3K(B8NqD3UhST=ML};g|bHs>N#;Rs;NocH^gN^V879Rv^V2xtnW}`DPsSA}LB`pokSp1x^`n=aP7%we*_>bc4~-4izdwH|*ZQTjg-{%` zRdK>Qt@_dTtvEY{D^I+L3G^LL`s0DBmY_Gy{{~_iK+kLXbjr<GmS%G13!rU$f1%#Z!*EnW%1-wDV#{SO>7$_G;#njCG>3_&Grnw!;h zeF|!Jczu|fP8d`{u;H&Zrii0wmcyD&1L~N==XcFbr$+`BY$KB~wZ`SSw?hU#?E$p4 z0)!_7I=r&X3eOAZuDFd_ZF=UMRGj~Lpx^5854#0hW_j_CjNtLNqctUK^1hW@8z%i zS8PcE?jEjXsx_91=$0BYn|FM=i$?-ykC(BD&Q7UmjE>itl>AA?B?RDjJ>&FkOjJZD zLF&{lUX{sN|Ni7!j5ed-?fHp{HeUs&injT$GULEN0jc*gSU?zPc*-N zna>3(m7q!!Pax($HY&iDwEXJW4Xha_nbMw1IH(0Gi{)$M_h@~fvqCqdiYqD!CG2j ztBlC`4_NC?yNqR5%m~bOE4SOT&t!|`;ZbdO)? zo!cy<B zWIfIESRt&aIC*dPSZ`B_bL;0{#E7ac@8Z-Bo*%ruNqc&N;ga3tHRV%WV@16^EVLNR`xHpVv_3ak!}HxV@gJng5xx`U|VVfnzoAH9bEE z_+DYoU#|~}g$*bl0yb81I~uNEL>Hyxwq*zU3JgN!EV<~YK1qW*VblA56#;_-~tq`R&F^m0(>WzvtF&ozPN+p*sYwryMbtG#mbya zRHJo@k&yarsuWQE;YiT5Y`bNF6MYV>fim9U#Ez94_?cSv7z0k?peb?X@ zYEJ?g9)6`E!#daUFH;+)@Hue@fq(XMJB<6LEsKzJAo3?H?~#=OQs5((tISD0RQDa# z74V?J_D3517p^Qg<%>G549>voPZ=RUy$Lbh2OIn^AZ^-+*M8T3G9V?j$VDgAnxYzT z%aW8Lhb3TfC_LCG*OE85N!2LVn>ScK_Dk@;@jmD3ZEj?_kTj@Gm0}GjvAOyc&O2Z{ zzBjY}p`qJ^(e!-%N`3Q3ZEJGpgzNL$(Pt->b1h@dptl3v^rAcjK4QNfCgy0+r?BX^~A~89QM==mb($TPnBR?fC zwVtvp_1HmogrgM-v<`js4HC5SAYrbg`c=59ZIRiITFWhj|Iz#+0J+g%6kA|L>)O4CgMC069y<9rVO_0Xs zd)Yzc#A;r}cxB5}>d0HZSTd{k7AVzBltxLTKLib(L;7!F{Gi^e?=M|AxwOp z89l^N&UKkUKyQ1vRRbPe{%+zvK_p;Y9aoCv{B|a+2XQ0s_gW%$rn>T~NxQ7Mf2HWw<73w6;=j|=q%D(CGRBn@~7;z}vIz~Jm2*GK|Z z=?dIx5i^e*GmKtq3fK^=VSiSM({`#4^U694BKs{>{ zYOnF1aoM8jt`mWbl&xg7QboLQDOY)i0RELEf1EQOtdII2!2tCbXdvXX6a!Owl2=&olMAUVnqjXka}xsY66Iv}{d z#ETUKg(q<5Ekm*f3)p|gJty1CO`$OgnCEXV!;rU_b%7Sh>?;4vY@_}nCgCl6;SS97 zsg7%c!HeUq=Mx<4FheY5WgGG><~w~Hw}A3gmy#ktfq4%U#otTR+N_b3^L*GD`csD7 zlDIk+%#K$y_<5eyIApt|Heanczy$G8dN9cG??3#0qI?C~TQTHq=J!yGC5QjYR|iUUULa_#K`TR1C!P8f z(?n8q`xWu7#PfiI5UsL+Q!Q3a=~VC;F5y(W;!p%uaXXtEnKLrue9WY*x%ZeZ_Ml-t zVkW8o=69fW8^{7SvdyP!o!E*;ZKb_;5YS?{1}IaqjiN7ohL{;j5(Yo6M}mBd9ZXOP zH-aT`1u)-XzTZ})J=l#Ly!pnr?WmBYC*PBDs9{qMZ~NvK0@yLcX=|2mep>tT+z!Cl zlqa*)bJK@(8H3D?WzUCkY6R@Jg)3*ND_;iu*$WbTip}p7cj#z@@isbb!LELLH?Y2G zRGlRed_m2gOh?7j^JgBErM@?T1c=sMNFbl^2IlCi%Lchs3E}x^>Bu})!tTtq1-jDH z<+uL8#yupSzK{3DuOB}Al+Eu#mXQ5)fM1~Wp#qG+fNBBlH-#n>>#?Jjg*w{P!a?}t z3lh+}Go04@erCeaqf1}tr>_6w(l|vm+Tmy1aXZwDKWJO<=<{D8THEx>`M86p(1*&a z+*}HIO=>Iubqo+1mc3ME*_h9-J5s5_>6SHrSaDT}MOI|0xc|%z zQ4{pAC%pX}aOQNEy-%>JihRanlmpXm64EAjPZp}a5lY1VTKn!(nji8dTeSQj$H!;P z^gbbOqZ7N9CFzUR8EP9|{WHcu4aM*>;7&d`;92?l{&ct9X2 a(Eoq9#8HB)rUh7okfC{8U{dY(&i?>&_fc#B diff --git a/summary_results.json b/summary_results.json index 58c2d469f..63f771638 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74374, + "Performance_Result": 1.7268, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9392, + "Performance_Result": 22.9804, "Performance_Units": "Samples/s" }, { From 3558c6ac04dbb05510923f95043c1c517aac5603 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 17:31:02 +0000 Subject: [PATCH 631/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10937 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e5e01f7e9..03890037f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 17:20:13,242 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 17:20:13,242 submission_checker1.py:3308 INFO] --- -[2025-05-22 17:20:13,242 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 17:30:56,772 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 17:30:56,772 submission_checker1.py:3308 INFO] --- +[2025-05-22 17:30:56,772 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 17:20:13,242 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 17:30:56,772 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 17:20:13,242 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 17:30:56,772 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 17:20:13,243 submission_checker1.py:3324 INFO] --- -[2025-05-22 17:20:13,243 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 17:20:13,243 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 17:20:13,243 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 17:20:13,243 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 17:20:13,243 submission_checker1.py:3345 INFO] --- -[2025-05-22 17:20:13,243 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 17:20:14,324 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 17:30:56,772 submission_checker1.py:3324 INFO] --- +[2025-05-22 17:30:56,772 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 17:30:56,772 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 17:30:56,772 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 17:30:56,772 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 17:30:56,772 submission_checker1.py:3345 INFO] --- +[2025-05-22 17:30:56,772 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 17:30:57,814 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32138 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7268 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71177 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 837dd962a..771034cb6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1258 +1259 diff --git a/summary.csv b/summary.csv index e6572bfe4..914b3dc8d 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32138,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7268,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71177,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index f60304eed..07fe69c0b 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7268, + "Result":1.71177, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ae5ed6c9b0b93e83a007f7388fcac78624c2433b..bd22bf3544aa83b5991a2179275e26d67d31b4c0 100644 GIT binary patch delta 1743 zcmYk7dsNbS8peM=QIb{AVVVpoH3w0=?i#5lod9hM@~&edt=SkjT3BAnv)hY_?|;wvJn!?o=i3z66nHOI z7nM*E+7O2X097;q>;`z>5U`R0U5YMLEq>@<~5xw-EV+g4AM!W!5di1#?jw|nBB ze>xqTZar_pMU<)C9nx4j8>uY7BYX(p<2O<>wde)!OXf2T=*cSgN(A)vk&S8hH2?Z7 z`zLVA-{$j>YM^D($D-xJ8}R{A$WwDHu9gT<@!69G-KP29JvZTlcC=0&y`aNn7(5-R zOeFr5)Q{#;`p=|_jR~-<-!37ITWS>>6q~lDu2VhenzXs*Ds`jn$r`Z%Y1SK&Kb_U+ zzp?)o`78-{^)Jwg{zN*w5r-KYWUB9>I3BzeA!R*@pRJ!&;Tk} z?!+fw_Olw#acg5C;nei779D`ldnj^XQpPy6J$B@K0_rQAwNRmEAqipiXMqM zG?{huS_e;)t)<5u5-x>(!WW{$N0{$W_agnF5zEIF-Kv4pn;OHF2zK(CyDWM_6t+2C z&|9+DKKW>!#8@ZE;yjAazcSy<$}~&2P@8c5_&YgN-7zKxVbMc%R-^#luW>-VAFikZ z)$FKC?njFwlcgxu4-CAMHyTRLl#g0$O)a7o=YpAenR2_cWW_73w^j=0c{H-?Xh{bT zYp7wnO#aIDiKAh|Vb)UQj*(;$j4Uq$6#|f=NdwH(#TxEmfqhR^7|6Hr9(?RV6jxDbHqW$#Tp{^=Rw~QP5Gb|4PHeI)+h=FnPAj~ z1OeK{G+yy|yV348HFWYdojDi2&sWA!a~q15@^~A*+!7>gfgZ}$@BOvRFf5v0T8d%M zA9fk`0Q~25NmoTQSeOvV$0NJoNL_}Tqt{xX(UkA7kQJSM_fy~2$S<{Exo|HHTE%Tu z@*4sVwe3=yM6srVLX@{f{~}P(UTB|n5>9AC0CEh?G}j)6A#}{^jVu=r>w?4n zBRk}%G`Z4z6Vo}q9D#ilKNS%Ditp>}Y zWtjPT#+AOpKD=;Z36@b*nIZSKt{MPPwuLPjn5Xxz1=S$w_K*JzmhJti;;73a^2td3 zKI%vAm@nncS?g+p@~HaKxx#;NmWdCgXjkaNKLZkg&TzBa-*P30$V9f2mUDZ5lJV6Z!NC;iQ&Gf57mZ_b_Sj)MOB7z3Nj8n zwehWs3K*X?+gmGqQP-aSK}SF+W}M&6qyfv89#P;8MKLvV(VzbDxu9 zck)gG!6-B}$4PA@a7C^;E$KT?K2mG*UbO)CY>Txtmxkl;%vM$C)`n5n3g)iTjKClgG>-geTtqxn$k)H6-S zL(K1h&XE3b(y3s8=IU3M0mQ&<1!g}t$&~Ji6=fF#y#%^m7nBf_rgu5MPi$U27RbE+8 z&Cj^7TL)eS>M}TR!m4n?X*RCh}i2z$S*M2A7JSKtEO?gNQ?$sk^!vTx6ndaDObI6f~EpfkX5Tb6e874nE!~s~4_K7zLkXes&&BsiF z34SHouhrd^g`dXsIIlmFKc{TFqi>V(1^XL66F)biN}R#>@Ag8kQ|}!S(gdNggwI zq)|f$EM13pdH72%H@14#3gQ-+R37ZRY^jOs6996}tTMTRKoN|{vNsVc$ZYzs6S`u} z`plD*o{x=2DT($Wk+9`!laR?#ecUogD0rYc&m;uW+reg0@^YqF5a7pF!MC}zO0Sx= zk_5PU{wRj85)8ODmoX7i2I6X-kaaYr@97@#VvNHUj+=r<}^c)z%I8ZGWrbqm`%Tx4d>oa6z{rZvLDLM9^_43HyV{;gVC#&Np>*H6_ zzmiPPflu`EG3S?qiP|`?u2q+1BWHz`FqN3l7hpUA`u~uWsF&Bj?J_S>PGVzA|Msf) z;-`9Q_w0|!wViEq^D!&GC+6r*Z`s*4C0YDU*FW#EV+G<-4HtLco{TeSNP2BX3vzZ0 ziC8Kxa?+Q#UECg_zf6vJ5i2&VrQC%|zFx)e^M14QTw7h~t}MSnpSyYeaqjAoh6eqf z9L=4g!szj+!hdNOj19vTv}}9b`N0wS;VtD$O%oE%^278UAgs>He!$ucuzT?}7MjKq z!C{xPHuZ?JipncKN5iEXRal#UI|szxG+drMV>?%=?0xw6S6#)Y<3;fc>(!Qb1I0MP zKk2(V)4j#NMai$B4^L2!6{^D-SzV(&m5pvUT!JGK>xKl2@Dq0QcAMzHOAEiEvbXNN zTIy+Oy{YwKy+|K2grJBo_aR29tT(l54uN|X?HZZ~f_=%DJq5y7KdHA0KHac8AmrCE zcg5V(nU=uMhI}&F*l7d4Fmb1Vb~E+!d5XJg36|>vsD9ddz5D6BG9jmu+9|o%m_46du Date: Thu, 22 May 2025 17:47:24 +0000 Subject: [PATCH 632/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10937 -> 10939 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 03890037f..a3f4e4710 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 17:30:56,772 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 17:30:56,772 submission_checker1.py:3308 INFO] --- -[2025-05-22 17:30:56,772 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 17:47:19,010 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 17:47:19,010 submission_checker1.py:3308 INFO] --- +[2025-05-22 17:47:19,011 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 17:30:56,772 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 17:47:19,011 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 17:30:56,772 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 17:47:19,011 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 17:30:56,772 submission_checker1.py:3324 INFO] --- -[2025-05-22 17:30:56,772 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 17:30:56,772 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 17:30:56,772 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 17:30:56,772 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 17:30:56,772 submission_checker1.py:3345 INFO] --- -[2025-05-22 17:30:56,772 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 17:30:57,814 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 17:47:19,011 submission_checker1.py:3324 INFO] --- +[2025-05-22 17:47:19,011 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 17:47:19,011 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 17:47:19,011 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 17:47:19,011 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 17:47:19,011 submission_checker1.py:3345 INFO] --- +[2025-05-22 17:47:19,011 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 17:47:19,983 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,13 +28,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32138 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71177 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.61319 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418438 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420002 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.10039 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 771034cb6..659c71f3c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1259 +1260 diff --git a/summary.csv b/summary.csv index 914b3dc8d..113dc131e 100644 --- a/summary.csv +++ b/summary.csv @@ -6,13 +6,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32138,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71177,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.61319,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418438,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420002,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.10039,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 07fe69c0b..7404fc60d 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71177, + "Result":1.61319, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418438, + "Result":0.420002, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index bd22bf3544aa83b5991a2179275e26d67d31b4c0..2348ceacfd47fe947fadcaf8a5e8f65deca133a9 100644 GIT binary patch delta 3045 zcmZ`*dpwi-Mn z3j%Ecu{Y{#)fTREgV4B;ic0H5?TOcpGwqD8*87u8FL`Bw5>dL1J2vvRGc%os4ZG;u z^)=o!o8~9h1oQs=Td2%XGXUGBA)5tPyhQ!26+)1x2Y>A!Aze z<)b@{qO2Fv%lHtLzwgbS6i@>c8I{6fWt^xcUm4fDpo~jc=FsjKWt4x8Q%~MU+C#q{ z(SG@eaTe$TN|%J%$kb)Js3$`%R0NmBU}wXv%RX=QY(Hbya~aFhsv)Pq%A>CCmD?YH2=66y zk0>yr#rZhS>kVWa*j|R6Q`tyx+wm-j2;d8YO+o+oy zA5sTau@-%7_qzD%8eu(Iqv2KQcz`=kU~{3RcJ;u>)=$|cJ*Z<%XW_(`$U+okjVe9o=AcMq+YC2p8qmLvG&h4fR7!F#v1*@Mwac+}~h3VK53jixZK`1Ph z`#gEAuwK&|-ZF4>J0_kTgA{QAP32j{HBBE&1g!67d6iS`4-`Oi| z$cN!M6BH&?0v#+acw2S-Q|&AW8$@-$C`IN|dqR0tDZ;!251SGpCXD_5l~;Mx3PO*^)x<+qM1-r_$tphC=~GXcGZYNmX}#Db+X+INz5S6e00GuVYUi;V}H_#U0#cZKjWc6)m|pl_q@=BKz|az!v6vfz)$2oBib zsvpF=@C{-B;z?%F)cb9_;at)O$Occ*j;KuZBu%Sk$Wrm9y4x(lf94kCWM%3h9CaAx zYT-sbjx+y)x?4vF;5MuAKpfnTnn+D1nunE6-t-%Y1lN#0>g_wQ{iL~z9e_#Zcc~RS z2IO#f5xDi@aF)iS==CvD)7_QEmch!H-loYf$fA-kpKf8ON0iZ>J10{6oFFa_hG^5- z`lj?cgL7~q+)v96waEpN8L7rshN=lb`7f(`;I3?N`+hK%D3=QbuLBGt-r_q1ony&#TowYws)zr7;Z+=K3+NJ zXm$=TAIhsnuH0A2C79V&@uY{hgRWx6of$tMMmm~lxk`K>>kQn|+oTW3pxPgjWZH`% zuZam)jpCAJg#gP~e#p{sw{Fy`TF&nn34s_D{!szM2v2}^5OB0io$iwDINW+66o!0OVbxjKbm%eiJl=D<+=C88`RA@B>{(ng`vxZbe&lhV0c z(M(BskM{QpiK7{xC4?#I6#XV=ag__X_o}c0VWIq#d?E#J1*j(^yZUVApQ%<^AcjMq zH3tCd{fyl^?$UJQOIM}wZCi0OnS5`0oF;}#oXZ_X36hM+d~_cI?qFef6L6Sfc3iO^ zf6JDl9@ZNvi`hv7*1T8V;@PrJ)WhSxrnHlNFoH@{8)My9oEJj|j{*UOL4*6b=@DW}gzu z>1sv+xM{RpzOok3wE`5d5;y?Slgy>pTkiE#+N{<@`A`EiI$ww5gDY*7o8KjUgh4117ZQAB7T}Yy3SDaz3dI<&$ibF5Ta_YH~JI1F;qM3Xo6T{S7 zL9j?1b-K&CylNzi z>n}cpO5UAY(n`^29-?X){aw}@V@A+oSTzMt@u^e4w$2*&(MB?UUT(8#ul+tMj=i?( zHowCAb!S@meYz@dxlzX2v+!*$kvd+yVgF?T;f+hn%kr`4xzpF*IZDHeE{#m|{F?f% zC3$WYlGST6)gzgm2y*|W*L!Ml?C3#(c(E&LZVbji47DMkGaj~Xz7JKjXBz{A{gXb#z)0$0k>fHIQgq zjD8GrTN2By+-GS#6W;vA^&T?0+6|lgRpn)J%x~;T6q4rNwo}b%A<5}u@+hiOy&6>j zig!YzF4#@u&JF~&!^G)zw45TpAbms5*x8Af#d#OU?L|E%{(5x=Bu)iZ?m1rLs9)K_ zD}!d|Jc;-5z`qpGyHXtmU#h~=cTz|CjzUE1V@V)==PUX>@6S5!OrXG6k5E?LY7GOf zUmVGa{WQ;cVj^9Ff6@luGycXaNveA7Plhw2eRdRGX{_Yp7M zw=vdrBnOK5mtpk?=APA2MP@~d0v%;4w7a422Ye#_ala8xzn%JLE6(#C)#rmiciY{n z*Z10(4RVWPrk^(*ThTp)_=W{O14H)GD>gRAd{H;~y-C_PBvxIP;?yBym-%n;TBrMH zP=3mb7AB-rki4RF+qTva!?L!;`L`&Fw85fEkeEz#Qq1O>qya?pADY;V2B6y~k}cy7(h;6%;x zm{5gMAjjH0C9SDjd4OG9v3x1?1Ri)zLa&sl4Wnh5{`ajAin!2-LJ!h2Ez&`H1*=3P zW5#qxpYq8w9QGttQG6%RDad^5##6D6qQGl4-yoiLLGBUb&xp$lP82Alx zzU>tMUaX0~1FuQ=oh$s%^y##N6}ZK`3r}bU`W+nu&xU_A zb*dctGw=J1m6hvqSn}XeXGq7s}I@^%EO zTw;|6_P!4^@&A|Z$`48TJ~N4OUA|d8=zwhMo4#+e!z5n23A9Dk=O%-9;Q#Qfbm}-5 zlznd#ly!|&+VPCX`)n#m1{OKbT40K{xqp2Xr8<3Be&xHvq)`Z#jero1BcVN^jzr^H zXpd0S&tH~b7Z8P-M*9BJG$?!b+d%G0Tai8Q0BUj@xY%#^6gJJD*H=&zCV)MP-!KdR1JI4{C;@R@sI{oKp8ez&Lg zQwdh)B`o9IV^epmx**vPW2;=I_GY4=TOp!}r%P_&g&OKhu>*|5@k04<{1Yy3{*0!l zCTZ;acvW|z-MGh;G7IVRYcX$iV;ALw=6!Z;9dX%dn=?6+ytu7?a$gonGV39w%4K0M zAOt;(i1t5S5`l-d`PR$4C=DM|;~>u>Z$*c`rD$r4%Nb>dBy);^ zk-890%*L3k)r>WCxAVanA0?TOr<=N?i3%%{+g-&yWnl6`%q+f%&4_aslfd8IeB{}k zq1a-chzIGzv7%;~jHsshpy%d|!#en#<@4RH?#<(m?s{Kq2e}v>2a4D)T^_@9+Y!O} zU>oB|BpD0MWqE^2rO{0vl#FtbaN#}V4g@Gbp3Kn7C<$yGoUfwI&u{J^o&#^X7%p|Lx7Q3G*QvSO&o*IszSUAf6e9twcIvQPlX}L^ z4{CJqK3qKwpY=|y3+x@hCn5hqwZW*Ew)1gpRXs?`y=yY6HO0I&jA3nc90yUqOj1*?~6C#H|XRM3B_zlrSR>yo?2}fX=gy#Zr|^+B>^r#kM8S&ruBC34dK9r zlEkPL;ied5`D~14wdBmy4Zm7Nd90#1R~_yL^=fUTBxz7|1Zvb1bOkb2;;S!QQl}Yv zh!+Do)`tL9(ZJ2VB8X|&6EM^lm)a=VV+;`WogwRzwxF`B?kK<*a{snNcK$qLp}PI7>v7K7MOuI#xu3#)l2!fw}@-Rd)ksw6MDHyzlCRHuXW zanOfIcOfF9!xtQQIh+AAR1f35{*`EJ!^e1Ztk|Rl{rDC~FD*li2>)oUcD^oHZxigm zU(j?B(SQ?hQt>hJ&D(a#H-qROo<>Hqa1b9RC_`JO5h^3$am3VV<&@&D{ct7q$>C)n{-6oP4}2f8hC}3uMGbZ}OdaLku`j&359!fw zQ`qiHMM8tas=Lzrx)#j7cs{C=119#z-H5+1_`dAhq0NlRHQIt3FMp8dq`7icp!M(c zx3QUCj>Vo^`!Z+`Z<8|Z8(K!^ENkNU?@jmAwK5mF2M-P>2EW){=+(s>8ECD%JAb@j z_Gd$99=&Wfr+KtN4T`FYLrM4Nkb*{Ylp2zVV@vJCYmZOx)q2X0B@YX-M=f8}R>51T z^#~<`RE5XcvQj+o?aTNG^8Qm&NLErxa8^BcZ6Jub@rTDyEdiLZ3%>hhlj=pCQdGRZ zE>HW+dL7cdNcrD z?{RlUUyO%@&fKjwdFU}sP!vR6JMvhGfJ@MDcD$O~tMu~n=~+m^-NFR!F@uNgAX%gA zngsaU@>sXWGWh+kZNYiU=L+mBW@O&sG?a^(AD`AyDn*tD<>xaixf%_J zzn+jlCkMB3>}Tg+xFd44kIxw&Yd+L!#*0$W(^K?{yKN>=WLV0qn9WKb8h#}ZZBZ*T z(&mW#et*-}b Date: Thu, 22 May 2025 21:19:46 +0000 Subject: [PATCH 633/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10939 -> 10939 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a3f4e4710..4223cde35 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 17:47:19,010 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 17:47:19,010 submission_checker1.py:3308 INFO] --- -[2025-05-22 17:47:19,011 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 21:19:41,283 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 21:19:41,283 submission_checker1.py:3308 INFO] --- +[2025-05-22 21:19:41,283 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 17:47:19,011 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 21:19:41,283 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 17:47:19,011 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 21:19:41,283 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 17:47:19,011 submission_checker1.py:3324 INFO] --- -[2025-05-22 17:47:19,011 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 17:47:19,011 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 17:47:19,011 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 17:47:19,011 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 17:47:19,011 submission_checker1.py:3345 INFO] --- -[2025-05-22 17:47:19,011 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 17:47:19,983 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 21:19:41,283 submission_checker1.py:3324 INFO] --- +[2025-05-22 21:19:41,283 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 21:19:41,283 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 21:19:41,283 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 21:19:41,283 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 21:19:41,283 submission_checker1.py:3345 INFO] --- +[2025-05-22 21:19:41,283 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 21:19:42,266 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32138 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.61319 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74095 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 659c71f3c..39eb8283b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1260 +1261 diff --git a/summary.csv b/summary.csv index 113dc131e..dad8b9799 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32138,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.61319,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74095,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 7404fc60d..54ff9bd91 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.61319, + "Result":1.74095, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 2348ceacfd47fe947fadcaf8a5e8f65deca133a9..5147bc6a2af3c499da17efe245edcd58f78d19d9 100644 GIT binary patch delta 1693 zcmV;O24eZURl8NNatH)TYz9fQb_iq$f4q(rlxPN8l|YFmi7O2tk_n^%L^23v1p|m> z07_=oXbGjs0Aiy6lqLa4g8&3AA)TgFWudIc$`Hy50+6m*0NP>l@9<@7SzskBi}sJU zTe|eaTw0f4i5iJ3?H`f}r2RuOh-C%)hh+duX4L?eCi{nt0$7^lpY4qgSc3dRf2Rpp zg8W0u-~`LMh7)XI0sm4g_tpH-yXH?PyKmgGGpt?0B^o5IG=E4Ykme7`AeR-)AC>_w zNyd{{n#3R00bptJel&PL&=QhC&|`O`XQM>N86eZ-{ICvyOq24XLHU7}kPJe|1m%a6;X#)DYd&z0ty%f`e?DaHsq=HC zi_TB}f~*@jC7=G%ZV8&?AL5bH56J{l`XL!avx4-)GJqz@0GcM}hjjpGnv@?6$`7=J zWDqoEp{&QsaML7S5Nw(?D?eY0=95&}S{7Mp(=0ka`O80@phU;hCCZORb;Og(56LJ? zIzJ?YXjX83SO(A}89>vdfBdix08Nweqrv!rmXKXGL=%i3Qii)G=^A&`HQu-kwWV3?w!!p1oQ+D9BS(Eg`MgcfY&W{G? z2UgryObbRQY!_b z00b=|nYFR@E3ARB!WPg+1Eqb#Wbe~}PUSwRD08K9D6fJ&1E z#5w>fO%~Ah#s@0F0%Dg9Ja#DSkut=xiUm|yz`kSoB$2(A^;N%#MR8R7MYFS&Q^J~%n(N=bT)g>>=+MN<4(IoMt|3fm$k^&G3A(9mY zAeI3nnOXx#nhYQ|3Lt4RfNU^;pe1C}lwU1#4rM)B21r&gfb2atfHp`5PeO8E#UHz~ zi==M{@>7`Y!k4>Eqs2{h8ciliV-RPTYhAf*U4P@Nwmhj=z#JR6Slf62Z- zS$~yXI-cD$l0Vm9lII?t{;sXOv%%+{l)GAI(;`8YTaM zP3z=nKicr3kDfl>qfb zB_xVK+$?*WTEsuH4QbyVCu!4G>WSleX1|?Xxm@g2oBV?F(&!42EJq1wZiLiLh2Gc8 z^d2QXFfCYXG*oC09xdvdl3U7+gI8xPxFEqN5u`okwnEzwEXCOKEhy%*=+IHz7^hf> zrE7n%W&E2pfQvl8!wLcuEHK;yQ#%pS0V()|v_GyL0D@y^L4gh)lZ@b#H-JfrqPob7d797DqF5J%k_UQAzWR|D{U@M0 n4EZl_8WB1R!0r0JOv8-{H&FvcO7M7VRHx zw{+=;xwJ095;YQ6+CL-{Nc)Fm5X%bo56b|S%&Gw_P4*8P1+X;9KieB0umt&se@+vy z1o?-Q!3mai4JX*b0{*2~?yLEucg>$pcHg*VXIQ(0OEgGaY5tH*Ak80=K`twpKP&@W zl8h&@G>Jc~1HjVc{b=xhpd}=OpvMknJyHf9JL?)ec3rdggIIoZk-e5BRuS2CQTuUQ zwpN9Y5bko>Dd7_R5m#zIBoj#Oe}`m{%L>{L%K(=o16-P{AJzeIX_9_4NI%dLl0ndw zg|Z$i!&59VgWwcfv-I;Am)}H|t!0swPO?SoN9&f~@k>7YrJWKoQ66!n^g}X%lzvDC z$*ds#undq%GC-!u`C%OZnI`2&gYp9{AsK{_3Ca&C!-FjQ*L>h0TeI@>e|^Z@Q|ISO z7oDH{1z9(6%3k|%OVA|$5Ra68NG6cd56K{!6{H`Q0W?Vl&@?$etOG#Pr2J@5exM~J zgP44aN_&gzT~*nqd5pGTb#u*SKreZ2Wv5H1|~b zxza`DC*L)_p!BI`iSwh;9`U5}Lo&*e(hmtCn-!!VmH{@IvIDQpnxr2#3czV{el$2g z&=S&VN>vuhdaMlhO=1SYzFD*M^Yw5(d&Pvsz2GjX=$(Qf92T{^GBmX;z{#| zWRxY<9}+?~E2uv#18g#J2iP>JKWr3W(`5c=Fn^#Wq|*d8!Tcd*xN%l7e{{{}&-Y<- zU;Q5~{U7DIB~Lf)6930Wk;Ie!56LJ?3P2WY44o{ z^le4dMw z`T7wl0g;TdqyI07;VpWP5|aBW{@9&e zBz-%OpTcYxzT9mZEpDRIXfjC}gE+ff>&k8G`Wx>qf7RiK^_Rmv#Iy0@*>IfyOZNTA z`m5~H@$9CN{JH*;JooVQcWvdJ4MvyCNz_Qc%%kIG^m(4$L-}Hxjp66f`s?u;^Tjw` zjMFdc(ev-;TTc2X(|GvfYEsMD_|IkZ;}3%$+`}IR?oU5A^iPfBMz-|(Xtui5DESX; zS|>;IKmJX0-e25|XGvocjk1MWEw9;FtiNz?^>Z4}S3fz;#>Y5KE%M*8 zzjvcJPVaxoFUj_c3s3$6vkw;74g~%JiSU!_Bp-jvPQx$|MfZF~l`` zB_xVK>@0iQTEq|8hO~c=n^#*YE0*JX=gggvi{(Mq(Jxphv?@`Sq$mQ#G#6@DqW8@z zzDJSsRB>8s1tmIyL(A%>V1_Vl;nivb)(db^04Yb9Dbe10Lojx150bhhI8+d~+Dhs} zX?K6vQ1(rCfb%rH!xB8@)Kh!{#%3g<9?98=G(Wa(0i0o|L4xuQXGw<7UJq8f$%a Date: Thu, 22 May 2025 22:03:40 +0000 Subject: [PATCH 634/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10939 -> 10938 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 4223cde35..4aadf6e9e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 21:19:41,283 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 21:19:41,283 submission_checker1.py:3308 INFO] --- -[2025-05-22 21:19:41,283 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 22:03:34,998 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 22:03:34,998 submission_checker1.py:3308 INFO] --- +[2025-05-22 22:03:34,998 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 21:19:41,283 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 22:03:34,998 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 21:19:41,283 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 22:03:34,999 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 21:19:41,283 submission_checker1.py:3324 INFO] --- -[2025-05-22 21:19:41,283 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 21:19:41,283 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 21:19:41,283 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 21:19:41,283 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 21:19:41,283 submission_checker1.py:3345 INFO] --- -[2025-05-22 21:19:41,283 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 21:19:42,266 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 22:03:34,999 submission_checker1.py:3324 INFO] --- +[2025-05-22 22:03:34,999 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 22:03:34,999 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 22:03:34,999 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 22:03:34,999 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 22:03:34,999 submission_checker1.py:3345 INFO] --- +[2025-05-22 22:03:34,999 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 22:03:35,998 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,11 +34,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420002 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421261 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.10039 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2002 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1165 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 39eb8283b..7c2210700 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1261 +1262 diff --git a/summary.csv b/summary.csv index dad8b9799..0001eefeb 100644 --- a/summary.csv +++ b/summary.csv @@ -12,11 +12,11 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420002,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421261,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.10039,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2002,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1165,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 54ff9bd91..3574fa0a0 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420002, + "Result":0.421261, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2002, + "Result":23.1165, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5147bc6a2af3c499da17efe245edcd58f78d19d9..d8932d39fe28a490cbcb30f07f4b30ac9ddb63c3 100644 GIT binary patch delta 1853 zcmV-D2g3NfRk~HMatH)Wnxf~kb_j0+e{^6eGYLiAe59 zL~=(W;?eiYGhj!`1+{Msk*dj_-a>fk0La{eWa7lYgDJKb_y*1=$@bf*G~bu6Ucr z)x&4s{nlsGI+j-f!poolVbJnDU3l)f!=SP_gsXNzp}23KlZ-DOxu+xp9I}iAk!2)_ zB%|a@C$F|4#v<)O!f5KDyS;IXBeaBc6tRDwLs^fM$=4j;vaYrGYjjaqf53h~`LG}6 zKn|m5o;@TNNpyXuT7I@Ael?c0tgr%??=}89=>fqs^w3rEorN=K3F#=F1XGsvSQ+eviBU))SgZC-rNHchwWIi%w6Jv~$Sc8=ppG%@2l5>3n?nj`~g zk_@1U8G@!Zp-Fl`Gz~p;w>LhZDO*B13ZN;=dZY|qR%Kn=mPAwk2hglzja6Q0-Aruz zoYJwKvX@#Fvp^2&0t8Bx**~)`5rhGM+HUDiP@?1M66HsuI^s#?hh&r`ogWfHG%Gki zECXng44`RJepm;9rpfrxVEjN!$SxbA3C0g8!(Eefjk{*e#?SXbb5E6@D_vB6@?Fyl zN}p<$I6oTg5l=clB%>@T{g4o{SwZ??8DNtsJMh}9N%~=<0GuZ0M}zYNEg_wMrc`C2 ztjEf5-y~)b?3*=9KVJ{$Q;xKIs{UN+qWY8Vo0j&5s2o-}_*Mp;t*At7Y5 zg8IWUz$O!SfK8M7!$tu%P3DgV^9Nc&I!#~`%pX#Q8)p^sN7rord>=OV)&J4b|52V> z@^sTK@qcU-Nj&NQkc_gV07OE604K{xI9W!*$tDDF+9m;rjUwSRb7Wh`l#%Y&mKsNd$p45OyMp@DWA|Yh6 zf(OJhz$VGaC!DrP17aNjnkEg%1`P;WLNW+=>`>KXRdU;8cL;2o_TE{4K;KqmuVsBz zc1*h{0cAU;<><1Hm)Ip1kc~cxCoLe7QI^zzNC>H{paHQAP)RaCrO5(f9RQUk3ut@e z1C?L_vC9S?JCyZE8Dd$*0xB$E-?4m>$X?6(Dq#r{kXQOODMblL`43oT5XmS@ zT0kU(SXQurSO&0UehnOdWK9|n8wI#DX+Sn;K+qDhY09sg@69X-g6Bo z9py->C;30MtgX`dwdnt7D?7jHk{4y|P6?7|l6cbpAsJ;!0f>YU$qE7x%K(y0tpOxW z1`rzskTe-UHW)zA60&K^ua-H7vK}o1Br6y|_MRI+8zh4#A-S)A;*Z_gMbfte`6vPkJR2{b4afPvWZ$2xzsfEh&u$vY zpX)Ela}Q5{*H+%yV05{hM2+;zJUVVhpXb>dpC;X^!}Ip zl5D@Y@Z>ME4;I)C1+OPT6eN@EBp-ij!!Qtp?|llRyDeKLr3l-Bw&c`6p(!oB32Bpv zT0cl?9lw1QJ8=^5smImKemlFUS{_uJ{DSk+=o%GSjuOz^2&s29df%+l2bB20v|z2# zP@^6^S~hnTx0D+Pug+L-L4r>rNPEg{jrJi}im~T=P|Ro1p`*ApPO%V6x5Iyy@o%;R zO!E95D+o-m!0-{6+KGrxNWmwh{c-I85FA4b3Uu(e$O?S%25`#HHVn#@Q_0X(1<`3wBHet8>KO{G4biUVpY!6|pZf^kiSuZwuNmie%bC*DdBkA2i< z-cE~*eISw;fR1s`%18P1yjdfmhU8?C=8Ke&4IwlyX}S11j=br=0M%*8zlmqSnEUn< rG=H-#CdvW?Oq!zSv->DL0R^uoK@=pDJt|ZIER&KdBL?0l00000t725ih)3Tm&ww2%7u3Enly|XQ$*t(DSTFt@hgjbcSMuDGOp+Qg<+o&7T5MbwUX4uN z++Ozw`BR#M>>13utAm@2$oZEvUJP!p?{v#;SqFoq(wzz{*0GcdEWY;&EOz{sf2%Er zqv&%KeMC__fSzN-muD`F0E#3cQ6v$GB8f;8eFKW5T%u?w?_xQk=o?VPdO*>aD3Thn zX}=}Y(qdT`{=WysI+jua#Si|@wrF>>*C{!U2AstJSrimK$A~W;WTzMb7D+_1NFtI& zj3`SyG+zY<7)iZ+8urFe-wgyRe`m5GWFyG)s4VJ{BKgeYEoM6M`AbkQaMS)EP7Wsh zRS%zi_gkM$>sVd|2rq*Igh9*qbm6(@4ui_#5U$zl$A38Sfp?)Jtlj?fa)QN;dz4rM)3CSP-W%evO$uhB(ee*yae<->lM z138SMdG?T8B+>PqYWdlg_|;g}vcd{nzS~S}b5Lz5%d$&e98~;O_h$-UWp^A2CT9GO z1e3g44`7lEfJrg{CT4Jt+=eKaqz44k&_h?rcNWf|C8VQx5=>dvV`Z>ck`Qc`g$3** zn1!7(CsSK3^#nBQSYwqfUvo1twR!P5+ce#h=a7oO_w-ER+c}Oa(Zr1JNHj5nXp#(| zNiu*YW(bya{eS(SBdTM|wEA3(E?HCB10bu+Q; zb4tf{Tvfs*vp^2&0tCAB!(6j25rhGM`O80@phU;hCCZORb;Og(56LJ?IzJ?YXjX83 zSO(A}89>vd{ICuHO_TAX!T5oekX<%J6O12HhPx)|8h6c_ji2v>=AJ4)SGuVDgryObbRQ;!<`1-lbeg~>m_MWpH_j^NkFMGL`95s!tN){=|D!y& zZu08W;XaI%bqlT8TVv`qpK8%4rt>Y!_b00b=|nYFR@E3ARB!WPg+1Eqb#Wbkq}Z@K?7nLpps;ON|Ob|Isht77SQ&_ z2P(k=VwVj(b|~wSGQ_fq1yoqTzGL|$k-e7nRl*V^Ag}amQi>9g@*l7yUq2!xAd*p* zw17wmv8-SLu?%3z{2Dla$eJ`DHVSZQ(tvExfS@I0)0AIzaSmlYS_TK%3jh3%z2_QG zI?9n$Px60iSzD#^YtjGFR(5{XB`?a_of0I`B=MyGLo&*e0uTuyk`)9XmH{N0S_4R$ z3?McNAZapyY%qYJC1lf-UoCSEWj$I3NLDa_>^(PtHb@3fLULb!#UHz~i==M{@>7`Y z!k4>Eqs2{h8ciliV-RPTYhAf*U4P@w~8~?eCe*9tZgM0YH!2RjxhW@E>+{l)GAI(;`8YTaMP3z=n-oJ^? z`-_|LENM)lQMOR4L;h!_!y^YJiY(zHtG+fMgCj%_ihx&>HRPH zCE0#);mKdK4;I)C1r~k&ktLJtBp-iH!!Qtp?>t51d*V2y2&^QP0QE#AB#J=XEPI<; z#6PkPY2O|vY13BfiQ{@^znxvVT(#E$W++Tgr`tS7$7^Ai*aQq&?-fLfa55#n|&LDCV>1&{5nNr&x%kYp{Q1{F^m^ zi#)%>3IY=>Fx&%EI}ypt?aX@t`IOPslFs?4~c@YoSG9Om)#2X3XaUb<7 zZ-+(3J`hO^K*zX82mk;8 diff --git a/summary_results.json b/summary_results.json index 91fa1da35..d36f8762f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420002, + "Performance_Result": 0.421261, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.2002, + "Performance_Result": 23.1165, "Performance_Units": "Samples/s" }, { From e22022181c7be32850317633b854aa745ebcb714 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 22:09:02 +0000 Subject: [PATCH 635/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10938 -> 10939 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 4aadf6e9e..f89cd1a24 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 22:03:34,998 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 22:03:34,998 submission_checker1.py:3308 INFO] --- -[2025-05-22 22:03:34,998 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 22:08:57,031 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 22:08:57,031 submission_checker1.py:3308 INFO] --- +[2025-05-22 22:08:57,031 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 22:03:34,998 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 22:08:57,031 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 22:03:34,999 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 22:08:57,031 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 22:03:34,999 submission_checker1.py:3324 INFO] --- -[2025-05-22 22:03:34,999 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 22:03:34,999 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 22:03:34,999 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 22:03:34,999 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 22:03:34,999 submission_checker1.py:3345 INFO] --- -[2025-05-22 22:03:34,999 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 22:03:35,998 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 22:08:57,031 submission_checker1.py:3324 INFO] --- +[2025-05-22 22:08:57,031 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 22:08:57,031 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 22:08:57,031 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 22:08:57,031 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 22:08:57,031 submission_checker1.py:3345 INFO] --- +[2025-05-22 22:08:57,031 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 22:08:58,022 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,16 +27,16 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32138 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74095 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421261 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.10039 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68663 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1165 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7c2210700..c2306a490 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1262 +1263 diff --git a/summary.csv b/summary.csv index 0001eefeb..d60607228 100644 --- a/summary.csv +++ b/summary.csv @@ -5,16 +5,16 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32138,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74095,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421261,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.10039,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68663,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1165,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3574fa0a0..edb4cba09 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.32138, + "Result":6.32624, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421261, + "Result":0.422108, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":4.10039, + "Result":7.68663, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index d8932d39fe28a490cbcb30f07f4b30ac9ddb63c3..86573ec64b65ae3a77175a838b5933693f24deb6 100644 GIT binary patch delta 2936 zcmV-;3y1W&Rl8NNatH*?EsYzKb_iX6B=&MY`ysfz3tTowzkHrf8h6nm8OO8ZrfONu zMl>75!}09qxcUCw%LBjJNYegn*q_9+=(zbMN}9i%{O{;fy!a!zjiR)Xbudego408? zKWw*?!EH3{C#`rM&9YlY@nYIfvu_tS?PR`)`omSxbkf$A<+i8&@vOP-;c!uFrH^<{DU zQ&RZ4kuT+A9RHDj^JaM5wDM>hvM*QDxL7oX(Wt+iq?hrh*U|XqHq9=Zla~zQNwWI0 zy$N6e0TYuR3q^mw)9kF1eKn63qwDc(6fLq7TXda9NqU_uKTgL&0kSA4dX5oa zJjhNl0xXhLhZDO*B13ZN;=dZY}uP0}@Po1Omv znsuzP(ze;z-8TJ>_R6-+5*J0GT;K$fP0aYdWD_&UCdmMsBm-@J#q z37V*hxKi~YnLxTeB!g&Huzgqt&}7C9ys>FAerOMXrb+ryQpq`kmXM4BW)D>LSQYG= zWd68o)@=7YLZRP z0F}(E0V+-E5A6Y{G|4{->>p?e*(mS}lT;8qf7?;7w8;WuqsZS7F?G<@AOS&3$fhZO zhv*#2dbCU)V_C=+lf2M1TR_W?*(r6o;ZiV}-c45r+57~nf|mYzDr;F?l@qKk8bGS$ zb*!L7GtjC8N;FAaX#kN-APpdrK`1L2Kr91LGP6cYC`|?s8wH>=2|yYIAZQ8cG^HvF ze`P&ZhEP@zfOO3Q&<>M-hc8>p0xMxzw12eSQmV}U(Ygdn)JR-u|By@|?H`gsEGyVQ zECX0Fs|K(%**|O)z|tiDY;SzP667B`O~4Z5A5sP&NH zcJB>u1t085kiqrv-umXHjB9y^rv zNEvwStZVStbxXp!T$-dG4bl&^gk%tOWudIc%J39R%pf?$)-3%z#^pDWWoubv4yBW9 z(fZN4<#+s&Uop{>91=4hH6J+0)~x(|A2RpU`MJ_X=O=$b)(xDJKMkeb5;Vz^ZW5G# z?2^B;rAwS2jrNEqogb1>mXv--2-&P4{jdzM$&?*jIrJt{d^C?H#Jym}$by5Av_DxHBWiJDjm_Hg75>J{xB%>^;{*Vx| zSwa0_8DNu%JHVz%{b8d3nDnmXv(AOJy2$fgM% zJCyZkncO%@2sTc8pZuS{OJuKQkySQMyQl$WYYTjIMCB%>^U=>d@t zvRT0cVi{nQWaJZ0+oS=p4ggJ)24sT<1T7&M1Uz=A>ai-hZL&KAwoQBQETC^Ive&Y{ zDm$iKlz_4w({gm#yP9^11!SX7;zP-(J&SO-9*$pYHm z_&_CCKN6X+KTj9_D*n6%4 zrK224^(6nNmbFznzZU%;ZDr?IUGk!=-6=s5O%hM~KP00pDFBfWB3VIy0Ad+HlBqR- zq{#qcqX3d71IPvg2wFlmP5IF>=TO$8Wq@P_1IXTU189R}@FXPnRs69#yGZ(WAb$(9 zUHEXfX|%YBPNT^rX$<1*a;+=3t?N&`yHtlC)*lY{5YNVoXTx#+m+bqK^+(yIgzO~ufB4cjgN7f#?$-nZlnGcFk0l_Wk2smah%?N zlV6hU2N$0F1+xzp*bW6VDCgNFlk6lbf5B}KwgYX+sewXMT6z=GCK0uMkkmSU`zns( zB#={&tC{_Fc2Tw1t2X%s=cUm#DzY3Upt%uJZ)^0vUZ!)D_`tMatSa3mtPa;Tr%59ByAy|sB=Q~i$XVIafxG_$#5KFhkmho@41x)h%9xDh;f3U#t z0hro}h(@H~6Vm>;b^r*Dp#=pxcwA%!K6?W= z826-n%$}atOVp5@lxbe3gscgn82Ox*4_}9oSN&(88i)Lwcmf=AUw?w;53?*L$^rz< iEsYzq`Y1gC1v4n;*(H-aDpUb2laVST2Hz(D0002EK$S%R delta 2935 zcmXYzc{r3`ABSg5M2IX2*~i!u4aqvPjb%tl24$3G)RcWG%Ofhp3}!6ZvsYvpOQc?; zVrKNqu0fPFS+X^D^N#oZJ%60%k8@r3^*Q(T{haHxTzz}BCV=yZO?J^7DTMPnP|pZ3 z+LdW~$L;#4*v-&R0pEg6O`4+nUumfHlFR0u{23M!I2OTNo_RQdgZcp4i*}Rfps_sx z`7ds^2ca8EH}+D;%h^CwA z{?AufU7tI&fYJVgup!q?zor*=`YiOt@JD+S!)kKP>w>;hWV1K=aXXZyuza{3;mOou zV}t?T46b|6@Jd{H>u8?W?le}55$C_5q_j8ImAR%tV0Hob&*|TDzF*Hv&#drqRTb;N z4!GI&(YE>*un~bdUYTDg)*P!)^nTrYE({nfLXzY*1AuZmT4C2mQq6pLimvGFq0snC z7f;;KMMvclNP8i6?tchu)*E8=gMC)?uWg7>1Jvf5^p(@)8F0RIJeLq5q0*N zI^QIHasB26>$<6dc37}Mym#>`YJG0@tkv$6W#8WX3j&1OnSu1z8Vxr{Uou#lbX-K3twSWwv;e0gTB zt6mFG(N!5rH0qQY3jLU_zX>IknA>ctWo4WcAAWpIj&%Ob@V@au+N^XqN8R%8A27Lq z19`ki?C65t_W+lncFfD5a<;@^)l#;^KkwNR0g2X`+q9pGJEzU0?1lKA&z)MG$>Nra zmQ6qkS(+9hWd+P8qGgpb6~(A6P6Wjr4xw7fOB?b}OU!X2zLffUr=iEpKgP9$QU}Cp zY-^p`h{RH`6`>gHt${enZuV7UGVv%~$voOjAXSwZ?{PwoD!}*U+C#AUW%Yt5?==QO z0mwsY7LJq7xu&n2_@D%7xE_|6MTj^M7~meOH;O04S^|R|-V^&n-u=26*T5fW&q3}z zYX;ZDmYX}ClXV_dgcuh#`4*6p)FqqCp^($$q$8=G&;)4lw6#KYqs(X7CF2j+--Rvj z{S)Qt9|Z~30tF`MP~FnmPzaXD^E^Q`#;fxYY#!VT-vD~Pk8een z;kf1%qsvqmQZ>i4Er*-#h8#e!L$#l+M!md4M>q6FR7SO}U4v|N=Nr_V)aThNg#Ra;7kYy# zpW(0-&6Xb`T$BJlrjHz_Z1s)Gh?>^LGpM;==Qo3t7{gzlv96rtN0?xF~i==qh;Wt6Q#15h?>-ojG;kg9p?nZc#O zTUTrD5VxnQ&~@L}IkP^jZ$D7n9N22U+(NbMT-CiniPLkqbW#2lv??tM2}>E`qex4n z3%`@f04rx*gE(XlNR2=)`49o2Z$>==Wcwcw(iMJvVE)Cm5f5;!{j`FqjYNAhV@+QM<(4Wp{U%Z5trs_TTSKq(%z4*iu>O|$-_kh+n5#W~Zg!q(6%_$7gJuw9 z!#jd-8kWO39gzS_P~nv(a+QEhH90-mA@QCdE3Y9eS1AQ_vlx^H10!QWN7Q8U${aik zfyz!QKlK<}N);M3;=cB0#q$mtBd?msNN#koN~aq+XJQh>5>%F?Sh#%?&3I4t-8fH> zwbvjh1GXE7XLHJ)E?C0zImPUmlye=f)E`G^Z=VR_+`-Jts+)s*y)CnL3N5~45RH3o#gQeIAjx^bypkP6qbu3{fl}Lot*wtDdU;3-_zIVo%=~NUs>7^nXk)AfnM~tIGFZlOTJ*g zk0d5=PU?=KU&?Fr_O_wN^2_zW(!_u44q__Bb>s^U`tEUMPm^#0X@FJ1?F=0wv85As zx<=4qRi*tDelps4=Wr~Q^j1bHEqC0DLAkIwi`aF@zph!T{U)GZ{EInS)<$l8bF9*Y zzdCUJU9>}5%E@3*Z{(|K0B~h!K1-1~`trVvj7MF}PKRb2OfWPWMcU~w2Wz8dlR4H| zr2%wN7{P9-%TL6;h1V^HaR(7n+M9#S-ae*%zROxr|Cv<)CuFHV&rC@omw-$!f)r>- z9klre3hapA2n-yLm;DJ3?rAUv#bl(Q1ocA{7kV|D{fYdzT!!Jc5s=A}En9)h4Y7w% zhfZ!eQa<}+ViDRq(ENdDR<=F}AbS|@=tovKN|!=zqb1~~1gkK-T^zietiEuDEgtqI zVALa&6PLpnMXp5v!$A1^6k7*3Jm%BPYk2XeW*uk0{>L9w;$>arP#-4?-LJ|B6+=)nUz;T5K%t zF>ug19>}XGF@t=iux->(`MPhr;TIFDqSyMHDB`(f%SFTz!}^#qqMnxE_(%MdV{H5Y z4Meng4ThpPSlAS}C_?+xzu1fmhgb+yEdCe5{>veT0_-XV^!sHBX4Y#DJPdfW#Ndsooe`ZPm zO?;eD6|5l_&vZ36Iyk%4Zm{S6<6tAuggKyd&%?Sk*{i|@Ph_^01qZgy_ogBUt+>&fM^f-FBMDbJ8Opuzey86TK$ic zpK*+xEHa#W7tfG!dRjXdeyQ8Lx3IWL%eI!^`b|g1s&6C2W^GSWh0_M8XfM&?`%-|Ueew%x* zIqJT+BdXLe|0(3Qa=TJ!TZ}*Iadl!?Q^s};L1EmAKRI&1d)ziLDN>qvDw^Q`m1?oQ z@eW=1WZENjHZd)QR|YPk?$f-bOL#Zm3L5n3YEAHNd$!9EgadVIFe=;sp3`-^iU#0+ Ug&8P2fK^FEWjT&<1rY1|FS>V;?EnA( diff --git a/summary_results.json b/summary_results.json index d36f8762f..1754435be 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421261, + "Performance_Result": 0.422108, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 4.10039, + "Performance_Result": 7.68663, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.32138, + "Performance_Result": 6.32624, "Performance_Units": "Samples/s" }, { From 45a0822ba6b2cc920dfb386a915efa825dc0758c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 22 May 2025 23:34:59 +0000 Subject: [PATCH 636/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10939 -> 10941 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index f89cd1a24..73988432b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 22:08:57,031 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 22:08:57,031 submission_checker1.py:3308 INFO] --- -[2025-05-22 22:08:57,031 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 23:34:54,567 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 23:34:54,567 submission_checker1.py:3308 INFO] --- +[2025-05-22 23:34:54,567 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 22:08:57,031 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 23:34:54,568 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 22:08:57,031 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 23:34:54,568 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 22:08:57,031 submission_checker1.py:3324 INFO] --- -[2025-05-22 22:08:57,031 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 22:08:57,031 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 22:08:57,031 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 22:08:57,031 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 22:08:57,031 submission_checker1.py:3345 INFO] --- -[2025-05-22 22:08:57,031 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 22:08:58,022 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 23:34:54,568 submission_checker1.py:3324 INFO] --- +[2025-05-22 23:34:54,568 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 23:34:54,568 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 23:34:54,568 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 23:34:54,568 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 23:34:54,568 submission_checker1.py:3345 INFO] --- +[2025-05-22 23:34:54,568 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 23:34:55,527 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74095 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75526 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.68663 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.51711 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1165 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c2306a490..5c59e571e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1263 +1264 diff --git a/summary.csv b/summary.csv index d60607228..e17376509 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74095,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75526,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.68663,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.51711,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1165,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index edb4cba09..04c77334b 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74095, + "Result":1.75526, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.68663, + "Result":7.51711, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 86573ec64b65ae3a77175a838b5933693f24deb6..5b471132c539489370fe0b23e2130b073f313086 100644 GIT binary patch delta 2609 zcmYM0dpy(oAICQgE4Q*`puIw!TEVnjr)N4advvej`5xqRnxj!R-~u5&pw zw@yMM5tZ8^oHCbWE+<6F<%sErZ2Z3Z{qg($@%iWbc)T9Z*X#BEydR&Y%g-;@1j)nB zD9#@Vs|}M!zyST~FKrV?bHta|ukE=?2|ENwQ^jdAoMsPYZ?*!HgoU7~zow7>uAfsP zCo?FpbnOmPeN&`c!V5=Sa3A)g)VQ)$=vlwP7LVloR)E!W^2B_V!7aG?WnzMln>>kF z`DqL{!JcEKyj)+qFg5m5no)m8>asP?B;KE`P(;Eq0W?*fJDPz04DH9Q2`2v8V3yJ0 zpY@q##tZkb;qIBC3{Qle1WBw-Gj)k54L$yNDcGj#);OQRdFlA=!qirJnw}6;w85Li z)7c6&By0;hB1}zKXGdpivg|CCCzkFi(|fX5Y16iNihFwIRDH7cIDQG(pEpm^8~u40 zaHEaY-5H|_R*lPNW;OqMF9gcIC#Mc;jl7b8ar~(r=&tipMr~~qvc#`Z0SxT|ybIu4kMzIEd^oW@hkGSWXjr?t3fz$&&y8l!y} zCXh)?%$TYtJ?muD+TWdvEN-m6l0JGMGQ``RVo@+^5?LvScLeqZmX2CtV?!dXWrdaA z>os78&Z^`=@|zvhM(R+F6wpRJq93;Rk9 zQ_cwc4lf-@6?)+$2%O`-z|#3$&1h@c{z~ty8ik_n)Pv!EgJiKl6Jp0%BwU!TjQpoY1w($3r=Q)Y$#JZ#NcU%XVCS%DY12z&g&~d#UX^ zU-ZyiDlUs<=d`WV|5L_S|6H?9J`016C=RG+W+7{x=7ZWxy|K^ps+9Pfa2FUx^3{2m z|IAR&%?rnL?hVf8>t%an5|h!%Mv7W&cnS0@R-OWrPNG0;sKfs7(r$<^4+3PMq1N#b z8H{#8nKPxhva3g>XO9%C${jN5v%36+65#HwYkcQF>GhsAnL>g`ws*2dve6O}l#^Iu zlPIvQL<+=~`WBLf-b%bH<;I=)0~~m_II(3peD~~`l<9r;`#!o4bmx-s`de{-)5nW? zO*hB+$1A_01f~{{RoDdec*2})T5+MjvKcFISlott;XQO!BE-e0>4DQrh#;N zL8zi;QThe?aiDkKdU&nC*Tj(K}{_NVlQOIzBx#Onl9{4Xwo7&qAl zl=|cVSyH3Kv;})N{}!9YQi!WE&iLx4Ug(>wE(4U0a7+K_Y7nFObj+Q~5H`kx;PF(P zc6GJs>ZpPCzS$pKZ3w7+)}Ir}tiSgRDEC5gQPN~?J_yL7U5df!e+QJ$EfTY^X0P}! zKT699xtV_~TYfD3nc({DlX&LXt7oxJtL4RYo^j|+OS4PgKt511-rt1<@&N*rI&pAD zT%!aI<|@y8^E+VKcfcFmj=gbqwba?Ufc_}9p=Ym-5rcbf&T0%Z0k;V;^SUqgceXNn zAZDfVOGK6yNmSSz*v9U@uS6ypb)l)J)jKN&O1;7_JPmkOIN?hvaK)NYu7|ye?2n`-o^R8(xZP8gfD2c>P1QsLS)6Wm?nu5$r-wD zkq8UCQ3b(RnvM;W8^Jvw(D!=}lD7f>9|3Tk56a+M#UNpziwO|&`eWa^Pw-U<5+gy* zHtJqHv4q&^I)AT&v#s2%Z@Y(xlKGXNn!ur;^63_kat4AqqK~k5lJhz!qXnA9weC&} zhI3F4-r7l9=In^Vk{6#C^LsjJYNkbWM(oW@dj73sSu*e9G)WOi0^v?;?ZSlMq75+V0%nmeDxTS;bosQ4?o+#4i7oZCUktdQB8>plsGAaX?@;(KuSlugC7U$z%q430h3w* z1EA-YQj#>^T>ATlh#C58cSEQjA{&V}k&%6-MIjkS9u&^(4Dsu2=HA`k;XPUu|7m+A z-(f>(#~!vFf9aNfijW@jay@%$g*(iMEZW>02w0mhM3#FDl;H>d_2^1yH0^s z=?xJ^-hePz3WW(XFH#7%uBnmUE6v>Wso#tI7k^jvL_ZCei@Ix`?^P8#TOTS(4$GeZ z(*CV|N^@EmzCu6wW``XO@VE7wLd=?ID4>df>N{;F@$!rSuEqDy1|dubj)^`ss7DuG zTX%RdLiO8-sM`*2?u;@?Q~R*idwWeR+RWl?cjGj-8@NM3`0ZeI$wvF6-1?Wq*sZ}n zMTygghI<&v8SbMSrZ(HKyy5QtW?)~zy4r~k?gKezdGunJ-T=Jng7Ua$bU~-+t*d?Q zii!PT`0X9_3C0le{2g)A;@pIO+ThmBRa?GfvW@c+Qmd>wq9hYdjoiywKp}zN>+L_P zoLiUd;XTI?<}W2I5k2|Wiv$^2iwU^EvybH19$moIzexEq&GSJa%lVDRD?An{YLd86 zi-#H%Qi_hlh2Y{*jp)kkl_M$nZb6vM4mm|ZI;lGM_pxi?v>5scsgv!3 zSee8y#Y881wvl(s5veZY{0)v(da#Ew;>qVP*SZ*r-kFLD$y6ji1RPM{< zrW|iGcBp1JO)cmgs^Yz=ZcBbDwK1r&miu&2r|Zn5qdYM!u&?LZikPo+ph07iaejih7h<+{gzo#8XtnUf^f#YUi2F>GS8kj?q wIPVO8hJNE94ABR7Bh`gMAP!Ox$PbYJf6Ul1j+5aT=rIo4@QCb`-tM3O0(8FDKmY&$ delta 2590 zcmZ{mdpy(oAICSFxwA8IqRS>b4r$RHA2}3S9TZdz-6zZgWT{=k}Vq<0G za)wkpnM)y#%Tbt1ZXL&j$~8qKcgAlzfB!yzeExhro}bt2{dzw>f4Nt=i_a@7wwovJ z2rs>;j8X(wANwGT!GC1we|t)I*~KR4)f(0VMZK`Q04#v39-W{!U|4RTYCH+8C(L)c z&)xgYGDv08Bn$4wUts7>r2+5^)+y+k^669e+a#|%s5a1ct6m1Hb->Y)<9FpHiB>-P zL*DR=jg8xKMAFD{!u}C!6s-*-0AN8}^_&DfFt&h{>u}kYpkWhM{{5jGQ=h|DsyU{u zm-E#9JXpT3`A6I~l{!T~CA#|s4?42fGP{d+d8UaaZ%nQh%*yPVcP zPX7yzWzesDx4HAThi9h9Zp+szN2mnJ>rYunv@LHKOCVdUf!A}S`~MGTIjx&`Cga{F zL=Y}2?tUtj4w7U^i!D#vQQscTJ^2>SrL`^BFtsh4$^o2RH@wJFP!aUeH zIJ8_>z?OR&!~E*$nT+j!b9z?di&s(&zanz1UT}JXD`VWQpH6rpbWsY;fBGZ7W4jw!{#F6frw!6=pVkt{d27pS@Ct z8bw>twr)CNFQb>38K9=Nd?&E7X#DmN5wwzC9pT?QHI1_9!i46pfIs462%tTpQWUTu zRzK%}&wAbsY=|`>RNj)Xeoa0~>an}6Qy1*@E=`F-X8Xe1hqMA$+K(=J^BG5({~n!g z;z;|=BMm$*9j`7F%aL7qP9$KSYtLz#R8OF+$ zS8;Y5wGDIN1=x#|9c&!?K7|$H#mwiiyH>lNja^*l`tR>@Qb{+tT|r-!&yqa*IHpH( z6a&!==wW(B*TtIAj=VV`Wn~NSa5H=?KJ^`iFC(H4H1~?uub}ZuuU0C%k523bDZLt* zT(U;~Ac`PC52ti*@vR!@9Vv~SZ@@uYl)z06!#KKr|!! z-x}wgkb3KS1^3{r$o*#~om~DHnM94*39W`5#^RZ&M)%OD!vwPf@-()#S};9|p%h&p zS?&s0x1}~)?D#{aQOmT!mlu{;b-KN=V~KLXIW61F_e9j3FW93t#jP#Pr)HuP7jNvn zhvvHSQ={yar6P<>tS*+M?z+V31!Q7Zh!gLNuzu^kPHqoNTzY?ERh&gxDlfJR^0SNL zznI$f+qi=k;jS{tB8?i!+6PO!OV-yxISa^u5cgms8Cl{*k)4t+$OP-U0teYCAtb~2 zi(MSJ0y8(bXDn73mlGzI-dquekiZ(PjI|G&L6e&%9p=A23%;{VCfx+G=0W>oUUbMF zgJ>QL$ix}GV`?iL;P4>NkENUwSiSGZK(^|lvNbJ1=>rEGlu1sJ+3VT0RAvz?#hv6- zRS0E;s9kx)94Vs`FgvcIoPV(eKp}1tnj(j*hXu;vn4{WGuk)bAnh+Wf!Hh$|%?1Q7{}Ori6zgW3&w9z+c4 zFxD(cUbYilb(i3hXoiw)zC@vaXxY|?VLMe4)1=9=pUxT^wv1I8a6vCD%W!MAXAktzm!9UO)a#ijehl)QL|=f|KHH2q`^ePr1+wqQ@oV zv2~UGY6J#UEA?GIR~}qek>Dhh0Kw|RQ|JFQEC2JOb9L2;cyupXa|zkecqqcKlg&PqoG5%&@@6E zzZF`&{l0&%a3Z^-JagW;;PY(LKY9Gp&)Kc-o6rbyMLb!3AUo>J`)vKDME2xT7d!ls z>)gJ+GN+_*S;qV7+6ui6ZX=-2QhVaX+?1&zl(@1*=dwh`3?PS z$b5}o!G+Z2y0U;jyIh334|)?W>XfJMP%3)KI7f+f-eVux!byxUVB9R8uy$$txhq## zvEQi}Ni7Fo@IRk@s^~bkzS~UMQCv3*z1yE~<5=EX?e2|=2#9i0uA-vVrISp>Ly@op zV`oX~Ex)-*I(;gFZ4!8qGh@uDXBHn%#}UW(%}#t6<1mP6tLLzqg%9BO2cT+=M4cMk z@H_AE25ikl;@E7?Mr!A+Oo3t{7Ze6oS7QzS8z?FlDm@0*Yb}~ z@s$0XqaVMXaY Date: Thu, 22 May 2025 23:40:00 +0000 Subject: [PATCH 637/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10941 -> 10940 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 73988432b..fbe3cf38b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 23:34:54,567 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 23:34:54,567 submission_checker1.py:3308 INFO] --- -[2025-05-22 23:34:54,567 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-22 23:39:55,380 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-22 23:39:55,380 submission_checker1.py:3308 INFO] --- +[2025-05-22 23:39:55,380 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 23:34:54,568 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-22 23:39:55,380 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 23:34:54,568 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-22 23:39:55,380 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 23:34:54,568 submission_checker1.py:3324 INFO] --- -[2025-05-22 23:34:54,568 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 23:34:54,568 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 23:34:54,568 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 23:34:54,568 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 23:34:54,568 submission_checker1.py:3345 INFO] --- -[2025-05-22 23:34:54,568 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 23:34:55,527 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-22 23:39:55,380 submission_checker1.py:3324 INFO] --- +[2025-05-22 23:39:55,380 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-22 23:39:55,380 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-22 23:39:55,380 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-22 23:39:55,380 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-22 23:39:55,381 submission_checker1.py:3345 INFO] --- +[2025-05-22 23:39:55,381 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-22 23:39:56,380 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,14 +27,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32624 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34096 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75526 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422108 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.51711 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5c59e571e..d07807cbf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1264 +1265 diff --git a/summary.csv b/summary.csv index e17376509..72a5d9064 100644 --- a/summary.csv +++ b/summary.csv @@ -5,14 +5,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32624,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34096,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75526,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422108,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420065,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.51711,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 04c77334b..04826538b 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.32624, + "Result":6.34096, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422108, + "Result":0.420065, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5b471132c539489370fe0b23e2130b073f313086..5cd3199ecd378cc4712defd45e7e688c7305276e 100644 GIT binary patch delta 874 zcmV-w1C{)}RlHTOatH;DBMe=3vv&w>0|lM@0O@d(>I-~-B`EqFMITWV51{85@#UEd zBY+}_NEAs#qDUeVMc;rTDVHc3%DY&ODEbBzu^v$LC5og*Y}#+hw6s{(h5zqCv5uuw zK=FgWvn|>k?R83yqXB0zKo$i>&oSbQ2iYk`fJG9KERu+15hKbH56xGB0Y*|UpN73L z)OQ1c%9(6`2-yhoJSvNNq)0yVc#D~ieEt&D3*59nh?9d!f7Qch-~HBS(>j({0m93m z0AbMbJzaS2xx=8cIE1TqL7}*BpOcI)9=WF^101r91d(MVi6o=sODC_kA;u!@LBeS2 zp}W0tizBpzbQH0FpF>%Xl*!i|-?FZ?_-S-eSipXNK>4sA=0FakXr4VJ7fEz|r&@lt zC4M!QwXCoLm+v+cTZ8Gf9OV^WxZ4Gjzv})>;j8S9Bf-Rs-;rRFSL*>xk^wMD2EfD& z?vdLN1(WoEU>bVpD*4XB8MK6S6iYz?+w-i2D@VgWyjdF%MmsKe?j4|)$Foa;{0f|M?C5Lkc_gV z^g}|(W(DboWq?hl?7(ZYCh3Qb0&tp~9}UhAw1jk;Qk8|W9xKCrlbAuUZ`LgRd_9~` zInwT_`g5s^>QAJQ5Rn@rpRHcjde8wJ=j znLirLe;;TG=`?{&Fn>rHZk$!jA6>Kg^L^OdSN}&#|3`Ul$iiFeDLDvQW2wFlmP4L*EtVhe_#z{i3aoYRj|NLDd zdo7EsvT@o)4Jcb%;9Dgdr(L1}+31saQUf9xNM%V6h=h>M3LX&40GlKupK#hH4TyCB zXqq%28#Ewj3CSSfu|rjlRmp9W-661T+IwdKeOr;emi1NHG3}xRl>%8*(o{!`I8hXCIqXf_bZb=Di#6SlT|7{0WOo2DkBEqCjbBd07edz AmjD0& delta 892 zcmV-?1B3j$RlQZPatH-J4eBFzvv&w>0|p(fix0yclj#e5e-1~{=P3G!qIdv3$A~Y_ zTo?fqNkpPZA`(Rsktq5G6iK;6(NNySazxQLposN=qAyV-HDc3#OQxm8vM&684~lgx zr2>i{{GDyl?r5)5avTjfivhAID0+?&Up&Z8F#;@-h-8sOB#RhPmUw8s3JfrkdigZ$ zjiJ692vp8we?!PdkmpfZ)FVamna5kqbma4wpkCmn{Xv`@O!})HKKt&sKAYCDyb2Ir z1_cO%mhb7pbI%{reosdZbLg=J=L%t;J8Hi^2l-e*?;g{V)e|7)A5!A-PDR>pRu* zvn}zfv8-i<6}Wu2nb;akujMGO@WS0LnEX}uX9{0scN_^OX8eu>le}6FV3G`gNiqN? zW^j+(hA5b%2L#j5Ls!Xn7S5n0q@#EeOj*`rWw2M05Nwu(1?(f3g`F}dQ(Ga)oaHUWQuUGjIfbcyq$(H`-n^FuPq zlF|_&nX&_~&6=biHVVLLa(*;8KhP4=X-ZWV%6hB}_f293!M<6u^z-#_ zKIKTer|Qq8E~-D-zG-Q%>}8-5^GBmX;z{#|WRxY<9}+?~E2uv#18g#J2iP>JKWr3W z(`5c=Fn@obC8W~?Ho^QMWw>!xF@JQ;=Fj(Gb6@=*E&U(mxg}3G?GpdTMv=sm{twA0 zOA0_F1aPv9gp*|?oNPh>r)?5|*eDWCQwLoe1R!V$*)+jphq4|mlN%=q!NzItlmGK~ ziR`s3vdYG37d4=4ZGmrRO?HRCwrTI31@vu2_FC3gWyiFO5>U2dT8=JzSJSga z6r&IXeP(+Llk6lc1L{(QlN%*E0V9)EB|HLsz>}0Eg8_T97$)EX1wIYxBX^ViC|dy$ zlUXS|0h^PSDM0~ Date: Fri, 23 May 2025 14:04:46 +0000 Subject: [PATCH 638/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10940 -> 10941 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index fbe3cf38b..7403c6ec1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-22 23:39:55,380 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-22 23:39:55,380 submission_checker1.py:3308 INFO] --- -[2025-05-22 23:39:55,380 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-23 14:04:41,157 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-23 14:04:41,157 submission_checker1.py:3308 INFO] --- +[2025-05-23 14:04:41,157 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-22 23:39:55,380 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-23 14:04:41,157 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-22 23:39:55,380 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-23 14:04:41,157 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-22 23:39:55,380 submission_checker1.py:3324 INFO] --- -[2025-05-22 23:39:55,380 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-22 23:39:55,380 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-22 23:39:55,380 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-22 23:39:55,380 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-22 23:39:55,381 submission_checker1.py:3345 INFO] --- -[2025-05-22 23:39:55,381 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-22 23:39:56,380 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-23 14:04:41,157 submission_checker1.py:3324 INFO] --- +[2025-05-23 14:04:41,157 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-23 14:04:41,157 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-23 14:04:41,157 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-23 14:04:41,157 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-23 14:04:41,157 submission_checker1.py:3345 INFO] --- +[2025-05-23 14:04:41,157 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-23 14:04:42,295 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34096 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.75526 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74958 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8397 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1457 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d07807cbf..8e7d494b5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1265 +1266 diff --git a/summary.csv b/summary.csv index 72a5d9064..e7078d698 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34096,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.75526,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74958,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8397,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1457,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 04826538b..0d7704ddc 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.75526, + "Result":1.74958, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8397, + "Result":21.1457, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5cd3199ecd378cc4712defd45e7e688c7305276e..a56a3439303868c4603cc94eb61bcf6b849f0634 100644 GIT binary patch delta 3516 zcmYM1cR1VK`^Uws8nHUmCek2gW3;rY6cMS>+C@vMi5jJ5jp(#$c0%>jQlovvz%kt+_?!8$H{~*d zg8G+9ucDEr(%Q~!18Bvv`=X!k*PYn07lmnd6=pXJBgdNMMEFDE9~P$Vg=Rs4AA^RE z(E>}3GlBj&Rpf7%QOAijvtUm5oPFrCa8!C_qIt{jsihT7BLIJ;qYbzu1!) zKe6pGQS|v41=svG7>ilLch*geC&vx6bQ@1Vo@1!S!@s=@Vqu6Ejm&_fRm@66&-;<$ zHdUysi!EUBcD&PAVl7gIhJ*~I#VTf9ml<|@-kW>PT)+l?NYbaoB?CHfvLK3$ow}pe z(CC<0H*o8wrgKWNeeHT*`x`4snywP6Cx?mS$~nah$Wsr3s}xwb-I6 z9ultocj$+8`!BFjhUrg}`PLk;(Sxn2e@cAX4=%--Df>`3SXUbA0Ssj#<~4q`)2@9G zv#)_@c%S|WzZ~Y3SJr2ZBZq~LsLr#ugbyHnGPy8OCkQ?U}R9Y ze+|RPORjN*iF*E$2Si9OB)eVrL&78xtN zk_40I4hyy;&gpWGWhG6gTRRc$J7^r38p$X=$XJfD80 zoe#n5`*a0UU%VJ<)?yvWfBTBEy1}SH2n-*~3wIGPl+J3@#_7?1c>P!)4w6K-QB}qJ z;olpw_rqH@h2SU5y8E34Q$SX_r=>pd%pbSpj-LHzUzxSC#94_f7$M0*SEtOnOEu*_ zU6iPjatm~}RzQj;S^2Se;U5n@O-*9BI4cHkU+b%Oz@8p)E;uvSAvhpXo03CapWqhHFm;vv#p0EWx9=coKD5eFS4cwhCy#B>4;US!5O za!oalRNLKksSctv1Q!w-veFSUzH+Xb?7k?7{oe3V-DB3z$Dc8EgN!3miYm5Z>^@7i z6#u#@f$rgbZXtOr2}XlakG4Y> zK*NajFT)dCt7c^H{{;5$Sv5Q^2`hi*o0daHziAGNT24YfbJ1P$VnDd3Rv`iF~1t~i>H93T3*3bK&nM` zT_1&b{vt211isY%3%nXRDlEH_8(65Sm2&1)tB7i0o**|B>X(TofDDlQLl|z6`h|ia zOuR@vZvJ5-W$hMvSfbD247D1*MA92G(7iiay#w}@XQ)|?ocQjF8x2bxtsbSHX%g<0 zxMuZGbP5@mh~WlKf#pErz(uf(B-ZW9nc@-Mm|6h?2cfmf5p%tVKnhr1P+aD$deNNy z_bQ+PEGKs8!LTRdjs(I%a;>!an(JyI&xMpKHUz#o0DA2>sYQ$syQ^AQMK&TH374>Q$Df8s z+6X0KxKP9_SJH!a0w$Za^2lWb18Q)X4o=EDF^|nmC{y+6&wghfCVIuw9h%Vq$D|^? z<_k2-W;&L$Uyof^%d@3HpWX?GzwLAxg+H0|vi5nGpz|RS#c%ZD`rm#XTG>x{Z4gKk z99NbtOhl1j2Gclv8qQ;{*Jz2t!_fRRNm8Fz# zUtkKkbJ1A`@5>eLr_kqb&H%GUU2mR|=a%eAka!e(gsSXkg7M z$7rHzl}StGe}26T=-7{%Q0Y4-m^+^l<+#HE`u_|368|so=U2>by=VdoSg$bKzC}zD zzWp4Y9gcUCF7Zj0=B*hjs4DGfWm1|# z_!c5*&&Y;vdKU0eHjYiW7Rus7q@oq*o1&znj&?21b@Jj3GY zhcJ{cn#_Lt5e+kdr6jMm%Mp*0OL#MchBZe$EJ3&o7aGqAxEjgqD`Wi|@cDyXuWN-3 z(NlcytKAyujF;F`$ed?avT_W1%NZ3b#E5?+VW2etsFq>2tl(Vq?GPPtChKww;{K5!)FDk#n) zi$dIR8rEGY=gnTpBR=hpr?hJ|Bm}G@V*y(!bp=$HShxB?`^%u6^cle7kNbBQK~!b} z>uUy5&E~Xv6p$kY!zZE7#}9nqJQz4>S&RmYsSxv7zbq_I<#EY0dw@i?nHwx;S3Fy0 zJwbV8zbux3^l>6#o9lHExWxpO2nTo-i6BV6JCAjNu%f|!MVwwjfX;t4s-St}Z;kR> zJL@DYpEOJ`u)c41Ce-_bLW~v5L35^=-lQ{n4~&m2Y-bd1A>=a-rNt9GpNG$o!*`P- za~FTNZMRLzPYp*=!*sgXgt*|hd$2cQ2nr6qcM)%05*fAqhD`so-PS#06jbT$w-$12 z&!l_xVQfS1$DfSryK7Nt!?D>tRtvcs^UOJhW(Ng8>TQMSKb{F&pVQl0OWXrbOoHQR z&NZbn+ch6UR6Mrh5FX?H*(@Q0XF!D;4SI^!8*JHs)>jtXc+Xw)-Edx=>Gj}^@PZy^ z?^nGaJ0NI)x*ORP5-j$aK_it$wn31zcT{capF2W!d)$p;2Zuy?iy#-3fMtjV{+(k= zy&M+hEFx9#^=ok^H2u-WX~(54+Eb%sAi>2meYN;lB)?NPWx3qD{v6h+0*-XxO~Gkh z@Tl1ARgXWN#kdeMg5%Ra()#eq4HsSWJo0b`YDO3wgxj-qtfn9PIpl#o?V z@(5Jjc=$roAUm%A;PsN~P1oBs_@bLB;MO5qi?B1H21>mdE0$bW51+^^d)wwlm`okK zlM_U~eBpVK0nB0h9fpPTf^ECak2yP-9>k*OJtKw&+k{tuzFO>{?#Z zp;xoHvul*t&=OF`SW{^klc!M&+Qt2UzpP!D2j>ZMQ!@b-0z@@4P!cetb^^2wNUB3+ l{wk>o;^MOB=He3P`u~=2YZSPkehQ=w0O~6IlSlu~{10*|wUGb- delta 3480 zcmXY!X*|^H8^&j{pD-CpBSsiIiH16vtXamsBx6nXW#rfj;iouRW6Y0{HA|LG5;L+( zp@ftznL(BivNR~$;6Kj!KQEpa&*%AE_w{`5`+D)T+O*hI2XV4Jl;C!{4c_5Y1NyVN zZc7K|z!dnum<@)fVOZn(1WxF+V-KPvU*j-a&tJ}9C1?ex-$9ytVht_F9n}olA`8rf z?hJ?{x+veueC_)5(Puh4=*#mBI|mcJENxW7!2|s0<4(##!;Yj;PFo&hZNa)_TcYt{ zacz3&dE?aFdY{Xh_;i? z%Er~{zbs4B!q7snVnMpgpAS73n?ODEVmWma9HV1=3WRyOsQx<%KT zmkVZb&tS>B^~xe%U#u>k89Mm>t!x4xDFhoDYx(wWCG1kl`tb=oHwi0b4&0c0q9#&l zoZ0-;hD5v0q4>GRab-Uc5C@=1a)b$O+r>p1C45oA0~i=2gn;w0MDI&6<8X8ZZ-|iPn_l3w+~D zNPzsdg)c7aI};P87EdN}7C3_mVXUoUe1VJ2DDl;{3n|`#p}KHB+pWB-ou60I6B?QV zuvZ-?jvpOega_`he4K+_61@D^EDNthK|^)<>6*6@HwX#1wHGMykv+IX4;&U}yMi1#D(4;+wcG`Ov3=f9sWUn%+KHWw3QITB$AiBTqhG#v#C(VhS>HVrY;%4P5aFd+ zm8y%qjch`c*Y~D9*T(ppOHno5HB~dlo zD4mT`1zdXqx@&7k8_u-|Ww$i8uEN&K(HztZk3wxZE+W!5lzduFk2_B;lE%oq?8c?) zCvlZ-vL|toYbSBb+lsahk}|u(dZLyBu)^=B>2uk~C1S)A4F#}Qiw(u$dXq8YvRN`h zJOR7vwNy97GliK@dQrp(1$@JKtsqzDz zDQkvkjw!=)Ey~1{Df~!R0mAGuL;W4#ru(8= zlvLI0;iaJ$YF2`cx^9n?{lPZNU6bj(v{Y#-Q_)CGITlakuEwgomI}WkNUb%+6OjZ{ zUD(9RLm0k?L`a=6R`ak+%bc!D6dPAt0R(cFu+p^4BRHS2+T=0M{&j`&**WCfNh(nG zlP9<<%bQp~0bEIx@;qTmgrc$R{hi#}6gM3v1aBsUTLF<;)^$t>TGpgrhKO6aPyEK! z)DNNPA2_#Ce+s49e^+EB(?3`8T#`=XYc5poBYIS~lqZ3yrG{l6bw9>az4&OyW)0FR4a;RAz4=knWs@wyUU`Q^pGkS7)-z`Q;z#5pyiIXW* zC-goXZOI>4x^#5qD@+s+L$G~7RxH)XjjDDt0>4yJSHt5>rgtK*WZd;-I6ifnEB_WI z@?thi{hEc2!^xIYf$yvp-M5uzDzH7G% z+?Z&&USHzu1G07Mr?-i@E)_o0K6al(FttA59r!cjmv=Vi9hsmg6<*|57ef-IST6;2 zl)9+2=2r2Ncet*zDenKeHbQQalg2rrW3KfE=aHqe>@&3zr4r?N&JwwvK(DHBda$u1 zctDIiN4&X8fo_Ov2_!=bY8VS)RTRrFyXH|;+0}ER=Z|BADmw`6=K9)C4?jB>NrmL! zX>|@pnN&^tESE%~MEMm_rrf@+N`eR5Uc3jy*z+@lB()d+@E9&<_AztnaQ7Ly zq9afjpY>>T7!xW{yWcIIjFe?b%May2Hdj&iv=!PK-#hohpCv3eRqQt?HoLKTbd8#E zWi6yNSnY{rYx!!TVjZvdN8{KVP>88P_x)<#7-Mj=M0@uhiNJloN+IJH4gv1*2?31M z8|%$CXtHAbb9}Z&nkat5#dvqZhd%~ky^w4ai+fHU(+a`IDjF^GuNBVhLIyzv^Pb!` z&%!~?$$Y$~J6~{F)3#}WF{|{x>7Mm^c~PyyeW@K?jMXmF2~#rG`#OQ?q#uRQ8k%KP z^FmC)^%Cu!f1Lnj|5~|~(|P>9DaCW{yWc=0QO==PT;4xN@4La6;slpb2*#$$>OI9vKIRDF?|jH-)B?(j>T z=juK?RikT@X4CBtPKZ_e)9!jfqa4y(Nxy8U$}7dJ87q%SCB4zDo#$#|@S9i+Xd)md z$_!br+$oEj(7n_-@ak{-5nC=}LQKnG8dlaws|Nrx1!MqbcA|&h9%Tn1>@LsP5xx6K zwJ;<@;%UL`;V`+*==+!A!;IZDxAu9jf#%EhI3SHT3C@kV|!hcv)Kbg_I{Cz?ub!abO z-I%;T)eg2nD14_zd0C%TY{`QwD0x8hFQ30xv>eF3h5Z?%LT`XBmB Bwp;)J diff --git a/summary_results.json b/summary_results.json index f0463adca..2fa792def 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.75526, + "Performance_Result": 1.74958, "Performance_Units": "Samples/s" }, { @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.8397, + "Performance_Result": 21.1457, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 1edbe8b54c951fe803d8df87ac2c3cdc4ff23ef6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 23 May 2025 14:07:19 +0000 Subject: [PATCH 639/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10941 -> 10937 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 7403c6ec1..8d07aa261 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-23 14:04:41,157 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-23 14:04:41,157 submission_checker1.py:3308 INFO] --- -[2025-05-23 14:04:41,157 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-23 14:07:13,810 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-23 14:07:13,810 submission_checker1.py:3308 INFO] --- +[2025-05-23 14:07:13,810 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-23 14:04:41,157 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-23 14:07:13,810 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-23 14:04:41,157 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-23 14:07:13,810 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-23 14:04:41,157 submission_checker1.py:3324 INFO] --- -[2025-05-23 14:04:41,157 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-23 14:04:41,157 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-23 14:04:41,157 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-23 14:04:41,157 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-23 14:04:41,157 submission_checker1.py:3345 INFO] --- -[2025-05-23 14:04:41,157 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-23 14:04:42,295 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-23 14:07:13,810 submission_checker1.py:3324 INFO] --- +[2025-05-23 14:07:13,810 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-23 14:07:13,810 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-23 14:07:13,810 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-23 14:07:13,810 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-23 14:07:13,810 submission_checker1.py:3345 INFO] --- +[2025-05-23 14:07:13,810 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-23 14:07:15,084 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,12 +29,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34096 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74958 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.2724 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.2816 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420065 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419843 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.51711 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8e7d494b5..c686973f9 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1266 +1267 diff --git a/summary.csv b/summary.csv index e7078d698..319f22ad9 100644 --- a/summary.csv +++ b/summary.csv @@ -7,12 +7,12 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34096,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74958,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.2724,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.2816,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420065,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419843,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.51711,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0d7704ddc..2ac3ee676 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.2724, + "Result":16.2816, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420065, + "Result":0.419843, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a56a3439303868c4603cc94eb61bcf6b849f0634..d216f45cf2aea0406a2f8dff21f260fdd414b01b 100644 GIT binary patch delta 3014 zcmXX|dpwivAKr|S(`F96bl7O-G>T@G%ACT+5=BPJ%jTtc^D3vXN64WA+Gc8wF?67q zLxrf2Y{@Yh-bzF{Mj>+eJ@x+XKc3I${^xqG>-t{Tecw-9AG?-^$VerYoDU7D43{|z z1N7y$W}O>9HxoU2_&|TIFBaLew?!t{PK&i4b$#f1a>Vy1^9udfLoBDaSVu~WGKMa{ zzP#|U^`C@=)JLrG20E)ZS+7yMr`uKCf*VOo@?)E>Bk!Jt*ck81PhN=i!8l`0eR`M<#YHUwqc{L@ITT+caTy(#6Vw$} zHOtv*XZ_kI883VJmllNJ=vBxStQHNc)TU+2(caE`UXRGdzjacR9(i{+spH~OSx0dw{((e7=AY2#<*GkCGPhDQ zS~O2o%gMIH{LH3H%)LCQt^OgMflbEmAY;|FPnktlD;TQJh$G`E+|Z1 z`?*zE%Xe%T`@sz@YjG;P@?2sO1_+bZc1edITPvqD_nm5f8-7=n?)mY!FHIzC5WM{N z?h4ZzOpz;d_)?wMuH}0(9V!jNnOcZe05AI6S!TzD`%O;|B`CINqNnpHDW4#retb1`=>zpk0LTo#J;tVv7S)KI_I7 z<0n}r5wwiZAWsx^4Q)}GYU&nQ6uP@&`W)qL*b_kzyU}^YarkH2{heYEkW4g=4`j)g zlTDuDBEvU}tEhpj?La!!`jd)&yB+z^uQmm@cHM38b5lOE^;7h~1E z+4NW{t`m}VKZQE>7Lt{?-E43Rx}RgHv`w26C=?hOGzBSJKMJbd*hS0>;g|Vd%c!9~ z1owtJ&x<U^5SPiYS=qU@>_j1x*$_aWEX;+NW?fPPCifZ|3? zK)mh*Z+6%(W0mB4$1t_XHm8}bh6LdxwA7!n7+5ZTM+I0dVE;r0k!BI7vb;sBSkX)a zi}6qCCw!Vk=2+7*xoy69Ik69GbFH78|FZG^BBMPgWN|%pZ4;QRJa5@M|suC0O1B>?s~&KRN%ld1X&&4n6~Uu3%Tn$dkw?|^wGA%fJ2x^CI&nm=vI1b>GCX^<;tJx7_dH$>~BqG5EH5}4gLw7yNQgZG6(IFC{0^@8~p*} zSvmhL=H<}Q@o|8%IVW#binjB-!#aVpNa@4^$##@q?tz%dcG+&wJsuWpPBthwKfr}a z(EU#^=hCklQrQufu$Cl|&U8oh{G`FYj9t|+J6rnN^~m}VQ=oGC=K={lH^ zunGQd!AvmgSapjY{%W}*gvtiy3ZMPB$W8Vl+9+QI3D)7hXoR=(uI0_fkLBu#+`q&0=!nQP*HBC)=W!y>= zcsweH$5PoVO}0=Xbu1134K&b<>aF|=B&6+}m5zn5nFybUKAJb;5HU(*^UgmzX~LgKi|ROJI~1EdV_17)$<0fFRag+M=jUV&L5m3vG9cDZ#~7@GNslba$(W zm6huc$LC=MZv_n$_c+PXOr|5nS;FNRj5|7F$CtD*5wg^fn>4wqcnlYczj+EKO=H8s z;u=vk(iJq2RW^aV`9h=Dw@MvEVDyGNHu|IPLyMZWk{7TWKnb_n)vQr%4IRW7KkhYO zP&&=Qq~KC^uAy1ZT$NaRnd_w(NQFG6tyjaMn+~aPhLG7%dOTFpP${c;)&J(XUshTO z;-K`|$IvQElV}v{r~Z2X{*P&=^;$C;FNuw!9}Qx2-#3@cMb!I$XHSKc89XeTbgHjA zmW1v@1By$?ju#Ha@y_d+Q&A%ecH*MKFI3;T@fGX&`lg=u)fwS^=#xgR|Lj;ZxssMO zH%<#fe7X`ugy$tqrnR1VHC9MBr!Z@eBxTN}k-h_86@kxD zq_v?=l4>C`-h&=` z&5AIHXpRfuInFfJU5kp3$OSUj>bKy?7dEI&at-=K#Qf$;CAEGDwBHx#JLLE%i@f8Q+`}b_Gyiygx6u*QqB8|Lo}DgDnSl5tWs$*}ul|mn)_r zALOm&P@M3+4kpoy7s|W;4g7;6SE91+{h@dPdwAxFGP`23mTv$*^D=LEVPYZ1-Bj%^ zNfmZ1Gh4~;;)S3|)G4*$V8c+R>fw5&r+JE&jdMh!kSW;qOOVk&miW^dIq|=cNXys0vP~yM$H+x;z6DrU4&^rL- rvNQE8q48|7o*8t3t)h?nzpW&FRcIgERo@(Hz~<=hfd8ul0oVTk$y;~T delta 3055 zcmYk8dpuMBAICRia@|}~DA!rcmP@13#Wl8OxfHpiqNz9u~x;1Jr;qXSedh7uUss0Sc;0-546v99!?tU!f zLg^9r|$Z3{tski zr?b?>@2}Ps;tW*xsb&~68BP}Fg}7#;dS??}^kB6oROX4a9cW9l{a3p!S|Jr1j5S^YEB<+JxX);K{X;>7ar`u!KYaK_- z%}{!@9pkpBSLQa+t}#;e-j)a|3CV-o(jH|;49;fpkA6Xl;srZ4&U^2PRely8XMOj? z#r1m66cymt?$&;|I-I{t5tZWjIAnu*^78MUP10X8V)k3s>bKzFzRpaX<^Jc?nGo$D zbf*sDdby5GpZ2324g@C`mF4-J>`460*tTiRm|seP^M7iI>%G6sd^{O++I)p`NAgM`>gn;B!ygMwVlzKR8f9n8ZIVM?e5g~4 z&;+qaAlKF}IyFij=6Kk;@}k5W%qeuPJ#gf-QMQ2B(tP|rLca&CpwQ!DuCOckK)KNG zhaCBRRRhZZJMCmp$n4kiLJCKeTe1-2_E!CQ-naj#5?iJ3IjL=9Ae%s31cV%Pt?(9+ z%+!Gs9btJBsW5UghjBxS+$`8?ayiS(_k+-=qk9dE9rHPndDw5eTHoFZYS;(UKI=v8 zG61+Xvcv>{`6RV%uK{2V0usuMipn7N)gwn~U@`VryZF;WZFy+Yh;ahxAScpS^Gwdh zS7BYNE>*Yksl|6sq-F|=I#Q!(ElT!B$>35YA;5t`5TAp*V$MK&q?(W?(}_&5H6GeSl2YQ2a(hVeN_@&lQMG6~fhbn(@Nj4idm z^p!8ePRyFP^xC}lq4ZHx`pWxe93CmQsh)NzQvQk$OC(8+PNXXq`@ZZC1oYMHqL<@8 z1@6(~B55uOwGfQTW|?-HbYdcVK!|_R5LSI4bvUM|=Kj&N-t{q5vTv|MUav(APuAK) zVY;-}(XxmdgM*5B$LINqeEcB|QWt4~uaiz>4VIK%IuFIDXzxSi4!gfE3+9KZev=ip zaU=8ZlMdRB;+|IIzb!*0?-jq@Jh9=rH~@QD73-lkUHSuH(x`DbXb123GGDQ<>AH6G zi7t|)T?a&D{VFMC{-J8hwCYs;5TJUjnj{e)c;d;6S6l}cWB0IQLL^6S1q?Mt#5I~N z4k@qaxF?~Kwv_2I<;vz9EpfpzQ;EU!{omh#GLdtMzd?t%gSs-mVGo=_)fcWpjmMal z^-OB24{b6BawxfM4S}BT20!r}*vh#8zhabACfvc%id1#;vqPxuS74Ap1WtyJ;Celi z#Fk7{9_}T9t2_q78QC@?|;E#@QF@-~oXHBwQ zr;v9phsIy>3MA0>WIp8GYnJogprrS1CXY}1d9raU`?mCcEPgiyl;LWuq{p;3Hr;54 zhPYc+I}?nFP`I{CAy&>#OInYdG=eQdDFg?2Uvu|{)2VGz?aC}#qM3h-lxd*v1KNX+ zp0`_9JDx}rx^cTHYw51W6S_=$v}unm8scMl2g=8aZ9~;-I}vN4;jy)tK8!X3JY05HTS=#eX^qEjz6Dne#5^YmqkvG?v&`?bm!V4E+6qWr};o1BQ%c% zQB-L@vh=d1<&G~ab_6G=ecI;cC#8zJFTA&k9)EIjO14V!-5>@-;28+UihqB57}_XL z_+;2|KrU+}E!uM#0EPfS+7a*e_`k$2oun1MGMxY-m+DP7US zEh}hzEI|rFF)2JM6XR#=_eg4s@#06b2~K*_4CCM6e*wXtlollIZK0 ztN`u-&18=?#R2Y-nTmy!bMZjI<@m*pLdoBP4{9_&;VEt>ewBGy?pr-G`;fdBpLsVp zBh$LAc(!DMbK#ZhX16S3Tdj90$56J267;Af9S%(6;-6^*wgqdZ&V~lU5*yN>-(>v^ zE{Neob3ouExAb(s)oJVa=P+)}ZOM_*;u+QyH=dj&I^p?MLVn;93vjLrN`E;zy#BXX zV)=OO>1yO?Vye_xZmzD<8=%dpT+hs_f~}@?aja~6e)xOzO6;Cox1F7nx0cS|O8Ma? z4EO6bi@8V>)=N4n(-(k*HG}bUiC5Cw*+td=0aF=Sv=a*MSNPnR^@@1E?aZ7Y-UQq<-oaU!vT_}+?E`8 zEhR@<&pP=^SGRp56ui#Qz?%58r-(;r8El~89rzKw%Y$o@0H9e?c!G%DeM-ju{u_Oa zv?~IN!Qv6tf+xfP&<11t(WV0jv>^-&&h{de7?B_{7ptd@1qS1_-j^yzuPm>oFxE$X zepqkJDR7*BhKMugCAOF-mS#_6bMCani|V&l(L;aXV=3Saz(J^!0}WUdN(^98Fu^FI z27C-uiS@PX)7Gt&R$v5J|5e!xZ$W9e2ZEPx46&Sk`@KIbM@XkjI@9nb`|VKCwai!v zgH6Xd`*unh=+_G_$<$lYjb=}O(L3Eq?=h||!$So2kV_l}@nC%SKfn)|qPdA=Dp}Nx zd23Zj&88XN${$?o76|tbe<)rnxjQboO*qPan5T;9_&+c8nd_zXIAFw0KM0PisAny& zN3o=bn*Y0bbTQ}uY#u1i+nheWXFJ^*cg@Y=+_8UjV_YO2n4!lrkIeh7!lH8OXLA;? zI%ykH!j~mg)v{zKWxpsOHUU%((DptHw{MSP=!idW43{kBVM; zDx7(BLporvX^pR7HXIQbj|Aq5o1G*8gJ&TcTinmsR5HIECCh)fVpDweQ3x zSoKvu%nNb$ecQ9)bCKLu?_)J>4UGsQt7#?bQA9ZW?d+7GAgT_5pL)?`*7W8Vcq^%MJ3I5zkvA&m z>^#R7rta)WV$lx=+?dSU76tWc5seog;~yYBKJSZu-YD1C#u6aYyGyg31`r$jW+3p&+2@c11TRaXPyxT9=ez7x$v@ Date: Fri, 23 May 2025 14:21:11 +0000 Subject: [PATCH 640/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10937 -> 10937 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 8d07aa261..7e818bfd4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-23 14:07:13,810 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-23 14:07:13,810 submission_checker1.py:3308 INFO] --- -[2025-05-23 14:07:13,810 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-23 14:21:05,801 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-23 14:21:05,801 submission_checker1.py:3308 INFO] --- +[2025-05-23 14:21:05,801 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-23 14:07:13,810 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-23 14:21:05,802 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-23 14:07:13,810 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-23 14:21:05,802 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-23 14:07:13,810 submission_checker1.py:3324 INFO] --- -[2025-05-23 14:07:13,810 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-23 14:07:13,810 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-23 14:07:13,810 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-23 14:07:13,810 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-23 14:07:13,810 submission_checker1.py:3345 INFO] --- -[2025-05-23 14:07:13,810 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-23 14:07:15,084 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-23 14:21:05,802 submission_checker1.py:3324 INFO] --- +[2025-05-23 14:21:05,802 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-23 14:21:05,802 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-23 14:21:05,802 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-23 14:21:05,802 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-23 14:21:05,802 submission_checker1.py:3345 INFO] --- +[2025-05-23 14:21:05,802 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-23 14:21:07,107 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34096 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74958 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70094 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.2816 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419843 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.51711 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.73824 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1165 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c686973f9..f038def04 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1267 +1268 diff --git a/summary.csv b/summary.csv index 319f22ad9..a3d7f9dc0 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34096,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74958,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70094,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.2816,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419843,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.51711,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.73824,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1165,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2ac3ee676..fa7f0dfd8 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74958, + "Result":1.70094, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.51711, + "Result":7.73824, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index d216f45cf2aea0406a2f8dff21f260fdd414b01b..69a1eb57812e12896786f9221ea332e303e14ca7 100644 GIT binary patch delta 2617 zcmYM0dpy(oAIHBlcXKZ%9JV^!+Ga_bRj5rmY1XtUa!GER-_#r`R4U&|Cv>{7xl4o; z$z{`ob0qxA&~YmfQ>qiAXX_A#tmzn zPw4}IG713J0YYQEg*aFHvpQIp*kZG*F=+0vtN?#LLl%R{&er)^2YdFRbu-!=5VC>& zU2m=H^;vc_L+#SrN!JBmh&Ns6d)#I**`kWhj-V>^`az z6Wera!p5OCgotL}jFkd;A7laSesT>hj!M?w87>{UP6_Ns$h23FA{q$!_O-jIw8cl*v64omr?h-n+; zD#eCfq>56^*+Mp-A%Kx{sBm5bmYfp^=YbY+6THatK?fy!qSKiod}l)iqK<(WlgL%A zgXcTaB-b6hUz26;By!20J)2y`ZLeAP63=)Jug{^zIiVA|-gc*bG<(?728H2*(>{jD zrP4Pn)C?6e2u6A`wx07SLQr{PxZ>b@#Zm6`{a3}Y|F*XB^Uh)3jqS3YN?^PIcRQCY zr0&PbtOYJ)o57t#IL{Y_?M>4i-;o{@ZqCbVTTF-$hvkpk z481tu*f(7^eyZ$B(TzDjM<$1>iQyJu-_c$sv3JXtQV~VIPZok9o6Wb)JS~+j-rfo1 zV&!f$%s9etgS~F{PUcGv=v94Opm<;DE0)b`=d{c#;&9(yEY^EAcHDTsbUkpc(~3a6 zZQ^Bez@36Nb)QONr*`O$8g>F1UH4%G#-!nA*gcYzqWcNSO&P?^=;S^|ilQQCAiJmLh=VG@0-mcI|1Ze}C}z)a#VHEi<~j1;FBW>wWn z)k2Ef-xzI)>FGtmputP=VA90T9SybntwVj=&4hpUIQUaLZw%pM>c>3oaz+{6T0=_p z`z@KOIKpw?fDF?LTMKIWKmOkM^m27~63sarNZXIoM+sbTHtVQFIMZIdmyTm-8d+)- zl60sRpgM}1GC{gS&|3{B)7yw2{amfbKnNyn_U}})3HGf8Yh|A0U8>#&b^42a&fzPN zS)Y*t7oyD|g$Vbtzpq(M8lv#lH20xbpddAyd_X!fR&P(ujXnboH-q%_7PCU?t3KWs zbpUUEYMSkk&>Jt&Nf1f}p}^-5%ho0vVjU4&-CRRTp;W+|`#vmlXW|Pn8xjkj8E^tT zHLabdESzHl#NQHq;!mahm5K2-9*SXopY@Z3{*Kw)cI@sY`d2sSvSo<)@CT6l1Bi$U z7sdN%HQ%8v^`e%cRBE~$HtT>QexRleaq%GUc^}vvdL-V|SCaXwx1-eT0rG)=f@!*P zIxCjmzUOS4u+LC>$#ZGFIa+-2YpXpKTJId}cv1|&&P_+^cNo_0VH@qn;wU8uL(!)y zMYD?}?4R-Psr7gN?#{4vq$u=6M<+6s;keHkg*27Xjr4fPbqI;dU0Z_8gspBiNZJ7n zOzkgGa+&`d82`hGJnZbnp#BP9g~E?iP|`6VYIn~I>s9o|(w%}+uHU7nBd<*+Z_Ypp zSeeoB`?Py2-YTmRg*1D9DNPY<1?Bv`TjDHylW`+28M_YBcbd`XFrDjQNqM97e5Y4@ zWi(46Jz!VpZ47H!y1<}+=WOoI%%)hZH3`h6po@uR zs#GJ9ugh|xBqaXz+M=c6IM^_uvKqWCx&Y%J0vp!zg&g!v;xA)tQ?>08LIA3Us9eX1K@_mMEjrdt zP)p>DY^Q0HJyke57Yh8^-QZ`^vkRec4XxCH(!DByK9(31FRY7Oavs%N98^l?@n_myu{P##u1=uJK;0%eJSDq)cHk6Y(WSKq8m zm|!geb*@{#P+u@6OdUBq`)t%0JVf77soc~RJJu@0&ML9Hu04DIYjFWP>v%wYL;(0s zcE?d!t%`b-bcTG?-ssQ`d6w__ zk*kz$(K%coPrPN(`3UHIG?a|Q%(Ip_JL6P}^PMNYumy6@zc1>j@I`*a*i%qe%m5^3^3_%b>SB1lR_|SFwjW6 zP#KF6%F#&hSgW1Uuiut+i~rg*9KPFF0*{cEC+rUJYcTCY8$Yz(cPw9dVeMl9|8T(d z>HMvyHP=oR^NIqiJ-A*)L>f!|O1SNdrafQ&gN>)A%^y1PnyZ0vOmLl=tPHwr7cuto z7<@o4iUr`^vhlr7&bivfegXsy=xp@=*}Yd1!c2^@!7Ju?-B_CwOgX#eYffEoT9e_KHEOB5rth66 z-nuoV|4aECDe$S>{_me+Vh_w+ExBJc-=3$^V;Z=s7(c@J3;>vF0ALLK|L_xD30Va0e*q9{xqtuw delta 2609 zcmYM0dpy(oAIHBlmRx6U2d6fgxoygtRl000VPm-zS>@zpbEf7TM{Y&?F3RQCDbBt# zHHlm+>M$agNOVEUEg798O1Twt!LK6U%!U5nX0MfBCu8^uuDT983;j?vtn-)u^pXoKUP1j;PpD15YhUyP!2Z=O< z3^G6n2~T^OWO=b*jdD+?vs-J7!P8vxy!@{iPj(-dTee?>=1b11Om4V%Hk)j*_aJFb znKBU&oIyIi+3xmUq@f;r9cFUrx#q%|jVwcDhY2-4Kt^TGH!hF(2czJ`SlX5eMVBpwbE;;vA%Prm%M9t;S+iOWv2pAot%f7 z_2r>t0Z&_4#8CM7wr)$5r2M56qP;JQ4eC!yIV*RL_3;;WI2C=0Z&=^9(_V0^_txyl zoraOjrHEuTk83U&El{_1>zQtasZoI}VWF8!H~)N8$b=Nmaey^zHcjcax8p5{_40D{LhI8Q7RJk%4hxJ-;h@{pQ0R6* z{G_xC!-Nnmt_VvOsFaREs2(tLU|nx>g8gYK)>p;_RsfaFqeH{(eX@JZ0D~!PShd)? z0<5%c!q{WH252zuTQUdTt~L+POY-Cfg2n39yq+H1?Pq`~t0BEuuM_ue;G zY`D@ZfPn))fR#Uhs5=xktrYpX8^6p;7v!3oCFa_wa|8X8iUW?InykR8kPl;awvDO8 zgY9NE*_?BIX?-{Etz`-pJnGCjc{|=PJ|uITtd`R?L~KXJuSPgHKc3L%W(R=GYD+`C zh38IIHPsbkeUsL(=U>FX#vQDf{e0^2%e`Y`OvCJ)B^CC@NRJhYc%Ipb2UFab-^GKG z=nl1RSUf3+hm&&|Za6p8S3t4+U&G?F-($#vVjasu`%0JMNm4Ig?lCr}kX7-1iC0&c zUFPzr$Gk1gHL4}EO}o|7!-0lB!}AXpz-GrA(R&J(%CrC$G?U--^|T}?#GMbtu*5)ox?CEp9nXLmi=fp2(sG#X`qy$ z_%$E6Oc--Z{EQ^&WEbt@UM+2J3m?LVpeM8JJ!pru$M(WWys>gXSWyp zm3S<5;rVJ;_o`3B{@KVL+Zm9r$B6&1n*Nup0B~^vEZW_-vR&`D5|qOS2#A#PX2X=I z&T>Z}L6XAK-4Sg$(M_)T6Mhyh&+l5Kftj#Z%u4u|qkWKVeFiB1)`y{)JUUf@Z|D`e zyMpX)q}li4*0jLIwLZ4ET*W{hSqDf7nvYSMa4^WKv6J4MPwmfIcZZ{RqCwe3JtF%Y|y7dVdTt2QPaeYhVT>D34|eJefX9zD-EU=!jc^vJ1IQ zWPo04D$yt6BnaZA!^kyk2n8qCmSvo-X^gIP;Y(-pZJ%B(^=OB!H~eh1KkgQyu=!`k z9DcP}a@)_L(O?-H#vR)qIGa~GCB~%^Qa3JRMLrTep}Xqer8p2M*TA**8q>QuK)DaV zfDoKSgfdAdt7wUL>G*dk>9Pa_=VJ|`(vcjG5q+~>>EHe)?WonGjK;Ha+xRyf4sX#7&#(n-gC3b^DUBeORpp^s|Q^MTtHunUirNb8hm&{12?)nK6a)Y<+Xj z(Cv(vJ}mf)?V~?8ER)ZrWzCGSqqX0j3!|cPk|)w09eeUFpR1LlVg;94ALZzP5jdkY@;u2%8fSJqnCIrUGRhEEJg*_BI zYgnOa(66OpW#=DEfX4Aya8L6l^zMcD-J_@8>^MO87cqskc^Hi2zDZ)-z=0NRc~T{q z1{HLVW5yx!xAji+2EbIQ4|C3m~SZ^ihb5uAs~0`)RW`RyD*U>7cHcFnD^x%X_uL zn$WJK^5dTeDJ%~nG$4Pg9S8{oSvevSq32c(h&c#tO)&YPDgpq!l>k5o`2P}~?t^@+ GdH)52`Lmb+ diff --git a/summary_results.json b/summary_results.json index 90eca68dc..ab7ee5109 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.51711, + "Performance_Result": 7.73824, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74958, + "Performance_Result": 1.70094, "Performance_Units": "Samples/s" }, { From 4d559f9752c7c20415726c690a746dac060f0a85 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 24 May 2025 10:35:25 +0000 Subject: [PATCH 641/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10937 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7e818bfd4..2c019edf7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-23 14:21:05,801 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-23 14:21:05,801 submission_checker1.py:3308 INFO] --- -[2025-05-23 14:21:05,801 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 10:35:20,168 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 10:35:20,168 submission_checker1.py:3308 INFO] --- +[2025-05-24 10:35:20,168 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-23 14:21:05,802 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 10:35:20,168 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-23 14:21:05,802 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 10:35:20,168 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-23 14:21:05,802 submission_checker1.py:3324 INFO] --- -[2025-05-23 14:21:05,802 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-23 14:21:05,802 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-23 14:21:05,802 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-23 14:21:05,802 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-23 14:21:05,802 submission_checker1.py:3345 INFO] --- -[2025-05-23 14:21:05,802 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-23 14:21:07,107 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 10:35:20,168 submission_checker1.py:3324 INFO] --- +[2025-05-24 10:35:20,168 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 10:35:20,168 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 10:35:20,168 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 10:35:20,168 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 10:35:20,168 submission_checker1.py:3345 INFO] --- +[2025-05-24 10:35:20,168 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 10:35:21,288 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419843 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420905 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.73824 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f038def04..8944e21bb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1268 +1269 diff --git a/summary.csv b/summary.csv index a3d7f9dc0..88ff88089 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419843,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420905,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.73824,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fa7f0dfd8..9232b65d5 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419843, + "Result":0.420905, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 69a1eb57812e12896786f9221ea332e303e14ca7..c095c279b741dea508619bbda5ca118fcdf57c18 100644 GIT binary patch delta 3878 zcmZ{nX*iT^*vDs#B@$zcMp-g5_LgZ#mWGHKvL#sx^VHDrj5Q?5ttceh3`1khS`Uda zS$k3sB_w7nWiPvsW~@VaNAHLC(|dooK3vyv{O{lYJkS4iA9uZDgJWgTKCxoo-nrld zAkgms5J(P0kx++GX#1pq?u^z@)j*oKmedy;*6n0N{+Lc_==uL}J9iOpoekGtyc)-; z4Ccbuf-qakb+7kd)wO7f$hCppVjexzT0K$X+pI*r9QEMks5NsDk-bhTOUKJrM&Y+FRB7B!W25`SIpVh25qbT zWlt{>-o$M-J2Hb={nV+N(rtEm{m<_K_QlCzXqb3`EE|6&(QB@OavnWbIvkT?qGX7U`RhDpG4HVU zxNYKlpBTbu*V3VoXCa|3g16`WZ{IvyyH@g@^X6&l3l*R#{FdsP{noLiFiIU6u|#OT zGw|zaclYP%>YTJ`3hRW>-?zjvGH3O!(ct4h%|}z?YUOX+9^LVqgv9d?ef%8c`{t^= zzTLI@Oz86Ka5VcTvt`RdO7TK`PI~BurGeec_m(Fm;?mF3!q$o7M##w=pC8w66pOjd zof2AdV9P}UHb=)JF$jskckU^1pfxASxdn4~a@@B$nfQIV?v%JN{(qGv0_PgxM}Ib* zdQ1!qF_n~XTF*9X>03;Vt84JbnOz;(zqfT-GLQ@TJSDCts~<245mU*nt1OkPdLQnQ z66d`15`MIQOH!p>Qar|K{SZfrt#jyH`BdW5*LMJ5Xj?1(z44r7c6lk>2A4aDx{3VyE{serS0W ze-^;VeaaAGlh$rmWfuLZj;9iB)16#HF)U+d43Q0a? zyOz@-uW~b76;ZB7dl1Z{=}LYZo1zVT)@`n zukpMlOj-J!zwthtJ)zgKnN&xf$t93`fjIRhL=WrQM8hv10%v-y0rN4Bb(qkF?{_;g zOfdW0RK&g(aOZC66~7LBIj!`cm+)0d#h38xf1U7E6SY^}9u{=juSlF{gTj*}JaX$# z@TrmG%u>zZkKjk`R}QF=gW?PJ-W*rpZ!9qnKJQV0^=T>Dre3g0@O0bQRgebeoh=2V zF&hR$1tU!N(n&p7z(EMX-frcv%A2s1oNN5faEZ2~@z{eW3$)nX5i)Xk_$tTytGwym z)6i$xZ4XltmAVS)&nzBIo=UFGoclO)+~nW4)?3kfXEIluj3U-X{O{_%xPDtk>9wc` zR8V`Y=fO;_fv84;X`PR(^rNjc;7r@bcYcN~T1afIpGjNAK8fgLYtOqTgNARI@L){+YUQ zW!&kyONWSN0N3wSl}@I6NA>o9nr9pF9H)^y^@#M5KzL>*+d%RH$HyoN2cSU6Hc)2f zJEI`^0z6p(dcjoO4Ke1T7fnq`8poaRbV*Ks=Z-s9>2mO3WitL?XnXbr-&eg--R0L4 z9cw;yw)9TFpCR=GTjiQ+?2PuDdf-2!S79b~>WvDsMYI5iisGr$GL?xwQnVr=vz~6# zTX!ZPG&-WF5JaPwTeJh;JRSRA`Qp&40H{;H-+3y%zq6&LOZ~`Dprdfs??l!Okuf0H zunp-I;Y5!=M4%9&tjS>Aw#LSfCBD}vD^Vm^_^h9Q1m2-0CRo~ouHF$|E?R3X`n|?` zxA5yFTYYAK)d$O=61QR9XzGnKh~w*OJD+!Wn3)D|%qXuHfuA_^$!H>3G=GoPYai{% z{{tE@1b3UL8Q_ZNyR^bIeY0{#e0!FNFG)jMf%ekv0yatmEsvM1bWcW;$fA`v)LWz! zG0M8N1-blpl(nh&$YP>6shw9+Nv!T(Nx#>`D-Y}D0CGTFG{$;HSr^civXITOt5E=n z23JJLWS=4m52#)IZWv2f(2n+{$YCmd`^%Rj5Lufzi$SB+h+~fSEB1a?TmOh!H}PBd znq7UqnQ}qNO_6MZ*8F-_3qnLq2#$V5AHpz%jC?R z&fhh66qFsp&U%PX<-#ydK2F>@J;1$HlRG6QlV%sM5wBUS$irMUMPWS!mUx(!(|w?H z*} zqryTYom`FKQrP`*1dtUq)tUadg-l5Mz4**&kN~c-pY)H*)SvZ90iyB|gr2k}(_k@A zBUfZ{l@=iZ=RQl*&Yx;sNNX4lP|v)uR8kgAioEY=Vg(qbzbdb27rrceK&g+TRVHoe z$cLc8E6`C`WQriPO({)AREY{!d*V`vNJB#668Y?C{7MJ-pp2Mc+Qsw-bOlXQHO+m- z4PIbkY+F-v#tXpnUV5e0AZE}{-O+28=yo6B1Fv8JJh?UTHYKXef+7{H^~4e4Oj}UY z6~TLxftu52@xmPO9OT}r9xbIc53{L!An@C&YS?MnVu^;yS$GrJQDfB@V}-_E_1@vN zsgu{HW15*5Eyz{!W7R5jJf&uZm##9U}UNc4A>y5+;)|cmRSBeJkX1%y=^RFYI4CI_e z^a?~<`4GRGi`sk@%J9A9seLw8I#!tEK&~}m!1bWZ9Q+C+lHtca(lmLiiQ#Y#3AC$J zq($=VvUql%$Q1GHqHuyA(PwywNa}x}=TECv!t+#T;)TaxS7}(UJyTv^MVM6 z5NT9sLjR3q;!g~#5({`)`fw4U%aeY;_?FDc(!5Lio9T>i#7Ip8eM+4g*+LJ zA1im+>`#r(fEl%$QWa#WA5urWzmxb%5+#h*zg(Y{rgx+GKE5-BnG3C4QcF1OKa zjn`by2}T0UuBPT~t);F_qZx$GRAmx$M|&`~IJ4O7X~e)on$x zJykjiyDzm|2~;Jj)Yc0#W)VPD_8=SuOo#^aYb$4!{04|mZmP{QQO5N9V8T*8m|=nn zlO%6x?0hvCPxg79qyp@(cH0Q3^=8z?+Z49Jy+y|tf?jCWzL>Xt-FPWhv0qVa9oqFF z;YM@dPR7C(Wz}M!cw)d>g?n>b`S#E6b4!Em2J(-OWl)}_Mn17ef+*OhPX zDqL-y&;h0s0$9f|$BkjOcJ)q~B~FtTmA| zx6UMml-YBRe`VMtO>bw-zHX{J)J6UPhk5RQG;GG<-yD+5o@|GP+Uty!>2d{TikUj? zr|wI35AS&9X?_c+0@QVA>Tb7}8GPlBrFT7xMq?9AI+a_9*sFyKiEsvh$;= z83VgdH<(Wg>rUEKL9J?m_f)ojjlfkXCrXAN>5Sh1U0B4LsTO@xVeQm~g7zgn6%f!5 z%f4~uO&COPN>yHPs{J`(R{Mf$*ZAz{oinh;ijIZ>d?NSe?^h9b6)#&CpUr**@zHpv ze(UZ1K#}%J#>)Kj&l;iFCSK%jt|DCUTd}5Z<b`84 z$uxkzHFeILD0}6PwyrTuwk%`pF~?hH?i66hMCW+a2vgaG=^kIYZ-0I~shx-Dy2pUic!I~{X=a~u3M;LmvZJ54lqRRLHO`E+Cmu_SxHfZ3N{+hWw77M*QO-H!;zWPG2B6 zBQ#hx@z$+@MOs}D4lAu{wXcaRShSn>aO>X*9k0%cH!aW)&I%l_l@;Oh4S$vu&+$t} zQojtJEhX?TSp4wzN-bi1a>1WZD`D+&w$&5PYW)25{DpMGY|%SvLC{+1X{Y$gC%@K0 zF3WmlJEVnA{I;KX+Vk?AXYJeDB;o?lUNOOn2^(za)SiI9A~Ll{R~$7%QJ88vn|!i} zU5NSE^f|i)fjnV-g=+Yg)vhhE8u-^T4n7he3QP8p8MS@Ym44HJ-vs@Kq-&0KB631E z^WHvbGE%XMZV^>6ct7}*Rcwq|`9@#s8zaf()AGyLuR*ny{pU<_>+a>owygknB*eFN z6l0==HXj)BR=%f1(`ABi3J*%G;R^SH2?~?q8;=ZIipL#iW*Hxc!t01R zvi@#2Z5<{GRHdDLwAKymd2eYCW{{)Xc@YvbwC9g6`Sk?he}7WAv#PfKjk&Pzv>Pn_ zZZ2o9R_n+q+Invn>YYP4`4<%zqTbEPCjOnh_`+5P>X=yEaxW))WO4!HSUCLnM;nur z3=NkW4!qUrRkD80QI`Ih?O7Hkmwjt~W>rDE75zDD;A0bGW`P5c1A22$iBje^)c_S631Cas@Mk~HY4D`EB)vw$nu1#;&5BKEupCI-)4>`z=6Ail?3Yl+#kgyVa z$4;$T)zko`N}DzdrONFI)yi4kQ>OMRMyJ1`_@eXgfppu9VLdng1Rup2<{K+WT#`se z@$ntS)fx7gO*Hu#o>t3?{uSX0Gly%q4us>2i$2g)9sq!7tUtPlCl>l3+q*BIdO78U zkwLd>G+HqBU>H%_wK&jYxPmQQoT+d*1tFJNaS;+%=y*XS#O?X=L!Q(#qheL!--E78 z`Qws9r&p!UB!qG#hJ?lohfSsUrzXmxdhTz%>?!s~SMWSccL6t79^1A{KS45F25cvhsdpjpO z>I<(lOmC<8zx-fY@VJ?06Veg2r^fFm)<$2HKkw8!99VXYXg_J~X4@&K?8Clwrb^>U zcxO#yv+{*E&TH1JT*c5-mOtXjlfx$tBZFScHNpXD5S0kJ^6{%&ort*_PE?@ySyTdL z8U0irZ)miNctR`h>~OAt%a5Righq_igtXnS7BlST50i$f{#CM{_HgxDpS;8H3`X?G z9|M->GMa1d+%V0L$!JU^x$^bFrS-UU!bdU8>%)Y8NWSOnGy1z#Fkw`p&m2Us9YVMR zD>S8r&kT6xZ)}8z?9K8ulLZggv6+-tKZDC25;&LV&iu2Ct#2Sop1FT=v{jgfQ*T5v zL0R8f4#WY?DqOIhrikuc#@P#%iuMXxCa}>IcL%FSW5`E7W7YfSWk7HJZ2mm*p2e7ahF-8XJd$Lk7HGrQOI$jV=Sc~)5sfZpeldrw=y44KW17Cn zyVrVCUiprdUz=h(l9pZ!7X*BeleM$TFe?f8D7SE*FnCd?bTjD+iX}(U*Hj+SMMKCE zsfR`}(ih74iNFAyjVk@hwY&9lvf#}56n{@?a#rj1eQ&x)_KfoAps7~>Qt8Z{(%AgO zH9IXccM>0hl#iUmeTpJpoco<5oNwE->B6;H`s~WT#mwzzx}Z$toDmK&BW$OjF8D;( z?324?Xj{>wJwm+yv`0rZ4GYMS-j~;ocawUIIVTcDxsfASW1|hVBwM`3sJ=q~Sz<}^;Q(oI{3RLOFc z<9cO3?5z}2%6LsMI2}(=@)eW_jG~WH{!4*74luy7d!E(MKzfV9^j8 z^%s1Vm{BbGXk$NopPQsciSJM?!ok^s@1)I;##3h@q3v|nx{B$<2@T-sV#8_&1OMhB z9>Ya(xTr2JN=5kz7j=UYr&@)H!IncB`@ipzR>JcDghFIq+YxsM3qCpRh2uH_4)m=^ zoA5iy@6r%f;-miNJF9jKw|@|-tzIP$@f$`urF&fB>;H(P|A@jqp0scq{>CEQ?>^Wb zH0kZjGI=q3O~3;?&|MxyAdvF%SgLqrNxwb34LjQUovHPDLnw+ZWsi#Qu{@j z4=sMnOT+08Hd@s6xb@CJ&P4fxh?yx+kq*&nL!#J4Bnn$7EJ|CfU^nVBJbujVB-NF4 z_oS+5Vc9}7$Jw-9Pt?!UJRCJh~_wu>w`=Tk3H=&NnHn@ zb*6(R{wXwmGg>wKfX3A*Fj@}Vs}!yTm?^UNyoca>mFA$U)b)ZU_WiWztJlhCW<&1o zwYMeC2yAv`7Ufn%tt%RR*0lsw{LS~6n*{uhyJ}C`6YTM05>A;|cmcYUlO#s7)!%cG zcBaN?Hx}deTna#0$aSMgHdkRbvhwh>sA5YzJ(gW?ll##tnzb{(Pnz3D0bi7Z>OIr9 zv6`{)h7G{@vaJP3K|X@&4u6XN>l;y$=bSec#9a+OnOd_WFwj9;4$EnCw?xMHIh=sz z)sWoX-1m55g?qB4=G8aP!R|MOATNkh)p!g{CQQP~OOLB<4xG%@)&z0oY9oG^ygF70 zt~vrDQ2B_Fd;sq$acN$ifamixr8;k#HZOw^vW(8VtxOxHH7IEpJETO=WC#5GM{{Rr zc=Ufvy8~t^muX?E&t>LOh!G~#L#G)Zdp|5<5QrQ9*#yHgQJ;B|B|gc9M;KUH-D4{d#tvq^$jZp7MvgcL^^J zgA1M8Pl}@zf($Q46Ayu--9fnGArJ)#iH_$n&J>2p2y<7$2%8n^M2Vg8l#BQ@`DpS&PLLO$joMWZK* zeN7<-Y5|KNNx*;1EuL!W`anS!C5%-6w7D^Yu%G6|~zxY@u5AOUAnP5Q( diff --git a/summary_results.json b/summary_results.json index ab7ee5109..d70ee39eb 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419843, + "Performance_Result": 0.420905, "Performance_Units": "Samples/s" }, { From 0b7a24d44b33f3ec49992e651d2825962810e4d4 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 24 May 2025 11:47:32 +0000 Subject: [PATCH 642/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10938 -> 10940 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 2c019edf7..a54a0f554 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 10:35:20,168 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 10:35:20,168 submission_checker1.py:3308 INFO] --- -[2025-05-24 10:35:20,168 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 11:47:27,816 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 11:47:27,816 submission_checker1.py:3308 INFO] --- +[2025-05-24 11:47:27,816 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 10:35:20,168 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 11:47:27,816 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 10:35:20,168 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 11:47:27,816 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 10:35:20,168 submission_checker1.py:3324 INFO] --- -[2025-05-24 10:35:20,168 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 10:35:20,168 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 10:35:20,168 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 10:35:20,168 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 10:35:20,168 submission_checker1.py:3345 INFO] --- -[2025-05-24 10:35:20,168 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 10:35:21,288 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 11:47:27,816 submission_checker1.py:3324 INFO] --- +[2025-05-24 11:47:27,816 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 11:47:27,816 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 11:47:27,816 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 11:47:27,816 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 11:47:27,816 submission_checker1.py:3345 INFO] --- +[2025-05-24 11:47:27,816 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 11:47:28,840 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.87223 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71802 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 8944e21bb..e4a7c9900 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1269 +1270 diff --git a/summary.csv b/summary.csv index 88ff88089..253874ec7 100644 --- a/summary.csv +++ b/summary.csv @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.87223,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71802,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9232b65d5..036f46c0d 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.87223, + "Result":9.71802, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index c095c279b741dea508619bbda5ca118fcdf57c18..6b9ce69e5c854f00a74c9492f092ef2092e6ec5a 100644 GIT binary patch delta 3835 zcmYk9c{r4P`^IOiDW0)C3NaHi6QUvyQW|B*V0e^mNFmFpp%pF6ZK;sVaF05zTf`uG@dnlVxj=LE>-#OFs5uJ>2s+gCg*(j)`3{4xIZ^Tgx< zy@$Ko$thtKW&Zo;%{$-)51F*mQth`U(^S8vAXqdIrxjM0)p|?1Dn^{=uW#L*rk#ny zG%*?dlnLDY7}4lUAg{aK1)DYc?MTqA)rtZCQag&%MPScq2Kcc%K2#Gvx=dY9HLM+F z(ogp#guP2?sL#}WWU(=hta^m0qBI!3^$+GhtMrz(13RnZCygqS%lS{1FmS_Z-_iD4 zPk_O1Du#iLwVp{AzWZJ*v}wDwy0(fgne=df)sKJ8?kF`TzNCy_+Fy-%ZA|5@|E)2i3~))?NgX*)CO5w@dtJoS`{V}jQs>{57)m|o%(mF*ciZi z2qklEE6o!dm!}NIl$B2))d~Ws*t*UzYqgOd=Mw~)@T&)b5ARwc)Zth6%+}pE zsgKd=T8qTi-2qTCtii{7?2~UHD$AP_7%^`+(q1}OB*zEV`hJDHIKr0(C9IHNeC%Oy zW9jqB3wh?>Mm)b|(Ym*)mlyRyf%ao-*Mjz1vFBTwjH=wpB&iYqyT+W&G+)28pLw^7 z_qbF2vWSUkJ(*MbR&7(>E7y!|=W6Y)hfNgc2bafWcIxJ(F_^s*UniMl85DmX+W(5} z5%IqCH{$avH6HJe$M_XB$Bk<@QO=5|b}S;&;~NZ{7^$@y)71UA`P{4}AcoXa-O0-S z*`wrh)@|dB_j%{3+bFMYg6zqCqfxZ&>W-V%6 zv+3lLr0+my4c=mG-CF68*ju1GqI?sE64Wq~0r>;$oMs z`&%<7S(F?H)7hx%BAs59=^!gg7SvDIR6*3dS$8B(oG+-wTsBb~0Asvm)VZ>-Cg%vK z#hF8l2UWTqA;J;dr=%VH9dpOE;_hTyo|wNC_mmR4DciBDzZerki^2h&wSGezU4HGx z$sb)0MzB!h=z%!Ue$G6}xlcL^ zTtcw)7h47f%K%Y1*Hgw1dpOO{^G|cf@~Bq`-LC8B$~Ck-Fuy~46mRH9+v_*0f^o?5 z_h)r5eS52wT3nr6B3eRHV_(ZGx~5O>KEK8nrB!ck+I*dwrl{A{JsL{%0K2E5HX!sc zPV7USgD|aLY&4HvKg^6AdGG5|| zpsF<93^p(w4*(7p+H5Nd!E>p{-UtEHC4;@gDT{@Hp&X|66ict) zK=)oBbA#%2)?^av)~IwKAT{_}-{#{gD${y`4W+8V+g;Uw6>?{!wPw2YnQx@R0 z<1Kyy-tg@#r8p>`)mAzw{}!J+E@rEkg4(2<(ygNl!8prp!}}DmKv86EtzI~~=oSP| zYpA_rs*gugh3Mjfq1c%rVF1&t4WQkO@AFMN)%@qwjSEunlDy^bAK2UuUDWO9zZ8-LqEY zCNoi#L#D%s)rP?3@8A1)jOECjfG%(G<2?3cV5IS1QGD2KLXP>etxdh6+lP$?Y zx~nIz`3lp$CQT%GCM6LC4ux?!(`zSVNf|nn9E-F%x`H%Eq)kXo@dN0COx9LsAr(=8 z-4D6}!IUJwQlZBDt6T|*AWHD{$FInxnGvYToQvAK_+H@da^GFV72RVw2My2dM7IMK zlRprt3=hYawhMMXI(d25BRkE7ElZB`wCp-vnc#uaj$5LZx>?cr7}v)h zu}ZNArr`p$4KuwMAJMgF9|&OW+X+cVuSLfTxulFS1PkO6xo(y#vHj)_i}1=H$dNpu zTW7TYbgs8ne6!nMaiyO<-FEP+tdy*C%66OPqrDS&JHH9q^cXpc)5ed$rXYp-w=qhU zaH0eU@tq|oJ}IK{OSu@;z5C||B6;YX_Zy#BM{KZr2e}_e9$-a+yS;*P?^!RLKKF7X z!3I$R2TvY6K*VSZp93YY`m@O{;vaHwtF4F)Q)^4Hlg1Nlk6d7tfL9 z2&Kv9PR1OC*YMYFnXh*`{a_~XC84vc?SIKG(PpO@bd~HP)`CjW{7bNovspFE5y=Ve z23WaA0)G|gNhQIY$1`T4zNKqnoXcOVk5w>t%WnoY9ZW@ip<3LnqkY*{6Ku~tl2d{} zt;d`DSEa#akldpI$>prH4zCLm%++)xXl#?rFji3nSo0M`*j>Alq&}Vg3iG)9#isWf zGpCCE*w6YiJbq{F{w6qRpX>lw?%?NVY!(XGQ>Wt5p1oOUAC9N3WYw$-9NNnDh|EVP zeQhAruebh_iW!P-3R?t<7l5RRiL-Rw_V%DL_x;p`I8_m1pq zy(p!G_!F`F|3L`6Erf4-8-}tZ3vqgL61f3w;V$CgOq?+Lq;O<%f~1h@F^5GLdcOf* zMW~Ni6{67c_HX`W!UeiY`#~$fluz+LTK=>71|OyHA5EQY%?^R)%lr(1=D#>S(;SYF zmx=%ryNCkP(mkLoo|x*fq(w_bFG9VLmbS+bD-`^6p0}#yDIuHSyc(ftE@8O>a{*y$ ziL%myhPav{@3}Y?hP#LrdXuI0dgGO(l|{?+{Y?#sfN6^?+n)=oGKyxa>O@76?qjwt z&b$*#qFhII($*QST~3qht_wghJ?%meCypJQD}vL<9T$M?5jz`#Jr(0E3Kc*B$LxZD zXyKT6A&q~#q9|&{jsqU zpH+{)sf0)4E=v17zU=-kz#Ui`$T40o+$Fl*w z_-IT4y5w2>);SBz=HcZUlXVx*;mw5`W?LObf!@~DRNp+eco3*z23^;m8NyC%)HmHq z)J_v*;FGjbjZ5>>@ZTN6>VKa3dBw;@O z9y?U@mqO29BB!(wQ?wsQIKZ>5JuGehC)9i_Yy=tboDDh%b^J|F&=6u3{q(;igYi~w z+cXJ0ACObvLYYZB5@K=CNt}W7*CWq_oy#{6gaIDUpD zVs~^=o)OM^8bdU_Zm_S38ek0Z+oP0_QCUj>zZP{z(SP}x<(@Oj`F&d0@O75aJD9Ca zLumjlxQk%6DG3CgA1rsg_}n(vEBsaDk&oK{z6o#~=Be#1c>d+e$Jh{|-niVRz0IRY(wG| zrbrvj75bkGDyZoMm8Lms9*3sTk~B|3qiJ25#?Td-j22pXJ6RMI0{K@60+E6I_X&^a VrJd0_0X;}d)KV84-oO3l{{SR7LMH$K delta 3834 zcmZvfc{o)4`^RUDJy{b`)-m>$X-Jlah#9*iOJSZG8lJI+Bsq#gvdu6w)~xlA7!$&i zdMF`jhGeppT}U(5A$*VL_s8%1$M^j4xvq20b-mB)ec$){bFNQ|OOs1YsGx9JK>u9W zehB1uFgaTAB2WNd3q@}!H@57%tZ&sCU0^SFgQ0S$qi&+y-@V5YCEwo0UYKs=?u>N9 zT3Hz84so!sy>7OVt7c7Y&HKGH>)f)T`ZA@Mo)LcBCN;Og&k#*)AioJfEvi3@ROq2y zvSeTR>QEOXb7~R)Dsi*Tg%QRapiI?QY_qFg{`?*cIF+SEqU6MiW!SjWDZX<}Vlr+A-I@MmN-b#*@~((rk9c7skf4e$UZda<<4gmbTe&;>fix=TFJ^?aWM+j}W8f z#QK1tmWg+Zkp>Sr`$mW&6s(*B;5}BVf27JLtNE!5mQFF?Oi$){v-8rI6cb6!2pc&6GKbwhp4Y3t}x zB)O5KwuEoH#rpO5)2ENqb@`doWae>!KW~U;139w>SE%supO&K;i4F2M9aMG#Ct*pv zL+?L^2E4i~Z|HdCWsdlAOB9OzlhM9qC8cPal%Ex`VQu92{H^r~iNvfk)W~(hxCvr1 z-~Y$eYh|LIb0-CsoY}H5_A28sXf=tDH(u$9kTqAyxdlrvQsTEc=_J7dFLGie@88N2 zA;4TST;*r$$%llHaC1os*Y!M$_RotMiH%J`Sc}Ue`}Q84k__R(K2C`l$QTAs!bDXI z8fz+KYu`qBrzg5CJ%_6dY)PtiNs7h0t{>t^vGop}t(r=C+;R&T+7=B*$e%@;CAGiQ z`SC}_@bytr%!~JzjXy+(aW;2lTD|TE0a4e`)v4y#2!*CY6%E~K&&-Si9MoB@eG66O z+j?mSOXk_c1G+S+OJVtz*rNjlt=k&FXKcv^nJ11cVWxec%=VWG2Eyf zzS!e8jT>5C#hpRRe#jPJ6W4Cm=9RHRe&EPW3hi}_jdJew>#}9rY`EV}Sib_0v45+< zkdk@iQVG$&a@Tq~{6#^wha%GBNB|$D?d7t;4TVs31!C`s@npZeE4+=W5}ikqFh4OC z#+zO@`4=!Ri`GD|$y3&W=dQhtVow;fZ>BbqW(x46{zT1IwLa#RiKbuvc+T`%6Z(B2 z^Dw>_H{f|>7;o{hwUm7&_!e-x;!=U?GoTDgEeWmgArYM zwgE5tr9M?%O>wFJ(9_;O{dR`f7iLpnuC+7Tck*7)j6t=9)X7(>jCPS?EHW09r{*XV z{H3U+0v3Ir%x~P9fKh2`r6mw5t;(wFn~%%DivTQY6@c^@4!BKa4fM3v_i7#)3ULw2 z{hh+RCOig&0mhvO-)L7_(jhzUTCbzb_BCLSBg_%0*K=?c+)hy zqpCz2>_ooT`|XwtUT`pE4Aj1}9xC@7){moHJFRwXU1R6t4v3j!^vZ(VviK8=IuS=8 zi4^Uj`U+By{4Y)b1Pva7drs7|xMD@_9dgV_5B#%Vx&k7`=(jjx?dtbih1fN*D zVwzGeqAg)HmPmyws!8XaBnYuI&VM&fpeg9a1(0RYH30)v%h77Nn^>!_CctX+Q5UBb zr$C#nzeMa>dF}cwEi?4SIvcum(UbP$w1n*2L)X!~+}m_%-G;knN=`%;C@_@Aqj~W$Tjf%w}()qevRI3bQ$K4NdxjFZxtm$R?A$M@T&cGz{kCP`wtFgsy_UW=?Jse#I22!Q+ZLK=bGM6^!QP zj3Yej*5lk_u4nvBQM)L?3U}HHxfB6(2wQt4e$JOX%1{g`TQf)A>5swizpN_St`P|a zXT7X*^N%BtY{Z;Z+zL!r`4F$CyT*Ji()g|9NkRKsJsWgd2-gnJ@;K-|2fsv*p$Br0 zv`*e=r8}QRbg5Qn#(;L2pxsB(rJ!9Tmj6BKG)Sn_SNfaaxJL-Yi6hOp%07|lK0uFv zC{0frJ(lpnBs``%jKS^nP7jAwOgfnRHSpST5+X}iKt52DpHZa*^6dq?yps#^Q4v#C zah!ko?Ed8&MkUGc=KA(@^?oVl!d>bq=PrIlC4Wq-ztkj^IUCtz!DT#2jXcj)1`hdP zXA}x%3`!RLNs`umpKzCL-WB`>+#{;p>)6Y;KMgxi%~;E1 z6-Iwqx%-cICCjI-oy#DkMKZa{pxRtQe0q;C6I9E!W!?GWYsVCm=lcfQmdjHFbB!v~ z+UH|(@%|R#L>z9c3UJ@#Mi~@_mX0{D&YYL|Tu2#zp_WH`PefmOxDe-_%#nZUze`H^kg~5`uf)m*829%7BCt^on6Xr0kSEG@39$!(a^_Qp7~C< zlM|nZd_wt|>Jv;qNs`Q7UrqQ+p^`L&Xg0?U(bqrnY}TvX(w(oQUHy7aNMi^%p=*`B zee-B@JI}wGe0!R5pk|gd3e?oX@;^bjn^wV!n0RPFP;`-`j#2~*)TAkBn(I`$nSwt` zBZ`A0>rZ!YH61e^jx`!lrUBL5h1zh~s^@uBz=rHgj2G_~J}BM|p?r9LC$0($yHWy9 z^#1q!`vpCKv-);iCz%fJ`T#OtDIZ@oABtm#ykUQ zO1lfiFTI4)&5)6jq%EzTL8I|B|EH;{z`i=q4IsF|kKUMMU(yNp6B%C!eWu;;Y~G=z z`9gx?fTHNSc<;O9Yi%Jr*$Z3bRV!B61j|m9dwpB^=FjhQOJBQ;z$1d!i?NH=LHzuG( zhA$Sz1kM)9b)*pI))~a`N+-^-LArhF^mguSOKanyUeY_boX@@&9$s~|Zos$EAM>b! z=dVJkDQO|pcYRw+fh9Y~JtoI|&`Xvfd7}W?o3tn(Q}COwPW<4YC~w>3aar;LToLH; zZ#ym?zjRx;q*<5s)JCEn7SQ?h&RJ)VP$?!t;zdk)zhTUlmi;*Vn{!&_qwR6!-7v6^fujenO2z_uj#mF495}$W9q3J(hZsN0zn_$g3azas9?HHjq$jo@q~S?xJ?7_mckCI=esPxZyS+wu=}G_0n&u%$>#v zh+yhtK0d?9ylbalMZyfG)aCi7x}M@^b!|O*$7e@xotA5^?rvh?Qn=TDzlgrAc+sxx zOx^>Szt$Vg8?Wz%*xh@c9N9?W);v@n#fUrH?pGhRPd>b53O@lcd1s|$>trb{g#2D<)u0fFBM$^30r~$c_Rb*rg4P+R4w Date: Sat, 24 May 2025 13:32:19 +0000 Subject: [PATCH 643/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10940 -> 10941 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a54a0f554..b8c6750a9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 11:47:27,816 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 11:47:27,816 submission_checker1.py:3308 INFO] --- -[2025-05-24 11:47:27,816 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 13:32:13,822 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 13:32:13,822 submission_checker1.py:3308 INFO] --- +[2025-05-24 13:32:13,822 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 11:47:27,816 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 13:32:13,822 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 11:47:27,816 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 13:32:13,822 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 11:47:27,816 submission_checker1.py:3324 INFO] --- -[2025-05-24 11:47:27,816 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 11:47:27,816 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 11:47:27,816 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 11:47:27,816 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 11:47:27,816 submission_checker1.py:3345 INFO] --- -[2025-05-24 11:47:27,816 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 11:47:28,840 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 13:32:13,822 submission_checker1.py:3324 INFO] --- +[2025-05-24 13:32:13,822 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 13:32:13,822 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 13:32:13,822 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 13:32:13,822 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 13:32:13,822 submission_checker1.py:3345 INFO] --- +[2025-05-24 13:32:13,822 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 13:32:15,070 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.71802 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.80695 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index e4a7c9900..75d94d2f0 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1270 +1271 diff --git a/summary.csv b/summary.csv index 253874ec7..8b9fd2592 100644 --- a/summary.csv +++ b/summary.csv @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.71802,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.80695,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 036f46c0d..328bc6f66 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.71802, + "Result":9.80695, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 6b9ce69e5c854f00a74c9492f092ef2092e6ec5a..5e1e339c104ecde4c00d7dbd8dd7a0d9ca1f9116 100644 GIT binary patch delta 2369 zcmYM0dpy(oAICRiqG3r2hmCCRsCBnB;D^Eok|b$sAi2WK{UUYa&gN}Htq2JT_XuPa1(iVwmK%$n%Af>;-6?f$dw^<( zgL}UVL=SP}c1KQ(BxQZlfWzkt+whCeXUL70 z^9`LVgYJfVT+Ls2gw?mH;$8^A0xX`999^`@6UBoPa{DtA`G-&gFbQ4p6PL?)CcsdI z0fqD5bexkRYJ*>E9@SE$gY(xCC+#fhi~V04BK*ev8(gLcp1g%o_Spw_QI-(^ z?-jpPS0e=SQH!+WeQz~AR*+zhMkp#i4R^6N)AgP`Sk3dqbnD7 zbI(s|4&6DdDu0t^dGD>#Ga_^^CJ@EDIzs2@wC*c-=yYsSQ0c9e$zuhFYY4sHau;ZYo^2AA3pTwp zp_a)*Ny8_TTgqZ_4~nV`B`Up09T)XN#DL_!qg~WtJTszGDh{(hpvR@L>lF~2LZUg* zHY+r8J2MI{pvGU}h4K+-fj*5dtA?j*c{-7g zCA3vqV_8I@T9~TGt0<^fXsio~!(O&oQIXX^6ug=mX%I`70E70-GLmL!>?;%V#4Clr zwu=xU)#3#AKTJ($w*8qL(}jAFph=M|x4DeCP8+-Sq8Vi>P$oPO0B;Zjv6^bcDH85h ztHM@OZ3x-VSrq=Ue38dOWLkM}7Xr-EK8O=$22;DnjE$PDcxpvsHwb@%q=9d=SBa9q!9Xpki3OU3Vp$rs8vTQ0cC9YK5-Z5jc z|0&x2&jDw(XWO`20T7o!p_XwHdzX6ekDt|l$o+l)?eM+Wj;&SP3{Z9jD0|^15ok^v z3H}8Ct(O>v?h~!hLNf#y+bkr`TV)-W#)<=!x`vWnPF^o^p6*V~s7mQo$zxclu=2P0 zzVtfn;5oZ=(V8cz+}JUD|zLRd2=Bv@n&mSsZC1CmY$N|IDZp!ncLd_*~Qg zR6xgN;J0{zuI#)-%bfM;IHj~b{HBT`ot^)wXEc0#B5?Dyk_*bunU7!G9f|{_t*TyH zewyN6-20qtR+qj4#1#b%T~x*gD~v33UV_&)FGkVy-s8(0*RR#YGImE+> zBw6%V_`*ck=I14eFJ!VGzRV@&s)S_gSrI652Yb>;&$?(+|5~~W>tTufeiT8!nvMhT zH^2!MbSTzMfYtm{Cm5b3q8*8~! z_C1#4j6(?4%szr1G0{Y?{7zPtffiDhoydP#<3q{k$**v-W8cUY<*TYj-!IXol(;vt zJoC`n+DCPPgpfvGn55TkRs{D2XR|`;Sp#78fT1Rk_U(Bk3pPhiLBKjA5@=vJ*xrwh z=ls{Il0AZL^obpjEk4s)slPa%q70tFSC^y3AAu`~M25QqeO+sC;wiukk62M-?bz41^2B?jZ{tu+_uwY|(9 z>aYskKj&>5rey=_yZ)DKWT(CiwtT4iAf8U9WR0fP#!d|MsqT^PY`4bAJ*nTDpY`y5qb4&SNxf zDUTTtPL%Z56@UKz_Uugia}iHEThjTi9R>HQ&+$TxmSmiKS7P3NCGq`YZ(?dCd2ZXF zEad9n>Y{f|59`gk*=@GwaqEANJ|eA&0zQvTPT_CeYfk20$!GA}`j$VP6l%6dzcqdl z=k;kHqj(J9#IUFRx7=FMks?h84Yl8#KL+7^utftWTDBwo7S;N~2QFUhuw z>~eAW<-SO#ik=PML_a@^*_lmWe=pLa`Uo>mE8yG5K|g;Pv&gJDexB8j`J*(mzArKi z6*v+L_zbxvK_=+u@-b;0cgpK^+v070+sT&d(K8Y~Anq+ancCx)&)A5VCM$7@)F3N) zockDM@HE<=k&G(7@#|qF=J1Xv9}NOHd1UO;f#ao1;X8<_a%J;k^uOo&q{Kti9=f3=tk-20fb;WjQ{qL*8QQIGcZs1THJfTG#o`W~^7tWx=chDuyMn|m0+M{Bi g5J;#N1hN70|HZsG!ijL)1Kq~qIBtc%vRga(2l1L+)c^nh delta 2351 zcmXArdpy(oAIEn=R?~!|Gmflm3!yk`BpOC-(>N6}myAa9Yt*T5PHNxLWjQt0cbCo~ zQP{f3u}P=UMJjEQI}M>!jvP6KlgrHS+c}>#eTZ zr&nc|78;pI4_sr<+gI6|Ilz(O1fIlyITLt(DJ?2QvyAe~gT}?}ucJqD*SFyhJj}FU zKF8(bkz5d6oP*6yYCK6Cg}0HWT3-#!2oyZjsG~sPGIrjhpG@z$lag*ga=TSe8GT9n zGWw=A_pa+B2bWu&%FnW-k==*qL5Nj=NAf^)r6$aV-Qlj%Y`0~1v{ExNhXG`PnowF| z1TtCOdlT=ZB$)oe9I=1;epl|Tv~pGZ?NO^gn96r1A=rF^AveJNu?&IH%(M@!#s*ep zH#B$%h*dwqn3AUrNBu}lVje=aBr3prX)%h7acW4B91-;;6$Z3?2{;^QW?y|{=Ff*} zZKrqj@Ju9zcLn!`p%HQbMDnZ{t!C9v#$(wMn(H%#W;)&%S3rEVjgB2bM+K07k#1_a zT)wS|-G~2(;T3E=>l>$0@Y~O)hl+}V!^8N}zV`dh)QD~!v@s#aym{#W zrU{n+{9cwPnfVbj4sRumQW)Az`&uo)9S=WcU0-!0J67`C9>{3U_buA?%<5@N5B09{ zV~ea;26*T8d}8pMsm6q`D`9UK3qlTJbWx1n$*PAXV2eR4*{_U_xZ(?w5zS~>ON?As z+Zyw$;o766Xca!zK|Y^^lqHP%qMCfGvns+W&z6nP`{(eAU31H}7Cx3)n`?~P5o@py z!9={=1#nQ|u>>IWgN)4b<5|ttu2C$nvBBfGC>`nD0n0)vD^IT3>#|Ij0IsY_TE?Aq z-(2SA8MBnwQ9JqpS5U+bZ*7kYO?16+<4|^25Rz8y)8u?=r_3*hn6u6d2ck;hfgYxb z#>g{?Fh){07geT#C6vIVL^$mP3=&=lD5Oc3%Nw=k+NB6_1q;?2|9N7=zWppV!R72f zdAFk(G94xA0z1bhXWh6xgj=9`oJlmE7be1jle%DX;=E9Vh|GUmG%mtKX6va+)Yn@( zWYko1+L9L!U)<^PrAyyd`#yAS?SX#vc)Qzc3p0z|`Cnt26mrB(z|!|nP8aaQbQk6u z%owba^fSe#&Yp|X;QmxWkC$dOzNj(Di|zSq0Vf}f;ryPM6CRiz+sW^jzfxs`D-+U` zCxfT9#avquZ^u>JgZ^&oxD*#;42V9Mtma29d00lDYr`%01=CFYcUx;P0PD6qD8gG~ebfGCG++65Fsqj1P2gPddBYfmVs}~)qd}JcTs#&h;Rv=% zCL6qA9!zbX$vbG!Jxo_yO6_-^Uc&k(VQEa6DzL={GVCBhPE9iB&m!$m>{QK6; zYpftKeWeYM0<8)(c$MH469Uvj4GuDPmzTdP=UWk_vy0}2Q)^0sC^mc00i zgVyIHFG39AUUIa{>&W{-{!2%gQZw5&q=zh2I)$?w$5bBuo;g^f`6S4~o`zYe!}z7^ zlnXUi4V-yJ***1zeZkM|MN4zmHrssMwc#JpBs3?YA@ zd-#=U5D9irO)TSP4kKl3xT?uy)n@554V%Dvf^c?W52}D;;IlnVbI$Q73-Ru`ETYp z#~(G;F-x8>kJvDY^u6Z$F6@uJn-UBB^Qz2SUupRDj_@b)9v|D>$p^`UMdg%EH@lWp z8%@zzUejSrD9{rzpkp7odod6YO*Q#=r*178(fLZ(-`0K0Uv-n>_`7@Dx#1^X3+h$A zi>3G9l>#0jA*F&?eNnU+vz21lFjMcVj~_Bj9i}dJdZluLALJ$7h!%lBy&~-pX<{&N zWZ_BkPjV8m%`y6F*XMlElev;F5pl{-$EIZ+ykqA&NS6kH4BKg8+Zu~SxA~tve;J-K z6zD-7<U0pK@6Zh9Nq?k%=7=L zxyA(VZ0d~<73|X+QeJoc581u!;0~D6wqKpv1~RD|(s>xL2JaPSp3%$rKuzAy103fS z8;5+D6Dp(7P19CI0@ZJ4xPFJotDEyuykYykwGkKBHL$?fRXrzUACsuc7_YLww;?@m zmU!L8glvC$J4KS-&7vC6Kn()1sLzD@E>)GEwa{1x+l<=XZHQ!ZSesSPYP6lFT z*!@q>fzv^ATZVR+`pw-b`Rtx~bf5yiL#CWv6SDHhErUte-zef^vuQRzcL65a_-7At z9Ew|xFQBeds@4;=i2r^%5`$wj+*}gn Date: Sat, 24 May 2025 16:52:48 +0000 Subject: [PATCH 644/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10941 -> 10938 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b8c6750a9..2dacd663f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 13:32:13,822 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 13:32:13,822 submission_checker1.py:3308 INFO] --- -[2025-05-24 13:32:13,822 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 16:52:42,767 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 16:52:42,767 submission_checker1.py:3308 INFO] --- +[2025-05-24 16:52:42,768 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 13:32:13,822 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 16:52:42,768 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 13:32:13,822 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 16:52:42,768 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 13:32:13,822 submission_checker1.py:3324 INFO] --- -[2025-05-24 13:32:13,822 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 13:32:13,822 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 13:32:13,822 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 13:32:13,822 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 13:32:13,822 submission_checker1.py:3345 INFO] --- -[2025-05-24 13:32:13,822 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 13:32:15,070 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 16:52:42,768 submission_checker1.py:3324 INFO] --- +[2025-05-24 16:52:42,768 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 16:52:42,768 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 16:52:42,768 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 16:52:42,768 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 16:52:42,768 submission_checker1.py:3345 INFO] --- +[2025-05-24 16:52:42,768 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 16:52:43,743 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34096 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70094 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73249 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.2816 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420905 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.73824 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.06579 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1165 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 75d94d2f0..88a90959d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1271 +1272 diff --git a/summary.csv b/summary.csv index 8b9fd2592..aeba29562 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34096,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70094,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73249,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.2816,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420905,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.73824,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.06579,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1165,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 328bc6f66..0d1a7d886 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70094, + "Result":1.73249, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.73824, + "Result":8.06579, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 5e1e339c104ecde4c00d7dbd8dd7a0d9ca1f9116..ce83e4626478418bd81d671ecc7864474789654e 100644 GIT binary patch delta 3733 zcmXY!c{tSF`^RT&k*rBE)-lFT#-vjAb&QE9VM1X<7!)y&WrXaa8S5a~^H^dqNm+WD zVr(H~rgk8@q0*LmOf=Y7Ag>oneOxLx&#omIfQKG2UCz^;yO(xhCA z{T6&Lv;GO8J7`K$!m~I`mR)9VCl%nl_T%k9oJ^wIXhgrm@&j#_^nj-we`eFa5`BLw zzf>`zZPu6CS=aEHAJFc>8?R!U@H^`gZt8;W-Ws!sMEBqybbb|5b2fct)(Up)>B$~3|co`imL==|gBT_sOS7#$&h*e<+G zQXkY5=`P!RaVEMTln$O#ky(S=2f&hA8{t_;lTYAixcPIof<;QOW+LwTb*z?EbV>8* zjhc{0wZaFVWRGfPgU8y^*HsZmZOu5fD_A>RE!S{*nfu*NP(!_7jgG{R9l)wIu#{B$2CcTNgTKm(*BTbDe}&K=jVf4*Q4;c#F+XCbA-X&s`C(V z*kP(c_E;N|pYM7jnD65^4}QL+SC=7R<&bCt>R~_>pHd~(06aPWlJQZ4JwM-_9i`Zq zvLIc?(UB2Y4BvvI17~zkRR*`*r=CB*Eg(g)S>fd8TVPqIMQrzx%qq z$ce@p%`L4vm*tv`5Pi2-AlDXL9J?m>R{w#Ee%V;03Obgofg2qZwQm^3(^qTin;xUB zujRC^&1Zb?Hn&>ZnDw*LPC7p<>DoJkrwa%R; z755ZqSbj=?ITl>Xm$iLzW12r7JqT}6?ksoCg3i0>8BZw)DVe$={76vW<)+1^h}6@- zLlY9bTWc02uR2l(1O7edlJ)xzvt7q&1>SwETJ@#;;lAsGD!5FG zwZVy8x!}ezf9Z@24iOG0Zj51Bh!BA20x;|RiG9a)^CE(Wy*MxWIblULTNi0yye6GN zF7If+Hwcj&Qs?9xGQP+eHcEQ;c zitE%ozhsT;@>i=~szDAuX(Vqb;yX4o{|f-yJCb8FTZo=cY{L+tdqkHo)Vh`m**$9r zqA8ITC8jY75+6!Y;uDol_G~9BD}3@gt6%RmaxAOZSJgBcOXJjz zYi>O@`-PstO7ueT-Yp0Gs7?tIg!x2ktWbEdvmS9X%uV>8Sx@g^9V?1~pMZ{4nm7=uTf}BPEjT25)lf}uz80#_b2+P~D=|zxLq@<_pfo_1SrJg({%lrf zEl{0H(~ojlt4ayXouOfJln|? zSpog)a;wQ^5=31dQD8pNkq2~Esm!dx>e}<9trqBM@XM576KD!Ckw;9;slX(2gJF!? zGZfV1Q47VkW@sbTb6!T&eW!o--7UjuW1N!cYZyF&AAa+~TH&TXERQ(OBaY`2Vigg8 zJx`V&x|NqHmu3-b87s3TPT(jy)`yv3 zSeaumVWzG_fK(_)%r@W_2`<4Z##?LPDX{yUO?H6*qZ0fQZI-jRZp9THLlk%V*%~Z= zDW^T*jpS~%M2jF3HDYCy@DdChs{t%by2C`#*2oC~Gd+6Qui!-v8-YD&lBA`$g5S|uu z!^*VAVgQ3-$FKrT1{DEI7xw}(YU&iL_|pszj7OD<+84zukW~hWRN^)d19v1TL?wqG zeq!jac?duf4?h_J6;bP7x4)cH)IG}k#=_VoF+SlK`%fH{sxZ7<+QGx4S^JGv+A`?P z;u(*0yiY7!tW4KQQ}^^?Z5Y;#XiONAuqK@bU5oYy*pNPmbB*)V8o2Oo4&{|2+gqF; z*hjQhdTfXCc6ck32gy`<@O$Nl+72C0%B!ZH*({~_#2&@S)SUpg{{ol(0)fh?0iFdw zJshDeTqy1q+a@vFu4kK`ivJ~-mZ9qtS7E!4OX%r~)FF$} zdeC;MmnoqPr}gpBBr?wxil+f#2(~u(Rqb+SiEo8;H;d3$(%=YQnC_k=CNLR;A3ViP zzNDZ%5#6cjA6n_xyGL_O-qL%!SSS951+^}N`)fZ2k*ybBE^%1(Hy(;5r@Q=Han0gOF<%!LA_q5!WOV{y_ zi{jhOn>_aK|5_PXB->8X-D-`4=lML2N}w zCU>2l=jmVs!w;Z3YMuuVVjY5KY72KVB+kO0qAuc1gs|DSJH_p5_BKL>e%v`WMjjjQ zInchL-cygOZ$j2ihOeE##LEH9R+ReXBE%t+#oOYZ#Uh~|7x}GUx0MaZZUYA5ao3U=%n-*Db0SWRmmv-! zmFqJC<%;nKxG|<01eZ85%+G55+v&GyUc@oUDzA~C2fz^Yi^jCA+rz3$`79|*8EIwy zLaEY%$%$IADoPTBn`!-FChVo?-vBuLXH-oyxRYY6~L8j!;QR9G0q$^(EP0;xto2#PJ4i|r)LNbCBp-6yRC z&`z@@H_ov2u+df#Cd*~J&MPt6{}zedITCbzzpCm@3=BP+`{nmqey=&XqptD?rBeG~ z*-|vr25KupC8@*qJ7kJ{P7w+ zZ{=|FvHo%Y6|4)gImxvQok+lcEcSl{Ul^m#H12h|HMk}GzE80zo4yR%*fL(JXu3fh zo>zD~yepPb=ni{#ysW;=HHka2-`>DxK;{lF?@14bM*gBX*6;}qakKpQXV!|i1X09K zT0}1Cpvezk^`xXNM#2#W6{iSxt$4>_L^ca~{*m8G_B*xLb|NmI!*GASGGPoN4s2oe zIPb}&eH)eE#sXK(&y`eITNH?i*uicFCLway12SWxfVT=rS;nfjl_dM<05^;uDp}%F%u;TUw|o zP}62^Sv{dW)#~&FsNO>}o_O$+>ay6NJQvq1y#8R`lB+eVRt|X_@iUKg~mWRzBniUfl*J=>Z-k1X*jv&!lhhU&4{neeJ&NdFFIoKx7ha ztA&ilHyy*);ZqY-wVxRW+h^hU8Wm_e)BnFGk7Ut6ZlaysbzlZDT}~JHj5sW(0o)-9 e$V30vNG*9m;1Kb)ycSTAm?AI5HZB8Ttp5WD+Z>eu delta 3753 zcmZu!dpOhY|2C&iG?tGsXtIrX(1 zW+8GYqLDC%8KSU=B$mS*=DR+>-*x@|`8|I;e>~51-Ov4c-S_>vuBYa7-RZKRtuiep zivD4gu&rpI>v3zSZeS+Za2MBs6G1YUj_*)XuzH6R-PLJyGhcrBY7D16xCDFkc#Q{s z%lG`kJhaLe1aqpBmD#4*>L#b#ub`jvU>Za}lh%&qx$_z;9oFO}za|kp+p4wWfD>n( zKLOa}jJ=1luPNoHFyh>UhTq_Ogcoa{%3Y@uFaY7bz@PRR`CxZXROa2t%J*`a zJtus1Z58qAyO-yGhVd_%y_JXkUl93MbQ>`B_RsP7J4dFk+&{2N*qrcJ1g;pmYQqIl zQJqDWFDu%SAI<{pS%TNEOynOg%clgW1H!&qvvSg4SVU`9^CQCuUA2++``~-`L(w_2 zd8i!f{X=fGx2l`5t%^Nx#W0fvBKFPa4|gE*VcJkz_<(gN?W%x3*cAh5e;=Pd*+)+b zBOyjGnJZp)KNzg0GlMa=ja(ljSulQgH`m(inlsj%JLLj3+ZQrrmvbmQ=RxZ);1)!2 zc}+hlamP|DR=TW#Z0l#TK*ma+EVae{`kjew)}E42N<1;DqZqjP^ddI7SdE-mT{8*S zD!ER=Huv2>pY_Z=)Iay75vm{-dppd?l7%_(~g`7<*SFBUcF zeci5Jhn|>`ZcS>2jlEskt6iVoqHi5-21y%EOPnBLo9T0p{2GbW9alzG+G^cOCR~GE z8M`q=OU7K{zKEYV1P(|^UpR|2GD4T^D78l1xL&#zx31rDY)Q`dbKhIM^~CxK)dR9s z+^`ycB_r;w^LGas&z@s`)%{X(KIhIcNPHIlRutcvV3 zia@_usdemBsxKd}hTuQ05V5-sCL-7pnQ$rpOZRhM;Qh%O3V2KK=h+mne;X~8DCytE z1T^&HcDuQAD|NbiH$kWwx66;X=S+4D9jy%xXf-nHT&S|o(T~*S%)_(N=RMg7-&8yq zgvQAZz~=;7S(Qn6juewH^{lpTJ_gT&aGWC91%@-_$YHc`{7}IL>5|Kl@7S}gs{5{G zg$54Ed1UjcZ+_8N?yQSxhWuQC;DH6%H@e-;a?f$d=s#T3HT^^F|39J{`c2m;!6`$& zlbC-iXI%>ot-?3Z>zLOnn(ds5_L#bn2fY&U?#338SSXuV@Ahi0ftdA@ZTK-6I zmC4aE{jHkW7 zcz|L7RtM!#&V$u<85Y~WbTK2678?=UwMwXwwun^9L$Jm{Z@O)UU~k)PmRu!%Ow?HE zUNIne6fmu;PFU_;d40-?Z}Zw{wit=pfS{aI0`q@cB4a3@!B9YRkIY_Z(R8`Nz(4Hu zKrVwPh={kpxE^_gdS_a6?uDnYa6DYSbb@Ps%JI04olZ$FAbwEB||lT(L(LPH_IY^!f(cQJ*$KDI0cCo3fLsbsMaQQa{KJ zj=5dw#hoJnpfr1!5sVBnbhp05MnX=AjlZ0Co1|Yhay3UnE5kz70v_k+`ggROY_7L&3e7Q74uGMzTM!`8sI6k=9D*G|xlL^KXAGFc z_e_~wb;Rb*Ta`2!*LO_+P5;{bQyCldkZrW0v}%q>Gs9vK1CMiaEd$(S%e-CXy$DQT zKrcEUC*qB)MC*LO1ZV!)^W+@L7*|c-Dl7jw@IL552ej_0D44m{Vl=dW7P24ch6Uye z#bTfUTiMqPFHxD?O&l`2nFAM=>{1i5QHL$@7%`I&B5^J= z4=T?Av*NuKf_x_jqS0=iebYxrT?8>p{N6u2Bm zv}u=6qr|dv#Ik4nl?SDPkT}^7s6!Zr6s#w2-QO`PR|g)afIKGD^a)S(0k~UsGXL3! z8BO?jBqXXNxLe4+kQyRYI6=Y^s-*D?x)FJQ>?ekl?FJaIjWW4haqnFzj3 z+=7&m<{oLhyWXD^qmp(@XZd!tb#k8z;ioHUR<47njhyU~|CpNeK-w?8SI^Do@@g!t5s?Hn;&mOU?q>x<`r)nvYi%T|2?_22d# zm2a@)Pl(qL#&KSw4EI&xZPT2fyw5I+lg_*QasozKtTRgs0ZBr-ljeM$SmQSGGa&vC0nZP>YoUnBBVbd9qkBISqwB(ky9DiHX-pgg zCv}M~cDDQnS^2FV$I*DHl~*pHx>`>I~f zbk^6NN`Ur5cP%S)wj}#C1&T~&*C>k#gZbk;yteSxMj#OJ^XvDy!8S9shx;E>iqfJo zY+5^}1m?8~3-8ftO`Y_`E&PAJB?rDyDZJ26pUmm4dh+{Z!Thh*zC2pRd`|mRD^wO! zoq&-W&WXA{m806467YSwJ0PUkY2m;C%Ypc9BWvnSL-n3catmD5XU8j#^A+W8!0(ai z8B}&@eNZMWg%p_C(z7;qoi6&@V`9_?GiM$6WCWmGAkW&UWUtFq>{K4uDLsDYk_2TM zZU~T_<7X|-GOy%zQE5NpQGwW(TgV=|zy)5?Q|UL;LBVrRULPzvr|zw`5gT&6c)Jho zftI8Ft!J6r>uzx8r5-#fe9+^%uc%9LuSZ{)&cS%Xj#5{(vh&f&lD3E5+230lQHw3V zd?&S6=k2rD>K<2TsNIm4CC}uV#3bQ1NhiDwcu`orrNvk4`9X5UHbPW@nS^OmSFAK& z{S#U4!W#`naD{wa5TE)ED*w^LCNK#4B>kZ$SI5)X%~B)>Tt7c@|McZ&t1iYCA?!UD zm5aEQ`{zRxcXFL4O};jpf!60$erHq(vjYh?;f-4LWZIb`{rCifH3sR(<(KiQWQKZs21h{$C}{@TvR`%o-24|s05W+1 diff --git a/summary_results.json b/summary_results.json index 8579530c8..91afd7aba 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.73824, + "Performance_Result": 8.06579, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70094, + "Performance_Result": 1.73249, "Performance_Units": "Samples/s" }, { From 685aa0e29eb70c59ef7b1eec84cbab6fe5c08c5c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 24 May 2025 17:00:15 +0000 Subject: [PATCH 645/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10938 -> 10939 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2dacd663f..586822005 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 16:52:42,767 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 16:52:42,767 submission_checker1.py:3308 INFO] --- -[2025-05-24 16:52:42,768 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 17:00:10,690 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 17:00:10,690 submission_checker1.py:3308 INFO] --- +[2025-05-24 17:00:10,690 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 16:52:42,768 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 17:00:10,690 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 16:52:42,768 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 17:00:10,690 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 16:52:42,768 submission_checker1.py:3324 INFO] --- -[2025-05-24 16:52:42,768 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 16:52:42,768 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 16:52:42,768 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 16:52:42,768 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 16:52:42,768 submission_checker1.py:3345 INFO] --- -[2025-05-24 16:52:42,768 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 16:52:43,743 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 17:00:10,690 submission_checker1.py:3324 INFO] --- +[2025-05-24 17:00:10,690 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 17:00:10,690 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 17:00:10,690 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 17:00:10,690 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 17:00:10,690 submission_checker1.py:3345 INFO] --- +[2025-05-24 17:00:10,690 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 17:00:11,917 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,14 +27,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.34096 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13835 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73249 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.2816 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420905 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421985 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.06579 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 88a90959d..f96065a3e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1272 +1273 diff --git a/summary.csv b/summary.csv index aeba29562..2f5f46fff 100644 --- a/summary.csv +++ b/summary.csv @@ -5,14 +5,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.34096,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13835,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73249,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.2816,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420905,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421985,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.06579,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 0d1a7d886..7411d3acb 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.34096, + "Result":6.13835, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420905, + "Result":0.421985, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ce83e4626478418bd81d671ecc7864474789654e..43f981dbfb71228d75dce499a943f78ecd3a9fa2 100644 GIT binary patch delta 3729 zcmZvfdpy%`|HtPv=}Tioh)p(f>L9if&0*LubRwjLVHzUG#78Q}7@Ond@OA1Wgb;EF zEq2doktNla=3HV$&iDN8-{W!rb6ZJUa!~tdj4@WU2MKs9kfS;RkRuxLJQl2 z0tT`fq3VHjh_>_x+o1@Gq3}JHjHLM+{B{(w{gUD8^OqBN)sHLim)RQ}NC(&TYt5;3 zE-!efF+zE*OR2c)6!fWuz+Su7oG$Mw< zzEiweA&~fF&8V)D!Enjf38GArp7n=(?P5HJRbKchbsEq6=5w;opZS4(30pEbbs{8u zy?LBeZNZhwhv#&$Jv%i+V+W+g*J8fyEXV0la>Id7b(e z-9QTEJZ$llw4)BUrOcje&T8W1tYBd1?_QJa@SM+6N@#yZ=e3NhGhQCmHbdd-8|!K> zXWd+%ej+@td0%hf^CXq)tKNobwJ9bLB8}#J|Kn!0 zP*Ksy9fzbute^=}*CS4M$lY znwYzi#@}j7Phi4s90A&24ct+M-8it+b>FBvL1$zm7Ta|TKuW#%oPFSIRye$+sXwhS zp{hsHT_-@CI}U7&{Rn$}f+I;4vqJO{&q9*F74WitJu{!0xHeTn`>@mY=eO={_UVnl zkb_n=yaCq9*RIiIi3w7qLCKm*(5PP?F} z>AH&0ri9|*jIhiSmH84!2gr1n=7ZWUe+V^}J}_ z``_?_l(%SBVP5Cq?-X@B?@`H00(rD;_(kc$r~>i4%T|@=703CAC%;unh6go&D)r?v zUh&dSFS$jgN?o;?lu>P0O2qxZVS_9WO$@H~o^Z456?w_e>Zg`fhF-QMW)AF zIp8=uz5iGKfYFabs$6H^%CA&XS+1=ljfG@+k9TAMwvK zVb(sR%0IRE& zQ52CsL@`8OO$DR(##oo6e_kJedBA#fkn|sm8vc21acXHJkRCXF ze+w*2Krc5^D1c1=1p6pdL>dAzGH&)2ArU1plR_7b97LMcaxp*X`jKZSX&Mg-tKx&~ z#tXGxYVEiOAFP_p+@#0vwR((w(v`WsYja0MzO*J*pi9>nd4e2cMH9wGf;0&VX@ZbF z&Zelno4!JP3gq4KO%~d#EU;$xP$V=f(pSj!iJP4D*y`PVjB_prty2sVwRGTAu~Hf! zsYKAn84o~fNWEXw4B~5)b!d!BvNhd^qroiMfn|?SD_8$U8JqGj_OJ;G)A6*ZtgWEe z*};)3{45!q<{MFeWP@w>+^Psm>&gU6_6XBvC6)K2*`BwmwIA`IL8MegGFSM%X=tfa zz$4l?+Mgpoa^xSV1k{m^K}h3x@t$eSy;Be-C>t?}*CtFr=0VkZ5g3IQn2%TweC7qe z@{Bu-z9ulm^~1szJnPG4&njZ#MQo_+`{++uHBrsvwuOuviQ5TUFJb&ZD`vXyh$7$P zi80nxfg^+wzFeao9-~momjn2Ey{3F*|Ix_GF}acS+-W`H0W;tbbEM{-l{%;RcS#6y zQ6^gBdbZ7@do~%Cmo>K1Y~W=us)?=^4x=sPNfl2c1w~&HH7>`m4Hj)0TjRhgp0Uej zy+7g1?Mlc&&+87+t&rTvxy{S7#XMRYd>(AexsLG{qkpqo#?1opo`QTy@nR>F^=toO zpT_Gn0O=8WDPXRsv#IVLdJVrdVj~H9bPtL}&n&Dm=i#4{b&@OUNV54bzlLD-!-Gq~ ziOVHtm6l4E&2)RIeGoAf`u>NzkBpD0o(85u$e8W9>2oZ)23qHOQ23h`BfJGi6OsrDD|_l+}LT=S#3{<0%aF`QNsscM1Cf$}U% zOegUC;btI*VNg?))?m^IbE zdhO=bEtt+0jQa{{hyD)TVOXSh(Y7HyhpqP-BZP0%Jh(?AA|8@WL67{PF{6i}w&z7% z6@}qGVjR5=kN&>{4g{T>{((rftN_& zn=^v%nhQ7979vw0+W$0rooKnRU*uO}_$i~RC4UvVCB zBr@ufq4xD2&Z#5B3t5+pQR|3;x)_{=(>b9-Dttb{yL|S5GS&V{2mxK7YZrY_fm{G$ zMY@hChA-HUs1g&1`~P}e9TUY^#vh8*nSXW;&|SWrx~6|4O`l{;mianVD!?fcFtL7Du7UYAaj^GRo%p_*{o9+A*V7922EFMQ z?3MvvCzmqb^}(G%htTL@q?ihIzc4&Y0Fmn~grkeYQG4Me z-5ByQer!*622A(nKBAE`80T;-X2A#@gTjIBOoOg=!y+v=&*}^Zehau({}+shN|8km zlJs&BDJBnHD?_;TAzPfV?75$v>mC!z6P)lR^2xrP@+hEY+rBy25NIYLfJ)4fej~#z zSCysQ%DV-b|x>RDt)Zee0-~$DfG0?V^^qw~wdFP5xru`n5@0&!TPj=qYbE z(%%Q6wr|RB{ptKFy81Kz&IY?*V(a{5W4g}P8Txa7eXDJeypf*47C<_GRSrGrG?ara zA^+{bypEEcr+jvaQqO>d4!^ZC?)M&Mk-XWwFg<=amAw(LY+bXpr<2wO zs@*3$x=;9ShQ1JO4gmvLww=c$?=2dde~WZKT^3V;$OQeyA`wZ(pR^;#oVzeJ5~MH6 zXcF6DE^+dfS6ja8E4p7u-c~H6m##1vTO|IV`dn^E%I9p}rT|eTNHyZR#Cso60Tw;a z>ZDplykmgUpT}@dF;7A1N{hS$N}VSYBz%;Uz8%9~cB+c7DZ2`IFuKuKg^q~5>Qp$C zZ5Y%dI^~Zj7CM(CH5+|IRk*rBEw#FDcnI=Vc!eAmwn2@y*Mv9o9VH9C3(TsHvvgK!q!6ap= z6k`yPElY{)`_AAUzxREf_x^eAbHDeTb3fnnJoo*r`dgI`IoXB$>VgAEL7eLNMs4a< zLND%aM%`m#SIFdfDW9TnIZoN#?G%99>d4#vSlI;h&xk&!rF*(;X+clue`nIZk^KLt zyi_%&ZPb-G*j5Wz?$hqV8#FMD`0X_*w7RgDpXN*g$qP5a5L6{MWigg#Y+y%j{-9Pb z>G`|$S1@BmvkR4@P8)uI5PtX`GtX7{dyn)E*#`LjI2$hU8Nf@PDSU3F~GB`va4{%AXs8+13dF^;xQZrw|vf#zd*%lCt$U-FgiBT#Z5o2 zSBE~V5#9eRcUU8b8*NKlQ$rlKHDNDa!8l-Rc!ttSz3y~?B7z4|&+)zV$=>p ze|+sP4zq~-JQZpvjl!FRZ7yA3$lr1tTZQ_|n2*t-wkf|2^YXGg?_HJelr41O?Ozg8 zQTtuzN`_1<8jrn*X#!y_!5=Y`tc1v)ZCww28}EQLtz6%w^RIvS2fn)0)3R@JTGE1$ zouY{=KMQ89?IvpKN4A5tg+Y7Bf{LwG6 zqcO&_i)(JBITpht|E*=n)p>X4&WYWXf8gR@*B4rZor_n&4NgkB*Nx*CE7f(4k5IN( zvs+i^(tmVW+AOZm1lZ^%o*g>x*)x{q$xGFC-p1dnjyUweCRr5cOw+x{;>En`CG%l0 z@#h$_?&TBy!=Qw-ojK2k4Jj^1wyi{pn|Bt{ce9Czbu6DQ?2nF5%`fkMs1c+y-DR@W z60xXZ%%YZH6AuW2fZs_A(8LKbCgY0Z;o|aV`x=?WR^FkN&y?AzhKZTjEypNgm&~Df zL_+!+3Z|^AUcudD4E4jj2{~ZMcj|AQ@*Mr~5otWRU{3DXe>)IVKUG^6acJ#T%n}!< zRLByye83WSy?ZS{Bo=6*rYH1;b%uF4_U|XoTvM>%HNh8h#26#B3%RE4GOys6T$ zPk`6TP6)9Qz$F5iTgNx11`04j@K$AR@@K3V{0lyD)cnxm$t$7{g$>T!Aj42uq1RcosF(z)Z~2th=CP9(6W^x$Pp+p` zI{kt+gi)@X+qTul@qX|NRWEnLo{{=+ikm;Gym`%Fp#I3xUo_>5|D9TH3H+A^MwUCv z>Ny=55hV@+f&MgB$|DJKr{BxVtfqqhve-Cr9>e8#B?^*+@*bJ-H zf$f%0w0-0wB2qEINK+-tpYHfZ*dS%8PzZTvp4{H#ag_l}V?R z(cAA1K+X@Ub8`=xT;LAzyk10gnBzkCSAA8;oc46RzYYsfTxlTQBgHmxZ(95YU9FJq zX|eca23Lzf6HnzLVBt_tPV#UhbQahH-{>A1+X*l7A9;p&A7U>DiPg-*+&5Ke}c0Mn9L{n2Nd7x7PA|#Me6e0PhZrR8frO3LBN`!G(e_BAyD1%bY@2l zP@P9RfO=V{QXU1?G?N=j_HeX~`U#DC@H5~D@8Y8~Wr}JQRrJlo`2P$*InAWz*-tFX z2^nh1uOwMW@kkK$;fhzZlvvTQxD(xQFRii@M!oK9a#q+}CK3Sh-Ss5ch&p1R&bMEF z1E&h#{;_i7eQYAHa;ZhR&DH0Lo6XRZ;FrlbQ)qH3A5ALrjFx07P@T~C)Q5aWvLDY% z@%kQ`ob2C~E907Pnz7CVtmYIIIdRxbwXYsrPw|!w{BN6K2ZMQ}*ki@>DKu1QhJnJ9*eG0YH)t^9!;baVQIju!~8)o}(xXUZOGl z2g-)lv5^)o44igy$lT$b635+E7yr;!)!j>cXHu;Q)&$v2Ng`L_ks-{~gPk?TBizh$ z5Rd}pirE6(B*UfHCHQNMe1vv>aLCOQVJ*af1iPh7o}00SM-Zj$K8||p-zuq(`6GE- ztx;m;37Q01Whtf_DV$#%2{+a27Oe7WABSp2Q&f~mt#-ncFA&v$PT}q~x5s9Vm^vYg zq9D=_eRbWB|5g?@y=-@|y*&4YbC+9TYYxGIwhU5yR| z*pWX-@{IA-7`gLr4Ca-Yw>P`pbBt)M@ZJjJr~9c;1}H5G;P)!`b)D#s%PJ?I*)1kN zARNZX)*cIP{U^BipCC{L+0Qo*sDmSPMGGX+ctV@hOuK=7TFP(v)O7s^vE}xA*m$}h znBGrmGm{;9wV5BN_~2wya;3C4FO7#U{@m|(yV9Bpt7-7;3G-U0^N zDlAt99KjFM-#zaUoaBKYIKfM~q^LU{-Ju;ARuRy%OLI=zGymu4QVeoPl1iV31`_dJv-aK4ZPZg+@hvlvmJ)V zbtO_!!(8S8T8^(6=$j_E5=K%yrC{CAAjXq@I@$w(Bm&{cc4ooda1mY6cu6#2(8Cb^ zap$~8oj0+|G6=jZR>BK+cln>0-V}0NFa_+3^esC8g_E`t>Uv;Iy+WHNG#E>^&-po3o_) zISi|YV5u#Pp4Uk`-x=pcJP@1Fkx88==lJM&rtL?7>c}}h5c?oJLsztenc@um3GxEo zR0NZCt3%SUdUri^aOC!piTRPqt`n^n`M%|->IP)>82ff23qKbyOG)~-y9k$TCV#V6 zCYzLcY-Derek%u%(*dlGTsg*tKKT`4Q0YIsKY7G(1cxZcl~MeMukpyC@ThZkGN59| zSHPhe(vCGNn3)`3%yDuO{B%GjL^|ieFq9`I5P)X-2?UoyGkN({Wq2#?7R{G5dcM+k zIOHBM6!oGZb@SGc+Tts=qb#y+Rs` zt2oo8+~|-$toW3>z$!Y2#MNZxz;w0$!lIybcf1NmY;&UTDY$S6COPjtk-x@e%Fbs; z91cN*y|flc>;@d!v?_NZge>ph##R6*NxO z1u?CsTLG=jdVYB4tL|a<(Z{Qdyyb(9M}|jzS1|78O^KeRs01SZV^QEk`21+gbi-~Z zx*i?>$AxNDI&~SezG<>tF57sWG&HC9c4#NPzzg>7Xi0sEX99cZxV4T+hs+*a-jx{& zi~LP z2{8va-(*Ly&dn(J)H_s?n_y*!21eaoYi^_ROGWaaS!TJHJeV#j-C8v8QQ)4 zAzN5u3z%pCco-jIs}nb!CbKDc2}@4%clf&Nlii8PAlX-m7>{i@hp)jW$6GG`O5fi) z4aZk2L)%&Y@8`&nLjie7cJf-lbW)1EKJXc7KwcBLO%haqf{wc?0|Wr9SO5S4!2hrC VCxav#1s$LgiKrmWF(!Mw^FIY-6i)yE diff --git a/summary_results.json b/summary_results.json index 91afd7aba..bf1623f67 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420905, + "Performance_Result": 0.421985, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.34096, + "Performance_Result": 6.13835, "Performance_Units": "Samples/s" }, { From 0f57d1fb271187d92dff7adf22a6f8bb86eb3478 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 24 May 2025 17:24:53 +0000 Subject: [PATCH 646/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10939 -> 10941 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 586822005..805aa7222 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 17:00:10,690 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 17:00:10,690 submission_checker1.py:3308 INFO] --- -[2025-05-24 17:00:10,690 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 17:24:48,525 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 17:24:48,526 submission_checker1.py:3308 INFO] --- +[2025-05-24 17:24:48,526 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 17:00:10,690 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 17:24:48,526 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 17:00:10,690 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 17:24:48,526 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 17:00:10,690 submission_checker1.py:3324 INFO] --- -[2025-05-24 17:00:10,690 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 17:00:10,690 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 17:00:10,690 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 17:00:10,690 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 17:00:10,690 submission_checker1.py:3345 INFO] --- -[2025-05-24 17:00:10,690 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 17:00:11,917 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 17:24:48,526 submission_checker1.py:3324 INFO] --- +[2025-05-24 17:24:48,526 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 17:24:48,526 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 17:24:48,526 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 17:24:48,526 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 17:24:48,526 submission_checker1.py:3345 INFO] --- +[2025-05-24 17:24:48,526 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 17:24:49,511 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1457 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0302 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f96065a3e..e5d76887a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1273 +1274 diff --git a/summary.csv b/summary.csv index 2f5f46fff..02ca1bbe1 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1457,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0302,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7411d3acb..6501de949 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.1457, + "Result":21.0302, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 43f981dbfb71228d75dce499a943f78ecd3a9fa2..050d2fa331bdb8d20d894d471a43ae2ba59cc5e3 100644 GIT binary patch delta 636 zcmV-?0)zd#RlQZPatH-izrG@Nvv&x)41dcmyhgf4vR(L?w`sJviB6-*Bxwxd>~gIu zx2@~XzPnV1AJ!ie_YlvF?UgI~$BHmy@WG zewjzd&FJ$yyNB|{HXFmwqxA>nHRg+Pycnln)}!a2)VG}UPp0wk$JL~kv+G#oWb*oYGFW9tBj^_QF=)AwU8PAf&BpPK4wOU@Yu~>if z-sWWH81cBID_O!K#AF>T;{~k9<(^krg<@nw? zb7$mYxtC4!3)Ts(N|Yrjia;^Vh1!T5Z63 z0WJz4=_1bOb#_PcIhj9v9YDkdAEvMwkl0RR9{0ssIJ00000000000001h Wf!dQ(Dn0=(laeYU2H__F0001O*F56@ delta 618 zcmV-w0+s!}Rl8NNatH;@p!bG#vv&x)41d*euAn$&VO{&N}q_+G={ z2lw!Yf&0_X4gFK&xREXWKANp=HA?;ko7TzEynhp&_ZK(gS<;w9qims8%WF0k>yO@B zeVxYh)mKil@i9)*czXZcZPXt|i~PIn=iMld)BA7oOS1h4!;`-Nvk(^84h4ghH3KD+ z?Ia$5%}&EG5QOhMMdW*8JEefEB$WX5L?tAOK-?^Q+gijwvJGk99ye*zR_cl4dS<_! zUAdZVb({Ty56YSv(So21jIdIfb&cMa^ZWs2A#x*mYYo(B2La8Rn@Tt)tcO=`9r&mq zWC>&f6Rt*^7#+hnhz)2SisUd!Tv@MqOr>9cW5>leUjr@)xx*SFmppR32j*@hq61Qj z5$S$>+XJM)(1HdN1EvMVCvOC=!_|gfIdf_icMh&vkK*Xw1~nSHt}D7pZ%z3r{#?Ag z^{eK}>`x^CHI)=hc;M0cro!h%I$X;`T%;4P6v)S2(yzQ778&|LW+?z0;vSWc$ Date: Sat, 24 May 2025 17:47:18 +0000 Subject: [PATCH 647/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10941 -> 10939 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 805aa7222..c1b7aba80 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 17:24:48,525 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 17:24:48,526 submission_checker1.py:3308 INFO] --- -[2025-05-24 17:24:48,526 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 17:47:12,886 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 17:47:12,886 submission_checker1.py:3308 INFO] --- +[2025-05-24 17:47:12,886 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 17:24:48,526 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 17:47:12,886 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 17:24:48,526 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 17:47:12,886 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 17:24:48,526 submission_checker1.py:3324 INFO] --- -[2025-05-24 17:24:48,526 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 17:24:48,526 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 17:24:48,526 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 17:24:48,526 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 17:24:48,526 submission_checker1.py:3345 INFO] --- -[2025-05-24 17:24:48,526 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 17:24:49,511 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 17:47:12,886 submission_checker1.py:3324 INFO] --- +[2025-05-24 17:47:12,886 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 17:47:12,886 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 17:47:12,886 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 17:47:12,886 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 17:47:12,886 submission_checker1.py:3345 INFO] --- +[2025-05-24 17:47:12,886 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 17:47:13,866 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0302 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1213 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e5d76887a..9c113c6f7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1274 +1275 diff --git a/summary.csv b/summary.csv index 02ca1bbe1..0906ed9f8 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0302,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1213,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 6501de949..c721785e0 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0302, + "Result":21.1213, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 050d2fa331bdb8d20d894d471a43ae2ba59cc5e3..e27e12114a050664fdca222a37b0a3e63dfcbcc0 100644 GIT binary patch delta 605 zcmV-j0;2uBRl8NNatH?ou6oSbt31Lp&QVo(;$O-?Z;f)*ot@j%PQGl``B_x7C>@0iQTEq|8hO~c=lcs4ab;WXg@0__aa=F~gHu?qYgjN;G zk`zUtnC3!lEA-y1;s+EtPZg)FR#2e>IJB(qN@fVt7GAA3V7&kr1(0%tnF{T^Hw0tH zb|9%sf2o_Ij|=-E0_?3#SzRVBn_pBu?(V6O*y)x}+-xYsjDB&-KgOuxcz+e<}v3E;%F2 z0`=O~CB7`e;ackaI-GbTfIl9Be&cOiRlQZPatH-izrG@Nvv&x(41X=lF1$v%MzUS_n73)PxQR}q$s}nE;_Py* zE4Qud&%V1rSV<}Y>WWH81cBID_O!K#AF>T;{~k9<(^krg<@nw?b7$mYxtC4!3)Ts( zN|Yrjia;^Vh1!T5Z630WJz4lLcDkdAEvMwkl0RR9{0ssIJ00000000000001hf!dQ(Dn0=(laeYU K2H__F0002~1~!WT diff --git a/summary_results.json b/summary_results.json index b9f35a318..758ce83c4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0302, + "Performance_Result": 21.1213, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 0386c20cb8a3a8fa461bd407385e7fd133ca0144 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 24 May 2025 17:55:17 +0000 Subject: [PATCH 648/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 8 ++++---- summary.xlsx | Bin 10939 -> 10943 bytes summary_results.json | 8 ++++---- 6 files changed, 30 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index c1b7aba80..5eb4c6cfd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 17:47:12,886 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 17:47:12,886 submission_checker1.py:3308 INFO] --- -[2025-05-24 17:47:12,886 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 17:55:12,014 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 17:55:12,014 submission_checker1.py:3308 INFO] --- +[2025-05-24 17:55:12,014 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 17:47:12,886 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 17:55:12,014 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 17:47:12,886 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 17:55:12,014 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 17:47:12,886 submission_checker1.py:3324 INFO] --- -[2025-05-24 17:47:12,886 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 17:47:12,886 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 17:47:12,886 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 17:47:12,886 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 17:47:12,886 submission_checker1.py:3345 INFO] --- -[2025-05-24 17:47:12,886 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 17:47:13,866 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 17:55:12,014 submission_checker1.py:3324 INFO] --- +[2025-05-24 17:55:12,014 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 17:55:12,014 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 17:55:12,014 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 17:55:12,014 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 17:55:12,014 submission_checker1.py:3345 INFO] --- +[2025-05-24 17:55:12,014 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 17:55:13,102 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,8 +28,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13835 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73249 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 16.2816 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74536 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4679 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.06579 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1165 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8585 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.1213 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0092 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 9c113c6f7..4b809f8d5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1275 +1276 diff --git a/summary.csv b/summary.csv index 0906ed9f8..5b1efa2de 100644 --- a/summary.csv +++ b/summary.csv @@ -6,8 +6,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13835,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73249,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,16.2816,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74536,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4679,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.06579,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1165,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8585,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.1213,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0092,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c721785e0..660cc26bf 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73249, + "Result":1.74536, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":16.2816, + "Result":21.4679, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1165, + "Result":22.8585, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.1213, + "Result":21.0092, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index e27e12114a050664fdca222a37b0a3e63dfcbcc0..25b42dc7249ff12c7f13883f80bb35700a4ba972 100644 GIT binary patch delta 3806 zcmX|^c{tSF`^RT26G_9=Lz?U}V|h@96`{NA8L;#c1}N7bHUv_8!=~lsyhcq{r*w->;YaY2!clUPv(XBS+oqA=;*cx@^b3+BKbu850NpiReB=k zA-AcC;H_&-SM{~5P!Fz^ZQP7`RY{-F zn!7l^z+C+?II;4T;!i;!FJH`q!KuZW0JY%%4iqhiLE-Crzup-NPSp+6N&%?HdIy&QRCL_Pk z8REhfnozj-CoeVAgMcO97ZU}&t~fkOIqekvoV^+~tk>>cwZy~3xQ9@4huz)D4_)6# zYYQF!_>t3c<9v)~7gt;=^38^6e!rFxdh@Q1J(C;XLow2?e=M|1IF@k~8XeTI7mO3R z-|HHh{OzptayvM4S^xB!TQ2_icE=K%A~_`Q-anq>evoD0xCY#)i{9}HNi{3WpXu^o zNS7VdE}suaODFpMG8p$Dv*^l^ zh8(?o{~5)g$<|-T6eF`=zXXvjXX+W{D*@pVVObPghSVu*g&eIQ#;XCPH<-#C!OLL!4ino6uNnf`|=Jm!_ zYzK{i@_Nh2SGt)Nk+0UmBhM+3Lxxq2G(Rar;tPX81|6nyZ?7EzA12@nO%-70U}?$_ z$YH`WeE-Gq*4uc^6OZD5MmWn|LY#CS2q%`7ykcrZ0RZ%VfLaOPec7m7@BYC0<@94F zxL)UYwf*<=To7VVrsV@rB0|9iAcoOC1&%JrZ)5b~g?1VRp zWsI?7%xof$A5z`f+|r7ze)^(H;E-0NVk@G01mMp?@L#7Mh)9qcRevhKA2kbZ#jN&S zS~0k%MNasp+$3O`8DJx|z(5F$xI2NjgxsCpcC+7G5e>jSx`WnkKSs^WydCcO2sT|^ z@v`6q738TW19Bq|s~fb_K=FtW_lcdWiQ1Znf~ll#xoiZ# z7xh35VSMJCw)dgz;(e|IeYhK8Q~cRXWMPp2m1HyM=GNftEcBGLCHty0^j!ig`J~mb z_TnT86W4dgj7*ggBjKRAi%C#& zXJ&jdAGwpMq0;6}yOT6h`a||i-5xM^%vkp{iy9J5!OT{Z5Jyp<{ib-|Ne{XiVZqey}C_%BtSZBtgo`2 zYNq^HmTH9gGZ?GJPe!jn#y?`7Ve9vu87SUWR+-5$BI1p~9sB-%Ub#Pzga$@Wdj6uw z$Xg!4IzqqXJ;z?Xd3DF{+60Ff-h+!p0+$OfPZGA{T+hz#rXH}cX$3h;J)%l!pD7{m z6&iptP@<-h`vla*;ez2$z@5*F57Z*;6*ITo`7@knvBF&jPa>->Hzbxw3ly}~zL)5* zD7r975Q%p^>j6$XaGi~Y5i`@QsSl{lNi zV%M~0W~`ScTg1Lju%4ThYe9!=2-0T9Tmw>j%xhG=W~)a0!>jI^BSN6ls)4=Vk?l$ZR=pEwKpXFA)4cWp3Nj2a(o=#*>6CUMW0yrEs=s-Eop?q1v2` zC}Hn76@fBQ8^0$tQ*O7l|5 zPW#I1@jI@()HE|{qYd4HWUs@BwDmS!kQ!+`1+u))Rsk&pgf3hhM7n*dt*dHv`EW7} z8~;T|a>&!h`HwVv$tSYaB?~PYBgb=L8vU}I;$Jv)!TWGi$g|2v+cgJn5zs9q>`fyov+R1?fB@yHbuvOeLVtl(IK?#D}OaCebUui7~0dolupwXtk{uhPP55+iYwY zx-OOf@GnX1HQ$M;o<(J_3zOh5zrC;(is1v36S>mjbaic1+v|H zI^O~x3O5i}SefUnhB1k~DJ;8TVd5Qq`$8`HvI(V}oBDuR`~!D3-d%7e&TY@+q)zBO zs1GBBJ^DbFobb^NhiP4xcWdyZR{4FX99rKnBrlb(@YHphH+gP4eO?}1VA@P_$@M0b z**)4S?(7?P!Pz*s2vX@niziTrnvF~1yT!J%e?4N^nfwIj;!P=xbC|vL( z<{|cIH&5X=OeE<9U@GO4bER9>zHZ~k-H)SJcTI@9CL0dy0TS?{eYf@!f>W}KLw8d7 z#Du{)YKos-rG%BT#acYF1$dT;8F+8hu{X)ur0*a1?DCp$w#ILGYkHS!uMno5SjqGo zJ}0VdXLrOJ2`{rR1P^_qty}Ob@U%hWe`}NBStA>vn15>+B^n<9A~P^@aEu%|;f);d zsTOyIeueE#8A`RsmCE z8w-$vJf0Z^YDY&PEImPly+lBeAgQ8|f0GC2z85?ofW04*+BgI0gRs97OqVLwU6$jq zf4kYlI;LDx9MYh4k0A0U+RHe>FUhO4h1-w+eEN9)ZPSa(ak7K5ht|a2z9yDNU)s@G z44e8HH(XLQOLF1NY)-AbY#W$(k`Xj0>!8u$r?`zyOv&cV*aSkqBzkK@^5f>Aou1ua zi(C!Q1I?8tZ?WaRr(=B$dQ z9cw&zPxDY6PKa-~7DMCcRB)F1`YZ*4=A2g7Ws`9)G#bZ{O( z-+l>45khxK;RKefQ}Bx9MIyhzVGsx;1gbkC(R)xZx5>c@Nt;K=06iDlPKd=U>z|tr zn?xWZDiIl=pLir9>ddHG*obXAmMQ8nA)`ZjXVsZ773$ucYya%pk{f(GD&TCzemBB> zg^OnX1*|_peXThB_jAhb4_N*AS0DJ8?Vt`)E5<{l+6rFd5Ru4ddipibIN9=Ge4=6q0TJ8r#cm98MhDu{?<9^NUAeLDzr8Y8eN-es{}2Al^c->b z$WZTjn4`q!Tj~Mf(idKex5z?`@)rQhfO2j_GnJOBUy delta 3777 zcmZvfYdq8a|HmgxL}NK5v0)>Ogr-Du7&gbcDx#FIm`2Dk@s&$C#n>1oCdUrWryN2u z7P}B~N~LH@NG#<1Ti5l!`TyMb-uT>jeV?!Q;q`cY+AcI-s0`SkH_>DGu|hE#dE)So#^vr zj}BeL7EI6R28C`ke<0S|Cp)+U2Q3+tSEEUH)L_%U$y?d_9n;0>mJ=xvjr6Y0{9_L+ zwmAr(!^p}&dFFw2uhc1OE!*GZ(i(uLfF6Z5wyHIv#Oy<|& zg7^@!%0~?Sx2x5BX>Et&ySukF9TMV=OMeY8n=KD49#dRAe-3s^Bj}qI<76l!f%Ox( z1(jIaREv+>w;GKVsBEL)JoT3;SONPA9QNlgI+m&YO)NgndJ-w&zngOnoA^wQ64%i5 z6`@q_OU5#XBdnFCzF$ep&2bY-lj5^~qM>eCe%-aA`qUvMD z!Lzhb_=~3g6h>S{x44_;O;OGVU~}R}$Yb4caf+}NqK|MElK3T;OZ)ZAd}iv}OaXOt zx9#_e&YhuCoBlzEt*W_idQY^tM3Tg&h~Wm^J2}@qbLO9g)gE*qdlq=5=8Waf=~)fV zd93>z*e!S12Sj|WEe&pp%O5`WEQeA2?(65-;z$YPxS|U2rk$?PL~ex;7g%q1Z8ePZ ztm#kWpn8+f31z=sf#)W5>Gv|SJB}=nRq@=Cg4H zC>9J4sQ;Aj%V9NgQ%)_ph9^s0wV9SwekmW1`+>s-SSn2oto7)+TJ{Lm@w~dA90N2A zG)59(EX9!!X1d`Tf?5bz*sN=2(WK=@?>Q<{^`DP!o9v{`z{9~cKfZrbQ4ZKaCdvFg zF?Z@iz@?FP*C&A`dlG|CMSBt#TlOT%w{g6m379hqz9}b6@%=ii+-i91n!p)^M7&Zw zP7;sJ|0Z_^!GW5ag~_lb%qOFiM)(t2v8iw4Y&ZY>lHK%k*9bT=Qk5ohB=ToSy?Jv% zi(R+B-X|3Jb$;HrlNNWD;HNtGR^pT>=Fn7>8`f#+(Ym_!OWb$PP^JF;4PW?~lTf1($VaWl3|+Igw#4b5C$yC>L_dJ$XhuVP@PcAQd|r@|5G5J?+Tv%ofoQ z-|*F~I-Wm1c$uXUmUU)nbdsTVr{#k`A~SR86x3$9$q0zT>wr|cL7AED9|Gj^T`3CU zHf9hPH-P;3iO5Kw#p!<87Z)XVK zXp4==eibP|=6lvF{ouon`SG%osa|g20fekE)dOVvkGNi)u(xkqE}T|I)nfr+{lcmz{jXH1U=J44-bCBV?CfN(3|}Kf+`al zd?@Aq6Z2a_f1Ib+_xAm$pAD&XtoNZ^R1-PQubM(~>smu*XPTh#;4y@(EA@%q%VKvo zY8Lus@uS!ut6f5904(i7}kWDPJwg_P#-`PfEbdb+O1qOCo-+Q z9{(OThQ1*k-b%6EE>vE~J%q`4`grTe^P-3g_lvJatB5F!Ra zE1dr(iT)n;jz5n$2&ARm>dQkS3Zka@FBmxpHmhV|e$bDHpC+fMJz!MC2H1aKXw+%! z1|F`MPT!)(ida1cuuquj8$6i7T}9c#>S#Wujxkb~6lFyfz=ng=Z4agFf$ViQMT=~C z^Yg}6)(ziefm-VYt9K5DLuujO{4P&iWvnOG{@%~}>(XKCB!hSjO*lokP!OvVg=FJU zP8M#wmDtIBtKm}}HFjh<#N0+c@Ho!+4YZorGpS+##8xY4Qdt+Jt2+@#1AC?4EV~C= zx%f6n+7yQj4V$1b?N9TH+H!lG9UM6V&l15Y-eL7xn;iQxt2``~nGP1`k%N?EWBw%v zA$4!m#teFZs?LI}sGvdC4OAjDt52D`kCd{3*6XePDIr}@S2 z#A(#Qcx7ER(|Ju0T!|5=TeY2ueq=Cl#^fPGVm zJyQMJN_9Mcr67pCC>g1CBg3X7#wN}3vf6fv4IC(fQA~6+a2OQ72Sqf67!Y|;$ha84 zK9IL%Y>fjedPFaq_56ghcgi7$J#IKewnDPPzi(Zh&F50v;B#QR@f#RlVfq*QW!!A6 z#~z-fXugxl#~(-{@!Tmb)+q?uS+ zOO(!m`8*F)JugGwQr8*Hgok@*59%Y>Y z#0CVWW79sDFH0`Qpc>9&kEjayHvxz9?vZpP*^JD{!ts`sbxzQ z^3>2-)%+E`^g;lJ7J12Yt#<QwWKyW)46gKIAN-e08k z6f@pRBv#CG#-Uux0@Eoxue)i;VHngDt?`&wKa!5W(3sv63`x>;$YaM#8^}Wz4hf*! zw-gBt{tm~0z}}huwQIMoZo@RUVVp*!z5W7p7hqYWGO0U|?jzPB#s~qw>IX4Y0^%Xb z6jb5^ypH)R@Ocd2r_yFasv6>HaPZW zJ#jHd@RyJIs`H2=fmNFbJyY-QoIFA}PrGQ0-azEmM&T@+{^D0shO5v*mrQn=>avlt}K_{ywMu_O}C!MNg2 z$6>9E5d|nBUn(H(=G4B&P4*>a{kUJsaiTFv^3z};-*_IMiS_GZHO#N6!#%faMfX=r z?QD@-z8*Sb(35)Jei`s~x>UryF|a$}5F9y-JlgXv2Z|U#EEVqlY4aX&VMU6R0G!5$ z$a3b#(M92C5jasNiloi6?J<-F)48>uVB`$OIcNh>^G5nnXdKwyG~j9{EZlPIY}vdY zqLUH)+5Kbjz?6zXk_xr(SX>ReuKay{IcD?6U-FR`0h{+`hH^)9#aQc;wTt#6Fz*cB zBO>Wb{|bxK7+cbILt~Mo-E`S5-(rKHAg^E0&z)E3k(nTgV#p(~w9(9?T}y5cBb~^( z0O7C6fI&=FR|m|I>fF}_XcfSKgIk{zcC`I~jmAfS_hJ3&qDjk|;WCTvHrFA8lk~Nl z`P`kU>xR~cs9tHBZ@t5MyWF-1!k<+Pg|=?=5YeZ*h==SK+nSR$I=oVWjO`b4o&Gg@ zrKk4`&L?s6RH#1@3IN-#LtET@?T-1Eh$%zogi=I0D99X%h&G-_g}-xVLdNgc0%U42 zQ6s1`;LhU1M%{qM_iXfr;9QJ@R1ZbcJTG&`)JNmj$I>Yw$&>a{C1nn09`T>3EnT#_ zzIRCaN*UU<1tcSw5)hLmvhYG*z?~Hge_|bNCg@*KR(Lus7)TT&Oa|^dVLud(YCmUZ z)M=!RJ|K~V>r*LQuUY_o5;fZgO$#|;t#fDfcIDs)(!sQSC6YU?pNdxyc0XDUP^;#e z3N^qtUNb)TudpX7y;0`@b?{@Ylgj_>S$Q{tY(>5L?EHPp;9;n86hQe&E-SWF7fTBURE>Y%cN$hneg4CPN*61tx=`bRsOH6ChAgrV^l}=(|np#nz}N0 JQWeBI{{yLoBU=Cf diff --git a/summary_results.json b/summary_results.json index 758ce83c4..8ccb3e0f1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 16.2816, + "Performance_Result": 21.4679, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73249, + "Performance_Result": 1.74536, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1165, + "Performance_Result": 22.8585, "Performance_Units": "Samples/s" }, { @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.1213, + "Performance_Result": 21.0092, "Performance_Units": "Samples/s" } ] \ No newline at end of file From a2d2e6f68aa3aa68d2231b8fdd34d8f32b3b8cbd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 24 May 2025 17:56:58 +0000 Subject: [PATCH 649/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10943 -> 10944 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 5eb4c6cfd..5b87a7fcf 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 17:55:12,014 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 17:55:12,014 submission_checker1.py:3308 INFO] --- -[2025-05-24 17:55:12,014 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 17:56:53,294 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 17:56:53,294 submission_checker1.py:3308 INFO] --- +[2025-05-24 17:56:53,294 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 17:55:12,014 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 17:56:53,294 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 17:55:12,014 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 17:56:53,294 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 17:55:12,014 submission_checker1.py:3324 INFO] --- -[2025-05-24 17:55:12,014 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 17:55:12,014 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 17:55:12,014 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 17:55:12,014 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 17:55:12,014 submission_checker1.py:3345 INFO] --- -[2025-05-24 17:55:12,014 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 17:55:13,102 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 17:56:53,294 submission_checker1.py:3324 INFO] --- +[2025-05-24 17:56:53,295 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 17:56:53,295 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 17:56:53,295 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 17:56:53,295 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 17:56:53,295 submission_checker1.py:3345 INFO] --- +[2025-05-24 17:56:53,295 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 17:56:54,369 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 4b809f8d5..36a684637 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1276 +1277 diff --git a/summary.xlsx b/summary.xlsx index 25b42dc7249ff12c7f13883f80bb35700a4ba972..123cf40cd3b6b0fc588c52e2bb3ca3c909c80a62 100644 GIT binary patch delta 326 zcmV-M0lEIaRlrrS>m&u5ujFPXlkg-ge+5}dDgo+=N=Ot@akK1gY7zg)Hl%rboU~0_ zsV9!>nf-Ql<#Mr?ZS)J)39TxWB`JzPG0lZ)D)g~l#*Zj+o+?gTt)N0ZIJBtmN@fVt z7T&BjV7&kr1(0%tnF{T^Hw0tHb|9%sf`C3woIf2Vi= z#%3a-Ba*WTX?|=w0yx9af&}Fq&XNqDy&kM|HycLf!YPIC4cxSz#L2yPVlsAJmvpny z8uDj&vwD3WSB-@lPQ?J#C1-?LpkCX$#Fs@lTuYr_g%fWD@TWfLH{Onm9QuGqApkkV zJt?2F=a=;o)p?p1aazRr{W^Of7{#0vv#(*~RsR_%k3;@VJOK{5uRlTk2eUCI3j_t4 YujFPXlR+v}0Wy=8DkBEuCjbBd02DQpo&W#< delta 325 zcmV-L0lNOcRlilR>m&u8zFt-(lkg-ge}Jqcl>qfbB_x7C+$?+BTEsuH4QbyVH%-!3 z%8BE8X1|?Xxm@mL8~uWHLaPd8Ns1y+Omm^O6?$)0@dJvSr;5{7E2z)`99q^lB{PI+ z3$IoiuwHmf$I;f1cta zFg6nr4M@%=r1`OJ58w<#3lfxfI7>2o_Ij|=%{KJPg;NTDFfeN!#mT*QVlsAJmvoEX z8uDlObN%wxuNn(=JQV{}mz)u1fqHH05?>bKa4mIy9ZtLvz#k7mpLrV=IrRaLLI84# zJ1QTGr{`vc>O9ShI4$D*w#n{E7g3Nb|2mGm>OTYJFy!CF5pc?V{R!$nvoa Date: Sat, 24 May 2025 20:22:38 +0000 Subject: [PATCH 650/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10944 -> 10940 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5b87a7fcf..490000996 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 17:56:53,294 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 17:56:53,294 submission_checker1.py:3308 INFO] --- -[2025-05-24 17:56:53,294 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-24 20:22:32,658 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-24 20:22:32,658 submission_checker1.py:3308 INFO] --- +[2025-05-24 20:22:32,658 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 17:56:53,294 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-24 20:22:32,658 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 17:56:53,294 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-24 20:22:32,658 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 17:56:53,294 submission_checker1.py:3324 INFO] --- -[2025-05-24 17:56:53,295 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 17:56:53,295 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 17:56:53,295 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 17:56:53,295 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 17:56:53,295 submission_checker1.py:3345 INFO] --- -[2025-05-24 17:56:53,295 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 17:56:54,369 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-24 20:22:32,658 submission_checker1.py:3324 INFO] --- +[2025-05-24 20:22:32,658 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-24 20:22:32,658 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-24 20:22:32,658 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-24 20:22:32,658 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-24 20:22:32,658 submission_checker1.py:3345 INFO] --- +[2025-05-24 20:22:32,658 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-24 20:22:33,654 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13835 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74536 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70802 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4679 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421985 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 8.06579 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.76955 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8585 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 36a684637..87e36611a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1277 +1278 diff --git a/summary.csv b/summary.csv index 5b1efa2de..92947a30b 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13835,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74536,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70802,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4679,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421985,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,8.06579,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.76955,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8585,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 660cc26bf..81c672d63 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74536, + "Result":1.70802, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":8.06579, + "Result":7.76955, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 123cf40cd3b6b0fc588c52e2bb3ca3c909c80a62..4f4518e705416ded5cd0468cf19727f72acaa59b 100644 GIT binary patch delta 3840 zcmZ{ndpOhm8^^afMYLI-gxIh-HOkDH=rP$aIaCg%imPO)C?64CNu` zoRdmv_9#XU5k-rloFa3W-#owH^XKn#UEl8?_jTR(_jSMT@B6;4Psho&lht9nM8;KA zWg}>jy9|KdJ6#d#p$sTy?~ns0I@3%zxkmasd4UHchraff3)@L{?w69zay4 zmC_m-FpcKlGyQwQp*PaC;yamrQLrZ|Vr(+bZYUp9M6~EBEgZ|3CN6&tFnSZh9_n|) zGpD}jg-5Noejqm-&p71+?6)bTJ|DUjr-_(a%iPQ}d@)se+h#N?zM1jrWs&a9V_RGl z@Z_e&lZ;n}jpT@tdu_h3s%&qTRlTM2a>47I^*8I(hqZfHV+@Ii-qP zPDptB(CSeg8~ge+z|LE4erzap=ah3E`%%)7hD#avwhymk;fs-~2m)&OSOopzGWUJ&RXF=)^4;lo z%+$zCwQ&o^nwRZdMg0rc_g7=IPomqIEyMeN`H4bR4SEC1P#cQ2ApOc0~C z!loPeC~|7>aWE0L0)To<2Hm*&Lp_}hZ*>$$L)mZYHnscxB_>j@n6cC@#JSd{%K6fK zDI_Y`fVI)~x-{6E_LCAMA2SnjDbCBKA^D2TxmrzjOoe14N%B(zd7C+W;P!%+asd1JG6m>w3bl#k|Gtf;6qDod1u^nnIWD2RPQ%x_JLb8H> z_j3dcIXTg2ATr-IiN&(_uH7IJsX2pVjIr<@a;S%N(yL;mG;|3}x znL=zWm)U`gDIfFd)Wnp7XarTfm}`Z7eL>{y4L=G=>?7 zXgGf{yHTs8ofJ%KvZxLo#H_}l^VW$r@6GiSbWS?)SP4I_oTLkDKur$V1&3nvzo9DY zDFn5weJ~b?j!~`gl}kmlGwUDgUj5US?936(@a9j>hjYhCu3{sb!sUr{evH;dchB2U{MH3W7_u1y3LU))ED zxVs6M{ypNc&rO_Nv_&%~ujbdDo8`0!+YJ!Tkhin!k>3&2qnLxk78oHnY`JJPZp0!; zrh|jp4!RWA0so9JA?>`KMb+H~_3wG@-o19mySeI1=D0p!G7>*U&p-biuG5DME#X;V zui^lLXmf=ZzgqG#5nIs7LGi2ifb0O;FXpq2tMn|%L?u(fZjp9zjp;|jyAGa*$(1&W-^(6pa4Nsb0b8n^1BeIlf!fdnk6apUk7{ z|CFo5K6?mu02|pNSloWO~ZK~A{tsW6u`(q41({I zu?e{S7QOIV_Vs|oMXT=1Lju=Vy4Ni($M=r8aHnl$Zx_Z5{Y-p5@o2=au9MdnkQv4K zr`p#W$yq%T96_}_`;n_d7f2kag2+dlv4#9Sgutl)u0(6MpZTfl8)~{qUK17W>%17F z!m#2GE-TZ{vhNCmChF^5SC@iPc}yqe#bUw0x~bow;>vly4It=XgWM*eAjSZ~K%$zA z1#ou-gd8#?&x@ah9WyGwYteFHd3vL|xr(1j=VvaBQ6J<*q;C5#36n&Dm2%J`NH_p7 zazFN*g@rp9C{y|k#4Uql#2=I;RZ2l%^~phuk2O~AnRM1ITC&$v{V&!wuNA-QvCrLa zsywmogAnOe(vD&lcwLb5y#WhBCwQXjxT6P$uLOlGqhb4-MDFbK2 zoZXKKdCT3W`ciCZBAq56Ez-W;)mLI#&mi7+q|72TX0x%KwP_+J+&bEQc&O|r^p%s( z;V5L)Pa&5yc-9i48il~CppjH3y1@CnYAU?IHIiPeO?ji&00W0Kkho0dP=b0X_tC!n zIf$6hJ$~jMqGBTw0KaSe4=D{rF5xeAp2g#eM;zYcneBjsrcb|7W&OT8h*JZoQJn3a zp9Y+g3}Z<^1gnW^{|JmCiNX{_vT$;i2KxnV*OI%TxyCyyR0wr&zvXIyf~5i~NnN?Q zY7)qNqRM|r_Y+w%w0nOoGdk~Bdfp6 z<5{2f{axvy>we}jd}2JCxa}(s1wd=NFP<*IsmNeVx9}FAE)eO;14d zoNAOuZzqywKL?CIO?|VGp0|i0+ZQ=CIF@tu71CF1<{z~?O!2-Qd@!=M{{h+T$@UL} zZ&0Q+-*a_2M5gQOmLU&4l92p~bjCea(+4wI-X zm(23qt%wLG{fROip2&1bpT1V@0rdT5Y3D}$bvJE}P0m_Jy;pvpBhx$zcj<>#h^FD5 z@q0BN6NL-$S*tmt^1)WsGCpq`@85k}WMvcTbI)6uUilKBFJ2mis73QBzX_|kc}0Or zBUFl1ofpK(z!^h`a98hDh@zQJEu`=X_}-GC!ZGgiGT-6TO=R;_TI@8}cW&dzEW1yB zS$6FrZ~aIm!`A#>noywDGtJ13#8A?;L+MdX8NBM7>f;31@r{QQQKmX!GeX&&pr5c` z#c}kvhssq&0tpTm$Vcp0TWOzjU>@|@^Pf*{%RS7n;*-q4dzz~(jxN{fs<9#eyQzXT zPlI7JPfas0gT~N23cf)b(=-9|XtG*r3jc^I5ZX~7w6s9_e-GvAdzzcp5wH&Jj+UzE JNA;cfe*jjaI-~#q delta 3867 zcmYM1c{o&!-^OPwgGv}ZgtE?zZ(@fHE|?O? zj|Q4`>(9schu+9)2q5=_Ov}l7m4qwtD{gJ1gIw67&j%hUCXz;?K02)2#PCo9BRXg1 zsP8F$Q|ftVj9KdqWwurf5%V3^4OG+LKFz?!nk)$|=HaV7mq_sl9o>bVp}o%8U7fQ; z9=P~PwR_ho`W?Dv{CulbXpcvR z>5adNv}owhx~F=|*He$h7KZK0&!16bqwE8b$?Z+3?ET3AlpV@E2~xOJAF7)etfS+j zXBk`idc=(J;7+aNPMgwxty1XNJL;MSZvWluV9j$rw!yW6L)3DQ>s?Y&L4$TlK!4|S z|6-8;-E4PicQ<~U6;!kRTtmX`vxSxvWqfw3WXvm6Qt-n_ zm^&t-GN$pl>Dr;i$3J@3SeQmT7bhdeb!kD#0>P!U?LlH0-g>+1&CXSwhd@3hu3^F) zXL!9@93~&Wmu{Fd_6`pf(P@$wY3=ufiX=bLg2}5th&5#H1wIr}tMoCHpIpe}JoM5Y zDq_8%79Uq0V!*kyZ!9kjv;^;hXASr&LtC7f;^MzfDAe1m9)*f5aEr4N|M@r3&^)a4bUUMnWHyx(<{aS^cUvzcsp4|F=A9ecek0qvi1?-*}Gu@p@gy4d5mtYTqk3#k4ekw$p=l zy7V}@Y%vUZI?0EI(c<$PmP!=w&QBV~(wz>h+R1bm&m4mH&)4K&`)9Mo1F?@Y3a_1P z$T^>XcUB>Avh~*~g^0|z%Esa6p{b*(annhL%)-@l--p!Vu>G_d*$@b0W#J1#;m$8OVP|`_gpndCcv!_iXJ6F9?yi}Fw-XsQwf zasv0lum5Ulzn;YX40o2j0z2#6ABHa}enVG}1OTB&_f(6y9?3-JdiMp?uBM$b z!S*=EsUCTh=K>QJqFX)&CBWo~FcFr#aT;8jR&{~7HjeQ@n@$7xcJ=H3%XsQ$VKh=NOILz5@94MO= z($6K82ebl(4b*&!sjPHcDU92Y{@9T02BW|VrxTQ_>g9~#%LdLK3r`tMen%0thW;mN zDEU|-HH~F0GeNJc+kD}F#ik|f%T~4K4C(k}P=+pSHeqwytS-{)?(QvpPofj=g$Zv& zy)nv;HoFZ!BdW5$y`vdZ_54*OkEmvZLMyCl1mMnuao->x3y+r=ReLUX1Thb8MXmQ< zS<`)}Ns6CSYUZ)bxJQ&&qQQ7Z+?~KX{O(TgyI3Es@Vek`?ExFNAETzG-VP6ac$=-) zye!y(PxF*RUD|)bXqf`n;}Pz{1Tq+fgY+_w8CYib!gOV652dw7q-1*2)I?76_NKW3 z_7;#c)$~eld_ma(H)){etxDAkA4L6wfcfD{>wvs(zFB&fn`T9ec(jIS5Sh>=n+@ak zA|I>9jn5jPy+yN&_+0yYu{T4exU=c-f=J(8W<&5BmWoH_oagafn z#0^Rrs-?u!nUaNWDR~;(^Xn}i+6`}LA^^{(S9EsV9$^amWUm02$K9o;JQi(SioN^Q ze6oEykB%Wx_}$Yn&T)#o(qw_6RxC2koi{et4McEfgJ_7v*q`AfnyECeiqN4Y^hLd- zEgp^_<|UtZYsciR#>_NJ>fN$t_dRt>FS~cEon0e6XlTHskcC*$AJ*g~Tp5wGZO9fx zs%;8^ve*XP*GCgxCXnQJ~m?}M!Ase9n48o{#laL#b@j=uJOf8>&f6;-G@@$R)-q0A_ z&iC)jiX#C8Brtl`^H->}oaIT3qtLg!ml*3?*1*2s^$9jUtQ#8xzglp061NxYdSU(` z<(LJr733`Oge-y9FUD~d=z`J_f)^wBc+^Csc*E+z9WRTH)xhi(GIrd#)1Bur0-d^b z5tUaP5{ggrJZ-D_DAsOKXf}yE8s~b!1Dty71`7$rXQbMYACq4z{=Af}DvN%!YcFES z2b>m!3Ul6^w)Q-}=V=ALb6O)K#!G{B)SkgxyPKJ7L6uVHrOc1H-b?8=uU7GzuPUWy ze7g2`>p}sz$hLWE7;dg!rxF1rK+PrfVWO9~<6MP1jzEMvL1|qTeb7r3Fw+>UMBqw+ zfcb6ZwF%1I7l_n!xwJocWbGNC+BRT%!l`F~-g3!CY&m}kclS%io-K6%Zc|`9iQD0f z!hU}coqF-3#(~n^nAulSP z>{TDTjYGB+v$ma1gQ$pklc;rWH4zCOd!o`J9jWsMs>yFwa~6+D_S+)34%vzk=V5lW zVQZ8M%gy2nz#cgUMP89+7p-%MX&fS}LY6~J!6EfaSX&%o7UG*p)Y>62Iz^yENVzRa zb?24d?UZN4tt~y*m9igqixYax_aiIBU&2=%ZdLIR))a&POY8ehdxArT|1XX9Um6eL zP#=ICMa7HrWV?5Fy#HLV1-n+CVPrLoO6Wj?2@jUgqH25Jf;`@z+T{F zarQ8aoUR~r2rYtoQ4$!*$1pOG(s= zKcgl~zLN1V{33r@#S?JNvCAC06tlRTPjhPGD2bDe^c&!sPKfmRw0k%SXW|SJ>cgUKtg6FpAl0s6f~Q;wlEPBF z4JZ$NG)cb~TI10n)O0SEuC5jpn$T;G+6UGaM74(0g{(9q(_37$Z(jYCkXeu5nu#JiXenjJOHP8uwQaTiz0H(gfaF#oy{V;qyOD-5ZJc8$RDCYYt{phnU6y7a&S83*-tKZQMHLUJ~HmKY%P zEx{WN$p>N=g*rUDW(r;PF1@ZWd5@s`)%l6Ur2dXzia(=hc{Xe z;XyN9dHpIeBq}HmpzbwFA#nE(2w}K-8K!-l2#_Z7$a%+2J{Jj zGn)9sVdN!~YIN>8KG!iu)R*^rsotE{cv8Zt?Qj!0(BDj5<$T60^(PGaDVQeiXUp8$ z(+-h+lXHSm Date: Sun, 25 May 2025 00:17:52 +0000 Subject: [PATCH 651/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10940 -> 10942 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 490000996..24c540bfa 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-24 20:22:32,658 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-24 20:22:32,658 submission_checker1.py:3308 INFO] --- -[2025-05-24 20:22:32,658 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-25 00:17:47,487 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-25 00:17:47,487 submission_checker1.py:3308 INFO] --- +[2025-05-25 00:17:47,487 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-24 20:22:32,658 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-25 00:17:47,487 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-24 20:22:32,658 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-25 00:17:47,487 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-24 20:22:32,658 submission_checker1.py:3324 INFO] --- -[2025-05-24 20:22:32,658 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-24 20:22:32,658 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-24 20:22:32,658 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-24 20:22:32,658 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-24 20:22:32,658 submission_checker1.py:3345 INFO] --- -[2025-05-24 20:22:32,658 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-24 20:22:33,654 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-25 00:17:47,487 submission_checker1.py:3324 INFO] --- +[2025-05-25 00:17:47,487 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-25 00:17:47,487 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-25 00:17:47,487 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-25 00:17:47,487 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-25 00:17:47,487 submission_checker1.py:3345 INFO] --- +[2025-05-25 00:17:47,487 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-25 00:17:48,458 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421985 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.76955 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.97851 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8585 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 87e36611a..5e9ad0bfb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1278 +1279 diff --git a/summary.csv b/summary.csv index 92947a30b..cc4bf03d4 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421985,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.76955,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.97851,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8585,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 81c672d63..eca779413 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.76955, + "Result":6.97851, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 4f4518e705416ded5cd0468cf19727f72acaa59b..81f50675b72e7d037784490ee409b83a99b52ae7 100644 GIT binary patch delta 2526 zcmYM0dpy(oAICStC^D22PHxSZ+lgkKT-ruqG~|}Kglw9cq=Xagi;A3cT-un^mirKP zin&Imiz7BN9OM$Y

pZVy?f@7pwqeQhjulfCm{fvr^_i?p%aMx!Q_Up zvoiIBHO;?DuJxqUrI~vTp%*6VRBNmR$X~|o7nfJ^ka2S(i@K~S`52eN6xHJl5{N(t zqfBju^M458)>^D2jmBYhO<|_7_5mqjCfV98X&5o9R^Ry=0CigQdxs9s&tGKI>|@Ep zAt_mfu7@uH677hPxBy;?Dv3dg7sJqZ9U)W;1p00$gsOJ!pNrvff$nuTBQvo)@wT~g z^#r-`y4m%YZ{qlOz8mID)Xmn6+p?0Ygn~&!AxwyqlZc9L!NPwfkg88oS}#~E);zV7EwH9 zYnD(G=O8`EeJ83EA0S7C)H}6WsgXfod@SPpenr|?Tm1z8-i&L15xFu(Er%+Er{|fEnz{Nt?@}^z(nhDc@2&?TCRZJIp1MV4b^06#xN<)$t_Z2%7Lw zVF4^e&0-w+N6!hB=%=17CX{PbB>FQtG^0CN)_L}Adt18(Y#)_=?8+L-0xZ%779DAw zwoaNT~14u3ankSM?y#S8J!Gg8Xc^HtLg$S zAF|LyEOoD_NvC8_qZ(X`B0Gqh4p+{xC8jgmgN=3%o~8fL7_yrpGPNu=IpVhq(c3ip zL5rOi=qc<8a3LF4R9sH%I{|U1wGz}v%4U{JI->A^UnuC#NiBITI!@nkKh_B1YF2IM zq)(8BsY~UfWU*?J;$>!0#i-q6Yp*0%FIaqJr-X1114usVE08b@BGv#rz0RlZ-!Az3 zWCy0Gr5G+~(q%atwr01h>2KBPL=RxP30}5DtITZ+m_01zxnSZ1wdnx zGBpUi;FPm4gLTHI3P0KE3|86@9WTHf=%$P3~yG8WjKX@kgdr47Z? z=eMI@NG4?nb`2>AK-q8z;K$;Md6z=zG?c@4Ue+z1hT%(hBhwr2_WD+T{jxuM>^SpJ z@|`Sa-ibLqOncO>=FD2B5_cN}|KNj6n%J;MYl@7RM>todV6kQB3@FJtx z^$eJ}jH2x_iU$T`>7@k-MQBZzo`m`UHc^8x!XcW2>PHsm!if0=H0`MZE4F;yZ?J7EG8X57<6fuahmAmzOC+_ty8(a zOOd@MivV>m-b&oQ-+3I+aT}1Hz~N`2-HUdF;1rh~3)T z^zPN+A!iSOSP1zxX?YnR8#9@hdDp&R&&$K4Xc;Q|+b9t`=0$o)zU~=yvPmCIwk(;Z z_}h_Fq)0_B6ZqY?NtwFlT1qwsh7@txJm`R&YbY-ieY^S+qdMQxa~Q5a!ek2ae=%Gd^7fCacZdXQagh8{G|n zK#2_L7CW#BK`ldG4OVlL%38Ah>2(3uUhq!Kcf4!sP3!}3anX0kcZ8SloeRyZ; z#*?HVZKGEplpCbaaQL$4dR26bW*?7>-3-AueKrm6r9>#_*fH=u`&$O5eNJEW9mu;L zUA70q#UMUv=K`rebR|u$hYv05%_-D6y{cy0EzPV>E{(kG9jncXM(_7G(K~f;!z?|G z-97V5l*06HVd$hh`#A;vg^nL4V@#c!x%OR{vhbs`c_$~w(<|`C=O!`P-RT%P+9d^G ze*DRm=zOBwqr7m`C>C6?m`<%vsDG%e72i_!>fa8UWz%Kz z;WD&HYNoi`oPRh`NjM@?cEo(u`*knUF)q-jvY&FlTf>TF9=)$VQym)q+WQhN(M880 zmLHZ{r|tb~)r_xe$GPr;q6V~c70jvrIlx>DuLJwRmg4tGBD@0|pe0f8!h>#~6+;0( zOOWI%VWPwVtJl>(<~;%5TRNgHJ^iAL^v1U#)HM<_M z@onC(kn;x5kH(EPyb3=?Mr|B0R!oaXlqW1@92c;+Uw!i!j D{aK0y delta 2508 zcmZ{meLT}^AIG997fY?Rwm z9AZWuN{%{C%#NpVu!u?yrHCT)u-#MlegAh~e_XHE_0Q+^{$8K!^LbrOM;ng{W7cbo z>@eFBSN+rajT-B}mq^xIJBYB{Oq~;eHbr66xBKU_@9ir*HtlXQ>uJ%>C*fEaF^GTL zlpeW>U%1Yrx7+=vi(_hh5xTGd#21x&w!hQ}=zkDJ!p{Ss4#)n$al3(zmf9|x?L*O$ z&gvEG-XEZ&X|Y5xCgRdDQJMjbOJh+uV1jtLv8^~Nl>3>*G)$O0bM{h*KTw+zt9Mdl zDM={TuA^wbs%0n(29F$a?CGz1<~>v#JWNRD{osK9ZpA`jtW;s}r~HDWz~`~TrQH-rivtCh)NdDK?Jol%0^7g4=4^h)NCRh6G!q^1uNkhNT&Arowq znH+5)3D=DAXRs)(EtuU=EQ&kq#fM*E%yu3-1K~vor)J)KRhwoyOvnYc3Jy6)zY+SY z&O9Uga5`^sFZ56NF(?r=Movzp!8@S3FbRnN*K07IVhEc8_u#XM&?Cllt+y5|%pkE) zHLld|3?MiUbsF8!aoO%tU&ZSJH3`;#sxMJKY z*b|i=)d7iGe2eS#eoLm=u0bxSWO~5I#6V4;n>MY}SjC6|=rC3>N$Km?>5$B0Ux#GC zj?4#ce$S^V>QDE-mlO9D?@DYkZg)BlRP^hP&k9Gg3BjJ5a|q6l&9oeOG{mok;c zZii~ZC7_#s3Bgzj0{#J9f@iowiO4m~k7YWC`_00yqTd=;4ev23(Id~00;hpeo!FwJ zV5_K6nBz8bRJ4`dJ5)st%L~KShl@cxE3+z^K^j_YA;0KXStEvo7PO6SwmskdHxled zZY#C~1e4p^6KW0`hf&+~xsBa+$6Y0vGRyM~ul#mpk5G=SldY(iCGivq*wfVDo14mQ z3|)U4r#2Ol^Nyeo=aXK018P@k5C-Um_CD$pP^bZmlWiBYkAY(iT7Zg3ASn{$M)taA zSwyv%s=}p+l0b#0^;vPZSN8rT3(^nNlH{*)ofr$bqLWk7hw9`$uM-R<`yldz3CGQ9 z=b(#~S|F}WQ;3(jF!h?HC?)gkrDnt%6GzJ0?x$%U*l!2qglyQbcq_D_@}JBRJHSzP zd5o8P>I1^22N_)?cgJ19174a92J(YiI6S?{;3vOZE}{fU5XmG$)~ zgO^f(wkp(Rk-47>+?KxTRaj-s2O8W7O*2rxH^?u#+giVZ5izaP%VRm#K`wGqt^ZQ|nHArD=wZkm`NVZyj z^XTFcDQ>V)$l8S#E{Q-g5QL~4pEjGb#? zM5)pFStkA9JO(ZiztyE1Ap#`7(357}+s_TCilTyAJ^Sf$_ptx?g!h)4cP|ZmPI@|e zUlv~7BJZJR#!J7-y-iJFcR-~oIl)2Mm4OI~6lSFH67S$*m}8;}H@*nFA>1k8mt~A? za>!^obxEFJe>bM^h5!77-RP^;eiR+ z^=eFhp9ujfWZ(e&Z55)F0TF~K*tlW)vfC~XBR)+m3mYmGnLI`2+%V^!ATCWAR^auT zq)J~FS_6rI*i#QY72^;-1XEToLCck?r**G1xk3j5uSsF@UskzOGkM})xhWxIHDAO^ zIq=A1AxNJZK30=%c(}HArf%R`MAtNq1U5H? z$x$~R^X88}MWB$CpVj=YA=q;!LX?Te4m6VE%Tsm5`l|8}aFuq5c*W4ZjgLnXpywD; z7PFg`ho<#rnIuH(hC2mmu9HClMeE3qIu_friCYAJ^dw+k&*%JQ2 zM~g$}#7mWHuLgWI1ulj{RBgsNcpr?SXu^#&*mwh1!dBJTB1Stb+hMJB)lfUPx-Q%? zay3FFo0~RNzRrAXrYPZsYs~E-KmR%Ng5W^9U=q(i$a#kBv>A99GxSjL{t@~7;;B{J zXxJ>X5r`*jn*e&7T&O@vu;ZNpceXv2vR54py*V5xnr}j9%;8)je{wEHUMk_+<4){Xt!g8#28XG}{C-R{bF_lyq92{q+a{0Z=8&-Ozw7Zj|`>X^Wgcu;ZNj(jt}k6H}i#2$!xYxfn% zcA||4a`!PfXKs(tf$^uMw87)`45u{i#ffp+`(@`TNsr;DEsLk+OU@O1Z>OAVYLVC{ zmOnKoMN@wME#Gv|KV8zt<*aQtd_y=rjCF;U!+zcV#GzNH3oVhBH~b zdhJaXJcu`S>djH5K}m+Yf Date: Sun, 25 May 2025 07:10:18 +0000 Subject: [PATCH 652/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10942 -> 10942 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 24c540bfa..bbf23f100 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-25 00:17:47,487 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-25 00:17:47,487 submission_checker1.py:3308 INFO] --- -[2025-05-25 00:17:47,487 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-25 07:10:12,708 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-25 07:10:12,708 submission_checker1.py:3308 INFO] --- +[2025-05-25 07:10:12,708 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-25 00:17:47,487 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-25 07:10:12,708 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-25 00:17:47,487 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-25 07:10:12,708 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-25 00:17:47,487 submission_checker1.py:3324 INFO] --- -[2025-05-25 00:17:47,487 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-25 00:17:47,487 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-25 00:17:47,487 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-25 00:17:47,487 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-25 00:17:47,487 submission_checker1.py:3345 INFO] --- -[2025-05-25 00:17:47,487 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-25 00:17:48,458 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-25 07:10:12,708 submission_checker1.py:3324 INFO] --- +[2025-05-25 07:10:12,708 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-25 07:10:12,708 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-25 07:10:12,708 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-25 07:10:12,708 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-25 07:10:12,708 submission_checker1.py:3345 INFO] --- +[2025-05-25 07:10:12,708 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-25 07:10:13,813 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421985 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422701 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.97851 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5e9ad0bfb..003638ac6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1279 +1280 diff --git a/summary.csv b/summary.csv index cc4bf03d4..dc906f0df 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421985,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422701,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.97851,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index eca779413..7e6b73f75 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421985, + "Result":0.422701, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 81f50675b72e7d037784490ee409b83a99b52ae7..1527791e0ff2202da8fc782152eb8e6153653858 100644 GIT binary patch delta 3006 zcmYM0eLT~9|HrqMyHePpQc7hvZbr3OCECczX6`vrJ@oD{~^_Tifnrf)6 zRf1T4jOIlMFv!=JKOcV^*Hao>Z{MA26ZzR_V76aScY@#~VCx-O zE9lB<|1@73-fTG?RPX!PC#BPt>z&fMa3-ZQclc5xAdApX%B{>Z&D3}qmAyFk*YA3` zt4^>?*j_J~9;@;q?kb0fsbrYeDY#!B1QbLao+Gdj^{mw>k_m|SviRMMi9 z7jR*ub83#mbVSpMe!AL?8BU7uECAGenUC(i8Wh|$gWoTl{tiM2LH~k5f`Zj!dg&QV zLOgE&Ci}~UiFu>i{*Jbp$aW6J{-k;fLaV)r3c%I}j=$_jSQB(IYAVt{fA7+; zrV#PYU~38~kJG-ft}bk-AoRl;sOY!422t3(VivtD(6(uuOD|ShN>o7BWjmn?c#6j| zB(up36%Zp_e$rDz4`u`37_{y2li?Q=gM6COF6*D=nf1okX+Lz+9&MtoRlV``u~Tc= zUxhxZGj|84Ro8cH##H}!fZ3P)?0uVd6*yR19JWJ2LeW_oFRiI|bt-k_IaANLa+Nj! zR21|zAN>|Xc>RbzVD>F>pY_74FJ3K~d7DvpT<`iE&CQ!{Lu}Z4kCTW;4kz+?Ihd{rXFNSb<{v9*Hmis(Jm-MMb>nLjOv$cgvhd_pI$1;r+PVM%fXX_!{-je3ea z*L(}}vT#Fm!-FL3WCW_Qv!VYs`;mCv-s?Dj91P;e%N7(+U|d+$`x(MhF=Asz&R*b@ zpx@zC>iYGTdx~q6^vWpH={|H^Hf7Ig*Ez@6G_WN$?vI2M9sv1xaKRyEiOtw~XXWKcjZAp@aWH-(_!Di8+& z#;1Z**Ob<9LMrOIMY^I5>+HXrS4(4*&}HXoD9Xzu^5)8M zbjkw&uij7>v{tVjB&^%sCMK`ddx3~-CI9Ixn3xYt0@go1c*Bw)((4GOC2HhsrmVFH z8~<84MPA4`9et&d_{b?$P&lC@TrXDcF?`QsDmu_@aA7Qz$X@KQ1)<#D<*CnTsC?3l z|L8ejLK$h`-)sN%wM_yxLw0DZG>jB%a5Zpiqw&L=v|+2EeMjK|mDn+tPwC`!wJPU< zOC$%C^&OmIQ}!lXs2J75DHOQ9LoBDRkvM zQtCc>jTbGv?9$^DCnyxF6nZc1F@*7%-#XAZRWDsf$kTrVk&8ISn?yPf(#X1w2vcme zwhr&(%u<^}Q;1ByoDVeLdj6u&$`-5G$O|9gJV~hi8)_{qdw=!+aZdl>7%E)RKe*8U z;jAe?tcZ}n&gjyLY%H*6vnJTSQvWR^ssv?=9Somx_pBVBg36q$Y{v+)bTS8CY7BK$ zg%PU?qpC2d-pPJ|2BSu}*cYV45qqq5RBCnUm~{lQFXyy`JnMR~;1K0eSY{FEn>-gN z*|-6C7;^0yRPJ15KSq%Mz}-~gFcx?fj)taaoGG>;U}^dc$$MpJM6xQ}63!ne&b*)D z=VF02Gl?HEC7&Km6kUz>cvVIW^aXfH67d;8+RKMe$Sto|VmXOtdax{g<$!DB_LqLHKH3 z&R!N$l|C=7J_D~gLN?Sw_Tmpq(?IS+Fi?XRtfXf1V~i&5C6q^Hc5kNG`vZtW?oeiq z+nR(M+F^Wq{l<(y%$GM6s_NG5HQ(ZS z4!oQzV>^BKA2_8&%x63i>$sY^;6y-ML82~;!@!>)sk&m1fX7r5pc(M5VI#Z4#0=H& zZW16o2MCj!@8**{*?ELBA8KeswoodJ6JFSqUj{fPwKv|c_4Ars$YR}#huYd>sO+YNPuS!k--43BP9&$l+j(e!a*GDsQ(av=89m|@8i|Fmv?T@Xl` z5Duc}Yu`wF-S&pB{3h9+;jD+UK#8{_1THO$b(CEFFsc! zHazS4@-=kDAudM|Y_zXy>e~FI0y%GqTwzn~l#w>IGMZA3UYbpB*ZDlE16jCLwqxm5 zSER}5r6lXqe_!AzH)8Ixt8j5^2-}%`;a&Xd1FREL$GhCuG}M6c(}zp8Ir2OAOG3tf zEvcFPw%Rgu8=JjF5hH!k;$?JeVfg(u^0M9)Bmm~?ZF#d5g#0b5{=MDOv- zY>*%I{Of#QMb)DN?hU%chn)a-_F|J-fbOL{gY^i%phb}DC%hA~dc;J$`Y0NtnOmWz zW*b{_{zz*qPe3x)!d&|Av$I_r2MU=8JG0FI(+s$JonUD+S z?Pf7;%t*K2?`Oq6;wtL1k?-)j9!dSkckWq6%Q)BpkbRfEMHydy?Gw!GedUk%gqo;Uvqi5+AWpS*!QPglUb3%u|A0b-z-$yum&Ga-F%06vo2QFd%%gh1H aVh@_xf|uC_=7|5BD$d*h{FZ&vJoG=Z=c=m! delta 3006 zcmYM0dpy(oAICRyOJpux5QdrCbi%BY%h}vF%q1x!nQTHLDirO@DwiC$jmhLbItm%O zA%NkM`pcph558nuE|cOj6#`r*m31Dgr{O-~?4zBuC;C=-W+W$`L_?(Pv_JBAw2_6; zP`c&|t)bZVJgZeWgYD=lFfL&3Q5(K~T!(R}X?VkCGj|3SFIoTT<2!^-^rmjc-m7Et zmT#i{hMthBFWeSOzlf|9OaJ^xEM2{AIt-{0cPOd&pmSpa{B^%htJV2HaSymcnsypa z8E;?yLCpg`t>|EPX>XT;!+4_h5adRyea=6rg!S#3T@RPGto4WLa%J?XOR=7VfSi_^ zO|cWjGfTPiv__=KKu&J}?Q&A#cd4l8gLC&TuPwBU3&VY1tQcbBRFq+|zOB=enHRT8 zCQydylg(j!t!~5%4tK6im?SuxV&PplIMLA*Z|dBdx)P>#>S(_PoLQyua7?XZc zVT4(od3l7hB{{$!KXLA>N+xC%(gtl2NZPT92|LJR#+ zRx=eZdz!D^t-aQsXHGh!OeQTA3EeOj91=p8ElpFQRiVr9m$4{0>kCbXek(Qbv$c>Z zHR!s`mOV+L_>VE|y|$$IoS+Qvljq;KhOaO5*Qqy!k4;4u{N}ah#-7BZO(t9DBEdp9 zB%Cf=i+?-ofIh2K$Ofdy1AM#K_(@11SC?I}Mkf%5f+)>rPf0fs);0gGy4;`Em}!1& z6umgnsB+(`%iz*`r{&eP5`%>K*UP%BDY-cJ@-!8D1{p-4gHa}S{Dqr*xHXKGyghYI zey*{Wx&%L$Vtpie3^AwL)bkPm^;it~gpMsNL=;k;;sMH7a9UouhuJx?PDF4*05?sA z%pfO<#LxxKkZ=nGx*!x1u6p@HL}UWVtMS@}Tr5}AHb=IJAp5>??#HuN3A~(t35zBg z=kC8Q{d&Dr5pXKoH{qO=LQV)>%9brBK6xLgf@Kh`607i&fg(Bt!zqae*=7Uy;)N-> z?;G(CpP#LFE1QikdvGr~Yt%d80*OcoD;kYT;7Fl8p;47ncBV=96L69voOgm+(E!Zr?~%PQbGVk<&wDEk4KX9V zMVa$?Aw#@hY?Q7m9CyaRl>pp)3K`lX^H#nK#Oi*ObO=rOB)8k zJwki}I;B!dS3zSd@nYrHpge>eE=fvUgegI70P{qfQwR6g0hsH-ki1iK);qea8FxQZ^|{xJ_t#D*&iTC}TnJyeCm zLUxRxrX!V#?1)*+&LHH@kyG@&4}*7x2uv(14;(rngBWO;`v@y6A$jxr1KcUbH8s(+ zK?ew~!Kw>ry09bnj<_=lexR8683wF9dvE-EvJ zszmLiSRYIFI0j9;&?Cm*#Q;(c`*n#K1rqNAT)m!FukUC4ee+3as%b`xn)Ds)4Lj3R zio|*f1>_Pu2x21EQ!>PSSW_j78S*}~A9lBa9orA@op{&7WTo&x@YWIJ>fn#QmKk@w zmdMv!GmigLT~+v96o5ly&&YSe@^^P@GQkJ%&lx_fcFiF$DRhO3io=L}vf&4%a5W4d zSw$D+G+OSCtlJ>l9(0`SQmjc`c=*RgvoU2oW`sIo@Lq94;dq$HdfZ|PIUDGb85LQd z4~Lv!aYTCOLg`c#?q9vU>s$?^XI@B?tM8Qk>c4*3mp^XL{F9QC=f-uI*TZzi$h74) zxKw%BB6vD?bEzW2ei328ZnlWf6ffCQ@hNX6K z&5bbQDcykeMe5xPeS*$_?2C}wQ*T@rnWgwmc-H^qSU=@ZcuD?GIqd)CN)3KmQHWTk zkp?(Kfxs74!sZQeJDdxH`viS4F0nL4b@zOIjOw6*yCDW%ft>r14Hbn^upLIx!CY{?X#kQQmMmCg7v=IiXkQfG*OQRgTFt7r23o+|-^@}LnMO8;BWtl&)Tuh=b`x_ShNsc{7<5YtskPj%_h|@V(aYCyVZz?oKgp|5ON)o za?-xGrqXc>9)m&N(PQLT>F~nuZ;9CPW8{03E8a1#E&5=JWz}?uzau40f?VM~f!~=* z&eb)CDFU{JMj@iK9i_{9gmM#oq_CBpVxw=@u)03j1#Z%dUE8PI=U82$%+6P_$Pzer znA_^?Lld9MqmrCG#g1%3m{V7AY;k*29(@mVhZb`;62=503j!cgM22L$BlrM8HCs*< zdjDEDYsK!Dmt`EMu6HoM_q|)M;_rehD}ep@Z}6?#%m#0Zwmln~XDPGC{a4C4Zwzb{ z35xE*pEiSXpxQ(m(KW-;tIZSHouE?{AUXolAqf$+jV1`R%zj?oM6v{=c0mxYm~o|6amHDfkj^D*X`Y9sxh zD@HyZtC$L?4K3Q=gIPb+cL|6xE7r}xqj-D58vY6ns@weVXmB{xD&WdAN=RBICg|%UR(~Nlo{ZV2SP3FTQnCEnA|^$Ewi+iP_3NbDkMZ zk^fq{`jENM=jDKba{|e?b|~bJJ`F3DdF<|{Ts5i4mpwk>LSssY~0Lf_weTA;F6Woden7ZfxQD*CV9jS770KYbq z^_40d%=+si1~M4c(Uzw7SZnp0UdSe>{vd)U_U|X0v8C?^R$%xUSb+-}Hw|pSsf Date: Sun, 25 May 2025 07:18:47 +0000 Subject: [PATCH 653/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10942 -> 10943 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index bbf23f100..afd7e1447 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-25 07:10:12,708 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-25 07:10:12,708 submission_checker1.py:3308 INFO] --- -[2025-05-25 07:10:12,708 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-25 07:18:42,084 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-25 07:18:42,084 submission_checker1.py:3308 INFO] --- +[2025-05-25 07:18:42,084 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-25 07:10:12,708 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-25 07:18:42,084 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-25 07:10:12,708 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-25 07:18:42,084 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-25 07:10:12,708 submission_checker1.py:3324 INFO] --- -[2025-05-25 07:10:12,708 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-25 07:10:12,708 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-25 07:10:12,708 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-25 07:10:12,708 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-25 07:10:12,708 submission_checker1.py:3345 INFO] --- -[2025-05-25 07:10:12,708 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-25 07:10:13,813 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-25 07:18:42,084 submission_checker1.py:3324 INFO] --- +[2025-05-25 07:18:42,084 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-25 07:18:42,084 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-25 07:18:42,084 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-25 07:18:42,084 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-25 07:18:42,084 submission_checker1.py:3345 INFO] --- +[2025-05-25 07:18:42,084 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-25 07:18:43,084 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422701 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.97851 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8585 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 003638ac6..a848267e6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1280 +1281 diff --git a/summary.csv b/summary.csv index dc906f0df..da329ff03 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422701,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.97851,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8585,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7e6b73f75..116184992 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.97851, + "Result":5.95448, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 1527791e0ff2202da8fc782152eb8e6153653858..1bea0ed86908f6bc2d981fde65bbcfd03c1c2458 100644 GIT binary patch delta 3793 zcmYM1c{r49`^RUR$dYYHk1f*77z`Ej4B3Xq6l09FWXKXTWwND3w%k)>Nwygb6Coj? z7*QchDcQ2slqH^!Z7dBQvcJ>w`@P3||8f6uAIJTX@8nb~A#;wI3Wk)wc4o#Koyf%67BO6aD%`Km)M0q~e2-r3M;ICo-rJ!&}g!tj6rE z#c4;KUGE@;R_YP1XRUs~NYTt(#W(krz;S#a&}(&phpwY!cB;KbZw5#9yLV0o1y*RV z?hOsH>sKG?u`pkZ6+f1)r0$O`h}=@0IjY7(yWP-9ZheiuzcUt$c17DJ2^P%NL>MQA z8W{zcIK~z?4%t-Qyz*Rr^DT1cIWpqg2jc`*D`y7i~7X9Y8 z!c0JCZ8bFD{SBO%8i9_bsOoZY*Y--I`N_3bW>9?z^_hiR*Uytj=W`Uh7`BP)m!jX5 zF-LW0?YML7m1WN8(qu%yJn`+1o90Jc0gK3g7HsAUR^3K;h#)v&q%CHR{oOo2Kd0-W zp?a5E;dyb+f}-~E?=QVru(A15BdKwX3Zb^D%>fgT_~@YzAFuc~oKrBc^Lmvjxm+KH z<}LKLY+6VwTgT_58$=X}fAXz#u#WaBOh$|tvqF-ELyK7(Ur8l|vm2f7HcuPH0J)5~ zm!r0L+_{Q_Fjd-iIxhR$2Le>e=(Va;bH6WCDmhONrmB527T23t)Y&zy__)$=GybQYQ>ur6rqC_$l-a(g5l!b?4jx>UIa1@iC<>M0^06&p9$q-o z*sY(lCgzc<4G8nN>iXkpH-Lq}y3zcPWOEz>nX1n`>4u_*e{}78CYzNIn=?~ zr#h%CrDAbR2{%$lDJg)3)!P>|sE~NDD>rG-oPB=Rsg=(5_RS{x|I4<*#uE1CwH zoCM`oDURWo)q_ns9&k8Pw3T5^NJ!+qd6wFCe~9aj1+1>77)OzAT))haI!c8Vhf1dMN$e$p4k%J6gdXDtE`gD~d;M8C^ZU*A6BXuNrSD zgU3Dm;}1L759mqr(0v_J@0w_;9LU{UxYSQ^#7(WN4B3uap&h^&$*D{eK}HG%*PI;c zZE(DzD0U*@FdK;s<8Q|3gU#_hD0&<@6$eZB!#*R#r$I#Q7(HwqJ(rl@AT4s5Z(Pf% zlqq}CC-}*wtX9`f{iSt+^U=!;cPvTqUU=ytlYJvEOzv|k>qR8pV+4fWI(dfX`+nO! zd&TmMY-`YGZ6HKuZRsjqxW)Td8kuI9k!s1lN2Zh0BBXwN#3p(skBn$6YzqvRJ0Ji0D@dBjw=tid|-k-R#(j2q}-z5~mdZBo8qkEzL61 z>AJ2KPF!ty&vlV*t>ET~XN_3LUtx7zt7FHd^`X{FK&>$iuLDA~fig3jzgBf{1RwR*kK!*mMmRv~#@fKrE-3kyEwQ*4yoN3^pU#I-rbs&(GrD&cakUL(Y290D11n9D=YckTV2sO9)@oxKbF z`?Uk^pYM1vOg|VY10?Tey?OERn=`@J> zHdZrJQ{SUyUez*ckBkX%`1^2{sk{*^%E#$)h_hf1Sym#C+85$15m(eUVV}2TO2Aow z(an>GDKGu$PlJZa9H-aaeHKc;7X%+rR&sI5aw!g0ROa3y%HWLD*Hf&KTFP`YjAn=l z3gQPX5g0(f`pcwjAG!b0CQ^Mo+lST2R_*v^evR| zu9nFeW9NmWzJLRoKq zt%o7kK5f5ps9Q=N-$?RU@;#)1%7+$0ZD+ACX{+5>a!R|1pkxOqy|V(WL4-=qgCC=* zrXofvKJj0XKTE~z_}vi)pOLQ|)>C*aL)gVTFO-N~D}3XInpY80N-C_;&yc?kwRHhZ z`8d%yGKi3o>dd~^!k?fmS{~f1Kdq5?O`PK9f=I4JkrVLl z(=4sLAFXp~wZoUyGp!d9rBQy7x6hm>0LJOnWl!6Moeqkrd|oPV>Vw-ofg*chyhmW1 zPClAuDVSg8398(0+SVzrm5&DGO#NhNeJ~JJlE2Ku{;z^|g z{I2#CUXsW`{C=xPNu#*(HWhaSwg{EOXO}G&Sy-Fk(Gbc-XP_>cfVMQE?(jYC;(M&E znTbU~&e883twhIT%OFjC6WcyNBlF;@QF*QW{W}OAGUUn|1-n(PML9=aO?{mKcaj>o zZokQMX&0YkZ}+Kxhi*&RUi$6QJl~}bkwJ+L5EBMD$0}3ifG<<&k!VY74>Z-RC;bjv zTN8Neh&mqiw~TAm%JLt~QCvnjB{kiKeUht&Z8Nyz(LM<8*E%fyL(10bH<8~pDV-vH zd}0<1LS|+2i78ZvNHp?aqEJ@eR*H_%2X-SDvTuNup1Wv1`M>~swsv}J;^>F4eAlIg z!d0st7(lo9FHI8Y`WMH7L45wj<^2!WllEKSTOuRGtb)oZ+ltU)**Z{sN#sIe)%2E6 zU13fIuC*WDo_y+h3$tCK4M{M-!lL;D7s($uejxNe+Nux-5gELK=3lom;@mJe>3hl! zF?EO;`5RrV#N5TG$p;z$U*6-biMk@?L3pEXIU!p)aW(E}jxyaoBQ4TWXrN4GyJCMu zTbfdcKL@s5VGpulEEhDd^LaLoPuO-G3gu-WR99%Vi&dEWEOuGlV$k>upnctbyKR+7 z*rXio&d9{dAAowWu&@GD$xezSn`E|qUe1#dZ`f3X-#!l%fbJufEuz*TT2eWP3gJl! z^|y&tch?JiF+hXF{mt23pP60csAgy_mH7*cxeIGp0W43E96+jvX$%T18+GJ6S$dDY zKLf;%fXgg{>8~mvejF4DA2Q^pLDsN7UI=3op1YOXNFYC>q@Kap)IA1Md_^Ipij`Y{R`Z z#PCH_UZ5uX3%fzXu*fZq##ZXTQmtLUZrQ``RpS*3;B?yQ@{+=b(+{9kGQb*m!N+Kl zX&EH1j|}z?jL3tcRDuVZykpsbwz@g0)YKJ*sjPyl7msWQT|>KXfuWpWTWo!$Z6ZKum<#2SG6*7gY>#J^$d89jB0TIi>vP z6fkG}h1PdI=}{Pc*Rf+zBgZ3QC%J~bT&f*v&k43l!7*BxJN%vj2qLDiXiFzanzLXE z`I49lRfqN!Nqf`=VmE_sXT%X5BR_Qh8>}j%W)ORA7&NFNXCmBo^8hfX#IvYIX{I%j z`m`fDhhPszTZ(xh^*$?%xhv(GFUq;y4Rxnul{w1NYm(jX2}M!%J9_isdCAxn;HNk+9gf`|?!o?Npc_pU4RlhdaHl2gu{`m{I(~IT;dZ5Wc*-YubN=m@{Z0HEKwY~O*-2waRN<+%?%Y5H;C<|kMH%z^~d$c^LkzH*Y$oquIt0a`im8Jq<10qF<7B; zAdqAr2&4j{c5CgSW=g97U0JQc@W5<&#O^*DP8iu_dwi#|!uh|jn^DjPXOoo|)uUL= zfn~MotaYAzBj4>C^Yj{j@h<00%3iin-XX(_9XIiHMgC|rnC++hB&{sY<<6IP)?C4jx?EX*1_22Wnisw4U-gpraig=N z>fR9Gi|2d5FO!R@H8qIWrn6+P&U^9+DMzAMjP6i{_lZ&+1btjzE}{^N>@3dzl0J!D z9QQW*=-<)Dbw)GBCQjZ9U90;_sIg7IbPG_i$fx|(mlmlF9$O^~a}1ltiZd++Goos< zTbc_`q+ZzILxA_G$oJ_jhOY_1yhrcw3bvHq4{6gzb@aEqKbDbT!&%QUZs;FF6Ne1_ zTbpjsu1|U1s<7gOuC1@Zs;AxEo`11^!R)Cq_AW{1`@laS>#Yi{ts_jo`#dEM4}uNyXN&anX%9|2*9E;{p<5NX5XBI~g>>}0uWb=Z^C6mG45a@;Fx_{NKhPDc z)X^OO$7C-fDTE9gLT0bJTmC3})pU6vHeB~oauTZSS2wH9V)xt$^*JX;@EM(ZQ@HNgQY^Q`-(&5637*GXtr zH~>|w9LiF$r-!P&d;2LPKe3#xa8viTEdML8KKM__vy(gpii{=XqldkG%6Q%){a5Mv ziP0Mq3~Jxjht(y$NbZ^Sz#{`Nad1Bh$svicp?9jBwlLq|{(uxzr*c zXmqbPN%X4vl)^`JraMaI^en06rL`D#=I1nN&645R#ixwrL}LGk-pZmM{rf!}uWXd# zuU(!AD_K%!^mY$aDNlYJ`x4fc8(Lx7ns}@@Fj?_iIvNrJ(Wuo_oS%x|t5#_8Ndtxt z7LL+&YWL*snQ11EDa@Q1wakm4U>yL>VjphPT5C$@9W)JQU*la}5i@^ntpEkitqw;D zRvU+TduP*^jiIgGF;yyRDWt+lmHFnlAIh2y8cCSFSTr+H(IY8gqO<{|S(0VjG^Fyq zE*8bNGKeQ`4?CQB-qZn-ANH=<%Hk{_^IgJ!0L)@H7Jc|WQZ*MnQ$;w`Q3@LIu!}kF zEubGT`n&-^6P!E(4_=-n0TzMBkZ{%OCJ$_}l8JcLBuEDahP}YvXMfG_T&m?j9E;O) zE+k@Xua&pVwAM{HGJ$zmn_7Oh-owpzK4`kIB4(pu@rKvm-3Kppn?zuaQl{2ys)fWL zp%2Q}<%Wd$upg}ELXV$B-W)BKeH{Rbe=Y^LK6Jm?GKV?No>>L~FgEB9h`9Sr$&e1e zL=rX}bzIK$Ze}ETMAp^ZI2qnZ^Dw<6SqGD8tZ{K}I7@QrKN+M~-RoeBs!Bg~iDzQC z+c78W@s;{Xx=D_c(CwMmj^X~mt4T1=NQ=TGRWub3vZjdXpE%dm1JlRKCl)_W_-DCA zO5X#Au@KYua3KS<#+_wlzJnC+?^~ekdu5gFzAtl<{PNw5YsTn)IYM2oh?sQOge@Xo zW+tiPsrVQX(SW`FuC3VbCiPbgaew%f|LsV3$C~)N%Ga6N9pPoNudQWAY6x45ekWVw z?$4DimV?D^Uku{tUZ3$S`IFi+|1L%Xw! zq|n^1-&GgSWv3Iw<={+NkkZ zsdG4k6LCa}Yz0=Y0p>`Ev2JC-PWr`IpOgPOnD&C(%)~KtDjMS8y7g(^5WnRr_Z~>M^;)6f=DtBNK$Uw_b`=mp;LyOr#FKpk)_U z@UJ%a#SFbK2H3tgexJ2}MIO^%>HTS6mrUM;D7AXwcqW~^)BxL|@sbvafo0I;*=jj^ zOu%$8gQOl3G5ad`GfJFE;6%}AfgcN?cn0Rr#e?nPuVibiM>iX4T&nsPGl}2fhX^Y@ ztF=z&`SA5xZs2>RPFDfcyYTp^<#Ymm8+Ak;44CWfFgk13d+TC_FGVo+(xqy0QTorn zRt4c&D?J;v4s{E*t@Fm$*yUl*3gRtkmRC!*GOW}#z!bA1Iw~lHxRYL!XlRh zr$|iuXUuOc4dn(X7cxzqUgXQVGfXUSK)i?FsC{bCmANCh@>LXaDgaU4T-o!C`i28+ z)A$qRih_tcGgDa<4+tH?7@xzw=fHL*rfC3|Sv_W#&V9osTqx_}05 zEX*DE@^x{CB2t!Zmy38ILZx+!86||hX5f>Jb9%9_0E}d1nfq3StUJ4Ge**`K1=J(*vhjSTM-PUBb7t^_{-s&Q}7ajeVaW zP+fJLNfI>aH9*;+N;v9jlQ9TkJw%qAlX5lJlMer3_gT~9TV>VD##?=-B9MuE!+kto zmz#SYy6vk5zkWg-K00{p4D@Ota@2BGX(he|!*~?SPr%clxnL!8QO6-g9Vbc!fRsYFh$&#X~Jf|mTMk3oE2HW=L z;PxUe*EaJ3W5Uw^lLQ~vxVL6njp|Zj5=gDPJWO3-hBiv1H0}_5YjRFWbXoA3Y22eeDBi%f z;UhewStUTO?Lt0_x?e#gE*$k++|L-1IEbnso@+orm5HOcMPj@HF*6)SQjC1)d3c$n zf6k9xv3}g4GdrgSlYdl~mA9u8_CX7Xt*;Kg=y-;xj3NHQnh?YurZS5p(dHNr3Asd% z7)y%S5$nAQGoWYsE~vo+;Ah)+Cb}8?><1o;@>!wd$X>e}V(aOADtg)V^JN)aM$9{o zyp^^ksvT&cL@FfFVlWq6INLPTSxk}yOD_hx==)90mlGo@-V>}i(@dO1&#-oko@zc# zTk=*th}RJcz!9HL72Kt1mgJ3Z^)PEJC;JMw7%zSZpX094s?R_%h1gN-RstZhSwc<8 z9HJvFN;8aJwPc3ADvGF#;^09%caRybtymk0pZGUgGs@gSya<1gUxsX6{CIsbayfmK z;Y#1rHjE(bJ3HaGU)R5o>#c{~A7@l|b*9~KX02-OXzxhEQY*`pRW(1#3wX3;y}pUx zcpN+(8qCFPUInfKc@d&lf6c;2Nk#`B;!|{eqC4>9n#res9!FMow)|Z5-Y^SG6Wly_ ztYtcDX-WWJ(uHqO2`0k8Qzd+&a8hq=KA}pWZS2;g;`v{jb%W24se1(> zyiaxZ2cNDCk7wc66(7I>u_~p#{ktR*QvASB>Kp_N_%t0?h@Xqv!Kl;aHTo&mi2Jci zV$B_0@9gHl6sw~%dW@sjkJyYtzB!Uh|J(=;wm&soY9Nr9eA0cY{Uk}L>x*>B>r-pa zx*QAIYQrn{7u|M@!zS5=A$wla4oKbp=p0~1vem#uwELw6t2-xGPM)#*+pGIA$=OHA;yy Date: Sun, 25 May 2025 08:43:10 +0000 Subject: [PATCH 654/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10943 -> 10945 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index afd7e1447..e01c84c44 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-25 07:18:42,084 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-25 07:18:42,084 submission_checker1.py:3308 INFO] --- -[2025-05-25 07:18:42,084 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-25 08:43:05,170 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-25 08:43:05,170 submission_checker1.py:3308 INFO] --- +[2025-05-25 08:43:05,170 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-25 07:18:42,084 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-25 08:43:05,170 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-25 07:18:42,084 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-25 08:43:05,170 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-25 07:18:42,084 submission_checker1.py:3324 INFO] --- -[2025-05-25 07:18:42,084 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-25 07:18:42,084 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-25 07:18:42,084 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-25 07:18:42,084 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-25 07:18:42,084 submission_checker1.py:3345 INFO] --- -[2025-05-25 07:18:42,084 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-25 07:18:43,084 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-25 08:43:05,170 submission_checker1.py:3324 INFO] --- +[2025-05-25 08:43:05,170 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-25 08:43:05,170 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-25 08:43:05,170 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-25 08:43:05,170 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-25 08:43:05,170 submission_checker1.py:3345 INFO] --- +[2025-05-25 08:43:05,170 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-25 08:43:06,167 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13835 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70802 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73721 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4679 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index a848267e6..c6523368a 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1281 +1282 diff --git a/summary.csv b/summary.csv index da329ff03..93668bbce 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13835,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70802,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73721,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4679,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 116184992..49320c150 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70802, + "Result":1.73721, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 1bea0ed86908f6bc2d981fde65bbcfd03c1c2458..b2ebb3645777dbfa947fec56254f41c14fd945b5 100644 GIT binary patch delta 1751 zcmYL~eOS`x8prvCIBmW~NvSieK!ntcP}5bQb{dHIQcK0eIt2!aS!QOMfaIn&UxA=> zzLe3@%o$`S(^F|y3ON;=lU^*#LN0BlHVZ){ z;*++hp^jxHY3!fZ^|hyZ;HcJ=HrbX2S_je@QHA3Y-9_*1yWJ#?PvK&~oAg)`TiyvD z1P-vKIk*gbAG9o}?@*r18x1b0JZZbOwNYaT^1R>Dm7s)P&!4z^Z9c<`@m0EpqZ1t~ zFy*6~;R0*vYzP6RJ zcNQ%HbHgGzM5ncM&8xq1&a!(Hy`Q=3PDE5zZ=|JhrdQMd+vj!q^X@DpQ-JT@@IPGs z5)MQW0JtSw-~VyUfAU==wYd%9lA<~HU_GchS+l21b(NaNx%E;N9y?plE|{*3a-EV+ zhhP~fKduM4 z9*@XnM@MpFEr;lD?3!4+xkDZgu67&y510t#oxfPK`T7vWv9}EeO=IOf)sqk3-MrpV9u6FSj_g( zJQ+g>Cw;Ekk40FtcLmVY072xPmOzPX{oRJpqZJc&ImznbZ@(UUer}?<;a5d5Yc;Zl z{p42s4dpe@n>_X>(is*|Rw%n-xsw}AWI$~58N5&dfT6$~#zEI`RDyXnu33DYPxJdY zbg6-?a8P1kk^cq^#qWeWXX6HAtjVQ3qsl%E9ZJAtJP@35C=l_8yQr#V>(_P4_>fL4 z4>^vHp1Cw?5gdDB zQKco(yfr4a`}6#WKSl(n?tcW&`py{TrNkIBN1eOVM-C0UPZ`~J6g_$Sui9$5B#G9N zOyhEAUrGawO{NQX@-qAqu&#yE_gSx8e#H-XyHZk|f5QI(muOw9OpQ0%{(7yB`sSGL8WZAscL*adp917FeSyUKwNZ}*` z&OP$b$02tzdwFfEm!^*Fa2uT5U6&I~i_2W8a$nKRDIEyp7%<^)5T$!cu3wx_PmgLI zs`J)Ew9z&gui}WXqEt#iXVkpw# z>T}>Dn0EB32mIrdA2$7zai>_{bPkn3b|?uMNNICj9q=iKco3ACpS62OVnpu;1k@LT zR)|EbsN3}L$-^SienVNz{Nb8FJH7vu#n#b#CW{`pe6zb^8D!~85}@}1j>35KHlSK~ o5gh~+3ZJ6`0h17hLH*-22;&I662@S509%A&%m&MMD3JO54Sa4ksQ>@~ delta 1779 zcmZ9Ndr;Ep8i0RD5^8zP-4)k`3Q!PnQ~drLY3?-JncMkfpHfzGnFV+ON#0o`LHQQ|DdG>N7dd*SpI|JO37Pb|WtODkdIX z_B}|$%2q78QF7aPq#Qyh4u?i6=8<*G5tT+F%ee7DWM;ACp6~sYJ(Q{o3+6zIG%#G3HT$2 z4T^hAM=W}PA~f`*q8>Q}N#aTbBpPMFy1-|kFiTBzesY{}K=gV~RMYa(XGLRv@&-;p zVUYUhoFk>%BP&KTf*Ew#YcpvT9I!>4&0&jsO|_fAVl4%ZT%i}c zd3CAV=Ab)=6@}#szTi}u2j!ceX7m%TUFnrki*Kt}0tV2a#Q(oI%f4-#0u7DZ##Q_e zHz3>gV@mlxD(+Cd{<4cAIE<6PVplVpX7q6_)uk%GXM-3`{;`zKB8^426T^do&Wwvl z5=6oV(00_pD!mFc2bfZwvg7>~`L$@WpUq8VSau6`7ICRhkypMTnkvI@+TGPpwA3I+ zFm3J)j1pwt$~D(gq$E@>6VTwp^(cdCm#SOlz)u`P8&sh{kaS7ev9^f|I(}arGQ{CL z;#s@HSNasy$@s_tUHj#?DF|EZ7zG8bB-~pUTdG+;hBJ1j zrMaH%HFU!nC2$A3%KBRVGswxF>l{*VmK@WU$rZ&c`P{o| zb4OjZv7ehpg0M6J_ZHfcP;v;K4!cWYQR|@`|LP$vul}_J*J;n_5kwU2UkS(bm&jd@(q)q8j|r5%dZ z0jO6;ff1CQw@LT6w&1*j&i^67S=PqV5-<0@LYIwo*6R3=j(eH82@+9NVRg78LONAFuwTs$>7+-!b?NQQ^aQ%_8Z)8GCPd;5&10CWB%a z9iV)5%YIn9pd5Y^p`3_OaZ5k%qog0yrSSyr3L;{hf7|8P__e;LmkrM|>K+WTn?Gn0 z$=+D5BJgcnOJ0VEUj`N#9@}H-=^IP|y%~9izxZjB?I*+L;K*JC76JevMgU+1005g8 z8WFE$j+B(f|95_n`xb zSIx{37H{{O`{U@g*44VFUpoZ<0)!i%%VbrYdUc=H{^IDNlfI=fIW`MB-qjLkYG zP3~BY(Vy2zYJ`h|SW8dgydWjgJ2)RmuJoTc56DO9XLv`%|MlKXzWsz(Mx`vgH{^iy MHhw>B+;QvkKWdpL(f|Me diff --git a/summary_results.json b/summary_results.json index 90eda5144..cf18c9b7d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70802, + "Performance_Result": 1.73721, "Performance_Units": "Samples/s" }, { From 7aee7c3d91d811236954626ce61be4800a824cad Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 26 May 2025 07:49:24 +0000 Subject: [PATCH 655/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10945 -> 10946 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e01c84c44..b00ca10d6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-25 08:43:05,170 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-25 08:43:05,170 submission_checker1.py:3308 INFO] --- -[2025-05-25 08:43:05,170 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-26 07:49:19,374 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-26 07:49:19,374 submission_checker1.py:3308 INFO] --- +[2025-05-26 07:49:19,374 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-25 08:43:05,170 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-26 07:49:19,374 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-25 08:43:05,170 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-26 07:49:19,374 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-25 08:43:05,170 submission_checker1.py:3324 INFO] --- -[2025-05-25 08:43:05,170 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-25 08:43:05,170 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-25 08:43:05,170 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-25 08:43:05,170 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-25 08:43:05,170 submission_checker1.py:3345 INFO] --- -[2025-05-25 08:43:05,170 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-25 08:43:06,167 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-26 07:49:19,374 submission_checker1.py:3324 INFO] --- +[2025-05-26 07:49:19,374 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-26 07:49:19,374 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-26 07:49:19,374 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-26 07:49:19,374 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-26 07:49:19,374 submission_checker1.py:3345 INFO] --- +[2025-05-26 07:49:19,375 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-26 07:49:20,390 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422701 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420161 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c6523368a..e92ce6291 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1282 +1283 diff --git a/summary.csv b/summary.csv index 93668bbce..2304a44eb 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422701,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420161,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 49320c150..aba267f90 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422701, + "Result":0.420161, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b2ebb3645777dbfa947fec56254f41c14fd945b5..f73b67fed3f7d0c8b967bd91025786aebf245ab7 100644 GIT binary patch delta 2445 zcmYM0dpy(oAIHBl<{s&)jtn!*sI%5`T+ZBzwKI|USse+RNRCUch3`bUbR4s-sVz)q z5*;&A@pCIjHk8XbYIIPFbU8vUlda$QJs!W`AD{0ZpU3y{e7qj7&*%Mod^-Fd`3d7R zR7qCplKMoASg4B4+xi)8e-)73^6t*cNx`$Nty^YR8Khp-63UkS-AFOOML^FnmQxBz zqn*eZh!p#!UpltZE;yI_&N74I%T6F0{=(h)C~8RIQCC$Gf9q~&{Hsf)` zQ!74ePKq;HNrBuRhjj3x}`nKGg39p1C*)<%rC0l>;16 z4N(#^iU^!+oLN)vuh{^eNgCDp?o&||*W5qPwXa>*oO@kUMT`uilNb;j0rrA|(iP5d zUQh(O!iaZe^(}K5B&orGkE>XK{vg7<#C@MND)_1wx?troVm}-ZF&ck$R4_X4dS9o{ z0Q4x^`5Gi-WK&ckKGSt>kvm=mD+7`pX#&DzuoIkvWO`&spf;P0)cRGq*{oNMgr=^j zdattSi)9aMuM~{aX$KU{5NixvW8Son6zd>Ce#rU_}eK%Z&rnS z=p^Pp<@{gI!)!M1J|ww0?LAyU6-F5>t2csuoIgrxDZcm_&*sXmXB8+EUi!@F$pR4@ zxsAokeUxKSVa(91SaieWgsIJDQvuS{So>}+ayfem2=CHn!XJ!|SeFMXG%k9?BgfBWE&F0M_vEzCN?I^tOMjsBCR+?e5lvIciL zqoKz!k72j51m^~I-XI@h^_j*q4=Hc2#W|CUSSDg}+#>j4C9eOPR#II< zCOqDbJ@Pu>?4Jb32uR^imNsw(1Wi61BE}dH89wvN{8JH3wa&wZZPp?0dF3uDm)JQZ zUj=U92UbGF4aY7K-3FOb7TE~W*v9p~@8cduV0Z(i7VQG<$BznNAU#Uh+B>15K*LKRq46i1jW$DZ4;hzrdrR z0hnr-5G%!>+uA;e_;xNjwcX&2u{~-1dkkg~Fnr+VX>IM5qL7ERQw3uOz?&ai=Oj;s ziJC>>^yI-clN;U~JR;b|sC?vuF3{a<4h`$a@$eoGqFqZ-&NfTekF>&}n z^2esISx~)hZOZhb4Y9^l^x@XhpQdSZCdwgzK1#FhYi zfF<>yJe2Bjm?!7_O$GFH$sL6CH_*{LH=zukT zJiFH^A^rFG0hu_6zUX@Y&ELdN*LVa4qTqix&IS%kfc0o2)gp6A!Qq6?B8i^Zl4y@dWQF_3qzAzImBLV>gqr{o zbFRE&C4-$4h;!fatMRcb&B&QuNqwCB(%HTb9$x*`yxzGpjVlI?-$douQ(D#nWdzean5N)tuW2ui*QCge5 zE(<3=&^EzGE?y!Lzf$);V_(Pb+GY@>Jlt}0m4dD>BD=_g4No(39ohv1d2l(vMJ{+` z%9OGwMhZ2j^sFA|*XPN~BXMA>9{20}Z{aiFEXS==nj(r;@VGog>le`TF!>l@Y68-j zRK!iR*(=Hx6}u=Vpq!W(j{UnA>le9Gpt;9keOzAJ5j1dK7faVl+lZ7}YM~#_tNAvB z8mSGfrc5>vvGCn#20>??mDy$>h01Il&=6%d%9|}-qkKr+dV4TRCQQ|4v!SYE;_de$ z)Wax|gK8t`;w&2EMO%xWi&{|*f265k{o1?c2Sup;`dcp zLaHVp#CaFQF%?roDnUCZOTR~JyM7PA?yb$ptN{tokaD01soM?H)i_~_Bgx8kZ0IqD ze)sb<_|fS(Z7}#Z{sqQ2-<}avanMyoL8EA)+BU^!IPnPGo!MEWSidt`&WJ2ee|=g2 zj7~YL3*Qg|jV}g79l(Zo(z|=t#bXFpT0^w_5*UFS{-`NV+XjLkjpjV=p{qZCD2zmm zUM@)Cb}n#NuzL!2`!X6}|ubcs2C(p4ZZ3ar{2XH$htTzZKri@z|CK zwBl#*=aQu4)#k?er>l>9-rD`b4+@)$-LG)$`KK(kuCHriAx7?!k}pkV^?J+{Exwak zEZRJ2VG=Mau?MQaxA9-5F^aith2GdJy|mfN(%q|-U9n~-SJSX3|31rF)5hIpmXl|Z zKhE|=A09ds`kh|=K+hkad75G1Bpo^(GpTL0O?tf(~UF@iAL zCf+MvGSfM#Q1K>0Rks2FprKrNU^-4^G7tg)6cqrd-@Cna6ZkgAL<_0h@mB6^R{leY z4)u@;8?-wjSbLpK|?lr%(#)$NWB z8L}s=Dob8(+R4qx@$!5M%t2hsGN9(CZy>m;Wr(Z_x)tCny`!+3$F|CQCQT-lc zURS^SO7k!Qe&}+t=86}3F67>E?5m8m&tH(*r@@!cGy2+^3J=P6@Emnz2j_I`rds8y kf7N4hlo@QwLRl{GAE6S1H~;_u delta 2448 zcmYM0eK^y5AICQ{2YG5q2S-!K#)O=iDJM2@mzgnjk~vKeesi`=gdC~%8#$qqr&$w@ zi00%Whb@Y_D{{h?glRuBNE{_FrSf3eq_D6RP-%500eQXJ<*=;HFFIOh+rA{kV^Ppltjzw~C!L?ZB;a zn&Dwb@ua`+#CU3;xGzB27#_BFnK2TTb-9A-7U$cIj!GknvMhlDfJ@;)@kIeJb}JfR z#DcM{Ql{ClXOHY}_DkebMHp#oozZQI(L}RC{b=~CB)>VwJ@h$#>{YX(`Cgy1uejqy zh{J_saNo-d0z zQn!Cv8Nqy4J5iam5or@1S`_|!(l^MNREoLZE{rlS(7_qW^TBUR_I84?5!G-RuEo3V zD`4>YvIA!FHy?7xo_wr)C;t@7l3miVk{SbwqZ;MgWJAH81MjNm6RIytf)^7y=T(TF za5#!t;rfcC$N+Y;R`QH0Xl=_!Er|s!9`7Glb+26Wf)t}y_L6qWjl#EL4R?4`pE3?K zMDvuB^)#{kgMLx_XJv}Rr=gXWeRmgfCzo;#FGO#4bh?HRB7*-zu-g!z$pG>shM=CA z6mt1V1Jsd%FnbHVfnyO3ciA$th8^D58dIO47?~)ARpY3K^$l~fh7|U$+Bi|zPIr21 zt8bGdY~CTTI)w-hCu(u)G64u#kRK+x)WHMzDzEB;&jDy-$kg1$E1uKLJZ?Ga*dYG_ zCDDXI4@c)VIsw0uBBqO-!SdIgv;5Z4ILwv6IdtuLHYe@CQ8MUtxv8#RVi3B?*!Jl{ z%;Ca&?<)@&@R_{i0m#-Osuj`hJT=sn8>UwdL}d%8#YVPxLm~^o#px6K5dEwGmW4z+ z{cCl$5JfS(+y<>NNoW?pL)%464G>%m^KfVcZ@IJ~+!4Xx^HnXDGXOKG;i9P&B6v~* zb)qbqTEiP~-GmWqWP`3UBWT0w&2KXhoHDi_Zr4!@!`{^@D}U}m+TV@6CvvN!x1+4= zuHry{H2;lhw{79ZCrAO*h4i>EthfXD40?pMz`-RG`Vjd(eUTX=7Yr=7^px@DyY;eV z(8uXPOIL&xQJXp2dSfNonekPUjHBSkWvMgmn~*#M;i50LivfR|!Qhg4P#4NG(;3Je z9Eo6c9m&kcpbRK&-_Cf;QwZ|1 z7BcOHtH6)GSpO)vLJJJ!K|w6#FD-#^A`K7-q(mmT@Ty!YszD19H zwW!HSG*CrLl)TDg5yybUE?`0Q9g0fzbk<>jGID?Syt@L_DmS+Hbln{y7 zjo;=^(7F-J_$s2rokU=pa|fqX%s4HsVi4rlIPp)>EZ>!M7^R&PM=9hp6WXvy3VvEf zrvCtn&J7N5WKuEeP41pwItsPJbs*Oe?9$nxR4`0$7(CwM=s7b)%_vKVQoT{vpa7cx+%Tk)W~IjB%NhJ^Dl|6>-p{WXTXg!8QcJ# zHYF^~W*|3kG@{lTH+0u!((&(yuho$yW>QBB>QU%&&g%ld#qehb4aj+1)qrUrCV=Re z{E&CnEQ`a#0_mU)n~416-|oLJ~~m14|QRQucGE|6Gg5Z-8wo zCGt$KdwJrOZ>z6PX@*420rI~qG?h1JRW;-Yep4yJuHSB6s9D_^9W#S1yp6iH0;pY4 zILv=75NugkF5C4>@F1}Xt4iwmYg1#^%Y%#ezYx4T{_vkj@XTISh_lpDl`>`3oiq_S zj-FScy|X(f|5aHA6h_lqV(8%Fn9x(zpuTb^BN-oowYjwLfc3)q55ka(4JFp~2=?nGYrX ziTz)9K4M=BmB!-V%;b*$c&-#JetPhPc>X)mAP}z^iy5`<_@KHwD<5+u4J^klkVPx{ zj|v_@sAQO~IRpaH*B$~y9mN@NDpS|CyV<4u1p(=@iY)apC1fFVO z@%1=t|M%me#fk9XS5xDscUmsr z?_H8Hvy};|mXy=h2PwkwvWu|ej%}EqAEuq$OZjE{ugQ0Elnoc{BgkgCzC*G1Y&Pxu znCgEFl9G95pLbMn?^}dB93$cLSs~3p(A-aa{^8~PgDXFkdzvUc=5=>W-B;`Oh~3aq goiG0^9gG(gz9wsdpW_cuw}K+aZ}nc=Yqf>^7Z6Q Date: Mon, 26 May 2025 08:35:37 +0000 Subject: [PATCH 656/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10946 -> 10946 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b00ca10d6..61e9b5ae0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-26 07:49:19,374 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-26 07:49:19,374 submission_checker1.py:3308 INFO] --- -[2025-05-26 07:49:19,374 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-26 08:35:31,096 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-26 08:35:31,097 submission_checker1.py:3308 INFO] --- +[2025-05-26 08:35:31,097 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-26 07:49:19,374 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-26 08:35:31,097 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-26 07:49:19,374 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-26 08:35:31,097 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-26 07:49:19,374 submission_checker1.py:3324 INFO] --- -[2025-05-26 07:49:19,374 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-26 07:49:19,374 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-26 07:49:19,374 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-26 07:49:19,374 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-26 07:49:19,374 submission_checker1.py:3345 INFO] --- -[2025-05-26 07:49:19,375 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-26 07:49:20,390 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-26 08:35:31,097 submission_checker1.py:3324 INFO] --- +[2025-05-26 08:35:31,097 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-26 08:35:31,097 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-26 08:35:31,097 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-26 08:35:31,097 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-26 08:35:31,097 submission_checker1.py:3345 INFO] --- +[2025-05-26 08:35:31,097 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-26 08:35:32,461 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -29,7 +29,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13835 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73721 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 21.4679 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8585 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0254 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e92ce6291..b2cff281f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1283 +1284 diff --git a/summary.csv b/summary.csv index 2304a44eb..9fdc2f19e 100644 --- a/summary.csv +++ b/summary.csv @@ -7,7 +7,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13835,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73721,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,21.4679,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8585,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0254,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index aba267f90..8752e7a04 100644 --- a/summary.json +++ b/summary.json @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.4679, + "Result":19.9084, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8585, + "Result":23.0254, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f73b67fed3f7d0c8b967bd91025786aebf245ab7..486addbe7cfc510c9794ea3c72e7bfb685e52b96 100644 GIT binary patch delta 3829 zcmZ9PX*|^5-^Pcrr1*_3CDmBQAR}QyqLF=uu`4RF%rG>{5<|+@mNiQ=jIw8$@}pve zq7ue3BwKc=FqSBxFxLP0-}jUIJ`c`=^WdE8yg%3HdY#kc(Bx1NaDW$-ZG7i0g>*m@ z=*esgR`Ab)AjF0&`$AK7d1AW6g^iogyZ7WjIO?uHubxE9jjl;oXKsx^8d=WY=;tQGxzS}pUh`_+h2itah>I#CyC7rs*DC!LneizT*>5ViW%0(2SQC+nB@swHg>NG+7)6LGIen+FCsTqx+ zkj?rp-ZfS!mpp)@rupQiq2#+t(5a2o?QE^)DO!f#LNk;n{CZ*>DT8xJt{2vLN>QH z;nnl5&i{^MpVL3r=nzU%SUw63sP`6ySZtWy51&}oP4B&Ff(YP)xFiQ#4G-^rX863S zQ@#Bv(3!pZrdmrVb6qHzcmiM#)Ss_R9VLafWiv7mp$bwHow1PnvB8@8KZ}w1>9OY= z>jPgijM_x{VIm~0l@z15U)sY&mq~I^GuW_6F!jbNYqTd)w6i7V(d zlo3xw1KG?=YlUpYWA)O^==Yy2BZl0ZP(r-JE4B3fe&wbBITfdN0%A*^;+Up2DhS&79=*= zY5DOlnZoWWw**;VfUU7#q-Po=WJwk~E#SO8rx2JTvJwQ$!isYuNa!ez*Vp zmVeN341KkyOZ%O3I8kWQJ4}bUo9pS7yI2Zo(7R}tZ1?r`A(64SSUWBcn^t~6T( z+?#z>9$Xh+IG|RRn@<~^{XS0%7m**4fqUPw)Zib>d&$2Fup3?9>&AOkbtkb9T`6|_ z*`NPP=Owplb>(L_EB;JXK(9V6T8k$RHw;u3FAg8W+g;sxiF3KU5L)_Irf8s-8gO{_ z)6{rqdu~XDUR%5>%|B7(TZ$2k1e1LuC$c;j#*(a%V?~W>#jc#DD!)0Ldw8KGbxL^Q z{3IqXjEuGga#2~V~9MVCo_!o zY+?DKZ(A%(F8~5RcyEqE8O+Awh9BRnmckdRyiatNfxhFdBhC=mR4MJj9Y8+L9`Aqr z^3N#1)L#{)AbDLk)(XuRkCRM*b(+D^Cd?x?F205jFr(3ozR!lFuqg=(g223@y8^lv!1@fv*fh1ZyT z_U5&G`g`2{h{f*amd5>;wdYfK#5BT9qtGruE&mxb-v^i^NKE7P)tiRm7Jc9-@3|*s z(_w;MJ)(U!x&k!BLZ$lfS@eOLt|sm^k&jJzk13_ksdu=ug32p9X5Ku0H2PNX%Zgs@ z^hojKl7XJxMr+Cx2B|i6UKIOLItvCSkOd18B%^4WPtbStkwSX6Yqc&EC|xzsIar_s zblxczbaBNyk5P^6_V|0sJ(3+<0>4-gw-);Pgv0tB)?`5-T6jf zlCBh71a{?uD&0GK7^Wwk9ec8FObR-jt|D`yC6m}eKdId8D`xoGcXUrtbAV#`iX*hI z#`{QZR{_h(l6eS~*wD6T@%`#-iYm|xyY(UL{|fbNi-FH`^&)@BYyEIgGvga67tL;+2RQnN4RcJi2Ku0i*@ zLEv>EXm#1f+y{EcXPxS)4MzyEPjKgtzv(p6WPK~Q>9E4^n?KCFv*Nwvu{>2n6 zIV*U@Kr=D@T=$c5(@xhh*e?h%`)rclbtWym?_sc2a=y8PeK_4KJJKP+W7yyQ+_ZQs}78x zy|H%pq>SBB)O83euMR+6_PM+}ukF&N%svY_oNk?9o1pv`N){;IGgLRj^Md2?5g5;5 zPzGEeehp-wg_Pz!Df}viD84bkr^qP@gM2&L@mIExePYQv46**1ugbauoHYfc{{uru9riYvHG0UZQDg@Ke>iRS$n{(=8ILkD8W&hNeG=UC|gQF2! z^%wloo;$JG7jVByR|89uK3VNXl}l;M129azJb{d2%9ZR}gR^D4C+~b59)u-xA+etz z3!omP66#nwlmG}Yr3oDdPN^B+Y3R4o!^j(#!rx1DDU{Ny5%fNq)9vtAU;1(j79B=J zS*ArglW0~PE?qtsDr$gK*4{TTC#$?|!yWk}Xv~dX*KK|C_R~^@0<9c<&YTXb_v~uz zUcVS z6+us#?A~vj{mh;!IXEeDB6VCVL6T?7v#YOrsF|HE_cQNbRA$|?-SP>|L1`@qzEi!v)P186FZa2x4NGISXZ%+P}^83ufwaRzz}iw zE>v@UV6V4t&nEBc%I*Hs%K%XvtvkP$_JCdtGBN-ubed^9nKUusJbt5auJ~lx@5rE@ z>i>EZE2hhFK(G26f3h$!XYB$T1!T6nbjp?ybAoNa9fJ}Zmzb^InFM{!O<3ON2#&Ts zuF?Pk^%ELhXD)2d)^X9|&2r9`JAo7{iec5VO}NobIk8I;Ksco#|rOJqdhlWXrd>DO*{vP8P#WS4&gu7yFxNUdA)ahumVjd&)4zh%> zfDH98LYm5jczhkq??mBam~frO9!nECwx1+fD~76{_9O3`^wsa(d&6NbxJrtSqyNJ8 zo>HqFbQ&afpuYIo%g!r>>FL%{2lhQks@p?h$hCDog)RmExvCL;#VsY67>AoN}BrcUx~ z*~=WyDUTf5@}J%Mv1h0U5^T~)cfHN=r|Vy*GjUrYu?hf}wKjGI_!4u;oj%QHgpJpC4i>#rLHQkM3NBD_ zI&1yR+?-^P1uGl`qn}STymVKvRU) z=7*s-Y1n!VONv-UI=c$w1^iID!0n+S)+@jqd4NRKNO9%3&FM$2{Nsg=Rds<}s;d_& z3Sgx-LHK@M>7HD?psgmg_1?}G+~)!aqmNsaD;__cZccnpYazql&7JRPw_dbqqotCWbu(9|7q+V zucDRVE>;&z8?VhTV1uiW@DA?({i}kNu5w9Gt}E$s6;N`NE^;MPCX{rz_9&94;8Ope RHwXEDM?~Z|%9T@u{{Z~xLx%tW delta 3821 zcmZ8kdpwix|K7+kXGKCZGjr;og&Z0Y)(oX&$$1kAIg`UPA3h-o8-GEHe>{KOulu>4>wUf7*Y$etch}xstMKFH!YD^}gpq=IHGu~G z8r}H*aHovdLB#H`@#FG$iV1LDrLB!Lkn7U7SIh@WiS9$upB#SQLLMLoM|Dn2k$XtK zV<&P>n$TBYm)x+a61Ci+J0a@Nc{czX%ku7;5*|J}Q;8&x@Nc_fCn=3ty9-m+8vCxk z(rq3Jvre_z-owSya~0nmR(-}$K0xm!w0RNNTG`WU@ws5OxU5N8E*THR1Y+p?Z4W=dEL( zG!9u*Mfkluy7L~+dI=Bz-cDXVjbgPohMv*(z7hITaDZIuakEQ0I^?U}BcQ)?yni+% zz(3QC%wV9m=^>TduTD$hzg*FhCJj%H6@R}IepIk;=tk(Iai5KF&%TXsyv z<;>%=({#d%gg$vzUNMhxDN2G3>r+CK_(Gphw!dOa(AT%SKI~l8yAR}&;$DwfqKt1= zNJ1fmy)@&j@9k(YQN4PIXmh`(m}pX-78G(KBG$NNFZjNw`U`Jk$mmQC`=wfYG12QA z>hW=q-&=k50o;2in5#Hq7A}P6kNTJ4d;h>o4EE~4vo}-H~b~I{n=DblP zqyc6WSN-;MYB2EIr*0&_8)sw;gQsYdFWRdS!@BMI$|W)qVl#v-9rpJ}zjRJvHs;zs z`?{xO$9kJg&o8@{=9&+ZeAgGCy0bV(#^~1KJ%nuQ%3OY0V{T5BYR+CQHxV#Yp9FI4}f+@0O? zW+g>;$tmnnih=pqA%!N{c(dPT#(t)d(cX;?Ehp0su4*5FU$Sb~{Zw^SnF(@jqDqldQ1MiYV&+QOlE3EmoZ0}I@*}TT9 zgC$Kavgneg(Xhs*0`Zr6UT`}Bj0=t&v4Z{* zyJ!^3(<%C)i2z}8K8P|-`4orQQO`)1=&a>s0s3%eWhy^Bk~gU4;!tmcwV#aOxrF01 zIQ$;F7`=9j(;RBVI9!S`G~vkQ^bq$3e!)}3ds~RP#QX+v{;TZ9wU{JNUdDb>yttgv zX4j>?yoI(sd6(pX#L7GjD?N6eXZY3mOlIX+Gzwcl7GrK0y=KJ{s z5OQc^*`LVQ;<}lNBbcS9n9&~Mh`7{n(O;i6Tb@$&&D1dZ0X16Q%`xe5k03$I{_t6% z#@5W39kW^jNj|}U@T;(%Tu`eF!%(J&7CRm;>NrwjlkD}#QP5powaj3uj?u!5t9|}4 zkiSPgsCm4+5ox`7uWt6zsng=xV&=<0tv&&z0fNy%j~_RG_meJgA|4YlHxiar9C6i- zW`A-NeF~2yr!~JoU8wMqkf=yu6?HSwBe52NMN1b)iqN0?#{>@XlX zlkrX~=%!^sd`456y9;+ORLX#a17W&E5Y>E`KF$JnjQc+t-3lAyq|g)#3PBWF8M@={5Y&FG zW_p+BPvvaDTfsvDGV^4wOhuaq7Z^=~ikXl3IOs69WDF6kiQ@i>QxU9~rhLT}3s(GB z)lq+|zCbTlo26-d(>xtB1QhQGyRImk4(c#%J%$1jhKh%lFT4_9hykYgM@CUmu{aKo z#bkj3Z1YcNwvbpoLNRL0Szn$4Bbi2ZaH@;Sa(z6I8JqgE236bYRb`Ps8J7R1{C?8l zweSFUEdF-(pl^5?kLG2OCF;15M*hoiGwy-1A6pdy6)36%u74cZt~i#AA^k3-KJ@2S z7nQ!)ku_<>EW%fKiEZ%IFeo2#%5m1nX|uW~tvy95D#(Ez*|rrl!t-!yUi`Z_r6M^Z zU(uS8Mw}bhLM$GHnHX-8&V+5mu7lhut8$9Jvd*~2x5IyI8eAh^V~@~8Qk@$kr$+(D zKQ=j=|IK=161X(qfqXQp8%Zw@eOeK7DfNc_LU1YvMxJ5JM9$FK2VlqmS*p{)Ig66 zy;fSXI*jep8bZ2UmGz9@`ywZe`phyR+7_S9zv_EySDCb06(w7H1mSAus|fq$cS$yi zRt2b06lWanf(Ix?-~zT8yZbS;e?U}E z34jpvcvrV;;D&@PUuVG{N#8^0TV4yRxAp+~lbN2!A1UV9QmCnZMXGVyYLs$UAO?_k z4mI^b_wV?GH;IkQ>0tgmuW&2bGS6l?9Xz;6c6n1iku-c3$eL?d>gcWr;v>5!hxF|p z&#Gs1#rsNetv~ zOsz&2+g*TlDCDI6In^gjm(Ms%l}iU}pt#!+=a6Nb^8M6ZcdN)eMM$)WhT+z6d`L1L z_`)MZJFAKuiS5!4jC>K$yG3_QUc2yW{5yI+Ef2+lCjBoOU89}Q+u5_1=#CvsGf*K42dj@K2- z4JtOCd5+GKOI(jTnJq^IE~m3O@(z^A@3AdLPn8YvVnX*SE`uycD}~KlY&;9=XEZA& z9{<`$>lpM7{$x-3W8~4pVvg*F6{?l6l0|4(!#!sy zCsMp_Q!#4Kxe$~ITQQB^Vh2Drtb%VsSowWo)q|}=dmiL0#f`m0Bjlwc#2jxMu(?fxQ`p>^IbGP?aO2{D#Sj%- z`A0r(Tr~Fqby2AUdO=TTXew{KXu1b#FkuU7c|NMNFSG^~-D3Mx0!_BxulhtGt%ZZu ztF%Jnb+-J{_!rVmlveP{x_phXZ^0gy)H=$p;NIxyI5?C>yDj+8X@J zHK&_yDK_AVbyQR{`DIBptYC0@o zfL(x{Utsu{^j~z95w808hI-?`<8tIZpS3_rmyP2aMr#`sL!$2mzw-4F=~-Ezmw(OTm|DpUOXH@7&W>lPoSvNOz zt1?jdhk%iwSZc5~S|fbvUPWJ`=4wZpHUG%bBh@({nj`MIoYKUQ5TxOw%eOZD*S3et zj~@J-{Q&%oznizppuPrmEvh`>?rVxpBfrux{|`&AkEX*G2{Jnb)8n9Xt@^5&^p(zg z@z3-aPkeO}R|a$<{4vzo6qjDRn*Uvds1ec)75D!DP4Y&0 diff --git a/summary_results.json b/summary_results.json index c0be6ca0d..b6e22e72e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.4679, + "Performance_Result": 19.9084, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8585, + "Performance_Result": 23.0254, "Performance_Units": "Samples/s" }, { From bc47b66df874e81649219e0bf4b6ea44e18091ee Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 26 May 2025 21:04:09 +0000 Subject: [PATCH 657/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10946 -> 10946 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 61e9b5ae0..a0d55298e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-26 08:35:31,096 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-26 08:35:31,097 submission_checker1.py:3308 INFO] --- -[2025-05-26 08:35:31,097 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-26 21:04:04,358 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-26 21:04:04,358 submission_checker1.py:3308 INFO] --- +[2025-05-26 21:04:04,358 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-26 08:35:31,097 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-26 21:04:04,358 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-26 08:35:31,097 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-26 21:04:04,358 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-26 08:35:31,097 submission_checker1.py:3324 INFO] --- -[2025-05-26 08:35:31,097 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-26 08:35:31,097 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-26 08:35:31,097 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-26 08:35:31,097 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-26 08:35:31,097 submission_checker1.py:3345 INFO] --- -[2025-05-26 08:35:31,097 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-26 08:35:32,461 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-26 21:04:04,358 submission_checker1.py:3324 INFO] --- +[2025-05-26 21:04:04,358 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-26 21:04:04,358 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-26 21:04:04,358 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-26 21:04:04,358 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-26 21:04:04,358 submission_checker1.py:3345 INFO] --- +[2025-05-26 21:04:04,358 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-26 21:04:05,489 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13835 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73721 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72081 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index b2cff281f..b12a46db6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1284 +1285 diff --git a/summary.csv b/summary.csv index 9fdc2f19e..42f52a062 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13835,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73721,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72081,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 8752e7a04..ed8a7ab05 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73721, + "Result":1.72081, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 486addbe7cfc510c9794ea3c72e7bfb685e52b96..d908443b8e543056b64f12e6f4cea5ae454fc405 100644 GIT binary patch delta 379 zcmV->0fhd-Rl-%UatH*t>r!*Gb_i(*1E(#&v*Hj86n{WLFZd+^002+|000pH003lf zV^DH$aC0wXZ*pZWcx`N*lFe$vFc5_AeF~$yE!n9<5w-(u$*F-tQ(Afx(#8?Bevs5U zdHX6(;w0cxkE@yec6L#<*r_i01?Q#FHOe!B642ZTsoNU8ub1fqN_=2iu+C_x(H=Zn zGrNPEg{jkY0Jim~TgP|Ro1p`*AlPO%V6*J8`~H){bGgv_ym zzyu2nAAqSHiRgqBd_>wG*9`!{F?66n2aoeC#}{t^r~G8YpjmnYmV?M0ni8m6&<38#WZ>L3$ zeISw;fF9$Xm50fhd-Rl-%UatH(fn?-uFb_i(*17~}>v*Hj86o2f?g}o&K002+|000pH003lf zV^DH$aC0wXZ*pZWcx`N*lFMqtFc3xe{R*SKEkAGw!gio7Sv63oOG|e`nk1su14*sp zzpr8^P6AnVIl6bw+!<7@4zi1W!8)N;jj|*~5h$j)P`etvx9fO`BIl{%w9^V|G=M{^ z=AmMSFn?{~)oKIQ3vf{YDMy&8(cXJQFm`MYlDZ@~R1mk?O6o&tci2$&O?QCvG+kf` zo^t9bJ_2Ji6VVCD*^D$lwi^MQVdy}D@(yQ7hA&I_J~%NO z`@T>5Vzh?*8UEb7yp5~ILLE=V05uh7gjt|o+eoIu*F`v7N1fk<6K@6Z$06u<-cE~5 zeZZp-fJ|}E%180^+^$iRr+FEtWt=bCbV-Vm6xr8t Date: Mon, 26 May 2025 21:08:34 +0000 Subject: [PATCH 658/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10946 -> 10944 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a0d55298e..94f71f075 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-26 21:04:04,358 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-26 21:04:04,358 submission_checker1.py:3308 INFO] --- -[2025-05-26 21:04:04,358 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-26 21:08:29,741 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-26 21:08:29,741 submission_checker1.py:3308 INFO] --- +[2025-05-26 21:08:29,741 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-26 21:04:04,358 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-26 21:08:29,741 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-26 21:04:04,358 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-26 21:08:29,741 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-26 21:04:04,358 submission_checker1.py:3324 INFO] --- -[2025-05-26 21:04:04,358 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-26 21:04:04,358 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-26 21:04:04,358 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-26 21:04:04,358 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-26 21:04:04,358 submission_checker1.py:3345 INFO] --- -[2025-05-26 21:04:04,358 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-26 21:04:05,489 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-26 21:08:29,741 submission_checker1.py:3324 INFO] --- +[2025-05-26 21:08:29,741 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-26 21:08:29,741 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-26 21:08:29,741 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-26 21:08:29,741 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-26 21:08:29,741 submission_checker1.py:3345 INFO] --- +[2025-05-26 21:08:29,741 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-26 21:08:30,742 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.13835 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22413 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72081 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index b12a46db6..b151ec96e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1285 +1286 diff --git a/summary.csv b/summary.csv index 42f52a062..b145e22d8 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.13835,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22413,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72081,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index ed8a7ab05..1c0f5c99a 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.13835, + "Result":6.22413, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d908443b8e543056b64f12e6f4cea5ae454fc405..a993fb405c4bdb42dbc5142fcd4d328b34457637 100644 GIT binary patch delta 1506 zcmV<81s(drRlrrSatH+>J3+8|vv&wl34hYMQ+VNGm&;BGm*|hUQu`s9Kx#iEgIre7 zepm*$BpKk+Wc{!XfJ>9~qe1$CmXHjBt}K-GSQ(yTi5Uc^*qWuE$GH3^vTQAjtaOqs zT0dI1{ElBnrc**D$|J6nen=*e(htcXnH8iTmH{$J2FNryKdb{F)1>@pP=25#B!7bt zGC}zvWq6Qff7b^NvNbC|--paSb$+gN(fP?=kaYv6guUwp+Rrl)WljqWox7M?9(gkbjJ_r1L{Uh-L-nhh+dwk^wYL$`9)R&@>r88jK%k z3E5>sG{N{GWw>jSu5s6_+4%WBXzr=-Y5U(?-JQ- zS!9)s(=KX2+1di%D%m*g5)H^kpTv_I5XmS@dO#$EY*z4qSO(Z68To|MHfcbt13=TH z0okAdK}$#m0goN3daO!ro9qsOZPVU63+UU5?6s_~%8qFlC7^7_w0|6}%cv}|fNbj0=USwPzxAE*Qih+Q`D*rBXP$`H#c7EoaU z`;O(4MD|+NR|!jyfV|QVOeso0%74I;d=QD0fJjDJ(gGqO#Ik|~#4>;-^K0NBYtn$& zD8Qvj1F}H_f|ig?Q-6Nw#W|GqXc-)2EByH%d(SnXbd)2hp5*`3vbIX+*P{QUt?c}& zOJ0<N#aTWhh&r`1t1baBr6C&ECWa~wFZzh89;0lK+yOKG4^MyBR^HiQbh(^F zjr7YrI&Mav=h;1!FSgkjejcqqD6cVJjN`>P{jwfC|D?X|vlL*qDa$BQ)2$o{( z`5qMWS#;v-a=1o7BMed6uB$f*xR5(Cgv+>7#2K0R+%s3}M>OUW!P=9?m? z7-T_9@^u_}*M9}7^N@cNFMw0-`%lpP0kbqF$^r!;J3+8|lLIMR0Th#8DLet5lbk6* z0gIE~DLMiDlNl-|8;HkM|0Mwc08jz|01*HH00000000000DytslUgc10Wy=BDkBEu ICjbBd0GnmYK>z>% delta 1491 zcmV;^1uXi&Rl-%UatH;u>r!)kvv&wl34cLd>`XjE?en=*e+7HPfmld=h zmH{qF2Dmg?Kdb}b(j@(8kba;gB!i$U3uQf4hNoC!2Ei$|X6ffKF29K^TgxIVon(vF zkJc@}Q~0HNismD$@yU&0GTG`M}zVMEq@^y zgpdi!4=KZgEc?4YaFDH8`T0I%?y2*0rHjr_{(`ImEopIydc;#YgT@~7R@KAvb8L- z(xzE-eze`vouKSh*%IYPqdMYA<$s4{lqH=X5<)a9I6o`{Xp#(|X;OYz2Y{x@_|ahe zKugFj8=?uu4=KZ4lXQ){X3fUW_d#<{m7gnJRDSYZ(+lj9-Z|IyO_QJ!1!bki>Je{2*3%Sbp`M#9M^1aR6W0e^^%BH=W3(6vDTf|ig?6Fhb(>(Mf~agq>hoc2EXKYy3V zUdtk@Y@Bvc1IpGG_*TirX_sg~Hu@x<)PP7vS<(X{A!M_H2gEYKCdtSroVG~=VjTdQ zCJo324G3C7G6;C=P}O5qa@%Bg2yC17-dRB3R%EYbeN}c$yC?x=JAbC-XkA8Si3Mb% zPvS`nh-8!{H6Ri~Dl2F}ECW=M3{YvZfLI4WrO5)?-uOTzSU~KufyWMIJyM2PRAY0+j|JZx30i~lHN%bWEreEA8lpl zS6%X=tlcR=5=|0M`adM2EGYny5F%MY0Ad+HlBqR-q{#qcqX3d71IPvg2wFlmP5B`+ z=TO$8Wq@P_1IXTU189R}@FXPnRs6Ac#2?kNtirEHu90jPK7ZzI8ZB<3(`YhD8iP2y zT-w|rF4f_O^~c0L#Iy0@*>IfyP5b_2{h@a0cy`lB{#<`to_l!uySDPq2BXX6 zBxp?tB;#_;oK{Xuz+`C=R|#_5;!==mr0EhqhxX*~RKHL2xn{O2JwSJJ)I(hplPU0lsQ;(~e{dRUywb-dH`32{t z(KX65f)dc&2&vl|y|0()14?{gTCmP&sL>ugS~PbRx0D+Pug+L-L4r>rNPEg{jkY0J zim~TgP|Ro1p`*AlPO%V6*J8`~H){bGgv_ymzyyB_3?G209f|0K6nsS5AJ+{4!7+58 zKnIWWEXNmb0H^$9!=PL_l?;0elh(61x_4fV#=h^fel}QR{v3ahQ9;((e; zaLOI9U|dt->mnYmV?M0ni8m6&<38#WZ>L3$eISw;fF9$Xm5 Date: Tue, 27 May 2025 17:29:16 +0000 Subject: [PATCH 659/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10944 -> 10943 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 94f71f075..4ff87e342 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-26 21:08:29,741 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-26 21:08:29,741 submission_checker1.py:3308 INFO] --- -[2025-05-26 21:08:29,741 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-27 17:29:10,606 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-27 17:29:10,606 submission_checker1.py:3308 INFO] --- +[2025-05-27 17:29:10,606 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-26 21:08:29,741 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-27 17:29:10,606 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-26 21:08:29,741 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-27 17:29:10,606 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-26 21:08:29,741 submission_checker1.py:3324 INFO] --- -[2025-05-26 21:08:29,741 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-26 21:08:29,741 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-26 21:08:29,741 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-26 21:08:29,741 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-26 21:08:29,741 submission_checker1.py:3345 INFO] --- -[2025-05-26 21:08:29,741 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-26 21:08:30,742 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-27 17:29:10,606 submission_checker1.py:3324 INFO] --- +[2025-05-27 17:29:10,606 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-27 17:29:10,606 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-27 17:29:10,606 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-27 17:29:10,606 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-27 17:29:10,606 submission_checker1.py:3345 INFO] --- +[2025-05-27 17:29:10,606 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-27 17:29:11,671 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420161 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421857 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b151ec96e..67d0dae57 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1286 +1287 diff --git a/summary.csv b/summary.csv index b145e22d8..77c024137 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420161,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421857,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1c0f5c99a..e1f343f21 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420161, + "Result":0.421857, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a993fb405c4bdb42dbc5142fcd4d328b34457637..a201dee4a7765a0984368b08830c5c061136937c 100644 GIT binary patch delta 3018 zcmZuz`#;nBAK%O+Qk*G{>qgr$m(*PPCim-RNk$sF$Su>{N+{ZAH~=i}9S^y$&Mb6aE>Ie*GrY>3|y z1p`F!ZhW)r=O=aJ#{@sLc0DOL4;iyo0{#(iRa zs?}_B9INNk4KMypz!0(v7yWU@#uhbOYh5r=k+o}AB-k$dABv$L1_bUd;}W8ViEPu` z_?8KN^%VhW>adLJ4B1Uabu>{%^&qWt04R9w`TanqO{u<)UH87RPE$1(h%-D-pONmA z=Y-090(BwdC!zq{gsQLR+!4SVRbk_rFFPvn(N z>|rkNKRNf}1l)%=ZZoq?be>nDU{1U&Cq@X0iQN=@A z^A;-pyQUkZD!mdS>$hTn_4T4>Z-=#AQ!><_?h>GChhc>%SO7zjo}oTs+@!03nSu(; zzV{qmw7zN@CO@Umk@hNBCace~QPRVqr=e?GLQi+U>UiTrw1#$B4w8Y8rIAC2B2He7 z6m$F+sicMYI~;6MPraIU50zJoy_4ktcVY%ZenH*m*fU%B|8ZU&#!P86 zb-e~4znD{oXJ;de0h()K$Z%*ze(4d@XsHfNXwoTOhI%kFm<|rJC~|{QEio2F6c|@f;4pXNV3)hn)8ba$Q4Ts)%SoZy0wQ#;FQ{?f3YFeidMV&33x>1txEZq<& zM=<2emK|~Fp)&JnArSYkA6qqxF)L|5AYtt92=v5F&_x~kpMKgMy?#_sL->0AQD_&Z zdsnc+m&utsBVH?$ZgV1A?e>`j^xH~EAa9q{h#FF3XM?79#7 zNGV~=4VE4(MS7Gy%sE_MtZFwnK=_6|P_{xgb|DMd#ozQ4?mX&;Il7Lf1#7KyjDz55EGk zbMRZ|x%LRk2kc+b^_?M=sl@ND$E(&sKLHOwL+vpPvo(SpXDCKfcr>zHYdK-H7rAan z#LLtrg%}i`nrRT(^P!<1xVg`HV)v-|)8g(6u8@HkoC6L;J8>eDgO_ij^Xpm z%tHIRs6nIkBy@JwihTtm-S0q2qa)v#Q5w3V!OU}ube)AY8_#`j+kghd+eWsZ??PS z37H#89r{ECJCt-ZFDsQ9g1Sj{fK*1nK@-Cdd&!;9ltzZh519#qeq{wgqZFGV2cck} z(WyQ35;W=xy?y&pYaeuzdt6#8pE7?-=6eL_zQJQ5?Ty#qNBBX0IfEhI1!OymRTQU@ z4sL@4(*tzDmXQ6`5*?6y%+~Y^5O20o2^bCU6dBl&7HUP*V1}`b--KW6Y71QNO}tmzpYUW=M6s}G zqiA@{KYf(F`aCQTxUv41=8Kq1o9;9MA8|TiQLSNz>bvdIJrx0o%zscwAoAH><<$Ah znJ-s_G0No+7{~`ZEA@^V9QBMUcuXJo>B=9@ZK_p4(uGy+%dcn$ADufedJiL(nJn+! zspp#m?-|)r)wFY!Y}_Zu`q|nTkllOa&UP#I+jw*nOH(fDH96Wo#X}FDD2n(aPM4fn zKY8%Tt24cKxDWRou2&{Dy#NXpqWCh$$Hmq@%7p>RpLGh~=cVW11+Hc5F_3IU(zhQ&Z!YfFl%05175vR;q>&@+eY|qcj2}__fGN;eQu*uTlP5~|Do87pWAtfK zv`D)y;P{ELPCcEP(>6w5GWNrTUTVvlQSHv#FC!Vh({;u;tnQAb&Tv%nw4O-?#m4>G zQ0A+!1-!g?+V$k}?d0kIjE@RcKIN{hs=NUja2S!)zehL|iuH%8Fi&8uptqQ8tUWZ7 lIfAu>N|&S&7!@Z(M^bJX!|&TUfBimxeE$4AUa#lt{eC?@jlNC3g7ZqMhW3tv_)QQ9 z{2Tk7T_dsf4MfhlWX~S%iS$gZ8@XrQ}!R-^AlJ^FXK`f*MpRqHTAsWhKM&bwPpagM>Da> z0T*L`V{iU-o%JHKd3xb))MKlu5P?sFcT(Fvu2)jqkKm-X?BVk@!1!-kSF#Ip%rX?a zqpvT`zPh>@f5Qctfiyad+?-x`7Jq|7(%LkB4DF_M1(xBjX11(xz{}Wnx{eYUh@N`B z{)6w|?q$JWY95woPiHUgGimRkS8p}QWE4(p{oa=RMa#I!D4n29bP=X)rlzNUs%U~3 zm)|_t@?q;ge0p))bq6YO_Ma{*H|NLD&lQ>l! zzN51O@;CJmeRr6YyUp(HDqxc8Lp`_MZ<^tDZWqoJaopkBK_Wbjf=EYp5>P}}kxeyT0SDWTSPrvHf?%Ly! zK-kHeT=uMy(V}H%@DTx_3Qoh3s9M*)a0Sn@Z|_d%c&sh2?78|!oqe{v=AJ}11~Ir! zaW@g(ghW6z_tXbPIcs@{VT|y}-z&xvR8Dtmhz~m`m!hYtYzFocm1?`5%Pnqx^*sMJ zyZiw+UM?@TvI^+<^!WCz!6z#3XR&o-lH!juPFU}y7ulok80%4c4e~34JR~fPrNU1~ zFiLHIC&uG3o3Jr;zwGCm2RZHAz&G!VFIEW-p&lFJx_GZEc2}Ov@(qrD??GGnBCfhC z)H${V=$ya({G!p0gTi#ga4zixT!KV|uqkx%?EYQCaDX>}#2WCf=`{>((;whi8SZGi zN$U{mS++!KI@d=IuAAAuVS7FTm5OV_b?UnK;{l$cjkt`)_H~cHgC??f^&rnSCH%k6 zQ|&3J&(I#seE0hgOY~A22%Zi?xMB0ome^k(d0`C}_O|MU`_QS=7l(j(`$5YgTT-Rg z)x0E4g01!F3JWyNL&9h*6@kb{H>B6PO60LEq__KLj4+W5KquP_&abb3uXO7S8A5(lqw)4;VvmbT6U8?! z9Ly>p`T8UZPxF#|<4&H`AvemCwOo$^=BrE(g9rL7M~r104h;*l`2qhhfO(92q=@e= zUNaOb2X>Hl)}Q!r(GJz1^!1f%z*@6`+KEG-P_W^y01R8ni%zdtk+F$cki7O(@yftg%x6J$-G`#di)z~x1Q-zxoOl5sFHQB-6o5CUS(zKV$#mek+**nM z*9vLT#jXO+3&Zu)qk|?#3mreY&;5*ljj3`zpjj4*7-3Ny(o2Z4&>aviYQ`=32h7S4 zA&8opSaoSseE%Xr`~JD@HT)y zgF_}CJ<^@%53ffeM#Pj=fhwKEo-cA-PU13E7!T2$1%E{TiFAZ~*+5=NL3_OnC8Swp zR>rWb0&#(Lvh3rorQS}c4&#TML;{K_RudWA-cqd3h1w9|FEPL3Dq#k_#j>zNiF^}e zvbv3<)Q}p-qyqhls=~dNIAA#S6*e~UVN8!ybetIrD;vvk39pkU&zfE;o_DrM0^fx9 zR2r-bRQN<2dZ|bX`prX)yCu*5>z5p02A119OuCLd?^IXE>;6J1(zlj-;K(2*^RJxR zpaxCOQ;VQNS(uNTL%EE3U06Kl?38v&VIq|ZA+w+uGSHzm1?k2haJtv^!;~ddXH4^w zA+!eM*H%#0$sxm)0n5^OyMpt@@4Xi$yma0ayct3P z!II8ulTSIWS6kVi)I?^a`#59@?Z_d(A3i|&s458e0tU>QN=34skTpDsP^L^{NS!-= z8?8LCHksE|Fw^qs*)lG08P^d_%8wc}SxlvtAy{?5Qa0_!Ce6YQNMUU@ z06e%F_vAOG9kcy50Jd3&dZ&fdrS6CoNW7*vk->ZPO+3p^%Gw2S@MsvprUuT$An-IT zJXi{n5Qrn`WvMxi@@acV{38`+We7x)5Cw^3s^q7PJ5a($N;UhZX0KW~3R zE@~sk=#o*ylj^5$MjU$_c?S z(;2OH5a>VhK@t-4_>N1~e5b>{DA$4s36`^~RdK2wO>YzX{iyn)u!S;K9K)D;CAKX z^plt6YAzbEZtr)QvxoO%&mKYEGbdP+L};r|Jfkv-BWX5v`=x$B;r<*G(A-7${(3{QD(mnFm3&*hkGBp!w|E=8n)z_K^8L=qg*=0`(tLyoEOO1KZDHKh%cJ MwJ?T{nL@zxKlzBG=l}o! diff --git a/summary_results.json b/summary_results.json index 001fda595..9e50a449d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420161, + "Performance_Result": 0.421857, "Performance_Units": "Samples/s" }, { From e6f1f0f1caab1a410c533896cf09136a9ffcd74f Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 08:28:36 +0000 Subject: [PATCH 660/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10943 -> 10942 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 4ff87e342..282e5dab0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-27 17:29:10,606 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-27 17:29:10,606 submission_checker1.py:3308 INFO] --- -[2025-05-27 17:29:10,606 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 08:28:30,715 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 08:28:30,715 submission_checker1.py:3308 INFO] --- +[2025-05-28 08:28:30,715 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-27 17:29:10,606 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 08:28:30,715 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-27 17:29:10,606 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 08:28:30,715 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-27 17:29:10,606 submission_checker1.py:3324 INFO] --- -[2025-05-27 17:29:10,606 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-27 17:29:10,606 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-27 17:29:10,606 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-27 17:29:10,606 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-27 17:29:10,606 submission_checker1.py:3345 INFO] --- -[2025-05-27 17:29:10,606 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-27 17:29:11,671 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 08:28:30,715 submission_checker1.py:3324 INFO] --- +[2025-05-28 08:28:30,715 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 08:28:30,715 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 08:28:30,715 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 08:28:30,715 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 08:28:30,715 submission_checker1.py:3345 INFO] --- +[2025-05-28 08:28:30,715 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 08:28:31,887 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.80695 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 67d0dae57..80a610058 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1287 +1288 diff --git a/summary.csv b/summary.csv index 77c024137..66742a818 100644 --- a/summary.csv +++ b/summary.csv @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.80695,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e1f343f21..2757a7a52 100644 --- a/summary.json +++ b/summary.json @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.80695, + "Result":9.42443, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index a201dee4a7765a0984368b08830c5c061136937c..ddfd6277593b7a2a2e1c8253fdbbb34abfed258f 100644 GIT binary patch delta 3763 zcmV;k4ovaCRlZfQatIEu{uQ>tMP#Zr-Np z?6BQl^l#%yZ-3EBX7Mz;Wthw-y)^rFe$!se=5cSZDw>Sj+OoX%q&J#2*F7B0tMri! zhogS{BIz$D@ibj`GLOf-G@Jh7b~IZwPmTto?6!P8jd?sgZa!0or@z}yvwd>38vDa2 z{ReX+q~n(?PxHF^9$W<^p}e?nf$)KNo_hQL+Kvh?%T-}2^sZtYFO z<$XgVTYpD>_3lpWqxN0i@g{L4OWw(b_p;$bp`o44XFDm)C#7Ky*6tiGG2ysknk3@w zM&6N#zZ-c^BEfFt1Bry&$j)+YzFZqRP-3~55c9lxH1Y1y#QR4RA54jzl{$Q-4%BP0 zQcQ$mrH>}wJ(_rr33;cQ`q){iOIP~L#rsq|hJT-vh`8TRNkm+*7bGHp{gp(V-N;K4 z5j?#j5y8`I5)qG}HzXo1eTPKE{da~DEv{}vBI4?vlZd#wT@n#j_brKtt9wBr;_CiJ zA|8LhdHezAK?kmUz+uD}7xok*wpi~A5)teDl|;mPUt&aA;(_q$K0|&Yz9vJ6wY)Ku zcYp40yWgSCWJ5$ID3LgqMO`eCd(KzXEV{4pEZS}vBd0nmNrPhaxmZMeRmTYN0=gvp&#DncQ zMglR7Q;Yz2BqF&Z5y>5ih)3Tm-+&z{7u3Enly|UP$*t(DSTBAYM_BKOD|zlqCVxqd znDSdPEiE>#3%^DtZ*H%9{roA-LG}!0-POTOM&#^En#}vR*LS+*wXB2wQfa3Gi*+oe z0*mkc0*g-2^0ei06a$W8fGCOw&~uCg^2~)1K#@cwiXXU_gX_Ies;uwXCoLq3<>mTO;bWLf0ugx7Y=f zzyAJA;p^?LE5XD};7BmZ>woqDCdmMpBm-b#26xMCh=NIaKrjtGbd-FL;S5?LI*KpB zlx1C32D>H+!M0ggz&?Um*fVo7wbfEjK(mfDR@po^6H}X)p<99Pl{}kN{9UML3g6gq zJ&7h}0#~Am8AOw008NqsG%-Wa)Fw1Z4~V9rhtBrK2Q+0%L`MNMWq(` z+r%d60ogS4(AnPjz@}`8=qQ0?Q@v3ZJQ-libBo64JDhH2?EI`W{^#i0X9hn z*kmdVu<4rEWTp+UY3QM|z43uf*%HxFLdm8q>#{Q3HZg?TrhoenWAiCT+B())Y1?#* z+os#{LbnXf5`Rab7~)BRhhze2@Q@6mS;66989nsLo%VXd`Je_tl;^u z46sS07G%@p`hTz_$nR^3oY?|C3+5@0z(tnhc zbIzb8BBOxg165sC1-mAhKkk|}dp?hlDU6@5hjLHdpMNX0EB`pI@AxH7tO`r0M5)A+ z`VYy3(*GeDq_Tnm#4 z#nOKgmV0UgU8!&vOFIa{lFd?E97~Nli6<=}k_n{+L^6OS%SbF)Mq)`aO8%bID{ZoX z*eLRMMG;ILbTmjn&=Rp}%HJkBhq5k{uMik0lhr{sKfz|FJpJ`l*0Q=PCs^% zBpF{~X%c@}2Y{u?`_bV2KubghL605Ex}*#|cGfj`?7C*}2eJI-B6}@MtRk}OqW0sp zv~~(FT4AE=|@C>j1bkNk1B-A83Dx$ROy- zLRpuU;VG7wL2!z#S^9a5%Wopf*0RV-C)uL)qjf8Af-*AQ5;9R9@uc)aGNF`yNCwHQ zApNimkV!H?rpftX9RQgoXN&JW7~nj`~gnv@^b0ibCzel!?A&=RrBhG>HEL&|X1BwgdKS+jrf^L^0VQ|0GM z7nPrU*YrcXvuhx~vTMO=1SYzFD*M^Yw5(d&Pvsz0l@P5iQ#XG_c@T0Fe;D$ubg7mXUC>2?3n8NdRJ_NH|R$bZiiS zpe16{1dkocy0lDgoFoJrr@c@9&)+4o*Rsee8>d~=fU>oPfmMI9aoQyskc~cxFEt>N zQI_<8NC?@i-~q7=ut_rV38!t+fLI5Brbz>`K?8!8hztTAJ5+U9mE1Pj9Rk~?y>}MS zw-woISznbM(=JLt*^X(sT9;8-VgcFcllamCA{k{#4Tyx0$_g3~%K()m15}zUAl3m; zX|jN}H$G4a77%~CY~Zm&S(lU{mQ^gE!UFak%O{EKwXClamLLK7r5~75lz>$5fF=1L z5-9(Vkf$X59C zKlYw$KBZUAkN44;JLzKTEgj`*Wm zmR0!m$TgDf!pFQ#;`vQ{8jr_|MnB0e*Sd1sy8i6DOLh2R{V{P5@ocdi4C0`j+F~$s`&4xSG^*I{I@N|M)}y2k-EQzW3A54gFK&xREXWKANp=HA?;k zo7TzEtalTi_vSaF>7p@?huK1{mfvj5*B`yN`Z^{}W~;B1aHt^ew3XC{ z(l*$CQ1(L`zM1?}V>1!afaGjKnqS-Y0M0P9AVGPDvn0b8uLmpLY(uYH zIi>JN1GCmqoZJT|CS%uiNw?^&A%BLyHm~pfsE+voR*h0tK)Bwbgi%DO&*&lUyl00i2VWDM0~=liMje0sE5{DkcPy dzj_~&Kq?ji+>=)-J^?V3mMS9#;wJzA008uCIt~B; delta 3752 zcmZvfdpy&9AI8nGM3^GPhHcJ?M2V0&j3ksMCCO=;QwJ3G8@V&eX+xTfOyo`u!<-T_ z#9B@{MJbGg+>Dsxv!46;^ZEVx`{VO^U!Uu`KCjpJ?Zu{x<+t_-5tIKGx=*zWR-Ja)Zse z(^vaQrt<_v(dZ&Gz%YLVs+8U1PeC z;>Wrx5&uXtnnCT2kZdG~Ffj)9Lpj>{C^J@J?pX2+iZkVZ;$skVs2`1_PEH@Y8?n*w z1w(gCcD(@{vdJaB8%m1OfK0BZY-Jm@P8Md^jHX7{k=xty^&VSqvtdBvW3$HOcB2|h z*vPXt-ja^Qfu^L{6AhX5BU!)AAVv$mll>7{U#As~f;w8Sr(gTwd!zhpU&O}d2BLb_ z^XlJY4zFpS=*IrV$?O2NCbPF^^Bo)_O@9VF)ZM`BPWej#0otyrFb)pa{nGMvB~$;s zJHX85)KwdaKUot`!YTsXfre8RDWAjdb!0O#wC|}Yd}ThA`0HVqVeUeKL2mlPljw%q zwG3p3R6k5A+-Nx&`R2>}Nbpj)D#R8xY#o+)lf(Yp69Zz;P= zB~`7LKF8eGaZOAxFInkjG}uTl>M1W?aDklCy8FW}_hdwFV#hzgZLrkZrbYr`-|twY zV0jbK*3W2}h!k`zvqk>1LPj!Fe~2d#?8o6!p}RTPk&j*}5DB&QGcc9XpcEuyAlhDK z;-^O{?^jbAfe8sY0yNk4#Hd0-4lObMGHoO13~t6E8TSFWRK?dPhnzAapl|BCQ*#Mr zoswQU!D8$$z~<<$@R!F%B#C?NU>|*)BpyvsIGHQ2tfnWfPg6-lyG`q>`ceI-Hbd_o zwxe-+SSK2<;;`Zqm`LN!og81kocUMxst#RE@uT{uUJ&n>q)-in^7 zDhaD6MKJoA0F5GMs6QAS{b&m9vPrId+q0B(!tTN8p`W%nbtHY?dhB>U4U zl#a=n)@pQhrl6kk@1oN1zHJ=qYO`S+UrXgX8Z$VbhN1P1O z!z&Iut%$38ndbelKY&Uat{Nv|Fcl^HcE^DSUW)d`_z4lf&%Nr@=ehL5HUb~L@YH<(nXigh|%iCo4n@VLO^w0Pu=my5#u89d`?Qd9RX$gJ9?MNKcs z98=_O!C%VOVgN}s5z@}z@!fAXUw0SY)h^?Ydtw24m&bMw`i5TH)i_6`ok)Ha)&fv# zpX8Dr>ngPt!32=B$4H_Nj0B@jJyI{DD_g@CEW9v3pEopy?n6|?-8oP}%C&S$N$&3&4vkJiDKobuQOJL zfzsIpBHsNymf?|;0@*Zp0d<||;5K-}r#8@QPmRMSq^%`$vyCxA6Z7(T@aVCC=s2nvq!bfL%H zNjIlyFz=U$)i{WKrvtNFt;11WY5yYTOwebm#+o%OD;eWX>JMw3zbVUIlQ)4q@OKS~ zKnwSH%Shy5`XbO0gu>1#r@Uozds`cjLCdr@*Q@W;u!9gqwCmH$ zMPY{_b1@DFkXJOgsb`@&5Tb>lR>Wy-0W7#!XvClnI9dhzikEH|Cw4jh#vUsd4aUyr&pydZct_BB0U_=!ShdtA z_zEq?`|vqTe{ELCMy$pE=JO}7K;Z0WybyJ&e_J=dm~yeD6g8dF;LD=wGr6g%c3;O9z2?Oq~_85tWAoq~P4%5nv?SP(DaD;m{(@ zFHjr3&~0P`_Y@}JO@0O$1f^n}aR%yUJz%26o~LW(BitO!)rl?tDjSp8t<=Y~f)&d%?|p@4<9+ZBh4alfD-pBtN&F!^)#4v{ z^X`zHGyb9}X|+a^vMkDj2er4OOrJTPMTQNE&9DbN`@MJkj{rc^wx1)8S6O48%UdUA_xs?YxB~i3_np0ON>4#Qga| zRUd-S!F?~w@zEX2D3wT^xmR&;TXTLn4KjQ617sl{(J4W7dv8f?xqt!nRc(VOQp)K_Ot^*E5hg^?XmaPQK4DT20 zDfEma{enS(pm#?l;})Y6->)%3C0|q%q16YAR4!^>bPCRF#1E|7&_`2i%A}!qMsdgL z2i&QKTPMd~A-F;_FM1EExFv&o#`YA~9Na+b^ogXeH&=Nk^`3ve--J9A4zEd95DET> z4zY-JRI!ch3j8Zf8J$=$bL#vDzuxE6TD@}>lJ+%iK;}{~UFgab*VI)q%QNP?Qr71* zLb7d(O~H03i%;^frGV(2f3mypI+3$+Ie`t?YQ5e0AYqt#Of!Woh7I}{J z8;`edC0-ReuN6(;e>#VBDVBR)xjMmAQ7XNDT6;3V2+VMiT~!F~u-YFDB_6^nO;FOi zJ2}h%Sj5(!tRfQ+j_;}xc0DJQ- AN&o-= diff --git a/summary_results.json b/summary_results.json index 9e50a449d..0ed42a598 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.80695, + "Performance_Result": 9.42443, "Performance_Units": "Samples/s" }, { From d53cb2188a09c5482752126523598c0ef32a7d2d Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 08:30:32 +0000 Subject: [PATCH 661/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10942 -> 10936 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 282e5dab0..7f40e4e31 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 08:28:30,715 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 08:28:30,715 submission_checker1.py:3308 INFO] --- -[2025-05-28 08:28:30,715 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 08:30:27,506 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 08:30:27,506 submission_checker1.py:3308 INFO] --- +[2025-05-28 08:30:27,506 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 08:28:30,715 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 08:30:27,506 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 08:28:30,715 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 08:30:27,506 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 08:28:30,715 submission_checker1.py:3324 INFO] --- -[2025-05-28 08:28:30,715 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 08:28:30,715 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 08:28:30,715 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 08:28:30,715 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 08:28:30,715 submission_checker1.py:3345 INFO] --- -[2025-05-28 08:28:30,715 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 08:28:31,887 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 08:30:27,506 submission_checker1.py:3324 INFO] --- +[2025-05-28 08:30:27,506 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 08:30:27,506 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 08:30:27,506 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 08:30:27,506 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 08:30:27,506 submission_checker1.py:3345 INFO] --- +[2025-05-28 08:30:27,506 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 08:30:28,586 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0092 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9368 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 80a610058..c83b06780 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1288 +1289 diff --git a/summary.csv b/summary.csv index 66742a818..f25806c0b 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0092,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9368,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2757a7a52..12159ee90 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":21.0092, + "Result":20.9368, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index ddfd6277593b7a2a2e1c8253fdbbb34abfed258f..0703fb74762fe74d87a7cf4ad08d38a4263fbc3e 100644 GIT binary patch delta 605 zcmV-j0;2uCRk&5KatH;S{9u)Gvv&x$41cyGkhDFCWV`S&ZCm?Y`_H~A&meuUx4U$YVx*bW497C1AL z?j$RJ*(SvZ+kv*^)IgyrExie8lZaYBNNOFweHABh638jX)y#f7yQrS;RG0jL^U~-B z7 zkMk_YXKw(f{LO|@xo|2O1`9W>CvkG`yqt`E-)DU}T4Vkke=T3%$5m6Q_ow23+FEeR z9k5_rTjR?j9U%%(s{agB$02tUPk=-2>rc@B1G6wD$^r$P{9u)Glm94N0SuE{DLes|lb9(% r0fUp;DLMi5lNBl^1alTRGm}3m76I0iRw_OLDwCBeBL>_j00000XuT<1 delta 649 zcmV;40(SklRlZfQatH;l{w#^Y`kPT80CM{zBgWfs9idm-ZU0}u0Jl%J3ReeTX|>w z;pK81H_|V&__!H=o@Mt?zSw4C@OilYpuEOxK1$}J^vinm{FC~YAd_tVb}{Zr$(kuCi`nyqd%O8y0#*2&SVcN3rY<~O71qA`w#*+Q+B z-)zj+AHBEwI!$J)uiR$iW0I!H(cxHpLB`FGjRyJ3=~_uu4~Wcv|@Cw~F63>Mf9 z1d_jcACu@LD}UP|#R%Jhw&c`6p(!oB32EbqT0cl?oxFV&$959%DaX~!emlFUTpeT^ z{eg8ts|saFiXu=Bo7D!a7vQ1*QjRcF zp}qHpVC>i)By~w}s37jNmDGpQHrP=1LmR+(nm%F)o_})cDLw&XGZE2%qfL3xL>B*Pc42P@rdL$6#prSL}sv({0Z+y^HnW7l;_x9F`Qe}=y{ukZb;u~4T| zF+g?68DSQv*S0S4brBBNQs+0}#5)1}=@|5xw_%ZUAMhvyAm_NF^0|0^*{)HYr};8Y zmvLTf(;0%~1zBX@r;#`P7oZ%5+)W$-=iIlSp#BH5DkjPT1+V_K)p(QnC|dy%lT|4^ z0i2VPDM1a18~^}#Y%g?~cPyzq| j5dZ)H0000000000fPvhTP%1tFFq4leBL?Cp00000Mr1&1 diff --git a/summary_results.json b/summary_results.json index 0ed42a598..b9544114b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 21.0092, + "Performance_Result": 20.9368, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 32f10367d52d7525f8a53b336aef2bef46f1431c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 08:35:41 +0000 Subject: [PATCH 662/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10936 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 7f40e4e31..031b858a0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 08:30:27,506 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 08:30:27,506 submission_checker1.py:3308 INFO] --- -[2025-05-28 08:30:27,506 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 08:35:36,019 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 08:35:36,020 submission_checker1.py:3308 INFO] --- +[2025-05-28 08:35:36,020 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 08:30:27,506 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 08:35:36,020 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 08:30:27,506 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 08:35:36,020 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 08:30:27,506 submission_checker1.py:3324 INFO] --- -[2025-05-28 08:30:27,506 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 08:30:27,506 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 08:30:27,506 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 08:30:27,506 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 08:30:27,506 submission_checker1.py:3345 INFO] --- -[2025-05-28 08:30:27,506 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 08:30:28,586 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 08:35:36,020 submission_checker1.py:3324 INFO] --- +[2025-05-28 08:35:36,020 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 08:35:36,020 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 08:35:36,020 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 08:35:36,020 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 08:35:36,020 submission_checker1.py:3345 INFO] --- +[2025-05-28 08:35:36,020 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 08:35:37,027 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421857 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416271 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c83b06780..e6417f67d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1289 +1290 diff --git a/summary.csv b/summary.csv index f25806c0b..cad361ae4 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421857,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416271,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 12159ee90..01dd80eae 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421857, + "Result":0.416271, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 0703fb74762fe74d87a7cf4ad08d38a4263fbc3e..91381bd00a015c75b2e007569c988ea52cb88a68 100644 GIT binary patch delta 3050 zcmYLLc|6nqAK%=^+@g_&w&g5oAx9gA%}lP2sR@}cSvsUt@)^mITpJ6^F{0Eu42f^w z@(tA{#guEEj_Au*j-ol{_woDv@q7RC{^R|4yx!0E^Lo4=?`z)o@U)V=!iF|(T>Uwv zld^#Gzm{)G-!{+ZJI#h(%*n_3jOcD@HOeGR6Y!_KOow|Gh4!IdLixrMurVX61E(z= zE&H+%FSYudhP}&{nZ$rCH7jf%Qdail^9T6?j(9Qz3wX!nxm>24i$BTI+P%y~&rARP zRT_KVda~kK&`@n_xDlzgb@<}5)=HfQ=i@;0rVd8cM&ql)(4U*OBs<52$@w9*GEFlH zxMd`^NO8X9jy%0KH00u`22SnJv znnSj5*x~+=eC}{alCN!Y0<4?~(-hch6WRGb%xlc;l~0D!HWoH0W>(jxzgc);UDU1J z}=^99v69pL4oDu@wPWbRl74rC<}QvwhvD_y;tFP&EAFHqg~mc zbg8L$z`(z0LyPa%=8kPeE58>1P&snq990<|DdUww3~5rdAu6;Y;Simjj}OJV>-vaf zQ=%uhRihVSf2?1bs9a+;$+sQYMO1p!^;~8_>-F=pVpi=v_63=ecwU|G8_--_H1q`a zQ6=%g=-|!aA8syzzhC4`5-~e;9&3|d8Wrh4~>< zN0kcm&X$|7kWd;`%*F?(0`pCj-Eks zid^l83Ij-ZDC^1FEcy@{up?@`s^PAaq-aMY;k}fR96qm8cv9T18OvF~UM2s!sdL;Y zhSzX>sN?ufx`cgb?s@L;k6CQs@4jY30!Cms^AxYMGXpJvtYZyNtN32jjdU?CN!#ow zHbit6SZf){cNc)Xr-{+u4VuW~SVrW3=Q;ivc-|5Zf_Ue;*O{x8t?v|=;jOGIW~^`KBz&jG#(}j(j(lWC3!GDI@Ylr#0xDU0E(i1T{&b{Ih$r%Sro! zlK6jcxcd^09@X!vkR$7QI-9Qoc|K%F;VWlFaa862X6%GcsXOI5TX57$bugT%-x8@BCE4Lrz0Llx3cB}C!9T{F^3eLgGU)}2 z7z(vehquiDcLx}Yu{1mX)$6XdSr2mdCKPYW#V`m8TwW-o4u%uFHn8Sm;NVRR5fT!u z$*4e8_7;e&3gRhnH;OAnv`ZD|bwrP6w&J!d;bv4bJCq=)$IcGiLh2wiFd zZ|A41g8HNCk3mSR)izJ46?yJ0Xm~4EDZuTO12+@e$rKv3P5IG724K!!J|>-@&c75) z8$P&3`ZJY;Avi}%L7}=pp|)8TfkGW-UfW)W$;9$xTScR*Q8P&uDAUUoFW2iuVYAbu zf%92v{o>_=Npbmqu~GZA2IKvgXM=;CuIO(&+kD3RF&=1V4T0bSmH`v^C_iPP&9)UK zfLupUrpH4Lm)u0Ci8LDU5t?UcTri6>x}?{}4jp|~tIjXR?DJ_cl!$5;y=SSXWrKq& zyf{|5*eo|8V9BJ|&l`)p9I-zy>7c?ZB;2@nbmgHs!-q9jnbTMf8a4syhFFn;x$C;l_HKeHR!PavN%+^ z5*6f?Si6#Ujem3A-7OhGnm{TNfv1s z%74M^#v0rJ$4y|WEmUdL{b|eS9hXBt^C(M_3l8O{Z+;B?H6Eb zUFAR8zCB&S1}|ZSi{53iLnvu(wrS?}RRq(Yt;--NeTjf8Z|x*F45mVS#Yjk$$_KePuyVGFZ+w;jU`4n1$R7@Hei^ZvN%|lPy#cyQ!$RVWzcQF zHVE%6K-k8uQ>j4MZ^M{!v1iy3()PNYKGJ~@YY~RZ@Oqvlz=CJnWOh zIU}A+G(Cj<$lLm<%|d-K9kxfKA5qX>)a4~eEJ&i{mH`2uPk(SKMqWl7UYJRh6p0tg!@}=VvRdDZBilyh-QWN|2R(vc!9=XAhqP&lOw;k&;fD7r$Ii&4@9%Q&fQ1b`zx!T+ zr~joA5|CpPSao?ZEp%!Cii{_oylp()Q1~ugG!4{or#REl@#vOzZRfFffgfa#?$qhY z7^QobAjImBH#QB%uIJP1k!|`96fHk~Oml5M?frGH!gTP~Q)dySMcLGT6m`czcPR{r z4f&L|?Q~CAPMyCac9ocX>lwQ3tiSva|4-$%`7d%1{TUf({P#7OD<^}W0JLWlg%yGl1n&7PSVHRGg#OmU1a9btn`$QAozjZ+`#$o`0S{p4aR1c|Y&Z`~7~N2R`?G8e+C=yDbYU zh|vH5STq1^2iV%!9ZlRVZcxx?yfB<@_wl*QT-UYFhmI*(=bX9$B%n+l?A8_~!&)&$PHYN5PPDiWdU#mX2hTlpT+&QOcfQSIR!wx6lku#pptB1~bgE zRQqCYC@(zwRhyLWgvvshc%rnK!9k>a4nR}+>qA|Ds*M15Z&H79hq zewrF=IBh^; zIwvL9`rEf2O4(oD!f#Njn77dh*!`*1VHj(BZ$yRtSS~;DZuWD%MQ7KEao4@V*<}DD z1d376pkP>nI3hER{0nKX#?d%VQtmtWW=pG1PDr3HtR17)*-G{6uuG$k?2E;>yz+7< zHQn6rD*F!C5AxO z%;Z&MJYRmQN(iAixj+aZ)FMvjRzCmYC@bvK2EdKyH;OLSmoOvB0^3?=>ku~50NxAT zM>;dBEVpOMr8J#=Jqo-fC}=qA`_+86(Jl}(1CbDx6dbK|@WPqXW~sDOh4EBmWXblw zA_nCyjdi7Qm3PkN6tudv-;cO%ITl0tdBsz9RIY$|I04GUT!sEq6c!BYMycXJo2<{QuQEpD+sF34Sh$C_(#XIrMgD zpi7F?l!TbjJjI1=$gmNbxkkebcZ9~%XrND5G(G&9>Lx+-dpZbtxLE=Ijg58sC~r;k z+#=TH_2D5u`eMYpQN}jYRHBEypB%dtJ-*KeXX8! z#8)&Yxxys;acKj*fn2Re8=PEiFwApf&sA05h=o~d>pP2XfAkot32BHxd{=GlX-^4~;b^*$XCnJN& zZQwE7_gn0NsiSQqsp<;a%5SiN2WJ1IGbnWmP;EVUXHCp-f!cEwST1(Wf| z<)tqzA_R;PvxNlb69#bvS5hCMkuDm;9L^Bzv1E7q=?R{otm1n0`WDVzJ{}l(U-twr zf&I$fhS(WeYrFoRk&%@pHL7o-8z7xfsIi)BZiN&NNv)WntRRi6uM-U=Ri z6$LSOFik%jDWI=A;xj{3)-pl>O6V(~5VMw%qvXe(Ec=|J>6a#R1Z&kcH_^DVxb3=5 z_*uw?7w-EjS%3S7T=v@oKO>7Ss}+UiT^W$=y5+$oqOP6x-3$7`?xT=rg6Rm*PnA(4 zTmCF+?&6m}!PX8M-!--M3JbEwZAfz#xKAJxcT^jNF?)=sMGn-Wf_K=$V*9A9JJi*y zH^i4r#6yCfIx9~ag=L=c2!llcP* z6?v`hLOM02E&YvWC_;ma`Ir$8gmF1W8Y0bRCz=)^q=GLNMGc~U)(YqDw;-O${=oKD zOoB=u#2qHSj-;F75`9{or2W6$T=A0bLWYn(W~6KNsz28h16&kzv(gX*ET`jB2-}GY z1&ObRrG0f2MGY%rqMq9Hbuqc`&8ex6$A2s+OB!BsY@bZv#WNSH`VGI;$&fd zlt2w~yOj(CSGgW^Gl&bx4`lUNK&{iX@CYuf)r`c#lsS5A9gCJgy->t07+sb5RnHjOP_p{MQq)j*L;*o zYyV<>iG3wSr|jv(*UEl(fw-mqO=~@IZO+Re#Sc6IzHzkGx4O0Mm~^apK4;kILeWj$ zft)90v$^8MZSIP6`iBZ$d9brPXGx5+i+oykUUHOU)hgMZWLQr$yK7yN%aWR{3k;8L zR6m-!pMWRBi1nD3+c)K>pAzML8=XgF74z#$v5p%r4@S}`k8?x+2*_c9oxJ!M!tC3w zj}N7Xdc)enZcLnQ_4s4zfd1k)m-#!7PO>GREbd8GimU1P2OD!Xa~kij$^(}dBa2Zp z>j$K0NlN-V;h824y^(Fov3dvhv1d?5VDE*$_iCljC#zSrTS_obwv48>PQD~nU@af2 z(b<{)MAr)jkCT5ejYjlfyI(y^Inw@H>AZK5EOl{U8a9QrR||_we^~-c=IhaaMHm(9 z@%Qc6L14HnE?0agjF}kkMQ+{W`h~1acYhGkXQNE9UQXJYCB+?%(;a?_)eD}n0Y~ui zzik*>9Sj@OI+VA<=TqzclR2HKo8<8N%y16SX&qNiQq5h&b)UZk)XI9Vdp?LVN5Zxq zxMLa}Gymu98_8qmlMnfylkWsA&y02ZoT~d?`=0UIKT2fK5Og|VLTqAKf6fj!!5l_( z4QhSUztru#E!mj0oniRCl=EUi8iH;iMc8LikB8Dd@3=Zs9I`(p5*L2ofO~iEO3-m< zIL^(w_sOA678sF|%KyIN>{YV>2%PPUwS^S2*;sqXHTEm)VaPYOmN`a$lT|bX06dfc bKpXh~Vx$hSJ Date: Wed, 28 May 2025 12:58:27 +0000 Subject: [PATCH 663/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10937 -> 10938 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 031b858a0..5b3bd20c7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 08:35:36,019 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 08:35:36,020 submission_checker1.py:3308 INFO] --- -[2025-05-28 08:35:36,020 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 12:58:22,121 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 12:58:22,121 submission_checker1.py:3308 INFO] --- +[2025-05-28 12:58:22,121 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 08:35:36,020 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 12:58:22,121 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 08:35:36,020 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 12:58:22,121 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 08:35:36,020 submission_checker1.py:3324 INFO] --- -[2025-05-28 08:35:36,020 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 08:35:36,020 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 08:35:36,020 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 08:35:36,020 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 08:35:36,020 submission_checker1.py:3345 INFO] --- -[2025-05-28 08:35:36,020 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 08:35:37,027 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 12:58:22,121 submission_checker1.py:3324 INFO] --- +[2025-05-28 12:58:22,121 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 12:58:22,121 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 12:58:22,121 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 12:58:22,121 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 12:58:22,121 submission_checker1.py:3345 INFO] --- +[2025-05-28 12:58:22,121 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 12:58:23,203 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22413 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72081 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74137 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.0254 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index e6417f67d..438db35af 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1290 +1291 diff --git a/summary.csv b/summary.csv index cad361ae4..3ed8b5d3c 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22413,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72081,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74137,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.0254,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 01dd80eae..9378fbd84 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72081, + "Result":1.74137, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0254, + "Result":22.9802, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 91381bd00a015c75b2e007569c988ea52cb88a68..7e508a89e1bd0554bd5fcceb48d208141630dc60 100644 GIT binary patch delta 1613 zcmV-T2D16NRk~HMatIEGst;>)4FCWJaR2}q0Fx0Z8nc=RL`8MNSh@sDU4kWQB%ZW?NG6o_56K{w73?3D0W6tS16Z2uA2teL zX_9}oH$Gqq@(&@MCSVEj4=IBaEbAIhu!RNuOR?Nn^GENRKiOt!d1Yr6Va*@`xv;ACd{B^g}X8W(DboWq?eQ0WwX_59tl<2x44_FefTl_LVI2UPCgVqg@dGUpyKIOi z7(b*8cTLhY?wU0lKi>zZOFv%^=TnZfd#e6i z>Z1CS?VFbN%U+%>F@H2FB)&9%NJd#w{UIS_vx54=GQcJicYsZk`ol&6HcjS_2J;76 zB05cA6U-k{h8t%U^GDZgfBt+QHuu&4(bE4>zE|>e(=PFUY!pd+>Hm<7vZMe+LI5Yr zNH|$W!pSBCaM~sTh>arQG&i5h)okbb|~x8GP!Y*5Nw?GKKVa?m&jhrBCBki zc2NUb`vxT&r(L1}+31t_QUf9xWl0Z+gpkb&9uUg_nld*cI@U;(kq1|B<EMVWUe3Hms z%lax|2@;TB`hh7$e+fth4_J~9B9Rgh$tX)&KqQ1%RHJ#sf3%gIUvq9=LXOQ$?!=??yLA?I|51DlSsA;AM-Ye=Qr_bJRUC^{Up0w>&k8G`m^sY)!~Qr z$HYCvv+-d-c5Yoo8OG4i^ez}W(&1iezP%OfArq!>ol3IzH*z5 zk4c&)ll$*(jD_FfCYXG*qYukCydA$t~r^!J9J{T#(?C2-2Q%TcKSD zmSXJr4ixiQbm%B?~cPyzq|5dZ)H0000000000fPvVPQ7S$GER&Eb LBL?0l00000%Wd2# delta 1599 zcmV-F2Eh5cRk>BLatH+m6~m!(vv&w*2!CrUD0$&Ws}d;DB=MvHL^7c?fJg?RtY84K z3_!`u8ZDtT89;0lfYKxYX%K**C8E=m(kzs9Ss6lEK>*S<3qU(e{vGCQEeouKWzqi8 zcFWRk*Mgc5M^3V3h2P{GUp&-)) zEJ6MuWpILJUBd~suz-Ilmiub{=w0*2$?hAkj7yV#5t)C{x)nG<8JTVgnJABVQu-m8 zP)a`}gJf2aepm*`BpD#nfJ~F}qe1zBmWT{O$OPqwl;J^^{aqh8$kwd=CR z)cLv6Mdv4fLDmc1l3&ZzUJ07yPx46Vhh#!2{g4czSwZ??89b)F9Yr1L{Uh-L-nhh+dwk^wYL$`9)R&@>r88jK%kiP&XBG{N{GWw>jSu5s6_+4%WB zXzr==tXcZ`dN`kQq}@~X=TaBdpViwYe%Z^jCFYMt zg~XTU56LJ?sy`%zY*tWzSO(Z+;tsHBQh(Sez^2Lk(O~{SOGKv$Y=Zei%5dYXV*cow z&7Xho!{)yFKU(@f%J)j1ZrUaOkBuUUFZ~~qQI-^dNC@C$83`xLNI2Pq08ZN^0I^Xd zoTd&sHV8n_60vE5#|~v(S|&G65`vA>-Y5U(?-JQ-S!9)s(=KX2+1kRuD%m*g5)H^k zpTw6M5XmS@dO#$EY*z4qSO(Z68To|MHfeuAtOG#PqygEW0YOVd1_6&9s=BO7Zky~5 zfo;>?I}7OBitM$lugZ>T7bT!<$Fy9n%cv}|fNbj0=USwPzxAE*Qih+Q`D*rBXT$`H#c7EoaU`;O(4MD|+NR|!jyfc(-AOeud# zKq`2^l6(+}lz>P^S<(U`A;hwR1;jFdCG%_GAZyZq*eJlINdvM$1A>-_O;dj8#W|F9 zX&D@3EByH%d(SnXbeJQlp5*`3vbIX+*P{QUt?c}&OJ0<>kP&+iVOz57&Pml-HQe zN6CDYep!#6e^TFa+&h^hgCAFuT24oQF5@46=>Omy{?PY+`njQhY8*GRrQb)h)vZR! zzhKikIhyru;`848W;9(i#_=#)sMYeDjrsbc_f}u0$!ztN+iZMH(lnXee|H=A2Jt-q zF8g^mOp^5eoBWb&Kf>_jFHo}&7S|30%2Y=tljYc^xo>u5)+ssx_LC7R xCL79BM Date: Wed, 28 May 2025 13:05:13 +0000 Subject: [PATCH 664/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10938 -> 10936 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 5b3bd20c7..27abe9101 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 12:58:22,121 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 12:58:22,121 submission_checker1.py:3308 INFO] --- -[2025-05-28 12:58:22,121 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 13:05:08,135 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 13:05:08,135 submission_checker1.py:3308 INFO] --- +[2025-05-28 13:05:08,135 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 12:58:22,121 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 13:05:08,135 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 12:58:22,121 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 13:05:08,135 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 12:58:22,121 submission_checker1.py:3324 INFO] --- -[2025-05-28 12:58:22,121 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 12:58:22,121 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 12:58:22,121 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 12:58:22,121 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 12:58:22,121 submission_checker1.py:3345 INFO] --- -[2025-05-28 12:58:22,121 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 12:58:23,203 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 13:05:08,135 submission_checker1.py:3324 INFO] --- +[2025-05-28 13:05:08,135 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 13:05:08,136 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 13:05:08,136 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 13:05:08,136 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 13:05:08,136 submission_checker1.py:3345 INFO] --- +[2025-05-28 13:05:08,136 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 13:05:09,155 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,13 +28,13 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22413 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74137 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72244 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416271 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418588 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 438db35af..c233251a4 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1291 +1292 diff --git a/summary.csv b/summary.csv index 3ed8b5d3c..9e6d9e354 100644 --- a/summary.csv +++ b/summary.csv @@ -6,13 +6,13 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22413,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74137,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72244,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416271,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418588,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9378fbd84..a0b58a7cc 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74137, + "Result":1.72244, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.416271, + "Result":0.418588, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 7e508a89e1bd0554bd5fcceb48d208141630dc60..b2758876e351359e21d13432eaa7d7106ac8ac6b 100644 GIT binary patch delta 1925 zcmV;02YUFrRk&5KatH-q%Na>>vv&w^1AhWH2sn2>=gvp&#DncQMglR7Q;Yz2BqF&Z z5y>5ih)3Tm-+&z{7u3Enly|UP$*t(DSTBAYM_BKOD|zlqCP|H$@>?=3EjF$TzeXl+ zZm)a&{3*>r_6%m-)xk|h@Ztb_hiX{Q2XU_gogKqSjZBw0o> zNis^li1KP1VpP%|B$lQgI@=q!U_whoM-e;uIh1utnSAx}E$do~pGy~o1%K=Zo)0@? zj^-eaXW2t?zKE~yR4d3f$FByomK9bY^xbA+Yed~v=sHfxvq!~We}AU%^>){lU}7e4 zB$(uNdjOMU08EkrFfoI>!2=*UiMX z&n{it^;8L){4LXG3g3xzeaR+f0#CAu8Dx`WfK8GCHp%FP1zFBQ2jR9uQ5F$)iB!ftHAl0)J?N%|ps?+az7%wppjp z^S#jASHnl`Dn2^2tTMGu?KnQx1Dhy_cvA8qnNV6jB!g^L@O)SX*d$U5vT1UCSP#gi zN%v7w#yNwQh>ap;&T}a1(lXdKiTeZFX3b*HSF)*}bfn!=1#B!3f1>xX0z%?jQR%K)0py@5A2P3{lv0njw*KT66uXV4OnQNZzmsxGU7 zU6afocg>nTpGU|P#?RM7xu@>WmD-hm9M^aJl3(*sVF{Hem3UJBA(>G6KO}=xRxp5A z2B>6)4Nz&4fM^durAY%)-~mBP#72Q%ZGf^aDMKtt*MErRzS%&fSo%-Ga!+lbD;4fy zX$L`AvRP`2W2sRm@uUSrGNH79NCvQE8Hpv!NGwT4$={QDrA-zP8%6%Eh^d2)1_=mS zA~sF=+eGJ3)}>|g7|TMonB;}7*#cUA%ucDx4VQw+^lq{`$mS>5?3Aa!p2}KQSLFn& ziw2Nt`F~nFWl*XTDA6SGqya=Sp)`O<2BEBA0I>`}$;=upp)?slY!rafBmij;fS@I! z)0EOIlyzAdLRmoo(lrY}J52r^=4>qstb}FJ{>h#+hk>O_u+$}3qDJCL`-fygY5$N6 zVp+ldVHv=ZSv7#A$^K!Z0G1~CXM5uVmLUJoX@3HiApejuIKi^6;RIV)z`qpBeKmjd zuKD9+_l;M!UD_pFqCw(G^M_zchE#PXYq?6oYhipZ{u+K<=DPO*g-E_S(eO1MOS#FN?&$%In- zA+y~P2@?bb(%DFp>Le=+*})|U+kv*^)IgyrlOZJ08aVI zhC#VfL*vTin5WBwd}tzO@URa2=Cr{aK`N^r^@uwYzM;p-wE zu4O)~;)ypB#8V&jiMP`t$375A3_y=@&&ucQ`DMLCO+kt>C1qO7*ZGW+l9KuNVdPE! z1*lF#?k1iA$K1D{p!o+-O9KQH000080000#vpFWc0tH{o8A)=J`zTuh43k$WJOP%I zlqo?0gOk=NIsx^Q5h^Ad1=86_B>?~cPyzq|5dZ)H0000000000fPvPNQYt_j00000eII|S delta 1946 zcmV;L2W9xURk~HMatH;7st;>)vv&w^1Ajd2E6$zIx$}`b@nCz7kw8r26eGYLiAe59 zL~=(W;?eiYH(*D~1+{Ms!80>+Nr=|9ZRXe;(Ncqq7$?{ZGX8O z#ekz2Ad2Dv^c*9BJab_LP$UtFB8f;8NkpO;7*HhT5=BFK2g?z~zdUz6uR6l6v_x?2Vzm6ADz$WPd}%Mv&)GS=1#(@|nk5%sBG-OIR;(lU_er9E^Lb z9zF-&w?3QJvAhZpUWNq-!#{P~HAx7z&B6lq z5zNA#nUkrlmU;r3b*!<<=6|`FnA*Gy-3olKpeb7-ItrjE%etftUSMTi+m=LA{|C^lV~tf_bKOjA z`|Q%SU9aRtSH<5leWvi8NY|HaVkYn;o0vg1Ne0*?8DNvV&x>r@CVw_b56GsWhtBrK z2R3C(L`MlEo3gCS%5dAn5N@0HKZ;HDq!Vo&Ypk?w+Qn@%Z26X3^1_ee@9{rV_|~}_ zNHobC@V-P7Gl(Y10GcENXkvz->6p+YJs_Hf9y;3_AJCL75gi54lx1B~hTA6T8n;d7 zKY(T(Ypk?wI=kCu;D409SE)*@6os0B8%j1Y69ke?%pjX218kBEu*p;!VAD0R$xIty z)6hd_d*cI}vL&LUgpy5J)@5b5ZDI(wP4^$h=2MQeb*!<{w&@nPO}FKRZW)>-{*FR1 z#FGLK$pq5iAsIxog2TfyfF`qe08Nv|!+Jn8O(u^5l?PfPI)4hF2{sQY!)=pvjoW6O zLeKX?b6*V~wX68((6Y+ZI<@2YSPyKXAmT~Mhh#!&`H&2=bS-HL`DI}2dcWP z3U*C0f7~@|_Iw^8Qy4#A59OY^KUZp3{&8I2@k@TqLxm+&qEzBZ{fA^i>Hm-nQdz+O zVi};488$$rNdlrh0F@>UNP!0gEfE_9ezgJ0x}*%TB!68amiuM{m15~X3ClgTfv!}z zi=`a|VaaBxEsmu|oy3zC5Xpqn0wNi}l4T^8EF-Zb86|&D>XkNGKx`EGyCS9zIvOM( zXo=V~L8n+V6#)6{(35FSzVPA ztS%Zrs($}ZU}wJL!UO%hKUKqM1N1Bhf0$_fS$%K((jtkDumlL5p=0VquZkOlz= zS|U14Da}Gzmz5!u6$Bt%vjDWi`8MNSh@sDU4kWQB%ZW?NG6o_ z56K{w73?3D0W6tS16Z2uA2teLX_9}oH$Gqq@_!GVCSVEj4=IBaEbAIhu!RNuOR?Nn z^GENRKiOt!d1YrmE4rN_Z1|B=>8a#Ggv-g8oesht%mL*mZ*>zF-@mg9tg%>V%xpYdnM1RDS+7HQu zQnT3+3ljv9z9bxz=p-u)TOlO~+kv*^)IgywlOH7-e_ZZWoBVjD_FfCYXG*qYukCydA$t~r^!J9J{T#(?C2-2Q%TcKSDmSXJr4ixiQbm%B< zj8iPc(rvM2{D*A;lRSUK3IY=>Fnj=}b|j(`Qt%OJe_cBO1jo>V0v$XqvI1Yc0i5!a z4TEy!e^fH`7ACD{adhv!9F1MqW!-GB#{32TTED&ztEN&PPQ?M$rQno1V8OV$#MebU zT+4h|#}jWPh^Id46K|(Qj(s4K7=Rw*o|Vtp^GmZrHOa|5&F3juGzFpa1tsMBF!HAV z0#v6VcN5QmWA58eQ2zr^O9KQH000080000#vo|Ka0tJVv4{LOj`6ycf4wF?WJOP-K zk|{wAg&Y6?cx*3oba`xLbCV$|6qEER8UptolMx>klO-w;8 Date: Wed, 28 May 2025 15:06:24 +0000 Subject: [PATCH 665/719] Updated results summary [skip ci] --- README.md | 88 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 546 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10936 -> 10937 bytes summary_results.json | 4 +- 6 files changed, 344 insertions(+), 344 deletions(-) diff --git a/README.md b/README.md index 27abe9101..902d46809 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 13:05:08,135 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 13:05:08,135 submission_checker1.py:3308 INFO] --- -[2025-05-28 13:05:08,135 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 15:06:18,372 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 15:06:18,372 submission_checker1.py:3308 INFO] --- +[2025-05-28 15:06:18,372 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 13:05:08,135 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 15:06:18,372 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 13:05:08,135 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 15:06:18,372 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 13:05:08,135 submission_checker1.py:3324 INFO] --- -[2025-05-28 13:05:08,135 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 13:05:08,136 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 13:05:08,136 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 13:05:08,136 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 13:05:08,136 submission_checker1.py:3345 INFO] --- -[2025-05-28 13:05:08,136 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 13:05:09,155 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 15:06:18,372 submission_checker1.py:3324 INFO] --- +[2025-05-28 15:06:18,372 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 15:06:18,372 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 15:06:18,372 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 15:06:18,372 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 15:06:18,373 submission_checker1.py:3345 INFO] --- +[2025-05-28 15:06:18,373 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 15:06:19,667 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22413 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72244 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22413 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71747 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418588 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9368 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418593 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9368 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index c233251a4..250be2e8d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1292 +1293 diff --git a/summary.csv b/summary.csv index 9e6d9e354..2b3ffa273 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22413,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72244,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71747,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418588,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418593,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9368,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a0b58a7cc..871e88898 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.431769, - "Accuracy":"mAP: 76.951", + "Result":6.22413, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -30,46 +30,16 @@ "weight_data_types":"fp32" }, "1":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":23.0274, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -77,8 +47,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -89,17 +59,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -107,8 +77,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -119,7 +89,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,17 +119,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":22.9802, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -167,9 +137,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -179,26 +149,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.22413, - "Accuracy":"F1: 80.0", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -209,26 +179,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72244, - "Accuracy":"acc: 80.000", + "Result":22.9804, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -239,27 +209,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.9084, - "Accuracy":"acc: 76.000", + "Result":1.71747, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -269,25 +239,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -299,25 +269,25 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -329,27 +299,27 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "11":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":15.6951, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -359,57 +329,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.290026, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.418588, - "Accuracy":"mAP: 49.593", + "Result":19.9084, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -419,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,27 +419,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":5.95448, - "Accuracy":"F1: 80.0", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -479,27 +449,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -509,56 +479,56 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":22.9802, - "Accuracy":"acc: 76.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "18":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":5.95448, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -569,27 +539,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -599,17 +569,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -617,8 +587,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -629,26 +599,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.438894, - "Accuracy":"mAP: 76.951", + "Result":9.42443, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -659,17 +629,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.290026, + "Result":0.457477, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -677,8 +647,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -689,27 +659,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.6951, + "Result":23.0274, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -719,27 +689,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":0.431769, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -749,26 +719,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9368, - "Accuracy":"acc: 76.000", + "Result":0.418593, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -779,17 +749,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9804, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -797,9 +767,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -809,27 +779,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "27":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":20.9368, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -839,17 +809,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.457477, + "Result":0.438894, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -857,8 +827,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -869,17 +839,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.42443, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -887,8 +857,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -899,7 +869,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -929,17 +899,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "31":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -947,8 +917,38 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "31":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.353561, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, diff --git a/summary.xlsx b/summary.xlsx index b2758876e351359e21d13432eaa7d7106ac8ac6b..39eb684ff6a26a234b09686afd69d1ee7d992931 100644 GIT binary patch delta 2474 zcmV;b303yERk>BLatHHVX^b zM=%R}W=^KITIvaC*0IJao9AX?YV$I5EAYLNXOoJ*3-wIl8#}Hi(Zo#PN;ENpXp#(| zNiu*YW`78p+Jq+Q0ns${(AnPjfTnDT=qP}uEbEdoc!8C5ZCese{U1QHjx|U0Tjy>d z(Ijuc`w~seAetltXp#(|i5Y^XV?vYkfM^~5QZqy4gNv&2eKs2RASWD_$%AlbwWvPm+)CdmMsOr-%fT@#zkv;j5^ zJ%4nzH$Jc_TOvA2DA|-{T~>zMCWdg^bpK&&KIKST#~Lebn{IL2bX#8NmZ4eV?Q zJSp&yOdt&&l0h^pI6N!^Xflfj&@^d0tOrEXWb!Cbd7vetqX3#<^N=#!Hc8jGZPqFD zd@nTj)$mcfijNL0t4ys^JC2X_z$OYJo_~~lNG6n)56K{#6+9o70XB)$f^3>xAJzl1 zY0`a^lyT0WC1RsUne!aVy0i?oP2&E*wpp{-^ObDsCmm_`)cCnnMdc@Zwi#N5H#TMVZ@Wt56Ohm`XL!avx4`-GJqy?Z{Uqhllwz^05nbdkCJlE8MH)X6mWc?s(;I> zVAmw`$6d2#&*u>`h4J(CQ0}SwbES6WAIJ3_zvS0ER9HeKN+q7ue@G^j{tw9@l@$yi zmH{f6VFOf}Bp})YP-)VD6nH?;60uR>R~w+LOUe*S(luhaZ#GaVmj08l+*2FqN`<>v z+CdPOY?j*MSZdTsJZS-uOeifNl3oEUSw>>XG7?LYQS$esUTKpB#72?7D`M)Pqd@|K zmWWML{x;D$lyzyDJjSw+Ehc%PYqo%vAG1^Ha>J!yGQFFu4zl?PHaq3%ucxw>)m1sc z>Y@RpTE6m~lD$%^lj0DUe|CnoOSnXX#FOR^$%NAUAsOVdg89QTz$MA}5=)c#!#V&g zP2P_N?+02UG6;I?P}U`7;IXr=!DH7odq0TfHy7D!Sz;BDT^F?86eZ- z{ICvyOq24XLHU7}hzvr=1m%a6;X#)DT^~5e)~x(|A2RpU`MJ_X=O=$b)(hQ|U(3{9 z37X_j@<{22WI`$Ze~=8KSwZ??89nE z2sX`{m7lLg^GT{~EsLzQX%?LyZMU=&mgso8METLEj`&jfAsJ;!=ZAz4%?i#B%K(}r z18AC*AJze&X)=B^7(dVwvCD>Ng7HJjaMvVVxuvI6u%5(P>Im7RtJ; z4EIf92Eo2rv-I=za6aWoyQk{Ur7o&J*}iFMzwG7N67xr+LgGvFhh&r`)gKZjD(YAB%EwQ0HM!PE!XR8w4O|iP$v3V~4UX zEt4B33Bkr`@00)YcZuw^EV9bRX%{u1wQo?eaoQyse~^tni7z!El2MlQfJg}0tl$B$ z46sQu@(HJH(tub8fTl?UvOxoamWT`j9y?TZS(V&2*&PDgroDF-(6<%YYgu2F9n&sK zK-rFIxmuS|Sz-a%=#%);0wNh@NezgEkje@g5X%6SBm-2MEFjhaP-(J&wl_Xd2^J8$ zY~Zm&e_5B5A(mAvpuz(79m^+)?6s_~5|$tV`K2G2Qj~yH@PH-xAQCA7k&LpW1w=xK zWd#d}WdKX&*T6y6qye!}fJ>7GWP=6-EfJfh{LqVYDC^QPILKD`^FQ{UYe4BRM^Zh> z|EXndmCmn4|3_Qd`Bj&^C~G?oy0i?CtY84ydu{-2kPM%M)piXz4^^(x@e5! zVYX1KMOU|_?V<=8#1~7?l$fX;(7jE_VaF-B1%bfaGjM znjhQr0M0NpAVGPDvn0b;uLmpLWJ9mqIHm9>1C!Q^IJ%EcjK;QYlXliyL;ehZZC>B| zRb!#LQ!zkQ!5Lu|sMoeC@NE$e*HGs-;lz6Z{OJ_*iML^qb06?11R&?Q7v*#I{IXr6 zDo^uyoX+EXxy>2oBwdi~ao3H!>%Ri!Fy!CF3*eml{u5MxvnnRa0tNAF|6y~J`6ycf z4U<(VJOP)Jk|{wAgd6|>cx*3oba`xLbCV$|6qEER8UpqnlMx>klO-w;8^Ax0)g=J{ o08jz|01*HH00000000000DytllTa!?0V|V_DkBEnCjbBd09hrISpWb4 delta 2465 zcmYLLdpOhkAODUlLR)kjF`SvZPHRb&afppG*T@#Rv}tPVltXn&zGlKsh4y7?JGoO% zj)zT7>gaNkY~#3Cw_N#SI22?j*y{9ak>Bi zKL-Fi0ceYrfmopFr>>^yD4x_Vf`PsgvDeE85G&ld680)y$vTEN{t+_wT(nSx1G9u;CBD13X4L@ zD}(#Yd?B}@Ro4E;vt{pmvh|w%>%CWhntGV;p}&`H)6u9?Ou{8&C4k_TDabgbFDTvU z(bevL^74qfKxAHA0tiHvKGI>0knRU85`GB;TAjK>NB8x#HrBP-hYu_r(Gn`{x=wBy z;>KT*U}Mz$L|haNq9v1gFbB!XU(MyLXlOB+X>e(r`BP$KP@PA7j;qK@_NY?lPqNOt zy4dxS-Xnfa9o;oe-UDEH_*;h;{GaO0cPlN|jMr5>?>uIEK!3qzeDHfX44zKmSFSOd zf=n%Pr+RVhc#*ha4o2?N>j(hIlx}nSLl&><+(7)AI^!E)x%e>)p7~i;*Q#f8xF}{g zjYn?Tz^3W%ZB~6)h*4+aa^b!4WG1c)-iu#s2m1!UJS%5%$1RI!LHq;?>Wpyb->k0H z^cs22cPyjRm9?x#FIiozUy6>U#c-dN`0LD-K6A&h@fV5kj>_TLYMnxft+8uY>0Grj zk_<^oTu0gbj7&8~N~^o{>;NWwq>gg8eW9WK?a>!r)7CCdwUxLBzSE8_-ReTijEjNb zyyWg7)8|>{pyw}oA|LPV$ZvsZ&dorDdY~3`7kO6|02KIgj z@+ZVJtQ|i=@RlAp*{*jp;MpAecYE-DN;uSlzR)T-RN&RTwW+Pgz5ec}w%q<)a97U- zB`;#ZKT0dtGWTNOx{9cRfZ-eo`93)mevg)>>mSQYW*}V4Na}&ZFgKq@Pj&<=S!8eJ zZcFOP7=6;`S{ADC`#a8$RcN#AQ*`V`w@?w~)U^{80Ty$m8XmZ!+GHX;teS@6n#+i` zBu~i@bg7KI9cg%Yttz8}lL^<-q?$R9TZ3wTGgjUO=|J2km=^7 zNEXkCoSOf$=$*>B`!rul4LLfx;r=yv_9;2lneP8P^Z}U%)`j<*hZVcMCoQDYeu!Pm z)`_D$S4N?Rd9*}s8A%ys1n{vlUYuohyDg{Va*| zugz80`m=hZzBN~O*q^VD_kQRBHbVUfNowGVDe8kLNGjj=$hn-AM0Kx>NXx&ak ztp^Pm57LV&WzyNLJ}oAWFS~754>pbNzezt$p_M_*4u!;5#MaNbIl<57&SzMZp?O+m zi|#mLjK)MdJz3R%9Qhx_B+!hvA{gYL8Qf-*)F1tmh$v%P-_R19!)%qdEt~%Acc;K6 zJU(fD2d$yqPX6G>>D)rn8YU|ziHObV1Q8Mw!M0K~ABHEv+wlhomFlXrDjEJsh300Q zLMv^0cu1NMJC_pKRY{S{srZ6YX}74^TNu+J?>O({9L3GN>yv}N@iAj}E}9p>k53|M zs)Dan1z%Ona*|vnpa47K#hFtH_?Vf2-AO3bxj^pR_erqNkTA3nY&{g5R23f}ms}td zYJHMUQu7=M2Gd(2oCuf_TdUP5AdqjNpr$`-p72n|#+H2f^6n+L-|DY-@ow)jQJS6kC z&n|KTnAdz>I&T4@NRf92_hQCSG9(eWqYT+1YC(MCWnR-2*PluS3rE&^Fa`7OG5-`*`p5vuCU|zk z7`z=WnI#cF-T!jJS&cLR5$m+k3l7||{d2JgjdvTYVjhQ0N?DT%@r?bOcj-&78V@y! z=Xev5IqT$=!HcU!=O-AKxfAlqvEcXdSmtF5h&DTNDo{2t-bPoBAXd?Ty+A7l{RsOP{cenS z)-A3u>K?tZl<#0tExg2oP<_M^W%HesbHa!z0f&sFp6^1Vw_2RJz#L9GH+s+ zxy$!|sc=Ge7QFNw_9IdP=@|>jEWy-15xzAlt@H zRt;U5&VuboJs7RQ2DP^DbOC~#^*XeyzGT!stug{qPB7fqn~ldPnY#x8U%4dBtv=2g zbx9lQ7CSy#KAzcBH7Ppd5PD*57x>5Vq4eOO3dG&_$pY`QQlsj~u%LI7X2w-?m&*6t z{d$c-=q2u-U1qObkR%<~taEMBu_-yd!GrAfE6$uwZRwGtuJd?2`Et|;^18-7U-wVg zeAuhEe-;$VH;QerQZ^^M7GGcf2Te1Pl>h($ diff --git a/summary_results.json b/summary_results.json index 7ca7b5bf0..a79042e5b 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418588, + "Performance_Result": 0.418593, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72244, + "Performance_Result": 1.71747, "Performance_Units": "Samples/s" }, { From dffe1421af04e42a518eb034dc060e47fded533e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 15:13:28 +0000 Subject: [PATCH 666/719] Updated results summary [skip ci] --- README.md | 88 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 546 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10937 -> 10937 bytes summary_results.json | 4 +- 6 files changed, 344 insertions(+), 344 deletions(-) diff --git a/README.md b/README.md index 902d46809..32f5f6501 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 15:06:18,372 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 15:06:18,372 submission_checker1.py:3308 INFO] --- -[2025-05-28 15:06:18,372 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 15:13:22,741 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 15:13:22,741 submission_checker1.py:3308 INFO] --- +[2025-05-28 15:13:22,741 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 15:06:18,372 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 15:13:22,741 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 15:06:18,372 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 15:13:22,741 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 15:06:18,372 submission_checker1.py:3324 INFO] --- -[2025-05-28 15:06:18,372 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 15:06:18,372 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 15:06:18,372 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 15:06:18,372 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 15:06:18,373 submission_checker1.py:3345 INFO] --- -[2025-05-28 15:06:18,373 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 15:06:19,667 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 15:13:22,741 submission_checker1.py:3324 INFO] --- +[2025-05-28 15:13:22,741 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 15:13:22,741 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 15:13:22,741 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 15:13:22,741 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 15:13:22,741 submission_checker1.py:3345 INFO] --- +[2025-05-28 15:13:22,741 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 15:13:23,913 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22413 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71747 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22413 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.68814 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418593 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9368 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42009 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9368 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 250be2e8d..7a3abe9b7 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1293 +1294 diff --git a/summary.csv b/summary.csv index 2b3ffa273..c32325270 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22413,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71747,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22413,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.68814,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42009,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418593,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9368,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 871e88898..eb9edbf21 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.22413, - "Accuracy":"F1: 80.0", + "Result":0.431769, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -30,16 +30,46 @@ "weight_data_types":"fp32" }, "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":23.0274, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -47,8 +77,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -59,17 +89,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -77,8 +107,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -89,7 +119,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -119,17 +149,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9802, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -137,9 +167,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -149,26 +179,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":6.22413, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -179,26 +209,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9804, - "Accuracy":"acc: 76.000", + "Result":1.68814, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -209,27 +239,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71747, - "Accuracy":"acc: 80.000", + "Result":19.9084, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -239,25 +269,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -269,25 +299,25 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -299,27 +329,27 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":15.6951, - "Accuracy":"acc: 76.000", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -329,57 +359,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.290026, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.9084, - "Accuracy":"acc: 76.000", + "Result":0.42009, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,27 +449,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":5.95448, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -449,27 +479,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -479,56 +509,56 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":22.9802, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":5.95448, - "Accuracy":"F1: 80.0", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -539,27 +569,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -569,17 +599,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -587,8 +617,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -599,26 +629,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":9.42443, - "Accuracy":"acc: 76.000", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -629,17 +659,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.457477, + "Result":0.290026, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -647,8 +677,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -659,27 +689,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0274, + "Result":15.6951, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -689,27 +719,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.431769, - "Accuracy":"mAP: 76.951", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -719,26 +749,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.418593, - "Accuracy":"mAP: 49.593", + "Result":20.9368, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -749,17 +779,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":22.9804, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -767,9 +797,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -779,27 +809,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9368, - "Accuracy":"acc: 76.000", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -809,17 +839,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "27":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438894, + "Result":0.457477, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -827,8 +857,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -839,17 +869,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":9.42443, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -857,8 +887,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -869,7 +899,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -899,17 +929,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "31":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -917,38 +947,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "31":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, diff --git a/summary.xlsx b/summary.xlsx index 39eb684ff6a26a234b09686afd69d1ee7d992931..6fdf99d04e80898f1dce9b56d7592f65ee659d56 100644 GIT binary patch delta 2988 zcmX|Ddpy(oAKuLUwneGQ-PfXwVyzC~hLKAYQi_18K zTw-BcF41v_Q;|rK5V?)tob!79{`&m)e4h97yr1Xwu6M0>t-LBJ-*)@1#x*$zMC=L# zvIoN0p{HEUlC+0`3{?$cbkDkqsd|;-y+5b+NOANX4v7qHE^+MU2F-VrJ+W zi6h9=YC}Jxxew<)OsD_NmR}XMH?+GD$ukTO*Lf$ry5_4Lli53yub zVi|2b;rzujHLA0wIwfyXtM_Q#iOJudC3a?~cB@vBtr7vxPTWCFN|=1o2hlFe=xuwy z=c{R+*=Numiw_jNu5#bX(*?+lc=0M(JGc4PlE{(iCC00l&+ER9UHcgIt-w!}{{%8d zb)y}8eYf~-LHjKD(3*Nh<;-l1%@sr3z6WN<;vEU%i43?f^@x~`5d*7rly5xPh1h#=5!|+5i)iYrJ+H^ixNfWd#dS86~Muju4f5WzC_xw_} zzu55P%3riWcANCA<`MhAl+qDt&ypXbolcN)6kcgpMA~?=y>DpFPlYAriZ#CXIOyfh>tgPUDC(9_cizJY2tBcx5=Oz zNL?JYw+}iO7cAC%W+BjJ{zzVqUZhv&62QuoI-7*@L6GQB3_-k)BWr##s}DG?!HQQi z8QH7e$3*W?73*VCJzc~rD%j;g#Qd@?nD1a!$9}{5U{q~X*z~*d61$;^xy%zMSF8)a z5wV99Bj}{p8d(4=kS<$Ect2@{4FVOj3MD85ecKwrve-KL1)_A4`yiG2Jcuk=M_N~Z zRs2VPQtjQtr{7^`$7_+bk$m*^F`N0im0Z2ZsZlwAH6aN1%1=V#85GDi)kBtjA@;s> zH3Z2Lo;{la_vNK0-WBoX31x2&VuYlO6fdfqRsV~Z{ZEiFc)7-_HYnXzPZ|w7!VDy% zB!R3>sM4hhTP=+eeZ=Ats1Oor9>PN1Nm!!!vhZqR zD0mGX4B2U>NKtc%A3;qk*L9mj<;=xRHcOxI|NFV*!09?k-P9yq3*k8Sd`7@0fRHE$1I8uMmMYRj)#KfZ_776J#YS$v5}r2 zO)SNiaTsgcM7K1Nh3pt8$H+{_IFM$?5XIks>?bLwiohj3w6M|}J}_n|0Z9f44c7X< zy2bau>IW6lZ}`JJ%P#LGno73OyIp=)A*Prl?L%wOrHaJxMl>D1t-iZ8nP6nM$$EPr zh)o;PT{vT9uNs1}HS1HT^5c1-OoZ`5GdkUr*slP?kZz^Eg$rSJp25#|+82j2{xZkn zUiy?A6D*w?WH;>J=jMr_fv5N~BCg^^`)?|c6daY!07W}kA_c*d7Dw{b+W8oaaK15JQiagtNlz|+~HrAvND`VGdA(jE5Bg2iwQ6Ge!_gjd@MBw@q z(#-oVY_|*SUUsOQT4VX}y19N!eb+g@!jr_AhEhQU`dN0H%TbU7jr>Tb(1?tsb`yvJssEn%w1kJ2kz@ngNtnfm+d4johvb0tXthrZSzYWu z_1XP|T3-QW-*BrGuPcPug}=^W^b2iwS<^%U)iFQYjm%J7$OF_=LMVj7VyesWgu6~b zSfF%qg|cw=Oz&E&d9BaqsdX^5zNDx3zJUpDRfc`;OZtXK@v>bX0H)d4;e=jHCr}XG z(*1BggayESSr3KtKJs&k#_0YxFEfs9aG;&Uh%m488+^JEiDo&vPy^g4MWpI48@`Q2 z*_HUt0Uh2w>rhSx56;ykn&5z2zNas5Z#sZ98z|fWE9D_1#(crcm7D2_o>;tMvbsxr zqcDr%-K><{x!B2^{kSs?r5h8C#p>^LT8y6&AF8J;CmLp})1A|q(YBJqPc;OUvXxw> zYDCZ=AgFYPI4~9qUakt^98Ny|SH9<$6pSU8yIDoK6$y)aHt&wRf(D_^0`4-2xDHYc zd($&<%DWW2SU8|SzicxV!FD$ncnjMn$+WeYAn-1LWT6&qVpn149oeW#sY$qAdrI}~ z)q(&l*bL8F|GMQny`}r%!1V^dj1?UAHqiVNw6%73foN&c7*yb)zV;YJQRHY41H!RR zyUa5AGNb`9Cvu9V)J&P7e&B=AqXbE;!;RdH+!i9Gf|}-vb!aSwK8uQ_rwkapp#}aV zM3zC*lG%k4c#AF{F?b@CMxv)V3IWAACJ>A;(sIg(ZF7GNFN!3$=-Xkm&Ic4d0|8%;$^e9{$^n=Cy6PTUEc#!t4@$ z1LJFs+aaAs0aC zC|H*`Y`e)wt4QN1r@Vaoa6N@&SKUIM(y72 zlxVtVdFbcbP{EtnF3t>UOlFE;HPLoiK#`kQA#kDTy`wZ__o zp0#%V=cB=eUOw|zj5m+*zvo3(b$5JRye#-%#?qHni6wMr6N6y1a^-N@ z=a8RMMw{~qoACEz@Tj?x+`Vfh9apq}UyH*0{*OO%Lk9bdQQ$s@Ix%P3eaU*j>VgWq zI?e;#GMrg~yMAdYl=b?wSABO!CqQCw1>tp9E+Kjs=h(U7?Eqn}g=@%NR+M$z6seX| z8j=@9!Q=^yX+Rz1`?nuNSp5=67c`-9m!;am9--Y2JCzcN;T9|)e-K7^_Nk>}25=B@ zr~cLu-ZP3kh5p69v)O8r=bFL>taj)Y*E@|UN7X!uxT!T&HeNFz?T?{Z@+b4i@dty( zy8U5;GMAkpdN!G5Z+dQ69t;bVpf>S+J6OGa7Zx5UP8)7-*ZqbM7~jB?LNxlCCNNkZhiq;d(f%?kZOF5ToZ z_xj~h4viE`SgC|XS24xhc2?){IRC)=^M1cB&&Q|5r^Tn{l#<%)32AVQDhQ+y4FdfI zVyhc&sVh)&g#e{;mi4>h_D(%_`rdir+b%~ri!84zpg5F%%irpJo%A%P_r~uTI)++< ztrncPnnV2i)0KW^$y=TLX3~rkzLO0|S=Z(Qu%@Q?DwQ@zgQ&>r#MKqUKH@Lv(YLQU z+#Dyrk_()TX3t+3NYZGrcg?U`n;2+bCvY2=}WfuIX-R*OwbE5gUmQ6;YWEWq4+#<;ZOF;EHqbQD864s?PZSJ^sh_^r&#jViY z{YJYf$d2sbq;4(Q#fc^5C;gl?{or{hJ-~rrx zzwq56qk^73jbT)nd|_eZ&*#NP1i;)@5rn$zoxTZuu{&zN)sctK?D2VQ0QAo1KaSH6 zJi<%QZ)ej(6oe>v5Sz+yo7-!~qZI-ps6EKSOTRTjjBrfVb6cEW*vCU}fmb!ijn2JOWprYEBVQ$Zm1t`exRny@q6`^+Q zc}Dn_v(xmbB!X9?!)XnI7dMs!k1Q-ZF^d033M%+B@6*@F!t+4sDlek0)5YU9yt#zB zX$KK0?1b`2*meUxsL-_>j1SyY_^cj`SKV{E(Q3c=a0o_}SBMKv<-<%FOJk2ekjL%>+>}?ExFkO+Vx-25j*viHP5{}JO?8nh;Dis}G zQs*%43B~A}Kq;soNX*2J;z`5J5krRaNoKf(S3f)+X5^@u-gCY0Wu2Y#v)y35{tZr& zvjK;@fsZWQY=hu}@#%vp^nyc(AqbBqILF~cKQBgylse~&xAr`!hc>#6ueCQ)>PEzQ z^v~OLXbXdjE#CV?+Y%e${ueFrX9l1y{SLvMYyq%=pDI|=gFDovEIg6f91}H`<;GsN zPXyuImU399p2O=Ms2V4eitqN`m4!&$9awkt#gsbs;;i_m`}aG`G394Oi0+>{ZIi<` z-hOsy7xaxmK(^?bY|*^UP$U;@lphT7EWshDJQ0A6Dxzc2K*xlH9YA3I?^0~N_D3`)bbMsom56J38%eU#`F-;8;N=`H~xQIHh&!?W02Vox-R;uwN z%6Mzhr|Mfg?kk^N9iMN4%K-n@xqn!XJnz9U1E|?DumiRP$8c6CsSE8yPakUUCE`oLCCnjzRu z9mh$_m#KC%DH9q(=~xSfx3efa8Rh)7;8GCRKyoZ`0>VWC(tes87b#_8tvYyBALIl4 z+S@dG&+P=}t0WWDg{*^TE;W#w9WHdTKmK-VMGrp7OOZUmwxjurFfx1b$!?H|+q)cv z8AX46tkT)9?nCIXvD&(-j?m$qiMDAXOD*9EO33EiK&=Kub$NqV{D+XMTP4S}l30{OA zO5Q|irzkk`<>X3@8pD-6 zS4xze=cYmJp2~|Zb3PEctAPlsr$d!1NdN-7Feqse?3>yK5QkLWvX1l)9#$@xJTT|{ zNhY4*DieB>LR<=l6J(5YV~b_?;@o<26hwdSE#SEig@rZKU}j4AzJk1>>V0C;UAk#z zWj#&vA4Eh`{U+{o`~wHc!0qfsPl+Bpj5KvQbz|T9=No!KzkyM}0T4srVj4CX_m{0K zm%W$5Q)caZP7b32C#ooQv*L68qh6vZ7p=Uaq~Uq{hm&y?vFYN{0o~bhu|2kjCD@>> zSGc+CcOCP2km|mIG6a5oJMC05&Pq0Jfac=!N4^EO5QUCbO5MEp=SdkX9^U%=qI~;Z zTVX4W2DE(@31MQVQoC`o_%fA)euaS9;f-(0{Gel_6-RJ)Z0Q<7FR5$c=D_dp$>Td# z^un5>eJy~~eIH*e1jig%MVPK4M2YAFmfv8~ZpYlttFrQ8$mn-6mVGvKq!kf+gjMYn~j2_YVeI zf`l1@bFc8q);ZXvg{@Ob3j>iq4!0AB%D}ZA3^H4RdO<?kD0Q+m!?y8nX8oRY>cbxvQSA8S zO+{T3vo{A0@Om4nMjNYam%n@JB>Mq?>!`hrwpqTKqvU=4Ox7^s_|@zR%dGzteaYz+ zs~rG_ecw@l_?E5RS@XRndn3DwjtTd0%o>G%CFxe#VyZ0)b6670Do=Nhw5(Os9ET+- z*j5=dT+d!R)MdNYFYT~iDVbTFkFl2yT18UHZ8=Bp1!S>+j*8e*xG!%zr>q}I`obTC zUzrGQbiXIE)E57Anz`S)pDp}s+$3DQRz}0NNWbI1tBxnE1^yC8UPFCawUnTR$*CXs zN9t5HN7UroZF63+KcNsn-|;8r8&hY_tuK9O`bJN0b0Z{zf9JfNm;b3vf-d!jWai-R zgyo*>3DlTZ&b|EAGiN+%clQ`9t^1Pdi|%1*>?^+Z(e4#`X{2fbopJqE#!E+I%z*M% zf1eHOcBIl&qI|V$4l^?4oOeBTO;>xTol+aB$Q(p>l%zBd`+6c=0)S9^q2+z>6zxGf z%tkcmH3|;h0UKaU(j5vAZ{|VI&2dO#!sR;DlY`ZY#&4GLzz|&a`s>8 zYK&l?849ppg8~tU-!J=wmOSvXAoz5^gEemtHyZtjP*)pvpR)YWnFwC_q%oh~6r3A_ z_<@l}PBm+ZVt%dg(1E2!2gC{2pHC#U^nFAHGQUO-XZ`!7O-^J!`My(S?E3QBCU2n5 X*q}#F`Tw8sx;2ay_ Date: Wed, 28 May 2025 15:15:00 +0000 Subject: [PATCH 667/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10937 -> 10937 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 32f5f6501..57faaa205 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 15:13:22,741 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 15:13:22,741 submission_checker1.py:3308 INFO] --- -[2025-05-28 15:13:22,741 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 15:14:55,380 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 15:14:55,380 submission_checker1.py:3308 INFO] --- +[2025-05-28 15:14:55,380 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 15:13:22,741 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 15:14:55,380 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 15:13:22,741 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 15:14:55,380 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 15:13:22,741 submission_checker1.py:3324 INFO] --- -[2025-05-28 15:13:22,741 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 15:13:22,741 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 15:13:22,741 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 15:13:22,741 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 15:13:22,741 submission_checker1.py:3345 INFO] --- -[2025-05-28 15:13:22,741 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 15:13:23,913 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 15:14:55,380 submission_checker1.py:3324 INFO] --- +[2025-05-28 15:14:55,380 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 15:14:55,380 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 15:14:55,381 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 15:14:55,381 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 15:14:55,381 submission_checker1.py:3345 INFO] --- +[2025-05-28 15:14:55,381 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 15:14:56,663 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 7a3abe9b7..6fd4471c5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1294 +1295 diff --git a/summary.xlsx b/summary.xlsx index 6fdf99d04e80898f1dce9b56d7592f65ee659d56..03153877aac66385af15136f128e59d12c9785fa 100644 GIT binary patch delta 324 zcmV-K0lWUWRk>BLx5Pn%90dCpqS=D?JD%qtl|ZVoTrM@ zRx7Ab4-PHshmsk>w1qdT4OlP0MFFH7VWvWR?+wA&u{}uYlHgE5+-fVS52f9IVMEzB z-2u+ibdDu>%BiRL2#n20L}w&tBhviXb^vgOp#=%bJDepMzIr`a=_VTn<;E$6?+r{^ zFXHGvI58T#u1mVvU=8^*{I!04A6AWpI-ZIFs!PrYvp~JJb%}3_aJZH_zYZtf3gAzD z&?nx`i=6s^Mv*(v3W`*iJ&5JlK;(XC$1<4*sG2a|V-t}LB@;v0<#0%h* W`~DNuf3qtl3j_p#PNGbcIx2tsx|_ZL delta 324 zcmV-K0lWUWRk>BL3ZuO(KX3`ccAza;HBhKaOLsz= zB%;;>Nv-3*uk0jF0$F7_x_8dp8B{IzvWxz}I-ymKvLr$02tUPk=-2 W>rc@91G6h83j_ov0}e@(Ix2th#hO9@ From 7c956a404dd807d8069710235ce8874cfc80cbd0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 15:19:40 +0000 Subject: [PATCH 668/719] Updated results summary [skip ci] --- README.md | 34 +++++++++++++++++----------------- dbversion | 2 +- summary.csv | 8 ++++---- summary.json | 10 +++++----- summary.xlsx | Bin 10937 -> 10937 bytes summary_results.json | 10 +++++----- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 57faaa205..e86b4e18e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 15:14:55,380 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 15:14:55,380 submission_checker1.py:3308 INFO] --- -[2025-05-28 15:14:55,380 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 15:19:34,288 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 15:19:34,289 submission_checker1.py:3308 INFO] --- +[2025-05-28 15:19:34,289 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 15:14:55,380 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 15:19:34,289 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 15:14:55,380 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 15:19:34,289 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 15:14:55,380 submission_checker1.py:3324 INFO] --- -[2025-05-28 15:14:55,380 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 15:14:55,380 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 15:14:55,381 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 15:14:55,381 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 15:14:55,381 submission_checker1.py:3345 INFO] --- -[2025-05-28 15:14:55,381 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 15:14:56,663 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 15:19:34,289 submission_checker1.py:3324 INFO] --- +[2025-05-28 15:19:34,289 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 15:19:34,289 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 15:19:34,289 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 15:19:34,289 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 15:19:34,289 submission_checker1.py:3345 INFO] --- +[2025-05-28 15:19:34,289 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 15:19:35,545 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,14 +27,14 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.22413 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.68814 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25858 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7414 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.42009 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421657 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.9368 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6fd4471c5..4ab81d9a6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1295 +1296 diff --git a/summary.csv b/summary.csv index c32325270..e7224e002 100644 --- a/summary.csv +++ b/summary.csv @@ -5,14 +5,14 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.22413,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.68814,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25858,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7414,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.42009,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421657,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-refere MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.9368,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8191,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index eb9edbf21..dca8094e1 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.22413, + "Result":6.25858, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -199,7 +199,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.68814, + "Result":1.7414, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.42009, + "Result":0.421657, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9368, + "Result":20.8191, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 03153877aac66385af15136f128e59d12c9785fa..91d672decbf8e409c546735f1c2996857869840a 100644 GIT binary patch delta 2840 zcmXArdpy(oAICRiQpQ|TQzC84WmIb}30or8OfDr;j%Y3`_iG_vw@4~%LyhD*by{L8 zi4rH&hTJ+aI>{|WZe84F{ifevpZ{K;=jZ);e&3Jp^Ha}HRfTO*U|S<>!eJngOehGn z9mG)5Rp+rb*+YQRoQd23n&reh+u7#iZ-`fvLFc+`SNIL- z#rCU8gluc%b)>cvQYqd093h)YQiY9cU~E+(kn3)WI#6XB=RL;Xo>AT1G1J=1OGR#P zob`&9clQg`0&UkE_@THlOzPW`qDjeH7PJnxY^t);rrRiIW4x^?BgHFvxJJ9Lr;lx&L^M2e*o6+(rX17eW9dY?)1nRY^dP*H{!6PG z_p?j*sa8hCUEAz>Kfkk>LlTTV%nYXVxr@%$=>V8>`1!bgL^w`1)VysM{ku)vl?wMG zHHYGjik@<+q>AT^RnF}BT4U9xYyPZXhVe=)wjZAUR;l0AZ2#GAKY#ic2+arGMncYc zL!;XQuO$&<2>W3c5zM&EVfl?K_cbwnJY1pmXt|~uid&;)5~crOIQHqk&L;@GTSl&c z(A-4DB|oqDOyXRQdHzX}$9=!eL;ax^>1bc6(TNplrNyhTW;!pfa^e zC$~!%QdJJVmGI^xAp!KI%h}{Xpcy>{7UF(LX0+VM4oIKWNu;{uMp6(#`P=^q>=oNq zKPZeSd363}P7S%fHZav-D2(*ivXjU{EWtF3hNPn}Lq6sCdqcaC2*L(xI~=E>!YY%s z?(I727dr}JaZx3uAQo3L_K-oO$7s!zNVbdfC(kc>P#)O%EiW)1<$B$!qY;M6zbz-l zNBd?;9&Aj6_EYs>Ra2~fDjQZsn=IzNTu3AfmAjp+_-xEH7yHrpK*}I{P4WD9UE2Y( z4{o%St@;G)$t`3_~kws`vpF)@uQMVrr(FQGIIdr9~x<_T=#sF#Hn3(c@i6hI7= z01cTg>HIY2a1F7pHIU~}JP}*`{BdgLz^S+he~QnAy91$d99d&0_(Iv(QEc&(I6J72 z^LeF8zKU3;M(rJ2t=g4EX3UnB-3W&oDDAKj6ihgNDEFxfR9=#;>0q@KKBl_hr|jx0 zT06`2^zcGnN0;7<>go(%$0YRTJ%vnCyd9ePL6NV;+WJB>7mT|T3keQjebgL)6w!JB zCNun95lXu>VH9%8!&RmpeFs0wGK#g#%+kGa?{Qk#iU02DKh^(jS~<6ja^NuZ;27`) z#%Z|uw!-)Nf!u_CoyBMydyNR39idCPnkE=x{0+we(I*j%zf*{+L~~tGw!Myw7X1iX ze-i-%vBy>)BcaJs@EtO@aVvOrk*0t8B(D767Ky;A_OA0lM2EOjfexgZo@tZPccL%> z?nJxSxhZ8#q+G6l_9bWW?mLbGJTlqMH4AWqR#BZ{=V)}GUnilQzb^4B0ac5SU;!IP znY=<3=b~w&n%exnmN3s2&6oOOQ{k1-oWQ&&xjdb$OU@!m)^$nNSy#9us}_my^rkJw zyU8Y$6W6+n)(IqQ=#iTmg1Rz!T+|06|Cu`9Z?e^*FS}w5NSu-C>5`wLPoM0lMeVAO zI$mpirI{g_aZ1nXZp0YCP5Sv7+lb+mIIZ#>xQQ;ycR3!||CF)a8VhCLB|=2R+d=u9&!Q^%EFzY` zCa!pCfbO72ZIc%yav@T(^(yDV zOJyA2b1eJa2I0zXgf%be`)h212v01!eiQsfVh zFG6|e!Mde~CGuBFdKLEQ;pQ#ABy=``; z!W59h;MngfCu9Lz!OVCWzhQq;tAdY*JK6+rP)@qp`)Tj`#N1-Pq}!aEy@u|Di$g`! z(I+IF){ZO6o)-wnQp^-M8_uOPLA(*?7}PNphru#%wE~FDfKrp8(vohaaZ&fba`m1E zat}+_kYndCMZwe6ecT4U!)y;`0#~Cq?5SZXw^@qYy?3Ep##+*aI$Umd&Q<`Yl?&D* zLhs`!1P-LAhYAi1jw@2eXhjJL=hL``x<@e&-3V7@W)A0`*Z}lIVU9>GlJNV=b-)ik zgGq~rkj_%t2p{5>Ybo%y&1&#AP;UF<#AFT;;$13n8uk_n+vKioOkTOO?awZfh&SH{ z%vkg#^myGWivj4rUD^ip?s{g3Gi$it9~xeE`Ve(I%bL-gC*EjeGXa@m=L%17gJw}L zz_e&s$r?#0^64M+T5KKfgBZa%GHskalh?lM}NZLn_1dK6@ITr89Sf#1!ER=KzI z#muC1s^q|}G@;d_+@XN>zI!j8t(uX2A3px*J9?hZ=8uW%2Ww5PO{_!_6ZZ9OT{>Y=(1@w&F7eRy$=QB3gZ1u2_%E-kc* zt?&IoX}KL9E%IIi|2+H`%Gvh01mt{czf0-erPOmX{R+rPV#s~1peH%+(|Uq{>f)LF zAn*h&CKGi@@StKDbfd%G+7{_|ut(G%Bps4k43QHYEKUtvUn-~+Azu8l5I7A~dUc=#qQln;C z(a@#ChS;CrOe+w!#X?uJr8gW^Z$&Qr;!}_t+ThQlQx(f}UMcK(yxK{)ymK*lL~j3h cgIKyllc$dsNJ($}{qk0HEWrPto|zu-KaomEj{pDw delta 2840 zcmX9=c|6nqAK%PW3BZd>WShHN6}3$|q`#(}9CN!b^gxDp8Hmx<>ko+J;fl8Jly73<+v>Blt>}R(-D<4jT7NE5)x(}$e>FuzA zSgWO@8S%t)B9IsjwRSOb#p`3zj&Rbm*046N$Sw_67zNQ^O`Rou8l>0E%euFe+2nJbEEI? zNGVAvY?%dfY-m?BPtV@8y<&{tlVx%2wi8JATv0* z{6G2t_m$$U#vz9b$q$AUJxjk2`|NPoSXeWSHLj^ImsU58)@ufghhUii>>NXxo`4+2 z)a+73O+%ZpKYE-OFt<%NzigB~$qsQx&Ts(fAzyFkn!K-fcZcAuE7=^{X*S>n(iVpu z9D>fqUX*D(y>P*0{?PqAg9xuTO8_TR;YHuRfL z4-2zJOQwfK^K_AY`jlH9L@p@%1@jxIY~5>Idl6L~89M#0ywqMaF_(G#iBE-;iUNF$ThxFAqHyGV}O-?OO^ESsxeP$w zOQ=pc==u&fJ6?^fj%deR|6n^`vz%`bF*U3La3;iIUIhtAB9jW)9QA-zPq2d@LmNS{ zg6GaA!~KNp_!KEWp+xqUAZBp#P|2c>1@B+H-1kAI;N>c>>L9kAfg%QWh;_jSB@bk0 z5^yA05r90$%)&O61o|)&a7`umSs?*$<}0;DL;|15fD3N*E$l*4 zc{it8Oqx*X9%&Tj*OtF5aFn`KhDjH)ig;n`^wm5o_Y8J(HECE!gbUn$qme-o7wOIB zY2Y&+K3J!AR9Jn%q_pX>K1H`yNQHHXtNy7qAw}-`3 z5kf^-L_37l@wivbo(bjKK+zPbG+C&;$8IxoLE=S}?pAg>CvL$Q9s1-k5QGdgUjVb&>_8)rrbr+Chv0ahP6eokQl zo(rNjAA=Zm!><~S6|pkwSu1C_736V|+uV+<;t%v@?QgBVfzQe~&*Ja(JN*Ktos!XKWf z?~BOs31mlUe_Y6J<32Ow_K~Xn#MC{5O$x%cU~(JrI-l7m353h)E+oLa`J=_e0wsWC zp{C+OAXE-ZM@1+J7ZqWNLgR~+;oO<-)h0r<@29D?v|6yVtNX5z8Gc2Hd-gMX-J|5E zeGdSp+uGwLUQEYP5uJ*C@OB6XfcbH9Bz*7ryF_8p|M6wTa*g)2P?+I_3jcw}>k$}^ zlM5}-omx!ceO~vgFV3wXzUkK&rrLz?GlXyfnrwy#Zuz-h-i#K2wCFEduTTXc6y|(k z^Kvvh-V;YuPttL@T`viLSSKere=cS+Z!i9I-GlXs`VyUY`Y%nNkaKjBf5sc<>M)$y ztSCGA!NO9UE!fOa$Tb z0DTe}!$|Ho>ZM=!L5e7YrYCWW24d6P@1pw; z+aFF;3%(&lxY-6Cz43`8e5(&A(E zunMG$Zps-7rqb*b;;AfsjINrc;XXGDLztV%OIcMeN+mv7>I?^~`x{o*FF}O;bpCA4 zzhy+IewF{L@}Dy>`?w>FFCb~d2^07zI100he<$pES~p?sK4d_U)PdLPrXvQBbi{6V z0>lSt!6-XOFA_j}3Ly*>tW6dssRJBk1!+2UVxX8tr^>+EMy|p9VyMdi@Xx9BO0DRk zSrRZ5>vTPvznHy@)lS`!9L4Y~5BXFbBJK^#`0~AF_2f9HJw6yN2;MLJRT6pW#fC$D zFt)+paN`WF~uOFW>nqrW?oFzuk|h>}(xdyevL)J$W#K)nPlG zvGjRGZVB_Ifk`r1zH+eaQ}B-|la2Yf4fx0hc;sAZ{%@uW=0ItE$*Tk5t@@o2;Qln%*UrH|A8DMV01uUl?O22*UyU%HYNC zL2s+UL+_zo{lO{v;li@gW%;B|JK}lZyk6hBC=_z}_I()z;n5s@WaD4vZqumWWIweU z#LlTWco>g=K7ROsMtwQCZt$GSis|~Zs7M{xZ_MwDQ(wo;WZv|}*{ZBo!z2N@=K@dGG diff --git a/summary_results.json b/summary_results.json index 33069c746..088858191 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.42009, + "Performance_Result": 0.421657, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.68814, + "Performance_Result": 1.7414, "Performance_Units": "Samples/s" }, { @@ -739,7 +739,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline", "Software": "deepsparse v1.8.0", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.22413, + "Performance_Result": 6.25858, "Performance_Units": "Samples/s" }, { @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.9368, + "Performance_Result": 20.8191, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 85039cef8a06fea29927da6f79cf320c21f010f2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 15:30:49 +0000 Subject: [PATCH 669/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 8 ++++---- summary.xlsx | Bin 10937 -> 10934 bytes summary_results.json | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index e86b4e18e..0cfacaae1 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 15:19:34,288 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 15:19:34,289 submission_checker1.py:3308 INFO] --- -[2025-05-28 15:19:34,289 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 15:30:43,578 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 15:30:43,579 submission_checker1.py:3308 INFO] --- +[2025-05-28 15:30:43,579 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 15:19:34,289 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 15:30:43,579 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 15:19:34,289 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 15:30:43,579 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 15:19:34,289 submission_checker1.py:3324 INFO] --- -[2025-05-28 15:19:34,289 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 15:19:34,289 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 15:19:34,289 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 15:19:34,289 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 15:19:34,289 submission_checker1.py:3345 INFO] --- -[2025-05-28 15:19:34,289 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 15:19:35,545 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 15:30:43,579 submission_checker1.py:3324 INFO] --- +[2025-05-28 15:30:43,579 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 15:30:43,579 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 15:30:43,579 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 15:30:43,579 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 15:30:43,579 submission_checker1.py:3345 INFO] --- +[2025-05-28 15:30:43,579 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 15:30:44,707 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,9 +27,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.25858 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7414 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.9084 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32038 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71933 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4ab81d9a6..70bdf7b71 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1296 +1297 diff --git a/summary.csv b/summary.csv index e7224e002..ad9ac601d 100644 --- a/summary.csv +++ b/summary.csv @@ -5,9 +5,9 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.25858,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7414,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.9084,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32038,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71933,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index dca8094e1..36ba9fac5 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.25858, + "Result":6.32038, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -199,7 +199,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7414, + "Result":1.71933, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -249,7 +249,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.9084, + "Result":14.9346, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 91d672decbf8e409c546735f1c2996857869840a..e4e4be1be9130c21f36bc43515521d59b891f86e 100644 GIT binary patch delta 2038 zcmYLKeLU0q9^XSIg^d$o9UJYzv#v8Lsf`k|EmL`zjuV=$Wrad2_FK|5m1vt4Q>|-6 zt<#L0>vVdO8=DkU*iwl)Ayiz8lbcJF{mAS+ z-ER$@jv$)AP${iA1oO1AH(7+BjWndTy(C zdSYZu5H<{Z=@}4;6B^)3bg7LwS_F10OMvby$Dp-PI!n!6kqwYKt?MENqLC6AMTMZK zeG{8f{phYDgFxQD134Ua@^*|i5aGXyvOnF!cNBA}fop`bMJPcckJWWd`$DYJMR#ob z%?2eTv9j&8%|9|byU^f3N=vmXARb<&`n`ebxQEeZEBK?+cNeL>SmT=6^k(~-vRAYy zQ;mMf^=3e1IO>+dh%5a;=H)=T_U^iz)a)g^wj_)|4q$=KQC%$1{6H6XJCX;K?o+>& z;zENVkR}p>8OdP@I=OI|P(@gt6c_a`dK7}2(E;hS6aWS5ceO*@6BrGE0^gAkKVt2X zX(p;Um#sE9mC?0C%!!yNIS>PrGovixS%3oQbKzEz4VqV9xY#ygM)~5Dns1ZKQr11N zwVm+(@4?+2s5>c!b(-9v(u)b>)?QucNvX~#6aff@HOb-)a`QB@&)pU^#vr!@Q3P|9 z55683@yzoaHwZJy_Ss{yl!?B~j#uzRT}SZeSW!$cvp0CWHASOUP__VBYj3 zROj{{MOzqN^Y2-~MJ5#Qax8B|R8ly_!&fch8SNCh*JQ+qN9BWkXitoIZ3I`MgYY|E zK9$Y^LhT)yn%A?(Pp~^wPv@Ktg8ZO#9WRxT4VYpf$Eds7Xrpo) zDM|z;kYSyK^+Xwz)GgH~(>W5X^9hOTcO}HQ$-MY(8AGRwC0v%O7S|K+?b3B+Q8pwL zp1n#DqKZa&-lZshgeZd(3M@5RU2n8H*@s(A)+PK1D#s0loS1}qTHR*zaCSy14%5`W zAIp7_#Mh=B8zVWtW<=QnWR!0&J*n_=f0!${5GYUW@%Waru_AoE8u(Z{f2j8KXn134tk(P3k;WE% zS53(mwcg1-i9B17U(&H`6y6RJQ~SYwv{5+Hq>oTWba-s!Z6!vYU3hWuYztGg-B;*+<2PAk4rO9;@Fl?alCF#C?`{3EJ;ee)qG$ zRsaTG8td#wo9MeDqqM($L=KXtYrno-(8oC+^E@q6q-%pMlE;xkc12bpj9D2zNU`8C zh1iW~>mWb(Y-1|e{4+VtEHMGy1C|&Q&F*IYdt!g>N!lrjA^8FGn7hVIZ)I$UIy*@fd9W5A5>^Q+#VzM$f_9dBNXCoc zJM&ChypY*r^d4*yFUe+kjhvg?Iz=`v_~Cj!Hk8B}F|A&KMcV|#lQ;3Ugpn|n9xU)V zL?Yt>B+>ppU+0@v;>#kjR}847V2$MOb|B6SF1CX?V5zV%=!@^XccA+}KDRS`3fRZk zRW+T`wa^un#d(hPR%y4H@ss&q)C-G>fO+`{J0As$FK?%X0k%M*$*Hpb^mND>nB5bxAYNVoXW;M3XGcSj1Y-b-=+BqRRu^Vy-T z^GUX-wL?$q#~f8FU$D(L!Fy4{-U}Te${ZnwQG5y5{`z45hBC|vRJ0&=TL|DagaU>a z27WYj&T;0Ot^oU1tyNUL4yk`0dSk2dUVOLz`tsQWgVV3{dkJxbN0V!I)jXdP7wP6J z=c0CBRkCMGU$AqiYC>5*a2PoUn+#a+`g&w7l|N$;>-lVA^Tb>4dJEe=OA`jY@g`0% zig5%w_B~vK$p4hPtn#+!3vw8u5xQSldpsyE_TUd{oYgH($uob>SGTM&Ov#So>t)}v zvLfpIRk(SR5BX786@NvrlQKI z_k+RGRSDHwOg~vX=&OT=%Blf8OzZ`*foB z&)GY>eQifG++a0s>sCL8%xa@d?s_%9aTwL(DzfohVrJmI>8xLv!%)YQASQ2v+vWQA z=W$1it_5_gO#huXopjN&%>z5F`%IXsUVsSSJ1yYOa-HVJ@ihiey*o|?G5g=&EHJo4 zgUtj{t^r_yK;Rk(<_ey>`h%wg2sa$^JGWpEhzBh!-HD&o6Pt0Q5XIW0xG z<<}~<>IgYqOlOmX4yLPnO1V@&w{3pYKj-tu=bz8x@p`--pXclKe&5@Be{XG~0lbQe z+@6F006l5CL6APEl(xUXH`UeU#sp^J4A)-~WcuI)J;rjh$h-+9gpe+#LyxdUuTlNb zGFBHT5F|Y>!>%jm3_#*d0YjO@7AU335 zr`ELa7mVdOT}w8|3P~2Eg-F3}WOID^y3}+4?{5Vb=olM>uX8n2#X$urG-MR5VFzKh z9fp^6fu_RrI4Gkg!IJK2@R-*gv~LYP&*Pk}Bbo=V)T0tP*ft+R*p4U~(N}ZjrNdzf zr_XNY*mghbWRef%g=rm2@eb#~l{h)_pN3jUWEkhNi6YUSHG4z`}0ELy%RRvOic54k_Iu$Xd| zB}T|l4;(b^%8a3+!^NraV!MLlK`O1*?^>;2Lrk<<^%!JC6l*3kRG(8xTkI@TQ|MIK zmON`kL&Zu`YL83oL_^GH{W{g?+H^;{c(Cr%weLfB@2sxJf@>R7ee0KzOUM{gZ6MqSQN~;HGkJ-kbTQW+tgJSFuZzCbn$A{#8`-z?3A97%qIiY;fKN zNtI@OeM02mA|$Q|YTDZ?$oSlt9Xxj&v| zOQ@SpJ~v(7xn{a@dK*!}0~N+_8_uXb5oVU?WQ9;%e@BaBV48cdl_ZA)z!hBOn|b92lRit zdLGPY0TKzrvdP!qA+z=dnuhk0wo#UU75-@ zDRwY_M|Ml`pt0PJE+S)c*dZIVi2ImtjcQp$ZCV~y(NvvijTUh=oSikdO_4ApUZ?pW zl9f(A8EyVZ4*y72%=%u5@5e6W=dC^G{2j&HE&K-h7`$g9;kIo)?#dh>;MMR_?QTp7 zSY_nlI!9ivOIE8XS((6Xip)u9Ap}o@T_=T6#L%)XCgkvu zlVzp^bgGhaKwD?~ZG;=4l+$_>p2A(r1|zV9EjlxFvU8;!q~<+@KC=lEq_YkgOIH{b-~&T%KVLvJYCyDUZ_s`24uK$$xO(|D;v^(_C-| zS?<)^koc-W^P!QRG8d_~KWaS9Zen`;JOXw#yZr`MF+4>RJ6is&E$y2sdqFt#R*pfc>P0UTJrFlUm`S`W}T;M9V-W0xL zKkF@R`g6ghhiJ zt)#XV4Mut&oHEy9`J+! z7$FNN(Up8mOr9*czwvtbmuO3MR?5~JR?)QI*9H0|UA^a$cI207o$b$qWvpuRm$3j> z*v^oxjO>4W;~$vV^8Ga+=)#8J^2@Vl_D}Q~VNz%b*U|BJie7TN;=#JB6PMy4W304% z>~Td+^&D`%-H*Bxb8usqsxMwQ;mlPiT;W-QMS7=i4jqqoJ+C+&Y)JmOp0)X$G<~v# zJ+bILp4(|Pb8s$6J>z=IYDJ2>IrQb8l2tLM(hgrTjIbC;uujRe4rp5S5XCKfcihv7 z_#KkJ6(Gts+oNRNN!Z4%{#Qn$ON2+8W2HQ%afRJqM(b}cbWl#LnK?2D-!Rgw(QQXb z?Qja6f44J#(e4)rf*)+}1u5ni*l$7PtO0=IZ0Drp6YS_Dex$t_f5zSl@;^QTk6ZoY ht04dor~?42fPW9?R4+dO?+tO_=i%4G-q Date: Wed, 28 May 2025 18:03:14 +0000 Subject: [PATCH 670/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10934 -> 10932 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 0cfacaae1..7f6f2b483 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 15:30:43,578 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 15:30:43,579 submission_checker1.py:3308 INFO] --- -[2025-05-28 15:30:43,579 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 18:03:08,238 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 18:03:08,238 submission_checker1.py:3308 INFO] --- +[2025-05-28 18:03:08,238 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 15:30:43,579 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 18:03:08,238 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 15:30:43,579 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 18:03:08,238 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 15:30:43,579 submission_checker1.py:3324 INFO] --- -[2025-05-28 15:30:43,579 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 15:30:43,579 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 15:30:43,579 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 15:30:43,579 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 15:30:43,579 submission_checker1.py:3345 INFO] --- -[2025-05-28 15:30:43,579 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 15:30:44,707 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 18:03:08,238 submission_checker1.py:3324 INFO] --- +[2025-05-28 18:03:08,239 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 18:03:08,239 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 18:03:08,239 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 18:03:08,239 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 18:03:08,239 submission_checker1.py:3345 INFO] --- +[2025-05-28 18:03:08,239 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 18:03:09,508 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -36,7 +36,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421657 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 5.95448 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57138 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 70bdf7b71..5b86c14e5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1297 +1298 diff --git a/summary.csv b/summary.csv index ad9ac601d..b391286a1 100644 --- a/summary.csv +++ b/summary.csv @@ -14,7 +14,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421657,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,5.95448,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.57138,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 36ba9fac5..254b953ae 100644 --- a/summary.json +++ b/summary.json @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":5.95448, + "Result":7.57138, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index e4e4be1be9130c21f36bc43515521d59b891f86e..5fb3aef36bb217add216c50b4890186ffb23b41f 100644 GIT binary patch delta 2616 zcmYM0dpy(oAIHD5$Zg2=Hyj&nUvpb0W`qn)8K#TsU&xd zCu@Y1R%AWSvrSCTjgJn}W-j8~DlXXXPHKy{yYs2S=VijiT@4`gboz3`F|&}plB~;h zFegl@mw*k0Bhr{tuInUhQ4e7wUUb^Tsox0K!?#)U3o(b&A|YEniq_e5H@a_`M12AK zRmMZ5cjVPb-I(F{( z-bQr^!?q5T0V3Z*6g5Og)LMyxsF;m$)ef%Z7C~lC-lSeWD`fZSe(?0+{Xwjh~{c%CRj|i>>yqh5DZgR2slH-fVnDE>wR*o*#=qTH>t2-!cm(tbru81kp zwxj~$pJG55`ZdRPfXaAzH@e$mI%OT@+lw#TrDw7=Y)X7@`8!|8o~Z*NRkJJn6fcl3 z(3IgtdM-EtikkI2gCh*ab)zM0`3XM%NP7V>Ozd^}t8vTyO7&{yyQRA7fQp{4xtuZc zU)bNCf4{fWb8yRsTOH&s^_|-?K-INZ_^F6uH=-!it3ebgmy^*k9N_@24_&~PLwx>G zMAA&0_8z?p3fZkF1uxS+3+-ta1VGGNIx%u&!(4m6{xE*wDfkvPH__cvRAYg4j(EQhoO z$p_FO#fJ@ioRvMe;l(x=3ODCB!unc>nt~Y?s^hT;0_(MR2v{Ptbu5<};KtFNlbRK2 zI2jV~QCU5d!5E$VO#yiVS-7=YZy3+uexzE|+*X;YC+NclF5kZe`h& zl*DGd@I-r6rReE<_5627u3HwEWsbq~R0>gaTV<9aI+F!>M|MZ46v}`wF~A2Ql-EgM zmCc&`B7WpGWSz8HR26c{0)W1KQxm`LKNt{7v_3n#b@uuOb?cCfLUO&fz4q2V8bQbV z2RB0Nsp_oeQ$W!A7s{GBt7)WIPLa`&aW7vr$Mw@$`SR;k1o0U2o(6@b!abi$=ETR^ zaOTpO8i`$LNCVN3aD@UvNK!*cQh9oVxb0uU8 zbas9`#zb1y`@`M3_TH1J(^pC|OBMp(WgIB$w2RAfv)vUNkY=Q22q2u29`I84h=K@X5byRNP&`8Y9O=UyE- zIQ%2$-i`KWxnw8WoSFpLnuWlkS?>-4#M|P%(z;Qkb#1D0q;*09gpLyaf!Hlz??|PP z{WGQU?L+)^&ZMTmxXjlj8dC>4MA1au(`DCFI-L{2iq6Mj#BGkbn)M0FPeOt*i z)EWm6lDSc%+*U@fE}Z)pxrEmbN;ie}SAJ_(n6EZVwiz#xg!g%>kowo#{5EcD>0^yI zI-$Kc(2XFrE=FC{+7aN64#O7kA`n0*Haa1I(C?UST5Vac5SE8=NgyE)OQYnB;?6yF z55u830#f13t_E!azIGbze?~h;)ufz&qni=&yzPR@G%w{lrJ<-h>Y=g)L&qmV3ZANZ zm@QA^bfh6v?Be5&AJJfi1S+lWRfU$CUmeAeei`!m%Y2SjReDQ4S2eO6W#wEu#xr0= z-&n<Uv$3d#oGxJGuGD+C|9rY_@1}I3PiD@N6_y z=_+?zv*lpxkT}BCyS8-8Gg@)XhR^xFy!z}ChRqEzEpIZLHVu8zfJTyOivt}r`wK?B z@(FS8*lzKMz2)qD;UR)0)NA@! zuNoaKD^B+a2|TVN2iH`k<~(;w`eEt!j_}4myX5j>Lc;c{o?`Nx;&5T7eT!X&cE^Bn zv4j2bm^^ltHtv#ssh#10%GKHWZ6S|s2Ak?j?DT_k-oVQ8E3rZ1Xk|gz$2oo7uK{r* zq_Xq=){XM$a@K+<8ZbCD5U!XGsp$X574Ub~?o_e(o>@sNGXgTgN8HjiOojt4ta}Cc z&B;!?-;1@x&gz4=_8*Cxs4+f&;`RPTPI=S9hNa*KHJFm39XP+oDEr(ueCN>myG0Ez zJzavEQ}#U8xlPP$KJncAmET$8A8v&8z{aMHR=A2A!wFf!4BfB%)~-J}U$HzMmZ-Ux zd(mOhpuh<;=P;@6IP`FFRU58?1hy#r`$q?VBL$(9VSfu36c5g^aK}K#0FcO8cQh`E z!#WD@v(Sa7EDTYTFvb!z`q^3(0BlnLfHlDXPg=Sh_OowfGq=K@( delta 2594 zcmYM0dpwi-1IIUOlM^PD<5Ib_4Rh-_qY`ayv1Xxi8OJ$jD?2F4HJ4|kV^gASZkt3# z)T$ZjRHwpmut^Pt;pE!MCFPo?o%v1YkKgx?=bz{GdcQu;^SQjf4StP&f^(apihvTg zv-OFawV^}20u>`5q2aK^&S-ij{sL~V zhEp6nHE&Ek$iyMXrkPC%@rOM@sWmr}0-FwHk_LV}Z~NfQF>hS`RXcyFT<*v(t`{Kz3@#LPN~6VI;`N8aNyqHlyeZk(;vwwDE0zuNw6+w>Zf_+nETdE1X2 zO1)eJB#OD3e2qBNikb#XG0PI^+b>PkQH&|G4D#{9c#_^3u5L?oufn6Qwk+=EUEpMH zK_l_W@6o?mjhrL={KZ%9q)_T~NQ5v@$&j~25#g|IgdT1~>kBQ+u5BEis#~wb^>oSv zgd;@WtORj{e1bS;2oW;TI4xI|@W1g+Cl0CaoGXgvBK>n*x|%eQ++sBq5$aSvkqNfP z!v@G`4Z$>LfLyHs2&hvP^-mWvh*IrtUl&oe<*d+l*#3vCq3W;jlS}3vgARRx)SYzKU(oVrQ*^3-|`0iQsW{>lwYq8#-{RB?Ezn{1>U5qZAC+>Crrv8l?l`Z zc%BZqcXCazJDlLN`Q%pHt12`a zE2dsbC^=Q0sg3c~UEnQm5Ys(j-Tcvi1?uIZVS}WTy^{*bZV-{BsN|JyAoZX-4QqfI zA$q`0B#-UqOP7+fyu#$EPVAH!#ZR>%vi8`tcAMC6zd#q{5oFnGz!Sw*z(qy{JJgK| z!RV4O$O#U@Xb|EH4RwZfWfNj(bV5}!dV;$on`00@K$ z$o?9kR3R{6MytXrlyRpaJFsz_jVDk2sDK~2zgb6T-sZ1!zjPTsja2NHrcD%F51Z4r z?lxNB0Wzgi7{C+uh##sDsK?-KpSAWN6aq~UfwfxM{57MHuAaI3AUhtfm-K-jIX@8J z^%fp3>vI1bR2b+^?suPSkCYAy32q5<>==fEZu-EGsaEyb)wY8Ioy(@s2u!dwu@62A z`W=JE>hlaE7#&7aVi!_T!I*hrv2zTNbC7YG?19%@K{;N5$YZA~eeY4Gh_i!dSO~@dc$frp0LzEqdU^5)pY(F1 z$;EQ>Z%%Mg3z?z4e3DESgt@^JZ^+}G>XvmU5DtbhuHGWB4RdGcHU)-E-@>?b&&|rH z%au{b+bWAF3)=eTN)0t$!W-P;IL8Vu39T*uOk)*0 z3(odg{+sGl>V2T+W9{1c+RHQEjc0=fZBmjOTjkv~`CoeE#(EU|N_JL$*LO4UP7v2? z7;I0Rfg3;vFeO<17lyz_^fHKh0Z+X%rG76AL+WKqX#>Ln>mG!4582P6UgDG1^*Ud~ zLe;vft(d8zr3Nmw+%rbjQ#0jiHBq^njO=r=dS5%Y!@!$^OzesC0~LHi=VTk+MG!mq zb#h%EVv=HYnJ?AH*_ zl$}0IItDQ@<_R-lt+qI%2vmUDLXQWX;8>Q=<-$*7LR+m3l0NwKAt`jqjx99;S{l&4 zh*v6{y_1FX#HCNGR&Pfd?QjgmAELV*!67Owso-l`;iP7jaJ?p`%q}M1j*5%gKH9?r zOUwUT2ki3r#yY5L3U?^vc~9T#`#8L~cEWoR_>~k~wG`RC-tC`|{sw6y9z3eXj9`B0 zS>JF3OwZC1eW#+;34oYhcfp8nYmkr8!`bU@3>j4nFFBAcw6=mEl#}tf%m8PA_|w$* z(f#qfxeq3KW=Zh+kT1m#@s7_K-V+N+-BYI0cR>MXg8OfzCDv+U`B>CPJeS@;8jM=!T+{MAbPyX5R!&m%29@v(pWbamn|utw1_+&A&6{=;tZ_d`+Yhv5B& z?Ec~|4^b*Ros?Gw92+?|e6R#@(J+7&bkqZ|o-hQa)<<_KB&+Fbk1Bv)ckUBbjd;|* zxqDdjJha#00B7aQ_|jYXDNG2aZDDV4&6`DTu57Jx)&G}UqNtUEp{P`^9!$wFkZ7<9 zTX0<8YW+2NpBHmk!Id+kur2eGHualz1~j20pT_$r);k0m7{VXXYJ^0KYBO9yg3fL0 zL2Z4Ip8wh*{VO`*10w%;-kp+v6B2yu9K@(K=xmlhQq0`$US9l<%N$|Z_x=Pjr}S1% z?_mowUscV!x3e?#S{g3xyQ2|k)8r23?F_4SQ~jjTJWvPmoMDdoI1f`TYT;FXpLCV( z`y2AIcX=ye9$+8$UQpBJGUzWoU!r)jIwE{j-4&wEPTWthOniN`#_Oo7 zG~?Ad(;5Y+9)%LC{CAgQt(XOa)mXmfj$jt+s<|_m#*&yjfEQU?&?tj%{}K!W`KW+E aTS5Onw2WSs7y1y`jCC2k2R3NB@#cS$QOKbH diff --git a/summary_results.json b/summary_results.json index 85cb8dc30..6fe9b76c5 100644 --- a/summary_results.json +++ b/summary_results.json @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 5.95448, + "Performance_Result": 7.57138, "Performance_Units": "Samples/s" }, { From 4d134f93588ac63481949d139fca1efaecf8e9a5 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 18:16:25 +0000 Subject: [PATCH 671/719] Updated results summary [skip ci] --- README.md | 88 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 546 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10932 -> 10932 bytes summary_results.json | 4 +- 6 files changed, 344 insertions(+), 344 deletions(-) diff --git a/README.md b/README.md index 7f6f2b483..a93f53a99 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 18:03:08,238 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 18:03:08,238 submission_checker1.py:3308 INFO] --- -[2025-05-28 18:03:08,238 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 18:16:19,774 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 18:16:19,774 submission_checker1.py:3308 INFO] --- +[2025-05-28 18:16:19,774 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 18:03:08,238 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 18:16:19,774 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 18:03:08,238 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 18:16:19,774 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 18:03:08,238 submission_checker1.py:3324 INFO] --- -[2025-05-28 18:03:08,239 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 18:03:08,239 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 18:03:08,239 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 18:03:08,239 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 18:03:08,239 submission_checker1.py:3345 INFO] --- -[2025-05-28 18:03:08,239 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 18:03:09,508 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 18:16:19,774 submission_checker1.py:3324 INFO] --- +[2025-05-28 18:16:19,774 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 18:16:19,774 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 18:16:19,774 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 18:16:19,774 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 18:16:19,774 submission_checker1.py:3345 INFO] --- +[2025-05-28 18:16:19,774 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 18:16:21,011 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32038 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71933 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32038 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74388 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421657 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57138 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57138 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41754 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 5b86c14e5..e17bc2438 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1298 +1299 diff --git a/summary.csv b/summary.csv index b391286a1..ff0d883f7 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32038,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71933,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74388,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421657,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.57138,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41754,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8191,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 254b953ae..54cdf5cc5 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.431769, - "Accuracy":"mAP: 76.951", + "Result":6.32038, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -30,46 +30,16 @@ "weight_data_types":"fp32" }, "1":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":23.0274, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -77,8 +47,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -89,17 +59,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -107,8 +77,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -119,7 +89,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,17 +119,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":22.9802, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -167,9 +137,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -179,26 +149,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.32038, - "Accuracy":"F1: 80.0", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -209,26 +179,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71933, - "Accuracy":"acc: 80.000", + "Result":22.9804, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -239,27 +209,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":14.9346, - "Accuracy":"acc: 76.000", + "Result":1.74388, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -269,25 +239,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -299,25 +269,25 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -329,27 +299,27 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "11":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":15.6951, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -359,57 +329,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.290026, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.421657, - "Accuracy":"mAP: 49.593", + "Result":14.9346, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -419,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,27 +419,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":7.57138, - "Accuracy":"F1: 80.0", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -479,27 +449,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -509,56 +479,56 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":22.9802, - "Accuracy":"acc: 76.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "18":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":7.57138, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -569,27 +539,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -599,17 +569,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -617,8 +587,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -629,26 +599,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.438894, - "Accuracy":"mAP: 76.951", + "Result":9.42443, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -659,17 +629,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.290026, + "Result":0.457477, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -677,8 +647,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -689,27 +659,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.6951, + "Result":23.0274, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -719,27 +689,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":0.431769, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -749,26 +719,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.8191, - "Accuracy":"acc: 76.000", + "Result":0.41754, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -779,17 +749,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9804, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -797,9 +767,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -809,27 +779,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "27":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":20.8191, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -839,17 +809,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.457477, + "Result":0.438894, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -857,8 +827,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -869,17 +839,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.42443, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -887,8 +857,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -899,7 +869,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -929,17 +899,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "31":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -947,8 +917,38 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "31":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.353561, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, diff --git a/summary.xlsx b/summary.xlsx index 5fb3aef36bb217add216c50b4890186ffb23b41f..bd90b79a00a7afed5bbb09430c8d0529dbb6dd73 100644 GIT binary patch delta 2080 zcmV+*2;cX#RkT&GatH~?&Jee54FCWIvwH|^1Alzi;n?{cJ0GzVkG1C*3B)W;F#_0; zh{TRWBz7bs9(=ES19hZaK>Nl}-obJuccQamz4&PyVZ9@+6X{B4*E-_oeC({v6KoZzV{0#Izh|R zmVe7p3^W--BWuOR0e3hhS${v>oj`C8yDlvlt?af}-ab3B-f!6eGYQiAWYn zM6!qxWr>I8tIz-=sh7{f-WcjTp+MzKHh)BH1bG~lMO{)PpLo2*j3b}Eg!KY9>GhMv z!ML~T;d9`9>!WEM%c}t4WmtePY&qI-bh!PG)2J+t^uBEG&;tsvVRzZ%e5R#<`1cbkc=5%pWHuL@_dT`>8}@6Qyz-0r#( zOw0t11e3gM4`7lEfJrg{CT4KA+=eKaqz44k&_hSbw;0Z#C8DGF5=>dvWo58yk`Qd0 zg$3**n1wwvCsSK3^#nBQSYwsVbAK~2wRsh~75HAsvq{C@gnFj%eI3`6XksRCC7PH) zG)V@~BpE;xGXza-LX-4>Xc~IxY;Sx(Q?^8O6hKp!bx9e#zRJ3`Es3W751?7c8mqkI zx|!Ja*`;f{o+@FJzhnAL;Tw^zFWJOQ;7K+ygKUxvut_q&CV86|*|be;l7AkMO+ycz z?Trs?%9eS(laJwuvF!Htl~Do9am?+B())Y1_1m+h*AEEw|)_AI0C|f2Q!A zb2pG^lK0kZfWG*(4cYlVpHRX3_wgu8B=1 z+5nq|9y;3_AJ~*F5gjFzY|64OE5mIQL%40a|1dV6a-^+ejg_`dx43P(EiZJ-&@9n+ z6nY_^^mj-mkOB|MAet2v9+m+#nZyHVnk*jH1EOgXc@&sD&=S#60Dn!8c}N*jBv`={`!zIA_oju~DSUc@AY=S_a!DaerXjtXb^&N;dVAjwkx25X}nS56b|W%)Nm(Hcjpi?E%m<=|4)!IcLxkkx{_$fvPU6 zf?bo$A9u}~J)cL&6vofjL%FB!&z0Jhe;n6${F2}DP+_cHSQWPP*Hc-`>Z+Vz zbZeJngk#X0uZ!B zbed9{g|aRyLntcr3z!KyiI!(Y5^%BpF{~X%c@}2Y{u?`_bV2Kubgh zL605Ex}*#|cGfj`?7C*}2eJI-B6}@MtRk}OqW0spY;Bc^?Aj^e68#ZRYCj|sO6`Ya zkjo0%4?fEPmm~vRnyeqz0dQ%Oel$ow&=QeB(3OR~gIux2@}szPnV1AJ(4|_YlvKXljI~Tf7>C!2-|_SZX=nM|6>xYsV!nB1~s|{E$z(oP19ATzH zNAC^6*s&u>>XP74LELLAsSl++U_;qAJpj(rbcH2&f6A$+xC6#!BBBAw*@QGdw&?+! zVQ4^t@(yQ7hOb@^R=U}SUb%5f;m-zUt)n=(Pfkq6wr!Jk(OX0Q41aE4-uhKzp}JEs zKy}F(VHT*@wl48)5f0Z-=eOa+djb6M9Q2vDVUbH8@F)Zzm$;+yv3PpkZBU)3`7%zI zabE1Q8G>XhlBZwY$h-b4P!2=>O&kH2-1ncL{sXfnCdvW{$j%VAZVdnc1+)1mH~|E; KRpQ-~H7b9wugf(6 delta 2061 zcmV+o2=e!|RkT&GatI01*5ih)3Tm-+&z{7u3Enly|UP$*t(DSTBAYM_BKOD|zlqCP|H$@>?=3EjF$T zzeXl+Zm)a&{3*>r_6%m-)xk|h@Ztb_hiX{Q2XU_g|WMU+?D5TlazAh9&{(AnO&1ru5#I*QoI&!MbK%H*q$Z&}w`{9L*yEPr4> z@O;=Ab2JBWJj))E^F@4pr&>X_Ies;uwXCoLq3<>mTO;bXTwfK=V7p-Q*WaHhe7)Uu zC775A90?|Q-5$Us832=H08Gr_Zn+InFi8&xrlE(9lJ7B`K}$qO@g zHVX^bM=%R}W=^KITIvaC*0IJan}6qKVrug;bSv<^l4p~OzYFzD;Tt=yC(*=A;7T+x zgJ_Zrph+@-CT0kl+Jq+Q0ns${(AnPjfTnDT=qP}uEbEdoc!8C5ZCese{U1QHjx|U0 zTjy>d(Ijuc`w~seAetltXp#(|i5Y^XW0MUI7k_S>q-)$Zo&Nxub*!<{w(0C{n}MVK zvTd`(N>Qj8xS?bdGeIEP#0;`YGQcLu0Gmvu0XAI|o6NKUHVr*=wl_YoDO(~sN+{Ws zWnET=+a`u^+jRe7Y(C{kTgMtJZJTa!+jLuA=$4^b;_oOFLp&+)kW3&A9+E*cD>ytX z1Ak~TiwDp&X*{e4MAKyQC{THzC8DDMnqc#gGTb&v*SKxgDfE0VH22l;QM-zd4lS!p zty4RWkM+PN3L>7Ad`KpgmJi7wn-x4CmH{@2)Pii9Tp!j0vT4$Nl$3GKpe16XNSX5- z%DS`+woT&xz_wYl*z=Wa>L(p(_tf~gRDVU~M>(y~F4GKDP=Y23Bc7CgNG6ol56K{! z6}%so0W_I=18;1a+#lKlplQ;7l$3MMpd})sfa3#IT~-CVCYe9(nl*bqkB}*hpRb2< zPu-s@wJZNPuJ8CIPOJ(`s6?s6lll+Igwp>Z8Kkm;0mL#uB{OV*N|OXcdjKj;8h?-i z4+vT!HVXV|1C(`18DdGgMlAQu1}eqUe-f5^Y6D%Va2HEE2ttQrsV$DBMxDfy77)pV z(gGqGz>;MomMkN&BpD@tPwJI6SwL(Q`MV;f4muhnAZUr$H05s-okLldmdRr*3)y0l z7rJH(X!$War7kyI3MSLL$?71RpMPLg*wSB5Wi6|#a)Q-G14y-e6&6U?g;JG3i6)6B z4Iq*Ur2#}T2xSEWh-CmuX4Ys4rO5zdqX3j90Z4-Y1T7Jrrj%x(tjo#}$_fIIu2}%u zVe;=VXKPttB`k~fPxhoa3@lxOr7pn|H4;zSKO_@M`-fx@%L?`n%K(xXp! zT$-dG4bl&^L}UE4Qud&%V1ID}UJ`#TB*#ZON&D zLS0&V6VfITwSJJ)I(hplPU0lsQ;(~e{dRUywb-dH`32{t(KX65f)dc&2&p$UdT*C$ zi4q@}7OXQGYP1KB7R_D7E#=0+t1}i{kl>RD(w=f#qiqP5V(j@A6!Teh=qRp@Q!K>N zZLnqhn{5CWgxq5Vfqw}W7(M_~I}*_eDfo!AKdu`9f@A1Ffes$$S&lE>08aVIhC#V< zDjD_`Caq_2bnmc%-cE}g`#>Zy06oS%D<89`=XQyjf)w+V%+sQ5^BIzo86^*2hmkk^7oa)~ r`8V+lIOe|n1kE3_CML=P3DVhJEp80}00y)9C^!KGPPQC%lQk-TM_a8m diff --git a/summary_results.json b/summary_results.json index 6fe9b76c5..d875a6351 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421657, + "Performance_Result": 0.41754, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71933, + "Performance_Result": 1.74388, "Performance_Units": "Samples/s" }, { From faf9b5e8d59bfaa49f1dffcc0daf264dd4ef82a1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 18:31:11 +0000 Subject: [PATCH 672/719] Updated results summary [skip ci] --- README.md | 86 +++---- dbversion | 2 +- summary.csv | 46 ++-- summary.json | 546 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10932 -> 10935 bytes summary_results.json | 4 +- 6 files changed, 342 insertions(+), 342 deletions(-) diff --git a/README.md b/README.md index a93f53a99..d18a24b63 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 18:16:19,774 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 18:16:19,774 submission_checker1.py:3308 INFO] --- -[2025-05-28 18:16:19,774 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 18:31:05,769 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 18:31:05,769 submission_checker1.py:3308 INFO] --- +[2025-05-28 18:31:05,769 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 18:16:19,774 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 18:31:05,769 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 18:16:19,774 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 18:31:05,770 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 18:16:19,774 submission_checker1.py:3324 INFO] --- -[2025-05-28 18:16:19,774 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 18:16:19,774 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 18:16:19,774 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 18:16:19,774 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 18:16:19,774 submission_checker1.py:3345 INFO] --- -[2025-05-28 18:16:19,774 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 18:16:21,011 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 18:31:05,770 submission_checker1.py:3324 INFO] --- +[2025-05-28 18:31:05,770 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 18:31:05,770 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 18:31:05,770 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 18:31:05,770 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 18:31:05,770 submission_checker1.py:3345 INFO] --- +[2025-05-28 18:31:05,770 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 18:31:06,739 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.32038 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9802 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74388 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57138 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41754 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41754 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57138 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.7262 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index e17bc2438..d77253057 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1299 +1300 diff --git a/summary.csv b/summary.csv index ff0d883f7..7957c9fec 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.32038,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9802,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74388,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.57138,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41754,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.57138,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.7262,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8191,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 54cdf5cc5..525dac9ce 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.32038, - "Accuracy":"F1: 80.0", + "Result":0.431769, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -30,16 +30,46 @@ "weight_data_types":"fp32" }, "1":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", + "Scenario":"Offline", + "Result":23.0274, + "Accuracy":"acc: 76.000", + "number_of_nodes":1, + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processors_per_node":1, + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -47,8 +77,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -59,17 +89,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "2":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -77,8 +107,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -89,7 +119,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -119,17 +149,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9802, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -137,9 +167,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -149,26 +179,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":6.28304, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -179,26 +209,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9804, - "Accuracy":"acc: 76.000", + "Result":1.74388, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -209,27 +239,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74388, - "Accuracy":"acc: 80.000", + "Result":14.9346, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -239,25 +269,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -269,25 +299,25 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "9":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -299,27 +329,27 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":15.6951, - "Accuracy":"acc: 76.000", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -329,57 +359,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "11":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.290026, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "12":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":14.9346, - "Accuracy":"acc: 76.000", + "Result":0.41754, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -389,7 +419,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -419,27 +449,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":7.57138, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -449,27 +479,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -479,56 +509,56 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":23.7262, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":7.57138, - "Accuracy":"F1: 80.0", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -539,27 +569,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "18":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -569,17 +599,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -587,8 +617,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -599,26 +629,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":9.42443, - "Accuracy":"acc: 76.000", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -629,17 +659,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.457477, + "Result":0.290026, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -647,8 +677,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -659,27 +689,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0274, + "Result":15.6951, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -689,27 +719,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.431769, - "Accuracy":"mAP: 76.951", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -719,26 +749,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.41754, - "Accuracy":"mAP: 49.593", + "Result":20.8191, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -749,17 +779,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":22.9804, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -767,9 +797,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -779,27 +809,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.8191, - "Accuracy":"acc: 76.000", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -809,17 +839,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "27":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.438894, + "Result":0.457477, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -827,8 +857,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -839,17 +869,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":9.42443, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -857,8 +887,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -869,7 +899,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -899,17 +929,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "31":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -917,38 +947,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "31":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", - "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", - "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, diff --git a/summary.xlsx b/summary.xlsx index bd90b79a00a7afed5bbb09430c8d0529dbb6dd73..d57463fb463c2949e5f257e3746357b6d043167e 100644 GIT binary patch delta 1165 zcmV;81akYdRku~JatH+)8bIuEvv&xt2!Gmc`MU6|a2HEmf+cDsp0s~RCY1IM$sm>$ z>>ri^ESXgUSeon~HVR;Al7F^0K41y*51l4p3GxpqgA*+48cwi<1^i2~+*k8Q@0veO zcHekqXIQ(0OEgG4Y5tH*D9s;|K`twpKP&@Wl8i60G>Jc~1HjVc{b=xhpd})Mpnt~> zWnEGR9y{wAJa%2P_k&n|bCJE4B~}sHby55AS~{?;GA^AGF3}(Hr1nEHq11jz2Dz-D z{jdyhNix8t$@*a(0GB4|M}zbOEfE<6U0EpWvNAly5;F)+u{BFSk8$};WZ7C4S?MHO zw0^X11y13YvUbUIOUOid#FNqw$&(`z-30y4v@Vka6Ho#SRFhQ`l7Ez~Eey)uKe0g9ZdG5g7zLcBtyID!Fa4 zI|Q~(d+#iuZ!5Cbvc4)ird^bPvK`ZMwJxKw!~(L>C-J2PL^8^f8W0H~l@&A~mH{eB z2BRg$3+8mQNDdYgu0`|kgf3Nf9yTifYM=(qaonMRokG8V&t1fv_)_-di4B*`j+F~$s`&4xSG^*I{I@N|M)}y2k-EQzW3A54gFK&xREXWKANp= zHA?;ko7TzEtalTi_vSaF>7p@?huK1{mfvj5*Ppz%`X@R~W~;B1!ah~#WSnjhN?0M0NpAVGPDvn0b;uLmpLY{Q`3IHmBNfm!QCoZKfTCS%*S zNxK-VA%BKHw=ZwQsE&0RGqoedcXkP3o0fXBhJ&XB>?~cPyzq|5dZ)H f0000000000fPvMMOe#JBDU*vTBL>?i00000VcHjj delta 1202 zcmV;j1Wo(5RkT&GatH;;&Jee5vv&xt2!Gj=<}k2y36{D9OVmg_Y5$N+DD5ATK`blS zKP&@SGOGr#G}%9F6u{CX|7>r3z!KyiI!(Y5nA(O22SCZvUbUIOUOid#FNqw$&(`z-2*<=ERzHiPy#$#lU5Uwf3)@uN;XcrL<6$X zC-J2QL^8^f9uNs3n-x4DmH{?NMn2)RO&SpE0MImPKsIPV&=QeBz+;E1E~}E;Cc8sm z+qC!20{XThdoAm$vSZps2`Jk!Em!L@DoZRN8+{UAT0kVDEU5vJ5K>t|17aDVl4O8N zlLf>&04hxu(Dud$e=5NOVwVj(b|~wTGQ_fq1yoqTzGL|$k-e7nRl*V^AiwnUQi>9g z3LdZ|pF<)gAd*p*w17wmv8-SLu?%3z{2DmOnlvCb3UF!CfNao!pe16{l%IKV4rN_h z1_#*+fBwhba}6jR=18h1`9HO+tKRdJHP6Z7iDdye*{T1Nqp)5kc_gV07OEF zWCa0;WdKR0)&PKS(lapk`)Xfd(RD^4U*xLkla`C z$98s+3>3VF*)Dv_+a#Xf#HaCiylC{3>~gIux2@}szPnV1AJ(4|_YlvQKXljkHs2A#ZpWyf}tE+s2}=nM|6>xYsV z!nB1~s|{E$z(oP19ATzHNAC^6*s&u>>XP74LELLAsSl++U_;qAJpj(rbcH2&%BiQg z1IA_|q5;X-gfu_4=>eQ!Xh4GU4rfV*uU-#Uy4i+axp7M2&jx0#qd2)wPE5wOZIgD< zTSNW~e{Nsi`c-40x>GSxKy}F(VHT*@wl48)5f0Z-=eOa+djb6M9Q2vDVUbH8@F)Zz zm$;+yv3PpkZBU)3`7%zIabE1Qf@CX_r(fO3yZ$Rs4nzJ;908Zy_n)Bt1G6b6$^r$* z&Jee5llUlG0SJ>+DLes^laVPw4SyT}0C;RKb98xZWpk4tDio9RDH;Ot9+MCs6_X?? z5FE8t;@u?y002+|000pH0000000000004l2(j=4dBomV)DiD)5CJ+k%000000001h QfhLoVDkBEiCjbBd0RCbp0RR91 diff --git a/summary_results.json b/summary_results.json index d875a6351..e5eebeb33 100644 --- a/summary_results.json +++ b/summary_results.json @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9802, + "Performance_Result": 23.7262, "Performance_Units": "Samples/s" }, { @@ -752,7 +752,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.32038, + "Performance_Result": 6.28304, "Performance_Units": "Samples/s" }, { From 3777b50e3c28b2906a12db486339356af58fbb07 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 20:38:44 +0000 Subject: [PATCH 673/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10934 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d18a24b63..0e4953f88 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 18:31:05,769 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 18:31:05,769 submission_checker1.py:3308 INFO] --- -[2025-05-28 18:31:05,769 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 20:38:39,189 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 20:38:39,189 submission_checker1.py:3308 INFO] --- +[2025-05-28 20:38:39,189 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 18:31:05,769 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 20:38:39,189 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 18:31:05,770 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 20:38:39,189 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 18:31:05,770 submission_checker1.py:3324 INFO] --- -[2025-05-28 18:31:05,770 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 18:31:05,770 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 18:31:05,770 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 18:31:05,770 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 18:31:05,770 submission_checker1.py:3345 INFO] --- -[2025-05-28 18:31:05,770 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 18:31:06,739 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 20:38:39,189 submission_checker1.py:3324 INFO] --- +[2025-05-28 20:38:39,190 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 20:38:39,190 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 20:38:39,190 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 20:38:39,190 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 20:38:39,190 submission_checker1.py:3345 INFO] --- +[2025-05-28 20:38:39,190 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 20:38:40,405 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74388 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73182 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index d77253057..6fa4b873b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1300 +1301 diff --git a/summary.csv b/summary.csv index 7957c9fec..228cb6a98 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74388,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73182,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 525dac9ce..b44ff0e55 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74388, + "Result":1.73182, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -229,7 +229,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index d57463fb463c2949e5f257e3746357b6d043167e..2ae48c31418b9792be98279e52b9851c82f54ce7 100644 GIT binary patch delta 1719 zcmV;o21xn0Rkl^IatH<3+YZ5Svv&w$2nOF)1&MVRlid(Yf7&aJ{xBoj*Whh&h;3g!>X0GA}=ODs*|59tRN&3+s{Xk1Z20>RA%DSu!PqD-df>UhG($8aDeiK=? zmPJ-N$ri02ty_Up_@%5}GTjm~Q6BN6^g}YClzvDCf61&M{jdy>NismD$@yU&0GTG` zM}zVMEfE=nkO|5UDZ_&-`?Ef9kgZwy`95Ussq=HCi_TB}f~*(1CBK%by%IFZU*wU} z56Og5`XL!avx4-)GJqz@0GcM}hjjpGnv@?6$`7_$Ne0j~DLmXv--2-&P4{jdzM$&?*t=GYIz0 znx&twhw~{%+C5c&E_G4;$@Wc4`(-Z!m6$&o6%t>XKP00pss4}VHsePi95ii zN&R7?0GlTBM}zqTEfJk2unFc5DZ`Dkiut2!Hh;bkoBQhjXzBkb-z#~#X_xpvHi{&^ zfAoJyMp;q-A|Zg2Wh9&|BjIEd0yu4x0K`U-aGE;k*dPEwOT?xL9y^qEX_?$ONeDJh zd!PKDze{AVWsy}jPP?c9WorwAviDEy5)H^kpTw6M5XmS@dO#$EY*z4qSO(Z68To|M zHfcbt13=TH0okAdK}$pi0goN3x~xiWf1B(Mfo;>?I}7OBitM$lugZ>T7bT!<$Fy9n z%cv}|fNbj0=USwPzxAE*Qih+Q`D*rBXT z$`H#c7EoaU`;O(4MD|+NR|!jyfc(d=80}fJjDJ(gGqO#Ik|~f5b9? zCG%_GAZyZq*eJlINdvM$1A>-_O;dj6#W|F9X&D@3EByH%d(SnXbeJQlp5*`3vbIX+ z*P{QUt?c}&OJ0<^(Pte>O;lPeO8E#UI<*MKVzE9%j4nDQ}Z_eiNU@rcz`4o`pAR^C~E zc)1+Mjr7YbK5oXJXW2cJFSgkjd>*bpC$BM^kCOQ){jwfC|De9*xOXy1X$C*8CbgW7 z{#?dC{?PxyJN%*V{q%D~|I|2cWJ|w~W~*C`l7GRbb#gT8-Nfg;`ORp$XpG}wwot3( zHyiWyC-1GkPLtW{E4SJBn51bkx&Q7q?hWF3{$2L-ZkQzL{Wtj~*?xlI$zQVz7T691 z6?hTeljkHKf6HpaFc3xe{R*SKEk8(ch3!CFvTC4EmzM5?G)Y9Q2a;OHe_z>2oCLD! za&+&UxihF*?qwJKf^|Zx8f8g}B2Y|op|&-8Z&z`NBIl{%w9^V|G=M|P=B{FfFm2)0 zY6I2_a8Up$N0_P6&U-^Jc5DZdx+FMM5I5RN>O*O_f7np=O}BvaG`+_XJmu6=JOX1g z6VVaL*^D$lwwnN)Vdy}D@(yQ7hR5VzP$( z8U9?qyiKddLXD?lfSQUk!Yoj)ZByaPA{?%x&acCXHv;(M5cC^w$3+f(z@rd=9O9mo zkK*aMC0(H=PxD2bF5>>oA?U}=(nwl_Xt3Gxq}CSVEj4=IBaEbAIhu!RNuOR?Nn z^GENRKTdYvcx7i;yM#+LNIYr&kW47eACf^XE0{kl16-1fFR?UmE4rPB`QU)G7>l!?EU9lU^usbhCdmMqCg+EB0A!k! z9}UV6v_xbOLMA9bqzn(T?9ckZLAGY)=lhVkr_RrnE;>K?3$kA5mi$_#_Dawsf00K@ zKO_@M>4#(x%?i>F%K(}r18AC@AJze&X;OYPC_m5=kwMUug|aRy!%dTTL9l7oto(c} znooaHWoubvrA@Qw{AjzSov=j5(uO$`8pXOFBO!glJZ9epm+3BpE={r2McB z08Nweqrv!rmWW+8L=%i3Qii)G=^A&jv@UUeG}0GChh>6 zCiRDn0&JSh9}VUYv_y28z$TbKqzpICD&~)_+5GuFZ0@W7qox0&e6Qr`rd{Iy*eHLJ z_|pF&8D&WUh=c%6mXUC>jD(X-2;j6$0uUQT!fEQDV}k$$EfJe0cd@tvRT0cVi{nQWaJZ0 z+oS=p4ggJ)24sT<1T7I61Uz=A>au?-xoxsL1h!3k?<}BiE3((JzA8JWU6g>b9n*5P zE~B!<0cnV~4UX zDMKu)SU`mZ>^qiE64`56UnML-0`f~gFQq5}so()i@;M|@0wNh@NehUC5X*lG77)t- zmdvk#gRDsdVxs_;CJo324G3ByHck1N7w1scrDbrCt?=i6>^;|j(qWFIdXoQB%i1cP zUyJ^awzBi9E_qSbc1n;$lf;+)56LJ?3P2=;NLCPlSO$<}Y7HQ1GJx19fTYO)vcUj? zmWWMLe#Xo>lyzwtAX&ixviE=70NNlKJ_*Tv6@P4J7s)`udzkIQr@T$#`AvKpkH?Ef zKgllFx^mmP{^+|)b@*ZZDRB?+Y`kPT80CM{zBgWfrd>Lk-ZU0}u0Ji$J3ReeTX|>w z;pK81H_|V&__!H=o@Mt?zSw4C@OilYoV>Ad_tVb}{Zr$(kuCi`nyqd%O8y0#*2&SVcN3rY<~O71qA`w#*+Q+B z-)zj+pS-vFI!$J)uiR$iW0I!H(cxHpLB`FGjRyJ3=~_uu4~Wcvw*Cw~D@O9KQH z000080000#0OxW7<6W~I7R3$(BhJ&XljbBJf6Z#cFc5_AeF~$yE!!c%2-|_SZXr~`-A^+U-FVcNp0)ds8=;GzIhjxbZ9qxXhj?AQ?`bxCljAnvu5)Q8d@ zf3TtKn;rn?X}ZD^Jmu6=+yi4X5z&a`Y(knJ+YA8CFf<@Rd55zk!&k2dE8T3vpxij6 z@STBK>qVU0CnqLj+qOx&7_1?GhCjD2Z^NpwQ2nVGpt|IYFbmXcTbKB@2#0H^^V@LZ zy#W5$1%2jiT;$vbJPHBGIqpUISUf%NB{rze(|j4H%Q!D~SwR*VNmpO}$h-b4P>w_X zO}qfkx$i$g{RgusCdvW@8X7?Cag+BbTLBA`Qz<+Fm6MPuK@EZ&004MwFLQKxY-Mwk zVJQ@o@+leu^d6HB9~F~jDG(bY&eO0Z0RR9{0ssIJ00000000000001hfz^{vDn0=z OlZ`4P2HPh90002P&>%nn diff --git a/summary_results.json b/summary_results.json index e5eebeb33..e92113449 100644 --- a/summary_results.json +++ b/summary_results.json @@ -631,7 +631,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74388, + "Performance_Result": 1.73182, "Performance_Units": "Samples/s" }, { From 6f830608ce212a122c913abd30e44d196148c7a2 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 28 May 2025 21:35:05 +0000 Subject: [PATCH 674/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10934 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 0e4953f88..4aaa2be7f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 20:38:39,189 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 20:38:39,189 submission_checker1.py:3308 INFO] --- -[2025-05-28 20:38:39,189 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-28 21:34:59,445 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-28 21:34:59,445 submission_checker1.py:3308 INFO] --- +[2025-05-28 21:34:59,445 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 20:38:39,189 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-28 21:34:59,445 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 20:38:39,189 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-28 21:34:59,445 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 20:38:39,189 submission_checker1.py:3324 INFO] --- -[2025-05-28 20:38:39,190 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 20:38:39,190 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 20:38:39,190 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 20:38:39,190 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 20:38:39,190 submission_checker1.py:3345 INFO] --- -[2025-05-28 20:38:39,190 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 20:38:40,405 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-28 21:34:59,445 submission_checker1.py:3324 INFO] --- +[2025-05-28 21:34:59,445 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-28 21:34:59,445 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-28 21:34:59,445 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-28 21:34:59,445 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-28 21:34:59,445 submission_checker1.py:3345 INFO] --- +[2025-05-28 21:34:59,445 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-28 21:35:00,508 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,9 +34,9 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.41754 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417282 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 7.57138 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.53809 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.7262 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6fa4b873b..47e9cd481 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1301 +1302 diff --git a/summary.csv b/summary.csv index 228cb6a98..e867d606d 100644 --- a/summary.csv +++ b/summary.csv @@ -12,9 +12,9 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.41754,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417282,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,7.57138,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.53809,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.7262,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b44ff0e55..2d755c089 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.41754, + "Result":0.417282, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -459,7 +459,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":7.57138, + "Result":6.53809, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 2ae48c31418b9792be98279e52b9851c82f54ce7..cdacf05a7af13c0268c7396f93e88921fb750a67 100644 GIT binary patch delta 2990 zcmYjTYdjNdAKuKldB#!>IZU=WG|?JS4$C2ig+fMhXwy6h<&fhXLz3Cn#8eJZen~b( zrKCu0PC10=>8asCPUVzgyz{*8hxh()f4qLz|GKXKb>9shjUIKEq-6+EPdd7B zM-L3d&&AfeFz-5ijoU@`hUZz^46Jnzy=n7^VH009O z+{*2XaRxJ`uiZzgTLRUcs$2L8uUpD@KQRw`PEmW8QlSjH#s@B|X{VZB4iI;RE9C6Y zaZtqBmMbf}!Y811HgQrN3h!D{_78%trP>w@WaBpsikN#lUPy0Q=nk?OvbxFAF%5R3 zz0IdDsmYn;YH@UR<3+QWe>iza3`NT(7TCgbmED+E$j!2HMznYVW;U;^dAF>4H_?XD zc7i(TMi@ZYEf&4^=`>vwIC&XH(9c;B)UGEy)UcHL+GxodWWSV^KX+9^x|La>Zl5)} zZJNj9o7Rp<rkh`Q{U;oXYB~?$vgKTOXml zZfz%%{~-CELi!~ocDxwmPxjdb05>S9ldaviD$KIS2&>c*L;lRMPqLh@Uk`zgNSS-2 zlNyVLRoxy>jHG%-!o}BMJ)yXw4phmGYx2jlWb`N&of8mCFsj3{;Kg) zx&n<>l_$1QihgN&9WQFYF?eWNSbbeel6ZaHg@1^DKCHZx5~Lnd!->0ytfH70+BAy& zeIU-HdacQ|N3oeb77xXDir3oiKAer@0*c`va^Tg{YCM^yA&<9#+@8yWka^7P8)9T0 z=b&D?k{4m90nt0K4aCfNoxeGB_Q>v0%ciKXq0@-a#YX45dWkxHD+tyt8FMZ&h>E8H zO>mOK@P(!QtcDzXABaNUMfUiCm2 zD<*Oj?9)8KeEG9tEn!ncGkRVUieHp`Y1rEaF2|u$cBtOcA|FL!aFFb(vK`f>FeE2i zdmx5HC}ezZz`bY>>lZ*u3IF1C&o`);ju?)bP@HNMGmBIAIhq{WEl$IC!dP%nC=Fc4 zbyxN8H)eC4_xbmKw7HNl24Zm#D)|FzYF+aq93Nh0jAmS;n~*e%z|$zJeH+yt_^w?#YaP z**SCW@SyM~bPb81J z5Sn1DmQX*rBwh4&O%Sj9dqDX7RMTSq*!m5>#SjgBR2j4q>hi-1zT0s-i%8|kO3HTt zZg$oI_3=>oIp70KFjm$aM$Q@5{rOPNbr3x-I$ zNvq$eY%pc2Mnhe^u_Cx#*+hU7v%rgUy^Tg>Y+6*G86g7MZsQBrZXKN>Q&UP2f&Glv z_$bk4c_Z`LC}J+|+*Gka>BRe4#)~fzdbgYxkPp*ClhZr`oG2DIpVrj!WIUDRV4oJl zJXSZfy(u@bImjFxIT%f*5@?H-CP{#@NdU*7L9$FG7Z6whU?71G=>Ty9h1*q(pt70{ zo;*)hj9NdMJ5;FP^5>36b0UvAWJe$!fIuJ=X^lyqj^~+1jg@4;JACU@fp5)lg*w1)=Q%u$iDRgekaAw$> z>j|mOx|N!=5Cu(1r>2TL3gJnFgZ%%te`Gl&rrDl?$N-s^BlqCc8jbSk#UML_buNfyW1yLZ&I&w z?k(9^X9r14W|07*3c}!dkQu;7jEa@&b&94al_VR?n>8)cvk+Qs%Z1z$DW5>k(x&dL*uC@#^I>xPJ$ zxA@e56-=VvI$jq(h!Q4Rl(?An%~!8quFn18(h%-FVwj-vpl!4F0foA@<1Puel9fU2 zlKmm_gysANXoCL%*(WiKDY8=S6fvB_q~eVz4k{N*JUTUf>jg_m!(Yy9+dFOB3(j%- z@l9>pY&ZC?%557ok0vHB11v#le0;NSgf&V$YQxoIu zIP1E|5z5B+_NMRt1cTffPUMK4oUV_LGUF8`u#k*3S%)_dq&S=)^nvzk5$n^W+R!aX z?gyQc1{^4G#}By=l5!$MwDgtPotC&XN{p2bWvzZCVE^_cJ|-o3#5gzU0!hcJCjRt2 zLCTRJR=Dt;L<`M87{j2}`Ul8Z6o}$Z1G@%uQbc*F0u(O=h_d8GJRWDC= zitMi8SkzrOXNc46=#?fuUYP9f=q7gR4Tw3O?r=}sRy6Wnc^NI{9u$lTL>2jgqzH7$ zwiCdEcvY&Ls{9k%(BefK!|t7#zrlRI_iPU*)B-D8)o?5D?fXp1Q;(ys_X;(`NRwxL z7t3qr7%)*E7z#CMZcN1#2kC+u7=R6iaR*efeMA7;4S=HqiRnO5AF!%6sYRD{XZp}s zP8yK7@ErK$I64tZ+tA+{?mkFMP5W&$nwcNRYFlEhVKnlU@{(!3wb2s|(ZbKMg+G3_ zuAk>?@IO|Sr}m@vI~&oAg#ffr66_#?;BLEJ5}el{#bVf%y0y|{!gPw+l^IyYvDgfTG_m)M12 z%ew|;9_cd>de^Xrx-v&Rrz!~Tmle%i_S}Eu=~||t<~-9XP>DWYhca;G|4@BSS=$jC%n;Wo7-&td8O zD6mE4>R!0+LIjhz#$WNTWQ8g-4%}C4K4KJXkz0|l&_3V{v<%J$YWEb>P zT(MH5cbniP_b|Fi=iS&)z&L(eLf6G+3vUj(@jTq>xKOHI7s(U*zYq9^t|{<;BAV*E Q1NG>6`r4ACIsno64zAtbe)svV3HbA93Ni+uJ8+Q0EJ3 z1Dk&{bOV}As>U0wQ;}N&#V18|bps7ni#n-Ijmq_|{fe9KM&bhXsHLp2_@l=U*6DrO zhPv02QoF(UW^BP9~zxa`PayVAma;=`z z^!TJ(ABsNvtfQeK)H2%2UvpiJ0dQgG^sTM@IQu(vm`G;a(&AZZIHdbnr`y-dJEr@5 zHoJ1|k3&_}D#vU5ZDf+KU79V!GeX4PmkLP=+$JXMC`Xk=v4x<6Pc;l*$7B|*Qt0g_ z?GZI>c$J4LDEy7s>(fW4*MxH5%6%CnKH7w>+%qSwz22}GYl@iQH96s2yYHc?}<0xfJ8cb*lZ^Ow(MqGvrhLX+s3 z9!b$|Y7qTV7N56oINj9)W&HZlg>{dW7TT}lY&*T@w%Iv7slK>uE~UDdI7Q$FP)5vR5zqTX(2 zp4lN|2bDZCiLgkuf%bY!c=js*()k}=#JPsf)}Cl7Q14@0jHy!{b`!9(3{#kaN>Zfu zQEm{67W^8LUB#xM2O-(l^&N&b8VvA27g0Q>SYTz*7N$XZ7$#h?!RMYSsyvpMR!?gH z|1GT}B;DAa^O_fp?#!QeMyJTZM237VP5pEzE~{NWgpbwllP_Q(qBJwNcO!b-Av7Q* z%2_)+yC`%RKQ$lKvYx+S{=3A#ZBKmqf#AMUy^fBHM>Q1Jy1MK{YPd+^gHk$G?!Y9F zZ1M?5Jgo{g1(o7H?Q{{_AMZxocQ2Aesnh*^m4?u#fulneKGQN6{10wY-K7(Yeq{x- zg*jd!84M+JjHAXi5Rm{PPQ?R3@J^MtASC}~bBrCHHUs+t?Z$l~xQMrcn5piq-($+G z%47YxFAdIj@^|F!d7=5dtDn?(Z>B44Fb&Y{KflNsnRN?LPQ#>K@K`a*a{EI-#_jjA zVRUT?7AZh1_WtAg^o(m*o`u#- zAY+xIcR^X`&10qRrr(Pc$+r1*EN7*fAZ<2E;*6#279HVUET*l4Z**ENODK&?Qc_;+ zYmBGKBxJB##q`+?l3n$ggW4CeI%dDa*js! zKs+o*ycF4yzs-X z2mcyqJ_%#7nkWp0hQx~=x8+pQW72A{gM`6w*`P(6N5)aM4%_sL0Y|xbiDa8VS6qmv zL#V<3K+@3*@riQY{PA2c?cTAAqV%3jO^1CFDWMF;^4$op|rPca11 z-m6-RDSvtlmxxrdlH-~1sSWjl>>#k#d(HcJW=}uk_qIH!9m4{+He-~Hy z+y0BNb=Uu1G*7b4=S4d!i7E`2YE)Aww#2(4S%4lS3q7$XZdTrMt@hOvWiiaGEe^X- zyYo~^rH@RuGiauC?rbq|^OfszY{hyZ>T!B$GXf_%GM@SxCc5hJ$4Hx88#~`-%R-AF zeDq{|BqW5-Gf)%5>c~NG96i56B0klNJPc>@w^$I4-2A}ZFZEsL!|262fL1U}u|3}- z*t3Q1df$s@Nw#9nE10qoUn|D|Y$CW6I5R?pJh3-u5eTt~&O-%esP!nkkLQh6QH0O^ zYlGiUF61@d6L+OqnG3_xTk|;S4;mRTO;D>T?p$#F4ee0 z_HpM5pU7g;6ZUzQ3EaTpeRvYFdG*guesOMf{G8dI!5s$xt4Q}H(N7w6fo?6ImPyb7 zv=B!?gA1{x%PDYykKI)0fmrk^xU2}87SawbdB-{P{7368?wSsJkSEm zUiiEc@@WM#Vi6@|Wc()JZy;Od0q5O59uQ+A7MG69!J3Q)X~@}!aUt!yQ%~_7J`NZ{ zzy{t`*5Ag9WMmE7Q&!{m0he+Yb01T|tfU^m?|W1NrH8JG6Z~?WMyS2mo!g$F0~cYa z7m3KLOs{WXr3aAOva7v#2Jx?YMNnxT*lDs7G+D1qSm6?$J*^%S*86tbnvX*_rXN&P zi<21ZQF+XcI=@Ou3HCmVIZwIFfTrFO6e*E+Inx!@66gTSg#$GQ8$AUn!TAuunJrOR zli-j4HjrOI6~iUw>)ZmM3rQ1P@=xEsgUZwt*5d0fs+kMX{6Z_|vO1f`0x7?iGd5}$ z#45JivQTrO?+im8@Skv)n0H$(*_?(C_uu2e{Cv z+mSdg!R`!yo_k!ngJ)eY-EdjIieOs3DLa`X!z>B(cdX>KOgubqOHm?Jp&#TVE&Dwu zEO)Ow*|wPfb!q-AY2}T5co6k*a&Y;fL=N!eZsZx<)W7YM&9crAp+m^6_W|{8$h>*V;^{ z7PC&MuBdfk?OPOg5~&YJPPEy;oqjE;6gFEDoKIi-LOdimk>{3?K64dEAy=;A_;vtC zq?8;5!RIp(UvnBam-x=m*Du9J?kd-zQ*Ic$?LC`czvbe|gQwZXlci&Iy(%YgERy6y zRp51-u+e``$oH)|JrK39CH;%h_v4iw;Oc%Oky+u18zTQ4Xim_a|dGk z#2e4;a>q7n^sE(0Z{Y~_*^r!okHpwn_2ZQ5U58;_!fZ9}nNi+Tm{Pc}>)7<$ZJRbO zw7C8W6kK1_UiN$3L@R&3EAd_+i+^MDQ>zq@L)mRgAj;Dz-({Fj6dT{A@-gb4F-FF51@s?$UPJ>!A gv{>%{y~7tV_R#;+aI5)#D2AJ4ZlW}33IV_W0mP?*asU7T diff --git a/summary_results.json b/summary_results.json index e92113449..234128b3f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.41754, + "Performance_Result": 0.417282, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 7.57138, + "Performance_Result": 6.53809, "Performance_Units": "Samples/s" }, { From 9a966a1feed5ad09c854305b667a685304abd7f1 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 29 May 2025 04:51:52 +0000 Subject: [PATCH 675/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 6 +++--- summary.xlsx | Bin 10935 -> 10937 bytes summary_results.json | 6 +++--- 6 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4aaa2be7f..e94cfd152 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-28 21:34:59,445 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-28 21:34:59,445 submission_checker1.py:3308 INFO] --- -[2025-05-28 21:34:59,445 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-29 04:51:46,842 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-29 04:51:46,842 submission_checker1.py:3308 INFO] --- +[2025-05-29 04:51:46,842 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-28 21:34:59,445 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-29 04:51:46,842 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-28 21:34:59,445 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-29 04:51:46,842 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-28 21:34:59,445 submission_checker1.py:3324 INFO] --- -[2025-05-28 21:34:59,445 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-28 21:34:59,445 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-28 21:34:59,445 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-28 21:34:59,445 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-28 21:34:59,445 submission_checker1.py:3345 INFO] --- -[2025-05-28 21:34:59,445 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-28 21:35:00,508 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-29 04:51:46,842 submission_checker1.py:3324 INFO] --- +[2025-05-29 04:51:46,842 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-29 04:51:46,842 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-29 04:51:46,842 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-29 04:51:46,843 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-29 04:51:46,843 submission_checker1.py:3345 INFO] --- +[2025-05-29 04:51:46,843 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-29 04:51:47,859 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73182 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72786 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -38,7 +38,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.53809 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.7262 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2287 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 47e9cd481..afdf0ce7d 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1302 +1303 diff --git a/summary.csv b/summary.csv index e867d606d..af2450512 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73182,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72786,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -16,7 +16,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.53809,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.7262,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2287,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2d755c089..97ff50a74 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73182, + "Result":1.72786, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -229,7 +229,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -519,7 +519,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.7262, + "Result":23.2287, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index cdacf05a7af13c0268c7396f93e88921fb750a67..a955aa100967d3e5ba1bba356f58abda45a89a01 100644 GIT binary patch delta 1396 zcmV-)1&jK(Rk>BLatH+jzyhptvv&w#2v)Tpl$>C-DuEJB5>FaHBoj&lh-47T3I-6% z0F=zE(Gp6N0mMcDC`|&81_20KB05be%|cn1l_8WB1R!0r0JOv8-(k+yvcO7M7VRHx zXV061k|V6vvn&yS0e?^a@{bdi=yunYv&8w) zXpi{P`5_r)N$H1#kj)Cx56b|XOxc0gW=+x$8wKDrIX@blAAe|x=rpA&3uRqahWjQl zgJ9pRS^D{UIG=K)-Bb1FQWw>qY~QrBU-mLkiTR^ZA@Qa8Lo&*e>JJGan-$a_mH{@I zxC3mO)E_nquxT=XG?+io647Y_n_&KsGTb<;m_NE^^XL1pxv&0@mi~|Oy^^P!c8ULE zqe$XQ|A%ChC4U7V5&}3`M#9N55>7TDfYUY!Kx`BVr>TRE4FV9fL~NSiu|rvxmdTBi zgka;e_sReHyF~U{7FlKEw2K;0wze=Rd;i2P(SU69NqngRk&LpW2Sh^1W(5z3Wq?hR zkxw{nlLo{(05nY+kPR9Tv_xbO@YtcM%c|tI$?g!?Hh=BCvw*&>$X?6(s_d9{Q3A?# zOv}}}jLH%V$VQ*UmlhDoC`)QUB!pB}(12J5s3aMn(qsX#4uDFN1+=~Kfl9D|*kuEc z9m=|-46&?Y0TmXo?^r%bWUpm?m9PW}$S?iCl%fQrf(I@T0Fe+PSwR3|89{DWCa7r-g5(JgJk$5B!Bl+{IQ)~Bm)KSVYUk&^EQd+H}Ppa z9xodGB)eSe%5CfVv+pj|;fM9d#685b@sjCal>bfp-gy0?cIjw((^&ku{71m&WWH81cBID_O!K# zAF>T;{~k9<(^krg<@nw?b7$mYxtC4!3)Ts(N|Yrjia;^Vh1!T5Z630WJz4=_1Y_>U2&P8JXXG9YWdKTM)@TW($pB)b0F)*HNP_?bEfJljlxCr<%gPYS3IdR>SpeE$ z^6xNbYgu3=EQ|J!wp+gT%COY4E)jqMe~-3X+6hZ^JYAyvXjDghsr-rHZk$!jA6>Kg^L^OdSN}&# z|3~>=$<1MGYuhTNsqRe`1$tKsNd$zSMw7Mp@DW zA|Yh6f(OJhz$VGaC!DrP17aNjnkEg%1`P;WA~Fbg>`>KZRdU;8cL;2ofA-#4K;Kqm zuVsBzc1*h{0cAU;X3y5TtB{d)tLMkh0Kr91Pk_=F3vVd3zK&8n7 z+TQp;C0Ic0vVq4AWnEH+SXQxs3JchGET1H@*RsA!Sb_xPmwsSMQ36uI1D522NTdWr zGRl$`5D6ic6)Yf@0W6tce**_ulLo{_0WM7%kPR9Tv_x#0@M!&;Qta zt^uXP97**g|EHFG{U2>*=T}|wqO9$dAc-c4FZ~~qQI-^dNC=UvAONupAj#Ak zK+AJfr9ri z+l7yLo5b^*_%t4m7ma?BU9NTIwsrm4cbDq$!}??59^%<}$#gKv|E7I!y#7$TbTqwb zEdE@7T%LD$`n$IB&icd4a4}I^apBwt8#&IKC`h7H8-D;Hl3pTBjqgn4J zKJU$MM$<)O91pXFS}niXn6E#2Z}oMW%vN8y&Bn(hO_Ry}ceinG5YO}PvY&UuBuVeT z$uG(FBMeXe0#Hi>1d}l-8nY@Ex()=T1!2IG=p-M1%}&EG5QOhMMdW+pBqgX=Nh$&A ziAqQWQE{{EZEF$#$Tp;Td)%~5TPY`w>zVy_cI9%hmu>V5)(NdDlqD&OKrzjQ+E(bJ zS;lh|IZqX*tyWN>9voWKcO^4~X$x;w8?au0ivmbF!c2vB-W!6kV>^)4CBdPBxY1Tp zA4wMFDW{&|128rt5uK2njY#uj+X28Ch883!?{Jo6_~P|orJHOR zlq;tczBe#wJ&U7z@5E^Ax-RKvgEi#O@Ot(7KCBuGbvP9RRF|9)W`TNb>k?lV;czW= zeicr<5x}4NpijJ=7CH6-k3s-)jC)o-XU{Jt%@Wmlnip|e#QD6*Gcvm;#pBmu>l2)hRjw^OF!N xCL5&%VZbH<002+|000pH0000000000004l2)RR#vJ^?9{kSZev+a~}3005@9gWUiC diff --git a/summary_results.json b/summary_results.json index 234128b3f..a3bed6806 100644 --- a/summary_results.json +++ b/summary_results.json @@ -631,7 +631,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline", "Software": "tvm-onnx v1.19.2", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73182, + "Performance_Result": 1.72786, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.7262, + "Performance_Result": 23.2287, "Performance_Units": "Samples/s" }, { From 00483430da2adf352bdfbff7e82c12984ee99d3a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 29 May 2025 06:24:15 +0000 Subject: [PATCH 676/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10937 -> 10934 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index e94cfd152..dabd2d64f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-29 04:51:46,842 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-29 04:51:46,842 submission_checker1.py:3308 INFO] --- -[2025-05-29 04:51:46,842 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-29 06:24:09,850 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-29 06:24:09,850 submission_checker1.py:3308 INFO] --- +[2025-05-29 06:24:09,850 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-29 04:51:46,842 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-29 06:24:09,850 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-29 04:51:46,842 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-29 06:24:09,850 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-29 04:51:46,842 submission_checker1.py:3324 INFO] --- -[2025-05-29 04:51:46,842 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-29 04:51:46,842 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-29 04:51:46,842 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-29 04:51:46,843 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-29 04:51:46,843 submission_checker1.py:3345 INFO] --- -[2025-05-29 04:51:46,843 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-29 04:51:47,859 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-29 06:24:09,850 submission_checker1.py:3324 INFO] --- +[2025-05-29 06:24:09,850 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-29 06:24:09,850 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-29 06:24:09,850 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-29 06:24:09,850 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-29 06:24:09,850 submission_checker1.py:3345 INFO] --- +[2025-05-29 06:24:09,850 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-29 06:24:10,859 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72786 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7205 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index afdf0ce7d..ac0040ecb 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1303 +1304 diff --git a/summary.csv b/summary.csv index af2450512..e20737ea9 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72786,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7205,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index 97ff50a74..c5a17a88a 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72786, + "Result":1.7205, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -229,7 +229,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index a955aa100967d3e5ba1bba356f58abda45a89a01..83f87a75b8f65cdb2f0b8ec6e1bf10c52d6ec1fc 100644 GIT binary patch delta 1777 zcmVPE$IwP}XH-$YcfoN7wBC>>&Af zh_kgUuo9I;`A6H?)8?S$1gmukmS~Z9QvM;CP|80fgIHFOe^>^vWKs=aX_9}~D1fEO z{n_65fF-y;beeyFCAdGN3=XiYYdF9b7Vs~{a$m(Cy(|7W*?r@c9bxSfE>R%yr1(QJ zp%i~e2Dz*t{;&*iNix2~(&YWH4ggD&_M<`jftH92f*L!Nbx9d$?5u0h*mcd?4`TVv zJ@#6bSjA)4MeE0F>A<$ixO7UmM191Q)(^>q()uA8({E#v{$g)4{0|(if zji2vB=AM5lKUcb_{N%64dZAnL%U{|nL6iJN9_jp$Oemcnl0h^pI6o`{Xp#(|X;OYz z2Y{x@_|aheKubghK~WaUx~vR0O`-+CrdhM`^R;L`$&{^Sk(D;hqVki!{NscrHl8jq zel(gRzBGPFMp;t%At6Mwg7U*MfF{WRnkM6ibpU^8nuH$>!Vk1W?6M)6ApDRr+%-wp zxNFud{CpoY_tf~g(naGZ-!=WP@Mmjw*(_0hG|D5sRDMWCS0G=IsERCi91l0&JSZ9}VIUv_y28z$S=4 zqzpICD&mi>S^W7vZ0@W6qow|%e6Qr`rd^`{*yxe?QvV?tWl8^sgaA&Kk#Mq%gp*AO z;IvKt4;w|oY3iV3gZ~395t}Ax>`>OFWpaPxBq7*1?S0aJ{w|NbmPJd@tvRT0cVi{nQWaJZ0+oS=p4ggJ)24sT<1T7I61T=Q2 z>ar@iZL&KAwoQBQETC`evDdP`Dm$iKlz_4w({i;gqq4*Tve75;r3FMX%90uo2_b)# z6*M500V+ubs5DtXtOKCZWC3k&e4r96Aa>b6V~4UXDMKu)SU`mZ>^qiE^4M!xUnML- z0`f~gFQq5}so()i@;M|@0wNh@NehUC5X%Y{5X%6T%&&oitVsi6qX3sC4af!!2wEaG zP5GG@=TO$AWpI$K@YjFrJ=cKJVUB;KdXoQB%i1cPUyJ^awzBi9E_qSbc1n;$lf;+) z56LJ?3P2=;NLCPlSO$<}Y7HQ1GJx19fTYO)vcUj?mWWMLe#Xo>lyzwtAX&ixviIBo z+8`M|3CVpGe{5$L$w0w-nC-%+yiMZyO?(=U$BRZk$u8Hra@)H8=(|gG_+fwjDRB?+ zY`kPT80CM`zBgWfrd>Lk-ZU0}u0Ji$J3ReeTX|>w;pK81H_|V&__!H=o@Mt?zSw4C z@OilYoV>4t!_0+{so)X$(cxHpLB`FGjRyJ3=~_uu4~Wcvw*Cw~D@O9KRxF)13eDi*p91*@7#3ni23Bp!dw zZo)7SgztTd$oIrfD5$lPAhi{zB2_9?QE!&LfYta%w%d@muM;4E#Hq*i%zit&ayj3t zCiw;DrO_42GlCM(+z6?+75Z2$(g&3Iz_ehE(NLidJet>cCAX9t2XD?;a6y7kB1n76 zZH0CrScQgF&0uwYzW;>#i)u3?~cPyzq|5dZ)H0000000000fPvJL TQ7S$GD3g#XBL>2} delta 1784 zcmVBLatH|nzyhpt4FCWJvwH|#2!FL7l$>C-DuEJB5>FaHBoj&lh-47T z3I-6%0F=zE(Gp6N0mMcDC`|&81_20KB05be%|cn1l_8WB1R!0r0JOv8-(k+yvcO7M z7VRHxXV061k|V6vC0L?H;z|35WI}2GkPKp3!Tw--tfThX)VWR++Ci!Q3;{%o; z|9{YF0+t~EkTN*IvaaC-TUfxq6w7@zfAp^T<7D@ZS9XTAOSnXX#FOR^$%NAUAsOVd zg89QTz$MA}5=)c#!#V&gP2P_N?+02UG6;I?P}U`7;IXr=!DH7odq0TfHy7D!Sz;BD zT^F?sH_tekp60Ot*wglt(-%{g6y3 zr5}<(GAl?wECXbc43KGZepm-Urb+qHp!`5fLik^kqVto#AnS#0$!~vYuLMogtp{_>9#mgso8METLEj`&jf zAsJ;!=ZAz4%?i#B%K(}r18AC*AAi;XplLFGG#Ee760ysMXoB%W%5c{tUE{7P_|o|y8D&Z7hlG&L3epeD0GmwNf!AhD(hnO2 z;50cu8k`?!iRd(?Dhp*@R)+f~F@s>=tXcZ`dN`kQq}@~X=TaBdpKRZ>w0~drGEj;6 zqfsI8rTIfL%98322_c&m)E|}sHkr5sY?{;`HVUw5GJiCfKhP4?sM{*RXakMg~er<-<(|6`*_;!FRBWRxWZAQA#NSw_OiG7?TUA%N312|#QV z38$%pjtv43v_x#0;ITtlmw%SYjgy36jFerQf#4gc* zZ1hQdsR5CUvZMz@Lda$X4~S)eO_GsMIBk;##5w>pO&X958W6NZWDxM!p{mQO`l{@hc2NS#c1+9Fx{S&a3&=*F#FrKj$tX)|Kz}5JR94V{SO%yh z8KBZ+0kIB%N|Ob&z43ucuz=WQ1CJfbx}*%TtYQHb7O?MFK1pP+Wqp;f1PRD5{lJu> z1f+roEXfCvNC}8!lqD@75<)C1SU@ZTSTer`4zeZZeVnlvCAG$3e+*fiybUYtW& zmzKdnw!)wPvG-g9N`HqrlIltRPc3V!bbc-RKibO9ue#(#S=%W=5=|0c`adM2EGYny z5F%MY0Ad+HlBqR-q{#qcqX3d71IPvg2wEaGP5B`+=TO$AWq@P_1IXTU189R}_#`Cv zRs6A?T_ghq?_stJAM-Ye=Qr_bJRUC^{Up0w>&k8G`m^sY)qmlK^~c0L#Ix~|>0p%q zP5a(>{h@a0XnNCF{JH+PJn!)IcWvdJ^@o?saokA1%;Mu_{CSq$L-}Hxjlt*P`h)Ts zv-v2QkJ2ye(eqF0TaJ4tlVtGYYEsMT=+9;R;}88Gyu%;*-cLU_^iPfBMz-|(Xtui5 zDESv`S|>-d-abuy-kaZyri;cn9%c)*T7I)JUw`!8>gzO_t-f-bjgLv1CX@T`ZsXn{ zp6B0XKktS~lHPxlUy|)d7@qtEvkDg24h3eAGzKP<=Oi9~%TB{E5JmTVMdW?rByD+E zNh$&Aib_ZXf!JC0w6%yIvJGkf9ydwTR?3Ry_})2lXXIkJmre8w)(NdjlqD&OKrzjQ z+Lq|OUd8t)a-J$q8?B&32XJUv-4x6arY*c$ZNPc~E(##!2s0(xd2a~Dj_p8Fmjs6j z;znCZeJJgJ78}aG=@xLFrngvvr<{6Qg^ut?trJPH9wANQnu%$}Ym>lLc Date: Thu, 29 May 2025 10:40:06 +0000 Subject: [PATCH 677/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10934 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index dabd2d64f..3e8651b2a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-29 06:24:09,850 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-29 06:24:09,850 submission_checker1.py:3308 INFO] --- -[2025-05-29 06:24:09,850 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-29 10:40:01,367 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-29 10:40:01,367 submission_checker1.py:3308 INFO] --- +[2025-05-29 10:40:01,367 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-29 06:24:09,850 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-29 10:40:01,367 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-29 06:24:09,850 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-29 10:40:01,367 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-29 06:24:09,850 submission_checker1.py:3324 INFO] --- -[2025-05-29 06:24:09,850 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-29 06:24:09,850 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-29 06:24:09,850 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-29 06:24:09,850 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-29 06:24:09,850 submission_checker1.py:3345 INFO] --- -[2025-05-29 06:24:09,850 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-29 06:24:10,859 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-29 10:40:01,367 submission_checker1.py:3324 INFO] --- +[2025-05-29 10:40:01,367 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-29 10:40:01,367 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-29 10:40:01,367 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-29 10:40:01,367 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-29 10:40:01,367 submission_checker1.py:3345 INFO] --- +[2025-05-29 10:40:01,368 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-29 10:40:02,594 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.417282 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421591 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.53809 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ac0040ecb..7ff75c754 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1304 +1305 diff --git a/summary.csv b/summary.csv index e20737ea9..a00faea88 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.417282,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421591,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.53809,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c5a17a88a..e6367b476 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.417282, + "Result":0.421591, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -409,7 +409,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 83f87a75b8f65cdb2f0b8ec6e1bf10c52d6ec1fc..1468ec586b0a8fc00d79a5356cc8b4615331f147 100644 GIT binary patch delta 2753 zcmV;y3O@C=Rku~JatH-CY>|6$vv&w^0|S*CDzoYfDgl4&Z0a~V-2TUDR2Ii^)h>7x z_wsX+3B+Uglw^QNmXS!ZjAW8zlzb88)i%Vaq&-M1O+9qBH*Ud%mWYlbcJgy5>yk40 z>f>A1wH7~@E(#0S4?G`s#vIK-9M7_cFuRZH`|JXe}$OKy(n4|{;)6hdl$@dt}pe3TC z_!3N6)@5a|YmyLbn}r4JBbbFfGbdA9E%gL6>sVuz&2uv`wRsu375HAsvq{C@g?gs& zjUCsMXksRCC7PH)G)V@~BpE;xGXza-LX-4>Xc~Wd=xlF%KvT9vbQC~SmUT%Pyuix3 zwk?UK{tuv8#~Q1==DL~K_SvOtyI#qQu8O~9`b^B}LkWE7mo$ZYeY|561juJ>VWm%V%;kJn(+&1lh6r1WvC)zsJSZUj|i`!<{ z@-2V2vuVbauDRz|nr$wpn7ODAWwxP_l`cAdqZg2H7MT zV3TBkO{UTSo34pXX4(Loh8{ZG8z0z|EfIemC6sK+vMwvbZ4*PdZMy$3HlK2&tz(Uq zwoSLVZMrQlbj#2z@plx8A)XX?NG6a556K{!6&xOx0W_J#18ABw9@Yb*X)<{fs65aT z(NO?Ruz5%sZkwcQ+&1eJdcGH$`)c^8UByR-mQ|+KsU641dSDX;5l>1!Boj)@hh%?{ z%?h3m%K)21YC$$lt`F-0*)-`sO3FBA&=Rpxq|A8^WnEeZ+a_^;VB4%&?D{+LeDC*LVDqU-M9536&_7 zcvAl%nNa#aB!g5|Fo0MFsAPr>P-&8YXb(W8Ndr>g0YOW|MuA^#fU+(rLo7+xh~>W7 zK&4pvPr`CfZJ;X^?qX>NLFjNSwZ*a2sFQfo0wS4ET0kTNSh9@7l4T^8B%^=i?@7JV zCJTs-B7axJ)Imps1OzP+o2LA2qH`$g(lU9BWg%Nk@a1&3!X%0$o)mycCX@mY$sm&z1R$0HCYe~HC6gur zh>ZeFn*1LP{tvW7behtcg|dGxD?=tL_&>U4|7QouzeAj@Wr3BbEXqIH&Ym_0B_~*| zORz+X#FO$5$%In=AsNK7g8aiWfF+Y^085kn!$tutP43V3#s@6H{h`wYEW!ODWpIFH zUBdykuz-IlmisFH=w0#0$?hAk>6`C6*@d zhjjp0nzSDc+7GluWDwNYp{z^FKx1cJgT}6F)_xGnZ|p0s{QCY06>$sm^%tRI#EE=dNsG)X_K1K`r+{Ah4~pd})MpehSxT~>ytSRw|& zDYj|)~&!P{7%*`nQjT0=#F^O`5~E5IzJ?XWL9u~SO&-> z86eZ7{ICvyOq21W!T5oehzvr=1mlO4;X#)DSsysa)@=NIA2Rn;`MJ_XnC2sX`{ji0YY^GT*`EsLzQX%>~A{N*2~?2r2B65~gsIpRy>hh&r`l^+s9G%F}S zECXng44`Q;epm;9rb+nGApAf}#4a163BnI4!(Eefjk{*e!q4|Xb5D(*D_t~x@?Fyp z3xBp|m(3F8N27l{;!EX+WRxYH9}+?~D>y$a18g#52VR>sIX`R^fYYS>Xi$EjC8E=m zrYw|oSsCt|L=1v`vu5Y#>*0LLk#?KqcajMuWtc;t$CvOPW6< zgltwYe^>_CWZn+2X)=GOWfQ zKg#z?o^IMD`j3qsi7)jZl2MlQe@F=6WElx3%SbrcgaA(4DpzQq%B(I@ex21GK-k{%EV zA)6IEAeMgtHc3W4;j~Q}5bFTYG-*IKXh6^skwHLXhpH~ClG`S`Ltxvq_s##MS3+C>Q{+c7Oy>oO`!EFc?w5?@+CB%>^;0g(_=SwRD08K9D6fJ&1E#5w>fO%~Ah z#s@0F0%Dg9Gg4U;(iVV9ER%ILMkbAT|ncY0`jf(14&NV$+nLd2tS9U0MbQ*$RLC$KG=d zC>`cVsweqBwXChu`L*c(Xe&Fv>XH{_ZKniDG)a8v|B#HbqyR)hh-3u;h-Cmtrq%$G zCIf$njRHuT3?Lf}AZUr$H05W^oI_ccmI0C#3?O^Y4WJE@;ggWuSMkSoc99GeyocE? ze9GG-p5Mf$@p!yw^posztt+>!>yN&>REHndpAz>F&&ErpgHiq`?R(?(XWFHs=}lwt z=lav~yu;JqwUu|)A6_oUaU=aQi;tV}=UIPt59Nz(HU^)E>(9w+%;uwHK1#o=N6$Z~ zZ#nLrOp?Kmt4S@Vqd%ANk3aN(@D6|Idq4f$&_6Yf8`;wDquJ_KqvT((X`LL+dN=WT zZ+t51dtxW02&^QP0QE#AB#J=XEPI<; z#6PkPY2O|vY13BfiQ{@^znxvVT(#E$W++Tgr`tS7$7^Ai*aQq&?-fLfa55#n|&LDCV>1&{5nNr&x%kYp{Q1{F^m^ z3qtO&g1`g|4EMm)PDFG-3O*t2k81~j;22s^po7PGmgAE*fK&cz!=Riwl?;0eSFJ~J za__vHj9u4d-E6SN{5k$yy}S*prc(E(;(+Q>aLOI9U|e0|^CBLuWj?Ipi8m6&<38$F z-VTe5eISw;fR1sG%E#>KdA%e=bwP@GO6F?~cPyzq|5dZ)H0000000000fPvMMPAWbDDU*#VBL>?i H00000gTW3# delta 2771 zcmYLLdpy%^AKn~B8e4OysTj|;5UD-OF{kiot;r!hRtdFfLP_X^-yC}AfZ1j-PvsEh z;}CO9@{pxAA|t1HI~X}cIXpc!G4H(Z=l#6*ANL>keSfacbzR^4d);?~U%g*dl)CEW zik|Sunk(v&%F1y&X*DyO&A8f{coq7P355%A2m&YT z2=gw}vaosNk8RntYPX6@Q@j7^u*i9b==J7%CTLLue6WG>ah**AW0NBHVDQhx$@cD? zBHJ_x`5(rgmSbP--)r#RE|h^Qs5f=h;~R2^^u3=>j3oL&u)x|{F7R?t9PoS6t=!hE zAm-p4_y`lcmjG$yL%(sva8=gd^dSA~QzkNlc4Vzvv`%1>X=hV8G?s)&im0ig$E(y- zo%=#LUsn8p9j+YpN~N3juW48Th>KRH zo%tibltg11z)7wE?EYLjj3(ryWhl{vygjDLTm8v{^_brNbuj*`|Jm%pQ`XvI$LG-z zgT9#X#fBpfP2+Zneqi!)HXRUPLl`6`*opudLVsT}&a1yg5-IaSrwVFbEF_YK6$U8@ zO3c6VaHCfHQp6=|nP+?*rrl1y*54hG7kfzzJ=rqB0h;HFq&HR-JY&8=5Tr#&vt=(I zT1ddrH@M~)(d@8z0xYehV8eAaJeBC?0mf6w_t>B72~8aleKJ@+`9FZ&-|CSzBbH(- zV!6&uZsPXCR{OM7n4~UL9$Gn^39S+w(+}#~UoAMYJ*e-U>$w{e)**{GC4wKp4yZirzdQ)YN$3eXOI~E z4Hlla|BgeTpd>+*6?-)OA<^$x0=rLR4)@ks6+*6R~6Wdgn-Y1pWs8CH4dO?w}wpS%QU?PFw`{6 zw0`xd_4X$^#C3xIw^GgM;+FyDb13zl8O8NB9Q_!?(G$)Jw9RQ05RsGR&%4WKQrN42 zL+S4$6mYfo`1jj6c2hJ4Js)$XkNp=ZTG3f2u$k57lskmeQ(TMuiPy92rqM{#oFm_{ zWy#?QNq)gZz#+S?yi&O7_*PBS`$gjYn2N(WZdr;_!38NkAY&wjmjvSDTG85JwG(I%RI!xeP8m&Rmx&E zacEgredI>DUP32yvj1@g;18*y_-F=&GJzozda+<__=A*=X>^L_Zp@FMT2dh;9B^i@wF-~5Rvsy zk;_j7H)xW+Jv)Ygq|KIxy&H;oI|4f+d+YGe36B4(vxDwmUx!V%)(ab>4>!8S^H~Eh zL5gU_2szG6`q7SS#Rx$6u+ynApi|sG4K=%AwQdynF=|11r*K|}$Vb_{8kk)@&@o|s zi5fVUQT0hh)w?vhVV$5{v@(UuHXHymLU9TB zZBFkHSuX0iL8}bhfj2y-ugYq}-1LXvUB5gqbNapM$?Evi*VhY9E_Mcv>~>+Tx*JTd zGLD$sH)OitciT9bE7O?J{7|0+e>HlEK-wDwtU43jATHi|LPf4wlL}SaO2{<}OLa6y z0lR{lm2LI5aFO-bsw4$t^KG28{JPBYz!MPUyU=wFZWBIiQFZ-KJH(unm-p7J8bTqfEWG(v>e-j6Di;* zHAzbf1F^rxeySt0mfC*<+mM8rkco}PCWQ(QlJ8}IZXxhdL)Kxj2`vWQk9UMF_pd(d zcVF5iwzinN_X2v%SWd$qiZhin?(!bVTo1kpQC&)<(n(+DKzZ1u)O4kAF5ggdOu;ak z+loz7To!Af+iWbelDsN&4F6UpFDjN7#XTLLE(s`HADYUN`m7rvsf6{^6p$w;E0T9N zg+)^TR7;Y>w!6O!oyk9&ysm7qB@auR^%^LqMoG{{LpF2qB0#<;^99HK_^{1fp(?w= z0xJCAR87RAw#Op^&pV@w%%{Owa2~l$`3y1$U{S`kJ;jf})mn-)gHVzoDi&LiD)`tZ zGd#0r5j=a9deV*W=J^-DRuA%8%|Cy#FQR}WF2>~}%u%t!&Sz5en0HSdAg!wfeT|DGHsFd=FHF7n*c;D)~A7z z06XpnydF^LWYJ@PU@hR}wR3BqC!%&(i!5uTjY!}Dbs6w#845W&f2~peXLComObTBl9Qt_=4) z4`x1E{y46FkMeWVAG-i>&1StJEtN7ZVKZwIjRG#b+zU_n9)J7IGT+nWNgWear~BC0 z&&<#JBIgDDz0{*#%xHM3jiU2QU;pSzuzwz9`>_lwQ=KgC)fN7lgm^cqUfQ6$;%^~R zV@q3WgVKAw3bYPz2d`in*xS`EcKb)VBpo$AbVC?%GuFuecJ-vaSJ%1Tf}+NKp4D(- z!z+NZaIs$X6dc#Cp`>lTcU0Ne9?`qSRGL=nREY#7ouxS^Y8gzDGkQLNJY{*B$x)w= z;iB~}oH#l*_2GuCzPSGFh#)9kcDea>j~V=XrGEOp;C?`$*r|CHIip#>cj@8IRLH9wyG% zjDO^7e`5Sx(e4LoOtC_x|9&c2Qj?Ql4c0N!17J2Q%hW|PaR&%=HQe%Y^tEv6Wfsaz zhqYv?ue4;qf*b0x&}Q%*8%|@uAdr_52($(C|3OdeN3#~t+8c*h)>!!F|83C|O;{Ob Lzd%NIZLItoa91o` diff --git a/summary_results.json b/summary_results.json index 8675bd79a..764797ff1 100644 --- a/summary_results.json +++ b/summary_results.json @@ -127,7 +127,7 @@ "a#": "", "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.417282, + "Performance_Result": 0.421591, "Performance_Units": "Samples/s" }, { From 17bb285fba193798d9bc040c2f56da8563d54690 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 29 May 2025 13:18:10 +0000 Subject: [PATCH 678/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3e8651b2a..2e1c11a44 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-29 10:40:01,367 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-29 10:40:01,367 submission_checker1.py:3308 INFO] --- -[2025-05-29 10:40:01,367 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-29 13:18:04,511 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-29 13:18:04,511 submission_checker1.py:3308 INFO] --- +[2025-05-29 13:18:04,511 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-29 10:40:01,367 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-29 13:18:04,511 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-29 10:40:01,367 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-29 13:18:04,511 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-29 10:40:01,367 submission_checker1.py:3324 INFO] --- -[2025-05-29 10:40:01,367 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-29 10:40:01,367 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-29 10:40:01,367 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-29 10:40:01,367 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-29 10:40:01,367 submission_checker1.py:3345 INFO] --- -[2025-05-29 10:40:01,368 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-29 10:40:02,594 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-29 13:18:04,511 submission_checker1.py:3324 INFO] --- +[2025-05-29 13:18:04,511 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-29 13:18:04,511 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-29 13:18:04,511 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-29 13:18:04,511 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-29 13:18:04,511 submission_checker1.py:3345 INFO] --- +[2025-05-29 13:18:04,511 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-29 13:18:05,650 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -34,7 +34,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421591 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.376622 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.53809 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7ff75c754..767efdbde 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1305 +1306 diff --git a/summary.csv b/summary.csv index a00faea88..3602b8874 100644 --- a/summary.csv +++ b/summary.csv @@ -12,7 +12,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421591,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.376622,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.53809,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index e6367b476..553f3dfe2 100644 --- a/summary.json +++ b/summary.json @@ -399,7 +399,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421591, + "Result":0.376622, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -409,7 +409,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 1468ec586b0a8fc00d79a5356cc8b4615331f147..538f35c770fa781a6c16ac99b45a5f2d4eec3697 100644 GIT binary patch delta 2107 zcmV-B2*mfdRku~JatH(@2R7cbb_j3-f9-o-;b*`OcRuINNAARf?KwsQF^yA<0CyxJ zxg!zD9f^oX-z(pM9Vr*ozA=<{uw2Ql=&V>TejG~jhOOVGA%7Ot_#0L zCU0)9d;R<=%|Z4IX5H1nO-AJGOPb94x7T;N<+ZGX{!(eD0*iGlr2>oZ{Q`?lf6(%@ z<#H4Qj$(i)iU-hhj0Ez`g%LoJL?ntNB2gp}iDFmvB~pjgLJDxmlw*xeQ#?K>sM(U7wkB8!5e=NJjZgX|O|z#@rA7D+_1 zh!JIphvuu$03)fFPs83A>N}x8f8|UzL~I0k9+gF1QY4>wyv2+opTC6l0ypXPlf}Wf zx9Z_@;C<_}X&uX}0O4g=fG})1+HrKa{g2bAERN%w?5h{y0L$pDcoBavho z$t1}r`69}zZHQ4xdyrU~dgyF#+=2-$5gkSB$@wC_zEiCr+Z?|d&{|elfzWrGiLDX!TCN>fWnl8x-=8Ubz1?*sn3xG1 z2_|{n9>63S0Fz_@Ow8bJxeZY;Ne>97p@)u=?=hS~OGHQUC780T%gSKaBq7)~3k%ps zFbjKTPNue6>IrDpvBoN!f9GamYV$I5EAYLNXOoJ*3-wIl8#}Hi(Zo#PN;ENpXp#(| zNiu*YW(byk2fft7V_TM|wEA3(E?HCB1ebu+Q; zvrE@@JypUcf6Mfl!gnHFU$Tjrz>{oZ2H7MTV3TBkP4Yf3vT2*xe3n?nj`~gk_@1U8G@!`LX-4>Xc~IxY;Sx(Q?^8O6hKp!bx9d+o1|;pHl6 z(tVVaan7J6Vxvfz^Bl^$v<$XQ;{L$4S+m&lm2B!K9clN}___$nR^3oY?|C3+5@0z(tnhcbIzb8BBOxg165sC1-mAh zKkk|}dp?hlDU6@5hjLHdpDVR1|2VGi_$9yQp~4a>Q7ZAI{zEdM^nXYOsjOfCu?$ej z3>%=*BmvPLfJ&1Fq`(7$mWYi4zuEw0T~dZvlCBZUe|@uoO0o2xgyo*vKvyc<#nKLf z(BW8Wi({!#C-I~OL^7eYfJg?gWEqJi%SbFqM#_cHSQWPP*Hc-`>Z+VzbrTjxOh-C%& zhh+duCe;9zCi#br0$7^dpY4qgSc3aQrwLesfBQqq-~h|Ih68M20sm4g_f`DSyW)?N z-8Wv@5!No@5(N@Zia#V1O7Vwekjo0<56b|TB;!jgP2Lad0I)P^KN_?jXo<)msIfy? zmz06V&bkJTUDvGrAeP_UW3Oe2RXlcGw0^vn4s5H8OQ(cO)JHsN{g6y3tsjy>E-P3+ ze=GxBk_>QZl73hRz@^Fg(ct_*OGE}iRTj#+tPD@FL=1vcY|YNkV_bd{SGJZ#RyxTR zr5~+Zfm8UMtX(qQ5;D;p@uc%ZGNE*SNCwHQ;QX)*kV!H?rb+o>9RQgo<41$>11%95 zgpdiw4=KZgEc>%QaFDIp`1w9$?y2%~U8Rf4PyTwW7rG_C{H47TG|6A&k_6ljkHKf6HpaFc3xe{R*SKE!!a_6}AIy$*O@u zU0S*m(j*bJ9!P4P{CyR_l7LrTj_#c^cLo*fqimvIuuf=IqAW>K1d3@c)V@UT^(J1S z$a$(bZM1?Coxq`WbyqM$n6~h0wE^n|xF~>>Bg~ZO;JqOjJ9Yp`T@oBBh&yd1^`W$T ze{3lGrhCA7n%-jxo^t9bJ_BPj5z&C;Y(knJ+jIcVFf<@Rd55zk!xygyE8T2Er(8Lu z@FxSa)=`|?M<*s@+qOx&=&T`shCjD2Z{4b~Q0G%IKvlsRVHT*@wkq&-5f0Z-=eOa+ zI|2Oh6!e+5VUfNMcoYJVKJKV|ES{e0B@L?bG+)N)GR`0BY(cUWNta*ekvIJppd5z$ ln>Ygc+_#^g`UA5mCdvW?BnLL$v-c=H0R$_TprVsFDt~v@*l+*< delta 2107 zcmV-B2*mfdRku~JatH)BY>|7jb_j3-f1{NfD$bqHx$}`b@nCz7kw8r26eGYLiAe59 zL~=(W;?eiYH(*D~1+{Ms!80>+Nr=|9ZRXe;(Ncqq7$?{e{H!O z#ekz2Ad2Dv^c*9BJab_LP$UtFB8f;8NkpO;7*HhT5=BFK2g?z~zdUz6uR6l6v_x?2Vzm6ADz$e`G_%Mv&)GS=1#(@|nk5%sBG-OIR;(lU_er9E^Lb z9zF-&w?3QJvAhZpUWNq-!|ZJ8I6B<^$7xg+$8pszcog^YbCLQF!}56&lJAi?z$38%mj`E zle}&ZV3G`gNiqN?W^lLMhA5b%2L#j5Lr2N?7|x(2qNDf{Oj*`tWw2|K5Nw-;1?(f3 zg*`JTQ(GS(laJwuvF!Htl~Do9am?+B())Y1_1m+h*AEEw|)_AI0C}f2Q!Qb2pG^ zk~iRei6&+cO_BjLNe0lw3_;T|p-Fl`Gz~p;wl_YYDO(~s3ZN;=x}*%ZP0}@Po6dg# z%{tatY1?#mx6Q!Oe}37vSz@Iq)C}BEvWb}>kZfWG*(4cYlVpHRrqTeLu8B=%+5nq| z9y;3_AJ~*F5gjFzY|64OE5mIQL%40a|1dV6a-^+ejg_`dx43P(EiZJ-&@AzH6pA69 z6nID`kOmLQAet2%9+m+#nZ*NWnlv8P1EOg%c@(HT&=S#6e*jIec}N*jBv` z={`!zIA_oju~DSVc@AY=S_a!DaerXjtXb^&N;dVAj25X}nS56b|W%)Nm(Hcjpi?E%m<=|4)!IcLxkkx{_$fvPU6f?bo$ zA9u}~J)cL&6vofjL%FB!&z0Jhe;n6${E}buP+Nz0xKNh>aqDSH#poM}q_eEfJfh z{B5FhDC^QPd5mQtTTJpo*K7eTKW3-Y<%UbaWO_GQ9c1$ptO{HD>#3||byZHVx@Z8Y zmS5_ue`{63B#I=S6o5!3lmZaRAd?jYAeI3pnOLJGlO_R(jRH)X{2vYe541#dn$nqt zvMwt_CM)0ElJO;$Chv!J09cx|9}U_Mv_xbO)Yze{ zOUgiFXI+EFu4~qQ5X*1wvDdQ1DjvHoT0dS(2ewtlrBlKs>LZ@Cen=*i)(^=bmldoZ zf0h9*Nd~wyNk6Ou;L_y$XmEa@B_e~MDhp*@R)(ioA_l=Jwr1z&F)qJ}D_hGVE1hJE z(vQ}yz$yGr)-IWD37P1Qc+&YHnNT`EB!gsDaDG?@$Rrsc)1>^c4uDLP@uR``ftH92 zLdXQ;hm_$#mi<{DILOv){CpoW_f+}0UD8G6Cx1QG3*C}m{?c9vn&dC?Nau%SLh1aF z45C@V`C%DAlVku*lk&ql05na;j|SrhS|TzCin37FWo5W&5-kWe&6{Oflg7ebo3gsC=321JF)J=un*UR)C zB|b1MSZg#?Xb&DO>YI{V%8i3pXDqlN!6y-|AS+Yl_p*z+wY=CkO~QQR1(Scs)- zf3Rizn>Bz7Lhi7Fzyu2n_rTOnM07w3J|XRoYX^Yf7+O%EgU5N6j&YC5$L#5OCA~y-L5g`w=4tV;&I!FGloVh6$cz3HP#uQ+ ln|K6_xi3FK{RgutCdvW?H*Ar6v-c=H0R-W(7SNM8Du3*~<3s=e diff --git a/summary_results.json b/summary_results.json index 764797ff1..0f5cfd9d6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -127,7 +127,7 @@ "a#": "", "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421591, + "Performance_Result": 0.376622, "Performance_Units": "Samples/s" }, { From 2cde15f5470e21ed6a959ac18dc34a8f9ada8ccc Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 29 May 2025 17:32:51 +0000 Subject: [PATCH 679/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10938 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 2e1c11a44..05e6bd05c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-29 13:18:04,511 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-29 13:18:04,511 submission_checker1.py:3308 INFO] --- -[2025-05-29 13:18:04,511 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-29 17:32:46,227 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-29 17:32:46,227 submission_checker1.py:3308 INFO] --- +[2025-05-29 17:32:46,227 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-29 13:18:04,511 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-29 17:32:46,227 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-29 13:18:04,511 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-29 17:32:46,227 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-29 13:18:04,511 submission_checker1.py:3324 INFO] --- -[2025-05-29 13:18:04,511 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-29 13:18:04,511 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-29 13:18:04,511 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-29 13:18:04,511 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-29 13:18:04,511 submission_checker1.py:3345 INFO] --- -[2025-05-29 13:18:04,511 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-29 13:18:05,650 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-29 17:32:46,227 submission_checker1.py:3324 INFO] --- +[2025-05-29 17:32:46,227 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-29 17:32:46,227 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-29 17:32:46,227 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-29 17:32:46,227 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-29 17:32:46,227 submission_checker1.py:3345 INFO] --- +[2025-05-29 17:32:46,227 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-29 17:32:47,201 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.7205 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70993 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -50,6 +50,6 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.42443 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.26503 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 767efdbde..fb814da09 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1306 +1307 diff --git a/summary.csv b/summary.csv index 3602b8874..993de4daa 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.7205,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70993,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -28,6 +28,6 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.42443,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.26503,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 553f3dfe2..fa2d9336f 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.7205, + "Result":1.70993, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -879,7 +879,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.42443, + "Result":8.26503, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 538f35c770fa781a6c16ac99b45a5f2d4eec3697..250292118086d496cf5563775918edfea263146f 100644 GIT binary patch delta 3884 zcmZu!X*ks1+n+IvRI;W{rL5UzEM=Rr=9hIy zG8HoRlr3xc*^*RZvW>wr^<2;O{9pXfi}T_>=Q`JCxj*-HotIX1Ru$g6ggpC4?RnHp)0|nhZ*Og!RQsu%!`fDzZd$p+a z*%`ouKGh6w_mC`zFNv`8{`{KIw{e}Bz1K6(6$7|7vngp~nz0A^LNbDxwXVwSbN%qL z*g{%O4eF)-k3^TQJ0ekWM}k>-JprO`q6Ir$(Po3$s9ZdztC0RVVGhrpp`7~Q**Vx} zi_M$FW}Mmm&4kgtxeZc9KJf= zs4(vh;B0b`FXkPI|9mEtGWu(DD8&i0Zp!tTx8}J7LXiG9w@3|+>&^^C3~)nPD00~?>1x|N9|{(-HTtW;E>@}cq0NRbDTep>V& z`Dl7d)5#Ghp5rDb#n-X zGS5UT>r=3q!qm5GNxV{ha&#yB_EDhWRd<*o{Pw{m)&u?KX!U{3a4hRCpd?l~o_^3W zEkN#d-Nz()ba}g|llrZ_vtz*K=uiJ=Cx%6-LZ*lhWJ{5_nH+Z7@8{>fOkMwyM;qL! zUtiI@*QdMbbw|dO!S3!l^@bSYEG<<(df~|2n9{I3()HH)6M~~T<$`QrqtWrLUbIWq z$GBNkSHdO1%>EU*ocI>)E_!B@%8x{4Ji91wHQIfs{!?ZC_aSNWCHt*%(lwifz~UA8 zJaSN({W`3oDZBL*J4tuRF(^UI!F*a$vEgtG?k5iGZK5#Mv(|CK(WFD*1&=KeW#g@; zqc#u+Z$U@eNKt1=VyF=?%w2GssR-Qol7?LzYx@5G;kqGH?pia}QO_^w16h(1Z9l_oP=_E=&mXA8}#G zlN8(7&rsiz6*k^at2^r~ZQ~PB7hHEAuA9M4yXyjH zCp?ff#{C|rYi^~6wL1Vn;jb0H)?`PgmPOpL_JvgtfNTdnLO>jyp#Gg6(c$>J2FHZ; zeN*oxPVT*ORwq`uF@>ye4P_#Q8f)xq8_tB=4W01OeDUd`g+W!qN$X)fZHY^n=}+z7 z&ZX&PUgi>?x$K@8^12#>biQYj8w11ANFV|gj6QDE-H$}$MWPFzM*UoE5k!;Ca3)|h z*)9=CY1~s%a&weQ`L+YfxKmQuakDHl#)D4OshR#qTx-L=@T25giTMC3;r)6UN#aBs z4{s&{;!a%)OeT!7kn`YS^zZJm@%4s6cb-6RNte=AY+;_62j6-0ZJSgU=lJH9-iS}F zAuww_pz0sJO|Pw6a`m@)Fm?Rz#UKL#$T*GS1noh<4E|t!FN)Uv??6)~Yw&d~9#w4k zU*;oPy3EnSiJoi2HFiz~D*b!23>Y!6iF9`-=r982NF#+33VT7B2{8oc5m08NY~AP~ z_)vGytbYYMiu3{;M@#I!ZQNh9JyPuEF}boOrnj8 zzyU;F=rqJi-&%l)e2n>-aWd#kVv_1ZdU=$$#TZ@fh1!mx07yu5Jl};3 zemP-2(dzrY85c7i@GLe`QQKuu=M1JqwbwUgm6F3!D^Metkq&P(U?p({1}J zkzGU^)Ak&O0Ru-T#f8OfGB7-k**uRscYon|oQ{~rqX<(X3!n;gAV&H%oFddN_qEc< zE-@vn^+O#rkvRB$OD=8VDyf_tV}{4j8lnL;90moU zQ1?D@^$xKW3@yZO^gQ7jnBky?m!vxGmh9CgObIj=+Ep#HUr^5AQ%lLosajpRY+60H zPsr3)kC@A)TsF}w<|aL4-&RTe(<)8Dv`2;&w+2vK_w=(YPWS#S=J*uHeY2>!b!9{?2=Pczn2>U&XoG7eznoU={l0#~ zK`8i71_{50o|c$&&a~jqR(OqU&g^iB3PC(;8D1mjgjMYA|4n}bIHgkz04)w8r#}>D zY}X4>kSA4z3$ipDknL^CSv7cw(_v5sQn(E&%zIIDSHSlRpKe%rkCOa%xo579+Zjw) z^SUqC%Fmps0w(Jg&QgfR7<maWAIVl)Df2ECD%P;*+}EVAI|^E*-w7l844-`7v#y{ zR}eczX)mmF>ZP{e9^^S2;&N`un5v;TLEM6aLF54_RZz01f}J7A31&boJ{g+d4syx^ zQIU`~DCDn@8}cXSE)3+6LrDUVws9W_C4%$|fmpNRR_ZcwoZaJ1?g!3%@sL*c%9eYqriO24?GQG%Nn20-<^%ltaNAdbt2sdd9DrG~ z`~jNRIg4=1clgrdb%LY!@#oWfm0c~r*pw~9!1xU(8I6|t^p&q0$}8&1TQpZauGh)p zS$FQ7CTA(CQRL^O2xP5@euz`l;-zTM2}awGxutOTXqD3P0(X!)iyEFSRr$wbjEB#b zrVYBI2HpX+~6}RX!HsT!w`0zkk!Wb zS$D)t-*v9fCcd)$NtW0OzjOAvI2j+SaKYT?X^qm|a+@Xn*sP-1K4UeN9p78=KgJN?c)#&7&kc;yY{rSq zizTsxnaR;mgo|P?cc>6@JMt-5s6#hP33et(-*z5(NN4t)>I2NdN7Xs3o;UtZ=o*jA z>Z%XY%P#9aTgwBb(tGac?T1|6pC#rTcGu%MpM%uoZQ(m1htIzjYk-Rw(gO}j&2NU9 z$b;smb5atA`+SgZ&RXSsVV}xw-7v}?OSU`jdBCAH*=Eb-iA*FnGl1{+t8$FtieQy0 zvV-qGeyUs*8?Z3#vZ@|9otCU>D12WA1PXB1@bL|FC;QOkj!4qjs?vO`vNTcnej4J4 xsLWr%!648DJ`hL@^#3a~Y!E?PLP-3%MAJr!?)kr?&@dD&;fNx1T=`Gse*n|aKP3PF delta 3841 zcmYk9c{r4B`^E<$*_ug&FpV(=Lp9BJ5;9~g(^x{u(jZHj8le$OBGAJ2U}=W(6q^<1ChzTaM{x>9=gpb%UTJLV$_ z0`b#>52Atk^DOP?-Vo;})&8{3;O{4(lzb}kpz`)s3dof+Qq>ou9Op3{{>5?mChh=9d zARQeqUF*n#hGFxH2X`xF{(M00RU$)1KQY!$6ZSqe1f9|JvJa{h9b^=dueHmD2M##I z1HEnEdlv$I@1@^hbaW7ZvjfY1SDlu2@3%TDOCOt^${(eK$cXj~*G@Grd7O$`7apZ- zFk&%JcIxX1o?5mHRd4Ge)SD79ht@KkrJ1c3hEHuF`L!{rW7Y8FQpWKPx<%aft%#4s zjPWz`mlhVwH`e>c*JeVzmdrj(KQK^q0E|O7m(3S*f0D*H2p=Wl82iyy`Gi4EPDaN~ zZIuq?yz63p%f~fNPgJ`w;gd^7V@XjBvOyNGcV6H5q9cYsb>8)?zb317$>ra)!|S#8 zF`VV@ra#6<6ilNtQnW(y#J=1vvoeiv$xA?too5Cn2n7`|e-GFd60iPl|M=&!&Lbd; z9#uVVLBL-tm4d^lyD9jmqo0TnuudHe{I2&l1e}n477o*R5Q%Tv4R{1rf8~XTP0nX> zKYB}ofUj<;M@JO}>vLc28NuX%#-JUZS$)A*A&u9Yq@;dID6(uW<00U%>a);CTTJZNhn%yr$f(c$tSUqPsA-8_xC6J z+vd!+7F)jhcqC;+dKvv#T5~PRG99A({9J)+FSt2%Om45<$H+CWFE&X!6>wm6jwqbD zQS8oYMRol>2OI5-7S4Ry*G>!TrS&;qYg~fV;E5aEV^430vCcbf0XHkc_b5S$rUhBE zZDgigff%-MArvha@5RKO74#XBjg#ugiXSp4zrJtNLMwN@{nX5Jvw;>wDw)mei+r5A z6zn*0vr#NScec}bxhZ_RH8;lU%Uay09cT_s`e~R1~{rb+#==&m(+5V@h zKj7{3o8s_2D{=vEtRE_mH+E&0H`csst#bOV1Yf)h6&WMgb?@H;YiGAurf9h=wA(VI5K;wcA$ zCdN-l> z1uA~(B?(wXE>F+Gk3_Yss<1&xD8(cAtxKT8#~Wn2>}_lxb_JwI zmw-3zU&-0JJFoSRXQ_nLjru92r3oDuLIjO+e_6ORi0%e5ZU5%=uc+lIg80GwoXB|E z%IT;2M7%LaDXqM?t<||7eqsPCEIeR*N;vq2c|Oa2UdW@btXVaE_J+%?b+oVAY8~w+ zJ*Gi;({w^s`;~Hclj*mMnA71NG?8oqnu@q_kU12A_`%bS+2|S^`$f&CM9(3q`P}?z z=)i|5ex42xvt0$fiuiwQf;#-plr2>d`~A;fd5-|vHlDl>fRUWaqn|X=DJ}woaA{|{ z8`W%0_hdOaeE@zQn*K;$YgkrcAekGGQX-|bTtn@H=MjTy&!K%J_AA}I(yzFzheejE z>L{Id@L~(&Vj5cZO(z^?y`bH2z`OAe#H=04g!BEaHCj3>-}M50gX$skH}w!bYZl&D zLeDzYk09-p{`~d`He(;e{GZ^DMX&USxx3|_6-^f{Ur1NyxE(p84;^e0DJP&C5o}Pp zX&w)jbR@mKk_Rh#-j{V&w^YRergafHnCwonc{Gf8bZ6LiAGl7@oi=f3kx}zZ_wpI` zb#R0x4!)P_p;%% z@wt*Njb4$d40)v9PWQ+gq`&g_g>k=llR3ol{9Q z5h!(&j#g?jK@p7DLO@!avrb6-2wc7c`l{Xee; zdF?~n%?sN^8DZ=VuX3$VGs?C{SsVSgRImWzS#A@j37u5bW}h|rbvD$ zO&_6lP!GjtiY_xEBFuOX1G4;Sm~EWQj)F(@947CK@ZbKL+N=V;9eba}Xh5W6nD_#y6hp?zRB}uE1btCvV8=*)*xdP6Adb{uybDboA8~!{9(_f}pHvxI zn<_>rdQ2>cAbv5{A@3%dB1;r#`WkAsI27MCn#dh8E!0-9F^|37{cT5&h!@bmBH8-3 zSj3_Xz>9=gQ#v|7yIu=Da+Q^8XrslJ%u^_Fcxe4AqnLyqg`m6TIM3I)UH1(&5nva8 z+bI=EB%t;33ES7@Ku-`;#^D@3b+B|qqENHA%7<{&p93)k-D#kN}P z-7l7UtJJWQR|mncoxUe^inAknwogdoE5d;F4_w5sK}^JSq#wwG`B*`;S-cYOc38qa zhYPPey>R=(6y`s&IH%Ydy6K?3q#d~zrHb2b!m$*9<7#eny#KtVf6pQ=OAR1lLf1&W;=%xn zdx(Ne8|Ugw5~u&lWpL%NYMNX*3ffiBG5w!&IEnfj7f1elE}otBPcef09%0pw=uC& zG8z~_0*(2Eoo?tJwQ@)gszel2 z_5sAnUJss&D&$EzY-Oj<=OUd=bK}y1zex+EY9`tBvsqB}9j^;R6yuLvpaJNn@)-Wh zwvF|!+wg?C)XgjJA#iBJ#=2DuB=L}ZDLl&nrUd0$=0f6NS+O%unTRucM@I7#69dH_ zxOf33}qlRZfe)f0BqqZRDUvNb4{>bG)g56Iko9L0`OEqE=P1kWgQ9IZ~l7 z3$k_ib2BkN+;UHODR^r0=1^Yl4>MQJ?C;*S+7G?sZ-A75J_SdO79Yi3bZkO8XV%VN zd^*+>E07hr@R@zPeJ0mU_hLh_@n_r%?{mu;rnGknMFjPNTpWXwfZG68#elhB-0nyh z?w@$s!Jd`e!Fik9?@py;?(zNpSx%65=ozMl%v4~S`j=bV9nB%T?BJr;y>6`Et)5t2 z1es|nv)oV<7aSg#2{3kRWf6q?`(~j8R+e+%npB0T!SX;1S>sDReq|HknDZ&a90C3;fkcYO-Q`^4zz& z6UVguK?D-yxN4NPr_yFlWiUWN{>cw+3}TMC3T57%SJCBjJp=Bhey3buDh&IAI!|Gs z`azKu@DEb21=U6=1_js7<*h37RrFat9Who$xDK-jHFL!$fJeBhlis5*%P!4YpVqtb zPWrm!i++~ssvlUa|I%#LM3Xkp4o70+N!9hFdhtLB-bCHy7a^$3%mpRUmuncO<{#)z zg?Wm8`B4JHX*TKXu2lB%M?2}7yuU}Pva)~kRiF^9y#Mc-x{tidBTT=lqQjFyPgc?A w38VL^Xz^^&!Kw(^|3%9K0$t(-fxw{ud#De;(ydf=c~t00s)_=`%71tM3(AQ+*8l(j diff --git a/summary_results.json b/summary_results.json index 0f5cfd9d6..449f2fa9e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 9.42443, + "Performance_Result": 8.26503, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.7205, + "Performance_Result": 1.70993, "Performance_Units": "Samples/s" }, { From 2d54afeb822a5fd0844b31354df1e50c40619f2a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Fri, 30 May 2025 12:16:49 +0000 Subject: [PATCH 680/719] Updated results summary [skip ci] --- README.md | 86 +++---- dbversion | 2 +- summary.csv | 46 ++-- summary.json | 550 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10938 -> 10935 bytes summary_results.json | 6 +- 6 files changed, 345 insertions(+), 345 deletions(-) diff --git a/README.md b/README.md index 05e6bd05c..418e515c9 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-29 17:32:46,227 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-29 17:32:46,227 submission_checker1.py:3308 INFO] --- -[2025-05-29 17:32:46,227 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-30 12:16:44,433 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-30 12:16:44,433 submission_checker1.py:3308 INFO] --- +[2025-05-30 12:16:44,434 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-29 17:32:46,227 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-30 12:16:44,434 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-29 17:32:46,227 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-30 12:16:44,434 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-29 17:32:46,227 submission_checker1.py:3324 INFO] --- -[2025-05-29 17:32:46,227 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-29 17:32:46,227 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-29 17:32:46,227 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-29 17:32:46,227 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-29 17:32:46,227 submission_checker1.py:3345 INFO] --- -[2025-05-29 17:32:46,227 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-29 17:32:47,201 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-30 12:16:44,434 submission_checker1.py:3324 INFO] --- +[2025-05-30 12:16:44,434 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-30 12:16:44,434 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-30 12:16:44,434 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-30 12:16:44,434 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-30 12:16:44,434 submission_checker1.py:3345 INFO] --- +[2025-05-30 12:16:44,434 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-30 12:16:45,520 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 0 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8854 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70993 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.376622 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.53809 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.2287 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.26503 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.53809 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.26503 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.376622 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index fb814da09..a3285fb86 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1307 +1308 diff --git a/summary.csv b/summary.csv index 993de4daa..1c9be991a 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8854,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70993,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.376622,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.53809,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.2287,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.26503,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.376622,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8191,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.26503,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fa2d9336f..297e838c3 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.431769, - "Accuracy":"mAP: 76.951", + "Result":6.28304, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -30,46 +30,16 @@ "weight_data_types":"fp32" }, "1":{ - "Organization":"MLCommons", - "Availability":"available", - "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", - "Scenario":"Offline", - "Result":23.0274, - "Accuracy":"acc: 76.000", - "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", - "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", - "notes":null, - "compliance":1, - "errors":0, - "version":"v5.0", - "inferred":0, - "has_power":false, - "Units":"Samples\/s", - "weight_data_types":"fp32" - }, - "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.437938, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -77,8 +47,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -89,17 +59,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "3":{ + "2":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.2491, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -107,8 +77,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -119,7 +89,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "4":{ + "3":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -149,17 +119,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "5":{ + "4":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":22.8854, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -167,9 +137,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -179,27 +149,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "6":{ + "5":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.28304, - "Accuracy":"F1: 80.0", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -209,27 +179,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "7":{ + "6":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70993, - "Accuracy":"acc: 80.000", + "Result":22.9804, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -239,27 +209,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "8":{ + "7":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":14.9346, - "Accuracy":"acc: 76.000", + "Result":1.70993, + "Accuracy":"acc: 80.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -269,25 +239,25 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "9":{ + "8":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -299,25 +269,25 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "10":{ + "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"datacenter", "SystemName":"gh_action", "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", "framework":"pytorch v2.6.0", "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, @@ -329,27 +299,27 @@ "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "11":{ + "10":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":15.6951, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -359,57 +329,57 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "12":{ + "11":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.290026, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, - "13":{ + "12":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.376622, - "Accuracy":"mAP: 49.593", + "Result":14.9346, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -419,7 +389,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "14":{ + "13":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -449,27 +419,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "15":{ + "14":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.53809, - "Accuracy":"F1: 80.0", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -479,27 +449,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "16":{ + "15":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -509,56 +479,56 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "17":{ + "16":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":23.2287, - "Accuracy":"acc: 76.000", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, - "18":{ + "17":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":6.53809, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -569,27 +539,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "19":{ + "18":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":17.8765, + "Result":23.2491, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -599,17 +569,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "20":{ + "19":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.437938, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -617,8 +587,8 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, @@ -629,26 +599,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "21":{ + "20":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.438894, - "Accuracy":"mAP: 76.951", + "Result":8.26503, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -659,17 +629,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "22":{ + "21":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.290026, + "Result":0.457477, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -677,8 +647,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -689,27 +659,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "23":{ + "22":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.6951, + "Result":23.0274, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -719,27 +689,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "24":{ + "23":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":0.431769, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -749,26 +719,26 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "25":{ + "24":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.8191, - "Accuracy":"acc: 76.000", + "Result":0.376622, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, @@ -779,17 +749,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "26":{ + "25":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9804, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -797,9 +767,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -809,27 +779,27 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "27":{ + "26":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":20.8191, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -839,17 +809,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "28":{ + "27":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.457477, + "Result":0.438894, "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -857,8 +827,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -869,17 +839,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "29":{ + "28":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.26503, + "Result":19.7056, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -887,8 +857,8 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -899,7 +869,7 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "30":{ + "29":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", @@ -929,17 +899,17 @@ "Units":"Samples\/s", "weight_data_types":"fp32" }, - "31":{ + "30":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", "SystemType":"edge", "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.7056, + "Result":17.8765, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -947,8 +917,38 @@ "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", + "notes":null, + "compliance":1, + "errors":0, + "version":"v5.0", + "inferred":0, + "has_power":false, + "Units":"Samples\/s", + "weight_data_types":"fp32" + }, + "31":{ + "Organization":"MLCommons", + "Availability":"available", + "Division":"open", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", + "Scenario":"Offline", + "Result":0.353561, + "Accuracy":"mAP: 76.951", + "number_of_nodes":1, + "host_processor_model_name":"undefined", + "host_processors_per_node":1, + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, diff --git a/summary.xlsx b/summary.xlsx index 250292118086d496cf5563775918edfea263146f..db9d0324acd8dd825ba65554ed9d5acefb2deb2e 100644 GIT binary patch delta 1691 zcmV;M24wlVRku~JatH+|+=)JMvv&xv2!Gj=X5V+p_DWrXC2AzDw0}q@koFJBAeI&E zAC>_unN|SvV!@;GQcItcoIvK_`^B?EKS~z2JZ)2LNW+??0-oecLMIvQxq(`XjE?en=*e+7HPfmld=h zmH{qF2Dmg?Kdb}b(j@(8kba;gB!i$U3uQf4hNoC!2Ei$|X6ffKF29K^TgxIVon(vF zPkxf^_+?Kpof0xp9&x4gLo$Jset$>?$*ds#undq%GC-!u`C%OZnI`2&gYp9{AsK{_ z3Ca&C!-FjQyFPG`ty%f`K4k8x^K+$(&QJb=tQ$Bbzx}1%5;V!5tjEf5(*0LL zk#_CWa196X;OdKD8Q!4 z{Lx_kKubuc32cJ-L&|XDtYZG?n$4f@!{)yFKU(@f%5zJeZrUaOkAICKi6{LZl2Miv zfJg}7WElx3%SbrcgaA(4Bml8dB%G!Wx;6+v&=Rs~g2xVJJz6F=P7;ER)7~fl=kF5P zYguHKjngh_K-t;?zwG@JyF>%B(I@ew21GK-k{%EVA)6IEAeI3(Nk%^5v`rci>j2O+ zX+Sn;K+qDBLBL~&s(&7#MS3+C>Q{+c7Oi>oO`!EFc?w z5>Hw{B%>^;0g(_=SwRD08K9D6fJ&1E#5w>fO%~Ah#s@0F0%Dg9Ja#DSkut=xiUm|y zz`kSoB$2(A^;N`ZUsweqBwXChu`L*c(Xe&Fv z>XH{_?M?}jXp(r+{~;M=Ndbt25XlMx5X%6POsxSVO$HDf1&}luKsFda&=Rs~$`6@2 zhq4|m10*XLK!5h08$cT*gC`-mui}s0*+tS<@E&Hn@G)=GXmJyrMw3a>7{uA-T32pc z*PnfNsSZD^KPK)Wo{bmJhU5Hi+V>~x54B6jvztcp=lbLF+{4q~wUu`^7+o$WQ6v2_ zkB*zs=XrJy<%?}LhM!0456WxI7vp#_PQR>2&p)YeIe+P&Oyl8?t4S?q<3E?tk3S54 za1VbNxIg{e&_6Yf8`;wDquJ_KqvT((X`LL+`!~^fe{nOOC5=fm$`)$1yk=vu{^-5c z*J(UoedRP8ALBHQr}y98M*U&5$iK^e-i_ioz5gb^B-@WLJoyVyO9KQH000080000# z0OxW7B2Y|oq4p(uuQ%}nikzp4(?%;O z(Fq({S9b+7glP+}RvWNhfQtf1Il@ed4&EDrv1132)Fr{8g1FOGQXfjYf5(QhZ@LGZ zr|Aky@RUjK0!k-MxTF>I- zJ~}ZO+qO;GMQ;uHGyJ)IdFxk=h3Zbl096HNgjt|o+p56VML1kTo!^EN?*#D2Q_yGL z#zlrc;86%bhPY?tWAXG{C2vrbr};8Ym+>O4vz%lra=-lQM&9&afN~u2Z{is+>o3)+ssx^pg=PCL2DX^s*%Z002+| l000pH0000000000004l2)ss>xJ^?9{kt!nw+a~}3008BI2tNP- delta 1732 zcmV;#20Qt;Rk~HMatH-()Z1Bevv&xv2!Gnno;Uj?M_8>(utbf-mG%$G1k(N?8N{-J z{lhYVC9`S(OOyS>Mgc5M^3V3h2P{GUq09~qe1$CmXHjBt}K-GSQ(yTi5Uc^*qWuE$GH3^vTQAjtaOqs zT0dI1{ElDt1k))Y6Xg+CNvWRT1X(hti3nIr>bnw%fj0g!1@el#dQ&=QhC z2$`V#kTN{TvcKyC2icmHpYKEFo;p8Qy6F7mFUY!qQ}Wwi+ATqo{7D`u{g6x`r5}<( zG%H9yECXng44`Rpepm;9rb+qHp!`5fNCrVu7Rq|83^z^U1;M6Sv-0z`Xn#IQm91rw zl{U?y^OL{)(+Ns+JYAyvXjDf$sr-rHZk$!jA6>Kg^L^OdSN}&#|3`Ul$Q6%xC|3fm$ zk^&G30h}x&;ba*JCz}w!X`2KfHj0GP)Irw<0SH<`Hcjx@p{z&C<1MGYuhTi}V3y5TtB{d)tLMkh0Kr91Pk_=F3vVd3zK&8n7+TQp;C0Ic0vVq4AWj#`cSXQxs z3JchGET1H@*RsA!Sb_xPm40AKQ36u_1D522NTdWrGRl$`5Pu0FmK7`@mH{l8UjqkO zlLo{_0WM7%kPR9Tw1jM$@M!&;Qtat^uW^97**g|EHFG{U2>* z=T}|wqO9F1K@v?8Px?P3qbw-^kq{zTK>%VIK$597fTYO)Vxs_(CIiR@0|;6|Hcj~< zGv`p&qh)|(1%Cs`-g5(JgJkd|B==SPu{*m+`U>8|Y!^P}Z5l0ZqSI(HNg9JVyIkwa zZR`58?=IEhhxNzAJ;bx|;@NPV|4sY;Wc{Ib>3DY2Nd8=ZT%LP)`n$IB&IY5)Z=1QY-O00;m8 z06zfdaB2Y|op|&M@uUGLsikzp4(?%;O z(E%JEoW1kJ;06CcQ#cp62s7oyYk@o!yaHPHq=p$B|e4XP_L0{F`_J^trD; zLG=f-DJIGS1#Z;aS#y*4C|dyylT;}@0hg1JDM1Z{8~^}#Y%g Date: Fri, 30 May 2025 12:24:22 +0000 Subject: [PATCH 681/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10932 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 418e515c9..8c5f83cee 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-30 12:16:44,433 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-30 12:16:44,433 submission_checker1.py:3308 INFO] --- -[2025-05-30 12:16:44,434 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-05-30 12:24:15,750 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-05-30 12:24:15,750 submission_checker1.py:3308 INFO] --- +[2025-05-30 12:24:15,750 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-30 12:16:44,434 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-05-30 12:24:15,750 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-30 12:16:44,434 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-05-30 12:24:15,750 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-30 12:16:44,434 submission_checker1.py:3324 INFO] --- -[2025-05-30 12:16:44,434 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-30 12:16:44,434 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-30 12:16:44,434 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-30 12:16:44,434 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-30 12:16:44,434 submission_checker1.py:3345 INFO] --- -[2025-05-30 12:16:44,434 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-30 12:16:45,520 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-05-30 12:24:15,750 submission_checker1.py:3324 INFO] --- +[2025-05-30 12:24:15,750 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-05-30 12:24:15,750 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-05-30 12:24:15,750 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-05-30 12:24:15,750 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-05-30 12:24:15,750 submission_checker1.py:3345 INFO] --- +[2025-05-30 12:24:15,750 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-05-30 12:24:17,008 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.376622 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419959 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a3285fb86..07f7d5ff3 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1308 +1309 diff --git a/summary.csv b/summary.csv index 1c9be991a..69fc747fc 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.376622,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419959,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8191,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 297e838c3..2d0a9c92f 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.376622, + "Result":0.419959, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -739,7 +739,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index db9d0324acd8dd825ba65554ed9d5acefb2deb2e..5e40726af900864579896fa20fc109e44c1081a3 100644 GIT binary patch delta 3545 zcmYLMc|6oz`<^i+*$weV8I7?v2=l1Nu9;|P5Gs;vLuAVuS-zyVp@qyBlM$-1JfdPq zLY7ht#x^yU5oO;QYu0{z-rxIq&mZTH`{@`ss z9iY~rLNAdO<&seuMsANBJt60tAFIr#w6UHBapw-aXFpJS?8S-yc76Va(E-ZsxYqGW zN(UG?qMof`%2=r^yn4Az%xa6_g8BHje=V@SBQzU2FKcqR#F5>w3uz=l!g))<8)gqx#=S&b9XolWAe(AW1j zv?UePaV*QCgUY41J}d8*D@XlkrYvdVcbn@Xw9oiojVKrDqZD~xYmlhQXdc1M--j`y^koL6o z%q)FnnLV^L9_2qr_&gSkm3IWpVpiuZXK8CrgIvWx1&cw({dM{;?28v!%o}E8!(z=X;nD?IL6 zi4+PMyOV}{_M_PXE~ftxDb~R9fr}+SKaE1FN0V@kJGbwPslD;XA%~~4_YPG#!Nsnu zt0g8BMVjoHc1@A8Kz+nEbi(A&o2Yu%Mk%Q^3Hb{91p&C&%pPpx9cMS3!*JZ`SI zS*wUmcvqvDdK;Jzs2+OJPQ>CAl~c}8&N-pUk?oG%ZzVGxk}^cBuK)Qn+|xQqSf6eB z9_W>lMe;YDnp<)&%DK=F2Cglj^k#_8%;Am2JDB5NmuDL#oeQ|gkJr&gmZlH47t1PZ zgB>sHWi@f9pMGz*vYlI=46!vzmg+m<*){mgQ@Fywc^$Y>7QgEo@#I25&P1y>^>~4> zPT@?9*6}2Ns?q5~f&DU%rI`;;D4q7HQ*Fa! zA$UG&)Kb4U+Z`p!I`f6^mKl<)2TvJDNeEy0xFpn@5@a$bgN-=Unl|vD{lV&RQ{cVT zq)PHcEfFApOHip(V0F8VR4w`tm&VtsdOC8tj}bdu*iOd>h`9FJTt*EL8C~Fby;XNo zBP~p|#JH7iGTGtHmMya_MEAqGw7hT>DA z#<`N(Nm09Sw!lDckCo)1k-=A_j=1&L9r>Z?w> zK@8E~y!-tj`H1=_DJ$QD=PqD)Eo6mrbxwt9bLOmS~jf4Oo5psbHJ*636J*IKAF&VAdV=m-MmLQ69x0tO5^PfA^2*s zczZD`oqdD9>+^jRCzW3iZPP;?j{0Z*>ysaAhXj;vq{{mp_bD#-tNP-{@{4%kc;7Rz zj%F>$&8^33#bLtF^L7>gTr3fM&Opl=vvRp5vZ4f#K{FnUOHWAR@w&^zw%p>NoJJ}f z3ObiRb`D(mY&NHAs9RpF_0Dqcj3FDAshkjQvveZE`l=x+-uLoils#`R%0__h9TaUZ z0yO92*Du<9{X0HA;W5&qJy?0BTdP+maSO)rWV&@+p%>W7nXoyMh8Qb9TYW*> z+T<`*`BAvd=02nmD&FhzVvId+Cvje=rqnwK@Dhp5$N#x#)4mV+h$l(JF4W=3ES0!h z>BFAg&_A1hDyCMp#UK=T^X<*{2)wd(gtAQ~@mu_ZD{KTeMHu%$P0`~3Qip&0CM-1+ zf7A_MWr|*B@~VF62H@$DH(^eNx1=oe_|B5t3)PbHOtTt6Nm0hVZXV>(`aSWaqvcCc zM->38=`oskhr1u<}qVDhj&${T%H4yQq`sPP#yzY8;#Zw~{tSiJQX`h@vh0yrQj; z)VAU-xH$-IG=(Y?c%JY=`K}^$2`bF94wSSLsYedTIh8@mKaUQ%wzKK==Jt|vIkWg+ z(MpfVJoN_nUvR5_qdmAS5>d33e@&znlHT?TiYBay05S(lFbpG(45H+~2FX%+Iug8cV+eVV;VkN?6cS#${ZAN+hCqsyP9r)j@WZGT)~=o4?7 z7>u$}Cp^W3FXQ?46D|E_I#4$QJJd%_%UQsVXX_eWshZp#)o8wx2k7A<6}-qs4Ea0+ z`EcF}Rx82t;1tONvR4x@gm#e`NGC?cNIDhamDnuj)v}+3U-|eNxn7LV<(OJYrcxpO z9h%;q?+_AG{-fK{{^iINe9VldJ8UsVBBm;2A>d~*C(jt6!B)69v=JPF%L%M_X`Ix?X=@T3!SGNMQeE@&LFAI_-l%?EQ zsfas)d)Hp&J=Dh!>^2Ps3n?ET2fHMW0Dc7r@ECm5)SoAh=(!Nw*#zQ2Xo3d>pU;Mn z6=4aFdCpS>I+G;xzf{hSg7w!0Jksp6?_w-!k$SSEnj|NN;+_@8*BTF@( zSwrvAZnF{B)tdt4ceEZRXL2VT!bHX%`swiIkY+?$eA>ooL__nsVzUmT!hpZ$vo4Su zl8f+a1vDcHH`!e3_O zPu0HN++co<+UbfcQhLiGR&2KT=@=?{Q!mWdRX>i556=cDJ2f(j_&a|HG589EYxt5> znSf-k@PTWbtzd8hg#(V%kz=noCCz7P-{w6sL-T#{U=_ML}@xTLb^>4Mfg;m8E zdE30zi*-Ov<}5=s(4f(0TV0znNVp(96%9EwO6M2Zv<{s>4Ff=Pg5C{dcC z^d^D|NQn?pAQVGYAR?e5MVgc^p7)!1Z{B^eFZQ0<^IPkA)-$u~FFn6h9n8n0I+0=+ zL<;BA2KsYoVJbls2vU61axjvh&z;aGA!7UnzaI^6bI{+cXqdq(k8LP4E-c`!4oen7~5A`@`bJ!`Jw^et~H-15E1Ey&9 zi`LDEot8=8Mym|FtAG@iN_;b#ep?MXyG7W|)#;cm&BBgnM!%qRcNLzxXR^nF0k7_% zUuAUbJogP_=DqX~u_6w)rZ4EWWH&Sa*+4^ezI)9wBL12BBByhm)^UaG`qk@dwfSJg z&hH&W!-AXh(+P`;+Mz~0pRx>=pGqtGwRxe1MU=s!U*d@!--9_HQNRyrrwm(*iTUv4 zLPZ^2^Q{v=&s}@bpd*~KDV**v2W$_woUI{@g-6nIJF}3HDl${_M9AI5Fm3AhVic8} zc*db6q^T1_6J@|e!*y0OFfS+HMv1M2D?@Rxk0xQ+*Vb5L{ddIZT?zlp4;Q6`6A)8q z%9fkyxAOW9`>{K>H0)ASjLO#sI$N;f%cm5V|FVUiRloVwlzJwDno9c#gouf5{#Hv# z7Tk!#a96hyao#$sL=3l86%O-ror399{wkc3Y&H!S4LZobf=PQOLriXJo`)$_UMFBW zhojAuW)@vC*+;GSlj+bK$APvN{dbh1H>6fN?;3O^YmEGk!*t#P;G#8CIZ`&+5ehGx zKV(vqt9nG-H3EfLlfdusAK_25m?A`eQ`mb?8%Ww*!CLnEGvhBaSH2XHMh{xIereuj zoc$eiQ`WS0t-nwAm2-^0@QiPiUeA8Mmv{cRXOZ<%&IIowpZobA3ztx)gG(OUL3);} z9ah27^Y!In&B=wsr=I0gOULFvFOIk4HUio%X6cTqc}7(_ zZAn*mpnHJ_DGZj(#F{lH;@u}U)oo4s0Y*j}CnwpwSTeXzQo`bZj- zrZ^JbnWeu814zXt=Gz(%vX97&-nCaI={z6ZJGW2%q7VhH`?2^5p&YyqPnV1zUphM( z>@?EuRu=l0Gwmj_gfngV6=#}UE6b1mS#?L&7-}N{c{ne%!g?U69IKRqlfYvNaY|yw z)3HiwIjTo{TO2Y}cfb-g^49C{)M69=vA~}CdWVs_z`^plnv9aaj8@`XTQtJzN7c?` zfb+UQjuH|EwB!@=9*?Csj^Hu1I3=R-JykP(lq(Nd3jgZ%(T#5FX<_bT-nEJz%5T@6 zAQ$f|ZNHn<@X}k|HnTo)gR!q*c?hw02||tDbfQ`;Ik@oforvd5h%sx)D2e|L2U>s(x1P=X!qURyO(h%dzLmp0HO!<4~)yr9UM*%NRGqbGCC=25Sn$H z@LA~cdA(GXcCx2|9k>O--`?ox&}JCt_)#lFvtjt6l~G;BX*;IAj<{`Z&LgMS^V#~j zf9*Oy@fMi=81#1v!s|A+Fhv?e@&H*7IZ&rA^ba9Wct~>Tqof}j-MlDIGaQx!<>^QO z$nApV<=4iEK8pt+%FXhc-fNY)DFIYxy~f$6V%pnIr5_|-i!VeFE#7TadWdViW%K63 zxcrHmk@qdeI}r;UOw@Y+~S+3XtZvoZC;Z>Jz40j?>Mk2`n(CT;YG^_>Xn?EeNOe{RG#cL&t1LH|`g zrhT?$ymWfNh1uxnUZOfAlxI|%B0Zhs?+#|dq}@m!u@d`t~&<|=~18@ zmX*)uKt1T<@}Z#QN4N@oH6taS==X~a+JG#oBXMlK#n&dZmn3ezmZbdl+j`>Mh2_n@ zR60^7N2A?x6Xy}NH#LNbD&wH0{;(?iZ`IDSj7u z$kX65cpMP~xQ&-Pe%m(H{zb~DkMi67`)T27kzPgHafT4tH4Y)5TuF|>Au>n)#d`_; zgBQgW=ueq|j&VKo9R$eakoOPau%g&mu1f}XJS~U^=pU5RQHF#}wR_a6q+qK_s(OR^ zfq1BF)oj)-B}u^a38t(wYX{Q9O*$ze2b|Xy*!ANg|H4VTHkS)M zjWc2k^zEHS=&=)|L83VvsqKQD2<L2))a09VNQ|y zgiy*w%3XGmFQ}^X88v%x_$2kTcH~uf-pQ&)j-i<(uj%mK4O8bhxF4*t9$-JW!FDnG z@He)L$Oo6W94OzQfVO~}FA-?eF?8c!h2$gt>}v7i$fcu})pvPz&1xp>*RabJbpE&3 zLG-42vnEV@XVw?Cd4s!((sQ+O+Z~#1@SX@UGtyRv9tdeMMF21JUOunMck7$=h_dIW zn!3vNtD~o*(8(+vN#-?2_rv#{B^#j6OQOe*&DPq!0pL5zl|z3jJjQ0rIb6?XJB|l8 zp&ZfWT%QJEdp?;l?DEDxR2O6qh{VSZ?%;N{Mz4kWQ^#9VkWO?Z^?Z-C5vv&D_bpGW zllfwrHu6Cw>lZZdu)7A_Q?9UAEtHAZG%D@cz9I*DfM8+cSGhD4r4uswdm(Ba@kroz zi{XvT3dikjLysS2Yau0R@2&RZ%4Kxnt$QVtVyGVK-2Fke+>1Mk(wMrq^$yKg_GF+S z=96xMd?;k7a4)1j!9xV)`KX-MdxHUU5%H&-R!6(0eIfnDynMvbmx0tnk6mA|v@V=o z5MyDB^k(p^Bo8t?A1ZbprLIfr7CX%GmNX$Ie@IXOcAATHn6foUPyX#9J_2wF4?E#iU>2%DlvLZC8_?%9EfHvVzzp&0}Ur1(|3uE0T%CF^>DB zAc@2LRqI@~sbZO{P!d5x74Tq?p|^tmjW{n;0NAHZg$S$4v2agSw$Tmc6@%LiUk`lL z&$ccSB|X1e@)LcC5rYSc(w=yZZ$t{u3+6gEN0( z#RXHBGTJDRF_MABt0=yxnjuUwjR)<2%6ql^JQc_Qe1gRDO-T4bsZJ*Avi)_ZQEGH} z*&%c}8u3H1AYJl9;_PnV*rv)7q`{0*zcaf!(ZQH*$O_k%az3tNqr9g{$jBX- zj|_vb(gL&-sn{h*@3nW5vk|8(Tg$N@&}D&V7O3X4p9#TF)tddUuZBl`^iK1D*f^1^rXq&mHM}>^q8si8QoV0h*sD*;iUeyU>)C)kB7JoAFbDt=3p3rh0WuB zxfmR}ocBWOi38MAd@t_i`DbFD_*5lp^N>dy@LN?nD_p60k5d^Y>Z$|$vX||{d9)II z1*`02s$F8!IB{p*n5P_?l8LLi%9}X^Z+{fm#IU;vHEp3K)&@57<~?2zF#AciwF=_B z*N4F<6YKbWWBeIhx*h$o+$=fZJyFQ@rF?m4o$#w*7WoZT=-uTHGjoP|r=-UlEyj!k zg}1eKUqnYP*{9!KwCn<1;_*&!eCRU%v~AQ__wNh&(~08C+&^iy{oYgE#)*cXZ1{{4 zo4i?9SJwXyw9b0Gt!bIFqEu0r=b;|ly1QsHweA`lEV~*wBYkr2^&a;yv<`vj<^2EO z)h|^i4ndNQnm$JsDMRfXM;xhNO^;)pBzO!V^T#hZ2MA=%2?B|N{<~uD43o@{8FHwR N(vB&E$5sB^{2v8;i9!GX diff --git a/summary_results.json b/summary_results.json index 3fa2bd743..37ce1b8b8 100644 --- a/summary_results.json +++ b/summary_results.json @@ -127,7 +127,7 @@ "a#": "", "Location": "open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline", "Software": "onnxruntime", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.376622, + "Performance_Result": 0.419959, "Performance_Units": "Samples/s" }, { From ca25aa35fa3e9502f8ddd5cde1cdd6f788d5c295 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 3 Jun 2025 13:54:14 +0000 Subject: [PATCH 682/719] Updated results summary [skip ci] --- README.md | 38 +++++++++++++++++++------------------- dbversion | 2 +- summary.csv | 12 ++++++------ summary.json | 14 +++++++------- summary.xlsx | Bin 10932 -> 10936 bytes summary_results.json | 14 +++++++------- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 8c5f83cee..ea9b8bddd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-05-30 12:24:15,750 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-05-30 12:24:15,750 submission_checker1.py:3308 INFO] --- -[2025-05-30 12:24:15,750 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-03 13:54:08,389 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-03 13:54:08,389 submission_checker1.py:3308 INFO] --- +[2025-06-03 13:54:08,389 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-05-30 12:24:15,750 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-03 13:54:08,389 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-05-30 12:24:15,750 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-03 13:54:08,389 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-05-30 12:24:15,750 submission_checker1.py:3324 INFO] --- -[2025-05-30 12:24:15,750 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-05-30 12:24:15,750 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-05-30 12:24:15,750 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-05-30 12:24:15,750 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-05-30 12:24:15,750 submission_checker1.py:3345 INFO] --- -[2025-05-30 12:24:15,750 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-05-30 12:24:17,008 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-03 13:54:08,389 submission_checker1.py:3324 INFO] --- +[2025-06-03 13:54:08,389 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-03 13:54:08,389 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-03 13:54:08,390 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-03 13:54:08,390 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-03 13:54:08,390 submission_checker1.py:3345 INFO] --- +[2025-06-03 13:54:08,390 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-03 13:54:09,484 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,29 +25,29 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8854 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9644 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70993 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70646 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 14.9346 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.3193 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.53809 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.26503 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419959 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422931 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8191 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 07f7d5ff3..6f0c51cf6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1309 +1310 diff --git a/summary.csv b/summary.csv index 69fc747fc..3c04c0daa 100644 --- a/summary.csv +++ b/summary.csv @@ -3,29 +3,29 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8854,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9644,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70993,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70646,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,14.9346,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.3193,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.53809,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.26503,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419959,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422931,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8191,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2d0a9c92f..5287ddfa8 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8854, + "Result":22.9644, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70993, + "Result":1.70646, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":14.9346, + "Result":19.3193, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -519,7 +519,7 @@ "Model":"bert-99", "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":6.53809, + "Result":4.48923, "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419959, + "Result":0.422931, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8191, + "Result":20.988, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -799,7 +799,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 5e40726af900864579896fa20fc109e44c1081a3..e71b8f115b911c5b4b94a67ed73f1aebe266a593 100644 GIT binary patch delta 2766 zcmZ9Odpr~BAICRy9Ws+j($K77q&1W4+&VMNwVUIIVw3gT5GfHG3QHU|R;Xigk5jBj z7q@cV2*b#Vj!WvO-YTkJb!$j&-4C%KHt~#_(c7Q>Iiv-x0I8D3(5e1 z%%fO&Du^BxV)*)Q@6*LB%lR|08CkZjlRFPIYcZX^Iypw*btn2)`DSPEe3{xXrD-kH zsR%<0Ls1&6Lseus;r!=!7QyR4=?{*pOIrHu%ZFKA`QllB9NjrG(>lrLa#T25)%-gP zvAFqV`EKYHqgig#@zILrKrNSw=84#*=6h<7t!VV8``Q>qd$ey%0GIY1NU(}LEA0VS zW$H4Wb`l)#?b+i2pN1ZDxFRP|ZEI%eia<9K9J2;fiQkQKIlBbU+aKq%?`CY}ejB;5CpP(_nipV*f3l2x5@+Q< zE-B)zqv`yKqYp-9{+RHOcS9#cD;2PmAYEuEfs^IRy2(0p@5N~Ho}9;W%#w^2JIH=sh=91Of@h+M8R4GKDnOCoG7n!F{`mAK$IW#eAIRxhQ%vS>c zB41aRk9(3k@}YAyBC-10LZ9*NAIq!fzyV>G{S*+vQusuI{}VYAf^0Kf1E6-K-Z#`? zr>hW@NcmV;G!?6KZu`os3ZQffuD`{kdx?25lQF;aO^RW}Y&feO4tvzFwfH|yI4<&YM=fOcCEF~^eBe4`)NQkk;C|h5wGqd69 z`|4VNx%z@<#Y+BX$bMpuqxY6vUsX{03IE7pNL5RwFI&PrS&aC!_X%+Z#|Zva&*Hf__&q(Oi7H0bc>I!kf z9!{dNl&T5nW`s9|K^$&H9HB9Y4k~xQq^nT*oYV}cGnVXgE9dVzrp{aJJNlk{cXOer zrrlXw>MPlEc0VJBaMmorZ4h^`6=j93*bJs}EBL-V7+*5isjai&)XA50WE2n@Vu2QwyvtgD%csR#97EQOM32Xtq! zOm7OALDa3)&PNQVEHkKi7h!tjJ4Zs8BS^0gRw+DVqa+%;9#>5hp;oTySJ z#wR@2yZ)q&8lYmg;7B{Q>#FX~I5=I+dG?pUv0nlk63no~uPR@FeV9);XTG__L|LBA z^9lEjD--?L-LT6&dDaE^&YkVO!wyf&zV_yf=Fs;IU;e=gSoAonkb}&L_S(|P_1M6{ zla8M>VVssNCnYO;P*{u*726U5$m>1O&ZYIYga8_YXRPmlA`H`0ryJkcmYm*je;MIU z;~J`e4W!7~eGEs{w$%)lv!jJly15>|+rmMoc{f{GWH<+2G&M(P5F!*)({<^mS_kd$ zNz&3rs`;fqX!jzKvqJQhn`9kd*9H4wD$ocQTsAnJ?9xJtG4s(PEviv3@dpCj$wrlx z5s5>`Ku)z+Wdw$5wTgmHp86da z2Z$la_F?pKRUq2|(vJ|6c&=XHE#-dc$&~EB9}u3k`#lPtocYduTE_3qF*d-hKlqe# zy~+2HAegyW)W3heWYGal=d*;8a{21Tl5*Hw9_V3DUNH;{Zl*-UV~*k^LxtjeVi*5e zmC`CPP`0Jf=h4#Qa=6X3)AO=_dP|J&o1%jTTV+;T3eC;OiR#q0N9!b z6*MD%nM8;9lj=c~Cog8(HHV0@xkS)jf*j9nKLK2V`c9jc7Xsy~ z*JVCia@?VB{_7K-nSY0XX{~x@N7nL3~S3_P-#nIh}+^BAQ7$udGc$OE@x$y8v`v3m}n`O7xEJ>^9GiqG)LQv;cUk zZvms)8a4`Daq4s`URR$<-EA{XgZ73^3`AS`V>SqvYDL$@qGkNJ-~CrAv+m5;)N`-& z23n9??3IT3jU!)#{Y#}aYmq-GSH}MSG&jn(oK-n=YqCBoKDVu-XKfMnJZYg8tC`gM zioQHEWg5KbiH4N~h>B)lgHQIYx*@vWcMKJC2KoyGYaOr+TyzRd-`^HUUHDPQF5wSY{ zuD!v|s<%lH|5lj|Z`eXK!Nu+-R_lt7p+Faa?&ehf(`sKAO?OOFR-4g~UJi|B*jG;Arow!*%OweYu zR&*?P$6G#+Fdwf(tjo-l^sDoaBtrY%%2zy5-#Bd~l*?Xgetagq-=h!|jd?id9I8(@ z+c9>h=6#}cfn(-Mwix`yzIta+?`7`o-g^%C3qf}RbvU7EMw8=I#c2CZB|~}F>gI4c z*Dp?v_~Q2mgq!uHa`G94KpBhZJLk-8VnIiz!>~*I>)u`A)!_p%Q2gqcpns%}Cfe0t z`k{9|W_@GqVV?85yEj1hU0U=nHz!^oL@^JgMb{^vP@r^WOJ!y>(}+&sGFWN?m%qSN(T6 z5C~QkDM^I_F9-D3EEhuNqb3}V59j)z6$D3KNe0UNd7+cqyBAw9)B;n-)qQo-zb^T z*Hp-B1R|r0f4$gaHgg~&T(`o#(?z%9=Z6CoGh+IVz^?{22R~`U5(^?gCpv9W{TF@# zD-#I?wwj2Oh%{{&OquwcFzEE5HJEVFFh6mH?u~LrAMzH&*mH_raSh@Ccybu72$GD9 z%b^X1S1+XRwXyE=PQ*zk?|6Z8&)f#xLTh@U6?vMc(lP3+r&iG0g%{1`B7Gh$+<;^l z10xERiWApdRl<`_DeCdT~_- zTTKX6u4Lv}dt{~a+>jr+PilTX4|R42T4Fw}RTxiE zgIL^h`|eNT=Z0!%M3WT&)N3~6K6&VEZ+mUOfv0fARVsT!U+|ms$E46`9@Y=y$W2lu zGs$!k18mCsr?HDe^JXpsbgJ6*8S0fNuQRoT&~yu~PEUQg>@$+=Sgrpyd)U>{Y;dTu z!%oQY95rpryz(4#U_*OLeOn|mry-WWYy>aZn;eX%Fl6(0r20v+Qlv6Kp0_uBGp%?F%fgrq#cq z324_I{(93dshRC0yHwa>Z4_$~VGA29|Fl>ko6p0kTL=o5D%2H7Odi`}GBz_Y1wsMh zdab*IglNz4T94v^zh4ZDy0zJS(=|7%E;p)i_;$p62%4>(7~%3=GYz0dzd&++vJ7Y=As3*I4Nv3{c>74sqMrrEiCM8}9)KVdE$@7|~(i;KL^f02TH?jT{ZnYPH>AvCvh|I_UsAoyg zZu?`BRtY{O`YA=Ha@v_H3L+dtQ?*6Zs!ZJeO0+5laGcFB^{}8SuQh0L z$>bHK98t3}c|JIq5pTxsdOsP;+QDN*28S<5V+9SoSyXnO*V$qIB}ZTVva;Z~zfMAJ zYohdU16+tq5%aTx9YmpyX`na>IC34FeuxS5HIyb59qsUFCA%*Ph%`&d`}rHGM#sCj&2vzR5mrXtg7s9c8Q!gaW@F+&*YzUc7$Fiv zVc>cDi;xVdqGfBNL_hH_3BDjv2~0?R@Hx;{3lQwL#&-RlEde%Uy!b z;r$%++OV*1H`r4k)dAOO^JIqJY4Gdq=wG!Aam>im^K^~*?kRxF0MGogdJ)q$>=U2u zr*~*YfCy)*XDh-{jeI38W*;KrsmgT0#0DiOCLdEENH4{ui^4yj-7Xs!UFG*>dTuPL zJ1yMtuRMECSAZKW;Kz*gSYW2^p8B&rYYNWOwlBq_Z&1%U0iw*3Kg(D6y8y%5Egas) zpx2g#J)u~N8n-6O|9H9g)(opR4_IiMYFfviU&nWg7SiwUFk5%;XeJ%lL^7N+mFc#U zpUA3GT0OQ$Ms9!{glJHuGx~uj1g4S}1BgohxK4Rr&-2*B4v4rrf)85T(+h}^Ko4@p z9f3iB#mKvt`Cm7c8VFGCpw?0mj9}EwP*;%B$li4vl?gT_o-!Y+CaD1mI}hS-VM-AZ z(=1zj5=srq7(#4=wU@AZwGXmhs~M?3SH|^ucT!4Qcu?PM&PdlDhuRa3xOCnE&Bm30 z(-d4hETP`2`3$ls`r_?oqUfs({OFL-ZH0Je$sG^Zo?{rbbCbng-CLS`*3fA$GFQ&N zM4(a4%g{5)03uady#kFSXkt(z%he*wmE~UGN?9-7hT4%XmI=yKI>?_-rw-8bG2Vi7 zW81-d4^)|N< zM6gOEi-Qw=K}60Fh=GMB-htS%WKnx$fq&bS@cO|{o4|}+vZ%;@xqLas7&CR$ZK*Rn zJWW=)F=50Y>a`G^`?0-vDWZ|G%={YsSnt{6x&EdG*97DcQfXD8zr&sr@4R*RTjcmk z!n-2&7n0A?^!oVs#x}vnnv6e&kQWU)ukPP8x^*{uY1$(kK6#5~3d>6X=HT6ducz2# zbGznpn|^%Bm80`Hrx~qx%ZU2LY<$+zUHmezC=bj;;x|VJ@P|_NjPMuOBeUn&6PGId zbN2RLKfHA9N{lhfKy5tYp8lJFwcd`X&B4gBrvh^AdN<9~9P7_IJ>T4LCo(=F7s%Rd z+=C@v-zLQnYmt{DzV8-uvWJQ>4RzG2bX4l)#V-i|3BfRD(CR=>K%1e$$<$>C&KBT^&SJ^_ z25*`O3oFDb)U8ElQTHVpp?V$`FL zxGMbd&TiJ)e-#*Z`axQy|83%VSt>VpR>vIbW2`txpP*W?kit+oL@>)U_x^4GU z3BC7J>*5PXix#z@RYT4XMs2jPKEhgJ!%Vpba1;ORz?JuA4TDEbLTK}rQc01b$l_pSQwVc+{h)D zJ^i~YdrQVi-g=9)7aTn=badJTSLtHAA^-g^nVZ^fU`gg#o#PP3Zf50f6=sQ!t{kuz z1d5;@2noGL^$lTa=qfXpb=1I1Ot>y~&;REQ27#O)Adn*HzeA7ZGo5tJ!C2;9-Tkm( It(}|y0hG@To&W#< diff --git a/summary_results.json b/summary_results.json index 37ce1b8b8..745bec904 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419959, + "Performance_Result": 0.422931, "Performance_Units": "Samples/s" }, { @@ -284,7 +284,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99", "ColorKey": "availableMLCommons", - "Performance_Result": 6.53809, + "Performance_Result": 4.48923, "Performance_Units": "Samples/s" }, { @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 14.9346, + "Performance_Result": 19.3193, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70993, + "Performance_Result": 1.70646, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.8854, + "Performance_Result": 22.9644, "Performance_Units": "Samples/s" }, { @@ -1135,7 +1135,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline", "Software": "tf v2.19.0", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.8191, + "Performance_Result": 20.988, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 520e76f91e04a2aa6bd7d1120b2b378d46d5dba0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 3 Jun 2025 14:40:35 +0000 Subject: [PATCH 683/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10936 -> 10935 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ea9b8bddd..5911064de 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-03 13:54:08,389 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-03 13:54:08,389 submission_checker1.py:3308 INFO] --- -[2025-06-03 13:54:08,389 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-03 14:40:29,356 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-03 14:40:29,356 submission_checker1.py:3308 INFO] --- +[2025-06-03 14:40:29,356 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-03 13:54:08,389 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-03 14:40:29,356 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-03 13:54:08,389 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-03 14:40:29,356 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-03 13:54:08,389 submission_checker1.py:3324 INFO] --- -[2025-06-03 13:54:08,389 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-03 13:54:08,389 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-03 13:54:08,390 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-03 13:54:08,390 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-03 13:54:08,390 submission_checker1.py:3345 INFO] --- -[2025-06-03 13:54:08,390 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-03 13:54:09,484 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-03 14:40:29,356 submission_checker1.py:3324 INFO] --- +[2025-06-03 14:40:29,356 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-03 14:40:29,356 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-03 14:40:29,356 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-03 14:40:29,356 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-03 14:40:29,356 submission_checker1.py:3345 INFO] --- +[2025-06-03 14:40:29,356 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-03 14:40:30,570 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9644 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70646 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6f0c51cf6..8381dd26b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1310 +1311 diff --git a/summary.csv b/summary.csv index 3c04c0daa..b9ff7b9d8 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9644,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70646,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 5287ddfa8..66a1c40db 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9644, + "Result":23.1965, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index e71b8f115b911c5b4b94a67ed73f1aebe266a593..45683e0e51fb0aa7127db04c47c08b6f0c8406c0 100644 GIT binary patch delta 1647 zcmV-#29Wub`z7cb_lWve;;jUKXuCNAFWHUM2*Cg_7BN~(*7YC#Il0@ z!!m#+vuXfKll{X+0W3}O&-TU#EJ6OE(*!I*{vl;>f@NL93AV6+e<_yxYX0b5^T)~V z8?WpPYnO0|28k!lACd{B`9m_uWd-wxWq?bP@ga4c-s5L}U>3f7qd{ zOUl4wXI+EGu50#w5X)~ave&Z2Dk8fsYCm2}2ewtlrBlKs`Xipyen=*i+7HPfmld=h zmH{qF2Dmg?Kdb}b(j@(8kba;gB7>kS3uRqahNoC!2Ei$|X6ffKF29K^TgxIVon(vF zkJhcg2}<4~(QXNuD35qj`XQN6e@Z_jgJf2aepm*`BpD#nfJ~F}qe1zBmWT{O z$OPqwl;J^^{aqh8$kwd=CR)cLv6Mdv4fLDmc1l3&ZzUJ07yPx46Vhh#!2{g4cz zSwZ??89nE2sX`{m7lLgfAdMIY%Pnd zv}qQdA8og^6PDP_|o|y8D&Z7 zhlG&L3epeD0GmwNf!AhDf6@;d1>iI}KN_4LXo=`Fr78<$T~>zsCNYCx->g~s`Fc2? za-`i;_2*I-)t_wNw6tILGEj;6qfsI8rTIfL%98322_c&m)E|}sHkr5sY?{;`HVUw5 zGJiCfKhP4?sM{*RXakMg~er<-<(fB$2nNa9QXhh&r` z1t1avI9W!*$ubg7HX(r1HVHs%6bYxPgN_XX5VS;Wn&7cRS(lc{jgy36jFerQf#4gc*Z1hQdsR5CUvZMz@Lda$X4~S)eO_GsMIBk;##5w>p zO&X958W6NZWDxM!f1#?&s^qrG?hx2E?Y*;rzOBe!%lfM9n08SD%63f4)w+zz5(~&i zpTw6I5XmS@YCt4}R94V{SO%yh8KBZ+0kIB%N|Ob&z43ucuz=WQ1CJfbx}*%TtYQHb z7O?MFK1pP+Wqp;f1PRD5{lJu>1f+roEXfCvNC}8!lqD@7e-c70D_B4*16VS@1`e_& z4Ty~bT$(f>8#EwjiP$vdhhCgRS(lc5V#_P|tOGne2#^TTQr{#Hvr@w0}@2o$(T#n;L`eha$ zH{;K<><-G;+H4Fy57(cQ*O<*m$$XT4S&yE7P~URge><5ZgCAFuT24oQF5@46=>Omy z{?PY+`njQhY8*GRmET9R)vZR!zhKikIhyru;`848W;9(i#_=#)sMYeDjrsbM_f}u0 z$!ztN+iZMH(lnXee|H=A2Jt-qF8gseOp^5eoBWb&Kf&tl9U|O)w zXsFR1JX$n&6}OZd2d~apa6y7kB1n76ZH=}eScf+4v%VaxF@KIfS1)hFs;Sh6Q*l5|B{<~{STL@s@O2Rn*D)Vf@x&Vm;&C7K ziMP`t$375A3_y=@&&o&n^t@gqqNX6leM;sjDcAXo&Im2Z*J0#M{{^T{L;g)X1CF_G tKSA>cvnnRa0tC!<6O*&}C_MoMPotceC6hWTQ~@cIk18Vu+a~}3001^w6@dT% delta 1644 zcmV-y29x=>Rk&5KatH+6X;#a#b_lWvf1m70bLcr`d!;VH5;YP}+CL-{O8bXo5X%bo z56b|S%&Gw_P4*8P1+X;9KieB0umt&sP7|;M`G=Ij36^yYC)mOQ{-s#%tNEjM%^xSb zZ@e-t?Gi50An~O6Lo%T>e@F(otYH4I3~)&@zQocb{;&=JOOyAb!TW)hhzx=re>;?Q zNf~(TtZVStbNismD$@yU&0GTG`M}zVMEfE=n zkO|5UDZ_&-`@24HkgZwy`95Ussq=HCi_TB}f~*(1CBK%by%IFZpX8C!56Og5`XL!a zvx4-)GJqz@0GcM}hjjpGnv@?6$`7_$Ne0j~DLvuhx~vTMO=1SYzFD*M^Yw5( zd&Pvsz2GjX=%UgWuOxCN25aGOY?_hlqJ<45<)gBs6Q+NY%*~N*fgm>Y!qPA zWd3L{f1o9z(*!ob{2^tyaaJ*Zbj{|^_hEBi{U0s;ALV-`PdDunfB(luk;Iq&56LJ? z3P2 zC-J2PL^8^f8W0H~l@&A~mH{eB2B>+)}>`|kgf3Nf9yTifYM=(qaonMRokG8V& zt1fv_)^F?UgJL?ZGm*cpRewoF` z&G_>yyMywzHXDP_!}aInHD>csG9RU1)}!Yi)VCb>e@-UJ;K$XZmebLn%lOA1`agJw zKlHtyes1WW8pn-n<@eESb*oYGFW9tBj%K}^_`Em28BG_BaXicxYPI}kW4`|6z17!g zGFyG+HX9$4G)*S=-`&Q&K|IgD%YNJqlO(r?N^U7P4&I!x;DQ97M3DBB+Y0SMuoPp@cc7ThqC-bTM diff --git a/summary_results.json b/summary_results.json index 745bec904..cc6303e25 100644 --- a/summary_results.json +++ b/summary_results.json @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9644, + "Performance_Result": 23.1965, "Performance_Units": "Samples/s" }, { From c27f18aa60b5736b546ed6c1cae353e927b25b37 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 3 Jun 2025 15:56:31 +0000 Subject: [PATCH 684/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10935 -> 10927 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 5911064de..c63e420b0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-03 14:40:29,356 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-03 14:40:29,356 submission_checker1.py:3308 INFO] --- -[2025-06-03 14:40:29,356 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-03 15:56:25,766 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-03 15:56:25,766 submission_checker1.py:3308 INFO] --- +[2025-06-03 15:56:25,766 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-03 14:40:29,356 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-03 15:56:25,766 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-03 14:40:29,356 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-03 15:56:25,766 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-03 14:40:29,356 submission_checker1.py:3324 INFO] --- -[2025-06-03 14:40:29,356 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-03 14:40:29,356 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-03 14:40:29,356 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-03 14:40:29,356 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-03 14:40:29,356 submission_checker1.py:3345 INFO] --- -[2025-06-03 14:40:29,356 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-03 14:40:30,570 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-03 15:56:25,766 submission_checker1.py:3324 INFO] --- +[2025-06-03 15:56:25,766 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-03 15:56:25,766 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-03 15:56:25,766 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-03 15:56:25,766 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-03 15:56:25,766 submission_checker1.py:3345 INFO] --- +[2025-06-03 15:56:25,766 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-03 15:56:26,823 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 19.3193 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 15.0091 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -41,11 +41,11 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 8.26503 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.22828 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422931 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419428 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8381dd26b..738ca595b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1311 +1312 diff --git a/summary.csv b/summary.csv index b9ff7b9d8..f7dad1850 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,19.3193,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,15.0091,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -19,11 +19,11 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,8.26503,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.22828,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422931,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419428,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 66a1c40db..eb5b361c1 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":19.3193, + "Result":15.0091, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":8.26503, + "Result":6.22828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422931, + "Result":0.419428, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 45683e0e51fb0aa7127db04c47c08b6f0c8406c0..45a76722dbe02c57b09867db278244c3de33ffd3 100644 GIT binary patch delta 3767 zcmZ8kc{tSF-yYi_V;f6(kZp{0s39$6tixa|6&aG!5H(}3Eaf{?mc(R6qwE^XQ+bN9 zMY3dwLAH>!C%c3Uk^P;z-uHTc*YEssuIudg`P`rTKG&&oe(hWmEO3C5hnTu13<7Nj zQf~{O0jNEDS&OUX$7t2S1GOaINXqB--S2V#vULK7fhqW3nsd zIS_PT#^lWQEnU$VAC=CCKGT3qyUEF!v=hX=V0Jy-p>2n}W{_Nl9&gk??LY1_^u1)v ze!FTVKdq92GwKYGfv;k8+v~Hwl=lr8EeX>AOKOn2J8dv~4c#~MBCaX?u`%Uq;3u*t zY_}Pf?eo%pX5~%Vbe}_jRatp>u>PUxDyap(j>)eBfKP#YDib59_ z(50zl4BKj@Govx6PxBeEweEubq}TW%X4AM=rUyLd;s<)hh;}`}->KK3WuE<|1)mCd z+Kv(IB67$XH_KN1^jrNClVz*ujfy%tp=CXu`8ed}of}lQTP2##<25e0YiH#vQ?G;% zBbLazlVKix7n5o-zb0NBFg%&Amh;!@w7`7v`BA-Zjte`CwZwrh4`#gl7jHC8-!WJ5 zvQcVdj5utA0!KY6bqZ!bes+%Am8TQK`ib5`AW3sZiz_E=7*$W~diDKaZ6TelDwm3hjw=+$;x=n0_4Z*NqNi3`xT z&p^pn#NnIv{z+Dy{xEPNXP8}23HSJCrobicA&qWk#h+=I6EdrEYoc*oP9^$Diw2rx z2u&GHiJwZgVA7W#2Q(z73SIkhh-K*dYRVe283|JRYb6B?bSdjQ0zzrPV{KlR;~*0EhRc&lc*qWJEfnO_li;tlNr?!@I?gmhN! zv|szQI#{k5ZyToI_ZgI8B1{Z~Plp;DH!jg|S3YH< zIF#z`;SxInj}06t-*+OsP|NaNaJ^DM>>D39XrcvFtLkp3?|t0UyC3l6ZL|foc5%s8 z|L9#`3M3bUB*A?I(v$xeC-C1u-w-LUf<{93czg#+TWm%O%ia>f$?NTvozR1pc^CeT zrC>XFqd72Jf5U_E5RlQ}&!JcXvL5_df316H^jAY6Az@BE4}X?+11IC3ap*m4aXl$`b)5FvK`K_x$1`$TM8y=w(hIs|`tonDNV}71QB> zRdfRnLR_A&n>Q{l^=l=jriEB$pEMhpSN-}Tc~~kk)EDP%mNk4dvY6k%NoXa1*c_P` z8fgt4D!#N)0x7|qlg9Vqf0Z15gr!c=^J7B52yt0U&CFTHfdcOmB6N};sO(I?k^LHf z7Bz4EwkW%+U|9MNo7l&A&miiln@SuLcX(k8ldu_#!;5tus1$7nrFXoEK5l?*5nbf! z*2kc1G*4dbhNd_*e2;#i^D@SR+M#f(Gkr1Js#P+(YtX#9sHN-iuyd5$;H@2s=aS=P z!N-47!p>|dJ#}>c(FrF#;)SPbHh`)IpcNm{2QGCFCYvFmZ_YG(Jl86z|8N?ShYt8i^n_LSj zlL{7VyVna+mbu4>;bI~|s(RPU@AcwArA{ClDk$F+6hn)*Kl_5=2yoOQ;LXQnJ*W*@ zJhFM~7o$SE1WFrOCs(fKobp~L52(66Lr-9~K9G7S{mF;YR>-eo$7 z2H(6IGRZ?z-0$||=G2dnCmI>AzVg1UAORY*?Q45;9;+YPmStamhg?q!OaLdSbt+H< z3pUM>t=@dAaa6K65*Qzds-=k#UOu!fx@Ws)W%PXak)3*l0>xBE%NM21bAe+vevBwr z{)Pg~R*&MYiLC|L^t?2yrWoQ-%7nz6C3N<7UAYh?fvcohx}0kBcSpHZajU6sN=Yoz z9wo5j)e3$exn_>E_Tih4^CsC*xRBV7q6?r_eGSxKsepoSLaX9EJI5$%nZx)Y&(-L= zgico1hEj+_R*u%Y#T`lA)`&cJc2QpW`55@7xzQiiZ8)s^x;)~L-pw(MzNwuO5fGNj zrkid?)8Zj9h1#jVRg=yXiouroOwV5LxlPe@Z&%hLrjL9iSGo^ttR4tWVAGxCSAYVx zIg&;BC@QFwE|sZ3 z>+!ft)B}$`(M1Z%C)YMpA!$8cH%o!yL`#jb78oj4+~r88Emb~@H~`x%aRS*>SLuzv z@%`bRL;afosGb0@-kDyI^Ez%-!)o}nGobUca*Mj0L0fOX2-Z9b=xf62^Z{!$?#!-| z622^EtBGT161PZL>Q)n)%K$q=2CwqFDWAe#lIT!46DxaFYSn6xuy~CT$Ol=Nfz;ZqPvFq++u*!`z%ZPPBY>~^H@ZvzWF}U^=;rV2 zuMXnac;>qCCJIPvz1Xh&lGCUJr@lr&SmUTtTYW{@J5bMSCNR48JXeYs1J}soC3A_2 zpOUa2$>%ElO$Qi%7t9fq46o76zrQgwtBDLl-b!rbvUYBO7Gsc6r)gOD4gqjY8x~Gq z^FVti@ZsqW6ShL}_*B(nicm96EngFDo^uq4qaTCh%5PH8#yvdx%S6@H}rg`zxKZKxP(`*Rd@GEDpAmyPt=t{R!D)0wW-^>nZWM3x|EPy!6o=8 zVL3eZFELrZH~M}i2EHbUZfL0=i(GRej-i9NdzWw@wSQ%~8k{OD1nZ~nS??8sdJJ#Ko9bd@saR+Jgws)JibZzUohB)^G>OJLOzV~?&pv{H7rUZr)4r| zcusAz-%ZNj?Yu(Rt>+@BVq@7QrEJkOJAV3v=+H*XO9w_f=OcjLn*FB7u5If3QsEpr zpm0&AB}io(nMlr9nZ-r%OI;@#fOFzrI()d^F+;~21FX7Yw61Y>z|o~FJ4z$D9fK&O zqYkc+QER{t5nwJ7wL994I+iTE$y%mwexuPPJxlzu7^GY1sQc5yjHQ} zbER&p_TBU#4(T2>Z6K>a)wYFid>LzbXKJKd*A9#dQ(>i-MLyRaD3XDWo;`>eVG!SEgew27OQCNlj zUL9Lc5dV03Zv14(RmbX6Ech1B!aWx$7DduB_sNXeqSjie#|N@{aJHts>_53?YJtAa zsMHRzLPRZ&*bu>`b3k!gKu_vaSb#uFvb@QeG~T!a6ZYY- zh=Je1cgLo>ehkk1eCK&g`h}qs&$+Zrai8c2;sVrD`fH%@jVKBGT8V}nd(-+A10CcV z7ugJiWM-vyHLtecge845>P!oAW6pbJAlev;;VWZFwck)Y(o{CsVN$!!jW^!=AR_^O`;m!L@v!GRwSf^h!nrjXNfdP-aIKgC5+^@yP7!Gv6Zx zRqfTQ0w*vuYYj=l)cj2G7i_2m-|%GfOxLpKxws9UFWAk@1bFsAYb(-Q+b&bV*QQMI z?Q4EVa+S{V_nq!czvfb}3KOTkRU?(<+>?C-bmH!v=#S-@)8~I$EiP4WZcwM!zlZuP zqdMlo3{E?mgm0}_E){M&O|8NFWX-2YF+0`Y0E2>p+`iiv<@?SQ-Q=gPoK#l**5F2j z(wB{=QsdergVB)pKC>+G(UU#>{@$%OB(<&F8nXm9nnU1gD?^?8CL&;q_}p~u&?5fP zyH(Z}(QZY_uqj<)P%>9=32|@SzSR8sUhl{Kt2!}xgt&%jG}7=!r8pFV`;%_?^h=L9 zAS|TQ0ug#Ya#vU=IbQ<`Q4WhW?EDiLBc$}k#}GpQ`HZov$w^q~`i@e3T-hUi#?qlN zgwVzpI z{b-xzjfDp`h-C3kG9E)yPd)hSbX|6C*F+s+gHtR@^5!XCM5z*f_0q*~Kus#ihltSN z@SBiK6z|JRnlPxod1Tv-ufBEnDaw1R4Ik|EdcKGn`y`|Aq37BD+=0f8)QfAbkCIaL zEM`Pcw@byFu9zD7n?UG8J7f)46EfRdU*0(>#q0iOwa-!$z}-UBi*&PR`*@`F>EeS7 z3-g4;#Say!q@2k`X9Qpd+)vSsvJZTCk07MAkB4m>E^p=f1S>7(w_TkmZFiG0FKwqk zYHNQkQmf@pmV}T2EiN@p^xo2 z2$kh=gkkDv9BN-FJ41}p#CfPIN3BZZmJ8<$s56=zYPIn{$)}Y7$W+V8g)qqI^jNAg zE8^qaQw^aB$FF1rdA4%%ox_Lh!j}>YT1B|8GMwu)PMN=AKPqdqp55)(tGT{wZl`jO z;Eb?8`Q%aAsY@JF^_Oy}Rn6Rj&M|D9Q5oAe%ws2yG!w~m%{LQl$#my%OSOm~-aI_7 zAW2OanUQLNe*)lDo;Od#98a(1h%YLQ?ceTF${G|Ny3%PG&W~}jhPi}=Qktmr`C+4* zp#6yXj{dAdt+*Nc?Ubh~^u>!~+|_++WuVuX{~Y5CtIr<)-ZSAJS-HACsGN1MwR4-> z#W&E23+BOW5{Y!K=QxLCZ-<^>5;@=C=wl#iG0YO_wEz?p&OYM3{rAXFu42W-YPInT zNY1*x56s&_y&s-s6G~ragfJ5ymRFI#H)dx|H1mGp4y~B|^n(8DvaZeVuB1gITven& z7}YCHgtBL>8+TAVc)ve7XV)|2VA{R=k>PO3TE$!B(L;S^DZo_HpP5S3>hZ0(JtXjuhxA zF8HevhdRdG(1Z(`Q(aCW-<#|d^vT&qJVtH35b>6}TVCru@zMJ{b~nRu{X;VS*BZj+ zH?>jH6i$*{Z8a1*9|t2b{f*TvCjUnkFbLI^CdQo7o|FU+rYQ+Yu??m%bdQ;Y{g9|f z%_DFg{{Xe$O?rIwY{O0}R#}VQVfT^y)M+7Ts;(M^#q(Ao)!Jb;UN{u36L#JkhcXbz zT}~9hk~89xgw513_L=G5>f&WStA7EG=b*KQsa0KOlf{mVd4_ls2Jp=IeUba4AevhT zDqn_;G%`ZS=miJ%5q0xSdPKVh zXaA*=^-~rxc!|Enix?FmY^^r`rrqvz-8gAV(l_rkj&SkDRV&LIC?aWbbFH8BitP5#ANoyH(J6mtqYh|>Y0 zjgs%Xe4F8|OxiJ4X{Q>d+K$=joBh=4b~8J%Ww7^c#6$8D__cYZVPYp?TYwZjd?<6G zvpVumhZ;_{<_?oN;ZKJ#ND;L@z{qAr{lkJ0R>mlUxfh9X>foxY!~yad`*#i(oHn{i zD};MOW`Ob?gtIwE-L0nN97RrM zpW|?VsBsdjt*97GgO{)hr+ObTJ+H!O1NjQ8lu%ylC6){JuCEozSO{t|+#PzE^c%3= zl{tYy&c{YGDy08!1~N*RmEj+|pygP6ilBL*h6NLU5 zf`b<+{1b2=rWysfPM6#7?S^RP`M>yLvo&umK9+}_uB~<1RjTMK<8HdU1 zE%=h0fcT?Zg{E}#qgeMeSvF~5W=(tJ4~(nAQe6fIjHq+zVMV!0N~4nam<4MMvBa-c z2KJd5q(nlr>rCwq>1g+z;kJXAJ!YmhLBgu8?ws7CqfX@^9Ci~KS_(>iQvHI%@G_$% z2*fd30zMV3=}lno{OgBA3ZDc6&|PqKyx*!inrd~u1)H;8a)Nw~BUqi4bAVx6D<<@{d1?nB;D)y@qTvsf+sPvjw5>2BGB zaHT!O1i8PWY1WAFO;j0>IX`bjEyCd3*;sm}?z4DCx&MYjcdi{&NyWc_Ry+Mpt4Hdz z#iM7|f`%5(d>V5Q_i?0w!vahB5#jJk zg;kV^iHWm@Pi%mn4PE#&OP25rdJ5>f({WBP`d1it;X9>ImIgv~EZ(Hjh|4w(AtJ+F zLT|%UMzw?Y6)fq{uQthL*?$58uDl&DRo-QrFB-Z)J|8JO_?feGQlh0I^9PGrsH;O| zzvs)LQ#uatz+?4kk{lKNcK7rn(Gb%|Wbec}8N0_-^Nv=%w}%P}8&oZ;1kGuH8|Amg zU66QmZ&hSFD-SOiS1J|15G@*NI02^dF}+uPkgvy)yrI}F?#oni;NFpcLo1#45f6b{ zquCVboYd9&tETm2xhjx^xUS;c@v3w**2Yf{C965dZ+GrZ>FPrv0h%h Date: Tue, 3 Jun 2025 16:49:50 +0000 Subject: [PATCH 685/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 4 ++-- summary.xlsx | Bin 10927 -> 10931 bytes summary_results.json | 4 ++-- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index c63e420b0..537ec3813 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-03 15:56:25,766 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-03 15:56:25,766 submission_checker1.py:3308 INFO] --- -[2025-06-03 15:56:25,766 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-03 16:49:45,225 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-03 16:49:45,225 submission_checker1.py:3308 INFO] --- +[2025-06-03 16:49:45,225 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-03 15:56:25,766 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-03 16:49:45,225 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-03 15:56:25,766 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-03 16:49:45,225 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-03 15:56:25,766 submission_checker1.py:3324 INFO] --- -[2025-06-03 15:56:25,766 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-03 15:56:25,766 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-03 15:56:25,766 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-03 15:56:25,766 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-03 15:56:25,766 submission_checker1.py:3345 INFO] --- -[2025-06-03 15:56:25,766 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-03 15:56:26,823 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-03 16:49:45,225 submission_checker1.py:3324 INFO] --- +[2025-06-03 16:49:45,225 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-03 16:49:45,225 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-03 16:49:45,225 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-03 16:49:45,225 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-03 16:49:45,225 submission_checker1.py:3345 INFO] --- +[2025-06-03 16:49:45,225 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-03 16:49:46,287 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9804 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9871 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70646 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | diff --git a/dbversion b/dbversion index 738ca595b..16e80cad8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1312 +1313 diff --git a/summary.csv b/summary.csv index f7dad1850..6e68888cd 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9804,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9871,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70646,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" diff --git a/summary.json b/summary.json index eb5b361c1..266846bf7 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9804, + "Result":22.9871, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -199,7 +199,7 @@ "accelerators_per_node":0, "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 45a76722dbe02c57b09867db278244c3de33ffd3..53a503b70facff4d1f3cd7ab27cc88dfa19aae10 100644 GIT binary patch delta 954 zcmV;r14aC=RkKyFatIFk=g<^v4FCZ9Z~y=p0F%Ke6|%Vso()i z^6?{50wNh@NehUC5X%Y{5X%6T%&&oitVsi6qX3sC4af!!2wEaGP5Dt5=TO$AWpI$K z@aKQ*J=cJ;VS%K2lK)f7+A5u2xBVY&<>%M3zc6b%B}jjwN#aZYhh&r`1t1baBr6C& zECWa~wFZzh89;0lK+csx!U{UpCytHMp|>I3gC)!~QLXTv?j^Wl=|U{w4e``&ojdK+qgG~=f!vVkGo-# zWcS|`mt^y~g(rUjvji5{4h1-NVn!yD;Uph_PQx$|MfZF~ZAghUXC zon=o`i})eikoNC!(ll+QtXPikoilev&esRoM89C2(5gULlA;I{(_E;!K=0KizDJSs zRB_s91qC{SL+kP`XNE9s;nivb)(db^04Yb9DbU_~Lojx150bhhI8+dK+Dhs}X=`kM zDEp>0;8~ikumn#z^%Qr&*o;IpAUPY6=EpWYfHMpYNKoG4EXnZ2>%mGl+0ZLjPAUA+ zz@+soj_!jKqp@wa{I%d|iaYHPrcSIPp#Z ze>?_#;%!*u)CW8Y0mv!tS^1bhJy#ngRL;`bGEP@P-(J&SO-9*$pYHk_&_CCK!5DA zfyWMIT~dZvR9Ar%z5E})!G-*IKXh6^sv1!VWx;TfjE-izDY=uAnWAC{Jlno0c)sy_6 zTGm$S{JQP`Xe&Rzmi>iU+bKa3O@9(!`adM2EGYny5F%MY0Ad+HlBqR-q{#qcqX3d7 z1IPvg2wEaGP5IF>=TO$AWq@P_1IXTU189w8_#`CvRs69XiKHa5z4h5{lX!j;pT^^H z+UO_w)mjy9T2~)q`!b7@sB_BfA9`}=zBl?+|WNYjvM*P@1yzZmZKD3uxXtf&3ZTSd2fC*nx>6$Jj@qr zwftsdzWThq<=0s8zUc+EK4T$-`&Q&K|C+M%YWPrlO((Urnn@V&n-Op3s6e~ z1d}T%8nYx8x(=lyb7xSwI?5*c1M7rV z70QwnMWC4GLhUQ`v0cZHC~}@EP8+SDLML!&RX>!>5T<`Eyjg9)dI2s9Ams=%6*_ot z2*!>bKvI_khYI3OTSo~8ws;3=n`;tm*_nTSRtXEW0L+NK9^hM@ro z$~&AT89sYGSm|yy^vZ=(3V$+i(>jT>`{=}MY}+>S{&nY?q?UM>BCL75D@*N}r002+|000pH0000000000004l2&XY?jJ^>+uBHG0 diff --git a/summary_results.json b/summary_results.json index 98143241b..667cb79d2 100644 --- a/summary_results.json +++ b/summary_results.json @@ -991,7 +991,7 @@ "a#": "", "Location": "open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline", "Software": "onnxruntime v1.22.0", - "operating_system": "Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "operating_system": "Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "Notes": "", "compliance": 1, "errors": 0, @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9804, + "Performance_Result": 22.9871, "Performance_Units": "Samples/s" }, { From a85df12269574ced8fcbef0a26166aa351e21a09 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 3 Jun 2025 17:17:33 +0000 Subject: [PATCH 686/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10931 -> 10931 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 537ec3813..6fb654015 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-03 16:49:45,225 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-03 16:49:45,225 submission_checker1.py:3308 INFO] --- -[2025-06-03 16:49:45,225 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-03 17:17:27,758 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-03 17:17:27,758 submission_checker1.py:3308 INFO] --- +[2025-06-03 17:17:27,758 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-03 16:49:45,225 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-03 17:17:27,758 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-03 16:49:45,225 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-03 17:17:27,758 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-03 16:49:45,225 submission_checker1.py:3324 INFO] --- -[2025-06-03 16:49:45,225 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-03 16:49:45,225 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-03 16:49:45,225 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-03 16:49:45,225 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-03 16:49:45,225 submission_checker1.py:3345 INFO] --- -[2025-06-03 16:49:45,225 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-03 16:49:46,287 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-03 17:17:27,758 submission_checker1.py:3324 INFO] --- +[2025-06-03 17:17:27,758 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-03 17:17:27,758 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-03 17:17:27,758 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-03 17:17:27,758 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-03 17:17:27,758 submission_checker1.py:3345 INFO] --- +[2025-06-03 17:17:27,758 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-03 17:17:28,776 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419428 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421303 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 16e80cad8..25a5cea61 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1313 +1314 diff --git a/summary.csv b/summary.csv index 6e68888cd..b68652823 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419428,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421303,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 266846bf7..42917d1c3 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419428, + "Result":0.421303, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 53a503b70facff4d1f3cd7ab27cc88dfa19aae10..dce9e2d2defe9af205ea5adf8eb31cd31e6f08cc 100644 GIT binary patch delta 2750 zcmZ9OeIV2O9>@3K`DtrXN6Msac^K8qdO#yiXvRvObBN8^*dj$LZ7a?h_vE3CVM~#$ zM4j1~(jgtEG;+#3L`YGIxe_+i zf5JC62i~+ALEKx{5gr_Rw&TS6?3=I~4x9B-l*jfyh)Ur|J_d18pA|;OV3JV9uX|5{ z1XPe8AG~#2#_87>ulp!BtKwfRckOGIC+PquJ}8QAmpR&JH1RW|C0VnXmPQ=s}d8JPv(OOzB>J==_zZmBAcv&${6p;<#%Szk_Ql7`yomR_Pq=@ zMA()&C|Syj33QwAifCJB&l1niE6f$?9S$e&aJ*+u%mdWcA!C&2HS0vwm27h_pbP*R zzs>XMaliix{a>A}EWz=dLCGhiyFAx9_&K3;nB|mI7o}hcZ`&BWfov~e#$fbN;$n~Q z^30_N2kT;77z$#Pglk)-!;6SF*Z? zUl4|cYa3~FVW^8i3Opd*1!d}!$fLqL4c?k`fJBdn!Fckl$$M}YUhQI_Wl7I|XT!zp zaJ_2OKvp>)5vRnvju>{HnLE{UtG~jt!_IWFCwCy$O`08^wYk@luU|RniF*{M&AH2? zamiO?R$@#mZ;s0pp^#{^i%#_HewzcYTzN7_hYXhB2cneKSMwGXY(Lmb;GR*2AVz4f)ro#7~jUd*o988-2)W|Av`?^ zimXanOLEcM!;7tQ$gSL4`j=o09H2q0&0F|1ln}eA7G;j+xNs&531RNc)oKfCiOH^Y zC@bLR_&3w=QqQm=p)wY0I1s~M+Zbm^pV>yNbnH81fRW!-&RRuWj1KBoMFr3ts;iUF z^$zOf$E$@QlD7CD2wH@?gy;HT3UuH~s{%74DyQ{HPWx=A}*C>>I8>PnY;Y~`Ff$NLCpM8I<*%5>jXp!ppJ z^RNz(@bL{WmOKx$(3awrM6m@@s-@=no388y_w|J-#+&n>#6mfT_~*CO;)JXY4MkxE(g(e|7c-6q1Vt%atcyt8q&@S|$E5uRXfAXk%s+RJNlEXZ{9yWi; zr*cVxxz=4EN3tqUX9DOEPOni$>7&uo{(+8zUyMl*vMtAE#gAn5SeeR~lRCKA)o8ZE zL-~6#y&==Dz}wY!2~b!GD8$+O01BymqkKj(J5<+Q;;Zf|)3}qobT(4Y&3d+fwCeaK zf~}_}1hboAC8sAfhbCC#*U~)t42Stdyh1z9x*(J(S9>8&>(3gp`pAkU5iC zTMCi<{lepHAHA-cml~;2nJ*4-xccx006XTLF#~m2b(fii`GI)V=mZeNDUq`UaYVP% zEr@;Vvtl;zgNRKp(rkl9j%w(2-suw@P5yb=)z6-LfGkd|L{ajgB|(mv;N?J)L>H09 ze*^mdV-@?>GlF6dX<{FCDc$X2z)sav)#9p=>^gp%7(rl9nm9FkIkv)Ql(fvF7NE7Q5Re^Vd;e785dMUX+-`zC5EuPxRT;V8easZSto@#q z96@IESlxnhS$)@Q9w;4ctggYiES+_jGGASinyo{v2pA<9X8}YvE5sgxeVRKK%H(s- zr~k{zP-HN=ft<=~Q81=UwLi0doDc5{p|#aDpv%(R>JHr` z<;qjDwZAhBSD4NK)6Sj_Fc0#LC=3}F@Jpo<@_guvJ37?;gY4{gNBw9j(t+v$!;Gx%!U2zZC5=nfhCX_6 z3`)+ZB1Ziy3%MZtnK7Y4a-5YLpBs>I}cB0CJShL$=YU3qX zk)LkGC~RQiVReA4V{f;3@8bBR)9fFu-6Qou$NP{!IX*nP=_~HHJn_tU7+cpYfx{x` zT;^2=v?V-#TS493-dwYH%kE2w+onqW`1kWVd>tee#I-Yd#Lt|MCY*PtiAzIIiO#te z>h|dh^`x+3b(EA*y1w(&_L-B3Y$uT<=cw!Bwej_uxY*{!Kbqb=3&TaQ-w-U20xSu$ zFf#nG$-{L^a|dV0Sb(~_EV+#m>pyXDRU1-BP%VMI?K(MXO`{116VU5|27^Ep2ne(a z1OibLV=2jrNyoOtCMMGy(^$+>erJ5^R(R(08H0+u4g=+ktqLehS#@?#WcVpl)x%Zw z?EIcxC%?Amj-g%#daPx{-7AeeSOb&gu{FfAuA%=YwEMzd+b6`6Q z^m8V*Rx4oV-0Fpvem^g~^-gjA+cpK1>ty8MLwWHE?>qD0{r!F0BevRv_}gl_T59I7 z{!F4;D2X?g_ol4l<$yB?+AjdlGyi%)Trl);v2f-f;MV3s^Jr z!{?HQqxpdwKH6wK3OM?ty=`>j&Wkd;O#^5AgxkV;COj?u6^oM*FM6W5$;{crQS+qs zGhg*@5ano0JLKP=PQhoJ2(YezV(SXNAjq-x0J8;8YMqB{r9_TxK07a*`utL|ZPu)$fn<`u#qCd|t25KhNuVKdg1+eJ*jE|0eKn4ym5r3Sn2Y3Q<4{&nlN#n0gvuLYq+p7jTqM$J93_m zr2hCvjq-+qhqRve?P?hUjXvMjwWLkNz(dHO*hGHA;CM-2+n~Y{lt){w1kNH}Rj(NA z^?o<~>%c~d#e8`cn|N9KNn+joVOj3T*_7z%19^c~((>6}ttQ?q*r0K+OEE){`SiGY zFau7(q5))B^o=*Ernd`~6@>z4x6XK-o?PSnf-f2e@qCJ8h0wc`S&wHfl%?X< zT&Jur&)GLL5j3r-CZbk22a9e-b^>=N{tP4tZK|*r9BkC~#o6rMUx^AZwDr&zWf239 zT@fR%M2l8yNa*LVVa%dH^yW)*F)96nX&&J)$U2ALa^~)jFUi7HT9q_w8(BVg z4Y@%qE*49i>fqfPk#2dL#<64?L1Ai!4$sHGcoQNQvi?^}F?KX$pMEwHbfY#yEjDje z5-8GWSz}ymLp!-yxyy!wihH%7EC#)adFPB|kxFS3BP312FIq2rhn1%)ZGFpJH)2h8qG`V1BkE6Ez~l)2s6=k8Y!4%<(;Hrmt` z^L}~Odu#4??b*->WycTE=Cy5AuS=N;{R)P1GzTN7Kj&I2gJ{GuDwv3Jdj|F+Rnx8; zg3hh49o;S|Dq3v5TfDM0Kh|`*AN~6^qCvnxiPI1XVzK{y$?LXk&liOAyYase{?KG^ z43JGfWE0OF^{Tue1`b*LCE+dlKk~?RVYV`1h|bi0z;H_{Ck4`q28^vEYf}3OK$S1x z!2}8&ffQDn*Wogbk33GG3Q-B-K4cEeSV`l*aB9sfHCW=M8{CBlw#20ygU0^(uH)FR zG=fvaIvpO0f3X&_rI~@>MJeiB3n>6tNa^X=cTkr8x*$S9e$&x=UT_|dDK^$pKosli zZ)+a{7hZMF=~jG%N8XA~hozfz>oGPINuALhM5P59k>Uw5=8aj%dD=O^Kl`bk+NgSs z3jwh>oX7U$HJ>^ic1?LdP3e^5u9Xy<>n%+tWST{!P5oi=%>SLV8gwuTHz^&N!|Uur%ohm;<>B? z$80;#fC2W$>l92(rP`V^EPA>m&7|lX!$oJbdzjLcoZ&$R1dZ!ap9VTGDEptX^up6S z^ivP|B)f0`@>p|{lf6yM2% z_*fpq!0NDflV5Y|OL)CEM~@|64HT-+Fzmx}J@4sVUrO7NqX#m4Sy{>MDk8T2syvFF zSz8mxgGQ^BzJM2Vq2Ps-`ma@iJ8yF!Dd4??6xdQ6T5wu}L95=MVNqv9#UXIc9N1rCakNbuwoDgc^1;+Ye{)kylP~EB? z`R~ag&U35CUya@KvE!cWG(wiKlD>wM(RLLFbErGI7DmTxaUf}d4)S6U1p|(rt;Hz& z|j82@eJ@N6aFA|qUowVcUxwb3%72+n5@woH7-oB^U_^4dVak8C2sm3 z=$z%i7;WHm%f8lyF~(HzkrjilqY0m}=SATcgzoDnl3ee{C&(JPS9@eF{I)%32ti@< zG1hB#{JGrFN8Moi+lA=QQwi5aiCJB4qFVzW+FmVKv_E5F(O+WrK5UCyoHSgUr>&`v zj;SZjl|0aum9)of3z8;bg8n?4HLhy&h*^N2L++pJj&c>&?98A^ogT6NB6ayasHstY15 zhvbz`u0%2;ApRtm|#1e`+nTv5A4HExTSz$$RF# z0MHJWG*0%Pc&%)=5z>L0#Y=D7@AK0=($g#^0n7t@l$Q*N{8B+KE1($37{ZOa^ Jrl(o>e*pPjDWU)X diff --git a/summary_results.json b/summary_results.json index 667cb79d2..b3ec5154e 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419428, + "Performance_Result": 0.421303, "Performance_Units": "Samples/s" }, { From de6f60a5db7001b63e63c1e8e402b255e2ba2c53 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 3 Jun 2025 17:25:52 +0000 Subject: [PATCH 687/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10931 -> 10935 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 6fb654015..b51578b9a 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-03 17:17:27,758 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-03 17:17:27,758 submission_checker1.py:3308 INFO] --- -[2025-06-03 17:17:27,758 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-03 17:25:47,056 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-03 17:25:47,056 submission_checker1.py:3308 INFO] --- +[2025-06-03 17:25:47,056 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-03 17:17:27,758 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-03 17:25:47,056 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-03 17:17:27,758 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-03 17:25:47,056 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-03 17:17:27,758 submission_checker1.py:3324 INFO] --- -[2025-06-03 17:17:27,758 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-03 17:17:27,758 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-03 17:17:27,758 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-03 17:17:27,758 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-03 17:17:27,758 submission_checker1.py:3345 INFO] --- -[2025-06-03 17:17:27,758 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-03 17:17:28,776 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-03 17:25:47,056 submission_checker1.py:3324 INFO] --- +[2025-06-03 17:25:47,057 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-03 17:25:47,057 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-03 17:25:47,057 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-03 17:25:47,057 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-03 17:25:47,057 submission_checker1.py:3345 INFO] --- +[2025-06-03 17:25:47,057 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-03 17:25:48,047 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421303 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.988 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8884 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 25a5cea61..523967668 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1314 +1315 diff --git a/summary.csv b/summary.csv index b68652823..83fb6b466 100644 --- a/summary.csv +++ b/summary.csv @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421303,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.988,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8884,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 42917d1c3..a5392fa8b 100644 --- a/summary.json +++ b/summary.json @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.988, + "Result":20.8884, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index dce9e2d2defe9af205ea5adf8eb31cd31e6f08cc..d34ed093ef743cc828966283a5ebaf20df2518ec 100644 GIT binary patch delta 670 zcmV;P0%85LRku~JatH~Rg9|`#4FCZAvwH}z41b+nB!eK3hu8Mjhr3PU`AvKpkH=}F zpX8ToRk>+hed67vI{dKuaJYwfK3*~%jEcWx-y5$!$}SsCZyM>Js}ITZ4o`pAR?%61 zc)1wIjqJ-TK5oXJXZbx;EVkJgd>*bo9H%yZ3{+r^GY(BW~Ir7g0O zR07l!m5?X`akK1gY7zg)Hl%rb+@wugDJPEWnf-QlHekH~7X^@VgqafUy*C77$MztpOM*iM zai^`MK9sh`hO%#31J2WQfhBm#si*h|jLk?yCnRSh()`$_2XKa=0SU@GoFy5)cs*F@ zCL4O?$|;5K3`|Pa3E+=i&?nwbiwu3hqY!`$anH)f?CE*CMpd5X^Eh3^>1>-VNj@j{%dg|e zoBj(>o`(FJcm@o)Z$Clx2eTz6$^r?Og9|`#4FCZAllUlC0Sc2)DLes`lZ`1s0fCdx zDLMi3lL{&(8 delta 664 zcmV;J0%!fVRkKyFatH}=o~ll54FCZ9vwH}z41XPoq$IMv_1SKdczzR~#^Z6?=qLHr zS`}_uS08wHsSZD^J{#^Ko)4Ey2czN-+4sh)PqNEK)0;;6=jt=^yu;JqwN-T1A6_oT zaU=UOi;tV}=UILS6>Dua2A_wkPseM_=A&dj%D${dFFu}cG47pAlEIJ5NiC+MKNsFX71tw3`J0+9nBp-ij!!Qtp?|llRyDizFX)v|}ZON&DLS0&V6Vk>JwSJJ)I(hrb zj^iZYQ$ZJLX1|?XRICrOjefy8p;d{}grEo%(_E;gMDO(`UZKc&syJ=6f)X9Up>=gv zFq1QF;nivb)(db^04bL?;! z^%VEO*i1w;A~~Co=Et@JfHMp&NKoG4G)eKr>%mGlTQDeBPAUA+z^wHwPVR#fldA57{peiHTeN2`yS=8x69!|FU>RaB_Ux0FK@^9i9aLRrA393J{DJIGS z32~mPPHhbU0Q-~tC|3aolT;}@0g;oDDM1Z>8~^}#Y%g Date: Tue, 3 Jun 2025 18:47:36 +0000 Subject: [PATCH 688/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10935 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index b51578b9a..cd6df27f6 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-03 17:25:47,056 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-03 17:25:47,056 submission_checker1.py:3308 INFO] --- -[2025-06-03 17:25:47,056 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-03 18:47:30,800 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-03 18:47:30,800 submission_checker1.py:3308 INFO] --- +[2025-06-03 18:47:30,800 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-03 17:25:47,056 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-03 18:47:30,801 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-03 17:25:47,056 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-03 18:47:30,801 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-03 17:25:47,056 submission_checker1.py:3324 INFO] --- -[2025-06-03 17:25:47,057 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-03 17:25:47,057 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-03 17:25:47,057 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-03 17:25:47,057 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-03 17:25:47,057 submission_checker1.py:3345 INFO] --- -[2025-06-03 17:25:47,057 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-03 17:25:48,047 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-03 18:47:30,801 submission_checker1.py:3324 INFO] --- +[2025-06-03 18:47:30,801 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-03 18:47:30,801 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-03 18:47:30,801 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-03 18:47:30,801 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-03 18:47:30,801 submission_checker1.py:3345 INFO] --- +[2025-06-03 18:47:30,801 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-03 18:47:32,069 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9871 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1157 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70646 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.421303 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416695 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8884 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 523967668..3f154242e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1315 +1316 diff --git a/summary.csv b/summary.csv index 83fb6b466..98af05f16 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9871,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1157,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70646,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.421303,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416695,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8884,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a5392fa8b..29344e9c1 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9871, + "Result":23.1157, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.421303, + "Result":0.416695, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d34ed093ef743cc828966283a5ebaf20df2518ec..1fd3a0c36a1518041f73078faa777895fa83e385 100644 GIT binary patch delta 3525 zcmX9>c{tSFA0636YLr4_@Xm~}wP+eD3~4dOWa%YKmiW@h*!T6TRLDNV2$8HQMKf6< zN=@04Jv&9VhAcyjEWh!4|G4*$`#ksDb3f;t&wXx{d6jv&j}YG-Fp@wMFPC_z9gMt6|=H3p?(LlZr3WXlz?su#@F$`>Zfe6s+@BPe{tLAHT>>r|!VQ z4W~d=j|ok80A_T!v26&G4XYfA+jiHQRkLnZ&2gy`rE&_=UBaGR#@44gK&mcJhHjHuAZoIdVclkbE*27!a8bx@N&sX`-Jj`tIMI6 zV7IbuX%V9vHlvT)U(~r)zx5;mtfsQjk=XE_`qp!!h?hX~xq(GHY#18zn!zC}7IF$O zUm6CpKO#{_uNhGe&OG%@N+8uvxbk~1U-JxxabQ#u=ZCe zHfHaiI~VxM-%(;1GQoI-JH^)!3PB-6h4BL`_b)ID7m{PDcfOv;`_!9YaN+t0chfbn zil8w5s~&m4+yD6P+L!@zLR>Qm_g!n|eVVvt{QCHZ97m=kD_l>(v5t#txj=|V6SVf* zjofT9X7&L>uV@m@9c}13+X(V16(0(!f8EXoZf4Itx6*>!B^7-P%E|5@XJPCL`r;-Vn@K6~ z2+6)ckJekoY)v~DW02$eEL=9*u{?jFK1IDj9vfA)wu@@Fvok7WVmc z-)Z09VLz{E_2)Pt_IGQ367ssNKsg6Z8BU1_TB%zW>re5~o0riK7Hmx$e(?3d>Ug8~ z-POcu%5*(I>4{N#hwSchoTyoJ1D2=$)bzDK?m7yzJAAW+4Feax))zFt1w+G&ZR}dl z&Kyq*P%gJ@G*d2L9~C-YKEtPGY=fjai_*a6afe}>)x|T5^{WG~6DQ3h7u8jDXDVLr z6Tt}w9{E=?+N<{%`rz8UFGZO7&sr)FsZUFhqsJpsfYh+rpS@V7Wr4QA5u&zVrG|(1 zU0O_{!k-0SG>f9n$(*qHRw0Q?I3}{UUmOkp^YLht-qCh?^v|#Z_G2X$$?mcC3C^IJ zO6mK_+BPOuu$Lxr_uk=q0S!OPYH(NAm1|jsCr(OeLXDPhX-DGKA*g1^!-oypJ~DX@ z6xsbo`rAuN&vZW`OV0NR9%X2q|4E03m+akEswQcJb!9&OBk%E%6Ws zpq-`=qSZH~^CDfDHCTzr+`vV;z}!lg4OuRZL({1*=R=;|C2n7uJ#}vZS5ps2oVu+z zz!at9)!%dZX8jzo1-h}X@FBWbvE35slq=aNoHdjiNOiWgxIcut4|)x~-Z3RVQ_Lo@ zES(D7yN`Kf-Ds^Erlf}+1aIzSe9-tZqML^w4Ol~FKGehD1vb%J9@`u)5wKj4!r479e$FewiBBFnl>>^ZMh@+a8{Jt9ld z!>aUatMZbeY5W5iDVhCH=&SI|Lc4xR`Ppb^{7@zI> zT{7)k@*vN+(%ss^veI1uNyrQcn!OC- zk<+5q3hE*cwM}}R&CA^=oQaoKj1jg_#s@Yh1ijX?{>H4fC23+K(SErVrmXq|Y7R9v z#33XMS5Y`qi+8;M0SI z9lYi55?;T;54NiFaze1g!b(jVDD4k5w$|ms?TrB-1X_wEJ)TJcQa9(HUK<2>UYY*% zjCpu=hLjwci*)QxeW&lY&)L=*mGnv#h{M~>WUA#(HqNEi40|a(G+IDCk0M9hF~30o zwbLp}U$zKa92S%7T~;eIO6q;aXNbXTxk6Hm&|-2FJ+#UZT~%Ha^;3*X1T;}#zYKpa zMp##xPmUBjgDF(ryK7Gqi~W&)ZF@TQQNifrHm!S9ZdknrUuSPoQ3ufAkPN(>j}v`G zVs}n$5T^NyDfWNZvt4NU`#C2+unn$Q>-m;`s?_!aQW}wi0S!>8Q{v5X>4(0+X*|d9 zpVKWfnecBs4z9})m>^sSG)1q&DVlyvi8m?RUre_4<-lJ()pu{2>{_AEW9Dj}ZUw2q z`Yoi=uL!5lBn6rM1?4kLZs8D7*DX3CECs3&Qs29Pt@3@51nNq@-o|ZOjOd-0s;T}~ zQaO&XQhdeznT3xVt2a?}vRYiJA4Sa?XI-^N5zRRyF;7d5U5-&Kgj0kISM=2lT=-|B zog*^C>DHvIV=XB<7ax)>IcU7{58!!#x zIDbQt6=RuQLDIpey>L!R!?xwI^XF*+EpPYO=53Pxy&P7unH*0qVC$ccap9kic7_ls zAPl_`@?~-NT!nhXWdbf=yw#$v8ezENQ zPT8eGw!V4`rAu5`KI>2cR}E|iowuwlvjC12dCz5u(d5d}>1t^He|~F3M}dZVE5-_Y z)k|m6ptkr|d*QKXIB=le)%D$+b1sy9Isg1PfgKH`B75P`^H`Oq+zXS{y#@O*B{=dy zhi+h-bOOvp=5_O78#cKFy)90Her;3jYd3uDx8Q19@X1i5vBo@fr}0t>GksgU8BGw; zMeupi{$bK~R%2h#_%&tTGF%96ZS=6X9ioy+_T^Fc3?7-N$3RQZ` zXV=50bB8rdJYv<|(n-454McrUq`@VtE?xzBfN@8Vx!aeU6dzUMc^zTg4j|SfAST4k zzfjQJZvK6SfIuN~)}K);-7yQG%sJdVz^E3@<9x1Y@|La7qY8qmoJ=^>-dYz#u^9c|V8 z0~8_mZpF#$0W?`ewj2S+DIn!2rnJ|Q212h&9e7@eIy7%eYc^$K2)HfPIO;2ghD{kfQ{><(N?&wwDrR;_&kxbIsS^irU=*;9K&+W>zRI>m zM9C#b_XYg;ID%T*MCl|nPF&1?45Yi+zHzq&$43g!ZkBI*RYwP5?bKoM%kO|OnKE3< zF&NQPIu6$dr%c0gwqL$@%h+D@Ig=N|)DftT+f1OgG_4jg3KXU~K!4}_|#B+Nid*6iaWiw0AuQwSBX;I^3+ zUYX}ttf9d{Q*XwcyW4@CCTu})=%WInjgn#_D`P7lVA3%-)qiF0H{lnCsYs;2DDip>E01QmjKA7-&)p0t?5zf$Q{z6l62cvQ ze(&NJ1v;q#Nn*}pe&Rv2B45=8@ah%qPp}i%e-30PKT0rQJF#TnOWl`7pclMo!20;A zLtz$qCQYUIVEyS6uG(?=e`lB^awf`kc~pA&W>9TJ_`d%v!Hf|&E)yla(fH}vd% z-$l*q#BBR$i3PsJV(g5BXYs}RI^Ru&bnjQ+oIC|a43wIjCfpSFKAY4XMwM1;+WM!N sQReZJxWkV&*54*-@!lV?htXD>Q_f~vlucsV7&fOKrzB#N zQ$;;QuVU)T5beczw^^}es`Zg6XGs|lCflbs}EO^=dv zl2mf$E$NAN{GfM9su3eHov5rwfhMZUb%nBa{j90kt!N{K#GvCnas4(Sc}`PPv)QJ& zop4@ju?tr~U4^D|?8aX~EQ7~`h8JqaoVOa5OS0=}o)$eJ%AgfTqwd!HiMsw_i$yuc zpY*URR~SPDt9JdfkH~GYSyr^qq3@}_fbG|Sf}l$0+2zJ;;!M9wh&`t+HXL$frs3c` z=HIE$VJo?27TaS{%Xq z`{((93a@q+vqtn=vBBs2Tsr1>6CJ2bU&I*JDXx&pisCF|Aa8oWslKN>7jjizFJ`md%n=Y3ASI%s}|sa>eDG9x+n z`10tfZ?5wK_G;?j#NAo{;O~)bGYPiGeeJ{67M&N0*L`}I&BB0)ac=Sk`*WMSt$E>p zK^K@V=i1aBkDc-M|0nD<#Ib!O?fLQj#M8V2c;eWbeMBXx)A#ddKH{685k>9c#>Zx3 z*U*z=#VH$9kX4S7xnAVYb4MQz`IqKMzCNdcd>#*3RzJXVHd(9XT*$6o%}w2?h>?Y9 z?+q z;YX3rhybnX#VbntFK;9Y*S6E(7CWSspW<(mJj$!LxDlq!n4M9zcFz!qGYfX~DH+waQhSxxl}JjhLwv?bX53e7 z@aQK=gZ)SUT$uRcItUEpnRGThzF|O6@xdDS!HV!76l65_g;{iK&{iNVEf=daCzM)M zx`|%y9=E!q=ToW_9vNs$Msjc3z4nbHmi=&x8p`tX2ZaK1rU&9g=qM2|6^cR6_X2*V%quY(u@I7qr`coKdfrR*6kdFCIc&uPiMf1s%Qdm>pQH zT{dWB)gU&Z7b{=!Gv4LDKU(sq;*^m8?jcsg51gWaxFSq|INCqCr~-+msVgFF74m!1 z6wqA^O1c(QBw~)h1z30WaW%~1M@F>*yHgTkEo9chj zat?l_Q$-KQT3@eMuE%#9#5UP4N3RR1ZKHAk<}@za_wN*CxNrZgyxUoPKDO_*5eqBI?^*@r!%qsAV~l#$L4oDWj`Wa3i7h-hCiKe z^0Mp!`R#1SK4>5UZZ8F7u^)GLg^si<2D27n_j=0JFv5M8;`+zNp<|50&4Fl!hq-j5 zqRVx2zyXI4GD@*WdRD$$g!%87CX1OD7A3L`$Uc)`R^)Yy z37MXRwYc}yABt^>-6sxwd!IfWU|EymMbREmr%9EL*n-~#?R`cLBq9H1YqCIX0Ta*)q%z0N^TZ3>Gf51}J+UB#OOc)j7nrD> z#v6m!4U4m7KkzN;U6v+<>i5;Wsm7M6Y(Y3#8u1XG)lTnI;NI@rJf>X2PTXh%TLV|F zlg`mMZR15wAbC`=t$kivP-($g@ytj^Wq-$JJN=Cw(G#!xh)Rt3Zz4Vwp@$LXl4r>Q z6$a_0M7#WU`XNOjPE1SrB`pxZ&_7-aPzDmszhyx^t+9XbRkh0G%~SxI!kt5AmK5`C zJ+m3y0DYNZf7;>ZAQ|+sF|5>Z&|4bBdB){wUdi)w`_Sj>5@=9a8J;*WB9U>O&$wLH zdEHG6$OIHXNDiP(2}Q1E=0RCahNw>NDbHeHCP ziba$UXCujjaP$5P2GD)~@LK)))YAyRD}ml-K<~d{laAlXsIwK@8M5J?Hd}uwf1;sC z2>MzZ(M-e;*+-R_6HXi~+01qw3=E993k*`s^S{9$s88`&Z`e=u*?o>tBb&X=2Tg6b zPjb)=x1BQWEp)eq0ZWAe!FmqDfGC=51}2=~vCq29k>6V+utvZoj-e8Hwp~9oyLXrh zYN#9TgcgO|_-_L5Ioy}>7@I+vF&YsjSNcK;!qu3(|4WGrLM7mw#ML%1>DP?{`a+Hr zK{GQoIYm^#4=6vV7{rd^h?8DhdeR;qfvyqu|DVA%SDJxjkIqR zqbv>Dc}4`mL9+MpCqa#Dkt0~BEB~*fLh6)#(z3Y0{%2SUhm-?6tP&~t2RLI(A>@s@cmXZjW^Hg8rm3>4xg>gs&g!LBL%;`7IrOZm3oE*kNh zg48kA!S@;rQD|tr20cO zOCDB^s3(C5Sv7m4-WYA?Wa6Y8_9}SLs2cZ`6uCMGFKfVU5BD09heQpEAg6{xswk@N zZG>Q~p=j%OO0}3SYP0F@n=T1^2euoq)FhE9Cj%Iu3ktmk-RA_&-}mjJ^$erL6Vz;mHj-nk zu7Z!=yW5JIP<>+6WvGe_QJE{ORN$`2yAX%mdp(IUO-DM@7P|GVf;$SY@hvau*AE}T zt5;j8KQU%eX?&e6w#tj2_?Fj=(O^fM1k6zTX-@i%cROt7-R>)M!Gopk)tk1v?@fcJ z5v|nO$}8y?@UO}u!y)ry?O$H-d(cd1$MQ+vhXXuT8vmR42E+Fn7t7BY!Q!3R`njXT zXw?mg<>x;QcVkaG(H3@ClQgVV$$C&?5p$4cN3x zV($-RW$1{7aTajb_wT-emc+)7mLRX_YNyZVZbwDU_(x_SS7ipJs?KEsb?;gm_uP4e zSZ(p?RL}`8IN_{XRRlk3TK8yCI!Lnr(A6ra-wTn$do#js=PG@Fae7Zc=elMEBGGnl zSb0^6Rc3uhGCRae{B3{4Enp0>tD^j;pTxHjP0@jk(avVEb+4@9N9%=3WCEislJ zb0F2YnJ(QsOR`R0|4n|iGBs8isYibF43XoX5;Voe&R3F7|%jn%6U diff --git a/summary_results.json b/summary_results.json index d23f7ab33..db29ad1ff 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.421303, + "Performance_Result": 0.416695, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9871, + "Performance_Result": 23.1157, "Performance_Units": "Samples/s" }, { From 630b5c7fa45ebd23e2ba415015d56e87facd06bd Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 4 Jun 2025 10:27:44 +0000 Subject: [PATCH 689/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10935 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index cd6df27f6..a24fba3d7 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-03 18:47:30,800 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-03 18:47:30,800 submission_checker1.py:3308 INFO] --- -[2025-06-03 18:47:30,800 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-04 10:27:39,192 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-04 10:27:39,192 submission_checker1.py:3308 INFO] --- +[2025-06-04 10:27:39,192 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-03 18:47:30,801 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-04 10:27:39,192 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-03 18:47:30,801 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-04 10:27:39,192 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-03 18:47:30,801 submission_checker1.py:3324 INFO] --- -[2025-06-03 18:47:30,801 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-03 18:47:30,801 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-03 18:47:30,801 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-03 18:47:30,801 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-03 18:47:30,801 submission_checker1.py:3345 INFO] --- -[2025-06-03 18:47:30,801 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-03 18:47:32,069 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-04 10:27:39,192 submission_checker1.py:3324 INFO] --- +[2025-06-04 10:27:39,192 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-04 10:27:39,192 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-04 10:27:39,192 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-04 10:27:39,192 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-04 10:27:39,192 submission_checker1.py:3345 INFO] --- +[2025-06-04 10:27:39,192 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-04 10:27:40,215 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.22828 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 5.96744 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 3f154242e..fe2568ccc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1316 +1317 diff --git a/summary.csv b/summary.csv index 98af05f16..d973effaa 100644 --- a/summary.csv +++ b/summary.csv @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.22828,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,5.96744,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 29344e9c1..8f9e3498d 100644 --- a/summary.json +++ b/summary.json @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":6.22828, + "Result":5.96744, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 1fd3a0c36a1518041f73078faa777895fa83e385..6fa8a6082013b8de943467ba57bc92d7a877a07c 100644 GIT binary patch delta 3775 zcmZvfc{tSF-^XW+sBA?y#y*TOWVv%sNEl=rvhPcg1~ZJUWDDP^{ED*9FluDUlH92n zyOPStE{5z<29d4d&Tc%@{d=zK`R6%*oIlRFuFrYB&*y#K*LCWh8=Wh!9X$e@kyW@x zjykFZ^yjokDuq*pR7FN?7%}O3+=;!XPMChcu;Ufqx#?}asGY$mjOW8Quc7x~4R83o zw9H%L@@*ud2IQgb)y$`X?!9IxNqQ@5dA?zXJ=G0q8M9!p38Mw{>V<|KMN4YS!Jzuu#2W`ng_IKvfDMFlt(BQ-y=Es$2os@OHDFQzQBVh}8 zPnT#(S6F4gE_bGkyYQ-guJT%$-D=Ol)Rg02nP2MtX^LJckW!nmO>GNI#D4j-2EFm@ zMx;AW%u;B~b%XlNr{C7!KFNHc0JOwLDQ(*Csjo$o8%T<4*w&k4KcDpX_sv)5WzCZr zfAR)J3FbUp(D9|hXI9O=WG2;1#@NcULuMh#+!Gyr*Mi@AN$T49HaqAI-l^PKxGkhMf#wd<~ahZTTw&7k=Y{@JZLL$0qd8 zE15|RO`#YQuPMIYOWMNWJCMFPAssQ@uvv(pLViPKnRr!OtanC|+uAF*{P3QzLYJ^m zqU#oPU4(TOYFsgw`sB?`0GK!syn&E3)-+0{H>&=)k~w+(3n`(t!%MFxE@FN6P^{(h z-O$(o^vkphm`Ir>Xjy%C`U@kyU|XfJmciu;a{SqJ(c&c*;glLp#4{q#3=^BDineLT zBy3||u05uFwh^A*c@sQQ@ke+M0 z)!MkB6D4y^Nrv$0%*+G)V_)tDg_E7q$rh^?CVIP z^xd=u()WBUX$VMCZc!X$_%fS*24dIeH=EEM1q>$eIJXOXlQd+oW#V8_j0w#`o@APWw9-R)AK?+lG{;G7ezcnw;8qEs!LSM)8_!gKWXkXK zFxC8Z*LSS!G{OeiayEj9181Rv_$|)@>m|3#M~-UT;YzeQ9oGCoy1KQHrjz(Q(&$M; zC{RF7(U55@gF~cfDcqTU@ML)Ltd-KMI@m|~6{8QhA5UI4hd)rLNsKsE1r%IxPtWQc zk(#Extlh>(*t@+;Fx;t>oi`LL#^vk7#V`@^WgZE2iV0^awH1|IIcr)X6H-o|(rDxx zWDOqz_wYb!rr zIqrOWxFYo-OF9|hf5MssMqzmR>HJ2Hx&3f`IqF@RrU^;e{wy_^^pXB7KYa&)N8>r| zWw2FI`5^>1T=h}qRZR$@F5YHwH0`|c;+!9{wVE=`vn@mV9O|hhSn`G@ieATZZ*!O6?^`30o(^y7wUD{cfajlh^I%thv zw{lFQIV)zNDv%Mbs5(7O|eMTz>}J^=JvIeub! zIV3m1{HEW8fAWJOJM}QGPK1A)H!T^8C1Z)4T3U|`Ap~87)+28SK_Ye`Bs9*)u`V$p z%aEqr9b3U)Z_WR$ZgIa=b~LUl^S_9BGj)qKlkb!-RMW@ppWSJ2n1DgO$!H;Ium$;& zkfey4%Okt8$R3%cTh>FyUget^7v;zggk(@1W|_yopUe7%gA#J-+qZ(N{qw$`9viaN zWvrHezg~X7$hI!n!^N&HcvlfugqX4Xd=JY-o)VG;j5Uux)VKS5)wwc=w8MPqUOlyO z^wEcnomk}7&~~H4>n_`NmhR>ArN)b>z~3W=x77 z%=6;9@dyE|hW0NTn!9WD9viEDNgBH$QgPg4cd5L>?}oEYWfKlmP6=rAi>+``v@62Q zSVEGpTzF6>zp?v!kS~--dUV#q7!PO?N0FdX42U5IGP1HLQ+9SoE>&qJ(r$xFf<%dt z5BJZ;N!LI$T*PAK9yX)<__$CvCLVH^Q>Oqi0_zm z{QliiEU#eyh`V-}B^#9$Y1I!0lFTiC^KnX^=mm~5IDk|*1V??$0W8EjB%qN01S#%!pxnq1B01V-`;4| zBqBONRK(o98z3AtMONUWP)*`84Mz3Z&(NGaiy3JM8Yb+=$)dKLb{YYO>73H^+2+EZ zBY}})s^JN!afSSd-{l;3VK8jsa@t^L*#%)wM3~GImeT z{`MBrBR8rYL`irn(^n>i-*P%=95Vf-G@-4wlHJVjXMzZ~1&F(9lyKP)aiEY_=^U3_|J z=5Q2Aie?hh4*Mf*%wi_D2g=lK>U?l0F?yy{BBizyZo+H0g< zSfy1*CLh5T+s*V=^JDG=W23EJ0$Pkb?QIklZt{1EKJ*hD7=9ur>r^C^kFTth7QkY-x!!k;(vu=y% zfoyOsmv_z^WKfi4h6$NYW(##90juL2;TA3@IKnfqfdX>)-+@3Z=>*ZIw`*fGNuQ9fzw3JS>YOl^UTmQa$VSTc75SXsbj?xmpq^@MIaG;r< zmOJo0CQ@kbL9o`{0`sNx_MjdyX7m}GmJ0Jv=z`F5iv<^%-_oCB)B>33oPl&Se?piL zAl`sJnCwTNO_LmBtQC*_a4DYkuEgg__3mj8!~o%c zdB@M8BxRW>xzw_)=_#{=D>WT0-3eb(XLOnm5iRfuA$7eSK07N2H(5hjq8cgMB*?8C zYEVF)hpPa-O1HRNNS4IdB4J@u(@E8vxgK;hO#nYFqa8fx@*|lU>)(=hsgWAul(T<( zgRvkn2Q@l|3rQRbPBjp%{s*jR@TZzv0BQ)wCu5zPd*8Dw^^SIcYJ=Q+rIRK4b9H7* zCsf-;H_jsGs~7t1y`W`oNu@zme*3w#L_TALn-So1`^ep>cSC6=5r)SNNukZMsxnON{|8>?y zhik7TJC3QX;&--dC1VBvuD@F|0FiWJ{A#yt|30`{8PUb{zaPtOq$il4?4+y*rjRq0 zjRfvUfk4p&jq4FH1i$NKIh9l74P^=NFS4KtLh|?BBETS!9Ty0667+wAcx#w!t6~5? OO-@r$;F(hTz4Jd%6(5%X delta 3770 zcmYLMdpy%^AKr2nO;ZZZSlet)m1m8LM$}_i4n1VaAx2YkmYniiN9V&fCPZ>fsn{eX zO0ArdbI$5fP9bc>{_);_+<)Bn=l;Bv?9bPz;2TAX^p|x#$fIJAa z761b61+jKwRajZlia?!l)v>s)DEG|jQ0C{z9~$bkVmd}zQ@ETC@?02t@g-g}i5ki5 zYgwO(`Nr1^=-;=_57+k_`XKm#$y76NXt{(R^&u!H78^9QF3-8DiMM`4%#x&P1NDMYc>to2=;_1`l3)LCkU2a3k-U;JP z9qwtv>}Ms1QeqlfFaDfgygKMpWs~ga3GDrFU+MNURiXWHbQ4@?;4j%g(oU(;_!D$E zDIwfb0TS5~2t;H?MZi-0{JLg1FT;rh8o)eoXtyl;!=G zVfW)3=7C`7&gDPYxY+H#ZEh+zuho6+RkbckW%%ILdX51!KWHB*E~ ztthP1oK+&m%1FU=_+q0Cua_cS!8W+V{|v1AE+6a8q{TLhqo5OerbMJU%!<*rvB-J#17w6|hvU7~4AEMlbo=Zzf7#hjVPWFxq1O7Z)!+?>AA zX`E|eZ^BqZ6O{pvR_eX#*K(;uVCV|t4hVf7T~o{P`#G_IIL6Z%$nN@7Uo|nk0nkF* z@9RRCQ;UMABP8zk?t1iC;L!Z61ViphIVFe zPlW5GYmL++&4Lpd!4tv1qJEq<>dW;+Ds&lskPi48!8`_K3^8KE7Hj$PeT*Q}Syi)e zsh0Gi_|Nf6qYZ(#mXfQP6Ll0O&>f3?gX-$^`2K4CGKD|>`&Dn-{my-qHuooS4Iw$Pm1f>IHNF<`#yl|c0uwc_R9Ro+`gZV zh_l#5G|#s{KtH^tBd{45mF8nk$rwq=T@`A!a>)3?Q?vp9rpijEFDK{c0Mm-(qJK;l z%*wgC1IfR0nBS?5WI4u3(~jUo@ln&z_?_+yb7TS%LFUDBcxD0zDxJ~YQgXpxZYjq$ zF9@xQz~3-RY0lTfGTV$sJU%ko+}=f8&t6<@8g(CXUYb8LU$@j>nLOq|pV!whnS5Tk zLzWPHWr=TvBHv6o~VQf1&5 zc#9pIjQBkljG1QY3$wmEe|fG%NYazt-Uq?Jf4|??Xu7YB6Z0c#r`y+3$5h`qwVZ9N|?RMe(oe zn*BUE4pIND2yga}VHZjlzJeHGf|uW-=y*BS8FAE7y+6&%#qrJ{`cA-L<)$6&k!B8c z+R3xXw@c42`$Ef$A!bJ8F5u#3<~xH=!zTH}5g>FKm37w?N0e+AVO9J~oYZg&Jq7N- zEw=ZMtk8>Tag!K&v5tuLAw03S+V|)t827(`{lj>%zyEj*ddDT?+v)|FHKDz4m~i5h zDJ%+|#029rO~h!g(D+O@F`7!mB+Wm{{Y;g4NSZAN+w!VR?gdHi->RvN_wc@`W=2)= zVaEad@MzUk)$4X$GcIqy?%~Za`$3IN6BYpi1Dms4AgYSz95Zc8LO-cZUng~+Fe?3> zYK>tfK-6%&Qw-UOBWYKBlxf+#mFvD-@0}j)%;~shS9=`4~>y(~H;+8m~kq?MrjgePpnqsu~n&^nx>T^$7 zVMu_af~rIwPR2wi7V3DzNH;oM+JIF>$dhCEFk89bIUgE3*<_mGH1OT%LOX+ z>Y<0BHUnEk{E49`w8;Jz5yAfS2&_Z;4N;XS7ZEY23NIHI#++8%n$zlK*jMB=``&Ha zD02GsSXgki6|Pz1|C;kxnG5g^rGm`G0T$@AaY&PT#_msWHdv4NF~cc~2mb3PU_sCkqcY|4e1f8#r0B&&VMpGYiIuse=^3SFQHvOmMkVTt$y~nRIrTU( z{fQSN9p9QSD41o3HoxBHlD|g%=kKV}wbVpTp}_1!thdBOj2B4Mc{ozD2wpo*nV^!m zrAaR{T>XlMSM_D))4<-c{uSYI>aS9Ma1oHuY*Tl6^??h&K`tatP|%^aP$V$ZZ)SEv zWHhpO7mBKZ&*X?bYt1Tz<7MB9P~r?l<>>ISboej6Ws-+veI1#*C|C+OK4wu<^0Tc- z?-8ND&d2A?j8`61@OQz9Q7b_VB@NXBhn~P|KNRIl(+`yD!<7@K1iP)nb=TozktiF3S?FfNDF!cN1JZ=Ck~Kk!2XGQ^vAmG? zXeU4=zEFA}273C86ijtX#eKN?Ev@LXR5fMlKl;qEHU(GXW}fSCnm_vNIh;3SVCxsB ze>sC{f?q`fb=`D}Q{+xDga(CsLyEU$tc{xYs)%A9Fi;j+zEM(M`W!-EI&5akBZ3Ag zSEXs$zemU>UmYpWyskNfTGwE}UdM{ZF1ZBVdv2wpu6hmXcFqJ@pi>HFDA7Ng#ztRRug4r>)TF36kesWH8w z|GpnaFRY=DCpCOOS#Y0{aoOdiuM03bTy%7;d?TPbCJgVY4@=~~p?p<+MrhW9k^EH> z2)%IT1T1&s=@Z~JcVj;2NV4iw203adkH* zIP2fuvEG1et;EDxhp;MKEu~jG#A$NjOx7e)3m7fDZcip^fq%wXTI{k5!-u7`$>d4#83S=|0b=80t*3L9^7sP*kL z)&c6zoHqUdJBZ)EGbi<4l7+x?TJ5d!9Tg;IE`Uv08C`NO%BD@GYnSY*JABZ`IHBOr zB#%nVLi_MXRQMOeYND^rolUy^^YjOhgT%#KWiO-kdCl&om zPTo2G%}(0nPW8pHzkuleGTXye7a@U1Q@Wy5S-Oqu`wxyWUXSH!LlqztPwbk#lo6`ud{lvN5KM?f)-kJi%T>Z1hU-*0__C- Yf6%XYv+T5vgR!hst^JaNnp->n0f^ii1ONa4 diff --git a/summary_results.json b/summary_results.json index db29ad1ff..2247973e7 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 6.22828, + "Performance_Result": 5.96744, "Performance_Units": "Samples/s" }, { From 81018ab27ccdb1d9191f0f924578260e9960856e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 4 Jun 2025 11:49:12 +0000 Subject: [PATCH 690/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10937 -> 10936 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a24fba3d7..e0d2405ec 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-04 10:27:39,192 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-04 10:27:39,192 submission_checker1.py:3308 INFO] --- -[2025-06-04 10:27:39,192 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-04 11:49:06,846 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-04 11:49:06,846 submission_checker1.py:3308 INFO] --- +[2025-06-04 11:49:06,846 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-04 10:27:39,192 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-04 11:49:06,846 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-04 10:27:39,192 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-04 11:49:06,846 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-04 10:27:39,192 submission_checker1.py:3324 INFO] --- -[2025-06-04 10:27:39,192 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-04 10:27:39,192 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-04 10:27:39,192 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-04 10:27:39,192 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-04 10:27:39,192 submission_checker1.py:3345 INFO] --- -[2025-06-04 10:27:39,192 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-04 10:27:40,215 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-04 11:49:06,846 submission_checker1.py:3324 INFO] --- +[2025-06-04 11:49:06,846 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-04 11:49:06,846 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-04 11:49:06,846 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-04 11:49:06,846 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-04 11:49:06,846 submission_checker1.py:3345 INFO] --- +[2025-06-04 11:49:06,846 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-04 11:49:07,916 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1157 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.70646 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71368 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index fe2568ccc..58d676f40 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1317 +1318 diff --git a/summary.csv b/summary.csv index d973effaa..5537fb0bd 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1157,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.70646,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71368,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8f9e3498d..ed332c34c 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.70646, + "Result":1.71368, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 6fa8a6082013b8de943467ba57bc92d7a877a07c..fe7b54de358e79a2b685c1c23055c1a041e7563e 100644 GIT binary patch delta 1760 zcmV<61|RvkRk&5KatIDCVAC;i4FCZAZ~y=p0Fx0Z8nc=RKL~%PY}YS=Qk6i7CW$Kz zAd(5B0YowgWd#F>WdKTM)@TW($pB)b0F)*HNP_?bEg_wzRAr&8$I1}O3IdR>SpeE$ z^6&6vYgu3=EQ|J!wzHo)W%iHOC0L?H;!696WCCgbkPKp3!Tw--tfThX)VWR++ zCi!Q3;{%o;|ImMF0+t~EkTN*IvaaC-TUfxq6w7@zfAp^T)5-1|x9kjSmvD&&i7U+? zk_n{wLo&!^1@nhxfJ>6`B$g)ehjjp0n!Fzk-Vd~dWDxY&p{z&Bz+-1!gU7CG_I?n{ zZ!WUevcxJPyDn-!ZcF>NRmNqfgiG{CT&ew#Odz!%l0kniD`-C~16+~}aA~rBSO>tR zN&3+s{Xk1d20>RA%6hB}PqD-df>UhG($8aDeiK=?mPJ-N$ri02ty_M_FZmS{?Uay- z@`x*?ACd{A^g}X8W(DboWq?eQ0WwX_59q189;AplNb` zSOF6%6hB}H%;ON!KPWW^7FN5K1r3WWs#LO&7$+8?UwG8c?G&e z`O&D3cvATx8D&Z5hlCK#3eFG90GcENXquEC)&YN@X)=B^7(dVwvde~Og7HJjaMvVV zyX+m466Z&wJ>p5{hh&r`r5_SPHY-R!ECXyZWd~lH zHAz2g6oAv@{Ah4~pe3Z!l&UP0^;jA1o5T!)eY0li=j-8o%8__bpTYFETHX;4^)B$#4a0n>`>MtWr$@J3#hPweaG@iB6}_CtAr&; zKwjy`q!c9}KqR9qX#tTCVp+igVi~}a`89BmHEBR>6yVaN0okAdK}*P{ zDL?As9LjpM3=Xmt{``-<=NeEt%8`FmPx60iSzD#^YtjGFRx9vz$&0dfrvynfNj&NQ zkc_gV07OEFWCa0;WdKR0)&PHEGsytWG;?lz4UH_>S{nIw%toL#PU<+gSGiFcRk@WX%l!{Hv{ z*?946IL`l)eSfn4D7$n#yJ;kUu0JHtJv{wgTX|=L(dBXyHPSEh=(rhuo@e(^zSw4C z_<6Mcc)Z4ZF^(7G^vinm{PX#ilm5vx9{#wR)N(fda~b{k!{7(^@P~o>)6Wh4Q{%Xi zE&V>4t!_0+{so)X$|^58PXQoyPOkS5C9>F;3HX zdjH*R)E`ER{JZSu-6)RJ`)~3~vi;z~lfM8^O9KRxF)13eDi*p91SfuK)syHXD}Sdg z1*{~M0QE#AB#J=XEPI<;#6PkPY2F?;Y13BfiQ{@^znxvVSnO1j{DSk+=n~}_K?!JX zgw%D3-q*|Y9wk07Em&hTl&Axb7S&C`E#=0+t1}i{kl>RD(w=f#qHPG4V(j@A6!Teh z=qPTCQ!K>N)z~ur&1%3CLgrXOV1I%IhWEhKjzn}o3O*w3k865>;20WEpo7PGmgAE* zfK&czL$91Ul?pi-;()3`aLOI9U|dz; z^CBLuVLq(li8m6&V;A)+Z-+&OJ`hO^K!>G0YmP~Pf-1{DkjSU1ukIIF>#anC|dywlU6A_0hN=JDM0~(lhr9Y z0rZm)Dkd8zernYw0RR9{0ssIJ00000000000001hfz^{yDn0=!laMMS2HYnA0001I CH$d6| delta 1769 zcmVBLatH+>;vXY&vv&w$2!Cg}WqPbuB~YSC;z|REWCCdbkqkmv!2n_z zfRdRtT0&_ufY>MirAYwNAOJy2NT(@PSt#qVGK8{%0HkXcfOeStJABz%7FY?(qWz=o z?59qd{iAgWmZ*`q(*7ZtK-xbfgIHFue^>^vWL6DeX|jLVD1fC&{@LF6fF;O3bbp$F zCCERd3{J4DYdFCc7Vs~{a$n6Ky=(q-virs@JHy%~T%tkZO7n+g0%`t`402h){9zg3 zl4LxIrAho@9RQXl??;3811%vL1U+^r>ya|>*jd-$vFn<>AH?#Ti|n;5v5Ls9i`tLd z(!On#aoH*168#ZZYCj|sNbQGYkblbx+7HVBmm~vRnyeqz0dQ%Oel$ow&=QhC(3OR< z9xKCBEHQ)N6kD_O^B9-kM3${(k(ExeMe9fFmf!J9e#JyPC1j#J;!5dMG^7DPj+<#N&=Smly zpZo<`H*iY+G?aEr&?J9|M@l~=6G-WYWDv~?(hti3nj`~gnw%fj0ibD8el#dQ&=QhC z(3FL;9xKC5lXyX}Y1XX#d@Y(!Qe|sdWTj2B==^BAr8{L_fi6*gG^!(>RDMWCS=ITRQb8m zMdc^oHNBwl*J^gzEOCA`+9RHHen>`HQu-kwWV3?w!!p1oQ+D9BS(Eg`MgcfY&W{G? z2UgryObbRQY!_b z00b=|nQD*HD4ai2H#FH8j z$tX*DKqQ20R`7sW2G}GS`GnIpX+W$4K+~iF*`NVIOGpL*j~%LdtV(X1><)o#)80D^ z=-Z0ywXCnoj%gPqplrvq9IZ=^vv!FEWTQ{wNehT%lqEGF5`RJ}D`-G015}a>P-(J& zSO-9*$pYHm_&_CCKw zkFrb0vztcp=lVnP+{4q~wUu`^7+o$WQ6v2_kB*zs=XrJy<%?}LhM!04kH>4w7vp#_ zPQR>2&p)4UIq9EF8={DSk+=o;l2K?!JXgw$<~J{Gg|0VO^# zEm&tX)MyJHO`DsFTgr`tH)kxkAi*aQq&?-fM(YqP#n|&TDCV>1&{13&r&x%kYq4eg zo3(%oLMB*2V1I%IhIhc!jzn}o3O*w3kLw12;21hkpo7PGmgAE*fK&cz!=Riwl?+=8 zSFJ~JbZ@*IjeXx|eK}ZT{v3bKU*CsSQ>k~S;((e;aLOI9U|dt-^CBLuV?NB|iB}TD z(>Cf?-VTfG`#>Zy0NuwuD$DZuWidldL5jPSOj2^Y$QcPO?rHh>wHtZSe*&t*kbe`8 zfPL=EPtg3cDJIJT1tH=eBXX1ZC|dyxlT|4^0+tt(u@@GTsVOQAgB$<=cx*3oba`xL zbCV$|6qEER8Ugi_4=N@bDQ`ZXCIJ8dPyzq|5dZ)H0000000000fPvPNP%1tFE0d2Z LBL>|k000003Pv`0 diff --git a/summary_results.json b/summary_results.json index 2247973e7..91f27f5dd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.70646, + "Performance_Result": 1.71368, "Performance_Units": "Samples/s" }, { From a233a915cdf3e1c645193c726e65e25e53be127c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 4 Jun 2025 11:53:53 +0000 Subject: [PATCH 691/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10936 -> 10934 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e0d2405ec..0389b18c5 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-04 11:49:06,846 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-04 11:49:06,846 submission_checker1.py:3308 INFO] --- -[2025-06-04 11:49:06,846 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-04 11:53:47,423 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-04 11:53:47,423 submission_checker1.py:3308 INFO] --- +[2025-06-04 11:53:47,423 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-04 11:49:06,846 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-04 11:53:47,423 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-04 11:49:06,846 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-04 11:53:47,423 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-04 11:49:06,846 submission_checker1.py:3324 INFO] --- -[2025-06-04 11:49:06,846 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-04 11:49:06,846 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-04 11:49:06,846 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-04 11:49:06,846 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-04 11:49:06,846 submission_checker1.py:3345 INFO] --- -[2025-06-04 11:49:06,846 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-04 11:49:07,916 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-04 11:53:47,423 submission_checker1.py:3324 INFO] --- +[2025-06-04 11:53:47,423 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-04 11:53:47,423 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-04 11:53:47,423 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-04 11:53:47,423 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-04 11:53:47,423 submission_checker1.py:3345 INFO] --- +[2025-06-04 11:53:47,423 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-04 11:53:48,688 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.416695 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420653 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8884 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 58d676f40..a09b5d132 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1318 +1319 diff --git a/summary.csv b/summary.csv index 5537fb0bd..7be78b929 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.416695,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420653,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8884,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ed332c34c..ca3496c3c 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.416695, + "Result":0.420653, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index fe7b54de358e79a2b685c1c23055c1a041e7563e..c3585f7ee948008f8d68e7d23541a15481c0f5ea 100644 GIT binary patch delta 2938 zcmZ{mc|2768^>o1V>cS&4q`AfS%%Q$R%k}H8H}YOgR*2Sw^1Rax(-s(Vwpi@XDr=t zF?L!+ma$~%CQG-ZEMwoYhTo~*AHTnT=Z|y#_`Y7xc|V`$^L);!b*OeI3ltJ4Je-ku zr6N@55{%!!F0Px!6TJx>Q!vY7PpRG;gRUy5zBCN-;J3rBt@pPesRx8ZJ<0!L@j(VwA!qJ{lbPDKp4ft+jFz0 zj+x(FLL0OQm627LX2?XmrzveLXH!F`kvu*AuI%?aTbIyw$5?>4vb=Go1>HVgn(L!@ zS@&IHk6YCB$h0-VfIuBy>Gh4-hR<6J=eA`H(lwMc`l@r}%>Mbu2jIyq%C1$SA}xPz zr+eL8YB$27Mic0myLeR!28DGy9#$o;E8~Yinx97eJNM8hxh}qE*4E)V(WmdV4oI7Liez`(xsi?e|etUoI&6+2J6WPPl8tCAdCyEQf$PaAS^G4b=Rn3ow&UY1tCh3}K#HL;-##QSTZpIoRI? zN!?K*Rm2@}49gEbTs{|M*%spJ(1Zint52HNl@8c)Vo{%y8J_SV49cD56+vB|*XH=8 z4PuM{blM%o`Vo1h!6Yr^11*mj9UMi?W`8M@TL0laHU0N2o66wg^8RW+*_!U`aheUc z)9Bu-iaqNu@)L1~`Y@LpZXl^$=u`~sCu%Gz%De6VaDLRMe5myg*@gY=+;YU*Cv;@h zU|16v(YJDyz8x4TMb_8-y3GWOrD0$UrdS!5U(WrVZaMb{_~v{8a5`nrK#}_H+eDb(RHXtm4P;d5UU41y>ogvq=elp1Ig<_TF+EgQ_Bl>vNg*7Jx zBUfh3vMR=lkGGZ}#-Y8Q_W(6~if=FAE_xWD({Nl1iom3p^GW@CXO8yd6kBwC6 zjI<*&hHl(~@czUQTca#12kAdK{_i+Z;O&sH)RBE>pPj%+kv{-2UXtA+)rd|=YFBOK z@A}kM#2mDjKp|Rc9P{Z#GVL2@BcCZ2MV(`HDTQ{Y&ShG(OK0|cFsUnS?MWVX2tOY8 zVaYPQ&O`uvCf9Pzk_Te??IehuqP^bsKxH$-j*#*xJ2^&>WZb4j@q~KM{ z!QBE3rjzw)B!={+yh?#Yy&MMZD?CIPUy#f&p~a_gFW^OoJejif-lD#nx(4~$zq^Xy z-Zk*1(P>%k4!!+uwsftcqQI#4W-xMupWU^6D^wQx$X z_`$&@zTJGM`JzpBXm^%P$md$Rk5fam6@l#Px0_&%?p9(NYim9x;$g1L2M5auvBY9n zWB<1u-^K83Eap~rn*`nH6kO3@M$RzgT*dk=r|_V}7b#fBeM7vCE{}RZon+yvz5@d0gMfOPWDwAcDVE?D815uUDxhrk zW&a>qk>Sc-QOhPBi|EeXRD^=pMaS@$h$lCuf4Bm?+!fX<^bFjFn{}uEl5pxb#1i?Z zi@=&rI)%qb13F&vICTEqJtx%pZwC)#I`pIM{ z`PW=6?9%MTcEp za4}`egAGZ?Oa1wbg76)u&Fx!&iPL1=U@u3*?jc*}r1( zfw6m2Nq_zQUSI#7eWuIu)py!U07IH$$lFVJz{!J{o`f9iBp;=jH*s1Fhc43Guy|v>WlJJ1Uk$3h2en#W zQauI61DzszHyYeD8OfiURvLM$m;&=;LmuJ8+u`CuU@}rS?PVwJ2H5l~2_=h@z^u`S zh<^LHyz9(8;B@D(Muyw{-S}$f$Wp8GNmT||axP_zvKIq3E&w);(G&t3C+k(g69_0U zcS!wSkRf-7u~mD4I?;Io{G?ZkB^0|uhHhMf_$P>EMt^&Gj~L=3OZmJ${-x^N#+4es z%Gy{PHW%Y3HZ~hrqFGZiWBa_}OpMZilEgY7+0%O8r#WEv(d;(syTwrc*CDFvX3&mu z*!&mX!f=Oy`~&Srtfz_LPb}KHr&bIbVwPL<<(qrh-`l<(nYbVD{I^240rq58Pfhx& zZNbuF+xxtfilwa1sWv4zu{MS%_Ax8`(o~jQ+d~F#y^j%GWWRi9u+k=OVkd3t`Kwx# zVGpwvBS#mlaz0;C?gWf}nw}wKmc9&3eHb5*+WKyL;SzgS$NAf5I&r}~ApH}-a%0X3 z%4Yr$u(N{kvvkzuD%54_B?R!6(#lxY6h;x~{G(Y%(rkT4g#AKmBup}Tj^x7mBfB># zWj>k^U|J%`bUp~UwZvucKc5b~x{zLX_^Fc;U1}#P_+$~1PPwOIXC0I?jeN17za(Pz z_H>IMdwF#X3#tZj{;M{1@YH|`P12TAwvB}FYu_)zYYo<}&=bMM&H1z3Ku zh2u(Y9j%YIdC*$z&k7l!ne2)Im)t(pJ6y4Sbw|Z!c7&MM*+t_(^0gp_M6{>6~jIN0v(+R zfRrF9YkODf3{atg>)`wE?=Wjq%>^pLa!@zqOOj=k%gV6+Qr8fVRDeLPF%AV?4P(%Q ySjrmGtW|Y6=n4y=fs_BGH53A|6|ME=9F(>MWDW5sKz!iNw3Joyh$J5^l( delta 2932 zcmZvedpy&98^<>@p*h@=nw;C#&?8+&$R5|GdR7^|A>=w;0)2@_m8Vtf&D9;N4n`h%&*bXHAVTcK(MN1n4DYwty@tF_r7xXf0F%%$lE4Y4UPlsoQ5_FkIFoO)A~pAm-r{&Bit^Z+H5Qc)P33(WUH* zCR%r^P%sVNMd@4$iis+pa@}!@uiLKo#qJ69)7ASAk6qZ_lpd$r0F*kAvh#$)8t-^@ zw|~&$yofC+0A5_L{Q<97PA#!PZ6U-ZR|TZ-(JAJf+qLylm%dtTdadCbsENZqz&35f z0E24BiF z=N}?+E*~XbLT&D;Hqq7k8gtu?@T10Lt zssJ{S`+FP`PEW#$c<74qA1pW0P#C2-GF0}C+fVJMm5F^!ewMlaD~m4^d}Nb#m;R}{ zP74X&OZAi5#um_pQ;`WCWlW?an+*Ady33(w?(+ZVJo_3mtA3~Z1pxV_oc4NdE~+TS z=`=uj9nQQ^=4l)wCB%d$9pW*OWHy-r7PBb8!{}BRi-JQix_aWrsOY4_KK%5!JUmao zE3p=F#|`nGKfBsClGK^|FJbWnf3|6?Mc<)O@XD(?t=apvF7i;~e^6;u|34FmGCkyZ zMJ|n%Kvc@86u$G)`G$9Oc7g(Ct?zX~y{x)puWNlK#lh718^h)CPqj3O{so>@@z1qB zpK%+o532c|vtQ?p7R%1UV8q4{Mpwy)kD4KEYr*!R#MqkgD|W8W_T|dY?8*Cwili3_ zew>MH3@@HB7#;Ai>YJ~biLE(b=E08)@b~0LuAr%9mYg2c0t zmfzYdg+H#^KngLlP6YG}BEa5M{<4kdV3=Qh1WPCEO{vH)l!>OVV114v@^YKg!(?CIoC07A;G?k^mVHp&N zOZX5~?wSx1S`Nv>K%LVd;5EgSzLcxA9i7UZ@=`fcWLW6h)`0iBm~*QBAGL;2_+%r5DrFs1hhLxXT66e#Q}8Ib4+)) z^6K782NGKzt9B)u150({e&PJKJ9CJLOEF9tra`whh$eP0eMW(c&QTbytOXPYvvBSN zgjo1MTj8xzQ#x6BY2<3HQ)f%qk%MU+brIiFl)IvhdBZ%xPQClaM{!ZEKly&`l&W;s zC8wa8UqiYL4$0Vn3V>4zVj*H^ycFVwmTQESUY#}2m@;+;)H*?zdo&0TsXp}XzL#M^ zDVs=q&R%~jq1sLDn}I1ECgRPTzU1XF7c&g5{Q_KaFm(=x&w_yaSx{5A7fPQY&&>33 z>Ur8!IiXJae$NzVq!xMfkmyVry}R?}vTm$LaVcPMFfL_r3;9ipzPR7>m608LRafh` zi@=(8W(Lk6AC-mR(vE|Cj7oyIm%I9#&*wLvrP~C&wUX4hD7sCxCFI|v1 zaOU$)!S6YWd#kffdW&_D6vE88bmeaO7fM}_LKKWzn5R7i<fkS%FNl(R#(bt{T(;zwCJir1-JjEBtIRl++f0CU$a&~%+A8)Izv1Ynlnuc} z<^&vFmC--7N{vb3qAU9It{dio!8>0BwlBFce0JvZdoU;BJqAl%5BdVU?k(Jnmt zLpHGQZvp#*k`U{99fwM%`+EXJnGy1YVm-|p7{Rtw3Rim%OF+R@a`aR&$TwLVD*0I0 zUHY}{!=n{kzzVK2*0m(+6o$H_>!9bL}5N7EthuThW&FVvMv z#d{XAcu*=WwL_6&6M(UFS4qtT`rqzYpZTgX8bKoGXZ9N(#ovwQUQwY88TT^iO9bm0 zXzoQ`#Rfb*C$JK_=%Z?S?ZX<|6rPDbPpmN$E`9pew$k<<(3%)Mxj^6jVx5%?@Fdv#an|=~D4-69 zdTC?5b};bNk+k0F<0|HTCZdnc$mEhE%bt%$!oz-?YLc9M(4Mp_Ys=(PSK`vDNX%hN zgl1Zj=E8yF7Lr)x+K;yJ4a?uoo%z}SQen+!vN_ji&9?Anl(~xpUNj`^eE@7KUfyE4 zXOh%+*|D5{)xRskLhZBCSYUE-yJ+x$d*i|`_x_WK>u(|lwD(JKS{TR|HuQs9x-;r{35W}89X#w5tZGS4Kf3@B_bvm@l2wscIhy3G+MxDox8J;c* z5a4(Uw6|K8w0{3j%*n*?$HfQl66bu`tEr;Ck1EqhJ2`4p`oIb1+u^PyEwt^QM#M0) zylPms+Mrvj(R6$3jla}_a>m)ReI>^O%OYhtp>tw2RS_xfBR1-ykD;J%5EwC)bRTWm zWdDI^cQ_S(cDZFB%P|fwZ>#quJ0g)WzE$0mo7QoPGs(7O_(+aA+`6(SA5r}GjiK`0 z_Y#7$+WC=<@vj2IBpN_n@Y@wXpPJ>W3C4B%m`=_5!$E%IB0a6T5Zl@brlpD?Y|B3e z(dYJ_wN)k{@aLMHro&d29$(ajGbk{LjP>1<0 z56?AdBNo#s_3w``dl?%D-N^RQw}$4kbM);LQZ*oulXR2VxKngWEIUMBmA$C1F14h^ tR?^0%h diff --git a/summary_results.json b/summary_results.json index 91f27f5dd..63907b4fb 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.416695, + "Performance_Result": 0.420653, "Performance_Units": "Samples/s" }, { From 72b02b7df71efab216a32c8db147ff0a7084944e Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 4 Jun 2025 13:05:28 +0000 Subject: [PATCH 692/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10934 -> 10930 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 0389b18c5..03505596c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-04 11:53:47,423 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-04 11:53:47,423 submission_checker1.py:3308 INFO] --- -[2025-06-04 11:53:47,423 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-04 13:05:22,939 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-04 13:05:22,939 submission_checker1.py:3308 INFO] --- +[2025-06-04 13:05:22,939 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-04 11:53:47,423 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-04 13:05:22,939 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-04 11:53:47,423 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-04 13:05:22,939 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-04 11:53:47,423 submission_checker1.py:3324 INFO] --- -[2025-06-04 11:53:47,423 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-04 11:53:47,423 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-04 11:53:47,423 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-04 11:53:47,423 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-04 11:53:47,423 submission_checker1.py:3345 INFO] --- -[2025-06-04 11:53:47,423 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-04 11:53:48,688 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-04 13:05:22,939 submission_checker1.py:3324 INFO] --- +[2025-06-04 13:05:22,940 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-04 13:05:22,940 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-04 13:05:22,940 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-04 13:05:22,940 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-04 13:05:22,940 submission_checker1.py:3345 INFO] --- +[2025-06-04 13:05:22,940 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-04 13:05:24,088 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,7 +27,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1157 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71368 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 5.96744 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.50097 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a09b5d132..0718dd8e6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1319 +1320 diff --git a/summary.csv b/summary.csv index 7be78b929..7e831fb35 100644 --- a/summary.csv +++ b/summary.csv @@ -5,7 +5,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1157,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71368,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,5.96744,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.50097,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index ca3496c3c..3763a62a2 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1157, + "Result":22.9139, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":5.96744, + "Result":6.50097, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index c3585f7ee948008f8d68e7d23541a15481c0f5ea..95afe4125ebed9cce3646f8a0470ca0b0a9a4c6e 100644 GIT binary patch delta 3803 zcmZvfX;e~s+lK+m)JhbOmF6rsltUt#=A5RY;#j6Q9)pf1jyX?|b^!;Qp6AQ6KkN^CuXX+J;diaQcKrqBh3ZJ5{g3;fI^d|W zLi%`?Nxe~eZ<7E0=P{Ibi8G1{xKe@|#Buf8N#L91v4(-0s+oa_lp*f!BJty@R`{^Y z?rJ$Xhy+)XX@-7cAR*ySwH>?!r)Z;0h#$91jDr-n>Zgbv5UGK2*(qY;;YJ`^; zZ7_e`x_9m|u{Hj#DP=OekL(BCZG+}tu5g@Nd7d-d>l9{FTNfXRI5yiLxe(MjJsGif z|BUhOcntk3wP1hYGJ{;CYy|0wOQ$6ypHHS@i3&Fi?l z(NuJ7^@$6-7w(={++^#sFU5~&E|ImT<9vF3GhgIRX7~;m8PZh?q}OJJK385G)BWPS zu)|!>7?`*@7ZCb2x^*_$QrXWY0>823xLEYvr+ej01k9etx$%QJ+3Id-RPg??E8WSi zRi<*>&^zFth&BY0J(}61ERH|Dns1dl-hKcgeELq_>_}3hepDelQb%zvej{*dyeRz# zS;h1o#OP%7PdnL1g8?P?gxc)nQB8@66*-AjN4<@jTGyPa_4^q=%Hs~1sp7>WnhPkT z=l{+FV{Zw{3W$-V-xt8X3IRz4U&ViQiN8<3lHdz+=gO5P54o;u6%ye-mkW&Io*t< zVKR5xnO;B)wI2RDb*ZAjVT2m7_YG>cFyq%XwYi*tkbSeZ!#?8oR|MUsEQ3sSLCd7{ zEEqLlpVh|8Xp=p^P9geD$&P~?YX)6louI^Iy5Z1z(s)6aD1NK9%>tg6lCNFrnUZ_P zqpWJ16J_usXm3o0?HPt}WSGf5BVtEzZ>2?bc3sq`V7U#&OlWZnVP7q|(kM^;2vSX{gIZr;}Ui$7B+^*({ zUtAr_aiB=+;Aa~!Xf)3IXiBC7Z_M|{R+}3=ZAYr}aYY&87TjJM*X5AGP2fW{IrH~Akapdp)_`-6J8Be?)g3G7&1=AO zu39b#{?RfnhYqO)m=qqf{R5dfR5w8h3aY8h_&AXkfS0u|CrlCj|MIGn{b~ez+nrT= zOerJv;ruaScG}_}$}LLiwnvawOp-V@oj6zV0iax-eX=+}(3N zuF!(`rnXzk1u%3pAS}bjQa=PY(j;k3BM&u1Cz(FZ zYD=T@P`m`mW=Wj`AZjREV5E>r9JS3JP6dioW1EN{kwpMTHEy!Emdm?F!VBesyoa825yYIVzX4P!ym(Qg8_RZApny zpN{GgU6*NG?QK2x5&gXJ z5Z&-Pzxoi`b2X7!(CKVkfQk^5anIu4@}&H!!ibjA-~j5TUQ6i%;=8#w2lTP_RyM*g z2D6ge5kATm3uP>s-Rdfn!3XjCE(z%!=jo5rj=sPK(maerqQ#u97(pd$!-yEME|ED= zE`Z)S4n&|j%+@5WH z%`cF^&yA%9a*#C3GBhvo*mC+O(n+@Pq~uaS&;g52;Tq4{YWf~g<9a|!e405_z=L^< zO~FEG(4v_zn^!$52T7SqcuRJlg-zPNeVXEUM(K|1!>eiwJ@EKjSVb>~)w6>*pr+|u zSzjC7{xv4%w5J>Vz%Ty1)o|#UF*!mBzJ(>g17Ss z9W@(yhyRj2Y=+M9PnS+t{VLDj*RCa;gx$YQ!~*Ou_XB7u+r&(e2Ysf1Z;(goj%Atf z?|a9`ecTr}Bo+ridnF!A6`Mt|UGl2UVNFckN}qpV($IVa-c}`h4$M)=k?fSG0kx2C zay;m4`2nDR&}s@I&g=soa>wGsCH~^o1;}~Phvi;Y4R2d$G&i>Uav>F@#RRYI74(_v zCvdGih*r1!O7V9~_=CO_TfxT4!rkga;YfH(31{n1SwN1;XPdNj0ZnnKeLkLQd<~7; zAi6@1HG1}DHdW&}gol^>7N@-$ni%msrToxAJbwHGMrB=4sFB%7 zXi!<_$q%p&vUM~&huQ>D1d^yc$T56^*QR<5%B+sd(@~E|L5wJ~qEc+FJ^A@m#MGv) zx58Ug++sdH^Q5k~S_2JG@5qAl2mZU^NfxqGVd!))Erlibu1qHA$NAmHzR@pEtT6P5 zcB0fH_D=Vste-tQic(nQ2Jw1Um&g||7r<~B1L~u`fRFDV!XxgRNq5Tg1xH*%ox+46&9V z#y$o8o7a2$EZ+VbKYi5l(1S{Q?#vtH+3JXq?U`RYK1zxFNiZWuMx~uu_0Gds94*jX ze+$aO_YwHr_6R8b?)Cti-z}$IHMbrq5FiVns~m0@Jj~BCTqthQxgUEWaugFQgkyZ? zd74kSW1eD@FT^+`dVxA(9k~53W(rn-~o^PuD2q zC;EKqezw+eK9c{1!;`R*2LpqF+qY+_R^QPB7lrM!2s=GwnJewhH<{3)&uymO0L zyx;09-;jeOz9DmiEfxdCz#12VJyj5sCde0?@AadMf4w$ncmKI&_B}t?e7M@&)r9k_ z92;`?-P6%May|ObLVMrrXL%54_kSjd7ZX;b@GYeQq4&`Ow)8^^`XUNJa02cWp%50( zn=P+-RTS%k7PNIgMOZXBO&GAf;dD0AuT6e*_#9ja6dxB7mi&Aanm@rVTZzln-$JG| zy5D6C^sDlG9=C!kz{Et-{4}ih^`oEm-(RhKDOzN9ub1B0Yuw5*{B1Cpi45~wZGAt&BOPe znudpMM$s?mdwGv!%}eNw;&6oG}^^E7_2J{J_(9 z9^n?LPD-|k9U+-9iXJy>=8c>?|LT5F^jycWM$Ene-$q*rtKaV}27fC8lF&G13~)0N z?-p18dA-i8N*0)UDbV7YsNB5g?e{!@<0j*9QRL)#`1+AhwA0iKFVRryWfk{DWZjz` z!#(d;O_>#?uHdpM0^KgWL$=!A+ijAoBv*wst(;0f4PjrB-#8xj_MB+40_7M%ZYn>U zy!tsF_R0!nansXoPQB4@9tP7zXZy9bSM2FJ%My1OaUhvUKYZ#OxK>T21N8s@n?KaN zfPz#vbz@)_HA~&%aIy>l5Qo#g8WoSjUZwu2{s(ncT^jUFjtZ8Sq+->@j{hDt3hB;DGAyH~$ARmNByc delta 3814 zcmZ{nX*iVc+sB7tkYyN3_HD*GDVkK4hHNurtHdD0jHPCheMxTQ7fO~HOWAk*LNc~N zq3m0>vbGW#`%u>KjOTe?{g2~+zqsz>xL#b>=lq`E?|C0L)w>GeE&R9f}kjGan94&}NJ37veX3k3D#RXtCAaDzBNw%1vg&H+|51GPF0`4jRVI zq1oo5fkUz)omDf%w{iVOD6zIy#?l;Z=io~(xOoCId_Ww}X{(y2?I29jn=}8e&R=`8 zEm!|EzdFhPjA?vEou?+6SVwtBLMm+<*zd{fMBtM!oI@M!96 zZOJ~PtbXOkJ*%RGAe1OHUzp*1@u?fVfntcFmrVSoOfj@Qv&DHkCrCRXqE9^hK8F_b zSpaof{hQrGx0j$Z^VhC!Hzb{&brnBM0yf-&K3A`Mh*3}5jL>XRkA_|}XBGl!jFPdT z?KzEp)eK4+&oq|0Tc;-BK6drI*vP@Jnc^?RP*Lc})SH=(<)903>)c<6JgHbw3&81H zp@u}6X-eb2)~S;>I8J@4bXfkm)tQE^{@1g@_-6l_A$mDS{M59?)Arj@_^$)2BK|M@ z18|VwMXzArOLd#YKh~S_lFH?P=8!;zO{+cC)gTIuj9A6D-k(^@8yx&FSCy4KM;SiD zaVL;3?b-YVmvs2_iqY4km^!gw3)zFaKf$qVQ(Yf?NKFo6nwBp0&-gdqgrFEJLv4G; zf>Kvvvr+=KO|&d4+D+61Vp1-p2W@#w>&XAiy1n4)Uc`5kewJhP8slUbU@kiyhDHc@ zzjI8C0d3mA=u1Y9`L?5I1g`seVBmFR;EGQr{$y= zT7wtXz~Kw`(bjpG_YU~OEcAk~=DnX_KDlh#>ynd|?IBKyF}AA}aM{s4m|PDG8fCL3 zvM$ItCvv%L_G#Xm`v5St&*v{Mc3E97wyhqya4YGv?^kkIO_zgqU#Q>u?xAqA<0G#S zcl4`xeQbbagGfnTZ$i1AHpxO^qIqPgjPl@I!ij=K22tn~RnXoq%Lp5ig+!ZoV#Btv zuU0ctKbZ?#RcpG{Y>Zsr)7Jj-@pL|1s|~)~?>Xl@weiRK5`aG0m&U;$ZU$Fo7EO3B zI8z!V+p31Pi+6l|PZsSn;GPG5A0?A`_v$oLlO^m6h__1*P3HV-veWFO)a@il_3{1n z8y5m4eH0{#18UPxJTqO`XmSGG60w*SjDhy9V=!b1Q=hxZto6yP$z4PD_7KL*g|^*z z8u?c?p8OGrQEEnv47B6cFSU`R0jJ22h>T4RZ=W;tijeK4#&5esyUulG-5?PmX+OJ&#p5f_Bp@m6G7dG| z$J!qNdztu9Mwbpk_~$q+wC?%%T<+w; zc7$s1hZozWD+5AJJu4SrSNgUkxa5OtBwOHs2#A6QUFrKSSA1`FjZ&WnCUUPWf= znPQ9?@MqMr&K~CqxXd_`n|s{&47V1+ytvkGfy-mGvP0Q^&dufa#&$w-llaWCAYYOu>hn zt*@w3Ju}API?@@BBpas0q{fp^$w;)P5qqeTDy<|zeHv-C2Kk)txID^V; zA{*HZH|k2Dkz6?I%zjCA0CU&k{|}KaRA*K)3o#c-M~uc_mS{_vzE;;y|ZV`4rPY0ADzhfM+4?ZO$ z#vYc(!i)h=CH$sb>fRhE#f7fJ)vMzl{_6VcCq>J9da>+sa_sZZo5LolXBo6s1>mHd*;1ci0*1SMUO=9MPv zDXN^*MS?BWtBgz)%^>_@?71i|T~RjXEp$|oGB-iz#v{uc{NaB6tb=?MAR0Jr-^;3U z3s((PXYGGfssgH2-@(qoY$nl6b@!O+_8!}0s+-#Pk_BnD2^C|1jp{}-{r;M5C)!-_ zXia{{mFmx{F#ewPpvZQmbf+M&KgT@a8;#&@+Zbhv!Z>&yBwms3mm=$_syss@!4Bj& zg-TQ`x(wVh{Bs}hSPRQVz#ryz@)2~dLL{{oMYRKNR&71B4f9QUn~Jz0FyU{t=WRb( zSFWzAg>cA6Pn#?=oAfWUNj^9|W|O{?@!_VFKdqV|RkPs)2%YImb)i$;J*!`%JV4T#| zM~(7yQKmzm=Bz%P%T`9)G`#rxv$0rsoUr;V`2t#WS-?UQ4KF}4QTeGjz1^b_DQT@u zmSJ~q6@W73e;GgAV7fD__&OnvIWq5kgS+uBWt=lbJ3)Uj6Mp~Yfx|-nE@e3*b+gA2 zqO1m>l;PPZ0KgdGuI|go;Hx5cTh&|S1p~v)D7&pjEE;BW36PDWjGH+N_A&Xy+>dV< zj}Z?-r8Kra9S#j2T4md>`;DkF0HP39n|_!WM=b;yTm~uhVlLo}-gnEh-J6tWiFqpf z1LZ$hqX&viO*9kK4p3YiQm^+W^Aj^SFYTZKAg#->SGJUx8DItOc>tbGR;&t$h2RS; zrm?&+_+*5RH0-j5N)Xf0L~>l{d(RA%^e9WN=WsZ2F#O0i;hlWeIyVq?W&zPHwVtb4 z?7vBdn|7l%gwE+MFAO0cUhADy&H|jC93<4*MO2tNCf8^oF!|VN>|r#saCTyr3KI$uqs=Q?621zG(5L@p`n>s1)B8|3JqRcc9h#AZ=-n^2d0hXlBApZpU|D zCV1rseRZ-&ODs+`jq)NX?74Ag|NMq_WAu8vhFI%B!Jp2Vb8}C;-<&LQ8ZG#hGfP0wTh9`JKre^w$E(fb=k_z^-!#)i z2FM+7(X#~Z8av0TN?YLj*M&t?c14p<`qKpO^p26e)w=};XY77{BcNA}yfeQ76esdB zhj8{^_A1DUaXtWPA|!)g0f8)8K_CGT2!!`?we<12=X=`K%ZH#EL?UL9dvCX%;Wymc zKUVPe)KoBk)UvKU^>*%Ha_UMH%G;ovgKVb&IJL)TvAtRF@mtNTKl{Q~iXgZj>3_Kt zM!-InwY=h+w*Y&)s=+vB_~BZ+XTkC0c{aZC(&$K$_dxYOHl@6_r&|nJ3dL^~8*r@h zz8kLuV;0EYE%iR%QTZ2IR9>04lrb8)$s z*ZJJBk18CYHyXMhnb&)p{IGz?mp{DFkYFCyQA?r(jNe-zPlcBgI7d5{n`wR#^@*sc^LUKV2&=!2kdN diff --git a/summary_results.json b/summary_results.json index 63907b4fb..a4f219177 100644 --- a/summary_results.json +++ b/summary_results.json @@ -248,7 +248,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 5.96744, + "Performance_Result": 6.50097, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1157, + "Performance_Result": 22.9139, "Performance_Units": "Samples/s" }, { From 3bc2376bbf4952f10ef4abead8bd8a6ab1be9109 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 4 Jun 2025 14:23:23 +0000 Subject: [PATCH 693/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10930 -> 10929 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 03505596c..fb389fdd8 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-04 13:05:22,939 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-04 13:05:22,939 submission_checker1.py:3308 INFO] --- -[2025-06-04 13:05:22,939 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-04 14:23:17,178 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-04 14:23:17,178 submission_checker1.py:3308 INFO] --- +[2025-06-04 14:23:17,178 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-04 13:05:22,939 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-04 14:23:17,178 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-04 13:05:22,939 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-04 14:23:17,178 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-04 13:05:22,939 submission_checker1.py:3324 INFO] --- -[2025-06-04 13:05:22,940 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-04 13:05:22,940 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-04 13:05:22,940 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-04 13:05:22,940 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-04 13:05:22,940 submission_checker1.py:3345 INFO] --- -[2025-06-04 13:05:22,940 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-04 13:05:24,088 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-04 14:23:17,178 submission_checker1.py:3324 INFO] --- +[2025-06-04 14:23:17,178 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-04 14:23:17,178 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-04 14:23:17,178 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-04 14:23:17,178 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-04 14:23:17,178 submission_checker1.py:3345 INFO] --- +[2025-06-04 14:23:17,178 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-04 14:23:18,558 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 15.0091 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.6397 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0718dd8e6..4342bca2e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1320 +1321 diff --git a/summary.csv b/summary.csv index 7e831fb35..03236ef6e 100644 --- a/summary.csv +++ b/summary.csv @@ -11,7 +11,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,15.0091,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.6397,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 3763a62a2..2e334eb48 100644 --- a/summary.json +++ b/summary.json @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.0091, + "Result":22.6397, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 95afe4125ebed9cce3646f8a0470ca0b0a9a4c6e..bf7c8e806797c0d3c16e727c65089cd3fca3d0df 100644 GIT binary patch delta 2008 zcmYk7dr;EZ0>^(pQGBH0W49tRw180Yb|bE~=7c7>0-0}$N>hVYQp+qY(*)7A=p}(k zVGmq0(@+sjQ!BegDR_NhrKC5l)>oyaWQhtc&i--l`Qyx-IWyn+d_U)$IkJfA2x+pJ zN%OK{-}ssnW+A4ms`eVnV4o$qix?;ul*1G`su^cEv4%78497-qIy`hSP%!0}4&Vb- zd37VK1^0xBmUZkcavgTmZs4edIQO!*7WZBfQlGH1RmKn$93rcAYIh^F} zqT}trR3P}iu5Kp@SV%ao3VIh`MyNFvtB{KrDND>k#TGx=fcK#Z^PLNB$IFukSBpcj zy7e&)rPf_nEKXF{W;-3r9V-+|%3O{7JzBr00Fp>LT zZg!6SC@%Ij;uM?>4AWb8|B|1=yypEv#&buf6~))Fmm@S`A7k4~qH#@ySeWd&IrVpU z9cw21wXD%~{q9GJKv_Xm!UjxM=oKF>n!Dc0iEmy45^-h>6UbQxKtA|7&b&kc#1;ej zXfPlfFm|`SaVMNnBxzIHC@qY5l{DZ@%=DM9jCfi!3R^#cNE{`kC_f`AC{p4@x#0Vf zTD>z|F5~D7YEd4}Gf#~N7elc>UM+Mt zjpPXiJ)_j{*Z+8DXJr>8A|UO_nlp4>o5cvM4G*IAtH|3>NHmE zXKz2elwN}H!kfHmE3B1uz;2&xSy9C=l{1Z zv-Ysb&fsY`^c#0)`(fK4@)F><0q{Q-Dg)p(B!n)yj(837n4y_?457>Jvai`S@d6L$ z&1#i3k3dY-qARz)_>J83BJ4WVuXTyy&1|dxGsurwWgIwt=H%^fez>Ci%m8SQT$ z6q9i5I|^-#BOpQ#|C&7wakqsV2huQ~WViu5cZG4`!FFY5?;l!ZL|wsl>KlCh4Mbvn zt$o4R$ws2K`|hRLP^~kZK^i@iW7E#E=r90^kdQc0z5!6o#AV`F8UU-?a^pBte#)d? zWaNiRTFsQ^bk{)G@F@9+8{v87_?cAo7Bs3O&p%x*?0_jS6hq^j1S-#9uaB;@e&`X; z&+Nkhr3!8L(1H(zSn3k-H=n1$-~7RLRcG%rZT`B6o2+tz@YG(N;zuB5p0NRP&zA~kk_&6-qG!iHYW-I!*fXzd5!fs zu>fC+{EqpWElZDOU^B_gV`uuUgiD0`Q?HeG19vugFfB^R!;F|%MLKBmPCWq zmeGoZRhwHycqb2wB&Ld;(8U&`VteutDMzgtk$YHBQ?&(4-}3gD6~2zt;l89o zrA_Hy?54U|9#@=~)rE18qW@D#lyLw#j*Se92S@@??hhgZp*$<7O)W7&G+oT-D3Vax z0EGpV_h=N_wMy}o6fk*j_P~~=0I07MtZJeHcA`+ODV)Wx$?NeauoN45%aeN==j2V4 zl-Zb>cS*3RjE8iul|Qo-k3FMK=SO;o-r`k+T?qcD<4t{K#UC~LugM!PIjDuSjDZV; z>~KyAx_Q&U)`V?HzT%1A)?nd}sXY)RZ>k{LwFTzGddVorg9$>Ke|n7K#?LPt5ua z&X14g>dTG}vtCOxF~4;}b7@h^q> zXHy4BRFq|H3)_=yuX*qB_N)L$nT=2EyjjC9!v??uo9w#b3iVk$p>7F|i*#A*s(@fa~8peEj3yi)|^TvFvirrl;88s>Fz) z{+Qo?HAuA5=xI`(NbKF-O*Ci0JYQ_GAlCyEfo>PHN?~wXyD{N*QFc*wMj}HO%Sh8 zX5(5qE@+l5WFwYAzH*)2AXD0NSZU~t6tHg2|rciaz>W4Ln4 zn(y!NzNRJnjXs}eYT`e}+CFG+=&#}>_d+ZI91f{Q5ob&D=|};xbTY={Suc7AH^-9g zChf<2WT>m#>5D2b>atE)s0#GF92%E+oF7B_0Veqh%S^Xos@b zKHaOKCj|M_Ul|7Z+eHk9lOg5%(hNujz9u-Np_dw$rYF~i>M20{&YB(rFCJLx4z?yi z@}OWC)zXP{PwpC0SYW|7Qa2B41U0oyj*cmm;#Mz8^e}elHG7snh@}>uy>4>d>OY8l zt?~^w441C4o=FE0FTMo{(Ch5X90)}?d8aD}PDxj}RcdYD@;}rSkFOcdDIuHXl+V?x zrAb`~tLHIK#fNIben*)giyhdWX#NyjW?pF2i4cJ82`>vNLMcKCKFeHWIV5Ad4Q}GQ zFJuCqs^#8J&@p>bJGhRwecsX$t}v+P&Rq#zU!XeG|Aw)@%{H*_K`fMgbHm*2XL&+f zYDzZhcg7Mht>v<{bEcp|^o&QwvYMT~v3gjD^(QgLx!vFp%%|Zn5t|#gN}>%?cZJL# zpG0ji{;hqhWb(9kQ>ZIizPFgq0tEH~0*j2w0D%|+w8P;TwnojouS~@tvr15w8G0D} zss){$oJVJOY3hm(@5hd2cAiLJ)h;(L1w1wX^mDZA&b8ifE#V}5!E+{)=4!K%xAWH! z{MtEV9FGPf@s`@0Kn^5%F7$&Y)e-{}ipfek{SnjP0L-yVSa7HA)Dq!-q+M1`6Epfa zb|{)z*}1(zEWqqV4(yazR9sON>gJ9& zuJ-lgUqRX1F?J-)58SOGDkfhNTg_8VH2{XnLJ336fkg(Ye-2y z2bi{3&2oEMO(`&~ti$S7GePaVDM*Oc>4540JPa&(j{KG$6hBN$f<=nHjfCJvgK2g2 z)SyJ~lu(E~$y@RE@#N7%V;9o%}{sp4KAfaa6szgh2r(aGrx&UD3Az-^>=5X`FNgnzn-r zG;Qi#D_&U!Zt&v}c~Dvg^soI;m9@Pe*qEao!7EGHZ^f5zgY`K76SA#)^r(W@b;Fyv z&D3)x#BlzD+Xl#<|FhB}A1J#3j4Yz6c?x%$J8GLVF`USQoD$UeK0diI>#q2sxAvg z-P+HKa$Zc2%}y+sG_G0~<%$nf_hh}94$V%n;?d?O;{CH`KE64nnt*R0p0**@MtreH zmVI+3+$GEF3@^#!57zDfh@@J1G>(vd8WD|VS1>Q^fiL|`wR=AJuR535nAHS9ry!!G z>*Xt?B44(V^e`?)?@iX1#XCuDNZIG-Hukcqj2*t4kVcJN1G?%i@)0g?47r~@811Ux ztQIA^m*yHky<*sl)_=gt>vv+U%YDX?c>%fAMH`8Ueh&t!9alA`%X<#X52YJ)zlYs> zc=$@hfj&+CN^8^E-+IWUX3kWHB%_D?-I{f;DY5u;UrWyvc~nlK1?5J=^l@_M4?TBF zAARrBphtYzE?9_Z);Obwd8G|u8`>Up1_d0~@6oqex!Q!SH3g?%VBphq&89jsTse1#6Wrh@pX61t5dqE=K0@o%*KoPxr;NGpPeM(Z-jVF*|bDV8>3K%sQiew zj_S=rWYbKQC>gRRK7yS-1#jGomP7vff_aP9pE|2v$tZXRKu3mD1c?%NAhT5Z4l7qw`s5&!@I diff --git a/summary_results.json b/summary_results.json index a4f219177..be1e0a069 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 15.0091, + "Performance_Result": 22.6397, "Performance_Units": "Samples/s" }, { From bafa3663583001b4b464a5c7a5de7c6d23ed54a3 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 4 Jun 2025 15:00:33 +0000 Subject: [PATCH 694/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10929 -> 10929 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index fb389fdd8..231fac2bd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-04 14:23:17,178 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-04 14:23:17,178 submission_checker1.py:3308 INFO] --- -[2025-06-04 14:23:17,178 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-04 15:00:28,042 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-04 15:00:28,042 submission_checker1.py:3308 INFO] --- +[2025-06-04 15:00:28,042 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-04 14:23:17,178 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-04 15:00:28,042 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-04 14:23:17,178 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-04 15:00:28,042 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-04 14:23:17,178 submission_checker1.py:3324 INFO] --- -[2025-06-04 14:23:17,178 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-04 14:23:17,178 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-04 14:23:17,178 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-04 14:23:17,178 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-04 14:23:17,178 submission_checker1.py:3345 INFO] --- -[2025-06-04 14:23:17,178 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-04 14:23:18,558 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-04 15:00:28,042 submission_checker1.py:3324 INFO] --- +[2025-06-04 15:00:28,042 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-04 15:00:28,042 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-04 15:00:28,042 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-04 15:00:28,042 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-04 15:00:28,042 submission_checker1.py:3345 INFO] --- +[2025-06-04 15:00:28,042 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-04 15:00:29,165 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71368 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74168 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4342bca2e..0c9a0775f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1321 +1322 diff --git a/summary.csv b/summary.csv index 03236ef6e..a713ec694 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71368,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74168,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2e334eb48..8c69f5ec8 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71368, + "Result":1.74168, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index bf7c8e806797c0d3c16e727c65089cd3fca3d0df..5490e8c380666be49793d4a4d8b28525c9630209 100644 GIT binary patch delta 351 zcmV-l0igb|Rk2mDatH*8N-fp1b_iVv0`DlZ)es021i)nCDwE+PAb)PcFc5_AeTvBU z#7-y(SxJ!Eic^s)m8umt%U-~0{3F|K;O*;#5J2M8<9cSlon5(H98{D1g7ebo3gsC= z321JF)Vm6Ote5E=B|b1MSYtF)=m;Jy>W7kB%8i3JXDqlN!6y-|AS`w%R}*z-Lo z=CkO~QQR7*Scs+DVSmf`H`@U&2$^98fe98E?trNsiD*CyJ|gXpYkGj-7#dKZgU5N6 zp5mUB&&l)4dL)VJf)vw~%u@2W&Zm?R xTFf`y$eaEPPz^)=O*{imxo0H?0BZX}Z7?Jmu6=+yP@V644pS*@!ehw&?+!VQ4^t@(yQ7 zhOb@^R=UZCUb%5f;ZFu8tru~0ADtMDZQCa8tha{z8UEV5zW1xfLUpHNfU1Hs!Yoj) zZB^jgA{?%v&Tqnr_X7CSDd-b#=S7A-;86%bhPW5ybN2kQT_mF_PxHq(UBu~po6Si+ xBiVA-jlAo>0_Azgzlj&Xko*1 Date: Wed, 4 Jun 2025 20:47:50 +0000 Subject: [PATCH 695/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10929 -> 10931 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 231fac2bd..a32f3a1cd 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-04 15:00:28,042 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-04 15:00:28,042 submission_checker1.py:3308 INFO] --- -[2025-06-04 15:00:28,042 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-04 20:47:45,102 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-04 20:47:45,102 submission_checker1.py:3308 INFO] --- +[2025-06-04 20:47:45,102 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-04 15:00:28,042 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-04 20:47:45,102 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-04 15:00:28,042 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-04 20:47:45,102 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-04 15:00:28,042 submission_checker1.py:3324 INFO] --- -[2025-06-04 15:00:28,042 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-04 15:00:28,042 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-04 15:00:28,042 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-04 15:00:28,042 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-04 15:00:28,042 submission_checker1.py:3345 INFO] --- -[2025-06-04 15:00:28,042 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-04 15:00:29,165 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-04 20:47:45,102 submission_checker1.py:3324 INFO] --- +[2025-06-04 20:47:45,102 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-04 20:47:45,102 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-04 20:47:45,102 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-04 20:47:45,102 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-04 20:47:45,102 submission_checker1.py:3345 INFO] --- +[2025-06-04 20:47:45,102 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-04 20:47:46,183 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74168 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74465 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 0c9a0775f..f011551f6 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1322 +1323 diff --git a/summary.csv b/summary.csv index a713ec694..82f6bc240 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74168,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74465,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 8c69f5ec8..b434d7e7e 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74168, + "Result":1.74465, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5490e8c380666be49793d4a4d8b28525c9630209..30bf6d9084b803a77aea79a37195fbf09e2cda44 100644 GIT binary patch delta 1740 zcmV;-1~d7wRkKyFatHFxKO}=#R( zKcoy!u&ir1!EPBi{BiR8#w$C++9h0~LE=gChh#!&{*VlES;72a8Q_v+ ze2Jw={9zpcmL~54#)ODgBTPl379e zVHqHkWPnVQ^TRp-GEK^l2IU7@A~Fae6Op*L>h0TeI@>eaPHX=YQu)Z#zH5 z3$kA5mi%cb?UkTO{t%Cpen=*i(htcXniZrUmH{+L2GBG)Kdb{l)1>@pP=25#B7>kQ z3uRqahMOkwf?(6ES^4=|G@qo(*0RV-o94FjqwSVHLro zqFKTDVHrS^WB^T*^20g+G=ELTj|SrhS|WDY5KS6-AZK1Nu*eub2Z1hQdsR5CU zvZMz@Lda$X4~S)eO_GsMIBk;##5w>pO&X958W6NZWDxM!p{mQO`l{@h_O=9+@0gaWb;)toF0p`Y^htba0g;Tdqy|JnNPlGo4TxocN|FI8O%@RA z0H`!sK${yMs00g$T{iI8p{z^F5X&kS(AEO>9m^+)?6s_~5|$tV`K2F|vMm9r-~mhW z@gq_KA{k{#3y6de%L*0{%K(AFBjvuk$suP$IbZjEWd|}#Wov*&%@Qn<27dUQ8FK8 zUsj_RpU<}#_f96s;K${p7Sqw6i}=SM`agJwKlHtyes1WW8pn-%>G#omb<0tTFW9tB zj%K}^_&mHfzZp%_#yB443$EkF1?Q#FHOe!B642ZTsW&xxUoFxHl=#53V6D+mqdj;u zZ|*8?DK`#Yow4A81fN8Z_LSQiZ9}jWW6!ssn9rg^M{#YOVj-4pgDvCVYy-F;WQG+4 zCVyC9xCf?oB%&iy@DXW$Tsr^+$IyZT9X!sn9G|@bobopt2Ia!3WY}A{X+4Rfd*|h7 z?7A-NCWAHR&++H-!8Q|^ETY(2CCzbe-lrDL+ItDkdA2P>}v60RR9{0ssIJ i00000000000001hfzXplDn0=vlZYxK2G=J50001#&_4wL delta 1746 zcmV;@1}*usRk2mDatH;9N-foEvv&wx2!ALqD1lOyK#3-aCk-Hy38eu_8P+c05)Behnm;5HO7n+gkjo0@56b|T zB;!jgP2vyh0I)Q9KN`FrXo<)m=&?gtmz06W&bkJVUDxdWAeP@;WUpn3RYZ2Zt^If{ z9oSYGmre9~qe1$CmWT|3t}K*w zSs9*Ui5Uc^*qWuE$GH3^vTQAjtaOsywtlp31x`@%D<;}4Ars{hPf9-|6H4iaWRT1X z(hti3nIr>bnw%fj0g!1@el#dQ&=QeB2$`V#kTN{TvcKj72icmHpYKEFo_{(&S9;s| zDPEBELbv2kLus!BP4b6$r1V2Fp_G0|2GOh_{jdz6Niu+@$@yU&0GcM{M}zVMEfE<6 zO<5@GvNGH>i5CQ$X3fga*P{6(RkoHzR@yYTogZztv=f%-c)CRS(Ws92Qu!enWl86U zgb>XN&JW7~nj`~gnv@^b0e_%rGJZ4|KhP4f%Z6xz@k7dR*Cbuzu35A3^L^0VQ|0GM zZ!15=uIYz%**hjB&W}cW#Fx$w$tX)oKO}@~R*-&J2H0fE4!kyNl784I0H?|M(ct_* zOGKwBRaq$OvNGH^i5Ud@X3f&i*TeagBki85KbLx2{mJ)DOZ#On1Amp6KN=MhUz$H8 zqb#ZZkPxz2LH%JFV3UbEz@|z4VWR+>Ci6#w`2#HxohGme<_{^ujkAjRqiZ&Qz7L!G z>i=lz|0v%pdAezr_&+v^B);^2NJd#w03soPlVv2FEF7^9tOG#PqygEW0YOVd1_6&9s=BO7Zky~5fo;>?I}7OB zitM$lugZ>TZ%aV=j%m4CmmFv95(~&ipTw6I5XmS@YCt4}RDV{`fLI2oBpIO6WC5`b zfJ&1Ew7KztO0a;~Wdn~L%DSWsv8-YNZ7pEmv3!!qUd#F_VF?nDU-~gA+Y*op9R>P4JaEHNPns)`9HO+tq9=LXOk$?!=??yLA?JG)2* zK_CyW?X3@Yo5b^*_%t4m(?&nZFW0Ja)4KY^yGwQWVSn}Ea1Zf(ykt5U6@SUTH(q^| zT{fEDG}1p;ACl)Cp8l?_qO<<+axsn@*_T;-+>AfZ@_VRQY_l==JY0P|USl>NCG%1C zWi@*7`Fx9U?_`n;eq2s!F&+K6h=2T{|ATk zXx6)l&p&(fo6$6FjN@UxP^;xP8}ro%?k&I0lG*Yrx7qlZWLYw~|L!*K4dQw6UHDFg2p%o!hmu>$je|F5EVv-SClRDQ<+eim5G=*m^F1i$v*^%K+#07?h^5QZpZ9k5_rUE=E^9UZ9TMNWMnk{E!V;+~by$@9y4iRywB)0E6o^0>~Y7?coN%s1W0oBj(>4MYA-JOfU- zZ$Cl(2eTz6$^r$6N-foElk_NC0R@v#DLes>lZ`1s0eq9sDLMh}lL{&(8^C1ZDkT8` o08jz|01*HH00000000000DytdlS?W-0V9)&DkBEfCjbBd0LZ>E-~a#s diff --git a/summary_results.json b/summary_results.json index 3d0d17c12..2b7f7611d 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74168, + "Performance_Result": 1.74465, "Performance_Units": "Samples/s" }, { From 4e3cf6c09cf4db2f21a3bfd1f36515fcc3a8dc5a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Thu, 5 Jun 2025 20:25:45 +0000 Subject: [PATCH 696/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10931 -> 10932 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a32f3a1cd..9f7af383b 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-04 20:47:45,102 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-04 20:47:45,102 submission_checker1.py:3308 INFO] --- -[2025-06-04 20:47:45,102 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-05 20:25:39,630 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-05 20:25:39,630 submission_checker1.py:3308 INFO] --- +[2025-06-05 20:25:39,630 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-04 20:47:45,102 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-05 20:25:39,630 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-04 20:47:45,102 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-05 20:25:39,630 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-04 20:47:45,102 submission_checker1.py:3324 INFO] --- -[2025-06-04 20:47:45,102 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-04 20:47:45,102 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-04 20:47:45,102 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-04 20:47:45,102 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-04 20:47:45,102 submission_checker1.py:3345 INFO] --- -[2025-06-04 20:47:45,102 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-04 20:47:46,183 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-05 20:25:39,630 submission_checker1.py:3324 INFO] --- +[2025-06-05 20:25:39,630 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-05 20:25:39,630 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-05 20:25:39,630 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-05 20:25:39,630 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-05 20:25:39,631 submission_checker1.py:3345 INFO] --- +[2025-06-05 20:25:39,631 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-05 20:25:40,764 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74465 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73147 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index f011551f6..82e36242f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1323 +1324 diff --git a/summary.csv b/summary.csv index 82f6bc240..0d6922dcc 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74465,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73147,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index b434d7e7e..9b4abd36a 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74465, + "Result":1.73147, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 30bf6d9084b803a77aea79a37195fbf09e2cda44..f3f39bcc574383e62a5d6fffed9c7c3311e83c98 100644 GIT binary patch delta 1788 zcmVe0?-bVe}^wy%K|H5xo!Vw zJO8OuX8&kif+cDsp0s~RCY1IM$sm>$>>ri^ESXgUSeon~HVR;Al7BWgK41y*4}YB| zUl#k5TMPJ?V!5y8kKQ$ZoczA=%FeKM372S)c+&hKnNXTPB!gU5Fn?GE zxFi{0Vrde8SOuv4F zYw5tY%D8k&xI}-%liCl-gi`w<8Gq!mg7(8Qz$M85mnQ3nbpTwNq#q5^541#N5Oigs ztjo&q6idt?IK|d1{XE9yH<4v)S!AV??6&ozbt`a!l3y{=ZV8zvk9bo0A(>E0KO}=> zR*-&J2FN5CAk*ahunvGslk%fM`GJ;*3_{2R<%g8vL6-eBA2`U?to(c*GJp5f`MJ{D z&QI}ztQWc^e;P`AC1{dA#3Q92k_n~sLo$eF1?h)n08NqsG)>MA>j2O+DL)#NA83il zAZW@$S(laJrb)aY*feWae!dpXC#kZvEV9z3x$XRDyQQ75M90%5%8y2M#Fxqs$tX)Y zKO}@`R&ah;2GArKK+~lBuzwB!O_TAX!T5oeh+Q^B6O12HhPx)|8h6c_ji2v>=AJ4) zS9)9dDRxai-1=)ZyKI&?KN{^3UphY|qbw=?kPxz2LHc1CV3R33@Y<|N`eCC0oF?Z- zgYyF|5uK(~WudIg%5dK#W)SR~HA_EV59d>kw0o-lT84%c|JW#!_|pF&8D&WUh=c%6mXUC>jD(X-2;j6$0uUQT!fEQD zV}k$$EfJe0cd@tvRT0cVi{nQWaJZ0+oS=p4ggJ)24sT<1T7I61Uz=A>ar@iZL&KAwoQBQ zETC^Ive&Y{Dm$jVEdk{_rsZl~a-6kGEFc?w5?@+CB%>^;0e_JYQdvO*Vi};4WPnPO z1;jc4Doqy9=Eesq!2)8J4Lo)z>yk3WvWf+?wSaxc@<}3lE$gdBppOOF$}k zz><9Yh?IaxMp@DVA|b@Gf(67ffF<*5;2>+#fY>O&rAY&_K?8!8h)q*|)WtcJb!izK zWGnpnAA8R=pnq&wAgP|@|J1U!O6S*Y|3_P`FwiA0%Gyo|l4z3n(*GeDWk~^ugb>LJ z0uajpl1!}uBuxeo8wHRw89+7|K+qDgY08h5Ift??EdwMg7(n)(8$fF$!zUrRui}sG z>>?Qifjqpnw?5o$63=hq(|9~i8~r4|T&v1W>*^ElE`QbGht-F}J;d|zlIdVn{3ZL| zc=b_s*=TyxNdH`YNS=3i`n$G@&icd4#W-$cUuN-fGyXiw@1bI`&BoyKaP{$cjoEyZ z%tzUm)#%0N^DV}`lSwl8aXG2QboA#U{_%(Y58mMqeeb8A8~UflaU);)eKcR)a+Kl= zHm#GRSwrt8KJU$MM$@!0j)(a|t(MZ=1d}T%8nYx8x(JwSJJ)I(hpl zPU0lsQ;(~e{dRUywb-dH`32{t(KX65f)dc&2&vl|y|0()14?{gTCmP&sL>ugS~PbR zx0HVy2d~apa6y7kB1n76ZH=}eSchQ9;((e;aLOI9U|dL3;p-wEu46u|;)ypB#N$5d6K|(Qj(s4K z7=Rw*o|TW;)AM?Xnt~K%O6Dmk*9DMirAYwNAOJy2M5ifLSt#qWGK8{%0HkXcfOeStJABz%7FY?(ZTm;t z`A?lvEOiN%sF8Tm{vnxA+CLzchE#PXYq?6oYhipZ|FwI8pg z1KTR&(kbB*{Si-UKO_@M?T2KL%YO>m56b|TBm-QUtRL0^aA}f$G)Owm05VO=j|SxjS|TzCArq7zQicav_SbyiAX~HY^L@zNQ-9~@N^d(q z#S5}t=$8CxDD9P?N&XOzlzvDil+q8$Aet4VAC>_$Ne0j~IX|oeK+~lBXi$EjB_e~M zDGOy?R)(7<@q%E}tXcW_S~Q=e%GR>TN}J}k^P}yScES=JPnRe^8r2bBDnBHnEb07^ z5TaSZ`C%DAlVku*lk&ql0Dm-1#*YT$2U;R_*$_=Ien=Vanxt#oHETA0z7Lvvs{CB( zZRMxfHT`hwuhs0bS>pU?v`2jD{E&>Yr1V2V$YurUhh>0GrtH9LvnJ_>jRJ6*oF5I& z541#dno^a8vMwvbeUq3$uy58Z{d_%~PdU==srqxNx7D9~-?X$}_J1-^iTR^ZA@Qa8 zLo&*e>JJGan-$a_mH{@IxC3mO)E_nquxT=XG?+io647Y_n_&KsGTb<;m_NE^^XL1p zxv&0@mi~|Oy^^P!c8ULEqe$XQ|A%ChB?TZ70ytSl!pSlcPBtNc(>4h}Y!nHnse_IU z0uZ!BY?|P)Ls^%W$$yQLgka;e_sReHyF~U{7FlKEw6`^&d~Kn!%h)W@fNbj0=USwNc`AE*Qih+Q`D*rBXT$`H#c7SPrL_8rS7iR`tkuM(CZ0r{mLld>%Vso()i z^6?{50wNh@NehUC5X%Y{5X%6T%&&oitVsi6qX3sC4af!!2wEaGP5Dt5=TO$AWpI$K z@aKQ*J=cJ;VSj<7dXoQB%i1cPU$^}qZMDKcm%J!zJ0(b>N#aZYhh&r`1t1baBr6C& zECWa~wFZzh89;0lK+$`aJNZ3zll%d@i=Ytll*e6DmSgGPrSQShkqYd9}f2r&&NxqgHiF9?0e(Y zN7-eg=}ja3bM+y4-r?!*+A2Eh4=)$vxRHIC#mCL~^DMuIip4e?gU`d&$Ky3-^HDM% zWnWgK7oX3!823&l$>7K3q!!cBpNsg%ANoIdhd=bapMGxWpBl%FeChYme09rFiZ9r- zPL5{1n?3lvH@_K8)5bU+<_oo2ezP%Oec;~m>nxcqzjB+6k4cs#ll$*(6Ci-ZsmJxqemlEzHQ%W=`32{t(KX65f)dc&2&p$UdS5Nl z2bB20v|z2#P@_F~G;i)IZYehoUY)Vvf&`yLkoJ_@8f`V0v$ZgvmBqj0i5zT8wTaV zsbtt&xM@9!qkHG&XzaQ!>n4LW=Fjox^5tzZ8*1>>3uUl#FjE%RX+ zPrQ~O9`{kd@pfF~&<7%k0q7y_N%@#OJ+CDes3}M>P01`J<*FdGyr<>#s~>sQe+H`K zkbe_TfJ5%$e00000L@P$n diff --git a/summary_results.json b/summary_results.json index 2b7f7611d..88554d0b9 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74465, + "Performance_Result": 1.73147, "Performance_Units": "Samples/s" }, { From e13c22cb917a8cd3007ece18db96abb613fbe8f0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sat, 7 Jun 2025 07:01:14 +0000 Subject: [PATCH 697/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10932 -> 10932 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9f7af383b..e86681deb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-05 20:25:39,630 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-05 20:25:39,630 submission_checker1.py:3308 INFO] --- -[2025-06-05 20:25:39,630 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-07 07:01:08,618 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-07 07:01:08,618 submission_checker1.py:3308 INFO] --- +[2025-06-07 07:01:08,618 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-05 20:25:39,630 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-07 07:01:08,618 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-05 20:25:39,630 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-07 07:01:08,618 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-05 20:25:39,630 submission_checker1.py:3324 INFO] --- -[2025-06-05 20:25:39,630 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-05 20:25:39,630 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-05 20:25:39,630 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-05 20:25:39,630 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-05 20:25:39,631 submission_checker1.py:3345 INFO] --- -[2025-06-05 20:25:39,631 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-05 20:25:40,764 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-07 07:01:08,618 submission_checker1.py:3324 INFO] --- +[2025-06-07 07:01:08,618 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-07 07:01:08,618 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-07 07:01:08,618 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-07 07:01:08,618 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-07 07:01:08,619 submission_checker1.py:3345 INFO] --- +[2025-06-07 07:01:08,619 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-07 07:01:09,776 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,10 +25,10 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1965 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1919 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73147 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74858 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 82e36242f..41f48e56e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1324 +1325 diff --git a/summary.csv b/summary.csv index 0d6922dcc..d31351de3 100644 --- a/summary.csv +++ b/summary.csv @@ -3,10 +3,10 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1965,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1919,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73147,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74858,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9b4abd36a..c18dcfd8d 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1965, + "Result":23.1919, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73147, + "Result":1.74858, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f3f39bcc574383e62a5d6fffed9c7c3311e83c98..384abee804cdd51c6481107fef16443a8ba15b2a 100644 GIT binary patch delta 1692 zcmV;N24nfORkT&GatH*5at}MRb_iVvfA0iaZ+7fLsY;+klf;t-5Xpqn03sQLvVsA` zG5{qrYqW&YWB{>I07{bpq(K0JmWWPMsl!?EU922qyctO?+-LfZ{UJ07y5AjIphh#!2{g4czSwZ??89nE2sX`{m7lLg^GT{~EsLzQX>L0|+HPqlEYb0FiSnaS9r2~|Lo&*e z&JPJ8niZTMmH{+L2GBGqe?P1PK+|OWXfS@DC1RHi(FEg%l;N&Py2f3zX5;7kpt+~Y z&z0U*eu`bw5ACvdOiG*|jrNEyogb1>mXv--2-&P4{jdzM$&?*jIrJt{d^C?H#Jym}$^|tzx@0*tPf6HD5DlvaFDkQ!% ze@I4IQvD$zWV3?$!!p1o6L)}3llsF(0X9wMj|TGxS|U14U=z$AQidC674t{eZ2o*7 zHuu&4(bE4>zE|>e(=PFUY!pd+>Hm<7vZMe+LI5YrNH|$W!pSBCaM~sTh>arQG&i5h)okbb|~x8e=@mok`Qd1_CEPPf0xK!%Ob06oc6W`l&>vRb{U%`8jy`Xi7z!E zl2MlQfJg}0tl$B$46sQu@(HJH(tub8fTl?UvOxoamWT`j9y?TZS(V&2*&PDgroDF- z(6<%YYgu2F9n;>Hfbt#Faz2_QGe>Ncin4;`w;VbTBIZl6`Ny z`Y5|>G`(r0f37|x&pSN*U0X$G{o&Ad_tVb}{Zr$(kuUu|ny+p-O7R7o z*2&SVKX((K_vSaFY1$aa!+fDu%WpR3s}I~;ew`(=1&{5nPr&x%k ztFeD&{F~K)Cxpzfg1`g|4EMm)jzn}s3O*w3k81{i;20WEpo7PGmgBQGfK&cv!=PL^ zl?tenTQb_iVve{UM-hc8>p0xMy;ZU1OH z|EW`E|7cx;C2Ay|w0}q@l=ctFAeI&EAC>_unNOKhU^% zBpF{~X%c@}2Y{u?`_bV2KubghL605Ex}*#|cGfj`?7C*}2eJI-B6}@MtRk}OZSBWv z>A<$ixO7UmM1RDS+7HQuQu`qpf8?@)_QNv3CCLDnChLcF09=}+9}UtEv_xbObY-Ee z%gXQ+OUxiR#nvqSJjUfWk!5RHWTlhrw)LZRD{z96Uop{c37II5cvAWynNUhUB!gsD zkbYPO$Rrsc)8zcH4uDLP@}oidftH92LdXQ=hm_$#mi;v!ILOwl{CpoWfA`e+xzgLt zPw|4R7rG^X8cKU5Xp%p~Bc&gb38nNyGKgjc>4#+iO_BjLP0kPN0MIlkKN^%DXo<)m zXv#uamzCkBNxUG~G;3CVz81|Vsj{^!veKrx?fhuFrJb-u$I~Usk4AOGm&y;xC`&p& zB!p;IaDG?@&?Ff^)1>^ce+~dmlkua$_<@#)T{c7$j2}{lyC&%xcg>oOpYMa_o+>|A zdRzG^c1=Ir`fD}2Y?e4b8toBZIzJ?%EGhkv5VBc8`e7MhlPNp!+N??XVWR+?Cg(?k z^8+mrou*V}p{&cwaNi_m5bT>ZOFv%^=TnZfd#e6i>TUHW-#0Dof0w-sRAT;UR7iYj z{*a8ar20cb$Yurghh>0GChh>6CiRDn0&JSh9}VUYv_y28z$TbKqzpICD&~)_+5GuF zZ0@W7qox0&e6Qr`rd{Iy*eH_t(*GeDWk~^ugaA&Kk#Mq%gp*AO;IvHw5F16pY3iV3 zg8&3A5t}A>>`>OFe`RvxBq7*1?S1lp{w|TdmPJC|_Hs>@qe>G$0#&5?^XS zB%>_p0g({0S-}Hh8DNuSU(*RsAUJEpxY0p&ZUfO%~AR#s@0F0%Dg9Ja#DSk}|}yiUqW_fPKgENg{hK>#Kw%NI-t+$E0jaKq`2^ zl6?G#lz>P^S<(U`A;hwR1;jFdCG%_GAZyZq*eJlINdvM$1A>-_O;di<#W|F9X&D@3 zEByH%d(SnXe{5JFsh;Hj)Uvip=htojM_a8h&?PU*+D-|QXp;ES{~;M=Ndbt25XlMx z5X%6POsxSVO$HDf1&}luKsFda&=Rp}%8!;ghq5j$10*XLK=z&+Kx-t!Cn347;*ag@ zA{hjMJiNBIKHO~*&u`+>csx!U{UpC!tIAF5>J#rSf7RiK)rZ49#Pjiz>0ng+CHvlZ z^-*@&XnNB~|6F}Yo_BcqyS9qX`oqh`IBsNLX7O<|{yfX?p<=Pk#^CdC_3?O(*?g4D zN7jdK+qgG~=f!vV z&%0rgWcS|`mt^z7g(rUjvj`U04g_h2?-7&aBp`ok!!Qtp?|llRyDizNLlL$EZON&D zLQ`6L6Vk>JwSJJ)I(hplPU0lsQ;(~e{dRUywb-dH`32{t(KX65f)dc&2&vl|y|0() z14?{gTCmP&sL>ugS~PbRx0D+Pug+L-L4r>rNPEg{jkY0Jim~TgP|Ro1p`*AlPO%V6 z*J6Lm_%~|-7lh2Qg1`g|3?G209f|0K6nsS5AJ+{4!7+58KnIWWEXNmb0H^$9!=PL_ zl?;0elh(61x_4fV#=h^fel}QR{v3ahQ9;((e;aLOI9U|dt->mnYmV?M0n zi8m6&<38#WZ>L3$eISw;fF9$Xm5 Date: Sun, 8 Jun 2025 08:20:23 +0000 Subject: [PATCH 698/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10932 -> 10931 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index e86681deb..a0946389c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-07 07:01:08,618 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-07 07:01:08,618 submission_checker1.py:3308 INFO] --- -[2025-06-07 07:01:08,618 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-08 08:20:18,618 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-08 08:20:18,618 submission_checker1.py:3308 INFO] --- +[2025-06-08 08:20:18,618 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-07 07:01:08,618 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-08 08:20:18,618 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-07 07:01:08,618 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-08 08:20:18,618 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-07 07:01:08,618 submission_checker1.py:3324 INFO] --- -[2025-06-07 07:01:08,618 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-07 07:01:08,618 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-07 07:01:08,618 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-07 07:01:08,618 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-07 07:01:08,619 submission_checker1.py:3345 INFO] --- -[2025-06-07 07:01:08,619 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-07 07:01:09,776 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-08 08:20:18,618 submission_checker1.py:3324 INFO] --- +[2025-06-08 08:20:18,618 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-08 08:20:18,618 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-08 08:20:18,618 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-08 08:20:18,618 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-08 08:20:18,618 submission_checker1.py:3345 INFO] --- +[2025-06-08 08:20:18,618 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-08 08:20:19,609 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1919 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.74858 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71154 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 41f48e56e..2d14433bf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1325 +1326 diff --git a/summary.csv b/summary.csv index d31351de3..97b45b45d 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1919,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.74858,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71154,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c18dcfd8d..9badd5626 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.74858, + "Result":1.71154, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 384abee804cdd51c6481107fef16443a8ba15b2a..6fdc5e6b674abf3275f2606ab52563e2f5dad95b 100644 GIT binary patch delta 969 zcmV;)12+7$RkKyFatH-vT4OG4vv&ww2!F4XTLz^nff7v;PZ~fZ6G{V!WDv><1`x{t zl+3Kr5=xT+#6|%qO#+Yx0SH1WVLNJZb-sOepOil0hsh*gq@-STd^yur%2}Y!txKB>!w~e83XqAAdSc zz!KyiQU)hj)-{}9w-)d(#d2TGAH8e-IQf0!m2H=H372S)c+&hKnNXTPB!gU5Fn?GE zxFi{0Vrde8SOuv4F zYw5tY%D8k&xI}-%liCl-gi`w<8Gq!mg7(8Qz$M85mnQ3nbpTwNq#q5^541#N5Oigs ztjo&q6idt?IK|d1{XE9yH<4v)S!AV??6&ozbt`a!l3y{=ZV8zvk9bo0A(>E0KO}=> zR*-&J2FN5CAk*ahunvGslk%fM`GJ;*3_{2R<%g8vL6-eBA2`U?to(c*GJp5f`MJ{D z&QI}ztQWdvPcpp{G|3<0kjD_ zFfCYXG*qYukCyda$t~r^!K*VCT#(?C2-2Q%TcKSDmSXJr4ixiQbm%BcMjYDXeEAq5|i_Q$mYKyVB#DA2*#(b#oe*3AZM%%9`W^~>9^YAW^NR2)!U3QoBL7L2P)d|kxDwakZgJn>e7 zc78|ml1)=%F*J0#M{{^T{L;g)X z1CF_GKSBKmvnD3W0tIARV=irz^(b2b2a{1LJOPoDjwwL_ev{BCIsx#L3o0fXkH)&E rB>?~cPyzq|5dZ)H0000000000fPvAIOe#JBC6kLPBL>$e00000qp-PH delta 998 zcmVwC1)@5Y~Wd#99*DL_-F!^`*vb8L*5|-Qc zkGAulI;!NicC{|S5;YP}+CL-{O8bXo5X%bo56b|S%&Gw_P4*8P1+X;9KbspLuzv*k zhfWi)1o?-Q!3mai4JX*G1^i2~+*k8Q@0veOe&2XyXIQ(0OEgG4Y5tH*D9s;|K`twp zKP&@Wl8i60G>Jc~1HjVc{b=xhpd})MpvMknT~Y=fJL?)ec3rdggIIoZk-e5BRuS3t zw)W$-bYNR$TskFOqCetE?T2JSsek>D402gP`(YX2l4O8Oll8+o04`0^j|S-nS|TzC zy0TE#Wo3AZC1wzuVr!Ou9^>+x$g;I8veHR*+xpSE6*xi3ub61JgiMr2JSqKxv82SBDt`O%>KKubghA!LH`L(1?V%l?`V9As-&et*6XnS1K| zT4#)ODgBTPqFF)uVHrS^WB^T*^TRp-G)>Bn2IU7@ zA~Fb?vQXA#Ww>b)F9_$Ne0j~DKS5+13=Sc{Ae(Kpe16L4bcSShm_&2NxH^evu5Mx`=Gg} z%FmVFR(^_I(+};kcT7sN{}SL4f5RJK-6a7408jz|01*HH0Az1tP;zf@b1!3Wa%C=f zZET&A&2GXl5QOi2ipclG4ye#tNs!u#Q;{l_s;D>1UV_#5N4DFLx33c*fW)cC^~`=d zyK=GEt0wsc=cUml$}@rz(A)^A>k@sem+1pad|+Cz#%L%}2Oce|yMkNFe~p7TXDqlN z!6y-|AUyAUkJ*z+AI=CkO~QQR7*Scs*ov1R<5)qp32%&>yM1PcuJz|@XJbVLe1 zBJGcB27urg8c?8v$9b0Hvp0ZK{$|6VTsV~sorRm$lQ_EfUXI4LZL@YdSY!SiZ&t7G z!>Xy&{i!&hst}xV2P_y@NEP_9h=*&K536|MtpxGZMg7LxagjqGh$IG}hqx!@bNc+U zUZQG3Cif|srDVP)b4qeb9>4mLSN&(8Iu7|a@dP;JzWxN&AG0MU$^r$3at}Lhlk_NC z0SJ>&DLes^lZ`1s0e_RvDLMi1lL{&(8^ar5-6a7408jz|01*HH00000000000Dytg UlS?W-0Vb1*DkBEiCjbBd05z7ym;e9( diff --git a/summary_results.json b/summary_results.json index 7e8e45819..e502f97f6 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.74858, + "Performance_Result": 1.71154, "Performance_Units": "Samples/s" }, { From e9e186d437714c2d4eb884a7d02b20ca9cd97b5a Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Sun, 8 Jun 2025 14:20:12 +0000 Subject: [PATCH 699/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10931 -> 10935 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index a0946389c..7666be8c2 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-08 08:20:18,618 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-08 08:20:18,618 submission_checker1.py:3308 INFO] --- -[2025-06-08 08:20:18,618 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-08 14:20:06,767 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-08 14:20:06,767 submission_checker1.py:3308 INFO] --- +[2025-06-08 14:20:06,767 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-08 08:20:18,618 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-08 14:20:06,767 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-08 08:20:18,618 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-08 14:20:06,767 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-08 08:20:18,618 submission_checker1.py:3324 INFO] --- -[2025-06-08 08:20:18,618 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-08 08:20:18,618 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-08 08:20:18,618 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-08 08:20:18,618 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-08 08:20:18,618 submission_checker1.py:3345 INFO] --- -[2025-06-08 08:20:18,618 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-08 08:20:19,609 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-08 14:20:06,767 submission_checker1.py:3324 INFO] --- +[2025-06-08 14:20:06,767 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-08 14:20:06,767 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-08 14:20:06,767 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-08 14:20:06,767 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-08 14:20:06,767 submission_checker1.py:3345 INFO] --- +[2025-06-08 14:20:06,767 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-08 14:20:07,759 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -41,7 +41,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 6.50097 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.93654 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 2d14433bf..d378cb0bf 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1326 +1327 diff --git a/summary.csv b/summary.csv index 97b45b45d..863c0cd07 100644 --- a/summary.csv +++ b/summary.csv @@ -19,7 +19,7 @@ MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6 MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,6.50097,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.93654,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 9badd5626..97e02a4fa 100644 --- a/summary.json +++ b/summary.json @@ -609,7 +609,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":6.50097, + "Result":9.93654, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 6fdc5e6b674abf3275f2606ab52563e2f5dad95b..5285713a48f5d3584886925af011ae3339f73f42 100644 GIT binary patch delta 3858 zcmZWsX*d+z-yUR*EEQQt8e4rC**&0_Pb1D>5)8p5Qkj_>^QX z>T9$y6DTFI)+#TzAwIPTWv%PWjgNVNtZcvSH91bkZDyuCcvY1AJiRuU z(tV@OqUH9M9xi@oFU`1_JHk+bbJ!Kx{5h!(B-+@oCMIf74;EeF3pY}j#P&TfF&@L- zIn_J);|fUO^-uP$;!#D>fRHcv_?lXY!zDk^C6Mw&x z7p<6ZMH-iEDqcxbl;bP=d2U*1@*eb&15?`Q;oLrbNA|~cm@z6w3>V}$(4z2Wu1<09 zZt9z`OxCaI-v^qQ`X>|E;BL$fxwyVvgWz)=_a%VNd#Jt_Pu8$XEJvk-yW-t(L*#j_ z7jsJVE&as)MSU0avnPF%zX&(KyiqTH-Z>Ta@Z(*H!lI3b!L_e*+PpFG$kB!VkBtR$ z3;RByJvr*f@bl|y=(C-xQv;3qA>$SWO}%DobF=bY;pBjk-T8knVnNayejyMQIK7oM zKs->(8v1ni=G~BXFTIh~8>FDQpr7bzbA#a*w?QKPx-ESER~7IV#8Ssnqvd=GAq?kRDg%C0nAf2@=IA;7m>+jbENRWeJ)Q`|I#%(U$j73WHn;(Z;C*@n$>fSwwMWE#Lk{<-v+H z0c8wdDE+U|-gh>EGD3QZr>V-y;ZIW$2JPl&IU)FX6gJfulz7UD80Pg(Ku8N8&eC2? z%6oTQz_trz)O;mn&dPB>@vqgaHs_DJt1MFoO@G2oI8wN^-a|NtYuP!+Y{B6sH=kyc zI$x8$SH$bv;|GPhNgb2A8l&9FlQ%Ku(C9G19K4_i11g3jre4QC!E3&(8+mjpoyLo! z6vcJTVq1kWd&GLJn$0oCJY1~7u90YF?cn5GU&|op23L37^9n?eNL%IZX+4Xng$thq zRy$XUIZ8eLIms7U{bG2#eIzi(GQ%LkcK7$@9RXLr2QK_zPtL0`$eY!?T1cMvAW6XS zb2ZNQFQ2&H#}(uKJ&!ycA$aHTKs0{=WPu9B4xdNz(K}1I7tdrmM|c_*(4Btpdkt2! zs*>iy-S6%}gVlc38jk+H{!RMAYo*@i>tC;FpN;jx3+5uBQD9HL?2$*{d5&J~ z^`6gTTTul*xbMnQY;Zg@EIK{J*ICSL4~VFIdfxtFXUN5h&Pvmv(0b|`1j}5{`tSg& z;NnWfWxXf(xbu7lNxBhGQD&=`G^yUCVbDc+(qlP&k~DBQO`X&#Bu@`W5^?<8 z5wo`c=8647bPR?Zs`1r7;Lu)hxJ~ix|2z)ky1L^r>WNE#X?f1yhynJDK~Qm$s6%>q z4p`PTE}vR3EX$kAM=Jcg&waF;4c$06&629ZtSu3&HWU0tpWkm9Mt7vr*}6wy<4eB% zz7O-9AA8~&$d-aD`R|RO(b{1qJiXYGufNI#a~TjhM1THDxtxd@fk8&hL=)qma(cyd z8g=-Of?THPj)gsI^*v*MH(gy)y`)bofmWF>>0TP-A}Pm*S+AbXvUIuxiuJMog?8W> z23a4adIv^22)7p?H?6E&^wABvNFL_raG{b)u8nI8VMo~OW1W0wc(W& z5)+{r`(&?H<~{rl^~jtNP5B6;g?x z?DQIc_;Q(4p($ffWjp>fFML=ZKq1Rg3IU3>wAYGq;K@aKCJ`;O1?6D^2yd|{Z9W7$ zP2ShM#XH!Q&nuA!%x81w%7PD$1<{Gz_8$UA_j^P4ZLX&8?!O*Xba;7X61l^6!kg`c zyJO@2Ev)2wyyN*O3S<8k89)TH?gUM75#2k@$Si(fwqqi{Cvg{H7@{-L92l^FeX@%!`4;x{t8yo^g`K zuDb@3r$*ZmUc&XY547i%f?*duHdpPahl~B=LI-!AMisi}{M=cM1{}nhsi5{8Rb2`! zb&fuswwr(pHElP3*i7hS6SUqwEeWcOS#N?OzzjeZcrGpkfXYq-3Uvt6u*N8n%V1@A zzMwF5ae4c_A>DIkZYSkUVOMwhL?|Xf0#)Cm9)X`v&qeNolK-?N8q_$rp&C=J%OT^F*VzL$)%5bSqqOPiN zEQk!5-%SyD>Db7+C5a%Q0A_?$wtw1Yt;sx7Z2v+(i-$->9aL2PDF^+9MswYOTH;t_tJ5nK zmJzfA|3jGnH0k63TQV%GhRY{=003#l}O4{2I=C#e6kQXZ#cyhAq`sM!hR-zscNwwV1gbkfSFWvj-sy|rmKb^Yfc485B62@NQ5aMF2V#5KbA$u zmLR#@$$$*7P&Xh$YP&9do{68x%gVgvC5H1sF6;Fz`B-=6T%ZC+&B}Dt(gagV z+yI{=?gL6w1#=!vze|AAD-%&7M^=B0Ol)UXW03|8IE$$jUEg)WZeXdZPHA!XySKJ+ zKzrcvO_A<4(}Gy56`iGssZVhu9^`qmGkZb510GJWe*z<_`U}FE5>qv)ky$Nmom*>= zhPZ?lm~?D+(cuX1C9nAKF>H&|{z*j+0Kk$H01yQL z04TrPmVti$L7KPy0zEXse0`qdJMVwc5_z;7nb5x3Z0@B(#miTnEL5;lwX^ag)<=EX zx=J5SsV+M%AJtsc_WL8|!rMT-FU4RM_Z+47WXHJ}v7Rx$qUw{|4(ff9*U)ZG<*i=- zTJp+uF1*D%cVLrZb?!iF5Ot} z@MXDkNAruH`Wam$i~Amb0&P4391;XHp4HAp@FuT8+V1!P3U@mn8@};ZmDE3~Q>qjY zyj(An$Qo-iifE1kQR`*(k#tCS(Lb) zR6V5-cS!^(h!762=$M;kc~l`n3=`tguG>JJXEtf_X$F|zXUev_5g{_Gev|zE?U8%@ zKj4*$;116J|7~R{+jATxSgIIsq!W@<3^`&5-75MV>jVK+u=IbL&i4J!h>HG1Fjc+8 Pp+ZPdJ;O7qbh!9GIG3kV<$-TUFtKleC`4x|)X6Q*Maa)Y3PPUyX`YWF#A zQxT`%BgTZZG*& zT)*CfsMl?0+-I?q`xx{|Y)K%z^lUeUJZ8|2i}qrBbbVQw>Ix|ca-G1rT`J8bC(^hP zh}S^$javsADX%n*jN)m1K@Tye>sJzp{z)j2bTG8qX#N_MgpZ zDVe@;cF^ir@uAZFJdWg-n$zP)zIrZfQh(eSocenX8~rVT&beZLFvu~EzvRA1Sq<#t zTE@X#Cz;pRsMB;`d#loSVcx|aE_Ah;iDPE~>_2gx2xrDv&WnTcz|oa^sEmo%5(-jB z^Y3y-Q=84>%NQ3-wB}Oy;WHDI?6q9D^=$W_x7<+mj}T_ETeml0eI#T9om zejUxb=;@C;H`c0>rELz$$F-Ld@J+$SPAJeX{cMZY{357!@}(2n8I?m&Z4J8^C)62W zLna&l>$~(p|1%SK6yw8j;jM&LsP@yP00sHeYZ*e5AP0rqO6UC)eVZwBW<8ybH#S<- zb->k4?uBX(3UTrLHU^%`5S{*T<@zQn6@6VUGUc4k_wlL3cKf@`>v%U$?Kq6o zlE#NYhNgREk3jCd*xK%VR@6rZmPT?}v!2DJSv-Z5BQS_0Fs> zUqYyN@irMML4U>-2Ugz5C3^X*rLh;R8bhDcQq+udG2dnoAFOlThxj~;URjff%U6nz z4gW};DD9Qmpmo}r-c7q_T!Bd|vGl8a`imKF(GsyWuExN`Al%6|>d&))5c<}WTQE-d zphfBEN;@GX@X1`cZ`Mr`#qizpV@cNYZylEynct2%h?wN~+yo^-wVMyhGu5;4j#K(m z>FTS}I$ zoXW(iyH=&oWCi~jP_O>Y3fA41N86`;BjdsRJ}E}#qSC<*?QAC%z!^ow%ZFxX%~ih# z9jq$ypil1mYIWb1<7SS+?Ml%oJQ3&(ElqjRdGlat`!&AE(Jy@BkKplu)#YQ$ZTyk? zTbxtr%ckQQ?<1Q@%A^;g4-90csezt4OJnm;MZDvZE)$exaL_J9)mfXVv(9E#qw#f4 zm-7x6U;2!T9{(r_7|N@kSe%;o9E63gm~}VSL>i{;55yS;+mxja%l@V*5r?g914G&2sFfTfE1|)T;XZ|A;mFdWqj0-4-B_*)TLB6 zb-Rxb1IV|fkW6@39-06)cNQOpeqYiiH)NyPqST1b_i4@CtLPDhJ}(hwg~0;VbllwJ z+E|4k7wdEU3o4*{zkU>Xb+tjYA(dsA((K4hS{0(w$M-;sKSd=4J;+u?31tuhBTVey z$GiPco`5briWr|WR>6bV(r%ggj+)<{h&mEmAJh;%>akJ+2P4ijt53Qb2p zplXug^elclt{4h|DVH1x&mPwCo7CG6;cY=Mr4rs@9`Iln!4m zd(3(qJOt%+2hze?=jBcRn&0u<}!jxK>w}If$AI+6A?+bS) z)zxQe##@j^XZ+3&I3pv#`EELfngp-`#sSMTRd6MEtu#{3Mi9jq4*Bg`*so<2JCO_$ z9__Ow(|4>OA>t0Qi<)N$Pq`o2kgfvZ`FzFpmIs@*k_}|3oq&|@Uxoty$w6w1-XXu2 zQJ^Z)vw{Hea=ohxXa$e`wopYEQGAG8&u(!!7O3D&O`n1b5zKG^>!3yQ#KDsO+Lq|5 zp5HM=g<1v5yEFr7pnCeJscF_AwZp|Pddz0@E%5abVQTb4%P!XeCuGF7P>F9sB@Fb3 zg-ZCDRs-U-h!Go{QU@Eljf+n0Q1}F~MDNVr;)nbO%}RnBRhErAF9h{MUPf+NtY4We zj4Dc$Dhkg_W^ZcU@hrB1W}fjq4ETvRK~;mW5ek5xM0z>%SKmj0tt(30E?2#$<)Jh( zanLMzu_f%gMBV0`_GLrY8g1%A)VJ=)FC9QbPgA^06AEAuM}=|s{-ve*iJGxWH|>Bi zK}GO|tOU@R;)jquY*}H@<>Eu{3v+x~$@Pt4g^; zK=$nZ8ZXjXC1U5q}j-ME`hVTKxpu-S5Hd7X^+Z#d5}#lNC|` zy9+=Ren;qA2!jz$kS>exP7~@^!Ti}5ylLyoFgox+s!LXv9w_V(tzo@g@7C-tdK8xG zzkIT()LH?F7CbKwG z{<`}-rg{#BhBn109Fbh@EiNy824wMcg@t$uFyc3C5Wka2tE{u)dL{ru588$RoE$LT z0xhi&TE>PjB5=i|bTn~;?f^DN&*h5qBzCedSn&KmbF_hJM8f+zJJ3$OC?XSe z7TBiEaRfPB$*T3b4N&y@EK9puxM&bBcxHGqQI!napO<5uZEU6~^!7*kj<+Adb<(xY zrYlmFQS9^`^_)eicA`Q&X%P*plL?r@aq|&%k+F)GNhm_tg0y!VtP88+Lkj#uf_;2ACXGIHx!F4Qg4Xc6#XH_hw zQ)JY?qwU-}H2)=uf9f9vsAl_*bKVUwO=V=TvvnHE#TH625Vi_AEasI6R<(n$ZWfzo z7GaB1`pA9=9w`IcxiTtrZGf*cYCSFW(lS)#t@h_oFhN;b z^YXhrWoCqFV&3>^Zg*m0zI2AI#@Vp|$dY;X{826tfz+tXNh1V3J;kX=;AbZleO;!1 zrB7?jtS2+$59@6auEf09&}qdXUPe3wo?1iSSs?|10Vnt4?9May4h4RfVJGXD8K#Fep}-Ea#q_~|GPZ` zpIOhQd}D!i@w4b-E?hD4AhWL~NwIWo?kFW!*&p$mx1jeMIF<-bbP&``(3s`+<@q z?BcqN5)TvqStmNj$AG~$8wO0bdiQsTX`N++Ym=9;-$ElrZ#Bo**a;+Py6_&6|NfT>bbP@Q#FM&6a6U0#*KSXm z8VHn(Grky~f(yM!)YMZV@^tr$ENc+uAj(9jo}Ai_+%PZ* Date: Mon, 9 Jun 2025 08:03:54 +0000 Subject: [PATCH 700/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10935 -> 10940 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 7666be8c2..845c1a67f 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-08 14:20:06,767 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-08 14:20:06,767 submission_checker1.py:3308 INFO] --- -[2025-06-08 14:20:06,767 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 08:03:48,544 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 08:03:48,544 submission_checker1.py:3308 INFO] --- +[2025-06-09 08:03:48,544 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-08 14:20:06,767 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 08:03:48,544 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-08 14:20:06,767 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 08:03:48,544 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-08 14:20:06,767 submission_checker1.py:3324 INFO] --- -[2025-06-08 14:20:06,767 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-08 14:20:06,767 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-08 14:20:06,767 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-08 14:20:06,767 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-08 14:20:06,767 submission_checker1.py:3345 INFO] --- -[2025-06-08 14:20:06,767 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-08 14:20:07,759 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 08:03:48,544 submission_checker1.py:3324 INFO] --- +[2025-06-09 08:03:48,544 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 08:03:48,544 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 08:03:48,544 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 08:03:48,544 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 08:03:48,544 submission_checker1.py:3345 INFO] --- +[2025-06-09 08:03:48,544 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 08:03:49,870 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,12 +28,12 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1919 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71154 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.76142 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.6397 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 15.5312 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index d378cb0bf..a38c57176 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1327 +1328 diff --git a/summary.csv b/summary.csv index 863c0cd07..84f4de2c9 100644 --- a/summary.csv +++ b/summary.csv @@ -6,12 +6,12 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1919,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71154,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.76142,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.6397,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,15.5312,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 97e02a4fa..1f6c1dba5 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71154, + "Result":1.76142, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.6397, + "Result":15.5312, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", diff --git a/summary.xlsx b/summary.xlsx index 5285713a48f5d3584886925af011ae3339f73f42..34b37e93273306330df0e53a177181205e2fb833 100644 GIT binary patch delta 3635 zcmYLMc{tQ<+nzxpj3tqMOk)g2GR->_F|v*^mdImCp`n>*yu=fc--JT4&5XT5mhzHi z$Wqzf6k~gkvhzf;?_?kE^nTxWeD@#sAJ=`~*Lf}Hc^$`HbM?d3@&JKj9pTG1L5zC> z1OWln-R%_6%{4~5fIOgO#@?(ga&fE_x7laAW9lzZ8-VR~C>bk7@w+%1&!B{j9!O|q zHawe()Yy!T4)RISJC{~7+}dgo7n zzjF1=rz69x55MwtGO+`fUpw|nTsR-hh}#hLsQ~=^Obl^?8YWE@ne$ z%y%;_4wG@v05st5Ty#oP_p>iXy?q05ZLiQ@`nlbj=KrCHQfBVfwQ8!h^jyA+VH3Z5 zJL*$O+Qj*J+l58e=Ek>)^`BwXCE~~F2gYYy&BM2rtrzotxs9(O{8b3!?8n=z@5Z^g z&%18vsdcFqctE}_%b(T$UhA0&pIkBv)K1voO|F$o!(p_;6q9FTZ3GE%{d$;q^A{fp@x&Y*IPC0$Xp@%1dymC6OQ|NX z$@v%DN5DHb3GqL+HDhCnLyfpLju}i5XbL$1&lnvq4Qujfk(T}?d8XQFMMy$?f$O*B z{tZtP=gEhw^9IJ@um*&2%-avz%zMjz?n)IObWQHA-p43>-dJpry76ia zR(~C3Xl)jEuv%H$aM#sQ?|JLmeEQdJ8~de=*+6^4MCl=A@80oe-jM1`H@0t8Msj>Z zlB`~3&vZ~S6<$HGMGN6*g#>D*q0VvtVYzteuIz+iW0uE}V=J9?)8`rS&Q>Em#O)0* zQ}8YNX=bPo=zJ0s3kh>bCy(3qeps7d!XaFZ>+ z#(pzf;-nQWZ|<}bqY<)9K;Yxk*Im(?n%GjIMl*yTwJh}TK&{K@m$3KP*Jh&Gk3~Xe z=YGw=y?4`ZOCmW}D6e?pf=~rKaVx+fPn>nndewA*ByWNz4f*7F&)uhd>z$9pamgm| zCvcb@Ts+p(9hb}`NQz7=pv)zCd7n6mBKajuubjay)sS3VPz$h~?Hs4pE9yBr%7a5v zcg**#+5VUxsK;$Db7r}@tU>iY36xvF=%y0MsNdbo-BrglSlFEB*J z<&0jKd)cK)vTrH+X-~DyXnE8(5$_+8{Ld}FxxXVHJhA8N(z>pW3zm_B3cKbm)g=TS zWlv(%x*`40~6V5VfAA{D(UHh@`8wvF~ugh42VGjS#|zRtrM|3hZhwn z%#BW%7a)Tp+WO=LZRN)h1D+e!h+;)6`$3!X1qObZ!ms} zXo1HWQV=EH^A$@BU5FNVgh7VD0O^{p@@ZEpTgdQ9W9yOCch?yzv={Ezv@b#Igl58d z$#mhISeb~zo0l1zG_vgdSzo^}oPD)Ppd`*dH3%<5&C2%~#Xdg*Wd@a0^i|I-QnMcB z9kT9JwpiM|mZ{Apojh*@9cmF~;n7V9HYm%g0E~4znblbZ#)@1DtOj%p%GF$97cG%P zOfNUb$0LZx{~QTC^6)j7ws38g)wq*I^&bgzy*`_o>pZ!Flr+{?TTQZpiYU^JFdDl0 zC|)u;?syM|qF-{{AsPK9>wAZYVUkg+YHmfY=2#~d;cU6qFHS9 z#$GQWajOY&0``I#23R1NsiJIX)+v3ZR4_KSKf*Wjn$kC37Nze2lj+}`qvDxok+C7) z{;o>6nmEa6s&=j%+(_}AS5=^F_wUp>m`TPV_};tg#b0MgC#Gf= za!2jAXDpZ64HFjh!r5gZugjw>nU?xRM@d%5LnXSAy2fc7iq`@Ck^eOYkE=SK)0Yoc zTc!ODv#AV=b53I_r{?>#1WfqmTtOYGdeptb_N1J+S5XxS4b)8?o^-n1kJqPwldX`N zP&ydp448Sc(Gmo@-THR^bqeEHi&=7`hUa`vo)J$(Y7EhygYD%Knv$XanU$o82+8%S zrll%X+Rq1kEO1BHXO%!$yMm67G!M5NzOmk^t2()1Xbv6QO#6U{um9q+#Bi}dTr3eN z-3a;L$ZgS;W&|XWUrg>;4n`0s3-Ce7v9p*0jL`4y+d7KT0=sy6APgY-f@=qlhKF+% zQ1$$<-%h2mu{cOD^>uP^A4fUEcGwUe>Eu`rIPsU097WtIEMO2xE5LJx#&>8cZH9rF zzxRA$>+%vG-DWANRjp3}H@cMSeK@*952jL{dn`$u?S-!8?-~Q>Tnu2rUsvMJDHr}H z9^KH=2j`SSKm^8iUi1FL)EMCiMLn*fWr7z=d{zg8y2eF4ce_k~xuiOok_JqdT7Bl4 zs@rP=Eo)7H7RInBJL<~5vdj-w#IKL&CKWa)4Y3d%1iF@)uO!kZT4h2K3-rq6%5_ze z&VKZrN%rz_CKWJzd-rgx0VyMDdd7 z!by~XaHr$ALi}!T&H;{K!mmt>~x@t)NrZGB~8E(PfTLL{S7tX<>X@vlL*Fyo6iI_HzT<6skm*FEiBNtfcQB15f zUzT@Q`(Fbuci~s7Q%xO*F>&3A)vm*Wgg||_mj`4A^RS|Wq^GR>4dcsPs?YqUIvtE5 zwu>%+dN4AEvQL%Bv4dX5nC2a2uUa2^ssBL9(9W&_dAVr)Zb4kTO@q&#$9D!t!U~9+ zx2g7X8P*DwdEIi*`#Nc-??73`3%?Z|mDF`&UbKTdHAcZ$;=HKbu|Z6Rp=>AD^DO2e z@f=_wLw$C&Q^~DzcjLj}=pRSsgd_9a>+CPY_bo>icJQ@=BV%+sokvU%{0w#GJ4r@R zHA}3ClEnwbK8gC$Z`8^U7I1+9SmNqnZV92|$7lTIa{uA|A4dmnuu#po5|;n4sR+{5 z_4OheOTa+o9!T7{jf3Z*=D^8fEazrkPZ@Ux>BmXyk$i9k1Vm);lOp-~epgZd&)4%T z^}0JOJ-%vU=n5*wPlGkUYEaiJbYs$33SR==p1sgLtRj^ZZxE{maHVLM+E23(C@RST zR|wnU(YOLOkEa4C1tCNZo#IV#2Um3P{|^s6oj>uYqueJd!su<+JNDE)1SOJg>^)R7 z6|^`e>O>cXB}<60kpveUJc=t2+HZk`i^!9RvRuX0VkkH)8P_9GL1`DQ*NtYK)r^ev zeFPXD-CU{TXF~46zcLQ8UA{Ta9k^@)RbNl=x~;z~Sz0_5F$qh2O1PZDU zhO+x*aeX9Eg&hdb0FxrY+`O|250WzOA{YA5I93Fhv{DDIGD3$*(D&6jgWYH8Mq+Ir zecrEX`!RyUPiB9fT6@`R!|Hfj3XE2l8tyM&Q3!VetVWIX<+L&&X1fzrEAazLx6={| zb>cr|FC=zsKx}{k$8l#MxK6$`ezOB*K>eJ3Yv>YFt9s~km}03RvP?TGk-m#OWGP-c z%5Rviy@$s08#ARBJzw!!qHoQ^nzEBn zT1!#uR%f{0`+&N^BJA$5nS$O^?dKyTdPW6`vQE^OrOgSe7@X5|^1(QeMHMyY-_u>& z<0JpLg{p-%tBU7eny$V)B8K`jxS@&sSvlKdH+uS&Q%r$Rxz|C~KWM5^RuvIV0H2WX zc2j?XNubFI1K3ic&vDw{r(JAwB3naJrz&-5_8jkzIoEVkghPdN-l+Nqu6&S>cN_3> z({{Sakq1hzk3YU@U$k%Uj|)=MorXA_D?PjBB`!LER&rcxkrSNsSK5;EYc+*DfYMLX z6enLW6DQ|F>P@zytQ4=HiR>_?ktCluj|=>SFT$ vEif1avgHAR#6kbRXb*?rj5YYl-2%Scc^!~eJHZ_9rGljhPR delta 3593 zcmYLMc{mhY`<^jj$gb>+rZKh=O^hWqqr7F3eR(a3G(?dQ71NBRFNTB}#xieN#_$T+ zheVQfOm-n#z1DpR?3E)Hsv{fqC656x#wR55VSN z3Cpz=O<>zkN+(!>SU|^xjoWJ#eU2^I$LVBFD(HV{41Xl1EAKl*6A7Ef$wBlRTYT5k zD2IxVugnqiY6Ab#TNBA&2wX?8y&)2PWckpIvG}B&IikT`7gTX0t3nr zna{>XE9y7%^fOU^o>S;9+DI0Pp%eC?3)=8iUDps~VtWG-kwpkQo5|>+(3{j6OPs83 zDv2i2dItYlSYZzMWhrR2ALzuP8Lsa7aSZ%bGCD3`Qs_C&QQZIH=4_Ch^m?1J(x%k( z5|XoFq%<+^&G8D3ClipCX@@8{eb42)_s#xY{CE{D!HRAY>Uw{?R#BKQFffOfZ~*Nh zvsQ{|#KF2~hr+d?>P}nQ_cDe8d*E&$5ii@mn7Mg-a8}5DqhgZPXUrK)*S>6at6}P; zO^nNY8pFNnz)@E^_)e+(sTFxSj!|6Eg9o0F>Xo?X!z4}THA_@|nlOYZH_E}IDE)X_ zxjB#J?tdgCpKIo0p_6xK6iM4}leb_JOP9_Qlgw&KONik)c6AO&g&Hq#x~J%}{9iOQQI%z9C>A|bQ#22Y`_ZdCc>aALDZX1$fclT3F zTX@4wpgbe)*p|-;^)T_K0Zj>U;|93+Dmd&SbgKV>ndvz3mQ3H&_e(J7>mS@jMPo!2 z@xYKTEB}ku&Gu4zuF=9lS@+B5#wJ7w|mNq4Mi@s`S^6cr_K z>5nrrs#AB6Pn_8D7az?ZFn1NdUqzYXq9jPc&V#McH}mx>`?r(dJWS{OocVRAO=x&F zdB2BnO(~{-&-nhC&b!hO*L__7%V)%OqAJH(_3*Y-Ps}iVLFeVXDs$T?eqhPS4gc4( z{;8jo>tEhz6}{-1e)#C)ZRvu-%}4j^UT3xY5@NAqivu5<^5+*1{6c%Pv_!}Y8|(Pf zU2D^WO-3OTR{70+7VGnK%H3h~z|p;hzt6$sHv>Wt>S$&gXOMcRnKAt7_VwGukPaWi z(Y0%|;Q8Pm_!&#%k(W1NVgm-P;Cq*$s9VMkU|ah*{ui}crH7WvWP=8k1e zJvF4)U2WOkqx?C;-ea|)v6YxBS!L}1Mxix=E~6Yb1>8_L8EG7Rw z@p7|;yFbRW>xPI^^Zl(uJe#1Nm>1=kV6pbfy%UC3(~D<639fam76D7V{&O7sxaQ@^ zPRD3aly#bMxc%O*ty_Zb0T0{+;NCo!qp;U&_;s*+?_sh4$uqUCcP|{f+7F8I{gy+Y z2^YHcJ0e~nAF)`09MQvqnO!BkODEG^!@W)NnJ(W2d<2Ke+caqNVV<}5Dc^*(f#Vbz z5eVp9c^Abow>+!lcn0|IC_*fry1X~HpL;$g;zn(hfqE_O0vOC>YiL>J4~3zNl2 zo;hOC{sg`N?9<)o{XD)Mk?%+PrWU~sOh%5Qu*0?fMu$M%MW>q#|AEgFDA46yuQ6{@ zYOeK}dly-7?(SIh>X;IXvWu9{Pkyaa~_|lu8W=b#$ayKOy`2$^@iT@KbtQvYn(T#Dn?dYE*qR5 z0@2iBAKI=FPh?oToQFmGIsU{u@r}T2k5>2wJ$4f9$j5Hk*tQu)Q)3gLUfscJi~YzE z6loVU=H2Dl^Jhh2*8RK*yXkpbe@!%J)c7nyywkxt3NCo^WeULzo-!oLQqDYH51|5e zGr~auewpKtQBbyTKQSdGxI0?~N{^TZIk-9Uz9qsl&_6Qu!p|orsXxmqjR~@v$kMFV zJT!Q{vh8L6RK-5mehRxEgS)VBm?UKD+ywBH%_hsDFbphD?m0jzT-Yp<|A>~jJTIXd z*jbk+R1R}HN!|4ZrFt%)1=#1RBVXIs#xoR;dp=1E3k!au`^&wWCgi z6JIvYJArDQDWr@N#P~`?=z=l562v~=E51^lT;+3$l>rNxyxEHILy=%6mDlk@(AYs= z=z-nk)V+h(Ln=&wIK zlaK~Y6|es}^0qpHs7~Fzeelg~y`8lpU+VX^%DmPzx`i?)%6ihjM?4`rQxU~!M!r{J zZA+L_wwsLr2#d)?z7>8KC=wjDWId$f^|ie6b<@pZy>N8wtiJT;JI5gSPeS2k4h1+Jz9QsNpZnX(s03N`OAebhqf=VoZTb3ztY8MV=j#K2jA z4ETIZ2mqIv0uk;Mtun=lU4W~h^MphzmR5G&n=rj+=XaCd6m<8bPKFZVq;ULiZe{BlqKki_0PgRe$;3=}(aI4-(bd7VQQ z9am^`%ys%QHS~joqE(t2TJDp$y9OFYM=b0miRC&sac;lS-!zxL z(9dEPIs`M>ADMY~#`W%?2=7EQ!(Z)PTkxMP7}49mx41I2MB$Uf5^FwX;lET>xo;w^ zNgTS(iB$&YBC?bGT~y!%?f4)!TU16Z$S-pcf)XtAMG|$>(cF{ImxtzYkc1pJC`w1y zd9Tm!mc1!GNTePlt~HPgXrZLTn{=}&^TQZiiVCl?1doC9r4L-gTcS=x44?!se#Zm~ z3ZtJ3^Kgw$^vnEOe$hFN%1WR-@tCgKVjXhcnr^yCIeSf4)#zp8{c|eVBZ$5TzYQ$D z5SaX|B9GtnLWei6Xsj3aOiQ*h@Tc&#{NA`+lCT^X5GVF?B@K+)x?u zuB78Sd*{_s`X=*8%1S!5D1;R_bgKMcNFuiyxtM&W7tFkPg}Inl$UqGXDX~Ss-@48i znmpw1BS&N1P+ZJn^G-3ghEy;2PElm5l9sBAOC_kvKn??vUZDjn{0cN*GlALiKxMtS zRsKYbEhr_%22ekiMn)H7L7sF#8eF)8PZ;b1MOkoir01=N7+&|1N{Bz_iA(2(BPwo0 z=7o_D?bxFlFWhVPA^z1LFlqH_ch0Jqwse!(% z`0aMc4*FpXdZQtWaQ9o5Tzdk3VXF@x!nU~-r#(_-`P{pKwq38^C~Gm$W}q~8wAzJG zdD{K)#^l-}B15Y&h?bCurK(VNSME+vD!-KeGK*e$iX~4F?XHo#F3Y^!##|xki8V)uA)P{&p+n1SRNj0UC$`LJv?Y}+}&b|#Y{89wx z@Xj;(j(47klIR@=7uFo#ankCSy@K~}DQolj+nQggYw<1FwKE}F{}!TFxk*(#@7yf& z!do#!>&UvI+Fa$gUYl{50;gEUm?wK%e~jl8IKz`Sk|-c0^CPlQk@PM07~XUgGAsmW zI<1?H;ZIygwBHH<6zp|9HFXIMl{f>bAE>7RgDQTiHk|O zYKL^4mEEdRPBA9@I$65Yg9%Ys3z!nP*YS8?;5)kV6ugt?|G#YeYOcVe6lZl~U@C>G xZUT&=jHnv{Hz<-CaQT0!1OfooJOF?=;D0S5@)N~g<2+EELeV(MH>Ub~=D!>&+`Rw* diff --git a/summary_results.json b/summary_results.json index c454f10a3..d4ef6606f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.6397, + "Performance_Result": 15.5312, "Performance_Units": "Samples/s" }, { @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71154, + "Performance_Result": 1.76142, "Performance_Units": "Samples/s" }, { From 7c51a4b515cb1ec37fcb44b657ceb888871354d9 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 9 Jun 2025 11:16:08 +0000 Subject: [PATCH 701/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10940 -> 10939 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 845c1a67f..59da6344d 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 08:03:48,544 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 08:03:48,544 submission_checker1.py:3308 INFO] --- -[2025-06-09 08:03:48,544 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 11:16:02,671 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 11:16:02,671 submission_checker1.py:3308 INFO] --- +[2025-06-09 11:16:02,671 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 08:03:48,544 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 11:16:02,671 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 08:03:48,544 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 11:16:02,671 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 08:03:48,544 submission_checker1.py:3324 INFO] --- -[2025-06-09 08:03:48,544 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 08:03:48,544 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 08:03:48,544 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 08:03:48,544 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 08:03:48,544 submission_checker1.py:3345 INFO] --- -[2025-06-09 08:03:48,544 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 08:03:49,870 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 11:16:02,671 submission_checker1.py:3324 INFO] --- +[2025-06-09 11:16:02,671 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 11:16:02,671 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 11:16:02,671 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 11:16:02,671 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 11:16:02,672 submission_checker1.py:3345 INFO] --- +[2025-06-09 11:16:02,672 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 11:16:03,793 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1919 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.76142 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73685 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index a38c57176..7ae5be655 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1328 +1329 diff --git a/summary.csv b/summary.csv index 84f4de2c9..a68365892 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1919,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.76142,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73685,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 1f6c1dba5..044eb52bb 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.76142, + "Result":1.73685, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 34b37e93273306330df0e53a177181205e2fb833..f141421abf4f9f9fbb6c72a9d55de0392952c5b3 100644 GIT binary patch delta 370 zcmV-&0ge8=Rl8NNatH(kus|!bb_iw&2DiPSL^IH{;t&NC1x0@GP$iS>Bp-ij!!Qtp z?|llRyDiD4p%u0RZON&DLQ`6D6VfITwSJJ)I(hplPU0kxQ;(~e{dRUyJ>BXq`vo7A zH4UN#K^Yifr84UVeJp1A1Ij|=M)J-YXwVJ7jetb6oq`=UD z1``9O1;uA?1h2!*hC#V-Y87`5Zdy;`=-vi38vDL4`f9MI{1ku9U*CsSb7l6Y5`fxT z3MM@8XnkAb%OV}F;~~z|iB}5b(=O>Z-j0hL`aotW06WAzDWBEz%VH#k+LDy_Iho|- zaX~4g6Gp1#e&kjE8K{p#{!Kgq4!N&CLHh@@E+)zX1O>1_E3^D4Jplzpe(_KxlRhd` Q0WFi0DkBEpCjbBd07(F%h5!Hn delta 371 zcmV-(0gV2;RlHTOatH*=e(q1Rb_iw&2KSUxa)Q;f;t&NC1#+pWS|*e1Bp-in!Y~ko z?|q8M_rwlq5m-r(+KN+=DwV2LZ6Ci-ZsmJxqemlEzx!9>T`32{t z(G|)wf)dc&2&tP2y|0()3?)7=Em&(bRA>(#E$X|HTgr`tS7$7^Ai*aQq&?-fLfa55 z#n|&LDCV>1&{5nNr&x%kYp{Q1{F^m^CxlF~g1`g|4EMm)jzn}s3O*w3k81~j;22s^ zpo7PGmgBQGfK&cv!=PL^l?;0eH?1debnmpt?aX@t` zIOPslFs?4~Wf2e8G9Om)#2X3XaUb;?Z^uOreISw;fF9zWl#k-+dA%h>^@L0wQZh}+ zd`)JQ6!d=n)sMXDKLgcq$iImvz#;ebC#e4bvn?jd0tCx`?oYG(C_MoMa;d3WCX+oX RQ~@rNk}4wx;3oh8004E+tVaL< diff --git a/summary_results.json b/summary_results.json index d4ef6606f..7e0b299dd 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.76142, + "Performance_Result": 1.73685, "Performance_Units": "Samples/s" }, { From 39db9168576e7618e34aa4dc11b134314db9f170 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 9 Jun 2025 11:25:42 +0000 Subject: [PATCH 702/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10939 -> 10940 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 59da6344d..f66000014 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 11:16:02,671 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 11:16:02,671 submission_checker1.py:3308 INFO] --- -[2025-06-09 11:16:02,671 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 11:25:36,496 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 11:25:36,496 submission_checker1.py:3308 INFO] --- +[2025-06-09 11:25:36,496 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 11:16:02,671 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 11:25:36,496 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 11:16:02,671 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 11:25:36,496 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 11:16:02,671 submission_checker1.py:3324 INFO] --- -[2025-06-09 11:16:02,671 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 11:16:02,671 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 11:16:02,671 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 11:16:02,671 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 11:16:02,672 submission_checker1.py:3345 INFO] --- -[2025-06-09 11:16:02,672 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 11:16:03,793 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 11:25:36,496 submission_checker1.py:3324 INFO] --- +[2025-06-09 11:25:36,496 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 11:25:36,496 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 11:25:36,496 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 11:25:36,496 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 11:25:36,496 submission_checker1.py:3345 INFO] --- +[2025-06-09 11:25:36,496 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 11:25:37,597 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420653 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419302 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8884 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7ae5be655..ea414005e 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1329 +1330 diff --git a/summary.csv b/summary.csv index a68365892..a29483022 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420653,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419302,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8884,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 044eb52bb..003daf905 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420653, + "Result":0.419302, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f141421abf4f9f9fbb6c72a9d55de0392952c5b3..8adceedf46fd2788355d5f4ae5d76ef8c17405af 100644 GIT binary patch delta 1389 zcmV-z1(N!^RlHTOatH){tdqwke(z>bs)YTp>jJ6NvdR&-XZ7e9_8tarqfJohD&q()5nEt!@U8`p(jBa=6` z*S&uJl;$9N2D9$!;3gw-_9ac`{oCt1-SS%2L4T>VQ-Q@gmQsPm_kMvzCun)ve{wmB z0Y@=F6vYGRIYt6`=E4Y|NFowN5|Jp9h(s|kph(IkiiYwImLrOR0Y$6_6a$GOsS%s@ zTQV&zmUR*Qdr+)nDHTxs5bSJ=wxfNgKnNER`o zEb-8M6&he9_3~-h8$*336sVlZe};&SAkU++s7s3EGmp2Japd!tuwLLMy?(Md8245^ zd=9*CeKxIQc@-eM3=0s3tu-f5=&DLo$ZZVFrg))qllgS9Ll<+OuqW~mUXSg&!vmPe**Rc&xf5c zM{^Lzv+N-`U&PmUsug6L<5vS(%L*$H`ff9^&5^a`1YzOjp+jDLzTWP-5=_hl zjs%muZVzCR41h^8048Q|x7>y(n4|{;)6hdl$@dt}pe3TC_!3N6)@5a|YmyLbn}r4J zBbbFfGbdA9E%gL6>sVuzf6a3VWm%V%;kJn(+&1lh6r1WvC)zsJSZUj|i`!<{@-4UIg&)P=peb7-ItrjE%etftw@uPDZkx`3 z0L?nqSZUjIcDK#If6;!~wpn7ODAWwxP_l`cAdqZg2H7MTV3TBkO{UTSo34pXX4(Lo zh8{ZG8z0z|EfF0hlx)hfE-S-r6GOOdy8kdXpK_$FV~v%zO}Dshx-Bns%g`+GcNB^t zo)maUCXfaX$sn2)93GYdG?~Q%Xqq%0)&rtxGI1!Boj)@hh&h=3Z4(k0GmW=K{idU59^l{d zpozkWC#4^fv)>O06$NPdS3f3`>?9t4%TB{E5JmTVMdW?rJk$bKl1hNOq7o88RO~E! z+FHa9*@m=#kCUcpD|N+keD9pOGjh4u%QpH2>x5Pn%90dCpqS=DZ7cNAEaL|hIZqX* ztyWN>130v(?@DF}(-z*WHekH~7X^@VgqaHMyf*}6$95p8OM*iMaiguIK9qKUiw$Mp zbPG67(>a#lDW{&|9vGX6h(;u56Vm+Hb^vgOp#=%bJDepMK6^b_>25X*%7s%3e=u;< zdJ-r1-igWBbzRaGgEi#O@Ot(7KCBuG)t`z1s!PrYvp~JJb%`&FaJZH_zX~Vb2;fhL zpx<~K7diF;k3s-)jC)c(i|3anW{K)N&1Z2skJHB{%Sb*W#r;=5@~Zy~l;eDL0R?FJS3f3`Jt|ZIE|ZcfBL?6n00000LmGs9 delta 1361 zcmV-X1+MzMRl8NNatH(kus|!bb_j6;e=X0oId?wi&PVRVgY7v+0x^wKi~x5eBDo_G z$sLJ^N8c;ofE_6p)V?v4cd%T^t>~;+FMb?HSnr4{dG1RlNsXBDTQV&zHm(c5Mka4= zuY3LcDa}Fl3})Tc!A(Zw>`R)=`?uG3y5+U3gZ@%!rvi(0ETsaA@BIRcPSEnSf8}x% z1CC;VD2fNrbBqM?%!Ls^kwhelBqC8H5s6}8K#`P76bmDjDm1`I>gCh0H-`F7C{Q_*e+>~EL7qotQI{0SXC7}cogKqSjZBw0o> zNis^li1KP1VpP%|B$lQgI@=q!U_whoM-e;uIh1utnSAx}E$do~pGy~oe+BFZo)0@? zj^-eaXW2t?zKE~yR4d3f$FByomK9bY^xbA+n2|``+>{0R8-=8Ubz1?*sn3xG1 z2_|{n9>63S0Fz_@Ow8bJxeZY;Ne>97p@)u=?=hS~OGHQUC780T%gSKaBq7)~3k%ps zFbjKTPNue6>IrDpvBoN!f9GamYV$I5EAYLNXOoJ*3-wIl8#}Hi(Zo#PN;ENpXp#(| zNiu*YW(byk2fft7V_TM|wEA3(E?HCB1ebu+Q; zvrE@@y|VKxf6Mfl!gnHFU$Tjrz>{oZ2H7MTV3TBkP4Yf3vT2*xQzShgn}!}b+Z!L) zlr0e*C6H{&vMwvbZ4*PdZQB1RHr10(v~{ep(za3n?nj`~gk_@1U8G@!`lMfCTe{P$kYuq-S{{WhGtg+Iz>FjQsfusGhZL`Em zQK%WXp=1*?K_J=046;cwz$VE6n@ptvHeC~&%(MYE4Lx+WH$Jc_TOvA2DA|-{T~>zM zCWdg^bpK&&KIKST#~Lebn{IL2bX#8NmZ4eV?QJSp&yOdt&&l0h^pI6N!^e`qp` z2hcQWJgf&q(`51}PxAJzl1Y0`a^lyT0WC1RsUne!aVy0i?o zP2&E*wpp{-^ObDsCmm_`)cCnn3`ON9+csUh;5YB08MD?85fueRe(_Kxlk6lOf6Z#c zFc5_AeF~$yEy<>#6}AIy$*F-tQ(AHp(k2nLevs5UdHX6(;v|q$kE@yec6L!c-Rds; z1s{|(4Wb1>85m)uGV2C?EN1xw%0lEu^3ED)&<+Bcws*B~Ojr+Z-a7D6LC6xw1SVXA zHZeMeaS$8OJQT@clDM*7^O#D%f5wiBZ@vax5;DOWB9}aJya(oPB%&iyiV^94d^Z53 zz|esP69c9N#b<8>ufxrTLAh{h6?YD9T2JEW-Uc-q`@S#wYOtpK6o1WM--lImW%j2M zfZAFLCOq(HeOu$pA|0;dAI}sE Date: Mon, 9 Jun 2025 12:03:58 +0000 Subject: [PATCH 703/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10940 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index f66000014..579cc9be4 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 11:25:36,496 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 11:25:36,496 submission_checker1.py:3308 INFO] --- -[2025-06-09 11:25:36,496 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 12:03:53,146 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 12:03:53,146 submission_checker1.py:3308 INFO] --- +[2025-06-09 12:03:53,146 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 11:25:36,496 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 12:03:53,146 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 11:25:36,496 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 12:03:53,146 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 11:25:36,496 submission_checker1.py:3324 INFO] --- -[2025-06-09 11:25:36,496 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 11:25:36,496 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 11:25:36,496 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 11:25:36,496 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 11:25:36,496 submission_checker1.py:3345 INFO] --- -[2025-06-09 11:25:36,496 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 11:25:37,597 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 12:03:53,146 submission_checker1.py:3324 INFO] --- +[2025-06-09 12:03:53,146 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 12:03:53,146 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 12:03:53,146 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 12:03:53,146 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 12:03:53,146 submission_checker1.py:3345 INFO] --- +[2025-06-09 12:03:53,146 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 12:03:54,186 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -28,7 +28,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1919 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73685 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72994 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index ea414005e..4d52c915f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1330 +1331 diff --git a/summary.csv b/summary.csv index a29483022..67b3ded9d 100644 --- a/summary.csv +++ b/summary.csv @@ -6,7 +6,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1919,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73685,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72994,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 003daf905..261b31be9 100644 --- a/summary.json +++ b/summary.json @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73685, + "Result":1.72994, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8adceedf46fd2788355d5f4ae5d76ef8c17405af..d54fe59b0685b88b4220cb5565985f6cf4a623d2 100644 GIT binary patch delta 1602 zcmV-I2EF;bRk>BLatH zBC_kE_T#m5U|VHeIwf49KjKO4hh##j{g4cDSwZ_@8Q_v+fJ>A0!#V&iP127B=?7XO zG6=e|P}YBCWq67uW)PfWYnFZ<x{b=0^oS@`aOtf1_!Ne0L?IX|oeAk(D$Xi$EjB_e|mGC}zvWq6Qff7b^NvNbC|--paS zb$+gN(fP?=ko7{h>`A6qf+qQsJW~20nNUhUB!howR*-&J2GArKK-1*>unquClk%fM z`GJ;*41%UClyzAdZkogkf=#n#<>zbBe3B|#%OWdnnnmYF+b!*cB|4riQGPV4BfeCA zNJd%G`5_@hvx4)(GJqz@0GcM{hjjpGnv5R}#t*ba?6M)6VEm9W+%-wpxNFvI{CpoY z_f&uRxza`DC*L*wu<&0GChh>6CiQ=ZjRI_%%pVQr541#dn!qNQKcoyd&MM}Q zuG#$gK5Xu*|D&b84%c|JW#!_|pF&8D&WUh=c%6mXUC>jD(X-2;j6$0uUQT z!fEQDV}k$$EfJe0c z1f+roEXfCvNC}8!lqD@75<)C1SU@ZTSTer`4zeZZeVnlvCAG$3e+*fiybUYtW& zmzKdnw!)wPvG-g9N{2a;>Ph}jEo-ZEel7Yx+G>S?E_qSbc1n;$lf;+)56OQhOA0_F zgh*BpfLI2QWNHl{X)=J=D1fBN0J6aVf|iI)Q+~+IIh1v286a7~0J8Vo0NNlKJ_*Tv z6@P4J7s((9ght^={(x-uz}XT{On=Fk7hA@|%tM`lI(&U#H1z^_AOfd`unEG@0CgcN_Nx z@jU-7`*}A^lJx$Y{E}=x!tmrTvkw;74h1GSuGJ-z>?9t4&1%Ci5QOi23ZuI%*(RYC zwgYX+sewXWT6z=GCK0uMkkmSU`zlW2B#={&tC{_Fc2Tw1sV?~i=cUm#$}@rz(A)^A zH#K^1m+3u9d|+Cz&SI!$PvYd>c{v&TzR&t{w8s27{#?DhjjN_o?@z@6HI?9$J7B@Mroxv+ zJY2_oSj7{sC5Xo%>NnnwiyZnuBryOz#62k=<N$^r%UORd;*ll>@L0S%K`DLes}la?t#0fdv-DLMi6 zlM^Z?8zwle)g=J{08jz|01*HH00000000000DytllT<1`0V|V}DkBEnCjbBd0JJ>v Ac>n+a delta 1618 zcmV-Y2CezIRlHTOatIE6e@F(otYH4I3~)&@zQocb{;&=JOOyAb!TW)hhzx=rJCt=v8F=ih zYw*~0&E5}U`OQW4T9#NvWYxFLFlT5D!P4Xvsr1V2Fp_G0|27l44ApNim zph+@-rpftX9RQjp#{Q3G>I1kn`X_*&)1^)BvrPSMONB0i_VX> zTiOXrbUa<6{Ag51e5w49jIyNjLqdpV1?Pul08NqsG)>A6>j2O+89y3~A83i#WkWQ< z_#tJuYm%;U*R0w2`95gwsekfwrHjf>zH9nn;qTV$vRUH%XtYOs>HLt4vZVAwLda$X z>4#;2O{VO?YqKWlhm8Vonw%dE&JVOibed9?g|aRy!+n#OL9lPuEd6{voKHE@?y357 zsf+4Qwr^V6FMAoN#Qf2ykoeO4AsJ;!^@oIz%?j!d%K)29+yORC>VFR#1=uv1KN`#* zXo=`FflV-fNEvRNRm>kO}oVZu~8)PrT;@R%8~*Q2?3le zBjIEj2`8Hnz-gNVAU2AG)6_x71_20KA~sF%*rBXT%jCvMLa=e#`{e)pT_Srei>$J7 z+C>d0TU)5?GB!&zAb%Tu5?^XSB%>_p0g({0S-}Hh8DNuSU(*RsAUJEmQffU+IaaH{paHQAP)RaCrO5(f9RQUk3ut@e1C?L_vC9S?JAagQNf}~U#R4iUVBfKP zlE_}m`YK@w5|Cf|fhk1^NCgjAk`E%05)jEKOIkoAgjiOvfLI2wWPS}CWK9|n8wI#D zX+Sn;K+qDgY03}1IES(>ErWw>g+Kpe@3{t)4s#^cll-4r)>i5KTJ(Rk)d~Y$@}jKm zlpu*Fi7)*hl7CT_6o5zwk*pv9u?!%|)EYq2WB{>I07;VpWPjN2eiNU@yOLx4u4O7*H+$He|WhZ$Bp#MEIw|= zpJ&-UlrOf~7<+yh;Nd`ZzCbgW7{#?dC{?PxyJN%*V z{q%D~|I|2cWJ|w~W~*C`l7GRbb#gT8-Nfg;`ORp$XpG}wwot3(HyiWyNAIn^PLtW{ zE4SJBm>r~PGP(cmHtr4LdH!AY^KO_V>HRnPCE0$2;mKdK4HnoA1!(zKKPHpvBp!du zPQx$|MfZF~@0iQTEq|8hO~c=lcs4ab;WXg@0__aa=F;c zHu?qYgjN;Gk`zUtnC3!lEA-JU;|CNuPZg)FR#2e>IJBtmN@fVt7T&BjV7&kr1(0%t znF{T^Hw0tHb|9%sf-7@LWRMkHqw()`$V z0C0w(1qsSKoFy4Pdp%g`ZZ-_cg;NTDFmTg)5-0cGiOJY?UD6eUHRR9mdiDA~tQrf| zpNavhOU?+hK)tqgi7$(AxRyG<3MbwO;7^C3-*_7rIragMLI84%ds04&=a(jCiRwJf zXK^}@)5j*uNIoOQ{Z~Kos{ag>|Y>rYVs0kbP6$^r#_ Date: Mon, 9 Jun 2025 12:16:34 +0000 Subject: [PATCH 704/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10937 -> 10936 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 579cc9be4..e2b98002e 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 12:03:53,146 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 12:03:53,146 submission_checker1.py:3308 INFO] --- -[2025-06-09 12:03:53,146 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 12:16:29,045 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 12:16:29,045 submission_checker1.py:3308 INFO] --- +[2025-06-09 12:16:29,045 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 12:03:53,146 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 12:16:29,045 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 12:03:53,146 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 12:16:29,045 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 12:03:53,146 submission_checker1.py:3324 INFO] --- -[2025-06-09 12:03:53,146 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 12:03:53,146 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 12:03:53,146 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 12:03:53,146 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 12:03:53,146 submission_checker1.py:3345 INFO] --- -[2025-06-09 12:03:53,146 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 12:03:54,186 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 12:16:29,045 submission_checker1.py:3324 INFO] --- +[2025-06-09 12:16:29,045 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 12:16:29,045 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 12:16:29,045 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 12:16:29,045 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 12:16:29,045 submission_checker1.py:3345 INFO] --- +[2025-06-09 12:16:29,045 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 12:16:30,170 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,8 +27,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1919 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.9139 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.72994 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1708 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71324 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 4d52c915f..85ce1508b 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1331 +1332 diff --git a/summary.csv b/summary.csv index 67b3ded9d..7b5a42f28 100644 --- a/summary.csv +++ b/summary.csv @@ -5,8 +5,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1919,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.9139,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.72994,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1708,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71324,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 261b31be9..21af53e73 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9139, + "Result":23.1708, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.72994, + "Result":1.71324, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index d54fe59b0685b88b4220cb5565985f6cf4a623d2..bb96ddbb699f3ac56464216acef65943c9b37328 100644 GIT binary patch delta 1575 zcmV+?2H5$zRk&5KatICwZ-}dN4FCWJaR2}q0Fx0Z8nc=RLI{6Xwl0HGl|YFmi6;#p zk_n{&L^23v1p|m>07_=oXbGjs0Aiy6lqLa4g8&3A5uK)#W}&Rh$`Hy50+6m*0NP>l z?=WX;SzskBi}sJUv!6PuAF2NEt5>MJcBoj*ehhz}T3ic1n0G7J{x zBoj*Whh&h;3g!>X0GA}=ODs*|59tR zN&3+s{Xk1Z20>RA%DSu!PqD-df>UhG($8aDeiK=?mPJ-N$ri02ty_T;l>CZ`c1y@a zdBl^_56Og5`XL!4vx4-)GC(HD0GTG|hjjpCnv@?6$`7$xJ<0S+&?JA7M@l~=6H4iaWDv~?(hti3nj`~gnw%fj z0ibD8el#dQ&=QeB(3FL;E-S-LlXyX}Y1XX#d@Y(!Qe|sdWTj2B==^BArJb-u$I~Us zk4AOGm&y;xC`&p&B!p;IaDG?@&?Ff^)1>^c4gh~mlkua$_<@#)T{c7$j2}{lyC&%x zcg>oOpYMa_o+>|Ax~Tl*yQUu&{%*}KnO3WXP3W+bxACgg)RDVba*{q=cune%t#2sMMr2eo`fK8M6qrv=vmWWOh*aY*3 zl;Or%#r)AVn?K)&&3*NMwDfA$paIy&j zoVG~-VxvenO&xS>5P+a1V$%eV9m=}2Om2UiBm^6$y-)tn-zBowvdAhMr(M*5a^JwW zN)EMli3Vh&PvT1rh-8!{Js=W7HY<2QECXzkjC{grn=~NS0ibEpfNao!pd})MfX5D1 zT~;NxO?HRCwrTI31@vu2_FC3gWyiFO5>U2dTCOgW^Xw7}$VQ*UmlhDoC`)QUB!rVx z6p?>Bzd9vIqDkUQ|A%ChB?TZ7LL@5)Kr91DGPMSfG#Nl_6hP8s0NG#wK}*D@DL-W9 z9Ll=143Ml~0NHzP0Bw*ApM>PTia)lqi)0W4^6=U&e9YS! z>(9QsREHnd9~1Ww&&ErpgHirB?R(?(huVLoqv=g!@#p&E^1Q>--?f!@)*oIj$8jV5 zGK-I!@#k4~59Nz(HU^)E>krCn%;uwHK1#o=N6$a0Z#nLrOp?Kmt4S@Vqd%ANk3aN( z@D6|Idq4f$&_6Yf8`;wDquJ_KqvT((X`LL+dN=WTZ+Leb2%WA_g5JmU>3ZuO(+o7QmwgYX+s)0gXTDlX`BoVb9NNSz@edT8o$g0cHy>sTy zpkjTLP4o-a39U+$B`JzPG0lbAm*~CT#1ANPo+?fot)N6FaA;lK70eK(ExcN7z2o_Ij|=-E8QU3#SzRWZl8;Tr1vHk^1TfIpsse&cOiWatAPg#cuTJ1HNFr{{Vk zgQ`5u@8fhCr>i>4Nwy^U>Z==h)qe)camc@k6JW@F{Ryf+vnwXb0t5$dh^w>tC_MoM ZuJJIdB$GQTQ~@fJkSZev+$R74007r#=^g+8 delta 1563 zcmV+$2ITp;Rk>BLatH+WORdxrRuF)6%>vL4lYfUfTgw6~VOg|)w4MFb zQ6;~>t91#MsF8Tm{vnxA+CLBC_kE_T#m5 zU|VHeIwf49KjKO4hh##j{g4cDe_28MVHx0(WPnSP^}{*x{b=0^oS@`aOtf1_!Ne0L?IX|oeAk(D$Xi$EjB_e|mGC}zvWq6Qff7b^NvNbC|--paSe|3JYbkX_A zUy$`ex9mx#SAr(_lRQ%TA(>E0KO}=_R*-&J2GArKK-1*>unquClk%fM`GJ;*41%UC zlyzAdZkogkf=#n#<>zbBe3B|#%OWdnnnmYF+b!*cB|4riQGPV4BfeCANJd%G`5_@h zvx4)(GJqz@0GcM{hjjpGf0~RR4aN_&MC`I5nqd5pGTb#u*SKreZ2Wv5H1|~bxza`D zC*L*wu<&84%c|JW#!_|pF&8D&WUh=c%6mXUC>jD(X-2;j6$0uUQT!fEQDV}k$$ zEfJe0c8K8Y_iAd*p* z^ngeR*{t9Ju?(i;k8}(n72thzll%d@p#ebC)wp%S8iL^pM7_!4nM3vChj4g zjh9RZqx^5$_r~iFf3-_T)0@WP&-KUUd55RJYb)=pKfGLy<3{>r79Tg`&$H|v$`{*g z3_cInAC%Xa%}2?6lzv%{o_|u`a@;$aB!eGUlUhzke=g%6f9U_<9sbbwe)_qge`*{z zvZdcgv(>Fe$-iLJIysv4ZsPOa{AM&=G{*5TTd39Yn~nMUDx>#SU#H1z^_AOfd`!|b zncRPO8}|nBJpV5Hc{faw^!}Utl59W1@Z>ME4HnoA1tvJI)g_baBp!dwYQr!PgztR{ zqq{BHCZQF!18vEvfkItcdK1zn5w(7h)H;6qDo)}gkW-JVnf-QlQMK5qF8KxLrO`FY zGlCM(+z6>RHF|HC={-t(U|O)wXsFQu9xa-?id)K!gI8xPxFEqN5u`okwnp0!EXCOK zEhy%*=+IGI8>d)^rQ3gC%lJ3j04@lbV+DZ;78u?GQ#%pS5h?hDv_GyJ0fJ-bK!FY( z=UI-=-T+Sdn+>CK;Z!mV7H(Qk;^f|WIT`!D&-!w-#{4<{T)n)FtEN)#PsIT>mEe>+ zV8OVi!k0xnT*rJ^#S^b3h{qx7H{Onm9Qr^cF#tWpJt-gM({m-eL`^}8SxV+9d1&*3 zl9JBKul>lY{xeV=hy0s(0vvK*e}d)@vnnRa0tEI;t=O~pC_MoMCOEFuC6hWTQ~@iK Nk18Vu-6sG5008xC>7f7s diff --git a/summary_results.json b/summary_results.json index 7b002ddd4..6e1a0028a 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.72994, + "Performance_Result": 1.71324, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9139, + "Performance_Result": 23.1708, "Performance_Units": "Samples/s" }, { From 7a4ab999addb95b42ef727e32a159003a4df12f8 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 9 Jun 2025 12:17:10 +0000 Subject: [PATCH 705/719] Updated results summary [skip ci] --- README.md | 26 +++++++++++++------------- dbversion | 2 +- summary.xlsx | Bin 10936 -> 10937 bytes 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index e2b98002e..aa37bdde0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 12:16:29,045 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 12:16:29,045 submission_checker1.py:3308 INFO] --- -[2025-06-09 12:16:29,045 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 12:17:05,727 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 12:17:05,727 submission_checker1.py:3308 INFO] --- +[2025-06-09 12:17:05,727 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 12:16:29,045 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 12:17:05,727 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 12:16:29,045 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 12:17:05,727 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 12:16:29,045 submission_checker1.py:3324 INFO] --- -[2025-06-09 12:16:29,045 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 12:16:29,045 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 12:16:29,045 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 12:16:29,045 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 12:16:29,045 submission_checker1.py:3345 INFO] --- -[2025-06-09 12:16:29,045 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 12:16:30,170 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 12:17:05,727 submission_checker1.py:3324 INFO] --- +[2025-06-09 12:17:05,727 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 12:17:05,727 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 12:17:05,727 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 12:17:05,727 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 12:17:05,727 submission_checker1.py:3345 INFO] --- +[2025-06-09 12:17:05,727 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 12:17:06,804 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` diff --git a/dbversion b/dbversion index 85ce1508b..8735ed157 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1332 +1333 diff --git a/summary.xlsx b/summary.xlsx index bb96ddbb699f3ac56464216acef65943c9b37328..94fe6ede7b1e0bf3fdeafedff66fe2a587bb6d3f 100644 GIT binary patch delta 339 zcmV-Z0j&PGRk>BL=tR$5H^+Y8kia^{f zdz)IsKe7#J-ySz<(^l$<<9cSlon5(_Z&jE4g7ebo8s!;5321JF)NPI4SBvx>B|b1M zSZ6fUXa^q6o12PT%8i3pXDqlN!6y-|AWn-DC;*z*l2=CkO~QCu6RScs)-f3ao! zo3(%oLQ1S4Fu?-DdthoOB03-ipOE&)bpt?f3>_%Y!Q(v3@yQ#&DSx$LP|lo6hMk40 z)}uJNw_Z-hzVEYsHdtf+9Dgof-iB3EsrRSifSO8h${ny^TvOrmA|9?|J}l#j*Am3z zF6vj_4vUO^Ad(n>j&YC5$L#5OCAC0JL5kaylqq>wVIRk&5KCjvYW!mjs6j;!ayheJJf7e;dla z=^k*Nrb{frQ%*g_9WXXC5sgUBW~BMCO%LD<(%o$6l?$g7{$${$ zbrNUy(TUmEwr$cbdTYp^;m_^MTfb^7RCg){s46%k%mVe=Rt3H+!r>a~{5G6;CxAbm zf_~#|Tx9429)$p8h&w4Ci>K#$B!j9v&F|xM8K Date: Mon, 9 Jun 2025 12:22:22 +0000 Subject: [PATCH 706/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10937 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index aa37bdde0..554ce2af9 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 12:17:05,727 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 12:17:05,727 submission_checker1.py:3308 INFO] --- -[2025-06-09 12:17:05,727 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 12:22:17,230 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 12:22:17,230 submission_checker1.py:3308 INFO] --- +[2025-06-09 12:22:17,230 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 12:17:05,727 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 12:22:17,230 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 12:17:05,727 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 12:22:17,230 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 12:17:05,727 submission_checker1.py:3324 INFO] --- -[2025-06-09 12:17:05,727 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 12:17:05,727 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 12:17:05,727 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 12:17:05,727 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 12:17:05,727 submission_checker1.py:3345 INFO] --- -[2025-06-09 12:17:05,727 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 12:17:06,804 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 12:22:17,230 submission_checker1.py:3324 INFO] --- +[2025-06-09 12:22:17,230 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 12:22:17,230 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 12:22:17,230 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 12:22:17,230 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 12:22:17,230 submission_checker1.py:3345 INFO] --- +[2025-06-09 12:22:17,230 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 12:22:18,386 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.419302 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422074 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8884 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 8735ed157..b9e1221b5 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1333 +1334 diff --git a/summary.csv b/summary.csv index 7b5a42f28..23e8e668f 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.419302,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422074,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8884,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 21af53e73..659d1cb82 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.419302, + "Result":0.422074, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 94fe6ede7b1e0bf3fdeafedff66fe2a587bb6d3f..5c00995934f428fd283d9f6dd2c2963609274f9a 100644 GIT binary patch delta 1308 zcmV+%1>^d;Rk~HMatIC*wZ6-A4FCWJaR2}q0Fx0Z8nc=RO#=h9|`j3)w&Ka~sWE60GpsLHN zVAmw`$6d2#&*u>`h4J(CQ0}SwbES6WAIJ3_zvS0ER9HeKN+q7ue@G^j{tw9@l@$yi zmH{f6VFOf}Bp})YP-)VD6nH=<&=Rpx;8z=Z|IyO_QNCC5bki>Je{2*3%Sbp`M#9M^1aR6W0f>zv;WTy7u|WWWmWWLgJa#DSf6_9!agq>hoc2EXKYy3V zUdtk@Y@Bvc1Im2^-zquO+9ev0jXsGlH6W5vmh^x~2-&RQ0kI6QNiy;Yr)|=JSO-_3<4fIRCQUE+&0-A0^6p&cNWmM71?W9UzHuxE=oYzj%m5NOwO}QEFc?w z5?@+CB%>^;e*uvYQdvO*Vi};4WPnPO1;jc4Doqy9_QnS)!2)8J4Lo)z>yk3WvWf*% zSirtx`6Q9Omi1M_5+oqM^aE3h5|9cWup}QuA|)V_QI@oTNC>g4U;(iVV9ER%ILMkb zAT|ncY0`jf(14&NV$+l#dT|bAU0MbQ*$RLD$KG=de<&U1NUA6KKeeo_()qRM|7fcf z2DXaafCW$ZoACgg)6o5zwk*pv9u?!%|)EYq2WB{>I07;VpWPjN2eiNU@<+yh;Nd`ZzCbgW7{#?dC{?Pxy zJN%*V{q%D~|I|2cWJ|w~W~*C`l7GRbb#gT8KHbFUz4^^(x@e5!VYX1KMOU|_?V<=GP(cmHtr4LdH!AY^KO_V>HRnPCE0$2;mKdK4HnoA1o~pNRg>u? zAAieE!!Qs<_k2aBtJMaq7vQ1* zQjRcFqMi4KVC>iqBy~w}s32~&mDGpQHh82GYl`tJb49yZ27a#;)sC(Pe3^h`8V+h7;|5Kg6a>mD<;YU1roKs%XE|aC|dyz zlU6A_0hp7MDM0~+lhr9Y0r!&-DkdBHVzpHz0RR9{0ssIJ00000000000001hf!LE# SDn0=$laMMS2HqzC0002_8fORq delta 1294 zcmV+p1@Zd2Rk>BLatH+nZ-}dNvv&w^0|P?a(zEId=L3JOO6)rol%R>ih$p2Vk_n~t zLo$eF1@DJt08QrJz#E$;_lNcXXqxmNCFPtmXo<)u;P^mQmsP>8N#>8cX3d_@BV-EW z=j)-|Q}^de?aDun>pOnQuX(7jgi4f3JgNVXOep;yl0hmf7(groR5HT`s5D7Hv3$r#8@)3U{%zgCKM`ma}&d-U5I9 zvX_BM%pZ*ki7(9`l2Mjae@F<~tf2m|46w<>9bnU>{;*MiO_TYf!Tf=ih)xsO1oMZK z;l^3T{LwX=Ki`MVef58|^naA^l|0?FOZ*=jMG{~7KP00pDFBfWz{xTaPL`2yvIzm4 zwn+eDqewVS9dv9EfS@H}(*%zl%DR8FOm3Vc1RJNlPyWx}C9>DD$SNDBUDSYb-@vy@ z4z+fP24tg8;!6#PWRxX6AQD10D|kRG18kCve8Op)G$7UiplQ;8Y|wz9B_e}>#|~9p zRwcJhc89>WY44o{^le4=FyeMxVr&77)oOOKN{WB!pB} z(12J5s3aMn(qsX#4uDFN1+=~Kfl9D|*kuEc9m=|-46&?Y0TmXo?^r%bWUpm?m9PW} z$S?iCl%fQrf(IBZUAkN z44;JLzKTD#vx{U91oH6OE_}?}B%a^Ir}22aX!Mipa;+=3t?SReyHtONAJ!ie_Ylv< zOQwTS{x|J=oAKvab`Rx?Z8ip< zhwBf@Ys}`OWIjs2tVhp3sc$*%olKI!kE=;7r=vfY@sB_BfA9`}=zBl?+|WNYjvLw1 z@1xo3R-@!!uxXtf&3ZmJ@p*54Gny_M<9L`Y)N1+7#(e$Jd#kV0WVZUsZ8kn8X_`## zzq^flgLs~Qm;JmOCP{k#O@2wXA7Oa%7qbo)*bW3-3h8r`>Lee3%}&EG5QOhMMdW*8 zClvv#B$WX5L?tAOK-?^Qn_9#_vJGk99ye*zR_cl4dS<_!UAdZXRhRsN^U~-V9Q~+li;R6Bk{E!FagWN!?CE(WwLnclirbWwDS24s1 diff --git a/summary_results.json b/summary_results.json index 6e1a0028a..9779aed8f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.419302, + "Performance_Result": 0.422074, "Performance_Units": "Samples/s" }, { From ec09dfd212847eef4274a3cb28bef78d4a280502 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 9 Jun 2025 12:24:37 +0000 Subject: [PATCH 707/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10938 -> 10937 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 554ce2af9..efd50b3ab 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 12:22:17,230 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 12:22:17,230 submission_checker1.py:3308 INFO] --- -[2025-06-09 12:22:17,230 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 12:24:32,083 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 12:24:32,083 submission_checker1.py:3308 INFO] --- +[2025-06-09 12:24:32,083 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 12:22:17,230 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 12:24:32,083 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 12:22:17,230 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 12:24:32,083 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 12:22:17,230 submission_checker1.py:3324 INFO] --- -[2025-06-09 12:22:17,230 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 12:22:17,230 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 12:22:17,230 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 12:22:17,230 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 12:22:17,230 submission_checker1.py:3345 INFO] --- -[2025-06-09 12:22:17,230 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 12:22:18,386 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 12:24:32,083 submission_checker1.py:3324 INFO] --- +[2025-06-09 12:24:32,084 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 12:24:32,084 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 12:24:32,084 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 12:24:32,084 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 12:24:32,084 submission_checker1.py:3345 INFO] --- +[2025-06-09 12:24:32,084 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 12:24:33,181 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422074 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420403 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8884 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index b9e1221b5..6f2c618b8 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1334 +1335 diff --git a/summary.csv b/summary.csv index 23e8e668f..acc0a8eed 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422074,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420403,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8884,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 659d1cb82..a6b2ecd4e 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422074, + "Result":0.420403, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 5c00995934f428fd283d9f6dd2c2963609274f9a..a5a988bfb10ac3f413901ba5f5ed9072925ffd06 100644 GIT binary patch delta 2263 zcmV;|2q^cuRk>BLatH5ih)3Tm-+&z{7u3Enly|UP$*t(DSTBAYM_BKOD|zlqCP|H$@>?=3EjF$TzeXl+ zZm)a&{3*>r_6%m-)xk|h@Ztb_hiX{Q2XU_gHVX^b zM=%R}W=^KITIvaC*0IJan}6qKVrug;bSv<^l4p~OzYFzD;Tt=yC(*=A;7T+xgJ_Zr zph+@-CT0kl+Jq+Q0ns${(AnPjfTnDT=qP}uEbEdoc!8C5ZCese{U1QHjx|U0Tjy>d z(Ijuc`w~seAetltXp#(|i5Y^XW0MaK7k_S>q-)$Zo&Nxub*!<{w(0C{n}MVKvTd`( zN>Qj8xS?bdGeIEP#0;`YGQcLu0Gmvu0XAI|o6NKUHVr*=wl_YoDO(~sN+{WsWnET= z+a`u^+jRe7Y(C{kTgMtJZJTa!+jLuA=$4^b;_oOFLp&+)kW3&A9+E*cD>ytX1Ak~T ziwDp&X*{e4MAKyQC{THzC8DDMnqc#gGTb&v*SKxgDfE0VH22l;QM-zd4lS!pty4RW zkM+PN3L>7Ad`KpgmJi7wn-x4CmH{@2)Pii9Tp!j0vT4$Nl$3GKpe16XNSX5-%DS`+ zwoT&xz_wYl*z=Wa>L(p(_tf~gRDVU~C)+k%yWltPq8XH+iNc5{r5}?n%_5f&_^dBYVoHJ;N$SC0WKvkDj!LCW>kGp2gp3fs>3ghSNq1;pV z=SuC$KaT4=e#x(SsIY`eluA6Q|By^5{U4G+Dk~U3ECW~eQGiX8`J=)7ftH9)6W9dvhm_&QS;hR( zHJd-*hs}NUf3);}l<$>1-Ly;m9~(syU-~~Jqbw-^kr2SiG7?Ufk#MpJ0i3o;0Aiy^ zI87aNY!HB;C1TSAj~&Xof3!?)oFoJrr@c@9&)+4o*Rsee8>d~=fO6l!w@MDRc8LaL zqfg>X4TxlvB|RV#LN+USKr91nl8k)9X`3`4)&Zbt(tvExfS@HJgMh~lRb5skw@r43 zz_w}codxu5MfO_OS7pbvixN<_V_L2*lk@Bn3&=*F#FrKj$tX)|e?TOJR94V{SO%yh z8KBZ+0kIB%N|Ob&z43ucuz=WQ1CJfbx}*%TtYQHb7O?MFK1pP+Wqp;f1PRD5{lJu> z1f+roEXfCvNC}8!lqD@75<)C1SU@ZTSTer`4zeZZeVnlvCAG$3e+*fiybUYtW& zmzKdnw!)wPvG-g9e@cfrlIltRPc3V!bbc-RKiX=Afi640IweS=N#aZYhh&r`1t1ba zBr6C&ECWa~wFZzh89;0lK+E4Qud&%V1v~JYFDE7dL7@R$a$(bZM1?C z9l@bhbyF}yn6~h0wE^n|xF~>>Bg~X&@4X=yJGKW&T@oBBh+AzX^`W#oY$*GtJHUCG zF0lkpIe+yOpMbHMh-g4^HX+TAZ8`vF7#fhEyu(?N;j`C+m2S47Q!boR_@jYY>nKj{ zgAbk>kR!=Ia%w{F!~sMDz!psL`EFbmXcTNU`S2#0H^^P6zutpNUb4EoI5 zut?trJPH9wA9qwf7EjOh8dZ6k-^J-NP9N$lCm8uHS>#`*kyrg^pd5z$n>Ygc+}EF= z`UA5pCdvW@^L`{@bCdfhTLBG|S1CLJmy?t!K>>u5)+ssx_LC7RCL3{$PaGuy002+| l000pH0000000000004l2*OO8zJ^?F}kt!nw-6sG50094CB!vI~ delta 2296 zcmVpR`@TGl~-skBpp#X6Q! zfyMWJfkh{1dD?$+If?;CF+dc>1L!$M0(s`b2%tzJ5=9b`D3XXoF)*M=$|Z`1@(z|G zih%(|tOpbWi6W^HoAz5WEiIOH5&U~ltYaw^Q2Y?=Y>T#|eW&C&8gdpxWKmG`93z2v zkey-#SR@h2B8f;AF`_K-(0mmdU?lbOY1kV>eJ2#CoXLNNh>al6qq3+=isUnox0rF{ z^Ovw*;3mC(vN#y`Ry}+Uyl;Irtz&r=AiN9<5QeSL_FTuh|4(rmmBn#fwF@4_z5JYH z0`V9=B^e-+Wh9a;Bbg)_C0|5&wGA;UX%7-hQxBc(jax9GC8DEPaWsI@VZe+q8?@X4vvA zx8#K%#oyz9rtqzEH;`zOH{gAVCT0*#k^wYH2GGO|LDMmzNqRsu4Lx+WH$I>#TOv9N zpef6`qzt!B(lu_I&VK;SI@VZe+jMrf&A@-re%ZEJVx=h54BSw%iJ2gfY+?r4BpG0n zWPnYk(g2&TiA`qO0Goy$I@=o`*pw|19VL`(%Casi!)+5oxNW-sFgBlZq^)C(m9|Z{ zxNW*EFLcY$Eb(^~iXom9ct|FY1`o*~niU)#mH{-G#RF)XG#=IiqG>XD6sSDV648HA z08OxYNEvRMq-)$Z>lAvv7n=KO_^4gQM~9YGrq-z)$H#hL69o}ZNUGSTC(F{t^L}A2}(hq;hgwpyU8AP*!_ro%PCUbA#jZKsLLwf);P5O_L za?TmFL}V0je4wh!s$kb7^T%DYX3yslGKKN;^-%7q`*WpsQINAe9viAeI3tnPCG|nj|3F15jzwfE0K@&=Rpx;8z=5s2zBGSGMp;t*At7Y5g8IWU zz$O!SfK8M7!$tu%P3DgV^9NcYI!#~`%pX#Q8)p^sN7rord>=OV)&J4b|53hI@^sTK z@qcU-Nqp)5kc_gV07OCnC(B4USw_OiCIoQWCIN_zBH=W3(6K=Pf|iI)6Fhb(>wnTR zxp9&ZY@GH!`9FV`$X?4Lt8AQhQ3J|-1K%n+)Y>H)kc~cxFEt>NQI_<8NC?@i-~q7= zut_rV38!t+fLI5Brbz>`K?8!8hztTAJ5+U9mE1Pj9Rk~?y>}MSw-woISznbM(=JLt z*^X(sx=hZqODrH8eG*?t|17aDVl4O8NlLf>&04hxu(Dud$D!~F` zmkm62DC?3k#IlM7R9L{iWBDYJy_WS=!V)APzw`rBiV~0t9R>P4Sy&d=18h1 z`9HO+tKS|6$ZNO{OXh-i6)6J{U4H1mK1`p$kZb13W5GC;C|0c7vF0klCfd=irTD*o8cE|NhI$ir*9@G)es%kvIT zf7e#tS$}xB9LJ6H%Pc-_#-C@|J(MrD*%*8tu0JTRF`JK)`6&Ie9zFl0zU8=gGD!wM zt|qmdj{aQ6KmO4F!8`n+@BQ?1L;ut`Ze&Zpk7lb|jgo)Crgd^O>ptDY=e_yOXu4>O z<6*W?tK~Nv^YusXt-el^+3G8|+4z{GX)?M0?l$fX;(7jE_VaF-BBtJMaq7vQ1* zQjRcFqMi4KVC>iqBy~w}s32~&mDGpQHrP=1O&h>@nl7;fPdR_}6!*Z`%tSOHIh&E@ z$F>82GYl`tJb49yZ27a#;)sC(Pe3^h`8V+h7;|5Kg6a>mD<;YU1roKs%XE|aC|dyz zlU6A_0hp7MDM0~+lhr9Y0r!&-DkdBHVzpHz0RR9{0ssIJ00000000000001hf!LE# SDn0=$laMMS2HqzC0002yKQ!I| diff --git a/summary_results.json b/summary_results.json index 9779aed8f..ea9600dd4 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.422074, + "Performance_Result": 0.420403, "Performance_Units": "Samples/s" }, { From 30356cb035076b6019102c35ecf0f58e66e744ed Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Mon, 9 Jun 2025 14:11:44 +0000 Subject: [PATCH 708/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10937 -> 10942 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index efd50b3ab..950bceb95 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 12:24:32,083 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 12:24:32,083 submission_checker1.py:3308 INFO] --- -[2025-06-09 12:24:32,083 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-09 14:11:39,123 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-09 14:11:39,123 submission_checker1.py:3308 INFO] --- +[2025-06-09 14:11:39,123 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 12:24:32,083 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-09 14:11:39,123 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 12:24:32,083 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-09 14:11:39,123 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 12:24:32,083 submission_checker1.py:3324 INFO] --- -[2025-06-09 12:24:32,084 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 12:24:32,084 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 12:24:32,084 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 12:24:32,084 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 12:24:32,084 submission_checker1.py:3345 INFO] --- -[2025-06-09 12:24:32,084 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 12:24:33,181 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-09 14:11:39,123 submission_checker1.py:3324 INFO] --- +[2025-06-09 14:11:39,123 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 +[2025-06-09 14:11:39,123 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-09 14:11:39,124 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-09 14:11:39,124 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-09 14:11:39,124 submission_checker1.py:3345 INFO] --- +[2025-06-09 14:11:39,124 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK +[2025-06-09 14:11:40,180 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,8 +27,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1919 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1708 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.71324 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5553 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73226 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 6f2c618b8..7345521b1 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1335 +1336 diff --git a/summary.csv b/summary.csv index acc0a8eed..f6db8c7e1 100644 --- a/summary.csv +++ b/summary.csv @@ -5,8 +5,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1919,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1708,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.71324,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5553,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73226,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index a6b2ecd4e..2d84e104e 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1708, + "Result":22.5553, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.71324, + "Result":1.73226, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index a5a988bfb10ac3f413901ba5f5ed9072925ffd06..88bc13facb2c284b770922d1f06cf5c5f965b29d 100644 GIT binary patch delta 1147 zcmV->1cdv!RlZfQatID&&T+JN4FCWJaR2}q0Fx0Z8nc=RLI{7ib=i|ktxBLolf;t- z5Xpqn03sQLvVsA`G5{qrYqW&YWB{>I07{bpq(K0JmWWPMO0!VbWn~Cu1p!FcECB5= z`FEJJwJfj_mPPwV+u2VYRr340T9;sn8i^+1eEQVmaBEian>%efNbj0=USwMf=8y~0y3y57d@YtcOOUe+-Di%;-0sD^SlSKAf z)>jEjkbwNs4@@aaKq`2^l6(+}lz>P^S<(U`A;hwR1;jFdCG%_GAZyZq*eJlINdvM$ z1A>-_O;dj8#W|F9X&D@3EByH%d(SnXbeJQlp5*`3vbIX+*P{QUtyUQ5k{5qvZKniD zG)a8v|B#HbqyR)hh-3u;h-Cmtrq%$GCIg6#0!W$+AR7!IXo=V~<%i6iLs^%W0g@FA zAbZaZpbe7YlaSn3@yB*{kqm-B9$wppk9nKK^PBiI9*-A|ev)0Tb>+5o{n>Yy>hQz* zW8xm-*?7rxFv|a?eQ&(}P`iJ0G`(po{#<`to_BcqySDPq`oqiRIBuk0X7O<|{yfX> zp?tB;#^CdC{Xuz+*?g4DN9mXK==mr0Eyul+Niz6xHL2xv^yf1E@rV8o-r)~@@28&| z`lrTmBU}1?G+W(jl>7@et&^i!?v!eR$sZz z#>XU0lga&ew{dR}&-3rHpLfF~N$Leb2&2GXl z5QOi2ipclGF=>)9)hRjw`jZeUCL8P_@LwhY002+|000pH0000000000004l2+mlf$J^?V3 NkSZev;wJzA003!E87BY$ delta 1113 zcmV-f1g87GRk>BLatHbqy!j!UF!KSnjL&qj$|8C%bRFvhC8deGzX0e{$czw@MDRc8LaLqfg>X z4TxlvB|RV#LN+USKr91nl8k)9X`3`4)&Zbt(tvExfS@HJgMh~lRb5skw@r43z_w}c zodxu5MfO_OS7pbvixN<_V_L2*lk@Bn3&=*F#FrKj$tX)|KqQ1zR?vW02B;(%pweUk zu?~PrlLfTBfAN7zuz=WQ1CJfbx}*%TtYQHb7O?MFK1pP+Wqp;f1PRD5{lJu>1f+ro zEXfCvNC}8!lqD@75<)C1SU@ZTSTer`4zeZZeVnlvCAG$3e+*fiybUYtW&mzKdn zw!)wPvG-g9N{2a;>Ph}jEo-ZEel7Yx+G>S?E<3+EeE4Qud&%V1l`` zB_x7C>@0iQTEq|8hO~c=n^#*YE0*JX=gggvi`79k(Jxphv?@`Sq$mQ#G#6@DqW5|o z-=oNRsyJ=5f)X9Up;dKLFhiKO@M^UI>jk(dfRrQ5lxXk0As9Qh2T5HL94d%gZ6)=g zv^#%nDEp>6z?;!^%S3gv6+ZyKyo%A&5vz50B0B)kf6N7S(4$i*MpUAwxLrl zoKpCsfm!P)PVR#fld)~vq+N8@kUztpo0qq4)mW(0sTiQD;EXT})N5N6__7FxYpC;^ zaN?~1{&)=f%-gU?-v>Mj0Z1QrR6Z6@&-Eo5Re74<#pyCmAL=Y8`7K%GU#F2*{b!&Y zhWwj20{Yz7pP>2!voI#g0tNGaBw=%t|0r7l4U=0bJOP)Jm?=R4gp=ASIsx{R6)GkL fag9$LC6hoZ6#>_iRw_OLE0dKfBL>|k00000jNAbs diff --git a/summary_results.json b/summary_results.json index ea9600dd4..eff54345c 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.71324, + "Performance_Result": 1.73226, "Performance_Units": "Samples/s" }, { @@ -1004,7 +1004,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1708, + "Performance_Result": 22.5553, "Performance_Units": "Samples/s" }, { From 5f99549d2b1d96275f739110f0f2b15f18f0d4b6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 10 Jun 2025 17:55:17 +0000 Subject: [PATCH 709/719] Updated results summary [skip ci] --- README.md | 32 ++++++++++++++++---------------- dbversion | 2 +- summary.csv | 6 +++--- summary.json | 6 +++--- summary.xlsx | Bin 10942 -> 10941 bytes summary_results.json | 6 +++--- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 950bceb95..8d79c1fdb 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-09 14:11:39,123 submission_checker1.py:3301 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-09 14:11:39,123 submission_checker1.py:3308 INFO] --- -[2025-06-09 14:11:39,123 submission_checker1.py:3309 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-10 17:55:12,114 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-10 17:55:12,114 submission_checker1.py:3385 INFO] --- +[2025-06-10 17:55:12,114 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-09 14:11:39,123 submission_checker1.py:3314 INFO] Open Results=32, Open Power Results=0 +[2025-06-10 17:55:12,114 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 -[2025-06-09 14:11:39,123 submission_checker1.py:3319 INFO] Network Results=0, Network Power Results=0 +[2025-06-10 17:55:12,114 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 -[2025-06-09 14:11:39,123 submission_checker1.py:3324 INFO] --- -[2025-06-09 14:11:39,123 submission_checker1.py:3326 INFO] Systems=23, Power Systems=0 -[2025-06-09 14:11:39,123 submission_checker1.py:3330 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-09 14:11:39,124 submission_checker1.py:3335 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-09 14:11:39,124 submission_checker1.py:3340 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-09 14:11:39,124 submission_checker1.py:3345 INFO] --- -[2025-06-09 14:11:39,124 submission_checker1.py:3350 INFO] SUMMARY: submission looks OK -[2025-06-09 14:11:40,180 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-10 17:55:12,114 submission_checker1.py:3401 INFO] --- +[2025-06-10 17:55:12,114 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 +[2025-06-10 17:55:12,114 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-10 17:55:12,114 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-10 17:55:12,114 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-10 17:55:12,114 submission_checker1.py:3422 INFO] --- +[2025-06-10 17:55:12,114 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK +[2025-06-10 17:55:13,090 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -25,7 +25,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.1919 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9556 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5553 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73226 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -33,7 +33,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 15.5312 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 18.9759 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | @@ -47,7 +47,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420403 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 20.8884 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0042 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 7345521b1..a369ea607 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1336 +1337 diff --git a/summary.csv b/summary.csv index f6db8c7e1..d26206238 100644 --- a/summary.csv +++ b/summary.csv @@ -3,7 +3,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.1919,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9556,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5553,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73226,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -11,7 +11,7 @@ MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorc MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,15.5312,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,18.9759,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" @@ -25,7 +25,7 @@ MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420403,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,20.8884,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0042,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 2d84e104e..c5d9cc69d 100644 --- a/summary.json +++ b/summary.json @@ -129,7 +129,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.1919, + "Result":22.9556, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -369,7 +369,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.5312, + "Result":18.9759, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", @@ -789,7 +789,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.8884, + "Result":21.0042, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 88bc13facb2c284b770922d1f06cf5c5f965b29d..2ea41df10a613d73a297a3d5cd792768ab80d87d 100644 GIT binary patch delta 3469 zcmV;84RZ3nRlQZPatH)9G@U?`b_h%e5>E2(D*?vvlgbDjf2X5ldfZektJ#RB{bVqj z-W)gIzk7KQG#hEwn+|&8WEvkgzr<2dQm%Vvk|cG|y< zC%v?l%;IT&%P^TwdRhMM{HC4G=5cSZESikl+OoX%q&J#2S3MlgtMri!hogS{BIz$C z@ibd?GLOf-e=MJVdOMn>&6A_SD8H>(Ph%brkDJfb;py+T(`=s{Eyw;aia(`WUpI=S zd`yx*if`Twj+<5yZA13uavB%&#vmT{7US$P`Sdy--P~sRWpj&?elkv%|28J0{95W} zW77M4+_amGPoqJ0TXfvG9SsI?aiN=y{vyqi$?vP1f7H5zRjKmUwaQYP>ae98%hPUg ze|#Co>zejz)a49c^s?T`(LDLom@h|2XT9Q{QHNURm!)5{{#G;>b8D{~F7F!}`8tZL zcXwhRweO0K*NH1x@=i9qmkl4b8ru1MHj~nPQX1x9?atv66OJpUNh02E)b=F7FA10|M=2{F&BM-%TJO}u|J@xheXS*gQU>Oj2~E5$@8R{Ch- z-J^;3n2>j>sgIqNx^$(_T)a=kWB56Vi2MDNM8pMqK_UX!UrEH-jl3ig!P6@e5j?#n z5%CCmLn7kRcSuCse`grc;_5~uBChT^iHNJ)eEVRxVjf4BChUlB;xT0oW~z< z9(3T!2OLIxabZs}VvF^@AQ7?NUr9u)_a#P@B_0T`?la^k;%hR5Sj!tjdFTGN`yJ{` zHbi8C5{YwJ)WssX=e#BTYLU5knECMJ_5Uw^y^P*;&wgAUhtm8!bTi6s7av>wWYV6D zfAfDQ`B^9bY8KCj*Q4n$p64gF_&STz>^fb1oQ%@q@RMHOxh<=m&Et4^%xV{>srGqv zx~`wcX?}=F+w0TJ!KUZy{%m&8pDhlO>Gbn_G0pPh)WJG+-A|^&(M|K`<{-{`qjB2& z>8L#<8}@H?EGa*>#`)Bi|0|i-)oNa9e|5-cIofh8n{(%L?tJ7&1`b2dUz6uR6l6v_x?2Vzme-jE+&SXQxMv&)GS=1#(@|nk5%sBG-OIR;(lU_eb z55~P^51#|?Tc1tqSY8DPFT*Ve!&Yc}u4CQ*r#Owu;y5nb1&`uheoiuhcnqJC3=qjO z5=oYkOp=U}FQUBKgcy~y2Z^PrhtB544Vcgp(NV-sehy__QYK%0e9OAlf8yuT#nuA$ z1J8$@u|RVW$Fuw)IZxy3JJkyE&GD-Nt!0H32z|Gn*!sxYa)MBoJbP69_4j8AUvGC^ z2_|L&M}kRSw+Apu2EZg4024E~TW&%WOwt2_Y3QM&#{P~HAx7z z&8-FOBbZxzW;m%9|JwH&67F~F9DmAY7QcQ zZ*QAn%eUN;7k(6fkN=s%x6a)_qDkI>_a&N`K{QDQ&?Ff^6Eg%&$Al*70ns${(AnJh zfTnDT=qP}uEbEdo+%`$qxNSQB0W|AaW2J4=+1)k+NBd>lW{H)eP&05t$tGrkK(dJ$ zWRqloO_BjNnMwm}x+XT6X#;E;dgyF_ZhT-Hfpme9Dow zjx|==Hr?%Q(`|X7TZU$dzoSqL@ua{*GJ!OBNCwfY;P9{vpvf#AK+~l0upSUilgXn% z<$;!njsj?c%|ps?+az7%wppjp^S#jASHnl`Dn2^2tTMGu?KnQx1Dhy_cvA9zA(>EG zJ|u%|R`7gS2G}H03$kf)eOM33rb+ivQpP!hmWYiaWzKUb>(VmVHi`QK+h)yT&sVak zpLC?%Q{(4SZ7V-2XobF8qFJk;1Wgo1JSqKDd(I)OGHKi#|NsqtO|C2O)`JnHEZ^K9wAd0KVJ{!p1MC*YFGYoT;K6ae$7LL zB~+qR;z|97WJ2lxkPK2;!2n_zppqFjK&43nqCEhWCJjh|2Lvq<8wGx~0m{0h46!6# zBbNJS1C?UwKMBh{wSlfwxQnG71fj#R)E385qfX*U3y5SwX#tT8V97Fn5=)kmSdxsA zzbExdn=BwUiu_#>QwJRl5)iaRY?|`7iO!*{OUvXjmW6CF$qQYx1+@5>pHdfVE(Md> z-DG)?El#j1Z0WD3vX<3VIl<~}14y-eTbDh#)T#tZG)X*Z0Fg{64Iq+1C@UC1ECWz7 zvqnoOO$HDf1)wwuKpF&pAZUr`G^I2PWnET=P*xCtbjM2*Cg_7BN~(*7YC#Il0@!!m#+vuXfKll{X+0W3}O&*sJlEJ6OE(*!I* z{vl;>f@NL933h7%|57aX)%?-B=8u!#H(uEp)-K@^4H8e9KO_^AhY>q}y{-LtEgjfa z8JA88m*|gpQu`s9P-;IUgIre7epm*$BpKk+Wc{!XfJ>9~qe1$CmWT|3t}K*wSs9*U zi5Uc^*qWuE$GH3^vTQAjtaOsywtn)HY~TbXzha`@5;9R9@uc)aGNF`yNCwHQApNim zkV!H?rpftX9RQgouKzY;rtdRzG^c1=Ir z`nxr|Y?e4b8toBZIzJ?%EGhkv5VBc8`e7MhlPNp!+N??XVWR+?Cg(?k^8+mrou*V} zp{&cwaNi_m5bT>ZOFv%^=TnZfd#e6i>TUHW-#0Dom%R*BV*Y4UNPKDjkc_gV`a?p< zW(D z>`>OFWpd*rA=o(Wee!?)E|I;KMON83?QIPxUz_W?C9l%jB^r=_jXsGlH6W5vmh^x~ z2-&RQ0kI6QNiy;Yr)|=JSO-_3<4fIRCQUE+&0-A0^6p&cNWmM71?W9 zUzHux-j;y!9n*5PE;-KHB^Hp4K8Y_aAd*p*)PP6`sjQ#@u?$d2GC-xt0%9Eil_m>l zbK?V*U;(kq1|B^80g(`5S-}Eg8NibHHE@tMX+Ufg;L@Z4*`NVIOT?xrKlI`p%DS`+4zd;g{Exlo z8c;SYkW^3de`;AHm2VfkBNJT=i?>Q!KnC~_Pz1y zL+!HB^rn&ix%#*~@9^|@Z55sMhnI_S+{nJn;^Sugd6wTp#bTR{!RO)XgYp`)`6!u> zvM;OAi%;rXjC&_$lVtGYa#D-w=+8y`;}88Gyu%;*-cLU_^iPfBM!xj>Xui7TD8(0S zS|>-d-c5Yoo8OG4X=5A@^MzV1zuB0tK6-EYb(YMQU%Ac3$0W;=$^Cb?ac>aMi|_KE zcf%yf?!PH6$>y^R7Tpd7C64OiC6n$XAAf4YFc5_AeF~$yEz2Qk5w-(u$*F-tU0Qk* z(k2nLevs5Ue)}p;;w0cxkE@yec6L#@+^aVE1?Q#F6(U)V642ZTskaq+Z&v98N_=2i zu-0g(P!Aq0>${R$%8i3pXDqlN!6y-|ASyAUkJ*z+AI=CkO~QQR1(Scs+DVt>o{ zH`@X(@_d061SVKu_yA1pNJJ;3;3LxhxOM;tj-dqwI(SSnf-l|xPWj1(LAi1&8F~wo z*0VUe_g;?1uIsXHHdteRff$AnKq^!LFO!ujBL?9o00000-kYv@ delta 3451 zcmV->4TSQ&RlZfQatH)u&T+Jpb_h%e2q*dXl>lS$lgbDje=kPK^th>7R z<4JGPN@npiyJeWnC%rWLc7D@d%;s@#uqv92+uE|c_M|tOHrG8I&a3p142PqB{37Wu zC-F30cQTL1f4wxD{^E8tTQpCO2BYk@d_9eMJUnhbQ-`O&+fK86a zm+~=5{>Z<1GdONqd9)4Lm#b-9%o~Gv*jtX%%jDDRcyx1{W|z&)OZv%pvHG_$8D-Z} zHye}Q=i{c`Y+G`h^VZ+;=6r7LO~d7VLnB*9 ze)aB7?4$Nw-ti`JB}?APhWE1JL!qIa&1X9)%_pT{4%Y4*E-~S_Vwxo4?MB{_h`$?o zPa?r?f8+y+gxko@a&5j`8#+*8xtI|1ym~b8?$N~iM-v}RiJg@?e5DT5Yq3&Hgkq(S zCf+@oc#jErr<(fMS*c4``pm`qR6K^ClZd$APf0{vuoomEfc=$3oZZMv5)nMTA`!vU zYZ4KUpf@BUE`5hY#Qk@M5iPE6L?Ytqo|A~Uf4W@~5m)yuiHNIvK_cSn{zf7mf53VC z0p~#nu6)2@#1|L#6eG4+?+X$U>;08P#Cl(1L|Nj2@ajH8ej>gmLx{D!F_d@iZ@b^2 z&SXPGCMc0OmqlGHl6%fu(ytboi-(yHPhS83;@8XQP511_)p2N%ori8l>Fx4ktDj8T zf0J?c?<70xWM9qV`S5x)9mezQ#1>zt@glumEI&?0i~R7jxW02+Ry&=?@#>h>&QDYA z^XPO_KaUsLA!gCuoMsNTJzw`{vxEL@d5}z}pXbYInjNPOHmU1=G98X?nm;!OaoQV= z7tNoJ+C#Eo|5nG6vSVwUO>OnRl6hUNf9Ivvhm4k^EyuDscRuINNAARf?KwsQF^yA< z0CyxJxg!zD9f^oX-z(pM9Vr*ozA=<{uw2Ql=&V>TejG~jhOOVGA%7O zt_#0LCU0)9d;R<=%|Z4IX5H1nO-AJGOPb94x7T;N<+ZGX{!(eD0*iGlr2>oZfBgcB zPSEnS<#H4Qj$(i)iU-hhj0Ez`g%LoJL?ntNB2gp}iDFmvB~pjgLJDxmlw*x43sNBd66aWv#ChRC9z=s89L@gO_J2(U;Z zl0_1cEMi1i;-UE}G{8ve<eh-DYB&BWueELS6FgQSsN`pDBF3 z-E}3HmQj8xS?bdGeIEP z#0;`YGQcLu0Gmvu0XAI|o6NKUHVr*=wtqK1uqj(2I!Y+nlx1C3hTA5FaNBhMVQfC- zNL$AmD{Y%@aocoTUg(ygS>o>~6hk~I@Q_R(4IYv~G%GkfECXmViwDp&X*{e4MAKyQ zC{THzC8DDMnqc#gGTb&v*SKxgDfE0VH22l;QM-zd4lS!pty4RWkM+PN3L>7Ae1AwL zl$H<4Ae$9DAC>_&iPVB@np_{&1F~t-eUy}O&Y&e?qez+a9Ll=147N?;{=l|bv)J>M zZ0aW+Y4_Clxl~2vC)+k%yWltPq8XH+iNc5{r5}?n% z_5f&_^dBYVoHJ;N$SC0WKvkDj!GEqv=8wB(&7RLAWD4Ww>!I9J_vcFO%0G_lJATQp zd8n|2N|Z`GssE5nDE%LjK`JX4Kr91PGQ$R_G)X|T2cXiV0V(i+pe16Xz^^tyS(lU{ zmZWRMa^GyAQY`%^VY#O^(3J{zv9yCAbU2pU;#g|bNjzx*kxVEpAd&$rS${@i$ubg4 zl2P*aq+V&01;j>?zbj(uprb(of|iI)Q~oy5Ih1v2nLNg_kS!*8p=-8)mLIcI>T<)S zU^2a%tPZmI308$I{q(mWWPMO0!VbWn~Cu1p!FcECB5=`FEJJwJfj_mPPwV z+u2VYRr340T9;sn8i^J{xB$I;?Jbzu(e!P|r zY^#h*r-V!NM?9(hkW47GACf^XD`-C~16+~}aA~rBSO>tRN&3+s{Xk1Z20>RA%DSu! zPqD-df>UhG($8aDeiK=?mPJ-N$ri02ty_T;l>CZ`c1y@adBl^_56Og5`XL!4vx4-) zGC(HD0GTG|hjjpCnv@?6%6|{EL}U;`CMZ9o3=gvG@A|+&wr1t$`;fV(&d-%DIzRaf zvR>$xJ<0S+&?JA7M@l~=6H4iaWDv~?(hti3nj`~gnw%fj0ibD8el#dQ&=QeB(3FL; zE-S-LlXyX}Y1XX#d@Y(!Qe|sdWTj2B==^BArJb-u$I~Usk4AOGmy^B{I)A#T{N%f) z9~S;@%`TfI&W}cW#Fx$w$tX)oKO}@~R*-&J2H0fE4!kyNl784I0H?|M(ct_*OGKwB zRaq$OvNGH^i5Ud@X3f&i*TeagBki85KbN|w{$%^6rTwy(flACDjS7h`%^#9cmQ;U8 z2-&Ql{;&+N$;2IC)1>~eQGb9@T0Fe;D$ubg7mXUC>2?3n8NdRJ_NH|R$bZiiSpe16{ z1dkocy0lDgoFoJrr@c@9&)+4o*Rsee8>d~=fU>o@u3PddtzDu4*?;Jh_)-HR8D&Wi zh=h>M3LX&40GlKupK#hH4TyCBXqq%28#EwjiO3+}u|rjtRmp9W-661T+IwdKeOr;e zmi1NHG3}xRld=QD0 zfJjDJ(gGqO#Ik|~#4>;-^K0NBYtn$&D8Qvj1F}H_f|iI)Q-0{hIh1v2860FQ{P`by z&o!WQm?Np4G;ggWuSMkSoc99H%KptM(g^zig#Pgf@G#-x^ zjee3{u65QrhRX`{!qJgG`(po{#<`to_BcqySDPq z`oqiRIBuk0X7O<|{yfX>p?tB;#^CdC{Xuz+*?g4DN9mXK==mr0Eyul+No6wlaW$#s zboA#k{_%(Y58mMqeeb8A8~UflaU)y$eKcF$YLxs7Hm#GRS??x3@6B&U(?w$(53_|@ zEx*~AuRnTk^>v!eR$sZz#>XU0lga&ew{dR}&-3rHpLfF~N$MuE1?(a4 zUnZ08Bp-in!Y~ko?|q8M_rx)2QL&OBwH2o#RVr0cZ1&{5nNr&x%kYp{Q1{F^m^CwV@_3IY=>FuVt*b|j)B zQt%OJe_T5N1jo>V0v$Xi8Np|70H^%ThC#V-Dj9kUH?1debnm>==9CX+uZQ~@xPlqw?z;wJzA002#Eq?G^w diff --git a/summary_results.json b/summary_results.json index eff54345c..f0c2f8e0f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -536,7 +536,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 15.5312, + "Performance_Result": 18.9759, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.1919, + "Performance_Result": 22.9556, "Performance_Units": "Samples/s" }, { @@ -1148,7 +1148,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 20.8884, + "Performance_Result": 21.0042, "Performance_Units": "Samples/s" } ] \ No newline at end of file From 765ed6c13cd56a02fd480033892c4b70fdc03922 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 10 Jun 2025 18:02:59 +0000 Subject: [PATCH 710/719] Updated results summary [skip ci] --- README.md | 90 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 608 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10941 -> 10940 bytes summary_results.json | 2 +- 6 files changed, 375 insertions(+), 375 deletions(-) diff --git a/README.md b/README.md index 8d79c1fdb..3f04ea61d 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-10 17:55:12,114 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-10 17:55:12,114 submission_checker1.py:3385 INFO] --- -[2025-06-10 17:55:12,114 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-10 18:02:53,733 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-10 18:02:53,733 submission_checker1.py:3385 INFO] --- +[2025-06-10 18:02:53,733 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-10 17:55:12,114 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 +[2025-06-10 18:02:53,733 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 -[2025-06-10 17:55:12,114 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 +[2025-06-10 18:02:53,733 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 -[2025-06-10 17:55:12,114 submission_checker1.py:3401 INFO] --- -[2025-06-10 17:55:12,114 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 -[2025-06-10 17:55:12,114 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-10 17:55:12,114 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-10 17:55:12,114 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-10 17:55:12,114 submission_checker1.py:3422 INFO] --- -[2025-06-10 17:55:12,114 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK -[2025-06-10 17:55:13,090 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-10 18:02:53,733 submission_checker1.py:3401 INFO] --- +[2025-06-10 18:02:53,733 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 +[2025-06-10 18:02:53,733 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-10 18:02:53,733 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-10 18:02:53,733 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-10 18:02:53,733 submission_checker1.py:3422 INFO] --- +[2025-06-10 18:02:53,733 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK +[2025-06-10 18:02:54,744 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9556 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5553 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73226 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 18.9759 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.93654 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420403 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0042 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.93654 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0042 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 18.9759 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5553 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420403 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73703 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9556 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index a369ea607..a9449958c 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1337 +1338 diff --git a/summary.csv b/summary.csv index d26206238..b7e54a33c 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.93654,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9556,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5553,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73226,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0042,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,18.9759,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,18.9759,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.93654,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5553,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420403,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0042,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73703,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9556,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index c5d9cc69d..bd18115c5 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":6.28304, - "Accuracy":"F1: 80.0", + "Result":9.93654, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.290026, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":15.6951, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -95,21 +95,21 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":6.28304, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -124,22 +124,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.9556, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":22.5553, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -214,22 +214,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73226, - "Accuracy":"acc: 80.000", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -243,60 +243,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.431769, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "10":{ @@ -304,22 +304,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.6951, + "Result":23.0274, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.290026, - "Accuracy":"mAP: 76.951", + "Result":21.0042, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -363,30 +363,30 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":18.9759, - "Accuracy":"acc: 76.000", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "13":{ @@ -394,29 +394,29 @@ "Availability":"available", "Division":"open", "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "14":{ @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -484,29 +484,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "17":{ @@ -514,21 +514,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":4.48923, - "Accuracy":"F1: 80.0", + "Result":18.9759, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -546,18 +546,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -576,18 +576,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":23.2491, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -604,22 +604,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.93654, + "Result":22.5553, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":0.420403, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -664,22 +664,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0274, - "Accuracy":"acc: 76.000", + "Result":1.73703, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.431769, - "Accuracy":"mAP: 76.951", + "Result":4.48923, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -724,20 +724,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420403, - "Accuracy":"mAP: 49.593", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, @@ -755,11 +755,11 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":22.9556, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -767,9 +767,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -784,22 +784,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.0042, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -814,21 +814,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.438894, - "Accuracy":"mAP: 76.951", + "Result":19.7056, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -844,21 +844,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -874,29 +874,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "30":{ @@ -904,22 +904,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -934,22 +934,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.353561, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 2ea41df10a613d73a297a3d5cd792768ab80d87d..244517a8926b0b43beb3a2f823fca70acb127826 100644 GIT binary patch delta 1732 zcmV;#20Qt^RlHTOatH;Xxmm(?vv&w%2!Goz`~4BEN}xoO#FGXP$%N7XA{m6Tf&s)b z03|bPw1m=R0I^X3N|OMjK>&i5h)z>VvryJ$We8;j0Z7*@0PQgOcbK!aEU*%m+xAcX zq&alU-p16r1WVLNJZb-sOepOil0hsh*gq@-STd^yur%2}Y!txKB>!w~e83XqAAdSc zz!KyiQU)hj)-{}9w-)d(#d2TGAH8e-IQf0!m7QVj5-!mo@uc}fGNCkoNCvsAVE(WS za7i-0#L^`GunquAllP;+`+=5-41yjzlyylNc4AE=|@C>j1bkNk1B-A83ilAn3|M zS(laJDVCT)aEh&2`gx4YZz9XqvdBs&*=_45KgkA8Q1%4VEg=)-5l>1#Boj*Mhh&h< z3epeD0GT8MWSX2G)&Y=dQhqcjKhP49K?s?k{E#v{$g;od0|(ifm7niJ=6{|#KUaF& z`6*tI^}@1uFSS>KCi#;*Qu-m8P)a`}gJ@Qeepm+3BpE={08Nwfqe1zBmWT|3 zrYw|oSs8Ac#0!E=vu5SzYtej?DqG7UD{Y$F&X2ZR+6hZ^JYAyvXjDghsr-_&nX&_~&6=biHVVLLa(*;8 zKhP4-Y5U(?-JQ-S!9)s)85vA^0m3HTkRhL!CZIj(0ux;9V zX90a%k-e7nRoOA^Z3!sfF)df?lH;shVgcFcllamCA{k{#4S$G)kje@g5X%6SBm-2M zEFjhaP-(J&Ha9*{2^J8$Y~Zm&S(lU{mQ^gEtp)5mmQNDdYgu0JtB=d`4o`pAR?%61c)1wIjqJ-TK5oXJXZbx;EVkJgd>*boD6cV_ zkCOQ)`?4Cn_@utYxOXy120tz*wV00nT*N>A(Eq_Z{Gspt^m9Z1)HrVBOTUlit6Ppz ze8Hx5azHfe-Nfg;`ORpWHpcNVU#Qjcn~nMEqxY6yXUS~&mD_B5OtLJQ+<$i)_XhF2 z_%8o>H%yZ3{+r^GY(B#9DFg2p%o!hmu>$je|F5EVv-SClRDQ<+eim5G=*m^F1i$v*^%K+#07? zh^5QZpZ9k5_rUE=E^9UZ9TMNWMnk{E!V;+~by$@9y4ABpOM6w{Q<(wwaGIVA<1K5n{^H~kl& z8ixFvcm|wuvnnRf0tKPDS;BUc{wP}k5R+OdJOP@MmnlI3hm+YUIsy5U6e=bJ=(qJ1 alRhdI0os#QDn0=&lawkW2H+M-hdEoz0xMy;ZU5v? znnSnjZA`68utbf-llBkEgwp;Y8N{-J{lhYVC9`S(OOyS>Mgc5M^3UeR2P{GUp?}i^ zEJ6MuWpILJUBd}>YXSdKEcey?(Yxl4lixR9*%{U@;SvoJPntg@6H4=kWRS}W<`2sN zmn7p$EKTAM>j1Dcc|RJwA83ilAn36}S(lW7$IiM2k6qX7{UDa#Tx73hiB&{)y{-Lt zEgjfa8JA88m*|gpQu`s9P-;IUgMVCB(0*73xFi|i(q#Rx4uDIO^rJ!gftH92g03u- zby*pnVu=|9r`VdMpU1fTCbDcTi>!2#-L`)6lWgDwCBI^#-4ZfU9`U5~Lo%V1en4#+iO_BjLP0kPN0MIlkKN^%DXo<)m zXv#uamzCkBNxUG~G;3CVz81|Vsj{^!veKrx?fhuFrJb-u$I~Usk4AOGm&y;xC`&p& zB!p;IaDG?@&?Ff^)1>^c4u1emlkua$_<@#)T{c7$j2}{lyC&%xcg>oOpYMa_o+>|A zdRzG^c1=Ir`nxr|Y?e4b8toBZIzJ?%EGhkv5VBc8`e7MhlPNp!+N??XVWR+?Cg(?k z^8+mrou*V}p{&cwaNi_m5bT>ZOFv%^=TnZfd#e6i>TUHW-#0Domw&wsRAT;UR7iYj z{*a8ar20cb$Yurghh>0GChh>6CiRDn0&JSh9}VUYv_y28z$TbKqzpICD&~)_+5GuF zZ0@W7qox0&e6Qr`rd{Iy*eH_t(*GeDWk~^ugaA&Kk#Mq%gp*AO;IvHw5F16pY3iV3 zg8&3A5t}A>>`>OFWq)$xBq7*1?S1lp{w|TdmPJC|{fFx+SmD+9ev0jXsGl zH6W5vmh^x~2-&RQ0kI6QNiy;Yr)|=JSO-_3<4fIRCQUE+&0-A0^6p& zcNWmM71?W9UzHux-j;y!9n*5PE;-KHB^Hp4K8Y_aAd*p*)PI0T2&t@~0kI5FNismC z$pT^>0F@>SXmjHOm0$s}%LX1hlyylNVp+uk+FHQAWBDYJy_WS=!V)APzw`rBwk04H zJYY#ah(t<2B%>^80g(`5S-}Eg8NibHHE@tMX+Ufg;L@Z4*`NVIOT?xrKlI`p%DS`+ z4zd;g{Exlo8h=nWERa-B@_%YsTcz{sw*RB8Rv74#7iDdy1W7bWeChv?jIyKvL_&yU z1p$a<07<6S0FovHh>ZeBnhYQt3?OKU*fiyb%$!46mzDvN6$~JI&kdk8lHrq(+*k3( z-VuLP%d#9TkZf;#%-bZM-^8c!c$_x+Nq)Ijm7CVpXMf*as>2VfkBNJT=i?>Q!KnC~ z_Pz1yL+!HB^rn&ix%#*~@9^|@Z55sMhnI_S+{nJn;^Sugd6wTp#bTR{!RO)XgYp`) z`6!u>vM;OAi%;rXjC&`OWbos1Qj6*6&qe&>5B(p!!yo$IPd_*GPmSY7zV!QOzPjZo z#TRT^CqYNE-c5Yoo8OG4X=5A@^MzV1zuB0tK6-EYb(YMQU%Ac3$0W;=$^Cb?ac>aM zi|_KEcf%yf?!PH6$>t*rPyPb45Ej@D1SO8@;*;zoAb)DZFc5_AeF~$yEz2Qk5w-(u z$*F-tU0Qk*(k2nLevs5Ue)}p;;w0cxkE@yec6L#@+^aVE1?Q#F6(U)V642ZTskaq+ zZ&v98N_=2iu-0g(P!Aq0>${R$%8i3pXDqlN!6y-|ASyAUkJ*z+AI=CkO~QQR1( zScs+DVt>o{H`@X(@_d061SVKu_yA1pNJJ;3;3LxhxOM;tj-dqwI(SSnf-l|xPWj1( zLAi1&8F~wo*0VUe_g;?1uIsXHHdteRf@L0TGi~DLet2la?t#0f>{>DLMiAlM^Z?8zqkF s;w1q908jz|01*HH00000000000DytplT<1`0WXu2DkBErCjbBd07zRr-~a#s diff --git a/summary_results.json b/summary_results.json index f0c2f8e0f..f9d0c7582 100644 --- a/summary_results.json +++ b/summary_results.json @@ -644,7 +644,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 1.73226, + "Performance_Result": 1.73703, "Performance_Units": "Samples/s" }, { From d2206da816b57ddc4b38397c7ecb0b8e3b7c158c Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 11 Jun 2025 12:56:18 +0000 Subject: [PATCH 711/719] Updated results summary [skip ci] --- README.md | 90 +++---- dbversion | 2 +- summary.csv | 50 ++-- summary.json | 610 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10940 -> 10939 bytes summary_results.json | 4 +- 6 files changed, 378 insertions(+), 378 deletions(-) diff --git a/README.md b/README.md index 3f04ea61d..49ca41841 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-10 18:02:53,733 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-10 18:02:53,733 submission_checker1.py:3385 INFO] --- -[2025-06-10 18:02:53,733 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-11 12:56:13,509 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-11 12:56:13,509 submission_checker1.py:3385 INFO] --- +[2025-06-11 12:56:13,510 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-10 18:02:53,733 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 +[2025-06-11 12:56:13,510 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 -[2025-06-10 18:02:53,733 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 +[2025-06-11 12:56:13,510 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 -[2025-06-10 18:02:53,733 submission_checker1.py:3401 INFO] --- -[2025-06-10 18:02:53,733 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 -[2025-06-10 18:02:53,733 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-10 18:02:53,733 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-10 18:02:53,733 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-10 18:02:53,733 submission_checker1.py:3422 INFO] --- -[2025-06-10 18:02:53,733 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK -[2025-06-10 18:02:54,744 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-11 12:56:13,510 submission_checker1.py:3401 INFO] --- +[2025-06-11 12:56:13,510 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 +[2025-06-11 12:56:13,510 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-11 12:56:13,510 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-11 12:56:13,510 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-11 12:56:13,510 submission_checker1.py:3422 INFO] --- +[2025-06-11 12:56:13,510 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK +[2025-06-11 12:56:14,531 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.93654 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0042 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 18.9759 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5553 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.420403 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73703 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9556 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 0 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.3198 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5553 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73703 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 18.9759 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.93654 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418868 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0042 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index a9449958c..97dc46849 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1338 +1339 diff --git a/summary.csv b/summary.csv index b7e54a33c..3f6d2d47e 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.93654,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0042,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.3198,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5553,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73703,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,18.9759,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5553,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.420403,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73703,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9556,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.93654,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418868,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0042,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bd18115c5..7fc4580c5 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":9.93654, - "Accuracy":"acc: 76.000", + "Result":6.28304, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.290026, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.6951, + "Result":22.8483, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -95,21 +95,21 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":6.28304, - "Accuracy":"F1: 80.0", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -124,22 +124,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.353561, - "Accuracy":"mAP: 76.951", + "Result":23.3198, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":22.5553, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -214,22 +214,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, - "Accuracy":"acc: 76.000", + "Result":1.73703, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -243,60 +243,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":0.431769, - "Accuracy":"mAP: 76.951", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "10":{ @@ -304,22 +304,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0274, + "Result":15.6951, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.0042, - "Accuracy":"acc: 76.000", + "Result":0.290026, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -363,30 +363,30 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":18.9759, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "13":{ @@ -394,29 +394,29 @@ "Availability":"available", "Division":"open", "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "14":{ @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":20.9828, - "Accuracy":"acc: 76.000", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemType":"edge", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -484,29 +484,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.438894, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", - "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":" (darwin-23.6.0)", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processors_per_node":1, + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "17":{ @@ -514,21 +514,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":18.9759, - "Accuracy":"acc: 76.000", + "Result":4.48923, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -546,18 +546,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":23.2491, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -576,18 +576,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -604,22 +604,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.5553, + "Result":9.93654, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.420403, - "Accuracy":"mAP: 49.593", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -664,22 +664,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73703, - "Accuracy":"acc: 80.000", + "Result":23.0274, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":4.48923, - "Accuracy":"F1: 80.0", + "Result":0.431769, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -724,20 +724,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":0.418868, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, @@ -755,11 +755,11 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.9556, + "Result":20.9828, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -767,9 +767,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -784,22 +784,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":21.0042, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -814,21 +814,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -844,21 +844,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":19.7056, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -874,29 +874,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "30":{ @@ -904,22 +904,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -934,22 +934,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.353561, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 244517a8926b0b43beb3a2f823fca70acb127826..f6a139baf861d9e5aee09f9feb88643b82e782a3 100644 GIT binary patch delta 1833 zcmZ|QYc!kL8VB$f3^xib35l&?d_hvD2nO4|#) zvBrcS!<|NV`d{o4mm}u0(4&~+v=e_;JUM^ka!@s+qPL6`j~F*M^k5SJJZ)Ipu}vL6 z0jo}4<_SADA^2(C@elwHO3}o3P@C16c608oJ3Y}TwQ-BPni*f2RNnM9leh%#b5_p& zu;J6yL(n^$V=>*2Y{6lBF=C){@#Z&#NvCF9riYWJ>vgB|ZUU=lGiQI)bSW_=qCkcz zNSC!9gphx+s3NUKCeZkl$+2)LTbpEL-vjp_~lR@Jgk@6UyXXVY9sSG(~yuR!WylRwvu? z>OhX@(tLRauW>nr1ZeU*CbAmrdljb)5_F;0;TGatQCtk$H%9`q(=mdNsZ~F zU4bzZ&t9*oExv}Z(>ZlPa%U-6$|njax*t{BN4U@KJ|u*?ulR}X4~+6>BB)_w8aF$9 zgb|+VT+8C`zuGme2##_$&m%>=qedde2igNdTH^eP15?A=mRv|b z)&qT(O!sH2_+$ar_>)YimRgjVlZGM%R+@n`jK_6dwEZ}Ouzof{)8%BPz+&ptPXv6a z=2e(bZ&$*_gM3UbGOIi4e9|06VoW){ zNz<$H2Fp3(1luFi@vIYFiAPAAXWaSr(#skcp-8-iioO#YJ~i=e5Yeuo;cD)`We8@9 zygjYVz$2>6gMndKXrfr^e25QLjR=Po5dr^1DM_6#2I6agQVb2^mj|fW=~UH~Ch{7G zB-#=UD6xyNL~}5kg&9C3x7$Q>POH_2p#6S|*BBt_O-PE#`>iSU z166w{&A#PjOTV?5XiiG)<{TwA6bIbT4|wnEQoqLxv}`kBdD`2M+-+f(S;+L{7HXtw zFCVu%j@t7G@IiQftuN_nL{sHy%A7_hOTIF@xA|!2yY?Ye$6rqOgi-x2f)jF6Qy*dO z_EY6EAqKdZH_G%#++q{W0cCn?EvDR91rs}a4);>o>(`dJ8%##zq5e>@{-~E#;|$cK*gqjT({NP_ma!JB%W(&+!V4%G8?}&}pMGUpP}g zTVHx@J|Fs1L&e^W8zG&u4Nh|S5`X--8Caq_Hmg=&sqr{1Vg=@(9UCQ(k{$$W8QZOj zSkaOn^S(~9eT3?>(5ZEO)9xkGwtY6WZDjd}aCt0b_T>G@B7UlXMpRUuqu+A*HWr?v zu5aK5gnp?zu5R_IsMWp62*^z!dFC87W*5)jI!_O!w!{TD-SID&ds4e&TeAMRyry}z ztFcm3di&@3(VZcgAjNv;RoLHxV>B*x!t{$90={ zi~(6bvkuB{ySk=Dm_M-Jiog5Etx~3@)pE9{c<*K0BK`pW;F9+~-UU|Dnda`frz*6^ zNTC1y(%5^J(GV>*XypQ_U~{ZIAqDIyD`&_jww^WS)M2P-2mk~_0YC@%-=gPCv%{?2 QAa-oF^=ZWK7KbPQ0g?AtO8@`> delta 1834 zcmV+_2i5qyRlHTOatH;Xxmm(?vv&w>0|eS~ESt0F3!(vkP?tP=RQ&b#X9{0$cU=i4 zW&%foNnW=HFi8f$BpCn`Gq_uBLKIBW1A=Mjp`+w`3}?_1(NTN}rY!5SGT1dq2)50w z1?(f3TYF|frnXw@324@_#wwfVdSYt*GIT5Oy^?2>ioXl>OyL_lt|!sNOyEj1F@tE5 z44_FefF@>t2%6f2Cg}muH1yEf-1vZ|Y>DV7fTk?#k}`OKm33`f5>5RdK(mfDR(Z{J zJ+aNROV@V2vhyr|%k-JTcOqS1vWc0%lWbxJ*(4cYlVpHR@;)!JX`9$2Js_Kg9y*&F zAJ~*F5gjFvY|64OE5mIQL%40)|0p)qlTNgCtg+I6wrOu~n_#f56n~HZnZmcu z-9VyA-hlTdnwUW}Ne0j)89);=1Wm_;Cg}muH1yEf-1vZ|Y>DV7fTk?#k}}*jN!Pe- zI{yJQ>sVu@ZPVG^HUmfdW!q+nm7-8Ha6`!^W`aPni5X;*WPnYQ0XCUR18lk`HkoMy zY#Mrh=xlC$U{kh4bd*rCDa*R747W`T;kN1i!`OVvk+zOCR@ye*?QPR-d7)c|W{JO} zPz>>;z(X>DGmWYl5XoAf{%5d8xUE{V{ zr_l4g(A-zUN9`&;I<%}ZwNC9gKGp-9D2RA}Qt~00P+C4DgKSptd{_q9BvK2qX>xs7 z56GrT_fb;DIfIsnjUr{vb13W5GT1hW`vco%&0^12vZG%I*NECXmV_XghBG`T;t2SC%L|0pTvoIy)OMghkMs=BOy z3U*C0f7~@|_Iw^8Qy4#A59OY^KUZp3{&8I2@k@TqLxm+&qEzBZ{fA^i>Hm-nQdz+O zVi};488$$rNdlrh0F@>UNP!0gEfE_9ezgJ0x}*%TBwZtx`(^``V(C8#%RRM$u2i^- zr5yyJ!?DyB$5NwC;z)m1sc z>TLr^wS3zz`~4BEN}xoO#FGXP$%N7XA{m6Tf&s)b03|bPw1m=R0I^X3N|OM8q(K0J zmWWPMO0!VbWn~Cu1p!FcECB5=`FEJJwJfj_mfQAE{-il{%ihM+x&%wqNIYr(kW47; zACf^VE7(6Q16VSv2Cy{QKWr4h(j@e@F(otYH4I3~)&@zQocb{;&=JOOyAb!TW)h zhzx=rJCt=v8F=ihYw*~0&E5}U`OQW4T9#NvWY^o;kJr+HZIyB9lyHguh$polk_n~u zLo&!^1?`7rfJ>4AE=|@C>j1bkNk1B-A83ilAn3|MS(laJDVCT)aEh&eS^9a5%Wopf z*0RV-C)sW5CqKysPEhs)(=8zr4#*H%nH&E%K(`q17w<jMYbnw6jLL*|}3KUaF&`6*tI^}@1uFSS>KCi#;*Qu-m8 zP)a`}gJ@Qeepm+3BpE<|)8zcH4ggJ)@}oidftH92f~G8#by*p1n#2o&O|xd@=WEe? zk}6xvA}ej0+s==+TiOXrbUa<6{Ag51e5w49jIyNjLqdpV1?Pul08NqsG)>A6>j2O+ z89y3~A83i#WkWQ<_#tJuYm%;U*R0w2`95gwsq%BBx0Rn_*Yv{%t-o8d%Vx6#6XOvC z=(qJ1lj|fPe{RAs5QOi2ipclGPACXjNs!u#Q;{l_suefOUchSnBin7@?dyaPK;qQn zdS<_!UAbHwRFnLI^U~-FDFg2p%o!hmu>$je|F5 zEVv-SClRDQ<+eim5G=*m^F1i$v*^%K+#07?h^5QZpZ9k5_rUE=E^9UZ9TMNWMnk{E!V;+~by z$@9y4B#G*R6w{Q<(wwaGIVA<1K5n{^H~kl&8ixFvcm|wu-+qGn53?;M$^r$Uxmm(? zll&-K0T7c|DLet1la(n!0f&>8|b(76(s=x08jz|01*HH0000000000 Y0DytolT#`_0WOo0DkBEqCjbBd0OmPeN&o-= diff --git a/summary_results.json b/summary_results.json index f9d0c7582..f9492ef43 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.420403, + "Performance_Result": 0.418868, "Performance_Units": "Samples/s" }, { @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 22.9556, + "Performance_Result": 23.3198, "Performance_Units": "Samples/s" }, { From e66886a09a0e6cc482987804c5402460a41c2140 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 11 Jun 2025 13:02:44 +0000 Subject: [PATCH 712/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10939 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 49ca41841..aec1d6e63 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-11 12:56:13,509 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-11 12:56:13,509 submission_checker1.py:3385 INFO] --- -[2025-06-11 12:56:13,510 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-11 13:02:38,761 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-11 13:02:38,761 submission_checker1.py:3385 INFO] --- +[2025-06-11 13:02:38,761 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-11 12:56:13,510 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 +[2025-06-11 13:02:38,761 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 -[2025-06-11 12:56:13,510 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 +[2025-06-11 13:02:38,761 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 -[2025-06-11 12:56:13,510 submission_checker1.py:3401 INFO] --- -[2025-06-11 12:56:13,510 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 -[2025-06-11 12:56:13,510 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-11 12:56:13,510 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-11 12:56:13,510 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-11 12:56:13,510 submission_checker1.py:3422 INFO] --- -[2025-06-11 12:56:13,510 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK -[2025-06-11 12:56:14,531 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-11 13:02:38,761 submission_checker1.py:3401 INFO] --- +[2025-06-11 13:02:38,761 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 +[2025-06-11 13:02:38,761 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-11 13:02:38,761 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-11 13:02:38,761 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-11 13:02:38,761 submission_checker1.py:3422 INFO] --- +[2025-06-11 13:02:38,761 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK +[2025-06-11 13:02:39,849 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -45,7 +45,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.418868 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422055 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0042 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 97dc46849..5c2484681 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1339 +1340 diff --git a/summary.csv b/summary.csv index 3f6d2d47e..ca2c84195 100644 --- a/summary.csv +++ b/summary.csv @@ -23,7 +23,7 @@ MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcomm MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.418868,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422055,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0042,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 7fc4580c5..bea575e0e 100644 --- a/summary.json +++ b/summary.json @@ -729,7 +729,7 @@ "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.418868, + "Result":0.422055, "Accuracy":"mAP: 49.593", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index f6a139baf861d9e5aee09f9feb88643b82e782a3..8703ce1943414ba9f6f9cff290b3cca405055e1f 100644 GIT binary patch delta 3019 zcmZ{mX*kqt1IA|zQ_5*9QA3%**hflZNit(8Gsaj;Gvkm%ktGy|EMqH6wq}qm%VA_r z*0D#HLum|Rc3v zKMs3>C}g`bcEkFGKlFVSr9IxC6OD1pcH*y>6Jec>#eTsW>+LvfgFj;%@_r=bH$~2w zdR@LV{{3U8Pt^Csdg?;6Gj%XVxkY|}<23W?tGn~J`5&)_)pIzfX5Ty7+Mk}7HSTS8 zW;IcQLSEQ8uyq&JQ~Xsbh@I9d62PW^&p2THH3$+>AIaLk_pfJ>cN`1{{a0*~X2 z|HR6QUKV|-1O^|+H5m6>PPX|PpF5iu^F7Q7ZiUcv8VJP~(cdu7L{K1cen>iB94z9w zXTN{-YRV}Sqh6;NrGGlwZ~0 z15&6qesH{!mTN?MmcqIYn_`yLY&ATnq|}7{Y+b*32ibS3V+~2c_6}_AnAb!E-`~57 zaVOmrFq|q?tkq}MX2T~Q2D+Xw+uzwe|ERpKv5a5re7Ia2qI?{bjsQ7Qg~DQ?6KdKr zM--NTZK#btt8bScpLdO2RBGn8N%JE?KW8HN$LELmeu#9JUaErI*r|QzM=cX?QBr&E zUpiM43VkKM<-n@Ky_tISu;^>T>0z3ylY+N=X1W0i2kt{aF=FTW37&?ujWAcqhl?Iw z!5Eub-h(haaU8U%wd+-_T5QoiB#+c7`Lb#LRnEx#DBZo$VKgwU#bw9r!^Nj%>Z2WI z(>Sejksef`3Yz}+tx%3< zeZnb@=?)Ry#;fb9bcBCA{D#)3d6>)T$U-pYJSo-T*$@A;di3V*#?v=s+!d zv8?WDpw@4f9%yOnS18-b>Y6HzrnuPKJ|0(i>^=T~d(%UI4r|vbsp^^auH$`jHDhS($&^GfC%n$mVo5BaRP{tm?GA!Ro*uYo?>hQM77P!xYT=6 z%Y4R5Y!ce-cB}CQiD+qv{VE1ksuFIhQKJQ zLBHgX^FZOQlJZQp0lwqPQ;mfSDSmEHkF1WL;ht5uD;HEp2Q73tUl?om8MUb}@oK{< zwf>T8mJtd^2VJj#iEhd4lWenhAZ~d4^rId-VL#o0}D>rhZ=5od`*HtFo zaE?XveIVX_a*JW5xK=1&tdUp)w!#YxsXBs2 zf`}R~!4;@cw8!yQ4uZ?8P^;)HjG{|8_*jD#i9?#%j&%}W)!+Eu=z+a<-`Cj=nMr+q zb4Bz_KYXolo98Nu=W0WV<9>yZH%_CugM-+wI0nGeHVfKz!_#7fBBU>-@Y)#gxy1eJ zh+oen#A&^ymJv)iFOIYQ-Nm) zLy%=aJqm}GEQGnnzJo7z86Ff-IW675JFb+p6By6FC(9mUzB5%E^;|Ytww>m8u{isW zsD_pFTHywdFC`QUW|hEyRtP2IG%!W1?VG0o847bIk>5aSBG9O0!7hk$dn9tVQQtec zl(fCA@BD?rSieuNd!GAIfTWHL+O!Umccr3Pa`Xc{o@C5zfu(2{0D%fb5qf!aeJEwL zWXdl(*MI4Xa82+Ll)G(cvqX;~gIR-Qa*7&O_FNn3&nt{O2jshwuK0C4>w>V9@VaP3 z6t9S(ydv@jZJ=DcQpPTu0|$v|Cj-=m|Fdyag9 zuzAZf0n{RVf=CIBpfb>s-MkksqQVJhDOjji_O))9ebx4Q$jHQXt_hxNvhBbcAT)Mx zE3FV~bgq#`7nM&~5SWdW```=}RLl}?b<5(1YsEzk3>tKTfxo))btJz8&z^mYqZn5B zjP1^H_v~eZ)Si?uea6NmEt|E5{qPR-8axUv6(V}$E(TG6Tmn87W|G=KPEbdh^Q@Tp zcpx1v_f(m~%x0B~MX%L>a~F%|?%Mli z*Wi)AB*_Po44x##nq>Dh03{z}VKjSgm1CLP&Xr%@V$EJkw65;U4%5*Rz1UfXraa-v) z5YymbbO<(~i*J5y<0)s=y=_>~d(Au#A_dOKndJ^LYfTr1J-H@!rDyx()w{;b1*xvo zEe`h}6<|!1FzX1PFdINMD+MA5U|}hNWE2#mbrRDjTItpW$+!^BJfj9esGh|9$dSUh zBia>Z8p=N=y_VZAPII|HhL29}q}So}T4F6`)^t2KXnVe;D%zzl-#0g2r9{JQVPc!Y z?>q5DkydNkD?u~=i5_z)SR%;%y#L)NaPq^_>X?e|GDQ8vO#Rc4j5nPxHli!w+8EP%QSqVwF zG z4QhY3d}0*jSv`QwMUj3H<*le{beR+|;?^Q2{FpQ?e!5CK%x3>~d-3l& zsX{?VwM!L!d{>*Y<3pzIgj^us3XNzF(8((j&GtQnJUM)Iiy0HWoDN|J zd`#e9t4`WUsGE_E))&SJVIV#hU3a$knB=^ocTaXvx%8+KEbhd5L|W4F&Dm>QXmO&^ zK0yz7tf3rjuwI2!VDtU^(@on`vI8EY5ta3T>9ka31K=atu(B?2lP0R7Aa&rQJP-h| dEgb|ft4mo5br>(YZaaX!Ku-_btmBVaIrjo;y zn;b)qQOHP6-OH(x6h+BYnq<%P{PFztT>o9K>-G9v@9*LBy{-nIMxTn)FohMR^3JI0 zzhQI;UTvpL`%jx3E-3b7 zj$eINJKK@=7B;)(FQdK~A+lJ*i{Vlv=;(}k^P+32_fu4PkhNfMS8iKrgVh5e>qd-_ zb3yuj*{!aL7u6qx{^T88nJVxOUTA*Uk-zS~H9)W5+`%l+MqHf)f8D$_*7{6{ya!T) zWyo?`k9W-1*7iWY(>$x!MC` z11(;sH>>WwW>AGjUrbH6v+i=+DTm=CkUd<`cmn!* zp`!dyzr!KnlJ>LPM)?IlG+l`zGPxq01E=K#{4soJN8R_Qajx{5=j-}Q!%_er%f4^z z&zH$Fz7v;>`cNByjUTfg?LK$ZJ)f1`nn?+T4j^^xxFitaDcqt)i7*Jsr_SV6*Drtx zBeH-%1fs}@{6%it>}V~Lzl>uZ-x6Lb=FeUX%{h@(_UR&dnz+jQuU zk#?0TJhl}j1m#S{93u?@mX-L6c9yC(XDxTzR+t1BS~(*|GRT3xSH@6RBFFAaJ!$_K z-=xg?g$d|GuU=U`=0(&SZC^pJ>PdH9j=0$v>&R^a&%0RcjHhr^ZsV~`)NvR;_5W}_ zpmWOp8aulRp@=FwoyUr*8|eTv^?OfkujaM`dB-QNa0pN3V`ZFOYkWuu4VwuaK1-ls zhoHk2U;9VKf3z0SSTpE6k{2(EsBNruO@IP zPkRy{n@EzQeqpQR6WiW|$nr?EW)|V!hS(tZDDI)yZipG6u|=U@J|{8bO)b8mlV0tX zH+?CuzWhqss894+3Xy#B=ICG1rHYoG@P)!D2W;N`Xea1M>4c<0r2=25MH-q~uFzu> zxStCOlg~iSG&UXRzA^0~5|AtCnoEkcz5I?_iM2dBn6oqd8)Uxs(Ta!j3LwXeMr zADw*u@G@YK?ct9@##4O1F#MqXq|d5NmDJM!MP0@z^*sD+Z#*w4SuR!GJBqkrj!#;B z5H6aNSnn^|!D!Lzwuu7Do|mn>ebDkYYlWYv>~hz!xShMKE=(Ad8t4>O#ev(gc1F8s zOlPjV<0X(OL6Q#^;lUtxvRO&V>9~K#pzM=?q&6Z` z+s;4Jj)h_wt0Fi`00(%Es<#t15dV-&n&0On5&}uFeE5J`v7uDxXsc&x&M0 zS;oEkGB$|=wu`K=m`G;2a2AYssD;ERkRQKEU-gyxeMb`f>u_GFOLuG7iNN?LW#Qi$ z@SX_ssuxx5riP8LM2se$8m%PR}m(5JAv5DBgx3@05u(#Y#?vGKIt?!F}h z;Eemt+-=RDR@QItx)?Ijm^Hf1p1XqAGg-rvN!nX%bCS`mnVWiAMki}a$-wQE-*jHT z>2?z)721l3)#Ak(iY-3#9yvYJvX?%6Xw2+Wf|1oNb5DlFKU-V88BPa@gAxM+$LhGn z$FeaY1o2Vt5~BbRg`cdi+9^xlT$KSoc!iA}{}ca0*5>=_;?FQU5<5GrR z8B3?ys3DnLU~77MYP~b2D}Xz1P_eK4uU8WxzP55U~}2jQPqXt zcck&f4%?r}4AL`!2zb*c9llfsCIgS<@ZXTqJG>entB0q4;zmzijy1pFDLUuOcI38a zt*%u?*$$cD5dIsc#-$hTF2$K>*v~JoHYC-N)R=7_+H8AG;OF&??Qn7sfLE?Ga zW7sXpS07SeZz{KaZoY{Sq5C1XB&}yr%^i5*J3s`^*K0pXnd`yWp6bj!A+hns)Q)IJ z>y_D}%Xj9aaU|%URl3bT3R^lJ{Efv!apuM~*Hc%Iw&Pa&rEU9G^5%biJ>wvKzUS1G z%fqzyB>}02quYw2PFvt+U$##-OL{4M%g%)zL3jTZ=cGtJ7A*0CP6%m|%rRQDYZWGY z9Qb?F5_F;Ihq?5^ZdcCSYR)8wX8Y*N)b?sMtzk8Jy)B(J+>b~MK;5rV3!w{(WA}Uc zUtBlXsZqmVKeuB1G*Txx1irw&-8na>)!E85`|@pRxXBaOb~C*gRUsZ_H=M7$ zdQ|ugllr+itDthWp)^NTbUk;X?_1AU#Yxj|Z@f2q1S-4LL$s1hT%rTpAJ;rhjIv(v zjXOVAV7qY6e`;9KkOiLE`I#uWZetVSd)HhXdh7kQ47RGt%*JSY;jb4VQv|8PT{nkS zf;A+yJ_2X|t9l!%NAADBZthQX09c)S#Kan$&CM{e2VdceOsv2^xEiLYO>22H0)s#v dav+cf=zohDJHkC|Y6~{wrkieszBXPv`46_8o+JPO diff --git a/summary_results.json b/summary_results.json index f9492ef43..8acade82f 100644 --- a/summary_results.json +++ b/summary_results.json @@ -140,7 +140,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet", "ColorKey": "availableMLCommons", - "Performance_Result": 0.418868, + "Performance_Result": 0.422055, "Performance_Units": "Samples/s" }, { From c1e7838d0cb5d74deaba3853df49ec31c46169d6 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Wed, 11 Jun 2025 13:20:03 +0000 Subject: [PATCH 713/719] Updated results summary [skip ci] --- README.md | 30 +++++++++++++++--------------- dbversion | 2 +- summary.csv | 4 ++-- summary.json | 4 ++-- summary.xlsx | Bin 10938 -> 10937 bytes summary_results.json | 4 ++-- 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index aec1d6e63..6c4cca023 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-11 13:02:38,761 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-11 13:02:38,761 submission_checker1.py:3385 INFO] --- -[2025-06-11 13:02:38,761 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-11 13:19:58,334 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-11 13:19:58,334 submission_checker1.py:3385 INFO] --- +[2025-06-11 13:19:58,334 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-11 13:02:38,761 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 +[2025-06-11 13:19:58,334 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 -[2025-06-11 13:02:38,761 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 +[2025-06-11 13:19:58,334 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 -[2025-06-11 13:02:38,761 submission_checker1.py:3401 INFO] --- -[2025-06-11 13:02:38,761 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 -[2025-06-11 13:02:38,761 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-11 13:02:38,761 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-11 13:02:38,761 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-11 13:02:38,761 submission_checker1.py:3422 INFO] --- -[2025-06-11 13:02:38,761 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK -[2025-06-11 13:02:39,849 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-11 13:19:58,334 submission_checker1.py:3401 INFO] --- +[2025-06-11 13:19:58,334 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 +[2025-06-11 13:19:58,334 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-11 13:19:58,334 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-11 13:19:58,334 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-11 13:19:58,334 submission_checker1.py:3422 INFO] --- +[2025-06-11 13:19:58,335 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK +[2025-06-11 13:19:59,323 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -27,8 +27,8 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.3198 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 22.5553 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73703 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1314 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73051 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | | 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index 5c2484681..02b327e1f 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1340 +1341 diff --git a/summary.csv b/summary.csv index ca2c84195..a10ce40d1 100644 --- a/summary.csv +++ b/summary.csv @@ -5,8 +5,8 @@ MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-refe MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.3198,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,22.5553,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73703,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1314,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73051,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index bea575e0e..984fd6cd9 100644 --- a/summary.json +++ b/summary.json @@ -189,7 +189,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.5553, + "Result":23.1314, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -219,7 +219,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73703, + "Result":1.73051, "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index 8703ce1943414ba9f6f9cff290b3cca405055e1f..2c7105dca152cbf5fe0c381839937440b9707292 100644 GIT binary patch delta 1750 zcmV;{1}XWvRk>BLatH;sfyDlEvv&w&2!Cyrz3`(|36yA(c+vnOnNS))B!f^^Fo0MF zpk!u^mQb1uAT|m>X%c`m2td#h(P>I)7RtJ;456$b0O^_qpdBXv4s*7a1y;gx+y2pZ z%T{4ff~79O5;YP}+CL-{O8bXo5X%bo56b|S%&Gw_P4*8P1+X;9KbspLumt&sPJa`y z1o?-Q!3mai4JX*G1^i2~+*k8Q@0veOe&2XyXIQ(0OEgG4Y5tH*D9s;|K`twpKP&@W zl8i60G>Jc~1HjVc{b=xhpd})MpvMknT~Y=fJL?)ec3rdggIIoZk-e5BRuS3tw)W$- zbYNR$TskFOqCetE?T2JSsr`@)a(`Jt`(YX2l4O8Oll8+o04`0^j|S-nS|TzCy0TE# zWo3AZC1wzuVr!Ou9^>+x$g;I8veHR*+xpSE6*xi3ub61JgiMr2JSqKxv82SBDt`O%>KKubghA!LH`L(1?V%l@tp9As-&e!dTxdw=TuT4#)ODgBTPqFF)uVHrS^WB^T*^TRp-G)>Bn2IU7@A~Fb? zvQXA#Ww>b)F9HEL&|X1BwgdKS+nu;ebC%f<>yLo zD?i1q>4$dNOS2`;k4AgMm(CB#C`(E|B!p~MkbYPO*ksBMyf$l+e%L4gr^)%z;QT;K zM5ifLSt#qWGTb+b83g-g&C<`;!}*jW?VhSXmwH?M$@fi5`(-Z!m4BE&8Wj>>nm;6? zEUEsG5VBc8{b3nklZiXPrb+!_qX3&G^GAdE11%AqCa?+S4=KZqvx@nnYc_wr51aey z|7hv|DBml2x@njAKQ@XazVv@cMp;q-A|Zg2Wh9&|BjIEd0yu4x0K`U-aGE;k*dPEw zOT?xL9y^qEX_?$ONq-17PJ5sHpTA3FuVs-{Hcoq61Ijl}Wvh~n(=O3~Z1hQdsR5CU zvZMz@Lda$X4~S)eO_GsMIBk;##5w>pO&X958W6NZWDxM!p{mQO`l{@h_O=9+@0gaWbs3c<7LbiTi7zc6l2Mk_fJg|btbd>Zu?$d2GC-xt0%9Ei zl_m>lbK?V*U;(kq1|B<@hjaF8`=Kx`D?(xd^|paDTk#HJ}f^x_=Ky0i=qvK9XP zkGHm<7vZMe+LWpDq0f=P) zNv75Sk|qO)jRHuT3?Lf}AZUr$H06iPoI_ccmI0C#3?O^Y4WKoW;ggWuSMkT*5r0(6 zvK%dtY;S$c+a#Xf#HaCioHqJNez{hao7UB5-(9N14}YtViF=6W<0aFzbP)s<|7PG{sOZO7T691jh0EmljDE#1CJK|vlL*qDa$BK&2$o{(`5qMWS#;auigMo`O$`6xp68P zIt!!Li#WIsUJk~#ZL_xMtua5rU#r*me$`a!7#AJin}$B&eR`lWCgIQbN|GpyZKG=bPimyZ$RsornCJ zcmbSp-+zMo53?*L$^r$pfyDlEllv%J0S%K^DLes}lawhz0fdv*DLMi6lMyN=8;zDp s!X*I!08jz|01*HH00000000000DytllTs=^0V|V{DkBEnCjbBd0D+i6V*mgE delta 1764 zcmV!w~e83XqA3A?cz!KyiQU)hj)-{}9w-)d(#d2TGAH8e-IQf0!m7QVj5-!mo@uc}f zGNCkoNCvsAVE(WSa7i-0#L^`GunquAllP;+`+=5-41yjzlyylNc4AE=|@C>j1bk zNk1B-A83ilAn3|MS(laJDVCT)aEh&2`gx4YZz9XqvdBs&*=_4b>sH_dWlu2O5;9R9 z@uc)aGNF`yNCwHQApNimkV!H?rpftX9RQgouO$`8pXOFBO!glJZ9epm+3BpE={r2McB08M|B@uR``ftH9}HbfJQA5w<9Cg~b? z&6JQ5Rn@rpRHcjde8wJ=jnLirLA83i_G=WVpe@Gc_ zoK?&pU9g9ZdG5g7zLcBtyI zD!Fa4I|Q~(d+#iuZ!5Cbvc4)iroAlzd=QD0fJjDJ(gGqO#Ik|~#4>;-^K0NBYtn$&D8Qvj1F}H_f|iI) zQ-0{hIh1v2860FQ{P`by&o!WISRj9?p5*`3vbIX+*KPktTdgqAB`?a_P6?7|lK9g9 zAsJ;!0f>YU$qE7x%K(y0tpOxW1`rzskTe-UHW)zA60vE@51BcKvMwzHBr6y|_MRI+ zYb3)bA-S*OkG&)QsFr0pS|Hip`k1##Jim!gREK{bRv#1h z5YNX;rh`%OH|=}l)rZ<;qv=f}{d4tkdEVja@7gLl>kls%#~=DXc!xjqy`O$==${(L zjeP0%(R_8wQHn3vv`&s@y_-Gwyf?oYP1D9W9_9JwSJJ)I(hrbPU0kxQ;(~e{dRUyHQ%W&`32{t(KRAj zjuOz^2&vl|eXJJg14?{gTCmP&sL>ugnm2b9x0D+PZ_Ze7L4r>rNPEg{jkY0Jim~Tg zP|Ro1p`*AlPO%V6*J6Lm_%~|-7kNIz3IY=>FgyTLI}yC+1l?;0eH?1dea__vHjD6o{eK}fVeuBT2ukYiksno-%IH0ByoN@;& z7}r$zvWSQ4m=DW%;*A9Hw2%6Yx8ov*J`hO^Ko4 Date: Thu, 12 Jun 2025 20:23:17 +0000 Subject: [PATCH 714/719] Updated results summary [skip ci] --- README.md | 90 +++---- dbversion | 2 +- summary.csv | 48 ++-- summary.json | 608 +++++++++++++++++++++---------------------- summary.xlsx | Bin 10937 -> 10938 bytes summary_results.json | 2 +- 6 files changed, 375 insertions(+), 375 deletions(-) diff --git a/README.md b/README.md index 6c4cca023..1cb5e4840 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,55 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-11 13:19:58,334 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-11 13:19:58,334 submission_checker1.py:3385 INFO] --- -[2025-06-11 13:19:58,334 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-12 20:23:11,890 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-12 20:23:11,890 submission_checker1.py:3385 INFO] --- +[2025-06-12 20:23:11,890 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-11 13:19:58,334 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 +[2025-06-12 20:23:11,890 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 -[2025-06-11 13:19:58,334 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 +[2025-06-12 20:23:11,890 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 -[2025-06-11 13:19:58,334 submission_checker1.py:3401 INFO] --- -[2025-06-11 13:19:58,334 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 -[2025-06-11 13:19:58,334 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-11 13:19:58,334 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-11 13:19:58,334 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-11 13:19:58,334 submission_checker1.py:3422 INFO] --- -[2025-06-11 13:19:58,335 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK -[2025-06-11 13:19:59,323 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-12 20:23:11,890 submission_checker1.py:3401 INFO] --- +[2025-06-12 20:23:11,890 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 +[2025-06-12 20:23:11,890 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-12 20:23:11,890 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-12 20:23:11,890 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-12 20:23:11,890 submission_checker1.py:3422 INFO] --- +[2025-06-12 20:23:11,890 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK +[2025-06-12 20:23:12,857 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` | | Organization | Availability | Division | SystemType | SystemName | Platform | Model | MlperfModel | Scenario | Result | Accuracy | number_of_nodes | host_processor_model_name | host_processors_per_node | host_processor_core_count | accelerator_model_name | accelerators_per_node | Location | framework | operating_system | notes | compliance | errors | version | inferred | has_power | Units | weight_data_types | |---:|:---------------|:---------------|:-----------|:-------------|:---------------------|:----------------------------------------------------------------------|:--------------------|:--------------------|:-----------|-----------:|:----------------------------------------------------------------------------------------------------------------------|------------------:|:--------------------------------|---------------------------:|----------------------------:|:-------------------------|------------------------:|:---------------------------------------------------------------------------------------------------------------|:--------------------|:-------------------------------------------------|--------:|-------------:|---------:|:----------|-----------:|:------------|:----------|:--------------------| -| 0 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 1 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 2 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 4 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 23.3198 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 5 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 6 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1314 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 7 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73051 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 8 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 9 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 10 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 11 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 12 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 18.9759 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 13 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 14 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 15 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 16 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | -| 17 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 20 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.93654 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 21 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 22 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 23 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 24 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422055 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 26 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0042 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 27 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 28 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 29 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 30 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 31 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file +| 0 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 9.93654 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 1 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.290026 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 2 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 15.6951 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 3 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 6.28304 | F1: 80.0 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 4 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.353561 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 5 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 17.8765 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 6 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | retinanet | retinanet | Offline | 0.436959 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 7 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config | resnet50 | resnet | Offline | 22.8483 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline | onnxruntime v1.20.1 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 8 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.413749 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 9 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | retinanet | retinanet | Offline | 0.431769 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 10 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config | resnet50 | resnet | Offline | 23.0274 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline | onnxruntime v1.21.1 | Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 11 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config | resnet50 | resnet | Offline | 21.0042 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline | tf v2.19.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 12 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | mixtral-8x7b | mixtral-8x7b | Offline | 0.0148696 | ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 13 | MLCommons | available | open | datacenter | gh_action | gh_action-reference-cpu-pytorch_v2.6.0-default_config | llama2-70b-99 | llama2-70b-99 | Offline | 0.412378 | ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | nan | 0 | open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 14 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config | resnet50 | resnet | Offline | 20.9828 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline | tf v2.18.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 15 | MLCommons | available | open | datacenter | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config | rgat | rgat | Offline | 10.1918 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline | pytorch v2.4.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 16 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config | retinanet | retinanet | Offline | 0.438894 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline | pytorch v2.7.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 17 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 18.9759 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 18 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.437938 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 19 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 23.2491 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 20 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config | resnet50 | resnet | Offline | 23.1314 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline | onnxruntime v1.22.0 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 21 | MLCommons | available | open | edge | default | default-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.422055 | mAP: 49.593 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 22 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73051 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8372 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 27 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 29 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | gptj-99 | gptj-99 | Offline | 48.6205 | ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Tokens/s | fp32 | +| 30 | MLCommons | available | open | edge | gh_action | gh_action-reference-gpu-pytorch_v2.6.0-cu124 | stable-diffusion-xl | stable-diffusion-xl | Offline | 0.342746 | CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812 | 1 | Intel(R) Xeon(R) w7-2495X | 1 | 24 | NVIDIA GeForce RTX 4090 | 2 | open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline | pytorch v2.6.0 | Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 31 | MLCommons | available | open | edge | gh_ubuntu-latest_x86 | gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.405224 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | \ No newline at end of file diff --git a/dbversion b/dbversion index 02b327e1f..c52d44efc 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1341 +1342 diff --git a/summary.csv b/summary.csv index a10ce40d1..c32bc8cb5 100644 --- a/summary.csv +++ b/summary.csv @@ -1,33 +1,33 @@ Organization,Availability,Division,SystemType,SystemName,Platform,Model,MlperfModel,Scenario,Result,Accuracy,number_of_nodes,host_processor_model_name,host_processors_per_node,host_processor_core_count,accelerator_model_name,accelerators_per_node,Location,framework,operating_system,notes,compliance,errors,version,inferred,has_power,Units,weight_data_types +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.93654,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,6.28304,F1: 80.0,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0","Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.436959,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,23.3198,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1314,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73051,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,22.8483,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0042,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,mixtral-8x7b,mixtral-8x7b,Offline,0.0148696,ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/mixtral-8x7b/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" MLCommons,available,open,"datacenter","gh_action",gh_action-reference-cpu-pytorch_v2.6.0-default_config,llama2-70b-99,llama2-70b-99,Offline,0.412378,ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0,1,"Intel(R) Xeon(R) w7-2495X",1,24,"N/A","0",open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,15.6951,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.290026,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,18.9759,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"datacenter","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config,rgat,rgat,Offline,10.1918,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config/rgat/offline,"pytorch v2.4.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.413749,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,18.9759,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.437938,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,9.93654,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,resnet50,resnet,Offline,23.0274,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/resnet50/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config,retinanet,retinanet,Offline,0.431769,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config/retinanet/offline,"onnxruntime v1.21.1","Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,23.2491,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config,resnet50,resnet,Offline,23.1314,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config/resnet50/offline,"onnxruntime v1.22.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.422055,mAP: 49.593,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/default-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config,resnet50,resnet,Offline,20.9828,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config/resnet50/offline,"tf v2.18.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config,resnet50,resnet,Offline,21.0042,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config/resnet50/offline,"tf v2.19.0","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config,retinanet,retinanet,Offline,0.438894,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config/retinanet/offline,"pytorch v2.7.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73051,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8372,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,resnet50,resnet,Offline,17.8765,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/resnet50/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config,retinanet,retinanet,Offline,0.353561,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config/retinanet/offline,"onnxruntime v1.20.1"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,gptj-99,gptj-99,Offline,48.6205,ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Tokens/s,"fp32" +MLCommons,available,open,"edge","gh_action",gh_action-reference-gpu-pytorch_v2.6.0-cu124,stable-diffusion-xl,stable-diffusion-xl,Offline,0.342746,CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812,1,"Intel(R) Xeon(R) w7-2495X",1,24,"NVIDIA GeForce RTX 4090","2",open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/stable-diffusion-xl/offline,"pytorch v2.6.0","Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest_x86",gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.405224,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0","Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index 984fd6cd9..fb1533764 100644 --- a/summary.json +++ b/summary.json @@ -4,22 +4,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":6.28304, - "Accuracy":"F1: 80.0", + "Result":9.93654, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -34,22 +34,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.405224, + "Result":0.290026, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -64,22 +64,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8483, + "Result":15.6951, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -95,21 +95,21 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.436959, - "Accuracy":"mAP: 76.951", + "Result":6.28304, + "Accuracy":"F1: 80.0", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1014-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -124,22 +124,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.3198, - "Accuracy":"acc: 76.000", + "Result":0.353561, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -154,22 +154,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.413064, - "Accuracy":"mAP: 76.951", + "Result":17.8765, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", - "framework":"onnxruntime", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -184,22 +184,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.1314, - "Accuracy":"acc: 76.000", + "Result":0.436959, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -214,22 +214,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest", - "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":1.73051, - "Accuracy":"acc: 80.000", + "Result":22.8483, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", - "framework":"tvm-onnx v1.19.2", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.20.1", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -243,60 +243,60 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"mixtral-8x7b", - "MlperfModel":"mixtral-8x7b", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.0148696, - "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", + "Result":0.413749, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "9":{ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"datacenter", - "SystemName":"gh_action", - "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", - "Model":"llama2-70b-99", - "MlperfModel":"llama2-70b-99", + "SystemType":"edge", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.412378, - "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", + "Result":0.431769, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.1", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "10":{ @@ -304,22 +304,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":15.6951, + "Result":23.0274, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -334,22 +334,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.290026, - "Accuracy":"mAP: 76.951", + "Result":21.0042, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", + "framework":"tf v2.19.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -363,30 +363,30 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemType":"datacenter", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"mixtral-8x7b", + "MlperfModel":"mixtral-8x7b", "Scenario":"Offline", - "Result":18.9759, - "Accuracy":"acc: 76.000", + "Result":0.0148696, + "Accuracy":"ROUGE1: 31.4286 ROUGE2: 5.8824 ROUGEL: 28.5714 TOKENS_PER_SAMPLE: 152.0 gsm8k_accuracy: 100.0 mbxp_accuracy: 0.0", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.22.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/mixtral-8x7b\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "13":{ @@ -394,29 +394,29 @@ "Availability":"available", "Division":"open", "SystemType":"datacenter", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", - "Model":"rgat", - "MlperfModel":"rgat", + "SystemName":"gh_action", + "Platform":"gh_action-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"llama2-70b-99", + "MlperfModel":"llama2-70b-99", "Scenario":"Offline", - "Result":10.1918, - "Accuracy":"acc: 76.000", + "Result":0.412378, + "Accuracy":"ROUGE1: 61.7021 ROUGE2: 37.9679 ROUGEL: 39.3617 TOKENS_PER_SAMPLE: 610.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":24, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", - "framework":"pytorch v2.4.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_action-reference-cpu-pytorch_v2.6.0-default_config\/llama2-70b-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "14":{ @@ -424,22 +424,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest_x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.413749, - "Accuracy":"mAP: 76.951", + "Result":20.9828, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", + "framework":"tf v2.18.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -453,23 +453,23 @@ "Organization":"MLCommons", "Availability":"available", "Division":"open", - "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"stable-diffusion-xl", - "MlperfModel":"stable-diffusion-xl", + "SystemType":"datacenter", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config", + "Model":"rgat", + "MlperfModel":"rgat", "Scenario":"Offline", - "Result":0.342746, - "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", + "Result":10.1918, + "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":2, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.4.0-default_config\/rgat\/offline", + "framework":"pytorch v2.4.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -484,29 +484,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_action", - "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", - "Model":"gptj-99", - "MlperfModel":"gptj-99", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":48.6205, - "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", + "Result":0.438894, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":24, - "accelerator_model_name":"NVIDIA GeForce RTX 4090", - "accelerators_per_node":2, - "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", - "framework":"pytorch v2.6.0", - "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", + "host_processor_core_count":3, + "accelerator_model_name":null, + "accelerators_per_node":0, + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.7.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Tokens\/s", + "Units":"Samples\/s", "weight_data_types":"fp32" }, "17":{ @@ -514,21 +514,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", - "Model":"bert-99", - "MlperfModel":"bert-99", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":4.48923, - "Accuracy":"F1: 80.0", + "Result":18.9759, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", - "framework":"deepsparse v1.8.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -546,18 +546,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":23.2491, - "Accuracy":"acc: 76.000", + "Result":0.437938, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -576,18 +576,18 @@ "SystemType":"edge", "SystemName":"gh_ubuntu-latest_x86", "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.437938, - "Accuracy":"mAP: 76.951", + "Result":23.2491, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", "framework":"onnxruntime v1.21.0", "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, @@ -604,22 +604,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":9.93654, + "Result":23.1314, "Accuracy":"acc: 76.000", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", - "framework":"onnxruntime", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.22.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.22.0", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -634,22 +634,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "SystemName":"default", + "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.457477, - "Accuracy":"mAP: 76.951", + "Result":0.422055, + "Accuracy":"mAP: 49.593", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.6.0", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -664,22 +664,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", + "SystemName":"gh_ubuntu-latest", + "Platform":"gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":23.0274, - "Accuracy":"acc: 76.000", + "Result":1.73051, + "Accuracy":"acc: 80.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config\/resnet50\/offline", + "framework":"tvm-onnx v1.19.2", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -694,22 +694,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config", + "Model":"bert-99", + "MlperfModel":"bert-99", "Scenario":"Offline", - "Result":0.431769, - "Accuracy":"mAP: 76.951", + "Result":4.48923, + "Accuracy":"F1: 80.0", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-onnxruntime_v1.21.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.1", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1013-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config\/bert-99\/offline", + "framework":"deepsparse v1.8.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -724,20 +724,20 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"default", - "Platform":"default-mlcommons_cpp-cpu-onnxruntime-default_config", + "SystemName":"gh_ubuntu-latest x86", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.422055, - "Accuracy":"mAP: 49.593", + "Result":0.413064, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/default-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/retinanet\/offline", "framework":"onnxruntime", "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, @@ -755,11 +755,11 @@ "Division":"open", "SystemType":"edge", "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config", + "Platform":"gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config", "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":20.9828, + "Result":22.8372, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", @@ -767,9 +767,9 @@ "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.18.0-default_config\/resnet50\/offline", - "framework":"tf v2.18.0", - "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config\/resnet50\/offline", + "framework":"onnxruntime", + "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, @@ -784,22 +784,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_ubuntu-latest x86", - "Platform":"gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":21.0042, - "Accuracy":"acc: 76.000", + "Result":0.384271, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", + "host_processor_model_name":"undefined", "host_processors_per_node":1, - "host_processor_core_count":2, + "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-tf_v2.19.0-default_config\/resnet50\/offline", - "framework":"tf v2.19.0", - "operating_system":"Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", + "framework":"onnxruntime v1.21.0", + "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, "errors":0, @@ -814,21 +814,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest_x86", - "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_macos-latest x86", + "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", + "Model":"resnet50", + "MlperfModel":"resnet", "Scenario":"Offline", - "Result":0.438894, - "Accuracy":"mAP: 76.951", + "Result":19.7056, + "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.7.0-default_config\/retinanet\/offline", - "framework":"pytorch v2.7.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", + "framework":"onnxruntime v1.21.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -844,21 +844,21 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_macos-latest_x86", + "Platform":"gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", + "Model":"retinanet", + "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":19.7056, - "Accuracy":"acc: 76.000", + "Result":0.457477, + "Accuracy":"mAP: 76.951", "number_of_nodes":1, "host_processor_model_name":"undefined", "host_processors_per_node":1, "host_processor_core_count":3, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.21.0", + "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", "operating_system":" (darwin-23.6.0)", "notes":null, "compliance":1, @@ -874,29 +874,29 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config", - "Model":"retinanet", - "MlperfModel":"retinanet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"gptj-99", + "MlperfModel":"gptj-99", "Scenario":"Offline", - "Result":0.384271, - "Accuracy":"mAP: 76.951", + "Result":48.6205, + "Accuracy":"ROUGE1: 32.2581 ROUGE2: 6.6667 ROUGEL: 22.5806 GEN_LEN: 264", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.21.0", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/gptj-99\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, "version":"v5.0", "inferred":0, "has_power":false, - "Units":"Samples\/s", + "Units":"Tokens\/s", "weight_data_types":"fp32" }, "30":{ @@ -904,22 +904,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", - "Model":"resnet50", - "MlperfModel":"resnet", + "SystemName":"gh_action", + "Platform":"gh_action-reference-gpu-pytorch_v2.6.0-cu124", + "Model":"stable-diffusion-xl", + "MlperfModel":"stable-diffusion-xl", "Scenario":"Offline", - "Result":17.8765, - "Accuracy":"acc: 76.000", + "Result":0.342746, + "Accuracy":"CLIP_SCORE: 16.368897303938866 FID_SCORE: 237.82578527053812", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"Intel(R) Xeon(R) w7-2495X", "host_processors_per_node":1, - "host_processor_core_count":3, - "accelerator_model_name":null, - "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/resnet50\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "host_processor_core_count":24, + "accelerator_model_name":"NVIDIA GeForce RTX 4090", + "accelerators_per_node":2, + "Location":"open\/MLCommons\/results\/gh_action-reference-gpu-pytorch_v2.6.0-cu124\/stable-diffusion-xl\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 22.04 (linux-6.8.0-52-generic-glibc2.35)", "notes":null, "compliance":1, "errors":0, @@ -934,22 +934,22 @@ "Availability":"available", "Division":"open", "SystemType":"edge", - "SystemName":"gh_macos-latest x86", - "Platform":"gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config", + "SystemName":"gh_ubuntu-latest_x86", + "Platform":"gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config", "Model":"retinanet", "MlperfModel":"retinanet", "Scenario":"Offline", - "Result":0.353561, + "Result":0.405224, "Accuracy":"mAP: 76.951", "number_of_nodes":1, - "host_processor_model_name":"undefined", + "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", "host_processors_per_node":1, - "host_processor_core_count":3, + "host_processor_core_count":2, "accelerator_model_name":null, "accelerators_per_node":0, - "Location":"open\/MLCommons\/results\/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.20.1-default_config\/retinanet\/offline", - "framework":"onnxruntime v1.20.1", - "operating_system":" (darwin-23.6.0)", + "Location":"open\/MLCommons\/results\/gh_ubuntu-latest_x86-reference-cpu-pytorch_v2.6.0-default_config\/retinanet\/offline", + "framework":"pytorch v2.6.0", + "operating_system":"Ubuntu 24.04 (linux-6.8.0-1021-azure-glibc2.39)", "notes":null, "compliance":1, "errors":0, diff --git a/summary.xlsx b/summary.xlsx index 2c7105dca152cbf5fe0c381839937440b9707292..b73594909eec63e2f50a4fc62536d381e3ab650b 100644 GIT binary patch delta 3694 zcmYLMc{mi@+n(Vy82hMXix^{RsHO?ohS!jZ$yTytDMqFSDP;*`i;=9uU?^nEN3vyK zDig-`B3VQ4*thI7*57#l`OY8by3RS*x$o!P&$*v-o!Z-Vw@aUJ@mp~jPdwuT06<;< zz*zuUNKuSTAnnGKv3L(>dx)ZYTnnx3HcSG7u^$gxq zV?KtA5BPZixkzo!I9Q#xL>xPNiMQcpmz?UzuP4q<~wW``@M7e`;xEs({$IA?r!5fx^LND^<`1qkcFB!d17uVf1D5?#@9dA zFx9%^c8RpfGfvn_iAJU!G!dIH?i$u9N}iTQ@J}E3?K;Y|R%UnGQoI@p@gJ|-cdr>- zTzMnWO*V~TJ`Dd>oHBW3(QIkCd~0)Xa$`2YV+GqX6R0n1cRgtP_s!*;9s7xOn3tUK z1U+K6{9pa->^I#HG!?t$^X~Bv{+3X={BO;DDwMHeIFUeV7WXri`QkyGW{(OVYwvmD z-gH-7!|Z--s^DgWKXUzdA8r4-u#`#En`DiEJpO@4Wfmsk_w(Xl6WUbYIBvfJ>fW$T zq4Ax)u5bIdv?4Oeq?$=n4Ek=V2vjEcFd3aO-fj#L(rT0u`ttn|L?|vx4JxA&7>TAG z`a}pRmw2FM7>jRNn}~Jx5TQG}%2A}E7kaF@BSV>U#Ls>Qz&Sn6l7P?mXd)szr(~+Cc{bY8Ji#{5rkf5GoU=L4x(PP*D z@l0BDWSW4f!|^d=sB<2>yWIA}%Prweq=(_c%7$}MrpYLg?6tEB)m(CM>}D|6{E?Ea zo6EE_js@#7jSg_sO~dGewThair*>AFZ`#%uQ-AcBTCQx)dt0L7L`KfL_Dy8C@>gj) z?mnmpJ0keSn-pZub>gX#1^lXoOF;<9SPv>njnivXJVvBDGj>$J{NAxuTVlEMqYSM3 zc5|Yi{RiS)-eBZwO3tU5x(vH9h%l9(^lSA#UiL!*gP>)zxX&LfOgJKc#ZI>7nVj%D4CmJpBEjZ__^eYB%i^Z<> z^xj`)P@yQ9D5Fyt^t-cX&7d<71ARZvPN8p~gJlNUUxlC(21wZb6VA>$t|M)Q!LlT< zLJ-&UDuc1UCd(%heAlASmn$gv$0q1%wcotV22UipB%quA&48T2Mp~O)m-+_N*!tpQvIELS;`NK7b2^+8)jH{eWes3KhX@X? z7AvMKfvYP z>qWp1xR;4wzP?Q(7k;1+{n@C`ipx4? zYtvFk8+gaT0UxJFav7_CYg_(lja|YZFALW|uwBwrD7#xCZ&s9OXufJT#$Ol7r|n&( zrjw;;C!+z*QtYhD<#lkv*^g1smdBjE#m+Ie9iEPx5cZe*KiPj=*6N%@1m>^3F&auC z+=s0}(^K_SF`!|ns1uRg@J}>h6sC$1XwT0+(TGzOXNf&oE1TFm{LryW$><)fafP5+gw9pparAH&k&pDarpB1txJ3;sSj=igma8a-xsz*9rS609odCJ? zJFnda&E+Tml&9ZNSD9yhw;4cY`*N@6V)Zxs;#x=(+{(? z0aV!GbV7^}Of^}B6@?Lk zLT}$QCTb^tDlX~ZCR&N`N&nbzFIs*n_2EVqnoJ{u1nJ@Z02$Hn@=zcR0g$`$s4~3Y z1yE)VxIqTw>H$J2q??vED$E9PI4&Fq&}aw_Sh>XXNv-$U8D`F-E{T^gzFW`=syq^;c(IgfQVA z@Yn*r`{SUmem0IguI4ZzI;iq5gb!j$JyCU&rD`X5mUWX-pi{S?Ji8V;M zMYT)Mm+9N2B!#r~)Pdschn5eeDU)|yKH{Qoqo3Ok1T|3j2}Q4s3&M@Ju4@$@#uH5x zK1n6&sa$YF!P)O7@;=6w2U&A|&cp2WNgwJOqd|Ii&UDlj^O}~?ctb4--TnVK-whJB zsiGKIY0%Gn$UJ~9{Uz6Y|J^tQVSa8scS|3UofKpQq8Cdamh#495W4wv<~>P38f@lz z*ao|@kZ5>3cZYytOBj5=UgvTAhr2dRrjE<+!p*07(LJU|p~WJU#utT+5F|ZwI-y^H z9&WLBUKCvsy48XRKJhkeF7g?`jhZCI*C9}eb`eD4vQNsycc_CPfqL}<$}xJDd~3w^ zj1B%fT!UN!xj__3&I}mZYWX_(VFXt}WA?el{pn|(7sat|eRzB~xSyqCy2(h$kSn%d zK%Ck02sbQLHMAXG?cFibwoZFG@w# zxo*zwm)3qfilZFGZ8Rnpq(6^3cFr+k7_&^1B*!5QVNo(kNjMPUwy-| zf zg=et}?Wt?9+Z8QB3@?4xq zOi-eI$+MZNXNMy`#FWLgmOW?2ImU=D-TP9Xq&Cv$@7Xs-_2YvsIKEx$$I@Ez0PjWOLwYO)p4$VcUhF1Hf&E5+lWl2_+xtGiIpy^FS9TpSVRs(D zYox!(3*~ChR2kL_!oLk~N-E4&%=eoAJX>H*${V{+?EIrP#s>3i`V3|hhF=N*eG`nVW*D3LvAXn;sUILjpM(6A$d>1 z4k$pjQq%&bk;#gBzzFgWMGfE|vY--7{6A5F0Du`A03ZbT{}voFLN-^@0m8}gO0u8{ I`ID9Z0fRaML;wH) delta 3680 zcmZ{nc{J2r{KsbuQmJ8zvQHX2$45fNYqs%c1}42(ypeM_+gG#4szdh&$+vm z$daN*N6KnJ}7*Oe{36=U1#0NM0FzZ;~SE<#q5WWz^u z6I)?uEV-wDa&eW?=bSDm*LHL@QX|Lmu}Wl4fFWKZ(rujcE!m9QMSEw~ga5+jSGe#F z@AR@FV@p-|`IUjG8jZ=Ho82!oex_Zy* zrqb?&Sn-g^ft%YUJC&2pPB9e14(irD=e{~G~Y)=>xI^r|A3We4YIm6;GRV$?~6o&w!- ztHMULfbx1%!Vo6E`Qegg9j{C)O44l4e(HNx#P8&|7a#It-zC;~<@7wNHEFfo{VO7T z|1bg7!sf3f$};pA-7*|gi{NhRlYw!o)roVjbNFeCPSIcJp~iq`7kj6cbP%FNznGf} zMgcKyckc;&cva1XKXYz~sr3`7Z6eQwy?%6@5Y!29e`=y&6SZ9r=Pn(8j7Amy7Sl-( z6Xq!Xr8FZk<)j#DP7~06Ie$di7g{nzqOifR2v1CZt7!Fnt=OSse6?>fV{K;pSU#{W zdfclg@R4vB{Xor2sl)Yxh~*)co)!&k0ul_2#IYl9PuO7A;?=Zy3CgZ&RNu0yRX}Pq zeQJ$hJy9)J@UCmx_hqN!g}nTomtHkR>1_^y;powyeVt9Y^FNQAeZHm3oy9M1Z3ZZG zZA|w!sd|4m$!+d6+MJ(*cl%|zj~pz%zljjoar2gy(x7~0^b?O|QU?EZv~~o%+wK3d zuq>igZ~pZ{-9tjyxkmvO?vQH3T6LM5DiopjOAn>CXU$+=)~n9giOH-Dnbmi_{cLo5 zs?9xg`)w6|z8;J350`2X>!aID*Q_~VH|BO~2IbS}f-^l&%&dgQW>DSrSZXcc|0>VY znx?uSpWq{1V%~OFx@32J!UpI)X<^+hs;lrrl0Ez%U(n4qJ$TQfi{SHU)IWNnu-mYB zNp|-0=xS82>3EX|O5tio_;!#D{0`!pO<$lvZcc02m0$oG=`EYX5-Ae=&ZXCy3^Nm|N#ndK+*A8#q0dJ!nD?ad#VTo^RYAc-E&%dSQ? zNZy5|bPBMz>`H!}by((7-lBi!JzrHO_MDOye7J?P><=FmL^jL;BdyMnJ1fA*GdI1E zD(WRtmZGZqlEVo&E3-GF;%}ahdL2J?R-e_gY~z=4A^+N;EK+K}$!V9jOx%%t<9qkV zr4&tmHAINL*>hiW_HS5TXodsb*Blz2OP#p`WNc^}pmf2?O>>uURWA4<_tC<83k+-f zUj>s{K0G2q7G^0Hc|L+7KVKW2N2y6M;tVC_MDSWj8DDi-h%+poZCJTlz3>FKFMD57 zYC2WRphYKMasICaZwEeSa?9asmvXz(d70sU6IJ@J{B_TKN}>o)#jOds-ogi_!Q!WY z%V`OLdg2KKP%50nQ!7FQE2sEF+?+{cA|ntcCy{_PQ&MRTgL)jXfGvglKrtC~tdOfkkJyI2W^pD0AdL;~d3&=l=+jn<;YFrQ(AaTDJlXwE)_?KoSDbixM?NrLjIjNTuAIP;_)U?hK*DUPW`@UiaOb7n$Qhys6?BnDiiOCxLJg~4XUkU%|@yuWm8zj!P;A@j1vA!%0w8HuQ8dPWCx z7bJQqh=4<+%3hwto@2T0LI5#M{hXw_Fa!rN4W4^a_y*J8DPz~s97tpT^n>&U18gxa z{sD7f3F%NV#ueaW`mXc_Id7qXfP8R#bXoQZ)U9?qHfT65$Os>A!VJmY@+Xn}acB6s zDo|LiwV})pv=1{B>MyP#Va-Yr<+WvA~-zQH(b(Mm<7a4vxP= z><`l8$S=$QGu@3=)Xbr>*a(3V#ZTLNPD2u(fJw|3w$=1R+bTq#(G-@f?0># z55B+<@P;kOJqjW2ycU3zjk|BM0XSkTb2ficj(iR6K)?dq&yqenaX@niC{brC)~g1OvW|*Bq46>;dl=W`q#vfJhRjIA6ZM|^edAA*w6y-kFUf(%2ZFlgJIHBEYLyl>hpIi#r;4f!L zro@)X7xYG%ELleTOAxSY(;j!D78iPMA523JMQWQx_Qp*#RJKfFrgiGK_U-FyW+huM zI9o{Q-Y6e83K%C(MCUr#3PLykwvcNWPA*pWb7OcDJBg30yS}y|Xn5-3i%;%uTs={% z8|g2a5!7D|Qdg*R<)?VRpJj_7qQ8g{mJuR=>zAusp{uVV+kdwxaEDu<<3=Ly#Y-oC z;8CGjH##0%9(8FS9c1Nz&-PFI*)_bgs(vJRpKuVoNnbb>FROdf-ZixU^Ql*bNsEGF z>R08=ZI$o8hs5%3Yg6TBzc!JTcpt_6AaKyrJS;O`aS&~m8N%l`FUtH_`s`PVOF3oXqk=}$Z41y|Jhbj9Nos65@T@48eiabNALjcE(J+@jQ9D{qvEG>ASE4 zvM)H*V*&`?gO1a4_@f&a1!27ydj dKp<|=|FxjkgM_=%>R=f{y!2)E?-GAj{s;a~5EuXe diff --git a/summary_results.json b/summary_results.json index 3a074a315..911392c07 100644 --- a/summary_results.json +++ b/summary_results.json @@ -680,7 +680,7 @@ "Code": "https://github.com/mlcommons/mlperf_inference_test_submissions_v5.0/tree/main/open/MLCommons/code", "Unique ID (e.g. for Audit)": "edge/open/MLCommons/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50", "ColorKey": "availableMLCommons", - "Performance_Result": 23.3198, + "Performance_Result": 22.8372, "Performance_Units": "Samples/s" }, { From 217472c6bb53c89c063890bc98b9b2c8a352edd0 Mon Sep 17 00:00:00 2001 From: mlcommons-bot Date: Tue, 17 Jun 2025 12:48:27 +0000 Subject: [PATCH 715/719] Updated results summary [skip ci] --- README.md | 28 ++++++++++++++-------------- dbversion | 2 +- summary.csv | 2 +- summary.json | 2 +- summary.xlsx | Bin 10938 -> 10940 bytes summary_results.json | 2 +- 6 files changed, 18 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 1cb5e4840..ae36cc6f0 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,21 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. ``` -[2025-06-12 20:23:11,890 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 -[2025-06-12 20:23:11,890 submission_checker1.py:3385 INFO] --- -[2025-06-12 20:23:11,890 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 +[2025-06-17 12:48:22,354 submission_checker1.py:3378 INFO] Results=32, NoResults=0, Power Results=0 +[2025-06-17 12:48:22,354 submission_checker1.py:3385 INFO] --- +[2025-06-17 12:48:22,354 submission_checker1.py:3386 INFO] Closed Results=0, Closed Power Results=0 -[2025-06-12 20:23:11,890 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 +[2025-06-17 12:48:22,354 submission_checker1.py:3391 INFO] Open Results=32, Open Power Results=0 -[2025-06-12 20:23:11,890 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 +[2025-06-17 12:48:22,354 submission_checker1.py:3396 INFO] Network Results=0, Network Power Results=0 -[2025-06-12 20:23:11,890 submission_checker1.py:3401 INFO] --- -[2025-06-12 20:23:11,890 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 -[2025-06-12 20:23:11,890 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 -[2025-06-12 20:23:11,890 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 -[2025-06-12 20:23:11,890 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 -[2025-06-12 20:23:11,890 submission_checker1.py:3422 INFO] --- -[2025-06-12 20:23:11,890 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK -[2025-06-12 20:23:12,857 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py +[2025-06-17 12:48:22,354 submission_checker1.py:3401 INFO] --- +[2025-06-17 12:48:22,354 submission_checker1.py:3403 INFO] Systems=23, Power Systems=0 +[2025-06-17 12:48:22,354 submission_checker1.py:3407 INFO] Closed Systems=0, Closed Power Systems=0 +[2025-06-17 12:48:22,354 submission_checker1.py:3412 INFO] Open Systems=23, Open Power Systems=0 +[2025-06-17 12:48:22,354 submission_checker1.py:3417 INFO] Network Systems=0, Network Power Systems=0 +[2025-06-17 12:48:22,354 submission_checker1.py:3422 INFO] --- +[2025-06-17 12:48:22,354 submission_checker1.py:3427 INFO] SUMMARY: submission looks OK +[2025-06-17 12:48:23,634 module.py:5270 INFO] - ! call "postprocess" from /home/runner/MLC/repos/mlcommons@mlperf-automations/script/run-mlperf-inference-submission-checker/customize.py ``` @@ -46,7 +46,7 @@ Please download [summary.xlsx](summary.xlsx) to view the most recent results. | 22 | MLCommons | available | open | edge | gh_ubuntu-latest | gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config | resnet50 | resnet | Offline | 1.73051 | acc: 80.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline | tvm-onnx v1.19.2 | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 23 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config | bert-99 | bert-99 | Offline | 4.48923 | F1: 80.0 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline | deepsparse v1.8.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 24 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | retinanet | retinanet | Offline | 0.413064 | mAP: 76.951 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | -| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.8372 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | +| 25 | MLCommons | available | open | edge | gh_ubuntu-latest x86 | gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config | resnet50 | resnet | Offline | 22.9942 | acc: 76.000 | 1 | AMD EPYC 7763 64-Core Processor | 1 | 2 | nan | 0 | open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline | onnxruntime | Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 26 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | retinanet | retinanet | Offline | 0.384271 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 27 | MLCommons | available | open | edge | gh_macos-latest x86 | gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config | resnet50 | resnet | Offline | 19.7056 | acc: 76.000 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline | onnxruntime v1.21.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | | 28 | MLCommons | available | open | edge | gh_macos-latest_x86 | gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config | retinanet | retinanet | Offline | 0.457477 | mAP: 76.951 | 1 | undefined | 1 | 3 | nan | 0 | open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline | pytorch v2.6.0 | (darwin-23.6.0) | nan | 1 | 0 | v5.0 | 0 | False | Samples/s | fp32 | diff --git a/dbversion b/dbversion index c52d44efc..50bbef7af 100644 --- a/dbversion +++ b/dbversion @@ -1 +1 @@ -1342 +1343 diff --git a/summary.csv b/summary.csv index c32bc8cb5..6eeca91dd 100644 --- a/summary.csv +++ b/summary.csv @@ -24,7 +24,7 @@ MLCommons,available,open,"edge","default",default-mlcommons_cpp-cpu-onnxruntime- MLCommons,available,open,"edge","gh_ubuntu-latest",gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config,resnet50,resnet,Offline,1.73051,acc: 80.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest-reference-cpu-tvm-onnx_v1.19.2-default_config/resnet50/offline,"tvm-onnx v1.19.2","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config,bert-99,bert-99,Offline,4.48923,F1: 80.0,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-deepsparse_v1.8.0-default_config/bert-99/offline,"deepsparse v1.8.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,retinanet,retinanet,Offline,0.413064,mAP: 76.951,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/retinanet/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" -MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.8372,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" +MLCommons,available,open,"edge","gh_ubuntu-latest x86",gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config,resnet50,resnet,Offline,22.9942,acc: 76.000,1,"AMD EPYC 7763 64-Core Processor",1,2,"N/A","0",open/MLCommons/results/gh_ubuntu-latest_x86-mlcommons_cpp-cpu-onnxruntime-default_config/resnet50/offline,"onnxruntime","Ubuntu 24.04 (linux-6.11.0-1015-azure-glibc2.39)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,retinanet,retinanet,Offline,0.384271,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/retinanet/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest x86",gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config,resnet50,resnet,Offline,19.7056,acc: 76.000,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-onnxruntime_v1.21.0-default_config/resnet50/offline,"onnxruntime v1.21.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" MLCommons,available,open,"edge","gh_macos-latest_x86",gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config,retinanet,retinanet,Offline,0.457477,mAP: 76.951,1,"undefined",1,3,"N/A","0",open/MLCommons/results/gh_macos-latest_x86-reference-cpu-pytorch_v2.6.0-default_config/retinanet/offline,"pytorch v2.6.0"," (darwin-23.6.0)","",1,0,v5.0,0,False,Samples/s,"fp32" diff --git a/summary.json b/summary.json index fb1533764..7f3ae5102 100644 --- a/summary.json +++ b/summary.json @@ -759,7 +759,7 @@ "Model":"resnet50", "MlperfModel":"resnet", "Scenario":"Offline", - "Result":22.8372, + "Result":22.9942, "Accuracy":"acc: 76.000", "number_of_nodes":1, "host_processor_model_name":"AMD EPYC 7763 64-Core Processor", diff --git a/summary.xlsx b/summary.xlsx index b73594909eec63e2f50a4fc62536d381e3ab650b..fbd25f7a8a20a5b87e35893df8497145f30ef113 100644 GIT binary patch delta 1400 zcmV-;1&8{&RlHTOatH(~WkyP~b_liz0)~#0E)nH_>?9jFLCLR}Xt#t+lt(-%{g6y3 zr5}<(GAl?wECXbc43KGZepm-Urb+qHp!`5fLxE_SUTUudP4Xvsr1V2Fp_G0|2GOh_{jdz6Niu+@$@yU&0GcM{M}zVM zEfE=i1Wj2e>#{Q3G>I1kn`X_*&)1^)BvrPSMONB0i_VX>TiOZBUX?9Tel)5hzEpll zMp@GNAt6Mwg7d>NfF{WRnkMCkbpU9Zj2{ig541$=vLTva{E#x-HA&aFYu0T1d>=IT zRQb8mMdc^oHT|&gcWZXpEOCA`+9SSnen>`tSyK8TA!M_H^usd1CR29cwONz&!$tu( zP0o)7=LcFMI!&p{LRpuU;l4@CAlNr+mVUk-&Zit{_f-A4)J63t+cz!km%R*BV*Y4U zNPKDjkc_gV`a?p>`>OFWpd*rA=o(Wee!?)E|I;KMON83?V<*hZJf$hB^#$*q5;|H zllW2tA{k{#4~T@2%?ch6%K)1sBcE`8+9nN%bpU9ZG$0!^AZUrmAmFh>RhL!CZIj(0 zux;9VX90a%k-e7nRoOA^q6C!fn3k(`8I>g#kc~cxFD)RFQI^zzNC>H{paHQAP)RaC zrO5(f9RQUk3ut@e1C?L_vC9S?JCt=v8Dd$*0xB$E-?4m>$X?6(Dq#r{kYD zErWw>g+Kpe@3{t)4s#^cll-4r)>i5KTJ(Rk)d~Y$@}jKmlpu*Fi7)*hl2MivfJg|D ztRMif3?RwW8bH!y0I^X3Ns|G8WPk7`*KDD3UR$GlDA`AvKpkH?EfKgllFx^mmP{_ML;b@*ZZF>w#^Y`kPT80CM{ zzBgWfs9idm-ZU0}u0Jl%J3ReeTX|>w;pK81H_|V&__!H=o@Mt?zSw4eWAJ&n{-C_Z zY(7fnqx8#q^!$_hmgC;ZBpLj;n$&VS`g0lo_(T5(@9>Ad_tVb}{Zr$(kuCi`nyqd% zO8y0#*2&SVcN3rY<~O71qA`w#*+Q+B-)zj+AHBEwI!$J)uiR$iW0I!H(cxHpLB z`FGjRyJ3=~_uu4~Wcv{Wh9`dkvkw;74h2L@JKrXg>?9w5PQx$|gzr2>Ig;H5b zDgo+=N=O7zakK1gY7zg)Hl%%f+$2p~sV9!>nf-Ql*Po#J1G6k9$^rx{WkyP~`Y1gC1w>0b-zJkhDpUb3laVST2H+XN&JW7~nj`~gnv@^b0ibCzel!?A&=RrBhG>HEL&|X1BwgdKS+nu;ebC%f z<>yKlm7jdq^uxm6t=VO>#QD)^kNDF0AsJ4h}Y!nHn zse_IU0uZ!BY?|P)Ls^%W$&HhQVB@s+$^ZGgMD|)1S!Lt2iyBb2aVlGtY@Bw924tg8 z;!6#PWRxX6AQD10D|kRG18kCve8Open=~NS0ibEpfNao!pd})MfX5D1T~;NxO?HRC zwrTI31@vu2_FC3gWyiFO5>U2dTCUb*RF+skHu@yKw17xPSyBTcA*8Z`2E;NzCCLDl zCJTsl092YRpzVzhRDuP>E*p64P}U`7h-DQEsIY*2$MQ)cdoAm$ge6Eoe(48)rW7S0 z6+B=`K8Qq0KqR9qX#tTCVp+igVi~}a`89BmHEBR>6yVaN0okAdK}*D@DL?e$9Ll=1 z3=Xmt{``-<=NeEt%#l=2@_%YsTcz`B(f`p_D-3kWi?X&;f+U(GzVv@cMp;q-A|XVw zf&j!afFx6E07;Vp#6|%mO$LyE4F(XjL~NS!LuSsQtV_!P$qELLz2^qd2FdVANbaln zWABJRs%2S@RM^{vk9nKK^PBiI9*-A|ev)0Tb>+5o{n>Yy>hQz*W8xm-*?7rxFv|a? zeQ&(}P`h+Ay=g4|Tz_1icX;}{w(`#U!^`D3Zlqsk@o_W$Jj?E(e6h`c#^CdC{Xuz+ z*?g4DN9mXK==mr0Eyul+Niz6xHL2xv^yf1E@rV8o-r)~@@28&|`lrTmBU}1?G+W(j zl>7@et&^i!?aM z^Y5~scf%w}@4v|}$@U`z3{U<7vkw;74h05eKMW+3>?9w5YQr!PgztR{qq{9zrlAPi zfwtt-K%p)zy$NZPh+02LY8}6Q6+3Yf$f?KG%zit&s9Nq-m;8eB(&!o$S&kCW+z6?+ zHF|GX=>tl9U|O)wXsFQu9xa=@id)K!gI8xPxFEqN5u`okwnn=UEXCOK9Vq6r=+IHz z7^hf>rQ2eE%lJ3j0w#IBzzPBrEHHckrgkQxBU11gX@6Wd0tCmA77ap@!t-KFt?tL0XbiQqqEa9Y$XD zpMmN)-V9I^{37S8%E+)zX1Xc)3-n0BDJply Date: Fri, 4 Jul 2025 15:51:07 +0100 Subject: [PATCH 716/719] Delete open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline directory --- .../gptj-99/offline/accuracy/accuracy.txt | 9 -- .../offline/accuracy/mlperf_log_accuracy.json | 3 - .../offline/accuracy/mlperf_log_detail.txt | 70 --------------- .../offline/accuracy/mlperf_log_summary.txt | 4 - .../run_1/mlperf_log_accuracy.json | 2 - .../performance/run_1/mlperf_log_detail.txt | 88 ------------------- .../performance/run_1/mlperf_log_summary.txt | 52 ----------- 7 files changed, 228 deletions(-) delete mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/accuracy.txt delete mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_accuracy.json delete mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_detail.txt delete mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_summary.txt delete mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_accuracy.json delete mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_detail.txt delete mode 100644 open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_summary.txt diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/accuracy.txt b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/accuracy.txt deleted file mode 100644 index f2f257b68..000000000 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/accuracy.txt +++ /dev/null @@ -1,9 +0,0 @@ -Constructing QSL -Encoding Samples - -Results - -{'rouge1': '32.2581', 'rouge2': '6.6667', 'rougeL': '22.5806', 'rougeLsum': '25.8065', 'gen_len': 264, 'gen_num': 1} -Finished destroying QSL. - -hash=5470ad247d575ae6dd919dbaec13f5d488c15366f13a7dc52cb0e73e062cc958 diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_accuracy.json b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_accuracy.json deleted file mode 100644 index 0679df2f6..000000000 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_accuracy.json +++ /dev/null @@ -1,3 +0,0 @@ -[ -{ "seq_id" : 0, "qsl_idx" : 0, "data" : "20000000000000006F090000000000004403000000000000B10200000000000012020000000000001E010000000000005F02000000000000B3A1000000000000220100000000000054010000000000005F170000000000001C010000000000000101000000000000BD18000000000000AD18000000000000FC02000000000000C600000000000000130D000000000000593C0000000000000101000000000000370C0000000000001E010000000000009502000000000000FC01000000000000A701000000000000F73D000000000000FF01000000000000B3A10000000000001C01000000000000254A000000000000FC02000000000000C600000000000000D001000000000000BD180000000000003E0100000000000054030000000000000D180000000000007B01000000000000720D00000000000052010000000000001320000000000000AE20000000000000090D000000000000FC0200000000000050C4000000000000" } -] diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_detail.txt b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_detail.txt deleted file mode 100644 index b8f994113..000000000 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_detail.txt +++ /dev/null @@ -1,70 +0,0 @@ -:::MLLOG {"key": "loadgen_version", "value": "5.0.15 @ aea55edb04", "time_ms": 0.009911, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2025-02-02T19:42:27.123680", "time_ms": 0.009911, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2025-02-02T19:42:27.123687", "time_ms": 0.009911, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2025-02-01T00:10:56+00:00", "time_ms": 0.009911, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "aea55edb04b10928b75d1cc318d6bed2f2e4a629 Create benchmark checklist for pointpainting (#2068)\nd67b62657a63c7d64bacf92d5ec3328c03ef0d90 Increment version to 5.0.15\nd7bbf881ca0b69c71cbbee18fa26cf13c7a67be8 Update results.cc | Add another significant digit to percentile latency display (#2066)\n10d123c982be145de8e818309440486ee1706439 Fix mlperf.conf link for equal issue mode (#2069)\n9b32a74c53fcfa6a7f6b529dec6bba534e10cc29 Changes for final report generation - PointPainting (#2063)", "time_ms": 0.009911, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.009911, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"a8ebd64f62d0349aeedbe3295d833ebdce625c2e","/MANIFEST.in":"ddeb472d62edf2920db1f8fa3beebe3e831557f1","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/VERSION.txt":"106b7e43a7c4da84939d61238eba5c8639ddf3c5","/__init__.py":"d013101621ef06a0ddc5e7d9ce511918a8b2ebe6","/bindings/c_api.cc":"14d178b64c7fc45d090e038c08d9b78ca943c383","/bindings/c_api.h":"23d9f99e00b2d196e095fae0bb453a391c18d601","/bindings/python_api.cc":"4dae966c92acdaa373b04a95adc4ca353937f154","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"02fcfe6d9cf958eeb4b6f1f4dbe87ba7eb4d7dec","/issue_query_controller.h":"ed20934fd3507a15949d501ac154be38e766f6ab","/loadgen.cc":"6daa9cd51454a699fcb55d9aa6bf9e54dd7b7a97","/loadgen.h":"ce9fcb5d44951e7e9048a83b7c1a41c8b8e0f7d8","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"49e63158ebca654fa4b7c5f3321054cf4d6c3a30","/logging.h":"5f6cc9ed4c4859641e05b1bb6a0ef860c852fcd8","/mlperf.conf":"80d879f8a0e3ff89863101be32a710fa476e64f8","/mlperf_conf.h":"fca712ecaa27e093c1f76dc4beb85741d88eaeb2","/pyproject.toml":"712fab87b72ba67ef2a068d0f9f47da65130342f","/query_dispatch_library.h":"1f18e9cd3ee4dc89a387cf462de1d0ceb1ece975","/query_sample.h":"c4f399103bc3d172079bbd4cd2b0ca0f22eebc4f","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"7c4ef762b985ee0504bbcd482f5985123f7c5a1e","/results.h":"fce22d5a588d91fd968a6b25c27896dba87bc276","/setup.py":"a5eaa6f713bd3dfb6603be2c7928f0c295d7ee30","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"8e05582d1fbe9dd2b809686684c3a0ac41248723","/test_settings_internal.cc":"a5cc85fb7735727eee032aa3e88b5d61c1f11a2a","/test_settings_internal.h":"2bb9e9ae53904cb0ca221f4a5d49ca7d9ec3b0ca","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"9f23d13276194588473120a8a6ecf5a6ed034a23"}, "time_ms": 0.009911, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "test_datetime", "value": "2025-04-02T08:55:12Z", "time_ms": 0.015582, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1194, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.015582, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1195, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 280, "time_ms": 0.015582, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1196, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.015582, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1197, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 1, "time_ms": 0.015582, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 1, "time_ms": 0.015582, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 272, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 273, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_offline_expected_qps", "value": 1, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 310, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 0, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_min_query_count", "value": 1, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_max_query_count", "value": 1, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 6023615788873153749, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 320, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 15036839855038426416, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 321, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 9933818062894767841, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 323, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 324, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 326, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 328, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 330, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 331, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 333, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 335, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 13368, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 337, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.021205, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 339, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 419, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_target_qps", "value": 1, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 423, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 424, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 427, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 0, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 431, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_max_query_count", "value": 1, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_min_sample_count", "value": 1, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 6023615788873153749, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 434, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 15036839855038426416, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 435, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 9933818062894767841, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 438, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 440, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 442, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 445, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 447, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 449, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 13368, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 451, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.021607, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 453, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.027710, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1085, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "loaded_qsl_set", "value": [0], "time_ms": 0.030975, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 608, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 0.054614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 427, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 0.054614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "generated_query_duration", "value": 1000000000, "time_ms": 0.054614, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 1238.574308, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 1238.574308, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 1238.574308, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 1238.574308, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 1238.574308, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 910, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 1238.574308, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 912, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "power_begin", "value": "04-02-2025 08:55:12.466", "time_ms": 1238.575252, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 562, "pid": 12432, "tid": 12432}} -:::MLLOG {"key": "power_end", "value": "04-02-2025 08:55:13.703", "time_ms": 1238.575252, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 12432, "tid": 12432}} diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_summary.txt b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_summary.txt deleted file mode 100644 index a05d86571..000000000 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy/mlperf_log_summary.txt +++ /dev/null @@ -1,4 +0,0 @@ - -No warnings encountered during test. - -No errors encountered during test. diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_accuracy.json b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_accuracy.json deleted file mode 100644 index 0d4f101c7..000000000 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_accuracy.json +++ /dev/null @@ -1,2 +0,0 @@ -[ -] diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_detail.txt b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_detail.txt deleted file mode 100644 index 89597bb65..000000000 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_detail.txt +++ /dev/null @@ -1,88 +0,0 @@ -:::MLLOG {"key": "loadgen_version", "value": "5.0.15 @ aea55edb04", "time_ms": 0.010107, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2025-02-02T19:42:27.123680", "time_ms": 0.010107, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2025-02-02T19:42:27.123687", "time_ms": 0.010107, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2025-02-01T00:10:56+00:00", "time_ms": 0.010107, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "aea55edb04b10928b75d1cc318d6bed2f2e4a629 Create benchmark checklist for pointpainting (#2068)\nd67b62657a63c7d64bacf92d5ec3328c03ef0d90 Increment version to 5.0.15\nd7bbf881ca0b69c71cbbee18fa26cf13c7a67be8 Update results.cc | Add another significant digit to percentile latency display (#2066)\n10d123c982be145de8e818309440486ee1706439 Fix mlperf.conf link for equal issue mode (#2069)\n9b32a74c53fcfa6a7f6b529dec6bba534e10cc29 Changes for final report generation - PointPainting (#2063)", "time_ms": 0.010107, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.010107, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"a8ebd64f62d0349aeedbe3295d833ebdce625c2e","/MANIFEST.in":"ddeb472d62edf2920db1f8fa3beebe3e831557f1","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/VERSION.txt":"106b7e43a7c4da84939d61238eba5c8639ddf3c5","/__init__.py":"d013101621ef06a0ddc5e7d9ce511918a8b2ebe6","/bindings/c_api.cc":"14d178b64c7fc45d090e038c08d9b78ca943c383","/bindings/c_api.h":"23d9f99e00b2d196e095fae0bb453a391c18d601","/bindings/python_api.cc":"4dae966c92acdaa373b04a95adc4ca353937f154","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"02fcfe6d9cf958eeb4b6f1f4dbe87ba7eb4d7dec","/issue_query_controller.h":"ed20934fd3507a15949d501ac154be38e766f6ab","/loadgen.cc":"6daa9cd51454a699fcb55d9aa6bf9e54dd7b7a97","/loadgen.h":"ce9fcb5d44951e7e9048a83b7c1a41c8b8e0f7d8","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"49e63158ebca654fa4b7c5f3321054cf4d6c3a30","/logging.h":"5f6cc9ed4c4859641e05b1bb6a0ef860c852fcd8","/mlperf.conf":"80d879f8a0e3ff89863101be32a710fa476e64f8","/mlperf_conf.h":"fca712ecaa27e093c1f76dc4beb85741d88eaeb2","/pyproject.toml":"712fab87b72ba67ef2a068d0f9f47da65130342f","/query_dispatch_library.h":"1f18e9cd3ee4dc89a387cf462de1d0ceb1ece975","/query_sample.h":"c4f399103bc3d172079bbd4cd2b0ca0f22eebc4f","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"7c4ef762b985ee0504bbcd482f5985123f7c5a1e","/results.h":"fce22d5a588d91fd968a6b25c27896dba87bc276","/setup.py":"a5eaa6f713bd3dfb6603be2c7928f0c295d7ee30","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"8e05582d1fbe9dd2b809686684c3a0ac41248723","/test_settings_internal.cc":"a5cc85fb7735727eee032aa3e88b5d61c1f11a2a","/test_settings_internal.h":"2bb9e9ae53904cb0ca221f4a5d49ca7d9ec3b0ca","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"9f23d13276194588473120a8a6ecf5a6ed034a23"}, "time_ms": 0.010107, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "test_datetime", "value": "2025-04-02T08:53:04Z", "time_ms": 0.032164, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1194, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.032164, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1195, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 683, "time_ms": 0.032164, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1196, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.032164, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1197, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 1, "time_ms": 0.032164, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 1, "time_ms": 0.032164, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 272, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 273, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_offline_expected_qps", "value": 1, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 310, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 0, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_min_query_count", "value": 1, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_max_query_count", "value": 1, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 6023615788873153749, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 320, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 15036839855038426416, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 321, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 9933818062894767841, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 323, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 324, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 326, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 328, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 330, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 331, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 333, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 335, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 13368, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 337, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.073114, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 339, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 419, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_target_qps", "value": 1, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 423, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 424, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 427, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 0, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 431, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_max_query_count", "value": 1, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_min_sample_count", "value": 1, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 6023615788873153749, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 434, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 15036839855038426416, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 435, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 9933818062894767841, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 438, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 440, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 442, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 445, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 447, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 449, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 13368, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 451, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.073709, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 453, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.087676, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 836, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "loaded_qsl_set", "value": [0], "time_ms": 0.099029, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 608, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 0.423955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 427, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 0.423955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "generated_query_duration", "value": 1000000000, "time_ms": 0.423955, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 1419.845293, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "issue_query_controller.cc", "line_no": 482, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 1421.803750, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 1421.803750, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 1421.803750, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 1421.803750, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 1421.803750, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 910, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 1421.803750, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 912, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "power_begin", "value": "04-02-2025 08:53:04.632", "time_ms": 1421.804172, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 562, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "power_end", "value": "04-02-2025 08:53:06.051", "time_ms": 1421.804172, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 674, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 679, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 680, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 681, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "early_stopping_result", "value": "", "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 700, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 712, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_samples_per_second", "value": 0.704645, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 768, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_min_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 774, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_max_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 775, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_mean_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 776, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 1419154830, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 4149, "tid": 4149}} -:::MLLOG {"key": "result_inferred_tokens_per_second", "value": 48.6205, "time_ms": 1421.810038, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 840, "pid": 4149, "tid": 4149}} diff --git a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_summary.txt b/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_summary.txt deleted file mode 100644 index d8e5f3eb7..000000000 --- a/open/MLCommons/results/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance/run_1/mlperf_log_summary.txt +++ /dev/null @@ -1,52 +0,0 @@ -================================================ -MLPerf Results Summary -================================================ -SUT name : PySUT -Scenario : Offline -Mode : PerformanceOnly -Samples per second: 0.704645 -Tokens per second (inferred): 48.6205 -Result is : VALID - Min duration satisfied : Yes - Min queries satisfied : Yes - Early stopping satisfied: Yes - -================================================ -Additional Stats -================================================ -Min latency (ns) : 1419154830 -Max latency (ns) : 1419154830 -Mean latency (ns) : 1419154830 -50.00 percentile latency (ns) : 1419154830 -90.00 percentile latency (ns) : 1419154830 -95.00 percentile latency (ns) : 1419154830 -97.00 percentile latency (ns) : 1419154830 -99.00 percentile latency (ns) : 1419154830 -99.90 percentile latency (ns) : 1419154830 - -================================================ -Test Parameters Used -================================================ -samples_per_query : 1 -target_qps : 1 -target_latency (ns): 0 -max_async_queries : 1 -min_duration (ms): 0 -max_duration (ms): 0 -min_query_count : 1 -max_query_count : 1 -qsl_rng_seed : 6023615788873153749 -sample_index_rng_seed : 15036839855038426416 -schedule_rng_seed : 9933818062894767841 -accuracy_log_rng_seed : 0 -accuracy_log_probability : 0 -accuracy_log_sampling_target : 0 -print_timestamps : 0 -performance_issue_unique : 0 -performance_issue_same : 0 -performance_issue_same_index : 0 -performance_sample_count : 13368 - -No warnings encountered during test. - -No errors encountered during test. From 0cf47355fe96e117cf95940f7d2b7270701d9752 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 4 Jul 2025 15:51:59 +0100 Subject: [PATCH 717/719] Delete open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99 directory --- .../gptj-99/offline/README.md | 46 - .../gptj-99/offline/accuracy_console.out | 24 - .../gptj-99/offline/cpu_info.json | 27 - ...on-reference-gpu-pytorch_v2.6.0-cu124.json | 7 - .../gptj-99/offline/mlc-deps.mmd | 64 - .../gptj-99/offline/mlc-deps.png | Bin 4222965 -> 0 bytes .../gptj-99/offline/mlc-version-info.json | 634 ---- .../gptj-99/offline/model-info.json | 7 - .../gptj-99/offline/os_info.json | 30 - .../gptj-99/offline/performance_console.out | 72 - .../gptj-99/offline/pip_freeze.json | 77 - .../gptj-99/offline/system_info.txt | 2616 ----------------- .../gptj-99/offline/user.conf | 5 - .../gptj-99/system_info.txt | 2616 ----------------- 14 files changed, 6225 deletions(-) delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/README.md delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy_console.out delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/cpu_info.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/gh_action-reference-gpu-pytorch_v2.6.0-cu124.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-deps.mmd delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-deps.png delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-version-info.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/model-info.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/os_info.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance_console.out delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/pip_freeze.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/system_info.txt delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/user.conf delete mode 100644 open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/system_info.txt diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/README.md b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/README.md deleted file mode 100644 index e0f2a22b7..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/README.md +++ /dev/null @@ -1,46 +0,0 @@ -*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.* - -## Host platform - -* OS version: Linux-6.8.0-52-generic-x86_64-with-glibc2.35 -* CPU version: x86_64 -* Python version: 3.10.12 (main, Jan 17 2025, 14:35:34) [GCC 11.4.0] -* MLC version: unknown - -## CM Run Command - -See [CM installation guide](https://docs.mlcommons.org/inference/install/). - -```bash -pip install -U mlcflow - -mlc rm cache -f - -mlc pull repo gateoverflow@mlperf-automations --checkout=1d794d8fb58eca31070dffc1c4bb60c89f0fd4aa - - -``` -*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf, - you should simply reload gateoverflow@mlperf-automations without checkout and clean MLC cache as follows:* - -```bash -mlc rm repo gateoverflow@mlperf-automations -mlc pull repo gateoverflow@mlperf-automations -mlc rm cache -f - -``` - -## Results - -Platform: gh_action-reference-gpu-pytorch_v2.6.0-cu124 - -Model Precision: fp32 - -### Accuracy Results -`ROUGE1`: `32.2581`, Required accuracy for closed division `>= 42.55663` -`ROUGE2`: `6.6667`, Required accuracy for closed division `>= 19.92226` -`ROUGEL`: `22.5806`, Required accuracy for closed division `>= 29.68822` -`GEN_LEN`: `264.0`, Required accuracy for closed division `>= 3615190.2` - -### Performance Results -`Samples per second`: `48.6205` diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy_console.out b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy_console.out deleted file mode 100644 index d5353df39..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/accuracy_console.out +++ /dev/null @@ -1,24 +0,0 @@ -Constructing QSL -Encoding Samples -Finished constructing QSL. -Loading PyTorch model... - Loading checkpoint shards: 0%| | 0/3 [00:001` or unset `early_stopping`. - warnings.warn( - 100%|██████████| 1/1 [00:01<00:00, 1.24s/it] 100%|██████████| 1/1 [00:01<00:00, 1.24s/it] - -No warnings encountered during test. - -No errors encountered during test. -Test Done! -Destroying SUT... -Destroying QSL... -Finished destroying SUT. -Finished destroying QSL. diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/cpu_info.json b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/cpu_info.json deleted file mode 100644 index 0abb38a45..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/cpu_info.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "MLC_HOST_CPU_WRITE_PROTECT_SUPPORT": "yes", - "MLC_HOST_CPU_MICROCODE": "0x2b000603", - "MLC_HOST_CPU_FPU_SUPPORT": "yes", - "MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT": "yes", - "MLC_HOST_CPU_BUGS": "spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi", - "MLC_HOST_CPU_TLB_SIZE": "Not Found", - "MLC_HOST_CPU_CFLUSH_SIZE": "64", - "MLC_HOST_CPU_ARCHITECTURE": "x86_64", - "MLC_HOST_CPU_TOTAL_CORES": "48", - "MLC_HOST_CPU_ON_LINE_CPUS_LIST": "0-47", - "MLC_HOST_CPU_VENDOR_ID": "GenuineIntel", - "MLC_HOST_CPU_MODEL_NAME": "Intel(R) Xeon(R) w7-2495X", - "MLC_HOST_CPU_FAMILY": "6", - "MLC_HOST_CPU_THREADS_PER_CORE": "2", - "MLC_HOST_CPU_PHYSICAL_CORES_PER_SOCKET": "24", - "MLC_HOST_CPU_SOCKETS": "1", - "MLC_HOST_CPU_MAX_MHZ": "4800.0000", - "MLC_HOST_CPU_L1D_CACHE_SIZE": "1.1 MiB (24 instances)", - "MLC_HOST_CPU_L1I_CACHE_SIZE": "768 KiB (24 instances)", - "MLC_HOST_CPU_L2_CACHE_SIZE": "48 MiB (24 instances)", - "MLC_HOST_CPU_L3_CACHE_SIZE": "45 MiB (1 instance)", - "MLC_HOST_CPU_NUMA_NODES": "1", - "MLC_HOST_CPU_TOTAL_LOGICAL_CORES": "48", - "MLC_HOST_MEMORY_CAPACITY": "192G", - "MLC_HOST_DISK_CAPACITY": "6.9T" -} \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/gh_action-reference-gpu-pytorch_v2.6.0-cu124.json b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/gh_action-reference-gpu-pytorch_v2.6.0-cu124.json deleted file mode 100644 index 698385a23..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/gh_action-reference-gpu-pytorch_v2.6.0-cu124.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "starting_weights_filename": "checkpoint-final", - "retraining": "no", - "input_data_types": "fp32", - "weight_data_types": "fp32", - "weight_transformations": "no" -} \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-deps.mmd b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-deps.mmd deleted file mode 100644 index 5ac293838..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-deps.mmd +++ /dev/null @@ -1,64 +0,0 @@ -graph TD - app-mlperf-inference,d775cac873ee4231_(_reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline_) --> detect,os - app-mlperf-inference,d775cac873ee4231_(_reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline_) --> get,sys-utils-cm - app-mlperf-inference,d775cac873ee4231_(_reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline_) --> get,python - app-mlperf-inference,d775cac873ee4231_(_reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline_) --> get,mlcommons,inference,src - pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference,d775cac873ee4231_(_reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline_) --> pull,git,repo - get-mlperf-inference-utils,e341e5f86d8342e5 --> get,mlperf,inference,src - app-mlperf-inference,d775cac873ee4231_(_reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline_) --> get,mlperf,inference,utils - get-cuda,46d133d9ef92422d_(_toolkit_) --> detect,os - get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,cuda,_toolkit - get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,python3 - get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> detect,os - detect-cpu,586c8a43320142f7 --> detect,os - get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> detect,cpu - get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,python3 - get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3 - get-generic-python-lib,94b62a682bc44791_(_package.pycuda_) --> get,generic-python-lib,_pip - get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,generic-python-lib,_package.pycuda - get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> detect,os - detect-cpu,586c8a43320142f7 --> detect,os - get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> detect,cpu - get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> get,python3 - get-generic-python-lib,94b62a682bc44791_(_pip_) --> get,python3 - get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> get,generic-python-lib,_pip - get-cuda-devices,7a3ede4d3558427a_(_with-pycuda_) --> get,generic-python-lib,_package.numpy - app-mlperf-inference,d775cac873ee4231_(_reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline_) --> get,cuda-devices,_with-pycuda - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> detect,os - detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> detect,cpu - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,sys-utils-cm - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,python - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,cuda,_cudnn - get-generic-python-lib,94b62a682bc44791_(_torch_cuda_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,generic-python-lib,_torch_cuda - get-generic-python-lib,94b62a682bc44791_(_torchvision_cuda_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,generic-python-lib,_torchvision_cuda - get-generic-python-lib,94b62a682bc44791_(_transformers_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,generic-python-lib,_transformers - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,ml-model,large-language-model,gptj,raw,_pytorch - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,dataset,cnndm,_validation - generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,os - detect-cpu,586c8a43320142f7 --> detect,os - generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,cpu - generate-mlperf-inference-user-conf,3af4475745964b93 --> get,python - generate-mlperf-inference-user-conf,3af4475745964b93 --> get,mlcommons,inference,src - get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs - generate-mlperf-inference-user-conf,3af4475745964b93 --> get,sut,configs - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> generate,user-conf,mlperf,inference - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,loadgen - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,mlcommons,inference,src - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,mlcommons,inference,src - get-generic-python-lib,94b62a682bc44791_(_package.psutil_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,generic-python-lib,_package.psutil - get-generic-python-lib,94b62a682bc44791_(_package.datasets_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,generic-python-lib,_package.datasets - get-generic-python-lib,94b62a682bc44791_(_package.attrs_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,generic-python-lib,_package.attrs - get-generic-python-lib,94b62a682bc44791_(_package.accelerate_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> get,generic-python-lib,_package.accelerate - detect-cpu,586c8a43320142f7 --> detect,os - benchmark-program,19f369ef47084895 --> detect,cpu - benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_gptj-99,_pytorch,_cuda,_offline,_float16_) --> benchmark-mlperf \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-deps.png b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-deps.png deleted file mode 100644 index b7fbe8cd1da6acba406d1f37e099f71ea40e6959..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4222965 zcmeEucUY5W7k7{#Ei(FTm@FeCw&$-Wi&hPxrIr&3VU2*r$13OVD z)NZ9Szi6RQ2M?oA9I9XMfGayl{S}c!YY-@G#2b_fy z*2=+NR8T}%P~_+jmz|v*u`)tJcK`i^po0@e2krW zhJXH)PnFC+e+YH11@7#h9uJ>0{nJO_{m`rb;zLiK(f{+Oj$3T03WeHI6=Eq{Rs~Af zI)I2NZ22ls!j@Y{OkpbmB3jr23!sE8uz;At7Fa;Eumu)C30q(RF@-I#07}>b3y3Lf zfdxbhTVMf{umu(nQ`iCvh!(cM0w`e%EFh+^1r|UFTVMe(g)OjvXkiO1fD*RA0%8hV zU;)v>7FYl!Y=H&D6t=(uC}9gMAf~Vd77#7`KL-}3GSL-cAo$ll63c$`V!twTBe(JK z!87^StN-yV?D}sSFe=gi<17ApOSR`;G+>xc|C77$a|0t<*1w!i`?VGArErmzJTKnYu5 z0WpOwuz+Y`3oL*Vw!i{n3R_?S(ZUv303~dJ1;iA#zyc^?3oIa}@c$QB(41desYshC znekMxnT)ugvzJdf@Xo@L}V=9ED|DSX0Z>c!6z@ zcRU$DW_vwF8hZP@)5#}y_(X`^mAhw@qs|?Djw3&nnu(Bbyhz;NAtm-D<9N9(ekF6Y7;y1JGI(lWh-`kS}_LSB|k zt8$8!{S|)N)f`15w5kH`2z_R`KC7m(0tFpx#O5{b*5zPBHdUw9IrHa%Jd4?pLvKCM ziY5vHN3*dDKcig)2$vF@n+UX8@}W=QOCZI4;Gta;#LfHY2ASU`cNK~SGu+3_^iA995<7CTX#cTk?`37a1l(7&*HK+eksDts z-O4e%GIChbx3NQM`3(m)HR^nL?TzYm4t^}TLy7jEWnjb;LbTA^Xs@GiuN<|e2S$W3 z=(4EE04Go3)M#f@_Hc*cm&5I{MKWQ2x+BM0?X7uf_X8hOAEKp9{`#{o%-4aNR(D{; zx|u(~cAw=*B%N!u58A_@GP12zgNh?Fi}jSQ0%2|RXU-6J^9$c)P|u+@vjX3hwt!8LOpY!Sz|{?? z_INwXPbwUX1(PSU(+cn;kuM>n<3F^QQB!bW8;cyx9|X5DNaso4dL+GU5MZf>qA<3(ZZBrhk(szzbmz@r>)$6%DjX|N}mHC&$}^M z!>L%IGUp|CtBC^7^5tVE3Y7wC^02dWgKxo)2xVhi2UIQf)yT3w^mgV_l?yFPl!+X# z+=}JPAIubw){0SXqFJV2n(va9ZB2`&^*5+ILv3Y|KWH5))7=l*L)H#a)t!TeUaLhd zk(cz*&mMnt3Rz;o?)xxKh|e*Os#Hx$j^2?mY1DOK*g(%sI!EM=nN*RHT2KX3-Edh2 z3%2)Bdxtd5t)U#N#S-?4UZA;b$~Df&E8WX6{Ig4lO6TCSIiB#AfDZbPE^e|xmGgWI zIFGCg(KfAwZEY;dd|-rCqOfQyM>D}JRw`3A(nP-G0UTqih(z3HJ)cDfH}!z_(Y5{D z#KO3!@c<_;YzSMK6*=B>FrYwJ{aMY(%i-35&;lHp+ea=TB#1@A*L6f$C@ilc2iul3 zVH8p67br9$=SOT1Q~UfF<@0U6UeCXDW+dfcEhi_>yQReq)j2QYaZC)aqi4t~4Ij#7 zy`1IAZw~CH(I392VKaBvM;-&Uf^!=4HzQnC41d6>X;i3$++BThfvnp#!ITYW+#ISk z4Su4;%LO+6d-;#F^3l0(KMIB&!JU{&{MAhcM_k4|0QUntoQ>p$TJKV%!2p)k*%W77Lm$ zMd6A!^=}TgsS1Rd@f%S^@^9Zb>ZAQ4ov)>ZRs_}%Kg7)xV9me`@BwZXHX%uZ_Fz_q z=`#X(zLCv}xDrOS0Z9q)qMVt{1B_;TjrpQ_yn@?vU=fxNwv(w5mv) zSvxp$A;zwmK$maP;x-6(gM-_)Nx~aUxAhmSj4$7JvIQ}a-`%5ntnAs$I(%-}3x|)~ zWGh2%sG#;NBYoR?1UIj)<;>zeN)a)jpeD~CCm3a~>@8c{7?wLj((hd=k7?v~9ccSm z-f2>6pXIZ1s%7905+^udRTi4dPmtq{XokbmoF{T2R%c&qg^gcsQ>wm-TE5t;~e z*Ho1GC@+_O&@k$%nr}PsrYBddaR}LR~ItX4w)|q5n~d< zJNT&AK%v+jH&>kLr@l{qfrv4Pu$tcGA~M-WPctd9FL19?A=pMv%V?AC$27V#kDYkJCGx$8gT?AU{F{e`llg5 z#1;lhW0xkr;XnOW<0c4e2dlV_kF}|On@nA|8#h)Q8>sV)6G%bX}({1V*+Juz(T+PfM zVA5utT-{-7Jm2>ugnKM5UhKhM6I5#iBbZOtQjRg;8cIpGwGb9b^u1wM(mkZtd-X7v zl#Ih?LignYAmk~fQ5haB#&?Lfm4VX9Uks4gn&^1EOS+dJ#9>l1s7P$I2MfTd(G`DZ*Ejy2)Aohr;XF5u{N8XQIbwVh^)8g-E@-2pAP9`1tqgA19zIg?JoM%3%LlFiYGNW^~2SWgW)4*=`LGKthg1SWhca;)}Qrz6_x#2=ET2?uV+gS`2(7wUw0|3>!E!8aaw_ zwt-3eRh4}kgZee3H4>HJp{1-f-65*cSmdQ1jN=ct5f8VKbhsAdAbBCe%EEPi&O9-+ zJe&t?c=aTW{VXbitnCer@Jeh)p81gjYeh|SiV(umP_^MJ1L$GZ?;#fdp`vFKcHAK5l<^7n-b0NlR_?<0( zbyn)Lnp`L6lIXZyWcFwW%+<8e`$YI5)=wXO5@6qjqm8zoUWC_e9FY6X4DaeT?BWEI zFnDj|XE~j@9EPNS7nwc?|^Ng#X7D5v9gl59$yJVe&vJR`DiH&ygcI;7`5Skw_a zWnx-+cult9IJM4vq5&|^*C%z-NHj&5 zee>4_L&yuT3PQpMSAh%PF40st)po}osLtpK*^Q`T&Y`z6{idjknL$q)S9)~n?;tm?|fUF;*amobGpVxqvXOpz#OZfch9ey<$S(+g^5}~Ne`8tk$ra?KW1sCO$}rV)y>wNg z^h2MMu4QMAwvP+&CY0&wZjbejhq#?n3o9$pGnAa=FwsV9>m(GzIgHc>o`j4&4PEt` z`^EbXHK&Uf5`k%i;rcS{h3J{1G(r@C>s~xcliK>w$9;^=@A(b(xBbD<^UkBNGQDJ6 zQ{K^8ek0uVB(7bGsEg{Zp-emhId*Uw#?(&A7X{sf%yy?ED&(J#tve5OwD6&S1|j4= zjd1wu8FDcrGsG9t-TPbAugmoV5ML?WQw^@GS)H!u@xT;$593nY^i(n0eY-+{ZXKeRU( z+%+`tUX*B;^r7p+?Hzph$n%pL(?j9@wv4G)Gg#PTY=)V+q@%!N!F%%mw|qjM#HJ(> zDa48oNv3-9vpKlh%4)5GR;t~@rmHJh8m$pPz?PpaUd3C7YhW`Vn+kzurk4Ysjr1fS z@^OlTTX_Zfg)!Gq*T`G!B;7l1>C9u12=142sj(>Oei}_fN|Ej+h76#csC9#yPXJZd zm1Ktl2}Iu8#)3esyx!sWId$YwGn`O7r_-mL zgZjijaSnOswUs}dqD_RC3AdxoJI!O8%TlueVCQK(8`@ancu?^8s>g((WMjvV zcyxnxP_LV#bRd_}ljv=vi@{cvfs(!yLP~K_k$75knILxkQTWP$(QVhs{(@|S&$7|J zeF!j1rm>9-2UV3yNkKbeq@G)jwyDW-=-_RN-hyC)m8}PJj)zNXtdQ|&3jt;SsMo_( znoBlU6HVV{A>kYB;gA|TX*`Z`w{>P$=xk68Hp$2S2=Cu5gH)2{Ch_QZAzB_5H0e0L zMEDD#p-=}R1!N{}l!lwB9sBW2i*pQ2!+)6H&Jp!7>KkY|-20-TI@#Zh@y;OGI!G3G zO^^$c{$-}xmd{N8XV7^ujW{Ux9*EPiGM$wi0R zi;8sMPET+Zu4~FWw_YSOKK%8c&^Wba6`k5NNpTrSAP9R?8Z4Lg#J)SZdl#8!eeF?b zk-J42L~QrJm(JfUk6VA_F`8Bm3b4~suH3~|CW}o713IQdnxwrTSUUvV%1jL8PoFa# zQ#^og#;mCYLXngbLOhn6z1EgOQx~sszPAoq>2%?!@|)1lsFx$%{afF5v*XO{Re;lM zFevn8EoJpI=IJ~=w*rX=O@w}*d~8XyiA>3Zia=s*y;kX|Dy^7~+US9}vbX z6J`7tF6Tk?FL87#j;)Is2Ma&4MBY;koZ-)8#=d=r6NYRtA?ZdPRfq>>;#T^Enb=I< zFE;hC2F*acR*JGoIQbb(OqAo+`tFq>L;?7FG5|^MD~Z8Y`d=+e9 zimB+ic$GFwp3@^VWN{9o*N4MBW=l@GwgQVrlL;RW+I-o}p4cH+7KsjU$1nQ}bl0Zt zl>tu)U+CVDka|}Birc}1)tq-P<%X`*oBLrW8z7_;6F6zNZHHa@XA8o;J*MMi|08t}5z&{Q5R+D8P8EH0>VeoMJ}a8LXIi z3sn|aSYR|T$#tm#7h^PDaf@L5DVL#7%Q5sF2WU~EN3-{)dhTCz)ET_aij5vCH$flz zi$31(rwsYMr(@7S;6f(0vNs`Q#nkgIkV6*kwoPpAh${>1aXGBu@+TgkvZsyO%%U|- zh0(`o97v&9bZc!k?(3<5?1-T}PD8?=A`4hpT1%|z%pclp1s#!1>U6lwLY7WSTm=*6 z5d^4hD+dAkQl@X(3)%+)covvT16rhQwHnaxXM7Pm>!$uha=JUm3n%~sG=2U}ubJz^ z9#d^#!fl~I9c#pl4SBc#>~5n+Pz;ys^9Te;Z$I;YL4x2*S~ts07b{@1hE-T{t-(d~ zwQ@;?@CFF95PXb)BNd9899bT}frQBE85aUklIy%W6XMh0}<%+0%{ zKE>v#C#V#S%D4e8OsJhET(#+xtXd11xVom`c4XZ6+e2p9tkk6xd(3@r!!Sn68;ox?pES!hJOSZ zRVHr7IjpB}W$gOm>7_m9raqB*@9d|!dfgQ_@)mqa-WSM|oPE?P#HRueq?$;55%uuJ zWa`|^&0XCc`;TugrSv(;Un7DQ<&$>=Tdj5OVk>Bg1?!!^ZoZ4WQt8fwLEzQ4Wi7r# zb*aqL&3d`V9Gy z*{CKC*3a1*tov{v@YQ=A zqxjLex@*CE7?X~g0wk3p?c#+(a_6LRrXaQT;X8r;QhiE&aZxcbAzn1f5^NsW3czcT zad012xu+YUV5sU+uNUz@85Lb%F27nt1G&WI-G5pFsu_HEL8LIC;9AEak&fvNzaPY_ z;o_8r218~?A(Ht8rnzyU92)<^Q;q zsf126f1$PvlO~#Mk~0H9QSpkcJMh?uLM?iQkD|P z00Zd|-fWtG;b=3#N6t)Nx#Wr!y0@qy1&(4zI=e|t=?fP=wZUTKmsvh}6on^-JP$l= zPR$6fIR9(z<$kT%^=Ofp`ySJ(yig@W{Q4C10|1)QS>Pe<>0WMXvsaU1Rz_(`OEK~r-v)dTLtvgH$*;pMUxWc+aGSYE^# zc5q@g$e||Acgjw-nbGcHmecrYTO_CD^(Q*}($IF@wLEW};(bef$awT#$Uhj;#-KMm+;#e4!cCmlWPdYz z_`<|~YJt2WHbGlmJO8oGLBfA~PszF~wgr)erz=9!R* z=_c*dQ9DD$mDb3GOv5K#1mdlVFw_)>PiZ=w4na;TsoF%TyDmyXo~3nf6t_m-8zPMo z%}AxyG^_wNrOw;qpeKu6AIJI2l9a&7;)9AK-8LC62#3_j)%_NK;A_%0pt9D-JY6nX zwCGp8lmHW@4BMA@+kp`*x^Eak$7*%BmH1}{*9y6CVk?zss6t)7Lr~a@>R}yedh-KP z;!`U60~$9Vzm!KBLOzt^BHs8ucf7lsr*S2ZMZzg{a%p!FwqjogC3u0I;DKbVJ?Hoi zA&F%;0URE_cZ1OO5+)A4Stf|1%f24fu zvw-`ZZC58a+paKm$pk(lj3~$!MSvevzaIO^W_k}G*tD*@01|oo!Y|QG;CZO!(lpv6 zxe98P_y)4cA0UDkT#W1UB*v777p;$a^mt$D8F}t_T-Ngn|Av2}Bb`LJXs&`uO)}$R zvV>po%2=TNC1H_{?QNbTooZ{1%M)*_Y$5yZk^ufZMmFmnLU`_++V?l})N+JcBBor( zG;dL-pPIF%$7**;`bxWc-o-}7Elyoow+TWTj*!Ng81%@oUi)VCZ8$u*cz|!o(R3tp z@5Q&)ochrz(RAy-e7biUb8p|!8L_f2tX(+Z-J#-GS2=gWS34HXc@CS(6UY@^2PwAG!hkB7y>A~>3n=LdFJkN>OX<;< zp|-HTRu^!|H5ZI(?K+p?I1x)M%n?QPG&#uSN5vLmAK`Y8p0J?HJVwgZPd9cP!q+~q zJ5;*-?&Hl+`+AfP^jhrcrjp60^Wrf9%7-_?z2KR`^4>ss(PBbDEZC&TMr0BWA`Y55 z=)2;9p__K*-6+oB=^QV*N_a1hTI>6}GvNGsu0^b`l#P%1`#aO&N}>P8@X53m(cW!&0U`ly|vY&c!iJ?E_+Ax>v(4e}q0y zCwM536CK5HzgtMNb(nl*t&XerNi^M5u9(u=p!?)o=+1*Q=)UWJ@jc1M#Jf)ux|GhN zsaa!`eI%H7E*q7Ej@7_E+haqJVl4t(%H*%Wj+AQNy= z|4={#-pOiw}u-ve1-;}E3ky`ZFw;=fb>w_M5GMgQ$^+4KR3AE2B&mKWpH2ui%HXH86X6ZNQdS&mDSdrd-nIc_@$! zAq-Wzt-m!9fBgfL^Dr`AM4#?qRZIxZeK+W!05{RGe=1SX&VM?b1s&eP zME5lE{*b5;&dI8Ci5jrs*BWV?)JevxAH8IpddL{UT^qS0e_pxpWpw_~}fod8&Uuwy?8J>cOI!Xnu{nT5Wp*eL62d9XNh zK^Cu&Ehp$YU2__;INCHl-$I|JSsXSS`hmE-%?^TtsLl2FYG=r;IE1`jziN%JW;s~X zx^e#`q{jyF^+BZt`JroVBePxlV7}y_oNwEDV_xKkY8A0ubQuE%c%zop-Am_SYrfcV z^|8X0Xev#hOua8R(@+x|Ax_9>bYy+ygFjMmCtNp}2u-~LgA;%5iNuqek#oRLLz)dZ zIS)G95gL=sI3EtgHI#<{VPwQ17TrH*2EYdmDHn!duYcpZKva5`T(PtFw1u!oa~ZrN!X2;n6bDHH4` z?FrUG=yOmI!fV)B3>1VLJp>yv86Ek^3m$~FZz#GNO?p!sM%&D18{!724e<-$I@KI( zSh*niid zw4-ie_2@Q-#)b6V)8rg4G9&dqTvPmP<<9hv_pjmYd=T+U(wep5gu~uTWo(p}IbOi| zV@>szp-YVgtOD+|ir-)mC|ynT4Y{A?) zZw&LfKqV(A#Iz7hnccxH*w8QB#=mq*RX_?j9sAieTN#2^hE*az?AVik;ZUVXNq+eF z#2}6K<385`l&ujP_^%E!#-rfqB`Fl$gUd;K4NS^<7oe}rHgMG#<=#1;KQtbeXM@R5 zgmwt%9|P+=QgStcC>`F3bQeOtJEpNhqMKCp)|v+I2Tk3yfPX4ChnAQ`J3vbllC8q7 z?{8~j@p}3)e!~K0+iPtD?auE~dfviDY)AIg$6x@RKA+XX)e35mfgasqI|VfUr8Bft zAazzorn``>q=ZReQxzb5koMTDDy_Xws8zjP%0;aTQwOWhj%5q50!TjpXX$*<8S_A4 zx!&I}gAP4p6P9Z}ZfRP2SSh9?-v~@#G@QS<;~SKY9Jc-H-A*D~SS~hkFq+ZP2Ju|} zDlrVy>$Mqtrut|RfkW$q)3~+eWPfPuK^zYZkszEtg)1yiR3HJzbnZh{%VDI63d}DV zT7(Po#h*sMVRNojU8&{qu{Y7nvPbd)Q>>jM){`W=ifX|GFFkvNn{Q=(BB z{IY{?Z<)p+xgZtP0)-^M(2kAlnWN4RGBBU7*#eTTjR&R?4Uq!EygzK)#-B)7g)9xa zc9C$2ED3EP#$C>(HIbdH>Q2^Waz7jof&i8%8r4KFjfJ)-q_|GLf^0LkbFei{vT?wT z8tdXGT0c8*2O71eXUkK8moPtQ(ni-g7{puWj9l3p+4{T=j5bCw3TS9B z>8n2RLzOPA0R7vN5HO06jyoKWLsR#3D%b!Gn=_U8*+#n{N23vgrX=nbH>>cRQ5Hf{ zdDCOiU56f_jNdRMt06jm0z949HP7?t8%DN}LP4WF*t^lNd+SoTDujXga8L*^bAU-4 z@gA!UI6-GcunVe3AI6&2s(Zp=L&JFsTEFVwS4O(ediEQW8fEpmx1FzBi znYxRB16rX3It=N1Yaz9Foh)bR<>ew(a zPfYFHJp*u0b+O}?*rEML321kM6I< z4^=isLpet-vhBGL@On^rM$T1=!-Wc0dZd93X?LK@;5Ql#n6a)C4|17+nv%K;96lfV z3Gf9Q2$AZJQGL5Vb7l^8P?Cm%hsRuKDjcfg%0Wm+sF2|(gk*!tp7~dAgj;t2&O_xj z=d5uH*)oDNt={MwMrEWFs86dSja|@01fm5M&3n0EhoV`j7cJDF9Wk70nd5ClSk&H9 z3FzTiIS9{h!LA!(RiXL^W%5Nq(KH%|ZP zjS~b~U)@MUM~rkO#?t%b4J6c_eQ|Iy<=QydBlPR-2!?aew|h6wArskNHNG?&#+t-L zJKg9LJAw3ux@PWimn-YZk`tu7PWE4!IQg-x&$OcYUxy4$9KD{`a&bGpmxESi7HEC`96M7D)X2jc0G)W=j$Dn~>Wh zYSTsYdnDoPls+9lJVxixNZeiDi>2-axp z4~?@);rF8#6?(h8vzxiAMN`=$>?~Y!LxH%5kxZ=fqv*I9@-sqCgLQ;K7ZrNOcZzG% zJal7_GnA`NH%hK7ka?t!!BA;y;K5zwuHttDquZjuOXn5r`*bJz`i|98 zpD0}T-(5KZ^Wd&jwP9ePyn}Db1!gLs|EiUMf|@wAX{}etsaSPhrr@D9P!mmOsv!gL zTX8~^a4U8KGO6ARGo)|e$feEA%)mfI%pn_g_Di2vfZFVY`6~6jgL2k}xK?I18}_>6 zV)d!WDDFpftAC#syVpamA1ZmI31(CUf z1&;9a=vQ{GH)y;#NSkaTVvp;$%K)znpjS+LIXYMN2bo=-(JEq#zoUA#jTjdZ5Z zPltvfUcad2>@J>c!Xq#eSZam10_Hq2hN)o=g>^9_(01Ck;*1UQuyUXMJc21z0!bZ?== znV>F_)~Y{3+hy}tIx>&7rm;~Gkr|5NBcBS7LKWh(Z2qF`>=2ouK@e%eQ13?-!u;{D zK;&!sqIzr7mKWInJe!WY z726yE>iY`)pD@(z;UbYa#k`tYN{8T!8 zk7)!o$U8S*41FrjFHaXczC(;cUoDuq2Z|6k>E9z61L&3v&l@Lde-}@Z#17c2`Rqov zGNdQe8i~CyM9?`1r?QWuoiog1bs8s+D4frx1d;BW|GI`C=pz(}nxl_YGGGTY3+ z7LjTZ7;ndLj}-icOw;j^zJuY4jdUHAf2d@}T0w)|;BDsiK{hiXU~7~eg5cf&Ctn-s zy~%S(x^w?9&~iehMbLjqRdIqbAFAQy44cJQ)RVMFkRDYc;VL+|P|pbuNfpu5jb4VE z^g}R3#+bBm+$5sz!(t=!NKaBm8O%M~@S;e&Fi`p1=q)LH-708Tq0O6N62h%v(nWxV zXP}Q4_qQw!$^v}}XA+62*lTPs+Fa1kNst8g9r~yOxx(vUeg{kZHM4+9k0seu$j@iz zpR`z}IH#~cB{bA-@Z`V~L;Lb^7<79AsR}O)?+zgXV*VBoaw?c>y9jvmx?b*SX$#No z^Wh)!t13^wqOzOa@esiQnnU2(aM^waFauUAnLayR%U|4;B^A5?RhH$z8VpiM+5M@A z4{wFj{Op9|yJri$l%e*IyH7qykN)`- zZ8pa~+>f+4JN{qbaXr@&NGx*a1grAJ?(x45NlKEDYD!rfeLqQdedU>avmaFDoiUEbCJJOM(onOn70PGM_I_uh}VbW3M9$1q=?iT2BV#zyAX$duk z;Z<$xiVTcxpbElKK(z^(WA%WUzzMXN0h(&B`|b-9*ApL*)@tiZOHaC#@ z9d9paq24?H2lNsf=W)_?-GSkSjW+X4%1Q}zx;};TFBA=hw8eU$Pwlh75W1jy6_F4o zkr5ZPQL)}WfAjAqY*$?($6+PoHr5fSKK(rz`OL+QN{cZPw2*K<3(0n2*ab31x%cTN zJ<#Eb+vuyf>DdR4d3R(veOCxXJAe}GCdZzdD@Z#uVtqKSl%U}k7{EU{vcdWOZ(3?Z z{%f{qZ(A1aH06llM!^4MP~0%i&PgA+6mCj2i{#8-jef+8 zis&BJ7QonzuuE8v$|nJbQ$mV?n) z!d`FYuN-ucirq3rA*M8@E_QOH121lj7{~ zo8%iGyE`w7B?~Iw13--%6J1#Fn>u6eAAe84oqKuJjZ>iuON_#nE_AuqNxS^!{Fgzq z4%OVFjskza>E z6jR6COy&gGdyk18_0!Xuk+W)O&mSG1pZ3#B-)AsO%9PM)d%$*a>S>1S9Q;wNZ+kW_ z*qs^zZD;}CXCClhP5r5tk55tjww$H9_b(je*x`fisydcp%jq3ohW~b8e*aNcW)yvV z&HH@PB3p;-6kzT?_Pr=D&ysn<5! zdNB2fEmjc}WqZ@kes#_yYa6i@{Zb@#FwFdsuE@-J2<`nP$tmPJ-XLK$^7s!Zn8ZoKX5@(+F~k8Y7^OR z_((TUo%+8%<0#msMbr$h-2>W);)S1h+0X+c({ z^4?!dee(QE)EC4NZVK)E)2>kjQLffD9kUtOGDf=LE?$)I!%;oI5`BhUD)jhvO`D`r zIX%jX@4SRGY&nYY$Z|x~G{ygnA6=9V6KtPO+w5$;u8!zJne>yV-6@!W-1n+&)>gcV~5zb#VKB`9SfP*kr<#H+no++xMj`4ZVm+fN!_>g2^-L_?nGIAB%hjmVT1XG3L43id$06n~T4X zp1lb>`YqV^IrVj!ORlTF2RJ}LIu!BdjwB6Zg~=4+pd-vDND7ld{Gy@tR-@;+~)$e z>KKEzzcOx`#ou6NdiO}4E8>izf=5mK(%k&?)(|TqepC9RxKt7N1^Q<_{W=!t*6&e2 zCug|sUv?xMV?4(px-T3;)5FhS zc47!xBrakDk7)jQR#e;92U%*2SF0#Ggd@ zobdPhj`&kbXOiA60ev0&;A-$8>^}?`%KxdKS9Qd{RPxDl)Rix4$DGx-C)Jcr9(BX7 z8Z+AM6v=qQ>zioAoVwRG_9%W|n8`lL+sw?T1k}^mP*G~Nf*{Wy)kM7$h$2z=hq#RP zLt%lFLyq>spQtG2j2}It%&C2WF|u7x>DAP|PH(b5eRo8@Watq$Y6bW8(+iL8y&x;| z_NV^Q$6asEZ%1LYwy%=zHj$=Jfq#@r7ekM^)E%nO?{->74jbx5M5n`(j1Bv4v|L!4 zv(D53!*Qapu3wMI_wdyX?VN>6xO9bv{!jGt*Gnme@Pkx2&|hz$PX5tMZtQqEkF4=y zYN~ZubJ@V;s3zhMK#U{^Z%j(Hf+NZ(M~V8$7=xYqhKQ5${q0xFB^5@yVTr&PnTDr` z6`rK?y)(Ek5B{vGv;6PZ|NVkd>Psf`1hhM;+cq@4{~ptJ)Xz>7#$p5eV6%4`C!YQt z+59+5&7PDG-*#q2IT~ACp$ri@=Aukz7IiG=z;mHkzf(@-Y?noLb~dtCI6Ju|jkv4x z(lFy)r)|)AhGy|+=;91aOiz9JT$unlfq6h z*J0+E((~z2eKpf6){fV}PcJ=c2UUFys(QRGC%IP)jQ|8jzU{SX;EaAteLc|98<72^ zX5K7Uq8+H z2uN70IcD=tDrGgYAWpKViS&4#Bhe~7 ztDJdCFjk@A(S{2(#96x)F)dDqpW5(?+q zkmG|w5f3f(K1wC@Yez=fe2q$aTuauI>uof<%a@ZsS5vPmE?ppbJu9_xtWi|aK*##3 zq72QP(1vg)fBzEnI4@o7<$1mv2@rkR9_JAYzWKV27qQ?0lp}6&ck;`Ae`nUOwF~|J z^BwA^m#5d!@f;3-4gobb>>N`RIm7DvD0+^m`23}NjYyE_D;l2N@Ae`86ndQF7-P(- z&y<5gdEWj~XxHUxZnIc;ewfwgQSF5I?3Gj_y#Q4>22v^qqDnTj5elK)RY?Ra$`kyQ z1UOhT)304k*BY;rJGqK9ciBhmEim2jpN{QGuSVvog&D(D6VLels;${JvSC?>?BkD% zUcRx3TlR!*BKPc3na%N}FsrysQ3nEU zSW&`1;3TEQh0w4VN$vIQzGu2o|*<2e^0 z*g)Xh*2ldZX=r_wa&>%hZ@D4_l}%D8pA)xw<*7)kKX2`a-*$`RGI4S~*qc9&#aV+B z!(V71UHotjB^4PFJzIG4dhK|i^Vh7Xa zB7jTt(j$dam)c~#+%0#Zi!u+$ zDbVRA{1i{TVHqaW@FNQEdeh70vgthx>i45d{g1|^dhcz+`}ASdfB(%gNu!R_MyL43 zq=3)Mw4VA8MG|ON<4s(-Gv-UC#{;Oc?EOaI#k}@w#vwjJ*l`DC_elZ4eHmZ*WF?mm z2giQ6`Yy|LWg{`LnJqgR*ooP2+YklS4`;az&41|mJu4}(I}WqUpn1fSX$Rx?cJ`%f zXWO10Kw~#cpI&~^X~nk<0G?$-&gGrMd9LR!T9}n#!^_DmHXM)$42#MvnK0Vpgj&Bw z{q%>|wNbvz7p|pW`Opo(8#;Zqhcol%x;{uq7cmauHO&_exTR&N9oqM16Yq_Re*A@i zd*h}VYlmjzhHJR9l<~r7Q0wBQW8pt>>?HX7wIj*x!`g0!=ifZPm4d-WjrFf~ z>^HcrolrP_$u7IPdE)Fv{ip_CUaPJZ!*ZV2QCiVnu_jkfrCJ}h3QnjO3@l4&d(C<% zvBK(g_R-yJVP5(o%d5Us^G(T*vej_6Rp5)O4un4znQ^j^h+QkZQ8BFZfuJw0*FKc% z9ugG&B#(+yWzn0OTAlgPCc5Nx!;wTTPN!P}6B`~W*0yX)n36+Qd;nT24Zdph}$eBm^;@>j_@B;NT^#f zBy)1CJXGC2UUjahP**TAsiMfLEZDi8CDCS#L;HJe;Nk zDv}SZH?(MuP#~Xux#Z*;)ngwUsoKy+DR?p8d0Wi-G%j}T=8L*cgDAJ<*3PSrcF9Hd z-g(E&iA;@Z3g@2|Rfc=Ec$YZUGc!vM7tczgeAePj%ENQ>>XTjgZ9(05)D&Ao)v?GT zFDrH2=G*qIS^XEds+V{S>r#H=^3Po;N5Q2hiALJTE`D!u?3B!2#*Dv@jM6%~t-$ki z{7F{l6|a-&Yjvl|#uu8*W9IiIwXC_OLuhzkU0PaXG8Z94ZY@4U}+?h+bYAH z#^21(Y`=4S5llmWZk$6VKUdXdZRFS?%xtkwK`*iOw!+%^w>qj_e#6oh%3MRdu3^QOZw__6LQ@oTq=Bv)8t~5(fWY;)n zSbfhuPf-u_9J4c8n7r*=A1%6c^-hF&t=m$hjYaY##XYW}cy6g$E@e*0yGSbz!#u9H zO56pWynJ9jxx9G>yB#%J3R$3To(5m$S%`tehjRW*#d`IK2V%@<{$7G&aoHI6Gvqd( z2j_23d~r`6K5S3WmApN%O-dpA#i~pbso~kPMF0hKmUF`Q>5g-^je0Z4<@NsV{Phh2 zy0raKH~;NzR@e}w;QRjX7H^h?-aP>3dQwp#O|$&v>F+0>HAj}V9l&*;LR-()#7skHN zMX{)!CxKam$#td4Ov$W5=Gz{3=REn9&b8wmmn5tPf@IYUqtbuNK7ZjoS*GaG&pLjQ z+)Bn_8g=WYH&mHbM1{E6T138U7!bov2oGyip3}1Y_2R(eOAgrN_nL7`-sgS{Y~)os z(y5nDEM0#zL78bj7O7U;LTM@M%w8`JIxcgh<5@EQX)98f*)5e@XR)Db3@!JcCU-ri zw^0_iHPmmj8<<&~Ym#hvGIAMfYGUYg$I@8hG(|gWFNe9V_Dvmzme8jORf&PI0-Clz zudlH;_$D!rb}Nu}#;fsIt-O-;HxV=BYyW0i%JZpg)01DXS~#JLjxky_JKzoEP*X=_qX3txpUUWsF111 zpMER7xQY3oq$cS2s(uQu@vP&pt7R&lD%(9RRU8zbAJ5vo zG`LG_`TdpdY2&#Zirex%yN(ZTkGRM15n5zp<9Lds>ep2Z#d-F@+^MTGoT=^}vD|#( zydoJ2LCUyh>oV-O%u#Ten2~YRqk}yw6#0k1ylWITHkq2E-iHUrbh#YjO85$jL2s z61%Hjr_#XUjSZhpy0m?`aDX41GoLB9G?h%h)K|I6tJfG87$g0#>W2UI*|vhR*#bYI zFWZ*aF9_m8!&CrF|v`tKi!#HGrnS-7a~ zUN$7K0P0dH_>1SWpSipdTk+DD^lqHAIns&n;G(40*peT3e|)tW3H$qR@4T;U!aG0? z`ZiM{FTxPg46p#5;Kn3#I|%UC=~^Z}AlZ(TQ2XD0qH5W@OJFdV=Z{+w<7w-KKgU^) z{8ocr==iYIM6Ys4=`<^Oae@n1*)l|Wf=-N^Xx9te6)q|9qki5R@n(eA*eJdP-1dF% zuZ_}Oyz)ff-5l+uBAr@md}}($ST^M6wfX=5#n~9)61H zwo1_6SoV~fdwc*y4Lig5yE(O zC~I;1{58Lu)ufo;C{ZYer^S6|NdvC&wADDWoV9sGQ~|b?z%)85bcPQ#Cu$r)IVs~F zOmGDkt9yk?i);_A4$3P7qcRxhN7r`sfHtZs+0eZIp&4eFbV{q~(B}r^P&rL{nmO?E z_uOsgJFR^Zsml+jEV=*1BK-4+e4h#MV~t(@_w#b-5Tm?r%?yDicOnBA|3z9DfXI$n zZ$uEA#k@=(<_hm0xR#XFz4>u(W%Ht9qQ2fx9qQdV1gqCsh*9Gh?N7T=^sYa<{wic5 z3R)P5E#sXGfF>&whcs-C2?fnOahzp;X*d#^7+ddQ^}4Y=>JL7yixF-4wa16DYgA3I zvma%d2c5pXtiXMHi75~a-qd7>e2IQ3^GyPr|tU{D;l}2EbrQt zfd&DbWUHHTfqWb9ts-C6VcU+E*V;cC5tC)lCki8%=pV_~DJ!jbv(DJKw=DM7u5Zs* zQ?(6-?5R-JZsWvKefVMK@D7~9vMM0etsmJb;9;ub$u8`Fs>W~teS zVsXXGD}XVxQo81$<4~QLc#woMDYJ>BjYC%k>^ElY%JMXzA&dHTQSAfFx!>DyNOi8y zO&GMZi3gTBIT+LKI5f1Z6neEO<`-VL$5d^<)lI^k>!v3vXFi)6o4en1Lx%HI$)*l2 zWR5DdJ6>3hgxLGRCFBNpzl4IbxfF4M&w5H9w1dB0K~qL7m{;8tv+BWLC_R$C1R$C` zw%PbImpmEXl8~(2OC&WPOFx=)nW(jG_Gsp%U*3$z>~X08&O8MkN85t%6yFMNA4xDo zGdhbR`(x%=Vs3wYSbJ&Dtct<6cE)b(ExvhspyjBYQaC&tk{vMh#(8Hc(Rl2QT1Ujb zW{h{nLx`3MDD=nIcTiE@<&9F2!QPk!^-M5U2`!av-u=&^&o(Zc!x=~eAAhm_wm!Oc z!U6PWzg({D^-WMQP*S9~{O|o4f8W)Q+W-3h=rex|XQ=BH-v-hvXuwfengJYKwqUb2 zC&!Llcq}{C_;IJ%WVnePk~}Jvf7T(5ZS;adWVIS_;#9BGZdGl*}@}P}vk8Ht{Hek#0KX9$gRT<>3#K;o}0qAH_D3d}( zBF9oqg;$j!4mn0K2Rqtm`f>nzf7=+f`T19vaBF+JlZ_KK;lzpTMWF`^Ur-_L5k$mq z2E%RP9l=CI>fq{D)KelwwBo~Cg&Ls(!J_?gJC}3;%%jHz;h@(&+NgK8*WA$h)?`3Z zb{4`dr&5fNxO@CaIW(7F6rQf6B3l(`k(sh)Jy_3I6!BH4{k>=1>5&EZ@+Kqm?-QCw4-g*YYZP5zZg1I zEzP4sQQJ+fJ>X(CfY@z6@6)sl+1ZjQ2?vdVuo3{+44|AepC=CdR-{*$D`f39sfgB=jRsQRj; z=nP+eR=<&66tUpp@MJeBKd(sYe3GSi=sQ7`Cd06awN1Tr#r?Q8Kg`@d-MUj8>rGN# zFeOe3u*3C1Utm(1TvrH=Nh~?i2BEwW3oGw9t-SR4bq{||#j{b>79nSF-dKgdHv?CA zk;ac6Cxreq7%&fayY_JR_oUp{h{dX(?qJ`H+2g9gy=N8DT$j{{3B3#Cu7a4o(?H~o z)f%#R6l_s)u$9NV7!o#z2M53B@Q0|9Q>NyQu8FzUO>0K>R*g;0RB>OIu~s1d*wZr6 zH<`M32ZuS6E9^0x`?fctmatkXZ6!kR6zO=Sq8V}n6m``4oZ4j~mC4#hMcrhAA=VS< za-bGixJKl*$rX5~G`+v?lr`<(1Q*lGF&c!iZ^I6L7+qWVpzuRwP!0Ujr8G7;Z^vTp z`^qK;{*x zZD3c5$9r1zPSmjZY|9(iKr^ML*FDL^3N*#q{)0iu)1Q|=hLZL+i5bO^hz|Q5T4f{y;tK1`iU_|uk^ZE*^~~MqD%*z zS5W*6{@%2jGf%-Q;m{fIrc<)8E7VFv^$z0{aRv76JhaFO$TydS3If+&)`lh3NlH19 zdh-_(4MvLHMT32UtnL@!erPQ<4_I+-=%hU6`9lL@47=Ql?9dcvnOe_B(I>gT`)msg zL&eX*qRq3nPrLp0UuAx!|L#tFpLuv84sZ=BNQjClR;P7BK6CID&j0IdM-1#B`KZyU z5D;0`S2R$}(xxNtCMX)!K1_8jx_g-|aNgbAwcf5D`hOJM`|#tL;=EWFOEl239Subg3zin{s+%ygrBq=Z^qhASFk=41>Ed;ZFFx$vP0 zWkPoxIBV0YXBxc(YZ$LLf`g|bc;otc&mn}x1C{vbu-c{;PA78O+!Y*N+eby~S2GN< zQLbt4+}@#J(MvWfgzqSG^!9g&mT4_`@l$AT{Z9WaSycG8SMC&{QhoG&Ow`Je@L3^R z=>qa409?6BX%5s)MgRc%~`#5H=8BHIn@{1oUNyn_OK7F_F4ye{|Go&3b zYkPAJP0#aasG;tQ%UfQf?=e5^_GhOf#El}_Jqn@%F@HZhTHclk*41ryf|j{Y_RtnY zhHUMk5fup(Kxs%nAte~Z3va_<{{ppFL?~HAUtV2bIoW?6_ zq}+I;uQI$?!Oc7b=j+jqQ5e|!HIAojOlZ0(&Ow4Oxm0(Z7_BjGeXCnhW7GfBZXHdzmvFJ~?HZTW@9f>|-bx~QV zY(nvcR{&~rXC$&;$?wx6r@%o$&P3(G4MaqakD^7TD#yZzY!n2@4uW0$x%}EG5Ttg! zfpZv^r4;2FM`HDkfoGB2;JCYL>9my|j$WPuB6FJ@h?GAoiv;Ea_)EzWB)j}eeY5kzU&eopBiP}wGbYGggj|Lgn67aQ9)3~DlA!pkP{)QB2gJR$W9N8MHy)ZaW+%G&f&CIKy%tQZ{Fz2}Vi}8cpsTIjYKfYydp4h&V_uRhuQS-U0U&p?@ASlAN(O9Q2#CV43G}|X~@pWG829uN{ zTSxDUIq#PHnp_qqfo!egO(d=C%U7iA^WZ+Vc@$je;L>O+QW|)(cfgV=n0z*{FBiGG zBWnvOUBO^(w*Ruz_GZtsQxbbcT~2z_(a0Q%FpefmUXS9v-d+Zs3HX7XRbI}2tkwQD zIEGx?u3B}3<}`3)&O@}+{b)&wKVoRnU6iZpAm~4{o8Rt{H@!E}c85Q zsKUGh7^92Z3vY%_>f8zTzIeEibBV*fC#}mih}~wXVii z<(I2|{MXKNb$Nm{WF|0)V%$)*qrX2(o zxuw*UZi<$ANQFf){uEsB@wx)F=#1616WB;F{>vInKVAKUR=2!o{uC-|>8a>>qC1tH z7pS2LZbzGeH`hQ9CK@#Q(J?jQ>8T2NFXp1AYnrn3orUu}jefq86O0{#|0Z;kj&ulut&} zrC@X}E03o$7+OGKCdKZI>O^=$*Kb-ztUBu8&ZNvZtxP3S0q_X!iK}%lJk@B3M z)uR(W^|P^kHu5uK3|AB;%;zy4rF}^=d-um0sIa1lY zXbWUs=xEjYf3k@AZz?A;>&wxdQiGD%uY#|uz<_4hPryfs%Sox!Y zu=b<2F0Z?NhF^>)8~kN$uDkCw2m^L#EY!Lk^P_%k?FDeQhtVM)R3?EC##e9}7k1eI z^!b6)>eF8qD7(ZretI4U&0k0hi$k(j5nFm9-76b=!Xpnd&dq7s)gc8de;xS{5tRbl zLuG<_WNyQDOfTCKG9w*7CTo#qAlW|ZTiw{RK#`VcC)c8&eOfO}PxxtMqGM>b5LeR# zC2@_*Rg+pK$f~5|8|uD~;pPnbIz{gjmph7tKc832i2rTDaql;h<~=AG0y%a_RT+S3 z+i+Llug4uK8$@O%Oph+}u-MlR-N)3k~GV@Eh#alT<)$s&ZHRp*|pNc=&BSVN8V+W$u%Mh-HZHiUg2 zGyV(({&%DBv28K`-C~~Fa1M7A(F(eEs>Jv5j#w#{@MO?rBtd@A@~$HpGv7i)+yd=y z=?%t8D;X}f5s9>gvfFHu+%x>#UC$JSbkEj>Y&j{pld$taM?&&Ue;$kWi?@La(uo(>gbJo!=k5ZhV3$sG=RFkArAQ8==wIC@|AqTgeWvk6z&x zFYcn+`Kv~ttaBi$Ri{ive&#pzHhEXQzoseM*0K^5`9n{{3lDVQ12QHk5>rh)sWbBy zM$QP9VgDmiS34X)<~A#$r*1pNZOv+@Ow5tSkMKrSmF25n$>|^2*s-hoKwT}$gmxyF zxfDE|AWpcYY>XkJ8*cOQjlT-t-btmB+3h#*`^!1_OKzW~uM>M}r>Wxs25=Ys zqatZUa4}9Yo5v;M=jgLM212m?ejUg0l=sYWKruM*`da{piw^tb>zAzPHJxCF+S19m zB741j5XF&JyCkh1B;OsH$4T&dRvVYe9FS)QKaWAh1R9)A0(tyc7p zN3)ctC1~ftd-fFw#`hO2^_ptS>R(u`VnERmZELK{Y)sQ($aSFv_3v+=MQ&xaSEs8CcLF= zG|^_c-tV;|7xAvJAgzT`{pPcsSganeC ziI{-_e_qg!Mihyhs;6E3EeMallu}oeHyn5-geHPCw>Np`xRD3D2`-fu<9T|8_ViWJ zFPdK#HY@tQM)vN6qxRd?E;ikNTu>Y0?o~tF&}*!`56jx0BT;eLf)#4xRRCGh8t7%3 zf2g#EvOn0AM6RKdh)!x(3voJ^l}!dr8F~&z>~@b;hFnaPLlU?oK;j;1rzJEj*P^|p zd85fn`@^3?+&NIx*2zdeqj*G6BIu$bSduvnwAuMrG_;t|LCr1R>b)}E!M>q8VFTY1 zdr^Z^z(-hp!Ni{=wgI!{tPqqZQ4e}h|MwRm+<~N#L7WF3)*Qtt8h7;o$|j{1^iRXRk=>hK;r+MI{Ns^9YB+Q zJMTX)nc*SSW-94m=wr2SP0<}?tTo$LlUJoQE*pk}nw~CLG}NtNc*^Zp&qJ@n z>icCQ|LW|IKY_u20~5VjtIav;n=)|Y11+L*KCgkXO_nFq6nbbA#3^#rB?k-g$*Sw= z4wQLbKqFI^7=5PJSERdU$ZmOFIEcpIZhV{$MVW?*n%)V0Z5TSfM$$vBhp|8vI5;2^ zEyHVSC!%((Gda8u7?&y{goAZQpf@^TqciRqZZ zt`I+j?*zVmxBa3a?z7y8NJ%7-GDt>{cLc@CfpkJOB~5(k48Mr^ot*e(fhG{~BleVf zO=&$=Gvin*M|4OgzaJ7}w>mEWa`LELd&mbchVd-wpvnzLtP@Yj9E)#zQTjcKb1XgiSU z+G*%h3~#MaQ!8kNyZ|blR&b%a&9hd}Hn&%SB&+vA*r|xh5&~|BCf@?*itgaAZ z=79|Y09c$VD)}rV_YpYbjyBE?Y@YuqOhbK#XEZ4A)y(uR*u9V8`UFI z!of4NHZ+JS89dl`?QaggYfJSDC;Vm2ifE0>pTk`KeLoI3qgHm`-gEGVN|LCE%}*ru z%y9w-jB`mpIgyRsecv)HZD9WxFnW2J8gHqz2V*<&Plsu98m$xi;^Lonil*vRe3$V?; zxv2p)n)AaZ`&JJ-#Wdei*!tua2EI$qVMX-=b;)h}nZh_79cM@zm(h~9Q<6LBpS#+> z3Gl#n(SnN6+@FAk|Icjvdtrg+(09rZ{+&FahC#rt z$Ct&-|M?@ma%OpmnXclbA+0vA1O0dBdFc0=cg%r*1JQp5C5>%Z0AN}%M{mhG#eNno zuaf+2AHF7}4It3ziP74(k0Pq<+{|6CskbaZ41FvgJpJjHVbU>PWe;6N%yn3Rz z-%P^TUgZ{;fNgGhS6w~DBgAUH#m6OJ)mCQ>Od}{}O@7*}$fLbM&hytD&M_AFL*G4l z)%IIedp%&l(t6DOGsT2#Eu;+?`yH9VmGVIk62!V)0*;=`ysnp%rmd;AxMdM9PL-LP zI~#THpDKs{d0hZ-yr6uTuhsmahVTP;=*W<9x;Lm(I_!npOD6iftTDfmS_^VeRMw!6Ysuml`vRl$95L zEw*IbJvzE1n*KWTDiT+=edLt-W^{SfT=LdFAg7Wbs*XlFS`8EGTnzPi4!kzVwnfGb zt7P3B1}AKG%@tEAyW@ejP8F(?@87?%5Rpmos~FX3--&H=T2}1N+R6?b(qA&^yLZd5 z$}u$MqLHlhtV7xK=L`#N>*jHmyZ_s~57=VwaXz>7*9BLB7A+pR7QDD{f{qrVEwqny zCYUA+ahf_r{?^Gj84sUqP@KgKH1MjWUNXAR`8r5ecYt{U7fUs;58f&M=uzGQPXind zAo?~hA$N2VeO%p&fyz)#rZqf8V6%Gi~wl8pT`r_DQvJEzT`?BuQLbOfw+ z1PVDiNEr0wy{hJeWb=nkwcF+hMv%Bf!0;g>-Bm@%OdMaGQUmqKEU4rTb0Q7UhY|*X z)0z=$0#Is$N43qsWc@2DN7cX$O>vppUW@wZs%0fPuX9;krfsDg$PEB4yr?GX0QjT? z)8uq4!OE@W4rtc66I11K^w|4z5ipj>b?LCEh!iTHAL6)=-klufhoqW=g1hovef;NE zE1gtQq`0%urOO$4M%aMx~&=3!tWc(Bfz8Juu%+SUw5d(6mQ^f|o z+sXj}`STX8WA-D}PpIw{8gtSH+bCAV0hd7VpqmYipAE|U>Ql+xqZ_@z2kbtdY`Of` z+0HJ#UcZ3yW!pMH{8bR?r6|I|H^S+5JQC>A~qHCH^5S)O#yp)udm_Vl#$#@ zY`j&BVd#^tKB3b6?kw=%Y=EuS_%ObQc71hqx#!~y`CB|6_;DRVu(x}eRY6z*EQMkE z1n)K+FX`2@4Lr}(Qb0n2QUQ+23eLeQU47ZCnkRC7w5e6c)Ht|rQcBDDLx?$VL;ZFv1AN`6Og8*O?*I=X1jAKC0Ku@%DM@HFqYeZuGwC_DmqXFdh3e?7b^p#Y7>%NU#ndj2C}kMVZ7FeJBrU+zLp@S3eQJ3U#|Hp z+|fMIkF|9)8%AyR3r8}hqtL9cDjSc{_uw4TI#gbMcwa!`rzIvZ$PD11sZIgbfhio^ z3Ku40N>#7Qp||Q}fjuwQE$BNetAo06Qb$?p3VvIDz;f(|1gBdB0#-%baAVCHjd==9 zDG;gTbX|&B#YUtogb_Il@h0`$J@gm?0N-yj%9zTmkgF#>qdy@22e^r84F@?Eez0z5GKYt5#Z z%}OI>Fpky?To(kyA$o5iWho@A4|;fmEnmlXMucJGNqp%*-O@SnAem=Y?uZKuLNkKP z(KDw|+m%62t3QE>jpL|tehm))C*~Wu^!}l1op3hT`m_MK$W|Cv(FS;*=Ew*}iqXJKVW-2e* zKxEVK{+^^6mk9cjfO3GV?za_Q&6+=WwVMstJ304YqzPD(hG5(am&LW>~g6G z7OTB#KVWQuc>rnk@Zn|)dr|T{fz*m06Yv)#tCr2Hz&4;z>Syq&CVb4>yuPo&aG-&= zdtY!s`K>dLGIOwpH40fumxm7KM;Hu)18fXvHj2gt`9jvL*WY*fXLn(HO-I00m4K+; zDxi0t5HIaK3Jl+gWx{hVmA}(<P%HJ^P6=gENB7E+xKtZi`Xl^YhZ5FVB9(denSGE@h;$n4$`BiIL(G z(faQeXVTN=A{}xozHcG*K}wvM>Q&0fqk3*aB)-PpmbuyatHj9$aq>y7Pj z7QpBuc03${*{Kt|w*KxnJ#dQvz6idHMT63JQv-k1vb$QnVXVPOLJCMaqGcY?o;#w- zwCpVxr=9Q#({Ix{+R{o1VsFdW-O^?{iRWIMQx|kDDgLq>el=cn z54QZRur&IQi*{tR&BaL*ljNbc@FQWY$PZo?=^#*}uw}-X$345;-JWP;D06DlJ0`CI zWRMCb3GUkwJLhrNTmqs~+){u>PBGFYd>%BNn51{OlYYP|0itt0de7{>bKA$oIS-gRj8hc$5M zZAYHskG|cDQN0Z^1ijO%uf#x+3+-?7Nt};tnSihf88>|39O5qo+#piwDuy}v8pXE- zq{TSuJs^qoIEfTcB|fW%8E{yA&S|#l=uh8L1Srd~B?ne(-c7HRk+RGKXi@vtOj4|Z z+jM6U<WOmC->G53q{vd63P7kKC2FKpYk9f zCI3BfU3v$0Pk^K@Wq3=4b#NoqS-#dAl4>L?myxNN19qEiKm!TX)0NRNZw=)68oa`u z-aglaB!50M@RwH7fPVZV`#JW4AHNWBR0g2$tq0^2fQvh$NS=XWiNIh=Fa9{c3k#?5 zwmyKac^18Ylg#jdXvsa8n|$Uu%P2VS%_u24!URW#!@=(g;JBcYMRk6;6W@nf+%4xgCP#-swXt+-5wCIs zcMm9of&Hz1w^FtsJ4r`l>l=m&fqJDi9;Si}0$o~|PDBZ(pS3=Cgx+b*3#@NBOaR<| zeGzNnb`wVIlt4i>?tpkJDKAIkZEdjo-YMA@SVHwd6J1d+gl#qV>quS51UqB~Dp_^F z8h91bk8!zmsQ?f41St}jFa`kEz`Bg3%sWWXXlEH?rVLh`Zj#{;m&i=vi!(~Mmcutc zoecSOlpae78WbYcwm*5I%i8egIzkXP+NP{kUYAl*Ds-5mP#&ZA=94(!Vk}I){jnvk zRWlRTdEn1hF-_47d@+AZv&qixcm%11=&q83`VuWxg%a%)ns2$b9|y|m+JK%M2h?vG zwD@qEQi6A`&aL%|g>n3m#*7gW`BcUEr^EeY<260XHX(x(jzeKFEkN1_+AAncvx?y# z3qm^luc#>4bKb1D_-1^OYadWcjIM!rM=VUdl^DENl<dXHYkByBsl4y@2zs2mdy;QUEg66?2YzwZ(uV#d=q)`Y-_oc`quEeDY1S z|B^4qHz|1#5~FMSdycVvDAsaz6W#gut67dL?VDN$)$i1e&iny5LG(dtKVu}*+KX9< z9lbX#(bOT+G+PNwS)MAXF&Q@64AZ`(EQGh7U1^ah?Iet|QTIws^U1?nArb4!T^NfCPP%r)^4$I8J2pXaV;j`{s(3*>t)q~*hwCTn^xM&FjGGl1-G zEM2^gVQTpU;+SKT0+;Iu6UbrH2h+8$sC3O}PJ0zge|R@4jl}F#&W`Ngdqp=Dbo)H7 zIm=wBWF|KGK+uionW0Pxk51yWzwBz!AMB`o+S6F3$MEp}+m)@mxUC46`x?Lx2M+6$ zM^p*Y4b*?R&WD_J18eA01K-2;)cu^V#a;71{QNpnO}K(F%mT(5d3{ zbD9QW2aMs5Gk`=Qbc(>lnmPn52NzH3dQR?9-omm(&9AMj4S4ut1)ie_Tt-K>TBo(9 zsiA{A)9-|o)!Q5DVm<72(A}`**kY5g5wD7 zT+fse*Qi~&Wq!Z!J&;K9!x9cN^-Kmik-1pPcxLi@hQ0+dk3zv(K4+Q2w9ka$`I60)J}7%zZ-(&+dko^^?FEEb6)_-+l%_DBx~tZSyrk(tU`_!o#$!qK z_RYU+9&jt3LTi=*7E2HF@}9k;HrmWvYs^dT(AQTK-4|A}a$WyxOXWZE*muWI1wQ{) z%2f5hUJ0;0L;$)Tv8Mm>!1CKWnheg$hx!$}X7%*R;Hv~#VKii_T8FQGwOmvdY=yHy z%*u3p|Ls2>Pyg@Epxh5}A5?%Gb+Y*UthycD;zNXN-Rg4p&6g##u4&JSxv|bz${om? z>e`?n=G8C|MR~1;a;x11p%0;1TmJE_0WshPR5zVCLilk;iF4wkGrENes@)5+&h08_ zM>k%o(ek!VwvH+%KcA7`o4E$U=n@7X=*X

rfvrE}>_K(_E+kRW{S zrVef*yt}OIihtlkJU_D#U@cF0dkz!Id-c4#H!Qd)7s@z2#6*fuV})IQbolUsa#3#xp8n9KNmCMhcxvs(P=Mi z>b6C1y<9>sXQ?Y2$yg_H7dQD<97CuJ zkz|iA_1_gLuJr9{v?EYBZI4z!or$-4H!P$wb<|{>C!#88E1Q;xz3rb=%5MQZ5*K6-*v1%prfhNb0tD< zfK*b^Gzc(Ixasat;mc8uo^-Y3_n};?+V^sL!K8S_phA1NxC~Qk%`3+)iw@W@Vstjh z6Jj!S%79{O0?=P^1HdW~Anb^|F4+shyWuof zBE#j-VU1vk!O>w2XI>bD1Wfmgh{#JYFr9=&W|*c=EQ~;TeTQF6;E9+z+vDW{=<{pw z0687ksx>voY6b4_W}dR0g_Q&U4>X^#lrfu>v9GVCU>OcImmc#phWgs^sy)ri9bDS( z-VnZcl7~#%)>d-s`Z(k8qj~+q5b8VQIC|MGxc5O)?i8KG;bYsh>Y*_V%-*7{)3$tG zP~m6av2Etrr6= z_wrvWeQ;L{GdTdKq>W9KyI+CA^9OK0Ci$d0$sA(Z)lZIcZbA$03x@&p=a?&)d^o{z zt45^u11&J+j5=Vrwc>J$jD^Tf1*6JdcG2JKN&{ccF#zT7{)M<5lP56g4-KsN{#223 zC(Y6Olaoi(2~iQ*t&X&f=dr$E5-V%3)m6r3w=V54_d(~%UWo%~rSehnG?2WvW=EIV zLvcr#Kmg`Y8vgq}Mwqo^5b$hdx7}NQ|2-&VI-)=+l==lt28>Ku#o9d!eAd93S{F^| zUsdN;zP&L$^s(jL)E&&r`*H;kdzpLD#r#5>r8~>fA&qYk$vWiGQ)}Hc zqjmKsowI^C!M4Ac0Ihm%<@cjZqWr_#+QBH0?e1P7TrdKJmfZg5#F{nJUgO~}o%Gl} zz*3HbyvAqg$kI_Ee=~ z5dr#IF@xt0l^xPeV4`LAoyh^5Osb+~z}uT}OUUp!Mf_Czz)8O~jokzF=IXl#awmhG zqi9wC(djDBhK<>RK;>?hKS!4s!t)**j#sht{6bmxqi4*h+cB-r8^JI3UX`A*sth<8 z7hg59vFV9tivR7rT~U{)z?5BT!rybraZplPVD{zY3Krq~3uZjpON@4&4#)Fe2OvP2 zAKkk8>YBNPDL|Wd?P!1%u&;G3R`GeCCWf%Ln(^Q_KrJLo85*uJ?PZ;^%XoFF_QXh~ zHEYRf_uOV`zqZT%bn&)?A2@C2*YV%vFv-9~^Z?75(G z@-PcOo|yv9k2nzRU8*1GU7-@6i$se-i$rV49Pa1qFR)<(=pwUgemB*qJRx3=ZFx~S zE7Y6;4_+)fL2naU82_3zg<=qQlmW?+Rn!MB1{H>r1_7lE8?*+N#ZU{4Ibu8j571rL zd^@igJHpF&^Cyuh1zJ}Mu<`|1k!$g<#DeM#z~-Vj!1c8GO>6~^@-?hd3w(Ig2LR!F z(>5rh`W${dOh*vxRiOhJf`K8CRd=_nm2K?O!_6UWY}W9{(5S$~jNE0`*cjMg!L=-z z3OEiBo8hYR>-e2AeLH6nY(b*+l5&drUe98)l-#jI?2}>I%$d2F#fr+Vo(JaN^MSog zTdjoT6+JV}XWiQ+Atsnk)g#A&83ING7p&lMxlEh%^zj=Jl-1F?Q1H126H|eAs4O4Z zcJ~j`yrC3RV8Zw8jse87ZAAOvi+EnpWzbH8{WYfw-VDj&50pz{eL;iB2jTXwj+W+l zoX_@nyeD$7AwYLLZN31O*>G#xfF?O?pB#v0e3MX{b`)Z26JDBTfL$~4`)lS})FkD= zDNRrwrEv}-dupcdS2DO2aTW>h!dgfhv+7K<*2F_99!ITFtUK2I`(A1Ll zW?TV(q3M3JTuY?&+mr?sbxLNWH57z_z*dQ}XKAVp%9A0FZ?{zNgk+g!i-oA+Ul?sa zWmE&8bUdz%=j-@-8juj_??HJOs(+~`B|R;8K?D-+nX1Wy6yqqEoMQkmwMVFFWdEPT zO@Pb@1-%PUMw44?jM9x`+ID=6TOX%!ek`E#H&{*ugXj-kLO7XuNFL&cq#NdMB$8c?%+qIL;== zA`$QWy0O_aw`Mt%oc(s2r?L{i5-go9&A^AQR5bt2aQVY}M-rxe==-{7=rH+RD*kuQ z0#oCe^M{V<7mbuP86r@=Q5Ag>bJ%~E51 zGEk_}1lEuJxK`i7ns!|Ib|&6|N_lPViwyg^{3HAD5yac2^{d*jQpDU0{;yxVZxY6>J(@I?l~vL+9gYy7NTKuA-0_F6 z^LHc9juUQ%zkG=c#>2?H=PPb!S~}pW2Vo>%{>vzj_zoOw<&uS&%c%q{z~rp-9a#tE zQ+j-ccB>DjC0Vw%7{+bH3%frNPWno)GaS@V@*TDH8*&T7oVlQ5y4)B!W{s0WR=VvQ z1OJbMQh>LIi<-G*pm4d@Xc$C-DGg;vo832~-~#44bCOm70FmW%3H8vW06>8gLj6-9 zzc92&%Ul`9`b$m~JlK13^14;9P7{2#;U;0`1;18T%!kA_u{(Fd5_RG(TM2SUi=fts zJ0S#gorc@CfO=!OA$dQvjH5V@Yd7kFMCIWEJ#2Oe9TxOtO_lESo1Bzx0eh(=;RSj9 zWe!j#_Iz&Mk4h68lpJJ(8b;_M%M;E;0)-odkYv4i6}T`Q29Q}%8(Tj}oB)!{-y_CG zVZS|lf~7Sl`Qw6uEQNLs91d4ch0=9e{IW!Estyv86 z#icd}kwtTiPHV8U#W{Vw;$)A9VqTsOdi0{JUu64O6WU%y8li?KHP}^1g zTQ31N$aP2(Y%;B{m7vh>{ksw!ic5~A8~vC6p}OFxjY)1mS$&%~&l)TG1!}e+Xhz(u zXzZz^T5ViRJzd1lACR3ZT-p4-uFg~LQT2!Led#)$jpr<|>MV96H-O-h*?Tds;=O0k zx5LYYqvuq(8iTO$5R90(hxkv8xHVDCS39G0*(+LAA}gGZ2e6^eJEMX`pGwLPg9M{B z*bt=}uB3tMAuB!bdUPc+n7q{5=lCoC748;M5W_8!f7Tk--Ti5+u_=t4k|G2>ZaA2wbhQd4?zOv~>lb3YzF?ngxL!Cf zd^hnI|Klu_Wb0xG4*g-Y0Z=~ldRO~xf)v2M^#O0f$OkJo7??qH3CG|o2Weiz|C{C| zbYM1YfHMf9ROX3}PSdSr(T%{6*N)zOD<%c};ev?!1g}n9&Ve@G3H%yh`_FggH3_*2 zhZcwEsMz7a$oA_V5M>jrij5bL`^62CRWdIUVW3lfI4V> zX<1)nf2!o1fjO|D$1>}^ZFz%A^Zd`QQ49Ps@Dsnf0UQTxLEz?;QAO<*vV*sedSi>B zh>pW0BeYXSM62`T@1o$92tCLFqC!UdB+fBnjR+mJ9}%8x+8gh-Kj_S2Rm+5ZF4tNx zr`2MoJ$X@bqNTij7l?o>+$c1TZCPES4O%V#1@bfwaPXX`-#NJVH+M&Sw4U_%X9 zq7v&-*rb*%7Othu=T|)rmWo(faKb5oO9P)CJTFNZedKi0^##;mW}hD83!yB4#9WWy>v#N_N#kt-64Gj-&T%eZT?lQ-;8g-`9XTTn zuv#nu3^%UZ3dY4Y(i~prwt+}~@%Z#WdulwF$WCJ@|$+=Te<*u!eG=*5E1cBcc{bs{<22 zfx4(6Zxf!X^Z;ZMf-ud0oXHd%0nEv>sPiU58Se?1$>ZYyHMNaV1(Zs@B>86Z=&Ady^+>eu|d;S(Pu| z-tpoZ+H>O0`WZ&;pKqpo%)yWW(&^~sx-fbaNU2UYY(Hi{OsI6vty8FQsMS->L`tv-}mzXu?Cf%9~Pki3?d?MLB;@j&<>D0 zxEOfg{STrl=hh4RKwk}JJ7J2WL~$@VhqC~KRbadwsWGs>&0M(>^hXcj=*-K-5wG4~ z1|R#lJ`%}NQZWX@$dJb@XV?oxk4=S%;GGfTO{T{+8qaXHvj7*K6&nny4M*!w&pvZd zH!-&c@>c8L=B(~qE+Ec5o^5}{T!5oego2R#$i>7toDNt#Wkh~c3eL`RlQ#8e-Hxu3 zuwDmLnF*5hpf1sLu2!$VXNc2kfM@Y0Jub;;q9%aA>F=~yNy>oYy8;}}7qE+JS`H&} z!eh|8g{r=OT2z0iWn^2)&OV4)o3b9+VTf=>7^DlVZ<6l13g)eHCd^C6tNINb#&D|= z^Nl{Z_jZG$u>yYL!m9|=$2;d8z(N=?gO~m}?-YXKPIM5(p zGeHTqby8LP2K-yGVaU1VH4qB%S`x*9u4<0_M)NY`Mp?hmg%L5SkU#T3r6Pc{1_oop zDpKUWH(2fm7n5xd2}HUJ^vVKuEH-#*P906(L4bX7=(H2Q^z$B9K!y@5z3CCmdIuUQ zj&8DYu+VWMe{QaS{eQiQpJ43w`%4hnuEBT*)O22>E-IfXVan?0?>|R8rWBN zv?MYclai%$m!=h+9E`NY7f7#lfvX9$4H%;tJFC77zwpoA9x}d724v15 zF}rcE3P6UA&?v7iXIRd9EcvQ{Iq^&?Fl^lkdQOC2 zrPr8tx5@}X;}FisxPKQ(QoYZ}aRO|Oe>0a!0m;;Qb*#yU+ukuW+AiQy|Nrgh{GXuZ zqtdKL^`CGap}(h>gfx6+dUip{L=g3dxILKJGIKNsj_B0U>D(gpVk#qscpd3QqLU~; zfm`}wxmz8xAq5${>B-BC*6#h10A2ltCM5eqE1+rakA<@JZJ1+5%&u=*7Ucf8t594Agra>PFc2CefvRpS1e9X z`(*wpJgPDV`;U}oiKCWGb;w}oSokz@hsjc3Y$>8G34gXy&uPoau_ z!KT?4ZS!_FKQ0&^>=hX}ScniEEg!uz(h2Fz)TE==nD}NMQ!>?Ycd!!#$Y-fxHl+pw@BG$2 z3xx$dsQjzQ=kSlp03_2v+v(^b?6;Qq;;K11wPg~)6-go_A~Xs%J{(_TX&t+3p#Q%&7?4~ZEQRr` ziZ`9%{8r>-b4qD#Bup_kb!4?`iKrD1bV?Itz1_KdZ(@v{oKfdN2obR`M+W?^==L;W zZF{5%eWjXrEC7y4&G}QOv5^Y_f}{AU~@KM@|tW7PZNqyPN*LYRfabvdwUYj~%CpC%s31aHW4h<%`|o{|_L4@B#$= z2$)y@M~DS_3zU^)htq1|ZB*&sZ3T2W>K_kH*VUtFEctNzNt!Y_hCIjP-m}8KGyAk4<-r{Vw2AKOb^ zz7!s2*-|9}XTi8SyTYXuZMTvoRv zp?|I^>-A=2X@OUR&`=d~qtzd&L2~D5r<8@$i}D8)aub+xYJe7?A%@UZaGoJ%IO|x) zemhg82u${CTfv=kCW_f$g8R4@=X>tluO{)xtR?_sS9^0be08&)!$?oB9B7c6--EB} z%)sa7>CD$kv8~t@vfTc&P0N%2a|C|+ebGzn@qE4gjVO9eFwUR`RLJk|V&uV84I5aW z<5j(opc_fzGQzb|Jx9n;!gL{HGXb{k0~Tnl)H_>TeFKSGOL8-TbE8XZ|+49hWlato}W+bCjtCuTmv9yC(CJLGt; z>dL9^cW&hDrKCKSY8&Usou{IYQ?y|Tp9trL3c9vK7~31qSkI;OrnSBt(`BA`>7{&~ zIo#|aOPw=q1QRL%TFFZQ?5v+~6%YT4oja}Ur!rZ#yNhMv?ebTSbM#>P=d>VE7d?}n zrGLLnq&!(cE6}wwD0vKHxK!ByMu7uMfNdWI=vY4pPQfV`Se6SN74mM>tij=IEs}^T z40qo3uHyTa;xX$Ay-Hi}^~t54Zh9`E0~_u2MC$d3nN#oj z;nNZN`S^!=-^^ke+{1n~fEz8M<1x*=pSAD`qII3SyMXQ^3Ej_o-qCeHkLI@R-2Dta z_lIZQ!FR*Nd<6vnUHz_n!Q;6a*=OCa@Kgtj1L{7IV`c*M{ndWHok9Hm-z?Ddg6}AJps0yc85=`JCXKukpxIbO8p*Q})UYtQVI8_t#K9e3L#!a?rjV@|DF{`fT{ zazy>kIT?iAGtKpo8VeiHHk(B7{N3NZa311>4C*rwxyjao^7^#>c0a%7&oUA8S{-h3 zfu`fQ9`?-yh^}<64!b}%>2z&5dEp2>&D+ME@2Y>1#ItclPA~iW)^&e(o2=%usZ|wJ z&uvQ}%b3<58pgG&x^4w(2D`rvO@77^GQ(iB+I)&kg;o~b#0n58Sobj z%mF>@_<<~aKf3&)kO)1h)1beh1xn-myH93|{fOtf4wX)1sVaXyUflXmvR@lWN;!zk zy@WszbEahPJsQEXGJr^CFmQ+jF}!E#p!!?%__{!6D0@$22Z&0_k`%bCOpJax_O zwl7NdX2l*}zOAuuKsAB!$gVC9i{}ZNd7H0$Abl&!!y?%34Ny79wWno?xX?<&0 zybH46MGVGMUF_lI0Wcotw+BgV91R}uYO=Xsj+TZOsHNZyKFty;Y9x4mUJG?UyNCV) zA>J2?6TkL=0H`_2t4?<9$->qv*(&H)h1_;z0qz6BlW6VMP;*2nkffvzZvGtK&V~_A(gJ3l8Rv;CaJ$|{$=ZA%<&&Zy4CmgjrT@~AWSw!gYA;O~; zxZcFZj9vDr1y0c?6@5i1&;*u0VSCquh2`T_y=J@hm~{imieOUg*e%87*1zOx*!H>! zTu|R#UoWIOIRKemr`AgWEr~DmVQ_OhWs)SO7>r%d+7N6Gv(^6w!^kMBf382h6sF}_ zG1cK#Kb=LY!QnFA;!IJcz}Od3*t$-(X$i*xQ(!j8)22km`w}?UiVM@ySM~Dwsm(;z7?jMg^&5!1*_m~K_ z6GTySOb-h$Z~n80gal&W8UE*xP4h?K&KFTxd(sE?e<~R(KkTIZW{bgC=5`|f{f)QC zM0Un+RwAU{p9oGEsnav-osYCKOxGV*0uNFNia=e;JL89Ws92ku)4U|q}_mtEt@_OrK20J$c4kg_D zYPD6YUxptGR%S63dV;PVjJF_e0?p1fGZ)Z&e2v(3jR0dTD8dZt7FKl+E)+-1wI=o& zUA(E~?DfNKRzquN1#^q21p}k8@$rSj87hx9#|z%JvRH)o=>g~G*SYU z^$laf>`@pn-po9y7Oe%V2{fGQ$jHIle6$I=zlN2)5}VmFmK;7=)uu(jppoQCs_L8Xam(aRn+2OV$gvy*H`kpy&fPPZm0&sneJcr$|0Yw!rMyM9ZuB|=r)5e(ENFlmt zSuoXrwCW;QpF^?2Zw`8CvCRPB?>mK=`t1IqYq9w(hB2Y`VoAASg3aZw? zwB5XSlj>g%Xr#q#N;KB3F-(}<#{T?HC}tUM(b@xZO3`wvmIWS!2C;&8W^GYG^j79~ z_8V$he7F6@^b}Z;Z>POYg(()him~0px`4qR`iktpS4^g8UbBU1N>&c3Hu~(RBa4k) zz!;ke)hLXJgIneBFNd$=@W6D*mWWm>G~RYfhzaNt8yD-}p0pctK9HkIhVlHeux+Tg zxZJ_T)G7bQJK!iaX=dfP(ZqHYJI#WP8W{6x72wX@?Gvr-WHuL?02(Ww4Qh+ zbu28|xsOj|K?75B_UY36A8w6hO4|7?79kR@YMEzJU2YOI8{gpzfb-D zmu>%(SqR0E*4RG$=WY6>DBPwbR`wRYMQUz;3H&ez?6{qafrg*yC9B+9dz@F)TnftB zWfiId?A4?X+sgbAxPaTLoJ?qRMnA3W)kcEfPltZKgba7xg{cw!hk5@XwR@np*=Ue< z`Mb}(QeIo(v1-RsYjWh%&MI3Pl8o9SXk~g9#FfH?=g+vdS9T+rmfz=Zhl;X3b0#-=Kx#dxVy{n<*4vd+v3I&0>{K2WOp&*nN3=}h{ssy|VY|0=5 z4DqY|=|h7KBG{^1jJr1-DtxBr-D|<7IM3y9oRcn*y;vHnShZ!FMr67jt4HUGWI*Y$t9fePvudCQN(#dX~GlXQ=#G?RdXfE%Td z*~i9oqm0^BF6jjB+mYe_*>iDmW|RSzw7psPXQziw*O8z~xB6!gbkuZbjZCTuvFdeMLDg++wpdIY_0h zf~6tEF&N`KA2Z(jjR2Z|`{>%eDO=~*IhKpAA}9&CD8f)hCb52m%8USy><2#&b@`j- zvphjvk!)oykERr`f5>Bw6(!SwKKrGEv0RmBSuuIeKYKEOQCDSy`>Vpr{lC0=7phH1 z#s9OKLyz*7!e%}@>}JnS?!CC2cGI^J$%!eHc-s3sfYIl#d$|R(P_v$MF z4+G4;oV&eTe*qO*W@H`d^|oHBM`tAeH+&Z2A}hE#jqLR~Z&Jmw@L@ZGj{S1G@bT^j zr}SIEy8rlg{EZc=PdI-F_$@Hs_gTYkpvFDv!4o_2q^0_JB0J9#n#@*9I>qX4@j#@*#C7X>Pe>3gpb;OS5V z<0MHDzYy~ZQ3N^@?3a<<!5=kaTE0sxtCM(gZ#p#dple%DQ4#b zeVQKqwb?G{tT}aR@*4F>?`sGZJVs4B=*`(}rtUIHyPkd}xTPZ)A zKVOe^6BvXNKxi;R(e!zsq1lnz7Gpfo8K)JBl9ED$-Dl@f(KVA6DfkgG+ztYL+kRK^bXdNmjAU3k`qz?r7hKAf^4Xm}EcSzMo zkNPCJlRR#@*6)`jNGx;L`lWI}lBfMe7KVNl^Ky`T;o6(rns?63WwF=Pxy9P(@;LD9vt;vyg&)L&(^tSAIc@{j;eka|7Gy(h;xcAuZ6R!_Q%@3y5g^|RGBs{4 zB%8)|*#t%Uq@ZR-K^*h<$5=aSYiH1n? z!R%{2C(g@?)@-fugL_}#NtG&gsd}ub!=%Puzrmc=amc@!6*|eCqLn0h^AuA!DqaI9hTFd ziUSdbxF%}6uxRYC5?8ClQtNn(eO~A|N*&RHaHt^I?82Gbi3~v^g5H;0&I&b z?%nLHRmXI`xGO512>WeAb{xaKD2J9DJHu3zR=<25i^^Z9Fow^)zl@qXI~Xm~5lk`S zxx)oZ_Um1=#&by-U2L6J$0+|nANv+7B{x8uB|wAfi#F|>&z4x6+3m}JJftgvcQ-CG zi(d*&%|vv^VADh#?0(#T|LgnVdeIe=7?!?crT06WQx|+c^g7|bPQmkApC;CeM}zM! zbKiLDc&#nDt+`##p{IbYX*YYKIioo>I@Y8*sX>tx^Iz5Ri(6B(p(9K6lgFEXSz)(5 zOKQSRyj%ja4M(@TRr4urELDYV&A$8M1yDr8Lkp~spLar68btPGaE-8K? z!Sz$`KZzSbn)Ye`do%$*oj)WX`L{si9!!bcJqkZ;AtIZ=BT6fCu$4w3O`Z*}coTlH z5QcGy~C_P3%xWlUYk+n#yg0;>RSA6-)>(z$T)rE|%N)RWGDf7XBtr3Y^^mg$K zx@n6YlFI*uqI{uxpFSTE#Pj|Mq=aX>^rmS4`=t)~Oi{!&2ejguhe&UWNYO3vYR)wI z5VxHxjillCochJ^Ht=_$Jq0h3-o0%Oc4JJjSN_0#=hEfE*N}r-go>n7)cw0FN71@# z08>Gf2LA;UcYDrN7YmV3*k3ycL=5slxQZn4F7a7G>9WH+iP&W~tCO2G+g0fi?bClDIEjAD37`0g=w~%(K9X&VV#QE3L zYj?~dIzf1tHi4rPGdwI;yN^@16-g+z`K6yAuuo_SrM}Y^=w5eM)qE4`0EJ1@E9p&7 zezMH!c;?sOGCi``Ms0UKOD(htz3#DF4!0N{(zn7ZfWd8hS>_F3MehgewXBK0$VM-@ z)502Myu?U6Qe>FLr}6|muiBk8&?-Nql8_W$Y?NB%4#3>j`fD8?zcP?(T0LEk&rxy( zV;vqtUuWFwJ?=5PMYJ~)sy2q7A&0^)SR(hZL*A4mkhdnd2kSC`(MO7LCj zq?qb%O%j4TGG?93f!NA^w}qy7*__X{TS*aK@u|?&7}XK4-;Xd-=QTu`(dt~w^ek@a zSY*c^K2<(%BFDIQ1M|!b4?V2?J(sEY)lXyEv+=i^9VT%P+*$%9Qf+b9gQt1=vyC0a zLcf2OpJusAf!<_j5?O~;XIxTYM^L2ospU&ZUz0Z)%}Ay9dxK84zzPQM!xM#-9m_gv zNcWs(Fk9;ExZzwMT)Ld-CGw*WP1OWlym;XXxx$x3cF=E@c+Uw5meG0=bch@M9A01(; zFEl*mb+$3O`rbC5aw`884?9dJ3FFtU^CZEbaNX1O0d7aa<5QR^hsRkw0~_xn2rkOA zIq!RW894vob^a67_gX9UM$)p8#VR+Wb~XEZ6t?-v&MT9cmlta`BJMr;SWnh&%vBRh z5py~{vX3#;^sha-;9+H2q#D;fUOU$}syWk5S>8z~dG%%#&#VzrZ>&kqpmzBkTz!MF zAXkjiS4pTFoVW?Ccc3A+eF^GZ45@%oTiaHfHo|H3mn!@YhNRUEhHdC6&0J|9q9l1z zGr!msX9h4MmUr%sc(y0h2oLXydDB1?Ux)sBree5Q+~wBO>HhN0W!;@r1Df4Y7S}lK z|L(rOoxTZIFcbD8pdctL>sUj7l zIZjS-GR&=cI<(-jTrZSVcopTbS+zgKw6Q)!D>+!S`_`>lg}6V{^qfMd77r8#&8wg9 zXG*n_G;WvX@CVo0)@Z7T1h2lH;MWvFO}ydf%ue{87v1QgkUnH?~U1kTLeXE9PK=k~S?h$i14|CJD zN1>-Ruf21OsJg+{L4&=bJ*rQ~xI=i!>|X1*x_5Zi+oqOyy_A+32LZxxjDv!uIeO$r zUJM+q6MkKZ!MR+PtOz2wB$xL={}y0_uMqCspFIEXU&@#aT>K>&Gyd2{`beDA=86>^ z0kLjKG-jDUa47u!DL?*Fq1sYjKXKj{yA9K(ZDYF^CfFA)c@8QLh^@KlGlgAdUy!w! zG!Rt$@DvhJMMoj)ytgKF6E|*b%c5JjBr{E!5}u zIlkKCH}sLu+NkVqQ?p%6dEK4NCJ z_GUAF6}(E%R>Fi}g~I364MPa4>R*%C%r_v6KXFI4Mu5)?MnLtFTAT9@WeACVe<{&Y zJ}Q)YO58)N{2lHSwL?4zIrPh>y)Y3JETE5cuz41O(c15w91uwXN6DE*7lTwbLTX3Yj)U{2HN_cH<}n(eFz?A@8*SZ=+;4nc32V9+jSlK)?M))xX0+ zpNn6~wVXJ|1)Nk%^2csR=ziZh_#0KWe>l!k7JA<*g}N!{y^tXx7|nC1)8@h=nd3u) zM5CFH&P&Q>Ir(h^bnJzrsbUJd9HwRdK<0!Ky2iYqLR*=-j* zsyDO62>CftsGi#%8Iv{M9KA9(XHHoi8I?GgzaQ9+CB0^4W9;z zk>&fqHv`i;ztl_MF$n7-w#?2n_tn|F%AVnCFLrZ;wOc%;GL`1o`ig!rXl21+slXW} zEiF+LlU`K*96Xlv#`0mD;;|mPS%*Muy4IdE7yCQ8Q9x_-;3gn$X3OKYV5vofkdn5SkgBP?!gf5^Na8j(b zW>>Ielx+FV1bKIR(iLeX(OQBG{z>rABqc;tBH+2{SqwNEI-#pvbX^9m+*U9>a0;ZG zdM!8<^~_qEtaY|Wol{-r>(nV}1iCF|KqZnZ0Q1>;BsAB!{uJ$JoDV|u(yy`ZU&Wlu zD-%?zDHaFqZ;cJL?`K|h8+wr1&+l;k`;GT?et7~ z()UB_#NPcGH!f`K^5n_%vjBevc5g3EuZuJ+NQs@%3+viEQ~34fC=O-p*Al45Ghug6 zl9A3X{eu(9#{ofd&>=oADUyd)Z*g(6+l6}Bx;!wgkIFsZbAP|UX%~)y zx+;Q`j|-G9ZZHKAAQ%Ox6gTcTNbeNeeL_l=f*ABUCY3$YksdOSnS&TJ-r^q*DWl#a zBjHuf)3p~2{duS5Irt6e^EbQFAOVSdX^pWJoMqp%Ou0CDR!Zox3MssFP!XZWcJX8x z0Jp{*#gVTu4-4|GEaP#GOFWg8C-ZwLX=*;XCwbbB3jX?b5_2&P@cg=P4PQ?&TNoRd zZMxpNDQjP}8Ccr6vk%k!_O&|(i*J`TCZ+Jr*#s3|VgG*O(?OYbe+#6_kMRrmw&v;} zh%@=Ve<|Iw@tFoK+=ov4S&b%A;$Po<2lVbKV$r?B5!jbcKQnzS6SRurha#Un(!QltP9>A{|Z%38}hvT)C-YZ&avj`I75o;LH97 z->Xe8S1*pHbW(gb5Sp!>Tdl3h3$@AeoBP@N#XDDAt4aG2(O>!*lTD;@3J_!d)(DIA zu0pSP^X_!}Cu6a%Dub1;>=qQAf*-lC zHIWvi&iiqB$O#=MJM*5z(kXsqes0hrVZ`AvCF6P6rIT`r@~-Qei9t2wsXB8f02Nv{ zXWyN|+ z>Xa@v38pEHBV$X`eEWrS|KzLUjF;t_{LZ_zPt>(bkrEQrAU?GMsB@lN1+yknVZrU4 z;k=Zu>s{=33RVo23*3l6;KNXUfSCeiP228bbKp9{pF|nd%8LjaCQGgl zY2OW|n%1a6Q)kie^;h;}Bsu)+pCkc@s7@(@pfKO?9w(zw`e83@O+3xIDb>DihNZ{3 z4Ci#M(>WdfX&63zyVE00pMIm7Y$|FKG#)~|J+Rna>+svqFJr9w_SB89qO4kFJF*|S zE!n28%T#gq0F42}eUt#{f@z&kN8x^@DCc8NH- zo>$#i@wD%r_O6l=zCiIr4bN6y2mWo~bD&6b`(ZMAl7O^r=qRM$Bc`9gVl zEf))ZFWF5M{tTXeP)qn8pL0wM(+z7|gkNqHz-=fZD7+XRLp-mo$miU9(eujJ_gFd% z7s>{UT5m32+JW){q{OrZ8Tf86sMXb)QvHYHbE>rT0A9!2$d0HdT!m_#1A*l7sYHI| zI<@LlY&2j84GZo=s9sN^2UU(-IO#))==^q+E2U^;%s^xha|`Rqo}0ysIq%P0a*903 zp@MTF7*)ODZ<>-9LP#pu(F3~j50?qRfBVJ98BUR>9hNmHFQK9RZO|=Pl4|DQ#_@A& z%;fb)jG3bB+a2d;DR;m4<5jiKb2jw zm=XCKgB|uG6?-A<3L~8PU>_?NDkXA$DhcgK5X$<@!0X=Quul|F;}UsvHRs1_O_z+< z#f$IShP zd(R?dW8_a(pZ}OXQ8}k9h+F>}-kcBEGDH3)G~P6CIU1=45J_A7#(I>Drc9qS4!2XV z>6(*YAtn45lC* zw$}cfgcRv%#~o296a)gEvRe@XfWa^dde^P?ncw}u?!Dut$T^Q$+PHchlawoFl>CG< z3pb6!+*m%M%RdY8c``lu{~l;h%-7iO>?G{txGjy3Fb!r6#g!Z=>3g*7u22}#P@dxZbRk#Qc8cq%nD>od4z-4`S z!=Q3myvP=hNzDr?9e7kXg|KLvW2>Bj=o}9L)p=Hu;7caCaKxuxzW2BLgt{KPSaNd2 z@UYNO)K;U&@Gcu~L2`TkrwW$)7kw8Uw(Xz%7q0Y`IIDG7s-!_;=`N&Ub~EG@H-gSbEHH4BU)ge;c}DqBsHc1>O5a5Ym0145 z4ib;Nwg58rNB#ZTRVKzg=sJq!-MsRSo)Z!PzWRTBM}1%VfE%HxEsYHEu@g*$iQm-j@8TScAy8saL0H;izUozrS6mPN%n>fD8p5?>^ z8EE^2=cN3{iPFtlSwZPugnSa~dM~Uuy4{YXuq}oIJMt=!HgK zMNs7y=9_&i3|^&EZ}ze=x2Y5$ccocFDj14mR8aB4oSAX2qoO(~{pC(^T-*(KI1xJ~ zuFDJun&NAY=S;DMZ?}P(q|Ta>idrrAcg3%wfiL}usHLBk-6M;*czBSXV%$99eO>(o zKCuGQK2#WR5?xXB^wduBVX!MHos__Z&)kh3yOcc&;}s19VeHz@^Pj_#Qm3<$r2F2A z^=vR$daUzB+GXes2rBjI};tWxZ2jzlUzAq(7)KtB*=Uf=O^AZrbpYdnA~jFZXoO8SX40 z@ZIcwOoHoS=%}gO6Jr0CYAunGuXd+Ik~eqHu02RlUv2v*M79QX8uX<*f|So^?~GsN zrkj+59Q6X~$s9QNwgvPl$%8HCQ09qE@)kaf>NT)X7{=a z54AYxTapBoFmz1d=XmQKpZpD`tt2=WfBIsD2v~Q$?`mK0Juhwt`1r4b&uRsOw45AL zeQ#UT6dziEkR-Ck^V?)M;mv}Gd!>a0@(N0HIf^+(ZKhWUjV@UcMDS{Fnp(mbW0-q( z#djtE-4wzaBUrsm_?=kh6j{71sUBd=qUF3(UT3Dpoax0fQ<9;fp_|4VHt&Rpn3!+m z9|=zb1Rnox34kjkjH6FH#bQC9dCKAJWma{ zwJ|urTppK2duNN?CTcRZCS6d5p{d&ry{KhK8lF0E5;OWWx`AW<^~Cjdx>#uSdS1_e zjDsUNUqa7H{BH>HtBH>W+>?~wWq+Z>QNLCW{Sn6Agt~T@0jtB zaxr%+7>-CUG~X!bW8Y3hx>tSt_c=a+|Hg@uKmF(1l(MFi^JQk_M?H$;X-s-gA0V7M zlCP9R0A9xuVIWPjBW@}_?w&5wxNX%{yj<-wNinO^5?vf-8jB^=&#w~4D->1LM1J4Z zDayR3t9O0nq?hr4u@G{IJZW-uvaZ@X?x--+M8i-!xp1Gh-_hXy^Cw?f#|jA|`6iFU z-k7E{JaiG-`(%QEdr7k4ciRsD`#eOTv`(}<@h>H==$v#NfOWbm$B1tWCouI!3L1X~ z(Ky?`Bz!G2oMXa*MDkT}>{r?ha%c-ih$joT|3)9wTJ&JLNJCWt2{!eb;#Wt{VH91T_{HWuM~Ib^Q2e>4)DC zpJbt=77I69QYMVpY}pVhT;;h~UI(BW6m&ePA6vgc3k_I?Zvs3>Ey}a-{Z6Epa*7k; z>IVMfpPxb)xb6Kr&A#!y{xf*)yPj~$gKPXjT+bx1ZOfnMefU`E@NDC~?X(Vqz=!aw zBdH%=#53AivD8$Igy4PrLcx%~AA*7Npl>G!J9kq4PA(0qsm#phQW=QE;b^6$VuS*( z9%V?qx#Spa_SmZta^1|jh4^H>=w_`dRdYjU;8#X9@_X_=;qf#B9zbxY>4e7g5)tD^!vXL$ zhDV}rUedDaD3SO51vqLKFP;xc<=#{lDXg!3?)klBiCBn40+MI>cXPl9>v2FIiX|~HD{ljDC@t;_2)gS961D)O*^g(!9Ay>Ix@HWgG5cu+HdOquccVy zfOpf%n}3Nqjv;xk1WWVUgYC<>hVrm31Ev@kt_`XGY!zyJ%M~m9>Fp(XtJT;-zJYrc z_Rt!RQk5bf`U#4NI?Fg>iLX7m*Vn>So%}}rcEN!ft!__@qWOR%B_LeVH zk_rTK$<~r?(VRyBLVY7cK@oa#SnBn=3KUA93rslzsNQA8-PsDs4k2m&My%8A%ny!X zLM}Q1%I!VH73}Y@ACtZY!MmNJ?X*+ewm?Da3utAR{05aLFR!ewC*!_tP=;wQ-1Df= zV7~yQLR8>+cx4{}XagiZ(VbqnXWBpGr0Y7Dnf=L58`!sTzn%k1$3+K~8{J}6))pUG zJ3=46CK#0CFpF$YMotd-{d@77n0NDMiSuC{BKK~O>7o(mq4AMRhKNzoejc6;izv@< z&4~gBj`R}oxG2#$5NqeEuEeSEKuSgfndfA~y;htGhZizIi?8OOE zmakf!Z?uoP8E7KIww#4PVfR=%@AjMW+wXr{f_DP8E1UTK>?N%WyA7Q@Zb#p`sf$m= zd5qA}WucT@>?dpbMkyB?L@B^v1xV7Jy}ez{*;0xv2EFwoG03Y$Ch(X=<=sD5Z9n~4 zGzf@Hb`4TKLzfq?wL>_J(mCu5ncJ6&&J%eOalnzvC{c>sGUrldVq9AK!fx>a+tBFN zA#!d=zhiGU3$RA%jdNdvk@2jXpoC-Q9}^EvOv3r>((#`>ao=#{k)fgT4|@6i`}41i zDobgqNPEyaz{B4&RJ)%-R-@Ao0@8T;ZsfAzt zk$N?uNg4N*!&dIKA=izEjdEtPfTB2+l{dvWpb0$di35qbXgHs>E^~w09`lW$m7=1pfHni) z`+z`u>OS|EuMC*lF;FB7dFN}+J~XNP`1nL8;8~|Ts<9j+D3--FJdfS>8W-vz3-Q7g zbvst&ZO(P5g3d9&z~@mbU=e?%6x24J&b+-;>Ao#}(_-yna1IbmJR&7ElR!B?L+MQ(AG{`$cCn3z|s=1x=sn&CuT*xwe71g-39OMlOd3AmIY0%$-#m!~bW zOq>QAVC$mbQD6s3iDqxyKqa6bmldtb@SH<4@~jk|i2*@GQ$@d|qaFN%b!#&ge+=17 z0oK3NATF_E!q6U>O+!_()_PwBtvehy;8A08HSUHNY3dbx^dD=_Q2TOn})Ygz;Mzb!V3 zT<_^=qlDgStH%6Ik@HZwKjf1{JQc`NLKLZwA;FT(pe}W)8_3u`E;(#U7*hYK3B?Bh z?pR`wJ|JEwKK0AB&)*A^;Lvxv{bu7nwE|}V0vQw$!DFjuCwue8{q@}i z1{mZO-f>nBz363Ad}D4NoKx>$`s_o|Yd?UnhqudF&F`6phdb9F^@!D9@7bxUCBVao_}I~SUvpx#j7CrY&M8PC zuWIFjwxN!4+?nrp3Eta2nX9qM$hlc@fl)K-!ao&KBJ-W7u{uLNv7I^sF}Be|VfvFv z#>{uKlEIQ>^JVskIe&oqzNBf<92n=y%qeoBC@hY98vsHUX#s;2YP=NJ+IFkOH?VBSnTztj$jF z%zj+sqn?_bTr`z|qDvI6UhFnPunlk>37LKEs2XQN!w9_y=>E`LhA)!QdCO5$@GxZD)9#C~arF`%we!4F6Cr9PLs%!Y%Hyrg8vP?`kySVwzX4 zn&p4%uLZ~2!;!ZlOCtCDfwcl}3ry-y)7a`u2-q4gj1p5T##3k9)@=(lN^|>N^{&4Y z=p_Ye*^o`ItrKWgS|SSHlx(JK-kkmH$9R1{L+*LdBKzu9`D+nzaa@f`=1euA20IZY zEsIXDKC(Gq%~ZdvMw5lhNb1qu_539vH&cFFt&92(13R8769ABv?CvrH6`X(XrnLhR zsId`nmf3eF(g|}Z0%A)m%pDhwTs8}TdBy-9@o|Hswzc*iQPeyizE*VW!`l=_&wM+U_&?}$vr39*+k)UK!dtHc zGS>$KK!Sz%JUoNuAOx?FU{>0rhbTm+D9r!drU^>w;KKJbAFj57Jgk&sZ8%_N{ zIJ*gun&B(T!@A2yKv>a6BK>+bcT7K1aX|>>vbVPtSgpmoq!Vf0;&Ty zGC*K|c>U+en~&qWow)eKwzE$2;1neU0f}Ov;%VdQ2YhljV-gTQ>NvhYc4j_){TnUP zA#%s@G*XDCWxXCjzc-F*#yYnY^&Ban^K?%Spa*npX z-d^{XI&Sjj=5x9bx|~@e>fZH#2-E&Hy=g=yqY#9eDpZ4*;~aN^JFjHjJ~wDz0`Qt| zGVmtT0u~e9=<)jic+YN~J~dea)D~&%q5cA=)HCwGJKvu5OqRTW>!60erhw~yo2@&Mwe5URv0 z+avXT({%b9x*P5#t!~zOjcFzRE^ThzIuXtc1XZ z5N6?9M-F4o`P3RYIW&!GTW*EaqI0(dkmI|1@;jG0^wJ9ZCawEYgiL;LEIU|EJ0fPw zhuCmskL*Z(vKp<~2ZYz|{MqtBm1#rKyqUvdU&{MQ77VnXKZmTf0do29#>Nhls5PKgw%&wX1@G`W>8T-LS^jWYlXply0iH zs+s}o4gfp=N-cx+uQ>lG#d5+TI&%Ze5W4wnE$IaBpxc^gNpU_Lp(Edgh4Ei-h@}jY zeV|8)_+YS~!{0p%d)-_F5F)qp$Y-LRlDh?rBg@A*F0H$`Kv9>5BHw6WTfC-Z>8S6@ zxBv8tsiJ)SX;jns?n`%1GD*o>HJkD_g6JV7T3s(Uy1hCZoW4)=x~>Kkacp`+;LgT{ zzIcwnnkKC-;Z<@KAoQ(=7Ets^K;0*5Y>wP+2sF2#JOwTWL5)J8ZF;Ezy_;vuRTnPN zk77A>(vzGC0A!B_p;VW(TQ$O|Q=308rVwANef0r^Os@1^BeeFS<02pn6x=><=DCy*Oh4crot0p2qfa2j1#M;9o zLU9{wrg`0aCIrAW!IGk#F|Nk)VQo$sRKH+TB?b}*mDVh0?IKqj&V5MKF@{-jlnGp_0urHGh8y@;w$^XO+l8rX}{*(C2H!Obe zIPD!s4rX24x(8v15Huvgq9!jE42nD2K=e>cHAV3+_=rgUt%i2#^&mE5;m9*se;~;c*e5S(EQg%Q$+0Cz z^1Yz~*{K5wA}aO?PuSI!>j7hy@q8`yz=bX?b3=SGpMNKOCG`Bou=ym5f@)G;kT&gh zU(x5)zPb0FM|D=V(}~y800#0irJuJln^w7!mBzexm~qDZt}@d z6!5%2pb$((hykd&+9)YwtL~t^oGWzhPY>!Z@{0wz!P^!$&=>~OzUyio(+}t$JJIn~ zeIaHWQ#rw3JOACuZ4%Vh5Yd?hTeQSve<1#ul*a%Fs_#QjU;3a*AE9pt>av!t z_4DRk?1v%^z_Gy!s0frm(Wwb^zw7*QKsC{^KommALqBE1pL*Wkdo9$(=Ox1jIvXk5 za$=yLf6&0}i}L%Y?xndvf1m?exhy_>r(|Gm!$li|R~Tv!R9dH4AwDM|c7*I47}Lmj zK*$D=R&lCmQJf-iQ0qhVc?IrPV{`Q3 zb$eOgcACV~QUhQe9sL6-pAte%{i?^tLpznvY4kU$Soy;ze<|uV&}iu|wQGE?um}Cs z5~#12$MW1EqDC2HRkM}kQ&YXaK2%xOJrQfpj>?9Pjzw6D&9!{RuK|%u>Z+a3Uy_&1 z4@edPHcGXxl$En<9ti-bs=Ya%s}ZTe*H)P*5~lPoK8X^p$T2#`O0DcLi*+Pqdlno! zxn8HByIzdaDw(ec?_yHbo*+Yb;)xG?eP3WzoJ0OKI!iVTczff$`Jnr|~!(?NpWkT^C zkNxiR2wGofTm-YveQc;iRqJX%ZgvGF)~+%jmt%)Q#<7^os)kj%??<*36CXmJ8hz99 z{~qKj4)J>Z!HnXaJVnIgb-4mhmrGD+8Q^f4mYLZYx5TqsL}s`u!tTrJXnMI#Pw*Sk zBeW!kr4k2n5YutK$zh<_2v&V)*ZrFh@onAbHu-vkFQ)5c9XaB^w|V2apSh#?j;K9j zxF33o%}M(=ZM0t0Fj^UkS1i8$vz$qxz8bovw@maYT@;{Np*9T0p)w@$n&08GEkmroFu53IIJ}U zTG03yTgGk&R)taXoAo^o_?% zSD*O1-LXa&p_DunIp+O%-R+TLAG50gUEr@5!~2u_U98y>~2~e1qk((t9ysIu%=sVf|nFLKJ5EaWkdYD?GP`j^~v@ z>JAl`-$KZhe4oq%H{zEW65AJHhQ;*sShq!J#YBRH0f9i5xz!tr2DgVPEerRnlO~E_ zWsccku(LO6md;y*Gck?+bu>WzmcL9F1n+pCEl(b}9!1A;c2DKx{;@}`HrLs#zma^8 zg)lbpFEZ`yj~C)jX+D^!W^zNV&e+x9{7O#tdm~v-QC!T8+BZx3;D=hDe7!%}jf;Sa zX0h=-q01gGtimcOcZFikzLMTPMd71$4-#RDq^-of3bxK~T$4cj7{VO@Rg3ZY1)GYO zvE8fSWnz0tG;&lUFP-sXc%a+Ykc%WwBBHF20kjjJ3)0AuXZHAml9J=ddgjze8Xtay zz|Rj4o!SQl);Vw9IKIQ@1(#Wja7$HRpogm|e!?SsClOf;Ge-hV1FM#0FRV+94Br!u z%2Va!KdUS%zZJp_6GU45R*N_68D&1s8@+T(9D~%Sl6IP4ME_^eI@yDkeB9;o4-CK6 zKEhembW$quPer8VBMeuI<<46nBWj@i4IRS;eI(gf=K^x56^n1v42y9;(0HbGydJ^{ zx-+j&!8f9YNL-UNxtdgz9=8$kDzpo9E4=y|VaDi7q;Vf^sD0jllM{(R8y#pNaG~Gy zjy3n>Bb)#ZyYb`t_TfX={EY4Y^7;<13$AbT^WQDDpCP{gkZ_M(Lh?)>@>kd`E!buB zY&2mTzhz7|+vwxm=b?{!=uEq%S68mU=|<2voXJ>-SqB0ktIEr_C&kh_HOsUET_>s- zW)?Et<_|scu(D@mi5Uhm>ZM>w9xY@-UB|gnq3JjN*v`TSvPUEJgG0N@LOI*cNn2>i zHFyNA1@u{|P2kzxJSjKo>?pj0VQDhDV6Ow$Xp+FrHT{I1_v>ftd|Vz0nO>c}Dn%P9 z_E4=`<)KWx9Ev=<#h7iPZJ+`28Bh?du?Jm!v5|YmCH3M{hFTuf68=akwtmOSPI8B! z8>9n^DrM)B)=xm_$D)gh$ZZe*0=pz}U7sfAVsa+iHrRWyBbdsnJ#L4Vzp0 zfNexE>dNrClrA%uOpBen=Fg^!;RcAIcmXmJ*=~H-jstdkpdN% z+ffj`{c341-(C(zFcZbA&wN+Y9B#HeCHK2a9sC0yeI`uDoyCXA*VkrDl0JKQkvBjt z5F|g9)2(!so81Bs!g+`X;U>`I*5nQjKC17zf4DFsEBh)fx|nf|w2AtyRHl@K+hTKT z~|?YMJ~Y^m=^_24x%xq0>*<@uwa_O@oo^3(w6~I}&lH=v*c>Httqtj*At_8}Y~JgbS zSO~}W5wdQ611cmMr4?}zmh4rH{-Oo7;tbtK7h83Nn2={ z(asWqeq7Dr8-mOjuaw+mdAjpmj_RSu}1N(4#aG*O;q6XmT|5`^Nv7Mis!Q$^Ck~8S|)Q)J7EZDkwlRCdM`;m}=al z!b)U0+Wt;HUhd<+di}UzTsgZjezRkmvF*Yqib)WjmepWKl+CN#*Nv|$)Z6p@#SK7F zjSiknlfH|7Pcvc^Ug-To!>No3?EDMTN4=guQdzO;&n7A))*A8J4g0cg6x!E$OKv2e z{qonHICcDTm!-AjIh)uVn|#iclaMcORc_OT($Xc-opGk*mcznc5Uztj@1;oEj7whv zd~QeKI(W%I3gbjJ{%j;)hWw=0JD3@PH~48uokZ4YQ$Q4FrNC;uoAUhH&A}-W_q;_E zhbZlhSPu)oV^)MGLY*qnIZuI3~E8l3eT)>z4^|DARvxoNu zTR+-yqEH=)HSq2%qQQf?%|N`Uec4yoQR_|rxY|`YyuhFeNhwa8$cG$$n%I>AQf7B$ zotDVM5V5jUi?$A`Khf|(+W@8n{$caG?lYxp4Z6|YJZeG-{c`nHzm zw#@^%Fo()i=NeIT71Wq~Oj5k|lZaXnI0v*_z4wOMyLXC=$SePb-b!|8I6ad7Z87)4 z>T9#9BiiMt>uKGNy2W}o;OZmX165y*U~rWuCcO5;6h}<`PI=f+XtKL3P&BHSv@5ZU1HC??m55;<+?r#ggO^>h>9*g>Z)aI&ca*_1q{k}~B>87Y)nmLSj zGh__uYj$*xD+yU{qUAk^mmlwX?^9SeALQcQ{oCy7{jPxiHJm{h2%F580-haw(;wfj zTH5Z5iaS5XX)1X>u^;>2S^hww52BN2#~U&%2*d4pR$b@VeTm(I8ZS#*&ut&WjVU{Y z#7paCYfvYKF0i?jVElOY>~+e(-`7(b?X&$P07+wW#)si>Y}}n~+>Q>;GNUd{TrHJlCBG$c;Trs))6D2l z7K#weMBGE>hwKDSgtD^v;EUK*;1uLMm`eJ6}of!Ym%xeEH38S>qh`hG??Ygyq1sTs<;J`J>&-$pT3X@?HInwA)xj7v|$u2{T2GGhseZoO4` zgyGigo$jx^Ck2pO$x^OQJgak;&@^VHUZwYqzd=l9l8PS;?g&9tWv>vBB8P@_ute_RyE5$2yMna`OWgtsb~r`kD) z_H_9)smKotdZydV%y#3+Gu!%TsaqJL5;60dRxY4N_@ZEWskcs)!Ug|k#>HE~!92p3 zKHT}u9`zT+>($S?P=A7LH84Ndykuo}m{(rT#Oiy~EYD^?T2r_q%0?mNsDH zCrS*wqM|8m{Jcc)eOw0@k@d58+PA_tXegZTLehueul6_Z(@_(rtFNheQ&)8k-gol2 zA)9y*In#7vnpfHcvM5&Tv-&5H(7C4DzqT;0oLIQ6aQ#BWii`IVhXBLCysMPGf0MVq zkc**an8V{}47ztSqPlYQOQSx@SvuwHE@70?MyuG+qx|*0byV&%?w=$0=}OKWye}Rx z(wx;NP(G3c>DDH@I%s7$xT&bkJ{BZ=#PrUMpIC8inkgd-kXZOZtsF1jqEj4y%@RPM~7XPm;{F4by-iD^Nv2*)4@I7+vnXq^;kh3KR_*% z`hBtCRZTF@J}ecR~ED^yF$ z#!7ghyx}BBmFsTWLq7`%Q3_=yW9zS;8ETCV$#Iln_nufh0~gn7TRiW<3Rpfs1f(11 z@RfE&-z1IjL6}3#)4#o?UzqXp=llD?o)X86!svK^GF4Vke(RQIT^lwYr-+`dULp=e z=!{IhpZtY{Qe@O~OM8*qSChBZ^X`ODb1J%IUp9zz{;YSFxrEL`WO17B{K^#@ddM7=8( zm_-WhmGbK9?;SR+f``*xZ-V}jNr%hT8?`5^N5a-_9lJWjs};%%59TNeZ(1W~TL=B0 zg?8D>Xq4-0YzuLgIoy9_i`kxWwYo)=#Ud3O=!gU1vSSU68a$P)`~Ut1&Voo+!)1W6brzU7%?aSO_Q;EB!af~LMD=h0uCD5-+wQ&DCTBGv=3BqDwvEmu=83qmCx<=ES=n(`^Y-^|1l_PHV&yRLnS__ z?xSk~Oe3ZZds-)-8?L9;rwlDhNkhZeV@auVR~ts}u7T}&Tl8|~nBPlrYSWe65^T7J zhP?$*Vuu)3Mr#(IfFf~pus_z*_3V|#Mx2n~EUom`kjl?%q*)J7FHZcBP7ewGF4+_d zqw!8Nma2Wa;niA5aP=edKC$05owJL5B@=&v1Yy;dL7T2l;vAg1P2i%OSB;dkPd^1 zla9hl`R|(yi?9%PWvGLLpB#SuGy#GCV)0jx@TZ*Zv2Px?9OJc3t-P57BjYP5(W%Y~ zNmUy9kMUpxH3*0OSN>(~|X``gH7Q+ugw>tQ_SfdKFK;BdEddJ>0x`}6R!f3Q~tc=&JSho7C#0k|P` zaB=jUp>%psTs5YtGZZW)13e7cQX^M$OFsYn+tQz{ePe_+JT~jPsViMN z_R0$hH5?4JU^(Jz6rHRp8TTLmpk`gx@oS<=GWU-Cu`mb7-5~`=-$`<+oY$R0R02m^ zx42Gb`U$akFd%6TV&d`BR-NqGF66+qS;yTjt!v9eup8M~grDyi6;qMe5LCf{l)+e8(kJAcO7+ zbj0JZ^D@D3aG`2G=`B}Fvit@*9y=)C>Toep{Cpe%@M#0#ZzpFcB28gT81C2qKTnScI12rApJN<_3 zMRpT|=f!s9IGDcuapxpJdFY6RE=v6DXY2mGX_zQGE9s7l2OoZE95|G|7gC-2=RbFz zIPCHYZ;i^Pibk++eQy{#-Q(@yVE1(j2tJCbl%7snf(qo2LcRAJWuJ^LO9jwj!}aF* zvX_J26tW}2{Cp19;fWKV72 z1#^ivBBc$o6DsQzA4(1nM)T{vexDYdHd$YuN{ad?|acl39{pfW#A~cQ;i}#>D3&*#;Zl+=X%=+ zu}{@W&KUze3PNHRFE_a$5hgsTE39YPrPUkLbc}@btDt-0aZs#ysZBKM&YC*$#KgJr zN23xd&9I90?d40lNx{J**T@CS8v19yd){dG(WLevOeSm!0-s;uU`yISLe_j&rRaUlWX}7H9 zF)0-;HdFebwA=BF8+D`NP4^^}P(3sDSJ?e~+fBw*-b8Waj|o_338B6!fJdZZK1z(A zkg||*mi_jCHn`HYThf%Q>g95_oxsF<{87gC0pnmTW19#kP;;DfC5mOkTa3t)wuVpU zB$9y^5bKd~&zPJdNZy6>;<9@c%F8kt%`P#wJiUC)UK43TpXC!&U6^}vUun5xA8Vq8 zdYrBQJbCn)mL6uknAx{!3nJ`SgRHyZ7GadHm^h*#uAHKIc_~cbxA_zL7FO~m zDhjmGd-k>wAwdhP^5@>qilHixuo5xJquYUw@Fd`d*qk1dY4hsx!Q(in?LzL!J>RpX zmMS>?@kvZsPvkG>iMUNX->;>x^Rx(@G?U`oam=V6bv|kiD6F&=ic{+FI+%U{V(MgL z!An|UCHm{54ub}gWKYd|yF-_pgz_|_S9={;cYb;QTVmC-vJr!BHLb5GbqGCgks|Db zb@X-oGp;C>$pc!#^W@lv~@eco{K_Ruz{W7?48qCw|jl!U>_ z`q0s$M$POv3SagLp65?b$<8HpZ&}&IvgEUc^MpacH<(*qDZV^sc(O9@jAIJL&=#lv z!X}BhOXY_7lKc5JE5Oq%&8s~))iBw$<}-wVka}n0R(In~F6=XxfiUSg+;yG=Vkx}8n1aI8qN#i-0Vyj=WHLu z_pX9n^Qy03pFo|IOzAM>x(n;|c2vn*BA<|7qs98W$2MZ(HUSF;r_-S>Chz2AJu-;+ zIhA#+7Wv)~+cLarojN&K&Q-9h8PlKX%Ke@XJw4Q(Vh6D6copEkH5%cT5r--D+_h3~ z)qd#Mx4>EM{|iw#p5u7jPX{fk3u}OeRV3u%6^=Ik0i`BWY_5%kQFdRPcrJpwYZDwM zWQ#E#_a3KQ_LS&c;=2Z;2Zm9PpE73Gr0KJ5s4r&uWMY-N&kjbf*#U+A?r3pr$F#hqTr!P=a~T1vop00n)=eK(25&Rd@3&dEcb_~(38`OL(*s+hm~<&3i)(^yEbN86?dz4|`(O!Nd z2S_Vjc71zPb(JHNft!U=2$+eYz3i2{wpX|pNc<+*JZ97YFqn2|^ z$@hER(|U*bypI+R4rbojIosvU87o&rI(y3TKHOKRfqe=j2=Tl54Hb?fdIzV;K}EiE z=D}V=_k!xXTG8-1fY1SUrmA^pCV|lIV6-%83)IYHWyNoYJLH(Ju~8v_RI=G+xDMXh>6R9X8iT2UyJHi}ze@6z-r zS@)8jiTGjVc7LUFT$2)(a5omkPRZtq4k?CY{z$$6*$K&^yv7kZ;s&Q>3%xBv<5c51 zn?UK$LgC$36vp&%>(o7{XPR=;FR^PZAU6ia>7O{1l#f4$;y`3Uck`g{_~87)%=-&c zZL^ihTmNrjUeAOPkI*|TaJ5gZdfqmiT*&nLnH-$OjmqVcsPI4M>Ixe?(;unp=LIRV zwsJxAoZCBDmk!_O0KF7R7!_v;L=?Wq|(1V=ms zh-t4mht~Gw8yyJl&KWPR06X!nEH;2VpB>CiF8m$G6Kb+>LLdNt)tH@lH7Yk9iWT6K zYdY)|<%`yQop8!eX?a?Ek)|%+JSlC}HGky3pwH%h)xp&51S0_vL+h_5{mCvSy=VOY zyfA-cQV8ES&)GHqc_IJ!)n#zmpkY>ebgmK#EOJF-k|#5ucbYjnwdFy}eX%hho@w4$ zN;dfw{{o%H-q|4okyE?^N#B5WRH9xO79PhF-7$Q`lN~8ZM(Xsoy8XzO1Y1)KL|0<3 z(ZL3(UGHOmw3jI6Av*_$!VrKr!M%b&P*9d>mRuPx4ZjU-nefF9XL%4(E-u`-7u+sB zc%c0{fSrXxu$@uV#Vl;@KN15$$hP*73PWYVX}?mnvn@%l)vv_gmuw z6!G{L3Y3TTBKN&jVbZSaiw5t4@DdnGSBT$0k*j=3%!Lq`2mra^kD2Rpe;*Bq)q~%a zLdbXMA+n4^a{xaV4`60?2JD{QdCj=a*#0e>Aef$k#JLC)c1n2Cp!c1t+PxvEhF8(i zEm}Z-H`U*6+rVskroFt$+*O9`C8%7cr#=eM@(U2$#QIp^{C`8egpi*c2k(PtI@0eK z2SN%|&b2-#gS5iD)SI9Vj17T<;~OYFL1G}*ku5`AIUPU;?m6p$N8a-PY^5glJ2hfn z+Z9aq8@aj{5)#6rZV3LurSDqhZ3n?K_ed}pWbQe6yr^;Bb7_z{?Tz)BUF@-7kks$} z(q2$FP8vg(DACRXq8xCJPmi5CAf&*gWjnFS2#g?;Z)ta+=E*`42KVvL#|q+Ij5mN_ zB{gyoFNCCg)KWP&H|&O5BhKRaEn`Q7Ru-ZEJ7J!;ZckNhou@m*#CI4pEZf9{-xdAs2j5o9)}9d7wDrCxYjrsE1Vbg{3-HziKT zg#gIfpI>-udgHd4_s_>y$WgV7KR)hlFNOv&T@HGU&HbKx6ye`Ug1F<7cd0#6%~MaE zmHbdJZd>2enuK2<*M3S02l36y#ke)}oo4ahXSqWVE!Cg6un!c7FF2mOkr25&S`la1 z9EjOn`}VCIhz~vVT$cOXRwmDgxb%*QK};*wQ4hv~T?+XAQ-UmnhJR)*FY}67RYKfqsLo8xQKX7*7^PC*68sfw|_h?clYATICbe z#6W<+k;E4{T6XFPOuUZMIz+NWF=G|#C*oGw1Nni(=4fZ@SxrTpRdCpchFr09rs06}UaGodA z|HTc$<4Z^y9QR4fJesI?l8n-_+!cl_@=Vw5`DtX_9t*l226AP9O5NCD*)N9~xH%Qt ziY6xAsjHzv04Fc1E**bZC$yGlQsan=b-BSO?<=z!C#ln=^QM?=+=bKU%;(%(^lot| zka&5i--))$#%)uW%DM(e(+vr zm3hg-Ufzhk1)sWj)$-XZ#aF>uv7i`qn+>zH17$(fO<@waqX69FYcvP=qT3*W^v%Sw zp!Ksk0U0)6N;*sP{&XNVdzMn%oaUbDqJF7DDR%ecBgp2PPDn5`9C{JF4U|8sB^K|g z2g--k?80PNW^z*a(9mZRljr<5GnACRaTfl%YqK`)(~X1Zv!HWXsibQ-JHgKCx1J=z z#`$44sK;h^e2_?cnWG{?hI7TeMif>C26kxw2(_m?Fa#l~{Mdwyq2T z8@F4;{+wnFgAEv>_F^)}?qGvqK4{m1O||l@?jCK$GlcBl@^?y%lo4on6lh&)=Lrx{ zn*8>BKT-~a3c=}&>tq0PRI$D6*WOS~lWkJR-658$<5Zjp&6P}j>OUHx z0(25oLA0CIZdvdH(p8Hij9A~wKLXq;4{yS!F@9z2L&j~a?$|-sW&K5#47GKe>=ko+ zlq7F6T$K)VZEbf&#q%M_x?-A{Sv%EO4Bz4ERL(>v#D&>_8pmc3_rSZR7>+EiJ!(3U zs`~M&MA4^$JNALyOqhWE`e+;)blCECPn7n<)6i+hcQ?;YZ!!*K1F;ZHFZP4J7jN}1 zr7NM?m1A!)Dv^>zd^2&)BC(R>J>ArFj9Gk}XNTt%D9CqdqM0`|&dv+A&N%soHq~BX zkhp#pU`Z{A3VDV6;}=acwuz{gUF{jx)lBLo`%jIF^a7Dg7KB^3vWVA{ZKTRtssDZ6 z<0c2e+9e9uKC{9=Msw==w)?GYpKm|a(&kZ_f#=Wth6em>6JsxZ{c}MRpr6tEi5zTh z{v5l%QflntLLA9dc<*rp9as=y+L6&^H+`0d*7K~G($aPN1;d6#X#49Y@m&Am6)Dx2 zOCcebekCPhmuU}Jx;v%i<&mTIH9qJ3d>x2OKkIO_sVRmQ&??j{52?ah^pQ8j)B`Gf z$}qrx@o2N9nqI^?GX7E?poR(bVhLA^^HpW$-I_t_~P+c-e~fteWG000dH)5Tqm zI4QA9LcF@HBJ)eJ8&vmiw6k+QYX*?rl*UPv4- zMV@kL@<_+u4acRQOwvU}#=6li*lDpq)nH%P*GRR~&*AEw3$k|f) zjAd2d6t-V%Y$W8;7i0cy?sS7iIt*DjSbjJ>|CBA%GEX1*k4K&4(!uwlNo1c#@`geP z=#fOr!fyiv>c^MVXfckF!ri2c;1Dhn{vrO<)s{R&_C7652<+lMxdI%5*g#$N$t^5I zJHvnG;396^;{R3cDEm@?@ItnNA8_$J($Vq(<;vdb3|RwqyIWk-XqUAAH1foxxNsfQ>mKtIpv!#C$W+r_8XNzT7M zjoar9ggW1ov%@z^J?-X3AVb5y=j=+5arAWe>*%3Rq{X+YwkD`ijnJctUOHPn zcUCEPQ{tUp$tw`iwrnK5+9s7LE$dUB(h~fo>^=PZM4_&e`dbXueHgxKarPxDxV8Hk$-q@;*p-GBiS%Y(o1=xrzzd5uU8KW!kC=^g6cE>v;PK`1%9?uvZm`02U z#o~0n6^mG+tHjGHfY`9B3P8WibKabKW(pb3c?!x6PHAm^o#`3q?-ze&5&|f0TBdYD zCyqX)TOwos8MJ55P@`t|a?{ALrpQ@YmxYZH${eRo^1g8yu`;Zfx(ts#1RtRZ#d+@b zymAIfk>uMdQDPi~9QjS#Rd9m@=}(bkD}m|)QfylVf`Y!LuZU-3u<7>|JUvT&A6nN2 z3Ap3A_EAq3`qXb9e9vfsjjjN>Qi*FobkgtA9r%LWNN3FWCiX17p;v z>w|w8^sMqq%tS)+o_m~6uf9^{B6Pac_t+r>pZyZu7Fxm<5R8uiP7gYDf~Hp?UW%w< zR6Qh27$(SV#Fc(Q$8?GQb%Op!0(+*#cTd0af4};w+h0VU4dxyWYwDv*qtmA>rEUMR zl1FL3w29zv0(H~PUl<{i~E_S3x5w21%wvr3A@Lo#gIM4MXZS4qW?jfn`?;YZxAza#MNLr*uf zIHA&+1nmpjEdvZK;NP5f4xS7@nF!V>)Sh9VMD5MpPbM2QUEvJVkk;sNY7ktn?}|Wk&HR53{GKBTMe8o=zth4M^TxM1AA`;t_^vv!xK!YDp$}hxO$!net~kMw_g32D>6& z!uOBJI4;VVts8i!m-S4f7CuNO3xjSn2mw}B3vP5KOcyS+-{0T{-gxr-CY}f}YpM?? zn=DX`vH(y-e(UMY3z4V0EB9rcZvMXC3g$tmzKKNTmNu`^pvP7@+0>QjPHu&XY$ZSX z!JYG_7jy8)_nbA}m#G%daFFv2K@xIb)&>JgxkRuZ;n4}(@sKM^f%tp$>DPZ5Z71c2 z14Z6-L(g7+N7tK_8-$`(Uo_BMndy!uC&%#%T(wPheLJZ{Nq#u{OnmgwIf53})Fjsb zkw96^uBa~JpF{+S{{BFwY%mRfCC2W(wq%h|)Pxq*ltS=PMv)sgY~e7_i>#)z#Gc9T z^P*XD*Z$B^Ihh0xXafBB(*TI<1!3Pn+ntT=*GWor zA>&MKhBV3&lF5i+<DE(WuP-Y4Hni!MYrM6?4&D6kZE)o+wJ$YusGKUaYMDFw+g zM;w%3o9xjvdj-)Kpsk=@mlhrosCsSl(VA%;F^)V(N-lpRfY2!mWPirf%3i}mrJw#I z^T89P&FO?f8a~|cM_y~bU{igin6LARQoBGtL2 zCn}$VQU)#rwW1m2kmo~l`x0rE<7#uaoYCk&U{u%qz^Kyx7q<};%7Yz(7r7C8VzyfJ z<5~OWOxgF&e0l499Q9nHloZ02RvaTp%(b+dSC`ed$%&R*bIK`4V*p-@0P2pH83U%9 zYg~Sf7~dKx_H<=`_$UJ4o>IB7EG#M6N+d)oN7mwr_J3v30kE{z2jGBGK{2v&rQ}&w ze+fw(5o~7}WFXBr_L_65d}5z*S#80IkKPKne^KmLo&<4_UoMmeEB6h-)2e-_IM4>L zxBUKj;>i2JyQ9bsXHQM*->%glAf`(WTUfG_?C&URMKUm16kE3@_Wei|GCa`~q=8c*<7y|I&&JG}OovlfyXrdXy4@tut<2nC~ z<}WPSb82_ya^Ajs)w{0qV7QdB*rc7Caeh8Gl#uPRx07i7(K6R3Mc1%ySyscYPgjm7 z&-4NTY)}MN#Ii*Zae8iIsfZHThA9Yx0T}6e~2W0<6?X?rq%ZhWhE<@fyx7{4U zq^&wzgUB5M`#Y|m8K8ql_?FOKwk~(;ED{gxV`H&2^Syk`anYdRYqyGz)$UW-OjOc# zqoCQwt^wfjne^(YZZ4(#H<{4S;j>HRS$q^wXFcHTlTbUn-^#vnw1K^&Hm{zH6Tn`) zky0G*bYhrX0R^8G?$-I+Z&JB33p%^!EZE!hg%OZeBYGVF$I{!n%r^sbnQ?Ggs zwv#hu($U3+61hO!1Y)RVj!twoff2vleIoffMHt!b8w~GkX~UsaqQ$Pe#%L@^PRbVr z&{5XkZME)ox%ui^^^Q8n$u<`xB=M2tG=;k{qtz23!N=TMy*>$#PIS_qhSJ7jt9jji zAUj(M;z)A+36%2W#e$VFQ9{sJys^W9-63aSdA5{=T=}}1_NS~;k;CxwcucahW`EDG zojN5k;}5AC%NF;LLnlcd0+8>*i){P3;38vSZXxhroYBUFe}Z9!)7l6ZaFGVz7c}ne zM>cDkS>G-w7<$QjRtfN+GAy2;8WrTCIe<)!4>qVZtF|tD>Kdpbz(jZa$WS<>%GF3- zayn7Wq~9MaB`-De4T0P9lmmfvn^wM}*N!qtAc;3qjZ3VPhj=pJrL$Eeo4XRzt$o&j zu2lT@ifQ*L1&D8xm-BM?VMABi%r&rX0a(Op8;Gqpj<->KKF94qM0DY>&Fqb^EIh#_ zm8isl)$KB~vbu8H_JW{kt^0xp8pXlkKra#wzF~C71%^q@-|5bW79cBdI~z<;p6HS& zOGq3#*?e%kAGvO5^#WCW$@|_VDrtVDLjd=P#x|$eew=|Q8wjdjRXSV)JEq&9zg3`f z;pLb0IjfG{P`e-WJs}5~OO%pxwVn++W>-HpCHm>GJQ7iwkcf7>pBC?M0xi~AWFH!< z9U}+I08cORX|H^=04gJH$Tea(#K)J8P@#Tou?O9}187oEdMH_U=(QGW4P_Wn_GCx=tPoadK@KG z?#Wn@4!J~qooZa22{A$$=XPsnaCnzDcrs8|tyXl`iYfs zGPw9nm$;V;Nuy!clg9$;JUgZ~{v-e zMojB)-x)&kPfQStR|$}ylC53+!%e;hOf zi^j2+xk#M|`}_lgIRA4a=l^%Ce*dps^S?QVxjxrp5>UC0xV&AxeolY@a1`4UaAwuVSX}P(pKi;ts7pZUDe2!4Ha?tFdr5vvj&k^!C(X!)2u72~QmTqc|V` zUF5d-cAo`%LA3<}yi51oDk+?u)vZR=!+}Qlk{JPQf6yH|YFsD%z=7;M|W*y zC9|%#Gj1U18>uHz7g}3!(ffBKRG#|8J@+o&ahP?%#zP6c*j)E*z?$@OBtbyl_sV-N zx|e-@hC{17;Qk|RRq_`a8TV7#)w{1@)UsvZP3j#xETEJT!))Rp_E!F$G-ZZg^O|4% zpJ2%30l0v;B6l0P6z+(}Su}G(2^fjK6hdf{6Br-<1loX6R}lmCoot{jW4mGb6A8MV z1{%%tor?QR^-0QmBj6O?sl0X(5e01|d3@c(uJGIEI-ly!4dbG1WHHwP1u2kEsO(nI zie|)k0Z_=HEU)t@Ej02h+{$n90^a5cKcn6-}O91A+^fLn8q448ftbg(+AgeV2U^x;< z5{v`hNyQVPq)g+4pcix~qiQz2&IU~_fpY7lcJ0%U>dR%n3{a3eOof@bO!ve-V$s+w zltfzPl}?pPuDonD43fRp26vbHm#J~89tIvg#fr4$k(K#CjsO+F1}KoGQb;dCUz97~ zK6s*`LCR5DI0!ILM*^;ZoFP9@%dEM|pZdG#lQrRem;VF;)(8PJ#ljl1~M(V$G<45@kUPxLLrG zji%Fdu1Hy@U;xArv@t0}p6O-3*1L>4llPletw!j;;fpS{!GUQ`zHp!=`sqE5pLy@! zn=oNmSycfVwB^9&vdYm~ZrwS!yEUg39H|a1f-+s>gA~nLYu$4}FEh3WRC2qS2>+X2 zZrG4dPy`|hK!~#PZcP`rm(P4KyXI=4!{lWr@SeAOD<=KvO&acppu*V<4t?5eSY)z3 z!S97y6~4n@15*=#ogP`L{9Ic6pH+}s{R8=}2cwfD02%{NND~lQ+JecW0m%WbFT?&E zf!|9XlafnHE$v}*E4H1qe!`(sA((xMPU5Ut5!(B#HwRzg#_hf@`^72+O0fZFBm{Hi6HUzra{;lJF@Vb%^QuWVQIIB7w>r>py$AUS)_I` zBLW84Q}f}5X-D=^2f#ZCKk~@UJO)(7#u3u-^OLW~rFjjf&OoMe7KhlcP4d)!XyM4f z!(4KpuzNoy4z`+#N&R3by3^Q0ZCw`KQ4f845EP`*(mFc z4D`EHww2aCnjW$>*S@jQtTXLEaK}|KsQvP2B=9+!P$ZZeZS)_a9=0A#p3ToBdCHZG zzKpGV_hr8Nla1Hujv`oqliZ=X&Xh8W2`}4o7sMWl33Z!r7Fj zmGAnU@>+K)#Hpm`KI3H#=c%(Z$ILgkmi5Z5s$6w3R4(8$@Ug^5-FJCMcHcIkdU%|d z9W~0elxFte{#^>p*HU+Z(SEZNXUsF2yJOioZOPHsmsajAx4B$NTJDyQ_#3840mrTn zZ`Q^d4~D^VnGI@Y%%@MxzoLH!1sWd7*xMX#+};yWjU zU#F*MOmuo7sV7mbWae$>@~2+@?JntBEZyx`(VH@ZC&x|i_GUaK8f2A@9qRN?P>K(p z7IEj{HSFg|oV53|c{EnN|434|JG3dhx^*VDoran_rl)zaj#R5gNemxh~CR1Uf^-d^t@sorI{z*FH zj`;=TkwM{8kMf1I>gjm)%-#%ooA^v#FQaHh$-1;bpWC(t{Xr5t@h=*(MY3(p2Mn*~8^rv&C*-{ro)2wjb-hz5x>3AIzM(#cOs>)*GU?CXWUi zY&6Dtq9H#347QC6$f8=cvenfp`7)thdhm64GnbTaF`f1T&- z_rAu9<&udvoCsRQcLJXrhUM2x7(a<5_h$T{Q)ILV(`3X!$klFL$n>fc!)seK7Pv51 zxtc@TmKTYMn30PdayHIygW50Xjh{T&b|?mEQX8tzus_(8dmVgDhb5I6Z6**;4XxsJE$J(sdUT=jGY0>gN8_?NiRLeA;19Za!I`5mWN$ zkFAemip0rM@3Wm^DU-_N_KqXG{l$&<)w1r>%j`$rm3URvjk-|=Z@sGY^$xNFL>3Gd z_K@XA&EW|giHW&yPIoZVpI0eWOR#n4zB&2JN*!Z%mB;h2aCFv|FiF+5>66&w_=LC{ zg}!#Se@5Q~ns^{XutK&{`&H()NXAo*Ovhq6jTQq93jc)9rHc=IaPE@LKUl1lt#a_J zTPCg8)4WX+fn1ujQiu_!IXylx|7F~~ckawS zgGNt8(0;h_vEazzR`hiEu^~KX5`ExF`k%Dy){vx_fy( zq;?-~KJC<3zUIk9RElyCJNn*v@x@kZQ_J(h@zH&^(-z*_H-1w*j47 zAA?heZ1jfEeR`3Ht8cs8AotU!8Shr(3?I~Jg=fC+lvL^IPG)|{*05RJa+2pnW?oX; zj|*{-%B$0^D&d+Ft|}8uEA)E)IJMTvJbEDSFe>A5$6E2}Ct-J(AC{CA20EiD1}8cD zCNrunGZ{3zJbhTCy4hp8>kkGmQl}%M;G_;J5_$s-7|9>1GqH?JSCgLTb7a$gXi;k9_JDF$NheeU=hKV>D2Kh zd0L;?^6Z~uWy7!ws z;Aw73<>rQqMG=Chw`#B5x#Dn!qbuK1j14tL zUAyx3dWyG+btbg$i=9E;`Uh92-00-mFYB67D0|nn;D9Vo6_LG_K-2;Qcd?|Lb_n(m z^A@QqlTo%)E`ocu)$CW$YKxBT&gbk>@)hS|kRz=*eo6NwpTKE$tyJ~d9)&cRP4{7a z$kc)1(xcowT>T^Utm$a`^y}TKH9zHWZ`S*2ltv6@25m6SeUq2R)5Q@B8fzz}0Udq-`H!<-gjm<@?>xx* z$Q{E;3Kq1!AKtu~*BAVIE{-O!bU$CgHDMfAeSj+GkFSdH>UXPKl4w~9oL*h7ovbaw zrC0RZiFhNKlWvnuZ)^4qy_3lJz;(11V|wMrDOLCOkUep};Xme_JkDSL=*Pwb2xFm{xS~kq$7{$~GhbpqKzv^cpj*Gl z=nx3&A=e?Fj<#xEl|~;kUkD_MTfCV1*m&p~k6TK>oL1_;FNBJ7AlR-9!x$f)%n(#m z`~+X4us@=-U!RNNjcP1yYL=J0us~jO)!QL4a@5cR|Ct;5+%+=V!Gdrlx*Be`6l$lV zviuC`k7E{y2D7}p=3G~PgtwCliG7I(n@Jcu97E1?D=6^g`AAF&DpVjGb>IDagIakq z==aO2_>7F8te?1U)k~8yzT%rk_LH^JuH!mMTre`eJ9djlIox`24=)oFW}jL5xDZ1^ z_s3bYTV9Z7(j*_2O`tw?bk#oa->s>dTYDE37Jn}Psbg1cb}q>-UF4o|`x3B(r07i} zcMwoK^JO@cL6+B{w6!%(45yJP2QIt#jMs+ln_!hh3n4kj^faAJ;dR6tPt3PO7Ca%S z-b+E!tTw)bHy@d0Yc4FxOe)^G>ow$a3YVPH^jXjmp?vP_rO81pHFf>NT-QgJTJ092 zwmD~Fn$61R%hKYxh{2^QgTH@-1l4W6y^p=o4B!h!J3npCl?5$Yr;L>`J-%H+a#>f|L+S@ap<=Y08f$FH{eot#&c?7;0YV7g!W8C*f8beQ(zqmh+>ht{|Jsygb#_ zmFjjqKKCTY6<_7KeY@0mDNk3=An^m2!upTHxaHhDsY{b!y|J^DQ$hCf$M9P4$WwDj z>N*VEzuv9b2J-c_xB{B5E@h?(dA4sTqCm)S)?R<>d2K>=*Jz+?FeU& z6*SAg=d$z}ABnHxGuGkC-`GKaSJMsH=v~frO%xDNE+n409Q1}TKd%)71$_RLvD$Ik zQVThbyk?H29uXNYml=!lt*vK6;`W*5DLOM1eqBP|4!Y`f>rIij^QZk@_ewh7S0T~1 z4n*P7_vaVOPPb#{l}S^M8hk6Btgb;%Gu1Zc0%Aw}s?}-4nN;JG%*8~2d$jW%6SdX| zS$`WVd~WE{Z>l!EQKvpUN%Rf46aQj3})Q!(uA zBA)RdG83D!bCd0TSz0O_%#d56bowRt2F2Vh;kGHjG**lhcqFi;b?Aq|q!{1B!)x>vLY^e3)q+qhQ`5wwvn3m0P|1heRc!yhA z(*S&O zD%l|^wROdB#Xt$1uNCIu`*Z>=+%Vlu75c#m#)IWPZZF`FBmJ9sw%B`$6gZ zMF_beoAk1APW3LWM~{~G>k0})lz8oy$0~f(GpBvQLFZMv%aifRp3xI8d*8VFHt3$h z8FSY$JGBpcoz|0>x~T80-dL#`#EFu@E*l4I9LmK;XYTJ$OTs}U(=4i+%|ZyzjK9-} zQH&HwpXlEtFZ6nLu2$#ip%X@Tp+6qq3>YH89gAikFYIawi%8&@9u8O9j?pHsVP!rw zXNx(eM)70yEvFnYEgjcaZ|y%GEw}M}wA}!WFwqllvHHw*{%k5n>s$OtUodz^;CTUE z^3H*Q?{_ATY`>|IZ|6}Hhrg_uCou(J5>HLGzCD@!#;b($9<2Y|CW^&D!$tFBT&B+a zp;Lj&a49S7{ySn1w@-vR>&}cQ@p!cxAZ2w_4VT6Z+*cba4N9a338p)y=DJ~ zQPMtJQHjfNu5$95?y}~9@aDC5*Ypj252VJnuHC8L$ZJih%rz3}yQ9DR>pzhEb4J8H z{6O_dK%{c-i0N{UVHP^;%@Un{ZA=+*4I;?Y6-0N{Z+Ew-0zQC$udYKR0;@V5*?Ylj z>7SbNyahC#3PP@`<0xt0Eku;lSpm~whKcgS=5|`59n7*m7CxKO`;v5Ivxd#& z6y!*TKUEQCzuqBp)cqcFyWS<-%qItVgz-^yq)QFz^F5SuS(m209&Sz0p!(Z>tr*uz z_5L1&2(BD__`*`^P$hj@(CyTu^fmnrJ9zV#hF-fLSg#_4)QEF>G(Xxk$U75&6Yh7f zQblpq!nB)K8&c!!M(4w`Fz2V^ma2jY55HUI>A%aWyL$9MUHw}wRF-8w zbYDRmy%IrJ(X?uQTKpi72{ zBx6oF36c#T`IQad0E2-~XD8`)Pv`KfdVlGLoO`j=vsrQ}6;0Kq>3+P^1he!Or%KuD zg)!%;anH`LjEM$*LB2fv+~c=^YYtga&7ZvG+aMw&etIH+6+109up+G47t&v6G5Vnt zs+<}!UyJIxqsBT$pwohFVU>|{H>0qM?QyfQt;u`AV)EICwkFs z8C~`(__g)u#6GN0m6Zd-Z&RH*k8+A;0%Jcex_oTE^3!`-a7qhrqI;LVF~ zi^@r2nZG}>adOu9-$FHT_I2_YPN#YvajMu_;LE>%ua3tvT=6cS9Jw+=vAxRvg(10Z zvofCYAgvJCgV-xE$rCMyAGgXCZYz5RFabYX;D1eH2o&ibB-lvg^APO4#BTU~;!nAj z?~9*vzZQ4-SRhY^Ugop@uq)JQEfPGX)k7@&i4!f@|Lkoc-=!plONN2${4minKbZWdgv#eo zClU6{R=9V*x_rL+b&XU{j1Mkgr3*TluvFRU^EUI$vFL2Y?qS4y%m5)#0v^L7IS~^h z7nx&GXCb*a|6MgR8<}06`L(>oPNF{i9V(p8btx*tla+Z8i|vho+iy94`=C{@|Et)| zcX^*Hp`o`oD}G-Uo`W6P)`ijf`b4xQ>R8$-z~i-)7P4QIiP|rp54?E(A<8H@s3HJn zbJLO8ULjg6scs)l-FRruDNXT<&ydBjxy%AA27z8$OXC@BB#=JoJG&~`lehcTq>zs< z%Fp_F2JCH57>Qkh5L{_pff3c0#>nr(E zdN*Ekl>Y;f;og^YT`Gag5E76IlwKyNXoO zgeA3b7wjL}r#~6Qp0gZh3a3>1^OmBb+`W*R8&q!I z?nrkvkok^=R{;pGFVPLhjD?M9M(sEX(>niIA=@w-N-?4BAus7`M~;P#n)T+uoV5aT ziOp4VXLn=J#(tew%t^gfrx`pil*)}}31{ls%G;}u&Yf+-bSXtw=LMN2By{5E7kr6F zC_{yk?Ci~W_f{42D$V5BymT1_YwHvZvhjbn`HDAowk1y6rod?dRNkXzZjjCD)?odW z(CFk&3%95%3^aESFGe@4ismzvWn@bbXV{b{eh*lrZ_L)m=7LPxr&KPsAaHNrev@xQ;>dn=nk#zi|c>Ts-7I1?`51mH6*`*RE;2Q?d2%!ujSeZS8W3u`&Roo`@=^j zjX)Jr;~^jyI^STOE(YG~q3Ze6EIh1UMWjFDef3PJ!tVWiN8{B8Go{ey@#|Yj62p0# z~}G%Xb_U$G^Da%%1;${i3L6!KZBrkS*6YV3}J#qF|=DmtjiueK>F1P z&g7t-PEc@d2wUhaN9_<|zl7l4)%>-OG>~w`ilj9kcIg%3KH?eV?vY^GiPmB-CDys12j{8jk2_{h zf@UuNvocUr5-n4ruc>%1Ab29ar$^mIg2_?0K(6N0^CBM!_EKG~jD#__LPsL(%PlJv z3Lqk#UC3823=3&?5ogn$t}eWdn^ehLT^X;(bR{IXa&NoWl#Le`9lS*ATOz_r6;S4DTOequkI4 zQrq*m2^meJyHZS1dGW(D7akX~K%zu9JTV4Jp<$I~KD|FYjhU$-k6v9j=4WtV!Jb9X zg?yxignv}m`F5tf{98cj?8lV*g@{aA$YXU~uCm%^PQ+VOi!HU!)FyAd4EXgd)eSRU zFyrPvWOFsd)$N}`@e7w@Ty!ocf$-<(rT92Te}^n2_0tffbo7aMs$ z>kn+q+2GNJCkHgvJ~q?T+p&cY{8;dE{PyDD!R|Mj?L6-rVq$~Kw6sJ9SM>#igz`yaqsFVUE(*$ptdD${uy=TUVm-vz6PVqWI-xIH)?HqkQS< zoT&(6`MJctK1zoDW*u}%RAR+tn-4Yt9j=GBiY2@Ci6CZM2zt4UI^k3LeKv>vPibT1Wi&@p}8svjkT(Uh~ zo?gFbuNyRpmpfLjvq?|_iNPQE_qFutclM+`sNcG~p$r8%Srk9Bw;cVR=Tm2M6_QKz z>aJ3k8MRfAFne#O2WM$Sg{%JuGzHZEFRk5Aa$d?Md>!FKL90F{5^K7x@%@dajC4u| zotA;8A)7PS9fno|uWaBx5)qqov*P3sr5v~M>0BkAgNdOFI$j?o^k{UwB+Giw-;y(# zMe|LjZtgR`3s`@a3che9nuJx1{L5+}5Zc&gm@(b4VXF%+&a{q}&ST4rkZB=&+M9Jd zD1X`Ex@8?V3qJ5e>}UfWK!_{>TnEw#apc-5(hFB>*8oxf)Wc_aQkN(1!Hzl-f=jJk zooVy*w%J&dzfEX#Eb+02o+s%lMjsSyLG53;h$J^)0}(#PzHI{d{HW4kWm zc?k!4I5`yKQVG6@92&3h%1lecpD#4bU z1e+JDx-@B;Ayt6E-_f@mO!?w6t}mj~x4cewV8(REs}Da#Kd8ls# zjQmygZkD%xIk8~10LVfaznl7EV(r)L1OE==JkKwWw(gJKU>#5LUX9C@U>%DatCgbd zmG=GKnb6Z$q42MUt1OH}l+i7cIQrh_uODmC9BisWINg=LI)Zffj{&%rirH3&f-n9X z5pjEX#Wq@IcHK&FSKh(lpT#~I-`w~@eA>#k(q`&RQ$lqwF=V^;-y1KvNk;-l!ZH=X z3+XZ=V~hYfel}K-a}PhttZqUq%%@xaJzM)=w~HsA!xif_Au826jwb;hdueiQv2)>V ztD|AxtEdONKSmpJ>&An6%E|Z`j#5r#H}`E9Hrp~PLHPuGNu1aE1{2oYu|YR37}1{& z5oA%@lG2auGm>P9&y^|xK)lVr*p(mNV+z940w+T&!q=`!iUIE?jykm&6U?`V8a*xN z_gOvl#(k%=97j#mr!|4QqZl3-Y?+w&1Y2<*gg1nm=pIBJ?@8sqwY+=cn9V5`@i7g1 zDNU9}Q++xY(Oo7|mXXY5NOT^$+%g8vEq0KE2eyzZzdv^vYWCuM>M+Zpdl|2i8#0@X1MJhB*k}h^psH{!TLZChl4HFOrMwNF?@LHL6Ema_LRwF77gro$}bNuEs^shHWDp`4y=*rSJWTe z7K%Z~^w+k!W+5@GIR6K^@4um}t=FsjnTD6pxcEI$8-K?xsm6>*vHetCH;H(FzUTJ5 zsD7}!tKep^Aq2~G%{XeEcfV4Nd;p>-9i zbTx*AxG)BOstjb^+_)ZYlH<1`fPyY`D{XcJH|jXaXz`cpN*wITrT7FW|EdJ^kwNOC zeWw^3Ta!dnH}{1X4KZ@HE&YdB58ceeO_B9{aBxwAp$0lv`TCo6-0q)WD-*BBG*GObLaV|J2Ss`QsyW!V>1*A?5BX5D`3dwrwvPHJZ6 z*!0%@Flm$fh5=+Tl zKFki0T?e9!dLik>H*=>Ypi6V+wJNdTnrN52c%3%|0Z+HwzgX|$i z)#adV(CE7uT`pIiBXi?YQoD@@!JkkNvR{Gpwos;uHru+tbB1}u)+|ygk!=jls6?9H^ zi_f2}4U4l|E`RYtl>C!s;dWO|4YNt}f%_hrC{bJ^7XRg7aZo&Xf3kd$OoSN?I+%Ga z%;$|9MO~NL>+|hXi9h-s!Pl@{6rra*IAq`GAX-KtZRD^9o3S4J%po5XpGJb7*Km=3 z9EUQT+uRoe_HwART&lGU7#d|5MQpDQbtP;s8n#SG79^XQN59eIZFCCnx_0$;S8As0 zrxr5uU(j^joPdMGP^`{%w0L`k=~>@X(9#1BVzchZDCaKy{_xC|5oXK4dmtoSP6B<_ zpj}T{ZW3`_y5<=vzVGng7jFo8hJEj?bDLQ?`M4gtxckq4+G_FV#nmddgC;yFDd~3C z^ThXu2krI^gmPa%5*a;kjI`M69&NL&zbxWC8evDTG@{-5`8ue|Hy2=m24Zq6NF(+# zB&^xOU9MKa!x#Xzy;9{ajJjJhjT=97Tc!&nRwpqw>A~)LOJ(3HebLdCGOi6fk*WbD zGT4%#HlNm61G)7$i7SyAvJ371d6d?>R%Pt$?zu_D)^8Sf$0uT>j^qzo(WDjPv<^EX zp++5(#US@=SX$quOnpgQV{TpAn_SL6w24cHO@a+9mJf-|*?vv?Hi4-axRm+UH$t5| zn%UQdSTPeG9xA_?r9}GqGSF6ajp$py(tu0Uf8(j7q<77Hlb*b&rgavdQ9r!hJvQyy?YbNP=&*_eDA~E?ZxRp-@+X1 z&i(fyRpy+%S)dW5dGO?=6TOf8?c_R7otRxvtEP<6)rOH`h$V!zFSL8Dd&oMp6M900 zWK*;xj&AawanmQP?ylW}&~ha84n*-Lp@anQtMxrAlI*nxN8o;bpq!!@7pwuWrH^XA znooz`((jOz`Bfcm<3zVjwh)I{S*6_fs0p_*KmxARo|T?}VGSj?HU#@tSro2nsfT*m zix!+|+f;!@xXoZnOnpU{a78|1n`{2ZQ>D`{ED^^SvZ9@-&EtJjHt-h)sFTozG1EcM z-S(%3&+f%!=F3=0XQvA=f8go80=H+%&# z>OeGjZvNb6l7L+CdxdN(Q=_8;Vokt&T`Gthg>T_@-Kk4Dyq63=@s%{ZmA*)!xi&W~TN-cL|b9!kuocbeRYh}ZoU*FCulEd-5YXfG_&OJ#pC($cJigfE7I z?ef@_-@UsRyz14z9BCP(5fv(Ofkzt)2xctrS z^YIwBVY%%lO&)^-bI|F$!y@sA7YEJ?kK?b;HB(Ju;8vL7X%!q9x-|6dCEr9-y@q>} zhuW+!L%InfpzD*J=lK3KnzM4;buWoEqkJa#;7pBPoocL4iBt)qoEa5Wb=zU+#fiKH zKd?8?h#4k}ZF1TBnQqN5ZQX0ZcC(-Ch)y_8hQ)RTcT`zT!rr((y~DS5T|HSjQ#d#C z#E zQhvL7s1nolW7|h&tNf7YCF zW`%c&1DNeSAaGqG0e+V`Ilao3ku1X*VQQeA0#eRVz;i(>D)^~WxS@bxms|2e6j~i# zO(fkR1m+Y?rfp{Q3mD9ftg!6Mkc*N7mIO?3?zAeg(>n`pgh1>^hLSsm(`~d5*@RGr z)lT~H{&!L6Uzs-eL!9hwwjg1Kcinwj*)@ zrKO}tiZIn*^LYo>jY=#ai$TBccP@D>+O&Mp@(s3mw^br@M|om%F{IrFUPfN6uY~$D zJ;T{wbKFR26;|hFk${p(n#-E7i6T{j^z2{1ylX(g>0wjD65`ilGVRagL7`Tf&IEAe z@e%`PL_v(|CLxFje&w?#vm4erYWC(O&_$nas)ye~p-x?iVQm?n@99MKN@SLI^9SA> zj6lTH|JuVZIPY(r#sqByC=WG2mBe6&1nK=19l%9;kdR=@C5I`jt^MreBwY=4MpyuT zoR}DMY%Ty6bTOsm^!irc1F`K!xU6`*JsyLJq5I zu4zIxs|}BI3m7irl6Vp_R!5BO6P(5?|Euv!FN(_o_S2xqH?cc3>xQ)L>|x?VJb4ro z+$ZgueY>yU5LXK>Z`JS)X``5RBwIEAP93izAa0-nQ>zJKXQ84Ia~i(OJvrX7@t_C?||v+~iCl=d`$*5>CM#3Yb>nQ`0T z=;^;+`5hmc&i*w90V!8am`%W{L*V*b2(O5>TeCbA&jI+t0>!IZ&cD{5De28dJ7lLN z5=JAMeKX*h#zjWCUiWk~8>-OFoC3Tf*W6mvd=K}}6_ra)`>H}4#bv7{@?0LVp8IME z#d?z0sx*r$)^}d~Tq`IHXTw~h0nomN6B>}NN}H^#@6BANRRkcT(^Ure(zTYc0;8UI zcEBEtX2#1mQ)bzF(lu$(9X6T#g;!)_j+S%l2<%3tDPlLbNTcAV2-s*wJ%Dw{Itt#a zRi*6Xgi*eK(vlWX&gH=mSjmgnfRzerfWOv$21mfhfVbw**2{E^Egl9lez^1pkYctB zwg7CF$Fcu?`$%zW%+9gIqbxyk)Fr?gCAA5N7a*i^(MJ^k+yz~8(wZrM;xMMA{LKHu zSwJWAmd7~of?UOd2upH&ufRYN5a4;W1gg%j=_)e!-U3$BfI8<(d-Nh2eMqdYlyRBZ zS(H}>p#)vKaHpD3{}$Bw0DBwj7Z{lf3v0dZrT62<#&WBINuf;h;R{N&??3;(R@Mug z8}60~%7cJ}vt3ntD^;cP1EprtJno-=ZFTEQ@Edw1Skg$K3Y-jIQUa*Sc2i6*dRKyX zkrV(31r&zpUU!RJ@#Eh+2jjre2^{*X#*h}$EaBZpcs24$5sglt%tpzXy*VQPp!tkZ z&KCR=d*=+}NO**9sv%9+lFwp8rg=eQbH$@Z%ffhe1i6h0AlR%;j$tmu9<*1|m8E%b z4zxt`noc(1|E)gFqqvV&){GqYV0=P+)s**eeOH$!US`HP|EJ9UxM<$Q#OlWW zscZ2|6n<#kbP>VF-wxi`e+XNT+t9bjC@c>qJPd0^UV{aaP`|}##rB^g5H5o5Ygh)})-1>nEOpH9)a@VB-A^1r z8byCdJWE~^(d6hO%rE;Oc9KdtQPBOgz27m0MWtofTzo(kAP7p6N!Lw|tQI_`4Ok@| znch4!Dxn9*ex?yCq)$Bh2wG~mdV_k<-k#m zVp6Szpo`brrxXYm*+`dl*1@C6?kPh85OPA~u zx!7iS8i(OgLFKo>sts(ZRkLh7BB%IoodmVkUFU@txfxN-*Yfz%QcBAYv3C$hruKyZ zt{&nI)2*uc)!@H3G$UX*-gIDrw@upF#08cO;Ew#^b{TT_lEWN516Gv5c@A096|Rg! zyP3vZ!z+Plm6NUwyYmSX)k`*e>q$uteQ9R9K^ov+Z%RUOOo(?`2JG-EJmwuQ}C9E^EMkOtO?$ubXK$*`5ycA3BWCF-SCxkq@Y$ z%yq-h7s66Z!7%gE>XCeWd1iioM#Ce^qrJ*|y}v4NP&*|VzlRGk^8*6ofT1zu@HVBqsnR0Drk6Pg!cd!1S*CuIK66Q1noDzAsOv z2dRgOKOoM#*h7W(MolPdok;{_#`8#;bZYqG%C4>3t& zQw{a=Z)d%ANl4utnMM}m%dD`k0MjxwNsLeZ`01n}X zLbH|D2D}RgU5=}>$Udil&~ijXq5W;IH>_T}wmejRo3#qg%RO|B*rcYLL<-@>Y4i-} z&zO}_*zHnjoeQOx69o8&Bv=|ucoaNRf|MD4cW0&*QnU|4#)wSnLeDQNdyD=20|3Z- z(=Dz5)uIqu+p|5Ck;qxt(e^NDK+(`~z9-FM-vls&`xRiN&AYug>aO*7?rCL&N%sLj zQY{QScP>7iU|^D>IGk^frzkzLBFcuU`YnpTf8TWTsmYg*!m;vAmS8@abV>fPBl)aQa?{M4}^kF9xI) zrRb36ZG;!8U4VaiPs&W zmg#qToR`7K=%{rqBwAZ@zw zT#fW;2yHPg%;Ng#m{w$}va&@n7@)=Ce|VR$IYCcC7>60{sX!+IHLP^dU~3x3qOgL_ zs(roVP1vzgkse_f#74x-9BHzy6^noUatw@HWNlU3I(NLk2_aY4+$uoQ?*ll_>OBgu z#utb%LG8npOacAf?Cki-x{+G^k5TCwv6EL_&I4W-r4Rv+;i`K72LYtWm5R$OUmVj% z6}}k8Rs*L+@nOrA%JO_aZZu7je@Czm&m1+Y##TF_p33d7)R3K6-?VLZ+(1^lTUG(b zc{c1x*Ng+VmwfM(p$zr^+ovt2d+JFD{>mvoc4uf zA1o=DWfyj35pul;eXGYfcw94=21c-L3{ON<{wIIr75$!o+H3qW_zJdg|3FOc6X`-)rS+W z@~|oJ;p7iDck^qA?`&doH_9^0Ra+`u3V!))OqIGX?JBq=vfC!*;#*jVOqn?!K2rJ{ zAbv9Q-2#=2o~($W?Z!Hdx9NE$6a9*M16LB&b#cY(yB1dy?H#{QJtv_r4~y#8R8as? z)btfSQKlprOyNqbcS2u}zRw%?0XLqj=R7eDE}r4+Ait(`Z^Ye7v=tTa%FnF%yNp+6 z-vWn!Z}tKW6+LVxoZWG6(}jT9FAS-HZ4<#BCHF`qs+u~bxs7@~KyL$Cuo#GIQ>(f+ zVl=%MwIadZ2sYVGcD)p^AoqbkepK3Z+AP25PKzu{L|3;+4VdY?Jd(RMc5jc)6=PCP zO=AN5>%fQk&vYkk)^UOQ(maM?)J>TtPLogoYfUVl5O&cqux_|5a7WKTdoOg48d7X- zoRK9Z1?aq1C*LN+55MFn_s)I+%n+*|AWz-foMIZM*@X&41p%Nsq!G~xniqoeK^_*- z!hY^g9=z#LCkJ`G^b4nfC?an!;m>W%eO{ZogjMBrbMjR+N?krt@6DrC0Vh&~T#w`6zQ+DNmtRi!fO0#uIm;5Q1B-1cR7lG)EpKsZhRU_YgAwxzR zyw~D4XzO~^>HsIbh@*~BN#FM$U;Sm_c$r%^T6_P}0#>b_EStU7J}pi5#K9)u5ij9} zZCPJoVeH+Y7wWTirPwCwjB(hK$?uHkF<77z0FpBfWstHhm@B|OXQy9^7%oe;S7)p~ zT<;Q2FiezkAd}hzX#oMT9H0adn?5=yC=*~7^O+iy*ml3=Q)8D~YkS_db$5R5!@93` zjn%#947^;DxRCV?px`?d2j0l;iZ6wxwGu^aP8m8al)rs2?UI1tLx

TbNqKVaeQDC^oPxHCB!KQCC#)t?i#%VN70bSzI9h^!zN zXC0eBt1$`{0}^=$R63Fu28&@c$y`YS{PEo~0St6b^7Do^wyErnvw6Duk{qV7S6-(C z>>43IO?9vyeR)_@lO9}ZVJ^=_{!c3D-NRU2(pDVJm31X{wQX%@OnN(zvszTS@u%2|*Dq^0wWhDvY%OQV4P-K>C4aQewgu~W?A zbLvviMveod;)0j`ga4gk*470DnKypxw%oH%`uc>6#cao(%%hvBsAGdbo1~4oh=T8Z zLJ2`~3xN7$;@=a>_=oqk$=FF}lpapg%%?-FDqMKt(_~AQN$gWEMiWx~KN#&3cJ(y-B_2(AV$TGgCgT%m3uu7iAh&t#en8w+Cq(Fl}D7 zS8MZ_im}wCp|>%8Rg&jV$2l{0%#!di+zgSNrFBr}bosS6`Im7kE_DFf93*{dKYMON z?uq!RP%N!-KSRPp2?~j&|@P^TKD#+888+34+BlU0|rQ#c4vHbK1YenR`nt=(A2v?jCXj>;aKB} zY2mZ8V_-T?K!h|&QC0Hjx{MJ*iy8SHmexYJ=#Cy*CCZ@RwPMgq_u!NONh0Mo4h z2fo&{Fe(`4&vcb;_>E|2p zW$AmQa*0BOT&%-(D{I+%G@fZaD91dpyA1Fl2##k=f%cSFf~`C64@|T`K@Kkv2p zIqq<~I9fRxZ?zkpy8+Qi#z@MSJ((+9{VBx^p{40CjS_=9I%%pJ z0VFE z$korU9PkbdW8)JEiGbKrz*c1C{CNJ%xfr`^`S_PJrngdEUHJSws^yBn*U}Rgq?Q_b zQ}LUm@|J>f82O>UEFBJ^o1b;7!0Pv$2N4vCWvJqOT7Nt5btzW^CB_e{M>9}RcUyd* zNNPgb|66N=o-@=O1jH3SH5x14>t!B(M~8O73o`i4bYe+dx2>T?h7w(b`uX!z(9Pdw zb3m!Q?3HOM$~)fi0{773&*P+q@cJ512m*{8K&R0)$0_XV|L8;&;`UcuwH>jIM;FIU8-kPMgI-!gEz_WT8Z$@N9uRa z-(pd+asZU2)tySyf$b{`A@&!uuBui_|BiB;#W9yF)yr8;Lnqf<-w>WVzb@{_SNY1!>L9 zgSi*Cdf&wDrg5@%zyCM7dU-4Z9>*_`k&(l5G~;>h5>+QAsGa~CN^5(yTKe>( z4`-G%0Ex%Z9Y`s`6LYY$tJmuadhM&$@40mMM*n51pL%~dK2&lb++3}bJ_(7s?4et9 z?twq~YEGhK%EaK6p$QviC%TP)0QKwEmEp?~VIlJ za%Q@b(jyneDDCddc}u;r^1}@kf{AG%V65v|4i*85MHuzPahLF>WH;Mu`OaNKPkJr& z%PReM4U|}PfID|Cmf6gB(+sq;Er-@}yL0z|$owLL?2`^UquMq$TC>m8u3zWJEd(3&7f46oW(<%ir6Fo~-EkVY({sldY48x6u1^oAOoJ+uf z?+EnE%q4eZSG{d2`d#%~HgpYN@?@TVb7nWsaRPmbp*;Eo<2hR8g#WF1SD@ z1EqTkAK`_m5)xc}Pftb#NbuJO59~E8ikz!i_KKt_RoLvYdwf(O81t~ag!ts z9Y#=Qa+z`QTcpw?DZuqBB}aAR6 zfw#XzDd~602b_WWr&h|QBoi(7dcVb+2r`uIlL#t>J%%M z_)$cpJ5RO_z|s)G5(?h+^qXJWiI`S6i28pfqOwxdoQ0-;?OElLtj38@UE!A=-fd3* z?^@4d`5!e{iI&+%?~wA%dxQ8Ha=NAyKOa_C8TT`-@i4wKWj+RJ37*^$J61+R1aE@Y z=>NIHukZaTiWK3FCg4hcdaXmHAfHcQiDNb(CE;BMgOG64`u%d+*3ZBp)t{<&OMnBy z4Hbl1_!cG$d<2j8)t&W;1l+G(1qkf}iuj@&+}`57DN)pww-o+iM*>_+%|kx&<;5SO z>aLFeEYr)RRu{^1Me9U8g6FD}kTF=k^Y`TUk6on=?aa=x`+-n<0^S1KVF4vxJZ9Qb z0_6KAP})t!@VIgyWxE;_)nYT$WL8&c9K`Zf@&lpL(X9+vFe5bxN$s}s!2jQExgI+$ zGRWB~SQ7o>VwWIoZ};rlB?xU>6O)A3%0H-oqj~I1(oaA-l0Nze9h`a;3)0oqso`u+ zx_#6|+&8L6Ezt#q-~$vr&7kl5`?~wzroqUwqc8dQAY{NDJaXU|DARn4^hCc=EiTOd zQKT6#2EeG}#|1FLkpDGWJ3LfQq1>8L8mKX3BHL}=1HC~t`-}~!)E1N4cUi9HgXjPJ zq8sJ5z!&fI+8i)d{HrKLA%c8q+ll<}-vzLl0$T{CxzSjkS-Thuc?mibdN2&81lY~h zsE*R;V+sSwhyUA{?Ui&N*HAH>qY|t|oqjSgex+9Wv`##};*ZKd&rAYKGy<)X()$A2 z0-+V91J39)CQ5U=16=#L!fX3aM&S_4oVg%y_}Q_?+f;fh`Hz)%Wih%Q5;;h)rotEG=qQ$qhI5jjV- zN3>?k2>c1~B%6Xt%mL+|@v0|=H~JjJ!0Ldx5m>`6$Dbv4s|GB&KVHQq3TZtYme-R7 zc@%L@Wf&H@Z$Y%Fe$NQ944+@p?vjZKnCP(Ngls5O~ zux%XXv-Bw-4f4V6>_k>LJHY|<{?AQ4R$G@6<7jn+(ne`-cwtf!JJ(KGJxxtq8aXhl z15@y}r`IDg$%^WxSa`0KR8JL~vFWRR1SIt8K~&4

>o-To`d}OsQ!Rdje(pDhzu6 z4X*Z8wc;lMJs;)q_hm!KUj+`2Iq%ST0DDU~ouF8>HHZ$Or=%WHrCM_#UH=409x1m* zm(!|oF}C5>4u9eWSg<9$>T%y>!*4wSOthrkxmvF%D#qj~AH`zM-H+!61%QSMK>w6q zsGTL-%AFaP8Nk#DlzD&3#U}^t&z#R8y=R^q$c2oRYen_LsDLmKpngy=RP*qO0rJ>2 z=Dv0>J@&vu<3YY!e5;Lqh?>z7Nb>(Ld*%KC1#CVL&d*|3cLZ-?3PG7bnUPN*9n^6I z6+9y&R@ls^H03=7HCY@iIF1?7Utui*f@j_UAc=?#02xb4@`PWOwlz>to9|3%kHoO>BZ;vN{ z`HOQt(bu~%=%1#@!>y;71^NWkhno{W8-ZPPjjSF^my7m-tLB_vXs9WcYCG`-@+jUh zS);cf3LuJ{ksg<%j8f4n(HpcEApzLR>mW*2N}t}dN`W5?egP|oHph>*tXGqHq;`S6 z(8)JvBK1Cy@w~wc>~9Z4!KNF^u@vSW^qI{&cE?WKus6Zi&?wfwWL*g>tRB}tWAzyN z02`3~y&N!WK@?{LCs!`7wT2Mf=ilIW9v8`@%6>+;`Y5Qvp`fkR|L16xYd ze~_qqNa^gN-%5HJ-=4f+Gmb{OQ69aaCnl;9`>+|A1w5o{5j`c>(HM_C5kraHr^6$x z8BNURhk?OgeVa_Nl6pe8f#!O}1=-U*u62$UWv1?H2Nxg#VieIm#twv&TNlKxE^tr0 zm-;+Q0X7ra0kvOxn{9(vLsYnFy^a8wXdKFbqwc(1c`5SpJJ#p5?+@48?V(Ri5h(4O z+-@q*KNgD8^s5Sqf_7EFrdI02f0R@vr=xlF^I>JNv0=P@LC-?9^s!3t6V1VsV_pvO zxYJpT>JRbHqL{8gw$&C~V%AriI%#lx1P{3LZhsEP2^5o#QhY)nRg{Gzog&ZMVGt%f zZiC%DfT04PGy_VAPx_aK5@zS>nKXKT1;tELo)gdq0M#u^G26{EFi|(?DwEBQ%WWE7Z4eL1mC!71mZE_)M>%H9hgZLII?uz6e!DAjmg(Z>1)Kd9&$ z)NxhvAw0JRDee7*FT1;2WX@|n)#?kaq=` z#1(-E$BCM@&{Lleu5}H-xRGB!P(7Zb~g) zV1Ht19p|^_vXQJ{Aw5#jA=hC|3x8Q^EoU+ak1j}V5)O^m+hN<=D7$SgFQup}Z2mud z?c+-0W92y55ygdX_TyG9*e-<%!uB1ct6&R@Vw+F&K(qEIW%=qSBY!dW=mGOqsj=x_b%*1vnYJh^+nm7Io{=gH6lPYAM1 z?#0(2v4L29td^i@OxWlW=2Ll5#uy>Q_zPZ`R6GGC9vVO&sDqrsMnh?21hH>Q@}y*7 zUbKHtNJ)pl(Xl8JD(_lG|28v`CKRit?2e{^T7dFLdms|zF{W35&xAzzgH5v4{hNRU z&q29a36xcxJk($RsgRs$oeH~{)w3{C&*5kiSkej>VaJE79F9O;1?Ekt!%c*O{Te!- zFi@fjy=Y^dq2!YaNP)RGuiQw_d?5sIzPgdZt-Dpg5LNoOxbjExxmDlV`c9nAJ1R;) zUnDhFUhER$6yb5=*m}q$){X2kR22pbon4CYEJDJBo|1yR_t& zx4N+ody?<74A?8`iv&x{Qqb}bn`l+(Un_92c>dOYE_X^v_Ftj_OO#ihfPUi*Zx;|R z{ynj7_a$C`Wq2g7-P2!_P}2z*fq*DdP?r7h9?Zzj4D`IadNg${jKkmB&Hb3IrXR8t z+Wem5&D8PZ;@fu8k1X*&yOo?`+@K)Cw=bZs;;ws6M) z!GY|gAezfk0TSv93?|sxg8v+Cv=%^>0ead_DaGc^6+~1ymUy2jSe(l7Acxq;oscy= zDR?$liZ<9zBU+Y@LA~wA1qpRbW4S&+Y5GqIZr%n=hk;GCaaxP+wx7;tkQA(Xt>*%S zU2$k(0mn6LTPJ;1=eJ9q4%xd1)$k7Wj_Nr_ZAJo#iI(Oy!dp>PL{wBdL^dFebcaZ6x+bMX1=8{ZnkF*ZvF+jXyb&wI{mUh|sa+s{vVB&3Vq^%is1 zzG;!Q^4jznn*1mWx+goD%#34&AL(h#nw+*&xT$Ko1+Uu__V?G1(As1{% z*!NzvY941&wb6D;#%)|r_ENgUH6$4x{X|f9*vsYBm*=mO4mzo!5vTYIX>Y{nZTarr zJEu-rzyMq;>GnSHwy0R_Jx19aYST&aW?5 zkqb%JJ?2uYTwK)RG;4{&5^LPhS>}1d2^NYLCe$oGG(tYs{L$-9eE>dFF^~3kzvm^* z!5e{(NiPzpL(=W3XP#w0&OiUxHmCn*8TFS1rKs0veOc1`?21^(5-07=4BZJH} z1ke!+gMGX%x5(yGW~{tPE*=KqP?|QVoD!PI_6!&`Z5me(XyW&*RY*Y4REt&?Z4`17 zn5~1>zpQ$%Sy!p1?p_VKZ7Oi`yJ3zcQhQ~2W0&n}zjD`T1lmY|tsQV3xkOpO5* z326mr*a3oTA?WpFJGx^`%ehePN~6UW7B&pJ;}^rZ%I1d)#utl}>n70e_TUW-$Owgf z=vXYVIEc+e*MK~Dbi}?qs8Bg{dS;PGH(8|0KeXq%v3+XQ7vMb^ejjKf=-$|EHmTW8 z``n63Q8aHHIvOnr(hL@5(j- zlAGn8o%*WQ#W6^r(oF0LErWKBFdSLXATsNA>ri0)W?21$cOTX!+G{qAuO00)I=QU( zT=gCwUoFjTf6FmbptNuQ$=n9V98}Xr_nIH5LO)jRlqNk}(kESS17R1>nOD~H^S}^{ zt1i$OXXvkcn% zm<=Nq_zgQ}{1H}5>NZZFXrXtK_`_7x4L z8K|L%w#boC|f8de4jH^I&1+ zUU3`L2U{Ar75-a1IC%~6YXd7Z5=3S2L2Ti5IV60T?K%|N)KBXu6}!IeEmbG$=78Mk z#FF;*;w!hqR*dR(Z=GJS)%N#P-i%+T2zmsjJWydjHo7l<>`$yd7&BQem(EI1+55@N z&}$&%DY{HPPLCQrSfQ}eEYX#oEe`Y;HvG|PH(bK%Hw&N zl?tyO@*(h`ZN^pC{wXHZNjCxmyAc%cmep6xc#Q(W{7^$FU=Y`s`zmVl_tHpUZT*!$ z{1^xdw!t~;`T%;_Q}gUWXAo%*UGm6=|52VK)y8}1XYstMOm>!6w1XP^zjF!}Wk z!E-t+w00w9^?^yu1YT<&p${*V7x8g{&s)YoGog zBb@{_Lgps|V&waauiEHVjw?KLi`4PG%u~Dehd&DFoPy*kGFU9Wv1g}VCllVeTY>w5 zD%(z-4fNYy(tPMPUD-)%#5I#u`nG>(aY{Gs2o)8ctj&)dAHa!Z7E5vqAs+odkd0)UF+p`2KVYXd z@V$XI_sMFQnD_qRKzbyx3DYWeap)R{-kWfCf9fr_#hhEgv1$^Mc@S;`s~HKJp#sX8 zWt#_<4zOs@SLfANH@3vL4eSrxSzvIg&j!2=udMkg)E(JrcVuV$KPsxJeM2`SEwhcP zqmpJy7}_OqL*g~v=XW3Rl--q8_uOI_vBdE9Quvsz6E(JcYj$?HOYL#^j3p>VR+9$U z2-yjz?n+8kRUNcT&vvdHI@R_UC&hjTu{;H7kPY&@o&}q4EQv4jem8Uw7#H7PF6#t& za1QUmUy%pvZh`aiT_g(4VGQh8&H5VMkZ`ZnGV-p-TOA=-jsROhg9=AC1x2r%U1HHr zW2dwC5D1pd6*C-q5)+CWhy`hGYjbqL2Z|Lyu$f_nIrdz3@a+YIH}-k+3ybQVGPz0l z?GJQHK2~~9i$CvW!D-YC`uq;690jAo46a6q=)7wrLb}Ah3B_lZkR|dr)n99kunk|8 zr9h|iIqz%3HtaLv5~j=1*c2+lBe0nJv0r-;AWvunAUxv_1XWH`d^zwg0K`Rf-J$wU zHxTL36pMMZgl%?gm%b1IbE0kLHwH`BFV__SUm?N9yo*=GD`JnfuQ9S@Fybv(B_~ke zHVd= zR+rbn@JjN<4c=&^f(F zSoy0M;Uq9W<@M7C{D<)?jIfheB%j3_xv`f?EJ7B#YCFxZIHVxb#iY55YIZr7HCn#M z{)J}gWuQ(v>Cdhyf~5!`CDrl`$_+&5TT!c<`;-i?*>Ghy0V9CccH9|yH#Tqwi?Iyf zup2b{fkv7PwFpd>yR)%O+UykQYNB{<0CiSAzi-RT`(UuavOhLRpw@o1mKC5j0s@3G z`15e{yiJ{?OwE@(rNfzv9qMdyR~OXBK1V7q^>;W> zBikTs+!n|4k{T}i+@l_r5Ts^Jtr566WbBNqp)-nbpUp{KP z1xpa_C$>lrd$~n~#f;7XShu&A2d(gJA2Fr-d%GOifk$s+Y+j7w6~KX>^6(*$WQ<+n zZQ|dnsoFl?8_%yfT^6ggo3b7xF^x70J}N2TO4ldx5o*XjMKt1^&+XW7XU2i>A6U6gx$w8ONNA*;+h#mr5!}6A)!@xWpOS=fTxHIu>-njF)GJeE+f>!qKl)fS^b+-AE^x3cX$VI0^enw3ozBdZL&<*WTT zAw_$I%)@N=iz6;-)@;Z2*UyvI@fVLSVbJXK6Z?-%*4wVL`kxlyI0{IxDG5nr4sjC%IVUA(Q1xf0V8Lq-bP58u-yW^!`y`+kw+nwGVPy z=#PF9Qn9#+{p=IEUXCV4cec;&NS$4D0-5yrG10Skp#W!TgRKcQGx4a^AO<2qT%+oh zHoL7mya)mr9_x74Cm2siaAzL=2kUcOl1@4=d`j?Nw9f$%Xbq~wKi>%c*$n6PBH#UT zSb`scUj9X2;B2{cYdyy(PK`#Ai71$=lJwX^don4*tAU(MA?p^TSR#uiDbP-~ZR4U{ zbZ59Tg75g(e6R7i%IvP2!0BPtPT8JN%HgR?;tSvHst}dbV9BRrQJmbxj4v?~_saTb z7Sy$3s>-}_8b@c0-Z$M#oHs3XSbLP4)IUG( zK{imNPInm9D3z{9(9GgqBbWpFCvUY#FEr2w#P_9xu1?W>*}~a1V|jP5x5{+(E`lX% z@KSo-m9OFI7Z6AlQ2)Iw9yQ$E^a0SQ{RDHYSASB8X5`cl2IDs+h5rIN8YF!on0`-K z?Pl(`HaV82sgr-P$Nyomwz_<0=!7vqu9ENKXNHI?&dsqP3^e~Yd-KQbcWRd;pYzpj z5nQd1I2h!7>no2^@@qBR=nIEMjRppM<> za_1#uL#vhKjjI)RpX@8SdKhexIA=Q53F56SOxg8YBxuvT8O zH#I2pA0*`j0ejJ+e!@+lC=ETD<(TrQ_1kAUR5_PfoY`haiV6r@uQqFS)# zObK2;u!Pda!XhQQ*3R-RiT|hilPtV>cXVIinakkI_X9t?GdB?+?8iqW3X-`9xzgGP z8Cz2O)kBhO9Q~P{*E*~aEE-wH!JQZR9{ld2+g!5MdUWSl!|)aY)4a4hcUFwS`gKW@ zLDiyff2reql5xe$89>Te&KrFU4!K2E)46nCmZk=w7Ea1M|T^3bWnPb!P>s*H#9tPh9jf*X&gaVMhj7DA@ zeTdF=ty~XMoIgF&*my^a%9|X{z(4u$c>Q>Ek!I&X=oT+YV4~(fBGJ)Pyu^Vmw{mF_FrW6zULvJ|* z0cnz8U(p9zMn-~>t-mT;ftW|te^mhyp-8AKqaz9^Smcz~(jxoWA)~z|1HD3BU(^sR z_g9)X?Cwau>SS?rb@l5bk@<@7Q;VDV{mj5XOe;_Xs>tMV(M=v$KafxfgO5~DNDle( zjh&-Emo@nh4G0N4CN3O;6im5Z(`SUKhQ`)UYo;aRAFhT)j;gL&OKN7P@YAE;#N?@M zx(`=lAqcVUgvB%Wn4BKz@CfXFb+Qq1O=;njlb@f7kz|SPTm4OZJ{!+56qZWP(tngP zk8i-f&2vt^O~&$%)>sWR18Oewx~-TnEVu=Wzj!c6YAKhit*jb* z_MiCFXKx~Wkh#=1lp*AW*pwhj^NZaerjk|$c4LbX7yC%dy)Xm|0W}Eil$JfYt2FO0 z>bU<48&;oSrs!P)1*~3I!}qA=lI2me*q|u1z_2%~qBg^Hc%&fNTL!%fNu2t&3E_>_n21Zv#fT;O%; zNJH77qr*OzJ<;_U1B;;sDMcGU^t-6C8u#p4F)Fn_d z%w3RHh9+p}pQfe)S@0tN{B@lg3M;v^t1d;$CZg*>T zczvu;!5uQH1nBXq+9cjvl^bnlm#y9C)_zk8jW@Z{0rIphlFGdF^3bdZ|9)*E)=}3V zLlIQ$T2f3ljM2FtD$<>|P9hG{1%n04qU%fl^{^f24c>W81nlL^)sWL<(|zV1{D|tC zj1K)nsu>$*WyMn$b?JY60Uv?cR4&u3jNJjHGT^g(=|o@-Z_C6$-v!k72sokUSZ!I_ z4R4OYc?6l9_0KqV`Zr)|vK9a);`=q!QHQo$v<2(%{_i_-|DW`jO2~@aX;(c?h&Cla zgkiF{&IZaXKc_;N4^2XxG|flvgfuCyZGBH_{tEh|3U0yk8N5bYTerp_F&jOdD-k?= z;y&*Tk_oarFse$ate9ukIlO!$taf6z9pJG+0|CekXPz8w;OTwPm4 z-PJk#)fF`Dcf41W28;Xz$38*(7U_su@_lBvb(N3(Et7{AOitSqzE*-h+~LW5l+IoS zkU@U97w2VWQD)Y~=Zz|L^(u`O?ap;|AJ0Yx9u0@0L$}lt2;FExe#6FWxF=7xYvFFt z1FLuDVxilJA~|ypfk+v$mEm}vgpYF`k&(rFL)GG4iDO2qeB zc&yI_UX#Y+Z`o?#a^>G_3*7I?thr%?=UZ+}ot_&mWbN>8i)o=RI*YnU^o0^zP^7 zP~Df$!w+ey*ezN`5AqpIj>)D*i}R3!_$#A4d8Aofu1jc|k$R^BDE2*a-HN#MNA72+ zdU8+MU0K1xdaOWwePm>2XZ?7o^s6UPK(o_fOtNy*+$h^#&L%4AXpeF(a z+FRGilD~_~MHg;ZIDcP-FB0N+79E!9j2yF;)*&SD#hY`#9g~|KQjR8KuvBL-XkXpp zhd>SU7tjP(&2e%m+Z;sxZ}#quk1KwFMgH96-RXt%*DBdvQv{U2L4%$WG$?LDmoSu9 z{rjPlvzY;H?^}&%N%wM)%2XVdN$n zbxo?-VH9h$0|?u@euGNj_py4EY9y0ZjDh{&Jif)+egtNpV4n^QE+IhLw_Ts3uoz&Ri{r zq9PInr||JfguFSZ3dVVfzxhOHT`}NfIxp>CvuXpwSC(v&Gdo_C@c6-uF2SSpt(`-8 zfvE4pcWj22t|z-G$%SZ}jgWzA12mi7vOP*Im6cw-?nwxVwf}iW3e{D!fnfaW^xIv(@1jgR^rAtk!SA19GLkh#Rvd$W&|u4R z-$HZ_-YW3ie*mf;po=t>ZX+H}zc1uLQ4~}Pq z^Q&d#5zQYZpB&7esChbySK14FTK|huo@@JWdi)ynBy6gG?)tZ16XtW;tE{S{Ud9S4ia1MtrsSe((;Kf30kNppoON4WDvOST7OINpUUdqbN zvc6K#9bLN}?8Q|A1w-gKWMBV+(mbj|^R*1)vd$N#KGKp8A69~qmdp94AEZG3GapWy z+_>XT)N(hk$qT$%mpqW61rkTi*KFn~hgVS6k_Y_}m7O|jh_vBFdG6MGi#k0WW64lW z)*djZZHkeNFq3lE5y;D%xb{|}Q|EBoxVn6yFSq(1wg6DU-F7_$i3jrf*{1?qW3q0? zI#)!zbXAiuUb{Dsa-dpSQ>H91r&X~)SF8AkUGzh4@1 zS&IQD3QAEK8C6mLRhU4J{rzgMB@%RdajN%yZXu13%hyi09T0)TBTi zE2!gl>*2(7soC1a2&dM1M#Z(4ff+n4FFuvZc&_cXm+y@Djvft&Kq{x=ZXisSEDdHH z=cyALt^GiIgh~)4wMcT@v)A2UR3TCBef(gb_=V@B*T{?WiQdN^oi(R-`}GtXTGn^g_7}q zufOPkK-9XQMq+oi8K-^;n**t~KUG~Ik?}(2@P}IVfW;4p!w|bwWKu+eHGKkNm{xwf z2P-yS(Zy;=T2lU;pBN1{KEU}IuvLpbKV_XHRI1jk0d_=e2T&{?JRSiy8elrUD;sY) zt*l7mi{*#@{}4u?wz_pi+{sctorn(RKVsA22DVL?(gRBj*kg3Dv6TM(lj{sX@4CdP zlAqgFEO|e#;{|(EM+Zm^k9+Mo(LD|OQ0ER)=q{v(XApapNtJcQeqZ+@3aQKmiSWZ+2Tg_5&v+cQNoWi^TBj{0OQgq2!aE-s^PLa@$1Fs7*+>A%4K*gJFcHuIz$igzjbW;RA}5F zLtx_1EPabyx6Aov@uz!$?vKblLqovl`TqcxSh4ZvkctZx%S*Y0rDb<1dh(%TUR!X4 zM(&p0b{S#cod~vamTm)}3Q7KHKk>bCwP7K2j>Qq0!k{7I(G!F1D=Pm_tNd~~i{dFe z`Br{O>FTM;dfm)S&iYYEy~$ybvwH1FDYCs*-LgOMoUfJU0^_ywza(pr680aFF7=^L z+-zK0&ryV4xLoT?i|KofVBEd9w9wqitm1gjzeB22Rjo23>eukr#y^dRV4^O9YXC?wwi_IqQT{+Ye!yK!s9p`7j_P&F~!n*ANl_UYhpyx_Wos~2%oRH zS6`~Gh9G$X0#eF5zBzd5%NraF(sz;K_9lphHz>2+NB%w(9r~P$&5Es3l^Znc2;OK* zi$xj|HV*=n2yN!S>N9@zc;K;@Pz~~f{7?Pa)AYLmkA$7I=u1A zm0blRF5)*h6xON|ORC--!&bE^LK!qC4zKb;IKO6fezbm~>o>H#TSOa$jAih{^YU_g z>PslROFG4r-qkjkQ)YjQwVoM@y^k)PNBH_sm_5uB=+)PI?Vu%FZKhIsQc@!>OCfHGD13q^$`vFPJtoEa z?5+-FE$0L;^<(*!!iW&NI|M4|zYN@@8#a=4yf2&_pI|&3OjN%+v_VSc^V}kGIIq=g zoS3xjN%Uuj`qXFAuaZtq+~X>9Bq89t?`@uykTi5$R#N?#Z;CI?g>1B7)03lQ@nU}| z`%YijmDj%qI;-u=jkS4rd-buoT_&oydnr<`4oWQTc)Qf)q44ocmoh0;xhbXB*iBZB z=lpzjy;R`WOs4(pyi>Ah=1P|6(Ejq#Oa8mqSzZJJ4k=72MiNA~NHKjx%txlhjkr7L zw!h<&{-tw=%^m&duiy!Rz@1mXXgx)wFq4>HHdl(H?A8Bq!8#vxkt`o)fbHa*unU%e z+Q&lVrAJfJ4)2^A?-!FzXy@x8dwUTCmkX#D{SWJ@q&$5lX9X0Jg$>g)-ZYSrGcjlwU-; zI2$r`?wg_HaPT(Wu+tp%{6#rvVJ(mkkyVV2_?6fo95ZXlqtj19Qp{cU%+i>VBniFc zQbe!%)3h2GS&}V{4}Er^>9geF8OkE6HeIq@FPWVT$V?m$w?}e4NUwiL5{juT6!`mh<4u8zV!6&~)4eE= zIPaX!Nt#rr@#YR+P&oU1dcq~he-V`kC2)HF-uyg&11aL*VTDjywXlW`@fGY#aV4~z zwRYbX?mkpPA{GKW4NYZA8Nq6>+4?N{XD?dUK6_8mn~IQ3Y2TV4S~S8wFBdar-dAW4 zzSB|XMrsZlmIOhX%bSONsu+8fDYT&Lkp#weO{`}|rwonP#) z&JZ8Ap6EQ|+DAWK5DRSTcptz!g)Ol3JRg2bTwYho)2k00A`Z63RB4G1uywl>i0R{Z z%pVNrHN-*(f>wXuN5AuK%J*>xm;H&$7vBlNf|^o#*v3DMpA1X88QW z?ChK-igBRodAHz_^T0Rqxd)~Q+-_@5$N87HT?@dMzVNycB)$_6)N)(@@_8RG6V?nH zIcca^!IrK4HIW|VR^Jf=hKMaPZ^*~tt)T>-_{;^*82ML?6J#@=-sZY+i}XAK+suE# zR49=_eW>9%_iq-OoT2mSMISC?srC_70noaC+#=CuGOvmX(LrTX7_6+E!zExOh1E^vFbDyGL}i(klL5nB-H8iwAF7jEBBVKRG=|K!ArM zAKqdv=SZ+Xgl8$Fi9%;@keBn1Hhay;7(CJF<%Es6Q79WasXg}_9T*^|^;psqnW8yaHb2|2$7LT{!IBU-T`0VSp*-EohGj%LS zuU4R7U|=95ei_;Pi@iqN-3pHG214ud;jC8lx&7QL+Hm$i&vP(EkxoI-8*phE?b9jN zntXSlQv5H^f+X>d!Z`aZ4TSe^zsc*vhgCvALU`~6ODqk(D)C+HqKqT34DWV!qNnVEnhroezcR7>X5ZT+P$k zBTw9ouYO)dLBV#k{mr9$qn}*MPBvre7fue>hns_l?_k+6=i`O4q4up_v`JEvTO6@u} zFH=COR`WBCJ_~6$KjnT@#-SY|v@n0Yd^LerwD0Iij5X0})Y#~bK_V6wmYdUvL%%PwLh-{ z1A&{@V5Mq}>q&v>|0G5=nJgmB5Le^8uXt%^hI|l8u_&MflORXNFV(^RNwtwoDZg5? zMoKonT7)%eL;>7`XrXFnYirBst736B5ixPa(`IiBzfKDOxth=HGe#ev-fM9XL1P@a?%U z@xe=t|Kev3?#RD9>sApzu$6l|L`Tl02=%Vxc$h@2;587gt&kjuU^AKFlz}f4dn|{0 z83RK*x@`8Dtb^^R^S&Z=V|imO*{0>g(F+PbA_j0$p?`ciMzqI|<1fm?!{dpkjKu97 zD%9E;u0K)Tn{Y`UD%LI5WBG2}RBS8wrYRm!oq2x<6CCNZzx-*>VOnpFM`{ky!#VnWRG{}Cj!4`zf zs45sW#CEvkltb|j5*f$4Q&xi&Vq`oWS&+%6TzY~RQAEE9hNx>pW`^8g>A6xf((`2+ z6_;Qvl({Z)6$?v6NvZErScST#W*)xIGZC9l8-B)J3Atrmywa_$t)6vHemKTri>lvo z+pf~C;B9t^N4SajH5SlcojCHkvYIi{`Jex%8P37PXY}{DFaKu3X=o^|V#;Au^^!%w zq3~B)vw;6ExnIisz22Oq^EXxZ8}rO@^4rMOJXzBG^d1F$ehrH%c7W9VmCIqyimYnN zK<1<1$>C@>b1v&6G0q(6+#Ho*4MSXB;8OJp|9wu_JMW(Bk`G!4ugiI9nwg z7gLU<&0_GUM-~5&vV4J#f0M<7DvA~;@C`r!GnAt`|4v1 zC!lAT`t|h!MTZaaUWju)u%}swtBbzT*Nj9iD6lkS@X8%~;2?S&%$#A>vIK7o#s5i} z5snL;ltFacn)1tnjAzXb8}Hv1%^_7%WwtJqF0L^Tq!8W-6)er1Vk(~$N~+{t6<5Iz zbo^a65~{RnoJo2bv&2H*uCJEWfA8uQ%jY`kv)5_RCw+R)Iu4FOgq2iPyi;KJ)doW3 zg#OTXLoM)hasf?-rmdH&N)zxH`NnfZ9NV!8!m&~K^HgSPexiP#Yqj;>Z+-8muKSh` z@4D7CL-FME_U!GRODo3cMAFPY?zp_=sfvjAWHjxL?{yKv-f2^$PvAF1<8j%_ z-)Qie{m=F+V&2@WJsUv#P;~!GY*T(&ERAmOmCERCrNxa?dQW{gR5gSyOJqVyMHk<= zniY>U;_hs-dmK4ae!s;?F}nX=z?(~w7}<2VL3yL(YjyV!DZ?sR)E1}`xh(Kx-q)qLKoa~=S^Nz3qSg0cFE`DPH^RRJ*{)sW6WyU zr?tAH8F2`cIfP%`?ReC!{uMYY7F6Lu1o@c6CYnQP~Dur}CmhodvFpfI>5OW#f$=I(S-84`$Jh;GDr|`&z z)CyuU#EqD$U`}(HQdyh;+J6a6lK%yu_#vZ6etB6zQ~^S7UKMhf1rfdwikbO30~Twx z?mZ`&ykPWGwp?)Jbv3ieE#kOOQrN&((VY{%r?THDRpJoL+7o+Cbbh{i`{!prGz#l! z?PVR_fr&yq*v=U3Jm~mtSZkZ2V*5j^bHGqyrLUi=R;)8YW3?CAGJ|rwEF1%dnCWUg_?wb?~_`}ae|TCKi}KS04> zpx!p&3GZ1@Z-Mf;t;*pt#;7`ePpV%^-f`F?e|E#c3-|JP=7yUMyiiJc6B!;=s$x3v z;%=6uu_(?99tC-^8B&lFz(r$bEIuI8-jy0R`yALnmdnN0B375 zggScPmX|Fox0Cv^LWfj2X&|s(2-}J3jEh^-`*b$UVKLmWFgvrWQt$^aei>CJ?nxT; z!Qs4cCzzTyjZGQgEc1F@;*fnK&W!URs_C_yL2{L?7Hl-U{C89Hj*!b3K96$Dvrb7F z;H6#sh;p;n)kb?A!7@FKaBSaSp9n@F^xJO?hpjjWnfl=EpW>uF@qiku<2_^lPwa~e zPOUe)8EFBk+U+5ntdmcB93Epc5=3Xvn(cahG?tQ*QW6*Uqob!k^mt4PV^LzXonwu= zlM+K`F|xn>2gN@)EFv1jyaxu@fQbALl&_p7`uT3y%+Ig{L=U-r!giG}%uz?z70|Ju z8w1^9P~)|Ofp>;p*UyGbdXhhjp?dlMTt5AP4yn#vwUDu+-tw z3$UL7fS;mP*)3#BwbLW4+MMHD5gc9IJ0iF{u#MOu!=QtKcH$ox>0sIeIK~C1ed~O2 zw*s`Q;Q~k--{y?Qd38BYX%*Ol4(VOfAt{D89TX?LS*6 zl)}t73pVA4U+=fqf2+aa4^GBQI}7HTI8WF%UL)c#SE*+v=GKOA1kn2-A?M*7+OIhn zDN*fD7qS#vpA@G_z~LE-+^}LoY?DT0$k2oclVNkJ?>m*hnQB82Bm}ejm49q2&Rqnw z>aWU6ZE$_pS*wy0F>;U`%R3s?TvfO7*+%-JVwnqAbtBZGIcZ0X`!ZB&s53{R)_%+n zaXmv1;H5robH`_Z6 zxet(#vG;wl%$5nHf}H97J4aNK7F!>wTU7=MbYx%~y>9<*7(aL2ftFHa>JHxR+or3d z701;bk&(A2?AsY(jh2RI&)3p+>JME@{{%a#@WU~VaQOd?&-(^88QyO+Q~SuA0Oyh3 zw>H~2qX%~Kt|=BcV&zAs1`vJt^p_r)rZ-cM{dy0e>)*bHE-B?~QSfUnKvk9I^_6NI z`GGHyq7SX^rtP(RfOp6y$rgiCC+?kj%vc*n?gK}T@YL%F+_dK>Edja*kx_qlO>@LcWBQ7L-k0t@F`V)TO-HLRbqOM0 zO|NprrfUEMU2wb)0r(63qL;dqcku9bmp9>Z-0I|_r=`u+jk&u03bHrATn{PGTiKyv+q91P+%s14lJ)fUDH zk)33S!OAvikFs1(mz3}9X7X0zpb84l2oMsIy}F*%YtvrDXfqII)uN(1k^xl*2In&V zSSF)8)b#F-c^%P7J5;h2UQuhgYO|kt_m_^sk^W;@<@_PyGHs!a2Fn413%4@e^HfXf z)tsVavDh(rwa%i?hTv% z{OUh2PSW?iBzLrW$N90__xfv`_}C6+@>VHRMHqP@L(B8({G{|#PE+^pbat&~XQnfM z+MnOibe1GTS%vC0zgI6c~1$~lx zdc4))`JN)+^c?N#u;571R{Ifc@jCewV?nzyqM+Kzv2=I%QU&a=HZ-1cK{C8F!5DoU zgpn=#d{Cd?corEQot2lDC#XC0t2JWhjl9Q0ysIIJUwR(=TVaKie=61B2cUQ6Gw7TO zuxt7$D_O_O3*wo~8E``WIS!ht-x>pQeZTRu=U*|X=%xD5Gx)f$e>AgD#%y~46^fP? z$8*)943|Xlluqa*Wv5Wp>*77w8NOb(qa0HNb1-&9pa|5H!Cqr*$Q?N=Gu~Z%9V}Dy zy49Pg`rV4hQYbql0~xkXor21Svzd{5{?6Q~RZrzCa0(OoBSxO~y({xK+fX;5ueqN& z{FGrtHB-oIsmZyWgoj=$x!xO9%$Uwq=RK|83^X|NKFrx`GcfZ!2?+$*;NAmK0P=MqeEdYulQ>0^tu&#>dO~+tkKZ4 zEtb%*qO?swZ=NBX`ugAVY$SJ6;j2|@6oO$S^=NUUxU#jln<&4`J<1_RB zz#5T49mhf`lu*Ha)fmCf)5+AhaH5HckTWFkd;5tqXj`MAmN&&knn?B5Ya}m2LaLG_ zH5$deLR)!`tLV<{+c{w5@-Yv*Pdpek-s?3K^7Rl?XY$;BD_M-kg$6IKyhlzIqRfu` zFtcg?*fdl!f>M%6|CyvU!<~vgM?jYrJ}EGoVG-~NjWe9N+^)&rJkyhewrl30z53JV z$lR2M{9;clwHK62?$eSgd=?m*>eKzx%?fFapAk1+KdOLWewjW#D#oz4LngbKYRr&x z$L41p#wBxumUQHV^O#{~M(C3(0>d}nGH)#V%dZXu2%RcSGcEax-b5EC-39sc=<6Ox zI24n6k8X&4xhZh$yLbD_wIdaL4ytQzo9^@#HnL__P8fBemG#O60c&y8Hd*Hl#+ginTqOoNYpVp+ictp|+L(Hz@pD0s!e;AvU z4lia(i8}qBSHZm2>vZZAOs%bQW7tEq^U~>f5l{>~hhUDXcIKR)UUJ10ssGr}>|v^& z2J?|)l$uaYO}iVkig^AWkh9YAlB_!Mcn;@FJ_SKod*A+srp{!JG_$9aRu!4@{Vu_p zdIl|L1?Erh)x+5h#G{L9oh@P2g`mJNsyL(Gf0rQHCs>8ts;Bgjv$?ITh=?(}C8^;x zW^&~iv(r1R#^y{Kmbl;AN6h^di#r%^#naqY_bhh2brZuDLVeCP{mH;`dJG2peqyc` zN{{p0XSorysR_JZ#*jgDh=Y9&pn<1x+A~U4qiVCm5N-uGxys{kYcJxI%^C4#d*KXv$AX5r*R9dPHTXaamFI5*^f5E96HnVZJpZ#1XqaXweh|W+p78$$k&j-?x8^ zEtfoOtoF7uBd($0AQ%tf_d=xxr9-?Id#RK%NMyC>8k(WApbCK-?F{ET6pUr5isOk!STL-M_uNis@(o?a*gfrf&6=t< z!>o8qSHGK8^pc}bzPg&|ysQk4GPnRfGYv}HZ?*~3LX%o%x&JG`F?mC%0oii<$q z1&_}#n{h1fMegIa_(%$vrH7Y~;>Zj_n`yA6&4f6)K$*0x6UYvgU1Y@+ema=APC6vC z<|>`aiK?*|&o&2_8krD*tZC1iJMd}S8(#R~XCJj*9F#wSX2%$vR;5mqPv7e2SR3ch z{Dq_?iURsW`Bp6@j5shle3ahD=RLs?-=7w+mN2AW6DIj6^%gEg_N*(ry>TV$)reWC z68c})^Bv_YDw+7uqzGiI`tor4lxC5ZccpLy6Wwcn6*x=pvU-xn`TG^?p7vOpxwk_n}lyR{! ze7i0655OQ0{y4M%g+FJ}Uh132Xw#>=K9vDukDb#R&tsB=bQUZJd>U1n9R_G}c75*P z8o|>4pn9Id;}!^(@^{xnmHS8mN(KomAGN4`l|CpvX%M8@YvXmcJDXdUS~Y~2y?XV^ zv#+RqwojI8A_Jg6K{Ij!Q#hpWypggYv0kZT^$}SQ|Ag%Pw(v86E5#p+Uu}?N3!PXK z*l$ii81BOa2P>mc8^>**RI~kF>Iv3Sz8~vj&l4H@2JR#dG z;H_Mq`(li-`Ek4Z&i3AvGWP&(i`Jgaz&4eb6!GMKotcXk$6?-sIHy6=*OUWH|j`Crb8U(MZ0z4Fc;ZA*!{tHPg%TUaVUwX zwCjT0qrMf(0>yRJ^5k9`_fT~A|5>VD#vPkNeu-qS;Z)&VzK`lZhs`?gY5u?1dJm{3 z@3(K<)}PusYuzHyR)&Cp%93qe5D*aAyU3CaLWl%HYOPQ}poS%cA+lE@D}g|Wia^;i zl0ZU|0y0BDMhGM%dB5rJ{lDiteg5a@(Gw*k;l8i$b$!-_!d=$C1mQ1Q8IPQ4zv6cE z0Mh+%-e3Q-gdbASll$SvyV14Id8=9F&tksc^9c>mm*$hLU?Pot`(wp1GlbswyX|DA z^+H{MJI-cb2>Ug}0G*dt^ZBzknX<(c+NX6F(}9J_-(Q-wu|K{F$N?ZDQVCKgE`0lI zdnCmD!Sa&>aC1OxrF8FSuG;STQXh_vcgGzje_6 zF3?x*|Nju&p9%p=espZ#=J<-UFxa6sioE}%Mdban8E;VFuM!Ri9t5ie|KVgxy^iNE zfd}spVv!RFsZ&RBl$IsqIe-52o1^Tbg|CW0=-$H|RaIo}cL#58(!@2!v*QIjQ-Xuq zcsWAzE^81q;IBK2+7bs?-6YYdwK_6e-x1E53s&mLI$5PEucZUYZ z8RiFLnuHE<#Jk)~Bgn{e*gZQl%LiB->m~Sac5-8SaYOC!d*#qu@84!INLTG~0%0HV$b< z1qHNKli6`QeOh%?7(=V6&nUD47@xCK!~TI3>pu89oj1PfhO%{MmLPYF*>&GsAB_(3 zb@TX4p4|!2RrO=)9vgJEOfCYVHo(59Se@a??Rkv{$+tiqgMR^`_;o7;0xavryX&Xo zOX_;G5m7<)Au{qPUy!&!yaL3USN91?!isuPfXseh`H-Z}9aAoc{R2 z-RUlJqN&;p+|F-`O{6|Y(KCRi^q09irA@1z0#&J+->Wc@XJ4B3rQ%UkYR#67ST_`)mBXBF zTO8Itg=}=E7zGA+aQ0`qWsInVC;a9q?dVXJi@K|2yW?iR6cL~IWQi7i%XpNjOr%I=Ju#gRN+-P~*P8;vwZW93aL zoviA%Hpoj;+ldJX*!p-&!`GWji<)RCcMhavEPk1S=M2QJuNx4)d^iqvM{>;sdpqX_ zPSL=Znxty}wb=OR^IzU>k=E_57#nli)G?UoS18ivKIl45s`#bcqc}OFaq5YlGoxC< z`k%mDmL}&vI)ZRsCp(!9E;)T5X$^5G%rrAONb&q!K-X+B=v**tYbi!0fNP2@PF1Od zK9IX${n#FLcX05fqM_)cj|uF;sEHNlMP~LcJ1FH-QdHPhbSZ$ zq2mlZAmvxwVW0on>KzYEpZEk`{bE#sWH<>#x>ow>wBWs%7 zEwly>9=KTF8Z3DsU%p*5Gh5v88@%~-PthKcLI|aEsz8Aho?AoGb4yA2O4Y+1 zD5&6M3O_SFy=0;xLeqbw(vAL~uRX!;$s%d}DqDMsT%o!gh!3r~oIN)TLP_^uJRK_o$7{NBQFnoN9Rkn^s-MmUWoMZ;GtFCXhY^VMD zq3Dl1cQmR2WqZZY9OEtUZ(@N99}#F~=R&EdDEWLZdtJfjk}8%rxQY%C>b)(x$311S zPTu$V%XkJVz2wb=ZoRHRy~PRPlfG3Oyx2W(I=#(F?bN-#rOcsbeD@ObyKhZwLFqgt z2()vUD0!_ehwf8$NOJ2?EKv#c?p4Awz_;t5E89Vv-NC?jF5VmJoRK&Cfd37)-lQ~d zM2ml8!a+{y??a0Vsb0z~hl?JF8!zwQwokYN@jqI6g!gE?S<^|HC~g`2E9@_L_en;? z&|}ERDw1RgW9l|hv-cjB2h*7u;TE73oUKPBfYncK=i-%bsd{w+iO_{CW{d36J>|+e zJyn6pHC2axZz`&Hl1kI6zWVXf3yi$eVIY2c1dqSv?y?%oD!8<7-#%J%xEHqR%)#fI z<6dctgZir(R5qt6>goH3cjIjo&cU0SC933wJLc(bv!x%9q=$4jiLEP++BR19X)*A4QKtAZ{27CQRUeKebR+ z30!(R`08pUU&I?4R>j|2Xq4{Qo`sAam8oHe5#-#$elEyTC$-SG;nT%wW}BJV%cCZ; znFX5#0PIQ8NhOEIMHG#UACnI5uPJCXgY_3yZ*a#c_;__rPl zweEaMJL(lyIu0EiIH4(SNN=^+ZQpv%fG;t54`&N&S}i~TLK8yLTiZtOHU#y})jxXA zv4Or1PLIj(W&lh<|5z|(X+ElQ$}u7VI=BhdD3etJNbsB^#q{P7B3IsUYy2o=Pku%k)}-yuec-U@ zl#cOuvQVT-5N?&$JvJybcL{BezAqN7u09g-*VCC108AFF`19W%8_XQ_=UBHsHJ$Hq zJyxR*W`z(wL?wkCntBVN4(*C84=g%z%`c4r~xo()Z|82 zF*v2&wdX!Gw@;xtTgZu8R35~G)>&>U>>1Y7Q58H{aV*7>~E|N{yT#69g7(MaTj{X8m zzURuh^F}|q0T{KmA@qVQjIJ)6zNJNOj6HpZaniFX}pNC(U^E>Ju_n@`ep|xskBVR?qRFkd+}wLFMk4;^h-HurbqPvT1C^x zY)<5Pn_QIg@9erKJD~Br=YFyo7shTNynWM$mfk$FFm#hq3+8cT&G z>l={HMr?(-acf)5*i&mGbT2LfMywJB_F3kNHdG2$A`4C!9|8lj3sbM67GUx$!iZ(} z8Hm{+WF)#UuOeC}HLX?Fn6UB6U0leOE~swuiE!40Gq1Qk<{h4Wu_!Vq*4fI`5x2Sb zt?27s-38+}HAEcD#t2SVK8gxy^*>@~L|h}Wn>-hd_T}oWBJZ8P1+sCV%yI?)kvnVo z$*^vptD45t(lu%t*^qZRTMit{l~8uCW`u7P_VV7dy?MasCOt(Ni%ei%pL~5WC11AV z-7lI3J#Wm{Cws%!#=qUOH<{{7+4Y3z>2#NZPPevtv$FYV_*o8c0cdJ;RD!lX$AW=8 zqjUD;$vjchFJOt6nHjkDDO4)Eu3Ki5gJ-FsYXvdvclR}b$T=AT-?{&cu$_DI-4DN9 zD^XcDl*=$F?1Q=3aaMO@jP~3Z`rI0`CAhm;7^YcuNwm?c zzPr(x{^nl;0N+vWd-A*y-u9SrThplCsZ&OX)tFT_2cOIxF{f@mIah5jO(r`Sm)F;0 zWKXuoz?wic^Z{39h~fHQv-Mp$SK5pjNU;I~!IMggL`he*=$4R#!DtCyK2l1B(JVo` z4)|TbRD%GjL<_9(+$gKP{aymzHtL*-O^QdTDEgT;G42LzcX32U5s#emtKNJ+qpycB z{&aGLEpD&`ZR9_GtmEAssmFqqsjSs#a{zF**1(qU=gWu(TWMQcKQcBucCNWtgm=5y z&?*j)wh78d$GQxytS*e-FEeT1iNdpgfY>Flxm$bDnsCb%$hXWGbMVHENy1cZQuymp zJ?JS5IRC(^|IpI+jBVC@-`LNn4oQRB<-qG)B8e$K+6~DT@yyYh#yXG_4}$aKy2#`K z_|}AB-Fh(I0yS|#>_@RlI7;?kn=Wre#MqTXwL>vuAD&w+B`RXZ$e`iH4J0Kgs5-4T zQZTCQdggHP5&oPGYnUa%*_mg%V?8@24U>eDlV5)|)*=qs+`+r}vO)&_sM?5MVcVC@ zD(RPz!SC%{jRUIzwnF>#IR~W9|HhwHr#@O7*$Z^fQZbDMPs8SOc)(-Xs4|}lNv_$QlC##M%`B^Qh1N^M=8v3(NH)Axss(ulIj=)1{fDzhav(`sCs2t4)lppFOB_ z;cXv5z$R6lh@q(s(zj}r$g-h`pLN8azqTM;X)kF6C&Ddt@zTZT&t9sr?_9K&W;%n7 zqbks?l$n`;nC_(MaZ#b6vZQ;L$LvgVy9z1QOJPJg|1Gi?@ISUX&{z9Gdx2Legbs|W zu^puSSypo|Qm}>nx<$r_spmv6uxVhTPpF67nVibu5f+v_W_~DBum6ODYrYe}1q}toyi?ycy;yU) zqx8=wFRxhXM)X5&wr{~AFba_2ibgQ1a9rHpJok821TgH=4i`*tc1!9WvBc47qz_A7Sdid^g$G1&l*O!tK4ax>L=D)_=)T_}N04;nkRl4W~=WYLy^_VWRH^Cn@xgwWJiP^z}sGzuOQShgr?FI_>Mz$6F@QfV_5a2 zs9iU4ue%|an3)tAp31KSaDOnkY_>PnGc**MM&Wz?JzkcKFv#$~SEyjv7HWmbog_a! zGHccY4IH^sUYOyIQoE2P2axUESgh}0KZO8Q6~4Qe7Bt$QBh+Aied}kSWdjO%%A7JO zOE%cn=8u}Kvy(s>btqom-z?W~il@EjwTf?g(YKXR7N$+rAI^;MdwqeBU*%H|G2$>n zJH4pKKT1MckLqs9pfhrFl>w^9=ksNT0-KznoF-~O|EsHf0ZWW|cHmeSQ$jtzg4K9QgLIEwDO2cDe`mMdlNB4(@xm$4BEF!MDWZA`!QuvbVhQufWkRFOfv$ zaqI5!8Q!l=O}(rY9(T``3?Pz|a77uLOrh%4f0f!jo8?8kk|Y1~(OGr&$Sr#I(h=DeYYU48CW80YJ=%wKRJsbR@Fu;6 zq>s>>jsx%qt5yLsJ8!pg2p@yRyn%>e-PKikleuF9=~H9BAG&)f#>R?RORG1>`!wCgX4r? zdd*58gaUhRlJEQ@=8>7!porC_?+tg!J^;*&8w3IoO$pU=1P1(bnX(n+Ih~^wm^LK} zPEJB0g?CCHy}S7riBy}kw+TclfsQ6cPsO1ivOJa>C*fcw0#d7MQJJnF222UaR`O)fAa24=regHz5g%oS8=|)_iCQnFe@C}Gx6j?e$+2S0Gh|2V84dL3 z`t~PP9vduff>WaBr8k+((iN0MCHc-?YPjNb>Og=zB(Q*9DxrZCz8gMtfZbFzSp(=C z%@Hp{rNvd2xZPr)+t{Jw&y{8`vt{B`hv9iG#6jATAYm`gw#bX;KSWs`@3e;_X{^ zezGnt*N2W;%xFeGj{oUp1z@y3rCY7oT3DF>RHEvl-1KPJ1}+x5%!YmB&41-4AHpT? zqh;oYD;x-i)2M=G@r!OC?*ME`mR!z6@dHhT@{r=S0BB$9+4! zg}F-T(W7o9AD*Q;eRBIq4Elvn%=6 zMQPC^Gx`Phd~ZdkU(3)@x>eaR`M5b407nm5 z83&}G{NJK0QL6-YgUuP9c_HNCKLK=Oie zYKhq&xBkMJ5qS&R5R^WnNZuo#92g8MZ83TcbS)oy!Q1=?kB^S!DsHA1To?45;Ex`W z>YX|Cc+dFTB7>+SKSIv9Wq^+7#qAQ8Py_^U@9A;_g@xtv=697fMoULV?h-dg+-voi zs{kWK%&PQd6s(L5=)e$0!n(;xIUvRjvnVVmeQ~(F#y-8bH+S>>0Tb?0wcwOqT!CnJ zBWANxy}dF)wtcxoTcLcSskjSE1O;DmE-y~V#=U-o4%Y_e{F%Wa8{ePnmm&55O_V^j zl1a-kd?@`M>@$ZHBuVGj1BVNiWx1p%`8!6t`90+#A0v0|HBOC;J<-B zKz5b=lr9b}9c;haK4gvp0URPxXsG{uZ%3U!fTU)IhDQJBk@&;|s_hf4mva zTARjtMBfsxzx)2ks&D^7A}BZT1rJsRsR;T4(zo%bSN^$~)Qa_MrHWfFihi>e^%)6UWxWtxreI`gJ%I%jm?8x8~au# zprQkbkd>U((<+$v3*XPd9yi@!V2V>h*V zIavU55Ie ziT)IU=5p-GE;C6E@QCDB(H)st^I4rCl6OH^SCD4dm4cCt=C)CYv39RkC~w+2aK=sZ zCYzE8LAk^I3*)%ehhl=A-sQaew?5a6XZG%$muYU!;-?*5HC@Xo=1+U%9+Zu~0m^jf z`zzT|)cuW3y;vX&>zdK{^~Xve%>e2!!1sC^KW9(8|8V6n&}Rm0%ncak4$Z7BWVwV` zhlKPA2P6kKa;MM3){e|a3=HP$bLY^ZOC#ohD0&6ZyAo=dOSc?5kqBLXe%A2U_aA`; z6r@l*)^AZfw26BugYJ(X8*uo3&#%{CrDVmO+G_;{mIR-6$>}Z(uxqgbz6C|0>c2Je zdk&voGkp5*{HJp7+1?I5W0lg?+X6)2D=pPpewHDva52}zk3DRHXM+;G1kU=X+;@W8 zv$H9L*{tlbu`$#;+DzS|NT5!((8-`JlenH z#zMb>j7Zq&yuj|L+#lMDddrb<^ZAZ%9><{UsG@n@T*9Nz5GVj$?h!Zpjy3To7^sqg z&)_5C^?Dh#iG6W z21sqtudHprNLK^;&>7X}HS0;1-RMbQ3358);RteEM!&(|k+hso9t2L8kakP8dj7t1 zX12QGlDvJ~=a+r*kiKl$k>_WOj-%UScK|GVZ)J^dB-{)E=b)q>Di*dqC^jj_#fa&w z0{Y50wsMWFLvpGH2?f7d(syZ8W&rGhdtCHO;98Y8GmpSk^EzY;%ITpVRr3Y(ix3(f zYwxi;4R*m9V_&Fj$?pEW7d$h7!GnYgCi6XUAHoHNVqL3j*Yy$#Ry})cq#F$Gei9O{ znI?NFZg=L`sl?k&-aV$^(9X_ploks}L*xM+(tp*NY-%>!#MrqMpoSaM0mh&1?3b@* zOkyqs9R(xACsw<=SBl3G^1$PAJwhH9oF((fdb0=Sg<)LSP}T%3rdzP&2U9tpsH{3v zY$fLZG@JL93Ug`}&66Gq zM-$aZeyK*W0p;$ZS(AeaKY;_ByPIbGX{RS+3e+957X6=^H`alg$=bDW=S82%B9JgTMBKWM+LT<{sm5-~$pe#~LXcLJjoh)W zSssY==lk4Ys42PwfUB~oO4oj{@Vxj*i_gT*U>ub%9q0CVxR<=tXrL2GUse{K*YLG) zE-Vn_QT&$|M{w7wE%iKh#QskGJJF6vejd)^oE@E$%<(Q;Yw zmgsGNcwTwlcwq=Vs^-8;1xg@R`VV4MYi;h;VxEzBunmBS;NsEEKP>@m2y?ktIZW{i zY<}^j32qUTBxTU-ozHCFEY*RobpkLl5;|ViqRaFpwk2{C7TP* zcXK-ZRt8Qq?E#-yup}%qum)G6RYuSgx!J$%cXL4;mAGV27NUEoY1w~-ION| z4xxr`Az->q%~b4{EeaHE1Vvw?A(m}IB#Y;DIzGPt<@5!{E@*c;7N~Sk&Vw+l5IG-3Ff7T6cop1ttPjWWW@*z(6T*s*Xe&pSZH;g$2%EbS3+mp03A#etrKT zL>rhwZ>5idbf>$GL>d<8xHI{dqLA;Nxqvwx^3M$3$4F*LACBDF*#Sud5xi4reLX#Z zdz$+s>zApOY5P~lG6YONh- zjitKhEC@kRU}!2k`KOMOx{rPA^VPQf8yDb#xB~dMcdx-Z+4;#TtlajO{CGYUIP427D>iSod;t$a$8hTXtWSVAZ3f8Y?4>-trK0 z-6#8&0!c!Sb7nSPVM@eURU6~rtrp*|0ASylu&Fn}nTuM>-npq9vT*&Ysd$I2fK?Ou zy7g)7Af?nJPpO(X^Lq_Y{9^msJN((#PjAZ!=|?2<&@v? zC{Rk@`CRC549JV^xg%6eU3 zel>6Ox4E+{Y+cEP6RO8b0cm*rqJadZ)PhF_BtTb)JlFh)!MBb(peusWn3UQ~kei}f zw?5g@0_QJQXJpM8REC%(-AXf#D;+e1Jqe=Cq^*vb1gG|`JOthezXd?G{}6MaI~Y!| zNi%}C+$*E}emc#Fa1nz$fCAs`oJRd-zeeFgn7*%{;^Ol&nBna;|2ltT@lJr9t!BP0 z{^Ce;3+4W5mv_BgWO3T(q8WJ07amZzNAmYF_uRjr9)1V$gEIPEq%}lU(pZU^(amB$w`xjz^p2JTd@fu&B7| zBCC;eI3F?d#BFUrx6WTc&8#dL;1^bS`>|t~LD^ro|5+fM67X3=Z8d2H%u+CX9NfNn zQ1iJhqU2{Z>-y{RcKibpVeTAf=gT8Jd&vyPrio#g7cfSga37k`4v!Yt%Yy)AZdjrrbt!|cWOvDrO*7Y&-gEIv>=M#Icoa1jwki;Ct z)eTVgMShCx5BkpC&%6?rHitD@^@7wdyjW#0u^dLB^7i>s` zw|x<8E8+T0x1E8OUS+ezdZD%7?9!I8`}fH#Ky%ZwEse0kae%DT8h~oP@G+#w2;6z~ zD0@Qt{F`F3n1Z6YmzM%;EF~3!dyT%${mo zp#6lEBqUv&%ujoN;|=gPN$k*WE}!l0o|#Vu%SKvi74kD@$AV{UBB{Th%unao)~A>( z9`^0gP9N-vTEcv*LLdI2<;UQy; zm-;KO-ZTvMCRPgpVq&;%8d$#yq}f@L+J)Z`P8~bb_t@Ym|NS#4d^Yo=;t!Hc%YO=9 zhSE8Jzb?8BYJ`Klxi6&{c4O1@s*4xa0unk(stq zA<<4NlFiM+A3Z=1AqF;D^@AgTJ{vjVUL1pLRnt~AWK;%X7WF(4q1zKfb1GWoVV`{P z$pkEOUtT<-_qe3J1CHd6#z)PHg{(JKfEmD13L9xV>=utLtI#cl*6#(edi zrG4e+<$4OdgLOiIWKPIc32l^`*VFLvEHV;;yL=!Lf=XSvUY5|lUbbQ=J}!&|K;0rr zaZ|V~ic+8W5=^~$qCPpFV^&`Ay zb(q&*o0iP1H3XPHKN{+9sGbL#d6HtRj_n{2i-6mc6t{B(1a#gAqDh8%sCd;ir{7In z`HNN8uakBDxr8U)HEnESYWE(9v(S~Tiv`&5Db}k|A9_@_T{Z#{HCa~^8;Sm@d&nb5 zEg2(Vwg8|^ZN3Nns;B`YBc+1e_Ku~7GyL^&6X`Y1>rEKy#fvN{D%8&fan-&MQ}EFn z9Q8eCd_VSrQ@m2-*qu0VUfHGG?_?yR{dC&XMr>>ihX8|F(m=FWZEjxF`YXeSzdQ8w z)Q(bXlsNcTzZL~OLD}I`Eh#C*ZE`>^iJH}9cUijwGeotgqHEgTclR?^-aZn&m17uQ zwZo^G4@c{&&0e1&75qH?`MRo}dUq+kyL4eCu`<{AGGx)PIa_0_q`X{3sZtP=2{5ox zV9t<3*KT;P0=ouby;p^90ZhE`=|n|a>)|!nD8S!9{1xSDPcJN@;P7Ev;HwA0QkU_W z-Eu!&y{ia|OWn8U57s4h;o^}ox-{%wVqp6dwjp=Sa^KgU5EPgl*&cK-n}T1GYt4rBGZ62_FDSK)&v}diG4>(j(W6 z0shB@5cYW9>Dd>Ml*~RJHOes*z_zE3-gTb7dQOj*zSdcgmtQg*5#QMxWAs8}tD2BT zEC(FsS@VipWti1Y@!E~xzsccj@cI7LCeI4Vwc`Blhs#(HLNgzxm>|{+%JU{`#u;ZuDi@~yN2qi7ed?e&wx#-t6*k_w{Ee}2;7=jam} zmaQDzJ%wAvzIYj{1iXEe;}GlYABZfJ#LdKki~o>Q3Vj^EvOQW#fORTzWwDd5V;o{1 zc*K1+1IegA*&BbK^pEWXai^_$~gG2N1}F+Nfk-ccUukL`;A! zv_w<_N4pz;7sCPeR~{kUFsW1p^iWDsao6USJ#*WeE2y+oaC6KZSXKHQs!LHL1m*&{ z8l#^a100FffCE(|tg6WQ_qTaoqs2~E7mOGE5<&eoCUwQ2%uqV-76rd(vtJpMu$2pj z#w>bhpC}udbKZg2x)M+eTYhD=N|I&ihQdy(1rY}yJ^Chqj!PB1Q>4Fm3QmY!t8N%3 zPI*ZM9m#StmsmH0q8&s6S5^LczmbzWJutB&zlK!eSf~#18ak&2tekXQcLDm0gCT_0 z*vNGapPMbt%vky(upLJY2O4b~n_6Scj^2sQ1e$c;3HAOom8=HLIhEpsncHpC($xYN z5K8M5REs3goLucq*iA#g=w42od|Hof^a*T8P0;3AO~XK<7>LI@Ts$MhaI`)a7-|W> zKM|&IDwm};SZ`#rxt|gnXL%{xL&#mzTrJK+G zUB31!iunIp^!Pfh#@+3FOnzPv^~Je~@B#cU@Be6Cs&yRB{U(l56XfHk*eOk)Y^jCx z8}GiATiM~8(Szlo#3N?tn;=%jnfSQk=Yr$z(bw`AIOHz~r{yNkYnoemQ{mm(4f7l*5Z8 zBBo~3f2d$7WIjhn9Z%PLAQ%q}*bVR2ynosGBr8f2I9uGE&#GWu9Lh}PXq?z7Q|fAA z4Ey6a8>obkqHg_w3ZtEk?2@ocZzQ}EZ6y7K<9>8UzM2yZr0rPi{G7bP{7BvZ{z#4Hj;+xb!+Q*Hjf$LJf0vd znrzUU>NeBZ)8|bH>mP>f{eelpZXL+_op-Sfmem7;jnZC$eQ3R!6W5Da2NRs5>g(nY zQvuAC2n47KHt5gcK#b1TAQ2rmLe9#fPp>C%mqMwLQbR=+v-~~=HgM+|`sC55b`28{ zkRua&IUnpB% zRzN`pZZFOfJuge9b2JB>VDNb?S{`Db;N*auUv~HBdC3|Q%>~q^JXvob$pk9o$!ZQv ze&LvZM|G{>IpAx*-I&oATgpz6x3!_wN2%-_3EXR`M+LqY36!Z!`LJ&&pa;JfE3_k3<@dWrBvZ?y6;ySnmUf!X8}67zFk|aqplh?-jOwGvMKsye*RClbEX0_mtgAJJN`Av#633t zzR51s`MLq5Kn=4)b61zvv!#~T?eY7mnUO}VVU0@dX z$RBN64|kYGnu0*$C&7}4iY2hKfWMRx>83GK?7tXQN{jw%i8O0KYpF6Yfp_7XWTi!d zr)@p(u|jN_iNF2!+tG45CknBiUO2Wq(GWAnymQ8_B=LkI4sCB^Q`{|!bS?Vyj%4-i z=BebKPpIP|s{Q8Q`wiWgK8HFzC+`m;_WBi4&E%M`ya7|S=hvTuBSD5a@J^dZJ<}Tx zgOeoK=?cj5l~37x{=aHbAv1dE%D*19tVbe)m$|=x>mNR(<9n|Pkm-1>?cej=U%9{Oy~^CHY?5Oip?r&GFntTIQjNsE9UnzntodTHfpNN>h=+DjZ1h#d7|o85PIm}>Kh%CuQQ zMCjRasQRj1&tO)2Iq+4Z)T)ZS4;^hPl`E3!t9bwu|pm4M|)`tz9 zMLr(J<^oDqFVetiDBgir2~0mUBu|ifsuM0yM?aje)Xu*1w2ZQ0XmwzI z|6U7=6q4J-OrrP#Kk~)dDQmN&ix>i!5=u+Pz3NuRt;s&Z8$Z-g-SuB*yVI&I(8`TS zWbG1rZ$QA3-tmn@`m`4vC!I_zxNazlC@Q$##(l*UQuG6ALRya21B_`C;C3Lk#j4WK zBi{pfu*U%82E;bSpqc_D>%o15cF`vJz-Qely>kAeTdj}nk6|3@O`sllZNd7V^d3I7h&?o4|JbE)~qbpX6A*hOy)B$RR1F!7HO5z2)lopMcG#Z!D+74 z-h=#`Nm1cR6O*69T_wmlum#9N{;w31xMj@N6zv*FL}&n($uN4uj-aOTa!mpW7S4kpOUV-~Pv{ca30m8=@9qM#5?^tSHb_m6yMRXTqzmX=qp#6iIky*5!`e9pi*e`XsN4xeihq!up&)@en z6mr55KRDY_xT~D6{=T;cGS8`}xkIZz{)?3XrdUS;_Fn|fuQt{;qZeZGbC>>zF-q^b z|G3^QGt5NCTa8;Lx)STQ|leqV&2@#A9y?SXefWIfOLw0GE@teMoC< zo@Mg#7NxM|Zm~6xi?VBrQ`Vt>zF;hJL_cm>G}S`z)oI!bbPd_1sDi=mX;Xek6K~g# zFl>2^pwk37+rHIl=q#8nkin}$+6gN&xK_d3kA1;-lFn!}jYH&(YZJxeTOn-Kouwck zFnHak2k~E%MMIQop{=w64eZ%cs<_#{=*I2$bAM{LhCv*)}Nb?Y0WEd&4Amt-YMkIKIiwHad!X(3i7}$I=w*L zqio3PF({tXs)9Y`_N#cT^x&`Z!crTzoW898hnl#jHw>n#{uh$if z*+O{xde(OpXtdFBLxRt8wTPmoRp!R9H87C7PSzoUKtt8I``)4s>@qFY7q^)*`?q;H zUAex*T{fBaWu196jg9&J>nU*Zrd!JBgKP#=D>_0pnOqaCWBCch;9SuT2DDi3`JBfO z7<<`$oBhXF4w@t1Zvsy#fx|mAYlg#D1KHOdur)~&w2SZsmC%YO=B09rwE`XG(1-B~ zu2(ofT)K)Qj%55hHTm5mXVSC378mmvqX73Cc)^Wg>v7)~3@D^as)-G4vv z`5Xi7vr_wXk>!f{CBhUH30{V1Ed{TiKmVq+Od=eU1ecX$Z6HD~%)d>X+jvgPwBj1P zDTiM>ZKgWoIj}ZUR2`AzSIDTv#&K0oouTrsm(9(6f{ZRL?H+vYd?I^D3ay+*F5&;O zs-S@cl7fd=X&LkiA@n6sg{_9`zYM#*x(jV!#)U-$u{eo;ia&?{{V}lk31;9SbjI`= z{Gl-qcxsYbg(o^A49C)UtmX}hRRFZbi@agxl`-J z06&a$&^JrZ9uywc4qGmQorKqbjD6dubXqm<7$_3cbva<|Ne!eMySDVT=P4j$1K|p& zvOkp(p4=5X(I;cVk=Zec6K>BwA~2; zfs{{{kpswpm3|{wC3!%h(7HtzT#z*!PB?wpF^u53_nC7MwQMNWeAq^?6j{@cc8OmI`UMcDY zs*}IAyig^eqtpZXEW)ulO-K0SSbbV3@5X2keI)$DmYSZ5A_f=HRiX?OLR~X6Tf@?J z;oaOiN$wO?)U0M7Rjwa7UwB4Ib?5|43P_a*VoWJ814fL*C^Y=&Q19l8cJfVD(q=f! zXRmI*lID1uydCcc7NLU&T+(vhOsoZ?Pq3l5VE3JYfIAme`|x`G{LkvA{{l&ejc(w} z3wJ@$B$J0OJuyi994KzEKcRN^xPz-KfabWLfYf-ZHnYwjZ0lM8=`4!ri(_5=1wny0?~P?7*Jj~-5rBnAH+L_T>g4?Hgvn7+bbRX>hKKVwKN)f zEL0q-F`hbqksLT`_gu!1&}tuk7$4p+K5$T)(X?)KUJP>W-fh=*r1N;hK#N?X~H`69c!8P%eJRez^aFYHR2-5bad3 z83zDCO8Z-rK#%d_$#8!HpT3b{ti>*oI7@UTVnZdMuEz>JYr$Hx$0Ku(LcR$Q=y%dY zmUWXPx7tpzo=XGf0;AaHEDEE%`PaGbSJFWdgT5&Y#^dY5SjMXlIe# z_6M_m{DHmCAV9lXb0NEQws7>#z4AL*HfHydhT87gEC3s3Fqb3y zLRN^Y7oRd8o{d*1rE%kO)c=vdXeg15F2$XRD4TGc=!eF`N5i*d^1P4TZ-Eok+8jXL8wSjmdc>7T_Vhgl5hjvEor8YfNjin4C5dNFf80PCRfQo>CfRxaS(t8g`2~kmyE+C-vUP6g<2qm%z2uLTPgral^0fA6M$UXVbow@Tl z*BNHktOWAD=RN0no?labp}Z%PdCEfCK`KSX0__-E*-5Es@yv4fqxLUWCr1%gJ5&H6 zq8r=E-s%bZOeXiye+SweHUt}_+(J-o#0Ft1-;Qb=n6E%JN)buwd`9&M0mO}3UR6K_ z1=gq#zs)=KR;be@+nt>fRlo1PjHQ1m51t00}m_Pbn3pucLVRn3x?vN=S& z6462qfwR&K*a6S;X`V74T!uo#Z9T~0BZSlcC2wH1DP`mr(B0(a92;q1DQ)_HK^vkJ zNBJMUer9%e6P~vyn}}T32F`@zxc-3w?B2LFvYRvb>9DknRyO=Jke8odSXmi8o_g^L zpMQWo|oWWmYc3DH`*;{$^5b9yn6WG2<*?8yNswGPaQX)-rlh3U0{p?vF2K>w*r0} zP^jphM_XXaDX~~9h|^BI`C!(UYId>zvhBxH;|-|5wPA9ZCS>WoxSqdz;DN?ql0JLY zIAC5LH#kyTaVR8V<3FnKYqWW}P#Uy5t47%!xw?A-TXvm;T!G}xH6 zEX_WkvjHiEtyRiEHy7-2JN|#-Evm&?S(}t@BA|JxxRkiKyADX%HoNt4#koL^zH0d= zrL8*|i-~(i1GUuDQ``Bm!(Lp0X0?&Ai+!+0S51E(>LYkcO=1s{}3s4!A% zYrq}7>AyuLxc1e;Elu5~s0otTY;mB!TDKY4+_OWwRWb;$$9(JM-%PDs-0ull7uJQB zm&lI<4WP)GHiXlkPN{n^=&An_Y13?K^N=#$U71LJ`Pa&G-xA9hB-uL?OwdqbI8PkI z7(}uTPg@pCn>!YIf5S=S1aP{9U!Iuw$cvga?OfMRPr=xyOG%xI z&7~GB+BJ(@PkNM_Qm>`S%A?`vr~oZdDhcZz{H1H`;*m(T8JVFR6#_A%PfQyzI{q4+ zI<+0_#B}w--C|n@gfxelPlG0A=r9 z;7DfX*rqxEemfncf5)y-zI0|ba^(ic8ouh-umTnu;AVa2X}A&z_l`Aq;J3%hZssEk z5IgDgt>B7fNs1EbH&&DCjtUAOMmFV+-1k&A@^RPhc%4E#3uwTM!Jy6$NT8`QXkHB_ zTGu+;&KKZq8+!M#-_~296g4n^cqV6yQeF`TS5aKy6zY!aUS8&d>M=gB-kthEr4)dN zw6;t-2b!$JB=Z)P{jcN4e?6id9Au@WC)I53)3DqW;ZteSFk0fPqj(5{Kozln^pcgQ zcaU=B;m-^9tOfF(NIrZR@YU`K$WQce#iJ!7Or7^r;8y7xd|_3NC9Dz-G1o=S3JSGS zyNcd*yihznTtMv(C?uy%)ok5-mD^>` zUSRqA9{Y8Bg?^#kPKp8pjqK=>tf*$d?~;Yr4vy}xT-j)3AKD5~?ZmWYf64sG(4x8X zY}@!x3Ioh;s`hE?{uGd=0LT-8WcWU+ef6b=8M;lRa=$xXceD*wmsrrz&4^(z0yF7) zVKf?LxB$5Cgph~uCt%2msz^2c#2#!3{Eh%9=AL)(`ZlOUA>GPzQ`KGP)O}rmRp~eE zFW_=@PJPsBEz1Q+yXFz*^~Li0@-;V`qSHi|EfkEzDM58S0RJJ?F1ps(*O}J>@k^C_ z)53^kh>aRUeoEf(H~vd33iU9b|6}a6xN%!{@(F$l%VmXlh z%nR@l|sHg#C9ZIB6Oxye2|^%oBx5qhOl_X^}QtK@K$$)`heu#qo- zD2b-v%%_@BJksvOmKVok;^H4WUE+c#GX-(DF!Ih@6zG!x%PPal+|Bxabwxn;M1423 z#YJ-}yX^7p1ReZ-EfKCq@X36Ont_9x#_46hvU5~mZq!2VG4*~(Ix``PAXBp+9>`ON0x%fSe~^d zYorl?{Z0m-Zu-muln}yR5}PcJaz(IvgA7EDwbR`J&P<^WJAt74Ku1gHrb^jex)VGiIGcQJVuN%~@te61YVSl`_ z%RK0b4BBgOtpzkfUaETlFgy|Elql;vvP?-i?HUV9I^lMYBfR-*tMoj5j`3&haK3^m zsau#z(BlB->bk;S>~xQ>D^>cR=x_k$qxB_NO}Fl@N|I^!L4tor&^c-A5Oumsc=yw?4;{rxcqBwrT2X3 zcl#c60w--#EUza`W3h0Znp^qeZFj!r+zr^Y;85@ff@EZzwyTTFNPfH$LSeH}g%7XrbWAa=O`?0VWy>^J_H$I#r71+f)>lr3MMLH8#+Y>>c=BMTm? zYq6_HCT=#aQ*|#Jd%el(*Dk3BU4Jf9ZsfZGEr@5F*U9=Uny``+_oO zu3r6H@t+{6&WUTxcj~>jx*(O+<+J{U6=>xvu9vmy?X*~v2T6NVsgjO^8Al!p2LAp7 zr>x0a%F-W#(f>LZXE;t{&R5IDWmOZ97oB$1Nrr%ptdPPLh57t)=BJV9oQ-SH`a!cQ zbm#Ml)%SsYm#?o5Cx|V9!DfnEIM1y(a(2qH(Zh#6%?|j1>mmX!kX9$jse;W$@6Xj{ zx@5-JaV#EwGtJ#ry3v@SZ#Tl^p9{JNyi_lA=?keqG?PbdVXrX`dfB%G`NwYDk+<~L z=oWiqeG;fzHbs9J2FxHC({CxOMW&W){WdsI>@St` z*$8~Em6~XB|#AGO}wIx%QbWn`mmA zj-s-@B{AIo2hbun(Bf_*x+}WhN8I74C;pD4E0?yw#h?_87-1AoVS<^Bn;>eZ>D_1T zb=s3$35H3PilF>8>HW7l8)aC|ab1T+K9}{eQ~uJKx)}Dr4OoM%x_z-|Lmj^3VAj>l zbN<@~<2w8Omu_qxKRywS>PWvLDsWt%?-g2aP#|UWt`uf+>$t}-My_~s3wQf2x6Z-I zf=}TJUsvz)3|ccfYuan%2oW!7$?U)&@69ouk(teuT^%=SYi#sT-wjnq@~&G7CAfFW z7i|q)Yiik_AXY*m%rT+rs}ho`*oFF@01?lb(8t=Px)J8vlWVKcJ>vuZyo%{$StaBB zpV-y_|BCy2!NsB(oXtg)!*V`*b$?^2&zms^QJwy7TBy1~YgaJg=tIj`Wi8Z8p^sV8 zeYJI3TK2J&rxRNY>tA1^CWs|^#SQKka~2Ak-BGn@<2g>C-IKFg>clD7DRWxJ-x3s` ztz45+x29+TD&wX{HW{Qhjz4D!zb_qqZ(4eZUsh5cQ|Nn7F>QojCzZInj>ve~|6OZ{ z?tCN)>Xo?3no{l=_hdLgt#GwDUw^Mv4_Mum7?Ph8ffqYlGGZ%E6GQ!f>D>B$<6rD% zaFaDD|-TAByKHxk2nvrkS${MxurKhTf)>fE4_bc9coe7>Nail&L&63R=*SvyCw(BPIU`Zdn1OKH*cZuq|~4j7T;& z|8nUYfqHSX!CuL?b4z%fdkoS)2E&VbO5>&uLleX(n=hMM;*zkWjXHvM5l|7sd+QY= zbz(D8%AmXcT8T!jRVqs)*K4utFiv3m*=3b5hse;0Z|xzj0GFo4dLhdsHsq;Q@MT|U z3lLeAL~xq%YnZjQnBcJ-dM{pdX{=1PG(FWyVSle}4Ta#d!e(vyQhb20<5?2_fub{T zO~d8GNG`Uc0Q>s_A+X)!Q61#5VU#LV5`~3oxdly(ozeUeqmXNreXI1Ckfty-w!c*w zWD4n2r{(Tr%PtM(YR%K$1>&EIAlnGBmf#;6cnn0^sV@&!wf1mjKLPkV$HSd;L%V}R^YLQ=HHLx=Q6Zkpbcia$Q3+5tMfz2nM`+ie>K?}b`>uS93VDN`gmFI z2$2hRGoMNZxgF&`z+Ygp+{Nlhja@K<$OL;u>65z4LMX!ri(Th~{Z1>mb>!x{&Z}w{ zr3ZWHllUb+&o-TWD4Oxz87Cg1y0r#1oN|l6G=umb#grX)J)vN_>NAS`+PS)U>R?4i z*m|z=lw0>SDEu$bAOst`fjU$}kUXzjB6D1Q-`%p>I&th7^H`LzvM0uOE^({X%RPM^ z{`@a?>YAw4(QL6{0^FvT$^QB`t={0;>LO0kb^+{r4^#XA;pwe^n#2Qid+ZERh z2=p86J?k>}3AIx$W$s*Cz|J6Xg7{IcL*BlLU_bv&q{2i`L}ThpZtZ9aX*Hh>+Cisc zuIH(;K^Ze_;M3joeU!O?dLSj45p*X3FaXdLm}QyIz-I|VpzJr((3fS$aGie92bSU5 z0Mq$7RPeMxNp!mbQGtE(GKL%(j>7gA zkAB}<+ch^I_h&0u;gSAKA8}i|NMWIM-V0|t7*Ljtmi}TQs=3B2ZjBjU5VwZKSW5|x zR!qN5+UgF)6lvQ>u^)tH$lfg@JUrufsTh-&ASf35)VoxSWf?n8kz@N4pk0TAh6Ry9*hLj)_acLiE!ydwp*NV-WdX^tJd$`^*!B5OirA=V?$a(=QdeQO!AZ_H+*FG_SW5sr1|T38R>ApKGCg`wl-*(>EaUaT2XpwLb zuw45FP)h$LdgWeLdaxVK@3gd6`&&q716=XdNVPiR7u~_F5+nCJc^ZKJy?I$G95@c9 z2zVtv3}C)rt47<9(%@oxJv#UDCJR%#6=l`~?0?PupTlA^mH+nAoO>DDB%&-$-Xxb9 zRyQNG-9dK2yEheVPusY84z^v!I-9E2Dzx@pJB4v(S-Ffy2v#Wy(pAT-*e)Ph3HHL( z4K+A)4Y(0v5&MH_a&OB4ZmanU~L*!zd)K(rq5HHVl^2 zbNubsm-Tv_TCE8M?DRc%P6av+^V<}TFT#HRp7QUx@jHSDK6pvh=4ucS6Rsh+wB1tc zaeec}GNtnm2paen(n|8B!#Kq`#(2_nC}uG|qR&QAp2hLYt@875g+R*iGiIlq5cBfz zOUNIdRj14Eg7@a(B6>U_L6*i89NI!80vFnAUx!(5QmFH^ zUDLpegoK0vaI?FJ-)j7X1y5z8%r+_bWQy)*Y<3<&Zps>l@`Th zLfJ4LZF)U@fHN_#)HgiKF0RsXeJr$!5*_8@xA=st;nI-JpfpitY;|-}00r(@`9|de zaoThKI4`-(ddBsoMV@b`a1y~_>+H!Qe>B8!3G>nlKKFL1#|$~XFB$TIT{sy0R(tK-__PXm2-htQb7iGKF4jwL_j=*5Jf{LEE_I<& z@PowWgl!1^Kp0rxEUkFrx=D2#g%zT+6o|nV3zPYQ5oh`6^qi>YUW>K-Wau<40}?aX zrO4D=k=2~viR$*m$zW;sW(Bdyz;$YGTUn+~$W|R$mX6DInhELlfwd8HZvRSRXglda zOUsy2>Un-LlvY;nh2=1LBAxDKPOp4>_*(wB)#v1%mOrEp(b}uRKS%Fgy%GMQcb}8A zA$h=pHvI5&=xEZNt0VweoK#U}Jkkf`g`4}`X;wq5npGJkrct&|iJLcoYwl+cUirqy z4+2tFzc}d^nIBxy8sSh$l)T%~|Ehw9=BuPmv4PBtU()=18mEFujYDjTxF@MiYAA{B z_Y`zxh=+-%c2gmgMm^qsR!&lbn^YeXr+tTIY@|?Y@9SBH)eZ*bk8@jF-`+mJ$2{>* z<2XZudiSs3GBFjPm2`A;i1PBWT$KoFtgQP~tp;c#vF%~B+C5;-7|uOqG3?|IqIE{3 zA(DvgYb%fc&F?*Fs|V z9=d-*t5X^svy@<=t~gF7g{FDZ0#o*K3J)kbh`%$pT>5d%I&!ihdkI0JFpucd1JpRw z$u-dNvHLKS@~4}+BUSCRvnK?fR?0!$CF$DxG$GD|sGH=Q{9w1$k3b(a44uL-^!ws@!GX=-1<%8fAm5sJYi3<%Vj)0b$o?uvyvJ7JW zK}BAolGy%QXQmY5vj6G3D)pErqm}`&QQGk1wOf{OvIWFQI^$Ub3Z>_}+)}}-1w5^m zf80O-spaO!@|ky?Hcor%SNpSOPIgVq)<^5c+j2$C<6uSbouo)j-Gf^5aC>nN$S~@v zYX0zbJoVby9<&zEl+kzhRvxy4EoZylJWj&i7&jScxr0`8h3obS zmQ~fDqGFXUgh#tO&_BfH+BtqdcY5mJ)GN29+daEduDg+8QTZ% zY5%XKPe0vl)KK2*MW(obTC*QJgKl~X@-;dpY?BM3w}d}puqT5S1I_j?lh&9+>rKqB z%gojDxF3IcB7WMP$Y;L-ceks7g2U44XsdAQOOAjM7H){5)|@^;8=_(hemAbG04pEL zNqdBlS-`xs3Gh1V9`wNe*NyG&)u-iIE&tq)lqi>ub~u^0xMAI5*dG3Yvw2IgA{*V@ zc?CRS&BjdlZO;Ma8rsp=@*rRJ4LEROt1@27)(gJOX_j9Nlj5(aIm&MhB4!;J8mudz zEFx5ED6ZAh^!OX1%{7)j8kKI{FllDdskvW$Cq7(@`?!@gU?wJQzH+qEu+BDjcvCtv zaeibl)Zf1bD&8z2vrkdyB8*Z_ba&4RkB`$5XSPC1XgN(r?UYvyPfrh`m;5+kmS8wj z*}SPu-b1>Ad5qoNE7bPE(0Fv5{4?bq8P9(i0O!p~AQ?9`v6 zV7A@6qP<^-h##XqXv7V)s3Ffah+3$A*_1BZhn&k>oQ&zGpz{e@R_8+O4E1C;1D%AW za5S|rSv|@Y^T1RC;^f$Aujr5W+DOo0fHn$}M8=3GZy?4P=d5zNj-Quh^~G5kH`FS7 z3xy337)j_5>)7SfP}Q1v8TJ^bU$*$Y&1@sh5kX&&C$&c^v%<2WEsMzMn!08Uh(;YG zd3kiAcE%p_)0dR>44FSCd{(Wn`*iNh^LT%lQOJlt@7geF8K13u_{=n~~$uSg{ZMxU^g|&0*K%E-&7fuMokr#@3M}Rj8YxzXq(d9xx|-{8%98yZqh3 z6$Pt)-u{<6uD-v(@WJ?Em8qn#vXwF2?_Otj()Nv<1}`%@OR2AArbpX56C&%lgKxGd zYo@=?5qtQvS5HY>%9o&lZWWjJ>*{V99f4- z(cpv;^?=23P0$|3cn=89@ltwMGu_lI8L%k^B(c2w&4y;{#wsdL62YMWY=woP zg~PYjx)-wbBza|T3=PyyLDR$)`B|Ov1lc2a%zlR7IzyJM!Fe$c)XdP->3>=K1d|^Q|uC%W2rh_A<-g{R*Yr)d0r4eqc7oQwk)&S*NqKz>F|T%eyRBnz}g4Pu~T^ z8ex9PCtrtXC5oh^WD7@Q^5oE&2>7&=uKKAOi8$5bOEC zy8Z@n-=u7wkUWJhm6ZN)_*0{p^G-4Mx8gv*J}$Abq&b*%Ec7AuWZHp;g1Xc^|KSQz z&mlB%`Xc?iuz!zS&kD^LRH$U9;(y2W8ng{$@`4$e%qiRekVTaZ1^&^K6oL^~gJApX zE384GaC`du$1JR{Thsz@7RoDMTk$B-@TF>K#1iUvhP2^wnf$$OhjDD z84yl{f8EWit$z(wyg@xXcX-awk1$<~RpeNr`X`tee1pPPXKVKackv_WKtubGlkW~s?M;=jU6S%gQt;+*5g%86x$LVCQ zU`zDY?p<--UG`}`oR>~c;beOtq^zvRpeR{kP~%SvzH9P0n_`;VFgIJ#9>z5uD?>m0 zJ;6Cjt#A*owKpG)1CDnrWy=0;;i22nzEe^iSYDFUL4feb51V?YZ$<3sG*kF{N}mYi zrRk&nD@3qubcIozGG3jv!QHDF_=tMfI0>n3)WfN2r)(hfSVhVVVt!w~;Pmy=6*&uO zs;z3*LA0TFjE9=-mX3w^!8XcDWQ$R``{z;Sy&#(9Q-Yn-Id==v!<~1uJ9GsO_?jiR zQSD~iJ1dC=JLJ0K-3=M3<3x)+js(8u4#X5-XaPeUa9nF8UX@T}l`n+5Wf@dWiJ*=S zm{${!=(2EB&Ko+QkIj$&s32{i!gr7?gAs+hKL|GB!q&$#NCr}3(5MFsMa;Dm`=$~R zP7AcEQ#Gi^Bg1<{3FVU~u!v%p?~9?Dy-`U#kn7`XySnu2L-jj+yUZDx6&J^)=R$|4 zxXnoWm3;OXFqrzO?Q6c*i1ypEO%=Y&z4>uZO{Y(KKr)YeMBT{2lw0&#>NW$`(U4JmXTT))^UIS$UT&Ivf#SKKfzz zShx^r?4PQAnx;lwPnF4?2ziUSqVDlbrV7zWcVW$u_t8F!aRFI+>0HV+8kf1|a=;~C zis^Y4H=cc4 z>a&=@%ks%1X7M&X7^$maTTe>-?r>?eIA_kmtZ*Xm|FkY(^H`lY4c%F6tDd|fYSCyd z;n0#>YCu|}&Ly)p^)w=NJ<+XN*`YxKMss%xA~;bdYin5!IMOeMZhV5e^tb&cuqt(l zJAF9l4D|f^oU#>gwoR8#UaK4T|CY5dcd~q+e6qsTAhh-A?hRnaERLNC5P!x4FrF`drJjTzSt}W<8c-TGWI#uWYnSy`B4hJsua@ll`Hv z%9wtvjB>CVG$7mgDsFU>BY#8@0&1-%iM95_q3)!^S36yff{staua3-A%ayy7Fd*v3 zisw0!ry6n%>FU&J<8JtQ;Oghh0jba7bj6NLsBgeL3-IGWAC_uQ5r0|5qGXLhm(27# zqBem1ue)ijE-;guHB42r_>|9t%m{%^-N3IK5OhPL?O|x2sRXSnsXDA`E#(~Bf7EA=`UTeh=&QRodWD`C zStJ6pQPsWVSotr7BKvJ8BelVom!dpFpEt*aAY;%)53*a2oAUGoe=@99dHzTRo6DhO zgGhI4gd_MIdu&;J8Xz=u3T20`7n>9}Zc&l5y*CzMf@l#Gle_N^3T#MqjlTJHLbPIc zqU2&rC~L@OWH*^Y`tqvp-y(G;RAc@yJWwY0o_uf!TNhPF;Qs(ptIy(_Iy0 z%Z_H!4iKzp1HNT(bxn=7<^?}R;96ve&;sAq+JM$+c6n2rb#Ybo0^bDmhz2=N=-z)P zaqpg)BK+n*Ly5dvwKkoXfKip4{7Q!JqED0jZJ{9-oibH!{R}g6tk-%k&5#xt&vmtR z$2`8hLn6DkNlE%WRc`i}kq43*|0g)hndROt1LMDFoP1UD)r!7sUHqdg$Z>T&TL1RP zi%!f7-HvM=49IcOhL*BHW#s$ham;y$ae!M+9+;PcT18mO-7JnZo-IV-R_@X32DIO1 z6RbAmNHp#40-gJum7vBH8f_G>+*SoSD#xbKhJm(JeE1!6Tggmw87~LNP|(T%f)ARQ zm{DP-!^4WmA(}WExE9EO#oZ(t^4L zT7U2IR{0csl5%o)Wl|2^9$-M67ZgA>&``61pIH;f|+g*dU>-`84!%7Wv>E0(oEl!vj!g8?{cN zx`+NPno&?IRK-wH1C$5xdq#&VF1`v$ zo}87IWl?powRRKJyxqe;#iYJJ&+{cBuX;MV&39W5bU`qQpz=m$1g z!zrZW>nk_p?rHC?3>s-hH5=UX=N~Pid**^Lr|f%18yYQ-t1wi!R&Boj7g%z@>n$pi zD3U*advE=#u`VH52QhWggqg{BHb^3uey+3TO_t&Tt4jo@1Unc}0N4+MmwLDfrp9y%|=J;)31 ze1goZbXEUL;2S+-{e-D)%3xV83yoSjqgr{={+MIRtP%{}cr-w_W`wRYD4UxKd^Gdc zFTQQR*m(-|(mFjQ8}gwX$JJ;uNcKx`>5Hq}ipHjtIJs*??`FWr|5%b2k`5}squ)Rd zOv|E7&hx>3wu>%Rjpy*H3~m+6Q6wNK*Rf&4PZr-+r|HjW;M{b0{A?8ZtD4JEM-0sC zJ)Yvvppm-SaG#tW>A~%C|7ee6xIwav68D#SYXexyz-sW|x(JfX0*1l-MZbIQ%2 z5~PQ)_Wf9>mljn;w+8`NGtjlH1UA>kx#CJ(081K1LDkbLD6yU~!=UA?2U(Bt=9xzW3hIL3 zIK+9R7at$zlU|bcFT_+#$f+dY?4aC|0&TGEr*DeAJ%>*yWdL`KuUSkQ0 zZ#A~HS;1&IM&T$l1N_Tf-2Y*5s=985l2;)-?3l{Is5*Sjyuq@oGl!S_h%Y0w+gfb+ z|Egu{jlt9{q`SYu*UXNrv4uaH^VPXG^k$@Q(R?+2Qcgf-M*y6s;3 z+_+E>QB=$tk{<%mZb1LlwoF^#Q9_sD*Gee_RS&*sPDe3!`hz65xK)o1K2oCYKhU}x zsCEvMy&7;ulQp#2MhY~jqH_@t+tF#(^kt))?3k>nSN`MYYC)txZ`b~e^y5DL45Ph+ zsgKz|pINo2#3iu@0O#e^D76@t+^DEa;BZi{s(yZ_gepV~n1eN3y?XWOQ(8a8ZTd2Y z6@DiJ)qEvc6J(coT=ZY9$)hf-^KIU&pXcxY3P}bLuagA-^I`p5%I-S0w)xk^t;(5% zjaD;!2ZrTVGPsLtperS3I9|A=X}R{X{)jt6V~I_Zp%h9%lR=8+rE`BdbUWXclbhW5 zyQt){T5Uu99?lvB%rapvK$v5`yUI{x{v^E83?(L6zx&f7nqYHsC{ke!Pd_~x6b)J{ z9IOlU{oLEf_AzZ;-^wb_VSv2==fRds^os`uWkSO7bPfq0{npzd;dEL69tv3P;g?x! zryY0NUEWE;aw*-17+WM1n|pFp{`X(Zbvzx*hnzjU=eU8c`-sBU2DU5K9#P`-VUEj% z9F{u966fTh2Lfg@jvuXnna1YJ-!|@~#aGz28ZDAc3Zi~m91N@qp2lAq*v!8uaY2w< zTqAK2sU*$Ph{Ma;lPX_Ec))+!xLf;EhmR#-b`|y3j8j19)=d5z-)gDl>JX-kOxQXt z?)AN$Bw|>qgVWwL$1(9;h0r7f+ItL&bI7mcqjd?MI^RcGk2WWsl1Xe5b%$%(W_w-! z7)b4XFD)~jHaJHBLkgkmGa+Kfd?j=kziRt%wCdEzB6X!^4-qR12 zJ+Y~yQ*z8bI=Tl=ol)k*26V+J7o}03bTBS$F0P=UqkQt~B|u&lmb^?9c+CTdVO%le zpzk`~j%RkAL2>g>b1^%P%b?}dC_S(3N>F%oLP}Yt6uu5R{j>9DVdcJISxlh#IM=6cfiW z%1Y)MrWJ7Zi))-P{8|~Yr^dV%fB^`hO3!*LegBN2*}IyIj#pAJi_utxY3Z5W)ukW+ zG1Q$b4KlgOdRuOK;Uw1+EiAXgE=FnjIOFGG?p8X~tNnB7hS~2SU>@k%z`CAheQ>~) zTZftnwqm=D-lL%X)Ml zsuwT{3WP3(D4S7G;U+8J!pGn^;QcqLODdTbZ&sc(B5t)%#4z>|r`EeEzt1QZfpPNA zs&rZY@ep`qmzDoO?)ac*;pit|)z9Vj{Q$LV6jKARjdE}?rfhW=lg|Ed%et>67)GM*rN=A zU`?-ZWYNd|ycV0T^lW=AYt#(8J-w|nv)7RbG}!6}92+gPl+HWjMb}@pY_;~=O6NmFw^HR7HhQ_T z3;!MUk@PXsN$2bI|>(iw)n&ROCSV) zA_{PW7_F0K#0=&_mxcPtGnWWf(8fu5a2uqS>i?Aw-XAiukJR0DS{CPbrovB-+L0$x zq72U*O|H;od@79I+ymXq#ED4 zqmI_8ruh#DdKq-)wk`zyv}B5*`@mVnvukn>^W8vrHd^>;MVKc zA+NJ~Ur0&v^pxJRW;dv$dkZpVSehZyS&fv}qe&aJPOru}n>&5W0ds~(vyvH*)!Wxs z7fL=hF?DyZ$O7txnZT7yA6Xsv+_lw6kb6^NnCi1Y2=atmmJLLn3CeqY(ce<(YuU@- zT~b&03juC!6XOeK+Xd7la(7XH zfe2xYr3J3N|9->K80lz8VTIyFQeV)E#sroaE`&+pU% z?08SzZ}B})>Sk>Ym*_dOf)yI>?J#KXS>tV63Y_gRxhhbfCb!;x?V(QQmG28x4KAn8 z@01r5puevu$UhiCX0<#Txcj*xZQOda1jeRC9f6#ty#g3UOQ{EdcU-=%`kA_?jD}h` zlql&VVR$VFtiTF@AQx|N(q()oJgk*4=Le1e925GLz`zF~y4&-GWv8f-+PL1MlQ25}f!)KO{}P$yQ?E zC5ijm->zFIZiK8M$iB|U$!xxi0JNWytuKa6j4ZWCMMF-+XnVyHLafyIEAz(=TVv2& ztOZ8)a%PNd+|Sp~KnWG&J*?{&Q_&}n(MSCtG&knP`5^M%4Z_qZN5GHKQC-gz9_`F2 z-RSH6ku%1|%tLvqq&ncTfhc2324iM`OG?^2o^lp--Ce#1NIT`+>LEVz6}^D%NnDkr z;|dKTNbz5b0w<>pVM4ETL_hmY`}?bmnZ12lhV~8_`{wd+(uNk8Sde6r5nzA}R@zcj zfPhkee#YH3#}*y8Xy(v16Z{;AI1rLmFb^P;tEKxXOA7*X*7?T>wRdM<$jxtPKG_ve z6xUm8ip;F8&^K5g-NZyd4Ca$M9R$F*SMJ>n$CfO7ITUIRmfrB}p1Ps=a_Wn_*Rq$5 z)I9Qzv)WhqSz{4yp0Sz-MrDXbD*%Gt2w#k;5eAF>m`>MV`bPP9^GNX0?*UB+2Oute)cSwDA(HX1eW5BNW`qZEe>er!!|9*YR3ItZn zGtCVeM;~sf_;t|9e79M%nqxSV$qQT%sfDY&p*Z|~A1XJ%3wOnGj9-V_Afxzx=n>Z3 z)44xQ%CYTfgLx94_K2xtCT~QCcNgJ|dsShzo^P~EKebrZ@xp|t(i0>V)M|i#2Xqd0 zu-Ch5ls#=!dKKQC(ab=jsF#8(QV8e!%<)IsI}HtWo6-yIq5t1)YUtfGz`M!6BQEo5 z0RBu`+Ln-5WiWk-z9Hifn`)iQ-vK`a%OaG-$E`hrgr#AiA;E%5Lf+1F^@h>lOs;Wg zouo7wCl%m#crO~845qb%(b{uIL%ggZ9&_~8l{`)5;SP(P!ZK~a`?q9$vqO9ytYlN( z@Hm5!<1v!C61zTfA<5v(gDokgnL@4O!2Y;l2lcoyVP(o)%s8!mjVZwRXISst%$=D+ zFT@1`*YN(t0x+O-9?p{^VEv65qd;IlB`cxqavDdcWbShlbo0DD2EsnW_c^HhiREeQ zMZV&{P}Do+!T#xH*%FcVNAT}il|=Ix`9pp)%xh@ReX!O67mAf_S2Arw+d5_i7D;l<=7HNruggysn%za^_a?sQQ=I@RBobi0RS>?cq zs@J2T!Myov&s%Oi#oDGvZT~0&=)K4OF+}m;h2WGF+VqnW4Am9S<>?=zV~qW0#MZow z+P2ILD0U=%T(AEBxzA~xlLP;Ik7+EQfbeLB`(NaxVW&qhCgznpcZb&^)$+|Or*aq= zumP*Dw7{VGAJW{fD!~9Z2U;;YS;ao<)BF2yP4r7T`C{E-(dJw1m}mu8$qLVmuHt4p zmMaC*xDe_Tj{-n7J5!dYY&1`oHj2nzYHArEwA%Za;At#yEowC$Zc$3hmEquD?$P6L zS5iXRYLHNZ{TDv5AzNf}GzB^W$!|rpy6dhstU35zAY=mn0xMw>0EUF*ElTA_;7>$M zpUW7x+NL4?BORT)A=_NN5J$Jj5%n0d&QMx=t30afi%{F4`6P=`BW-M}n^b{aF+@OH*3$oJi& zAv;Lz=M@vt@Jj}k$F<)){jbri-s`e|`B8Ggto8I5@1QRCdUS29L@)nG+>0ntrdTSY z&chi@3yX-q63KtuE*ccyfd7U6eR1`9@V;TsLY1(D#Ad-BE9;}2^GwkoW<9yIW>`MOj(h)Rcd&`{Awy=;&BJl{nTLm&*ThS3#$FqT$#3 z3o!$-_N@RKm)d%bsahjWpIg%XzWLqhLBlo2$CAP_dSYcuF;judpZljk9!?_IHdQ1& zEJ*Z!UCD8*HoaosJ(qSQpe=4@SZnQ-S!al5UA-@|O#SWWLNY@qm!8^x#ezx>9=$6edsoIfUx z>PiLf*u`=wTlp)j#OE%=X!tstMp_fsX>Q&~yfkW|xhi268yibyX7&fh)G_Pu#s`XYJgXPHwod0{L!AZg@UGwMmP4p8sV#ki7PqREw&Lx%T z(Dws!{N@@Ol@nxpd!_#L26f671y&?#-fO#*#J2}h-sz>dcbr}SRwqtvnt}Z>3wsi~ zcZR%F)Wj-TioAsUZhU$u0vgrMO$O<=wT|wQDW|T{_3)rOMw_6Kor1k{WaV-)E(`{i z*n`)FcRC-~dTZ!pDh9uI4wU{VgFxKP`+T(Rc>lX1D*oMNPq%x7s_m(>uRf!El`}pr zj(Ea@+csMRUWc@O09j$WAY?xE6Gs_t2e9nDpXX)G=wY}o6v38^v%fEYahWFmMVB>Z z?A#PtZ4yU6b~W-bq@}__cy2lug4_ahD7OdrvQvUc&F!<6MzzDmy}DMTXMJ4_pFPXD zr4n9}lk*@lGSc0pxN6(UT2#Z^+aE-?GW$I+qiK!oyz_CfR;b4r5-M?^p09V*p@%uo zU6n3QRWdHe zlO!V5Do7(>2AVoDI;ssOSTKM}fc+(mk>mLAi!O-G9wWzJ$ZxpI6RQ=lb|MmQ;rLrI zwR=G?B&5Z~)fFc`?Y+LfE(t#k5w%1af@xS=S1cQUdTgIa9+_9Zob}`C(Ej?XNj;jc zet;CA>1D#J!N{*;Xm4K#HZ}(zge{*=hLis}gO7(D2>Qo5mjBXty;*6qPmPdXQviB2*xEyP>!4?epd#zz*m@@LPYM#Hm*6$Z-?v=59!?3)x;moh zSZk4^cwcr!TO$$8m$S~Tq{`hri%bm-&H&OB(_tPBs;a?5y$)t}Ro_{M!k_pzLvo)zu9skmscz8nHieSEo>OedH*4p-BW<|7ui3g=oJ{3; zfuxHi4L{Aq5KQbPsjSBAm&1y-l6F4mfw+r)?zuT{`sR#k+o%~~-SmWsDizSHWL3U- z^A`|{IDLI}rkLF#U@*%Cw4hHkMkcC0uaAqj+18w0-MiB9q1)lRZH++3j9i~cD7=iG z^FB-Zew^U5@N_K!a6M~_L81YorNu%kRV+4HN?LloX(elPCSb0ju~C+B^vq0HpTjVa ztgxo9k0jb;*BjIRh&xx?`}1)`5pi3fAEQK?`uzDbqN2h*wcbqM*tnQVZ%NWs8a`Ud zMuNA_+A$FtGFq`it@WLGFw*o{8-1m5F;UA90x7I8Z%F&N`l+SmeqmwZXO50*3a093 zzTTMB<9JpSL8hW;|MSO^!BPH?eaG)k^Ni0UQ-GZ;EuA{xtwF`a2|u}K?-;BgZ`NGu z^f$iK5(5_3KnX{OO9H6QV;|evgGwCWfaw+4a(6Vw$=y`+S~h>_FGAtVVEdXS>VPCt zf;PY>ZJA>`V(-Vk(2VRIJwdUuN$;Vamj3c28k)@F=7SDUjb@(`-D+%!0+XE04$Hi& zOO{gdHLZfarf%AExZJKUE^&ux6>O=CWA<}w$VdrP*Z$ohh{=_ndizM$GOhGhbZxC6 z_~2&sZl5>&n|F40s9VOvunU5Mf@^P&RU(SkjnQe-?WJeFT6hGEw(F8lm*bnj1^D^n zX}YvF6BCp6h0jO?VkBC;tpw-MtS!x!TJ@J1cfb4SkCSs8amadoEwEG9{&c5(Y$@GY9hyJ+FHMd)&Xi*) zaFVme_J8gi_{-Oejn#kLya%-DQm$X5-BBuAEj8wSDI6pFgk*WY5*;Y?p=m|9zP<&h ztRET_?~iUwhDmwb)K4bh3L2G4KuF_vuBa?;b1*&*-Jji?LM^_E!X^Wc*<^&)I>v+x z0Qku`{0Eu-ij3hK0CPu%YjfO;?sSS!KU1q*JYKo+P%!O|?T0RHE`g3!`7I3CbOu4f z#NJuPj|AtrGlfHSf$rC9xM?0Bm+M9u79Iy)q_<37z?cW)f5iDrwe1lh$02JE!R*SU z6DLfKZUt-!zzYKSawo4G#?c9JX`mPTNqA?;j_sOkJLGJO%iqkaggp1+a<(fM>R=>) z_qjI9#E&T{MX%4&x(ww!?Ct4!@%$xCYdDVJagVTj^i&h5LWgQ#a^FLs&^me!`D?;+ zXgGR#AhQf#*<7gwW4`1Eg)MBLnr`X^yM&qfl~{pYvxfUjV!@RL2s1I#z^bKH<#;b9 zCdS`$*U-hqg-B(PEyOEhKMER1I{aWUt-$IGzRp^iehT~^8iHi#f$joWgZZ(*5{|UXI{6#bH{RR990K%`bZz%jJUtKkeagq#L z=6?9c0?09*GGH8X4R*N}^C|P(*R?O^1)l>i_G)>3;y9nl1G3K<@O@0bQqR#0t%-ya z7w(o`8H>JNA?`ZVLWVkY(>#A8eBq@OAEwYtwj2a_P?G1ilnM22(&_*2V*_@=!h(WY zTYZ{Uv*rHu;z1>nh_t1NNr@}675nWm_+=E$!I+LSc;hZ+Y)n!R{qxObN$9#^lKz<) zb3j1F_P$EFHeM$5wMXWTdSXYOAZ%Vzq-QvH&Xz+d#3cUy&Xb6`Iac7{hB};&ma_Tw zw}EdCi5hkg6kJD7dgP#f=9Nw=p=zzZJ(7RMLjZ{Pz{Rc%4-eOQ>Tl>dt8QtqKK|zH z9{41$JhrM&y#ewht?)+;O+zF3;_jb96&h4=KNDS0pmFS*nD}#k=I!r^JDaTTm8&NY zZ$78*xU*QdZSWV(ov?9-wV{U`!{k^E-!<_5;0F!$h~c@4@0N;bBm&R5)#3cXlGbMe z1ynL3h!Qvk4OAZbWJx?t#xu)*1GvrKncz$~lhgP6{~_$XG2krE>P+n00S@At?&@9)h& zBbfl<pBWVt)-}OVbRLa0re$Iic+Ca7$ zT%Cuxx1ELD4Vq}!AMh1koF5HJ^fJFJmBmYf2}-a=No9o^!}Z?P!^gW1P$|1!8n70C z&EcA_8KVWsSW$jLL_`K`j7W5UNIclV>yqqFRN-VA+HTVaV{m60QwW#wk?YmNTx>CIrgDq@9)qS0o+lfxY3Nf z;!tg-F;Y{jTSx&Hja^#kv| z)C->1lP~u9Nw!qMn-FdK0yBq(&0_>Ojs`Y)O+2F2)YG#DJi-#}{$;`warXDpzn@~g z)M6_ncEan$XOCL=iU3E6KKxsx$=Rgjb9{W|eKEs{aLUUsq#lCrvb0S0NMu2ImAUYK zwjX(XR=_5b8%Ic|zTMvserIC#dYBGib>FI$YoMgM{tRKB*V)@RWv zZyb|Ub%YKLCbOL#FWL0opRSZ$GasQv?N>zjmbEL(@La{c?Jgr;;z0L`;h?0lX`lYs zBE1;45N_)J36cE#{0f@^&LaIv1?>VY5fKs2OP5rorSJFl_THvTJ&PK7JMSFWirg`6?_8zGFonc{VB>$C=Vksgn|Bw6Tf!IpNk1AFjHs!+wI3mi3s5?ye2>DkDgQHlR06>;4l+!&95s zL@v`Of3;ijLFfxkF0M-K{0O!DKtD_`P6MC-Id{um^-K+Q#b)kGzm8tUT=8KmdG*mD z@4Y*#TMdwyJx@xJU*h^p$Ta;4$i~@Rc9D?4`#y^&6a%6DYxN?(rtiv*K38&SJ1?_> z>_7@BG2Gn%9aD7`3Hi+rVHX6Tns{_0O<92r{95M6>J4h$7Y83u9F2^P&HX3ay~M+# zqRjq~dmq8;Inymm_RgLy{!NhL$~+Ye`Y&GMMaI&xL6j}6&Bv+Y1}GvXjr$< zD`GrJF(QLa#PF@m=O6yNvba#1*v6pA@+_X|D8?|cqdQhJ1}Ts4xhCxh^YQgnnCIl= z*sjeDBWru*fEROt31Q1FbZn%J?1}Pf;23r)OMbuP4%gX$mQEfHrw?y-A;7E!vMQU2 zw~zGeJW3zauoNRJ3n_a2SMP_ua9km;2))q=tj%wa)PmC3e{Dz1_z3TRw`0<`?E@rz zgYQCOl%n#_3W!t^l&E_2vxps9d!dURSp_b2Wvg4IWdY1xcjbq^y|cZYPmFZKaL2$q z7(1M&uZ_VJ4HW1Qn`w>vS$2QQ73r3~?Z1xdW~?5?n~4HaU0T|c$q#R-J`_9y!HqsD zEUYKf2O2n$00F=N`eQ&c#~YIYL~lR%5Wbp4(p}>sf4S&LtTTwd0CB(cN}ke zDnYA1E1FS$>dcwhh1x|y`*p%Fw4_H(Ea;~QpsXC}910$@i79*g6%6DPOBx&c2Ajj= zZzc%Z_iCt!X9cqO=@w`g&GwA5^dEeiZ}&VFz+$h+*7r|r4w(Eqv6On=Jonp5!WV>* z!59H6IKJ?5g4qgr$kx~M;qbNYed>u+&kmw2Qg@qY=)=DDlGYEGt=LMXjymg7VNz03 zCpbE_yEJrlKRMkWxpeA?_0HCMe#!LDKLd&Zn@d#v4w;!el9_pnBq<|v-<|~p*{U~q zk!-FQH++UZf-Z~7v-Ht({Z}4_yAE|j+3fPs42CZrg%@+Y(CyJY@bx9xj@KTJJsEQDt;^lDQ|zAx#z#q#4^j6-DLv*}gQ}|) z4FG(>t$$B2udY_{HU?C(A=`VQMh*s6vCk#=as2s*)l_Q~D1Ud0bCSIj5`-MJg&7@& z7%hf%X!#+zU6o%g*|StA+=*hFJ@yF!7V~+B2}$+8Qq?l|wE=4txIfQY$(?CZE$j1| zeTE9a0)^aq_bTh9&f8Tj#U;lS)YBiPQ0&LcrZ=S<9Ae`b?OWToM%AQhVq&rxD5``Y zC4jRGUqhuHhAa+FP*)v6< zGpw_t;7tukc-d#ca?`W}W}bHOgF?#SfsxK|^n?I!Gh0rjx3A+1Si_07*sCTF7H~vv zC`8?UmZ4?{PWeYS-XXZX=sODE>%1dNi7|6?b{)y$wfGUnt9Jq*;TgzLVSyIJ%8i~- zzf47*dQGTUuXL8XFB;Xu5HQ^C&SvGki5rk-S16t-pDTiX?R_Y(OkcPEWhp}zNI`e^ zEU@dduN$dG;F$kpSQ-_}eIg3^_gxm+PYdY(#*rJ1L2Kb(%VupKMnq107pHJ_qg%*G z89FsI<8RdRoUSkAsq%O?khbenF%XJS@l#74@wYd({eDHyYo1_oPWU%{MADUH5od{Oap7 zthMdizYi;A;~lCy#mR;+gl@d2sCq0PaKaB)pIb~96%_P=-q#hZJcvNq^e*6QPY2%` zWz(_DJN&oS=`YXyQG`FDf{#NX2ESnM10CM+@0(Dclm4xjwR`Xb!ve)))GnV)Wy%!xWZIX?SGsf48v#E(RnuV#VF41RCQ(-J0X5)TnVc$19{%eMoFm9% zegOv@dZSa{zF7?x>Xqr@76V#PUV=yz8JS**i$q|%aMzvn>4Nn18^ocK(5L~x7yt_z zN^&3L2y8OBE1F71yAAVag(?BQ?D~#nFl06Wt9{7z6AP~xy)wAjYe=UfT~xrfn4f(- z95$k4VPO$ik&u*BG+b`U#m1H;Gjd4Z)P^M#zU!2L*dM5h12Vml63NuI{$JlcMVfER z%-fV}g8{!f#v)NSya~0+>-z@;?tPKc($TRc%?{{8$L}nWsFKSNe5=xpsBMqOY31eR z#lu-Ra`fnwbB(f?f#pQqaTGy!llXu>?CpBW-o46TJqhWbg1~{ zN9kTpI+YEA!e}G01AXr9AGx_BsZmKB&1l%ZQ)T+DO|K7L5mwDPHWh5qvhN-^Q7dBx zySNInd?~58FJJCQK$@(gD&(4jn%t~>$;AwXiD~I`y{FkY?x=nO-jlrCp#aFqaY)YT zzBs14KG6gNcx+2%9y2(+XnyM+EM>_@B@@S-z$-486f>`;^j9H9U>~HAR*v?nCdwyy zlLlUZN`+J2)#}aznZqH;n|SqGw{F!|dk*SZ&V(y+Pv~B}9rL}+8j&Qnf+TT2%F@md zr)XUqf~hug{>TQuxOcF~K*e^)Y4ZI?8*vg=1Hu~Q;^&p(c#(>4AYZGX!g0L8f*2hW z15-8EH(dyEhtf#77f(nxe86zg44RCwfo1Z@>Fb)YBpJ;iJ_-3_QzISWmo)wb{_7;t?=TAMIt;AOXfDRfX zDOz_pxVE*nZm@-iFV~yiD@?xBWFH}WLIzOYmh!XexP%m^G*igN56?JciGV#-uJ$n# zO3{PuU&`p?!)QxBd&t#R+m{dXd#zeQE}D)XbwC>$COXuUv?0AwHn@mNPO5k)Mu^F2A_gp?XcbNIwgJXXJvwj~rBv0g#)*_v`9HgJ^jM>a{hJ zav2IL=3H)bsI~=Y!fURy1t9Bz*>y_jp?~-8U7wMIg++F@N=h-nueuC^cHON}Tw2=N zgLe1>ho}i~-+(~B;e7~G0i^$z0MV_8!5iHO%WV}Vv!FyZ<0K1JSz8w`?p^!XVi&*p zJveCqHy5Zg-t36}k`Yiy36+9Unc~na7$J9rJm@nkKdazE9(&V?O6AL{Eu>PuPaTu+ zrF$MIg4w&<)UH+hF{tvcCLL zN_LxHispkIAqQ5EPykpA4aI$51WB_8FgC+6{C=l4y66S{tV0x6jXv08AN+Q1OLsn`H=?S+VF@7 z0h6}}p$n;Xa;d^LTSH{-sN3u^jMo7n3x`;z1HdeM9`%em)q4sSLi015{Tn~-)I&1t zs=>mXs}tXr|JfsOSnVD?q`3B~8Az7g(zg8LB)2Ow*cIumAKu9Pl%t{uu+~1W3NQmn zN|~RjcmqaBOa))RexXgTOJaGrhWc zAkzOO?F7~Xiaow-OaWE@Q2q&2?f;v`L)hl-C#=Q)BJwDA?R0vZd>(+Tj+JkaJV6Ph zATR%d71ky{H&-4mkS7Bou7np7KffHnV291;qfZNC^q3Q%Y^=yGd^2|m)eTN!hS|x} zu&4+as9gd!vS=a@FljQ4Il-Z!Llm@zsViNQ#^9t~gIsUo-fR2@)n#Ssi;cSt5*vgI z!Hgp6NL?}B;45}{bcuRXCaH`OMVP1FBslIp7_p%RqkFLZRrRP~lX^AH0%m^dKLeD{ zM6JG0*2hb*qCYAlYS#S#Nkk{?oJnfCdlvDy+pfm~9;@B(r?;<^_wpwyOw!4c zx`T2#-0mPY(SLd8{*AN{x>c)<)fsw0JKeyFV<@JN^H`y{wax8c`0DGA|Dn1@aBjAR zh48oVH2h3=sGS!awqul|!0{zCs{)=g?6h`qD$Gh#;QkX%`X)KYVYFdIgcUy zS=Ica4uqI%rq!onv^Y95st*Rro@u%&W)b=#KAtnEIbEK$H-AGT`*u%%o`!j;?p<>j zI@QWm(hr_3T1H09T*eJPHO?+B0%jjxV7rc03`E+6R%r{gc9GpsxP74=-SGABH6){f z>mR{=I>sE!hhbR!Cl0+r?E(Y=f~2ef$UZ^PUywk zv8iDf@kV2p>COQ-H|&7T-Tou>f-aQ|TDc%RQF}_r#h;%(N)ko^ zUOiE}e&aaBTQz}O>5wiR@mmu5^J)H4P~*gtQ(Bs8GlVOgbdSvlZ3*WJWC5cQDS8YJ z3)BVYHfjBnjJHUiE-#OS;y-IYY{-q!Ez-`)$tlRp{FX_AGtTew{r1$f5ScnW@t+zE z@tL2vC#=2mp23j-9v_cSoIE)yAk?kRoghm~dfF$pIg4bsIj$xt!Zmi7NycvEECG(F zU+sWDOh(R*Lv?K*AN&&HGp^$93hO>toLfbPHD%5AaWa9_)}X3Dwlq`_$fbv*Cbr2i zwS(9HC!jbe%R&%?sj_k=JTm|9)mF~)v1 zFj6;3S5HRfp^-013-7l%U#4F)A5ACB1ftRr?SWj3fHdSdA|@yMbzeL))>ZcgoqHmLefal7aD#2Xa86(*X^ZbRnCT(lmu9#vESWdsRR zP_P*iZ2FJn_ETvFW|ShJ^D8z*_(A>nqR_d+@J{qENJtnVHdKt%auLh2L1(Pt^SPL{ z9~FD#5XGC|=pLRnF*oPrw|9U`dOwDB;Q>j6rc>A{7bCa{PrMHzMGJkkJh!fKK>DRlM+>-CVOFq-w=`gA!yilqd2)~O2moaB9k zEKq=fagB|(U)}YpVv@$^w{PF_Ib^7$NVIl!IjWy|dUJJy2!n=~rz}59pj-uiOZAHc zJbv6k@(ra{xXzI^had(7FRD8Zn03$?F(+^X>f=ks=OiX3_StkzIa>oEQl=Zv+~>PZ z8qzXo6LGmZ{^3*(<;5S{*8+;o?>xZ3`L<{g2Cyd$sW-QNE|rbeVkK?B9MbH=pLplC zD;o>$jRi=v3*4x~xc^}Ppc%@OWA?lWQBwV{Kdne*5X$FIGoZSUoIbW%yJxLX?dCVZ zVrux@p69zu&x7$4IG7(_0{yFHqNvtS%ynXYF$AOb`IG+HWC2Ya4n*Qosd6&&lF(yaA5r(ua)U7CsFj99U ziRnPXCY;s@H(X8-H%^{%(-nn#D#zlhi9|moSbGc{R-tRh;qB>P0CfW*vH>6@J`$+z z8L){8Z=2%fphKmW^9u^Xtaf|2IFU~)#xxx0<{=Y8#SIOL0xT z8|?@IJw-2yXPXOGIBw^t{`w{DRBZlSgl9w-kRD4&Im+yvp zle2pC3Y0-~d=^N{t4P`yqC0|VpNe7f->4$?OHKbrd=R1tZ^iWx99oF9K%GiQESHt> z)^9Dh9SNkfVnTU|=Je%VGktIM<@J5u=WFZ5U*Tf{Q#+aXNa$|R38Lk3h=C2LT`zEr(!nVo*tQMexCsj9YE2&6sS5(;z z_s%sqfu`5|!e%=h2Tq2jXE!tc4ZC&+>9y}^T0jMaK<`+t$}?Yhd3T$S;auMiE*1Ev zBM0iRzxcnKyU!M0U!@9GaGP03L(|8;_C8ZT^!i}!0&{exT3#`h;b8TJV=#pp&WF;< zE4+j6ysfdujT<)#+pkh#pkX&&(0=4F3HD_H1X{f}e{Di4+EB9gRDz|Y36>6b6w+F~ zatjV|@p|++pvVG_W7lQrR%ub)w_s5LAJn>g_in^8n_{#Tma$PqfI~ zw^~BD^_5Axqf@0DWK_wj1&%kc7kgpMp+`$(?Brks z0Xiq-kOmnrkHH`}9X+p^d{BuT29t#JdLu(0dFofg;RelEeyqyQPKlCNmo@zaiN&6W z*EPSWJx_Maz$mo(+G^#>Hwc@*st*4h!yDvS@!MceCEk3{lyAl^$%6`5i=3;tG-d({ zlJ1ND*o=^=6$fAO$DDmeB_4Oq+Av;`d`xjLS5v2&(Z_M_Q)yAYWvQ;~WUx>^z_ieK z@9@OFA$zM9lx(IZf^3<%5zy1JoA0j@)7l5}{>lb}hK$kFfc-+&6 z0{}1LVA003VoDV^o7%dhHFFL~CcK zgn2#OSIA9MWaC2UPQ#G~4W^J>BqlPF@RU7SRaUmy+MFUFN>KLsT7KZm$xD zQ~-4mPTNAV0;mARB>zOEjwbM1haWj|1THENH>CJ1KSe(cFoiZPQJzs#$U^Gd!6Ae+ zofypxoEo|>Td9x6|EEkxqKL-!CfTvHSS7J-AIc)v>W}|zB?f&<68CJHfHMP>Z#TaD z!cL^ZSKqwx8lbHC%Uvfr<;ztufPXKo77z}DkuzTOl9~?nk5*H@_K*XK*PyD6I#35Z zQF_^RansjTkWxg9`2(9~3w67_4IjWh?w_3kq0Wkr)X~|?i<$=P413B@R<^MTU zdB-o@BD@8^H}x}Alq6_G%B!jx%+y2$EVKdu1LinOOG_Xt<2KB#_a;NuA^aWmh!C+6 z+MW&mn-nnvDmmph7UkCPdl9&*7{Evs$dV)QJ5g?rMs1aaQG_W-o8VU6^B^=|{dLV`tvHsX=vaAtf{Y1uRNZS zAq3tqTwx-QQGI$;GCI@m+f0cf<4}gjU$4s84$eB{t24r2W^O(MW3mTB{Z=nv{MM%w z6cr;_>57FTBA|6mlNJ;d|2{SrH*4PRVX&k^%S!8{7v~BP6+q&b{C!3cpn#0I;$dxB z2~#Q&nJC`K?*T@L>NcRG-RD}j&RyYHv-TD^Af4fTHj&UBDZ!jo17f$Ct0fH^`4FV~ zGwmS%!_DRw#8bWU7KPh$wZwm0F_4DELEWKJAEa}afvT8$K5{5vO&x}NgWEPssddxU zRgfnxKi`yw4>gwk)RCDu%j6!&tm7kTASw!dzIR~M3l$(}P-gb@47cH@OgjKK5pqDc zTWf)TLrLCh}EvuSfb#1Z1(#9ih?71<{G~A7 zB!bL!f4snJ+aVAJ%$~Fq5>p5i1Bc2=&lL;V47o!O0KUA7YZc5pZ#nUq|OgUc@W(Ny-ck}XLBoxhp+oguN zxp|=vB*zAWe#jtbT!1nT6_p+Xh*r#svKnA$_ ze_GsD)xjv%mYm-d&I&eQ(|F#u-)>r|!2fPl`eGQJ4C&X@_wStQ7;f1Pq<~1qMl5D}_4HJUX=7Xb$cYu$!G{ZyX2jUaScFA-s&1*29HHe>b}g*dd%kLOt(p z*7qS+C%)wR5~;uJpttV#hsN-~^TV~{!M8|bJE8)QYrYBx6#wTM;un*=f7HcYn&K%n z(87oYk+4R|n|spheuEPhZ}=ityiyvPz)CLoQTj;@y_l$)eIVfkDi?CRZwMF>Ae?V^ zwo*bQTtP^2btWhaKjL0vh(s(BTn|W{QMwFmKS;(+B9V~v$g5-JSgP%^$8~l@UQ%Po zeG*Zaq}F>}XGUbdNEhiqclV!MyUvO zU=+{3*`bn@r!%OTI=fJXt#cr=i)W&!?_SUfBuz*TDk4K0T*t0lxUKy6n))v@Z+2!G zIr0>Z?CFV|{ZcvkB{tnIyOv?}#1Q+O;Y4ju4Z||s(4>@}zk3xwtU^}V+63z`-mc-YdqrZP#sWC7AtqYwu)sVgLPy;GG2~#sO3M*Nef@ZcR}++oQ7m}&GkGIF07$*ett}tu-9h0PN`?yJ_$+7P{(KrG zd;a(@fzD#XCJ2g@ZU>Ha-aU)EC}gzz*{EolvJ=1W-uK>Le!Ej*byY7Syb6ua{QctD zFu6glE!jr1a=q17t|3CaEoCpy?_nsw=%WSp_ti(C5=HzkrH4`vAT_G~RE7)*`p8}c z1-ZD3JUo%axwQGKy;xg1y=m?bsT|NYG-)K#htG9KLw_XCQbApq^vNC(b&PgW#Ka&7 z)C;LZd($qT2||ck(1ZBGepXsA03%)>i4;U_r&pvMabiEA1KOlU=RqThDQusf{0RW9 zlu8(kZT~;ba`LR{znbO1?g^3c-adpPH~(WN@!{ zgVykB>W~Bb?^HoRa>?|L`kr(!m3IBrCwBiQ`#4apG&`!AiqGg+aa^GOKr#eerSxhB zp}FvJnpz$=pbO}sDVR86$_JY4mv*5_@D&j$R-g%>rT2i4&W>a z61_`*{t^3v3dQ;x#>8X-MMnH}Pv?j-ARZBI{qp+HvE!(2L0ZFm_bI8)c4ykb)r3s^ zK$79`B<+q5!L;K(C}GUwOghgc1*|^s+|x9o# zarT&~xdjB}9FQ_#6TRGekMGqfR(O@%|TyO>Tqed%(wTWZpzsPB1W}T z57R_CoI6Fp9At?Ac%&w)v)#A}pp3st7^JX-AtvwsMtNQMO)14>K2l)~{9iXnXiUw4 zY(T{EuA}zVA1x-mEM(L-HTB+%5?R2@q`LLClI&4%VuDZ(Z{X6c=jyc4ZK(!s+ER^o#J`38q3PQ zDsvHMeyl&(_|fFc$(aupRq|jM3#ABuFn!dRHC;n~gG4I0@`LI^G%9){GM3N}&c$*qQb%187~&dp+A)zK5sGoxVDGMxd_E2-@`HQ*10 zJ1uK1ROgMJ?>Er}@l7qasAHV^F1n`=5uteyMi%tEE-#CWb+T9-5647tjv3|Qjx@e~ z^91}w&CjqFfJKT%r`#VuWk%{ZUgpybebixkA=I3e_?F828(XEYitwplRw3X)ddy9_ z<|HE+k;%d`y3d;aGKOJL&~kZ~BQKo(?z!-k(g$^){LOy8l$#kmx#^W`zdUsY{t9d< zen%C$%x?BE>5E;=9E_7fK6F@n@U>pf#_>D7|8mg{29He3>e^$-hEYvooLY7DH{f-{ zjM`vE!|8t`+Hfpd(%5*wD3nB4@w!=Q?gtH#(0$4Y-XzW~7Omf{Qd};>6pmRiCmD#| z01H>7tBo>clpMK|I5Ry)YqQR{DbJd(A|V3i=J|+s4jQd|j3GLGrIW}$YVE)xFIT-j zSjf>cSah263hCqYQI2zO(zdp41~Vn(ff=~HLwPZGYN(_Hj`}j!3YWxKaf9!q8xpJ& z=S1fV?N9&lcclaVxc!J`_OoY{0{nk}!;VJ6Tt6_UJ93mSvRgTYym0wH1~8xX^2&G{ z_umHt7$(k3@`FE@;uP#n{TX1WnDdm0Jzl>U++)gyFSoX@LW9F?c$Xs%H8AyOV1tw; z>hV&WJ4UId<|zn5ckI2C9=Dxur?ceZHE5Z`T6Ig zW4KB~+aYE(G`fD4_z-+)6mGk6va?HP+{#KBoCX7q|M^&zFjV_J>Y-8@pm9w9Wh2sm z)NP@LM*JL#9-dIDc_A4O%^tT{#Yomqcl)D&|ApxyWy%LaKu7ieY5`%OWVzWwFYZ+V z2$Yf63>(G6H@fSq*vT08~$KdtH!ueav$v);W`3{X~V_1`M; zF|D0#Wr#0|5dBM<);~k_QOtDpr%sqG<90(&%gMKVDm+q1LIa$Yo6+T!&ERFd@q>l9 zy3T&+l-(af1}%5_A`N2m$}NxefI(b*QMotC#NJRfbH0rl27A~CwUtB9i;HIiH_rRH z`bKlL$YNC}*|z5-0}S!WOAq_rxQ=~4EMVqroa9dYgTxXymXC#D$7dv}jM|JwE*cL! zO*N@vkC^TM$sfaN-IdI_y&R)R+8~O3TY03MYkRggl4E-{Q1bS` zyQjlNtzM#AJE?x_ZCPGFCrB{pH~HS~r&2oTD$AMu8l z#EPySm$;!K?)Ip)Z5y%s10}G8s*X=3s@qsCL*`Ec~Y4HY^f~G$C`P#>+EN*#52{ts!g{k(%QOR)D zIj0uyEa^G3SNPRl0hg%ZAOmF5nAX7U-OFfjv%=h)zJ$0iT-H`B7T3f* zhJvPOpjD`4FYaAHuIarn64hQVCL}H4WzdSo5QU&Pyn5NBDHTUa=?*eDlr zyKksL&~SWrM_43@oyGai_Qr*cDV`Xcovrf}``I1*hwaW_Jk67V3(fwM#rhQb}92|Oj<)zDH;Cd}HI3bUVC(ZeMVe?TF)(JAC-CQmc^NCrE^%V-ey4 zbBvx}9=He_b!|PpJmgmIvFgv%e60oyP_LiSu^-FZ^6n>}-Xtyi`xeIdt(QW&B4m)$ zP$tcCy-qpQ-{t=PojVC5M-RgUuViFE z%6P@YDwsKuPbUEenB9?i)r<>Mg+V<`gLQ{=Ds`uOyvO>MW2*Cz8E`QSM{ZVlEhvk7 ztQh(>tV$KE2=-H~_LC*L`^I-wy@y!T_Os$a*UsjRE8b2-^eFw8N+SG0;8!WftOvd7DgBK*m0AsFI{89xWi8p8FWzC@F7tbo3Kt z!sL=?qfuS=3IPV@&Lpl7K6tq%FBbKSesqc4(t1xXXq}JT&6F`|R)Zk&Tgk+Y7iH z($Z`_rgP^_D}H08sLOh&Hj90(Z}Y*>!gywuu`fEXTC2*_U-)cK0*!sowy{52+P`ai zA@?Kr)Qo%R?f&d65M4!^C$!*LDeBJElRju~kH5`vGgDEDVVRCIZpiTW^Dk<}YPsk8 zsxXq%CX)@Xo*mp;S0NIO{k0B_iqML9_2=;T&ai>+uB}*44ULhGwe@C~TDadvpOAogt0ag7YES=B;k$GCi0o=RU#)=O)&W4yI+MCw z2i`m+#q&-{Z_nxwwe&o{PJ@bo!viQ6?8LQ zzp>%A@%AhiyWw~-W`JID#}8&yIL;7y){M#cTHDn}JaxHre9Sr+%#c5ErNFp8Cyp=1 zcJloN`!SY3{w5`i*Olm%YkwGO?z$mvvN%Ss6nn36XJaVzn`2JO=a&;txEp=q{B}2k zlZ+f;%*E##l^I22FSM9D_Om6>y*ay^jpWk~lj5g;zuvy^{oD2F%eS90$BV>$z8?Ls zU&^t1?Ow(+4MyjY%NBBPa(I&j&1(xqzd};+E!7f_zz1zO~E=w>NjH zvHg@4rEhZ%91|B>Kemy5Jyc197sILS^uY$i%MYok0O|)Se^7=-JQIh*>G}eT8qJN# zPjJAYL1A_A;=!w`uMCS{m36s)efsMzbZ28@`-OrcdKIZu&%1+_wnLoYvdUuHC{VZ_&=|Id8+xfcquhIQAeEIQr+KV$2NtK+Ij4CNhTaJUaljXIh z=8$aAO;6|lD1770CZ_4$pEK{s*nPj_^hkLc3`Xk`OEMxakg~kKB$QktIPQc9c;C#w zWpq_@riK~rq-EdAk=0&_fhcb_bb`?cI~unX(`Qh`<7*Yp`cY?y)fivuVjI+>4vHn9{^C1Ncyqp-z#FT=_xtqg`X9Tp@-8{Ub)(UyFOqDcp25n2} z=|y4UOp67l{S!ImP#I}|`O|D}BWv%awU`QHSOo+QlDfd1QimlzWA1JL$>P58Kf8@% z3e|^;XTP_U(s5qrl55H_UE!flt`&Ez8&ExB@p>j!X@HHGoUv8)Bn-8}y5#h@7w zx=W^e49NXD?pyfvsc#<+;;LjpE0Yh4&*n;O&F`S$}`n78Tg@a>4(E=yJ03#KtCk;+$jOmPoM-zKq9)1`peqL+g zR1TF)zS!)*Sg>4$rEgw(kLmmvrUCQRS{bu5)w#$nhaDo|r$$e&21Z{0P_)E%BvR|e zd5-*tDVnoQ)9x)HaiPYa3yx2Jx9XSm#4XzLgi}_|t_;ed47->cKTTi;w zY8gAtd8f+2w`Dc@q;~l(CQ>_V5|hP@o)A@f120le>*zH@T)U2()jR5f6wHYJ45qmKwYs+`MlB#5cB zM>Lr)4$hIHB*eFxDwhW+%qk7|ExZ~KkYS>%VD?Uugr9XvK8=8U_|`FLJed?jiaKQB zkDDJQ_xsluxyLw$MqLq7Wv$4CzUifl!HkqdZJ_~*_x!5f5mQq{a3;699Mf+8! zApZ}%yJ?A6D}hNSQ4MQdA_^VWn{|^Ojyo$S>$g7%qFbuEVrSM{KPZNq$VVNT)pGA= z7;4A~+5F+;xMI9H!sWI6F)2LrC{O!9dmIHl@4Jl0j{^pa#Z$o;{4`Mx^e4w~tGzxQ z@md~lr#Hxrj=i*vuk^#@`L$zGQZk3jj|w=O;ld-B^7E{F;S_kyR~{ItEXZyeT5L1+ zL%WY}s}w)m`9_HSgKh4c+w7@$T=N7oS+E;su>ei`0{&t%T=9+RI7O+%S3nVTU z0^1`SlpNf4{qD_ti!SbdI?GIOX>21N%PTBYWAXRVv>nnj8jD%5|*FCU4t{@yW8`xaA6nv+y=Cov~*+7 zSC1Cwb2dVk!6#eSAARJYHIU zY0A;TLXp%q9^-7Gu~*X6ft4CHEPfsZep{15Y-r2Gu`UyfcX9EQxJ0NQK>dXl5D?fP8t-z#=X^Q2!=i7# zdb}9=y@5<^;Xim`qa}oXfD-yEr4Oz7-n;2v>l-B@CJVjMGtuLQ`E^cuMI(N%Y(k@V zx0JvD3kG}ks5Elb&jpCu=4~F%8q=0dBUz`_+&shX zqa;dXm-N)wT~Aqj*UD!>)z?JRrL7$;R*cb#QW3&esPojlR5=rLyp0>}O&&{ofHksh z`sp^e=1a!Fco>Eui`tn!mI2!mrso9=BD@%t>rR#LE0*HPXR7g43Sg@%8-BN>?WM`b zjgdtjjY}mY+uk_etSolFQe~*orj3LYJWWUanH1bLXTn&D2YP(nTFP{f`Fb~BH~-FF zUE3a-{h6iK{;WDbXHt2pZp!9@NSWAV%YjI6OdDVDklxt{VCA?nl<`@OZe**~?^puA zywFxxvjSb@+3M#d4QR5@l5CU%OuN^!KeKH`a#OhNRu#uipqU93lFV5a7bsoR@tw9y za-WyiD~x_sHKBU5WNanOsD3&_7yq0`WzgLHwjFsLD{IK|W5zZjETz!E-#d@;m`boM z1I(`uU2Mo_B2qWca^uVk$9}e)5&0~L*)^uMU9A}O-}P88VVQ-G|14M6dW?$zX`{#f zd!B@0iLtMPY@tufc(Lk)o``+b_SBqtK8Ltq@z3%dF38Iq);qH3ub;nCY!9`@%5Jrh zqj;HswvdlP)xxW31(2M7-x&AD#zgPVb?AyX`b3}EZjo*MU|U}jwV2m)VB8B@k%?^R zWY?UAO1t6{eIsLs^cNyh=v>eZXSP=N*Jzv=G0e_t#Fr{~k4v6M$MfdM5ABS~GI95v zPV!|LD^JseHEY%#e#z*4#nN}>Hs;=BtoEi0X~JuJK7{t}Xvg=Lgxw+|RvqWfu9_*V z!(!q$k2e3c>@k0=PNC0Q9jGI#JEQqN_T8a(2wv*pv@y`MbXBZ4qX+(;_9t#`z%L5#RnU6mzLD zcxV0oAwgo*sL_+CE-@ZC7%Yi|mZA{{ynddDOLG4LULZVE`r+r1vME7x3!FivIGycK zG$CA(IW!8K-gvr0{e6;Qh}Y(0)l2ELkZ!pVsfmp{1*FnyN8_T$G>z=qM&;i{l~R(& zO+q#o#|xW-5{x6PR=w@3#D+;J&!&?{iwa8oc-P|2iwvjLE;bkneeT8@GehUUMaRDF zA^-Y}7~_WGj!nx>!{QiE64US+_c%$;lJo$NuU1dfN|1#VZyy#0PWbXk?$=UKj~Wg(N@>=m5*Wz9%`_Z9NNi}yHqfjGfO?h+cCF#+XIwDkIqr?##)>gU!?SLCg3(W^45~eWXe0HR3j4i7 z0Xugl)fzb^>br%#0?!Yk)Ha1zP&9ogHw!VeExuc2@l!)A`GaS_|Zdm*i zZ3$gPzbxeL;ZD)g+Cp!TliS2S-B5{*-28%qU#%KJ>KYnu9Rfq|1|How2M#)h_oZHg zi4I>VK3!Ip!+G@QnWAcg?VR9eexf?{0~S5@{;_Ega_p;;_zP5rwg}hJ>J`lg$@88Q zpSH~U3)rAqar74s)L*F61Ir9W?aT4*|n?PCyz%h zTVesBQ#rF3zM=jp+)ioyB<14*V zJU^jgE&~oQOxbj!@9VT3V}Kve#x4e;61P^dkG@kLE<00(SGKB;MhUIGE0Ixl!Zq9@ zpN6_N#Qd-vsk6*JsgpTaVG^QH9g9j7e%R}}6ugGJ*7RM8qg7I+4D!j9;mbNI~ zOQi$6#hC}M{2aw4VnMk}#!&2y#@Jt4O5llwTRAtRr zTZ6FL!!G%tC`msjM>6G5=}y~3mGu$}&(N3S)RcY zp=rlI2sO21!BH-9mV}T-Oxp9hZ-S#VoVN$&-Hp%Xv~Z(sa+huK7gAhB{oiCtBX#+d z^315|PI_3b1#<7Yw!IQ0Pgk5n$CD$zo3fpHVzt7xX;3M4+Ix(rZ?kLRSyWf8L*(~e zB8Brskr6w|wQv*t)fC$1*4pv^kFB>3i!y!Rhf!Bu18^;rTvtLwLg_|P>F!n#ks73i z7+@?wK|l$S?k>rpSw%oVnxTdP=@>#9W{CHC{Jekc?)Nvx!Ewkq!8~)vb)VN6T=;x9 zS~v0vPpy3{4`m!SKaSd5A2sH!v5O8mP?x}IwL@CkEYi%h^D%|1fH8mBXDe6VV~>1m zaR|RZ$^*`fk-fd=Ke#OHPBn8BbOE@1uk4(}OmRnh*XmUNHR-yIn_7DPsSZBQD!E?M zQG?*{@ID|wlVIFG+jZGI;|vA2!VF`z28bDbfpqmHLza;G~aS(ms67cKs9zX9Tc%PC!7w`25xPuVDtBK+F;r zZ=9gYXkQFY=6snt_)f5yM0@%YkF<*A}Y|-oAJ5FLM}AFH|%D>Qh|r+OOxnF_tY^ zxaun$r13d*fhMwwY}Ovp@K`?7MvKGf|7e7Dfz=U->K z$l(i>FU}1mKZ3t6G!=S2SpY|x+e16p$?|dI!Q!qvC+7)hEoC&cw0^MHvrH?$9 z&ODGT^Cu(M0a6d#?CL^cB3tueT!d=*47YFhVMmqk^qqv7VvK#D8i_%{gg#X~ng;)+7Dw@M6frHm0zw z+rC+atIyly2zGRKC+Fgta1AZz9$$R2J9U(4Ivy5gr!-di(8IOwgz|@pW4AaaLfOuR zhsF&o?KK;qF2wP(=A!_J5nO$d1iQi|Kj{AM#_c89M*xJ8!MrKbBPDGbwbRUD$4{xU zYV9&#=pb{TO9oMUTH;d9~Q%kew8jF!RPU|>_&8Az&($lEI$kSRYy~FxDfINwB z=*OSV;GuM<(hxS*9a(}Kx9XKXyzTLrp|&nww5nZCJK;FxHp*>ZOe7VsKzv6;LknQX ztR}8H5B&$#CPy&Bz`0lWmFWKSvw@SCnwJ;C(_q=l+*wuAeBfBMJ9|&&G%XW9VdU;YbN(xS(Iq8%+aH5%Pa{@v~YGGd2n~C_T47*c^uQsmKj%C zSWr^aWxB$1g(juQ8FSCuwsgNJO%Xe+ohEIU-McKTGw;TtXS1_ZS>A+sU=;N0KSvGo zunU=87F9MIlOtk#v%*W&4(> zs^*4vSN)GftJA<6%PR5jrQ&ZY=gga#aixcV!6}0q(qQNuYeZCAUj|BkLsVM{d_r;i zXJ=8&RrjoDo#x6kvMMcHveO?|g^&N3dFO2qP@kMN$V0p-rh^Mo#t*(UUgdlFxDKKf zkf0sJTQhG71N1J0->G}99JtO;D~ug26P9~9)IotPvkcr1_^-6&hIOlIs_4!A*N5yRJ2;_^2Hn1bYT?_h5f4!2IYd?E*ki$@dJww>q7;dAK&(G zRgU?Pp^?(9wo|C~)%>OlDKAdk$1{e268|8626a%?SGF0kbZdNgx;E7un-?u6I)a;x zWn{CeP#b6rTheqwudPJOt7}39)?4`s`*z#%v0JrrYd?GMdEJdMzkGl3L-ppfu^voySdVTKdj1*@ zqa`oOr#sL4usJ(Vbet13Bn0&aeY4_Px{7G+GuLzMtX%zDC(mbCf6IIeGxDg`^m%ZQ zq9IHLbp#N3mK%LOo3V733CE1`x0?wV3RRVAwZ;#c6Ba!SO0*{A*}pULU|!_aqb0qE z{r_5-Y6uiJJ^1vHlYV$y!kpBXEkuyq-`*Us4{Ye%s&zyAla}OL*IsfJ-01vxGb+w- zsf-Yv(Et9bDbQHEwA%ktK)J064Sz~f{`@Dl&OU2O>Ro7PqWE&GyCTWf=Hy&jFTIomo377Ng(Sxl;=)=23jtMk{>^=zp7VY2mCph=ESu)d2>SWb z4*A_CEIRXBF*zkJMnTM@CK)qI^d+00^i;;CU*>nS^o@`cX$1XTAce^O&tyOKyFp2r z)k&FNW0{BFZ|{F)2nIoImj+hfpI&yzh^2Qk`q_wA+K%gGulv!PR;=?YL|4Pgy!&+r z0v!p$dUMYg1q}-`RC8zrzDa*eX||$W`sw^?GaL-a8Nt#>zSqIU!=_e-$jepU55hdWqi|&kQy3fbslv#~R=)3-V?Klr?vn8z zFa*1vRl&&A!blp@CS#W98^v}fw}JMtqUMO<-I2*0+9&8*x%#k)(Si}LHT4A$BKpR) zKZHuqvLz8V95*HL8atyeW3M6J@`i4_XmMgstEuC?8AAM9z*}KU&;P+b_2)m^X@8#Ji4LGd5yw1w_Oz zc+PnA22(#4wY0y_FUV1-wdc{=Kdqmii#8Opt*Bvb9_{p|ykMo4x~ifO#k6G>sx2WX zj234Mfbv6Ucu!^IfidyhDSxz>oTi@=e7-^_t1_+queX`X=D1dy+^TuyHCqy=?eQap zPPaC(#`#YIrK~6K#f_ppCSpQ(vEDLrTX=N&_fM**s(7iTdhY$cCpaS)D4^pE-B74C zQ7@ICx;7z)>aJZ=pUuIS5eW&yS+2N@q$`QE(lYltdC|n(C(m>;> zBb8dI<>lAyi!w>*k8+ciNPoT0?8~zqIi6o#B!j~3h4)oBT2vfmZumbjIoNZeB>g(=8krn$DG zXks4*Vz;;-=Q9XLWA2C*<*@6bNc;w@*TL`R`i5Ugu~1^r1)mzPOpWHI8rnMb?!;1m z`VEaVv$UYw@D3{RjH70N=7d3u854Zv-?NDClOvnlzwde`^+?S`{W$#{1i)t2=swd6 zGyh~rjL4B``A>!dLXx*xv`f7IaG`o8z$1D4L|-t+*F#U|!NjzTkZy-lp_?w(q5Iv| z%|u4r_Q;SKi}pv4bQLSXRH(S`r;2EYFQ^yQ+n`xLegjdn$5e!(vT`q2EsQT=!2fb; z!xQm>9<M0oh4JM{2J#-da+w1h|8Q$3nuY#IdKS(bTA#c3BW#tw{U&XMH^=1WS+fdm~f_i<{od@-o|qwQtMXl*9iYjnUU0sg!HR`ON>Ob zD|GYu-?Qk6bKHuA_QSXL(~IJ0yZezvkE}QvQ7kZu(kw&Ve#q2Rk4zUXO*Fdhmoi90 zjyZGH)vNbYH)hG0mLH$n2pq#cdRI7o%uZ~J?YG_jo36L7pY{n9^V(&__g$xgR)vpU zusNCVRMbqoseL8%mN+g!+_-I&0TWE96E9ZAbdBucz9hwds?-MvP__ zP3T20c5y=99)-lJaGVz1VHL8o)&ZL(PFF63%&MR!QK+RRU86*277L^Zf9s(Vi8znP zWsFCj4jWiozqFC^rj1N3Zm0KZAq{tpyT40)b50zaFZl3|GvNVWat+2-aJt-ie4<0C z&!gURIYrU+X=S&?o~9L^d8N^Ne?IfTdTzVJ0z*2I&AHXka#10QC#O9+K7-npRz9@M zE~{5`W>;&X$;xJP;f{%yV|=?b_O+(wS=A})|IDjHuX8s4b97z*1M1|02qEoTEa99cY}D=5oE@NEjDGzAI`gICg-Yn z?F_28w(y=QOwq3;ODrtrN}u9;od)y!T6J{E<1IMi<_nnNu+XAUS)r2>XauALWoKqcl8aG*F6#t>Hyi0oOb>Oj!=~`K) z-5Pdu^42pVp#u54e7Z@}@8z(&l{9u@Ll3KXM=)uc$Jo%V7DH7l`l6S)JJh};z4mcx zf5}mk-)>j+>VJEZQSA%WnP!uby1X`ZKzP(MJ-!>M2G6AS5xZArj`w6pqq|9B`+Iv^ zb)%W12R;`5tw1j75y1L5<}R#=K%>vurA<&RJL$?bg_Hh>c=&7$+2##v^J3ZDms9>W zBh`OY?vphXRThmVTxglH`y1%ob`29Ss9*?#A1rjTwA{6g4RfCNzUS)d#X^W!D_U6C zZYsz}X`1|*8|s%xBk-HfJoOxrSn6m$pR8#}$B(wE>owT-+m_1UN*EQhVR)~-bc(df z5?4@DWvlIxD1{zPSGh1$l6B``O?tT*>V4R4qa~}Ko;|G_h{ia!uO z&p&S>i&9xA&58nNy0v4^YO{ywKr#KSF_r(ew*%id%bw1=F*HJch=Es)YJ5}r4La$L z!=6X{@txm#e+yEUMAH}?%-y||u$O}haQSr1Mc44fy&dqlGV4}QX7_tO0}P;WFg!b_ zCh>>x?OhNW|FU3Ck3oetInSTtI&^6OG-SYqhcrfs&ma0pg1Q#ojU5{-V>UrI=}_!^ovi>5UJJML*jwf7lSxrnq)6@n(9dspL>@ z=*ljI|7O6X3z@?DH3j(!wUrhH)&lhQ&NYV|oJ}z-ml!m)dU_pGR98LK?4_+GP)1di zKgn$D=nkJQW~0Ep!~z>k$YQ0=0q%Bzba_P^B^Fi*{*=-tBYw@W9968V{cXJiXv>_Cu;Z0hnYLF>NwELJGXZWjdZ4Zpz*`Rqo>#Ji81!H zt_8yG#l5Kf7)R~!-%1RPRDr7bKM7VdLB@b~?F1Nd%QIyw>H(0wlF z)UlT!LGX8NVsZl?+GcQZdsh1eTy4Rr5MZ1A^ar0j&d$xmV^k4~u>1G@-poCFZ1tFl zGD1#!8N|+U`%dyzG-VsQP~94ss-E<8KMQ*pE)`njk<^V7D2#DMGoiFUQ?g@^ED8P3 zHxX+bkLHGbQ?&U!-f14^XfZD>F`PzNPkPHcv#&_5<zvCz zwFMju^j-#PTDq0h8oT0?Jc)u<4O!CL)_toLBE~`>1UN4^5EV9F9J@%+i#=5gPypgQ_G3R9Mr*Yc64d zSTbgOW#+Jlhk{e=f z?N7Pd4{BynmJ{%KmpvImID9aI0>2Ge4@S5)F6)^(sB>~`-}vKqxl)`|qNM8~$>O~0 z`%;*L%fhqDZ?Ptot2=F@z7{=Elbt%FQbtI-HqL>i4_Jmn2O1U7s^_A&UQ+oombt&b zrQ&HxyuyuLF%0V1`1WwB`akW!l0GsW%fALKIUhPt)w#twboxSuh#2bkL_K-5zMv2>yD&lZ?)iYbuDx`cQM{?Q zmJ6|R-VdoK)rc1r40v*mRoOS~Lr}WY;G<%`uA`M5U{~h~a>hA;4>RZ)VmK7@z%RTz zP01C)lDNdh2_T6ii-GkH7(?g}k~X40{wg{UVJSoeWq}Kwh_0TQSpnh;h9jb57-{xv zPk}2tST7^K``}D-U6=0n-%7y$&wK2Y*}JyJG{!FR*r=({hIghCodb>*>mR?Cyc$>> zx*9in5vU5$^lB2#-sag_D%4Fw%GKuO)BAO6YCNfLhB*O9C7&GSEGVx1k`rP!aE?o@ zGF!cWpK`Cr>U9yJ%hb+CzxeA1Py zA$d8+#JZ_Qb!sZ^z-X@vMy}bq8}VkqTAP2_R}|(}EO|)Q8Bk~UFP6I-Of14R8s;`@ z`}%yMMu7+J6>qG`A%J3K5QKD_aIS1k2P%zm^rKe8+pTjgED5fwW6G-yn8#5}VzYb*^OwH?*4XKS8ov`pO(bk#(Q6bn}G zI2F~UOz5tG^-*|20*juRpjlqp(?6@O>D&)2V4?NabcB;m zO|5c5+z^AG04c_*x%pjHNkwZPYkQ^Zw+>sRoY!#2T9cQaI{dozcU~|wU1Jk3te?uJiNdMH1(fd> zS37Jb)Mx;=I1CjddV?)LR_7g~UcQC+#^slWjAlvcpXhCO+0(82B10&J7X0QoVLK70 z77}olxby9%cWzrvO|g;M#X05{aSq+fK5o{--n5b>r8*cKfmv_UQ#Q^s_NrVp2TPIv zrCHfq4AyNDIy zB^X?&vM)6Vmr6bG8c}X>X$>3MHDnp;Td^wYU|hOG*enk{E8uhgIjWL$*4uhcr${`n zkRoMVDmRu{lEIUSKhCf*!;ME?dE~}M_IHNyD(Zi)tH(qF2mf=bqk3vCZgV8t?j&Mo zpqitI)rVgnH~pOfCnGfvb6CrGPb-AoieoA}(uo_m;Riz~`rTOX`jt1pq|PPCWz1Eb z<9C_)C}5%&o_ksLoFH3I*7$jUzjAxUGNax%JBzAZfmhfF-1OsR$WY~sQf<2wok!P<>METL3Zmelq!yy+XzZ!fwkx4dHyepa>c>>OTcNH4|F* z!@5RmY1SBWA373~`g390yU@!mh(6nkJ$+8P1+AM?o9sMV zkF-@5%7A_<^h{>8!pzg6;a_ zC_Oo^#RFYOLd>{UHzMFY>64+*2_YFGK&*vxuMPzIIN`h2QKo3z}NQnKpc z3XULgd9C6F8%jmTPm$Vg6)cUKrXVj#@@LQ)d{h1JQJr{D=rWANy8QF@_zn0{QGTcU za*U6!#`&vTOW3c6rBZ6%9uzht?e+1Igxvj&CzfT*OrOcZm=X zS&;hPXOUaVUA(d8eziBHOqA}Y%~vKrGKpC3TEBu)X+zI_zmd_H!SO^pTh>r?|P~$^ScG4T63+#h}*YQa!b0@_3 zTeF}cr@}Jcr?bP1Odz%t&5fXyrd2OX<4P1$RQ8T8JAYw|}it9)v z*ffPA+PkTlnI^pdoh!hGvp_tElogL9@;d)&+J)5RL7jEl1xjxOg^X3TGTt+Cs7d=@ zJSH6eL>Ot1h@_&yagGTe@?3_FiHSWV9@+!yS6RkqfX)~2915y_kZH_$qHvW&Yp2V~ zC^~k_RSs~JC=;f9%&=wsqe3Q1dn|7TEpS|${no8H zY#AlBA1qGEK7X3=VQ_naXNM&26Z|Gi^+KFDtM2}+Z%sC*?SkjGupgf;A#ZHaeNGHD z{AHs*B!K8M=36sVTa#Ut;yM&7>RU*lY~qowk!$&P1_#F8zPxG6B`(z+hJ%U5Nxx_@ zDKxC#Ov&;==*SB+Fud!_GQ)c{t%mH#4r_8!c6XESW$j_e~_ zi(!-2Eq~hhQEmr&t*zhgg{-EE>bRp-bE}ue)RdKba(P*{xJO&-$1;~HZM|{%Yas5p ztE1yh5?_a&g9Yy8=e}hF8h@R7|BSCK%hk^1^{yYnd*>qbHr?h|B;&V7)p|Fd9`rh4 z@O?saOI6ZhX&XnmFUCpQKgr!~48*wpP?z;@wKsE9$8!$_RX&+LVYsCN6yXNcaYQ8c zq{+LXs2=Y72>%XMDNU(|3P({YNm(f@RQGcrz`9)QacT5 ze_ZH%bvgs;3|r_)nMn1f48nTm=3bP~@IE;vP;0}t%bL78;NorXRaq+~B|Ca4A;}@Q zqZ7x@NI77b;9uWZxlA6~c7c4D%33}#6=AxJ`N}^yqs1i*iXnP*HHle;AvANXhzNU% z_QN9Cm&QWy#+Ms8)#Q%F3Lyyks8=y~%}vaM3s=0y^kqooZclwgMFyi*CcVuaa^Iv4 ze8AbuknoN81UxGIYKF2!47iJ}LLd)>%DW9hB|wglTV9!-&JH;)cgCKQ6uJ-bL;mY* zBfsWk(MnC%+#_dw*Y@4%WeE`9k#}et4)HYM4E&oQ98vw@}Fr&Ip{|Ud^Rk7>$_Aff$*VjV{<3B+Qa!?#s_hO zeiLEMogIb3RiKo`3=Mo1U0R9$@`ru)I;jg-9X-?6gUZQo2fS}%^*cs3@{_wZ6YA=& zBt{%iOmpoDFN;}y{>EcDdkR!cTJt}?>hA#WKNU1XIKuu9jtho+GeOXiTYP{2-j_J( zb~y1ipu~-Z*wRJCOO4^+sYfd~uNITDKAUd{ZFnI8LsYXoki{5yk-IxpAz={0Q={V9B~Nr zF!h+a5O(&^Vyh#;we@nAqT|!%uAaGW0>?GM43AKXD=jwpc->wAjeXu*{Jk%KEV@(P z?R$}&5p5x~7b-N*iWzU^I>cwCez(9L+2}v+vNb+ax>7-8QuQ)38p%YSe-O8oN`w2> z)!kY6i=I(E@5V4$K5%ma5 zw^V{|gCa^dZ@;HeRr0;RlmXx`#oPum}xs3q}0>~^0_H>{{u365*_vPsnR)UJEWN@>Q zL9Pka-zzZJXQbCWV_UOSS;-D2#cQ6x2r(H7o{8(6Zl(x_BYY9EJm_vL0y7lDI3diNW7_*Ur zjQutHqkBoZBt%|$WYp`P+?L?75*_iDYO}!-3KLx8+lIp9r6qWEX~?oTT6s#4SR0|)v)Y;CmfJCKH?FBeslsrqdxKSS?(*ix`>F@9H zHiDd209Xau=lRwd@s+P`@a~uN)0Lx=TUr!BdumiD22sUGxGg*jle4BkMFkcxeh7>O z);n3**&d)iCn=Cc~FNLv5wuy2W<%(xl!0u zN8q~mszMr1`Hw#UagmQS4$b)T5LVIIn`D{A}&=gwJZ& z8iJU4Z$GSHMfwx;qE1c?C@B0@v?d)!?Sg#S8)M;DXt^oYd2iyFs^^h77avY?aKt2dB*9bMe?A_m&X* zV}O{&V7;^U(Uj|Wz3uZhCvB$=^81JCu(|y09XjpbTq?rOLF~j-4sB*ps;WO|xqkL3 z-uuGjl6a(daIL+Q!&XV#Kr=&nWczKSa)%xeW)Qutgnoev@1hnoZuBu$p|WcSuGyzk zptA@InRac}edrAtgqA>&u{T<(W;^ip_VtgUd7wlGU9+IzG58jH-kabN-UDnd3YxmN zojRnfOGwcNkSz{cx6$KrtO}%r;Urq!rS8|x{?|Y$C1I^^~$)Teg0Z@&PmsH(U+j>Yii%p%^yzDNck|I9@B`(Rzc8? z?07(U*S1Upk2D&h$IRCOQrCMxUrD}nn+l;cATXM2rnUL`<1WZ~;WcY_ch^n{JbFFD zY8Lev2$jatltFDUfN0teHZdZkGN9a50m(Irmc*2wglUT=XzNV%!$&dPQ+%T+z36PE zvu%{u!FQT_d(ang(a5SDAGj2-6+e(<@4(#qDylw0@P+}X?7&RJx`&I>+dGc?OSZ|< zrZFx|bw_kV7X%k`p#+X6Z8a=Pp)dA7neNu#&)c8452Y8ItW7RTQ@|@Qz2aKe z56s(*t%a_aK9qHg0;Wi+FssXms52-SzInNYAtq~mIMpIy@Q#KKe8zS+nz>Gnh(b$7 zo|wlzmMxr;88WQdwAXjZiC+rf*sjv-Wc?^WwhtxZ{~xSjy4dyq!Vm5 zqj+$!n(oMxgx0A5E=-N-FX#3|9H=+_K%)e~uzmU_uCAp>782s+Uo=?GN=NnMEA)Yp z=kDz-WA#KBgYnYf@lEcItW6CMKM$0MJkazI-R}p)4`>yHUOd;9GBX+6+zXPEq$Z!$ z%HXA(Ks=m0j~c$MaLQnR(uGy-K_&;|+pqI+Zx*q1x9rCM^q$%uBsaO4=``$Hkn*{0 zd;K!d=<6{LOG_}L!z^(bnA-K^Y#OlB{VOT+I&E)LDTt2@x%JHcIB*-dl5$Jzd*Kf& zx^H1)-J~rCO%T~E{VY>t8qpmbD{88mH?ktZ+Q2kmd`00TS=RGl&;FMiN6sz&cXiDN85m%x;c#wy#_15n$?0-BX?q9kVS8viz_}~>{vUVzLQsVRGTtI=h zeSnj57`RsEXzS6vV?~xlsohW2^_F?MDMSi*lU?VEX)03#ZPF`}lbF5dO2bj`gJ{}l z=4dLj*NHGgWbHX+U@--jQXxDJ9U=Nb>9@{iZ8W$$hr@~-wwkTR_)Xe&FxT!o_SowmH-;uS9e zZ^r5RT8`^rt7ZT*u^-Y>dXN%K;eBcUh#vDo#jG5Y9>2frWq zCtj@9bG{rG=VxNSGQOe+Jo!9azNW=Qq4RcVsOR3C8cY1T7>EZ0b=2SGI$$w*ho4&{ zSdMyJBJGUHh;wCd0A?8z!mY&biorQ*lk)Xsy~WjiH`y-z7|Lhu3B$+8)d+~uw-JcshEA0K)Eky)x_GF!xI~a-ocfGdm_BIxpB>%k4 z`q~_MxR^`*5x{g$UtsFvT_=vlIIcn)WoIxynBlODw2Nt(l3y49Ic^9uvq)Es%LZTG zGS3cTp@l#Ar(?g(td$nCQpuxm;`s|}JLmW8&_?hMGEKFlXkmx=Ule;KMMWg`$7%m} zs@d?^SVj}zaX7k4C8z<-L!I^PP0kCQQjjtUAiLm|tB}lihg<|T-6JGyat-$Bx03_e*q^U>z5VquIN##@PBw7p3y713K0YY`^l zLk~ZIe5G>sLm`=b)j3>PsF;Y(5@m+h%@Le2YuACNv(Y-WE-kpPl8Fp0<~VBE=#?}e zc7swSf0rCZ9;$a1@HC9i-Q462vIYN+@#-t&VYd$JH6L&!5nrm%gyLHlosmZrvNDyt zGwQxzGZoxE%Qw0b%k2MQ*E$_(d%!)NZ~TIxyQBimPptx$o-9|jZ}Om2d?cDj#CZJh zfLbD6(rs2wT1@ZdagO*;c|=s#QM$?b$hz2cCh^)%RMiHlwYT%k+X(DR;3={1H=n2{ z_Z8F`jPA5YDJqxX1|N5)-i9T>mpm=pvGCOUv3_wnD_UH*KZV!ZI{A$?tA84%kq4xA zBE;1gB``&)APRX*6uVG}{I1P(>@KYq(cGe~dA!nmMu^NDgivd*nu=J{XjQr^`h$BT z_ob93czH#*FH=IttZ4fBS>jXP)6qmU43Qg3^r=A)YBuWWmG7s**gLlok0oC7K8QLh zwmhWa#Fnrv*|k(+182-ivj5B+IH=&KW}Yos6dRTP$e^3$qJ=|Y;MybS*lq8&xah={ zNV?hds&w1Ut@fotV@bzhBekAO-8*Z6U^%o%ZqZ}s6fE6abFUZKGzFT4X)+WM9L6(x zwYLna+*pjP45Qax*_*&#`2A8kILzIuS!vwh*kBx5WjSz6swLwUpvobAvVZt60B|?+ zJTq*ulauQ1@cm%reuBejW=;W*&6*F?mpXdX{l{5`v!nP=RmM`{G5MYI_*1T=JPU!x z%6`zxD{|?6n#MPKru$EcsIct#QFUxX3pAdc#D7RxCRdi{Kk}*+YRIGslN~Q5(K> z-@`eGYz~1PD8DOg?uH2ys_`bNO3NQJlPvfx3*+3S9ofWP_LDWp?dR6C`9!M-2{Ei5 zvKlq6Uo)z3+N3v^Wf4he{4NmAkXUo9l1Uu>1QFHv-f`S`d5pSS zMVOd6Il0Nva)j?GgbsqFnqp#1r1klE1dI~njgju&5jAh`8iZpE@!Sn~GYC9KSd_n@ zPX#CzA$kC4lzru7$T?7tG(krxWIa0v?yjD7UkdbSgaQKJZ8FPk_0J2h#lMZHD{=X7 zlHJwEoIJF8AE|sdB7Am8(Xdn?hcECQ0PvzYtgjf$;yq4BD%nSzWROwTXr{WnDi&RR zdy(?vwM{EM5GPp!@$bahZi5=Fg(>Wcf9pur#x}#@p{>t<=!Exj&eC$-Y|9F_!}+OQ zzTw~6H)Xt-B|fjy+%yG8s%AZwcaop4t5qt!^vk$r^{5i8I{is}tR_MN8dlJhY}F5M ziwT*Z2;p*B&(wobnT>rYB5F?4(Xp~Is8!uOIi=opcLrOVlCV3eqR?G2Gd&yUhML#( zYixXG>XCLAdI(uroT)I}qxr5V0U0)~*ICS@U1Y*^aY*N?0(mXLYPQuZ{*g^Q$5x*; zMkSHZNB8EgtGiJejkhMJgMAnyDkhYl2%jqn>PHkNb_6-t?y-1#-sl%>{&dO7lTopC z;~I3hIbU8X9`5F~;?+{IPGOxZ;#KW^E6&lhl*8?y-7F`k1L&B%NcQAykoF1cD?}}6 zd4BKM>{yj7UG3Yx@Y8?t>ELAZQ%+m`fFfWKIqR#<-R$IlvHxxptDPU+eP0kB6m)m$ zDIo%mo{3{wEsP)RTJ*m!i7d&bi3zxE&oLFB1&C#-Ds8Bu*ibIS6MSOAL4N~>N0q41 zyoH7QEj)c@f!}KeGyNUJGeTTCl?*lNh{Uk<;W_Mwzy@d#SaS@8&`lFt*fG zR35?w7j%D`%&x|qs*!R_)uzTDuMfP*yOgp2NUl{vjn%I<=cg2RR-3U9W>HVn>7Eh%Lt@eoWZ(>cz@)deQGhi;a*vZ)zwXhh{x-fPB-EayL80+?U&Scg|dfO-DF2c zN0*ZkOZI#B?oGcsEcVdML`H2fCsO+RCH%;bHY}>E<-mkj;nod5pZ&%lr}72%jQQ%S zeAA@tmm7#P%G!#pz!UfE!S|k$8TiiOF@@{$P9K@llpcR~H!i5w{0nsZo=H4))qfAm zWPAqyzm`@?4r^B8p9bP+uiNM@H&Q$Mi&TbkIm!r^Dnuizhc zeO&(y)KpY_KPG2pY*)q`AoM$70Hmb4N|8KBnw1LlLUBa|$aJ0HIJU9l#hwOi0T~Bj zh?C?tF8_p2jZu@_2&Z8bd7ycnO2%y5MXVIgOa?brCO8nCG`_zd($!}5Zx_iXuta5_ z&M2K|oYgpFrvA%nh3BJ$JWV9=>s=Eqt)ay1mrdlVH^nCf;i6YOCqDx|4)UBAtdIa+ znF?zSZbWR8I9y^a>k5`7)5pYAOub84(Uc5nNi%fS&R%-Gjmt1Zw_42lclCVf%9P4q zCynF@dOA7Ov{YXyzB;A_rF&vG6TZ+1iY!Q42+?$qS6j;KuQ9Sk^r~am>a6}95t}GH zVe{kPR~eGK8dH=%o}{uL)W3Z&V5CniOE*=HBqn7vl*HaOpk|K%V&-XN+d-HY@8AES zx4hvADt6jm34ClvIvFneT|x9@GixhbtnlI`1?A=X{8sI6b58l!gN=|Jw2+DInp9K* zt!-_1poNalSDO|f7ZY+lfRVwZr1WXNZ~XiPHppk}Ue;#H*7LS(!bYAJ@?GCn8&LZ&vEb>pZzd?K8;Nuo@%*Vt#v(tmsNora!}qm zDUi_tmPVnekf}`cBFrrkxyxN z-al=^)^QRH@-7^d@#vz#9pmBj&-k}!LmeUC4KIb4OFOBCz&nJbw`eBVo&R|&`TT!l zK?JzAd|f`>e1wn-B4`9&xc~w2LtxU+n$I6geycHB(y9mvG&?d|2Ofm)h9kfO?xCW+L5YoLI z@}!I_j1Xh3LuPBsk_Efu^Mi4F%Wf)=pdLSHR^Zl>7cIrAQ6IM8y~z*C#(&=SG!7VQ z{*x=x=?@$$VI)w_hDCc@kiP@6?BpStm~MvEW-wk1iFpo*l45;-D(47Q=P6cpcEpk% zZxJGyj)X(aLpT(uvv}ZI7k5Y@oKVkG$V}MW`KW&x-kRg_|CG`5^EQy0C$5WKpVdJ8 zVi4v9@bbado~kAe^ey6U`AfAFXe$VYB)Kn^_6yhz@IgrLM>?9>XAOJ;_VUTA5@^Y) zI@1yL-Ua!+JDn_je!v2%?-6HHSnB-^fi*p3SNjDyJ%?Lk#Y8mv!Plus|58HdXt}-L z^l7E;pO0Y4=~OGX_mVnF#otAnBFo@b_Z$>?-iFF%gQ4&@1Gdzrzce;BrK)x_hMGBuOv?jm1JHzx!Ps86iOGp}Svi9{n`=B=C45b}jH|<9B6)y{oCeTLeOP&-7!_5`y*qbapsH8y zgPCy{8FBzXSqFJF5I>(8$Xl!L_jxkzZy87}oM?fupNcFwvr2F|L8qDt`{(>@S4tt7 zSmkC5!L>+$+jjK{DiK(r(vF_GI1ACAdXVAH3iSul^@m8V43HM7sQw;102k@QTg9GY z%6w_vl_%&}sWuy1$^15>9##hx!ojQkLnb3vE?zhf1Iu#WwYz z&>@WnvXII){(ni!$_c_J=|j&dVC81I?scU`cOKt9^lVV$SoMT-9XQSb7CvN0dP)%R zUinSM?AE@^UD5?u9M(gaN`Z-#6zE?bG55&uKcvy`U*)C#IiLY4Fg2G`k$EoPizG7S z9Q{iJFoq|_l|2jB-aWeHmrkqDD~;}jASdcGH2J#!ix8};}6s2N}zNLp(b z@A$>Z|5F;a_8S-7~0tE)KthTaJ#@}87U3>i3ke<*6OWzvPdHnGUK2i=FXNC93XBRzj z+L*q`dFab$OZ6r;Mzi}Keh5Zu;AAFHeoe*KinecZ@SdVF)5kiCAp?fa{Gn&;R$_wO zP23qlwP*f)srd)~ZyK6KnYJ(}B&f)*xOG(MBc=Fw7YGb6D_rjf$co#F$1@(GA_pL> zM^>09P-YB0fV@(?jvnXDn;MYGYB=l-Y0U3BW_pGzoGpu2_-!8i`Kwi*&|&qzR{g!K ztSp4Y$n5^(kZ=8%{vM3~%#sp;AV$&dU)TP}i&H$EAp0r7Y%S>%Aut!xN5sGw-j|*I zh**>PrD(B%yd>tTc%ohb`N7u(d2dZvOp5}$p=3SBI>=pF`KN*hb7O3xmAl==;;Rpy zkw3TCLP+xZNoMgqzDgcgmZ9A1V@8DwpBEITG5j=`NAEhO9#r`>b5O&xCNmX8yY@NO z-9Y=KPl*4sp6`3`zYC7PpM8ng`1ggR@_g?1;4X#q64B;LJv8%Hr~dpq-S*Nw9_H~r z_y)km=h^<>?*!~$6JS(R{rC+)vh}&1h>ssLL4Yo`RLgBt258t^B`F`0glS1zg}NdDMKCnpt812tvCHEgyhjMc0==|@xme=bswA45IU;*zOZbVN$=?eiV4m^UU{a2s$7I>fe8 z$TdCWk32C~0XUIJ@BYBo}tF5?;!<)6c9^*(L{~r z$N#5Z$T(GJ2&Ml&Z?i{1E_Sy6#PMhCZxn|NiBLC`yqhbrc1rI-<~VoyBhiStN%J`> zs`aA~+*yW*^8?xTIeIXg3tca1r7~eEhlL*E3nVNC!zBJ+sB0oGgD@%=pfJZ z+trZF+S-v=b-4alVJM!uc;nqe0X1WY5-2rqIO$Pi3Q8}Ni86zdL_US$HA%%!^h~@| zr`1pB%EE7k>K|EgL9M&VR#5lRQZ7?{{9;l3)w}(O>Q}me*@h$3l~_y=4Mz5H)1gXY zQsX=30n-%xar#Nxdyvrj&yPy=_p`rw75{Y)6$}n;9iLyOB}Hl3%?rPaw3jgAhJ|=4 zX%Zcr38zg22JE4dqUtnR{PJ62KJaco z9;w6#shne1ehrbnIXzcHB*4<=f0ez4x_Tmn{WbtVf}fkqNYE9a7X*aV6`C$yep5w;Yd3bW-i_w$G z*g(Z0{jq*CoUW}_T0)%%%=&-aI_ocW{iiBoIaJ+%>o;x;nSfT zP{?$_cm7ln*75M3Mzws{=WI`&I<>LWfi*>N;M$@hO*qx)cz9GOG=s9dY3qnoLdX9$ zMAH9*q5nqc5ET;w1A(4zd>*|pkObRbQ>@J2obpckrJwi={`XzO7pENgKu|z@G{x7q zS+U5dE%@c3Mhh#ub5tLn@6$15(Y`A6Qj635An=WDe7Wh8h*wf3{PG0a2l(SVK{x*K zdEuXuzJIj+`yMKGN-FZzQgw3JCmk%fB6Qi;gvao;if~rE5eiaM`&9*tAYSQ))$WCw znwsPDkVxx_sK@?(eguqVo{Ck#SM2oX8Fi8GlZf_TL{un71=vrThK7d5du8?nBBapU*Yk*ZX>3hRA~Lj7J;xrb-3EAnXLu zdM;k^`rJX8%)Bf8HZB+HcyBei42uhEj`1XLlhw}(p7jlz)l;HDeg2xddrRc+=j^;t zN;6k@EROaNSL=L53WC`wMtVam0pq>@xH=ZN(vS%7x2x_ZuwQiZ;t{mKt1D`ni9=_w z)VUV*u!MoOwQhYtdWq!yUW@u!$ES~a~ z4}(t5zYt@xd8}-(;Jj^>s!}#j;SQps3j^);L<-r$!oqlccDBnaU<4pY3A8M%!FmYt z$x>jbsF*B4O@rh7=o{m&cKu|nZEI#mMnz$>TE$~`QS!=ncS%q@86s0YojP6@^T$1= zONuhCD;jwZ?&tMS*U|1ufsZukVW#z;RSsY8hI{{Dkyr2a|6M@ z``x$$E^iq!b_OJ4_X&%Q&7CQ%RHBmKW~|gI)XV*Q6f>#=eH!N8!c4zl3??_Xlc!xY z-7KQ1rBeX3lNZk^dD%O~W!)DxiJ?6_c|9Nl8V|h8o#H+|5HvYEyU2yE=(cb(r!ACz6(^XBw{AU^?Mx_uKm#JaE-lS+gnV%#xGv+5OXWr^Me8w)yiTly6cIhW{@7 zz{zfxuFzGvZigX<{+Oo+B}c!Oo{NhbgCWsysF60wvLS~cz#4+~?NBE#23!JF z8v{0Znx4}yV53js;@+zkTs7}_E=E`ixCF-Gw>=FLSKG(>6#w`Hg<=}e9=!keP2)*F zzwqT3w5HM*-ZN$$EYW1L_fN9G&i$)$xbRjGmHGQ8M+)Y3tKD%(gT}3Ng_8$XWd2j- zQjml7^xZLSj$nxscDB$<99`(E2RRdDDG+c5!l{zG@Af0MV3XR=Qx(R|FlFwqXun&g zD3JUr4YMTXFh~YRGyAhO57POBe28o{dtbpOilBTa9Li2dJH(jCsR9ZiUV|1Tm=~jO zP>DDUN6Jgwv@LaDws5b-nJ7bG-vnnMEt zOZ}{5oh%naVyv^3MAFM|5k{|>)7Rdd4d^JqZxUKhC&#)BkbCEzs6=h&|_=Elih~3jx;?=|;zR=j~Y$ z-w~E~7L(1Ml;N`bP#x`X1YxMF1&|sF=C^*;Qxetzu0Kl_hupk<2r9VZcJ?43>jNH!$9k z+0g#E;@_ED;h7_&bUu3K)pv(Gs&XydlB&5WS0xQw^rxeCsPA|tZg9(WF@j^i@=Jr>{)TPzEc#<7j?P*ys?V+|Hsq4 za{PsZy&7tg31ar6Ptr{rB}M*scRy8+u<9%S8*}yTwp#kpgybt$rRzx2_2rv+8 zLgZZ${&!mWpNLcfzS*bM+}5@U=Pd zf^G)gDFXYKZCMoJVN;oXC|R|~_>Ra8x^tw&lK|RvetLBo=?*T7+g^j0Fc=58k$Fh# zO=bD;JPdo8;Q9MuSAx(22&x|}m`-+|W}y{**zx2hBqo>G*%kZr92NESubl3J`rPTm zT%@GLP@}V~3NB`gpEUGgux%~{hyTB@Lt;s!Ecd0hSP)=#X+YAa#-PoVBfrjF9y z0kEaui+Xn13$9+&q`58aqB#U+eEu*F{2!LPZw&4Bk6UK?qTa}G3DuD}iS{UC2-M_p zIE5sxVHvQ!<%)Q;0qh~^e{EF1H|ce3@dLiEr^jrnE0dL-oex~}{8xv!-~R`cIeS?8 z#f4n9!tzUn92Y2NI%ViJp|Mmb6}m6p14iS#+W(*t55CI`a2hCqxF;E%r3p_;5&4*5 z4$;ZEBc86I-OTmocqpvQpAF9f-De~&yIm)H{zs9$syRwf@oGTCcu_1ZNw%!B`W?#o z96aKsRekpvgO`T4n$GH9{$;=|wxj+5S=Ak2B-hJ1jjA|Bv{(=TJ*dlJv~ZTW>X80j zJ@?0ZkqIXkQXqhs4C*!ZgpiPsirt+p9AK!9kM{|h{Nb-2eIIgc$HU<2bLc*RtPl{_ zyLL1~1D^3+kdvkneQ2jQ44>wJG{UEhn{`u0d}JZ*cGIh!xp4xbvO?gXUUh_yms=z6 zAP>Q>L(^}7kqg7u2$=N4t&a@quJe64Ueav(ZQJPL7nr0$GH9tWMHx?f@T?sx(Mard zwe;3B$-f_XQ4H-r9{Amq++$BPhfiNausJro;tBkr^5agnfLj*Q{8SA~Oy)*$$%GB3 z_1BAX$C!cMi6=a2MG(-o;oE0-@puO^;s_|kaS>#7xqqud{>OlmodyFAG7hY)tj{qL z?khQniq-8W*Dj%aOZ%<(krp!t!3*{THguuJty}(Zz`Gl{$U+npI|>?DD!gwN1-Q+c z%G{7pdwIW)a5>D`(*C)07894j0b<+TvQ}_5To*e=SA8I)KHu!+ZsnQ?Gt|Ma?`xrd zG5tp0c|c`eowxbTpJ;F0VtJmxp9@fD?vesn9|UO8a+8K0Vr8vqm}6ldG&oU80#aT0 zgR&0&>q0P4>gmMdLuAwErledf6u&@W3FUZ1IH>gq>DLSxbqB}0@xv|n*^_RF`J(24 zk#|~pLq6I ztN#TsLZ~n=#O=i^!RG~()UyGjK~EAddd`d6T(xj;^A2>~4lG6m~a81~a4F<3rivSJvl+ zH|kFSRR71-VR`3YfE`!IES543@!OeUq?>&;-?y!z1KeT`kk(CI`)u6j*3W^I4t)*U zOpDDyWu$;7J<*YV8E%&E;8EDX30fq7=` zI7q6E_Ciy;@kq2qN6K}3=Y`tc8W`AhPwi423=P6$k<0d=KW5FuJ!Tkpvx1xE1Q$L9 z@4QT`D&UJ6MfC%VHzXyleKHw)--YU^zZzDgmk_urL;mh5-H3LJt%|Qfe#T!pfv&xI zf59YnIFr9$)%$BN|7qt@f+5rUjqnG`uUtZ{JekgV3hzL+@zm9TR3X^k5Fn?cCMSkU zjfo)*DYfu#X^Zb%-5i<_hbcG}Ie4#y>a`@KSm$5wIe)-iq;i-8!{>cc=fCL)*-bY$ zH>hCP=;^~2SsDj9D=kx{D=10qgR()SblN|+TE6)p$xT{QQxhEg=Lfj6G>K6u@ol&9 z)Y6$2a#`vW$Ba>=+<^@p5enHgT&$V`iLdfH&5slnDc?nb%z;ToKCQq3R6>bDv?hNW z&t@X4&z;>bs_72&#=gEIg+s2h4-1wYtNu*A?*1R{T&?RkFH-8UJT&=WiwxBIh3dl~ z8&65DbiV^YzV{8lE`@=CtPgbCilt^Y-QhB~WmB=GPnVA#*w4g3(xDw-^a=Jo9`tI$ zyI|GtrqKLP=^&dZD3CQbX&pOn{N(rg$6M9Ln3-K z+y5`~hhh53=UuF}+CMK}X))mup}JdfK>(7i6^70Jo$QipIX<)bs@50pxtvBNYMg&M znVQJQN?7uMtogUAuI?k=`5%v8DV>B|@9+tfyV?<=RWi3|e`@X?<*-gRCT>pcdJzDZ zkj8}y!+3i{xDYDc5{)C(9%Xc%zpeum2w>tQSv1?u0{Mkv_)T2(CFb3{M(>}L!(=<* z|E>)Zx4FQ`&aMy7PK`~=$;%V=;FrUbUMKUZLgZa7I-j2Qa@MQhXi@CB8{Im4XL8@a zrO6*h%1`Z2ZbHlt^sMNJ6Rm2C$~aO2UGdG-&mnM?qz2lRSXHr>$IQCTpD;6*-6Dqd zw{?J*zTG)J=aKf870Z=kuXcYSn=_1#i!P{P%bP|2t#AXbMxFJ2;LxFeR2$CHmHf#&;0!P zb9n`Tkm$iuYaW&p9UefZP89o;IS4J{FTl0KVn6qjFIY8!jx*3+c50l|80P2J1V!d1 z;|KBkt((@kR!_oHqJ#T>1C?o$%9+!_F^LE2nvP0mm;eZq*=!3B|Bp-I9cTU@AD~c5 z;VKMyM-9IqfD2Z0{r%BfF`uxUkoXmH+*S(!bmKINcr&XzPv{^F`7=U(U zAr7&70pZq%oEZ9zH1qsg{R_v89sZjNn zd%aW)m_&zH00nJcLl8BakdRPeO<|$Z+OTUf1)ZoFY==4fy$1UK#5a=Q2r8g_9{KX+ zada&dU95-7JNK4?LqtRu*2QEGt$^yoX$s(+99$s}FuF2m-3|M2n4eYC@LYTr+tNwR z7%S@faQ8!)zkvMUe9IuMhMQlpEJO|u-)2&2%}vICvg1sB7(_xSUvHLCsoep3PnB=o z#||N!qm=jp;f@qa*0aa{3~0~ZDevO=b0O-?)esU65eFY2`kOovuO-=Qxp)`fm6!a@yVk5{Tb%$(R_4+4_)h@wLk748C zQSZ}tO}rYl-r`-L_f`!`PynE#Qp)1FO`Zb`NsV^>_4PUo^uR!LznS&7@FOH_w?;1{ zr@GaL^Ip4r-Ta7N7{N(S`L)G8>Bm3GmAeEQA50FUod#7Gmi~8wlwUP|c>I0$&vtMN z9=kcpoOEWqBBZ`_OWz+!SNlXPO8#-DzK{G2>;GW@6PQeDUhbtwSC+d*S2YhdQNU0H zGk|8idQ7BFX}1+1Bmjl-1Ns+zy)U;L?~xSbH&(D-16wj) z?o)TuAum~Z9ZkVMO-vU@;+EepJDp!M{|I$`yE8uT8AP(fD`j4TvD!+CpKimQ`-+>w z+P8gnibDV@GSvTPI0T^X9btbLl@z35!}*RF{eNIN+o+c>Q=qD>rG?=u+-YJ?*3j2a zgaCi%a+4AMi&M7gIP;XeaTg)-jd>i;=k#aij?nxla_H<;D!L6X_BI09@+6ya_e$oq zskq4G3=q6!6w0S9B&lnXb-5E6C24yQHO#{P?)`>5{pxjp5%P6BxKIh792dka)ac||CE#V?K; z_rWKS6!{$&Q^5U%e^_b@TdFLj^XglxyF-U;$Keb5TUenJp zsd;&<)`O+&S}^#wppni)0l|8$_&LZ)QBR}8!_!+hYxUrV+{ka6ST=8Xer9ufLV`1> z;U1JDec*og#LK3XWE(&3(h>YMphEQ7V|G;4@Xm>F$GE0TEG;JZyJd4$$hN>X3||F|Zv>G}Wh0Sfh% z_sW_->B{{BC_CX>X8T}s8OUMEQ2Jm_-Bvyh5vzwbi@G%2B=N*wY%PtQBfbK|I^4| z0U#$g+0HyU>ZvP?8ZjHK@fs+#*3B=>$hc|QSBTGyHn6zIo#IoxCxtckF%8k8Cio-r z?waa%G?RA>hfiw>THO0$W6GCNasUG`x11~8mINrWM71!TALI2k$ONU?P}aSg3{lz$ zVxhx=M|AVHb(~Y)fdf@2%FDRUFGUkS4{Dcxu8i1wWK*s0&JMeIf`H`i;zE9t!m7XI z$*9+)hqo?WY)-q9^7!!n$H%B}I1lSBC%1u}0r3LmwY<6RUe-%mlE;s0+*Cbr|CQJ1 z8ao5S{ZXda&r4e*iCm`fPo5n41?@9#_U*CIf8LfTvStwG(f)_8^NoL(>EQbMGOA8- zTtk&77vQ2nO<%$%N}Hug#F4e_-HD zxU%@7hjm|}KBVArqHaZ>O>L$~3;E((f1+K46T!1_$0}TxL%hm*u}%TfR)%2nXQ-*S zrq|{Mv&+i`L(X2i^mGbHm@q@D1A@LtD8{^hf5UK(91<3mrj#gr@EWb1^H_!W?l;U&px7(-8`I^nR z4u~-MqMU+XH{X9b#BN4Ug6d-n|F-0PrZ2ktmQuBADZ`g*r~~~T6LR+j&f~9z5e5Hv zK3_i?|8lP$68(l|+jx4O%r)H$CU<|t8l zUs_4Ux1czkXe0 zB2l5o_OS&HV#4G?7qUN(zv`aT^lxO#9>g@a?>f6z<*jcUmdQY3f_2(xWIIO|cHMCM z?0s%It@P`EG5O_7`{gtFT}|FSzqC>7QmpzdDvv(C>7w7e<3o}AQ<|#W(p0jav(4BZ zM&+ny-ww)hesR==j91dsJd-NviUx?{m7;7ssnU6iEhA3OMeW`AZ%djY8|(K^@aU`Q zOY}Di08h$ue?%6KR;zvph~vdMW>C>R=&(aYYX^7rw7q&x&UX+wJihI|`$+B=_m^1p zbH5$d++`MX{2BbSH9i}Vu$Vv+lD?O%rWmaWJHgYihEEVeqvRYgvtUg5X_yuI3G2PF zG!6rc)F8U{LsV+jaMpub2z?6~EMlk+0r`mg3_IepxoJYKD;!GUAoaWMQVjo7J%ab~ zKY~JM%13Cz3iyGWEA-G$&wuyAm-G{x>rAWqt)T~~y6mpY>_XZZG8NK}NEZCt^)=xA zFvLcGTz!%T{*^n@C|{;}@3oI%W@Tm$jkj$cqb{ajj~lwC_RK~8HS$k1y3=8DApB89 zIz)Oc=f&^05ZT?0jg9Maq(h#;(&;w<=AH&=)W;D>St&;sn4bt0uvQrAx=*mdMJXW_%=|OuEIeYzThvMSdMh1q&((;HGptJ3GAzpeGKV ziH`CxY2gXPJaVwH9owU3rFg^k0<9Euwo>BuG^c(=22c;1$G+JnqfkDe#DbLU$w>XM zFg6RBhXbf7`8{s9v#a{)Hy%SEUTCSGhU`F8xu`+4bKaeP3Rl=LX5WfvZIHJdlmcsi z+O9&Mk9Ys9#i3B;o#UEZUs+HGNVZ1LtBUl>>Dj)tlW4zdsAekl0kcXu!E9jLt<(lay5ffx*m z&GYD5<3y=xB1dvmj0H>%^Ts#f|* ztE{_?l433ftbH|A&+kI~l-M7wo0=#zzPDL^zbDOaFHZdS9$xyIM*YV=vW!O-?^eGe zw=(I5J9$h?BFR&Uv5p}vEdu8QmM+~tAYl4iAiWkGf5Ym&89YqZNC)Op(+q)>h1Slv zxPN<>^V@%)l37el%vrIO4k_-~ZThEG?gR^H5WBi}SCQb3Nhn2Ae*Lnm-9(_v?TzK@ zau7T`es`Da3Of_xHNRFwP@TF2eqr`(PJQg9>|VVqpw1<*8l5a zeTSqC2|#k65Esa8@3O^zEIo!itJeEgm0WP6$ZM1@#`Mh{Uh4r~!b~4_z!Ir!if{jt za;qM(q$ewX*FWb{TU7t;@ZTru+W^io8|3;&|9faEAQ^iJX_ZbM+>T z)IQYLpItos3|~$l_wsS4_*M(n&axr)p1m`<@5d!M9s7zq8)xSKW%E>n-%Iu#;1yX?+C>OcbfG;n8W9}{>uVu-CuMI zT54Yow$>~AX5BB|*iC*MXb;KLSNHBpSKAO8zGiG2py5#7RI#_Wx4QwikPox0Ej~Ow z;q45$qx;re**Ib8uqz5>L$#P{iYtwUJ%IwLQY+I2|2XhaxEwo=k`;|t&BNQHRCRb3 zR}b8*#w*sLA=^LM2jx3w|LvBg(K=1O{;1<*jI%zrG+tv;e5q39!2x%f_u1isNzLyC z9wvkwBS)Ygpu2(zhE~pDkb1=%fRtu3(suyI=OP1%jRImD^t807{NSiU%EcUG(kqKWm4xnn%PYwL13jV-h_pF#b`WBdRc9^N;4>!+8PfvmUAu*ohLxZL zGkoDq=YGAnVzZtT4_}Y*-g(w?nUmz>=I40VI@fKUvpW}N1r<5KH>iu;YJQ)1vd+21 zg4fI<`!?anbxL@`qyoUr2gzX)ZW`h)iR1RVef7@f48+lhL%##lk_=-Ps9=*7%7A|W z-l0mR%iQ)f4>va=^}IuO6!ps(DhUwOAbWokrq8OJJ9lpHgcjnGuKmS*K;!^QkC`mY z{e#`Gg*@(Bu>x}tD{P25@}KurGx<%U_6)}FyO{j_>%rgOqkPYIRX9(qU<|Fhu--V? zc8OY^FZOn)9LuYghCZ}E-dt}Ou1O9~fBWtQ$ycVaMLCK!#i~;_kB%N~_L?xO-NZ8` z3aNj|Ai4&ZSeW)*>HjP+pU%h6Avh^gnl$k09B?bb-^7#0b@HjvUM>JD4~Iw*__+|~ zti1eeT5L#!sZ)e$3^osRL~b|vo^On%|1f7K_=B)ic?tebJsfK9>DcuvG=RtZ_sV421*y_ry1Ew*_(vQLde2UVA3d4K;V@l% z6$$0buX{y}cT8P+0I&bJj&nH@|G`cw-^tf?HK^Son98aJQoPz6StBJCI$@&)yXp1i z1NgY;)y*v9u=kM>G}O>mVc6LPV^l6RtwoV4~o*WlB*=H7ACJY1Y?j@Eb zRLNI0QkOr9P}u4`ynQe>v&JF&`m^f_B)lP4Fsj`mGA#T(r}wJReHxH)Aw(Xew&YDo zTH`bohD1$Bo6h3m;&p|?dZWk&>K|LoNlUwtnwt7|D#2?2F_vjt0!*F$314`Rt~Kvw zYG`Q4_1WElKo@0ZUg0#A2CZ$D-C2q?72DgM2pR_kH^|Kn--CV%JNAr*CzA^HCa^q` zLD5v_9TcM{Tq?)y1TBb1C@Imk-o*Pj9jtj_R8~e;x@$WX5)_>us(cW{SLqS_o_)ym zPorbD@%i6|^zwfVX^$K$FoyA?6arCSRKn@zs0sv>saOXgQMiC|zWq9ux4`kVpF0W4 zH(m?Q#!4jC^Vaf2JKEkJ?-z-%TVd;1K$%=lNr~d#jjx*~#hSaBFYCXln3iw_#L8Tg`v;Tz;+ag z!0gzaf=Z{;o%|B$Y$4*33^L)P>EUZ0$Z^x>hz4*6Y#&TMwAg0hFActr&^Zghsh}u zLwu<&F|pjJ9V8&`M)5iES1a4ly`Im7s_ruyZ`vX?NCAAJDWQbKBKxp^T=3pWJFao$ z$PqvuOMkW}V~2+H4863qV+}m!_!N`H^%F&0_f`y^r3o+1_@V~UI|o#_uU)%#T>*g) z^b#T5nF7XCXLBT*mor_cjBU+E7E;=4Xly)$bciuti&cmN@D{1j0tItxT4Mbx36f`7 z50o51c2dXC4F&1Uh5$2HB{yQozOPJ?RF;wN=G2UyNm0nudqe%8cwE?dIzo>l`t#5D zp!V1 zxGi)hZhOTTqgt`8;9xRCFB6kYq$8*) zoSBWCo&D*NM>C=C?!FwhK|^mY7U~rT_C}#}@RCwo$bqG_9;xa_qYVrS{f|*k185mB z+$IH4Z#4AUnuDey!fL{{&AC_t+hubaUCWBW(?ACXD&;L9loCImumrK8FO}s?nA?C_ z0`=@!{D+Spo2_bu=;`es20qWz0uH`T(NxMbcEDe^hhymEKZz+l(?6@;|4B;fFas~A zTY7)`>$?l%KLk(H+p5dSMM%G}Vhd{MVIVFHL}p3_Iz%SwGhxvd!vlqnNOUm|+Si#L z9%|)e&Q`GOOG3N5zo{}{5fak#J5;8ct<+U4d&}cX;6|9uZ(Xl#59T};a)zu}3voJwh zSOf=<%ZD}Hw+fpTanbAPOm}Pkd^Hl{SL-qy{;Vpkc1iD?vYyU0vTB3U@Gda0>Q zj7$OWM3kV!7}ja_3s+$05pVSt_x7QvVQk~FMua|-e2(@t*>BULf~s1BA#S-jjbr%! zVeMui!;G#f2Y=~1ksa6MXg6_DO+)L}Ib0Zx2WzGn!M@aHhv@fdQfJ~Xq*4%1-M4v6 zjAw{EqcyqG7R36AH4^i_;MW%w=gB^u#ktGz8M{$wT$LIVtNLoHF0rCEp=l@I_t>kP zzxAR2G3{JlIqR@+HIMwpIYw5ttLTTpiq;hkIo6j$mFdc4w3&}NO*y4LcWBm_vTL#^ z`$R{Z_Jht|Zu0Cf!`ot|Ggp@HB*`tej9)E?z}u)g-HSO{*}%##lqMV1c}GbmlRI>r zZ9?x=^U>zFX;(8%x)On3zhMqSTWgKXxe4c2t^ZJG0RyK}v3s-j$8a+X|;rm2pNTH?c6ro{Ucq zE58ri%5m^eF_p)Uwitz)3hf~hI@b{51G?)%Mx*GvMjb1uE7~~=kri5H3r%Vi*k6S;$)ED3q zA!)oUEI&rxziBwiO{^f~5Qb9*~X9hLEP z2EYaR$8=sRo$;&tKB9c5)zsDb?teONJzRlDN5{uw)KqY<>b3t_^o*cR}Ur~5prB~ZM4*bIneXV`|*O=_QSssCxn z%#?yJ{$}go<>4>e8-XjMyVchx+NseV1)6N}HH3nl&Hna)=KI{NjN^=nV!F`7JkA+7 zg|4MBcre<~pagS)2VpjuxCwylcApCMjaOz2NAu3{xc3Z%zWX&{;ryVT3V(-F_OeVV zzQUOjAIq*18_Nj6P49F;*^8F(B~O-UUJDHo7}QNW-JKozvsEMo>p4y`I%BXvNuu*{ zlQ2POG4#$NKi@U(f~L3O;qA#c@2;3~tv#Qbk?~7-OV}|>1!Qt&*|7Q|is{n0zWHHo z_n)!2&6VyIi{-N+q++XgltCft@)>$t$}7bTZUAen4(uZ?h7LOaxZUbk_OYRVC2?V; zFVx#9=!Vox&4)Logf-B&U&)rR?#os8_nv#uD%TA*qh={bHOO66tq*s%&Gi+dcBecypCBC9Bqs zEzVgLY4VvK3jxbd5gNk;!cI|7k$73PvqFo{7JTdHE?K$B)3kPl)-5kAHj3lD#uwNV zGkO!#2#`?Ver8tV2RMm%CUzn|i)L%{yBL0=_*Nip2S;ShH~>eD;?(A(VI}m&*@R$q=S&H;*~K3{t83pI-6PzFx7@##m8iCVR&ox z*mjE)4~dIkj_y{sPKxk861N_nDmP0tm@=zaP-@Mt;zs?P&_-W6z5esqlBYSn3GcZc zz&7M;V3MNJ}b^O&j|=$Q;_hs%^Ttf6?b#PmR=P~D|-CgV)+9uw5PhAOe^%& z-1J;+T8Y{raUqPKW!rR5VNl+c+s+}@+5y~BhTfTZ3oIo~kB-mw z*zFLyVS}08#O}#X+f4%zuT7TBjGOq%;A)RzLXe?V*EIgY4<{|dfPSkUu{Ecjv z?^Iq{94xgAce|=gPjoDMVO?|4)r8<3V(OHt5B|XA-jyKo% ziP|NA4k7XCxTjs9)f#h63ebp-cYZs&p*lE=NTA>F!H`2b8nOeQY}R&l1L{?`7LZnA z-6Z zJDz^gJU%;rk)JPBN%oF27@MIBMKAxP?fLX7$azX_E*T$LR3yh|^<{r+QnilEiy!y_ z4uQ2LeqOt;GtHl_n5)=Cx1=AUyOWCbAsQzNYko;_BY738od$maXse>l7e?frOIur= zCt(VJWoO#uFop;0$$0!g$*-3`y_YYW^b{#2ns8V@B&O^vevym(5#9GC3y$H@BgN~@ zA|v7>DP)6a4ctP9!1V%=TLNY~0d{D2-rYmVlfx1=nnvtf&$_l9*nM1zwZzKAQpCk< zLZgW8;Xl2$jP(U0bS*_5#LDYN)8P%7sx!MQi)a$WAXeGoAETYqc9!FK&`L6%)E{2p zBe7W1HrHZfx69rzl}JtSp{&KEpYI;eHL6@IuNE#}O-M?v_+H_j4AG2%n>!9MJ1mCVlQ2|%RWQ3bkgzx{_&NkI zoE@C}LhWm*2W$6R^y{xJ%S|0}T-jWc=zP9Ad^J?$^`oq1j?**#Q?{0aMaB-AJ)SkM zjb)->M8mV4)iV8kgbkD3gcXH{B3hkDRtV3nBPR(|?&0CX4242oS zeo>t&@d6Lbrh8loun8l>)z|7jI6Bg9;<4i0FaJ8`v~>RwR-CabPdm5s`7yo?oWxsp z^>$?s-7T%;LT5wb!RYSvFy2nEIOWb;UfPGMBe3%|vsU&4u8is^(h&U7Bzk7{ zG~YBH6FuwV@!D9=@z4^hQ(BJETE>1!brIDcJ&&p?7RvsNmoDD<7q(!w;r+|c*?kV@ zS5wqsl1f1>gv*SwZf@hv?VQmJv8~e1 zlgG8kmcqj?Tjbk5?an^tv^Fg7)e@euI(M1Z*&?sPIS52mdPl0GOPr>zojI)mO!qZH ztL-Xfxlh?(x@@@CWc6AesW@)175Id072{F-&qK&HZEW6(Z~RCQU;kQCWssZmDc621 z%Dg@xkk{^Ryl>R}lGM%%bdc(0K#~W{%)cN7WRMM{7@yW1 z)lnaNUy2VRH&DL#c*q8eE$ALA$Q{Hu+|$kKZ5`&sHnMp* zJ>49_s}@w=Y?)bPUA|MZ>f!FIxENj)jo?CltR7ScsxE~88kHp*LzL9Xe zJsgdzSq@}J%n<`irs3lXGI_20Z1D*u9Ek&NV0040*OoP_zY+YVJ>}ED{T39rGw#}^ zuT~ins=2j;r@cKyGFRkPZpYk$EKm6L&tQ&3mN14$Y7xS8nRw5sT^bC9*Qx2*tz(RC z@Dg#cp7Bq}S%s8WKJQd6ycDdN_vVm(I*^kABw+ zq}sWkc;S{(>{V&P1-!xBR!JD%RMo)!)vA}1^hWYReTzIR{4R-J0fS0(fm+(*geq?~ z@3LFjt|qc!VHJ&e6FIBT6<%ZVQ*JPqnoR%bB^xa*+rg0($@rW77^^0k9;7;x27#lT zT!`kKe8s5a;mt?coW7~NpukFCn;ps+re;U1su#zc-M=@y;ij4KmaKy)HE!2W`G*Gd=fDPQvAo)B z-W@K8fmC23NR|Q}D1O9Zm{DgnN97!uT4yGA;z(qs`XGygqfM zup=QGjrn>n7t6uTp%JREP*_qT2;mg$Hw>fvi>dDoYGTWub%?VMq-a(UR5o-4asn@t z;{Ke)F%gUPUknJ&p!!NZo?pLX&^B4cBvVEyTbJiS4Rd^J+L2Sj0%qFIZ=G@#j6YUQ ziAX>TwQ;ucLgDmDHS=As|&TBSM7v{-Itmu@ikr<^Hs%CIB%V@QjFt-*;_q2$jW|Tl;w^_7esj_ zWxg0weOP|0K9Je+YhACL0-5^5a?Op6@qx;5ss!SwV1T0@}=|bFYy|RY3fwTBPs_kP)lhY^pjbLx)U2)94SbX!G%#xF- za@0^{5{;w2dBd(vZ*QiImq~hmN{{Yfo>c7t|6751RXw@I9>vZNhuz-l)4gRA)7dU2 z6g`Q-8{ZeHt-0|&zsA_|UBk&y*H$X?$bzWl%$Hmk+kqLvbV#yi-q5mU<9NBt3AyTm zg71Se%@$=jMMIh2F9dQVed^1lrS_UQDXBhr(j5OJ!p=WtTB02)s&yvP>8eQG8!+kK z0^NzV0Vr%ivrEl>c(d!TcZR(;s7B!_%bkQ4CkoMbb@CUSiM|wZX@NB~A zRId}?ptUHkc9D-CAb`6vj%=^54f=)}bD zktYh}JWzg3pR?+9H(E)73`}FVQw#eu?>P{^H_|Ep^}1#$l5%Ch@(^IE({mysl^Q&z z`5M}>@?ob>7~xwuM+>I9E76u)gQFu^3*Kv|T4Re1!;k z%->}P2ZwTOjgaS@B|RWwL+G840-IAp9$nb!%7!19d>!eoe3ZQ zma)a@LlS2*9wb*642XjGy2`L~-^sa=ME0qr2ftbjDb(AO7Wr18wTUp9j&YjSNM&fc z`_g;0I28dSSb2MD7YvCO|;kodx~V&D_qo~^bCg+PEKUo4;bkkVw;DcZb;l?am-eUt2DdTR+%4Fi{xE zjGG|lEoka(-4Wn!LIcUpjW5HeCDDUG@~jBKRNj)D}$JuEik)}t`3d} zGeHE#0b>j21j^Vhjm)oYPBL$rH5A7qSzTDh_A$nRjrk+w@mgLIU&5>nz4Z)2(Qm9T zuS^;)Ud9Ipy}2$eyG}O1v@-_EiLh+@U6Z9+r~!)eBa*~ObhdBGX=f*$j`Vj z?mp8O@AQ!9Cffza8#gn2ipI%jZZ!4r=acL^8>DX&qH7P4`teRb6Q>cY485AlM}MeD z`HWfpwjXz#W!s6T$`f;a)3pl`Ysl?ID(gtBW#gD48#R5w}m)E&|1y|3enlgDhF zy(AmkxU}+7xp%57w#CmI8(*nRT$syWUwwsZTi#g~iSSANfL5wZ_w!z#P<5~JVXJj% z7x$SUJov%MCbC?d0k+k%4K^sjGviZQSCn3btM z&8-4jkDmGZuJO!nCQ;HDaVHS|ID5tk6$cu)PjUGBYZC{E?Ym@s5iGWrjW6;^6Ma5> zcw&LzOG(4>w8A&>Uw(%AT*w7~H2sWO3se4u1(MP1q)ioOB~k4;yrDbS zaBJX6OfV$ENEu*7f^&Y=E3ox1CS*@J(@8bUCk&KrmIP-F)C!1OzM`g)#-v!tFE<81+_a0c-Nf1MB1L2Hj1zeiJ{7y z2V#{rAAb3I4 z6M+;2l7KCx7|RV*HTI>wN%CoFX+5RZb3R+ZsnBx=aoFpAq~?t;@+@z207(jf2E$X# zN@PeSt|MXdW1xj&l$4^+0}`kUlZBC%J?rpor0-+HR;6`yb={R&l9iJKR=ygM-1Z{W zLg*7XRd4YO`5Z!psD4|W_H$%(f3422Ci$m5*NONTdI{XdZ&x|X_Uwc6<3vwSnV^Oo zO-$Bgl|F@r3fGSTnG&JX7E5s{iqvPC+I(A>!qCtmFlVm;>)1X}j_ zOp4RrF#lqPQBn&da@MngvkqjptYbImgHTL3_e1&W(k9( z>v(o*W1mTMM(r+ZEwRSI?1`rG_A&=<>ZE!>L8n z>>S#i4?pW>RrSCOQ?csWE))vF-g`NFNC;{c=*v$K9-83d36Z-PH?g>@(u(WYzorFh z!m;x!g;!|}e0Fp`T5&B*jE=q;rAaaBwh$KwgP#%Wb779Hl1*&$7k#D0TyguC7n{>9 zV|e1K3%9mKZGwm>lLqCkd5Q3f{4czx^TS*=!(I1sVW+2g#lt-yGFP=OZn&{Bb)!Jf zjr;33?}v_5*#X?lHL(>H*M2L!L?7pqKwW6+Gv>3_$%C=0yCaxpxiUkM^mi#IWQ|yb z?V8X6_vP}`0{e)*A3N+YHnCxO*TXA%*WKNosk7~t;L%F+C$h5*XfWk2ulP!pCr=kJ z8(zS4{AyXqWx(pij|gf!2nY~ai>q6NNsD-ra=Vy z$E7EQ3pM+PJ({8wiOMF-bbE~hMJIHVAfMxy#GEu)@Xl|rXK<|{C{yS4*Vj;^;347j zeCSZU3^Q9?p`7fwG1y;xO9wZtQN7HYE#$bml~te>p5(LhLU=F`9uU|Mvhv-D@bwdqBI@oN6&2~6W+j6jW z+a%D|eX`>w)K>_UVVj8yWGbO_Z^9U`XqbIaO=!C;3klSurQytZl|e}`TeWL1BTg4m z8&+XP2)ib2KsL&98Kn&TbW&~WV)ks6|=SGQ+%b>CJuSFo9j_zXY}&7 z9({ze1e=Tz9u?%r>`czZ%`X6RzJyi(?(YM z^K=0O9?qq2*ADra;54nv21kW^hu#$GU?Of0uin;4w635~R9mQ2^Bk`~--Z#){&|SD zk5EnI4sXb*tQi?aw+V)2f7g=8uw0l<^AE z@L-+k%dk+}^FHHUv%o_`4q)_0Hr|r?wXRgD2=C2qT~AOg8(}Wnny!V3n0f@sE9spn zyIZv)Cfs;$4@u7|eTbcO{i0MyJu17_7ZbPJC`WvDo&2KCmN4mxfT{UfE=gDBry+!qywz>_BY%-JP2uL>9t!5)!%I5p6x9@__nXo zTuzfJci7}aWRW~mFT zQpq5$+2=$f2ZtgULwE86ujUOYsmLg;wQ4qjxO>^U^oBWGbo{8pQK6g>dF5TSd#vE! zj@){W_|4)Y@61P6%I#U?23?!bJ3DKK@Pk+efInK}H2GpI zQj!<)wavNDlI_Eh_Y~@D3%eE7?PY^3+}s)Oh=O7xObX80HyW=vU+Ab9amlR@KZ9rb zj27RX+MpRE(Dtdq`H~Im8^gD`zUb>;crmVt1*fU$(7rBt&oB~7N5oO|&;FDLrrXTcf~(PvchoB+e8GUo@E_B&7ZEGR{5@@VpH)obEWqIG(|6nc4ub;Jn*gYjbzBV?so#!URw>Ox=K<` zvu6vG64N}#=LUH!tsa!cdv%XDE;->Ioc0)%7>}QKu`JSYeNJIkv+q>i3m(R|;}s7@ zcUi+V-aLu(?53ueXx5)HFRFROc8(zgrsBFs-}cunw}u9Ulw9ZO9@%2k*0ACEU61d; zCj!%p2K_#ug_h_6C%vA(1!?j?IO%48vl(p>p zGK0x7#x9kF?8exIgt1JPA!Hskih7ZgLTLlbRT%Kh@a ztgJTAsR~MOrvD~$PY~mb?``;}|0b9(Pn7G$mN2}hHMQ{{$thYedo0SWc%9_v&6XwulX@bP$j((E{4tv@roNv&rC8VD=^t;OG7{6+`g_o@Lfl{A}L{4onSG6<8@&bym3 zAG}B0H!k`*&O*5iJM`Mb32p_c<7}69?SOM!=?LpTHVdB`;SM~0h7PyF6d$5|t_-^8Q21(LWn}6L& zRP5BwZ(QC_l5X3}S8*@VK|-kJp~KdSI;(k%s1D7a>+Zu>@%`x>*mloupVj`-(4CTa0YN z7js+cdYOJ`PMfOkuv(Lpj%ku^8Mdye{Ijv_SnxkI%nqldn+DSf3!Udhzpq!ihP`Pp zhK$3UUQw?#!zIO>tOk*b+HiquXI56m>01Y*3-zTVdrB<6lh40i^PZGAVIJ#&W zk~FeDO>Tx0GOIu9nE0&b?H+Y;iulbos9!L&VqEh(MzeWy>#gzCQeS~+KBg7)sPdQVFGR(ExG1Al=a zk|OCkr_vbCPAIWx2gNDWTxl}T@7m3o8(gZYE!5oapZ(fW3?oyX)k}m|VV*tP4cekP zw(QD$a2gH?vm5fUA)XIz$xyy3j|eGuFUiT~Rm$qHHrM>|_f7)O{Lfq?jv`CM-w($U zzKa*v6dON3r@xWXX{iDm4Nw^YS79MSFXKlODdT<Rmx)!&sj3U;;_#W&jsrFybl6&K?S{%`CXS6$p^7yA`g0DkCFT`>G%)eec}!S6(ci3Pl7PXvw%r^p?X>Lx zEF`Ziz&76%Wm`cK&r_ldPD`9?gZ*tvOl_;`4D5K;Rr^=l^>*m?)DCv!qUu0&T@kyc=bGMu&kmvRTO zkAL^>9r5xzVM~?awKv0-qNcG;V`#DxGo9th(}LJui`bb3*RlWRxtBK)Zk!YSdp;n~ zBB4m-P-6MR*t(;df)l!!YpIb9)=|!c|C|D%1c}H(s%K?yf9uceIr{g^RMzE)nv&*N>ZbV~S%77ZfXhOB0B^reg*`pGu&4+bWSFaSbWyZ{E118`?m4;9vex*BtwU?TWux`svVlvC z&Fs1S(iRad7#6*2Y=rjBc5=s(g;zcfpAD0#_bvpji=iL0mnUvFnwH@B zY|pO6^4En!v>ZmWPTN}32+4#sHejFdLMYr}Jq^bYeNK)b21 zCZ~QKG)aM@Q#$U4p9W?pVS~u@RgWnDNfGOuc~^P$g%-KILxQsp?ymn&fT5rV7@_5r zTMZZ?xB;#7TcqscJ=x*ZXIo~CV!B7I3r1AE_Itr%pc}-tWbG-6Ck{07fL-m_1!T$H zzuzIXHZ-BFt*V-ah>Yi5iC2%qhGB>bE#@nHO1!`414U4|^cQEIC^85bjNL5yB ztlH=kb{wf<9(}POx@D#_Su;o2+gy2c7RYZ43HC96WWmw$)ay#K_r5b;aC_cAJ&V;( zIqNFeKz)WeJNP56w*nV#ceRgg$slSA9}z#IE$3SiyKNl>6j`n&OHs>Tnf4@<#eVL! z7g6&M8e@dE#*(@D${*@Z6&HwxHcLx0iL=KbBe!VubgC3*VppI{MPw#G;IHABE!EDz zR33rc`fcg`-jnaAk1!^5st?;MY9+LF7WcewZ?96xmsWhJWl@**csn3aJ6fw?kAsxt zczSw>r}JRlibt8v@v|7*6K0G1_uq?I*9tukT(@{~efMZYOe}qw-PS5zuhiG45}O!% zEC=Fs{bj?fL0+D2TBPAe#@1FEl~`bzu0N0g(gGNxW4xzS=mS6nJWToZNlMt=@f0`L zbHepeU@C5nJTO?OLGC3$#BF>h-JLEaIUVm(wdQ zaWaf-WZ7LuE521QH%WEqR(@#Pol=J*9mxu&3F|h9n@LnIK2w^n&IP7HOq}5yu`oST z4WZ!aMP{J}9LebuiQH0mG8K7cagT zb?>1of3h-5RJLeK?1b18ZxpB}H^4+%1dJDV${EGhN0%+{+azcu@gD?*Yq@z4Pv7oZ z`*SyL=tCq~4+O1Hx7&6aQ@j~PPlyRd4m*SA*-2dwsfph2k z=V3V&h;#kk-9X{rf5YhU=`bK&>* zaY8Y&Kp1=DSK0cuo!!x1-NB$f0g_~4Mwx&lkK91fczIC!9IqT-8*2J%&~j~A0O$0; z>#qW`mgSoC8MmRg9I4UekJ8Z@Q@d-Gg|!VmV6_Bl2xxg=XO`H~cZ!meI-8;=ZGoZ< z;tT*$cj4-mY6vR^gX^nLV*Gr)g-z)x zLQ6?aeqjXu(Jhsk{Z3qRrwpYpMhs#$ecd12^07_y>kwn-IF5E`O-ar547Z!4B&y$6 zd-#gBR&*KD1I2xzq*TrJuEIKR1;b*~^dJVge7D-4N0zhJ;*;2a@4*QfpzG|+vh--b z8##>D!zBkG)T!yv!Mx4Y0I1~Iy{Yqf<>?C0&V*=oC*%MCRG_h*(euTZ8$S5ZKt_*)Q*)y59iaDAw%7WUk_Hb9<9<0xF(DT zMGn0p3Hld;ob87NV@cCrmZ51cmo)0vM6@SXT_AWVK;1JU_)xX;qf2w@atszQ-OmrQ*vIeJ-DPBsAEa`eo^lj!C%3~a@sG6z zCk8O>@7|_v`k-LJ-pTucvPb7q?3wI6bftSiNDSVL5rX>Vsfgr{8Y`d&d7bG!Jw3?3 z(jxzgE)x81|n?PaRxO% zKnbcXGx0MFyxOdf2y4Tf(t5I}9;_smF;Ga7cc|5cIJ0^`etpVP2~vzcMwj} z`kpv0xvCMVvk|jPQiF?b zE-TY4(48h-pK&R7`4#2tJFl1ZoAXpi@dK>&I#J#w;WDGg`>NW-ftpHkojrIxt)bO# z%Lm4OHW!^9SsjC}dFHuG;A{aM|67c}f<-K6!cJbx9f&K`P0RNwQ zWpnN|cCr!nkWyy@N^uz3Nc z4!TTkc*0be;InS2wsxI+?bUJ&9u66`Zq){qZLe@^?60>S{#Q@5YTA4l{OgOMEwL)b zVPk0f0iVzldg0MA#u?nIb4e8lAz{zmg_By*t>lD;h_7!b&R<%@n*aAvuPw;Tk`b`oQx|z;e zH>f`qFbgD6;D62^*ln=?LfO7Drh&D*4J8ghV)$ANB1t36l1cU;hiUTMRJwmaU00V- z`(%E=d6$_r)617*P-=hF-v0IJ+OF0qpjX-(P^bPlE*SkgW^d|6T#PT62}IA&xo-No3}NoKkgd6}0QBy&5ax$< zU4X6s3X<@%V;4ZbHv$lN|5hNf`^Et1gWxyt1d5Z4Clg=R z37~aG486hl4EGI`(uMb=O#X^~bd4~2gs#x!plFom=ZB}H{6HozZ2b8JkAemstj>Iv znl5-QR7|ZvAS5C*M~!4Wk?z zKVjhaB0KA;QzU*GsO5gRdXW+@0J9xi>zO)Io1-wpSF*ljX1i}6YrGhC+k@@gjiK=b z!Atp;UA5R1sQkRUfd)0*Di+Zr*-rMWQ?z%Z?y-(nkd^xVNX%$Gg*KGY+&nw6iy0Zy z#KJ4JOUvzK1{lUOqtdsBD>Z9^{irq(u!b`(G@g;6hOu1)u2I`u6B<8U)GnBdPpzLRP;wY?!2EH464a-v|Uws^l9n ztb+62o8{QkQKt(5DN8s%f@+#wv~CQUpp@}92@%s!Q{Iz>pCV1m%I2QVyVW}XmFyYr z6InW6U{Y&^uv`ZtF=Ca*k1XS^y{E2QIGkq6HNZ+re6=hb7|nMWpCH8rKG#`^@f*qt z-=nulDiah(Pu30VZ_@P{q1`ll>X_sYH{&*IqIQ`9FQmj_K<8(dMW`UQs&e%cZG*@$ zIk|83(JGW`3kkkdicE8h^Il;i=eL*BNHz9-j<@ec{3J-Wk2l=YP*I_Ycg9KAEiG+G z=P*m($bga8enxI=Y9KjMd|2=B{6|*npU1dhxtx_lMw-ut z%ez+dx(&p~fS_%p3FzSK$X)f15ncn+y&k)XljEHM%$7xvXWqE=S6A@vIs@&tNCm?yWP&Gp~PdVkLVsviEuswFw zt5T?Z*nWPr&IsO|0DK~c`JoDRC5x8w3z#cTrv>NR4$jpw(pk>98t`c=`LBumQ^L^^ zeS-a;LUt9*>mwz=?nCbqUL&`6f_TrWV4j~(-}Cp_nue~fj>9MjG;DyQRJ-IyCFV%D zT+F$>4t05=l3N%S&hG2kp&^M9ezm!`eNHj>tx_{Ht2Jz@p!J&d;s6G_1=~jYMB}oJ z9c{$x0>zJZ{P!Lo->nHf)2jv?2Pi;l0dAGe?&P(@lebUIwSf-gf^( zHjm!wQf*L8`b?fxSHpU~9zWHZ10*X$&tYb~A!W!_yaioL47lKi!P)uB7Wqm%N+wTb z<%~EK=QBQ;QAh(X7C>r%NCYPIS~{q3aT z1a_6Sx0#Lp4sT zsebyNXeP104$7OQQSYDl)}E}!T-OkLbxP`U5utfPe@#DbA=quMJU4~rZPF^#`(1C8 zg#G%prEv>3Ne&;E82!RZ=}$tK*>X{bkzIAm5df;I7ucnEqDfodpqP;yUC(TrP`59X zk>1pKM8?SkU#wedj=3;vo04Kwn3J-7F}0SPmyhpGXQ$?#JkIM!`qquWs-<=Y2s4lu zqH3|p${#)(CboN%DN$Vggo>nGrDNJdadA!NE6Q>&)lOCEM7V@?NzH#U1HO>0r(582 zKi|^=c|8ereP`VE;A5IY2JozPCbKV}`ERte=X%k+xuI*C&!Rb1(y4ukS!{n*f2!9& z@a&tyIJ^VQGW`@#ivvX(V4Sy_uC8T>X$wMG;NDXx^$%0HGrvK9nFXPOnpHm^Pd3L$ z01~sqGWru(l?PdPF7ohv!TN%Ai3G6S5oas^pB=ni>8@k?=jp$H>A5+S9)2_yv(Wgj zDob8Yu3|$)yL=*CawULs>lW}PHGnB!Fpsp!!19C&@$m%k%fcom1`Yf5|KT04lN~f1 zwx3QQPP=P^p-}3dCsuv0)|Ib^|K4}t*@v+4DQQkL6Z<)zNgxg7ZIX=&fhppRvd7=! z^mUh&3A)q)Rzieq#ka;LGT?-Q&<{UeJ&1qPZ z^y^M*v)w2Ud-Vmqe`ag?-mK=shpGJ&tZJAhC<*P^}Le}go{$KypsV%b_<1%F>oFKD~MTJ?u=AC zma(w?)1Ph7l|A-HLYtI!rrUP8roA@BB)O_91LwSIO)$c5PUg0Q5IIt+l9V>#K(i0M zxk0oC9=mnY6f^*9C<%%{7nV#l^L|G#@6A8Hpb?j*Nw#YoY{bnGPfafVWx zpYWj{VJjQ(+xS|lTHUS~`Q0SJuJ0xIea)flt?=Rn8J|h;<1k#`u=d$%z6oYdBEM{4 z!uzeaxQ4Oe9&&TX?Ba~d$dIAn>xRepvAQVT3|WER`T!L3uXzAK%^n2RE`~)wf+ASU#%tUA%|eC6yS5aeRY@^)>xZWWtjDz}9SfXW zkCK3S?6G+rs08O_FLljy2kxR!1Jh(MWdX;HhV*P`+H8`d4Ppi=DrtylVnx=-esS$NjBrv$V^?2v+9jn6 zFV+S9{wY}rIubw19n>KcZmi9%+1R;F7EyZILJ_FBdZaSg)@TEx z=sDB0hG^%-*>PkRfGCmHVSf&$O{7~Ehb?g!x(=M^vdU`qwU>wOI>b3)%IaoAcW;*8 zXg%cPr%zPU;X{Xn-4^uC%;{-vs1LYD04_8$1)sk!2QT%HO;gWNssv{XD(MDC1Y+#= zZ=zKx7nnm|h#g&gLoSzBl)+(}tM3pCKQIXr zlk+=!_AF_*3J-DHAPdK0eJL2wOI?6=jL|!VY__Sq+}sW%eKW#uONItHr7qJQXCWSk zr>Q$p5Y+ANk13m{!6wf&za#He7ikR)cK-9xk@&yj&;E}e^L5K?(-*3pUmiNl&n1T_ zb&)@SBtsLP)WsbP3hqcQAjPdOlRH_I^R)~OE4Z3WulDx2C5S%OEt~TVJB5dkAtmVZ zP8T(f5ZisrRNY3C(m{#_zoB`?g|UvC70Md2`Pb5f?nDtZpMM(Y8k~BMP{N&EkCyZVZF z*{;9GK~n@cC486iJ7C-+inKEO*63~z5C5Jg6B6{?n=$0EFtLU10=!_)`f?m6y2!ZU zV1c2JEWCjwP(Qi3`ZQI?de@g51}a6=K?N@ens%dYr?*^9*4FX$wpd|nSBHXxDkCTp zW`&|gfIBIPR*bU*f0>^}H|95gy=o4{&^suSI51#{VAXLoNI?$aF>H}u@dVHhk7nJb zG9>oh9gJm=OY)#c^y{MSoui%cSDMeqDH@BX1^sI$)0Hs@0e}Rf73GahRE-; zv}6eSSB+r@ha7~a5n75)|CR+@d5NkE4?zFNB}e_@X1(GGeL6~?vzz5MGm+GneK<1H zfSeo+{P7;6QJx9bsa@we$pxQMlH-W{9#Vz^>wc$jz!E{7O3r_AJprf{s4>SvCah%6 zQl$K-1Qn?7yPLbfOeD;=I06Ls7v#{F+-3`=TrJm^C40~@2J`{oOee$9KFM5gFXAC$ z>VhKIIIM46CKQb(i(ulIIKB~5wo45nAqr+6g(RYkABoRAdVTs{45Sf3{=Kp>yQ63TemPoA-9%Gorn4IP2juV)6JmIe*^K*$y4EGC1BZ)$i#`8YF?01MePPel+TBzGw-bJilh-luGYQ?<62Ea96~wF&C(v zvuPc4@V?-)HNMa?>KvhsP_hL4RG^DLOtrl);@My=9d75jA0pe2Yb;Hau8;aN)Bl?|l}A8+1D%*41Ykn}l* zJuLQXYLxp>-n`k~V{T~G_nynHf9XAsqHdZ=qabs<2axk14aV;G8*-uM(YAY*y+mBQ07QgtApj-w@xZdQVJG>YNQX#MdD{S+~swAp9hlZAb z2q+k0-^o^!&FYbv;`r|vyy&cl4J>&98>TE7{MLj5zIta)1UEIDXD^;PawA=7Qm)~GSrl%&m^Z?%O73uTV$gu8+n1;BzU@+LO`*{ z=I1_tPfzJC)aDOc_w@7G!VWJPZd>PqmfdV0tM7cGS&YcH;p*(|cDt+Z!l1b_+1_a| z)A=-_o%Es6!p4@9?J>c`uSSM_qqj!6{3N)t2`{f@tfY|r(wB^LR(;%MH*elhHyM2G z7|KB#Jb{yci7sg$KTQkXdm-zKh~^OeyOr)VKG;8J40M{WMJm;2a(ZWTr&G}0aN4yj!D8}jWR27at`|v@q_P8r53S8Ak)V2wSqH5 z%#RowD24YdWL&v))?8{MPEwc_Y=d$!spPV1N2`O7HGVZ=&`j9C_l)Z1&4UQJQm#2;eBlfdaCc6NFBC{`P|YRfia;JPQE z>yqM1Zd=7fyvp~rcg&LBev+xCc8>Dx+UyS(!v$JN`&D2QoTQlBG*9Sav|N3*7B#{+ zsM%Mq%5%w`JjuN=YviV7bBfKnx7uSuQ%WF!vpNEDpivxToV6|w{>c?7WUrHl8!-SvXYt7L`! zUos&iyx6HiKWxDQLI82K(Dg&j7T~S>-anS`SjsGO7%enuw65Kcg)n2ph6Td9Ax*x@ z22@3)jN|9FBf>Jok#utr7i4gdk@Y$lT&eJXc?JGy7h?U*#s9T_3<=V@CvIgw=(lEJS1&b}BCY z#}CJTTZ;5`3uqG5S_RxjE`2-AS&vfU!sVLEB-5B@;vBKHS=S_t=o+T@(Kmw5pdLelXhelSwGqq7)AJQliv4JYfH@G`ccW=i}qmLrkqjy$~`|WS?8);Ms1fKuhjUT?Cj^N zhVumt(nUO+vm6c)&wc3Z=-3z?GT1XkNOf|_)-_# z@cd=5c3-VUtd@Qh!U^U|82eyReZLXtaKS(RHsUo%nLC*S72m5bl@W}?RgG4}!bUKU zHqab=I|Jh!l(#`P)Qw;g)wcHiqxzcoD;ZU~_(H!k8)-5ulwQ3xC~&;Y4u0QUA8lvo z%8ODay}97k!lAiVW_C~{ujTa}#jPe4pZSxLN3-H+kXC z`BimSZpq}Ju*+U+fqftyg0+6*=Pb*g{b&#Rse8Y;`xGFKN9|hW9X{Ut1)3gA%kN)~ z96Oe+Q(~THa*sy)R55Iie)b;-PZCV`)@exPXrP51avjO=FS_z`teswp! zR`azA^?9*z_qz9dXl;9Rc^X<@+8P?sM$`?;ENWwAW=B7zC~&hUe|s^kY6^gY_+15m zv2sGy6c84Zmj;r(WtDjh$h7?3F zTOzKs`)#X3+KM`G5;Crd?t|)5`Pzek_+51IgO#t;wqySZFP57#7#8XxO^LrB!XU0( zijJ2cYLApZ2?}CXW^&(#wjiK8+2EoUlVWrhL&$ymRt2?iy1=kz9K57j1FS@2a*}lm zK->5rGF$NhVZ4DQ-uvV=i^85$4yWI6ftu$w3_RTJWo@lW9LgqJDeJ+823=Kkbj*N@3I4#UAWZ4%8jp@WCKOQr-DY#7-*~>gs`9qM^0g33<7Vu7vvQ5F zRuVn;Ml)FD@PmU|h+s_9X=617werfLNG{AgiN42Rd(L8@Z0;==K=aw0*myqm!^4H~ zf7LgrB@HGNfH@F!TZ``ngD|=a5Fcks#QQ z!O1$ES#_DWg-xZ$Iw3PvtH6$(gzSbF9a-zD_Liof}{BXimLt z7aiU6oBV2UzOxuL9`R z#*y3&+#p0jL*Fh22Mh|7LaHq;75Wk`)31?|V`XJUVo>+eF$&oAgDpBpx(ZI=sxeP4 zWA(|&0u?`7_W9e`A@LBe7!Uqe3dje=58rE+1eO1u+{m-|5712Rxz?T^!*w!@o|V-{ z@A2c_GdYL!;{=@%t2+gr0H)LK`mjEWSkb_@X?}pJ*WYgznaN4}eT=%jc0S(aq^0;x zd7;j>2b0=_d(iIMb|1ZJw4N9D1>K$c855>;A?H&TjlJq2D|bv}rmE3QLD48jUw+Hp z+V5zWSwC4p4YLZpvt!4oL`n5OkFk3y#!~-n-x`@Y{rNKWcTFDAw8HY{9=$+PVHr;P zK%P+@vDA&sE!VX&lLs}(M_foyLiFG)A+7q5U6a#A{73e=>*aEv1LK8->$8)K&R#vq z0EXiLr=D2`+hOq6giJ@Bd0c(Kew(e|i&q^-UYxpr`}R{q_pyC-mGHe94>8t=);S|5 z%fkJ@6ZfPLw1G}DqfJ9h^T3LHC#a<p*v#Rsg;pTI{C?m?H~8i-`wKALpMHzWtH~i zfu(?F?#w*?0{^9NY@hEiIxpzIQA)n_noDZSkr{-R#2+p0>ytjnktL=v?19yiS?c6_ zHf;OTeQVWz@~qr(`*TH3FtXjK(0k1?X71!W$_=dsZn;kuD;f`lnGeZM1ojkh`jTs+ z5R?*o!(PA9=IB8&@u>RLktb=M#rH}%rcb2ZsS@vd57FvLn#V&Cb@}K3Ad4FE`ZOcq= z&hqMJGs0T7b-Hr|Zi};1<6XH(fiCnQr`%yDHhqUS&TnrT4Mg7JaIw^1+tfH=uT^U! zad3(UQLU)g&F@IFJg@VtXTDxE2kDPIrZGs1`^d<}{V_eb@(!uUUR6Gf=}o;OH7N>8 zgj5euYKC!0>4Iq(c69)9z*fGTa`-yqLJzlS1+bh~va~c-fQ9~*fMyBF73v7Avq`AX z7lh=u`}_9o+o9`R0qO#eVtnR{12mh1kfebD1rz;F&q5wPj~o6lGh9l`%K@2H13^a1 zvu0tk8K53d|NQy0{{YkJ>+hScJr4@9{`&eXfcABHa1}MAJXdm2QqjOj{uIRMITm;v zA)zJR@K7_v;^meBFtTfP61GZ#L1NeBD!jf8fjR&2@E5waU-Ccx8=jtV1d|%3I7p_6 zkmnPkMOWNFJ9QOCJ*GKTXk^&qS$0!?8G?fv`RNtNtW)asRBa zuamZ7;*chu@qwCCwSj2KZTN2D;$~n+)4f3@<7dD!#%M&(K@nE3;ee82Hmlxz!*~&l zcEFU^)*|ztY}Q>MDQ6Qa%m7r?OxTd)Tld7G_1eOvnx$r6#F{K7hGW#bU0S-bsW`)| zI3bK8W_|ngT~^O2W<7-3jj5Vwx_MjSc|zw~ETtR%K#Y?Nu3lafFZR$3M=ZpI<)xUU^kXZjnMgF-a?@=wYSS`flr}1!- zv8Aza^ldfbMNAN`jQ7eb>Pfg}yPaDE@PHNCK)&CSJKERHXX%F=bxNAn`R&~`GyLG} z^e)SFxb<_cRr2!k-YBO1%y*C~)A}lS3rzm_VF^Q_wz8{B4Uq{UT2!DRK6}dqYaPtF zoiGcC#~77~t1-mF=9rxW{ftr2mpbmTG!e78<(=y~8NDAei1z0DdFKxF1za1dC0G;S ztwk)_PO~06);mxL$zmZdURai$hFg&&fl>oXzJ`WICg@y&2WjBzcY*c{dVR3lqgJL> z8@_z8h7&)Dh)uB-9n_qE^FJQ`KUos_v;Wx( z6IpVv;dAng)*jP>Y)TJSdsb~v4Ye6ntE_u4yl#yAeCRZs#3AH@{y5eZvM(5CT>n@u zQcB{^Skz>Hzl{JNPXI8YYa;b&NP4?IA`cjNDa!nJFyu1;x`wk;7D6c0^5KDp*r z0d0>8jP9nP0!Y5nGR5I14(d z)G?5MQaYvGW;6U2dg}^g!I~YGp_qhO6&)Rl$x=qdYEQTFTGi_9r$TZJC6-PXfW>1! z-^5EC8FGrLjGvUID}Va@9V6+~akTGZ8d$9SD@28uW?19d1kmr|cmp~=e}!OF?sywv zB4tF}gixVe#P2^Dc&T2$bU|Q$q#L=A0&x(01}d2h5G=wyYSvCkN-6-Zv-Ou(C$F;U zZR0Kj*8*}5_Cq+AJg}iC|1ZQ22{&a3N|>sA3{bYl`9J0*^i|I8CkpK@XMz+^5c%&H zk#Tn~N8)!XoTtzrbp9d=rBkiB$GP+8MLbt5kx;CA;io%c??J83^|KBB1Q6*Hvvkje z6z_ESB31zDz`(Wcae@&jY9U_8BSEK+FEbVI5om5EfBBD508d-Loc^!uLdR=P9@WjR zH$t8UB_^EK=3r}YrVE4-qg zGvBI2{qdKgwb`{6+v*~Eosx%JL{*Q92f*3v=@Y#P)%gmV4S<P%^-_qFlan{)f>L4!i1|{z7Ah|6B+}L+!G56in1)tBZ6)kF2oWD}FTH@bkVq#S zs8XWp#uYeI<18}DvWHHfX*%gjCdoe$rRwQ zcyJ_(hEWB2SWEIqjgv$YVW3i9aK69f9+0Nt)L2Z<%#mMOD*UeXsq~lcwokGIW1qB2 z`X{KHzk6WufG_5c$2*y8$hY|S;{zk#FMh9o&ik*%Z7HB1Erc9n#*OSr4)Db$)cN@##X5*nDS{@>u^D+Ldp@AdmhT3uohx@c z{r`r^&fgt37!0v-s(JqyEvW}t#fK%d)$*3PW125H^TKGk(3e)YM^hH(KiNO?GLw!YM-8%XxAvL@`y4>hhtQXLk1Ai4-X%41{H&k3?PRdjlWKF@PVU0B5mvg^7u$>pVeM z3q>><)>p7S5gtK8EWx5=vl$l5URa!2KHq_#`9K*1x-^0?{cx@FLBXba&HUSUS}+|Q zEZp+Q9W^olWDBZWENlY*azEi~P{RK_&8rfaz|Xp?^ai9IC(j)%IPjpQislFim~NG8 z0T(Z%f+EY7!X<_06SFsufC9oS1o{%BHef> zfnShLU(6g8s%HF_dnfMkKQ6r3nE>MKB-4M{cO8Py0})*$Te++@R&DeLowY}=4(OK> za+!`E!!g%wu{^f(*b}t->F=qhH&```9Y6yi&}G~s=Gj0mN#9`$3273EMri>atd-M_ z`y`p-9|C95>dINtQI>Sa2b8Bdwlmy#YDNHVe z3=H!PRTdT=-uvO{9%K|lsO36oR6;Y;B(Uvg_#rw0L;OVyk6Xxphlze*{Q8~ZSyRvS zk6}zlWPyWi9dFs1JAPesS;$8b)u@xdPIO$1#M@0f!xZ+~;W}Zc~e@ z8ZSp^bu~Q0{bG@~KGr%E=cS%I@a!C4bNUO(^b(0{63$aMuhsP@SKV-1c%^MoF;Sj6 z9#s7DS;zstkkpIC&9(a?-#?$5$&?(PDIDdrlu3BL_dwpIl(pH0(3?=-ZSV}EDkTh1 z*2;O|nuNDNh8jo;x35~5iL!KMNY(js-MTcRTvt*at@(56n!HWS_fqRyYSHvu_jtJn z>>=k409w+6%BT%(aIV%jA94uG0gphzKH__H+CC9!4SYr_WR_W*#^4R)sR-Wp0YFRit3m+*RaX{IiWu1RdkN zM0~pB->*t{CjVZwOm|gdi}BdN$Lv231p>}t=(9cHiFgc+8Qw$)CQpp;Y#P|WJk+~DMO0F`<>>Ej4^;uq@ zEXRMDyKi|bsHHIZpb|yi5gpL-r4=!d=t$oia9cpHOaazi3~mmMDeNkLGI2fVF6O<* zvN`<2CH=N<-+CZ`-PHITygn6mg+T_KDF{%0ohJd7p1ZkFrOXukDbp1AZ>iPQ`pA+{ zZRk+mW?nM$bU%lRxq0@>CPd?(`)j|5n_C{b;_abZhmV+*SSr=sX%H`u{bdZs(p_J| zPT|a2j;(vwmXs-;xT#dDM2+V?NmuHvomu6AYCKV5@*kH9o(`M{eSAW{{~s5NZuIeq zASL&J0i&P9+G6XZU-Yfv&*?L(Y)LUkvfztnRoM<5VHIfJtl+TYyRm7S4>wfol87!# ziAihtuD9qR&Yq`Eb*nM!sPc1X7MP81!j)!&E4??z^Rpx?`3L<$cw@_chEE+3=m#B^ zAl()uYWtK!wW=0|@En$j`H>pE!^bWih-3NVkCl=X>bL4!@FTbQHPqEJ;tW0IKxccD z0t;w2*mGYuI!rfE0S<^(!2LLi0%xC%k+s?!Nb&sFP%Ezan&TGk{tdK(@k8QT0cz>3 z;hpw-cki+rtu-+6UC#>Vls2q-+Q2vU*~<)ax&5xwS*97!9iuwlDz??C75nF<5oU+$ zr69lvALsAK=*1O+rNtARYqSRD8|PI?rZ)9w-RNC%s%)=MXGL39^*^DryY$N%<(t)i zg)VitLX==uBZq%v|ggR(kT{2 zI5zH#7l1vXUFfZ1qcd9TBLdXM{`oB_4gn-8Y~|x&q3122I))rntbzgp;*~0aI5}R@ ztq^i`&Xcf}B)%|C>5)Vgi2E~vj)iWWug{%f$hJo~k~){Xm6_;n*_fXQianj*vF%g` z)3Ve=Y9cAVS1P>yIL~DU8OS0VD1VYsyS-tvFW|OZB+f)xTvp%0NrKMOBikiRy)N_O z{|v4paIu2=&;EU-;MoU-7sR7gQ%n1f-XomxzC^Q$?DyJYt+3{avN==ic>u}Wr&7p#fAk+0 zD3v{zDX0|RgJtI?Na!CuazPE}b4<=){50F^KeI+&D&6gW`|8iuoKlJ09M{-?t?}eF zWi{hJ#op1gVTL$03NNIdPxglmAh+YxpZKFzd(UU8c)WQPxjQf`;Nsiw4hBhx9Ie*Z;Xj2NnvEo#y`@sRq2idM(Wg z@6mo&-XWh}yqI(DLuUE>^Rv7;=ThCm6Fs-Je|l%ExGj%Py;V&$;}8R2+Fw-N}5bBD%Jt+{#-Y!k6hPDU{xSsY& zCjuISMqfRPtalxvF60pdXpM&F9uE9V_x+=%Z~#z1@`Dfl!NLN`%F!<>_PB}9C+9H&fct1;#>Gra-Z5aiyL&;6IugrmYn~;i?Z`%;7HYf zE#=>h4+k;K^Df(T%VC%JY`%pD9Q(j|wX~um2Y%eFp-<#eYu3$!O8xoAN;1;Iigho^ zhZN^^92W*?sq~bbs;G`YHGa?WkM_8W*S@5Ed)v*jL43}Tw9-4B`Tx=N9q?4Q@BgRi zsidbeO2*SrMyRZiQAl>iF%OFDJq|@iOGPLd*?V)8y;YKsJx(?u94mYCyKX(B`u_i) z*X#QYJ>lHvzOVbb-tYJIzBrSnJs4m z*M0AD*Y&XQcuu}Bj^Ux9s&nKTM;@TX@PI~cETE;4I(LSsu&SyaV=~gxb8EBpV}znf zCFsz7y64_C)l{saqGAexKtTl5?(kt0iPL+(_m@L64ak93&*Fw>myzuhnrm%aRB%%W zldT39L90QKb^>SV(g_?)dq{HZby3Ufvh_lHRP^PMfXSv8p6(RR+YKeRHBi_;#T(x$ ztQkMDo81$9*QXBIW~g=%HA1l1`;<`MJUUv@<)e z^t)ksjxRs(ND=c`yDWCkYGd&+Io55~?cAxO7j9ik+&Jo2aYAEmLd45lwaO%ix!UkW zv+0Xw!xsago3W+Ky%u?_Z8j^BHHI(ZpIWLu2e@Tv={EGCdm@kn0sj6EYjiS*Fs2)JWakZVz5&v+ zIVo#QxjGOr+*74C(-&2x^w<2TRQ|Hpnd z-s^F5#9Bf$GW6YR68GHGJ5P1=w)_g&@BzaiDg3js*92yDdi z_hp}=_L6`dZ&`$;#~CJ?I044lQDA439~Kmp2svKxt3Tl78VkR9^9E*1t%8s`V+>wt z|H_SXNoaI2kSl{rN7(PzP%YofPD!bb+akX@zuF7nE9o7LC%Xy&L;Ztj1+Bj~ppZQZMt=#nY(t@or_*coA zCQtdf?8IzM&Pn#>F|um>m+}uD#sZIpww1U>;W#JQnrYJGN|EF2*}jqNWFJw<%WibLlGmYgo727R#5Ow>I8rRrKv9uZIzli z>h^ZTi}B)}pMADRdB0W(OfhUu5eNd5OaTVC6MaJX7>=z@4ofu=R1EpFh zJ-TlGdm46INB-J{nC^M=NQdHhbC#gY+~h%D&y)v-l2-(kROnlZBriER7(I2MRa z4Le-a8lyt<(sGBW-8HY=D$5+m+2F+o2eX8-lXeC7Vc-WziCk_Wj)Q!^ z7JLHEh5s0TF=T!51StqMR7}i?AKB|6gDg|0#oH2{7aeKdK6;v~!EJP#rki(O6%Y7u zk3)QXJkmG-9kmFp>c4JGJ zhzwBmvm~&Q;me9?N$xr7e&?Y@g+gE z!{g=_Ji``TOu4_ty1mv1gPFME5jssRI$Kj$=fgbw?OVtX8|I@|edv<4tV*z|W`j%< ztBX^_8qTt_8ALOs3)yNML2NNNEY?5?l|xY$S?XsC`#p}q#Nov!E5I?K^fLPTCqNl6 zS=zQ$Otf(SiFd=|!jOfj(8Ca-xswUg^4?RF`f&KuTb9y};Jxjxw!|kSkRW0c5tS8c zSqL)iqw@qeFtA|PEyekYY~=SiPYJ;yX^(%J@Il97D|i?OKt)`0j{NtGUh}+Jx!f=7 z^p@^X4=p>tIw5E=kcFtM??4x&QxHL0Qk*9%LlKzTP-#XrUt`_*_o{eptqMwhhKGmk z9-wcP?To-$;6Jyv`|QjNs)&<{i@V}(8T)owI*PZC&HufAtLD6+D9(Tp(UX<+)kQPt zcs`8`eRskj?UebA@8DaH7}C?o&R5Rlwr2M3MDbuavut)!(*7QaSBXVkpw9+LH0mw^ z@#X=9&VoK0pF6*Nx8wEA2OEoSn`&U7k_odJYL|=U1|RYzT@Wdg!CYiC*m%CT`11cQ zNb2u-d(mB<<0X+}jdn#ckOEo3f;0l6GT%i|g0H^F@TlTd)FKNCf62G4??jfgv>oi2 zZBg{qXsfKQyFyXhv=vE2YZ?J*?S?H2s_ujHXA~k@(Dnr*ke7bZCCig$4P*1w=g40+ zp|`_A*WMoHwS+L`b>7Npf90H7$qZPU+ z{!D*y+(fxQnK+$$f{`y*d?yOHLA7X{XF`0Td8XE}h%h=#h^D!@d3q?_ARMH_P?KVe zZ-Sg2nO?C83k%1T0_KK_H)zL^llEBnfZ~c+^m9N38gu?ZOPJ=9o2+l0% zV}}c*9@S}C_>qRouys6>QVE1MKPDMU^4)S!*kM9EyGRt!Sg?7N zyN0s&Y)-_1LhePUXas036JK3LgS~gAy9ikjF?qGq5t}nrJ zY9~5IFqBmb{3;WjI;klsN8pZX1?-)Gid@M1C-+ELs3cV!zJ7tceEixu#bNcwTN|ckYhku_ zjE&730EqrR8?u&pTF|eZ;WRgLP>~S;e3*pjGF?G)nn_H{=#4rMW+zD?l9e)nAMCzh z1V)Oq7p-rIYu~w3pyRfV-?e=k%1k~(pF}(qIyHt_5-nxR-$GeAIr)C9O;sar?Z`n+ zvaB0BmORQ?;p2=!W36)UddFiFk2wkk57h>9Ngfbhit->o3UYrPBdxOD42Zx=n}A&* zclU`eZ^Yi+_#R_mmj3ravUo52QTgA6Kp(ul^R)@6_Me_w!h6P+uMdWthM!3G(3Znp zoi*Kd1byapATagBTk-e_Y;kjf_?blO?QbidQyO54XTQp+P+;~;DJTS?9R1%croom! zT)x|;l0L^$mQxB1J_jk+sq#L-Q`|zornejFXQ|O>viA1&gY$5LQ0BKx2e5<+Z~y?0 z6h09EO*;%(V5Zy~yv2mY%jStG7gcZ_$!SKABHT6T#G0IvYbL73pp*!tn= zVUW|~tU8gE6eK_^HM$!yDBM8vP;Foq(=6G3`h}ecCAWCr@9DmFx%9xVnPMqb@}i6iMfle5&-ucO-Mz)bhRHz+|5Z*FUPe?NyCw zP6{;kAH)NCe>xt0b4}jR+636kq$U~8^qg@J95w>?TE72DOs7v@ zZJ7cdkI{80nm-yq$7AR99zrVtjiTe=r-LxsO!CLAovu}^{Icjn2>lNCs?(03eY~B> z(&;m|Du z0||dfRAhfNPTYm^87`3~9wJWJ3|e{l30CH^wh1h45^25E`h^k=_4?bCe0#b#Hg6`ZIO1SH6clgEfun4xnr6_wL*)9z@ zzfgKH_?aLkY=E?BzBzJ3C05cGCM?N@TkJ1y0C;C^riOD2l);aMH`nZdQ8JV_FMTlN zifaDQ_xT79{_nZSsG5bY$^RbX&Y!ji@O(Dz)Wf28L{UxX{6fLw<|}%#+EapUH&Ma$ zQh=*;yYaTntgMsOu;cfFV)DlZ5r))1HKSA$a8p8#`EXr$h=^M$x|6SYrc?K#o3UnSVTngK{qt2*HlI01@@AENB**a)>&|6xFcUc(f|f ze-0D3V;WS+=o|>lfWgB77P!w&xZ7uG%O&^X|i~+c9+q23Xw#-%mKOF)=a4aPaXF zew*(jcTwuHb=XAxu0A^w4Yb+SmgT#` z0D(jf;4#LDsIKZMah7aB@=Wlx-U=l&<=?r3PT#74qW!G|?{T)P<;}SLeCC&d+Q1KZ z^N~RDS%Bod+1c4nNJD2b+7yq*AwWBWDPV)Vx($Z!@je)98@ii!e+?mFCx1`+v)PZA zHxv$Foh)>NS+~QMz@*Dd{1pepN%rLJfGdC_(c#3q2g3LBZO3raqGdS6#>N%{2%FK5 zdT)NTEj<*?;Q{ZGCo5mS>IrSG&4OvuL#e6;>br}0Ag6v*hM3-vb2slJRfeJ(4hNn9 z)XYc7Ls0TY4JQun3}%!QZ$N6HDlhMYBJEp=^OtJl$1?Tdis(C0z#)hh!m#q96+rZb zuQEPK+{G{r*9;UJhX#<8?!TuG z*XqK{_21|4Q5YGZa=TV*au(DBIzG-lb>}CSGV{fnyxXw7gVv|-#3=Q!f32RwUX1ZE z^FZYRR!lY}x;KZwarUJj`d;bEaBCt|7R5v--^y5&PkwrF^4m_a)L~nR9!&i|h2h&g z09{}zUkU)|Pmtr{*hF5hB^RF^v@}F?PP8HI$(u0Ys-|<~r~wY0H#CSs&cZlF=4B?L zzN4N3tCSj@Oz=3CyB%Vyc9BCX8Xye}wk%TM<8Zi=32?dt_3`i+2=9y9GITQ_XEc=U zoTE_tu_{dm5B$C^UFyF9D^V5~RVInEMKboU_PlXFZTIDH1NtNh9UI(E`A`XMJ$g89 z?!3`5VPf*3@%9A(s^ zvNbjV62I#DdMkXSLG}9OfEBT=SCSkj5NGSw@H>31`oZ)!IwoD~)492-Mrrno*9j>h zZ9JDxG|;L5cVHcbi=<08$G?0Yb5rPWn4u(E9-09bck0t{alj`2IWK|s=8N5Ko9iTy zbc(yk$$3+sRgVG{$E(BS>47yvK)S5E?t0KQjyMbBJzoa~5JnA{vv`XexDx^FohX$I z;B?_kHCroOs!s}`f-(mE%#Vmt1N39<2m@rZ20PLNn@t;^O-_>{=~}G^R6*?ljXti0 z!x7GNv5f22g0vkp(q}!&fS=95kEx>JTu?NmfoDaU=wA(=pVrqOR2ciT2a}HF7&wcz z`@c^gopseO@y2~tm7d(B>YZ)6NI#F@-iB}RXDj5kDHI-`eeR!b;d*C+uHwT^AGjj6 z8*9K>5Wtft6@yT4K|L!`crVaRAw!I<7gnsBB#Eqj7o+F5{eX(+KD|-usg^XIJ+|%az{pwq zbUAoUj!#F9TOHA^(CiiDuECHSs-!748>0 zdlbZ-y%QdLpWiXP{G7MY((@f{)%VXKJPw!s*3VP^qZsq?NHqDCZ{3ct{B88O#0BD@@dD9G<195$Drn@8)8Gq9Zs5_FPMvD$+yI#$ zqS`kW^2$u$MhAU#?WT*Bv*^YJ zz3p|gc9@EUtRA53J?t0FV*|k4P)Z8xkNYag7jpRhd|gqN-Gve@G?bLaP^|5JbmcVD z4$P||m{$pAxY{R43vi17T-6mO3TG(B6R!@vH1`FNKwx_mL%2RYl32WRE`RSVVi)s{zBjw9e-IC2ZG?9E9);*z` zT^rNs{LxH2M233b>eok5>i|>ogyHV@%Lr(Ku>S%KhqKaqr-%ragq9g4(3`{{)%J~Z zw-slB2Q5efQGoo-;$QAy40mAb_`kjO=Kp6+$dDR%6=#joAlL_`L-ywWnmkCGr4g_2c0XLO4h|0D1u-#K0J-^5CU+#%g$p8AameLy;w~B* z!7y6+4!w|53xJc!zkmlA+Ei#sLQ(o83F>o@e&JDFiO@IOS{`D{im)rh7bNtPdJ5_U zyAj%{mJ*jk0f%aHXhiK~ymLo;gr9<%lUqA9x9r z!8rd}KeH0O^lI zCCIcgIn0gjAjFShhV_8CDO@3e@(*BI#24PV#IUs(NR?sHk*(=AtWjW#B%+pEbO)VZ zjCo(zzEYY1Uj8AF@VWY z!SZN9>o2N^ zK7aoF#0hXxxNEV3j+5LhM>T1GIZ4)pk1$IQT3@fWK(#YEPt;ZW>(6rtKOl!09SDg& zLAw7gike+BjzluGy8@}d;XxpmH5YG#=HdcMGLiNod(y@;^L&7d9fWsMdsZS6B1NtZ zo7@P?%wluDNI;~=rv9I*Em5>PV)%-v;a?5tgK|$fCdclCue_JLv<&s`rrHcq8daC=F)x=z57T%rXU6vx6Hd-{c7L3ryx8b$IskD?3Ywg7|iR$AnZEHF$L7V1AcOTBQO z3s6l{XvyEoTtEo~i43tb_x0=RAmWJFV&Q>ft@LFS$*>4qx<7o324l_OnR?Xm*Lb^M z|LU)m&`qtqE}E`iVxPBP#^7E*KMo9hW80NT*e4XbPe}xWyg~EPwyI}A7){@)o*e04t_I8hYc>i-e# zG|KBls%y{;#k6HYY%{L#qymBj0NOn$#sb4U&Y`? z+*u%jfs|i$Oz0;3TNo>e4igMX-lBevW~6d;e>ElVThlnD9X{GNAv8cWc7wDMc=qH; zF@We|8C>QRQ9YELoIIaf3k`=&@iNGXS?jp1=505Mbc5U)D#--&?tVTQ&m- z%b}a+rwX~_Kx%6W0E?IPKR!#&8HaSOWsCT69`U%YwSKvfVaak9sJkYJWPX zqEuTziD?LvPI<7m^*jP^etN7=69Rww6j~z+U{P{dGnycba%5uf1i+dR$B!64j85LV zeyF`|4vCuiBN0;_(yHu%f89_D1%22M^%raFyPhB&|Hl5a#|#~pdF=Z7lp7JK61Wek zbaW;Uuo!cz*<##J?ebP0;QZx!-7vG}9CM~v9n`AJ5Jm_t)=q_xH z6Nh?}9JGebg_S0@!|D?4XALX2X{IrUO1u=TgbV0m!GtWfR~6)PX5(MpthCp=CLHB5 zOhPTxx$G<~jo>CVhmH;us7esVJyRdl(okdm{J8Y1#>@B*BbQ&c$wUjpP}Og?SrSnt z=-i}B2xsX>Z7YX%4k(VPP1IQjPP(_Tp!5}FIPnO6-nw)rnxPbhm~ob$4y3+-36UPr z=)j>(=6}~P4xBLJyTXZMJX^?|d%`(jdtsoQU<@>SszWEt^>d%(gtkSMyaru8jK&I+ zHxge*`8d*XK<W6AthUAsHjGlha=T4U%m%$b=R+HYsh`DHvk0IAMFCSrSCUkN>V2T-#F-(`W6Ud z#XyzB62tN5A6w-zy2zkHGkj{-!Cm59j2J+FZ!>Q9zqbd7-zxZ_7`@CTB_q=uCk0&q zvA(7=gH<3O%*eBl*ssroc@_NyRPeu+c~OtDLg;@vDx$LjfW!(u7J@TyBNN(;wm?|& zmU6=!<7?8{sI5Vw;1rW4FWXD)9me7CMk5%OG=peAw6C(Q5D^19$Bc&$A3j;}^xTVd zz|c(2!*Vp_CBk5xOLTq!1IDZng&n(W9&cf2dFJgg0G%TK_l5^#Xh@E95RaP5p{Gg? zNk2cMDwCN>({AN-iexcx*X|~ewQ?uw#_Wz4hgQ4dwFPi9G5zr5;MQ#m0~vH5?B4I! zx)R<=nXfmda};B(h(Pv}zr0|K=EdK&TiH~VtUH_rt{1$D*(>_#dCx@^m@cp_D>jND zDDJA)sOz>qO0T%%5Hlf6?DGo`r&p{)uMdSwcR=`%R2?p_gY&|X3Y;Z#e$WwTq;!Fs z29e_b)pY*0wd4%mVr>q(?{x!a=pc zu<8D6MvkJb*zoV0zwgEy^%hVlqqcl7GI|27bBXXSDA&a3%gU2sCSIO-JN|3`tb5k~ zzW>Y%yK=tYD0mgQ_if(x<0g&wY^hf78%kcL2hBfp2m%9*(R`@0N4`QZn4Dg8EO1Nn z%FKKnkI&hXLtv1i7Y&5Mr|Cd<8z^41L1V}TZ~p@XGy^>sye0b+RUZ5DCiZ`TRdR~Q zV$F#^vqukl{L9ESacDC4KY6+bwJUMld4>6P>o;J2?<~RHg+8u*6#i9O`fvk@?o-C<8^61E zp}Sj95z?7DA_r2NJ(Npss#NZrl9LQefmSiuk0p=2h7qmegAL|a0l4@E&VIB zVFPFq`|#n;!!R;ALy3vzr%EMvY-WHS?QON8d15tz z6U2pyN35(fRl|ZMn`gBJB=WkN#==&$v%Wnc6d2yjzdO?KfYj02Ro3h8zXZ<6yy@yT z*90A@le4omTU+0KpQ#OwQ-2PaqY~3pwvFm{l$(Iy!#&ipB_in0C1l|CmX zx-{Zmo=F{?-36A3@jmqW_;!P%q!yOC@h2A2L_+-t@X}IEQ*I*CZcF6>r_cPniAA;Z zPi$5F{{%GSfxGXXQ!_9tw;z4^Y}?!5>b1m=Pc=#oQFee4G$pZhg!wm zr0|1)|IvYS&>f6B2k<{M*+6N)jRSrp5gc$sUm-n&J4mj9G$a_*1e(ApK3LP^w&{Yl zFO+w~_T@*i?;?Hf_&+>!pB}=%QI-d!q%{moYy^h^9&kW|R4^FY7aDLNXNm!f0!Vpy zaE>6i1-nU22oWV)t-vzR`*X(#I(s^FaOpgfY*;KD#N>ARuG_L?d<;RSBr5swt>HUl z0Tns9jB3{y@ZXh$~o zmk-fo*PhqU1kvedfgr5PQqJAwZNRhX!W19zGYV~Gx~G^VAUcwb(3It?1P8wv4j4*K z%!O$fyrj4Hnlx+7dA&aRPQ{39-g&nBfV|*vshR1O6(B*bCyU!Vid#SX;XbeP$kG$nqJ)K`Bfj*2G1u}_vE!VYqtV~Z##MMmgXpMZO)qF=|g(@1mH%?D1EK0AX35;BMgD<=~G3( z0kl`cgOA!xioD(RO3=|c%Saj+m8{a3_ZbxQ5&rdR%}K6c@a42NoxyF@Ls1*<43j8x z)MB|*u)CnQS_^t{!kjvHKJe&q6OlA)8zfn|9ZLiP&+qD@+gI^;AU5~I@XnYXURSs4 zGJ)gGDcHyJO)`8yheUXjR==G0O^|nyiF9)vPn)1%E z_pyj9=|BDNXXpdXljY~;TnUnZ(WAC0CZF5eokZ$mrP|;0OZDF<<*=bJ$e$3K!M*Xx^NsO<}`i;EkmuG7Q-2noYdS$TO?Q`Itj)N`SXgO*Mh z{3l02=9X%_?{EC1tr-Pq=tIwHNBX9}A^yF@Dd^QQ}P8;udUs=u3o{eYa zs~jI%thNjhT8>_~P0{rsP9jKNNHG@8XQxV?uk-T{_0L$jaw%>0Zt^KL(o8RQD&5{n zhLg%86lPUyTMl#Rav8lQ+_~|ZPSL^*XDVg7{qiHzXHN<#x(~JB)eDs}zYfViQ^I3K zmy0~J2g=K5-0Xy`+oz>Vz7JqS)(6G$1##sXdHs7H!q6deil8D0M4eW>Dff0Xhis4u zkPU|~_boPJS^N74QWZtL8nez-T$e%L3XlgQJV_jKA zv7cDUI7|w6q^Lf3s!SvahEN4GHQ_=8avUG5JC|&e3!X{fwY+qYFxJAN>e$_?E97(v zY;9Ai*eJjv5=OlY4Y}5aE$4#4HLo{q>vWn+P>|YKh#9qMeZ*I5Z9P4I!_OnJG8|Gb zn@yTaT~KsYSb8<=#pz=uKtJuV2(H_@e0HvarV<7US(s*S z2-sV*aS&|ku}VtzFD-sS2fuy&d>QTWf1W6B0Cp5QJyVf_C!igeI)F8<#ud<7J~O-t zSu{l5_{XdO_hV`;ZxzabKnmgNx3>5?s9C^5Mp}l;@%>g~U?N0P?K?vOK*pgSPV#{{ zIRi*HV-ds-`L&8YH=nuj1di}&>hhLW9OWl{dUgf45FWEYn9s+SK|Ds-{OZMtCS})f z6n*|+5B05IL<9j`h=N2QU@RHB z@UAC9FNXKTh-&4LN{L=@9Ikxto384$v9W05`P7D2V^hDvRm%3sUabPC>0!M&(? zI5Oi~7+1Om*KAm>qPBLde0W44(V3Jl$W#%pWi^3s!^8Uy@`mS=Wo}Ws4-f?F2Yh{7 z>s=RKM7TJFo?X%JhHhKSp&Ee z2SDC?cWpnue3u7^#368b-6kPPbPJ7U+_27B7-yjfs`MTM)tqJ8JlAHW8!jG92)$ut zYyILfN(G`LH`nxq=35?S8J2z7y&UM`{$4t#;hZ(odUw?x87aP@H08rzEUHr=1 z>xpigi>l$D*N1e`G3^-I# z%L#oPERDSL^2UWyHQS zR|!;Hu*DYH+K#1zI`SAm9RNIpPCd6FGntCIZQ;1*XUgX}YZA=99(c^cBv{&$d;V!- z;cPuIsq^)mwYH+=qudg=k+jZ>4w=#(HicbTMqeL&EWIHi(c={Xb2JuHlSbxNr@9>% zgPJuKr}MfkTQgkoV2(MA!$Ng7!zkH*mLKU2kSRI98pN-W!4 zHR`uDG<+7wUL+wU)tJ-abBw5=AR=@gw{Uy=3Ur38rL7MSVMpK{wD)-TpcxDdZb?YA zo8^S_q(*K`^b*K}?CjXkr9jA=3K!f5+mT>h;5c=PUDq{LNx3_xLm-?NuJ)h65>|ivjeRybbH%gfU!jt^# z4_`O;l+jpM_U}14dVghq+e81=c`2$PRc7^X%Jp%^2991qOewKvr=QCCm1H($CqJPx)CK5KlFsXHlQc1zvxk>P@qOY{g|t3T@O4iaOra!%I0Qf zex>b}{L9*jvFv$d>%g?xM2B)2@%@#IuV3k)qL$aMG-P98St(u>YJQ{7;Ih!J2($*& z5vK8pl_U4acJt)dt;|c}*KWsG*Q`rRJyl(td`c@+_%h{zoE>&%+J#9VH&1=&OzJJi z4*!<(s@;;em<8JITwbRh#xh$;MqgqQ#yYRrigyM)XNaAPcuHzupAK6UwA-Aaykn4~ zMHTc_Y51Lq^kmBtQC6AxqIycKv*!@YO!ZXii|(7s_Gw~IgWpt-1jH?IEmSUye|V=v z->H>WeJFA*U>koo1(V@N<}AmVJ3>Z}Bw_ABf@jIz-B&!NJXJM`I}HxV2DG+u>uYOV zj`uMaOwJq#*!zgQBEGQ2<+jJ6CVg0y*Wc#yZ{_HAjiue^IX+}A9R0q0Xhv0qjki7C zfBZK7XwzlYu}2m~X;Dhcf?g_4#Y(LVWoLQ4V!9}p;wqWb&F(7|ec-FX--4HKf2&jK zj%UeCAhw3VKpB^!H7ax{poc5DCz312uJ~h@7TnQRos*p?lLZ-*U0z(*B}1 zMkQmnQy&~0+UHWAo;GJz8-g49rka$NrU|i-_WTL49lPUr3w^nIX+lpZYk#ZjW>9qP zyLWFs*V`LU+FUPd9B$c++`I%0;khy&`K;`$%3Q94%`w${Zls7?9*8ye27&Q6oBu2I zP3OhW!7lSbWq0HEAM=qCn4euEx_PUb3r#sg^AtFV@$}C`ZrJD0`KK+@ zaZRU<-BnYjTkN`Px4wE+TfpHNc;ossWVn^^;uh-!$>z%$BRc0bgmt|^Y2$L#o0!kF<&(v zrwe8nvs^Cv`0YN{MKiOIs>1_!ajhf?SXdq#5K~A19$?W|+sF|IhWgkBr?#Rl; z)%@Y%MVL^LstQt+008ght|8<9uKa2$cz9?K9^*n^IH_0BvFqt zprP{b`bI;AIiV9DoC+4H7#XefKqcCs1d>N&niY8}5qt*5W)JI7VadiU%9g`-XfFH8 zu?p&Fm}$r=Wnb;?Q6pYDr>+``-4wd>LfSXnly4(i#(rY)aRiSH-awSGzLUV&+r_5v zBBPF%C42x6e`EY}KjUUm zsNgqOam;zXu`8W5eZVoWjYlVM&B?#e_sN>2!_>;%^rf1s^u4-DX7p;%k>$#JC8(!q z^*)!IS>qJZw0UNSpZuoqT93Qx+B}CcSyC&j}F~f8$M6!7A&X}u2Uh_7g=;lfv1+)T>oQ9 z%DoaB#Ro2(OR972aWX607!VO|R?`iCu2?x#`38Ea^Gh;zcXM4Q{O3xs}mYma8mL z00-Yo25-{vkWjeNb5DBYPrEkZV_9za@zAs}1_nY;tN@R|SOGMH>XyDyi!3e{laMqm z+o(g4d%e=3Z7@=o)vH3uY`LH`=$cYhdjl>f9YBE zqi*?1kij_n>*uzI|vhLf~qW8C|NC?n}z)o+X%9wgqEmS?0l&?6yu+ znEUD_I5@r2l%0|HtTqJOi`q64-TGa{cXR>K>@{nWov*550Yg7JiXhIiOUgUQ62jGV z=SRQ5im${Q57#MADUeb4N93$G#jn2i?Y1m}5`x&r%W&8FHLv-_(^50brgJ^ckGWl( zB6qJ&H8GXG)8{YhPFbB|Oxj%Xt!Dz4M5>Pb>3pcClUf&IeQnEq=_yhZWJlX?ELe6Q z!jHxu?O(GhEl_|kpOeX>XL&C{c#>a@sMD!YwlP0yD^_oz4f&{WbC1=>M`R2~;!+9K zU<*3jqo57x&w~S{86+!VLE()hn2IgIC)Hcq-w>@NwD$cmAYL7nUwA_$_!AAJpT|9+l3whePtc$4yeDoQ(F|0MKB2q*GIaaf0_M40IcF2EL>C&;OJrqf zoTUvP<%Rgm?Ul-SJ{U6>?Yp({=Gn#yZX>TzhRjk zxKZg8&^9Y57_3Yfc3moUa0xp`SiRljQ0sgyrfNp(Z- zM6X4bW{S-+=)}8cOs)?V?WG7HaW?n1Km@q1Xj!4rsY>aTvyr3SvANusm1{P>fwy;j zqqEe=TCy&p%gs7gkg{jz{7fvP>8P7)Ld?mb{kfd{q{hN!L8_Eto*va9bN=W`OO2S; z&91bW`zcwu>~*OP5P(L%^n7*dTq@xSS$x^5xfGfs5Ujj8)vcwP&ckU=Jlx!?^6t*~ zfmJ@b)?Ooa;$+U9d10d>;ks7k#f z!)2^T@_5ram*#wXWZ|?B9Z=nVH|AY#k8N3Q+Qp>zz(hB>99#g8r_`RXiio0i;11U=dKc+hY-F`!OB&DL6t`n0PMPoWW$P0yGT% zEPHn=Fg64JkZe>RaXJB2vevv{>VX<*kU)xF#Pt$RS6h3_zpXhzPPxo=T~_wVEch*{ zXuGU@l*VF>%7Q_!2R=hkRHA+baNWb|=;-7Ni-_ogb!Yo%;v)fw<~=I|!`?mL?s58#Rz$i+xo4cl1@&+#!Y!s`o2U*+ zA=RM2$!ZE+08Q9Rj~RR*)s(i$pp{xSn08UhFRxg$$96Nh*um=%m)=;H>)pp*cG`{T z?*5jiuwGY4A*VJ4Y%NDcd`hP2#LChW+g!?Peph%rsMlT7T}$m#76|o3XSY>pKmR6v zogI~URoldiP*cKf+OUA-zi`Fl)ad#dzSKpk@Rm1L(l5X9UTGSGNOy+99|o-Lmx1pt z$a0!RGvZ2yLwc?9H;w6ul6sQE0I^6+dQJW$L}wsX!!5!!O|%cmF;sRwbzS?XosXf>N4D|>DFjBObl2_hh!pr-cgWo${) zn1Wt*04bcfNyF=3z4VbfRKSF;@^lFMEOM5Z-m*{P0HX7t2;_*_`|V5%eY>+_z? z%F|Mb3su|n1>Lwl9uX)^RM+YeVlG>3`GPlY-jnX?3#CdS`;FNR9-=&HX{$PZddg0$QUbeuF-atm^tnDW4M$ zgEZi_%spuzrx5#9?}JnWCuCi4VyZc4 zp^U&o!o{Q}s})WWjd&ETr^>9w(rs9h_0!UFl?Rjzo+`8kBz`;aTC$xo@*5#X+qq@A z*Y?)C43Wmtdf$k~Ai4F*TWroZLSppjSoyy-*7^F3e7L7%;VdF)qJ5aQbqbPP&E-Y8 z{><8&*vQmO|5g#~8Ug-?;8=aPN;{X}8y145MO_!s+#~wsa^m75 zh$z5FK{yYT8m&bo*RO@}j9E;4u^~C$nQzqy;Z6>1YU&bjEmi^qK@`!Dp}8Imu1h=O zB97Yy>Wu*!#5 z5;orpbOwY?JIP3UTB`IWc3kxt)5@hBf%DYbS<3I@OgqpJo%zJ}>eY$qH^T33T(kR@ zM!qQ@dHZXK-AXvmDZV^h*-ERXlwXYjQDl=px@x8L3-HQK7J2My5IN)%<&+cV8mQFl zm1LiZLCOuOj8K0sw9(|iH1wEykuqfvqJ)Bl)PGOZgy3e@uAe~$O!kq-)$3qUY6di@ zMGN>%f?aP*?Zc;f2SJ=`ENl3Tn*J9$k@g>XQqvy;*?v}eHF z!{!f=WnwV2hmIXfMf2g1^iy75K3^EDs2gJ>3Xu!7`ImFE+rNgv99Hlb@Th^t+dOoa zO+4zhBm!g4<2c#%q<$APbhLoOOdL#x3#I`n$yv}IT7tVZc6ZDMpi+KDW(ChAh8BQy9UKH=n-^>zF;GFt+Gjy#p$gbW6 zE!yzimY|baR(UA7LO8oqs;kbUwq+k&-#Xl^<(!WORmGCrrD|SqS#j`<9Q0xnKi6+W z$+8+ju8}StSG7#Bi%ez~^d%EQo{#vESRyurqc~!4bDYTAJ-UMsMr6Rla+Kow1o0xp zf+mjOLT1@7W=Bx))YriC3gu_iG#S+k~{&1Wxv ze0GdnK@m+uqz;je-(1&;)EJtLl@9!*I9Akq!y;S|@gUxF%)TBV^&5>AcZzC88$_Py ztAo#!{BHdQ?s&HVD&S{IocXL8 zy~#E=0Uvq!q-aJ{GfOE=q3A#zzgux^UL>Rulq|stgNTF%QejAg&2@s0Du9XUMFN2I zdsaVuSowGcFxF4l6mWCEh}JxFb)AkEw|9Ud{v6K_9{Vh5*2V5R0HjRvnPq*NgvYMV z2pOMEcRQ129Zv;=GY;s^Gj3osd%1}Z@O;2?K1z)mxhAiF6?J`Z5KNDNQ78{|cY3N1_K5dx>?&mC!|DaPBkdc}V(-G+0k zmW5kn0-RBdI0XAk=N!WQKWFb)f(mKi6*Ap!>~)pOqW)@#n*`XR10XHWANL(w_nb@zdVyfO)69F7-%M`W9Q{&ca!Tjs_#^V>;uhB0oJHG2 ziN~Ld8;FQ2%=}-6!T`9#7;%Z2qpIjKy4xwTL|(p(ZL8Hl%E$uT`YM>lVp)B*ju7yw zQc|kmKKe4?{PX4$@v27l2kMw+x*mEW#B+fDIM5A?Z=$>I`i5uueOa2?bGT>z4*i`x zq}d?S=c)S2{8X^Fv?i28lmlVwba&<`2>(cUbUw*bqr)2(-5}<-{Ix7%i{z?YK>j43 z4Y#}hwrV;{Q4rV+=vW|3YJ`2k0m&U3t->C+1Tlb&G1mCXK$I>UhbWzxNV&bvrSMN^ z)Xk%PMXMvm2lf{x%BRXYK^H`0bzYbD%oLmKlfxD%0H7Sfey+p<1JbJ>D|Nya$W95~ z0&{Z;vEDZ|{fGgFPi}9l9EbrlL-6!GK;5nw8plyux`gGkMvFnIdXryd>`f_=d@o?B z=d`9D^zE-YkU8j+b!%fxI`NJB0oswi%~`!;!Gc7M(Q5SK@CpH53#f~Z?(Vf;T#?rt zykrfV9@Mofdo=oU91=|MHd1li@lY@<;LJU*)T0%?zE4}X=CehK<-!>s(=i_r#xhg< z%gGAyKEY!X1JUpP?}-6SvfZcwmj9Zc1?m{zf0m;d>a=`s zdM)%Qu*+ouP4}pJU669bwtG1z)zt3%wA9&Y{6CCveHpO^Ml3W0sZfvO!R!l2YihkZ zoO|+Ov(G)91@0y_wSZ<&{I{-!{2Hi|#6n@`b)C+`F$7w3=Evi3wDg zib@Aa?{tU)kQZ|K&~SL-N>J&F3ZNEtONTp%gjH$Y$m8H+1(mckh&z_LV(}Q&Ji&^Y zo5>Bon$W4`sRw4T%KcZpy=xJ|)zaH>Yd@lZ*BA>l6!imKfYyP|+W+Co!TpZ&XgGCr zjs+yfD(dc0jhR_q0inDlyjZ<$-nLn)C^ghb@OGiIEi=IeoZBV(^5ypWTN!*4ynE=f zA2_=}>D_jjX7XkLcBaa=Fig;YVCqhqSsB_)-!mL8#*aS4A<=mXk+icm0@eWlp%wwt zDlOmW?qj(SfbB+Wz?m5cwXa6aaKQfL3{F@w;^Odz6wJQYQ6%5#7dl@@g*Ofjqu|;? z1BeCazILwt36j65uY+*7YxawcKIS7;J}F`R)}d%&BIR|gsKI`+fpI&o7kwE5hUuU7 zc!_kffvZGFA~x9`Qv!yeVO)SwY{oWg-6cSCUe`k)7b z9Al2%s-zLeF)XvoYK8-D!R_Fj4;7iZ+t(uzE46C7by4gJ5Iy2-(r(}x^T9}m|LFGRJ3YTHkz4Ul z&Cc6c@s3s$y4cx>D^6G@-)?XFf)s6_rDU3@D>|`Z-y?Dki_^u8 zpZcl?gV9B%byhaf^ZCIHq4km%cqB$F&O^oWPW!?J9mqVdr$w%igF99SjF3ItEnBax zYKrazWpOnv0jf4eZjLjI8?HY4TUq8_1ec{Y1X^NXzEtB(A0W)(gu&f$u-%eip4npv z%7+o{H|eLxr{2tK!Ve>j*b6ds{O?)DfJyN6&%}Q*{lA^H4`~qpweFCjLJ< zrpOyWG6{%$QgXUAV-3J&;VNM9y?93(_Sp6>A?fuT5Ir6U6oSoNNN>QK+O1e8QvL*R zmSbvaRzOOcQCVG6GYSHN-tpD|3OmGIR38JRut$9~Xy6+mt;stdSU8W@a}7!Dt2vXs zvd_rZX#g4p{qFU)1Y(hZ7;O>%fJxo#>+@?yY;=0W*Ia{|_6J{=id=11wGz|~)Wx=g z|I{-}>M_R6y`|#eSYb1*M2{=aVr^_7hnOYg@#yUvd zgkLU>_5=g^vA4mPhkYEDvKcKO z{@+kYxCL^Muy+re#5XmCL$=RPSe zbA3`H{-DPBNzF( zxK3*mXt7}BBcRMVS*ITJOav7Rz4KqHwZeiV!qI<@VLWTc~~xc_=ylxIQ38O4y(Pc~+kA@)=;CH;4@d8i4u5r>t4)lYm1b@pVF+26T5i z4Foep)%wZlh71C6GH|V8y6yuC`hNSpYT;13B>bNE@9|Ku<7 zH?IJa`mlQ5o?q?u{N>noHNodkuY8e3d z%XZRAvZNHQDxj@X^wbx)M!e%BQAdGsXZgFB@dv~2Gj{DJV(k_MCw;oK>hcs|Jc;oI znHaT4wd^9H*G0U6&dn>+Trjd6ele!=yXT}RZX~rdeQt#C2>Av1dSY35zO{C3<3ab% zZFacDo(Wpp(vOfBc&1$95yY%BuD(dlQw3nb_oq7^_8AoH`o6Lz=ZVIb=Uw^yIOq*E zkAES;`_dY=%YfyUE0Y+~?54etD!`ehpXf-M82ka{<)I@j4LfVUPHgqQ2QVwgLYF-n z8HGdUL-qZ!CX4qq)5Q&Bav#u(QCBr{_1e6D8~yQ;@{A1&zgWP;8;tgsvsUyv>f}cZ# zJUIknw@NOgg7MfY6yKX|0Md>OkO;9h^1l~-_Wc0rUp6w=^Fh9ssY#E`vV=Ce8IOA5 zfiLq5O!PIx5Pyu;N(>shDtS?d#PJ=0x)Bi(fE5qo6#DyXJ5GZKy?0QIM}bKMM!3NY zR;P5qI>a8#>jEj-%b#b(W>kLv^+)mV06KJ?JKsP4Z8=QxhNcmAN(}A>BxmZKK6Riq zd|K(#=%2P*(KO_5dzAcy^_is~(+?k>0D!bYy>Y)E>8}M>LE!bwu!X(VVG+CC#A2eM z>okJ`yN8OgjNI34`$=x@<^VgEO)TsX+$tpn+m1iDCKs}=V6S}pz}u$nq;PKaCv=%bq1E`0l@cWdbo_F5}j!Fl}h@{!%n<>`^EdHwH zwO)hHboYkm$F$_alz6HbBSQV?f`Tbd+c5^T6__dMTXqW{pn zU_(6=yU`%(G6H)!=0*R|nuk{5I{%{p6Vx-54!d@=SJP~x7#SLEe);)uZ}49$qhAES3sVK}PAMUf>x9$2 zX4CZfK@%}p6%Yh-=S@i|;@ZBwei7C4g+DOu1L30zAJ@;{>>ZCf`r7z4+T3J z&XR8y^SSg>7VQ~5l{p1wjYG0SZ55qs#nyhnur{S4&l z>d15-*yF|=>_V$r3DL}D4$}u+>*V7?>xWG^Al6r6=fzAM4Hcz}z)dDRlREi>&j-@} zjSrfpIZAFPhqVaS#>nZ#q#DmS3qUDz$OK1SshKb$DvP6T85c(z`E#3I(Cp`~-huQC z*IBUWh)!oanziyb51V<_EylYnyuvmZU;qMQZBH%grW{Cx7|Rc`~NiFLCjo zqpsCjMwExF+!=E|p*qOY; z%;~60#Q_Qh6K%mkM*?c-ys<)$H`|qN{q^yHj1zlVvej$a=);`()+|ojreDnek*GBQw~SA-&76;?IdVopDb;2NGnTU&v_vti>fCrQFSQC0fXF4gIW45uU)h0ImQ{sS0dL# z2%q-t2)@?3vg20oOmuO*Pw){Mlwm-sZe9trr4UNTVFFp*b{~{i;;YLd7>T*jze!ph zgTty!6mwDaro+lS;(;RJp*g*VXhM}%mAbSl`7UE=4Ecz#!&3DSBypg~*pswiIv=MF zHEn~4Gt@|!Nqde9j!HNzsO`{Sn^hTbpegdl+ns%X2ZuT(W139NFRm0 zlCzIC3giJJg}sW;#>05g7)C?9dSXb?`DkQuH%ajQW?IqdSC{xInm>B?T)H&h+Y1@z zm%V?#>Q6&7<=UD4F;eG+Tx8D{>`>Dr#HO3LBT=el%sP*fZlp;>L(o0hP3Z zU6DKXJVyjOaYUELi7zNtn_v1uiAzXMTq&rwZZ=o+LIZuC7f574ZEQT2mGcEEjxyl} zS`YMTGmyZ?swPxFGMyhU_@_mE31G4muY10hfK-#${nEB#c zHUs#AE_s98f3DHhi=OZPRn5-b%=}a~rm(xxH2;m^&U*?d)BHhl_m+P77h&#?x5$S> z6cI6`hX+!F$0$liw8hal0a|S9!ZA)QI0sk z4V4I-#<1XQ1-t@X{FN|u(8qEOW$FI@g*9)C&YnRGc$dAIXuW5pmm{brC>`sL_V(2Srq+>-fNQI#M`7pU3Q-1;NzI>N z@y7JnCxVuEeMjdcYnB>1VZIX&=|fQG+g}eoEFADDiml7}EKgr4v@V`W{TAYK|Vo0k#1bVz37#2m3Ynocib3M`>O6oR&4?NAI^=T#a+ zN8bQ#+Lcg>^K%q%DYpfgHm?b+N0y2L z7Fw^r`oQLq`=$PzP=SCaD(- z(rNCy4yFQRf%C|ezd*soh0(mlQ7$^y&#vI>Tl_UX5%hShkL~|FL|>^G;!2VK8icy~ zL!DgtlplY-lDG|Q3@b9&wml1P|tQBsSIlI@zOKYYiSfNkk7c3Nzv1De%348Fl|d1 zh+GZHy9%S2Om>-N^t`V>*#GE0ssexX)_4AD7Cn5p!!C>%P|eNn3bVYToT*B+FBz6U z5YPTpi&?AqJ)e`C+dCcW^+jgrKC160l)=~_L+~DHBI7x^3od!_&sju4-^g;gL7{gf z?RKRvLYk$1(1auWFcq@Smx1o$lNWT=H;7K;rkfeM##kF2e1WNgsujb@t-%A))#L!tO zJQddQUmpWNomWQw@ZoU&>C(3To??T@av-w5ivoVur+^;GdORc`F@p;`d^pUZ1B0pm zz!M2bwAIy$$e7DX?ap3Qpnu3ILc%xJK#dy48;4Jden@9&-@KR(Os}LJw&;>q~Ih||+s<~n|guCD}x<(DI@4hk8;HL*7CwG9wK_^;Y*&c(5W2 zU<%I-!!-VR|Ibyq$G$iuz8n90SrN^@xx(;GXKq2-h zeq$bK1IL4`gxD8ZS)~AWaTB{uOafXj@WVo>6yH66|Lkv0a1hW-1fdEbFhl%(i73FM zL#8L!p);BwGz`W;Aa#MkimYrUl&Ed2Im$d?`gGcE>yjJETFlhU@Brn$;#TodcN|EY$yq)kw%(?>dz~i1~LaX2*GBB2)1Y;C1rR9GUfY?56<4^OG=ir zp1K7k_qS?!H2H)o)m!mqr{ykCc8sSkys9YZBL#0`hvF4Lacb+d z)k)-GWPWLW=leDwQSzd!N8)aO?OTgXZVcMbWFko85u`h_?6z8Q_*7C3uX$Y3%M|`l zqd?Cyx#dN$sN#&gkn!+J3&OGKbQpu*pF6X`qN15oDgh|Mo5F)Md|ngB^Nz-S)U)D* zwG5Pg0>ibwFHEjZ4$gfrCY=+~)z|*;e610F&690M?t7}a==|GlMk2c-JT;_cmzlHL zp;X9NjT5=8{-jfZ3G_{lM$z#O>e#o4E>V8mcMGapyl(Vb6aF5FdG*{m_gy_`(%buP z>PK+8(a^zHvplyq*B^$p`#tuChTwALLAb(ekFdi>Gw@<@ zm_Ir8{liI{o<;_Ypn`ZYpCsp8u7C!g0gyRXwMQm5-%W*Y;?q>s_}s?HJgprN;pU#& zT@$H*W8(3I?cE`oQdACldoKsegM@b{7tn|Isr;!eolvJm_%$zsTaW>y_&-t{H zm%|)g=RapN{8}mE08Dw)1bruQ067q-s#}Y}lCS514Yb%;i6o8z0W(iWll?pAfhRdp z?6Cq)m0kP3*(I2+oL6~r@%(d(i(z_C4aLJ*6E9j9EE+`IF@XQ4F3|^=se`yt6i^Kw z1P?~{klb)Hq?)G|OS2gi%VE3ICF8Atf4tgPz;pE#D}@s#|0*Rgz*B!+ps4y*lLosfD+uZ8&>TrhjV5+&YYm7)<9UtoQoK|9!h}4C< z>~hpm1)#^$Ar-sr4y}!iCKaJ>ZeioBq;Iq$y^ThM*K45i((L#;AeX)@=-^(euQqG4UdFgiNeWYzA&g$Z${45jF-PaBQ*<^^H!ZJKn$`#k(&w3GiKRts zMN2=kE}T1Cc%wRO?Wt8kYJ@3$=A!Jh%)!pXB7PcWAlS^O!Xu4_T_hHD{=nbUqXMhz z3r@aIc~XuA{P;5%(vp%eq!k>Zme*KINe9_39+S&m@SBZ?WwqhDGneXpcItxee8=qG6nm0^sV>dkaRze)lgs5@Qx8oLZNGz)#^6{QOUY1;S)T6GZ=!TrI zA^Hy-sRZv_-#j=5bu49dR7s(NcAqThd zD*7Z3Zy*QM!0RjKfnZ6{7kC_!iVaI~^VKU$MYTy@f^hvbDEv}J+15$Gg*%~Wu?+|q zdJOLV7>$ftnY|-*c=@{lA1;b*GGcqGp?RajwtzZ-dVb7KAZe>8ipBA-;J_(JKO*82 zh$AbvZnZVVtx)L`xN=*eL~68m9{{cdv$*2i_N#0ny<)jc_pW&ddA114Rl7YmBbH0z6Np~K(tJk z@~$!SQhQK{S_6k8qrT!fivV{E!$Tp|^Cse-!=OQ~jyNl_!u{mI5pjATnM`h6*d{Tj zu^p5=0EEkjG&eVkojG4vE31oh5)G@DrdJ$Vd2Do0!z;mf$oRF%5X7EicdoT=Re6v< zZssMO(L0o39>A=)iR+Mctq~f~bqmZ=W4IS~{ox>|y>q8P(b|aOjnh2!>Jc zPIypQ><~$C30E+lHnk;GMMl0(TRKR!d-W*eX}a?%CADb&TwTg81ydiwGLs-3+8rr46wmu(y!-O+u2*h(4<(YxV^Vq?lHgy06B+DDA z0+$T@HC8ZM-PwNVkqP?!)Jp&~>tOkd&mCF$!70rUV*i~Y?qqE_&T#siW;Qg(y6yH_itI1^GpA<}RtKHkM@MLDh z2zSx8%;k+KhF*m=`N`B>uUT-VWR{9Xder!ID0^ z7L;uF7T?lW=D|HZ_VwKZ8_va}B~I1OU}|S`@OU=@xayCG%vxzYJYN*KMMM1jV=uLX zGC_dDbrcA05a0h(x7SWWw@6@~g5RlXYy09zX$;5!os-s6U_f|pfU(@iuLx7wYl-EO zg$G%!cV@c$`M_?%ESe8guc`;FEHa0^V$ued0{ul%CdR7f8EFG!HXjnx2+`HKq{(l) zGAQ!Mc;zhhi7L5z%LX~*z=!sxTo>_T_$-{4EtXA4w${Z;p ztj=GmqOh^CWtXPk$7AEmgI4Dz96K5}7wS42ERZHO9);Cfh1t44zWq7Mx@cS_ANs>Y zKI&7|Y3am>%|G!>3juzXIqZ>KPUal={ShNSQDfdFwmghfEZGvckQu7tQCND+M{Ro{ z_t4Ldx{N`W85!;3{xFBBN(aIS43YE4^T2VLdgz59M#~K?M}0%~pAAvE0{jdzYzIm< zdaZh&IyszgluXf1Hp5plg(0u2V!mDP-sfn=(*`jM25$kEo}C5(x;XRfyg=;hRkLm#zirW|iYGcT|_@%?-g! zy3u7o(kGVrn0K+q0pbOG{2rmIsxG0kI%w(AlBEm6Nr-Mtnr|4Y-{ z(ZE16g=8BSj!h%jq?leyPfC9#v6{w9`Uf$L580de&{nb5lCkPbtvReMu zS)t)rP7W9209Og_x*`#*|*vF&2?DH27=Tza2Ak=Uxqp>}qdhW)Pio!Rwj|5$|)Pa5upxAcV zY)mBbACI!YZq}SAm^w(m*K8{q2hLEoPELgMj_`FVB)T0)Eif|adUIV804V)EKIx8zmC7>}*;BjOR@rY?}z%Cm;7 zF!>&ZCE)Jv@pL*_Fg@xU(e38VbNBsg)B(RhD*741OUm@;{o?qjNkDaoT)OuQ_WM%D zK)pFhYmt1vg3WVq^_TwdUB3UxUz~|ZBL~)I0y?ZK?qghr%+uph zd={FSrnkES;!v?4(x2}!!!ab(2dso)qXQW+rw)w_mv`h->8orcB@}SMw-U&_)g_P@ zG=hmv9x#YGnR7}odDZo0ot4HPfw~S>f)TS&$5V&0lny5WMzc&&bGUgYd%U^&nwI}b z7r&XRdN_cBfj4ddfmrC`gT3_WJHdT3{8J{=hL7V-CX8^OYB2oB-VQ4PwX&jO=;Z>g z8O+0`A5p|3#bL5k9ssg3wW@{01l;=7;GwQ37q8iw5H~&VEZ@3{?!ssiyB0B9fLTDT z^gm6P?;V^@S^V!x0slCw(h3lytNsDdTJ39hQk%GL{Go zf1p(;n>F16R!!7S;K~G~R~7ESUingx_@SD1bWrQWW%~}ZwR2ga;^Z6g%=2JV>oYKo zX}<6~Q|75ylt0-1g%v}Mp@fPZc!@j~1ql{~8=yiC+Kco4Kv$3t0qp`vZ@ud4TY9Mq zytshr7^Bs}3)9zFp%hh+^ai^_(O^beWCHzC*4qT)Jb9& zf6*Sizcw2{d0bLq{fBk4V#1FM! ztmPYM>-Lb1Ni-o#+M(4I2WHBm~B*OI&srWh`_hf?3CJ#L4E8iv`XL;7L>l6pkiS ziXphAY68(!(SUA>7U2VSBDjWvZp!7t(pb`q=j@S})jP__P%~3hM((0UidPNX)|Z&@Zck;DLIWQw4WEqkRO{9*+>tenUEl#>~2AQUw45d2y-KfrZ(34!bN$ ze0_0O1Hwva|A>~z7n_~{JL4n=bE-eM+2=UkW%i^vS@+Ou8&GVOiy_vO6bHN?04oCx zinx&r5iv`5&p$ngEdi3)T$D}mgk3Ocz}`Kd`Df!~1m1$fn6YXWEh*}uDrOfDTR~_~ z*a&as-41*&+YFj+t|+g`3yK^TZTQITPwNqzTnVQm#bp_6O0r^b1>X#>vZZNgQ8OHhV~@9@-CcYH4Yq~)97931k5`pw$xI`R2#eAi3hZ>E(OplL zKo{d2I7sZ6y6Y)V1-m{39k=!&!0_&y*+Liq(|8AdF3e}bS4$)4bN`Hdy4NzeJvr~6 za2HsSC9db$Injcr@juNSo>ULM>Z_)yl+@w}T6WFlzR>W3r+r(_dPGvByl6C565ba%79w=9Pqw={pX+BGvDv#%L5KoVTCDdSjkdH z=Y~yjr!~Q(^}^CxJ7LNozR4T6-k&RLnotiOA@_fl@$?-NOhb77tYq&XZQ$_H0@i3f z)kF9rJE=obsxPVvO(>B>+O*il%(dRIIBZ+6v2jE;XMXrYoI?Kz>ygcOhvRQKwj>&w zx=5#g&LKoF0LYcq26+K=4k&N(`y1f!W-%L``1U@0@KaXFSi<4Jt!{1#0y@0d@Zfb8IXr46 zTQdlso)Wp33m}(yq#w|Fw1UjGKCt4QTsXiVKHaui#0@tay&zpX@?>9tna{YN)_y5` zQ}Bf+fLa#Y#C7p$imS_MTfQIB%Ys8p1z+b`@iU_F%Bx;$HT=;d;e%umzr6(un@gpH zuTvCzXCf0QJl6+_P!f|YxUqr{aXC^o0tFbhX1IGv5~F7f_Itq6x*C)t!g3D4I-`&b zwE2Mva3Ec&rl?t1%}B%Y2R&4<-}XgC3&wa`#Ag zJ5l|56=es`xDE_cZ90Ue#w!HP=bC**5F8hcV6bSR_DxF*__0D`$7(0hX8rDPlc8H4 z?m6ZoiidI`U^@!_guvF4mn#U#u1a+zA;%#U#w>8GU3={Wk5QiJPsTx5Gg zmF793HEZ!+S7hv1fckJWBGdzJA;Mh>b_9cC>~=pLqs1kd(?+XY4zpIYd46UGs$kxu zwpqj*O~@MRLnf9kyNf;~5RSJ>67M~YFDHA@j4Ne?3=?9JF@L451Y^%OX?K$InIPWs zIJuV;313eKJS!}T%>V`y zAXA%VNlaYpHU$En_GP}8%!?(v&BR>sYIq4zG#YzV5T59yqhsT<0~C&r6%y}yp^9>5 zM*$6LhJFNBdEx?~pk1ositF2=*shY>ZSzNf>%4}b+bqPona^2as+i^79#{gQH)2A4 zISMr(>^img2~!65rHm5hQTip8wC#}g2?ua1IS>qDz<|mis?u4!FM(V<9X-)hvNk3WFm+*(xFtl(o8j0j?xayhdFNN;?CX|ixhQ3c_vsa4Q_6$ZOV z^1)p*iq#d2UoMRRs$DjK^PJdV42qw&7>D+W<-3@Gi~HSK?^MuvY$&(pR;>Kf=Li|IK%Qzovj3 zfVP9I-MMypv4gRK9gzmY4aH%@AD{hxuwtu3Q>%DP48mzu|1fC2Cjf7g_Vu?CUUzJ0AsM#Cl2!GmmFFp!Xu$P%JFq}g zbaLXbj;_RXuu^?x_Ll|C^cF-5IBJeAGWMWykEIDJgo|ubkOIAXaBir*cYu3h?-P*0 zXx|a*64aNoPs~!jq8kahN$W}k0G33p7oYVLm+3u@j3pfzytn1>6|2aK4X>z(aKUn` z*8a!J;yC>N0+GTOTPK!L6*g+4pri$o zAcwfPt7u%}SvkF&@(sD9RSIq3YGN0=#y4G|2uUyQIJfIyXPt#oU8(t2AeeN~4smVq z|9oQX$CN?u$J^jds_5;#41|wWnq4PI#U@VjOUI{=EVc)v#w=8h;zWFUW?2TP9Zvfe zkCj=%bfO_8U^bDRGwTTMyqTGEV1ip>XQnQG5t-TvclpM}a9&3+@I|i$4yY90EYXi- zG-sjFD5_uQ1Blo#Wf1#Rz%-UjethK$V|7dg-z2Kum-4Wy2$^^)*0tk1B%MMZmB+oT zP*D_?o{!#RISdAS3}!SfaP7b&hS5=%F6-9UX{ViXw~4L9XTQ+>j?GI*pqNbbd<{@TLn~b zF1jAoSlQI0-TYX^{wm(rli-Ez+~JGTzlQAg$*vBGby8*KdB)LPS)wLuT8KujlKlEc zS)vZ_KpHmw-bVW8?Cv0k56=xlq5Nl%biFd}ieI}9o9^<_Uu!@=%q168h(KkkZ6X?p zJ&>WSWa-n1u@iq4N99hKCjXS3mV)o1&aUc5;WaDfNv+r%+3q(*>jEo)Oeh*-y~JN} z9}cU^7N92|G<@%i>N_P#bV-W3%lw7K|6aLw$syd;UocZ1x-4iJ55)+4kn^#;S|?Vl z*+>yveyGbX9Wpl~7=?#)4L6vim`|e&6dXTsvo`!!B97m*^b!oNDNn<&i_f^>hW@B* z7HSkd+ck{x1RRG&x7K!zAIz`}o=WHxfayR6%9JB?xC{tjkMcTuHs(r8X5uQ%aH8Ws@uuWF~ltWOEt&?+7Q)ECnsLx*6 z7XDnp{X=5_}CmJ}}*ws}!GUd!IN&qQJ1XPm@+A z6ll38_?T@B>0>t-rYwp}?NiM}+t!IQ7qgT(Sp@nZi01)JLK9E|>k`e>`fANKr`!); z<|WkrlkPBQJpfz%m*c8L-x2AaCcuCJ2;-othpMcO2C4KX#pDz2LDuOW9l`$txbKuG zlN`7WvEIEaWb0cV7^vovb#!!4XtYcG#<(UR>H^G#nXe!58XP8Iq!i>nC(K@Yj=V@$@1JkRb;*2iQ5G6n?%4 z3V%b~F+EbXjCb2P7ay;h+iRmVNv$)U0rGtDn>}L39#wDcL}8)T$1TAp70;*&Rqyq0 zMZoIUXo5vq8-(y0dA2_B za^TG(wqIG@E56kK^IbQi7)Qnf=_{OfJIVv*gXi>cd3BU<#i4~Z@G&{EyNrBRW^2cm zrlm5=z_UA;t>k?FQzvn~FG%*(Zx`T8GHkWI>)beOgm0d1*va4+TBM1Y=0p7S;0C94 zz8*gYfcD$^btE6NO7Qh%L8!Y+@hU8t&UpSoZ!XUaZthgh$t>$R>*zt-xD38#JA00p zU|pq6BZ_i_Xr7bLo=hdDBXtfRrZ5Lt;a|u?I@r{lFxd0u6t#hILc#nP6^FTE@F>8n zE1~DuKF0=D*=|_$`ON*l^sJ8eVr@1K>s}z4>Yfv0nyn(J)zdrvT-y@5c28+$mp}p5?vLy8Y!s#PZ*jTvT{Q3NPT^rX?^4vBV|6D+skkn8HMp>zVOwk% zR#9*Ix@W@zRYX*=i52prW)9}z6cuodD05VhXNo+>m zvG>U>l;Gf`i5M2*D&3U@5qF#MC88hYNXP zAojnPr~Yo4e+}FWO909neY_TTXs?8;`XBSG;d_~%ZZ*P&5&_7-xs=#Wng6L-V$eOT zqlC%l`$Z*HzuS6NB@2Pj%2vos0{}qHP0Fq;(*~1{7+3 zPy43c1D3~u{TE6G-QvZ=>$k1q5Tn0mY+$5qm8;?O+uI0)#9$E^VKZ_(!cK#xx6Qf6 zwZuZ{&TJ7tuQIeEZr*PR!+L-jV>2M-xWYEdKFu^#+aa`V@<&|&J~0BKIboPZ%>66M zS%AuE%__*AF{OVwWjMIU21I#fn_K9KUL5+}sPWM5=pm^G1@f&XZRD>BU%Bz{J6(XYn{#r!7=lE0 z00ZnFF*sVwvz*ysMUlQ8u)!e8YR`SnkT2*TOg;R8+%#`%!u|5lInC$lvkH*SaY3|d zm*sRjhh&i32TZY~2ZEb>( zrZu0+hGMD#h@t>pI0_6p)Cy|X<3I}VNucg=03I1w%xwM^dNjfTc@S(i9l(8KtDjl- z#I0625O@1KFmCt$o5qa!S1?#p@$*~T{#6yc8d=9Q*av5T!}eYsGf9}3)@Q=o_F+)i z;-t%v^0t$R`l)e)uaiUCCZS&~HhJA4tzg}(iQBy61!XlKR2DDh!9Ab>tdQ}(drnG9 zssaP#cCRlz9qr(?%jCX)#9y1v0P~OyM<_@z7Ly?#JrQq2QQJPoI6I}B3PY&hWi z^T6ce`AFRCC9^FO)NYS(Y&i(s$b(Gxq>ZSzP8{uS`|+r3i^N$%eQbDklC)1Dn3xQm z7w@MZ_=+a(Xy993vAn$9$t8K_&iIQW*3j2?Zou@!ZhgIr`zCKLm?3pIS}aNuU}$M;yESed$Yf*7VR| z$+-9d-Q12KX@`&RwG5r~1nDxWPWF4QUAW+wh_k{%`Jl~82Qg9E6xQ4iPe9oKjoBvWpiT!N=aKWsK@wjv~{7d!@%JLt@KiQE~^QK73y_eBt!0;?EL;UM3>K z+t>L>mBV=5m2KdgyDxsfO_}Mxt%imt#Y=z6)T2^+d>*+qovr}$c;Iw`KI>Ep8ZRJ8 z2h9(|M9Bk|E{V`_F-2dTxDD3ddN*3ABjex9W`N z>-^bz3N~|!NOh;C)5?|@9?m~-?fZLYI@-`3J8t)-cG&F!;*QFS z{$+2@`NESzqj{eDOc+d~~z7pHrc*1`>I2FWUnW(-|$Q_zaLvy!t9pKwb<`gYlts@Fa%u+OI(Am?+rHH^t(1 z?i?`XiRUu^d%c1k`gH3|P&CLA{>O#X?%BH`sDo_tJwo34in(kcX`+3`Lt%%^&8BN- z@LYG7j!UM$X*6ZEq>DmIVR}?0|69qWNh` zN|Zndj2EO_3(oGPjoyP!1rDG3&*>_UR zfx)~6IKGwOgHITjy<{flGJ^{qnGp7*6=HtDVmIO^Mka(-+7eh%CpcYn-UsmRQ`G0vrSRM z{PRtg_&YQVA-p~7a?(w=`_v=n*^EWxS$3zZ(`HMaKUPpZYeDgu@ z=;byp;-O1(eGxzJsh(-gtYqa!_N@K929YjblKntF+>8&YI+7ozP&JUy6$W(7kgC{E3N zfJ`Wpj>wNq9@)aA?CV0bKvMU9;y!a4*VB_aBkD2-%2=U!RWq!!FLz)DqCoxXx!W=t z>dHb(l0E|pNSa1Q$#{-9L}gX?U>ibb3#a-#+PduRW=0Gc@q|`$9l9No%-XIz8SOS- z(+8k#JX^3K@p-+lXWoWB0c!-y_P(uQ`+8x#97k9ix4$FFtVVjzO5ws&9E|%^vX$u1 z*ylz+?k1$)1>Gh}#e}Ig)3v%Mb4hk3$#64WsMTk_(4bUyGEz=cTN}bS-HosFymrg$ zw_93Uc{f`-JGFsmMeMUbcjwd-03DU)Z;IuC1cG2Dx<^w_P#qE`rdszGnTg!^B%4Ja z+z36IoYQE;f$zQ{MLoEXQ^F~e`!4)1gz)dH=z*-1~Ldb zPhB#|`ue%y2g$w{y7r=zHW+c$^?GWvuS?Bit_j6gGB|Rc{hIaQCrHU0I5ip$Pjs#{p zbkCa3EQvZ_IF=|O_WETOMx=v$kD=XO4c50Uo6tM}!a&Bl_w^!ZNdn+-N5#Zge2;nj zxb?0Crb;ZV=_!mcOAvG8(S+Fss<1!&poR->lL@|ZA(#R%(x`!A3ze)Vil<85SFQK> zoI?Y*{p+Rp?3yQ+V+SCr{&G{_rFtku8vR^V988h0`taNhwI@V0tIK%h!GJ!Hh(`tj zpM!@`^UKHR-v|N(C0JwrQ*$JM@q#FA-yaF4BHMPrNq=qlS2P6&ZSo-Sf`R=IR#YUG zIZsZ0u4a8g5JvKBC_>RVZ3a*Z+}%BlIf2BO(-HygGkY;?1+zYk-=Xvp$7NISS|4|1 z^Wu2(yr}aNt@$8=`R_^6$JqMQ;>HeSeuq>phZU&Yi(*=t2 zepcu|)TxItv_F5W>p~|n%zr%7UA8*gmz|t^u20`Fgt(#L)K$Sx(4W==T=pNHb$jqX zj9w{xFCI*ecL=2p@xv+eTQv5U|C zz0lUG`$}=cX)p+;ycWYWKH)yTCJH=VkSKc^A?FAaYr0o|EDOFdAG_v-$r*PY$O*-~ zU%R{xsb-bE+yU$tsG)JT0zhTTeoakcvc8`Ep1*7oYozrQJY@ z91Q21cTYs9QD$b>&&>@b=ls6x=g*&?`UN9}O^dI0X<-sz;(R#FnvbxKdH9eeR&hxJ z5$L9jfa7D{`6RcbgwNy0m!>qk#M?_ja8m56rcVJz6nxl)(=fkWj-84a0Zk8(GK8eI z%k;CDESMVjSinvm2TLE#VUPvr(#ZWf7z$G*0TVC>?W>hQSj#u1OMSmf9Vipd^yMeQ zk;5Aek+|;um2aO~S{|^TId3s{)^S1-CL%e{bO+Qa?I?Kii`bFMYoT+2>+D)&P%1=9 zo$vYzj#A&*lpI1nma9ibJY;=|LEHGg0hqA76WuT={X5PQ4 z&r&_aQ9hXKAQYeFCuL!H96xU~&JoD2SzzgJtt}|wSkIh6-Gw$F=U6m2k1zYgfcPU4 z&u|})(pD>#Wl9-ujhU?{Jod1m-VEaQsnd|tq|Fa5;%KhX_5eBGYVm`0RMZ~8ZOzJQ zTAIIP-1$YU^jIsyZP_2@rEciZ9mS~qAbE@<#pqmZF)-n~Ds^psGga{C9NYr-8ydp~PQJ(ja{d_;Gyl41XT4&x2^(J;S{b`j)6z`kz#FmI`$3ZdJNZ7~h z{pwGTBv-%!Ck?GE4LA>fW;^O{9`TXpJ)8P?6*olSv}3@ASO_DV?S__aUdORwUh%YU?- zVy32Dv;v$N$jXs*%!Va!(Ui#v^J_Xin`sv8JsteeEJ+J)Xd4{@CGHzSv}x{gazW8-ms6mjCpg0Sm_*__Z|)@) zQ&s#<8uA{T?|tdFe|&92X*Iw0yF87ceF+OWCW#UBY}D7+*V6_luiQ?C=}Ni}@z2^hcT5SKW zw^6f&dNAu*FnqFbHf_hxV#B?qDv1(E^Mx?o`opWBBe|g%%qe^)AL3rgq2KCss=@z2 zH}t-;hc%gGC5pMJ1ygcrUAW+Pp}C3>lqReReSU0Gz&F0|Yi?@;f#?0>4nn}%ob7`U z&H!;cN(9rpGa)kMkNpMeFu5}`*1>4zJw~$inegQ@2v9~Wej=RgzXzWWWBY%v;MxB9 z45I8)5e0}I78oZhC7S(GtLcU7dzZ%(2U;F9+3UbnQBD@p#EG(;x8JBd2v_A%Mf`NR zO&Dd$8LCfnKmrGUVPC(DHIav0WXF$hPvV7<0&k%`5u^{C)N9Jhv|wzi>gCI^aBUKa zofg&sQYV5V&?lKv{{D478z^Xenfbb<_l8F?Ni?&#>=-nhTAiJi1ejao_nOAgxdcZ)*qROEn0$43{8d$fv*FEHwJVH=uO&sUjARuO0WV5W+Q z<;sZ_waW%S*bal4QF_Fcym`JO*)?_hF6=jJ`%Ch>|Vd&Vu=;H{o&ICjOZ)6cMmo9$as)E@I1l z;Ha8n2%!<|&^UAEO!xOyVAztQml9=^^oxqBx0` z7)ysDzaysaKF%h) za_Ayjkrk$&xPspK_wk~#|E~yZMH--*#D~QNbY4%UyO*F(FBQk_5ErygAJXcqhc1v; zqAxed4Z^v|@bKhgKSl`WA7+@r1o$klg$lfw7nQf~gtGoiqUr!KdL@`bR8DY!+BA## z@V;D~LMfmERKi|7wY3+2(4keA9ONcGUu@p~7GfgB)up8yx%!oALDDbdu424g#$BrU zuYC~fpPy)ab_GQ7f8SB~u@3eolWVm{FiGv>6|r~z*oBx$^Ws0Ca18bcU0&GCO)k{h zTkxvzsv@WzWSfle`5R3QNY#DcNPspj?1hyzQD?j4EB;0$?!(ZaD`j6^O{c~Wo>x{q zUAgpA?T(~p_zdXnTQ%@D!Ee!OPUXC}FGpG#(3mgSUj`z^HNSpt{NU)4`t$Y~pz2rR zS^Osc4rAg=?1w~<*%;6}cg`D;&D3)X3spgmPD6~oeCzhb8-xzpJMYF!g+Kc1`q~Ts zb^t*Vr!7=PG2(;FO|gWcGy(0{_;b@UCrj)4W^oYi4hm^5 z!{@fIS=H*drM-AC$?*&;D^Dp6HMNF;f!*)D84J_HVE~j<^T`;#{;%NwE$0VH<(Ups zFwN@XvteQa_N{DK6wYDQSg0t(#Y(FH=uJcJj4mRnMk!th+BrjbaPsoL>5vNT&;LEJ^)1vXBD8^E`Y~+ILWgjL1fW9n+N9j- z=ZK=&4es+8i}301E>AwcJ9xo13#;zb&B- zfoD$&US2^&6tog(>?_Uz`fpP-58Xl7EX{1P;6^stUgw*5stnQTSVi3}zv%j#E(3M@ zU+tUig2O!!_M(mlKP*qyfb2#rnZEs1BWTHX2d9lQ*enfGdRv&xFE4I_f61_;E^rWf+fmE1P zKX_cT^*&6&1D^mW!7i`;MpXWYm|qHc#_!!p1f`uQZNp5s$mHfS5ZktdM4c$00q!9+ zx`zp}Ok_dQY+K%=`V0q0?_7VeQYl0s`n-}?epimtqwo3WAsLLj_;EnH)z}4dTyDdfF)R2Lcn97$ zU*0(njt~nAza@L&8ZR(iR)4hROUck46=oP80LpFxG>f*7Qgl=Z31Xbg|8nLhV+`G= zk5YYnJ@zj{3b;k>05`jI(AD?JH`Y2gK!s2nL4G2(rd|wfuTi$>>1KrK6VNKn6 z04mz$;yI|CoZ3RPWh{u}>Xm1@GKJUE9luejxLQ=igF31q;p*Kz`%0f9##UfCeG~=( zZEa=ud%)GQfgecIwX@4bbA~QFRmB^aZnGGJv{C4RwGKFV4Ce0Aux&X%C9_*|Xs^+q zY-)f0mtUv1xw!56Fc>>edGA!nvzc1w!8juAF&i)doGgQFTA=N z6Q6e?y63|{Ahpnd04xrh7LI-Y?v8GtOek$3%jL;W%j=5s{|3_HLEY0mclYd% z`>{|4-oFO0->riOIW=gxv~Urtd*HG(iFWV&9g(TKyV0>3!i{-lF1v0R07%~-4+Bio zHx#-3`=B#jmZK=#<;B~tU;pD>e@N%@R8{kw3G#otHkehz9Gjva(t)UkCOE zN>HFLC#$fqu)T@U>#LTIj-_=>bo4(cUKEH}{APFX;Am)QpgE)y^1vF3le4!G8Jg~2 z=aBwoV-xQjT&k5;k>;vixqT^e4&uN&nFc)!0Tj^~LSr*>s`yCyMLDg&c{jsq1YpX% zS$j8LaP)s{=MIwWKaZt&eb6PvX9u|PdsO3a;It`A+HKANeiw7-Y>;PqR<1yk6S^~L zm%jR=w0&+LCPd42z-EG*2vzn(` ztiu!g(8%En9`9j~ff3kfAVO<2yE@G5!NZ;~CVOV$@wHw&koivd8@L)#GNQ{{u%}8- zwl-(`V?h;E1?P!X#G%vy;OU>e2QWie7$olmGClLdb!Rn2!IS}Gh*2q)r}3pJx`W5b zztM`2-h*R?sjAs|Y*O>-B*fW7jS)nz4iib&ZBhcwF_f8Hj)`2j_172QhZ<^fa?Pm^ zADYVY++X||bD~=5Mn7@o-0P#iTn7)V(+aF73~xHV`{Ya&pS|5?dvSO(cKo|z;E5MW z!80Hd?Zsd-?J_NUN-ub3nbm)d#&rkj~|T>fO-pH$JHZ+ zMa9LM5JqG{ur?EG@uZ7fTR#~J-p%&qS3Yk-#EOqK&L;yDa|Yp|TdA)>6oe)Qu0tSi z8uM7<4qPCbUkQ?(&gmaDx87P5UtJu&(%9IDryF6g-Q+jvzUvyH4O6Z? zVp2)vbT-IT%{kmR>*aM#{@0r1q1h5~myX9>I?tCZ$(Af#B)+izgP`-rl=N6#M=PlenEdad}aZEY3p6i!8c z|J-oU`SZxR_E_G4MleI}FkMPsMMaQZ8ph70fI9nrb@U0Qvf`D~DcE$9(BP-p-rO`h z_)GoxAt-6{nyz5|_?{asTNopMvgI?26gX=%pe_W-jCvLYkza|^v>LE@cPV*-(R8x2 zva*WfwFpH%_hjTS$CJl^P~RAr2PG{=Xpx{`M)?(1o&1|lEi_b1dB?0%7-+MF_rMWl zrWwn!zn{%RtG(*_>!VzDcd0(ZY21Y)S;40|Du(Q3`(Z(AH)7@svtyd_Zs(sT(d!iZ zfQK9Y*_*v1fqyhKCMC@02C+m;NoiwuVD?KNmYdHGu~ZCgnqjf@-sfi6Z#vZ;;RUEr zNpkGwCUv6CkeL3ZOG(@&uPtG}LQD+2X}A3)Xz|n8mD~)2TDVVo1ygLc=-`t zi;f;R*0<(T1~DRM@rX$(pc-k`Fj_T^%joWJ*xt;g+5kLHF+^4p>z|HvmqE_SN<|Db zaQldM^J11{=i0h=jx1zFTlwmf1(+?Z2^vFC9goS<)Y3v454})TH!UB;mqKiI5fEcb zth!e~HzM77brpobaN4MN5^9I6`_&*r!vNce!&-zpyzJXfogQwz!97K79F(f;Yb-*4 zfBKv2FKyqC;vZ|I#7=ASz1Hla62#d>EIoGtd&2rzL}SD;Bc|^}KuD+4_AQE)aQ_fh z|Ib>Zf4xY`o{HF_OysbO2KN=0lVi(bRhjQN^CAhHJOs|i8N}y(sie<-YWaO-da?%d z_2Fstp1M1~G(>rM)m?|2WAXq-KV@=}ffPEAZFDPdoD5dlHP!P^D)k$X;DC6jw9hd6 zD6ctW8*m4vVKKD$s7sH>mj2~Xiu-NjL|yaT*B0!uj)^$QUVHw!nXc!<@mE6Jz3gXzprX4bq2a5@~wLw%gmJCbS?EPEmiAW z$i-E?CGapf+8cC9e_!@_UA7a`(kyxTXBu?12a+t-dS|rt&YBe!X@!dMT@PscFM>Rtpo%3ut-U+6U5mt|{Q zH2AY54ws8CY+tP^u4td_uQ3B%9WwpXLDeTCSSjnwstxQI_oYz zUk?3vo#XS~B*)V%EDwh41#vpJ5&KnAFBXn9m`m)Z;zx`!p;HheXnA-*&#A~0)S~HA zpR@%hj_Ae4#_Co$7)1{E_vT;8H>h^OH)t9{Z3tn?StqxU4`f*J8IyYvLtQuWsky5b zPyGBCzNCKb@m-w=cNZ@#maiL9&E*f+^}UKGCrCJipjG$%uiq(i@h0RLs7!;33G`v<&0oVLzRMy!5iC~r(oV@+ zfN{+L(O?^#HPi)qH>EJ~9)>}w*Fez?L8~zK6jm*|bCS1|F+30me_OHfh#tBlfG+^A zFIV>&$_GHEV4yqcoDZK_3z>Q@pqJU&`C2^9Ql;n`MYQhRnG_w~jj4)cn7HgzP$jGX zSjB&3>8I|dx!yCw^)y4hGYwfJ%sHXGPc;IL4Co&1Ju}=hD>%ryy5y~ECvjBGz;_Jr zhW{Q8_BlJZzs8hA-h2J1<-GoG0*vn1O_sEl*N-6nrw8{w#LuQs1J>gy#_}TPVtHpD zt;(cFt@0DG%dj&2o=_v`s<=L;%l{RiS!hrWvwq>VL#YhZ0B_QkwDSBIqDP7a-sz9cn)QSCgwAQO1Y6@s@fjT=y zuoYzRS-H9R5V}U+Z9HmbxUMd2S!X`K+oo3Og662QokNaaPSeV~4%7n9+;AVM^yxlMi>t zwe3-&Ol=drL~9V4dl3;VCjq zlEFJ$0jTbjH&1Fp7}Q;8(uAkIV3N{Vl_zPrF_s78eS*JgbWwHt7ZHfp7VD}kW_HOm z`LG^r%nkLUU^&>h+Vv&r$p`hPO|Dut=P&!P(wJy9ZBMx*nY$|US^wbWIjI7~Dw@>* zxr`Twm5j4avnk4=Rp_ggmI*B^_45Wng|OB_KTmHzfyEn>Me2a?D=z(JOBrIko?XFA-48pH6$DrXa*$;zRL&Qb`;x6bAZ#H~T<`CM1MwktFo0fhz* z9v&VieCIhZ4_adYuV@p-hrA-nceV^0J9k}5?w*u=9xe>2C#$npKjl%|u^r45csU$a zRWu+gMbmozdqWSx81L$XWKKNX%qmL4-;1N5^(??#K+1`tEWLR@~7R6v&h(>;Q3$k zZ}G(K;+FY0e2mdy6c;xDL%yv?-^mq1+$!Kg#<98oO5gLK?#GgX&0S|qKE6bBZYye$ zRELNUYJS@G_Ia7Lu8W4bdgUEp=e@r2I89vw#J*W0J1?&{j^E}VaUs}YrKge)db>-)lasCrP3O65Zk3#?8!~)Gxz=Z`XZG~v^*oF1 z%YyBG_ZNikBm=sT2-4xOJGN{YV?pK5(Y$6|KSJy_=a0MYMc-;7SaMYsz1hN5^}+EA zB&=F4E@heEx6Udhh?FWVHi3r6*SUU}FzOlUmvM6XX_&-YfMYZ-U+zy^+mB(_#FM8` zr4YsUk}4xE)uy zU!oqW%orlV6mT%&tbss4*{<8{}t4bM^ErzcuJ6XYo#9er#P#tVrZs2;bwP%EwBYrQm zxkDVr&!!GZ8%~wu9S9e#jvf1@gUaA2*ou#jcd4nnf1enX91Nf`cQJC$P)b(@0+#?^ zE|BE+BEfGdZ7nUxlYZmuDn`Zqd^WvoXiGMi0sNu}3}Q904;~V&YtVf!P=`nDrgDBa z9I3N|R0@2iPAcjrEPV-ZANy=SDwaL*CWFa0-*pCn39`wsfN<6#Wtpb3NY$p ztQOu^7WKEqun0Mfcp*Fju>9bNQaWU)|E+gRwkjq{7(f^yfU~x-Sp|mTX==0jb5IA* ztDK--o$hYZ7_4&6Mvai%hwwO&mKO27;Y+Q0xVDx-8uK1Qs zDb$2hamiSS=;m`nruu72b<6U#0%jQGPr31Jw>M7CWfRrg*9K#mDJz!Qu!F)GfGURy zi7eUpU;}5FUC^F!M_Pa~eb2z9!r{yf8B>7Nfj%@83I!h}!J*XkDj0jm5n0c`Ck~# z^5}0s9RwxQ+<_#40CfuL-Cp*uMjb4HNqD2OttkKv@ncqbf~XyU20SsAJvEsHDH2u7 z?R&hkQ1Mqk(#w0xX7!`DNS^jx0aTw0CY)KB`=C3Yxo>&x{I)Ci;%GB>-u4>mWTNY7 z5-CD`COfR!PHA)Ie3aZ*AHcqgV$6b9qR$dl;ypP zP@o@(18GPg_Oal#iJc*s0VNdrc&tpRLf-Ye29`;1kUj^L0`;IGs{z$qEO@l)72`L< zzv6&Mww1;ujRQIclbBqfGbs*{d9lDaxTeH{k*t0L++ciB4CJoBXt7Y6bs3{8n75Ek zYek71J8{QViN@1kOx3Pv3(Pbd@Vk#ccBeb|Iw#cEeolugO}jR;FS6Qt#o6rX$9Tf+ zjtM*Nu*ZG-Y9t*11{qL*kuAq9w3}1KX^eEgOpcXzy`ggkt2?dyc!4S|Q*Mt$KTv0c z#HHNbp%rSD!@#kfACAB>z$G|vRcP)aMLq$R10Or%Rl08cUW;aU%X!#i?KC4}gGPRK zb~oA>h>lRSf9%hVK?lmBq6-!0Ibz(5y1}O-%n7$7IOd{dGav57+HgIaF_|!vxT*B3tmK_{mvbR? zfN#lW+m3}yo?Le~w|L{>SCM#Kgg|(13YT?6H zAejL3tPoG)pv1SNtw@9<%1Vp7ODB2?|Itwg@ z_{LoEK;50X3(XygcsZuzL&4Y3U7F%SR1}mC`x`x)vFXj}g0dRE0DQ2|z%>DnM5>i=vCryT-+l{tnk7%=6{a8d5q=~bpdr<{TC69N&hPNVMk=ceOu!<^+q zmY#%30XGH@w9hQ|!JA9MLbUJYq3YO$Tr%7=I5k(UN}WJsH6c$$^+UG_;63VG7#YeMiX3qIQIiz7Y~;Ob%rgz7)JLR=tY1)TUsVXSQ9d> z(4o+Rg5a_VClSQn5Kd55_W;B?D<|i-30biJeRjaf($9B=3P_ha-N}tUmeU<{Xp$yn zS%~!B?D#Hc-Ksh*;QeYfZQl+rgdHyYd1yi&?yh)81eq>lTfVrz5f%00*)AW(B0iyp z{vRZg2YH1WG6`kOt!(pT{Ec+z2*0}&Cc^AKRTyS%S>#p9gh;5+Hk_`Zs%msj1T%zq z*qH`uYQ|fYhHn5}bfdxqDEd5_0l6*I&jc|nn|!akL}EVTsp>vJF&?I47+AkqtsK>D z8vFgEwj|;oAm?kx&dI3)6c-*GLwF$Schq4s->OayUA~rY5X%wGkMUX#pDxObLFY#& ziGi%U#mCv*{#vg5&fE3@9obBWx*fscEL}}trFP5No@Y8;q;1}AQ!pjrz&u4Ic|Bv# zc{hlh+i4yqU*6n=4Us*Di3Z^#v@f2YI|@eS#M=(gIU#ii0YQFCY|^Pg)Z#S^^Opyn zyA=bZx@Q{W#0>M$(p;33LQo`Og|2f%4`l7jhdrxbhtdt24x%nD=mt$!xXtnLMx|+S zENA^8)F^>ZRwfq#^+Q^x%y(-9E0muhKxwg6(iH^v&B5;&j|~Oe?g*&(w!shOINokeZ+mhLwU{ zH`OMuGObyvx64uko%W^iR;Wj@%kNDRebd<)(>ji)wfw%&bR^XVwL^rW#o2BE2vlO; zPJQxl!mUCmS|er*zeFqA$5`@yJocasF`fXfZheBL2FS|L&WZ$`OG;yTEm(n~uCeiB zxwF)!j{}K!g6ZkFI9`FWMuWGO6P@&oU`J+KE^c~imkMa}-K%?(bVgo&K>y49*f)6l zXW4ebC|2%W+~W2fgGmy04zI9gNVs)Q9Lhpj4f}8xK58|2UHEUWEQt`qUZHzE%f**n zcV(HtjAh8GT!cjT&A;aLYNX#CCM#e>qNsHh$S^t&NF-ga7WZpFq4>WL4buj96H(ux zh?Uh`29#IJ^`-H9jm8vAHo|{^&@cRe9tS{bG61KUwJ-`81|f89S&OF=5H^f6HL{nNtq4ZG+vDhu9q^oRJ&a@+Up?N4* z{4jQT!`fNwU1iuN{zhT&-k`gadsECFY>I&a-liZ&i;tQu zp`s5#`UOE4@f+D$=Cb%9cQWygVMs^_#WS0~0OpsdS^DeXwU`DY!*5QT-&bK&atz+8 ze-=8{%vE8?|_^yg`P0L-PIZJ7r_rVpCb6&)W}8-)e}&o6eJ#KmuPGKZow){Lakq3%98*@^nwgJ zh^H&9!c>7-C>pCCKUBcdUy1ly{}$0P0~hN&H~B0&;!mxDG%ivHy(AeYir;4@@1j{X^yffkv-kYlH^`)5mBFZ?p4YYNtWs3l7Lu$+2d688*+tU^ zKo-P$4BxgI7h#RH^gp%OViRNV(>2_999P`3W)gwJafs2(gl9Bz)_8dCLO$|HxA?{&DB6nDpd z=}yvnCs*Nx7Hy8Y2cNz3cA;Z0dcUV#=vV#o@Y+wNz@$m~@#Sd@_6e)h^~S^cNe9xY z{FAR&(xcADBg^9G*u)c_fBWjkwPQ`7PzQ&mc}VjgBb#5N1uaY^?9?GMklpsRa@Muo z?ZfB-GJ>~+prc=kXGCG*2Z1B%1np+PEp6Ax2V@O#g?@ay9vqPnyA`Va6VKZrgp|_2 zDLj*6?`~AP1{A;shKmn81j;%z=Ecz_t^oH0<{Kis4jxrcyZY_%FgBM+z zXE3|=O-UN`%m{gzk@Q@^W0~2u8a&PX$N#x}^smU>_L=Rkp<%_9flU47%pnZ6{UlQY zaU;<2mEVjNOvg#p(Nl##Gm7N^1IsSJ*8Pm|qB(k3jD%!%VILS9D0}7+9=mYMHREY%x|^no;j=h93R z9i?s&&L5KhZgJbevw=ch+| z!Mx(($X<@hPlrL>;;n}0jZcBVR~7$lO6-A}e{m8#@D5DEz$4o7_b$;s(!&ac=B`8Q*SS5wHDDVr0dLpS`!qr>)%5u#PW%fhr<%-G{fQ z)dt+{gSV|VeIUuGE5vl(qdQ)!$-5DOL6fL13{gH1dYNmUlaQwgqwb6Z$ViiKb2^%8 zyD}ZZL6`DY(>K;^E{9La1i~(NFafAPw~$Hza|C!Se+Fy1O`{G|WrOzJ&;}l7dkc_#L{mQ*+51Brr0KWX zO@;0pL{&KKCK)BA4b%$KbO95C$QV!AM1*!VMNob^v2L z;OviN`rDt+L36lZe=U$BUP6%hQ2kDYTZfbrR_BshKVOrB-Y ze7DM395~*rn_?eEE1!Qz3*O}?l}yr!zkhT_KfZlIN{pEl5SKw43Z9e<#g1rDzdUCB zuD~d50N6ze3-qoZE-LqdmiKd*g9T$bEYOuoc9PZ;zyS`ID8=|elp%W)WGAs z@qm~dwL3@&@*E$^91Jj>S}=-*FG8B-ZB~NZiRa4GW781|99Bi_hxQ_ANYq-0=fuOR zoJMT|VUQ$R^8IdMu@wK!qJ%upu4o=AK)qP|7 zK|4UKHL#dLjGJbHy6dq?h}PTBQw2q!{2vdAc@Crle?{bj-0L%pxC>F)M4$4TaI=)~Z%v&nmu1U}X9 zi*>y|3J1x{DTaciy+^!(ho7zcB*$9dvXLy(!-Y3ssT2>$x;As&z3#UmTQYfCNek@Xumrz*yr)ba;KRP z!#l`v`}-5%TZb`IB(EpkggEzOYE5LTSURDFu|$m_wg>7OHBL#&^;CB#cMRwPbgw`- zQ<9wHT_?VMh4e9c%Ld!`5hh+qQwZO3qMAh=>)KWJJyn?B;*vi-T|%y0O^rN+xcYpY z?M9Yr*YeSO;Juw$=HyrY@^R#t_FkUdD(e(+I>YCWUAAe@+@y5BJlHF z`M?{gxcM_sk;hc`jj!CuPXkIy#}2VYw?yG*1^)TSkUK0Q zx`W}bM@m{wlG@}PSzsXjX|YAM^IsfDjU)qU45vnXzJxFA)ef}dln#q`@S_@I9Qqhbjp2rz-dhl8uD zU?wLA$7=|`16Nj72s+Q5;!R+YeWu3$O(Y0^SE+rxyfBIhBIG;0r`H&u_xj$$hh)rV z7o?>_84m5`y8cm`g^VcjI~@FFF_o?5Km763IbW%db9{k;@vKrO&K1yf`YBzw5k!4< zh*ipSPt*Pr?%1U>PT!`rNFx+*|JVK@{Xdr~OFB0o)Y0<@srV(4^SJexsdJxWg18y< zy(vZAO>BL4whx^=^v>`;{mz{jcuGa1TDmFYduD|5h^Q#L`>Qc?0tE^W|#PGG9UJ z0n7S|z@Sc_Bfr!Te~(jSE@)_w0j<_dExmiX001#EGBTmlXBZf6kO>M3Dn1cAy7Sy> z`H2Gry9Dgtze)7L?U;LVTqC_mxC8bwnKd#5n^JU5ghDzi;q8|^U z6#ahHGK1!>sj)@+g#*WBIbLHg2|j<PU}T`jY~8pkTZNJ=Wcn}Ly$`|1l!WEC`Y zLABTKNc~}SMO_}j4l&1NJL<|AQe@N%^zZNWd)06C;DBz;4pH^i0>K*EAE&9pmycrR zlsKC@dUz~X#aOp??0?3{cGPlx%Oq_3do43bYRe4C?THfm6|v01B(*~Pz{p;Ylg)P0 zuhyIAX4U62l}fSc!G}C{nd!Xi*zY)aGugt6m}p=5X?}ZPzr2Rlzp#vq4BbT_L_mb& z`2rS9$mmcPqvY{J$Hm0nr*iP~*BKobJ^x76kCYHFYC^-X9PYIx`983ru9%f*$o*^Kn@_O?|HnOUsLMUF$ z?XB=W{B4IRs&M$3FPV!z2%I|6BYKIzTM|sjn|nkF^s|Q*9E9uM_GcdI66cuIHa0Nc z$Liz`W@_7Fpr<_<*FW9N?kC$tY3YN}msnl_Bkp)i ztJ2Kc!cZ`@iR0}RT&j|RjSA&A`3q&cf&%FoYV#OMzB*lZp*Fn`dofJ&OQY=RZXr=} zhN9!o6n)?&OZQYl0ulbd`|mH+rzf3UcK)@5@S8PH6AGMQxIsk1b|EB0^0;J@{{5fz zH`?rG{U}9lnwVD5=-!k)eppCUbSEYpV>x}I{*8m+b8J10K z?cTWq3UGdY`zc=z57+f>T)%$(8#GTpc=Tw>cP?=1+`fYc1y5U8Se(z*68-lO9to0u z1OhwmU*|5*)&RK3C?G)LUNh$D>advF#v0XPWcPd3PO4>+A&)1O%#~T^FuD__Rg9+Q z1O~oQeLpnBZS|HZQUo?AR6>(dQeMhk-r8g8r(lnrxioyhtCNy9g9{(Sbjqv3*Yo+po@ z&2*5q$hxtyqpQ=GsF!*z`3n-V5^AJX-{iRqD@6^bKLuy9i+ABv2o6$)7ppVgsCe;0 z61hEYZtl_#(DvSNkNivlkVGv@b&=;gz|O?9TgqtF!@vHjBmi+oc(=Cs4KZ&?FR1XE zV3W?3eh_tCpasJ`RgmcM#xyPR+36_xfD#4#9qu7>b9Xle@sUHpji*+j>AW7go3_nbgZsWi2*IgS5p*48s>I?wYlf4?~;5oIF;HYJ?*;g=r@N`b+$UIK<{ z_w6S_&z7v!2h%<0=8*E{s8zeEu&!w+vlLC#)u?^%Iw{ffcl9)#Woxge%oi-jPWlDQ zj?AqDQpba>pXz$b4TrR%vJ&V13tFaiicI%GNjc#G9VtIkko4W8q~noQJv|!CX6Luo z!vVG@j1&xnzLFmIMPRDvlvfkeWkuVvSso2WAUGx-YPjc`H|3d z@%tcz3THe(FGe*wI(o`k(Ej}k@Wlxaw>AsE8G|<_Y)`8l;bCwJ1iP!xF^vp!2wapq zX`Lg_;xFI2=t0WQFQ2<3`(tppMv99I>)&TEel^t zsDdcO@6yE-?9zy~d2pMX=0dF5rKpV2nT4HU7-5E|yH2b<6cH6al6e1eSW4Rs%nPhV z_0@Nd2FKo21v|Ecdx{@><|fY)%z7gF-pB%{jASbF&Zo~$9Ns_M^Q5Ta@a27eZ>rUI zYRci5g7Szczxh@&KKD7MKV(4t(S$EB$5-mpDe|XGD*K!(2{`Nz-`rDmf~$qbvt4W< zyZxRT|J!Sy7&)KqCO5ODA|`nA-e81g!5iyR`@rq}aIF;=)q_OcdvmX($SlsNUQY9W zrAZu{PF9~=LrOM&AUVT+a4En~G~9`U27|TJS5Vw+GFgkF!G&DH{CF13x%=Q%Y;9W` z53Oi)Xj|Kjhf!9Cs90pAeF?v>9rV*+wg0?&R#r~#g5|KMFe__`<>1XS-_;Wuo!AN1#ONVcti7nE5n1KCb1V>F)RyJexi4NQAKvG65P@D*F zu%ApJGyz7LkjM9u!uJsIx5e?d25p7ur2IKh^*4czckHy&snz=W`bXlo1m3@Yzfe67 z0|{?KhtVnLI~#+;$D;)Dh7-4JmbfL5h#En*Kye0g4v;(8doXNcqxLu{DL)DH(|rTf zHCZk5zB5DkUZ|=|Ub^()2rnZtC@E=gu5yhu=^@m8G)xV$IuK z-t5n5%RaPK?7Y4FNg&c$V39u1f-=nIoe1r;lr4jDUR_YnnM$sng;N->V8#2<-1Uz> zYaP4!t+o9%?5R}!jz*k^>jMX#UHw6R6oW_h?y(#aBnuPA?O?w}AUYK(6rCm&+)$_A z;5WbTjS71yRr+V=3mfO!xxUQTx|BY4=cgo$F?ikn)XIEK0dwfyZc7|9na`7H-#1f| zkL|RgZ5+lNSnX-$&wY$#dq}CJb3e+hB5C7D?&o*vbZqvmGTw&QUbrs19AA~sliR#f zuHx?+c&lcRvsv?FaPwnT)imy2w>{OPA@4E;VmyimZ;Z_fE;w#FPB^_-8ZYY<7Ik_5 zdSfy1MW%YJ`S9AxL*t3IvTsvQn?~jh%4c0J_)$=DPK?a;v&D-)4k@-Mc33vw;Z!xV z`Qhp}$AkeV-?N+37}(qxuREKw+<}cR37<4PeFL6~M1o;##1aD0N>=tF4PDdI*+0j# zoWfQl1EnxOc}VYm`j~Zxhx><_aiE`)$w)^^3eoNLabXj;h4-O>_g6}6`enZK*Vyz) zwms^i$krZy70zg-Q@JE!-f6!$R}x3-C*=)&1EUZjzJuA8&C)}iNuW~sg+2CE9yCq09)APgw7!ra^#dMR!K z^v-uMl~fJZ6H)Suf(G~>N(qpHboW7GVi>ftP|u>=2z2<`9t$4i;Na*` z2n`IJY3%Mk0=g_OAhHi}7D*t1)74bmDbbo3g5R;K343?%Bqnxg_&#_N<#QSJ#6;J} zhHd$zD3@6a&Xl(GTOA7DHFe-cnW!0d#+$>FHsJ4AU3PA6_n)@=jW7wlV&mDL*LPwL zk&b?mRlK3xqjQp(k|s2#UlKD)sIqb zOj51Hy)FANi<@0}_w6V5RH?svISzY|r?uecqRjOT+vs+kQu$z-05xt?Bg&iK`kv$* z!t~ZS?xuA8P-ZhkP_v^vG;OrEi|>`9bqMo^sRQBCI_W=X-p&`tzWW?z!lGC%&ZT|X za}UQtMM%)KwbG;2Ya7Au>b@N&cAwn{Fx1AKz25Ye>#9BTefQ=iaYeIAf@8#byX>BI zOnl!VshRS+i@eXlUT^2Ax-b2mrZ_Z6f-L_4+E)YxbQ)!s*3MmtkdQsN(; zBo&swd9y<9@-wv)XU`t<^z>A2*}D4~qrj%3b)0CcZ;3jStic+x3KcS)DyP#=ZZFC0 zCLrjo1R#oXubXftzfymX+!Qt$QW2Zm@gy*K}Mh zRVtM?QHy3}3Y?<#Dj@N^^sVK1^j_76nxT3!-48sUR)r~FyXRAsdbn8PXb%g4Q=K#0 zfis4~2Xm|3d0r>6Tq~J5va|EJpzQczjfYv`ks7|Dl#UM~HEgx31ER0g*Di9#_4t*o zU3lzi^qJ^}Pd$drEPAxJ(8=_D&df2N9YQN(QbQ1#zNBbjKi2D*1T6)f+WVh%syBnS>KwW->!3_X zJ}~KHj_{svd?mHx?hw;n7hbvowQF2S(z!C{p5@J4@gp&#_E|IpdvXpUE zfUt-yxAE#g(-;Wg++PLkL8tb%uz5#364%3JDtSv{Vio`zG)8FSXUD_9jMGQS7_O1s*-*iCjAs=v=e~D2PRcJ=t=^g}h#A^6`(Hrs zjg^qQ(6*C+o?bTP`q2+SD!;trq>FUt3N+aR)3tSHMPCDNX6O;q{Zx8iz9Q?03PnTteH;t2B?I=VTEbsF?{w)<$oW3{?s>=QXn%^9R{NMT#vTb_ubeD2G5=i;e9*?!afN* zXok=Fzsd7ew6;yYo^|`ud#tp{tY=NTT2K-`yZ&KHmHEyux+PK20+I3?d?8z3ES9uMGMHCIlC! zK8OA3z1#VIL-?eGcp8#;Uq!s9<#M$sDtiN!8gOjVaZ=Ey+Dl~l$C!Y07QrS2-ou?f z1TlUf!t?aZMQU6?x^viZ3_@^gBJf`vpP;t)PG|m|xoa;2gB@N?7zH$N!7(F3_#SUO zkv{bmF)%#X**m&S8!ZIptqQkqW4i{<*S2_2{pGz#ON+Vpnv?C~#ipCQ3Ij=J@Nog| z1xu+5?b!BKK~Ygk;L5*Ri0Je1FkPSY7Aa6zgTUC<%-# zcKnYOHx^bo_}4jzx|MzZBzF<&FDpYc8ul4f0$e@?pM)f3Ql@>cYYahAp zgSCgpYYRG1FhR~)KH0^!^u7M8aj0P-c>0@$H4eheXeknX;>q&$ozewmnWs`H$>?Vl z>WDxMw&vs0*c(q30}8vo$mGt87d8wg%}xXhTKm(5c&?Wx-|EPE>GV>0eLC7F#+@j^ z=uY36;+1`TvRirI&8Ck!-fC*E$jPfs>a1h#)_5WH40}tHiNa)t2W`4%ChkFIh?#jQ z;I5eXau)i9dCB*em9{iS4$$T8V_$(w_|VLN7=*N!t#_ z`Mec16QBq7Q`a+>|2A+#dq)OFH!oY-3ZBWc_Y_E6ZjrG3vSX;5jv~mO;le zf`q;@^GvN`y(y3qb^tw{l9knQ)5ya^j8?!X26*|fB|4+s_B@p+^W(>lL7W{{mX?q( z``PYaDaFF)>FL>>E=32;fV`4YRGCrXpecx=po!q3WM;Pa^Z`_rLNi_|H(UMk-^CeL&>>?l zcDSiM5*Ac=9%3p=+WR0R;tLKwf-vMuI`&#-Q^j7(y$^yTSYE&bN|L}5|KnD#|8pf{ zA9`;`q^BePKGpx+eQ*5?9rmA26$4~Kc)c{u@v%8{QlFcG9>2J&K(60$ur*o(K#PQg z5)_5wz}3@>RpoGH zK-X5|nuk9+s`O>XG=`Nm^vhk<@CpjVaOvFCk6eR4vmYMneF%f`7l0;pZX1eQvqt3l zaWdQ9c+Xkx-@18|%Ejf)`iApuq(|fYu)z77Q>_vU-d?qgMC3W_mviwiU$)sl>Y_S# zF62S_@zUVxOjj*bEJW}l&hHT76V_#!DayI6F#h^@X89a7iSw?V*pDAdY}98cX=k4| z2e(em4L>Ec{V*^Xz+GRof7863?k$JnnaD!=Nj8#-Wd>5Z$y>pNkL``izuRK8I&_FVcOp${be zahIY1<8qOpy9}?(-TNtFO_o>O#hRm^a``IlJ-%s^PF*@I7#E5V&m#1Oi}vX#^Wu4Q zohrB94!QhsC!EU1-VC1{>DWGoCK`6p&AqwA+ejJ8V3E(QdF$f&8>?4B9c!doqz-Pe z7mvw$xM1#^x6|MvLQHYq2Z$gX$m@*q&|icbX`}GxC9>jsEKE5o+gBZX z?9w#&o4+L14PVLZI32)C^qdHr+Pif;`%;9|Pps~N{dg{4iNcQqKETJa3cj_p8bAM1 zg0oKUj4It-{qbbyZbM4+@qju&cW?XW*)(gK#K)PlR(z*sHTUcvY3T%woTd>XB68Vd z0?H{M?-yxjh+fX<{XBYg0}&WVYeX-YcZvFe#X#ZxO6Iw#T;)v10=rZ^n zUlgqz(8&-ly(H=O(H{-&=ikAt-u12uA1=aoca(w8U}V&^9))7c43Pa|IZ5%O($;GP zg<3qkNC_cjXvJ64cxJ{1u%W1d0rm5A+U(q4$Yi|nN0)t+2W*&23_9C-QVG8FYTeBJ z(bC#VfP>?*y-2hC!e;`xwPL>d#byY<9v#oCwpaX5ayp))(zlf@+P=|A7osTQyLzoqsKFsZN{(QdE zwzcH7YhNGg=$sJ{xRN63+VNI?%E{G7@LcMjSsv#D7k=t<(eQt9f^o2yy?GAT;r5vz zq)&f_$z|bkQmzN)7lq52+S=lYN!?nH57N8adEyzTx1Ok}T^3=K`Qsvl3r)}Wmb2f# zpS-d>AFD1kwC|jGk4goco4hCfZ!Gl}ely9R2PAvhG`L7$x)#F{p|B04=|9lx-q~px&*x-&| zP>@?WxwV%#6&J7^RgGRHdwDnF%NJIsxd&$uzt#fQg7+qutjaNv)xgh5B!RCsX4x)x zG)b$tej_emyRAEUg|F%orLR%pYHvDq`3^$R$%LMqCGD{=s_wd<TMghTc(7SI? zGv-OZ-gLA8VQYtCSgot)@4U>C!7FqPlYyE3by2X_C^%L8YiPpq0q}??RbbERNn3Ln@M!EZ zIK2|jSbwytcDz3=dY(3g{Ljq7mA3KPjCPkE%7uq$TBVJ;wg>XTL%+UdRp@O1)?GC+G|^{%AELKoqZhv1 z$x|8&ixj;z&&5CCj*SdaICw2ReD)byg&ps1(`}4eS#@X31%Bw;T`Zi~*zB74^_%_E zPa_wXvd%eaClf|OGMliobe}dB3yHdSqaSiN^{ZBQSY?uhWMOhV8Fsqt;Xh*ZC%)QunXX^g+zhpY($1{5^&2rVVM#=qdajBSfdFLv z=gIl)^v6fXuG&u;Nu(N!7~dH1(L8R^_gj2XX-}|_J95~0<#XoUa3d>0FI8^pw-bR* zF-NunPYLTr?Uw!=rV1WUhNvbt?68gAgo?FjyCJ(Dlb_}9<4Rwsuyn>0nbi^tBC#8oRid{gbm+Cu*Ix6-t& zxTAIrkAWTkG|W}t@taxA`}hTXYEpcx6af;)sXlCT9#n{nXeDDkbt9c1`^?PtO25u; zA9gAy9lu{9Ry{7R9$b0y;PIU+&{)Or9=%d~GJ8ypz|BHH$o_6`aQ;xm*4A@uOiqfx zLT#{${u)a7IK^w=iny%^jh@}%ucw;+xF)~H{4OTwpdO55t2=%?Jeg`Oh=?m~R819R zBWY~B7QF#sJLOHDN|_%yB?&>&wcNi4260igky5+l&$~x??}f{HVT=@8dbE8;-a3C} z>!x8clIyB~HV{(Dr{ORpHyAHK17#c_gYS0~_R~&IKsC~R$>H(VEItfc$m5l4g{tiJ zS+SS)p!8cM4URL6*|`A7tuJFr_nPWsyl@ z_7~oK{m@4P*?$7bl?}y@S$DBMflm1>XO+G3=g_kf-}JA`;^Ufpd11MaI$e5{=Od5! z(#_YJdE$mh6>GayC@NEZ2EPen_WouU+x++_`nO;0)o_YD>|uO+1Wc$hfiT`vKf0; zldhM=x78d8St!}opB%~~n&NrAt_R3on5ZL=bYqU=fk|#s61~EmJEqXifs9$mOHNjn zC7SkX3-Eg>X=rA0_#*R5o&^W9QAfV*Ux;xB6DaTp&OKNDuk7^x$U%JuIGfKw`8fqj z3(mEb(Q+bKq(U!dm(-I<)n)!3_2?l_Pe6F^zj&`k$IKiH8*P|mya#l&b#)_w@$$I& z8G(~)IRdK#_8QMX)$XJ^Ff)7QyVxBy0(swEw!PJu*2}(fz;eo`QmT z0wA$~g>4Kxl6sM%XCuZbt{nBnulxTg<0Z<-V8Rq8#XkduyFw(B~_($+x- zF;Pb)wTgx@w%0UG_^OdFgWq9M%R;}dCAt(;CsAdk|-}NVK5MBWFl75 zQuLm!P6RXuoVb#QhSIp}4Y71Q&&}2>ybPD5RM@uL62|+;Lr8Z27YD!L*D^+JR__L! z{`dB6UEoKRq09|6i7#e{0A`on?FO~4!XLK|O%EGSuh*Kn}IV|$)fjO((UHNQ*QJzc}cCrORA z8!d*{Yfm^ye(L{(IXl?mHl|e4R6kFao3v4qObk7KLv2>OMBnkiPGIW=qj4c` zg@-_%mSzV}J+qYc!=A?B{Gz?11}L|umUQ)t8NZ-Df$K3ZJhuF(b!h%2qi~ zWSoV29U>KTL${<507~Z=cC`-0ss5bU?YUIYj*>6KIGHL!w7*ZYr4QQI6-S`4hQeoWGDyh+Bkcn7x!Inv z(8zPu0}G>eTeD9qtjp-ZNrT+Wi>I(myL5I6;_M~i^6}N(s|On$yfAJ!i;aqdjST|N zSu-PdS1>BSPu9cH>ythm&Z`DhZ9aMkLHag zs#H=c2r~UV2!~7c184^qNJr`=ScHUlH>TsXi`vCFp|e+1=_Xm8@ab6e$3hgruA z><84@c6F73 zB64}W_MBcZX77o@A8FGvdKh+dx)&5YM_K*{obmvs6gafugZGI3QUk0tGXs}{Z%aL= zEOQ{$#}d$rPVJKZ6Wqkizfe{x&2a#$sMH!@>=#^^)Q=P)1m4Ey$%3OKPK9l!$op4rso$IEaQQnf z|3B@%&41|6RyLUioAsETrD?!oK6(<25n^MtK@s8=*x!oj{kjZSti|w(tk>4JA zo~P0LwLT@XJn}N+%^P)<$Eu@a-J6>o09E03oBwrIT>K0K zpN|&G?EFUEI0yq&MqCiy@UQXOykApnm9{dhes0v*D;oh2OkkOr{zdi{y!d!YSmqa5 zY3q11yCCI@%|x9(y$Cl`pB$@w<$Xm*E7t)VnfEHAQBXcZ|92})r1b?`59RqkVJ$5r zSnQRK?xmr9aycNeDrq$Y@N8y=Hv*Z=+AD5!JojJUO?X$K5v*TLf^E_Gje8r`+((yS z>+5rPUl535eIK*M0WrwVH=YzQ!ZrN(Xm{+JO<@(tTEu@%O+C9E9X4!Z^{dIL!hQ)` zz#|aK2?ot0Wmi|ePJxUs``tSmBnf^Kv}`eJp<#8|7RmzoFcmk~pa81w6wZbpsY;H9 zjiCQ=DM(}y{QT@O%wxDv{xo%T(CF$a*-bLV30r+aAfVsenq9eBYSc_juBBsFUZD|- zgEvGkCxzJGuQ7#}I6u%yaWe}sTr%?*-k>i%#VIdVzQ#*S?~d4{No;ZW`kE)Ch&j(c zWP~lm9MChiw2TYEFy?2~PLKk7%g)k)PT6nC&CL%#QGgW%98b$4_b}}oQ*NB({7Bs3 zNknbp`SfscMUyvlq~3Vfrb(PF9Ix>x6%xvIKHzSN6F3Q*^Y>+GX4hFgq)85s z4!!&6nrDAaw>|+ayDidRkU)EH5(j}_wcQe!C-t+T=xXi(PtoC+e7@2E<-HJVxqOBc z_ucSlm1oZ?s0}7x$F!jP5@{?LkN1BV)vx!{?<;zz*bB_K@DF@gbam>=kSFij@t^29 z<@Yo`dd`(~7dy-&Uiw9}MoC~m!w)#^%S82Osfs@yzR)5;Zt3{Q6?Wwqb=|V4ZP#&_y!tImf@bhK z7w_`2C*qsN{KrKBqK9e4Avd^h6w0ziM4bsMM4pAwIW|d{bDamnMH?Bd(-q2D0mdi7 zr!vv}Ca1rZIo)9%X+0r8Dx0NzP2=qn-QO|h!qG6y`+E!gqszX;33X8me6CAJi{xC3!GKGN@2!rH$rK}r+CNPWd56_))>;q(KoTW zy3o+;sZZ+Ut;vQyI}$ivLX-NqntS17_e+T^dwxC#JQfFmM;P*WdwzC(FYyu#J;tiJ ze*amUV}>3y(dZ|Uu`z>Xx3#VI5gb@7>h}FR&idACd3GvxKmpHjUHj;2^=f0%u`2<; ze!a!$c)NdEY|Nl;n*=PE#9z-PS@w(zS*o0^cruMgK|u$B;Wtd&nIzvtg`nfM4iD$E zUt>$2b{BLb+enrDyPl&$j}_l1t3OgMb3`+np&3b=kZUZr07coF zb%*{>U7hMq%!r9gjEL>*bJ%a}rZkm$;RI*>VNDGSy`;!HEvcX&mkNlM z#aeMvo=P!0Y-}W7%xBluI8d3v%$Q~WLpRO@{sxjtJtBZpRgx$}=jRash34cXLO=7= z%ntZ9(;cTa^(dl1e#2uSQw0vaiuDi`&O5rvgbNE6HBW_z!pU<}q`M@4zF?ce*7S&v zen+{=LQy zsTgSMLFotnh!Z-Q2?=4~;9&A2=N=_7{)6rp5fQ72rKJKeD+&kVHsg{ABbk`pNFcaW z=^r2b!(6^MNFLmE7AnPMAlnz`R!|_jJa9kef$VPVy<23wV)T! z$>4_zN#c%l{)Y>UAvcc*hkHA(baW?=#h&v+B9SH=zp0XB8|;fmC}ij3w2emNpfE!q zhKiA)kaT_(6*`-!a);-sN0t*f-U#KeX{Ay-+oI zJ@Q!Y)C(8tdj`;&+4+!_rT0iUv*RJK887xi<_&aXmWQ8_PfpI8U!7c@Z$GIMy{$V% zdl%A~U)#dFK3?-OF5vWtkAwPU>t@X`5o;PgqJBq9p#Icf?zR2;Vxf%FssSyh%E&Xo zF%eCkXv3B_SBiz)tQCr@X!)08p#C66U`vhVWfuV2N4p;{{NQaLhOB!ln%1a&^J}~r z0zpLa-emokd;4K9QQoC%Kl)euyoTFjH@+>AMLZ{OI^3D2q?xhbJV|xjy(X~>I6cigRXMM^nd#)p0 zzybCgJn2)+4$J;}>&XkH`*CBQlMAs^t;5qTp-c$PqZ9P$B+<&)(deMRnX}U<4OHwE zt8`HdtfYbC{ce;WrUcyDQp72yc1QUTH`;q}nRL7vE*Z^ht+^}byzfYPf75E%B}J*f zX7ioW8yc+}Q!6&^wLmScIxbwKKTjNZ#&Iv`uIY*NQ@LKd=i%~>fq?#G9QIM-Pn4r5 z0va!P_<9j`Xrl#Z%MXI5b!wiz8NJ80_& z900nuT?-e(v{K4^2sIq5UrI_CIhr8Qc=H5-C_9x6K8U_^W7+iN@NK~W`|ehM`=@Xe z)AzXQkF_j5myN&N(UlJ^8!vJVoai!|@FW5{n1hje_lYlYisxAouYeKP@7FiCM3;}( zbivyryQ6(CZYsV~^#;4>se9vv@JG%jotMpYKfEvjWQ|hh>#>y>J!y>}=UJiz!z$i6 zL*vX_&RP;+LlQC+5kp>47J`8vP3&ar`gYxiBa~2S+l|?&n&!O$RJCpI;-L6jbxjJ5 z!*1bpy9Z8}UMD-?#Veil^rTDm8Ze;17MPB%$?c|ltLD?Tp*NqE7&2F5u1a|DF~i}! zmQ2s9{k-{eWU2{?Qt4ihqm z4KH4x@s?Y*F|I`ioE%8b_O3)U2dCW;cN>|fq#xW-yHnv9Ay-Sj z-8I!_)T!UlYTr>a5f{}3pRqsXC+n@m_yoz5wxR%2Tp3fGm$B zIP1{!ET~1W{2{bn}N5Ld#lPA#K#XZsoJag_{jQd)nPxq?3XayGePfJ_uOgTA1H`8)mUjx(?>2W_y zgXfdfDGhIl8M^Vj*+IW5(eKRDWeRQWt#gBNdj=vWm^JzK%{V!u+c$4!87}<*{HN@< zb9ctXr0EHC3`u2a$-2++h}5rE?Om*f?Tw#TCmwyPcFZ^2*gfnrCeM2NwnYcxao800 zS(8;vos*&dD>LuByS+3wVqgQ$_jI+E9|C#DMe4zm6otK2`H@;z>YN{oIGiV&iMK4r zQiSt@g&q_RnjLHwiVX+Ms&GQXCYnIUzicoZ_mB&Y^wiYq2r7UP_ys-f^?OEg+bb7L z`APvdHTA)EMx&n=`$h}31;reEab;yIxvaK;I{veATBU04XJxh1q1*e3+S83V!;OQv z;EVA?vJQTJ79ipv4QW}FPt_`}d!juS8!7L2^37B?k|R@saqr8d{lVU=#G8}!-FFJ ztwj6R8zZlof34~t(Kje_pMTp|QAK~0NN;BNO+0O?-1X~-j=j^DC zh$oQ)vvROjv2gnnAQF0U&_vI!Hl-^|AU;$L#>5dewSC=rUqXIr?(M%OE&XHFg3KEa z|0Yhxp(%HYXJJF%Ss+U%^-~4UX&X~)K2mb>ev~%4>&7VKkC|84!MradK8s=vomacs z-HtX?>`F4>+jq$6@m(|c+FY=sIj0_4%6kX(V`BE2yYrv9!gfra<{IVe>F-nnTB`vt zzBBc`N_|Gq(MU0M<>096{?~i=+GyGDPc+&yCMM~a54*%RcfKARUI1Fb{gQIBnN`0f zP<*yPis){~Ug`VG_y__5n*Cwa16{kV(tz0JQq%*O3-lEEhXKW5`KKs@nwn92T`8*3 zL=-)ji4Oky^@Hn6m{ZO6OmF;9p14S>e?sEWm(dI+5Yj24TOWT`NyNwNbtjdPp?x2v z4+Bu}(QE5z;oeZHSZqh4F}XZYAhgC}gUqsl;pIM_|#*pD5KV0>MUJg80%`p&g));M3?SSmdSC8DKRqcQ5?@`%vr z$SII9%-Tr0_R8JCx`PkOTntdjpNKjDnxKQM@>Y+}$uYGh*o$Bq6MT5ODB__ei#pk<6k4b-5Va^|fAUc2x>UIB zFw?1__rE89O3>LhKRagwTF`u|g~hq3i?s)3w)4$b`sd5U%IJCQr|A_VBN!=ihXN}j z|Np2PgFi`2;r`dCfl8N#PCKwQygz2fDsv?Unmk@*jr)A)a;q1HD^?cqW;0(GBZpwa zx76Xz*~;4SVPgu>2SffaUG`Y6u^&EUd<8l6lZAHd3|w*Ra$1Qo_RXnKYL98HBPqj6ADKdRU+5;M=MLHF0l)Fx!4v8smhMBY%=0?r%77) zm>B2e4QJSq zA%Mect&WZ9@;sH&POqTmiKZnBzVXmCIhuCbc5~OGt%Ft0QuMvOzA@BImVEk>C5*_j zfK`IXc>gzb_ty7q2j>LS@5G%QJGa;{3F0iy83m~$fhmnU#dT3Gv4a*+u}*c*e%N}} zGE9?gwBTTs;4WRO5lT#-X^e~X^EONx4_ou&mX;P>C zFXj(|3DeK>Qe*p<$R5&arxclCaU-`_|2H*JzZm`y7NmYQA8m=b2Cx*1!T6IS#=$2# z9N3!w>Av>;?IdSm)-L0p(R(cVk#UNptsU9JQ><~tEvluv6^jL&22YPR?N=J2pwvhus1!Fl#N9TXQ+{aaz_;PIFlWJgy6|4L)*E2i8s z>Elp`%I7mjxr6f`7cgdm$FhWDNyBCrw_fNO=aX*&vg+tO*bx6&Gvtipj7-?@L3arLtVuE|X998a4aNozml>T#7p28+IoT zVo?2Gz3OH$MR4&L`%8Sb86ycjD|_nucA^$544U_sOQ0b+lW_IInXHEVdEkkNhD zZe>o?h40@?6iYV`3JR*6jEjrw0Gr$J{r#s{Rcuua64~0{-~U&nepBS{VgH_;4JysQ zDxpQ2!=r5=zVhi;k%DP3=gzg{?wlg^0!C@YzhUNGxV(wU#rG=7xIstGgUY*DOdlsQcGq96=Kc(vAO}6THJ|cKPyo=+r+VBERfKGyy#&pQi_7T>W3A zdbC>6kQ7L-ZE~UpHRmEpnETHCPhI$*KeyLE$+SrScVtSgwAKD}&+Fv)!7Sd|W5@9@ zvg)^5O9%vj#NZb=Q8I%?fbDdI&Aj?Dq&%(;zrd0!_4EqLq_NrYbuV7iGlKLLJ2|>U zfjP#m(uSedk{RH-Lw{hxH#VjXxLgIpr9|a#wdJTX-|0JzqcsqARyrF>fOs7Kz)CRT zbhp}w+_(diBHIw#sj!kY7ja>g!Ixd|3P zz*B1nz=OFNUm zc5%xBD6`f9a%)2xY9GWJwGuRbSD%854#)z9b}%RVM_Ia0U57>LkpY~O5$VrxR6Ro; zYWQ81g8AeF$D#>s_uT%)vf1KR?w-PoUmw?EJfIcVUaPXDCLypM%|3jpgiNp!gLq{> zo`7)La^m^@JP!}iB6;}u)1wJ&zZ_8l7ap5|gFH7-m)$%w_b( zO)S7^T!%J_0Lv6cuIk!ECEFcI^SzAvZm-g{=+1RM>taKN*7ywiRcxst4fS@)`1=R7 zI<6+27JoQs@DO@CXXLl@X~}KJfDP6UEau`O$)Pa5Wn9p;gjWV6>NyJPqs%*LUVm!X zLtnb1@NPEkI?OO}c;aWam=Ol`Nx^6vyF8?lZQ3Ko!mmojoig5lvQpShh^`eB>DGLd zMivm@J*I2Rz=*tsJNB-?lVFgujK z`%`9QpBhvAoMu(3+G-4uuVP9SW1|%AMzmOtWh+*H&F$3tx#>M7Y32>n@?dVXT%AXyA1jH5Oy8vYZw1R+xf)zkQ-i}VLqgc+E^kJh zAtr{iau#f6*7VlEL);98L(sW!Rp=F;`*exj7C46YzXA;V;A_a=`X%4_$xso)Y;XG6 z(3WkhByjB0sj_3ZCR`Z6!kL(QQV+Y1sU;0p(8B*_lxp_HdX+7(S4w5kO_F)2#4!39 z?iiX2)LQMGXf!L<9P(beUU54t_T6Xx8->_Er55T$u(3H{tsEGGn%NwjXk3-(tbQz# zjkqDtOYr!6Mn@ENsV#h5@VHVz+n#mt-H&^`=gVs^oAC-Mp2J0xO_vhH6GUPq#()Q}B+-`>YkJbPvru$tm``8Kk-dZ%)1G_Y>x$#mw~8d z@!^Q4FGw{j7MV@?^3dJklQ?*9w^Iiv_w`DJO?uGeSw$arcZm?`T5)7+%(tDjyo&ck zXcCAVe}ahM&%_lN{H$Jx;=iOnAZ)w!OwX%zLe)WPeNP%om;Dk`C{ueo_mlSxw}z?@ zr47%<*IW9`zS-NIH!!E3J~C=1i8*;hp>paJc<%zTQG%jU=}}&vb7^sAnnFt_shLDY zy1<==@dhJ%z2B=`!j1gBYALa5y!VNshM(*pJ{)dSC@Wt2piLa|Zm~2gv}&uZ-HCchUUK|XUy(t-DD%j z=(v-;u89(yDwZXGvR8s_7@>7VP}0wYc4REif7@OAkvw`Iw8DU|g_SYUb9X>HVoQ{Q z{Y~CmUKwlCsaY4F?=B|3p``_at#R_;yZ5h(cFJ%lJeC+~z7rEs!Mi$n z1o0r#qq|W}ySaI=&oN;)yZxObnsQ7YwKR0KY5(xcOA(lau|S_$#P@yOMwSRb z_v=Eh72`mP4+1hFVA<-O^TcZcOvq-P?L;DJhFND*u2V#!Rs4 z=+2a5m6es{+<|cnY}HZ(qzQ3@^Yx&fR<6rxU+2GnneqzwT;Q>kGyp~%MMe#bG&Hj) zq>_R{tg)M$NS)h)sd{2^awHgwOJ4cMC$Ncw^CA)kIL782`Tu2`aU7b4jC0|6aiV+_T6-=iELF?TAjK z2zc^{b5OI1Fg6Uk*)Zmb*wZFnx@6`sYyEch#If*Ce!hj(DZ+-=L=$=rwU8MTKda(-`<3Y zyinunm^l8**jQ#4rY_U(oFFQoy^SX24btt*N2$2pbc}5!u;J|885CKdaCi~x|M7gG zmW|pgL-JCS+YLi#Tbm1MarYHwi$C~D^JY;R_l-w2Z%Jj_p9b}^e^}ejOvyFJox-L^G z=bc{HecwC*X}qgzsp*O$isKt@X=5n}#6ueG-#3@-L8#rnI&Sk5)8myk32mHDIQRN? z;w?6hg7j2Tf+x8($NJRjC4bm~0{W&cK3~J;?XVupjl6dn_EnSPb-5&G-k)pHID_?6%deb^v zDxQqJNYxiDC=L%l>yL5AC8TihFkYdd)iRlGzrKvld{(Hv^#V{(_<9~*cdbNTo(~Ep zZx@E6mX4ULlzXh$d=hJw*{Pa=u>)Ea#PfjF9$=XQnF>oj`H#3MH`k`HZvz`^T81!B z?}GsJvm$HywN>$GTEiBA3ZQ1k52J`{f}cA}nJyqgfy~ukTrn$OR#9}f(O(Cui77Cd zY#l1rr=p{?+BACbfc!s#s7sujgt$S|0lPO8+|OZc?|S?C!qp2x4ER0O|3TOavC)Z% zbQxZ=3`GXD)UXb(-MdmR7F4gMF!8nSf4{qStieNbKI#zD`wwKFSHtc=gXKf)va*vp ziHV8fAbolarm&Qei@TV+K=@?B?j_90%ew?C4Rkyg2d)4c^&QYjA9Tn3jU_20(2%K3 zk?_^(hA5@ap!_bgyU-Tg4OHtne=K!{ODa)6sEgp(=O+7y)UJbg;un$MTaEvJh{~Ci z?4)#nlZC&vD_TYRH`Q>pM6zgexr01 z2!26x8PQ^qwO+!kK=HiL1uga`skp-$2uz6Xm5jYzdBekNg5N}j8mGTHpCeVceYK3? z9PRt2=3rr_ss}!`2O|LuPRP0AJRZ$f-Wva<#Jg=zH@BK~4v!tV@A48L-u&S%n=$4p zYiqf7)$AG7$d9wwA>>f!xS>tkg1fifaR)d)Xi5!$8BY_}K!g7o{j7Xd+YZY2VT1qsrQ=@EF-$xMF7bdqD+XWbXq_sxRAx zRN3p~!Hql%>km$njs^w4$lNd0$LVRH{5F1eqW#X}17)8V-jy~tjxf;zY()*MKgmd| zQ(jAeq0jMths+lx?xGXlY<-hs;OCFiKCH(47@QzrkEE20Wx26}QP(w>)JZ{Rni0xL zHXhcMtv!R<@wAg1TcTWL%~ZIZOo*Tqt|eoy!@pjEN$Fmh#iofXeVNA9woZjwlRD*= ze84Ir$yGm#042xS*PObN%D8KdV;7`6IHtyH?GT!KM=UIreXnChdh1_K0RLzKiHHmCoTpc=@Z{CD}$po0eK7Wm*rV^D0UllA) zlE9~Hx;YU7bd!tR+{8vkMt{v#;CrT!deTvah1Ri&<#uBg!GqWM2FO;aJ7R<6-4hMo zwKs16^`N*v&{*NJUmHdF#L}1Pzj_(0DUs6r|Ldrd^5Lse`bMROUbXz~S}NbL#vzH5UBKwHyn|I)&0|7Gfvd%nlO(q>L>4t@OgB@q zirt>~#P|oUdI8>Y>8Xa&QqtF4+N!hl=AH3+LZVC3(guR|dKZve)_kigc(unThd&JT z$$By{ToSc^f@a+kbI9jzB>L>4m2%AvzjF;R3ml6OriUrE{7fUq#rfuAqc4qVduVN& z1{cKYiXQLD{MV(ssFxH7;;79#r(9CkIgFJ*yDunf_cv;!E+`NC?2ggO1TWi2<;RPU z*XOjm0LuZnRkt)BWjg{Tg_-WuQWOC?7IZ7f347o;{0b*PIz9GbCnfCP3t`7OFI%!s z>Jp6gm&okSK1{Q(@ytR;8Y)oUQ$bk{y$kpAb9GG1Z5-WbW1!#GTuOvK;HX?4MpZrS zxPdSk8qK5#`&{YGI)Yh|K)QS=01o$N;x7A&UPf8T^haV9Ka^ZZ%MsFY$O46rxEJg9 zsQB81z?#f#u1mk)w*Ps{H;?;H`4I?!U^pB5iP+1g8}B>?=3xozT^t0IUV@9cs{lNt z`Rx4!n44+e4}@6TBRyEXj zq{%ZFtn{BFo#nhY+SSw3lXKD%diFYO)%*uwl8};q{tH9!@bG+ep1w2hCV5u*U$2n7 zIDhH3a$MK}@+v?DgFkLPdoG)BI+V_6^f;Fs#hEyCsPDqg?!$mMPSJ>JQ97wl`QJ4C z?*R*od+JQkfB$8SwDbp}bG4qE-ug;Pb{$Vjvlf{an z20!;i4W64iNj~XAYOf2If-#zhtsm0?H;#(p!9iG9xJ%J8rYPlx-kdU(qP*edwbNMe zP>1!)GDX0A4CvQjaaVd*d#h@;8;S^s(bz< zC~(#Y5%_M|W%?7fw$SkR84?X{I(p~7<-HrA9*pg#c;z%N_^wL|j zPsc&-I&KC|)z}`X)2*Xss=rnN7p1a@y{=qdx1D`#*2x1CVY!BSI#2!?`X$K29WfL{ zDJ#4_gV31q))dt&yoj)#e-#(>V>|mLa+6+b>{-TtGyDnvRkWERT;Xl zpsMy>Si=d!DubrO!jD2ucPM`9)~b*mWSROZ961ISQY1qq4HkTg z3QvE{BqAS=d?W@oD_KFsu5d39apYl(xiTRttfzEsKNaoLFs8qnqH;e&L3c~6S$!;(3HcDb4T>i_#RqP9izFp zJ>rDKr>We7!%yTJs3g- zV;NhtglT!tE5;6k#X)J`-OqQ-YJt3<^0Y8+aHeaqKar*K=-RIge@bM;vV$>u@$*5g z#bu*ap#MQ-U9ea$ottzzaToaYRY)=fch95Wni|0feS6fb7r+2dTv&+hrb6>jOg8QZ zDP~6tZr*&x{)0NZU`z;n1g*sCj$h3^f=``<3xy&hZ*vO_({~98$?q7g1!Gwa>b*V1 zr?QrDb>oi^Rcqg&Fvb-@u_^<@4_vLV_B~o-^*(Dhv3YstP0J|t>@~-dlP%oujX6+J zv0QhvmINJU_~*~;h$i0Bdq}ulxJbO0P+Tk9TFjZ@T;J zz)Ed(NuM*q>3E+HSg^Vg4mymnx{xLBB?E5We2()OUeW)CeF)CqC&5m!L?+{3MLDyHQ$Oo+|7=r(!@7Y?w41KKO< zT~YV8f!Ay|0|ytE+2)^4Vp38!^#p~ulJ#|`*Dl`PlH1-gAvek20}co`H-W5&SP$A8 zSr0*jeT#Nl;{(xFx!d3K^e$Y@+M1j5L0SX|@lyvJe^#8?n(eveFN15n z!}xV`bMvTxr~fEgd;WlCH*WNXg86-;P;&j(%0jiFL?O6KW*`5*4!Q@E=09!K|0Mo-S@#8X znJEu&08vsN^tkFdb+gLQ84e)5@tAO}kC_(0h`)}#jt<|xyBWPSyk*}$xLgQvbABhR z?()0>MXSf^$oIbeL-4KSvQ!@fIjlzqEcam^dmmffvkw#vJKqME88X;^je`~Hhz%(! z620}=yNGnPR?({#TL}jEa@lBibH{*mOv$ql|A!KJ|@4aUUAuHo#9zw>kIX35a-KyvLe4g*` z{v#vj-1oWf_xl>J>vg>@YWm>(^e@#|0n}ArX1|!1HzQV2l0S zJ(x0#y8+RVS0%~zLs!7LbQS1$SbjdwEU5;1ET~4zClK1QM@Fi@0XfeK-L5Wa&=_=O zRI=}^F#>rEH&Ha^bY-2tobRV0(PsMWidYhMYAvk7_GLfT^!VD;!7KFJ9VVn;!I>3 zY9E`Slt3JzoX}HVM5^taMVPG_;R~8Wk287Cc|efI^;Mv&D`P$2)zJU~HNv49Pz?GB zK$&sG>-BSyZS5wn=mId|fLjUFdCuuzaReZK`3U`tnzvzqj2GhRF+e~k- zr|lVd2r3JcFO_AR#wN0boF8f#>I zKQCJ3G-3m#XXZ{Nwl9pjU4)H~H+JR2m@b8{w_3g|UN10-UT11bnoEM z%GHh8YXeYBzh>~I56w%-yK=u-B)(|-j$OM)AoQtK^b>dl33aKKrQi-=3aMz7cQ!qm ztbTlyJQ=gseo`t?w*qbAmoDMgol$02-#f^D;C!a%^cuy{!@ywYdaaUCh?85sz>vD+ zgoL~ct+5su?jNU=GfN-43_+QcfaFBU>7i1aS=A~Gbe)Uj;`Y`w^0llg>>?U)vy__5 zjKWpj8|8rZR`d^*u-9joMy{6`wCi8sE*fQQK1yuc_{p=Yv)ByBJ;UzK|1v;StK!%3EM6{omh9yYf0S(Z_MJCq$;lpVAs^217#!N8h;ySb_9rY@57j@oT zetI?JL*k3yu0;ei(|x9<41>$Re^)@NTgFBG{C7b#tLiWzp&rb6W3*)oA)@9re_73@ z6aHIXS~3(^*JG!MN~QVC4xZxosgQWZaf%4axC5WM&j;P>=8G}Is~!S2wv~{ESK{eC zwxGYh=Fitip2lzK{~r6WdX!4IoF8~{59H+!pf*;!y2AMjSS1)*Jn5c3c|Y1npVQy7 z2a>LfzoI1LdS<}by`E^v(f#zIjiiYdb(ZM$lC70hQe8iHBU;7#K>|abp5$rNqlso} zhxMUhPy#G0eG9Zd(5-0BUaPP@E&*^O$O;`sA%v$)RgUDn*PSlbwHbezGq@o?e`>= zckkqnRt=+2@0z}Z3s+QWIrje9kKE>((k|O`w^6lC!4XKW8MOK-hn3l}l0u6sWN8hB zhF{M{efThNTdF$T59FvbtgITlKqS?irG)Kt zIy=~%L4l&(rf?iogtp$cf)rw3eFC@9M+|H zki{1-^=i#C3TS=~D6!AeC%{QwSTA(jE6ragSrzci8_YMz?b&Flx%{}!SrWFg<|x9K zM%Q&_2G{gVWOAHKN9QPEXVri06dzW6-_PqH3%+#2yW*l~(OHz`M9C!~(8r8w(b{C~ zLsDLsI$@dSV^=t@+r;C{)H#b8@kdFI2YSe6G)C6Ye$GnhnM?sS$?z3S{JTc%?)B%v&yF`7>(uS$R*o17-*!K z&9L&@!LBtelB7jnL46%evK1lS$+f%!e8xkl$`?b6cV@cZ8j z<^*jy32-qEc`T`oM%!MjLh4=(<_Xr1ng#y zf=m?%k3`7CWhyd(2;mJOT=&Dp)xw=bYTThLo&q1GsEI+iTaZ@i{D>_dY@S^FImiJ6|schVQg6apCbcd+@7Q zG?J*1*HZRhzwGoC7Q8oF{b85V4a(k~;^qU<3PN4I)I`VH*Z=Fn_rL4rTGAl>_wUy4 zGLen-83Ry0t#;%Yyr6%LOQgtEAAt?4)kv z=)QG^iy2q$FL7=*>x@z%OVf}_=&sw{zxJeRy4->JSi&Zb83_B&j)2zEwmbK)7 z2-nwCVYG73X;1(NWr7iT+nk?nT?BIc962}}sR93IQ_yrFRUp#Q?a$sOmX#?|v;LvG zU)QuoZSKO^V<=Q45iRKPIcAutyidEorE&H2`K={#^06G+QqQ6V8gFeC+qZ~H=5~)I zNMPoc(`i(Cl5s!#etx^plj@3pjZ``Q9wRW1G2gU$=Io??q- zOBm3FYHheJ5iWbF($yLGp%p~OnSl-OoY~MlK7GwAOa)hPQ)_;mezial1+8HNv>6IU zCn;2_glj~X-qvd~gxWs~Y)}DR8b$M#^RQ9_#_uN6m z2weP?_lW;9nLj?tIzM3eQEl)aQeT zYi)v65D*D(u7&Bu)NL$9S?#$L=gFpYjg?@6Iy2Ds5Q;?Hd0tS3fEXnumPEobml_2G zxYN5^SqFM%i6F@KjJPMH^i8SfnCi4>Qz1uG&PM>wAb~NDy)m47f=#S`Lh_(Yt9xhf z4Xb>BoNdr=tg(xl1wafBi<+uepwSfGw^wO-fjU9bLgKbpuj+cA!J#>G_gW9jF%jeI zC?HadIRl7XTtUmut`h9J+a=dNvNp$15oU1}v`8KU0lHy%X^axNYFJjr_2d*PFtuoo z&!yDqBbJi0pMjBax1n!jzPW#%3fttqGH=Pt&3@YBXG7B7@P z4me|Caeh;m=Y%qW3}9xpL87yt=L*ys$~)a>4lJDXs#{KFhmL~gXT+hqR;2l@H0)4} z&cpM2LW6^!Er9Z!=3=GwJcl%#61LAgXjo3emE;<99+_(WAbM}*z3SewHpsb{_B&!$ zP`3c>ZBe`teGq$ZH4pU(0va_dV!}|(8@-!6D`MXgd-Vs-y!HxTw4*=jhEa?pVh!DS zOu;r+EF@I}Dt9m_0@w=kTdv`ZwG->E4t@C<4Fb?i6yuU#m2`V{xlK?q@SVINw?2p^ zoS-Qs2Ynk9Doyd_)0R88Odmu4B`{#Pnl?v1bra}J4jH@y^#@(hr6}Nlks(NfK6j`q zbh-SBD!TIgh)3YXi+(xl&=m_j8ZK%T?d%3=_PIKDyz}rl|JP7ZaJk7ub^rwoB}Uo7 z7Z%n07Rx8BbD$sQy6ucIF#3K}#Y6(kvENsia0uEctL$m;>!AKyPjI9Rq2Rz&B)ou^ zuyasraa54Pi@=M#Bi(5-72CBK9sdB484H%l@vEH`>-SvnyO)CvvQ}3WkU}9MMx&lZ z2hHh_gh2)Xxq(Q-GO4KMzKB3uv~ETVh$(@h=!+|{Z|i}1G^-xQ4RqT`p>{;;4S3Uf zJZt?$LMP7U5&##^wm-~i9ojGU<}0Mh2WrrsZgzwo7sPuu8$`*zQ!On3))*j)k3xUE4h2TMLpt$Gq2LjFHfgciqxd2y9d9JUo zZP$^YvyM$@aFwz&H}@n6?8`QL|IXW{kCE`;x91^gL;51p(uncawtwJ8VA0fNsui3M zZjzaigMoFh?#?@F~@YZ_4}tlKlJlxc#}a@87}^GoM;pk5Kc9JdYPg zQ(rlXDd#=~=sgTQuLTkobU^PzfdIZUD+~E=uGlBU#T`VQKYfg~n0{YRp;50pmH6tl z2MJ(7PW2M(?Fns}4Owc9* zI`mamCF(JZV+!y5;5C@5b_A8V9s)skB)8tu54{X1z8-+}ib(lXZ_NnZfxe#g`R9mO z%lVUWrKP7~J%AoEMT+dxTUptWmag5YzeqMe*!H{|*t~fR;DU4|Jr{%1YEt#L_p^@i?se_Cl?Iv;pWAC)qaAlKoB;KN5u-j*dt z%lQlxXAW6H(Sy)8%Zk+6HzqX9X~6yGhqnYm^ZU3FRTaM%IQhCq~i63GeW8io?-RYVb0Y) zwkhGFEb*1poR~8jqg$WOj;FjI0WG*rH+fDj>}eE*3deHSJ*%g3Hvj?lTgskiL)s^f z8q>+^BO?lo&?|wC8OE^~n0muDCwnU;{uF3@`Y0zzg7h=eMBBVP^E?}-on4>LsbVRv zzy&|-=;6a0eOcS!PLt7PFfUj@yVYu{lc7C2_Xe(Dtb0oXF z6v`{&X{jw2ErN}O2F#GXr5kH4iFLU^NKt$xvYX5xWGo{>V{U8EUby1PS)oINrVX|< zRL^H&NZTc{F(8$6lv`X?Z#Z11bm68|@oDth&`srA@f_NhPU$XJxLycU-(%8&p(*E< zO}W-w-A{=r;WHgqt|Q-?ZTZw)Q9JD#Y296wxJf+2yr^`NOU^jy(m z+}-;&zTse~N}ii54WsPaTMngyzz789XB~j24I+CZo_h}1tc!LN14+`*T}kB)nnRx2 z79Jp}4n7NnZlgZ|dA<`6Zr~9ivPaas%OL6%X@%dIEqJ~@BmC0=nDQ8<2a>0YxDdpOfynfuf#QMG)Q(#$ z#R2S92{wpQxp&~~Iqq|>d52TW>DybCE->mJVU&4w3%~c+(mo7m0bbB=i--`^boKJj`eBf@WMxw zFdYaz3NqY~>)Zsh36k2LphSQ7H5DR~h}JnH)20<5I!ps*Uf_}sB030)8XTclSQ?hzCVkb{&Ireh!zX=-+skFRdcEU3=^ z$U8JVJp5ai6)Do4A}9Bb7x&R^j}dMtUtWG@3ROHgkgR@Pns>-^1v&6~A}xUE^HWo>I}YD)y5=?IG@B#J!iymyW7PA=CN(eQ+~` z8gT!veEi4HR@f}iy>#79)?J&WZFoI(ATCqJGyjHP-Yk~6t~n$?Z>r-c6cgU!=;+v# z2Nt@n9KIsghD0FFug=A++$PBzsQ6+#bS&H{__{DE<N-waeUJALWl;P%blFAsP{ehSc%@-3RWxChmd2OcjSZv;VuBU zfuJR=Hn+uT2bfO&6;Ti0t2`YJ^V%I8;4P<4Nsw<{YK;gCEkwJ8t^(U)u0O9xq=x%pN-^j*H>u_)ZdHbW4s9k& z7f-kGP?fL9qQQD3@akCS<+zD~nPt-Z)_P+L(kqys1)`yhXY`ui_d+VcEmWrI51mhA zgJsq~s+@VD8*)iFTGnT%!v5lg3kN`I6?%Uv&Tw9K+JJ#|FTK5i4fU%%N};njYhoKR z1ms9(3~R}%&VeBXFgPBV$aDbKx{0a%wo<~H$aHjIE5Km%dR2q#=W^Y4oscdnK!0Sv z!4QCXkYjd5rW-+Tx1JXCQ~A>ggWNp9f3&8V&Yr%|aJy8`O!sxc!wVmMLOyx0yHlETrw$Cn9cRQ`vj`08ut#(l!ySN!*1!l_~5 zqhV~qen^xLZL23~W(g=LPQCCP0!f^s(BFREd@$T6@M08DM}cbbxinKa_*IF?q;H1G zfCtE9csoz9+W_NQYA-;IUv%ALq+|MzJk*pG@3+HRMB(y*jb!>Ni zJKhA1H&|$^OGsaF#keagB_%8Vx_;@>H&2hRBi&gZM`a7of4!92nmE)k za@bXF!Fk4n_QlfhANRPL) zWPH~tMrdO@J0zX=WGJO_R!>%s)1Y}+jPtz6?oW!bo&?FM%I&k{-m4p9_C`JFcb`?T z;}zcw4tg-k&7%v~oQ5KKEvBA|%Bn|dQ1i%iMRKo0^X*Nl>*pmYDYeylpNcB_V!vNb zUB|df)oj!|kWuJt=r0{PGQEX%r_G=htWNvtIVrdnx@zqb;L$4|gzp)qJ3jQg4h z?wrdKzWsyTLg3J9-DJwlFG|mG(j+a*nxBugxbUJ29o_g%+cNgi7eagQRF3bh$hPjK zt4&+Yu3Na4UeTR8)~(g9BUJW*W9I2V#_(8PsS)xH36osyH<8rX%-d+$* zTyV8`d5eNC^es)F(m*j>&~8sE-1$SKnhCs^4@p3iTAU22^Pa$NPF}(0HO^Ol{AJWd zFta!+i5pt*-)PQ}W|5}i_?Y3gRxqG3z7jucTBGf3mR!A7BC6`v6e3gD{pTO(_l2&R<^abE@`wlqUm+Zh{)?Z zm!W(%otwieZFkpD<(!yI`LE7>I*n?fN%#dkb(ZV)FKoOIfzIOC&wv{@DkFu;Vxjmi z*)AfuneQ$-Ic8S@b*;0zU7~&u%;z^n30Gj*!kw_2i_EB4n78h0UxL+*?NKo>oFBYE zb>os;M1*!(^-t@3QDbb*5FT4I-|^WCoo>*6+L-r<9-?gT$dEOZ(LB4ms~SPl5J>-|h1cZ@;_d^mOSjv>_S{MERqdSGm$ zl8%l=DiO*uiKXRxciO{{z)>MWGZ+BFbnP04;t_a7T-9{&bPwy$w`w~Iu|_h{)xhFR z*E_KN*bB`DczSwzB4lS2O$T}#f&_Ya81JW`rV@>*gZot#uD~$96y%(L56v4bzf1}L z{)9pqKJ+0`GVF+ZdLLUd9GReDF=wR~V723rHSD~Q2=XK-$1l-KV)0uw)plwNIuFnE3|P>^9E0fQmWx4&-(+%}aLwKC zE;!C~U8zV4){t?=I|D^l&_1TO+I2#}nJ3ebEB@_UcKod5+o|*zVNQ)5ewEB08Ap!u z?5fqdn`|vS`QE!W_f){8^qxW6%aAB6nj1ZC{4JcJOtV)>cwyvIi_M6b*Ciw9opIXR z+|Hg^QW&}t`=K*i%fRNBjA~GKlC(TLUX$)r^Zd+=?E+Idp7-6xa<^C0>BhPZLIfOB zZ*d#cEpH4qUptY8zwfn7AQZN`_qri*Z?rhX*z}cj&@cQXEtQQ)=j~yvbYNtv{~Uit z^BLPW-CxGnO3yYQznXGAF$QfLI51ZJ6BBl<$6&_R6s@Kdo2GK(F>`^5{I2;__S>Bd zLFNA9tbDl^+cjU~o|xk;&(w$}wtDm|W6E(YM)6yrJ@OP?odI&#!8|H%o=fjz##EF` z;d{atvJ6Aq?ZRn_1-ZAACR75D&JRaoi3o3;$0qo zsfm7>sp5R%$M{X6i4BdPB^@)Ldmgyvt-Mc{^Xd6e556>H zrx93e8ZlIumCcsSs&Nl_pP8A|axhkYyml}&bW>$jb9*hi3&67O2+qfwF6LxsU)l`W zZc^PCZ>C#t`MH!h?2bBarkAy0AZ~ID- z^SYXn@|}?AxraAzmYbI)`IOsRRAdv5lq!g=f%Va3FT=Pm|11#`N+srA|kqnd^;19#uUQFW3bhMAp(+qkVd8)FJpgG3h)r4N%+fOo?@y z38x?V8*9xAaDp}W6}Q3d%`vw z?T?)N&*8Dwl8@i%ar*l+3gw}^z7o685#Y9(e5q+7U-97z`kvf6RrbC6u`bWH>3R93UFtcZPyfSSwo|dl^UH~p4m^jm}zZf^VK43>IgsdRO7 zG5~Dhs12H_}BnX;4{4bw=J*9)k>ChHP(z_}OJFQ+< zJdE6uIUhjT8r^p!F9$az_mx!RSjo2U_3<5|}`{xoSYVzg=ESw#ZAuE=5Ys zq>1-1SxAM;9_TpLt;AG5=Xh<{;gc5V9(;sXJV0yNN4tpVP&e3;6y?2);ge4?d@(SN zy52rnygD^v`ZTfZf4s+8l_F%J{60K*6m{sk*xm9HjD+=NNRv{WiOG|S@w+v6n~N#( zSMMiCT6bwC#^ASd8`xhE%*9c3s!%mb5;m9LIctnVTq-r5iMGL<&0>1G$D~ z1+*fJK`YBU9feZS11~-Ulic*|HWnXOt+WYxzd>G5$SYDJ&AFxDyzmS65|!6eK4wP_HQjVGh@nn$n^mSqwqLe7YrEtdvpxDF5M0RH(fL3Moetwx)PN$HPsoJ-1qTzodzfC#LGIr z{;KWS%X8Zhu08L~zap9W-U<%8%O=eE%0l1CMH}RgL^PL`k*CIzJcbO5`swK%jLQC zUr~*#CGOiPQ8$UH-b2B~saTm@AU7p;I7P3~`10KXed6z_4~9-q#1|d*yxAJbEgj5w zKd5oc%2fK&X(wEd3ho-2=wD&+X)@801I{LvAn2Rf+JeZ%Vh0fRD z-MnsHXxstgD{o|!7Qer-=I;BqXQfqfeqkm$pm|ODd$$PX%;{wl;7&TNc1q&1EL^28 z-g#@d7MU#ird}6B`2$ve=mi+nzai@{v4}nWT4KQ4{<<^s$F0ibP8$LgpNn=^Y8Bao zE=uHj{vH$)H)#KU4`1hoz{x4ZiUeQ-C3%dxlGzoNCMwH1%B&v?@2-kyx$kzV*X$J} zZ?cx0oq0}}ZG_cKFgDhu;`n*jb&T2sZd7b~<~K*{X{p2q(88s7=1O)aW_PGK+0i^( z2ak0h`rcbst0G{NdaJ+yFT3QR5cEElzT;k;8!a#fqXISZYlb02af!-mZS zj-dKAnP_|jgCN4+n~rUe3i>=+>b4wge}dvd$hX*fUYx_|WIy?C-XAK{ z-=8=3zIuBIZL>aPa+ahqs(5#2vLx#3Z9xjZz3FrBV`7rs@#|Llo*gp~)8f!NIk$bu z)8gXd`qlsn)`a2Vou51jiMtQbid$Lb>9rD9Be!Sd`;mhO8U<)b1bY+i_&P0{?&Ub3ny~)lvPzzp`<$1D7u%2_~40OzqA*3 zrSIJ<1oQ+xkSIFJqB<=yGO|~8-qKj_n3;&S1vtDmla(bWh_hu3kc<9%xLgjH{`)-DkbSCli$2~1=sQY`{0|#;AMvm>xMA9Ie`p% zyo0aX&KT?Rws3RHigZwO3w3wLNsD4U24;-ev8LO)i%KezGAPs$63%Yg4S`bnuk_F2xBPI5BI11{Hfbmo}7G# zo144&+c&O1UQhD+Ih)LO`?_(H?6NzgqC2_DIoerV@yilGqt3zB{4-8Psl9_{-` zL-SgX0}u{4uQoI1aJ8O({SxM8s0qUsY=*&Xs)VRgP1mi}&L4StMIeuntY&Llr0ueF zTX?bdP=03HI+Y;uG{P3P*xaIxUYXcCuKpA}R7-!NePw#f1%IT3@ZbU0U91ioUA@K) zOe6_7b36&Hz@=?=cJfPU+1auK#b#a5@XKx3ael~Q%x|VEg%mSTdUT+;D6LSoQ({H* zAxi2<=gX}7jWb*c7L1KH|9JZE@`aNB5e>yvNlROEM?sFRs6D*nEfK_JrAxp2i`PCQ z?tFK@(Y408;_+u{_ZaaupWgNNhZ~#dO%AzFbqImy8`|1JA0J>kwQ;Y)E?RhDCkjLJ z(ZUs5&XfJzD1LhVIwIjD17`A!gw=1U%MhI zs^eCpLrhG3zW@je#L%fbq~Ku^`5X|`A2GuN^8nD8>rrZZs)&;M^3&4hTrUmr3A$8w;*$#`SMzz zr-!?FuK0ZvR;4{AqI|BHp*5UCo_ARFkpMdNVag0{xg{2xRnlB&Z&7wn+C&jD35Lha zf&(e@LtVBltX`3Pc>(hkIW?NuzB=PaHWJ8~*Ni{CxPRYdV9$beYaw=dl-8B$JU z)zGJA`#h1J6EGd3Qr!{CnzS_@Tp-|7s7^*k{HFeNGbDVXqH%#2D`xx!ou)q#5;C7% zj+76w`}-&{pv_1j~zhO3-Ql=t1Qja1VU z1t{k|+sTv;0{UTs51o^gdC7o0C2HhecXzkp^!FqppSt^;VJ%6LbdC~UUS0$($hiuy z5Wh}Tb_3Pyz?YD#T+1}6%_1ax(zH;$jAnq*k#m?D9b`+xQdkSwb`qi^1~ij**dc+4 zNMOClYftl1On1(@3KPsSmFQRl9B*LpI#}_3L$c!a{c7 zLiT`}i|mC=pd%qb#F%a0pY^{k!MJHe@miRxuVc}2U>Fxws^zq_qLJt(TEMoM!@goD zQpB~KfRt{KDGRy0jt93B9)Q$`41|tW-z5F0k|15MqN79=a{7?c_re1j3!fMtpBljI z74Nv!{Eck0#fR!`V&;7v4ty|p3^5#rB4!;cgr*Qo`5_2?-3UfDtM9PzdwT)lwfI5 z_*ys3i+XhH>)6la8xW%%41EeY`-3g86pIt9r9_5;A48h7_0Cv29#+S8{=ZtE48reZVr4>aZsc z@Es8jrJD*%l2>ohKK>CNSR|KG5CENfcg|dqRSaX(Vk@;!Q;UE>^*Im{8T3~=7BDe0 zYZN9`@satPOqLNqhWGxx|ZmMzVbSJ1(WMFpZ?Tt-MxnKd2!n(1{3~ox4{lNG5PZ|^O==zkZ zPu-}rlV}Yl2OOv>TGQH3&wQX*+imly8KycKCPl#6YM*{>ZvGCltjCueXjahL&i9e~ z4e(;A0<6A@H@?2K1<)#Q1hn5jNsMHfTZ#T!%UO2b!;6weY)1MuG80=vS!*;Rv2wga zHo?^^(F=wEEl}Nf@=7y%h5ps&(cEFTZL{6wu$pdwHF13gv9^)c-mob$l5CSXQ6u?% zLE6k2SguABxS+pBhnr zJ7B8e=vWrx@begwb`B0w^96kT$k130dhznW;Rgdr|LIXjp8uVMdzVg!?d*XGhlJ}p zbe=_;IpcTN0iWAyaS}rQa%vX(G(G}=B&ZJ{1GFxO7#ka(P&$ChmQYp>O_q(G3BP>$ zSe9DBojlcVGGOOW9(HMR$tvq>L=kPMM?b2B#b<)2xt9kiDJjAK4Xk?7?})*8az&^n z!fsJg3gNLB&Wzn#O>r9aqEJ*;?j6-8>^UiC-16E9*)2-NA5vAsg^d1fOODx3As6?b zBU?VZ-Av#BN+3y7v{468^nyp%QzJMHj3F6>XK!sP%@_=z9(ol_o^nx9kvGxvZbePa zS=F%6P`grJRb&r{NyD&&R{cP!@qD_vd=@`?QEia2wl6RP;mJ;5vzLv6fkL4)y^$mn zQb2z-2#S=m)a%hIsE)Q1R9sTL7?%P)Wo_+ja3_|s$a{LU+~7jzysp0hALT^E56;B+ zGPf?uXB9)pwgAoAj0LbZ<hFo!b4e74dUkn_6#ul$0`D&k+J71f`iSuV3r7)V4! z6lo?3z!aVHEC(&0Wvv+PJE%cJjbdxdNl8g5;x)4DJnR@w9_5O<%@D}!l$ql?XHt&e z+n#vF&%R;_aE37S01^3=1^(VFL@oJT(1i${x-A@{wlnV`xlVa#2&*xca+^r`d9>od49db}zuNd?H| z*h&|`-s8SYvrngp?xwlnw=o%&m6gQO8ogywn)IQ43%M4P;V6z(6#yGvzk z)%#_YzrXZYX~LD=zk(7=K{gF)=Yh6%3=(G|*1xs*Hp405?b!mS&51BF-})2X2huL{ z@~XgA;b3NEb6rF%C3_=Gc&{N zW{RA0N(SO+5C=*c9Mo*wfv^>E3E*#QLoC5*bLe@yh{Wnu0|7N!*tz#lGHU$9P0XG= z2}wrp%(L7>f8Ftzew#CnbF$sVp-n)INWIiD9^{D`OSX4%g3S#LI{+3;!LPXQ1;n^* z8NmY;YkHFJ=bO{k8~nI0`64RL>qk&=bm_A7oqw7W^`Mr2CM6IxXg_t*dZv-gjEb|i z($JxEhrizY3P7{J;Cu(r4nhJ0J59G?5(EybgIB*I7XwZwk)$PGQBiR`JR~H9GJSv9 ztX({JLZBCRQ950YJAN-x&_4U%fyvfDcwq{tlug4sso*w{o715NV2|*|i08gV1;iUF z#CGS>527?*gD5<}?0p=*jbl)`ne6%T!w@ofST@eKu3;WN5 zdy~r|4Uqwq`6As`6X+4JuDQ9nd}L43pJv}r>g@ca=r1;lf=APzyVgvTW6^EU!vVXp z<0FYuqs_;dg@^l37r8DtR!YVH>77xQUYemvG_eCnJk7)}Qlc3O;WP@B7$@SoH6@7I z^#8Zf-CY}i>;SeE)&G?K+zuaXqo(w#8_jV01{RrBHCGIDA^jci-}gf@T%d92;H;4g z5Hole7M4Ot=OD%wwi4t-&12dTcu{iS?tTFo3fnqjUOs(;_Lv^#oJ!r){T=0Ohuj)A z+5KBHsP*CRt+^*Bho0(8!0j%$>i}U~8Nav7UTOhn9Clf`4^fI}SAUU-l7d1|P>x@| znTCdJH!M95<-7?AzRLks$i@6QU|DFnz^3`fK@o(tj*) zN?{>aWAT$4Y0&bAolc2Wh7-o$s?9`l|1GFv0$%2AjJ#8v>(So;o>5@q-1j73ayHG> zogIJ>@xYp&0j$*`aB_cP`DfcN3`Yt%=IiMX6a!;{|IeNoxi<*so9WGp(py_~-tIOaUHV`loAWc10jv-J zg|JTy$h%3ox;u8oTJjC*21JL1Bpo?Uo(jbre@8vN1m$eaG>|GP`~ky#VX;W^zyvQp zI^li}-#ZCN>O6zR5`&Ls3u_$fgzzIIH1t=v1z^cAY#xE8ApkI_TiJ~qWWE$;yX1Nx zktgi(5Sm%MZe=I=q55re^K1n1+MSXV{r>*i!?x|nR8axzUvFU;WgOzE`(e6n+XwzG z@1)kg;;gJIo7zGi{$}8?`-;u}<)>|Q|Aj~+`!{sVet*GSMmJi)TGEWHtQ2C^EY0n^ zKXvIE6huHoqN|0yCR3F~KaS*j9Tt}k)e?U-1d^pfF5Hm*;ByZ>+Sd~5t4p4r;t(GS z!3zH`$P@v0hXP>@=41Q*-lgr?SQ_JoA=8bS>FM{{p*Hw)sT8rj6ENVjEAehHO@S&) zGR#AZ8RLgH_FzU_b%ZNtNP`xohg}OcpR2$k*?W9TCZ(_(7PH?uj$4?m-jY>7d5yok zI<@Q|G&#ge?mG=IMz>yoa5DHd1*p0`WC%1I)1B^WPHmIu-*Y?=#~S_mQejOH`r|+2 zduu*2a^&OuJxYnRc787y%R%s|VRaB|f)PWqnH!v(QD#WG;sy&93(k>X2F3Y5c0~j% zX`{gD*M%F_4prK%5uB0`!>IX&(}usWKvM2GB*Q~_z&Iaxzc4K zkkpEs>J*1*r=0=rJ9c~_kaCy(;=j#^>|`|h`-!8LlY>|>{4)9BTcX1U%O^rLR?+UeXs4-oAH>?Ih1w3gVrOO$ zO2$5T47fk%(c(%hX>!aTAshPdA$FynGWWkfLawH?SjsX% zI0%0g_i*Z`ean~t{s?eQQEZGSj~X_-r7T~l-hLnYtMa-P7_?ulQ8>8ljO647D4y^j znH=QGAK$zo66Z$V_1)HsPjUPEX|4=FG~xBZ1duJQ>MI6cYzUQ@H;rRiYdgqacANsO zQDkXp=b$+-aDzaYSP#L~%f5gAjsWhCLUfHgNnzT6KPUac6tuE2Sr#PUlvT8;NfIlw zpR?nuer;{8Z%KweO+$hMD4SJ7(q#gbveq`Ut&4!4y5rptL};rg$1MqXWC*s%+OIcI zJS;loG^ubvjLB(1MgpGTzXHAJcmEsA4BL`Au~nZzjig&u!iG|5fry@lU%{7xT^Xck z3g&cNmb{So7oZ@F7;Og+RPW9bdC?#!=HDB$Z)iw-G;o`6V*p9!Hu?@VrimZL$R)0P z<9u@`A*&y`GpISO-!+G~`Z_9A8$W%aYd(D6(#-T1WmGTuS@7w18Y)87{Q*NDl2yY# ztU8`lwmRa$f;*8EA$!2`fUvlclciYbEu%r%iCg&v1>7(wc;5g+B^NRkmrt%OEiFYx zU5vY{DWcGfUXuyCxJarlekQy*!6kq|Y*JzPT?hOG`Qv|&j$>AL{#jf?S%#Do3MHRG zRu0{`1BAxvcfi^sJ{PhY-htkZjyM3-Ws$OpE(hD$*Zz(9nfeqlZ2t*=)7XE9ckl>G zwpO&=cX3D<3sJ6~+v?PV|CD0(FJZ3F*(0@(}dzm~X#SYSXy_Wct2)v1d>-$3*0 zUy=RL0c-U4wwgM+V;-iDYw5Oi6zRH#+WQP&g*(_Ey787Ml%1su05TSF7pQ0!8Fn&2 zB@73to!~+>2vprcxbsJ!ulTcy4Hd`B<@O8N0P{`n*Q?4V=>RU$vBbb~&eEyT7TxY^FLYedhwM(P z*E*z|XGW3FEu%9=J#Vow7@|^&DnMz zgMbw7ALck8Qt|O;{5r}uK-~CacKR!|Eedkt83K!|@+5BFe2(nR*;&)ZN~k))BnuTN zfJCh!*CFQo6Czo-8Mbr%Jse0r&jW&tcX`#1pu*8W{Xw92e>HJ&PZ)BOA;IwYcLF8Z zDcF_a=$RifIf?rzrqQxq!EGU5>}24;Ud_qo`&11oSpoWOAJXXe7JU%B28<$GDIOm2 zDwN=-pjsq~0|@^r4N0th%^92@Q<{lsmMO}R;@%9EZGMd21?0;99EO6~z6<}lQzN=W zFX)i4INs_5Si!7TFv<{C`I56L1kecsDB-l4iWp_rt?3kk8IRD3B?m=S)v0iZU#Wg) zm)#IYBX*hmwM2jAMM%z%Q?Mz5@ugV-`V$7GwhA~bK8oVqkBXg<#bVG#la!Jo_zMvdGnXhHRwa9QrAGD#`<*eHJRm(E5F_`>N zpz_-{8JNm0D+(ipxdArB%@1wA*j83no+(0WcOhhrs>RR81porj+jdSF*c-ftlDfU{ zOz59Mj@j4$$rGlIz7f|nB|)h_l=CluCF2JZ9HGE-go;yNroaqbACmpzK;W2(+?O)- zkh(13?s{6v%KEAMZ>yk#)J*F}a|1!{^h9f{s176rHs8jUN|9=)>&{Z+opR)1#0h_G zYwP_B@2cpyD*c6bX#f@ij11y7t8W;9XK6)Te*DHE{J%`v898BOhQ6~t5;!uoXg*Ee zv06>JTvqn8>`f)ct$$>->&2&wqHdMH+GBGvGc)xEL^sFJuxgbnf}A2R;NEZd z%gLFUGR0=pe-;V(Q9cVHSRyAvq@X&9)RGBEXr{skinGGBo$ex#Yy3WzC*_LNTEIF- zwW0yC3cey&?0OQA=*?X>sMCo<>isPJPxbSUm3a+FSMhZ%uri7rdWe^tI^ZAPX|0C?Zv&bq@D3VQz>|OTAPO>-Iglv*M z`(Ed!_qRUZ-{bLn{gFub{kqTVoaZb$XO0oO0n%Tt~g%$uR#Q1+ej{fK5`>~cttyjyU8 z_1w3W$Cy_Y_CXw1{YRao{_j9wnm6(<6a4i(run9$p53M(B9zXC3SdW2+BgYXc2^258;vvm2Sgs%v;?f7v!7k81o`V&T5O|%VT|W(s;pSYC zOx}Q_NtPMp%jrlR0GkYs9(vzb8=|0^9IkxEqYq#)+7|A8+)e26*MCGH(!V-2HdYu4 ziR)PLLeyoT$(C;XzWtKnQ0+BRoNV3T*pCMU`I-e3qFK%TUnc|61^z-dvmRD#P6vf) zP*_-ZcE3`yN-&e*hm#n28Rx#vXk(X>zjqq-iw1-LE{)i|r6ES5vk&8D1dA6cQH1F_ zLI!R6NL<_t8vVZqrUrsVIwI-*4i*~ZCoNMi`#+B+paxA_HR*#1h~%cZW?ku8jr7qi}bJy!_&thU@ zBgf@O-pK#;)SEwAou&Khdzb~@LPjWe6`HD9AO!U7IiRWtLW6EJLCZu$A_xmbb?08Z zq{U^FK>{KzBh25`qUiZ#+g2E^; zB4Rr9BJV?4pkLV$8%6>E7xd_}SaJRc3mb$WjeDOx+2E?w(exVS-?uYh74%QMo9uXE zl39Q+#PeepZt7=q&ee4Ah_}IzQvsG_$4&q^-Cv)Mn=|BZg%HH-n6I$-M@9l+@r$Fy z{|USJF;b9go0o3e1Fj|mjkdsT0K!kidHG%xFpt8|=Y=zCeC-?@xH!^KLG88(-TQp) zXkBJmoz^TH-tuXX`Bk$p!M^8BNQYh9yN_Ij`CPmE3I404gY%)Kurzu;>ORAkFjpBF zJD(mtO_Ehu$c<7-6lKvMHOn2~W0abYh<^0QbJRv7-o`C(%toUy&p@ngQ~&D3@WxTn z^6GJ7lu=8JaAH}fy^pJY!C>U?kB(tT+?&={NmL-q{O$g!DTL68jgRUHKg5yEx9dAq_ z=m&xyc}SguS_y=s2{>5{Az~pT!5Zdr0F9DnK~+AHmN#&1Z)F3CSNUtpaVgw%xo&jX zuS{nuBrN_Go`H$#Tb*FNutve7&#%1hQNfaRG-jY1X~E5)LM7gxE3MjYLF>;q*sq$& zC1KjwG{{mDelw{aq8XqQ;oNq~5Y5IJ0L!xYwBY&Haf(JsMdbo5Zb^9WER?O$e<@Vu z;epv=SMzHhI(S5IWMZJd|8S^^ii$~hrXE5O_x3m~gqa|?>pF-N`fQpeXS%bnAGtP@ zUMiA1r+f7?LV|+=15p7254J_`VSjiFhr;U0v6}JR+vAuo>wUp8md=WtSTw#6Vnj$K*R_E@zUdcxR^VD|~{@|B*(jOnodAQt4;uU{mKmYk^wdIO}N z1p*XYNhYIT{R@{nHAu4{DTMQul1oYkV4V)hLnep@Ix|(EII1$&42b}TEx+ACa!fbh z`b_%cb&Jz2CYc>f*Bfl{@7yk}XXWGO)PTqDo6*q10=<#L0qeHEt8<`Q61~yh)$FMG zx1I~V*5~QmcDm08Mln%TOSNJFT@4R5CJj6xt#+^Yt{pOoLpQllhbk+q%#R3Ha7QADQw_^5iDP9?n^BqYc7W_CF9 z-6z08%!ERXHwomr5-%AC3R#Ni@tfuK{|E&Q`V`bmugrRPF<{$+=e6_nmm1oEqdoHQ zp%f*zpIdb+e1jhTc_Z^SF2e(MFU6r+42K78KWu0gGezv6h28e|KDr_4|MQ^R1nNu> z6Z>=@i41&Zo)PN;+s)ZLlbx+~*fflDx4~b~f~sgJdEhy(6A}^ebV3P_0Oq~rEcN(C zR5v|r)K?35CnqNng#?@Z2$FOlq8TZ7_U5)!u-+t)^eTVQOqAY1fTG30XeL9`Zq`kg zZwBw2 zGXmNRQ}Io@n1A6oynp4oK&`TFXMMtMOA{eA3&4P9%Rm<14k)D^FwRgJtEnZS${5j4 zuWMo-bC=QZ@o8~!as7pAi%FU1X=rCGj-Q43DBCc$u!=Dy zHre{cKq5mM{((@k$6ITtOISV=GSU0JuNi7y-&b>8Qwr{5teMUnwbK~i?ikNKvy^{E zVzV-$I?k>7wVs;(Z&|mLSCe)r8f9s1SfQ3-9-vdTONCf=MUw?2E1)$r0i|cphr90g zKB~5{LkNQ!9??Qw0E`6&1?hb_^9q)YlA2m~HX|Kf8|b$UKVXw6c^Qb1PR7KtdJ;(n~DTSC@@wu=vU zE01*omvZII5hDm5j!>IM=tGD`CK||V!6tzI8Mt2hLOeb|frD2Rg>%M)iz}>pZ3!%u zo{&k79BE^t@2XzD!R=*p+$ue6A0}i{kuv?4Xa29PK7zmTx@`JJ(Y;YdU!qQn_2e4g z6IT*GoopzUND(e;rYjFivV-G_5|IpTH*Hr|VJ4oPn*7%hdB<=`(Dz3MolTf!8oGh^bFDEG^ z)Q${aLvj!(`#^%z2BbocWPUyn57ZJIs# z)1G6)vyqO7qV7>HqSNTj_pbjPR~#>Dvq-PccDa|LYdxRV#vE4K8D?nlO;-%cjtl=1 zYfKz4-anTcOY%;m!1ZC>LQzM@ymE-!EixjaL!!nL@5VYb5_-ChPh!Y}j(r75B)p3u z3#@4M4|g8tn)Sv5#lU5*P4rSU$9`5FN@`V*hXpo`Usv^RoMN(gs|pbTvMKNz2#KH@ zeC4=@!5<6KjlZ^+75P6Zk0oZV+WwB2{b#A$`YoChJK9G zEw?Tz!GRm)+?cW|IQ&bRii!$AXw|kE_~nhCl_%`lq&UvgBapATdyS8=U+qlmSKiXx z$Yay8cJ(k##SiK`KnzD0mQNV6+F#B27C;6b?=UrBns?1bY#3_fq(a zwQYebO9^TW!14Biz>8lvWKz=+Nh8ZId#vAU$Jtr6iI+J5%5S83SJq<^_@O&rC&iwa z=wZqTlYOb|uDq7bBhNeVH~h*_4dMZI(!Wz=#`G(}%^wSag=U+&&zTP6n;%$LRe%PS z_K>3@iG)IvY&e1k)Fvo%BNMxL5-t-Bp27@SO6IwJIcwbp=}NN|K+J|Pr+TssWU|a4 z`GT<}=5x%}WDI%CZL2K9w;Tw)Fnr>dL16fvXZ};$9kgR&VuJzo;x-orfKjX;HqwJW z=>s4|$cDuX+9d8D08~4!4L^hRY<6Y0l(-GF695OcAQP(oWHDHRcCs;h8#oLyHWNqy zl(^W|TU%Ra1B$=y#J{Vd z0`3;u(SnQ})05WH?@p|BlJ(yj>SVeiHcFxUhNtPB{E{j=7gG2ExcuNrZGh!m56zZt4HD3;PCwYkoww7+&HbR1JnTTx<7WdBlC*zD*zY8`Ei z$tMuvase72zi~HJ$7-RL9Z7p(r2u}Igc4B5yY%_C7udy1w9nVBSoa z7LRnv$&i;6NNc96Hn#gfo2>fe=+T(5(M`UovzDkVu#ba<&RbdydkzHfeBhR|!GH(3 z9!IbJ%r_RKuK-@3P1dtv1FvB zb^IEu18=Y~3<_NGK276o0BVAvXfAaA+eaaA(Z5ACYaD_L5krpEtimZxyLylzNtpCv z=`w@FuVbKS4ICfInKa~aaLGfpO`hkdp_s1>RCLlek0g3g*>YD*i(`h2-*|skIz6I_ zO%V1*N!|BfvN2Lkiat8h;IiYKWsb|o1n6~}*pdD#kv2xW%_verk2OhaOfK#Bd*#w^ z3hCqg8J&$;rx<^^on>L|vOrHID7t_j`&7(msi|)*jyH8dw%$2&vk_5-UFjW1nLb4s zvKl9zjI7cbcuEHbVE`=mAg$#+-v1n}5v)T8R1;y@Y7SOba0R_>h2&R%Z?Gio?Q|h}!8py}DT*gg!+0A-l0pHlI@ZAKgG=e?H)|bKU**{1vbU?7F zYVdIW%>#mdbMpbww=!xvKk%0Jxh^=KwKad1qKnOe*FJfU(CiHq`eMlSZ-roc1!y@% z&h$KK&z!9raXThDaoP+iUH(0yRHkiquVSAvkR}2QbzJRwcrqZr1QR7RG*O}mo|`Hwlw-@s{JQ8P0$`n?Zkd*b3GL%>pyIy=&A1SAYl zVLX4~fXnbBB$j|dBL(#p+O$#OwfK-X9r~Ob0UYL8i*B7f2dsuhpwMslft~}15PSI# zVJvG9JF8cFrkbA2&jc~#RrzxC@at~vchrIp>TR+c15_gYcilK@{f~+>>N;oF$*$bf%j&<3;*?2JTQw3Y zxJkuB8!MV^hQ#S7Patz}5EM*W)dN{72ozDGe>AteKiAeu=S>goH zzPmN1z-9|9ODp7jU8Wkq&!gZ2nICP~ZQ?{B1L%D>su}_kJ~-bGeUQ_~4w6oT^|d$m zct^G`WPBDbixI@kcigbbaj-K(=jx(Ag9Hb)D1vb9K|S$>3pvIev>om3SSHgO`M8i2 zT-ScCkISqtzXe-@Vcxt0mML<$Wis=u~QH)Z**&~`in3N&$KW-?k~ zA58YOTAWx(5OJ_p8=yt#5nVvS1TPGB2xJ8XkA@Wg zyTwH$NYJZ0i)iiL@E(mVR}rQxfu0-r?pzAHjrf^lBR;SKI}Sa@L)11~@5)q4hA|un z1=xBh_MT&I`E|nKa->;3J?d%dImtpU#fLDYHIWYnlAb)-Uu_CA4;(l$^|lwv~1k%z_7w+Q01nvY0}d`I88cq!mB;UZW@YMi0|LO>-PeykaNF=V75D} z`o?+gOsw2|*W&jFeaCe#*Uu88ycZR_4D>A^Pf9w!H@YIs9sxRlGJcw{Q4;Lt6p7$x zFjo8xKaWgb!HC@j1pu6_`>Jc2_kjasGTe_=QXz!Fm7jk*Lqqpy+j4(&L3k^VP}L>B zNy^x$@2_qtI2ue83E3ciCQLgI#_IAd<9pahHJpD0{nx!q-TU95PFfx&rdi6A)|RoN zY+U(~n>5WV(2VT&h~@V?5rl41#yJKzp-v8Geog zHpnNp3V^sptQ_2HfS#JK8ANm>QiFq82w#dGCww0R1|}aT{_$027`{D&g7Jc5v30>4 zfx1J0XBw-+Vkw+XFG6rVYwt=8^*f}HQveCz<6Q=e6viY=zd;^U9J($)hXlbyc&1s&i#@>VEAHLeFNZC#>W97>NU$V0^+l_ zvC%x&86+Gi!-hfW<;N%W)l)-Ki+X^Mj&-1Sy{ zGAM8d)*8w~uY`XkIQ#xpwlxnZRulBtzt=A9QHc)m?MzPPH|IV~lbFh%4&P0wY|>`W z3Ku6A$O^Q~{cY z&Z#6I%>`DigX$Sf5;aomAWTWTmzc{smh9osJ_p{$cjF{H_P6PjQ)R)zum`0bUteDb z@+4T3aXfAtK~8{!@5XUZW@}(wG!Vy=08J#OMX&8fH)2y{YnHBaVyUThHNBzZDcR&y z<1{-@EYx>bmc+Wy&v3T0(68`wDJt%Ho@WZf@_2I$#-3{^uL`8Zsg5 z9Megf)bbj#&v9Gf;e0dh!?QOy&8H*VPBC<=B-!0E2Eq1GQCt0(LR>7rKpfPx0L99X z{F)Pgzw7U9R30K_eS}$WPo|!aa;j=HRFBsmPn^W+CV}ULJ}ZC7VuC}QuJNZ3S)5%g z=zef=_}NAC6+SOWgAkCOoS4Y0xC<*sBhd=PCEeiLSN;ali2{OyOoM=;q3{~aPK%g(=tNT)p4j5N*5sKcPcaY{A^6zumSd~X)rm*iF?ZBuxboHLbW z4F6t}yKXpTdQi`Sa7h(FO!BzcNA$4C0-6hXnF?44;^CCG6i|^j-=cW;E$Mc)p_eDs)9=cT*_g^Y>qY7|na zWmz7EFU0+Pi3i*>a~_KMemNUY*VHsfsxBjL7K2>uuY5bKoc@kUDoy{0E8G;3yH{Cc zc{JKMZRe9H)u*o)84_dD`w$x?W-!(jY=9fvXcDR4ha&w*QS#jRpT`Z(m6e%yLdqOzY3Zz4 zDZPHDA(m>?aPp*qrL3wo(3p7u7G7i%zu* zR-4|rfzvi%Hr=oNcCwVVk$1{dZQfUjBIk!PKbao~gb*bL z%;a@-A_S1yso#JhK)sbfV8GFiyL^yxh&O_^A6UHyc7i1BIgdWeiW~m~0tHF0K@C>jmum(t z+0r&ZW;s{kg+by8UZ!YpZGD|y=Ki6PYufUb?-BCWFvdfn7Y>qd-Lt#c?XK(){!4Q% z@b8e~ncQnS#z6c!1+B7C?Mug-7L!cM8jdqE`b)}pxoN-@15-sT`q%fD3eaE)*=*PT zER>?&FD30cg3UWeIFJ4rDEOKl{7-KPOgyqC1)pA~O&h7&6k{-;ILx=|@X}j|>?Pf8 z6~86cHgUtd03%KjN+_Vh=nRHAUgfIPfh_@ih!Is7oA)sM)M0WqKNQC6=Tdc>&-7Pbje@yq|5DlW)t3t`cqXFexn}$Vk9Pn?P=C(vWRoT!M z7a{tCYhO4TK2dU({m7eEQv7uw}KUhEZECfX$RuJx|qXMGZ_y**ncqtK}jAsl*Tqnw{m<2 z(-`;p@iNxgl39v03V)Xf_XVJnk!I4Aa&B$wuXT(WvbOB{tG2w6&;mZ)-kJ}{(2j4r zO<}tq1b(*p-0}YFE=UPAZv|l5(TO1cct{DsA{MWJiE1Fk<(ne>ULg*5n+7e+jDQQr zBK)AggJuxbD==Fo_7Hx@l^LT%f~3^MM6qZ1q{iw9p-4pw*sUE7NGF*Fc3+s{612ce zylMwS114oU7+=Waf!`~7kU`=99ITgA8C@ocSl+b00PG*OfJGoLhZMB-N(F7e6jIbF46TuTK&xf+0oE5lj(&f}x3SCjikSx6$5`6; zUQC0;1j#1QZinHm;x!LO%=gULSJiQ{J?{prw_%J+oa&YeM*+ePyW#UaoCwp*4)eT$ zU*)gu1?j5HdTDHHsnEOZT?|dRX0u7pkrUcuWGj?pR-;)LScF2)v5osy!^!sZfYbuk zh+N`kb36R!i34Y^BRss>NH|8$tS4LiozpLIEYe>e%7V1Kqt*pI{P;A};08q9;vzbM zJ$K#m`(6HW6pgB_F@)0%zu#*os%I>qs{u%Vj}gYjqX&Yd*4M&p>mMymL_>=fS+K-H z6ArjS18!eIpbyfGvdjZhL+12T5Y1>k@Vxyh$GbQYq@rzA4^J}5VeyAmIu@SK#`NYA zihqM4V=wj?5w9Z1eJ`%@@oCx@fh}Snl6Qvph~yHdEULQK^xF#!uY0K+w1SxKz8 zq3?7D)bNL@p_3g01xa4cf@&n_5htK8U?CpYb`0Nz&@rXAO=-UjD{LVt592M>0JsYK zs@#MyS)e~uB6OA6faC5H43}hxjzjxW)ENl)5mwOpL641d^dhu0^z?yxYLGCrhFqir zzXNKB)8L7n$AWuGhpT6foKF`HS`qJp_eC9l^F5pw(|m<<;IAp^mNNWLU$Pq}&p!ujI2%PGB=WF!s2~2s zLuXAHveZ`S-(dWlT55dg2t>k&BC9x7Ok`bG;lX?_$Y0zpffFqab%RJ$$V?LPI%aGAE9SZHEgxHeO{;!L;>2uSbix_E` zYSEJ{BZ23D4A|JF9w1tSDuWMM1v2hISGS^hemw&No*x#)%fbA=w`{J>d+po6gaKn#b4{i>> z%+_O>D(w#yIG*8k|NOM$)l=C kU-I2~_e8P*pZ$8wKlC2k@tr+F8FpBHTV;Euv-gy zuYg9XU-^|e)QjpJ`4zTmyJmNcYx(#h(*B~%5O8uVx&oRe)bRk>2|bu;AYqD@{rXq4 zgrq$SCghZx1jh~{VdnC5R}+iOl>k&nL(g94KOYx#!APN6otm4goT8GUOn>$|lCPZv zJp_;~diwg1sY8b(u&Ff&qbLU6VGr-CT~zzraur+3keB$sOY{7{TH5rdfCB;KO^vTO zxfm|m#&x51{ZqUgA_$;UEix_Rg+y8CheMW)(n?xK&(?P8%MEBUV@fLEIiTjD(A*-W z8=Fsq^`OXp_WS0s$QS=FGKp2&f?iLtb!X3s3qjok_JnvT)E94U0L%7DSAq62oK+n5 zR@-_w$t?*+Ws!{l8VZ0$0dz`r-~|se>s2qZPeWF2Y}kemT;dHI5qicX4|azD`;p6n z{^gB^tYK44X|()+gY7prQtqA=9UyPmqDg)AJxa;pqQ?7ccFa0iojBgl>#>&^l&1KEm8M7s*zz zyW;)VHi)Xm*pdNGeS-I=8&n^pi_mb*5Hf32gzCU~%x@82tE;S{k^sX{m+O#r0sFZO zFG9!RTKL4M=|WJ;)7MTHu{j6!A7M21byfX64CHnsDJJc|aFn<;PlJ6Y`@%Dmd%y!h z!v8W}!SWPeYa1&$NE2aFx0lgp9BFIAwrCt8D9~hldH%X6z!j$UIK1Mhb0-qfLt{xC zJLc%b#IMN1_Tqg@;;uT8nF*CisCT6JCTbiCaJ)W;Rd6iSNyfW=*@L?q(+hZ%sK%UM z<@quHY0($ZQzJkHe=3<5FDqkL!6wx2aDJ7=1uUQnECw&+lLWbwM=%qb1S=t^kPGhaF+f6VQ%0*H0*d7P6JPF{i`w|H-SFnahHzd&m=2p|kvsnG2O`fDbPZm!OEp@wgWpvGp%J=)Mo7t-#I83UWYx>hiLhL{4# z>=BQfzU$CkI2y7NgiW9;--(X07YLqsgVq<$`^wN<e`c zC*8pkEo7z>5%Gw=nSB7nfrx+u#<+er1qmhE?J0IIF;@2+n{gmG;-J;`Z0JQkM@0mi zL;JN|*xc|DYBPy#CIG!IkaZTOGumdpmr)&!U)#4XWB2BrnVn8Z*W3W~GpRABwU6hbk` zv8y4aNuvlyy3$B_5uAT!wB`E0vT4qDv1TT~Wbs^;2{E+eC znY6eZTuk=%N_!ik;%NSv`a zQY!SVL_5=-u=>?U-Rfj$N#fg$+>wxWh8^CU&$u}+#gV+9ro-yr8STd7`rng=Vp5vb zzrE)&o8veu$Mu8iy}(ba+r)~t;1Z>au~k|-(;jn#30y`v=|9)ROk(&hNvV=G^(aaG?9Qa4371^5V);d$h5$cAFtM30YBQ1pvI#d8tt8@^?Vx0hfk8 zs9qK&=n_E~3^)PYk^~sekkK1#`t($y??Utgd0`t`QIk_~-W z=LMRLl5EhPQizRI1{vEy6k^_A$n^v02pNIxZ0PoRXYa8BRD(p+{WIgXs52FSgIENR z5;VZ@X>IC3s*Q*WwC^eHJ~Sfb!B9kCMaq-@JCP57P{OPYtCrjSO^jG;OL(X@r+SJf zu0ZX<+dqpKq-F!6LDkSkvvxfrxYKByi&6A=)b_cc`7SMNcQ7CvF28fxKm$LvI3Sh; zJv0!D1>5EE!00$6dxr;YqV5NFS@09HsA3n_kFCwp-$jriM06s-BA|W-9}-85zy<}$ z3bge39;y)>2OJJEMC_JJQD)FGQUZlO`m;9mV7X2QlMU)THFOia{C36Qn(*z0;DLO_ zkoWDfSeQv_0SArWKTpg{04ZPF-V!s&t{K>qYsv*pPx=y82L+Nws;3oIx)SgIs+%6Z z9oDMD^oe=dn=J3ayiQ7Ntmx~d(w(lZ<$!m6J@PdnRe;C#rdv~)O|~&qXPdO0B8Aq` zJ-?n4U$)v_U}Vp$Kc8Cjdt3kVOHA`?3N|_C{GIEhPDQs3N)qLT%x*8JW*TkkWQ3ykHzKCGNs8aPktbtx+n2D;8$D%P2jXjd70+eYVGT4v&qsEO_E9T=OL>} zslGUv))3)0IaGc`N#$0@x3vU;ceS;p^7grdcP?43Lvvj)uwM%bY){O*mONwn&T;9G z(b5E2ZNu^GZ;PKt#EVx44#U4rjMg~9c@pRliNO*zVy?`TLVGETkK={%*p{rtPV0Q( z0;P6MY|pC$pi^xC91MID4yG7eeWa**4z)UO&(K*R6foD-y@`jf2qq+BQn?*!l#~rsH7NhNuy7SeC<(Xk(sR@7R z{IsSpMR7K=``eSK6%`ReFPo>h(CC9q67~x$deQ8Oy83}g139{2s@a+Hpx$NBK>@t0 z#=t;1Wkg5P0gwy^lW^2*{h$fDg|bx80&Q$;fcl$+UA2t`Ojf9k5=v9ytebDFd1e}j zm4OL~NWfp)*e!!p8|Zul;-gm3x!VbgUOJ#4g4aZ`>JBMph^gOYa~AdX1@%nZOq+~A z2U3wklF)*6=3rt=;1CVy$}|wae8Tl%ac0)`L7(RH(0ORy$sjA3v0slnPD$i@?!Avs zm3LMddP>ulzR!as`I{HcnEJwYLoy$cQBd{Z(mNho+i@dO0onA2tP4S#`gTXBHZ~6$ z4wSx-%X`zl#yh@R`uk>VRgDhd4lZti( z=2$5B_6q16X9V)F^W%39oy+gi?cTHvXpR5Vo6KgfJ1=A5pbjxNtIVTS#{ALk)mff| zcUz;R;4Z&gHBzJB+t#XCxH&hl5#oA2<&w}ZrlN5_--{U?zc+uEq~w?gC^)Qp_%U^9 z2Bgfpx&{L9a!O@O^L4h>$SJ^wxT|MU_{pR=A26o6u7{jhn%0>w8Dbgpyihnf|9fW| zI>+pUEQQ+Hv1||iUe$o5Fak#eq!M3<_#L@ZAn0zrJZqtoP`Z)g|I4vIWc$%xw1Y45 zVgCRoQ8qr*X2tUA2@TcLdOA)IFlqI{+3Qdc(WEKx)|A#N4z1?1mS77I&DXt0+s0Nc zjVS<`G6)C@OG08ZIE94=IwJ^Nd{K4J5$pmvpkNW9NVC}|ll`*v)5qZgoAEUslr1(J z9e|2DJi|}@77FOlY7gscUfGz=fFquCg3N5pu@#rE#ZcK}q@*F)lgy`NX11cZ=LGua zc!+~H$cI9I@2r+~fp-Dz1aLQ-mP>{?JHr02EvBs4vkfTCX)OeLlO4D|5s_ky^!W!etPxc<@@B~5u14$Yt zKf}({5IASXBz%3FY8wn*xT2H^$D&N9Ci|D3K$Qwnd35Wu^H>*33$|WYNj}H*yX#Q zcWqh1ySESSu{P%2uc2C+_(&YVG5Dm#Fg^E0`JFH6T2>R?Hpff0*$cJj7duplT1vL9 zomTtxMt^j=TwH&7!QopmPrCNJO^B$1Cme0|P{)ilTOmQdC9aMZ4s&#f;2+Z-Z!-Aq zMV+fVT;a?dmOLmLM0h3;@;gRyTH4bG-M4>nSo|37Q1OF)yh{1b5vvL(|2lWI#a)Ef z7PEU8?4LiEezr!`eaC=A^UuSEW}1qPS*I#sIs|i=Uul$#^)l73G?M-y}wbc zgb26WiWJs0cNaOFh!sMwuuOf{hXn{1`0E`2@xC@R6H7XvV z2OF}da!M66`MTua?rMD!djR75A5kD2hc~KPUnD$TVw9r}c<>3;4|nzjN=Oq(3b!jh zeE4w4i~4ODNIFs-t^^zg3Zw#>%pUc4vZ}dv3@w|Ijy3=Z@o4ScKoLN11V__J5oL}$a`=q?nv%T}}MK1Sa4I-y0sR+yOFl)43 zH?X(NH(gTm|7h5 zV@GL-JdKFu2i-1G28}` zvg*ba^DYk#k&$_Gn{`1d<5jw|tYV^0OQ_o%b>|o<@sqO*n&d&={*h^wV|Cq}j;zqg zmI^;lD*(>2^&Gw5;GEYc3+I=9{^}nQ)uzPj+Ls)J7cXt*)0tK*tE_+1*7hvB`6r`U zn_wot$G5MP=MDN=uQ_XK=}ojzpN!;QC#R^uAtO5y+Hm&1o{`m@=E0S@m$!_P4r44w zM9Dmlb-N`d799G{#`xEqKE+4=#x)P;@1LAH<&WciT;KEsgK%&orR`N=MfccyIGuxx zpO?f=~B1n+E24hy6{1DQwgA08j`r9!%weBg@4{m_rkuy^KkOwf#-E0 zS!t63zY9)qKUlGudPGUdJKerAHQn=)cfz|ibyBW$dS3>)%D&4K6vpRjShza7%u8rr zI7UBfbA2}}5QEcm0$=c96IhbjewRuo&{X;QHIh5ZkYJ+nzqwWUoEVzv=8}Gyz}Wak zJg;P~E#bgh*OE==oK-j;M^648l%NwsF9E-sODV2Ba%5?(@*+MCCQyQAKaSSPi#n~no;iV449+8jErbx-QSQuBUYI161bav3vp5nVdPgIA$3@O*$5J&%S$@l2H-*Ew)_JuAH z``*WFo+2Q)7a%Pze(Ea0?zyselRAtevZWJFUw!>d5~wm3=r&3l+Qu$%>$=Hw=i4V$t<9}jIqXL7WFI`HxI|&B+OxlSIp@KL3&IrsZG$5VT^mt$ zgXChfFAAcq74B@{xu5E$ZVm@5_8_BHt+nHA-`bj_{P|2flZ;JcFdd|PD;3y>VYa@1 zE?&msfzE6{7x94e{A(~ zNN_m3IQ2})EhndaJ6q$r*lGIV(dvCTAh%QX7A3*mp|U%?tC}2m0$ypE*G`{h=sA(q zz2EmTO`bfxy{(0P3E_vTi#xwUM%a7jwE6jI z;XG&AW9LpD^NL-WJnQ~M^P|l;B`h%B4|1p5qlNHr@$ht^>VEF$m!}pS9PCLY8GHv!a}&@c*8SLwcq0)pv2lR3l$1xe zY5Fx5UhEyUxJ$>$cgNQPpz8mOEX$K~UNkzodlr>9a?7CYoOvQ*&m>Y!Wfv|K5|M2^-Aa zkC=V?jF+Ye-F!)%BYz&QrS=zq9qsU8kFO0KmAx8XbN$;bMGuZXuUamBP?FvLJb=3G z)hpTOsq_qrv?!o4-xs^8o^5F*F|)qzNFwm|Np|+V(Xq+lk(i3U0@CR8yKkVe`tfwl z8bK$gOGl1KT=IWDG@B+2OJJgl>5xR6W!qiWFS+R%Yis-8@I^chJahc^m7^4w`B-2o z^YR+IXmjvr_~=&4L*P6VQ1c>+T;;{N7J~9}^OxtiELTjq&HKIzIXbfR^%ENQ_nd~< zGqAdF{-TbX&G@xfVM!w5Ui(~JjI&L~J$AO;$^ALU!#ozpuUjtaleM-<-ubA?Y$$tZ zWA#KPBgeUV_A$N8`1RDM*Eh~-CDmME@bAp9$W)~lD|+iwpMNuObo$OmO!@#VWITd5 z=$agJ9GGItpFTYrc2$f?(CRJzwYW2+&payV5`Q=ks_NgoXM6Tno44GX4ENiYJR&lV zoZ`b7`jKKUq1jU*|M`|)6nPfz7x~kg#X*Or$uNV#s}JVolRDj=`h;DrZG9hmMFvasSZubD`(#)CLXgcz#SNnu$`H3 z*Dl^a<;2f^jppl%46t%JVC8c59XdkWB5vp#K5leRrW} z1d*LCQo#&9A4`<4Qifhn&~-0O$fBes8ZLY6ZP%Wn;#Rq-rgzrofDh69Gi@E0ra6Om z$DifpwHjG*!x3ZWbIpp(q#nQ~?s2rJT|MEJAErL6?G&AIpvA~VmwaD80W$gNoMpNU z5)pCn^?aQ)b%&wN`CB}*+&Nc3aY~o2(}aVe*Zg+C9q$?EHs4%d%`HX6EAsNJuf#qZ zcDEkCESmi^Awjp-VRHiCt4(4=ukju)ps0uk z^a?%z{SSj#i8+GKg@pCoF_2{8!6_T=(9!Kv?dV+l($lBk@U>Nyl|7+nE;xB_M(u6p zPotH4r4`?|P2zfw4!6fkUEqaouBxQ~6gd{N3Gnb-0PwLIT$@AFbTh*Dc0bSYnzpvn zU_d?teXS1%B;9K*1KDq7UENWD@J};y!_oEi@C_W=hzrI9!-ng`FTZ6Qod)mGv4S^E z6WDrBDc63MZEK9F+W$>zQf7UVjK|>b61)M&l#4*dR#cV>-Ma^V=;GB4IBXrSvnLnI zO+S8apEyaT%hwe&%);2sKK<+}#V@N5TcH+p{Sj8Inv?8Au@WsLWVtk<0Nd!=&q|$z z(%&MNIr658afgTJ)|T3aLs^qv_OHBT_THM1J}C6~u$LqF>$V zOvQh=IqPEBQJeL}y3*6EjHxtMvBvNDTFDpgvvnqHj5aH|=}i6;r>Cb0>ArjqGvFTS z$svKpgwL!-#R%wEj}b-5i!W?U7ca~e;gFwI;mwQimwTe3!{`0-f<=X4Ht7cu%Kbm; zC!aHwoZH956sPc=-;wojOBeQ9rQD$e7vFs^K zVZT;fuYXiI&AYt%;=4+y;5&jB7UYJjh#n0scWvHCN_BP&3Zm0xcHQAO*_d#gIBpWc4%uS|__&)1}+x+DP$3cK-`KB|Nz!^K*V)7E3~RG%Si{Rf11O zdQ}c;-RxUG7W$&&L?fk=*vW35lQ{bJH+pxD?H=K{RoyEuNXxS{ zZBQD$LfFxhuJVeG8{fohA!8+%hyTgP`lmXspL`VrG;axtis$I{ZFihwFFm>;ZHD(T zvShwFk7!}>K?nu|!Fz3S&WQPZ18459BQE_9al%P@bvIfJilRkKGSqu*g^j%SQFrQ8 zb~A9u1uAWGox*$)>T5PL#XZpMl_3kk6d%QzeNG0|Qu3aYxL*Yjym%ct~$yRzT6 zQcvr%rOYQHZ0+qEdcWMiHd6K&*R=29X*_&N3Wl`&FPZ7-*XY&T2?6%S+qaLAEjY#L zrF?g?l1ke6+uAwpqwPQ9Lhv=3q%2DNPozbD_&pHUcK+u~79J*Wbwo1w#S=|NEA>q_ zE_?d!ABE#J)Ebr53G2K#MfXIG)&KAwF=%fiS7d*^g5wR8S_Rh1rK zu9e|)T}&s-%R5#Zf~4N|9Xxt=#(gr6-=!{CBb2SqVX&bmCu`O{Gs_`E0Gmth{lRyR zq?E^RFTC!3nc&xKXwy|NJ#`E)X+hxzxAJ4A-gFfmpQiBe-hUas5_0FhWrMrh$y03c z{^g980c=}kd??noe}aPh7zyV|kUKo}^OFf01Jh|GoblxI^y$+tCw!+qcXSYHYvSz)F}#^2{H-MQW0@$n}af!uV?-Ey#U8n`Ke{u>uxNO zGM~F`=jc~aA%Bdta34I6jIRx8F25F^KBc(*cI$M>#?<@DSw^T*XL~cs3hla3bY{@~ zh$-EA{iMXXYdEW^v5UqW!)bfVm^97ln`bGL~wZM1*USmN@8h)+KTZu z*NV4IjWOggv0to3)YNCzCXNO3=Ut4W@?Ca*FzHkGv;TcD=g(ykL+fOL_<5C^wpaKa ztdHV5+nf?n%dr^zbmq7(eesJW4YpM{_wcfE$-_ez2Kjwwo`!ejrGGecOGUHXKq2ok zU3yfb)gV`ZCzHYK=a(ZiZ00G`rIaJR>yP|L%Iqz?msV$(`1zBGCHB|U5c95lQ%==% z=$Lc(0H{!tfZt6&vJ|Twr;+Va%hfqmfMkz@u|%5x;r8-?y>WX((fnIsQ`3EzJ1y_i z&#_+poHW~Yf3g)@fETjYZgRUrnD-`R>C9GVs7h5Mo$8YY&;~fzbC?@Rf!`x6M7bD5 z4ZQvG_`2+7r^aQAAJxr%JEfu9Jyw@dX397p&~8j~MaEpA`s(x-(_DBH9OfsJd;e#*eF8&k3xUQVP`ZC`ayV9KtLDY?Ao({w#7wz(3Fn-c>46Rla$Mao^!Mn8Y?;tcam?EiW1*#SoYj*g8DgEz60*S*{gv{$;|ptZEI z+089Yp?9j7w6v(GXjSZomX;QP@NYqDQ`{FX&__-*dbr#Xqnb07%*NpVoY~$K*Yo&M zkB|@oBcm$dF7OQ+V@D=h?r*{AxHFc$w|psw7plQP&!)ZzSY(y1p9n&D?|nSPVLp0y z6`$hfmtA7qh|6EkKETH2D4`pGo zNxbuy>(dr2LtwMqaXOf5TJREdYxHM_ zAlHQBj)xwUP7lk@EG&4e&AT$a|M20Aan703SA^eN4BmM7@G6wdRdfATC%uw0==`5I zC&-X*EEzq9^zY%rukeai)5`qzv)&?_nwQ*u)eI-x?tZ*DJ|3leYmjyp7q95jwbs+z zk*AD|2z3LVdK3uJ67no@xqa#GT((;_#xGg_L6EI}`H8z0ir$HSPR zt@YcVHs{2hR=O<{s!L!~=ii2=Z6^u7d!E1E6AiTUgstcso{-0|WeX+H>5Sj=P}- zK~8JT*w;_`sAN5-%lF>q3J<1s)#LDKic@5Bz5r`Mx#xcK(&M5c5ORuGoRi#rTeK^sk13uN7M7u?KHbEgQ%mQ33^8wIy981g1^~JsxG%S zF;CF>X1R5~HRNE%9VkzS8Sz!(>gOQ$!&hr*Eq>RQ;{xIC=gGC?Aa(G1XR!-z_hFBd zUT(_8IHw@tH0P(CV2u;=t$8f0zuDKqM3-@A&^0F2$p4VxvKYDgviUwJ$d5UmmQNoW znria8+IAkmpCSQ1JTJ{BJ8iu@XH=}-lBMjdUH8t+ew7k8!I@2U^Z&5*7En>B-Tv^P zgbE@J(u&d`ptOJ>(j_gOf^>H&N=V5;K};5>23i5>F)aW=zH%v@BM$QHF!Me zI1Eqh{j1G)ERr0R3{Zs{b~6oAokt=o3%g#EX4_UMHm4LQ2oq(`_vIX$sv-zL@?&f(;2b(lLzg?CF4r3v3;N7_O@c0 zIrGYhVV1meGioKGX6M5^es$1R-x1u^b{KS+{3U5HW{G0X@4)xzPqIanESca*f|~|j za-vp>uHLxbckr3Kym5l+sW1b5m-4zcwZpW zcIMn-J3+4JK3lNyOY?-Ba=dl)chmr!C|lWmG`sm-l3oPv84jzZk^b>1&wc8-2SysZ zzZe<#1W5gUVhes~zyBSst_Aq%{h$}(CVcr(uRW#KMSsENsLa8ceq-&e$lBR2W~rc? z3hs=(uUw={Oy_TYR(vyQbdz;G)tb(h7Q@9`u>nBaKP^XZA(*c4Kt zA|frEU4vPc6m?$NYnKlmraT{AKeF}Ft#K*+HgL_)paaCqIx8Lh}VU! z?*t}R5ZPG%++B#))%gbY)DetSaeqVTedWwgSBA(v&K)1^xzPYs_KB~Q#q9vgM~k&# zf3*@$&Difpc?GhvEUl?wR?Ww|_OoiTmoH`tiO{JUGoDGunOuTh}m@{UdaGWet~d}}8A>W$}3 z_<^D>%s&X3tO%YMvP~2Kd1hqA!QSd$In2jHjeOqcjv!Y6Pqz=uJ~V{LDHU29A=jOI zbMy1;pp`v3I=V)1e~9+cBLcO2t-^Ewg3TFZ|J2KSdWu;VmufJJ=?3s(fz7TFj2{di z3kdwV*)+?sK@>CW@t<1@lKOuSW<~hm{{2=sbHP9kLlB68TPs_9^@tS73qvh!LLJSW z7$_PTKxw3yMF2of3#4B7gB8^L0`#D`dvX7Vt1kA8C_&}w@>u8p` zX0KnLSzE)aV6QcVd;4fUkIgy%{E|2E@#hMhGFpBhyws6$7Puu)$l<9Zz0tKb~FU|ly>E*1an331JVhBdPNf>l3Mb_zw& zHWG!HrwaR~$M!Vi;oLi7?Ad7Dglt8psj6*+PpWtXo5zuu`9)vqLmbKteV9rfQl7WE zDs;(vD+tYHqtfbVtyo_n%ZRH}*Q1cm4|~c7NMvaJF1?UF!5v_^u416iw^31v+kB0c zjpv^J?K|7VeV3096R`!NhwX!j^PltclkC)jXIqaDK+JU?t>lR3G*gmk`bb8U&38nS zBfcdFY^&EjM#m&}$|@6px2@9tkeW3g2SJU8VqH6Q3;B>rTi;{YtQW$W@gfVExx6s> zBttY=spJ!_EBMB^vz3qFlvq~_bO)^?x?#;J&$P`vSlxDhl_7lIz7-J_^|0>JE#F|r zA{*{cJ~WenJ4y$>8{`A=6B3KI8{(#hjeER=h;)R@PDg72Js#bP*468crZTxfe4Dmj z0HNI`s?5ND;w(_O8u5^)e4PvIB@L)rB;-K}VdCTGHPD8jAb?x_adBw#)^WF>)MKsE zVeN7T&3v5u2gSonUIO9u9KmSiBX5Di19|ZecuIBk8|P=_8izOE>V7(}TU;B~{Q8N- zbc!2iK!^m3Teq}pXBM;C;B>~jz6$D&u+nw(#f_6a_Djo0_A9O~g*`Wc1xVLOkj)x-IHn zQ&7df9y1cQDnhTmw8vJOCNp)`euM@+gWS1ISIh?6J%Wc9?5H}X3Z+`z!B5=qbXipd6WpIq0S z&>*W~d;xkD;j?!#=6qj@AH4K_1ekn2*MiukuKe1`FJ|}-KxRzz(O}n|ZfmW`7F)ZT zLerCws2FXkt+Wx}de67N>)=w0hRCt0N{FH%PbmxG^NVS$0*KyqjCZ3S-49M}heu+k z`9m@J4N(xu@t&4Z*Y6--bwOxhdRypCahNf6DL7eGN~hypdkeTmj9a&!l^S21&fxyR zLm}94X&uNR=v#<$HTKmbi<~!@i|Q!3EUtPT-$z5Rx!RHCYP~4m-(8Gt`w*u!y@Z5# zV6Tj1d07UFJtSh`LqxP+K_IcztsOmJHY+Ew9FI5#;_YVNA_vKS58{)UyI5gdy6cR+n*4soEOSLtp2!~xMOdaBqv zegbLZz`%Ddt9d({Zx|y9biD;_#i+2@i&j@C9jq3$D>rRJ4Rr%%N#X1pd&1c(tt%)cj1Ju&`$jXi&w zbJ30czgO7b|DX%#7}oc;3j{9n`PH#Rtv8t#@#k*_sRtF=Y5l5`H=e5D;7(ED`UK8e zDGaey5Fnp0oQBWgw+~E1q{PKl!MGzQE!_mFa+}#kG<9YopwGvVc>icjVq(}oco*0* zf3B`u#HAYA_vJI@FMHmlsOGRkSW1c6PzUXd1(zY%dw4*D{v`bGtL1zgE&)1v+9YG? ziFT%efq_DZIKAkJ49L8`y}jK4U$WT2Lca>Lps{=#wy%@@p++emS=TyVarSQf(B3>+ zY;xmd?|N?88Z^tf5yi1x2i}J+^C9!!e+*th{Ia`#uo3?TdI2>})x4T3_pmY=;XUa z!zfXPqHbNc1=+#?4w0&0VU3(gGX_;$8QGzBo~AoAu36V^I+p8_JkgposM+bDxpvJa zt1wud8I(jLS9Lwu8)KZ;)Niorc02LpSDEGlT114&=4htrak@^Qsa(X^S?#pnRK0mn zo<2X1S!mp82@JZyOdZ2DY^NN>wgIKUG-+|o0d(Knn&s0@ZfmgY!UTw3r%AUdlexPF zgAgI3Xc45^wj@~TW)qs#2c%vXyzMa;NAt-rO;=cfF>NTy`{H=xsK?vy#Q>9P^Uv;h z5C8egolac!nhGY-ltq3H-{Mme1lzEo_sro8J;Bln3IgvlNsk9Fo_3qEC+jr?6t=YW z4{<2`)y|wew_-7|sZ8Lg+r8wzp+9wI&H^a+*L`25uV=&g(AgyDUXZ*QKULP8rul}3 zSxde|(i|5YwCOvYdg=oXyZ#T!D|0Min)>U2dAiwp1 z!*JKuU|q$0ygrgHL6ZwRIsV)%mqtmdZ+Cn6)82uPS+w|O;Ip+c2XlZN!u0kw-9*we zFOF9DO9d$1E6X`4O6R$Np)~WjOuu(5#f5dU<0ZySsNdm!HCaJzJ(Ur2Y0&ono{mjB1ii(zKA=wwXg%vkcm~zp<)N5`gxUs>xnyL>?QQG+% z$C9Pp-Cn{djREudF%;ZG0j>CF}O>9w@!|V~#`)F?D+FoNZ2REEr-du*fF-bCy zEK;mQmA^<6stUeQzIb1!L*z#?s=q<`CzjuEr3?s>W2ME`bl6o&y!Vz6{r&_%&WBdtpSu!ln%+>+1>vudlJ2?*Zxo6hxU7NArBcOeOWZEXznCm z1C&4`50`;~^lV4^hTzf9M^#SR?*j45?{KOZma_;6P3+ovcqBh%Z*EaxAtuvQxWBpB z_4v0Jzt!uVOYS@qKx`p^G){y8L`eNmHMkymF)mRxL{Fe6wc&oyXTE> z+Nq4x8jj$?J_`&iGtSXG342dmg%jB&Is6uHg|R8=TlQ6ATwKmC4X@4KRt*dC zz;BR;@8r}JJE(nvNjYyA6t2^-qWhx2#Ioje*;t+icYJ&t9K`1z2mHczc12J6{_0z= zKTT6qX`eq-(}xs|A91|+VC={)vVd^_JZ^jR%u#-^5`XZk00{|b0w6ZgCptP>_5!3R zHotrCyiS3BX9dDAZi5B2q`mXUk7k%e{2xqSLcQE-VRp2T!2D#)q&mgj`BGyQ$!|kA z*w+asF|q5zdDT|3^=Ebuu*gX)%BP|F1XbtR#Q^{zYh(B(<=BXtZ82R@QI}zj#mc3s z7rFpI+AJ#$U7tqLJ3W}-iYWGmlkn(d_(G=A-~S4g=cDGB>5^qyJ9~=i=-K%&|HdJe z4TTp7ILWZ1J)!-y0Xez@Ui;W_-w(0cU*E(D0G#x0HdEKq$_^zKAWm>{X*YT5Eo z4jP}+USWDqoMe3gPYzAw`)yDVh%gfksngPbCxpuRTTJ(lL4&r?QM1mAC84Ffg42D> zW*|1p$eMy49Ekd3Pa_aPXh#PV(%3E2wbk`qwj*)8#h^Cut+d)KXz#mQ)xXVe>LmWw z`PNrW>$z3WJ5f=a{AkPfD_rlk$c5N>W|aOR3K#MzBt>-Ix5jwwJUE}7OUo?{?UxO; zeIRHc+eR++MDq1?as};0<%2IAk|CXGcGunHOf6+Ibb8*ktDkoW)-5t_On2YILm9Z> zx@LCk-Ba7aUq>;Z9!S-2qcp($hh{NToo>WL}J zr&+ie+|ysvt3$EHqo2<@t%h9GCn%hkO#$4YHLC47<6Hn#U7wYnM6E%dERc9=x9Btm z)-n8(HU|$5hh7eb^0P(7XE^S@ktrU0p zfXE6LhCWx9Pn_}OO7^;RVA2)%!*Cv|ZVnyUG@~QQ&A2fKeA?9827%Yk>B?F^KP<*2 z_@!a)&hZ%)FK~V!sCHM*JJVb9FfEOiTrkc-H!*|O*qD}Cqhw}0%o6}Vm#HON^)aMm z4HRl~=|Hlv?(rNxy)St|YxJ_`0}6fM)H4y>oP`u{8J5Y0jPS(TxcR_hr`c zK?hb-fP!+VTis!^jKpI0hr{>Yew&*3VV zFe>u5w-l%!4c@%*Zr!ml5I^hpfC+^G`C86!Zsq&)Q+)7_@t@9>j!m zhcdt#t-lm7+dcmU?4%k4F<_cLKJT4CU@83KbCy)4d%Qrbjb7a9}b>Dr(Z8y50eX-Y|FWBC=s@b>(y|IA z(iCM99^bBYSBtM$iPrUMfZ#1Xt`S?OzppptrAVGCAV4n^7%k~8_#IaWShXss6_k-W zOK@8@&dkivK44_L0ZAfx$>_+Fv(MLjrVce0DRCsvUXrK;1+5017;(@um*WK;>{1GN zGs9^#H3l$l@o`n2KkRWR(o6@~2`7l@r3+|M%E#^C{R2_&Xdi}Qn-~v46Oh1p5e3Ne zYVFhcX^p3c@+Y=C{mBQbPnn7IkG2%dPbMp|GU~;Vr;y8DU(~f5c6p`2_4Sw{B9pnV z0DwzyNhzHwb3efPT32UFQ*a?XtT&j+R${jBs(f;?G9H8(dCQI3#H9$Zm`I8WD+5xe z%zzJU@*fSenvQF8G6{&L7Q1qITFIwWcNZPsNZc#Z{d(KxCDm1gSaF)%ruTxvT^YZf zQ!7yeT=ecTs#^#b)QTj>L~8Ss?$3|>SN3B}y=O3dq)+E^3(hrNQ>MP*_^&v+J;J&X zy<^J<=;l4G?%qGGU7C3`0;0Nr=H{gjpO5(o2x|k*Pcxu3SLJe4EvWMw(cyQiPrizX zV#x{6rNy6L&Y<9@n%v**2}I&G2ppWGR&lDEwo+Qo`)jv-;3|3(M4Tul*)IM4`w;~m zexk3~E}qqlPJs8hIosJeQ)oyWIR&lUq~GZ18`6oj-UbX7O;s)N%`tp`^AS6<5SW{b zrRzPJT20^UouM4+zAlX%}x35S*b%O7;k+y5b_;92s(iJ<=w| zuh@+{un8uXFtp8L5;*a*0z=%UK&vB(`KjdV z6j;5&b|Vg%cs@do&p&9rAw%(dr~QTK&YzP`cC6C=_V5U}G5~@6sZgumB(fgG>-Bi( zq)M#-fr?teL#_q__u&zs{w7$ch!#V?JyEM(7&NEAItI@EtY3NT+^W~gxa`jr^0uz+wO(G z#~rMj6Zpq)t&MQJr6;*;>fCaZX@FR5O6{T5LUPHdo~*OiqaVp-=1Q()fz5YIC~($n zraC6*neq|Fd{0@@V*Qew1i6+u3&_QGhT`1H)m+aj-I#Fu`!b^X%PbCTF%Iq{uB|Rw z*DDB&dSFxCZ>;8bvQ)t@ZV&!wUODU#dm)SyL_ubqa{lxEDreBx8v=|n`A#=-6C&yT zw$BR9c(@=*%JrrM2ab|dUMC0%Mh|8gYA%n6$RB^|rDjJ(SJlGgY%W!kk>R@7FDOPJ zBBkD2*WF$oJb)R79HA7_pc36!3jzZJl7AN~;5L}5S{-(POCCs2J}n`(UbL*N9>cMs zLYKL~VU>L8A+y^^3N~LGDb!YvonJ2QNwGOtJJ$|N5Qq(BQezA)ZKDdy9Vji7Gx=uU zLs610bm@WmSRk@=Ff-Qs(w#mfWpNHL3m(g##(+rS`uP-Dsl>G9sD z6HjtAxhWwvJr2W&<9cgYSaU&rcUh%Zts@hau9s`!aL?9UkbiirrkBu3agS4z4_{p8 zs=?^jUdGirTEjL!Oy~-4p6!o#XF2L#obA$EOgL3bc{#4u@kynj z0;xG%4`e1i0bwE??+afBC8s3uu%UqW*(dRD|EureH6N`O_>T+cKVSHrY{w}dR@>g~ zO&uQ)<6kS;QW3LshLZDLQ0+4be&Zz4wLxiv-!%D?ao*A&x!VS9j5uyndY{=sM%`6L zm7&+~Ar2=Hrik6)<}QyJAN(XDVu4`^aFlPm5gtA1-$O`$@^nfiW8gZ$^(kqv9sKT1 zCK2-Th%`oOnLsuV^U-!u|3XxvT`%AtiQIhpviLT*;zZ=+&gm9GGrP32yW0fQu3ko{ zfrJ%u4Vp|HEUt(lpFRZxwCl;wnn8ND>iTlNdBbu{ncE7g6WrG;{F-189Dm!;{z zONN@-ON`HJN&%vYqaa_{%*b=RaU<^X@}zZgcz@xy;M|;Ip*JdO=BidkmFJ3=|;an6nhv zgpcLgn{;HCSfe>tT!M&P3DdXEfC}4U^=Mxsh)aiSd9C?wwd15mymQ?3XAAk?5}jlU zFKP=-NH5&%xOAp>?ujvPpdW+V{+>CT(;88(2I0KT1Uvhf%i$Eh%UU5zK)?}pLV~Wy z-DCMk^8&}!e9ct?0v=~(df-%^1z~^Fqo3)SaW4?Md-pNu#zA$1G_xS@%A<|yKhTBw zOoHWv6^&56LA=1ZLk3@My~v$+)zyv3MLmxPZeJd?QODN6+^C`1!|d)NV>OHZAR@;s zrE}*QfxywxD~NBSg|o1y4-N@?YR}HdtLyd;og2F2BXG#q%(^Cu=_sE+zs))9I{R+p z_w>|EOr70*m>ZYd$~_}W!;p0i-S_&~n1;oa8|L{*UejV%IU=cJ-_&s=L&|u_Q}3G( zbAFSerxc(j_s}UUzI_+oLT*~zqjxI zd=B^Aer7xxug(m;#=Wmok}V3-Ede&p9@O-9q1T*w(NQyP7>k#0?L8Ik>hXd8W2_w6 zK0K#+HB%?>#@83ne5dA?mN8~&dHFT~rQwIJ4w9WzIp5lUK`3qpq;9ur{Qr(fz=pqJl^8_1|XY zxqTl@#)7d4bKvzFeq5^)h7<}xw3|xZ03L2Q8_?CEZ{MP7RXMcBL?+t_%|R~{Ri(*#z7h8WHfg)Z^TNFD3k-BVt?9_{YQ;jAoD zEQ*uVDL1eS!cDea8ugrURggu(b7j!cou_d|cfsa`Q2qNNsNtVIGv3$Zztv-Bnh?8C;67SfsuW}XzKyVyXfWvR(jrEpXm6XbKW889-JN?Lnu zqH@(cr+C@ET5G!>IbzMP?5sCURvlX~-}XY5vHWwQYB0BZey+3TVSd;q(dIM#^z+mC zixNYr+L>a(^csGg8J6*&NuvvpqY2fh@q%!mG67*l($bNHi+{z?sOrK# zv9sWO-AJN2%LwNU;?q+X&k@&!p4koO%hO0X!|H-67k;nW#YM~0i<6XoQYqEULtE7) zojhAmvL1=M7z|fo!`b++|jWY-HO9`uf;2w$_uM+^kvrzJZ)@YjW z%;rKgRm0^4amdGyJ6>zO$@hLcpl2$F#$z%uF)%>H8+Y#V2ORET3bI+7bJ$!S2QLT^ zPx7MXRtv<3dC^UFKp~%3elqJ^&u`QG*gu5bk7d!iE~|bbW{qcrY5sP%XjC31l9#-$ZszP2IAAe0BTFb6A(w`Yj-}& zkSlAe21^66PG-RFZjXP{z`*uJS^n8MMZNB~%>;W%FRy*i{`mVn&Owbiwog`2q=DiG7>l5_N8*uaI0Z}%l7j_XYd)A z{M_tuZ~gbZpLvXrNrCvE-TwDiPVeRW=oTRl-n{tzBtXLECFWPox@*fCzP-thKcb^$ z<1XtQqR<8em<^0;hc>*93;t7DkVePGk}dxD?E5@bOqjXM^Q=XW4KSE^q}zken>aut z&ky#ltd5k_`+vcpMHDvL89A=~5Ao!nkY_g^X8yiV7IF0pcy3zW6VlVuo#yR*$9pd&C(B|9% z!W)wDE8|SM3JT}V%Z_P7TpQL5-S5e4n4Pkel{2`Y;UeKMx(dxAdQgiKn4aF$*OMkE zTLlE~m0Y;GGUOR=bL}E6-7IeBLN-!r;phmBm9@+`D5%q-KcwE9 z-81goaM<0tV0i<1P9dfAAMl@;Nkv__oEcH}4ZfbtnZ6a-GtGvswDCop0) z?SVg04ukB0DgCK?Njg_jMu7{KJsRn+jrbJu7oYsVM}VB zDj|dLYxqr5Ur$u;*zMDv5(WgUZK+7H-I*jz85h&JgM4~3>bv3BuA*as+Z7e=v-fdv zl6={cYc0%Vemu@_??Ak>O9f@QUL*(wpq$$zDee#8sI{9 z*C;i62dT{ZkQ2)_!2miUq%{kYb#ZZUsP*0=t8-D&ZaRSdz!*(HKtL@}Ymid*<;%Ub z$x3`k4ZOE`^WI&vE1-Kymy3T7Bi9;Db3dW(W+Zxvz^t*K%gYQ}Hh-QYBxy%ye zKYrv(X6C_;S?Qog>wppp2#sZ&rh7Nye+WP$W>z|n|6PrU;MYp<2aB`qFmao8jTX;e z$l!)|#8vu`#!zChj2$SdmDcUP8Q(l}agZ}$Sea94wYn&h@sLNW>`QZ5Zcff5t+wkW z*Z{9xx6N_ZlDeOv@k$1~nrSTy6Kcz}{^L*eGsK!AW{PiB9A{JKEP2f?W;lsSU`Nah z6hyCJDO9za)31N(NYrK|Zkn1&o3@l(=rYMANsA>9tDPjHN!$#bs87Fhpv8yx!1xYW zESX;hmBx)3H5{*(hvP-~E=7g;?j&4f?VLp9e6gtwvkz2foV?Wi#dv$KURx=0DETtK zu+}|ax?`^uZZhNEu_%?yQR^AP!D);@!Wnm^nh07MnN+ee`FBJJyYf$+)KKHq{I zfk7siOVihg1{eqdQPdS1i>afN?Ctq5{HCRV4$S64CcS*R3S)3OekyxN~*8FAtA$j1+g|x=zmL)e+AwluN`gy@FYqC(9CSB2_3vgGob0?zRr9Zm2sJBi!&$VOE?_n&?}GyvgLumuwQq zU6|5JKtd7#(-nyz02ADvAG zROtPE8K_BMkd#UtST>`VloeJ`z=4oDAD9E5^!00ROab6ub_-HSJ`Y9?{EvqH?h=%( zpLp$QA))I%sO8NsU}DJZ`g#Z?58gm}z^4dtaC`tE2YaFRpVAfn^3TsE3A5*H$Q`)% zWOrBTg|1ox$#epEWlSF#mqu;xt3bGE%?~B{(l+jy;B4IwAwNdcj3X}Aif2Eur`A5{ z8}0Z`-*8Jvt2})Ptb!^zt-`>gX%r+Sl?cz9Wm-~DZlDA$aAuSY#Tgs(ddbT;IX!z( zsE+FFT-mkLaq2Ak79(F**X9z1tH|H6-ey}lrDPQ94ldd&K#bRa!$mQZPm?{_Ry$2v z{RC%L7Fnr(BW+%LUqjQ(LfyDSR*7KEo5`(?VadZG9b@llfv#7UyeTr)+4dBbQhstY zjSzDWUI|F+h<~_TQ6oeNf&2|;6Gu$9op)@HPNiMe7N)ttv|GPLiSp@ENWXw=rMi2F z^2TrPYB9!+x=VLb6|1sSGs%SM*{YV~GULdN_oPySac?GEX^^{+8szsij|DtDEDEjy zo7e_q&hVb@ML)6k<*8n^CHG*dnwHk<{e4i7@Yqln4)3^vSp-_DYgr`*Xa4~H>ldqe zT>3`U{E3)YpyQbCh20hJ$Az< zN?xs#=X(o*7-@OFMB{1vM1||aTXgcOYfOME`++pye^qU|45;)H>-w7E9lrQJ+H;Uc zHThE|l%?07I}wP5V%Z;^o2M#DK37+-lOHjj0-tmc*!|a|sy%ooPO3~G=p_5LAszKM zGzePwwHS4nBb7Fc=(;05f%1y zGg+2@`>g-mhSyuRFIE3bk_*D46z}5Fbj9(@{y^}(l{WZ5H@zYT2)ebYtnhd}B zv=2YqZMb3Uq6YOg;5;wBcV)}yEm;i#`HuwT-iks0EaL|TWpc;>Z-ub&Y?%`;7^2kN z)@BIG*{%IjAyew9`rclJSO-@JlkZ;2Crz^uT0>?=5C-M7#Lzs-xsr^p;b0cX@{nya3Exv4x0QU^p4kO9RafVoq7jK zEvMC%7Q#oST3wEs+{9}6MuyD+QP`0of$fELRujXRcf;2&`s!-S_ZZuJshU?>^tZiSvA_@-!>SCNjCC;;ekLF*hAIn&ygz`}h~7f(7a?^XCG$ z7Kk_WJ+(ikT5_~S%P_nXwnjyKT$aTiV(6~Dbl+CCy5Vs9b|@=NQ@F!@GU>z#j3(<% zs#|ksAqIRa9141VzrIy~FRZg?FXFWp1H=q3ix zEq+lvHO!u>X7%*=$Qd=X9PBuK$Qi?7`iO89BG5IZm+jQ@dAz(bJv?tA*2i5(6pF0C zG7XBzmKaJNHWH~c`wT^8WzvW3MsxHQ|5<;3e%aIe`QyJl{=j^ih!vx0&OiFz&&&1u zDy~3Hkprzo$iO$##jE2XER))y;(x<%VzR`aYT;~rqfYqqOd)xmrw1-jIrRe4!Pp5S z&bOzWXYK%bN`&bhWNPZ=<)uJRz~u_=D$b983s<+TOk^MqU?xwT@e-gDOK@Gh{npQK z{1;h>n8oND? zXLEXBi4>c_hnWl1axZjayNJ&A1}OYjEXNEk`CN~-qJ~H4a%80-d6qtN~G&;#r z@%B4-)l13_sq3B|XMteAtX%F@7B2ijzQH;f+Cs@#3i8w>Wbp@bJXHScrGnX~D zbLb_E9RHYAb5Ue~2`L@+W>_KjYfM^}=(~q#?wnklyeJe6{tzlSyay&p?8HQC@*J0L znR`TczC>(#w(KOe!#z+JB_X=CucF2fp^+tXdN8d(|HWyd3vS(^9NzBhJ9mmKUIfd! z%{H=z984eTFo}!03})3e=rw`>HTM-26=8x$QK^yv{`lvI%WNc!`MfJtR%fk>eu6rR*T$VvBx=t{7(djt;qo9336xWI}c`buO!`y$al@s5~Z* zVWvPt_&c0{Hr>1f(ge0jCTESlopvA{| z{xDqG;@j(w269-eK001f4*SnXA4rHQOL9}8xg9gU0~!E6GQ}Xl_|roZY_TxPn}N+v z2s`jOXjeOdPc9^mr0z8UJV$g5zkKyL(WRu zA;xO7&XXTfo9`o2nUEhhGvF&EFyCG_q1~tvlJPzyCO-HhCHO3Yyk-ZoCvO#l2o<8R zuM-gdIggu2xmmAFPZRb$^g7#q@kwS;!wdl~KlQ@9tyU9C=_Rj~H3SKt3JWu+=9lPZ zWFax$;gQpWmU&RCp`>OhxK{Y|E4I;==$5W9LiP;>2Fn#!I$~fJu7X!7Hp7A4r|8ble^0C(O)JkUH{lwy~T4 zCa!9{BN@KFr|0R2vJwB+0xmSL6l#n4X$};c420XUEY*!;nQdN8jM^C9Yj+&k$PqBv z#ytG`m-+Mf-0j2Uk4O3l8^LzlvVeR1;Fe#%EOyeFhx3a6s$5tyy94EO+uSSP*y-Z% zCvIIYtz+%zpKE15?Fl1XTE?;G>5IYMoV}Hnb5>{Qs_-$2#AbKok$IK)E$@I9mD7a| zL(b*>P;K{fz^i%{o<4pi-s9V{6Vv3qSvxY5?&Wc25WA3&Ksa+1Xfimc4P0+N^;dm7 zK|-`>Rm{xime7>Q#FT?t?bM<%FjaLt8!oOf*%o6rS!&LUJaWzXz&{Br+K^&uws!R# zFj(8n>v6mkNeU9HgRQ_$XAV>e*%?*IoN%9=-1TKoel{>o`+QH23Z` z)5ibv&s~LkfBw8^i2UDBs=O{EoWT+#^$najwEa0nYd;%&d0e)r%ju9?09U}~C527Z1xp*i4vGVq|ni6ujB`vo%*q@b)Qx7|)~ z%N77427#cBKr4Q^E(X_d{CMwbh+G&G^N*>MDzL1!g^hS!y8I_}tTZYs%M%e0Xlvol zxnwy-PJ;Ea=4ftKV-qRmYB=3YEJ{B=ANs_@UL?f}R5>_t3}p*IO#w2W0(+U8h2BCI z&5j-)C;3NjPPLX4r5}2DQGT|C0sBaIw!@Axs5Ni2Q4lX9(8w5^D-FF(n_E^JCzHjvdFC-(nc>*$@|F}6xmV&7t~yG_jT zAwpT%{J2(ER9MaEGEVK&iz~(L+cTa0)(bCmJ5>}xIcFKgq-8|1rHo;IGP*+2953%& zabEp)JaJlcWUwy8$v3Hau@3mUFFz zMTew2q3dhDvJ17@QDWEt?1-%Q`dM5s-UliZP;=r|8Y$a0ES^Z-wO6=Cxf3bA>3lR=JU*JXkP*ZJx`K6 z-6yWQNBr{-#11{zzw&p3=^;Z;0OuIqTS^q4g)CfXPGrpCUTcM5#orUo6rq%|VnIZ# zTL-G+mZzYfy$g66LP0^{{p+jV-(on`^f`tu_$AD;(RDp=-#M*EO*HqZe^F$QVx_nm z_~p#C2a}kiuY400N~Br&2E?3#vBEGQSZigk0;YsqhnChY2o3Y#4y>K=A~^jFzJH*4 zuAp$E(6=Yg&5aS+T5uwjLpa20KNY4HtMX#6!rp^`)k2qTl;8U*iNpaxOhAjqhDWBA zE+U1GcOq1T>KbZX2dc; zM_Y1cj_rf}H;}D{NP=4tQW|XZ)tG-cg%&sF;>Brs-@N=>a`V-Q{VJig>^(*H^qPk_ zb2BHMC1jOHC(Bb3H6M-3txobDubpnXdT`k0C>h#)^nOd{_jsTsru#k9w}0NbhkI~l zwDS2W(y(rXwv^Q;W;?m|Tn|T@(ViS{mx48yXKM)UcaV8~L0(C-Pv`0U_S9T0&wkUs zn@3EHBiTsD$+$(4UX!lZ>$0ya+g8jE^?OW~Y~1E=29S>m^jKEEyObICi&!lXj!4Zb zU0b-i&2u?mXMtARFCT}1DlpGgA7cJoja&qTWCzOE-d!C%-h3!gA8Ar<0Wl%Nbxx?y z!9KhTT7xF4i{9e#9m7^pb!6R+%qigIT-!vNpnmto>a%^Ks5?lp>x^zw*JX-Sgw(V` zo%J}uct1)3Rm1a>ReFZ3+L@5|9Zw)z+;-cbg`Jq>H^flooy1vKDk^b8h-E{}#soJG z5Y0nbA*~#)JMu2uEs2n=qixQcH|zsZyabC~6Xp?@NCV~ai&N)s1$9*rN~ok?sBXq= ze{9Dk7CagNWow=*1d}SfedjuQjs*n0-uH!tk&2cU^kB-`+au-`WiJZnZb0TrCTE_q z)OI)jIiy&E$@-ZD*scoO%I@BOrey32NpQ$j*yPXD{mOpPL3AXab5j$_5`%&CSbCPc z1+9p@mKs<0Em-Ket^*n6O@mqMW+CPFOOZ(BK|lygW+^!`Ljc$-E=Zn3HP=DnqLv!# zo5Fu@;lG~;wIu$ZG>sVlGe!I|I#Y#R!*Ws4(qhCYWkExYQ^J;_vd||2POLQ1;N@^i zb^9E|JL5PWpMdP057CGCQh}ph`p6R^g;tC1n5PSoiwH(NHymxx z=fr=E2nZkzgRnV>vJMBb9CCupFJJ^7E+r-8jc5!h{y8@@k8N$k4LhR(K}jo1&&-U5 zfRrWht{cp*F#1Bo#@HC&@BIb*cnDj~)jlO5Vx2@{a)>92O~v6X8Wnn-QFI^%cnR8t zH{2;PYyRONCY1fK#TOPOEwfzH0un)`mc^ zv-85K`X2f%CU5iHcJi31ij`RUzexgnA=AyStfq# zqne;l*{RR*DS6=I9sW9;SKRjDw*2+NVZ|U0Cb=}DT01$stTCdJS~h zL|fQSzKzfPAc}4LiU5Zp<_C}aWN5cxuxXZ7#nP&HZSPe8er%Ts{T58>^0K zisY#$&j)b=pF#6{XZqi-H+3Z(xrC<2DD(X)J>v)c1rF`f75)q=c1Q;7*rZN#bG zI_TTpaT&I)hrx*J9WwH zpW4ZqDtoDiTDI#qu8jf-Q1R(l5O*X|9qC`PTFDk0)7{?OG=ui=LJ&@hi`%Un4qfFf5gmUNi|I{rBx+%IIXIz-@%4ca?K>bHeT+BimzET609 z2E5KvyeziSk9KRD?sed!0jBjX@-bevF9IT5;PNpbJQVB?0O;X@DObRKE<84;IE&x@ zr90A^=`;gr4C-eWhe9-90BZ90Pmw0p1u_fnt`AYF!GNT0uNJ&wQ{%%%O2TwplB=I4 zCaW;Xdt9EI{)jCnM&N)&-)c&A7VM;kkZ)X3*&Ysm56&nEYxk3#vaY8HEVHboa7*+% z=wPbd+8^md3QH*X^8sV91Vs_Th=Uy>W17N*ra`UoOn15{jZE;|K|Rm1wfWc2F7B|V zAr-&bCe0LRr%}-=MHmTbn52Uqn)KWLd~(I1>iyV=tJ)*aWvwBhfo0NV& zpk~HlXs?C4Cu>TLde}~S0glNPVMhzCq=zy zqv^kzzkgdY^bd@4{nbc5MS}N*E8u$a$Qbt*QQ_gQk!i^w>ij*FLrgblkC&!+-P_yS z$k_PB?mrO`5F*E@2fhyEN66D;g(u8JO%`05yO4$j*%$9H->+E0ybc>%+Xle#^|Bt( z^aG~<)oEP=iMLA`pVP(vt@O>LK1$t(sUPoveop!F#lWC2GZTzuC%}fnfCF);?UNL< z*x=@Zl)A=6?(gs{bD)Fq&&kR0=|h(KxV*X$a=o?NGXQch%E_9~CS4C$xghvmPxkaE z^D9KO!`;M&)PNjsZ0wLuYybzFA>Y#)A#c@|r+Ey1dZU|BEY))g7I=X+hY3)H2;h>h z`I#h)1tG!uCy@USa)z}*P?`xH8%j6|_JAj^edcwNIWkfs;D8W64S zu{b))+*&OtW;f}!0wNRx0VVWTh-$Hz_UN>dnN&TLXUw1B!&+Soo5>>&<1C;QlY>0e zB(_x4rSci4%(~3it+#JOAL??HQ@DJNPk+8%6Yll!)^cPw183_!qK(_`fy%+Ix{u#m zQYAFLi(lvwdBOZd#DNHp{P$|k{9INuCQ}>e-NR;K5Cj4ufKb@|@o??w`>Z@|ujc+q z&wOctQiKnL#{FZrI4$QqDc(St;Py={9}`Ss2ut{%yk8Vf6a_DG0;ETo8}SS3@Ij)ewtfgIj8IetW^e%+84w^>zYL?p?Wiw<`42aLZscb%qr5^}5?;%6;=c;a~t z<&~WS*D`3?v4=Wkhp9(in-d{S$GXWTyh*|+1+JK}l!F=^i~Q&+h<6YO2(*C#ulsG{ z0NIX~tzWOXykPJ*pjNJw(<;dfQW_sJ3*m8L!^OBYzUe$se%M;&I1dNG`e|b=GZ7{n zX@%+9T1H6p&@oiy8dxngVMICDr?xVL_F->h7m@Q$j&|!=8 z?jqL$#J@pka#opz%~vLDiWqDmq0*<8#KdG@?M;}SHiGjCgC$UzbtVbHy?#Z?cq&~! zIIvwfIDYqB`>Eesy@ifKq}*n}{lCr*PGi~mZi2r>2%NZLk{=Y0~yg@ zKIFC6`SBjHsm1gLk@}Y_zIheC@nJjv&fSv`87!F~2l!ui9q9j@19G#Ai+*FpCSc%U z{~%#eU=OlRr_JdQu&j#Y4Ba4@^`-lNa4nF33j?$T#cLI*l#+cfOp-rsF14ua&2`>lM>({IRMqI;kXz)Q1KKd1rT;b&@*cu!Nxvj!p zb<-Ln14_@$Cn)%ikVYNm7aTX>Covk524ru|$qEgO2za%(LE$o!xb7trGl*_$8??O` z>)Ds`PCC!nn1dZ18w+VBYRJ>XB)dmhs^OO^Zt@cxgReET&y99iSvh_r&fJcE(DSq> z0macwmo@_ABJHC!t}yk$+8;uUe=gGcOZ-wlU#!WCExJ52;TbNV3j6;U`|@}w-!JS* zi^@`|5Tb-+7b0aT`M6#7i5@i<(WnUxvF4>cHC`*)Ni%KM<-g7^+eSh!! z`Ml2`Q!{3sndg4)``p*L&UMa#*E5V}p^7gQw4wE>Y zO&E)#! zeJkgO3T0$BS%uEi-_y9IT4!MX?!u6*z2+oaV+F6Q`48PkEd$}bRh2`lJrlO3z-r3&HP4E9;H}k;^vtcqQ%uAyaEZ5#>aH8kjozEe(c_j$h^sjnyxE)_^ zC*093Qa$ayV`ZGxO+v_6Pl!?Ar_i2*_dRDG_@P?2Zuk=?i|rU7N8gS8N5jt7hj6*} z@wd&$%AOJp^lkNr`P^=MCX!z5Cwk6?XC}G7Zw-n5QEU3`dU>Co>R4h;$CA;fx%UcN z31j-hpEmDx6R%swr-YkNe23z-`xnX|Oni1{Ir+gbb%U?`rfaT?jp1&jZ@ce@b=F;3 z<%i(=_=~n$@C*)Sn$;ZO|Y?41iPEU>h_!Wxe z=b4_eDsY&YR9iwa9Z=6Wy?Ht)Y9g?)Cge~vFf0WrR?i8f!orvC4_I?u7{B=R<}U6x zZ_R(=JAsh1dN6qnddYn2tG*%0cEL_L4m6|)o;)iw9CyFWnMY8kF|4k_RsZxl<3E}U zRlIUZk=`5XF%#VlTh5v*$M#aw@V==3j$huH2u~p3I12 zkMs`Vysgo_E&F%Ps<`>?Lx$<$N(cZF*!!7JoG5QOs;jH393!|b8(&CuAfHsp#YND2 zdGfGrKa{@Bt(e;`9UP2X3&Qr?_F5q7HZ?crr5rx<;5Ls@%CJ$E+DL8#yE#a0LE;{2Zv0jhVj6-r4N##=Vg%aj99ydM z-_Z!27B}6(3gODT!$y#3R6f=_AFx}1-_i7O-GZj3kcgM){(JXWo7#D=8}i%$ml)D> zCI(;8czb_6EKZ{cc~4MfYW5dF9XHO&)sTIslh`vu4Cxx;@IzY-PqhC=CLS(6BRj>_^t@i-`&A~)S@jNIa* z_uf^#E{4+q{m0CjOj}wyHaZB7!Y)XE198dDU2iU#%!FkR1<)Bba~7I(Y`Ct^Dqj^! zp_0S6{7^&wELGaz7<7%2lC~_}fo=BZd=H`-tihF{u@C34!zSO`dk}5N-JwkfLY-3DS33)(NzFKtp^&+($IJJ_d zH+v^Yz)FGpdEskgnmRxMr*Q$byFj{klHbA6TyvSb4x7D4v2_Q12$Z)iJwmrlD_&EKb+=|rP>-hmkp zo};7Np;dx^-gENnfmo{qn`w%6$v)BId-v6DhT9%zf6-lcman#6X&aQiF8DA-nkPMo zi+}i;>#~^y__w4u@_T*zMvXn5C3-+k4sw*^jmeG7_TApH8|oy6=Qw}6R(A05ec|w& zu|6B4ioLWi>ehviUv4Cu*m4N6gdUnOoOy8yH3;G!2J+c*b%$00U0QV>QowRAxV zQt%~ep_t6wox1aryT5f9`^U6V4?^u^gW|c6(1)i$%0RwFU=Zp6e${z+!OFyCW{R}g zYyJ?xo4Dg)!BA-9-T3suumISb$Y5f0RQv2q(imS*FD(^~6A8HmeLf(_`DQX*(!GRq z%MQvbu?Amg+d-}E87C1R0DpaEzU#Eq(5Hzau0HGDc3Y%2;>^n zz*2{N-nn4`7}8s}e&A?m#Gk0XBZnfVkdhwM>jEKGX!x>~c9+mKza#Zf{Rv&DdYFY^ z^rzy)X|s-4koCMdb0VdtzI$oZd45jQTDa=M*VMy(wgH~be@2^MfHJQ zon9~RrJc&Fg*2WW$*QjvZ!o~}d$OHm*fj8;;Du?|G@Y#|fcb{zDva(!%!(=u+*&iPO-oyLFy78FD(`={ckkXe5TBHUKIKSx(rIKG|GN&%AQ|=1s=a%~4Nam~gm_ zRu``YTGicdO*EBQ&mjE0^|-Q7i{k#wLG(Wx4}^Bv2nj;?Yn0a^wGnS|$;YDTarLg# z!X~T@*UGTKQ$Bwp&&2y~zF#Ax4a$UiGTsW`rluy2R>)jJt$pGod>E;J$f{d`fac)A zgP1ahLuHVL;e^2_Ie{`qu2} zrUJWHU2GC!V*8;v@#s%=2;urZe#}SCBtFP?=20iqVVU`miXzk#SD@LB@AKzWscWg%xZ|_Rrn^=(Hiq2c>IH+etJ3N&6KDMBtdmIlH>n zg9@Gh+;dua7OGfF*+;t|bGzY=#E-4nI-F|P+H?toG!-PNO^{!oKT-JXB86#}@576J z4fXY#q4*dK+%=UdP<}uO9mgNvFI{0?Ga$}9=~}n;37o5GZEa=y33(nOE)#dxluLeF zC{AbUWZTW5J|{q)Z^H_s?F#t<))2yTAM+gFh@&AU zs<84|$gVU{M%_$%rKxmWn*vIL%8K{x|5@>g{I!@UnWFF|zseme7+?R_B+B4?ZU3Ln zRmVbaQHX@R*ybLP=CuDzA9Lx;1F+(;dNjc`VG5XV>2QA{mOiL0S!agktP>7#N5?l99+jH%m8A z=pZBo2`ueCNZi=m+>HG}TKe5dDEb6}A!wMz{b_EQ$k?d~aaFj_)N7N?D1tSsUGaCAyBA2@)6)`qgsuRfED zlZU6FztA?%=chMqknDzsw%!mTIY^~V{^5L{-rdMZHq-U*3X;m!8Ax))iVjGHDJDlB zwcPHpFr{tz3i`i2cKc|o$5k=C*3lqF;!n*1zU!#wfTw50nuW^A$%#IBDPQHmE4u~& zh~HLN&~Fm5J*rPUvfKHBE9lry8$V!N`~d;BlC5##Vz5O$2-*#%cVk0gj5m~>qWk}v zLAaK~w}-;?$b-OEvrEt4?V^4DGEDFJw(?Cqb-LG`5_3-bpORKRmXpls@htS*`fMZLh4;@ncDHP-ftHKNZlATXBHL;7g*mDf@J|yA22P{VsM*fSWCBL`8|sO0bhX-(7?NUZRhL;OLiSUetc~f71|IpTFz@h@A+?& zsqx>GDX~#80L7Jz@^b+sadn?-?wx=dEe{@Smrphawij}Vped?k#Sf43abhRvp|;m4 z(*yAVY|I%@LMJujATw2H=IvEJbfUiwVo=l5)W$WN@#}GGO@yJ?#lw5imbt5;*hPkh zDY?V!WKYifi1^*lE1uj;)88HF+h|ah_^9|av?4@DOvT%`=7{VpxiGF8VF~rMAYrcI z(4CVcA@VGeE^T#C*2Zym@VIRmSbEUyr}(b_2%(B+&!n>qOS@up*!Mt?UHBT=O}aMV zJo4N=zUEj0FC|*sOZ1&{DNcRu6UY0Y#IF;Gm}5^Ovk$d*Yzpy+nb?YZeojCq*Ek4H z82cTh|4`DsYJKg&oNc=f?FFiN$DEb7F*9$tDN_?)w)9C~<}bs8{+qEo@4s|#$R|Nd zHU>p*WM`OhHHke`1adaeB&E5|>6qg=-J0CpcBwyMr3vKAjyNIe!HKp#IMVbT!D?se zKS9a4zpOTj27>TTP2=gXx-Hdu`^78uPeU>=WzpcY}e;nCt(je2QxDX4nmR8<{qul zcT6IVAlkG3dnn{tu4aaBRNT31Tdg+74fu8F+Wt9g&bc>YmDqkymQJY`=6R+j&islh zaqN*dV{T}KZC(kzb^Js*nO+_iGUUz1}mAUNooZVo*L zK7IL;ng1P1LPJ5Z^SkKjIVD|GOwjIy&m=NSNSOQvK#i4QKS_Lzb7>AxQ9*Ow`&bs^ z*`u-FdwR6EOsW`QZTxDm@{R!zLKvIMX;ae#6>kjL&Nm18lik z6MT#wS8VpZFZz7VKdyM+E+HC5Xa4sEpQRJ<=OBp&9on4hrWNjE#{ZhuGT`3cz`6;G zii_j=Z9%XqcHPho8H$Jp+`j#+;?kFl4`k5JL9fd5c>a?ozwiEW$TK_lB5s?Tn_KL6 zP~49~oO#`RI1JH2?+B=3*WM15JIbNY-?~$cmg+{`eT^Xpn5jrsO#Zp$F#C7BcKUbg zo!{9Ve$SD;P#XM)n;EX`g`4R76wsBwR^0Z!XyID5BH}8~`d4RscVEz%bH@O5nF#{C zP=p3%{|MA?uv|ih(`8aa^{8=xJj-cxyN&-U+$RJ%Y$TUNN7{{RR16HaH%Ba)txcP)A?p|;VUpvCN%YN4on6{D^zBcI?8MFx5>{y4y`>!_tN3CEBTM~r=h&)N zW>3VOa0{+B`Ofw~JgpqsJ_|8~e-{*s0Hsbtl&1gPw2LSV+m_<dsBtqxI%_++F zG+2EAqe5d2Y82`zQ(_J=Z3#c3#u0IB30{|ymA!#BRqRmL9M8VHbw3GQP_sevkAKQD z!W@_i76@7U+3WA$C6;79r?|`DEJCFZ@%7aEc82M4xX^BrJhIEeBk4rhGCQ(svzZ18A?Ba1Bi=-U7k;F8RDHC!(jmYkY_}!vX z|B@hpcjUUiyF_R&Sl2t>(jmgRUlAN1$D53J#el5>`0~+R5__eY83VC#45}X z&DI?`Qd{Oa{rLQ_I0hqkKTu|qLgxA3b^p6N_GuMh^BOhJh+PMBrK4X2@bmlkFYR$V zW3b;A!~IJkcoaN1Xqgx*eE!BOJ8XrJ_(E>{L3Z{gwvRAvZUzTav!UgI4x9YJF>}YJ ztsv-O-4(I;UqdhLT7QD_*R`+x%XoAj{t(Z*38cqQQ{{O-5t8U7a`>x5#?zs(=)M=b z#T5fU%rEBR8A0{jc09mQxD=kEH6#|Eq;UkCCx87X=W6TglUwE>Yu;;YprBZniHnnUjvGYP!teNr752XiE2r?RsZ5^eSt4km9vAhZbRkghE9AOEG|!7p*!rza__9gZKqh% ztM&{{#}940d(AmbBZ8yCr^H~^B7n=9(^JCMy7tG6(fCOYMN;ki7rt)swWW@jAFtqG z%RFzUVR7H1%p~9AaH)#ihW;RN1%8hVlcQmkP$m$^EeSc?3tdXQ=>YN;mbXKxk>%;! z>DcpC)^71u5L18E53SK>W{<;raUE6eCm9U}J}%NLUX5(X=Nj_)xmq^<;PTYM45*Zt zJU?B5#z0rjqPIGZzJ2>v5Xzujsx&rOE&o{64XJAs$s{;=4I7&bu)*e;*H0hx`czAr zk)16Rz8!rAS0HUY_=FPIc59Red%pHiVoceMe(JBI&B{kXa@>o{W@Cb@sP6u*AECaJ zwBG{sQl+{tvs$}YwMUR{EIIh-#AU%s-G}D)4VOvblLdRA7Y6Er2E($>|bSA6*$^6~6uVq)lk{n@9u5q+D=$5gVsl~r#kg$^qH$Ycob5n!Ll z=W6YBd)Kt9a`E-$jP!K=a7y%E2W7Y}kPEP;c30hf3j*I1i^kx6ikoqFlK^K=zmGj% zMC389NQ25R^opC%Cvfg}=Xe<3l zhsp&Vv7g_+OhNgX${*lKO-JY&cLwY}OrB)x4Iu|bMa6ug^Mwl($>* zt=PBW^T7$Vfw~#aqBDVkablNs&Q~r(W@KcLv!W65r4NgUNCv9xrB0yiBYd9ubtrz- zb`9=w=xp^*Z`awuS32PJ7X!4mV&8>b3&C1rG)|Xs6OJHgoyXwz*Nv@NaEiu;5_=yG zM_@GI{q~MbMx_zH=Pj8`A$NTJAU@-P0)LF~0qvpPgDH=2_Ogqcir-VnYquxScB`Wg z8lFm3y+02oZ@Or}A+t}{$S8Yp*vDkLFBj^L9le`#33x4RPTvScZac;G27q{ zw~lq)-Y4+x;=sqxK$+dY6&Q$SIy$%&8bwkkx8gc-90xaJudF@vwVH|k^)z8}?~+u% zk(Pz%7M$QMBN|&Rt?sWha}L>5gOp;uhZWo`-7W7?ZY(g5Oe#NG;gNMFlxom$3Y*u| zJW)zu=z4aaIhhE_r5>wLI5h~GCb;gUrlync^91JDer9mcP)Y%PU&7=iw1UUK9H9%M zQ}m0>k-NhfBVeUk*-im5r$WF^-~iDpZb9!AexFl5+XXUB`@&ndwsY&}OV^rqsBxgm zFlCIMpH3NY(~{dj8ID1_@!W{B+R96h=pTSe$ka1>}5PX9Fo zGc`0cp60I(;p}C!x_ZBeck?^$qipS(;4yBSjm?r4j$+@?$I&^{G3pg2R2Ha9*?M$8 z`jQ&giSE=p#vYTbu#j?%P0Y*^AuUA6>r0!k{l(hl=g*Z5%G1*1y{m`S=+L(k@W#1) zy}{DVTou8sQ(eh{@7rQi0X!H~mX+brTm~lHf{hlvbr9E@F9=SerIppd6b+AIM~skd z|A$g_9D(8n<^GIahh$&cEG;edKQ{AAltv%GB_ul@=W_idf@bK&M2k309?}apvA?W6 z_)gbNN&I=<(QrRAhxplZKAi=V>ti!3J!AocLj@f612cv z#RhM{shGE3ihtOd1{X|{+nP98NhNgr#Hl>hRu_hZ`dhB}n7I21@Zy-OlQk1uK!=WA zWU|5eN^NiiJS#UhvQC0ihJS2&C74OV;eb)MlPDC;Da_1{jEb^ir$Co1m<{{+afYv7 zPKd|1tI-v(L7P9A+QCp<%OTUh&Wzin%CP+6%+YXi^zNMazG>p<9f=$$(`DcV^khXl z6yhFqC`MY1Qp2yC9icGlp@_&xIxykrz+dHI>_f)dL63&F(ExjG?ZI(Xul{di{aABv zVN3S*d+4H>Q!{i@Ic-%MYqTG8DM@vW&JX(TSx|M34c=HFQW-Yr;b3~}Mp9#QK3BIB zI-B4XNHm-*9e>b0Q#@>(n+uZ^pkOa-RC~7<-fzF~1np6ORm@&Hrf?!6WZJ!M|G`D0Q-RMHMsb*oaBhl~N^XP;Ab>hS~fGb^~E1y(4 z6pt2l?MhMzvn!N5kg%6ihn<0ve*L=l2>*rJma7djLK_88Y1_Qdf zIGlUC)oX?i$&h_^LlGB8S+u@jPiivC-%`5a0k56PF z1i}1;Yf6L#+A}>+;`B&-<%2*@5IS?m)ch1tDeUl>A|i9h%}8F894K*tt`{QYeVAqY zGTuJ@rCmZed@FiCF1ojQ)cg${j$lryuwRU)n;(01?Lm^W!PWTB^A6el9299COvk{= z#b_l;WYr~~0wYGj)!KD;UPMgzfDGK6IgNp^S%H^OYOXjp(?&JU{se zTHR)Xm=I7G8QB7$E?c<7Z0|Rp!NoH;K}((iRXhGRVqaHQZ-gckI*TyG9JRa!ttd5f zKThXX_kx+)LwSK4*)ZQ;%Xdv7bdZ{wt)r>=s6LRUb^HqgAv3X-`Z0pmp^&U?zmQw~ z0}pop7wU?B=d3q1;uHh()gDiJ2N|)zKbMK~ogU}g|JU=5Z%;eZii}M2&*5h3ddY~} zi|00%4?KECo60y7k<|QZ+gwSsUeDH)g~=q&JR z`RSX-;;ZwT+{P7p+S=MTs9u*q6};r6q?a~e$j&`QUn;r#i~Gr@!Jx*Hnq#astG|Aaf9rF0xifQ)P(IGaAkinm<6%riAR87`kv(K zKam|98@r)_K7+%B9j(aW+yU(wg@d8O>+fSk_P8>(qpiOpSmnpMA!75P6bqc>6LvCL zmoyer(JeWCmilm0Q)#xO*qrqj(NEd9L8U2FFkdRpm?1jj;G|PNm+@yfgXCod!Ai%d zD?dFYn;I;=Q27Hk>?q!R)I*I&$IR7QCF zET9Olq7tN&W0>wJV#mH0ogK%=%2{hKN`fn}e#Kz5HVoM7H5ATdWVqf9`=-)#3tMzoJjRbp7yB zyYzpF5Y5eIMEFZRqWS+@MjkU0F3O_oEkZ znNVws5mIs}_EHb6s5r@uFOUZ~eq-$x7X}6dbOI+55gkp){5Vj22Aq2VxW#-nJ?s>8 zf=xQDZEZx@-ziwv%u>qP!zH;Vp{LwXz6kW-{C)V+KT|yX`x&mI0ruhyk)7B=)^<|S zYi#jLu^8Fn`nhcW?C@EY7S+3bmiK};4)W!;b|r!BdL^uHRKc@munSB9f5DWjMs6o7 z3ro*5Bov)Tt2r|vp+>T7Iw~T<(l#nQJguy(e|8XNb7qmB!F>D~jSMhw}qzj4(zCwRi zKk`OO2ySX)yPT78rR1xK~vxArril1aa@7AN8yaVncd}X=i5Nsu)4CC+voQA ztqii+%W&-mZB_^FpK!}5t}!(>evl}LzQ4nz!ZS~G{ZI@kGe5rzpeQ;_ zPcKR@qK!F6{~}4FF812mgX8Fbb|~%_I=vjjGzgd8E*T-4w9^LsMPPB_@tG3Qb{uQ!f*K+bi% ztq%m_Wy$`ActM0=sy90}J2E!bb{&)Khusu_nOh?AgVY1(+w(aIRmI zdwRIK8oE51W?l!ZVpOIDe#*Ji8zfKk724`TeGRXonD+7RmI%(h0n+5_-bEYtiM6Gy zz+m#{p5i1eOc$Ub#IJ)ORmEZIbO;3fx=ReC12L40Q7H?GU7D+*5fLM0a~p9|`MU=Z zK>)96_=^~4q7ORUw$v|MdLO&+l#Dm+!uX4Y2hJW2);(=|8BuzcWrwb3KlpnPIRW@e%LXLDjt z4=tM21NOy3DIk*>u(4`?K!ni7uZ7&ky>7A;Uj2KahyH^%8OAehU5he$UC42KQ*d{A zv@USj1%i!QaF0LR_The#DkESH$@}onxv#2SAH)@==-QJc(FN1#!i7PPpZ3Y%6SHZ3 z<6o8vfEw_cYd)NjZC@Jik5M5AgNQI3TgM z8DHw044sPUf6ZywzDOC}B`ZA^@`=1a1OUwX^neaW4!95k-)PLTf7H{MOn~HPeJ%UG z58pGYWy7WP5^kwtkSja?jZ4nxx2_*GL8DWD5ha8}uz+I$s&PNnW!N#m)tcYKc|`WY zO6o-7Y=C0Go4Lc$VN|0XQesp>#Xh$@SnHbrb%vmG6{sU8l1;fI?2LFi)WrZN(t9Nv zb0YWbp_O;Dym#+j(M9$>HhKDJnvHl|s(kGsR2_ezkKoDA&yQ#ohqB2O4=-@Z`ryv* z4U(l`!T15&Q5R7miw1I-`<*(_wKS9WkLmPfUVcJM78saK>rD z{o7{bJ>oRs+5GneEege;Y~dx$^A?PFU3qyrIm%cAnuSj<7*)8d11>770t-f6tOnkJ zHd?2DEoF*3jhead!F;itx;z3`W#2D21TajaPG&|%7X(-g28E7b7nSgNLGc#<{tdYPJ;B)eTffwV zJduBp$Y4pOjF7p<8Qiy1T3rtw(!n*8$3-Rs*g&2HYr5{oSYP_c6vHGk$>;b6sg_m(X=bf zRG3R$KpJT|d4-g*quu-K6A!OppP!WoU8u32J%Di}hOyvVqa9o|wOeNxJY{en&jIi# zI1gcL9bmNW3SqKagHv*Y3cI_$@4E!}WmO#>k^0KMDexA_`D>rX32yP9TptMZk1zf1 zVClltk_K4b1of}fee4VYuv?uo*N;tI&F!5>@l zqt%g@=&KLEAGo+)&`2c!M(UYHlMwsHFfh^tHk zc=^4yca9cB=K}Yquqa(@9`{`R?+d5%uX90HynfUK9mg|<^X?IXjSC+>rKW`;l!SZ0 zNiqiT#|#PUx6do-RKKI;S0T&sXH7S8zWrb_Dofuhlz50=HnKhw0WHbat~?7lv6T-` z%0`1vOf4*ViX%>Ex;I+B*VZo@Z`38G1|S9 z{4|Bip2&UXnQqx}x}4@?z92l~rBEdn7%Q~gbc+qfJn|^ms0Bf=CbrO{35w>dzMrs@ zP8RkVYkGTEU2&0^dYSjg+`_`X5V_RWXtWco5yc%VfG@I6WcTMXig^$QOPq@EmX^uTap)02&}JQb z6KuRfEpiYf1?N!Y!C$xBw_)@jFUxUt`|80lZGUB;mZ*;ub0h_N2ZziUY24Oya}DZi zCr!1sxg%}KTB0m>nk+nGq^pN9V0mdo5YfR3{HkkaeIy2Xzu@8a0UA=^OMfSZ@MU7i|U5rG3yWox*!a_ZHSRG@t|KNjj~Xn;zWCIjH?`jv}D zT3Q1S-qO;NPi&8u$OSw`e`0tWZsyg?JPTBq&R_<}n&qj?T=I1!uEKq!A^ip8Y%!@X zeP{is`7AO-QCNnA$Q0*+rMWGx--0h_~|G{@p(1)F=wEmW& z)a18rTvgbFgu_ST^RA((elg71V*6p=Z}?)tSrIM zkEwxM-@kuP*Nja>mHN@oFZ&T?L{(t5Z{I#dlxb*a*ew9H28}QcTAG{rcdt=Kp+HkW zV}}$(AFh6SQ`Zk%L!uV0!}~Y`rAq8E6^JktEdzNLfqa`aXA5q4n1zZQ&Q~YjaH(@>K2YwyAp)V9WQ! z$O&f1n+4mrt?JiOoKJUO!Lz3Ix?q5qAPBdoCr#bsb3IK&NQipwPXL)1bH*sv|KSzJ z2b0n8ueo2oxMPs7y(b{aCe<6Vi^Y&k;Nn9Gw2#$fRqR z<+WiD{0;(yNwIw`;_lr9>+Y0SHX-szzC3b44ZF_%H1THsltVbl%-*O`_CJ@7Bb@y= zq|-O9r>|Bk--8T$=hcw3;3!g+MoF^0(=HP!1tP$J>sOZ?QCV~e)TO!8>cS;>g~vE* zc9J^RYQ=r;(%0QGbHNI4@PM0VhO38NpyCfqW|#ehM~6p8ry=L$#EAi&9oJ9ei|p=w{qSm? zB{ob3^G`q#5s?qwtWeFXQ50GRW7A*(RvQF;3W+VPtqYxUOh@R(80E05?vHuFm>O!K z_hJh!N6z`S)cupH|58izb$kDc|8yws=c0%PB$xfPmMS=^B!5$}RI$q(;f3!WF$q2y z2yI34Wkx6kI}u#0?mS?Met3LU(?#j<9Z|M=Tc{FSPsYj5 zpJ)6NJk{Jj`UU))$ z)R_hy#-(On=GFZ&0{~^$NJDv+hfp;(GcBzNd@@2cYy-IvK{c3Z3OTe4Ud}HVuTr1S zMWGsm?PAHb>3I-HgB$+m=xBI+6`PT3FBP2DF5GoLI}3aNb%BV&jEtb@;43;TVlR@* z%NlHdBxtqG#6RRWs+fA~bT9aNW=H*KS2V;A;Ke0B4OiVh_vFS`KWA?2U~;fEDRDf7 zE;<^sJTbcM55$Cget1&Q@+BCXfX9u%uGi976mX>~4_G-lTYF!$wb{>kArmcm0dsD! z7}2+oftg+~0y-a0rn7kcSQ=mc@T5IvOaQYbCC1>^bnSN=G|L2^h5#xp*zm9g^3G92 z&}-QR1z%QSQg(wu{RMbcEYA6%`KZ|Gjwk~A^yr|IgWQ{ll{I#!^Z>c?`NJKZhRV9~068vY_=C3&mi ztUK}5bf~4!_d7kAy1bxae`NzoI_nz+=qpWw*9@l+#6U`i91KiYs5;{z&ku`pW@XuR zF<@?hu`y#{0x;}h{JD~1bQovh`7LSBO5o943=Y?fuup#!57J}Y$j=O9T_g4 zOTrn){}WI;^&cZLYpM{01~P+z%PiDluq25#Ro3Mb;oQfwt2Xjw2o9S~<|zh^9nE?l zPCec8cwSGeOF+GvE;MIfWth-IKRE?{UE$Jub#FSL>6cw0P>gw!$nyCfYtQEjVmPkB zTk+{CXzHgH;zeB*A)o6Oq+E2J-G>E^TUr#s?hC*Afrc{SI0}28pbB88C8tx|_!5BH z%53==pzcg6t12%q+PuoQL~vn!be_L)oPa?{cY7cJG#-=B_dLLnzyWoLFM!hyV_HN+ z(K$!}32sKG#})AVb-z_c+lr+lHhHjDv4?~pb4!LlPYIT{Y)*50@<$;m9oAtc4JVZ* z7xIFG{HjW;n&u0V7atlkgr}I0s}jR!x6ws1X7=duwAv^Gb3~zZ2femjKcZF1$F9BO z?BNU_#VqrhVDJ}iTRl{jG-~4R($GI3Z>>cLy&z=zg zKml?qAi`FY391ehmYC@S6qk)G_zaYo1Km#4uXx-Ma>XFx!cZsFM!myoZEBMRljqL? zL&N&d&`?Hf!Ul2jK;s#~L}0MDq}ESf`5;{>!mc906qj*iXtyAjm=0KLo}MyAN(K1O zuTfcoQGs{CsMt8-7SV=Um5*kzS+5)r6ckZsCLu5ZCOCNmk3Ebu@cWSv;>d=& zshP#EJ(MCMBGy6o7_1&^*qWTnhl-W~XuEtGIfeWVFR=SD;$CGS!LVC^Z`RtpdGjy7 zqakwSV%p`0D=iy0;HE+*bgstkJNIXZAaFR6@mDa5AOkazGI9%op*oYM){>nB*rW?~ z1IDCK4eZ|dDgedeeI&Syzys-B)&Gp9&Z{n0 z?s4FRI?P_6Hth0O@6hr;?G*EEaoT6(@pi%(t1`wWr#fbc8IRC_7Q3RZXq|xj33^5WymX2c(tvp z@`>DPs724g7e^o{WkCHUn}J{tU0AH^LLXynQti1($!Phke9T#J zSXjbwL=u)oPa*8bUx8_|TR}U$Z{*YI1seznp~w&fF|&E80w&shemr@7`JzZAirmLl}~YXpCjlfIm-FJAOQXab|wp_{WUBw~pA{HRzbqAvl%+jni-Zo5LL zBMbE~07pMpX2%gyw~*5&K$MoK13hcI04tdPiV)kaYMAfozWoJ;(ss}2;~wGT@6n9= zbL51z{^Q~jPpC5r0s(?A934C5`qjCa9KR*1PD8fBVBcp6{qJIQ+A@3(LnXiY=f*tF zusI~*eVzH1G{toX-?~sLLrT^XpYMu+7dW`T`sXIm`ZMUX#yWC=q7?}9-r1q@X|&CC zg7)!jVNC`44ISaY6ahRT!cxsPEY*N;|Ks0QozY|BAOM4r8()dwA%X2C%=-%$1GF7) zkJkk|be>R-orB69j67&5_9>DW-iNrY_3MgDC_Dp_5e?nNB~K5C-9YtyE||~Vrl-+I z2rnV5n*&N^ub-=nu7y)mHpm4>q_CY`R;>#NtOkMUZE0&OG&-LTsUHgyol+306sB2& zNrJaP6;Ouat`p)PB}#OljmE_H!>U~qUw4Hl`yXY(*d zy(gCk`B2G0h|d3T)Y2{tDdRO^e`qR_A<2>fkNwr(_uZD>`>i$p&CNa9TI!SjxoKzh zLzAn$^_@jwbd&W`li`szm37hN$L-bDc;hIkZL>_(CsmaqE$?}0h0XtxCVI!bQsBO4Ud6VIN7&zq470D#AHcz_P^cPsu2@>wEPgZLn=26fbDu~%F zK;1(EvXCNro2YOxN}!~LT5Z~!{c6i%0LVe)Hy0QxI_0$iq?1aNNV+c9f;rvOuQnW+ zngC$=_pI&b%TfL_)V?|onQ3*dNZ6sZZ?p8{F0ha&;3Bsn2jZ=BWn7nxC|&2GRlCa| zf}H_fy}-(4WX`;Lf(=SECVG30Fgk=Ti4aT4Hm<~jbuan>m5L{UD?&CL@J5a;6Ckga z_I8L6kBtqfVfLDE2@%yT6(fm@#F3#XPP!zKJj06$hM4qzI_bV$mx5^H&`MWw=B!N)W{kxB?Z2!1t8E&Yiem?!7KgKc+PpnD z73XaV!6ks?nfp(=t6h<-Q9gj(=u;kFKcq()dnkb3KC2HpAY=)@c9(*@Ry9Go%J=qG zNVNsp{k}ph7EQ3mf=MVwx*^9177yz42r=3(l5hY)pQN>H%i2Tn(t0M(+813!`$%tK!{UK z&8{-5+XDXFbL51@|Iyz8KEk3eyU+)41g_OEi+8U2x{Ox-Tk`Im{Z ztaa3~%CtcU^4J?Y%j~OW)U70+{2)6B&}kp0?^}=X%|VOA9A0Ih^oKS>NwCI=(<|UM7DBfY@aZ>V zkawgDHZVcJi3Z$jR3egwpzuCZ@jy0;3_27K=$MpW%>GM!7gRg>Sn#CS3ydzlk=)ok z`50FAU&oB^zk5)`{Ty-*2BG^Txc(cC^ZuhYKg0DvRamg|knHUaz@}4Z2ErQF@Tc5Q zo`OJuZzQXjm=PH8Zb-T-fNp{Y@c@fKJ}m_8z>qxlRzN`Mha#?44YX%g4$^I%Vub+;_Ssd&Cw8 zA3>O@F~2=}-E(rnQ71@j6U zSjaX@0VO&Chy&L`z)`D9?LNIAkTd03b1_N|HuI}4s04TBIln(m;#<|suXqbIHCHZj zPz7Z$)2b^R$O?^ltVm*}-7Bl5O|R~1FRh(;leshH<(CEf9y5h)JpONo0LScCbU6Xu zsTMv?(>hNTc9VCHvt(cVHKg`||CzL7T13peZg14a`Lx1z*mU^ES)3(;tHcdK z9zHBQQ=ly7G2iih>B|Dpc+((&Dyjl7QVDV1GH_Capfaa^RP>&si2mT=;?g>>Q_arK zE^G|#j=m}HAsjLT`Y*f|>7TD$ZY&32dNTps$7Ylkm&4s7Pu32*3aHsJbF z1$U}j^1;ULOMMoqHWXdd){an(6ERzh>lb)3-jTZ_zMO`j;xDdbJl-b<(3>vs>=feS zhYm?mf>ijJ%p-NAgisJbTco7s7xyXNV{c&JV%b2a3yH#L*M^u}|5SZ_J-);-5q6_> zcV+qiF3a@&dKhpm3dFRI2i1A79H z*z}^&{B&5HBKJv+44+|xEVtkYTQAj!DxUdi_e*LrLT1+=rF0+5an$0-*fymswIZ%6XPutIBi|<>^=6|A9s$9LIA`{^dBDYB+$n0#-WP zn4>3bj-swpNe)A07~jOd8-rW;ZJ@U`f3~#}Y#HpDwv|6%pTm^tuw6_qz_7YbcGc@Z zHoa|r7P?0r3>C)bD?tV!aCZ~I6Kc$NBabC6&Mp%=E!#jlRRGRN{RGEU;GH{tfS~r$ z)7xf#e3>T&$!yngc00nJZvG*DQr0N-AWsX_k7jeTs}t! z`yNYRrtQ5FN|)W9-h;+rv9jJ{T(4BTHM6(stuMHIGRppfT+HbmOz1g!`9x#O!pF;j zg`ecpd&jk=Bo%@f`EcvqhScTBEA)}0(kf4m58hxVO zZ@{1iddZuUzS+{!8t}i$`VC&5MT$S29&5qd2wtkml#xAYM`@4kU&AcuqHr?NHOwiy2(%@LNGRAav))@b=`!`;@@%I-`^iU1Nl6~7FcW$a z0fQI@B8rAs0R6ouNH+E)21x{wpw@Q9Q(Q{HHzg!IJP9HJy_ZdxfFS@q$jnfA-x~d$ zmaCg<90Mu$+0+5j1)fXJd!ghVBBuf^^*tc11y<#`OdqK6Bzo~fG$2lNj3`1Wm|W;U zK8eJC@FS;(KJF6FYlSoNO@r$X!D)#2TmcvCk>v(7)iV7R=TrTX@t2DfS8i?OAd^*Y zwAF06x@(4;k}&g%V|CIr-B?WP;ll9t>zf=DDtTp+*gUfOTchrEt~{J%59%c|4CXClDB* z&Vn_X6X?w?p`J7Rk+*lMI|9u#Yv0mMv6^trFw3&VjndJd=EBDMT|3)7Ezkw89eAOC zwFXM9=@~*ukWpX=+E8zEY!iTK4}oCQ^-Vw(HUR|9Ay_tcalk4TvJ6m2@U_>X5s(Ie zj}3f`jg2qJA}r%8izU@ii^>^{T`XUKmax{)TYg~~X_ZkJ0r1xrOywx)1ZcJB2Vo(L;&%!bp7ei{0js?uWnQs`MlEv6a*P-nFSRa*7K~q)H3= z#)Ycdy&TV3!?{n3aNN&GBU)V|F5iI(=#r#0!22!yBbn2EY?WYR=Brb0`F9q}NS8CX z3BXDp;|AF~VyGK&9ao{BwiMt2P!(jNX=VVvtq=`1bMOaY1MvLj2niRq#0+gnPl(I@ z36Wcmu?XFsL@81ru6l`}#}IA!pImwGfia zm;v#yA4DJcUh~G00Q>-ST?m2Pr0FRH9Rc}e0XvW%&4Ls3GjJ%Cv3e2Li8dV@C>((p zirLa}#&!HbKuHV1V2GjK!XzN4CZxhYIj|ev?2jucJDoiU7Z?43cUGh>zrUAnlu> z2s#6;+1p3VeM7m|}pWm`JBK;(|b$#5$gAZT6eCe^O+S>J!Jh%5jwoOY3u7i0n zVi#{J2yHj`0lz1zMb}XqO1Jl0UCL;}u5i*r%lS3Ayky@Y)MpLxvMtFC9JMoi`;n$+ z{g`^A%=YX5;O3|$%4^9C`jX>RGMbol*s0)-NRkQIz2q|}hp;yI3+tH#cnR1n!5KGz z+9c}E&U2QH#vldruCBhfwj4(>{rN|y4n}Zp!EMk)Xf`XJtHv3+XkeOx-zk9J;jmpm z#f{dPivv@!$LzbnfI=b-5GxR9MQD+MawWV3>HukgFl2p8DA)mP6{MwfG5!1|2W0~l zX?Tt891WrV7n^Ean#t;vE}g)++n`b5Ep0oMo;UT*>6rFe9ATH}r8v+$K)x9H)@z6Q zO$;9;A8uJl-N@SSl7rv;oZ=(q!3T0t zQZ7SRCmj^22Sp(BuGzIXL_!`g);1u#T!xMj3E=R71gD_Z38gdv*Zt#*&yV)9nWBMr z7t3=??7^P!9IDDYv{5qfXq9aR7bJ~sY=az)t{a1}SxTn&j%iIQTV_8@kxJB;X1$-D zMJ$VB`mrkfPAC2D@s>Y>QGolG>Y->|bQ0je3Hp0F{2=Y{1yeE54{oFe6bK8!hO{FR za&N3bvW&tB=H}*f3|}A-C#|BQ;vS}Vk{tc?d-?HMK#bmFDm`@%0;e>4_CQqv8Bjv= zufZ|zo@yT-m1w?N9Wu}!l(`(~3nY;gC|&(JK+H9fFqi=Y*$F zINbP|TG9*pNUH(3$SwjFdtF&&pU0(y@@@iotJ1&~AWlQsnZB!=$)Icc!Htt>@JAXj zMjgd+NTR!ennw;eBm@)s%_n=j&L0-yZDN+Ft_&HONyJ%7<4~>4hD%TwPuLM$0bFWHd5fR_3jo54}OY zkJ*28Ir1_5)Y0;WVR;y;yau<07Hr472AWfQF_UbG{al@LRvZz!ZJt_byt$%DcV1{s zogq~!Snw=_x2%b;)+SSdK%bi{&D#IbV?y)TO3A5xxR{NQOGV*SJ_NGaSi3^8%U{`1 zLMG-w*`LAVW8S~GpLxn}3voI#J-w5Z$uj|Tgldp~DT!^kCE}Q5XUQ|p;&gdWvJwr5 z3(o)(SmXiw-)rv`d+K}0#WIas%lg6AFby>|%GSIk%?xNvas{M)LckOcysd&L0Kkcg zcTcf~!gXf#y-RgS3RUiy$Cjmbt@$9XM;M$%gYvAb+cI<5ScU)qP>B=y9J-34JDIsv z86Z>_vfw~}Eb(5?`^?C;e0!+?4GvOvmAOos|4xDQ{PuV)Wbg?^X@QYQ1B$+A8S4K(|Wz>{{qo(g$jEU+n?unNk z^^j)W4=JJW0^Yg8&3(jqi5VCXkNne!ftEW>?_89jqjW}R0u3CvJdp0Re;YPL=s}YH z-9OZ|+Hif$jl(NIvr+_?QEOWoOryjV^Yrj)+03t4ywkH_A4m-7d%wdWoQchN^1tF9cu+(Rq|Q|KnSugbmP(utFCa z0BjtTx-kLz^e!;?MA-MvHDA=dp=%sPMdQRQ;Bls z8C@*BWt;uW%_j2jY$~YEvR~Km5no@!AJa{c4(8nPhq=KC7I(!wMldrce3Qs53E?G_ zS+gUPzj6BDei6qSQ)V=>gjh^Bh#Y7k1yS(GKr)>G=dx4NmdwL6qmiRZG(bx`NPhPKdZ@udaLRQ4D) zqu;lFaI0+-KDtrz2}k;y60!s{;EAUoKPS7$s-1D2j-hA;{c9mS2tpO5A=f^mqU)(F z>3}Uk0NDX05z-_Nbj0zaaURJ)n&LHRRKP{)FSSGbTLN^EHpo2>xCUS8QTWs>=MDN| z(-$!If>8Gr%dH3f7?@%#i4m9}j|oV}&qvhs{2PC!eD}lX0|`UmI3fY!>r3PN5swRO zgPz|gP=^lXGsgfuR(FLT(F~7_?0v4_Bgn$7V^qmwv4T{%+^>@lTIWHYnXInn6d zj@rMI#5Uj=L39t{bX&+{Tv-14aX1&02OJ=%TFbJKpcQtAgALmp(_c4-vi{Ov-zlFh zQ^L#`gmzOPsiS`_#E2t@ghs^1AS8j-IPhE!j)uz0w?R1Z2=pcT5DCzY4 z7hYgQJGvG~xBge*`#4bg5eJ2%g#b=4wM$ZO7uzus$8CUw)lZ+knq72x(cu?Rf)w9h ziDaA#EuhSRk#p@<5hh+X1Rw;>3svlg%Ctk|ll1e^1g?qrYknCbw-mHMY>?u;(2Vvn zu+L3Mq<|1XLgkY+J|4+I7JvdnVHK`4GMox%(2$=D&UDc-VzOZ?MTx_3NPdA@oTBcZ z5>TN9q*nkP7ez3*MRXG06~^96mO~fX#UQYE{(aH{%q5`6bbY>$GP%wV@a{c+p~i&; z%)=qkXX$}`z;5NJhe#wvq;ky9LYoqu>yQr3gVgPAcOq=ZU_gnV@PEQZj`sh_-&5WD zf2g_+xF)ZzPqj*`qIIIks9Qilk-Y^`0ohBoitG)t32`Bc0+k`k3dkmesO%XPl>#C& zgpCMcg&`yCknh~s_v`!S_me6m5b`|t-gExzynpoJi{TEoMgeZd;PQ5;prp9DB%EIm ziXk5fxI&f3=mn>^)FUSc(`U~VkiVKZ1&c_zxkpfyI}brGF9AME(TRWvO!K8bqAHCqU0P6wN3ty@sULpmsY( z8W2(2#?zH!|ISV75Suvqw;RdbF&P|$e3|Ipq{~~2o|V`#u}T|GbQTc zM@HoTg%aGo|F>e81m+2`+7{oJVVR<15Mi7=iow1Am{Nak@Nh>6au;K5Sw>u(BB}qH z8VKN9(8~v2Ud%xYy}zYUv0sN;2vD1oB9T6Z-r_&K!jtFvx>+;WWC{_&;BX}#c&^?6kL!eW{rGz1p z7lH{}iNSQia9(*+aGf2wb2(9XjN=($*6A5e%bKZxBp5>-9UV7wNdG}KJFwTlftp$U`A0JxVCeO4 z*>v}0BMZEW`;H7nEIUoJ3<%jAp;Tl=?|^u!(7xp;+>!rTrT;+kCNq`82&@9)@dX6U zO#RKWT~VUBWo}?Q{>tw10@Z-XTVIu`AEjDYBSYJdOe?* z3meaGt$Lc6$Xx~I41{xi7+m%8*}*NPqljHrfLMydSJ>H%i69!5%=iS{CH}9O?c;sc zRCHuk>$JlS_i#m77ZatQu@n&}h>WyIeD?F9w0-1%YOm|Yd$JX~wnE}eY`gpV7I%N( z&|eot@+sz{%n}Fmq?9SpRfDj#fGk|ZRt5(<6iebT@c-3pDzP{cvcZ)PPl>c~K%F@TwACts1Bs zLl*=h*2fX`z6#@1#X#cKm?*6eK}zv2Vo}b;pZsj*CJ=P-aq?G_1GPENzwm|GtGLGY z%a~@)1EMFTlqL(gRXvv*7ED}YVNw6IlB3rGz?IVfwn9F!MZNpBv7I#VfH!dq1W8Br zOyxUWX6>_-e~Pm}w3|aLNN+<#6Py;P!G_|oohB$GL{%x@f zg&|Zq9>_@mUu^LdfLY)a?IvMKKf9vt3dn;)ZrImkM@Pp(FzSM33^ukD3?wIZ0O;*f zvbs6+kp@vpu|Fv5G&Iy$k)9Y^9D%K2MBa^*G3E6+Qy$ z53EPHrL1{wpQ<=8beaMJ4#v9Smh$Ggj8Ed|z&`eGZEJtQ^lL0jD8qVnY;0}+!Wjo$1^fcG_)HjCeG&GX;%eAPkckf@ zd`5tYTK~lLH6UoGSYmtZX5%KXRzlxsgnA)5fS-nVBL}E#esUl7Gt*T|mOb~%qmak| z+!BI6BUYwVz<%lt!lWKY#6CjcI%S@c%_@ua7 zzZ{4}w4d9Ed$l$Q3JRj~7)e;cicY4%K)ffYPlx9CO@9?n)NOs!JV<{_I3~bmUevGV z+81*GItSrY(Ssdf`uk1e^iNrp9Kt`mAV9P+kc9_7*^~-Y;4Zkw{y9?vMAR99VZgU`9a~1BtdsKvvZwb z)v)H^J1AoLsCR!=+iVPiz)N7B?a-9$Ga$|&eHvVZYkp{FRw@9CeY}KAJgbez0j>y< z7gWqPHnBH=9U228TtG&3BC7#rVJYvZz{5F0@Ffd91R_4SxVZSe2RK^VkT)m`7iKou=G$wJ;bp{$D zh}%@*C%zQ>^_>lnF0egl!cFIm00X#%8m_a}u)QqQ$k{+2Bax&j#5X|0?q?XO4UKK0 zv{xbQB~W~lCB|?5JmL`Xgd`kc;o&X|>*#j*Q;jz=%MujXf(i=#N?a-4NR18`4Jv!& zhc_F5cG&=8*+8b$Fz9|z9Z3^YDgbP-v!4t zz*Xqr+V<-vZ#vxkNcRJ|MWR0=r|$3}3pYYSfmd}|5?gNvZ-6)i{ey@g<4~JL0xEXl zMu2_a?KKI)4pAHq6#Mz51X!zmRx^JwKW-_z)Z6yA+ArU_`+9^dQ9cpm3f5o0h8r;& zzokkTKdtoSkKZmA31_KauHO{@BI%t6>s&{_G@E_x|6JN^km_-}bVUiOsjEG1YY>SP zP5iC#6JMBs=eWOW^A`oH*A>$<_k*7k@)>O$nZGo*AJwhni>Rle7K3BC(aoow!59o( zb6oy`Pq-e?l&nlJ@mhB)_OeT}Q|bAspv(u)A(;#+v23XF1x}v)e4S-XwLc6(I_TED zA?>f+6qLB=`f2tYgkl|4U9cv0}A2UJNXoX>dwK>16mYO9w20E zemb7733nNSY(f5bnv0-#NDq1?#EbwoTK6_z?($`T;E|pPiW;@3*cOei!t#gZCE3gg z3To%d`Gp_}Vm%!8?ae2C!35`<-aWf__7_F(x37%cxP{xQ`o4SXn@$=scB0nk=Oe}C z#zU6El=aV)E!}TV7?T)N#!>0hebEbZ4QV@0DSPE?Q@wim-9YDZcYir4%STg(vN683 zjm>^17}h}Y3#IzzjaAv0107(1W9TN5}VvrO36RaU7!}> z5m`5PTuj}Tt@OZS*U*!`+F)+WO#3Dhl=?_P+AL%5{`Z2rwsyZ!lFId#HU#XiJ$?H3 zFR}k#xx({_M_7A4>&o3Zke(}X6B#(1xI=$24^K}&&I;y0X_Wt{RTqCodVr??^uF?Z z7kF97iPi$T!U-V*Bwny-I8X(@I55{0{iW!;HvjprHl+9lcx(~Kg_O6!BVP_7voH&; zK|r7qD~gI0+hPhqqi{G&F@IyzbcWLeg%GlEB$k$hg{~zs*qO6L{!g0SuEz(?4+Zsn+l7`2|3QQv$H7!|5U>WR06|KerXkG^nhBk2fsaR zmBk8piIn$)$|m8@dD-$KPK>AWKAtPR_7=xN;v z=zW6=3fd%Ni+>2B7N|xM;|5(yp#_Ml5h05Zjh<0f1_ESFBh3M@+vwq90th6?C%^>g z169Lx)DTdx-VjhT4ymHi6q?KD2NYrO*q~rraD2PW6fo;Sc?sP-3SUdq8Z~q|FCwCi zxL#l?tEyvS_92Hjl#S|5OAb9-1~{<0*xZXg+xo@EBBfN+QOM2{_~Ep+1@|hmh7E*M zE*GCf#yNqr6C{s+-!S=~s`lppwg`%IAxDwRy5h6P!FS{d3K54)y#EDNhChDtL?4iO zlylH~xZf3MT`(}G!NiP))w)|xfI);8itr^x3=mDnV0GyKN*W2w~quL4fqF;oz$!fELQ-2>Ky+-A7^Ve0KEY6;CDNs2q3_> zp;ZRL0Dbd9z^*Q`@8oU<_77$Wi6VjIdk;&ZF~}!N5TeAGv{OXy@<=)*GbKd{p$aGh zamQ8~9pahw6`3ndNQxkx^i|LmS)HKcVP&`ck;+hv>Y=T`fO7_RXz9XZ^i`-i;8Vf>%Q5x9TRpb z<7-kCmw0Z$R*s40-v%zU2@1?P%#WB;sfGevvE|+x%|a!&b%~bz;l+_rdq~A@idhf{ zXkf~vqpqI!wqUP#aM1JbL)Kh2e5bTxCMssO#l*n?k~TeC+gA|2a>jei-8$MQQNbe9 zjnrs-;jT8BTr|bX8Ner={ZbDo$oE z2^|WXEo{^WZHIxKFK71Y5@qrU@i2|2x>?C}6AG0DiTN}vM=YlSZL(k4@W7g(*Q>%xXpjQGZF=P_VMuLf*FJgZh;x`b&dJ-%{Rf`qjsU4*9Z+*%Ene1Dx3lL9szparGlujpZm)`ojMrVWQ`bGxE>_u)?&mwpP;f%8q7r&8_{!!rAF-zax7Q;h_ewU2Ncq^K%nc#VQ z(K%$(-M`@4wZFc>Bkt!u#L&_;+}~d8;$Po@*MsLie(dT^TdZN$<>YvT7vmcE1@SG_ z7*6kXznALj>$!yR)8gV18q)FJrS@Vz(*h-I`~dj zq675~ZMN-G2q=P5k1O>=gX|2Z#*M>aR#a6&gy5oAs!vd|H}blgE1@4U6$UzeNI+hI zWF2%SYf;!@o)_rG_>MW3PX~RjwcXfuRdn=0HzOf|&`CXni>{c%@ZY!BFwKZQi_WU|z16rRh4syfzmz){dh!4=BI)GR+!!CvVjssch#nKo@6Cuzy`#^vK~HdJ zV``w9{OlNq_P(g(=@!u399ytr(!gv}v7XbZA&=dDV6ufduSjYMN!-@6{7gRpzxGcp?|F6Jv{xXkc2hgfICagS;~Ca27Hajmsq_nvs^`Dw~BymdZy zwS?WcqX>tc3Z|D!tDN#lrmPNNv*tPEvd&q?xhC{mEmesrPix5G2J&Pw$;@>di?tae zxje0^e1&;Q!AM6sDaeo#?Ek4|qE}EAC(=m>Cr%M0gFm_X*ux*OQUWPh+jx%JTGPF{ z7#u6|l4Uh>w+j;At%D@0c_Ec6#Y`{Yo% z$pX^GIguOQSrJ%ot!FsyG;S@8%`^3_G0cF3DA`&`(~-}MCB%W4r+$nQ@ALb1suqsZ zLDP8u*j((E3Hg)qvTD)H=>r_xzSK|8Hi#M7+TJ?Eqa57dYB~BCV z3Y3_%O1Tu~;z04lRLQQ~%##L0=177pZD7MC5soHj=zG&}3}=6ff$_E`{N>1D!D;K6 zer3jx;&k&v4!eCj9@6L)5G5MluFs@l!}Uogm%9`K3NLEJ@wRjqohl0SO+EVT;i(;O z&Wul26wnnnYI%yWp*UDdK4T>$My@4hx>wtUIl4h0ZA||tAuV++_5GN!>Z>?4{^cd7 zbJ4?KU%9|yb1ZYoY+<%GYeK$uX<(pcamK*pP`e^?-Z)^sakGGM^}?$MJ3}*|L37wx zzX2dmY8-dy$aw&>rh3Z?@-3jw2KYQj8|SoE#=^PVASv>v_TjW)m@6q28XuD?w>}lpg094$G2`I)cMf6wN(|`LgCd7{SNqTQ^Uem=Xb9gLWw+GIj;rEgsZ~Fv}+qwlnn%t z_39djlkeMuyc7{r!*ol=`qj0i843Wy`+jT3QI12t8DiU^qQ4)h=`Jgl5%}tPQsu(@ z4mQaNlrHG*75PH#nLQE-FW9`#814AQoe`9p{X8kJdF(51c;gR=?kZLbh29HrDD2D+ z`uR8bsFc9@_#)u`0dVl0s&6DXyMOugB$&~rh{*|@VJMNw@143gC{{S*foiugs|~8t z{(a^FyI&5~YS5R5Xu!T-&LwbIwJF4RFAX4BVhO16bwiY&1u$B&3;n(FA%+%1xaP4C zU&6Od75uD+;Q$ncTNLju)%~qBKK?J@5B}P^(>$AC=0)^ZoT>7?;@kV?npMy<%607V zJ!IHr+eQJ(Q8SKWz=`@Hi_?N6U8Xx#%Czx8<0po)>&*}hP3y-`bk%EGeK z?)>nvDr{$Z5wlpI8YiV?5Z7T38(@a^2FZ=+m;kkrAhtrqjgi&lJ*_i)qIL`B$dtK< z0dsB8P^Jq1Eow-Z`r*wU#*M|x-Sc()A@}H!yP|H}YNuLae5yPb_e*pRa3w;qh7Gz& zh;VJvPrqwzaGaa`zEOI$@2NOpVr|E&6MbZzqkCc1a;1Jk`};AqPtyAXMrR}(88f7c z?~6N~<Y8@w{&;oN2rBy zQ6p}Zk7-?TDI&<&iIQw^w)4l#@l>6Qe3;vMDi13O)P_iNO0e+gCr-sSx8V7VYG$?4 zvnw?u&MS)RJKuU(M|_U7%EWGr61n6r>#wZO7jQ{T@(@hD*Q1(dt)R$v9RAyP!Hu~H zr=*DDT^LP6&bQbSur_r+{1B&Pibk9;2)166K?7lZlBFFgJZECE*kl86gNSU-68%wl zI2oloL#`G-%%kYw{0ET$hycFCshxpZDSJND>P;;bq-<*LL}|f5pRwi_r0QHj2`rF( z@DCTu3-SIa`7s*h_enZ;rVP~*r;<4#O~|XFFJPhNGT0Ja=+G2jIdMWVxQS)FNx}U7 z>!$8E)}hD|c#vqAt^z*pX|9-}hVRs5nws)<#kQQjUp=^6Z*gxEj_n_BFX7;+H!#lFIcq!CjS36b3bYOU;i+|f2B5M88RHASCJI;tkXpQG#G5Plqz%(DZR(zh-R z5X`$NU|2arP0z>YmF<@?VlYE*|Bi=6zB8(@4_&pf`9*&IO8{tdkL3rf z4Z2_4rC50hURN&Oze-qs`MO(QfD>m%#;2Up(n|>vRIuo43v$H@2;7U8^2h+BG($J7 z+?*U-y?j)gIVWO;FP3=q<;z9+xhs4e2md-*(vzbS8b*f#)xcn(CcG|m=YCooskieR z>HR>td#bwi=QAyENxX}mKJ}Ad(Bb3YP0U~REAjM1vZ7N(cG@@kifhBoa%V_bbv3m& zaovTD*cB18LfgjQ;rAgK(96&3=UD9Bmm5CzecV&pn%lT2>0yjTcI!QV&^x~~HukEO zCXW{9*YkGk{+`22-dZEjMAZAb@kFK~OXr#ULiv)fxK@zY1+xRw#$tP7B~KWs%+PYa zV)XWfU-TZlma~Wrz9b%dK~3-dV^N{iPj>Mn`>5VGi~7B9EK440PUg57@kSq?T3t?` zW>QZ}_ZbIQ&qc-w%v{&CSbKhXwS&kx^X6C*9G$LMM{ku!50(Gj~8g-bm=S3ntpYOfQtGEq8JU6v9p+eE2aEUv~V^$%YQPdGoETRf!} zVTgYDB5eHkHaX8)8LHr}vEJ09JpJ+b?)cZo$Dwr>T5&U5n7p?}b-xzZH`2o6A27ow z!M|^P_VeO{LMb|5G`e0?wWqr8jPb9|@0TgaQd+5{Y5U!l_rU}y3o>y$VGn}+L|2!z z+a7YXk*n6I%y5{a+A-E*0Wv0@nB?haHfqrDI?y`gT)G#An&|ho1d?;25z?L%*Jo*B z5)KZW&D7y_zsd2=mb63XSK(a04S04<=o9*T_SzVn6@j!?(TF;(ip&{$M%+f;mhFoj zMHa%LynZQJ)0G8y8J~sRS4XpXWWPcdnRYg;lIv&lYWb{AMb;Ad^#iQhs`ESE-F1L4 z)O;?fvlg&FYzsLGTBj`r3!Fbc_;#Rayp0OXqYu7P?oT2m1AZ(N zZ=x}Y-1vK&a2*8M6EqRUZGb67$`>1*(prB}rj6UDncS;^S<4hS@?HohlD@v~IMw5; zYgRm*n|G%9N5qXU5IpG(={)6&Fqvt1X^kc%^*h=LOC`p~$07VcU$ZfCR0JXxViS9p zaSf>P2GkLW3;|(v#z24p%7R2e3#lrh606k+>MgWTv`DX=vD4j>!vCKcBA{OJ2J2RN z005z2%{z{3$^tyJ;jJ*XkENOrdk7JKpyLESiI9s2-52578)cMa1C%!hq=^*c;h;eM zKcoiMps0d8*^T0ds^3}AvMmfOfg)QUkk!Jz+8_GBL2$|Xf9mA{mUvcCc1&-$Y1UVjz{lHdvmB? zaM#6XevSu4-wi^-n4*09?_~M{`7^^UNJ*=Tis6EM=>Jl%9D~?3`?mZu|9nfU&}?|hKqq+mwbSZRW8o{6cvX+0IKy6=FI)8eFecOif6;`$Xm zu~jSKG6puXEO1s<+1JW!qA_`u#> zIvW~tMMwj!ISLVrsrdeVV$|gIbp(BcZwtlxS&lSTsJT^3zj^OMqIdk#n6#Ed81a-| z;A46a_te_at{P^)uVs*(nuwI~neE5bj%6)$JROK|aW|LR*W%PfF#cNo#JdPKu!UwU z$%3gExWvO}m^8*@%Abh78n9WJE7kY6O|2Jm)Y*Sg-FNX#YQK;#DX?L>uWBi9rhle? zt;ds?Um;v7ke!`kaN*b8aXI1 zE6ijS>xG*7~iixE?Pj?%crabiBc|`zQy{MCr z$z1xQXoip0#{9T=8?zAYMm^)L7WKX25{(}D*|2;AgKrv}$M#L|Uq12B@$<|>flP}B z#)@$bgzy=jc1djiOwN<1yILX-X#dqXZ9G`cB)!22uIb%1Ui4v@PSja(7+on`QriAi zG|p(9l+;}?7n9}sL2-$4=M*OL{?hjj#-V=EE6fGz-u;UejH1O>*~lDCzJZ}CgW36~ z@O{594~-P?C1nT`{D%UYoGwL~DmE*NFl-F&A3l@)ijrLC+`5dNG#(tFw&(A_%tanH zjKO-j^FG0uD6ZeW7hm!_(VY=UlFF#UYtaVnDsnJPDU?)a=!N%=79s0}y<#GGF>CX8Uhm&;-VyZs4R4=rGFs^K&+o=xx$=@k)$3fq2*p^A? zc)Zsn>iCmS>42@_`&@{`9twtL7g879Lt`vm=AT>~x@GwoXbC4F)7!nHi9yr78tUq~ zeo@oA6r}P-lLZx-3o`?H+c_f)dx{4YFp)eB?aQ+b-Xm9wkLUQ$EAdFrCB8~?vU?YM z&iXl|K>b!Vr>^Zi&28vF3O_6|_R_ApJ%dzjrK0Stt`Ydj4%b_MYc*+fc6}ucn$^DC z*zv{r`6PvbUSa-mo{Y?q05S*nLA>gd&-_P@6t+?&T*^~YRTSrAm_v$N$E`LpNMA<6 zB86;hgb8N8T0JFpV-!;U@_OXzsJqm{+5pp1%Wg$U3MFbmK9v1Gk`FG0mEpaxmv1nJ z;Ir8%GPi3H^q+sGeEq32kU`P|U?KzS;Y%oBzj3^NG{wQI z%pCdmKv@ygjnd)Zl#(cQVzGH*7RVF{Y<|^*vWf~CI>|33)tLe@pbO~m07B?TE=&-> zT?)@#w$x^o_h8utU|S1CW`R;Ac2VQ%RfKSmK|*rLoADl0AVmRmM`&$~I+G!_7u1(I zFgKKJX=k`(iHyhApPxP>VW~sP2AQ4Xt)Zv9VoZkf%FnYI9FckV>Lt&IVaQH?dc3`A zzU+jlhk~}MR_5i+Pye`n&Owj%$BB^6rx7f^iuF=b4!CdmmtT%Age}u$wddg0;9y&V z$$8aX3Y~91RBJ2tnYfYUm7|6|4Kc@P05%y}lx45(NDa%;wFEg*cL2 zrdo8j{{-xwQ{-+f3)o)obr41PJp+5;nv_X|C#$Gs=^nN0QCHQCaP zN!{c8k$eM2f<{;Mu3ps%VX$+AwZ)6;-zw@~=bs;IxhiDtmuK4JS=I}{E-hn#;d&)x zCGFMZ>TvWM(S!!}<>le%q1b%4!4GNOuLtjCttSk~y#k}@kuAS2=bJ^1d{OI|!76^# zPgZ2U*1}cm5+6Ix!f&2 z`6(H8PK~K8W#K(kBj>Cujmv#HTOO|LJicd-4W)o7Yu*_3LC)I@md*NMRRzqD%w_(3 zjhL);#+<$a#?{#9d;5mCoji3|*X34$@wzX8!!-_632IG@8xOjdA_=dxYcI@l$)(}5 z5`%}!^R50Sye1ZW$?$fYqQ`7jb`BVvdKTuuIoL*VmT*|skf;AB=#%p{8TaolE1jck zOm$hysycM=T{?YSra7zcrnuAhrd7eC#X;_qo1L0g7o5g=g-e|8!lcaFu%)z5O?k7- zX6?GP9>0mNQx&hQ%~~hU+lH4h1M-3=0m8m>N>D)&c4&MS_I$g(&-3yHI&RUh{3~x{ zQAY#z5JBty-dnTy^|#UA9bFs^q3^JbYmkMeNy2y1I+%ae*|#V7wx> zaCMQeJpXZEVY18ou!LOdqeltRhO$YnRSX~X3`SFOV@Kc&&s+@q&Zp#8szdvXT0BHQ zW`=C9wg3CC2adRCFRv%`G(MMw7%(2Bh9Az#aq0Q7ZOwO@JA>nJ@(*X`1X;Oi#gvo+P|PeXAA_Ff3W1s}IMp5E)K{LPe&x3+YmysD zh?(~+vDcimPGuuD36vC=Hgd@p6M`AE_eIb5FMV|_!2@4q=?x&GguR$!fD&4*bT#TkQR`RPPq;kJ}8XE4!jY~tr@u4(Jqm? z?X_40V@Jy1w%0GjGBmT-g|0T*{0b=!^2R)-K_`t>wXADRf1S2d)haoF`$|s6@5D8^ zvUVoc2QI$+d;kA0rD*TDThEJ~`El8E)VRRY(q$_=asGoL(-S?Cnf2Z<2@$k63M2eX z#=ufX$^{QY;U!q&Hn4-d6c!gZcIrqZc)l!$DBe)HNEeAGaPd2N*4Wv57Agkh^`_{g z+0T~J%m8b5_t&_7Ec4vUMql3bRBhbBs^y!$+Mt-$%IPRRdUVsm08@TB0wd!c)>^YG zPTRITa4AE5ZDV68P%d(9$t?%BQRo(XD0sPsy}isU{dnEf^O3LeI#ki97`Szam96?9 zmer8dt1x$UVT?V>;_g196CJFOQqQom`phWAt*;Yei`Up!9KLARFO?OvXE4C>*17b) zjz>OHd#YY%k$O+ob?rw1+c)(T;(m_8OQ zN3}LUDNjg6WWGNRLx%Pohx9Y>fuqMc(&{U`YUOj%v~Q&{+-;_`(Rh2;snSL`BYrb?xp z4zBhA?vo_h%B3?03i<!mDxMJuQ(Wo$}0HPYm%hlVFdQb;D=~H3c zHF*8HZ%Y2lt0%(`o>NA$9PJ)Z2ZP}Jd%%h-t6#i&G8#mGWAh24U@j>FnRE3qwmUL6 z3%Zci+l0`+M96~`-__Oi8D?50KJN15V>p&J#YvEg-~%?1?fV1mJW20 z6VlWwHITJ}FsRUf-WJ@%xJ59gkJBkuD4NSg)DU~-720~x+SbsRcRerNWqH6K&G8cDm`0`%<%{%Q2wZqt9Wp(7{-DTAsXOqoZB{{b* z7@Ahs=SnaBiC!V=apz&3^X>oq14OX-c@}~F`1KFl$%iXbWEjuSNT)_cCEOQz(Rh*p zD>c8Pd}v5T?B>N+S(U*U$D-$WLBOr&<#@#f)ZV;#Zzy~|;C6x|Jvv=mQGVKgeX_t} zKM&6}0yQ}>KVfvX8e<$2OU?Ftc?vg~o${wyEMs-aZQMU-@|tZ3W+z{D=6!aef4$<0 z8cu*B*vBpB*O8Z>?CH&{F0l2c@95BlmFQ~A-DF@Cket=ad>pauxBLKRd*}$ErhWD9 z0mDU_)M+i&ih0vmeLwn&UXw0?$~SyU&9kDnd+k$>Y`$;P+}vWP*0V6$9*d;TrF1CS z4Hq9jf>z}A%0~mrSE@nhOy~1C#`U6?WV@Sd0+cDv>xMWp(caP*M7&)A-aOw#FWZ#v zY-vOKINc~0*$|dg{_L3*Rg~i~^c(jhjjQ}dC}UOkBYyTg?LZF`X? zMH3AwyotT<=Q|BYIv!7PI@zqLRhjMYtt^B$k7mw#FQjQ6V3S-WE4BH%$A0h2CU8i|8~ZJ-_l>JS>9GW+|Eg=!os%ph4$VTJuzx-tAyH$s{)Aj(Z;Cz{Y=W6hO ziF%u*4CFWe9LR!JTk{pLoY*uCy*`Tt9n33J0Lh|sY;XiKkpNzZcUBfMHNc*|AIvZf z;GOBm#JrQ=cnCF{yNQ%9iYE#Pc45l6aUH4$b57R?kyK1g>$jTh~Ft%-KkE=wU6cF z1ykZTAGsaSxh5Y>3lNBIt}-d=;OXuSR3^My&$U|OP`kSJTgXLrqdr}|(7RNkmEB8? zI4xVUVVTfT* zuP6|wrOV1O^Ya=pQJ!S|{5t6})hB-zIqsj}KTPk+uxXf^_qELgVW-W*;p_-@zNgQ~ zrd17Z0r6cg3SM-MNDgo;YHgdJ5a62ltIOF4M(P>&0m^A$)ZuQ0XI=S&oz4iu8m!d z27Aj)o2%$cl#q#*m?NkT!aFcoYbQU{)Sy|R@^gP56HRc!P9*d7=Y#&i^uKaTQaogQ zg=^gIXNxuCVb1=qVW#KF*TTB z$@DiEl^!UM%&qt`u)JcjZ};vLys?|s-}*_6eS7yB*zl=AS>eyixz8YI?>wPv?_QOC zdv4-iTdBG-)?|`g;X9=GOifn?Rrm}h*SfCkyy+>KpWLu0x43Igzaag!sfnA*BL_|o zV~obOhM2$%r9q+lm{obHvb1Wm0KKT&$BtQL`C93%ZM;!nEapjNkiz#4!I?sAadCW6 zJu-Pi!el*H@==)3>vg+zy~{7PD|)7kY)x0zQf)#f___dtW@+C*AZ$@r(96Z+ib4EQ?0dP+IHb zGit3xJzsa1c$M|al#ITOa4t{LIQ2FrHaY9ny*;rq^#<>+UFB6>IkCUPN+$ZEiFQ0L^&am5nSzM*KO!Egp-{n)hW;5#&t?42f@ZK4T{o@Mpe zT|QFW+ji}e6yE2KLqp7yF^eW>I43v@=Gv_)d^AD-QjMm7kBya=8L>R%8<}u+Ah9@T zfcOF#9i<>O4+#wwgqgFY?rhB$QNal=FW?@^J%%1;1w*3;`j*_;Ta#!B-1+=z%12OK z+z=UzQ6oBKS7b_u#cU4}^ljUnq^#%LKSCzv0tD<$Kt#v>;@Z!j9oLsB`b4zr*W%;@dXT|L!r0*WD*uEJ(ejgklrf}_K zbluqLrKG&0SqE@+x>{;fYspRa2lHhTSI2G@XBKhj-(njwWtGINWg4zllbG=@U6H+| ztBV?~cJZPCR`g0iS2`SU7NqIyLTy`H|DJnWE`z}SSw+3y&?Q}R(!}3{wad9(3b|SX zW=qeGpE$wD=%$e@ychMAg!;evIW5kQdfA9Frt>L=Nt3Cm)Xgr-J8M0Bw-r2*kGy|u z^YqHYmF=PV4zncZoVQjbg9NeF%b4lTQ>Wjq5{puLES(}M_k@%zOmR}A)jcd;-JZ42 zJnqmG7VS8o*ny917;o)2%xWlzu+O;jhgnfUuMmD}+A?8!n8a)-0Zzb9g|0 z%-^{TS05)3(Sab&g*7hoI%DbMyF11fF)7g{k7(k1L2g9j~il{lwaN? zYL$SB>mK@Zod0ZdZYXaUWhkEKTd5*s(To9>K>-66OfDMDX#&s#Y1tr=__RdF5*KSy zR<tg0(NV8&)<&=`mu_+GwARVVXoqfeIn-f}9-^EO#`_QeTs zA2v=M&~)8<^hsIpjP(86IVlFXq{i5^yy8pEn{_Cj>gJ|?0%qEU9EJ@_ifKcvWyJcB zeG=dD@5;JUu4itpkaCAa%DB{EL{^loSk4#i%ZID$9=hsD+j!(~N>eLKDqrm)3MP z^|J4ks0i2clzWp$w}i0!b!$75vU~B=XgT?1zfi|f`_t;Kf$}R`+9&O>34OV*D)*@A zdB~H9p2PzNH1S~pFR}`kTSt85Nw=QS6TW_$T!fP|;&p2bqpo?Gx91FT8*R4Ig3Gci zTdwW@WAYT9^vTI{Y_eg+li~8?k@`+g%i1%6eZ6_E^pfmJ;;4f0OS3FryKD@0pe`d< zW(Tkdt{raJHB5-h;NZr#ZFA8~p|zB}FB$Fay>oQq*Smt6>d06(<1DwSWXd!{dDZlcXUtkKPU`gGl_!=vfk#8a|q!Cl9P92)KYRp;sRI5(It zkwv{7pjh_dgD8eU_eXy~_UF@7;hTmnG?0R=02PIGx zU^})*E`RmvC^~|W*~F%#hXdS>n))ERLaB5Fi;50?U|d)!^6}zNoLFi9DUaPN`cMK5 zI{H8WX4u^XK&TZd7zG9&rZ0H_g^!r}d;l|U1VW1@28rw;4 zAdW|}P&mveepO3H-`(tpcBxrtsdJ{rv@=V^*tn|C_B0M%yc06BHqleoyGKYi4zIVf zNhwYL+DHd16|pk!Y}3&0rxE{(rc$^ZCG`J|u4#Vs;NfZ@?3S| z2cc_%0=cssQXc;-fNsUsM$B!zgv*CpqAzo+!p&4K{@P3S^rdx7ZfqR*R2$k$FWHjy@~+yzRj(^^|#aIkt`T$=YtOni^S;Pm0E{dzV{$bHO$}c<@J;^K(bH zA*03w_I|Qw2@DESF|AyFkky-On2jJ%z;y(@Cl3PcGcW|6GEOn)PP?VbsfWNAUJ)`8 z)r}M3v~P?r0N!7-r^Kqgt3cY4;pwG;Br2+xBW^jqzt^%g<5SY@@m5u*jyzp8wUxDH zh=~P7WGKhaaSGvP7cV~8w0X1PwQFNJFzC}5mduTp>p!6cPzGp$K5voBuLoT3a2unP zFLa1Jdh}b3CqmWE_D}6S)VGQ~%tjZKlQVBpWL6^D5C93k2_QsKmE{EUkm`4vImth3iP?lGQ+Yh z!2s8BG-Gnjf>ERk}+v%RTPqS zZJT4u(d_P?v#o?Kvyz={Z>$SwuuYhnYG1#$a{cs*i7JeF6MZ=yJms=l*p8hF%s&vN zw#7u1JQG+`L6rY!z~Ym%dH zM4I5x38s1B$Eu2bsr;~B*6l?T#AxBWC(fSDucyn*%>-%1y7Crm6i@d$ML1Fk{DIk; zA{#5`s+LwM>WV>L*K-JbJkJ`&J4$g1W8=PADpclr6K~Vd0ijJrc1>5&cHvRc)%D5O zIWFC{aZ^>jxa_G>mx^a{aThUI=6r?;ogfE ze>NufSrpj$#vRYdpiVT1G)>&!6=6@w*GLZL3v$1xE!T3vy0%yx%9g>=(JX*@9iVeg zE(c>KoDIv#3Qy44|BmtHmMHM*@15>d0~`S3w`$%QNCFYFPBZH&V*@2 z8CIJ%22~XVwFzUH@sX#z20v_p%E}zTa|IVDKI0+M9!^oG%Kq*HhpxBmby1<$<#5NA zKh@fta^%N(4OBVZQlfu0R-RHTvrI2?FpLcRHD%DNEW%^lM{#`O7oHPoH7 zmoByuy3UKv!8mVN6Gh?=-JcW#BJgQTM!And*4$V6!pQ+RU<{ zg`n&tZf0#1O&OlTq?v_~eG*NFJj=bO%%Q6n&T61ZwVbJ2@=w~BjvDo-2o@sWxNaUx z6DrkB|GwTTu=c5N`;uI>-()~6W5!77k)}u`BYRI^4E9pzMx8EZmN_*SBPhEA5n8=3 zfPP!9t%A`vdTKMCc1=bM$w!ZW*PAzsa4W)SO?RIAfOj62mIe%?HZqQ{0(1huY}<|x zxPvdSE1;b)VSq599X!(ogd0%`u`>DTG$Xkc^mTbl+01A|*S%IM24L8o9z{qRBgeiYmC zVJxH(LxdV+KhPb_Ae$c&GLM&3Y~&w8VhDHUn=a>g7*nRKTuHpFt)|ujAq5gAj z=Zb>yZ6pGB)l@)lLLEK-SQveGu>+vw0AqF>;%4jtfGzh(`#!6D^w=?U+cUngD>lQa zfRfSzv`hn0{RW2y2G&x2!#s~5;AIB_217Vu2*$ycHDh_78UpB4P-^Fv?FUZ7#OACj z6p;f7@jN#qP5qx3L6lwbG zf6dUlBm1qKAH7u8L(F9J){go}D zJ{f(}U?{oQ;#c+_z`qN#Duev#i%qeDTe^#hK0o_8KbJ=`T@0+Ao`zR;s z^*4f78kgpiW2rMVVU<1mxp(UjcGx+z<{xV&~ew%fk!>T^j>W*)5qHnY?M!kC@pr)plyhaYezg zD>a~L?aNYJE?H#!xwTQS?X$00?R_4Se|BbeVFL`*>Kf&}WjXmMVa#s+k{*k352^EQ zso%{O!J$eH+~txzGP~5#Bm^8)bd9blRrJIOEwD#q)(1lgl;cF1vj(byGdDIz*t~K~ z{+l6v$(nv1`l!*c_@D?7{wUP8lkHwt&};Y%ocpJsea!enGDD{?>{@NCp({wn7 zmHrU_@WF#$iF2`n#s=o*&N^IHwgo0=?&Efm$`mMGaElN+?F0oux%>Ol85?v>57xMr z1jh30-0H&4wFytog(MohHcmmo1#u`)F${Z(|F!)y`bsgi)YMdTe>Z*n`>=#_Gbp22 zcehzkP$g`k(}u=6Iw>&FW7dKdr7q^Ks;Y{L6_nA4!hs>fWB;O4%c~F=O7Mzeg|&d0 z*-@Wvf||hkn`c>}6Ga#$OvreAf1q7Kb|gRAeGGIqBK!&p)t_l3K&8+a4rompKsaFb z9)trxm$ZQ0et2fas*dYvn@oG=8^XQth+p1ax88f7EYog#`{~-(ey5!T4MDi0@5sCr zCMX3GJ*rsk2l|EBnsxg8MVBju#JUbU91qd+C39nDX83kF ze$wv$*!m85s{8l-qxm$nrIa+JP!v*;)lkabTPZR^WE5FxJS7w%WM$8j?5#zSY|1VP z$4c4rf4xujeV*s{`hQ-p=le}Lob&m--}n2zulu^LJLT~K8?5 zfUWdiF!~}rT>hMUd1_jHV~%wGopm zHOv?t_C3wld*YXH?@B1UV`g7uSIQ?7#gaQ;t_!#Dg~@A3jd*r}!l|0%)N(uB`;(^A zbdzFPS6$P@;)$LKF%RY5*1bn;a>c~LJLj^M;=iu5n{qjQ;)Kee+k5Fd=x5uzy=t?U z{ZPRREm=SAl-S(i6B@|eH0OZ({5Ue(9T8Dl%4fLBrO7q|t&n)_Z{tJKpzjqlHf1$c zD7<*l>h__w6MLa9<$Pq9<@pJ18!-Sn$Q1n5TCgNlw41-{Dv`%oss6gSxbzZFA!8}f z%N;vj6&+`Chhs4@aQyo9LTCnja!3yjN%;juZoHsfbj^kh-72<+HaufB+Kv7L@j`{; z+3OfH^gBp1uCz41Aejz~45*Uo7EUCglLV~=5VVLLR>T(QDw?#lPR+gqrND0F53#;a1NR@EWjmloh`Sx5^N% zO`|E*xH3%Qxw?tqz+FT_)bpfOf-cHb8fIY8;JK{=*Ny=pXUA@qjbT+wUPzWu&OWHR z`+{a)Hhv?P&4GG2O*B6Y4G@;}k)DN(4j!hh^bbPrl*{c{OTCT%q$bU-g@r1-h0U*& zM6J7XoF;3^eMfnDcxw17uE*WDQO(r=51p)NxxufTbFaiM)mKF{m_DlLw@rwAt#M9E zQSot?rMdat>S`;~G;^OGyAWq7sX^t&!R)RaPWIP0>Dw2#!H;Q^@95l*GXBX9wlOu~ z63D~do~uiHM_%c&8n$!J3~wx(l~GWbk^6oQ1dWQh>9CmTce_f4@t;XvNbWoHxF|hK z84jOC72Pz>F5SJgKlBD4m31ZuGd4%m70$>S_~y;-jC-vRe_$@RL@8rDpw3h~*CZ*o zE4Cgust-F8^COdYb9>n~3v3ru@1H7B6~1`<(u6p>+J_5j4%s=i2B)4L4f2)KP5;ch z_>m*$oM+jkt6eV7dHvfywqf@LNJk5mecWI++y3l%_0NHi7t80IsNOGsjM=okW9901 z*s()S%;BtSSDP@eql&49>6gZd&1O^M-K`i$_V6nwTa$Nm%Nb8K(@(husZ%}KV)vDs3igWF zwDF3?$;}OBEcNnLP_SJ%GxBk}RDxP)cE_X+Po}PYk7;L@+D!IQ49~a}^ZNBFxe3tb z@7N9`Uia=7ap37?E{vURFL55T%U{Q0|MN*`kk-=x{sveF7_L3n){~cU_Qg4B1>4-% zoH_NGp1yvp+O-i{s$H|8rDZ^fgSnZ<2fa)uPoe8slc5#f-g&8Y7e3`e_9>e_mD6Gl zH_CFQf{|-aKlfmMQbRWSiOOWm$UeH1PQVFNEIA-0e60E|4vnk2|qlZC=!Pv(McRs%BB|(2JU0RLa zBpoK+zWr%&FQDLRfAQfwvL97H`oLat zp>oUWbdx!oJM}Tl_@lVbGtqo;q-6I!|F=CU?Skn7zh=FX-Abw42ff@3rUb>tOc!r* zIAHn33c8p#N?q0dal=k-% z*#=hs+^hTLEDEyKScMA)eLGYgr;6WR;$i##1u_M9D=QOcZ+Fz5BfWZJUpe*iHDhOI z#Om{|Er!uULB2cl&tZ4HOEn!{RAr5TGdelPRZM64n@)E77jACnEd2RFLMAZnXk47# zmaUn9O2O_nD!$J<^3^B7WIWNNBvC`>06-oo@20)wPqHNo$KME2XA~di<*nJPdtLk3 z?LE~&76VB`wr1ZNH+4r=M2?<5Z9c%sHcm?voxS4$g4)FKlM4^tOB-sL-2TYE$36?6nn0*h(YvtUm77o$=IESmM2N#}HwveX;waAIUwN6jZ@L`wb;N5;x zfydJNV}>Cmiwi|#_}g^iN1xlg0Ufq>rGrDM=M(+>xyc6Isa^|h9=H$M$p>|@{ir=r ze8;FLO?G5;dey!F8K87^Q_h*XYFr<^J}CMZOotApJCAIX(@~_Zr%hizDiWP|xJJ)w zm+9OywOs!vvCk@cS~hBquiVsW+qOq*`n~<|$YQR(CMR)F9=_eub{q<7vGSV;`S~P@ z$L#u;OZ{}knC{_tzR42DbsGi-IjPfB9nHC}83O!S2bgwUN$XQ<^0jX_Nzbqp1~jf= zu|h3(wzI;l>xWXHhyyBuOf^0CH+_eBPgSGs1+LpkE^P}i2SE0({?Qqe_o2(~y?!cT zS%2naUhjvwN)_NL=4&;WA2z>eC|6b^7C^-dwsCfI{lC!CT9+?fX)8Hs(($ zrjJFij|{d|{{*pZEzZMbb(+a%{CD5F`SD1n^HgSkj#ZTTV##Z>6Y0>`o&YwOwRG7T z?tV4ZZ`E?nIZ`E_zH-{4#)`Q@<+mNyZr+@TseV`MU<$V3*Eq$%ztLr-TG`v@NJz!U z0HG$PO`zg;*7${lBohZ0zyyGhnD&sd24Qi3Enhy&CA@ig?Fxb8fF$bv6l+$aeHxAc z4Y_uo)u(f2Y%Y>{HSG+8z2nQ4F2!jSsw_0bM`<5exM-0{=_5|vv5g3Sdv$Ui4Z4!8 zT67jz0)eqf5)8-1Gf`1YZEiqY9LhsPP(hX}qFrSo;3s_y7KNN7|HV+|i?_x$4#SiN zcI`Qk8aSy{jG-?@%xU3T*H^QC_3CgElaSYsX4^gu?`1M;cssr|;yfjG*NzUiTJA=3 zlKHy!p3w9!wu>p6G1Sp?UD3>Udk1+(zpf4q4|+TL*H^VsVOxXzc!I+|zY;dH*d$kWq^5#90)zCO-@pq~^*A{_o92}=Wi-85) z8AOJsh%q8leQJ~b&Yz-BvdJhmBrDwe`!9K^yGjSm*Y%7K?dflGR$07KS4riHOmXod z)7n^e_&LgYgIQ&{SHntCN-7*JmY>7DHM=BX!0=I|KcYczFlw-{Fma{W5iY6HQm*PJ zhkHjRd)s26DDm~nv7WJ?czp;}C?X6G3St{?^MO?1i3VrZu=6E+=~suANL82m#-MAr z5$vWg*ZJo$3Nz7#k^so~(Zh#Sko{mzyWHSn+q8qosCu=CNMWu)%466#9%9@w9p}kf z24Q#o+>~?f@Ex(fn)6;eC;cVBNZYv8?NRG zp6O@2s-N+Aq}EyFDJpo$!~Pbn&5qU`3xgZ4DfBOX&6(oBzwpF@w;D9%(axh$tOe0m zrg+Lg%KPr`P_OCP#2C<`6Vxge*`ZZ4G9^`3J5yB;Mn6xcR*p5_8H6h;x?pm1AQ7@m z9`#}|i)ee~IceS_MHNbY6Dhj3sm@3we^*+K-5I&J{eGu68`#@@5L$S9ikrQ0V5Bj0 zn(DT$ZD)>TZvk_DaYo%cR%YKHbs8XO{LG$wHOyN(@o+&p?UOe2U*Hu|cWhF$8PWE< zc({<_9x`2b_EVpeZ_;Ez?by($P)^ScKHjf1FFEIM)Ze}zQ?oBIs2W6f^;Ay&EInQE zb9m)nDaJo|7YmLv+|EPF=#G{+i!OROcX)T+#JVBQ>Yz?L+CA z(ag3p-We8!7!C1W0@~vCr( z19^n*;mpkImo7c4PYh7xo_a~|oLIwKBspxW7~_Uj321un78Fzu4;N%w_lSl-fP1-- zh`9WEKcxnQI2ULoT@Y`P0&fOhE=M|nIqMWU=~PvXj0MnBp+Ik78;%%)5kdlcv$>ty zYzQVZOIL_aC1U0Qv4DmDz_mtXX(a!G9qws>DoWzwF0NAK2h=j{ zI+chH`pDGu3jJX1MMk-y=A(0Kw&E2o9N6K;?;r0cq!`Tj>540~ABp$dqwy5p=?8P9 zTp7Z`F|E;;a5Dii+S4#(ko;;g_LFhao<3>I=pLFi$z)(6v|3he=gvm+=;V;riy{ih z-u*f)FyUZL5aO zZGW;Pjr6E^nav9uV%f;{^;%)X?B;h|)!@C@lylqma&y>|bR}-%)=pOY$V(Hv;*FzQ|>y!^6NmUhXZZq#XQv`+^R9)>jyaE&_T(LYaI>bTKI%+sFEn9fWaxg!>7o5{Ey1G| zZS`|KF&(vF*+DMT0v(r?PzzCN30}&&Tw;J3gb_LGnG6~}1Xqyk5)ocQN)bXFu*Ktc zo{?3}o^H928DKg#aXbC5YE!oRb$7&C!=P2zDHU}>mbK-o#L0uI`E$KTa~ErI?R&3h zI&0^(^tpR_Z)%6{s9BYucEwze!o);FUswM}Sp{&lqufvOi`bCZop?>&qu!@7DL^u> z@l4zFwZd7!;?w7BQ@{zzpY&blr~1sg#r})f7VkGE^`Q6cltOB_!r2GmUfLnG-qAHv3hJsnfT+y`jtId+$%C(GiGbuQ+cAH<=Fj(4uuJAJcEO#)zUGgD2r&z)%G@0W^ zg$8bi7OL<&i%BnY&QaKv5p~9UJQStR#pnHoE#?VyU0j+@cS(k0jO383)@~WD)GF}m0DMSc;ekIVIV5*_Pt3n-sYl( zBmpfCydvs2GqV@t%px_@mnMx)vR!``HA`22&e`O4eGCrhkvEJUo8l8Ie0}kg-kkk2 z+rfg!dzmR87k~SJSvzpxi>w%$HsepdJIxR zOlDG)keIkg1J-Jat-2_h6s0c3+-Q&`^j$vmC?FuNyW7ZZjfxipTSS%_Ewc%XH))71 zbo){JqPv*`qQX$_mVNYrW3QM9*2KfZ<5$+HuV@+ON~jfv<$+Pe2>8~m&IS`q#h}fM zwKc#@ur}{-gC2=tv5xEAPms}bt(XFg@Ytf{JUMJk&{e2d1K+T)$eB+Sp-h0_O{4y`@r`Lh!?j=i=Fg+7Z(cRo-Fzy`3W`$1@ePF83;=C zRxgboSRPh&ZO^t-`A_>LOR@%^lXS}>$xhqXnd%21?)2WH{$Rt0uA)Rie zK>up&7UGh1dr_)=SMW(vCZ=E7$R1Sw)Nikw0I1h z=2#nAvadW^!R93U^P7h<@=g`Q9&|_Ny5@#4V7emSqRo$}W|!Bw&QB#WPB72CzkIo2 zkf|mDsL+G?h?fjU4@w`J>#Du$Q=O)owl!uBYRr7UvoiKd*Qtezr#KINlw%c^G(nn) zASAU{Q$Y8{3qb<|73aC;_vgq!mIt1jugO*%snK}x!pW|WZH<1-_>fup*%!O_>K-X5 z9chRH4KYdcfCjp#b(P^uo@)+H^PA$##J!JCH;w7uCDxnIo{r!e)(+nFJiVnRqy9=WE&O7R$!Mr#yg|v^rA_GcEvo2?ZxCaNY5&_L zw^4F%s6Ve1+^WlJy&FgMEegPP3p8+a|lHB3GSF4(SIc(Qe3hOLE5d53O))JIR1Q0D~w zbPa)yv123cZ+}RIiOmdWfnt6J$IKBM@`OR4@_}jb@u*%14Wa}uT|Fu`cWo}W^hZtT zDBK$F%a<*tU^X%@~dIbgBntBtH+ zKxCy!^Xs`r(|qf1@7?YD(=VJ*w!EL-l*F?CsqgG?%;(0Ok+@uYQOss!yA4R_nzE(k z3#JUMJ&0Tk4NZZWQa%6Du<51eS3bv>vYa7wT zR+f-(;;XAVHwTUMw)|Q9AFO(E&R~B$#u_|1JTt~$``qBo6eFYi8+`>#Kg***ZNA&T zOywP&mgpk9gppC@)TzJKbTBMa!q@~sMV6ZPm>0?g*K* zQ!q6k?54;dUSoixe7wVpd-v|Q3A6YGX37nb6ID+RJL#CF|zLBkSrd<1NJPOObN`=0-#aWX_1HX_aC_ z@myS$0(oon%VjF$UB7Lv#|nC0N&hPW_FZy@apz0ML^?qCGU0U|(hsri&xzz# zTWZ}QnQ}=m6*^8Bie}~5xP#i_szUv`B@adlL`o=C?IUMf`{hlCspBR&v3$)?Cwpyg zLSstOn>+5|$Gg|YWNhG!kfV@R163HzHRV%(%k0%!6oj1*!pD8Zh_ip)g;(zo_VC1xuP&;#5o`^=9oso{5!SL&_;qsqH|4S1)%Iw-lF(t zhOQ>9g4%lWG?BTgCYA2V%~^^E;ISkCDs-{H%V>rjm2aptIl3u^L-@6`Oo z#he@*u{0)uZ4FdrT4i{zkb@vbF&q?QTexrn|8y$f{(xGHtb*jp*E(6Y{efgAEj0%f zL;{2?zxqQi#r;1k#Ku^z=c!S3iF(`h?!X7^8`J?@nE5r424a^n$T9cs-xt_Qn29cp z->ij3P>_b|+EDXHHGKm2EisBG-FAvc-5*;J_G^=cFQS2+79Vb1NF+NRe)o_XLaaPtU znSeElO)2qa9mN7t=rdOr&P^3*>Nx|5+9as`ok;x^I&7fC8Lm7wWZxY=-gsY?_vo>i z0}zf1X%&`R*}1vZ9>1Xd_x6ta4j!#M(T+Fu8&B5KgwAX>5PFt8o!{(mU05Sg#_sGo zsij^^6t7B~Mw}N|a<1I1*6~864zHdnO_}Bzw07;p;wiETOt>3GQYJNO&xva%CGm1? zQ&w1FBmCG0%o}Zgqc=8plWY2$<8w;RL8~C@s9zf+C(XJmCl#N{eoI^V;ZNONRgcBi z)@F2fi!-tY#)cpBkiF4}Fv^dmLB?-|jrykh=ZhS=l>4_1kc zHD94OQT+VA#qnag5|L?HemOqU_o=4nCb{{6@z9K6O1A{G&SSM0Z%GCtVr-GB`}zZg z=9?+<+2EvfgPTtm#MpqnMdZlPyA-EyBCo&fg;gdSawHC`%)aG{qcCVTBIE6D#J&H< zjRB|kUbo>pc!j{In3BMCX%Ic7Esn!g4)z0R;1DVr4!P@%gf)YP?3*`rtyw!5RT|z> z7HXDj+VZGH@kG>6vNj2(N_l0jQ0SEpA}eenif6B=&yJG;P2IaGERSFE!Aam<)%z8n zmyVh?p2x6^KNq8K9ep0mudiWpATR&)U|Vx9=gKXcW5U9UuekjHuq||jL(6*@EP%Q= zCFLZG4qoTOu?cF+L^>5ByiFW8jqwKsyx=lDR;qSU<%u4b6kh;1xY=$m?p_v?622;f zYE-H^tVd_vx-DlfZ?C=SyhCV(^F-%HnDUe>Z8;F1?rKa?jz(d36M_V%;$YmgfJe7nO7_U{2Grs!ZXR68NNK;l*iDMZumLB`nAq+dQeX^J}#_Hq=|6*AxjC`x{>*`Z)rv(`Kf zw8edz`bUlB02o0|WRQ9w#4>F_1&=qxoGrTI!`D)UZ@F?50c}@RpAe%>@qVsOk|(nV z*B#cG?34C{_5mW<%X+1p+1H+2&2fA@Z;#=>6%b>rPEr43gp2#2*PfBA%^CY#PeCET z@Wb@!u8J;un(nozi#n1CHk%IWK42$ET2rBlbG5rpE?<29tn+k9d5Bes=R+1g_7uZy zmM3S|`&6xF$$!2kMv(W=S5CVP8)DvOt(w2~v(pvB3Ii!`k=p1N{5dR779B9)A3rob zGv@!*T2_zaK<)&ajL}&xlcU1E20-_|PgFx+^{2GegtBMLa;%i(H zQqbT-2*%*{Q!iWblJCq?!yFO?5DfBYgoK9qMi?c=f{}rN9QYNW3{71RokC}ndq;%* zSQ~xsm}AmaBj)Fk(+>R*Wwm5&Fny4vV&^K%be|NfMv=_4YL$G2RIeYjhM2=ahCb7f z>xuR|=Z3ZuN&}wppuI3kXu=AG0Tb=Y%u(QZWQGzj!cxQ3mJvag1U%WmPomoA1fx3{ zSlssoY4A7RNiYZtCEMJ9C(|#uyhi0t4cX>d2`ggorBZ;q>Bax$elC;6zVhT#(l4VQ zKnN1+ljfc>>#8TDn^tfN=e|vV-|JCCoYf-(GY|cE#X95I4((IL!-yZu8oN-RXl|A3 z9A8}>in$2iO?VM7O?{tCyApV-8~H@GdG=?Li43-up^O+IJut34dw!;Y3sTv5f2EYG zs#9@3vuCsZ-kGs}0-4t`I;7ljMQmUhI{kdswC&1dgNz*a)~~ah!eUbRCl5`R>6xpQ zcn-o-=}HX7cEu;Y?=(gaPRM>Z2e@w%%f9xtI{BL2##7WemOZ(_FH~G^!>mdi+}WY{ zrb1_&BjnQ;={(a+Lh=uFJ?X>C^kXy%ogJf_Z$FWSNi^Wm)29T1%F!J^C zUM@=Q^U6wjk=gMMT^kIZIpeoS0oTqm^rtZ!jll(N&U5+j=-3N%m>JW!5})Z^y_$sy z(FtrINwVxa>vAb0`XJ9qL&Ipav3J1T=J`w0TSV(ZM#zqT|NP@;Zm#8aoG7r}60m4J zf>CHyF(P0JU-|Z40o!KKWJc!11(y^2!Z#DLTlB5hQB;h~3n!dBmbcGnw&8UJvSkgo zrWGS1C`Rl&U;Syl!`Gcs=~0WmKxhH~IS(8!A1Vc&d1nOw?UW)!4B!O3)Z?VtHR{l` zX6-SgWE;>pn_tb+fiBVHJ2}CgU1Fai(SLgSbRN4@K)?y~)_(TPmFbN(#40593LE+` z_gxeeIvSBWESAyxrK*=}uPsedvfxQn=i2;XklLYK-}HEe`1zVup06%`xIjE@lL@?b zNzffX@v4oGnq6b2dgG&cJikXk1^|k=EG!=#KVUV7ey_=%Pt>;ej>jV zIU`vPyznR>TN_7HPmo2ykO>?c9%7CuMs5$-f7wTFUF_eW^(o1TL}9dz8v3FklK`z~ zs2l4bYw%bG$&fs}ganK#x0Qr!&K|*Q-Z%shw z)HFS7>$ENQ-PT{G`xo!^4`gc9#RRk84-9ABCI$HZh}Vxsl(6FLYP@p3leWF~y5M52LX z^$KaaSx$-K$=AGbXIeSF8VVZdCSRd_IWV%izhJHQi(V!>zW~34GrKItYfIAHD_=Vg zScq>AGbs#ymR4VX;IjQi#=zF7`g*%UgoNu;hqEJJ$DQ$C%GI}Nb6-s8Okya;4>*mi zW-JmhLXQHH_ua`DMK`7JJ8(2850vz)@gMhz%@13y5}jfM?!<|st6r3 z$-O%C6gG9l@H7EzVrhF42n5#H=8lE+t~mLoA&J3gGcx+fq9XAm&GggenN3R0n0P`a zBas?|bYJLtF@Xa_27oyzuwx#Co%408w>Zu@xwuYa@M`g``7ZSFC9sWB0Z5KiwWf z`ua=;YS3!22<}nfzi6ufTS%JkynRnlkH9Me3H#SL;p^kiwm9_dpO_fBM*r9*+eJlN z6ST1bNxBL4`J$cij)W9^_)0W^nSljEr1Yx;XVrQ7cL&a=<-^!9EE@4+c2ixW!ZL3c zgjsfju8HW8e&O93%xAtvvmCh?8|y_+Qcb zYxO?<+QLpdNxRc_?nwvEsxROyyw z0kr++X0QRPF-|KTLolJ3I(CdcCA072P)J6HwNN_gzBOf(_X@u(?7=(6qS_hcVHwVK z{y>m`&^Gn*epRzmyblI-7s!W7d*3cu9A{@O?jtHEaN*#+V0b8cb!>CV{pOeUMLWdM zX1UtU!Z+QA$Qz?y;yxFx)T~D~{P`O1cNNqrg(43)nG3(plWDBjqAqXR%Q9;v=kd5E z0sCBxKPb|?DFzDr1iMRqIlRUjZ}1MS&m$N20~6mm1^EdQ}!!?q&*>d5+S zrAVfVtgLu398^wxu4ug+#mk!IKd1VR6r?tj*K#eSKD?zn@6SBCUW4_%K_vo}GDmTT(BsRY3 zv^DMdq#gWj?*XBuE*z0&v7z`k-+;DXCZ^=GxCosW6e554eHCdvU=!<{=d$|7;n{hy z=W!nBc4AX=Lc#K25BZ+T$C$zD;9FBu!+2Uz41#$M>BnKaLa#r6xnyA`;94@s6DDVi zB1y$2?~9{|YK8hyLW)v*YyDnLBg~ZmayA^#yL&eovqv%+w_hXs;)yDgP96hQ)OhI2 z8!MrJQ_K{T8k|3l96n-KzlR}>yuXCxJCu=bu*}ImB87?xW>pis%Mp1GsM^uMq~Ty4 z0;T*D2B}^zl((L&!oy&C_kv6&+7HGn87x7JUMHrVpGU(3itq?flE+6zzD_8I_zk%$ z38dJcgT}&Rsc=*a>0cF{5stEzVPI4LQM15L+oV^|u8)XlZphH#stssX1n0|0|HD^5 zlct@?KL7ByX=_rs{#V{m#OJT=E0N2loL%5qNTCt8t97eZwX2jAeI*}C{t6vGc>i9S zX}#c#>oVAoClY)e%T3e_af1NwFxuhGY%&w4KQ!MID56j_zUZ zEz_b+7g%T@oOZXSPP7&$A99|}`}QF?0a@l`8pJ3)x0rP+kF6B3u4r=oYO&w+bdWsb zrE}CAeisM6XSZ~e8gIUBdzn&ps>|w3TF1Rthyj7ItZJKu{I&Fv^8LBO-hGLuk0ko_ z7e;4~PLFdkQ|_;NKg2A~nyQ&ZYt7G9(>Z;5=~@Z$E#i!L!$(yU>JoU6U55#ba0>`1 z6NCiVrh+#P%T@W&Y)P2a$axVbXT@wG=f{d4}!dKQp+A5Y(Rc-OcXr6X~E{)O}8 z-tzYGWnlx5%atWO@$rd3tl0QV{=~8i9|d&{H3!vSf5njq0NxIDx$vAt@d1GAVN<{0Zi1`2M04QL z$%%|Snx3A{qmjWe_=$8FBsy6qHy*MnT^oex8mLPRNCwfh*2{VNJQ2>q188JZ?bcngPbyIiB_Y988+UL9$_|brH35e+BKlZb61HeQ zgxzF7H2owDx&A|npss>J{UI4{rRgiujcsH8N?$sffE8_G%+we2DR_IW(4+JIRocMt zf-98HZET=2#A@9#IOuhdC)xj>RMsT;-!(0TV)2k+eZ&q3c{!^?G1WeYdF|Q=1k!E{ zLZHip(~WD2iz`QUYtBx?fPU$9~L)rB~vRh%{zegsN#_>Nc5kO$Y?&clf?} zN$*IwC_rnE{$U;mr`4g&W97$Xie`#U(oG-?(S)#abfBfTF2k}z62=r5D5WrVWIx@I z;nYEA{wGl7uIJV5Qz^t5it3Y-hvx$h$;#m)qaV)`Z^|6zvk1!@vRw2zY?h-YSyXYh%9v3Gu5;Vm7* zWFM1Nwz`(P544RPnNP9VcA!u>H~EmjK90(N6^`OQ`Un1dBu0w)z%gz0A)JUS(XT=L zBeTh31^{(ptnRSmfqiV`{x>HyE&fq}LtcqIBcSJMgA4medU+O-!_ zE+-LshKoZ(oYF3j&4(2M4vy6#y)dqNdV*vlkkT5S6`~jkC8{QbT>Y{HLcI#pm*b@O z1G`3Te-f&~6fS3L3!$h0nh=d;tmrLg zB_6Tn%fHddNWosg@0u3t-OuyFG@4 z{(q-vf77o2oudCjUPs=5WJo5xV|?I0hR#Yayh!N zlZ#?x|McloGQO0WgL9X3T#&*1K11WlgF*a&wrY;J()Z$cWg1<^$XFMv5HTFVHggJu zAMe6KXOUe8J|G1Fel^&OTrd9n*fiF5Z+e4n4Ui&(6!n2?4nP4K(0u`4NFg*MHgPe))39O@<~4CjEU3~5MVCxSpEri|9d*RC$L3MmgI zP0e`JXp)0@KRtPL^SW!cY}gQs!oe842hp*8&5S^=$iK5)|L%&>HI9vJSKAr#r+<_I zsYST?+={MNU^T~ah+I1@&PlHc<3nQsUK~hgRxQ*FSeg)yf(i{`YM!Fgu6bwg=PCNv zM}8~ox83wF?80bG;=ip3DC75rGrR2{*3j#ozlSq7urd@mavVOaLAKf>em*|y&Qd0l zc>Rw0S?uiWyScf`NYrnj4^P%D?0RcUAPJJKu6Q@Vmp^ODQAqA>< z%aUJOixm*m?>mZ?!iC=?Tt7d1I!^qDgu84R|8HlABK?%^->zC+Bf_8Klw#7%TZBBe zh%^$@tu*lCB#C__BHj@jRACB`ae*?yJl}@|ogx`DNGGwt$?OC~Db+?Qsk*?x#S7E| z%D>r5f7}NnV@beAsCpMUklWubtER|y`E6jGU*tGoZGLyf zMWzG?WtV>wQ6vXvb4yp^ZT`MkTu(WhXDoJ^T(_~vl39Glq$wlmDEtIy^eTj;EEErA z5^E0c-CG{U&M1d>M~>SB6#AlOP*{^5|48}4wzoajEDuOqH){^|PShX498U8p1<-C& zC3NYEp>~{oWaMT(2OZjW$psgB1X8)F>9!6s$Q?-AMR^iskm~$0mR6-sl9|J>m)f_R zu7zpN<1hfh+P=f%AUMRgPg51{f=GnP$pRR>M@M6S&Kxk4l8|sE_!}}#E@DX_!pEmb z-U3N9i;9XIT0v$!l2y1t%6tS89aXQh0_A4D3KmY<$ibc$Gg&rWnPv9QooLgBKH1Uq?MFl$Vb4HY$`-)U4@WU9#3I-oA5@4R<5(@&zX*3Qn(o?dEU zJy9m&U?%DLODABh)nv*v+~?xarbUK0@}w3hnHFdmHEeU^Cn3Rn8`JX36R)1cc7(Hr zQbE=gcSPr=uaM>n*WSG&pIT9cU~H&#sOgf*eb~V^UP~+8gJ0GO)>d>Mayi#{d$Q-v zhtISvOqbh&&#=q~E|6|K!DXA{foUM#k@_)Oro3Ov`oCrUlc+Z@OZ}I39EDO4xKKTQ zU;!oez-Bv!4ciVM{*<7tt{w~9f0niMt_n*yXs_=KQ?J!wPU1$~Z4&Ht9Y<71mKoEJ zSVFHs`TW{at|WynFX9;nS!EW0jGX zHg-8ZVGltRCzdt|JSZdEuK)CqxJw>Svjs4b5n^MBJyeUFiFov#E41xz>>>7mS1Lqe zki3+FV0A^YgUAy|lqLra{Ley1L4zm{;6F*y+&YsaU$YQlJ6Y%1pP@DggTKzs@h|Q; zJ1ZS0#*F%oURp0>wl+TH?u+MI6H}jO&t(P*2<}%`YP!kw+AGDLCH|Gh8zCR9p^Nh< z`$|#lvwq+b{FgIFWo*E|x%`yxbM^C0LUa-KZIm^->!|@22Q1eqXGO~kh*CSSnP0J$?iuS`7s~)_&z!9R`iEXKJsFIMq zDPj7Va$kc(Ld@m8Z&x7OL%)fhG_$hy*N`)^JXlx_@s58hDKjh6ASSejCC42ASOV*i z{Sa6PDO{HdQG~!h2AFNrpKt;ruUKG`x^!9%&{q^dbXN}d!oorVvADmPa;ik{b@#4a zC4dER1;&%)1p#KVCUiCuh(WeCPpBxK56X&4Q)+%RkTpfXfQ#78Hhc&bqvF6LUX=7! z8d9Db#G>TZe220N@xl9dAq0|!jxoZHs2Qjuct$lC`EeojlZk=DzK|A^`iLbj0_*l` zTS};cUy;tk^EO_C_0@>c(P#4#rLWXI09`0arc5oR#tfar`{`Aloyx~!$hvmI)kw~r7Z5Tv^}QLlq5QuG^^CHcu!U*s)Zn-6U! zKI(D)dQKWx5u?UsWo0#Y*qmaK|E02F0lH`+1Nqa&tb71dY!aRch$GiTpj99o9Q35~ zvK$gaa%>T8>7+X|ugRNxOw*zBDJD|Dx6m8tR8a%nE$WQF)eI@ET{Ph@oGd3cqfA(^ zpk8nh7VI!=bGr%BLhw{TUx^_~f&c@LkqrRYI}BBR=;-(Fmx+cbkk8=W4L3JagP`Am z9n%@$X}6PJf%X!D1i(DpXQQk7^O8W3lRm%3jmhZ1M%Kky7r}ZdW?y6~Ikm6&n=w05 zk6vl>xT^h>c2DFgG-M|D^PkH_@-Fv3jR{Ja>%9~$8~JAOB@`PE337y0UNH;F|qhH%*B+OQif(zG8&Okyg`AmpWZ-5#bydsaI>%z8%|K#YNoIi=LF62wPPI{?tK*$S|@*u zLu#eKvdZ?y;tSQY4dYTSKgu39lXS%c2-7Sm-z`72Z&TMlc9UVZAL}gn^JKuehP2EK9)T!b#eLMH@o!F9i8bkK>2B zahm?-{soIT)F+Sw6}k$$;g9FNdR!N8IlGT^tlPIBO$-%+Sv6b*=T}pVj7UEp%|Fu5 z2P!pU0^Ud=7AnkpwkPM+H-Sw_)3M7aHt+Yu1q4edP&PpN zBLVJ;F0m#qzDbTU7s=hEL3%w#;^Sv-cJ||#tx_4`C5jmQrA`%PUtNM)Z)%0866sBl zrnpNn@4kJNF&{o(-T?gZ7~n7>wIN*?cwZ;<5rkG<}S%c(||MP<|;cs&1^R$|0ngVE; z9P)Bb#Hs~vKQ_ciaF1+^5{Oncp>xw0fln64w1EBr%#B!!!I@qup$UmjJ{euQbgrTS zBz{tinxH3_WVY5rF0Cd0o!QX?=&r4~v9W0ZqZ&a3(6_)}GO-q!rl({>YU>CKkmceX z&*$tP@-X{tWTh7}&-IXnh2`2tftT&{Ax?9 zwIGZtD=O+sZzV4jPaGx6)29&;j*VVyV#zqUXy{J->I<=m zxxd*SRQ^o`Lkx`MStyL3$S?wO)~$CROLxP)oNRYn`6x8hLY^KeWF&A?NIRG+1d<&( z5-0|12(X?8TOrIdhrI~EN6j4TI6!X_ape67T)?fwYQSLgOw!th&wKVurx!0?&|5zM zW7LtHJexOf9__I4D<2#VcCFV=!b(eCLnA}0pPm6 z!B|^qQP_~>8O_iT(!;?a`{xR)ip@`G|MM08KRwwXtPTi!Wu)bkg_ZJ{rKW^+sKmp` zS%qNw2kJUWFF!GR7ChcXQO3>s`ufA90;QXVkqWds!i)4|FhXd4HQ5y&sGP^(u0Se& zqi!;HXd}Pj+qSm0gxMdRWuyqT0n_JqLvATpIuR%b9qK?qb_8AII&^R8k+>(ck0}vq zI%p!nOJ4sMy3VM88%LV^Gg#5J0H*;uXq+RikAy8MHd*B*aJGnoplvVP_FvM0#_h| z6bu&>Y1^;8R`b^G=3;A!eX?2QiA$um*%zG8YP{K)SDYi>VYNF!ed8?yw2fVIty03I z?>xT;SF-=zX8{&%%YUjR3ZwebXQ5TADPdA)z8+c@=K9x?B?_P?wAdL4w&9p=GW>hK)EF; zD!ul1xecA>akL6+LG=Rjz8-Faxi6g%4GHDX>AM?SESAG_&7_e{X$(NmK}?J#L{QAG zjCmDWCs;@|+j{ z4eFh1+-B~hsFcy1SE4~-JVn*b&nso-(U!-JkT53qK5PhQI#?xRzd4dMRVn9Bk|Gb8 z$%t_-`eK9}MXDA6Y;?fI=sLD4266;VkD=iXNB#kw^E71PVdN-yh;INHZu~Pqz_6 zu;894lyG0ammoCxqm8B*I^hRLy{X0%f@9Ei+8BPP`keD7S{d$E|!J=#R7bnrE9D+?}$N_x@`5V;jP9E z_m_#kstP~0n3O#KGH_ubV={glxCHtU6~fQ={2)Xeif&H zdk8~NO$X!)KVCha%%Hn&HmK1^oz{m+T|{lCy|vqr)yF@uZiH{sL`d~$qICqrcsm{SCx!>Gs6T2xKq zNOuiZNzm^R(8e2T6UY&<8GS?9+8A265%v&~F)Kt0Xk!gv);#*@3XVD9@e9X z_!BaTEPgitLWEyp>MOuV+N9VnHV`?>wO}8iAP@xY?_&`iOrvR{xk!j-NZ{iW6BDbQ z2>6W71(Hm5pLfbg<_iGc8w1Wjb)tByADfoUZ(v%zTA{b8z-cCj4D3Q0IYmlt6Hs1p z$ed5IU>@VeXO3JIIOiy1Y zi;Gk$w;4BaB=|Pfb(?gmz8setH2stmAAlIopz z+2I66)0j}fo?YTlewp`zOb8Wc$!YDj3LfeQ{K>_ecK^`d)Sf=Cy(l4(6Qjpp|thte` zu!cfAy1hh=!YGeryKd8_UKM|DZ+&SxMbGFs8rEb8)o*Tr0Ahs`kkk})62V`$h!&S0tKrYNNgUhw6v7SD>}b^-9Y3O zo}|D6>6+wW=wPQ~6&G-mqa*ME3UXu2h=UzMW*V4v+1klVr)>HgRfzJJyr=Q;aeLWM z--%qvqHrbyHJ8d~i*}Tc)#U6&v&vycEJ1MH`t|gE31*Xus_M5oLm(7L;AkLXDx5lX zD#@HVRywx)>7Da#!TeagsX&5fkM$LpRp?gZP$auOOp~eqCAfleheVblT@G4$4JDB>r$7oC5?D$+g6F!J2tXZYppDbiH8M z$h0v^C6ZNsbyFKPw?b&lajxQ;b=$?@U0sP+>V?e4xsu7cvE!VZe`gq^gwn2^&<05zEx`NR~l&}C(pLXTk<9Cu9 z1oMWLQ`-Ga{k!G4HpNqnq8DM-8A3sWm)w&>+X~y-+FuNkckaUW_3PKztgI|2ZJ0vu zlPRl51w;T)(vHVym$h|Z>BbYdns8AGX|X@3fUL8j(vG3y9`jvY!r3Wb0)QD1e}ACP z{mt3@@tN4O6EjYvWhuL>SD*Z%X+ETMz7<_cSXYio@$sk8y{{&0d7#Q}J(R7pqz^EV zJ4kk{f?h5hX@Ka*U~b`~5z_1s-<*p9gd`Ytlz6pSd*T-Y(x98kfZMvH=8YH5FEUbq z@EhD_P>B;#Ck7OlMjV+#WF0{RG#=NBt`IQ3@J%0gS1?Ymi+QEkW_y>`#G?@9`rro@ zk6t5*wmqZAXdnBFbH?KKPj9EmDK1`lQZAk2BDWh?;?e-5x*lKjRph(O7u4m;-01m- z9WS2xnrk)rsK2hi=j$jo=jdvgq6$yjfImlraUw&CI9Ly>9@^~3eq#G-LP8*o@z3;t zUzrDz29XGjbil^vQ${(1tOv$5o$wMFeGJM-0*UX1p1zUtyCZRu#9|r>h*pmpdJjA% zWslL8BU|KZKMsyXUStdX@}WS(;7 zWX!I3HO(7qg7Yr6HGa%U;-xAFk>^5lFRP>Z(Y_U&#oPOJg)wA0FZ|ab)t=GY${}&b zF(oJ#7=p~0;O3$QVG}>D9wGCGJi+|XD}!MX=#ihmX2CUktGT&3*3!l%9eGV%vX1bv zyPBe9&*_sV;BrC6Zvcc-o`X#$sg(^YLXyB!-Ef*@Z2~KGsRvCq|Fz&klVIo^CCh+x z2*5tCr|MBpu`B8kz9wl3fEoU_V`zi)UwF;dR_Oo8^js1@uZ!wA&Op*g+(Jp$);UAAmfUl%qWfHHYt+?zTls5-gP1d~F<+%H0HNWfqWm8~S@*E3=^-tKuTmMe2yJeO?9xQiG^z%X&y>eqI=PdqX zOJrAEn_qby@Sdtg{)now8HlW9I z@#lusOVFYzFxl#{*>?Gt)xCa)SGjx(hZu_xET5XpjTFwRVMNvjK(tYZ&|M?$Gq;Xb zEtaX9y|pD$Sfuw)|or?1 z%Q4Lx$7Y&##yi_T<@3&3xmfXIMta{z8{eO82V7Lc%x)a37&YB>aYx=i2N#03L6cxf z1BnQ~F7eG*sbNPiS}c%33ZB(~Q9p$}4-FB;rqBF8y1qLQ%l!@iEgel7S_h?JL`I58 zgLXoZ6+%lH5u%}tbF@o}jL1s%Dj7vqOC^!@7K#?V-a;Xg@w@JaPWt`6zxR($PR8qb zKF>X_`?{_>!)b+ku(N?ZZC{|nJpk-8 z30h0HJk@A6b(UzzFu7fDk>baKgy`sX{cY}jC9%~Izl}ukTLs`thP1H;kW&T)1;x6+ z@?)y(wphe-A!Y_Q%ghv77#sutdvEPsi&Lekwnl%dfRJQ-l6oe{f=|bHaLU(XmG%DJ z4)x;rQKkFV%$W$qxE^|XJ$3abT8&3?%!r#fcinkybf*I1=T)KpsYGe1XJAmZ-5nDw zYFq*I;*sce%Yzm#GWI8bUjfI-8q_$1?8zy`6FT?R7^u%mET7^m6&qV#srJ}OwCDJR znE@`*VzG!%3zu(7UVZvoQp&n8v&4nepFBfOaN4`hhr`rd%;i?QweuAykNm%{KdZ}X zqy|s*7Cl>B^9MO`cb*8Az1cebO1N|H-n4XQ?$Vvga(D}7Nq1XVzpy^i zcqqu&+Xr^MD!3!omV(5fqnaVD(f={8XBt9E;MmHIuQHMV30a{97;|cuQhv_+3rpJp z#Y%xHz);Tf&r<*j=y7xJJ^c3G2``t}Ngh=(*RHLiQ;rm%_Zl8CC|??aC<4__`-#yt zU_0}*m1SjRG(et*={Hd<{+60QpH`yHn=q<^jcm)@n!S647)^SbS;2N~S}lgsSl`I# zNcjx+4M2`b_GhLtSIq$1y)xs z99Onay=x#6n!N*OS>eu-*IS&|*Q_ZYGN;?)S$X!?}azOhw7Ge8fRfdg|myVR%@#Qi4*YZ+BtRmFYK$yfSl$!KK1WS zRoz?tqhK;92`}uSiJL}yNZiG5KjF(Q74Mp*(7m`Fu{wZN&)%S+L6#8h&zH7*0-oE^ zfq)H8p*3pbV+|C>>ZjcS5`eG~4R&4y8;YxTl(CFCN$DO|*AzPFP#jo(eCpScZm*n3 z=p@0FlLVGHkv7LtDm|UFG;=Boqyw#;Ok=loB;@wUL1rp(0Vlh7TBdVjtq(%5finvTD<8(MqOT~eWYySIsJm} z!j9_GZ;#8YM~{$o{We!5?(-2(^T!#suBOk7by-Q}q3=9gV#FS&O8=3k3oM3`ck3Iu ze*UrVJo;lKhO-1K7ReMijbL@HMS~%n&Uu3tUr$FTf(YsZap?yR58)0K!)k%FHen5O zfnJh{fq_5mbwzEcg)?=oNnc-|^Ch#6tPY%E!t~_hyMs!&IBEN4dP`V;?!3i6D*sDk zhfBu;fz6hHQn9gYpX>5TXRszs5oIkA+EA|I_3n&?5!QP6qgvC)nj7;x+*_)*K#Fsi zCCus>LmeXC4J1X)-PP!2J3!t$;6-sDNNv`HQcV6yd{BwqhfZb(4`xusy27=)^+0#Y z1pZS1hh!mWG#qp4zI-nl7vwxwAZv#sd)=xtlE^y`5Ym_vtK@sJywjRQ4eIJvmm5dE zQ@8$Cx3@f`DJ)b}Jw9=4!VRg|bNVYyzn*fEnPBk1{?Xz3ixqF)>t(Q`WF_tJ+*FDe zSgA~XS9?NerD;cD-%?&yQo$7W4b$TPeX3zm_j{PF=JWFAZ!Q??-G-915%5JTSi`bU zaEWq=T;kDHmbu@YCJpgPNl7VKKE3tOYv+IhJ-S%vH$Hq|PW5~D6{6$!V0R~tJCRN+ z5askuO=ncr-P-F-+-h9EI6P)2w@|kCD&cml8|?jzvtV0FNm#_KE8@b;<${$p08Qc( z5}a|!?d@q+=MnZ0yWJ4Ms;D)Gi%)kq!nz*@L~uC>f-qVpAKE%+RCTe6vS4J=st2sZ zTj0qQYW@jm7R}vz(=jnf$;!$Keol5TbYDZ3=7F>u(oT-!gA!g2IHdO0^A! zy>+S^d3LNF*QoGhcARHn+yygs=Xh_88Ovnv2yZjAKZ@r>14CGRwFDUV@1PAD?EtHCa!yUy{wyilleXSmg$UHqfiiR+*ka&LFAU^o zy{l1uQ+NbtU0+vM*Ss#x$`^qL18WOr8(C>89xf_cL4Y!xH#v)Qm(+QLFd{GuM4#LL zkD3|{T_FV&`kZN1zHa=oRu62{d>y+4fin_#SR5*oXf~j6Q0cwdM)`6d9JfB*ba4LC zSElA;*Y5MmU8du|vGXenV#8L`&0t<0Rq-DapZ+ehJPh(vr91Wq#J+ww z4AE{@bNl{)#}SWM@5(ODUd`N_uLuv$*sKL);>y4p(&DX~Hf^d=DUr?6}Y z6=nxY>NOqIDFf|6rg;2u=U8SirYZw|F6k>#4NKN!aJeF`?U$gBDf;I%t95=F$gW@q zmZ4GO+C*q?AgUgPRBB|fpOl4G5m|#!JMbIyMJL_i+(jT`?hzz=bdONYBitfC`QmZi z)$MOVWAUucH%Em!(zWJIE<@1`@HLe8C#vL9G`JrRW(&#ilm^`v_D}VFjhmIwRc~}6 z_f>D_g{aU&{8fSK<$4A}21Z(2^$Q-4_tsuB@u#^PSN!9R>yrakOE!zlnPa{E@yzd6cv$LX?1#gOzdSGbI&h!p&+PlZ|E_kG6A&8vx>lK=mAADY9ppq- zy}wgag*qAuZS>x*GMGDI2Gp>M$^MJ{;muc3q4ye5DAxc?Ollo8Pq=^g)@sOxkP*FY z?DgxS@$P6%NZd;86!@`xjOT-$Y6@FOk$=Qio)O_+fuMkeV*;`pmQJqLYeCi+Gh#R` z+4Kv1Ts?4yl5l~1_7>~@GBtRdKJ zVAO36i5TI~oDuQRqUNCI0qLSLL~tW0(7PTmG2C$*Ob`U|%7x1O34Kei^` z>m~ji)7NWb`oQ2zUEayLKA_&V_uluH)<|vlJ|z~jdZYc;jZVh&r@gUbf3MBEx;z(C z{;1pThYw%%moBnCTPI#WEiQH}*J*7C+L==6tI4st7N%PfSED0;-@SErn6vonfAlQ; z;=AqXH)Va{QNf_tIz&bmMx7V+f66+&8OQJO#o2&5G+cpP?4sUs$t;FDA-LVhAHl~7 zYQJt{NZ}zj-nn3kpbscQwB8P+950n^8a_D09jMYsPX!BS7~sCCLrhicp@(i)Ufq6- z2&$<0`R0;i3}W)Vw<1*{#IqqqIf3+I5)wiK7>wEZ>GG>qgl$HRkd*S*@L_Oa*#e@Q z)3c+qkj(a@?j_+6`AylNTfr>~Me(ywmYTW=d#T){fi)f&rI7cip<(zzoEPwav;t-O z0QzA}yq?@-{w>(9S%=yla04k>dAhWQS%6qdQH183jl6bEt?7cVqF1xj7VXebaY|Zu$a$yNZ(D5R%tt0( z6cGkrmI}S)zMAXN+?S~?e?i5YK4eDI(}$Vkp-0O$PJE!Hmr+tuvL;w~-e|Y8Q2YY! zfXMG(-tjBzz$3sQk({-7IQXs}HAO=zf&_@pI5!Jt>SEZY*9+c)@y(qlP>>am!2MuN zI7AE5P-m^LJhmc?fm-hG0E9BD554SixYfzG)Ez^;f&rZ(GZj8P9)Su1LuM2J=mn5} zYQYiAi>TgiaqmA$Ff(1i-j~B4R_qss%wJ4cbM-aRp;u))vu8pj{a0$;_R5Ut2KTV$}M5Grm;@B5xNz-C^!^ zZ(%~PQNmbXj@hrC!m}lZbEsCEyj+fS5&znAO zpfy)8$K}PX?dF>I$Ae+4i;!k(Z-&;Mpj_3yuig~k&}s%LTOCVCN~*;FCgmoV5W-&Y zYYD9+$QYTaCe{ebCEA3IiH}rtz~B=axz5YS;^xxvr7zJPO+K^cnl+TyC(Bqe_n(|q z`LX)g^Wm)iM3voKJtma=y0{)X{LIYl*T;F=&elXE$JybU&XWBQmF4$t&y{f=T}p>r z>th`d7C;Ry9^k3%kzKmfWK+zd_=8}RuC{7DfJd7P2gpkNnxKtcb3wYqxS#?wM?F(h z+j+C`BU-tKbKb-{XlZFZdn?T*{6vfxuv*+$-g1iM~s{Ce04CGsc8 zWFy73g05o@Bfd$nf74av2qN>MEAq9ik?%!A&V{~L8w#u4Mh1}|N1a$j85=B~5MK6W zZy?qs%i7MvI8acA$HQ_(n4I-hOozP-cg^D~T_`Ng7YP1Gaan=QHX;KLyh^hizA@Ii z10Oz!>-#cEi^4&vY6w@%!yiWWW_&a2nSVWv>vcIKWQKnBtCMY(X2?&-W*whzl}IgG zv?!X~xhaVPUgBlQPF_aW!jC#RiQaZGAxpRtqy^%VX*h=LLRT@p zyhey4UBPC!_VTiA>&+7q6Dx2s;gxTViQl7kyiJfL<`9?!Sj)OX8oEpJRKz?*s9Xc) zdT>loz>NOR>*jUE!)67QG@cXOyL&fU;MGgz9=P$C6!mk!+-MiY`L@nJ50ZnYslqXhvngec2n5^ z;)@e^5c^GaZebEbKo_W!QnIoaIGxz>Lk4JwYJ!R*Jx=y^Er(6TxYg<&{km=oieWotnOrluhT=3xJ~dlf;-W@c~d@8Vad8V##6`(b*t?lJtRY)hv? z&cBtm@*=1L`%4usBFa(WPi)5+na`4TB$gF z3cq^wYOJxFTW*8`eGM~?T<2wDV*{Cx^*lZ|FQ6Jt|0OCsNbvh(({vsPHlyumEjqj` zG}d9(gatUYZuJMVJD}I2VTODTs4LchK<_rw`)OqqsL*z-sRX7=m$kCWFL- zoF`UU4$IHmGVLY?k%zo{I+0@pn9b*R8R#nVt}1{TCwU!BKr@Fgh{R&>4GDTsTe6(=fMV($#WhJ z&r_u-8cNB@v1yRVrGf*V1S9~!X3=~ZO84O5IRPK;#;)<7V8X=UKY|R5T+ zx0epFG<` z{Ae}638kg&P?Aq6SH;MWm{Krw6Jqp^Js707?|t=;T0WdZPg@xI6a5}<_F+51>~-Xx z`6!+?(0iI?HZZPLZJ1DvFDLj9qd&DC4HO8dzNpw}0?(OEq!}hS*rgCp0TmjfPC}m$ zn~{-GfcEuB_m}%<7&$H4sy~6;JjI z>=(3;5d+#lGn@%(V}O3iG^9^tEyAR7r!@{euMeUX=Cvn;5{rg=guUZQ$OPAdoCREPn*LI!PZ#&q{F=X}P}Fe1&@Mvs zb_gXhj3g{!zRV;tYMt;K>&=_Ia3J=wf0GngN+44yBPh;qn3V@P83Dis6g!wAYgy~z zWyUwQuYC4%7W%MCJPf-)&V7&v1rb8v@^}LilLVsX3iAswBc>h~c8WP0g!gO2AR(dj zg6aG0%?LfC{QvcV07OqmpEX|T`hhtIfGIL7!Un3j@~%kG+09uRAd}ujbzLQT1jTm@ z+w7uj(8qfe>OuSV?2sS}%ZNei!r2Mg_R>M;KEx-J|- zJwS=6Jj0)Nuc*}_`H^m{eARJJH~5qnoZQJWPR85e!X>ifJ-T}8VTgP_IL+`44i-}U z{^J;6Zgj3doDMqN*+#oWNvuymT2_`yd>(>uh^o68rgW%!{39;oY^o)Sh&-Dy>PEvz zwKfu7U_L+wD}bjvIxwMY@=|(JioE9+6l6W1L~oE@zTA{3{C(du^CTyosbJ&&#DZDn z5U9Yay@qi83F8)+nQd?-l?`G;Hd?uz3rO74S_G3KYcQz0G5pwF^IfYEM}9N7I))IP z3q;%8L7eLV9sLu$y!pP93nN(a*zCoxnib5jk$%ZWlL~IlrO;tMp&xKY<1p&uWP?wtB4kqyEYRpfN&3vLy{M%sA?=B@MuUZ-)ezi3 z)VLGsP((wKdO0+7rhY#fDG~^wc|hrA-hgJA6a!413)s%qJ|xA$i0mlHhK?F78qxY> zNTY!XbML!jcI4v}7{JO$he39P3rd84Jq$>LM9XnCiB1=?!c`?Y06HT=O2|IK;${MW zsW9T+S}*7a@msvBNe{YsmS+dGRmt1(2*U|1z(a7)%$aS!~Yhcf-^% z)=XKr3dLF){1h<)`gV%4gIl}uB7{`fJvPFWPX*317m;iu$}M}i=vdiq9$CUY9MVNegViuIMauw<=>y#0Kz8>K09jQ-A87s1F z5ca$8i}Z!l!;Rxl6&d0dmDF#rA@s+w+(0*UdmYf`S-rV?gr;LYn$(aD03OqrhHRJj z@@T8_Xe_%^RG-PpY%XN`4YUni7Bv|c%u*E55WFjU!A(zvYwvF?Cj7~t5ukQvh|lY$ zpm&S$>F4oZgM$m%t5`{^!6-rGoTv|;m(qZ(R!|CYyuL!&s(d=`FD|4aWKIYJ4wdM} z&Y!fTg63z!PINYR4M|^ln)qF)xwP23cYP(dsfig91G^l+Vl12SphLTT+1FP;V zwY7CQuWbZUSZ19Y49Q&7@w}hqnD|G5O~DlB$Wy`6<;ZokRy___cQoL-7V%=9diLlM z!{u%22CFPXhgpedRhkd*c$nzuOw&=HwQp#dwfoO`mR*><(M;3WcWPR!BzEA1(6G=E zEE(XntFb-kO7~YHvG;a6Be*`_E=7mj(#-AHW77rGvG&H{*^X+lNEbNda z6}xfL2GMJ1cCjlp1pR9+XBp;az~yZ~bRy*7-f`;x%N7VvXC7-^31H{u zlZ6OgR&Y_5nm6w>2RNqIm4$1;gWB6WZo+y{I4WMgwU0&VyXPLuAu$|{#0b`D8&Fm* zfBZOo9bGQwmB#-k;FY8)Bb`qcd~yq>_}^5p^o17nHh z10kAx4(1J?ASL^dz8G)IgLiPiwGk@tcnm>S1M*%BVD_B}mKhPGutfNU$vAghd0Hch z1c#gN1-8$(wl>Bo4fP!^r=7hSbP{v|RD0fzC(jB{;eaR>O~F(h)-{A80eJM{NQjDB z0{I{-RL@*ka_(%AOn+9R|ozU3U`m(%7k^9Q1> zfws=xA!{y5cMGd7|IPMcO_0OG87N({dGiR5si2jf(u!fFsi68mt-V+83$R+y@dm&r zwKjT!6=kKQ7G55qC4(zXH4+hLe;Nz!36GWIeoD!L zc%mSn_}Lzi*Rz(UfjC>9Z2(>qDEW8h_2;bfo|$y~_;K+~)v1W(9J0DXF*{|=t=W8Q88mm}zXjohgTrejz93zm7?NZUh!#hI;x5}erITP3*r8`2uY97KJ{G-X7NB_RjyJ~Lt{twvsdB%D8E)CjbQ9wv?U%8b_l%FPQL zS40wKrn7_JmEOe!^72L>>!K$EjtEKb=FeY5XSN0dPX_3zCXqa7nXAv>(@V=LtEylJ zZ^|GHP}?JZC@2%cLyNWhdh7a4!F=~E`bbTg3D8Um0))FW{Q-uJ%fW`D*n_Qz!Qw^w z+qNZ=Eg7hAXE^lqfYpV9UCLd=L;3srPDe6&Mcrk%5^y&_usK)bKDdzljd^)g73}zf zZSx(4Rb=Wi=5h*#aLz2r=VKY4gWjAF<@kf2N@p+HL0=huSz% zq$?9#v`163q@@$9T=i)zP09T_O~!0?82K*?{gl3Z5D0U|BLV(uBudA^c5tB?`_|+O zo{WXu13~$n$Ke7oMStvw;RIr%B%o?GZy?E4;5U!$Cf!9Q1vj=@y!FxokE#d|VtdO3z;k6qGRJd;TaWT7MogEl`sCR&_{Xo(OCr!fFc1nj z1&gPX2}nZME#FHXX7rx0){QB+EM=(R3M_GEJ;=rk%^Ky6{HZZWcLp?HL@bEdO>Zga z7V0T7ywg#{bRM(z&2Q~rg1mRy987({0=^bT9f~y>dt)m+D?mN7bO368bIA(}O^1X}nIsTwCvuYbplHd# zUA%B)e<5f&unuF-^CMhNOh7uK!*|Xe{7UB?f0SG= z9EJ+O($qE;l^x&L!o<}2=&wg&Ooi zJY35JH+nXNgZbkEV=3pt8HXOUqg{kM^g_Dx8O4vNoxR;Vr1Z)xK}AkeI|k3j!wNrCnwOi7DKEgBMq}- zU)Lg+XrQLYNhK&2ZUl^93cO#XoBQ*=%(-+L)B8xOyC7&k%gIKr4^mJE@p#{}>K~5| z{_kW}-||=KD%km~F4(>x9;=vl%;TF7_WX6MZeS*tRi81crq0~(zn06x|M?#gAl#wd zCLmVRn~0B=dRJz~bZ0Pl%=tFV1V+Zg9Y$y)4OUsjP{nQ<1i2KZ;XM71JVg`oSc(s; z@C9jlLz>x|1iAt5&PaUtdiYusrr)GMz-{0YqiwN{1M{G5?E@|klnUFfc8*K>$dmM?NSEX^Lyn(yE#4e&P`ByZ0JAQoAh3dhz(LrU zYH2CbsoguDOY`liC%8;8N*9M^%=2f*^$5|g`S~Vl4+JZhhxoj@K91FAhmuz3J=!|b zKcF(gORbGTU~)kuL{$@^U@41;2dsuxYf}i0c5it0Hx(#24+9vNj9=mVGlC1C|0K;keGtwXf95sv?q;H{nUI0z?Rd zL%%>bK!j9^P^CxRSx^$D4+vuA0M`V{vC)L)An>5R9N2k`p${=l?Dm#v_0zJpPEnXK zjAcBTd0^bF{)6)-xVdvN$cs00#(MZm-jDaozP1LcSKMYpOlz8&ns&pg-IBF^ z4QOBhodg*e@ydW}*EkRn*LWd7U}GjG4MD-ELOy{iQxzz`Yo`k!XHtvdA)Y)gRYwdz z+HLvok+|A>%evegy2SXRHI+8j*6o>jy;|U(FCIV=4*>1|enT=x=54e_5Gtx$nKYHvZ^PRcsswEJ2NFHNJyK$&O;lV+_T8hg%Sn%1Pv z1A{tCXBr*MUoT6&tE-043}gZ=jIv^MQ{SlQ?@$@Tsy%3!+p~54({n9P?si2 z$~6EiIywOTgDY!Dgf-~f_{1qAm)5p_sWBwIEtmsM#9^bmOf@L6hdE$`e1J45IlH-x z2I9_eyrg#hi&wm|_}A-7H39;O?%)oJQ@cQ!X*xpTBQ zP<}NIz3sc*TL@N%U0{9PG)N*a39<&PbN3C)See2I?GyxTv-yPnN}4OI#mU#}cly5d zzqzkk(Z@ev-O}4C64{kN!qCuZt{>)6BYy1T`On|p1%F>1adB0|#o_;`MJ~&;FhYO2 zal?j-HD8k5A?k|*BTR7dzO9kV3Nvi&d*IEfTjUeLUklGJncu(6gT${>6m4Y=p&N!@Xf@!dcwpY$vcG zaoclqDbIa*ogL6=fv+`|2fYD7kP)0=7Z&F(D8YOX^pV~T=|Vt!9~F3*frGQS7^U{3 z_K+c3Kp*FX0$^O!(plIJWG)sxLOg-Pp<}UYZ^(et=_dG{!&zw?rX~CfGW8#W zAdpvv#?>*mHNMJzVwaiTVPTf$)8H_|yAe}hkT7;$NU@`~t z9GN)LTxN@|E}uBcU$tdG>vi6rg0g+ngHAzbM(s=MkgoH z;j-L>!~$R#s6;Y~Xf_+(VN?e$|Mqw;p5KPy!_}@jpss zqtU-UV6l?aeu@S_I!mG_yK_tLPJE@$M%ld*qeiG;Egv$)w(Pul+7=d3gBjr2hlYlA zUi0VA$FdugM?p)G|G)t!Jsq@u7dT7Z8hb?pMvj^_!zCx!L!Eh078PX6@enDvpv4yD)>bZ$hHh@X09O*<%2vC+|w zm=voJ1t^H(BTb6QPzMEpg0dVse+pJ)Kl&P0netpA`=5V#t|SpF`RKq-a(a2^;bn9R z;C}ZtuybS3Jq!y8hNG@VJ_lqenif=EmefT%5wuJQL%O0O5I=EkJ4T;2^mE?f$G*8~ z3`_7nbQ)6Jd#JnrdwHGobNTNBOGQY}X6;UpnyjPW`ty|PWF6a=2H@_jO8Jyf%Cn9@lnXHDt#g$&Il?jf98HN&1Z9a~F zrMmoYR*+;$QCWNFm8YR9q;Xn2qcIvo5i6D99tMesOH?Q0`4M&PH^w@s`vm0-sF$~! zfgOU0$zbdZ1L zg%X=s>T6)Wj|T$R>hjgAiCB2Hz5|fblu;wdBh$=;lHhSUawY*@7S|81jt%hrU(-vp}gpW4+;!`eO1ul{|D7t|C6i$dK!)4Nbx za8}>8#4$k_SIHRf(AS3eXfkK6}kR6F=vPew8Qn4Fn8~Y_}@?J5m3yx$*q# z^roDhw^QlSqPl7&=dD}L7DepNTrFvLYRhZ>C$M0TBOnBqc%tCzhFQhe_l4j9H2H2`amlv-25%<>AB;;M?NODKe6oEb9qr<^SdK>i zs=b(n>xxc_OM#t}4jM9S?Oe_m&EwH^0KUXcC{n?{O-#X0)BM@l!p5+0oH))y{ugi? zo$K?5!Ea>go?@=52Hd*`6>dvHgzMkvp$#WyEpR5Km;15)kJ!{8N+>y`wt!T!nx>Fw zVs_FLlAJz~1@7$q+cIisp*F?fFDCwdfz++H+{p7caO3}E=sj)KI58(<(8$WKH;OC_>ai-UFIqvRh8jeznp2YFb4A{oM_)~t; zokh!aP9;}$yB3a@8ph_7iD{#RM&sVT*w%S{lN^lfdE-tJZfY=)pjfmUJn;|cHz zWqTJreX0lIJBY~BHIa#kJ_iOW=u(WTgtvUw8z;EqfSUCJM6g7ppfw$^tc@nf3M{&E zKxD6+2=h_;`1SBk>wnbLP$~14A<7D=IK@I8NBb^L+<)8M4Y0EPkGO^0;%7xUl}A4c09V@saGP3o`>CfPdP4`-7u>{%p4(h&$k4 zptQ-KSnheJ#jG4RZLIhw7reTl#kN5Zr0th|={-(p@p-;+6+C$yFZbwM`>n8e(1Dc`xep+x zd8*?wmTqHuf#%pSvHr?}?X1#Cj3ydS77esegjF&11)9-8ZWQl;XOS%JF9$S*hpU2Z zZDT{gHbF?t4Tda81M@Rtq@$Y8%$>?li$Pm#Fonc;K!Kq{WGfw`b2D0g0bxK)2y?nMT3pC)K0yn<)x9 z%F_8cZNnDC^$1b5p3!UfnT|c~j?sU;KmIc1?90!^zn{UWXQoj3JMgkJKnkVu9r4$& zXEuGd{EYZ3XEsQUKCVA$6xYNxPxAcgEl0g|Uzr6s>ee>-yfX+M5%c1zWiD=}VD$?1 zktrY?V18P2@wB2Ib1sA1A{|tlXnHZg``Rq3GhcMw9fX+%bba2cb zq1m$w27Fl;km0k=dz2OMU`fE!dven3nFxy*;l$%3!md%TO{!Eq`aO}TNQgqTZ->c@ z;Y$GY!~@zB0i*zySN?fCtxaU@N9%Nuio21#*aU#loum*v_#iqUSi(0-6Sz)N3cxV} z!pVLXIvysNIs$1Uq7JpMu1WYrx-Hlhu-izu8pYfMIXjr?k>v-1tl@&`sO#~A85=3Y zSJf89o2Y!zQE_OEnzHZ(j)b^4BaX+~lAEf*Fd?)?SDghCswOd3Noa%4pAWURFb2V~ z4udyTa`f?+x)WJ~Mr*cM&AR@1_X<@4Mwpjck-$;w-(;`S<8~D%oNr3Sj)ipc?nJ#K zGcupQIeYrLkJ?rkOz{7-7E`ws5N--;G6=ZbA`&-x)2-Pr5^MdwRDS(t{MV`s@bX%H z@5*W%vN$UA8P}d>)bLVlx;RIGnC{TfL_(++MoS*xoEBi~-oMBk2NK=7A|mU3rj*Q2 zgz6LZ05*KyX4FMYkUeo2@#@bi#|&ChKuC%xVYNb3{8mrIBRS#W`>(NBYvT5~6SfJ% zSO+LL5#x)5-n5e1JS91q@xfm|^T-;KG!19^f~f-#JM{%rkh0IgBo75S3FmmKhSg6C zgxUqnhREbNC}%^UuT*gQq|JyYTMvM|ho**+=_w26a_-Xn6*9d<-^Cv=X<-(w7#mk_ z&qHBHkqO;Pu<|ST8ZgKO1D);%IOUq|bN%2M zerI)ugrmVDoHewx|MlfFG`WSEMqsI!&HaZ`5b4!RysnWnQkN~%`bp@G{H~K&x%T>a z+_k9aXlvM;JPH+I3DN>E;%q_XHc-_7)BDv-Uqmwvx8QEl3lu1vK7_^q?KTO(2-d?8lb!fGq5!IAQRP%A3;iyR?Kv zjf*LQzR%QJgqZLSNy}Jy?yys z8IMLsEu*xyX!GTS#m`m-vVy|a#6(${ZgKmb$5gvW$WZRTc>LrV0vvhxYjs~`wI1kz z;W)+wO=?BJnlbBmGC=THASWRqVZ9C0>2Kd0iT9yBCv7s#uo_DY!63(mT|f$3>&~s2 zLnb^V`X!%GY*O341;0*U1BMKb1_-v6D{xz|MSK;3+DNRiDcA}_OK~wT6jXIv#@&Y4 z>0uxXF)YC%m=##msMZRQaGxBppg2w_aAHj`M;43$CF&D7kfadj0o@7p;srE#yLcV^ zKEg$S(r0mmv4jXrb1U>ai6<7vqBBJ5)5Zp8&tBXYLJFZ@vVON)mFVPmkKBSdw{*KZ zh#uts3h{t|1kVJyA*E#B@i`gEH$~>M6wZ_d-B>lzM0TW&7T)#1t2FiY5N@JckwcRB z^<5U}DlRz&Q=BK3JG73Sj2p?{mEIO_4#_xalKVJko^E(I_o2V6VnLjGh;RDaoqot) z6LxKB+#QwW-q)Z{n?HbffL2w!PIHP4G>L@qGItCj0ZL+MQ{x!HKPvnW)zk=t>!TwI zU~~P*DG6O&s-&vjSey!^3$oUOC&mjDo7cKAJg>y;T@i~@5c~_ZsW}TIqWP~oW=5O_ z3~`Amf`?fr2lFpY+%BS}N}uzbi_ZsNY+wdIsEq~B5BYGYHdc;6Wjx`4{Ef|tQcRf( z^%ax3j3oeN_4OQ~TLwZ-HfI^2T7nF469{<|5PzUbXA7Jb;#B@c1olc3`Ox)i^i=h>{&2d9lp%T^9Y3(y@U5pSNo zChB&vt>eFs6lX2h0jlC0z3^b95NkoydZA1>XZ5`^4*;Pap@E8IuZvO!;3?7kO#Fx} z>$X0ihcj8S25`dwo`?%J=+v;?-@*vv{TBeMn`{86)8xEB&(aE1u6B&{xost?$QRky zT+pqd@Tx-RstD6u#?%U>97}h|?6eqz*ir)i0IQ%{T+;z5)cm}<6gYi7!CEDTf*HPK zI>!Ad)$wm8=7SS|FbFw;o#xn8dEm}AS~1m51heGi1Smym(yVz}1RP)=Adv#5P%jm@K@;E%e0si&Owt+O-h4QcfCWbk z<2jc$Xy&05cEK9Xj~y;xE=<@z^vSzibz?oAyC^(&5Ty^rSTg;FRv!<-s_oarJmosI zyClihj>4*-QCzRg=aZr`m zT7QezWhr^kkYecYH=@j&hTS(I52*cA0%r#U^tdD(B%?((&?=yRasxIf`F8=-BB;c> zVLOoJ0zP9Ygdj)?;!vCu7!QwdKq(jlkl_G9DIoeX)qnnR5s}Z6L3#n-xZz2vH0W$P z2(vW6ZzA_VAfN-Lev`(DF+&w$O>oZ;!LcUD$+ft#jYE1la4N-P%B=_#^Clq@A{+`C zh?@al;=&T!3QI}gvcpo_b!N>RlH-xFA4;X*=g?1MC1V;?G9VLPr&~uVQ1FSPi35=_ zBYU$3tR%>j6HWV@Z=s_z@6(tQVlq%f7e~X_Q_;3sf$#5y4=mTQzTVHUrRv|9s(%Hw zo;zj&i>Y|@OEpf;9(>LK%^guzBrsv%rD}p6y!eE^3+J)3?d`2mZA$;1eLaUq=8_Bt zT872jdiqM-5A5&**iwGeUMNajSuyTFD)6mY>OmDtjN+P2>PFOlbhN_U-cecbkkvDC z(Dn1HjKv+xECPk`+9Qe0q2TeR56YgF;e@X70{RXH#@=~+##v-EM6xH0k#@PDeWlxr zr%iC#b$Wv`zBKOr+3SXJZ}<@*Tq=yC1a3_we@1uAxo^8s4o|v@rw%pZ&Z_TW5S=66 z)VC?Iy8ddu+qQ`hl&g3Hu!eBsg*M_YmZqM#zhj1q(*`V;b~%bUw)dQ)P;?8pjF6zGi|IcU03CQv-3gb8lpt_70# z4nhC2xn4A-htFogW>%>OD!QkDD3Ls7=;AO7iPd6@Q<~I zGc72r@hKo*(4uDis6~i-?~0%$N4&d^f~LC~-;WXw0MK z)d<5o1AHazqWtLSaQ(OyP9>r1kO05fu8ML8X2yVO?F1ywf!rR0!@Wce z<%?QL^fN+C$VVHI@hpd)3mWkd1%esz1Yu}(Mi50j!qjRjR`?Gt>=2mI3bzp@Cow5d z<`)mJ*o#p-2Z8)>DUhcIThkxy-Op+r3r8ce6u^){rvswN4Ty`V_i>=df#Or0b2O6} zKa4w4Y^HyGYJGuw=(Y0U?N5Dz|6|HG^~Fc(e;K(C4Z0oYP-LzgGf zMTlsL=)KDYl0L#%6s^icQHO!L_#j&(rDy}{uKNWm^z%z5-n2DMv2b4jB?;T+{@EKT z1eXuIjHerLfBl^`>*>d)@G9VW^Vg@?9CA?Wo1mosHs;6b43rz~PhW72k7jHMNLVsY z!t97CfAKR7KMR#n9tMRz3XA0QXR^PTJbwr436y~A>YTnnVgOMg6-`)TTk2eP%7c6Yt^rkFTzbLjbrgE=CPI%}32 zLMF2xK6tV1wI-ZnQfUAi**&1rXb49?jlrdzrYq2VTDSer8rFm`lAQi*$x|imMhwBk z5!#Pw0O!#K)1gROuImn&TpD&6fy}0;oB~LYavkYoY2Pw){@)eSB5#}o&}wfMDCVFo zWTP5N87&|_y|*pZ)&zZV73M=9+kSuoix!&xk?i0$j0}(J`r08jK)~qJt6DpG&(ifQ z%Pl*8{^ZL2cOOc9YI6lb{`+u3Ca{qoPgRmsOe69_;!ESssyK+uw>u%jfq6mS!T41P zn+!i4vk2K2oTv3wT+iGu-PiI-+ttv*V~ZjQc7;D*8_> z3{GkBo@`c76yBhujEu}sq0;y}ckbwrFYt;QHD8$RW}yaY2QB`AaH6a_8T(pCIvbDMhR0@>8)o_|34zD)j%-4ONQ^{uWV?Rd98eMHyIr#$pkQplRFcfXp`1f0-K~5`ZwjK3GOgjA*1@;=1bxj!TZZ@*4y#qS4b;mF)0AsT4y=p1AtHKz8*5<#ZcT*5+t%~j|N%D zF^P-!(d>WYHuwJu^wP2$_lOF@yuznoI44nL)6oz&Cg*7M79uXrjri-1_LPe3y;2>S zx>Hn)%!+n|I_%N(UvH{gi(=7IEGzo9qDsQo#c5QYf?VNdl52?T)DhNK?p_8ph99ix z%TwawUI3UTekv&jdwz}#H@f*t?}U~g=XOx<#7lGmv`Mq}lrq~o zU2>xMz)i78A9`C-hiY-u%E#7Z z3r0#j`sH`J*L`GI{lOn!cZ^k3RNTkU%rx~iptUCYBDB_}+Jb3&W_qtD zdo2Jzl9@F#Dyg3Y1COjKAfjX%A&*B4r-=yCPy(ys1Ou434es$oeXwY#zeIy%!#qc< z@&Wn=5Z7t?aW@}H5NyjjlRNg-c?(G2`1*oSl5$9ciY^!EL%MlK56gZj zxtbw_dsF2wNNa5XR!?Z4F-eRSEWh9~qiAohJy3#fc$0iDQi~Ny-*93FnAqdUGqScc zLI-|&B*X^2G9H|~2kit(L2h-mBObq}+I%c6=?Jdt5pTFyi*H0xh_Jw(Cr_W2;hkQy z#>pXJ{JkB`x**?OL{&G5%r+U`C7u?kNoODu!0*DaE-`Kdhf%>{W9@l(HXm*cH*?{L z0S*+6Zn0N(!f28L0oWck-5#or*fz~*5TVM|OrbSQpxr9`S23jNfsQQS!qd~0W5`lE zH1}m|$tf&T(|_18?Z5jYbAD)ZtFQuS36MV+p(lvu@7cc~;jZj8Z`sj`)5%H6?cNBeiTp`EVkI%t^lm>34-Xi*P$Bx)aMfBNS{edE7iqUJR95Sm z4hWoVEJ@_T1|Hu|E18gl8Nz3v^N(*91T>+My3G+DuxqgfHi)6DOYn z_;sO-q_tAS!*v6jjNDm|ESrhaFU^TjFJHWvpDUR!r#A7fEw}rIfVvJtLET5r zaVoKLvWD;n#n^vBl<0&iKu3YlUR%Mdi&SM#d5C6INT#pw(Rq4O&`+UPk zzj_lEdRS;+rbP6rbabG%wP7rT29YxPY`Ow5-a>$(AJ9lJpg<0^y7gF*%sBD{}w&zsoV`?d)vL>LLq zKsVP$>8H~Rpcr&z9h#v4ZXHL~dig0G?^j3k{RrPTuy+8%>u5~eTtWEP$$??}V+@U( zYwtMTd1*~VS4-Qn@^Uf{;qG_;X;Sg_HF*f2ajGEmVGe873Lx-UC#sFkuL{FYE*Bw& zT0obMCdQyrtvuWQ?Y{?`YvcasZUW^VQc%MwycCBb@{L^ADG=p|*n*>Y32^+Rbz~z% z3>UnoKw9BVR_`#L*>h=%TG0naa#oYs7{t>vyt3eB7@TyxQqB%38jYu;>#1P`qXSQD zts8szY}X?XQ{WX2dpuQ*l(Z28$=$8(p zX$k5-sO=#xgq4;g77S|6W1U_EYT^qx!Ol#7^yvLpON2#Y3o|IaSIK~`nC=Y7C3Y$_ z!GdKUBuwXNG`V258&M;Ky^f&UfB=l@d z7vW>M{MzF@9)do)!yIh{@(3tr_z(uzC*qJ zeO&Vx&J&4|(={1PgRYuK^t~s)x4Fw`YR$WrlHBFzp0uv^+}Xp9*oN`f6lcM*73!E` z`+a!Fde|WPf>XNsb%9A=lWdrZ*blpxhCcg_ZVuz~l0Qbr|iR!V^G_$K-~$gq9m&j6%^9Ps0_Slwk^Suzes_Xvcd*cA|V^xVUJ5 zpYqK9&Ngr}4X_uF>}xJ|+LZ1)ylf_+mt-M~$#(p&5fFe}8UexG)V>}`OzrB|-w{4d zn1)~o;NYD{@)xauBAE2)%fG+eRot*(C0!M*y^zDh{ID831YDCcG7NqZ#Suc7aIkyt zUh1i;(>SN@U%^ru606e`kbh+7UpzoSFf1e52AwO=oc6280iYoJKva)AtA>XkY5mzt zV=4#;Dz@jL(;;Uk#F~<5zJmgU6d*lmb*`DN>UNhh-;$8$;+t@$(7&7$jERX6+W4v#iem;@8OrvLao`79gD?nejLm_ zpr@$RB&z=U)FHjjx35kL470SJXfo!w?GaOd4q(U3<~05pQkO(OevD!c$tHx+ffqb{ zhEmP!#>4Bz)F>(U_BI*!Pg9%M{1r6J1eR}#DmrEfM^(hQ&)e4+8MNJvNKv#7_*C)v zu3pj#UCHEyEV~n8RtGcn(lr$Fo>XF74hVk<3eNtbBT(9b*m+Mzbu&c1AS^HaMUaY+Oplb$x(=)&P6}43=28o|1kc zOMaAaK(f#kPNRYfnyj-;VK`a@QqmeB_(%Rt8&o{Zhb$Q?O>Kvx2ugPHZv^Y5iHq@j zrFnn1A<&QqYzT3S`~JM~>VfB?oPC0|-R5z_IDj7Cy%^LXT$7^8UQ^{bnox~Vxx`?-?k9<^Dnv@ z`&IA6oXmEvwOxhPPaD5oQ?~<73U4{4NmA1?4A}!|z8zm;kHK&t2HMuLP)HG!#FJ{-O7k ze}kzaz`)oSjQ+3ZdQ1$LR$hbKu+s%KH)DVT<=EO8xoF!w8o(P}=Ov^vYAnm%1}T}@ z-;Pn3%}nqd88OwgX#=}K@+ad~*i=c=F^)F_3=72Xbjj&q^?N~*ge{hyv9Yn9D`XZR zMr zva1G5J2n>$3&yXc_{I86OLUI=0FJ#FM?|O$TV2s_z3T)~E#{h^rq@v{Td@R{Rc;#a=vJ)}YTz{_nSqDn~0 zx|A7t$vrlT(eGJaPyL-*jlIm4yP4Mgu->_NzFIjr*}9HfD#RYz(!G8m_Ap6uiDa?k ztprR1I-^f%ZiW(7Hh`O<8e?HZvtxg0qid*=Zxm=9f)t4cdU-t`<=`*uUa=>KWuGky zTU2t0h{veu@xUIy1&KKM=y%Qy$Z~mlmH}a}^bymjfaELC3jHW*?Zide9W?!n#`KV!(jkI69q$goMECca{fx~F3_cdx6X^C+0``8P>jv17|z3-SNkuj-hN8?&6-D$%rMYIMX5mQWrcs>}= zd8K3ht-|lIN4ordMQwAOqe@WY9z$@<2@Qod^V;G(^uxD^#Ti@j z1+YD4bF?RpmL9-;D{A|K?!9+m3QAf8vO>GA?n`cai}`u%O|UI=E6Uy!5p%eP!H(eR zvu6r})fr5NbwCy9h3nl0R9K4wM|D|IUWYE4Os>dHkY@MO&a6pu{do|p(bplGV~=|f z<1A=-n&5pP$Vh-f8+ceUhVbtQJ@!ogNWO1Sr$QQR*ta41+3#`qm{O zpKtQy3;*VOsFK0N}^-^;!BPZt<6#r;oMt4N?v|zP?wNZS1by z*mh1|=W{^b-F2hQGRqd`G~rz=vz@eGGGVk|l$&W!hMvB~u)n0NKW=oAO3yJgT3#)c zus7SJ&oyQ22FNAAy(7(d`b^iVrzeNzCUXN4^zgg--}h0H z(fRvM+|!&}p0#I+TSh&&7s+<&aJmwo!hReO953X#^b^V&zl*KhKACZ+=Bl_<#Jn$b z&2o$~uB==cb^Uf_ZqEAqCoSSOhjn!=iA$@1DCDfu$BpSy^SKbVU9YXxL$z|j$5+6v z=zyLfo6}Y@*7@tsDExB;HiFMxf3PCt(81d>b&SUaJ>B8y)^1_X|I5hXD}jS_6uJA;Pgppd0Bf^oTGllTFt^?rQ@Q+j=6_y za8%J;|095CzQES3kxYF=fo8$nO7T^mJY>09hie(&n~Z9bR}M3Ypu-~D^@ z(z|CC6^L{h=4?$)<$LrxdiwdBS|fKYkU8%E*XmUsqL(MX6wMB{?aa=}$;r;Cam>s< z6tGWLXv45olQs+we6uuKX<8Ov^P}zWWfQjgxBv9?I)8B+tLFW;-eu!UA~T%`m zN7kYuBN+@N7*W@%IkfEBq40=+9>@3^J2@_l^P{fY>7d@;*&*hlC01ExAqnOJJOcF_ z;lMaJfk1zqmzOjxEe-t>6TF3kLe{_>M4Kv?9mXCr$LKQ&Ny(`A_@JPm%i&f(zKmJ( zL&9?jf{s&_=79?27Htq;Q$?d~O#%pINbSY}GBOU99LJ6Lo*mzv}gMrgwI)PMdy1&2q(l$$-v*BPO-* z>)_&NOHRxVcFG|8Q0xaeQ37LIDr67KM=!pc>WaUQ2r%nNj{e$ynSPx5sy_XjZNqvU zUE4<*m;~o{@Si>pG2|95(qtzyye~oYsZoH&D=nF0#NI-NpZ{XwR@O;tiplIY%QcR; zgbQ07$^(f;^4;=)Axp`QX_}8-u4Ff##wr}By5Hhry@e56B;6`L*`0G1EpW(q_`?&? z^ldE4Wv4PYTOht zxx-3nhrir!cxZ9f?XSFySHKtCR-xjs?tb%V`BbQA=9gGJvfp@aO2)XTbj(`ttQVgK zDRQ;uO2z=s`CM{5GO^@xsQYJ62^p=6fz~Sf)4BR0u?L$+OX03cWv6lC;&0Yupxu&O zKebI*`%{wr)#+rS;|g6NO_$_wf{_}j5WQvtOKd1H(gTCJo)|ZU;gA_A8g}u%8b|%6 z6T;_MAQzX^nM6)ivrZN|L*I+gaXcjngVIWP zP;}gRV|=;f<;pDi#XHAkx$Qjtq^6yPhsD}SoxB>XH{yjW>CcZkrTqM>r~Ad&Zmm$$ zi0dszmJ}A!HJ+%m4Hwp5NeVqQmVe6$KG)2~r$@8oBT9cm$KTt&dxz`)^8f}SyP(p* zQ4#+Y#V>=;f-_x`g9GvG{OcUw4By*oa}8ikcs0RPwj3 znvSJg5Jlr5Cx^7c7jk|E&?~myHRq-30kc+=)7W=NcH=yOinjx95{x%*Vgi$ZmO3)1 z=4Of3+tE?2z#3?zX|ap+Go!*ydh+tlmLJsXX(P3Cym8rgfA8!xLxvU^AODN@TEKVn zHo&cmAzK1#RME@K#Be#W=crvtkpHRN*?(P1YD0!O+7NQwpwLi9OW?E;0dp9U7&c;g z74_Wx8>sLC4P@F?JgiG#B8Bj6W>98hOfQry&@cwaAcB!wqt^V^W1lerbV6tT6TZ~c z)%^iZn}vQ_Cv?XR_=T1J!AC?zMQMS*HE4xGy1Hbu*0^𝔙4xMjpP9z(2?u5Gn`% zbzIxJ9V}qx5kT#EU*fq&9vLv*+Q2>_JS5~w*1@k|=J}vHLWRnYOpFc(M-6g$Lsw$_ z-W!#V7r$K5t2!I9d&cki|RN{j`J20rO1m2ECAZXE5V zaL%6W|29KDy~D37tnCw_H{~BXQJV5gC|T`+*)9IlR&HIP_`n)A)lx9?W~R|g3U92h zpkDIlyHkklJ0_{S`0>hQv;U!Fefd}e{=SAuN`2^hsk4smR&&I__FQ^Wm3=x9dFk4| zHnnw?eH{^fT^Vfxwu6rDbD6wb0}Bf`Yj>t>pQ!w!K4z`&8H+=d+OF}E= zW=hfL;dlw6Oq}QDRAJk)xQLIH%x;RH`}n5tk;}@9MZKD>&r@dQb3ghfjuv}8zo}E{ zl=ltHOg6FDuD4Dl_AS>Tl#6Ve#(wyp&6t5%e&vytIpl!9P@UL2?=NW{Qt@78`hhDEBci>Wd`y&v6T%WbU)SI-f+YBqHrz4JfFQR*MyS6s1jvJQjYjP!#%Vb~q>9 z3|UcGzop+6CocNtH*>iE(h}AwvN5H3@ym?p_Y%AAckJxXY6yj$rynRN>~@op56@?b zi-rVWZG8Cj>2eLrtp54ZjgGg+R5B6dEtXE6N%>z)ngwPi+nQcmmALLk(-NYkrlw*s zNqhJcip%>J1|KfO-X`QS{mKkfnTAdY)W-VHmA)y*_7d^$PpEs#_i*eFUAam?HqvY| zXK*MLQLAd)@mwI6>*hCnUr$n+Q5gm?IUK#WdrblBmb~`*0>t}VJlxwxAF(}RNYXMy zqF@nA$;jZ+Uh?wt(yFq%4g?H)DE4Dw1PPzmL8Dc-(Tf(gos^LgqZ(T*;!$vRE$q%# z=&P)o*6T$89wPPdAx?Y~=MV%o3xE}Sgz$)d>Z~V0a`xPH*+pfMBA22|qP6HIG7Qobwk9~r&F=21Y z%E`3=U3|j{(wk8*4ulH3&D}dVC`eWr_$7j_zi$BiVpz^E${?+#MhX?|cd))8ke5;F zwk)3q4^csf#mhZr%a8g^d9DF5kqZizpmrlPk{YPC!=N;~&dJ%Iqreeo4C!A`SlE&f zFR>FFA)SaFt5v!7>wlD0z0P-ykLE+I4n9jUw(Rx?GTpjGIXx{mS)bRsIVB|MT!_&b zPBx$9Oi<}I1#84$IG^;_e%PTlKHYlgwYx3EbelLOT7HQFx3{O9E0uNgFCnbQV=$v+ z{cIbPHp&{hWH}M{ANmu=#!4f6)YzoWvje-A3GrUqIt_n{iCy6~{px(+fgmE94Zy@r zCn8(&J#oVYYFWe`ozJNQ32;>D0fqMTs(6GfMqB%_@nALrk}7;#Tgxt=8xp6kUh3E~X9Zc89FH)C;rj4yaRZ4{?S! zSzAxiNf@6IbJq#fxz#>kyxRKF_P%;BfwQd!17scS1(Y!_Cg}TSCx7ZhoQg8yQBwPT z6ZEcfy=9o6C*vi3>jlkizs%+!Fog3T&oKuoeQVwLw3Su*4UyjZPf8jE%sNjmkqrRhcQOd;HTM!_{EUjk51)nzcxAJ|>7ft0En5;DicKS9J#YVf0NfO%z z;x0!yI_2jd9v|AT@5*aY;yhxk}(Y?SaEk$R#C%6WP2JkIX4Oglcgwi9*8Hm?`L2l ze2N9V_vooFJL~i9@i9*d>IK!6A;GIO8Z+_yCYku-1l(%5gGIE7FHJkm%C238*{(u6>joY!eM^YyJ)n3jbvfkVp>i2#9nw5?u*;C2fA+xx{1>0d$Mp51 z``#9LyOM5s@KM(@)+KKex?^>YJZlc#*W56Ty8KHxmk{;04w|++8WAv98=jC=;%pDT z`cUa5yGKk!1j#}NCZpIi{rS0Nw%U`JIBNTydC~Pzdu*)M*0Y6`0p}8fW<{ENdn9c} zo&C-{38P*j?01GT4z`ennNLER?uT_RZvZuBYx&dZ$w&@h(S2jxMkSY3vjU8MFA^Oc zWh?9IhSCae&i=>@oM`wy>Vt{cR-|Tc2d%|u(8gtPB$E-J`X|%+1 zb|Is!-rdc5>U|h4#U>h}$L#jII}?E_8R)RRPHg|=K>tSP5#5XQ|4G!|3R) zcsK5Z2I<~?YCZZawY4DEt>^O4{i&X?AA?@#gFH6^-39xqf*#&?sKQ1c zOj-^&DP$87p#kb4Gq4HYKtvW)xM5wQWd||}=m0>bxC4?C3~Vu$^sM={wb0pF15$GG z@ipK95R;MJG+_JlxzfW~qGZ2iDbYjV9bE$=$G1lBHLPz;tMX9Qm4H-zeY&0+`a(|T zdD+=bK#q!xi2*xQKSjPBbV$Hdls-Pmq46RdUfb{8V`faRV1VKgTphEAENMhszNNv? zuS;0IXn;=z%6C|${t3`-=#VLbg@7S+b*j8C8tTOa1So)R&;fB=h+B|Y1s6hYR7y(3 zhYvLFQvgoc!%(CfH*Q>xe+#d7%LS4J6CWQL(7pnD%^b&MZjcy)Zj(~r1 z(E{KVtxr~81@w+%0;q?fkma>>cjp=VmP3NHHd^h-@>V)SRmZ@90f5rEg$1VjJdEj| zu7yd15?5%@0)q69$lEMR?F?cf9R=&u)-FU^Vpd-R3jZ~u`Id1B(6ADf8BBRXF=H^ zwTKX@FH0ag;@s*dvs$dkWMG)5E5oP!muJqRw;%hGQF}Yl3>{JzUC_pmzNI*~=M_1d z(&fm=NGo6Q1kJLQkG#V`UjGUuQ?27hrIQAJNr z+JPDB>D*Hpv0k5EOUe1X?=G83X^~%PJB!{Qbr9vdBB8*|M<7u7G4giqx7#gwUm9(Y z^!@Y52*W4Ul$_`M#FC%n%pI%sfU~$uXNho__ke!K69>9TKY#&s}l6K9_OX@M7jx=U94(m(1c&8t23W&UFfx>ob#sPpUeFlPf^)nvM*C0391tO z&*db6dGGGg))G%mOMrE?oz-?5-s5<-^Fz z$tdlI5J5oF?gF$c$c;!@Nv~vyLC}IY(+T7kKj{1M3CYqCt2lysi{>kgxA?a5vc>QE zpU**}Y*)PC^AEHI&~ILMdj$|2Kw9y4pA#%PbZ*%cXxxS-;2Y>eWPmqWsvoLi$3R$? zkEdC4QDVqRNDc13r4=TJw#94NMZxbSotB_mcDxeiel7&lebLW))Hi(SMJ*ssK(5^gteLtwO?tRe;rArLCFUIBvGlPEw4ZNoQlahjdj))Jncq7}9?;gCr; zoAcf)0|gtHMH_+VISM6Yy-ut$0D}ieQEU z3%x@Fv-F!wgB-DsxwiC47-9t!zsH>{@gbVhFXUPvXH)D(G%Z#deXrGD*x_*W-n{zU z%68{2(?$e7-+q;A`V!KBo^(*qIxnJd!M<@)h%mTFs`Xi(<>v@3_Llg#bgs2HrVXd+ zxUGyEdFfwF$6H@Va7-NPFcKFZ^YPV zCfsGZ9xc=;)3Cgp)$#5QC&TsY-}DaW0xgvGD$4@(Pv@)7bRV#h86R$%++ePm_HI0P z!xK9)NUzsbF>Lcbm09z;U8!wznc!7t6ccBft^UYz=*0nGp28Io^>v1=_;F{b&GPS+ zZ886@mgwJj0QV{~CF9nDPD&ABV?)_C^FQSoEwu~ALh#3i;6cK?&<`d-ygR+nb@UGaK1!B*9Wp47P*XAgRI@(beBy zsnqfS6%bmKb7-(%jI8ZL1YulFFfumYfvJp=Y)&8EW)&9`0gGA^9MbQ9`ST64GhCr3 zeClBb75QXU^Ox^ILE7ox%_Bd2SXX{tu3Z9(BuYH2*H2JiT*;txEHW-G5W>#_ApSBi zv!vS#N^uls1|uGgQIMXPm>BJcm}Ho4N{$**GzU3YaW$oq8JJeOfXy@)5UPWu7*VyY zuI^h<-!j1*=A7PvfvZ5oXa-!LA&OV@izsnaBw&2BUxAsSIFs^^-RIA2cJoqE^{2x+ zl0+2CGwVFSPV^;aa;vP`1ZlJw@zd!SJB7#(^os_Fe6|rq1*cZHjGn=YYR^o4?H%*J=*}5vAovalus5 zzCP{AE2}N`(VkYP)S&#b^HvH#=&NyIhbFE^Io!)B9rxK^OgR2n_7=foBYaruDShLx zQoQiiP)1Ezq57JJYZq+=n&xS1Jx#?_OP(nQl5bGQ&h?PiUC1PeKA?AVV!g&V>mh@Y zOE)s}lf14NMsoe+d+Wn(RVD`$`oNm(HU>OdyN3QvcL=0AcNsU9<`bdE{j$(qzIEQM znWbw=PjSLAD`A>2Q~ZYvBN@}daS$_7?2DJlVjbExfi;ag&)Q4m`h;!{;BRJ@Y4su| z+JcNS7=nA;Pie#@&*v$az7?f7=cQz1o}Ffx@Zg*;h?A@@m6bi2Y@De%G;)U)4jqSX z)pO3qvsX0IA_pfTn}OnQ9R)RyR`aOAHOzKs_iV&FEU?mf*O;hbUt>Y6u1YI#etuk| zAo1S~hq|Y;JjuoWuXCAkRry{i+01m|1Iu8#zE%Sw6{|7C$9KPWHa4h!^(3vC^(8Y7 zYX)01E6n6SmYRZb!2)fTiH+Rx$w}G;9jNXUxWQlpo5W!(D?9r#oWSist6t+BB_vi+ zCB1@~o{>Sj3Wdxwf(fL=)^M3<2eJJp?^As0Y#==)^FyTdnN=d_}j& zrl$KnNx~yJVt{soSw0y9?5z>VI$i_KTX+z(L#X$y6Mqd)AOFy6Ci&_WKQLxOfdz*S z5;G1Y4BmnAWfu6qta1!C5|m#E6q#Bz`7$X}r5H$$34ZGYnTsO|5%k`wix$ z*_9mgQoMbQ&aE~^nf(%Rj+%qi-sI({M;JA&s5SV4El&4ViFjYX%K>tO=iuik{UJSd zkRGvKf%c1E>kWUl+NPqP+aqQ{#_#eg6pZWtLEY2-1u0AcV|KL!*7}b-uedPbe)GEI%;h?La&BC45Fbk z;B;PWxU;6losS=ee5yGyxqvjxQG2`8ja{kpsjZEHl{5_54dr_Igk$;Q_XP$--PtSM z&~AuxfnMe1`+{C|#qCSHRM&p?+aA}8xV@wg{2fN!my+-&Df_b~0N1oIO1t#m#|&|# zj%oy-!0o7_ySTLe*%MtO2_-+xZaP@4`iK8uYDeaB@pUQuqLdnZ@CmE>`gG+k-ssTP zqobo)(EGK4oHB{I_k0@G=nhzI&lqI@OfhI-xt}Pj}uJSs&hq9iq9xYD$V&$pMMh3X(HVnZs{2Fu1 z&T)Z~@|i!t(lA+J2k0wvP=qUP0R9W)8Lbi{OlWmu$c~l)1Ppelbij&xJ!I)*4%0zU zc{Eh$X=-f1agrKz19X3rNEDdgs0epNI@Z}$G38=v20#qVx=ue&Xd4~I2Uk}QnmdeM1>tqJg#Y_?d1Ur?NN zVpUj96>J;|79j+xtaRuO3u{-;z#HtMlRMskL9zRLTFnl|0IX9A)=11y}jm@ zd*Qneq7yDO1cZleK4=BPafy7l)4%NVnvNFhMSi()=KvRPP%tJh_ie#fR22Qci|~;@ z0cz>|dkscEwc-E3{(WK3Q;}6-kZup3-@`o%%h)|9k}@?w_6EtlClNe7TXVfp*rBT! ziV{*A7M78TXJ?AETgc!jaJ_pC?vfx{YDt>{2p!afT|GTg5)$ZOFN~ic&#bJ?xVjFj zV0K|)p_mi2Zj#E%YuLa4R#>Y9Z+v}Wg1iMM3?w!L{Si7dqoocQPH9y;+ypXa7}&Gi zD+7JuJSxX|@`MD8zpzWHme8zG%k$S*wpw^&_1b>Y^A;Sa7dvKV=z&q(31*o~TwmTn z_W*`lz6J{`23Y$Ee1CSdiZd$h19m>ZXcKEM@6qzVm<%%Qe)oVQ4woW4C_DOUXuGVU zVh1n+X&ZLM>}C=0386(TOpTB9kLV_eA*36l>lG|Hk-dpHV|cHy$ozF{|KcuWGUMS`-?kn9SZWlt-wnohxHw<5^l%npSZhoV;K` zs(Y)oy!Dv1;b}NFCLh=EaE6|3K=1G#PY1I@nEYl$x3KL*oamlCm7<3|CMpFJVni59 zx;KkYmZ}?{O*KBl$;;%HDv~W?IWA58o~x|>+s?Xn71{F4S?9fusdv6zbTem9X>UsF zvx%CLdQ!_y?#xwu`~q#ofz^EhMaPlBn(p6JhmQfS5IG$z(60RY46R>pmF2wAdc%rI zQ%Lc9)_1lPmxSSlWZEw*U9@TT=Uqyin23)X4ZhzwrK{HyE0E!5B|n~pzrOlLXP*Wr z7hNKep>J-Rvi9b@X&A?dp2oA191xEBbZ~uR?z>GNm|qzM@77213qGgimBkH~!U)Yw zNSZUw*Psrkz1>z}GX_9(Ym%^fb`Cd;fH?NFRFBzARt0}=(SH1YC5k)O)BMpW%}`;!y6XYi z^+^r3O`ejM^e#IFwzP-&4tfO~=P;0?A#vCI!eMV2Xctq@d+InxiNPIumi}zeG0O9Zt#&IX!%1Y zk`k3uRwjbR*T&K(tV)*EuQ75r!7U^ZqyvWFGMQauOLH_}RyqfcAytmcsIWY@GIrp* zfI?qXn99fPx6eAgw{F4#0#Vu@b_;k*;a9mI7S=Za+yI)UCNLr)rlq~}FA?c70nPA1^P1s3Sqt8S{MNxbA=*UGj^TV68JnqC^LHl_O#S}5XmR1J~pX6+V z%=J1yf5pHQi!Dz=vF(f)>~i7q{~#Oynn%jsprsEF`gIG4OdvfL9y~rdxe^ngo0H2k zPVeqsd0tCiqJ^QPgb7>wM7#a>?~V$mwL-(G8Y`vPlFvUz9kO0Md1~RnibUom{#xnj zFWUFqce{IjoT4<}N;vTC+pGt{zz3}8rQj3sVE3`u(SyLdUHz<{^QMVk$#~l62fTRk zkmqh+^faN{4R~&q0~a9zuu`eTE(D${2cF8po9j{4=XW0JZya6hLYF#OaQP6AZ~u<7 zCCf=}N2BCIhIQPoTF(5f{_8ds8cxxo8z;|P_AVqAc1>o`YF%%uc;>b}>ubBZb1`-5fM)JS0VEPI%+*be$sINf80^_`Yh>r&gMprfr)S)joSu z&dEngw%N9go)ZgnQ$Hu)3zFA_aK=5Jol7&{w+G?8#OZcmpxv39Yj+a8G4HF%wVf)N z2!AC98Vn!3`bp&{PnOG)Ke5It9g51ccv}7U&T!JMyZ7!tDHiHZ77@#g%fCFBs@s(4 zALRC4n9nS~KeyxbGNDW5TyOEGz98$Xb6K(#V`E*{e76fg9TMM zrlRzp^KZ}|!aMC(=qLm3_syIa{U)QOIOrN5*x7Nx3{_<{wZ(55%E~^V7N>*--2y@! zRJsC**IAAlTj1yBZ+jo<;U{iTX>5bR*BrRaJ_LJ*`e~yQ1JJyDTiZu@gZbc~&a0qo zCOA&(z!>4^bhn?DdA|4S*A@^|pmr+icyaOq{@z14I>F(^vb8M69s#M0BiQ= z&*A1T+Nc^I#m@Wu`Ny9cAE3<3I5@-%*NDHD=H%r`dV1DNCIRP#6~EeuhcEnAhD!_g zkoKy+x~W?(j8j^p9ybwH$|&qxSJueU7JnF*n3%7AhKF_clhVwat1KZO#@mdDJv?H} zBSWsg3)$F*ZSj(iRu!lWJVZB3ey1^4tC}w9_l*h~H=*_`%xZDD#K-@{{ zcKPLi)s8c_)BG;8k(@l<`2m5$G~TSQISTB=4>odqq&QwLMtLTtbzn&LwjS;u8k0Nu z$y5qGy*i<+YB4!!lAUku>Ro4}!>{LN!O~-vr<$mJ$3H6zT`rS!UgjX@_?XY!+-0

OX->~JVEm>K-ADWAw0@3+cjTL1T{YHTQ68;@R$ z2F%eOf~A^j3eYp>z~V{@6yV(8Yj=c1;zq^<-D4soB$Seto(Ey)K@HF`+B!O}{LADp zK1L!XIpT!zKI*Ee`DJIbqNx&>)7#qGvg=gffe^Y0D7iFP^}Gr4qn}N2U(7Uw!K^vS zv{~m?)HbqFD9{j&*>@AQ=BH*cyU9orFOk-i{xGx{%xHj7GL>cn;VtGM z-8Z_t2QpL;fP%|&{Kuk~Xig|f13n43)fItEiUgqZepPih3YV0GYw)&t*>)5JK&nQ3o z9udF@#1CTPWJtwb#7id)d%gf)t7NG%OFL=lQGx>g(Lsu;;SbSMYAHqozYepnA3rC#&*bYk9rFGO!~0-OgOgd-m|xp-G?fXDj_v&(UOfJ>Du`hU#{P{ zU`A_Q-g7jMzD=7Yzq`JH6}-37w=(77Cp_~{9(84@o99|Z;--9|<*By~SNS=(h3$8+ ztSJ%ZOp*mr(Z9_L{EddNaChJ`HLa6ODVqZL!1K){G5JW|nA7G-w`fMy@6UEx2>!O{xO;k2X&JJt$u_V^UZCFL5l@wBu&6=mg#`ICu6X(kz`{Z`M?bDTq!hz2n#{7#$y*%<5} z?>PUh6aH?O^a=ny?Q(j8THBq>=jm=Tt|FvE@d1fyUc4`Wx2x)46huI}k>RJ!CN1{@ z|J*C(Cb1K@L&;^w)+yr6=0yKmbWToAKzk^_*q{UAZd798+Tb#52UOAnsS}s|yi`?f z?Vrt(2x-P-gXe@^Uw60U!-tq29v&F-aHzmFXb)5+^h=j$8g$a}yKbdLdAe_R19!<{PMNN+9!zkm7+siIxF>;`O-QO6b_7J<5E7 zYSCDhQ-M_Er8&6C2!cF=%k$)=qRXHBZLk;m*6dov2$;DLI5h5}Itb3xvl|;>z-Ih& zaJ+zm8~X5_x$1cayi^u#f%42(84Py;#43SrwYa=2Tn>m=&j+Ea z+}zxgV!_#uZ*y`wvm%d<>>7u^93AuNxoc@@O{^~E<|&{s#)^odqAPrSgvh#62U4+f zm(>A((#XonmNZ@MT-*K6rlxle`4aCO{mOXh?)u{+(fU|4o_~uyTXE+MUkAA!^10K3 z)^)ZJs>(p2M!L|bFK0TsecPnfZV%luG(QTE=+ojQ_69(rRpnJj6j$*)<98te!k`pQ z_`-ugWx+Rf$70fKNKY=%>CQ>f4ZpS{QG2Yk98v1ZyU^d^Pb`qxx~(4Z_VK4Y>yx`l zx1C};-U&xNmVAEDJ|syfo(LUnM)3^Nw32fK!m}JlK#6<|+0h?l{_0sd;a=BdoH1u- zZ%|X{Me^m6s^g9EY^xbh9aTt~-j4_9*56!PyOW(`;Y$6i^yih_T)G25uSUzQI6f)h z+S*rd#Sy=+tVACgA}kg8V!yp}9zn1ACqY0Z32?~@(oFgN{pX+C>`S~_HlsS(t!eZ3 zM*RpRrHau6)pP^v{qON=9T%?4&e1uruP)=nmMu@;R~S^!$z)_-Uw&;zsOU31gn@xE z3p&$w2*hpgKqQpJ+#;`}M8S;rCu@tk4{i`N zU!nV{uyLYZEMof*b{2LiX*~+7^o16MNQxL#Za^eW`>`Pia&_32uB% z5bZyVgJw7g#@z7o@?MTaaXf;8?maO#XNBnt8_Lg1$BIFO1Owu5pu?GMwY|632F|lI zs~}wfr9%($kMIDYtg7k@nGrBH#G1fw#gL~uxl5Rx9j~bVfC?cIX$fv!+L*&%PFa}=Re z3i(5pm~s*d!C^Kul0OeN*an9yHr?Nql_BpO1cv}f$-$QE`P(iDDl##Gr94ggGPkke zy?VPjy4*bKejV=%q&F(Rvh8@q zCeFcu^HF6R2>`UzyRTU)WH-#zG%V(OOE65<#U3~S-S(hUli+8I(Th4lE-y2a(zP1L zIXY`^`uD0g&`}(bw)uDB6Ka=V=DWO1yxpA_Ff{McpOLpG@FJ$w$ifxx2h0aQe1_so zjhAhc$GL9bk9hySZ)XA)M_6ae+qc6PpU!4q)4?oDItsdHg#Vq~<2+{nAFgWpW}kV< ze628F7rV3Q5e_j9OX;NF*5+W-uX!^GOK~*IYkC{Ym`Xz4pP5(LM3#)|e)Qv_t@a7{ zP?U#*NCx6kF%Xn`-t*y7g>~daj9GU;2C_ZUcuoca_<4+0DbaW}HmCP~B1}xaNX$Ry z!>Y=kTpmz-|8^r{R(zGtCv34OEC-6y>v7$*gw)xwA5rJjTdrchN>CM+xsG%a1=fIiyshB4WX*w zq6#Chff5@6bSDb#-PpLJECB>9bv?aJ;)#DY1jxD9xo5%nuZjK`#f4c(LJ@0H6`ge&f<32 zE34*$XrFZZ<3HNY({kW0Eb&=1ysEDC8@Zc4zzVL?`F{QzW7=*7p=uJ{? zy%>Uq&{l=a2Rg|=-7q_Mpwk>-;;v_@DD@N5;LH$%)9bB|6yXwi!C1pBfP;Pw}3NV!|v~#YXTvUGT~k z4RE3oz1MUuC{RgN`9i(SvU)1QS|Fk*@K1R9&y8RE*z|w@JRa3&-43j3(y}>)IH|#k zD=*Egy9xvPCJCIERHA`iT{k|1d(QT#J1JOJ`8+ppOOyWQOwrV9E%pvrD%%saYS0$r=mPopz<}7(Z~0DE|l{yzNKF+4^xo-^64F(hy;8UTm~6ZdIUt zkx?Y0iahQF=jkPl{Y!{2=@;IUbe*j-5x$6G%h=uL?0_wQxNLMDyd(32~XljbBqLL6l$hfg?DL{?C@6EPkj&D@dhm3;Yom-eQ- z)El@j&Q4+B<S@G9_213&8(QLy4bFZ2cwFhVIf1LS|YY%GA+T?tIcf#e$J>VI+xKiL~CkTMWa zQvOobhtd@&rNRoba#B)ZH%M^6+?dc`VotdcEPDQvWT4WIM~`p-As0S{xoMJ6Fn9MW zLo6*RDM67Uwt?}M4o3+3jm!!oF3MuZWkndY0yUZ>e?x z3W9`=vM(4(W*qqS>`~?E^0n}8uYVc| zCrq-r4S`NyW(lkQWt`*+M`hgBRF374!!#Ndasf(9!@(heiHq^?9wzG$Icsij4OisS zBh=zxGIH*YUWSaLr)|$0R@QT|^AC@ncQ`jb^3QA@UtLBZN^{b_@NbEQ^}AlY8Ul^w zk8tZ#%U}19jgomkbPuiuOUZiN9M=P8h4C7FA8&1v({n(8BtfIoeY}z>JF#(6zf*r3 zd)IE+V}x#kZP_Te_2Bc0zN)Z@;mZj0J21{-&^6b+ddtCt zDEDKp(zDwkRwai;amj+VIy=C=VB;Ru>zj=8oqT*qb0SMeEw<%-ame^$%0Z;xLJs5- zXFJpN)x}!xXJlYf$;}U9o|WorSU$zY9sAFpYqkvbh;3Nl(N9w|N%MLY{r)avA^X_s z{9-R!iN}6ZcftL4tR_~=S=7RR=RU+Zp`|ka{Xn_zKlko7^b|@sokdSA%1X856jsjX zt+U1a(rIu_9uI%Q*zq`ujxt@nY%(F}-XKu{IxeGVs>kW3<*LTy)D$C3-1VZb|v4EyRc<9pEd%Uc6GSy`ikHrxy;*;?%M-wN#To zv15?fgy$N{@zllbo}w67#i`T41i^S^DXC|Eu!;ZV?<3#7`JiZo)<4hc;YCyG4<4YQ z0DX_&UswQW4d@nwx)g$c=O0E=!d+|#1+*Xj8UF_*W6IO~(=UehxFQUSzl^HM0Xz+Z zWF9xN*c1{pP|j@Cm-v+*d}CAer99IJrErdy4yM5 zeraRr683MD?hkn7!uxmF^PAsZ(^1yaT8k{W)f%w>Mq|QdW4Yhk3vXqmZv^j4JL6=d zYVeTl`a{zaf^?01r2BPlFKA#3%(P7DEZyN4u$8Ar8Uvx3|LidvG|UvwAY0|I?rrJt z79l_seBF-|x^;9g!eX~9fs?PH6MeQf#O!g{pHqBR4^kdHxhwXoSq_>82b?lv-Ia>u za|4wJbLUB4%^cycM{4rUXo9}qVE=c$!h!qVes}dt!UUvaQs3Hm%eK%N5&5z$=kxX6 z=8;B`koP>E(cOdURQ7wNcpuI?_LnmX*`YE}qjPWsyF(8& zL6~P3?kJA7EPZ(N^HMbZ^LbVWj(4LIOA#hI8}Hl4-TKUmtFg%AV&rOq@Lax@yAk0 zkqsaU7#or$IJ|$vBvAPT`A^8hc)I-8jdCHa51q`B6P3Zg9t9!f2^?Cv^lwTrPM6K7 z0_1A^J^f)tPB8t1e}_B7>kHQ5C0>ZA}K%?(;nI3~bWqWo7s2e|R`?h@ z;86QlyMfx&k}qCxfb(EfT-=C>c!GQY%Kr=ojoCRnZ&mpWzCiIS;Uq@EYNqb@?;{|F zg!%aR5GW42feNJ^njHo_2_MD*b^yvF9)Qni|3FjZ?VqghNsb@zuUYZ=I(rqu@95|? zEq+NTZ%=VHwzmjGaNGVFeZ70F zVfQ=w+2dt_j8C8FfO=kcFs=uBA(ovZ@atDt~TLZ|3$Xqk8}nf-;oHi`NrjsD8TtB z(7na1JZ-*^Kp>nfmG^LtPYK8?>C@j(x`y7LCH$k24FcF8D@*;(MtrZ09_^Ax>W+#; zZe)wHP0LjN6=zqem3J)(i0o2tdsm&E_`C8Z2)uB^ASjSSHEa0u@KLG|9dBE^$-bqBlCU`bPIgU} z%-j&qqFw{Gl`nquOo->|@n}6Jp{w*@A>&Qq z51;&j_OALpyTni$V#H(L@^(~c6A&fA7L)Vswe)SbemFE!DT4?~xi*&TWL_6b$qk_i z{xcD*ocZk+eKX0OV3;8IK69t%19CBRG+#dL}2S60hZuZH!+E9y8-@yQO9{HMg&zpKB<;?)$2T`PXR zLnHT?UT@-Jr}LWN%WMQ^!zg`Jlpa?Cd)L?8w&8-Y1E>1^VWRb!l#%%wGy{-zO37H+ zZ;ZD(7F`^tJVpVmuwEr{Vn&hm+Y&cGb<`<(9D2#%^vK{8gc#e%nG{UKWUXdkTbnJH zjW#8-62a@rN|N^Wd$6~tDLDBXumMv68ki6UhI#U;>wlw_*`BihAEPKqH;>E6xq(0< z&6nuA>Rp}J3PODG*U}%9MLhW>C<>gS7CUh=4>A(#?mYkRKUywdk{aZQAtCIKiL8r@Aj(`mL&g(##flF~DPR;RgYmY?6FXUj{40vl2N++QO_ zDgX9(MIqV31&oOiWmz!6R9dh(T5<{A@-mL=$rpAgb8ErUCJo5nhg$-QuM*z93)tSa z16cKM7)_3aFbbdmgi#e;-ANO1kcmVw$rypFqH^94ZG7Z-D3B$fYXx>E+yJTymUo-* zpvrhSI8X-wgJCy7_TaM~3RVqS{D3q9krkyEOyUb?r z1+>3H;^z+|1yP1`-ppwSuNzP~dwh-|rF{iGWJwhxLt*%H4+___7 zASWXz5~Er_-`XvXrw^Y`0Dj)-+aKhPLGOh)ch{E!gf4iobiAFh-US4%@vZi7t>)64 zKz*;+J^W6%p~Ex1toP?OJgu@ZK6rnNR(dDQ0pT-d-V9RQs_7Q1LE3gvdNeRms6BU} z<6mo?`SyF@aa79YsrUHpAGo+)DM4Y4_olydPrMyC;7L8+$gFqSe)6{v>DkRN2U0yx7cm1+;)1(ERoQNZsy?AOpWa!@mnwVR(G8E5AC!>J zMV&0QdM5Q^nIv&Y4_hp6-Ry`UY>}Z?kbb#WLq4*$$~7Ecqd;idv1m4v7-z{S?q>?q?> zptfGS;=gNf=f4zQ0C@@HeNvQ~dhFp*CwUAxOO^9h@_mcH8Q;BkDJdxf&P;#diB(-A zBO?^87I0 zqW0MHHEsiv7owi0F$XglAJ~`d-k3$CYUFW)XKWp~H^SUnl0i!+Y?jOekTSnj5BGy)M!1Zm?t^8Ngh*sm2?i`1oPKfofTsKS zYHSvvYvOKYP3KxS(7lioe&lg0Gc(wlh3w5gGQUT5KZ~$=7fHOzokZYU@7FZOohD+iUraa#rI6uVT;^ouJ?~&eXBMN%|YjrcPWw3H#XZZO*+HZ3&>Z558U&(-!>znzri-L$N;XAFj zy%T~k8|evy^OttBxH%ur#R>m3+zVAvcI6d6kCngmU`7SaWsY9PxP5sbaYK0B-|J{c zfRn*+hXLauQep{MBjXOqA8^AKaxn;g7%4xI4ebUefSrSr1ljYr63C3KD~dW~f>k&P z?oYn$A|IJhN2y?^Pxf3f@V4#8I5Zz~pJcMu)q9aode$B+x5!jd5PI`T_pNbiy*ql^ zH!&D#F<0P5@1<1u&wQ^^iVWymBX2$1id$m46;S_;Lj3#y6N6x#W*O(M*Z(8zOTek@ z+Nh5qsZf-Xp&}&7R3arpGGz`SG>C|dB{G!CP|{#Z2qni*Dj6G)q>>6p$}Ew2C__^G zYoCU%@4Np0`k(81-<-sG&a?Nvhjp)e-HMFa({F?wf|_^dIpvCD@cDJOZ^JsS9!@_M zd_zOGg4WHg8{d6=dFO=C%16T^6~++fX%dZCw<9!S=(JOPbbCaA?$f8CrQOYq>@P*F zC+tVi*2j^iFQsZ^B>i*;|Mo?5=a9xPc^3)alnz0XH2k@|_%DX%F#UN&{#44jKbxgr zDr=!43ukW7!BUryiJiGk7aJm9>@7e4(4r|_hEVCZclQZa^h>iZ;^H{7?e(aS5?GGs zf%iRe%q{8%{Cx^?YOg!@KUJsYqG{qdrtiUz->runH&<<%yt3x68kFHt;*{G_UK#%a z4kMjUwUEyTzLD+N4ipP6r|R z=;@t*v>6aOnqjO<)M-#*6qmLw;q{&T^y0LpChMWGFXiRA3ch*F7+;WluZFqM3YnD% zHh{GE9jPBbXak&$gH`eq9rQMK5Su8Ee;yfMxYlOZ!Kdw@35!ud?pL!-+N5!=JN&t3 zw*5;x=~%a3+U}+T!}D=*J`TCsqdnJ82)>$E7;SGcwk}=rmYQ{gALKRAr4A*Ft$d%( zUoZ~{Dc2A1H#M|6ThrtS1rA$UUD<(Ii(F*AXB7xKT#(wFxM$NH3+Dpg9RAdeb#Kj< z8Xi6rXeDx5BlDyUbrs6}Tg78GkA1h?D`Pp*IEN43jzWp{S0Mc4L~kYT@?5c^@046_ zme}4`7gnbZj~bUL*UWn)^hj}F(hc_h0oS(rWh!QS(6th5FD0*jzMH);!}s`xAC>d- zk{;e!Oc@Tc34hkTm=PHWkdc zoJD#8p|Q}=h+y(UT%4m+rinN-&XhZBTj74-&Q|g(`}0HYIVXmRi3yT{hgp|^4KEIY zi?%ix3>onaa3#pg&$YF+ou5zGleBGwi-oUHoEh2-r6F5 zAZID&bt`!FF2V$jxc<)|yg+Ffw6!mZPhGk|7=E8)Ki|>GX?Nay(BHrXJ!6#K-?sl2 zhX0&~#x}$jO44^E8`xsPWJqWzp*mi?(-JV_hk4bFm7vRP_w>9U7)o7BaA5RPVUjHr zL*(R?PhP!h)n;<5K7Yu*z$0xgS17Alw&XUGNYif3Dn;`fWvQ~)-^z5)A-klveu9yr z+VSJAdllP>g@*Q~e#IO!8~qkiN*+Y8D7VJ3h3ZPUiu=%{l|^o{96WWN-V{oI;N$^{ zdW%T@v)0w?3QP`^l%^;q?^|HFX=>VU%PV~m*Gq}pB)Ik*UoOw=wEz7Tn|XdgJGwR1 zo5L)`pY$6XJ>~0}z@VyWZgFcdMrm`+ojaFwk1pco-Z)kWsw+mLohB#@5F(@xjKU%* zd5f1U3C0KvUo;K1q1lID{Z}Q$=TuVCiePutMtFI76}B=~??W}6E)wz~r{W;jct(ae zDTJVgsRD7Rr+Yc>p~#{>su1;Zq)7OYwUhy%$F8SOv_R#XZgzbd+$y*_Y{jWd>I^UUf(F=486IYqa%ge>qtjVdzY!pt^$a&T~ zTnK!DpzS#~>frDYA75QrN6mEdc_pB%tbC47o&!UMx(=^w*4?p#ouogQ`C4>nVq&5K zEf@><`I|~#B)-D*zB@?xVs%B+@&J61T#vN43$GO7ivDm&14B)JQ($v*i;8^UZA5%G z%E&BmaoKI(pM3HpZ&RW1EDD^%ij)+^nY`6YzUH5}-Szr3*HlNk^0_6_bG&9nxJX@Y zId@w9*_+~o@RL(}{t0k}dNmZd{%g67wS8@|+&5=y3#JTq+8PQx6Hhib83x2+$}{WiZN&G{ED8eU@vnFz}JBkxm`np zg)o9K))@|EPGodQ!^sGQ`cIyqS=c@s6RpXncTN5s{c0K*Tu}P}QY1*jgbgePv@q2` zjOY)H4m8#ue8K{L6*NMLYXy_y42ChgZiOQYQ4}OqgAF`vq^~)}UW6-@aPLd2s@M^f zx;x-~f=g`?^?&tNVkbM?Q2vI2*d2#zAqWDbVc8x|tcT_S$^kr3T@UQRFO&ja2xr<+ ztFOEbLG2@d*dkU|*1-JyeDeK}ZeIlTkUuU%82}u&tNlwBEC_u1R2_O`TZKOgw>5_% zFNUB3>3^Wr*Kf+zNj1`qm1fxY?A!^EI#%rI!?J@bJ{#UykB}$+?&S03D(Xtc+qRuH zPKbzzplDHZ-90jsOe^1J##rY3Jok(R%6`_LI_uu|nP!DJ4|zNHc2_eJ@vUdY(@yo@ zUhfuc-{@v`*<)lGWre#s1wFc~@n;kt#Lt_bzogY*K3f9!YFA$OZLfby|8v$ky;cfc ztvaw!k@=s$G&H+BQW64M7Yx8f3Z3x2bFRSk?f1~+?%Ntj<5K#jCSP64$)L$*u+LM9 z`zin1@CKmArQ~x1cobdr{Oj;ApwY{m3m2B1U=A(10~7$|)hg@7q3YhyJ#TAo7qk+i z@5&U8FBl4c{r-KLb)uq|_^R_O9k-%igOvAAo@7kh0N?>IgOI4GM&N#OauaHQPEFi( ztP>9|YDH`r-w(#tSpv6UZ;X5!-a@*zfF4I`TSpMfpiUFO9Nv8xfa2ib;7YZ*W6J>j z3TEwX>a@w!)YLs#UJfsa=#q^rcu|vlLtR}lfdkz3QuGoU+Ss)2FGdSFuyP?;*-H}( zkE)R_5`a8NfBNq!2?;S`O2Mg!2>scsd}WcpX=-WV{-cbq#>6Z^2MZ_8{75M_)r8E(b>2H(2F>EnyLS8y{_Z+TdK=X&Tip>qSbG%jeINjR2Em9sp3UY(X&Q2Eqb@RdL*@0GWP zRKAOv!aPbo>p@F}yAA_fRl4`C!YIPoQW$MFP&suI6uDs-8B;}4>}Al`iM?I1C6J0} z4K@HOOMhD&reVkpEi4>KkKLTuGoaah`SJuJ_2l=0L9|Ktsf+9tqDS1Af0)SH#h**S zd1DQ7c&dX zUJQcbxcrS#1)&27OlJZEDJKY&@8ag>267WAx<49($x(pjCp1DuZ9!j2NlZsUk2L#0 zRK0A7KLP+2ks(BeyGdHWYf;XJbHotA>T}ee^w)<3DCW$-t)D3cT=W&tv@T=9BR@{`SXJOxm(L+htiSH=-t|?&e*s+ zDG1!-f?%s|F(soAjWc$J*6%N8ml3_1js;$G{FFY$a%P;cc*&QV?uQ_rtl&zGb0rAL zi1uO{deB7(odPYi!K;mLmbA9o1l~{|y^lgH#P`pkwXv}eBPgWNbE&7~KH#Ja1fOW@6ISQf@Z*6`u-$G{y-HkEMO*GL;6?Jmm5s_(lA0u0Ta(2y*{*IABZ5{ zpn4PY0>W|1Yc0s`AXh?v^O0D#pV*qLXodtyj#+m*EQ1GnuR36*&?$PV&e8c^E4~BO z*pQNv5=b!$^dKI5|HxM!QKwa2MS1z2URpWU)j$`Vav>MUPi|C0|Ma`2df`53k=^F?zRZl zZ+TyTpzrGR<#wLU8}C=Wt6%8)|6A>CS_w;67s@&KbT1O>U7fXIg-AZj+3&}MRMsw3 z)Qh+g9VR37M8Ir~F4(!VFPDpY3a99433BeyPU2&k5>wbE>%|HNItEGX15%^lJ-hgxzmrf88Yno}8mN)~J=oAWNJ47S)6+9v)X-o- zl!z*$*9KWhE|KKtG+agE2F!~kGhqMaW61t+7 zjV+PZVyZ=@)05DUWyb%ZJid#Q&&JR04r_kk0inFz0I5F??iTP=Tqn#$T9a7kjDV87 zHCDM;S>s%roVbA2%1%F7b)_r0rRcwI*kE}r{r2OkcXh@N`=BD3^;ndfWkrUdfCrfSb)N?Oh>@Mw-vI<)4MBp9K+Y)4E z?y=pa=<7ti1iZph@L=0P6}2@Y{=XfjoSq0p__H8{yPb^^toh^P8y|n~U4yCY@B8|C zd+)e3D@s@u+|kkf_)Ci87SWH#&>)x;A!X%O>i|@?i2(Dw#S($$ujY(EEucBufElSc-FGJtOlT{Rjl|oBzIhgAncjW0^Hc&D=vAR<%pdr zcSheW)$M{Qde4m|pJo1hlyuSU?NUKd?W}G|w;Q+#=q{dS(^I;tD)b~0%_rlAD#9Pb ztA#gb)`h`NcOVZ&zY_!uuUD^Fy?V~b$O!-QaO&kv$Yn9>YrhnVHa~uRRt{E8UhsD{ z8=)3*@7}%sPK*z;h4Ew*73wX54J7>}P;gYpUA<^70kF`XNOD*K2%CQ z9A*TCbQbYe0}!-s$H>4V?>{pIqE`}8xHM1z;(>7)`pWRIee2J8V}K?>17+yjVzqV< z5IO|(-Wk-Dsj=&tZ zZRX~&YL5`dBN%e+sS(+8^4C&5FibZwzwf|F#B6?k$gzhe>NX<(92cEhG%_%oj8A;j?ccM1{iX_oxKsdQ+3oQN;4C@*+SbVlz z?^;!8_lv?83Q9bj7ZEd47I!?xkm8~$!oBA}2+P-%o7*HWvkC-17eQizUVx8;ii$0t zAi9ht5QCv1Pr^V5F2A{`YpWgBcVncd+HMeN{_x}!-*zxnw>~&JAC16ZqxOCWRSw=U z0!Fv2v9U4zC%nuKy(0vqq3u@MvNiV)3soNPiKK);u^eM~!en#c9&B?);!McSOc_+q ztoxW6V|~)RUxB$B-C3@mUkm(NDioVh4Nj?j>;aR9fRh_4f1v2>M=Klhcajgkzq5x4 zr)2lIv3OLh^d3fNt0(pdtgR$~?EY4zrIwGq4d+oRHqG z{zik@mYU7AhQ(*4MrQvglP@PjAkr-&=uMc6=-%=AunJPJgvt0z61|*Z(RzamOo29XFG=2NB*V28iYKfF9!i{4t;djL>~H29Ulia<8NT|^~cmu~zl!FHEM zw`{N?bk5NzJ#4{!QUsx0bw&H-7x*m*b3owjrY+{;f>Ng0UP#*#@Tu2lnX)8Z z@M(dU)S#>$&{bh(_KD&@D?|RIRsC;nxpcSqI-#5=!my3Kdu1zk44Y}mG(`lh^BNm@PgX{nGvSRd~xWS2J?n{UMO-q?CrPHHCU_6sPbknm1VkAq{`<`B4$jbr2n zVc3Y`If2tavn0s})PKGcSO+5(=uN}KCzOziC9VM-$(ZZlN^>k+w`lQVAqffH0t5Bt zQ|SH0?t6<8Xn#ZQJW=D0!{o@IiC6`1N3`kxpH$03KV{)595k3ub)Xs}A%LLlmA%63w`VtKNaJ3P>bg(Q7VHvmN842LZmCbNTZ5S9IT`w|+q330*|2VokU+t|tQG_fk2*l@kT^ur8pXC$_Ec*GLS(~yA`$&}1utTqph?1v73^=! zf0I0b4KK~Z=G~JTvm)ke(Qi|^x8{Zxs$V4_Lk@Vsv0xt0Vd6A3Q~59yAE454LHHxQp`k%^YVzARTZ|>*k=@Ta^kF~{ zwRVE?64!TVJ{#LpU}oT~(i&7Kg` z(0mr3eX-ZL;_qE565b0OC_hE zB|zr0kamg%I9Q>l7J3~$3L*r+t8d>rA`n7$&?FI)pp;2ysfg$o!$96eFU6659-v#n zBvu?pw3TIkmyy0cV4t4F6d^J68dGc0-v-}peSFd8AyFYN#{aRC&HrmBcg0GW9(tXA zqWVcGUugFSp0yhey+|+3{Hc_DA;i);Q1P6R_%qrH_K&&R@nY30GmEUdHPvs`v>Klu z@uLmz{9-h}w*>$U05us*8YD$B()a`svub6LusaaW`4#hrW;v`%~l?}+} z2_q6mX!gwMVe1Ao3@^w)xY(39M{NRVUsHUA-iR& ze;xe{Wm`X*3o0)Lx3@n*lxPG5A$IdIS(1xn*COjK@>*kktdpC_fkKFcbBf{LwO>-Aw7H}`2&smsdBt_NLKNRLk7>Y+4~LwXo`XX8U#CWQ-=MQrfm2qxI|5eQQ@fwr(wm#kY$5pM8|)wqJ`6p zq{R5=VlHva*|Xhx?aT1RP<}R)A1QGt1&P4i`Qgfk>%Q#Ny2#&t!feV{$b2QYB8TB+ z)%CIu-?c7$!MSXBr@LipVi_D5X-Fph1<(T+B7q_qm2F&Or$OX z#r!)X4J(CGmZgDB;60FAPnbi!(0L3bO%^2N*iZjxw0G-y*`b2LT4*(B5Pe5~(&pSqJRMCQGSvhK2{~@?kx%x^7WN~0Is?Ts?(E`eMHV(5 z%0*3Dsg@Qiq7I_F4^@8y3yWSF6amj8O}E9krQzY>1vqb&@b|FiMB;`XkSw1b4*t7V z5p>*CS2PI7F1S)fOYUkiA3m)PjYD;%uzXcUnc~_1rOe=>@jvaquiH?aKI(|t)ru6C zF1_%jt8_+HE0PW{7rnSc&0b|4HX2wQCO>Taxjp+*TdL-KiT=J5TU=y;lg`lq@`Kxi zh{RHCyWQi3M1b&n_e!{w`94Y!57Y%#qbbZNoo1b9aVwsUNONL$$ss5u6--65pr(a} zjfP^2;6wZaN%v#AiQ;`E?(qt4jA-das&-n1uE*1GUOUxY%u-@;Y}Lct7elD4;2xf! zpEh3fR2Fb{Ojw^xSVUIUP`(BlBd@M0FHflut!PAz8~>7C$~XowB0$m=2TzWofyKQT zjlxPq^$L?nPh-|6#r{pA^B4I}BD6L6rPKW0{y8wR@mo9$CJ_JkPwcjN=v{4h4FE7E z!nAA!?lL>gR}o7G<4w+FZ(pB4!!M_L3In@JYHN*28ri%Zle!6w^9%n;D*L3E!sDfo zLjLGaM1uZaMJw;VW%&CFU!01Die>EtJAFxdNBx+fUuik0d6E-g)$oOVA1G1}7xZ#S zZ4A&})$aYGoYW#JS-b_GsJ%X65i%_)EX+){80o14H9zeLkPcKO=zY!5D>{wNQ3fNf zvCT+#=pCi7enF^jiIL}rCpZOZ$|EGw$HXkEedzh~&yZ_z^78i5OwqMR#D~$y-Kv9g zXBczSG&2FPv~?qr7vssvN%H81$Hu&<@;qz=ABX(RS_6~}a5*}mNvaIufI|lMcg=?} zXFcWKJtKNR4}~H=I5~mANAMi)A0Asb>hp+RL2h`1QZbQ=`hgk9$y6$r8*!#AF9KOsN+*gmAg7AjnxoQYE1# zp&>pSU@M7Q5KJV;7^oYN7xmn@^cR=KYX{M#rq6N%#|y;hlGfso5Gzs?K%IiW7bK3m zSdbPnLZ$g70sPvxHO?Nm?GYEEqM(m{ z`2|Vf9*(`U)GOZTWr~V8B$3K71>Iox?g82|@E*9=n0j6*4i2Gal2GDru>Qr=ee*h~ z+kGl?6aHr@La2O~{<}9wQ_S=#`=+ms$%ork?8D5eQTF*0^7Fp?sQh>@tV)~IR2O>c z5XIAfUB%Q(>2-^Il6vb3Pl?`}j5MQjgQ=f@BgsuOG;GmyLIRKT%60$*A*SEM#sye3+oD ztP%1urC`X=plXJTRP;zmWE>irDWIDZ#=C<9Rb@5#gozMYI0@+3Iy=7}-Qo-+mbVAn zbcAS_A-I4b;?e#wJV6*DqG2ZI82P6TaEoj#ykmjiRNkXOkZ`6Q;mHy4De zt8N$Sb%%geqVM3whZswXI;7i&v>P{=QERRMVr^(@`mS_Ns1FOA67tpHqBUl>OAd8? zb|woW)SCzzFuhOOiak(YePE5W^a8YXxqi21q!6nE)!;Uq6;x4(SRtqnOEC`iZ%ee7 zsCyy*yd{#J$HHG>@-yK(pzKFlrHGxiejlRQ zZ=<#PhDJtYl0nxQGUx|wZm1{sehbtGUL=U55&|pk$0V0vOfDb}Aw@-tKu;X%Yvp^l zVk-P^evRS8r|e7am!ySKID{56_=~5%mHF3Gg*5O7%=!n5N?De!V*l&~mtNCWb%8ax z>Jhf#Z1ggD_Z&p5SRGVawxcv>Kgf4s_wM-|Jv#c@%t_bM`1kpwFKB!AlVWH&^5u&v zrer{rHXJHPD^R>h*GHShy__86{t>|PRIgXFUWlYlA>AswyRV|(4%hrNpiiInLjjsKmc(t;F`Al|>dl)e5G^B&2J=_xzZh9XvOpcTokcGvC^1lN zcr*>F#9PPQzm%EfAc*2^j!$;Rj4Q$&6BYFGFm8i%6Z$Y;M*%=wnAnH7=Sll_S_R4n z=Xhio5vQEFdew5a=#AIHYu3yoa?~I`RgM!tAE?&u-p$V)*6#(^qQGY?!}#hiSu4=_ zzHQ>+G~9%G#L22Ig}6(w4Ukxw5avGHmU$_OW{Z>;aKPuZNEPwLgsb?^MXmZ@$>Hiz zcGLb>nWt}c-iD)m$zqGxK6=}+t!q0=fP zLh8ghKWCl_wi5E>PjMlm%t=Irnu%-oTfxoV!zIbhzzp$}3fw#A5(eUdS3D3dI8npp zC6$<%h+6^#b^ehgs7H{mV!TN|jcC(Hj>*xPk|Kn26r#?5=Yl3{b#*nVs}p}06Z84) z?*R#ahW<^FMOA!Mv5K|!*JOv3*RW;nlb^2zYg{iyG{@f{Y4eyZs5B7E16ad|{xGO( z2-*Yz0g^QUe1G2uRRLB=UeiY^BE|g*%S_01*g*qcb#--wOlj@YBVxTq1=X2o@S8yC zB&-b#c){f2RHHp8BlB!@7mn1nBg=UsFK;oFzsdlt?9Q9|{(UcKcVKn(wC#T*s;s=+ zGT&tzR@(R~2w{XE!^0y+W@9C$(v^=8bKSOEnPOjWxprH6sSu)%6qoG$xO#*C4pyGp zJo$HJw?iNt6bH3iAH+RT5>IxlGVOm+zsLI2rrC1+Pd0zzIycmD=2VO0{+;z5UznKY zdaq|$?*rlpgCl=n()7ZB|UFH<-JI5QAn2<-^k zbN$ZpE=?u%HUZ%=HvZ@|Q7wt2zvpNc5)~Pc=8#anNG78MkkPB5cCPY*5EGDfLpkj_ z=&065zjo~)8f>J^R~j|Ei|fvopM-O00~-P%$>Cp&Z~%^k5iS{X)K;}umo$Ro380U@ zY`^TrTpeL9!ij>{G&)#+<;WAX#i727$(A?DhIY4}h55qAEF1v;{X)eXt@{?YRJ1~h z@>|4L%eEia1N*``Gry?s_!h!||9kmx{Sf(Obbc+aPh=*yXw&WVB_QvQxnjSUJe1+~ zGLX@NfRGUG4&j1zU$5;Ux*GADj&c~^)ZL0UJi`3c)}@UhH`|(S@&uNY#5xxvBZdEv z)sB2@Z)X5;m`0<$??VvL{`vDNjJboiBg5gIl$DVhBR$+fibuNZi0U3{<-Z!Dzmf{t zZvWFGsDT=uPECy~7_dNKA0Ug%bHn;zsu)w=^S(Uty+zWk^w2i(_Kz!k+=- z)9W_BlpUFxQLc{^UU%>UgQ`}yK-619m91e(fq`O1R%Xo1hV@ecP5MKhCxWIXlq4!s zLd*?RW-Uq$q6Us~Ex6Fa#p=v>nTME+FNi%6&2 z3#lAC3TKv6{#?CcYk0%m#d-g{rY@OV`NYEeN3?d?@y(1S553b;m+s~j`!KLR;iR}J z@A+=pfUH23RNdqg_Yupq!5raSy(5{HPz1j5Y;{YIbOLkR66Dm7A8uW}Y17ns--bNr zOc0kd(JNWEgoP#px^Ok39r0%X5x+OKPK#FFV>3nH2BqW?X}u6o^0~hk=uPUXYsYJB zIRj@f^yVh)U>~>@QMc#wD;OZ>i>`fP??s^`9r%#ve089N(2SfAX|tTAP~@5c6y##2 z2&w`fx}a!HMdOJcY76U4SnwuE;_BB8{eF(s%hW%ScWy92xgE>%@zp5?6?U&LpS0HQ zE~0ISiH^RBwOS1z$$*=Z^cXGjwYcT`*y;$Eo84@#YlN>yA-*|a=BK;Dx5qwyrqL%o`S5tw{-SFyrSAINCMC+DCZVe846v+EY9F*St zh%MhW!T8PQnJ=Jdf|zFg5@woVvZl$5$Mfv>_TH4hjE@&aXZi*&R~q_SOELXfC`xe* zceuvJdF+4wf_ALh!KCud;%?oog*}TX)VGr5w?dtb9qZTwhAS3DZ6EzwWkgmk%J#V@ zM~1m^woWtevrL^bZ;`^n7kmbMR*!skVhV2!$fAP9VFXikF~(XgL2MJ0DGOHf0O zeFP>C+_IJSuJ8lATbHyoh?c6e#NTWyz??c%G`s+VYu7LLZes}Nkx_+%sIxk9^kiug zQj+UL#UyJZKL`6#S@ro!qYJX#7)man!*#}R&z?Qy->%pv8Uj9MTg>29pVA@kwU%c? z0t3a%bYX;X?I!v4`1;S)v{174KduxZzD`1UI(cLIj!X-1yFxo7`FxAL!G0 zBZxy5v9WZHc(ZXcP;~(XM_s&_48Yk48#4da?jlbuR5fmEg!87Nfa`H*9#)Xke=<&J z0V|I<1BE6E@yJ?usj(NUuD$2Tln@uM3103`g|=fR`X5>5kq_R)o>rS~pLk4TsCecv zM$)h8f*eIMLR=?|iD2oYBy}xS^rRvPh(s_ea}-%nu1m88KpI(d77A4gTY{|bK=T7L zJ3G6hO)*!lq{E=6qrPHQ%mQYz6DW&{=r>92!%n0j9e3)Skdv}1NJ7?<0U+_f+O=zU zfFo{z4aXMAKoL#{Yzy9D-o|Th?5Ql&skmdiix#tCMFJHwuf+H=G5%S)q`{B1(tlnU zeCCcmojYlCw}*{FrFH3)J0^TIz1Mh9an?kvRB3AP)d%aTSIp{9?nx-xeX};d(o=*- zHsMU`TUv+F6ivZPGu(J##AUvR3|=bW9MEp=;1k$#+}twnP-x*UBe6v_FkxP8B@9Ym zp0C8tjEUFtH)AR;zbApud=MJ_+#bp^KXO2ch`eyY_6|7}lniSCTAl8Ni{}3Ptki<| z$*cQqsV#TlOJvvGg`X2^(FjjB!(5>zS+Ir&(E_XJ)fCNszB`L>vcC8{%5#(E zS#QEf!JewA8HEZ+H?ovEY^XI|K#+0)2keMfFa(OQ!JHNE+UDZ@$a)8A$4FdFa8k>E zfHfWkgNc4zzAUD|xbhy)-t@G@8ek7jzONt}{YX|;HZn2M0|A#=E^^Y zgD~^(Hn9Ahmh0Q+qkj-QG&dG#NX(!Ec=@{eJA2faL{xrWxbo?pU_d%qtQ-r`y4&?Q zaKEpT<$JcRc_6g#&+DY~AsW{${PR@o=f2#LMkx`<)p2v$+qlSO*-m}#$kSdolXTNu zVtQ2f6h1i~b~wKFlEtbAbz9{028=#Na(!M8kGD)*N(yJ^!F|M`c+xwIR$NE|HN6n7BQN|PE6S0zwxkb@-2jM9l>u)EN%2KGu(4~3%&6cTG8|tZsE3444 zg@<`2gF^8#gtRJo&}zXP9}+WXjKJq6F3)cNev^FyjD;^2s$ooFF#gNUBtUn=TAPma zQ+e_a)@lSmnh1vfn-ZvU_n{jhfVx=C7=O~E^lm_nQoo2 zgmW13e8&hR71W;r$$+)Wmz zP_Lkvnx^DC{=Tc|=+JGn3tgGRNTIF9o~tE>My3y6yaNd$@>f`5a0Rb> zafmZ{@|_z^(J~ufDwH(n``O{KOO)GT84A^ry+gn!-6;-7?9bJf=yrwU&kIU{*~#RP zJS$1!8H+2UE6aN?@TM`xlzGl&3-drFYlU?J?^B&i2sew&O{_;eaw4zJHHkVi!jY@% zZuUdC!tJ@3L92+UXm9AML<0z|j-n{gOB6$VkwsA_S_5owDr!NVsv~DURi&yPJN_pI zGH`d0fCZnjNrm`2m4wSzt{mjEfit_Psm?KbXpBj2mz7mK`gJoAmvv(wn&S94!=nr+ z`i^Ib>c*}hu7Qg}p_wg>hfQ2Yrdoxal9cEEpxLMEiv1`G3Syf!HRg@urZNf& z%n|NUy=Gew-|;OiD@$VXr@wRS<41dMLx>Hm+3Isk*Lh3Lp-fjIjf%Q_`8xQ8VTCwc z>4*xkk7Sp?*VU1e54QEr?@F`)4+VaowiC0-p}VaXRE5T%``}SD7r0%o8H1^D#F}gC z>D}(S_h1PdOD<}lh@96SsA_k005Qa;J2|-E9WTX^r?~Yn-`A#D1rrt`54wx=SFYKID(8vugvG%hwu1VxGS0ZU zhD%s$mffDtMNxe?Y+o1!n_16r%QTtuDG1AbQK);>XH+SiFMGL3JJciYHKT!=oFh#W z43!%48Bh*DRI&QN8;tT8L%D}wKXem6=EEMzC}#%e7#Sa*1#(wdAp#bq@%Q^!GRe&) z^}_bNe}=7EbYvio9vrfkEU!jDm+MfiYU4K+1xI9W`QZ5EJ;Ff^=atjO5MciI*pC_E zm7SLw2TYG0J63MBVu7?dGlhDKpCg!T02J?7TOSH@Po7+Gc~tO-vV5J=24(AW|wq1N4u*GgQJS}jWTeWHWPH;M3 zyf?GxFi=$dK92c#;rxb?XvUn0Lamahtbtr_WNa)ItI_a|WBHaWTWMhq^h4zRv)(tA!yoE@s61PUW-N%h z2Zmc%DQ#xkt{am08zD?BD5XOMQj(H-sCd1Dx;Ygwn4^UHjwP_McsRFgiml;xWcs4* z(jgg*yeb4qC)=Wy1XqPDgvBl6el;`ieuXplOs8`gRQwi%YjjRbIeTom_QrtSAYi}f zi2ou=)4a9Ej?W_ZPA(OJicp1LU*dSY4%N8c5No_22YmVt^~R3EZE^1eqxecb9(CI& zA*bV+x$9%V;{|+S>Rw7bHDj0Ve9Dq#A9 zLTUQ5WieJ9)6*0xZ?s(xWL8NqlZ&nt=jNiFU{B^L;eQC#E(4pq4jiTz|D(}^9FFb=Qu?0&|M!XNoRVXQ3QLDhPEiPS2+4R zQ17Wnq{0@>Kyl1)Sf7D@Iw$O>qT`ca1SUj;g{fG#-umq2L%A!TA$Q5eNl)0~S5z|w zqT@l3`5loo%EjyqC()1j-ISW*mB5I2*Bv+qYvq{4Mj7^<#dspt*WugoEoyQqKQD59 zHYxKCc8S*T_gzP6J9?03$iO|m7E#Tg%T>QnclMtb6qDR*Uv#`SE|A4B4|tjS@q+(L zsXC^|%(bhjsa701Rb0)E(cO7f+fxj=BezA`@J@f$*D~MMd2!~)DWNcR_HAZQ_GoEI z248BLnbgv86<<4jO?kB&$3}@tPtic1C!Ov+rL&iVejJUuWd`=JI6p_@IN_-Q(WJrK ztVe#uA+GW0^Vdn*Jv$w=a^KyHlAt9_xr{k|L++GXxl#?~sM|a^MLbyyuJUp6hi?N# zA=!(fYg?f6t)wKC<$j|VL2XHb65;cnPX|(wPqd7X90XRGuky+VN%(s7KW*`4 z474a2nrM8+5v&G)>x^UI_f)$}mhw6}JM0mNRLj2j*?9Huss>uTwp0B3DoH%*l+gES zcamWitZB88G_9a1h!i-shMzL<*eIdH{xQLE!H`L#_B#MdC)=rEFIRFapD<9jYo}_j zG>kfZT^#4x?jlutX3SneXdCxVlkcDPP3}p&s3{JR*eGBWUuN{Mmi!+n%#=;Qvzb2q z?8;=}i2(o2dOFc|JNsrdD8rVUmqje|?UxxFXf#81T8$dW-f^_A?IKHu^zjDp>~t`5 zS+q~Y39ebAgFz=*Jl7Pxv=Q$j9@+4FH>^Qt&jyIw-bDHy6&ve7Wi4q2Vp>a*fyaKn zeo&AAM3XnRJm7~R>`E=j1E0Q5vO$b2u;_Ki$opr-$MqN~CD-bb)n@EEb; zT##e-#6!jmwZ=#VfiE#6vQoV6tR4Sft~w`W<|n)ApUn_x;j&NfB1eM8#K1d1px^s zy}Y$6M+0^+lV_s;gE3n0Wk~Sk1e0(R!d2=7Mmd&UpNS?{DhixuUXnLz3WvNmq4VTz z#JAussn|0}DwDc4A}AnjX^mD`zA>l*zwC6s63R&7LG#~jAfFDvPsPQ=3OXL&zrUV1 z2K=I!+McXuswU?%k6zjXaZkY7pBeze# zzCGf<`5&Y;<7K$5=Iq0fu~<2Aw>F2d{3IB2txQ}@E(I1d8D4$~-9Lt=q#Ue@mVl>dGF)vHca7L}daT3Xc}ZclpmG|%YaVUU47u*8MeW>VZSIV#a12C9VE|} zZ}C=xF*HL3HFYX({lOGKsSfD=CWjzi&t5JjWdrF;fdeObs7X-^x&x~cHkrY9M%mRp zE~_bPzk^9HSpc)CSO@$Y8tB^Om@#=#0v})i*kL}iBdu(d361z#qfEr9KxP&qDdGN{ z<(HlR7a>JV@UKbC_;F#$g6;84)HNxW@~k-=R2vu7BE4=iGmlP%|IS1PMV!mynQ8b} zhQ>tn4*>^0+K_+v+8MT`DF7`i_#N12JC%G!4UQZ+QvQBuVvW4K8L-c;6td7sVRi^h z0Ap}>Kr_-qhLRC!w~J&?1fhc?SM@J$&6`G^LVgM{ff)?_FDvzoSgBImw+3@4Ns>-) zlTO^!zjj++`KL(?V#HVwed=Md2=t8vuj+S5vJXjH5sV@T@4ID=qG`wv(Q}-vU0A$c zTB#z1x{+*ZghwRH+v|c+YruVSi#&Vw0`YUr#s?f|d1Q$?KcMv&0uyqFTdcs)BhbwL zUHq=$F9EJOASG5rqd;okj=1&J1PmRUSpCz#7z1(+PHpaKg>^?4MA_F>6l%HuF~5hu zatQqa;||ZfJ52~bXg&f)9lB7j{d3iwGju=b(0U~Bau5*-?J3*-Hml6803mzCCiwP3XLthEt4;CouCG1h~2wY2+q#+Yf z?3z*r$)yHHf_*0gWiS=wU}wl-9ey}+h6F+oU)P=hG^q`1Rf`0kR_I~BD|%;DDwSR6 zVhE`%fH%`a5Q#p?7#(cII&Aq!U=2ybgGdg53mFjf99cboP7C%^RiY9-VlqH#I2Twa z_SObORBUyWF=JrFNXZRW-PdU!0W$mh8Vf4({PQ)Xq#0P)pZH+a<6{1{XW4)#gwT^p zf|D^6#v#m{S))W?A$+_HWF>tFtLyk!P1aT3BY0qA6Vh&^st2fah%LifL*tAy|J>8w zRr6_EVWxOdeM1X_a74wzoM}Ay-k%e#5rtZt;C7aeRet9Ez2CPGSB^q=9WbKUT=W|C zP7M`*Ry4(}CPjxvMAYMZ#Oq&G#@R%T$c{Rf;+TkE?0{+Cd4(vLlB3n``ks{6GO%TJ zNL3IXHXS%~2B47|KO~Cv^Q0ao6;F?46SPvIGNa3gwiq2dFu7*|yM5O#UWM|XW!05+ z+u=GQYkSrkcu$~uLo6n}!WUqVr1K0r-%U~hMDTcKE(2v}JMc1@>;S(l*OGaY{DDx2 zHydxdwvJI{M>UXoZT<+vq`xof@_&86;JXhe-8&bZhTShwa=S~l`aE|{@eANf)I!81 zqn|G8_o8jk2>69~vjAXHBOz z%FEXqum}3yf=7;9XMY|gQ@)(&n3%Mqt$t=GN4E_hp-_g?jy5e0q0D7Q7)stC}IbB~TNr4Z3yktDVkfJj12?=cnkWFBEycrC= z*&OL3Hy1)0)9D(oz2aXFK%s8n_@`oabR}(?6=(%t!l=79q@o#QWG-bZCt z@q!Z4JKQ|UZZ`6dXCK5f?;~*@ri~}7tgxjn!0|eOjB*1_8jK@qwManAk-cNM|$0IXZm z40=--!4!&Teh$5S{n}KREA+#@UD1QZGyY^>zuElt9MUM4q(!+ez*d$+FXRBC&l=x= z{U{5;JClTG|E}m0PaZ7c2+lqB^?)CyPC@GJ+^!)Mk%%3|e@m%MKn{qp8J|3!J3hf< za5Yyyd|d(%5R2K%AeQ#~GKeVnC*y1i-zTs0yRJDw%e6IEZj~^uzwb?Vzxs~AHc4e4 zMz06<{cPbzoBtHC*q3cKM*u*Mq{%#!M?gapP|w|ay#2{6z@n4Kw*>7ZZ)xB~Dnkhg zyJcQiRmm-3O0s}c4_@Y@`5O-Kf`e@Qj;)jhmlzIJMEt#s32;ltm9S6i0DE#Tsi?wM znTUqZ{&aCW0ejGv@%|lx@tXj95F@SLlGwKkpD?f?1>xAsH*e~J_ltLd!bj2t$djp& ztP(}Y15oWCS;QDh18exF-}ucVfhsKa)e3D2Rm9!H1GUo)37e07Iq!=dfUI*pYODxK zccL7U3CfxC`#W2q_n7Y4448qGLtxfW4w1pp0$CZ9AgKdY9U}x^>}@K{At>G>avt_Q zo|qqdanMa%?samb9VecyLEnvM8);9=l#2c=48YB1mMZ|Km$krL^N+fZ@+4WBU9tst5 zD2}7`91#Rh27^HYDHBta(Voca}yUXuZa?KRZWyur-eoC>>>0nu$(ewqZS9*1zQ zJmgVoYe0^X@RRnp7!pW;H5d})hIdJS3?BNu8b5egq!hKS)8q2tOV?@Oh}T(#HHh+t zJEGo=Ly~wNQldG7o{D47S4KD>O`tEpY}1PXbKoDkK}f+=HS1rgI)pX62Ae*p2vFLr zE8o1y4?lLEId;#kXcXDy@hEeR^3ROO)HnYp1Ms@?uOq(Zh4Os<%Yx)3f#2Ircd1u= zA?ZCrqBf+O!)5dPAY!@`om%3LUYuwhswb#125lLj-IzuEL`J8lqSwsNiN-DzAD)I6 zk@H7e3L|ZL{Nc|7cOKrH-8(8o@){}0!hz>af&+yKQ?m&cd*zo&iUrIGAPUaN?rJd%4r0Lna2$tMn6T@Uo+j73;%tMh$|DR$Fp{X$%^uPNeXlrIikEtWlqBQ%~ys z$cr<5MkTbH~}4fgO}Uo@VbKe_?DDr4H%;lqZJmLEss9{PF}Jr8j1G{YivGBr8_o z*3#ClLA*QmrBW&rJ}YXl(0e3>Y@f9@kFAi`XLYG2IgWK#mu--gkf1?>fxV8LFO@pj z=A}!Q<`p9TM^LKUgVPC}$?HIeo@I4*b{+(5(11>@Ym*Eph612e_o8Z3e- z%+te%*4fT2LlaJjOo~9YO_n9UH~{94w(iYl=(4SE`~lYEeVAvpofLx{SUK>LGeBa< zn}h@DLvi2$l93Ha&q3d)J2`9Keb^?Tpr%D{U@>r#zuV081sv>to2;#?n?)XcvVmr+ z-$X*ovCjg);$#je;XooeuSTMh4d=8}EK8A~D#&X{jin-sRAA6I_f~Dbk}&vIGzLnB z8tgT~w?hPB)@7r}X}hj&?jXdKTy5hBoyW{E|J3!8EH?gYf0nK~9P`44m(=y@UfOWI zd(mk3YYm)e)QgFJ3J#|*BCgC8Xz(~LPCkrqbgCjW9 zu}R=KLz0WmZ=Z~@02>p=`P89|;e?9icT(45WoF|BLex;wH`eg%xCSdlIu*Yhd2kV! zw$5LPp~C4~qm;CetZdb%F&{aM)CPL{zU$~xFmr@IYSS>o+mj1=)yVv zIL%JKb7hH2Sp##qCrE}7C}G%qLCxn4;Ih;pGuZoh_FM|p*QMoQy2E@MjCn+EPx(o*RiyzJU^D5~8LHCbEm2IHV^dH=2+2=BdZ}px*S=b5r z9I06Fu6J0FvkXxH#9J^y!m{D+K8Vs7jBRH;nvQLiH-aswaE>4xDhZf5bCB1FWuOjsOY#GKDl`o#rD#h&$Z|)+!gxsg?8s0V86DTp``de>S0*@oGL=qfs7RRaU>G;2kDk% z>C@8_#eJFCk85B6#hYW|;xYi&SOrWqwvK`vAucI-J5I;Y5~(j(bK1ls{O}KW<%6_$ zJvxL)1PmycWp3tc{P-wbw-bVE87I*1NWBr}fG|hMZf}$d7(%^*PU&N(X!SNoTSxa` zo`8si)^6S}O@ALWWh|m2KmX5Thl>Im(o3Xdt3RbT?V*b1lCYO_L+|_ao&;h4#ceuz z`wT#DOrDDUQx72!391cZyy3iJv!>&b2I%Xh{5b6iE?ox5$v4*!30u|!TyefE70;L!E|13hhN!>+IZUjdSU9SO|A#YwP&Io`S zImxD@gEW$V;beAWQ$-D8clu!X7fC8JP@+X5QO6f%0jg*(3hu<+BHrHj4onj$zKs%_ zi|TpZa`V$Ezwj@l2qr$eJqW$$KfbnP5vfK`p#ma8nO^&1z#wx!9R}hCp(g56SNFF6 zJa^^e+J8$SjtL278y%Qbe&&6hi&=>jnipaLaPlsYR{61^RvV7({i)WZJPX~l`>yS( zy9-=PRYZm>P)P9VsVU!J_JMsf@v=!9v^6yi2q56GapOiDKH4Ied`tb!Bd3VbI`*g~ zc1&Q?5r_6Cr;9N2>;#Z`17rM-Mg^WvK7y3MW7t;UL0G+NZ-S2`73U-U@Av$QahBI1 zyC#{=1~Utb40L!>3kpbfOXElJ4CFWyZS64UY52;wDC<_KGpHyT7#XDyiUlMu-h#up z6D^}Xk6{}L+lqjhxITF*W)?2W-32NR2DU6DAnjacMwvd~-)~`0)`QDY*zCe|dgrPI ztlhBl^`JZ;#(|erOyHH-dkP{%H_^91utVb?{>9GmNYbdxB>{c!q=~7a&26sne)u0h z`*V@6<6knV3j2}50xRtI0N<4q+8%rnW%g>HAcZmpb5Ol6Lg^;xo*9U#;b>8ECsU}Q zwQ^4FoZ%YeCLf+5QHQEf;uRjUrqsQ}!@8l748V=t4#FjZatpCk2L=F8koq~H<6yS} zETkme9P!?q=Vlk2!a%V-2|${7QG(=UApy_=0QWI>EqBmP=UHzMG%B*e&bhE zlP{2Mz7Y;N(WiGcuo@XuU~@6;8pelDf+?^l4O9(`ak$mu-N8BYgv$(@^A14-p<;Ct1jq@5gBoB& z#A-Dp_Q5#M|7WvXR6g)u0YOz8KMDCZ)ZlesanhbH5RJ`DVDe3S|a?CA~_dIqA1I>z5~>R;zTcy4sH(& z_cTHC*Ds2-N*($}3HMI?84dtKr>2L?1BF*_+_(!7Qym^LNg+>{wTIa0x1;Cu59eMrGi|3ayW z`*x9^Si=zb{S~Rqk&619z?sEEo>b7*Iaf8q#fx<~G0DZ#7Rr=Nz z)Fiw>?mTid;PAOXg~JL#dDa4P=t5=r|KaO9z;bTizi*)k5tXDuWK<$VLkbZ|Dh*8) z($vtPP-J8zqsT~ubZhT{R4Sp8b}5P|l@v-z<^7y@dGtHp<9+Yr`1}7nk0t4n7xx1`xO=;Dwo8aIaNC{d_|$58pu#fEfs=(*Ek<|rOB!ZabE zc~UkgY0++ryyO{gkTW1uS&hqwh=6{9pde<~bM_=pZcWC0VA zOy_>lw}P2U09v*T83uD6v9v&Ps5#|52C7*Vyv`*RRh^!kH$GJTsTJ ziWf>m%;F7d!kB0Gs^!P5$T3~p6a+*+H?kBf8hd&y{Y^)KM9ZI<{bNoWd*z((Dr=U5 za@(jZy6=am6ZW4dAWy7}%z^KP0&u-PHnA`IOAok1tP&6u6tiW4c?eH>5&(+2^n3S| zv7MF=!$dkr#Mn;24?-LEfS%b7m}n=VVc^Cu>LWm>46*kq$W}eIVIGyN?Bw0Y*!dw( z2*B~%PR|bZ1uYtpcidU~fS58-QKNQFdh*Wjvk>GD0f_AnlCvJ`Z(T%l!9mOg*c{?Z zmb_SfL)Q)W5~jF{{w z7t~gl7Ds0^NfIUXz7)@Z-3M^G*wS6`N?1}-8WXi+K@Aeg!Q#d&%;|3k;l$#lePBfA zg#bf32uHw-9cx2T=?E{4?nHV!X~}`InZZE82xJXqa2x~wBwAtMAzmoxk0s4-0>F{B zp}W;EGYcIa%d+$kBezif_Ts`s+aCb`wNOb|X9jX%q`}tIHSJv`PzmQD4I9_SD0RR# z7=a+D^i`9fg&?I#<6W^si}$k-{Tr_ zD5_JN(N(!zt#0?dfmJJ|0m8ns{K3vefeeg!UCl&%kTSQxH79VTEG6dZj3>G%P)ol2Z0jAVi4Ng66_i4>{s5Z4St7O(l%s8yjB+{3K% z8G;v3eXPY_xlA#u7t@$3qkd~dh0I~&wP(h9{kirzu|i7v)hJ1hssCsYs1ro+Yigsm za2)9B1D`UqL|R%0W`y!>%QEi)KG1#Gg6ipx@T6ocdO&O12j`grR=*zT{Q{M9@8~Ce z1W~czW>NJt3~|L1!P@R$5(>}(+Sy>(CwhcXq!CbBdl3Pk1wUvmsD>UVv+L<3)ZE14 z1~fu=*+K2t8 z8-xOFo7ZS|#NW06)IwL&X7Bm?)}2EiCPJ_u;=WNc;v>Kj^vZ8QRsv7mQ-$RphiO`$ zT64jRZ#-8K#6b2OOo=$(^P_R@8KM`_!dOE?nYG-%4(V%NYrOxbZOZ{ENUOTx%0;fi z(GOm`Ba4&o@7k|8Tzz8RC%!XaVw8OSck-0=%=NnrPFV5(#R=GezX0(V)4ZWExc)#I zi~lfkbxp;r63RO7nF463FYOy_*^Cq{B5+sl8W)lpFR_ifExf}@=%NFA02x?PH;9eG znhk=CB(LMk*+Cu*I)l;Ji^~$K(J{ddVR38fI5xr}fHKN%+(+N}u{cO+A~7YUX5b9` zj4@h+D9`Z>Y466{cY1dSL1J%hWczfUSzR}OL+I|2O)vcocm`rD!5+)~sMnmvbHgM)}zcnwXA*wLUe zD11p5f!QLMg$1S1MBGuw;_@&Qoz=Gon~8YV7-a1bGgHBbCHIdB?vsAts1CQ#O4p4Ha zIG~}@g!|#lCDat-hP*{43=;0|PtC*{3=AxJYZC=ADws!ad#J9$Ghv;nu!hcq(BB!=J5JMxOw2vWKJ8_HiG0I-Zer zUt)UYgSvfLncP^+hFHu*NOdEGg*cQ%32}=OAXLc+XFm$69+HkL$5ks-uI(eW>}oei zgi$nZ!qY({!j;RH&1SbWH#_xB^HL0lo07!|BlPw-}bl46I7>^T9U$Ca`n zLScBSqsO?fb|VZ6@mVzKwlK?n>Y?qW;fC2_a+5J!BqR!kgklH! zg!B4tfF2-kruH<&wzf>^eWqh`(JT;9S!LB`cm!2cPVxFxv(`*zWG@3BSXcAOX$dL? zP_TxwwWTq-TOVIz)5uCub&Ap#@dI`-^6^^mzf1q_8re&NjAMRxf2~E_idVxmfB$<# znm`CeTR+@`#P_fqSR+XT`(3w(J+_= zVylxZbd`X#2JNWEUPK?g#sOm9tOwr@#0_mz*}Meu-aCATP43)bMNgok4?)Y|b)>tg zZrj#Fx(nD#D)3EX3`Whz8EOMTG^>fjsGngJOHbapaA!!D-vig!KZIE^#M)qLy zXB^%aGoHn!62cm9=1qlj`1oxtS3yF8HX2~;UL`xymp1`c(7(2i$)4T$&GaFw(-it? zx>K0rzcugW+4R!(?>o@g!yBOD96GEN;%u2zsG}{cd8N)w4kLdB8@*oOC@w<-hasa$ zD7%$AN7hN1jEhKCAs2pr{qk8%F(9^hL3d$2Gm%2!F>Zaom%hG+;VLXrU>F4Y1{M|< z2SA*(YkbJ+B%H(jJv;GG3CS_eI`%g87z!AWpRHKQJE6E>DY7EF3rkwqSlS@QSB%-O zX=Z@E1A6t!)Ksid$msAL#=a8dXANS~K#g{i%gwe{OG*y{M4Y?U0MkKkNx{5fK@bYs zOz52p#hL*skgf!ToW$Q$V&4=co8uppt4cfypC`RBQ+*i~@gnwYg5D%x4*ynZ6VID58RkaZ$D&pg2Xm536C&i zZC8L!(hQ6f-j_84W5x}G=eEm47-N7JNwd7cNCJ!adIyvvssQjZ-XvEzdSR>AhI5B~ z^{;jDwC<{M7EDX{ig%?h?6gV*HS@!_UY7fveL4K2N(63%e(8K^E4B}0X zgVvdz^bQ(i5?BDYSVd~ANu#=-Od>>Dg#pl+D*tib5p=j1%CnJ=_W045ki6{8-M@;h z7>oeQVOF?}xrrk0H^hD1gHtAOSDid)k1mB?~QdM=>Ui2q6%J>Y zH^z{Xo504!JP(_mK=Wku`Ha6^fV(#xm9Fa@dUXX6#@BT!r7>9>^?9 za|3$Hyb3xfk2)X;uSW_(QrCpj@O2I9uw-;UP@@1x>jX5yl8#8yVH`jkQv)0I7tA2uXx9 zjLzs;cq#;EW0tRLu7(j56@M0ZOvPv5obiO++hfv_QQuB&2xv$PAq|<-p3{W=W0cl8 zDB6|?1q4Dfh>#_pd=Zq#csJfG2p)L=IHHlvJuCCaD|vJU%0&h!s#Gixy0PQhK30p$ z3u+oWbHp(Z|Fr(K{&0_D#0<ha+s)$X)cf_Ub8bYGKxm$f9ot&A}=wtfd^#; z|NQFL#ehol5#JalGUl$V0=-LLPd%bMm5=ijX@7SXg4VfK$-s)}&J#h!+O%gYhie z{t>8ta`J+KM$(BPuKBX?+o%5!{V|~>z@N6_FIcW|?jca;c2cnd{Wfmh--MD%0}>ep zUUs<7Ui^3sXbU7Mh7e4M5E-d<=rT&tCI!GW4}43bx*h?d%^7z{t2?&Quk9LN9J{07 zO7@UrmeX1^NRsuBOHlI74PdQRdIcfRz#7ZBCskmSu24msgqVOY>p*Z(!Wz4Ctlw%o zgvj`Q0=~*Z+dcHn&(jjkxS+#@M{ok~A-`oaXuje+`eI7F4jD#pIo+_6T(qbF&Dz7F zLBU_6d4Yez!6E>L%K*o;YZO*90}+3{gg%CJaUVpv{F>V?NFPri)rey z_%ldwv@!LL@MmnJ__Ln%Xy7o}grZ75i>C5PfepZkc{xv!vI9JPd&-;z++1gVNnrJ~ ztb+D~?yUguwfEMtB}+;HT?27nG6GA$S!K5hR|V7=0Y@X84SfkqC1Hkm4(m6`5A4Y3 zg)+E8dmQ5rH!umbcS}tk??Ti-1&X%V+ZLyc3K*j@f4;F1kax8WZxA2%wF-@QH-Hur z2J6WPdiY&{%aRm}ZEqi?MawxaK8%fwLYZQk!-l#kZIOwtf}}(6jhfP;zs{NrT7FbM zXh={x`Z~qz@y$cGfk!vqvHsu~xh;1zfu`j9akN5cTLjQgfD#5BJbrhv3(f&g#<}$w zaCjBc7_lnNf5PBm1yrd2cf+4p*o=vJ3}oFg6uc$+DFv^=xl;zEUFUqmn- zE}q_yQFjwcB|I3GRz_5G7DQPU`~Hct1~4DAUbaYRm@&rd;a-Oy%J@A|k|Q7H z<^MsVTPr%WNXQ-)^u*YL_?ik$^@yFM+_Q%I$;q;T3S!nZ)8Mn!WvvfVgh)zEa*Z4cc5L#7*pT5je*1# zQ4>BH8SZ2x9E&H9%HM;FObL!+nFA=(2F=onkz)UkI(~e3=RfLrdC9PNGc|)=7Fq&q zNgM(dD!^zk-j;iGW=4SAU%Y5hMA=cu0dTBV!QwzH2DqySkgCjqJWU8w4;)D4pv%lj zB0=&R0HYaY?eDA~k#U(&MBJ%dWPrN4YWS%2+gJl3s~XWT!zf((Y`dNHFV)UvR!?G7 zjEBkF`sx=_T>br2>>AC&{D}#Xh>psMCs$1++8wdQO$o3O)XS3#b+v6EWx^|TWse10 zk5Q8kraiK%0Ga?OT44dBeCjvUa+hD<_pOwozro=;MD6}pu9)9?R6b@7L!*-D5q!(U zv;IA75q?O0{olhj4qYTsX3*u5w9cUu|Dv^ae{f8hz-m9&@fvY~* zP0{PTQ$@xJ{fq!hNVtHx@OafJE?Q$N>3B) z0jTHwU~eUs^tW@m@N1Cw?YiKNiu0|dkDA`0Otcm`bx_kj~Ee*mjw zL2yZgityn)Qz1s6wA05e0BGN=i)zU7%py}Q%K9vcu^M`)BSOfZ0tbK&HSKHqG87#m z0^GYl&r7Q_K6;$#h8ddj!q8=#gla4%o~9jzOb2sW`$vyI=+4T)3%h_C3rxY-uJG&Q zRcfz~uYkgp{DJt=btCYcThyj7L->@O&;52~V{XJ8@ZhjW=gxndBkTTGW4r%OWb2gj zBcxGN!=-Gn-uvM*O(Ac})Z_Tm&1fnAqC_G6pmz&| zJ=7r6^3|6=)W!0tm}JpomsW#OVd0KD(MTAs8$hs09N735z(xZ>y%52e@%HCNoh)n- zgi+vZ)|d&X(h3iZ)%jB@$U~Q?vZF-b;}T|*qdhus@2+Y7Hj$|cd2bmNE#Px{{ZEdO z@&-1~VbRoEImvmLi(xaPO6p7xz6ZP%X{T8a9vIP4?OL1HfQZJ5G9q8jFV0LZuc~$t-=Sg1NqK{&wzy3$yMb zA0bcy#q|xoa~3rKBL#~Zu=d8}UiS>b`hH-^!y4U@6WG2!J(L7+>`F(QoibKoDQ*vv z(*a;4)eaD*ihfL*j7YN>z)}zmo641XpzsCaPyz4pUj3g<{*-w%Ov#I$MxTlq`91_a zgy5Oq$&G`|ki=n;jO@o4S=%9jPAjIzjp%v+YIGRzQt$y$8b^L?Cq@Dw2C5ObpPf4t z!buf{lO4bz41_9rh@+ z67aS=g*W7khdP}mU`L3@C@-QCwlUx7Ms1~wb5NGF&))H!QL#AZe-7hsqYvYu4n!h7 z)hy>_XskJT_svCMkV zi%oGYHv3uMl(f&liT*&EkJTp~!BBm2rfD@{?R9|*Mh_U(CF;j;KFTC znL&=E74HTlL3}XUieZo;pcR^o|Mdo|Vw&|j40+F?qA^{xkwBAAK^mYhsWr@U*)zXA zn>_+z!4(D-64Cn<`%(!MPBp8JLpz)tfm@m zkiAKvDmV>fpu&+RbOMG{_Av2l!!u?*?_A=2z;h4lcj1fmb`wNODGa2TfWgY zLp@y0yPddTcvOXwt^r;g;WMO4a4a#^|26m8DZk4lTR~RoJpxED> zq*S6XrrOl13GEGZS%axC@p~2&oCB9e$u0z!u2{JeCdi&j z`1+t}7okia>|LpL!ZC489tCxQSR8w*moO?^97(kh;KEXiO#Hk=;f;*84EcXuX5>wD znU`;{LVK2PE9<(jfQBoS>5&gpdUQQsuq1+6Pl}VRGg&b}(zw9Ej@93SO8F1hhgXPj zef-sq-k;oKS=xWtOjt?;Y5KZdErZ{){ebqXPDLTx2}+OxKmI#Q;V&=6Sz53M=^B+q zqt?6hMxaK-vQwPs5#$CCvjl*;epvp$8x@C)zkAksz%P?EV*4#>2S^Mi%itkkkd7+r0qha1WD+`+uVO1%;zxgXFr$_yjc6TQqi{a*J> zQauv9NqxzwUK5;M*CdT2F+S!W;i$s&i~}^_i7>|RlkfmSF*D+{sGqbwQ?kbvDncYA zT@v=4UQy9baXNoL6=MIh)}0eZp7q)%U>9#bPn4kfi6!{QM#GH&feLV~Az+v&YG)9m zs`g>$9>!q@N}V}}9{#lkOG`a)7^DRx7{~1-;qryiX5if4LlY$BzeDp~=B-|+hM5aZ z;;L5@xT2sH#NyQrfKqZMv`2?weGV3 zh$o0aD?na~0F#Z^5XP)$a(L6!wOwAMXgJgfwni1DHLknY_3r?S1@>y_$5@8-99rD+ zf)@ij(-_P_;RDS91^EXx7%AS>`&!>yylX&m*56O|XbGp%6-l0JtYY43&g}Oxjw6Xa?l#ky2SxiCGPppr%5zp3Jr{5kBW!hQg$FGcN|jFjCc3PV+HEh5meVc(V?%A)Bqa!@CXeERP8`Lk9i z{hnVBl|5z!R}sU@H!gVsB+;w1B){iBQTw*KsbqeQyP5VDq=-r~J~qjkX6dJQzH5egJzt^)LG9-^zgo z*!{DTX2P6FvRJ@g?l5SP3Jn*LJ}Em20vBefOLWN-$w0`=_)eq3hVzxSQT4%El9`UE}zjiY%OPJEI84?Nuc?p6+j%VJ|3;@)28 zi*sFa0MD>1>P>jM%Wi|mqtOH152`x!py*>-pC9H#h*<>k6r&7pvNFK3?ST)M6G(wA z%i_|#-*^uF+8-q2xPjbc^sQk=MuAj^2Os;fjP`Rgu7G6%F6WOPK3OGM+n}UG4>z^P z*^Q~NB!Zqi3Z%_OgOQp7VGP(5GWd`yv;M$=y;Bgk{R5uUgU6MWP>PUw$&mH~VT5`@ z_G;|&x+DC5HuZoarSV^S7ki@#JYjOKUjiwI1{#WYj=|OKlww^8-6G6;+I!}8?I5cu zdLtM39#pFhQ>??WKBC}PgW~@vDXb7ehgM^WAXh=O+m{+msAfJHm_i@eJR))c7y4Bi zmV^~6w@&hF(6W-Tmg8WU=gBYiwY>pLq*?^)Ww3rpxl;HqbVEs9GXptI%yRUaQHBC* zgs==#RDj9QL5RFqeR7kq^#0(;!0cC&-XE<1KqkLjzxH&ZB~a*jo8Zd@(5-`L*5N?B zM!&5)c3h`a%3q%A<`fqf>!8VmO(WZ$T<10i`5xpyr0A$3 z7ywzG3)pLN%}z4fgz$Kv#>d8@Kz-gHFmI9Qq7oq+-Y$$t0~x_8{BR^sK{PQ^(*Rls zyMLYswC-xJw3#w$6GX2Rfy^S{AyfpHZG_+*W_ii`^XJZy2o@8UoJBFtXmAT)0^EXz z8nEZr41CXsU?tIn$So+4m5$yNR%Im7-%-raC_bICMVnH#Y4;C*m?@)sBBYP9HqSR1 zzz^2*(L}%u&C>kKI^A0qQU^2Bnt{KvMlIu#Y`2#p9TmcoQWGmCSl@ zd9Xj(gjr91se*zr(RH{XyRG{VmeE51Kw`K;X(Wb*DcKvIA7y;54)8)v@;1lzC;v+> z#x(bJEFFFe__bGk@$^bx(;EEY3ad3I%ub02vf*^MUIbM;FtAq20@?_}aYXs7qzHun zkzgutmc)5PuVb!cRqVwnHuBJQ)9NnJk2at&G)1Pm5V~V@vc4q_FcbhJFPZ?IG3kcr?a4w*AP)ddbr^DDomm$@05PP73B^dS zNLp6OWdcIBkn^%`sBh1&!J)t&ie3RzL!MlW?o)Vl=?YHz5{LoTL^&*SM`@ErFp9KQ zSsqDQJ&oW_0KzyD_UYATu77Q_#lj`;tZt}}VQfWcU)l3(TN5ROumAVxy&k0_MXQ&y4}d-jH_m<#p51_aR?s@DN~^E9FTuKB#s4IQ)%Sm5kunv zGE<4kquz&T+yBy%DWGQxV?Y+FUR(}N?o>zMIhqsH(^oCW>L7D_b{5h=NgZ2KUP_>=gM z+`9ZCK_ZleS-alQpVDf2hfbumj*++CB~H03`Av- zp@k)V7|&=1E)C^Sz90{pq=t4NmB2U$xFJL!z>VdT#J_MGFhWzklW0Wraa3bjL0H*} z6psw$S}8y-^aRtA-a%4KC>)Va5XT>X+6KxP$e<)}7!PV8i`+3*NDtdTU(xb+D;Ngb z{91yQ6r9j0y@|+!tVg3HWyjfaxbg@(9{G65<(PqE9r;w)>z!Z~|7AuKyAGlJw_QV7 zZC~d3>pX3Aptl9lD#I~cG|^K=9Dv+eOl2Lwiiy6ojA8sA=DWHU0jozcO`Q?>fdm-o z9`G2$n)jNrjgSYK^aKJM){lU7L_BXBJMK_GqTO-?xBI{!um@5Kh33I@jGT0z3)!1S z>JAu?Q1G5L&1mT;bmkq$SdC?*zlT6SHTMGo14%^!{jv8x#<5~FSOo87{%XF}50tR8 zPazejD(R|Oaah3TYldRSR-)0d!TEd>4ehHt3bT`NCuRh?AuB+8P|>f^*md+APF&0h z1W;oir=nlzhLXD-gq#klDydk%UTo-0^Y#4oOV;aQF~-|>Vq0GvxVUJn>E^_`&_xEK z-`26ryR4_Ju_>ME^YDk?MHE4_7@XMrV3iM%1inrHE*NQP{III_-*TLsm7yG zSZB!3t)SQdjt$d@l+O(6k7T{WK6rbS1RbCTwg+i!L`TZQ`7KC>(X#{FZq=bK(>f}= z6!Ww0Acu7zl6xsOjK!(o_>^Dg){3iUNN7R%Vc5paN*4SS-3~(%0nY(L@^zGv1+hOM z!+>1PG{5C^c^3&zfJx!*1~?di9l^mc)pI{`HEz=5rr@#ZjQfX-xqR({B<@tQF%Mk3 zmk@LHlVfE^FA$fJk5}8rQJL&*aT*IEXAiTxGh?rxt266>D4l&|4U=S z;5>6ILkHq=y3>Kc9LSSQ<+B%J(E(|bJKCR&%|*ANd55F*ktE21YvdFTLP@&gIdqs} z2jG-Z%Rd4w1wm{R#t~`hO|7WXpoRXmXMS_yKyGG|^SbeG8 z#rP%OOmGB#EK7*?*b5n3%7|w`mq{#psbzAR0*)v2KLi!vc!Gk=M+eq=5MDMoXIb{s z&)7(M=&mC(qWj@HrC0^XZtTO`sb>Rq@yW^YL1pICfUxM zUhaD}R>mRF{{Bi06^!%aH)`^p5)l*o)3ReOkEWGOWa1X>eQtd#1uuu46BAn^5D<1n zV(Yb@IK_Q0S}%PNy!PR?Zn)<&`K5=AxkNMm^fWm=Z%XW&{9o88e@abd%DnaWq^<(zZMty#@6jdm#IvDVU_+BrOK|blUpc zei-^TqOXnSZ)s@}M+`D#C##0(ADl*ss+R+_1q@eBxJ`V7@9T#R(Uz9%pbdYvwV7nhf+YgECRN*i%_6gA< zC-jZp@sV}ArK`pSoQJXvBO^gMVPRo`F2kHS-PT0d@ybZ^>5TssmjDigrh zsjYf@BUh{nT7uzEuh*!_l9`2_ynPWvDbp!)LA1qLLJQDT8)2-XiwST~mJQ&2q;+W* z5s^Xh#bjR~-JqDXbfwagp$q62aAu+FJeL6_N+dj$BeP|%4fOXPnwoSB3=E{;VZMtB zDiF=)6-g%*F*6jU>-x3c%8(*rv49XeLJR15NApTGc-4GIiMg%Upr40B>N&EAV82XF z!HcFFFDTpwp(b>aAkKwid(aT}z*fnOj*d15ER=N}aWM#4*pHikB6lOQ7Z+=UVD#53 zf>DEmBl69Y(1(V&&iVLu4)e8gXf%|Ns zpY8u}FdBD!e>n8(P*c1Fa@3D+xXg?lzwx2T^s3@3@dM#zZwIP(r1K5BpHmJ$@!f~> z<@|tH`K1rc9`CgJ({B~nsC{)@gn);UlWgCk|3KuSwWIXm8lG<@&Ssk7uX*=k*(XO7 z_ljkd%Id#lr!xW~NfSbbu-b!g731;_Z4yb5iJk~E9?z5e^vPlj1PqsA!5~D+8|)HP zKztf|A+{k=S*|KB&Z2hA8xHawJh;sl_(Re1wi5se{h-}Z$SdSLhaQuFn0(5l43TbF zHkE(MI5q@8B&J&-+|!dH!XRpcqLyxqs;#03V{|guorRx1q+&%dye6UnuOx;WtE`r5 z#}Hh&xGE+RifDiX8iQa6qsUT zl#031Q+tCAX?kt(2{Bg!KF0p;T+^6Fms?E2mpTf$-uI_%?B#8qw zn4SSDa4T>~+FzWUU)B^(Jd;afBq4Lew!?#cjbahcH{1nSgg><{KW8l3dCH^5nm0>l zDXEVI(_e^8}_+_fYA zHX6ma;afukx>wd&jPWRPP0=M>5J3(B%Cxbe)dg^?0`+O^J_y%5{&AEQ3+%K$bZb0R zk&k7^{aj3MVbxxy3ukqDwH|P4V+k=mV03?Pp%K!_RbGIXjO5BHvjEjAf~;5C?6PF# zix^yc6Q1YT`QJRDA^ShO_0#p!ta3d3HcJq@0qFr{>rsh8tD*3k2vy^F5=W;7HOQ+gcPtVw`a(jiP#FCo_+!swLn-?%`y6R`U&y&kr zdO0jiSFBvVAp26+;M%DrD_h?6-udm+QguVNg)Z;q;^L}-;(XQ3(+xEi78V~+$qV2d zZ~Xk(GJi7NIqxGEhxF5pVgVn}A@p0VxG@r2LWpLPiFCPwKn zHLPQT%EoS;6q2s){XuSuWNT3Deyf`&ucge0|MCxeN41R;b`|S@>RXv3w3TiX+Hr%aqeWvmq@r zLmx_aQ-h%JW%rG^h`V70pyBFwa&=ved*$U-SQn>$w=^Uq#H%Dkxm#n# zgmFH;zMCv97bAi09K;vizW^)mO0njJrs6CR=Uq#-iHL+p^)>x zetkY*L&eN{_r`&4ehL6--gL)lt3}$@!lbbrf?Qm=tFE{`s?r>X@*&7T%?)2~$kW?A3-L$3n^=FP!}3$rIg|tQ(-22d@@>>H~B+t6?D+uM-)3$8sh4QK1Z7Fc?y;z2rZ&fzcg96h%qp!=h> zwY9S_*NC0m5Xzf0{IU{@fVJZu>wC+3@rH2}CYxh5cHq{Vu{avm$iBXcKqJ6U9EysH z!wWUx2u9#bs^R_LQ6;%~dH2!l^$}i&)dC;{_}Zx)XVgmxgJfp;ugj zH1l3W#O&gvgoGS4@TF)cfM|>F#RcOPL;k9w6rt*E1;B9akt0W<{c)SRw!V()z-^Q9 zTqc{9dp9r|FHe~1>}b(&ZR(^T^)q2|Qx52eIVNcx50mpv*%@Zhr>gw9+4{@`JwyDj zfr0a^m&W?#^Sni`Tb&0k2LJm2`6}ay@hElo{TBS9iBTKf1f~;QOCsXlN+DZ&QVoj-FlG7!9J?=*zI zT?zpyH8l>)Eks1xj(i=GD6XvJrMQls=~(I8Bg)O})u%{qIZ{z+3VH*1SXQ6C+4|FwfxfvVoV~qA+ znxB_9m);w0Z#|{95ar$bZ+W!76A`g!!Sso3ZEfA<2upvDcb9n&?@oE{w7EEXLPJBT zrzEvT^ZvbiO~G~sO55B`EHTLQBF;M>~p)*0J?fd4GG#@wjaVne(7Tgm`ns0pr^VgR*DU&eM zENp?wW-3V9FWMcg)R`cf4YwT#6d!bTeFn(8^uNwT3P-4SVnjp)euyxR<4SY0^X4wn zUYv~nLtkD<#GQeDLrUuT+2BozUOWOz%f$TCbz`$Tta+WQ)P3HlnFR@)I&7WSQb~R1 zVmEnbx1>`+TP!;&M!nC^>n1qM31v0mA^rV?x%LkFa`HT{s8aX#VU|yHh|ig!P#+z{ zPOZt<@USp8pmRCE^opK2w^vQU?MPl}X>RCE7;~=|egiVPG3G$uZ8MC7{4k{aeV>U% z7;+qcG;cq87+&E($RBA>C9d<`>^n90Ry>FD2-TJmD@2W56{V5WfvNM!# zaqQ0d8QK?Z_Hiyw5ll=>O6297_Q~uF|1yuhEw<4?T*GyZ_3PWQ$DZzB{m{d|=xRo- zXt+F;n>!^);EpGaRR%@H#Z%yn&WFCq0oXizptJJpA0M6p0BnPr+GNb+V_`h|xg+^% zVp^IC^5q?n2Q!!qPr+p2wSi-gv;H2pW zl06}4_qxIGgm7%?w28kC3;cVRrQTsb2Al%_+m~{J#~bGlwoBVA+|ye24v$=ImO}p` zL3v%pzS^rhHot$=d4HGeU5;Dfbw3_;#+uhA9_>$5?|t|pJvv(TgrD8r)u-pH;e$(>~ztf~4s>;VTx~=)}A5e7Q9OBn(VsXblHwAMqj=@FbPuUuu_*kS5~O0zrZ_e^TdSADGOp< zyE?yeWleCKU|4hMYt$!?%ZEcJFkX9Zmf?H6^1~gcyBDJ(l~31QKc{JIz!9!Ce$mv4 z^#eksqR(coP7PbFzu(sYjb!!LAhCTGx^q{2mw*1r+PU-VMVsntIrmSstg(LM?O?A| z!T0W%{Fu(i2RRwj1lmqSNAHih&7G2Z;)H;*a`b7n`B0JQL5iX)veB3R8^m%GA>7^7Yv~&JX2q-tnrq>QAkpW>$vqm6T`8W7lSo9 zV@p&#My>F6GBcZ}EykN?o)EACr?(6a$9PZ;bYKvv9*79&b3nZIz-@<%lA1zRpbEY| z=-{w?apB$-;`=t>>tvwERtsD{=HkVRGVp@sJVqQuXlyJOfc}=2!>kI?@%-5{c3^K4 z5OnJc3CUnL962<(p?=1*j<>?MZ-pQa$i^wzQ(j>$4Tk?Da>{u78FDrospCXr$QsN73b@uCHssxH>onDr$ z8572TUbkx6d||&0eM*0H`*#>zSB-u4ZBE?0!_`Z-9E>b}CVTkJd?)7#&5va(!rTwW zUTgfEcKx#ZifYq2R%(l|uwg4o^D!qg40y^**+{RL$%z$JY5) z@OIKo#s$U5n=kN2V(+}v;HU41cm-O90y9Xoc2@7t(6 zRb;9MH{Z!erh~<*%nOgEi-xIhkr;=6kez#{q(MmkSPW;x`8J0nt%)m*r_Y#m`GLz8 ziOAdsVRFs~Q}Uod~R*W`vl-Y!=! zbL0H%MF*uU54aT#G)?o#d&>8z;o6EJ$8yxag?fHu9J<)Ailc z$7!cUtZiEawZB*FsJ?tK{9bX}pHION^Wrq6=hSS*IeF`LoUT=TQ*eaifx6;_O&_Jj zZr2_eyF?`^p)a@e^Yo6oFAr^Awsmz(;?VHdEd7!Zzd8Jr_x9gHzt2o8~Tcm?v$ zDGI8~d|S0$xuRAOB!$N@6WETE`4LAxo8m&r<7?n1KlbI zEUhHl%xQd%3kY_41EKgZGKQh!lGAAk?%0Z{3a1oB670Zjo}36Yo-lj%koFZ;Pa-FY zd9|;D+~o+n$O_o{$*D4%;`$6RlGW6crXN;y{*>5c!5=(Dg}e zWc7lt@(T+I8p{J)8(O(K*$M$4YY-^L0&iKg0K}Xp@%`{1?L7j~rSnk%fC(gE_rz?A zD1VQGOvdrUIGkO@&oH%{A>$E4$pk^6TCU4-Pk^)Sfn*TT+|sg&Vq%=|p0qg!uSdIU z3Yqf4wDU0a@Ps2Xl?$mt=A_d+>^ey;54t zXg-~h-LT%aL;LwSz*lvZVzH1s=i$0vEkxdhbCpP!Un@xW}t1cwcqPO%mEwV1cOa<)v7V+I%ud>^u^ zk*yVR|32u_?dD|6iRxR-VW5BbnDi5aCsWbh^;=1<n8hWUPZjlD9$V`u8zTM)F5yr|@#(s?Kn>_CJSs&MWf!G82MqOkaE- z5D@GWc~IPz<+OOxl5e%Wj~}aO6w2?3l!!jH`E7j3Pn%|3To|{=(Q5boClWjpU)$_m z{xGla>AM$v=u__rm#lwP+0S()a$9>>Zq%7?^=Uib*Hy;6c&Otf%~|$x>B~phihg-v zuXVpQ-&)iY_h+Cbrz6cA$bx11qq?Tr{l7mn`IX#F)jj1o&@_4IM{oj4;RL0lXHuXo z$N}g+ZO$A&Xyi9UtUPiN9>E+8!ke1X;z^vG8%SAJUy;28W7G{CAQZNgS5ScJpJQrbY%FvPW{&R8hB;WnCt_k^0usS5 zS*xE zSI3MQGw}F?@F9a3KD4d{;4h~fvvFGky*tu$RGm;XEqLM&RXmZmm^va%yu6U78V`|n#KM0 z>hhU}YLi@^zN(WZa4XuDPf&SPlzn{#2hrz*v{Lc&7$7^W=?p7~$jBjs>l>V#AGXB#{h=w_Av&WMj zH57Qd{`sz@n()(2B}BK%ZE_QD+lgAwiyP5|CKdWy=)M1Nr+Z<0%;M<$&}GZo3Plp8 zO?oGE@z~Xe0{QAk5)`kvMeHe*pZJ>D{zK+_kErsFR!%>qAHVMhUF9EZ;kxll(S$y? z-sQ7acBFa7x8F^YNn1LF5mNa%{?prW&S$@!y6e%>T4P<;Fe5ZN%KmZljOKG6ieBnQ zJ&H9r9yI697)DLa*~TxT>kpe03at8btl%+UzTWkhT_q20b?-wFIobVf^=!`rh60iu zg%#J02L-=^HxGh_r|hV! zT~jzif9Cm1M>CXhW7)p?i8~EzAFUa?{}q(b8#Wp*x4+|3vIb*QqkB)~t|R@e<1JUo z9YL=#@TP?s{zr3r2k+E^irPf?n zPp3e2{k+IFy;EIXeRJo#sqb1Oh6j#}aU2{?#A7?Dt8?LP%iGKsS+O2W2@+Q)Or2T) z^{s}nvE6b-AkZ)YJcB>1vD1EN4F~@7sRBF4Mge8!O5d!kym$P(dGmIq+RmzqHMZ$8 zG5~-|v?^5p>xuSW$zqV(jL=F>eV}~~fXd*i45N#HESo?Eyx(uBTw!&{-Q64t-Wr5U z9J{IYAP?u^H+TVapPhcIQSPmBx3m!_HNhp|Kx<_dEm}k^Z9+mqe99-&7Zu-Ho#lvC zM+d25XBSg?L-{RXe~*Bhnd6?0b7#biOwodNY$ofXg%M z{m&>=7?|3t7xs704?C0VKW&lnVdpzTT64w;1aVcrOLm}k>BOvmBKhGm2=46igy$^F@=MxW3)qNYA{Fgo9_rqqqaiOEEZqkmS z{esG?dv@#KeEjl9&#@yTEdNHLpt3~sgVZ)dVEbkqE>_ym1 zIwyv{KGMX=p(j7tP;g?2>C7Z4vxH-Z-WRm)`uS^Iw{Z3N)#jILYu+_IH2rqw?v(ra z@Zs$Hvqij|nOyVz09%FXQbZ6x6%v0Ebgpat_NhG0ydO?{6F0wkA>*;)jJD0b_ZMdJ zoX2jv>Falc@8a2IKi53(_>>~Pu7~cu1QL;4q!i%_2k~Nx*E=eFG=4ldT$Mt&tKl*Hn!(eSU2bGKc953 z{_+tuzi?+X*mFT3ZZ|8j=chR~dr;D+BT$7vdV@9D{hzN4#Vk6%u8b@kpNU}MvfX~{z|9S_q1bJL3HG!>OsW=6*p4{tZJ&!5R|n#jG#@~IhKuHfdEC>Da)fb& zDAzvZyv+?z-Tn;lC$VLRKqXC^Ig`WCu+H_BKl2l`2nkTw+rE9so05X#YAhRD{E=Ty zYPV$)$~($oqu-U`6#k;5W^i!u8xq7q;nSVwFl{fM-a@hu_;(JO-11?D_aCm^qvcT7 z+peN4RCv!+rTa&6nyWy!I1WRbeNIP$v@R;tN?%&tw*NXaW-G72mlv@))k{8sa*Hos z`1i!(=Dy79QdBC-8b0w*pD)0FT^=#Xp~GN_>bCaAPonFuOUCj`i%l?3U{+RL_SZh? z&$Hy*`F*cj8M}9L~%Pt<9D6z`~E&Y|NnVBTGGim-tX&rUC-+|JPXApEaCWk{)V&Dk^6<$ugx4`v+zxu zdg*UV*|Qbm!1wQ;>Q$@1eD_XRUGw(z+%@UvIi#4q9cir(x_S9+G&=bdHBFEAo$E8RE-}ga!C!A(udQRc_dJFLv%6Ja=*?w#w;@x$ zh2w7ggS>vY9I5Mkd|26Av-^cZE8X%K7rQUA3RKXuy+u5D&~=HI{gQzJMgG$O8K-%< zb9W8zJ}@vKzedCGD!9sIb%;t(Fih7^pN~(@cGO+XaJ|2sP+4NU&9+u5p@X0E-fch%dZT@OZk@`TgRcq(S~GNshpvJSNegWa_at29uh2{S2rak00n@ z5fM@h1k8tRvV!G*_k=gRb8%%V*M9sM9{;3o@n>v(=;ZLsbG|)t0}}i9A0ta$s7(LL z7jyo$T*)JJ^fDw`|DO3o>}`Zow{%6$8xgZO5;OJ-=_Rs6aeCwz9f+a(zwim%5$pE| zmCk?M)lUP+6>DD1!;HU;UBBp38oJJ|&YzR1h(`+W%{#B+=I$VVv?{&Q28Cpc*fKiK z(sbF9)Df3}afwmZ6o~XK5wK7Gl0?!po_cBzuazewC58c7*D2e5p}NF^H2O5XI3+0c&@rGr+9RKsT!a>nKx#Hh|{Nx zmjA{l7p`mC<5iO&=FWa&RbQ@&h69=<)cdSmD!LSZ!UGRpfjK*SDLSHKgt} zwzUxi5g%3ju3Jy`f%e+2@{}z|K?C`wKwqEw{A@%ZBKydo4V#k_GrK)cGghnpqNGGP zM_mUdgR2}rFE@pp`O(z0ZIf{`Gn1#eZSe|RZo*7>6#fMv_D_GK&HL-Wf3G-hn=!-9 zreaVKn3%{UC6&bO?Rt-qX*4^w{2k3@{FMmM*zfjvxKSx7eO#AkURPEk1VttTnBj6v z68#uDHfH@ke&7KI?T=oymnmXmR17E|}ZdOR6pV0mQ-uS%4TNjQ;orFaMPw|Nr$hGk8=n3Q^pnp3rE2IJ?r(yugAvd;ihDL zJM^Qsml=yW{Om9p?|>TkqLTAACMFv2Q@AIuz~4?zQeTxJYU(ar6QR*A|0V!#s~cL5 z{a@t|kS7sA`F1GZyV89ljZIDX;Rgu$SH;DdpFMjT&oW}KW}r6U1=F*#tFj*m+X27*%tC?Rd5A5)US zCKMP@vtXa6lGiW2V`pcF-{^yI<*bx3YFB^%v&yZwA;=AW_f7$Q5g*kEo5qkf5h(e= zcm~^lS&BVP*IMOIcL&>i)?M=iF=ep-mF6@|mlei=yXU5-)BY%G4MLpx{%$hw;| z(AV6@3R=M-Lz-BStwXzGWBPxs177UNa9y+!z@GnktXZ#((AI9Xo|~IfXXxD7kDn4i zhBqWSP`r@6_9)@5;q?9*rosLMGbmV#-7m606#^s`MA>BmPT?H4-hGT*9_VD+sO6=N zpxfiojh5Ve+w*<2B2PhyI8Zg?B|y6woeP6BmfdtynF$oLtHV+mUihnE9)?e%+5Z|g=7qpI5t)(4XO_SEdP#fxw7HF zvH>yTcvtNfMCqOWk{QRqIXic)HT6XF5=`k=!XCwYu>aB7*kQ^lPgTfj)OAR1{=_$^ zVsKX=%)_m$>2GEM#(5TN()sC^R`vIa$0tqMsRln13=O0VQV!ods&;p4nQ5up9G#>H ztf}!U-P36t^Vu7=d;+*V8CTQPfsdt1Z9-Q`Pua4lct zx0U77@!wrabzIddQ1#Us8ni%F??~`vmTQDdoE)AW_*V7$6MbCIvAAFD=HcS(J!i9` zmXjcz{FdKw7qj55ranx8jvTDNN$tGCE~7-~OD86X%kH)oRJrKDu?O{<4;qh5rJCj6k@ zrEYfamnf7lKd*_}b;Z`(WnkTxdQRWbxQ!DLs6?zM@-8YZnm+(nm7${(Dwdi;h#&3l z9Ys~fXH9%5skL4`m?E=J`5mN@!TaIHf6x|-W8-^zeoTq^sNF2}CI9@!t3NLXeHxaw z<1}$Z*r#+2CbT<$1<<9_*0(iSN>3;z_`8}HM?0Ti(y73xd@3xvN6U~Urs3+(s$o%> zs+8;$`V&mELYlZ5$4delPnG-*Ay{S_Uo+k?8(rnu?<#k6t&nlE!cK9P+%nIbpPgAl zY}^x|wNwY+0!3SS7wh)e~lS}3e#fyx@L!UR1J7jNa`!Ic#f#3=i#osi#H^rS!6obDdxGVQMc>9A9nF+>;}-!x zEKppWamKT)3`6B>bTl;EDw>vK!-Gs>KHgK&0gSJ1q()jhJ*#le7D$nz1V^1#7&P8I67Fl`z%Cor4N?-QJ+`J&- zk#$w5>XHucQM4o?l8NEv$q}-WZk+U#3Sk^_kKsPIU`m|sfM}Yjb>ldc8Q=b<5co02(`o&`Q z5gU-IB7Yd3(1-eCeRXP6UcC~nJzBrn+-Auzmda=0;1J|AE48`Zj^%bTNnl6y!Zr(u z6NZ%$<6Zq*P*az0HmnX)dt&zpmWE50-&pA!4euJ1Er=VhU!EH(W;|Y2ZgawXpLV-) zzwgY>BKl9l1rk2@b%nMpMJqw)Yc;=i0`aw^jy4ExtKf);x5!hufu6U&J{|#E9wQh?;Oo4>Fg{6KtQ869_AJmGFjkOM*9_)2Rr>#m zaZVHT96Z_G+PVpC0nH9}uc>%mjrToR`T~aqNJghYCi56f`5KNvKZEB5!Wvhy(7+B| zM+k7cPmJs}m0Ws%(yOr_ab3yy(jE=`_Y~jlC&%EsgU@hP9(;a(sMd=YzBnQZ#{b95 zr-UyH_F2Yw3h7tt=SLo3=stwNHxhAQ4MZX( z#>RCEIVDtW-nP#R`t08U#bs2r@Xp$8KR-WiNEySEnx)d_{dTCdb)j2=a4bAMoh^39 zXLsoZILrM2F2>w(sP=;g1Q4cjDz~o^TlfEm#qS#ZU3Qfoe5Tx;b@YOjtwjA>#c71& zy^4X~z|`0ML^YQqG4ZQ<+xnT{C&P#&`W>txjrK#QWFm&%JgQZ*pdNxZXI*XU#Jm1a zyMw623u6DBq*RHs`&Wuwmkod$%?u4MF%sOEn+#7rg>B6Wq?Kz}@nqf1R$k1|t>`~E znp93?y2OaEpSaUD(^6CHI8BkG{)w(ITg(@`XWrbB-!@zt%)Zz@Z+*P^ki^8q9|oia z;#UHosD+R+)MlUB(c4ct7W_)L@&lAjOamVI`zuBa?);SxEq>l}7y4FWlA&5gW1>S! zGdz7-?`6t#dpV!lZXuy(VNqD>R9|O;`T1=+PMdJJTU=4)^e5=Aw+p0dA7VG6L3-gMQZP`U_&{&Wn^7d*#<^Z2ugPOlzbQvmBICylHB(B0ZM1@q@&~8eGc>*YTT^$sak+29(4TRkJWOI7^s;I>dndf zirkO#%VH4?(%1R!s|KZZX6L$y)2!Lu@AR{?nt46bI==kk`bA8`35nu^9A>xlMg|>i zWt5k#_a5}vG1-#+maAGe#k?s@*l1c-%!Mu!09{5Wwx< zDTA&q%`SW7rK(&TC|*T1|Ow2^}!$ zh#z~RTiSJIrhc1Y{e_I&Jz%J^>xUW0W6;fhDhc{?U8a{#%iW^ zUHc#jI3$@kVSzpZmojBjr~_zt zv{iBy3~vNtPTbJzWXH2CimBbCB-X&f7RKMofH3?u`=+iF>QGm4fABV(?mIKiP0`P)H`YWnxfUCeg1+u*3J{-gXSfRBPh5!@mP+~DOD z%QOWhc+j?^v`Q_=om~25xR@a1sbnw;sdQ1J%+KFgBP6E&fp^ne_k@lkWDzFQNb! zWR@xj0oF+uF2Yc;kv)VeC1~9({117=K8cIqVKBuwCAeM!XIsa zO}A> zgQ=w+Mz~F;>H1k%Qlpl8E9lAYLd?X)EG6=XS|4t|k|Mz0H*hu2&4a}aznxqVaC5sQ zy6=7E5*;H)sA#aucvUeXj7+aSdY%*UYmb;nxPoIc3s9r;w>21FX^j3cMgH>f$GTbGl zc|cxXuJkOvgot^^)&dn3-sHCF857GMZwOuKJ%HOBA?f;^GD~^m^&gds9JKUF;`)K9 z0ZRRp_mS{*11J}{m>O+g(ly|ps;GLa?1MxAF}$WLR0z|y5F>!P02E1wHo$91Vq|l> zB!31TzTy#v>SjQ&cUwX|E)E&=gy# z(IYAkx*P#=qpWhlF^9uu6*Ziag- z?Qd?R$p(Jr;5z--t3m9?xqYQ_m0$fFodr@W;d<}qH3$*3e7C+G`H33ni4fN7k9Q1s z?6|(bmHm_s`|~_C{)AWY?Al_`ZQ{~hp?TmHaQw~tW$s)MC#9PAH=zl0RBo+wA*$<_ z5}myVLgg2q*2$5lI^C};R3v{m?;@y!j*T~LBVNDM(ThMvrR!b*N4%#I~`-R31$ArnRZ7HDDE2E8JN^PwTBM1Q_MR<$2AcQsf{9^>4kXCs%5=u7#$@D z4lWxiyUmmxAhrQq3_Nei(_3zHDMO6&E3EaYn?la836BsP7|?{ZNYP&S<+!3Q4);;t zn#!Lw&_3MsW`ZYYVq9vT`z zT9BUJ>-Gk;G(%Uw18T#AyMtGFOKFbw&1ZhcJw*fpyvp0nC@C98+8RY_SE2-8ZLV4c z{XKG~vVr=wreM;?Fq1L#_bXD?e*|RTxQYJ?deBA$A$yJg!#!47 zCnti=5KBd$yARBo7dHzYf;(J@EmCJ}t>QFtF4tF$pZOJt-HTCL3*ILL= zS>>UT*D-O=!lAHr+(iK#RAl@jENU+piM$F?HmU}9+cM>^!^TG!RL8*JWM zk7Z=8^75X?lbw~@ySv-c9){in(cx_i>C@`Zp5Yrma5e*V?EL@y1z;4fwZrS(Ep2vx zRQY3t@l!FJ5@L5Crx&h-w#JG2WpQt2X+VhoG4pwOqy91TQ$#Owb90NX!|b;HLj4vW z$LR2ILr+hS+%Q#7(=C8MSx*los`cXG&wE*~1k+M27mt$CufEMI6EwK;&^edUZc>*+ z6;-93+ng_V-K2Jv`|l*dKKc*(f72iS-PK;e$uRy;l%3A+vTb%3cs-<;61Y;z9a5tn z1d$Q9dZO|LzZnKF;adxO(lAKnxpv0N%#03bFy$rgRhE`=*5ka{n7zJatFHQNkhk3~ zWCX*Vz2rBxW1fV)*&Z#k(K45t$E)|q5L0Uv=Qq1Vz&7;73nk359exM{=9g4n{>k$9 zs37xu4g9?bf{S>fDg|9k5KlZ-Fn z$0;A5rs&}bf1gVl`PE^Q#|!uF(=*wRfBj)o*fN{wz^?nfpkhKUu&$0=qcA1xx;z{C zLfwH08Nh~gvuzH4DevMckjagu-5}UC!^l+FZHXs3Lj?=Oycn#o{TL2imJ<`e`4(=? zHs(>nU@5}S#z)AlRwLPKtwBYE_%)#K@s?Lui=BYvq0dG#5dzR-BY<=61)-rLk~^Wx=mD=}uBTY=@oCL&xpu5g+<)Ma0H7M!s+YED!QG zSg57|CX*g)HmO>meE0gV617c#x6dw;3rEsokq)aWUd=`%eM~hdJ!r}>mco!S7g+`{f!UvD*2RYwfvhM|agb6_q2=i{e z2G(0|QdKuiQn89rXHYXIHxM0N^yQ>b9=*y(7gBzyP z9)mdX=|kMm&a%)o`^tiD%Tb?jE!>`2Y;xk90Vg-dky{>%UcD%i=%V{rGjPN?xN?H! z3PLgxnV_qtkH$zgcfAhG&>g-1r$jtcQ+ zegEw1U8(5l2Vp5={pn~?3mrj=O*rCSC->(Mzlmyg z4>?NGNqh10rL~uZFda)5f%1gy>mXeYzd~e>84?fv)Pv$ zK)bWuJ#1idbGv0b@;(Gmuh^Y_YfffA*94q4WMRNtchNBNq0#5kU*@kPY=FO&Fm1A+ zH)hbC)oE4eHzRTK)MD0@;3lTUNAXe6^8&R4#V+RNK}=B*6#12E;FU=$3j6`gf)mXjla z^d3Ak5XmS+LqUb;QbL;sIU2p@9v@Nh7(VoMLjkHNqgjY&;)xF?AP2~ZFC8RE6T*{SEeus87|D+I+Sx1s%6}l&=B5?&f zXsFx>y9kUGtPYH}sz*j(eJJTvtYC=i4rnzRE)i!pJwl^f>iu?%EN_$uo|M_BkrDZv zKyNx2oeXK3nArV1cU$8T++8xLcAEjcmVDiLx_tS(e&e-~T&29bR9+LQ9Jdlr=kcva$WfvcH~8DpC{jCJ63vQ)wr}=3J~QMi5Tw8}&6KtC;TO z+dt;puVc2i=lygsT;Ibq@*X-C@xEcy4huWmn_N(b-M>f1ME{wq&@f=!)^D&NZf4Yh z0oUCSgawU&f~Ti!q{;Z$>efy-FWE$QYC!#qQcKc1U0D1!S%xJlBsqD@2?cu`FfY48 z%YXdb4~tQcSXq~aW@=*MTjYki6Fb@iJRZ1EA9SZMW2{*GJt?kV=gAm~-`?kE+-2h} zE&1g+U$__ti^vQwbD?LQ1r8elT^TEbwZvV^Z|?Tg$7fknCu?e_A-zE=6Obb{1vhzjw#TTh`fkXyV~_vm;dd|a^RI{9_&IQU4~`&`u& zP5f|O1QhnIt)%=)H+JOkNq<8>h0BS z+qptFxM--K-;AD2M80Whvz(o37{zSAJTngF60?dEg@*@%GYN(k8J5bF5vj^NJhwLC zVX=w$Dt(<_9fhmf< zVOPhu<^ug?i`hc`+buUzlLiy6UKIoFWrl@>1OgHMDDp;1SYv{r(tV*5R3R7GM}(kO z(;th0!_l47M8b*ry=`sO0f+L82L4sZUx!T7ykYwjH<OJ(Mn3?2N={7pjxEu3<9!!I!?~i{i9DNEWmsZ39#;zHZh@Y^5R?WJs8XM~?34wwarHq^W;&k#`%WU#bA{FuT<)c2GW)tpXEV;xWP^`^tT$NO{FCv(|8j`uc!Qz$y$|fMzly*C9ztY!<-zbvZ*_GQ zZ@?1vh%IOt^g%M_=DmJWRApHp&BGHsA)rABl!%}Etrc6rac8<}F{KGWa4I-5b!pR# zr;6ASAP;6pVE}J>ULO@j4(%)y@-NHP)+gmMX+%8Mo5A=d5;#X0hu}NWgts`uZ!qs) z!ELQ-P{6~H9;F(hVIghlN5gWPm!u&R^(F4tkkGzMpMdFn5RI*y5t>T7a^$9;# zo9^x@yGoc@GwquV8m)E8(f%GX{6D&KRhwDO-GA>Z{KTZLqjUKUi^CQYPrpqtnv?>z@`-EqH zRmv<}-?XU5=Xl<}HSk={yc97Az7OO?pYcg98K4f2Z6Rxdp!8AV0H^S>yc|M zfXzaa`^j$Vjoxnsn#&+~k4896OQoqMWoI`}`8m`oCzb+Y95lb!v25{V1U(-_Dcv|e zUjOTZW$Id9Kyrd+P;ISstkZ2xgZmsO8v(P%mX@YGbO>8JsF%*7r(d?V>)mIV{7_RP zhCA%0R&3o}T@HVHl|gvpo`;^BD-4cWL!*h4Z@Rk54-RH~FeQLsQA&)ROpO&6*AvYH zi$>sbVPBtsca~*nT4S+hv8iw0F`AWxdXvX3hTzMa)6CQ-d zx2=XI7R$3fM&@xkEam*^dJuWgpHQ^Z#0%pT0+AMl0J)ZvJop` z$<#0Y_$yAHf8&+KrH5tQn-^(oAHTcSE&0r;3Bh;XU@7{CL4NPYK%LBn0T6H=mHCar zEcvoIbB~@W9+WAYgL`NV=(ZL}>OpHaf$S^!WL|>|!p2K<7td-Mwz0?Oc%aMc+y+We z9pj5n%AS&z$(i<)D5^dmsoyew`t&W%4QP69>fG+OX8~}rjOFeY?Y$U8#K$-(aU63h zxY2dCEUO^T!^Nq%iNS$vAc2y(m2mS*C8n2Wkm`2UU@r86L zLzQlEdpY~!=R-F`4`-U$QHL8q%Di9I5lGh$}40TDV2zjBYl z9?b4wcJd_=fCtmU38|x1KqjyMvM3?(X+h)uuje}LYqDJ5cV8KCNzIweZqrKAmEWgC z5OoW{xf3`v70~ldS6`amJ5GR5SZzF2L2%okr%na7H(*iAz(jUg#&MCfM>o}f)dnK*qyzH5herI z4f_Kj8!!_F<>NoKR7uA;q0-#k+=p-|x$^&^qD(w_Jnr4?DYA4^rXedoSTP-heMT4S8-_M$w#~M*J(+pnV6z3 z8v0TLpc$HKh}zqd>@xtzovluRNh7e70YatpPE%9URwF5w5v*0g)kNdxpX&b_U4cW? zujeVCyTsFE@J51a_Aoi64}k3t`ENjGE!e`tbYwn6aMF3W1UD~C9A&eyp&W~-jZ_6L zzN#A1fm)rbl;46&qWAU4ICaR=i#4JV`?+o1e}I!dLowGUS+L--^S|N@zvgEytG2>m z^Y8J~dVj5*heHDE%F_G7qxio4&H{tVEtJSPJezN=Oe-Ru6gu(27At1EA*g~=r+`zx zP6ueZ)_z{YB@=YyNw^MPKhAH+(+OM&OYKPP8X_VdTpm5Qe{uK?|FtW#)!wi$Wn^rI zx~pqg@2_voQ&2Z)Devr%%c*O>LL5UJD1o(d%qyHV5QD8%W%URj1A4VIHB)Fjr)g5+x{f<*v{g{YkRN8!5VrK|KiQ6*(7utr=>o6PLu`hDDI!?Y;;CEqUOK-;*?|%0g-xHY zJ!Dx6{w!N~R8YeW2m4I|@;0?~zTTsk=^Ej!SeZ;vmg<-=Bd+9BTud<#qIO8tdMUA{ z9ay1!r@>*MQ?l@F1I9C7CzNx8{qvR1U9YTSgR4PXm^W3IuFrL zC<}wFw}8PfEk4YE7a098Na4gycHWfZYAZ+6?3qK`aG9jH?h%tW4O-v}V5S=#{(SO1 zgn~E1{@Ld{g(Dp6+b`~5ZG3KU6Lq6{()B*kATd0zd2bUS#%E4ZiB;L;_2>647wwIo z&bhGHHZnAiouZx_>v)>KI97f?Xd4pIQg0(ZkH#R*($|e=3uH-Mcc^AH!ja#!Pwo=< zRFK0Lkn;WgQ=2g_U7M#IeMIObR@S?A)%C6PlWKhChua~7K%)o`Zym)LM6$4a4bWH1 z;PDXySMllEyR;;Ln-J=nW3%ww6<~o-v!rGiPFki$wM?Ffrz;K&dNL%5XqxeZ!H@zL z^((gK%9S6v!`0ae%hsxl2-~4Q2mV4NpxmJ2tZ=nza#4Kxl=}DYZ@eE8rrqz{znlRl zS#$X&FRQUi6|zf?0Zd%%*DnB-L(0&9I{0~I<=7-vL*|eyT?`$5Q>}3dg&Nwv#z{-J z5CR1My?MaWbo`i5g92o&^uT%f729-22fW_b9o-88a8Ehz3;K&z^t%umZ7R5+G#)2@ z3(Bb*9vTAvCIwg&@j$MUf_oOEvxkG6a0XR~l-AJDIJ$8dh>w}(01kNCmoLoad35b?MR!5ODKJUnijJ=X^r0eFMrA*e4;XX}&Lu3jw$ za~s#?znR!r5WjoNj{9})?b_D)gE3jhmIu}Bmc6}W+5Eldk0?nrby=BF3q%3LPCE^K zUxDdVAUl7s;w|(}ejc3aauk;l*W!L}&F?wuz5RWN75(=%{`*Pw_LRW?tFBDyvkh(f z`fF6@XKi|%f$kx|a(b#cToj8M(wd|r|0L#Fgv9NAmf9zMU0Q6D6ZtdK zzJ4_h@P-2d-Mq7eiK!ne9Rp>gUq$N?J3?Lkl7Kr0M-D)SW%Z)wujkt!qhuO0n3?GY z5~f<1w|Agk-Wk6Eo`md5tLa=fu3vbNS9ij)gv3~X!<9*AG-X7uOFaqiF#Gm2w`HM1 z8B{a_&r{?IxNF`t8R|YNyf^>Se9 zRk~HDJxqj3_2Jmr_PqRt@c2UBO zlml1BkULAcsaYD&i_pD0oeQykRhK5e2?l0N8iYt^@-bvTLpdHF^#rZy{u#{WHY-rAdRa+jwELKo)GFNN$>~Rb8~WZc)WPqx!_Zv&+D@tgPq&^ zcz&`*KD$rzXfo)vl6MM_xFy#g!=?G94&UD+1DbXctFTLdGfjdPUwZ9SkE6XAB!bEe zk_S%6((BOprJhcT87ZImK*!ouG;o$f(D~O#+gqi>a9F?H`EtxE$Gtz=C)Ha!*L#{I z+1?t>G>7Alk3BUN%SYp29StW~H2A6!f&yklsEHU{Ur#3`k!^H?FCyOG3I9;tdHIq| z;^ck%Jl=ZaJWT$atTf^GAge|6a4y!-i5q8s7m`F)CGp;$Ou>)XxWVTtDk*TT zr>7wPL=@p>Bw9x48(^OV0rg-4Yg zU0irSq{PGo0%cnJ#!c^~C0AiEtevgdsBeNuvW|6Klj2JdjDVMU-UCyD4D7q52YBbg zZ*ZIi$2uONB6~2;^WRCF#GA`eojr{|G=Ba1brXUNXJC{6$C_NE62wIxgk0Fcsm@_n zOb}l5xJa2!)a%zIK~_L*8+Z}|X0hP6!@RLh#PVdvZ-C~l+o9}Av?iqwbKzRPPHw|O zvQ)H+InV7tnB5&()YRV}Si zGlf~Z3p6uimX^8^D2Lj7SQnF!ScC23?zX>wJbB;xT%U@vUH)$~1)utEW8(rU5?$m~ zP>|;0_F%fnH#osA^|s|bE`~V-5eV#wQm6I=bPJwthbZ!Lp$7=a$v<~45b zPZ~j3Z*r&6{A0>3 z?ycm73&-XhdVZZ7iGDXiSUI^VFS_@6RI6*|N^Y-{k)=#F&5Y5Ho_Nu8*+1z`7$;2H zY|frFG94|EHa4&7uk^igeppjZB}tV)JM$FDN0JeCdVybgaqQWNIu)>qB*pQc;y2mx zQuOh=^0#|F<60gG!g0q{98P3Y=NBw^aL4j&BgstQD2>TN* z_FaK5=Z1f6$XAogUtgOaEO)o5CaH#>>u&qdnJH8vcV?j0lj_15ax_EGf_qz7n@>gM z-N{Y5)dcF+BXiUKOr9Y0^`f|ftFR~6om32z)(_wkULSrA%&H2xafAw#)B2MXV#Haz zpe0o58eO>#wzG;ZHLCh+$l5@F&ZCF{iM_HHKPy$vK32pEZOlxNy?q`fQ8}Zhpp0t}46XM4}1CiWE9mDA4pyeGi`K3SQ!=O)By!!n#%q0pM~jC z1cru9sW-+pT=ZP%B}lcaJK#1syS~29akqe_Ij=0$2PngMekWd72@Ggey*m%x%1VAM zddlx#{p^Kv>08|7f< ztL%6CTG5fJb*9~zYRa?Dz6F-L zpvul=cRhw39lRL4UlYXNf?w0 zTC>oA>W<6dJ?lhcYOiTI6ftD+RSfTT!KpT-V!%oJseFz2ea=<+!l%Z2dnA#Ru8$lj zMb4himq@|8f&TZ=)6|u9NcBJGn39m!BvrVwnwq+nc4)^iGN*n2zTDNC`8VWn($9k& ztk$cGeOy4o+exgwggjAf3^+-IjvdPKrg^Mtd~d#WRF19qMRM{yi;s4R9fL#Ow{N%3 zo@UY3I~l(-Y_Px2c`~GAY4Lu0-#1ksZeAe%{!+DCiXvi;>jSEqiQnKkvYyfHCB zUaV->4a%=ZSX6^8!wZG1))4Z+Vw1<;A1GLVrG6b>!aos3)7b`lkmFC{BkAvag_MEK8Lm#s)QpQB+L)x_Q~O(gkim! zQQx~-TT74UGM2W#qTCL3sAQlK&>K{a6&$5+vEtLQ^}0czX{gLRDzZXg;!fc%MCENGqeJpjunx&qr z?-4j?m zE>9$y2hK^^bFV_Gi+8*OXa}%34s~Zf#1*vl&4GGm2ltP~?MKtCzx2Q}^%UZ?6v1jz zWOQ=8HOe9!wCO?HbrUk*)#VFxG&Y7oy@czgo{g6Vile;2%EU%_w@Nec3D68syzUJ{sh*)|J)8Zakw`?+a>r@`rkQjJ9@Bsre## zA3j{V#?HRd4!#?eUOVjG`>R$h-~It;VI8EPn1kff`lQ&{Aox7I$a{D5t?acA0uIb@ z(-uMC4!Bk^8QPTT09*DF1fJk^-FkZK8|eG^bJP=9DJ3U>E4fm(X?6#Egc8BU#U%^MhnZysms|anUlf z1C)2ll9C7rE(-*w5mYuq_@EdZiT6a4_mGb%JpkEMJ>I z*kWus$d5fwT}BL+9*~rXH*R&VavIf!xg2EV_(+(d;^m|@{W!VnV9OKw&sPhQ4ciK;}hDzjr^VBiJ|~{MT}0`k2$Ivf)R(>tp6*7aH5xY?J~Rx-Q(Xw3_(j0 zg)2+eZ}-1q^~Cq_W7Pe>l?e^Yoi}giJ@oc|Tqjx-@;{mEW9+ z2rUKz{*r6fEVx2!9;bZRUO}wl4!$*r5_|Y$9K@oY#(31#PnR0L5($x@>%VpQ$+Y(+ zDm=e#+FkQ^WJM-cf3yZA*)Du@7qvhkBOua;^A_(`<}4K~6j3Uu3&Ez|SZ;?g(bJ;; zn&)XDT|F6~@)>eXhR#HiWOzxvMNkyDQw_hqQew5__^m+hMYUmfrZBI_%c%UbKt-T` z*2LZG7HB@YurBB_YSE4CevqASh#WF_b^U~%oLF;*e1S=Ct!OTT`@nNwBWG?8eF^lq zfd&~bIe()lf)#?Hj=dVQ$;4RIwS}5A)HJcywh?)84s#Jm+By(MeZ`5X3|wAMcwMuv zCPLigw`%~_AyZQ)DkaR@k=Xi1lOo_s z2q61p;^077Rf{vIFM0X`SX7T>j=twmz%6iH-W^X>s7k~#&?e6Ht!^H2<%KC0R1)4#CDGEcu&Q>e{Tdf zn?Jj{_=P0y-=_sFbZA0?rikT$)7;{sDR}%e*Hid-d+Xdh z&$hB^`|6a!Iro>*%|xLtHlwxOB5c;55I-uf$TWKoSI9}kn_Y%$BF~%zqF#cfns`}z z#-VO$`PH48C1mOu&2YmE|BuTJwSR?7hhLcCf90!;;o?OREUJc>(cw*Q!?%rvDWi;N z7x;!}XMZt2=MG3NIbqgdUyjcAZ<(Vj6hP~L`EtEZ?WWJsK2~-e29e~XX3UEo8*3f+ zJKxKyh50~8kVtu2MR&BwL0E`y-0HVW_g)LwKM)`;-ld%e=w%nR2yv3R_Ybz(LSFo& z<6Jo-Bh&7@B)bmBu**gDTjlqkXO;R~Rc@>pQ*>5=`@I)u@=SiwxU9t$Z)KGlsadD8Y zzZA^=IJ$EFBs|1;!?7#{raVFc37NBxTQ}Mo;(mcwV5S9X|E#O@6OGJz+vRd6&{%OK zj_}VP_W!KX`3fco=YsQ;clK&9f^5zT9Cz0`5eVAxuaZ{o7iy*SK53VvfF zao_9pjv{Hz2w*XH*46%c{pdO(hkh-woTdr;cfA0s&T2C(_Wov#gg7ZsywV=*RA2p4 z*zGsy6$*RhF3Rqu_FhAl(>{y)b+}~Mh*nFlVTd}G{kz@HK=Mk2gn|sF>J;7$__&X! zFTS1iJN1wLHfqoT^*G3B3RaCi%Bhbtp`HNXrT!qgDvj|j8)NHWoVi-+xpXIgACVMEcg8D_6 z3EywhG*Ecs0?q4v+a4ZZ3O1-I zuQ`-V)SyQ{(O=ioU+Z|Jo8^?8zeo1}vGvwrP4EBv_>^Od97Rz{3sh2(PJ@tckQSBh zmKak}IDmk(bayE*8m1y3U87?n(lL6#*!Vr>Iq%Q){o}Xm(hCE&z4m-P;(pwZ`yM!( zq$D~b(eu%usOfp=a*zG>*9Zy;=t)V&MWBKBhQ<Q zzkg!AyTvWMJ1a_QAGoj(bfsv{LD>PG7=K#P$B(~&yLO5;!vs}77W))BF$&>_QsaPK zhB9tP0+fY8wqVamPmkWp%IfEDtibH8KYddg@$8=!mfN>M9iHh%_%Xa73q+g4;1vc6 zE~i0`xFLY{=II%V@eW7@!`8<~F%MoFTb5d!CBRY*jacW`P@&~pxB1Xg5U>HzgIoPh ztMU|_$)F(2@~H-g4ycAe>Kaz0%>Q#6WcC3g_E(zhji5`n)d$EOg#iRcA(6NfXL}w& zuzdQum;I!4NiocB9uGw_u`5E*zG2zaJ$!&u+hQDySf+%5pc(?_tw^U{5t53HX`xqO0 zn@IVgD=gek$D^a0`^n;| zI}cBQ&(B3!mgRsbj+oSb>+{JKMzP5U&tM(; zwpK7111%k8Sg_{=gv5^|nLZ}B!N`_)!)7M*wBP4*7pFGffa;iUNpRtG;;n#X`e*$o zgB#ltL7N-;$G#6S`AkVh{3q%sSLh&w19oOG5tGUz3u-TaW zOBND8-RC4b@Jan5TF>={lhi@RO{Txwv1^=|>6@SUpsKoTrQV6^RY|QMF%Lw}9X2$5 zBGz2BPWrCvv=J^d9`M+IJOL2rtG(qBp=PK(O%7(|lm>7+Vu0k`6QbF1b8Vna4zYIH zZ7QSZ;$EuFz$Mys-U+>8ED^V`wIkqK$G_uVtUm0!f>gq`` zP7~pK?M}B!r7mOn)>48Xf8u8Jz@ySuw)h$;(+VOHtcIx__UupGX!0g#xiUn?R7OKRg#cgwwZ(kCCy0e$& zyj48n@=9|aA5Bzt$1(kKeQVwqQl>Jt9Aj)ZAF77fS#Jw?0-BYjZMr4@_Mn&paj?SZ zE$tX^=b=yDAdy6yyyUP+6!xFn(i4h>A1t|7zFbRFQ*?B^J5Z?a4~8<&zP&xMy0Kw> zASDWN2Dbo=q5*yV`^fC<7#`y$K>iC8F8?Vi!tSC_D4-Bp@KNJXVEF`f*tbA1-4h=J z^BM{de|6OX*o6HSe-}cV24Eipxb#SaE9W2C<~)`$W^omK7jKxp3d zf4U~}Lr5aJy`hsD{+y{~8k;8(>)DmReBI{u`93_MU>xlY`4rt*t>mgi&~&7vM3}K?}nV5m$B) zxo##}Lk;Q%(w)?;NlDR!Dj6xPc6H^@uNG#zp#-%fxnt|~d;xQ0kKv#@#rdbkxR$1( z^km(FPl}f2sg37IKDlN!QpnCakfHt5?4tQ!q5*~fuLqKvf0OaT|GG|BRnnfvqA{*o z>-Bj=7tF)}Ut z4h+eJ};Q@+l>`a`n~x{U#d6* z*ey!qtf(HAEUcb1DAcw%J);%V8u#Q;9oXv79q_m}d%mNm**x3b&;T&3>98vk0@M~s z)*wn76fKM)vH5r$MAqBD!gMy~BjR9%S&7qBZ?Ed%*t~|~6+~7oFXExe1`{Sy_vQE+ z#%okluiO@d+kH!XCBVC-|Ceg^Mi59vj$fP1l}p>tcz{VUOnbN|-=0YM#f^GdJsP zP%m%_)8sJJnd_lDpBhP4B%ENmsP={arW7Zh~;MGceEqDse z4LP)<2kR`9mHFA(f73P2Olzg5rA-<En2(ig=k6>QR{Z*D^%eai+_oD&(yv?J zduN3do-;_EIy8V}7I827J1V$nma@P!G|Z1W)AA$7Mj&f#<6O5$mtyc@0m#-hQTHXz zprvGA3M@Y+ejle#w)Wgy{Q~yoq2h3-b8~Y8%Ur&X(acB{L{m5CD;43I)qaaFDwT^n`Q|s6Z%Yj>(dLws%H802`Tt&sWwkl;2i>wHnw3 zDu}#rR*iV_%$X)Isyv-VDc-eqz%UO z1qGbx9v~3^jVGGZwge@ZUzba= zaZQ2)&WFFvksf%XwsKa)(nwS)Y8kY8rQUkc_9V7}s102sOecJYUXdF!)HpR+Q zmvGbRYodUxc6555udQ6kC(q>3%h3B%w$~j$^WV}@SmexOK;+3&KL=Qs!&$^pteLAc_1f| z(P=MO+sxb@GMo2XW>yZ*%{@myx{rLzI+A`Wd!mkEyCz<+ZPX-qi?fb-Wz_tVphF5q zP0`2*O?JijOwjdv?KqT6I|k#OyOMdNl+!nm)~8OpH1(v1rht!+H5QLTod&2omMzA5 z|A4Bnh8k|co@|u{iD&c925M?t%1Jil*c24XxKcpGy}4Q6esuz-8@?seqib%ZcacQT zhQc0kzs%$le3@(Y!+Nf6|En~NI{86LI4w`!Qf;ZuTju(?6L+6UHPm;9l@BsmT2c>2 zOCUK%?WAb#85!5geGLCoD>x76(Gz`&~eG4NZUSVmL)hrk*3&$@nM zB853I*sn{j5rUgy;mKbVdQbu=m^CwWQ2dbu?8c5q$(z0dR2m*4fF2#=1e;qRP#o=} zrKJUP;uo2KrjHClX#+vZAMe8|E4TIP=vjqLEKjPh?^&0Q2xqFlV;+{Nr*WAdn~8A$~^j` zqYoD-O?6js$v>~^MDLKlyHIsh$ExG*76rFj*O z&*@=9Aq+%wta(@Ri@7gHns)@=p&kCx@Cf!}YMz4@72*{|)=Iykb<4L!;f4TZNia=} zfhF|*C`&|i9H& zniGghR{<8FEHE0xOP~ydT^RI~9&Mzpz9uU*8B2T!=&(;I$jmU@Wav~eeCECO8`bo{23Kv|oH!co_j5uC zlWdiFw56tZ%M^E95O92k#;wHjev8e{2n5LLblOyz^pp$EPA8k8Z;`w6UNG9Z1n>Vb zRlCzki@4G0#w9`HjrQh&CHO;#|9sW-%J!M~O z5h(SrJsHOYti>aURm_3mcaGvs|0g!yiFVf2H3At1MX#l<3D6qizr5xIJ*EMhaX{Qb zT&Wx1Me(qL+WOL+FZiU5;DEIUgq4-Y3i+s`Mpf(AU{y^rA<;Hox3@NA)n+7YJD>)= zX+VWV37F)1k>1br$a*~kIXA>s*WaK>ohlm|(sTK#W$x*B@2HusQ!~EA6UBNO*u=yb z!IT56UwK0qh2MaZi0`fynjQj{l)CyknPB>NLqkvgks6S^SnmJf!`~odnNmK=OR*HgsRNi(ZKYD!A=B}cWY2f(w9952o06Bu(DZb20G7?%Glenj9t{hDp1gdUqJ~UVS&Gw-$>j zQ$w!Kp56yd`WloNdzBA+;S`>m`O_UQw6z&dm_RY2P9!R|tT^fHz0^zLjrbt_N(%bXUZQfBYZ zcNfn*$hWm{fX$K~vnkPh9V6kx@4oHtW`T03gZO9kg~@=MY3)rErD~vf%pZXDXsXsa z_7-+$w_{2bi%rg_n15`KzAjXEw-(+eX@LTl;N{X;+-yoDzYYGib4`3OQ@PMNDE|;C z;nog6Q2_UF6|4Q~&hy*wSG_?I&Vc>VMjY(O3!f>>B42`j@98}1=9RK!m54>lkQ$6e zdV=N;MXS9%T^Q=s9n^KFxxRl{Yr@iAr5D(COwoPFS=LDYb~qtR@Hgm3ag%~Y*Vaxt zzLxz^TAvVQ@Rs336%ccs_!ug9E(l_27Y-8;<`JLug6S9zoEQgx0r2oJ9r^2;2NlWX9*VQJ?Z6DrL`Qj2YZ9Mx>`Wlvb|QlMNzbj z7sOfcnsvl+2svlnH}(P1YY>mW)W}~SI)dg%A9g*Mq-p*TWjb8i)t&nKpP*CSF|ez7 z4LAcnOoRs5C|_F(O@Y-$QW>b?d=bUpho(C?! zT3nM`5v*zv7QRWCiA?xS4j&46{*Ucx?(_P`Z8Q{l4pU{xo%hXscuwhIU9e1VJD%p; z@Ko3>nODZURKaBGHNkwwjXkMkgtvM7*L-6StwZ{|!23Eys*RM)?t~QSsMedc!E8y`7A_#b3vvO_? zZZNsk7+${Ye(?S{LX`x~F&gf3aFGA88A|hl+-X)i-=`wk58JkEHKebd$y5k17xnfw z7-?*jB+~>e*RUAeOVvXb%er~-3?&FxY9#nuaiD&^YRT0W2U7hE8V+7VaPW!?Kk zi17^2<&5!lgd{Kn9oqw61N4iQb0O3S#HCyJ`dn8&flY8lSMs5hA`1qK{xt5ZR?EgD zXa2*%TGl8Pu7z`H<*!#ZPN$j)B`7b~80Da2)-^Q;R%h)FTj{ICYFP73TjI99$=sEs zr@U9ngOE=bok;cP%P*l|XeaW+G;9GX+mFm%HgzE^7%epp9(Rk;@)y*MJD<0g>shqop|NIuSQzj9NqwdHnCcWVCDq`$eQ(MsjwRFPrZ;9@b0#%yPsVTrf#0r@O% zJSP6-?^#J)_Sjer&nMCyHbsm4*n6VdWthE%>Z7N39>)tBYwHfX8>^_RSA(>y0fsm} z$Q!S;(9616UoMIADQgO;bKa=wyWJ6Ap?k1bOvG1Ny!5Fzv>r_A`w?cn33@9&-*vWE zt=12!%J(N}uAisd9)GdNQ(yV$VB4OkWni%N>nqMDJ`>D|@`~%LwVzkDn?+1`a;P1U z@yFOnK1o833K49D&bl8+G^k6CxKS1fvAkdA*!jrbK1_V&Y}Ksi1i|f%Z=Ipf0ANPf ztG$J1>sgF7($X8YmP*ChM~soKhNb4HBDlFmhsMUvb;Opc5w|n7uruxRX@{ud2IU9# zzS#)O4}I2;5PwtNcun;udDA@hgN5=w#;poWBb&|FMLw^oOBX67n=Ocl zwc0L8@>#Q_`SgismeHX03d=FbhavJIEV~s!Wxx8FTC3Eas4d>htcxoDZ;To(43Q#P1(_OovCdgN~9v17dN`dtqqZx58b!+eBgO@bCC^8*#d$>`)xxfgi z4fRTng}+7Zotufd}8q(Kx@o{ND=s$p?$iAFKTXKWhTT?e)pPdq%v zV}CSEI#@Nm@7jd#a`&Lwc+?y+j$Il~=9p%dl-08A{jO8vCSEjA7q5+lC4=Hk`-5_v ztqt6NV1A|EL&hIS6fBQMaE#l@%O|lZ{?j+vAh<&G>dpu#CNEV@K&Fyn~AjTI%rl=P}$SdyPCt zHOnx&Y#Mj(TD$!I5zNl=yfg}o!s>g%Sw}gRQQO1J^a2H@(q8>}rE$U=_Y5&cs?!4Y zIPP8)dG>1E?#!Azaq*j1U-kOEwMRI~f#1I)=VHyvb8>!=NJV|~*cyBimR7|kGJ{CI zy!(09LQJgPlQ4dwuG zvhR|P0)eKLdU55rH+VMY?S`+@Xr zU?mW*l&@X*i;?k_Oe#2S(%=2&DiH}we6~ZMUQ4que1dJ2+uYpB_H|p)=V)oC9V&5n zOWs&)x-3oI>Nj9p1>D4@NuQz$KgCnL0@BiZ7uiNDJUrgUYMCxmo#DdM z$F9!Q=j~ozd*#=5!}6uGv3So_RdP}=VPD#ooS__P?(HGgRC)_q?>f8n!6!%mGB z@tK0Z{^S_=nV(nh{K@6;=e(I&^ZshrK0dSd>;n_^yBXa%0Kx z%Gj8lK>TKpYy?DE{irC@h#R_`TE>@{&rW8$E;9Fh4{&U4E7jqAL*h9oF`jC!P;+tl z`0!z^X+;c&=xCPnY>CS+scPk<_U^iwGnPHyjA?I{_sn$Y+>o;#NYF4C^FUkD-jbB; z3Umw=JD2K?OI*E!%Q}f7j=nUk{E!}T^WF_P^UfDmt&uN1Jan;Txy4RS@{!5*#g`5s zS3cG$o|zx35{)RGSoSjg{1I)|q1pJoU|_gZ*GVX0uj@c8LvTW<2;xDGF~;m94I!-M zg}R+x0NgZM>gs-HSn99V)ER{HIX)Y;+7)5ZR3b|bMn`i|m09(lSBft;Vr89n63N7o zw?9O4iRR{NjOKa~9NxWs$svg=I(4xGjXK~@6bj>(^IXU+;*VZ5K5+d2VL?3SfQX_2 zmmYh5FSncU{xbHxYtEhP-E6!Vqp7zy{;o|Sdb9WA-9i=%7Rg;X71Cz5mw}aG^;>o} zw)NNLr50-Lr2Sdq4Y_O;oHlVA4;DxFuU?(lBadJM0ybDIYG@W!Q1qEp$BMM}nZa+C z)pcnni5XQJ0e8Z+i1|+Z`UrY9GHh*J>EiOpk@>=kCal%K-cKD2eGVkY2%|>)_#z?Gh&Pe~JvWc* zysRviS7>u3dA@dAmhT54_Gt%i_g9xUB8=$yY822jFQ1UiU9f^)>XEVG*%-0is_I~y zOzpb3)`jKN_A>9pk)(oxvi-odi9_0rWY5Q$L3JoxWbzr+5y9B%wdKU*T|@kh+m?$L zwl4MQrG_11=Ww0dt5T$VTllx$z-Io%zYAnh5}#U^eaMpjTI9N^;1FS*=YC{Q6jqvl> zb{#`MSuL>{BTi;B@m*q=Ub#i7&5=xYbkux9@``%ALy?`}t^WGsGfQs739#lQ1>zA{VLay@8JnMlk@$?_pE4k8t;$E!aU zZgN!ZjXE3W^?iu8l1!1=8kJt<;h zq>59mHxb_zXS~waX&SH~#wnl5MVf}1;#ciC_?hVnvUay-%_gak$SvLV0R7g-orM;% zp>|x!h~gTtUlZpf#0tmpiJGNnzFLjv=f;(t2v>QfRsK!g)R?Dxm&y|UfBq%B3IF2! z0nP$F=Y*n_u<;<}DC27vw0YXgtIAOPYZ@eXR>829apGIO^3yNB6z4fTcwj^?S{_xu z&#N0==4xF6r`0T!@W{VB(%)aM#y#6=ZcHd^uHd0$W;ga#iXY z8?HO6)XU@XStY#wWD@R}PIU*R+vVO@-0v>4WIE5ply%RXmxGrQ=b)G`C^BsL?I7ej zEng%-TuSRqs&hX5QRs8<$y05SA3nHsElaiw$MvM^&v&7iGj)u|S+h4fy9S^`>aTIv z2!@O5w*2P&-%qtnOmgjT+yh`MKzaVmjIrISwe=FsHo}~Lxt&tc4xv4k$@&G${YSrM zXR|@|(E?oSM?OFIuO|(V>hxfx0ZcW-L222#{o~z}Cr;eGV%c%(^l43)F1xP%Y7EqS z_AJeFYr*s>J0Al*_LOxuw%i1x{D^KGrIz3bC$ z{7%0gQ!|N_sqsAi3z6d?X@D)P{3zFP2x0M9ufneI$SJxxSaQ_{08v_8R8$`v{02e7 zDibDA3?Ki7XS1jTb_-yQoQb0DMQPEAiG`rOWko4|R-f)^BdGNi72g*b)#Y*M6lr3) z4JuQ7kI`8D_;B?o+ap&61qF?V58r@kp7Hdh(%}M_AX&m9@kwgx!bX?-9i_uIl#2k| z7i~)vP-WOuk_tK#MF*Wqs%N28>j9mYtMMH(1A`@|3XM%oPVb@Q2>H?Ok|@f>xkR4H zKa98@tHeefu&R}T$*zCgsY52r^!?>ve<;aJ$(lCh7OKmOMh4x&eOqU2TFi8$H8|uj zAAQWa{MOf9gAS{UIUnfbMk#NR@_4oPMs)1oFBd=@;aZvVtc~~|K82Ha|bK{q76(>rmeC z96i_`o#L~?>KWE2aLBQ9IM4lRWQ@7{^{Wv9olHDSzmU!(YG(cQ4cCPrJB7R_cSyw8 zgDFYB`HPrKDL0eHG}SPJ1a{7L*<&;ka0^dlz7$jO;S($fpY=`U z?c=eD=KbBCy=fj=1D|4$HEX7_pG6PuSrS$>ZHL##@nBI^--h=)d-!m{NQHd}o3cSR zC`rqO^WF>E+8&kDk^s3Q2itKGh166Dtq{us%6$yiiL_+y{lQonJrSrnceX7Bt~AXy(J(<@-4yqGaBp#MSTQrHo=c9Bo`)`8bX^Jx zvU^SyzhgDQ(mb~`5qWF`CFTEj2MSrBO?no(ZjTf0yqR2?`J1*P&%6E0aN^e7f#7P< zDEejGR(o=j#TE}?VgI%Dg)M8`3+;Ut%}fQwjG6txzF_Ca7 ztz*nv9(QWUtj7`$uF>bDGjLay5moR{Y)9)|@5)yAtd8vX1EOF(VDFbVew~2%=l%oj z&n3MpBw}(>c=o3JLT10W^LDLA&)$053%OqNWW(MP(=#3~v18A?%}Sw2)eL4L26QI0 zU?=Jg*7 z<)XXkOZn(aKl zJm!I)&0Fx~*(^;KPXpwr2om4Mn6BPJcXo2pV0LmfCg$|X^%xOAts-o}qG7hCgBpBB;r*mAqd^v@3*Oyr?x-JbR&QAUM#cK;$i9w(ebF`Ah zb+XRVa5r{G0xn}^r6u&&N{z&E%6V^=)Ry=>`SU9m<*Ijl{5)AtSi3W80rRqAsByaA z9MO-AF^_Q}Jf9@=e;w1O@umb} zXBqRZr2bcjsQ92Ct~kQY@CK~s7JoJ3liEP0Aqv7a%*UuS05e6rssS&8M3JrWs!}Xg zld2DaxY05?st+FV3*D){U=YMvGj7x~>mXDHa>JlJEC<{}Rn_jg;_HVHKbvh)*w$pNBepr?sRD$s&*g`H<|pyO>iLTMH0Z$-3ZT)t;!(HMB4 z8su)I#aUUUjlI|2WvL`{N3`sCTtuW~ee5uU5N`zvGVGlDE954~HhEh4+JQi-y2HdA z4VXcHzE0GK4=IqKbc8z_8Xo=EW>;<>q7H$729JewAT6mMIda5~#1EuM@D} zxWA5gqwinbSR(i!5U$6|temV-y8}OKLmXlf%m@8`dk>Q(jIX^KiLm4tY_)ghtFwJI z^4A#hjQYPx1l~5U#MOU0<(y-CTS2Poz4!J;aZT1sLjzV5V=CJD(jD57mjw-sWL0eF5d39zuOsfqumu2&|!V-vi9C7}N%P zGcek7GBS4I&gqGRx<#*iGbS{&nS|_-gaWu8oXxWYYyWXL9R&`z#?#_T6)@V$Izfv{j$|TN_tpti`|=7 zVeyzN#+zWV2hO(i0-Wj>xy7=LNmeOPebp@vDJMSML>=H&x90l_n%19injzO)`0B`e z7HFG!xFsp%NqYX|`xRJ`~xPr+#JLvL{RFb6`+>Rx=B} zAcW}=rL;2qNL46DYs@6EA8CX_mO*)*$2eG9mF#msoD^}hNzyCJ*(XxZQ&JYlye^E_ zSZ0!Ze3JHz4;Ifd39pyM@l_G=VKOW7OOG_?_Prd3~J#o)hZDc#Bn{hDEpEl&qE6WS z%D?Rwl=|91%457ZHl`9va{7>DETBeCZQ?#|zqQuAfmhUQueI>-)FB(#ZLO(d zESWYJMl7v{x0=U4PGqUvtJ|J;p-$-6oggP^g#<11NS&Hs%%8_Fa<(DI{+?a7@Bb#) zY}tvT(aBFzS`n^yEzXJ~;;!=@Y<*r|&lmAO8;3qMz>%MoJDA~C@=}|OuJHQZ*=4tx zGeD>thqtnjJgB6Dms`%;myBN7AFEgy*Zh1S@cVV?pdjnJOcC_;sx&70%PDT5QevBK zM5V=?59~&Q-YDbeF`EOD%}P2ovI|X%pIA5q%G19!;29er>7$WePQmzl*S@Lv(@27H z)B@K5dk8CXkFUQT>Abn#hQwFZ-AtNG8?Rx}!T4xlik5n=8vy4p2im5w&{jrbP`{y1kt{YQJEdH^mghswA(xb7M$I&$ zey}0g;7Z;vrHS0|DbRB*|Ao6QE0g=Y`GXbiSa&ztZd|WRoRDDFnUG7FHTn2AK%IJK zK3H&~TgMm)Iu#`&G`hYb?kh{>4(6t`nH2c)RZm)4n!yODn{l5yKp@_m03eyK!^5Rp zlF{2ZNGbF)6=D|JWA7c!phEB+t*gBKen)FdOAieHE?m5r z4SGctWRNWvCyY7A2|1?K)~+BAJRi9&kMMKpmlyPU!7jra2sqxs%;I`|da*l|0gwvS;bGPiy^fP7)&Q*gx-cPb08qt**Yz!_AUgZS zreM(^K~gj7bRTBFE#;V}w~Ds=1*Sl#`p6SHYA% zeF#Zs0q%F$CXHzM9{h~v7dH1aE34_;-qv9cFE6MG?gulC>ebgL5w>=Ed!3FP!{nx= ziJl6xhp;CXF)}{*ytLF-#Cyk&f~&ffE5SjRmfFcxioZh5JoZq(_`-swc1tLE#7f`J zGgUpDH!|MnmTZ;AYb9hqIqdk6w~}L@!9EIoHYZi5- z`TUr|FGWzryNI%?WR^SazI(}$6I(p89sHrcIelbX!;D9V0%ZZx_Kh=M3=-zwE9nBr zxZA5&lhaR@*vgAK>kA8>e8KTaA-tAUc+zgA_{W^XuK!D&<`~tfKqm#4=^dr%33m0| zLd8Rq^;Q6ZKGZ)dhIMOOny!xfSX8jM%idlt7w&zy;AGHA&9)+w^%FK@5x2EsmKKjx zy9sDiRh%^CVUOw`)Hq}!Vbv?}xI^^azz*5#t*52@qCZi$${1wreBcx?hTl+1o_Mf0 zs8T$z%=6@vG>6WF?wI}ee;zLn-^)%bPWe&sPQ;)U`%N3A+G-0f2eo`}fzPqD9W1|4jiouod?A_^oGj zY>A$#nVHj+PY8f`RX4W)7^7k#9s8ZO%mrF!+t%+02FUTka=k;J3(-j8(lO6{$@&mD zPtXlIHhKljbda!U>l_Gt5wg@1E`|mMc>q5zE)6SAPY++9n<=DV#NID6-uUL@g>P}> zKY@B1irgcu?PV=i{P|D{xUMkfLF#U8Tl|__QEbrCGiZ#%7V1m4wsC|N7iU8aqyvKk z80Mm|i(A#J6WF=r0G4VuF5?p?1|y@RdHfr4%{sgxU44QVz1xlF$#0wQyDZ|dUe3!K z2m^tbl}FZA_a>BT+EzzSpHX=eurK}ds-va7V{oja-rn<@u4N@D|bD4_t$bjJS? zOvdpb;jO0@f6qJw;;;<>JYNV=F)`L4Wh?*qagNKwHYnpuEzTlRMvx$~1NWQ?$4F{w zswXI!omPaHYZcyM)(7H0Q3(l4>VmO~alEF1u%u(Icn6*&<#i2gOagazWj(g;Mn{6M z21o@7-nxLmCc(zf?Wih5=u+^-Wt8nDsenY%mU1@jqar>4?jBh>24 zt3#AIQsdkE5FQxe!~f)lpIF`=Q4XgJ;3xnSfN(uH{TT-#m!Dq_Ef1HYkD$M5(|pNclMNdNP%GCJ=4 zlC)(Paxpz__BGWYpUu!h;PyaaDW8vbJUx!*iX4%UeVffJ_=e?Nck~DbWrw!)^3r=9 zVyaocDvU^(*|Pps=om¨SVqK15p+icX$Vn4UC&>)Hhi!(XgM$`e$8T z-BytC{HNTbEJ~-83szrXgC(EedY}>u3>=WgD~2t) zKRF!^ie8ly<>?F-K-sw$TGP|=?}(FdL>92nNMnpEK!@Hb0fWIDrC~0RW%iKL=5W?X zHK_(^QlPQtFbLXdj^I8&msLs<(}mj#%}{WrwWFgSPPgmUf>wn@et%v|#Lax@dlQ0= z0x~GCtULg3m}3N#ocH>ReHxtwUwL-Gk#^Aza5Lc0loM9SPj%cV`N&tEnOyDfxwdcA zW;1=J(YRAb{y}b!8bUKI3X}b8^vI9VX96N8|2@RIK{G5Bs-Kt<%96GG zA2yIy(EIl;mf4w^lp@VRuuu=(8_ueLt@CoPs9##LsUu8y!I?>BJmY_89)X|^E-o%s z7&R+hW28ddr<$FdOr=zCpigFp1iEQ2uw18IO8%yWOQJdsC!*j4En6kFwXF@RZ2IZn zzc10AL|6q3+YCUBI6XBrrH(7Njkg=CG*M+AQ_|Iq4Ph3~gI-;OLBJ+weIglXx|GkC z5*stFKJxovWopOHCkEZRvCnSi1d=c=AFAu);{$Im8fdKf;AJ>M^Q4)AcPPYg_X8#{ zz6V>^XizQ#Kl>mr&qIhD=u++(Qi7zIJ5>$=7;v6%Jr=9Dzgn@(L7DdD&N7wwmciiD zEf6c;t&6@8DN?rZut@!+Jix;#CvP9g)N{S6b0l*NhW`IPEX3Mn{w?1SU&alq!oeZ_ zBwo1FjH9pCLq9ShK}Su$wo*q)Cf_+QP~LU@MH4cZM8adV4aRl@VLIm)g^{E+OR?MC z9_56VxZ=X{MWeZnjw?A@Fs+ggmkG6lZd*@lvAm9RpOflUSOMCb5w-$|9Gpv?H{Jq} zEDyIgQTz;jDCmdc1}n#FG&VPq@r4$!)tMJ|J)gjT@>R7pVdXq>t3Pq7G37xAKeQpf zw1=oQ9zXsExRaT)NnXl@dRk;L;&KIUp%_KVhpmhZD1-HspCI=(=cjrl;LZ*V3Znk^ zz(fTF20B?9*SIMGefel6@Z5tdDk>;#1fV#TTocnSB4{sy9TNP=;D+SCbrq44x-4k?tSwHQ4$wJ_~My+#jV z1ho;M|9}RqsB`e}=l~xNq?q2HI3ftg0Q91;4ED7Y{>1H@H?w;kfe!8oi#D6B9K|`P z1M>RnYR-7deJ;4V_FDt`_VcTg4NfA7b|lb7>1*S`6u?~~BN?h8Cl>${o_o+*RMnO9 z$0`CKS4qeDBn4yue9s+5ydp-$#?JSiId$sZtrG}M3?g5;bmWK?6#Tm{`Bn&y76#s8 zigCTI#_+f7U;3W8B*@%s(=MXvn!Q!`(R%$=HFw=kC*C?b822Z;Q{L&+Q-S}HBUV;c zC*!R~x0%F=OEG|@aJ05K)`dYtzlPkIyu7tYNF6A~sH(B>X1q&!nWkXd%&%#bWLW1_ zjQ!oR2mm<_L{)K_n954c(Q;Pn-V7l~8ASGl)YOSR0*Xa_)bVwV2uec1dB#Q)_(0+3 znOD*Qm%W^DGPq-9T;<37zVS%a#+ntby!aT48sb%$6;X(haj6jKa-yfMoxcz@FNoRA z(utttkHdj}2MS_$k7ggDuaD-^xBB|#2!*}Ep7^RA!Y?EYz^4S!_Be^C=xF*6>yzLg z_K2>{o%T~?Ow1yjlcQ{uT!L^93k&03E-B%KT+^1q2OmmLsksWYp5K=*am#_Nct7kM zxz4m-rV2lVpg##~R~3{)bVv@aw(ULmOrh|CZK$~ccgQe>x}`F1HRG>dP(F(Wz_|Fp zwG>{%L}pT0SZZaZkTByJ-*9L?=digEkb#!eDFZXuyN1)9|MI+*FVt1_NFuNii>(1? zg}wP=7X#4P;e$2MKhzs=4{xjEyZx2Y(EfBWYB8;{Qg8c9j1?cSs}Q=#4wXw%u+ zZ0x)*eh^LUjpouRGi|#dr+)vH*43*pN-332n+0cp)-<18yZv73;C6mib+Wz zI4S&l^6SQ-mB;d}-72VRSxM47GxUbHAl!Yoi;Wc+wlTGvLXj390HA4W+x*wl5AlMh zpkiZCN*I!{9X*mR zz!4BWmf|&F4P&9wjcH7QEja$CUl9L6;T&*?-o1Nw``*3MwCL#QEGVoleFW839@u-+ zt8_>Op^+9=BKAHH!EkyUA3Ss>#X8 zMNR3I+vI{~T>kUs5HXe12hJjilxyI&GNwC^f7|%12f`x1%bY5>$x)pf;DX+RNWc$* zM3q62<(~h_c#U;yyP%GbSdmc z+VQ4HSkKR|r80k0DdN34y2zBCCEqUMVi8Bk4` zJ*{rKE-7hjjNFQX0sM@U#GF5j3(`SW5p6rF%@&bMM|&8JhiY%6(lNcax0ij{ezd$W zOEEU>)2B~*%G!d{Gc#6@WUD>5^7NAdmIG@S3Ux6A4iNSp954!m$ZvFju$a5%Exzf} zd?m4!y(ek$9Xq@81o6RrrkO>P-XC;63uhM^iB43BEhUzH{fC9$pE>+fmc#XNvQcLr zfNz0kwgg1Z{5dr68~?Y4x&OYdtO3sX{O0_Tk+S&`3OQzDwv<99J9$zb*n5;Q2XI9B zpGjqybu46%N%0I!!*>8oTNpvJ?)?f<7{N#?WviI8^Yg0!%_S;E^aA5j=0C?JDfAZ# z@dmgeF9e)>#22!Fz5*oOJUBE(=1BoFN6+#Jk(tfXuOsfK&){1eFS&vvwu)6-G{;Ek zO|5`{%ks*dTEN0_^QwZ0Iq(G4k|mV*-E6GIcVFo267_>aVBQ2%sp(9RT3ZC^j~=-B zqCS3{6JQe;H)KKWe}-DT*m0_weHBw(%b7u<3QrpAhZz>bG$ta zF|)}m3qzmNVHf)Rg$r6i^N8mF$5_F=M7iA<&k$JCk>1{XCg1+rS{jZJjwi1`3qd)3 zHUjbb#&B$(J%4`T+_`7>c@c=!Pe6bfGot+yqmCl1LLnJ+-LZE)S;9VP78YVY^qWuQ z{OT*k@I8SwZN<06wXx}gssR6KZfx0o zEwfYq`*Vsm0K=rKw>8d4fP$=^xq#e(aq%6tJ}6hdJgNe0r5t+LKNCrcQ3pN0RU}BA zTHhV^9Y$Nnz;$CEk&uvp!5NAj{M6bsB?DZwHgY8L*6yg=j;_sFS zxB(-U+-mfVfi}$d|Izg(U^T97`}i_XrEQy|GE1dNg=X4pN?FaCWvrEoCX@!-&?ZDm zlxAr)lP0CXm?WiErAg+rl0t*#|9LI3z2EQnKgaQXFFUQ_dG7lf&g(qSd*>rQ+}Rp0 zX@+{d3sV|xvQ8?Cd>h~M#b;(o^u zSu%B}>L1pHnA*7TG32+e4~*zL-akj^XCLOndT3bi>&aAK*|MzHqeEsKW6s^C`w|z6 zcj1>1dYlo~LsmP6hr4rE$b$@3A%kOCc=acy+3+poINj^7%suEB>~`S5O)!%jx6Gl} zWdDV-ZC8e=1q<1hdLHF7$OWw8fThp|IkjqS-u#@3&%RDuDn7ZPB9hn9sO)3@i4tx$ zN=qR0EAo4z_HOTgDvY^7auwy2Ryupa3Vp83Y62oU<}lw&2%-E!|s|mWHdC;Tw)$ zGLB`8VLOg?>3aUR$OE=DHD`Uy zP6m6Snf?@4iDUitbm!>epJF`3E8*!c8s86p#Cps9v;T72+_6 z7(oytwRMaCKF7Xur@<%Ca0)-aAt-DXtmQgarA2v#Z1$?qQV;-K;$yF07YrPaO%G*k zI{e1Z&XHMlx~vuYfsJP!2^O$6$-2%cWvGV_$7Wtfi@G z0uTdU$FY=FmEd(MnE)V3W?&0pFm8&DA<}@s{HCm|OohHlvQcm!sBl3SE?6@YPA)_- zhOJov8u#ij6_3mcnOS}tsL>?-8}^a7vhtlv`lB<-(DghxHQl{6IZI5X%vK>3jd09t z%R}qz>oesSSU0`}w*uYJN~?VH+S*!&K{$+-6C{&hC_bhF<0$T>-nwwLK*^7L5{<>$>^zOZU-``i0Du1IlO5joDlzdpbiUxg4>L^`bj{Jsy^V;&^%* zbX7h2l{c{-4}9&BnNbwqu{>j!(vNABZ1`N0rb)%q? zTM}+zYIMpa!$|8rt4n}!`KQdDO^tp56DHNtxR#fn98N)cV4X=ymODx8j zKpd}LogX{|m*d!L*F+T*jL_>nIzE=cu|E- z!TtI9QetA7$f$;ZTJ6@FGzML~7z4^p?wyT;697Ph%gM(28NUR}Nq(ugcuA~~pdbrT zfvp7KscxKUl|rHZ-dk-5c+9nKzRz!#ZyIM-D){o=R*PfFN8Aj2uVz~$wb`E7Sj1pz zWyZ~XvM&HJufc=Rd*0gt-_IdmY_Z)n$# z(xEQ*VCwv-%kGV5WTRzqwVgeC_B^x$434}}q9IC7L-bjVefa}lwIFWS(uu1aX|>-1 zF_0Jakc1-8r3z~L;&&{sg`bg&9{0^{d8`yE4d$T{>^w6J%gFI~ROw!VK-<}L&~ z3?IdXIKV5`!qPG&bhTqn)hI@nKf!QMFGpdU+F(73p#c1dP7gx`e@UzSB%pJmYu3ET zj>PXiEGjCB#c*vTL!s{jz3NF#jWNbi8Cq6g<)Tj@aqN}SJp_OuhWpayLj^&Aa9VA? z+0#$xguvA`#4w7Vx(TqYCuwSH8@j{ryY2gzw=6US7<4&oPs=HJz5gR9ySjMYlAbnS z_z=k3lrQLNZ?Xael5zPD>6Zw+w51yJJ##CDK7O>(zAU|tw%gUD&-fnCQIZ4(2EMBX zd5;)u{35DXo$p~}qz=;@fQ+v3Bb6Rk(vOFcaD}fDDdq`e(;^axe%P9^kq@-#v zO@R){3+dT#t>l6Q>KZ5cbn)zWXPDZ_EZlwvU4wm7z6J6#J6#PZmj3lEy0CMls7TPs z_wR?$bmY>3cjAceh$p(0wwnxh%)6&N|0-q?PW3O9wh2Y?@=*2=({(}<#T0{>tzjlX~Yo}KPTs5!O| zR~FbEf)=)XRh#DrER(J;MIzsXX->jkh6q`S{@9#_zBuB@*Hbw3(4HvzU0G}m9(KW%+v&#^p)=fs@emkPSfFGmi z$_>Qv<}1V29JrUo#cz#;;kgl$|t z%IDGj9XZYe<8WhGq39Waa)1zEOkWx)`F!af=eav?)uzLElKzw6sy%05EN5r`W+rYh zhT(Qtax*hl?R{p0W6A)-B?R%7#s1a!Qj&g-*REej$t-KVlp!Zg^~c6DOiZn4;>oK& zP*ID4eG0Ct!~%#WF4L?lHz{cgqRdyAkBUyi(<;G)aXD~7obITD)_Z0b3_VFsx>cns zaA0tde+z+Fa(sXV4N*epLwkBU1Yj}fs-5uNyd73E#^+=-&}zgEBdo3sZ73k?5cUvK zc9~0$!TR|Kn1*^@gy6*avxS9Iuy9=4Us~*0)#DxWrTE#iYV=#=ny%SzmTnUTC+391 z-##EzI+TFugpU^#5b&m&(&Weh^v6#7pBz^Z(tlViGc4}wr}vt7?Dh4oYoFM?<^1W~ zZ{MWjjW1`p`+A6%vt$&feqM+<##YJZc#gHppA6Rx@fv-ddg0-p=Oi&5&%MPrrOT)9 z;X4<_UW0i*{t63&H;uPybL_rAZVZrb z!PO2|sTdQ#0>%~?&_lXS1xZe5ce#KylRp9bW-vZ~K%k|P*_@S{*Byewolg^r72Bmt zm)6}md<(~CwD7V6XuF8MH#W^>$gRgXr*E*`yIt??AQoG8FeafkKvdJ}+a$YaxE;EB z>%xWRJ86X|OT<;i6HbqE#5q2TRi%l}>!Po(@48}O7`1I~dEcHX<#_gIY7JuJ8pGY5 zU(>K3SOye5)r}(?u7H4g4k@&562m$DD>#Sek|;&2YsIuW|aR6%l>s}`)0Oz4ffdrOF>8P)0f)tn$v5Z zd#*{+h;_8z!${6ScSsoce+-&rlc!f}5(Oq}3iWjyIp@?CRAxUm^1zg|5cyR78v`n9 z1%6`xY_%=V+EcIn+I&pEH&V?lM^Xk~j)B#2G`984w$W$L(a#MYW8{J1gbOf2u_70M z#ek0u;i7c5*(A?BgLFI3&z(M95xv2Wb%fV?@0Zev)5mEuB5=(x*9~3ke4SSS;H*je zHHbx+D;m7Nw;tOI4wNZ_+Zl}eYXG2hXBAZS#GS?&emCJB#J2@|pL-1ReV+0NU946k z)Y!ybAYk%G)BO5IeH(dsU~>Fguf6tG9FHu1YMl5exh*)?#l1P~pn*|$aSOtEKO6|_A>cqT8K6kCaSMrvfjM4cA2ltFuM)c{&!+|*$p2>8QdG!RcEQx z%4~W$>sd0@Ty$|KT>RF5OXt_~;zX;|Om-E``_6k_JD96P&hC4?mj`YXYU2ywIOgkc z>eKAkUUm_oAP%IECuON0EGxs64(iy|q`1cCbyjFi77$3i=}GTVIB1%|3IY_vd@d}# z-G`zOngDhC;{Ov$sFnHKUzkG4oc99W-;V+K5Uax`(MJH~agDfskbz-h=?l8$fq0L@ zhiy?I!4f4y<}TjHXZ#Z2$?bQ(ox@pQ7TlPPXDiT~@N14^$hV_d1nMef&H~khbMM5I z-8Z-KX&O2^IXib%8C$jB$70BHPUaM<(JG^gD%mAlwPStcXHh&BXu@3IJ-(pXKW6l3 zn&P>(RGetKD6$EqC=R!%Uu&@$tIY^lIyM+f4RDThL%a8|$IRK@t9_erbQolr%g0O~ zrGDQ)?{oPH^W0~m@AK{b7 z2`RbZ7><(IrXQnF@XQF2y}s_)_~OdSTXoJ*L+XOzASNM^#9j*10h|V0l~2G4;}8=s z0}AZ=4N5ClmKI_lS~p|<{G{G*&Eg9eu0vuEs_bnHtVV!5?eAan&~G+C+a(yK&AU8< z!8D_J0(9|KiuCeF)n@&J+eRH9Il@mY72R2@8Ik|{($c6 zh#s}_wk-#zh2v24sF1u0i)ex(6=8J00dFjh_x@5^mVDGqvLFa%y;?Y+_FvGza(WW{RIV&;2qQ{s&*j=yLkKarH1|3+l zA?R^&u`V{Gp1yu#9~mNzbTotuM5chlD%tYP^$|+hN5@C`Y<$pEnKY#B$tj5Um$l!m zc+oI=RqfIz8A@l)ofKK|m}TVhZ*a6m-i!~WdS3i@Z}Nre>Tb|6wsvM-`ZW+(0JzgFZLt^-()CrMFe6K+5rzM5plZRh6d{!=$-t#G`-MIpMT`K&AxbOk78 z>_OJ_nTEuUaXHRISbDn)4G#KNbDSO#T)U4ZsgVkhyqY$H6=&lwD6fehtS0*rwm%jD z*!ek{vnL7zTsz4(chSXDcK&!HAKwE=UufD&V?RJD4Cc%Kx3VSPo0{$0*ny$KMjVZ=m-k|GeAC+`;m1Bo2HfTX?!c^?E}ZR%9iQ#*Oyb z2tEgLnfYv|ww%j$HKlK!H2=M>@%hvGq;q#)a_8(a)EAjj^SAth;=^vqd4bDk?Hb#$ z^YRjM6+vUM@$a9zE>ctYC!-4Y_H3|k?X~?PeieQt?J9E^9n6qJX78*&jNeB0T?F_6 z^#DW}Y6Fg5@$ix(<|qEDeVYK%)xhMMmZk4YN8lk<&X%6eO} z5h#jVz@tiH14~Z8P=s3XId@A+v`~N=6F^OH-?&p(b>g}|Hzp$Z+2Jrsl48@8AH6NL zdN2v&wR)@2OA}|7Qd%HJcNJMXi>A(;HH(D_vBW(}49dq5-a#tRN3)X1rXRsn(lp)X z0v*&&P)ZEyc|kxvI8RPbS6ouEI>)t@L4VYG5j%IT8R*Y;DQc>TPlNf&%vyH zOR(DyksAZOUg;4P&eIx=`yu^eWtJ&gQ2Rg5Xk~fDQ=Cde0cxrn7lo$~!x+=ItjAeVPCB@v=kzC#Yskv=WG& z9Xt<%9KkyityF~p^r{bn{R_{4I6IsNS_g(;iuvs~3S^v&(mH@p71+tGke2R<>aNAG zhWAtl!Cp(aaf69uDUd;F77#Nf$iv_E{7jIH7;2*^`4)KP5%shW!3fyDEO87~Ji6tG zt2Xj9e>Uz)5T@H!UdN<=D-K`S-=c)Z45bhJ9Q@N3U{J#7SWNf}Dn(%RIY)Y(HmEFK ztU-t}Kx)_(P(LUi11cI}Md%|S6h z?Z1or81_4F?UnE7#yUersMo(OR=yAG`DU&IPW6RP3$&1 zU=)I2BqKw3aq`hVq@Wr}Ko1@OCk7Xk-~7VL-=p$*!;3=%Re;E1Ll0P6C+^}nB}U{l z90OTB(EE+BDgeSoyD!8A1pEn#q2+8TOEBgxzHe)@->%0{cl{sST5c@XJo+0<-y%>$ zf2e>!=~{I3B0|M*e)ssLz_uCO)qW47@epZtj_E3!Z3X=+xf$YRdmBX5kL?6Li4(*d zvkz$C{G*v&D0hgK!)nUUJMdcUwTmK3skM|p!6Y+0_pQ6T8Xzh>F4TL!f9Fs^OOhs{ zUp6!}X!iGGUnB8+QPQ6TaTd~#E$}wUnt=fgnzKUTB;x5mpF9|NlvY$>B)e@mTeU;4 zdApSB)ckMxl1-K=sBuoK)h?ZSY`Vhyg_e6DUrqWxeS3yMr>DKR)4bu*)lH9C87y?$ zM>gu0t-Y$_8Gbc&i`i{3p5^|?vVY&Fyw++}Jp>lcbO-2)JZC-bCwGytosB(<KCcSv5P*8YK!HPKm0@MDv`bI|A(fAV?i^e{L94ds0*r*|bbE}g>(ZHMN zYT=AykW(&iqsQwLg_6Xg4)xuWXU~jmY}o5gq2@XY?kF0EsuT+)fY{=gY{m|wHUU;z zg@cW0Ms`eZG$Wx>V0OE>m{Y_WfCsPEVa;qqr37h4nfa{ydAP^qhE*D-6ujX$!jg{& zDV+b~$+QzsMnIqj&xRL{@8hT7_K{x(Y>2X6d_`a-PVu<0W9MA{3@t+xEnJ-1Ol!jF zDpt@>&N|vpz}xIx!Zl#xOUk&QZr>9pcmpcTCV`F)o`?DxgAZ|e`KshHq#PP2;imC# z!cE{R0+v8q70go~+cFpeiwaq~F6<4@qh1DFQ}(#6H$^i@kC_0FJIBIYvPdO!RI$X0 zhN9>X`NFLC?eU`CTgREMG(1;UTAG|5e)+N}4m1U{e)RS(?#;o~r(IU0axfDVysULN zw$3|d(|xn=bA0{>$sP*7!Ga^JzE;qzj+d zl^Sc5O|h{l7f;MUPD9EiEeQF8P_`{UE=^vbsNlux(4s@?eG(~8*WQZv9?EP0GSGj- zgFeKg+F?dVj0o;&sL=_O9N`gYSs~2fRO={=X6c`b(1XYRn9pQ)AHzV(eW99KQ(Jx?imPK4 z42JN=$g1d%Oh1&oOE>SH8Yg?Y#N}4@meqcr2Z}`{9XZY!FRy4kkDh4dujGA1ne(7X zL-<2~sM>XeO?-R9ZV>=fR4J`>+SN67V{%Tw-Wubq>~q5ErQf+mH=_qssuaqpogKNv z_PZ+l{Ihi6|CV1d$NRT^?%!yNpGT_RG)<-=f^Y^(QE1fB6iDs_ew~bpij6{wsu(=x z16K78mDkVybLE6U=dLh>>X+sPMfk=33Q|{YG-weHvlkE0wjW5 zM;KYj`i%gW2(Jb^hQNC)m@yYeGlWl~;MK~7c_5eyP^l2{h=#-vGCbLR^!1rDd^AWW z_y{wQALuN{3VDNQ;sAcFbdiBkuZPW&G-pE^F@{=#IdV4VF>8c)w)eEDlT#Y9GyDMl zH7(TxM*vBpB{4N03JQ-4wSbS z4iTX_ZIV$UnuxD4J;&;dncVW(#)+y$Qy5jR{StdkIZOW<3cq^m=cecIFKfa0OF1C z0=s1gzZZv$Vm!)rZM}3!(t@kk-<9fJy51Ex%rrJGt`!PmC$pq_3@!%ktc&s&HY9uDdjJ_%0q}X?WT3k(q{?Y+ z3JSi{r%vf+IW&H68223xy|Sd>!)8AZC(qU%q-`v9m^?YAXvD9FKT~z86%7!0k|_*Q z#bjmEer)4rWD;an{SBcWnxIxVtqFT_ODWJ$&#T!#*?e5=x%Q#JweQ#(i4|A00}6ai zhQgZ$Cv|0ta$287s3)3y5L^?tA0-cpm!bSeSUHAZlO%TYvAw3o!Q%Hr3yUXAk{g{o zsM7V9Z=vAf@Sm{Zt96Ix{5J$!@T5t9^OdC=ivYtVLdEnPZ6*k7&#!HsjwEPooMQf* z#+V=%{ogU>{H+I3XJ_$JUKY-lO85uDA{u6llP! z3~jwJfCB08xM06ZN;^bQxKBLFw|VY2fjsvvki9Qvk4NNe$ZGd?6W=^igv}}5C%fWB zN%VFpKe1f*q868g?1uP%^!|Pr5iRTc>Ag3)`-qvt>vLeo< z*Sns#}YeCV!>%kO%73i5`AvQmF6k}o$e^Q~FPzqp`V)+l4*zZH#xU(&wOJUde9?2aa8GS^ zAN#~54@fK-A6oeg-iHVr#HykaiHlW4Bv@*3goNyWECw$?cJA;EB^MhY-=uG?p-C(% z_xkfE6h}oa%fL8?0B3wFF%}$PsQm*0FrKCI`F(S9>#MuRhz+l)m1xP%M72&^ADWJ} zpGT`PF|7jJg0K`{F@%JpeMNQyLP6^LU0r{Jx{B5aG`bJGSsRVv5CnO%w*O(z`Q$v^ zC0kr`e+lN)A$@bg9<4$7BOr&tPFI>~G8YIm|I&N+HCKpM4b9ttEn5C~NrR|fV8L?d z{-3*}6 zwE_Uc!~J41;1x5#wq%GPd>NpFvWmrW1xZOs1!Cjbe1vCuC(e(i2Q2A|iWm|aAWIJ- zf$pK5|M36A`U#=r$&=li~xrUPoV`A)EXQ@Z3xzTOEz&@Uf(+rqUN2C zFt8lcwUzx71V^Jf24`&Wy_7ePWLkjcfczisO_9nTRDG)iq=N&{fcK1Q&>E0&F3~MV z72`vB;%LKPnl6T`mAiZ2n6n2J!$?K#O5|wG-wdyn=O# zyF)x0&CQz+w6wxO9>9+U3rtk`xw!incCO%Ydum6cp&B@k2Py3FNd}Y4lD4bCh-M@_ z0@t6KAL4^dc>O!6(T0z}RrqMYFfg~v{^cDL5Z^9T7FW>Fpzs*b^~#G|RYB;G3hsk? z+vUxJNu*IX5A%s(Y)^xB?9LL0tlUn3 z7v!<=t+|ZQ=XKC4sCyb&T7Gs5b0iQ7y$-t`wK2j_f1y8we>H$%v-jRQcL2yete1LY zrvl)$x%L&7=tnYq>~{Zzmrgoa?fx0?7=?`Ffh&i=03TgM%DY!n(IyaAMk=o}06D>^ z1u$hw%V1Aa4OSJfk`nl3#Ih6V-=x6}3Y^$$mkW@I})fBu}f^ax{Z z)xlu4pd1B7=~VM92Kz0TO%`=QEB^oofEfmsDAlQ;P3LyCHRc+Fr*-N6-O)teD?CR1 zC1x#>w`yHazkp7JIETc83waA*7u20ATdtoD`E~q&u#Fk1xZ@(c60%;pCq5~=jqx6B zc9&CieWHa_(~( zhMW^DuoT01_zv%0G1M17GJO2wk4V%y|AI}bIGQ21DR~Xg|5}m?W)P9c2L?rW(7CnH zmg21Q0v1Mz7;+gzsCt_BIXgSUvA}$aEj`poC$%GLSUiDOuGlgKu}TOV+#7y)BL*mL zW}T0r=|WQl1T468&nh5zuPia!qC+5lxR9MCwK1}^918BNeEbg8nIJJ`h)~XAH{-#_ zp()MfAi1mdzMBpTT0N0`ohcup%gp5fRqw3b8&TZi^77jkfNeA|Magj8F~9fv|8RWK z!LcF2s0t9Wu^B-YF#=1haP~k{tRhi1tp$sn|0v}esOpQ^P ztvaBU^78{7%5bYlt#?KIe08p~v(wl84ysg)QXVwRxvJ;qnJcBtgfxkF+ zKK{q$v+>Dq%tJ=N44Vwm2PH|L{`3W;6M2_%Mj@yjyqvKdOWPK4Rnen632HDN)Idf+ zKxi;p-Hg@?F1gx)7cS^N_eP+A_9ruQC+#kuC4gU2ZF`8n7Cr|5|S5p$B zhO-stB*$ELdAE4N$!8#Z3)@;?vdeov8Hy351ZD07Pm`-oXC3 z-Gz$P4x5JNoG?RR*!=rD<`(%`YwK3;kM@8)xc3FrXofv={``TCsE$IQ0Qp;1GB@9@ z#DssrX0a6qxg~+4;V%l?yIn3o?|7H`;+?-`_(HYxk!>S7-{Hq>s~}`UVfMbYwN(%3UM{LFqvvj5p?MG8fyqdUfa-U3unKU513)h9*h;(9yzB5G z5R;a^vFz=bE^_fhjnYXnZ5)C3!=?}PSU;(JHYVls&fo9?rwu8g;%7>y?i%{vzYvjm zgf)I*T>TU+FgTv5v#Qdz|dqv7| z@FRI{`)E<1pmb{i+(7b%{`P`Y0DDBP^+J>N9~-y)G7@)dd|Y@y71zD-E7D~b<@)8z zi!N`+YpbgOHkAiYNZ#>fLu@#rIg$rBZKUd9ZB$qH!6UFBA|XZ)@N=iCFC7yOs26gc zw8kJZ!faNQxYN|FqiAC-R%A*c>f4~l1D#2K-MXV@-hk(6Ytd2zf(ebX2t-yya6~Bp zE>tjVE0ai|c4cq4?$6|{(RIJ|(EWS&Xb7g>7#tpMXT@eTXPfD8vTjND(NZQIy1#BJQ;?w+IC z_i-_Jg9fc`x@%DpOX*KB(0ewh#o%5Ka61YAhllJ_6YCf-nFq<_5hCLSpQ4ao1W6Jcq2g#2FdM^SNz*cY=Bw+f@LqD1j z^At9Z3Q?gpRtl3KKoBT==9x2R1ZzK2V+l>1X4%b1_b>`0%K2sZ;r|;&?p3|~d6Oe` zTK;(OZYzT+r0Aqfu$r(bDjjTIWm0>yfFH!}Bpwi%o&YRKar$aUzsjBjn;I;E6IvjL zR}mg~5{cCg_Sm>_7f?hs_LW^-8$*>Gv|kkcHjz5!7Y@oecpOs(+<#K0Sa5OH20eLublHD88^IXnApKAkR^U+0=fqNV*-ZWdC3$xp3$JU6;O z>Wi&^PnbQf6L+M|WU!~UjakdXz^d-x7A+zs-_CA}1>wocYI36u_wy zrc)8bU<;vFk+h$+U<|ZD1o^{Moyam~KtICCuNKvt%uQLwoWkGZv`_okv14uEWx-3e zmmsMtU!01v7NxSzQZ+Nk&pNd*&p z_tz(8lb522#3fUm$sDyNul{5rZxgu`@|olU)Gh8UvB5B@Uy0H|{a)W)lr}Olx^(gj zR&zriB2e;DtxKYcigi3l`DQUe$`d9`Fm`WJVld8ER94!#^v{+wXC67C{H=s@{`kBN zVv^2kap0!u9=M|4?_e-A_ooJeQB}GRxb(>7(>^8Ll=IYgQWW?Se|qoZT6 zDgqY9f_cM3{aALg2orKTfbV!lALIuDVISj_5(*v~yufJWqaIpxa!vrVS{x1?_z+zr zXy-tq6>Lfd!>N|Z04mYb7Wy0G17ZI1ajSRV#qLkFF>l_y$vub8w+8!Wm+TeRiiJ#X zLV(Z^KgSwI@6=q~1W=SPCye1-bH|TOLFNM?E{+aGs-;{beA8zkQE7 zx6X3o-xvWUYn-q8NTsj(v8p?VodIHTQ+T5l5-A=NP%TF|vRgrm&F7IQg+%+TdY%#y zJ(5OEve{}vJ1~ViYlicHYN!aHmiD$RgY&0|h{!K*6h^~gcX#fo<>5>Wl^gm@ZotE z`+#{K51XXFaj#BYXGB_%yy^>YMe!e)G2oJTTD`@rr|X$gJ821#(d9$%MgVVSfa?eV zGMJ`PGS*-st%ncMuLPGilHll!CSI>O7SaU_ib;9;h@>W*rI3I6t8`x``;k%9UBraIMwW0HAE`r5GwAE1YOK61NRNemAtqD<{vG^*ZV>`T zGOWWmW3bZ^FkSVmS>w4EaVKnnS^XL)0>4ZoNhz_-9`2Aq;OAc$Y>Uz(5|h)u%8X|W%vomy6n&aeq3@qMQ8VD);$)>J};6xIVA5YUQja2>UaIxVzCx;xX%fL~Vf& zAzp_KM>1T(@vt?yptZn3vH&%NxiH)Vk_fnOO)aR#anmp`UA?T2%Et~Sj}?PoMR!Lf zAF*Q2o`Tdj-L{h4j&QD8vxA?=Cs-(AJOkdl#1xd?{2gDu9I%}|Q|5vUvUV=&jm3aM z5w`&OBt=sj2`uU(x=uZ`kRvSNg^(dq{6ga=&cf`SwDiz=fKj;$Q*xW9DX@D+>c$xW z@lQzV4k8GhYzGqhkurPi9ZO(kJo?>AglaTmx_AZ&ixrru#16rWL7rw1I|#2360Z@* zqNJD`M>0a3QGeA;=bcI7NTo&X3IgpHRVzJHPbV+fW2@#LIK zQL5m=T%Vq0-y=Mc(N*|`FSPHthj>Hi(j7meQJ0-a{{-JlPThLPEb(rNh`|1g!AFy4j+f`B~uY9=Y~Y(*LuWk zYJd_cYQJXP!Mw-+VU*)fi=X^)|B-3KusQ6pIoQS^xO~P01^+DrAQl%XN~;Z=DfApO z?FqUE%~C4=do*^9X^R`W793#XL~Z~QX|y+9{e5$r#Q!u+k0gNTq2HYZ6t92sEIq+KYHn*w27cU{QDa#NB67pC2@U?pt|S4Ch14hkAL86W zn;Arw2@*HvQYh{W@k68~+yXfDB@Ed{O==B>J{5ckeL6u2{fL=25OVU%$ZD0v{Xf%r z-=q8&DgcLFV71mZ<55(jI!pv;S3bY%Va7c@kawFr7BD!Ao1NQoEFbUPHgfZQxO;y? zycj!OrhEw~{s{f**2_y=`M=2WVB&IiC$~y#S>?gKQUD!sjO}pQOYe<;n|%H<+}<31 zCGg6O1s9pq6(<=Ua~Z39{+y4G-h<>tH|}OywDwpHXFQ8>9@<)_DDcV0XQu7w=mmz; zu5FUi8$}lWJP=`3C2v8duqUVKuG4 z^3>Y3w=4X~gvJvCU$BA}`})#W{a)!cd}|{nP*F>OKtD^9ApSM&RB@GoSKwJu(S)-~ z3K`G~{V}j~y^+-UqYFpReE#9UbC4`BnmWFIK--w_HkUD~W+2C7G$<2aSbunMxdn zX&{$W@Is*ngfO?N3cMpA{1~W2&HlcpU%QA26ID#twTswa)<>Psv=Wy^b`n0_R&)Ar z8IN#<+r@l8|N76A1+-FY-*JuPH;cBQtT2soH9x*%@u4YHs!5pS2s?RI-w8+>dK0Sn_PN%c;%WuG5(rG$&3*FPFPH+2Rj}9L^yC}*?#6| z#=r)Lta>Bz_aJ#gN6mu=4?f~#4`Rb&3=n(TYr!AoXjPH{2q!`z_+d2d$P9HTy8yf- zfbf%;o}L~Jw{M5Td6ppHH*QyCWLV%cOcha0BHAD<&U(yi*t(195rO!_zN*u#QT!+9 zVdy<+IwdAGTqZmN6@xy;5}$kLPhf_AKcZ^im}3Zo!+4jMk?Y3LC>b(N_J>^=!DY>r z;DA7?S$yLPj&{d=!7lL*Z^Y-4Hm3M9?^m>oE-Vvol+6 zaH2*DMA1!Hb}crxxXrD8s{tsMrux2( zYNiPd>`OaQzt^Rum}Y2V@^`-Ep<6vW3=xSau_Tl3lizD{sjs^&5q(*}{p8`}*l`qJ zpW{Z{-(#6-3?FOC-24!nGP1ofAm#TM-$>gFMPI2K76WVW2w-&-$wuhU>G|PJS%U)^ zwZ-+ZPrAnq9a_lX?5NH6(P@;_Z$2(s`D2)m@8~*q>lXYE@A45``J{yzd|Xthc!dn8 z&#a`BNxG+F3AMlF=E8@MFq9{Sqr22N-O9S@WR|BPUWRkV6{a@fMiWOfhpN zbFSX^!DIu;Gs%NGS#y{tis$eH+Xm{K$c7UL&a`nnhN}I$KZc8B7Tbpgr4wWTu{nH3 zPw}t18V@$LKn+PC(>u3Z{$=BQ>fZOHUO~Bj61^}(c}{RiEbR?U(G*=ip11D?&pDpW z-h^0pez(a*7-cpx#@7CxcH_;(c|C`4 zUiIzqQ|II1rn80*tDt@*UpGoO?o_i1s0~Cy@@!x?D^jc>-_xwgdpM8u-z|60-|~o* z=ss9FMjnX5SRd9$q6n>4r_8BL^A?|&e@F) zuJ9T}zrdL0P z^VpGo?p=e&H;iZ6bC!SQorSqw58WF|Tw~qYGiH1QM2TcJ=;?h10$Uc$YPcYBWjZt_ zVC9`iUW*Ahm2e0~t;NeDNWgCRfU0ih-%FM+ubIBqN;(mW&&idsJP$VBo8t+ou-`3z|d} z{Lh@BVF-{!sh7k3(`h7(sDQ_|a^DBH+%6(=A>odvDri8yprEN58*lJ1AAtKO?LEGO zc_xN;EK{-P$V4tMa7Id%a^&>Gu{rv)V)cFvU5E|?WGW(B9;Mwm{Q^fX@ z+lsZ1Vw0jf_+3GM);i_7qPWzo0RPr!Ga3^^1bP<)9^RgVlB8d4@FoD@6!_#j5?6s9 z8wj`c$}=XDrFS1<4Ng4srpYx>BY}Y?deg{mnCINAI{3Lk7xxWwQGPs!NGHJbmDpSv z&}AZHJXR1F6)h*Ytw_Y%94z7)zR$)KitNvAw9W&JJasN|bjGFKCZG$(k@y_b*1A1K z{vwoRP_mch{fT!f9PXf_H&C&mZ;k&5AcbJI5LM3{bII_($99O|Gmw)fbn4T_jXvs; zf)hOQ#`DdI1bfPGXO0If_&HSGA)FVt50|OVAn+e3c>-2y$dm~UzL*q~f{7%GF5$T6 zU@Ip;Ls)e+seAH?^~B5~;{p@~Hx5H_OBRg4u;y*T`t7IzwYrX#T4P7>sk|B1)<1W* z2YD(Zgv5orYr@ZPV9@LYB6D|R=M_xzX?=rd6LKrDsKBmvCf6IA;ha^@E z{K3!g>D;y`)gM1s*QU-dkqDiD*b6#uN|b;A0u!#7e*7iq)XfPk+-jmJ<qb(ikn;an87~GQ_b3X`pKQ=aNf%aec3lk9!u#vtf+Au#gV$bbky#5!SsNBTsPwHCqDJW#q;ku=t)YuEnm^r}pMn4m93@N);+M0NarZwz4A&^;$$ zi(21;a2zMojsw+3x+ahDO4wY|_(lH&N>cc@41;9!1SBi7ZfZ9N#2?|+7Z36wKX1Ux z%7?56ov_w^5zpX^C7K&~<&dQ*$*{&Bv$n%(0>zv-jU?2>ZC`!9b4a=S z#ZE4$lgSvpQ4XG~Ikur5p}6z4uLzkm2R-vZawZI&iXGI2^%(k)h!%ep+xMZ%okWe) zC#FesC*e+I=u>jifzM{(Sj4%NN=p4(heNOd6R@nwHL1&cxM?^Gg4uoFKu|I;RNjOI z)%X5-8HSS5JKD8H%II(d-X(mwHRXc zLZdPlu=gXc?mJ|vkfvZVc52)?K9h*0kC%>AJT-9hMT-YpV_xb6AJXsv#%vC)9zms z8g!5|bgVm2A#yL_fdCH{TfF#fxE+!n=8j$8-}-}=7$sOa=z3{)3wR^X0r9$^13o>3 zZwY?B9SVTE+ZA6$V2!{D?vaPEO(Q08XXmck2+|M&o!gYLqw#hT=^$BaHWMxhfT}0j~o1jqGeild;Mg_x3eD`DZ%|s52-}!EhTyVUud_Zl_Nu&v36zb@%St zMPx8Tp^TR~=FF6N52-mT{I~U?mxZK+aqF5C6RN-I8dChd*yw3}{=jC?sWPryL~5Lm zVp%|7_#As4P~tNj0LB76O%6%&0l}bLza`fD9m|}61gdul)dlC<+75ukHBj8BJ^PpR zf}pDz6gN{#$W*#^?b-`25Qi0iOngkW!B{-fn-W^+wv7?rQNf4YI2*M!OfeRTfdDzG zMxp}2!9u~`h=>_YfFi0DMkE&-xHZVw@;Mwt_cir2 zuERk;XFrIiLW$*w6hG%c6mt_5fvAe;m~oF3=iBDo=P|xygbu|!Ys)ryvMhqB2qAFj z?naWeGqxtt3io|>y4{K5IRTY)!`q(a(9H;2F^JXm*4yw7RE#8aev@-(*hc9A8!eI^BFGQcc3y*B#=sEP6ZY*h zYRrpI^!(V%Nr{w*fQwniv!gqZKVwX8?Fm8iP7d7Y?`rozhCl`ZRJEW-zkL5TX;&@C%a7cpJ@Hghmx=1n_Kp#WSOwYgd9x;Wl z6Zr_!=@%_b5z;Xe>W@_+7M2B?6ktgspfd?;P1Sh0`*%VRPNTL9!bjh!}Qm3auJ2!BQ_6>PsZT5J)&i zIgi@h$a(RfYrqCo;%f%@cj30jMrDYCdyE*IFCT*K}j=cDY@5qR}*I;zxGO$R6zQdfY5|W z+7lG+V0fn3+u_TQcphQW4ld}R>rM}Cqnzxz4G9vCDu%klLy<_CEI;%`WOu{^Pt3Eo zhZ1FskbBB0n8}`LwP0{FnndPvzxDD#RL?l{U0K+l=(nUAgWa+D2DLeSEL4o(wiyA> z;af(v7;9Nc8u2R|Z#x%t=W9VBgaSVnMah%b`{!(#_-fr)fRqrxlfFVn=b1_kX`iqc z$s-73y+j%7t{W0s7+F^#b-;iIBC9$^k54(4_Bc16wv5GblpLOb?<>uDl$1c*yc z<`wv=xqW%F!4syCklIfS#R8=MmL^e0sqYprrqZU{J&(`>rV>2Yc*yh|^xM2g=Lu5+ zY`e8#&nBETc6=W^&PAc%DXJRwv}YfJTO3T(zg!r|`&rw3|8}U4iWlC+Q5+1pQ9ISA zFWE}pasuv?Pe=lwVHRA!k`DrB^&L9Ac+n!c?S+iI-VdQbgpFe-jK%Q=yaJij3{Rgv z^$ka3!tj|3gd{VheRvcj!vryeX6KmqpRoU64&nk)UFd)TWHk(Jp7sxpI;k2_M)s#Z z+-Znu}9fYP7mh4NaAW{As*ds5d&wGT2MU+n0P1*jLu?vq%_NVIT^ z)`@Kul(UJ%$8QlC{?1Etyz0W#6LA^6z>MxK4?z+#*tX3^bS5_kN31|%tVtvUD1l!n zW5!A2X~^B@b-xrI%EBh0EP*172{Pr2J4hsy0PqTX=oFR0`@x~1c8PhGsZmQ1?nUq{ zGIOq7JHA&=r2OX9KM$;)%^1+%RPp@kpD0R3u8hFn%SyFGL-XSy?&dkVpH>68O0Zbv zua&%GF<^XL9cnJHIw6M`b{Sfufq^6FPF_WMe+K$+7G^^T9?uEX0o&9!8PEog9zoj) z`v$|1LmzL`s~AMU+|tZ~$zO9v$C<`i4Gq9ZJXzI9_lQ z4qAE+D)wzlAF=63QLtyEL4(+eNord-EiNpgZ^H!lz0=K^0}*7`jVS{e(jkyr`7oVuZgA)y4E(iYcAH$HIEUWXx0IM!;J%o86*SSw8H~t1b{zHv z9OSYnuF|&E5%MvF9QnY>g@fd^ht!!wLr!vIfYdd%HAK!f8b~A#Shv&%E$c;%hi5sM zcvQ?+=i*o+Os{r}PUD7PbG87aq~jS4_5SoKbRsV)P^-zWRx-E*ZL>I199SoP=oIWH z933_z^b&E__0x;zym1#-Ur!J9<|4DQ5fFGVAS8s9xJp)b#^e)xvGI-1-UY|MC~FsD z3}h|x-ZZy&_w}8;8&CImp>NC2KLae>fShzVhTF7p8ysC6)Ai%|7is_4H4|V!Nhd*> zkp;*`g;3H#TEbSB2G7_wlS3yQTSvHQdtOgC#22M^&GEA8LfcI)(O@MHb*_dj)vq{% zK@i7{(QWiEny%0U3vN<11F{;^!YDk`R3JQSWS+ttL@qQ68Si3x9@(F+O z>JDmPP-*UM#Y}?Y4f6fulq6B<>u(xPL38o=HM_uU ztJ$Sk0F3aEdRC20nJa0PzYo!CbU{-p3UgZ+9Ds4e7m8}g4n`NmFNI~54dz{7HY8yz zO=tLk2$aRE4d6=%Lk3?@dwep^1~3#d>(Y`mBW9wkHk5(~p%FZ1SD@-+hSarntc+-Z zuz#`8v)5fjC8VpPxvDAduOENLZA;#R34q)Is?n+v9Csdk@u-)6#<~uqtRCR35*&*&!~s(>v8v?dgZ5q+e)#qanGt6; z`7=eYAyy_r_q1J>V?>M%z~|}C%{t?b-Mw)b5C>7O$VuUd(9k&Wbju-R!kHp1H%xZy zh$h|>Rc0PYzTC4|NoI|ZAl&HD7=>K&9Qz-@ORYQc>n$@nz)BBONo;4L9lNWL9BlU<|~MKz`-``dNL9080WBQ(jS5Hldk-wN4pC4XM4DnKN2c20J^dY2DQ87V!F=5AS}%S-bpqo4}GNqYD-ph)sFt-UVJh2l!Pm6={@r`Kb9#S3<$W zyVR54r%Ukf`yj(Z&!?B}c(-e(I(`yajdBpQ+sv0xC_)I$T2}6TAh(e^HjvoQ-WW4( zs0|U78p(8=hk?727&4ub${{AE&NqbCVE#bn^fYZcvmecVxK;+^33{(`b>wB0h$Lc5R}piQq#~-PXUD|_@JeH2+Sx$x6DDuXVwldgpMeP z`is4i=7T~?*Qq9h9&bu0bdrS7g(w|N!7kp5uoR4pjghi(U)RSkhsZ*b2ETOTO$F}d zWoX{_{wk@qQc0z_YSl%L^x@+b>dfU|pSIr|!;r82az`&{{6D{ChiSiO2Z3nk7*TmQ zC1eC2--Mg>@j7RJW{2e}Umq_{lSqt~1cjVV&|Q`WBra;%mdlao{pgUbpp>e~Sv598n)Cn`6S5 z*|T^F(B!#v?&@|+gc_dii^X|djbleDFmJk!>hsDZf*d%^AFK!?bf@-%=;myZKliL3 z)%6smGfn=8Ff$S@?^UCWLpy7Lyq`n0F%ClFs8`e#1E{7B9>clh0vN1+j=!e@n1)NW z1a7~$XZuPhDT&8{OrA_$Ig{+N5YrG-ZGi*xpqx9%Dim}{QG&vQ!JZ4W-9DqHRi02) zUgenCs)}}-s*1~Ox8iL5>uKA+XE-nly#vQ-D27_?`L9!)b>Q+ZBRNLL0wl`FJu=>R z{$Q|fE)Oj_F0_bZTI2`6vu6!4ZJ>r@nuhIXTGW)tXmz0KKUt~~b)KjO(A#fCiN!RT z8;S=2)N~!1!sKTV9neSPVi83FXr(3k^^mglH$qNHGYRPi_IF1I?ipynW~tfPFL(sw z5QwMOk*F2?_M{SJjL!{JBVuF*+F8V?- zi?=R{RXD~p52w?@E9ppV&%j%MbcNKvWm*CTLC^FaJ!tHD?f;Lh?~bRs|KC3?x)WL& zh*Y)=Uzu&LtxSrQ_J!egx*o7=*4#rUm_+RA;pl1>m)AecQ55g2kse+rG+tyQY2!ud9 z2t}wRz;io{7|U>X!bOMJDd{%d3)GDLqA-EQ-TED)PVB0bl)>H~vokZD()50Jdq0A+ z|3b3aOJvt=5Pcwt8uoLj$vnPL*`w1(&r$Dra{7+JbldFfxYyT=MFGx{p&H64sLN+2 z-GdK0!4O1I1Seg5?o{5bfEb`ShfQsHGrT7$eo*bS4##%v!2g%FG4TA^vk-er&|8>s z72>}4HfGgPzlC9qaSQwSairw}8)Vto)kos$!jLuh;ZhwplrVs%eQq$7=A%@kRVybT zdS}@@&<>9>7+(+$+z;bx?v$>f6ElG0Ed`!^wd(7Wr{sQwP1ESh+U;-3mV!v`!>)s!?Hd!v zg-AL+O!ne@gumv#?>-Xrxw8-SpG3Z4-wzCrY^g}4PG2K=e_VnLem|v2s)TTkA5NFU zk1;p0ZUIVbf-Pv@S@d4GW$QWCP9jw`v`4a?14I3Y=VSvyyh)6zHFLphG1H?t>#(T2 z{66%N+6dW}Lx=~E-tEE@|K%CMt)DZnt3eATL{c{ZVRCLmTeUIDJw*uhKB%3PSP0`J zL+KPO=2D@JI3U)=;x2$a6&L7RYI&NCQyu++MLOm#Ca~G_oDnKa-3U@&Hzop5M(Y@M z0$ufjsy2;b;SufoXjS*3am);SupHEEZVu zQ)R~%JmK}qdQs>rMqlpxp8qwLR??hTT)9Xfn2o8u$LF@m`-YSFTS~g^0~6vv>c{$m z)f-KRZ{mJqgue?@3N;KA#C2^6I0GO+NFYW}F{=?P8SX9^wh$hr2SGU<)aylLm4*2* zET5(Xkz+4b|B=`RC=#**Q-+}XhARg6*)_0FAa{xhY*Qx(5uJcyPtxipa@1LId)j4 zgL{d3BPz_}v~A1zH4Z;aK0Y)iO9b>S*$sHgOK{<2undD3ZCm#YAiC0$8{&uI>pyRR z`o9R`i|5Z8>FdH|sMdL@(Z)TDHPw;V*1}UX}FqW zOe6vB7j+L;CYbh~G@%=077y;I&Mj;M!jD2_(cogCace)R`W{{HvQ8S1T zMh`2sa+w*d!~oi|O{OFxL~lDhR*|ZRj+5{aKZ@1!f-Qh9_*b!qly*3N@8RwH9fF?W z?=+*9^v^b|aYglSOOoL}J<9mK0i&aTneVr25g1Luru9@GC!>w7o<0PjOW6HUjNm1% zOCG#LtD@aZ&RySx0&+F;wslyM*B=XS!Pt_NN}e-n4e^Zv$-)tsE2vQl7S*8}*~#Vr zp-#X2@uJEMgi0!%6BCSv;%Jl1V0R3D+*F<2z*}%ln=#fXL}6GyNN*|~hFUi1v^wG|#5QKq z08$#LI0RZshADuyEgLZOgDV4VAVR4L>R)DXqkXES9SXwG#dwRy*^4eK+yOrBF@Jn8 z2JYQaO5mhlZZ*BkaPdr3qdY-a~ph^-(`8{1^0<~w6ms9 zH>4?rQmakl`}vbjum6p@G8Ut<&(qQep$DG}nR$QX&*Xbk7~Ef$#@`*g@a~GA|0gj3 zEHD&(fV-3#d}teQAb)OyG5)5~e6ldtIf1JH-eDdyFdBWrU7EfSmW*0bTo)z*QE2vC z5XIpPrYEz6ilgQyluK^;`x~>kQ>tjc=-PuzAZ|gf7*N2QfH(YeV2CdNkS)|6pQwk% zv1X=%SMd#G2zcGxjD<~eJZ*e}4pL^cYuHQlCQQ^mIRm32Sv^=R?>xp&r8Vy$0(BuT zkgchhc+6l(CXj6s7M9X-R{$iP->o@I!Q6raa87A%ULH;9;+Y}`MR)`#mQf2%KD5MR z4qAfU2^fHkEidC-e4h^ooQ6?E2LW+pauAvJlez$HEel*4B_~0Dj@FdLuG)r`0sa^n zTBal0@T%;#y_F}$>fRpP_W5VD{jQKQGgkjadT^YJagp=Vt2^Bp#gbJs@0JwTje&g| z3o&EFVyf4rYiz!HhpyC=!Z-^ei&FId_kmn%q>x=Pt3WX=5=IZ8ZMy;YBE%IAb`MHG zE#SWi){)!Uu|$o&7iNzhd62!tV9H9E5bI|24UJp5E?70o-~+3L{uB*hTcM03GYpJfcw}KFKou^5Q(jd%UWyTzAcp-6K6|0*Dp7}lCr~dnTFa66C7=uoYZ9ko<@SxXIxxbq??(L6q zL$mS@kn(BZr$*=8O{tC?=+haIElAInb|^{kkQfWU+t}5{Sqdq>9r+Rfm@_m7-Jlyz zU*E`w0dlWbS?}>&@+v%;I+B=y_CpWV>f}zl0j3)I+ zRADP=MdR2MVmIf0$p8uqq*O)a&9SC!CogzK^k}uU%}fd&4;|?e`a#vjQq~i zGDj*`S=>f7Oz<~~Q^Y2vdb~06dXEZgFo*Jp>4MgYDlL#qx0hRB89T8L#nxUS66&drY5)=-|h0<<_f%r~`!0$`nI3{@MCo*xtAa zv)V`9m@S1PX~Sz&(FVX2S_M@iAW6@i8WaJ4YecRxRD@=0lz82$QQAI&HwJo2Ig8oI zm2jyxEsj((Y}3I>_?-VHFYj@QUVIs~OzkzGtI#wo-9>j}oYmX$LZCF#IH=hpcT&9= zhjp;n*u@XA`Fk8BKm?YB|7xl2lASW1xnk35iHj!lay<{x6{NB+&Pc{8Rv3 z2ulyTX$%gGFORuUV%!W>5I1;&is~XUe)ztrYbF9Cf+Ne({?aQNQRH_?4ipI%@%b>Xz^dQ zEW}6Sd6S#V@taM`5c9ou?NAH{HbFwJg}(X{heIwwOY&#RH0k2vm6^Wkuc|4M3u=gB z$#q{Y&l#^I^S|Ng5T-7yStOCSsoGQPHpkeMD3k<7>cz`h2%vq0(3A|0$R6bF+wzsv zqBu@N?G9f4qi*W#$ZO)UDzGymxr00*WMSXgu#*O>XvpKW%X%5b3V)*<&1^WQSfP#S zm%N2NEaD}gZU(HxVy-Y`wh&z&f2;$G!5zdzVFU2>1SgcpWRJJDyLh)v z@_yuMiLqP_8=~p5=8#p=&>Bb}hz+AB;~sHp=EDUT7yxxonQh?nfahC2Hn?tpGWH1k zVv9x7SgwC`H-5bQ8|1UYM0An55l{WW1AGjbjebRi-q#*jJf$kDcsB~|CnYd>r_2+& z@(@Ngz);9X4s#NpoVtH>XbBCDE6Y9!+I*g=Y@96WfSDpNQ+(Kuzoib$H7H-kCS^kZAh#lB_jWF&}>3$&lo%}mcX#_mo z8gcu`^aE(gxBB2;?mnH|C@4j}8gmKdNQ*_)@ROaZasUpW8t?!ihfh+R;+Zit$7Y;I zN2=s$rUG`}3KsAbIrKM?qgW4$JM>M&o&zw$FPjg)7Z4|&dQQrmOw57b!f`uQyaRcE zlN(kkD7q;uKVKSZkmqs;8y+2`agJ1}S}K^3ve0&g9>m}g=50vO&S3SBAeP)-3hjWx z5oN#nc;`;Cp22vK?nEA`YyJEVwh)>MX4Xo-$l+w@v1y`0t30PoR=ki1#FsS_`Yj%> zA|qMU!KNr&4HLn@qlN;PBPhTAXg00&VN z5@EfjK*V`67b0{HHWwG0IjM0hwt?azNnTw1{Kib+b>|5Uqxm)n1BQtJ5a}mS7x|82 z-_7E54_Z21XjIF+5xE@uWu@L=P}ffoH}M826OiSV!R_37O4%hqDs)EhFX$(NZZO@$ zz}_c{I(aq{K_1>NDrG3r_K++A&3_!e10<4Kw(7+ikoSRe=7VZ2c7$ZII0ByPCEVyL zYfkjsS~u{01x&mTbp>)K6$$dF_wNY5oLRUOUQpG3j2 z&_Z`hLoQa%9zd47ICt3wk|JQn{Ja5U46@(D`UA@a^WFPjV=(uJe%Ty{qMddXl7c%{ zAMgnI*kQlLvYD(Y^|=2tr6#FSp2voC(xqZH+;q#Kx&Xb%52w5|Jimd*qz1%gpSyKW56EjQw=%5t z!9!%(fHoUN8R^0PwAq4x$EF`Iu|Wfx8as(3KRC!>GDBO#IA$cjS3T@$YFV zBho=Z((6^yC0}Ft$?ncMJ}Z;#ZdOWGM^lB3~O4b5=56qM6|F8%wR|K1b=GS3XFOW zy}KTYz+KUgx{Gzx;HASN{4+XfP7ObeHxIw79*u`O3k?kTNZl&1;iZ${Q0=q@!=1Fo zwGF8LSrS5sfi=NwGjuz2vIeHC+DP>l>4-Q@;D}25MIK4Pl5+A!jNO}vER$py$~1X>?WXK#BPda`3jgbOO$hCF7*+`pgSK3^MzGn#vSc4LJj@{Gi2 z6j#1dg5%q+y&tG1^*s{6X7^;N!GC-W&)VYunmVcQ7n@C#R^%fSxQne!xUog_!5CK{ zay@m|-RKsMp~zWbXch@9&6GYO_P|3w`n&;cGJ3v3T4@R{zxXtR19am)>-|q!fUDFp za*o=3iC;7z*Eooz=ao;%4l|~UA;X9i``fmH*@L#6Wdw#lw>$y*BN-*pKYsJhJ-7*^E*4@V8ta!r-*0F=4M3l z-bj0oZjkkI&FRS5KKAcu8t%F7tTf}{Gn8j@cfO-(;n6R9bYv8A>Q=$ko&};v3$5%x zgOXcoEdmPw7IgqVHfV%BV=;^`#X}54O#vd4eduJA)bZvahhPwuR6@tX2MNDt&o#}X zO4{i1o;Q$I0ePAPNT~jb*#$1={29|Vvx9DpLq}WhO1Yt*cag~P-5Iivq$uCAdyiGev#s1nlvbhblfoIgg?EU zU-U2XR78;M#!)2&dmb96NNQZcgMTo;|3U_wKy;`@!7qJ&h#o9-L}L#Y3kM(arx7Wi zjMm{-_%{K^XcGLD(!w14Dy`q4dU)%2GWY*+Eq0vpS8v^TA{`Ra&pnIziRpJ7B`OAq@}aI{!Md~}-4_i&5M5u;(wr&@j=oh+*V68t~$wR<=j0v;8u zNhLU3_4O8;5&zqzY3Ko^H5PDk2a%r&37kY`MUS;~(m+uE%&H$4P3lppJ4uo-GNq(l zA1HY47Y3(;mqqnynV!qe{IxwdSqcZy=0KF5W^jM_EuYRauL#yap1@;l0y!ZW!<8#w z$PLUlO<48zeGKN&T;E1>P4XSMzQ8=3D+nGv>j?-)YsBJ{DgB+M2d)wW9EiM5+mD$k zdhcwP;VX5>|IPxvYySrOgkNr6)wLsL6+ot95b<7$%AR_1n9jb&j~*`#K-%Jq7G>|a zzHgzdEYVd;;Z8r>-a0EoH`#`GV5KDHLaH^}#VV0@05!OpTmP^dpygBysX`!c+)rI6 zO=Vcv_ej}EeMY-B4XpJrjAP+@-RHrcfE5}1Xhsq~f#j}aZO zGy_E?QoeQ@SCeuHcEeHg{MaCCJbLMeg)-oGk%F7JxHKPyR=psj8?ZgcLup${Xt|3t<(NN<;-CL~E zb`+mVsr&U`z>>A1YepGR+^ejvb!T@c`^L8&lL$bSm&loOMe+4OGk%6%?jxTk!s>bS z#cX?g_OiwZOZ`x+HWc)HkCK@0yB@Lr`rM0gsLe0)XheMNE@^WK| zg3yYwaD8aTa3AJv-6IEwCoQDxn6BgDLHjxI(UE>bxAxyO%6=i36=B_U0fvZQ zZIUe#A%jAj7P7yJ(3HLpw3%^G4uyGr#6C{x%G^R0fBCtRG zWlemQ1q?}x2b*derF)NkOAFK7^rA3>l<>VGJ;SObl>G^ z=z8cbpf<6fOu!h{5t*e<7E-^*(>;{Kh%qx)MSbYVtVAmghK3j4Iz6l3*MFq8x9(s3 z7Pe0G=aD!VnM`72JffE}3XidlPFYX`&)Ng0z=+SQa=W&L(u9+7g!oU#Y55#Jn z0miX~9fnbOdJhQ0a=R8M0N?-lMWes(#NSQZJ#QQNl~I;ycW#l>iCMBqMEF_DI-dHk z6098FvjlMf^E~4}Bgx$DuW2MAiU@=?15t1@Y!1l96{Y zfuV2c5wZP0Q|jH2VT5W#Zlm53C^l_Q1Ty=qacC*I4#-+uAMUv9M$`sFMVE0UtS;Vb z8p&J~M+Or^m1VY|vIp_}4uEpZ?aU23381(xy`CCObaw|)qGDo*c4r;&{uPY!4Q!Hw2bHSE}0@q*P{Lv)$ctD+2X6(yD*gro(&@wZG2? zV=!Q|+OCUM^Oxivn}+%4zoy57R>_>R3A{vqJLFmuHnzpS&&$(fk?|e}SG3XwXqs(B z;*|QwC4>4y&njVaFySk?NgG_G7aB;A=>n8dTnpEcBt3$g5plqD*%)OI z2$&7;#}YiGwmsum<2O`;%N~hV8eEcUbpTPbaG>+HorHCiDb(9oRrRFX0bV#LD|l6m zt7%$6{Q(5l;G#?k0@kHVhn-M^TqB3e(dy{iF^?s&5#$s}H;|%<4hMca4#r$Ej&gVD z8_IvK7a!2(KaLSO3H%}!ln0K}T$SA?2D7f%96zb3H<{sR@I+<&PxkB@^WPW0SoQDu zk6B=8$5r0KQ!r&3l-cNWpjiIWPEPu(gHE7B{&Y@Hs>TQaik3BaGO$!jxCvx-%HAMS zqN!zx3_ujda8qM6O*aSJylFwSa8gl+43oVVAqcP;OW2FUN{lVcueF@_@G&aSQ+4|L zK5SJ-577lXYB3?9tWqg7ALM)x&n-srXb9#+;O2!x>|Ndu3o)8EfL$DVP&yQVH}UXr zBVtf#iYO{(6xv~4N_&V|P#zf3k^wP+zT-!N@EqYQ;H_daFg^rCikq>SHV8nojmxdo zFLMh!>BrV7a=Peqa5rGaM3$Xspy$Y%W2j2_Hch97R--{x%Ddw0dmzSbByBYgaWdV3 zZoChQUNe!M-LHT3y<8*DV6HtS)O0QXFFm~H|8V$bc5gjbx~N)&%GBkTo$nSeTJ~wQ zP3X?wgGZ%tUn{d9)F_I^V6KPt8_{g{QAKW~R%is*_ z+!~v~b?ZRjK@~S1P_Ox6b0<%HM070OltGxe2)B$Nr=Kp$ef*8Wf z^5VRXjrQH3g~5xzpi~i?+)$!HUKs5QvkHu;y5Ol#X{zVA)O~<)1dr2fpO&BdXJ^U2 z#Whk_C$gqaJOnT`$2%_)AgfW*Sd)L?IMjXMqbZkHHXWmpQ*q76u)|}slDA6FQ^1PY zPaHv%#CXzX$+jUB)94F*4$Je(eSjqd5_jQ3OH{lA^&86o>2y{IEkKbj$aaWXY(N1? zJnhGyfOK(*IWE=cOAIxN=f648#ugrw8O~mNFrD`rKRSFgZ}=O`^J`H%Bn68jb*=J_ zo&xX>-C_Jp!GN z*M6rhNb&@BhH5yLgZtb?Oa+4S0G2{bBBmPvzi&cEfw{noW<`GMozibW=rH}WYQcV@ z^?(DG`$ak*1hz$@U7|7}y(T!}Uedtlk7zfg1>h3WyIQ!L!z=I-!M;J^wJM_x7wPf; zv}%f=@TdU7N@8`)$j`~TGo%T^^G{RPbNC&MM!x<}ezsY?6xIV|1R zFeaX6Y{f(9Lyr&Cm1pum#U*!m^7sOU0dtZ_iIVYWTN(I>+mzR95YPKrS#2ENDbf}7 zA>jl7A^2qnNFQ+^hYM$L0xM0gprzs1y}0_;ug0fxaHU`owa$3eq8*@p<}5xRa9;TJ zfzJo62@MU?f4&tbOnT}0!}e;am|y+Y?v8+aK-!ayQ33*E+FfA9{?#t9>!XEi@Wpf6 z4bTo_A8mAefH59f*KPnf>6M4Ym2}epx4kDJHDhouo*WR&l76dDKF4pRnFIP6P3gHS z{Tga>#W_i>*sY@XPkbtVf?n=!Hh)u%eK)O{@BB=6f{gc9OCE;5`Lw?Lva=>j;%Cy| zjou9IiDfsmPDCsh*;z$&OQDUP-!95ke*m3fUsE7?A%j9s-uoC2>>W)OUf8Zpe4!*g z((n@eSVL)hRCbIZv8;tFENxL0UTM)_{GbuTAE28#4+A`~QKP{@GE77#XpV*g{#>I2 z(LomywbN^Ckr@~>Am8VEx_E0Dato*@Mjx#1YzcFd&(auLv9y~ zvP)N&(L3S6bgrF8%9I4=@ClAf=zO}A{^P;)v~C&@g*1oVyXQH70U$KB6^6}cX{0#V z+rd(W1IpJ*X#fsMN$lCsFCFrV8kHz&O3|R>S-CwU4UG_NPwjWbh zT~;$|6=MFd zUYRGhdfPWA+jpG%T*bo}nNfS?w)vcR4-<-0cs0)jQEQ8U(+lEfl!KhNrNne>&Okd^ z0PV&d%R~01#eO*XmjBHeO181Y({Z|WA9O(5p9UTll)vVpcAB(zu=S@m7k1N(p?6FUx3uFO4mT%3UAUm8_x>tK%Aj3 zK~8XRN0a20C+{dxW0-jt5h##l1h zOdqe7BYT-A+jB>dAw{P3H@47MWWC}P|7a7HN^rn`_H7jFV%OX=obPsSXuUt%1~Cp# z4g1S(?5EVX3Y}SeW`p`xpMT#x9PYN-Ya<`F)hm2!gl1F5-J91iATnAs+$)vEP)3{d1>#ED~&b1xCKjb`MY3;iygAODg!*0!j3@jTYy*n!S2ZBnR_Sgna zpA__sj7DBOytI~v%;t!)FU4=*Ju3Pl_e5G**5x+Xv4yha_v+ZdrNE`h4j3NPdzus= z5GEK5M-LuhL)Mt@VDls-hgDO8hzW^6L@fZzl_*l|?-8GXGPurILAzJ2@#|L+5bx2u zvF+AOPb+jM9|@^?f3{-jb3?JCPXR1rZEw)Pne~cvITd>4$F)iR+PqA?ZAwnz<9Ph= zNC;@CIWf-+hnx!a>bNoTiQR6G`u*I{>yvTc`OlrV77=qn%N5DgGwHuEFv3>#8&sF&+M7SOCYb}6KQ$f6)%tmMh-~mS3OjNzvU5MQG2%e-p z83@V-(v)wQYmrFdMuPP%vzySp@C>1bhrrd6Y{`4n4*KgLq;>b%3 z823;9mm8;z0th7V@*f_K2oEPBWEwX*S0+2`pg#Wgr*)hL)+7`C3942x;smLY)7rgg zXQzK_%c8+A(AP6nvfRn4|LE4wZCp;69P7n&={qfCJTRD|IpagCkGswG2`NMIZsUcD znttd@FRqf{Za(6aP?%ig!@FsbX_;R&c0Y2! zQ6VFD$Y7?1)AVxqCYmsC6EuoJmuD-&gD~>4-o})iJOQJSg96X&Pi?yLj_o0kpoUQ` zgA(0SeY|vDy$xx#QI7JY<9Bn#;zUQ*0;-7!&~x6Vf1=y}_&WVfia3Wfry$o)JV>Wm zl)vWbV?0q{ZIorcwPEWo%+=v3(eQk|4@sN)yG-e}1Kh}HBw`)}3|lZ2yIRp2_sT;l zuMi#$ocybpm_pzG22!Ir=x)@=9o#Epo9_c4+XT-Du|2~(@m!h1FBcVbOvAzMjp(E) z#6WlDl{bdI2&{E3E>~2<)#$Y)UjUgN=W#G3L;qASDXtjSRP-Rfef@f}jx8c55q%L9XagyKPEi#!rQ9vWxi_;(V4T>Tab}oJ zd4uR^O{Sip{T(*wCw4nSC%dB+Noz z3FiK;no*122#FT7YQX&r3aCW^Tu65rv}=M^pxiYFT8D>-hbABx6@Nq3J;raCDm=;H6u>9MOY!En zT*Q=9%7g7vB-9rsXo~1GZOTA~*FWBI=C<3g6D!edz~>HehEMV?Y9`Q<>lXlkhlH4A zP{q|8w;6+7$fk;D+=M};OWb)_mH`N$?bC=KcX(%xmkCB9$67`r4Z^YV+%P2ae!v0% zW6j&qqly6+R$YC-;1;OLGdE9+4vHc!(!a=AJR z;}A*E*f{^NY@;_Ixd1H}yWK&f8bCw%R)<#YEfHOM*ujh;ee$oHeN+y38SACwqrsg0 z!T-4p*y?5er);W3>pbf+7K(^g%wZ-;g&ar@_X&eZ(c^RHnYra|!!zeJt>l}b8NH=Y zkvpI|kmuQ(8SW_uOl)?%EMxIdG+1u_bc?$|5GR5g?}Kr6C(2+IM-&SxlA*YY#0{)a zKqUsGuK?&Kaf>4%3bUAqBk0b%1gRR54>3{EsBg&YQR`2$vF%3lFaV5*Vn6brDO!va z7))8Z5``aY5zuPFttSwUygQJrlln8bQ&=~74oF|xdQO{YxTK|}wW+49#~kuxdM;?i z?pVw&fL8(8<5HoCJRd>OFF;e9RHuecLyRfQz+5G+1e&*=+bw_@y1-RxAyBR8%{o|v zy~xQiN1=rQ!`@9&(Bl(J5dnvOX z?^HY$8dH?60N-N1!E=GmKkFtdF2da>t`61osO)lN;iOgy`3`xu0AJn#q6SiQQh-=QrBFER2kr^O({jotOQG3M})r4o~$YYVC|vv?#-u(wsQ)M5K3Y_wG+OD#m- z=%Ll8@%!*(m>~w>Vc`ziqO8nX><5u*ue{q4 z0QcrA|9Js0Qvoh-#zMh=wn5nR=F_ti=W3O^Rd@D|7!3Rp$3_Hyp6-|D6XD){TCA`9 zTK7myZ%weP{06Cv%I@3dz@6*>FrMoNXpXUk-Iui&LGIY#mRFO}|LgJiA-A0Sagm2l zDD1a%d8o?G3?9l57`Xq%O|^eSAx_}H8VwiE_>r15c8*m}+th;u(8|}oR_t!!7+MdA z?N@An!1jQjCHL~4pX-kZzFp?4nzZuz=?n*a4ZI>-_TR&-fAkfdU-i)aeSJXF ze%Bk4bFPU`TYf9jHAR=VxV2Dyey)Unknw^X+ffHeQoG zG#kA@$dro~^I~A#J|;`bL8FG8DBTh7s@h~_%+LqJtYhH=w93^El z!MVU6#8}=3P=^q9_-sksLOh(L{rJ^Z6Awxv7F7J%2eZ<(?PP`l95!JT_^9fx-C*xg zvTO$ZBr=H~HE|IFyG4BL2=`^a#FDk*LL2b<-2apyfcasFbE<* z8s@RX)`(r`7J#tu?l|)nq)_7h;b4{`q>?Rkf(&h7B{~-k56E{Fy9ch!!bj_6&1+7; zxD8~4B=Kay7@N0L>3@kr;$;6g~A z#q_K)sz(%7wYV1KuUiv0f6c?i z+oaNqH`V1hNZ*|7XzlRBPixRZc$@0F8+ZQq(a`<9BQ%8Tt!PKfLA@y0Vrjb0BdHcu(Zts7*DZs&3lyPM-8h9qsRRQ$u;?4l6HF0z}?>!&!0f`4FOcZSK$)*SWLU!p*@GGhJrM@5H z5gPKc*d1HZFJQkD5BvLxr`Mz7*onGwjO%bV~5s0c)t|EyK0a|PZ? zBNkNMvB$`piPk2f%Hpo?q`kvr-E<7KrAi|}s4}d=_&4DES`r~PpGFEH4bhkhEF+rs z%)n(|kG46OfNuD3X|6#uyY_40REB{D47Dy+xuyLEikJ>ZBYu~f#nbVK+)C4pp6T>& zN>geN8#8k4&)id=Y>Oh+%AGK4wvV#i;3qG4PTc}xiv|cPrjw-<3Bc_)JBK( z0hyP~vAbI<0w!eRO_;4?MuE^pC*BFTapO&vI?59xC}8l+kmesw83!uL_xTXUHwFtr zi@d^bMrQVxznmr*nlEv8VU&yq1EqnsNLTP-O$tXSW>mn>s$~$juh!9S+C~oZsIr zbqozsaKZNC?j4M@l0X5b_ENg!wD{7D9(dHddtEN89Wkj&Ik_QpBzJU9j_(y||JOc6 zH(k>Okr&n&%O6eEfV~<|f1UoRea)Wv@qF4t{Y_?9Df8nnhs$2j@GCNu=lv0SW61_7 zdy`0>#!i3zS9t?UmLC=rhQ4X#6%qPmI;!aKfE&Lr)zO%buy$0@IXxX^t%$k+S{pyC zHXv6-V8CdS$_@>f)1)ObUBKo0GUyset9Mc<rI?Mt zED&($tQC;#l(vg?b#<{1_huO|*|`OZsg)uLe=_JPGgTErQD)~heeLV_`WH(LQ1|qY zBs$d8s%iF(WY!cdzp>3KOm2p}hsOsO|LXG##l|nZZ&_t`UZ=#aOw%pZwd8S#I*(s< zjNb>LM;^SB|7l8J;`{DkbKB)_i+yh_p1EWBKWX~;+1YD*ZrlD?@+4oPkzdiZ@~i9n zeAp0PLVDaN8Bno69!@O_A>_HU(D%{S#b44n(FJeuxO|x#3pJQ(YKHs)*RPk?wxw&E zeoT^_epRAl)eZ<^$6iJ{>`K4?U0FFVAij8E9P>=Y%MxY_XA+m!rz_)>%+9by@$mBJ z%?b%fouzjB*SE!W5+lD}SLt~iPE9c~x0(0TzkzeobhUwkZNkq_NzYlOQr@=)9mDkL zk5y-!orqfC6>7qZb-1RN(0FcGFs)pA`NL~l7e7ai#os!KE)8_#+52Kk7ppostwuF{ z8W5imBJ^YqjzqOb^!d7;{xfyz%D!~efa6YQW!CWF|6OKW-HKB!yg)&LpJw@{pLrP| z?4H9}Le^z}_UxJLIUnzR>O=22{8hp)E)>Fbx>HK*PX)}v~S6rlxW(T1*$zOc#*VdP74YNHs3h!CI>Wj#dXS@VugeGZZ%jH~AOH$X9S}4Fp1& z-hh%Jv*tf?yA}HF;@P>N)V6H z(p@lyT>&iSt;^)t`*-e)2djq+J3p>Gt9%NT>>W5Z9#_5W?d|<-8F?xA*gH6QF<<|R z_x`l#be8-1IkwgV_CixQ{M)M?JNUET6(?r zrC*!#Z*#l3WorGX6jk28f8E1}v!WxNo7gNKK=w{NvU*Egk!?5QjFYz{B{($%zM@#0yb z57j3~Nv&q%{%_HWxJ`m#PZ`E_p0a!KDKRh7~Qy#bscZmzp^_T7$2Q>H|J{<84p zsYz9z>;HXnT|c~Vp|J3LQBmW_NM?3^{`9KU)B~CBuQEiVO83^VVTHxUeaF|SJmg{e zi3nXiSNm0Tqlu!`DyMyya!Via&j0x%V1eS}sppJ%9*IeWU9z`6mbk@0Q1xlw>c{12 z(Ql)#+Q`fack=(B)$;kc+$IKdDtMEazxO+}Xd{OgN-T<%Xk8DK5qierO zp579&;?$xw>qRVXxFpKQ3e0;lh3Tg@ZqwBn8y-yCC9r(rCdaI*jQXS?ud6*FvpzbU zXWrJI6uOlWKW&Z&XZx8K&uWw2ZMPM_${XzUuu^66#PLu2S1qu(BE`SIRZyO5)7dFe zWj~o}d(JegaDJ6kf7-ih!J}Zg@4U&IP0s0ADec=YFhxC=sX1KK&uP8E+OKC(vTKQ! zfLC2z=XAl~%axTXPg7K;J+}5Z%$-m3k(w%7gyW2@9nM# zd{_aY(9qC5Fk?jfd)dQdqOU2ut@ePajLO9eyp@E`6?eFH39qnB+Z|J~;ckJ_Ler0) z9`r?D+n+jpx&sWj0|3t=@84f}NlH{S3ypgR*#DV1IXRxwvBwzkUf`sXJ7gGl9zB`` zBIT&w$ixtWDfylb;Q+9Ut=Bv}x4#*x!JFPgq99HmlmjYU1}+ z%Zituz7+%=ZPpNG7<7Em+mqqCTxGjvZ=RJ=f>Y<(NR4}z6FDYkBEYrxHP+SEstxGH z?~BdO7`B@(=ysf^zTPacZ>qre^D^ebcMhwzw9T6}N4(?xnmGxsLlWoh#w*o4({pUR zWq5t{ippl^ai4#+Zb4L#ySiGTZ1ea{wcKuVO*&gMd3kuuzE|Gxp1F^(osdr!_#Q1O*#}j_oc520kIsG) z@S~~xVFH|pt3U(ZJ^eIFu^aW-^MIhqyO=yJEF%R1&qysNYwpNg$nbt`X=k^Tk!d1v0v z;Q2MuGvmno?Ps(sUAg5JKb(0eZS~w|x&4;@%t?=GqmPe)4UhSz=S%Ic zm;+Xqe7;XPzAE4&|lug%;NaMdaG!~F~IK61^}Yd$prz-M@a}<6*~}u8E8-!wx*6Wj_r*@V&ctzSZN~YT>!Qdadko zic4ZzCb{RP&OG`(*U@jzb^R^Lj=9ThjtWja5nXE@dGC=|w~qefGsb%W-iy+My7ad8 zty`@L2e(-b4%94K;JXd~W2w_{-^TKUT3Po`EwA6I$PD-96w!aqn*FQBbttVPW8~t? z524ora^Lti&0TRQ=3_4>-x0n>P5q&k7SFhCIvc85zJW7d+t=uVa5O%xsNp0Z zpNITHLXzcaeUdO!vwiB8@#6l4)YP@Z?Ox-0tJ>I6orS<47Wl;L~{@vf& zHt~sk9L1dBA*;@xc0BLh)8s)@kmLg6sNnIR87YPrJ}lBY4(( za=dU|(j!G}P1E)#8Mj`h*ga8Gzq9;J8)}-){Ggm`f%`liYSP_XD`xTya$<8D)j*BG zZN|UWBrMCh9D$R$>Y{ns&v*Aeemsj9ch#|{s`g55->tb%t@Xr=m${(qX9?M(eMySf~(| z#C68I`E2`@FK?s|m&>(hF<5V{=JKVnkzGov=(2m=moFBxIEU0>^m|>#e6}8Ti+Ol@ z?tAC!=jVYYeG1jP$YFe2wo>l9JX20i?sfe=&Xb1@UYFxn!T*{cv|lm@Q^jSVz$IS# z-sQX&I+b$(ckkSJo;L2p!sb=Y`V|wDH2v!@iPM*ftozx~k@e6x_j5zTq2*MLyuM-B3*sVV~!z$fQpf1M>@v?r`M?RYlBiWQ=doQ{TwJiMklh-w; zb9iWm!cf<^rF#3G4Rf(` zDY$1D=xvTEsT&z?6Ki;1HR0C#K|L=CIB$mNXS5+rSVn3zSWe`dnt4s4NBVd#w}`0P zIL6zz(aW5>W>cJI>K$=%Np@VuzG$<>tJaD$S}%__sdJ^+>6}+pOfNOK#jqZ79(O43 z)5S9!Jv!<0TR+(e4)ve#@?9d~c3_WgY<5r068cS7t}qWSQ$6tWr`WQCuZ5TDI!{E3 zk-LsHSBDhO~5EPs?S?myCl0jnaeTzdp0sp;g@5N-#>r9Z_D>z zF1;r2C0__SGcKMvF(LA!qIw*N!HvJ_zQ3y2`Ks!Zq{X((>K>_uqO$C~nsozP)|<^s z$~eKrSF>u|)NIba|VKks~_^P%1gpzZw0*v<_`Il?3XSr5} z(5vZ9vfTDOYU<$OhF$wB z?dL?F^Sqpey8pqC`n%;LHQV!U=VYI&a^$f71OM>+kBnGA!w2Z~v?!IhAwif!KYQ`L?LH7gSa~ zby`{y?c6QD*0r}t^!s3rlb@XR;L_Zf=g+tgbPNd!#&%bfAMI^(Q#}91*I}ijo`jTk zov&(~;fIi&uk&J$f17J#E56L+gYfZTCxuy@-!ssm8ouz(>mBUea@X?FMZPwJ&zToj z*)NTK(<+J%boagQFZ#Qm1f|)liLC7Ht&8zK*FRf3ZqKtzZ0>sL3P9yrpBOFK6(+aO zYN+MOy`N>bsuLsF(M<5>S(#g-f}uPia^UUf`l z5}w`js_T+JeRn!|&~K4^|C-}n1)6nj*X|*x@zYcP*xX8{pHDen?(@(3{b_?gkH*yX zwP$a4{BpMSX=$d+c~oax1a)72eY1vl{}WcNPcd)8gV+fRC^U_}avzGzJiQ+AVraQc z;K-B}YvZCHvE9ty{bj@L8M^zwOPA}gDY9=XniN%X_RRPlT*s7`+?boQLOfZ2=Ev)d z%%==?WtUHh<78Ox)hvxojDT9(;RnE*gS`jr;KO;QpU8C!*6n+vG8FSx6+VA_HO3mBk@zd8yd1jO0|Lk)a^ zVqW|f4B5Ose>MQ$Z2}m7!bOhS z*^zh???{=a$;pO5R>~il&a|;LU$Z1KF76eYl4WoS11C89Pj_)+hbI~UBfzRmG5=Q` zm=>P$@?r;D#U%UNr)zrgTQ+PM|Niz`^Pd$V>+p9m&p7}=;PH{0k+*KFI=T>Bimn9f zS2TSG+B1!p_Z-5w+`c4a2xu#^0QXAla&U0Sf!!qj@Y9`TzR7@wzgH&DJaeYu)-(Cl zs||rn?dPWZb`Z+1gFe67+Y{6lD<6ErXLWOB(=olpGYksDm=^@*PnMH&LET~fIdj5` zR}JV2E*!E;5t!+F$+XD+PPB?Se_862CHHpxv&kfq!(+p^AJr1CARoHoAI@M{R9u^k zCJGC%gG%PiZkWOlk<^;E#(&QFGn2XOlUBL8xh;t~@c0#CxMN%To^Qw4jx~MpOtwu8 zu|2FESt5UPweMNWiW#A0WobqI=v$->H%Jc-@-MS(kQAB4`!uxzpj4c#qGT~d@(}x@aL$AUmIgf=jCitcEruFp2#MDH%$3VT}~06 zaMRbXBQw*STnb_Wx*H$d%4MG_2y!qH`TT7U7d!ie#WOPByy4@$DEI5fPEYl{wo_NR zt>Z%g;Lf=uVRGzNnWp-k=ifiNT)7qNzGvwRKhe;+>na5W1eOC+IQsJ-oAScYWf{Q0 zaNIBC&Sc!Y)hg|9L|b{2-6;*R4eIKmNmi=LDk}4aJA=io6Fyvfu&>5?drCxU&5K&q zc;2G%<31#3NXc`zpY)BG%InW{!(qx%^bo}=5>weeqofylIMeX;sb3wf=kVuQS8shT zE17Sj?c)7ngUC7mtr=?WE{!Wf%Ss1_9CjTt=*f^W%2M3CNB7B{J!d%M1-Af@`u^3Z zi=W5CUGMglxBrUd#k|N9+&MOy$di{dwap`cF)KRPuNr zTHBk$`68!hYR01In3b>7biLOsDGFU0^Qy<~!ut4SS=`CT_FaCr>D-mqyV_eCxY!-$ zEmogS@R(i1w0|O2gs7Dao-i%z?8p{WHToERcIOXqo#ErWJyM7Gij1NruiY|fVZxI+ zVO%r(zKU2c5uWe#Cj4o~0kO0VfjzOC21D7)fAy^R&|=neHnaNnx8*McC9AnUn0CM3 zE|!I^k2!p7-I^y&T*?cDgfpr$_I8@Zea%f?mHdmXOMZgFshb5Z`=u*~ZiN(my8>h_ zD5ub%*HnMql}Pzh8a&M2*8*7n|SiL(=EZxJ|*y zQ~mjz!<8z&j{*opa?vM8+s@7<%Q4&BHu@+=83M1!NSd;;@ctTMN^-WDqmO+a zN?H#H2ni|UoquidvV}bf?8{S8!A*O~H7kDWmcXlp8zsDG2M3i+Ov`hTtkc?UF>11z zcF?snWvNW~1}>ck+$@Xvg8g#&n6^s#%XeQM9{;YXA;iG=G`?i+p*DpzG)&pqvG&4iWPaXF`#-?On9HnUh+DT_$ROUx9H7nxD{EH5_{S z7Q3c;eF~qZKpb;Noo|?ny(2#)>3h`CfC^7E->Udxro?kQTvfgLQ$U=+Ds^tYbbKpv z(DU6-X$p&*9*U{2)nC5GYjcAXm1o$-Qn~R|I8&F`YW!S5A!*~*#{NllF}O| zZB)CM+f=KK(sUmwAA4?EM7KG!+%QtnK!oj)ujk2bZ%*|3HI1x%qL{fk0f3f_2B6KO z0Wta|#;I~U+a?GLbHf7{Y|73zG%%^xXnG*1Xtaus0yApucozqdCvY++YC_el~W{iqp7|Jli;mKTw1xYC>JRny;q03eb04E;Apqb)V*VE*Cv8-2cIbc#5NWp=j%Mp7G{T%)s zz&fQCmZ0lT{noWW>|)1%o( z6nM2mXu<}qQ+^5tISXoRwi+66o_JgbC-;kD-*BDlY}_S?0kXF_qaIXT0swVstx2{C%%Y@Olm;XK>#&M*u48e*VRDk5_~B0&!MVM zh0ewCH(PBPQ^8@$GwYKAVA4o_Cd8e_ce*7+dwEa|<&mkhiBwEd33P@W-V3HI@lG>W zuhB6xpKNNjCc!BD6us;2sxz-67^)I>r*$uEu4p1UQ}<>I9m z@d)Y#*70BQhg_R2^5YBDH}F)n-9Xe6ClL~N%8O-Tx4fBuOLN6X?%6@qG5#aU>HfV6 zioc_5-;?9~wlfQ&e;cC%U+3_YVxEQf*$EYqw7j}PbC+^t#kzwE? z8=;1SLMjJu!GV#lE9HO+Om>$K8Z3+(hCDh&g~kKpQ@P0WMur?BmiD${Fa%E8Gmfdd{iDyhi=Rbtj}kgva%UzlQcb6Cr%o@KFG_MQtXNy`WQ7LGbAM?#SgO) zors!5u^>3nf%6=CZ<*U`s`h4u%_90f+^kxMVFdueQYmonx;smND-_P~PmO=SlBMBt zsU;#>ORs8A_k1wTlMAT&rdh}->f&nor#ULNUA#1OEB>8HW|0F}^npL%mzb$3rP#w7 zj$l~KC#{PU!=zmOY zgveR-F;Pe~(rd!Qqg@y?1|~5jwUue18^R|9!aiEcp&+|0nfe^yL<-+;3DDJ*Nqs)0W@y4(V9)C^_~A znA75R>NRy~zH1m!C!Vaoc;#&Jn5JiAWhDaSy624zO-)0L6sFcA={I z23RAlDPT;A54H1+23auhb{sWyo;D1|MLMG zmbGIHByAo3ZN#2@cqv24A>D7WJS8DVY+RK&{L7ia)NI zS4KvL`q4x2?c-@SqxH?}_I~2No>;TG<04uCP6=-1Sk-&Ff+(o456X@p*NSRtwh6@9 z+cVJcbCSyl4UFh&LOYDEj`me4{1G$-%Z_Xn+frF5@oK5jTTNY+wca{c6@QsIHIpz`dx?+m8Vg{vWazE21{VqB7Xn= ziW7xf%*?`~N4QyPZEl56gQI#SZQj!6t?`9L6bc4?yz6b-5~tq1oHo{I!NX48S1YTR zzWjbIa2g$(a5>^N_c^*ZMTf^;Z8(Y@O`mz-C9ufc`RSzNXLfNP>0UEDF<~D*GVN=D zZ6{P&Y%ALLHaa8i_nG#PO+MsIF?c4U8FfqwAcE;H>5L9E>nBAzJWPc^7MuS(jS^m8H)2eqcq@51kWPVQ9F2N*Mr6xo`s(&GAu8&!#g4R92G%niMq* zBx09lr`JT?C^&OtIpUy+4OH{OUKzrXap$GFXOyl&siY+-<)Y`ok$qe7+P;mYCkM^Z zp8vY))7xa)>#0-3gB?S>{C#_nQI z9^gdAvS3mvbkSo{m-OU#RXM;(z<;9d7)~%=X+=v+>*DIFZ-qmo?*0aLX=&7|jBp`5 z)bytm!NNyh?8gQRrs;5TncYvh45dC>msY9=wcsy#+T})*vn=RXzt#W-l1OH@Q}snIrE zw=vEp0Qcm(=bgr_OKe*vD+#~9Lfpw}7OFDo>b|&JlK!F&$-;U-MT1Z-qxOIeqQZ5_ z8J#)VQ;AIzPW~))r*22{$mV)qKPQq) zi*%;jeML2e0A}^)U+>H)|Bp37$AvGfg3!_aU5SdR_bIHB9((oRB=?8QzZqs~p`Rlv zdiRNm#Xk9^ww>4=%PZ-h_3D0bPcxEEl;yBu+j6^yM*FHtmKM^+FE3#5r2p*wLUfj7 zjv0U=$2bp)Ye01SAu*8J+uKucnO@O!S?1bW?Ca3gdyd4>N~in4?DPPa3Tp8uNKg-< z0&p?ROa@3AV7y8#f&7`pO-{}$H3vPpzmt5NJUu|`jqpXl5}>1f$&a5Xy7gP9TC%)J z(ql*>zG>OM4Z5uf!Z736z8`BpS&u`*XY;OR5&7C&C@B#AVA2*72;>rkdiPoOw)qu> z7WiGM)dU7&VNb(rg#sT|vQ4Br1PX^E=sY=qHtgFFAz*(qaVHhXB zod9R+g6``vP=oKH(dZD662n+H!3PxLYVVGeVHE)@vZF@OQF~M_4~QYKn74DB=NfWr zTaMt>bMMKsWci=dHb^3{y9w0;Fxm7D8py8`Q4XYc-fM~2{_b&3=y`8kteW5RSdzf` zu_H#RI&ZE~VnNi=s{7Pcvc6Xw5fO^Po>*<}K7${8eDGGQ1+qE?JA{V1%!h9!1QzwL zA9WbEVtWPzbVmgTwK?rwRqKb+5ihS%XDZ62<#gZPB_y#%YE=S zn058GMX0E__OP zp=|hQ;H|3B$x3Hj)T=8-iWFUiCxYi8LI(AsSM`0GbGLa?(6rAU8Kh@)-phUj)B;NX zsw7$owjlPD)h5G-xg{sAS(NTH=kvc#6w5Z)Gi=4uSxOn*yds*~qt@zTuRi(s3wzh<*SO5IgQwj-X}o)gsjV7|!s=O=0*qrM*EUbJ1+lofE@iY=aT zrld+Q2d0ztqBW)(FCcJy(2PkdL=eY~=9QosrwK1?JQ`&e*Q4bU2U(2~>qE`Gt*-pW z#r2fOHxeastruZSk2gg}Sr35yyrGsHT75YyYrDA*R@Hu`_EB3cyquA-#$Mm8>C-%I zsMlA{;(_8k-WQwA(eWqqj2TU6D6<97H=Q+#=EZVmpj%&cL)G7Mf>$W`A{NT0Rg=5`7`;nh8*NM2PZ>?Ai?!Qn`p#)<&yr7hI>92zV;ZMB~ zNDU@;jX*gEb+HadLP-e^Kwm!Khkye?p2&}Cb=ww6cX)w7&;I^m9PLiddW~f8h5)%_TUpUV)i0uE& zw#MxQo7 zxed?82e+Mp_%^1^anU=RRGTg$zz!~Tt=JJPaI|LQ2eee3yf*?F)=c!0UX z6Klq9Xn@N#VIP8ZOe-b|hFRl42GTnysaZxV@Bz}<5a2@DwQNCe{Peh3jO6m=pk-7D z1u-i9n?0pg^(QlhII%CNjt702^MNP=+NT@@E{;FhZ4Dh^mx7&5Z6M{$ez)`HPCzcb zo2*(+;oWllOL=ikCVNAqX8t*Pl_Z@a^u>=X|D=^2ONgZ>_gP9VywsAVXLIx_rRaJ>UYO0a%YG0@ z_@-WS*q9C(RrK<)IpErfipX@pM-4GfXu`7bmFUyxhSxZ$_oV(nxI#1O19*!UKh_qN zlhZx^U14)-Vg#TB$qrfu`;%O?>!CeGP|SIa#?b|*&7Z$Q}@eAY~qGO z;PuCkX($xt#l5Rgn!&?>iTV)sEHvKoL}LyVb%zQg2Sb@Sm*2wh&n#>|-nE!qzhF>r z0)`1>n=D#%=f334I4UbD2Ej~|I;%^Y9gxD<#;vV+owXhNyn8l_G(w*+vK7%%HCysu z6{a=a%D59W(NN$KQnjl$0oV}ReqI}hV2wKD$+#eA^54aO8w#Pn_qcjT?lx3&UU=GbkaCH*=!1L_ol6%}-iqcOx{sMb z^ta`PNN~?n%+GqT!U#WT6QNZ|Wg% zd{NA_AfiO>S8kC6`+O9iEh8jlhhxuN%+~(fd%!cn4@!V^=@2va$bQn4Vxa7A)^|2V4xZ7^S&QusU2 z-t=|6b?M<6Y7-Bi$j@tSaRsWKVWzzDEEYbe7}nj5Bc{w>7k1VQm=bJnl|4~SW*_Ul zzj`HcMK8C#Q%JGLVpSh&%#wDq{KXA2u@zMpIuqkEqY+k40*8eq+$R3n_(tUlycd(E zCA+Mi$5%fnpGvXjg``@Sa+U1DwDkE5@iX!Fd;>jq^hS2(m5mOS6qSVtjE1w0hlY=6 zCrBrC6-GuRqv&iGdi7|OjLMwO^VN-vHD{Dp5Nww)X-O*AY=_j@g{*tn^?gOL!<_Nm z4fpKC6|dsry6{)us9=R^@I*{Z9S93~994#GZ5I~>?U*p1JV`!l z)$*AlX3|`OERAjzIL8H2kz%LvH`zVwJJ{yqe@aB$xwWnZq?_Tg!Nt8NVPaP!pJn-H zf)J&6g(tDbeR*^3kGw)aljvC~0O%F5Z*2R&N!Qt*uw2WO_Y4~d|0!!6+b-4Zy%ElD z{+w0va}q_;G0phu?+rEw`7SYFv3{$d-bq%x`ut129j|!EJ8emv6d&I2`IPkA|5`6Q zcy8r*-A^hxG*ndlz@4bj+hnej1ehjZ96kjHkk@+hJea9Ur7XVUS1+|Xx4ynk!Dn;h zA|6GpL^jeJrl~+V4kU@2$gm6(svnH+bzJw5R*W_bm@eSP><_lpp9CD*^<(7I>tS<3 zCpg?7On7MW@Vmvti$OQW{d!b3wFtSjtvT5G%l_8}kyK$;RTnGy!!NZz) z>hw2AuuGdF4rp*(NKb_p+4N&X1Xa@*P=!D+O97iNmfxCr!=lGe!tQh+@Ly&rqTY;QD%!NgG zu`>w^06EK@Pr=9!{`uidwC3aE1LCa219^GVnz7Z+x{(DNjq*~!lEjiYPTTeh7yB*+ z) z8#v&Mwo-QjMW;Q!&6HDS#P#a!pE%Oey<=S&6>UJd0e33~!OG*&%K%OBI^Dypa2S(( z{aVO!*vdoR#6uMyg;9CD<3QFD9+8oiAuKrDmhoCOd-QKqXJP7g6Wz0>JD7Nk^Zq}W zZu^_+tz4NgwL3S^@aT?hf)Tqij^-$}=&+~y6z(Kn@{gl-*K}xc{ zN*B5_5ruSbP#JyyZN(S~_r^UAS=Hkg zlrEXS`kPHlI=pv+er!P;MJp}lCesWrA;PEznLb>*#eEe~(`CCDnft*!LW*?v1%4c4 zyVK;jEH6w=p?G)}dy}OYMS zWf8;W3Le>Ns}sM%1rZ_H*RP14U~U_qew?a;jN(4yZ{Dnzrq&Z;+!7%K_ksibn%N0L zXsPv&PB%NI++*T{p*yi?wcXJ7URS5I zn&QoMN%?aqAQMdjIq=oIZuKa*$eYj*;Jy0s^~smEz-kk_+-;Ygt-J-E^Rk=6r4DEr zU^ZDO$~+W%sB!fQ_i_9a>EIuw^q3;2Cqjx1-Riw3?KZE>-D`;d19t{vO6Qr+BFou- zww_&KD&h@);dN-x>&h;of0q)MrTQ83LvnI~z>ToFuJ{n*!I&g*-&=KY)*Lc^_xTiz z|Ef_in7%{RVr%hTbZNP&?}@ZjaGZTNwzkAYL{K1~4+3KKf+|+%wRC;xv*dwCipb@jM)apI_$94b!%xLERFz)RG|tOn3&K1z?VUoY`zo&Z3xE= z5nkN<)&5~~WrCqZr8@{2TU@&xX2)Wyz+)!&= zjcmyWD<@;lqM$q@0|Ol}-*TLZHX29xeb{H8iE6y3CuY3cfk~E1VdG!gAs7_Yj*QSk z3sZb42M2V}ye>P;puqpq$1@yi4L_bkw;uca`ERELKYZsVU}Mu1==<@dcQ-fdw&uGg zrn>lPYeUl^C@qavQG?FN5W)G8wGU7nxs{xSdBrnuXEPQ*aVP29O9L8vpx~E(lD+K# z#f6vGuZx>kNPDC~^UVz{?y)RK#K*Ll_HOT`dUT?UrnFdzBziJTn%XomdlkG+>Gz8{90kaY3P?!?vB&)PD7-FP-c@`YU|f!d~Th z(e!jq21I}&^{k(nil(0x`h;o(P80-#-AU2gf(!Byu{m{Hu^0L`BD^C7IG=Ljdup+k>rC%Dy;FWC%Z}6A#Qtd+|qS~K(W3vX0haE zkJdtRa#0I-W*uzq)cwWskdBKH8ta>$zRAQh2rdY?*<4{H?5L5N% zPgQeNfjkA2@TZ9m=!+WQ!;FOudpOoy1izWLqIAGTLqFe$4dZ`mZj!>#oyzx-7>o^j?ZH>Z{`~p#+K5~l7q{S_T)JDXw6wGq=rXvEEc7xd zDJlQwn3Iu^yalTYT`){Dq#XyBQyn1FLYilWl`Fs6dwmCq0n`1ikX$f9z=ph;xlS@V zx?tGGPJo!7HroO3=Ni}@2EYW5EIvM-JMjJ4Tb>39htA2-J}`L0p3FLeCzEGvEH#J* zEhys3z|0T0He}F*n12f!fjt5`U|_MfzHU-C2b0J7f29zJ``$u(!afGePaK(zuY+Yq z1B@(LgwUIC>_IZP=SGY43tRz;nU7#&eKW7-3z8ywK*6T+UNV^~4y0^EMkorul-gEM z;CkKRK?VAl;BVh-)O^+h?p&do%2NlVZI*^cYRD){jU4q-u{Xt_KufXYCQA_^YS}@! zzv@tkjJY2x$je^;CJE+OdCYoUk*g>$;kZVG9U7rXkX_8^uGfLRK%zDvZ8f4t^}BK^ zCb@FE?uBAnc24dcs_=RK?3|ZLyoHI)HopqAiZwL)KX_Ep*QWzWjTuTXszkRZxfcKyF0r50=>+;x1Z}~`%Qbh_6I#=!J9F#_c}Xw zsNLUI)huT$;-z~FPnEsJ=!T`G+jyh~eRs{u-0iFfvnNZC{Q~K#E`a{J(1gT~Xk}&R z{nJP9yZKf?GX7;U)gmt)>Ez$$S&p!vY7x3INCi z6;|Xy_;+#rf|0tRHJ7xZQsUF09eNz1qVt#@W?oFgEZL35!dP3@kp-Jk++=@mfB&$O z!yA<%BJb4V`c-7>_9chnx{unDKV$dTFP^=YoFqtY*=U4bQ&2r`YpR9f8;pr-L8xOv zLs@jQOg$LK^;e8U4$#3I_i( zRe6Aoja>^{+Vfy9^T)_O`(p4Y3WkPS7dG*%?e3Q7ygLg)ENslocwl+*9R~h^Qy`Oq zWjAlZlDa5h*TmMw#tN)x!ge_qzycTEOl_V{-T3_cfOjY09{6(u7EBdd}?oG%znd^p7sEd1IRVW$;<0RWYZ`uR7?JCOlFPU;%TFm~n=7I9vJE-kQ5L1EA^918KMuZz7GPy_@;CdnI- zJr;iInnMclK#3Odcu_bb7g^~Mnizd+Y#*uXq4Q)cTF6UP z!{t+ykgMQ7)j{t2c+E~Gj-Cf8=h_5vRQOYR&;~;}qs~?vA4NCbFE%zm8-0~R76H0= ztP_`ksN%2y4o5eKSMw^F2)Z2-%I?Mv+q2&&tr~>@tvhbi@7P~ypf-EYL zHk*9KBGQ)Rlbw}uCN4GdUYm8x!{w^%{^okwJEjq-5uIXxTkL0y*1yePl*_;Ic;l^m zCL|;^aG#K~3HhRHxX|}oKXB+_3+S&d35jm-6R~lwZ_MgcnybA@f z`;&BXE(*z9;{e_F1-y(R8po!;#5Vh4F*Lv{YG?qb#6Q!TIg_NP{2)P(m>Bi=5k}yy zg9zFoy@8PB-;2>H$fsX{8n+%C0ipbznnLIzCqmA6W#Hrh$>-<0m<2rR zv6Phft5=+`LFWP%78bQQq9zny)({c#c+#W<0v7H-Z+PLsOaD-{^T1dEW5l2B?cCAv zA3nVOPrL$lH0FqSMT+tM@;9;cmxBOQ!DY$@l$N|#^YZ4yrNl5#ZCrYO+8UkIedZlA0uXOs$$-~u}CeXBN+p`sAe@#uD0;iJ4PoJ8yhdjO^ z27aWl2I8WUQeW|efmTMsTd6Ar+Aa){a|StXu=!7aoU1( zLGSvlTUg*$BLYJA0nS^uZo!LUUAuP8g8rYpkYqv2h@YQdj46EZHWfogM@!~TypDu~ zSI;zv`Q+p}9VM6Jc@~^AKeA@P7GU^VMuIJ*O9j9#pWpt4Bjtwnv{%repB_oZ=uVq&5c>^YI@#uxEGZuxc*Na&A~8D1(aqBvJeBce^>OWdAu+bDHe z7!Oio63fk^Pi`g}y%OCDWMz?8ddO-n?IE3sg?ha7NV&`&_jN@%kE%pZeSS3yZ)x;O zBpgMn_2Qi4sEx{}nHxB*y~TE%Od4TMIV)vo^p(n|y}?PuUwDC#l6>BnA7Zlj2wPwgsuWK#g(Yy1Y-8-qNezE}wRV z3gwX~-jh`j-cYn}K-MmN@0P&J)0Yk-mCt+49?@i%xv9w?Gd5IK{%xFCGQY6pjQLJ% zac8Vm>QUjW)~RaZ9lKn@1uX|be-ONYK|ln@@vJc6kD3!Ms2|$O$SB~%@@93;S%Z58 zqcmOvWMHz*qfaNm144NKUub&WHPKldUqvXHDl$~97&D2^YT-Cf($)EoI71`sh zb-W^cI8Q=!JC0u}?*9066Fh{{MsRE+#=Yor=B9KAMUUAG{^vMj0$Rp657nW*fdSY1 zSURjXZ{8>u=;O?Qm!gBirZ*ar7^_xwAh@r)^V-hfaZ&~0C!QtQ9cFg+-#6kC5(vXH zoPuiMjddoq)YW0a{0q_Rwxy)MWHJYPS4hoXsuqV5k$!Y%SruewU-~wWWspSO6c-S1 zDS}xA!`;0r_&g^SKG-n23|BgJNrWPIN?a5V&O)DI5eMTG_&y^Y?%b>A4j=mEfzbwP z-+Wk?oTXMm081XoHlN%e!K1u){*1zAxA3yGyu4T?P`tYsz7-rXJ;5Xu188sgQAXfy z>Ie%z5iK(!Y|~eDcIF2wk3oj+CT~E>069k>I0Q|9zB*8N%WXST3(P;hE_h`B6OY(hUNoSQe#-Rdk(b@7cvHhPS9Ge&?jZW~{{6c#^fbgCIU0ab_vOy2 zY3#sYY$~s&=K5couAS!h*yugpNnEi8ZFEYALeO;f*l=oLB%XZJ;Bp#sO~L-TLD@gaG*edH z-RyhzTPdU6qn%}4Dx)2ZO_yl==7M{<^_u21y=hr4@hLrXpR>{|?`e%p(*f~V z!{>nKLxqHaA8wC-%23mKdZD=uNq`Q!vcso*LrSnzb-Y+X(PiL7YvzK!8!y`4IPs}o zH?u_{B289P)*wGV!Kp{(q^vnvu`kZ((P(gi+hR$%`YuK8OViNz+9N{opp*abVWxx@ zv7-4Szxd0*K;iSZZ#z-Hvt!k+m(k;cR-V!aObXU^YY6i*hCQ(i6g%q1#&9&syIEV{ zq=7w*)r$J_SleI_Q_#$r$s7LtDdO9}r9pD2&AJ>W}|8sdWu#R9Tsjh1ybrLHvLuypy@J+|L0C+2Lf&iyi%vG ztyPOVCnv#3z)i_jZ??$)U&0UwFDz*PiR+PvQ%=&5ox-Z{u~+Wpr$kCW+e_y<4N+Fv zvR=6)PvMUZwk65B?|oVlkLnwseBnuRjg;%;-u=!g5hfCC)A!eB?Cib6#Pt`g{vAYf z#@n~CEiF^%F`F7125wWCvVoq{d1uKFnKJ{f`xa7P%L@&Spv$+U$B`Ko0hN@RhX8aM-i3uup1H7|aSsIap>c5*o^!s~9u0>- zm`KO$7?CV0!WuaRBcX@NQ6`xC&Q)|^zYhpjRuBX7yfc{60;d%`7`;M5LYnu$Ic5pA zclC^Y;ernfm_3TQnn6JJWME-w(-i`tNFWU#2`m)kk9VBJ;pO8iGF2DXNA^x3R40Uk z>W{lgU{O_7g$VBv#wk=27;jJ@Q1!^n%zRRfxRCMjl?5fbeFFpVDKN0F1K&b$n|YoB z=Np;#!@}CaJ(@G*I*ktm`vQTx|0zJ!xd0$O<93wi(yyAt9gt!pNm;kfC0s#mQ(DkZWm&vlsj|n zLDg`EO+>KSw%QL;|99AaTihvOY_mcy1B>P3wG3!>mKQ7)qQ5@l%O(x(5qzkA)wcM4 z3cBg5ngZ*h^tGq9OxeH8R$r>Qk4$$v9zXhR7Atfr*lHtFZ9ZZ{=$|&Sd!>QtYs0LX z!b-1oOULTM;&9E5)0F4kvxE7@)wECaNd(4)bOXik3K#Ce@6OCYcb~FJT1V_}l0GBp zP}>c9{l8NrRGyhq7Yvf}^du)AHfb+!f1jvJ*}sU#UbBOu%QovT@52Clu7^ydJSvP( zgO;FS51|mqN~wHgK0TH^t8KrbJb+ku4JME>OMmr!=H_BH?Z9RXq-f$&QW9|oZ-2{y zhRakz$V)89cGh(F_#EbIs%8zngDrMIf>5Ye{Fs3KT}YPqq<1SdYb^w#L6}AHH4?vh z{@-f*UAZe<&UQrvx1^OFf`wiAF9RMOuze&C4@{lUA$6AzVJZlDyL7h^VG?r5ZQ}et zL#2q9X?Fc5pJ}lEkjkpAZwPcIiKTOmiC;ZbwrF@)|nErZ=A=!bX^ zFK$7{wc3kvXHA;Rb+UkoMYF*frf_cGcgXP`ye=Rhz^qm1YP;h)FOl zm()#knhnv-r+@_nP2f3BOhlx859n#2vupE|H6HxB2j0T6aylBfKIyA~z#AM^y*^&q zB&a$*I?4k>Pw_{O@+~mW;sr4f6d6ta%aBs>G`J29E_#UAMl{u|l4l?4u-@#79{mxZ z*u7C!->j(q_Kj+>k4GbgkA~K4D3$qcre$Y&`Z2JuALVKm%?6`usys6?QijL}5Pur~ z!Pi|WUeVL@!vo(|z?0@-wq<$SKyFR-s6HKPuCId{rdE_);Jwt7VMF_9#KGbv)8;f| zWD79`h4HAz2!~iv#Rw|Ow@GX zp@iwBqD3jSk*(sKo;jk5V@6b+k}&MMQ{7OWIg53HNM zpm4h_+q|H)r|Lb?}Igd@@8_MBqlYX+;4MjsHt%E72GfNeZN*!2RG7c6k zS3zL=*uUuLXw|w`@8i66dyFfhakC%N&{G9*>Z~z+;p-8}m|j<>zGKEkMTIo$DPz!bA|pJ0%uS*k|o+MjN&z1Qhy7Y^8`OE&9I z$+^J1ztv@Cys}shrh}9|QEBM2mQ(w*{L07<4nUr7QDJe|oBRTLkVf@&xwXYZ2&0uf(nW)jOe;;YNzn6e z3Ph&_Z78Ejk!ld6G=zm435p5Cp$#my5SyC3s9QWd!`e_f{Os{TF7nfABVmNV-d@Fh0yq8K`xmA|W?OL+B zW^A^tISIxdbZzl`^AW0k`@#H?nYVt?_ej;cYeak%PGc4XUMSqEB42YSS;U#%p@!D1MUxU^zF9jZMXF?*v{f*U$m zzvBe-(iYz_*ZHPV2UMI2nK{&55NuSKUpO4iCNxp|B+CrqFN@r289N?#&tAmrTa-R<^3O78K++42O_t#_-YtIr5EBZXz>XV*q7Veq0`cE zfKaBcVK#Oy-+pLk+pd0TV4`trW#kEn?tx?s6?>xo{iESdHkWfg(2?)(Kl z$}acAAr>qJSFqIM6=d0caM z*-lbJQ)7n(M5-$h9%zZe;#$Ag+Rq`#3mxi%Lre54-|=4}rX3jnv!NShcWi zd^*AYuh;cH&qOal=u~B%X%e-!=SGU7`i6$+=eG9tjgt)l9559J;TyI1nJg}Z2`IO} zemxDdrlX_d_V@Os+UTnxkg&=tbi{$6YyI0x%}h2ltD=~b7Dd9E-}Ug zxcO@hwpj1&AJjCLLO*Z4*L8QVBQTEeF!h*0!+Vd?2Rf6hhHdx(``u6RcigOqXWbur ze8%sM$r!ooj4m-3!wgD3CQ!*l)z>8JkGo z6mqv0UWsouE>PET=F?TwYVzqSNh%OG*goP_a1}aI!_s2mYVLH)nyzMkiSlSl3|T5L+}>yt{2S&s+K0={z_9aHKda zm4<{~=K*kdu=pIHNbLz7KT56Dz=9g*eM)5Aa$0RiGscX$25$#33J=D~BRBiI4};r*fu-JI@Lmqz7OqxCXpkwT2$v2n9b=T7hAP47M_L zPY_m7tgd>DuE=9&?K8YneJ90SfB8+Y?^<3{r__Z4rh&El4^)q6>(9m#eDoGb_XNXc z#(Vcjo;)d{Cr(XCN&Nb?KBF;xetl|j@ofGL&>ye!)@@u|T&Wb$`$1%x074$cJ$-Bh z#D~s_`q49_yeTGv`X6RWITE6x=U`WWe$rV~zHS9(Q!v-FTIjx0*uMIbi4*}TfWiWG zm)~xz?;wc1A38Z%B&{nbL{JyqgnLUo3#(9jO^lCwU}6za2z*6(fRnHJ;5QtZ2isS^ z%BSNeA@P^Nln*35O~ACZUQ)J5H3r=+5R4mP?tmgEk2LI20yGDf8jb@6*^o96_!!e( zux%>|hVC#>Afy#0wqpjHFJ8yBt3X#|mJR{V0T``c4QtNId2y8p5oaOB=2Gz&VEGj8 z_wV1aad7sNb%g(z@=pkc08Av3%dyDwvA36t9)fNO zW}g=ugN9&%o+(&mRyFrm@K6HC29(~U&muVC87U(WXFP(aQ^70bI{09nLy0rA?4n%H zG}}^AG2dvR-1``z2oMs%1e-fwz^!;}(OgYBz|VYCvmZY3!v(j9>#VHwJeF5#e~Z5; zb#n`j;=FSni!cR+0?}*$cxMumOyXf-`Eh6gn+vpUlWZDN`VZyPDaKrRE?k$i^cMSa z4h3vJC-cFN9{ZaW3^3dX52sSEjPe|*2n!ryLbb+_29w@B;8rfFs-iIKm8yS-GX#(! zD6Js+;((_8a-orfsEEg9V4HxLpB733v0R?dMlyupNA(7H?XiIxH%N40^(_U*Bo?sW z9s-k9>$Kb#EEK){E@Wvy)2SkpC*1%t%K5BMJK- zAUtdGCl&sojrykgKWVP!Sc2mnTuGq{ z!!XCi*fsepkVL?U7ZU|sr}`P;IPb^I?~;5TfFIV!>7pUU?r-d+hkTQJec+*{gE^a# z5Tr)zht?7%K!GKD-pVw6r|l~uvBN_Vrm)YIdAPf5`q#Nw2~oe@70GWj7$$M1PRtsh z*tHdvKB+o%l}hQAj69S#CqN_?k{*}A{~fpM7g$xaDfiX#X@ojn%WN(`R|}p|siV|h zLFi8&(l2ZY_kOJ$mKGceEyXQe4G;KoTOJ0+(Tw@zQ}cup6I1&xfp*wbP7$-TGl+#n zEtojIl}_+;>cCq||&dUa9B zbb^=p3N9jr4Wuap_7Ew`!*;fS-R7_1PkmkR+n0U$Ra`+~dArsqAqWISC7}l|h7&Ia zuWsX0&k*J-djKb^HKvFfEU~pW{>nN!HG})Q7uYL+aHN={Mg;ZxWZ%(|t3cWagpsMK zcwSylQMF*dMFP@wmGf8vb?wsnttB=FT3PCpCJLX5Nhk+!!fyFY{{9lPxDGoHeitpO zLi5J6wLFBQn@>bZIgJo`bJUyfN6i0e=*d`2D{l9D(9VZKoBkXM;zsMIGV1FR-AibYCcqx533$2sdi|)@O;17z28JJ*D%PrI0AN&Z z_sSwc2PFcUK{D3ikdiT>kn|T15sSRPHXQx|3Z-_LL|}e-{Xtctg6@ApV&~y}a$zL; zpIwQJFHN@pF)ttr6z>eZxTfq{N;7L)CS_ zW7)RvM?;Hfmk}yL8I_$?lm;@Bkr54KlWdhznJH=5dy{O5D3n5Wh$OO=kA#Zd7z=v51li~9Tx$V0B<1UGab zZo#cxyGsU*j6boow!Z1*<#n=L1RJXt17>KWaKDkh(t9=IR@IS8n^nbY1A_a`P6o# zJ*1;@qr`D;Zf;A!o}j>%f47BV1d*D8zoSzxf+{iom3-WG*Zz<;v?f97fT-&;F){5F z`y;0boe9h$yN$91Ay};dHRlKh;5xh_g^$Rys%l0p4!KE5`($@11DLIN92w;Tmb~+R zEnE6YG?JF@+swd;rrVZMnL7&4WA)o|_0=?*Bu(v6*i9d1lbN&p{e2fXWB2Z{Q`|wn z!XB(;au)7N`3#cp&3j?fP|U; zV`2W9HLW?-#j99!GQ_SyS&modoX6!*QTO>-OzY-Gw((kGLu<}nAT#B_)UVJ(% z`UMI7jwb@fhZ7SsDGYjhF@X4hLk5GM^XFf`BiTu_*Hd|K1bx1>wI*Ftk-O^bKhFM{ zDETvVD7@}zw(-E=)m4WLN-JG^RVBXFp7b@d2s<`$ru==d)4PfU8yd5_#m9%9U>wJV zh1Sv+-4<~XiwU$HWfyh|q%or9xEKOTSr?zs2Ap_z{ZEut^ZdxVyRBhYp&-_;{8-~+ zfPHVt&L^WY6sGg)vsdN&S&cIncJ5w&;HvVO;8r!8uvUGFd3ki#HTispcI6vSrsLjB zjUU=u7VjC8}Qs12F zh*#m)G~>DJmDKCA51@YHPNamymxU;NXgw_S`YN1Y**ZOix07xn_n`m#=C*M+r~E#P z%(m(kK7*(I?n82E)oS-A4T1v=;lixHb+&G-zNq<;FMHF4HwC*PZg}FMUG-EKD`#br zRadzhMX+@)rOPE-!!&o zE036ab2C%d^%xb1SFbhH;@D)^khtdhaGoy8cP%beU-J5Xc2QV2FY9oVqJuQ>(Ej!-AqStXAkov<&k;f0%ZT4mhIQPZ>Nae?9-VsDuf4pl z7MgyAfV)9+I*Wg>4kS*wlshZ=cr4t=NS3D5r`KnP#(>~FIdEfY@TJH@E1)-mT9?o} zgvl<(LRAcAc{GeK!P({9Y(f-xjf~XLke*ko-wv%rC{1`@lA3%jDM15CH7F8(?SUi( z)zjqPHZm?_KkYG3A{qj+EgeuI3B1%gTHVxYNA@0)h9Esj+#wxB#D-(jbHxE=I)=wFTT6Q zfy!)KvbPTc-(EN{Gu~%-kXKT2r(8Sdc*``n4?g2Zj7O_r_6a5#3QgU+ zmd{7BNQrYZ$%jbqU@+1G?!XThUWbRX5^IdPUJ%Wbm@3@=*{}qRO(00neXh3pS;hRa z4|?oTa(-p#Fr;TX^>?1`7``g73kzXC^|Qs;lXcyvQ0E z?eS|O308gLQf9FP+j!hf-oQL`YoIBtv`J6u2ANFmPH58gP?LFE%C&SAEa zi~;|448nFwt7%Ygx$;%Q@j7JPe*SLI*n~#My7HT$5)!}480R-^n`EM<`DMnhr{sSq zRZkCe6pQ^F3TtnGPhoV>>fuEB+h|75mCL?Cn?X?X&Q%Kt2(=aDY)&g|q^qjBM%&|c z_g4wj-+q_U)j?q)Ink*qI5qd$kS4m=^sVyMxJ&BJOcrOER!7SD^@LS}D~r)k>wTG? zdV?U z31=|RdK7f_(a9M$hIwWh7&X3lc<;qGmfh(WJ{gUNHLTouL(#w4%V31_2ow>vi{$Ln zoiS&KU38_F-N^RnVnbQR!MHnqj8f14lu3V_8KW1;Z`BSS9%L`i?4W)7lEvqC^M|f= z6pue+WtANY$BkH=I^EknbdF=yrThaHwcq@xD;(9VVd6$fIq$kEd zdtv;<2cxz_)#A4ZU-OtOv+}WHD-plmL+b=JBRk!iHE%HBk4jH(fMNSzK%5Ke4W4u? zLa`F!;uxus*33`duKz9&Ix^f2vjdN$2)X%j?b)*eQ(JF(RhA0Io|L6+Epn4e)h#rU zM<<72OQx|9#zNo{yAOSF0^OkLnxl)EnR#iNQfq7LzYMhf;CTv~Ha@XiFKPKrT0T_# zi;)gQB47(W5gA{}YM~yWL9~pH2^h{c?gN{YX9L5+{4h*(_wL=L-^cy^>E%PjwxCag zM~KM!C_EQp6BYepG?#1Dg>eqQwmv^27kf6ziq!Srz0(YqMC6OU&9%FfEA4ET*a!9g z;F-cKYL|@ze&;u?-9vPNc-RlT*#0#?MKHEH^!#0e(zz7gut4=2NX-0$H0}iim^6P` z)7f*$FP&dwe0XRLAD@G%hgqic^oC?zmray(6hgqjDOCR{$8#p?b7x!L{ypr0k?Bfn zoSZNsu6z5zw{&c5MT6((fiGGO>ud$-89e6IXwa*8KxJtkl%+H-?cg8?MZ}|*VwW8| zc8cQu@+I@lTOnlAp-E5ss;+eF)v@>Q1uJ>>%27Qyv06`WmF>u|Z^a3|xt~&tH=y0x z1K^4q)ZC8m2j({Fxw$a!JIl0^(%cF`ncob)M(W=$S!FfY{on{>opfEpeOfIZjPNR5 zn!F?lWa^!phSwf`8l6?*#_8~$y&_k0^|5;!FWg(LsC#hA*2>fVvWB*29EpfcJ651%3}^UHZ}w@RGc6D_Py(F)#vu2-gDtij70zX<|7r( z$_7i@pf|f~*1S;$@TZ-6zAIa&NX%#Fi1*A+2K7&e`hG47@BDgaC53`q@PJ@*-?r+g z#8b<+1uY&)YFJJYG7r9qW2^OXc5ciQ9R+5T;&^2$5rfzX=-067>AcE~Z|GX?UQ)tM2Km+3Tcf&5 zOr>S|q2s@%|bY9O5@+FQ0aqmkiOHkcW@iAH2*qC&u{F=0X8?Sr=xM7Z_4rWjV z9}eR`pQ1&t$}XADmWKAS+0lF-J^oZ3+kR@yeV*;^eYHEsAShz`q#DM6%67s$;#sGddkrl=W+Y%7eEEgACUWz9q zBoLC_=eRR0-pq%$0&F&8;Us}yR=zu>?szT&BqeqAm3go17&Cz+N6L9gvq~d?x_R?v zy3J2TFmRvBpHWbFm#Ch^Pspi*#(8Y2yBtL((!Tz8bf#|^Mn({ht$l}!C&pfPUB7+w z23lYYI!iE}V14#^_a$S^Bg^ryvo9{J406y4!x6gqwrdcYp;p9-yRiH-s0lvMa>v5g zZ}Q0$Q{c{ETh1tY-s}Aw{`m^&Qz8BB->r_KUk0c%s81f!HP#^p%LJm|7MBMncOo3fg5zx#K$m; zKe@oPb?wT#-s2``9iUbGF=U{=)bn^)=6PwZ$!dyPkVr1ty5-*O zUZ$a2(0kz783fHiB!}9s+Iw>Ch3Z5GdkyG<6xV-BEj?7RBlJl2{Na{Ea>7oi=GT6o zSbO{C*;b(elgf$bR!I~Fbp^)n5F@NHJ2y1B@JV95@DDHxBN`ed>+bvhagBE*axG@h z%vyw06Kp3gJP>#y%)#+hWNx^iREsBeMqHx%$)PyY=bzF9anOV=;#ATw{Ai7|i;WBF zf?|=OXgKGT zv*PawaWsnN5+Hx&>Ne5#9Bm%8M;dLpZU1VU<9nZ0Gsx`gWgLLpIXTAVn;IG#$mm6Mnp}>mSUR&>j|&P8hdPg*yek{qq0b^_yBf2n z%72Er`A$t;@11abnStC7GYj9OxlZg8pZ@U>{aY4E>G$G8PMkdX24|1#ey?(T$<>%1 zQ;Am3gHDPqDnOL@FIO==oS}u7szt{_vm{F`h(KtX#+d9|=(B#=SLirVf~0B#6Vp|Y zY;pBkT|{~VC8z&rjh{|TRwDy>80vN(pDKOxrs~cH!84AIqUg`3q9BaJm=oz@^Dd3)v$-AA(nokhz<#5*;VIqIT$ESVbn#;3mBI$dT9IRJ=ZFTR_|&I%4b z_0Tp>(b{V|F)51-)>u*%LA!$SP|Z$Jzt)3u{Ye8g1GE$hxTp`*d+(a-ng=f3ARA3XQ44=^Eevu8e$e@%UVGs(bfMP4h$mEv%l zZQo#L3-Jp=oH$_GxL@S(;pYlXeYS}0#Uav{2G8x+eYG7E(Js|>b#)=*)MMMXuaf%n1_?(n^vmr2$KsOl;b3Kn!%oYW znPzDb*)9;Oh3+qfkntDM*<=WO^-@<|ZS8(nMn2#JTejTR&~;d8)AR8Nnrd5i$?r0e zOH52mcbYUO(jT0>yh6{1!K=6&>Oc+7^PO9_WRl7^L8N3!opB!@UwNp6)6N1RNtFJ6 zLDsAjRm3GR-NC^@ew!Zok0FP*9enx`N@s8{pSw;)C`qTBeeOL*sL%qk^_X0)0s%px zr)Mx2_{(KFHo>em5%YKVU#INQ^76WO@^3#M0B%M(nfjTQQXx!CSs#&}t#B?x(Gvr< zGyBCcRTi2tW(W1KJ4sJ&NNUgPNTdZB7hNb6l?UPKhS5Ezp{S+)8dDH0uhlUdiaGnU zV6=zUzWvWDos`xF9{p-0dC@`e^j;XNA}= zf|i7Ugdr@(5A6mfsOM&y_FDeF;A_=3+P#5(wOIS_`MK$q5xTEXz33=DF!cQ$6DOuJ z6uB+fTz~%dPrO6$V*}*}8}*zwSjbCF>YPCrz!%)vLscmK~Wt!PZNa~#B6 zJ?p&KzK^owUYO~$>rBEsZ~DC_j+Ml&Q}P14rnisxn6U8SHm=mNcjD~#6TcV>WYN+& zZJ@Rfh=?oL94W>U22g#~Ha=ZhMa7|2+*n`;*{nd_m**%c5F;5$hujPCtF1W}jOEaV zCKHfI*dPYBF}ge>;X(m;zXSB6jN#V-a%f3IK^aKGM)3B4(Ky;xl*S#+kf|6$qU2+@Y};mtu8{4R`hcSMrP^L07?pX&E&sEH zKZ43zEj~tbZLs+UeB+LC)XE3H7Hk9Fv|Z@xlYho-&vYY+0xz+%zyCgzgQ>^vy}JOl zQaa>Rx4pbhx2`-PJ49M!(0hO!Q%cME4@Mr6xebK%zwutQf*~W6bnr@lKtyEX`Q7H` zb2K}}a}^9<1cHdD5A0njLG4oA-2D6j7}MN>f>zdpOOK4mQ19UZQ5p@nIHvJjFJ9!G zL$f7h%zaVvHTb2kmz&?h7`JWf+0L4X&94?0*xT|{3*6|K`#n)HGWYUlX@TzMi|bYv z+MY-1FJVs;EmAzH=;-L}dx+BR8|ta+Y(N8ieX% zCFY72?N=T%oc3vVxdd|I~>3fHtCx<{$*UDY){ zI%>is73xbT^~%F*n`x^nk!!~6*DRf1WYcnarc0g z<{fmsh%e5uJ$l@jnRMy$UTDY{Uz`2+M8wZ;5DvNh@85t}s;i`w{#n({snWNkdA=L8 z4Tz%ir@cJX7QIW`bF6veniXgv^8<+ksXft32_`LQk#djHbiCiRAGEc5@v`|q9*Ad1 z+i%b^3T-b}KYR8&sx%NdxT<~UZ%^C!2}G1{hgx%d`jCD@Otyr~FIN5>CB5r`K7+ambT2XPYrXmkCp5&ai?K_jewe@@e#C^Phy`$&s`s8BQaa0349<%4W=H zM(DYkj;KN;`j0pKA-%;Xx3k?OL7WeSyQ9!ZBaJF$0%rC@OGLLkY_b}l_7=8K$5`=*x`yGs7Oy_D963@Ym(W5n4rWzPi%20kD`eAsuo-YsSmF-=!VXA^^~NCs8kG`^qw3 zW=!sU=dl|%Zg3d;9KZGImH5=mObaJ@bH_)7Z#vYv4L>B9JQRo&V&hPoh59lPHzj2S z2n31o5)HyeldPI60{$oV2od1}09^+olSiwLhLg~Pz161;w-g{>KyJv!%P(NsHiP-RYcO+QeBGy0dE~&dgA{(MBnIGo* zS(?C{868dB-;)I`^O&GrAd>H9dr<#+-H$7s%_p+iH?O10YJYNeUzXCotgU1DGa~wB zEUTi*#VN}wL%y^=~x;cC#TIu6V9sLRBsg#vj7BI>@=L;Y(9hI5c zf?|)%T>7;kSyZR3T!Xkbn~)hc2NhWIR%K^j`Ah|Y#LN@tZ&6QQsEbWN z!?fyf8`n`St-LQ&R$VuLLGZt25B1W9!T%zAHnxiWPvt6`^c7}F-|o^3tnDzz1MU< zXBHFscZf0U>+3@~DPD6`Eyds8-~FGonkbq7u+F31wZw<>?Edx*T4TZpI=7c2tt)8? zqP0U9JTX>kj)UDl%8Tw1k!!23OH zX7OwhYa(Q%>0|5}?{7YqPG!v*xzl376z#`Bi(YB8v|H~xO2_Ow$632!9f$22i8Uc> zjE!n9^|*D429H_YboUrMIqIAo_|CCtlEZ=LgyhrdM}^(ic{Cg=`-7fUmCra!9y^|K zMTkP7EbOezAiBTr-oCw37psu{0tdPB=a}*6kN2;7R!}I?&1icSyZ$R21cI0(jkI=K zU35jxxfXq>Tr5!TM4+CDeqDhym#IMoy(Q$^{;3eT(he}?h(*C!l3 z|5}G5EKLhqMWfdv8}e*2j(cB;f}mSsX6BTC^@eSoxqH9-nw)nla?p~q|1lunc6h0M`coZ3G*uu4|)>zCsWc%n*x~ze^qoPuw8trfP zjrMFwdOM)1?vNF{VSQfL`O#=0<=I*7#L3d3hF3C?eHF!K_q>v)MVuGsdF=Y@_owK& zohq;otNR?a@UB*OhOH%a53Ss6xmwmOef83xWZ<@Oa16dq@hS*;% zj0nlCT;tcNat z)Nn&m5QqE|ef|x&!wPE~n=JIHLq6)ZH{v5>%*HnPnxA#UNP2z~4fVw0LdRm=A}`mQ z!gKiSc%IqGktQIU<3HIyB0v{FhkV{wbD8(}nY$7|ln>C4z}{0ockX(f!UGwP;jQdI!I}WHW^9faBnc@_MNKXl^xq zg4!$Z;nleD8<{c{bHytY?S(c+<@{;R|lFg=^Er zc_{Uwp6v5(j!wCMZlnu1PyK9pEm3qmz9d;O3cU@J#K+!N$jkjBQrK+S4kTN1P3Pr%WB!G zT~$?e7Dj<4_NdGy&Bs`WScC}!L3*yAeVGsc7@NLo1Cmqb#mcSoD6lgKnmj`5_xs3g zc$W!jj7?u!{)MqvLHt8j0!m2F4%??XvoB@n8jFQw@wrma;J&@N)op2+w&Rz^ z{z?^F+sq-Ip|I-KWq6&N2FT{!*9x-Wmen;DCp8};w&2mskj1>AUML*WZUFFn(EryT5vKJD&je0F&p7iu%gn;; zNZ{kgv5$GQ8RYHlF~>a_$z>Am%xJ%$pgLl2C2Q-98%DUZ;!i7=QGu4w?6}jfT#11~ zq315`qw&#ZYzECb92|1F zAHSvE^!JYxfweK}R6K`uquXk^>y#;NvgXEtkD2*Yxab&eGsl&xH0=or9e5NURIn$} zqVCEu(d(=c2fVD$^|uOW@!6lOw+Y$UAo!mBGLOp{l<$m@=V!M=3f}Jf+k0BmbL2B> zo){S!HTQ#%9gpxwe!BpJVw@_S+q@kulgO_!4%)?}+}v{@gM9X6agB$!WIVA3 zcX^o1S4>L7>H_DNF(x~To|!r2-ln}Kz~tGU&#b(4#O!OndYJR*Jv>|*_7(kpP^Q&B zfA;JpPP_qyA#!}&ws7jZ7u-qq1fSmPs?C$d93Mw-AXQBzhn;B4s7P*#pWvBJ3fLm)RFUyhIF z;~400nZVMVQdAUrM30-!%_}K6x^mf{i3#zYeh~sN3_8)Vv3J@=Tn3vD$QYZSxw=Pt z=Q8&-R!nX%4@*~oh&C)Gxx(M<D@BOMm}8yFr!7tLqKq2nlGC*r_rb_HWs_dYMr_v~}~q9)2RL1-#Xf=BlM4Fr(;&>>Y))%IT>FV$wAlk#WU z`Az`&LJ_Ak#dYe4#p~%<*QriS zEDdZBNy8&{{PX3qATXAR-p?}Y%RdGE9D;5GM2__brky2uw6nc!Ql*z~SiK?>Kzre2 zyOS9DVk@p28AX@*GAF_1aIv$$lP)PrLsPlquQ#jA3gHRNn$wTG(XUZT#I9)*<OY+W>qMF`~V z`upg@@-W0@QuAQn+t28seP@OrI|_#P7`8;tASz}|W$Vik$ev2O7Mfv*yU(p zCtIJ5zJ1}nbi8cKbD68P^ddP=BNCeaGbz&xep_g9d@pi9;ig?VoO92 z_neqmBS_5T1c(y7*wg`~%2Gk2O911%9)MnKpT`lhIehTgND8$wuRMS=*?O=&N{;SR z$YBk$)@;G0Ff`XiH?8UH;&eXj?v+;y&hwp!=MCnaWgk||ID~Gd^cRTy&q4#dU;N7De;1_U#UGt=VQ>On8;_lamSxSq;gEX|=F}2&qhhc?wJeaRkH&z)`lIQ&W3{U@;utf?G_iMd>JR@!KDV zLS5~l-`8o_$%*WFJv;ocuJ4uds%2CmB332(kwuROt=kHAO@{xpZYWj^+C6k-G&1*w5M&up1x$DNAdbe(}j9U4-1#&NXU$RxKZ!H23^h%yrge>_8C8?O$*hHA! zY+bT91+EMtkijm8a(i4ki<=I*Q`v0rXfvU>S~ym9xFyE|Y7O7~!Db6Li9$6>(Kt3xGZmck_XKVuLjgH@L;c>ceoC0(zMs3Xcm}l&V0&AwM@mEBd}}fC1vvcW0krv=P7Z&ypU9)C zPr(W8gK@Z$H=$q4-Z>QhJE}A70d?-q`pWTJRF6L1ydP*=1pjVghy*7IPTw z`5!)htVe-Brc6auRnUG|2O6xxa@^7XkI~bSrt%ojIIvw4C&wQ!Ex30KZ+jRM~IJ4HQy5T{Zx_SR*nAa$}v@s)!wt@D{kw5QWwx7cu%d|gY)Y)mYAvd zUrsgA_?G=O5w|ey#x5!c6|RY)R$(J5xusA0gw;EK{0@S6=O}w{oFqmoc3bD^P(O?~ zU?alR-;@>x$C&^@!Covb;HW?|MF`6_rs<`Hxz3zKSxp(sr74bMTO1A+zxMg_=To3r zk_ZoXXCoC3FE+OpVk;FQa6DVL*5ls)4~*6YRmNbC z8N;lKIZvgV4Wf(rGmfF))nr!Of9}Y*=!JMC|JsmNk{+G(5H~OHQVfjhccTatM{N*b;hP!T1t*)M z$yWhjLlfA-)=~D@SuQ9E0;_rT#%K}FA_Dg6EYdZ)9WVEvmbpXo87}3Ar3g(lNh%g3 ziC>jKWV8p75mQWD+yKIH;eP10?4w`p(e03Kq%oDEy_VubRjx-?0GIoo+Z|h$4j2BC zyG~QXWrjDc-j;@vHQ4$}<>L8?r@l7|=vlhe!eY%OZ1ip}e5s3CGxR(@r#xoQwxFbb zm5j=S_MJf-nHG_5f5vQzw(&oJx9 zVon5kjV`x)*KKW#@ZWJLtEoQjOc;YOOM!LweMJ1RI2LM{)y?lu_GZA3YXuFN#;V$U ze0iaV`CHe^l1XJogF8rJ+|evnkBhN4!>K0bt(i?I|4>7QLo6A z%@75*CgBd}C}~!G)*X1$ev(QMGtJ`s>y5(5t!rW-`HKs)zzEX%65>Kt5U~!QeEjIq zW`$sp1JOmur$eON7DAmBDE7!=d%+rLyt#56(TVf_4MG6W9u)*@slR6oWe%c*j5MgW zu@b2>rY_L3&!o-GIU_1cFWZlhh`19_8a6zBevmAjTTEi&DM-C%hn9C8cm)6G5rr?; zY7$nxExX#HOiuI%&3T3u6iYhM6siBZQrRa21X{YXE_Q4Q6YACzW?o(_G3wH8xAJD~ z{HL~@Q1A4u-I?{GpJS@-@t#yU8KajMqb@9&s-9k9Yj)CFw4Q@^v`p?HXI$S2`($X* zeyFHWx_0f4PdjmoNX^<_zGEXhK0E>t!acOcMq~IaV=HL~)cf2P2JH!dHz@G|%ltJv+uHC!s7Uw5Tk%^fA5(zOH zXvs=C?zb}q)Pas(bqx*E>Cx_If~H$1^OhO>sEs*Iz&JZ}H!+kSw(SikF6RCF_jbV_ zQQTjx5Gt;TopXbG-#&|RVdUG0`I2Ba>_WK^F#*m)+WUOy*@zp+`l@~sf1X4}MGc5E z;>MI1(UN@B(WhGi89=TC_Jhgi$lJUiC`H*Lj{)A-sGs;ubpPp-ym6_IXOXM8AK(4H z`%!6zd^ujeba_yPek8Fpt{T?Cg70;)a|5x4l-%-#F&kI6Lz6t0A|;JCcB@Sq#pUl^ ze8H+_6{F`6qyB*>rS^7)$@z)tE&RbRBrQ*v^j}Llq~{5T4eq2Z9y5}DOgMHUWQbihi1;u9GUSS=c-O-B z*~0kwi=qr&1OC{=w_9{3zrMZMe-JxX3Cy(GM1{ z3_ZrY$Q`Fha%H)hhC1-!!zcvB{ireuhy1k88!ui?*ZXD_3ytVDt!)tC;Y@2vN={B@ zEJPm7qxV_|ar!qqUj1`$H=y#QvbfDD-;nM#S&3;Yj%c-NczOAZrq5LwIluBKtFAI* zcwS*gRLZsTtYdUtr!O8me!NlhF|t%d1jm8<{zbFqewSn0r-@aP)84j@#LFmZVg$m6 zC9onDQMg@SZ49Fw@*m__ru?5ju@?mAYYpQ-Qy6@VG)m9u!RjqtmXo{W|5C>lQHo)Y;7oJ5$bt%4EBxnx~YHzDWQ3GmWSIMM{y2xW*0`R=FI}XaO z_bgR96+WB?Q^4aQ8HNI@6nPTys_>W^*c||(%vv`uAl#by@$ig*S&P8CvN8twrLBMg z%CftP1F}zcff{z%7G~!2o@<8oSJr~Jkcy=Xk#w0kzKG+mm1Pnb>hedk z>fN$C!Qu`WwVS2^lBZ7>07wROS0aDy7dp4S@5{?L#W2Z~rEP?4tq&5yO`vf1?b~BHT}@yfft)2i+>hHXF2w0E%OWNp&dTE@G5L+H%I@B zm#$x7_f8{aU$^TPwJG&qBT{MVQ{8k7G}@KHk01T8=Kj*HWQo~AsK4>!vJEHbeFSYQSPV;QnOb)pT_oC81+eCn|2+T)(+Tq>=t$&=@`daQB zjwBRn2$-1(Jt-0(Vd=9)@isHV1ta4q_n6klEBCi#?fXfRhT#Yp3KFBC9x6<-<^J}< zf;VdT2?{^VDUf%KNBG#1Zg9(NcY!{CJTlHctnD*?gPRx^_-Wp}_?`Buk%* zZ!B*>q5x)DgF(r28?g{aa1|$hm$S5Hut>S;kng|SPy7NubIp`ak?VOfXJP$L@pE0* zfj*W&`~c<5Caa8JPnW zZcT#V3T$_i?CAVA>DtfEehlO>ZXTY(&z0}rf5tI1;#dC^5cxM~4;cKO`)|-bCF9YD z*Dqba?3sBvn{pgtaza;!ADI!#QQPpiUP83RcJp@n&h&b2A(fO1yUpgM`QyKTa9_(k z=jQxqR$Xh$)QFs&^o{U{8bQl*DmorARu2~EPxxNr{xVT6=ln=!y->qr@E~|;StNBj zoX0--v?K2&Zee2ZOP&5f^2za1qzr&1Z;!`$hVNg4n3GX{H_SeG0h_yL+2B$8K}kq7|E1_{F|Dkv<>?Tl~~7HjG-3I7q;6H|)a zk^KV09#BK<$I|=^-6XKqJm+xq&#~#vn>HD;f`&+8C|J)db_KUI(^Rw&1I;2v7Z`V> zb$!?n8bDQ+U~K90$PiVKKcXZZ6&V@%yy~>PJjN~FY5gMa(0180#X!PySr$&xolEgA*h;S}shG%LmD=g%BZOHTGyFvmOU%bz-R|58 zSt$W1l91pKkH$WVAmAAOVgmV=tUokvtn~4}TZaBOel+bKATe0y54WbufVYL(cm%Ox z_QP;I$&7a9=Bdt-VE~BVs6{+0*!{zQb8fdhrTDA$Ih8}gro1HdAlE|i4#U5HqD)lX z9#GeE$$#QWPvNA0bT*8)F;j7k$(BvVV(a^SfI>u_?moEnjVSF-l!E1ja?UicJShTQj)}g zjMm99HzrYQS*&vb+I!OAAh|g>%A{5B7I(M=1sl%lhJ2*@${6x~GLKYY2^P=WiW{y{f z;dB~++9v?OYwtx-h1f47=NZ_~=DbWb6p}TNU_Z+=XwKEG;yUVTB%^@sZ%i`0w+4}d zm`#s2iUMv?_CO8D9`_R;tvK(t@V`MK3cF+%fTd&Cq(@?!)pGEfY?Gz{ftRW~xdT8^ zK+RA)f1an$Z-8d6Vz3gBE84v;y)4fmi0{wf8*kfB3BM-jb&b2^P1X_wiuWmIVT%2C z<%>f&Q2?n_eV4fQiQ+O=9krFw1g=x6x^+_8uqMyuGRZs|Tk~w2QFag3TZK0W&0>F-Q$`ppyu3MLN{r3g4j@RdJP=wY&ZjQZxK02E4A_stWA#NL z%>ht4@6hbHFaV07$(M|aaH@ij)?T>|?qAf$kMiVmUEO-cHc2>+*Wmt0Uf5KL1tZG< zuqI^^W~^_r!>uT(+6hh%euo=87Nh+DGm4HT@Bk<;9x}jwZANdt zn&*1vtAz+%cAOe((khc}18Vv$)#}y1nCTp!=YM>+Wx=VWWi#K69ym7~K{dnUX-0qe zYZ1-pT5h4cR19LEp+KNdJ1sKuD8@(dKSC%A;35dMB|_h8MEtG7U*VupFZK9h0yZ&v zH8sY6mD26QFbN|?m@ZNv2WG5+nCF`lGt31qJo*w;URhl&OrVBufAB8~Pxy}jD0@`d zo#8_Yhj_53AMaO29`@9|Sb{iMU@Bn|d+_1zA+1A=q}l*T@5>DEH0=HcG8C7SU@9gu zm~cU&oh6{V&?yILf@v^#DXR*{Yflq=9EWdDWk$b<@esp-3mzasn-36Ug4hrOseut{ z(xuGWSp1*hTZ!+{?DCmWqX+l7m&Zm?R;wNhbYt54 zVV-HP`<+y5yZ+Bno+*=+d`0o$;l)ZzJ~>_w!H8iJDXyaUxBb-ewn6|05@@w>M8#;L zMuzV8{`evD^)Rm z3_<7>)*Y53xnIryTh^uLM*IOO6J-?iWy*xi+v1VE@7qqpZ~~9Oy(lR>Tw2d%B9NTa zc{KO~pG>vVh1mxAVt?VUl6550+h3X36+L<9q9(x|fiRoR11C3zGk*1Y<-#2V;-Ikh zqVYf-GfgXU@oOTM;RJ$5F}TsCm>LP&R>vO?DB^`mk~OQ3QG+z_$)|uLz|xQ)arWPzd+>QLnHkuSTPRt{WQ;=j+{nUk*3Vx z!#1y8L9++Oxi|z(;2B6>Q^--(grv~xJAikIWkR$xG5E5YK-$y+?M~>bBEA%rsH^h; zKTHQ2Q|_oxO{g9@a${j4&CMA2GY;Ik9cNCS{0u6MAZ!Q1njI&a)7#t@=Sg~%g7_LE z)2BKgy9ooP0^N?pJ21vFWlj_QF<^1RP^M&mCr`?tdSIf(#IM|t5C-D;2-F8ORC@%^ zimZ%{{fg%-%Zep>{Odn58;Fr3MlM!wOCP{I}YDzb4x)Eu~bM`Dw=n=^#m9WjLD8uc&NW7QAr~<0DNpUB{nE=WzhYkRdGO#t z!hs-_HiW7C{+Z@LGK^7D1Xh{7fIL3SZP8UG!F{F_#Uca;j&Lq^;12r#CYB3T4oA!w zX~?*Edifegj>?R2@YXrGxZXE*yu9q^i=4BrNCS)&_S28Ape)H&vLfFl{o|W$PCbtVspw1MkIXoLjnXg%8WIS+r{)4$-v*so&P&OT)^LJ;lt~F z-C{~gST}da9`~H1v#F1t`%-<|$#~G{E|5yILNJMLe=U7Tex2SI-}tH>XkD(0^Y(`f z+?}GaAxT|-oNP{n18iOJ1V!_Io)g1Dd@BvF%6gEX5*y<@iXIlW&n?kusDpHDy@Sgv+PL~~C~#mp`J|-u z2zHARY?1x_{r)R2;0gcz` zG_|yz5rzgDvD#n0!V{9tT}yonxJ|aM;bCE%orO@aYqVW3n(aK4W0jsgH{G(yKx`RB z;0n`aJcFgnt!xuO@ujPz@#*(dW}^0AQFxiCe>^uJrdo2RL|UfjT7e*1Xh_-P46#iA z_gq8BFbT15{|q$0>=J?7bM!SI z6N6s(OehGq5zmL6MVNd4)Qp9;(okoic1oCE*tUlr(dQmUWSyO#ocxjaAp{F>P;$h* z$a=Wg_#huAzlx;kP5|DHbvsQz+$k|AnA;OcvS1NDZthbs{Pj|c(`O|9G#jE85Z@Ku z8HMP`as0EKS)nv*IL7ZB1^tmMOaiD87}8Y(s682k3{P)i<62pK-e=@jn^{;aSPi-! z1O!Abm4*=@1E6d0nH>U9rcVHK5<#Y*-R ziSv=BTU-ePMF#LAo#vyf7^yHiJN<=tCU@=k6&2|XNa0lwL_OGFT-akNQjZF6;hz>$ zhJ!{YDmX3$sA*{>5_ABuLH+Np@+~yhf!I3VE|C?&sEpSUB;Lx3!@`r}0`s!zAi%CT zz%c#5tf&L1?92b(S(zoNAj(p@r2JoG+ZV8vM=KhT%Mz25Ar6ik6~}&%uypW5xZ+u zwVX~o99*L$0G@WislOe+-B@*Wp%B$LvV#zoa~(SLvTPvkA6f*O&p149bys7X)LbV_ z1Lhq&-sfRYS!!eR5~fPf!-s~PpFU;X^AcdDg7p&ceMuGpm%CKwdNdX;T?TO_EHxig z-*HdA{Jqz`Ky{eTq#@yg`VH&p(^&2x2l zNvSTa@Xs??HIrZ)a3n|-kfe&dy06aVP~m0$_mHV*yZrus4Cw;{@h3`7z#J!PPxFy_ zWybzWA%Zg?YPy070O)GMI4d^)|EyGHnzf!qIbrpe_!sHvN`#w*rCTBmToDIf4#drH zBp7;nkb_guy9KD4cp%3vA7M~cE%Uw(izzZJ5XKdWv>kQRH1LZt;Fw3KxjVqC&mepw ze9b`g9C_zl`FxZx?4tZ68Aj)p5A(pQ00QS`V|&xs@xst^E1zz1NlA%+Q5`@o1d_r( z77pUsgdZR_tEKC=M_Uk53(G?!_(Ifq2Kl@2qu5KR2eBW2v2qt`7AJWVj;QWJgz zZr(09bHP%C2{BOsDqjWzVWKw2VYt5SG#5Z4r!;c01{8-ejbYQlohU+s#XZ2XI3-tM z^CSK+;6>tFN)7bN=Bms0gKmIc08(=PZcldXFAGv1azf&N5{|_MFtQV>N-d5U1tAPW z)gpB7)x;i3F|+BW^50wjSpxbGRs&rB{8`EV+Ih~p_ZPOlTBTF~j)rm5?SrThwC#pj zgnr}Fp$3!d-+(LXOqgVLetNpU&-0r2Ozb1HGrt02KJ8D-vS5h%h~wl}`z+9p5!g0a z9mOO6h52VOh^JfrWw=q=7?J8f1EKvtKK-BI<*!fA;p5=oAW$vM9`XejLcwhL^yI+q zAFpgd50IQ4Jj5*FSg(W~>3$~;rEv1}Pyip6QzZC6*a!$i96O!(FX;;Er}0lK zyZ=k9xp>hS+X)`TZUk$Os_Aj4iTpKy>MSK zNSs<75%U8e>1jj!Pq;%QQy}Bgy*2)VOQbP?R8J~LCYRX1^t%7gMNInQQK+7gAlvAD zxAm7YP~b`J5^+_B?X9z{OtNxvl^OfBfqQ-=^+w9Kk$a$y!Id|pol3&>Jo zPxbLrS`6&D4QgQm$P%ig+5s#RWHs8JW1E=%}ppv5^2d(VWb;V4H42%h%(>m;IunDqXA`hu4Kb0b1)qMb@1le>f3sR z(Z)#fW{MVPrREXj;<+YS=>oNmm46zEWR2vj*_TDFe!?&cSbcx%zkn>%Zeh-0`8{_h zzP)?pMhYiG{|tZC?sCcjBx!;em*5opD-Zh9yFGsXJc$GosC@{B0MsIVi46waxT|p2 zU$gT-@1H?$@A}e>PAsuWQKpL|q@(Dm;nz5V+K zLZu{E*RnU^uO(F@GfcKFuCTNSRA01uwI;opzlIfI7-q!GXSX9Qvjs3Jv$PW7uDvLx znmu%f8OF}2(BD;@{7)?jky1X!32SsMoBYQYk{}cHl;vD_kRO){%93GcNFZ5@Pi}Mi zmwIgdqr%_jIG&^ynq^3q4Ol{E&5h??$VwW7Ta8G5wAvm`Y0tzRGG_h^F&@mu&TxU% z6EwFa-8lsT0UOQQa+Q}Tw;-y3(#P-ojW%bWP?5M4{^Rp zK?I*39tyz=tqQ??BoPNB9-nr>(2!6f2Cz*&lGI|;He-@na58E9dlrPdawq=HEpl^Q zU;U{2f)0iQ6oHMGPbXzg22^9QCMyVc%sTICA?Rlr#pI(XbGOly55mire=>{JyGeNl z>;=LRjz=6NSoI#93=*BAPJu$Msa-I5`NhfT3`Gk3TSBaSN)Z(YVz}@CcmyV3@}PB;f79A{c9NPd z%p`5ye=ZzDNj$s{hbWSwCU(3so;>XySp}?HEXhIvhp<)uEC3)1OSk*DclyNXQ>QZV z_(Ok^sD8nd)adX?NDCeTPwOu)1xyfp8mxL}ch~PAiCj1PCXr`}%l*04e{52?RVBDL zsp~Exgt}sMO_x(u*Z9-A9;)~bET`c0U;5f3AXmOg_0@JneUvxM8|M)udfSU8a?H|k7 zv#(i3#unKnyOb?!QqiI;A!Q3$GAc7<44IHMOPO$#WGO0=wQMPAQ9@*^$eKv;yx!kV znESb(=lt=z-M?SU>3lz*_j0|j>w3=%pLgGyzqD$f(;Z9*V%8@d)}5MnKgh79Y2hPJ zZNU!XmkU~-yp-}lsCXcEMzIW10};*ypE!|ZC~5i4;(}a@kjZB(U!Q%yQRZQvOTKD9 z2ZOeN@|L`i#sK2Z6W>*jRPj+Ca zcrpaLmed2Iw%0gt*y>B$@e8dFP5@fW3j5mosFa-YGOhOmch?SO^eK_aKc}Tidyju0 z^{e{GuA%9kUXFA=f3t|EfIsQ7(FXa>H;qK53Hof)rC+~$qZ_vzE!STPZsah&&_}zs zs8W%=ID?dS8&rTX&E!I%&gD*GXc-~{OCfo{U7`Cx!AAXONgkN1*SLlgXIWZi&fv`U zI&bWyiO6{VzR5^=+sQ{oAO&k=VegkW4)1}3B0O$oJj3IzQVSk!tdk(AQrkj^$bu@?LG?wLzZ$T0Rftf zSIIWSm+c1pJE%6^}GrRm05qWiYe6wU{{@BNUj*MIK0O}_|70kUt0?n&ov-zFo6 zG1O8pNW#OEv2^j02PDfPGSR=~(sxhpz0;wx8B%60!WYi={Z#Apy%28qn%y@_n5uqY zhR}HAFPS@0#Lz@fKu}}evUb$A(L7gy9cMT=Ts?8uyqhiF7$Oa#;i=JtIF&^F5VWXI zvyU@wW!} z8*RpEl8R@>QnkNPyT3caHA}p@r~T37fbzZKDroPA<#`zH=&1+Huq~{v<&_FrcVbE- zV~K-wb$(SVtd5k)2P_6<@&O<4dd_06n|N0wQLq#sA5a*3N?J_j%NxJTDCB1= zx)s~VX)>NNykE7_pM6&rg$cO7lC*|<(_t4oP(#E|{?c5G>;HB&oPylmx=td{)SYl} zwe74=qNIS_LMpfEND0VqFW@-^`TMtY=#R9g%h@-7h@67lS)uCqEg)K`!JJUJa-6C) zNw|775!VFYUAx@wB-|#JK>$il1+q$FbOk}tc|Z^TNYj1Gev)5k4F=AeTL8RbVP-b1 z{h0W^ZHMgK$Vt7p@@=!S2cj}~{v*q)C%4(^K6jvkAo{fCACucyxC=9^yJm#=n%XIY zfm~^Jsr_|Vuek`gd)naRa06JxeZiUg!f(_UZ69O)*gnq__NBpr6Eanv6qS98~xq_t8~z;UI>Dig_MJwRNOf?!Rcyu3z@ z8acxf-5tYC(Xxb81Nl)9D;d+nAU^7{m$rsGS`gcG{)lTw*1 z2@P6BQLB}Bs@&4}^vxEgWkk)#emL9QP0TKMc5W;JBXg^8S zJ@wcBvoL7Qz1fUElASxJt_`$9^{6PAh2dINs1;8xO@7vTq@jIJxG2MO7Q9*UqbijMNI%PnG>0CKY- zmlWGd=g~$86c~OhSD|(b>(lc>;8V7Tn&EH+4G;xf3f#<=F%RtvA0;|cfc_X9E|?FP zr3_1;KFwVTF+!dLG=r|b1a?9)<9`Yc!73?8u(7qz?oZf~l;CafAQ75Q{iaQA;!IrH zfdPb2`rUOS3|gO>)1+`;t2rF*m9Xk~LwkN7Xn%DXNfDCEzvKTMrrW)HlA%#@YZ9R_ zesA79@^dn~2#`DYF;TWdBj-yuVg z#sl2K^ln6rXCy7{S0x#`s0^3!Prte4-h9(D)XMtEjM|#{avI3=lef!^VnzD->fRv{ zlQF6{wP=twn(ryuPNg3F{{8y~H7Y3r&nmT$w@aSI@Z|_|{CksNsmsVn@<#pojYs|! z`z*oFC^gKgJ-;*2v^82;2Br#vW)G%MA(h9;Rsi?pnlTSU<=CUqR4xM}^X#*fDc;Td zS^MC?$vW0^(=xc=Vp#UYC zj`+(@w@a5au6{TLFfeFFEHv$58=D<&WdHkknyaP7k+|a(Bu^NZ?nV;>Qf3Z^rgYYx z+eYeudiClxs8eUE#0kgtGNdXXQ(ny9H5iFd>65waQr4y{sQv>;>Dl5(FjL=twkXZQ z>?urW!%of*&PK{8N?^mu=YCHw1yZUk>b$+9L$@!2CH^_sc%5evRVaMwp-}$rO#f#< z!xL{qTMG?@=O<7Kthxgt0s$~Hd^JLd$eh$DbSI)HRy{gUmixs<2OLDitUim5sq4)< zea{kt(XCOOCI7)hw%d}G2ssI!`kNT-|={t}}anD$klt%r3n@s^oQL zk!;rO+Q|Ga{+hfr@_^pFiaibKm`oiUo_P1=G80A0uFVY@g}@iKpc>lQ)o{SG#eGu< z($Qaw@)(8u1!!;djE?37b^%GCcamzAo<83WW;j%!xdR>;^UI4TQg_-dd-RL9wtfeP z2{Fjw$g^XG@*$ATYR4L&M@DYH3+(S7u)b7x2VXzI=QwTB*g?f_DkRh)O1k0PWwzqu z%iE%Dr>7r=TwVW`GZ9rrVbM(Hk)GLm7k1kJ6_`5tWWvB;+FP4%O!`IAvD4Ji7oMH$ zs_ewL0DeH#CgZ43z^e;r0fGVHfVXA>)Qj9w=3}&7Izky2fRqS0xugAjQN^G$Ef7-! zA<14>`pewvKUdFM>+Zd>^y@WOLnWHf1D)A`YcnL%jJm(YmQ(Npa@nyc)2%G+WJVkR z(x9)Et$bzifh|`o`kE_aUh7n3;+P5~ruK7ryqclHjP{D>!RpgAI;-`TY~Q+dMtd#O zZ_=Xt&jCqEp8dBZ2!BE7>r&?$a+a)p#DD+^h$6_TGLAAptVt*XoHOS<`JqwN3N)Uc zwd9Pk8%2O2<((dS71@G=WHUWFPD@_@AaIFNL5$ULE^P;THx z`R?62ucgk{k?_>RX)>bH`I-)57Yky?SuQ_Pjo?p{BbT(k11o4tn6 zxTnzwGk}_-P@n19;oWIXW(+E_KJb8?qJqSNX@Y%YC6GhwXo(zo)#`|yr8S&b2T=W1 zZ!-VhsSp{m!_|MdC3N489cqT8_uOFclWLyql*G}q8rTlcB^8pX{2iWNZ3hgvZ@GQj zwp;>` z$;0)253C<530r6|jH8Y09{NAZxqJNI7ArrDw@G+im`4uFax3p-0Y?hpLK*Nw5*K|lRKqdIpHLo%?S{DHAkql@npNYCY*)hI0QRwrA8!fcq#e#^x^S4dkm+b z4P;=vw4W5qdC!Ky=U%Pa`$W=C%m}HSMN%H z4~1yPtXU^bes%KxStWjcH~4xkZQ-^%Ibn)&pAnckEKC_|CGMcKdyy9!lPWn!)XOO; z`h&7}@7;S;OdF`k_Vb*&2rp81bngE!<9HvfZoq}OBZ>m%Cn=v^x{BrykRv?dtX*gbidWHQ3X<7#hz7}w zGu`Z$x9v&YiN|4kOxhgPHu8t)Fu7j;c=={tS$k=vGj_!Nv0Ih9&p&X?=L2(Imhah> zEPyWicmZUKa3nybpnl7H(-TjDLSvU<NsHi#q+}3$=5Gl&%Er5ue zLaxuzJGS`z$PD<#jEAQ0MPDv(dQV}Unr@dT1DUI4q=9tY^8ybYx|b_yciFtez=#oR zng0sCD|gM3F|8Ujy7pz5@;v19KHAW|^xvKJS7#;4ys|q3T}j%(Qh3vUl~hj{mnvFl8iegLtaKLfzOeYIE)@;Ov4;q=vqDRj z*9$AlI=PKXr7(Gb{@dcRt2P;gd|paaTD_=RYb!lXcwYRSM!Z}QVET>*O`AIB*;Azo zKZ1`^@viZ{K&dNA1BK_u=z#9Y_!)#gurJ|hNiiT}_m;XBy@^AButw*86=pYtzvNHR z6^pkrx121qFLyk*v|~H~KMqOt3>QKF!NVFS5#TCdKxSl*C*Onwg90Q>#;y?be)~+L zEu6f&im>zev^9_iHaJMu$k&$w%{i)_@?^+;!*1VZ+M~GE7^F#YR$i4{{r;9dc4awc z-o8tVH4jLoHDS4{;k{d;(pKePKP81gbEcApozTrGw}$DSkY_Zv=6BJnkUkYavq6;w z=PE`_B;WUBDSGs~>m5aF=Di46g`j701H@mTc{}jTh)9LJLg(^e5WhQf74ci7nHbww zZ6ZkmSVso>pG({;i`w7-@vH(fPerl9dg=U@g zKtp0_EReZ&2M(dan4vrqnyTA0-~9>;vUV!}w{IJOG))XssfYg%M>y_SV%vU#RcQs# zaYoAES7)zQYG!DBC&;2B^QarCUkcLfT|u17o=(Z~-!l1gFt$L%>;KkK`UkY4!gC*K zVgfoN+D~x#m5@IvmlXW0rMXN4WQs}~ufPH7eZ&T#>g0mfA9gtzM_!tJd;ic5?b~+J5u#7e-|h-QEri_pY(5`MMySA z%Im>WHJRdV-LYfR>A~Ye!4zIdpaS>|iNRb@VPAAeu1ps{=v(ngDreK|FMED?8d!pl zQ=l~f6rz3;Cb%*Y=+mqwt=LF`KVc--Xj7wrQ?d5!>szRTxw8Q9a%Gy!mC=||`0I?4 zK|bhS%|7;yyKdx|7CLN)pjtcCt@W8F3r}p-)7+J;k1m#?C*Wlmawa#}GF8d{{+7i{ z4%kk-f!&_U^afaj?x8!&|E)0#q= zJT`t9jTr;+VCMKXG=bw?^`CdADeS{Zh39bMJ55#nkoy#br{H1wGo9&xP7~qW!x#s$ zrKeem&U9{g!gFP;qpH${f#Gc-RGgY?BZP`PFnq;7NuJF>u*xFX5DLsqJ9!7|7U$iK z03bOcxq_a-s7@h>^&R_nkx;#HnVL4jgHzYn*1NEq8AKyy`&K}n-ue+_M9w`?#Z#+vCvj=>WWB?HDd z*FIZko+|ZliH+XGuRHGn<0UWh`Q8ZvqnFTE;KDYNWANCzKjE_34j4ro4hj;B1E~b*AUiwsZ`*bl zZiI>N69$C>#8jy2BDvq@)STM{TIb^Q{5yyK5wugv=$v`u_%rp1;WnYb5CWPBrNx|g ztr){>d-(cvnYo?64(oF*6-*jrT}CvK{frJDuOr>eq>-3S_ArPWAs;WV^Lt!=LIBP! zN()|mR-HsQqqgWXbZcLa=4{V`CKbaI8LndZk5- zzs9si4kdj@^2*-HklqyVOlePHkTq#?nEe*RKt*(>%O{19diew1P^Q3=(tiF@y`fMm zQByDUeh&y#fM>3PG|RzW7)hb5y=(BFmDZf~>O_))q-t(;E#p`3`^uHwQ*F(#&O49Lut?4B0Qb zbSy(k{_yiyrsqsi3l%Y!igMqxLI8k#OHsBa>=hlN*9Od|oUxY>-5k+x)nav*uV>T8 zSXu21XwBCeeu97@Jt(q&f1l%U)SRfz&K45Ad&z`YMv_dx@@QWy2M-?X{a#ZZy&xf` zVMWWcw2CviZOVWcM~uTHuv1?d#52VUI+nSiexV(vO~PoAx}k%fUX3P&%la#xfbPYg zeDW$+sYMSD0Zu%vDAA0SoG)=#$@wKKg?NcPYG$5Ty!1sZqD-(1U%f45u3kN%CsRU0 z<1_?}%D|BdzRAc%i}uvpU%%c3=T@pu_Q&gNORBg3r#{B`r}nbhp}&KDCtA-1Ez1;v zO5e$5AZMP|OUTe#75R-)r9e9PU`CN!mMX zsh2hAYw^knG^s%y^4H6e`ih>-yTt(ZKyK7((r-bQcx_g<8^t@qSwg&K-10(Bf1v3- z%xc8gnB+t!-ECo1zN@hxz8N&FBZGO0fzO-o0? zn7?D0?#pma?1_~aL-zw2Xm^fMmHDRKp`kVdei8KkfhbkePYg{9O*QDL%FZFD4#R`|HbQ@oEt>FoW7}+O6%T? zefGmcEyOl%n+#qg96jIzo}{M`0l%)L-^b}KWGM<71k-+1E1r)n9nplIefKv|)NO48 zZ7GYP5C~@0o*rP8;dxPoCpHcgWD|0wk~*}Hg6@b=QCk+_Jo{4u6z4JwB7DBbD(@Rl ztL4D-YcfJg&*Wdh*^&gmrtRDTUVtc8BK4eU>Wlh*b}& zfPMs9Q5X>nQ$N|A!>=#_?W+as(_meda}#dJ};{)E#MdE5?B7*Gz9YW;3<1h;mnA_bP41 zDqrb26wW1;@%D$W6)myK9*b(4}Y1nsl)6@6P`M7+q2}(BO z#U!6GD9R;O*nuc9Q2Gw(#l5 zPMGD>{35>4S)}Q;Xcx!q>^@XiSGUsU@*Z=)^8t`w!eX{1jCz?pn#cGYqIyc%P*PD<3bLVJ)Zcd^)0U3 zT==7fO1shgNPAiuDFS97B@nbcMBGDZ#cpW=+8_`LHVXKLoc2J6J6n}`RNfmEbDA(! znIfk3ZbybuxU9@*siBfZ*pir7#^WzdHFVukj)MK;YIF5296mXZC6vO{AeEAInqf(J zWy5#4Irk8%Ou@4Fj8D1aWgz1&TJol1sGe`BVFHR20PTD1do@)DfZT0{-a+&%4?ei} zZ`o|pwgb$#NNC76S6;-@ZUP)T_g@lH1+6NUWvnq;H*rQ zeY#;x$53Q8G9}>&L`5;n9>foX6MW;w9llos%j^~^v&L>H@vbOOC=(5G!35+ujb6NH zuHO*6;Z}1ArVI(*Q6=0vTU#+I;*n(K{}4u+dT?k;mR*}g{oMF?;pTtVq*`Nhb`AlF0=btwHw|=%X-?GMAYf(?5Ac@Y$vPlIzM%@Odj@3?3pq($-ar zpYhxo=^3GwJoS}vr!XLT1n}NH`~7SY*RvPHLJnheNGWCW>#iAAZt6An=%Q+@nJ8x6Q0mE$D2d9>*0g_Junke!4Mtr>1y|sGWP1X6;B816B zwyw)wW8^|hufwaQc2N*sgo$X;ylzIimqdx}Vl;-SP@{JcuU7>U*#;^l0fYzrc|jEY zo-2^^Lpd;o2W%#DzgS$8fWW<9n&#ysCKd8 zODHylCB@YcmXsKSFv$#ZvEzzY($Wqzc(BY^x73;*-SpJ(@0Ive;4ab7lJ{sP^>w*?c?o7jr(hJ+*4&-B`%bR! z+BuGX3Q@x|1`g>+5$|&HkAha%Z=12^W~5s|(7&UI2!`Rhn=XqlF1Vo2fmbk)4nnf6{|Dcpb`f zsr~Uk-HlWMmfN>)fB2#Aj@j_7467GH12C-cZpxyi-g!mx;mT{lp&#=A^}bv7p-H6| z<-Gdj0gI>wd#39oCbE;vkvMy5Z(1IH;9oQ<$~O$=#p@+cJ`LDp_)kYP1yUAW3-7hj zi8sj}yP1Ffy7I#&!9vjDM#)LYZFA!U%wVUyHFmj@_7TL!G$jniYNlVDDDnW z8kYlGkm$FuP4U&5|QUdr8lbl&Lui*n{AVgNc8-?YQ$4u_K;c?Gl z;)2WcT@%{og{(L+YTfubd#d=mjM%VHZFAx$vQvc%79#1Fm&c6b5Tb!}~ zU?KY}!)_$JsD>|U>VYowftzPo8RP~9bs+oRrEps+dn|VKbq$it0IMM=whyR!5_F$n zkwdw#Z6e|TMfrF9S=IV591xN9K`yG@T{h%NGbE}>34V*!9X%eHMwS^R_+z*?O713o zON;JHFdil9Y?uY#s%v0Wl(oo_9;_@B7J?=HPKb*gGvgMlLn?W{?^`l;7e~l~j1zSm zo}2`OG9`XQm!36H--=z5vN^@tTD$v${X?q@Cs;zDZRcLyYAhwVrVLX=q_3U1kfB$v$sW43!_g0b2maeY2Fq?J80B^i1>KIL zXw>k~HO0fiFhrb(WVMEK`|0G@6}X*&lhlubfGR=A{?sWN&I z=pEKCqJoTA+C=eOU5UpE1652c3av+PSgEtL*KQemIE{;OFOgCG+OrQu3LqI7f}~Vw zhS?LWtd?gFJk1nYeeBq=*ySz$87`u9=g{bbT%?YyKP2e`?AnZk(R;(I21dPdRHh|7xD^f}~E@4m_J@*Q1={ zUC5fbyXyax{>%NOJo}pN5pFwJ0dy+1VL2J|=ko&+k^U-5zk94#fPhgJ$TlOO6I!1L zzHl^4?f~l$^C_W40=b0QcAj{3H}HYzAq6r7xyHAmfa7pi7-&xXl?-ncqP*uf?m(>p zAAE#D#Jt371I5B?fq6;sH3Bb11ANH{Ejv@{O4Ieia76{9+u{1T!&jx+cD&t+vBR#a zJel9dqeg<`_afx2ejvzet^PSL`@Mun^9|4&}aG98TR+zp1nR z%QlK?LsiIf>D5vKiFO4(rP^ispJFS>qCt$Sk3$a73Z%w(bp7eV!$6Lw?37I?{Mt_^ zN!L<<3e~j~4WIyKc5kAf0ht{EHFToiEzH-d%vU>2AT~1B?y~0WSN5qW?kohuCUR#k zNmDQQrl?~D52J5K9Sbgkd?`g*U|m^6JLc{FMF|Kza@yrD@|E0bhA~93(^RdDUe!h zU_4Lw9$XCL0RrZ@V$d7VnyR!LZdww}H0y;98UG%L-3xfL=ZgJfYy&e46h=+y<48RJ zrl&@|?AdSO5w#gq00pGB4XxSXkTOy$G^CUj!@zrPvN=y7tq9)e`Q{PKW=2gmuX$H= z-%NJ2tk7&_#x!PiW-wJK_Ui)NfGNZ+evsmhElWCXYN`O3Hw?9@0`efXeLyC#ruZH!$+i}z@TRMD z^l;ysB$66FIqfszmbAyuucP=r#fOkDxG;41=LdA_-^R2dE`e&5p-My`ER;pF1 z(M7L1#0~ck5Svip0*iB1O2ms_<$3n(MUQ~9EF;Hdj7AWPqjU- zUcKrQKl9S^+d>oe9|#pyvG)Raw~4{d2)5s0agUIv@s%gAOX$wRJ`l%DV$uJ#2k87W z`}6xmUBXPcN)DnwYamtwAN5oz(?#sbNR;|u%?1OUXz`axWSh1QZgVYl@+Gp$(3KlC zz~KnNv?y-SSDEF$`|`Z=WejG>t*1OLG4c_1FU5-DOHKy)_4}C5vLsTpv#QJ$DwDblhwuHG=|AhuFhrSPS-a0Ys%lmpz2?I-hAEyNZ@-j z6yW`ovhFIL%_WAHVEE1qrq^`NoKlr{@eFT?Az~57dDx3x~?8y?(FH{tM)I?R6m|0phPP z_sL)l->qouIT`1Wo=(Z_Z9<97cvMZ4fr7c#C>m5O3WB6qM(U}P&43E@Cru(CdS)7# zLyQ9$5C#fwhyGC~8!FQs%3{R`E=PMWyJTnIJvQ}}9oVe6-c0H0s7z|!urR?)^fzV8 z4HNIug9phY(gd{mlSXckfsOyRNt?Y6Ud~)t2GpoL)?QkyD*l#!=Jltb>jPeYd3b;G z%RS{yS_ZUUY*a^Qac%Rz7tj8q)n9HyvOLvsk+b&p=#h}wW6>?&S^0-gYYjHkYdQ4q zR!KE11_#yNv@5ZyPtCyXllKnl-!3Y%>HsUqek;9(G6_OMe;ZzV2;loppZe;Z!YHIcHSWY%iBnK$C-P9WH_ z^~z)&@7Qyn0~+UyC1;?}Pi-sssn5d#-C4JGZ(?(RZv39vwH+p=RT?~#Y*u-{&83$5 zRUV&JVjth@zi?8##*0zqKePSEWv>12d7NmfI-js{Mx)@n+4}t|d5+A&KncTt<4(ML z5U6Jrm0afZ1cg%f(O&rlZmu6*jIUR(oO)z0zcD29RKXkEONzFh>1_n_k(_pjLw{_U zzDnh~foC{p&XIs6&}+9O@eZ=a>N_Qm5FiZXr??0cz%Ykvb!A~5a`U!-EhK!NfjBEIrtVu9cXv)N;RK48mo&{p z7;~4A{?ATM(i7M&g~rHu03%?o^ro~MCGVF8hlQDg+1(MeBtN`6SCq~-Mfk)d+e$}+J z^mV7+uU)t9exA>DQ<^69c>2%J@TRBiVqcN*mo$%we8}eEiHW9Ui_4}c=DTx!s-LjY z_cu9~j=11l4ad4iTJ{@tNiKlAI@B5-pm|7{o3hi^$0NbkzxZ~}$L)r5XM|hpJ6x@; z+4QFK^A4~6=E?Qy&z;vPdC!@ZBUj*(x0;6Cgd^o zpMQ`nD#?I_0PVJ$64ml8Ntwdn;eLH7pghc_VZ=yzEgOMHk661xc`(rTG!N#VE7NWZ zVH?`=rvfqAWe6&|VI5eM&)$FjT~WP%%Bmn|f!4YGQrjt0`~^DF06{y9jE#ePwKrWa z;;g;>(_i;~2f8Ke9=4dJG-F*;k=V^~b$PC@M;5qXIB#g(D&oFZ|u1v?ZRp^a?dz5zduAqF9gSDUq|X*t@s; z#|1_A(%Ml~~+PfmZkNC{?@{3P=&I)B%Khl2z zp|x#u`Thb4Z)l%1;Z5Y5+Nf$Cmj4I{U>&viFs=7!>7!COvVx?Gt^=>27cfAcZd?>; zE6cVYKuTo2+|}2;8+lg#O%_bF9kV&bY4?d`%nVs8aKz+rkOAae89?D?#*n)p@3eJ@ zjK<^y5yN)+)GoaEYKd^TQeltz`uXLhJ(NYBTu~BGcb&=idrdj75`K&o>Z)M{%towfwolygH9<7r04M=SGal~>l_lMmcA zK#^xu{^f#cpKsPIr_x_NF6mmHUSRd)ndnu!g;B8?gXBOAz7EDwp2BonS&2Y@|Vi-M&x6D1=Q{ky|#cQ0o**H!Bw*lcBO#Z zCKGp&e`IFA!bGU()qIa&c+xi>E!e^1X@!-C1YXF=d167Q2Ey+Ohg5`tz@^=W z@|C7zii_qS(9qs@cU1*7AR{7?$`P(8Sf^dSLovZZAbo-8p?r~1>uu)#T*p3hx}!a! zZRNv`utzx|e(>;5bJBq34>7xTVpvvM7VJ^(O<7r>3Uf{yE0?re)QXlNgnk+`hhcJA+X|Zxy z{}wX96TLXVQ*lv+3Kb|@+6$MT`x#W^8PSe=F95N?s-)DIuY_gF3aiq>;LRv%Gr{@; zp1oZbhBHZnMm-)%gbL!O7%;=rl8rdZ2`|1m?YN*sru_gPDGVM+VUW>D0$snyCu}}U znx6Kma#;d;qxz|SbGxCpuwzK#bncpZyjk`8*E>veY2e?hDemhpm#nb5_x!yi(aS^R zIZZjoDv85HU9SvyxKt$mefbCyLE}HL-cYEOCngoQTO(i`yo>?}urI*B+6q{-Ri{_4 z)WoqSCYxZlm!_ia)CgiM`6t*y+(08S>n)Zkf##i!Ee2rM8mdX1OY zF5lWQ4I4E&L}t@=f*>Kxv4~#MLs+x|P)kG)_tb72&uxkq%~+ERP*;YA2D{yGJU}Nn>_?lmAQ>o$ms3WcEnFRE>I{-lMe^% z#eNw&K}uxyH~u>77uJ;W_nE3PU%`}JMv!7PKon4Cb{q8d)4wvV;hPZhs{a;3 zZ{-8DKYpev(7b|NLc}LsDjV%(`N|?caM^00^IMth3 z1St`)6FBCm@xy;1y$xItsRzzSqj{VFAol?JD6uK{VqT0G#m%^0ALmqKp0v-EX@?&N zs^(pxV-hhBGp-s1v@^45d!MZinnu8aUU5eZbz#TExUavmU+Qq2Pz8*sTGb^KGM?gM zz^It@TJ9xm+D#@)b~t;h$+MH`b7a@~!O6M5Px+8r-%GNB+&M)Av$9 zZ;W3-dL-SfAk|))Zd~6nJ5$FqS2uCy?Kd-ZY_l~_N2NCVsn0o=RBb?%)nhtPRq_af zQD;YX`Jgm9)oRp8CQQ}9uIFeDW1pU)wWF}EKm&|-&9Q)OJ9f0EBcwXfA56ZGP3Z{t zAIvXT-Q?Y}{>q{`CPvO+l<$`H{#Z?AIA%H(1u8=|1Lkz4CRmoJ*Kobq)Cj>1625zh zj0$864AO>rNsrQK&Fw>|3s$$5-|gP1ckfA}(M4G_TfL%x3zk}pBpGzqrMO<2PXx<| zaQXLvTV5vt5~$#Hm)hb=@xkw5@i35LaTT%(tFDeUGBT1j>z9fZ8|MnSLEu%hpdc3j zbTObNos5U{WA8yxC+3C}l{a5M|1-A}iElTfeOKf3A^l7p1nQROF$Pb?h8H$BgT#)r(KxH*ej+3)Nij-uzKsPyeaBUb!do zTI;5&1%o!H|5HBP4)814e{cokxp*{4e^J`@>^lM&D5$nuFTbtNNten!$HJ{HEEx^=rvx#rZ2Zk+lj59tP7OteFI zk^MX3n5#&kMPrOCam)zbT>el&arVFgeLU^x7=D&|E|WDe|6wfFzIX4tp&nc?@m9l* zXY`25IftD0a%los=iq3DbFE7C^!UA3vZ;{`Kly2=sqguGOyJG0b|9#u0!)*YH{!xa zVS#?reYzYN77U9ggoHCL&X^I^vfO1&J2hee7O?<%DG4UdiIok}oS63=g#L z)X74WybXaQ_Ft+SC}i zZ}L*RWnR$~1^%{u+O#;(z)dcdzO_?;O`kR|N3F`hqjD@y$RK$M{7z}!OCM7tf4fp^ zvxDhags$hA#}HhnU9KX35hq)3sqCqfDhl%1XzE@I@nXvi0m%N;>H1=L(@JIOe?rqE znJF~^Xiu2b1pSwp%E!KV0aaSBw7zmKQvOMCN)hu0HYk5CkgwqSZjHONa+FH-o^5GE zJ6aw|#GP|7cHs;EDd2F4)2ASytE<+l-@N%;BQsSkGtQ6fJvSy$nu36azD-*%mTo*a zIT!r;;v!=sBN??Ts(A9#=n;Q?k(C{+xsS=bvXtzomWxh3px4&w&-7PIkZ6uCSFW5_ z5qAcpbarW*V@avz9u7=UPbz{;YtmQ19$#Pc?4yrrX&uWUd{I5COp@x2`LjXpv7JB4 zf#Fv32WcLT;pD~7X08)V+OoIbrLXnXulUT-xj3ms9EIMaZ5!p$y=NwFWto?-{jgjA z7JT+Msi&QS_}$9Ph-A$;?|Sn^@1B?lL@aTi79v*LdYiCa863qI_tUFRHS#AIp%&MZthJe1z#31L}$BRAL0pIlyL zE?e!zPf$KryqYHGH$HP8e90|G6j;Q5QD9xsEoXj^Ty~fO(#slYBVB(Rq&{XrUls`- z4lckFu=b~8gaXev1FWI_`YUEvauy6CC1_*{9auOO8bjcf&J4H_1h8@^?3o~v^hzE{ zpN{RwaJ@pMe%!mCy{ zJ@l91$?5#D$}6{P-mA5mf2-Hu%5*RrN!Hq&zOTHik?=dw5;t`s%f9nd#2l*JYRr$B z12=UNkQ(TwqYSjDYh_()gBC6F(_RBrBQS{u5y0utpx)IA^s6lTZ7hYnQm`0W9G|3@ zi+9V+n>zxK`p0|~CwfK&!0FwbIMYajCNNAi>ge&C%7st$V+ zo=;%9EK_U5E5ik9jm)M@l}WDxb+X+(fELWc#Y|;!!71Dct5o;6l7xMOLP_gapTFPf z`!b*9%8tNv5~mo~mK3p#RCtRr5TG7y+xl_|qs2sYzhUT*V#Fw13=1Kjq1S60Xgos< zykvoznystORN2ZAHk|t*Z}|ZQ5a2*BaOT&2@Z0yOv_M-*f%co<+V;s8Hm_IRtdo1p z$^3n74&KRsH4j3S+x6Bq2@$;BOnCe@XIhK(y~>~eb3wP$=56-w>akASbAPH}vcY(Z+5J8!9a!(;EH#U3uCIN}{ds99P^IdO$(w9iB3%aY6Dmym z!WLeP_hjB9vn8G8K^HODhO9M1O>;AeTD*r+EIIiJ9NiDwLj?#qJ@fda;*D$CwrR7| z_u1EomzLhM>vUM3K3J99r=iJo>($q1&q|+I`ao2kk-g2NTjg~=;d5r)<&(ZtRkf({ z&+9`a9XvBaN#t_21C}rwsn%|?OglX@WHp_jIZ~l z;baLv3CVHdtdr@oO+|vRj*d>-HbX4f+946+>4M%h3aFhsV(1)b&PSyl^Ph@2(@xUN zO0F>Ku3bEu%U->cV~*bdTh84vyy=>;81j9}tHaKemu3GB+PA3B*Lp7~qMCoP z@XudHd?;x+=>BW>L!2tz zu3eA)y`{XmvzW%h!v2y%!VG4u?JghnH2PWSdWkZVqY{a6Crt4r&fH4@9CUQ@)8H8k zU4)tU=K|;_voW!d6G(ihnq!&*z%6b*O`Q1OVI^;s21V&LZT{11MhNHk%btXeuX{U< zxowbf_?b>kl8O4aU2A(Mx64@mWmVbdZ(c4qT@%7o=7YAs8!2O(@M`ibs_+p{|7XBV zHB_w^94<}LUvwodu4x8cg}&QnZB@zy1{C^unMk88irl563H5@@3(JMIgceOHzZ#Q$ zv@`T16Pg;VS-o0_XtL3APA|WJ1k${eL0HEq0OO=H*r6Dofhn$Bd8%tfa-x$DW>Vti z#w8Qm4gokonw^3e_rqTuCzOYCMY$a(gRx|~VB;W)4{?s>W{j#?x%#(qJ`!I+kefR3 z5CqQT9*-V5L$=vSbt1hm)1u-cY&u0|A*D^j^;cG}j`$TELhRgVc91(ogJ*@)}p#b=JS<5J3HlOvpgqbG~gW!2tkJiqmn& z#p_KYUp7mqY#M3%aoaJ5SR|M1ZCh%vYW6yWnfDx3|3lqaE{>{n<vD7l+%r(GU9Um zM44m2aG%XGf`fLUJ3+a@_!O9jLa|s@CKup@<0NHvq^-rTwF{-!7kdJFw2~siqToH) zQ5Y$`zPhGV6%4r>J8{Jm`2b#ACrJUp3|(s@`H&_6#hd#|nKRNagyTs=S6447Wo(R{L->z-vld5a>E6k+ zjdN+y+1(sTNjS@_yUNgYqN%Ar_a-mi;9)ZVWyk&llf!0D81UugNm|WqWbf9&TXjd5 z^G%p`{Dn9^kV3Qy!1AGU9=sl!@_`{KepTW)2#WT(;NF~GLcx>orS2YoApOmhPv4Wh zDTtW82f&U7vrS#SSUr~dPdwM@V~kg#Kx18aRB0sJg5;@528X+u0f{ulj_-pfa9z+q z;iz=GUEc=|Fe282Np4V=ohNSSw!%^vcSz~1av{yLSD z+S5}dpE?FrC+#1?XJHRouYs>CsIH5rEs&ie=tY7bgjqqR3lUavP7xzKfX*IzmQ_>@ zC{kp#7G8Wc3MkP5Ud^olPHIT84d%QuvakSYj4hLfBq$BHB2XmP^v+yzbi2QG7d&!WD2oKlbM4A|!D>tMF4 z@*$7u+pX7rP5|*N{CCVG&lYoEb_X}pyxf1n%A##=cGB;M1=z^#@9xHwcU@OHl-CM} zK(P`Ws;tMXSg|5cs6}_F$*ewAG2B|^r`Zh6=+n-=TQ+S95UXeC;i-FOLsrkD z%8fb4S#T!L0llWj9(52PZVr#!tB|f3(uYceCZPXVKX}+hKficGcQDy zWeLzD6*p4`^=tAJJz7w5+>$f*Mc*N5lEUV5LFHe5nJw$=aQjNZ6=r1oAJ+j8noydD zX0BNztqvtLnQVPw*H1a%+uTnFweat_9GXHo2(0Hvp}e%6K$v4kjtqL6_SB|RL+a$9 zt1bP~$E%X-YW*=s`+lp{i^gakil@`hv(4dS*9|uqxx=BZs>7v%X`u^8_LqCM1~^?- zix|&^{ujlVs*h4ZoF78q(hoa%;)Fu((N?*RXUa>-BC5)*XClH|LsK3s)OYcS7jWFg zh{y&i-PqVK22r@`%_w?^T{zSct#!`apZ5Z3Ic@3|Y3ONZw$x4iJ;tuFi8wded`S*3(>m8MAo&)i7nh^hLK3H&sm23ITIpTK#t?UH`E!_u1+(gorip`$aH zP$USMXOuF@Jcl4Ua=o#<>3qgcGpd^m8$SF?0Xuty1u$~#*j_r5m0Qh5Cu`4@B1v3>2NG5xD6IU2^Ij6CNQQOt+mjVI)0pz82wLSZWN>c{E8Wiu)v146h+|eWpSxhS0 zkFOtg0hTf%qdA4jr8yHADBh!*cqR%^nGu$i1B_gDlI>f=Z#IhB6)!V1LH}j?qN*-fcXi z#9x_i)*zX#UB2Pz^XCuKPQeUiJR>eR9e~b5A3`w|mqQtgLInQerne&w!whH#Vhy8U zoy4tRI-Ou5G2qOY6eg*%>BCfptdeaC+{Tn)?A^=UB3ZWd7)JBbBM3GL-lGWzvqf$H#tNzm@b%FQ&H=nP3hs2F=)Km{krr z-uPw9A_BGv!%+Zz^QFL>LTHv_nl%4UL{duxc@B>s0apL;WZ&r2H)~V>%bFnE7 zSdty96*B4g#rcB^`)%E`CqyhRC;=x99=yulgX||rjEp|O#0oz&?rN{m+0&SO9x(Cc zk~2~=Qz>s>kzXZZ4+h7=oA$~OqXgj`)s?Vt@T!nEE0iw}7Nq>Ko@eN9`x*D+&9gtc zr*8b8Y0*=C`VKEjXo`h*Qk(9k_nTJyfirvFJF!bMcBXyE`dH*^^`e{1HJx^@X%QK% zXIDL0pXCb5A;XVDEkRU%F|LC~#>OdR@Ur>Lo9eg%@O6sfj5mATsUYU&K&%QFCnn)J ze=An{6aqJen&-eu2&QX2gse%~xZ0W&;$h->yDy-D$N3HQO9Rk~j3O+q9_= z{naT4mv`#~k{`QTZS~ism+e(ff6dgXspVbu(R{^rfNwSQ&Kk6+FaO}>*;h9@u~TTe zaOK~RyxxAXU-Jr{Rb>~$v$sp+yf0tA>@ci}vsx_%UMvk!93!ddo>i=%%IpCYN#M#89;le9039!k zgRUu40})$OJsio(%IYw-nre^>AQ$jjN&d+Jj1nNTxDLM;fhUfb;?pDDc=a*HK1NJc zWM^wZZkyn(Ioq5&FhSHYd{(eP|mYa@G0U$$;T$rc7a%z#@F8FfFmqhwV|f= zQ5ak6*6A+2Np|cBDlbz<5{6O_WGMB(ty{M`FP*9*zc`IkEc?5}?iTO$&~MmgE6vcB zEn`MZTqNv3zieB+((n~~Oajo;d@j+lu(drRK?tpl@Xs=}NNx}>1#poSAmoDU%^PWJ z2SL?Uygk924*L81%VYWaGXnrAP!h#O$Koh_j>Irs;w16Z-h+BmG>kA2xo$64Js*~0 z@}sbiIhU0+MNH4+x9iohGt9LF%Tx*Yx$%;#n~ z+EWlRsA+rPaI?X^g7?mzJg%Vpr2TaaKj;2Z9Ja5@iR1f9p9Q?5X3bJGjZRCGk)60K ztT|YnT>j%}!^z_H8jRncR|X6s=lk(H9lKlo z=FNf_cTO>)%NJt7MudB^WKnS=Jd&_i=p+z6z}yazAFK4olc4n>1B-WfC{8){bdJP( zKenbj8AatL2xcQNf>H{YHsGUG2p7tD6}wqwtFVxYh-~*<&fFJs%LmJ!HwT&?%Lw)9 zMyR;w#7Rk{lcG>DG&F3`?iWuRJU3H0<}~Ov3Iy9o}93mH>k1U9+-hW54#Rn(|b~^7*WZ~^yC7ErltS% zY^@va`AfZW{vD3R{F2+5ij4hBjh%#u3H@17k*Ll$Q~DbAvkS?EBKLdFpzXh*G84rv zNaRkRqi!&lf8qq1Z}sGcWDmP9vmO_2 zeIm`Of`&7)JaO-p#qRo;xd`S0pl=+X{Z@u&0Wsuo@K`9L z!L1EC3`hAY4iy}a8>kM8q%wfbFl*mRO9DG{fywZWdoFo7i6hk{SMDZ5qnlNwq;|TN z{noeSRnaV=JsluwMJ#1JCpOFqLNIp6$cA+s?%WBv^km9`VJeTu$3~7jU30<^pLnv7 z=9Q^Gc7-O~`P|E!Ff-t}s6Prj{d@Jd)7#g%t%?$90Ipdf0jC%{F&EsS;R@Qp!UYJK zriRkLt6n8lK1#z0WyKOZ(Kf|D8+7|7f0UwQLzF0v!7{#4FK19);}w>(3XRVHpffAj zMTkwY8FNl3wBR@N`CAg=3RZZqk>{tmr+Or%o{(ag&g7<@2ECE0viMa+Ra+@$0XaXw z7-Z|iV3HW*&1Chaz3xS`6;pB$<;b+2paF9otEmDLqg$z zI)a$W{MO>i&^9LQED-(`2tmv_a2kQ$p`-a;nbu9Gd9F4yg{t-1Hr_iQKubfDT1Pnv zPH7wCe*bl|k!7HCenM#InQE*%~hqq1DORZvPx&DLsjjsAj`UrcY z^mTr~quq@(|31(t%A#MFZoFl^Yc2JI@Myjth&tsfR@8`(+*@={GFFXq%q zhSq1dv;cwor{0 z)QgaoE1)NVxMP32dQF=Ki*}t)R|@DycKWcfgrzQ^sS}|u^3|IA;lH6( z+rKM%23>Y#hoq&B;h|M=I?!Rc(u3xNn1kgxO%Z?GG-v+CSfX61jfwM6@`X* zJD%weg+{(TZ5uM^8sTu&{Cp2O^{-ziu+DMigWp^y-spqr5TBnq$}T@>WnCW@wE9OF zGGniGo*7d;*;VmIc+@qSA&^3GF1y@*<2+nEtw^mmkL;@RO0 zw5|65CTnPfZRZ%1$S!59nB=y6ipetJS5#!sEjrnsvtRilBs~v<1>79Wz)UAfTj53rY+jNVg)OASDc)(hLIp$z>b@UwKop;{7_w#(}K@5LM-;4xh0X-g5X&_rQg9&LldVw0hV#FU^yAi>6 z3)Rd&U@T#MV`F=yF(wtkmOzTU>LCia7{K<8*fnHP(g8N*LH6x5v9olVK~P~K%=s%7 z4RrzT2WmLRwQ{ybh_L|-aHErNZKaUW>{KMhHgBzY=2+Q;S{=})zuG^{UENb)J6?>! zXU=Ru#z1f)eyQ^50aRFl=j3fz(v+k5{-nFvU*JhstM;khj=S;Te{Z;Ldk-3mgI;Pq zCM+Hx-=lLo+2xibx z0Ew-qk}kxxPA=k5=Ef^TBzSir1t&Dm;80;jT^^eH5FQ$OALy2p7z9rNdyj}?NM*kM zyyM_TF#&W5>Ue;b04pY4iWKmrh}A{lXK!!s?%EB3`~9vIU1S9XxCJFpat7PSZUadZ zJXBMl{o`LI>sRhY8{|}K@?K#uHh~rdht}c-PrvW~@bprw5VQ2xac5j%qy8?fHSe*; zm3v~R=-ZgC+gidsFZ=+YKV%~MLN4AfJy`_ zjIPfjj2_jda@I&9jWBv(Hx>vm3~xi)4~At4m-JI-&V0|RdG8OuQLYam0VvwmK_W09 z`@w?;6A-?x21>4t2OT*1G|2BoF&;=3&A03thPTjy7S+g$q=j^Ih`T+p&AV>suPzk{ zz96Z#khlU30h-jV^=fbUNoenp93L8ni4 zUh%7NW$S$ZhaY*`0<;=PBac{sYFgy1GJdMRIrW&(mjbSx@z;(VWK5r-kAXkKH=>su z{axXIJ8`;4ZeB9L{esY&ZYqpW&FS{mac9Z`;>}qIoC&P*u4)iGn=NMv}{1D(1=L% zS(ui9rx)c0L5Yakz|kP2sev$GM9#zL&4}v2O>Gi@X`=Ol{D4qZuaOGOS}ub;RcJ#; z2N4cXT=yH?@y6$vrj4)=LkP(q`hA2> zt@W9Yj|BvMA#M~3IE|S%KrmQ>cnB2*PDq-Azz|Y1*zLA$-_B<7AFIB_1^7%krQJ75 zKPVfGX-FGl+g~anoLby!9&J_huNjXnEdXj3R32giESQ;@HzqB?7b9JlZ~&Y#@CYwL z+UOMW%-hi>?Zh^ucSzI)f?_S$Mbqea(}k!1ui=|?^nVQB(4LbGjbmY+4FJW!Yb5y} z9LB(BVUCJu=oL54UjQ-F0QCYxVAQL9g^yqZP?aWyq1r6=)vLdi&2G4-rau}RmvE4O z{s!%vgI|#TD_0PZKIrU2e`%Iw(nqr$*f2>lbOJmz)m`~Na+S7o|8oTEIA_;_`%dk2 zHf`Hc%0;0r5CrYb0}mpxucYHI;;xqWwW&IfP=5D+k;!zOui zV+t)4$glP@mu*p7i76Asa4Mq@P zjf`i{o+&j1dLRcU3O0d_b*_Yphav+2U+7#9&U&Q0hE<#hfb37mI-& z9lhG@yQ=xDcG8)C7q~~;Gk6*vCS)&eLlGjD9Rhz5cR4)ZH%~Wj31pOn5H|*B=e_H| z$V5+KKa*+d8kL1z&|!}2}l)#l<15q92@|UB*TS*uI;~R98ea6b3=+1%KXKW8Y1*1)1mOLre19z z?cCg7H^gcT8#>6C=x9C^-NLqz3w>NzK4~QPk<^hCa|7pqQw8)I_2+2f2q)cmr6~Vb z5^*Q9_gAq?wk#Z|ZfM0^PP`V_2IEbx5Wz@`1}gmpr{bqzwp+#3mq-s$x>GI#l%|U? zX=V!9R3HG;1Z*nZaX|hc{{u>vK++VXh`<9_o|-_Mb48Kqe@zgX=q`YC1Z7S9+gPLU zO7w6-Eee>sluB}Iv%tTBFy2!@qE13wHyTMqAtV0~fUyZG0zlTGqqzcG(T`cP&M>Yp z0;@H6z9Df9#hN165W-L*s)oQ=4EAs+N*95>G6UhyD5qsmU?AQWyQWXiDYX)`^c;{Q zbWH=KaA+MHxaHY!)>3BqR!i6X#UY7EV6+d;#@XiP1+BNWkLx z7jI94@zo|~Aj1ADCj9&Pz|oupvWvH2H3or(`=kf*%cr(|$~~^8(YEK4g+&k9V*0oH zzZkn5u~wJw??)9MRL^VjTY>J}7C8DVL_pYGwOIjnTNF9Bk#In#8H4)oZg}e`=L`vf zwN3*U4W{rm#B_tsDpGO`N?r2cQwCTKJP;3|g9H(2*8mCFXhRVM^$UJoya2H8Xb3rG zt!Ds@Rl-n!-N7T&2MjihzqDeF1>fK9f1iym4Yk`~fI-uWvu5E@L2e-kMpuS*TPq73 z#4z4;W0WRrX}qUTCj(pvEv1I4^anUhQjoj3%L>7TX+8j}z#Bl*vOu=+B}*3QEO-RL zom@8JV*Q;63-+(M0nRXl9=?3}a?LMr4I)`LiaSsuV7!2jN8J)Kixk?jr67evGGtA% zSwJ0$hEId2{jbc634RFbP5wjeO1Z*uv{2$B1GDs!E{+B4%$Zy*B>Ou!1hJ&dypNVG zGdljnwd`LwYHIrox=-lE`D)V^ZQwZKn6H6=#ur3S$8V}QKAgW%W~z{KUVogcC4Zaj z^^hsieeK4!uH7T@!aXrmsVw6LJXWgcW4iSHFX!^6S_*r!X)+e!HU{;PC+K6luB!f= z(wh-$v%>Ze6LYYt>sMr*xLVsw&z-8nU0E}}&gJEJQ?M+zD4n=K3HLKsGXy9!6}EWr zQn=_MhbNkN0|eq-qYZv6Wiv=I5lBzZSbh4`srh9f{{P9QKyw_!{}xCqf#QFMO@9Ee ze=56{6LN^pA3F;+bNHTXP$QcV;rYiwH+1S0+z2^COAzE%zBVIy#TeEG5@DjU z5gJ$IefZ#Mhw*xu`X6*A4fGH$56HjXftQsdI~-MDNVrD0CKW*GMZNiQKxo+ycQYIpl{qhKsj{zfx|^+U=T^H zu7k`}?FL*^3e@mwFr)7xDo^A_UjzhbSXfx-A}Iiygky@I-%qCBpDdJm-1R&JbT)L% zTjJRpF0;F*X{9^?Kj42)L6hl!^H5S0oo7muVk8A`giLiX{5oRmC@Oxy4hv7_e6P{x z`@MM?3&CNOlHp2Ii(f%UE^ca*a3KLyY!C%bSWe_VjwSBJ9>Dl8UYPBgL2u2AcaWW*&B$y(k?ySe(K|SomUNM5A}Q} z31Lzk4Ll-}`@D@O*bk@N6*`uqD=w(j0Tc~WF;>s=c-CSvZFjE`lZPkI99dKS3r*Sx z&LhtdPnjR5e{lb}x7tcGdELOX?Kc1JPr7v!Y`XswLSYoiQpFeVasw_3 z0WNhUOOxWSP_+n0d6w-mU2-BhCK^7H(2k#n+oJO<1MK*2+ zon|>=5rNwQ*eV3XJ{jq^H+G1fvav zip6yH;SFcezp)8&$|zX^=yedETn1}`2?^i^b3g&mSpXfF^39!e%d_x2pgJBPB7pWa zh6(`&o`84-EfA)r!9bBWRUi_AHID>#8-O}U6_D5fKy;p?M+YBEXM#c$ST08pxqzlB zFC7B{55!PHo2eNntzxVZE#svJ8Cxbr;|GwJ7wUnw$*UH(3LyCmfj(&DfPVniD9jaL zd+<@;g5KS45Vb2M>|{J z_@P4|UB{84#TSBfM;=d)HZrsyFp5(ZKWPLoWtic}E$3{7q7Oz8w-jzXr4`$wuSl@! z?YWl0eD<4-)~Bkp{k~cW`*7Twt^IfAoaJu*_=ygeF6HtM_SUaD38+t!; zw+}`|)4dP&RiZN+m0EZ2`NPJM3SCc5k9aj2G#S4H@?DfzaQAfw(ClbP&NofEJDdO) z0uRCijU!Ai%YasAu%NRAK5~RfLJ6(|KSB$z-=Dko0K~Wd@L?Eoml#>Mg8k)6>azoxM_HXb3DJj=w;)4C&p*Pj5 z4V?-JdHkX9(gH?MR~j~XfSB9i@nQm#?7ho$g`$XWDf1`*Eg4T%!EcW_0FblLXb3nv zQ@=XW-HC<7QDg~)85B2*D6l%(VK)Qq*szkDC(vX7z9O4dF^EQVDGai&_DQ@1}B@v=XrbCol$>otgHzMHlb zMkH7qs1sW0{!S!;m?q;TcW@?)TbR~#3rUuu6Y5-I6c?s3rKMuN+QHt%r|ufBd;m)p zl=}WLn z+yAsa!~0J-@LTy3B!m&T7OgrqBm%h z1p)~qen4O$RQ_Rj2-b#Tj>{ZcK_oGE+q7v=0EtOKTMLm0uFwB^_kaHO4y2~{T;eZ3iN*N`u8O2 z&WZY0SLf|EuSEDd8bPP#mCdzX?KLW{Kv{w;ow`JjMgtNqC2EGslN3DUD&W9)4S-BI z9y&@?Q7~$3k|XWs`es4|1*E{BAM)^9k6 zKceOS68Y4CY0^ax5g1q>P9QEE;&>t7+64(Az@n2B<^toe$#_t9*JU6tgq#}~aX~n7 z5sl+O#V!dDK@`CUMXDx%1~=glDFBHj8fL8xB3-7SY`qLH(&`}d(18QDo=?(M8lb2H z08Idm)waRHMu>C^$XcxX5r}eh2tWZH2{L~i7UC|D1qga2Z4+G!2vbgmZNVI!s;=uS zw?R!^2AyjIrO`VHiIbpfQ&Lvu$APo}iIh>G_TNwb=>O-FhmA3;&nB{m7_rzo_ZwvA zC=U75X`Bq`iQSK7#+H}9!hIJ1U|PFr>KWL*L5VN_SOnK~%PttC_Qhd3Y4vU&inRx| zO%oJ;*4d1NhZ#-Wa!*iD(Bf0f*?0i5>oYs|`@VZFc8K}T1>w_h)-6sqZGG$ z{+1FLDUG*ppIbCnrZV$SP0rQcAF=DDJy(Ge+?OF4q^o=%l3BiwhrUlVoa&d>4w~wkW5I&);tszVdOlMe`u_6o)vL>a&mYrk0-G?7TH5gm8=CYas#=2Pm1dJPT5mXg>$?1_ceM(*OP@dO$e~RAHgzu-;Al`&aerLO znKgBF8_p#7+U~&gu02N!Z%D(py~{Li(D>58{rs6E{)~Wu(DDj(gyb6TyySDfaX?eW z^s&;OzTI}>BWYYIA33<0@Hx~iKVqC`d@@g(@Y390E6v|v8gxj(4BrtB3Q&`8ZQe1@ zx}28uNwXs)lXpQqXt&LwA8=7>X5N>N6t<;5y7FAOv|>5>_quwFw5xJQ*60t7-&(iB zZUhd0W$Key&n(bt!H5ZF$q!(n-iL)rnV2&3Tx?7DsEl=2-N(RCoVD&URTId~A^k34 zWaLO)qkyJ)s?N~RCHNSjDpWpnz@C;>_MW={ddxHs<)*%TdD*Q@=BoTw2uwHVD-l&U zo9M$*zc&iB#Y*cq!RMBf<3D0|8$N0c4#{h2mK|nYvv=cQ&;E#6Id)}oavP6;Gp~fi z?HCDRwdI+XoZFo1YmeO0ev8kIH*t4ljuOTi?p;;b0kgTeGVRuj9*H`xyGEd2936dd z+tyB4?MXo#$MIzzi(6LNv5?J_wKQAt@xsGQUj7U@=Mt?zwr%iy)mW^0FFg%Mfn3lF zBVU?s1{5X8|K-ELWpnR5S~w{NKvckSA;LFJ1{oeq^}UqSRVdC0BTPWTh#WcLVPQ!? zbyn^v|7>aQf}93U%f=O4U4 zkS>}O1I45d71>gd{vH7^P-I4h%{PK>v+Bfrh$cu_?s21w5=J%}G;9Tt{h+Di!GSdf z1!dl^d3g&uY;+XrK%0stHK5%P^3k&FkTe{QMpToW;Y)Y!E1;(9`WG12a)Yq&O5+>!sosN6ffF}A`Be@cj;dw6e&w#+eR(f z=<+^I1pwP&S8f9zvnIfc+5-?K1V?w;c6y7NUQ={e-*gbPkeu>A`7m|hAxGimF@Oi8 zgIO#I&fKUsY2;nGy21%sZ=8*Jn_bKeJQy@TD!h+z&&>{aq#=CM#!mrSc_*PP%}9qE zPKims^UkRWWhE_M4*s=AntTj*J492qGHonjpYWd5$%r3Yu&!vd2nYKJj zH}~N{4!sf)U7Lcr%@P4u+0Gr+wk;RG9wR9VehZG)8-f%mH~*$(9hXJc0nbOew3Xli z@W^V6%&n}GpFWv}kY6!MPlYs4N0)7iGWM|8fY$A~36VoGw#9o>gX75cchuj4&H#2D zs>sUL;|DQG5V?uwX+Y0bICJLn30Rac_&|tKi)Dwpx@$DZ@1-N)!X;D76|*1WmKv6%PN1z85L97g%fK~&c(@{+ZBJLkDOSG$&%e+Mv3CI(;YZCU^xsrF;ci!lc%+fEcWm&Z9`#R=D$C-QA zXZAC8n}46qbkO{>AoWQ^F^*Q>w5OI@J!S82p+yUN#BuwPUL1|kpR;rXS9`Wt%gZ_j zE8oMsJ}8PIaV?p_|HDc$WU;CR3w{2f&`*mr56DXsD3i&iJ{6=Y-Tme$U$SEu7=9Uik_0KR*TB?`i9d8Vml?p?S&G z<{P~#iQZ$S|4qUC=fL8v;ldXsb<}vnig!4gtAn!cnhI7B8{u_Q$C$c?IXPR#J)5<3 zOFQZ+2WJ3&VqjUjY+`Un#OfvR3>3fyO(FRtkJ@|Wc?~gi7fgl0&b;QO{NVGsKypCr z)F0P$Gp6)k%s^XK;o!n?I;;}U627lVOU%kfCe z5fW8jaB{47i|KSyJX?Ey)LzH4#qH-6`ip-?8>htLy9`Iq)bY1`EMS=Yqw{c7U}=RU zN@PtvP(es&mp(%1O|aAxdvrs0V7B62{GU?^7-Nlj-1s7cY!YFXh6&EMDvA#|NUQzS zlffpQ^8p?mE**oDrTGAH!3@D<7$2m|As!--)j@moc(L{2t4mMcO>$8avS38hQd5gD zL^55$t`8eociO2ue?9{_T_BMWSs<1O1W4DcTxs31a9hna1|f(*rBb zPpY8mFp#F42oyE@Z2j4C4|&@PuiIdj#HxfR!En6A0&I^y{EXKPre)SO9&_<=j1}wa z=9h0Oapg2l!NtuV~c*Nn=Ar9)9pMaCs)bCT+5J&5?Xw#v0N*p*g5bQZB;hd ze4?3|$tJnj3}_XExc~|ij)yGZ6hOqza)0ezc0oH)%1sEl1~e@$_`>-3H~~cS79BG) zL9LtRtdY+ia4Zzs<;%bYh+IinT%0W{@`J;HG=-3SRAzH=3F7h`f=o7phYKQaG@A%zn{)fprGmKAiRVc(efrSbgHqs*ZU$n{L7Wa_$gSMV-%S91 zi%`Mcdnha#K=5QERiZHPHK^x;!y^??IduS&_;E;X2@C-m7XnAg&6@xWd#{1O3P!Yx zE$ciOdHR+Cc<6Wu&=a5p&H8Sb3ji61IbAb0`-Zz4EwU%FPkr&Hw>P6y8!#2t$L(OL zEVJ+82dBR#f7RxnK~E&rAi~Gf#;pm5Dez#H%>yQaZ+hHV-#&)i zG~E|hGQMtZJAYAbQ~s0}!@sKXH5UTY+rRh?&+86CYp!_|X|A@Gmp%0LtrYBf?JSU( zCSV|6De0z`d+9)H$)a?lgf!Cm<3|Ss7i{;Uy5-1F9K+ayrXD3MEDt1Ge1C;6#+0S> zV&YW0vxjrYi>sP(p67pO7R)+(K-2O>W@WHeKZuvd0esIY69IyaW-tQGeLB0d#69}M z4OP&^@bd^Ph4svmAkbN%)@e{K&3Ps|$%o#spwCoKue|2vs=7O0eOc+FM6m88Bav4@hbCWvx*2SY#E;}wQ11HW6L`rES!25d^{ zF4ko}$pm`ZBQ+V@RhHbyLxm(_$gonk$JN#WuFO|w#Y0L5c3pY+G`nI36 z95G%j9p1a6;<~_^%Ywiye;X~@Ou1jl;8~Kz>cDi{97iM%dnmT-I-C$jCPsBdiL+IN zxv2$g7w!rn#8x~rsWO9tQziKFkCVRAIMO5f-p}%^Y671YCVzX=h?Fb1BAKBm7p6E9)I2$xfhIi#+`irfjp6yA%PXk z_$s6u>9F|2*sArup!2U61Ej*yr6B8)cs#cAkwUQ71!3bO)V25^>#~$o+J%(f4tqC& z6CWpAX%BtpbLz0FoFA?C!gCWs^7i>i5V_0 z*z)K_U*34*a@)5ft}M~|BYRU@2rbXB>~wzXn~E8J9C0Zi+tm06{_Ei=r{lsEUxjp3 z9+=PZ8e5Lgrc!Gv>U3WJHlDVn4RJXWjA>34rA*ULKg9J%m9NjIi`bUOpS*DmhC_4@ zv<&yA4p5gYA(U<|=^b`#L9r*_@YfPCa#5Amrb=7GE|(<|A;+CU8JKOUp!8%^s7@x& zI*m8hgFGGzk+hYypFhsFkNhx()xA*Qwcgu%FvGx2@6sh_=&R=*T@YU_z80v58yO)K z^!A7Iy1A3^*5v8Wtq>TTe29&F$5t=ZfBYL!CYkbUw7v+q0-|-TqPme$YQN`#2mB-@ zV~n_wwEM+koWb%v3i&f6C(ze5@YAG+n}XOCJ3igTkyx-ThO`CQExghl6qNA}G6P)^ z#pi!BwNPl@-yM*SM-^Btm;rVF_WCd=n-FE?mkP2@sKb%u3@J20C~pR4Ih1-Jgcu3P zi$P`-kl3dH8-d5%vI21-)BgSOfERrDNcEI%0Lc>`<`V%b0^8*%cs0-{21hWSBQ++l z-X>lUt^pQMlr0YheKJb(4efsi)E(bN{r;~T#FO3We`c)#q7Lj%OLGBaF^-OpTYi6a zEc)|ywNcp2QTP_vJdQM8rM_#O)P##fsb1nSjXVH+j3JdFNFl&ko$9a#i2#cup{hJ0 zOEY89sW*|+6n-!m0-6@SJ}@;!3I`B3><;+Q4|ZF1G;nMHOY4)umUue0yzREOeX!W5V&&AS&iM?*RkH)M@hOK8i?G;sJinO+7pWfMIm%8dij`?gWU#a`q3{qoe4jAiMr50gnrH9KM!q0=IEq>)e*GNB$U7=TOQb55d5m@ZmS=ou3qutv-20Q?<&^;xqK4LKkf+ z%Na5fFz=py)JWdWCS9DTm%CE~-p*-NQ{Hob`UTTV9M39H$ox7=bxn*#k}=NohkqG( zt%=;y47+jlstwrsab9bg2n(9moX3Yh+-K0*SS?#OPWquGs}|#%tLYd;o_F&qj&L-T zwJD2#-+NF|z5DGEvRz}xJJQf~@y&t0hqoB+b2{9wzN7twV#q$MlXjI6QJ{yz;Ahl3 zPs6jd*z#SU$5-NPuiLfn(UkwGnw2qayxx16`^lj}hp-Z9kHHLv$C%aGHAm0dlZxsA zD`)a1do=IT$MBrBBlIq(N)#=Gcw))f*piV#y=9AZXUaXBq6RJ6Cw;#t492jFT{{nZ zQJepF;vv@QW)jX#BA@u=yuiry5Pydoi$geL!l+wSTj`B9{sPKFx2~(?l|bt{ML{*_ zX@Y`9HK43MG~qCDO|KlYPA>h`L|Y`EX%xi!SFByL^%POr)cDTnx|k?Vt)6O{y!(;T zgGaaV_Hq}53B78YRD5$Z92_qWJ;|h*a~T2F$6(x(rX$1o!H0k9ZS?w5`AUHo96-t; zZAo6wB=(to{B$M~`U%3%$CeT$G>fiBvr)mpKePnZf@xzzlHK{AjpFsbDWocv?A)dx zHo7x6bwG9z7wdEzAc}wzJW6<rhj(qVnH##_ zA4y#`$s7E0RC-;0&(*w4!GNnuvRDmh#MEp|6GYaRFVTAzEpT`RpTu?-G{wfmuP$KE z;ChJKfvhd+6f&|m3XG0~nKi}o>J`kqRJu_Hd3I)XVe}3X9Pk8_IHcSArkamIZj1;_ z#d-~n;E*079!{I?d?LthN=_-xuO?N{)}#@nDOl#XHa=mC`K%b{>F`yJ&@Bvuq4U(N ztWq{_Au6c~s@=O+oNrs9xpX*zTU@+)Y5-dVD`az}lhB?>Uk2;%etb^8}gk9iCCO@P9R zicZ3NI)kIFE-YtNe+#3Ljq|6Ij9l*~4lSt9_6G_(nmwwn@kz~XOXR}-Y8ny*^ol|WcM|6 za#Rw5!A1WOsLBDa<(LFpI7WHER0qMY5At%n_j=cXtsJ6ygkL|G`V?>xjbA*SE z1!RYscQ4bOi-WoX)Sh`r)g2bLLh2zpqj0OKo`Ni^{HZ+ew||XEXuLYSl}LOOjBx@o zatag#VbuH8_E{&uu5r$=@Z7>o`CQ34#|fU3Ck*&3E9R9- zspHt~XU;W)&N((V>qn@l+1k<#pSWNcmx9$ATB5Dg&Mx}Ou1jSPz>7~{}zt(0R|v?ss6jytU_ zhFKQ{JK&|FqB5g6pM8cOta}6$p<0-nrDg_GXJ*M@M3HS;U-d(!?z}68Ty!(49ikx4 zmG)X=7h8nDSZ{iz!KPpe6tT^VlMDk(UyJ_LsLB7UQLEz^clLnIgcrh7@j2a{+16me zzawnTrZ>}(lspa&fkaS0w-K^jN=$+zNrZv$YKzxR1Pz0VkM9-*(J|0`P6o~yUaE6s zBL~@SQ_AC3Ny`ads#|9C&tD}?8G@Fw%W+%U%h4bn)HPpjpPV zw1EuvM7rd}UTfM~idL~b|G=v<<=Gndu{bU*vn`Q#(`$)%7Fw|m^o5c5nssF^lGoT5 z0gFHNaStgqU&sV{)kw)1GTwkv8){kCeSvwe29A@u=8?KSdjSx#CR)?cinzq&He8@B zC!a^-b@h6imUr29u6ZTRd8OmA)fO#)*$FH!hm_8yMNlxl{_{#F6-9itvIMyjQ?yC- zebi;+Q-n3r!^O@$F>U#(8eQz8s*Uel%C1||3KdRQ6d7no{RyVD-M6aFN;uLnzqk}e z;wjZ#*T4i^92{+A6U+F_K057k@+_s7-m7G>usGN}k$J8KLvfjB>nF7Ych5I2o5V5Q zlN4?au$4_1Bwrep8SBROAJ6JZ?r+M~nZ7bw;d}iTG_U-$sF=FKSH~4|y7|Zf!o&D@ z-P_NNKQF~CPYjOBR>X3$wHYwdvg4jy4h+4?JRWI5tnf@u)NhQW#L}jhjg{408EIag zYvfJEs|_b9W1_MZG(cBw--rxSC$y6&HE~jJKeQyuLd2-(3B_Bt1UO`=M}$}Km_egJ z_M&U{_CqjasU+o!@AuucaHJ_VNf%t|pf0-810XkAU!hfIT;{S4L*F^uEwiizEk&)_ z-ZRLsLX37@+^e6&C5tMbN2pBI&R_$NOL81l9j9|TJa3I;nTEd1naEwofn6}oj^Go| zY-`(0sjpaECcvmktzga@PQSmYGHw=Q*QP9E8Jc;szSWyvD9;31hq@ZKJQYVZx78)a zQB&in)Y-L&wayA^v^}oLoD1uj{wtP|xgNWUuL-nFM8D?6&WGa0bDv?61Ad}Krp$Uw zLKCdDvLOvmI_(+u)Z5 zN{#75mZ@qM(MeoZqaxWnF8_WlLj0#WWMdD*;zZ6(9LzXLX9gQF^z^CSW#j1Rs(gxg z7aBQ>24#o7bSwI-G#Fg{gih-x`9OC}bqt zK}lev?&lU*>6{?C!G~GEAO*rca`ZsiX4)97OI-c{@9WF}^vh`8L=H;8NJqm6H{L+_ zEM$hFH(b{ZZ7oV+Y?O|583a(S-T1qfO)4$hg^p>D({168S5xj?DU3}o{=w%!kay08 z(KYQ22bZjcwaVyloVyv?t{iympMJ*>eJJQ9d}Wn4>~DvunNzV=(l@_~As1BLg~x{x zJov*7wJL9aT>>M+vlBlEq(0{!?1Jb34{(YhhxXqA!3X0-bM-Zc+G2x!{03q-himfA z;t*_7lvta3Q_|T${qoo9oTXOPS{Pw@`Vi{`_KUWr=BM_IBgsp|<1=E*Kx8g9(Mk)_ zrQIDZ0b3J;gUVoL1w(mfF4obTOdT=tEER$Vu9_apM*NnlvAsr?9YHWVB7q5`4A)6rrp@3wjAxfB9oFB$ zK@+-k;2qg`%V-+QLC2ftC-{=FwIbR?h&n{D=Ig*4eg=L+o_?eosj-b z>5xY?uI!erWH#>#<=I*W0TJdB7&;?9p5E>O&k zrKj;WQ)@R@XI0^FYYC&V-fmwCb0?8{tC(y)FMNu4S7i6BTb7MCD)ILf+OeTa?lZDG z(%d{Xb**RX^kkvz8%A@ zD%FWV9u1mpKBnH2FP-6z?N&`rW|_z+;KA7RX5?AcLh!*3kPCj-4>}_D36kR8 z9j%RvwS^9WyfQq@6x0Ob4`Uh$M70xUk36I2v{!_o3AkUI-;$Zma}vWFH&b5lF5@*q4<7VvB%kBlfQ5n4xh zpADXi)~RY++ESLj1iX@*D0a@hh(%mnYb|r_$E|Bc;9${6{s_s}jV8f5 zvCuzJC*#NjpfGi{^^4|n&!68tBi7OP=E6hV?%f!@^m34*fEM^On^d7&(f#Ax(I7la zcVDtPfto%VrNTu=DRakZ&9$cGg_<2LeI0p+MvcsPcJgdkfk z@{@<{m9vIXS?|FyGnG-${|m%~*Nj4Js)?UZNSewyk(|vBYH6K0^Szl{Y z%D!^!`#zrd!Og!j3$yI~^qCueN2vUi=#!RiU1*e)u9q}!TD+;MnjETf(T|NWl$G(; z{odc@@n4uU4dm3E<M%Gv52hnZ4l6i|8YR8I@^|%Xcn^1#mpasN3ca z_Db{1l<7XF7frMOXb`~m^aE*IYJHl}Z9Djzs*UFTg$9f0lH{vpuPZd*7i_%V&!nvC zD*3(m`;xe>qHvMva+Jv7{z5r zQN^*Y1Ubv$N>_2R@AZ&7XDPZ-LmcH}1SZ{Orfm%7hIl8fu^Wp`8d z9y=C$y8?3!*HiKN=URi=_wWXW_>YUPhEt72tsi}EdGb=aNM$WZkNu&CCvVvyyvq?i z&zbbZZO!c$l|98A?!LrjbM?H~CiH@}dw%X=TZmYR!>x#gsAHL2g7QSl$1f|%lzXh} zm)TD5}XdWH6;0aP}! z|Nisyf_;PFpU?psX7f6wkB*L>{+d;n12Ovo9Yf<5$wzUnvt_CI3xn<7N7b)?$i$Yd zlnewuFIjs!?Ot|RA(*+iG#v*Tq1x>GSM%qw)Q7w8-CNf)00`taWjVR4&fNUucZ)Gf z8Si0nhZCM&Uc@O7aPR%x#iw%g?*p#__5(AWaxAU9b<-ce7b$LFL8--MmHUI~zk4#N z^Hk_A;0NF$mhB)U9n&u_FK_rHD@#yHN@{uvJVNg#mEm8>w~-)8Y8YIIwV=B5;Y2JR3i_y?I$fuiu|W7|ad9f#yT=`irMi88u(VtYY4LZ#W|D%#F`hkq zfOJA4X5VSm*l#7Xeui9I=M?nzrJ)pP?y?X4eydfd3 z6zW}88Bf;&av!6UFi=QUP}~xyAhp*HrUAY_2;&3}w1hUBU33QfE5G_dxNYwnVS4yh zF0M79`=395zN*wpDXc$n`&ZMRN5`cS!sZv8-;bU7uzBp7>pewT!NL+Tk@HUMWnE8j zY#il{8DDnjhKuR_@^;{_ji2`Hn-v@fvx~v zPQ_bx%oFP@+xOa=i|dfB2V&`;4t=S%IOykXOyTMJ;N<(dz3k@K8lIlDw(1Jwwzw7+5!8@n;113miu8^bCU!{g!SQxpKHk=Ex2P zhO0PA&C6Z+(&hG7hI3W5+r41Y)qpmYl%n2|96P3=9@|#=R?QD~s0ewocXT8fTQFkK&;J;LA>TC-1hs z{c;9#hsWKA8Xn)}u+M6C6dqjjJ0Zy0Y{6BrU)6QnMfzeaO(n1uBdQ`|z2oiWCkIq( zbBk??qV^u=tW(oc5B4a)=^JLMhKX|fZp-f$ymy_e=YDAa%QW@zr;|=vPvk38u4dRO zsA8FZpBCk)vZdVK9hzWiTo_$?bh|o>fIC;7So>1vy&Uc}$F-Z~?BR#DzI{Kr8jz(W zMvQbyqGNv48h&2y>ouEA=;q_o}d`loY0E0pcb_4DxFEywa}nYIZx zi?YTn)r)IbdGw0bFx(Yjk0S5;b7kd9Rz*ZKRiwW>Fj&~8^wLJkic;f`IbHE#S8b-m-d5l z+(t)7b7Y--2<W|A0U3H+Vc6FqsFK znSCG;ZV09-!w9v_Cl2nbQj{-K_41Mi8}J;ADSTgC+?`;Up?$yFpUD-b*>K2+&ImwG z(>bRp%@T>kmKR_ma`tN)u6}TlEgQlKta|brCf7#6ij-~Iz_xX(tceY%1JDOrWW3IfCkiyCJK&#omo9M%jI&8gbS|w-cb$)nw6t||!>6RTb=76A_tG7Gz4MKjP&ytXYGbB$ ztL38F_ue9kL8FML>(P<<9b9L1Gb)Q7x#;U|X1#E#5HCZwpKZ$A!@w}ZrF(p6AWNS6 z&eF75WIKH)4U=Vvle_XX0dG^zDe?O$mHOgX~WfnOeS$D#2*_!QV-jY#b zD`xfZXnt+1g{b_=;-%O=Z6`K<8EW;W5$Yn9cV}gaFU{**m60` zrPRALM?HYh5rt{I8gMUmWtsd(d=nj??)=E?5rztd9oviV%OAlE5S=)6jcw&j$E5X( z5|{mg#5&wG?d)1tT@@9*O0sA}13fPfzkXH!K|ruatC7~jeeBq#goLHpV~}Lzt&(hM zx(hv>-0x?U_iu&8+l>dO!WXlP{){hPq(p(U_>Gu7H_Fs_$vU)t!8P#OQ_usWBNzs5 zJLfQ?BklmRDVF2VKqOj}DUt`bZUWbZD>NVb5B_#7YX^-Q)SB)sr@1v#HJwLmu8ntu zZyc2eq+ui3nXZ?&+sMesnItSM+?i|62$6JGm9EM6-Lkgk2M3xr*g{yKtY{|01gq}n z;0Oe}$YIc+U9+;{JDtb<-sp!#-fsv4qiDFhnMX>r;&K zn{%7su>OpLR~1N=@1Aqa5)$pcp{sjXhCFfHpwzV`p`pHh54dd4gCTccY8z=S*Du9n z_w&uj##Tj`g6iUmlT)5p!p1vn;Nj({y|B2v%yMiOTFB!8fq7ixd4+|n0D*FUm@F?n zq|Qei4=g$mtERN-CHz{Q=Z?*@(-BJxJ~~Hg_J3S5$-ix1rW7pbkd{4ZDtw;D@toM( zhfmxYo>u$6+|41}~6>sI6mMoGHAe?CBxahYlB_8Vxmr%Nzx+$v6(vOsh*BRShjcC#t%RbC(W|ez5Cqdh3g=F#XJA zVnWY)E`D@$$KF5a`7Op6-uIXu{UB#x&{6c|(=1&@?Ube2#f1FQP32!=_ff9#2UQid zvD3+EnM*q~MI1AIIXp;aaqfFUN+#?bwK;#HVX~uU(K>nlmK;xHVVllS|KhE*Tb2%H z1$r6HhlI;GpWHX3m2D?2KR><$>U-V+qzhVrRZ{Y?I}<;2%Kf$+fafOLM%g-0-C(@%i_uz3pdAq3T|7KCf5>lI-A zeMqFRO`Umi^bS}XJHbwyp@PG6^Yha~${~v&>)+>Y2joSvqF5qc0?EC%(0&V;K$(P? z4%}KE8yb2HJnKEk&=m7rIS0u$9AvOw-vbQd`t|FTkWW51zPu858Wy{B5YWOU^MF*u zcMuXZd9EsL97>BTZ5F~{r3?g3ESje1Y^|5U4A!bD$l4Qvw7@fmGL4T981?q`5uqc< zZ@sX8V=2t;XJvf>ko0{Jh&qS5i-?H){@f%}1tI#r7>Wdys?I%Co|cyODk8!MY+O}V zHdIV4jJg1O?oo(ZXKaN6S$cVPpcde5wcEERa;49oKL{Jz6*;-umnV;IzqJ1xmg=dg zqmu|>2-?`?8AAY5wm?h@--(d?fgSxEz#Xr{v_mn6k)3?ZwR7)4D)kJ~>aKinu-?na zZ4m|i$In>lBY<`1;U^s>z>9bfQgvS`iO=z%m8_H8(i!YDHac4E!G?Csju-y^PcA2T ze`&sGWUHdG4;+W`H}U_}1M+S^?Bi27-SQf12To5-l0E8$B)>8mIbdW2BG{fDj^uUx z-Zs%}W^uRDeG2b?>$FR69IfetCou8v%%?JL_Qm-cWZT{A0XJK4e2Yv6WQXWzY= zXPu<%H~fVwUYQwEuI?v#OC>OOyt?ii{WN<%eD!m&`GkM2z(RyT^&w?mUqT)ab6 z-;A)BA+S)`b=T=9l|+;FdyiYzH(bQie;#6-m~(Afd()l6!Ljs1@5%+~>hZA;-s$u# zUu4%mn-kvAie?N-0}N?zgP>99iT;t|BD3@UedSi_L|^j#?coynL;l2U`BQhQJz6<} z7&9dz-vw^LmTk3GCS)ZlyxkkrBd23qQKr?O$rWWV*|T!J@keMu5!|G_j-r4rt1nzs zyM3UKm^8DxnCIpe(`f%zp-KA5Q)w>AxX(^?qX)NcEi7m*Awo(*>4aa?1xRl3_^42z zr7+9bZ{3-Uh5Xm2q61P-$~F>4CK+nKUp;QvF!GfW2;zw zfd1|cRz8MG`(uWHv@{1~$~+4W4u0|D@8>Sz6CBI{6Ig;Fqfg)P+O_Qvz`}En2OzU! zM~~8h_ectCmUa+d85R*Cu~_6xWV7ildLP_shvSuV41%PNj~q*K_brf(Q4~w`cNgOnnttE~o$?tdS>O zWhRoU3$|2Hpf?xT4yfEz06gK%%~R15PP^f2@+}29$HvBdVEEb5vu7v$YNaRrlhn_m z@Jy@T!UNF2?_`sir|~-vz22Yi@9~FQ#B)5J_xrh@bDitDj`?b*WvO=B zW#ru_sIWyP`<(*6G>^tFkKt0?3d z;p#{K-ZdRFNmn1uc&f8vpqKBLrx;-gYplzYvAuZ=$9(LTW4fJ}4eO`v@&{&SF&&rJ z_Pix>b8ieYdUcB94J8z}#k zcKjcz`2j(Nart6sXI#sTn+tTMR|gm9^#W%ikMFM!mPP1Oa$IzlSfe{s_SGAhw7T=u zK`-~1i>+NUQ{-bFH+r6RZ2WtZ__eaKnn4tLNXJQHsMO^A@ZlIapXgt)j)^Ds+wY_@ z-i_+{8~BtnzE84~wlL*u3o;nPd)Tgf!1+Ml=y0jF7pm`l>uV4hN<75SQ?>q4NoK+} zPTT5tlHW>Zn2+vJeK|kylu#)l-WoHi@kH$J%!1CGyuq5(2Chm@F~gn*tX69M;ZU2g z&k1E{bamr|o~>J5^1w%!I(PbAggz$MYkqNO$NP`Z(u|8|N#DJtHLt`i_xgyQp6`QB z2|DFDd>(&pRZsoxop;0q^zD;|{G}MdHP-$0o1?x4+=NGmLRC#0id=AX%7S&VjjG(2 zk=Fp08-vpT3p?vsioHw&%i?QS*OzD*L!tKY;du!O3DO6oq`UN>2zQ4uIQGC@q8E0= z8WQxi?ogvjJV&Yw(l={OEQr}n+7Y=Bl4q0IGR##V#MJut9zm{R0cO;@!I9o{! zU`U4v^T`{aae?dvo!SxWHt+`d0ku06(XV3P1XsO+?FqZA)AnbzER?Dsb~c9j%P?f^ zcZ)LseGhzn#lbXfjxXe}7tFh#gV94%U*DIgC>I_#H#aVWI(o3`fpX)^z$qM^s<{II zm{alVwSx--8w_qI%-3A`PDMuc90cT2`I@0`UT5u-Vqt)-p{#|)H8{+zcVV2gq=p6s zSg=dOu7@k%W5>?Q&(AONEs_?5umb46UOftt4dOxg%}p#U$p(^AI_!h1_BnZ^7b!(b zHstc9^@32$*BS`P114q$F}B52yYe(S@s?j76zhgQKU}}Le%*yz6D52Td%mBBWa(P_ zFf%1f?R*i2Br#DbpkxTF z;ZHZrWokzj>ag=W3i6!od^i6ZeMv3e=v_><<7;|S#OyR>^V)#X&Z>TMf_rRETunS! z9?w#0xj=I3>q=+VjhlUal|kj1KNFb7PP+Tw`}~QtDKIxmqR`RNxAH$enOR-TDg0@K zu1|b9@&p4V_vD-m&+FD{i^vNMX)l)}_oI`JhaOC&9NJ2S_U5a0B%_ynzJ6MZ4gG3X zD@^|Z$2N8TFd(i-#rOUB+}2l_z6*z_&*eWA9R6%`>5Mc#3MuZIB^BkN@Ij;RuFuCB zJXQ~q=uxq8sI_IptNvWqxNGytX(q+1m9-M!s!ytNdcNU7=dr8GC$lN(Zt1p`*^HDq zK1mat^e_|3P>GP%jc0l%-@fL)s&y$t`8cC>*VA*|Af%Oqc{Bfae#+?T(ra); zPoS2-(<&B>0fVyv&7y*o5kyM_^g<+H&<*{l%vH8|6e>YE2$JO)aAQL@AdHgfZoCV5 zm=t(`CBsNnO&%T|Dt0OiWl|GRW7B^bk+5XSs`< zjUZw8wihX%E+;SF1c24d%F39>OmJ{jR$H44y@>M#csI&pK0ZGDFy4g#$~>ol5V?HK zxyr@FW^m94=Q!(KNM{3+l9<3Xf!(-j&y0@-ydEW{q4ZWBYmZ=x0DJs996mMZp^)EO zSEJ<-06y-P&GV6QfnVKlz;1%MuuJFuGi2xNxWjPBwf}c|&Vq@2F(+M^47f11Cnt_7 zUx_j_tzdtdGhv@q!FYwxhORe0D;IzFjgd?&`=Mjav9ejIcH{$-hKL32ahv(#z|lvN zwWQ~FWw5Hd_}JtEwV2KqZe2bLx$tHA$^rznKUg8!IE;I6`1v5^H-x(`Yb8uMk!92G*tp=!T{YXPwy)PGcNLveL-B3zVO1BuwW~l+ z`Y#lUXSFJDt7{INNol!l6FENbGE`=HP3lhn%Jzpsw^R!QwQDo{UZxiJ@MXsR^BVe{ zuDwHVA4z=Z%kCYbvKa{{LZ`WHqtk|+U9e+cn|_@yAROJbg~}^Zj_|p zrcb0-ybo@(Ub#Yn-skm6*%hw+UG z*(-3k!Gl`18O(KTTr_hj!Nq`ZFz;~yIA90I794}m<)xc9DQUyn&LA%^bGi%DP%d-? zUqHdIA3l>dh+*~(nnQ5#12;KOtN8hT8*;k3s70KFrEBKilfMh^_N33#Qajbj`2as1NW4 za{K43E2(H}D|gQaC*FCSiYvO&FPZ<&*!c|}?uoqI-;Zb20kVOYDQf=2|8Mh76d5UD zI?eymOfBe{{^@w};zB+&yuVNF!Tf=OwV5ja#sEy_NCCY&E=%ew5^g$&;x%FZ>!k^d zJ^I?(#G&bEJuWeKd;c{D07FJbqFFME-2}@!7wQ%~lzbqCt-+koS(Wg-A3w28@pJ+Q zrF$q&{XU7uGp6&3+(*Gr+>zA;k|dvq>}VSbxY~E8R%&Z3wU|a@Fj<7(oc4}>RuLZ+ zWd501J6A#%I`RtbW~sYs)p&fAQRkCF?^iJH^t7xUQwE3CRo6Us#U|^gq=B7vslgOR zU0M4TK@RP;(~9vL3a#xKMQH*X5m}+*0hiU2!X3?+QmwoSpJWRA7!psEqHN!dJJY|;6;9{bStEaX>PfvVR#{TQC}?+!q1t!9x_EO-t*mN%Z49#f z`*If=nulH*R7@XMDmncFltztII-yC1o=0 z^Jg;fMm+DjKEAZ$GyzZe|vPXm9 zDL_JuJcYSHxCwnrdppwCmPK~dQt)phFhy+wM~$^*c$W^dEx7+W#A*u8A_>h?uaV+W z00|7UFv+Hr6a4TSEZ2PJn%tfOAeS{tSV;6o%8^O%LO$=^{)G#ux9Ho$5kWZ<%ooO^ zL_^`3OXV5FEA<0?0(Zav_Q+>&VGNsm4c{Ehdr}rWTJ{LPa~5>qOF!;qW1N3Ie0Z;B ztHUH?CZVmbHtM=uEGql^b@KNWSjfpQT5XvcVT9i(0&Pc+-8trTkrz{mqlVdzN9vq~#UsA6OwqWdS3E3lOul#J_ z_FiWxooDxpc7DaK$%Cmk&n)HAxc&NRuB(Q9Kggit=-u$%bTT+7SXzuoqQ_8?5RADl z_e6=BT4!wk>B%tkYKp~pYCM^$JcrmYDjaeuMs{~x-RvF$x1+R!(6L8=;WX_!?%mlV zQ4Y;&cOg3kTZ0PxJ}GMFpjI5Ao$T77k}yD!(UsdhF_rzimhJ{)bDm*De?Ks|jk|1X z81CZSAdRLm&qX_oDXkv%GU?Q0RTRq*MVv5(DyNjsiiXFsXD{PJpN^UF$CE)mDH-ELoBTbkPkGfr_jD&EQN z^71r{b-kBw-dqeB#6dmCcrnIkP_6mQYhT2Vud?9|@#V_`wFa*$Rc_~`rA_X+Pv7yo z8vR)_rVaPq48)5!_HVL!?)DoO_gwM*c87J7K-ndt*RWefsv zsM+mOYiqtA^;s?0KOvW{t{8WV5XwP&AslRf7!d9agHW$>yR3O`v?u=~XOuN3h9b`! z{96qcD;Tc<%Pw3Bj9P~_2Ob$n77jrH7cBC&)mTzf6B*_a0X?Ie=#il6YEiKNe`Rp; zCkB#dJOFk@g|CsS4Vm1EbAVaEc{=4pfax0shPSu(>at4W z?WG+p<+I)P6X9n_l1wiyva8b~VPvbNwe?@@0;o6g=Z>}QbzIXI9>(A97_=N-Z3&l} z{|p)BAi))OM{MUSWh)bJpEotpA#P*c#@=ylg)%AAa$l!7`d z>v~D@xUzuzc5ST_mH9j-{c-k?b(VcL$YOtfh<25(vF)%do^2|Cn37xgbLMdCu8y#& z^@S&R7P2y4#?ZlS`q}Blgo+=FH1Xn!#0`!kSsq?W#o|?{%eDf7c z=kF8I`a*H}rAzHO#HP$5!e2Gr_QtGNk4}B9PoB@pE-M`lmP<;N5?%D#*6n;Jlv%%4 zoc!a(4TJK!wI1I%!u-%h+zzJ{n}~Aj%J5Ek2eut|=j~?F?*CtO!D*n zwC13!OzFCAeOx+0R;nGi)W>SGkgsExl9g2!ctu;#N>jkwCmO|Q9kD;zFt{b3NQ9;K|>52{cx_iN_vc{R_^95o?#Ft{a_!06z74Q&Q zS*7BS&swCxS*c(THyUj>UeSoF9uh2a-`xBvjG+lxO!OkMEFDdrfR-iU?aJjVbAE*K>csHy z@OrQ0*(uQf5y5af-d=elw;I%54f;or4>$?H_2n|D3p-4_hU=__`tjo{DD;HaVC;Ax zcmdAD)E`{|uyqK?VAb(Glfe*BHEZ`q9*C8-Gi>JNC^lDwk>MS)gkqK3ed98FI01GoZ;{< zwqPyeXO)-+A{en*jkB! z^ow#mkKV*FZt+ZgyQa=)99`d2;EL1S_Vpx2`3p`8L0^D2=Dkgio$rmGdlkuz!7&Dt zr_vi+`ucb^$F6VlkJ=>?+qdQdZS;Gi63uFMHPwUG$s;(gf3{%i=*%(S-3O56B8U}x zs%uWNuA+VTIJ-)n4sgkv-n{)OaBYFJ@W}tO=A<9fQ-P00whO}}(S!^lfT-=2=8C$m z(hy(=$o)I@gGG8^l+7yk#*MgdFm9WSA7a7=%Y}2;MD)QhN`V2HtJ9n|fHLHxu?HzD zl+Wm|U%&dd{`8UXVgxJpa71B z(Zs@*R%5Qb0i3;PzI@HGf|Y*-n&SW%MHj=|rJ#Cbl;GAcUk~o22d&p zCioJ=U;&w@Pt*H+;ffWsV?1M%T)fdX<9-)qfQ`fK>i)JZnf)baQ)~eCc`_8Ja{y=j9+Uo14JrG+`Cgq`~X&a5$jZaA6R{uD8rC%z1 z0n>Za+RqLvCk>%r-9PtG{r1=tT^F`HYfHe8vvUSsK5lloRq>Tf?0dVj-%2I>Kc;w+ zm(BMx@49kxNXJv3K?6uul*RrwnBgBs4bZ_%zr#%a;Mdl)d2_(zo?Xgx1Hy`OX$C-^AVfg*ce={Iz`vBUlJO|1VAFc{a{O{_-WV8*l*lZ-UjOGLG?!q zEVHw>Mi$(Mzef68io8(v2OQ|`_xJzS`PQ%Lb67D43ft%T-W&R?;M+q6;|DI5mMZl= zf?_I1LYXrgB2-_qVIHCwMgPrqxTs zrWdL@e{&0M4Lja$05CGOdYzIo4B(u;trK*N2v-aX;0#{?6F7a4u`_P-(wIf@KyeOv#@Xv9=W*;P%F?pnKt| z=qnAj##OhsHcxHB7~>{yo0r%6f>)M<^&DmV!A?$ok3=`VNfO=gMYU=|NtQcPhAkQ8 zM9JeFt^#!2!|3|O$YSq=i6`Tw$+`A*~P%%H(%j%>vi%vf2&Uw z+ml@`@|D@EZ5$ubt{&|7U9DIJEe$6H$3@Mfr?l9wEe=O`yxs7#Pv2vQwptE7Vu?0v z_-&~3cXq$BqJ({D$&)U}W%!V|-C<2Dxl5v6k*D>!AB_nooeP___c0Fr=?>mOd%?NF(pKM8x>TzRc|kH;MLtUpY6s3{yBKRoD5VjJrO zh%=VYh?9uX`Gme9U^{%G+G(r%Ms-42Z~Vo;#UwisTSi#H7jM+vb1mDJy#EYm5tGUL z%`;@hWvqiwNPvGoX}hK7L1TD6xOk7>_1QRa$AgP{EddHYG@>DeZyp}M?Tki0E%Yz>s4f(ci1cZ>e0#>tQXF~=yjt3+6t;dd7Xz@r1}{d3hPK8C!2K$@;v|JJv9$C5x*-C9BfB_)g*=odLnufGd8U32}CLTh5$0nMOrOe;j*=Ye= z;$+pUPzS{eJ%b>6>x;$^Y|U4Y$PF*u+?kx5)G9VU54J(r+3%v>k-4}4ALkypHS0|P zdGF2pU*22GD7JYRcxOD)UVjj0`0-)y8xtj$-oyI>hOuF0XRDa(t*&L|-~3di9RcTc#3|aZ zkI%^z+^PFFx19nS4P_oi6A9XBvarrE7U)(I=aw#vL$@dvNb%^s2@DkAR!Y`Wi2+$x z?cCkJ+8oQg{vAq4XDasNmgn}j%kdI^)No81e&6hrXBaFHdjW)v*9~{vXu_yq1uK=^ zcq%WX@6FF>HFyNG>~2|gNStnKjYIAL9WRtWKryX}Vji)#h3V(o0+}zjklWf`?}r3Z zTRVdD(fV9FlY*Mw@$^2}x%HDDdq*g6L{G02E?xo?VB7Jm7p=yv8-7XQ`;*l#9)C%B z6HoubA4Q2Jc?TLCNHY0n0WAb19PEILC_XROlB*+3Cl||X6M`J(>h7xj2*6)*TkC8k zmY6pg%)9#fB02K1fmlF6IWPlEU=Rfpz>(nhH>Q~3z15%POG0%^9MHFDOQKqDc=O@M zZ_x&7Ah^DN5WF zq(~bhVDJT@m7jlgA(=h~M>Y6^N<_$NdEbc}Jih$)*Pbj?rBBxD2~ibE2BG`!*V;ba zp9!X?x&Tz4hv~Z&u4$-!AV9v4s=mLqTAvB0g<$!zzQueBjjn|L1qje5Ro;TL?6S

3wT7*U!;jbY^X;5I@J*6vi85O|e$R}WO(vmXMdQ4Nk)x_12JT*1+@rAA_&%6Ct{ zx7aIsJiQujQ*jQ(x7>X&bF3hAFr=`#>4b^(DOsL4ccGD&n{N~cIl+=@orcSZBxjFm z?p-1+GHx~0g@Tsp)SVG!f9Q;hO+5M9VeT%UgE0+saHnT3A?#jp3bYk7Nz3 za4RH@jGX*du$>xVI^9M{2(nFLl$?TB(OQObSVK#Q(QnxNreDqj#Db6^Wh1EW=H4`F zeT9{|vc0KlxY(1)?=U-bmD9>!*{rG1*M*?|)JMeh+pD|A#>~L-fX1g9WMTO_55IQT z1cg#}8Qm&(WEsKMdz$2~1;6sEJEZO8n7AjPudj=Qt{S&jf++~(U6-QnL+uv|gsr)z zK*zD`{QPwXdt>qMH3?C`!**XvclSOpVna<27qh&0ktYic@0uElgTLPy$!0@*(C>%2 zP)uSBsSxsW_u~B2_uKxIr zprr)|{16EHmu}qXOYQ}tCd5&LPe5q`#1T7U2(o8W7#{paKAkV_TUizGz6fZ(msu&D zonlq2z`{Uy+`uSrK$_p9&LRwm3n+BhjDQqEM4nw1Eua3G)2ejT)bk(r+a(-OcI0mm z+}+ah<9d4@u(Lr)_6mITSa09fMW|o2zyyH_d$TaM`TLUdw>_BdEdyj2sBeZBHb~i6 zmaP%ec)1;uqhtJ3F8B!;bjJx2CnqN-#OHb8&m^SKI*tg?xX_`Iu9uCD3%b#;@$hn}hVd78PK3o$An z*OB|pR5d*?R{a;Fq&=Ka((@1>NCec5jyWjQy?ZXh6@ydg%Dt3nA$jh;E zCIVBl>yO{W>DUal4?S8c8lR{pbX`bGYoNMu;kmz3oGB~ccLu+c?R&A`hTwJh`lXYz@fjDQIkN?Xl7gsm_(h{#>H!hcuzMpq=~Hl#(t;s$bP7G64Y^UG$fmxdt5=Y#O* z4JFy6t4UNtj~^LHJiaRzFC>WOIg&6@)zgj;1<9P}gMgoLA5GE9-X45x`Sd@ntA9eh zc`9~3sm|qsOad z@kuA5uB%@7Bf=P@d#zc)@8c_8UnhbQi&m6>w@P424QnQt7z!229X?Seb^KQ2L!%xjGr95i`MNMASHC@=-B9_ zoSc-8zy0?%ur$4nHl@V+` zS9om~{TmO!Sn$KW)fO@Js;0&gmC&i+!u2d{%HBYsfdoX?5yFP}$jE0v5C}bsrrUdaZD3j!2F50Hpbo9@L6=D- z0VfR}=n_9a>4OZ#XXS2V;s$I0K|mVE0$Aw1d-uw!V1^0sZtk>boFOp@vK-OIVBrad z2NGb1_H1-iyLt~gF%%ROP*z{LfQ##~y6EBjQystL)IURPtDU{$R{wy%ffOF9NLiQVNbZk;t9EABfJ*} zfQx9t!skqZRG^r6`y$Ywo~5ymU*>!s8%xvu@g_bF5q;6Xhi_-yT!guZbW@D0*nssr zTwY!PjJQ2t2f4IFo2l={xOJ1XnN*H*;edX zSUc9r*EAQ{Gr`Omn0@_N4uNbap&JFSof(FuYsMrbgu#(U`e0XC{U5at%?T`^JD!zP zPs7-XP1J}fS9zLanJJBZD2{mKvE1mCgtRmw)a%-^gUkgU7Z(;W#@rO?zOXHA;HI3a zcy^=V%Z`vcMDc%D2#0VbFApe;nEY;&_hrhm01kr}NYU8HNF&ID1=g}r_H%6<-5)6l zflivkkzn)Kz~E`<3ZsBrSYBTA!SB?Ia&qR!rUeJ?H&6ELWSLp#p1!8P2*_{n1v}lk znsXcV5|m%eWG@NylXE;=q2^-{)in3wy3hcAx(Avuk@8m#=9t*$ex}Dj<=M<{{zl@q z*uv({J64_^Cf+)l2nhMDLk7J;nVILJqYLbkaD=gr(Df<}rB<9tkZh zZ7=Ra|L9Ax&DfFV&{U+i_q&4=bnC%PPBfP2&?{NH&|JI z3&NV#qds zz4zGMaq`MrluNXeknGQDEsiKn%q}EoUP=1ansuf4Lila~L{PF12h`UT(#W`u%W<`~ zCY@gpS)L51+Q%66ztuWg>$w-Kygn9!TOIRzzRn$;`xlx**dhH}`<#(@kdHqz@clv( z*?dk%Z)y$S%_ahrLyFor)AY8h})@T`m%nUiaBboDcWRepN5P&K#|7=hMA@@ADIprE9#3;B%W~BpJ1}o@*nSBe$gWK z_gnw8K9djG@oZ7z0rNiFO!A&^Iu%0=AfNM&J{D`&X36v z(N7VbZ>}A+@t>SaWpm-`%tMR1yCW@?BLWb4@+lA9_;` z>*JwNsoTd5@m@Fw2K(#)B*yUj`TAX@__h%W3-yMn^lYzR;?G%;+S+I#d&6t$52%?z z-N`*sI6Ifx63W{)m*DoYyrKs_Yml%wT94f#W|iHuaY#UYFH9oV_w%7qPVi*xy-hyk zxOfW*uFW7Mvp+cs)vER6n|EAX)Q1N50oYhLY|ScwVSz>8p^x`FAtzH|BnVS+{f_uo zAL~q5m`W66Mk?$ZJ8EB@Qctv{=}#r;U~8Ffk($TY;DlA3~s4 z6auQlvse%ycbP1x-VZAAbN^Wod|nzW=S7B{)ND54Um$QksZ}yHWt#-UVtO%83eVa} z3noBhWPw~c5*-=}&Y=iXN&0dkhNXFZKIQJhCe^*K@^35liyLazY8@z~1G6?wnTg7_8 z!234-u?*H&Cqgjc$|Nh@r@Qn(eg>LR7^!zZsdgm(_Ha8nAvhy&f~NOitjEd~A1 zE30nIzo~0qHSw>EAxUoV|g(%8%soo^Cr|v6GZI-1eJ1?;p%5#6H zrUwA(@KcIt%8>X)%VD$N@Ry0i++U!etOELkr>cd@U0cm) zYKNmfoDKA{J@3)4>MP1gIB%5!GbagH}{8EGos5VFr#}Q2+DM{S(6s&YEMVB3mWu~N%W~xYNnLSM> zQqM*)FoTa%fgsQ8vHI}t@b8&G&8t!Yh{q%f>C}bEdBF#Xq()!L$ zd8B#f>H>OTg4wybGx=f!jNY#B{(Bwxp9lb&=>u#Qevdy-2$;m_p#4d4P0ic^ z_m%weL?Pa@*P-UcPa6g}f*nLp?w~Rtuo;6ektL0V1qGfXU>MjO&H2c%SU|#}8=CU` zFx&UV%PMghEa=)B|LXw8i+NG?yu{nv+-!i78fXIz_fcb!kn({lIx;TK>d_P=wV>6| zeYZkKh>eNJuOaEoT!RcTR=|R+u=euuC2(bc(E$qMbv|PSzA)H>M#8s7E*ptNp5>sI z^4=RUEBma-)+}){C-G;)0ePr0rDDp4Hmx3$H!Lq|HE+-vdu<0jLWGGJ_C zp%nFRVttuZUjM#$`S1g5&(pXa%GV+{-X%I7O{`zp+;mxaw*~Ikxfai*HU`Scla3o zkp&^1zXlfLnG!WTB%~35{Ch~|+5>IiGYzWKuh)IJU!LJa>A?g>*ozMITma3-=RlsH zbh5)ZDQ;mIMUT(wQ&N+;-;! zdZ>RxLxZw1u$6u0=jRQ#d&1rs|i|2N^1YZiO%-nSxk^& zaYZ4|fLxCR1ZXy6>UB$^AE@6&mK?v@7zNcJ3vVGU3UZYIKKoBq6G?Mm0yT|Tvv};S zX1V$irLad=p5EHoCFZfGybL0oa7tm)UAB9V)|<0#V4~W1fooeB_cbIdv&tjq!c*ag zWsa*A-%ZgRY4n@#gGYxD701ks5yAvaHpTh-A0f5?43E}2LG1y$MoM-9Qqr3W$*P7w z)9$;Y*UBT~<5!*E68%352W9qtHWrpYJ3B853k&u1)YKw{jLtsustf-ida(0}k}7{Y zi+ID|AAp$(a0zs^r&6Ovmz_oQo&GHMuk0cgf=+D#Kw1`0S6m@fyjZdop~F6+t5 z%WncH)&v@0z)^5Kya+UWNp*FyNzWB-F8vxBFxF$ae1&EC5c+GIU>su%1H+N^V!qLr zNvxT#9SAE_)H_^ZO%}+zFesjAe^I~2ofM>T32z&XiL4l!05_ytm4VCw=OeJGF| zlslxP6T15P$YEKO<$lrPl2cL&grf5rB9v3xNIz3=x}YPN5Yl5mLn|-=t|Uf9HCy#m zGus7HkW1a(lzA?Z82;yOMW*7aT>tq%ug*>9chr+|o9!(m#O|DF#F-~qDEYd* z(W3CzgX`f8oLH!@PsonC>R&r~RPOAk#|!~a(631|JaoGq^d&a<_xl?av2Q3_*I9D5 zhvNtOWyJ)CWR-VR+f$=H{n2DzXvK=`Q+0dboPUnWX(gacT%L!(SM1abm{J{Q6Szfwd3=jlp#W6B@Z(+bf z@lGG$6(P-Lf2I4s?5qqCgMn^IZfb`FE7E`DODZY#B5{WbwUsuP=94GyOy=5hhl^%7Ubkgn$V~dU$yYyRw&f$*5?z?Z)|WE5Q8KwV&92$9L&$;rxyTY4T3u%YpB zU6AqQv~K*Sz&nq$R`^ZY8bA+APEJPZA_3vCd4I|;Vs_Rza}8W-kj>R^>B7#QT2#cx zrCUJ)t&H+q6(yzCoQJt2^z`HFy&lJhF5ZxzLLY?4209#CHSYXJ$H!yG9wAv)A0asd z;l8S>s$p(dPfr`Ta`1WXn3F~!N7XJ5sC^2$>wh3CmJyVbnmGiUl@T-dhwDt~WLo=BpcM|cRL1G*M)&@ep$18J?tV`zm^3f#9qy{LP zIsQHcQ^Us9ksk>RQC}sOPesGtu2AwVf6L;?4;nEqE(O=Pu;AopEE%jUmerlJBJ_uR zm_g#*{u(^bsw2qAK466nc+w=)u(N+#i};~{Q&yqucG0erIc1MMQjQO%nnJ}tp!rBg zCv<1W85Yoga)0p4facjHR@P?Cg1kIwH8oP$@zRR-z-SE2JDNBPYo6!jaUoneXh^xP z0iRalwHAJO~2&V6%~>xS zqLfJ`MVu6&tN?6;e)J&+B{*=0KySz4IXCH*%a=caNCsX|>A=M>83Y$8toefnDcabA z2XOIVHdaJ;x8nbm3Tf&V)wh{QU5Fu3gCn>NiTofn)j5WN8KjJi;V?>^wC`+n62bd~ zQ^eH;Tn1UvFTs6X_5@or*cC>ICdLp@a2B?dGFAC2G^-mg8&7Pr1hwr`v>kFuY&eDJ%cS&F`*U70c1rG3sNbm#aPHFrRc zJS}pGyTd1W5Ce)E#==JCYFVrI$$w`}7Brwx0t?ncoM|Va+$?)f{8k$&t;w^ zR%+(BFslBCLK0?b6}kd<{6V%XR+q|`J-ju5`Zws0k%`Vw%kbMwemiAt^OAJ9vJ;l~Qnz1dD*~76D|E2yPGmxi$)YQ-rFG zbW#z_tSeW8qkmvvq?1_16M^I*U3y3eLT5y6S!O$dhqOLnm9-lM)28!Qvsz%8(8qrvk9G!fGH|+`U4Tm&GEXU zDR8c!q=oAGbACZV0g+C*EfNVosa{PqN4nTx=!bJwB7nG~$ovNjlnCNKoJiF67uMFo z0`aM5L8J9~0NATgMl%%F9deA1kN-0b=-+<$=uxwA@$f+QfF+_+g91yexVRYl!LRdm z_4UJ-mn{zt4_{HgVhqjHeIEIY<28YXWa^C>?R3VWoSiIwEp3C?s>@7)Sh&9O5EheN z=XTQcl#FUgrxu5jd%h0QMCLkL5gzQA-lUTDxHSHN$CVzr)5w*=^K1%*JR@9@=I^;e z?r9FnuTFT;Nt|fCijU_2t$J$R88J)9LgQAskI;nQGD5`rywE(Rf;BvexSE=08S1zF4zGWNZ%jkRA2eO6C0Rzkmn(; zs5%o+fbxNN13TjmsIWyM02%ibSue_XayOjA%sF3Xm!DvDQj!XW7?Z&Blrv! zlIZgC$AWZd)hB!e$4R(;J0R{jfYw5=IWQ;yDS!HAq$hD=Oa)oY(Q0kHvd?x} zK1BMt;G{by2p|7zb4sse!tMr2LDCxABRvpny?uKyLQksZ_VDrJ5HL@f1uv6tAn~U=+ptN2{VyFAZy8H!x=*I2g3}=NSn5#4ci=Bj)77uj*w;u3x zJF6sNCpHjLk3CjH(xBkrL@l_*hQ;P!S@^?8PSJnS=$Ca&N)+CmcQN#s-W0z#^8L%757|3E-Ic+w>n)U$tW{gRDlK+5 zEt7sbwPyO-V(ba>F(X&M7w%w?*^~biqY$_BTS1T%5p%O>^TAz~6lHc&|9ZNNU(80* z{mGV=mYXN&jb`gxW6Ox_03=L+?frY4H7h9PzjkyKM}z%Mk>vmf^0L*uz&ElDvH&#= zjrkHiAj`6|n}C3Db93|eyfgG`UV@zbED3wx@#G-L?*K>H|LoZ5#{)YJ;3cs@D>xaf zDv`D{(EjfLS_GAzt(C(4`;B14FM0#Zz?QXCjzqC5ufIQ%e73=13E z)t1kbi;|KOX(EA&SMRvSWBU!DDnCWQ;^HOD3>C`5L?KqvfLk4)(d7#IEX$7}j%@vq zWg20S04plrtxyAIz~cb)gG+1(Oluuxn$GKZ3>O&zLnqn&8L~R7KEW(wBj`4jeFkV{ z>Ud&G63Xe=S0W!9ty>dBFl~ouRkiG-qbFHh)AXASuHCE1Bsi|24Bk_}b7?_qaH26m z8?%l#ODZ*l{0>PeyW0o zD)Bll)-yq861Da}TU(6~mF>ZmOlJb1NHtYee<+>))fNDbCqaWBV7}o_XuQFh4fO?* zp`@_Gj(iED+h$R%HuRe2GxN%0L24AAit}aqC zG8Xl-K$Ef#&c1|@KgvL41l=k=&~&XpP>hf(x)VgHVaoID`O9RFq2uHj7G~}0Y}hqU>z{(R7d*zqi*kyI}Lw+iqG{vqxdm6B;kOC6MzGS zBTsid#Spw4VnWa@Ns>2j-#!l^XTQr4?JDievWx+K>RX_yF#+*vSyj?$9ZYzBl&oq$ zdk@}c${uLkoenFfov`PKt~BW{R*z#KraJpY%sksAE8{uH`QW@TIVC z;H`v}`91UucbL*YdPgSh-f44M1Ehy+(PFK9IZmsV503f6YQdgqGEtOce=PZVpN@etGl%)GdZZD7(vy!OsA`JxHDra16o4#-NgNPWK zV7Q(;`T4WmwR-{Xn7x}@dEC{Tkd`8Tk%$FVDGq=*=HHoH+S+DkTSJ8$W?wY73`2qd zbwexQc!(I10JKbGxDo;EEXyI#F#*ZW8uk&08t=-<1qWv%i$Ayuc^K#4s&KjiBas0D z2Kc&xem@qqJF^mY_A9|v2jNHn5cf*?yEV1DMgdPpJm87N=)thy{NUgiux@K^Q(fyuir>ax`{i9XR*?y z7Lz8ZQzTkX7p{JBh+uFEzJlUdX9BuSh&8s{2xFVLx8nXH+wDXaymT!ivCVWMo94LR+81o0^g<`JbrnW;2TnTHUX%a}4IQ*)YBrc%a| zZB8K>qEVWNjm)IPmZ7o}h5xl4opZkL@ALWZ&-r>W&u(O*4 zLKXWa>MRn*+!<=$gYs!wUH|k9YQ_=QFIdcAfrn045X2PP0}lb;FrP}~P1sq^nL`c^ z)OELf*Rzo#P_uxdotyMfQb|7gHJGl7z%v>@dGh?Vi0!<^s5qVT`aytAC}xw8lNOi) zxZ~DPLkwE>xLmGmYuR=sHr(jA%mF)t6zuBiT8AX>Y>OMt)+3bT@_EZmQ8%w0cMCmt zZZvwI>t6+kV;a-jpXYsj^G=SMMC6%IGeU|A@EUSzJ(N*!FZKtf4jqL3_!#ql&={k( zX_F|8xaU9|19@oU1VZ9Nn(Yj8&+fhGk%mn?gyx5AGyD&-%^A6N|BHo@GgV>KgCmbh zp@^~;TRzR-LOVBv(RsVjA%w}A4fV;_JsaEGCD%@*zb8sATQ-*La{)7E%+SI1#|-QV zdOAjP{YH-bd3oN4b#`%5#_{P%GWCz%UC_v$9-z`|5w-eh)t@NA>Pz~}AAj9n;!zzg zM?1V`tzOc@*>hPv_|YPHXa3k7D=L=)>zD(k%$1f-xxc%3(#s^79Y&s_j!Kpr7xETL z%S$Q0PJHp?fTY@yZ#`^B5A6hAw`{Qp`ms}cWc(7I(Lyt-# zvOz&n)rW;eio~k1QK|D2Jloh6B!{hDF@?dG3SGQ-(G^NGQtYnLyy>KH;GUy;%r1NT z6E)hpGh?6QD8EOlXJdktzJb9`=#($tzICdYDI#)#R8d*kbU#0xjgy8BqA5vRw_dDv zYJpYr7Zhw_s-cOU`TPWUoSFit)XIt7UD{p);rZ@k#gU?^TW5;X^x!KP;$ko(No{ize`Kb>bdi3RluRFa$ZIexO ztnY^FVi2I#{temhi#iMT9D6RNdtlN0!F5aga`kFD3u?EH_WFFnt7i15`Ns=+Zs7Pt zcy9oH`g*Ksl1-LN#tfW_bXSWr^A-p;t<7AllXOeUGS}BBafTn;tfGX5Id?9&$t)>r`KK|Jy0Kqs8NMPk0%sj%Nm`^uylHjP_xo^u5>4Rh=U=yT94%vsQoR>-io2 zt9$j-hta5JmHU|2vU6udaT>+o&wu^?w|e?4}qDA(^UR+oUqOHLcXOc;mx%bG(0u(65W-tYgZwP-97vE(S;tI zq@)KeyH{Sf$JiJ`sUVYHU~0Hr`G^TRs~23^gzEns3#+h836f6GV?&&i7ta2v=;%c# z{XZ6xkcrO8-9UPmpz1(x`$q^p>@?+3J{7Q8eMEwk zb>dBiHhH6UQP6A1>ld(AF4^#W3&eP z*>{2-NV6`R^%)Ekrkz&BZP2-2AzsCQ~AF1yXa>t`8`{-1+>__E4(X~T0 zt(Ho!^w2d;&-Q#89a4T=^g#ZasD8Oy&PLIibShD(~`D z+2+0&O@~t*_k7LWfsL1uF=)u}I$Ys$T4S$@sm}WK!$`?vFMqr6lBVDx7E#Ggd=ETI z(Y^i}G}NSHHSX=aCSyH?W^1lqJV9q1rqk2-O_G^EfBrnS6&F)b8u?b`Q1Cb}Z{x|6 z^Un&?(o8lECO;WNF@&p*pqjW*$J8IHJLtmx>^ug3uT2HXY(9ZlW-|6=|w z;-xS?FEax_1`M>5&iGc6AdG*D zzy7l(RxUTYuMX^1#MU6ck%IvYl6Pb7nwY1?lKIDHI}6O^MS(RN?sL02Zr8vp;&THi z8=YWu&qqLeb>|MVFc*BTy=jzv({HLPzIVQcj<6n8(lQbScLs z#8s`iY^F9b8SR%pe_(o2Wa!hkZxc$C0ryyy2U`->ANIvvLHQBHkL%H+l_h1Rr6185 z@wrx!yBE~_p1phPZ}}=e!JvwZ*qG71WS>Jh@#lL`&NIYi(S;{*B+9C?j!YsoA7|}L zipQsyS6mw|uBHPWT^6HLpbtI`gZTweC7vj;a;o@(JX#3G_vVGOZhk1&sjVyV-Hzou z0L!uhse`L&6b*s6_uq2NW+Tg29=6?jjdk2?v{A@yVE&&MsPx_M>dbQtKxm$+J*`7e>8qxB`p|D5QtvmB%PLqTVseQIy_#Rz5g zRuvhA#Vgov(Q%n5Dk_N5F*5_AHpviIt|=-h5y_O**VA(L;zg&5=DZ)Tu!88 zW>)=hkk8qsWY*jaRs(=W{GZ3aZtfLTaKI%9WCalxU^Qv)!>o|IE7{Rdk=-zn&Lv2x z@#fjW5ZRph`0&&ha@fHEH?4eniQI-7o<@Vw@1kIZR5_KzzntzHIcv~x2l+&v=5!Tk zLl~v&UqEBv1~!KS;{*hrp)Bs;>S=x3R{0uhx)b+2EtZh5Nw)aU+pLNET(aNek+M#2 z7fjhg{iJ=9%@ftz@pWj(GYg?5p<%c>T_`s88!eNN`1GYgEdf3v!rn1S=yC_)g1PJR zvO=kM;vb(~`XE+kcb$i{TJ`#ikB>{*8QhA0Plm0(uSn;qF?`e{Lsfcy+1?Gg#rA~U|_Z;p!pMs*!74o8!!#H zSlQUv_*aWX|2bSaCoeD`9wIZ}8#iX)%XS+f6#siN)?-})U~VHi@Qn3E_*gEXp^!E# zCVYpWABSF?-e13d?SSPR^8!-iRtsPJkl`c_=}b&_W#ou03n#q`Q}KmaU>2~wo%6jV z78Vx7XOZ)T27{hj`#xXY=d2sWC2EyCF-|!bES+e@-j&mdG{-P~p`AN-f|5=Ko+2!S zzMH}pzOyYf96SLk$jv+uF^MR<_O=l@6HVbG;Shd)HjGnx6l{s%*ThuJf2)Q zS8$B7ujA$dpc(i8F|PqdAPR&eNM#x+Dso0pX zL{O&->PT(lUNJE-8kFPNKL5CNfV}$+qcOMEf1R(^nSNnPU1eozx?#*DM8ChU_mdig z*sC1bD1TDfPK5C}%aQt)zFE!eant)-q6>Gy?74iA`9eC1k#+@TWmDh3fA8#sA!8)* zM4rZxrf>yca?+6=Nq9>FYn+y^4M}T4*u< zb8mv-n7YR&6#x*Q5q=nX+-%lgP=JdkfOLxg^_1bU5YoaR;IX!-xpP4^bt?^Z8jZeU zM98{om)hqM>&{RMMT!OLKMXY~sJpnjCKqzFtokyD2+OAl$T~KSSMK|@+uz?`HU-tp zZcOxdh35xD{3O?|op>JhX)wKaEsGc8xyz-Do4%5!n+3NDPl2k0-~Jf#L1HZs9`;|& zlIuPkm(hrtkW2R(5zFN9+xgNA3WeO^r0LGhni|8+K|ElC*G{XXutJ+aW|KZd`_r^D{=Il(fl4cB-vZVaZ) z`KU3V4R+rX zE-!KP^DCiv3`|B*Eh_5Ael@3B9YPI^Fc_VB_<#byHB`_c=TGUq)y|_y$$*XD=te)h z6{LC1-++j_yT5;S1)EN$u{Gld($rPl^y$=6TL0#{6w2PQt>7ETNKc&dOoyY_q4ASV zUDoqY`qM|ajVKwxf_|x;ot?AoqeqWi`?|l(@hkUTKsTZ6*}vb#)ov~?o9e?oFUMzQ zfaC*#xa~H(`>VC!JT)t?t+g1q38Y%f#wOm(IJ)>-@ZrdPupZ=Ch@ zb*YHHeS2t_XU`Jc^f3J5hT{BDIJt6Gsls9buM#f}F=Wk{N)I;{+z;7$7R=`*sP}YB z>R4N3EDWjCqB*wPBm&32{##{F0QZ8Tf{@#pG|ccL(Q5hiEpzAJ3S}Fny4v&TZdQ<| zPj@T_qX%a@rb-WvGfcpQN`axrx^O*|k`D?_Oz!>oS~5os_pGhUG)Z^?9f#(SDY);` zP_i%_m?HY*$=Wk%Fq)E`Q&VMDzcJh~Qsr%@WceL@JMLL@?#OL2;e5{#uQ_HeREVRgNSUSOJ0WvG7Zf7TuQ0U5- zL`4r?I=>+ePg}X?teF}5*3hX>%f=D^0S5s^b#(4x>_L@f7_3_#F$&xDJZ)<%xzZ?sp9kWpG>0&Tk)|6PmF_wF8<0?VmS7G~3vbu4>!> z*oFN4CzIFU`0Nj<)wspW!n1n;ZJlOLG_jt*&EGJ~vL0Rrte)m?tl$D6HrA?y7AUbS~wEeZk6;fp5LQvPN;eyO_O!ksW&bo}^X?PLZ51gzgVvyk@( zo}uAH2RCLUKJ+>3c?TyDgKE*~o7scbwun1-Hrxh-h&zRV=}*C78>feg7kLBwIh1b9 zd?hD`K{=;h`XDAQGCz~QhbT@2g+ z_5Xz#X6q~hwy@OJS__w%2jQgjId|@GIvU=TLQaUVXF=yd5FrfM05+hDV6icl?C3~L zgoj9h`n@~$&`zyzZFR5=y$~pabhwH~T=s7|CJryIp@5L22 zMk6#4&4@FS$+tj-9Xh&tTx~psSqF z^qfT9TR4X!)qTz`CZOSt$^?ryU<;eT7l8X0U?Yvc#t=7S(3CEC4$9kv=QF_1Bs)4| zT)1PA1%Vu38XK7LT$G;EW+Hk6$e!0JKLi{I1R@H{ z)D@47tOh2W`{R#$(kHOgI=Gguux+3{hpwZAWfvs7Pd+x2b5{*k)gpGyhDFZ^}ogz|+Xg zfRgSoSby%^!#BZManaJuc`;RT9x zQ90B@!gBsWLFfH1WSlsDd@zlV|MkgF%l=2>)@Wji$afvR`u`% zonjzTl&oW0>4cje%svIkQfcO;noXv#RR?P>z&@_9rGkT)hoX?AV~wcq$zkgj`qksI zl@mFr#(Gh|9s1Q#pJy&PL1#1aD`Jc2qx%XTJ<=%IpO0caLC-A*XpF}F!zfhR{-4W) z*KuX4GhN2v%0#oWBh5t%&=T>;tgL*7SsZt#mrnMF6=yfi|2?MVP{P%QqW=~!xBK*8 zAOT5=nGHQkTTU|;v9ooN>Vi{Sp4@mxR&&K8D<_XWWmXN`EHd2i?-SG{YZREK!^VF3 zo&EP+!rFBVkJk39e3|dJr^tK0>c~;kG?+o7W*6O_A09;>z%3q;a1{iPIxU>v!skI&;KFg1f-DA^EAhz@_=ynBgQz(@8y*E^uN;Q) zO}u+oyVj#CW5NVqEQs;!?Qz3ny#4qhs4vRNUKp9v-^bPpK`($2?q6;kcsKQGhvZo8g~? zzuwu6@1s}O=k)%`1Ak}8WMt$Sw~KXkJ1z>#Xh5sg0rOJO(pr;k3Yb}vVESODf+YP;{U898 zts(LT?DC@XpAVXb2k3wLYQ&umw`_Tjo)qQYPCa%zV1-Woz;J3Vl!R=w^a=xy!IK{+ zw}j!EnUi#a&+swEx zu^Ep2LYPBh5VVbWhj#DUC5@Y9IlH z{n9fZmmJqkcz9X-UGR_&*C?5(hn-UIavlG!Qr$thHkD!YTo^{x?8^8yY0*K9RqDR1 zg29}6x1V2QaJEekA{R-l9F8x^OkP zC>4mU2ic2nqp8g%I>qM`p9kqJ;~s5Mk3o@!Q}SaDr)9Ol~3L*j{61 zTal8?$H_pBRhh0alwq``@D9s)O-%>b4f^LsakD-yp0&+}3PGpoIRXGsvBA$Cy9cJY zHDQOx`7@Wz^b-TVI2gP-oXPZs>H4PdwkO^@|7z<-3MP29#aM;9{rz)YQ1GeymnSm% zQA2C?58~QuM(aUfP2t1T6af#cFueyvk0hAspc_2kUlNhqwNL=3@UFyWoB~9x`_gC` z2=m%MKPDwbU!}LxPZ?vBL_x9692Ylnkg5R+zXv6VFnCUZ5z_c!-u({pAz&d=_*HgY z^K#5V2=QOcIC!8g_t-s&*Q*S-Z;wYbv3eUU1767O6B`Uvokouu^&UajW0h-{TeeA# zXEw_JMM-7Wi$!DgjE%3u6V35e?%6~K1Q|^H=G5uaW8kVaZ_n-ub?a(NsH>~9(j3jC znjSgw5UBJPx^bIu)|49%Epf^4HH6n#Y{D6cgNieKom(rSV1I7HFHvrwN!GCUM4dc& zQmmlpp~HTDTXkir__AfCXoxo0wr$(zyb;=6+XACNwTrD>SyWmJ;})!jI~EJ!7`)X5 zsr^HMZaQK=exKx$CHaQ^-Cx2GFjxBz=g8UBWEp=hJ3)^eyuqRNq9HFO`As(*HNKME)%DX`ymVp9DU$y;wKwuybyg~5lgQ56^ zP4^GAS|Idx7HyVn20vziUN!M7+yU0%ips{cQ>D=0haRe=>Q?5LH zEHCWzo-!vwEP{09mmkIm*wEk;V8S&_;#L z5yrCZNz9)UPwNXIP}pWe9_)p~@fV_9)&Ir~)A}sj3~C8R&JF$9B~hEF4jVdD-$9q*^g{Pe0r**Jp0y2~ zUmeqZFI?clH$&+OZ4knn&IV)r&w z*?gFtRA$QU3;~CkoDVc#egH}fs~gB!#v+9 z&;E)29A%td>^PL+1h{$)tJ5+`vx&h2sBi<>go{TXMX1picDTYK;E$}-A}V+I8M|EI zdLn|%-PG~vbYVspzv&kkcn7duA7GC*YuxJD3|H61tF++d%jNzQXgQ+Tj_gAHtiYxf zma*~&aJdB-Qh5z|5F3r4Aw;gYqD!Obw;DXZCm}dfaT^jIf4YKOC+AUj;|!cB;Z3Js z>2KS{0U%@-41FN@>$du=^mN&avsBFKEkU;YY=8I7JVt*N01qq-R(*fE0q(*J()Swk zYO*uYWOEC0-ZZRMIES-|-ZfYQgMYBch_PG@7K#A&xG`#RezT($T-E0;Tu9;jiYnXz z%a@OXlxl`r6zY<=X*+^#MW#y<`?O@%q3Jc@(xpqQ=wpZf#Kn;JO3$ z-8g9<-8_=f7ME%074i<74v>|wZ0XVxnB?D_q=V(H0oS}uoSvRu0Gcz{Y-Y{nT!8|8 z8={dJC=#J}atL9{Jx%kVH*N(*Tyk?fV-z=ABTQz} zU^)>(-t3paAa8hY!gR;xs}yum;{Cor!5W@3gB;J^PxjX=F2Q5F7Mg>;w)W+O9OTH+KZfKs+>_W0 zrdt8bBux0Q7#0YO(-mfM=PXSL}c)`lWs)EPm zoJKMMF>CIwo3LgO(0=YZ%c}I`tA2jA+1SXpRvo3^XHIe2Ah^!sOnSFtL;7ggY5iiPWNRqR2s z=YPwa)dH?sxd9Lh<@tNiNJ4mrv*%b1_RYHm>d~^fC@KT1T;Lk586u5-L6vmNYx=98nm(2N<@Ey?PW@UuFa z{zgFqX9*-EP|>`E2Q|8|^0=e>o@&&|*sC$GR?@pu-K(ylQ3Rqek?PZa>>ZOV*JOD6 z1Q$F#f%~huAl0M6O*I3*=X`10f<-RIqv!L`e}^lEN^VYMc!A($nlP%R7&+g-#DphR zli_4dl^3eVt4|+)&v2rOFNLad!WK1|R2NQ{+JGGx+fkDpjN9JTv2j!R*m|j8>JXq` zQ}8RFq@@@ECbZHG4nd5%742F+gz=VNLh~dyZrnJ7sxIxLct%trw^PxGB{Iu$eg7f3 z7&QQR_-P8BTkYf0x3?61e>UBo@qHz3S$#^eS}gHb=vw;p2%hrah5}}?uUSjyp4hSx z7XMRS223BIs4kjn)w~~iW5wl0`<&>vFK03;iLj&WBGTj=l?-#2Y z{{;`iB-JWlilliwHvW($gL1vlg#F>)LK&w^SlB9V%$PB8f38t^i8t@@u3@t-!UWv$ zvB*2G8ouctnL_W&RRnoE800&cN}FC7kLVzQ{9WNM&~+H&d$4;=P^*>^eE!pBX&e~p z##=&G$^k&Z+;=xh`TIxhFAFvdU6Q*pWH4wTrp2N?j;Twc z8U-JTt&YFD9mySy2j|^ySsH&Gc-STzx6I7|w&9#df0vBShvoEmh$6Dr+_2#^Efc^W zx}4&ICk^)YI!7fJ`dPQXrpG_?%InYfO6(qFnp~QJGm58d1)PO?D)}OEDkhC+nPAk2 zU3G@R(j)ZsbvnpYn<#srWJ$09iU7m39X!iwe}Vob$Ze?C%P?@q31t50FH}x67p;f) zt_0O^JZ29XI%qJQqu6w8$OE_+GVf}~TcwM4QV-Ue2R@IS=460j7$_cZq0+G1Hwd!__f<8O-by9e?Xb3)_}z$E+P963H%LS~#x@TEaa>e?S8qmK{XRyWqW zfArX~hG4{OvvI8j53=xI;C>6#3<@21NG2_EpK@me6CS&M<3{VM^h3cI%Yi;W@tY3l zX!8J<+yjdym5^^o4a&`#Pe%;PE&o6pT_23LF&5j_$tf)80MZ~;p!h_;FTj|&c)^18 zAd3?09d^$v}AUa_U!v|Pnvvm74m~?&$qa1FdwPEy^ zVZ(>VV)Td`ieoCZlIJXC^iN<;FqkF{tgaDGae?d8(}FU{W`eVnx5Hu&h;fG?25}nh zFH+Z>7*3xB;BV2Hr#XS!ov|cR`!1l8KF|`T;$>dz)vxsB*&uzI%yV(3Qbp+xS4ER- zx6~@T2Lp@M(zbsry=(YM)>+~+SOwuvSmalwnPoC=;j1=p-Yg}$rYPrKf2f?p3rAd7 z+GQ2-?sFiPAUsem$TT+F$an}&E4>>lIN5Gv_V#I3r9bQOdccK|G`8)gcNRTaKz_|^nKSb}WD zLFE&h;y>&%6FwK+R%&hjUs*N~++WI9We zmzS^5o`|?q0eG8l7^dKIi7&46WjivK<4Fa~f!QU$Hud3U)5zYkxPIeljMLM;7CW<- zOIL{gJxEUe4Sjm_6!7A%Pv8x^ehC0`?dwf7T7!P&czZLbWj@gF-nnKSeU0Z5$}vztObe8{ZWb)8y9zcm9=aqPOUgy?*cf-tV%l~Y2cmV%CL7s{ z-Cj)p6iQgwRTm|Owu7(0xPIa*pu{LQV1j@)+UhQv++~-jZ3@C9j&3Y&JyYHdG>)-e zndyC|r$7v8hAPHrDR-;UDcP`L!>J#DDipYAg#b`=ag=2i4fD=u2A4!Y1fokk(n;7i z0Q0_+5!@->1eTydm1@ZG=y}DQO59!X;1YQZ;YwUuN9=bUFk9W)s zcS9f``1iIB=|1a-10_#bQ#Rb!5Bj6)yK4IzDbXRSOCQwl3jMxW=^qL{eC%jx)Y$Ax z_3h;8lfz#mGG;D8H!clkJjF|eI4@^U$r_@qfV&oA0J72qJ&0R_UG}%l0COl~hFb~x zfRGiOOiY4da}}?uo6LCgCXOE;K2ynUv7DSS?8@)a@~3ZZmt&4g%j3X-H6s_3WwYhY zTy3UfHX5Z+-C@G$7KzcGuR4F4J28Y%{hD(+@)_xdhj>qhJQ>jBvIX z4ZQSYkb6Qh7Ud)3tuqN7{W=RMh4UGcAo-qBj|^^$R8``TY zIiOAtyu3VzSDJ9c^Ut@9qAH$M`$;M5L{kF?3oeWX2-GTjW)!23<>L8=4&s|$IJ zWemK%VwfO6i9EbK!nArK4L}-n0j_k4g+E`WFNua2~&|1Pn=Hdp&KjlYU5f*{!;zTo^ zX6$LC1R_eNii+AU7Q`uU4ex{6ZOs5q&ToiUr&z}F$?LAzbV4hsn@OQ;|PbVeQ+Tqm}V~{<&HQVrj z{BY)1LMi_^6%!h$QlrO;%72*R_y-59B>Sk#s-Far>Wokrmn84r*xS>RO6oL@*ojc#AM`s}z1J{;8MsR-@u>Fy{F-F#Mv8-_GFPf4-6}xr= zN-30E;?mOkIDu&wLllqbLKZ<8vAdybBc_f~aUd)Dd%YssX7%SlK-t^+BB6bgM`_c1vgyY%;~&vPIZq#4!%Zk2QjMlj`o@ z{R=_n5IQDJE_`B{iyJyYXuE=!ew(@)Z@2|}GB!Q^L+N>wFmyh*lS%h8x%YBnWY-zI+=f5#Ol+oE4oZ- z0jhJG^UIGa&TRdV{B*}$`f68c;D@vlFdiB9^~Z_wuQJ8{B7n0M7jvOFQf>ea0rORa zK@=%4M__kO+cb)41iXa9`CaK|EiDKf_)#4P5fcnEtL@gQyCbG*kwujn>&`s&Ogf4{ zNi?$uIAomomHy-Dm-m@Ju&b8FI)t@L(}~7#rwCBt>+=6-p>hCEw#cL7@rVi>NHjo+ z-z-Z%w3`wdnw)fya9SUtt}Xcv`PKc`T>?p{47)EVh>QicF+#0pxNBD=Qm{Jeb$)}d z%Krl;J!y zAe8d}Vu;};9$Ne8^JmW%i;2A|RltC!2x(RRH@9rWfuotR9JckF>2C6=#w2DD+6CqVas;wJ@|*6oQrR#JzzB(BXVI zvs&mK+NR6kFp>fHMTO$n5t}6kt2(^`i9^eey3pvCc?e<~obJ5x{b755C@xk|Fx?y| z@g9@BKJhCaj1>T`g|HxV_|vAa0|KiR6h(G@N;9Ma>pM)_*C0U zLqwBO3^DEW>TmBaK|HW^APo;aNUtvOy0BALS-deaU(#aDU@{K(*4wE+JldS-!Qd4; z*hThlo%zdDYs0V`0K%RzO6_5cRWb;#9-Fl(XbqitlD@XiMc$>&Caxa^%uzc-kilE) z0AN$CjoyYR7y(fF6*qTY_X`V4g1{z*$`D-N%?CrYquFv2)`laB5{oH5D#o*(6TO0v z)wUJ-m@m{RS1YLwJ7W*<8J&aJtgI?kHvRuPy3{145cGci4^atm@y)QCl;WogEQosu z7f>pm7<>F+qf%OJ24e-62sq3w5d#wzxYvL*ph{%b6!ZfC(4am0?S4gB*=;CnBJXUu z9tqjVz)O|87~zA`29XKqH3<8Lq)$c|VttP}%8+VBr2N0e6HqNE9F(8ej^10^mpvoacXzq-X*J68A6Ilp(cAG zzIwv?yJwLy0`zAH5||iCOsYH6P+NHc<_EKfhodq)hvgiao4L)a@o=B8x3O1aV3V5j z^De1@JPJPhW864eF1$MWpPd{Un=GzcRxR;rEHh3FNV#b|=u}QNk{aHAG#Z}jH6=l& z5Q;}ZW4U}feWT({oCzzWAeU7Omr6>OVOAF(M$c)Q!%Q2*Evw-{;T1EE~Zzs6mVGfSv#c3kkm~5<;k=e0 zK%N+7eH!jP9C$6_fAesOau!%kswKY!!?n|cB2fpUlJ)oP+jpvg;sS11j)vvZO7nCu z2kH|38yBL7#|tEWDyV>%%U7Al$3ayi)zG8+b5L5rI&ki~*zqMOQvs2N|P`yfA+>esgw^i|G|W?^cNAcliiOiEx1bWoPR# zdKRJOzd)^+1Eh(i40fbD^sC*3&z!JE*pCBU%({birub^ulzB zntKaL23N1Uz;{TVp;;cCSAYWN`{wD@X91vCLmdp3Jrs#u7ALoWQSB*{dxi*YO?<^AvsNtB{)usYf* zfwusvH~7^r$|zpKm`M(7+-Wta>&X@+bqsG;%=rHv^D#h>I5M``8Saupd6!6V@WKV> zonCzf{ls#RHvvl~ws>*VF^h(_-#cJl7E`!Wi^Zw@x)Qt(MlXlEz}_tCOChhmL)2D((&;u`WhM0WI|pBBc3wFs#){%aX^)A}DD}D-Y%*;0@ILFp*=ztxz_h9NSI=JPv`!i>0r1Nvz!e zP9*H&BAwOXTwCC|;6iCkL|Gm74dbZD=!BYE=5k_V3w#KumiD%w-fBh?FeGSz7=4Rs zZBd2N{kN*fj)Qw0+4TCV{+p|RNQi9<_)y}r8ST}e4$67>?Qx)iwS{xNsyrlN3UE8s z^`5(a(Y^Cd%N|hWa+fH-N#)P~B_^ggRC)b{x851cs+zUI$`TTq8hX}VtSWy2u}_Gr z&G+-EEw{Xij)k{{`j##Fs2~%J-VX;Z`gH<~=m99~B(xXg94yLN9OU>NIQlC@Vw zO-5$3lGp>uO89nWTpr1XUKLz!y*&pz*9~VztV*^Cz9h0Tn3$OG&k#8C&Z$5AkQOcm zr#$TubTU*eDPH}a>+)j@P_lQH7+Eyq&ArE%0Qvg!r%z2V6HsMAlp5$_Gva~pR>2@F6vc?7 z{a02o9D%^SmdI&op_Q$rz<2@+DA{_S`2zfdvd^>~oV&!ku}p_);9f)%O}8&0F46}I z!$+^SK9pHaE>r5A3z2s^*D{i^3LyPHBqSt(b)wRH>z_=dU^O`D`M%3=F^JT@H`>Uo z9xniL_z-Qa;gL|jAxIy}TOGPBwDqosVd_^=(cXbGpd_Ou6A%X9QH)VgWXA9O4*o>7;BN^$saz9sX}{mBFfliK&?_3hTUJl;N6Mt{$>ijFg^;*SKHda zccVbJbbGos=}ptUKKd*oZ;Vp+C%@rE-LqcbBR?ycmv^N(pyT0=(QZ+e9K4)7k7No; zaA|lUXj^{oxi*S8@4O&_vnNiR@MsIdqf`BS_~CY|nyiNjLBYXRiv{acz$n9}vaUDgt-LJbU`q zhOw7iGj?-AOSj$kBPw_x=sE$OAJ-d}*Kj0!cY3uQpI6cnaeq*5FFpyYz+&UKcVc2) zI7ras4^s5gb1=ao9(rX}4Az4vEhHIGTChPK!sA4pfw&rv@}vd(TDeRJR3b@2#Monl zfD2phKZOy~LaqPVQ63_$TXfKsD-nPoCkPob{U86WTYMM6jXiI}1#M*0l$}q9BXX6$ zfHf!%LC6|2KS1cL^asn$Fkc4`9!#eRbNYCQe`yzqA%c~WszT6p0t@0r@M+;&q58uS zvG(Y@F(^O?kRayVUn9YA87yKcz?CCV%jmqy6w6yAY60Ao+!T-rz^;ai!}0qk^{^FC zy1|0R0+s;Ux zZ&R7h@HjsOkAGUS#4xRLX_r|)Mlil=b&lh3UhaR>>ioKeZ)ll@dCb@faT>mZ!rO$U z4_wAFJM4F80Q@0!Mi`o2{T}-19XqHrel|(#!<2h+fVmY94jx=KnPCFv!}TDU)p0z- zr9BZKvIXA8j_ZDK_`;3X0(&4AJjRF1GrTPb69BP-HF1p&!hip7BOLBJ-Hx2=nf>cG z2?>Y^;L0^2x7@u5l)oMoEK$NRqxAlC8A=~H6AucMrx<=U?CtG|R?@!>>o)wBg`n#V>s?{E!H&3tAaeMU zpbG(G6g@hqQ3|%fbVG(OxU`1kTcKroBrz?3**fa#`5}FfAtsB8CIE&_+cF}270w04 z`%ZjrPD*F(jYoDL?yfBNzJpaC54dVzi;~f1czaYZ=S-*DeadvqVU;jl; zPyY4p&kcK|;seW}byD^n*sck{_@g9|GDOA$NNs_Z#zl)jZ%^023U{_un}SiVxV!U2 zmU(+f9|(p{`1q|bq5wMiifbK@2?n9U$3_ghcfY+?z#J$7xF-so%P*T94R^0)a&ygpLJqk@l*G_d~teo^RfNW#s9e>+4Ak^}4Q&qG ze^r5{pe)b%_~`80*T@z)z~+K?23tMKOzOfU6FuyDr|&&q!4G1U+f^noR*cJaBNely z+`^Og|DN#kvj2LE>r{jv4i=jAV%K;^dxBQu-XlNSvX{rAz>fhL3M9t)40R<5bT#W* zNE^l`%J7B2y&<+&jv~>MgM7R)`Rj&ntdo=Z6^7FkhU~xU^6kTL1mFQuof2!mI|At*W{>qSVwB8cLsDA=5R@=3 z1S{Cr4$-#=DH*G_e{uN4*$JL|&FK5VeuJ5|Kxj`w z;)Vwh$A$Wi7ISpwW%El--OyCW`_$;d+2-t=M&c~VLl9TG3hVV+7 zK$=Qvt6t)xzi+uu_r-I-UV zhca0|$y^EF$32XFP1!Rc2&zp7%viusLw0^45fPgfz#}sdOh3!Fq%MSmp%jbEFYxNqGflHW zRJ1$w0|A*smSBt>xrD&*@rTD`M!OekEFlPJGe+r4_JHX_HFkTt)ILaFH7GaLkebho zn`MZK6Zd$2l2!w>J|8PEU^L0qQ&=`BzngadpK; z0JOQs=T+O4b3R|YMhWCVc?^N?~*-2B$+Oe3s_AyUWym0K4Fb(bt$ z5Jhc!kp`*k$Z+GwM(lA$E~MB-23ANZobd4QCr7phOTPy)AxkW2qbTvGAlw|MQ-A>$ z1k9mxxWB*($H?JIT+~uRHP$CH@>`v)0T8IY13bCy?y+aq5yy%!-uUCs5;5Ps#*~|3 zz_x6uMLvqkO0T4+p4SJT|DQI9f+i}*(mi?t0&Kz)$Y*l4BjU~ z+Pk0aN+E;%)oX_oGq)Lyehm+#_r}8y!?)z!Kfg7PHYvVq>0`h2fj=@cHN2nd9(Fv- zE|#O+wAj_PuGt%w-JS|217+kj_);#}BO|l^blB@~xLy(l2ON_{v21`yp5Q_h#!hS= zXr!9~*h>`Jvu=U$GnD-TSk&|JykrW9yy~nY_u=EpeY$zmrn@ChwrufPcL*r7f=8u` zdaNNHI28q8HF_=(Y9TEQ)jM@V-DWphCKx|!QkQAPaj6D%2hvW^Ub#d?#R8G2nAQt$ z8UT&+r!d zvbg`%tC6e?1<9MGk)W-^T0Zo!0VD!p1?PCI8f-xdE}Ed{8WmT2YZODT!MMUVrMO+8 z2p`?R+#I|-HsuM{$AGp_IE+`aidfqruofV4$;Jp5*Po5+3${R~T4xz+q;q{^X5wT# zdB%5dS-@bSfRc+{wgbzhzclpS-&&=kQ2zPJv!DBzItvN){1sFy6ynaDUGkTFasGqX zn+(#Aiwzq@fmOT(@a}56G_0GqJbQZxrebbPBFJhZ*+nuk z24kj36avushlN?d##Q&2nlE@{JW;0~(vK5T?mb z4dghysKg8`0yM{~Xr;92#o?pw9P*u9r1FboU^l90uxHs?+;#c?p;$A-$rogiZhy?l z%=TA{#h^St4Vm+rR4l2GeY|B*TN|B^Y(hmtX)wOioff|hl$ zFvI-v+!KT6t5d!}$#YQleGcMXj}1qVF?fsK2T^%$DY|P5A^QscB?pSZ3{*Cbg4ad< z$oJcik!1CA0I0LSy_i96ZV<}uFVNtK&vKhAGF445_OQzZI# zSx{WhkjNJhRf^E>mx@ulVJm`ON8xr5KpAsg?eQ&FuW5q>rlOG$L?1p|I5eJV8pFcV zk|j37K#JjD{t5kfisIB3Lu%J=Tv&Fm#KkKVpFy?68lH8LOI`;uoDV-$|?l0V?)9;>P+{60yIOH`CSyi@fH)=1<0*f8fnvU1qkr zQoEZ9?(fV*IeE;ClR!ih z&BucawO|{n4dwnn4#d3@!B@ntUk(Puwt$v1KxDP?0Ca`g#Y1?wTGo=#*>ENi9f`fuoWYjDN^7b9N4RE-F(elpH+%v)LVQaFY7HAiWmym2Q zfi|96QlW?{H{eOYvKLtJWs-i*fIsL}GkNZrnOkZB6L|}0Pj6vy zwsrR(6S)iCzW9n2?@Kpt-HHyHj#+5)O3PTLD3$%&DnhKkZ4tBlsd#g=_`gRkx8T1P z!n&}JPi|VxM6>P?<^uj#-*0(s*AtYyvB+D)Czhk^dh0lpar%HBV%9PVEQFXM4a+26 zGq4_ihkm4~TXNYY! zY+*}raWO0#6e?{IKO^z@D%=2#Xxjj(vZmKA=dNDmf2%cH+UW9Vul%8&>%nTZ6@70y zKOff(dAo!hlVS!|nm&n>{cF5f6ahVb+}Qv4x1`XE9#c(6cxT++Z}YtU1dtMnE%eTA zms!;F0HjhZs_++=#xcmU`lzvOv$1cR2(J(H7M`%i>7y7=|Nh!2h{}Q!;6MZeWrbyf zN|0#(4o9GqEK9gu5KzyGKYXb7=>NwJFzIg>fZ&1^(I7;Fm5!#tFXQn7b4Frmhl7}c z%JR<{gIApRP}&p)A24w)7VT1;tgJs*%qv@{<7+FHuy}u%#~s8ieObxn%e8Uk^@SRb zE;EXNC~kcn6%nVa@XGTgNVx&5(}n z67pwTK{=K1^iW1`?AEGNZu8tHub_}ws{Ey%@0Qx4@wj)+Ez^A1EjkM5n!3Dhq#L5#pe*5VObe)rQ zK_2H8i)=Ro6cl@lrI>&x_av}Ip6vGsKzYu?9I1V45!Ur*Q>Tq-U=%7tHYsR7D1oUK z`#Oye>Tx#+@L9sE?c#G)e|s|MmpEOdpYY>hltzIhys%8{v#3A1W-rBM@Mrer`RY*r zl7T-m+AENl?kBhOO&SzFh4=o0Q|lE2x&l!#@~HYVlcrEjU|@xs0O}@K5cXAfMx-kYKaYKN1?o#~djD8)+o*6m zpRuE~b+bp9N+bY6U_aa}m@97(gFT5M1@UPOgK}$|F;1mmZ86Fwzq|&vNeFpw3&e8w zDW7G*PqWLn1EM(>vMPJbU>@)V;E4RCpp7MID?Y+w7yZw^5lg zx$lU#d{Ak-Sp6;J>Pdvf@%j1Ffg+2^_=LcEsE_ubw+>|-DB5tR*!F@aqbi(KyYA^w z%@O~3q%7gsI>Hw*+Cl+fiaHH{x921Qg#2~~pUoA(10v52+V8}(x=N57aZo94qm(f2 z9i(%M^k8|%O;k~`CEuh;lVniF!>s#GRbDX`d>o2gp=iiGT5gK;w8V6nL`a9<;OmQ5d z?Kc$R6wMH6H2|&vX)FF(D;~4MMvz)SQRI?M!fH7*7m~AKqrzC+c#t6Ojc{c0@W9GI z8OLL8Aao1iO{2q?#f}*unGNQUJP+(d(G1`Z+#(AX=iS^S|LezmB%v zI&PqHw)i5$JKeKptkQI$KZPn~DAb^5<&2)#O~yc)0gUFyD{qwsaYrIJ()gEzF6`(Y z32izjjLcRDT$86wbB+E9_SpIBHLC5nHqd1us+OR{ii4Vn;Rj)I<>lojaBR__B*+?Q zU*yWTNTxvtvRh0)sm`LU=9#(fsTsm$1|;LcnM{tNuw-iEBCZ1??H>A4n)F^l^Af0a zao#-JUQn4d(EzN~dn)Fz<(m=WgdS}`orXzg5zusF!ThEjjlrAU{6B+6kdT!d(5Fpg zGIa>c*j0T%oOFchRCs*Y7%C^jl3_>2Pu$9`eCYCDzUUYswDixIn1JioElIwAhEyja zSM(*b2^u4S>Ar;!#&Y}p{?AG4OmYCaR8J$D6X-54mhJ*5%JA&P@Q#4?JT z7Gi(_SXQ#t3r%&nwSxKo_<9q#n)CMm|70Ee*vA$|7+FJEBVueZ2NjC!Ta;{}q+^Jg zG1e@}Qnnl=+0sIbH6zJzDq57u(jp;=QvT2DQ_S4|-|zoFkH`0MyT3Pe&iTBT>w0b1 zax`3a?8VO-;Gl8nHG1bsfu=E@>;e>Q@i^RfjS7WnlBmEvak?c^X*T};`xz}d1FC?1 z1uuI(n(fkjwDc&xXBM%RM7>6NXd8vv48N1w&d$zbup%7B$}7TQVg4fvI2ZN;-UtED z-E@qGd+n{1;6*3JaYSY2dHrh7i)TBd&EwG*VqdWQ_*mxUDb_2r+HG3-4~2cu;T*9I z;x}=d8o#`tXZZ&Qr+9`F-A~8;Q|MAihlh$IdIr8V^*+vA39)dI-r#|J!tV0$ifQyy zFVy|KmHkH%`q-tsMCnGwRXOR?m)2%!?_%rp$$c#41pns9n>iwK;iVb>t{ zx6vdA?(A%q*Sw19yLbvAiKF;s;M-x(J;Jx6xLd$sP#iP3>A(;`h9>o8JF7F5vlH}Ade(W@_-*q=n78ipYQ^jhD&nzR>IcC1MHKOi9V_{czJd7J zz~Dw=K-Y?+^RK5)AJ^`Eartt0sh#HkBCB0dk&7}{`ZcEeRL+(UU1WWFL z{hT73T4cZefTn=S{-iOSM~LE(8aiI2sl0u+Tjah=1vR%U6QoVFar(mD>B z4c22R80k3Kp}eUR`2cPA8GLc%RNU0WMxWW!@1PEqgj?hZjkJ$@hmPhPLR z`}Ung4=Th~>SZ}teQjjNb2i@Z=(G0dl`B_V?}=4HXxNY^E@OC*YhKOIf@tDe&7sm2 zoP^}2tX_^&_6px2TmY*=yru<+_Yt8F`t+>fG}IFFuka|POp#~C9@~I6A4zBe^^)xu zy7+SHR?+5r4st8j^0qEL;XJ;`^d`Cg&8F>@1M<+glbqyA8y7vD`(DSfUy%|MJvg=W z?g|MvGT0Nt+k2TiFsPERWAXFw1B>3vw+WCxJ=&9lJLW52er*b&Ce<^Zb#Pgww4}Y1;inZsiFV5Id?;^Py^f}fxbyXh+06+_0 z7`TeyXF^0PFtMWfQ}!wpBI{R{phX*i$=xKh+&{8GqTPW&E%~j3mh;0eGJ8@whWct;eM)c$#c0VOZb7lFPB1+Fg7GzIQ~^D2m*{l5c_D zs3;mdZ4wVmBzF1XBDcBjrX-YdK6=c!%0JiNI!@j`iB620MH;?(y8o3HXe z*W?~FG-}iB;)K>6n57&k?<+bwdh{9z851`Vrw1LYPS_d&Y+0g3cMl=BC$MD{^Fh{I zpP70J_0WaCJ8BI3IAxfnxcdB0I7SH)3(r6lW{IucKa-)tKI}>pBtKKi7Suz6?#&;E z?^;ap6r&ajuib#mCv+MCs7amEw-BNhc|! zuM~y8&{h;hX47{6w@isn6C`CN3^Ou{IX~4=7hok+W45f|zxQOa2?N-KC7+t-Z{Q>G zTHWJ6d$8Ns>j&+pb+kD|a|e~lf^tGAb$W&P z{9D4LdH@MxvIUX$Tp|lX{q&%OiSTtM0Ypl)pZ&P+d`%W|g*g;-p-W@Oh$I5EQ`wG> z4c=U(SBfNxclFl82Uxm*6hzk(&QQnb8uG}H?|{VF7n*ro(S$-)p*xPLh| zMltbIO(fKBp?af%`aUYliL%sc;lfk#s}V|zwpSENb*snTy*7#cBXUMggQ4J;gECoa z33RcI$YW{K%3{mbtqJT-hoZ!h4-m0_O`kC^G8)g35mG2ZNeOEESPFaRYp7VDzJ?V# zg|N9nL~c}|(%Mhdh2lyub&vF0e!|xhLrlb~$9Id0sx$rsG^a1s*WA5s+^9$OmJbb0Qt10}d+Fys5Q+|9*kbx#9B_K!(Ciavj*VX8m=IC^nn-U!e`0 zggzAoj8ptT&1io;!qoLA5ptG1EC53jm5)CC$h6X$-?MR001rA#NqV}SRmE7B* zyRe=w^vk=C>%#6WH-!BeT(C|d2~?Wu8Q*jdW_b-8Hqas-6N>Ld)`K3yjM}cZa`}^f zCIAuL_Rj_4%!wa*wy}I0^nB7X!BCH3X-2%9wN#irtUwZWMNyE=@+8;6_1T}QH$D~N zsCI1P}PvkY!fUq(|NKzavL<)+H$!JFd z&0vLq|Mp}hjUzZEZs^JFKmOmUAa+*jNvVP|)+3`mL<~=IL9-*l0KY#vg_z8_j(W)1Bo5lGIo=&r(btI$^Q`+P(5k& zKz%O!*u;RW$L)FlUL2n-Y~6@SRH~55y0xnY_X0a^EC|-ggqu+ZWwkFWP5b}$I( zwOKyoqMRd``^=Y5K41P8Lofg5dN1zw?fZc7g;8Hhq686UUTx|9n97>V`0*3wAAx{& zKKqv_prFFIN`b5tDbs|fe$)#Yf1}ytRI?Yo4?XuZ3|IkhzPLp;ewB_>A{TZX!1a6DW_j5guh3JZ;`ZT zmZVkR?tCQKs)D!4D0);3?j_*@Fnm)wU3MO;D4i}gR;h#OX)nSOi)8~MVo)Xkjt)Da z9-t8Y0adjObw`> zaZc_N)Qgu2VF={%Er0r4zD^u%_-R)Kxyng<0ZWtH8C%9BU~bf{rKkx`pnO+UJ-xTT zUNCRx>IW@UWj8x#kL-|imw&+PP0@cJ@X$)Fnt$JPo~WGm>MfmFv&QMev7Np#QK{yo zLJbn%)~c3hxdj)%r(oGEn6XhRtq_TA|OO`fwq0b#Fs;yQ#srC{P{v|YGE76p7;d- z=;Hw)K&GkHzJSdK;Jdhhf2Z=bohFjCQF%Mdy|5*V-x!YK{Cx;wQ^<2%?%sjFsTsgG zNojLLf}?p_C?L)_*`W=PJ4b??MfBqq`LOhm^jl)bVfI3NRi-PCVS4KWN-+K0RV*$k z3aJzP=#oWqJ17&PzIk6KU#6X%;>IF_+g&EyqAbV~VRN%BzyJQXFjrh63Ug)OAHWjl zjz2i{r82sk_Zae*ziu8qdr^Y_9vJ|ZE5UTsb{VE{1D5-xYR+30h5|*Yn$3no5Xg~T zuKrgS;1cu6?We#B1HG4RpETn9A5sf9f4;za=N;3jy(&uP2&?#Q-tzc)hrN7{nd21M5PHgc(+5sah7#fPi z?alozqs;%gK6i?BaO;wwu^oTiupxgy2-mIEJ6~4>>>~UXoI2>ryFXxj^ZOM6V)GCW zl)c^VE~!W1?B)|M_0+DYc5(4Na-p;?CWEcQ!V~qis|8~hkAQTwIloaaf^AL^`j=CU zlq`0G0C8Nch{PSq($A>ZCHCE8odHfqb#tB?U3PBvG{St1MgClwkqM+h0q+gU zsVqN+W+wt;o&pcdJhxpBHWk93pagT;qSHAFB4?YHP{nB-cmStUyWjAz@k6IKdvolz17<7P(j{_TV8LL%nvR_0Y{* z!Fpq7A|UF-+vSP(|Fd6-(0ti`1xll+f&K!AmxS1 zf2iP6Ed8R?_Y-$XtH_qfZT;orW@(#zHW%n+I6~6yy8bplHc5Z`2rMk}AbGv+3Jo@Y z=Q@)#6v!^Yiyu!AP!KhIqx$uq_97wHPk4A1w@&UmKC^Hz!<1kjcAJ#f3VE&lG}PQJ z|0F7{YP)off=S`+0ezYL)aO(8sLvEvO(YGr;`!Q((s)&Vq9|7>8UhrhF`$`ZdZedS zQkY)$Ftmkc5JH*)Fu9mIjUFfoO*vY~fL9+?>I02Qd7%=NzMlpAyq7)-UJ3qH;ykC; zf3uIG#H}E%kK7M;?!0O-k@7r?`6!~0wRu`<6}>^QH*&aU6*;<5^x{EIr=cON z!hd@e0Reidf|3x@gIXDQ$@f~A9KjTZ0EDfxWpn{IU0eLzalh+*_-NPf7F6(NmG|kwMRM8K)J;ld0DfE!YhJsAp6 zE_ZfhR2e0l=X-QPoxwLv@Ds2pOtOhb8qb7*BOB!d>r2{X9$dFyX-=J9wLvZ+D}ROG%} z_I(MlBCwG(kpdthBs0F5wq@cx0eR$r`dB{2NNJ(c_!fo35etDxaWnD7j)xW|{cL_4 zvAk$T?7@32H;mj1BL4$Ss`*k9;{Y5U3P~gp6(0*6sX|FSkg|I%U;h9~=^88%RKPEK=>xwZkr^-gcR9WXxA}ZW;!tSRC6;^8Y|wJ4rtMZgt%{?za<)1J z`M5m>L#1|ihj^F3D=EK*a|HIDb>lnRAW9bS))ytTeybNN`u3hZcje#XBN(T;pNZ-` z3hB{l+ghpD)8Z|r)6r83ZF(@!%QNH;GR;&`PCXA4RY;;g{wF;#t$zJRzE{UyznDEC zl8BIu*-j684DVanB1S8fpb#fcbYQ)3}a(R zr$pUagX(m$U}X}R%s*{5vaYY+YO1#PL_CNzVU-KrE~{r(rAqcMTzf0aE`YO}Dk=RC znlUw>L}p@KwN$CvAQE9U-AAAj%mIq4v5@C#ID~gdKIQG~QT5b*uwK?he9NE}jisS`3XOf4Yu|%ZSfO-{G;-E|luLqK_z%FXjDrXn%zet&A(I<=_ z|M#IXc1a^j0X7cy+URs~+)H3#p7$e`l!QwzBAm0DCs3EWVypievmU)cLfh!rSD(Cx zNdp4f{#9us%_xpKsMJ-jM^QzFFMIw69IIpBDL0AN4ELI3&&mJM_KQ^G?VG`Q{jmzxfS6H)yAaIHT8&r6)_O8Za%|f1OWVOTke~tFs7#Aemn-g;>(}eFA2tnb+p$S@Ylhd|>^3S_ZYE$KQE+#K{>r)F zcL%h1MTLOO%wY$)CEud;qc@-<--$eYe(oXii5W67k9a0)&Y;B+Qavqvov#a=_36ow zsw(YolOAN--uz~E*|hXuvx}uZbm!Wkp}$t{f&tfTSEH6QvOmpxdcMn(eg3~vE%9xL zO9YamQ@?(X=j$a$QDtNyMR_RtG;GXMkvLDFM=8b*$e6so3k^9c=&V_ zW2oFW;OYb0ZMRQ?&Ht}t>Qk5RGS1OdDS78LYo0IUw7FeEBfVM_s+W>|?!|y`>8xPV zc=+GiE~N)$vDGFb%k%N+&b3taG7(HMQ$%@bXZ{BRb^Yw?7%C|WGOWBe-&3X>w)C+x?O8$Ycb3z%*gsIfjRC|k>(eOIGVGeC2(3VG7 zkx>0QkkgD!v`WU&;cjdutf658zDktD@>N=G*EfCU^l{L*<-odfN~Lxe+p;7q)?g=V!bPAVF8TlKU_r?> zd2_9nWK|e<4E#UR{dt%mqzpGv-3|RW!^ZF0@?QV%1e>S7pt_Mc2MVL52nj0cGDRWR zKVv#}-=m6hT6!0739`}`2p-m^e}5bNAt8nMM8z{DGjB$tUHJSz*-)x_?G!^A8vkp# z9aPT>6oCiV!jaxZd#70X<-T49CKequb@{$d0b1#+l$WkeWM{*2>{DfCob`R<)7Cq? z9IhJ8e+*F$;hzs-o)w9gNJoPri~VUE+QptXJa~YX1yiNcvOqyGcEO|hryaF4cO-F{ z`}|g-)UAq=O{u~xyWn%7#|bIK6Lj|zOVj=*&?I3G#Oxd&_0&C#n?pt_a8~}*tCtu< zwcQkGParTkN_}4_`kcjG>`8!>xjk}Xz)61k6QEV{0AV(J?GUO&>&izT=QAjfn!$3q zTj3tAI?Jb5*#9#4D?jQ>q#Ort-n_9NXY+|nq~nNoV(0go6p|9iS~!^BUX`xL~$cNxq5^&M+*kAe| zT)&{txVcT0EX8aCvJ{oq$N$)7bYlUfJzhNh!)cj!15LB~<2KtZ*6$X3-eSb53t;0^ z(!B+&6D|SIeEiQH+&?Wk_e8jHL50O`PB4l6xZ1w`C8_*_{xmuoyhr7XzN+zM*S|Bt z+`oFz^oL1J2^A|r0`@37JeEta4VA)V39{qM1JP6`2qj*~eH&ILCrs)Fp`cbRNv_;W z$|#QhZ2I!I&0;GsFRGujHM;N@)x%*Wq&ya^3TKCSbUX_1^z>#xr!rm=`F{MWF79ko zQ^`7K_gz|t2q3boc#*^vt9z=niklMz+MkCyX?0y`&%Arh-$qhL+szw{8^4hl%XKAr zeI#&B#G~=6G2sPpH_UGb|SPnzuXr+m5+FD*a=2_5SDg@4_mDSx%TJ5$gP~2TFBBDG7qk zDGWYKK+^CpA7chf{tv8jU~^A)6=b_0gn_I0o37oG1%g!hVazIzaIFYV&6tS=DmSiA zx=wt5ty<;vdX1VBv{C75*jUHGhUEnIK%MZDc=lptx|iIG+Q)JT!^!sEU1b#+QQ~En zh&<$-K5vm6Dhxi=W4Twu{v--T zOMV4yvO^biYwY+d^6r32&1A#u|v^Qs$#+7H;FOVGR1P9R;a`86YUv=; z|M)ehC<_M77Bqqh;D4=ZpbEZ531sYEwzl*`%96(07;iXM)XW%J=S;CSKDl8qoasSn zn7zGodL^EnX+G9x&@?Wom-!9tFN%K4yZ2u&FXYpwucW0EfUBA?(R}6^xs;QOfUZw> zr906d`QJn#)%K%&vP+8$*w5l8xH`-%PBMi87A5z3=>Fnt-G;Se@T9%^fHV63E_ z3y#5YFdc85g9X5$STm57WTQ0eaJ-6o53GBgJa3c7idtym$4|}T!@6|sI`?W+^PgzW zgdU+po_pbOZ8yGIR?Q)CF)$d?vw?h-!4n}vB#(t^+g26<0xZ*QBC=(?qLK_pOP1%h z#%;!h$8wbk8XcHVv!G)UVjCJe68rsB5I6a$^zg}QtusgLtUXP#7b}L#&r0Do|QA|Nh%4ZfH_)3B~!TDdUoO~ z0VTt(o5%6ysi0&^Fk^X-hcxgPBzh`Y*)4wSzfqdv-GBYh^GUg%}s)CL5IR{bus} zT5~NQgKAI<#mV!2wbX)l3oLmYR)R#KK&iwGxidDADPkDYM9!&tpG!;A>_K8W7+gIx zxl0pK*$ElRH*D+9`6tL4uc4Zvq5fr9@bQ0Q{0NuI^sV5 zC2#(@$>J#!Jw)IV((8q|;0gH;{(+P)Fk0b6u&^+XX;jF%%V}riP4in<4X9Sf(twCz zu>hh6N-LH)JET-Xv0{GcaJ=%mC>W~+kIv5A`c==F0ZDE%tTO|Ki4u3ZU zQFQ1VM~Q&bVJ9|}1?k0yWic90hnTICN4DCT%2fw0K*#g+N_)yzFd1*JEWd*dHb1D@ zY94|Do4nC8_b(g$NHPd{nfa$rGOD<|wMLa9PB%~Ka{JW}Q1DCRer=R>^XbmyJG0J- zxJ!ZhA=dCE8}{gNKXWoaoh_0h<4)xYr1qjhJnP-CvbH84Nksie_LBRDcV@XVLeQ)# z9mK2!GLndv?}a4yxKD<05p#h6hg4JV`@Udr3Y{S`-h&4ZqS(yXE@L?^5@fUod)fp+ z$9-SCD=b_Yuyl}JoP^glwOc&|tIo1qsmsKrM;jb~Cv4*i`G9c}CdDb5#YLP55XaEW zS4+iR{vKa$4#e;zHp!fGll}WGjGN2jMKjE_gyg_{zzdv{OFg!CR_P3wkp@-cc3gfm z>8OEGFeS3gS9~cgik)|1|Bmm2x5hq~R577Zxk$naE8Jn|_xrSMr8btSVszfk>;d4! zk%|7-Vl;se1s^^na+#z)SuS&~g=y~Chzwso=S6hRsYSUo%k2L46HL~Rpq6geet^w*6Ewpu?W;7 z3LUaezPS2iUW{4e8bnzW)h8J?0>M%I201MT=aiwtl4pX3mh`B7{@s7l82*h-3|;Vj zbqXYP%XKJitA6kswaOZqVUUQ>a8xODkrv{8)Xd(QsEvoKGcmir< z^A9xMFHxk)IHxC!G7~KsU%7q^taG-ogA5R)Hn#a|);Y`N50a0eWYllEW3cOo(?Xdu zPwoK4n4@I4!Mxu``!djOdrp3S&$A?u#f2Vt&ymBdb-t%N*(733=^0Jx@e$^3tS- zN9~NVsp`7FRb{hQ4V%6SEZh1yf8h7Q+v4U;1iX;6@+=7?Aw&3iW@hG60QCJrKvSQH zmXT-DcwCd(zVrC*MRR&YJS6O5Z}HmG{5)Q4f@Sk>^%x6_ISp2#2~lmic~sI{&`*dp zmt#Bi79#S6{4V9;ibrXH%KrC9k0zU6_Onn_Kf2y!Ww)01Amx1Q=wB~W!<@uAWC%YK zC_$Gz(j?ItEeo?4DId!#9~e&}BP3{mHYn&?=2{-EG9{ACFi{vM96bL)Lg{5`tZ{X1 zks}#6V1;NAjhFwbXI2%1hN!CfXR(<@?cpeelh!dCWlEPO4I3Oa;cnc`;h(wm++9LE zr2ymX_@|#o|LgNEi1q)EKmY8lBLypvn(D1vZ%w-JEy6M%+3W4?&mrX{-;ZvNfMl0K zLPAivjg??eO?oT`CI4uX_sQAaR0F5?a`>=xe}m0qUKV(jG*kIs`;z%A?}c z=~Lp$|HQ|Rnj1X1_n9?2uDLnP8=MNdZ7*q_lCa-W**xnL9cyq091 zkL09woLynB5@1gRB5n`5sY046t56OWIY6|J;VNtA(VKYg$|w>~6v7c1?eI9y*?c+)`K!w6; zyelf|qq}hayl~=N>T$>CsGKI^ArZtyth#JN$5OV9B058<8lNjtoTPN}l0Hc;arz`eyqf2bEg{IRa-ohDU+03& zn>Y8W2T5kuH312Mc?q03zHXyVb=1q#T;f)MW2ekNmg8&^GOVtjEoPOcPHB&V$}dyj zbr-yaUQ=F<&u3q-pA|dfQ*!8U4WnCUF1oN_DF5wIUxNugt6g11UM(r>g@g{DqH8*D zV>-CH_=XtX5`mmc$UgZ0PA4BuK@?z_NZ?EDhHjF#L%;N*s{BHYEm`rGyAE5C5v?az zgvRw|N{TJrs}i=!U9;w(Z(oA=1zu9(=NJyRakt74?N8$Udw1_zLi~JSz#Jg)6| zCcZW~L4#jScBk{|)uE$C@%i42KqYvGf?A#FIxl(}HHT7iA|dsBkm}UoFX% zuDMq`=t&_o#`K4VzAuJIAylV6cs^o|++W;_J@vs(34Q2$VAX7>Tya-V z?Ka8Sud364dIR5RD8cTe;7OB4OL|W-m*4~3&OcuK+yP__Wa5Cxv{DywkfRnqFVld? zJ-VIgB_Ti%nMpKqlW(-r4q-=8V$Uis<`j%7OwL?~K`DujJQ+#VX~+y?jjE`i(0!U& zKQQO=0ylwgSy)f@Ss~6N`oHD+F`qtu%p%et>GYHff?kkGHEGFx7Xvygk=>FcV6TNZ z%|=Eg{^EXDRl3eXK9C0a;@+VEmPBY_35O>kl#28a5^v~Gopl9wT(Op#6~A)6%>FzD zHSPL&TwQG}NdVd*8>%(47k4ZqBcXPS#{HxUwxa|Qq}e==Hz|m+g-k%P_XhqmCmJL5 zxB*|Bwk9{N$wd*0;t$|JGoHi(IpXlb_+h{Hb>}u6Fg9fnqbeWY9QS;0-@Aa2%B$gj z?$MBkT^~$3bj^+b(RXil%h$&o>qp9yofB*$XSuM*N-@BM&|=*U;Rv?vI-RA`a;uE@NsWx`ea3_amM5sA*a;6EuB{t2`j=mlM1ReI z9*)R{N(fq%&%sGdadVbY+qU+iTUrPVPR5jTt^gMT zEKrZVyD~Jwct*5}NsoW2W&9}ZsHKgK{RFfb`)|&YIMLL5Nu`{{D^Y8CPGDkz_qC@^ zGDTpM4Tr8287g&qLRjYnKbh+I_fw}grr&2=4NS7YlSIhn?cH5Et!Y`glE(FErIJa? zW7N3Bxx8#!{zz3b^rfHgsR|=@v&F zTW(H>oGgYvsgX?xb{{ECfi&?Xghr-Rv1o$*Zozc!&di}J_#tL)FJ8W!zo2Q8Cf>WN zSak)Sg+vI22t|20P_%~`38;m&^#*-3lYj;r8}FE<>({Rr&{?Fa z0HL>&l0v*auf@iW*>3v5D`k!Ku8Wv;akuTv)T;dDqLzkOU}IWm$5Lb%iaPjqyX~RV zLaL>X`&xm)j<81^OXVDwhe?IYf->IBL!+HdyZ11RFmLp8_gn|7qPF<>_@MAyFxksy zmq2dY9Q0J`cmiQAyWK-C)|HrS8rLp zW(~51Apj&vLV;4JhEB662w+I7Opg?57S_5Kq9}{Hn~wo3ym|XpX4lsa7GEb%to#nu z*j`-l_fYh6o07pMETk5{J0m0|fw8g@yh2B@FXf-=y97^>%S14IWW`H{j#llGbm0Xg z$XJKY=)_xAPybX>(jr+vqxM%LW}KU>LnI^hq~xF2eqpSI1XaV~A5c)EP>R)_;@}l4 zt2`f$v+bE;RTSx;Tq8)s+AUd5O!qEuz#*MHe%~7z0f=7AIrXL>7W)7yY zZaZ^RY2pt^WkeA#xY>QRAS1w(vGYgsfX>-&Z&;;D71msmR2(X;p>ptgdU`UGg6~gf z!F?mexek}**uRzy3eoM(t7JB9Rom^Qos}oc9vF1lsNx~(%py$!^c@->{eDSgVwH?3 zzZGpQ-59jBw92|DNQV2{Ukvsli zFG^E8WW`7|`gGRs zs@I1!8WpurGkZn|C|$P7z5~an`DL=XjvIvTlaU5!(gMg^a&%+E2p%I!!+tjWR;>}Y zsiXl!6xb3(=U6pO8q9F3E^pTTSoH zWiHcf)^rk4@|J3KMq+E8gvhWMkb&2%Cx>rmo;p8BM*MLDtjcA|Y_L3XZU))1wkXk< z&-P>M)~(G;VFe#3Ya6^Mm)l9wFRRY>bFV}ec!M5e+p89J}XCj(o`f*o!cfaK?sy1Srb%9fueBbp?StW_rG<~ ztdz^w7)gbBS5%R}qb34JQzbE}{^tE|U?UU{abg0q;6(Lq; z1IT95@}a)#E*Iwa9H3$R&o8TX-OJ&p!_zV07B|6EH<$F+I$m7q!WhUn zc62pg6CyIDpcmknuRgr6sfB-kHpF$+FNzbHlJ)RE3uV%P{`I1-5;6vyflsd%F;{-=*o?}1WA zQ8!a-w7phFw+!09-n#+>EeI*=B)APrv=j*T$=rhFf3_JY(DLw{GbPTk4q>= zG~W~jXud7ueqUeQo-xLpFMV{*uO3~!79qdOej}u3NPHI=UEUGlhSo@&9x_dbPl`;$ z%QlLi2R2%WcF-QU$gu0&2|xJuPwoHMj(S?@(KOzF#mC3rcMh9+u=S)EYVXlB65=*W zD<|kRTq;+k~|VHpyZR zj}-HQQ~zFrdV@XNU{L=E5Xa@eJisH#%R=co8D$pI}ni!jf|nqRp}1I%<8f z0GHgLn{4=U~`s)ND+k}Q~yHg%tK9UvZtzM zIvwajHk3F`V>}XY$rUolCRbJ<+lrvS1L6?jSIA*cSDsc30u#tAAv?6=w4?SxASN1rgS6mXK>x!)0XSSQEQZvDP>FZRkMisMf~BCn~8Irn$a z8`KU)Cukq%6y|$c*`am5ZqVW_g|1&W=;*G$Mi_@r@#Qe~TD;<+S@3rsOa@fTX$FOa z)oCGor(i`AunP&erll>zQzRw^BbAavfjX~W%Wa5ZbPn{+*4zqpg4qH_*>b&Wnjb+| z@$T(gBOMal1;U`HA3j=aB8->_rci&pymUWg#q4u8rG4N&l35EiI;}5mi^Ac_lUuXn z6HLoFQcr$)n49C=rT2@kf`7=U1E!!+$7UtE=jb{KfXJcAm&ux)Mt^xCKe9woPf@rW zJKr3sGx5!RDzz9uwonF4V5ZWKFVSY9`=R&N4SpLrguOOynoe&=1UBxXeku4#fYSs|M&gGnlry3n4Cr``fI046n0 zMy{Q3DX-_rF_UWTcWtJ6eImf4?w)?r02P#%rpAAP`T%pUpr3D5d$OjDrvJh@56xD7 zS76NJ4I`dI#;`28@^%xUbxTuz|E}BAP+!EL)=MG^_li;k`S7IK6MP+8K&Np>tw|x| zkR+219TWht1@Oa)bhn~r_JH}z!a~os0BU%an;U@5Q--2Qj+*JVQDeqD#J?p2-WmTY zWa&yNtR%-ZIOpJEG23hFDEKaJE6-igwZSS}fO!(=lS3le1v2qd3Lyl3V+E(zgm(bE z7iWV+l1eBM;z<1%5wZ&UBNlsQa^BMJ5+cLnp2!_=#140^;7t^O#+{mJyq|Gc2uL8R z;gI1cPqavKCu_DqO1rgd*UDqP3oZl}D3vdFKw2?Ce4tDf!OygDg<0CnV4Fe!$pF~$ zO~u58Tm*)?ce%P#(z>)D>f0Z3q?VNX<&->FXxe1~V7}fvq{=D!m2ZIAqDPqIIA=n0 z-xuM9Q~h&3uj?Ig*5Yvdhe|5T#wU;ed9CM#+2zkOy2`%@QgrLT`(2xKeRk#71HU&r z)N`PT*`e!haohBE7Zes+5&A0Je=f(CjD~<%9Joz9lBmPplMSk_@dF7yWXBrz=@TN) z)K<`vxLF4L2#vzWm17NFHyuOB+?y#4BkJF*#3*7+I-^m&2vhn03ZEjkwHPRJs@K!kfn6w9 zsw9lJ+dgoX7IBj_aRPLnU<)L)C*AYDBj#^z6$Q(7>dq09e18x^8@LcDel0@mdL7!o ze~LqrW@@LO0`}(7!mm29qqM8?WX;^>7?Y)2{`x$=M*Vfon_qu6*38YQcLy{68ykAy zdG0{Ib$z$@<0boqEa*3(-)%Mkt)VBLAh`mlOe7e}Xn*}T)!!rFT(|C4(q@igmxDXR zK&SGb$Zl)Ytl2^>b3(iYc+l0P!+L=0=R~K8exVWDcR043J(B6oLZx$)^jxT~9MwdkY|_n;&xc!1Yg#g_A`1C-5*T)t3Nh#B(2h+QHf)&j$&Jm>AiReo6mox4IHCjT%D6T5@2;(x!oxo6KIK* zCn=K+#ILEP#F$1O|Llh*j8dOEZ=To5DM!D zKeKUT*eHJZIiOlq*hAx0u510gFUtw3@QIUze3 zRL$AOWn!KJRtBw9(V6NG9-R5g>;}b2Y{5fVRM4sT5W$#Z8UWjNCPt}mZY985~q@pvCJ~?m+pIgA!$y4>U)uo0L zUD4#TOm$4<9g8*?9^yrCrKm9|Q$^+oq4ywjl@I>mM8Ri>1fcFMG-uC>o++f5=GNcg zs)cH0EU|@RS~1V#>=(qp{4Tw%!1dR!k5%A$SB9~~IIK%z;Vh>-+RdU=t{^Fo9)H;L z>v{Xfbyd1m-yUeke-%f*{_;fMJW`(Mh)prozCKY|*`yN5G8T^^e$mZWA|gz;b?w}F zBDl5?53J&!h{)n`AUM)Sh%Zjqb_X85Vdu6V^`iKH`7rmnwsDg_aw|K&E9iCj`V>!0NFm#@W7V9qJTd`}l5yh2L={rPW^-m+^PO<>d@YUSc%b`2bwp`I z10*rY>?ThCoPA$aM>ar+B1EJ3+Asd;xm>Qmi?`F$)79ya%p3@Fz<*UvL6St2Eb@i+ ze1zx-ETN0c2lVSVMOYf(K?PHr6l45w?znwtn)HAD2|we6pLd$i9e3|QIliD>a(b1% zd{goY9x-&I54DjuB8J8-9`{yr=C0EJPUl2bF5SXc=9~Z8cw5Bkw?_qNlLSjL5gDUv zZQP-%ea8cFe&sM1UT>fYg z_|X3RWnCi#V2n9SBtTcj3R|rnh}G5%)Yn%~-s<|Rqw_9&m(!iAmBVlmDI$(;v>*xK za-BDCsJcxPZqTRCBtmL~LRWHmT;l=KZoZBH0CefMm|{zKKGTh9z0IorG1DL1d}kH>2aLD^*@YXElwgPQ+lSt^8b(;U*i zg}RVR9O6LwY}7n|*#~>?XsP`~sdy4u$@Qv72Y-IE0SRCi;Jk+7|5Iu6p-{!N_)Z3j z(Ll>JJByX7PO0!42?ESAKMtiGVk9jdJN`#cgX%7as!2C zCoW&2Xpw&PzFl5FQB6GL(3UxdS?NSuNHm2Ah7e$OIKjf!G$T?^JdWViKQV^l+EM~? zBnAnd9J^PLy~S_IH)y-%AORqNr>N~VVp~gbU+cTRGD6F8VE6ZCR!&f+{B)`P!NEh; zv9#d_v<&8hQ-ma+wgZ(829zvL6i2C`fJP_b)KRv3b=?xq8<<5)1YDd@6Yb*3?!r23 z%pGv%LUpG}Acq2s=K*t>Pd1Ae#LvU^Dk{+!u98(qj2@FM_pv1G373aEStgo)boqU9$Qt}Z* zIx1i6;K5PNA65$9P>#FS36|vH^vE;;hB=$efwHJ-Y6pwx@EXs=raPug9LayvZ%%t^ zFhjPd)M}Jw{t!`=&B4lsKji6oj8?0B7ig!T9j%k6G((Kglt2-1SyoVJA$AgRNwOUG zfdVK}H1#3zJwe_H<6jKG#ZdLUrBktAN+}@2JRoCs?Q2@Aea1Z*3Rt6IhuikXcEi>} zorwUQfhIj@hX%{g6?j)2KRd07=icLV$8x2~;0LH-@QbE0*;1|@?t9zm1pkS-h^>TR zgNjRI04E~?K2nl46^jS~!as6$HK(L%_4Hp|`XQe)N7`pWdNi(WrEI?RcyY*~c#KY} zVB4?iQ2=D)i9pW)jS6t#8XotGM;qbJ!~<)il4lCJ=6N_*mh}ydZo%=h4mI?Bp||LOCPVx$vgHh4d5c zsG^(Va>}qXBx>ku*9KN4!yWo|0}&C<*KW=8#^5|wCIR}s636`_F_$7k zYuid=D-xzVyE`f3sQ-$Xkfkd{SV3hJ3Zkx1XjuQfiiF9-<1hQUqmZ?0!ewyOrVniM zv?mQK6;1;qJ>nJ=wEv#ryl$0M+Hq5czgkuPBy(Ea+f;EWt4|ugD|#^TZJkE{#3dau zbEyB1LxShqI=b8Gx%W1#?|<7q*q;(aVIc2x4w9)Aj-D+`?@+A5g$Vv8rVQ{Yl(NU6 z?TjA+AusT1a|6&1IJ6wQP|4uhP0kn%0l4NWveEg-w4t0wiOL)In@}GiMs#|`2pMGV zh2$RLeBF55EDi}*YKN3Dqeh|2VwtEfq=u^&Ak7*zO5H+!3dReGPe6y7RxDaRU|b2g zR{GuO2sKCqJ(ff`Ch}JI`1s^6ol1{d_-OWag%Egju3fDS|I}NEu2TdXFhtFAK;nzD zj%|lro(XuoDpQ2Y^vmmtvpdq1hLmC2$gO}}kP%aQg=f%CD6HM&|5o}UukP+aniaEu z-N4YwT#5OB-IqhXW{e$CGo&vhk`Z!BFl6r01g}QgHJlA{J)7s@jtHkBD~B!2Sh1F} zyuQhRUqVX?p%N7F!J9$K&zBm-fWS#wx}0+WHWnjm=+=fr5F05|DxS$7kkKaZ{OB2PqGiE7EUYMyP0`9U< z_uQOy3a%~ydCgHcTtkxZ)$wt0649Lh>Dp-3x+pvy;@TCq2RD9NF%lzXY#QMdvhmus z&8Y-iM}xnR!~f3h+fKPl&|+5*v`DJV=H9q!IHt=8cIti8Eo%=hqF*;vPuJi19+iNA`QWZp#r z!bkHw{vb+MrTMiAD=hZ<#>j?~-*D^xAn-heO{}aNNI&!L!v_-`Zm`#XU0OE!;IbYG zospw#&b?JuF_56JdgrwjC$hHBS_f;>WW&jdx~Wn|6aMr4+D8sLto^3vmBZwkerzbB}Mg|Lo+IOCq^6(>lcBjWM#zKj(`osVqiIMfM*U>0H1S)G^*=NY!_ng#Qiz$w zW#s1OTF-vxoVu0GU{4G*s?|W-bgm&uZH8wX)2-=+UEno}j#>u90F#)Co4M}h>zScY zdO6^R_Za{qX`EKeN9#sN+$2b9-)=nH{S4<4(k`yNAWf_P?;KCk)gNMMUH5`kKw z&P$t0t8zK>#Z7|IIxkTeUvBvE(jFabt@54Mg3=1~a)DO3Cw?Fmm~kfz?Y9wvQxm^? zYjkgjjGdt9LpMy_F`niL{f(k=Qou&_G(?#T`<}_7kB9$g*ig@S%U5Jwr_;Hf4?I09 z{^d?RLfSq=$=9q>71j#)_E0CGj-mCnA8DcSGOXFdXZV91UaGBs*4Z6*XL@_5K2)LP zeGC>6izs0+piO)Fi28&hZL9=Fh%lN2nRA-zIa@hnI?1_~!b0@$^c5s5I^qSr7GpVo z5O!d#HT6`H1>LRWkimR+VnA46I-o7y3Prv@JEm@vMMEM)O^a;EyleaRHx+q^SsXz7 z^W5A;+BG)AnuoSx&zo`+N>d8`Q2YN@Jo)|zxNH-?;*{(+YH_(Bjp$6M;B|~wmwWaU zsWKf5hKrc$q{MClpKS_f?a^Wt0yg2$FawcbA^6GNO~>X~I%v=!`K5e0DP zHvD@@CAoL3s>*k-n5t9O4*lxPFkAWd-Ma}CEOml`tVECaFfhjYb%OE3DZ}r+t*M%K z+i>2Oid)v_(Z6c`ggB@%*W*4vu3c=iwA-ZNbvx|WyPJXhd+nAs#`WHr45+CN5UpxV z3^`SH!2HQk?os&JGnW1{?Bv5!GCK?eN^VmyTk+we<@BNn)~DH%5!`%3v5TY{0qBu? zxS|b?mwX2P_q`$HWvl?vm$>&EhzNYUT>!tPu%@aMd{>5oIN1)13=N$uc{E~K!lf>S zSCcI_kKfRQ0Xw5Jw|+8?YuQLCua?W9LBDh*FMFVI#yA{lw6A{bOp(MUJdfR1r}X46 zJ__)0-@KkAcqlD#bdbcf^D9%jJrNBiw};dNGiT0B@T%+dk}FA|PQdI3oH(`J#6t;e zA@63JN2P=;9=c70OE466&bcU(b%YrL!3yZtEXNz6j z7(?YkrE{D;baeDukpelxib*8`CWdtLx(F}C3*V_m&adJ^W*86;ZDp>^4UyE5@=y=# zrYxp)f<8m8h%!&+n`|0>a9ylpNB}*X{9eUa;o)Jmd_r_DpbH?Ua*>tTKM}`vJo#|p z(4M~QIDB-R*@{1aveL1XE>6z4@^K$#Xlc#}5JH=ch$rK1SR&$K8O8_qBVCby6EtuL zhhjmL)nB06`gUv`J*~x{wV`m)3lxbxoxUP9`WPWkHIJK^J*ODyeETc%C>a4FH7E?x zz>s@mUpe(%!SP_2KF+bz%YQ&~-aYzKe);M;|4aR8_9Opy319h}V!NuU?eovC5rvky z-+L5?c6a{H*Pe=3Hj*&h^ap!(>}Yd2?e)^di~@qmIw>caO!gql2s@a6KO?6lyeUxL zyNW;$a)N2$*yIezL~Aq&)?Wr>@QW^rdyC}Y^dJn;X`VIivvxB87pa$~NWO;n zMMwZo%dV%;o>> zSjv%)n&fXGj*$NIRJ;8}-7RmJ4;?zK#Uk`kq*}>!;+XDGNt=d`q>v`-j=k}?Kwy)c znuB9hnvNWN&@~K8U;6qKB}8Yy3sFa96onQB|0^GNR1ZMr4)`Lg}3Drzj%-4`puE@m!U@9fvuyH!=~!(x;76Tkdi2K$Ey zF6R(X6XYESsoSc9-^n;3`T;ADH!p+cEANQvi*EPbC8|b5_5K+dGa62m^FSMfd{8Xn zDQee`Pd6vvpcFf-#wCw)T-;mqjW5rob7gQr3GOwpTa&svV>A!$rMc6QuBh!YO9ftT zBSU_`^$nL4Wj`E28;RFFXP2<^ge8k9*cMVhflgZsg5adzlLEk0_yzJ7WWF{A)>BRg z`9}CwPoR^y6CWScykRBxyAtlomztZnu;~GWE@F9Rsi7dxIZU@E-4;jthezoY7NS)E z?o2pXd*$EkzVg9t;qmqV=6EuFce|Zqtvo%gEt4&R_<=3#`7zRKNXDBe$N%P#7r4i~ zD}h1m5NU%uVYvxyK?UA(ZAZeG+c4PnzEVPJ#|hBO^>6*8E{7yv>76w((dnnL$WqKa z&tp3+dT@M{RruC=+vapSsr$6~_3S6La#uLJdmEg}ePr$ZKE5>I$L9X`{lcnWT=Qhe zs0kZ>)+=7J$p6C{%XG^RZ|_F#4*1e*-jp>Z9&4^AoOf-vefF2-@B4&1_nW@!ea8EH z&3m>x^2fwuelTRvus72O^{Ch9`eebS(*+Ng-8;ET7fvTs5pA7*JcZo&7 z55q>VW5f!Qq(x7R&}7Ad$&+>ohVNG5Eb-Q;#l}lHfMBKD8S}tCTxjT!_YKY`w^%|} z1uy8(QGT(yHO^Ws_3)+|^$1my1t%*}Q|ymU!Ue!_53)9n_)hKGUB92t0^$0QJVQ3> z_^5TI&+_8V4c1L#=WgN8t0* z1>INRIXwwHTR9Pi8Ha|Io%nI--i(>dEBCxtT{DLVd`EKFoi)oxPa#1xaPxPr%;36v z@b&gkl(o@PFmM;8+)1Hslkp53SaG$gZolMk+1DXug2AXd%|DVZe&h2wvu*jwehNXcyY?! zqwZxdr|n{x_^-x`towL2OsJ>SZX||qrP$4ld0Q)X>auYEXJ^!7P2vYkE?d4J)6CfM z`pX5n#ruAl*X(eJ`8)$JFTZ(9!;cT$Jf_S!)-;{G_W<_)v?rHrx*}aQi;V2EJ+^u^ zm-GTfro!>nte2|d_=@gI9E3}kZj&oF|CIa0XLNN=g;Db7bD=Y;x-1=8N&CBqEP1S1 zWauP`ObEAG&6s&}=Pu-Wav`=G8Lwf;@WGnoS;BgKFtO8~1qoNGFnBw#vCdT6=3R}9 z{yr#!PTgnIB*1V>$1Bgz`u|a)CJf|f>BWmZ_m^u z`{7GfVvCt(s+~W7{vuH&ib?S=O)@(D$%5KTD1_o|l+{5U!#rV8Thz z#R$h7QpT~NAR)4RvQ=JCn>jm2=#OV_9a+b4g+RLq@8u~fOK&t#l)+gaOm9;mLnckT z8VIB^mif)kt!verKNBmyrWhts@tWHZWio1^#=GN>KVEa#c1$Xr%U`baoWCcnsVd|3 z^F2>WN}XoS3U_{e#q!Cz*I^~Q50(BgVd8uuSYF0{Y0>leKmA0U%)Rtr=D3B@bpH1-)=SDowuJ_(B@{f@b1>%`Mx;!c(J(RrK2J|O0F%6 zlXsy`BWT+toeefs3(O#r4E|2^oQGOtnl~XI!7l=@ICD#21|Ln(%ZsYYz15v>9WHo0 zoOstDN!vmmw+o}1^k&gpJ(>`Uew>qhAyQ6(@=|7X75gb0TLS$=Dj}AYPF>Z7wESHH z5)4lk-0IhkhrlqWIYzzsXu3uJsfAlhEle6D#Q6iEbOG*EK%;JL)KNzmbS#Ji0;7Ub1Z;E=EU2IqsnSOgkS^T- zAy^PAC`Edc-jx<=Vg&&K=|Ug@F;qk70YZ}hyv%&}`|o9~S!)!HNyuByIeR~Q5;Aw8 zE;*9lZT@`Q-O1h1R6<-hxQHtpJqFjSV%P-EBS#oQ%vx)tzETWYtAx#PYzf0MMNlHR zp^1x#q&Vq5tOiE8sgy~WneS2D3KOE-4&M?#ly5`;C@(E~Yaqrk+ z3MNIJD_c|LAA?s9TJ#WElsAVY+^7=-oo(>L#>GOBW+aNnd)v9WBjB^NzCiwY1y8{l zRlR7-1v>?TfPXU5XhD5v7=1&7FS-|6k*M`2S;eW9j3?q&*IpWXs%u#7?pM6TnQbn$ z2Pfq#(@CQ!O5I@n^k7Ug4mJ_!9KGlMeMm)d{i72+S>H|h??ZH35E{@6mZU9D!2Yd} zT7oJi^ef(l6Dr)nlmjrBg;qxovIfD$MB>DWk`HfD?iPrO7Df}?5rFHV7Uphz0rYgB z^?#xZ`q0-1@5Ir=8)ZW;`y3N=gvtQ*Sbv2BAfQ4$9i+j6A-7tX5)`Ru4u#aTaGKUQ^9qzjgtT9yx*j$CugV z0;3>Ar6L0t*LASk82(Y9j(qxb3w~7V3hlIWTg?iu=!I58DuWSfo1*1yZE0D6@)JPj z3HNFz0-Aq9zY @VT4n0e#Imv|)j;DguOdFzfA2gfM4x-mQN$KnIh7Tn(__KrsT7 z4+vJ@x)psqz)IBW^%k`FFdOTFn#d^~bUHa8S9{oxAQ5P@Me|;je~r`sO&hpQ0+jm$ zP2~p&B2b;Aho&}G(clh>J`p3u6`QNP^bx{?YQv5xD{E}nP%!v!f%MOon*W1=M0gad@8Y(`)58i*tfqpyrEk5YpB*hu=2ZxhKult*N1y6&^I@uEdu&6 zF`W}ZB^a31AxZ_{MnIus4GJk!OUwPc4btnODc-YbhiWQZV556M^^4HRuqI+-5fD@Z z3zMlsNK_2>O=>yXVt^rkI18TOZGJGlgOqX{`iuP#7XX<>TVg;t4&HOF zHU%zpQAM~hG#ERF5MU6B~I$Bx5;Zu zauyLV%3Jr+bDf0E7nyG0N+@0oLnt(z9SzC50Q;vpC(!I-xOI1Te#nzYGluh{2Lu$4 zU9h&st;NOz@<^7L0l9xyGPC)=P$LB77eI`{+fBfp=UdytAb00)?yr{9%+0Oxtjaj7 zvk?y0GH99W+mXIBNp^(^@%u~|B$t#v?M}TL9xu<^k$UDW_wO@B5w3~K(E#1;08pM( zLd)ye*uT|gGztzLTyqboS)X;nmJRKC5aYy%PWFD5&O0miNatT-NxI6DO zR-oCwhPC!#v)!~8UqFQaue=kjR;JV4XB)-jrgml8C#6gJkea?Uihm0^_g7Q1q|xx= zjXtak_mdgZk0!ib$7GC5oEMW1`H3l6#6*@xMOyzYYg2PpaBH4%_=UNp{DkHPNk`eAN@UP7u_ILMo&iSzasrEd{V{C4e6}*MO?Yt|)e%sX& z)Awwh2<{^Osb6fV>Q}FdOKya2{216>aQU3(sX~k1Y)ieiUi6f-WM6a+8(lIAHunxU z(XUMDYrkR`YgkaawIeFwYU;fyvAG?yS3iuK@UfGu;jcvhbg{E(GQ8^c_^8R#wR3@c zgatFchrOJe%dIBTqYnM0tsY=iWy^w?MnmqsoE6df zC|u%$xn$)WcjYWE>-&uV8PtHqI_9RYImi}MI0{LLiG_6m#o}sp8lOMgL4Yl=v#vu) z*DiF@sE68&=`Wv(kg;j)U7Y&t^!=DdXO>Svmut9CP2Q%h9X_)+l3nDV_E+$B4!zfb zBglBz)xf~B%(g)@GvjB|jhFEA2g*sy9`;stMczoAsn+9jBz!is{`Y5Q@_`PqC{*|f z`t%7mh^Qy$!Xtt#2}&7*mF4eycKt|)YchzsG6WR17exF!9vxbm998&=&I>3)=b<;n z-E7$;u<^zUiDUP&MG|8nJy&6- z%BD2fza$RkX%rJ#4)~@vx9fACm0RKZWm{PszZK(%)lSwOGg6f}qAya;z0~n2>n?ZR z>)j@gsr0sI*wq$H~@CG z&$QUPAm~OMNvQ65_Z`!KefXc_;$7y@y9Rx(9!5n@&R8bq>HGuqxbB0ib>+=eK=1FQ zIHa3{fC8QoqMNM3$Q1<9V&^!}(nCRSpqtW#>zkZ}7+gZDpv8{7S`tttf9XV&^vodD~FX*eQhlds%{T)E1E&w@H>ZYjvt<8)obtz0IO z$bN2akJoAX(P^B8&(eES;I|lxUrINL4(h5xQ`avHc1s~K_r)G zpB2w!`Fw4delYEGSy70V>%bdB$VG0cdgJITkUpI@|fNdiuKh z&Oayfugi+%j6IN#W($c<{M5_sR~!qL5%yB|)2N=kcKDf7=7E#D_w1VtXkaqi+9~7x z)IhC{`<*JzU!MI&b>MegA3V2FJA}gPT6m zi(~$}j4_`dwNCn;eDGrKEQR1hfeR6jq0VxIY&X~GSADzQR#SUjpwc*G` zbfWs$?Bay!h7HyN%4KnoOi`So?ECC!@eYBqPa{*hn#IE+IMuUl@RuzH!GksXV`Mc3 zw_-6`Rc?#t35JFa5=_+-q%apTP%v$KUyzQaCfx#>C9ovMSBDF)k(mRR=-Js2Tc$~ti z!-W*5BpY=H6x)Ev1LTeqSG5!HF5IXhJ#D(5Z?a_5o7(S-YobTF8}ZWWN@CbIbNO;)3j>RlCO_`}HL?Kv~e z?RI=BQ5KU%*13i&nYz%XqZTah&r%;^4CZzp+}vNtJyRn2tvT;++58WIF5L6w5?>p# z{&CMQNc_$4$Zr_Zske4-{=EgS&FxOPyV~@>m*7ZNu>apck_E%iH$eRqBdtEvV2H!k z@D^$kXo4r8<}EVL!3Hf(+_x=fTY*VFL|)_}`C2VPSAw3=3?ASZDubc$N0)oJ-`cKZ ze1?l85)ec7z!hCkZw~>Q1sVR}@)ZDHX$u&+p>g5$D{$9Fp-zaYfvk|w`%u&kHXrK= zF$3fmY@weYP~o9-@N2BWBY7|fjDj+Do1?xB$ikgLPJ0@@mj_hWaE@<>0Itfw4S(sO;25~6N+~N>3v_OfOMsaO5H8E?TLh3-@F4;A zcEnbM#~8bIWHJ+sWJuBgNo=JJy)nVTTqqA4h8iNcLpi(P0*ab7aCJoxo3(C$@IW9* z{Rkt9YsLcg&7rRgJxv?y|ADfjIvB=+pS-!8okW(#6Og3=?C7Gg=?yf*c}9 z>SSl7zS0;I?!#R+(At$R1C_J)^nemb&*P%T$)3tqz`m-3Vc4w@+Y=|a7xg%0h8Fm& zTZ@yS^hq{EpGVY(+oYA1-6Mrv)@|*Hl5j2y#FP-0ZEx?!*4;XF%hX3=G2xf)n7q7u6dza#BgzT_9_rIL5MA?g~32mV9I;#4%ro- z6o(m60vE!Oa*q*FIk)}e@>v=8HXqt=rz_2NCwaR}g{ZKe5HIUqxO*CB`_v%Zx^CqA zJ!i;lJZN^{DjTLB!^uBiW~A&D>5NxkG#S6QH89Z6qNRu!!w%%n{5`4l8KEC3eo{p*H2{XthtcAKN`zPQ=)_xt7 z=HRWW{SRat5JNkMyI?=h#7afFs}yCVUa5&m?s`veo1DN48$B)JrAsoY8Aq{m{fDU; zrnTWJF|kEEvI{%A`p$Ki8FpC>;&eQGIeb{QpM_DZ;SYF&ZcmK&@F>?YN|FPJOYCI7 zk*?{>t3T-YJn`Z(gOM@f8-l?|lXhZ1aWX2*MX9)vb6nqb_49yFg)wKP)EDdX`U5-S z-U-)O)>D}yc8+r#;cG#yCT~Nrt^79oJ5RP1M5ga@&iy$S(Z&l4n-Vu(DRssYz(`WAHE0aQh7-cMmDWPkzwszP< zUW>flM@ps@D>6Q%X+@fVPPH?-n(j5`2^u-(B924X=%Xd-tsBy6M7&XSE-t8w1(L>U;OHN_%*5)Y@*_}$!DtG_PafV5_BLJE$%WY{8jql< zH|T%xAr~_RdGbV*y)QwN5RDTc)c^&DgTAr(!OviXpn$S47vy+Ry+~1uwwSwjr}^tp zasICE2e{%ue?JZkBP!DNFOFYbd3*rA^>qk3I#3wfz|9!B?ZD@D8o&89x|o8WPaJZz zpc8HA@@r5hc1tV>q|O6&sj0a}sS3Cg+`C`Fx=$s86AG*qo{Da|U1lH*2FcrQ!?e@z zTWH)4RunX|1Rce`RUE`qVfpK!%np*vxXYwf8IEId<-H%;Aat@*BVmRjym} zJex_U4Bv(vIX?5!(b2cR<8U?g&m3XIozAy+G#|#au6>?org~GNo5iRTaz*m~%C_{`phzAr3?Dq9cSy$-HW=Y}fA2W_rNTJF$Ku zM`H6<=Duk>Y`|hYgmTk{3hNi5a?hMoeOb1A{fhc_nYwm~butMIy1KO&%`O{t!;irZHGR~wAA3c?<=G=$U zbH za!F{n41+hW5)&i-dC~Fp&?WWgJy8>1*;2{`+C58Lz;`?@$f(d?-XJ09IKHmUo!z@# ztf*$i&u+dX&y{mcj=oDQg-aIt6`e0z#-co?>wQEutHf59qj}@y9V+8}!m4Rlj-+h< zrIgnGg2aW57$&JZ=pVz+-j7~{6Jv@psZ%o#OQEANk{I`6%|Lo(K0JPL@eYRBU6hIQ zI>}SMEK-qsEc?8?Y<~X^r@XABgH&jH6m;zu>c6Agha-jZQ==zGOcf?F^)5=4U>-F# zx^uhk7te3Wi@0Y%wHq8r;jCH`j4z$zkjOSYnW{8PPfJZq=G4SiKKoZ?fFJylONwgKK`X#JDj-RPGQtbsdhHe+&A2WuH`CL zMGp-ZuFG>sv65||Oq%3t?l}G3h(V#8e=fMS$qhfbx_Be$G**Yn$Oc|QGu0NCuN9eE z4Gjz`(CRcWa8nJEkf@rZ;2sCD zF;9N{I4iaIoz~}NCW7@oFDAA<vn+Hr9q%KoPeReUo5 z^|W6v0q(5uW&HJrXO&a?iMI7$apkl$CCX&-znaiK6IZlGqQrwP$BKxqO4xx5d@Aw2^Q zB7~qrDst>_u=zByoI>MB)lB5!I zryw!+Pphg*9wTivz&-saF|=M=2Gfu36Td)sN^8ubWk zUY#Bx3I|t#*H{*1P&SDFAuscSphB{&J8{8)ANbsVLvxaMq@!~lSO@}e5WSySz7)~C zXd_w6f#H7AYO{^+D||zkE5wpKlu%ZF=lTnLid`Rf6zgB8e{TDlMhcyZu}Su9KBzSY z&FiOo=bK4H`_&raR1uDeXAZhW#+l`$og=p2-84ZhPrsOA%v{>(#=yeN1pF-ASawx0 zoTRf=jW=Gh*T%@J^CP|YeJ#6`*(s`J)lkuqft#j#B&L7O#k3W7LbIA$zI6YF z9P~k?M4MQ%F&VW5XK5?)TBkQyPbyTgT)m7;mNG3_DaV`PN0_fElYP06Dc5exQPyA| zIQ?+8DlWu)j)MCZhgBGJ4H{a}6<9Jb#WM+aR1N9svsM`9V~)xthT8YFus8yfSW`aV zEZF;*`J4ZQ2oZM$#@d`BPTkZIs=D6VWpy7x)2gX^@-1qqE%V#8t5=w-n_-a35zbg9 zKOd}5S(7kMA->fN>k!O;`N{jyx8@X6RSI1;UvDabQ(jjqL_T#*kG;4-vmr#FUMWe& zl{NlCzC}sjn&;`Zjue8x5qEXja0V+So!DO9yz=muN1eb#J8q&l^>^IIYZv1TrP!`9 zONCctik~w>$&>Rr%%GX@W!gwz>eXO2DPBvsVQ|$l;5zws`LAgLF-Db>d>w8blP&Mv z!qd~ca?9Q6(zw=322lAdepuTnH|;au!)nH@&ADW{Gf53UO`*p&G0WMPvqgBR55LN| zZj;ODWaKt~dBx4mm{+Wy@OET$M4Hd{s(D5Y-XY+}`}I5dpB2mmxwDdBDBxmkqBf__ zl)P>vg`edQcbH6%qvXRW8L?;2a5nCmf|f~wv8&)6=fI~!b8p*D*XY8gJhE2&=y!F{ zWSkkSxo8?oo$TI-U7Otrx1=KAiQ=VMgx(q4N{rR24QFbU{=L+pjId%4d|_VsXP^Y? zEHJZxw(4aks1L@k+3SlNk`4r z>uXs$V{-lrcu5J5OA)J#@T)KosPVJu`M9HDDn0xm^pbjSC-CrlLIj76q!_hcF81nG zcD1+TLlTxVX-#0P(!7RE+uJulVFB1p*B`U;M7JXcZ&YyeG>Z=2Nh!TEdu#I+lfdmX zG{H*pTLN{*PCjlvaINsIbF+vNU6=|3!T9xwy~T zvff`DeJ@2SiDmLDdyAJ|6Ev7#H0$uG!yGf@(Yxske@UoO(x8>1D}S~Hdahl^2TVIj zA$iUZik*5UA~d>7E=>vy=kvpF*;cwgS2?pUxC#gsi1dY`36S{>2rH6_8EK$5LFqEc zy$mA)%k@dB8HMuKhod0LTt|{D0g*r)RDJVEv(GgojO>f+pmUg$8XW`dSE@aT2zt}Q zPv@ajL%8(|H|$prvILh<59s-USEk>x?Jh(ndMW)cWN~QBAu%)~ZiKy1f|*Rjc?;I%_^Gr(FX`!D2J5L@g!0dBVdjQ_lU zN%2+?O#T|N#Sq#a994s>%*Cl}==?|mjt!{TlN&)-q!V{S6T&ApZ-!zhAF8f(A}Sh7 zAXYq9*OCxe1&K`nrGZA!zvTai4^0&52m(JD3f1Pmevr(;<9osZ1%(Z7*=%bt zfmVV>RH~TO##u4UtniX`9rQo5eopAccbS(u?06X-KIrOVWE5;tw7N%nxwX^PT6e5d=(L% z&i?%LVzX17lIik~&qq|rrZC{&E>tr0t1lrX0oGVDJaCl_fiTUG);ekL-Ogd+?l+M{ zaDnHub?pS}qZB2B2I^V8){`rjrX-V^1L0S@nU6mvZa29UHP&DgvGL>D zspXYc*x|9!)XM1Q34@Uez)ThI_@e%)p-^3XKu?djn0hcF6l(glg-3HGPY3Gd9r=q6 z)7*unC^f1pVI)~8kMVc?C#f`Gl#3*%uN217 z;v|-sPE2G8ykS)DiMKdf@q?>{s!Iy+f_E1CO1Y0&OLxU#O-11@k9VokXU}t%Me4fi z0v25HLOX|xi5zCKfJ)DN1up#S*Rw1pk)kR?d9C@Vr#K}iOXvPz{p#{}$TiUijaduB zL&eX!qa*W(R9zjD?*7`H@K^SmH%~Z8U~|lXCj9hjHf|E;=2#5xm`%hg1(3}QFMXm1LjJs$4WFSzx?7A5gJtGQ1$4v$9eX{v) z-du^eiMJ@fJ$!pJCA`s)kGP{92Xr}c1F%=x^EuK&EL+s# zGk1j9J1?%N3Vl6l;-7X-w{%m_E?upOknQjrJ(Kg8)uCGzqrxp6)jU8ghObs9W7WEd5|c=f>{v;e*+9Pb^bM=>x-BbnxLpbeHF-) z+`l~xtX#qU+3*KY+yRPP{HD)%`~*oe0A7f|?n&8IzFg3d(mH>}2sPKiJO-4+CQy^e zl>ztB*y!i!cEA$=MtY$7QFeC7{kxO)Z?8qYc#(>1htCCT^jxr@@J2zkzUlmz>)kd?L2iHZoGC!3Z( zf`%8dGHYmhXil(hK?Zm<-1B#XTck^0N?iOKg4AS3#&rXf87Zj|tIPWRC3vx2{6GtI z^GfY0r{9l^D1rAqmr9%HQ~?1=4vB2vUlummerPp}fu(8_2+Kc4m3oVbWRPacRgKWV z^LeI^Bk!=dGOuJ_v-DNS-Ah*b+U?3U2Y2@ba!|mGYXfLgr;2ex*Cv=aCXMB(Xo;&S zjM*!*UnalXlSJ~S3j@KEevO=b^2l2Q5;yN6YNWEx)X+<=PwIl_6`)2dQ8&hiq)%c`RT8j{!Mu2^NqS^|lsL&<|dC%5aaYh>Mi%-at9 zbNb*lbd|2Kt$yYtCQC`c7;L932UPp02T~Q7Pl2~P7MQyooduejc2pXN!Pz2ls*6#9 z$1n!pbN^Ah;y(X=>*HuLxrV-71}E#O@ZlqCL6*}&udq$_&#SYbCHZY>w$Hy4Fj8M` zaOsgArsj3;=wuLG&dZ+PXL1jw!Gnn14)wp}!4DkybkyDjHsg)WSJD4~#nw%1dkhpI zJ+LLL&RlzCFVfQEH#UzhR=wIzlx?1JedW8dx7+WXs+q^cV^3L#2sf{9ADF^xVqr~5 z?{{)##Uc^`YYZDO#I03?M!@EF3+AKSk){<>e=` zS3r}|4w?&Sy1P391P;L$EoI1Kss5m~%ZK~RL!`9@Nri*%-{`3>njly%Fc$U44p+hJ zSOm!aRp{_8!idWm09#<*@DelxfI~wm)8b(C^BX7Z3k%G$)fQX9F|c{E7=~a20D~I}gRf6Mm;c zw*UF%RkXknEt??B>$S1$lkQ!q~Fs8#mU{6^s1n z7pUqY>eE3-O(MdepVl1X-kLzk*2@K(D*X-xnhR~JooTh$DWbdPl&H5*r}o~<-L0Lt ziSt<{242mix+CQ;-G`DyQYQcGK6qZbTE2FqQl1qrE~=bbi(T8w@RAqJoqx1Qf0f9NO0d3`tB_?%FmF&xbfhv%=uk6GM#@*5X+RcoH;lYiuy-+V zxi#GOpV*ZC9o1Dk^vFJ1cDMlP9A_l7pap*q@Z~_eu&|mkAd)RyxQ2N+C|o^HiNvQ^ zwa=zeX$wS_8Xu)ZWQoC$ta?>K zPnNH_6v2X*s`>8dTjnnY(E9svpUs2hP3ikvyLz*Wr(&L+rFOMPE;@bQnoE4X}rccT(L!#qSZUr*2h^V51}6VyH6yafynwc&RV zY?5LD-iF+Ih`R^EDYXHfe}uuA3~jEzoRasBp(fqrQ;Rm1O=GFl*C^%4d8J4|K-87@wNYN9?Xz{tx+8OaTM+ZF7Zgxd@Dqe zfC9S{Tpd!i5I)F$1k*Yvbe%8<(*>eq4-5=&yKUxD00%3wPPnD)#4QhRZN+ZpdUyW+ zM%6^F!!Ho2K513v4xKDTRS$izEPf#EdZwGgE&X3Y4K9?RdCNy`g93TiKL%@$O?FR> zmRCZpU$j8W>!0ZCjfyrlA{Tw7OKZW0dDE6p10DmjBXKZb9QE0gei2{bT~TC}{*&rs zJE0$QxxaL*NyzO0knqOCY(;amBHzuW6kE-!hSS|SSn0hnKYr|wCfKjyIRsF(8V3bw zVQM~2V8}h|icpH*$bhWlL*LM?3C1h(LB-3bbtL6L?I%v~8<|`BygSa*|IQ!SwYo0M z`KNPBY~yUn#6t>GH;P(+!E~b+8#vWe&mMH~&5tj8n$z`t=u5&pYwzf*JNqL&71P(o zvXWl^6#QwG%XCwDx2(V>F>^1uG<<=b?sk+%@{iZ5ZGFypNhJ zU6|%no4&om*LmQqJhRT~gOd1SXkS$G|sjX)OPmYdG<#kTvbvfHXPlm3Tb&!a~=0C#Yb<)xl z+B%EAx{69DuZ}Wvnrh`KQw;07cdbDK;@MXE;I|Ry<#Rivfh0Lv_h0h6Lf`*j4}#+% z#7m;^Tbg0-<*2~AYmOn^en0+{Y#5Gc{TVfRMK@PIV>CZ4MQ8TF=_fq1Cg;=zcL&9N z>2>S=7ONrlfH~RFyji>2x<+@uCaYWdlf@0tn$sDV}};0mM? zpD6%;8o6(v0fFYN5UqUf>Q!T$$IidC5GCT&trxHnSyA$)e@{}Yf$!gU<* zLs59WB;`yPu)@I`7YlHYq40SCZz7Zn_;_;>DsUhCf9(LHEHvUV*6B?`(AGiX$_Dyw;A9p6r|`k%Pa6ij zP$vkY!-ogT(dGcyONbPLprYC<$W<~kKabY?BAncUd+(wM^F?U6K-lk$JRYV5#^@&u z#uEJzCK{kzLNvgi01JT;`J6cf$RW$%jl4XsrC*D4DWr0q`~}*Nk#kT_!U(4H!||;< z?4vjXX&(pch;HAHfd#UwYoMmu)p(UQa!89tGG#MzzS7r74LK$F70`9V$C03|C;P&v zssS3~L$H!1j(w|H8w0^_xNT5^n&>An5fl@t1LX>M2A1w5*szu;y~DRB=BR<6KMz(u zPrsLKbjRg(W(no_LTPUkZfV<$!Oi&>l9YNnpAi3QP5rAqQ4&X_ZFmK$a18_VFXj77 zk7`9kVI}mkeQitv>Gf_SONEzuift?cR{Eoll~Ap+hyg(d$R!vvDY+gJXS6CTGXzp2 zU=9=~p}-LqC@)$g6kgBYQgTh&Q^{SG%e2ijkWDSV_kktpLpcpGsHZNg-YjmMo>cnF zeSd<-boC8-e!x6-?Ip{=#Hnbzt71M+=*g&{(RdJt=U1j0wVp5y!Amq1uJw!Fg=H?l zPJZI1dc=KjM_X*W8K{s3_8CrgJ^M>=F`QFrWm{j?1-fb<4(m9$g3e@?4iP@tW z;naTcJZu<6X@H?Uf9PVxw`v~;F|CW?_K@!_3(+7CE`+<~-rZyijCk9S2g?J-_VAHd zcahmCt^XgFYBk8-ZOuL_$6FOliSeZzeHMQN_J^_$8)=mWseiiXD?eDxITsuL0lT97 z?dZ-RU$u@C*ZAizm(vUg26Ilm+cI+0f7Svw*S*1%EbU8lPS`-0TKmVs9JE> z?gVC;vB@Y~*$n!DV6a4753t|HA$19I>4G2X#eN?%klGgoug!Qr{iAhnd?=lPDLCDuR!*7Y@Ck9-?y4YCrQgQ?rUAboe;(MfM* z4sMC9Ldf-c&tMTFTpH^Pp1W4-G?aaF4*v7(*<%?mE@+h;@NsYmI_cNytec?1NL2A# zio2P8j{Al1?_%2?=$~`naBt$DYM#M5?TQSFNZ-z#CGSNmUNw)g{8vI!N?sAabd;3! zlqO$9bVG0uAq=SI`7ZX)F?fA)*YPBzmq9cpSVY}R5Jgl41IH|3pm@6!^}QD6a=Hjs zU<9uPHR4%tJSVR{dBTka=sUWubc2q3)9#J_J1>N2O7P>JrrDR8AxsT;VEOia z!Pe)ffld)df-s}nn&%<{nP+eh`pkOGwf5oWiHW;gh4p~}2wSAuBAW6c_ZJ{!jtl?( z&J{%Hx4y<6M{j+05wyRNRSvLj?>(TZPzO5Wk^s*^t+#LeDjvzg%Mkzy*lS=3B7Adl zHCPg%)z$mXsQidJabONXY- z=~ZUk^O?2Z58$URV808gzPek%)pvH2vGqc8pp~8G63JzjS=vlXVCCcb^1goTb*q_3 zIh~mLWp5ghKE~Gdn^Rm>J;EbYE|hKk`>IewX2hB&meX&YiN$5*Wxta)R!)?g7Q9t6 zUsXJa+u=PwH8~twb8CL|3WU*~(X(=Ms{-D5Ec#DW;`C{tC6;$DOjw=rbs?G8)knvS zv6Nlf4`$SpDqJR5%2gN}5|AQg@v^cDVtJ0^_U z=W_~gY1-s;`r5Wd)4LxtlT4e+=C$L@H*d|cc&0P>+LfE<=A71{vTeLwwn+zligL~- z$i5nvEI!$_Tl$`~03Uze((v6Yo*Uo&2?kVz-8cn_!f5$}`_$OY%`F-$3rQSsQ4l|R z^a|`O#(+k+w&%Cfn4;dZf1tA+zM+Hz zovA=HgnA(xI?A4g1Em-YfC1$Y7s!GTGG*vKf;#IpM0R{}Dr7!>6Pzb1DtafH=34_} z{Vec3>|00AAJe_`i18rRSo#)k*O9}KS|IE~-gjv1MFViQI|zKMC^r}-I3W-9eQ?MJ zXXQY#kDej`Ka8xcSs62kUjU{!gkq`>Ue(kL0b?ES6=RiOOM>(9B$rnz{Y$>A`FR>q zsA_|A6f#0GZ}k%?B}^!SCs!iGz8oDn0_zQOK=x4(6<&3|9&;BI5|yijNR7nMPV zM*esO>xq{;j(%lGNVT`%J^At%_7caZw8kK^{Kt1)t&X=Xe@QmVn%**ckm zouHz7=;GmV8;(zQm2(U=Bl_H}o&4iQsHxJ=?P0xX zX=JQB+0=OBe&_T(JFnNIr=UU=OweDGlGOf{%xI+B`c+3I)4#Q=*2wHJHd1-Co0mc@ z(Y9(W>dN}HtDJX-5#v@u#Os0*GlA*g+=pLop2$7-z?k-NlaFd!T49WK6n%!1s;>cm z!v6Uq9^+ndsmRWaIs7Or;$QlUKKdL0!W{vaGkIdu%?dK$EoaaSPt zzyJ>L1#oO52(hGu#O$Qu)vIYBO1}klG-A%9*v8Vhx7%lmfeQdU29(Wg7!wzlixwj) z7PyXGfG7#ms)Aojkw-Tzo`LxqIwjFVfIHpV6=<8t+zXc@5%7|qG0E>vjY8W4B-cP) zQUgDQ(@Oih_2N@vV{_mNlMiLWfYgSr-6r|QC=3$?aX^pnPWgYVQ+Q5$ip{zN0Y8X1 zT5w~bdMji5AwYYYz;uI2LCTo4Q#6Uz12>dK7z#8|=aMd`0bw1j)wZe&Tr)8c(1sEs z(U{>To^tJ%Gu)n@Jbru?-l;#}?tkD|+!b`DqntH9zQJqW-Y_QtW}`HN1EtXs7=&6= zVc^p01uO9#nnRq3yOO+eMM_Ev$xPw)@K@rm@TZU7Uz(XnG+7&@ zX@_2R%x`lVw;fcaF$rqa6#}N_Qu$Qp+;)TB9X$!wrhL@s?uCMWcM&~{-<#*zd2|0R z%N|&k2@%Geu_IP$33Q1gOHM_1JF9c7QuaP^;*<2^o2o6q(Sw85hEic7*e~}`7?bf7 z{kJq-m*F!db8D*SD2RX>P&BD=y1k3&pL?gsXq%X{LX;ID@{nH1kBEIRhA0L7~O0OkltNWXL)M@6O(~a z|M&27+TozF-Z0~e&&juoR=!`PcZ#bfaF zA6@>{$i|xn_4nO#oO`FG?3$|X-;IAKvQzM8d+OfzlY4C=Lj-d-be)4!1sFXb zoD%}>R4R<%r$G0$WcI>n)$Vl@5V*#nI(+i{xiKsgXdS;!{hLF+8B+i8<_Sm0iALG* zYCeGKLTN83{_ejc@o{)K%5m=f`3R20f9~Dj;`(+guEieBx6}fby5J)Vure8AqdR&6 zs9O$&ts|sGBW$C?4UKm;0QU!B!jOc6on0=J$6F4gnd7sayLSCBhSV=P@ol%q07r<- z)Tlv)2yRD@E;y+JAOb~cAxQuzn%7s~ThJ$egFl3J24fQw;++fdLeY5x`$$jt1XouS zH1L663z={sz_ z-)$8r&rv!Y* znORvV#B_H5#Q`@w0y;n+a@EolHr{eNo-w3t0B_+me+h z)RM3@fYA2!Xx^;v=FPr*d`fP8*=D8x@a<)HjJ02^8Lwm1vj{J?pn`j_&o(OBhe>FO z$Ix+OtqlMPy7e6Q@Ff)1%Kbn|JsK&xdFy3xG3arorz+W&)PtfOqbmeOyrmnSxx)%K z{f$`8rldIZoIQ~%imx5T&et-xgj(d&oO!kx(YMhSJ4?K^ItcE%j<6O*wKSFNl;28s zcKW@GYW3SyMgE<;wCSdNBa0zt_=G<6w&0CvleMR<+moT&5H#9LP{-abw!x8;DkB@( z$CUoM%JM476|c6=JLlZ$-z2eHWju`&Q2BM#Ld2`-sXGf_Ayet@^|$b`)bcHV+$zMa z@Hc`GicW4`9Zl#oubg>QWX+9ai5mh(7)Nv=okyM2p0}zsr!Tkl-rK!`R_FE@^-$N>1dqxA^eZ0SpHrm^fkMrt&5d3k zijN+S^snDC5m$6IzT>oI;W_sAV3i>g^U&XJX)|xDb&6@L+1|f-{f9gkuLhK1=pzoI zLs^(kPfGLOXoG1La7Y$YKwgVd2N3a7{jjEO2l!*59riKb4SLv3EiH25C~q}5tN!K* z2_U-WPeXa+sknw4figW<9T?Ehh1JYu>lPy5&@-go{0G_65TFnGQc=zuX7YjCSL|tT zZ!f0?8&7b(wBq-G6Evs-VFd|9ZCS=5z%_hQ^Wffm+7$v4*9X}kC50K%CWtZ*u0IU9 zuj^59f%7_mfWY@N;Hftv8`c7(^yR?W9i+Q$NH_M%VRTf&1#iib1e_{Z0Zk5e`?6|l zEfF9T4X_}liw&%~ZOk%oZaKnitUGM$2ChTiV60;d z`lXnHCIdmgq9Y@1^(DD@wP5Wc`$TayK>e@)D)OYbiNQ2`CUkI(pau}dR8e@-a*6>s zRNDmZiswISb?D?4HA>3qq~h1F<6ACz4ZVvdEx~(UKaVvrX-^8x-|D|`>u~8xyLP5c=9ae&~V|+uegwj!NFhDf3NH-%rGH>a5G1%xeH%e&fztH2GxMl1stc69_`;DT5%rUk# z&L%I?`5h%h<*z5UZ)4?d=_j&Tl#6ivcL4s(bIhrdk|$-Dd6AThhoH_oKs58B4xOQR zxO6Ldf2nadFt{S8CK&o~l?;ZpOPyI_&J!Jt{clH(Z{A8-#^RRxJkaeOzP>HTJqHvG z`UsU6<1QKdZNZfOaTS}2v@QZ9vM(t*;p7$$EyWY+_)jYxOaw2 zAS;Nqin`L)KIp;Q>FiBX~1GbDV=8= zo!W-pdnD7XWbhCq@BO^OqT*MVEAyFf()dacbIHZnWHk)4dT8S&oJvD>jR}L5|K?b} z|B}n7MDew&r43_E(YstJ$yiblmRVtRGo!_jTqZH!dQRXco@i4um~lDA?Wk>hbo62k zW2$YUmBetR8lSG7y~hb!c1`TZzP-SLBU6`F-L6n{x!~(73t0)lUNIBb>cQoK8o$r7 z#ba0s`Hq#JV06x#@l*l3V_OH7Gpn2#cI+Cl1()fS(<3u-(fle-WrMS>7RPz0mqH{^cH6Ia74u zuF$L&Q3;8TyuY+w`j;7I^oLkr4_Eyb{4fuOiq)W}TtG=@KR!K%5ZF~g^^%g3vKybd z>mZN^fl49%Y%O*VuNl})r>m!AimRb&kvLg8Jg^2W4zASeJOHSIwNf_;X4Gd0KKsuL z5av(9G@%!9+)WDel2467OJqIv1xQ+w{8byBP(Cu>pMMS+GcFz`p?XswFZFLq;}f&g z-MrCoZv{{>3fu-c=5kpIc<-QM>x5wHKTr&*K;D0Wt_WZ5{qxJtszCx{DIV^Y07#Xs zowA)4p|$&8G%{i3J7tneNecp zxDKs~L0)?DxaxQjN1Z(a!6HQ#ZeFuD-tLuA^(0~SlKRayY#<(WjZ1OweXWV7jOsoJ z+3Zh{6@Sw^Nx)6bHq`hN)V&VdTz-{UNt5TW>FSLWwJhcBd!L%YR*h+*6B4qXJbCD$ zmgY`*H94TvZwUIRdiqHC*Ye3Wv$m1yPdR3!-}Nw{A}{HU^x0FC+|s*JCWWWP5KdaG z#9UUTx~1=OD&yQWbhm6%E}Kr&o&M6X$Pje-w)A>T*;(5spsQrvuP zs#kG`#idl!zUk00ntKDsMA^wVk%OT+M)R+5yaRJBQz7wS%K@vKMF#kfG z!LOrvvUguW5H-{uh>*Alp}YX&GpxnGcZF&GfWw(g5GKu|bPcW{1t=e^sema*_EQuZ z`c6Yg+51}xd*O)M^3Z*7n9ex*DJS*6wGAcvPf3HpKUDL&{t>8$+HNc!MGA(5Of2FBN>$}NHj!>NyK@f-tG#2jNbs^>LYNnzzRX(RH&%S zxB4OuCQ-jZpBU?)6+5+=cbkqzZ9BCKpsJnFYyc(Rq%jT=TP!QTIlQz?t*Eq=fqXGT z*iRt5dd}*KIJEH~-M$y(-TBZ6-L&iVae#>T6R&0Gw|wj|j=Q5c=1M{mkVirK)NPO~%zU)#yr| z7lBrzuAZKPxEAeg-$n?ZTAYj8AwnqPP&n?F%UQG~ZO9^LNJ`{Z(g-Accn37HRjIhO z25Df0tqM`SY6d+O^9a2)RyCK}7$G6s?dwcOS7!w&INSwyA~v;NiCxT@k}*3o?$d#A2^iM%W<>E6zjFG_A&bT22U)h6E%v4unYGN@MC`ZY zGiR-y*6aKC66jP$C-3fWUSh0_(%@8BRmTVPzaII}h0$+mczC`c1VbQt9wye4x-((T z^0ljeG9)j*KSQ;VBv9oADe5_E&0@)O?{XVYGaj7su{eGD^+3UEizPGGskp-h)k3x_=#v`Pa6-a%>@+y70@G*fo zrEC%%6tG`Ly6{`MbEX`5Y3RMPHPx3EIOe^gX5TWImYT|?w<5y6Udqk=3qLR#Z7QFe zGhSb3(WahaAZ0dL3QKh{z|M{jO1MG72!*wT*&`s3GfGrSUT;PQ_Zu%x2BHPb#dYg8 zI`v0Miyr=W3Ce~!JfHC*<2+qmAwRgWK3V!H8T{in2}zF)*N%R0YJXIzuSQY35J?J= zI&gGoNieNZ>vx`SKYfS&2?5V8d~C$&-dqhba74Z5@`7&^mB{tk9R0nfcfH%KZnL zD=0|ysVpJqxNZIN(di>vAe=*)Kx8b#3i>Y3W_fdbpuuiU{rdIF{Rs3D>#J7=UV?|{^&qA3WviNO14PvNXN z3_^WmOrUUmdjtJbYp~fHfvmz5>*iJjeEn1@Q@1+BZMU;gJ0IdI*MZKOA@qS=h(`r| z(qcd@H@n#PU+pRaGauR)&`hrF>=675z%=QBf0=Lpei!|-T$Nt`wlEo07g;dtgf$)@ zXR+ZOFTA`i7(3KW0?w(-ld4@`x)B14JUu;Yo4~i1_!M5N2DFr*+5+0I35Xidl@iA) zz7Me92ia?QBCYplqZIUP73z=te`LJ}T+V&lKYkL~TsEaLqoI;S(Xc`^w0Bli6q?$Z z84XmTE$y>`O1oSrT1rDIsgRZmE$#pNIOD#b`}gnly6(qgbatNK&p3|v@g8920x~mT zo93KQ60o@{Rz3RIu}kKe-9RhunAQ*Z{hT*AXfuavC<3+}(SeH98&iwk3YBx7{jO>8 zYjWa>+$~3B1s;^}Dh-B=nJ{S?mRLpkZ=LrJ%;A?inwgph0Y$g?%Z!Ht!yL)_{(bdm z^>~K9YNR8xP~I=Y`d0kAvF)~=?_Gw^UY2Sm9DN$kh}=@4k3;F|+HGjRm)n27(T!ko zf$kqJ>f28iw?ZjZH({N`eDvyaQ7-QAhQOt;AlTkW<)SQ_q^IrP&Fm?r(U)7mvY zZ97w6ewNZ;<2>-9Wz1EYQrf%C_zDE7Bj?RCZPB#e8WLASwe3G zs@J;=9@Z@V=Fl!z-1_iwYhA9;x&n?B(KsO*J{Z3BXwb|p2r?3V5L+83e?uz3Ax8$6 z_uCPxnEbl;ix0YLFEr?C;qzPkA*J!EwSiZ4qMlHyxRcd>-6h=pm3gn+PXAb=KQce% zUy%n2PyN)4|Gm{=9dYyZJME_OfyB z3*^UoX!G`uRXX2XITM*i_$2L?-irP1Al-TMlgqbhzDp%eh*poJXzAkE$gxjRGpox< zYzZ+;3^DJiefwrZWm0KEkI&`HY*wXGooAoyEy>Gg?=T!Q3XO97b||MS+xEfJr;a?k zGqY8MvmWmR6ZD1ZRx4vpg@Ai{I8HJ%9C4gD{cc*dZoP0BKIH1=qIId?Gfx!EfcVVg z%TKm>R#}g7301>&`}Xn8372hIwic^QYq9DCDa}Hky*#l*`9&(+t-N*vqEN z&UA2BSi(N=+avFcO0{u7hFqOhe4NU(1iy~`3Ho!FE>sp%*GM`bb?i)gh2Z{@%M#7= z7KV+(VGVlafw*IP{;i2*i4w#y_6 zfJXZjb0fwnvz7>!!FpEAs#RHf-egAEmGpXZVjGbTH{xKG(kGrpi^5ST6IW31<_xh( zG;#c1lhlU}7?Dv8s)hi0AvZRqD0^J@ecKtP&*688c>~UskgqnwpN@0GkQ_0Y(Lj|NXCvsNE5Uh){Pj!zz}=cN<)=avDl3!{)y=zvy{_h| z^l7zf#P5Rh`U!>mkJV5`G`G}Y1T52v)ti~`bvCN2{x~9`*Mg##m1O~KJCki%0vwf3 z$9@)l=9iWxqHw=Vk*PT@s`OpcFbE;8ZKl;iyBV#W`rQ^*5*4Tn$W#iog zPqm7N+L;G9q(=^N$pj0e)`gqZ#nmRQha#qEi^Yk3P(%x&n7K4L^KFno*}s>olTVed zlUR`(cX`v%T@Hy(I%j*ds>GA>;`WU<+_69V-gL2^(4Dnhbv_SOGoM!3o#R~BemF*CNw47QjA~#D>aBpvB6Hn7 z?Fi|>OpR5C#jkFi)oa=yXfE8-BWFR^PYY?n>(u zmumaXSLqsu{t{Z!(V9Oi!#?48lKzp~d*_YUozuR|#;mjU7CRz;imNX0VW9i2?9mUl z?c)50Z)EGf;aC(J7`K|sz{Mb4dvEaA&L)kBvD8(47R7hgmBmRf+-^Ehy?*V1Qv>?% z#~n%+>`j^LyK${@uH4+S++o^hJ(#{B%a?G4Pg1>`+|ps_*(32_uf0=kYWtRipjxZ5 zk+RIad#evx4@Di6%CHvA^s1zmk+`02i2SW9+BH4IDmz}z)ClB1U>CA{u&|ivl z(zW2$sE7ybjZ$oe|Dt!{o3x*L?jCdAhwWWxKAf8NB)Vw8z~$A0&4W;xE5KB}1h$Y7 zT^Y<24E@xstZwu3>q)xg$acIBs?KrJ+);Ps8hiWrpv$!?eU`m5W2J_`#K74&6g3{$ z9w3Fq{NhTeJIkmHTal^C+ub#`(@p-ozqAVkPY?or^FeqO=yJ5L9^-{7!8Y;K`E7TEAbyxH`KKMrC9&4+W{2 z@@kyP&zB(_$)K5_#=8vbH5)eU2lW)3TyGFeH@bbvU)x9~y<@*0A>ru}9FRANik3eQ z$G>ZUolUz7ZE$#qpGE>)^nT!CKtYY|P*|w%f!(S3D9pTY+3g=gEv9!Zw+|4vbm4;R=sE z`Mq@+qw*={)YQa5sn#Dm4omMvaA+o7Eu!6a9|1Z7coo_hB_pG)OT-P}vW!vRveVNU zvKuLCm;ZHsK8FtwSw>$_%^3{PedE$xM>Cstxhh*D*}ZK`j5S8wIMJzC8A0RWV>*~912IB%6#D)nBArmpRg z*Idoa5`-{Sm1>TEa`+-<-+L@bN_XXf*f=Zbm>m^phW{JD=|@2O_&f z@O~)VjS$*3+-x-_>X2E{_CDg^r(yfAcTS+xkQy54`4XI=DJ(TDfaY}C;!4U_dI$CCAYbwzx2*;I|Lh}u=jE47wS6C zzBS*{IKH4pRISv-F+|$V=+RTJ@n6fIgnf)T9w2IUrlON)Jfo~F;r+}T$tHar!)kZW zR(Gw->ERiPHF~|F#Qs#)WHZ}mK4FrGri`#KRzxkACfinsPNVbph8)5ImgcQuqYuy8}#&z^nSRfvZD^vz@Xqh%#EbXOXc#RQlgG zA8iDt2{xBG>vUi!8~{Ol(bAYDYRY2u=A7?WadyRY*x9AVbx8DiueY5U1;A2^tY zAz2Ms7`6$ybLv*f1AF9sv7D@sxHI9$~GZ@Ifyf`G=986PYq6SFho4OcA~V$PVS z_)AzuqPWU#2VW!wG7!?ugq|>thH^gz#^n?iOH+H3vml^SVkG7rlm+X0#rQj%+`vg0 z;eVYdCnsl0dFV)h1i&@^a*`}pYk9;i7|Y#zs!ptE?;cj@kBvCGr#@Kod?kQ2WHsj zjyiOi*B|SfH(q8qZSIbUNo+crb;6rA?V2|4ZJF*_k9SvQbiz5|{-x6*QA21KgS!=Y z+raK>HA{Vh72ILCudO8{q`XLYK)pI)6Fws~Ev?YI-p4U6txrDx<{0{I6Yo+xgf~AJiYaS99!rvI|$_^0S`m0SQ$+4soj{Wt=v> zJ9ad0QA)Sws9uGF{;+d=Z~4*z0Xwa%;w23Yi8aSmQs3RMH!8SnT9=t;HehShU{pGA zapxos3FfKzZ`r;$mA(o2N1KdWMrU{)Qnu2&ANTJh(W<73_Z%xiPs!X5ZBnw)Jfd9X z!o=^l56G_1&iZ{(SNc>(`B|@*@w#s9m*?dpQbF-%PRV(|kI0gE1~1$Bqb}nOZzKzr zELZ^dq^R}^rS<&j{>LA!&3iSa4YPM4uI=In>oc}6y#@R9^|J?tq=FrvW{wTJ@WkP{ z^Q~-r-|=wIzG-cFONYUpso@YW9^jrx3RBKJWUeuLg4bg#^7J5o^5C%>qPGM*JSVI1 ztrC=d^m$hmZ}0odp@#=73I(VeePm~|c$=Qw>wmi!nQUP?BrDV(KXWtwWFsG1S9D<@ zUz?D-EG#nu60$du78MW&YWAQ zCCL07pi?ZeMr&~*Z$g|Sg*U9j)aLaSMtc&72O!xU!my;*gcD`RwYk}&yR?mKQR$G? zB<#sG(zgZY9eW4%IfZ_XjfE-_9E}7uQ2*~0HE%z)6LuhoMuo}KXwKdWMyNF4UY!(E zU(okYB@Th%XAHf6;5_zox*#4_fUTF+PI3agwFm7M5r;4~zFuwUq7SxD@UPwTq$rYR z`F*n5?9;ZhOCKAm&b8O?fs>#cdAP8t4jS6qC*1X3dZ7@-A~Zy1rddw#{j}-M2^Y|m z_whUBS7oY!k{C{+Dqh?rm05Q6Z!haFp18d76?k=-g}hR=UN^2b)hpIUM&Rw@_7geXY3wdVB|<9i?{x4N zZuHI$--S(f^9*;`60S#;Pmim`hXj;lYK4#APcBw+}}x3lO#Di@NDj2bd|?nZD+C7SAvH7_t)krf*jD5~zG z$-kC=|0c5+l?h4*r)n)1wp>3m;ogQXjawG$ZM;*|rm^SNKi1zHXGu|X2q*=)2REY9Wd*2Gpu$|^V`7WKEuui;ODks9fztMH>W|5mcw zCvXL_=DwVGBKGNcP504W*i!rYs<#5!mc*z{Gr=b|IeFJkd`2E~SWht1c9ymTHw&(A z+*%yQVhJ(I7<$6Y%6kVRB-M|MG|D$>V5j$)*k6>~r?W*=Xiv$dZBtErK7=yBiY@!` zepQCfw@S1zic7ayrN@=G8mD<^;LuuRqTaHfsWV0QzVZ}CyDOk-s8CQfQ}8zh@qyE+ ziqt*dQ4r{usOtGa$)-3ooC;u6eru@|(K&W3jBh9ov_qhqoWC}7i~6dL}C>%yL1E2cNzBi1cJO4(5|7GJM}g%t%N#c~J? zU9TaiF#7T1GNuqQCB*Qqji;kkqR2WNYj0Kkf`b!qq;T9X9M?$9`8mEdKtcY1iSxd? zRFj1XF{&?5Z`bfJtM%bK@Ko#2P_+w}O2@G6w!qr7O)piusuP>Jy4Q%5g&lqQdCM6s zM>9F*X8804cVE5)ol0R}+9>s6&Elfd~pd4}d8&M(1_ww|t%(21AaSKThn z1G$b& zlRy-zCFmzw@qlVGZ4(jqu5Z+UQ3Zh11MoUHjFeWfBhMKS6p36sxlq*RyH-S06tZ6r z+=aahTGDW|&C3Ew$|-w%Dey}?qIQfcf-_+Cib&Gir4jfe_36{6BbDbe)y8P46=X(x zz46aALkyReQ`?V9G^)Q7P6%6i>I(YT@(Bwofnb38Z8K{q)aZgqhe7kxr=n=%$9%#3 z25pWN>Vuv+s}}5vVk2u=)O>sp=Vsv@p~^o(Pw9B!eFRq*L8F#{`T#EJa}Xb z5b5~&>_6?rAU&lZp$}@y;p7yP3X-(mym9p5N`^-Wu28z%yrhTrS0@?{RDvh2Ln1Eh zk~!du=O9v?g4QvF>h8-iMW+FUtY*q&zuw{(`ruHEtMycwI9FTYA>>D}JnU-ThOITJtbK2LgEIA$?Y%`+C( zRt=R#pT!t%@$%&{2)tF&hXajHvhn$S!%uXU@w>bvu< z4S5^eS~{8~@0_iUPE+o4f3m4STk4;qDbY$vNBT^b2)b3oTkf43w9#tAqqJRiHVp^b zmA4){vt-?ssnIN@L;UQIQ(WzzV}6YhAihAstimCS9F~jxOW&SHcs7m zy(Wpuu1N(=N=SDk(@mUr9WmRjT5nU4Me}kNq0V?F;@PYS0vRmy707_FnE;|A_=sJH zHg4OdMFV!?8ek$nB)CH80Met&$;|-K^&W`r1<}Cu=$=`Of0;3!`x5IPQXvdZK|TR_58W)RZ=D3$sB>qZS zhy5E+`4)3>GJ&vh+@CA!L}_3o89#VW@Mk5ss;0o~D9X_&MLck7C`bcJtl=ZPivkUC zRzYmIe*J0Cx`QpyBBSX+e{rfQ#qrkS|{-vhJ$ zKV){>I5g>m*Mt(#-==)QOpJMS)4%aaX82UN~oQGXfIYJQGmo>G|R#*LehugcWv9KZK$=8 z7odZb>Q2C(_BByqAL3SyIY3FtlQw&bEmD$_9h1eL%A@C_7 z(F+Xyyerz`o*AO+2l@X8d-lrtj4zY~`S9UG77xZIQ>qUFq4KB@=U8ZFhaJDkRiKH2 z?Oruq3hK{ZBkvEEE~R6xM2B*XET#m;`%^%r=g zPV5haV&!Q4!tKZ9=9S!k_r)_cK*G}F(j`IvO^-5p4;d$dWAvaTx0n-YDmbkcV?cTo zaOP2UwU76`M$fu%>YuLZ$VBcUMGdOHEiTUb;`BkVI8Ud#aAxOF7cy}^#;sPWN6(LT zNa5WF2|kYP>+@QdMPKs$&&vH*{`thp?c1XraZp#-xBYpa#iK_I>(!azDm)>@njt55 za=eq=vvbnPM{AmRn>21^yw<9e`sNofeI}Qp(GpEMbp7N7th0;xs?GW*6$r*?Ki(U^ zd$P~6CDB(N%kI_MDEE3YV`MID{^buGI1Kh6aT;jqhlVgVF0Qw&l?RKY~o#M4j( zahSA_+JYlic&pm>{>${#v0V2<96tLZN=b7BEt;a57YrxOUXkGhnw>bx68=+y?gd7yZ_G^ zc*Lj~B2iPvWPP*^o{>}&2Xdwf#?B6^wl*-vogv=zSymU1g#NmbZhE!Jg| zt+<0a22t>*A-oBe#9fNiNmKOCrZrC^E<#dci?KEN#UY09Ys~fwKSJq#K;!@A5+t6o zpnlo=9zS+$Hl=a7UL(>BEp8u}=~-jW!g;^dOF)1L=jVMWjpiSK0_O3H7n9+$r)*%5 zZqCg?ldMwdfDK9MZSJEo7k>B~I9Vsre<%_baSj)ylm-S``1%dM6&`LFt6PbI>ZgWH za~AUObgREDC?GHtS|8IauYcZiMlUQH>*gwyw$Wc~6NSNEk@op7{?1Uc__4|ll*SThXzoLDcSkSNIZG?-X<_^NO;b47Qo}n>=pP4svnD@(ZEv5} zm8!6=O-ro3292IOMij-LHuBBT(}*dG$aT~guq?Foe-impFvbq+c-Lg-!I^WFdm0&2 z#U64BGD2gn!)_bs$rdzt=$i=MkzS}if=s*~N!XJiy|FU&tRV=4UNG-rf(MT}4sK(K z(wIY0)G4)kC=>0J*bS$2^+VHL(ws*#NK8cxfSO<|Ms#>TQ>R-) zvMCF{faqU##>n459vn{ATN0}njotRZsi0|2+%yVb0z=22SA(7UwSnd=r?P=W%W!}+ z85HVjF}- z$_75Q1O|#CZ9=^UQ>8k%6?Yv*%i7}DU*ZKAEdqg;%5}R+#P9K zRn}H-YUbjx{Pj_K8W?Y`dEUzU6@~%YcV5P9d5|Zms&&+A(euPY{`w*5M_TKDb^704 z7WqSV$tlz6lZ$y%=FU;;D3evv|Fb?dy3`WtV?O4N2b{7kY!eqbvoYy@xN*SVP2#tX zU2+~wwY$By5zN91gMlB;Un}1DHi9wQgXZAp)qLnb*hSF;k)_ zhsY6EoB*#+1bWCGITA{eCV187tJ)&{ga8bH6`O^Y(OZxcMm{j+tE_X<8fmZpU9Y@2 zp{NGIOVQDUw3+5D=ly$&(#L!SG!c#@Me|7g;`1$#NRicMP9^h1aWPvYUy7bhMU6sCth4N+%IdoDZ0Jqm< z9;#Om$H{^}LFthq?o>6%oKygJDCq(@!6va&n!-?Dh5`$PDv{fj^eKL82xkagseUcZt5>x>Jipt* zTMtGH&`p5^jRky9rb5D)~*y>wX43{oH<{~+HcE!%fjdoi=><84KddW zXMRt4mYDDPw}7ZPYN45;!acFJ+kGaEU-1)uZ@Rwr&h{4%$P0gBHgW8lcw8^))Oh$Z zLNBw&FVQ0Erhwko9DaR^sz)zFYSvzzdVlef3tt3y--Ioz6E>W$kyN}xW)h=n0~DfI zT7BR=GQ-2a-g7#B{5Xlu;U%UEV9i8IWyeBg+o{ZGUS7M4)LE z`3VoiIyC!gE?U0agqECZ{zF(f28f%f=;-LAI8kIenw%KS`Nn2lv+oG9{qzoJxKy)4 zSNcA9&WMVTWUo;MGsp1`mr>!wy~!c4azu~=O6X`A6x`w6x#I?N-Cjm*gbb8S7of=) zzlr58|C(0D^bs%aLz7AUO?YsTm93_kNDr%-^00c{dD_t z$2+VJn5Q@2a~`xf$m@NgJkgL>yZCO^nY$&&=zpt@sWdi>r;Ls^+-4-bn0(f8s_vmf zKiC+*Um{%88saUYi@>Hg^*bj}OF~-F{tF(fbtPjyJm473`s?%8fP!Fr8JXQc4rk&l z=_P$96kov^Xv<8#zPZC9lo;JyD^}?>W72b7_yEGtcNWAk@0g!}<`(+!k-}kTfwC^n)@0{V zdmQPZh5gad8x#V!-LVviw#2xZUJ;iibTn($`1vS`B4$}3}pYtV;- zmpCzIhc+;i_x3I#!02-6w77rpJ18KLZO3`zlBri3~ee9&gY6XHpX&3Lb--;j+9 zXVe?4VU`z2!Jr(AiYz`72h7{}=jXg&3*0>5;E$*0hooSPJ7zDkPe^0WR7URRfN+=Z zTt$-P;fybZJ+6NxBK|W`-K+sj^hW4TeZlktjVDjd6zFaG>F8m$FqV6hIQx?=768Xr zP2#;e<$l%b53N9cD?MFZ8GYYtR7vHIn~Ay(Y_$C-tE8QPxsw3}>Fdcxc4(j*kvtiE zD_sw!%{hREp%NZfDBlWIQvvA_C6sJXCfbnuQ`y;L;eaJqQ2qo3u6Gs=!AQGHgDeSF^k%pWh>Dn z0K#DbAQw$sht^93!s zs|PoXtHT(BjIlDiGHzYH%BY-$wwj8#WU_xG2)x@$GR3M*1w99?e|Fwi-ObRO;DU<> znG$k)=}vr-e+;Cc>729wdgKfEcH`SqHCiok(~#W$&>rw~doUX3Ub%Ly9gDgGYEKpn zff6|rwIRsl$Mm)_`gHKHx$Rq5W#jlUusj^6m#%_)2u=7Vs^|I)SrkxkpiU;$Ou+Tm zp#j-&D{pmF1z(%rX%q6A!N72iBKxxSB>L2YQwW9)6xy?;_7h-m!RK)=W@_rcVq<*V zrRDk{y@j=Q82T)%v#g1IzPtY@Y3p@SVLD z!;v|_PB+q2G8@zV4A}To#c3y*4nud1q`}4Hah`JYB|9LDVc`v9k0F4wufK8_KMLTH ziQ%KHY8lK%5-s4R1p|V!hfqYh%EjWjuu>P$p+PzGF)ACx4eAvG&gR76#3|T3f0e&y z!1_7?K|wWC7duVphaaB>Eh|4t(B(c;qH~K+Cc&>O0=Wx(m=i@2s~g5wRcb!Z@4 zSF&U3oC6>@OOKCOQm4RL0C+;I9~+lsfoTfI?SB;ZLJ~m21_1g6E!*JXO42%*Bl-@& z++@2LP!XIRk4ul8#p|DNlyTK17dWWmr4uFHUOr6B9n(P-$R2@%2r@bv?W&cw;6*Zu*zsf2Mfh=fr8$3mnVSXpyFI(+$TU zAuC)Z#v+PKC6j;b$-!99-4osH4>w?_ZJ-*LlYRnL>_lc34PfRR(#7=$IYg_+)=y=A zhR(ga|Si9Ot*`qrBmGT+*|Wkr@SzKEBb9TJ0F?Tfy(vK$SZiTN*dn z8j@}L+_}?Kv4n>w5qOA(T@F+smN+EKoCfM!)LURLec|V4Q<u6^7h#oguK90I zhz=gw

r9ARNwxSU;h6X3d^WdKmHwXS$4!!A(cn33Yxl*ia-)C0bVam0SQ$DEb_# zerBSakyTwl6fGfA^35m{!SXV}jf>ny-;$F(rXVrU@ko0Frv&H1e;BUpubR#o= z;;}Uuu>jzD-40y)a6wk$u{hOI%TF#~XhmI5MBj*u z6GY6(G-!T~`{K&c&uG26)SwJnb0hK1i#N1;V*X2xMbHozn5R0V;HUOjAwlk-6!f`s z#-z~mZ5V6dQuwN_X<(25FTqO=Fe5D|wivi?c2$IbN;;3E5togoQQHf$hY-)a3PPGZ*)pZPOVdyO#ZZsJ#DusG(^uBJ!eU5o4GJjT|DW;bwmF zDZi~Aq6d(Bjqe$~7`G|m%*e!&>3HbIt%AkM>Zzd+b#V8@SZms@5qwqJV`Jkpa zn3w@dF;1pv9&_%7U_U?2&SlxLuRaK-aLr&Svu)bB!UazWU;YcarcaFQqK#T+a_O8~ zt+ezeov&gX_`k`GdxGCCmSi2b%BKNFfgKqc6>KjL5>Xbx>NRJc@efM|DUz)pxzI`f zZe$~rVew+eZ_!HRxreORMEtAw1 z&jMGIdcOod)=$pEW`FFtoE4|Lq}x5s2EY-6EyJoKTwQ>KIrtV876RN9+kuFt;JoJw zyxy7=h7J1%64v9XTwZ_^FW1M%#~3B=P+h}VBp$ow@TbiJExQA3cj3FQw zIj|)6%gWjj(og*d2;*dpwvr1pP3(V+e_Z0ENaONE9!}2F%4E|hW(8vxgJxK7;ymu#4YCFjV6$n?2+q@?+Y!&!Ok$X> z%+LRcgB4$)e483_Chk>+=?fgiZ~@sUtr}e}&wGVos%sLVd#iL2S%dKEV>x&zVxVJ< z8>78S)3AyLJNOh$hYcXjC!=Enht4fjKuSnRI3K#%t`)C=O;E_}!mJeS_U%m<=@rFh zkV_o@sZ*ygc{$A@z8c$;fz7F4J2^sj&_*j|g7$(BrLd@%8q7f9 zAv3|39T8ydT^$`$+Pk6wB{T}dVW!?}cNMsh$)_y4!s6mymu-MvZ-9v~3L!P?`Qqw- z({v68G5!|0M3ZkEJg|yzNl5T9DucS}Goj<>(L~*ZrQ69nWo>du^m7zMb^R=|F zxL>zW!N2BAfBM-W#r&tQ|L)xX?As;(5!Z9DWmZHOotl*bbIx~%UFFWgwqm1{?uN_? zBdbL?19eWpj~@>EL^j5X)&P8G^h8bSO)h+YDDI3X#sDjb@#j;N=dgASc{^Z_48+#K zryf-a3p`+iK7stzyY%Y~c#Mf`K>nF*#R;&O0bjJ3Rm<0}1P@X`_Ra!-UNlf>*E{g7 zs!*sq)v`^VECz~-)bRHtuqw8$T;=aii66A2fS;Khgn}!~qdN+6m60DVX|V zQ0Yb!k@4MOiXV5>7*MTnfPPktCJy&bNN47Rz>dL(Pu0iYpFg1FTh%w4%oMwxlT>dJ zZ+yxW#&e|vgX~yTJ{_H@4ev2d`D&3K`@K5b!IltWaABfLC3G0Zarnr@9Qx0M4BxtW z^JX$OXu{#a0!+Se(es#Ks)*5nbSFuksrJWQPQ$KbSv1r|`s84YW9vj}U3M)BODZPu zq3}ye)`nhTlv$DB2#2tAAO4=Qjm@)3H?TF)G(j8LGbs2IpBr(@@N~h&!$D`z{NaPv zMCrSUK66yRu?Oif&ryc;A}nNu4YIVp*tr+;dWV_0`{EP4`AqwS_yc#(iR+aiBg%;; zn7%?+Z8vN!d%eELSM}{`FW1~=pY44=3mk)le+6Y>H-(a7pz2S-+JSeejsbaWr!HIq zd0a*gqrb`ki7x?F3+@b=Na(_XC^RN91CSSh)K{T<=tVd(rJaT@_@YX;f|{P?n?#_) zz(-3jg9}Yl(bhjIlqM5sFiqkR%_C3DBg@CQp1PgFCz!3_O zP$!^<8gW{y;F%1U0fuQQN|av6)Y>l3lP&{QGxq!O zCt5`DBRC!zX`-_fCL8}Es~(JHLQbpU^BO|0MP=>fyITy1-T@n8CU|k`b}|kixAnp{ z3yR2K>5wn3wUlcF;WUjJfssmgf-hH&73Sm#T>kiM?j2hAK>A`q@OlTQyniJO7*#ab zLjFO!x*sU^1j?YAbRAj)Uu0&n^%}grk;717b z8RF{302>OS$~h#U6F)QX2Mi%ED15sEdJdiyYenz{!ZyPvaV*7R z1QjwC`-lY+_w;1f&cmz&rwY6=pEhn|g#LRzvtAN>F`C4=yStkXgHS=}DB!}ABS7HR z;y9tjYtkCM&|5iXGD8q=7Etp>W3&uI_=s4oWv5*0TOmpe(bzclzGtfk}(ds zp;d1|ZgVQFEh0CcZvs)^Hb#ESW#*$$CSZ;s#)0`Aq~4c-IZ0W=UyvREhVjSI?SshQ z5p1EMq2ZD0G52ZZ-n9(bEyvERu$?%v{W-d1@eS~m#@4LpcQ25y^5KficZV4tzbbp` zAAGwg_G6nU-NLR7Q`p)0an4b|2Mq)&#@IUc(^cG%#U%;8I*fJ0N025hJ z#`8{iu28@vA&Hu!sG3u3I(1kFH~k3khj~!)>EQG@$X{^g)4}~*iw=vH9V|i%-mPms zyw(bsjzhgB-`e(g-(xW+yd5F~rs2q*`#(pvdg7spffa)%Z?Vuu3Dr=h(d^mm?M!!g z*_@A+ooDN=rWHEaPm>9;)c`>cW0**e^?yhqg+)cor`;I%k#7^WDLLg5l@y%W(ZICx zB+Ua{L%uai^*Alb5`mY8PF>{=Gzru2Vp1T@G*>%<6W-{O;Z3xX*4J zh*#rw{IL@U-~j-Eck9gul2I`b_(;S76lW|Fg$hiZa0Xl9c@aMZ)hqLvAxO?NSD3`c z{^n%T;HFi;DKS(Akf5=D|8*h(R6QeK9v`1pjAsY5%ebpDQD1`_qTcyg2Sr&0FvTcr z`9X>0{`^Se0=1(9v@B^ej2lp16C7t3#3n06{BJyMkT#@$W!(D(CtyD0-I4nh(SS7Z z;lIP+9;KS3l0JHG+)5AE2Vb}};0ouB^>pB`0D^!u&Z=)hHyt2MvaZucACyGaoOS+@ zI6yp!7Qow4@h+Y}|9qX9?v!b71Pj#n)i4-a?<=VE9J%*s;tn@9UAB{+1fCRIdb6UF ze=s~$6v3EF^I#g~W*Jx(*=2(S7Rk4E?H=;_IVhYRj_OGdjd+!i99eBZrP10~ZCDM> zQalc?f=-)ZiSu8GX2WVlz!M67O=ps9P-{h$m*yajpTf6q#i@s4nCG2LB9NJC1E0Hh zRg6LaAGH(eN!&<`xreu|?D|mR-Kn4BsEu*-L<2RByu|+?AI?(ty=DKmI>RE>OxKcN zFyzil1`Q!Rx)0uCL?6TTspC6JSN72tQ$`0%UEU-5^h8n(YkL#_7e~QqL@!920Sl)A zA2omb-#Rm|LfPUml3^; zyiW1a7E04D#0S}j{wEf#M(K>hgm{TQFhBH({6>F%-up-2cfRLB_(WT!up6k!$>RGD z@lp;>bA2&|OMocY7?0F_oQ<(87OWJCprDx(29jibR7lc3A_TBnIj~SrEA#?|HrD%- z^!2=M0G9vgX(X<5kNEBvbN<@W(gfD6jE?`!{2EzquFJa}SkpB^F4MibY%tU3w z7l({%(7n@;>WAA}Ufy;HC4R(`)(=oDrsv1t1Kjrj7dzfRHer)`E-F4s-@UJlVdz@c zMkZiu0iKbV9GGYZ%{Xh|v&HGdE^-fKHJreI3peI%*IGzXq9?!BWj#~Z$bvH%q|le^ z(09FuyA1<{l(pBfUA%ZLGPyinSaETy-oux8+6=}uw2*CIN~JYgJARwMB1eSKQ2_tT1L4rIw#Kua^FO>q5yQ8 zE#Zu^3xwKu?f7*aAn{V}0=TF{sHCQ*c0&2-Pz)--2FD%`M9VB>c>(SNXZpta;4-=m zI5OSJ=_^ZR0^J9XN}oP~_W zd8Ze@__>7jR73gx+d1b*`s*enmM)5YVPv&{k*j(2;Ly;Sho0Z=agqos)Bvg>Wo*tZ z01OD^BBbrekt4oD0Z4WkcPxQoolM(Vm*D_$^N-LXmGcjyd21s+JD5RH z07(naoE+NBh@M4;-k)!*$7!9_o(N{j)LBb50QwI-(dSmzse!hT&~m`=Wl{BK#7}>C ze3U9jq=@aq$RURCSuC3<@P|U4{bA#wrx(NEBlH;^)VM-^#8L>grlC^=t#EYDrSKEj z0;)`61{xF6Ae*Pw5is&w^M8+McLJ|5+eoBDC|t?;nS;S?q~*IHU8}KblJ)!D!e%?o8u)GK$6e z*Dz}!M;F7KkDE;#F@_(%Bt996g~QYt*8@V(zKxj=LpJ`__3O%bZdw1a$%W(yLI)gW z_6XLZ_k^YiTy4kX!#$d(G)cK%Y{y7n6ZiBnIUJozgVKdS7brL18fD*>TQ3@9cU zg3?k)g{2vQ!Wp;*AyYsy2>C6$56IXA%1#yr4G}i{DBFhz2>Yf(qI5@(J6f03-UVN2 zyY?m{S%v0zKz=oy@Lp(YfDdiCPw@rdPOK0@+z=XehDzWn(u|@FSC9rEYbvJ+X2s!8 zSqM01Gy`zPq8HtAJbyL{<>i=6DtBN%B6j(EQran?T-2iFub31~QRlQK5;NePL#!|M zbttM`@IBDAWw#UImvw99BUJ{I-~y(QlvI7FnCnmToNG;}7QyY|Fsoa=%{A}vq)^WX z>);3^A~{h;F=juPxWL-Et^q633?e~@P3@i@x^r+2cj z1|_RCy(L%`%3UR0f|{^Z&>}AfO{;7fTBZ)_>p9P6Jw(G96BZU$=nUdeIp`wRNJ14B2u8y=J#;+nY&B={u(?dq(Y1Gz4Po#A;O4(2N zKG?mCMjmyKA}>!*@s+eEIFeXrFECp`n5aya6(LQR#d^RTd`qqP^D`<*M-J-0g0MqB z=?Lm5_mV%=@y)cNiR!pk;xuawZ#iMbMwOo@U7L!1sggY76pj>JTavy z{i>F00UQCC6tPZNcVfEb`dhQ=<*=Gz9oMCvies0xbBOFAOZLujggTNO^!N+>R^vQm zeUiGU7XeOog2g+8p<_2Pxwb8^L@<2yvrAgL#pi`mWC3Fv0j8iS( zfOrYZZyThgKaB&ea9)T8<~6s&A@P6`CfwUQKLSXV2Q3EWN#CCeim%#q62qew48-h0 z@Q}bFU`<^Z987d#d?F%c%q~=|gjV|fZF_ayemt1NXkYMJ8y^Y^qsXHF)=0<3Q$*uS z`8P?jyGlNjX6_kI^j*${3N_RKrzP19_7(~xw$h{?JkZ$xnbfB?I5Pq84hb8xS%Pm; zbSCHggHPa}0CJtkiYZ%0ta=o-ED{ZYtT5S8$-{t7CW?maWgJI#4aF|ga(y9%o~W2qRPuhg@&^R=9S%2S?Xq#n> zXW=ukl~7%p&Vx$2l3Ar=)BsjIE;<&9JB`N?HezC!!2)O=T>=Cd6v84_UI_+kq7%lt z`LVbs4Vp1fB`zVW-;7qE=6-YO4l;&>Fm)(-dypcjrlBl|wCVa7gV$hF5!Y8&ct$t7 z*vY@z!6n&j?H92{jA;yh;J{B4Dlt%(f;P*G^pbHmHipF#g3?>`#f7|q=fhWy~14d0eLFW zU=rt@D2~^sIL~QftZr~n5s`lAIEj7@S}enzE<~_r-!b1}(-z!a2WzxV8y33`oq-b8 zv^pV3lMK-yxH^7lB__>U&|UvGq`~+*ZbpKix+?a1L(tl&E+izR;q2_}@l0X%thmd2 z7_Nrv&S6GDU&c?1|B9kTYX9wM)(Ox)W;L7P{7gJ)5A>uSG&3qYLQFfN3CWDSlp!5h zX7ksdG_j#wSZ&~-0Lzq~2VWoYac(+x1u$7wJJJptqIoUoygrfzFosC)2G)P4vJ z2PFHUpkdRGY3HR|g4V$fQ;*oI+q<(U1??Pnt&(~0nQmaDtmF$?b?pXpL4rTQ%nIC8 zE9-Owo51eZ36!fPEgvtU`oTxf8k|p5Kgv7+nTQr2;`r+Zd5%N;pjjgfQ*)3!Dp~%> zF5{yhh5ken`aB4y8A&BbS{3pP-_rzby~k(&V(=F1?oY^-KG#1n=o2&9XIa#@`{Wbs&}CoU+>ddU7)^$sw(Itzn8U*tkOrMp`GE{@ zLR{0SEuH4@Hl5`}bHnP@W6j~4e_4JWS_iBh6SaZZBD8a8*8ji^Puf{965#SfhX8hz zy#YrMwxX+8^Cjc5Q2x^D7T~<-#3(6)khblMKRnBb{X2SpkLAYAG-rKTyS)MG$_L0A%Wj=w>zX!H^Z zD4{?+)MHH*{lN81P^D6fw08Il00cTJK;6JoL)1p%M#D7`eErz$ee*E7b?4lwNbDDDL?9s?^!#O~5cbfnE8vcSq-S6ahep)T z>UI*AxHmcG;Yc89P6o|y%jq3IfisD z-to^)Elgu0bPi{kHds+ZW4l;S^E>#(K>{G7(!KdF>K;l5#U#=n8)!iIT( z=zq!}MKiy92L;8WP-|l*=9Hrr7!7>fhIaRdcA3auI)!XKsE)moOE;3)IqCuBAczzw z3YC1^N{_Xm0;1+0)8n_td>Lg^aPX2A3i=k>YelA|r`V@WJ6#!ln4Tt?Zwf$u05UNn zB@eN4Fp#0{OQ{ZXBX}bj7B(4c4d@$U=F8*w<02)kqFoH>2cJu7e58kC$A!nGr&fmw z;YG+91Oz3OojWfoIZT9%d`SxBWQ5X02ZaufsramR%7DeM0Ge{bw9=^xKpd>IDw#r{ zf?%Zf_5LJ=u^0|<8tDJGL5W-`AyLB&3=V#qzC=uWt6^ylQ2&K5C!YrDFT29<&$TJp zb(@HU&>56R+?}~mMstJTyPb?>Z=3Jk3lXS{P-&RafCh~Wb3Jtzz+Xz-3%8C6B<{ZuuB>p|6=F&MzK&2i$&>Mw^-1d=x+aR;S+c3=!U z{VG@wR{53<+9f#TiDn1>-cetdB8+<|9=eS_2+};Ug&im=Dyo2Ti8Kl!4lQVZX{wL& z2u%cazRHF;n#m&?Q$~Y6K*yvRWMMKa^ve>5>EWjB8%N3uV>TMPQ zRZx&b9Mqk3e6#cd3Hjd3rs$ey6XlS9)*jD9dX{*a{)7Gc6Tjr}`)Cn$FL0yU)uPt5 zjP_eI{d)Tz>}tNZ9Tsmm8ADM?5?hF{E-EK>LR+(8Lv&#dVm=9LyEJu{7|ISv$aC|z z`fp(e45F;37pPf}*8actj~*m)*BmT|_qGr}u36W7#CDOxeB^^Xoxuw?{KMj3AZXA{6Nn4qC9 zWEauG$69?#1gA1lMU3mFGxxp|zjt7&YwaSl%>o~h4DlC6c=f=R^=!++7ip#?|2}lF zkYD@9RM*sUc@Qi3l+WWy6RPX63!+OZ3kV;gG_gv|*g#fHx7bUCA*r#5kXn1Ai*w<` zWCJXaiBK3;cY@9yk?K3a?fNhC-_;N^W^0g1SoBsn0Q|pGS8VQr=3~Ws%rJ)=V2zwI zZAw@rvTS|kV2>P&{g0GOcwG_Zj|nCK54#kazhTHm0%}x`pldTyN9ay=$0_?PyZG0cv19v|76o%t4=!Dz~q z#>Jq5L6?#P771;C0WLIE zQO=OS&`fIOeAgjCQBf7(@crcMNfEn7sdZo>BiZ+VNizypQCGsJj8z_kym%ak0IN}k zk4RE}-TG@&-`lKZ7?*ELz)r;1^S`|)!-dZ_Bl+v0i!)aUhO>p5P91mY@9L3>jWrt| ze{!mvch3Hm7jKu%P@lemYktIl1%L};)4)<%4swHQQ+lRcd?UBYwqpUMB zI$D*Wzg;1C2XXP$a6{wm9n>S56}Y{p(IFtx@~jmSck$48*7pbD19vc!hC0X)qc4-g ztLd%ydH^5CKUGKp3bPxUj<7xW-5*;pmZ5~yi5wz_8zm=_upNoz0)mKUg5HyrM$HSj zxx1>Ag#clNiQ(VK);J_zOs{B5-Y~;83WEp%2yWD|-3Rt=$gpS>=-$ z^1D$T&*k0`4SfL&v>Tu6%vhR_|$ z!0%Q=20*^?PnK4I&JyipWMs(qmew1%c7pq_$94hsqP7>k+LtYwkL2c%j}kvgy)0A(cqtQ5m_=0-H?VxHHpKW4eA z#NGuzq2f0{bF8pu8|EB|M@?4o+a!36$s%s$+Vem@zBqw4NnhU;QCi_k{Sx!TBL&SQ z>r!ny&N{XYTmymTW*Pn)H8;8Og|JK_&s@Dh-(5`{A64X&iZ=Mz-Q98&zjNdP#>q5;=W*)21ISkQ z|D)?%z;eF-@c&?#!)BPnn8OYx5r!O-wmD2h^f)Wbxr{k39n{8H%vf>?l~XDsryM(N zjEOXoFw~q0r6iT=|GK}$_WS>@|8rfx>(|gz&-eTNybt&LzVG+VKpF*csR$;68HKE< zJ+$?T#s}AK+&FEJ=2an&Xg-oVw`r5K-q+1G>PC?x?L56#N9EHe2{ z{)NqDc%;B|7@^-|)8k9!Ug-I6S;9TZ|l2M&bB%818&Xu7i= zAj^eRC}FhLp(6=H2#I#VlUp{poJFKOhv;&g9uxSeXLqzhfDxY~qT}+9KKjVY@D7RC zh6A4%M=70mRva>fcs_OG#*M93+)LY(me#hF-Mm|;Gfbv|lR6c9$nJ@kUCPO+#uuiL z3pwsKS66S`x}i(2p?%6Zes=9_iwnC0Hu@B;l%OdX3zXR;R755JK&zW*LY4HpJQ6!Fn&k!RPJz60stwf z9lYjf2IkRssnvJ#-4B@z*4N;*S=yiVb+@We3)H`|V4rAlO_K@WQ#0g&PQ? zNLpfzMWtCWai$f--Smr?2KCCDvN}{hE`3nSo^f9|6~XP%%#rhLWnU@8hQ~qmZ0h8{ z$k_M!%evdko6@f^M{mi=G{aRnV2}?YY7w!}B(HO2_*!>+xnJd^&;V>FGShWM?XjKA zJub$z`l_7WYNsA&$CR5FlwX>o!yZl>s+xwEgM^nE*&+ORj`{O*R zLFqdAM7&D^Rb<=c`H!lZN5jJt0VaCEDF1C#-I{4pw_*Kr5Pe2^u#J0`U5}Sp3+0cE ztP{n`>cg$6^lE?7U;A7xa;e{|wB%w1(^X21QVt5!73b?@@~5Q~x>9wL?l9 z=cC)CAl+g%CG3Rlsun*q5b1;rIEN3PGKfeR0S6b(ig3Jx(YVO$|n&N9)-zj=mcMz)Kzjw1=>i()xc|&)nd}=Z~dOFRT z+kL|qip*Y{aR`2h1fG3QPBHm+H?X4I`_WafMA`nb^DVDL-gRaIWsOy9VgYiw)QM(L zjlkApVyBg5O$REJci+iMOT1GuRalpAL}J_OC~|u;zc>%;Jm&2stBPIiXR@?SW8CJ5 zF81a(i6&o<0#OScNfq8p0mXNpR$(=`cjwNCfH&->$}@812VbyR_f4v;^>=L=uaJKO zIZsc!qHoFEl45W9t7#Csprq*A$V$sgWCo+qOBgLUb?-nq)5_ukm$t-Xp@uHAyM6ma zHTdnP?g_V}X3w6DdE+O_2Q?boWCgAOGUtiVhMq{zTE1;=1Gj*NOCDIYTH?KTE>G*I z@EF7ru)m^w_n^G)XtygO;bsR-OdV!U4Arhs8!U1n&1{i87WbEEUD3wEARqPCk=s{Ezhp$>A&h z%yBQ5!PteyN;Se~zvwo>WI+afZc=8rKm9PcSulHvTH+RK_6$7!T(@@V?eo3Y4cL)P zR<(DMU>o2aAG>(BC*X|?)N00cD(CwHOMJH?haRAfp<>71EZjS;zPN^Jgowr(k<|UT zqRDJV5g?En#?ftS+`2y~0zuw>^xhx0o@yne7%asva|TAv-;fx}6p+0uQ-y1V$F$BO zpC6W$_d^D8F0;ulN(C-xbFWN$Kc}k6Lb4F)TmU8H(jO9M9bH)5l9~}Tmaqip zyT-Cwv?U}7XWvCX~r(loY%D%$zmSw3ZKh>vUhn@SiKyX#A^mJ)jiBCQP_JK9%Cui9mD#M)hW*WsCOhlb7@j4vve9leEhTK;IKU z>%b@P@S}X~chK8~&ZMlHej|FS@k zmOtO?GOY&h5cZ|g=`ECS%lu6iUsK#B%Z&#bN#xy!5GX}&hxA~feEYd6%!3~jJ|nJZ zsQN<~Z2d-$XHAME*aUaAN#^~N9GdKS!(^_zcw89dOP50Dj~`&2DPv?rBteC9p-)Kj zr$MVME^@fO$wKGYxC`+(^T?PQ?jy@u^`&)M+F#obUo=Ekf+T0NdN=lo?1&u=<&YzR? zjFQjZ>*TGOfkUgcdi>h-@vnJ>)tWZYrSz;i?tRzO=Bh{cQ4jK`@7!t% z`}jikb^YfbHZQrBWBsU3Md&1zq$(3JLaDW3i~V5AIE6axWne0jBJS7OPtE}@SfFaNF#=NmXhbb#bUU%oZ-}GTN@WmMK+|^>D0FD%N z99yxb>VdSQbNfL`Da)omE1SZovftjNi*NRCrBxCL;pcwKW6zbB5j0kGW};II8xG$K3~Cyre|%JbNRHh_%ulXOW-> zSG9cy-!#qa8g`oQo>&QF>3)OD+dR6TNwb zeAUIP`M_cOF2ATAGROI}<%NH`%^!M-zX-cf{kS7PR18fPjM)xiC;%gCJSB~1^IXCb=fqaN%I9=5i23F{4@m=?HN3*;iUlN#INcvhKtuf3e4Pg_H5l?^8!lX z3!3c0a0v%PXttXX^iSe~bF^0%7G}lP)%~qkw=SkK)QCp4q*8p?^32M1cWQ%Q0`ln} zyBM}IuuW*EkimZaSyxloM)GWHz+0(^gt*Y{*pIG6^7hjr zE`@wnK`k;#^8huQNrNri_}!eWFfZi@3>j2&@{DpXBI~)h|CW)ri5^bEcO#{y_3PKK z{eRjSOLjPA%$~w=rKrewk)J>RvGu^deF?x#9`D&X(q!dSE`5Z#+MX5(<9xC|*Ine2 zR=M_dY2~{*A?nx98#&x*>SmgFYf6Mu>b|PG49!c0Rq=!z5gck4xna8LGthP7*BwK-j3>pOaW|qI~Y32iuz5qtVYM`Sd($4-KtL< zE@()^Loq)QnNAT?(?GCcJsjZe)dsKW0U0l1(ALG_Yt*MI(S$t0#bAC4!uVESmJj)r zIlanAZbeVOk~kkEni{pyX%PQ>6Sf{cJlv-^e}~Q)1zh*%SMI^dP$gqw~@PW!o zwXUlwr4E$m7(?hcVwcB#>u-rqEQ%Jt;N}gJ$)*{gDxVM zW8~>w7Y0fzDlI$Bg_uJ1&&5mKVE@!%wlarIIX)`0{4G+02l<14%Ej*p(+Uy^YpVfM^#tbF&azE_Y@6uNHVJ|bPhCg5p|=N*H+pRafE zDVocD>OSu=_`P&#+CzL5U5?)p|KZJ;L<%puN>Me3M#8PU0%e#Pef#7F$PrpvStS}^ zUf2{B(&AvC+kzoxx%a_UdUn|gSvemycbakIK+}o&Pw7wT#p*UQ!tGGP?bMBH*G}ff z#sUtq$o5nOpVTFT=cyrK_6_=BZpG6`FF zthJpdrL#M+$TU#azG4)BHz~^1O1(xwgfb(Q9}xX^1o!{S=rtaJEU=d zC^l|ynJZGN!d%o@ATq_=O#=8(OHo!J)v;qXDo(iyIN^F+biWM3yD>c$6e*-Dx&29# zX?QahjcDQuA<7{a+>ULngE97A2@`{cN8}mjBeVXc{nv%?fEHSIO>c&- zl};*|e-?9B!YUeN%(xd-x@_{DMdlu<#9aM&+Z3M{|5PR&A6Ln{6);!aYs1cot!`a(Nosl1CDBBK<}Ct zh~Wr(rd!Hvju`>Ks$KrAIbCIFiGAaUs}x}4GdElcck07oJE%!1_4uEYZXIrGvdgc< z+Fg1U`3ZMQxjy|6P3;aO(Qaar9~tf9^LO<8*{Xkt9yilE4?kJjrm6v-o)-JqI!)!z zhK>%e%AbKd(1tZ&vKu}W9TF+K1$cwgp|m8VU3WCkox#zq!#9H3{EGZ747jH=ir z3Wu8*U=n>)#7f>ZYg+t5$RP+SdPk;3e%*0{@O1)vcAS&R?uTsoLTP3a$Uh?==0C0! zpN6-O&L#MtlQ)Oe<3-kH>&-;9-?k>z>)E^aNMlXsN&ux+8y?3!Zp@>z+a<Gz1V>KvLv)yx#T`vH|JuvM`J0aF_ zCB?^RExp%Rvj|&Cf|MsWVs;AHo{X0gh4KI>s}Z9P?G_$X+Haa05Da0m6kyW=4XUKp zhF|DBU=Dj5I-=UOcb;fpFUj+=BovM{Z3^;(KUr&+!n&PmD){Nfp9LQCEmQ{#% z!5nc5JN@Q+N*32Ddbum}6B&-WvMci8rCl4se`{(o)#=61>s?^g2wKehhv3&#&#@t9W$dont4nOpg9%M*Prq)u6F+Ma8yR2#`bo8K}xd7E5G& zn|ZkHf17)}Mi0q&QvukSb>Cg}lLA)PrW3iyw6)o%GZcKoqSYwgqPw(2KoY2x7hz@^ zGv3B1B}iC&pq!Db7#tF4Wauf(v0@AFd*_coCk{DO5WZvXiX5@Sxie?>GrP=giXS1E z`4GvZ0*|m7WGX$tRpbUvO?uZb)p6jesN5&k>D&_Qn(b!TdPEz08F;u`?=43m{O8%T z7fFqCiTF;dwWYO0LDwDd-V1i+)2 zt=qKvi)AIEG{ls{jNrxU%c@KZTQCQq){G#jX5gVza`-vX8v~9_)UjRxZ?|QdC|54o z0!4PgF%mxODD`cqOIb>rXR>=hudpD=W&-v=)hP`0aA8F!%C@TTk5&wC~cV;`9|MgG_e*=Je-N?j^oI-zK^Z zSKB;c@i+^F$}^&$F19Uv)ShOout|;PN0@TN;>t#C(ht@xt}^r1-`1`sLQVfD(o9SA znEhRJX6#blLHbMxTj#3o%%u~5X78*?rQ~GUw@<4}rZyQ9EAfhs(8PIh*Db+{jfoJ? zFIX|(6Y!R5%$(8Hd`rSWdJ+2k%0qtM4U46IQahjcKb2Cs>Lhra1d89lD!~;rWZ#vuyQ)e@ z%JfvNUQ?q#;O4F*qZu>FthLk&VN|7|SFz^ec^%$d(D;Hi-IE`?R$4vueN$L~dvIw% z8Gq|vgw3nVszD(wNA%LUT*d#t4wnJ`JnWQ44(%tM{fpiu1>W(g-H8Zy8Fn*Uo+Qxk zC!F+VG+9K_D&%Ty^7Wo5BK(UN7y~w*$FGdyU`IQt@&M+Mfm(U8H{S z(*>V~$2GNLiqmHAFXMXU{!g3NTa`%vtusv}Eulp7U|az4VImZLSDZe`dmh&HEBG#r zr`{vxNuLl4mT;=$#-}+sP6PpK5O*!mVdh{vpsLC4fdwEYC9m8pjoV<0I(wZvS9%toX+d1>tKJ=6$Je;OCv4G`I9C-yOd=-@gIk>ax7On$t9L&7 z@Q~38I4sKgkRvMXsq#D$uOFG+M-V30?Ob7H->ScH7db~|hB})O z^shPGn%hpOOlC^T*4j#8h=NOR;UEn*vb5UY&e;hnU;!Qg(g!U>G-wgapO1#6?A>mYgkb5nT0Jw#=kypy zcA@mfpemv|qdT}o5&&p6bvrXNGwPF0oraNQli_!gzs;M2Pq{zq^aCUR*E|9XZ}^y@ zdr#dOL<|m#icZZ&4SBOWGo3KVU%XnVXLP zquHj=-Y{Sr5YF70g9+-gfYX*Dx`>v{+1z3|AwngzF}KV6ckf#8*$UDZ)_AvW-ANIX z@t|gOK^7jmeo~*5ThMsuxS=O(IgvI2!=OlDcuh%lZ-;LM7ZfidoZVGhdHWpCw@rP0 z9=B=GZo-cmHb;Q-IHJyvVu%r9`9B;U)4G&hqf#@jZ*BY7_8QC3evO+c&2O~DW5axQi=m00hJ&LWliBKU!ao~EXK zc?_(V^KNrdE*QmGNJNz~g*VDRO_Ue^8mJmukt>_Eq=dnjL|}?;JNMIS zrZX}ao{6oD{f6UCZ0{GmiiBXnqab!ImdeI_IEOhiZO}!rt>P9GopNbT?gVSdbNx^S zi8i>CLAM7r8sbDH47)g#+Q~hUM$~8>=}XI(pK`N;DQqOP9?-Yl1eU;uw4}*$A{ae- z+Msr(JAQ2I!$M`^{`G&?od&rc-3qzYWbOrig~L(k)vbAT)#mM_7}hC%#p*Eva*s1X zQ;#J&em=o*YA+hZ&0kMg`KV)NdsQI3d`tFppI@c)%iN8-zj}OZlN-&(i(${NpBoap z?D#PLD&l)(Jf;T)kM<{Bq{#NM`Rg{EK|%T2Qb4aUkbZ1nk%vfw!?u1;y-B-ilN+V? zweyeQepBJCccqRj?dn%E9||P`y8+n|kf*}YqI8GUIt8FwE%1yAw#|>XeHW962x#1+ z3JCusH3uRp8jm!Kybr>plf69i0VX0uS zb!yihAE7>nNYE8*F?=Ei7myrm!|nW*UPozFQ@yAxL;_Zusk0kUgU5s>H0x?*?*|u} z8KWr`N5I)!92bL@2UxW2=2z#E%bQ;GCWcTv_mA@Fnxv^yR+K@c+3`Gu?P zuOFPW@}~=LzF}JJ`$3D%lUBaQOwV2UwbTj4lr`gh+<UvI@)?%b`haR^@Hz) z$@LeDEf5uL6SSjJ)&%abCexTVwpeofA8Q|6iiDt({pj`__1HYb=I=brj3{o@sp7?r zYu4P%cG@YT-r@*G`RU#8U6b!%+04=LLAVM#ad!}Pv>bd$s*nj1cO0EZRVMX6_2-`k zkBd_`=jI8&dkH^BoMwzgAIgfqCeaOs{XMOtU~+Z15MO%1;I`<2Z6O;$9?xy_{6BP_ zPU6P!^g0kzcFW*Tvq!VFj_2OExE^`)__&U{;g$@4mEc5>8uwv*9?*FITg+T=b;t{k zbh3)y9hy?!zGYpLd5Xi%iWWBUVuTp4unVslTkihH=C>{i8alJVX_N2Ao9xe|Ubk6m zJOb(C>5Zra1xx8E*G31XgUxp5y4z?WGZprB?)L*si&iH;@Ve@Ml#c{I&;O#QmDXc?Xt=J z^pWS=Ts@neW>dL>spPycPP#{`f$YFzv+o&NeP;a*^y{3G2>+Kay|8imQWX(Ap6zUw zu9eD`N10m4hgG$>Yv&3kyI!1JYo!&?9AWyYUW_)1iw`BW{b{nta$jE)Kfzwi+Q7fT zh!(JX-Zt;nn8KuzNlC#yGL5=PzJ}bWq?d6-ILoadqcJ(>y1C@FL#o!dg0soLxNDn# z;2ArL?oUBCTEzR6-z<>pz zZgYR0XEVR1$z0CaU4sx_@o{5cjU3~9JT=&{WrryJ@SMC4M z?RqI@1KEVJaL6TbCQK0R!+i)WcB`nO-TFU~3PGEb0TlTlb8n_KtyhXBem)SJde^KR;3SVA}T$peM zry0sF98Wlk_$gp zG}ReeIGLECM{-jWKpuk;K zgbKW@!9aB`YEJrmGfiW^X)g-Xf2*JYB9OA@-3pHqw5tUfVt>wrc%ANbh2hPED%-4_ z<1dug5X7et0717yQ(rQvlD~;FTY)&2mU8P?$}&)LK=<#z_s{!Y6Jw%c%AzW@FaA2B zz0-IX&XFY-w#;P)Ipp)eLszE3Fgd0V%kf$J(2Z~^feRJWv%qecL_j$jese*CrffL@ z-$_Ag7=2ga74u&5@9DJQ#ohUMtH_OE?AZw&3IAR6bu>q!uMgx{Wae(N>eg+P%ymxP z+WX17hzkDv^Ut=g`5q=47Xuw)F&FCU~sk0g8xc&-b6rzhBnX2}V{C z!1+QVc^IQFJkPbQE?(dPd=r#fC$14S)9%jD9RnIMcwR3!XrF;Aq+2m=tWc@?U$4iF z7=&sLU%=EMASmd`twjTe29^#u?kqn(&h|~~QW1jXLIR^37?Nbb(RZZ;!(d*&0772f zKl-`G_hsWIhg_D}i6IOMOCv1Cy3D_bm;U4G8->u7FM;bQS;ba?codnBWmVV=#P`=y1#O*F9c1p$oi`J@9HKc{$@2bD^a;u*M((V*;z@-c|2>otFr4UUUymK#Vc8LBg7dFDDHvBA z(df~z227BD53l`bN5STaIX`Kfz{|49h7<)eUXAaiZ?d^_#6~Ccp8vh<`+qsVol1S> zJA%Ujr$gH*AE3z^M)aMqfA!1SvHwnxwNwWL%H2h=^bFWTtc9j`#l z)!9^AGYrL!p-=GjM|W^ZhB1s^QQOw#wW4Clv_5A5$33#P;rUxS2nN=C-krNM|O!|0lVq2Lq(KfoF z-*w5$rDge(a##ol#!>fqQe1MVdG*Zd=HnO}Ji(BUm(8HGWE^*)UPlUIYFtuG2bI!v zAuKHH&kmCTsM&<1Xid{x7e9=Uui6=mom`Jf!W(5Tnq*yWC1&7KG(ZP;@@51QDwCJ-weVW}hvla0;Gy%{3cQ^~s_ zcG9+ZTJKL7to0~@R>jklRL@vTX8v4g(t~b`0nce$TlH6Pu+fduMPu!fWJfMJkR!X! zytzmhAGFugI%5CtTbKU?Q!qe^F@uFQho1j<$ff~1sCUgBL5f98O$i(HQRESb=2K`G z3g9zMSyL^&iO?h^tmyeUXayeBN^8$NoW44jO4OAn$VhSEwHDnVl!z85i!%n%k@>Wh z^2?R$ZBoM^MspEOACWGeO>G>|1ZI&0y|OZmJVHEo#iN(bxTOOf zxO;EN_Np3bNv1~xW9a}ya|myfSR3TgpL#%IF$?J zE6Gm#+1n?yAJZmwr0<6n9w#-dX0iH%7sU=Hgo%KBn#~XxpBo z%*mC6iCcK4Yir)o*WLm_b%tpp>B#MO{frX8Kc7zakKjm#v7V1R6% zqe{21?LJ{mvnL*-j2k=MaQ_0BKYOM<%p?3kVg)T}85clXUSN$HHBzb7H2{W}HDYB} z8zS)qWZn1XuJ{1bGH zjMHjq0;FT?A-r5AJY~-9xnXHOxTi-RAM&x(jPKtS#+SYxSN(M-L7y*Bu>D$E*7V~- z%5Y1I7^B>VMfSeq&k%4skdfQ#*yVO5_E<;YnKV8edp>=3WzwZ*!( zWVd3TdX2XxrWKY8XhLa76>PxB%(dyD)$@9^oAD(tHT6DO+r@XCc#F zce~NHx`yKt_fS|=tP%r9%HUm5vWLIsT|xP>c+aavSzSXv9P>YQOl2{L((=6Q_w~|7F^o`TAOhHeET`}9vyYj1`J5Fs|I`~?n zCv><4-g)n+%g!I|hSUg&vhU1-ZcHi|*!s?M2?V&2acWcphpVAS(i;>>2T2k$KMIs-xCzo*7y=}b~&kK0t zpq$Q3ig~WbHY=k;vz=YT5nv*hn@91ZjzRn=4kK_>54K1Zc0?L(o0av?7cw+}DsG;> zJ1_q&sPpHUI%^3EeAVGAvz`KX3#U}+^-#{2M`ks9>$>Hq{1i2!k|02PioV;>+I5>q zjx@@~>Bc6gCic?d5 z{jk}+Z(g+YO?WVAz%RQe)+>#Sb!2>hwG=9LgT3I&I-N)J3VReMLgadevEF^2eCSZm z2Z2sAZr$`>1v3RAj~1P|XWMrUU=@iOBxGDj@g{?_Ph zDF)7HAYRpOeobhY1ZP!Nn2Y-otR5aNJADZ&`HM= z1xL07Th$yD%eCD8?Xt)!Na?f7zUUhG)JHyVMi}uIOPkc$F$A` zEPTrEUdfbo?UfX(l%G@bGx8@U&aWt`SurIWR}{qguUZvc7#Ml6%KY7KCcF8m?LS#G zd9yAiMN0pKl5-ds^KzWH(bF-+bHLk8O^!#d&ptZ0__HgW_A^DZyHl@Ltu0(DO{IAE z0EDpFm!t655EWct`PbJD4}z+c?rSs+E9x-AlbY`_WnrlhBWT6A`j7}x`2%|Fravrl zZ(={pY~IFHT5f787uoJJdeVHDV+bw3cCu>LZ2MV*Wsp?|&~yl5rWOE` zw$(IAJIMlvg0`Y7`z{UPgVO30dwPNKPn={}H^yZ!{#p9dPi7tttNxYa zT`BGulO)r~^pC5NiK3p#DBEEVvOG9si?(q^&^iw{jGWxNf*V^euU56{oLeW~5^BGt zr|%4!kM$>+0IoT6$3Hl2|Di9ftAEjb{O1SSwaA%&tDN1DN7N_=Z- za9)nBHzr#Ab^Q0QsCj$`+O%qQ>BvvzOY5In%ARnWOHyRqslCW$dm~cgAv!*sD*|I>sCWXAD23s8IYiQo+`o z|9I%cT&4|jN_$3+7{NpYkkti{bw19d4xXGK*7y=+>w(nA6R!?y z+E?T}m|s>vtkGc<5xC5|M$naNj>?l~K;{!P1(`{LjJ91Ss=*zJh!G+5k1CxsE;!egF84gtZ&k#y_D-8Fch;du3321&irvV43V<0tEH((P#C z5;DB|`b(ds@fE$^p{9+XyD+#10K*-1L;)Cnlfo+>wr)LuUi}ix6JvSjy(#5PVg0dl z&`x{$oPl6Td4YLoXyj`oHUMqe_lqX(gz46x>zuIExIC~EW}2({z?45eAGK*S5F{p< z;ty?X|JfWehFQrx3uODY%W9Q3`Szh{y-@9%NKJ#(g>ZeBr) zU)MT)*Z*MOCgf=s--u@2({|A%mYq4c3u|~zCyQmPg?4%@ao%WLc!)eHO^Gp6NQ+}n z>xhYs>hY#_T@dh&ZLnke_7?c*7(`aP_xv#XC$w6#-H)E)*a>SLO+ho9m9_Rx$El!1 z_%YwwuNKO(f{1jbP68(F(a~Uv;`eFbF?f_h;%Q1Qn95r$VSkjCyLhu8LV!`#YW;33 z6oA8qT~84?F4FD1SLti|kJY2-)ivVBd?VmR=6HhN|M;{#TMmgurt2~_DB$jDyB>LB zR6zBT+x&X?JAKDWsr%LdkX9Iz4k88IZ4X|2#}Ss1K@AnH2N3BMt!E34t=hMqBEk74 zTkosNluIq;!c)8>XC|K$va16Iry5_~z_U&VDc>7$K_RUtOlH)^Y>O%*DD{J%UjyHc2Pu@c@oZ;k-iDGSH5zS; z%73O(T+05_4;8K-bEa+e$m7pdp5==zLb0ydRJTo!>XjS23-0Lh#@)rqUVD9dr$l)W zV!OpTZ~|gwX!|k}g_$}T8A}oe@60-4KROk#N>^F4U5I&&3vF6)4|_rX!Si*iPCccP zOAd4)4|oMwkF2vyK5&SVLu?_jO>&hZ0O(o&c*x?t#=PkPx760s=g3$zhbH4MY^+tI z$%Mz(M+Ua92|VyR!{8Z6vrz#v7AA=Ey?ZdO?B;N|oN3^l^PV2E_Asro1_br%G;66bk^&_I+M}IqpRya19Bdc=#F&Gb3?vm}Qy>$A5gH!5oB}csghCI_ zSa^q~2(-_!F9zyIIlc|_Z4k5=%(J+*)JeecWEp-biZXDE%UkU~)^HsWpbWGrDD%!% zzTMrR10=L}7SX|CAl*lqaKltGvHoIu3~kyh?OtStEASFq7%u3_ZMyuF`ce@9UTufI z9cc|(MJ5w*L&d3BPv38Y7k2|xL56>gfHzP0d{mp&qf!A(P42yTP*mWjbQuGsu|mM& zkc$M~Rq{s$7`;wsH4!6D3ri z^AXA|jzGs0NHAlt9VKVe+5DdOOmC)u_z^L+Q6M`s`T z#Twf6ue;ukBt59Ne#YonG-J-+jq#DJ)!B{7Oy6Li;v>}m5x_NTMu?l7KfY=1zh_Lia}ASM z`1Bux`EP|(<7HVTbJd{Io9T6X-t=FPHwxWpHTR<(Uw%7lzK*or1FdMq^UpPcjeVLC zySM>uXwjwiXTounZ?*+>X|2dnw2S<~RHh4ZOILCzCB?2DPI#py{VKxHZ|bLK?PW?M zIjl?*>F@{L3`7a1nSrd5d-pyW7dsCaVOqzW=dlzW-G`|bi+5eou_sp*1qggzo(y?C zlYHycs&&)4gi?!fOY8;0F(<^_NNJoTFUm4~{N@wZ8y+E`{J<$7p(Y$Z?oXRdXs+ z(s6tjH4<~^y`oXKP%Hq43-LJ*GDy|EL1S3Ur-fD^GA%_0ICy1R1jVdj5$8MkBI-No z-OG4&UgRS69&%RN?okvw(`K?M!g+(g{}$R$moZ*Xe^6Rt zlp8Rz?3nTvQ@!YisGEix{N_C(W~&SL&ibt~PEA%b6PeJ;Sy^sPbQuGqLxY2Zhpjqs z!N#y^+4K9>M2k%Wf>u>>w1ux`=Ajva*+BGNi9a;XfM9KqbL{XUcb}7=JyB2mEL|oj461&G*_4UVonQ3`x%2naI!SFSeFjdg|snZ<(`)a zZO_ZU*5ErL^3mjr8o2&W=&1Tzqezx7i0edqlqwslclL|_2X2`QFv$_V8X;05dn@6o}S52gVJHKx~ z`}xE=M-82%uRXi-!YRl?M>3$H^k3V&_-hT8=RFG3V3~T+iNSJXPxpQi z3Y=AIxfVD0+J8)~wljPh4noG0XR4;e1e-J!KG_g>GLiWtIR*lwRmVV)Vm8m@&6zc(pC*@S;f+>!vOu)MJ*pE>J|VBzXmchkD+7AtqvR(00$fPuLdvO? z3Z4GJ>R~SDW%!MY0p-C(**@&~nMtT*6B02KF7cXFn;FyMRmU5!@`T( zMFs4UsQQa%bwXsn_hud$ttfxMHN}#k@1BT!{4*Z1!DIuqlXfF8^@&B!_EiLhrrJgT zg8jnQL+Et%Gs1^**a`nOPuJVi$FF7~T!TKZq>=8n23v>xez^Jfmd4a7mI|IS{ZPHW zE4uzR>&1J<4Q=_`g2f&et1Nis=#k#y#VhW**;l^ig13XdU=3C(-03H+6g<8md5$F=#je{GhR4Qzp z#K@TEfl^?-w>a;+Zvjg}ldixxAx$>htr41K`BMG)0NY#TEmEnyh-r#k*m0lUIdeMn zG)aLpkOVF|Xwl5^Q)U?%%+|Vkz1TxeE4e(EQcs|)d}s|^i%@mvI5eYNFe(r0^r971 z#5=KIDse5TTCaK-K_DADpq4z}bNq`!eKdYW+S(Qvx%y8?J;UI)x zO;7oiJ8bmEaR_%+BHy3aG28-ueYv@%MnOCrpU=kUtJ^8$Rr zdSnjq8EIr(X485I4uHnkV(poUpLGrSO6QMXvBS4dM#~ECs87iPzYGM+)??#Ml4?uu zN;~5%sMn40sWyh9CChS+ZP}M!HdWXG?MVwl0jGQ|+z#>7p#!@o>Zyc!=`SNioa{*v zH&0ZNOjx%)hj#irnky-2@eMMiw-ot@5ZJ$r{4QKT!huWpUU))TfAEC-B zsJq9C4#?OsgyY`iV8rqCYNx<}fC==lhhAgfYJM;XOZehWM4Kte~p3f#W4PW z;4t&!ZelGS@W^#_`rKStA|BM)9&@+8#4X+hy0EAqoVP|Z7K)| zpW`Alk2h>n;&Ya*SkY>gsvxCqI(daa&mvXM9G7t_WD1{g! z?MiQvcAnvg2iJz*ECpv9qYDC{Oe!kS1LI{J)YxzSDZhLoKhL+s37;~_V=P)i~CjT#sl(! z2=}7=NIw+Z-E*iYKgzvLk+D$JcSuqGK|lziE7@kk2K1Xe5KdM)nH~7J@i`#e=%?Xj z5U-7x8Gm;q}%)z94)&mZ{jUNiSHnwSmr6)XBGMSnq)y3*lyxdk+GnL+8mtU zQ_Py8++org5MeG*rf5iJ*+p+3ut~09&$^^RR;PwJVA#Dl7N6h}B2ti*qbvp^=Zgg3 zQ{t@JN~I3i!@9dpg9m%nQRQbb7Y#5_uX*eFPQFV8n`pMd+OO!Kg_^8EG{3|yg4zadi~$wOZjWPBLNC$}R8!6ul(gRdYIG zhyhl?rhY)zXaroiHd-{&HU&}tE~2v0%HA-nM`|3Yk~j_B1`| z%X6}`Imd%OiA+IEz5IAlFn7%>Z06y(k$v8S-9r<}=gH7Rl`97dPC?*!5SEz!36rj%*L@Gx*HQYxTRVgLht z_6o0GFiOYg2kg53Qy8zNwcnt@r6A4&_b7vp(5sSMiDFW&~P)LUM6bxy#YiiP{P<8I6-Pg^b~0x zL-s@k#y1GNe@5lL6DOVb_q=Ti+s6onDxjk=v3YVmCy+&i$#k05vu=M$t{$~=&%v9m{nhW1YF$+uj4yCt*Y?+32oGQbSBWzdHtUGpC=Ftx( z&hs|8H;BntY;iX2wPGlK_MX>WPNTjxstp<5fBa0-i{?N4(YTDS$1&dvKf|-p$$i|0 zc3=Pe)&46~Y>@&_5YCH4cl(>p_M=DPGo%UuxvF?{8mr-m)9#Egr1S?Jrd|GP=V$$1 z*FRN1#2$uw?)`oBpj|1}N2LF7N{x#zRZ@+E|M*$8i8^JO_j<^Sh0>8K{K} zkYaUCS0ry1b)@aM??DqLMrcPCd<6P#|6ZpPf(hB`ig}9juFl5Px1{)I?MB9%*5v4- zRUuFi+mXz7pgc&->}@BW&DS(VN#-+2NvS;2O+3@y!=}RspDTj=#E~=igca{((1FS- z)>j<8cHS|UYJTThtFdEfBtZX#k8SLBeNrsV3zY6hjGdnKK zla?x=5rYD(ls3464a-J)pi2)RW)Y(lIH@BdFtlYGLgtO`dgKjgcShQ7a(dy%AL%Vm z!9%eB^2@fyyeP7g!m%luXIle)<9QWZWv(bffc)C{ew;MIUHXQYSq3bGJQGkvE#vgM zo&nbrCQ%9$J&-lO1klVAm2X5n8#%uIVj$M7+EqafbC6E071kkk`M*F(CCEx|XRt$9 zTY3qOmr;-Qo{+VPDp^pL%UgQ|p|ixkh<>YXvz9$%l)+Rt;BwP86e{wr=9i}XGXRBN zf3aQ8(7{SDg?d7k81@z8T@_Ur-cOaL)QzW3GJ}B1#&G(Tqk-=QydGnouUzx2u%+XB z2E%|}27SY^_IYn%ykD@nYR-;kElqBV^8bG3QREneBup>gf!}nJX&iNN&jEw>PM=`V z^DGx-Tg_g3K>o@|V-0;qG7AYTi9IIFFf&(I}lGcU2or2zh5Q&w> znwA-d?#i2BZ=xNm?4Nu?``I2{-VE@dz0zA2#pf_mK2PcwQ*RSSTLz(-N>tdt{7M-A zkF*_7h5I*{prBC@k*nq2&FetmWG=xGXD{0 zN-Gz|>V7W=Bx?~P7XVs~*p{X`el(dFp5DowCXthZGSJX6|8jd!smRNV5yte0CS7Lr z6!{L#PyApIL^$q-4`^9;z0U}l%$=}?^={ypPsRM_r1ppU#7Jx(J84g_!xs$}owi7Z zhcsjHwh!L={dU$?D^;|-ycTvFWPB0d{9Jd7clxj^IqudkbBaIvuGF5w4|M_5vp`o` z9B_<~X7gpKh$+1EK(FT4$DUKL;)Jx2Uql(OC41YJfTfvCUk-*LAZW(P>eElp`!jY# zYj=qnK|&kOLd%&q?#y~X>XGH$&rN!?W*ObYc*d?=>oC$c$7IJ-2~ z==h|rH(73hu*)abp4-p@nQ0>HrJL9@5-JLi?UR&{7a~|FwE@@;osUE$wfkY8uCtX~ z!oK8}`S5t#;>O2Dx0WRBcc?!v^7CbE96FM^cYWOS(P42%Q>(PjNiFAFk~?eDSmCS4s~7{`yAqn%k4glXP+n?P)aZhw1$e#Ku6I>MM(uMq_1QcogF79xhy^6vyW zT;aTm@dPM>vG>BK*e35dPl4lZOaNXjl5u=a30l^fopx)Div^YR zL@to+0jOBZna-pa@wf)s=J(|Bk_Y38Rpm{N#aK%`Bq10m1^SQ3d`)$v&a+@k?ON#!nb z7c52zAcgy!b_}2lB;Y0-)IQ zs+}l>U2;yXXp$BKqapK!{tN+Rm|sc5%6NBRReFKT znhVaBzzs=cFzO<`0i7T-@rqTRvIE8uw~34L)IXw0A3Zg-W$eE`DS6;iiJ<3-)1|RbNyb7i!vOr#Pk&kX4S(dz6n31aWP+rlexo;Ff&_=%s4XB9 z_EJZiK1T7gtdfa8q$tN|rUg?Mw|3%k?k9&jYmId*J5`x(n6ppggt&D5CI;{K*@Z=9#q z$2fa(AOgdyM`o*1H#EYvwnU&KF*JHqMD%MG-Mhz}n9DO_VAv9q$+iIng%LSzsWLX% z%pc72lg!{W`Yr=@ttXO#9rJoNZB3oZ!h20v)vy&&`^)XT{$wUibC0OXoJN4%{-U^|$$cMGrT^YX zB(Tp4HhNO=0rtn%MxsDftnDvXCP${VwXt*F+Q`nbB!Zae$?R}j|ACYI+wsSRB=wEHAJlCSTuQ_0$V1)aSjR41 zMyPr-++MEi?;|;(vm-wx#2+T&?$mov2|}guJ?t`MZnntGh`>WjS50jfL*7v0n`e9;>@L&qN66+w(K?ub31{noCV7Sj(+g z$A$ir<1*p1u4ZeIn;|622lVT-*rL5jr10LZYG*zjzgTlDDztowgYH~LS9SoxojN13 zkxLHKA+4g;!qlu8Q<}-0LPa#{*iesBBH!JV4o5ybB4e0VBepPx^;C6Z>?xyp$WLoW z&Z*7v3Xy6dkoC`H>wsonfBdo0sgk)D{kdp2{i8UA8oV&5BT<@O*#D0)%rm@U2w1U9Bs>AOUo>A8r$crv})qkfN)7Q3V7){Z9wb>0(57H|@ zJj-%*J<8M&)Yh$hQ9mM;%u#G#W_=+P3U<@>>c=f+eodx+WkM~QGFh!lfbNF-0Kg$0 z*ig>WJ^N`V)R3kJUzyfXd@6Hc(l}54UNkEdebUyT{fdlh1C-4`Q)4Evcdk{S^7EZY zf*WRqPW}6jO=EXqyk16Yd>(j)0RxYOsFgr69Fo<}V7M~jB^6Djq(s>A|7VweXkfU%{qR`3{sN_j$@zXYxZHVDCD>W8Re^p9<26~2R4br%KrH@f*SSoZjt+l z_bxfij?2a&Ul};f>dTMk$(YF{c(a7UTmw8OedQ>AH`iMnP0dOr?2dJGbWG3f_DSAI zm2&SHDrO2M!@hT%sR`r@>*d0Z3ko_O|7&HXaZ9V;yfWOhee0x^&I=nZvVDGSJlB*L zZrZO_s;&Ned;h>1<>PY>ZyZtToNX?r2_ed#@HL?Sbh#q~?mvKP$N7|`+z>YN^rCBe zv^N9z@)=`3R|%Uvo>@5T;oW`j{e?s!d2v4Rl1}c@?>+fLyFZL=Z#_IN2T`wpf^1P` z#89JI$DhtIeq^p{P<7%K#I(AlwzFBO_`twGz@Fkq>bFAZ1JQje%*)7pj)ul!T9;Vy zf#Z9SpIk{<)Xi(~Ub%OwxZ(Q^jWPOEt>j}@)U=L2123lgP|XivH!5Yo643fb)@j;B z#ngjFxI@G66AXVv;o2z#+JmSXTU?M)HD{EV@$&y7-v0S`-iEtWq8%Vz%fAIu$zF>O zs196t?%*O=F%gew0)>>?9=V>WC4HL&UKw7#f+PVsbRq+yo;{lriHX)tU9yCxP^>c+ zlo}b%Prgqd%J$lvysz>-$HG7h6{y7P9Hy6P^HGW38Sz8ejRg zTQ%hV(1_%s+gjvwG-FPSoh$QziA@19G|uXbL$7{Up|ZqfgSJer4dt9;^N>lNi2QVg z3s3F#rUQ~%3Kj9^qYh3x4-D>A&nSC7hSp)|r0C)ji~)?@A9ZEh!m=JVP=-7kwkn9u z)Ih#y-uzNUV;uVHaJBdMnMh3Z7ZTyVu}=-hyxb2im>h4&l8ES?Tj1>AkGN-tCeNVJ zidNgcn79)37HC#9UOoCugCNeb509dH0JmKd*y;-xSHq+ul7&widP5>L(?+FwQQ&oe zjIFJ4x-U&X@UjA}(lhK9Q+^7 zPoMvAeuj!f1WquqU;(pd$xxu?i%#J<)aW}nIzV(urI+c--Ye6L1srQi5xH-9ydLLv zZv%t;UKiIluIBb$*!Jm!H+&~g{L*79=BIC5-6aoSz5aZDyIJX1Nq1*W+N?Tu@L~Ty z3RNm41eH05-ksA`lbmI>x;Fu)qf>iGtWBn94I|-Mwr#uTL=;LzW)|8Nq65Yqw5L9> z9iWhF9?eB*uaFV2X3eBh?|_>|6mf377B*IsW3e0qX{BUh2e7dA@Fy)=^c77?d#TXZ z3{K=Om~6DhXKMfBolP%x?!BwFheyq$(ZNit1p&q3S_Echx?94((D>325puH)TV{r_ zskpgl>Wu~eUl-FEd%aA_vcZ#=nY{FsF?X>vSIa4BEC7Gqj~A`xdCbL1YkpDVd-IOv z0?BGM*CC55w~^BteJDturtI6d zPrhw}el&Xgzw@RMb5kS73@bHqOxyGKV}lD1N8IZ4g{C%kF%K6H&%SE1jCI-jf$KYU zOHH)7Mt1%aUe`v^ttZzV*@zXXKlC%!=f(HYjEBIm!8NY9U?=2%D|>TSk|DvLQs!ZzkgGP*q>)pJ^AwfyphbQd%{BL=}pz}1s}F2UJhZgHr;}k#pI;(^^ZRWE~94t zbHc#2%Q|dG{1O3H+nR%pyFb5{LapuRZ@TWi@vkAbEUrARJt+C2!>u2m+!!$UR8ur4 z#w+6EoktewU3lHU*M8kDcSqNNu9a#I+MgA2;?0Fe{ts7g9!O=^evhjpkxXSOLn5L~ zg;IzPl?WwCW};3Ml6fdY2vI0y%4s6W6cNf0Qb_}eG9{EUJ;pNq);{&TpXd9#|L866 z<39Jf_ukjOuC>;+wu`O~T7xN+`VPczM;R+sSb_wH_g@SOT7#X4X(a43Ce;i>y&pVhLFYh=NGG>px=C zcEQXkNo^3g&_u09wv`L){%dbt7WSM4y&n6B@=Y;46gx2E?BOq*rLI zIQ5*J0zbU19ZxU+3VT8^^U{Q{u>KU|%#{U-0#TGb^w-j#dh_O5L?10bz}4@9*HCvf zSZqqZc5Qu2L4BZzl~D|ii7Hz;7ow#O?Wsb7x_EC{iSNp~-~OXKG4CEs)oALd6~64%*D|B)LCknmOz59%j; zip?%!|Nny2tp>|K%|OzT3j8Mrhm63ZnhE^kWtq6$o@QpNAwY;gv9a`>p)H|3VHLU*k*ONc{lJIkJ+6Rd7a`IpL4(hD>eLX^ zzCCz4x9z@D9LWAF5L>L35}j%Q9fjdksWjAuW@f<@G7`E8K#`a}Mw*v;a-zpPZ#v=L zDY$8Vgxe3a{C20OG5CNis`0>`KYxDiY$5NJD$2YNpA-!2*Dyd^n+f?${prLoQ;|c5 z4(S5tZgj{s$wBt-KdDP;)HTU6K2~6An5upAk|Z1E1=c6-!BH^OR z%uf`)=izt13X9791ez9d5vLtL7jeQ-p@I7=u82HpY+LvKOgBPFYWSfUP_H0j2&&`eo<&Mzq~y;DmoBwhdJ+ud>J$#+e+-tLoR0X)lHy(S`Mhr zhQoP`5P-U$Kw_3_UoYT`YEx21#((e=`4!RMmrS#G3eKn`lZASnslYS}1(pq@Mh22A zX!7|jkuZz01$!#FabvvX8bz_prVHo}wnDo8vB+gDjzF_|^t!HYDIkSPDC9$tXjL>d z6`YGDkysEx5ap3cuFYb!)KNFdFfRnj9RyI974L|QEs-52Z1+;IQy7U$!8e`)V|SzZ z$B*%eDDAbVug?Mo`=~nc4n|%ZOdm3$3Zdfwa1HdTF2GiKhuYYk0XJBveZ(@ zk)le`Gs{G2_B}2ki^ZAp8|BGJr!Ph(%8rqsuiTL8cQf<7lmGlv2|^Q8v9J{ys7Yf{ zd;yfOAD;%O6Vzk%c25qpKd>dl(eL2$9gVKXDku@vfe7rovAK}@e&=Ld0HlQWWNd7V zw>=xCt;GA6c<~I6j^-aZJ^TC22FoX_f$8tk(yC_dSVtbTz)UJ$r}4qi;^OeI>x(ov z-r)SB4Hl(*$f%UMA+#|mpswV-d8}pw4=c{O4F|$)iyjrvv2T~!R4_3>zI3BZsDcPE z%{X(_mi74R@AsRSgkg%>B4frH%IAkBtQ^hQau9)S*|KGKOw9E>kWV&{HyTLA)8>%F zIDX7o0Iqu6wFF<mPaFM3zT@?UmX?+z4g`u=aFFu=p}{^! z$q3smg^eFYV8O4sUx4MFzj$#O1JBvCn2hm>%>s!HXX%4fZ%Cm8r`Mm>%n-M#isM%s}5!UPTYd`Ri8V1#ICBakoQ9_!+1}VMDp>t`pvvE4Evd}o4wol z9f@?Tq+!(osrax97b;hsDm{nuFxE&(-Fs1YO=Yz#+QtByBHa_e= zi7v5q!BF{q0gG7;ioo+e{pnI7{=002goTASm=$pVWh&r*|K06l-ceSp9)_EjniDqj zA|C>?402w27$wgd@a89z6BAbvz_Anv8=ssN!)ignzvF3>YvK&UHS*k8k4oLx#-Sao z#E_ybN_r{R{?jm?xd`ujHZqq|uU-vBW{snVpT0i%l3F0Dq{M?=Tnk`by{Da>*cBP` z3pqJCDkXXdHMH|Q_*LZWVZUG5Z$i`ed5FTp_j6t{!U7qFjdKL~Iq3a!7cQ_I*W~1U z%le*LKr1BM>{9aH469(tu%ZB(KF`mw*URXc=8YQ+2!LF-Iav*)pYdpyy{xyem%n`9r87j=5!Vf~9NiIrMddKwyoR$>* z>{DWYGpEg}+gdD)jd^s}9Y`LqM2MG?Vj(#VvPhOCuxt}Z-WQ6VOB+OhqSJlL?;D>; zZc`(7Qw(U3d7jn$6{>L~;x5kkDk>!dngwM!M&&F86rJ{TC4TYlDnBNiQ z?d7#6j-H^TTcd@e)%R}V-&ouQe&qDv3HsLSSm1KHyolq|XhC@TQU2!E#ZI3c`^^4V z@lYK-wKkBKhO4P~iFjeG8)i-fhPe3<8GNb<66~+fFVUXl*8>3>A$W0CNVUx2^UtgcTTK_k;c;E}ISXyZG zKjXg*%YAuAV(waIr^Qt?-*xlIbjRM1P_15!KrAD|DqZ; zdEqFP5_17c-9PeZIZ&ofuj6{UT3U;7s%nJ3+Al_UM{9zOXj8#L*S)5&_~}%&EC)CD zq7cou<$ExNuczRSWaB_*+Ac^F>Wd;GSfT2?2z(kTzjv z0R;kx7kQ4Ju;@q3Lb5J zq_Aeq0)&=$vd5M&llZIdqZ>s+XL>OgV?}R~19SlNFW#Mr6Xa*lXx{CKN-7)dl``1*Vr062hRh6#(4#3^CNe!lJ!A z{7)KnUWEA@dP>q|@l~{M8q&zkF+HPWW4V3({IvYWx3;7pTSNI9O33KJz&-}z+;-pc zsSt{fa)KsrB-sq7BD+-li#z&v-|vY*Lirt2EMe>Ebr4{(hiI-5*pyhm#q}#Qv<8h| z8or-kaZ?e5@-G~47$4cA43{c4#^Ir&qDv8M_itYC z=X~;KK>05)M9+olS#m3r(XbPnP z)icr#!M#^DoP+kQ-cv`11zc?uItO%OE>icKnqGiLz7tq@4U)V}v)#AO{P>Wt%|oW$ zvzv|h)VsBltm|MFA}P|SDr|TLI3r`z;Ex{>kONvF$5x6+;BH!i)Qaq<*!OAv$nr4U zU(mx}(2CPpmXoRq77B^-a5^X19oYz0uxKTJ{WFZ%w@I~xZ6WM;wt2~UMDM+E7s76R z>FPR%E~^W$&*a92x*(`R#83*!ela4Cl~SUXpFos0Z8Lok*s>9WXr~7@^01PkUzcJu zpOT!q3CF?oo~o)5X+08hN#ivZAdN{u;W_`?b}0-idD-Ne-Jp=M=Ua{nLgA|E*s8Tk0mz(U06|)z`wEnwmL~spuxFtP~Ow zQtIZ3SPYVW3Oy>sz*wk*yerXFfg(E)l7G9Q)H#lb$$sk8DUvfmjgn8#nI4~!o*3$2 zN6wV8nF+}67Uxa|Vh;xdEbT?dUK0~OVk7-3Rb3dzhjB2b{i$*E@L}KqDv@0|L%)>h z0;mHxi#0qyl{!i|nv}88ze-VEg;6c!-{v_iLIl4(9swwg@Qft=rI)>MSQtU=)0%XX zz;lge#U&*qj*%j44roOeiW#yDGcd0fK$OWf4h2CsAQ&zz8d~aJY$>h#_U%*Zqpt%_ zMTh0fm*aSnsB~H2?igscwUUy%87fN9p{0U0oIWRiy9uQxDR_r zf}Hh2_@Rem8zNc}$l)IyNlZz(2+nHDwry)wRM0XocDAI%XORSbQ0^VUiUr1*;Rt`6 zw459cjNiKu@-d=R&6H%v>|3`OF%)bQ*G#b(2S4*;XRbvAcZRna+UO&w&{~ju<{45V zKnUMH7jNQyL+(t|k&0&_Dnh3+3t4LcQCjcLGy{?bFtf5&0rZ#k&RTpHxMnwKrUV>< z4TB7EotC`g1*S==K0Gi1I9qL#k@o-zHQyFz5wv(To&Ap3bv+c2)z7vPUTEjsYq6>$ zfGk$Le3^MJmQz*y>Pr%!(v76>IN>$>Qq`i6azqMl<59SFKkq5N3PJ}@vd`pN%n|tM zFU7f#Gy5*$wx&rS+pIx25{y%87K%MVh=x@vghM|{|lFVA3cnk?4m4+TKI3uh-+BEAt*4M0FLP|>evFIU> zNkffw459B|-nyts8MkuF*+eTl_#X_}TPE-NUJ|LvMXj~V6}3Edg^nR#$yTni%heGi zCHNn33|k8-=%l1b$O*!^6xlEdzqOSpla?^j+XijJ&Uo%_7~yuI18@o+s5wxyib+UR zQG5CV=ns7zexU3)pxn?B^xS4^a_6w?B2n6l;&ry?9)v(5s1K`vp02?p1tPcFLB^{9 zpQfOd&Ovj+c1T()Hqnc-?pq|iO5P`7$b|vVKwL9#mY$nJALKC(&M=GPn4y@y0EJ7O?F-5ssFUBmYd2wv>7kEXO;xLE2vpE(LD}8;(9t1`ZA1eA zda=nS1kM5z=pM?^D6i$WE_NwcQw=!zki?3GK+|{hf^enJV|`-<5m>4QG+%_S*G z%KBuHHon6jVXYt#(Ry)JlFhJ+dHHglg4;#*zx%=_jQ?g==@+Ap4+I`Nb+A6<-<3^? ziqq|9`1`L$->?08ImrsKsB|wc-jUKS1^@#WH4VBzb_WLs!w#(O;zJ!tB|7&s)*)I$? zf)#)O!!R|DTlb!kRHTrq08*5lz~+8_*zP^XjK&vn;CnB$)W|eomWubxGftV_K{1+a zq!Lr3b3p5NW*A+l!R&?_BoIwhD8eYT;EjOkm~eJ$AYOSlh%Wc1-0^jGVHYpf$oh}4 zl9c%gDtIwOOKK~&Il?2L7HA)wb}}+r0siwmKnhI`#-&_bEfUNN!RiHLVYfs=khTh_ z%QJx9Wa55vQ%Ssze8UFwXn11cWLOb-l~JZp*r2mVykC0b#%kQmN{;W=QZ35mVIVyq zDJgzfeJC44-yVyjkL7tn4wRLais@*Q-xm{9R#u*xo{sp9IU`M=sJ96Zu06;)*}gA< z2M-sBG`~{yiKFt2M+e1a@m9R?0Tu?FgEMs&UnpHDurw zc?eoi_^4b7g~ApfPmmZA zRyH%UXfRt_K|I?4aw1a?);~@)iUMrN8A74F(#SGa_T51;4<{bfpD89VT=8j%4LlK>E5dejVbj)zpG>-} zvNA79u*-BTkv@9=NA28#E9!cxgt&MGwLmBCeEOWKPraL=At!hdF`WZG$)!nhWm&7@ z%tqqV>s(`&CbWNG6D(VO8=?2_HC73m8!}wD`gIGj3K>OdJZp~BHXk9Y@$x=5x6QzH zDSps}oF!Ap@I7`K9B(S|Dp8poR`)!R{VheBQBr_d0+A3&DzF4XVHDR+3V%xDH}qpx zK~3Mt`yT=TMyb0SQNEwtTzh`eqD3PR2ihX?xl4qg)F|@(r2wn?i=fGruYacB#MwwH zJ{_QhIA(35f~SvZO$CoP zHq;@E=plqCvdF_@l4&16uf{QO1!iU5IC}2>4p_w07wN960%@HC^=|O_^GX=VK2q$E zDqZJxUh8AzLAIdej$!LvBo2QCQK|}vNJIwf)34Ex&4r*KicI(N3s`hBBKtJ#KtLD} z+6%bgJAafmd}PCa|4fb`9xIeaF8mSJ3VI8H40HQl0PtmJH@6kzIPZ0DTyNj;?%uOo z2R^6mr^CRhdjJEMB(G{MYDxV;r$fkyrDyz?xrug6-a`PxjSDSLk%DKp4zNlXG;xeW z#)LB}jcJtnAP}ACGl`g=ik>gWDPNB3G^aCg!pP6|vQ=;-=Ol3r7n3d>6g2V)6e0c1 zjqtsEu62xFMN!ww$#Db@yXC2%TmU1qCt zs0^yMxg-L_7Eo1JOGD^(C6egl)sbwCR39K)l)6a@jxKW~8yl9sQ*^>=gal-2X0!p| zPp=d&#UR7z-`Ek!C%+Jrhb;*xb-EW0->8E9S~z5jw9?Yg_NCZs$=*wlE|k9-nzX*Y zfdMwagi^Ij5M22;*?*iJq$33>xM{*Bhvj5;&4a|DWA*^@HNfA1Euf>uxa~gzzM?ek zZ;1a&q<9GwBO^$82cM~&t<9sDZ}b7gyCSptP}8rXD8sX+;fFqXmA9SS`;<#@<=}FK z(s!>HEubE)ucrSl`9qH}dOv-)BB-F;;rq{<6K`MCpfrkpjZd`BGa-M+wGk_7fI~n) zs(%~Wn_j$lacgN~B+%^}c=zO6B3C0vZth%{AK)E|wSb219EdjwrHMAhp2`R0CXL_R zAVnU#_Q6qU?V2?jmE>o5fKcr8^E?Tj)3dXz@F4aOq+4dLwM z_PbSp0Ky25rWRCj7S3TBScK6c%-F6Jfq`5=H6RQxf}q62Bk|(^!9Jo8R6}PGDFgs5 zvn9~?(sU}dpw+&s3PJUFCB5WT)X+Pgr@Cv`F2YU{SQlL}KRLy0@IDuALHKPmpM2Kbg2bs)p=Ep)Zm7Zl$I?jNKB+^KumNTtForvPN56c(-cmy z9L5npK-W7?CYO|KBT)m)E-y;f?k`>#H`z{VGCv!Je80^(O(M~ku_ zsMsn@rgi-2yfB!!1{dXXvM)UN{0O2D+SMS6F zKhS8WTZ?ZNMCA3_h+d|g;1%zEcsexV@?OQii;mN9wCbh>c@A7g692rY-@gk`pW)PF zx*TCYmiG<>FD*Yrrrc}G&RZk_*Mxgl7jc3k*!w`hs9aP8`}q zbt->IziT0eg))E_yaqmn*X8Ix(|eU4JD?cS{RGJn0r?3M0eGf03+fDlL69V91uw6% zvnH8jROF#LjlGob6bc(KAy_e3L0!a*$iuI%r&kRMnpFM>nESx^1H3IqKhp((YByn7 ze==G7W(DMd!zk=-;xuTKGO+5$D&~I+ng+JxI6+&wqMOj^!0|EZ3J?y zZmeD}0F0~)T7Q;ub2eoY83ay0 zTT4_Vn;RPT$W%qjDZ4do(Sal-RFE7UmE96lkM1Yx?Kctj zksM0nDtaQQ;>};EHU*`7P-bRkdNk`@`ja;i`Cxk1`|rb3Bevo1X0`)>@I*T{!6T-A z|8C6+kB+XzI&y)}kCBnl=0Br(PoG*~+NdND1%QbpwA7Z(0&Fmw3d2(R6#0GJ+woH& z8&*sZQY{hlZeYbE3F?m3SF3=DnZf@3HjmJl$d15U1vO^~SopVF zyE;xsyt1;ALPSUO?U6u4&8zOxvzuy3yM;B3C03M}O^g{;9Gr{{YXg(y*L_^<%AXof zKAc>QzIC^j@A&g~O8riaki+!sMn_|-r3Z=qyS)bo2fxABC4C5yhfdIjn=X&i4kBuw zAa=EKG)Tj0+~BB_ySoxbI~0AI9B`<3xGh(aOg=-u%XM~i{Cx=FPcG5x54)MZ4mgmr zyF1UgVnN*m?4NQWLDfy#!b#6kuva;}PO2cQ4AAIY7J0r#UBE7hY5;pQfLtwPz+Q@z zuR05wE%K))f5C7q2hs<-qeu6|>{CHdr-ut*dr8~9xKxr~pG142hygy(LIN8Ibuobma!fYR%a;Bl$~Ay&Y!C0YtuZ z?A>146KL;n4pC6wIxbGdh5x^S-HuyO24HHAg~4ci?AJNvBJoUCW+21rw^ z@^3U^47VP(Tx1A(8+G(wdOHosdCbcm_Y?mT+$OmT(N(Po#NBDr+rNYA<=(Xd+$M;n zfA`8Y$jpe~h)w}r?~{nF;}8^#vYh$COA<;*n;fa2M2O-8DMlCO(#(T%c);{JSeTD}=wkWppcE$I4v!y)-%1d5FjGe8i zo7;YaViZuC{5661`#FO{pNd)kjvlbGwIPiQ{vFHj`6gMOW)}H@ft0 z-M$S$&q}nU?9$a;W@c`_N@+75m$N`pAc_f6{~9oX($l24_UPJ4rjIBWPClq4oYf}? zx4G_k$s=0vLBB@yrC3{C*k6^nmmmm}K#R^(zOAle_;7I@`j#)Nxh}Q>f+ApB@qP_5 z3V6ToNQI#0r-BbT4=;j>W1EdH9^SySTGe|6(lt+*S9bOGuCspkcoO?S(l(FfsDb4Y zGoP%>LrVeGaw9zRFP;q1tCUL#i+mksTN@H(t5K1(SX8zs(R29y{dAK zALX9+9F)~YVoFv!urPd@(PnqZYQBwo6Wj#V=)QcC#VGR1$jFca=0mn6dr*ZzO90NW zJ+@hZWc!tFltu$ajdC2lJo4@BS{zbgG+_yp44{irgg-X~4O8DwC=7ocZ>_4@`8PL- zLIK2z_W`?!!!6Jz1;Yw|=>g8BRDzq8JbbtU*^b`aDT@BCtrii^wJFo$yVomyC|$r= zmgB6vM3?!@xtnI|&a9vh1^%EhRo%44wwMYn)vn$B!Ml^Ig_XUsz_{|YRt{eE!5PDvX6Z;aLy-Ha%^t zii!O`}1$VJ?8j8 z5&haRl6q~faWCZxvR3{6l|hwRN8^3hpdXUqWZos&=lbf;C}OkA8xG?mix5{?P9V#4 ztP2;X;p~?mvg9XBFJSVcP za?F8&4`cmJtOQF&>b7c&pkZ5%X%=>d?WlVk^2-$%2yup@RSVjC|A{9zX*8mnm0ZDM z7~t)#LOL=rb+>z*BrB#@X8ZQl!1OuCu~QJ8G!PaGZs}G5q(Kw<yVtHFV01+jjd{B#uq9J_QOQ-N#ED>>!# zm4S^eVb=u375*P12Ied+F$|RC?#%aTY=Xxr?O0 zLwGS}k`FHp4-c0^);|oRC|mgLDe)eiGr}^#d8%Onvp+09J~|~#YEn5m3`FM$zd>7= zE{m4(LD4^=xv{wY85Fb_r)O`~D_p{WKxOjq80Q_`YYRO*JQ7`?2ZfNJlW5Vf1zIUw z6Wa}4Hie>o70h{omk=O{ZZXhcI3tpPlUQN^m7qQQAE>07iapu}$umKfT&5?SN)$0$ zvAfg~wGtpC{`U5EwxrA0`hVfTj!FV60=fX)+Zs9j>lcBU`8L@v1Me*kOyUa>pQDos z!MV}i+UIZfMNp&6rUt)Nn0llh==%CqtiPeYz8Z_!iRTQR9=r?8G#u`2^s$6&;?>ao zFw+>o{50$^v?qATFEGD*$B30$k)nD9EA%f|-#A4|3$sJl{wS4xPyRGZAp4wnhPu<| z)j2oHBs9(9fK^bH@V_Q++PU#l!n6Io#Q3lMXSbG4m-{=Yl70UU>R+sa}$N+L?tuELUwPXCU`5TAW%Q-YzzQn47uKJ?h9e* zA@BZQSz-5D=7m`J6u*hrB_dQAI)(wOWWFfv@OO>EbS-e3oaBxuUiF~h^d2sTWh@4^ zNUF`^%>1wYZ?!53t)WMD1Q1?hn}@sqEY_aC!>RzWt`cjk*<-h*(ZcFiBsmYlqlST~ z^hjd{GVMd#yc&>7<)ZJ+7hWz~T)r?utMTtzm4GX;QmQH>9_tJR@Tj7+JKcX25CQ`> zow7t!0jFcq4)6x23O%s1^AA7dbdoV~Pk z;qPCW@+HaJ%;L8@T=>@KdhmtZU0C4$%3L#D_k&C@`enGcdTssRg*;ZN6UCFU7I5Bf z-&ul2(rSo30w<)3&$f8!8*VfcfnWY&&IoAMKH?wu;0Mmb5M($Y)Gn&K-*k=TwM9F%@=-eMx)gqC|Tex{Cfb+ zhyBL9l_102)7i;DWNwylGN2q6&?X?{qsbjbi95jqFfuTEoGzc45Mp*WB;eNHEB$Ru zli9y9+o()ot?Yeq(aW0aEd$7NrgK8Fqlu(X%MUN^pv;U$R4Cx&PlX!^LrvBNq@a1% zoJV$nKmpQ9klm3Mhg8sItY#GY{!YcWJ_!v`o|7D3OhSUBHCddPLJe(gg4i@^PkPl_ z+M{tgrkqpjW>URk{kxw4Ko|rl!v+a0wDu=DgULu6t)yeiZYl)NdpinwCwar9geTe6 z2wJsl@ThWp%h;~JbX4jF&Q4ShNMFK{o|K$HJmpjM-b}JWAc!15h}f<}5Rfju8{IWX ztT|V$V$E!#ZW7|=Ci9p$UeiWH!me{ca)iHIqW}kcrqm7D!!sgeB??C$9u!R2{~LM> z{|dU1{d+giaherG*@V8F|DUpncK!o;Jn(hGe#1PR$a}((IToyL5IO{Tz4Mtf@qat5 zx^Edsa`5o1`MdZ{7^Ki*f4fKnldlCx$`Ip%zZH+xQJsEN7>fNBLSGfa&%;i0ux*7w3nI-n4-Q`5P-1R-B-23hP4Q_5lJG-6z!n!nR_*nBz`eU6 zcaI;yuiB!Cm5U^L*q3HVwF(U9g*%9J!gN^E&N~2#1!hlnJ0$n+CjdanK`u-%Xq}#j zWaYnE6g0u8Q13hXfb0gVLheZGJ2OA$_p`Nt6+J9Uwg5rUU8#YrrC$+<@iX*u5c&-~ z^TES?5MeG@#B>8oU5Rx$umZz2Y*Q5wWKcSe%)~u@wCd$3be-u&CDWkc#05o z7D~1V-Y7scP&)@%O4}-2BSEjEp#sPKS-rWTQR*LxZ~PozisIPu#Juq?7_Csn}pc5dx{gw6u`HKsYXPI`lJ#sLn+eLN8>y^e=G-`yy>NnV&g+|;f^`fToJv%{k` zeK+40){e@y`u909?ei6i9t-v@>E0)!i*}dChsRvVI zO~W=>ig(V9o2|ITTV-1HLo<<2eJu4oXY%=wMwR#ZLhaA1GuV_}WG6UvSK5~-DiojX zlu_d=*udmeGTmIHANw}4$b8cIcV?pGE2;b=-JG3~Pd2Gol}ih*iG7)}ExO%HUHT+-BHO>Xkg)bJg}<+hAixD4)vMC*CKsY^y9Pe`I?+ z$bE4C)!F&q*{7%ed3glr@gsf@7ekwx$lWvD!a&D_a2@~oi7fXojzc# zaQG)$WEWa^XR!HCe`^2Mp+4-;Q|jEsIX4`78qoHd4en~o(v4V3XI!_;{c5OSW^v7G zQXA@!#rX#(0*o&TJ9Vfncz;Jrt+^1xMAudk#S^jIn@~lEE%q!el+oh4+Ui%$WXLJZ zZ@@tSHv0lLr3n4r%b0@nvg+(KeSNosh2rGn3nP^iz`fFRdP4jYN2Ast*^~MN^QB

<~QC%OkvWd??Z)|+HdzQUme32JKJ zr30}Zs8I<#;a%2le;?R%0<7Ha`byTXU?Fm2BucnFPZaDiSU@YFG}Vnb{E#qV>4gN1 zg1o#Nm|FZ_m0txw^aT)mEq#xkUSx5E?jwnt$-JuJb>J+3opZ4;A(h>m6a!fWsZ*4) z*V6Z?JS6|_NE!Yq3VC~sVrnWt7;_f{4!EUmp#E@E+3r8XM>L7zRFenGU3E|VI3FCy zvijwQogvp)pBTLCmE@HZeZ!wzw%Y6UpM_6er=n+%3>g<(Wik6>wLtGgy{oZIc~-`H zP41iCcwPXOifTZ#*5?O4O&{W~3b?q`nv11)hi#o-9r#A_sp)|F$!1lv)@x$@2a308 z_VgRHxR3j*9GOy))?XJbj%C)Hvn;K4<-oDJW=!%T0sRMs%C|W_H6rs4asth(S z@)&Z7SG;{&+jm53;qukHt2{dNG~p*tJu}TKYj6F}*P%=C2gjc^sn?GTmPiaY9eR`# zFu!7I`sqh8?w40BwHzHkunu(V2dG4=&;MGu;Yxq1n0@}r>B+la%Fh|i^eUGPKYGe` z&!q#CDPX4CYo|Aj`^^vZI2LN4rdQqzPR zLZH=v;Yvbe%2mLMgy$zS2YDmWVF9=t_<#dlMK^8Y1~8$GZOnV@+XA48O#A4lT-0GK zphjxJG5J==6Cg)0t2Cm=->4(>u8gZ!my}ts$Lq9?9=at0=@^^~fljXk9Zl>GtQEo= znxwEXA!A0i7Xg-26sK|1==Ax<=KR!yQQWQ{+AU7Dz+2$OImdtNqBbSQc% zfj9gzU&Lvsia^a#;I*0_yPm6|9|KtX+hTqa2TRGr%_AaxBHY5wJeU0 z8jBocB}{aAR`m1_f7`d<-2=Pk{z(H1YWZXF&=`-WMR^T%O$*GmgGEIuxm|CsZVjz2 z^7^SJq%O6X!A|UJcc#Zh>ygJ-J8$g&(ElWZy?UknebX~#d2MAS!tR4fVKE$&eeHdx z0(=C%e;02k$kMq@F-fRxXsvN9?3~jd^wHOsC>`6MNPSfI^1b!XkIB3BHs1SrqWLv< zz5~$C9O=CLmpP~VRG-!MEo8vo<*X}Zp4Gh8@MMeb@tG8|9s4FIDRnRX=O|>*tq+;q z27j_SIEX(qRh2RG3@-CIU@e!hfZ{|U}9fBcUXOI zrm>7c2GzszaglSNFcw9PHX0OwAP`-<)HJ3tR>1{qt2zmh7IDB_2w;iU!cGJR5)=qR zko!?c;@yvN0&~AkRgXqD+}O_+ZU8Sab#`205@cP}P-L#&1nyk~nC)l>aD1`|{4EID zAbL*|7#k#juk24BwFHC!oJ?F-d?Tb^MBE4A8R@`s0p*o=9bFH^2?8xjgsp>#nb>Zk zPHa@jXrdZ^!Y+k<7NsM-8-*978UFnSfZfl}#oWMHxS~4HIEx&asG%X}tnGUo_3qt% zq8G>RdA`CbJrAd`^AT$tPtPs9tl01F?SgKJxVWIZZy;sUgZ&wDASXctL=Dw7;vCj7 zX3`|_&@`()0j{5c0P+jLj=U#`oD7_DqfGzS{?iYnQhA%3g?>-e=Zre`(mTP@Vowd80TJE=hrV*kJ|0z zx+lK7ZArhUzT)xpsodu!mS!!>fAF7e2@qc9pg;0#a_v>E?($t%U95jNG3@Ad+Bgxw z7h3o9$06H^-GrZk zF-fK09GvoPYt402pFGM_^*Fg?%hrSQ+kAdx*^)5f_;mW@GPN0JNxQte5;1dglYS#l z=Y5tw{Z)V5BvvlU4 z$=$o3FJbun_;&TvxikDvr_EFQ1HMUZIPALO=uvHk29No>cX_+n1GZSUxI4#mXJ*$} zec14+ui0{5g<+NQI-82XopN*2voo>*;nkmu1>f9>3T>MeR`nJ}#JK&UFf?>6qF}If zm1#u3|LhM7fbVK|^gab4Z&}8)>5hI!(ATe8&ZjeH#m*-xMODYF$vPo}SxeK-1lkf7 z6}58X4p*G!h5b7|c9rrX$STCd7zv9G5Qjtzw7A|tbH+(OY{zJHKrlFmeVue((bc+i zp}9aTME%&IbMwrhUx67A0nYzT8A!&DrAmq2d*|-mwcEENmxv02_>lNzjSaS~f|5=P z9)o&%fGuGu0=qxuWr8Xz=xi!URH=udfjXL>&FB&Z4e(1KwAzU#H9J zZj{dzTtqTl4Hpehi!F*?NV;(Jd@Z1UtK5&T)82x)j1*RHAI!a2a|4+c677!#C$35c zP^nU|Wjq16GPL0US!t<2C2&{vldV?2|&;l$Iin@n+ z^H4M4I06lDZo%IHIq-rGL-%gPfAPnsXj=iSaG(BaK_Y-I-JNhD-5(Gxk+sB=wuz;Z!7{0hjk0 zmDmB8S1>+{WUu|&F2!J;DXR84nJ$Ap$=Ht)9ZPj;KTr&@o+JB0)-8@(F6 zV@L3z_^mawyBc1&@6%Z$Gqmr)gp!-q%AC2c_Ydef*Rzg13?Hew;g#IQ-EU#*)hB%ZFOY2KeLqmPk zCZAx=^K4XE7E#Tj!SLZe|2S=gdUW=8B5i`EWqvO!I^t#JML~VxO|~@;D{6(=gLLh; zWsGI}H?cJw8C|EI#Xth_n(`BAf;r;8V$KW{0Y(*1(AR>PZ`$T5pvxLc$M%}>?dHl1Bf zHbh^5E=Zs+pv_be&HK`ZqW5G0XY*}7#z7!fqB}raT2@{mdL1!!YKhwbrZPm;i;SgZ z1L*rxIQCZp)c-#nabXU;Wyk7b2U@s6LX#F2s4~uFXJ^+Jf$cyYPMS4PGo#?u3?yrsqCNdwTulH*Xc;%nkk^aqDL zJhq}i-hc9B)2J4=;!)zZmYDeIOZN1^^>%zKe+GJ85@L2UZY*73$j$xjetGF`_DB}@ zbFqPY*Z;3Jo;BO<`bYF&wClx-Wn0cj2_j=fa(VXY(@oxE!@xz_8_M)!>jEQq^J9`` z__l6UkzAoLHNEZTsSXC5JYncite>J=TNPR@DWg8f$qyFl0HD-?=6x30-4V|wDGY9YDTa3BA>2R%2o99mMW$>gD~ zXzDQ@k{JCvWa5OrwvF09gS=^Cmh2|yVk9kt-SW+I7BlE-Z*m2wJUA;F#}ij&bTaT} z#3Hlp$sC`nSEXc`{n&75&y(eo-KPzcGJ|p}8KW6^gU6Sh&AijUwCu%Rw%sL{gJ#BT z64>mEJTuCEFDjsxSVKS{fH+CScw(I=#!bec;>?386}AZlz7(m!CBh-Cqs<>Ob72 z-LyYHy%I!QE$9}~<7^<^HJGXWNodZXc*Q zuP8sK(NLD%s-rJkb7?KR)Tg$R>9iv&vlZ2UPjzR~CPIamXB5tz3RnflizkcA+Q)_l zvaR}IgnjNrv#WLPcE25^Big*rdzw1#qOd`QrS!3QREVK-ux(-7)kOafK8R?`hn@zo zFm!&sRetZ_+8h^Ko2*CkZ{wx6#?N1Qxp;$V{Z7_P`ZJr7*;iL=|+W^NLm_Mq-Zzp@r@ zVNFd94OoObuUOh0y>+o^Kn8FQp(;usf>;C5fQai;Jle)eyGRs*0{( z(Smfr(tB=nV4B7ZPD7fQX}xgF`~r zsHq8|?*&%*8R<3Xx}U&b_!!ZnhA3lxIZI~M9{~6(@L-au${afO1i`Q8#4&%Aks3r6fNwf zUZXiM2^Y5NbVdXTn9Ag^eQ#ee}K zIzrbaf{fVBdm%lAE(nV~D0CBTgU?^Rq9AV*-?vURpkQJ4!_&3(fhl06*xuWuKqjeWF#Rjn1?F;}SUKbmL}^D@&?q_TY$<>ST44)yS?BSW-O zv!o*&XZbgY=Gs&e|PI^j59#c$S@ zxRBx4`!@E7=8NYvPCcv8f6&?g#E#FcF#3`4fMUW_VmtG9<59*v^L&p4Lk=HhINKFb zV*TvhvF-jw7iNWPx?W#%ONinz^rJjAb~Y%!@OzmzHwld&PYvDpnLEH}#L&6&>xDGd z$D;N&TkP^$Q>Wr4Z!r7JyKvU=3U{>>8y|IDaA{GXT-Twpu9g>zYmF_dMjEmgF>GI$ zG?F;`eqeC)MT*bIk=oPo15eHU1R0!NY6X+#ejE1;jqH-=uB1#JZRvLXyv18MTtc|B zV%&`HW&Oe4$)#s%JDazywb%5_+BcOqIW75OeN{?;#k1s1-*dK~=DKCJW@kekC*QO` zyJT{T&_7bLik;6dWQI-83F)pZ{3W`C(fXgE-<53*#WlK7TKzp63`<6bV+0;us$+EH z=L{5ng2OwG3G=L-ZnvF!G!YRP zSa0sMWL<#5r8E%+*)qXV%}28bGFqAfBAvmo>yX9HcOklo%{&Y->lw-Zk}&CAF7d4GwUCH+ih8DAVu6yM`jd`> zIO~_4piVe{Ia^?Urmyq5f1FCL_0EMG)V*X6KTO|SFs^-*ACRA7g;^6@gA%T__eG3m(mY+{UlehmeXiQPsm}h*= z{#6*@qVE7IxfWeq?R9o3_m%}9Yz+5qf))^m%pa5yu~E3@j8+_!*e~BP-Smd0j<^Ia zTcxy2`BXavmayn}0?-XK23;^jltxe_SQL8tH2a#-#XQ%az)*m6s>@wV@$G;ot1}Mh zL{WO=iyev7l&AyFIa8l`Kp(+MOXiJJsP4L*kPxeQ4ANzbzr86Ffx{&{*l;-cRB#41jB)oVS+SUe^20Cbtue4m?XL zzShN}VK0!xd6d`(2lcUcLA65s^fq7b&t3(YFzqqKNTe^CNT&upot@=~A(Umu_3SnY z>QDF~!;&@_ztbWpjNihxv#{8#y}ZuC41#>pRf^scTKc!|NusbaAr`6V&$b;+UP}+h zJX~CXq}c?KJKhf^FfpbJT+BM9%wfnXckGP_j4^h&>3+qdkuM4z(pqyl_)}^Uub7!R zdINL1J?;chA=+QLz6p3kPJF zmVFGYjefa+0dfYg53A;8`dE>@e{@{LkehoLEQU1PDYE6OV%B`haXq?y|AB~mS()i_ zw%B4Qy8@zumK-=S@;>~UE{|@?j4C=+OrjngJi;^@JH2hKyzmL0)jLR>E`O^P-h4Hh zf;fG~bz6f}U9)x1iD}O10YN^u3E@-GgU531yEM!`x=%X{ssH-2)>q|(^2uaRhWxkt zZT;q|Yj__mjVLuho za^aKuE!$t8%XXJsKalCyYO8d3xIQ4Md~%-q$S!_K9z7F=!TBaxuT^X%l)AhxGlSYk)^T%jc?-1j^q zEaOH6 ztShgpPJ7W>WWFbUemNKq2kB;&mm^be-%lI8Z)MZjrIl4U{oT)WqG&bysotid`nLz0 zb9`T0d{#@g=s6KlJ@$3$Y+Ynqk&9TC@nMU&YMW}B)-J0Vk2}$70xbIacfFsQW2&>Q z;*tVDz3_CeTl)XU)Op8K-T!g_AcX8fGLk}8Wba5+_Rda(Y?VEZC@F-bl#IyUgzS}! zva)x`cI-XQx!+&c?|$6(dHf!~Kd$TQx{Px^-}mSBe!ZSA_qbvkAuZjlEulXdT5Kg1 zxl!zi{f4~yOyc)(HQO6BxY36XOMKc79>tsR4=$%m!dVIgH7Qg8^do^U!l&PR#O8R* z-9GMk$jICrSQp=f*KJ-TMrp`#L5f0aBkb33M57abxAo8qe8CJ5ThawO`WwALOwaYPk~2)jKj$_a7G+2 zulk)#Nk*gq1B1XQSfzS;Qb~Fg!S4pnrFz8vi9iA6sT}N}Ag&OAB``)AApF+g zowz@rs?8nw0SwFw0u~7U1OgTi_>5ZzG5jRvh(ySDS4=Ap5$+B!9mj#Y16j3Kp*?U~ zh@e39LKgL)5z_GeIA#aW9>lyU5N>3CzH=k-C05Lt1*#Lx zX51*%U@;;_$9oUO%DOI61D3`;z)rZXnqg+&!f|=Y?kb%50-*Y7nFGZQ;kkg+0I_kc zn^6$Iy8e)~*KWN^1tiFGd|_}m?^HfH%qafDgjs{BE;uD61%X~=zP3wF`+v!}JJT~? zBQ2~V$>>UVZ#?(uvzMR}5v5+CxHni&z#e4s_v_!H*8k2b_|X$ zVCd-Sml6=I1?+TB2L_GZ7yjhBX)RlGyu(jQ+J=W>^?OPPGbi#KH=5~R2#Xb7@ENT! zMG+HA{ETo8^*=TOtQ5)6kmT~P8>OqO23@cyBVVi>j>>zN0{r_^qlI_ivBe<`d)Z{(O6EpM86%Y2ZAQ*z2jx-aA__jdB&Wwef1O zk^ltP;HRj4l8#!fdV+86CqmSdzFPl+3~Z)#^W9$}qX4J$^2mq4-rh^78?J?TI_WF* zaOfvlDz6$%e5gH|x$2AayN*hFV&kkDOBLLuDaLTd8u^haHz&%;zpa083&tpigT6Zq zPdQKlV1vuQ#`;V@YWvCLhdz9VV|02{wDxI~E$#Hn5bQITou>+qYFNrTtu7v0kJsAg z4jy)fYWj#LkIa{r45RB)v_DUHUzkZ%;)*{zFTi!WKHd*D9RYvvWmZ17qt>=|4P#A% zJL#DfTQyh)oMi2uo|>z>CtIZ_rQCj z;=Ly~^8}8MP6#Vq(B>zTt(e^m&JIOG3&LKOH}lfcr?8U=kvuak>i%K&hi|pa4*vP8 zRelPhwa>R~Im+AHz*x4RHP0YJY*0{lMcg%AehRMN{7oeUnOrSgcqr)(-lmn>=BYk3 zxli6LsxNDACzfx=2g0~cRHEE8(5Re3K>O9!E}Ykq2XkJ zsR0P8`e#qB;g``m7>e2t?-+n6){H_0qH+v{NZahj>FxgHoAdY{VxpT3;Z6SGnoWu_ zdFRLP-+A91{VRFcm2|PvErQh`WH=!~y`*i+!PM8+=^T>B?j0pRfBh6p%FpZ9wu+k} zGO!N3>_OAh?>@1f1dN`8X(5bJgcR&jjgW|fQOdSVRz&0+G<|`8_7;qZ)bMaHFc=6X z2U9q(3@*V^2n@_I7})Lt8-yK7bN}xW$N*Lx^MiJvnSj%T%`}Olp{a=w(h4^KweA9~ z=asaNWj)tW@Km}5p)n*b3C|;8Um*I!ggHBOYjo>i*jxwxBWlnlOOH&l$Rh(%+tf(N zpmOAY@D#uTUo$bus!c$!_=_D#{sGBn&drT5uMQTnBg>(LX0`S}z~I9k*cE6tm5C}+ zAfy1yNT@%p;F`NAJ*p#&9hunq$%^x&Kp*rFqHb7N5CTer z6mbsft!x%|=1nc-Cz@$8Gt38WFDZk=LKHMl4zit94RE%Ca7>kiU znR7oI&Sy*)zB{oB5&Y8W_36i`I&2^6C4G!7WNNKs~mJofoYcQu7X*<8hu6PMM`3ojw~qoF=r1zuyA&mJ^9 zFW`Q1>Xi8OhuY;=(6U3H*W%M9N%cDk8HC9r^7 z3QdR}>D$|*vci|7AW_0eFjF&5J4Y`lN3h1+U&2X+zZBa*NtdH5HBKwv>m26eI=?#> zJ@*LK{;}9}Jdm7U|0J&NKW|URa>GL3aHdlW=I)`^2-pY)gG-$vQPZPaG-%GFUX1l~ zT{T$*3I&SHr4xmOww6Ow>8^5lhGUk%&Ag2L?T&j+bR|!8LKQI`jW@gYCsrlkYxtWu z?V?MR9fJxXS;^_*>w3in8m8OG$H<18~DFNgiZk=n87(7QZsVIgv{flDuY zlKk!k7BxK17-jkkK81n=ybu<8qFPYMthK~jE%^O57wP}WT9I1#a2;C3KBic8^E=-m=XJa&SIK_R>A#VX zq4B#h(R|2U!hNw<^||b3Y<@=&@vXhF)ed}=E;pfm9+TFRqj~Saw%9Gqt)DSd68+rC%baA! zCSZYCtp5@9x2#qsdRg%pnqg}6ULzRD^-w6d0H1xs5>KS8d}vM# z;{4jU4=2w^u2DIlVkq40^*XEa7q&~J_K$`tM#{+S`~qhDEOw>mK&5EmMvEd-wl6Sdx@ytr&(ix_o64{j2Xq>sSNScjzYR3XRRV3KyUf`f z;&O-u7aknN#>OCYAmM#*$OF@p0FmE8+ulq*G2dB#1r8$&amrfVJ(d`F{@lEAW6WO; zG4%t=7TJ8ywx`dTLZg>TY}x{(^XmZy8uk8tFznhdLD78LEF8zTRaF&%^9RF!opmu4 z!nKJyn{dpV`oGvP`5oZrLu3vN_eQzL_Tq4f?xyxT*pdgWXn$7>lqLei3gy1caN-^W zEH*Q&9s!D?NWjUS#R_J9@Bq-SOw>9H%>oq>S_lwde*20fWX@k8HhGXw=uL|)N{rS4 zxpXtV2UOM?8jh2NtTY&g0OKB(oP5d3+FE}HNoJB;60b2q43( zt*=LMX-(75=luBW;_)?x!F5sV{NwEsbK_?jvin*882+@KcZ8%`+6l!D`626S*qN38 z*9;P}Ty!r76@d8B|FvS4`+vueQ1ckP$#9@Q`)=A9U1?k9K2&7cwEz+)C8hFbSsAj% zmoEo%w>PvWD_1$b*l3_p0ez%L*j*d@cq#G0bDZkGZ+L!BANwSSEtEUhw$~dPuiAHy zIsDZ*Fo}pPLpz%~)5}9c3^EmJF(a`5RQ+`J4V>%!lAAxnEd@QClq;tN1^xF9Cy4b+ zi;LP^HmopoBLzm!L>)(gxgdOssA4$G$5R}FL|Vzbtp1*a6cmq{B!$nxv{f{u#6b({ zzsuf!CeLB1Q?-uco8DeAXiky&J2z+YFkF@6Z*E!gdMjrm_aU$?MkIqJrJW%Bn<3}n}&b|0D+VGsAkFq8yvq#w`S*%}m3J#b8H zE3L`EPp|o|wKURos`3H$HAJfO4?OTu9-R*_?N9YI-H{}B48Mx&qom+|p_UsL!I+P} zj{m9ODaOx6bX>tT;kdOVpZT`D`dy2MK3(?lf=?|BAdFcnrB-l(E{B1{MkBrLaL!Bh z$jy1{6OC4Ob2E4@))!wkc9$v#jH$|PILy4>PYi#~v9~Zi)zF`1ZjkV>1i#+YS6~pkZ=hvJlQryZAeI zI}+5B-Q}(CbI9~cW((TO!UV)w4U;hoAiX_u|b#>;Yv>2g`fIq9#UlD;J| z&6Qr(|Jb;N-|28MW! zPhE{Zs@~Al#Ed^VOL`t3-((rAL#+PdryNIQy)azY#4K7O9rp%mry`?uII>h!e>cCq zqI@E1AC7zTWVLk$cS@pVX?YBO5_oQ$Uu4BG=w@TGre{(cD%?`I%|CR0ZoN_KZ|t&m zAL8c;vh;1g)COzi2>lM7z={Y@J?yzmt zh6@0~pGE4iRMQqC)=x|fCAyFrgx|hA9>gZIGSaf3K$^(8YMlr?O=mnjMmB##IuDw( z1K_p*VgQ}B5a&E%zVK%0Ody69CIs}UNL~+uHyF{8ZXggZjN_VS6CMgMfQ@`|-qvRf z!BF(cz;pxQaYVxcyH&|~`crstGS?$`EYQ}0soPf#Q6+>h0dUu{^JHhv{D3Sfe8qhZ z^i6;lLKctEC1@w)4!ck!lcm2DYMPInZoMWzhyVRaI4uCs}V=rzD9xA7!jIkkV#(_>%O zud2cU;K2%s`~F{vIfB18G(J=L^Y*DUlT|Fk;_Lk=7TcxSE{D!W_>uhgvOD8o&;}VW zcqJQM5X}*>?r<=7YzM0nR0fipRJoDBIu(AUJ!XKV!jMWINDN zJbcU@^O)CMmdWqH0Uz~QChpbq=hEO&*0EecA6ZgK%fx5p8diPh=}1JMLbd$n1;Pdi zK7py@qJCc3OLd!rJ3glrZaUIG@}y2tdhuYfcabySdnV=$bR$H^7{d0wA%vQ(WEdxh z6+E4nN1^uyCWkLNoa7*1Nm9>FHJNtj#yQ7~Wc1xtjc0rYdOOdiZgXqq4m$ao;U99` zVXf_Hj zW}@})sT`(F_-w)~fR*!}%JzsLK=E?$@MEe%sZ~-O?XH2szcImYrp{p2xZ2TY9FFWI zJLUNMV@DDKZeQE_#AvwWoob0;EmP;GU?@uBamOw&q5h6HDZ>RSvenMx@!)gw-Cz@~ z?N{L#V#3DOTdy{>Egw#Fy)d0zSu&Xs1<~gDw%va2use#wEyvNW>z3}Q7v%O6KKyPZ8y&Bs=MTXrqah(Y16q5*Ci$u+O9r;H7MUtQIc6(#%_uz)^)Z+f%2IED;v5wCA1nt2sPnBHM1 zyi9t%9J5(>Khxcy?_f607(x@WC&Jj##s`@Mw2fR3T+A{uJS%K_8eYWPmq3&Kt(jU= zQ|6?M*^)^^z{$7l?XFgQ{h|3VzO0PVrsMa|h8Hhg=DYmffnf2=92;Pc-5}oE%Sh7a zS>JP&cpaL!(%92O0`F0|s^e13Qn?IR<3j0}e@1L>`)zf1G0G+O^?!R5HCS!@?25{p zsqPeQlj=`Fg_gBI`DCi8u|wTZS3e*2Go3PlMdk=tY+OHn?CF(Htc*3mlqLh1xJi*& z3HRkMYUZx4gh$8O{^8iGVq(N#2T!gVJ(H_XS-ih+`6W24s@8u}BSBU23qN}g{E*b) z*A%dpfvX*6%Jd?H)C>bA=3FgRdmNE|OUu(Bt3@;6nc=H`e~wMkOe7xuY< z`wUlt!C~9x0MP@t?@VOy9EkMs7VMc1! zz)Vj?Mb%Z;8nXDZ{(c>7D5+UlojrV^7?J^aTmLo=%ExO2qkhetBxUro!MVZ12zgc} zcpN}%;#7;3NzQBd0k(j)1H`nS^ya>JL6Zxk*THuWg#q#~$X`)aR@{aM60!vaNK*KG zB?d$qb};J$YY{B6&{v8W01v9F>=L9PHV!Wo%+rv;hlOWt5eV@lP!J%OD5a$X@T=!1 zDO?bh<~Vb&fCs1h7j;wu#cke<8xC(+h7*4uYv?8lt3Tda*ZY$4f&E|APK{xaQEpdU z4ws0brk(FbVuzLdiy_jWCeoYdaAL+No}2>w0wX*g6rM(Z3^nRB)x{5Z6xZ=69@nqn zm}l@I4Q}!dht*&Uy36L0K>Xhse#vg#uJ-i_!i<`F=w8uRcY2cAbtQw7)4R(K2~$u9 zQ#hJR)Oc-T+R!kWKA3FaFBx`T9mlhs7$@^skuLu`JKs$_KRT2?>}X>0E>@H;`KRCR zLeJga8ox@^vuAmtZWI1KWg8RJ)CsZD<))6wINr^cD{Q~pQzkjHNNNFw!zx_Q?#_LR`|gb8CI%;*ZI<5amoG8F7njPc>N>tZ5eZ)8Md^^Wq zTe##U`!FSGp6dih(0TA@=#@&HaU%CHpJOZwWuiLOd4nD_T|(3-e7xWcz9lkqg-E(8 z%6Mo+vVt6v=8U_{GNNaQWnTxi-p?%DF%+C`T->($v^K6`ZUC8fxMWYVt{6_c%l;QX zcMLaS3qng>1BaW=5WNh4&QHHGyHz-F6M3H|c+_V6JouSpny63poRd=<@5XYON0*J! z<#+0OtcyNR4MP77DvKVUd#J=E{q>^&-{m&?Xl$+QX`lk(Yx%vB?;XeGi^ELM#bm`G zUCZOG_9`H*Gy2ZbFO@SPa_?TV^W@8$Z&`gw3VZIme6Zy8`yjF2eV4Ucj7ltGJl`P^ zD~VT(v##Ay{!pW$urE~3{Zy&4WPmbFwmR4U*_!>57N!P$R)~%1rEE1mimUHE;sp6- zmtUAEWSCPjJW;%{@f04pI;e!tVP)3sEs|b|S-IOw&iYv6%y*L|1Co7qm4E)E?-ob* zbpFcLmhnpfD^phHDX>`7*9Y~u%xf|N7xdoDp%a*g13b0B7lvQxG-j)))Q=6EtHo~P zJ5Bg+oOjxq*ZiF@_N9NM|L5Mi#qB#J^@xrJYTgG(F~R?$qQG9P!)3MiH)v&VGw~2k zn1lQ?L=

5)TU%d=xTs`r{5|*K%j8SK&-{7=CBlW3H{eeY`rA`~wWovEqU-ikzzJ zTwTDrNwRk&wEsv?G;6{hYa|+*qvKD9E4`qUV=}^=up7c}mN|`4K|*kJG#$O00O(9e z`fI??IpoGDgk* zgt9s?qsbc?89jss5NSz|y#2!m$Y(YAy%Jpc1q3vO3$1|B1=h1vvc4Y>yhYeC-QC^d zeh0!(9eKYnq_ffIXipF+4um}>B7wsqFzuGytfGB)69N1D01--av4Q{qkkSK&wB{(}|P~0Mie0 zo*azOTaOUJ;a~Jcw_iFRfxbdv{oLG3sF1SUoNX^KO6v>K9d3-{nBVyIlce_^ItJM1@^cP)8=Vvd; zsHv7HHcf8(T2Ci9Wn*wPo`Tsw0pN0MSAwg%L9bO=P%xU7mUJWH$B%USL)C=QJTrCd zd=F>R?4=MPdOH1HySP{ebFQ-@IIrco^t_`o?W;GResc~V**laQ`)Nn``nBDMOi_IN z1Z>X7kEtc&u#?xn_O$ZhLn77K0B`*YJkj zck}yd;!T*Ts4_G4FvSWLeMjH@sq@CC?lgryaVr_-{~qd*yJ#Y;9)sI)n8ByNrYF*e z>-g$rq+|9VqYraijdnSBCn%6-gz)^8y_IWo7J|mh{U81<6>OKK--<|Q{mA`$`y4Gd zj7Nm!`q;UtnNuH+7JPmL?V0cqp^D88*HM_aB z-4E$azcfG(Z(AnqczRRgAb1LE!+{lSW%wg@@1s%i&%vOg>VBN__oaB_>)w|(anH#I ztv?^ua@SkZ)%0J$#kfyd-|ckQPP^neq;vio%JmtK-hjL6sc*QNReCT6`ybowGV0Uw zo3=Hy$7z^HG)GVdB_=+B8@wJidO(Hv24x;=z6ZjOKZdq4t_iG7@miqFRdllp3w#`4_!9}0R++QN)+`qHqy^_yH~M=5@sABxr3MVMLvSw zeDq~7(a6p$7VCE?nUY~kUbAm4I@?KE=4B!-A#uMy2MhWFOPpwXu&y`m`^G@68Eo(P zYFUwc=K7sfZMglaXbyCB1pqE9iRt`F2= zL9+WCV01jcyMY{lz1uK!oo<5;Z(noelfMv_ba7Jr-AnrIwB~6*!VVSNOamlH2{^@L zvXH?-Dpi0qZ7uZZq>6M1h=DZkL_YPKssN_J)L2J{7NV!?-O8+!Fw$n^=a-Fi4h+zT z((yGQ1V3m1--Dh>aCQM#{cZbphMJlhLX`(2GeLabab7bBs*CNAdk01X5khb!3c{55 z!^z-zh$(|3{NC-<&+j8E;+aM?K5t7l+< zWY-G@%~ubpSMJfQ1XIxy7qV+n{+Z~*Q)w70;B8=zVrio=OnfaF-0dLmnu2## zPfc8}HzZ!dZBMeA2z@8Bq2e3k(Pz__mbb!hXE`XO8AZ^9Q@asuTIL7+(fUo7)NxYGBFZmVP~=;_feE^WhT+U~TuG@T^^Xdt7pU5j()j8s1sksIW{$3tDF z`3tK_8YUac8!3fPh;_anO5L2vaxgWQvk|Z0?$XUCM(K_^dg0w%y$}B6%Imf2nkyH& zkMM^mm!!IS4k}x1mE2qjjvAzM+(eoh=89Jd9QoES{CyU9;T)fb?``|l-NrB?+UuS4 z)9w%O?q0Xo)G?tnO`#6vm}W0K+{S&qk~f~Ezpaz8lDPPI-EM4szus|cSa}X}H^B>i z_HkZ}!hp-_y$?q!6x?Ks0uOHqA2c7-ci(zeK}zn#N%!}kSY}3kl~{=W@^POAXZ@sKIoFi4an)kIpJtc2J@x(T zl`U4X;ODz^i^=L&jT)a@nnS23rg7Mb!dTgA)`KmrFGbp+DaxB`xMMVZ)JH=C zl#82Q68712O?Gy}3O44h#JjeBKR2-HYrcySnPSBWb{p-&A17QMp`XaYz#9Ix2?5a= zKu-`v#>YG6BDZA(01bva>S!&4REVL&85MS3hzNWK-yhGA0+P+r1HbI&Po>Ko98(}c zKq1Bt$fZJCOk3yx^4Y9-hCsI=YZ2H)1h{yCM}IC;>zoWjZfcYor3SjXr=nX=eax9z z5AVGWO)Ap3qfG=m4+ak{P0jl~a{#vi(V`Aq&w^eXR^n1+PO3^*|-}N0Cbb-v9q)(Q=ucEH51YlS?KBMP# zb#)Zw2zX5<9>7r$p>+YKXr*PIsfmezO$XB%7U|gOoXqC`GPLwaSLN_oQo;nU+4ErB z4vX*tTWlflxsfwoHSKV~9%fl&jzuJg$;CzP2MX{QOU|=^L~TY^Rs&oDa@~N^17Zx) zo8eGH%$}U^5IEdGHdKa!@%!R*1b)48-PeeDh)duU{p zwH;BIjJNGhgCZC^$1s(lhUW@)lWo?`%zfkI-wQV;$9W*N|XK(*_ zoI_z+TC35-V8%;z4_xR1FeRjc8@Cw|vrvC5+T6XqE=xN6@d>G(UIWQ9z%9i#rLb}q zSKlcbwx7W4iyi6d#n7?(y3kCZ6L(ttcZ5aG8;=dNs|}UL%l!?#_LgwQxyB0`jg_XO zFX-)v6-=k2WrL7oaK~qJX~<>Q(f09n4730&WJxa7V={qoY9)u49~f=NsLucVg`Hy>uGTo|d!m3Ou)r$v<~ zW5{M(Vq|=x;$@FV3%2Aqr<-9n(m3%3?}Zr8dgH3PXl461vEb>BA%(N46FwbxztV7- zN_tQwDqK!OIA5ad+K-kVl`mQ2Xh_z2-`4X%c5}#L zZ)Q5%Hlt$qAl9$^>rhQ&0>!O;Q;|3M8m0ZIOl`N?zL-=Vr#>d3!@1)79~~Z|pZIjj zgd6RRj<5avLia>W+1LiR*VkWPwY}|~W9swU2s@vWxwd9I;<|lDSacCN33F$XE*kt! zU>;t?n6h||ii(u1h;Q)Z?qKcHLPL)&Q`K#LJn6(JIIUE-<}x$S0KbN%CvBT{YjfA* z=h6OTtehMZ*kx`VZOSDQqAqiO09qX-U?le4cX6$pcYy9QZYd~PL5W>iRVrDK{q}Ep zNCD9g*z7a-7&mQ zO16Xt)RCE+*&;Vmr!GE(58w*GM?OMfeCc~~A_Dbz$rr%teii$=LJ~eBz?X+496&x1 zHY89V6)aOQ<-oQMgpK$#96mtIQi6387PlYh~jGL>jy)hbrj9EDlDPEYI6m=%C1LyuDS*WWW!MLg7CheGZdNQU?_VL zM9|C!#}&0qOY0p^^7m%>FI%kwRr`yZk7=p5g}1j)ASVbK0Qz7YYC&H_+CTtD!sp)% z^g1~t#!O7XR0j+|LPrr15u|96C9@Igh))#1=EN$BS0<~}k#H6`G8 z@KjvVeY_-9kr|-|BAtbwJ~_;plAJlC2wDWaoPP_H(INTg2jJH3T39isHHU8u+zyja zTQfO+wXf)S@C9B!~faf#0iDA>N>X1jE5>39-PyPBoy4C_=IW zDHaD|wO2-0k7O=;duNC4Dw3=rv;icWR*+ij5Y%Dl8oD7b|Ewf0 zDG3E$gG*_P5(kn}7iE9rxp~vR<%x^b?t89eX>1^tH2%|&zuv`J`Nx*)^AXD&`kV3T z>dp%OL3o7ZMDJ|FGI~GVr<2m9wHhq{yEH2sC$Hx7J>) zoVPF~M|8PbnYqDu$YR3*9s2iAWl95A%~D6r;HhVG*)``nX)|-&LN6?qC{1F-OR?XR zJm(pb2Tt!k{1bQTVfoFa)#4*T;@lRXUwtp&W0pHwx!jvtN*2Hq&-{0G`c(c!gZhXY z#w^lS?2mn^`W-$8C4KQ@8PfRj))Kqg{9tzY9*j{(Qsq3iOk0GUus5Vjo)(n&?s}gu z;y*WnS(@kd^;=(9r4VD~tn_2O9IetV_p{T;Dd&+wZhZ4nzID!<`aihOXFO_^S4V{E z2=meV!UyD=)V)$bK=_g2H(#=Ghv8G0`_jmP>H(xxl9G~GPy#mUTxPzmU9=qbahMSy zO#=_yYSDuxs@ti#Xx@tQtGTse!t)jmlOogdlr^i5-Sh@A#gBT_AOMRC43#^6grBCi zwA*rw?U4&_D(<8xcuz}n-4$b$sa9JhErMOe$&jHU?_Swk%r?~68g}Sh_4i=3-mF%B zarX+4x;pEBsq{IZuU$LmzT&ICHolXHy}BM%CiwWP96tcqg=~VQixN%vn^Q3c z-3sYQI$W)dxD`srcKPf6Zib$o9;~VaWRKQvB1}C9HO0pwohTv0m!KO_GcPk!-pnlF z_9Nskg;ekZi71X_nDw(~#BhY-C7@xRMe39gLNIp>=H3}-m;vI!U#M}v2`my`CRXHr zYxQvTI2zbNZ19Pac{4n0v^-JE$_kwo{AN|BgX*!{Uhn}z&Lf0)4td;f2NhQ!!tkTH z8L6|>iIqBk)auCI> z*l$LbaR*|ez4^nG!(wjBp7vzou8b~vQ6{Y{J1hhv!b%! zwW*RW_|1DlEu0x=!uu<;=~=Bw;rnyykHh@@sfg6hDBbu<|E{AHZ3E9kG~>SGOPeJ% zd?K%D%<3%L;a!rI&wPsO7(3y9VR`Nkbz=kP36yr5)+8)k)}I2#Q{Hs{_|7Q7rKUth z$D=>>;=7igN_?dXsU|J)9S*z*D@fZ`upPNI!gdYRu69riQC{%tm-B=pt=FPF(+Bjgmo$F)2 z{NtF~c+^t(la|ToeZH43h9+jy6P=9F23%HPo3+So+wPDm94xywVx)e%1(^NVKw01e zv>hb4Y%f|cNcHcmV=u3a>rMYjJ)}M6GZCkHMk#|{COPx8so@1d;X@Tk6yPWBT3Pqs z|9hj;x{QQ;yDo~o6IxqnmdE14e|=EVGRu3`YNezZ$-|*~`Ug+G;urW`qi)F8AGfL! zQ*1piezIiNW5mAkWzNh?f<6a%ma7B${Wajj8*Hm~C5FJF0v8a7c)0xcA4c%1&k9;gF*x;_h{vbI4*@*&Gj6v2{9564Tu=H)PqIqMJlbziN zflGi)-9k_=FrZvYMW9HFk_Z$j9!RL!_P?f0UWMG2j){p8hc;0_D6y8;t-$?I5b!8a zr@mY0ta2&pKBPUN(_0DY#0|s;isHR8CuZ2FyoA)f^bQ6_rEOhk_e8oiV8+3``SKn} zM&M$H>l5+Eu+SHpom#rOfKECE^VY~Wgt-l;8?cM0*$j85R#zjCmMu7%mDSY3OKJ)W zsgceV2!b`N`@k96yL2_?F497VL{1qmT*%#q8A}ls4ILylnu>Z41#jwiwy*&)8n+kGg^N33t!z=q4-J|JZKh7|rRIcXnzFB<@Y& z`bu+VTl-RJ1iP>9=eUyW?l|6-+yVA<^@t-P0=uWyh3wn+O1y>avY+10`~xYK811bm zt~X-i@W14da#_V!iqik@h>DVt4mwdp;e=}YCE~bfuH{-=o1UJs?LkR-#p*hVYMM`w zBd8PNM~!k%BbwN%s+kFA-o7P@di&HoH8p&&LS)RyeMuQzKY(HI1{E=50ma z-97J)F$!o3-F+}ov|I~y#4d7JtrEptYl($+VrwoR{_ z|2*o(Oxm|{W--=uea8_^zA8$})n7w8_wx80pI0V6$=)$F=pTHX&iQmQ{b_92E)k1_ zcw2$>w9x#8$?X=$Pm7M6-%@r2@9UbMVc4F(-#0KhxIkarK4z#$4mNGh$yxkTr0Iw4j$-|HM7P;tSaoV^FwzEEBcSCk1i#-e!uM* zOa&VGK;DB~B7>L{6NuTlZeZkSX>Y$}JD)NUO*=FVkM{B>b3koAe;fKefw+%=fyMAc2>|wpbRsS(Df>VL{_dsZaZWC- z)6iAc{}`rtPYZ8qr04*cgs>TG0Ldlt(?7}{ zxR_~NBgQ&SPYG%QVv{VvmxK>d~=7&`X$zD>AY2hR&K-#{g7I7sOC zcz5#&9@*IeM@uRaYUm`M%-+bik0%NrVibQ-PhT@RI?}rwLX9iU?!T zO%JX`Q_*)*q0fkl^9=HF)e_@~xr@-?>fOTNmj&tYZy}i>!{!iEO3E2n($dnxldJ9T zELL}%OL=RlUqHq|EY7Br+nj61ywZQ+{v$7)#o5cKgG$N!As=W!{LySPGII`*)QBH{ z&RfD_XeYD5ct&9T&ehk9@#jS(nYFW?M&1daG9Z3ld6jA?+kO_go+7UjPOo34 z@QZ<0OG03vcW=LFePhZD$)ne=7qyJDi{G=euU>o118)cbdw=`pL*H?hPV^gkkm9=4 zZPweD>++vSn^haZt&qKmys!PUc$D+;;hoEMXx#4Q+o^Bq%6_$GVg~O6TF45(JQ7uF z4YJGQ@r0<`65jN{cTmCElUh;43)$W_9X{(E`H?A+R4|Fk^`zn>mHq&T~hzUO{UA-K%x+1un1Jw6X_1;Y<2Ikn9l z%w1WBrl$8MqM|YgB@FJKnpqRXyL5&6EsquI3<^@e*cMSpIEst6-M77thZ44{9`oO^ zU3wZ=6s>%yCj0rg4#ttopkkNOavyHdvv4l9UBLS`@#9xrHQwh4&>}8LRhFua^ulio zS0@O>P)9NX-{TS&!L$MM2&bMMlz{K|wN;Nv1;I28%15r1n{%OrGxPJZ?uR0xj5)QH z-^IU4ekd+3$k(@bNSl+nb;}Xm`Qd{BCs&+m!yQ`I-xMf*{@41emAU&PQ(FrPu`CSb zl9o!f?jvF-ka|hm{0=gP-Ehd$Z5I z#TOfVy1|K$D7>IsVAN&v`s|SI*3WWi7BG$>3NO+Za67f?#v6Da+~!5f(m|9%7=CzPU!mZY}HU(%& zeemFc0CcKgC*b|2l@E$@S8uNspM8lDbq^J^XTv83`Q!r*y4cU^`g!D^_W!SAX=L@z z?Nn`qN)8J44XA?xB6N}r@Jrzv35kd(lR#PxdgB<*5Fwvw=qV0pdf?-i2OTvD{s)f~ zm6e~E$Zm51(Hzjo3UCcmi&OYgC)6n&1Qjsrt~dtBjDQ#aG1A%yVC0hiI8_d4A7BP7 z+{K6C)!7;I4D=Vv_yu1w}=S2LA`k^{FPflIAB3Oe6-`L&yY? zH45|M z#@yTAZ?pv7GYA53nq`a4C*wl zOvm0rz8ZKf3QK4IZ(KO_hN*lc%l{<>$zeESpMhqAqzEi*TBG+h7_)mky(J&acOBB- zTp5me&RJWK52a(-Q5KKIgS@}Qn+X;t=kj7s#6 z`48dUc}8;EoIb-}d1l;|$5s^DXPD@1`94_dH##_tEvcJ!SHsKPw5sOIpLp3z9}vig5T{a zH+Te1Jx2%xmlhkK@ZNzVl9!j)Ax+Hf9kOnR@}?%0Yt^S%Sy|)U;UiEW>y=tMrNeMp%S(7oy){EF#V1-3&_*@XWEIBTYnNqDrv zjSOVTn`Q}^k|tP$AT5@NN(-7P%nLt|#1gEY$QJqNAgeh6<{8Bfo}0*H-wdXAsIKJC z0N(l_gkcvK7ZsuMJ-&WqISuY*cI~U64|`Ipw(w}R*LTA84N#g{EhdO|2>kmo`^bpe@1V{(EHjyvwxzxT8}|a z_C{BYRqI7xtk-oEP}sPcq~5qpbzcL0{13 z%KX#Z3;*EvR)$RSNs77Ce=j!n=Wbq%X09uB^Z=iDI@wy|MEnmPbOLF)LjUCXyydn! zn3(x?Ray$LEC0SI-VSrFd>F}Nb()nnK+0y7TwPTzJ_5GKqrRV6%E#iLyvVp}SMZ)# z?M%@Ed(2&dqXhwzQU435$S0|%_Qklnt_{1dza@^}OJufQs(d>ToH#H8HpLnz`&LXmx^0l-l81IpUd_U3wbNnV5P_ z?h!2Ri)M>}hZ6&&v^;!ORe674M)VkjGxnP`)YNF*SE_K{0!}8VqodE59dgRug4y4# zTI^DlbOc16-(r_qs3)z0si{k&9XUC#IlIbH9!#@>#)HKKmVAyoBs$pja4toAeqR%f zU}-KY|-ZMpxdjc9y*dmBSHB zfjskN5TrTyuoaNS;R6( z;Al<_IM2U8EALH^?m4-U*v_INh@C)WEG;pCwBPI3q8Qj6fqpUuf4_=tE{H*wNPe^ackL zA!Vrmpnk6Kh3Xm;wOa;Yrjgbd=-uqE0FVpfg-6bp6OlkiPwxtskB^v~!GFaFGM}J% z6ovxAfw$IHRu{Rs@BYwL#9V`m6CfW8-b2loqUuf5Ukq3CSCjhl3pHkos04lb*y-%^5JgI%)?*FNxG!;dIydx za}#OqH*H>3b|}S~jKp1a|Eiz)JHIgxw;EV7D)xuWl5Pb3TZTWV<8*YPW4^rGIYuVz zGWNNZcKInSokNikId3Jj=T@~`)cT}Ge!Z>dRzP_WtJ$+AU);X<)DT|8{FwKn^i5YM z`%fj(v-|R(TJ$;KT(40C{+SuKPtB_jNW?`B&ejg3yvhFKbUk5nDPM_&93&}|<3+XrG7RFpb z3Q)W^cuZ14OG~!?>zYKw;1rZ86uDr13rm=&CtqQY6MY}7ye1_Ft~m;np3X*anjZbC zm4>e?4ctQJH(C3;lA3w)nV#=Io7>-*xyd3Y0MDLRce!H_Wgt)zYC@n`;nlkk@9UYm zuMv|6GA1A1mWMESoxRFA4F8vKB+;$)TyIr-}A%%Ptp`Og)Kv+~1Y5s#^scTVDQQc|lG{`%3 zECw*^Nf0nrgu|k@5*mw7#sX+Tf`w+BOhgh`R^W80+8y^nI@CYM$76PuPOdA6pYH5g ztDg6Xi4jPw5YQrL+0MhIW3J#HFVg-8rM*tOB0#l+dYo2dxBx|TVju67q$VgKfJ~2o z!tXdy=mkcCe-I9Tq1*OIWF%Uby9wx|0-$EbmBJ6m0=C2gAHR!JAXAbV9EETK+`N4| zB&h}j5AfGK9~#}?#G}!g`v`$Y*o+do!5?S@2e%^l@p?sIWkdqw;rOL1|4nD-FR{Pa zY=Z>$n#tvDaY}O`X!H&T`%ToJt{_lCjz0g3?HVBAA@cg0%a-6NIe6Z}i6YW1+Kt9r z^5ZFnzq0}9LGiHh(8yu<_l(3Z{_ou7mx>Lz%Y_T7)}tAA4H|}azW&9GoUa+R(q{Iu z6Ebw|f4oXoLjIcE!%WwdED}lT{%9m639saYPy_A%Kv6OrnOozi*E7G+P#0|Ie&{gv z>5Yi}8^d4Vm;**#^C0GM=7Z4RsmIRLfmKzKVT{`Z6ih8ng39{(NQ_`R@=1o%`WJ3DNTLYN&Xv$k+i+2{YX?cPHa_zK>O+G3Q3XWcbaD z?C)Y;l_qRWQ)p(&k5zseWcs6e(~Wzk-@i>$2-}&49}ArY*$e7Fr2{&6(! z@^9;rrcVG9C{l(< zB1#B}k`RrCC}ZZRkTOK0F~pIOOl3?+%Fu8OkvYXFLxUkx|FuuO@Av&(*Z*AC_x(Ok zPmXgwd+&SS_qx|w_vX=88<-ckmQQz3wL4$<&9Qec*>rpcJ5D0)(z{APo&Wl@eM2UiTm#E$2Sq2iPDOi z0Ta7NcY-&{%0{vo83xhcr!gkHb8MfaG;W`knwg!A86uVtpk4LALbwVJJNDulj59I5 z7O~6z_ULpe=p8b)3{vW^h#x_pLG_QeojrW~_*yc3 zcBE{Y9aaq@Ddb9~t3bP9&}_Ty5C3{#sCAI>Dk-hB^!~n(NIj5G6wd;a*n~OmjOZqR znhRsWw<(?eN$|;Y{W`kwDjz;O7l76VW^|6%#F+sXY0GxROj;ZUD_S8Ak%{Pc!T4y? z*trv;M1FY7FFg0u{NhE;)s>i)3K<_ElGxhXXV(g%qlB@gfvDL+BC^~gc8lHK>cCGd z>fgKdy;w>zAJ5t3Y-UDquLUw=fesvI8ne(bo74#yPb6IIPC7X`Jwt8U)7>pZ?ZPjA zkdd(ql@~1^0+Se|^f z*mH?#c+w$Hf%kHe^^-!bcpaH*?RQF|<(5pCSRbWs@Le@@?F;i0(SmT+Gpu)7-wrsw z%GVN*IrZ#|f}N94Ig|U>rH_v21$v6G`4^8Z5_nlY@yG8;B2&Lv!+ge)kt8V0l#3xG z=TuGo-Zx+P)?S!5F!4&YUaO4?Bn$>`?^pxGsJYn>P|eL*Gl9VM`^)C;zP@Abej1m5 zRlpuO5d3$J($0$@a@NHZvJpj`tmuNsNDC+D{fX(d!4DpYr=*$o(3|8wJ{6tY6`Wzp zQFDLq+Sa^5aS8^sFzr(-oiD02*)y0noouOQ3V#V><5LmWjV|=n{6yPOeRpuo%i8KM zeSH@dI)e!Vl|p&^*jDz~OYxG?onl(oegq{a3#?sxP0>w?BTb(NMqB~rU{)P%cBuM1 zAXMF}rthySUX2`!;$Ih9Jvm<5cW~~li=tw|pTVGrp0bEnVlSVXT^UhRyznN{H&|fJ zf|DHD^2T;&(ZxM4(ce7B#j@$onwn?TxwGpuZ|oL3_$!-xwEFQZVF{~`_g`G{ar@!m zqq#eKxO*32kX~IOeJxPen6`Rx_qO)1-{-?Q>uaf6&G)$|{zZA5YO`}o zj-8V*FIkNa&b`cHwbT2*l`xI6I#qnnwq>Fi9jaOKJZALAGIRZx{)X<3OfN+f*7dx| zDPAF$zvkm??9o$lNm9kUh5DcF-J9w>=APSmrMjo_y0X|lQx_1DeSc3r&?~jDSr%P5 z>n!p~r(NCC*>_dYz4sb-JN$Yl6wdyLF`m5SC3Nh`M(?R&aI<&pPiERuz4WD`Y4DX| z*XFr%mzHPG9Os|6$hGd$ou*4;t+Q!ttrKdumdWRFOlf&kYM5#@ET<)dGuHTrck*hYt?734iDoL`O zJzd+`@bJ4+9r`_DHKf$Oqw%wmInyj?*01bppkHZ#Z>2Y`e{$fr>f)8dp>v`$-NRp& zT+(3&u(hyXZ{11jZ;`jaeY`K}I!|LHss-D78xsEl%lWlfy_mwnmERVgA zk|OtUw!LpAIXtVUyFx(fi1K1}^{n+9-dlZXSYlr8`)BOg>(`uz3+nQVyWeM{+Wp=W zv>&F5+Ja9qmw#)Gyf8VrSW|oAFMm4kiPOCm)8qX+vdha?WW~hHgWjYsWTdOl6J7V# zvRQWhgffLw7Maa3IO_-w{aG2SbZ?+#q|-)XV>UZ8$7d0VJ&Ng*S62$dn_)oI5wzRG zNHw@Qc|Svc7ASixSOI=A_uj666I^&@&zA{VIvbQ=&~Trlapr{iTeo$^W6D5?Ap;L#-+U2#L!!~c zdqwCJS1`+yMCyz<{3jsGE4;aC)vCxFH!i(=x%)pXX=y-P0go|l6ej(8sv(t4O+4fd zLJl4X=AiKA&F@niJBr4;NaM~R>8@@c0)zdHS#6yZiQ54`uiGy7pl!|q6+xURiaSI`affPrr+o!l^yb<2J0o%h z(3KB%G2J2FKWRzQ0laLtg&7c71I8lo1AcV;wMyOX@0-tbJa|6%{*t0k(W5(MT;4O^ zjinCB4Ma|xKe7^nfg;@EMs}S(ElLp*Ov#8}W3!<8AvfuB**AnI?#|ZS)s(C# z>ALn}G_A62OJ$(SOx$;fIv#x=5(S(`q0nyFIgK;7|EaXRRQY8WSqNkyeUa4V?5Fa(ecq_cpoQkOW}cOZ!YoVl?Z?@_sTftWNXf=k`8`` zFLY+?taXLjwZIR@wBD6@F=FiAZNu{IUG+6A+1`pvedntj&iBi$4JdAT?KCl!Y+1fj zd#dNRU(Di0$}N*2KFa43K4wf@5tw{^{PvQGMU|iLM~iN}&*PV&UGnmE@#HgEfaQ8I zXUtwVG_XsTt+QQfX?apW<;mj4?Zs>#KIBt|OC6^Q{fey$%M%iS62^uH2WzXjGj(56 zxU8PGxfdfDm&P?LpirrAtY@d9wa@UugV3QpewRz$zV>0=ZReaKj*fI*I{5gQn(yzm zEoOdWwkxwNe17$Lc9v|i{~*U9cj#-U@#jy}$f)--3;HSDeaWb58=w7bEOelhYuQx2 zO+K&aDW6?k7P#nqPtBSy@TxE0+Cez4u6i&)2Qm*#Ug4oD;n{$)qV#r&D=TPypbx>- z>?ra<2BFBJwC!833cQ+M*4AFYI6^2!LO4aWo>ku;OW+}l>4CSfUk}7jE$8k0BQu9% z7%R_^ZVfO2zJuSF`mSA<2)oxm)X818tgOsq@cyj`%n$*eATckGmfDC<5dp(AAPWSZ zBXj^^2@*4EJUk+8uf3R4^u$41<5?<5J}IUhV;UlO(%XHetT1ztu%V7U9^R~U9tQL# z*%q)V=U{!YYTgb#y`@{vd<;e#gHX65uAJ|YhDO(>$ZJA|bPRy&6%RpaeS3=?Z;kZ+ za2L4k6FQ}GTm$h|UJRp+(O>o*u6;CLFc9t`di`uvtF+p5@M|apey87ym;89O0lsj2 zvcmYRksc+brQe^kZR`f|DCmQ?--UX7j^|)FqF=d8b~UGK3T9i1g8c}(R&&wy>s-VN z2wG@f;DwQsg4@e84c2#m`zB??Pl8EOyeI^HB(!aA1fd8~c$cmw2t@$_#Jt+>cg6W2 zs+ZG&mz%2gUKX~P$coB(A7k*wY6rN`YxzYb^Tlr;Ze?aNqWBog{c^pYoDe!6^ylu> zf$cVPUZVM(7o%T^i=jJ6O=9^_1pO8`aoPTvZ<1tUcCfKl%BTVWNsK>+q&5 zBk*sYJSI0c*MNKOI2c2*6@^`$J2e6;Quof_(`;Sy{Ea2m@wYxVEUnDQ)~aYREeUl1nlblNZCy__A9Ii8T)zPOnliT1&SQny zJB2Gh)v6yX-kR#H;by3>&#vf7--THvC*h1)`{vZutv)~?N1wArVq{3%613yShwHh7 z(|uU)EO=I0^DMwmkU3rVN@%;|e9F13o7O+KY3IyDtu#K7}jvzd2B9VlSb+#A)$AN ztK^@({IW=-ZzreimAeUItw=gWKQDcja$KXQyY+0^ywbprqjl3~Un*K&oLR-=mJrTi zq*yEK;<@<6cD2P78a{0^lYCaijyLl<2sLXi}8jVY^dY=N@Yoqc*Uvcs3ugYODl-M`9FsNab_&voj= zw6fx>WpW?+4EJlGxAW%nbpDTnA~&Pfu|-5V|Cy}|-Kh0EKQ2r!kl9Og?eUdlTzhND zZ(iGP0$y=gvYDTplL>$BEJ3e$n^(?=Lix%IgYPv))&4k>F?1J9+GqOOy1G_^!FjEG zx>(V}^Gf+9j8f)e=8$>54@&oS31VRR^=gm%(bFj<)5D8PCVLZ2hWu);?L5f4|~bk^7lil*Sl=;n8!6Pg7s3^cs4o;<~r})8ai@4;?y0<|oN*-aIs(2`(fV zAdCD^bJs4ysDKC}Q1e~BS}vjirzS0%QS0<<^pHkB4+-`FMCR)9<@k`iVfi;umM5ObTJ*>~P*9MkH&2)zAu&=ox@I$6-(ZY`XA5T52 zmXnri@;q3@uF-M%#i@$la?&!GPH^r)gw1S6~MxYyGx?3VB~%*TaYW=EWwLfy2{mRRvZbAN_Hz>e+>Oc*t|8 zs0g$V3g%^BUZl#!%?2%G_$wW0G)z#`a00-O6rJt<_HfxY>s!XAX&Tnn!g8DKQ0xGG zxR@x!ryDP@WL;X8(z5Oy7OH2AznXy)QSir3jR%r*Qdx}dEReB~T(D&PvgK{Sdh%jXvZLk~4pxO2Y3h>f|hbh)L!_~%RU z?il<=S;w&!(|UVW(b>}<`iT2wNK!QnJ$dMo@|);-PUB7&_xZUmKz~=d7XZ~`$cjn0c>2X(ek0;DPYy6VR_c%CkpXq$4X<@M%tXy1Y5wF)`vbCl`1e;d) z@O0}_Bve!{G9C!lg+kQB`L;Les{=ISGK-4%N#_6s-eGg8dXzmF6Hn${M(Z43#~A!f zKaZiDb3bmCKO-s1iH6t^ge=l!B(Ec{@q@&D-0lMk%#rdxzPAJ$clg6s>;B$bPUGJW zRfy0ze*F3c*PP9pJVveN`~dtL7T%1Dy9R6X3NyEJ=bl59&=m?BR_e*q_V!iC!ibi3 zU<}v7LS|-?VPTNah+)EJJ6qdFb}ce+#A?k0)4DE3fdl3c^6B0yBH4-Te zvj0|-!t2-XPB})tV(&KJ8NycIK>ay7`s{0VIs3^k<1vL$+S~+~ih7<&JgV00%fLV| zdf=6nl^P{hnwlcYE(6O@wjYz`fZ`7t6)xGor@9e+=~0YjSO}|x=AuKnCywG{6D_VK zEPDS;m;c#7_3FE(0EaRmBN6%%t=xq?W}=dJb*asF3}y@c>|Dfed%n>r=8fXKskP7$h`nA7lXIQVtaF62JBfDp)0P!xo&Hpwo#q0Y8undUW*Q{u zT95fsj931~kZS8e^)x+_YvV=DT`g_@EdNR>dwJZ@pS!^+J|8Il!0$YqD`VNq>xZfL z%tHU%&0P%_EXt$qgcsIzz3L)q^tr4xFJCnJ?Os4xlE8g=?H}1apENZmuKhY&RVT!k zW5XbuCvLub&Ob>wv;Gg%i+zx{^}g*2-DF*iV$Zlt24<4B4ojTeuR z3_!MlwBTG^?`*7~iTw_oLF-DU$Y7E;jDvbAt^-2xVY zKL!F64_&x$fedZ|*1t_@rD4tHn0lgDfOz!ygsQoVi#*YQgX4T9B1~`>#G}N51LTUt z&WZ}AjU}V=aQ*GlA!c`CGAIH8RtQN;>lv)UBZCNBL*`a!^wnVHHhjCY-OrfRj~Ey% zgCdLRj;^?oB#ci8xA^6&SC4OEI^$J|{i(rx>ztb9S! zY8GK~#vkjS_eW_nIn4L>x?t%y>$+gkqAROzx7A_iX=K)1l9SfiQg2zQ{!45M7Vypc zE-iN76<)fC@nP^kf@`!>{-*w%Z@CS3E3LS%FZ3x=AF2WZWuolv z=c+rnTJ+3{q@eZZ1xS}5Nf30+qZm~HN!+9SEDQu592^9hgiChiTUB;;cA_l-ohz)j z*P3Wp(4NgJei;t)D1xhDpF!PWMWY&$(ZcH(9ZB0j%&dfI_rcdZJUm1PZ(tAsmNa=4 zu{%0CVu0b;d`z%V1l84HYB?{jHCSuJK@c1<@^hTm zNsm)?JBA0XKP<&N;G!(ENgtvKq!@%D4nIOPOBC8MZF3%StZzJB8L+KE^JOI%<-2$9 z-r?YonW&0E6n}ZB+Td&}3ge^-f->RM^L%0KcAvK46~`{^M6#O(w2R#+9m*BW)foi^ zyo8GC>s!X4qJn%biw(==t-VMDK?=Zp%rfsVHc!DM+c= z&REy{WFV3FhKFB6gY(8Vf2+-SND+uVLKP=pP-o@yZ+Un` zr}TK`SJXAHx%PHRY3;nY5A*Q9m0fd9G16~lhb*I3K6*Bf&1t0OoTCFDi|=gY4)M}L z-f$!F3p*t*>=eI%zjppjqJDb$k5gPjO~C8 zp>Y3d;4)=1Ut1-JRUL9Hf7}y@ot3ROV_(Bf$==ZtcD32=lezRl3i_pC5c_NDYlx+E zpJjNAdm`!9B3_!*^z@u&oa`mwBSh356A>5XVuka>%_G{0+eKl~RTgJy1gnFa0wsz0O6cSb)PLBA5 zY9(L5NT|i9@s+co28&ho;w3Xws;y5KbVUej5Sqf|@87Q0MOngdwX|z~2!}pq6daPL zGBPu(Q}trV%}cmp*p(^K2;%KLI3QhRKI=DbENtJ7IeS1Jg8Hte*A&5g8K?EO5n#9|@iTHy?8 zHwztm&(P2{M&N_o>Z+zl1!S(F2n7{m`-puVh#*7`!TBDAluYv5V2D1|VdYw=c$Bfg zo{2!&UmlYK_D%uKTiwKj7YNNReJz?V6jzZbSEO-y1>e40Bp0GNZ{5=c`sXCP4_K$= z5PVxrtt+(**_3veKn0i_n!9&p9!J~TBw52cWKq6nF80~2^mlihHkC7d*}LU7tz~7P zDo@1jCtDd(FRY#8I=*cbvWj@QI_jU(fBxa^)`#1#CRTI?$8DqZ*v!MwP+N!1E9)R+ zya2B;q5fL}cBOUdSCmGoLpTeyr_`qTLRVj)9#UZ^r_gZU+NfG0)(0uI(##afS&+=E z@GsbLkI9TiWF9NFxk;gNBY_b(s5SMHVgPzxd>d;>F9Cf9zaCPQnRn zF7WBPF~PC92r5Er)L&@LF2nIBzpjN7w^Uu7t{sujR>}Vo!!sj-7Ys1y17km+TF2ij zRN11;dnIvb-E?#wdJ&b4v(#W%JTA+j zf?^E3X^2%q*f&_WPLwX79LHWh^CQ%XDcusjSg1Z(U>6u*{K_t^m<=%%*{d~r4o}Yi zOidhmb5G+GGE=w@p3hu>Z#t%oraWYm5LfC=E z^`6r;4A+{Nm{?+J>WkF+%F6iwdAD}C8%_vC)h+{_6q<_%s)c(l6?#R~4+!I1wyX)GbSjjV*>Vbye)@#n|H{|NYc9A-e>RGR_jUf&0QNa6Q$k1M$M)@KPJ!CMp`~}%haPT-kK-1 z#6&p1D`Na0Fqk`t%vipK#MB7KTx6|??J6+h$n!)Exg8Emzh1J4W;}R6n77`TKtw1f z7%{s|mI2?lK-+Q|E(D9|P$kR!1Gcl)ZQ8VqE2PS7YfSxm85v$#&U|L9FyuwO9qwJ# z`}e6HcMo`jH2+&$J{utxo;j1XSOOVUuvrZ*KQLzU30scdhQsvu@nZtS+jo|%gax46 zG{Tb=q9w*+`w@*JlJsQfKsRn|n}cQEN+9KkD7w!>EMiBZfq^NvteECwWye!YGT@n` zb`&;6r%s)Ebb4Z90@fGhVSm+xS0W;)Heenilj8@-r3q27-Za#Fgi^DBnK?8ftSuDJ z3GP_%6O)fAfcA>W3$#S}zJb)2)Oiw))gU|-BsLtA!^}Vjqzt?gJe;n1@W8Tplc*k0 zgiVY$H%q}IkHf>%F&O_p)09RYPGR%FM8rrWn4rqo`oT;A#Y+H5>>}SSQ{9fKU?3W* zJljD|Wd$q}Eo(HeIjO%|GG_ANzfD1R z_+-tD@I+oo!}Yslc(&~fp9p78(4M-p6J-1=*X#IUl5{V#rP5wrV=&1-G>uDL!+VpH zIVI`jc+kfRHzqC-`crcOkH1-ed?spjl3gQ&WV#Y-aHLaD5?NhW$=z8(M3xBB#61Ur zK?q4R3~y4iE3624^!oMdAZ%VF@-#_F)^kchze!124c<7fRS8-)wzjq;k+MepgpZn~ ze$N2Ius9$UZ7vE3n1^y97ztEpa=p~@n;yuyfj`6laC+co9Sz22D z&H1t1NqTf>6JZE+PI43kx&l z=jR8Eg9L_dIJwTiZtcA&hO(yy_%X6a#k0K^J7g=7`jDpki?G#f>Cr7AMvcaY4~L?M z{$F$i5e7Xd5wYYjAt0>C*U(1qfq!Gbt|&ah?|tEF*0tByk{ULZ!5Z_%R3WVc2Y4_U z!k!Ukpl~6WDydb8c0B-F(YaAI2w$3!Yk_x);+WVVF|M={hT~X!`(Z*EzzgOXF{`G= zheGl9?T)~_e5Zy6k*=4a(-Z-rfHWU3@0Ht^2flwS^(s8jvabV%@ub34 zmGz4IlSqU7pxLlK?Kn)xUK)MnY_H*rno_Qa{2D|qA-u*|xn2jF4*&fZh)g5#u>YhQ zSJZ7@f<3P<$u#HtL_v%zq}7lSx0DetdCDex*BK&I|{@ zaZ`Dm4SAonjSayeI5@%^SQt3OLiiw)SQM~_Ph9Wr#q|AYNUh1+QokxXbyLUyC24If zE%I1J#FN5|%+0UjnfAiiz^#2P7Yp7`_V}yiPl}4%&M1VT~T&BGVxL}pB1q)u5YniOUX;Rbx1SU&G|a&vnDF7)J6bPe*5c#Ce_ z*k~3nYqJQ_hK<(ve)xv*gUFeHn&jnw8ypM)Z!yUE#{O_ZRDre3TeF+B$0lq1X0%`S zp95j!W9^5{))+h@i#X7V}%&FgLt@{k1sSjc%T+I z;#EOaexxbYI9r^cq<%}<3JB>-LtJIe8o#R`REt0c$IG_S3P?{{(fD(MCB!B@eEVbCUQ-^#` zVFC+4o*H?58K>(~@hBE-{>g8-jS&!lqLEWP`|;z)OnGvkHj)ficjXez?-Bu8n>4JO-2zf=C9$Buj<9cwi;K%S^;U#XW*Gb) zf6cfZY3~WU92kSYej0ZkNq<4$++gys+c`046+$ey{l}4ys{5QiO;X&K6Abp}SKuyE z79}gtXAhYA_|aTI#UsK7UR_j_^|-k83|d-SgAsu&*mh~Eni1_=-L8Ccy#_05F-&$) zfWV{+S4)05enkDlF}ZtrdB!R7s5qgAi;wT5&jFDDcG3I0Pv7W)Y%i`D_Rk7CY(BZW z(q>Oz8bq#~ojum0AYN}-X63_)gl(vm7pU)Y&!rWtqTti6~$3Vf~ z8hQbaxzSO>hK1`Qca!;T#48dr4ktGKJ&s%3)!-ZUAEhJgI5d*qx>%c$Rr*e@iB~W_ zgU+jLoKCS2*H$ng{;-d;+iv7mYU)SrWlaXz zM%vqWPrAes0Yw7?1olwS3$BnRpUt3)vb?Bc;2Dmypn6kr8zRyX0{?@MaT{$n-y9p$@nbJ;G z6}sD=O-W?Z(`t}rOvJ+(=l)LDC98s$r?m{bO7#v+u|*np``SGzDs$YbvJ7fycuHv29VU&KNg{Rd&Uks1THuPLIG} z>jEHN8;lGAOFCdHsA?EzMG^v#0RNut#IiC$;*y6#;hsr`Lz|DcH(6Dc>2JC45)}~> zqk90TPScNj4gkjB@L{|6+fk7clRrfMxHVy#O&Cu|=1C!M4VWUrv$i}I;7h#=5Ze$m zi;K72lsUrs)Le#2^!@PtlLo3x-;Hp}#w+#ckkkyq`3y36RU>k|r6PBe#OXi05@_I% z@9i}BxA0*=1H{&oTR!?CbqFB8pj|Pr!qJ>gCNh-EcuP;lFMjjvPh(89+(Q(x16z;! zcrL{spFP9ESYc#5mor-a%0z$?JG*vEED{lseKI=9@W(0zUTxF;`Lj568$N2?iCnC= z@?~9d4TyAaN*}&l*y+iJ+xN0jdR}`ADDV*U1E=1y)xOmqGGwrAT+UhJ_ars&aC3{U zPfJU~)Uhl7mYfZ8=vez~J`6#)t?VjE#)&~@*e3OmodSrMz^7$@=Deb#qk|)^W5GQ) zlUV++7%${gNqg!zx}wED7G#-11K0qNBzxZ`2K~yM!uCRCDsE1+X3t>`Z9GE z4LY`02uxf1_6HYCfnR_rVdYCctQ zVP?^{m+2a%6=H4zf0tUsc3o`9e@waKZ{IQ!cUyqyUHdw+hCzp!(4$8} z50Mk?JD8osFvc5$4bgFluTS~NAt@nI(OK#R`tX`w+=5%9FyRvCdO0%Th}z)-|ahX-!Qc9;|s!v)f81mRU*pX3;&48d4kNMi^! z&F<7GSaE5O5;u3muv50SkJ~qji7m)3`N0b7X4fVB^1B}?u62#aHTTe>lpnKy3QrSU z6i!rU0aPKs;(P}r#*`R?6IaZC3EI%?^h6Cxak4WiR`4aoi=&c(R6Tdv#~9XHq%H)y z5gKu|h~L!Il*q>b{c_V*UDG1&5-^+MlORZ3q#{6)O(+@F<8aX+SQwg`u45czL=zMZ zR}!-jNPy#i4tS+W!`)DG2@oiue~`@_gp3b!rVqZxX25g~5plF8iC)aovhn8tO&aw@ z;=Ox4IfdC-Sua35YnL?~Ifgt05--cHtX*z#>faPKlWMazcYpn525DIDgb~TU@KW_R z{zm!xkD9dE^GfHe!C{v5kU0PMuI^l?tjrH#3=b_%+4a?M)v%^&2)cprZ_e=?vn5dr zi&!sn5WXH4$*?F~muT6}_#+@XA0_*UZisSf6Od)~Bj?UN{)gxm-6pBV(~^?dFK@gg zP@$1z)?=*6v}a_*v2dTQt>pIEU;7DT4pBBPHiA5XRsWlsJ;X^PIO7_hU!3TiQ{aWD z*-X5yukdVmgd23picJ)*kd&yXca;Q~lsf1S+eY1_?H z@JCo6wx;l^estP2-T1be-*&{S7rRW;wb9@Psy`3#)A66G(Wtjz(?GmK2&aZSqDTz( zM}!&(d#2?yv`dH=B=Xe0-gS7Yv<%(bo0u48l2cQj3|&pBMXZ4)?H(Z+p-NLf0yMiJ zL2aWEqTaaakt5RUz0RKxA(OOleY!`y?%(=l6<9?;Bx!TF&Ku<;*9yDqe;YU+(9XFOvhVE%l%K`Z}aQ zO3tZ?Svfh8+>VCXNLmPq2^V9@pSfuwx2__9NPgxdhOHeq^ zdx#99xQxDe^S^PM1zvO*2k}@PkqDqaCwFA)^XJboO5V}BB4DADvvclgL;yv&lW+h1 ziJS18#JSf+lWQQ1acO@Y^DLVXN)o*P-~lW8UJU1p4AbnbURPJ4T}b?H$n%F&#^9%D z8b3iDa9T-Bjwt|W^6+pWJ2;pi6kuPYM`+bihPlOm*3j1F*Xs*&e zb2BqCL9#eOHH7K~=cj{DoP?2sE*!%wi|!;GZW{(Z_&+*ORm=lg*pE>|E3QHNxeOEl z1}#^8b>n?QQ=-yC8ZI;4R_wm<3x|U!9^^t?kbvq zbA%_3v&0pq8Ww z(CIUAgba>H{!>4GAfpNR#)7%2mU3%sZP=TCL6V2voe4cxMN`uOM&YHuf?OL(GF9lA zBV!9t)-TY;GeMZGarFJUilB(Bm!7AFn`Bg@|2108XNe~IwJ#Bf`qYkeW62ohYg(c^ zxo`{)05L)*j$ww1pwY3*28B85ec9scCFImO(WV`Oz@Pk?_n+Ab(z`oNQu>VSGF$hvr<3BVfZZT>YQ>NcTRS_4HqcbU z(3WVIV%_y?6J*fj`axhQJYDS6Gmi}Yh^iIaNs95LhILW3WG6r%v)91yOT%iUuEr!G zkA#R1$nr)i7zP9q&I5xl9rT7E^sPvC)Jyn(Z@VL_4kL6TU%9y_dZ%Q zKx#7go11?;a4e8Vrl+D1Dx8fL5EZP6|P{z}qZdvBh?Iic$Ms-HeQk9CVIIdW+|uKtVA&VZ9G!v{t8*^1-?IHx`dENp}>ucy&exrZZ)93?`?M374P4_Pn`v)oh14{ ze*WCGTN@&odX)B1Rwnq(DdCejwB0w7Ah8z46e3bmy5xwbrKT>5+3EnrMw$a}G3g4y z>kW4V1%K6OC;bdy8~}$GzykV^cRwY?3Lx&L6-yWaM1<~*_9r2<+n4Pv_V_M>d{B~c zi4(&)42en9&opt4JS0LN$IiR=esF;+Ip+8O%Dq*|=2)>}J+;{ROBkuTx++_A(Mvg} z%j9|*uc4xHH0sQY-7=3%eBka^Sn+5OXPmBBC~0VSylF58|HjHcS;QFag3TS*+91f# z#$d&Nt$O9WDRa%v%q;4mFIBLV6aQ;vnVwV7`WTngVUn6Qz!EhXeGiPLoV8|P6STezq;U@&_ zmG}uk&e-_;_3MnNC?hXcjMl}Lb0|w#Ff6?S9qAKjNO7IWefqN!-!F)aU_fOar0O+} zsl;u0@Nu^pqc@Q|3UQj0+UKzWFp5r@G}iOLh~pp~aa7`{)Kx2xaXtcOgXXEa&xsS_ zq;@E-u_A{%{^h^Ee2P(z%XeaP?&em2Q%n`r0}lk*S^y^0I}+3v9TE}Gb5QjMJk}Kt zEG{l4UlA|QTR2SW@I4P~i8cy{I%)k1Q@9Hmnwv{mV@%v3DPGWz^QV2o1(d9vhc6Aw z9+pxxeNfS?yM?|~pm86~WNgvqf@2TPW*N`l+@~|hZGjjnisSknT|R=`ZXE&|0`sftS2s5FeEr%^ zd>jnPFeff9QXXJZsd@|&H9U}<|0ZglSCJRn00rLKvJL6775u!Z85!J++}!XB9bDvi zpYcQLaro_MtDui6OwRRsfD35TIbm)TZH>~oi=i914GYIcJzUW&t(f8@v1=9_ zM_(13494|+X(yAKn%aNB6~RoHL!f^!RAz}x6PZB+=M?dCX@5+QIGA){*SAfY9k&LA80LJLfY=fB*8u_f_g|4z7tj-Wek&ER0f6XIgaLWeyLv92;gr$hq% zHlSrSWTF;ASOpH#hcciq)%aG7NmqC(v2`){>7o+s33a6n8$w4IL@)|Fa!_aWlK|p@7Ngx zbaHlKF!X9zL=fvihH3%&uUU8?y=Hpsw-fNcfZ2|1I;CkPx^~0FFWd4ro!`#wM8@?BpllAs`nGP{wye=0WlYWa*mw7`Ej9 z)1DiFohlnzqUs&!9WiopG)@-ue}3V@9U?6)omn}KQ8QNosJOJw6~}y3R^8f}rEU?Z zj;5`umPH`bD#sR~Lj8?y%dum~%X5K6(|tCV3Hb@5z%Vi|L^3 zv*vz~BI~>NFGXas-qAPk798$DSE`qZfa<{iecHah;81(fX5~oJectCHW}bLIyj<3Koi_xWIkNk$Gr2BU>Q5L2o% zYP_Y`HvvTimO&}5m25~-G((ux1QW8f0U_U@xOeszTBMa|7p7>rAWK~!^WW}$ku7#UW!JTrX-WM@wtCwTvry>OT>7FhXylkGU7c2izWku)DSL<6hz`YbzJM`ke zNk4RLN+*&3W|(9K02fKLNPgQ5d!k+X`eD0k@y&ykF;xLan2m6>sc_nQ28Se%%ms!V z@}x~Kn}Bhz&yfTYBc5L#K+zWT2Q_|Yq&0>$S5Ui0L*|E5Bt+2hvC-d~uW1DWl14cH zUp6spAfErOeIBeBhN5&|1>bL9v!gQWjBM1 z7L{6r1HSA=SDLt{5$w7!0Xe|u#)NQUcS2^tn#+uQOr-S__!4Il>hqk_`!kw|PqKcj zB2!c??RO8(;vq7|1_-U&@W&7kZDPiW5hck?z{6O2X{YecokGZnuvwROh*aPBH~!VxLI2J;T_@%LXcqC6 zT_M~LxG)w`(2;R#Uj%aw1%@y+BRjhW#7eT0;lnN^C*5@^>TdLe1C+f&b-41Y{t})%h;V_6 z%ertMX-&=i*!!)$kOQ85e{r=UZjhkV4*_Ne<}?rwp;@ew^CE()g$XFV`*xBai45s4 zI+l*{!k^cEyYv2$RD<=KP(9#}TTgs9z@ezZbj!B9+7s%q&ZT>!+d&$&n81DkkVcFf zzKF`~ulF9aV=Zsd>L5sY7Jd)vbSsegP*DbKMfr&;{wTS)v4Rr9V1)z>(wl4n z?W%WFwsaaMs!+>pThT03a4UJd1 z73zmOj?j`G)QaJpTckaxy@-G9J)>HlO|DYB=#u}>a$BzDV`S#tBUOC_6ytnb32G9$ zT#{5Wf)9n^E^@%pkhAG5IgfeQ90V{&UpnnW8P*gX;AfV9rc4FYdj9;aDQqTWCnHwfEptKT2$;}(O>!ORMCa4-0v&*Jfkv;^zJ(YJFfYdW$ zBnQgZk1YwFgu_QLIb05Tr!mC23ZwG~RIq)`Zh2(&$S|uxWwvi)7gPzxb@;t>3%=)= z2ONV4c@w0OMBkClQeaSggjE%d(v!@t2q*&61&FYJf?qK8HCR<#(W_UlCh+0EMp)+t z(gkaProF-n5z=s)mCysuW5w5^;H6Tj1GM&!$U_K}Ikt5hDMFggNf#V%xmR2D^OL6a zX}Lp>q%+t}U+2#>Fz6+SYHZy+3I7T2XJ+C)S!)Z4(0OJKsB|(mnyN1RCsAz&C?k&I ztH7s|%1q2`0UMG#k8F2HBp%;+`NI?)D%fyTla)d|7C}zV$FR{za9VUf#27I;JOP`D+Ig6d+?Lg3{Q_9 zVQ_ijd`r*7jpl@q2(yYB0*AUN$94EZqDl^(Nv7E)?70&)b<0%iVDA0;rgSB?r|=Ed zG`DwO$6BFUA%PrcIFgHtbBs_u!P;dbgl|OH4|&B;o^=2Il0Y_L_wcX`3GT;Xz>ob! zx?{HaZK9mgsTEA9EZ;qwV(;eV))qk$-9FX`go!y zfZMeIJu&@qJ>2rZ4}P^Osa-q4<|}dPSx%DHuR5>Yw9%-_oR{!@CV2TJxP}cD58)B^bhdxXsL8-|m-df}~#7~@E;9L-V`zo+t z_&H-7<9yvI107Vd-f%Btv8v=GW8k;+j{zSx*XTXiN5^!1&Y*#iH zLAZbbW+Z-+ACr&*9#JY8H?g#r^#jS>i2{2b(EsVwu$n`a_TRtk!zt9I8n=p|G zyuSHF-40}xeEjGPi^PnC{Y~_!g@DN`AUXo0ejYAv`*Nhe5DUCLG0!Wh0CO7%E)0f3 zB)Bkix~=*&HmU*YHr!xna%&PjL1`soCXpc1T3Xs2!b3yJw&@udXsxXct}=wTA+jgN z^cWcp95dXoxeD>i3>63Hi9VzwX0S6WBZGZEIuOnpde&JX3l$YGF z;Vd}uc5zzC6==_l0tEc-EN#dk5>@*MbQ*7TkOHc2a3FF#d$t3kn}b1-qn`h_EB@O{ zQ^1V}CZ{N*0cL26U%=Po1kouR+X-YEP{U1j<{XqFnxlgUZ@;V!_W$+E#qf;6h7D}Q zj{*co&8M?r?~cid3a1JFUb-kJdD&YHqYM{n(;dz=pINn6ev$r{25`++izJkPybOyY z_pTjCz8TYs>g3UwqN?SL{`vE#gLnaBqtf*REp(4$w3kr@QoZ7|LciD1-rKgrxQ^dBO@EBxeDx;skxY?9qKvz3r%E$Lx&{V zV4V$==CEATOULftPDxpkElv;S)Kh-HKcPkB_c z02E9N+|eUo0R329q#p?T&w(E@ZUG>lsBhe+aU~O#5wN-!&{agl#@4Zn;`mcZ72hg# z#_Rv`ypTqydI?*EP+`wLHwga>;x*2>4gA-^^7m)-$_Xn8M4F}fW*xOGjOY(GO2Ff4 z>z>}Xnlxe1(1^iF9~{{+bnHRLB_1RgM4?Aa*-?C#)L$Ia90KjOn~Z12-{km*&5?c| zDb^7J7|~w}*%=}J!eBKFNmqN`>;7}P3ENl0C{`R&6H>tchArVE=P&xCdh{%{ctfg0 z6lGF}3ziHI4}(QxTM7p6QA|`I2m$r;r%&C*BV`%{oTYe;eehNOM0|FWo_84|x+BTndip;T zQvlFyY^_%RwiqConVR59JZ1cmyzh($SiLNCB+$=m$AXo??&QG4(m-N3Nv`OG1j}5X z(9lpaUym?XiTsM*1fU_D7Q}8w1sB;WDA>qEflJs$Ef8oN+Q>kA6dc%$&at8(`Cu$gHF5Jtwg6-`W*lv6l5G$$ z0zw91VZ$%4U%#$Ip-&9bVz%NGl4B0_{(gto?m}*-MhLB zO}x5l)LL{s=U;!rt<0jgc=c&kqlKCa?l1P#-0?_gYG!pf^R5&dCJ9z*u#jHL`VRkJ zZ}te~U5w~F@%eX;N|1Awt|F^g?&jL*{N~c;(sHk^+AkJ+#W#Cpm6oI9A;*nTFVC2< z3AZ39LA=pU_nY}11X-KodLI8*5Q%Tf{xS-n#I?7IMI(fg=#AM6MSg$EGI_bVZPqWP zP$>RDNOx(svzWk2Dlkx4&Ac4b5FP-JsR@l84AWR#W(t|7ZiK%95ZfH0;=qu}f ziD|CaH|Ow%#*-4Nb4G9wCIaNc zLwL;n77vyt_M#q1M|tb{HM8*7i&c&T=>V(-j*ns2ww3#pgRzo^YKRUx@wsB9C5tJP zcyp9o4t;?Aj(~V*z>l|mR)R#LU2)5nCIc>-nDpTS8U&t1AG1qK%d;MBCtI)_?}DNO z35(dR*T;p{QYf2*VfGGooXP9PMuq4_J6wCdd$_>5%?EBgQ{Y9rf}(V;W#OVlbX*I~ zus+HL!I27TMA_0S{MpDG?}Eq@?=`HH{pG=x(9kj5Ri_Z(CD*Ut6upcxvwzQ?kOIf9 zEDVwT3TBnXPjt=rH=no)HsH4=Zx(8nxy&u!-~D_(7`Sb5?ZR3-^*ty&Z}Kagu@_oB z`pGQs)V&rBFQ=B(0LM4=e`wr$DE1b`wbB=pP#U#%GK0Cr@dfe0m@ z?Rz1y>5cD~R&em5BO{Nl=b?CjYZGsJZPtT4D}mRKqaz}$GSKf^p)=q3AqWrcPaZvr zD3x0Qyz4&r+PA}m_p}u{CP3pQs{@jNr+EZ7XX*Br9UR5*KxiXoPd@*5+ zW!`bk(3e>VZ;chNSrNswdM3XO~UouKG{8gDaf4wF*%O-VHq+HN!@z z{{_c&mCk&;>21^T$ED~a;^?o^3A|g?Wkc50F3}!53R; zN>O5T4SSFVNJ35U`a5)He;I0z)7U5+vm+}hR_;(m`;31$IW;@c^}54SS4ZdFwvm!# zK*p~gOU-9t>Uv#QN5kcjRq8e2Ia>@G$6G+k522R1RqV(y8HLGwr@_8Nm3T8VJ}N31 zcau4oY@4HTxD$VDIrH%#T3Jb^KCmX#*|%@PM~HF*{);9s8_R3L7R*NvVOWo1a6H;! z!J3sTYv)Aqh^!7;TPHch#>Z!U`)03k+W!hS4^RJ?lm{-td-DD+kj}m56dd(X@%?S) zCz)%|Wl?7m((b}V3D2mK_VJQ9oF>Pnr%e`+lBkt&0Z@5Li;Y4N+zXu4!5hN#{{H?uEM$3Cn3VhaEL_x}0lWI}EepOW3FqBv!1Z82bP%HY@inC1v07 ziS{>a~%#cS+`pJEw2EfRB$AJ@qangx-?vsQf}y z$*s^p&7%HByUQ}lzPh1E-;%o-eeVBV)i$tL4JpFBQ5Q!mRfO}@{(bH>!*nw8&6rl+W$s1)xj~#H{GI9OJ9mD+aYY-6-cekLuk+y9j)tR&bkNk_ zv283#L{J|@2Jy?2abFa<3|>bAAp^rOG$fc3ZNGb{=;-S=qqq3lXHnCj%LB)*GJ1Pq z_PlvdtU0!5gI(@BHMov&2kjbqoK9iX4*4wt>_Hb?cJu>wr>V_-=g5TTV6)3Z$PSQ$ zyv6ouO5eXOkOc=!avuWujmXG*NO9DtU~ioQ$m|I~Q!$FgSQONv7GO`VxcZp$@h=&` zw{Z!?RUD%T0o!YR0pi`|>twYi71G|PlWzi%GzYoN>CKrj6~;TNF$v{8fVVYQ-@kje zBvoAx7B~*y%FFRs!F^Z{O$pxOfn|9aj_IIjW6;_z={L?_j^jPQzE_-0wj@>VJo7LkSdjAFA(d&i6iW(8I zOe4f#KW@xaBR^lLz=4M^=jKix7#LvwD>?=Sre1!?_v!u~(T&OENiwEg>6 z*%{2(h0NGxry5I=eJ4~%iT4K=L575EWh7z_v?1M`Bv9;p2v9{pY7D~-{CSjd1qF~Wua{}gX_ziQPP{7JE8tY zTfZe8!GXCOUp0^BAkk`$OgT03kKt2TSNw`^%2Q=W74obbYdANid+QH9@7SPkPnt8n z7c18LGN!?HgWN_TDMTas#(Iv0#Zi=x`LHFLN3qeeTma<<9awEeYm{5~Ao`3fmbt4x zy>mSl7?|o(hp%IV7jD1nYIyYMhYS5u5`yTB3X;Fi=F0}2Wn7a-oA+?xIgr>O_Fo8o z7pXJ4x&xauP|TKnt*42f6;WMJEN!Pv`$GyHbZU=2MUQt+%?O<{la|1eZ@=thgLp93 zWAaMhXM4#(*Wui!VG9OKq`G5rvBk{7asBq=occ<-Ey2jr-tcel+qJLi1+v7}kssM% zw(r{Y+Sa{yPmHhp7{c?J`D!(PY(yKc_j|zAx?g>j29jM0SyjM)M@54G^yt%P>blwu z|5-$z6tsN*lt3At=!7RBp>ABjKTf3$kv?W!zG88dYMT$*lEG4GpT-b$meyhNfx){5 zWGGv%qf^@Y(~m!X)me`H;%w_kP3Mi5U+1iBvpb1Yg5D=s+Z%T3Og#=B?Kb#8)TjCs zl(b;W9_vn+H$5=S>*6J^JhpinOHc|6v!1o-GdFdy3@u+>S6jd(;5TFL=9Y8l^Q-yR ztX67&VRp>1$u~i(Mvp%JlVTQmyLxh8q|aiWjl$ zt>AWz$n`?XFp1!c+q71RwyQq8aj?bhE|vbyvEQZAgB!Q+$^x;pJu^SqB4)YkoOPc# zs*E&gf7^89+ep~1_Q8|_ScmC!SPeVtR$htdx$OC^;jWoyZv%5mEh9uE^DZJQDKI2# zZI=OnzY<_=7gR?vo7{wjV)136SaKxDg5srmtQk|bs9_spQPHOzpWMc@tH1TBo)g!6 zd^-(nFp>!D@G+hcc~&6bz^F&&C;RQKS5Glr4g|`* z)9Uw4oBH|iM3n@u2KDMqLzl9YY@a%j%>A-N@fuGg$nIe!j~z7zlIsS6QH4u6F<*-_NOah3JK{8q~9 zdHi=<-?Tx-Z^K(kgQ#fZdnGKD-!K#njr^_to!47BP+9yl9lvQlZ!kZnl5>tT<%s@J zkLLj7jWuAd)5GoJDCP_Lw$?__3#P@aSTq#Q?t=U4!;qdQflk~qz-1gE>@v?R@TLI^ z0hBwzfor&*Pux+N9vjZ7!Z7@PT_&PZBQ_OdBHa*QuFMV~70#Kr^TFTO&o7v&uQ#by z`B*bE?Ln^VWbAf=(~+ga0O*>``;oWbZs5;mER*Bk3{ezMZZN!I+&k-NU9;p}`}T$K zR|+{w40`wWe0~AbZ4kvtb3dYU$|iqnXx9$-Y#?;ndy8sPxZ)y?Q0j)l0LGxl8(e#>er~9UX)C`mvR`Ha-TVoIs)S z9D6|Ht563Tn%}8!K4rd+`8CteZ^7W&O8YE7zpfh{PRy9}TfXdeHb5Jl(_3oLjdteSy#)79@$Dt3`0zQWFFP}BezKH+POlm9=;1D%!5_t(rCvWvwRY`Ujr})$#oF@_QVF!_M!IG2skQtIk=c8SRhPxPh+q z4!FM5>k2%9c3yqnzsw4x3Lx~go6bfI@KMZPo9KH>?l@BEewY^yi$XMb7SgW5K#k*^Krk4@?PQdy7Pyq^tgd5BcIDWoE#p=BPl zgr7F}0Oif;>EC~3QpeZ%U3+p{UMm0D`OHPD>A={{u^E98&|r5`sedKIv@@8ZOvThD zjo+3nUuB)g%FNjVPQ|ZoJ-+*HI0(|3zBiMSQt6VL%kFMo{FL@E(yOYn&Xcoe4LSD@ z?pr&R8QHr1>f*Ji2<4nUb?VgE%GOlA3skLji|@zz7W`u|KOpL8p5Hc|gzddD+?vlF z|MJL6yTw)c8Y%`OQvWuqHK^C;hNcA#9e2&#*ML$EUiI!| zz$m3MjJjM1ww#Jk(Vr;rZlXI)PB?wtSlE;FvregL1O&QeKrJ`|aH2Zf>9!==r;(y~O@DXkA>{MO z@yZ@wMhcv!!v#!{`o=`%mcc&M@b~>pRz~<4yB0k0(avUlr_(5>$F5uw9t{ZOh$D}A z$M@Qke%l)<23CxB0*!FfPuPQ$dYrs>1!H66uT|S^Lp!4+K)SQU?;ao(&bjyJkrz?V z$eo@2x3Dptxv&9i`l)oP&v9Uk5t#lq>|pU*9OVBEVU~)rrmX?qfl=}lxsT;h{Z2^Ws z*QLPw53dn^X1do*LXBMa@7+85o72HQN%yaO%r|(Ju-5NYpmX$v{kASt3Hg<-8#qq3 zq+)i=$+E}a0$#STF7c4A*5Wsg5kUZ>GN}sZ=QSv9N-|Etz0*gIPdN~b-*ckvg{3PU z*KNWS#HHyLG9U=rBd6%uO_Y?NM;v@EPuOhTd-O;e*MiSXI4W(&KuGWntAgGD{+;wH zf6`uWnCI%~h7H(-s3C(_mgG5cd4xgt?)|nmQ52J4kgF`Zewl8Mb6eCzOgtHgL8vwT zSmxnIz78j7ko{=hu><&1HQJ1;Zz1|Xfh>8doi<+fw6y@B#9JSP2is z9CLs_-{8vRI<;y!l@&ksgR#>ybe#L`cXPKT<=)|4m7+{G7&0WwnVTtY`t|#11D(gb zZ`4=(%3HM52-jwT62RJ)bn{bF?!Ye zuz*gxI)X&E>AfK7(_eM9epg@qVQh2L%ER>ia6W*}^=MnKxSHy3b6%efR{H^q2EC5B z4`q|FZRJF!dz;g1zy&=m_3bgZWck|L&zQw1)!pFyzhNED|)y%{!~r!q3c;q|f0u2PEca$v;0zS;(D!m0E# z5fN{h7kUgG`p~>TZzu7G*}fu=m-g9;n_|;hWJwPX)yeVdnIy+ctX{t=%w`wh>-U-~ zn*CC<%e)?blIN7Lg>fybjHyN?E@}svVQG)*x8V3=@5EQf=FOjPwQsoChDY!pN%P;8Lz=Dl~CtqWjy!)-8*|g!YzS8$K zKSsBpd2s%8XcRYy(F*HLsF^dKFr6U8CmtAo-omJ#py);P40#ZV89<(bX=T|@0z9|u zWB0VfWS9&s4{O^qqxFn6pwqT(C8;+0KwA+n~lBV!vU9~?S| zki=i;AJ^<}R@|8OUWkmeNava1(ZILD74VI0wCUa3rM7t_KU4utm+Z4;47=eU;H1%ZJoIZB7pzX@*Hv;dtkdiFZzo?O$cm+gdd4-9 zgN^|>#k_;k{7)E#MUQOoL2ssII8s^h2`$v&2a!An8G-Rol)~m{;QrugiWmRLQkn?N z5Ve&B&h*lJzz_1})&SKX}tV^^;)kG&`>#<7(p@%I1@j9EDT=;;6GBpGlQ|CMy+R> zI7-E;7!5yk7e48$W4h+IdA~NVD(X@5wr5RVGFI7zhUA)a`!g;xT;<{9@rCH3GOZ>G zhFv>ophL=}1@q?{2$-gq^LqWpjXeh5dz5l5`;bnbrO~CyS3X^^zfozZee?4pURG7P2F}vrDC@gn ztG;GH0k`QKnJ3O{3`>RsKz8vKD||jt#<$pbt6RcDmQA95TZY>W{jV;n_7&Rft0+ss zk^1}8cJI%Th}Pz;K>v~C{XZ~*-M@2q^zUzZPdzx|t^v#@TprlA(XZ>*-_zLWHneEZ zN}|ENi{9%j$&$y8ABQLcxZyH@ld|II%ZdKGbZxv&RR(iDDus!xk@tS&9c9P|GbjB0 z&!qw@oL%41Q2ck5E(?0Ys%*BBpaJY6l*%M904`>hhKn&Vqi@d5%+59 zdAHona&bYoZ?t~?E@xg);~=;6R7NewYy7xzr?!Sup$;GJAbim@m4m6BscF1wdAp!s zs0N0FP^;xBR}T4H7@GNW_p#?1oR3!$s+Pwj9Ba7S&7vSLwi?;-PW)l8`l@MsDAATZ ze(P0Nac>*>OWWX<1%~Zp`U>);9hnz1DI7qY{-FP!x@M!73_bu|8z(si^7yzSjHPZK z?L2YfMCovP&5jsFgOV=O==ACTJo!uDe$GLnV&=SL_%i6()37NtvC_9w@7_0!mq**c zl|k?ZwPJhR0u(PLSs~ai&LcN$kZ1XM;g_iZZi@}*yhm{ey4skV&+S%EyZN8xF{PZ_ z1^E0^*_oeirR+n7Loc*lUi5;4E3X!oDhWsEP0|oB8G=C#YWr@ z4k3&hbHJ^Qed?Pnn;A8GIdAZ#zO-42P>kdKi|5WHNs=&z2_5>_##Y1%v!)>t=Z{4QVL0-n@IZbvUA zs4<@zJvS_baTASD7$~m2z;r+NeU_}<{Nrq2kN2-a5pZ`mHoogz$-rcNYOYShaXPKY zj|!vKLsqyNbbq~Y@NM~CD&A$u==?Iek^gc0pSL)wOWCO_lcCQFcFnZAO|PU*RR!7}nXWF@>sTPU~+Cfm1*r~BtQ-a8c_pjnu0j7Ks=+jCs< zcbMp&%Ic_Lo(eeVjG@muV%{c#3ghy)Qo)EMJV8t8-QTS-OX$l^7*<&}l4}+2@CIV@ zu=<1v<;#)ip6x>agY#+T=B660e9wwWg+B6C)z@(4V7<7MjE>4 zo`E~hIB5G|&e4D@F+j&=WX~BdoZeY3^CAB@!wfELG^uPr%Nyh^5_#SG`bdk#z(?uoBX%yV!!!F6)xeq&dJ|9z||{ONCt_9`a5y^j|4 zdG_4ykEGAStMXs|ZK(#F^={OUKDm6tr{{B1zu}8$^#XWgo40H^n~pB2_qi>O z+!t+5H(ZK9RuCF+<=l}OkK(=Tg=SHU@V!&RnImm3)Kj$WozjDMA?k1e58n1!qS2&% z`$cV@v59!8486O=d%)e$x2STVEKRozJpPk8V^ZPimKHr4+P*7qH?O|DUEK9>OvOS? z6mt08X2LF|V}@Laf3S=vc^CVt;gY;bLcuhuGc!x+!qy%GeKVbX7n{f4jC6nfBS?$S zfqUHXMLJZpYCwB%#d7 z*ZPH0A7rr3cy^C$A3B7K9?zf6<*`W$^huFgnV3AV62m$7Y|I6Q+RVz82{?R6GZ3L; z9K5Y3)bhWuINHW8<)N?ZaVetyniD|nGOK?PTWx`DS4DQ!5P^5#3$M#N%JX8E{E8(U91Tmy!YF4~@aGTEPMorzhTP{B~C;Pz=NJd}-R_boTeG+X8-J+MG0_2lRMyN5uJdI{eHY8(A7(p^-D*gP6a6_w*~G~C`OAzs!IES2bpLV;+Pts#(4 zG>`%F_(_|G@ES#>#4PSoN*&KbUb(Mr_a>6rQehA-F!#U$xk3?(7!3Uj|&&+&#Hh_Qq~I5B0e7TY-@0=NzpB7FAFt40`n{T{8}B zRIVf+8!g-g4NM}C%)dl1|&aiJDxZzYhd*?_c59$QW!WuE|Od7(Ba)6Ua zU}MFHl(pkr{R+nekkQ878txN?9vaHhG%4{_Q`%_wh)_^A?UqIJ=In`D2v$ z$EBOM;0zh4e{}giq_t-eR5%E z{v>B2awlA7r_)hemLfB4Gohw`Fbuzi5dI|2@l;{vmw7m@+rmVKYHA#>EQ5jI82#4Y zFPDE3olIE|VN$)l&!waBZFo%1{2MtDL(}H@I^?6LRb_}w9X&dA`pbDTaJnc{0VxU@ zhYWuLqH3b<*$*FV^y*nOM|YCWa|oA{owB$qDc4{HDCkQ=5)EjAt{29%R4heoAb&ue zO?K|xwd+{fc6@B*Ew2*A+=}x{L|XV!yF=bNn;IHgVCcB(93GdKKCp(07gd%$Y23^D z>^OwQb|w7gmb_+W<{J4*c^?izL7o{xT}lP{Vyos>zND!w9?J^J5VKPxFgtQ8Yx2?6 zeTr6d20fx{B+o~BL7JDK-igN7^J3)5Ck#VEGY;-N%345WL$fR0KJ{E>W#vn$fH%;e zdOvPg#Hv1)$+<6ntndtxjwn5?=v@7!gJSNf-4Vxh&yD%=Q55#PaY5~le!I~D(!j!L zc%zA-&b83N@c*O@x5x$P2}?NxQ)@~=v#ijW^%0clEEcm;@rp(EDGtV;*A`a^`= z(2#s!$V>^ti>#?=4USVRqh9lFpnjcIVRp_chdx=N`avPccY1?JD02onBeHpEHCR6H z7iS5qZXa(ab1|~HhSEWO{Cviq_i54hB~z~SHAYxj8NY1wy{|cMh1G+)c;xxXHI7I+ z4NdXdql*4asw3oZ0C!#BRk`6@$x6rolv>K=th|}BoqP=^VVi(U>E|<6!>GSSVh;v2 z-uFthTiPTPw_kcFk2Q4wuGJ2TTrt!bXb;q-D%BrtlyeBcnA(1$(ELip9cMAFS@M}K zksKcKdaqnDTvMa`039`Xo6Yp~7lss3T(D$3U zQ@|mJ-VMgHK9l9%XGJpY&|BgU94Uu}PIVOjU(^jtySjRJ{Z>2B82~9d0XTCuE{%2; zP@2P!Y>KIb`O1jAW`7g2(Wo@jgyMr+43DP#$?ju==Pnwc^Z^0?$ZI@Hq99c+#n}{I z4J`3Cpz!n3gYARHiynlxRKN$=Euq@L8~QN|yM79*ZI<2)F7^9I?jxml^*u$(ccLL{xHepvTlq2EQF%HI9^Pv;vvoz6<)un2@$X0R=bq4sk54+>r$ zdT|iElIs&heGy}o_`IJ7#$L>TaWDeEGm+8@X}df4(#zM+uk{#|*e4f;c~_74xAx)R zFT5EkZ&S(nwf8yoHwl4*uWi_~4bVd`d?$aR9mJS2lY=l+goVf>_SLKDKjBQziO}A6 zA*^Q%h?T0MiQhODj8q^O{v4sGS zq@*O9>-lSFtK>TBKp5^B)X(lKl35sU?=Lt66zw_P&hDVvHpOwxP3F~w=k^tu>$HF4 zZX)tiv9Qyp*6njDEH_* zm>=}RWP4xF6=T;MDr7b zq0s#v3SAC@D{6!Nfx5-B{(H;HQUv!fX|>8kA8p*9b35c*2dTb)6^tUjd(62VyLSf) z+To)mY02# zBwSxYJUjmWRnA3n;`y+&Zlaa!SF~}h*Cs@@qU(isc@&br<864+Hqi#k(Mf*ewDI*? z<}(xzgocjQ)T(9EqMn5@jVv|Z2}iW`=^uZE3!+n^Lu4&ZMPGVR_B}v%cI{d@O*y^A ze8Sk%3?ZqJ7org%_&}-qZ*OE~$(s+rem()VzUTX)tB`;Tl8uex*1NBa%~QG9V5Ur7 zeCjKsmgo}b>}BPY578;NfO!yd7_UvD^Fq)L`13$WoV98%+?Dt^>6Arae$lJSlNl>b zUT8jx){_6jC6eCbIYfp41T%SIiPx{6PERzxEgZV}GSpB)TL0&ytwt;89A(!oMPB=d z!|W()6a@i>xiP+TKDSCiKfF8stcTI=Pi^@WdYVOzQsvcA+LKd{EewMGG4H%=Q*Noe zg1)^d9ZVi)Lze@I=(#9mbA&`Y{nOaE4|)59l?D?ysnu%TGFku)QGTq?{~TaiRlY=| z$GvwaUE8Fi1V#H9I{lOGSoyEcqBd^g`&XHF4U=nB4g>k19}y5w&vf;9d53oe8v>jY zFA<1Q7pCZMyJKtW@voQY*$!*}i1QB@%9jy%i0*PJorzlph@S>cxZi*U+r*s0AP56A zcf|j0ql8t0&u~mr()w}*h*++Kl1D$|o=m!g^IyMxBcgCAFxho#2G?aZ(O79P6pAIp zZ_(|;zY5_DNDRr~-+`W46kTHc_M^fb@+^Nn_F^AKJ5MINF))R@FaHo9vxb(>zn|^Iar9%uuu&yN3^2e*cW0youl_mAeJZe&AI*-s;^AERc5~IA}Tb z52WT>pj>f{q?hrlH)C7*^mCpJHVZo_InMgyW#WnVSx_RMf=2zCeV^)g8#HTEX)ot_kw7{tx(S!N61N-&Rj=h7RRM_c{!>XQDi7*5^(aPtUybdQG z@W*Gvs!rH@ei)6veS@yV2FT2=TFu{f*x~QVLn3;M;4k#yW0lv@fg9gp!`pUfsQ^J{k zcg4tAMjwA0yO0bkFh*qqyDO>qh5@(hPTtdZxCj7BgbSdZ3QimSUNK0aS&PKYT>&$f zYpgG{hh%*VnCg3F<>E9O>7`(O1kyxjelL%MojQLQ)nSkR{qIoQnKVNnCy~!R~f}r63H-O=l{aoug!QDmKd?B?{Rk%eb0;=hgU zlXLBUmiMhx4*O5*ym)ZW&R2e()=McrH>KZqZiCNn0XT_(q1z6715SNigrlnJt zua)=~O`05rtI0pG;?Zqb7NPn!8bAFTofm*YDLjj__Jk+Fj7t zuezuZ2F>JPyX521E{(g{bBCp9K2$se;u|s-?*Yms4a!y8MPE6Ia}Fh5Y@aNqz?wmx z!Z&aOl;}06va6q8fsD_%3$|-W`z^9j)bno^Pe-(c&5EFg6O5l8$qyO3W%FaNc0XP( z=s17rt%$XSVV4;)&pAE#7^sC4ct41o;r6eC2E+OsmHrAF+Q0o3s=EA*rUSl8@M3HR zvTy=)v8%cYjTgc<9ZYtRISiY2^KGOdrCj!5)x*Hx4y~Zi?%fLnquPN|kRBt`N{nYT z1^24z6rb~1Q#-(=#6@0$<09eicOZ-r--9Q&T*$DjVLdJ~rcgf5-Nq#ybYXfFs% z6`H>CW<*bx{fPgj=}k;xg{FV@XYNxjt~epg0h>*rDgTeeRlU)iVasEuS0hf z1y&Q)Hn6yHq$d3ik2XjoSoE~rRwU!P6jY%CYKhCq-Yp&hML$uY5MTExh|n7f8Fn0&0k+UU#~KZcY1$ltEO(7Z8I0EOu_a} zFV7zg+}~uk3MIm;Jo~qsniToVevK_6IQ(EutTXCQx7c11-cmMIacfQr0EH|K)AwNa zVt??b;GLiOd7%lU-(|s>E_G!Ev1yO1cE9fzT6lO|XuK1YutL0c#`42*Z$&j?v6}Y+ ztTJ)p0H2!0eg1E@cKM4v!q!TW=a=b}y4c|A{>SZCZVv)U3_z%iU7+lZU4MDR)ZDhD3*#-W3jQK+=OzrPL>`X z1&wOt%(>29Mg$7;HzvvhQ9L5NaYuekNRgu&`T&JZ)JObk1P3hyQckeW)S=g;2* zNsfm{;SuMDo_i#6v1Z?V^EmQ28UR>M1gIrTDY>Az;?xSdkoYROg{o~X3r86$2(RGD z%-0`pd5xtJr-$1ibRPKa^@yJA*_?2Oj&Mj9F&)}15$|TS+c6g5)g_a(?a?_3E2Pd} zP+WE;%;S24mx?3kxQO+~)wO8HbS_>J1RG0vd*L4_>KaIYyR9)w+E;=|O3VMNxa@hW zBH2k?^N=62xTT1uz>rngN@P=1YJn-b$)SP6aBc`(vp@nS6YJ*$@Q$iAQ(yKgS_t=- z71gDfIq2$D1Ms@fRnC2>!~0pN2US>T>(c&b#-}OnJUB$?0b;N6=_1& zJkJz99#Aqv0A!GeLQY))kj+Y);53)v!HUc2&odrV0Qlr00ofW|^V-(RgMkj||J)7< zRq$fV^6!k6!YaA5wQ73|Yv9XZ$Dyh+GAmx~m&xO(z^vf8QH$Bf0iCo}(O~mYF82?! zp9^SfRynRq;9f@6cPq>efC1!<)aZ7weg$G@VrWMcmuD2u%mUm!2{r`ra>A4ISOE!^ zJd8;w2p-MZ6tjUUg~W84HL|wmFcLChAcs}F-I)>J5(6a@zEpin&PQP%ei1FQShE?r zLl5%3i-H-QWk1PX@YfwRW(=N#@f`CJ>0TA3GYm!VQHfVA&(_*u$5 zL=c*;kzeX`9p;P+%bZn`Ug|XUfjEQD@IlUPYF&5QQmdrD|L(h^bKt#W?J`QXlrIGH z9TN7ed*ELJLdvhJKfe(ij0#@5wBlC0T}>tZUTQ=4b~X8<;0RQ$f&@pTqbSzA@Nc0l z=!4LnW+VgH^{uy8`AA-`m*YBbPF;cdRVG~&PDhV^0hueq1)E$=VRGpk@ojY}MO+ZE@0z4@r^u7vuPgf-leTsy2aBs&!0I* z@3&Paq_kPNE5g*@DaC&r|6{G@>lSl)FrSdRSb_-1XF!#ms7?CL> zkV&h_Ifqh3^qOkC0@Atacm*bAK~@?1Qki)MK5eWdbadn;W{kdIa~Pmg>(}8wCvlKI zjIRt4BF3V!=Y~adXyeWZmh+J*>YNT$dIW|Q*C#R_jn@Vjq3sorPi{8BhLZGh`R?^UfBjj1DAJ{aAwEYLT4<-IS`!IR9&*FZHH-VA)J+#%JF}kKGp5Qgs=yq! zRH??eQJSGl$C%@-PR>-M)5;)<0)CpRYBInLoi@nZ z)%#B5xbuhA3u+}SIYqOS`7;El4@un0H%H~Q@sziw`15zWiy7@^^)z$=cshWiOd~ux zIWy6RGO)MfX%GTH!_}~2AE%CMno!X~X7-tU$kdjvQ0g~~5-)oWYF!Uh;v|fYlv_r3 zOJ?ni)x>&;xKV1jgc?Z0>?SQ1C7gNry&s0DQ~_ieq~|Q@4KhTNB$y|<_n$(54qYFd z7hT1ZPqWx?=McCD@IJFNOwRz8yC}8|8G64#R*hc0+ni8Lo{|UOaKFwyAJ@&cQBKv( zl^kiD^qFFshtr)JCcX8Ls()emVn;r~^$EJ1L)`fRx)J@yv()T*ig)4eHY*K!F8iXj zNf|UrJjeVR7u5Xdp1G>hW=6m7rgTn2yDY-2glx&h01?U*x!9BGa~0EE1pa_cC{0GI zC)2wZC&oSC21hqCP(JhPS@vW@*R0(vl6w#+>z>OS= zx_8SmltxEerAn1a7h9$(_qFxE#`o7|2~+OT7j}YN(3Vd1e4kuCy<+jXTV3NDCYTRJ@t+^m?x}f!^;kOHO_eINFt$K%BUJf>4K$jdx(rB&S5+YnVaFEIwwV-mdq)F+>`Ci z7W7C~D;1|Hb4tTb6gv0*gm#LH1P`%mDL8e3@bU~qb!i(taI-XT)ygrX3T7zr#tlQ% z$(Dq0#o9GUG)|VDHph#9CuYvil&(qio12qL$_>g#vu7zXCr_T7P!-3pmNk7zkBaw- z+31pSnFoE7vOVO*nQa}Bu}FRcZ7{Ic#0otR``!(e^l4Qa9=o}<{*FJ-SplA^dca;X z*GNfXiYaany41V(6nM$o;@4!ionW+VZ0nVOr}G+k>yLc95(SW(=hm&6q6LxwbxX7Y zd^J|>@Wlrrc8@(S+g3O2D==S)wG~vCgEQ=b5z3Pcwon-V%b;V&%i=I7-ineY+?ugq zZ6|GO^x|HX9|GmLl#T^;L~ayOCm1i#9-JU)s+uUx+`x8F7xO zt!7<|MJ;>V+#)CbEfd`CReyRwb;@N|;z@3?ppA~qqO1&tV~6OJTOcR4qRIhFZ-@%^ z;aju?JcB>r2P#V$lp(8Xt@@F(s}Ch(4vBc?ldVj78Reb zhbGS8ZxA?-c3*a_*90*bV5Lc1Q@dVq9oYnn;Xt|Y{>FM8{b_(H8GFC6-`~&lkJ(4t ze0Wi%TRb(RWWurzX9R|lU#qXa&X}nFPFa1#_Q2fq2FpYrRs2)why$8AT|fUdZ}E27 zZEQrTOLj<5f_)%dfJn#66e>hLyI|N-S7sSe?|D_096~y9fGds?uTWX^lD>%HQ2w1a z5v6>(-TgI1D2ajubk+0E`;9Og2AY%X|J1_l$MKw#R>B7uFM2eo#tH%~5I&&XcL`BJ z!Wv~$2>TE-r>Nz!*StT#eV8*|T@b{KGfj9ut|~7qMOqfzf#&JVf~j(5b551`(Y@W~ z`7a|B4W(j1_wT=-DTYr8hg;Yb30*6Z%^<6?ZJi8&4-;GT>)!RepBHhFLW z$WHsVowgtt>E^#OH>MIf9DqoTg~`o%&B2k6%;0TsIBF7>-o?f>{Vy#oNV{IR#S-KW zea}K%Bf&YeS5b?h5ZRyA*q!+w>X<{fR!H+N-4o8E!E|0{(z%&P=pkD)m#+K&_+#3M z`0!I{q4tY@_H3>7`cyjRVEu^?KMSRDn*LjlDUkN_9pv|uw{?!I+6KEZ& zZF53ToiYdC!M3UBleZ*3Ruu_?$YC=NJNxHvSugK0%c-&nX;@d6og(RE^?m@U64i&x zv?5El8lf7CvEy5|Q3zuNV_KRCQU)Q~1eGNmDbx886bg)|Iv1lwmL`8(J;f}y+p7E} zDqvxj+oOB;+km1t&`dCgjt3j+v4T6^&x)f|41zTB)f#lg-HqWX16oIHf8o*0FQM?j zkn?BMcC1ElozJ|U$Q^0PRkXK+TEanEllk zc6jnFFon2xna(sPr_+lbL?7YwUS;%w!dhI1Sw92yq|5)(P}K!@j78+>8c9PXu}Sn% zeM4Oah~N5^g$g8W0viuz&qV312u-jDs$iU*(@WKd=G-Rl<=$0?_wx)g558TUyW7K{ zl%3!p&c34L)z$A$DX#a03R0>cxf()xiF0;*flMMjCwLrSm_%l{+p=C|PcXrq22!b}1d%ugp) zCMTslYckGa7+{PjpYm=V2!W#d5Rg2uFg+wtw35=bquD@%k^kf<|3-+v)nz}6YXAkQ zT?vs`#N7Cr_a!rm_lQgXa2&(8>o`SqAr98O4n4g0ra&WUG%ENHXR-nbsp&NEvt z!J<}=z6J@LXI(cw8Rl(^^rV7R3sNrZ@q568aRQ*p9Ie>w+Q@e$tCjfvY!OkV6laARk*<9kJ zMJn{pF9zmNhBNfZNvgT8Y&U2I#Nunbs1mWkK zHBjW3zXWO!KDsPWsI59a{h=cxe?!M-`x3IB+RV#K( zd=-ry+#<6m^T5GA2Xkth+q!pp89lA?!%+2|1~*-(s!E{#VCCn-MgvqsrTh&ui&(bM)`SmcA6`o90&4J|bb}AQoCE zXAVzohWYl2Pl;#~szW}@#;Dm{ENi{XN^N*`psL{>T%acwK(`DDK#+71&O?5jg&zje z*){KIXqaZql}~-)a0w_LBtVhuS#%H_;HMreLsgrO_!XO}VI-ykUxyDg%4GyvukSKt ze-Ku&r2M6sc@*PFcAr?n(Z?ZPw9KZ63bXi_XJD&Aeu^#b19k+GD_v+|Uy3%~*-Hc6f~E1xlGV(cp&GGB0+ zhX1|aGJ|F z?dNhliz-NUSP)_x=dtVZS*6^U@CvjIiKA9bEe z(!W`%^0`o_*y`r1gA7{+Y^990;&87f~q~gT=Xpw&5%JJV-<) zP5K1JcnhW^W+G(~bHDVMbO1$0=Nyt+P^^BR&xD^4&bK-v^X$Hpe$bJVws_Pf-JYe+U%vyi z;%}85lZRvE@(|d6WSfRM+V~)Bt|CVxu-%NwpyE*SZShwuE3~S}uKMH^$=)MG@l#)Z z$X!RkLpi)6|Kl-O-BAs8h#2axA@@^cv-FIDv4T9fhXSMC=Ia^K4oiOH!gE<>bY_(o zX|=w*-P@v+;pwId%nyLwo$X4*KgK10$Ji}g6M0MEssGVWW$p_cc;IFouT_azb80C& z0RklJmBVXQGPQjyOkmUO4?ME?J7i{T<$fruB{2_0i?x&n4w16S3BDW6-`l^7lEEDn zSjvJH{ z306~iiLgWVu+?D+^eUpg4GHFpA%!QNVR;k)(<9jQdrZ^jN0P0 zEfoJm8T&qaH!CmFZVTBYaww*;MB9?0_bzf|+Ii%6rU^)4?kG zI#2Ya^B=i7Z1h%<&M&fQL7D&#FdEyCnGqGNr{Uo{ZtQ*9xH-gT_S<*_$K{|YU&KTOKO{{wxJ^V zc|zt6Sc`4vewTX=`{oIDAEb{Ev>7VG&zh;6>pnvV*n+l~erORCEFOcWUNM11&GVP9 z4Y}m0uUL#e*T81*lJGAs5`)p&d|4X&s!E!1`_I9|Gk^akx?3G>l~Y@N|G>lRswqA5 z>k{~gQPqq&ys-xQ%L)s8^R_n@=ql-BsmH>GNaBZ}MH1{q!|D@b4DADbD*cWKV5!By z^wUBYB<#$`?y>yEz1RNtZIS5n$H(AtTTPLc-(p)t%dzw4%|kJ64`jvIS7!g2$(0u_ zZiij__rFL#RlyDszD&9f+Hyu>AyF%R|)EM$(SJ96xZdjrMB!{@gp#o3sknT zo=$=n>Ir9#L@BpoM!IvSYJX=0^XXt0f?03XA8O6mRC}lHF3g9P;$f4v=afZ2Fg zSAFHbMS5)1APiPLS6meTCetBd%x>S9r7u8%#VrGTkwWxu5y_;ISp$4y6*F0|i)xwe z{wn$zY0xLA?j&3ysF65k+(}m#7$r2O1huqtAI@_pp~%A=y-m3MynJhZ4eIz$-{txX z7jC&$2XQC*qOk+n8`|p7FfThsa$kqmlWLhg+|kMO64;mfJl~^#F8;=3 zpcIBQg;lLqAFwlt`;#s2M)a0wUKkpEB|by4#T$+N;C)yeww!aR+_`m_19|9q7e2y^ zi}V7ZLI2BzZ)#51NBHqRA^^OD4}DnQMkpKUAw-pN~mA&91wC}~K zqjvrC+TCZzj>AF{h@+nWMpZ}f4Tp_}`;P?&h1FQSzZp#p<|A|wa$K`rR}ymLWhcf8 z<{?03gPB;=P%s~rj{;7zE0OFnbTtxC_2?(X;laf+A-KTEZ2?CEXliw}0d5!00 zf%Chs{5`JUxN&xClJRxfALeVwK+n9&+pG=~aYsW2NPN@k386-y(Yv>7*;4;QbYTDA zeJo#uZ9h~CnTOTsIe}`j0{@;wn_bz><&*ereW$Nm`1e*7k_jF2+qOAe`v)6qu^y#0 zgJYi|lXnyZ9Z{Zm0hL`TWEwf&oy2!4R}o0vHR#{}O+aDFp<&^?Of}mA6yrD=7P%!M z=XEDBE3oFo|G2j=pKI1xGF2oCt6HU@9c#&iDaZtu9{cfLX1(%*K70;ulg{I|oHgtD zll7qYss|tXbMxl&Z@w^U$S zE2+R4HmpyH%@; z*gRS>Cb0P@K=}CQW>*{mhNSajL*Fl8lUiR~9WC&2O<^2H{l6tx?wDj3OLj}BD32t$ zYT@_Eu4@Z|T$;YXZ6wW+MSJm3Zrc`>POra!nRuRv)&%#k_j$OAv(x>bRc&DV8$B{` zKYU0>smx1wsmu5J8P^Ij|Ien~R@v}!Gd)$)?lp0i_89*YjEi46KcV-hA6QUz_}|1E zWp!5yMT$wu7Ri+%)_XMluvayqPk9qFituDSyv2?UxZOd*mqPYBV(1All$VU7?Z4N*qq$cB0>Blgu9V90lm9v^1PJj zRf0Prl|_6grsPx!o1i5&Jv)3e4;@rVS}-B-j0T$!rNt7$?SW!i*)p$S_L5%*A974lgkCRZY^kEh%pc}f20{+H& zUGwg!Zb&0UoL7C1kzzk?(xkInK@cQO38eR02F8)_0X?b@HLpXL9~WnOt-LpAcRUlJ zV^!!cUANX!Jo6Jmb#ER}Uv6;o!B4ASbNub^2G8nsL_}8FsuKE zuRtPJ)Hz|7Dt+$Fz(mCpG~}qbnee|zB;Zhwx7#m2UURaHg)J-ZOc2Q*uhbW+Ma_;JyOWMdR9|Xf6>V2_U{!G^dI>{vc4qsQ`lDQc z`u^zlez8&(81bx<1}k}q*hI{XG6P^5)KtG*?=B3~#-2N3T8HQP zJfH8GzjKVP67O@+J~$-$QA&}t?@CViz+bBYk@5{R11;JI%ndBf_+2MKJ2&vy50g8% zY|NF@Ag!mUtjKN(T@LXk-bA#UeYtFVoP@?m^eyv;FsV}l&pe2PXj3y1P)>=&AEIYh z*2{N-P`{8=P@ZO%Mzm+2U$%*f4cSDWQ5;kWfG@&9aMAQwAcY<)CROU&?)V&ID1a?y<)3%y0sRlvvS)yw_Q4@QTVVNk|m>rz-Fz zQDl_!BdvRDHGMd^WWEFx`7uPU+2Xq=nPSi0xjaGdAbN6s`hB6k;d2D}A9EiOA|hK{TGI3@k4K}GZj|KzZM1ADzo&_L%l%jZIHo$^<)w!sWN$hcJ z3xB=y3?{BNQGOXggv1DOD1bTPMpm|yW8oPB-!_h$t|c*ezp{=EaIj` zp(L;PI#JUm@{%{>8poxups1yEhF>>-d*|u5Q4CI z$!jA%IQUq^?w7Er2s1JOb9ZrR2n%G2N-Q^QWA*Bh3%Is~p@{SYnfS9MeHH)dJ{4^& z%o9d?2MrVp2~M;iaif7{Sa6h~es9u5RmAd|CDddPJ%^A{UIy`{IzHM8__uV}1l>p6qrx z@G>6AydaVBg4p)w6~*@<;Z*$KtbtI^x6h}6N+n1oXPJ!AyTi@xTu2_Uok$EXRwcZTyv{ZP?#tqOp~C@%0y98l$?+<~mB%Nc0!d zn#zz0?IPF3w7>9QsoYb5!cqg_m;D(zksW*qKagqH~=ut*@u5b-w<3?mndHA`zwR7^r5z-~96AFlO3C zWONe9GW-LT0N`piYH+XUPkqcL&%~7l?d3?_2kkELDaibNUn^>{d8VsU*3$dvPh3K$PQ^;j zrhg^V?l$cXuS+*Dc+Sf;>#S+s{3rn|)$*XAUjWsPp&GuYiLZFqCb_qgBYA(4JcGtR zmF!nY&Zp_4o@cYW#kDJtF>b+QK%Jd;C*@*J)A}6-NeCvDO)@`Ry1>pU=9}m>MB-4b z3{lDu(EKJRrlhYx!}eT}EKU-vKHT2-R7rNIJHR^6D{3S1#imx6$t~(sXcK?ojzX@!{Bp z4I8G5uN7wX)~TlMzcB|pUEk4JE|1Bu;jsz`f{Ed6K*|nXv&nZE7X&u-b+}&u^&*}( z3`uILPT8vek$i<<=A-G?+OBu;sRJ$Du;NSZ=cXdZF8ty2sQgGMM|cZ1D~~6~Sxw@n zt$~4F2rN=Ia6m&Z2DywSfe!CAl`VH?9r>yCug%*vpeDcGG9voHqh}4(!d%?-UqnQI zx5I3Yi%#o1)VTkTGIz{8|B)U2#<<(wQ{m1kWAt2E=g*&?B_ln%E@z|6hqP#gqMhb0 zDQr0vw<7Ud-DbR&wwFG0B4Q4)q6O8X$(0b7dyQC0?UvA)TgQ1svN3P0hza@`qT&w1%d?BJP31<053Im^ zDu4tV4O~~(N2o~Qujs4cuU;A*Vj>b{7bhVnwYF*;H^s0KnQrEW*pr_~GG?vpTq?0a zIKnKoceWxTkH7xfu6FTA62znmXPi+aD`D-RoQ;XbaoaX+dOSp+V*lEkxMB%l$5pOo z5)0BrX{HBu>^M>d1r#rPVWC!Y4#ih^t$O8onj2o5x8sPlwmUHvk-)#3^j4gVzR-w~ zr<9F$jxD-x3(_ezns}!{a);)63C2D6zqA{j@!yENE>f%tA|5A%6!B5f&Xd0fr{@JXBV%{(DVtgO80jTDx?*}rNhOP(u$#-%INJFKm zERYPlRg-}&-(DDJ^>JoN&SxT7*3jnPqqj6riruAy{;BuMtJxxzW{ms)xZAhBQoKpJ z9x@fg!OJeoD3`Vv)SE|0@V3SG|F5Vd0aDdC$S$#n0gz}C0d|K<@S?P0BNs}K#==|nDlf2h-VJtOIe@hOG#2BO#OePnC9vrVzZqAWO#rP?N z#Y7W6g<%jCwR>BK_stTGuc!cZg}`4|#ksOW@eXHpRxewr_BcrU)Q|Sb>r>S`?;*9C zx2vNB|2pi|n~!SVRq>*wH`RSeCWvVV^EGYUhPPL;3o`mndi~AX>dISOqektoPPH5N z*4;$#jz32!IieAhenjFRx!;w3B75Y%sj!%G%dUi&2%}q^GrV_iW|PwtZpng+vi=Wh zN%p~M*B5TlJ_ip{Rg@XknloW4kwo~$w4cKQhnR8Z&>T$cCap5gBE^f)-(iiXnmiyo1Z% zR}{TJ+iB;7`a=UoK9BqU8=cXc$63~Ns{h;J=ce9EUV(3UuX$Hl@#;*`!=B$x-x_3C zYn0A6HJcrq*r45ru4})(Q#ao=ukvz7&z3uHeKmXU)6(+a3zygqC}}jl%K+!Q7fiW% zfgMhiK_5W85rf@_pjnr&77=ygn!OgAuCauElv}SQXsJu5nz17p>^@6V+;3pSo*}>) z)nwl3S(79*Uhoy!K%A%zOWEx`NO2+1zHj%|3_9y(_iP~x7uoQDw5Cn}4_j{@P;(l# zkDnL}BgRh1*peb^*`kcK5=Sbegow&kDWxcbj5Sh9mP*m8B#qKa5s}a)+LU%$Bq^!i zb$!ar`+k3K|4cKZoO7Pf^E~%`ZTA2Qtj?B8Yz#od&G=6r+GF6AI$vm#c*+k7^~X5% zWXAKb-gMC}{SkF+NLAIu5g~m!t;{@sAKF(n6UXBayzCxYXcuyG<~@s3uICKCj6L#z&z%i(L#581}5q z0VsMZ;V>%urHiBb7!-(TNOI^}fX~o5ZCSf6^er@UQhI{ijZ9p?&!*w}yo0HaM>gAI zydUb{cUMSmBn~LvH&+LtuZt#+9{s>?rs%nGq#n)wj9;K&@55f3YY~?_@+mp18O4F` ziQb|jpE*|_H{1biN(r2nl=fD@V3;xa61PJm;BB|J+V2Vr4S(YDpb_=j+S*FdX>C$+ zaB!ei9}n31{N1yj(?Wq}ig8ZhAy%mWy2llXY?LZuh=Lf0T#7wLGSDr8kX*agJc(}@ z>+QoL4k!3TOjj_92RL`&Q)G_v0xxtbu^{j`o=H0Gp{0cd2d`A46P6iIeq(A(K7DBu zQCx~uFkgz2Y%MvF!x9LfnDrh&s*}p2iTxaxS?$TFh7Iz93s+{U1`e?KFH+8)IyGY= zu5n=Ll18DIoMvH=z^v0585yIhJcE5F+OCr8FVgO%yh^}QVlQQV>{wROwg;x+JJucf zqxs!r<7E=!r_Jvi9OA?EY~uO+)YYNHEHyX(gf3>e(Nd9YSL+x9h@ctTB4qdipG_`{ zdR?0~nUD(3f=aSYj153b`vPH11)*qzp}(Awi5oUeh$K~B*tGkfA)wl|`ilk~b9Qzn zhy$$BlOvR|Nb&FO3us28E9KB?bq`LkjliTv32NuJ)UqfVa~D z2u5fS!q&blNdF$R{`jE?i?Xws5;=49;~uWP5=-vr-NH5J3TcW@Jdk}zWitGf%KSunji@-N-e3P_jz`c9)GgN;GQ4x_nuC% zuGH3RyKuUjXk+bGz4*_kY&%9j*R9lz5WrCk)N)fju9H;N;}U|_#?Oj9bgs+D5qi8H z>UP*4JO~s~Wzl4~=P6s9@ViJ`3-x0rLudCpH0@_=ftFC+;4+vKLotm@TxuPo33?Nc`!EJWnuGRKQxj>=gvN?YF)o72@CP49aaIl-NK7w! z|4m+06koGzu?!7|GUFC5S%XovFgomjnF6!qU`z~FL5d!tEH`_?wuo9$=6KTT7f6v9 zv>s<7#h#F4CK)lVl=m~bCvcrR{+z45kB&!b4Pl+eZnk?)y~Aj zS5nD91zcJy&|R4W%C10I@6^mTn>{XGt1jq!kAcy@Zb#;!&gIo%2fNuG4;}!$OH4g% z+Bm}iZyeplh1a6KDe38}@K-xQ5?FO`&w%|gs@)X6KIr23ed69|s|CJ^9Vf3ydGP%T zVMjZv-`ITH`9SuUyem?o)IZZbS1SJcb=o|Eo1Owm?K0;yYp6tg&|hBY4|x_AMd1x<5SH<-0wqq zdEt-V`%!Yh;j;2XHH4g@jO!u70mxZ4CVwLry5QmNd#FGc(sYJ9a(-D~3}7WeiM5vT zTeJkMWH43?Ip^Dex7K^~g>#dy1H=+Y#Y^XYuj>QiM92%-ogaj7?4&FCyK^|TdUxJz z=k9VM$2;%wf#BQiz%*v3U^fU+PzRH(O+bUZm3QJSb^ zW^tG~C(AU%{}PZ`v&;HZdHDqY3vnB9emVvP;84GlyK{dEZlNIv9h=s#t^!yhM$H-r z0Fu6pg|}&{*=$!k?Xulf^>hapbCLJ!*=~;;#^VDPCC+opGw&$k5>GXTehPd1K)8Ol zdj9e~VAU=MT*X|6ElQU)QOQlsgvM_l{GGxAZx3H55N?Qx<7AyRtL9r3I!y&#&if1> zn?KG;xd{{eM*v-qRYT{c6Je~mXfozsbl$)H`}G^GqJndwsG3l4sJfBbFy!5APZ8v1K7pO@MG+&oeLnCC5l&(3`llP>hP0a}yqWk@o5Z3l=bnzz6Yz z+gA=Pn({Xf=4fN7fpBmrfkye;0x3ALF`2B^mPMQafURnnQ-amr$L90iRkh zetdZ`uXYDS06Yu>&2YOD*k`5AY+iUo8ZFQ}*LBCo1Q2&tCiFSJ5Xo2yA$hADA8ldf z?8C4;Eh*8UtN3@w;TX1F34iJNu&}TcPf&$q06AB3rPl(@Bn?rb0gS*?U%+87vFG!1 zsJO711^Y8|Q>((iqC}|V&=c&8P{25Q52=WX!oPk0ZiW~%T7{8au{jhl04Y;mkZ{^x z=>KpMB_04Xz;*b8(RFT~YgHwLz`&aMp`^s<&p#9AaUwHtJe)*2C(55RL3huF3}=LJ z`+A1x1~WE~c{N5gY`npuq72krPEo-o>f4RTVR1ABJd{*E~(@|B~L}7(13XG-$#k~elCZ%$DiLX?*XE~4GYxCDT z;Ddh!%dGI=F(goin01PA1d~+{;HBdBodg~yGI?qA>srm@&^6!Wo*+nso(atBvpJZC zd>ttRw?eY9M`h7dBP>i)2nukGvzzr!18%X2Jwzr@MVNca?tFlMRAni~LF<@iS?7(iNd{?Wt8Ke5dp zZwcpE0VZ-XePEgc+HfN3u=URvJu-CzJV}HAC{Kajzs{qnzWx;oCjnW3I)FXVGN3!W zb&)A>;5qaRrT|GP<-z09vl2rVBGyep`{@Og_v5tvRhbj9L+@l7(Y$Spy?p>CE8s38 z7ySTi&nh8r_uHLa3dX^?0LE&TF#Q}^ZUq-agT&}MHrwx+A(T}*8oYO}T)zCwYIEIO z+_`{kchM|`lX!NM^o3~d;JI!WoBT>JE7U@6a^c60_-N(nbE`UUTKUYJoBbsJO8zHF zB<;TB_1{hldo@O08p(rNb-Wf7@YM2%=OI50SHlWaLpp0&rvMHU&?amNUb|IqIgS-q z%K-?;BcX!&zR~+PQGgWIW4_YMIap|1sklQXjD@&z>dF->w9#GQKiOO$kRh$Sj>=j* z;OhEh-m#p-G3|A`i*&AJRL(qYX=$mc{@3SoXj!O>BGUv?fn|B|zf2RrC~X`n7At!q zsFBhjmQo*cGdfBs5K3s&((7-le?*1Oo)=xo#$q!UmPj2Vyxg-=t#_%K;FM;vA1-8B zL>(A0WSVC@5Yn9}lM@RW8HC%TwEk@w_|_IAIe?`#(jp1*niR#C*BZV0^ij%m9;Ffr z=|L9jVP@QB-Ow1a2?7kMN%z?Yh22WHB`(QR)*sK8oQNG7-My`CWKVN_JymO2m~z>85(kjhd^1I1$pnL(xXSVA3hS z)^sj&70j=fHjlt{tJS}#e%&oyXEVH}{`mBiufR~^OwY`+Py6%^*(OUks+SK;N2`sm zDMmkmi^L1rf%m=MI(_#X+11hAWJpc$%6^B81%^VbhOSVP2V*Agx?8|cQ96nI3NPnF zjzmwM;73F3*mot5XqY%CrC|&q_e&bG*OK}wA#_ahhsp;1xCT-=&n+Flq9pw(ZwC$j{A>(GzJ{7KMt zMrY6qqWkAf&6g!zy9k)UYoevZm(gKMTap`qv??^i@J8xKi}6cUb59}7_4EKf zbzKtQ;qS+*75Yd9-#OoOcX5Rj2GwK*Xh6yg=48X#k{stS097y!;X4r4Q6#}ZuUsQk zp=*oh0`WluR)vo+8@>k<<8GpJ{SZWtt_JWbhG27 zCc;dP_?D{R{t3s~aKUf>1xKpgmBQZ=VS^O2c_DrR62!Tg2036(n2dNIx2mYGU`oaF z2tc)WBPua?yI&r~d8*AIung2*y(7^c(b|FfOy))xgq%ZE27l)wI`i={>F^c`6t#}V z?769Zdi8PvfFoQ~BJF{Q0ACFufsgK6+hWxvR<(*505Lat{h@6`gxs9>@DS>copZ1Qc*n|~z|!GvrCynU$~Ijf1nOOag8+~`+VFthJC!bT5HR&uaGAQMmDwJu1)?M*xIo~^ zW_a{?k+*X5_wU0Wk5`O3zJD(?22f9O9vM>@KpbnGsOD7xF{{8E0!~>r7RAW(Eojkj ze=rgP9PL++1nfni#HwosO)17SCv=ec zRs?g=1F;5}Dr^fL$9PKgaJ7&hWux8W{&5i78h%xa{ic2SLNe_C~|@8^Dp}=gf&7 z0ZJfs4b%)cJ+}uSXw&n5E-IRgPIOsr5Nc-R>5GXnU~{Y~_5^`I76}c>_>}b$-uYbo%VCI zGm4w(fTN1Se(lpI8{n(5=WXDZCMYXOIS62sx^ehAMrbJR&U+(X1uGZ|V@&*70Ux<} zZn8^eZv5~;{nDxcJZy$ltkqTRFHkt`5s=8!}yMgeX_{J}DL!@$MFK)9w#g=wzP3wql?RRuH%nDfx zsZ6dp|CDw=V1)$VeJ)naMw(>rr`cvqsl0i#JEY2 z#82#Yivo_;9E`hk`LaBi94#s3Ubvi5_Xyq#fb~d;P3xb-)&wp*VYqnanpL+vw&sw2bz?vo3*kPfz;N;+6m{kWEs3amPaESjZ}a5THsFKwF%l@P72+ZX zK$8WpcIwnJPZRBZP10QDxbN$qT>Wdy_|iLJqeZQQW?7!v=kHjKH;iI^|838XT3O^B zd#pKk=1ue4BSs99sB?%u)k`PJ;X+>Su(_}q(y<9hB?2R;V5}PhY%vf;u9Rs)>*rHY z74x*V8Hma>3)#8elw$zwjop*v3BNxZ3r>zDc9f4`E^g^-ez?tsDLtfk3o6tTg8c4| zQTRG#rH(#V(lIpzR6IN9Mp%R}tMH)_b6tQKaiRl7T3C+(TKHAJz_B|NALonm&>~>! zsK-a+@K}Y4>=x7p>iRh1U%l%PF$Ri?FtO5xb=M#bua`wfEPAO3bXa$`QDpg$5LH|ICnDnhc z<_#ArNC7exT5L^$_X*7rV4uy;6*B@VYUV3lfK>yRp4#hUy+b3qIo{b@-zVLlKPIYG z9ll%mMm#?2>;4^ElQ#7^D>}DicXXdUZu7=A@BRe7O6cCMUfs0?-5bY0?=)SuzoX%i9R&A<3_u*?mMCU!`xH_fqB4T8u~*Jf(F*#j2bW8G#{<)n!9%sd7s9aVFsHr2@0G|HBk}PaeO?6HMBtMTbOtxN89TyCz8sZnhjCRpMHSrm7DwMsDiZtdgm2qGq5caJn zI%=}JHd-hmID8BBzTI*T&@aV!i`T9D~j>CfVtv%Lg1Uh_DlM85rHmN(L;4o8=a;$O6?1 z=W!6^z^JhBwXM1!?JF8$2q(MG6I~k-oE3x$Sy-fZ_rjJn?5}KkTtcfTKqMKKozbYU6jl{$rq^)9yTlxf$}z)Ip{f{*ShHN zhsgQSmhc_$eD(rgo7s27>_qeZ-dBFS%4_Kht^~c-41sKM>=nr6HitYLU4t<=z*lhN zKS@td7aKYMp}iGkhe!1(=x$hc>PQI8k-I(^MA1V zFxucvE)N>JWXU}zeuA*;3(8VU;GmR)00G#7%=lmMgJQSdC~|bGz5FdOD5o0dqtdi~ zs6~QwZ+Y^Ka_)xI91Fi?V`~#owOof(u>yoh4vY)Atpi1|V$kF%OZz~SsyLC)ocJHw zdS*L)!efH`7u-Sta$q4F(Pav&4l%%_!KYF@!N)N_i&{SxnaJh(7L5Nv=9If!ORDu2 z&N0Rd3ug+LXRz-6$yU5t81liB7T-HHK`*7PgcgLGl7#j&cko3Vv4p2eYxC&gZ7L?P zaWIUP#To!}=VmN6qk)J(5|HdEQA>9e4t(0W&}&OK;3micE|Z0eVZdt%QR)St=|D{p zhB4JS-55`>{~&o?*KaN04H9{LWaIVi3P6xKRV@rgM8&+JbgNCVo$%qbl*ZCQw$Nqrq+M}ZhR;eYlOQ{ zk2B}fcQaJoZf+>r95G+^e-&+acQEEobNSk4Y69?hXXB(nz700lDuBtbCSAZNgUx`! zz7qy({#;5UK5m`ONT|cu@y#4DoMJZ%BL8eKt@OD0_H0JI%@0w)mH~k!t^o+qaiq_f zu?9SK@%=uDr!t`hqRG~W{#87y9HiiXgm897?;N5HLntO;`$6s5^noCrWXll)WqGmhkz3Awgu4B7`g7Uu7`*d&d z@n8Ly&4&d77uFi=`QD9q-ib_rIgtcqkIgHuEC*&5+dgtcy64^f~uR z2DqS7V`M;VY^()lfY|7-^sjHNK~-_VMSB;p2=r+eXt?`Sm|zHm5bOK)ZXnVKgMd>7 zGOxg>1iO4Z!m~0tMgYCO&2uP4O<{t18?%2(`QeK`oD~hoi?Ii2jFijX-8J#$a2eQ= zixhj+oulfER{@$YX-lf6AsZoee#l?sebTfbHX1Lu!aNQkmo6}=z~N&|Kum;6?>cl| zfk66%JCms))8;0gLcpS$P}flPkIj!(CqS>}V3@<62BCL4=f|FFHq3-LPgtcH;GEoi z#lvG=RpS8FC3|ZR9i!d*;-mCQ|M$vXUt{IpeQ9@(kzEr9C{3NNdNpXC##QEMAOdUt z%05hCLWjTrmI)*K;f1OWVjeA;&EcjL-Qtd;RKrHAwY4=9Qdna8Y&mcat~YHsi=&mb z3V}-g7Nj>QE$-&n3%$;A5J7_2<7e9iaSo)9MMJxZGMG~Z1xyZRDIq8G<1|A+SO89h zQVr3-0zKCgJ-~H20G=v2-{{oLZn$iiEigZIz8SSXR(C z4c`n3@PK_+5S11cq;JMSsj1F0(|b(t`iPq;iU#Dwa70%?S#0*E0UCmd2n9-BocJT( z?zGJ035J!;3OLbKOI@$kY8gw|^~cYGf>in!4wG|PnRWGm(#bPD4yRV-28gVJYV3>V z!mfZ6Q>I2b-A5!Q0IFax`U?Vu9qbLaj;;RQ7{T?W-`Y*&b{+>WWemp8hf*ZPng&7& z4gG4|!MIXrZ$!7t@mm@@dvy^_T`OQ?Fcn>VGlF+^osuBu*}YpK%u2!oghc>$IlT^) zWi|sa3&ASBg01_@(+OS`*f>a8lo{7eq}FJe&q0UKHYI|2ty3tCG| zc@VUg0OCtW{a&X9%^32!&ZCU`IE_ISv^Ju{bWJk`HHl?`Fy$iM695Yq&7U`K1Bj*! zx{iAS0o>b{B0#)4WCS|3;M1`B@T8xx*f(UR?ba-e={^TBpf+Rpb`aI>5ze`dzdLWHk?e3Dq!gtvrG3A0L|i8u1j6OZQeyFG2P&lmRUHl=X*X^Cml*zB5JUg zgh5X#;1IxHtaD56vCLYE`O(rr6CqHKB421Ic~_T=a3PXND3>J}S!F<}PsGpgKm)h< zKn$&$_HHPxvot@Vrwc&L)}DZQo9idPu4`FnI;5uQ$-T(jod@0@ z8$xfXb2PTKDYmY@9~qFM6Fb2Arx?f4+6`SLPjJ~)=u%L0XQEJHb~~g4S$kL46+-c# zg-t?|ozyI(Jw_!cQko%1B1h;0B6HdTC~!oMIUw7mFMsM#B#D#}-cSK~Drd-KVs2Ca zoaB_0YTRI10L5_a>AvU*K^P1a0(g!Sd?S{D-UyUVp4ZvnNpqWxGRAA*JaY5kbRrnbT<D5D zrH9h1*Sz6yDbb?U1(Fu~GkCg+-BQLdpPm8RjL+p^|4KT-`mLZ>FW2{95@`D zMVu$Xx{-6RL2y7t%3uTBE$IusPm;U|KglU{!rru^;x~Qqz*s--Q)aAtuFpfn+(x3K za8LR{ljz)xf~^9>jkT$YkF=1OT6ep%9CNj4Q^)O~$f@*-ZP zu!gk#;7Y2WOKI1y|4O@1+)~=rutYRvGw^Xl_d=REv*Q{jkDo&K;nn;nn;E-l^mU8S|lww}L6w{H<88mXo4uUxq>9K963 zWCyLy>Db(P%aS&YI)&0;iFikXD31g(^W=sA=Pz63?kk_rQxAu=&Naush5xYjCO+$RUg!>>Lk zT0hhRsbD%AP|ShI+_mTvFN7Ox3Q}LhUH$~E_^AeCXy3pbWPmuN5()!L4m<}Y3j3!j zt34oh>N^kg5y`6jtNT(bMO+()HDD;saypFxYHR411LlMD-3%uxtcnQ`t~{GF5lxnR zMr9`cTIM9<-blKFOccDAMg-ng;XTNV519=Vn8Qy&K30Hcl5Rk<-hs*+?#zkk)N8K9 zKPl@Gc+SG@c)1#ASiVP?_XLUg6q-I%gB*09lE)_q?R6?P;W_qMJ1~xy#k$W$Wa*-w zJ!y%xuTKg8Y8Mvx18zQ7H@9tCa^4m~i#^90KO!nxh=a{|s9~(l`I6mrQYiGKb5`NZ zqdfDr%Gs=SILEL2B9goJ?cXa~9OM#nflH2`4t?9!M{}p1=I8|@)Fon<{OznWz*Be4 zE{BvOLsb64JEss150+C4PBwZ8Ct@qE7)2A82q)nmryg1C$F#u!7{YUt!8b<$qC)ey z3LLwTz>1F;(@ml>8;LXH?o|BGDOc(-7?Ugz;xP0vidkNg9h#v#(Dm)g-Kn(b6L1V( z;5~4_qs4Y3+l1^yjPyu1(gc>oj*AXu7?}On%ZNNa6lh+ZbFJkYj$tb}f;mlw`VydX zz{XKaZTC8;OKcoE&^5({YN!iy2&e(}CZKk8?lnmX_UZs7IZDIiHv5qa z?B5s_pmK!f4xjf$jNQ>({;N$G>O#&N1Y%I{0pG@J6ahiF;%!DW^$- z-x*;n{Ji4ze3I`>{2YA%wFh$HL9{)lM%ol(Lw)*Kky!WwW z`}*+}t^y5vLnHzJ7FYjxIP|PO)F-%|L7xK8OCw`=o+B>ePn}=Fgkxhp zb2wW-j<8S=YI8K0+Bh8~BiZ4@@Q4an&dbhuV*9Hv0)V2<0CKJ?QZxjySVyNVT!2Ln z+Ggt-1?pye7>6*X@nr)qjT9ghN3HcgB_~CbyNuMuS$-W-lL%jt>=H;%9?z0o?OpOz zx+R49v@0%MXAs8^Y<4d5mMz5t4%!RLtO1(%!`@a~<%gfF8LGgA(w+&l79}^c?>O4s z3Nt#;Z;TI}j0v`A>Ij8~OLQ}z9QNI9HFoT>4r3iC$0jHT8zz@m7 zoF*KU%Q&k@XN%tKRGQ!Ldsu|; zD$(Q*56!*k$H0tE;n*_X`65ZPM~^<7Hb%ncD%x025Ri+_VYNC6Nd(TQU&NqB=37A8;M|OMT9CB9^6e3}!Eu=|zeqpoc5?NG>dZi2;_#sQI~4ht zaD&1<3-sNj<4KzXFwMY#+8kIHpj5zp5<{f%OT{bc+B1j)*dVkdy%K%vnwpR}@7p^e z1?LC!EAovqEJc0GvEd-Dbkai&+aW@-xgZD`P-h@~g3|3BfL*H1Ln?U@%5+gjM=sy) zx%$q!UCVWU`W4lpaUJj{CE)0=%IRNd} zLe2J?TnBntXImIi0ic;dVaY|-W*4mvMKRO@D93cTLyHOD^Y~EA z;(krT4D>}@aNk#B;4JK{;~F3SHU<6m&@yac)(iCI3Fyb>T!4Wuf%)8p#tGWh1J8Rv zD}lA6(i1&*+JsXIfkA%I^;(^%^+mQ2M{&Mh4m-F4PL}ILXHaK~F%M@o4C`Rw zJD&rTDkf-#ob;eEqVN(d=;Z;u62}H2U;&G{Jz$%$aTTBeL~Tk<1g6MqGdof;8@5m0`vF~7U%(dKpB5(DmOB=RY3%$8Wed|xrsN*a= ztyTK@fx=EVI%1qS>5$T?$BI35%d1vqC~5S?_&aplnIL}_#lhRaoLO|aE1t65^R@s0 zQvYTg2FIwOu$l!&U1Bg7i-UMMY!8`F?1wWWnQMqr2yUxUCV&aS3QgjsZ-&ejm)i@P zP>v#|vyL{~0(l^>n2_kCC)jtvzoDi>uw^#kZiM#ALMmXSiHn?i2S`FI(O6?|0EBZ) zP5+lGQG4WyGJts>OrWGjA4hEe9Owv_^HGAdC=-IHj2Dr3n)?!qgYApDhYlZ`?M}23XcQof>yaft}L16|s1kj59SJy$dgwKmJWWjY^LbkqG2hwJa zffbbB9ApFJz#!pJz93Oc_+Jt=5@yeyQKGi&72FJXu}{P?2#1EP(oAbi;&CC>=;yWgaw68_B#gYHiFl^7-cTU-=TaG6?g!pc5fL> zj}wvSeC!#hIhlZpTeTYgnEE%F@AvSTrRkL8m~nShz$`tYq>Ez;Kjx&_P>$oaP!tGX z@zR?7om1$&LS15wEsUN!gctNGuLZcErOn_;<_raPWH73fM@)BBngpspmpdIe#x}~V2oa#|2Blg*Z9Xp0LOFaf`sXy!QUcoPiZNb(K=)b$n)n0M}*Ggm@ z@l(T;`0iNC+b^aOzbp2!_88eMU^-lNJGMmpj{o}ntc5!af}HP&r`kE1rN@SUcD_Hl zg}=f9QqwHKzk||a2dY&ft~T=M9Wa8d0w-@aD|mpK5Tqm|N5cAt;gQ({$LNQ_t#%o} z9W6XI_i%X=tfc<^e=NJRaUYS~06cQ&VGIF$>*A``(6_A4o=iR*MQ#oX^=GsG5NU5| zI%F((DO^szM~}f?jY*LsfmA0i{Tl=YiDz_TjkqM36;DVWmT%y(4()AB91xs*=nsNe zZ~_LB@w1zW8y1OGnT&d9`C{U1dpYeUee&$^pm14%p6wpqxG?w`_wW3I3Ao^xCkDC0 zMOaTJjz+#dv?nx)E1zB8$#`S3>`_6dcqn1F9qtG!l6?-LCStYoaZop#!>FK{$S>yX zUc3)MJKfDDjKZebFxO!p;45}DIAEAp&3ODr*BHGQU@nGGbkZfJN zvNmMQ5U*iEqb4W)AD_~d?kuTp!P_;JOuJsYdrk4GRF>@W_PSMUC`{z%*0&1TTqk=8{GiKHhqKCLC}{p-}6-aEhO^sowbCu zw@lfEv0?Zs^?(@SU0mApDVPy-`gHHkTD}PK7tA&imQf1B6|p3|a(4jh0TyKr_8Srv zE)OodEC_^B=ZqFFezUT+6 z<*_RzViyDAQ2uHUXY@<_W$LXVBULMFcEw5~mjcKnz6x+z1Oh2`f)ZXJRG`5ZI;{>< zs%qu#(!ozBzmWXJ@pF-d#?$B}#wq+UQR1RU)kQ)jFDkkbGWb+WcF&hlj~{%#XSrpt zIJ@=J;4@|;LgKBu$bEUtO#%psX=3o10pdcDppLq|bH#_PgWF)l?3ZHPzAsT#rXhzY zU?wCXpvLKD6B4n3LyhRO?1UKx8VdncFv78>MU1fz5}F9e72LoAer{}3%!HW_>K(ks zqA>D<_)R{KjqOBo!ML$#fn!t1TDsU1T2N+-fm>g8A}vP(o^aSP7*;Mbep(8On@BVo z9Oa6_LEiifJQRRV1(yRv(%!?+zqz>rh{GB7g#{dN>Z#?JxnPviM?h^cP*APsd~Qz6 z9{^vr`b0{vt1JE42BHQ1EY;nnHz7F8`)1BJa1bEcx;-a2k z#!wHQk!Z4xZpC}kLNE{~K@fsEh(9q?+&wlK7UUZ!+=iRtj?2Q{GUn9{sfX;085!bJ zfB!vI5C@6&;DMcy^bPm`gPc*p*Wo&+EI*R|oY$fo8ZV5(R6u$tcYyM8WB%LOmmy{I ze5|W#^&b z(nZZK8Ho~F9_9ur8B?|S9Dd9m zJ&u`PG>DZ?2R7Zp`c(ep8GdFY*TM#sJLyae_>POtxr?_QU68SYl&9j z+cv43aqABEshxKv7lYhbuVGEZKw!w{@5mOQ-f-Tj)-gY7-k@Z`qb~q5O98)Ntu7Nr zp_~shl9mKY{sq^1q+Qc)Aq@dXEJnM@-g{?&SQGCp)l96p0)!d(l&3Ht_FFXzHUftz z*bbp`p`x7y8*bXq=m*weNR}@PZ>|bVE3UO|shshdx zC#oUM_yW24o8pb;59An=3I?^s<`7yskA6ibZcH?1`;L$X#y! z{CuMYoMQ!P2X^=SS3?IU9s&17aR+*OlM`v1Hz0tIMlvA7rK9>lz`H4Y9Ra&aWa@tP zmj}X=3TDzfNnhERVl{J9M1tP4q-Av*&(ut zz>9#yDos2O0XvupYpOLyq`HmU23;I;3BZ-F#$~6m%f6yR6*1W`ULXFea8x;K1BWl` zlIRQN_F4MR|8tkfc14=ae#jW{ky6IdjG5OAkfa`#gLixv#(w#3`xg_n8G4WI6GC*y zrh3rrrx63RLjCgxAZeJ_qtEApU}|!5GF%5peFc%hdEQvaa`OdqflBG&NCDFyM(z~g zfYAbs<^?eL8mR8G@yS=?HXr-O zuA`)-`VfzvWFu=*9F=R>9zW#|+eARXVK=j^1(W2We;tCcSvc0JnG?-SDllJcPckFn z>F1wlL|0dh9h8A)B6W(aNL%%1iUM?vOU4pTw7CTkIVlSAMrvchs#!9e7xg0E8tDww z=)4?|L8=GoBr><2p;WUO@?Tf>kpe@&y27w&#CQ>*jZ(nqdYIt*M2hlwBjYwi>e`u6S?i}6KO-BNQN8}CsyvEMaV zGH^)1>N9_R*b}CIe9G7kn>uIDR}sym1Pr2ZL8&FG)5Sr~bgP4D20Gs;zXWkz6;k5Ob1K>WFdt?At*T}l_vugfWAf-if6CGAo(kyr{gsN(~qI5 zsv`pJf1*f^(a`wAy*|YRbJc-*uRLLfhvfztHIQ7B!w=6BsZv-&H-23FQw(A9p;4Y5 zV(=9}s;8JE^G8UDhk%O_4Q@&AWy?{mCvN!G+)S@8<5p6d=Q7M61G$J3vf!vJ8Qd9P zMNvGwr5*8AAj%kD1-A6rm+-?u*S)<}(Z;{TnV_H);=JymeBQ*pfp21_gm)2zt@+sYqZv#+4)v3M7FOQ`m4qlD5zpP*qu(nd z%d#?Odz`s`*bcpUMdaYDOqsk@s8c@9Ud60A+jmG6y3#4ESt6l52R1%b-j{p^4f$?>sx{6+BPWFnNvb(?+{bibUJ9f^-@@_DAb8I*Kzf!G?W~mKNnD@jk~|fn|}o z(QJlK3MG+(kb(n!a)&L!cc~*Y0A3h#ATjvNS3B$hxIR1=MlFH3`^0C@Ns^B_3a&YaJ&E z0)j8mo3FhMpeaUZ#iG{e-H10*1>lg*-`h+#!^EryVvMmMOi4f)ENU^NeI(B_1*s#8 zXYc7z`wiW|I{=h%mJ#AAmM=dCTy~=5CfAF#wr$AteOl77-r`sP?e?r+n%~U4zxuXQ z=5MZHXU+_kFuc43JKuDl@1|~Zf0Mh*g1$dF|j+^#`&Dy1~tbXENVtmVO#uEY9D% z(2qmZXY>3x#^Yj$EKh5cfTOCX;kBX4LJk2-96deH6nk9Aq-&TOM172_GrXK;_={=M z@VSN)8M2%ec#0G!;K>en&9o12KF;a&oI+#}Z=XxskuIbpOvPP<(ce1okIXK?T`-{JvpHhV+S+=~&2EmL zF`Io2!9-IW^*@)I#}h@^phc*ZI7w3jtSSdOiEp&9&Wao#Kr{m>JXL^fOGot=$qhx? zfpd2U_Tx8fXQBueiU<;GjE6{1$I;GleoRsY5LsH*^#QGg|MVjCt6=D=Zwz2M4d*E-}aN4$hrKLZR92spJIyf`P@;be31 z^710DDysrF5Gx}bXeI$5jm)4jqH2qkk*ZEMhBqVYFqv3e3j&DX+SvkkVP7+poI!}u z_OWjLW{x89IaF|uka}r4bi=Le{wkKd109c}h$%IX*XV#`j-#9XP=9TF~ zU{%zg{oL>2>qf=?!o^2AgZepVP2?h(Ess;3^J%BtnV=+-dpqziXLb(^`Q!NZq%%Q@ zxMI*svKdRs12)A`rOp`pji2^rnj=`r>6wrZeF6UN3Gc9@3dpvgJ%#aXi0P#S+%ZRB4bjV zjgE+<3((Co7sMd$K$cM`i(dpVq0daW>w)UIKLk36fnu1f(*fLnln(+4&Lqn2EmsYc z3Uq`?04Io$Q6b4b$np5*d+(T&2f?aT6)b7GY?uNaPfmb{erliJe!GyRK3vF$c1}k_ zLG%B=Se%gG0T;^2=YJm16*&&OW7F25cAR=8x8p+JG##M0()&Z61!J^Fj%`g=G7oq@ z$z|@+ZzfAmi6eV91%DnsR&UZ^fek0rAPn+ECkwmK69GsP#cD$I3rcuf09g!j`x9Yh zSPB6J_I3`GFOA4V;SeNTwese>z!gGUP7x}pC)B9A=!fS_=nX$4yi6O4L1q5 zd(o8Jdbckc=KvX5$dbkT zk!lI(4!>8>I*Z*HQ-DtlX@waX%)t#;87O`LAB_p@OHO*Iq)70Pxk5dM^oNraMRwM6 zZcpR9ns7s0n(tRMs54%6>5}t071K$jsVTSfR1nEY{R_j@&GXL>TDbZ!*)yxIAB|Ha zQuXsiF)~ZmW_SmvHs{Wn+1vxq1#V+A{OurGouZjmgA+>n@(TVkRW;fQ*WYr8FT^EH za}ixcvg^U}GjT;o(OO_1#e=r$EuL8x(?W_CW|KH;)}VOCB*c89MK1peW*lC{h3)DH za1j;waN~gcz>Z$KWo+M0)*S0dXOPL!uHLB~-8sWswC?ctLmJUJU|PogndK2Ryn1Nx z3DeU0gPiUjWIzQ_5w&IePPN^AJ35%X==!}Y2QAmKJUCj{(u zXENnpA^Ze<3t~Z#`|xdzwNH5?CX!8*QC|L3I;)2wE5LIcfQHdSHxJ8+@@Pe5xr35`bF6_Le0dXa z7abFo^&Ei6H9)&v;vwrO_keZ%plxHzf#0wNNBMXoG#om56zr5VynOl6f-dPIBsjAL zdxdPo>v&7UJja#exW$j~@`ZF9G09rR2Y>0-&7`jA0nUy&Io-QB90pke)h%OcqxG*{ z&IA*q#}sN@<`l&Rc;L?BBr!)v5vEwDOvUZCfYB}3J@yJHCsWU`YC6=sjn5b=k^R+> z9jwJ}Y5DreY|C6x-_HQ^CZfUxY_Z=A#SKS&Wg+Zkzy*mi=&}`0BjK{swKToHpY)w zR{K-JrX12eGdD*eCUM&zDly`RBVb%{F--UmhD)n?rYuk}C zx_s-0>at_|Zs^DP=FMw%8JD0s>e!O*6N7Y)w*F;*E!n+(abE1X&=1ds-M`@C``5=m zn}3=bRt<{-twoNz|1G&P@@O z)-i5UUV(n6DpQxIu0EuDc*G&=Ig9LdV%7(kD)w~C`nsd=*`u&_?`Ovy+DG`6+e6DM*JE~cI4*%oA|vM$ap$1QA6sp%?*=Fht# zcSf(UwOwMGJxSYtrRMn`{k#gr;WzxtO)6iPw7ft5Z0^CiI`{UJ%d51f$NlXreNYD> zNmHkdmOGM{_ot%C=Rl;MXzd8_DlWb$H|AmC$932j4(m$~^yr8<6sv*kij-_x6Y=|~ zPxmkRj(uutq4LdG_1Kp4nc?@nS~5SokD6er>>kkkQeQz=_msrihS}hR9mjef_GU9hB8VQIlCL`uu3#><3s88qJS8- zdelB@sL{}LikqFfJ5GNINFT|hr8Q3D&|vcchu}a{m~-8MF+>FfEQdE=8ya`IVhWPL z9pY4AN+$1DT$jU?rnvb*QjbB$3mMS|4nGTX2QR$tqS`E|VTHt>LxmL$DDqJxx^Fi6 zVh8shUkWv*d+SPe0ks#^je|gk%?7;!^uVGG8d|pM%IwuuyJGfLUdK81EIqweE#((E z%*W}?>=txhjUEk1!K9f5cZWsjERksZZKMUj?14HG{f<&n9_F=TV3Obnhr!7e3KeZd z6fo@0!C+{wTW52x4ZlDhaQ>|gDwmw11j%nGU|-HCaw;r@aUc)}D6<5njoc)Ss}0Bb zqBDg!nt=G-pj)t^uB5marlzftQcb~y(G8x~HK*Oj=>(Vg&#|2AX120u4_Y30y_ey3 zjoE-8Q89Xt6)SWw%kAmQJKHAIo-Kb~iBV6poileXQMjk?7PVgU^`m{e3quT^{hoT&WW2J^0kT$mSH^{{QY`hAa$RX^5;3|6dO zzxJTaKUyUUk}?UntHuqlYCo7Edn~)T`nz+1+KX2`e6!`JRHvt(4a(~C zvi;kk=U&6|w04JYXtOSfF2H%&R+{29GtlkgWV<@c7iDSgU#7I*S=FP@xQnw#^)C)~ zH{2cC-r&$8JM+t{w%z5WgC;6{z1U~u{lG=XT@WOrd3FBfo5jT{4mJ5T?O&HStsh+W zym`H?Z9;8yU0D0uWnZUP**>y!KKZ`l)i_N2oOR!J#=DQQ0UxfcJlQ3gdO6Cu6GMzcR*HUkL*xa}Bz3$E=L0UOX983ElKbLtu!AvC(R z4{pbpNEC4Bevgz-$6sVK2%cBy9D#`B-GVNZQFbynyNGIw;j+Y*3x;`xleGp7A6^|^ zMl}qQt^p1BL7!%!FxpZ5Sz%$+X5-zvcSmpU3~2*2edFlSqgpxWepo?N6~x6Lo;#Iw zNun=i09~VA0(EmWI*lfS4d@bqx39p-RqEO&AeN;HcHG1%=@Exj(WfjuJ0cK@9h+}rcuU7R&dv-!To_>!7h-*KAP^7OKTnlE|$>Z1K+ zR=K*iQjhFw8IK)Qyo<-I4KzD>N`G#Q{K6r9@?O4d&(_Ib+TIvvW$;Mh)_3>$7K3zJ z+uyxAv0krXRg6{T%d{S$lb=j}eaWymtG&^>S7;4n$zkm|3Xj_MR{BW4`(#m_SM$r& zcUQeb$6hx1)?T4-IjLoNvzMR7@&1uty_#fy9LVmPV|=4*qIJXgxNm*ukBYMXetq4K zyB|FALch1$N1b$NZ$I(wf_MCdcNNjOHxitlrYmH&&)XDv^7APTgNAz+wF7rFeO@y+ zw*IfBKa5sIRvEVY>|UGual^r-IBmS%{BUS$e-{=T@~}tfGr4_F3pajM`Hr_m^%`4L#>3+tk&^YDm$*^qHbfZSfANBFr+&7*jwAN z8|v3DF<#b1T^pp>A*)uM35lDn2Zrflv<205+@=Q(wB#Q^mrJgZ9uj&Wr~HB4|v9bJwq%87o`IMuT2!!g7 zsY##!L?M zKR>M;BtOzFL8ioO)bPou>}!hvf5E3q!x2=D%~BBO1(`rN1yQ4X`99TBU}A!ok^WaA zx+FzG5p9jsy&{9+S$iL<^9xYQ0O9T(w!NgJgs~k=YO^;+2a>9ZZjD{%BFj?vj+l%o zgplh58*iK~YQmarAbPs@V-3n80RjZsaCu1YscK|JKH|0E5C9d1uYjGv7dX=fv|r&Z zD7jWbyM_}?dJ>Xcrp802psz5b-0s25V+s)il!tNm-D}_O1(IWz)sm*O+Ix>CSVZ*N zSNcTrYO~_P-R>KfbW=Fk6!yJzc!lrRbj218=Qz7ZCI%{3TZ)d!Xe68ud=$3l)u-f- zF_EaBPu2_ytdjhzwD(t?g}cwc?myja@7yK4dgZJOn-=_L@tdSkq3ZFfd$)X^==ST? zdw*)p?W(V7JGn`#x3{&bw0yceyEN}<&NqkZ3DAxHV_0D{``(0w&)RDy3>wth{F_#= zT7c)*0N-q}dv9CRb-N`&dbdQ; z`L91EPrCZf(qPt$wzl>XYmEeZ>pvH)jA=>84o&_JMqMs-3J$q3M>204Y3l6xeN4M) zJpNO5+HaL}BTItr>?zl9v6%5W^0ULoOuR0C-5=E|A01ZRICp5QhRO?-&&f6I$zSE? z{%D9iu6J3dLQAq_TNp~Vw_{bxBF*+|f19m z9|C@#a8##zC&UXc^zJu#3sh7LsSJhjGAq{BasXVrc;XnoPsIQb#j@R#Cr{QSB!wY^3lOfB2wWpE9Y#0Cg9l8|TLw4)QR>vF^rvJ$Szwk}UxPySfpOfzJ;qM2`>V z{2XBqMPU{W0SomW+6oL%#`nF;|7snLAygcME9h3ciyWIE)W5_5ixkqbHL31YJu-_1 zEH*zj#~?qvTo`aTIJq{PA|P0tdB6h7leyV-uL{|Epb?U3uQ-PU!IA;MG-PDDrWVGP zKgGPy2={ULOdO{RN;bPXG|_F#cFkiVCLo_YFdhy^j%(2s4#hVDgs21V!H zcwr?~pncV8*5s6#+s0OIxv$igayZ5)##zU=ZU3G~htTtz4ZM2Z+V^AM$#|Ql3HfR- z&cB;{>w960PQos$W7$0qgruz0c#`?`Sh$OyqgVyW6s37D96pAXs_}Y!eMzN?UGvjf zFMS^D*{|h({?#2ljRj8b7tV+I78<(FsCbYQ$Ku*xqzyHwh&M_lgeWeEuZR_(FN9xX{I|7P-Z zTI8s_<(sWE+H=21?h9zByNN3Ngz2o`=Eq`G9BTIiO0**~61D^`ft zNciZK&v??i~fW9gC>_{_k|DJE-zTf})DW*Mg!m8B1 zD*CSvcRkTjcbU3Ir_YkLbFg<94?tJ+<5|L&;)sdL<{395U*QOl&X2zuCOKRIyz4lt zk54vcq4umf2xg8fUvoV;aP{rJ zqu#oQCnEF!wK|^5Sqwc8A{NQya=w^I{EF%9=wqj4a;{BMSq$pw;h_0(v6=whX4(!= z>_7fZU7fpcIa4d)_q_?;I0$1rk>4eZqGRr=cpj!G<9?+Y7L3+SOq_^;pcGO@BQS&k z$~5-C^|b?Lzh1pQ=bZ~hYmP+2*XRv6J@D+2E&&4}1LGWq@5$1^_`ejKBWi*Jb!}%F zv@D{e8Y6cYn9Nb<_V&`TYS1XDE&&|i%}i{D2b>vxZTRAOBli1+F5a#^`r$iiyx|}Xbfv>uMEY-EoZ(f|6)JSFRy0Y$#tlGwnn??b$0WzQQyN8 znjM;griLwdnfW13;j{V0Sa+Y?Lmqa^O?qB(8Hd~Y=E3R9^Ow$Y&(b@fGPM17!$zq> zt@o=R+z2&!6L``1;2iBW3#0?G(`TN`-z;ynt9rZCj@E>x8JUOLzI^jNIC@)1)XN)I z?RS6XymnrYfU=+dqW`<2FfpUXMtbY-|p4?yx_8CJ6<84UCe*&+A!**v#H+7!XhR3 z!167GN@)K%GPH2YrjhkMT00ob*Mmm7!6p3}A_BLze%oQ_bLOk4l$?r+YS8IE#2%yM zAS_a0Jz(PMq8xtv*Ax+6zscKQf}*e=EZ8{&N?7yhnL}_a9N@a3JlAZ3bf8y>(4^!RxB`f9flS>QXU>F68HO&APtrJ~jtj?XvQJo=2Xaou@kB5FRRA zEYV1_2Wo@tz7F7zv(VDi2>ga}2Z%yI(vxevZfz zY`w>rD2N$u4v~OnAZuksLBC|JWMM&TcR^cOJf%{deWn_7-fK?%^Z5(q7a13{;(0Qc zMg&!ZgJUtpJC;qoNPT(vGVfK@Uqb~xZ{NPXj6~L(`)0mm7QI#<^i3l}qp!^Br>aJe zzKzXu;!|t{KzVgs->@ZoNP1ZL-o5J(Ah-t9y}4PP=k<6#fkcyfpQ{Vs!Y1mxc$FmQIk;A;Xv(o=N0Q-5F*UF0>N8HPCZ#iq zzi->sdGIPz%hf9^pZ@HYI1cqLsD9=oRUr*`*t^Z|AWEs7NZY*@kaD}bWe9hubh}~t z1^42?J2mDLjjLgUWWJx;Wp`Hn%ue{V{~P>d;^f8H|4=U2{Vy%*#c^j%LXkszyw|)z zjA^R?e6m(cPbD4QTHwNQjhnbWsz}DiNBUs3=aoHnB-jG?>o`Gq0-tFVTW3Gs7}K+k zx^!O^m7e`cty;fayE39KsIACx3clwZ7Nd?%zBjG*amdILAOq-za_U%{V)t%VA-j#P zv|({4j+G{&Yxo`gJ51QdreL1gHB^y!RNd>RM_(sA*~ZjO_VMx?$DG?R7LWx9XsMi;M+s}~Ylpu3q~ zFW8rHI4#q^IwwD$WVhMgnDPmKK(5* zUN=_<)&f!ZpwfpRPiVA7f!jXc>V(f-0@3l~hatRKlEsT2R}>n-nUl`FQ3r&<2*izqJp%Z8!kh?T>vuOXL=G(fX~Tr(wfQH5;SPt4 z|Jyb_c%_XS(8vOG3NzuiDZ&ynEQqI5SwN5(@FE1^Il*fa96EFfPB4=6$>0$UI}57t zULwRl=z--C5H(+CJZ?v)6w%YsK|Vx{dNLB!EHIy+B2dAwF{oxnaD66N)q$ew&5*G; zOAxfbZxeUJqvjECw%j$A>j^&${TL7V1x=>D1q?2f<-HW6Co4f+=I<6%ZDkj z-2p-pncYEMfVXK%qMw~lg(MMzuL882?WX@H;Hy)ks+xgz|boDbZL16IZUUb$%hf)Ly^M;!=sVTq%)&VM(Cdocr-AA#G) zukC(4?(O3Hg~Cp_7{iS7-+%6^rr`aQhday&Sei{Vft$eQRdjTt6vUh60`uPa_)s%; z!mQCh_j?_;%&y)i@3p|i=ZEaAE~qQ^dn;_Pt(3y<0q@ypwh46z(5d+LEj3yCkQCBR z@|8Ovxn%cn{_5^JK3L1PHz%QnM|v{98ngBRkHge=^seKa?j`a| z4c!PkAh)V*YFgLs1c%hoF={;rzbRMOVH0rRf#BFZKH7Mrfqy}KQ_q@Dw*q+`iMM+n4~U1IX~bs@VSXO8cw=P*ru(HedF4#uBqLT;s?l2 z>u|@WfU3|ac4B|I+8A~6=THoQ-%o?bM>7s}87nKqSS)6}-!>yn&%hwFJupR_7Igl$ zgVipBcac>0?t!V@xdQw_Y{(Va-uoz|(|XgyeHYXeL`v?X*cUqzC|gDjw|lJsW07H# zN&e)6!Pb7qGZYMgMS@d4d|fUb?KJ+^7)N43sLU?T)wjj32#X0=i^1>y_^VvE4rO~G zB9?kl>1?mIEY91D(cOFEZ4w#hA$G0unxA^XL>CdgIuf#JSpY zDc51?us;ME$2UpI1>GF%ZwV&^^QK&Yw89dpT77yx72f;K{JvRY!RUcZ`8c0_pM+`) zEe5P`u%22Qj9Nav+jF4BFJm+V=PwSA12mb=O?t4rYjys+6V5?+7Wd0$QV znAx4Zf=^fdZ>s&XcdwWZeq{VQpUA@VTZDcqC}?l3re!-?zu^8@T2@N@=6vl}qRike ziXFe-PA540%9M|mR=&Zhl}%b_E>cU?zsrJ%UJxS)P5c(;0~ zZtn#-nQ9_WzX>PZ^V2#iWTEF2&Y$lO5D5xKD)KPd@FcwvAwrkTPh0lX&Zi&$bn;kF z3+Qos^R=gyS-*AH>7oPgMJM{$tO$PTI>{`daxhcx|6V7WoueL(k(G@YylkX(U6f>? zo_eS4vJ17HdjPZUJHxOnIqsQ1aeP0bQ!zw$IO)EYB(faIO|spc-xIB0o5IUPjc*+} z347a}o;<~7LT94;R^(z8%BN0NmPlxO{qChI`=+VSrdCJNjxnR7tfr0Qu4&>{Ns9#yP3~%%>4{04f>9G`l5qw zKmaeF3H@^k4C`sZn@u;-LHl&L-4YBDLS8=RM~6QNVq?gV0q!ohA7bun8yls~0qfCU zZ}#puRRtp?z4=)12kmyLl6^a3Wl>RLIqHiq@F%w1#Kg=oQ*HZc(QI<*4xTKNP3-l$ z%4VeOa_J!wnAfdIHxK=1Ltd~72Y6q5cH|{u(?vf8O7;vT*U}5#YYnfe(YK#hh_*QJ zT37@G;tmh9(Uiu#4t(_qZcurRTM5maz{IF;Qs zK?%oYVg&Zc?@tKLM`5?euI0O8O%@L;u2Djs1lZfNK}w!vm1FITWM=I8i&7zpMyEWj zyv?6iHxHHyLV4D^5g!Z%4ZAWhEG~^=pT=s8KO&KWo~uitU0vkY5||4uGQvm9Sw?d@ zIYCXmbj@*9Q}KTvo*C!JPLp}Pdu$nbfqi`~k~4r;EkbcW?hz@kGCBE5b zjr5#!8Y85*Twh|!47F}6+Es@i2Us)4)C;~7@Ltp8QEp!;t~to;xurl$^cjNiGcR>AtWIIaNA}}sp#2TRw=gf zzPb`MJ2*J_8@QnZ=;z$PJT!h6k~6G8IOhu{u(b5_B$;o_+|SX{_BGxu82SVL0SCbE z#({iT$c+2EC2Wp7Yzn*uj!sfr)m2q(AQ%z?#kxwiPKbXjn^ZV76Cc6-TLypnW56p@ zfp+La%77>OF?+-&PH~#}HM1BwfIi_-Y746d72-tQ{E@ zRj%~}9^Cl=60K%%z@&~-LAn;e8agK|LMmv}YI~d}CW2Ov+|bal@dS3Fe#3c9|LRrV zOrGA25KQpo8hI!%^QIeC}KX_N~UrDJcT}J9Lzow8VFtwWll-Mbeq{7xtX9u~HH?K4AO2ABL zqLkbDY^Bd3maEYO~lYYSh2CtXrBFU3jnOV|Y%HoV;#m$RV|t zn9EL#d4z75Qqhz^hZoivzeM4&-08lLX)271jJ(j@-Uc^JhUJ1aG_H_P9x!DDy0cqb zJE7!KZ+l&k@h|)?hjSyBa1n1bzj>2)107)d=B-{{C!kZmcy$d7!k3obxnxX;GV{;% zOob&{$l8mfH#FGunfW6$Wot>9qc@#)yKd6@&+f0*J9|@FTZ^oW#$~F>?dmZqe%y1A zh)*+nnV08vu+@25&yY9zU9E2!<4imLMVeGevc9dhHh(y!Q7qwUx1>o2OHw4zk}=Ou zLE?aM_1gJx>ZxZl?a|co%3G9 zoaBzv3uhFo{nBcbNC}|s;3tL-Bu|49)}%+I09Yit=b%$UtZ|W z$nETS@(?9UWmsGhgr{foVms(lo)_|7p-;UPkNM*VAxqpgJtr6K?Al6-wzfVC+xEB| z>`f?|p^tVGg#r(h;N3c~AnoOKqhWXWtn~imrSh4B)oO8$x}Kyv^d{;{$;y z2-x|qYk(CwN+yxhXf&Gs_3OyUJgnrcTbAHm8W|I#WNgg*XJ!W0?l+ZUVH*pc69Mwj z7eGV7LVa_4dv3^O9xh_wj4hp~S9Cx0!@^zVqzLOLG%Id0# zsiC#?6$rSC27$`mY>}z!Pr*k*2-yGAq@*SdEe3FjKGtjN>4kz>!DXolU*%&^q`en+ zVxF9uN-4a`#YId+L1k`SolfvreI^|5j@!_ zpojvR1w)Ua&OiL}AfzBJ(J)#jK*Vj&Q8tFHvzbDW z)lM5LEIbB3iIlwq3%YxKpq))O$5TF0K;sqjg|Xc;_EsI$T>N&f6Z7%o0J6H6$giu^p=WGXjpA? zPY?OnSi-^Xh=Ync-)V$to!$%WEb-}+!!xd)Z<8xi@%#9m-vuS-5s0|B^!MULyf0p` zUP>%*n^Tt+cWI)`);Wb3{W43g6ifZpqxYV;(fpLN?eja&x>GpBd7Z-)Lg4kMw`O(7 zPJbOOd(J2&eh!WHzQ!9ry)qW4!p&8_Bw~YUQtwJV%2Ua%+kpH{D$gIK5!~Wwtl0>H&#}vs%JD^fO_yo zMUikdAhsc_pg_!d>T_!hn;|WO4+Y|bo_(qBW-ulD$|(+x0jG%1pXnlF(4USch`tUD zF@uL3m-Hl?LM9?Vzm;(1UcFjEAQ%Mct^WRqj*Lp7F*MvBH-NS_Y9Awa3N9}aN!pb< z#wHU`_pL)UR4q}p)OqR(9mB{7a#80jEQ3Uj9pk3@EAFN-&2PCm?%sPUW;}XZjaE35 zqM=UeE*t@Jjg$;Dw(_2|WRBQhd=U&q!sPON!81-IPFiaSpxWH=^ z*2q*r>mk*XgMLeCl=ltk7twOa9u0vPAB(41C;2x@ZR}r);H*E4gYMtG<`c~K!o@^q*{o)Od?nqz-Vrk_F&j&SG{SiT!l|j>yvFK z{yop>-QDl4ca7@ay9-`yLHP`R&zwl)eO(Jt)m{Vp8#;<&pQp5F^J_x=6Y zn`6kB)@TgnNL313^NQYHmTRON>BT7AlP?+*?sRxNdYEw&v4qOCISIO^XuyxB7MY#P zew;e)dKU3$-6r~H38hegdG_3z^2z7CH_%FYn%$NhrJZRL!=@juI?PN!iRdI-!h zid=A#o1MEjOC=~MMRxj*UoGq6qIG0cJuF0$zaGB8I_dhr=;6bt&vM1Wg+;9!+cz6< zX$C7F@YMackh%-;rwk(@u+{pDUu zLQRovWFaPAD%q!D>-Yvm%gfrrZ}wW->$6{rOvD~mJmna)=RY}Ffx(Wb)?Q!?HAn5$MDZ;Rn60^*SXzu7&kaTOd~3f~_7-7NcFQkaa0C`RkXP>pX{uvx;p zVGfFnV1M?48L^}$1^$d7gx{^>_f+fuGe^8Q}BWZ(OJ<8?X+}NOWKUW!a#hEvq zyg}D(Z>(^UhS|vThf3Ws6zs%|s%zN`h2nGQ-%j$eRZ~^7!ymW~OfY``auM{3WWF*zP z`jPrFeH1M_dkjcR%*>%LfUw6inWd}DDv&@6gUeN59~&!_vgFOp*-0(0aM{~}XY#`; zM<(b7Tp`g!Pfu^*fnmXrq{kd3_{!3e zcO6K^Ffepz+%H;(?DZFMmzBR^F!gw~hs`*vd*VkkUlLQf)bg z$qPm>aD57c1hC)WF6w_uw`11baiWR}7|>Q&UE@qhmLMbV!zyI;{f6}bNl5o>{!oMf zll>S-BltQ#-k|aOC3FBzk*$xrlEo;*#l<5b@i~RCoeHDqBiQc_sax}WHH{7IS5&vYl`3L*3Ui_j$ zrr&zPMfzdFT$Wd)PlRA)cKImI#92|rVic=%6W~>xRtcEchO%EB8J~}zD&@wQP z+c?1ki0M0*MAq2+hsB}W8x3b&&V%JwXDGfUM<4HFmiN!NxJWrp?2*A4>R>~?ZEb7# z7Iw| z9`&ZH)6orn?d&WbV>BqL%R83#cI2${_8QKHtUmI8NE8eteX|@=dyFqa(zF#bQS;NF}(|Tz+#MOKG;>zIx>);Klw<2P}SR`sDDIxyB za7xN}sk8KpG)~-1Lt4B}=m9wQ9~~0I;f^3C0Ij&jDcalLIO5{#k}JE9AFPJA@Sf)^ zFd#rnvtl zfggJ%HQRSP7%@3IPts!-R5Tb{*^Dxf}{um zvm3(NU$$?#a;oi*q0!{nqspm0(qD7??BMr;>0B?{u8gM)G|DLfDIAGAC1^W zwkiZ}cl%0XJ-@SU(tOpF%YNMVF$nTgbV86PGju&^^=B%Lqvq3>qwrXl3N*H(ymzlP zN^q|wPdf!P&lofggU7KA@*#$Gyys7^V>Cix~TH+^Y9pc76nE_+Q!HrTcJ_pyU; zeH(dh)gfZO#Z*{kl!$1LRqq+)nb9c1KYAl-xyDDRV|apJXZHYzN({TBAqfdMinHKC zbO8QQ+D1lpnLh~UM`T`kS$TO20HwQIQ1hVLQ&JFKV{lViMn)J6TN+@${Jn@ZxwN#j z`nyKIZ62^}ehJ>Nr$JGp06r9?N~(%9U%=R6F`Ji{clY-7|ADqEFe*1R>`3|j?C!4A z5>-^Bp@)$T;DQEV7~g|$WS@MgwFnb6yL2Q00ftmTCDJ|~t|dN4C#T*1%1X#VGJNhc*eFkI^>}5^lnNJBWBMD3t^_W3KLj z!^{|U;W{@RxiBor^O-j=_Vo0S41llQCdl?S4>@GC!1N8&D8^u6#wcwVr^5aImIn{4 zI$Q755+Kz|xlFA{IN2W!UTbOY0PKSR3jpkT*@N(sl|^7rtf;s^Bd%{`^b+=JMJ-!Sx^B1>6R^^Log@Klx5UgQ;pt(a zl4pK#sFRAyWTAzCm_J_^XI)f0P>YZwd;1_93v)Xb=_Go^;}ob0y}Ya1AhT-Oskb|&1Ym8*dq-}z1|J;E z^u1A~iZzjtV1Zn&iwc%4^|dVGrs4%_^+W8oYDG#E7~s|Mn~bBk?^ESS)r>uw&Yk&9 zyYi_s$}Hu@l1P&L*t;(teaPM~q9q-LhpH~j?|8fIf~8}EGTO7g z>Ju@Jrax~YOpg>JLM!e0z!^n@5@w8xuU~@u?%EyW!p&_rbN~Fc@`OsVDh-`f^xJSv z+x4$;>+x@VYU#W#y6641htr5_#!+@Jh&>iB+dy@7B)4i2LxQ-j?EHZ{*K#U?ayz85 z@zYN~^uen3>r@UBQi&F;9q~-9XPF)WkH=X;JpD!l={Qk+7-a4VFVz5Rqhn^Cu4Q{b z?q>Hl8OdsXwtGewwFdJ)({@!EhtY*ihrT;q=x^*1+RI^GTH(r?pT;?096~8_~_60i803-QIQD3op9U z4Ga2URDRT)JM$|WH7zT6j_h1x9!(pc(}C=@pT*y6R~MC*b$P``5Ane+HRuZZ>BV)P zOYS;`S+g#JSreQbd?uS(<(p`g8{fw)mU^@IoQL0^bUd?}w|jX(E_-1wgZ*ycZj7?W zwXTLHVVl{GsJdRRI7Ae~zMVHa4Gng$a-_uYePv;nI@W!pI3WeVX4g6OmVHm(nh5m1 zqpjHEP!0%FJJ|h^2E(A+;nsOLGfsn!X&ucI7rC*HjQwhXJD=S-(QpWeN|%FGQQy-` z^b}=2>wM}$Zawcs^C_Ue`XIo@;M2K)jZnyJ`x!6cwFsw*C;&aq?uR%WweNZ(EzkPB zbtPqUQqYIQ?tg!E@xmjAi7(r=1{8rui%;W6#wZxVr{k3C7W+{r|HZIK+yM406QJR< zO6!J4d%NGI)+zh41x>Gg9V*CKzi8n*EoASojEVR1e#a%g=k}U~;YmedK><~jOGZrI zHBd~spn|>}%I(m=x`L#?r-1*yx67BwzAUQ!Hgi4kH{vsGG@Y69tA*H4D5{HfI zu#k`tb2Ow9@C^$kMn?Y7f#U-l9yY)WU^+iHoiIvexl0R3 zDb8q>6AM^Rgn|3sLrx$c=cW#WJ+N@|C353WmMNwTp040kCPY35kQ56@#Uxyg``Kl_AFgd&n=+(##6pT|9U083aSJ zQ07r0AYb`mqrVuqzV!ImFEl0o_umU}t>l!Fns{Wa+kTdCG5Fw=gcWIk9IlfLfL|eS zAxb}H-6<56;PTU)heyGVpAp>O{0{*#2PiwD4&aHS&FHV}6*8{xj-KmK0`$$)*2E(&2VM%dkPAT84~6 zAY!j-v`4~O@9{cX-9w=r;9V}RZ!Lvi=IVkQStqKpm-`2Dyqvs`#jkL3-VsboOeC!| z4l?wrtuvkho;-yQR(VOygKJR&X>YV5x944$-gFx+(3hVg0FGX|U6=WwF@NTB(bMt8+%P*NJ7odSaMoZCxvsf`WJG=fGxv%! zc3TH!?(DzlU{j2GUG7KX_l4Oyew#q;xk)YcQV?p^P6y&|JpY|U)7`2c=%ULb>+~4H5l8Q zRFAz$X>~o(awLLk_iQNf>!3Q9@@=t7iz&NbS(&bzBESdi@yk@!`@?z485U`%K-=U79m3 z7)<8NTT4`ozq1n|t*rrvSmV2zSdghTIaR5qT<@A`DH9e}+1)gRZaR2exOThdmVp6R zq~ct{nP)y|rqA`^iJe1|{)44#A=lO3?KHo+)C6yiJtJ%tO z?wTz9SiS>oSg$OL2ZQ$TdbMD~ubrTo7iSuG`wa{f0*~wlI_o{ybw|<&$Q)13bij-p zp6MmKOfuRAUNi7;R?_U~@6QyK``7K=(CF*emO6uU1t>< zAu$=z*3N=h#*!p;J@c;<{H)Hh%7T@nV*y%1Z+Xi(Q(1ZU2#V8;6rRA-r0Rl`q= zxIPX?zJ5KZmYba|Cba>eAT6!7tvl3Qm@rti!XkeVLnBXsvB0#~{0{hr_KASs#dp|S zEd_Ze9Rq`pYp9~)gYwGCA+?kvCxTzfKpQ2yQkP)R#heLF9^mM)Oz|% z9E?c|<_5;bpA)@3>Lo}{xc~hk?;h{lLVBbCm4t(=N2k-2{~7WyW&~(!HkJxbbtOai z4beSR zNfyhv!};~&Ju0WKV9H=Q97t$-s$LfrUF70oCV8Z(#w%=GTvX*@(uUTVs4@#Pp8Az@ z*eyX?{aF^)BpNGJv%eLY_((j%TE&Ka`+0xdvDP-d4>RJuT4z%+S>7xDR!vjzuA^hj z&)W|6bfXpdncdyr!Wu8dMJ7jQi><5{2?cnol+T7!ZVtg4B7vEIf5L@o+49FZj-B=O zA;+odY4*q%=h5(hy(i*QG>EP=7IG=6`~SY6FG$?WwDRHusfwZfl|P{ZHihlKrh{F# zss^$LVp7l}UFP0~wvrN`w>$Dp>7KlkxE0%A@bV^#G1%preFD$Cc8C5b4sADS9yEZH zCKIRb=7`|Ga*kZ+sZFd+=UDythy8`{?b#vB_J^aHhKcuus+q^8CfD1WU4U$iVXu0> zy-s^Dzj<||8tbvy)8}`voIVz5K1!;f-QqKrWFas!Sv8Do|E$mK=+8^9-%5qepbBVU zI0>kBnvjOJ+W8bqOzx!K%xc=K7`dn{ok05>C)N`y$%hM?EZdoQ>AU|u-q&NDwISfo z{e~Kx<<2YTg=2-R{>Kw2%}a7q2Yxwa=6ys5+r>tVjoids1xFalK~<_7PK8q|Nu>rBTxBZ_5`E&)Dw<(ca2;NEs>up#>I``{BP4 z4hO4@H3~yygw34Dr(cs(e!kE<``dYxpss3AhJWar1DCkdgF{KOp!azE_!71FxJI?U z*zH_T1NU0hb!Hi%U!JJaw0oE)_Jxs>!}Y#;YZt-3dW~06H}TAasmm)LIIsFj;DROG zscakyf4#fCb=!68ufWFE7l)<~v=sC^;dg%R_#O3(w#U-Eh>Z=3-}dz% zZ$|a`EpJ)RV5|VmUX+!RQtOt*0Ph)+E@S3p=dm^)sz5|9Ac~IondazJ zaMPh_kM7*BQrVj#9sCSrgyO_YmLmASHVAs5{hRqJk+Q|ZzJ>J{%ok=}xdnS~RMASv zU7qr2aB5|ow~b&ps`*p4rm-+%Pn91TOSVo5K_5Wmzm#u5*VWxFG^?i{DP}@GfB%2i zQ+S1qt$yU$TRc1@E0^|NQg-`F0un+G;tN8Rv0htoEUBwYJ_W3Y(VKz-UYp+ZXfPol2ojzq zf&BV6I6gLm8J8sIRa!4*U>y7x>*~T|)GLko!80Qq%n2WX8yt}QLdFpZ2^S#)=@sF7 zfg&DOUhWYPPH5=|i-C*S1fw1+$mYl^DiV9Y28RbMc9DSg7``eYj@eiczCTdZE5jaF zb46+@YvIE(VuBBw$ko2a>k<+S-~`JCv8$9hx1^-3z}$xsykDCiwGy8FR%lq*^v|mq zp^6{no<0PMhhPYYi+KQ#t@aEL(`!m^642=90-C_vfMMCcItsR4Eum*G>}=UPI1pf$ zr=ra8|AVqXeUSDvm2=$vS4!)AMA-|`KkvT^#wK%=vLTdW#RD0--e0)tyd+D*_NbnboMYo4evXg{KUnIQ5KO!?Kte=?{8+tKu2j^TBN3Zb*X4SA>KS0d z!#wvnUAupL(-a7R1Wu@B>x zYsQf--tM8AJPNuov zeU@lDFWxN5c4Q%*y8H{hWt+*}iBIT>9w)W-T?$yHD_QGdyoKsD8_H^m%7}a-a&fps zb0E91!4Z_NQ_1hMbXNG$Gr#?iqY;sC3#S_hB@BzQ`>Ryy)f2{7Zc4 zjIltX75L3S>HP}aF(rvPPn{uDv8>|*C-?=tm-Z7WEGptZ#|HnXR0M@|8TiV;B{pA| zSo!b2^MrOC6mAi@xwaR?pfjhfjH$1e0`K-H0J`sH*)6m?LB5fzx3^NFz#L8bJw-NK zaA^F|(eVwkp*{Tom5+>$ZUuJ@eg;Nu7I5##D<0

IbK8g+9X5(P}%Gq0BRnCkqk_ z#_nmd|`~eKc!bJ|g!NvmrZ!J?MdV3J8hxj_% zna$Jr|Iw+SbP3#Sm6W{2)Kp*VK6`I!9%L$jwjY*F7S)eQ5X~Ih`uiuD*ST11Bj;ly zH-z#E0`_wz8mggQCizK}v#TfQ4}R7%(mG|!acsbk+(^|j9rd6(G+f%>G-0wna7PQg zHe!CBe)(kjw{ujT{h2QUbiYTJo=mD2$Tekq8@!P)#^^}RtUlTvP;>h;PltGxP{}xe z7yqQcDV}|A`AbCb*vCTh(>r|_&7x0SA%`|iGL#zTIs4s_FU{TrAMy08cOVdh`aj}I zuU-hTjYwuQu_A4%d*(6G?`6ke5x{ri$ol9>U=kDV%dX_4jt|AG}XnkhB>(TUF-M^+B%u|h;PfY8!LYtD*{h? z3Pv|^c25U)g>U zj2FiG$Y{y6B$b+@TNU@GAbd_Q3TyHG6yh}Tqd823{1Y!9CxHr~#@^&FR_*$46MlLw zhEcRU8v5SkpBpR39&Od`u4QC|S{o_RY#A!B_TBDoX1vu;qoJYmcUIqOe@(psk7N3a zFT;#Zm}J(~O-qmcx2&ypU2p`(i{{MjeYaneyNbFxI53XW-;7I&E$34kEn| zeecn|AXntc6`NbKciIcKalw?PKQQa$vp_Q-u?~EQ4_|bi*LomBN=jtdgKJm_s-TVO z;EjEk`k6fqwjj3A#`^NHUChr^2NfwP>`Y&0XM^~Lw|BP-FZQ_hU0NkZ|H?^$$omrG_JTY2;qjMx);)%)>voVUk0LP(Fl4p+^@9Jd|IY9XoJJ)f{h zLV>BPc2#`*ucLv#L^`^SUq7`FRyrE}SgH|SaFP6p-Zj^Pp3)0yOQsD@-xt=n- zm3C4*M>dYTVzQJ9b2g!RIF5Ze1BGN@){O;`)IsArt>M7?2T3pB*0r~SoFs|Yh5TJ%VKM~_S*v|k_GsYF1Y50f=MgfW2$T8~TU!?Oc-8Fpx4mv;_`qp4Wd#Am zWpLzy0IWAJ?x)H&j%rSS4Ai8Lya#%2FTZYq(GJo*IQo7ryI(kIkz~IX17qQ*NcnxdoFE*W! z_7cdbQ)J*EiYlrqzvWgv@_YTt6{^m?%`ykCAi@&@VvQjsl3R&J76QG2Q? z>@uC`=OZ@z$(upbOC!PSCKJ^*W(GGXJ5(HtcKX?yHOQFeMjWRur&G2ZnlaURw$(~g zrnA|5J65xqq`kH(i1)Hn)-(Zz=^U#}m)xFxhxO?8eb)2)rZ@dfCGKTwv2-Vn8!`R8 zr?EtSY}1s{YQROAG(iCftwEyCs;R7Wy(G)qZq;jF_x3gJ7Ua|1=A1mG86?bJrE!EY zNjFQK9&2U7=6ba!CsJR!n$3uC-HG5n*!Na0(dI`o+&3ut8Y{^Dt?2caaFmO?p*qsx zY1A^Yag~L!lnUpU^9n_S2%$+qht64lJ5nE(im(9|!NuV;A0LL-?>UBfcF}vwq_+Rm zC?qn>uRb)T`35XX;y_QBKc) zUx83+22rFNE|BQi(Q*(d@3)`W>iV=xKJ^J;>x)~nX++m(KW8wPd;yws6X_Clo0Cc3 z+^PnWQ_n^$I3hth{ihnkeDtLkfMg%cmFfER{}70&da0Srh!?`hU>8cJKLLiF64_ZL zr%XID+$ENAYiskA+pw1Orjuna4y0w#@Mr2z9g)XK{NXVQsHnE+0ioa%!E1bl>l2Cv zXL?e9w!`)v0s(CbfK07Bi&Gna;~S?}r;Y+@b4Jl_MfxTMjw7>#)}}I|2RQ=$?!ONn zyGrB1{E9pSJFTKZykSBAo#J7I%svInKgddwUkVibOGwu~;&X^|WVizk;>gfvir1bg zatj@y^thiDJtPY#ki)$UE2X92JV(Y?uagHBLd3pfucobe+sHUf^ll8+803E)B0uA- zcRjNyMdDK4gO{i8l!jcale+(_Tbo$<{zPMq%0Bx`#TTGgfGeIx2*OH%Tyj~0A`pUQ zr7ixo&oW;LtuvVYn$2is>%4?TZt0Va7Wx>%62>+t6BaNiIj|g!05>r%D9*5YpaqKa zr$3*Fq{;-)L!22v3uX|x9tIH|)MzzD{^3doLrF==51&7){%f7@$MgKZ6&f}a7QD!l zQc@h6pMri+gDZ{NyPmlC zQJUV|{Co=E^1?!-WlMM_5|1nWBC_)IEmW}u^ULu+T@N-aQ{|B81y5yVW%VSHGntwE z>q1TZgxv^m>K?6j;Q+K?Cno`*kaiBFB5VbM6VV}#a_RzjGep8UaYrDLMICm&zQW$! z@0FFw&F6%swE(bX%b)*Ud06et2DW6LhayRo5Hxjx&=|AJ_&*dumQFak!E&G{Yq&N{ZC&B zRNA?yTb@*eH&kEa=gm8Rb?iEiawT#D{c5L=MSZfW4B%R8nbx?}mA6Li#G^4WeAy|?$D!HA);am3{0ZN$;~op*2F zE@!V#_P#DF6UH}9^b}`0se4uSKe+{h1nHxvWa5r0rx1mvezcv5%1a4GwOb$8%CoXg zd(0c}O%2q$&EGDqY^?XPD=gl#>G4y|?iV%5kqHkevv0no<=&3tEpbZOU09lbHb0`# zTEdYk-!aYi^ub9cF_Dn3x28I7;Lge+=2oL!mB28`tY$GPG{tVW7u8T-|LFTksrbtg zIybf9>`Gjn9-~p>u+K`hSyn^$=ZX3V3J|d69nJ)%`_vAu)HqAk3S1;@vOh?#@+oio za*?Ka2-CcB8}xq%(~H<^U3?PndFakQTILQ@;d-Gm>FQim@HT5L)+2xTdkp8h$baR{ zQPQT4TGVA~(x_*!ZtU2OwIcaw*2E}k<1bPfl!x|T%t3mPe&S1sldzI{`{l^t^(l)C zsGUSMtzIbr2m?)}1e3l%N$&w?*Wo@HS(wj$+Sq^C^N7O2RxRG;>gOLdkf?_3_le_G zO;ol<=3>-A@d&u>&%8g}(`pR*SSi5X^DXhl78;&;A#-+G4GtjXKQO4hR6BPr3VO{Pm%b)m^u$|EdMs{-y&p_ z?5!jjDO+Y%Mn?8dWMpJ-LP%DU9YV7ACVN$6Wy?yokiEU<<^R0z@j8y@IQ~yVxBI@X z^E!X$`Tc&rM*Z`3H9FH*=SV^}M)wxH_;tsVx%n4YhVF*Z)l-7c1M_Eo{;L90M7cUhyOR6&p><#p-|qUo(p*J-6M zd8d={5e)>^MaS>=hAa2e7V@)V|DMBuT~1N|H3(({!8yhdy*z!7Er#lYKvd|}n-2f> zCbBlSx=t@NKkrYrtD^uyUQ zF^A;wt=?0q=kfHFcWwB(YUQY)MneOO%Sc+L|FcVA`z|O|X#l^2Hdhgy<$QXFxO4ZW|?*LQSc!qvJDe8MC! zk(j9BdX#cMr%U0{qYjuiGLV#@Q6vAJvk#0CJeHy6Nu;TWGg*3@`@jIj1&-|80CB>j z5g_8`=Ee;8hAu84frglvf$sbSDD#_JTczTkz|X777E=uZg0gvief>L-GYZ{vT<;_3 z^OhRG-)Ij3w`aq)b_8hDxgn|=QB6P`RFM8uUrvE$ zUORM+Y2`?t!NmJ}}F{mXIBce(7C#g@w?g3&V3)dFm><5J>{a^V4CqN#tE_ic~np7p8ctKN%3(5NPe*v%p3 z5-UNikgqNuibAr2&4jJUc#Za42%&5SLm5q+NvS<^4v|K?v&!wE7n{X_zILuVN#oa` zheJcnU*@fKFUW7-#C90WE6S}V3)Z@R356kv1}A+|Ci~-t^Bk*hweOdU6hfk$mpwXN zgUhOt$99F?Co&cyw~du*y4Ma-GdrJo)($^C(Z4Eeek-=X4I7$Mj$HHo&PA1Yok9gH z88_q2wG{02#Vh7^!r0P56LWo4R|Jj%o=`6e*Bl9cWX+R|<8z&?K3LjEv$URkKd%8D zrFPu92n+V3lpiA-e=LKip8CHn=emkgd8-=ndriI?ONj5DxrlT!&T}KtFzSbawO}4QJdVMox=`taq9*o?6ebFi4->9GI*;uR<+HPFkxTX-hCTnQeTA6Ua1~-;{IUwzU z_w4F(yB*g~>CnC3S*e_qe_VNBXkU0a{_4uI;oZsaxh-?j$|rHW-aYyRUj zxz92Qis2+RRA#;>kE=FipD7Gj_{nEt#>j{FFEgRdFkNfjjr8RlW^#iP`Gh%0`zvPb zK2wO?OBM55lF6&S>iKHfQ3eHrr?;c2r^&PD^*>2sjO#6*3CfN73Gpg15FxIS;;JfH zh`OIDhlyZDVIcu@Xr`T{7q=W?HdG3R8#gYqX`FUJKol7eb33jo-+wau6<)EBn6R)g z9=#|H=&K;88WZvF?yhKLn1N20`0CXF*zaV3xdn0R7#lMPr(92e2IED-o60GJnX4-+ z$QX5JeR9HI6)B0t37sP&*L@%s%7zz`Lo!o&Vxv;?9ZBPPWL1{)Dyg+o?{S1UcWJG}Ili@G2 zlQKrxgX|W}x5Dtiz`s~W0S*`&8++-X33&#nQB)`wDU)>Odw+Io+G8l{^YD1{Es80x3yJsUng?_{9aU|q6_#5AIk3o%r91c7*ON+B48&jf%T#DvJvQN|;M&so{6vmt%BRgcus9NJ8a z6Mo*sp3)x$y_KJb&T1aSjgatW-reqsU5=};zhh@)d!?wbXLDucmvxPvR+m%isYk6O z>wuk}riL0;X8xJ53@fSe_QcJ^g0j+n?zm{RuyfH;rMPwOHu8H@R!0W!R)+HxWLuq= z-8(&qs*LriZd*RkiP48qh=d_c>8a0%-w4~M?Z=68&y7b;Zm*#-xQ_7kJ4BjSE@8^d=JAzTb`cL=&XOyOrZND<8WIYiZk?Py{fx`c}-Qx<>qHzm4hOM9HCc>Cp`U zR)~(^XewW*chppC10BUMp1+731*Ses(5XB*J>sFcAD?;K^Ef)G8<6)K;8qYo{>BTI};p?dBUfW9Bj%MhvYF+U#4GPEeR+wO_Xlq-jut0wbOMVI=CsxSG>M{QyHOK;c zZu`A8oe#pU1a}jbBe=Z1yof+J0NXEuZdv5T;=^kKIT91d3*eY%TnpQLz$*ka2Vgy5 zwKljp_xH!x7$wB3w*a#oF&4r@c9N$?9~w3n24-e-K)v}@?4C}Pa7|Lfv=WeUeo*~a z|K?S}{MQ(?re1pzFbJlH;J}urks?8Wzrct5EsvjwKLY_AQ1eIhGC*IMZ2<20?j(Mw z$(!i8xm1{w5WwdT(gZlbb5PI{Q!+y%3XcLWND}erv{6*V^Y`~Bq8Qx~e=C)&@$g}Y z;rNdqC(SMwBhT#Z<%tn^e@)R)ckh0O;`lNk;M3S4U^BXN8_kuqOrL3@P!5iHiMJ+s z;F$%W)oaVk%e7yGH&n+TgWAMn8G4L6&bi)Y+uuyt^lJybU>s7rBDy%Q@gSpL(k(Kq zsy}vOObx95ofXgLQ7;b;^lUVX8XLv#-o5Lz4D=kb5&M58++fJ#%d?i3FIjiA$W|zu zPjzRxR8;SRhbs2*raC*GRxPe1dpiH!)F1(j(s&dzlj-rop>K&!odHerdiiHv60ede*O<8AsK24Iz*D4NH?X!@gtnYb%rvF?JXFTY0xiH@zXb?fn0Ky@lFD4VTS z9n9GoAkn*Bwt~458QSsQp(5lRzB&%CkI!G$!gk)BZ~fk{xsY+IN#1az(%?b0O;3UF zcaL8sMzA<)=}=9dAh$DQlhL#C(of2}-zdXX&>v5CB`o!w){BE!%`hfn?PUHz*5z9&z! zb+#uP_%`ydlMuuD1^NZQgHEv^CLxb|h5ENHqhO%};3KtHL6s+UpSTUN5@q53=NZ&+ z?M;NuOE*n2EJ|s~IPqFr0-uc5Y_rjMJCXAw9)*TDuX2L?@TlVOuHBTiKoamjz46_@OfSm*Elo zP{|LQDlch|Y9G#;F?-C=t6V{I8++Y1qVy*C(poWV+m)hS=hk;AobfbEFYjnQ7m(LI zzeFT5ib+88TF`Mp{QAIj9Ae8tcvH|XwS9nMfNX4FrbPirshusjK`qLoy`cnIC5DFL zB6GwMLr?sB&$?d%!0}rt#Ac4xxpIjf{<*paaePY*jZ#u5B1+1fxE@I6UHy7dyR+|Y z@9&Q&Oe>jtLecsM$LU(vjR@eWlOes=+FyL(QH>mR0mzVUIN4HYYj5Wrv!o_geGGek zdqAWEg2x;d&Vh<7&r46hB;HqMu1tW3h>P~aAuBpxzlH?t9F*>x_FlehMGa=`#_qv4nUx-OXomK6%n=_8#O6tZWVQ|0Px?f;L& zwSeD^khS*dKq(4Qvrm^)TKzs7a8t$i;oKuxkzFdjPb#BZA<5lB^KlobQ(>3>>RXGSI_^UbDmN{gR}jMiwk_P17! zxkK40CHb)$TB7Cj(ol$N@{4n{tIoN5c3$diZHfK+rOZ6}mF`?>6kF(S*SHYxDM6`x zPDia+=MrtO0hY(p*_%(F+_;uh|LgOh2HV?cHB-&8j^i7sgWzR5UvNjv$Z^$gy{V1$ zhWI1hfpq7gT+;fTfy@POtg1lg)d8o}2M;dHnO8G~<3zow_UeF2IBp{tUgRvaBDA>h z!)kI7R9cL@a{`Udd6>XY`Aeypw6#;-ibWqb*j4A0(uCoM%?3C3{n$Oqj82()5pjjEvFqUc_P(|1k~}KR)s$AW{7pqmso;hBNcj{4#xAP|!cNa58CU zHqvi{tTB12N1f+p4!nc0zKg{_USqrrRmafE!7d!$J@?f9-(izrAmL@Kb7ro$p(3E;Ip=clf)M8uKLR4Yv&ooO_Sk|a#aHpY(GLGpb6c&+UJ-$OhFog{*}f>QKKvI|d5obWTK zlGY~c$Q^2a8SBFN7{b-An42#~5F-DA&% z<_c~*{d5PbMo)o!6iK)}8@yQje56}*#B=eU9<-@Y@i8zl6Ae0JIhGG*0xSfC$6yyR;?%@p8IywGwE7Zef^o;pTFLU ziK}p1&8`+*MMIhQ{9)!UAp)3-QcrZEZWv-?@>m%x{7d;;Bnz0QIw6;_BX+Dx^^ca5 zSQ|Am6!z(8`=-(;H9(2-7`CMY^)i=RzIF9{{9u<&edaiuLpF3eJ7}bmVsy^$LOBy? z?i2qj{jh?H<2UqL@n+sh>{UzxBacgL46g73m(T2 zv`$VG_0_-cg`Oybe&Q6mWm)RA)x0`-wEocITtG|Cx?THZ9s?uDlSk0#F;ydVky`wS z?PiM|Wv# z)+33Jb0S{$ZAQpDM9B%V&bja3g0>&O5aVJ$@}kMlpKx*eS53Q&z4UWuPyN=Q+^Dxw z+}_GMo|Z3GfCDk1vxKfS0ZLr6pl!@Xt6O*f^OrA{9--gq4WnK%hc_^vmmaHDA5e8% zLGcj`kfmZZ_`7XMV<>wO5{ynrv1RrUtHIJv<59ZI+jNjYiwPGc`q+O zREv%3HziGj*{ceqq}}IW9ETEzcDRls>tDqqS%XC$QU33y3(3@t2fr_c@$jzT;$y=E3f+-TwcE@3yMs>)|UL2Fzr|uICYzYkCmH;hc&qx%gafj2+KfJ zVjvUD&ydH&@^oXPgscSM#JMT-5jdZ256s&l?{8rndj93Tzti{Ng|joS&K}J0M!TKO|F6 zW#;9pGu+JbJ~s3?oIk7Wy_`fYRtpjmGk{!W7Ccm{D0+Zs1 zP-S^s;g&LNwffikiiE_Y_e=~ zD?*+V=QA7FsRcAPvqRr$JURSkhmXiQm7ygH)13z&S3 zw75U$=k7nWZgX1+y@I{6YUx1dSbZ?c|K?%HDtY4B8GF4>ptsQnyZ*Z}oEA&b9H623 zhE)VF4ZvvE*Xo9eGEznSk3GTvN$B({uQ7l#IY#B>?W{zx5hJ14a9DZZ?_vnHw2w4f z=!EMAaM*P+@v`W}k&`ZmWHrzbhJVU6TFnE_dELRi8YWxQIkcuSUF7 zB0Si{7##KSU#U*eG$~&@l`~!Vx&eeS$sB8nJlnC-O_w)vMlv##DeenzIcX`EEjK5} z#=@!Qi;h3UXoR%1B(lg)*UuQ=oLr2k-bd{e6@gSSDn_H<^w4o}>)*dl`@giBW%3HE zDjJ!0>b%t6ogIGuS8E%Nbt=@S&n$BQ+`gbWi?LF45ZBtO(!!@_4?@_bt5z09ZDPU=b}@aAjZLX5&NbO7PfwL zeDM^k&B0-h1wJ=Vem@0xtr+FMXcfFT{6$XZt?tb zPo>uP;NZX*gd&)I*o@7}u?P5>fw!iWpviGqScRCM&y|M5DzAs7(D z)6=v5zF`*$@njv?f4FVNd4LY0Ur$UGa?4V&F$@gf`S6Wgz@q52pq`ndY%14Lqp?9S1bq8(zD&h7rwK% zx!D9AJMvFy>*?t+wCq7^On>pO;^Fbc$0j9(A}(`5!LdtG{Q*ojZ(<>S9i)lB^P|1} zvO+9-bGB**18J#(D7+6?8W7{V3BYa%cCyuZkT)Q2Uv{pQ2kI8`Hr4S0HxAUJ;#@_a zKV!le;-w->nTooU=^GW+HmI`D5og8uLJ|}ip`4QG!y7k_0y^Mvk!ZdDA3tC}qH3Bh zoqzKbq6a#6mWAOs4Bnsr2=91wU+6j`=9?xa2nq`q>WlUOR5CO7*jHKKkh2r`zu;2J z`9WX^R*Gu?Ji9OXKx?&qpxXOgQ4waiox<5xoVL39*B@_IZSU`>kb``N&;THH_wlI8 z2|V|oNoHCdc)ojw10-Kg0GbG52VMe5C(y}4$pYbM!GHpkAiH)kIp7jVVq%oemB3A5 zJ>^B2A#2yNn3~!c+L*sP?jfw#;(IQPV{6M-yMxfR09H-n>?{ukZ-9tqrHa1%nPr#A zewFB>*b8rk2}Rt%ONe*_=IoDUM$9E8(O8tHFcTCryN)+}WfKg1ot$3U(hphrwX_8B z33SCu#~VaKP{~xoS--6PS5VG)p6!v4EktB0z;5Jc*7hBE?Lej#RVd1SJNK{Cm6@+{ zHenK>NB}^FmgKpoMWp>B*hbrajxj!^=FYz7;FHT4J)5N!nO@w#7il<%wa00*pR&D? z^z&_ucAu%3=izdu_Tie3X~*=yyb=h4st?AKK4fKpoBZqZ_u8kw8A&Mw4Z)MR=S33y zYI4NM9Z1f7w}qNMuGtt)>ff1(TCd$1Gb)^c4Rs%dps)J15tAQX!Kx`n8C|RV71a0r zKZFf?&#QCtHMrA&i^!v@l za$k1E!LqXK+Zc<3$$S`zm>95`XtFGC{klZUZ4vvA?+;T6(Y-TKb=M!C-{%x>jh)H~ zy|uiH2^Qq;1Fw9Kb@A1+_($R<3!=3i&TrPi%}lYqxNn>8v^@m?X&7oJvX_D2J&dXX z5&xi20M;DJqxTck5vZd4qh1SL#3leHxhK1cll=M!^Tj;=q9q0M%d$!HP#5`iM zyAlc!1gCUipft%!?U-xwV>g%zqZCn$ZPerD#$P16dGi*+(Azdc{3Fg7_S@1(iwR$H z5c=`)QFs{6#I)qzw-;@%zOfWq$ zfdR-1=b?@KiH;7;p`oE#0F6wzSRqi4L1Gv>BD;BTk$;^JN+t`HVgmg2Q>dXtAyQ(-H3#&o8 zXKHdX#zduvzx|w#gtYYO1`K2kS^-@dHH|#9n|y|3R8cM!sSY;FY5Q`tj!v z6NS4o79{-wl9K|})UO)4KB-adQQ;%^1M-j&IxYs5a7#~*Jb!PK-xR1n(kitL4$h84 z;Lp&&{e*fsI7ouvY}InGfep>d6_=4KpLY9vZ{op=Ihh-rMx2-Xc4|pNUcm#GN+#mh z>+*6n3m#qMJ_0k+Wq8)y!NlA2hpj$bIBaZxs(tD&J$xt)6k2*5=~D{UXclM7=*F^4 z?$zArN@%qg6Zv^k3GM9Fx~96id*!+e06PALRmSk#iEiib z@*8di=5d}NfE=zJZL-sdSob@B!TJ@blNKwjYAOedX9l1*KoLk-@@5MhAGI_^5r}+Zf>+!oi#m{=_ z9~)7>`lWE=Yol6YFa0Uwm4{e`%wGCLK%e}mW(9OD401v<2>?9*SR9(9Bs@1O{^sH8 z?MAjDvqpNTqcW~*n|J!cM=Q7Z2q2P&JNzj=M7c@x($ zI7aFf2op>{bjdzD;Xu8Nj^pr#Z(b4X z2h=CwujG6@E7=&Gk+{c ztZPK){6P>F1Q>t=pD`csf<35`5G}5(0`4@2TI8>$g0Q;=?r`RdY`BM~srK{1hhT#J z&QWtF`#*J3sNGfbpVKDDf4+*qewmjl0?OKKQ1UWl(2V@vQuXO7PW|B-r9OBPasY~| zNmo)-cj6FwPfx*fYUF_9c7A$q38?g~!N+N^+7%F1P}Rv12NdGodVR!gEvndCF>i_; z7x8vh7tJ(J0X1uHJ>fFM&{$lo$u(JjET6GWFLp+8azdlKNuIFq=X(3WKflpNFyDEyI$7x(@hrTb${>Ul{j}G$xBLVLd*1^zpChFn+crDt{WFq3@NEsCJ8uk^7C>W z(HW9ymOhv(JUA_`ky)!#w-`=(1jmK z|GqLuz7R)_LvRjw1tV?{#Uclmr7b|F0JSOt)CGEXAnzyrI4(N+D&pBv&5&W_;J^n2 zv`6nU)$Loint|E^Y(7;)ss&4d&42%vXI_VVRfFP)SQwCL`4(`kxZSr*K}vutgFFJI z2Jgd69%4wJrUi%`S6(c8Vas;MqyZ@$~ugytqPO8|WhZo9r?+R@Np6 zk5dvQUmeKAg@B!-CcGoq8E`|A1fnv4^VtKK;!o{Bh7XglFg3uuKqB`sF#Xf9G&~}~ zz+t;{fOy9KOOANz_Z9iJ_i#ZQ0tc}Y+esmHEv?n9E^wH6>{{JF{1fdJnFN1i`~xVR zKL|P&!>@vIZF#E*^kc9mc>)TIXM`^lq-Zl>{X)TQf^i{BhOFr(NQn-DMet^P<|Bk6 z+4l=;zUsnimyXp8gC7A0Q-TyPhoULe9Au$Dq>ZyBSO@Zr#4gWuT$bR37ePy#wBBjD zq+;|zaQphR?$MB)oumCmu&EbY4M~^%z8XTco>2eg%M55`z^R668qC459y1C4a$gVy zMi2((41GHvbMqXXF79BU$`R0~1R(6ydQa+c(XaDXfBvX1x1RuacoSqqEmxJ-`sewp z3V;2w-aOt=2Gp|1XuUAVP~nj_T(CS^%UeHo0FL2^VMLa}@(8bEw$<^@^Wh>^wp?@+ zDcLaeq3`Po?;)%&12}kZZ8<=2Vm$A;*cmserR5Ef?KvR_t^FzCr=@Gixkt+={JDJ? z9N}hR^^xYP&t*V0_1oiYCkyR$$hYPOZ~RUf5Z_IeJh?(ZK+@JB8u2i?a<(P#5=tB* zwHi+OVCUg|R1NL!7>M$hv+qm<7Spb*iGIZb1moH%4;}Bk!-p}4F;gALDbPOImC}Bg zWXr6pp_Uc5_t$dDG3$G}?)CL9jJ*U-=8|i3Y2;YOqjUA$xn<0P)nhXugfeqkBN39C znc`c`?APhL@9Jt=*#%}bP!L8M&;4EA`93}rce*^pTrto7EVb#} z(ko`b11TCB%I9>Ks^Rz%dP())#>(FfpNzRSD7bq9!r3aA_rOCF7?;Mi+GYl z!Umf0RArsgC>?oj%#iT@*6VX4vIwTD>SNCJ-~^Z2eyD@PRY?L14O#Z{Mc~jkLZQCka{~x!awI zUGoB3+QBP)XHki;-ZQna35V{E`TBJ!E@ViKOy6@JXR-fZL?VxokGT^9qouBR_pv8C5(Ga>LBw)f+3X)Xp z`UuMy`5S7FH#Ry(k;3cbTU}OG-_j!a)$}f|G(1vl5N#dcUbq?#m|#dom@` zp2s0CFE1h-fdvWsRSGyHPL3e<4ZErP2sdKBv$Jy!60adk?R6pq`B7kCt`Kg`vrcgggqFYEuC+7yju}AaZJ;5EI1~6?K7Zmd$@9t-zfHh^(rdf?ilK z-O436Y?hV=@ul|k1la_v59Y;DtEr{5fNuu~XxsZ&>pkmccSo?87@shvr+YaaUh)$! zG6GN~$CO7Grjd^zSv1S#*4|z^KEB^%XS*D8vrj@rJn7m~M9MY~=0u2T*02GGuWwKQ zO$-)-a!0;7Y!iZDW@u^IH5>za(#BIhTr7g_j(Bb~qR=F|yMvI%flRp0&txHo3SIAX zzd}aO#$17lyMh_e-8;AdsoM4LU3)Ca^Vp=_g%Axl?vZ7tRPR^Gjc#9LrX(yP4h z-z!FsD<uJM4!Zm%Jnm!XL$rzUFq2pfEger&=?}Pb%QLw;sD*W@N{|Lq|I*W%+R=Tl`?p zDd1yBV{ga9H`xgyo8y%U6nkz(&z=^)=S2VVZy+cbA-79L%}z6uUg`5xWH>7~!}LG$ zNHi#d%*+%c;cL-Gl-TCgq%7(ocl>iZqcR4+jPynt-nlntwpj)H-dNkr)?%rL~ds?pl&S z@1@<5UlgRgVn+s=s6 z)NI4NrN2P0ie&*_yeAT1X7cnYSRc$Ch}=PiqX>(0;}>*HwXR@MhR$~u)Fjo;tFhl4 zK&?Y@-|Y@2J{1Nv^|p}UWW{rG5)u-pW#Hd{nKpBX8}STRWWmPv3G67qzI9Ki9>h8E z0`}?64A%&ALA$&*RZo34q1!kSw64HBXm>UFY7I_U&;9jwz(V4G1|M@6od^}POxR16 zuYB|Hjt+iVS+N2F&D2ay7Bs0Dk)m%5>xDKxYQgV0ZG8Xstr^T5yHhnVX>&XI*SU-N zoF%;{t+bTnwr0M#fdPz!AT@J&AanEIG7Y-@)yantlL7|Z5zkILSj1psTbzpLAfFt3 zvxmxD(iTfr2olqsLiAUWv@`gDz<9i+s-hB5dROo|2_uX_`XK-S$><6WUUr>_NmsGY zsYm@6n059Rmpz|A>z{a2v{427Ff`Pc{VD8|(~SqZJica`G$5suvbE)vmtRS}3DOfH z(m$`K>L>d$Lml;ke?8MJOBP0lDhFK&~r3nZO79E4M7Xwo$QKL8c^qgm? z$LvQUDNy4ZRq=d>y@xHzkye0`avmJ%NER2Q1hs#SIwQi-<0lg@GR(6-bhw2AE-(>L zoR|x8i^>!f*`?If7(p|EhN7dZ#3po=1s41MP&JOHt@^A<&KH9?`&BDSKx_fR3b#ZZ z1Mn{pAB7tdKAL|Bp35^{D|ATQMa|@Jbuv3uH|lH;YkWs|FlO#U*?H>X*U4I@eqeGo z>|uv(zLfK_tKZjA)eczu#H3C-ES@pk0<_pkOpn&)dgn!X_;F$S!CGtR2h$1cSC>ZX zbX$SV^`+LiXTQK~2*>mW=|_P&68Quw7tY}qTcy%*XH(BM^cGZZ4Stnp71#0MwURe0 zE+uPip2UP|H#7QuBYWj8r31jyQjG9*DlQ=<&aW&|sIQ);@TZVemG??x7?HU3n0d!P6cZM*;@jMxO zIS#o>aGyECZWSUW@RgPIFfI!+0eS^6tO~*A@4%M$5D?((7{{3oG<=ZILpzTFm6?OH{H#nE`3#8$609i-8LgWLdr zm7sAFTo8hr7`{Y`aaPskSYsJ0)CmHEs=gAkg{XiK_|{R4UTYb?kt8EzKEk|@M%0TU zOF5MFDM<~27&fAM+=4S}-+9Q(s8k>kq(WG+JsFYB&|j)qoAd_lv*>DJjzwNQGzQ=hup(0~{*+xHdH} zpg z(2IrvYjAnM`)H>bDvHO*|4J7)3TOmgKq8$XkI>}M(6yhRl^%dVMo6HvrwAH8_%NGb zRf_i+)G7!*0Ny4q`k=)3$;si62a%DU-eQbsouRN@_VwMytVJNXwMyWIh3^9ptbs0a zpsxVV1yQ&pAil4q2A6-orLCtM47ik;qL9rn;?za9mhv7RLST{2j$2$=d4oU?)s|1W zmslo^zJLF|*-mb>G+z&q1jQA9f1!W7KsBfyAFP#a!y}9W1@td*vv~z5M`NDn1<+-0 zLQn`fPM~DmKFm=*0wjl)ETD$uDsF$#v~ulEgLkCK!1V^ggO7qBq>i8y$yGQ8ePDul z3b(Gyf7Qk)ZrJMl*ZRL?xNMkZc<$#3y)fp3W3ui3yxOtwd%f^&>AMQS{h#t*UGl1O zqDL{X4qWZhQZ9J`mz9nA%V1uXg9$yMFkr+!!LM~O2uY;60kf_`1HGUoaao06X{r5c z7?@##{zg2@$9J(IE7EmH`B${*ain6 z6+OKPzkU)`Puwe4xR?IiO?YCw?egm-H3^rHuoF;jyIb&m`K(7@=H{|>VMt4{z&gx$sV~jL zJu@>CF(_`3XFq1Ub*oOmyy0mA)6A_&!!`cJXaPW2tZgluCyHdQ2Xd0;T|h9&p5$pcXy!GQ_uLSanp>qX>V1&Pa;NB`hcrDh0Jp0e*gJN<&zlZk~2Vi0yS zI^lR{8!M`)sA$T&9AuuLKfKIr^+ z@0~6EM8p*w-$Y)}HKfA!N)_@H(Jqw4Rkpmokq@A%%g7|G8{T65N(`bm%Q1M0!Mx;U z-vHW0>44%<^@Zw2rMaewv=!we^1(FC_D3Sw?iFkt@PD4bufcCI6$Rm!hAcJ1wm1YB zHNQU2O(5V~oia?2-;ljQMBs2qN~)`q0^(t33lIsHDs*kqkY8Qm14jySqJW~{2L7Wv zZkxu@=u6o{R)`XQC|3i5_vqigecMD}5wdO%CySp7S{fRNS(@w(Ouj+zcpt$}Sb}{o zooQRh1_CTR0q+fcX8k50$({~;dRPp+7}g9GR%FnQ9Ae}Y75^}#NF|3UvH+%=9d39> zn)_~je>~d^4Gm$m#h2oH<}M{8BYB;q!eh)e)zi9_i*G8W+Tr)G!rFDCjnStLmxl` zg9^0S_4aEag#8YeYb*G&x!{sLWo2b8hKF?r=u($5ClO~35|x3Vp32Pa(iGz8i=CEz z)~0=F=jZR%FSoZ>92_pH<56thgEZ6JE_%Lh@<@^4vW(WtC_pYf;3`mOi;9UETJI_V z!RB2Dj?ao?LA$G5jQm=S1yTmd;8l9|Wb01>qHuRUEG!&8fMmQ1Pp?5iXh$w$M!-nZ5i?*)3_@yG7`}_1}1-Ak}FbSsF(C(@Vt$;TPpE3;2Opyzsv{BOjcTqjEvgF z#%R#kU?=3KF-kmn!iWT&T*f6UR^>+CxqfeXIIe-A0x`lnA>LGL{_IS10OINgDaD~zf0dp8PXEzq|$K zR6L=H{}R9yX=rI@|NMCcf{cqSWyllbpty!S`{CmS<7x(+Sz*b29SBoT@2HqVus4i& zwGg1yRTf1!`x|hOauB0|hyx#CP{CXdVo;*zPfkt{XDr~J%H4%=;ZaxbWMgBKG>jl_ zonQx4?FUG{3}{D}c8ZJdC?@h;fvikgnz7Rd(D;EIY-y@q*kxm?V5CwB=B#k&{@UH$ z2DkML9DkeTzLA*Tg}vKY!29>40hS<&04xhK&txPmz3PUPX}zivn_yD|flXh(;+;pC6-1)i{IPkXCo|KpK&hi2t0IZ9i zbsNJyg^Zbr^n_3sj}2%5;98C{Ol#V>>|CvpSL!6-lv<-&&=ziW!^^sz;mGmy&9Q|r(1B6E+YsM*sp&~}T>N!n)JaHId;O9ch7>8rr(0_8S(KXk7#RmK)Na~5JL5lD z-Y-iCx%^P``?kGZ{j|dNK&UlbJ3)AvE zIA_s(MOSA6*N>)$#}WM;TyFD=i^sHsxF{4#{W_S@;rwy*^h07|E+8W*6fsbDZm_c} z!n5KrkU7ge_idB!yARm){7B-jFlooBu;`aOJ>J*S*Z<)6NqjdZIk^B1VN1{-X+n#V zvz^FenGO%69Lt}d%_<99<{id5Actwo?Y@Ihvf0)){F`TC6Mu)nWa$KV{L51+UaKWg z@vC&jTB{#rm)lJn!Yex3>f&;SdB5HU>~VF=Euw&uoZm_38CKCS)CT_4a>!$v*j|`! zZ)>vvwUgCoahjhg3iVNFd8)pyrM0yJ#7FM+Q(`M{>>Gr^J!DO5Pc37hP?4YveFbjz z3;<~678h?u9>IrojhZ^ICs~+U5o<;Qf;xtPY61p&f>YHx&;3H6ZKlT=dVB)`Vk2-k zMms?F6a>&g6p9tMx%tc!+sw2&CTf;Iz_;(?XWiNz_n{(so#Fm7^*qgj_lb!|od`TmwT{O~U&r<^$=K(UJH2Y02mK3P6LYOGC)K>PU4vBrP* z{6tJSF#-*R+WdMsfdF}tsHTjWe}_&&Whz*9H8HA7zN1W=tdn24`s3mpNvT)V_iH&T zimtKUOd`LOoes5^2X^HJ+9i?+2?@S~+uPd}kQNLTP#0ncdE<%T`-$rmvnv3PT18E5 zfS$K%q5$A2xk$4YjfQ$TpdFhD3!xj=uUj;|L=%ER2{bDto8olKRZ5#+x6Otpv-96^ z4Im?^ntmGlA&EMvAfOdmny_m`4Exu=t?sTs;PFa-M(`|ro+$BUWEu&jRICFA379`o zC^moOf`KQACg5GmfvIW$2`+X$klE-2$8HGaS{XA>)lgS|k6fJg8&e{GryGqrBkAKh zA0O3z09Q#aoP6ZvA{RsX!*G+n*!rS#drg!lt7**0?1+EEMS3cfdrGsD}-d+AF z3lBI<5=|RktHC}f;m$KnXx;!E*z&0N-2pIQIb3PUV!nnCg78!!d8YXjtAb&XmgnAD zKNQCK?ZcBJ7ztAey!a^j-~s+8bQEex6(VBh9cs!U4)Pi~`MD}cple}s#_Mz9Hk7T} zYAlIDsl*KCJeGyC!~j%So8C@7yQzE#l_}l7s?xl(F>M4;##-RQ&{u%+FB_hn@`{Q) zsdFv1k2ME6{xc zwm{u_&(75tE!`Zws-w@$3h#8QYzyGYd&&)5LXhQEsY5Gx_c+0M<<;uZF6DUna{At1 z1$eJaS$TQ&#>U{z_R7_{r@p%)z4YxSwFTS+R6^x}Gz2~AFHeSlR!`cRJ$puV8b(V? zoBk;IH;Vi=c1V4#UA9X|V7KHGS7o07?ZuD_z82L|>I*DkivibTo ztBr84Nv^H4T`|;qzeGj?k;h_^HcutpW(mY&0P2_p6!LMhdWCmCmj~hDIaFvM{NNk% z_upqiomc@~r*ByDqpT^9RgR&2ouB%waBoz(*BcrDP7&_3VYqP4oC;fKRq9;+4#Hs- zF5S&!EEPYS(P9b6H#>StLr$Ir3&TxMD3>pB;5}G9g7dR=#rZbeZ16ea-M9o*6kaQ+ zK>LuY27TvCNAO$@fiqoXcknUJl`FRvUx^R7EcVdCIb9jP>p-fFNI$&N-Ibv{Mg9Yy zwj=lc{y^t@clRLJsdAtjXf%Xt5{w4$nN3j3VWLnBaDcajV}V26WTl6b zQ#s;I5OQ9jkcZ#PO$RrW6+Aa8b>VM;QpM{3(Df#8J*MsZ|0h{ytTWcJWv8<5l_HEm zwj>Nuwopl^NZO2ThOw75DqEY1v>+*CW>5%~C@M>$P)b^~{NLvt^UQo-zrWY(d4?I) z=f1D&I?v-gj^jM9kJ)CbYT6Kp`CUneRnr?aZTgU0Gc#tb?WSACPbuCfF=s6zm0Nf} z*StEWFE9S@a+i^=l!dhQXLgF@+V%i`Q-6lqM*e+lP4(x5J9ny1LM{y8upVi&MLQ|C z_)X2au4B!ocC7EQCimQ_nv*m7Y^dZ^a@rejdn5l;sq`EBB>C@{GtOx0rLZZp?H%K8 z+UPY8I{$f`xvT5Y@wc=}9kpU!PaB?7-y;Fb%9ya&Pe+anv$YQ1ps&Fup2|`W^OQYi zI46(musiIkSs(l7ix0~gZC{^`PY?G$o~;TXRkjmpHs_T0YA z>Y`z8I>V!Md(|&or+=}_EmqT!=}dzc@_jwx{K$F@8#WA2t<$k*BUQlhK8YrDP7I|u zj-_zl{`>DAkIq1A=;T&5n*aF`7`uiYEK!e-lkdTin?)l?8m1@p&`m^GXt@VKMIp4f zb2V(>OT4<2eP$Y0Ja|Ry-?dIHyz7-QGkyrWZymzkk9bX+M?NS$22;=AilycHwXUF$Qxt z3^WVwx8#kp#*R^HrUuw^+i~9*HEwQN2GAFj)S_@z9r;FHvqR=aegqa}SaS?boSiTG z{isr9OQ9?|?Osg5)nHy+ApbjkH`euY)YbA=?Pb%Z^LF%1SR_R5F8T8Z6c2Vh2 zCOib+xB5u(8&l5bilM_Y)K&U7*-Xp{I%2RPith9f$j3NcvI-2BUBrTGb%0{K*0tWa zZPmNlOBJbG&VN& z#c(l(RX3eJeQ!waDxDNK!YOEk1y8@_Sz6?!1fFCcel{3SdYpUJ#|3itu_kx6wIOl1yR3f+%p@7R#(QMO0@eym;k8?Z!1%6qTYaCz@_{CFs| z##k$lQEfVPvTM(6Zf%Wz$?M+rQ0W1+_{|eS<79pJo0^HTPL+rhP|`9^G!>qRkram8;`2OS9a=IP7b(*{dq! zzZfWI#$Jd@=VM)hD)imguxZm-O!XwR1p!aB1`eFg*Q@2W#vVuRn0s!)MgYPY=(EHZ zA3W_BshWlNnMC@IN+$^XvGf7oHQ?T(70pzI>mWn}rumGN6~657rfXt&<*O&L8)kD- z9|AXd)Rmok7+)%L89D!o({vxVia;{_B!lmU>*)M!;QaC4h=MC8yNunc*|X;v@BJ0_ zm*z#C&v-Y?6M$;!$g;_9JJ4@_5C*;S5;TY*MOc87*!SnHGkKOlo_@hE4`DgQA9;iRpS z0orZ5?)U8awqQb8RE6N|^~Fn%Z&Pb^n*yA)v@E!aMF&w>H109R@!;XZR_*__#-yQ_ z(OJ*Foa}jY2iv-Q{$$H&{U3}r<#io&sI;eAeBCe0y#4DkHw^RrGx%NLrMZj)KbE9D zsAGe&jJwNPS-pAuIsN#N;9r`Dz0m>A>U3z@L__~)z~-m?jw4-uws84SZ(B?7lxpS# zC#Q>C_F@^#<$+$~Vt9vYP1+5)$0_)jxuL)cAR0u-3A7KJcj*K}2W=4FE8fQxAo0j? z*4KpSAJ$J(^B#ADTlbV4Q8--x;^R#IYEDU!c}JD61Kv_+SXPPUf`_xcxeseO%1-*Z zg`3`kFR!X?G>XqH*6Pt?9LGMDVK!&j3H80(v}+f{tF(etaIP-TC56tsJ8@DQ4ULuU zb8~as%h?DDrz_B!aSVDcWvk-dsJMpMJnR!m8REVFSrV7pjD?}NXP;!yYuoj!UUNCI zS1a;*yYgvB;ai(O_bGJJPmkRYEgSdj**B^hh0Hr8#>z=kn{9G9rn;kN0gm=-=Z z(D<0;f65<^QU1w+?7T6#TR~s*IJc+9?M&NDzt&+yV^z-|*Hm%S5i2sl z#iaH(KNFC84Q?1w_ST@vCE9Rg@b1yj3x3*QB>+>k>FT#ud1zB-1~Qx6IX@(N2tYb=<xQz5jB_BAJ7g=7dQ8f;jbK#7F4j5T>Tggn>VF15JQmI}h>iZ2*c_ei2*Kg|L z8@o~g{WCd09`?-pIlX&S_0LF##>5TXUUjuK!qS2*+{(uRf>HqJMo(Yr=E3S1v{3IjIu5|7o#W zr-L&@%}_dSIKzoy?iC-cR_nC#6a(?mnxXdWq45{NYV?cSk8G#fb6);)J%$2;M$NN>1`23hCbEjwZ z%t8VJCUB+wYzi_z(|}O?5G!6lb&Q=TE7|!RYw`GR6jT#Q@5& z+^ZiOcNI7U+5))_Mca$5`jjCrZ<4|9DL_jq0Lfvml6=DO6|O1lZ+w?BXhnMqu6m-q zYpXeAt>*`g$LMiq@DaQ!13_7Z%lf zQC#j&S+=_G+=vNjVHTzN!>u0t$OtZDu7FYsOedYG{xe8e;Q;XWX&Eu=!YCj6gr(VZ zmK1Ky4+{Gp0WK8+Gk_97{Wq2884>_le}BkYpy<0t=dT-gH66b)I-?-=s4TpdyrRzM z2r&uaXGi@F@F3^yyY{2u8x5B@&)jfLo5(-BzpT0EKWkoD(o()om9RVJ&tT@Tl~^UX zp3XgZXqUSFnzUl0h(0-2qx@e5`EG8qSY_G-!S){SvQ>u;D}oue-CI|8PH92b@D>t} zrc z$5TbzNTUR2&WOuMA2@|_ZQN?GL&{b_r=7KQ>C*W1Y@K%8^-UwDZ3(nLltGU7N1y-X z^;TR-n}S$KXdgx*C@wB2t=yw8r^yQ6m|KX~x-jje-b_Vz42@~@o!_d+s`{Mz@DO{#s;a7R*R z^7)v0XI`i*6MDVLjj1=+;nVS{mMsgzJ9}vRk&&F=MW(eR9|kX5vP4r$YhturR&AXY ze{#{B0~MtOcEL<#vqG5J+MMSo`mSZmQxw_O=u%SMq;U5{y#@^$c$-mK$ul#Sk`uL$ zmWEre$_?8J-~*kmkTox^>c%J2KHFe;#-u4zh61*XAM>?Lp7nVM{}j^exYL-X+iLce zvw^H&Kbg(;yr6^W9KmC8?r|f_vej*$UY^OYIJI181~WNgLMsTL#Q|o)pBa>`4F8Jb z){;xoue!#dZJp&l$T+6DZ)UbgD7=kU&z^r9j?vc!#V1qmr$$)Kh%=`aY29yL02$}Q z)61g?^r;M-awP7@nNza}EoV^mFEO<5CF%TYS+Pj#vux)$^KoVr)%1Z7i9QX za?s4)8D&@sujFrOsMV#*wPF4G^hu#;X-tLoUB~Bpk}LAp6NztFRjsW}@4j81?bnud zkH6^Mb@xMygone^*5veiR&SJPr}xc$u3kD2TyiknYsTAw9UG{sBS9dtfI-twZVfrU zfB#A9%cf=W2<4sy^K4G|Av4XWz6h97!@_kpkuJ`Hl_(`v%(2m@%US6q<)c zZ#VvjTRjgOwte&D(gj-=o{BLse@*^@Zx!f@=V_o|!JiQw$iAutrBa>r?Z7 z2NQ?gO-dTBAoiuXNj_=3u96qKN7dI4I0e`M3mky`)J$BITOGvs*tvi>V~D=W_nUU? z*illFW?{n)QxM#&X&Sr76WSk;SK1XmeF3D&-#I+fn29Y)IM-|sS$ZE&R8><~xubbx z)yEn54%#VryYt{jaIBXTpTo^u- zP=0A%KlN{?ALx_#fG~erYsQ(n`b&yA@^de)@(bm_RzC~!16=p$)ypR9s;gTpGNMF( z#D|g%TiO1`ts-nQW)ZwfW700IFFwOnHaP1HYO(t>2fd*}4~5Rpm=(9`*)O2HrJAig z3`x!>i0F^3fd{;{7^&5`&cUjE8c@=>xhz|mYy!7}4X00?`oT!O@)DeV#-X9fB=Rrd z?GHy9LHYUL>rR$pnu9n^cbrB@oM(t#Xim_*>uQ?~{cia4d+VyxD{@+Hn0!Oc!@UO^ z9p0#D#7m^VA8Dny4~@{Ihoss21pf!yI=KnA6^Kx4Xq4@fh-(ih^AWeuYsy+}ZhF%R?y_^*jroc9#-B^Mmr=K&-S_2^D*~jf@{RSSzt*YmgBg<)WXKNqZ=)mLT_*0k5 zr$D$~7@QTeZW?Zf%dEQJ5hCd*dxU)ZUv=&tBDC8;pTpCq8(eJQK6maBzwQD3theY*8L#rNK0WJ)wE27lqW_Bl z9pyJF=na`>uWd+jQQEeJ2q@a+j=XGH1Rb7-yFdRJ~wKGRq%#bb63!E1(Eyf zibE7LEa+NPItg2$$y$v@mytKNE_3~OWJ^;#l#`Ba5RvB5`LPOKTlwUN!on^p_betu z_uTfv-MmqMEv;{zZbDtK8P7e|Q2CynbKo>bOpu!BD)ags%<`r-6iTdi#N2Cra)=TG%;zu${cQTsAHb#ltTkXvtp2ImigDN7hf?CW7jDk(d!y{ z&ePlbENW-4uWx)<*2UF}?;IL9mMk*M*m%#z%>%kf^@|OY1oyb?O6qRSf!9#Z8cj34f09KE$ujA#fr0(s{ATU zL$TD9U>Ck{&i)nRK`hz0JTzXH01qo@~u$ZQ>s@eB% z_WzK#W0G#gtGS^K4OV0~@jP?yMKU?O=(s=g;y=^c~ZEhEzuq)R{tfvevwanjdY{d}_w z3hp~6{6_>a7~|FO>e_MijYV3gCd?9(weoE7J{wkDh$TO**Vm9rWaeS0R?60rPZ@Uk zT=xBO#J%zRxS&BK-@c3}p<*1gr};N;A^j>)ONP*e)sY(@&BJOg*Fn_t8{0sZ6QyI8 zWZ3R~H;~K24*2-kb(Y%vHKgZTJ$$C zm=ETBK!*6a_|rGs@2hYDhX*gKbSA_e8DzO#8jE#1tk^^x4LUeC(%MR(k6CnR2--{l z!;)UpfHzc9p=m41k#>x12zgap{!t9Uq^7MVYEH&rg`o@LRc3eIt<4u7b=mQ63hiFL zd|4Lqbmz{ULp>?$;h^WjxMj`#1$|#qIjCb9&(k|IA%P1ofeqme)K5sCR+rHpZ;-^>EC&SAaI)2x>^%)d;Xr0UlgoM_szS

ihFKL%@EMvZkQ!jwKJx>;l1V;e8Sf$C!4f>HEllg}?CqbJvBVZ{Sg`T=A`AVx?C(QT+JJ(WxA_$4IVym@ZtrcIkD87+aZ{Hh!Z zNlOAtx8@;1X`089-+bKAre7`n(qMIH=cb(y(w?6?r5D=yi-1yieef9hTEoLd6)x%S zgDrxSmTCDP+VtB-cWwVi!83C&MAR(3dPQwmn+6^U@!@^A){cAAON#U5=m3sq!u5#3 zf?c)~p~(P4aa1}~y91=**x%%Rwt`3mYvci4?{U4W!aJ!ScbkpcKvbUI$@Sx2nIj3{ zsH?Bv*h3CYOPVf&`2kOA!TPj|^ZF@RrWSmvjE|+9oud#L!i$hB>w4RBovWNo83Ng0 zjo%aD*Ew;t1>NiA<8p#H>`^Rql$efTf!gUg1c3SybpzWTu+&0DnUyG*ozr4E1TH3> zrVos-p#ZQrH*UQVEtE{ryKC1!*Z`&nEZ+qBAAbODQO`8<(Y60cNIul&gi-T)9N4?p z@6@U8!k32SuDK-qyhI7H3W$Pc+K2jGq(q6z!JpAX$?ECRc0Mj|2ZM&Hw@{SGeu$dO z_92g5H=kx$5ABP_u^>u^^gMQDDhBX7o5y%5A{u!-A-qp55tChjfa_)J!j(C#TDJz< zdiEPRGL!3Oha{Nnxo%>6FxjE8Dxh`VX5CgVv*#!Ux$pCDT`mg@gZ?vId49H!#m$GA z?aLR%G|GNf-u+$d-ms+pOL|8iQrA&4of>DCt_O6D8+VJ!Xzryorzs#BpH@KJx#lfe zC^CRJAXW%6tb;?5%*nMDX5ENFtXxxH#rpGHR5~f zMkGwMPcKq6Ad6GI$VtQbb+B^jGu7z9c9?h&QM#!M(=Q1=8rO9`gk!JWG=t z%`NUj>|aeDYM1$hWp7OaPXoE6B51{#vpk%%Ghl+Er=Km#sxEwFSg7am(Xf+PAk6x|jp+3K)rYHUH$Kf12Pu*}1Uo z3WE9-^SdoZKMri8u|y9CLZf+hN-l5Z^&-q)zDiY?H2-ctlLuj?4nq%a`gP;@9P5?m z_qoOTCq7=WrkQV19_2C+riXvBRj@^@^Grl4twzuJ9v()2vq(64qUK`q6Suc=!Yy&Q ziNyT|O6(y$0;#+Slia12mZB?HB)M0g1c|?fWMA=1qq>FJYON4F_1vqkC?rz6&o>cZ zH&xE9@IIt>Bd%mhAw9JG{`+Uc{n3oYFEhTA>nYN5_`p6IZPEHg7C~YmVR|7BtKV(Z zsL>4Y%+O_bJCtC7*}Hddj(5a!3i{}s*p!;|;FGr+&5OfZ>_^y5c(?X*$%_{+hGODp zNv8os>o-3UF0?oqlo^CS-O5>0UIND$#zBs|KH@|BFX&B3z18PkEz}6J6#rJ-wYPoO zu3uufbir0gmAQ7=CHAl{0gUwnN#$5!M83wIXc1}DQdNSv&xawDz5 zUXC*B+rR%bvQ&?PKF{_#6^_`&!?mpyNZl>4T%hMsubw(}=v(#5-x{P%ZquesJ}_qM z17E#>+=hcXs;PpuI6wQk(EG1T(Ln4?eO^-jajf(4k#UDYDw{X0S63slvbm4^sndgE z9j6wCr&G^NX)l;p)X)mTwg6nwbZ*Jl>;Fzc6HEaVl}@JRpxOwz8Zfy-({17fg0Hed zLC<@9!5^Wa&fOB{SPUbcB0t~?>8QKrM~JQ*?;ip#%4vGI>a6?j8z)8Tk+0&4!Kw-#RumP6lbuCi zX*c25git7-Bf9n?{p6)h;vBMtnXEXf_)=lq{}Z{f%5x4_Dk(jO@C+94XXtQQiA+pM z8dWooK8pJLgm0rOYz*$^ApP_z#kkO=kBfuJ)FFO zo|Go0v-qSBJk-<`q75m@8nP|W-tOt;)ATm|B*j@2GbDUN=$d6k8 ziwTMHM!~uLND8vndmyt@7?*J8xPckrv#Han3jQIa4eXEix(*+9^NC&)M=Ee*2$y%D zLVpDO0VawE62JYAJ0}#l1?mZhzFKT1vc{f35i6$C2~YI$gHvS%3yQ1j0tr9bqqI0a z3zT$PNO7WXrLAHgbO6MOPUhRZy(Y}D9*0}% zkqhIm-}J~k0cXMqtzP^VF2CsI#7)1+$8!fs0XJqT6ncG?M-5j&OKwvtB9Ud>ZjOcb#mvdwye;GVZrt$IlsLUAM!zYrDAV zvwAAu3i>ea5tk&tv**f6PM$m|;vQJT2sNHit_|{_uj-VTIfxRUOXz?Krt9D}a$)@k zhYx;1XjyStUlE$vutXr;gBa;swBhaf8$%AhYOPOQkdldZM83aE-gsREq-h*-0A~)Pw}H#v}i$Z zfR0(;9zFgtZs+T8_V7>Z8?N~|=-nD|=+2yLdvM0+rukvLlwy2=$Jc0{v#v+XghZ3~ zy-#{BY)*uUb_JDZHk zhq=Yl{-QYBf_!m$NL88XS~_1cCtN??CYWTZOMl3syw3%3v*}IL?O9KIE4P3}+GS-b z<#$%k6MRn(D-p?7D})h00s17TsqOtre`p zg_O&VHc+S?3H}z4ZnH@rT$HSU}ObI?Ogm9BqV$ zT0GvjuUS+ILOy_bX>ish`|-l8VeVF#weWnD66a5Et%5lZ%hB5tq)KQQ?+%>Xd))QY z9J9Xg!uX|@xc;C)yBF9@L*spEKSk3vT|y-qE}?Ll=C{U|$=X8Pfqj&$-AUtP8h1Z8 z+nL`-Hid6j`!7m(z!hNwpRCO<-tkzw@Z8T$2Aqz6k(}Em-0@}9(Jd!c+NWuNQi$&d zr+y20|G_DOL0@9NS^A5z(BruSEtT{lyK5XLZeRqM4HU9lsuT! zUkh0vtbphy{%4xX^k_7T)b-TZD3m-1@W_aDq1aXD=RLC}CZ-VB#VzCb`rhcK4a`PX zdI@EIXwwIPMLLd|FxUxEK=6VB&WKjj1gYXNg0b)1C1^j9s!7{HKhi#H9e-U<3Er1F{u?1R?xN2q?3hc$+s`+#yu*HHAUfV%S8;x=8AX0<9p zyVDYE4ph!Vm?9Kge0PLe!6>&}qVucUZADWN11X`{6pP>8-!>Md1x)WI?vej-X@(0> zKBCSO)z}@vcltUYB=YXXH}mm{eUjFYAt`pdXwRUm8RA~;a!l8LYr**7!!Q)CpWj8x zB7rPg4i>t|>xo2%$w+@EDsb>$27H7ruRlZf`tH5O_r0&bDyZ;{g652`Vd)LR zM;J91`PZ#Hr_JDvBp|O@7nV*jV3Ip`v#uM8FAQPBQ;$OylYZGTIMcnx)i(3W9Ds=M zXyyO%D4sZRqN{udm4^zRRjJg^3uWF_eA_ezh2Sy`hGkl#Q+e>svZYH|H%FtXQ(qNC zT4#g()32<*ATl>P^E8fDQ=Krh3W}0_axDl)WU?ZQB=i?Q>R0vC(~E#1Brb&QH%=C* z;g*wP%U>O(Tv!&rVYRP5)qxegf3oaZKG(SbL_Rt&E@V(?Fv2IfUHWsFCiNn%fH2I> zx1qqK_DV_ZD)u)K4#inp!q*aNjbt)~!j>+DZ4r`yhh9-aBgAqPG}zKIJ0NenkuDd7 zMHS)m5|m${aohiz*W$0EJdqTTlO%FCF}{rPCPSxh-#f<5O&i?rGBvgFvt!n@_zd)G zI&PcJlpC_-DqmCmwje1=@@|*@G7r)7Ic;-k0Pbh)&~chx^knTSM z68Sr4IDkL0@jm8{|LLbZui56}F*7Bv7d87vAKtL!?c*85lB>lqN{c7#;tL!EwNU7EwQ}DFWor4_uZ#yw!`}mQfu|pw$v^z z>~Ouxt91fgh{SBTq0Bmo%Or$~p^xgLtq=j&?@&I((G_yQ%j>VDbd&qe6HG3$5iFSj zM*RWV3ytFd(PGCOvhMvnVT_P&at^iBG^3!vbaW|XEjF(&#T7;dbH?E{AWUh?9T9?q z#iEafp?;T{ol?X5Qw3>{8Wrc=;`#I#WJAgJSZv;ad(;Zdh4LxQ^HCm_%_prRWvRra|{DxrYTx-Zl>#VD>eG&)l`A%~_`*{d_ia~YR3%6Ku zf+QWzeSpKZ9WZMGPo!6m#@J_Rb`Q@U<{Gm=659{TVUC5JTYme{K>2nScI-we;>xbF z@|BB)bAv&w$`eyMNOR+lDvgh~2Pa8yQ`B6HM{@II&+oLoh;KARKl&HUzHY;t@}8R8 zX)#tz7+U%l%M^s;2Zw4pb%}U!@Ny(+k_M?`YkFo6F^@Z-9t_&xT+V#3nsvZr`f3ZcYBOYyCt&U|SoC+z`l{5ph~wYyd-jb1vPJUsF_F7mHkEf4!rp2YQM*|lwrG3ZdZuMje8K%43SI4 z@U-&j>Z>2^HR)&R7`3uZINM|=93NYNB#eOp zAnrBJH*)jF%+*3cdu;EA{K^+cehJ*)9ZJkGK?nU87F!@qWk%M2mWZ8+9IIRHtaw#? zFUNS#KQ*&W_{)CJz8mQqehceqYIz@6(m@^TsWgnxfQACB>NUu)_)}0Z6ql~2d}V+y zhX-n8MI<_N6+?;@FIpaq{7~C;Q)$x^w{OfkJGuGZJHNQxbKkxbCKf*#@+){M@W z4+4CBb;{^%OV7=TNNiRPdbIL zve(HRuik1tUJOyZgwU*N+Yg3@6cUN_*l=B^eCkakT?y}$+HO^n2|x!y7I9aVHP_sU zKmTb=v;>H_OvG*$Nn6Ay$$r|sF{P0~l$z>`DJ@B_rEeX&&9>B<1O#hFSna0-#$f*X zzXtQ{EX9;I58;p)*49Y9ZtH(_3!Rw0z@4tRa(SQZZNvPug4^DV1us@+Aqp}=J@`}<6(5mzRI?dZ>qT> zim^o7;Wwl#jm~tQM3+Hhom|R1AuUAk6&BTgjQ&k*;FQ;rGv#C;E;sN-P5k)wLXx$# zjNl$YA(?)6VsO~M9cR_QfeGI|xio`{r>WC!3_+pNEhZL20CMB*bu~Id@Y1Fy6%!Bk zDZd8!T`XsS$|aqT#lZ+#vz;(jdHe^7b-3L#pC@RQO56(F+iiUM!<7>sz@ckjR1?#+PH`!`tpB^=(18c3pyzvM-@h_^SK5-PnTF) zYK<6?LEk`wJ15i&j@DN>>CBrg6o<-d<5x=TFNe3GW*H{-pzy+9shv~L$E^6XiSL@A z8UDLqK;K6HwuO?FDCJ0st`#*rd-m-4lNbm?)>0fR9zag;g4zC4J4zc-O}=|of1&tr z4fb|iN%1A%u_|{|&3yzHufvDu?3%c7+YCNzT$8oP^-C4$J-guAolh$>uLJ@^gyVLK z03QV+8QX|C5L-vQu2?~Tsn;li%K>vOB11*?E;8!3UQYe)bg9>rwHAq(PFa(a{W zi+`t@PSDj7KFk{%ifDdiG5wHFDSnruCeUf_Rs~xFES9d&eU@{`&z2{V;>2^HbiSD* z)QRpz{-7t}hY7ua!Aj9UVa7Bpf98V|%;8^2Z+nUp(& z4RY;AbO2{*ECWvq6)j$cumYSqC`kDtb4&XnJ40P7_@D#}EPCR4kc}00-N|h>HNEUO zXLuCpp^Z~FpYEoiuiCxLn>R2I#!$M+#3KRonG|M$pO2o*=>YuV&m%MfwCN}>`J++) z<>X|w8t)oZ6X`>DfE8GC2mfjq3o5`eB`anQ3Zkr4=rJGWtcfiw$}3Ljfjs!IV$6-dPXmB5`f zV*3BM?vwXnX)|;N55Dju>sp=a-!12Cr%T{#Q}1`rBT{q*JrY+!^LFFw_5RbUs)U!x zuf~mB)+i;KZ@QlIjgXF3>&f7ni$^>ta`@P^{TlTEZ+75NlC+5Pvr3&~V+ z$eH05#J19UG*hZgah<77U#PVtuu(sXrxZ&9`fWeQYFhoJN)z@novxUoqO#Cdt)_wc zY}=2eO`Nlpu%Ij7AlrR2-JP7T+HRmAx;ur1zJu?y|CPdDkmxF3wPXt-I8<=ksH}ft zeo4Ctb0aAL-@sxj_GHV%*39Io=RKQVc`;Ba6dSA7TT?T@d;jy%Vj*7}GpCsv<{4?} zdHZM}GHCzo(!Jm*2@6s^k+j=Q_{K91sBMGg%q>5=`y0}e|F9Lbm$|v(iJZ0LAJ3V_PJ$YHEMeB9i4zBU*8irD zSW%{md%SMyxE<}fNLP^8y?yywvfRG=R(++vb#^^D&7!7H`Yzjj6FcZ^9q^F*cX z&zYsj|A}Ux!hFd4DP*dpeu}2G(INlg6d*-Js>1r;SlkmmK0fQawrvAhsW->gW!{H# zR@Z1<5}@#&SEx}m4cAD^~T|V zy@}`FisNLsO&1D>2?8y}pMt5NX<4a*yl!Z5@lUB0Y}^>s2NaCFoy-<0C~YgQ=`5Gx zoib7lTK(Xyq*5n+QC1|jQ*OsqOJe47KIzY8mW~7|H?g(TCWsXMIImv4dV&!ZN=ur( zeg}aDKqqHd0|BWuW(HB#`^U|7O*oNTx2>1$SdyutTNnb0F zAI-8|;kqtJH{w=6+R%8Dghn4KhFA#m_%K?8L+_s)pD%-vR;?=RR&|NDJfIi5g#>ju ztF(}239&t5c3!A-=Kn5p1{Bl3l=iXCkpP<}r|WdCimFLx9_0zC1zk~yFH3i;X}|;p zbF(8pOhw+7r39Hot%hOBZ)EKE=ZzZ~f18ILbX=%8L#R+kU67ZDQONP$6Yh7JH;HXW z=uo%YsQiY}LWT$&{EY=SAKF~fJ2q#5UVW8*ONs!vv66QgXZp?BX0cS>}>HlerZ-1*eCHJ%d`$vR`aCag4mu&fV-K z_D!)h$f*Wu)%F4)7f4&$5UU5>zxD!{RL-xB*H4uO&McU)nbeODC~g%Wf^;v6SV=qD z`wJd5A)HT4CPxUTuhJ5+d2eskJAzunJyzo`b!jolQd6Q-beT9$g*}9*_QX5qN zV4`AR5VQVeEN*ONnK)FdOE1PD*rfd#*h@15e8-?R&!pVJo{c@a%N#7`vqs6_OlVV! zyqG}&BmYlbbzcn2xFf57z>rAi%(VqwjCJwj^So9!_jGY<-`a<+8i7@U6cm4w+p;+a zWH@p#JiPcNrCrx!TN}*eged{elCgkt6kcstt>nvu;DWZaDlf{Z^&&j>2!F|tiohAY z%--?;H5JB#<#SCBz-HC8TQ}RtFP4)5Cn?s1!IrX?S6|=T9($@Q6XbEK`zz}7!kj&a z4+n~qx06fZMt+iMUU3)|nH3_Iv@*yaP-@H+09^# zWkB1CNUF)@fU~X5{Z24oFkl;v4^RB`tJKkf zkuR|o@GIcQ4rhd5Q>kuWf)j0W@Iubej&(g&oBKpOTNAbNWt*c?NQY38OUBFU5-&Cn z>0ZD9R9kXYLfiZ#O|~JQcvD)vdnYPO?S2EAY{etv{;?nT3HSYv1tAVolu(Frg@Hr? zu^^lo=XmhYq2=vkp030#bhp+R9QPmI<;BCss_FsMh{%^o zvqtbQu$);@Q~&bcn%e23VlJ|v53XcrfHA|;R=nU!hKKRKlZo0hyKP?ln_Bk5ly6pR z-p!FVKKW((o&L9V`KO1q_J#P9kq5(P-q!eDRUN%3OqjAkjhudKZJ3i9Wot&|I2Da8 zs)8MaPIU@XvO)o%f(6?WDMCwyjXhTvw8@eHb&Czc#xv}lN^uw7b|wPwwl zkIX2Rp&R1Dg1-8rXLC(^_-Ydlfo-H=g9df@Fg-=!fcf|9u2LOhBn>{701+#tnYrLc zBN;R#aayJwfzwhyl`o&qER{#};F|`5m?#1r`fs=$+o1F#wgzeLz=u$2aB-;~ z))b}X$5)T3QSTOyQBrlF>*Ro}b-uh<9(u{Hxa0DUY5rfcBetS)(SavDQtj2PRi_!g z@TPdjoP@rgjwq*FGT;C0Ka+}JiG1~`tYQ6>p5nE5ex6=j&5!_D^h6xJbj&fl^!`7- zz3h}M@o?gq2XZ9Chd6L?0Q}$q`$eX`IM`7;#UB2}6)GMwVfbiX+L0v9X#ELVC`}Kf zDVvC1?@4yQ!GmWId1Q(a1CSb@8uEg{X%$dTG7Cu#omwvE3kWj>*YHCIzvwLSXCC4_ zw@9L|`b3`G{QrMk)^vUrq?z_8$~+u3EXx$fyp#|5Ec9}2iBfZK(Bln~ql9!ZL&upj zZbs&3v@v%YjiF(;{s>-<_HGP|MoO_B`V8trJ{&%b+ObKK@${eJuV0&0?9BTwPV%BR zPqUTkO|(G=hO;dShDlWLQNMn}JR6>Y6T9`|!z1e}BZT7g)&A}G-%a_>wvkF_kIJ31 zQ)Jd)f35dq$J)p-(!ysHXplBdmdUWkKOKBmaUg`m9y)!ttEkf20cTHSg|?`t5!ynf z9R{|tp;uC|Dj&xpL=QHgCUvsX^1tsCMMH0e0awwR2*4 zZwF}j5=IV_=^4O4i&ayBA~O4k)zpGe@NRLS84Nheo+cw9WReop*bl4Bk*8!bt8i$H z_4u^mU`wwV6DItcRdqrd@G&OlaaH6LVQV3;O-)Rmr`eI3c!1Na<>xAcCfPnXAwnsz z&#=eJOPDCFcPt!<)H05Z)_Q%H8#~6! z<#J%+bE!1E-t8#|WB?PECZO|1E4{j^7?h-ZaR5*lhXQKBYVfq|o@wlw;6V-C!9T}< zl0Gg)zs(Ar7s%8Fz(IxiN87>wBD+1+c|e9N_VvhM>1sA%J?lHHAp)z}UCM{W@og|0>;VWY4i=ZZ;#>Kqr+(Sve#3^%I$(*iA!}{<;#%$uh}SfZ5eTjFFR8tlb-Hz* zub!1f@7oS_x#A!_tzlI@VT@m<1HN*lIoIKAHdu#-=ClTz`R==T!0hZht4kez`o`B} z2hS{TmR$2VPa5la4BRCHYmpih;`M@6`}m0o@Kz#i8IP34RD`?fQhU{J+97~jG)!{M zW7bY-KWxQ4Zf@@+luE5ZVdhLsoVG7+-@a|q%n`vyl`unQecgslQPt%poyAiQ4;sgH zPsXs$Rnmk_r*eukVKX-PbH`a0bDYQxdX7e4{QV3|(^zHB=I3r&U4;`L`B|9fL|v_f z9sjbEna*dx6X37df!oSGs#seX%{kJOuiHyK=pYubFCOGA76q&mzNT3?2Q0#XbyABt ze|&vy^hrLgCKG*jN^yr;o1Vw3pUuD!JPWmjbRQ@OQLPy$_BcLQ8Rutd zSr*&hRoHz+6x>ooBiO?BHILiPlC=pq%g_Pz;3j5$?~R!0q17z(@KtySrMGyrt8?sjpCKL94SU!RI%zz<0QE{uRbwO`K}s4P zDnmSiK*vZRZ_=68e7%Pn2Qe=rIxIxyiz>yiq~#*A?HM8+P55)9jghf@W_izyg=L2N z6q8rToC`lO9r@*@o*Ggm%ScbNcigtK^bh%1HArR~v8Lbdiz86wjskG*T@NPv(gwD2!1qC$ZZ6mct1T*p*O>hQ3&z#A zKP3SVi}|x;#r12dK0ACzM85NJmRvh+8Kgf0YUN|O|UTyPro^&>|1 z?mbD4zG$5}30_6b%7=qeL?`=u#aFMW8dwnOHMn49?f6W_ogLq; zt|}}L3n&IS^sm`+8b|SiC?1j#$XWvWI(IRS1u(Nw89{8$oovQ|_A(%VRVgkP#85MV z#!TsQ0Iy`^35!WXBHHL;h4t!#q9~S)xLgyM;#cN%2AmZ$E6)BEk+37u>Oobk|6hU1 z@Ug6?t@nWgXRheqT{8Nw@0vG%9-XwbmI9fYeq-~IUGQ6DKF^)kpUBQ{|LL@Pe0lck z`+xV=Ejrw^j_-q_iRn(d<4YSFoH?nAk9)pKM}{}a#9y~qj=*#o)ugWx3Jb3?T#vbf z=og&|wI~j1>8MlPMwWPZA!%eV{;k*Y_!LBwjva-dfG5e93+<@;Pjh$9n#tP}LV2+8 zKKZ+0YtB#g8EHEJjZ0OSvFrq%1hbuYJ^`*aAcu(TLn~JxzB#pA^43@Z;DBXEUd5WJ zKVOzX3Ctiy>!QNTV6s@pRT7JbU^0!R0Xpx&=^frvm+AJ}VkLan7n7$)&@&MoNT^}~ zMB<8bqe>BnA-$8wpJz52@F#c)+m3WZkU35Ayy+J*Co=bT6qCI15xEUkrntaBeKoU_ z6aTC|o?`7sWGoGKrv*}sH`u>UWqn#L_nEY$3o^J98%U9NOf{Ry@GAqG3l}PN2e*{8EWzuu15L?nGRc+FTJBgxJyE=c9$O? z6b7OU#3dV*PLT85zkji!J+5a08?QM{QZViand9l!{zPRcr6Z%r(>PNf=~I1;kk~WJgjDn@*MSVoW)eRu>4t2Hy6#lcn&bmuA+=l`D07 zRd%AESqybwt%UqhEmwxfVWAHaeS*-E@NB`3&3W2FTCZhxLya2|;P*hG`TWv$%J)9i@-v!ZPsc_JqD>Nc!CU9tty^ z2X>Nylkw~MSr19kGQxxi{CB~zX{Mv(oW63kAgo^^$lBlslNl`P8msP6XNYG2Oz^V0 z(SX+&x&)`oz(Hq_`h9kvxV7Ulv5iJJeOQEGeQ4}tp0#jy-Fo$;FIlR6ivrq%(tsY9 zZh#ZrgTP=CB$UBoGFA25xqD0urC5{p4ckb1e`Gdkk<+8#C~akSLAP$*TD42qn8G(! z1%=8!{~Oq0D#Im*cs$)?bB6BSTFO3X|J}&zA}lrBna{Ro0S7f2gg=yD#=|~6y{XKX z8mKZI*yh5H$bz4)9=G~#A>?JH4FPOlV$K67-xWLRV}_JwF44Y0`!Y?RZZdA1nZaOG z*67oC0Xf3Wfz=xKWCI82;FX6k8ZK^u%WK~qL0LUPt%p0O;tms##U|6VgN`qwPL&A_ zxR}6*8(#(l;K2(dvOy$OO&5-i9JOqKm=1$tB*RK~4I{qRT$(3-LF4r#I^JV2(L!lf zV!%|!h0YVv93F^sfYxW}1fa39&xPDyr6Ry31!g}#${;&{uOWyZ6l-=*UE}pk70viM zP13mh<*QeMg7qlO;@v1chAz4J-B19n_kPUq#7j!8gKiAO^r(^%cOw1)VkK%4bY&n% zRMOA#x8#zTc^7@9Prd6eCGk8)*#;g|JYM6ttc*=~u7u0X`p|wOcB85O$;P}~Bmv4|jvMs-VN0Z+1>yz1s5& znHgDqlMiU)^0@S0QT=t&7L}aPvIHVr1^Z#FsU|7(L#m1zgQ?~m& z+6%iy8AcXxhzgV3EPQ|L=l3zrpsZFMJI>Fmgq2v15k*`8TwF3TPW{*EuFOl3IdM_x z%-nrh;@m`;aBzxpf-AzV9-bx~r!bLCND*{jBnz7<=9M^`9$LM6jhFr(v?>R>p+^l{ z>eiB%q)aj5M#~3q&4^8P?$Xh^Q+NwHvjd^rcfZX&nYT(AA$=!Zi#V{I&yIMAXF@;2 zZ-1OUR&-|RDno%+K6zv~Mim!BE0buA(&*t9%NE|*afCEkRJ@pP2J!u8i)qV2VXQ*& z=Vf4ge`&+PwrqJ^PUFmNmUHA4kX<^*W#ZvH!2<*b{A#d_WUzq80-Tz#4Q)$!J#w?h zP|t?U?vnYKOyycpBDck$sc$v$JFD3Ap|b|b96ixysRAW^e_^^tB!+wL*|Q+{{L*$g zrzdqxV}Ox|b-*Os(B1O4JreGGv2kbLhrzT?PVIouY`Xer=qt_WLz});GnMh4Vj5C9 zL~sQ#X05k@!BfTK>FfL(HS+#xBdvnvDhQbxByG^X&v#I53{KKlW>iV?gu9!S_XM{! z6vTXG&H;~X!e-q4anC~2sWKjXHTU{$+fT}g*F@i;v*`<4!|90JOoI zz7J(?XD2-e(Om|4Hssp*L&CksZKxzqX^X2*+jGl*^$N{VwcV0Fqmqv61q}EXRC^i` z`u&N*&WYj@GVc0IpU9x^7ja%{4t}Ch1BQ6_3@yk#I(F`C5{LY;_ z=V8yR9%=4}*O5-NQ8kqpBpd4v)ZR`HtIX6^#F$!VLCxR7czA|@z(v}TalJIKYOJ~| zY7l1EZF-wE;UHxuyc}sxl!rY5RmSs)E0pNVB}z|5RTmIDAzMc_ptG5mO-wj zQ1bPR8C4bi4o(8mE2F5X(}87`{pc!?3472U!d2xHcI5g88A7lj`V(kjG}DOoh^)l% zj;kg@mk8v<-MD+mC|+t%6zb=4(hFQz6vHXv0H@tCEY+GkFGCVagN|47n~2nCKtIr; zy-_`1%tXo*S>|Ph@yw*ic1Byq_Z0@yZIy+AtX9Bwp~2d7^;kZ@jmKFlSoQ-T!&K(> zHVJCsf8Au!ifn_MD&Na_E!+3>zdop=0_DNl{~YM@4+qR<+wVZkFrG5+PTz;qqMu+5 zi|;lA7aeXEIPNaiEs=b}^9XV9bCiS*kE0MOiE45~>a^M7Ormb1zg3=pq;b ziXoCRO$L`Ao{$p%qgfRw>sMXo(D9UpM;~MGrT>}p>DQEbt$?PG@kLp`5@%#xhLm*pGdzf7v-anZGd_A8kH|srLbbK@qS=mBJFkq?JbC|LzBZjzc z41T(GYh}c(>{i3LV1~-uqQoNu`G4fNZm5L!0bUpI9^)3M{?sB#D}Jthf?QywRmAMqzoU z-Q8#kGmkOfm^#P+KkP!b!BYGIU+}V#q60E%a$)DlymPZE(4P2`8lV2-WKnkMW`;g6 zv9R_I7tK-sy~E`#o)XhIH&?Tr1wT(Wx8AIi8jiPss;o=I+3!!h-V^_MfUr)?fz|mk zOSE!Bc@J~j%lo?dyy>h8>gaPSEbOIb+-7xcHPenVwS~vRA5@@R-Xha@%UzzVmx9hx zrbi*HDr2)iw7nD>MSFN+I9yVgfKp+vIjsK5;i-|n6}p_bMOQlDHGB2)ul(bWUzzBK z|M9K_a-F0<_`VA!Gs7-xX_@%;4Ii)3=>P^#5jUWK!R~*$zqk)mDSZ?|SX0gu$*m+i za5`pTYmRph!&*r9%Meyux_0Tn)XcAGX}SU4Q>|k#)p{GL?IIf1#Zcln(d)5eo9{peTkR-+dU~3rjG_9* z;F5;2+CkrZoi1s*gcD*G`ienw(qm^W1LykAw%7Ij&d{qlhO)>WT7(PdSBIAkVNQZneQ*GFQyJ_4PV$HBk;J7|V?ZCVw5VYLIZ1Jo zn6v1`Nd0!rs^`eqX2_ET=Kq^ERljn?Kzd9VP?08c*%@CTIa<74oT$0sOMX+7D7)HV zUG877TK{+Ogpk~`Fn*BLdQGURUB}zX zQ0gf+VnERV%5nuw}gaHV*c$i(!XdM-3F9WQ>6uhWDMz4+kt5#PS#RVvqTi&2>2ICc+t z%QK@_&X*O8zH&uhSS$8tiP?@<3|yw@5r;C-7X$95Uw#pjBXGB0c9M?6>pk2Fe?sQI z-AY0{;4v8u$eOW{%e=jtlTbF*Y)h=&t@9BnG{Xq7XX%Z zt`PgSDk!$WsH01kF4gY8d)-nsm8P9^K0mp5eqLx?rb`$hQM`fBI%j-+eb-FbmiPMF zs2yKs`Q5$ z4z12QKRa8hZfua+teFXCkj0T-PD{R?K#g70_E9PGS~)-ZUCr<><`%TTb zhqzy4PB0j3aS08V;-Bo`Ar2DHz04?1c;iq#kOL-GNQKdy$!N|!Sk+E}m!#Skzd_)( zZ|L|}{HbIW6#Wv$Wz^NLZ0a=$kJXC*kFa-v>-m2F|KE;tHa0U1JIEn(meZ()IUhph zTsCv4x9|6} z&pxEyuh;W>Uf1Jv-IV*4pG`?@wy!*joY&i6+RA( zAr6>=`U`V^XmD`w2ZOhDs_>uGz94U<-g=5Pz%uG#}6{*eAmGDG5qW{ z#s_yQWQo+xv#?|r-G8HE?kMa44Alm^VceXqXnaIa07o=!-~NEHy#zgGr6u7=_+{H{mFPL`!UR*LP2KgtW?;p?RXA~vz?ui^>1xl+d z&hWpq)yxbmBi28u82ga@g>% zANk{#R{I?QGnsNM8&_QXi~o9>(%SePaR>BY20f3(?y zq{e_mOL$S~oCQLuT0~A6ZiZ}eQ%pd#I&)9+Q_ANGv3w%3CgGCcS#Ig^i@-Bl|CD_o ztA}=BA*_>@FVd$t`Bjn><#$>9mV-he*2pEb=0z@0-3|@_bNiq6t@(_9=I{=IzL ztfHU=K|DfTmuhlouM_OOe&zkcz7dz>d`7j&sNCybgPC3Z8h$@BxXH&K_P77J`cEHk zyZ3rd=gBofhJGI4`%UBI8o}jCe(q&yQhu?0h4Q@{&bl*iuTS>j$bNfP5AS{F)x24e zGn%a~^>Fi-(-TKz-TLF}__hcQ6!?ON_4bxl9^pe%n9lSIbo`UYEL zy~bc=CCubA3AN^2GG;4RC$h^i{}!(2xnYb}4|FqBS&?^gM-dwuI0!_b34Cm-f`%XkV#*ca_CrtgL3u3sCHLMhDKv|424S2 zUSMaRr}s=+H2p;#E&pI817oI18Ikluihe0FErr+wG`DiFv#!obb7C?9sdLF5!KbFC z0%-OnFWM{|%*~I6c13izUG!-)NGsDK=P7LIavr#^b+cwh9*~KcyRTFywgq`(g&D8% z>|CFTOOr85GEA@=_)PKDh%1n5hLhdmdD7M5g@}-eXnb6L_NkG#oZ;(VUA?t}rU0s= zre5zV?V*Ql%a#u8Q8qv(g@^`htv37uG)<(8xg@nLRkEZwcHY@Xd*?UnSH4`i(-m99 znyfEEn>`8`t_1xVjVKk`-YFSFhD3UR0VeoV`IB#!r&xz`--lB*qDPD$FD&R!+t1@I zJ8C0>^#F2Uq{ty|eZKBY(C)_< zqH5KurBW}Ij}|L{RonLaKJF7fU8BiEZt^C2+1Cy)U(`ynQYJ30Xo-L4ar*D2w8lY0 zB`Sm07w)+5x}3$@B19(AXQR&$e5q>NmBj}qa%%Jpa;TNbjlrRE|6Y$)0eD|J1}Ui& z0_$UpH*$}wTK)CcwF%2fWRw?&Ul9fOP%YIl2cCDd?LY>XwE`+5{(Y@HL zW51$C2{z#=7$QhtRI0tR~kzxDgk#CBMmA z*>`pZ{pra+JFX{rOs0HUZoDW+3Sk11GO=Zd)4>ZMVfLlev-J)o2g)e(EHDZ}YSAhM zw3+bZn`St59qFlBB!ehJ?m)gu7?*Sc`?&tu2j`5TFI@s6oHnZ9aa!HWrICaY4H zy?>AF3SPw*E?#o6#qWtlTabJkg)@YRE*A)fQ(#^qWX?nsC{mwMAu?Y1T4FKm=_Cu$ zd;B;ysn8CzM8%p{+vCHlu5!JRD0eT!-(|49y|3r)Uw?JfmlQN9n@M?|k3$W50!Q#Y zCuu&n0AtXzt$X!Z3s6*7KoYrlWD3d!$TboZ>-ab|14?>G(V7TG$m!2(ZGCK>`l-ut z>X0o39~-$_pFS{Ad(4pl&#Pp$$j)PE(_uf~7*(IvVRGmYPY*wzMyjziIGc;p3E~6i zbg*&VUA@Y<5}~z6S=FoC=k~rSjL?)rQEsqX3C`F!DMFF*_!EBC;MewR^;!dx zwj>*|EX2<KVzwAakwpUFo^UyGF=_BWDrAWlEQRzt{8UP{z{nLb5--)c(WX>QP}xCSIUs z`=&Y3$D()+HCn#n&={z)fb!gV)*-A{RFd_gMT??eiXeY<;|qkf&q4DHx5#iRsD>{)&s@*2uN}|+Q zYq{x3d-Dg}PrbRHlVbc`x1G%>BLc#8dc(u^&mZ?yXIG)d`|pQx%ZBqdy39cIa)zhl zW@>!r66?mv5NGt?2E5kuO_1(b+#-T69cXu=us-i~<>zwBW} z;q_IaQzq$nLaKZgZVS^9rmcy8y&ZBv?wOJ{h8G2>8WMPmX-9}WCvpH zCLj4peV-|m5q;!p^<)+G^|eIWV5c>9vzZx#`6{pL`wYxmrVEvlthdvMx}#Wj$rbNP98#;FSbkDe z-Jf0Bz84t`njoBaz_w`>8;DF}81==|2U@jiZC+K_wWnC;R%Xj_+* zpdD8-I{Uok(8cD*a*q2?*i+K=E41b9w7jb3pByQD&l=oSV6V@Hp2^qNj^c+U87!zP ze47SGy7QcX&sYitwfOni(fpzG@=W`vX&+3AqDO69Y=g6`l?{$fos^#x;8=+u&>!2* ze$BM1K3c_9`{fa@Rz16|vz9`#q<*pu>yOUhdNL7zzHurUEBEi;k2n7l!%)=b8x(g$ zN{fym*MoCR7Euub0uux6Txq9XT%9o2XDE^n$-2Q*wBzLV?%s$o_}z+?4G9V9@~Bd% zOM*-F=F2A};08XeF*q^Q)^zn-a5T=(ZUfmT>;o@nvq0HZmQWlv-u!7)MCX z1P;+4Nzad|ysu5-`A~4B+7pL~_I2GMb~N_W3VU8InHHLU`suDe_5((_vB;TW|NPj% zYM5?D3+GbA&6+hsu5WO#K`e0bS*2dcr&>!09M>GeTKWW;W*i-F*`#5%lqO%9`)4!S z{`9u}Jfgk8Jw8V5m*_$B=B4;nQaCoa&-;0vGPPhqBD<7c{ghUc2i^}+;{+e|M+sUH z1D>rLC;B%W{KV5oQ1X6$PK8dl69v!TddVH!K@CEMiZRElRLd)7r4+-^oHG(ay?F)j`r_+W3+)MCV2VU2<_$lCYb zbPLm`Z!BVx$lQ0~R)$6FCz326mAU z1t%MujODgs+ez>arzKhY>d2FryIbU?cLXylmEqlp_YHqzDsH=X-+p_{#-`1hrJ(N0 z1p@$mOO-av;y9kdAV+rU5y##N{)Dq_w(9s;vx!(~ynS074n{$HQ8tWJ2KZaEXRNB5 z?jf=5rQ`ESb3)pe;PCO6^_~ zm{NEE1!Gfp?AVcs@b|p(9s$n&$=cV?aT7>&V@6cx{~YOYEiN1CyfjLt5VWSa&$T#%_oeLW zZTsX>T1jrf)&_Kh9kmQnG8D?e^o_r8l!kK;IN?Wgd$*k9$2gA6Te+okhK;Z|-hvFu zcTOie=h1X)fGTcaEVr|t&bIxO@ zykq?>OxfiQAP^LWrOEMuWF4GXv(f7L$|us@zjG#Pz<2J;@{V6Kd4aw@Z_*0p#?BnN z{GZ^Ovd|3tGvlo?eOfeb#VGDApirtXZb{EvIHZVCF{>N?#$Ow6r)^_E#FyOFK6Bzt z3Q2rgHCReYlg!|X+#*y3o!9W}ONd?7_{cl<>+A<3P=3T*xwByIbVNR&qF;xprY(FGA@+kd$s~RuI{x(2zyB5Nr}=}MP%GT%35dFQ9Iy(j z0lb#>g+}_MS;T#rCE^t5s`WbO1hD5LO15pj=KH-S7)yW_=U^1`XwF|JCx|Km6~p9* zIc(sikgZ4W2&v%Nn>5NQ{OTmI%eZ$?gd=~nCQaPgjzxbarldxC9CM(u<1V@L)a4bJ zTh+BU^v|VZ|Muzjx2www%t9i+L+)cnVa*fr95&QL~-YJ6G6eTp9$Dks>z@YYdyOX7AVjn$eVaBl3&kA zIKZi%m-yKSRY+bLDx)MhDbkX3;JehdN(FupRo@2alE&&RQfTcHZT#i!x8Hs{=%a2+ z@3Frvr&Iezy~!Jep*^K#CPh#DqNELSE2ZA8seuh{VBQ@-<_5jF2sPvrIu=m?;3y!t*)+i2-53I%oE@=?Zl@qPcs&}^SE2ny1yz44{ zq?gxkitmMm;cgbM5rD*0ypNf&%cW2I|LW5;AKYG;8rF-SFBJn>Z0#4llHo~yWoqUt z{c!jLK7<^$S@sk!pds4}*A_jP*->cDb}|M}abwO;=6055TwrTR$$YKVJ{Ro8quenHhO^>3OMC>MXfAGT3)4XDW84o)bt70 z+N>{YDLB&OdI>MjBC7U z&2aTSr8V21q)*d@yM8Br$@yKLEXYY;Bmhwhz>A(Q4_~QG&cwPUGkX(q;nBE8dLRI| zv`B!h8@(68hkNk{w>lCy^EB-Bl;X!brZi{! ztGf$->Rz>$#c73=+tf+j{p^MsMPa<__2Lk*^arV%zOuTzPFHu08?z_xo!r@4D7v?x zq#--{4TAsm%uP-nEUSt5#|>lnq3mmnCuvAyBOpT-om5-tg)B8b(pD*C9rGU^{$;4k zyf1*vz3yK^(4ECio1!jFiH9_x&%aF0+zC!tp;D#%qzgA6-Q!JeWpWD&Lhp~859soE z?m?E)J9NQNI1%H75pprtGN$?*2^NqBmZ=V$|I6zz?l01f%#J`f4qXKPsxAF$hocSw;=*bGH(DK>zyJvnf>=$8V;H^dMm;G(P1QH-g zrJ{8eX#O5b(K^T_Q9vgGn(tq~etltn+Lrq>Q<^5U)&c*mc~mz?yB`i0@Qfg>ZWKTJ zF?UhrbaR(R#bN7_xQ#t$yl+|91oO2ZL|n`s;q!@{A^eu_^hT~wwY(MD=QbJB?L$Z)3twP{qT;*BP+0ii=WZj)bRSHQ%H|H?9jeY)Jq&P)uk7nGdl(b)S zoE8#G*uNXhSZ1`Kuf#BLT?rKNTfd%6?*~yV0F&QSMO&(1o^h>SnV-eIUO;>1Dg6;T zGZzmC?O#Eu7GS|)A!6sGhFlV3F9(OtX`oMuu(d!es*o3?O@>VAuV^DxVB)(X?&6K!t*UJ(j);>MY*C zY37%2(i`LDh0%XopIcC~{~3$pl;@*Eep~A#i_ZJixR|jMdezuA}xcob;?C3x8HI6`5Jp^rV@U1^z%5-Hj zKY&!yH@UTYLDM#E&bBK!JY$_6Tly+kBD$eTQNTm=;`w+NHhKQt#kS=~z^&~T=6%?| zRPCj85XSf-EI*;j1yM$IBGinehW#1Elm0fNzd!~Ukm^LwQchP5iV zaB3w4v(mRsT7iO&k1XmZSBv+fFW%7FbNC0d#t>s`Z#G-o4k+hZ+?aJ-QO1(98K03t zL?~4oF?+|}VRQ;b-{($3oKd#C9sjU*pFYOq_4?(k$QaZU9(4bZGndCcubZBveHNO; z0QTl4vdzpxIe4@0zyGQoKR`}>l9?6fN{jJjI4NmrM-{JQD)@U^|3z})yB~ZoDV#B9 zn3$oiB^FxK?SnfzjTsXZ0p~G*>+AhKw*wOjo;SbjXd3Ykg*Uci^o{0wgfuLOxW6HT zK$~v8(~}Zx`yxPecs1g4_i8or*4vSD&}~Ji{jBe*6x>|G_e8_e?ak~}IP0VR4pmpk ztKmDa^QY6Zz8b%kN$ZB z*5QB;-cJdRhV~SjB>s+`vQ{51G~a}VBqwMh;oghA1Mn0=ZJh2*cWgIQA~v$xBw~|F zgM@OohVQ2T!amjrT9}KblPmTKHTEk!skGNq?feW@u0g>6YzF-fkTRzYC&aZ0J75?9 zh{uAJJ6)X6o9_+}t5>V$U4I{t)yk)Wwm9`}(eux%_^`bRu72ey*Fsg#lbOVt2^ znE(G$HBcd+e3={;!tEAa>LP9bZ|DpS-k#H*HYfTKidG3tWmoZ8$Q7!fDut*zUkYQhK4qS1t25If<0l9D(4i4W{xrz~Y zW*-t0VGeegyk9^P)bK>30(q|1qs^lHADA(I+rE3!5HpDRDsPw4ciH{^&~Mz{kPGWp zuO{C&--*o-8s;^(pZM^XvNjc1!eUvN7S!YV_myTAJQA`CYu9A*uli5lJ|8o4*IE){ z&iuG{*D``jAl`B}<=9js0XW*Z$n7zhyw&3QJDDP`1Ri(90=JVzZrVv`v9;SGe{5w`h&F9GBfZ>H#ra!WQS2_uu)3G^qVb7>GDT6IJK{!4eRfPfL%ztOtrnRCGpD1Sv${p>|a zOZr{dGG2E>y{jAcvXd_K7n#Hj$n1V~qB*ay& zx|~YVfdwVYRVSePbz0$T*BJjT1(C|KVS0U;8dnfPGn&E4J5q7PNwaB#k8s_aok_lF zC53KC2}ciai|OXD>hy+bl60PX9plua^|Et zfz5eusuCag2x8V;|KvCfeP;uMnNS_Q5Om0utey^(#;3?*Kzvz-jSrD&`s*kZZHjcQ zTCGBy6O;EHn>nBus#m;cW#!*V_=+yVWo1a4Hf?m=1NxS-)j?Lb{2=8bLE)%-D}_9+ z8fUYjpg+>~TQfTq2Wcck+@BE)Uj5H6e;oV|J;qq}{^h&_6SzAv{yJE$%xBHs*=vdK zyXS23ij|*yJv_F#Y1x70q@3ie&&(RMH~Y4~2J;@lcl_@;Pu{CRtx0j!T*#QGcPsBzK}wJkr$#G!s@f+#BvLk( zedv?RK`FyX3B)uPtktv(h}LuZBMOn?KnuO0L2L(n*J$hg8GR(mDUZtiBnS^~`(OFI zA^T5w91-QooUb>k*yr;z^?D*`k;bajQAoJjdYu2X3)H}sS3Ud;&Xu zQMRK^=%9mt#_x4TZ%6AX{>VX&wusnO0J^Z?ZVyQPS1046Gqd|by1_C z0|~1d8{l^1Bnb&i?gUPq)oJ78F#fF0Qk7l_MkK5;CQ76w*Cqx4vOZotifb;sx zn97YbZsGXRMoU!pKQCHrDdFE1g*@&dLt{rQ3}^mm89T<h z(>Yj4p$yTNB&I~W-EDbTT%qTC%>=v`shjLm-WF@p$jWM(ZH*p&SJ#XhF(Ql{U=?&_ zF5KLgw5e0~x5CG*eZAkyk`uLV(C3+N*PTSM)-S(rijQ~B9H8@AO^pq=WhQ5P@RPaC zO8ce}P2CO1?TLFBnw?!bmWU6`8)|Xt2IWt>dc~0s^!(%`75w5i97`py?Jugi6ynUW z8*B6(42~`0LxIl<$HEU@2;pRiD*+}kKY{hKOA@{*e?sUoK!{j zO7KvciXOaUhvs$ZWvN~uD_BO|tu?FFp}#bi#)m=SbZ4F_xrjgPk< zxqx0&E$miD-$J}n3`j#j-Xazv(-wf|5SAG+Y5%=_iNU5M&o)Ray5-b|&b}?Mx6R$G zO|C*;!tS>TK?W6bHM{VKW>DosMOFWVrw$nZb|fpyp~8-4i^bRJUeT3~oR{taVtylSMF4QCyW1x1^RG z+7Q5?Ia&8A?CPn}BY6$gdojX_T_(3}yU{$f+rE%s$d$jFAUA=(`C9rZCMtXyZiV>8 zj6EWZjn6tS=S$oQ>kjQ!I`Pew?d9TPTn(t0Q;E+>c`lMCqvof-o*O}u=^45O$rrVc zGJ1vMpciXB?Kc)}RtmBfstpz{b)sZr9=1Tr$}W~GgU|z#GtyTHZToC?KZd`d29T4^ zm=Z1%fPJU@;S>;qwh`MB*%4BYepTw?h5K9R##;pwQc7wzbPds70Ek-cg`8lc^ou{> zB;VR&RlNsJg>b`!vehlM+)~KAz6%29WX8-(nK~E-atzx->qwvD8TIcor)72ETt&<- zqECGOBUFu}pzmaEq@qQS80OpUlYk!Iuld>How&N~Hn-rer6wMhX~q&iC9hd6{X?aM z=GG}G_dk~IeDJ}5(ZQRx4L1KwKa~tX^z_`2f%6gz+r3v>Pq zK}x;ObH^0<_|~eGC3nDV!Q&?j=aO7CdSgI97$4+430IMZka~RmXz=)%(kHX{vO;BF zUgdOVeRL-OIQKPgFF2FZAnAgYhD8#3-q?-X54flFQIv}WW9_FjKXv0}VnRHq9LSJ3 zQmt9z1V!3#Z)#FE>ZF+IOBMw9ISOdKU&N9+I*Gs`#zvt^(+dx4-LT9}OjS89Sktgn z1=F3o5qKczt$*c8mCliAm#f@ZE(nLN`8A3J+R4E0hXX9Vlhu)PlW4nG(X`e8Mv!mx zzXJZ`?$w!W=pgFw^iDWK4xf=9V)JHCNEIKGx~gDkwgR1|2|2vw?#+kusfDT6{^zmt z0ijNIMx+m%ns?ywKmBH`D}RKT+a2^DBb#Zg7(J?ckXQRL1DjjYGS7b4Z(6l$clF_N zZB8hLX5s@k_iyp`$oVXS>b$!8*@|LI`>-gKYi*DK%+xDdilxm6#nUDEmGgmYAdqqb z_cA^M4w*D0dnauqkU)XXW-vgbR{D|4IX(5gci+9o)oJ7qNTno)xMUmmfHjH_;<`Zs z(Y;JCv4#1)!QebMY1%Ari4P3^<(I7R$Ro+h2}c!syxl2XlS;K<8*pOQH^Ut3K{Uwd zBK7jD2bY}Ijhh$Id*HxS4NPow(m|%MZziuX5 zKzb#UWN3W=WwlxP#Cw)Zb7QnG_D-z%lyjDSXU1dFVQLU>%%cmrk`vJub4csUYx{ju4_#G6 zt|o&20pC7^fMwnCW3MCNK^hU?q*sn%9Ut_=dUP4%!_$~*Y3Kcv_fRRP1rl(H!bmTR zSV%9Y9=Ttr`;JL#?lE;CIxPxPajeS=xBo$J#+={0a>IV_o-+qEj7gZrfLl3aQJfzA z%2Pc0l}GorjdctDqvYHwWrDt1QYQaZ?y=uzOfgSC_D#5k&n&A$y0+({6Cp40pO zC5S^sx)w5>EBGDv^r1_!HR0s;dwL@mp&SMnkA7S-PqgC)g*zxGL%kC@Kieg71k@%f7TWU_hU z9*Q5sN&7Gytg*+CI?PQI-6d|d^u0t5_FOvHss6Z)-TfCNs#CD6Z2?IM0DB+>t%gwd z!$ZjiKVQ5{jYe)ijojbIcdslrbK=}JC%*r&bbQjviPd+0XMZWed;y1g3&)xszS{u1 z^cy2BjgvBFy3_}9#R=e~=uo`M^l+v_wCxj7YUq4DpYr-Dnh!l{bvxDq!0($#CIm8ux6J7YPn381S1vLXoe;kluJ88nD*^38Vg$6n8@dKZ)Bf^yKA zhX4BOIr~1n2Gd5rVNQcu>o#RF;1XUvJ+LmgtUIZd&?BuYzw+%Pc1PLtTth^g9==0^ z8m^TheBWi0w{Am5s412{|8W2yngm@(_}Q)F-qJ_w69_E@10;?wGKOJPDXE=Fb%?LJ zWa(06cRuNKSL=mjanaLmOOz78PIL+zsGk zy<1xIpxfQ~yRCNwPkCR`mIU|rY6Ne|O^BON(NeH;OK#Vcw$_4C^~<_hF}|L@D*I5M zwX26J8O^cJ$rvlI^cyzSEei*#Lp4sgw=QQ2b7tf>OT>#FxUNj@pXe0DSgx*aB9qcj zeupFwDB5V-5QXyEUmu$)U{0%w3gQYM7M_*l59}_lWyYGDZ#FOizg}M;F3Niwq6o7% z05G@|9E(s8shgqGpdf~4|3-VFb;*cecbgQ+FTQxuuCezY`#z-u8>HM1aF|bf^Ze(C~06Bv{ah=*}v$$ilH`^qbeL?;Az&)%?q$1Rfl<(S|=k8&) zxo0!WcukrbUwFpLsq)L!EfXH{1ERXvqc-$V5!1@`1a>`<4i>^ErZdIz4Ji z)f81=3S-EeYx8;nVfY`tzQCp%SU~ioi)KqYm_XUVFC(GKt=3R(pmHQCv0g(PBt?Q& zsjS_wA;ZS3L?!!;(f`JRx(+94Fo$ai8>KqaMpKu-0B%`7HsmJ%zL@SrTa;nJHm8n) z@s}|XtwYW(FzonM=)Hz0E+h3}c})0vXL$+r5yr3OXYrtpL`)>cXuS(>!iR!kg_O-U(JTYCnCBMYOk$cQ*P$O*5xrmBI z=97@ww)-8HgiO|;mK;uFi@FhoVzZH5SOO#u7aD&@ux_ZhlyN|K2owV#z5 zI_MMGmSixLaOmE>d&hSx3Qy0Xs$IT=f=gK0Uu_D0g#_+C1nPrWAa7~IF- zSTFtKzY;i5EWaWNoa@5*(IcaG-cxY8zyE~BPAR!L*SkRpjC}C?AMFRRSYplk4qhmi zIFAxXlx}4ln)2%<$JggDHEG>C+4)tQ3Aeu4TxF$ay^yzm$zA5bg}t~#vz2c3LdG;| zcHX=Jd7tDzGsWL_-P1k2zL?z3zeN1QCe!E7v@zsbXmx55HoWwxMX_MO$db;&FjW?e zXtCCdN(8V$hxMWs;iol$y%c+V4QY}td3O}wspG=pPO_emCm*DzkIho9Ve8jr+Cg*d zU8z{eH(Q$0TL37(TcgG}1JR_fVgs%`VseV(eS37FZ)=2SJFx|77waHx>O`x4GdT1I zRZl80kVxa^&39k1QR~SRF7aY==bhj1p`5(F$JJk?8hrYCv5pj2ckbKU1@qr#C8X37 z$_1%Z3I9=ipdYA$nnwgrE;F#^OAf(B!u~+F59?Z2ByR!0RHXo*@@YUEieG7YQ1H|$ z8qz?YrSz_+*zQqCWzdP+}jcw@Ne>Ek(H99HA+T1Dr+pwjXBMN0=d_pZ6Gf)+bjU+>y5(w{+|vSLi5$-&DbSd|vW zVb4OWQ-s6%2`~gC9(d86(9$;k3A~Q+jcEA+@pN|?of4hAc8!nDq8*1>(zXcIeHk~9 zCg*i{6nK`Q*O**rZWa9iLD2wkrrjOHuJJc{`WbXoChpiHwfVRS6Z{Wp8>&osEJ*4M zdeR@2r|p36!#*s|msN_5q0wg_QX*$@tk;g8hnLy=>8Y$2?${l-!dDDIO`$+VcI)+O zKlw^dKvkTwG6yOAf)#k0562 zT_bqW%0B{UHn1%GYRsgA0X@nV>)`DD-pF0^dr#gw@<6?5ds{MbwXa_T0N$hpC`D2d zCo0@nqk2RnXjhQ;Nu30}EWI&F{Zy>n@L=RM8J6+&vHK3e?{!XUSLsd}Z2{u8Y}u_0 z9|gwH@H-@cp2q-&6(~D<>BhmzXRe zR(#D2t>a~sx;1rb>H2X=?0{pRyxG2oR*DYUD?9QR)kE$ zD@1OoZvYK(8Zjc%CJpbnfsR>xUEI^5K-sQBK;@o(=)r_OrAn1j0yp((_m$^YeOW~h z8KBk2+{p718Er>!#ud3=S!)S=-SlXHr-pu_C2*QR%nAwJ@QZzYEi#dV68xH?O>{Kw z0p-i&)j~j<&@_^bD#^}v2;(NF9r2{f4~gzLlPx4$8|aZc%wkgHa($ByF<3IVO~m2S z13-#F&WWBM0|WajUDxw5-ByYD0tU}FTQ{rhe7F3FYB!XJ&Mqt1-drrE=JyDz^khiAdhB7;}4}JPaT{pX76xB)}b$oY!a?=>fcKW=|auryQ3iaNhpxYDSD?wAo z>Tnn1F@s1}z#VeIT`qcc@U_04Fu}x{6JU=quo{7|eus7|cMqst)sjawZC>4iz4UO> z%V=gpi#RMhfO_BtfM%=~g#`=hH`=v@t7X#IBGx>;0`iP>9@tDS_gd&4N8aQ$zIZ zIQtyoznxj{odQHSlM6;32H)}={SyMV7h^T5Ko@df;Wh4iPw{&0I)PzTzQex zXsu_J#Jf%OpmLrwL3amej=sc=Bis$Ew-g2=j1M=bCr_K4&YeN&k+~NtOdVGHYUjbT zx@{iVx;B~gWP@zq*fSv`-P?pmSbY56%x*O^%2@T8;U}}XQ$m-a%Uf6IBk0|lH~=LB zw}{R8CiM#5j&0O6+vrEy#hYH0)q0cZxog)3kqSxs`NHu&Y{?W)>3oy>JZB(MeQlRFwo!7d zt*mZ8w8QBcg^gru`)hM`(pF+}Tb~3>ftb}7)eg+?L8W~Y?ou}tU4skyfen#^AqpX+$D7{CZGdlF z_JP+Gp$(mBX%63ELAJD@dnagYt4edCFzX?_`LTD7ynZ(FHBrnxo~6_7w2|$G?G=~Y z86pMC!;V6x`jAq!DzzYh`6v_kZfSL=%Dj0l^M6eE_mI=G$fr}xt67=|niw?Fr<)?Le>b|0OShemOzqZ^CulDp|>hiZ6MZTlg$C6(+zlSO6w$od! z*)h_RGiOwK?mL!(XklH9tgDZHS=b>Rp6oxfCb1+*TzDu`$A!y@4di89#NHtbUyQA^8O7S2 zpY~-|lnL!(&5D+kRsX-~UHLWmex>GNS8jsmu+zHZw{#iFYn8j8gX3q&7y9(-<-pAy zdq*A#6gIq3-=^Wbx8(QZUX%8P_Yoel@bz;(IsalFGJdh5Ot|R}Aj9+rOisF+88`LUV&BemZJ6Y z`p5Cw{!qJ5K0dwmQ8V|(pT@tu`IjZX4^C74*N=a4&eXHUxR~!;^sv+Er9WD+@-sc} z6V|sLzh}vNixL@1j!|PYWzGY8_dWx7YF0xDjT^JBsB!U(qG75rt=ArnNI z9YD#it}dO#rX&#&1-X$v0v~M;{bYlKJnP109%A0D`Sdt=tds@=2B%13=5g=aoGT5) z5|J`!`yEQX08h$48TSI4(o8cwIIj&;lGa)CG~_EIXFB%020AVkV`@T`cI|ln7t%Vn zs(ix|`b|s*Z+jidq9OcxJzZ;k2iswu?KCr#4>Vm!9&x zme;`KdQq0n(5+fOov#5e&Uox7XR&XU@pZos0>cElSSBN2-=fZ$U@8nlIrv`RqC_A@L}JXCqEl%xR$58tvMHTGC2f&#OMTmo6!xVOB6xD!N8MLYw8QP~Lm zefnC@3W)_B^nmgv$dL5UP8TU>YUdq-Xf^jU>RX5ZRd4(QSORvmS!dF#3*EMdQf$NpF{-j3e-7bWp)P)I@_ zsO%+`8=4P`Wro_pkGf5=K^lS-td0b5a75y>I9?_zf^`Be{vQ=aqFfWsp$Hh!Nj}~$ zj^5N7B-Jp)Z(l9(Zo!|me>rE!<9&%IVkT9!gw>=~<(Rq2%z%Rp$AL@sZ`jv_XCh}# z@NW%Km1*yzb~MRIQn!Srl=i-CgwP zK7|V-MjvuUk+QV99`j;pWnohMj(!=p<%NV&T7pADr|H>R{9n{>kGbnK#719YI4~XF za_h)_cT2w5WHd|#F|?DHJ2Yvs^U8u_o|VPt#GAyQ!gDG2yPmCP3F>ob9lN5)vahH%k`d}~yBBwe ztpZY75eupCsIFJ9Etu!W-*tCBGN&HAFnXP*8k{+%^L9K+hyTp4Xi4?K9+deDGsks+3x z29h{3ei)y)3nE8qV-`ZyqSAe*5}kUL<30z@tX;QG5=4WwSbR2P7ywp6&lRDyfB-TM z+(`8f6@VEFJv2UFY*QTkbKo}#mz3T3RtD|5Td`RBxm)6kEz zm7>xuuq{@cw%_ZiK_ zY}^Q-b;a%4O#XW4T+L@qUhTr@H4Kr=z*`=3!s>3TdE<}36R9N~X2rf;bN7H&A0dFz zFNcT@#koH`@^y)S4dHo@O1hr=u4T(w0+IA?8xsENzEcp)}j>ef4d1?SRc;T~D zbq)o+E|$H`*k!g^l%;2+54{JMr{`xx0^nh{cjIhH#%Ue;uVG>8o>gl~4s@2390W%B zjc8RL2WajE<1;Im8EuqY^3%TQNJnzR@D!KoO?m5yxLxoMFWt`26R)le0~oa-p-a3h z8~lW;IuYw9M;(3esRy$!-=TFgd~g^rXjqXm-*P!kBsm?Z-?u zn^w7{wmR}@R%%lCyzAgH^ITzJe617dqA8O?V`UgYD%iL|gWxMsQb!<5kvf8pgsCa) z%J`prdYGe5Go{Ut5s|v6@_&X1YSOah>0)t*C1Ps+tsA}T@ZB}1V4dp*0qlhC3I?i|faY?GmX z6pM14%8AeNlaUXoteuqh*WIt@#qp<6&)`{ZY3K$^n7_Hu$x2e=kQ|?8Mx0F>mPXPg zqMI5Salg$@pUjL-TWfz%r=WXWL*Il6QzsqVvis(Og5p7ZMzQ2Es8dtx+&31LW>>r% zcWoQoe9&9IjS$)@gX;e<)h#_Wgf}L=J$<8==owBQaa@sIW$r$aSa51uXrWEKywf=s zqHv3-b2d54m!kR-q?Fe;O)lgL<;vw5MGMQ!H@omZGDT^A>>exowpjZg`3>7Cb0p&qKrvgfnjW;&Fejs0hR;(U^B z&g@h}reBgTB)rLMLtnTX{4*0#{_QPJ==JvtdP6wD-ED4oO1H2dRdOzkxrxsy2r#kI zoG+RgANUSgeYD+XEj3MzybM`9KEuUJyZRCDv^P<)eT*;QrB*b^7XUzOu1wcr1v?dM zk}|fiDf_)z?^{6|RKO1-yL_bRNE00s_CZ3sZ-4yp$L4cri6A)}iv~G&59oW1Bh5xN=VeiXW+}dh`BiRz&czTvSVb zbDJsmO|h$Kj*%F(R-5sfq74F!kf`o$b7@K!oD@8R9je zO<|MpkMd9gAIweU=X9g0Xqg|*Bc8Y`K0u5;!E|-=x=dLYO!Mz1OB13>6PuqCMGuLq zZ#b>oz7(J#(PW6b|Ca41z3CeT+@D4PcX9u5G*Le1;G$4m{F=f+l6A30jTa9WtR45f zoZo_#rZ3B1z@I}@_-l~>j;8L=GW5)wpR~@|fMvxx^U4VSOIvH#t~ux<*E+r(yVdC# z+V{a}I~O6qT&C?Nf2B*+x58789}DY^D(U-ue3rGvu@c}0alm}2e|u!plj4sgO|oJU zs_xyC(?l!#YYJV&EH-&{Tdpthy$X|2MzB2t70@o)=q|G#P#tsss_6pH$xZgag@+y940Fw=Ko1 zavD-&k6w3xBGaWVFkCnKG8#I0AgB%F#*g<(B+br612_uSO2MTHHehlW41=*U*Q+5j zOAwDhc{U5LT97lwyOsvL!N!_-Vej_TD>6%9Iub5$k`F2pm7U~oLnekEUVa0`mhw~# z9z*AoiL*OOA1qi$^KQ%kZu%;w9|dJAAP#L<7bA{z?)L5lawiesrL;GWnF7ENfe(p1HcFeRI1y|8{Bi?z=DB6JS9pDh zFBWfSZZ66Tm0<`_wj?#n&d$*DDZdP?k3izWvJBra?LgsP)#^Qc z=$G5H?+vuqi3mM7t8n^}aEKQPA=aKhkkIZMbc-^8CGsceEt& z(X`f`FE+-aU%NfELnSK)=^D-GTNDP$E9?2nMd<{bAH`Ib zJzF9r4fL3m+>;Ky`lwZ_L1JMbjw>~6vwcz<{7l=-V=e5*C&8DyHT#fJe!gUSOu;m9 z^kn)J3X~FFpcV_y5!)i*^4p!TmbNp9He7xA!R|dte_igm46t^qXZz`n_jms^r9{v- zWjcPk(*3=i;|G0COr(CL*X;1SsX8?p{1WOoe93(mX09gTrz3iPv1&>`X4AWT_rK_c z`TX-|{k@Cbc~BV>8|p?$XOo#5_S@(YWRm|_uZz2hY)}8JIF6HKFB4S(jl~C56ji~k zAks=n`($*m!hO*L51D74O%OMR9`O78rhMi2flNR1M$df~R}%ohy9FjuCWutf0woWS za`HM-XN&xZbGQ|DgS@@!VC682euxj+4I!l0^(8f>or>Cl7D$Y-pneg)j8pc5@p|UB zR_1GU>_1fEr^MuVsI()jPxS_?ln7V#fY+iG0-H24)&DwPFsVI?R*m>HvzP$5BG#l9 zHH9NN1~~zz{{H)%<~(4B!nO_gVC{3i9==YZGM2>7OB8^~tZ{#38B0)r2bvYaO1})J zipT@-n@`~pQ8P-Txv+Qa;p}d&l(4a7%jCv>wZJ@x!tc+6fAPL-FG-SbwhQ1?Gde%gOhjS&8&Pps(%ES;;pX$pVq z^UrOd^3ZYz<}E1WTY>F{ZI}T63U})UU!){7+La<8FKr@=O?Z#TY2TTgRYly)5wY;% z?zCu0tk}VW2Vr43f(#K)e}Aj~L_V>?c|T&c% zap*==CX_b5ALxC6E}wdPjqriP>Z0)SmHl#&*LJ7=Ddn~uZtdW7uH5vAgTn*d-+pgmCt>T521^=E)PFjB zKKbs;q|dD^IaV&NCF0wwl1NmvYxr5`B!mta%xAIgwaxGP&Gl=c0e}3s@QhuYa zp7tq>?XfK9iFwSR6Gb~^eKRx9e^RA=yl^DQ+CtCvbb-CaXXF2B z_H^3+r`h|PsM~cy@1W%v{S101tuJh|iNfrvAEoPm?E^}J211UB>qg*2=I{GHRaY#W z@}+{Wus%`4sgH#@OrKs?m&~RYMFf7$;o~&x%cVP!1(r431hl3Ht{_+qjfkgBcKXut z;AX}Nr40UtmGTE%{46BFc4JttYKy8IJ|DKUcF)o;@6Rup=fOW@MmgKOTdmh;Gx^>u zmVW8&r|UmMEgioI@fvnZL6Bh~ySg5j{(T<}eR0)l)!K;@QzovWr8?|~$+o(|AF-eH z2=<_9%hL;6dcswRA%%A9$(5%E!#w1!?NubIk8RPmS>&VllGf3SI8&Ql={o> zDm&PA-haI3`2)Fdw7$)>UP^_ULebZx2?&TE^n_aZq10RT5P}ycA9x+>%`mPw12$8R zKpqJCdWSlq_2ycYi8b%KP2l?Vp=<$fO#GF>=kof{_qUyu>4<>zLMFI@&>UgxItn$K zF%HqWJ@;xyRVt&Z$K+MNlub&hhOYEPhw4O?X}}o*XM5gPwZRH*VqSf^Z)7UUjO21(2{U7x8C+ z92hIIZwCANuoac7NIpbxV?2!Rcacxh?c=oGX#r>?`iwv(?6r43$7psaKd}qM$1%C27WPWj&0MOE*?HdCC4uH4@m@f+Hw&>ud{NL>Bzhy&mB6 z&$6$ED?UQ~G5U`MHQ&{DUfcfx%TP%gbyC{p|^6HUi;y>m+15$36!rQ~Xf7U1x@t&8n%3sR%-doUuR zRoIuv4=)=fH}{}7O;zYb)bQ$_T4w)`u&pYW znY6f*qOdsf`K3~e8(%K{+oUz71BzesPMtbgvz$LNQJs1FO*USeHbZ!Obn$X#5vOdw z@m1RMWuSIfGZ(viuV}oaLxf#HDT{UY@6IOYts9i-+~{1^{v!2i`ur=m^nffiKD0@D z_0<=a_%%nfKDDgBj`0Y6E=2TMk8IAZ;6>6~7V)jKSxLD9Ofyr$(Kux=p0+YMA)!|o z?trQz;g0Uf2h4V)A(Kme-J894O74)9LD5<4COk1t&pOmrVO2NJojW(Z&W=)+0!-x9 zHKqr8KVBX^B`rJU)~)t{$x4)bO7EdXNG|DQ z&#hFG!VUZU7ZRz62vXZ@Q-+G@dU2KEh!}s!20|yf$M^KXFcHdl*MU2lbj`Fev5o!B92PYdJrK^bd=-2;pVN7 zx^G-x!oSWLeG8HvgcAbMk4hrLpnBO|`y#*zDp`dMPUVcvzE8bI_gT<@CO>@q#lD$4{(La#xA1fL5kt=>j}Eb(HXjkAC60klSlPb=AN@?*RnQ&wQD^a* z@w%oue>OGU@^As~i?5vnEv{BXzlk*xmU;{Ii)*7G}a4y^UG zC^`hJLr=nKsM4M4bqQ10j}{*cov|iOyArk82CJ%$G97AK*)lo5hzOSlR7806Bm%lX zL$oJUtG5YVcyM#Ek1H0^p<04sU!C3INXm#x3l&z*`_X((BCU2ftw9)1#dg43>4Cyu z@b+mkM6nh50(;0rv7><0i)PxVkcrxi-`bzLyk^|+=oscbp~_P73Ud@XV*drP@Mko- zGsa)aWWH5I6-D;;qp1U{P`}P-Cm1BaWq0_RmLKP2Y%TKV{R%|jA@^1TnnqgU(55q=q_rWATkphU$jZ~Gm3+-UagQq zws;GtAG)~Qyc3Nu)`L6jO*x3Tr65)tH4tD zA_o$(oGSeX3TyekGl>^Ty&`uqo82SR;&=~oTS6g4E=>X6xD-2Yzm(n~CBx$QXfrta zzotk831;U0eE-ns7c;tdFCBPBSF?Q?>c)}>v1{q{X(K8({3gx>=U?7* zTfK-J^Gmm}0X=t4IP7qCkgt0r|B0n+=a@##lFZu@Iw{Xn>((q=_RN1_1#s%?dhqUz z-1jW~*d?`JN>C2bn=x-zjQ>w60!~gsD8UC!zSAnt1VQid+BGF*d}H@k z_Pns?bFHvb?*;SeG!szG;#f)yPOJZisqX;CvhTxIiAaQ!L`1_#Muo^slI&z98KH>m zJ<7;lAuS`49g>vD9-$$gq^QSv+c0X?H6Ilw&w(% zl)*eAF&p!O8+}CCrxl%L^TX}bL0i;oQH&F1>g&K0K0T%)g3@Eu@vD41_9dv zNf7gb1vMkKKZw6d*z}+jQ&@>7ym$V?wxrr(G5d#lV8b5!*s;Y&w;ao;HD6T+a{JI|8YS?ghpv47`-!10@NaLv!G=lkpI`G&}l$XNziyi4t;nV*CdS} z*923{#B!$+5T|0q4&pSdMEtj095?JF*5PjSQ-q8a(S$h2_+9fzGd#$ynv#fj8>SAf zbM^hhnu{|Vio!I;|BG!oz9S?o+-{~tFp(h;2$W*TX`MAO=^awA=~JHSe~~mJ{NXQK zmpyZ0_jNkjG+&{D)}!?+SzP#p_>8BSQ2^m3B?4P8dX!IKL#uw{Xq7%uz3@Ll3wy8!FWXJL>IlqUkn=3X$-#B+LW|hwY8E zQ*?Cd(+Pyu1Zk1roBwmsNIFma`r;~L%>#nJI%FhIPrzORl+(ukt8=h$#A!WGj1@G5 zhKF}RN=I;#f7d7boU}DM-b=16*_(bp>q<|%UH;gyrAhs|x(RX)tP&%9lF z(xd!t>Ddu26}rnIHFr-w(#{Ju;tSy9R*KN~H|(CF`C4Xr_j#FWf6myfjb@jH#PLZX zx1179jO8Do3+xtAXE(PLd`N{U;4)C>J`fKCRF9rf#dBr=vI`v;%a@j(fy8Wu6!i$r z%r?y+-+;8?jEsy5B&iS#m3JGhdtJVKxiMaR-*$$hw_%FBoBd$Y7;LQWfN3M7{H7)# zuC-rYqksg=a@IW}ApVN8vj|EBL&rWQ642YZXeD}!U0Nt3(VEe!S?oJ;eJ2#NCW~Sd zI7FRqY&ejkvv%#p9~Lm67i&zAWK~bsi15NN&cQ>6bhx;aKB+uaH2_@{A%D<&75#?O zqqRzhBVLYncKPZD;gy5el?#A0*s?VMG&ka=}O{oE~+*WBZ8V?{DvX(husP7sD; z?ttOhamL8h_RC^a>`BKC6spt2-C-#S2HIIkGn4j^EUcsks!=1EhyvoYT+OYTQ^3A(+>C!S5*Vq72 z|Mx{?TWy(HSOnm+BaMBAf&_S+^51iCAG%WMqNJqFz|wYq`*2NA_UzdqfY-ISkvI>V z0%eWz#Bu51Eii~&TZP&!GCI1Ii!AIA@nOu4&;>S<^;}RTk{JE^IuyV(!1Ti7``R6`S=i@gcU%gfqXcKEpN;6ImP zbC@NOiy6Kc1!>)E&C6VL>`7rMuU_p!k5Y?X^r-UPsnOB9#GQq_*K|dfeVh@H3Ytwa zW=YT9NvA};W3{MT1Ykea37cImnWPtIjxrd7NzpE{=fcfttjA1!7{Z73S4QGnD07O4 zFrp|CeE0rdY^*`Co_8rbDKSXlkic$a_wrugD+`l#DGDvG|!Xh@?LGCR$n z=iLCqt$&RwNymqW)6|5qZ1)n|dwNsht4>gLLJ!wd8#7m)QwLRR~s*f0ezW zBJTW?1cgQ=I8ATFiNsg0UL87fg!r1&RUI8#5u?vu$hKYrsxX9-vKI7hD1>*x4hQKd zFtL9d4mSIan>OBl{P^;|bLVBJB1=mp_1&gU;wF(}N4n_)ZZFCs ztCKzb?1gzjw?gDjS_di|7JQLqBzEGd&bJMe^+vuoMuY)-nJeQf+e!xjE)caJ2W&mF z8k?GEP+ZAkJNa&Qj`41$?lts*c~}qlM8sw@nNK-oGx6QkEM^xLsv)r+8XRP+!;;@} zG&G&5cW|xfAyMMQI1s4|>d?-YCfiRig~lYYkN3*5(_&|U6`D`L-PWLjCP>>hqh}z! z5h;HykB^3UU*wT6Fi6s(e0)ulg-mJ|nJbOp)oVs-nuxl&IM*B!6% z>&q7L#ElI|+qzqGJey@Rty-*l;*e)-K$^HZ=PvDBgH3-Hrz|uVA#np9PS4L2su}R_ zQ83}$MHzzHD+1`IA7tvfm~6uf=WLp-0jb-{9F$F)+jt!k%T=175(RzfX(-Da)>1x%-OK7c+2=ExrxJ8zVRTad&q&u(06yUv9XBCjMhqjt;88n53sKs@mEvMZ9p! z&dAodbjc6}(&r!c-m4J3Jy-~22$cOkX6`dm68E`My8S-!T!=GY)9$aO_^W5hMi-dH zLcQ_q(b9@cs|835KaHl@klc(=&6Ku~3l#XkOmGl=AA@#Q71DD*j1(2vf35}Pr=9E~ac zPDvAfMr%y7-|WoYw7%rE?vIS2{p4m8@i9cPjGVRnwmgnGm=8@nf@ur5@DNy%6AO~C zWQFjW)yXn5HueF$@f9YAm9RfGxOHowCX!C=97l`-k-Un3eG+4Yt9AAHFbKW!pl%{~ z5!5=;k84sBHK!Py+D^)%#jM+PM!#3iF$e!5Qbc=OU{>2lWn!ZQhEb4 z8VoZfqNh$70+(VGcYW3M?1}6(7CZ-)&{GVJjhSiQkB<*SKmpsd@4sWC8=RcX0X;7z zzK-;J&CF_KBj>eqxUC=L57HF_RWn`Ehd9O>@s;Q7+1B5`|NJom3gKNmH)o3{Vml&_ zf|~fdq9}=9p2s#WuUZdXpzJmZ3fgakf2KmSs@5Tf;E&S+rq`}%b1{u8=;`T^{s178 zS!ILo`E6u`Vh+&LXN+^Z#^%Ld;6GBg36UW&B49lGTY62IJWNG_;m$|n?w7%4VX(|8`LWOvm1buS>7kmW;mX5H@jYLh0xiOPOI;) z4iPGf`DoS7MLv2IMBGB)xeP~oQ7PA6zE$FfnZ*EEX6MIg9DEs;IdE+8~%zq(EfqrL-m z!i-cvFwTHE*r>ge+7mUn^DrSy6f&MNGMkBQ5GiQ^qNP~z`9Nv^I<$*VuvuAwR{R~z zF{HcO+S-V0N4y-aICvRLxXy1QG9Vs`ONBI?2@jvYR$bE7=(D(FIfTCF4W!;|O)V|- z^oJF%+v`Ni`+B|*((dn}Kvc-ybwF*Uqf+|r2!xAGZFie?Y`iVrc(b|%xzxW{q{A2@ zx66FKo!TcL?E+WpcSC-r1Exn9*pnPD9HcYEmsU;JK;fU5I0-Qe8t*ccYq)cGZ{p3G z^C#Le_n>l!yno-cUd&;97t&=TS<}j)C?SKM(g$?Q#Wk$<#k$zrNPiPsI}{y`c)SqfI`Qu=7re(HlA$A6bkc@lJASEbKF4V5Sg14~CfF+M&TINKz%sT_+nhM?`E z?$Xole@#41mxT0MHjuU2&Nnnm(kAE&sRY#*;)JD9ecL9Ber+U(#Xc z?JM<-2UM2DKOrHZi)|MwQBw;r7~fka?~;lmhQbVFc5Pe}aI zwzw4aq1${A0}R&?Ok+TJ_#QMy%Kkvmgj{ZAYZ86>nanoIxm2ac zWPA3+niit@LvSIxHE$#L>hTWQP(B#~#gKj1WpI|&Y~HSbR`1AvwQ5qBO0P58XX0fZ zw0Q^li$eeero93KM8hDfKB}Etb~EWQsGY1U@9xu=Zp2pqZxa*28)=5%h2pl|+o)(V zP%(mEqMU=$xXmbPR}YFV2tu7<$MF*;;Ylh95(^TXSM~M7aJ_~A5m%$9w?UoG7+S*+ z=tI0mR&7(0O2mt_G%28zF-Zs5T|I%yAf?veOa>9ycK;pbcPIfNCH_ie8Yt!t#3=Lt z%Nkce{ST_Z5992Hs9|#p3QlGwx0!%G z=SsUq4!pyH^RqPqNNXgL(H3apgdS7jDP{%z$O~uj>U{pFJL)a`MFTE8Io&@wS`*YO-X zR)bUWw9P!5cqW7pxsc5$=FqkIqny*`N_q~~i2hlMIu|o2{9XTw3AGLe0^zVY!Gw^e zgJLxUNqXxKnCvq%<=#lZ_9vHEjb+s36`zLc!M5*T(|89hu+~|5Sw_KXDd; zt^sQ9^t9Qn0o|u!G<-j-Icj&d(8hEBG7}>@OT5y44RUV&cSm`jNhKFLZUfk=E`4aJ z6cG}THVvw9GQhUdo;PlAp_FU*P`2^cDs3DcYmj)hxT5|GmR!OUZHydy15FpD=U^#Qn zefI-I@8OOzOSm&a_CP@#$ohJP-~|*qvmnd{fuWs)sY>JFV+YyIFHrhz#(s-(>^crb zVHE5Tf7-^mmwJkCM1&0t9A&>RT2AdJ^3DiZ1ZofzlJ<+JKTg>l zo}8Q{{f$Frv1A4%DUaml3)M-_Z`BVd4c-GiYLpS}-#BrBeiV~N}*wQkimVe=+) zN6rBvr{m?Y$>{$TuhHf^+NtlLa^)Ug!v#BofKZ7M{m-!Zm-Yl42Qh2_V%C2M==@E& zFlLH<`}Rrq!~0|T^I_us)Hn!)=Lf(nP1XPz_DHr9Cw75GE{9l%-=K^H2Is*J4x%jJ zzJt8hekCkJ9&e4B^SgbKW@tF^Koj={ZB8$h9L{(2O1p4g3C>4ovL~6B5mrU~X zQFP{FCiUo*qFt!tm`{WSbnhl7C&%3576PBbA0KT$ucTCsJDm%Lny6qpR=EBFpORW` zc);%fW=02|3tjiO(a}dSVxpo(YpcJByEi#C^)E8O5+dtr>3%>nrXF;=DYt^$@I&sB zd8!KVlii#bO&%t|s&I7xiQm*DHe8{qxZClbxNo9?z%mD8wDbigS{}v4h5m=8|3g17 z-1~6tf$**UfRDHS#xat?b_54+_VI6c4ETYs{;&M#c3_PvYU*r@qk2SK7tj)f{|cK2 z%I&b@@|M^Zq!MwAm-p=X^MOQVQBhreHsTm3=konz!=ZJS`^Sq zC6Ro<^(MW1Irp$Xb^2rtPPUo3e?S@s4k5aR!0c8TjrjD3BqM+_oj0Wm;L8A+>lc@8Ki+@H`|8fs$swl6TJ0}>MZ$+jfCCBW+O z?x`egd* z>Z+z&4%FGh7z*<)1}la7?sd0Y`{?s%Bh82{`a5iV6XISe^+$czS?>9e%UMO6@69}D zTImq8@u&r)W&Z!&hh9TDp24iAAuCsrz?IT9jG*`Wj!(xnV8CyH%Pa!z*Z;_|_uwN% zPMxBd12?8V^DT1YLz4o`!A;pdhcDVb8f6N|5TvjH%w7nV5l%`67%op+0T;A^We9Pn ziGpgnzog;(`;AARi;s`5+$FC0x-7KCyHTwR#zsZ?U`%s^K>;4YyH>LcU|vTKD~!$j z(DRk|^z=|h(g6qnH#3N}Ph|kC0Ds-me9`j58sctu$l2al56R@uAObxaDmY!p=;<*7z)n>B*Vk)fT0$q` zEc2CA|uE&*`f(M8=5gqoTt9bAwrh({!-sN)&n`pYKnZO~{9LWSUcT3A>Z+~M2mflL93 z*Mdj4IaXZ|XC8bJ{p$A$eOv7nu5$bYZy7yzWw=@I>q|h>OQAP$0-WR{QAW7&*j(oH=8+Rd%3DEoSWYwz2Ow_96{u9@CXdiIRc zwS+b-%+}eZ>i~B>i?pa|IqdXj;P{_tLh!szThzRt@#^T?874epbl^(zA|5ulq` zoAU(o57Edm)xfZuq23k>KYNhw02><#j0HJ)cq*Y-v4P)mbab?KR>Hk|-lz?5@}Hs- z3J41F&gPvlA)vQ{JrFCPhC#SLF@jCLY#0FuQlq84gl4A%N1iVvL4i8x2jNy6?s{qXTrDd$UW>9kzXSA3g|-pbos*{fYt{ ztTc>bvls~!ufP`)LB&TWD9EXq`A7Crk=^Vo+KmWe=5-*6;V?i9BHu6(NJq-~W+I~^t4Qd+I zE+6VdpL{b4nSXtj^~kBQaiYf|Ue=>mH~v3~A+de0A-}E|B@`#MrEl+6X696M-Vt&J z6V>M>;lEvMBH>Nhy$fy^b0oPt{&?Tv`t3ZkEoQOSe?M>0 zNOv$DQ8>5tkIMF%k6T3=(D^V2}Pd-{WrQ1Uz*;`)AdgN<`-=ynAXV(XQFHhFhiBjsj z2?=NR$#r1JqHQtm`7AUVkqB5{(@jxinZwqhd7Cel0roF^%ZdS0kT87i4A= z)=z==&C;2?*fW1kihZ5nprkcQ6`kZ zM0^O58?s|UtmuUM7D^04vw?RM>U@qT?OEISsALC7F@5Nyi~y?DeR{p>u8hYQ;<(0N za!JZ%Xs^(bi(rX!*Kb-Pi5Pr&b{;JnK@4=$qunGxLSRO6sE7>`Q0@Bx#39UT9$@p$ z3Lnw2I2bH0E)pdJX1%;I(0cJbic$300zfe#v(OQc!Gmi~0kpdgAUP5xmrAy#F-8%c zrwTe6r~T=hf8`Yx5>kd)Obb7rL%@K^{~b6)y?Shs0icW68Yv&es?E@*xvD1t~dXDw22;V zqx#Al8Du!~+*g}hXym*JT|Oi06*uo0FCM+u#~kv=Qic0uzI?Dhx@Ld0XzMoeM&1^t z#Ebg=ZGXw}=KNH<5{C;H zG6$#Ec60MMD4D;@&81`Yu%|%XP$4{^Sk?HjJ?(;i3#`hlSouyuf zWn}c2EDKK-rl!IjUbNqHh4X~ccMX@XvSi9n>dKPNddyB0@=58Mt)zw#{7i)}1<=Q8 z$RM_w03av&L^O4sU2z*V=t)FSl+y&U{T!eIC2@#BEadAOY}}~V3)X2GU{9@(GN%SGz_-w=lr~^q9PTrCa^%fuM~-QvMdK^ z%x^1a)cnkC-<5#Wt86zOkfGC{joLt<)%uJs8 z28+@6X>L>FE(FC5Unp;`1N7u$=Fzh zpz{3+z9RKK)eGhuzaFvY>a%w0e@er&j~O@~?AYal{@ z{SZZ|SNtHF)i*Fl@9G*d_lsaKna*x(x_WV{MZnIqsr~QgP@pS()XdHX3KtHt1FUok zVI11xPt5XVi z{IU8r)YRwq}^t1)9 ziLMZKCY)ScdD;>H>Oxq=?*APR6b&Q-6%bztJWWm3*47BT5^}$834ZqRo#-(0;CZ_` z-=ab==ZRTw-Z0|lYq4LBA251Awb2CWzc6t2aMVyrtbPSxx5iJufed&*ZhoI1T+p4LFd9s3_xlujQWyh-nMO$3mgCu1O3G`}PGR+hd{X<2fv^2(7A;#TC~U zcxmo%#KqCUq2PatVU5S&!vO)38dDuUas)j^MUa}B7W+SKgtMjq;3?cUWN=Y2*c=eN z;TMOTE8>|%1qHap`?Q&vnPQx1L7zVT9;E1zXHvk)!s}m&#Aeqd3uYHoCMmOj*1wBm zPulsRdrC-(2y%F}cl@Mk{`Zfq2TgQ+3eK=4Y2Nod-ZbwzIH|vJV&t|5^Zg_t&Lk)en-upxTt)ZT~621A*=coQ8w!S$qr9@xW84(~Fd)QU zICG_RU5v8y?-YgmpC2Y9x0TJx>SO$E;JbIMZRm$T>$Ji8ng)qfg2XX-1#ai(j+pg6 z_iZe5lg2HT^ zDYm1M<50{)+7GK)nOPFPOtUsMaQExaA^!)*&G-})GDt`UJl(cjh`npr$)7M5U=YiwL>4^jn6~f@7#fzXmp}mxsmzPpi z-ASl5G3jOVl@GruN$BhM@7vtm++eu+Xb#BY(CDaXby`4LJ`p8shRLuvAQWE*Wy!n#4x;LZ`l!tlKK7uSlBKJ#3xVrD0oNV?}pjl{CclL64DEvdR z7$w|AW_ak-k^y94x4|8Bvy%g*tXRAFTo642kXiPVCwCKKzkIvlrbqv@0T3i3INPKC z{fgS!`z83kr%*tHi}8_V&)aC9=LuvCrvKp(tCK+i`g|2|eOX)EZeoDj$X+%s9Lzk< zU>+79OG`#2c&+V)Xb1(;DhrXKI%3U_IjpqPdARP(wcd<})`o}zjI-tsbj0(teQ%Vq z_{N_pR&o&zKP_7`^~i*G%0D^j^v^NL@G>MmRn?z7y`}4-uPq;k4g9q-h?2N?M(;|@ za?gOA;dfP5`GY&I8vR;)UHRVQcynraFxRb)7jAV;g{KGKYH-)gcu8)f*08NJr&>(;e%6SW@P6n{FMKgDYylM^bF+pbF-!?omJAKyAm zIrn~LdeN>{dg>x9EM}oIc;rC#>BEO|$gp7DW{~m^hhiFoE*$xbS! z4Y)Krb-HS50~z%V5EX+S1Nnr2gYlpUV(;d@hT zkH4po!g$=2s^58dNB+PY*;WAvybUqsg|mNG=OGkQK!xujKg%2)-3i*!9>x*G`~{>b zjrA~&Ce$tvQ|*xBNG&7KnsKnFw1i9$3cd$1XqR_D?dPYNSs-%_k8Z0vNG1WyV&s3w z3P;X`fSeGtxSRsv1*bmm(`ZBp1xj+WHdltZL#>vbjZJ#$29Jj#rT`!kj0=viL&iOr@0pODMly7!ED zbZ(50gceO?wubN1{R&x>4mDY9|f%3&+e0U#k1(o8|xft|nk>*T&BnLIADu0mqRneM|o4)1lx8Cxr(?5 zZaIJcHz!|5(q-Y@WHgJeqci-+^LL>L9xC@S+O_)zTV%i>u%bbOBUJk*``8nHrB_wm zJcxc6zY5YOD%R=X180gaXZ?Nsg{kMNC4gKJ2ktF8Iwo)Cb4gw3D3YGf-`2bF`<5X; zW26la&0CrvRZUfjJPj(bJ^PmD9sS%QTI{OTpIi(h*RoZed$yS_oiS`b_p<-H%BPRV z-*y)q9CqAq%gT^^LS+Zzo!jzGq_S@=XJxjGM@Fd!9H9wT!<$`6p{5i&HTlGHu%l_m zr~XA=2CW}bar18QWtUUtl;RRm=0xL+9B%0*4O%Gq2RvTeUmCBz0t;{LZuuNAv(p?OZGks zscWum67l)_NDNMFHWM1CDPCuyAW8f3*va*Y00&MVsW8|5_QSCka&1ix!rPAV_V};2 z4Y79op6-b{SH#3lt*wsHel%bAS~J`A`sVqHmiGxOOQ~N6)-10_-EW)N@y3B?z<&QF z{TF_5aa2}T3iC5Fclt}N?7Q{s{)Jdu#T;^~pH-;DLXYL!nvi;~3baMPcU_<$qp@1; zP0mmV-zRKqGp4-B)|NX_a{@yW{_oyV!dUJAf9{&yNsHH4dRAsw&skYHjMYl68uEMl z#Kot_g-gD_=9;acuWh9pGd5u2&z3|15EDTT>qtH!8IOBO^EMb@cAkUxA z(=Pls(Eyn5AZCh*=|o6qeid%U`Bvj0hC`-$PI|N>mSr*SpmmT#d?BDk-~;}c{c)Xm zTU2BOEhA8Of7vgB9#0`><4D&Cg1(ncP>@hER#gY@dD=0t)d7Na+C6*F@HJuBP(k&% zW+sHwE+wG>qP|#!Md*Ui$ul&-87YdLCm1#Iy=#L|=;RMSvtdtn`S~PsY&i zJ9;K-1btZ46XPw6k#~U4a>VD=N@$*{rR) zT-2AA-b8de7@}w~_fQ#Ep3`%pyH5NXcJ`P1#l*x2*f#zm_Mu@MQ#^}tx5FVMJ7Roe zE0CA2fhW33eHIr{QlUv}>K!-*9vXwU-o-$kjIn#FiP_j~kS*5WSt0LTmkjH|HN%qd-!i$Led2$}pNOdS z8Jf&~Rn598NrBn<<-ap3kYqk)7@kJZx#}*!z;W)+$x~J&pv(5Gug#4891c%c&Ey?e zj#j8|b{Boh>=Sb}E;JEHDHpfdb``7;tjLm*H6054N?pK7UT~RooEIaVt zD%A5e{gE$umX`&wGjx;s#o&qTR8{$018LUPAsQdrUEf--zx)5GO%jq!t*h1Rt4(L~ zmvq5YoVj~?<;->DCZn%aEYBTxQ&PNNu@W5pG_}xZDFIV?Y2xyc@PiciI(*ftd5Vw! zXM*#58b+IIC8^63)=p4`@83P_L-RGy%ZocB!*@&1U8;qiSN-?79ZuKv`;&%uPUxgv5C0#4AqVE zi2P}?AR#%zciDY;STER8zx%N_i zRGdrKK_gi;H5~r)dpG@c9=`!K%f8Jl0_Nu#lTx<||9F~Ns?-&hR69a)JMWgPL2o_h zuCZw2nE`t1F%3~F(VlkoLksH-Sfp^tP(ExWn*twNA=zMJ7{ z2B-9-l8u6oGm{n-*6K5|baU%@s=}((`~J#bwjbb)ceL~IX3OaHUYYEDh{3=wyz%*`N5;&; zX(SkL3lD_&>h&+nGXDO3vbJc@AoXzdKz{1acU+vg?!~0H(IVgMxgT=n-Skl7!$#L8 z-c5;4xAx%pN*&c_?tV6`Xz|+e!_zY~kj}P6?O&Lh2paf%jO^h01)5l-t>RB| zvQv*RvV^e#>*?N8NG|!dnDE2DCShz^^YMIGfUYL8Ee5$sqwbwfQRU`9r_s{DD$D?Tt&r5vKg-57p}~l_fB*1zFI!wu z`9V8b9xMvf)_dK;_~tI z0klmRi_bICXnlA}>~RwddNmS0z8up+!ow4@HwqDB-@gZ7!q@4MN1E(W8CJ8ce?)j9 zGTVm~^bc7vTvu(|G}`*K{lK2!ouf%o(T{f&afy8rF~@fxUab-~qy&3-f<+WJ7FOII6i9M%5G5mGbScdf)JXJ>QsfZ^Xim%;Of_gp7^ zZKYK6bLg_Fwl)O`=dpIoHDxb{uZuw8&mBA!FS*XhA-8(_a^W0PD5ih1m&Y?R<&}7? za^vNaYl|F&X?GXUlG8jF8fu~VVNcfhNARDF$^97u& zau_@kQgw}2GN*#m-DGvX z;)vZM+mV8_uDqGQJudF+J97G%eX%Vs7{7F1bEvss{O{$M_kW$9XSi7nj(5ML??L;gnA6a( z4zX2g zj3u4FwLAW>;vpCIDu5KI?{(b<-kixD}^ z*K=>Q9ER{*=RfV`U1C|*P5usuXD&a{K&-&)Ho~WL5uiOni>b`dTZW-0O!d_gUKE53 z0b33i+CP5$SPjnLCn3Lt15p)_y>5P@x=s4@5u!J`z+A7el)%KmfJYg{#mvwOR1>4> zc$?&;3R~6-kQEfeeVrJ*Am~emU(Hr9FCacEq`JlkJwKP0Zha9nFDQyhM#1a|tGQaO zs~&!Y`hD`;uK(-{5T@fyBP^3%G+saP70?d^mj%@p>R=kQ3(R-pjk9*L&^nmn1LGI@ z>0oOOCzSPIPHmv#UbNWIS|(zcf*3{Z?A%s6Rflhd_lb<{SQc?2B8n%mFJZDs8Pgbq zXdokFG~7*2b|FgsATVRj?6iF7GoF%iP_yl==V7PPD#U`a{@rqBhjtAI16evBfoZyZ8P?*x{9O1qj? z5;6P2*gGl#_B+@~`ld`CFsc4sN*FZrw^B$is3cM%UP;QgwHS$8r~9Cs1hYG=g;tja zNLe8HiaP9YuF#76npvzdmb5@=Im_n|iV>@1Ylll;M`GPNdPYY-p4Wc$on+oMnd{bP zHa^7%J4qxZ$qA|2n@E4lo_mPV6eJkwx)7mIy}UAT?j+ZA5GAi^V<#*HHrI{NDaJAR zK0Q-11iWH$l5(;;#yfaV{w5OL>`fP9*L|4W(dk6RsG-ZC6Fn?@l}om&uN*q#+15IF0raHz8H-PX=GvqCTP z8#a+RP460va|*-s{CR>&{1n&VEl;ZWgfXB~+w09+uE?m1ckt%$uF)frl9oQiz$X!~ zeGh}oQLEM0IkM~(pg9>uKGWa11DZ2BIZ4JJO$3ZsJf1yg*g^u{fZuIV|KQ49C#iyzNu&rThbH(BtSxgl&j z`}bD(nbhXTb46Q4*S-DeyiG4=_^^~qcq_hCxj9|447DwOS}(2fF>c? zU#|T9cm>U;e#g%@cGP*OQUw;TZTd3R7riaO-rp%;yN0q*gnTt8k1EHM>#=)Hre%ql z{W*PMDz($4Piil$3nYE8i5bnz`ypJbrvL0t{rNk&d7FMxIo_2$EWKDFqu1p{6H}Vy z2KnS*XGaIW>kX0iQV&v>a0PS%&1TW8s~o*gUqvNzzN6BjewGAWheY*(&Rv4Q(pnc7 zefYlc#m`gISDZtZ3xo|$Fg~?aXLEOVawZjn_Vep(pG>dF+|HPkHkzE#UY?4gLTba2 zq7M;Vwwx@!Gzt=T#FwIxw-n70u{(?rZ0aDg*KQ;j!-`}RH}btDe$ubF z_@=Pbz=x&AmEW6{_GOhGx>Re^9F2)=+R?@zZG(bc zT_bO(V&?5e2mm{{n1?}3047X;F6Vyiq+FJ8N&c=*iC zn@6GJG_G$z=to2FxfJpfIm~|&KFf)RALs7jY@LSiPp9DIswotXHzrSBzMMZcdB{h{pM5Gfk zR<&~)y1GFiufJk=462ib_U%+u7!l(jwk}~N{#ZmnS_<5F3_jdDBo7TIuze=x#znAX z5BLo?UlY_RaR64Ni?L--O!fDj|MA`Ylnbn`jo_Oi%S03N8!Aj=Gor+tub#dn4J*-u0f73~B14ar4PX-f|-zS-tKE-i?SwQ@+h1>-c z!coq30+~|r67}&~U$P^)O~~Qamg&~kmedwr|E}H>3wH&}i`})g)B~PkwDd~PfjxVkXhkD>7%<1Yu1HH1&+H36?q0!vCp$i`5SKk!_ME` za`bsBWxm2}<+@nltBJQHw|Q1(lGk4u%m>VxtLyC<&M0h=NM7X!_HpJ%;?C%By#WzcDIeN>uKyqT>`Ay+$ zyE9j(eQbD=j~@F~(-EbZQ@B36>)6aY8aOrD(MRjtTS@Gk*F8#GUS0d5{Ey1pWyZHq`*+G|H(09I&J3uGjW^O z`pVghu7BqDe{aiN+eB4uL~Gi(t;TKUtnoMbQrGa~G^`B9<=(6|)3F{ysAlu7oo6&M z{+s>9(enQgbrx_@u3NZA5l|Eq6{JDLphT3ARz#GN?oyCW=^iXVNl69iZjf#P0R`z8 zLO?o(?wXmqKKHrz@;gVj?#;jq^Q~{aYdz2ZIddjtW7stM1OX!>`|*nHwb&9}udpyu zxq|B#DXLsttP-6eo#5S$&4i8qlg_BS#xfV+Z-2Zs*gcEdZ=wlgpi^AqH@ri=u{-)i zzW$SywXtR?dveFrbxd(lQ3FRo%^UC(O~Ilg-4k>aAYx2{V-=j*=2OL>I!pHFgF$Ng z_+aawvj9+{DKJE$KaH#s{C~cI10CXGj;KT7#bEILWtYDC(PQ#%G zvBv>7i*pciy(qx;bP9F{pkhn`Lk2P_(u_tPi*^o!;h=tw(DOd_Tu^$V0_HS0ll_I8 zV>f1JXAm=7d=oKPi`Ta#%)%th^co-uAUP75x`7uT#MvUC-u%4;K}aluf;7PVfCv5i z!%Fg{8pzT>HroL1SL7!2e~ArXe-WU-?D3lfch@4W~QU54eF6 z^_$^4cp0~Ifsm%}zIYwcy}p@7$da#sKvDyuJB0bMwFpB#M3Mt5e-(Y8ERr*d_=7XI zM2p_(rVB{Ptc&;lN1ICrk``cu8emc==LSZT=AWOs1ja>_Q~*Il+N%&R@>(k8wZscI z{Y%O7vr9`KVF?xsYO)u6bD*hzvytkvH2=!)98E9Fh7+svT^_?fk^ZkVSn8GibcksG zu=gicv^`(xUZ4?dYd-Cy&fNd_ga)11C3pt^KJo&t-GZKjqVx{jzz8(USBZXq9qu!X z)6icGD5#?~mAP_Rb&4A$U8uPS(}%=l2@(Q3du7nqNAoya`7~GEzAf{kx#NLc;7x+> zW~XrLetq)6qg`E$t--bL{ysMiaKde8yr359AB`BV5zHFjg*6YTsBX%epFdCeEFnJR z!=-9;bsz{z0%;nc?Q#z{8i#u!+z$$xkrmMKPQf*~#nzDaSI4eZkhnz?^Z+EJ`v>!a z>l;M^94_|X8bHo{;m2mrlW68>+4Vk!VjkDR^5MINMW+)`VpMR2sf>dk!~9Nge?J;D z8-#x+U+CB}fI0?d+>IFt!925ZB41u(&<=D~)%jiGOmEQ!wqIVq!EJ;Dz-p4P+akwS z_dNCA*Y`~4QaoHT=54C4a@AtBF48Z*OF+dFii=N5DyY@h?5X@Rz59xLvWSg=789a{ z=f7;9b|C-U&xW1zz>!`TYdIoPx=1Bs-$BwsmDNbeY6&XJ;t65dBuG-=hsGv_C;S<*GRBfGetiM zMtLw0(WIqsXI_5+pY(_;|B8a3+whCcBc25Tu|hHyD=nfgdhy%d7W4om_Bw0FJ}Lg~$<@~W@LW0RS&uzRD;#c#IP zQZV7~rpqm!_#&rlTWd3>3W_5kU$PvU8AyRE;2qi>DmvjhA7`d=h{DaY=zF^{-8Sfa z-J`kX!p@odBZd!s$EyV+`M=-y^vzF7;L!YpGESeHAq=6-8hZfii(V!1UZ}fzXI-`>4oXb2pRUd z3G%_xXLs_Rf{7N?`}+e0-tS>5MhP3{{Bm0<7*ZklRYb3ZV7oxRRPY|@W%C=g~uL+&N6b%C3f3!Bl#0u;4f2L@-r8EOJ{x#8As?RI7u=#KT*DdH|VQ@0u04Ea8 zK{oL6!C)Ve*!`(?=`v&hre+l2Igug?00Ee3qslsg7KIq;;Z&8XOfD(cPXmh$pn%eY zviAv0{1DK?5_TuoQ~!I`vsW} zjiWmEae8WMly!KV$Db;v_J4Nho-JkeNCd4IA(O@|e)Z~!E8P0IrfjfehYid-fSdla zkAsHVJ+@gw{5lr?Av2ujUJKLK+>T9Q=jQAr z_5H%R7jJ{;Y(Z6ZW=Xg1>)YwpKKaG^W8+iqyN))6dSB$;KsmFNNf3+s5G&vidwFjH z$A1y*YGi|MU~22b{Qi9xyrtT?djlG9gGh6j?AtbhtGbN$kfKbj)^9TnykhL2BG zxfC1(pJ$Y_-u(9I)2e0>OY*NBj1e#GM(o6oX@Ppd*BM((@}DpGPq)q|=?Dxsluumd zD#Z@Wuc4TTgoU?ul4ZERt}h;!38$m)(w#~7x}VwbjEsNRUJmON$Y$M5^9q~+%~c<& zRpbg*bgk}`eX5VEjU_djfDFh24el%FzK-Sm+A;o^UQo{OY#*qvUTh#O{qs#j_x7*i zbgaz<*4{(Z$S&g|37zm4X71fmyZcq6gVljBL2*BqOY`z-?T4f8`JPYWt^E_{EY@xF z3nO1_lJH^5^ew+fVxveL#jtuE%v8_f%RVbx#%3y*SrJJ14xan|xv*)|StU9yhpZ*v z-%5F`ERhXOTIL2eLTbE83FS2luNeEh#nQGU+_bgc#V^#G+6>9@Whp&6E>pFj+VYpg zTOQsF_d2D}g>Oog7A}JGRG<`$0{@P=q);%47c@8o&W(UO>1cOMAnn?rn|#{;j$2|2|ra<#@8C%+nY^nQJ(*z$JVkOARSc04RB35ZYo*9FPvK+?wqJ-0eN&U8QP5~svg*|^ z$O8Jw%a`AQJ%1A9GR;tv0Ni;{_XQl&&>JHXnn{F+3{bZu&@TToKO_hLLP!_++1U~{ zHkiFj*OP-6q1{C+hzJc|x88ef%2pXR#xT;UIhcz@C>|i7&b`1!K>?HjF#Qw%+jh8Z z&YVH24hW`PfDdES1d<;P;12^EtQA_{4G@MG+k=GsKmZ?_Dm zUV65w>#}M3?-ggUU^+=i*x@Y>rTib?@>=*W*uAs;UwRZ}f2~wxLc&C9Q6kJzKqGZC zlLHV~u$;(LwZ0dv^#sVKf0M!tE22qd-GwJR6iwx}iyw!+Jr{PEA)La`N^t^kho2Wr z0|Xw$6{VI#Ma+W#od5n!2IS`oA4;-0s9gu-dvQ16P=6)<<1*LNo7X8dbIYxDzAZX| zEYqZOdDv{Uqtww{GsQ`jou;wLl6Y2Drp-3dnGEV%xM6Vc?uAqXuTtCMC~0}FH|jaI z%5p&z0gwHI@@RPRQ_bn;BdT0qEjUbM_FL@0N`x8OMM;(MH$+}w=8=OHAD!kmKWfk zNL`+xbGIJN?YvgKVXjP%%@~kd@?2}2zTqT6n$XQ~_kD@3nHVYG_G6 zlsFia^{6h$6{KyZWAhTi*esSbvU2LnM~)_U!3fJ3E1=MDGa>MGo^?CJ72C14COTXC zyJ>04JasttlRu`S#%x2VC(l04TdOVFItSO%1m61s1pa~$!J;#(t-Tb{DhUcwx>jXX zRk0U&V_B`ngveIq@9XoUD_)&QM}so#Pu-aP`Cb03`P)1_2%Zb(jG8NkmCv({;d_-H{mav(a<$^~kzb^G{u zU5^m(7&VJc$Y3fBV979k|BwDv*^pUb2tscJj=Jr>HSgcl*tkBZV|!IdD29;YlG8;v zJR`swz!q^nTSqXIFv@0uWLSi&{bg%oqXEIq(9oExs{<7Y8n(aSS)uAOMtYG9T zwNTeQkpN~6P&0Ujw;;75dG{vJ8u~I6uFOC`gw!86wq_x%6;*zIK2pz_6>~Fd zA`|EG)@rOSPr2W<=@A^Y$b^nRWdAy z<7vgk2<~e2*>aN6&KSGLg?8`%xQ9LR;LY-4ViJB>m6_R_`|ROs+H%|Z*xq}lyH=46 z8zA?ysJZx#2bSzHC321>!l}J23eln>s?f@Op@x-a(Cz6(t>VC*)0eLZaHXfOZ)P9l zf;LKC-)2E^ZQgwXj>vMcWoKjg$M|Qx)*qs*MtCc=FxIkWSu0&(AUVhAac}#^pub2T zbX!{9btl_0SC#``0$koPy~AwpyVaLYIo-9HPueZUKE#T~_Ll|K((Bj8ue?t7@ozz6 z=D~M;IaM~E_s{DYwt8<21Eqr`l3+9PE4o~~g zEAekFckg|1*?e*5G)b1LBlb*S>85+o`eT&=965(rcWAD(_3rDgA)?>?pT%Ed*1?gt*gboARn$ouK%Nl@h$pBXQJ{J1o;k$6d# zhnkwyQJh+D;Bqqr2OW4ls4z>8LO|Lj!XM9zgF}1*5hipm=6b9lJBG~KY~i8!v|q^| zGBG@~K>%H7h8WF40O6;mYOWsL3jc-RachoNtC20r*SI(Ya>-%=jy%u7bU6Llxs)Ft z{{*>H7xA*d;uK!7E27MQ4N>gp^2 zwM4Eqwa9fheP}&^>GB+U3`FOJ*sP+!8IGXPz<~80>?w(etOCYS2-6NGgSr24hDFc9 z3JgSMpp$FKT?VqzeF(EVN6c}45IMtkfhX=Hgv(r*F$cWKcnyuQF_%CcEyW2iwLVf+ zjbP@!d6NXfkSzxG@-j1hwDYKdS#taZd39lvh2%UxJcQcVZGmAjh|9cwa7a4u`0MA- zzn`)F7Qi}W{&af2p>X@Rq%~ynG{d>~lpF9|!FA>u(2D-@>pIJ}z4G(0dQtUlrv{y* z&d*fO{h(BQrTiLPbzP9-hrgZLb8R0GRyeiLdBySknqaK`dO1{9+Z*@zU)_hIXL|Mj zGKzu{5=3FsL|btBI~vUhEy-m{we!#2=7a4XL|LNkSXkS((6y8ezjv@7 ziS5sii?a)h-L&5d=8~2s${d>jKWxa2(lOnxi^XZK2^rQpI$st`0|Lli-jyaS?TB`x z4cEDb8t?CCu^NjB+?p@8J8~9-r7MJ{62;}?NNN)DHRS&dC#sxg=dyX+x7As1%n=Ys zviyaU+lL99^WS^r2f5nzN5!d0Yrp(sBE`7#Bo*fG? z=C(om-=6TzMU7wojE}C?Rv)@E%^dS&#W=e^Pxyt#6C8bfgFKUBz4KT$DT9oC+vJSe zJ)foNYQE7#Np{~@R3BPc_{yYq+ zN0=5$4^6mo3bM1fi?hUb2glFrJkrz@_)}f38=8Tpv%|XI=C`VNUr<*?nm6c^qmcl@ zOQu4XwE${yvGp$#1&I%WK3sD7+mjMw)JbRiZW>+B&qSSmmG~{i#>OH@_=q$L=B991 zmWT*5Mg9E8kFZYyKC;tUZ3PLCPmbE!eg>4g$i}z_md8GoTSP=vF~OKa@{@DC2V_X@r_Mv+@^Bco(zryZZ*o2Ad<8^M8Q z@4g5BS%U6@1Uhdl=0s3Sk19X^MHMSGwT{tv-k6v}nY5kTSMb|?gfI#tSuzOLfYUs( zuKO%jO9J)u4eZTLOlX1`<(tHDpZsTLPF{xL?U+l*WLqSCuHMl}Nfzfe%o1Q<5Z5`t z4;mg(!#o4IW+C+T=j*QsQ!rBxfog(4RDM1WZnzK|heSx^(j$0DaHb(AD@w}i)FnG0 z6a5TP!^5eXadCcrBnT1$NJ*0*nL^eB_6}ToC*W#)ASFc%iI4Aq^0d7+!PMC)Utj-^ z5i<0iV$O)Ji6T00NJSn`P&t;%@R4m ze~T;&Ql-P;?Ac%{_>aPP8p@+|_>G3aK`H>)y!i$r|I7i48c4wil@a9SgO}|IB*+H9 z0S_5PMQA`@m8VmVoY-ovJnnx3-jH`dUTtvD1&zSzbLS+@%*=9XOdr0s^L$yPdG6-T z40Sie(+eN&-cY&Ufktd2`XLtZmW(2csiTL=%-xkwgSv}IEBhCtgXck^^Djd$!IPR1J*bm`8=@0?Vi8o<*C)GvG?8$Uz+dvaI9~9? zU9I+&_Lb$C%*<{+>PKGvOEHJe&V$1OZ?}7Suk)?#OWoF!qRrRv%3Ii8?G6QBV-*@7oc8V6|Lxb$a?O zFzOon`ig_DT}vKJg^UEiIRg7+?EMOUw@~j zdZ*UDByiKy^@8sAqVy5<+bE3{j*aP&u2-sysT}?Se^g-;q8Z+0jHAce_u_5 z9KD>ut2%j)oFDJh>y|RkT~l>pFeDg@xagXlI)IN*c2tqMO1b4tirlT>!%5_SnHl}V zw<0@)o4=Jte$kJP91t;6R82K6`V0W5APtL$X=#logjSYc?)0bn&djMJ>iP;V;fY*1 z7K(-ewKMM}PFB{;UH$atjm%{#^^*jgfkx`wS=+-m8NU^MO8%y7D#Ijpd^&*Yy#$N7 zd+e7lX%(pY%pP}b>!$K-9gCx!5LIow{N%`lOQiPATj8U;qPN}Rx)jx}TFDxuU1?#p zJbWafd{8W^$Z9@!`0Ae~o~yy|OQSmRdl}*^32NDgQu2OjCV4gLtp1gZ#9s5O`d3wW zz2dT~Z7i4QW7*r^wFb{CsbyJ~1gE}U8K&i(<(u%+9pDR&CDoy6Bh3!cS_(_YV|TRQ zJ{Ql~H#Qg_-=pv8VbrX4W&v$38Np8&sKJU1nIO9skFas%G{W%t)rMgin{DUU3qV9V zGiBnaKLSk=cWp`6RFPU*SYwaU2|Bv0#EvH+G9b`Hm-9YnC%Ak$bHf1d)EN;B8l@`+ zftGj5*4J%-3eak_yIf1)Gi6AI+buu4f4o&hg+U$6QUX}-30KtcX!{FWZoi-)$#px= zm90Qu@_Ts$yHMHW=6@oty!Gd6hi_ep$;s^@r~Cv#xJyn);gb;gDApW zg&xM(P3Ki0{YTN(D^1zABQ}Hi{=N>VnIYtSTD%j2@tR)r1#aD5szGQnkX(lV6#?3i zehdWMT)k?lf|^ZI7-;+lSEQi01J7Hx$`QPTv*|Sm;TF)Ye|>C`+nr=7IEz?VSxfY-#?E4Mlm!j>?FiVA!vDU#F&Kh zA!{WkN=8OTph&%uJ_(GD9^mc5Wg=<)7?|?VN8R^2;!amb+rPjZW7`FV1V6(28Z<2jpSficu|SN}fl+8~?H8 zk=PCsbT?vw+TM0VjHd|c=HtJJhZqiUL_z!y0L$-CTeQR=d~7IrNRke`WZFCuP+u)> z1Md%k93lBpATg0g;MBmI=K!RA9yNo~d@o?Yk+4aR-DRFxI2!)b{zY-URUP?eG%-7W zwlxWnPxS2m*M(%x!d{LUDKyWdrC;}$&3t~-zXY%X*u&XD-kgd))EOAgA#PR^rvf!C z_h)D`UPFMRzp+FUASnXiQL0s&PO(rfI;~%J*L5H!THf>yi$>lfn>0ouT2WH(gL=gj zzq(q5-M(`dDgNe$et&%Rb%6X1e$0w>Aw5E3B=ErVzp4kP(tDkQGVliJZ|t|&Ix$15 ztgnBsurf}sLtZda=0@f`Xj@ZKroI?%?v#3@&&ETby50Khx4YJ&d|6i!$y>QCTI4ds zWC^I}ef+*YDYm}n;v;D&Mm@=dcO}MeFwKwLm8;p)=gl|h0!zht0s@_~Kd<#|f9y>t zyjC44BfL@8zq`k2F~Jed@m{<%o876f$aW-gM$Jj=t?qRzStZk>URAT$KhDN(7~25B ztjGHsOjqR1!TQ+X;E}R-THJNd{&nNvu?IhG&^`()0KC5b%@SI#aLk}yIi*vu!`;{z zCp}iGhT)^(5Iv{S)zdSjhWXhGw5{@{Odc~C8VbQ0VHzjijuRBUH`38-R%m;{+rR9j z=b4SBG$L6j;q`LAbMhoTU=q#{Ae^4#c@(+r<|ZYjw%nJC2tdK2beIrk$VIODiV1)I zkaFrBUMd+`I$@Nm%oD}E<|pVNae-mwgsCZAKs(W?+=tpRvuDT>MymJ3*x4=0_7K-% z#TkY?lMd3?At4NR@3j*Uyn4lk)Eg5}Q*}0h<6R&P$~>^Qp8_$H5LT0g<#uww%DWJN z?=d?EWoNv04{-()fY(gtc$WJbT~JyjqQ0v2cue4vn`^(Y<9j3T5)DlnVySua=Bu_y zS&ORr5+V>&asPS27EJoX3-)DY_MAFYzDUbN5E`A)lBGfe=gz`Xi8S(bxB(;C5Qt5_ zrba}i4&Z+<*FiQka4Ko~r;Vjh(zqGjP61~IC@is?z?*pmX?pO!()@p2540Fa^+~&T z_ikuxYydJB=E>e|8Xspw67yi;1;@K`|s2 zDRccPaPmG6z!)3pIzT|dOfP7E3W7IGoIrxL4kIm?EF<{e_r=8yIn8pC3>xqi;N<8B zYc5UBwXtde#2HUQ;`cA9DIE55z%NI(RFI+fC6a`gcnXqf;hIUIOuoiqAqcW)#0Lb5 zkel$WJyui);1Fb~7=fvhTGX8v=J%1w2x%#K4oHOW!OIABkf`3g_}e!G!jG{2kYJXx zh+Q+L1I|DYOAP(8H^=?D1QEB3Fseo%yoikv=EJ9u!xd-|VmBGZy#t}j{spe4SFR2ic)?eK4+}TMWDk04xy9aIWfAXVYWOx0cqgfWfdO=lqY<5uwqnTJKMc$P zQc!LgpMX~I&G42sqWP;sLr!PS3*r=t>uSW8&9(>d>^mSvw!kf( zyHk=<>GOl-`!jamxZJsffS^;iKQ5uj?RkFC%Le?f`UnXI2g7Orich=<4^zpvtCf{} zI}p4jR}cr2tEZCPn4Gmj?3@502(aF2-}tFMw`6hcskpbUN!nAPd;oP>4Trsy;f9qz zsO;Zp{#hscoy;O{>1QI96bkV5E8sfPUmkT~A}9p^`Z?ljkXTEarTWK=ZexSH{3$2w z??nSfP&8?8B^1^5HP86jG@h`*sHiKlJw;o$4{CkB-FDkA#m_o?Rz^WQ274qA<-xmj>X^*Y0rYij&}lfdVpspJcF8eJ?clU+G;it5m_&`Sh~ zDcc(#peIRra-G1(V~;ep=BsudD3Gh|HebqTbV`R;86E8|hW?5Xu$veu6_A@h2lBC_ zy)MYL9HZ0AXPc)%=g*S%5)!-e3g+XmqM={3yv9Gn8Ywwh=Dy&Wkgf!jH<7mW&MT>` zh^g(x1Al)z)M^C?g!=nqKFGmdYheOkO^m^K3kHkr44Y~cnyYlUO)}x`P|HTWJF~P6 z2I(e0rY}*b{lZhC?)Uj7q zyL1Y;JO6qan4ZEUC*2dGo?vt{4Hu0h#3TQ&mvM4}jspx`BETGybAu(!BhcVn4gW=} z`R}HkX?cd6+&3p0kXa8dd0-9|8mg{@XmUzH4hkd|GC_LbRURG+7%RV*GV)eH zdI>}a>bydWWGX{F0slxy{bP71X11>bdjlFK`9)|CFhq|e^JG$bnBkM5Zvfsn7P~n1 z=MOk+PlG}EA^4_fPQxXXfJk!k)X{iP-`H3>#K;W^r59k8B?XCsT5(V_ud=b70qWQf zC}SNXHYrx_dlEuKP9i{aJo&{)(e90Nw zCL$&K%^?~DaTdVIa|q5c>>Tf>i4ejntkOPM89o#z9b$8oKbkm&n^Li4#cjpUb&-eMNd!N ze1Ga0>6tg25p+RPnLAPu=49z8N+H7RZ7H*JQmTtV-MCFuWW(tWZajv}heC?_^%AOO zN$|a->!HG_57eOnj}@Ov!mpvSlA+km`l)EVnljJwOFn$WMG8pCou4C@$6_$x1xZ*2qr#^P0d)11Ut+ZTt6mR)`ve>!i>`}lUPHAZN4I9>)U3Yf z`<0htzn(mh>|jqK>cki1`| zx+TR@;gosT-zBWelf{#%iv9GLcHE5!`7F(Ng2SR2xixuvbsAq9pGm^-%v-3L?%4 z3(LOY6;et`r4C{0CfzL~^V|xLB^@uNcEz!T zVP$1SFb3*@{{<3zQ3N!@eX|vY5%O+;_GyKM@zqZy@(B^aEgcxC_d%u!j zK|ui#c>^MG(g$L&;(TPTd#XQ9>EU&czXInjnEp;fSQYRdkWlecNHq0)S3)bSz<8kC zoTI;g0p#I}%u633g$>r_AQl!+D(T<6_X(*hF#kwYyft!P2oUr@zd>-r5cT!D4N=bk zNgw(pP2?an_-`cqKmGW>kuZ3>KzP)GjD%@uepy;UVUT3-0l_>%SX-+*A_r@4`Z06{ zh`I+LLdYcc9}hBljuf0!@OgHBzn@Tv*Dq*A(r(~MG&QH08zo&-YJwx$-0v@M5s)DQ zm8^$!8drDs#;n!vAG*Gq@si#sCdp2?gmiIm5$?)^V+#(Y@RS8+uiw0hgr*Ok{aAGl zUNz8XHtYb%?i{pQIy~%4xl6&)Fkw>C)jbFDcvW?EWHk@Tm2ZS#D*Udt)=S4%nhQSZ z;dM&*Uz48siW%}wKt4_r+}}hdD@q&?v~^YAJn`r z|Ej#lg^ls|t$CE>rQag9v|ty`OZ&+_KRw^YR>S|g;lsF9LXoQ%Pe>U#34wVA+d@>P znz^FCXQuFVEbJ7)orRzZClNBGa1Twv?7;!c&mn(pb>!WG4pvmmR`1{+)YtIc8 zrN;#b37;iV?CiY#1fG|?_U@~=>4=;hxAVm#^H1f2unRru&EqlF?0Z%&Q&!NV2V zy_k@S7WUl#1kC4f2)Zc8GFQNs=#8VZb6=MRqxkJhsd{?WOIgx-ak{r#!36>vDjP8T zsVMFumjkdgVIMMQtB$;-zI}utAj_RQigz^e0NS9Sq$GlrdJNHjmd8%uks`w5bVu+M z!hi$38`EBrRd@eCtQ4FTi0ctU45I|P1K;^i%9aXJ6qoMG9io~KYaK!V8J!_1#ZXe zB92)aBr;u?4NUFCmv84pN*EFufFE^{O0u;PJrguhIF7Akp)^ zukX@qw80>d6ShICNKJm>)qQ&HYUezVMedw4p#4X z;85uW+QXPTFWx}hw4B^&@MUX2MRNpg4TG3xMWIRg=njMuU?AphK88$DhL+bX5rl=7 zAP9J03ZI<^p&(udO+G$Y5Xx z?a&X^Oma9wLKbck*bf@D?d`uMr!cJqvi^MqvwAjW=I@28?0kG`r4|FdU<8Gci)hn6 zFq$`uPbc=fi7*bwF&YbxwK@O6-$XB}*R`jdEPq0D{djaXN@VkA%8}`jt0O*_BVbk> z#}#0u{;SL4Q$mOGay{0Xis(R-a_Wc)zpqKTVDuu74f!4ub+MHyf$-|AKiHa27(B}@ za=adUu%$qW6BH3q^g5yy;7zz+R9bg@S3S&e*DAZrwm!MxZES?a;J3T43@zityih55 z`}-b;8?6Df!comFA#KwK*sg4teU>q`V_r^~; zi8XfR_f7UDsl76$Ic>b)Y*-a$Xq#m|e|Vn$NbQ1Z)2k@#y@hVE4(}T2>Y_D26drv{JQCYx#m4BYqh<^a$2W1mj{ya^W&?o zNedNUlTN|sXTX0c>-l@7eNpBk)=MRP5X)k`Rg0fkz+vo~qaq@bj9P6d%ivAC{(a+F zhR&s%`DkCV5AREH*lVRWW7I^%hqy)hc=4K^pY0wcdo&5&?#G{69zQ|zmy}z}D&q}t zVjS$(t{M1|kKM8oFu%wyF1|)ohVl%Fh@SUc*rK{z!Rz6?{HQf@)}bqAK)%?-$s_e# zFE%0Eq+{+%8GatWoQL13KNMT%A)`Ho`EZlP@$Yvh>E3L$4e*O*s8u^Z_z+TQcU8?} zPoKjognn4Jt`23bw3X0P@rvx=P0w6u0;eFLuIihCoRnD=DP6M;^*Gpb>7$=4biRN z_io%Egf}94l#tUB#N@OU(#Tj@krJ^bv1eamGL~I)nmLL;gCP6$Tk()OQ{zh-{p4=y}b4 z3!;l)4c-H@nwmpE#`40>z0`Vi@ZC#-=j1s9*>GiNgX~AU*dz#?;bd4i`Kidre1tAA zC0#x?=F&xYZ9;OTf@s|#Eu{p4j+%~jGy1Zi^}$rm#{)Jx?!U5yE_~61)CxTaJu8fg ziOB}-lm#53z74mbLUfcD0aH0U2W5qly6%7q(RgI|v9s?!D_h0~yf-(vU=t_tA zb4Dr2$-K~?Z3drCB(?JpY2&ox;F?G)5sB{I)N8Y^h+MJbXq{9q>*Fe)pX}P&$NdS{ z=2G?#QERAEf4{G-)`ZT!gNkiDX+^qK&eq?>&T&Kw-w*+T3`KP;{1)+gWwgL-AiIJj zSgT^agq)#DX~=svvTSp2k0)>u&8>_*dO}q;;c65B9h7%2C%K>O?dYRPLfPR!nz+mK z%Q{2Aky??FVaGk7bG7VcEm`lPp*a{So^J>jxNYzZjlvdGw+jtk13iPqV(+JzyJ_+- ztebo#Me)Z9ZS5RxDQmsT+V!ZAcJ-OQflJ)S-ULzCy3(BRYoQ|XVYV!D##hXm>X~j^ z>3DH23FfO686~(TMdp?r36P00n$L=_D(%+R7`DnFl}M<3YbnK(4yBbxT+gjrwH}(F zBR#VHOKGec=UQNysX2(RZ?dPVkXhy~6fW7API+$~TcD;V_4o>8>y-q*N7Y^Q(5a^S z>b@)zeEba+=SDv||FwI<1zWbBU}VWnCEUr~tx=0x7e-+vV#>VTYHAE)(${7t1STA3 zDobXsPHg$2#0|~%4w|L|>d2!@mzUqP(yrN6Y!B*;&fuCjBR8gvWa!uID|dQj>9(gs z>hZs~S4N-G2?VRz6;4c(;NFVYbq$p)Za$je@ZS8zvoe9gev0=PDopp?y<+fVX{0pQ zPiz%lPwcl*6ducUC{ll5w7Y}3#B^ALpRGU6PEGyo7tDc~zkMDm6xVU_*lIrtzvjVs zA7_giH8(@8KSW=Yipu-@T)fFNo04O z1%#LkK)ih|FnnS_o|BvsQ18br4a1?4N{0-<0~jHhls`dg;tahDeC5$kRGB!_U4gs+`sh}RGty5<8scCY3DI_>}5E6Cs zAiwmdh4B$%BKwA`$j6|d0oYKH#m?i?&HK`01Rc^x%B*#Mr<_P!f+1&jtZiLZzFsXl z&BVmS0@O1j5I(A#AGXH~ufv08*t})JT1g9lbiDndj3LO7cx9q~BK0K!!QZYz?L1lm zo3Bgg0^3Z3I!`S+eyiuArxhVzCIISC8a!{hB{0^an**BH1P(JluJRymV9zJ;&;*&R zR{XJO(ddc_l7v{oh^VPSx$3c5lsZke-}<`UBVj>7EjZlr_al|i=Fk0MtFp~N;>565 zRiah#)7`)NMOLb!QU*SJIXpZT#UA}q?GSWP5Z7(%-?bDJ@)l6Ia*x#U8ahkPPiK|X zxxIbm(_IFev7b&$hElJCCp!AzzdD>$PM`Z)jmLU=;o=8bO)Su*^l@?>eL?hoUny4r zApsw%w99r->=fH_2am#cfmavqjT1~7;%SeHLkH4~o*jMh;>6)m{hHeH5aH;uo|BVP5m>2&q@(3Fv)Kmq#v@gH zSwC9u$JylR#U`G)G#KUeopg0EgRZ@Gey<$Umu?LMd8s0!y_M?pn+du}+uO&D3F5`W zR_a{($ARGC+zT4uw|&;@(1m5K9`m@?(yKfI)u*Z8C+bMui;k^Iqp5NC%ZkQ!pEy(K zwJwjQt>40x+S?7}W72J$xjU%JS_up>^r6bc=(O3eaPoRrbnu_H+z23Io0Mqih}~bsDJMS`jYXD0fxdXMIhN&4FFgEC$8pt6>P(H>@2y9u2XMRW? z8DO)z;>B>=a%B{igF0@lt#V|;CK+|bRp(rNq)t0dva`_h;L?o}cITSb>7Ekv*iysh zwM%{puE|OK*0dR+7wYS2^_qP7IF7fP1NOGZJ%nc*-4!l6-Pb*wM~P*EF`7*}WLqrd z)wE&|6O+nNoeQ_ZBftFKN{43KidCqdA@~Z71W)_iLX%>x?ecmjj;kJhjpE?8Rl_FW z9SVmauA0bwt(E~=t?7_sTqb8iwTRs3Zke$vY#rP8uYxyslz0k0*pxphHo+!$#D%@|_+JXR79Q;Mum83c<5Dc5nTYjGY#x>^E*msie#Gc&ycCK;^l0=T6~JkxjXdY3J7-SOevN;|e(h z>MD>B5Jc*~e3_aQ2dZA>iv*|^cx)%Jdmk$ z?NZoDf;OEVOI_!q1OpwdYG>;pCUJczdsR?+beBKFWX%g`*CT~5-rC#SIWpJ6w{ZTt08PQ4S;7_AWQNUgANPAw-%u} zQwOQwnJxl?fwwRq2558M6@AyFp-N)tZw5fMSwr53bg9kL7)%bhP?h3@oHmyL6CDd# z1?p2cCO9fhZS3u#VFq>i;zh&E1hA|^5fosfRNR3K5VPa`jc?jPPim3iownb;-pa&m!5UI&}T1(p5}v~E?IR3#=pK-Q9MWZ94#eCLs{n3(?U=ypX9 zun((2l*eWuQQ{ki|LWp~u(NedOd8RZ)y0(XEN%LPBzcWur}Z*Dq#g=%UDu$DO`qC) z>awN#vFlcM!>F@$RZQOF4VuG+m)TUh6&XPq#g50pdT0lF`>LEF0a%8^9rBWx3HZIa;U7YW^Tq7MPnSjDFlq&-CEAH_~Sks9n? z=;=7iWW*0g#P?^P{A_K_COW%{KL=Xy)SeVnf0dKbW$J~aP*YdWG*}zoKenQJG-_C6 z2Iq~80)4H5-0D4nZ=y9f^snruXHIvUWRJ3{-pUH>;awHlFRiC@G#UC8KEWVwheT$PtQg|*+HY}Il z@|D6-^iFK=Ge7i?_Ae;ZX6qn#2xfbiwUlm1lMh4E{%O8HPj*r0sGZU{V^%uMw{GmQ zPThU|8*Yka#B!KKy|d3Sjq#VUM_ZQtFU;=o`R88`d4hZkDmvj2(K|<7FC;N{$UJMO z2@|#|QLYkMY+5oxuVlj6)1QuZ*?K z@2e54JM>q!5;F?AkKcOo9ZbacqHV3&j%^Edg9O3A*NQvIC%#P97Wc~jte_E?|%<%^7o+jN3L zm9yWJ3vhMa-<(q}i0%IUKOnmn`~@YlmcGIGDWaagp|Lg=e|Q&K?^+t|Hp#-&sV zZty_bHTz$aoOdjusvNbByVqtz;Pm+z_Q_2-xI+xXHaFh>DZJHW!c4lvvfzzm(fE$K z)J#eC;NW(1>qK9OxPE^m#~!}A{=npk^>CQ#kas9>q)v>Kuf;$qs;@k(bjVs7diBZo zylpxTOP8fz%RD<<={PA7GniYM84vb?FXRp81sI$=f}2C%pURAgXcxQ-t{%Tv zt6h0l->5~BgFV~V-XTuNVJ;s&O>-ZMUBizl;MdQ4Fve6Pw+l^9DlR=C_wgXR?7zP%4xcZYHa~~sFcwG!se2AcL1({GLw{Nj@bTl_5{TA@_@gvhzM6U66PfoR!#o`XssJL* z|G}f4qXq~FRoLp@|4K-ZodjlRfWuuWb8v9LZ#qCC<`N_}sepP5YI}MK@AQa#NNWnr za+koBvou;>01=T~1{1vo!X$~ya0G_?a8TRNpSQ~DVXH|C<2-QswhPkuY=%Zeq%xt7 zi-sOuYierZb6(bbEGhXp^c(?uK8!pwK}veKn33QJMgP$RNH5?RqZ>rcBLS*S6B{VK z4~R^PhqwPYbb*%aDUwXQ4L!I)Wo0F;sC!B7;J4xObMVsUgZt28>c@w}-BH_FN1?MA zcx^Nw^Kqoa+{jQUaS8Se-4GF53I^^*Z5tbS=V+y9K>w&vi+Uz6&sXw=@C^JM2qI;{ zjg+gN^T(!~75qqSJUmJu|K%5?$*!X1v&e*-rtc0fs8OJGt#eCKG^VUInYFurZUtWU zEV#MO(wuof6ruIEcfiUoteBzd6Q*CUTGpock=h*_>GGA^V`bHjG6!L-IjKUg`T5Rm zDyjMx(E9DO_jj3CJVU-d+*ysB;dh_x7K_L_ymtigdE7(OuJk3efyy?ffbDu<=|Z0Z z2|QC01!6bcK@<#BuJC#vYme1@xhDNFq&@?tj3HltaRPu%ZU;lMIDuO!6p247oo=;V z+~3fdd!Nk;*OIMIFjIZrBR`v`VQ?kEYgUZ>U*G5&iv*xex-3^9a3lEO0oYOvP@;0c zIqHaYkqIj?YE=uOuatttJnV@VZA!GMpx18eTB@Z{&LeJc-@;I6uXn1CRY!42x7Ebu z)XLW2nQ-ZDd&KSsDH*fVM}O%&>TqK4hlhuY2ILDr4rSjvMfc?e>a+wo9LZUvn40W# zmkHh^wZt_T7yq(e@xLIyZ-6bCLM=y)y5Xa7Qo1oVKhh|L>(Kk^w0H7S$~<4y$ZaNC zJ>Ij*ghD&Azo9gnI*<8#{)!B}PW@vpd8~b=Czi>Rtq~2d`!1U71pI7bFwRjaY=+Yg zxi{zysT{9uF{zcvbAD4^d5L3!OVrNj%$0?9?q&KoRDofuM)t(M5Xnpz+33))fX*6o&g>VQK_=g zy-YDzIdUFoX%Xq~{QMc}C#D|O`L!|!z_DVzL*Q-hFP|XSC^jjTp`W&QQwXT^7CrhD zj}M^_p2|S?XHI6Q<-`xHPv%6#dY~rvk)|7k>q;_g6+f~%>xDaDZvNr9_0*so8gA<` zA^qu=JTA}rC40A$7mt$pt)6mjyn>%16vx_aalC25#a3Q(Wfg$;>4NZ9>DfGXjhlbB z%+Sd8im-P};4|bnmxkK+6FuLd7uBVwuU$$C2ly__KNY1_UPrj2*D#n}94eZc@F<6} z5AOq|TK9&K?PKGEqY}GGI~abFtvPiiDD)j{HP`#?%dd@7XxAQU4P;L4bz419f&!=Z z{CQ#&w?Y1!Hxnou+wqz#U{o!EbX<95q*NZ+Ech&?1M~|z;&;ot69sJ6V^wO`@skq za;C{%*;yIY;DeoQt0RHWN7g&t6@!06_Wt5Y1DccJ!Eyj`Ed-GO$f$O9juvf42`xBG zD22cn6YqU!?gRth#yp1;iZ!r@wjQt1f`7+RRu7T5NC`tadDzcHkVLnqK@65T6#nL2 z7nrJo{31Gm2@_3_cptK~+#!am6ovQ3Znks3C*hRdJI@H`f&cApZWVM{!ON3*9~tzJ zGh{>Zo%KZBSoak=I*#hGd}#lFq06myZ^LVtmIDQsR@gb$dw-JTkyl3yUksS3VK*ly zL#)aLHwRT^<*?hj`(E6G@JcLzLVo~M{!B*)$|^w~c2uIv$o+~yv54r@sZ6jaRe2ul z-MbDs0njZje*Hiw@)Qm^Fath2%ErdFG+tZn_FBH5S*Mh?F+pszrE2K zQty+@PD;8tkgvzRB#xh7g@0yqma{7?NPlKsppv1lhR)w<7o1Ef(&UV}XX-!hUG4C~ zdcN~z5Ot9@^jX)%*A>_UChDI|nsVUmhJ=D1@nQVt`4&2!k>)cfPE9wlSeh-1@AhCE zeA$?1diXQ9&TjXd#n7jvv$9{Y0BqiqgQ;nQ#GB!J~*y; zd!V|5NB^$8=%M*m|1mz|H`T{bn&K@eT{ov>Z;?c=Y;Wl*>pIEQx(h`rBH%nCnnrE? zcL{&p{&lZ%q5Q4!;zC!q{(V_4lIqGAcnjTs)hWd=d|9^t)E1T1Q|El~s7kb2D#NaL z+J1N7+nwn;GNZ_~+{Wr{Nr$YN-g&n5*AGR{Y~<#?^b zUGbBC?(yPWdfmgHLlV8YRZGOyB*$TMbQfWzGY|Uv-ecSA`oj-YAVJ?GL3btjF7`Dm z4b^CSgA%M&R~#YYHPHvxK9x8^Yk<9t5(n7==yY~Ox@m}97>S`P{NO=L$E73FM?TD?4pOqgx4*v$a4~d`R6hFxkhgzm^t5~!)|Vt zHSy!yo6s$GCf7VYxe}Ku3j6v9Dl<&VLDq2<0v|To3R9 z;CU5S!4#=9{y8Ikas9|@X=x2ivC%X!iBx;VU_5EHuD*8*OM*`%Dnw|2iM{D4qJmAo z>*xy6^F|`FE`mHZ!8_CC|7l4vJ)fFp&Lj6k4e0cBtq6i5?y6!m0)DEs9gL z@V9Ltp)5!J^r5zTw@^3fgJJhAB&3y0XGu4mq?VA17Z8XJ4sJlMD*WS8eR#^&kPx(F#C9*)a8GJWZA`A?B`hU~ z1tX!wyv?fbKQPo_6P9UNX`oi6eP}nn@U<%*=0oH9;oSP^4%^mGT|EBY)>rHG`^w(Z zWhFU5VkzaZw;!nN&YY(|(VnHQq^+4@`Yt*p;mf`|t~afc9!}l03-K&J67)r4bkNwm zIis|=_MV{QOm?*S;T-oOagXo!Pj*fD7#i0_D%%Bzp0_((SLo6oY5P1u;+w{O_aCQ| zg?^3gjQn+A!C=#=(X(3z!gK8h9#z?W5*JfCB%HnX!p_2EgB|;ZLfKjoYhz578|wVB zEc@Ib>zTg#9nh%Chx=2(&Q9Fk>yj4`e7|RjR=~Lphj{X|3oj}svsWJGHuO%?XsxL7 zIh{KrPxV@{twgKEqt79;5M-kpeqCh^&cxk)f8>F;tIAAQht-G|#}(z-i>fY>j_WE3 zIMA3EBzE;#nxNbdos_^Lz#X&-9r>3YK08%d=pbP{K5SflYk7ToeX4cc#=~hxt|lGH zo*h3X2lO0|326Qy1}k^$NU^RhoDzm!f41k?L|J{tdB@HQ?#99h9I-G#`GuOrK8`hy zdQGSD3Q?S5n)G5MRc(hM>w8}sOZ6LbFQN}}FVcOJ)`biFZuS;?_7wW8$Vmch42X<` zX8&%e+ytn=KZ(NGbE|CBV=p@(Fz_B^U>fk=dgr2V0>OJ6{uj)LLWsfkDQ&BQ+c7>f zGxMn-P}LABq9tyLoHA3W7uN*~v)7#(A06#NQkz<5+I-#Z(0$3T&Wh>>4%9$%zF0(L z>IwI4APK{4+cc^v(Rk$_h*ZW+T|+?#-CpTadw1`)LT$Bn?TnR;Q$NmPPiSv_#$Gt1 zA$Czw@q*+(9yJlF;Rh@DW-uhvv4FT65bR2?v~g}H`a$b6SV4dN6@wzL8*@@ceXJ^D zuK*v!!lV>rz{BwEgYv`>>XvKESzAsw=)i)|@?=Z&^?~W04tlH+bbb2nVqcLBkBaP?3k5N2{_1T!79IGO&!YSe2-ir6F~J8P4A}%` z<3S%0vll^iBNp>7Uv85KxuGeR|N2FVxG`qNuM!pAjg8CxkWa^?K;RoHbKDy{JA}Iw z(1sp@D0-X7NTo<9xD_Q&+`mY}{@9v}4k9&MWPoI$zWMba>F^Sxvs1Dj%QjDKUAY zD&DE%zH5@I|7;H(d2=WLKHR*#`UvCfoO36EF#WZ6;S=}z-M$?SieBIm@)g*Yal3*3ei<2lrbkYr zZG^(PJw#69B&n}h0C4l0__!8&im3=xCOW(HZp8`fxI>ZcDY(hHa~ZS|=?+1a8!%pI`0A z$EDjGzZwD|J&_x;i`ez=H?qAvXL$#fZ$=+Hv;`^O-swf*t>VRv#`PP+*2KGlwmdWz!8*4NMlm_c1j#NFtmZO z4ag%7XR-?~cN!)Ajp~s5bWrJwasDU2;A=sWT8*wn4A`j32cz90P{dhd;lbF4OT0CJ zAfw~sLFcp}jzQzs(lV&FU>LZg6HmMfRwNykNr74M^OrBXF?p*NV5r40SjuYv>nRe^ zC0N*|!cK$%r2KlVDM6 zYwJPvKf0e=WJvt>+f(L#H8pCnBqROt=jrdLP4`M}vyIfwV|7@*&T$))mo|2@Ywr$h zG&~slRolM_oT1QrYK1`qPrDY+9x*C7!%$j^O&}{joJL#RT zthCd)*kS1jkGuIG)#uyj+)Xv(8sWXRw@*Z?%JD>Iw=9pG!a;V&$)`OV8PVPI=FY9y zzo8%w7i3vCfGzk$z@zb>;pI6x5LhqoZ3u*jdwLuvm%2_}DE(t1KsDm-hw$AK#N07W zjQWwq-n#N&8CA%i{8%izhsT9@u2Qgp6sH^A|4KaEPY&s4gcyuqsg)~LaD3!`Vrrco zJXclc@Cw(QjPkgL`xG>iBMm0@jq-RZMml*6L8$ut4gTCdEkyK~h_J9Dwx;Ngh;Q3w zXjd>4h~8PRBfKCh_giV7Uw-NF5=B zO^PN|?|h49h}!_sg2k-GW>^1^kS=y}mW_S^=0uvz0lu9K1s~Z-a?abUR8M3HW-B^C z%$aQ)p?BX%BCpX_3xSiNcmQEa?sS7Jb`CD0h%HPG zfcQ)`Os-n|Af!%|w;y`szvK!qee^8tS$40X^T)_S<*54;*`G_qkM_EO%Q&tK&4pWEP|1W!^@2YG7{J>56A%Sqaoe=1~{Zx0#G=} zZOe(2-|;(Gb%cg1ysH!2? z(dHp*4tRb4js*xd{Gs=0Jkacz(U5_aDZc-7j!FW0jfppos`la{@B z#R{XAvdPql8#iufBN`t#aHSiYm8&YCk<3N;2T2V9 ztE~|lX;|}Y<56P=j8>`uIri;7dnx4tm6LsKJ?ep~!WlDT%HC&gcl;Kry`9yT2%ae` zrI!_Kcwr#>;Z;gH44pmGU0)o>mCtK0hnWleLJun?G5l!Fs|^)ISJLP>r7%^ai^fyIkYSYqjDNo#-b!1?RY0YtiXOCT~Hgc_>~ zKAQ!+QDf!UPLSn3Y7b%P?`TfpBHSub5@YJd-@9jxmK10~`-c-S2_j1QV@NG2(EXR7 ze}pWKtd~?*{`G&@J!Wkj&RqgDU#{QEw`7@@3LuM?RRp1Ng^k%MKVf{&FrJHKG#+Qc zSa%9(!64L}--93tp0WnD1sg8^Z!=~9FIL5=@e!XueWm#}$RPAPGOel~pP%hd!;vg8 zEf3Wn@JsxAt4G!P%^+n@u`pl8O z&1vfj$j$>fyy@j+GYBs5uX@)e5L%!K)mBkaQMfnb>&eDP+Z|hMBVXTZOL@}kGBT5C z?z~3ST*2SJt>V$*8x2=RQ}#zN*b6>cy3w&qJTS7D@H``a<@4+xyxHkvhvWZf=CzZ2 zuwD1)d4cVZ^e!D$8aIt%h9$&a@15gS^7>}&F@_4;;^ARg%#MMS!wOmq??MceDFH@> z*kWL4NKRrEK9QrIpi_gKD49y2nWVRKfFl@;I~1qrpgr2Wc{8PlEX&HL&viy_y)e^6 zDa6jSq7~g}AGH!x#~#`1LYSb=0tN*)dISeSAB1)Rj~~##_=4SA-j$Y=XhBDe2!iq_ zx|^y6cn%zJ@XdHHPaVW=RyuETVE>=b+{$1$2V=a2>bqX26oe3r;9M(u2PGC93F3$c zlmYc2MXU`bHX61BJ)Cnn*uo)VO!;wXhVErt9US{im`-wxq*)ih>tL9Hy z;Bdfx4Op1IcV|GchbxD^WqA+a=4M1gz+d&s%1UxUKob+=2^%Yw!SI{uCTIw8LS6J4 zd{3d9%&#LF9v>;|?RU<^Ja=rDN<^Q+y=gWh`j4KPBtOKOEEwCL8bY&85PwUPvnmS4 zwfANwdtaCs1KA*jPhOWQx@Ju$2z4UnYDZA6vgGtpADBn-(R*h)!C>J4rOs0)u<#i3 zdeHctSMmllJ~90VYD%fp__#Q605;eoKEIAsN~|oN;r&G1Pcpu6^QxZ|+-eos`fE=>{R+Injd|50S@i=Mul0GmhKA#O{*>7(g;O*DmciMCJqm-1ug4H$0_Gyof zG?XZmvLd zL)ty<3Jp_zB0v5c+XPihP4v#0jrlGa4+NGkPl1^*IC%bxB+a|t6o#m7CGv9_fwgwsZt|fUTuToM1tQxB07VyGR5h(^ai8^7|8Sb#t!10{oNF)B#H@m29;kw> z-#>mxwNb0Kwst|}U8mlL{~kcYOv_ba<$90rzwULCTD58?GWKR_e*@r=S>O?d3sVB( z*eGEl>YFVFMjt)yx;k6i^D<_j1oFN@~UL{ z-)YrR*9iQ>eO>no;b;xV89LRz89a3IqGQvYC!Isr;vX3w4nep(<|(G#*3Kuu@ZX-Y zdn`}mOdsKgF>=3jdXD#hS&vd*NA|(Z;M(A9)Fgddr7PLg1B?4H0jHMJb6E$e;I>#!| zh(S8sBFgWd|E|*JgmManl|D=bRhI%cW?NP7!9nO?Ix|#~z)y`ePmOz)_Bt(ji~UCT zK>_e-yL@w2gk0q&MAk7kZ{9R3h7_*zl~@#Hi%MI6a68XMj}9222NCwgaBZrA_8Y}- zHxUUq=Le@W^IV4p{yVE~5(g5Eh!{i$sg4s2uRG3)!sN8r?(})1yU#cv+-ZuNIKm;f zdGIWGTM(^);6eMnOJM;tx9IVh%Bf--b&G63w z^eq@Dk(xhu?#3D*hBT}a0WZKNtu`aEoA^j5Lei8ZfbV#*(2CMm`apOG8<}b z>$Z}BF*IkP$y0MtYMAbB0a2D=&kE0(JJ2iJYK7jR z-_=>M6MXJ{Aoz7iUo06H7Z+gnl)@ha`y-Ux9@P~cx!C<0m>#i*^A#|ljdcOfKrGUE z7kxar^7hS*&X2oyk5kklAHqd5bLLRXjSZx7#c|oZN{A7?28I+W1qQpSDZQ~KVIH;t zI{~zoK212R#P5J{5z5BZ`D6GmchTBTijIDA8ISRUCiJhQu`%hXdxQUEoaYq6H$Tw~ zoud2LuUy3}{pX(w(5T$sRIc9-ko@8*K`$$MKw_KcC~VrGN7lygbc9pM+!C8nq@<)u z<75-d9@#bD2lcVs8kkQ9I6_nnsm@z@c;<|aRfv-- zn!KX4Vs>RJjLTF_-KRBj(FiIKvY;M}RGjAT3 z{~Ta$(6huWp-l7E_k?Cv7K=4mZr34Yh==NzFgF{78>V4*J1o`otBTG|Jd%CCcBl~5F#atav%6q9zPd0ACJ5?SrXOB+ zrFREX+xl#G#!awdEmhyL#FbS`H<)*;%e$FJ4;sC4vK);+faAb4axKK~sQ-(;PIN=- zpauPdxag`v_vbSho4<}NV2B?ApsdDA%wM|$^$i(50Lf&hyR_*bGg453csD>K=N}bi z_cMER60hU|&JBp9I(*|uHCO{o|B{7_ZKBuCvpRa`B^Vsa7Zsc4I;erntRT9GG zefz$4e;*oR!}2!xa=D#zc{Ud_>l1!%1S(>RR`9gWZU8wz+Cun&Q&Fk~`~QdWbu028 zMDzQ=6+@g5v4-S!n5=dStyY^|+wf2o_Ynq%3J38(u)D~(7So6}(o=L^jGwRe@;ttU z-^4|&fE;pNhmzF*x}I5QS#!u*CA7s2x0>d3D&CGAC%Z99x&%^10wQ}G0xGImF^vIO zD7acvaE;$-?vFlE)P=EEOxQjIu`dB7@B$lPUYVO11{YA=D&hMlWAajIYh_Z6qKAjF z@eY&MRpmwr^?xz1km;qk)L?APJy#>X{nZN_G_k<+c~^73bhLlX*S6IHjNnz5YfTv{ z3MslNf!JP~jGV1XbVjr1Q~^M?jTW&$I=TprsGj6%u)qp3ePzEz^zHNSg+r2GP|!LN z)KDnj?D*fXbpWheHSQ$~7*lfFw%q3r(d8h33RQ^+yu55Y7Rj&xV7ub>j*gD3SI6Nr zRI~EBUf$X-4t+ajNL3(U2Kf8yeBO$LK$sUttO~?bfpRt%gNZqxe=O`p9q)HXu@I_w z>K~CQ0Ot~JDfHqvrhvz1@4^&NqOj`deK1E>lRrkO<$>;oz9&@nbcB@ppYx>~8rc1! zV@{_L<4XzIvI^1b=)`bBCPww-^JvckYoBJARYgp7zRNjQ;ZKx`VsPm>in)?dW*Bw` z=$mLl+!b>H@Q4Hu9MwMOP@uq~LmSf$Bm-UmFfzn{e&V)Ws;Y9>jiFlJ>tus!F8y6E zZo1sy6;$9S74fkQm1Ud`errq10Z`Rzi0VP`5)MGRM)Bt#6m&9?S47!}TTpgm&2^J! z_FGq!V6efrn~X24?>oW*6Gl6}ShccG#kc2``CPVMBi9@cg}>a?d!>l|r6)DE(3c4I zYSf{>NBdtMeOKm#0z~UDgYw1zwUQtZ-3@`%<{Hm16^~E<{7SSbBRZLkYqh_%B2S8M z-dz2&yLB|nka}9Gy->&6Z;Mzfy330A12Bh}V;%dw$?Xtk2@P+1+_L4TJp{Lwk^+HH z;5CkDSH}X?^L`Btd5L*U8R~5*x#>aG#!fdcReyhb6QsqIn^O%kWKjUmnlp#G6^u`g z+^=l2!~8$JxpDJm4Zb?jo#N}(m8CSpa|>Q%>~qGOObq%!$B|EKMw{4<9j3sERa=_f z?*g{C4qd%`nb3PEP&1Y6yLK_!USN&@(tsi263YGju{_6~d-(m@oWcb_A`_rObwk16 z9Vk^K!^$7~y#IK{5~qF)>rMQ_!fZI~^J1}0o3y|mXrqG@6N%;@q<;2aSK-bWp<6;l zgerB?uze_Cpm00061H6}pc9!I&{qlI%jRDou=Dc6B0IrlXB0H#wBb{~7!Jb90lKCoVj z`oJ3=D1WLh+H>l=C#S;0AF@9H##1sSHmn!b{+Tx-6%&`5kAnh4YmdB}L^M5YrTmNj zDvsxV?A(ikQ?NtO9% z8A;;Fk!WvsrmzaD@SAQ-guew6D8S+%0Ohn}Cfr`h9e5jK?O}Fsw_IQqf~hIM6mnwq zYWI9ccl8z6Vr&?7#V}JH4y44#SpqmxV&xzRbuIjI0qg@NeIG%w2w*|p<;%Zt8W4MB zFcCHDRDBgQ4?6^@aS>E%js`@xGl)BCo+o0^&9VjnCemKMnQky`qLC1o{;PUxg2{Y) z-Nf@uj!~}5w|ht&1{fn`2?ocTWDjysdYaW*!wW#o4{Mwp^Uz-tU*B)rI_R;>TW_sJ z_oE0m`NXZZ?~KU|S$OE5imVk^>(6qcf#Ls-0nFC*zcQi?TZWb}j;;kTqmC}vmhau; zXewuc`sN|-P9+J_ko!=iauPxM2cmP>CJ>-=&``wIvrpq>#rNY-l0=mXL*j=32m^h6 z4?R4-VE%ju#1{(NsK5V0fnV%hhm-X?v1zYfX#)Ex9P^t0q|1Rs6l0KQ4WSDe^jP93 z0<$b~wa3@L=X5vbVL!(Y&M|2EUxx7K2`j$`Ek+k6bm-&h+$azZ#AU<HWkqqo7}&Zo7MIaUzpcdSPdi{LOv?y#mnfm!Z~1aP6io>2r}&U3G)Xk zRJ;kI99;u6Sd~;KFyfZnY`BqXue_4~#1BUB zb%`-{db*NDapU>1n0Prf7Ir@YVq3mIZ@P38R8u{zw zWr*fFT1YY&P=Z$oRZDE%tW7k=yaNT_w8YOX*r!@j$hm-NNR|NgbX7?DTsP#ca2>MbGGhTUE~VgqXg$pKNW zEp0FULF5#Hej-)fG@}LoDKCfJsK)^ygr$swN4COiVv)CBn8{EDnh&A2^{0RSx(>HC zm5Q|kqFwkVsQ{&35WaA*|04Wgo^Ub7k@p&`uMR@pu||Ld#JZAKuPXTtDIPOJdy}~c z*yte0c~r6b#B5FAiJ^n-Gd9>oo{s^-bCGUe6w(eqjIEoDy}E5Wl@uYBbm6yN>4gCY zJh@w%2SZ&OvSrjr+I5)uD6qd-K^7R-rUBf?qsN6W1DH7_LR^3#cM$4 zUGrM0u&Yo+F;Kr_QgUa=$;kfh?QX-VKXoB|(-;F{&i;nS>t$prA81m6%M{82&jiBs z%4t+|AoxRr=eCIg5l}r(wIpmtiEyBZ3hxTR@F7#r3Ee37foml4HikJ}nD@v=9Le$T z6Bs|SFpE=tG1@j@rNY;O0q6$&BO;7HKZe|ag9LD?N$wJMYxEn6L@{yke=4MNciNi6 zrErj_MAV@7z^3SZ4n`<1kJzpce2E1=dJ1B{Oy=X*SYg2j#9@AD*FmENdH|d$95Wa4 zm&{7Oh>;}aA)EIukv96;or1ST;{c!%%6mS(3&j%L4tgZALKytWK96}o^zoR=XfBM0 zpfU^7xD$Y?lL5p{JV^85BedPQZ#Mo5%%BIsF{`19@*cxabiKfq_d|as$_>m?m3*{e z-MZB9YI+@v5<|p2?pgyyVczlCv1R<(7NKtE#9sqIC*!m&wMIyl>>p!`0EjLp#1S9L zo#=yrgbKo{Y1EAj_WAaAHx2CDcH#Tvn|@RsO_%6%RV2Q-jQofG>B(IT=jgJHEd9Kc z!90C=G$RDE2D2_Sl@XRRvBEZ|@yk6f!!Y}0~g>qUVTzrhmqulE?v?_h%si_|Z1Csf$9d%?KCRTm1zjWsup%polTBk?^- zVXAp(*w$MzIe`7)(~e07(w5D{3}EOolJG&H{&V>?@j9LQnocJ|&7=KB>HGWDX^og+ zxFr)KiN&(tl~0&{IFuXGQ}g!IDdRVep+d97`ExXU?KC^jN zSSB_WdK{shpVJ(tj03pV3Jpx+Ko=HL@y$Sx`_W{nYMvb4LRbb(TEI{MFmU9lzRF|! zdfuj0gTN}40h~AjUxAj4xB;+KNtS!odW1usTf|SYTCa7mz`628P>w!I& zV-=36!!RCH+}hvW2dG44D$ev@-|wo(ok3;k0bL~~JQV4hBJHi!2v5XubxnhsN@oKA zFM67nDW0GeT${23N^(qj(<6k_h3Crjhu?$95VTv|h5*TS;kx@*zxrs1Km9xMGaFpa zLbv7wOGpPaaFQ0d5{@=*kLfafFz=mOHrQ+) zkb~4$%bF%MW2ut`913#wcp|T$>^|Hz-ye!N4JJL0(J%U`1C|NRfrv%LYsJI}a*d&PnT3>O##@Y_sO!sOvGCiECZ4qu(S3og6lAG_9yG)P z!o~Nt=O5A-=Y(aOEi9qJFbO*}gTXXd+*qe~2gTn!wGsq{sM!mbMiU`|lQ6XHlkOc5 ziz%4YWdm|K7gC;i5}J)T2gqVZs8-`qOt!M^Aa#TS#tQN{urF3L#NrD;?-0IW;vGP^ zcoW(Le`|$lUdd<{tIfmH3Bfl@v(*R~fI2mqpae(62j?bJhitcFZ-Xud%KmXmzBO_u zNoq)LhZ*NY`Z2LOIu_5xr7J{?eEN%%i#nCq8Rvsp~ysO zLq`CxTd*6am{=QWlYGl*+Hw}yH|wVSoR~)_;N{Km3RSW|*NZ)i$%DQT0b;Yvuacoa z1thlM^r~Fi_&MD2rO^2CIc8LTpk)_ArcIzd7sXSZ_y}DAl;@TSadE-^zlaNwz#ThY zWwG4{^-gZx_YB-~V1l=#ghl<80(KVxo)JA*cS?SAbaa_D^*Q2Y_M%5lliuj=Le9L- z=>Sv@?Qf!3sQ1N{lnUHu#x~@h>+m5<6R9}x!;+Dgg zM$6&v6c+j^L`Dog=R4%@ya9TqW~@}Qgx?4_X1JSPBjmvexcc^QXj;h(2m`ZBT)RXe z{$1X^#jdFaP9~ZV$6+Y$;QJ*v6HGdrYap4$bCLJc;tFWK#ABlji>d~iEpRl_U*_wv zlCr|b*Y~9Kf)brdxoUit|Fe%9CAcN8BhQc@)Ejd;AnlsZGx5w z?Oiw%oAV(~+r;;kHJQSZj@8{$1=#a4Lxf=1zmm%(KYFVcIVFf3|V;3ClkTl2K&#p7(cz?2O6uY9?%@Zfc`4#LKBU63v(Rx+JZ{H zI>}{hY*((d`L;5jnL1m8HVJW>RSE6T#0LKi|B5}&uF}_H2=Be5f+7JCykg~_wMQ5# zZ7u(=IZV(Dpq>6Yj;e}LsL1o-afA_IMLEw&cid}Jbi?hp)-eDEOQbPP6fTa9KXvkog7`6r zLsUNy~64mS;*rt-msa8 zC0Dg~ME4VOH68o$Uv!WsyN4()=l<9jT(He3-1;^s*%2QZf$|aOw`ee2MfXmRgN>tE zAa1>0;f*X;6}16lVG0NAYX&8bx#d0_?utK@{h1B|gUObiJK!?1lJUg!?aJGnqi}qB zN=g#gcoV33dTH`vo73hl!Y96B7^&Q&WPt`Nv~1HMAh^sTBMCf;cLNxRoXP9OTK`x3 z;%+X2+r~R18LM8`T1(VG6JS1RI=; zaLT|a#gS773N=)ak{$Sy{zV+Q=kzInpi2{+^biyhDNq`u8x~{><=|aymx+MC$%c$ z$!Bqu_`hx&L|<7_6%gLLaXUv3uXEV5_d1bcj3@gGR78|daj7iJP)}aodE5y}e{wO9 zD@bH`^Zz#Sz~!aR+@~`er%yuy1X3o;JM7?!pB0TyNPGktmzgC7_k+;+n^ZaIBh;W1 zM8FJ|AYB}%{qK!LH*7dW02&-O4F|^*Ar0H?yl_6SNkz$>7H-pGsOuf}IaEYaO#qX@ z!E0ALyn#+oB656IV4;8dtQcjO^l3>-^H9VC9FHofjs{_%1o0u{ z$^sO1NK7TvmfP$<&HZ*C(fj~P_-Od+)PObRYmiFGB?nM2aDf{t%XWy# zhjVMgBO~vlSSjst2t_bZM7r1|%Zbt)p#Y7z!Ibp5al9xS3|JdVG~w?x>` zEzy~ng9^~wNkfCDc4_i5v!;yebG-8Ml|F!Sczo12pRNt|fS?oHc+@M#3Hlaj!e54~(D(cbw=MyCsb$j|sGR{v zA0DoGeD*jo8J^?snmGi@>DvQ)>1m^3L-Sp&(9#7tFHxL{YL!JAC}54pSAFOIjmh|6 z?>J3pT#!XuE@Cy}BbqGu9a9Iu%?>m?SPYfRN@AHgDwm`Y3d_fy{sKAUL6|;X_$YWz z<`<52OP!{4Y$P5C0RUW{6l^fu+I0wCW-HkE09!(-GTpU{VL{*#1{{j^9B>FCz!GVI z#x`-z!NH&45KYfgfWtX00EcMhtv4+Ue?!`XwQ!hLLdZhHUs#3cbyyx3L+AGf zjRZi{m*pe|tIv+9ADcbxX7Xwd-fl+>!P(xQTO@6Z@5g%*!~mfM>7bPNo2aS8P$ydl zsO@8kU=%4uki53wExJ2|722Txu*5H5iq^&&kYFqT(V&J}z-ONrk7#bb+6D%zLZ>?y(^wV+TjH;X?z!>oZ@KwV7DvlG{ zd>RTeHnM#RkO?I_M)W2O7fgBj>=}0!e%OELa1-6Ik(B-1y&z_I?(9mMJ8zyQN`Rri z=ZIL63~;&1geDYqG$Y&NM}ILnia7v zgcy8&3|vNvAXn@aV8hqOiaHK_;P=1GFz_dADgG6c0GZ9*U)9V%+6tIv+@ zQrGz%A(S(y;+zhd)z3G%>?QodE|K4&;&>JkUk`}e2v<(ih49IrjC9%aFHY14PQ(GM zA~__QkE}^F>Zvt>V@)&?gR$ln_!HC|uzE%5H)HSJ5Pk-X1q;FOGRETK+FE^hXm!Am za1U)h`Iubu1GjmmY#AVJ4dcWd^Tn&mUi4iI z{hPhT#d=LTUeQmFb_mcvh|knqb?VPK4D%!zRtCdMG7fQuyvCpU`s@;$a~=1C-8;HT z4!=`41c{RB!z}D_0U`JgN6zAv(~xb4SQhmZH4utFmb5I}oes>Jwa#@8BlNE3bt#g8D~3PIVOQv^*|S}5y#h`{*JlvsdosB} z-m!Mg?%lV8f)b!k=_F4lNU4G`RaI4zzjoud=^&y)3mN=3XhXEmabO}jw4n&3|M=3b zLJ0mUE)*L$y{OOi)hQdcZ$E-5B$YU?3M2CnUP(GjtT`}6{utUR@yG$2rxxL)Lv-*v zEG>7C0AdQ^LID+A0_ONdP-v*NCG14Y8}LU_3k`02KZ+hJ>S?KO?(~wp(zP9U%y;g* z$bn>+uoRgiaB>(rLkH&FqjDxXE{MCW>K;P&l{QCqvpl$TBi^-Ee(6y#JS(A{qw%(Z zD}H@#SLOBnAq^f%SBFyRV^>}?o=pcWdZ8>&zufj23>_=Y{MR!W$-(4&0#+gHT!X=Y zSx_EQBY(P3JcSCj2EHlP(}?6N9ua+LjWru8VEbii&~o~F#&Q2^s9f-n50E-K5;AV8 z-Jd^%fIMUUG*6WO#p{*;^&4}b8Xbdjs?Bc!!gKUC0!1JW3UQh9{PO% zU$?X{yxoNe34$_amx(Wc!HhMD)7XW2ru_m0gI?Dnom=Li@AcV#n)govX@s^PUY{L+$C{jepxTy`*m!!v9U>)1YuW=mry;!yFIx zrS6sFCoBB7$8u@lHsU4jE8@(u?&B|S!T@iXAOq_n;$CFJw&#oLlOZ z{%9)Dn7G0|`!FFqa$p#9kDoor*7-vjNr56FIUpx%hO;qa(CwTJbVYeo5`jbIa9Zc= z?FX?_Q&K|WRrpc-@kAXUCCuS4OT^k38w_GtD@uv0jEy|V@%xg981$wP4O~NjBk}MM z#HER}&0F}xy2hz|3Sk@}3us6X4;@)KXh3Y-TUkDg&cWL3u!GjjM`%Oh5?GBK;Dt>5{8y0#0dN=qPGPSyZoHYg4ZN2j{j=P}~~y_Q2*2>)@jC=P=fWRHoU%}R=kimp(vu$t8(5j&u*@C8$M($n=)U&rAN zZ`?S3VSdR0M9tK<&2J8oa19gg;GYOY1qvli0{Eq6-QbWVoFa{^#9dF9VuKJ|!v_il zn18)5J#Tf^-CNpRph%42RdrsXI_5?HvfQ9IN~EV?u!Dv^s?t9&Jooqs7Fz5@R!BNK zm&HPtAO-#(8Un|m^|2|?626XYfLt_xiNOqkqhudK>XsT9m7lhs#;+v?5^o0C?lR8w zD)_%602jNkX`}k8$c4mJs@{qCNOTn;>!8XR$6p8DQ7w`PP-NK6@lK$cB61g}i4A2% zH>!%aR9T?p4s<^HJMb30m4d;l{iw;*6cT=6MB9Uqlg-55Bu+O{dz4z4xFO_cfJw31 zte6B~=oi4OvA*_E@=CC8FjjX%+^mDjSHB446wR1HKBFOcYuEg8Q~*mtKnKuI457?$ zcOQ5HqK$Az!r-9s+RL|SQ7k<2gfG+UPkAfj_alf{j}Ei}?aGi#NM#63qW%B%o}g)Y zmnv7{0m+xNn~Y)rC9)jdFgBCm)sy_y8OF5y`&$H*o{#$Gp6|O4$pT05j3jw7;bsw z2>zFLspRvVfxwPTSID^*#E>y`BypoQox0rzt@-g{o@upXJMlRrm3L|{TLz`+xfwBr zcuQK45MxMuA9ZCQ=9M^m3Y{pLlSo&eJ{7s|Q@?YG%z8BNk8p@pne_&EKy)I3~>Szw$D_5YT~( zQW?y4956tIG0T(%4dmu}BA0k|QuNm2QB*U*kcD%AEPboN35ew0P8e_ikNZ)AfOvIi zMvtXyrZfO?bRiC#Owf-3ky7o)yEuBY5y-0%f(qJ? z?bia(pj<@y2^3Le!i#oA`y!I&z7X5r4VLqUzD1|>z||%@xxTrq!9U$S_^RGM zcQlRO7}EpuHs~K1Ur$~htJirKdk4#i;u z7S;NV0B?R(feDffw=s{PPi~@lSkVobi5v*HZTr(4F~W=ZRD}yQlwCA&5RtpTp{&D~ z+P^6EyYBib6OO#0^cmt=|H2mecQYr~;mDQ-ZJ^dLiI6dy%=9zI9fg}m?kM6B9z1x! z!HsOt6pN`<1JAc#mCiw&5d;v95rZ09fOsayvG_a-?ro~_Ks}b}C;pC$M8azFa=n=J z>_HXGG1ovSi7xr&%ZxG{=)2UN#+VJ0u-bR0zt)5$TXk zKz&q&xv6QS$oP2zU_Pdf1360?L5PrP3zf&u6VTjTRA(ba{h5n1=a7wR#ctk{(yq?u z)JU;}Y`T>CrBq&{zn;`Kof~{PxdBA0J|ZgR27cqsXv`Gc*NnD*2kS@eR5f-jCSzrl4{&D?lQz%FyipSWB(+@CEv;a#?ayYujb_K*5_`= zb-^qZ_ee6LbKz#dx8on_KA^@bgu$7#0rojYoXJ+4kx89HCYwNr(6&&H^yp|L^_?0p zosn2l(FK?`3WpOoe|i$xO|JWKgVzslJlQqLB}kK*sN(}g6FSJD<2Z%>F|=55J7zm^ zuJcZFU_iWo7LuzaDFtZ4L@V)>kZa&u*}r`k_uk;5EYT-ZaL{B7$lix!S^O@ z)b%I8&%Z>p*^BF54UY{CQ{vNV(S%;vifmEXK#u*2`C~vIDd1hG_@58V0nE4=#(hgP zk*;G%W;fkvf(htj@{dD?!3fYKIvu!hxg4I>gZfS64> zF?|Z6kmLSoZf<7gp}jz-0F7Xd)sYq^;@=z4!XZPdY8u+0n0d1=$6Va?R!XPb#^_C@ zPLGEsUM$2U-e93wD)a7JY#+qGns0w^v6t#cuT3Za9pz+Rt~x>H7~T1-6#OLrU;+~e zee(3^H{%%7zBPnU8tINFu@k0dpuN+or}0u(9EIS<%Q^wakmJm~Zkx&#oCa=45R;FU ze|%NNW%_Y(`9ctZy0rhTSj00GEW%>9!$2A%HXvU595?Ra*rS;Bq?FlyUIXgQYI4N< zsHgrcrsB)O9Y7WgT)wt&aEZxM|8>kI36p{lmOMRdRl%Ii^*4n4Fe8crIlj~yy9TyryClp6Adt8gpbr7v2BU5O`k+gr>XKAuXd6+g_fJF3ND>nq z!LPX|KY&ZLs$L6NnOavwiaw?jj;SpCa!CKLQ|wS96BB}*G7kXzi3oaW`zRJBvkWr! zkp6JhuS}W6O5vkBd#+xG1=1B^oYP^P8tY-vu}}I54|$<@0g;dM0}}_Ro_>7t}AcEq4811)eX$b#z)Ol9tTTX zzmq&26Gi^_>a9m!S;%Y$B;=Maov76XO-D-xX!aUfq}k^oWRJs?7StiN(?r5Q6b|7} zQeBUoew>c^ulUHLJealt`FRr)%2o5#3v~8Dvk%-9MA4SOGmUv&7uX`U3X?S~p#vpd z84Z{LZFwyUb*uWdatm$DN?0f-|L)nd$41f-iY{8F9zy|CU_fImuw5EN40##hk0PxQ z&7KVRgX~}k`uk-KWf|*4C{cqlBIRld&LD_o2;G&Zew+i_a*zW${Vood4DSNI?NzKodiRL@c=xwaUl6-2E&`+j$Y1kcVi7Zt ztPwk4$=pc9qRNnYMm;U|!nBHKDRwBcNdiY;gmWOEGSNfHlo`{0W-Hw-a|;Xf8aDc( z-wv%G7N>q~ze-0R;{poa3Stu89#*}|c}W-5}YAL@XvD@#l&=oMl^6UN@f)jfo+m}YmTpSV^y(z1ZY z9XRC`P}hk$^q^i?IH0<#2D(xj#@U$=bpa!D&)*Otr8+r?IV~q_sYI5QP)9G>F0H6&1VMH<%1|YmfCLoMwj6)hI{`~$e2gjGb8~47&DMjmKkhP$A+SYz0h($tN%Hzc-dBh$2_V$QK!$+PW{4MLqS@oK z0*up__X-|U6W>eTyxD>wF$@~RREaGq9CCI7gfxI4_qR=&X^WNo>+z{@4taLxj1BkF zsGOzN)RRMVY{~)5B_hoz72$xc+*xSiz+<0`MN%<{V?TNMAG$=}L!!r>MYJGv`S(7d zB|!>BSZWu{l)RB)4Yhg~pgEh2$8aW%j`Dl%XzX&3l$Y0osiBr2Hdq0L6+ZSFXdAR~B#GNs@BsAF zWQl#WKO20!2h4;tO#{#$aiItcvoZFYgpNq}pR-&LHO04sJn(jc*hEe30|YftD|QiH z1QBkY zY#^kx5ql=8ml3LCw54?Z@I#}kJv{or$jAt7+VmD257yV9nSezX3S!#v1aKfqYin!c zbhJ3Oxk1CF%w^A;%Bypw>OnOB^2D@#cpuG2|71M8loJuMockI$h*VrR=7Kz%_rz)! z#)y>QFJt00zO|$QCvgNdXwiQ-o(OKA22H@@#G%MkcFVzi@obeTx=9F7vN9CIn3EWP zchuSRv%@w=P)cB#JP{^_2Fo_T=_Hvl8aZg%6)Y+?$#Xi0CTs?6B1Mu(t9y18!%?$| z#(%bPx~+D7CA=*Z#xbTO7I&6LEn*GTUw;u7(d62*o`Bg`1u8l?`GF%*`lVQ0k7-mo4>4h z4ShQZ2M`F!(iBH6${pHT0ba3~4smp8H#MyqQVR3=jr~mN2vkNqwyAAD(e6G&w;Vh| z#N*O2Gx|8)Iokz^$5v6WG^lR=JeR>t+Fd*726Ym)N>O2hmu@{cb+S)+>Tc@W?|mY* zWpP{f?K;$XnTKoL%7fd~H&iXQl3aPc?fZH6#<^yeTEBPgR#D0Il1=(e&f(5~4Ci;9 zw)iqH^LE>E*^O+>gv@g4HJpGvM}RX9%6u_$ zBfx2}^^y%HHK2jlsnbc-$w!l(iZq?PGavtuJLa`%(NEC_B^6~!+TD?KzG-X{PMAe<9xhLP`imaXkN7qc z3rU0O<;BoO-`&G#9S3iUDl0e1f+MR0ZJsu7rvIvE8!^RTHTfAy-wt)iT}&J8xDQ+N zJre$4karKXh?=(2K_{STIoMS?45YeFeTLMD6DKTD+fo9jPH;<$8LNl7S3r5x&!k7Y z@LA%AIK1@Cs#|-%ealgNkPgpV3{eIEc*%gd!ru_=iYCWW2sq+-B*n!&|HEH52yhLW zlmIxPFjmL4Yz`V;FmO=hg!Rw3pNs1-=CbwBRvsvM@-OIPgv^5SNTnl4AK5FBH+1)- z6FCH^lLQfx@VTG75sYc&9hiCKHD3z}kd7q@IN9j_W!kr?Hx_v|haRPXLMk6Lc*}Io zv&Ba_>#DN*n?xcQwiK;_D^dbut zp4=^HqU!xpcHKA1i6y7EH4X+fde^C+5D*eNEqKzx0-abx|Jft0ua#fb7M}QGJj1=Q z@DpDBQ-$MnmUN@ayXo(+zShi3mwFoIdzA52|M!u1I@|xTbwnDJlp|IeW=Igp--E|# zha4n*Xc8>oC|c5yUNpf)9LU0x>_)FyfYZv9H2FEOB{^b;XH%@K%II8fZptKskgL*u zQp~`pKUb^lHhh4zZaXIbAGY2FF6aCI|G#3G^JffW6Wbh;Gi45?Y#LG)T{+9loXxo* zq1iCZn#n1L8cJOv#}P`K(dLwuB&RYfD~F<@lKR~r@3PPL^S}MwZlBwHHmJ=+>9ES)A%Uabg(BD(w)?69`dX|5@NZ4D^mgR^G~B0|){S_~YJl zq@ub_wk+jzo%B{2_;vH3l|bI}2L( z*@6x(KML<|Kjwt8<_>9vI2Y9CWK>7O{toj+e}Jz=D)muDZ7~FC&;Ds1YbB+jrY*oCMB3pT4!?#;J&{Uf|Qcv*}tFBF^dyjwN!oMS@ZmTE0lv&=5Vy; zMwmQ=wJlK6gOOb0YT;r1)P^c}ldqzNQFVGkegv#ZHI9a_pOPeIkqkvzXi$-iFHbsN zduzo{aA#BDB^7SWz}PW08SCrci1OkYH2d7ypH7%??50#uk}*fcJPX1q6(7WRsGdjd zB6`yCea@fnNOFs+7-X*b`B|O;MZn2|oj5+cNSdfp-O5}Q{Kx}QKIGQj&Lk8zjbUoL zZ^njU-IJ=J6~20E;ap*)yJiQra9mO{oEBW}j8l)D_+mq*Bec>`kEtA05#B?F>_n>k z014)`*Z8kL?nZxMp$rNe$?_K){o{}I)=+h}#wmGk(8}=e@GdV3sDl+i=qdgioXerB z@xV5W#xLyn_^bYIZbxit`;D1CJ#xy$Em`LiaJaD>ZQHf8OHLD@wtvjco#q`R*J(9d za2T?R79335(Qxu7L-r?kHkl7N4u~#pzb}r9f?v6tQX|V7(PWpcznhH9;p)bePpCIL zo#`YH3X&;hJYU~jK34rrGxdoHnC4_e@=-Y1gb*DZCc@P~A@Xrroh=@@ba4FBTlz ztqV8Rsmwb3>*%DbKrt0)Q?iA4lzO>8QfjO(JHk{4Kuk>ipUyTQaoVc;r zso$ognvvfdz01;7udakKeXI2}w>Iv9@FCe%d%W4;p9N1i{c>mW)v$BKuhu%#>5?qn zit(Z>YwwEq#{c|Iq&>jF+r9&$$aoV>^X-%=O*wM-kefaR<>e|Cs z=@?~k2%X?IaLjxF!lY6U!$*E$SF4cX1todR(Wx-S|_V*SH2FA)h#I?>wDsLoqmpNcbT=dN`DD zA$x3Zyy7RAwji-;a;>_o;>G+|BeZRuR7j!laQn`YoojuBa=%HBIlF}q%PmaBwBT_u z{pn;5GieB6cSi5q+8ZZcF#Ukzf?Vf8dy;nA6`o|b>hmb6QQ9O)#C^uVjB3;xd(Mn} zV#x5}!>hEWE?qhdm&BBd%m48D1Rra`t&HY3QJmnHFsOaPxQl&&N?7(#GhXvUXbXuyeCUjrJf*5Zu{Nz=JO-AsYpS ziNYP<0ZBiFA_wnD^XSpj`xs2s8B%wU>r4LOkf~uA^dhLyMu^{0j1Ssh38|6^UyI~g z92r={W>M(Y&YNaj8$FbLt`Qa(eLNR~;w!sxQ*d=Hfx;JjX&tsca{9srYY{tS!npSe zEksiMj-R^x{QRZt>BH2n{IpBv!#5vkNM)r;wv(D|N?ha}`usQ3-q6g3(=Kdp_XN&m z)2JJBS`Um^=y1kBr>7o_Sz!E>kjNtujhmmzMxo*I){i4oBDdB387S3DK_EG&^1hGH zc$>F<71jENb?e5d+kwF65*-XF?{*Y9@0CxrV3R+ljH3g5o&yYcjqNjW+WBQ&Uq_rx9lEpynO(&4!zg_Q#=uH=p9sY(59LUbA>_bFAZB1B_^L zQS>5B(W(74>(m)4?+b;i1(AM@I(7CNJws&Rj3Wx599*xYDW{p#XoSE@xJ0WLV9(*K zU8iM%=&s1GekzU<-8ZVn1`Jpg?}w^Q1t!lls6#|Gj~ES!;Ji>8{@?R@y3YmEmTIMy z+vfBFANZcY_7x?gJa%Yh<4dRx@!k({^v(Z$nY1j)E62Shd>Tb^dgtdvicMj$+I9%b zLJvO_H=ro?`!c4DX5IaMX&+V|go|H7k5C?BSg`V$t!tpoeZIXZlX zQzKs3l$*l+hhO{Su#upvIv7;^L}%$JO-x@%`9>_o!`Th7b@s8qy1GPow*#xNgU&u| zQ$4*+Ok70mV~5QTZ^mCXl~h05VamOI?Lr%MAA8Cu5NxZ@Sf`uAzs8%{lRS5II^C!v z3a8ndgFk$E9bHR$j*eI&AG2pqS%8vV9%nGPk%nz{Lm$d=_eMwYl03k`(6AL(g~@gh z` z48qbS8EC4#>(@@mM6y9W#g-_50~NrcF^w8FG=h^*n|lF5qK6%>%ffjCOQxI3lyD{)}QEX*(z@G&gbg@Za);9QDzhS?6jnbW>qJLTztcy!Uz9N!!Bp2pMg@IyGn zRXVi8KdpvSVSpDY2m2|tS$@)hR>7Ryev}Aq%FzIra?GS7+{sGT(6-*sM^G1p99qaZ zA^k%@%sPDTQpZ}mmr`h*os1ceTW7ijL)N~rr@f5=ztZ_ZSradYZnfVeL>gZBl6fG5 zjTj8}yLAh`VUt;@G-0z@4Lp1;2sAuy<1f1n>y560jq_u~HN?Ww1%e@rcgnmd%KSPc@uKZ!Szlt;z504 zn2Qy|NO>(j?wc6QI_d8feAIXKd|3!E}1OYYU{bMNxt{D&sXo%yNumY zo$giWgU$rzk3Q(UlyVj+j19{B@~fdA*A2S)hX;=p7d{y;<+TCosnmEIOfr6c92+GW zny-7XzwL!l^)nME-8la>y*ddl4am%~l)>~9);Ku2%&9Xf#`W{_eTpZ^=LLGM$*7fDRW@T4!678K{c`|KkX#Rvnz^ElJhyEeFSsLh53-jcN zXH<~y7+Oe25PD-MFmy=st!wW$urlR(QHX>w-Z$1|fX5Er?=Bg)B}t>0Keay^&Pmk) zKaq(lA3akmG^E`lH0r9N9(araMMOxbN#TE6 zgek|B4FTw=n534s`ND+@N$|!xOt>FMm1D}21#yy}41??G2e32^pp!DVkp=HXV&Z$Mssib6Bk8NA^arZ zV#k`Gdgz@aHe%Qx+@xd;UO?WeJdKv;9KwVFAFxRd9#-5TC~?tvXP4OknOg&Qz3Aca zb@lXs=wQ#tm_K^>HB+q^GUVR1zFHUuocH|WEbt)o+ykzljMl{Jsbi8r*8)MTkzru_ zHS5NxJ^DxpO?*oWoe`4+4%rxF)oj$HHfK7OIzpr*0_e9VhGa#J2&hzT>jtirF9>OC`4@%HQ9lc=3C^zNdy&7-PfGN~I@Q z3{&(9Bu5|H?ahPX3MBE}oRnIwui)9Q48=*x#hhU6Zc$n)CO! zN+-8a8w>Zo_FC~8*PO*0?z~;u_IzsVDknC3PX;mex_I`;huartzr1Wv3*oi?Z6CwG z{Nba{@7CEno;C31+u0ubntLPMviSt8Na6QrL>)6Vb`9n~@vS#mIzdmKKTqKbpJAb@ z>JP(oq!pzwF{fB4HO|pfOg3nyr6DW>3L^E>%1pykUvR6{9yS|jev-0F6nI8#UPznn zbg$)IXV^%p9G5B02T=PZQN#>N>!lN1z74khD|rY{=e{3ULAW@`>1}sG%p4$*=HPLB5q<-N zW8mLlFC>^dvMB&4qCiB3Q&stA@8#zxZ~C1&c`@GzV&G(G=US!2K6GoRQBSs)A?kCu zljBW^eHtok`6SiwCWo;!&JLG9Y|*L}yg}@DU6VFkfXtnT-^jB)8V?j$;d2tcW-{@X zbD_`A`LCY!!QTGWhRoUby;aESoj+5Qs*?8Gx*%|oa+ak-agg`AfWkc-rj@tAsl1IV*LHUM>eBwUF zm-XbtelBMrc_;|~E-Sk88m;vHkmh+s-(W8WZrX{qDFVn#X=tk2Q>X7p*}Hqso@u}o zOyqG1T`*I3wm^gOh^}MtDD+P0RFVWE`a-Bx;65#d8%GDScf1hn$NS;wlWk+6LrqG2 z+taXwTEReP1(Q!H<(gB$(5PI#{Id=N|Ct=if1+)XNW~M4##9(iO}ugWLu{NfXcVJ$ z6>-$BJzg`n;W2NTkvRGd2upR2EKtwIh{=76^QSDoZ#UI z9Twg4<*e4D8ZXbEsu%YtT5u=C0H(wL)BgRE1`pHU^Dq_8e7DX=v)?FHO<+3z&A7L+ z(@vizPw}75^Py!_L2r!y#h(8azRsbNJ{m z9UWtq)pymcB3Tn4HM&CIiyGt&Y016Ycf+aSO`KCH#nuTtgtc3A5jkT3^Zwn zpv`oC8Q)D}=o-Z+$X#<$eq#FHoliIzoqK3_D*+p{`ELhAvputsI{A^$=v-wQ3Qvxs zX%_a)G}+wGpFd>NAou=m<^h}w-D)>15W%;>KDO=9;TkyH!d6h0EWdTe{SW0#*5?8E z120VV4v#DG*$&yb7r?O`#c?2UOg6bEUMzc-!t=m8FW?o-h6 zR?DrIN|OKN+muMpfNVv5HP{EbCfYaR8MOPXkh=W4-U)YSg|lJ8Zcs|J+O+2DU*ggR zBa5!xxQk(87Z6X8`yhH6DOyI_2u|oLK&@PEgTLaz7LI4zG(S_A&=m%HAnKfktt{5` zzaUvXHAId|&HL@esnfgB49d(X!H*)?4RYyxE8-OFzKkBtopH#fcBI4*Ao=orYT=~V zE9p+@8bAyR+`82j(3+<;NYFO@y+bBJuOiV|ZIVG}Fc>?7;UdzK>!kl=hr^T2GnMK| zZX=De(LDyPM2-AEB_iuXv#WxtvTXt@Q!o&}8V(L-Jy(hy8stX>seqs!OIUPjeuAcg zr^&sSU=@-`jhT1+%-(tHGgn_ll%YsQ^CFOK1zfGEXPgqQXvViZ{PM-=c)lXetV5mN z@LP9jVqW*L>r@^FWj~kxF}# z2b&7N-BV={|F?akgz9?z@V?VK0uk`breo7xl_#89(AZ?Ixng%A<>GkW_7|GmRKJzm z59GWhuV+Xc-5qB_dCS<~k508}ztt(u+?Lx9DrvfVb6zNuHan7j!&N~kWx4oTPJ$WE|#9p>2;$MNab2rbA9u3A3 z_+Kr;?$L7`7C!AO3WLW&9&mc+G{h31hyyifB>r#=sQ0#Q@8}Y^ad=X|wn2F=sdT^6 zUsI{Xl*VvUSi1eUM1+9J+w8=Rphym3Y-$OXcG^}K&sW=>8N z&WO!Z%L%8Bh9ZHmv-Yk-9DygtGS=BHS@TEWDjR>C{AeP2>^BigS5S`0e{_skcw&Rs zbc)8_nL~#jTr%L&;R+IZ>+_r2jJ4O`ysUFL3bf>iX5ga?eiDzh4ePX`9f8Mb>8%7Lwg*rn$CjtSoQXdwAe>J+X*D|Ui8Rb z)RPn#%ug6>pow1TIXNA-Uh>;l^+nB2wLSuNPsV~{M@nZ}q*33_- zIuWj8ojDl2BUQlKPus{F%;@}_uLYxo8x^q4;e-zQE?PrQp%Q<9t|52xNgTvl{q)vS zb$B@9?c(&kyzAg21t8NWLqT^OdZK6!OVe@3q=?BX*&z<#+uRD^_9d3vY36lJ{$-jL z(4@7Y(wr`;3$V23!SY{)78_45g+wa(jhi0M}~@My0Q zE`!ZL3WSc3{bJUQFBG+~^~(tWEid|FE0}g+A6D@F1+1 z{jKu>&QmoomqrAjKO+4yYg$YN_Wi2G>wNZG@PxHdV@`g2=ChD(&@G_e-o=RFJK+~fhK&vMc%q=~v zuzZ$(+wN_6)2`sx9zPUCez>G}qqTnpQ+hE_y1I>Y2eZ`tW7;U>IwF0FWk$r7uI66GlsWkK;6jb%|QC{#zNQCtlW40To@ zp4oreU8i7_atMsV!fdeErI+Y5l@sm%YL zq~}9ImhGFDmyvjB%RV6$oGQN=C$H#+Awy6x@tUT%(*_NRAtb?Ze)SSB?VE#1S!Qe% zE>kRZ**)xnHcX~7=U`3Y4Ni9t?=sZ4Zab}_7U6Z~tR8b?cEF_8R)JAXpC8)b`EX$e ziXPrl(4(^h^&3-0#FB4=n=CJmS@5o@@W97EEwMEEb=tewqFktWbmj)EJPv?=1dXM1 z$~!;j?}XbRl2NXuJ8jVZEDD{9it=MxBX{@7&}B=?Ue2#s-zqkNlaDlu|@WJMm2j7n*dR6`~&rfVgnXdSouZ85;40MB$I%;sSjp zWXQ^U+?BPth0<^Zp~#scjx9-K83!8+rqi}e1eCxhb^^U1n_gbpjqvFJ~eH!D1KKO}&X&^42@w@Jjx5 z84WW8jL7|HMPIQ3m`HNz#aNhi^u{kAQTB+2RY`KCIc22~%=z^<{mGhi(~Pt$0q9x_Z;)+47T57 zDyvs<8;lHmEw){^ZrAKGFU~8{oO>=d_oM1~8N%0XYtE5|9b&3&+_9qr!-)hs)em^; z({bgbmR9EP7WFvWv*_Ed=k?4ZGkTr;R=@FBZeJtM^Y*7J(JRH(D_GWaD%_*=is>$# zTT1Vr{Z%xmB?r?y`rV46;5ArK4%M(50H&H4T)sXVyf4w}lEU1dShk2(&7LH-DP}+brD z0Yy~^4qx+d>`bdmN)}uO!z!^iz-3SXz-2G+YiG|eH$|a2G?_J^Z2Hvs&mN9YASg8) zqXX8(2T_oT^V3a~k~SCo^@=WhzQjE5M*c0@0NG1`qs%T)l%ToY-;w@BsX0%c)82*0 z(kCkr94u@Y_oQ_mP=>Uv`MGyLs(AG9=n8LJ{q47%k`H6x<`&)Yt`()UIv?=2PRUf4 ziXjISK#b4{@v0$Lbp2q`&p}i_f*4kw@$2{$<)5Nj+zmQ__l9p$M6x8;uuXeKG~Q!0 z%9nc^v#cEVE6U zJ=e=6X_n4gla2DwR5smwW#6D5j8!n^!E1B_>KXj+=Ykw^X{pW+hz1DH0`%XAl&U#i zACv8e_;-v-R6aK*{*>PY=f~o!d(+x_hQUGxb#k_!^wtVx8&JP0I#*Z)Tl2}S0QRl7aUz{D1^!@+@9{zG%$Hc_B#(!}YrW}0OGJ`nx7c!E^Cq4ZmN76* zMcfs+Ai4in7xuIAs51KW5+nZrh}R(#B4sa^fr5m#{lYomjkdTiI(#nheGZvBkm!kO zrk~hm^%_0cN1V+fuNbN~!9q^WKY%C6++pnu?lB1;IKJC+<_qLQK^F9=8{qmbtMBaD z%vKvhw{{&?mF4m$Hh=e{OYJXy*>gLPv-T@G12=!B(R~@QbagaindBUsX$*(yLH$>7JwsOdbCixCE4E)94P?%lJle zU2w}mTDfyyE< zG|Y%oP%Z9CZDI;>17{-Wrk{ZgxMyr6I*n6@kobj)$}@6lG|Ed70k!|E(L<@-O97eB zf6NIsWt*W*F}on1)d)cG(d-IOZt?N%(B~4W3DlM(n(i#-aqDAKlU)T+0O|kwVhl>s zirSDvKCgIT(a{v|BpA~-k%dct?ilma+GL-;9 zB~Vpd3Mz{^;vY0EY6Q18pC>Y^k8u{JipznD9WFHRB^8Ug7$c>8RCM90!k54{M%j5{o1F#r#=-xT?`~(NO;Xy@k5mBhxkK?Y|A`*=qz^82Juabl1n1 z6^_O2o6qqW;0x^}5dnZxTL*_zn6Sfb@qUEEN^69*^<2(*Rj?fU&B<{z9(n`1E-Cgz zNkX6^No0mMl0+8YC4*;3He%P&1 z$PV@-x^JR&w0(5=(28-Bi6LFh+@9o5wsrrB9+k#XPechS_h@wIR%?Df5H@$!s?N!V znET#Vm5TFoyggMGV$koreb{q*R=u;2qNYWm-oT12;|B+ANJhZ#Yl49lpqLpEvkUf6{8 z$L%(m#@-_YgPty^TxAzAnzN(;*^uE}bLY&cCu*YOPVGxoS9q9_QEKHkeJ%Ve&?L8=b3rF6&ZyvEE^bPm%Nl|X9oVpS;g0Xj1r4sEhUR6F?^6i)vW(Z zBZ;-oZQIhJ7gRrB=z0V$5xphM?%Q{Gpyji^@*?1M64U%ghdEQ1x=MEL{NoX)mcvrd zUY2}}9oeOCirzQ);KLO^?I>6__SAn&e$G+8?;c9>pc%0R>-SE?;E*wURUDwIn!Sw7 zr?Y@J{G+Iy2P2+#KZ-=hONuDNm2!S8l*|-Acp7p8E(Ucnm9_>u1A5D*>NX;5kfm(t zkXk$wHGzl~reDdr!)&+Id+#+vN${;aX)Rlh2;4nqL2-VB2EVCHqncX$I4w$ys>(76 z1%TKi(*uDbdn#RTZC+kpd)G>1>%H|Bk7G~Smjsh)1|Z;hq`#TCK#m2YzN06JzdsRe zaBWy4P6p3eo$h~!f;nwG=7rNsm&Uw!e6k$nqaD{^Gjb-%3pfCd{lsER)?__bCONK% zcd4dDsaZAwJSx$>i|g%~$LS!;)q~-WvDLcWW60~8A4_H{w*dyi{V0qVAy-?TAJw=2 zsTFUAC42SX^V$GFXid3aIIru_&da1L>zXeGi0GujmeroV9V7F-8|O}T8V;*G1dJRUVa$wCMjom}V8ULW_RMk#mtCpJp? z(!8{E57zmq<DH8` z)i_lk9OMeH2P>itNLa@14>^Idv_dslX>2+Y;=g{Y&fm(dJ0@Eu4^4!Xof=%%c2Cj> zt2S2Vj}zCJS=%cR@jBmtzCubOA-+HH9QkE_|K~>KWU#pf}@;@CURdB_pbtPI(Xvtm-P~&%SY+ z*vnIByfw^c|LEwyeAeEcF^t~87$FWcEcm6$9)4=!ed}MAwr$rgP+FAQmP7110jj`s z1vmQsO#McYy_K`mF|)!CFKm`7=Aolia{t-04=y1%g41CTEV6Z;q%G>h;c>Keja2Ub zl8e$(wBo}hZKH98qsqTMKaI}rrePb2J=3X{aupyI@)meMq4XAz=jGw?>0rt^OMpNs zJ=h2pnv~DE^E=xH0=gl%jTm~O&D;qTQQ81N{hPC;jN$D>R)-eiULOvpD(_r4pGTOj z4X3@KA}YV4-$vpdiQsnYR8TPSz?}3h5w;tT#T=k5t@&eIqt6@lIO^0=#?tiU(!kLZ z8udaBz$=FY9T&`sv*Je(A|mj-XVG0s9b z{xlMzPGIn88#ni&IM!8%FC`+_kK zxQq~CDgH@h8~1Ot|2wAK(O`fapIys!!JeHl5wQLC31tK9Q0!`o4+M)=6E&VaGd0ke z9A1Wgz*N$YvbF419Rq`;>OA$>UGF zO%DoUKKRAT-}1Pi_!sYv5;l7c!v?0!KuNVV8&DLA&A!@D{(@hB@N&!?Q!L_=gie;1 zX@<^$t@BF!lRKZPBAh~2nKE`DXyhg5Mo;6t=>fe=D|;EtT$aa2i|o)!cCB^hUb%xP zF|Z>y7ew9zg;}#zeekaU-&d-1mHyL!?>Ryd8nx{Pkm275aMt=L0)U!7hS)N8+dy^! zdwgTwVJ`aRpstg?=My~7Iow4n0dI|Bw<)tBX7Y6@tIn1pr3$cXmo3Tozu;Y&s1lpiM1A_*RJ4V6d$~zcJe-Kgn z&!6v06lxSz5{1&}h@gpI++SD))a8^q-_{s>B6SXnIOYCu+rtPx*?0+NO-!P#%a{`l znx_Yq?5Wehk96#3-95rcHgmrRv9ToJr3)IdXA8bBL3uosrn0^3=i7OYNyqbdG|ufe z+O^}=eUYDWW5<=vYrA*&ro@^RM?jN*kAN{>|HtX*x4AN!21XozeQ9#tCkEeVQ%$W3 zYODA@%MfnpoYY-gegIt4_$otu%0>vLkhK4bH|am;^S4%%3&x}8xGZ1(PNk}rE)+m? zoAOqQ`6?ebwrJ_ly}Nh!)%7K~OM|Bfq(ku&k^UP#HB0{dzhx?H*`}G=I8t5sujE3Cd6(K41Y8HhcQiJBcsF(YO%q=*Qj)0d3tngEjPD`%542XeI zj8rk1=a<71inY-e?YtBsR$dg{%|do_+oJMj?+4bB_Q_bEp`$V35!O6rcZ6jt>hN|3=>U6U|z zF|qteE4K#ZbezsY)DSijYQ>&5YFecc?FdzeznfI`RR1Nb23BkK_eeUh(jUIAJIBl8 zFW)D0v|OQYR|o--nB^4z(j~Hvm1)9nKV040yU`v@A{R7biZ(dl!fPDLDy$hzc*|gJ zYzA^9BRa89M#Jv|GFk!8?4=It@?Dw3FCdy_lvP6Bryx*~nA@skd#rB94&#I2Zguf*L@P+D|?P30Yi2?o}} z4RAc4y)%Eu@GQr!=`I3pPk(d&Eh}^FqLwq74#+5x#?SxhgbJ@fdeP3xO|##;6q?)VHeB1_D$>JM(rTO5V%zar62$2S8y&7EPKkZ!e zo{P(&E)n+11yXuz;2jwvB+y0iQBm(1+-_WOeZwkHPUNIYREFr6w>XgY*E$dE^ctKH zkFsMxf>p=Q$^~XPMilQ@qgK8{*WrZ8vc9L>{dIXUP-E)MN|qns;vX@=L570j`EG!U zwD#ZG-kimGfG{znxl|>5_)S1EQ!bZ>s!5$fhFLpT3i|Z7RXrsHt<`z=P^hpS~@p>udQc9;7-OUdGK+!5Fd-$!XZhI>jvHXN|+-MFsHO={Nj zgF%=j7<2*4gViw)G*n``Iux#JIez@Qxf!RJQF2U(!C#ut>tvk;d7YyA-rq4NzP-OC zqa>-4`3GGB)AwpeGOf9~}@u&n91HH|7cw`};Z)78T>4ycuIu7X8SvUL!a z%+B+fd?dX%Y?06llTBNYC%(fQ~3m%HKF>*-I<$=cm-i3??`J!l=Mh~pmvH}1= z%K3-U4;1%9xjY%tMp?y_@3x)Vo8qQ&*ue30foryg?QnIy4e%dRC<_9%T-F7}##`QLJp0HmCPgkF3t69{SG9^9>S zhEC8rPZLw68NeA_jthLxwh-LJbLo~0$=z+nt$%!$BFzunjs{kq$gq)gDAK!NVp%if zmy``NelD3G+~dGVdW~da{XE-acvE~f2eV6r9#e?#6jW(bpd}-3-5RX}!9aZOTpk{G zl56cBFv+lyoCr8xd#$PGRR>=wU#wxbY+`N+gtch>9J$Sx28~LL{rjC*ooptL8Luvg zl9^xC+3FqD>@x_x!cqKYr*pMYgITt%%OA6dnD|?Wc#hEB8qNqwJtl;5xLH@*z^zqu z>zu#~XDE?gv zE*%UuWZKM&yAFo5pePy1K~@_4LPY2Wr6-8t7L|QbQzUp+BB$v~jQTPCZ7}r$WopR~ zaROqPH=#>@M=v7&-%%%N5cP^`Cg|jugGVNRRU>&V8>kH$=nv4F+b1)_B?9DcgdxEO zNa9PQ8ci^=Y3Tgd7zHBz1Ebnz;;{tbc}Ay3wy{_>uzx>~lBSy*@j9@#p?J0sLJ=z0iLw zaCmLU=FQ=>O@rtc%5$hj{KJySit#R2PJKS^Jzgoq@A{)J(GLD%R%~y|ilZjK0Mbn< zT@3@cl0r~}H26>Y%Mq``s--P?;ch59Q00Qu?5$9C*bY%7(fajm7atC1tv^aTJyzH~ z7q_@gb|5IV9~{<3|b{0G9o`=J3G&d(~&AA zLJr|#Zk3}GE;PLS`cH0TgDkwNL%jcxpK0I7eNjZNMN?Xd>M5u8{Ik!X!DV~A@)ob* zZFIze{^$#Qe7Z~B%tb^*PGh5l1#w-8C?N(*`LVgPkt|%Qh z@*7LYH#yC8Fa$c@T(gOPLjMc%I=3Pg*^jx)irrS7f3j-{MIE#zARq-~4DqyE0W8Bb zA%qgmi|j4kPqT=v@2r{mMrnb}3mR;!juK`a{$Y0+0}lELD9Jcm*#K#v6-6NQ6(V!t z;~OA7x@pGxJy{N$d!}U%Q~EAr&6C~ZxbW$3JMGFw@n~d21;2+}4UyD5xQ(Lp@Zu_?$6+PhX6Yc4M7Ps8ExIL%^_ zz=+w>o|hhdiSpdtVZCl&Q-kIr{bNZ_05kx96*qE!_R5cqgd*G1GL3e*fTYpgB_0n}s*quk z?Nxp|48N3?HWr2JpM1|6iz4CimnUyOxB;dAZxC|1VU?r?rPMhutX{LGY$z2@xg2V! zqY$AAx-Mp}Mew53&$-P-%V^7U2ZwC|uFGvKFIneHKUnr0Ibye%g(-(}N?K+77ePJd zhRILlCI=W%Y*H|StkS6k_D3diHS}Igg#*;s% ziZrNbLj>rY_j12@C*FycfM;><_G4dI9|WS#zJ2mo#KsL9LJ{@NPT=*}u!mXlo6k{< zBV^ofKQ4W}v187EIq^i7RJwRd8uz2zN46AcLCzb@Suvd-?~E^dYInNt{osaT@6P`g z-MWhJ$n=*>jQoPaz5kX5vtzQWwEg`Tmok%O#iuKO@-A@b_z@RK@-V5bVb<_c1Z5+_ z8yV{t9e*aDcN5+O!-j-co{u>C~+B;HFmrg}*&ylWG#85n~G!PhI-pghD0R&B_Bp zL}J^$B~PQMW}R-qU;KXI0@OS|Ab?UUZ!?;EAp}=piO}yF+dHrspgW%uh-j_!(vCv*#t`7ueJ!xb`JmJp7``;l7hqiCeY$=dccLnlA2o}LJ zleDw%20?&Rwr$)vk;}5mO3%FC+SW0Ry}NPV-&P4E98lel`jHk@eTi_3|=G=nR}-(q34hD|&G z=p$QUr#Jl1Yr-5dCxuWeFqEv3lIUrN4;>OSWcZ085};NcJ4~qLZySlDST|tHqCJS4 zC)0{bss}0lxFF{(PI97O7w7-kPY@iq&-gwFJ^fd|@!u}70m*WU#5yct@G`*-0YzFZ zDoJ29fwrnt5AFyEPg9tzuzp@HYMM}Zeu6i7NY}J%x;6KxeH>9U;{Z8g@_eiN!euF? zKOjuJX{@q=>3)V8M?3h2KxkcMbTbB05MGS&HA+LaRL^P1mLkujy!z3~lz#JHy&vDv zfAxN22N%CMi-UCWS0wl|1+D?iwIv6CBNC5S^}xG^;>*qL@BNhmw-~xuK=TKOuO^S6 z&gOGYw2+OZ{uLCtVW{PlYtC8qK3?2Rz%~3cPr(#UzM;rzxlBWy_etXO?Rpi z2w6R)QR9$_cbCXN)^KXW$9uk9XC*s;X@dqt>dY1)_y4s*g4JxG@Gxvq+}VA_@0UAC z2T87tiOK+zNBdg65_w2fYeYbhoU!YT;?pGID^NgA4K-Ipq)J;^Vq()u+&{`smnyujehOawhL z$MYROi(YhEi$Mu#w0c5W!L%I#ot&bG&U-+WUa=V94fZ4cNg01Si#MXA$@bg2Kg$0%CE%C8@S}p|7U2KGm@aAY!@R1h=A%)ce zT_KV118z%+3E(~KrSxDa++r!v&$U%nnq7FLfWReju@0uR{JhGS?NnOW^m;`Ym%f9F zqkA{A)t6^rmealbZ(m7Y@l%&t-g{7>=RpEZu1E;6ASlB0x{d>DS z!x6@s_T0C#!%Ywq-7y-3sD}!+?wvx^`2z{CNRhX2=y`(wh5w)&2I99SqIq4Db7*lO zJsJ$FF`Pxc{+&*>K=RxT@dsj@KEIyc4_qKck4Ppu>zfaPKpQ8Mmn}gVtXA9Z@22-+ zT8afl4`V6f$L|LbP&vUT`>m)4pRqkZxW8xy){bK)wMu~iaU^l;;5Cid%H)Ri>+b@* zZsXu^%bY;$F52%YC9i`eXVF|hMSo$z&^VO?4-EaFO#{mz)^R}9WW1&?e&mFdqxV%T zU7y45d_eUVDcR=<)E6%DL{(=dSi?ee%`8@J&TloxfAZi)s5yYk#QL(z{^h ze|C0ql9s^aO!~!tLa0;9YhbR0tEeQ7f65yRsc{gN1d~R@Mv}&X?aF<|HqEF2hVVbP zqslI5#?*R#sa5e-DQsE(Y!gNy0l#`9JrB*SB=RcsLb=9}rFFGX-eX zp&cEx$8zpYsrkDmD~%1?T{+D#A%V9^646xHAwrW5!atdNP)wyLw46|~Y2c35jj~FP zL63m|xErB$O?&pO`~iWCw%Laq(WRi7lwy}_)B_VtipKt_SMU8Y-!q0`9bC?~n{+XF zeFcI(y<1*E6M_U-_ao-^+%8lywzPO!)Y=g*Io2uCwG5TgRi*lc9RM0sUX&WmA_#yX zFz)&A^#}miNrtCm|CoCVN2gb@c32Q1T_AA@)^oW}g6K@a7+T|rGQ#N2-yfThaFNat zVR`PqMPkPJ2IWoS7rK-PM~;6HiReeUMyJb4Pl}(*9Rpvb3YSa8y+FN#p{HwLu_RvF zKq{Hzj!{qS>GCysXMy)ZN)caV_T?aA+%Y}_@I2|73mNkQ!uU&yMvR)^&Axgk@Z{!e{FZG%5WI zutoZHk1E$|?`2MVi#BmHCeJ$K(YR_9Mxyit?SE=+GzJ-+=cD@-_&TU{-D=!4<_2ub zPLJN;I{I%g*?DgX$fT5@2pg%BLERO{vN2YzSI((j5W2meN3UV^XpXj!B19eYn2{ai zTG_d!C$)kRw6vcTSM5~DEKPA0Jha1M;a@&upj`;_iewIsiqZ5TV7#W6dG8F(s@h*? z0!u>X3Cat3(_knTT+}EO*}@PAEIyRFi3KFdge1yTEKg1x3q(1@a+R!tzk0BFOOI$k zNr}${2AuMqa|d2T4Fb*S{=!*Wh0d53)Q5ma`1-X%5_$#Hk9j%W8Z?;wQ*W-<;5XiR zCsJMlW4i#u|2#c)CkxT=m5i!Ne?c89Sta92I&8ufbV4O$UB3_L&#UNfnKu zPvT|J6Erxfp^}| zd-kxmrS+bh`*uYvl(?kE>=yRJ20gB&Vj=f~qC1bc_t@NPzbQgL*GKXr0=W6juO$zS z5Q6mO80`uC(x|C`XeeOWtww*y+wpLrb?RrJq~ZiwmlgNJBHq7sqtdZNm7yWDIM8De zMj}=X%*~5*mpAbM`|RUBrx-OtW+8UCg9EkCyW>~cT9LLioIhK9^Yjc(6+thun^Nsd zvXkPYh)x(3r65Y$!!}#`ViF&!@JGSS-8%-X8MyZl7!-}fD3$-kvAxieDDoa4Ovi4Y zUspG1(!_#gM?;Z_2Fp8<34o*FYxsXlZY4Bu_E}Hc&U<_Sy~)8J)&p2h>Qn93LU3Y} zX(D@9O$dGEsU=kv=8xu(JkB?%#!DJdzbjvacJ+Se=aA%C6jXDPj=YX`(~F8oOy`Pm zo3mA@iYbinNDiE_{~a8_gMVC|0?Dj!Oe2?$zIvMs zA6Zk&BX5?Jh2fb&D_`s#Qvoj5T|j)Z)FY|WGouc&qeE!{Hm+#Sci`2R7-P3fGyf-* z?4e;WSJEmyP0JCc!tW7Qg^xYG#89Dti2bWVfo*ciQtAMJ2X>n?HQZ{LY}(^{-Sw68 zoHBem`?1DNATKb6((ov@r$=`02iwFk)w7Z;w4|E_lg{wR~Gss}nw2E}&sT{*4J)Dnz$pJ5&8tM>h z!%_rPsM(K?y!-L7OTQQe0h07>5E zQ|t@=T`&Oeq4e1@b_0Do7C$ySUyIF;M z59)n+6D}WvQt-FE`0Ou07h3Q07dzF8NwqPLTjKXyA>L#k^Ere*eETmM6U&FZdDX+TzY71tCB z4zaz8HgQ+-r3~Wozs8FS<>kOO(=_`3y)YF~Xku`wJ+znr2CAt~2*UROyh!m;als~5 z><=bN?{04=)MMk9bQEUzB=63opPp(ShV(!ZcTSqL=ptwTC`RvH0Zz?2IDgb_jsGFb zW`_(fv|IrT;WsLrX+3BE_1dBV;X_{)g$P$*QLOVon21nHk}Jw62`nAS=i(sv6MvJU zlY{0`8PaEHYR2W)skWrApUjwW9iH*Gzy&pq@7Y2EoyN=PCgjSqy@GGAMLsERT0d7& zoe~L;%*wnFdwIq&QL&mjIG#tNi<*4XZ}bZ1P;f}%5PI-q z+%H@=pIJyXpuxR%Z(F_6V~LAC=j9FWZJ*F}g^~TIEFZ)kcV6z=c9!jv5$n_maHo&E z)xElY)4TcgaY?!7#&z*N^`_~3;JZ^6-0M`O$MuPPtO-}yACQB-!iGA{dVoS+-+MgffFgEM{ z8-|X2nG;=p7Pph~OabsRJLj+h$$JzFMHzjt`M3-~B*8lYy^DCRWXwzK)Zh+E>xCtyHn;tv zdy4!*mpa43-k41JY-_tStCgNn;07)_PSGo`gy-(akPuWPXAhc1NewS*!!$&5PD)g> zNyAE)@rR@4U)oj7BI)_$+q}gl%bxWCPkUU@^G@%$Yc6yMm*=78ab*9)muAeBm#D#@kiPZzH>pq$y8-ZKlrx5C+NPS3NEM86-+&vd z`bX7dVjvEzo$}>k=^P9s=g?$r$v`$X=f4pt#ek*3=Xou_3QJT%A8og znpr8GfhK8F9($*NLyqQaN#bcsih94)(9`BD`})#-;!C6Bj@T}sqSnz2rTxdWuN7IQ z3qP9~>wr&Hs<(AQXn^I{9hLq;M%~+Py_9F!BBjBdQzJxpa{N!~;~L^uQ59;%gA6!k za&2obh!iYW7LB1M5rCv}`AN*j%(P&_-fn=@KYx(J?i0dt)|AVCo~Re1;Oyu(2)Rzl zY)B^Mb5U8j7XTzrQ_scx8rKiJ>b(BC2d+AoDUpKVls6l8gOf@j`{nJI|4d4ku50Vp zub+C%dCTh+Y|GH?k1IR*=7+~O{jmMWBOaX4?Oe?TAe#y$OZ14jb)mzxGpFit1o*cc zlIJ?FWBiNTk{y|H-`jEQoPI-g`3#?BS8c8)Kr~xcks**hvi0wBiN(b;KVTXyu@OPG zBRDupB@%1bpqk^F{mJZj{>0X)9M}tHd~IltED?m=3+b(}j+!jBsS26%5`#`aCBaoP zAy~YQ^B1G8HD(tKKj19<$lKc+GQmGP6hO(y`-W?d*}oWb2}o&}N<#MDoSxggQ=|Y; z^Dp>^eR1GoSG6e!Q#13J+NOG-nS^j`U-7=w`cF33Www@vbzJV1V}nF***9LFKp`p2 zT!FPbIqwu=u_p7RKz6oc1s(ZI^{<{j;rf$sn$JfhR+melSJN~)5q)!f8a0T?=53XQ z#reu2b7<0VK$+j=i7*o0PA@16<4yv3J%DO`w&3FoIBL?-mb1iVAN9{sZWV$=WAFsq zW~gB;YhZgg(R~9JGR`a(!UwdtsMAx7*-^|4nQZX2rd&t1yxm%+>Ex&XQW?!jig7O_ z^y>O(nuc4mGq+AfNJ^Ug=SS~6C^{+2;Jw5}Q>Y3Y{+iy*Kl{;dA9+_3j9g;>3MV-K z{iRNJk+U~1YwGYrr4YX=y-JIG$KNvw11nlu5lT|F7#RY`W;ER-O|$tkmb03}WW-jG z$ZaZcf(sQm0Vu~$(VhKk)Ta$1|6z<03AHVd3H(j0NS7|su`MQ6L}Q&M=TYF{e_19uCg9zUS(GwCJd=}%o8qVqf$1FA8xa@>Bm|cOL{V)PH(S6R z5X-qgjBta_+b5w_eYLB&)4OAJ1u29#!5k5it>b z_;vFOB+!#TYF+M+knGY3JKdm89?A)N!4bQv zv(Nt_lM(>UcJY=U&}Yj*9q%V`5IPV+dL_?)3vN`7=}JtAH#6Uud4Ri3%2QtS(9&;g z6`ss_pTsFAOGyw|TSY;5s`L)vm~)Xv;llT5yh^}|-zc`=y^Qf32!2LLr5rLfqGDdR zVKKpdqn)qI#k~`qy=*s{yg3+>@h}8%+ur}N-3M=ybbMY?Mz#&(+5qG1la-xK9l66r z&!4ezijSbHA{QptJ~Y8?PiQDXY6&~-G7>2IS2kNB=qTb2@S-Y(o#xnG`Q9V4V(iS# zS~r`$zpkl8)@xBK^-`skPV^Tf;9qxYy~j~0*|Ywz^?&+g~ON9EMqG`v{JpNbZVrKGOCAxia z;oX(8Iy;nZ>pujXS|Gb_6Z9su`VCuDDLoDn1&?(7!F+q+dx8X!9h|)F{p2n`6OSx` zw*3HjF5K(YU;x^y|1JHpSAF-LMlU<=sLueZtAIiMD#LP~_h1p9aP5 z!x87X4|YPujms*(fmtz!K&fSk2&_7E;n2UYuANjeznW>li6@c$N4Tg1;8*x(ufzj^ zoL0+{+uhQY@8<>#P!M^4VJfTB*+uG7#*la~F7OJ7mRXMxP$5q^_ncd*ch*d8bFaCX zwcHcvTFeG>``r$6(BFq=w#;SvnOr`+ zBM)k>ct0>Q(Yz{1tv8L_^$?kxju=fp(gfG2vi-{M0IriXfYRtt^Om`%sqU0YLFqn> zV=~^4412u@gi!hL2^$3r3go4_Yk8)V+g!RrIaXAK05g>{0Zl868X!}XEt@FVxr%y9 zJM#yXl_MvH)R68+F+>0oZ~XWKDzR1Apxuuqx^LQh_=7tOK4PVOQ|Q+Z zM``?)Rprw1O?(cX@ByiFG}X1}hgC)@%R69zHm`G`%Vr#=MX4DVfCwjyY$ zyuAAM{cUcFvN}ONl7QQN;9`*QnLizg&Ia*~5|eJ!EFd_tzasv`0MPD#0wK`AI7ExF zk!oJq{ZT)>mgux~j+8z$#EHoM95X9duFNslNoroSY7JXNYKT{|`aTE%YxjcE43?w3 zBytXB@%skb{*(Kd*1Q-YsE@|>jqZ@Njg`tl9KnBXOZv#pZ}@1r=cquqcU9e|$f*GL`+@YC`X)oiS!kmkS|YYVgpW+s-auktX#3e76l#@j;F3eXojGvG(S2{3_YMK zCsV8;6NyET1pcW=7zU(_Z?b3i?!v2c9iaDAj_@_Q6+i+wK6j{6dZNN@*2M2-!6cXS zZSVG)MxJnV%NZ?uw>?yW_n3)S4lUYuzL0JtBh+WdAP3zQ^@nhGjV)g z&yMg#1qiSIr1ziGqGDPtU0PN{@#3bFw;w<0-43!S!JzYsWxYFSTVuI z&L`U(aP9|wSok(h4*|Hwv6^=hz5=oUSD|gd$IkrsPKRN~pp{uPk2}xD&aMTeVEfNi znWv|JYGlhdE&*az=2^qgWy!8iHSrjBDOE1I5xyz&0$|sxR(>);veSPqJJQmp=k^n* z>XxrcZ)=c#K&8-Kzwfl0kORQU5^ffwVZ?3ppqbKnzrN$1p6OsYdpxa0lAJ0@1lfWCdquaegXe=wGF$eq#-o6W~3qE4&bbc0+1k7*bcX0*?nJ%I8FspGt6$z{KN%PQI*Ha z@AhSug65esU*;d5gsX@^a_)=~st>R1{lw~t6H;w3xXI%D;fFXH9j^uARH71*G?=TE zAcP5hhL~HU$dbY=B_$9D*Z}s5^Vi*;5oK_Eg`?2GD0rw14!H4;A%-Mi0~Ktia@WHL zDJ1`H8V3_$5|}Zif8-%49yEQFf=18%%Est_g0UFk9lC67M-G{7XUHv8|EX*J1!yU!zl_nCUkCdI$@haEYN*wbAnq zPbgVQbs>R2$^y$w&I<#Q=aRwT@t=w)K!!OgWT1sw-e=3ZCRG%g@xkjr8r!sMcab(O zDu2faqSa!UIo7~U9nc}@2Y#9@#rwMN^ZPet%9Vu%^m?YTk)rC z_u>*$8E)Z|M(o(Sb+43I+@i*eRsU7=Oqx1&l(n$E_u?BB&z!zuZA6>B9D`7QOrkRt zPE({lP4qv96M854Gc6%@Y0*rDveG7N7M-Hm@EIun$D=bAIiK`6D=|65f%(q(It&Vn zBXY2AR!Vlax&M!?HxKABZU4VN#xi3YGsZHCjAbklqOuj)iBc+&6w_ELTZk0ZjBPN< z7AcaYv?vT|Ck)xLw1^}TQAwo|k)GFa)qJ1t?|FXrKXao#%XOXCc^=1mJ1(wxtikqT zsigfVgN6RBM*AscxeS0{KTJvF@(B|y&TqP;v%hrvs)^7?981D2!z?8RjwHZ8Op>@b z844Bs-ll>0l5~7>bIb1zZ&FKhKX~wSn|lir z29L46ajWO)*LJo?if`Q+*=_dSQ?rX#Eq&4}=E+ij2#Iv`G&*bE&SHa|B}J^t@+Re^ zXWdtSZTl^X+EDmGn$JF3T9GwZ*->40L!}jO?r6Kq$HylukT4k{OZo!mi#{(ygR24O zdq9~`!e`hK_72R zvfd(+6;C;}P>375eMY!tza29&TerV=kj~LrBl`P3&he|epZk}t(DZnhd;QI}r7r+H z;(xZu3ti;DOx7ts+ez(7_wlVeuR%x>`dn?;TlwRffJJ-fu?zX*fSy75D#@M7P+!rC zizK?9KL|ySqcPMUlSakPfEsxma<5PmD@P=FuYumJCaUE>{`_+ⅈv7c^UA z>%)E#O~G7SvfNWpqt%p-J3Up2wgm3W!zUAHfZAtHnexh;?fHcq?!Wfxg|6Oq%lwS^ zg`xwF{)60f?@xc%JeX{3oGE%nE=G7_VvhYqK9w6ASJl5iO~0}5c*(XHnvaL9^$6&K zlAW*r7QR?~)ct0j0%uuM)ogs^O3pITma!7w`|a4a?Exo0)9iY8YF2%P8=Ijg4hG!! zsFvPnA3$CJ?In`Q0UcX^;@{u(i*|4cSfW71#)4T+FVb@s!BWj_~TI-1L=t3?Lwc{j(Zi+F}Zw- zWM-YKKvh&q#?kz}fH=p-yG)XyJxSx&+s^(R{QH6Mj0ts}R3C=~4~V(@a-W#jxu+e2 z%*7?5%Fu4KCZN}aR_1Tj&lk zY`<6F*&Uo#>9J?d+&|Vc+8n)!=&HE(g|sUg6*7+O`R0x~$lg)sS~LH0ERDU`+Jw@k z)%1-Tj{j8z(b6JAvt%CnA@>w@t!U9Hrws0Dt8I~CR}Ro7B#55r(~b8GAIDu5Av4LB zHxL8qAcRz43B=2fA3uKBM6TdBvS8RJl*1X>lzb>2ekIRj4eTQnv>BkcaTQhvQJyUu zs($+6n)XA@^jdAar(OET{GZ*3<}~n_DHN(AE>yaUTqV8F1i@3E=8k(=95;mDS9-_P zt8vtqJU;Wn*|8P_ilQ+>d_0l^7O$Ie^!C>eO+O#Hp?P-1H!9c7KOO9>BpBs%Nth<< z!mvxbw@&F5T36e>vCr9{?KR@p`ziP3k8f`~^q}>4N`weB*1#~zWyN^OStN9p+FE{j z78$q#;tt8AU7Bw1%3rxAu_}eR5>d3E3X^_e%npx28bb02oI9F&dLMQ=Ek}|h9fWFDoCkycm&iYXD+-C?Oi zc~9p|0?+$wVQaf=jFIbJia;%BEhK>Dev+C1JSf%6<0U!Yf+3q)L6%J6Gz_mCQX~zj zSgu)CO#zQoiR1Xj`~3XgI3Fgs@Ter4iTGCL7-e71%CZ`mSoSBiu5lH3zXNuc4#8tx z!eY0rbafr3ZvE`ImzRpsIl1}gp25}ilY_*97ggmYDL@@$g%ViTEP6BD8xbqhqFGHl z()43-qDeJK;U_@D>fq`zc@IKAu3KuvEtS~yP4~bsLRWK z>^)+M{MWEc`%d$+Htj$9{?*}bUUgTqdKoY76n;as>~~gn^hVA$K?G0-`^156E@l~v zgh4@vyN+7o>5)u4xoF~d@=z+L?&79uWY0z=AakVOX0%rL`NW}1%tX2S!uEe zdL(I<4yv~xjy(E88Ho0n`=TI@8B0X1%JzU8Wl1<8XdtN<)9|b)X|yFj&_VpC-NC_D zk4b4-Y~3Ev$13OdPM%Rz{(?<~%|xX~Za>!`4XLYg&+=f;}ab z1Um3APzhho*EgY8lmR=`3dSNlcE0BMQ^tVa=ZyY6JWjP@dCne>+b`4>%d2_rT>6H? zO%?S0j_)foAH|yHcK&8p>aOe8H*1bK^WOLv|CH#9Uj`|SKs*e{L&(O~iCX-;5=eaF z+4Aq-k{}j_6+~MJX&BG%7-qgf4e#*NiL|sC90X}s#;eK}i-KQr2c!uR%P#21TAKhW zC@FEt^}J7GiH&>ihB1wT{^ThumJ24QKE|%dG|}Q>UQZ`BF&F@_u24Jb*!I*9wCP9@By4b93>1!mHP?b{A(o1SoT)cQu8ao9eH#RZ*{ORMzPy$ih(B=w|#z?HS zgBBv4AW5qxW%Sph0`cx@t7`^DBhNGY;ms1gn4k~;mx@srJCQjxxh_UYP;mD~t1=Lqk8B zTfD7zVA-QqUts6NAzvRtn>-Tv5R7`ZMZ{Zw&pZ$DBV32MEf2q#%RVfk>#&Eh&h|3l zfO5g=+NwoLjN}-XVY_E9VfJ~MX79mMr8VKUZ?SLVCjTci|Har^onx~lv0~Pe@#*uf zZLX0FT2jK9lH zI^CdZpGXK0dLBz_J)kt?w{zlO+9`!@2O(VQPlZJK#;nTfOLcQmzFEwkeK2J`8u&@*?&%i=Lo# z2cWTZ&O6d_ju7`#g!on`6CPR@+anw<5B}Wt_YUKCdH?u#EtiN`o1xneiQDY-%A`=6 zPS;*3&NG$i#s@b!-sJJ$-AGMCU5MunopN<%b?M$%(W95$F!=OxuOgpV*b%5$M$=uM zNUfmiAOMu8OQR7U_x<(P1^_B^^5n^bya(D{yi*XaDr$9g5TTkU+4N*cn3)7ob+-AW zD+Y4vT_s9N=wa%gSF6-jKAUJ@sR5N+2ay#_5pPjALYl~uCsw5lXP5cMyLRpR*5fWu zJ`D!LKx&hH_y%5E^D)rp!}h2>%H7JQBXcWj=y646>hlS#f-}~Z<8y|qt83M_rsd9`VONB<=@7)2RO2k`ti%`tubsPgBF-ikL#hdQa>mU$t5SOqgj^a+WhnOwH3a zWhawkz;+s9tqdx8H7?T~G6rnai|NwC3}U!h*>Z5xI{IY{N6> zr8EPg#G)*;Lv8th5-4I|A1`YW2{z;$fekzvsWC?YUlO~LD(;RH-&2ZH+_yh=>Qq=h z45ee6+F0Jvbx?b|04}YRAN#H+pbmN?C^cPR|ClEw+Rtz%i=?sW{?U!CuAcd&TjCf9 zFv;&@d1^uh*yr-5lo}GB$ObaN*VlLUuC|$d<%?oihdI|T5Md4bJm2G)eTB&LgO0&P zC@OVt?<&8kw5h_JaTLl8R%K9^8{gBf_?v|TPY1>#QHu8GCaO4vv3IaNqIB+OZouG2 zkomnFs(m_zTVt$Dsp{Q+=#oIA*4!P z+Ptc>THBwL+yDFIX}j7M4Og}T(-M)c^LXBRKIl>h<|kxHY&`3_enySU0cyrcdqWw? zB3k)*Yw=NDap$f-EG3o=Wl`99U z&uARG92vIc)62wTn2x8&FVe(-;3sT@LPw`5sKq!pVz`sMe2ApLvcoP%k9>g&NdocvCpb`tNyrk5&;P?4t5-541? zs-&jQ^WkkM6RPeEx#xOEG?MI_uXN#Q+T2)pPV_Aju@~k6m&+{4z)YMtMQN$sPu9zfGfe-JmpSf_s_N3P z4tyT%^83Xlvh;B>kp@9gf3YS>Y13C5{<5F|FXa@9kF$an89vQKccBOjkxHIi*O&E_ zXic1u3_ia-C&gPOgK2JMWmWHf8TM$&KrOA3-TDH2yt_*p zhF(Bi>O^T19US%8mL*PGOiiFE1~R-*($PgD3U=C|=cX9RprLaVwHx7CQ_0S-HwLPb z;aEbDlXnvH%dZ-n(FZ67=9uNxk5)Y3UFcpi0TJ6VsN8HUZ|yn+4}AlifC9hTPqd!M zmF)6|PNGvfGgk7@B%9#g46){5Z3`947-fZPj2AzEuv8>gRrT5d% zj#+E>U2;Fph7+s}rGhwP1KQ#%y|1<#mzrAnPOr=SMkcO>`UzV&us{auHAni1tjBN~ z8$}s|<|wEchqyUgm6TU9MZm}4gI`KPuR4o*HK^iE6Y>$$Cr5(is^3()vpJR#2_F~y z2v=gP{kQ}cloQuP8oCU2?n$}h2S_;LMX(a1sSZszhvzlDI_TvJztWR^`u9KI$2e8w zI0Dy*XNlKKSx<1N&<0XyaO-Ijjln|b4(BC(DhNOwYaZ+*vO`fp*=?xuC0c5BYI8GR z5k`|_Vig~!942fD{Ed=8qlz|`!Un}_1{kVTxaErh|MlroczxrRe8 zdd2gx!q@x+b~~`6hDKCJTi46^Lq6~z6ahs58CZtJu3LG{W2WmM8L7u)v{qMRCE|{@ zvuW06xl73gA~6=YJ{oxxjo}4aoIt)t17wghn_ZR`E99Sl2AkKi`@PCBZf#cn?C6W( zdajp-3=gb6(Ma{~>Eq5d3K1P@^Qw!x6dWoS+pz{)x1=4>tfQT z&71pDLeY_GXWqVbOBo)G|3i@*hXw%aV#-;rq*v@zZ_a&CceL}dlmY08sX#HZAXGu~$z1_2NzaPlrkn$3X zIxYR--_KdzEAi#ylrR3*--)fVkUYjvW4>22v;1sQiP8Bupe66nAn zClv2TyA27afAkYeF>oAEWK^226ulhjk~Ahc39H8H|4oOFHo%CE!D7m`xSQvJKG^mp z(u|!+>HUNBXo<2KxCJ!G0R`j%NLGSy1;k?ed-17sWeAH7GF@d1q=2gEh4@y|8G5ai zgC7vYpa=x}K02F-(LaPwLm}zVSuvqy%a~tzTO~9bI^|}`nmIq!_Y0G+1$roOa&yO) z6yRtR!h=IXtoIe%mC2}2W1X*BOr1R0lyh0jXw!Dh+ylzE^tSpf;(|?{_E{+ya$f!#D+<7URerjZx z+5wBEgxAU-`<4R}^U6Ej?TXA{FvbF_Xts+9#!Ew{U}0_TIL0W)2VhS41rZi2 zi8h>(;%s@mt1Vh6 z!R~Xg-7fu6QKtk4%jgFQLQ*tovS*U3)Zy2$3p4}A-M)KQ5S>1)!?PcEepf}#^@;%p z^5y(iOIS4`;VmF-_+r#W7opv=#x}lY9#ATr#VRDQx|B* z;O+u5@lNf}1`Zq^aS0vFMV z^F}0=l5g;k#$8Zc5lwyHdDt(CV&O|CY(0R4f*`=EeyA@H?**2W2Z{;DwOxZCUl^SH zA;B&`zozm^_-p6RnIdYx{%Hg2Cd<=~2qSi}4#C=N)@h*o z({!ajr8Nd1DSLAHLuf0M4|09#3=zJLGW7SW1Ynh*Zz_*%r;#+>nx4 z$cC#?!L<~F2(XkMZ!=uF{ zlh_C#h7Loq?=2o6hPdFsMto2S;Jj7mO? zVn)bV58_QCOGq~HIM`0J1$hn;_<+)bVEuvgSs4Tcr z;`WUjj8zYx5tbe~_0s|hKQHU&Gi(nU9bUgi+zH8d8mMNkwSNI3?|b;XMMZvO&pfRw zUzWIqxx^l8Y!Fb&4j8nO+Lb*=>ECmp|D%7qwteKXm4e@dGkp3KExW4Xq$tMj1z<(Rk;piBNyjyrj=c$!1_EEx0l6O#b1EZlX1k-E6Ruh*hiC-=lp+a z9NI$G4*5_z$Q~rjR;T=cN8PCyHWas#=&VXWB6ppbbJo|4{SiFvWBi|)8XqWLOXk}} z?FeYc43fpTl9)QaltIT*)KTWKhu%s_N%?b=*FlMAsxCb(;W$JW#J0LxHfvS!LoLZn zf&UEqF*UXLAKuIX&IsDH%FikLe3(o{|DQ_n<6Mz7@}hhf_dAq$haVZ?VgwX;p-+2_@z;#^cKLp5N_5{) zH8GhfI2{aSJqsDIN)Gen!^rEd>zEiDTf$l~>Rg?9d=cRCA=V4Bc3Q=7O0CP8dLM#Z z5-9x*Vw*OM*R1+QEg|Gg@FQ+n-ZXJw6j{cG+kd7xF*{RbyU^(}wym>0T! zTt}7eq{bZBh+0HPc0&VZ4(Ol-J+c*1Ko^dzGOCM z_~Od%`Mc9tt5QNH^0Wv_z|oPMz^)7owHVOJ4Xd!u?>Bz%DR}EC3Ikz%L~$myR$^kJ zVoSSZD33d?y(+ch^4Lq(~23c8l}?iBx~1 z8MWYMB4v=t3ZidQq%ITD&w(OjJj!|)`UJdN2X_gF`xE;fIiB)E;-D4jJNrVW`k9>) zs4OwLz!*MGk*yUdK*{{ux^>P^0-Aqp49+QqDD}Wzq%;n9xz~JY@yY&|jBR@EV@*vF zAW3zshV>o*_fX13E4QVzys_uN+N60Ti!Y>x(~tnjZaZc?_pr>-iF*-pE7Y2u*NW|M zS~PXC13*MQLgVE>kil>)qnH#xS&H50K|eZf6bJh-Ev?jJJuk`7bx8v@*TUx_P}U?* z=Zwzn?*&`xXEiHbJt`)n=sT5Z#0nd!6is(GWE!bT+rl)zd^q1x?X439!rJ>fkF562 z+1mHf0rS!PV4LIML^9C}pJ$kd+&GAJq!RUwgSv!(%}Z99mjEZqAe@GRA)Vx#bi!?C zi97{GYesi^p`ypv|I!IsQ!1hB>K7@(iBU zeO-%uDn89c9-tT7dRG#OEJ0w~|Ni?_!HMdWhLk6nQXt1v=o8>aV}X}VHI}37BJ`C5{l5R|M?keh9Nl_CKF(#pi;q#x$d}Oh- z$bloNHb`_uiTwLH=SxD;vfnKdH>e;EMh@AfY4b!CS6^mTV+ zA|J{UUrE8D*2s-u6P09(l-G+T4;qXCX>*Rx_JCeiIToEPZ-X`AUKg^Y1}T_?J2i|@ zNi8TYj-WB}myw76N7?u=rW+~RkbZiZ+%5+94P>cK9sE4kLjS@D$on!dh*v=;%ishVoKf(v>g$b$Fj}4~;EMGr#_d^c86?~*8s$++7FGE0!i^E9 zMDnHzx4YJwb3yh21O6j9*X*d1!7SeUwLvIAPatp(QhL+|t}Dob5H&k3qoYUYj|o!c0YM`0$8GC8B*w5Krg1O=fIP^1fS}Ec{@YnwsT-vH{P!tQW z5N^=sU43YY-1X_p!pvv?IsnnCVdNgm7c*@8o_;DPz?YUIE#+lZ8h`nu-Y{-@XNU>giAc^pJ)VW4;B4imZXkYWUzFV+mNY$ljG+}PDE z#2PU<3uTd_K+1c*n!`wE)Y^B4sQym@DMZg|$Q_bQA8O!F>8*E(;DnaB*t4#*E)vWu z@Sca4V*ycS?_W-lX}C%^^LdM?;tAo@hRb&(T!V zw&g4#mG?}#&B_7XXh_L=slEK?ErD)DeI@6Fa`eJ4-Ar}Cc-fm|H*mTiG=|!*dKLiN z3Hv%54>IR@YEY^SV%KmO+!V02e}=Gr%(_8)C=MtyDFH!7d;q#ag7j?Ij` zc=h;vStsEW8Ka7>TFEaUqfN*&oemr)uA3wi8O6AtBhEvxvWxZSz_&=lM*4UULz?*; zwOVNH`t|EU=z1riCXP3W?f+808nQuYS80aW{L!NoGhg<3-P z0f{*4{QV@|-jO5FkV%kJHlPD>(428=E!P{*o-w1ljQjSV)Pt)Cv>Yv=iEuPNpV(E4 zr8tEI!3(u$%-EnJR4+2STLG*U!D7XdFv%~1T<%9ttIWbVWw2AjK+Xh6;GvIlWjB3xd#NN{Q(~ zV>T*{wsObOQc%xm8Eqa-^kxXT@U)O2w_I1tKt9fW^Ab$%l9r-eYgJGINV6Hc>$GSv zSYe-$t!b3rj`*xr3okEuJR#PENMp;o8K`C39cJ|S{@uIhk7;Y+dz)Ti61~E<{C<2! zuhH+{hDhP5O5Wh8QFPK_>6P~F=Uw{zo60iW)@yEh_NIt+-?cu<`q7AP=!I!RlwMfo zl`cEQzqpQvG~3l}9P@OcXr5hs>Tf8oH?FLzs_N0MA;X7@@9Nli?$LO15O5YL^&x=+ zqnCc!z@sAtndj#x&t$;8SljGp1g+^ea9|eIyyVb1P+QX!&kQc67L*iHWhqc!!16{6 zcgUvyWSb!{l}%db^u_6%Y1VNogT+wGLzOP}K3GnXx)W{U^o?9M%Vf5Rdj{2q zWOS~Vmn!`bbU7`9f*mhdQ14_eAc#Z(P<))+*-MY|7Ki{87kDX|CIW4;Km(LD!dH#0 z4s90xHrj-Tz3G%9fH;vItvht}$(i{Zm91ElH&%tlDOxl--Md5c7>&z>RM{6kvv2bhxdZBG94rvqO3HtY$X8a9O;|K(j~cf0Z|Dy)3IW2Yi&!=Ss1 z+ZUbeTRwk#S(%*~Ue6=uys#K{;SmFaDf5fM(Lm-#&!ir!n!PbwL2*q@XLG>EZRD><}SVRH^ zE3RqERZCmjciPX;Udge*YB;e(&)Pfa1D!+~50+3R<+g8+cziAyDrz(9OCB2{hkdW_ zD9j40X8!R+rrOBL zQey3vJRYo2=n9zZD0$H7!^rs&7ZYIO4>J86%}tVk>BE3*Jk> zy(-QqIXfgeT&B?4_Ti(ky}ysWFrW1=qE;;L2e$|OYLzpsQ|57sZ=prQ*wc^FSpQLc zp8KN8YlD&Djq~)&4R}%z_kM7z{0|tQWdOR*mvw7tz^Nn-I?6EHVWU;0rsapMp)V#( zDLXUZ^{Ua|tMrY+IxmtJ`_lF8JVkmCIzx{%|LbGiE-ncDX2ahJo{hXdw|!$T#Y54P zVK-J^mi(ab1E)`)mZMHk3HxPKRSbCSk;)BFh0j{NI5cqm`zhun)Ra2C*-N~#$l1V? z5t0qr94#cV)d9AhgvK{*k=+6ET-=5pR7(Alt^4_@;0`S z6N5j_jE@33Qxxai2ZIlf>^0O!J{0Fbh%+EYT6#KmP^l4w)49IZEy+5aoFyZv{irkh z9mY_VAmU_L3FkH~if~hkNJH%K;tTo^kXT9;5Ww7YlENj@tNbo0)R(twH|BzJsigH&vNdK;N-Fm}`mr zODD9oxSU6Ibjx@hCDb;1w7;a@aeGUmu4Fr^bhm$iizbW{R)Z|@l5}*% zFB$)+82{YGa%q1&7U1e^Um>s)O%z73ws_@Ngitb*?+TxL3e}uwfP}A;Y$;Ic14ry$ zWff5`VH=+<=E$#Kt1rp_U$8)DrHY(8;ONl`#YDmhUCNBYcy@XxLb}~>;F10mFe1H@ zT*UDrfwH!?_GjM#%o@P#dbN(iI=;M&Qldz*hhv_3L`dxPCqSn+>^%>M+|w5cI54v= z6Wpv2?g|zm-;mF#;e<6S8UvejEs|Eyw69Y$a17KmGWE;%;&L9ZFd?pGDJmX0)u{(3 zru-zKsFI=Tcbg$G)Pm=URCw2yrQ%{uijsVFzCb{c;J*8{(&;1RZ1w7hRa%lijCE;T zCD%jdYy@BFYMIkNPFV+_B?8*81ncu|lrur@oR@!ZWop^fspi-R1x!?`1fTbTg$UQ= zkgy<^g&wCOcKUa@$6pK=1Do3pw=k#8uap|mJHNlYjORa3H^`$5Le%R~R>ew1k>OUBDyyqS+E`hD{f zwjPK&W+pb-^mk(ll*Gtm8j7V!R6iiEo8yZyH8PcFc0WcI!;n~*@L%F`&wv>kM z1%j)@zfde^0z;ZKYN8L3)BdOoCetM!pitfW_Fnrpclts0U!ZE)5 zN(#T^)=KY4ILIsQ()|46l(!CS1{$YFHu^G^az$o8O3(z)`Ki^ZG_X~GBX7V*9|H45 z3_ltz2{QnQ5^%*nA2OU2dG>wz04%)vo!}JXYEHfmbT^Tmjn`CQL=Xmn5i#&;$$-6i z2H-bIA7AWX*45##cWV-=_-0Q0&=Sq>P!Y&}PieR?T(-Z$v|t5xX#Z8bPC8Ms2M32+ zQ7uyMJA|W<6J=GUN)B_cy5m?a6ql$ng>Yl`FG@0yvGInUM;gPt2xl&Cs2bW4{lUoz z7VHNzX1pjb{oDU^&Z~!!iUx^Y!Sba=+>YXU96l8t|yjkIU4Bwjc%BW-`Mr* z)LZ17{=qJwSIqb3_a^@FPv&LsS^RL~1ey^sh5Mvd@Chopc%kA6_?>@?zQ-B=&bh~P z04}k7)Po6^6)?esMT>zIIf`oSONblHmO}W6ykEopt@h(3zIH8{^Ks%oBnKV;zCC+r z|149t{ZZr_7#JmPM&W{aj$W;vnL8|;I1On;drc3td99vg`Q@uJZkPwfbgh_~9;ru_ zCm>FtP&+^WYBQ=Wrsy_gA#*~Yk<*&|o$C=Qf1%MNQ%4-sg|rTL<*}E*Gd890p+QXt z*j+pYGAOWw#wxaWWj_K97#Y}A?n%*hYUIUy8K$`$OtE2S>d=2y5W9!|p4JEClKnx! zF6{*Sn%f=DdSetddC|0#)yW8Wx{k7uXq(J*CIhYiv@*&v#uTa`rK(qZ_g{i)WIkEG z?qV(-4(sL06;*}KrWe=0I|T9NEPBN;D3GXpAysC-x_1tx_u{?gE8+jgOIX8};d8wYL$SmR`-&B=@e5SC(`Q z{IR{d0x*MnY@`?Z{~9whw;9WJ629^2vMiAwMAJGeiUbY91rFi-igLox-_tXE7@)g3dxl;ia)Y2%0kWcKBmbsZepG5{q&eFYNiW)<{cD z1<*rGD5=Uy)=A_#g@0tfShvKK_w|}|`}qpL>xyAHEQ-WqNWuWJayU(fF1n;GA_BXCqz4q9LCDQA*^ZNu3D6k+(Lns#I)^KN~WQ( zN>C-UB=S-fdQjS0e3sa=C!>ywT*NQ_r*LNOr6Os5_z%$uBwQ1yRIDB2S?3`=b3=53 zYe&iqf-@q7q^?07A@uu9x7GtSlD4ZaeX>Lk6(mCkdNCQ`JoU`t@Ghg)&w|4^r;NW= zW%LK14=LyOlt)m_HGx{#CwDmbhhjx=rw#>IETy?Ytt_L*0TuYpm zIDnkZi!B z#pm0oY>Bg)#nYWBSo7;6p0@1#vH981uu`)r&;lV8sBO}kXxM7$>RJjkH0tx)A@Gs^ zC>9nPi%c0EhK0p_$zP^f(#>+L{;#s3*Z*5K++))EOu)l1qD$r!y5Gc)rAU|`o$X=l z`^Q$8F&gu#Y^T$O$hlaQyHflScZpK9>xN3n&FUlagtTyRvl7TB+^R;3Kjnr{{?hPq zOe#l$D#HfaTasbx`vil6>4%0Lm7PU`;bc(r85~{$Nzez?4>1*flT%1MVaW+F#f#s+ zys2@-p1gjsb?%rDR&-(eh7$lnzn^_2WW43Daup*#_+T1Tk` zkspd4mHe2Xv}j5zgg~?$d*t?mRQ%;_nWu}^KuTjtC<6FHwHvnju=F!DEfTjWeo(kv zv(za`>{wRROUBXROD+gOofc$}7cH)$I^FbiAI1Vsurl&w!-1_&)>5~s@b zfbgtnU^TM7nR4lwRjh0IZ=#F_MODo387dh&g;0ONxr0xR9w9Lr#@kHjViE))c;EE{C@~Nfs-j~?iu37`PrQ_Ll3g-lL#22MGa8ZSc+&`B&tYt}8X;3oi zAvxCe?w6ofJ4PJw)ISJeLfl|=Bv@8+-S-)-6q{RpgBcXnapm3Cs_@1sbL7SL{GRYx zKvIqY+rN&?&7ZT$5r5OCC#@s$Rr@O%_J}!~H2=-=s1}imq4z^*Cjb~Xn;curcigOE zG}raiqyafa#{`$P_zbchcT}ib+hF!X4eyz#NXY+IC13puut%^znTp9_Ye9GQF7^Y8 zD>=DyJ?l0w&twb?Quy@~$w_E9N5HYleD62HP0HGs`A7j`qr z^Rp>sw=6n^qQQ|!pZ8T&x|yeK;*PQ9Xv#b!Dlk_cuw}OO?Bf>^1)}vys%e}#AJJoo zGXmrZQ&#k*PnP)cY0)4jPOSWgO_at$jV(J@2RHO?!qNM*PMk|gv7EuKXiK+xgMop8+_SPK|C=mD#W4N~X4EP};(1e!R@2fuEHI*cO6>ddvuJQORI2EYTL^ zrR2jCdrn`9rCEMLQ^tA<=HP5(w4Hib5CPX&2&~-TfOS_DOcS0$h6NRVxz-h&lZdRR`zAx8N!`r6fBK01plS=0;E2DF`GTO+_nJT!vW2L%N=niM4|Qq#|+ioPte*_D@D^Bvsj)#g;eLl{nxbAp9kn}k;+*b$(Y!I(ENx}aVEq{s@F9;$SjO<-2fdSc zk`6230IVVwtHmXk;{8;-`vL=(DWF5j|5e}^9)I#hubY_UC#DVpfmfhBdDIGPfpfry z8P)0SaEM$KmI-u3F2k&|^Rp{PJ`M0V4Kk|lX4H&X)ll3vuWege>c{Lqc<|{f4JV7} zX~7eZD<&8*2a6f9sxW;Lk7MScjL(V_6t$=f@v4U)0=Ea)b^w96(AO353d0E6}k#XxT2>|H;2^^|rPh5IoHU}aHQ#uvay*1S3HBjZajA6l)tlE7FW8Tlfv zSZZmXPk8j?fZ(~`rWhc-vVUcmhp1(x&C{EJQ#2ElMn1X{cFBgBJc9gC0izUGJPed* z*@{J8 z!K+V!Ul~joKi;oi#qdXra5l<0y0x(BY`Qo&H<>e6gd)pw#fq}VqX#%fk(zMGWKnRn ze$YarforUxGK<>nOyG{ImaMf?7D)1O+ z=uR`ZdXIdbKF1;S)}sDEfl^)X|3o=-TThRF3vI3WM70j-W8!sQ)GMnDdU%+p-=RZy zoQPPQAc2S?Ltv+tQE8kN=pH%eL}xPUV_su%`vcp)tZVN25`0*a_$53MU0Nh*Nt$!T zHXw;Vj2lWbpKkCgQ&=5MtlgV5ZMsfp`Fg+lV`H?0C!9?$#1=a{`gQDz=YPu64OHi* zW3vAQ8+eiT&|K58W51~mROyZvfjS$O?A!JgHigysH}P_a6GyO&l_Q)cUcH9~Rc;^j z%KKqixL|@fgo@){2j(Zio&&_-fRj`Z8StSXlOjMCWD+~8LS)a9WnE`=tvpo5e={e!^Dt}5cztlKG zfA+OQ_Km%Y_RYTHjOa!cTKr!uXP!^zQF^VtTR%Ln_uR4VOM92jLAP|9zkk3V)zs8a zsk)k^^79$(1uMg*0g{3q8V|6zx~_|&q|FV?ny?UxSrag*K3gU4s{qtb1q33lFKHNDd`P7YOAa`Ml4qSmazr}P4>A(J`&x``{f>= z{bV$2YP#$*(sVE!=4Q&);*k+!kuf@a2~udx6?K~K14$W}we+4HRLxlJqI&?}Qv*X5b zGKjuK-~uI*)9~qi`dL9JIV3e!35O`c*(Hy!EW9B*JY$K|h5ZA@G*2Qb0vZ8ZS&EXv zzP7Bq{DH72a3+=NdEuf*XOD-iLEiM0kZSIS15LKJ zx2z%HS7|gX*W9bzXfy5-kCBhY9#)to=mCvC`Nu2^8g?8@U(5fmWfj_GBNq!7xQ=t_~`)wn8{Clzx#Eg zvK0`Dk_4gnqR<%zmj(;%s3uA4r6%#P^KoXiuu5)PK<)DAcCL=iY9zg;^R-ngLgNNceeDsx-@0C}am z`;~LSu|E}LUroayPbP#q)vBBm+9&Sr7Uop@%+qJ&gr+OeL;C4qlG=fW`uH32#8vD?DV^ zu1Acr_7g6GNmz1N6`c#&1tz_Xd#os>`i$o2R#pV32>5ZoxgFKjmF!KBXQG}BvXTtC zEA$m46Dy`(;u7=omTsUBf&h<7dyONE=Cf8FjhaXUxpfu5~QD`Xny=jdPt~UgHfM9Gm}6; z5oTmAYd>^SRP4IJj^vYi%JPRJwy=X2MKspmND4{SL&`Bn;J+|xQdC5)3E0z>`L3W* z)nBS4(1mFtlS(&{0OhJb!0{y6nXqBQhhz0gskEp+5{0%DAiQh1xl2W%l^R;sLIe0!Gkv|wG!Y-T(t8xzj-6+r$#a> zDbSQHa^aFd#?LmK2 zD8$G?>fupP{9EZsMfs?)vq1u_akF9=p-I`%Q{}h;zbMr%r9WL1xNf2tZMYp>r!-S# z_CO#kB)3?Dqvn4)*^tYG#36}PiwOOV>4h1*TQnGgG7+?Q&z@2_uDt`ip~`#EvUO{j z)h3=xpU*9cyNX_zsz%N8W%pq*N`-C{?J4pVVnkf;$cI6@mL#zo0yTV(?y`O^-^w^8 zL7F!r7b{5;yz{{ickIdY5Ts3z$^Y{AT$gxBJE(FQqvUH~ycWm5v=j<&F?DLf$UUA~Py9&-7jzgpLjvsz;<2YNlTJ_rGK6*TMG>y69vs=?}R-AgdVwk{BQo!BkLK2`mr_&a*!fY9x z#N`rMyyPshl1!Ej?XtcsfznM30g{9TIgN0|wlJPBhBv-ewlW?igPwHc*NaCTiJS3h z{oLH#yq?>xM#kg?ft^kl2@W3sK-B2CAJL zwJr3v=CYkOk3V(Eupz&|93t6RQnmg^P)1k-F;`jI8XRm#+1&Uobg`q-rrB)D_~57F4nY+>ku{Y>)DDg4|bzdsnphWSB+d;fp#y>=!LCsV{ z@%~>1C?=usf`3E%VrUMd?o^~O98|AX#_2!gs)`Jq?p|D&K?;x1u1&Tge`<42C=AO_ z8sr+y;z_LSa=LLVo?!Vw$N2W*8ZYPkbW(dpSszthgl~BqjM<<}F^i+0cwF*! zg2e%gH9Nfnr|Q}*)Xs`-Iq}tvqMs&}LL_uad zL)=8HwqFInO^^RZbi}dF`59;Ck88Q-#Su-tCkbvDb3#ZOcapUO`;iKer5(FN5I%Oa& z1t5i-032BU1vEnFB~J7;4{^0QDFelV^Bia`23RNXQ zXg!Q5>5(z&iXGFWy9l3Y8;;LGnr2RR<0^z+O4EGzkdngTRV^Xlfd9tPv<(0JD8yId zpFkxfgDnonE&%>9oB0){>BzN2HzG{8lwUHl9o@fg6tfSr*aWH|l?uDoecUSP{SSg# zMqdRrt`xzXOo)LO)0~S}G&9M4tuzB7^I5#fZk!_gz+@^(==C7feo#w^=X-NhV<198 z>F2n){hqcv$2RXK!%LYAK`S{rV^yQFQJCGvZE5vDeR<;Z^5My&S((NXBg)ntSy@?` zFscB6To=YdV&&`yP;-!1`wtL9$=Rd$_E4gP@hgIIf)MK$$V_?q&^I7xOTOWeD*7Ag z6^cUJ_pzhifs?aC%%Jj>=PC4YLzO9IU=hofLG=lY>UFosX*Zmq7XW4*f-hfP%o9mV z_M0M)z?G46Tk4k}lWjgeL%mU#t`FY!W#hbMmfBmV^-V&Uw~84x#Pdc=|^GkiIU=<|P8<>~e^Dp6DgNhnb8sGe( zk%B{&&_lsAR~Ovr3sCQ3>JV$5%nzm|j#!PFc2OlK2qn75dKfh~0`PBi**e)k zmZvAKTP0R;nM~@&3J}q(j;!9Ss>)>Fx|KgFt6nLe8}*xA$@%l=jk3gbTA96b=gx;{*EMGAWvp+1FWOSS z<5vIPm+mYekuP*_!x0fs@@}f!d={gm%GzgDwbXnPZ4>OCb;I{=uhyX>Qpc^Y8XIA1 z?A^vZV1;L=E%RM^^3xfE#UhH|47H-M3r#$&8co%P`nq8N-?8XvLQL2iraKr7-6(a2 zqd)xv74Z#=sU{{D@X{ss6Z4-H=>wzj9`8`Qkw!#J#@O|f---qdHy*R;~ox;CgDqwznf3h}H*oq%mBeKpP?-b?lw&8K-aAXL(cGMsN#?Q{OUza)szjtXX6 zL|~|3QJie5t(68bAD9l_JHn_CQh{8*TfQZQpxOV-C8$O1h16s z1j@_1hTBO`aY$Orwyp^hKp=L?;|Io|mhm6~&OG&h5bS$B?)~K-@gjOg`+L`p7WD0N z+^V^1R<|ShLPeUMbnc+IYxAmD4w{h(%Y(z-&h-Dgtl2*?ZZ;R*4C~Y^b-lA^t0|FJ zzf@X`+j^{_jPx3jOk5)8X`-xpP67V{af{Lk@SNW`{z-PMJ3=~lVRQfhT#OUO;hAJ1 zJo+elD_vWl2b4)$p5)6NCgB&qU0$UJ|Dcs}z)gCW>NOnAqKB*5% za;d2)Nx`!thU>n=nV3eO8x6t}eDdTo=ebb3Vw;lF1_5iP-w$nw37(0cFD8X^^x# z#JD(U!j3JY%j~&?uMQ-HYpQ)kB-h62_M%BWr#yc!rp9OVv@^qYuT980vgl!^eyz_U zs~n9>HILIBA85s=Mb~Zkcklwc&$VAqj~O!3sqF10O%reb@LshW3JxUTXDcoI`s&h` z1wLPEvNr5Fu`qQk^P=xe>N0%~+lZT%?fS8Mh(Y=L!g9}})`1;$=TsCN0S|iSezf-U zrP0Bcz62k*woPuvUY|^EJ^yo^#kS!&qV(>?u=kDCPk7CH91wjVY4*3B8Z9~Y=je_v z?L${tc*SS5G!5;V>c`KUCIJn|BrEC+nQIUdM_mawY6$nfFLmL$de7u*?5+5G9081C zoDi;#z#jnp?KpQ3FOjE#^Fk)x?ny7)SLt5yj2&M#!w$W{;dij+Bp3>Lj93O_R02A- zi)qn#y9HuCv+) zlmN~!5%PesIOjFaCL2~HO_0s8(gh1{4cVc-d-s|buBVif5o;uV>#v@ua@mNqL(=_7 zlaZ0gyemUD;v9_Y=XT5}%p#V=lq>MkM$SUhmTYPcSfON=43Ei4D3^mijCT+3Z9bB*rjFP$IGAsjNQ$9U^`xupz9-a}D#Hz+QJlxcZAG7n6KW zCZk*oaeo2AkV!>7Mc9PZ1azfmVPmnR+YWtrDEje-k^|8j+{1g~RWU40%s1FDv>$s! z|Nf+3>_#|0VO!av?YEw;jEgieTeURM^Wz84n6CFNyq~NX&`I}6W_qU)aV76d%JUxE zc$)2v%bBq|J<_o2D5d6!oF=C`R` z_eR?8Eo$EU^pEOqkEaw5Z`ZY;cY=2tGrrlpF7;}mrr&<+1lti3hn!GgaN5Hq10aFR zdXI1^Iw4Ja7Wp9(P!d2b-iNs?E%(ZW@f^Uwa+{H9LB#Z=&;T*pfsyuHxA2fz1 zrD)~mNya(2DA?+YvoetrdUgb!9){X1U;cOMYqN)a2)BYvj%HRVCHPb4xkz``e zS3PBqzi!LSk)^AnhdoGYNy6VFcd`o2js4G@wlLmj_wJt&W4$lDI@cqnZsalNGiou< zZ3a7Da5_1}_)^TqMZIVFHLLD^=}s_fFJC@WQTN>Da;;Q}fjrixruUOP|9J9iR^~a) zCkfWUPxjwh_uMb%Rj|*1UmW#E2OLc)Zu;ich39T(U1GB*e7wEv_+|OsQ!otjy=kY` z>0A~ykrZUoQ=|4Yz9kJFn5Abji+Q=FLiWlb%;pA@mS>&qudDuEx0i=YAt3IoXq z2F%Nil4FA=SxFyNIiAO1EiN#m1~E>QVd6xR9U=M;1YdF|C2j{&MpDaN;GLQK;}gyH zA*q~|uyUcW2}SstsD2Fe60crmPfn~o_x0SC4OB`qiJp-pdXQ-E=H-2O&`>_DQYyU; zA?$sj>g?TnE1D;g+)e>+s%XC-Mn$*J|BX>ip+#4 z12RDbXMzg|#aJyFuv<3CnmEOBLs}k2Y7j&S5i@~Oz_QKB%>)qWZ^YA9_69^-xD(2Z zY|M5J?@;p2M72n@|ErXIV|g3`rJF#<;aT_j8Oquf2_78#RKkVO+~)@!UK{EuupWnBk@a#zzpA!_Q3_YL-)-qwmsZ>5Dl1O&$G5c%JDHMmWv4T8n znKWDelR%1#*xW$q%+X|rjr#51?`Yvpk2rq0<@gB``VJXV@d55>xc;G%Cw8I#q}&>~ z%JQ*&qnSGvcRp6PW^sI`w%6yCr&FFwG4=KR0b8G8-X|&eaMC}|{F=y{8_qTCEglI@ z`m*~0H%8okX*AXB@x5Y;c;~aXd`kE9`@Ar7(>U+6KKCbW^lwsFmD0UxeZ(Mx4xN7S z9C>mwzh_#(_>-9flPdQp<{zFU$vO*Z`C5)YJzVqY3sc#iDchz2-iJ?Dsch0F#5g;6Xi?r`)0>vzgU%xHwJ zU+ymTBk3$0l*G6$=HyQ#gF_YV&)&goRuVdj>I3t^;5lB-Nj{6N708Cz7ugleYauOW za12cdOPq8qbywWN6Kj}c5(g}`C|o)bkPn%<5iIC2hZpv5j6g#(huL|3i@K^Yu>$^% zLznwkf2E$@!JrJA!)Nn;)eKw};cD`|?xgu~wJ0pintk zw)h|);&!AVSPcb8X!#Q#F=HCdrI6OEvAI7HONC^|W2yRM^G*2zqq7Jv;M6Go02VkI zE$~cP{?nR5(ol6}pec}eexyH{lUqcix_;U5v4OS6&g{5>;1>^ruMS<^PuHsH!D}tPD>vU5J4m-U&2-I?Z|l0;iZctE zI%%2az~hq_Tzl1`MH|!K_x_+eb@4ApG>2WBGGo5UwgH_Jqu#1F)g0)oIc%_DYP^m` zQpi!u_aAj9+)e#v?kNALoPf@^h_(4 zcQzN6<@Voq_4)BFtv5m>KVT~=`mihKQk}oYXRUErQSJK-dt?=9Jt5SyZdTKbUA3)m z-=0ei;55QrZ)(i4_F?BUg3k;ZO)%lnsJZHz;aj@QL7C>iQzI$3vU2Q%3B%!&Lu;yb z_#Sz7wKjF+;Q@z-?RNhBEIcS^+U3g^!>#VguZ1t26SbGB#$|e{>oo;QOCHB$H@-{p z-Fy2hDm{izFFo+1_4Er9Qx`b3>pf?f`oo8-y*rO`Y+rEl;d~Fryff+31Tc%92t2|} zrcgSIE9V;-(Hjd=Oseu2w$PXXzi>vh)}5EE!WC@8xwXgRQgaEV6c(NnnODe}P3y!S z9|B$RGztVa0KJwsk?q`>MJ@bu>?bW#yG*M6AutQh;L$8@_NNE zhT?2+Pei?eOT#i=k9mDWlAofS3Gds*EB*x_egjFRUqJYR5Rs5uQ}>8c>*VyVLYm4B zDru!LPjaT|(5*b%P<@zfQP3(06S!S1ZD_I@m9Ch~WMpP~ulY)f#Q*BoHoznqM#{)M zHYW)#;kdmtcRDi<_ykqV4eP6Oe$CP)&2T`tzCD{luW%HY?pc)BL^+}4#BSHN?Qe5q z)lR)wc%%K$sS``)V>Ez^lQ*H?izyZulG~7p_OE5S@|lkFM|J!X&y`JE8dhvOo5S+F zX)Fc8@QH`nLNf4+bkj2oVh3K*53BofWNlyF%nJXynyN0?a+=g0sC$3n?V14pp>@+p z-8b6vwf5POG`p{9#nZcv9C+HIuJ%&t|HIXL$5Z+L|Kr-jC=s$#$fn39C1hsr5!o|) zj}Q@(kge=Z_6|u1na7^VIN5uh^L<=i@6YXbyZz1|uVI{XT<3Z|)_pv{t-)J`*M?ZF z1{$8rCp$&LJmyNZf+FEBjyWPmGi0ixhv<0=1KLKPv(&$sVg6N=k#m?$b1a`|>Yol=xGAP?29byoQ~*a^d{(!ET?+;XzkQ zNAvcgH8{!=VwY8-?4-;8joe{X#U)wtwyQl#(nN9vFYb>YoX zo)2M-3!ZYLojA9D!EFF5L{O=}BMrF#SP!B~%-HPr$x^Q@JCN75j zSnKucJ7-r9UH@-j>A9R5m&D`h{QBzGkew6kWYJs#M{eCfX`}QOJ3jX^b}RkM+?_QR z57pZKwbA;Hx(`|9*b~yE5tpZhSKIC-Ogyh`C?P+fale=U;(4@jA+yi1%TXG3d;g0x zDoFCuk|>6ACOkr&s!?52eQT%X2(Esi3V-LXM)eJ0iq4*y1AbKNABxe#5yB)ingnWzmx?>ca@^#+8wkidvDjG`kJMaLg3e zB_)PKp?a>T$>af8?>~`CnG|#Qd)M{XR8Qj(jqp#)z;er;g~Pj;f+i&ZB4pZ zRN>!%tY|omfBV~APjA|BCVE9D^Fvsuiu23bL|*px6-SZHg&z|m_&LVeMakj#eTf8w z@skgA>eJ2*a{dltcm}_o?F2j|Br!m*7q<-b8iWk9b4scMSnxl$E122Jl|!TK&iw)z z$6Lh10O8XQC`?shAp{~5canzOfY5Rg&pAL>F#QapNaqTamTbhJ7%^T%HUxw{t8W1r zKSGX$tfK(S1tUA>M~c6LI1$zf!v2$9U@k%in3JZyQm%?nvJ+$ej?^Ler0S_eIJ*8K z(nbMA9l__J=T=qc#&PlmA|2$3av>ZFZM6H}I8$Vs33O}~ zeQij5MmDt|)~IoD(Y#-g?DkwRARqv7NbUnf@?~k9_PZDe(g55bYUZoBEK}d1XH9)* z1_ViDcbPBF)HZ7cYeFtJ2^fN*4w!v|kNMC;EgLVaWs zWk%w-&ogNSj=o0hksdyEo$Q4RdoFq_3n@iF$7@74Vo)hgb3c-b(EJVzGEt0J;2)uK z^YcwsB4C}olhb0c)RdPCnD-~25RKcIk5#BZTcWPjaJ;QcQ#8w8IXin_E(PO%xO1d7 zR+$A`4b=~wT3hF}zWzxE)?8nAQqJCqm*>I!GvS>9)M!cC=FyQZdnE~*04xn{fRkCT z>+dut;Rlcw3q1oST{&yjQ$bSV$6DhP-CFO@Q01<%xVTlz82!i}vMjP5g5bNiQ@%Gl zTJDLwkNErZl2rU3FjOLy6C8I4MEm=i?N<(G1OCdL z=pATAr?jDkSmiNd4V7(|h7zlb@EQv-s6&$^itHDm>qm!&M>DJrOPxMrxlY(Q()sZj zKE?N{`Knl_6z}7)qP^`;>{&7_m>S8@$?<#^#s{lag=NJ$jjW;tSYh`JcFLMpQzJzg zFD+Kn%cU_JbsaLo5!XIAavfxn)Ya+5M!4HX#N%EnLeG(+mbO>9tC{&FsuOLJhMhHO zWHnX>rxqquP-}xD`h^y&xR#EFXnOumW^Dao+UET%wPU9UlDmbdxRjy}&30^_O%dCx zWq1y21p#*j&D`0w{#8rmtk2jQ+k^cf8D1#BLPQ3}XHsFsk#X?V?*LQGyd><-!WfNoN^FYySiKQ%BDO#sG*XvZoGUw$?Expnd`C z2-%~v3~>Mec!*f3FtS(%0#-*478OPSuvQ;*md0d>`q0Wh>Ow4sKplc{cF>2j3Mdt_ zM+rf?25y2%5$1EH$kI(1KOv-YWSw9~r!a*UG-Rj$4-f=W76U?&2k`|=YDn%D-2&wu zxT+P3T7NSIMi2<-k+t7|>Od>bgDkU#D%sT|F+LuGLsvC09DF4gZPrwg3KS;Ih^cz~ zjWgLSAY(#wVlbNQGU|ryXJE(+?f;4{O%~9d5T9#AH3T>bvT$fa!vSP=P+v>Kz+N4= z4&b(;K|ggm=Z|mY#6!Au#P$n}za7DiyZ;zk`ApDAhTVD~0VuGGE5hzuBLOlzg6qN< zQ$=sG@!{KF;Fkl{ERWmfOIRZOy&0y81C*dA000e){fB@+VvG8p5XpS@`&*rs!%KkJ z0SAE#lxZ>VMzi3vK)4ppYm;8JF-b{?&d8!U;L33r^2ujM!;lnq&vcE=0n}l+KHWsZ z@R}E{cA{hf{2~Z+i)p+3k1Bn}3R^=J#;M0v-n_vNh3%3hA;Fh_59S>l&ID~CJ=bW1 zuxe3b!+bK0$5=%t$i}0YS&!9w-n3EYWtk?w1sm8;u$&`H-gvooO&;A5J>_A_t{p?y z!U^=pJHpgO#+`Q0luS%YzJ3j%eJ!RIHjQ{E!aEJnG- zi5SWhJv{@YWs(H!@9K8OX5AC474Oe1K%tAU2WyXj*#>NL66H!k1d3O$ZVt|OR%@01P5cmLJL&xoLUeDFzcMlTvX8F3*Iwa3#or9M;Ak;eWU+j`-I2Z zKUmDGn`T|E;qr5&hX%`535cyI7(=t{dPz5-Lj5jLr0tZ80TfMs=QNALD<=buZ|O8R;K7`#(1{JL zocA{GD$&0w!rC?>_bi*a$gf#Gy!*-7>%#?9|0tvHapZqr>5s#)=ov#hUCck2zTC$? zv^y0g90?lsrC)bk{db0xls-3i z`)6nW^p7ylM7~JBM9MCQqP)U9bq7xSKO`SxQ8w7$&((i@A`B0w(pA(eY=x=VaP<~J zZr%=#uUvvnTCQ{uY^BnwPj1BQ2sL01nQ>^DJ8+2@oyCI)y0Z(1=8g}ntR_7vczB>w zk6~*kd&qiSCPsYZpWDN~*L?W=v4=$tuj_&@yS$^Jc?YkNs>bqMLnH9}cOiH?xA((I zTChhxr0YIdm-de_Qoh5(wD9LpudeR?>|kyQr*eS>ouM!uOj*)6FkfADCgc7|BkbYc z)G34S;lU4!oUYSgFW0(cef|pWjoTxo*ESB(qH{kZQMA#U5~s0Z*L@K6*^ zOw>F68@-TOSm@W<9w%}IsWM{ z;tY?ByZ{FIaD$g1Q2S&LB+>}MMA%OvtqD*EzV~O~r+P)1qNVXD9^%0_xaNV~a2c2u zPoT69jy!#iL4_QZFOb!$01;r~xP9Ai0TUc7MGWA?Cn2}{kh~(_zxM_IJJ7XhL21_+iF93-Ss+r)6Tund-K})jHuYGRGt1oX!||sL}|e z6|tJSI%Ipios-i|Mp|f4zC4u8>Yss}UFEP4-fAB$9SF>y?d8GHh3Vj*neQa-0If9y z)DEecKRmzn0X?N|vxB8)T>mQwNi?eLZbS3&7W$Gez|DVd!%1cbbCSDv?;1e9Ssbh6 z_+w8XD2_Y^NPxXO4SM0`&rgD`)6j4LeSoC}x|m=9_YI(;c;C)vatW?QxiE-BkS7K@ zStzi`lpEkW0$AeH(bILVdoX)SdRuFJ&yUp~oIXUIla`D8kZ?Z_Vw6%#9 z|0?pK@_%6Zb-jZ{FfvyCZHF|~q^pr=p-!A7Nhm#bM}NV9lWc={H7_w z?p@E;Re8NR?BZeA?{oaRb_29|*KA2=_j4`DPHZ?zD-LBtzqML3;rNDpVZNIrD_I#S z7WTu=+%#mVq(jkfwWKL?`&$(|85!yV&N@x~(5KBdZ?dq(_7`fRJ&e-!+*MV=7k0a) zzn@r>H6I>a>A2Lr{QA#>IE|xru0>1go-pT^ZfEH(UK|+v8!c=uHSB}>xNEldj^Brd zAmur)SiPc*JMvcezW&?zWXhxJZgp_NbZoKp^4P6rezX6)<+dk7Bid8%Ldc(Kl*UC&s&2Hm%B>90clZ1D^e}eQ_LC#};j*O)rjsFU z;q^iFJD>ASZ65hI1uJLWFLB$l4xR7$Qlk~b@30GdQg#LBT^i9rqQ@_A{@BlKReROq z6xDsjt+W-TMuTURphknxnYB71i67d=OWU$ECv+r zF}r(uZY_wWv}oJzxUQ}ovhG6Nk%IN?{^;ziW_Ylv@x9flLSqWz;r<5R@lMe~g3v=5 zn~9D#AIyTL@@Q8V0T|HzSf5$Bu!GqWy=1V|m$$#(Vx&gHBFg1xhJ)GBH{x`6yiRkQ~MBffnf3EnY?d#4#n ze)!fOutp+b^g`YeodpwoI!sdF2MIpBPtwicd2l>DJhs=SXkbNw2;iOm@Fn+4qD&kd zZ9oLu0G!bXT7@bA4HD|Tjk(DqKYVx%tbjStet!opBI}THjtUvK*>#b_#XIk)1WCHP zySpfv%Eeb#SD)tvo@3eg6amlpa~FCqU+8{FK(ki7Kb8&|4I;uGAr;kx;YGK*Un*d~ z%ni|#Z6d(@OiPbc(v_GXw{ZcS%*W3!tbTOk?RgjswSZ4a5TNEQ@GAN8PEI@^3V99u z9de+w*_JLKe-2z2=c|Edi5O(RLF#&8Yiq0K4O)SGt@5kDL8KFxL{iQ+Z0IY7-HZDN zR~UVBb8?!1Rb&TbKj;(S8o3Qv@PKEXv9s64zmd`^`}q(l#d0Bl240szC{0%iXFt0C zUOM9w0MR3E6_D+2gHd;K#_5X(Yax3{`1trPZvwqgJmig50??HHfv(VS1ff9l$CGZ2 zO(p*T=jWSOnRZta!bQA1*@lvM@*XrCm=NQ9GcY@*KAAfHOKfis4SR;to0FrgxgC_C zW`3R;%ld?i5vP+yHq+W(qo~oKdpwYum-syQ{g_Uc^JX+^9Enw!t2K2wI4=Dg<++(8 zP_2ElGAQWMRb2Z0dD1{8TPW}Tjq`&m1OZ9xExzA0N4s)CI+*>$i305l8kOwrxeJC6 z?=eTh8+(n{O3507f`oyGgOgKMkuvfIzY=W?OD(kd+_@({!qk;2L=@|G{olV|V;6m0 zCf_!TVz_>N#9)pnAjbd99X>OiS0rJuF-!e$K^mVU6fC_t4f`0g84N=ihCy2-f z>xqS#j(@v6|768#<>#Sp*{$bwW|k#I-N0>p8>o<-psAaq@;Wl~4%eSBo6NfAU-uKo zbg&Kjyv~jC)Svd|@A0|ic6AurGi4r2idG#AcJs!%P8Vv{$8?)>UH%N zlxJ(p^IUEPCQ;TiyqS%j!Z+h4&~6m<2*1W9EH9iAxY^txVZ>Lz#-5w&n7KV(txJ(T zQ4CAn8Y@T3OWRQT5dmNQs?TI7U3o)*|DAX0yU3nEibnae0Qh><6YS;b?=+`ig??@) zVanu_gzOf7yw#Yk0JZ_#b$GeqTp9hz*w{VNC$6n&m zOimh3>`kKI0vG!_4mv#gjKSYu@_erh85m?Q`#n~~HVhXgks~4Yn;#~#uBf5LVx=Fc zHZ96sQRJr;FJbu=s9(d>E$HBtK)($?d5$ecki~d$*(T`KN>XuZ6ots!?QeBF$S&QH*lOuMWLStzk?&3Js&BZK?a^gv_)lQ#6XoGWjc+VUw#1f`CCiNllLw= z%z}?@5Jv9by>SHu5u6|iJziFCY=%)+*Tp1{W=JitDn_H$nJtE0H)Ly}CoQ+blQw1O zrvc|QOw9LTXjTG*qL;7{A>Y*2HUh@Yb71M&3_?Qmgxa?K2dTT4FJHd?$M|Jgl3ySa zRlcifYQBXD6XI90Bj2zxKOYG5qTo@5x51;CK)rqe{JN5!LK_=A6}i z+ANuUdDfF;2fA1vm5`>&&U@Mtjr#tTv%uTC{$o6&cXzwS#6{E=rt;U)%p%dQR} z`@yo**7H8(+aM9c)a=1WW)yez1unA4SJ zej)_)R!9uuA-Tp(_wHR!NAGTZspuLRxdqaon}3#e+>zfM92}fI4}?S{7>mPo1W(ZO zZooIEd-|l{ZIi(_V6#8h)HE-t*ZSOKxIJ4OAQndf+8bGJO?wYEph)>Vy9*Cs`fa+ImwO7pcU+QFf;WeQ^= zQh3xa;I#Se{q6EO9G$B7y0#NO$9E>TtMoIJLWqNB&3KcKV=?VlSQGnS9(=o@_- z6n0@9T6w_Myv?_B{DfFE*UWB$H(dCU@w1hKZ=Z5r;15)UO-8l+ZjfwA5%tQHyxez- zcawFf)`^ZP!;M_o{m-ma)d4@x@NTxtm+4t$LH3S7>vm4Ee2*1ItcU*3n>#%`4C)=f zEtRsFlA={`>bnh8$JpB6yG_iDOjb7u5Mk&qN*|4$?Dp+eI9Lsi9}QON@4S&NFe{sc zSg{ulA77?Tam7*8)%0t=Ztu2Nr{Rl5!5T> zx|t#o$%n_d1kQe6{Pqe9ytd?CFO?J(odb%X82->TJjWj?818Q$*;(_OGvz7^v`~w^ zVxL5W>zY91<>;rSd*p4sC^)6|(55adKi^*Y5NVX$cQ$P{cSMT*{j-M&s|Txqg9#&q zo`xo4?-MjxKw!0IJ%M2DVIvx?bnoT^E#B%8QMZpg&|8C#8t!7=$hWa~)Zew@G zpI%WAy!DbpB-t#KE(#~v`F=p{z22TN>bWsNL%k|oBtF4!k)bB92n(;wpi_P8A`?7L zD+DPix=5t0B?E(P%@IK6O3eEi%*wS0k=g@_4>$_fDJfZh!p=KxW@gtdKIHV3%iehV z{oiB&P!l#2Y2g3wv9q(2aT%V6ouEdtDI!$Bd=j;LBqnwShTOp*oF)b}+7G1Q;8Ggv zMeo3@{~KtX{Q)3{ou#4ca)f;gF{1s_M_X?0hSFBAc$!(BQ~8CLZVJ4Sd)C>eDRvL$AA+WW;U?% zA!W*qNdwKz``z{R>r6Lp$hf-lAwGjYfe!o~>TXxrXQv;!wLDlF(VoDxL`7BgEiryM z2~cK$bS+Uo5A2qEY;4v)f|IBG&h1@T3dF6|E>Gmysj!iIqn39GM=Iln=kcpDRqIRV z`CbfUk8V&9(0CgR+RM#<^Yh4TGitjCzHlYI#y`T)W~##1u3a0iy3LzmWSG9bt)hJs zm0&xxn^a{_4+Hz?F!rrcOF}!l=MS|2d>HkndG_oA33|LLG6=@?M<$?*Tay^OKpnxx zW2?8qBWI~U<+fGxn66}HTwWat47P&dkKTI|_xJ?!el45hclpTwVi~^BR9{C?K+5%- zhFzrG;S>Fd;Xx^ry%i_4%Q(Kh2H)o=a_}oC8q+Gqi=qUNYhAU7c5&Y(Q;wQZ@rp|F zxs4|Dx$Q}_4#@~;)>8eEW^z}!Vp(R5R3|ARpYP>SR=b2!=oad}e;^?4&ZAZ2wdb7K zR%0O?LLI|fBP&A}DgApP`bT9NRCP6Yc_(-Z!h6?ie7s4^oV%*^b97Y~x$hSDx?l7s zy+aRnF0sa2S{eLeAY_fdwR=TaTX>^qklA#agTf!Y^ZlBRfD4v(%~6WkZLM{8@Y5P-ae?3-v9oPl}N z#T*q^rge~(Acv6u`8^~;P`i8!4GX*Z$Glt%yiPdi{+C>C%8R+6Cr(<+eay^cgKZ{VwI zGc6=;#Pf**`Kt+eyJaFmo6nG3&3OmaOY^(Kv9H*Ktv^mJb1#OXnf3&moz>BWgUDJbQ0oh z6JTqD3=74Q^YZdErw+k(>MtI;TDNBV=k{>xQ|SEb3$$#8i@!;r69uA;-6^S7@NI1` z<;^XRp-|iV&4E^Vv8Cl@WroBgYH@u-Lbf>XrRmA;-%agx4-B+7<1RO1by`gI(<`d9 zpn?^XVxPeisl zJ)pUHGc1A&475`QSG3UsOfyH0hufT?JKJ$@xAa42 z#`<;{Z$|0YVaXUcJfjb*htDAQH9ST_qN0c(${hbC(6+O)`v${;8MNiZ)3gQpXsPMk z4^67{Pb6{YVdx+2ey76{FHe70*utMstDcYtdtAgrrg+kWbw6hiKqxW2kwW-h z3g=4l@iW?}v(h=|nd8pGKc(Z&hxlOBPqrGh*aOqP?Dps}5X!#!Z#q4@I9V_FZ)K$f zwyh5g3{0Mni;fP2ISw_%C%@3p&|tzF@HXG}pira$EPR6lE$aqiT?yb}u@clk4ip!^ z`w$XwKw#i)*q-w-7N7*kk|AV>{dY@UeLWW#n*jBjj*E*B$fiF#ONH)9nV2wuThgob zX|e5pBSfGIcPyhLlXfUb{;-WWaWSj;# zPzBgNq!l{-+{$gh7Sk2nETJF1{k^RX7sRn0#b7`KaR3Kid)i4Jsl=3_FM(5k2_A`j zuO2@qp`p1JCgHEH=XSxKKgg1ijt;DAiNSK_)zPlCvYw(MAvg-X17m=Ok5jQ1{m%pj zHa{a177~IX(D%Dh?VLtG@L2U5CJSKs_T5Ko68OQgU)U z^BL*ssj#YnPU0z>s;1^GaN)iEGYUpdTt*keX+*<;LZIv|A#o8bH2TJGi@;I~Y1s%t z7w1a%>S9jbd&Eiu17ynuQJ*vfhaRH}dz{1D$Htnr`5u)1i;F4G$bw}SU?G39Rs5IV z^yT6Oa|H!qaOEoyV9a{{bNa7I(6|mh4!R;LBH~G!6VA8Zk_w-vPXpcb2_NkrR@knc zhh48nFCn-r??}8fRj1`Ho0)&|M7O|Z3S7ztcX?jfE5QQBO^@k~cj{f8j01-olF5a6 zgNx7OmCvZ_OP0EOWj>vrt!SHbyD7}tmV`TU8NFbt#^G(sjNw0jp4i!bo6^*rglp`p zi{j(V_gbpzIfcS5`pV{$DSc+#Uue^IPo^nsPD~SAf?4h=U3vGWje-^{W}eo`=QgZ8 zSRu_;_cyitx`)iJ?9t0l#Ws^CH_kg-nxfywt2(TyQ~PzRl_xn=YP{cFGd$wTgwn(F z%TYG%Ka9_UqVmqZ$NKUl_qzkyQNxKm56g%mLnA?r6J50wG5@o_%kwVTOnM$u*=@dL z0DW9ogO?YVvlZtXyep5U8XBNR;V4V4QBn#yzw}dzBQJKPH;rm{wH_A-5Xm43<$g0y zB;-Ux%R)?w(sNjz7|VKwGKKjzikby5SAw`-HG7X^*{R>R-~Ej)5Pic*_Kxor-8n&* zOuE~*ubG>--eW6}c=|MK$z$Zh>@0I0N(_$g0C~Nq%lheEO{kbKzi8%tyOF7?rp6Ff zxY@Z>V!oItM@rp&-bw!);@H>KQGwGRt-lM_BpW{{Yz;sJ3&xJzkcbzIK-Bi>fhhqf zeJ6NRkib!&4Nktu$l^De#-)Li43E!!`yP^`Bkv%s3f)&$X6CDqmNOmQ-336|$(PLr z`Q8=kBM^d!$;#p(GhYDIW?&q`N%rd!3O+wRhrf8>(%=tkWwuu*C_aDw{9fuVoj68A zRrMygF8P9c#pS~K!TFVyFgWIE5KZ1o7f=ZKEt*4IfMff!yZhn1@ynOA5J2iuPXkC? zem?6lC{2F0x0h%DcIyX@9ZjvR7oI+CZbEOn3tL!Nq^f9%?z0f~_H&ctBLn89mtG1# zU*3O5$$tY>#Y8JYLE?09InTpoc?H&AaNN(5he3 zS19H{b#@L8-ULr;q(_o<6G$R}0MQD&rQN3f`~1sqfk!+NJu*NC`Wc139n~)WE<}sm zzsVOE@Cg$>_bmqQfxpTfT3Q^SuxB14A zCx?@b`d8onS?igI;VFTyVMa~F#8%wv>j!s|uRYWn(L3A`vcjyiOQ4-7M$XK657ssb z5UdWZSrRzymUn=Gt5+BLz-8ChhxFRZ31%Fe?`yJRFvu07@}KnW6rIWmb5gl9%+GV; z_NaREU{@|JP(3SG2RH5W!>Ns16&a7|gF3|ga__U@q<+n>=`~K~btdaFn%W+V`dO8X zTKr;LK#5Dr5wxJa=HMQ)VpgFLV_{7tUuwB}($NjNv$6OaFSO<1^Bd>^n2Qm*CdXSt z)Ne1Xp24}XJJq_l=Mm>cU^>|$31LO~FpRfGM4eS3N}7JS>J^WB#RJBevv zHzbN;_uW4~zWuZjp!)mt=Sxjp{33cn8>f;nh04svcCSh>9$jPF)c;eyL{ zX?AWOuO5O3W#61zq(O2zgas3Tn;sqr)Ya7?Fh^BwE$=}Ma##QzQF-(T2aF4^nVOo4 zd7GOvHDEDEYW++I?vo9Y)v8q3)bY9wHu}uL0sOxN+uT0GK1)Je?xoQ(R*>=A2vP%r5j?7f#bD7 zE3*NASG-)R``V=NG1!^Haz@Xey)B#x9#- zRtCmPI_E2BP+#(8E#3buX6IKXmsdtF1k0!LBJ-E}mIPm(8-2fL0Nn7ay! zEokm9;&qXe;`~f8PRTFrD#!m9Rqzy&tx}wm*JjzxX6wAK>O#sohRgh)catY)C2v-_ z97Jj+<$3hWMkFPE{3O6Lz<^$ztf#thWAYn}c3+szl;`JgO%LSA!d$hHmTzD1_^1(k zwm{AHGd(&w7S*6;u~WJtNlSMgietpO3kzFzFVD+6dw$a!EDg->lQBzsGwnHW!G;QpQbeh^*9+Y??Axm%Kp}h zfs^w(Jj4AAaKnDVOZnY-*x*qzD9E7>oq=|Sq!Xt3VF0*3`E2IqAD64b{Em$4EWE!TATI;rk%1Z` zFE1%f2qghGAf%wc1t;jg*Zv#v#RtG&X3~=!2G?R5u#MknOUA2E;e%f`2UI8iaF09m zStFlQqon6 za{obej_%NET&x?RLR+Cy`}OPV$6ID2BX1uJmZce4PsMI-9_n)#IhE~Mv7ow%ag)}(oaZAkyTPEDTdx$e}Nn!nYM zo0IE9c~b_?Br{Kzn9t)q)}xn6)`eCEDOPM+%igLkV`}O{-@2cfm+mGgK`w^#ry5OwJ!*`#o0n$aT}vUTK0f#%>Uk(AndVk;0?s}X@4agZhSF`u@xAschzSVF zxzyDfhS4U^fG)AMK=VkhYpwxnUriiJ;?(x;*39ByXYJ%%Edf=-Q}>$rR~M1PI7*JW z5B(K9zC3JcUplbOHdBYgxC(0p3o~oiFRA*G)J?+#ylVZj0J&4J#>C@ za`F?Pr1fp9P%`fB?m}MSoHyr=XLuV87$2o@Azd1V(DUq_~aV2U@rGI7Wd1Ev30 zdRW|pRJuUwzwu{nyqc)6u#lsxegNS}aIY=V>j&E1{py&0Rq>69+M{XkrY?3DHgpr%oi-r7OV4(lD}G^aM)8OeC>NaMVI-n^ z1t$!P!(*gahk3b#?`nz~#$FS2FLXT^jq9CMEiTZtU7VY(AjWxg+^vF_ICuf)x!Uu$ zV_ElQos9I%lZ-q^_?d4Tx=E_(}Qh69rbQ}0sg_@{6)?{PB z_?4*coi#|7G807u?9CM&n=CW4^9FEAKd3M~{WBcSSHH@DGkw^5^yxUE;d%BS9Nz&4 z&DSf(KPYtL?-KCx3Idr)LPy6G=aG%g1p>;dh#hnk3^lxz>+eWXPmkFWZA^Lyr+5e} zn3cQ3a*meP)*7$dyz0ZG$Nx*4XJ`d$za+=^{B`pwHRrC8Kvz`{Hb^+>2faalBPKqA zQ4O^`h*H!!u1Z#6Zb}v|GWhjrS|ZltD<0e`y`7JkopDa@W^wU~**|{#5=JJq9HkpBPVR&0da&t7|avm2z(2U47y<|3F&=fEvQ5g6?-&GX#lt5bls8mk!(7 z^|M8J%kg2HCg}X{5%{*z`T!tl1}0pZCp9PTFy2ex7SRrVXJzW{xggI4$4iaPWyeB3 z%}Ky1mul-i%#lX?Ezi_AZ3U<>b7d6im>;`|%1mTc65@P}k=E`q?ddNO_0%BnZ@Op5 z-IJxwtI$99^*-B&C|tIoC@EGG-W*Anko>}gC^c<$bB>_W*qWw8uPoFq*3B{NGJx4Q zIO^UF=loYU(D#(3dL0ZGO9RFYrS9HvJwMcqj!7~+a4NscTR`?BYuC7kp!;E10gp$F z*H;_e9s>5A0V`ohdGl1u%u=Qk9hI76U7u0kN8QA8J1yTkesQjgJ1sjWPAGxRXHfbb z;-D|@Q;v^#VeSm(L>l*bZ|U zK&C#h1tqHk@E%9T2AoN4efaXoOIaEwGSBAD{z}%+;J!Ta>`+$(^kGmOeX3r z1eOH9y5n%Z!6ddAOyA%W*3^hcZhxoE^>yekaA)S{>3;C6rU;4tH^4<=dNwxfh>^knuCsW>94r)d z=6D5hL#p?=%qwVn(=#%zY$EI+*vuEYgWiqn{`u2vV@`i>ZOzo(ij%Cw5=E64GgiS? z-Yd`&RLLt002Qu4lG7ZWhRo;e6&Yhr?k;C%N0l!gE(*%-Z3O5F1$p@kP=Z;M=@scs z+BvIz>f2j=rl_T5Q9`3Bv$c)Z*2-;-X5puk*z$ytB=By=is-utWvOx&XtY#&%(&@x zbgY3cNBzEoJh^79DC*L2?LhKTec9Z`R``_a(Qz%F}JJRDzeAqE$JeqIL+QSGI>pZ^D^V+ z;TSK>vs~GO?T|Nt9fv>Q%_>++xwK>)HUv1SvxmCcvfc7WCnGv4K7$E#LbR%lJLN6& z3DcGmqb0S$Q?dmMxkx86Muo>f2#icd%j%)$G6biaFFI1EGfp}S=KFDYVE^z0Kofuu zlLP|6)lx(F1rp2^wfn$_qLd&XHZh7D8)y6v92_iyz62_u%ramy0m0)GbzTL!d6q3i@&#}BZm^n=Ro3plqM)^Be+Lit4v4jIU4v@QvkWq5)N zADS-F7=U9G)KO&`5!p=V&tJZ@+VQK*hLZpZ1QoI{umlHM4cP8PEy7JBJJ8g02HtxM zf=oeA^7%q1phXwV_3kiyc{5(`K?fh#s9o+1wHIGA*Lg5MRHh$r7i z=?qOOfIUu*_7Wt29#9}?`6wQsN_F7J zZgPU_BtQi&)d7$T|6|wQ`VUqKf{nLg-g8CFn9Bc)ZA}x5Hy^lpsF89Cxeb;V5HIjk&Ux$?Q%?Cz?N|FQ^|#Q*iY7mXPlDWXXn6P7 zFDG}Btt8i?$fUE55&aqc zH4|?oE@$AXSj}k1Nd^6?x>C;h3CRm_eZcdNbYqz^S>X*zCbB4M&q(fh?Dm4*hSJS9 zp1Gc>vF9#YG|GyN?#r&nm(0HhMMM`b)q)gh6m27lam6KI4%;k|hXoaMV*?gDxRrMx0)Q_S9yxcm9Nu~9 z)03+bD^Ax0cK{do)s=o_Q7<$A(<~S?(1{}`>0Ro#;z)ymyNP7EY<1o=SHN3!W}n5meq;=Z)C<&d)EjwcO)u8$!}J_x{r7huwJkAs7kWeqCj zNq00GWZmHFy8TL!?e4$4j4WjYBxP)N!3x-42m%Sa^Ju|cC*a=c1ncJz6uMlNL#(h7 z>Eiu3J32ggT6i@>VD$^dK87n~{RXHPFl(`V*qhsH_8mIjS3sEm-(iId51T*yUPkN05&2$zZXDMg#i?%z;M>;^h5jdW6g)fqJeUT4D3Dm1zTYXm;*rfGEkQsX+Unhd8CzEI=)v!MzKQZ$W1LI-F- z35R-mmiZ=*Ddi#@`a%ly3YnY$WawV8Sa1?N)TIk4?K+YYURewgKMfB@@)6TIHcrQz zE%?MHF+3WkQ>L-0g98mkMwx*q?aIX$w=iia^8Tri)+1{YOBl?Wo|*1VGR%vy%sCoN z<=0(Lf0jnLza*;gM7huXht*qh6rs|{9HRlk)?BA9T$OnceaUevKtEWv1Yjk=BKQ;1t^r_nk2$xo z^t2g^)Bvlh^Gnb&zBDxrC4!z-@xce_3}}rn9+a*@M1JI4|GbQtv<%%I9bfZC!CD2rguWCX?_0{5 z|Gq&fn<-R3>H8#THO+Qv2Z6#twgQ;8W!Vb$Wa={v^l@+y$=UR`Fpb(>hHdvJ{+-6g z(KDeO?04u4%W1>2TEFiudpFgYeg#&1jp0BxI0U5;eiC${0g+_?>lMRinsFovyi-4lUnH!zlEx<78E|LQ z8K&~Tc#zOX@$l}8!);yH{o?VDSs$;ql@;B~ZQh7}%U8>k(?0t_ivFiG^S47E;c#i( zvC_KitD@eXctE|Bb9?gF$v;hdZcPRU#OBxeA68Y=zh>^ATV#4+GPlp0?zK01v{DE) zL|N!;1TJv1xOQ92lxx1Gljltd-W}FcAj6ZO)OW&f^z@ndxIjG}fGKjZ2!F1P|5oMo zatL|8H$y9G6#(|ypoL3f2t`q=Vh%8 zjrPrdA-8Rvc{3c~n2!Y>MZS8_IB9H=>r_?4i!X?=ANso?#IoRQ5wN|%lhI=%+g#rn zA-kkU^!S%Zkp6cXj2(%hqNu^nGWs5SVQytI=dPk9THD>RsZyjJ4f@M{pF zqiWy5bG{{70@-T2VRc&CjVJ~u?hI5xZ6K)iUX-lDeFi*gpa7K>@8+#?_}n@l`1r6# z!&hBzB#?-)k8Ma7bcc-xT}(LFDYJ%5e|;KVn;jUSgYsB$(edxm6FFG!cF^hOx!L|I z@3dXFVA0wmb1O&(-P(YIP*DKu^h^8LJI%I#e0SU5J<)UA6s7nP37Bm zf0|WFnn(x@LKK-ZM2gI_%#tZY=Aoh}lnR-W3<;T4k`f_PLS>d&C^C~$QU86P`hDM8 z-}+ncyWZZM&Uw!B-1l`~*S_}NSG^q!L88KOa0fG~&=#UKaAJ!rT| z12{R^z3)0;H$*YDiZq1JJJHa+5n4Bqs(=rIo~S$`TV>(gx-Z$fnGGerz8JW4_MxXO z|DdxNoH$neasfLDt}X4Q)EfCrARYp`V`niANLwd{&WaO=1*QqmVo1P&_j9uk3kn#L zd3d((5DW`4Sr1$aEVU}ca^847n$@ez&`dQg{ADjGDajA94H39wPR1)VMC-qF6?tts ze*E}w(*rs(59yn}68WZb*}(+`A~0LLju#EU7RvAO%#1UM**Fr)(Tp;kHzE@Kuy*7( z7&N|=aJ(dW?z(=#4SR1R2n20ovSzvh;m=)#v(!2a6{&#KoQ*LPWYi-Nfy<*a8`SW@ z1fGs;%WX292mHWsSsvk?b&;I zTcE>dax8kZqhxXX^mSI2%Q?o9;Cijf)(uHY;_iBB90YR*r*Q+;>s0{*-)3DZ+8@cZ z7fiBJ%$ia=+Y5zO$jH0}RsHstxruTANS^AV_rfq~kALPmLhrLMy_J&q@Zp-tA{WNV zVvpq}CTP@=r+EIW{(kOa0hi?riiTDiHojNzdk|M}@4E->;P{`Mf*)IK&ciw5&z+V1t7o zHD7D`cA$1Y0j1gG_%izJ0{5mrw+mUia9K+0P(%0;CnrOJ>&FforkB6*ST8%C;TQg(*sM3_m#6T9X|Au zoR`^#?6-nuvVsDulFDeO7Y$xEY#`dp3?&|JD0l9X_(^&G-L_gttG#BbBVpB%)46H6 zBhi|5JSU$)`?F#sW!}5hf$&wvcNQl;3BS*lN83i%D}T}|CjBL}j!iz&iAUcqqe80Z zY2N;E-A*|x@|*s*b=}ryKgAEH==L3dtvcR(`;ShR*@vh?(@MUPt69lQyTtxHn>0k# zy6@a6I$RBVa>#ygebM1iH&MSDSaao>OFw(Y8@mn6W4tNSM)gzMQf!9TGCB!_Ad`%&3bM|HqpDr0fB4ch@Thyz)Sd9J*dt0|-)3 z+i&-O%bKz>NS1@AE3 zl{(`!mh-@EsgC> zZ8}LzJG0{4B=uOKzglW0n51oY_+&oo6&>%*x}zGprw)7FSh7WEWyC1$0YPW2>RGRs z$5J+yYVck0mt7-&WwYuUt}Y#cXPq^DIk(F6xvo69V)X2MPnkrEtq?@t%P{(EJ5fgC zr+LP0qH7!A*D5Ft5?fk(4ljZVlVwuO1Tx;0*x1;=tYV=Z;PE5s_W@8%ErJz0;SEiF z|IWbw_b_2cHVznxXc4Ht#unwjI~+)IafCsD$U)bx$>NYe=6M}I$D&<4JgVUX_&(cE z~+s*go@r6-&0fRUtb< zKxiEN{5f`p&!i{_hl1-wmwv@-oWuBuH^srz_wox2>JR6(A#v>H z*u^il#az7~p8x*bR#LL#Z9>A0bS)2#-6=Gb&@hbvoy;7=EwReFzNP_|wdVZi=dHSm z4VO~hkI+KweL}suMaS}K*M0ddey;E0P1aR`(g?1`2~V%|=?~V?_u8c6H!0RmfA!d&K3DM+~CSAY5TnQLlaZS zF_rrAa@Nmp>#LA+KeB#jOgW6q+OQxMFi*s>XoiE|IF~NHi~fbZa(vM;xMxu)+&pf? zj0l@=r=s$`3+L~-Yn<%0o=)4FQ}u(lU^IJ_<>R#v_^8U!&II~rvP3kd{>XW6 zJ16SKz3G9fkVKWruWzcNXU$d`8IPNI#sQoh7;b1gaPpP=;GbE^&^rc~fx5^R*Vf(r zuv+@J=){$_?prW;S=QHoGEhxQ?9v^kWfV5C=jvyff9^|4Dw={rK&AW+7Lg$qu^Ec| zzM}oV=4LBD)os0*chvg8$CF3b zZCsmimLqd%=R}mbr6&FFuoD-D3a&leoxNH9G56WsC2P`m8(7%6?#<>eHQRj1ncbfr ze`GVff5i6cR=t_bv%4~v8a66ET4){koF|Y@(>WRSQBHRS-Za!$V_6)HT17*{bWF69 zn_H#dinV>Cqn*qDhF$vn^`KjM&en_c_pucwyOhW`)Nk3M6?DPQt{Hle8(gsHf<m8<`DtcO>CMYFciAbacg$zBFdMq6E6AwfE5SNgO0qLXqKEw@4zxNeut zZT+h(>Dc!+bu-Jje?^7JL-R@-xPmQt`EtEFw$FtW_0<6lHgXyo z<8wRp^mLZ-E{i*K4t8fE(`bzjcjQmcs$ybKpXu)uksOl?%lP@{-sLvGEIzn#)8~F- zY%$i(Hvj9;s~!^*$3@PYJ95Y0?GB^({%k6Rps*%+zU0sL03N-b(!d?xWIw;1IP0oi z+Msa>er6;YmJfLupc1>MtU=@IE7{S45Qt@VZGMe=@S zZ62X-xqcb!F*Amx-q#C4OEm_L+!0Kb$$Md5)_7w6*om9+r}tO)JvW~2>9~5<_2A@y z`HkV@ug(qpR9>6=z^a_sa@^XnC-L3av-1Hj%HAGdoo=-7Qb99j^TFSOG_x<6SNwT9 z=lgAMi}TkQ)|Vrum)cxom*m zA3?@oz3Vd7JyKgu?{h*Qf?&sbzvrgy%(JX}PonR42CcXHA;~H6`?{v^lu>+3^$z&~ z(?3EhyI-7x<<>J}Slum5cr3!-<-DsyU%rp-g`e{rVqQz679KL(jf{Ftc*97}n;wP?6<|GQ=S;#N0>jQFn(F@r z&NKQTr@=POPWbjE#S~d1BW7{e10~zS@EK{m;$V{!{O|QS^#2>s(EQgSd2El)1B`!i zxVoXVs*5|MS3xf7jvkoV*!`D@vjyPO9&ez62H8)`o&t5mh`GyPm;&N+N^;t$1BRQ_ zud#zNJ(wvJOih@m0j24=$0RF+oTr4-n>NAMq8gNzK7bFiZqlgjfm`sW=x}dAR^8iv zUJ?oEh^siWq&G3+C^*5oX_G&E==N@2M`yDrkeQVQZwOAp1jv&aP5?!=9OLM`)CB$p zE^9$1@+&C97`=;ovLqxVn1Q!C+?o}`otCTT6%*SAO!f+B2ZJuOhrj~*u&K7&dXZlHO>Me=et$1nJE4&PcV~n=x$8j&!4Q|7f=RcZ1pI#L5~OzsY~wsm zB_brXT4R{z>o;!%Qht3M@rN4aDkHs+5W*8Ce69vq;kN*?FN%b)cU?Vjh9z88#Srjn z{a4ZX^ottaiikj4CQCR)E#YQ4TvBxRCq(K5%zn-b3)i}olw`X%QuEpK5A3@twsaSI zB^e)b)4F`-vFEf*mXQ|=WeSRNgo|7CZRq*1+IM+wryQ`=Ud1f6U|Z}4=FfDCp6Tit zm^E!ODlzdp5Q;QH)o^poGUw;bpPV{nPoEZgd$&f`$_kE&+)F5L z-zvHe_g&{Rme{~N#_e!X7gaVR!+t4Q)tf7$8*9~VmK$}X_iy>QW^#jGY{)x}uMcVe z9CC_<%({}%b!^;DQCg8#)?JZZaaJ%i)8g!`i=534>AmAa`=9i%_=*JwcWP27s`20E zzhB6eZI%7bW4PBx`_k76qlhiqNvVpf@-tbw7Fp<&;!%3Qbm8BPx&9zK+ zt=D~hseab080I~`+35T_@8Pj!kL$z5MedZ876s+H^6nRvuWK{qEK}C6U%%#woplJS zWXC;FnO<9{{E9gNYn+&a`l*xW4jmFJDsCC>?WL2E;ZrG`h)OOr2s~)b`mv{Y+totJhP$~WYJ1dmpyAgd>Dea8hKO?R40ONetUGhi!9~ruroKQ{Y7LO zvMC0VjbJ-znE2b@SpOr^`vfcB^pGVb)G%G%A%(R~eH|rq>C}_xGCf}T#QRHOT6y+q z|CM~}kz^J|M$-3e@9gZfUnFmfH;x1Jw{_o%i{LTrgbpLZ)Y$k2hKig-&yRT*5sjD- zwO1Kh4H}x7=5G!;57NNFLbQ+AY`~@hnGZFDT!1!eGXq0;ON$OwwsnV?U|w0va2DGW z9tAdFzhi%gM9RdD0~1AIKonY)CC9+eOdSfD}-077no`wgwd{CmfZOC zkj-BDnA+OfaxihHQRn6fz~%=2b2QR8C0`NU9Vd8%<%!enkg9?O%^M_L_%#v3igu{nQAxvuv%bEZ z$_WmOca@%Ez|Ts`;XUDfYQ9Wnv)}R!ByX~2ONKDAjCY$|(9_d{qY5K!e%__XsI8~o zCmyC{+H5vEf9FnK-kO0bx6OyHGi+oYyM~S~rL@1!+30Pi@7<73eVab5OgJDC`!qef znSI)9{G6CK%v)xjcE5YyQoSTcs?0pFVb3#9vBYy(h<|UJFLyEXja%cH47XW;N3X z8TnrccOK@BOms9g{W?Bs7yfPE>5OkjWR2)Iezcu+Cy&%1@9Bk9Kl0FmUI)B+;@t(8 znwMV%_Lb-dIXY}T`Oe>Y*H-PF1)6k622?gsmR;;pltdlz=(CL2%jf!+heXysk}jqd zS(~Yz+ESwaOI5YCd*7z5TLVB%yR(i%`R~IC@ii<4RmP^*#Iq?z^+7oRb|=gzr8eL> zY^C7x2NQzLgIhLl?!%7wr^vF<$3{s6lf&!k)vM49A#wQnrnc4zie^aiK8t$I?8?a4 zM`u86*6ZRWw__UVF;EWJ!xZ|ye7Oda6ATlj31U!}6DY|GhWo1Z%%zwFcMSt#KYUz1 zB&WhlPz12n;G=@&$k8dlbVET|xdLE1RrtXdsmKp*+dlor$u*o0pEehyJ%0T7c-wI~ zUV`;w3K+8QHNho|M~)nMgc%W+0|SFM7>{D0TmY=A%rGHrF5dUJs%jN+r2%MuN9D_3 z{3$z-AotL3K`C`RGBP^)RCl49sOJ=$t9Tqu19fMs!oZ3j#D5UB*33ZW{s^*=vdhG= zX8$Jm+7dfh2ZvY3&BAZpI{S54BP{7LX1gtwG2jThEvc6DlWeTJqV(V|_$>g^g6igI zavJp!16?7$Fvf^KXbFcU-Ajde`VBxvt7pKQ;>SFCg_L(9i)Xr}nqp#NKox(-a!N@{ z;~r*k8wsbZqWc!;@2^=A6tCj#&DGd=Z3zXMKLj*ZVbf}lUgIMUIaLv?hlJD0erl#S zk98?_ySVRSP0{9Xr%~ZJ`Nvp0x{X4CqS-w4ku<$f!gV{Zv~+I?k6w7=ojsesNsPm` zM*GZHm)q1!W%_>J+U8%&bt&8gYARdW6HEde9k-Z|)N?E>Br>zrDE z|3m)@M;(^t|3AK0zo1#8yNIzwx^cGb#=;N9His@IPTRe*(~E1TY-vQK6amkTT=omEAu=YIA>F7Subrlleuq3hvk)TSMn3$Y(a-ttZ11V_pI-!&=Uu=n?ns>2D zs5E9iAaQK4DbC<#fk$u1ukUuglO>Dp7Fq#TF4Z9q zmcee7>-SXP65ahCzZZ@Gx+(DeRT*(J=xp`FljCsvage@T_<2l0f;;AiC~I~c3>Wv? z<-|7jFI(uJnA~M9_x17dVBh7GhaS4-@LA-LKf)XLc%9FB>M9CzOEzCc${NI{TM@Q% z&u-~m<@HdUs+?Oo_DnCJ!q0ZI&i0D+<0+cjXAYao2)TFhlryxg1(vg153Q|?7Lh{x zQ~nolI5R-jWBGvZ?``1ukH2v3ng9<;IZ(gU9lweFkfs8i3-AmW;RphzzhLl>H@J_= zg*G-ax(+2AmFW?70}<5>3yMl5hP_Q*SzY{abNVLIHbnK5h<&eHFbT3gyaGRlt4Wn% zMo!sqFoRm^V^?&Og%`ZJzTf=jM|E=L&dk_-tDoRd7*n}Bq-ngq+ z8?GDb`SncDI ze+81m+mI$H$4hKOfdN~D>xn2lUH&znLH?&UpXr6N(Ez~iq1lrx?)zH;Qs%y~v3>vY zx_mrCklC-`u+lQYEOU%hJ$fC;*>L6H;2`9)TF=}@b&EGOBP%5uQcxTHqNCf^X1@bx z>7PXdj7UNd<96-dt#4F-j5+3@HBoKjxQ0GuCA!r`(qZ;*tE;QWUbZ9N28WBIB~G7j zfn0kY-oRne(bkWOaEg+6NbIr19#`Y2yw_r@GVw8aqFJAdnZBIBmK7|}UuzGSr6DH3m5#VJzP(We^fjS=(qJlP4VMuz({b2Z3R95zb z!bMQZuRl8;op%*yAsRXa{@XwIQ7`QnZs%w2jat>QdSJ6(8iT*x&QtGYf=s=7hVkaI2b@;k4 z=;)u!wnA!-Nf`>HL5dUuiL+oP)Rz9JoY^6TdGzSf)VFV2*)^%*)7u231(=c@#pQ!z z9DwaCzH1QqC9p+ z4YG#yEaGgtcF7g}gAw9o@Zb`)xye@c!YoiIWglQ|XG4Z*4yv;sRGC+>X^-p)|6sv_ z%!zm}aK)jO5$+y)P4=YJcMTbt-GGgFTX%qe54-(Q7|9G4@ zMv4o_v2deAWstoTyAlKaq$G|qize6#0_Z#l9Ho={?J~w2$1p0LX6B1(fb!RUV`F0n zP^}TRZt8De3JkN?7cGA4M7L<;fAZr|eAMNDfRAXvr53?83}$9p6=|=xg3?M(i@aas z55Nl(5fO>a=A`Cz_4P;H+}whP4J~TEj`ZV-RK~i`F?@b2DW|AN&nmS~9wZf*&~ zzek%0uw5y{?Lk+kw;UR@LSnsdtk;*0Q`LeEW=`KHqX}KgAe$BRbWiV&`_^jvNuw&y z$}Qmj{h64K+;gg`51PQ3eWG7=Cow?mO^SJ-nU-@TZTzLT!{U#Z1r#-GQ9O60ot<7^ z`?kJS-cxR+j5DoUg{0+=cK&2`fY6*LEJ}BuJ0otYoSe-MkAD_H#o!+<;i@#~+Luw~ z|Cot!yXj?vbooeei8qb?p1LbKzQ*TyYxkqUJ6+3p*k`N8$n7{<=4y1tLvpz~JJZU+f%E zyQ5ND1=A#wcL3F^{{prMnuXg*OhEgy-F#KzU%vtuBzripj{K#JT!G=G%nt%{!p%iF zR(Kt%H;$vkgZAqJCLDmrO+f*9SHeqE-5nK(wyv%@e4pS%%yE>=vLUZA@fzAfv?lPx zNL|251%l!#oH3`T+iU<^t{7ND`4rB+Niq{#o%niShY1hq;7EXopSXEV(owG~NtYc6 zAoPm@DNu31){`yTMX|Zn9+8~b<_=>1aw4)gVdD@O_2E;|U*+1*!?PBHqMwX-+o#*< zskx4S;c{o>jq>;3c+g4?nvbA8w!KRfQ#8-!mb27C~ZXH`ms}+;&1(x`D_F_yhP=&mps^miK{8zc761cF9(SihM)e+v}a^VQP7*Q?7dQ$CO2Rh(N>X|x2Wrgp{iCP;yY1m2LVyxMfSf6c`Op2 zEBZ0f(F`CNt;CL_=eYlqk#;34CrG(RO=zl-XP`Pgnrl)ldXMUZs==eBp>YX4es&?D zmMmqy=$*pCTj3eQ>*ME)b1CVkQIRJ-P79G*I`C=ja*J1{XmZJFVyKxTpUG_?d$_7lDDPhH5VmK8ZwahM^?G)Cnk3ll(Vaf=3B=S1w55noj^?&?|I-57 zA*A0)hymv{U8n|wHN@hBe33g!cPqr<}3H3i9ijgkcLk8HLA76Iy7TBmebfgcm` z2Lze9-=FC#yZO2xZd}{En*M_y&||2>ANnkMBP+Ze`F*5m6~+Ph6%-WEG-w>bzQ**I zs~8Tv0|s4rqWX^Tm@TEw+EA9%JSrA-890-C8 z`~l7qdk(9%a)?T%03{>458Av9%UBduAbh=6U6EyfE$y=)>d;ffrK#OY9R&b0OfbXX zhuN_a^#Qspr5`@%hyKCb%RYR-e**!&(jv=Rx@_4BFHqhAZSUs>JdH|j%OEDDfmK&N z*0bJlH6|n^!{sQ}LbBQ2)67*mvnC6ticC%0edHwX36C+X(O zWmeY&ZaLwdx8cQPG<%*br@KP)QcK<8hrfKbMfL2fz%TbUe5qVdPA-I}mf4w^R|Lg# zXl%pc#&$T1F4jL>o(VuFvMz=}K-w^9l(|cEuR%dyFc=0k$>u>)@#KEKun`JGJ}B-8 zj!5j2LGgi$6mP4GlT#VCGgUFTY$feO(baBB5I4RDTu;u4g!412iGd3PLo^)KfDA#! zOU0=mU{uYK*G)fkV&|$(oXts5vItVh=Ey_`%x$QD!veVN|h$)q;cAUS6$#B7&7708Cks$frp^=fh zWh1Ef^Q#1E2RwRWVNs3~C`&>}XbW`2+-DX1rE{0!H^c6t^3Kz za>=jpy9$^u1*2?;9h!vU%+X#tLyubng#$eF#Wd)<;TLOju57FBk9csg)S{A6b|2Lb_GbL;jhu-z@9_C10ItZ+U07IH{vTcjQv|9ZM^M^nxnsv(b#@wt5=$z$`2XWhY{clfjmbhx z>yUI*u7-Ne#2Zfm4Vdz1Q=hAVA;89OHz)S+$}s{fcATGD0%V6Rx;9oseY+E^g!{_; zr*YrSakfDKkzU|%{P=28Bok+USo6ulaC4Q~eQ&D16i^I+3Vbv9 zWAgF?#ov2~d;W^>ktcHhs9`AzVH+A|(am}0InD9%*Tf-UMYkgZ{}(f)X=s>rc6F^l z?oB#3c~@!UNnN6wV{{dT?mgt=G1Ov*Bk40vzbjY=5NN%$`Y)%4Pd{CpDyOU~x!wEs zpxVVEI{0Fxj=K6qXxiBK?^pY~6cAc3QH8ni=!8OCdvbBe96tSYY`cFKx$?fBNT(uR3qfmzCTA`9%=)}wtha`RU zTfjFbO&;okhIV_`O`pY|?Be1Ep?XMeNV7a6Loqq+i=Kz(SYbHZrVp=x{M1#A6}o(0 zkJkCW)C;M|^w+PKK~!hkFD9m+Y%}Zy{{Rq!O<^wJ<#iIsy~;}m6tAL(4y~Q>?7qyT z`Mt{s&M9BLixnE)H+AWpIdy&mM^bB5_s###kc0nY(xP7uCk?dDq?04ev<$ddBie#k zHs-U_y|H`9mAzRyHG&PlJa%7WjF^v+FUSI%1e^-j@s5@(_!3pMjqhL2$dzw_GDLmvX1^lR zK}$mRofIAUuW5@%2rM`zverU?c83%*mXmNtf#ZJJ%kPjLn}p_MWeqMm$Yoa`+CPMK zSaNdm?MS?9KjwGJB5@|id(cgL-o1Ni&~c{rm-o!Apr8wa4bYE+z5=aplpk_IFyF^d zybcPbsD1i=aKOxaj7^+%;?Yv@*am;VW0b?zO+r$V_|#ymw`!;<4EkYV&LJeEL)~C^ z5}^A>9sVBdwEvtW*7%pfSn3nF9a#3uz1%ou_B>vJT!WZ)W99lSz}V2q+?|?A+BV=M z!=ILWMNNE8jY@~nne?B9JT|qzigv3DF|mpyj@0*-HE$q83p#8B@5Ghtbb^YaV*RC{ z;grFgkDsaOcSCK~_|2}rb_8eqdTc84{kXG~xvOrTv-C{vH@n#E`dCPuG`u2@S3c8K zUm>|F^>5AN4bdVqaTmdOa{UkcOBVc4f#j!^9VR!&M9$tmi_KU|%O5g_Afisjs$s## z+d})(9P--RxA&SO#x}P`Mn#bn0K+u*lko%EdU}Rg4De>fy*5Xu9dG5CzGs!*-pvRl zuYjIEvT5KznIXd$4fF_65|8R*tm8}SFjasYgV5H|IZT?Mhe7P)fd89|qJ@Xg34qZaa#1b# z01Sg0+~fP_0+CLnXvVap!;uF`E{P&!!yfy#h3Z^qI}Nh{GGy!e4(XmT@eGsP>VMg2 zM09j8u~4ROm<0`}3ccj|iA6w_WvEfkj&}(CwJ`Acrxiyb$N5@KNr~jLhIMYmA3vJs z@!tX^kmRG{Nbiw%sf2jawXm`h?kr57dz6>wj~}*Ns5%gQPETaCwuOX(XW94PT2c8Q)3hkSR)kXG z0s8v>V&)YsceO9wQIpAM{nc1I-JQ(=4Bml1u9++b5-Z`%f+&2l{ozR~O9XjBF>)0* zo?S(JE-X&<^z`8QZX7gz?z{q9^5b&+Er5p22tWX5*34|&xY3giRQ3->8XEM7$NpGu z8^J+8i~_aN8_tKMD+uyQAen5z5hm!8%o_|>eWF{zG+stv)%D&+u6=>11C(t>Xjwh$*%TZIKZ&}&(=jBd8$`T6YxSWjka~H!o zIc20HR$g$~zN`@Rn(6uT=dc8sBr}5Z1OSN;$_losQ0)8oj;%OCFbcS@oK9K}0hB%R z4&8liG;)!QoP_WlxydCOs&gTILmte;;mBBqHQqlyenvO`&Yf#Rwb45fGA<93dl;Ya z*Xtq}#@#Y6Vb6o_dKP`Be@g|72OMhtNYw_>@*2R~z{FSE!}$~t-aOB%_FST(j7XBV z3}VNbBd;2x15AS-UR`X8dxVXHJ_AaYR&n4&$i`QAfvRS8id#6JSon&gjv;3w)3SU% zw_n*XT@T*8TCyH$tg#_h@ykJJA8K}mNc&Q&kKv)p{}R?DERy6%o9PSA{60SFH_fwU z#zKd8>t+d=efJkQpI~(TMT&v@o4@ho@h!we;I;fd3!Zt8VJl<~FpAJKs6#)- zKQj~HL7D?jQ4qD;La&is=wAZ8(M}#7rMqUxcc|4l#~nT6YrFeDC#Lh>yG%SPJOUW=m(dG7}=xB z_RX7&@5Q8vVzCxL8u25*nZ3-vtSCeeZj~euAeDw2RQJxS@R?OedDSl>`jB6v_S9bv zFF6>nO00&+LVav;^~3Vq=BMc`wq)ig3{+SP-N$Vosp{%(ZOzj6M}F?YMt~2FB;8UN z38Dy=E%8~bdqemiC(Boovrs%v_1~HZ`KzZ!E=rg880b3WNB4_%pdD_)gkJe=aDLH_ zW&4AioY$OVmQaZbVG1UvauRJ9nl)=S4({gU96(L=Z^y|s(;!`X`0#*?iPF5W+T05$ zuA$`G`+63xO`<-oT7W|6ulLhXfXdgiuF7Y1d)Lq+87TiTRB_i?ts16WtSRAv)LSwg zZ+^K_SriSyVIn4f>yb1s3!Ub^it;rU5iV17w))OIbM3cuuN{^b3-}>c+AG8E{&z*n zWy_+~ib5J=cld56eggl5xE2$x3x<;hF;c^3Fv8|BK!8rDDz!|Iao|;f;9>t$EHA~$ zPltmKqs8yckjX&680qihp0@JA78m?8Szx8bZa77~zYbplrx{epylwaGit#sa(g(5U z8iFjDB8aIS;Cu3CqG{$d+RBUG)mqXhfp;ocwPx1VoU((>cfYV!gXoQFq!hYyDs3Yz zI|JZ9bl^^lKiz_dfVtg09_}y&Ky)EMSGD3(2AnuHsv%WHW9ACXb@BudFVHzg z>+SHXG9gm~^3UK!N(?cuciCY$k@XfryfFCUkrC+Ig|C0-r{3m#GaSaJs;(zvsrZ60$MY0$0LUE5f4K;G@66iTi9RwqnRGZs;va?$=ONC43*mKpf zvq4`XHFC;3A~xE;Fubx}X{`GTb9J-z@$=2mjJ&dd+BL&7KXdV&cFmwm@wT9JLpmkP7A_}^m zudtS4?%QuSGBzewY3CQFt8WXRVZu!v&5ngV3QnN`oI;go;D?5XH{k?w(aLPG@_(Rz z$36~`e%K4$D9(gaWU~+;R5`xt!Hb?O&YG$ z74)xN1&+ieISzVuaof`%oDr{KVwwW=zX*>A(+$AF)ufuo`F1Zg_1J%YO9p6@QSCrk zSsiqvfVjZ-a6OLMzQtd-LCvzRN2B;GT>%m#ybVm} z_F?efR$oIz@~zaRJ2(Gw@`0t56|lnmm2SV<+epxH()*XpesrjQXG{&5(Q(>i6mnVu zHql8r*r$F>Ot+iwnC)s#sifAO`EtQJ$16o6G;VUo$zYXGe+-K7Dds&zUNC$0TR?~; zi6X)^$5E1-eEo>t;)hae^f0z^j!`>GwhlNM{mRZi>G+dS{OkPn41-@%zdH|(MDB9k zn6Z5M2%AFg*3h2KVHq0^(_i>~v)X@YWAMf1i-KVe_f`i9($fg-a=jI5V*Sahe?V+v zOopEQ+^zG{ldbttHIq^vx0PGR@^d(hpPenMlA~OlsTi2XqP6=8Epq<(AI?aSDo4dp zc)ERQK91Q1VLj((RnSh1bZWd-gV@7s@Z^2+@o+)V>GIEX>cY3!q-`q$!zfw{1`o|H zVU?t!nrTQ*^}{Yac;mS}9fcyn16Fu5mP0zy5-w@!UV$A93=hzunw#i0gr%$SuJgYY zrF)~mm(kit@!iYC)qu$dnegh`DTXE~$qM=Yd*6RA;@u28BW?)w>c9@CC(}Cc-%HIW4|ur5EnwjbNuogomHf zlBG~C`c@sFbsO){ffc8q#n5e`3S~(v*lGDg?W;`W?QJekF(arbrj|__n4-!GuK6qQ8eon|kBf@p+nn_p`ma{CgY z0$s+?mFj^8@RXmR-pDq{R|H{as*n$@UT{vD&~f_;gMTihHTM3x9C%F4D6r;gI1`46 zxv^2=oHa7Njv}u-M8a&uA|4FQqz(V*g&BN8Hvm@JE8-q;Uqgf80vZft*}&+ZcNU6v zZ%|ByFjos$Sbe}{OHNVIcEPvr-W71IrW`Ny{qqvMzbw#A3_sxroqKy|lKO#ysjGh1 zjktb6cGU`Zx->NZJr})d-n?nV4bMk82g*@PKm@rl@t03Be!;{%

tYAh2<)flMY^(7W&-=GAx3dSJ9cPuZeMj=YiCjPwq6+R}l z0OQNB4?&ey3)}m%!AISu-;z4UZA=3>c(dHf@c3azQA}B zq+0|hb=qVJxX(_qc+jGKxlJk?O!@px+(0{M)!yGk;zm%JZ8?JAdzNLHs?*G+LkAC5 zzcM;bbD1)`1$AQb_pkwn9D7i&m`_hRX2_*{S4>>Lh_|NW`8oaay($+UPP{Kz>9;$f zE+KJr=igIHZy$SE`jyTJuUMJBwYQG&eq2S$LUkFbmnh6AlqYATtxMf0w1R%SzV6t# z(B)J{>}2qfppSq00<9aNa9_s&gwgp|&IddnU1zoIuR4q86uwJKwo?uiCa; zECw7^QvzC3DTvTjATwO77vDYBR8|g$_jM1dC0Jo)4`c(wjx{WBCn0EGuS9S%cG|=P z1!&re1zO!uQko)@-ReZg6MO3C&!2YixpFzr&`9$Q?*5g?b^akC`u(oRVS;wQm2v*} z2n(Mf?FMADzs9E7*fq7aWAWpDzq>|b+MJyRZB87LYxU&uOH0%VC)V)F=mxZ8zl4~X zn4&^)layKV;z+Z)v%)2Q@k{8&{!(C9s86 z&vCG5Jj8&~#I){J#CvrL>u!{z4mC0|!v7=edL-R_p!TQY^PhA2-}q|Ma@RY#GR_M+ z4j=T%E#Ajdd1rK|FmD_}ZrXi1mEPF3GG0d%3(28XmRa_HqOO$auWLaa3mF|7Lz*8) zCKOuj89Vke)vBW;M0cN6oL0oT9d~25#%JCiqjqL-Iy-CIE?;h|wtC~c6NQ*uV{ZBfLdrugMyq5kgUc~%ElJ|0P>nc+ zj4o<|h{02)<5+5%+|;B;jF&}hI~B4|*{L~pWi8r|? zoN&?wzT+c!x1q~UWbE6a@>oF94PcIuOH9mC_sq=%5y0cWZVDC3VSmz1P5PmYGr@p7 zuEk@di%Y@s^72*KbQcZLd@LH5KQv1y;XysnECEjXv53`Z5ImG1=@dW_VNsvjVNlxV zU{)F=hyo($xl)-1V#K=_BgngWD}b*o#;_070CDVqfBf)ec;odQa)Pim*{~oPp*wiZ zEl(I{gyJ>TwY42n0v?nNC(==y%ppzq}NwUyR%vYCtJ`?*eR6M5%1j2 z;0j+70a`*)K7%AV*Sev5csOngWoenwsf94U15JO(f1{`igYKS2&18?Kq}%+%+60HC zr|$KO2&8y*2@ro3c=r}*u(659N9460LBH(^ruHtQxD;`p%@hrtu`;zns3VCVCq;bl zQovl(Z|ftY*hv|Ok+{>lQE*5qLWdi?PZ zx_S6pQs7sIOVJ!hcg2AnUA4Gu{l38{HgTLbE8Hx;0LH1|^?C0kOjJh*wKyJMjDkx% z?w5B}AwJk+=caBRL{BUSFWin$$D4>{!2Y#(u+uG?1l54&L5H0PeRTYDkgh3EFE($L zKxgax;}$a==k-;s?lVak`E|6K1t4dNp(L{)k%Tkw*1H*d4ASW&%L2gFCAmBr+ZBE}WpQu7Q_05&9n zo!Kf!i*K#MZuDd7l9<1wrxv9u~%;jw{x#bieNcpG;h%%Sim3z8OMDD zeRHj@RbDg(%|VkwCgAB5gwl+B+Nk)9{J>{<6sre zUtQ|S^#h2Z&a1Q6?r#(RDWrvLr1-G@E)`d9${5&+X5ArGYB(vYKCXig3haADao zglsVIaWF2{6%!KD0%W0%LKzn_{t@`V#v`1ZUzFFYpTKZ|3c{5|8bDJ#?0a1-D#qFY z(LlfA@vzdTfsv7x=1>G)Rk;IglP)>UNt-rpdWqt*q}z@~PQxk9qOs;M9CHr(F9U=8 zx&F(aXUW3F2DbZL6xU03q@p_U%nxnstHC~#rPD!-yHY+S9KYJ6Mi<;2v=xY_a2Ax-`bJ>_Ua0}xfBu_yL7 zkn;|MLnxH+!PfxyNW#u&Qe}Je2qMl_gNM}RJ#gS*FX?zrE7yM?5;yp#b>uUPja91!!3kC;-F&JpWg}{-ttjzC$ID5WWz{5k#JHFV=h;II zdB0fyG4gS_#DX5_G7$HcK3|Kkdqq-&fybQvSa(&SRT=UPFz9JWJ2g~Q&#~jy#hQ@J z4DwW?z}-zZ@6{`1%o$c7JkoK{C~Npz(9Xt|e~xel$O=q66&-|S0^+o;Sxw@@TKRS* zb$X!oLHmua0}IRgh7wTC(lYsPu!+CxB&E-M4s#t;DSU}SObrWVS_zUP98zXeyJ_P_g31ORx^y2p z_DIYQeQp0Cd5Cfi_CN__;X8u%n9IN_28Nmv{(?jtRvuB&**m$RB^{~0&6X1zrn2y6 z-i1|U+mkv&9%n`s%DA~s)HraaDEJMN%PNqOQ;RA7#dD$IJp5cQ)5V>h*L|- zp#f1=lsR$26n0_wNZ(NL&1o1YLD;NRK1k7!s=xToICwb?a1Sh;ADzj22>b?>_!hb4 zvrgb0NFGcgT@w@uHU=bD&imt$0uwFOBvehPbI+dRcX$d=o$cMZ^MfY(qJ#dlFF$?x z(lBo9oM+H)RJ+zVWiN1?SB7-4QK)Dve%b#;`!4)S;WtGO?ruw}lpsMf zb)Dklxi|$3C8@IvT4?uBkGr6@#WS5lKG(;FQMAgM*qq0wm#N&J4SZB==Fc1Fg}Dhq zm?{C$pLlJEL5j|JwV3-!4w&&ToF|{iWClhf8|(TUW(2QO1}6cvy>*{+=GD(u-?C+k z`v|%%Dpt-3!1o=@MZ&RzcowFZ*_;0T!cV}w?v=!BnFCgcB0?F)kWzWUGso zQ68babRw}I&h+EKW=)x|TU$NL5ANmR0X`9$q?vwZKl|Lwz%ztXSi;x$kQsfX7P{!A z^SPJ8+%$18-%#P4P%O-X7j!fMDk*a1=gp*=%q!;$MjFmuIuii6!i*3KTjXfe&?Mqb z)9CUF3)ce>Rz-CAWnJgO@!F6z))nhS?C~Vdr+6TqLR!20&zPt|SlNY5-N8}&IG&eO z4!u3L(Z%UyJm=o=X*Lu?!!^+SN;$W5o=Wb%cK1`doB()mD=(-r|byques%V1?BA92; zK5QVz)k&vf)|@VSNjw{V9I#h6@PO_sKFTdv9&5o*${VZEHHBDK@*qSUz3|Jd#Uj}l zD0pVJ=T5hRXGp_7G&hXa5JVnNS?@{~Fl@=a2EYLd*@SH)jP z>deD2b1#j`Ft3k<9)d&^VZdagzbo8b_MtaRf^pJjIgpB;@jn4JB9!|iZCbxRa@7WG zDC7RB93M~Yk9)Szwv{$ zMSSBX<)yBr!XXj9TDDIf8S4Be}D~M@VUc>y^eD9=8Jw5`;5E=+07LlYImJLM}Soa0>D3DIQTe%2pl8Wcg1hg8t z!0H%fD^(sIXBhjA2Nm@dF{ZSzun_tB4Cw<5+rLP@+9!{TT7mhi(f2FZVo!&OBQWq^ zO~Q*K#k~bDrK(k zCTJ#AqGBVVWLscKG(LVDP+d1fl;;}cbj)Ml)5G?a^5B^+N}R}C9)bueq`R!GWw{vvpagK=JloRRq5-reL+Nn_^i zItRS{mQv<74-}BU@$89(!w`WV0h*YP;Z+bNpdu_$*pF|!21&y2`b%MHRHz&hfU9nm z!{bx71zQj1r3VSTn)G>q&~0jEq@{n~Vm9~=rAsVmp1u3_B_6YdUg6|o*Mz2Y@2DVx zk?EsU@4o1$DDb}OjG!r?7FrDTIGY7i7}_J;2|-8+cf=l1fHzn7&;;=VRVwjc{rc8KB|;dUE*9rA z8ucSmNIaQ6gDY8!6DR=jz2N1_s3t)|!b90Zm>}^gl3}0X{+B)%Rzp`Z6l|N*Okxd? z&nQg((BN@O%Aic=&mFvud+}4~>g?ndw~6Mf?!Nv@suhfbDS;O2>HK35G{CoqEi#VK zYhl}Ut1?epIgGVyA?H|ZbkOB1iL?3M!NX}AC}i`f#z8VqLyU_BNb0p-Lkv;O{nBGt zpOLIn-2besyu9mM8_Q1UF%Fk~{kq?OnWPgbN<^+lc}SFFZXkkde8g2eEV+}(Z0eRd z@2O%g{YRPR&j zZT)*fDC*=BtVV4*%fe)Q;lgi(=a=2ckbV&Woo%x8(vcJ!>*!GQu4Tt_C-e!iUAL%v z^?qQmwUrzJ{=Csg*hED`cl5+UUYC9C2xIk3IBMr3D1ycmret&(r zoT8#2X?@k1kW-Nz0ah0e2tP#?-%HqRVF8M}i$ByZbR1F|gpmUL^Io$Af(x)RnQYg2 zULYt!JoY4stRN-2BITXrlhQ1h8xKaXzLx%h(71)gx60`U%R6u*%wd2?b^t98 zeFzo>E`Hmb2lGxMk^u#`^trloPh1SF{x2pV=Fp1b_Db4Uqa+g!8NW1O)s$(321gK> z6;jG6@{!{x%yqtqj1+*uLEdoj-j2h@iT8j@B%;>qdEqBsd3)Wi`Bw+NApoR-$( znU{}%gT~@In?mn`z{Rvenc%^CgIFLKWT-61GV>EU*9w;P{FrGA8GS-Bju1pan>}X zrEezT;wY8Hk&q>Zl87|d7HRMeQhd5!3b6xuMxWHhzL;jf7>D5T%V)0}pCOGeVb|%S zrP!b3jXw9qKiG#IR$forX(C#qq_F` zzQSvTDq2@O>kH&?*sR6LdbeRMcqsI8ji%euSBP#yu?iCLGWkdx{yc5XH9phRoRiVw zK4U?EBFPVh%s`Jw`9F|5a{r~mvq+HaC8>!vwVhpEX28>?N2^YgZWBoDodF9;dM!P@ zM1X;p0b@XN<357yP=P4KI@qa3j9*7|5pS9wkr8CC~=n$L++@9DnP|5 zs*yf)^A9keaQ7k6A;l;X^$%V=&8}M&6J#JmN37MI7njX*t5`D|@xcGDcBnfDe(uq)yKKj}o?955O0`+6W z?v+ugwX6uS{=va!-1MFc0hk#OxgM4}!O!{$pts$LhmSMHX z3XSmyDXM;UZ)cemerHNN_k#Wp@-MT}-a%%!X()m!cAE!s$lrJdMWB}q!iYK6|5}Vy zR;rT504H(3KOIVJ0DPv%Zs|G~FTNy`Z`{x}Nsc`Z3rhjI+8eeo8-wq*Z~an&H;gvu z5xKj#*#V7pWjE1g)_(yDhwdl-8EP8}RiS>yfSeF*P0axGPcH?{YPgY}t@6Q~IJf%> z5t4)u-?Ebz&ICl`D@-9!o(5b-6l|z={BvsXE5mznx--~n6v`^XkWS1otPY<>^rvF7 z(hjB`1zC8SW?*k1aEX0h_HICC)xReA$OD3~YMem7^1roFNIGv#Dt6*mk5py-jU?jP zL+aJu8b#Er!)Ehbr$7i-V3gTz{Lh)cmB{6q#Ag^aB4ky455|X~dQb)?4uDmmsJvWD zMdcyd-&6Ysi3prLnO57Ur0_y_Mvs#9IrtDWh+Wckl$7otHm`eUYle3w*n}<`uga^N z1gO2IGTJ9XntB|^%Df#nSKOs`zwv|uf#LwNdR4M3BMl`X>ge){i8U*;P!^K@WEs5# z6$3O!i7|~lD^wwyLdz>(nmHBojB8qpw9Tj9&1-Pv)ml@2soO^BK-d-3K%P9DU+wVn zlyIKdt(zu4o>~&l>BG?B_P6u3ML$CDYM_dcpH=2%gK7Y`@&so9Tvf~2V>n--xI}wV zP}IyiLf=Z>I1thI7s$ib#V?5e8iU9g_x9~*elI~4AXke6M3TC1ujeW@B`Cp2TMz5| zE`vS@bU+f{0!xbngK@Lx7S9(!HLX1o%u8Q7m}&P{-lOWy*^j%@J!!kO>cJ zdp+vTp|U18&WnkK;{5c&c5GU>W4LPJUm8sb`0K-;Z^ErIRsqE(Bw-^hM-V5` zTx$STrh9pe9p$++gLivNJ||*1UdCEnQ4m-R(F)@sx}{1+Xzd`5l?A!P4EVOu@@%*Gc1HXZOC?4fBVMedt5FT6|s|4=? z0t4!9c+v_bof#N6AR)m+moB4snu9Pmm^Kp+B=!;BssOf(;72PF;RJpkixyn(|FlQr zeF?~TBW~O9#Y+HJ3tsrT7UE|)0?y<9SaKh!_usScJH> zD6hjWLJ*q@qGX53#En%;mdOEBu-LYJJ1-HG)5m8M;XB^q^bc>gek!U4%|z{pf`z#O zMykT5{kVWcbVtaqMVSGV0m-0pc@7`0Srl+ABRK@2?rZ`gR5uV3Bg2myK+YTo&fL(i z7%JfrI+rRaiyI9wbri%}kRx$%L4&)AR&B>x96j?1&SHZX%g!p9oB!EmNF$Gkk6(Jg zm$#(y5P00E$f;Adfu>hYukl@&834 z$vG3fNt?_rE=nWmd1PN)zEAKy{2lUPAyOXDCL+(C>m<_)NDlKxG}aW#S!Oo2fvcIn zKx6q6$u7ti$&GfHf=klR(KK`&x`oW=&}z1U90H86k!lzRVEK`b7fukJS$MC*ydTn% z8tn$W4RuCS)jGXJ> z_W&jC78PjV+|9^no(BRM&ota)ek6*CbTdMO`CXWbsTx-vSLHye+!945}DT2uL&B8GsLnFy&>moTQFmLLR6QsK#uE@ckka%_btIu@sv>2?xEKF5_NOqlV?u# zQ7l}-+HmX7IXAb%^16>l)U}JniB>y2)kf42H-h5H?j7yS7t_6&qP(k`(U#|{(fk7w z{5|B{sQnRqtH8|6b>exXVW{dNoos1E^y!`RUXH^oINu2BB)e?32XSD7diD>5!8KnBEP8HRNvNeM;Nf zg|8;!NL8KO9TCq5>%>k$qBDJY>GU(3u4#xW{tFfW0Y+lBrh zsgTxFH142yDH=V-dn7pS{N-Cet=6c^h@UE~L{~g{!FflR3poUGv9oG*X$QYX~BVO17#z8jp|kSGKe3uO)w5gn)p-%`PvM%Yc|KtC$Xw@fz!5t{BvN=|hM#B%B;F!i&d01Zsx>2z9+W z19TY3*YfhEBr3Ji2iq_jHJ3)DJ#WXv-8*|D%x}yEjJ)8ya~r)APE9UCX4U@3>dS_ zEL-!F2n|VoEByQRwww66*f^*-ASMyOhzg?3C#!iJ{Gg*(lgX$Sp`0Bn0W6H7bRkkj zcU!_&6Apmo#S*G^eP@AjFb$302P6>CWv0;oCb$j{wt6O`xt^Jr10@9`@< zk3K5CO;ai^fYaElWK49Uzm3}#4MVB8(`ci*gV)rf={6N16qx72&wnx%9+s@@3MVO^ zED_iYCH*7d_hPHs3 zYFsjtTne-cIcl`rV_uDv25@7Tp!JmdifIgV6@=a^7JHrnaBUIHN4xXR+eP-Dg5T>A z`JFeCVJXaySD;de!!P2W(}`aL5`j>Xg`Q&=m7x8SiW|3YPXe(J54_Ud7COCY`2D}d zCJSsk+`5=3c7umR9LQ=anDCez)51f#-x)3`2nGEU^x6qeEEwoS;}PBzLq*S^l1Zgv z8ao*kHmDy_QG73s`i$7ebRfqlKL2Kky8$5~Nr0FdO$jZ{sW5odjNT7&NLi4ksZ^LV z(?wrfMWDq?Rf~z@hU6X>`pT}6Jp_B-OnDNNQ=I_b`0a1k0LxAOLHB7yuLupxr%k#B zq;qz~y{a}_JQ9>s|DfCWNEOHLH<)3OqPvUv-;tOF$m$Su(D-iai2@=H*s&v`>5BqB zdp=yNfp!!*?EoO7T1zOLtXwTXSbJ`m_7;sw3q7*9&P%pn3rPMlbr zJ#oz179$d9#k~$4@)N@7OAT%TvGCZ-);DFz?5TMyO!xr`<_XmtwaN!N<}<&$zQ|CtORj(U#*< z{Y5Z7pprG1&UkGnMp|=(713hl(ixg^t6fo_s{DFm#OR>wa7o%_JWkPNSdy(lY;SBS zJnKf|nQ4gB1n&4LFk$68=Za6GXvDW?Pja>DJ#Ji@t?%sT?_y%!5P3j&z~M?Cr*sM( zk!@ z5$Bu21#Z$sHb6xamO>TtZnNx?`ZF_&wiba_TVmX1!_RW+xNhg}-PZRGbr%#w8rx`t zVL|&9WQBtnQ6aJkKF8Me^oW91j%Gbfp=IS;o&`$ErQmli#40Xj)e^@~Bbe3EaoU0^ z8lg4Z8AV(`vpd4-$@|;Z=r7M86kO8PrSh_KoM|w)uWOC;L;~*9SAIEeopgGy zfINJJ`(gbxKUALxOHGhcvX_*S_3niA=M?{uYBP!6q{_$s=xHFODk_+A)Ka21r-Rot zhUSgoiib*aFA+a@$n`E_uhdKtlBt1+F5=lwKj$*n*p$*#|6 z8~9_~$f{;dngmBUo{&!90v`F?`AccEol4R*ybO8B((^vILRBHKtmSvWq3VQ>FC&v} zLszq=Qy6e#u-A$pfEUm1qASSyynA91+mcc(DAnWMy_cyTgX^p9v)VddEw^(YxisM0>=BvudI4u&UF%(!rd=TPa|Q5XA-aQrL}+&uD0jNpGo3m5Kek2ai2^ zO&74M!<>2=bLj;rO>)pSA5QyJd^|7hS}5AW51U=BS79W<)mk0J2Y1cEuOsKXl#avK zXX^7~DnbCmQt3pb3WiPf(b~h`3|vp?$R>~qx!k5ztDlt85g_`%O2-0FU{;%bGc=7^i)wcoI2H`b_E3B-HWa+4;DGAa4eK9Sr>zG-{Mp| z4*;njs?)bb(FO3AMFUg47`P8AXD4Zq9HzCu9I}iFXy<)NV|pJC7*9 zGcCU-3Kg-qQ**pUP<+~PjC)2Bok!Ay$iiU`CZhyLQ(n=D$iw)c1Whmf>kpY@}7Ft3t+n&T13l|%Tj2NF68tLIkDcNVj_y*MooQ+zZ23&-A+BT=S}{8rx|P`suntCPI{JzDr` zxb`?W`*>)!C11)q;xHpt_5iEz>PUZ+R$b@Vdw*8BHk!G`7c~NBTnlOQxMTCM6|Ma@ z5P&^WISb0@Wy#}uZiP?;M~tsR$FvC!44lU6l!CS>h)sEp*w`)xFO9w}Fb40m-ld&Q>6 zhL?Y)IW>x&zl-DZE~ip$rMOzu3B)`=97ScN9?04(WLtD~(V!59Uoc<5tr=vCYllAc zO^gXv6QNZ34l0^jPsirv_1}6Ia;icqY!GDvhrCS@J*5aG#0$(GFiBcRpeZU5t9r=z zIAJQr$AV4{kng`FGg|Q9IoN#@Z&np!%9S$iD-KGH=Y2xY1eN#ZBPa2(ZyyYH7QDo?7uj!?~u+7 z12+!L@_ZkBWg(@(p)074Rj#THb#+0nnLlh-c)5H3{JRj`=n6(K2SWH~+jU4pUl26k z;-7k|^0TZ>jg!uyL2aJv{p=T`$2yM{i!_lL>{HzBPA%5TbUY ztv(lTXTfubJT@KYTsb;*0Ux7_0hMp1xBjE|4N~s2R>Gk;WIJJdL)Pv+BAMA^BV?a$ z(4hUaJvHLsKRdIdJbcQ4+ZY;uI2v#A*$g!o39^DrOGgI_bm7#gQ>s2%K?ivx0~Ur& z29YeWe$*NvJzg=4KU$ z2<|+k&Hwa|@&>I=+*mZf*lZvpmh9%q z{Rr$UFotMO0S%q)so~i6A5qHm2tBIRmt|r?Qz1J7&P|&lH26s77KuQEW1)+QrRDjs zu&}k^vK01Z!t?T$p;9%kLiZ!NG5*FecRGqgXVkj2LSE+}{jW6y%n?~Hhg^gV4;{hS zFP-Ri0YQk3U<$(jZ5`p>0F>5$wjD7ae|)qVJCIvX)r{*HwSUPD+c@OhvyI>R#a;h> z-Ua^C^Thn0#)@-69Wthl)$m>&zGEg3g4KD;`gx)9`0>$l5=q-n+m|{+YTxrdL5h z{}+7lFCc~rOt-xx*3jNxg^aN4*qTK58Vj+ksdv{zI3DBFk z#pcw3`1aOtUI^4ddG&XdLuLzdr>vpyL}bbaUiD4_9q9eb7wW_nHV?Vi`DaA*7C(} zgB%U`L9UHoJLF@fRl(Zgi2m@6NL>GNe~=eWjQI3x4Mwj@A@2wd$SU~c z1SfCne*B<=OlalMd+@**v9|y4m;mvp3(V{AERu^B3w#3Y*gfg@|C>BT9 z8P5sj?|%@B3!2K3#g531d2UUulM)2z(01o5GYm~GqeNt!`B!xD2-E-p3) z8{C!T*IV+FsHH7~Dfu`f_i4Qi{K}-eD(9Lbo}O!{)DVvjQ5Hmtc)&5}LFbAgh}SQG zpLa0&MQANiht%2@+)R<7AQhJ(Jg@0_e#P`?mPpC5 zZ!5}-xV~g;AcVP`BK|NO2@2qxpD*w%0uyV*H%2pNc&6q?mp!dhO(D?z3dASgCM^%s z@c{Y1ty)-emyTq3d)_*Srl^kdt?U2h;qjApv;1ssjkx5}OG3lA?_}D2g-ej~*{#K% zJ`c0(B;iLO)D6IE{xq8(Fq&5UhS zqLOBW{^NoEjeS{mk(E}ug50P)2k}-DRQ70G-mL(tTJS}RUD<8Gz72m6+RpfQ0N;e) zhZvwuFS*%O1Yqm|masDr93wp;<#d}%DK@igQ-vB@?H+FedwQ-!VXg%EpfIoUit2seK19E0Q?GLi8f6dNG$8aZ&V(1&kIX+sD$Na#JHs5f z0%(zyVrf^YQzq{kZ8GYK|AQVQ4{O96p35I!bQcIgeg!6>vg(PpncU2v({J@!gk@ac zU?BcNfILbt0!!Eqwh*NvbTb1_Uy15oER&vhUJw2qIKXckrbfwHtIl8F0nn5y2*e!9 z$8wAe0NkZuM6mbg2L}!2?>>9BfI~5B30{kZ5+U#hhv)r!_a4dl zFO*&OiWMu0sJequF=))%B?@zXvN5_sFJM@XZ}H=1jckoX7|r?#Zs$2_Xkn{l;l*bS zJ=AHi3Kh?>VgK4>yD0t2!N4WTUJ{ntU&GjGiSKO3c}=d?jC7pjwok3jNsgO7 zX6tSL=~W$e7oL$_wVpyb$fVx+FzUe3HS7{Xijkc{wQw zWMfF$WCeqwkPE=f^}P^RnTd!CsEy^ol~G#UV6U67Leo%-CROA`hw{)jWKyESX4-5+ z;fPB4_eQ+H>Slk%#-`mBFRh2FMJ%8!fS$AsB-MT#)fJzHQTaJ{P4Z5F~Qsx|e}Zo;;!6d@vf@4a@HwU1)iNWU+wp!lzn?>kSrs z{Xeu-pcw?}2=o$)7&Se+cx#lRkp@l?C{i}}Lvr?}NYnvU>qooeb`~pF0@qKJ-3#^- zt5>mw8vWyOZf)RB&{Bb9(n^}fhY`Z!z;P45Y-r_0w$=#iUqM~7J{wFfGX6Pn|P#6QgJ$4te3Ftd+FP*l~mS7uIQrcUYN^M`DnU5?%XP3N`O z0sZ_5Uy56vLDwic-uWvyC!x6aDGM8==`c}$IMtBY0 z6qo*}kejsa{|M^I*vu?vRE-oT0!ShQc)R}6l$&RT6yEq0WGs~ zv{29&B~2gn+=>&IAd?Y^*LXI@Jpg8CO4P`7LegREM%bpmoS|L1A##y|?EZ|UO+YEG z)|iqtgvbX@=X(AhN8Bf;viMhxlqn(F4T{>I6JYYNYQ)_$Ct8X9VD+~a0#B zxq2l9R#Q?m1fA5MNwCsJ;L8CEF6pz|Ln7Km<|6mgYThYOm@96z3ENG*6F}!YR+7vF zp4JvTQn>q+wt}@g>UN>1idT2=+Xfo(f)Fm*W>5ge+D_nD3cXz(Bq>NpgE~yx6L*$< zT-a_&gZMDWrEGsjR9Vf>0CW=bDVv1Q9bz@=-9U$BH)=R$D*C(f1le7L-iCZyy5d^S zf0NsRH$Vz~oQfAp;R*CLJ2InhogPcB4Xeb++Fjxp0%rQ5)rmfzHiw#ffVrM!fB%4j zLP>Fm6}gn+$)2{S>g65e<(A|N=_YF!55-`YSGHeS6!@eb34K*P@OhNuwOKy8me5!0 z<<%2wmldp#=y#M?Pdk^R25u)T1!DCgJ5WcXofUFi*lgp4l+DGmjJ6Rksvc@h zDr(uLO|RW|(hJ65utf+ud!NamS>3C8-wWz&Gq$n%&FOYquR>#9eAo84yX*(X3+qS= zwdtPZxmWfqQ%H+7rsEcLpB2B6!hv5wC!y*nv+m?|dqATtjNFcyXd=4uZ0GXaZNh<7 z`jTQUXo|Wu{v_VcN$Bp*h~|fzz=1)DDxw+jCxJEIBj9zkWr7UfO&dLG)X{f8^J5P3 z)$^>uhz`3y`G>)f6CHDajVgdf)AQz85u)(kQeMt(FuHS)U|b^I6n=WRQ~5^0yYepG z&%lfDiHuNRr{ZI%hR@7wEB%y<)*(U}`i%R_py~%umEQu9ewXoR! zH$g4wRdC--WWA9%$J-{kBr_SXt5DbEX32gFOh$f%--WQmvm7jl?>ZQ1R!-#xjX;kc zowpyT>k0I=>=h4}Ru{C3wy0>)dF*K@GNrb6%|P*Dz&E*;f;bRr?0Us#lkH;!q4>41 zpkO1>E3#3nN-UjeXo$+nEpz?o?*K@HIq^FhX=ZLH+vag)ibaExE`9s;d;q6;SLbW# z)q6z*D88e2rrkL^TpcK{c+g=k1%@m~}Rlck@Or-%omRDrdT$EA*B225* zDIK610W`&_aL}dF02%Atl`G#mr7k7Qi^ZBu^>vZ;dY+=v)zWS1VM02n&)=bu*Z{0|73@h(wrMSdxzw%D=-3X%A#NNb_j*Z`CS%Y=kxN+O4`1$!kw zHi=z9LAZR8eAY3rwY$V)gtFpoBOUN^3J**|Mt~I(u>Fl(b-a?bHhm41pkr|5&e`@D&>OAcrGd5v4?&_gfIV7t&*kp2BGO_290(4|S;uroR6;sssV8C4F5% znS-jGN__=Rc^a1&E=Je0Kx^KOD1L&XaPncN)E5u?NZa&n8FxCa0HA@5l5_tTd7Tl%f&iGjc%I}YRXG|Pk=*|j7%a}_PY$JXl%8U7q1)BJ5!YP` zN|ZyjWJbztZukfqT>-~=M@x>Euu}n1i|tM_H969?=CZZ=-H=n8lOjaHAdq!S)B6-p z@S4#LQH!>&)5SpcL~O+)AgOfTjZjiQ?~MpbrFj+&!*q)5uoS5hx0Tt|Y9MtN$TmUo zVjeX* z8Moy41I51>6QJ|eq3G3_PK(zk|JgZkt;D3>oIXb9Yks9DJ@YS>{wyH2?x}k%&vm$Q z;_M2R!qaH@j5PEuIxWGB!6d;_#e$pdANm-cha$(QCWu8NnwxaMZ;o5y$wS4*6( zehRatZ98VaeM@Lqth&RKm7JHNSArwfJAeB0U$(cYo|Th=vM&DhvL++DNKlW6`FUzl zg%8IGvec$S2Qww8=Ww6d%5$l8j9j*JIatq45^ANUvgaBa#WD1zXUE zNO}YKITvp{o0^i>RF0A*&Sk6Mx=KT6dLHwjzvA8%VYY5p%qFgVdUL7>)h)=kLUr@J zK&9FaT5{pvvN9lW@y!USeZiGpUQndEfX4eJ(d`ALOsj}3>Yv| z%miZFGLG3HIY9J_VJX6rBRy6cIJLdRAOY`&r${S=aKvD5{w2v$=qUaX__x)6cJrfI z&e|NJQ=6tRsWx8sMK9MlV2j{Y?ZP&VKj@UEziQL?(8=)wSN03tIldd$9&;yiqb|Q^ z4!OElr)JqQzF^t8O+lt&d!~vOrR?q&sSv}J?KRGs;6(UasW+A=Zo;{ZDCo{o@|=9C?oAHL_p1!=4_uDx-?ZXH{fG6?{i}V zs3ejFutf*`JV*KhDgr<%iRCGdJTG~wM)h9$5@{qZ6bZsm5^M>xDbZG2(gR6~GmvaV zIl4$sfu?{F^%D-`<&+g)9 zTvVbFwCBL2gXkm4PF^v#v1`NYZ>mdtQx8n4z9s9w#h;zo&1DI=Hy+YiLlty&$A@9;+&oL9qFh?DGn%THTU<~1h!!s+Y&ss-2Y&e>9?tckUUeY$ z3uDM`A$THlrS3&n_8vvhe1M*++t=jy4PR(t$ z{--CC>Hh>e6oor;x1!6~&@F{6$M@y7(QE#5gEIuu`1jqRhG+j5X~#Z1n*_4$u7Q`?{9xH6PHo?|li^em*`~L2Z;X;w!1M4*o16SLQRGX}9=U9~ROaC)%J%6w5B&_~bLtgNYKOfj~f~jUqz@Ek%0^Ncu z?>tcs3jM!x7=%X1`1{TJe^H%?(GIY(JyCA13n||^z-NT~O!dkp7B{0C74|AQHbI$E z#rrLKBve|qKid#BEHWBGBn8vU8aZ>v7zlIq%9W@8qkn1>yWH`ctEd*}QjC~`I;g}i zT8Lwi0a=l>sct(1@3XmsvY{LccBy>RRWdSaB`>smPZ2IF25HFEm&sFqx&QFtQ>0J= z4MxASf<6!>o#2O;E)6zl)Gt&b*RT8exoT~ag`j!@$3W-u1ZKU;JF4Q&volC zQRR|`bPxj&WJ=IGE$5(;E{Rkr(y)HsPlW$3qdPT6n`EQ}AdJH|j;uB_wSNcwzwgJ# z7pCnukri-kGHoqvIwe9_$~9VFBs1is=q{j0Df+X?pl6ckkLy*^x>mD8?fVD)QdLPt zKRO^&iHw(^euMM*+Hut-V$K8hS zthLluj_gm0(7cU#>96t*-C&CshriNGdcBNtoR9WLDcS*3(@Ru2PyuT|`@Bib$ngIm zP`6X_-L+hn?iP>4UEKPEejl^bpFA<1FyDU?>J&Nsgi?eWQofUFp;o-vTH?k4jye@sh(2mb<2t$5T-}Ze zSXAkMO#a4`yQJN-=U=ANrGeCHg9SGFO5gY;cj~LRoOr1HB#8R(^@h%VaXMd&f(K^5 z->6kFfva{c7~SQT90xI7KeX&zE`@+z=Pfnz6;;f209z4tD7#g4#p|OAl?}vTUzd%{1wbKhhsZyWW#Iyr^6UL7D8w~*xT)ykJXWw>QhF-AU>hP zsjD49;ZdjS$s?oH$bZwQv7HR1cD$CB~fpQa#KPmjzl^i8c23FD~@NXU-&v z^W++{X~fUaryxH!UJ^;!zdvmmWhe+8V#&CuW!+S9BTIgDcQ39O*2wrrd|8jcj%>o~ zq!FxX6fiCi!urdmqI8}vpdsSnnW4>-v-&7B7k0k2;k@lIjXqWDOw?=E1abhclhmza zjdr-&jqaq#DeIyOlp|woo+M1LHH~6ozZo= zhZ{KWtls}Kv9qdl7%qzMTDM&}mcEW(hotSNCr%Fi&6paH+jW|P&btvO!$fo>^T5n) z!o|udsxm@jUE^}t%tb#Lj$r*H#rMk*bRlGuJ`&JIqE&&CG-mCMMKQ5BTGLI6Xhj#> zrY+Iw3dY?jVs{_Ha3*8;eT0)dzRC&uQ5sOrpOn0o`hG)Y%n5zT*@F|xr_tbi8F%4p zH1N+TODx*(s{QQ#5Dq$`I1TtRIZh{Hx%KIxd zbiTG=hLpIW?Ebz-e?M{J0%j5duVd9iiTFr+c{ICP?Zp`I`AYXKDPYn@wrh91Q3G9l zJ`zcYc_kMzr41G_gOrFx;`Xbn(@mMwcQs?7U*K4!TMz&RHJzlb)p@ah(r#3*`+tIP zVyZgLw$F`ys|$vxyIikl_6LU!72#a6Yt3ClHCGs0U1|Af(FtZOu4?TEKJxVi0C7Z>n)(Bzp~K;p#+qU ze^TaSk^IsBE6|ce>+*dHt+y536v#r3lkA6}diytW8f?+^QT6%J$lshkb?u{A#g^^OlyXb}QcluvCk#FTXrbJRU`ek7QK%_^i`zudec{ zDr6?W3hhSu*>x=e|N2UbsQalxl1_K=f;~AQ!`p)q6IyeXmAC0l945(R7n;xKRuYOV zGBWLNeRfJKBBIWZCquf6EAb6I$tB;=4*eW|^>D1OpJn&;qB&8F$K5sjwpV(d2o2c$ z3ao%%Ub|}6$<0@}nC-sd|IhDMDu>ZWP8V-7m>DZyRjF1nNo|d&&KjO@(Cs2YPLdWP zyurG)tioC@b`4ytwbOxgwl~@l^tnVcj}>6{c~cr{0*N=^O_zC6ZaZTRr<^uyE&HYP zh)Wz^Hvj?1*!)RKSCOQUrzZ_OOhLWhzw=s5l!f&2^f~2K<2%s@;7!DFvcb98)BI(% zn&>ef#Uv8g6xg;1#wmxezuUIMe3{zC(s_@nKEP5A%PtaDOQ3}imnc_bZL86zA)=Wu z6WlqlM;)aZ?bMx24UxskG1p5o=vr8+W&FsR20G&)yxEA#$7gSbG@{^0kgmYT@$(po zT`|3Bm$grza7AJsbdWYfd&rQ-rg(dliyY>b{})oDscc`DY87u(gp%b;4wG_bYqbga zI3n)Ch5m%Qf?Q`tcaDOv^xVdKZ+yw7Z`s(bN1f8R4A-9j7K1zXEZgm!(tIT0ML|2? zh3{zd?So#8_!|HL0n|cK#V@S?w=dKZs(AR``ryX!(da8~je@t8ukd4CWttnBnEgwG3Wyy?40OD z>Mn}jK#BjQ$Z#$MyOmWWFK^hY>6(q)M3eFrAIS&y>!{5x8A>`$R+c4Qfuh8vBwKuY z+p)Z`5ij#nuPZ{zc0h=tTx6fn@-S(!Dzsr+vFb@z?57J9D}r5dO{9MRb~C_C3eEa? zKDb-rSN?`We?DPJ)MuZ%S};AzoLyHRl}NsY%us?c|7tzK3|#^>K`vruO~yZ{BOkYS-=FId_-0I{_e*V%MZA4jHn(|22U^521GD1g62YT%e^293q`D zXZLd};;NQw4kim_C-@(VfDslCJBhk%GedVbFQzIGr?8L*lHfqyRPV%hZt7w)Lv92J z!D_`TnPSZGueiCSrA_kF0YpocE{-6l$XC@38`A2~A1xGK5V)pU&-(RZF)k690P&a; zW!;KD>TC|-ydje0D)AhGHzWTt#y02L`p#+TsNpmC7N8Zs*$~Fv?EAs`|0x5z#EopN zlb5H%eNerVpCxWxto(6RT?D|!w<#r{m{DN)nx5w6*Wc8V0;O*2VCV9U1T#XDK2Xvl z1+A+GJg{4wE24ZN`Rj5Dok$ToNe+aAEQZ-^M37xz1Z9KNCSveH2yHj|qm5g{f08sW z^TNmsg(It8DbhFYYgyTFM~>vuD|{5$-=ZxJp^Up<^2+M8B6>vOD2_}xXNHE=B~Y>t!F+IMCAB=wa&KjSmgEaQ%dLbqdE80X8QY_Hv=&p!%*I`v0~3 z{7mCLcg%I`+lSDm#PDZ%#B15VA|zr!q$xUP#H2>8UcotZ zE6;xrHSJ@`K^9jGC%fsiZ%Xb2UqRfH61p_we*uN4d}I%(mGcK0ft^aw6^}QEV&1tB z96|_Y$=DFEG+#sGj+Ao_`OB)_bghx8Q8LA%;^Ise5=hl)5UT}z0%I#HE7jpYce0fy z`06bsHYxvjMU>kMcJV6j%*@0l|0RRY?^dToKx;c3FduDl^avVp<6k}>yrI_binM%`XZ@HD2PRb=I$=RUZ@}!ebII+0sXhMht~dIQ z=bOKC=h)6U-*>j+FBn^$JY~=(=dGO_c5Do8tCV@1aW@1)kE*F1}7e~R~4t5z2@$g;d>A`;x-$avW zDp9?ZO%DWU4m^d?Y3&;xmVu$#q%!@GEiFA773V@p?2rZ%_&yF-VNgc)3>tWkr#>N&t=jG$N>E{6ijgHIp(La%udq&YG^v@kM(YIBdU>(qtG=vCfF}KGG1W0=G)x&XbFK zeM#@9^@J$`z-Ejs>OX~E)$er#mjGKNUD0SVxI?{oxghZYSrOWJw%d^ zIVNl5geJ|JrO@k(ZS|L5F2}wxl*4#P;uE=Cld1Gx(pHLbOR|QDQlKPS>K6p^SbMnsLte@z`^AQNp-DKS)Vc621Wvo-v7ZB}lI!Sl1X zsWjj$XDg#?mDvRZXY@0@pvD{lTU13$z%AF2(DjVVD;lN_d+Dk0`F4Mu9IXNXC^mAR z+r((bRC&Pl+*?29ch^3FZvCch2;V*6M9T5T6{~%ds4ADX>>7;6r?@)-NWOQ zSUiCF>m%}2_dkJEM zBtANlz;~gqXbnalx={ZAoCV^t3_xTmDt$HEd;DU1#65yD@~+KYw{G1sO|~pQGh`5{ z)TQxJOMD@*xkXk~aRpl~&($YeIEx86N24pJFxfl*IO*eiur35+xYoRnoCUOZk16b~UN{p92qCCqsqoK7^~K^Oxz|BSruwAE>JSk7U%!a}Ab_S4_Kt5lt2SvAC6lXhu9$)x4-|G4BYN>u@Qac(~}^ zWt9X{hAJtG_?s}x@>x>E#0GHtoFL2H8PE_CuP(_$$iu7Nao{WKOtKkJD>v;~KWHF$ zH!&8J8Dv|>_WJPlDB0TqYq7e0}lGgvbaBLQy0E{G-O|Z+k zl!V7zDYD#H0F)#}ES0})KT|Nus?Df!RJAaTIB$!aTHmBC7mhJN#;uv^ZOZYKbYlc+ zv69v&lW(j99%t+v$fPpr!;3Z{%@f!;c+-cZDQpxYT7RSQE$uHL-m}_Brf!G+*Kzwe zZb|S1Qj|Lxb<;1C{EfhwLx&pf<+Qm7vh_5fdtAL1w}bijbsEA9qf^t zbPGQ#SPw*_$a%v0`!x8|ROYEPJECWE@g z7XzN0A~{YYmc^)~KT$GfUtg;kz$853@&q~r%j?&x7wgnZjh;o&D3l203F1?wdywWo zYyJ9Llc)T*eJ;Nz;32m=he+EbirtT!eHVC0xi0bk6c94m?bed!;S_N(zJ+3&7oHLw zN(RkojSG1i3cY+sCBFAl8(E6lQd;=%|YLvChDItYx&{t9|tu)RnOnxbf1kLhn85pD~ZV)lYQk%!^?** zc$VZozmYoDZp)LEtB+P_=|Ia*;sttP)oXWky*7)1R>YYTqwjbyo%X(bTXhrd7E%Bx zpe#^K5}UA6rD?~Cz6#X$Z)IW?iCfKjO-8_YBj_P@S+crqA7CWwPx%i~m!{KCR{lrU?&JtB!o-3JQEB3T-b%{+(oXyVmq+jT*Q&1+0 zW#TU_^){bP4143H6_uQbD5NT-Gi~4JM?*=`X6OdRKbdwv#5(iyhZDsxHNf1{oxw?I zV~NYk4xc8M-e!M%)?5xSsf;8eMnglxeHd7ipk1<;QT|=r%$byFE7&dVkOXtZ4!#1w zDZ{dW$LD~s`0wOz7|H-i;%fR09{fl$bc=-%gl#o0G(l zijdt>ZE94kO))PGjQ?=$nbp`;c2j?zPQL4(wiY@DGZhN2k zsYRY0SMFC3&|0({>c%f(8^mud7@`Vn)!(r5KimJsDre6T4ajWY&ko-PfmMsnK$5ML zHDN0;uHCOwp0d#o6PK>83`73PunSAX)MFisFC=P!1d5DB(H;m*fhCU>aq0s}kTxd? z+<~e_qsb@u(z9pJXHQvo6LuT<*@+=f$zVVn!Le`uaVnT88Cg=S3XjhTbOf<|RQ{h$ zqza~Py`&fQqaDPTO~&oTk`yPPuJ@V_1w4E?(n(Zbzo>-zC5a**iJmslZhNBzMV_5* z_t?AqGWS5gSF7*84`iduk-`KRt+RnauMYr9Vxo~i3K3~50dU3=07uXLlzZCn<`=Qb@cEQrA9!uuK6tImj>3c~UT$uyLK1hPGe@YQHO5j08|HhiJ? za&`mH*8E6O_K0|aI{)kCU(G73s${;CuohzW4Jgyhx4_4KkpfZdy^l$s!Vs!Wv=$)@ z5gihWgN|UiutGa>FzVbh{-uBmy!cx5G5K4SvHVvw;aW zNvM(p1oyQNt-D|y@^b(9amZHQun9D#~Il-Ft+TNEknQhane4*D87``xM@>r z&`v79xzeT+#IQ1$o4r57kr;7sydq8`Ypz^5A(lkt6fRn}%uf=2 zWjX@A%ckK@TGc@)7oI&utIF)YA&RhhJ6dI)LPUoh^MiJwED|IUpld$G+KOmuPniKJ ziIVhU9jAShsaIWiWh*?X63G_rQll5;PJm+@`B%j3`0(xFxjqh^VZKQ;XVkf3rfA&w z$IIZ!}dyo7M-E_f711OZA|((9BEW9|~=$xzRx+BFP7Uz_W3 z$Js4z4Y52{q(_Am=f5@(u{`DodC|*@Kjk{K%al}a;7rADIXDkEFO^h+wS1-^OXznf zI;yWqbtm(=i z*yR!(*>>O}nJhbImw)9_y znlw60OKX#hK1%UCs64$`aH47xaTL)9ohvxT*s~6&yVN-GSq>~e#jPogho%bvD-ELj zagWlwY0DQM>}O^+i>0%Hr6AFqb|0XRsKK+$?=Qd2iU5UJ6^mtnOyHn70lPw+i*f*> zK3yZW$xaFxA>5F(hg8d48Jc;73Q&KCAwbyao>@6NJ47?4ap6eeK6B3eor$bW1JeS1Wl-e%v#QTA= z1$drXH^RVY+czqm%j?2R$8cV=rI{>Sctof^vb9n#%U~I~n{{BMdMi?zobP*w@k(sb z?IK7-pcw!%=FJe%2+&SHqQ~FnBmOZsTOp`r^y1^MElXvd2E|uD^H1vD@)3k7WsLTlDXo`Kmptl*UD) zpe7FlcFqYcf2~oATdQ1${`NR{mfhfBjqBB`CzG+wKT!`!bfOT` z#?4qKiWi_V-+&R%Q(APL@O8^TTx^09{5y*;JAY7Sh6I1xkVc}(IM6hFT1fNGz*w^q zuB7l#S7|phGJsGLW`aZaS+-u9sB&PU=bfD~NAfIos;Ym*MJ*B&+o+-=KOXqrA3$rU55T@?g8m>V)qei6;c@*{b($d--ea_jpdA510q|R4jd^ib2 z^-IC=Ld-o3Qa?w3PUNkcB>j&6WbHwsPBJIVrqJ227g-t7poa^pZ>jc{ zO)S-xqMw`|opuLPgFmHS@$}9c*|~v!uP=i$Lio?Ouf0W;sM9|Ccs#$Wd^!N)gsc`y zYT4ve@9yr)-^!Qi3UVW~ zwoFCL!U_4ga1jrVu_R>I$$&V1L1l<)7FZ&}3j zP>~XN3udt&dcn)STy94Eh){Z+se{m-MbfWZZy65L#h7MBX<7siwF-Tw*%Pcb>R#! z3fY=)MKHNDXuky6>ao{Oui37(JG-`IHVp^U>V>e2jk{CIBl9Jjj7KxF_JbVg}p zA2~Z^az`bVbH(m^NOd0r%AomvA;{RjNfCYp5_gCYMU%+ve8g7r(6iiuvmf%PKl*A-9X=KzTF;4<5@KD$Mb$U^a@^l zR*PUk?Fg zPgnKo-u;gVc94sTyI`WF()LYXl2$*zWO01|C%-Z6bA>AKwEN5WQ{DR#*!{Ghscwk z2H>T*PI#& zhnm|c9gOai9KlbbX}X-kSr$sCY;a!PMFL|YZX@J-iS=T*b`#kx7FQH)jS|%7kIC8t zdW9}Nax0=yX5RgQ1#4oPZM{}cd-1q+EBoHOw?cWo`Nog7h-Xq;Gi1f~?>gU$wN;rI zZ1kHR@{{mW%C9Va?w=rl$iC8Yz`yY7hYLd0n=XC!bBYq1K@Hlmc=3lxE{S>rVIa!j z22f$ys3?$1byKnDNK+Mx%E`BRq?VC62-3g7V>te}UwtlM%UAN z7k~Tw{K{0JG^lb1;N4d;Rdn_-PF^xKM0>MnpyR=toP0702Z{B?xh0FJu)cvL3#0-V z?Q=$aFLF3s=&v>Slob(MS3>Z5k_h*Ji7R(@x0uf|vMGYovEoS=IDGH7x8MvSR3$N> zm?B9efl54^rw@NSm&=PVV<)VQ`&Tbf)`Doj0gV!NPCWJvun1ex+VEO9&|4zna&ZJ0 z7`pCDd{blit@+A73WBo2yHFV%U@5h&-(u%o6*0 zv2$-dt!Q~a5VH1R4;FO~FaO%2%F1m*{E22>*%HRHyQ?{%l>m%a(JsmqaUDMaF>n(} z7VcNx1hPEuWk*s03!g7!BBx)^{T0dD4UL@J-ok4@d_=91T_5C&%6L`qY0F1$ND&x=KD)2FV07im%wnD z?=p;Ef{RG(`5^FPwGtaPXdq+mr1z6HtJhx{5I8@XXM+dT%GPcbl$6XzBT;m>qpbw{ zQmERWkPsl?;IAYQj@IfRcc|tlz%f5o)H8qo_1EZPioS|xjyIqqlsQ+J#@#J6Xz|Vw zBBCP^1w&&_co=M(-n^_qPzSa-iMaZ*IFBTGvM|Ne_(oKHN_hnhvx)LWQ78D^1}Bkh zga|WwK~zJAd96nydy~z!F0oO&pQwB`DU;K*XR;fdJ#&V=gv5N~wNvx5KKvMwx5QqR zfA!V9fk}xa1-0*d^@yV%k^+QfIl zt0BuSU~(74h9mCSG7g0)?Cd7``Y|r4f;KW)7!cTX-)o+e^g>E2t>?T5xFb7~8)rqk zjYp)wxA`Tl8EYUDzzb}I5`es5SGcU~3RD8$3(_FPH4MmvuVg|xX46Irn zuYJkk3lAwQSYr&rs*1VB5hLa#juN;0_W`(6uIGM0Yk&;n zS_%hEyR=-YKyiJsReZi| z0hbiMD!#4E>yUQ|-IDRo6Q@t)97gMT0iYo8(xpccG)>fC=Ws&_;j5lhIZuXWY!Y0s zr(CbjK2*^%oNWGX=!6t|A<}D-~G)UCA_Rjk&_m zvrha@^Wm)|mWNRF6mo~;I6}|%YQ|u{1F(ZZG6%8Wn73#Y#Lfga8M&zV@g%$e&77L5 zKTBv4R~TVNQ!F`9r+R2fb+9i!x5x?=6bT4nDUdD|&$69VhP4bKJcsPT>JFaBTI;8c z9W>}*#T73#C*B*MTcMjTYUnOmcX{f#ixzu7FH*&M&rd4OT^Ki@xYo^Bx#D7m-YQotk{}jpD$S4k#VTnCLXMHfySMuCm9C+x3~v z_IL&uS!GvNp@0WMiV%LDp530!j+(_#f?C0O&hRE4-C&fNm#gqVgo`0+uiXYVc&?2o zVc9EA$sMs@NU(A$zbauwG)A$vspE+vl7L)}6i+98RU0#$5E5o1R6*#if_NIeOBn&lK4gc9EQZQn zd}JjwoWcXFV@z>WZKW!j%UV^ef33gu8=?K!FUmXZIfkBxAC(Ix*yeK=q6EAIUlxCi zq9!D3Md2t&YWdzs%k!ki9Y;+~&CYq&nkEZHc_vvR3AsgzNuR@t#LjIbzJfY%wKgK4UyLlc}^{!U_r#(x`)l{eCyRh7Vn#|X;Rm;`trZQ zL-g>IN^6e=|JCKq%s@tSG_y;KLDUmEXJcaY1qC#{Nq<5SWh3LlvUVaOpu`Io+b{Wu z(KXdK6Yaryx8lka-j&971~;hRo6|9R^Mg$Xz>_AMjuWJ5I6tt>RP;ny6obLaIB(6R zpz9L)DX@QBLc*qxsW?Yu7!)YvcP(2k>(<>hj+wb4XQOURmZ=~RLrc*892}4R;Pjx| zrIWgD%Xbr$q0qv3%RjaLG}+3D-gU1M*Cp+P7^2)ZI+y|UbqX)2&J!JmYXPGk@Fgvx zc9Us>ftLnbxWH)kz>Tag-;-A6k1Hko&3v{PBUzE`IDL-Br#s)kbB*#Qh5kgLe!2sS zMvFH>!YMIOM%+N52kx{H7m?DLj6G;G>xf-{H z0R^4D5rQ`?S_UoEt`B=&Kqd3PZI><=iH-U4s?BWR0ZIQ6cNR~tWy&P)mhwlTMryikrn>l=qp$79nqI*s1jgH<8Url0<9tdy3RHxun8 zg;&siOmW=}c4-eC8n$*vz1fIn1w{<`8L^nMH*i3yhm@9;ou?5K>M*V$JSv%3b7;Y2 zsdcV!b93AJ`j>W6oyohWKS|}akot)_G(eatR)&4k%<^ao&~U&wrDCLPp{6qVd1B2e zMh=OWngW+k8U9HWf*|?=@YL3f$*nqd`nYE3!y+CdW*;d9DSMTQk&HnK$jv9jLrn^d z#F2RPUuT6fL$KLyer zX>*djO7V)R+X|GJTPqOuHd=D$ZN+glC(&^N_n4zOe5%b>Wm+tAlHFLLGwC zX8h7V#!n9))X@KBnPJZ2K2Pd0wA30B&Ve6%W7E|TVo6ZKcU^m{^nbw^CId3eMH?g` zbAls#j;HVD<+UPRQ=(0}jQegZ*^JbdVmo_6G#vDR*M_;7AFS937hSB0I{W(Po0~`_ z+yNt@(%B{Fo%d?TckB&HQa8Q6Q}g~(VM>zoc&5!nB*t*GfeD?b&yprY4iFi{%vzg} zc;nSNS*Q}tGokElYrJIdz!r~lJf|2F9 z?vtEI`P2d>vxAiC+C%(1QRFG|D;7tX68+y;cnt4h7Ve3Op6oV)SVC2RST64e4^$`g z5Dm6_6o6JPK;P!3e`)FLpruP2i;M(e;Wipc?yd|$U0X!khv?yC#<$w@g&Wkkpgec9Fu12sA(gqEon5d-4BMQ0~WoB)KbsXn>z>^NFPAolsr60{9?h z$oB$7(7-cmcnxI_N~_f{`t_GYhk-|(Cg*dlq$F7G6=?~HZp_Ld`d2;V#LRw=yk=C7(@@`D zbiaQRhej>D9Q}9E+aKh)8yV!i@NZmnO?nEKUQaUShMM`;!OO0VFXHnRd27JsELL1N z2f)-}Y@~7H=xte^A|Hxh@C{a_K%q9-u^x)2}1AI^*+&`z4zLSa$BS{S25v5JW8* z8vE|YzuR`Oa!KqB`J)!ajmm2#6^u;35X&V9&etvP*jZ$_l{jfxrAIeR;;(sb=FCx4 zD8Oj3>D+obJn)^wb!HNly>2R{lKf6Y$n1mEJ9adS+6OH+=5%t3Ev z3HQ8g+NJ2}PB}Y|G5(>($an1W#>LqM>>vCpEs?gICcU3%^AiDCq-U-uwvmk1QVXmJ zA>zfgqD4-X;gCJ* zCH9jeeS=d+m8)H5WVTPTG{@0bqLto<^V`CuncrnI$CD4Zb>{vi_(r1>U(P`ONzWP*$g0b3vj zaTC1wUxkAi-;1~h%V5}VjIy4#Y-JgJEP`V`ITb8d4fd-&H#(Q}XIFDA*w^SEp1$iu zn-$xQ`*^xdr*pWmyxOYcokRR@e&v5f{9^$6xPEvOYWx&x*NrGCYF7m(mxZ+_vr?~; zzb0jX!&u^IH zBP=84mvf=c)nyhk*agDfy~@>9C55aY0njygC0(OIU1vxV^q;>p5ilA`ru4D}d!m>k z6yyQQ3BkOU&}gkHQ(fA|U4!9H^LAS=muj?wnvkvf ztH6yEcN3y*-@SV$W5<$Z8VSAPqrsx&sKJ~o_7^b_yO)gVCH21SKX`(d%NV`VXyk3d zqGs>bEygr@Q|=MLJXzwvPv-*OK1xS~AnGbW!Tv0t6!rz1|56{uoaY}%EKtZ1zm+TCBWSCpppH`IJ{hdIrPJAxP0L)5k+ zu|x`C9!2k@s88#X5?q!-D=0}gkm$c@B~Xd<>(ghd1Xl?gA(Mo%vdVOi58U@~qG zc;>C$muF*xj__VSdx``=v|6$gRBGfrR*Uulx{S;;xD(vCr zv8p!vcvkd&O1$2eyB#@!(pEHVi$3P$82caj8A+IwWt1VQH~F?T$9M+_Ul%@#sxRwV zC{wJ28^ixw^C^0y&RNkSmv2a&Ba1%M@|$nIQ3EJTM~uDT@9c^iOm@O;u`lUJ?>mix z!L=ZWo<}N3IEVMo;)zmV3k~N@*!7F1YS{wH44F7P`twx1ZKY~KLj_6brCA;2VcY%#~n4=|1A%@P3@;66Yj>dLWWd;(_R_kzw-Bx&nK;qlN{&o&s(zJ%z5Gz_in z9y~pJO1N<|BlG*4&mxw)?rz=)82s5&J~LZf*8C-xr&g@X{+&JF<^JuoF>!-{o8&rm zzg;YOP^vptzm(@58=dj%It{;DRF4y>4sY~HfgpXBa*|^TNc5ktZ_<*41+)Oqt^RW! zC>sDd<6egiuADz7Ig@&o`8%W&5Ptr$%n3rdEF-;4H*VbcykULUbJ~QAO6IAY2MBor zxoro>JLaX2cVe=jOqBVGeZDGdSE%fvRIu#V@2;!%F<^#xSL%;T$uP<(xz!`!km^0q<1GNLX#H++sk4^7Wr%yWbHvECd z|3}!HfaSQqYyVGJ#tfN-5RnXpqDY295{iUMGAEfbrBLQfA%$9oWO|eeDMLlpLS#rZ znOn;c6`4vh{Xf?ozx~_q`#+AiwoY#4tS59vY>-ELY4Ptj+h@w?X znjBl_@pTi}M}%zMis^&u(35z}M*pVwZ?9@(slIra|8M^?L0P#F6RX3)q>7P!g=VGV zM=VllS3^Tui?oY*d2WiSVj_`_Ydb@Hw*4jm!AjvSxSyEXQ@t$ThaNT5l{KL9>}Ah# z=S}%;{&dL?vJr$h!c|aUrWX?#bTw?+)SVGj@oQ;ZCH%4p{(k=QgcPHj=MJPtTSe~yZZayKj9;YYEY%$M?^l*+{Ff*kzY=J=uwGH@kJ){<*h?%1Cy3$qx__Qb|&hxkzXEls!2G`8IXlbTP|plClG4ofHF% z@J?hAa3YH@d<`s~uiuNAjFbbZ3qnAu#^5cWh@SkChMi)Gaq8E$(sdGfdHJ4ve__8n zzo8uF<#+xsM(63^q@CkGICZegdVH%(_J;LexYa37y*^C3@b+c-+cVPi1UFe2Ck0^I zN;kh<adNMELE2>DD0CqGodUtUaeSplJ6qW1+ z9!2VjN6PTjU>7zYGQ9vW9!*U-<)S=GRBC9oV~Et1YiNWQiA#-aY7tbCL33~IuRkUV z?_C&U-tQHcUz*#XcEl{u0I{&K@QYzp-~betON;7xJFL+2+y7FG0Oj+C8^~fqIh$LW z^%#t-_@~NRxjKr+d78BZMkc7pX6cya#C*HoGL5bW#HtIpTXq8x z@0G|ihqpS@bFkT6rOJ|(_&~a&(9ypu(zE1hcg6{ZEW>Wr_JxmH(d~|9L z0Ygoim~zBtWgBm6sQ;7VQ~+kt z9cI#+v;mZN%R9<|_9x670wPc2Y^0vtR z-B&;HaHEQnbwDLonoJZ$vnn(TUSP8`}l0UAOLmlpVjy|AGp6 zxsan~@dD5uFW%FU;6W@c!I{jWbiBNHhc1+?V)TVS>dU6;_Cxd2_vf(G9Hni^sk%%q z+kKxuu{>m~tG%#uWWOebJ!6$ApeueOVcndHdsEqF`SPlNyvq?nE4p#8$un6?-aI&5 z(f^qWLMB6`XxS{FIGgKo`9kPA3uuKBoM`%_Cr1$<8uG2hGRB!qu(WMf-*rTb1}de% zv8(Swx$d&-hDDv;CMH?3-@t5#WB9{!SLEvV^XhbT+>E=waAchuzYu~nRJ7Vb^A)s0IbC6-%s>_@I~xm}a1#w{il5#%!GCDM@lYD4hSD zHc8e0+&q4Lkp+rTjm|^&ySzd%&=WY0*Knt(QCTetCU@tfM~+lj|4Jzp`@AF$m7=1q z8+}s3h}3X$@;_QAUI18-i71T2htRSA7hv-+HKS&WJl*zSrRr>Qi~jiWX;!1h~#}2z4y# z@wvaAf}sC7f>cX!%@t#+DTY)yl2Cv*&&AeN`L_Yz7OE9Be_`%dl7n=fvSs=@jS6G~ zdp=0m%69!juo+SO5)WTWPfwrUw@1Z$|8~u=^i|567N9|S6V2xJ z8S>>yQgfVf{6Ac;K#l!>Zm9n0p8xqtoDBbwl?&E&)b(HArs9p82~ia0X{TK`ew_VV z21bkDIK6oO?t_LZ{2RfCCwnRjHilTq2sL8NW522fjPY!Q>jaMjeV^T9(8rw9^lUtg=h!+O=yoGwj-jkL{+Xm1a^11XNvt-wF4E+sOdu zS)2jlTfUWa;IeyECd5ct{9gOXMnt;AJKgjvj~NRo=^%~M+}4udRJr0W#Krbums=Ln zn*B3mM;X|UJFgz<(|`(5(b5LU&V<^6yp5fI^NW76zYVsfo@KU5Wpr7z8H2ZoLO(yZ z51|En#Ex6nADF3s(Pl;9=D@(knzO6E`z$mj0z#Z(cENrE<%0epzd>11+jtHZfeQBv zLadCc*fVR-%pjqBm|^lE4rw!+wQ0BcK{R~7mOg^^pxG3($7Evh_lU`zzQpwWd|Tu7 zoxLvX21gwTqIV`Y>hC2BDwKlDynd01PGz=lZ@r~$WkG`eyt-Y(qd|k;ricE&wUMn; zRFBs#9lCO3I>smPfgKUY?g0=}9JclI8Qrm_afTFhv58o|ik_1pQAWO9sOSSO zt@=X>x6U`ZZ>y&=sOsnECv+2$Bx5?>GQC5vD6qu&Oq|JuQ0P8lOJr(}e>UaX9XYc4 z{QzpEKX9i8j3kq%_W(SQ-8wgRcfX?3YW2`=d0XAVC9{L6Jw-CDV&yqRm?Q- zC1L&W`W6)7g!5N^`0Vf`Z+J|zCwT2P^!JOQ*AWCg`WJif{ha1$Ti^wz&cFk)1;qO+ z`5)i$b4K11=@xV8Tn{e!;!o8o?@K^#c|V(}@iSj1J;ZDOp$fw;n&J${lNuzrZrzU? z7g3ri!@ND-+hW&$c{Gc}(*Mh|3UY`)+tVw;wDyY1`mI)d>sL$6>7Gz*_K7%}=Ug7ssmEE_Q>j#6@$bR(**a+SFO z![&a5nl*$?hVg82%pRP`VJio<-Amx5YX)okkKYdE<4+5KyjAF>40r9m}SiV8kG zR00-6KFa9vf#5ECmV)=E8(9bU)7!uF)1tlgQ~I%XT&hVBtcL{*GGrPoi9xS&kx1xZ zHRO88mQr*O?$XbwYl_^4lO!0b6Ow(h+;oGFD5MW0_QlUb zY_HD>6QLL__}`cMocj<~hm~j>#@;>fP#8db)%nHtu*=r1^xZgo&TeJIr)}5ER!d_= z#a|(O{c>X*OKBOs58^7LGt5}-i~942aQh(wPL9Hh)DiKvENQj7&s0Ko;Rt(D9yv#` z%v^S0GR74&%OhLSj2{@pPLPpJt7qHpD4P-dRNn$i>>~P>I@c<@q(AO zso^F=r6`ox!8Ci9;PBzn4)Wenpe8SQ&S+4W4N8!$TIT3AE;crOx9W#k1;x)*_K}p! zoExa0l3QjR0IFzCa|_TV-=UX>=fR9%JPz-KjhRt@RH@<)a!H{ED{RbA=8Vdks*1_I zQg#u=f6;x`1?3oz9zA;Lu6kp#qxa~3-)HYGL%1!EFVKd_@J z?-=Xg#Jb=Ug@+J#3E7{Q#t^}^v4#+(9vG8Mq^Ip^sQ#vuT~7(#8CGZtRet*SJ*ApK z#JBEl@I=n4Q{v)i0kvsYQ&$WexJniw0kpsGTUYf%ms7#{7uL>sqUV&0@YDsPjPwZQenc$Jel7ux6u4RyKx-GRzq;pj0PF zE;3__R$o_W*=2*4K6@O7>R02N?{Au4a&4qxk?dq33+K`Zc)&kp$@N7OJ{&=(BZ$2l z7t8LxEENXwGB7w*cUP;Po=u+)wLhZ3EG+iU@QVo|wJFn!UsD)un;$%BiXola7S-J+7Rz`D{hm`05*qsYql>sm zU@9z&#H4CP$%p92ZV^01TVYo*HHuJ{7`%X;x#ysU#jVmGsn`J%*Cc5|{d}8S7tBYP zHOz#fpf^oxMQT=aGNf#wR9|8ohR>UHg5oxQ?b@}U*=^gl&0>)*_u5+MlT?X1UvoyH z;zCvR>2{xDcLpAuoy9Z1M~CmxU)m+0O-Fek1{*1Axw|8!r31^!`*@QSB#qoFvdcXR z!y7$Y^Y_xWz-f}Tzr4S(T*fIs#l4vKt=NwPypM6_N6L;_Ldi|Xk$C0thv$y0_G$wr z%`84a`|rG!cg5Y6x%0Ibr#_qgq{Ln%2bV4ahP<=hHDNUO9J8@kzqE zoq=#aog6a|`S3}mUH;X2+14mo^7?R9ty8wuH0)lC3v%_Scc7kMSA`5pscr}XgS0wE z#yyKTq6{Jn7%?e1-CJsOg=hBbI>=#oC4S3(KUc{31*Y2aqLhaMEa0i7j9+L`+zo2v zy{4*MV0@TA31-?NvwqbJoU4>Ef_uW}^}oq^`2VLD2Jl0(fA6d~0K)2JXxktat7%? zd^7mMFrmxS_<5EyXKg<>`V9LgIw9s^Rcx3&S6P#`bAS{YO^M?GgS{T;cxEuY`v$USOWG8 zu5uXrKlVXkJ`=mVQpjGK(o22J$mso#^`6&feg0D(xZRvP!zB{x=hy$Ma%S9OHK!zU zw`;s*NIjj3@eDt9V&e)uqv^Wk!|rm4By;KH{efxBFKu@8PAa`$TT|6)++}}zcG8WH zf+Uf?Km4m@2__+515l zY(G0WgMmdfw1L;)@i31QVMUd0{b6BeL1*+|qu7*-qv0G%G#9}s2-os`W}HzAI0tws zjpYVc@P1zmfQ2GahjJn9#i=uiT}gtJwek3?#_U2Hca@>Ia6MBO?L~H`aN4}zM;w^y zzrb3QV9(;(`5%oEDjy`)|Dy6~)?2p}n1>=~6lQtR7f2@tKLrFSTO#ZDy`wLIR9R)h5gcJQw{`2V*E0}+7MD7 zuYOy*_rY@0?KBofHH#?!$9#8{aL>%Tb?p#=jm3V*6|6a)EfLXGZi z@fCQfLJlNnU39&1_Vd_-G1Wo~#@!(_wh3;!mY4KZX6WZ??M1z=1`@FO25F zlT-B{lyjCvb)YRGn__v zA@<1nT!#tpOyzT9p19okG09DwyU|kC*0UTl*30@AV<+n_vmk}!=ZoAi8_|>A{+z*I z8bH|OVLp_|Alh&Cnquj=2i;)Gl~Hia&WtDKm)?mqhiaU2W$D2(N!kzyk+HaKRVtxC7`VU z>Y^4WzaFJbNZof40iCSV6(f3qY15x-sGEv0#H<))9;w!Z*DN|NF;xWpEm=CdJ6Z%8 zD@v~_zB0oq&*20Z9F^)&2C$0?p6A}c>xE(&BG0~5i^>Qb_gw&@klh*-Bqy>Ujmta= z)M0y}sr~|nqr8W3@+T^FZbb_uD#h!kIzRkf zTI=JSWrvouNRk-I@S%q=IkBshd2#SqDTP74Dl}X!L!Hj6k-B1gqaisf$g`e zZVI)E9{guwX$2N=NvQ=!BeeC%R8NW;6EfYG3m}dh7Ui$nVmW^QWGhk6N%DrDDCKKk7*kRY0&K#ir$Gy4^YH(0H{E)(hq8toVh;ihEKBZKJG2;40>lt$6O5r8dyCp7IL&N>Qhb?FeUvc4KlQUt^ zzJ2W@6T0pa??%iG-38slLXN(5x#ASZ6Bb!N(8@U2WMi?FIX2_+#U|*T|3~|WS~uEBf;%A9lb^Li6ihhe-8loF>>b5WqC!?_O^xIaH-z3pS z^(PmnUKBq#am@fIvD-d%>Qr~8dkJ9-;H5C4f9Y|+zrKyd0>sZI2ymM+mT_P#L!Y#u zOG163^~q=gkOH%6+N}%fUNgigCB3_9%rK7&&O@6e71rg^$Gz$=N#DV!@2vGQk|SCe zgtp(lvS3V~PuHteL<57}KQ&7XzUaQ<9E&l=kbAl+m2JSLO|%Jn6=y$%1i(nn_9L=n za-nGFxC;90S<5#AS?&Az(E=a%5#nP%gxT+lk<^Nwg>(z=&+%KNtS5#&>}_Ut?rxoS z|8M^{#87V*&*J$Cf4X~*9xcDhF%!#bLUD8J*Q2^NLI=>_Hfbkq!SQWUSLK%7DZ zD1~IZbJD196ZF#P+1^T8M`a8zXNHf zcpRYx?oB(Dv>BG)-wimBzhX zr1$|ssF*fBp&S4tc+VbBEW!GLGfHZRehHtD8E=JmH4W>k)aBEg!p{P8geQkyhH^FYMIncn6;`io8{*kZqeV2wL{3KBU zdH<4h=#O;Sd3cw$m446MbY^iORxT&@uj^HPNm4yNGGBJ%MV2k=$q}r~E)j7AG9Hjy z$R!XGmWof95GOFC{d3m{Rrp4PTSB0q50()QSNB%_IZ*~7l^5V%Rsj*AuMXAtHk!;W zoPy}Zc%dshOVVN$G_y}1XZZ0TMEl6RVhUFd2WQ#aCHSsFi5ScMUz*)S_2UR5NV%Sp zET%}~Dw{6_zW+SKR`^taQy{0EWd$*X!Y(Z1mZS=F0jm#t1{xaO;5}{l0qZKUS?)tm!`lhqA+N z;_{LZ3V-;SKSZl2djz9*?%Nl``Mt}wJe7Gi0DB01QxvVH4KNM(>pO@i*1wmSubGv6|$#qX0achxr+ z?T7lKF0C+bTJ2qNm9OKq@9B>ku)>84FJB?faFi9KFv{_kV>)Xr`PIevxyL0M7C+8M z7A!&MH=t|hVGq;g`=5fT$!;ez*)uAH<%1cw{2;@sK|P4o+kw-=PneMHT?r_`Eg zDNiT_%{=UC{Ob$P`0w@xqvMkQpQdrDM(|y;$fpvF753KC6O$~-@!}q8yzOl|<`=vT z584`0kBVdM{>qPkJ|koZ0~j`OUzvaB7dPCEBK*G`0#(y);1w|Qp27gCTBpUUkLA

cMnY%3U(Yi{MDs0VT}57WMUZJ`RI&*e>vYc*}FNPFx`Xp$wm1Yf*(F?t`FQXq9U zDA{#)SjZ8+${hI1&LS{MyYoz?f^kK{a2>Fe3fj@rt=i_ z!6jc5H?Re}EtGn|E^E~{<5L6Hc(19E(r?;O?fXUp`8O5j=HLbm=+N&8FO!zpIXOJM zKLjbU0(TQvP`_6&@@bwt<3Z0po-$QAOy)cb0}^ykq^xjr%gvk6zOz#pHL*_85Q5*sEk9NM{W&BW~lf8WXo6U4hKmPg&^#}CFE zJ3$YKBko*wde2jDAoU1&m*lkQ(bkSzDe~n|ihG14Tn=n$ZXCET=8gWs800^FMVp|x zq*rHC)7*?CbHlCE7sz3ctV8CKW{)P%f%5HzmzlGUh0iBznWfVdM^Y3hq)b%<{EbAu zi6!=>{2m!1&_s0d2|_^S*PIL=WJijdCW7jOatN8S-Ec*RgATzhZaaH<8!BW6mQzK@ z1gHQjtxU?w=dwGY@{c?qYetNw7O~5Ie*Y|>ps=#S&CvnhrHJx0p#}ts?`<(m^Fc``hR56eqyx zrcbklcx5-qI6%|kr!t>chnJ_WKQ_Qhlzz(BGy|$5nfB3&c?vC4o*Z+s>8qH~_Y3R% z86P>vL|op)M!E|9Kp5#r@GC{?01}hhvvAACjdv+uOTY^v37a)FpX(UfHG`ZICELNZ z_kZCh?NNB;8=pA5hYlmdSD_!W6TEk6a}$q4BI2jpf(Xz<=C%o_hsAPMwgXO)3{D*- zwz1GhRxbRx2Tx+?pFGz?(gwh#kwt&$ar76$rQs~drkn4`l`;3xe|CA>kFtD6FSUo{ za^g4Qg%M!BvS#UzT8-5dw-)Jm_c{yXVC=-@S5jt4Lnyi=K|0CW&vO%T4iJe?OWY5B zZ8M78NlL>-L9l@YA90U@nGMCRna#R1afV0;+_0e=YeoE6_56Pm^QIehMFbD&>_pa& zdxmK_aXg~%0&axxqn@4n&Rr5pFbs~7ja#;SQ`YsDU0|5ag&un`79?<5&U|-TkiqPV z$>8B)ihv_+g*6kre?1@4=<+XihTqZAC2sMXYIhe4(R&s>t@r+q9k7uM-I&EVl6w_g zSPCjqt?^(DRnvPItthhHhZp_&4c`udTc5!iMZYqV6A>2Lf@S+;6(?#Zh3ig?W5#9p zFkEFbK`90)WAss}DZa2?2ESa6=B!N`;mDG5B8n`sFEvx^F^>>d1o&dpQz%OOnfh{0 zq|(5X#;mjt8XiJ}Y(=Yjx9b$AX&Jjqe~R_Yg|Bb>eE0kL^yg9e7&Lp+ zb~g-CW<~igGTbE;_Rksv!%^Oc0L>({{Es(fz$9)nd?0&3*NSeS2Cr3Y5+G!@W9awL zRov`YWXX8@K{wlmWbEajGZh@Jh_FBZjG`iWMQyY$=s zJUyxvM@JDzI;8wm9N_wX-lJfyz?<31w%C0wV8VhfpUC3ey)FBu3Ko%coVdfX(t4(d zf>*67`gldre_y)i_(6LHMAx;;I_P)@Z60`LMS9-T*{W@ap7y92GX9mUDr1bXt->T$ z^)nD?qG78TAKU>Cs?@D90&urmb{3jM%L-~{N>jq3JQszT1jxEv>SMfN(IG*Vkv6GWu>Tc(#*O6m8GkSp~d5qQ1= z4}v+!PT5M8403$Of?xAD_X^?i7qjZ-l%G1H5kqw?ev(4}f;{w@93Wd4`I+Bo|u>$BJUm&^1%|dJKD%MBv2e zM5-s*bmwRYek?#2BW{TVhahh)(W6>#PTBG!vR3q1f`&Z-Vfldyi~t` z+RfnL=(-;VotdL5TRnEvzke@NI)A%A&Q&Nh-W7jI?7U@8!vorFm{GsW_gGUlbK$}2 zU(R_qt!m?sa zglWoHj5pF3@LNHt3zC^Ht^bV||8|=jeoA;;SjQ@z<@LhDe@KbySEXvzNCM}tZB~9G z7cWi#h!#2jU)`)N%ctzViaWb(?*SDSa|m^w;2y2|#YoFfvogP^7i1OG8>8t#G!J!h z<}qdyze-3@v)<)h^6Py;7Oxu0BpyQW5Kj!DGmtHtLk|+6J6auEX zy>D}AJ@Ca+=pwW+$U!Prn(q8n5XyGEz+~9PG81@^?Axn&8D8OO)FrZ>Lz($}CY+58w>74svact@Hh zN|S^v{9JI}^PXF`(O~FiWX|bPvi%l(@QC!;=OdDOZ0R5mSvoIN~QrRm0p?hh7q}o#4f-HUyj;>FHaJ)#Ie_)MJ6D%tF`sK@O2*iq3 zJAA_uV#0A@CT;x2fB>W5JDWjlWn+{u{AIvHS!Ge` zGX+I5(!S%OC6JzseF(a-yI#&1=}ZMNd#^~p6{=tGNM?HlcMoq8;)Cqcf*d)&Jqw6_ z@4t}tT;Ua*5=;<>ReRcig)TF89fDA=ENr1}uYIU8SzXk?qIlpc^eL4T1-7<7U-RA8 zkQQmpwX_bjRPw}A3>-w~BNPZC$mgXmDJ{w|#xCJh7D}0+cnD34rixW~ zfxYGw<%(H~jGN^$KeFgY=`rnB)vu;V3QQJ`E^4C?WkX=22@r)~2f7>je>ezPEU>VY zUZz3`+$^DnXo9234fj1TmCOO_QiL%^|1dAh}Xm>m)-xZ@pD}K_8k<#jL zC7s`Dp8__XMKo5lkx!Cd=hm%LN5Z^>T4s99tgEXU$u=}$+TWkCzaw^*C!exNO$q`4 zmG0YOzA>nGpsEoXv6MLsv;B(hgWTd|HB8n^2w92Cjn{BmM>fYN9Ob%{I{wBGSL~ejdTkR| zJA4pZ_=?N5q&iB9Q+3zX>;l*NcgwJY$3f$C43F}Ja%yCu`;vdqeD>3XS4yRMT5iZO z1zZNCNV`i~mUZmTUBI7-Kf%++(>eTf;rI3Ye3^`ip_iwTmW}1_X^dWn4y>Lz+NAe5hs?QMRePI~Ko2deUh!86f${=SD)1JhD}^$l=b@49ASs5wGP;faU>hf0yw<=qx zlU-Ar5xM9$WJ?Sx>K2wk8o2_lbj|s>?~?(fZp@f8+j9o>CsJ-T)X|v}6>@aCugG+y z7K1&$e|RaRpeS0pS1jE-aI&N7o27K!VidvJwNzJnK8i23#A*$Vq+ah$e-TVXmnDc2 zJ%b~&$n-c)6h3B|vY!tWMmAJT-ksZzMdu#QJzC3TS@k0fUYz6_?C(xZX?nx29ny-` zbuI32{vGV>?5?usSY_u()S-?ufiNK)ypw(vn*klT;3nJZFm{I9A-<@JfZtbIS1QNr zCC$Ed(OdsvXNw25!k6KVm$RlugNCqsqb8nTI21W-x8Ry(mRvrj1%=)#c$9lJSHe{Z z65%IvBhS9BR0=X|ph7Te_AdPoN-R$9Z)(Z#|FISHeqiHTXghp($^EUG>VOdzH+!+w zFwJDe2q_UdpWC-gkmCM>GL5?18D`h@`!GNi@?cZRUs*O2=gn9OxbQrPH%u z@Ng8(9Ppd}v+%S90HSP}a9VnH><$_&?wumR`g7M1)GypQW0ODp{Vr9odqtK@!h4oG zmu1itenTC+vEdE2*L6)0P&?#woh~?M)mG=lrVtCRX!=$R#? zS)na+pZ%E4+#^?d#*0_!qqEx!c|Q)LXrn~g2h}Y42RSK92r+bwxp zy*V&WEM3)2VzRqk`pqX%t*1qpC60KqHt*wy52s};z+4xCVgE4=8a6y+vAjIjgCaeH z?u&(3{suRBN%Y8KLrc+Ll3V--1UGT~c(Ux{@Z_?8H{q$>73{!Z4=rG>ER&F~zJgt7 zr)axmnsU9>)7s$7g44?0Zy}U$*bWFsj5wr_3pZe6ECEIqK27PrhNC7OKS*WHti?g6UB=8}GWR}5PpI_TGNa&(W&t9PnQLtkZhHf?j zy%2MvR}c|p{z9+Vy@6(a)oE6&FGg)-UYO)8cPV>GBh5$`(5kpTHhEB>QrWnN3p(0oByVr zBK5&w!pA{9ux|RxN-6t0e)2By9ehB+3DvsGmIO!vU^nyh)>_$lYm8_rgHK+nG0CHq zvU^M(P5-rxS%V-PPXw$9KJ?FAu%k+nVlTFb@NS2!#)YTUtg2Tf$P)D1KnEqV0vB7^ z9C8?9pPDtre^nMwF!>1h=bGxrDS{dp<)i;Gk9#5Paw?&*m_3E&Tf+*K0*-3fwCYrA zv38J6+gPxt) zx7YN)7)^B`07&%9_uijMawZl^wk1G#V~z%`caifOrt4W;2=P`?9d5&>Osd29EH+RZ zL*+f^viX_S72)v{fPUKpon{$@=@T;>Wx6zS2a-$ZZNK`+cc1nQGyHm?j>=AJ@33e2 zdc5qks!OCu^EU{e;u28x>UtEP;hQx#*BzaDb{jynoCj4(13Kvfl#KI@qwp@3CY&m7 zI#Y#k1NJW{($AgSw_hr5qtc%Sc#GOIgJgc1M<)aFeAx;tWL{YSNwwdi_F|2^plsmOsdB=IR**CQ53+kAuM@&)a_iE3)iO*9XN{ArDx0>mmdfBy z!nc%t`h2n6R|_;N&?rC==gEiK}13C^LIf}EPR z2@tN?r}3*|XI;S&plXnOTy%L%L3lNoDJ+#Ka%q-G1^q&g32`)gc z7=+O!w0wHW&4GYIanK1s&zaly=8<6~$^!{*Bp&#Fo+aS6ow!l#EPCDPi>3eU4v$%t`{?-5fMSY}4!`VR#!X3t zM6D-eJ;UAsO{$2r#Y314X=j^hYHRfM<`gd|)c_7}s z#kq}iG!sBy$W7@zfHJaJt0=TuNKPRY?od6cdgEt}l}?pLZb^~Uc+?JAl+c<29()jR zT}E>U2iJD)+pph5IkTeG*0Wsd=+$(tLONoV3e#ip987S+7F!5$`_r8(Pdm)X6~9E! zcv)-+*Iw=mtdn=f-pQ`AR*$wV!G~}wloz(C-^~yLB3r3$%^&_pqly0!Q`gv+z^WGI z5A5pay*AzN+Pe3g$|)0{g)D57T){qb&{)_Q~$MO#!P1r*Z0V5I?8*^Qy;p>(+~2f4TRyKtt=c3(s&MopZ-p^1ek zWnTK?2Pa?PNvE^(4!}uCvzw|KIY@*DD{l%R^DvoL-!JX_eOF@ig)c9gOHw6OEGZf? zc(A-)$)s#n8D>&T6Mx=!&XP`HW z;`dON(~xLm70Suo52=CBL8ZRh7`#yowSZ&kah9SENON-_ zYS~CY;s)#l2L3nq2G)|2#?V@=OkYJdUqz*v4B}>kPePH>5={2wpJr4@OON2{q?cC{e;a7j3SUQLDy3P}9BB zq*4r=-{!aQ%)tXrDGnv#wLhhpl}VtYZHF6Zs1D1FD{#6sFQK{N{z@qq>7ByGFiZBq z2~vjYwcLCx8BJCP$sME@V}-y(SSAjOc@d{&I1;>fFN`?i?-$uZK{0*IIF{{Aiogq( zVE~w0QX`5(j#vuQ>iHG;S*RTJQ0@zjM~+Eq@?t7-QEgI@EjG?t+lM1~# z=rxhPEK}S-7|ouB4JkexFhjPbKI{1G@(WArfbSFbqiU&GNEZHqzJ*>TYt1VeV(!K~>7IIt}S^$j)ism%IUw9?drc*Sv^Df&8UY{g{n`SUV!%HP?z?z_qifJ)fiu5x&$Z1>>({hR<0MV;^fAc`WPt)dUd zj{#(Mf5CnjSqH@JO;}ze*eMrjNo2;u$o}62DP2@~v5A88be&MJAlFhvP_*#?%31R# z>b!(?0_HBTkcbn6?egl?E9l6OoHc>2rR0jRX%fI*s znIZ!J*la(zBc!0=Vhp4>e@TI5Ga5+%6CZPN=Q_Cr6L!gFc%#iwAQSOL z5&A*iex~Jjq()erIJ<;6-I1o=&LI2~=E#o{n2f$$9&FRz$Vk*Mrz-ZOU%Kb;A?)bG zH!l5S@Mox=X`&<@+=jXMLOwn zBzG0%NF39+Ve8sd)@LQn0MbK9H0Gh!ZRhaKhw_pGj~p0T7`B^ZMA@T|nDz9`N-J|+ zxRsK4@dRIL;MZQ|For^m+H#ldCEx!!D{MczU@e_6^%&M(0}7m zPO)~6>6@Q@e@O7j_^BF9<{Ab&B_RtE3GGFDzzQV-JpBh>7gW`Uj~;zkzybPsZ%GsX zG!Ze;)0P!KpjRiw_1azia2vTeBg7e^%E-~kS(rliIOIK8tI>8^xO1PxCNz9t?_I8T zdQT1@$@Mq;MMd9zPmrTtTSf)+t9;sjWBbgfcMl!e?$NZFdv8V`lY#Cxw3SY z@h{C~OT*KZF!%igW4_zXd4$U(|4OQn+qkaM|4M4WNu2*F>Ij}Q=k@G zv^2tv7Zv4ZB&+~9pbcdaog{{!ZQIKZ3L^l^R`2Hb2w7W?vDctqrz+mGRyDOa)+Y`PDjwfjmFnva)gf%+nE-zWAsd z8rrYyI((*f^n(M&k1L8oqiBjgWl5C`G>@{~{(qKK`KQ6%8pp8a$IttT#b?#Gmruw` z4{MgDnY`M6m2q<|hZCPSb}1cL)FrIxv)dLg_Qe(V%`aN;`F{S}1*Jo?cB}0h_!eE& z_8l_dQlN2&ZnAag%447FF0pRhW8C&ZOE2snY4QGha`Ew{{*$`JrG4G#!7_-m@SLnR z1O$X7&?2@lou|T~P!HnLft=hxhMK0yN%jwxIoS&OroLcR?_W7WdUJ7{W+s6LlrSg{)mvHMSppQ1>;3`+G5+~i+6U1GNT zi(=X0b#L_X{mHA9Y#lGzqr3gVh%7Uu`o-3>)^$EV}WLw|&QY#A~CCuYdT!Iv!kmkDR_e zZ8BRN>YHWLa%F&e{IC}*g8A(Op6G^VYREqXSo_2zc-^eCXx-7ECZn}X7VS%$aWOU~ z;oEXOUWt5p7%W@1>;w){ao6?Pc6O3@ZNB2$xG*;>D|);^gDzdUGRT4T1no4du2xRK zK8igsfoibA2GbyK9(?eA(*gDFu)@0Yg!2s%h5ENFRf_a8_?O_=n!bq~E+POA}!%Jd*t&gI*xfLFE0j2SZt7~d2D z)BSd5z0S2Dz*~8Gb}oAJM$g82?xukU`V$3DVj7O%Uehm#?ryayGQ%@K* zO4q)AM9!}A(}N4DTUp(5dN4XVQ%in-^&dNuy{ju9IKFDBWB1GJOb8n@ zxs&_i8o@OfK4ip*I@qPI1QDuA203{)H_5iW<CC6jD-kbe6~b z;wOD){v~g~q|K8jPo{4QnL1;}#ZS*V85o?P`S*N!p(VhvN#W;r?^19<=?3kCr*vVy zt8PSg*S>vQvNLAOm5(oUbacigd3QtX^v8gD?W~7vxt^L@4JLdlbE#DXfUx1G%AGM> z$c>v(SGL*jq0s&JY0=9=Z&!WhZ2S%cRm>AhyZ7#`%??g2Zu6|gH=C)_ZrqrU`QGsf ziH#Y0)IZdY#}aX-O}lngz#dGTZ)&L^Vx9!f#{tkesdNty*>&W&aZM3p%wF8rb-(}( z{44zlksE@xH!S#(^!U-Eb~Xd8=5DfW|2Nuc%9%EiGp0_>V+!>xX;9Gnl9DS#szE04 zPFslPf4Ljm1VhpYC|JN;juwA|2jK@9q+Gm1ljhE?J8|Mf`z;H?5<0JHoq~&zgKvoM zkz=nTeT#Oa{&nnJ#~kf#vE8g&t|k)PxcACF=Ea4IXiz7{eT2k5t9Z{B-EC`mUG6id z=cplVB6>fWA9v|f>mj*mU+nkEiQ!9q%xm7f6g)%j5W(#y#O6%l23p(N2J1J|(ux-! z{_ctA%6L8Mn3$dVT3Xu|+O}^1(6)y5aS9o?WQ6*Fojwj}Tfp?)kW-~ksJ<(}+8{76 zuzH<3jm1yzPz_6dm&`S6pr!5_?0~A!=bvYM8xqP%wg){*T$29pV)}&5)LBo9%Gyns-?%GuT$)_*%)+)kCcx0qa#2Wrimyxs$-RMnn zA3oevz%`|XtBhX&d0|WR?~t(bQg(J@lw2p`;u@fRyBG)~W5Xyd4Lk;{C?eIe>q0hj z$_=xcI+Pg8jId+mm$EW9)}+_h(+zTD?xM!d?{4Sf=mX_nzJA^Gz;HpjvB~m9TbE;< zG$1x-Sn#@6S%-h6g+bPa`9FT21doc7pj$K4zp0n7!g`a9ZiboM@Ellk$MrQ-P5)wnnRIMxK!En<*43NhtEN7Z;XaQ|B^&B< ztu6RVXjD8Ooz=opwJ%D^&Vg|W34`GmyodPIwtf4mv`bs|1P8pz;==_zpmi8HbLKJQ zNy<;F2h?ESr%#`+=bka|V4_#Gg_&0sBGL5kIKYid@N6b_LJjbClCFJKm{_eSIYk>YloM{W>Z~+Z0gAdPsfd+d4UEizBNPK6ou{*|SZ4UG1>e z)nDC@>i|!`5=IztKafh&bX7(6TK^C$!#qNhl9Eo4oX(yL+0IPaQqv*P@4suW&GghS?o8ms)Jz~J6Ql!LvMxvt@IZh&YkMUIup ztXZ>cL%aV`bL~LNgu9V_#8-YK>swpfYx^`Wc+z76Xu*3|cMsPBov+5BwtIR6goT_w z>U6AaJ+1ia#~$tHb!lZd>+a~SQWAto-&L){*RQXCZ*_}Fothj9w9NFlpZ14s-NjaQ z>*`fmd2y2*M)|t6HYB7Glc(tk+e}rgHm=`c#2)|aQ$4;(!Jx}ol*OVE59mVe9Ua>b zZ><|Kcyjm9d4zt6s$36C2C`S?CnO{^HFmFuNO1#K@6wYe7q;)Hua&%yDYrhU8vDJW zVMg-vh6jLMtM~5MQTbqod(RO^@BIV%QbrrN9ds)@*1@0~E5*3V|f(*-Xzo2*nyTn+QS zeNWe!M3H45@0zFGuyqd}7wO{#M?L`#yyPe)Hv*Cy7f8BO@c**-(5~rNsGa z&>q&_JEBA6)>i#=Vq#+IAJz@pcl2n7J$v?8#S=xXgV#M-2cxEc`v!&`JYVnQOn7>F z-h64+scYAr`n5BnGPG^IU5@#_XDy}Afz(F5-D|t7_g7B%zM?CZ@(%&(_?q)hUwvdW z(=XLGmSdgB$A42qZ^UaddGDmsAQjzMjS6@plv!YPet8W1*j}KaA|mgT4b|xnjF6TbcR5!dT*lbB%1$!U zX#IxO>(*@|x#MR0sqOi39<$+v4w_B11cGnhrES~F_(fh~YTD(3A41qpQh!itg$?gY zxQmU6jI0SeE`^J8v$1>djTcQ#JQ^Yoy460bU3r&tE`hitxiOoPqr${oW7e!2pEF;T z_&{Efc~}{x*gRmg_KBuyo5S_vSw?&R&}%uI9C_pFtNei>rS_%#i}UVm$q^+zmO%;K zwryL=f-?DQxAsu0+S+RJLnm0p*JTZBlzx>((?-4!Buv*RP#iyXM+TauS_@kgR5^+kpFS zc(7>e_WCAc>UKJ9^sZhFE&J6*yZ5~rt0V;Nu>TwflLaTYPZ^)?BhXsFdvA;I!fW;rRyXEVTEz z#ts=cu#)uTMNcobqWNf?${4E+Sp!RfZWL*?nm7M*@CX~L%z2C|2EF<4!OI-q_XuNo z#KTwb-P@A3&?+A8F+__ojriW{sY;L`XBM60V(jiQqYiUR46U>Dmu?7fI*+z<88Y3* z9nK*0yi7Tffh$e*Rjo-OF~n{o44I|DY}(hy*xcN_pyi)?_ilmI`|RGxepis*b-qw- zThQSvr!zg`s|;1HNx-XV*Qa*JvZ%IHl~60Hbn5i+izByn9$SMYIj~+cZ&iqLBwVbm ztHYOD$LWc?KvQ+x`Ai+!nP*=uh&NT;rc7CliTT2U%|muAu`Vlh*oNzGTDrB(AF`MqD_;>iBM47Q zi@s_Wf}Gr>r9F(bw^%6=p_%Et)`TLfcqW_kbO-Btl%96#0`qtkvzYpfkOq$*A9`gz zZ?0*-73QLYAg(&QG$ojvdtX|*?#!7p>YZ5g4{s_zsQqQEW;1D!22Y->3*@u~@=xfN zUDv0Q{`<*uPDx3*x^XC7?duXBu7)P$%af+ZW-nZ5%mTJ5^!qjL+7Fw=V~5z-**QUz z8*fSN+h)i33r`0h%*gxj;zk+LsgvBiLr0EKtoQVqynsA5J!jLq-dYu(_T{Aq53~tf zmm$~P!S6OMeTfR*`5KDEJZ5#N0JH6^hswjNedYG;I&eU&hzbpBu2qkW&3Q%zBPf9q z+^IP;z4RN_6fA&_uvcLp-_p)9BJ-W%y!-s7;`51#!-CfYX#YtW)@N~NTqin3vetJp zrPiaTJK9wlkitBPp20A3zuAJSv_qr?;^tmPzVjgk+TTh`tBD2BKS|xJ{IF(FQBT-+ zfe#U3{HbSV@{Hb{I&Ip&%URW{S1(l(~12d&| zNNyO~+O2={KDKqe*4Rw4m{~_Fxmn1-6dDUn_bD^xPc~4Vjr{Z{OeQx%dI}qE8yXU_ z7G_ICQj$x=R6gq{dhkzh8XWM5(VE+?L&Wr`^XIjhBBj7fT#4|q8I@P#RD3HYL2WC; zGEA#kvl{1Io?HWJcoG9(WdD_6ugm2yZlV(t;ePgC8IPDUF}?D6D7b?+g2HY9$#EYE z7oU94MV|mjcEMEgcsj3wf;?cuhJU8s$#kQ7WME=6Ev=!XA>@DX%`B6jVLd4CUQ+(q zg|F*pdMs*tq!BoR&EZiFJG#@P$X9Z2UtizN&3o0MQL&^Ai2uB)X4M)& z84saf>3SrdKX1e9KNN1KZ$z0ZP+I|PD^u9zR>#!wadC0da1Q$D|EFzx%9iyxWqenY z`gBnoCd;U-jRk!7m5GUoR>uOubVz}Dd2aSwKBmQYRv?<;!)r61ZD(Uy`^v3b!!t1t z-l<4)jTg=v(8qT~ zecMdC2lt&P4XDK#vgd z0Qg{%@rS|vbUiFGJxs6$-GU;$1qe;dlKQ=9gs~N@%-Gn1A#tV9&ASFQu)hKOwaz&g z*M~iuYM#YEagF?H0f)veIn_7Wb#6~Y$5rK#lAODA)gDY;aC>U^f2aY+K@Xv?zg2EI zaNr39lIXRqw*@-Ys#dLk!)r_u?GE6zQ2s=E}uQY=b(fiAY&_Q+t{ubW35 zYZz~$J^bK|*2-y*=#~G^KPx%vlkw7h_x}A!)0g~x7G5_3R9;z7Q1Gj(tbOI)y#}0u z!Bl#;%DMR`78N>5;{La!CQtLz2QD(wj(YH1vzNeyBhWwg-0tPEub}G%=OsWR9$y>` z8c8kl^3gf0)EJYsu{ZE|!_?D=F^+w7Qc?>>G25OySxO#w$s}Dkbn3=hS}Q4#BoDl1 zhSMk>Tki0CujP&3%E-`?u0u@vs0t38O-$VMfMPc3$hNSg<2FBLx8d%0Kj$Y1Vno%e zp=HWreid@x`MOcxT&>E>9O@maW0|?RhN^0rTK-L`5sJH1wyv=?3gbZ%&}PMkTjOTSHgPp+O|gBaq!L+Td+Kjj3`!8F^=D35wws=*Tg z*SMA*wC^z4@t8wIf>ZRI)A>@D@uS~`71 zRF=d|hXo53?9i{RHT3%9$1NGy^mzERmBq5AkXPy-j>wL|l>>!M!>m!GM%8Q5q(em^ z@DDLEVC&W+i7V*`ZY{&>1-0` z`v{ggXXBGs_fE}=_YLDDt@%1uKCXwm%Xfjn0Kzue*1MDHyf(DWhCy14BNa0-#=-=|lvCS-?-S0A)(@WD%P z)YrfoxitAAeIHk(8xh^q)YRCkHz%!AmoByG83v#c9KWC6h%SB=XuVEc+v^`SW5ir3G+;GPUk(joT;k#`+{vq-eR)iCMGXV2 zLAxEI-nV1NYFLnY^!WAl4)&ThGh@Qov14ll)w3T>SPB3=2=EwuP>T&YZI!S+C^{6x zQ3`+=Gde`>Cty)WR}=vEZH5gKjMIMj0XsK1VJ8w2cIr2>zjtKRvAp79!^rXH(>H|~ z{sLm0Bid!!0sMkc42r*ccI%c`!_8w(nEz}=q6@ug^LFC@8T_4{eLLMO&fDxZ#P#@~ zPp4M3jP7=Eb&qdvqFv2t_$t1f+R*{s+~2R&><6c{vcO#gX9N4U3Em9i)Q9;9gB|;5 zdNyd<3dA^#Rv;ItwR8sfA+@Z!`qIB=VaQB^4aXxg`BNXrGv58_g}yCXwj3mrxeAzh zXbJ(I(cZXZ9~PLMoiu;*>eZFS>W~OuaSDb}bs@aUJ-?_EvG->A!Gi~-fA{wBvDp@Q zZBKl>38{M>glac{L~qs|wcd53OJH2IpCC*EA zDvpE4G8!xnDTLKdcW5oHO?^4K&(57YC)1J%GKelvuc8{9i%7T*vy&8J67Z?j@~W#f z7_?p?XLQ*&8ZaqhSCfVf*Kp_uGS4dD^hp@98#hkhzPnF@MNJ+r^72|0(kOqjp2scu zjcEfndI*G{FuQqdwC+%jy&Lf;A}quGGFl+}2@|^5QSyzXY?M(R-?JEoO2W%6Y2 zxivFtAqC#}fT0$6nGBj7m$*h_{Nrw{z2UB%<@^N;8qgSY((|xMi?13y^XcvHv@_8Y zW^U1X;dEm`wBSlw$#p&^j#JV?fa`XfHh(`at<&^gwoXs40R^U3!*_0RN!guHO6QCPTq}tP4E>a8`#yb|L#w;Y@jQ1&hBWp) z+EuR<5fO1h&fn6dp(QNun9LB%did~=)-Cz-8~+p*Zh~FEdvAWMTAe~?y)7g}kM}=e z(blzD6M}aII30!D6e0MV)6AJ2`Gv0gzO=4SHz`x;#ZGOLD64q-6L>*AxqWx+f&_8n$#${e=HXzV(RB9U!FCc;w>VrN0dhFSz`l`(N zzoUS91HthUHK1SZ^4kHhQ#`eu#Orzk9a$g`0rt2&ai**{ zJP7Bb455?}>@II_Z(}bTa+w&7WoEk@TZ3@4*ObTF)`PzHYi#r0XlndIrUay&jxsySRC&OSK_O;tcY z87|2rlnQ^&wpl%Y%@v+_|0&Oe^lg0Z{Q1co#c?OxH0Z562Mz^&h6J&N56j3Pa(e>> zg#Gga-~;pXI~eN6U5Jb{@O+q1o6En}HFj6Ui_d%eHc(#tV`s++6Se?pv^6mq131(h z-zlrvWP%Nn33RJhLdF#U(ic!bzgNZ_+%%`o(8-WB-C-E37qc3cxz#Y41gNJ`C0&9{ zYh}WJ4v%)-tR?l5tI+MmJu?8K*rs3G#Q2=coYH1yaCzkAgVF zE>7(~$0;Axf9j}eO5q!^^U5Wsz>aTPb~_T?*YkPOwoTUtcXhLM_O+c^T)kSSbN_V! zYK+ovxZ8MUea`&Na<0L89!m-%ux*kCdH}Lo4IaEvsyUZKp#j<<*QXBTmWervXYPpu zPp(glK7L#t+3|5FD{?b7)7tYCL z-u+*EgXJZSz-OxO4Qvo7cgLgyxq>HfnV(aQ9y0C+REkLmEt1x+r#h-_+Pn8;%I0?; zKQ_o()VrljT>&_T25)v9^LWsb!meqU z*J8BMLzzeMyJdDaUtZo2Z-ORSr1&GYp_Q69Zw?#5q{sh(5)xjf&zJ$)pl$Pb$^mkZ zk{H_fTyTm1vZoH;Xx`cVN;9yo-S2TiygA?F(XGatdYiVrmSIEG531pH`mcz<+truE z5vhp`6)6(j# zl}Mx%GBl`2NanE+lA%;&3@MV3%t=U+kTPV*R3Wp>^K-4O&-wmd&;H}Q&N)eYzxP`A zy6@?nf z5xQ@XNfkuOmH>Zo`R;?dtN@oaTbMQHMpahd-mxt4 z95|&*2e@fQRSfoHuEvAhFJwcZe!-?W>tFm1{o2BIuCZ}{VjJ!I8pkbbmVS7hQF?zo zz%upT5gx4W#a9?TT0hLJcX;Y#7E!c)*8oSBgoMIhj;TbVY=h09|GYohlcT_^v-DKj^yABd)u+ zD7%R5+>etCBj%~JvvdUE&S`dDc^kksLBLS~AX~mgyOp$3-8Rr+8N*fxIQn&i1A1V0 zls#iGY9Pe#Z{f#B8Zj|9M-;G08ZtmOIcYdZ27m<0Z)ebU^{9C{$L-SJ*O&|zbgdekFC+?c& zBcP2g<|YCT)aGB*#Wad+G_pF-epgiTV`2pVoZw2wz4wlm_XYve_B#{?X-Be5@Fp4g z|5qfgEEb6)vbm4L^F%#4F|$3#NWLrbsxJ2d>Wza!Gdp3y#=QPBtLMN)Xz6)C( zWJDWmYbyBQ6+&;YxVi{I4V;%cHykx3;B1AigTrpp&%ytM)2Zl1D?l2=qP<6VUa{}s zG$GufR5{?_Fr`;jLK*%@`ZrvfyRCeYmkWxJbv#n55FH7 zQrG@bVtYYqEoQoS2r<-kFJ0QAa)*om`C6uBec#h)&=?c=;0k)%p|P@XaC7?1Y<(+*XS2}Lho$vLZ{!$P zaFPt8AP9u5g>D#fDJfKl|jG^3Q(Aw9h`!_b*?MCc?kDou^2d;ue z*Sj4IrQ?_|;S=DSN_X{Ax`7`1ECx@U#Oc}0Xj8s_|L#LHk`|l9*))$3vYU{5XY%Sg zlg!!81cS8C`=i%p++;q!K30h)mwGqs|7WPx~U`^Cyg+rHJnF!4=k_YC?WZ zFMxjM7N#$LYR!>0JZ`LQ^6vuEc`YLZ^a64yWK)4&v{1TDs6H>5)c+} z8EdWU)V7k|w9(;6lp4vlBcU}^_rxkUpO9u;Je^l6v9(5q&db1R&60#^BWU1ab z&3}~ZM;))Q*YC2^zG17uCUj$9T+=~o@jwa9#J^X~Oy9ARKf%U>UgB3<>gQKRudQVl^-@0B&k2;g+D&q+y_|N`wkw zE8q`;dj21j=%@~8iHz*Tr)kwuj5K0rOqygez}%SM#70|qgMfyLK@4zU8+zq#Ut3U_|R`T<-y3lHTmBY03RQ@suHd74c2~TlzLsZqy-MiO<4P%Q;+5D&W%$2Nxvu~7@3n1S_ z#11Pv`^M*-92^9*0LlUQsqRxl!&=;;`!eBaeJ`nDQCTQPj!WXIeIQ~}0w>>tY$Fe- zK*25O-a^On@qB@pyqN?Sr0V>}0;Y!u#juksh#ub6$xff|Cv5$Ayf>bVl){#7ixcI6 z@ecmQZ<+ZBEA5+~CVrcE&V2b%%u(dWr&RN61tr&Mj}uS9Pn;1R0{FhEFU|dzzCC3| zt4%@3yk>rE>{3kF$|fdHsVKtkyJhFOU=B`zE8qq|{X-ap!bJR4Jg|V;$^OFzJ9gs4 z1*?dxtX)V(y3Mk0k^&}vjk+l=!>~|crNtZaF-5sJpswz};|6)o1~;&(@b^-o3HqzObMS<9@IWuoAwgDlcaw$Uh4BVBm)5 z#Sk3<`Ys%Fosc7>A6yE_c2}&dBq2KvKjGyXn5pb{pUKm|2+hB24WV;m`W|*tAPa&85Q>D=V_=|i2$2m(&wfA)d%QR z)}dQjhEUPxxH^{$k|m`J7vh!65?QQy$fV7TOB&X4LZ;fHXMk5-Y4*A~b_6>$H?Ruy`Q$`)WQnB@l3QYHIKe5e3 zeeqXez#-J4-|BngJ++Le;8Jp%)s@de@r*bILRh7NmbyOJ!UGJ4qqOI`ER-o72uUh| zLTtI05`Rz1pp4w&9>_Qf`T`tZDKJOUP5VXfN`m|VlrNDw02XN%ctPG|g}Z8e5o)6( zL=NfJbm*24vs(&ZB!bj%@R8d$Xh`wEOsEMql@D$(g~Nw^mCL%bdc-pI>?92Y1dd!W zHz$H9NcJ=#{p`g_`JBv~>-IUzKvk|xkl5xo4#%lVM)fAJ3s#6m)+7WI>W5fY48Q3H zEQ(lLu%9WLO!=X?`23Owa!?Ex(w@1c%3vLgc5rV}-GBb2Lfmr0yUNP`#g~>cNJp?x zr-qjGI+}}Bu`EYqO=wLETl)UUHj5PdVH%%}U+Y>Jtd$}q67vO{&8Kv!upbsr?1!J| z0ue~{Nc2@A>KBYYM^9mYivxYbd?)t&a)xet*?J!z*IuCQGLEBL2=}0$d!1f4xvnD{ zi{uso63kH%!^3BCt)NATAOaA>0$#K!+k4ZqvPucufPEDv>T^iBstc+Oywi?} zf`z@P$!+cJnuFiLJpz-}R8Kxx(CT>?5~@b-2P zO&oflSV$*{08d-^0XZXx>Q6HdfIpR`hyjdH*4EoA@*G);7$mrrew>Si0Hb4WTv==D zi=Bsv)!{_DYY5rb+&BMIqbL^)Wuz-zNnQPvZgXnQMm?4oTWJ-O(!zo1X~1sA9Yk#Ho*tB#KoA-aH65Z z=Z2WC#0uTpsn?g-$Vl$)PVr;xU}dXrf+At7>BS2dVsI1J;zH=e^TJJU0bmP$0xU9_ z!K9 zn#(uYpp~x}IPxNoOPsBG1UD!dBYf0k42yn)xOmDijb>73PtR@K3F2#t?i+fFj*AX> z@CA~O)V^m~@S(gO7{jRk7{?rG86N?)YabX8dk%9i8703^^?fbdf4qTn11^BurV)HT zVfVh8dJu1e?t^>W19sG@xFX-)1pEm3QdO{p8zDq<4c-1B%i29%T0S%AdPMRCF1|MR zDdWmZgO@5wMyZDZ{^QU!)Y8y>F0vTiP3!Css~CYRE&Oy)`>dJQO3^ZS#b%nLm&X}E zznV?+IJ}|msPJdu8iX3&v-;9&nU`Sbh-8pJZia@fDn~IpQdn25TKYhB({mWmRPWy+ z|ImxnR8}JW#oTc0%o+2#R)CX4E(pMD6;G1mnSUL^_&SSVpM>2+#D^+u+4o(Ni>c%A za~$_7fxKj@e^*&~G}cpDG!Uo#iWTpGX`~@on~8e|boDF80*Jc=e{qQPfmmM0l4|`! zsZ$oZ97ded@ayuK00X3vg@r{OA0J;JASiw``-@c} z#Lo{-^HWB)vyeY6L%*nq3ul$fbhC2(`gC!gJF^hTjvQFH;_!fUC;IBu1Y%#gqVatigon8$3_XYxml!Zj8qQFvRvhepq@Uk1XZU&i^22fmg}^+^%I5b{rKWtkcz<*AfLa zMLJAGO2ImvYA&OT)W5O;T*Te~)`E{I=9@P&19{!J`}1@wwvrY0JD0x-J+(P#>+be} zAKTXy!1z1N?VG?%s}}-GCZ+-+>O+vgElfb`1cS=}aoix}FYuM2zkjJr@0^lY$WI_L zY_$aIJGT^XcH>qEaR9EgAzE1x{psm~>ElM6U|*%;x9on3m1r5*s2f#A-;vYRJvW84lYd;3$e|W>;R|OElt05cEEo^kB=H zjFWIr-h}3fT3?Ygb#Spe*l%JY1d)&E^MA%Xwne`2K3J=YW!vcp9=Y_5NJudsYF zfnDohEXU`A@$c}zw)M2#vD~Hi(V_5v1BA5Z%?$*iA!EX6LpU#7XNM1VUzc562V`U- zAWc6+G@b8Dl#{;@5a&WU2T^W<1m-A(qec0^f%dq-eMPu(iE-DiENW30!+oRd8Ss*c ziYaK|>#yNL8lVp`O@S=*hRnPdFhVKVO&lN%ZNayx026qB%&3<=_AN*CdG_s(B|Jf- zJOjvkQ2D(PNMUy@l3=h6TOdza4SmuQF>MQoyc1w=o5-J{CD^n zs*4S}HR4f9xGME%2tar{bhLX6zWVfty(*ts<=broBLj|s7!-q8$E#Px&_R{D$VuaO zh!DsCRXBE>%jlTST*>wne3aV=2C&kf5+ zZz%Ykt-mkXV0J@n2z#56*V@8>(A7Pj5*Ndl8IHRDR~@>|amSE2cBh4{8VQv8Kd&ds zz$1TuX55mo<@sKRzqEj(?*ea<7J@kdP#m$rkO2w>uEeUYuCDvrGZtcaK(}Aq(2!&0 zv5Kvl(ZSKNsJ>oD8?Ej=Dq)$S3#$kuTU2~AGnAGBZJ!aIF{rYydf~@vYHQaJ#UU|G ziBug>u3^WO8Rt-1q3YR1?Ho~&B(D8BlJ&iUex> z1k?(`Jhob>f01OsoNpjb#0dhIfrJsu4d~X`X@H*1gH+iKo91ev6-$?{e<8X3 zJ-zk*NYMH%6*jLvG#}gid4;W#yGKTYKqXl^)K`TfOpQ-9qOWF#>Z4GB zWA%jUnmRikuY=n?ecee7M3mD9`shmF%Ll1Oz7n!mpz#n5NYBV9fs&JYUi0l+kLpQ0 zP(B9WU_2A%V64|ZapLf5(kLmhZEI-okJ%0Q+E9?L(Euk!v z?|~21*I!SzH=I!E^SD(;rGaI2wI6Z1Her$AtX8nMm-%%qr=(O{M@LXhY$XEUi|~tr zA|jaL?gm#be*e|mflfgB3f+XWJK!zd^}&$26SWaibd^EB3T~#_AVYqGg+?n#^xP&c zSfeQErO51yfohPdR-65JUWy<}m;|@?<^eRtPG9a(xIyt%9qh1qHiMUi1RPHYKVTr< zw~UN6wc>_v@$**+LzRSPI2L6oRu>fl3Q2GcWB~x1AQ^4iNbi1!1prA{O)gcPLVRNY zVTfq~J|EvkbkD0c#oRhJ@g18?!(kvK%<~?1lnB4Ty`>WcHXraidN=KV!X#<*)a`?V zf%roKNl7+jhf+b%QXIyDu-8KK+1-*a2l>!4C#RVY|4BC0Bf`T8CUgr|YoJA53Z&cQ z9P-BkG3-4Cu7lp=aYL6jc~1ADu)%TT1uM`L9Q|?V{c56vg52>_M6^9LKK^jaCc;zk zW7fvugZUE9EBzXBCn`o+(!HQeuG(~^foT~Hu6$9JuC!BBW-eTmHx^^*0u37y395HH~=g^{5hL8LNguMv!0yP_7MSCN>Oc2Xcg9P=ar&CYP3u z4j#^8BMFNA&elV)_aUcZo0XK~aRN<_6DOU4!Ah{EzN@Ok!PiJ9J-B~=M~};-^27Vu z>!{)v;RiKbUkrL!$&#vhF}#gvO;Gxs0_v_o_3q6go7eG!vV5{hK>W7SCopF&iZ@`fBnqrm zY?}MO_KqPGMizl4UJ#eXq^5r9kimNe;l%L&^T{Y%6ybqhdG!e$0E#KK>DD}ABEdx* zo1}U}D{3-_bx15O2t{b^4vwvjyaFzXMK?K9b2}V$Fj!DsjbL6)Tre7+dUc+jc%-P- zYvqT;tT&U@^^+`Xv`0jY0YE?j>fZdOt>WUP(98@W)yxDpSc`?$g!d&<&>^x2zXP(?41SDP8Giqjs;>iUoB0u^B_xsQYgf* zR(ZRj--kN?@M_qv6Yip1o?C>wW2dob+S=MSXbQ*Hl|n#{^du_xxx+X?(ZSw6w1Vbg zWTYtEGsp3u(N441!e9ZwYBcE{_u{A;f6sKlZE;lnXs`z+PP`Hvg34*HUr#G^;;%gQ zy>RW1W^-fFpXWa%Kgv*3*kz};t*`CQgi=Ixb;+`8Z-uCTh5Ri#gvk~mHtVy{MF>`? zvp9nK7Ehu7;HwilIzIsz0(*_U1wl8Izur_g%}O3(v|V)5AV!+#nUH-}^y$-cV!Xt~ zC8R}#>lTQVh$a~UOcROb3b;1>^W2X4EiVet2~vF*Yt~%I3_5s@VC}T+^}U`TAfom- zeC^(iQYM744~&x$5GPEaCUY2>Kfr@VZ%eUFM$w&^aGX5%V=ETt-bOi7ufWgRhw|4nHsbp-}rOH?r(kb zIl$6Lw}wt*)h4r?9Y_Z5uEpbq-)B6?^78-<7FUyR4~{|-ExsehM>h?botkpcmn;e?j?pT*6ATb`?M<58 zrYo*qy&7x#<6MHWR>GIzK{~d(?7}$^j`_tv zh2wes?+mN4=`G_v@r(IO2bu8n~r@_%w$G$==?a$9hD z^0=?=v^KguZU9|Gb)Axg`JOm$Fy{nd#Hp8S$c921S^AztzKF)gLto6|*AsjlU*IzU zH@S@pcPutA5j{Po>t(O6AfTb5;*HYBo1Rr5Zi}0m^t2b7cPKH+ z%gc*#?64w~zwRQW9=Oox9HuZ6M)p;j-;6FRPIifU{sfvtxO=;T86I+D3BKE>Eoo3q zBOOFyzRPWXIdCgQluHk)K@=Td=Mg9?$e=`&6{CO74RfavT}4wC3s70A)O}HVTP3(+ zDe`0ks9^&JniL33N^yM_<_uogK(Otm=vS`%05`zr%2p6z9hmI)BUcKu?$cbq-nui+ zAuz`go*uCSthJ~-X@CWAl#?1mmJTknJ)m7)Rz_ASdA!P_wwpF?46V4(jk6f@1O#~r zHdN3XZXim5(7Xm{WJX_saDa-GWDXci%3y-TM=^rt zhRnj)^^cF8dbRCc#`^4hs)7%K<}YUVju~RPSkcdeul%)md1wQx(aV(2p3mZYii@DEG0@@2x&E6 zJ$we>x5k#ZGU(Yd&SQK9kU`yc+qZ8Y#%xT8w)=N<*Foq+>QYH(3e1j#gd&4)J`eo2 z@VaCtJcX;>;qUp%bNe^4K`T%4SWKPy8l_;J@oU5n5gBuQ%&rp?T$dM1#@ByK%++n)=D5KkjiiizXKy7iG!ZrE|1S>bh;3;zgZx*$saM z#i%s~|50tx@cnyR{zOYLU|}o3M~%5)rU`rr;|%;G;iB#sT#^IPgQ4ld38SNjOFhI8 z$Z{bwVLOmSl-TTRzZnpy9VPe39+l-wftwq?!>GYIOgO1>4L~St9^>wZ@gCWpOGr}--hf8uP6hR1;QXnT`8fJ(hs2Kou210BW zhresW_+s%HX!f>4aDkNC7}O6Ls}5QOoSfe&gzAkR0~(F@uw> zdcDX~bpMYMn_jWE&=4FWLKY+|5>g1qQpOnGUtvWBGt$k1BzQ7}(6oh0U0@(#pj=G1 z8mIBSkX>j{fWj||*@ziulIXPFateYh+=GLI!LzKO;1Z_0(b0xY$gC@b#+rD23IRUU zz%YlF(+f!p%DydkHq2&>x31e2Q1r&(%tp0ulRdnG$xcieTM^i>yfU(U8Rc_i)7tQz zC5Ivs9Gn^kyPm4sNx03YHD;EL+HsVESz^C&qt~?QVx?N~V?}dAxoC{4@o$am`;rpB zwrgtb`*cktW@~*>T_JZslrXKbz2fil-yTQa=j!q{;vOcvj*ey zAnO#aH3v)+%1s}q-YqWNPgh0RY)_Y`@G>a>>v)ZZSy0Hy+p(p;zud2Shl z?@lM9S5I307d_~9TrFv!66=%>Y^2{%LaY9<5Ir8osQzBLRx zmOa^5E6|_F;lQ0=Q{7GX<^cn(+-br0W!zD!yElJb^dyiyw{nrGYeOqVawZeVWmXV( z7QvQgAK{PtqHu5Q#!Z`I!2D;9LnA?)d$V$iHSf_Z6P5u~$d2&Wzq8vL%IA&?>miEL!$i&i-PaMTk2BY&`TzOOuS zp(BmO|La#R1gC%z@aY}{DMZ0>?5Uf)vlpI(41BT{nzL^J;E03FkHB^>lEbTtXsPw|jFHg>grD99kVX&CH});Y39?CXJ{-W* zZZ8JluzNScH?Sw1iIg9tW`@lo8)A&}p!;k5WpS~Zn%Xn^Spyy5E||*-3zvWlc#m^L zx&9roYs2#1{Oouf$sEBrZ3(cTF$}c=4Z8#|Mj;25WvJ5SQjgS#R+I3tW2n?5PZP4N zQ9rP$uu8^c;tn!TWUHyj30aR{5vq);SW1zRYdZV;(}4fqH=>cC91vH6$e3e#026Ow z^Lr+C+tkjvXeZTvxwPY$@B`-QJt&eT*x#m2B09F->FMTxqcfPe^XXMan%E4FxH@h< z>Kg9pbaA_JO}K^j%OL(IcI)m;XD@%#D7Wi#@Y0X!TMM6`miLX?o|?z`@P*&y)UHd1 ze7(Q1Wr?gkoyBO}e)1T_Jwxlp<4b&MlU*&AqsLCKKPUAl$RUZ-R9N5o`uxq z?Nt=o@H$X@x@zFg<1Nw4rd7EgzSIpI2_F;~KE{1JUATwm;-HD&QIE#69emSoYHZU7 z*gN`e=^c6-YCdd3Pm#)T4{CYK^?JklC2R)b%_iy(R!&r1Y@g3jds5mJ^Za#gOPcCI zv+I0(H|Nvk>SxlgFyB#G{q~U?E#BgRF*lhVYsJjhx7g48_A+(e>A?3~#BJ}cw!U-T zZhQ8Oc?HfryBgjyIhE(nSX3iF?$)0D%dbHue<%HEJJ)>Mg*J=n$cQTv)+Wnmr>f@K zJN}xwWhEtz9a0rmP*UPuwd(KPpdjyGzgBTE6}%l#e_G%aZ@$y>fk@Tk-023#)PDj&Bf<>iT&uFi?IVU}3f@>5ztavEf3PyZmfSNKKkgx9IUhJd2R^AyuS< z9f@}jQN7{1PndvQ;`IV!i5!6<*Clp*@jd3~a&5^9%T%S*U}<=B+AoQQ$S#f-WP7Fy zAi@Qbknar*B*i55EkHR%0P-MAgpGwE36J~sX9xvJ5Xt;!-O#5= zK=iqVJwnJmAx9IIT)ONUh^);uSZP6=lY<$SECQNPZGd?Yf+PF96c`oC2tpMc2_=%6 z@tc(l`e_#q)Weqg?9)N1t0IFozKsN{0Tp{_S zAa1@b@~qGSu-Pqp^quqY8O-pVAmD*g7(rVlF=ry(g7*xin6F>f@i<=yqS3%uk`7H5 zs5FwZ{XnT7BFI;F#YRYkAUTK)OGyz2*d~nDbvhZqHC#mD_wL<9!Pxa_BzkYSm;C%N z1Lg4KI7V`VczGdpUiV{BhJ&g3X~PyD=Kf6wYWb9t9WBfhr#N7rAH zI9pX6zXp3t$#vDiN|N(@PKsA+ufJSgdFW%wfV_dy<)gnDOHXp>2>v;=OlUr)im^0m zb|?L_`X8QUle6{S)kA$V!PmB~qv`ZIS95Tzz&jCLh(@ukg5f#)_7NKQr}}<=dIKHP z8ka+7-KHOp1lfv@Wk2lv6fpBK_snEnrd!)WNldQRM8^#a-{kx;>zOfGnw9efGVceE zxC~@J(=Z#V*tN^5O@SK&G)xLEhLg>7bSDsvwrik8^w!%Oi=jR9GQcX_?2n)ZStXg;j3qeu)?+cr4wh1`;$&4W846kYn8QpTW zKQEP^6phqyz$1U{&=~~`j!=Bo(=5MmjxLe$6aAQ26ZpHP~TN(O6|L<-F(}Bnt zCIYk~Wxs|b2oRtZOSP`9svzMJ+VM=po#^M;GGl{eWPsEb*s+5h;iI`m%E|hW8^WXH z19NVj9Q2PE@#1jp<6@M-!7tzz?J?_a)ZgO-tMSOVb1(s~nuR zP}I`G3*J`=k=7elsLzBMfXl}eh3=nPLZHE*?&=V{ys;S`CnS7)QH5g>%_~WA+jK}p znOSi$$@OVrCPJd_U>rHy_U;CPY4>3=xclJ21*5T2RWqCx?>{e$7GOe zi4TyH`1tX=D+5ZPY@rW*WUa*}%?2tH;gl#l85tNOVS~^s(yn0z&JV`bztI-e8C`AQ z>7DzbJ3@#@?9XSgZ()B2ORUK-d62uy(0j0FVEOljE|FN>Cc8bXbyX|@1HzYtGwiPm zvfz*`KEFdTMY`ji+lRsrE!=EzEqw-m{PYI-yP~G1e4aiXJxo_8w(r)f=I4!JanBfx zUFO=@+E&EJF~8>CtNKbL$R%Y3Ip%kSR!t4->H1IV8m@B^dF@oX+BNcr@!(U>BGp^3 z-U#o>4)u?iDR2#Z*{^;h_sz02x69mJZ+Z1UeR$X!eam2`*Q{FZI~{+1)4Q^!l{jrEV$d_PZ&Nb-l6{cE9;Y{)Ro$vMHb>nfj^LBx%t zkRe|~*(Bsv9= zmbVV^_#x@n>tE8ZPnfV-j=RLgb)LJMa5uv1W#>=!m@bIm?#j2y1NCKrIQbnoj|fv2H(Y-%0tXTi3bI1f38dCo37pP7a60e{z(f<(s0%KN$l2hD z!*QQ+6!=lLW&2VjO!V;l6Tjf{QLB4L1q5!w6;)JRjP~Z39=Poy(05_)pHT46){LSv zCliPLbj{!MdkCqFs7E6+v!;(_2Eau5&`_8);a9JEilmX0AgRSs-y9nb!Rt5S`GAW(VGz9(nXB*?aFlLT{XRQu zd*ckn($=;!V})|fB}Z7wz=BlI*$`qi)4Uice^{-}aE)^Va)QB~A<8)JlIt_@J?*$j z*xRExN!%IlXyY2l?V(XeC8sQXc_Azq@f;F6J%R&y6^R4VfYW{2{=Z4uWlKEHH)w&?)&HI10L!64V+!?tusU4v;Usn%)m zG~CR6(V0>3Ff0D4s`uCtH`mwgSIzWSad|sCvFB<@g(4YvITvL31R&K5gcAwckQ}u#sM|G}o zEzGtZ@9r_7(TI=WOEtL1MME4pUy<;vwf>u5_!hF-N@Xvh?;dnj#D4* zqu{wwTFiFFoQENMC$3E&6nKJyAPWX!?c29+V;h0|Q?n@$FjE337@!&{N5t+69Y9Hk z3^qE3;0_r0K0t3Ka~bx)T3k!Vo`DILs!JvD(wf01;PAk%Fh=XZQN@UG03+ZQAkMF( zhkO1YiV>*W6x7wfg&(`^)6EIEk|;F@K?J*wW;7}nt_kx4^r#rGb@+ae4}8xPCoE=W z!(j{|f1ris(mv|=x&rwpKj2rpqgMY?F1rkgvj~;^6`QE z;3!30AuC3W5@;RmTbG~$X|CoEDVIdzB??Mu$I>p`+gb`1`z}~G@MRz|6sh^)MlkW4 zxR5^tR9V4s_Stmp8SQ#RbH#2%q|)3g4HgE5RkaA>Cw?3*e~Zi^sp@j*<%&_c4`d)y z0JKM73L>CPPh#A;4ah=MR9Kh+i& zwY4W|UOH`ikRX{jW?}2xQPUVM(e3)kGHaUU#DQ$UHJ76HZr-WyTJWsyzV<8tRy*)o@N za)lMGX)TgBe&i1qcBX|En9AfbF}oK%pU;@be^KajV_sET>1@|q`!;Bq$(OZh*olfM zF5}a~D{TwEGRLv8_m4D}=49Et2ZC6|W8FKszuU|;U&rm&!5`H&Pkz^Lq9_N@7x}Q{ z`M&xwJrz@uCY8hF(dqq@&o%63nj0r`^f4X3pOkmHiSM2)d|Q6zoU6-K^&AeXynE5% z1)FlLw(unCL{TU=ou)T9+beinUHJHnK4HsO3T2tN^YY3TTjpGsBUjg}dTUxdH=hV) z)fzmNU+I3l_iA0{nmVb^3#OsnHX}`QLlL_phZ`;yL?yj-%HmqLv)SP)ubiC7?EJ)D z+r+`>PaU%SnjDfJg{io15@%^x92{?7W*%Ywi@i!Lt2;siNq7(JPiIS$MwX8g^3!BT=IbZbcpCy zo*;38SmFVo)z;yhA)^9hFo!wsBzSE{*0Q>K3G_lik&$NxWze)j8A;&u)O4qrT}4>= zBR_xsq=7IQ=GmG$IM~>@xVB)l$0rj;(+cvyvZrQiO-)VcSo!a3Ok5HaEK)Chg9YrX z(~#9c%3->>f9E4Lh+7qB#>X_)<2ZZ(ln_$^O{{fh0f?7E!ruP_3fIAGkdS;bY29xJ zbJ$i8P>#W8kK-tuQ}kqtKny{{Dd3K>^; zhiGA73N$%~+f=+TA;sde5g8pQs!fLo&8J+j21;8f7do&B@h`khKy1qj@N1>`-__6n z@lL?J@=Ae~mV7vfu)t)dc1KKO!Y5`RqP;|Svg+v-UV`wM!@fjfKFl@&ph1uoLNJ!# ze!UDNmtMzg#||K%vrE8~oe=KeZ0{lyUC(K9h@VUmH)xhqKkHDemF11FDAxmH^>o_aEQjvZqw4G5fYY<`Ch#EJh+hY>Xi}zPk^IlW{~=UF>@QR z#udAPP7}8^k3GIsMr-eo_$5D&Q`W}RVB&TBllyHwwfcJ;wCJ6j%qFLH13#9}&fd{% zqt3$j5clPe7(Z$kZFzm{TWPiXHk^$Xzn@g*xo{Cw5v zz@&k+ufR#OcrpsLPh7U@RO67^Uao2I62UkO)zvL4SlrSb{49P5zpn_V+p^_Y@ueMw z<>f2SeN8V6Xnhj@K!Bg0uc2X0P&pm>%tJGHkUkNokvk;?U+FIm!Qtu||UXj^*Fi z`;AyEWU{v!I|7H||7J%JTBhA>F1KFGUOG zx=1DVizgJ$oMiT>T(Qo{EPG1sT~^$>z6is)i^osv`WHtV&``Eq`DH&5?Fjs5dfo5Y zCyB8 zmYbB%;S;7aB3qw4B^P3_(9xOJ)O5H1NyNeyaU37LoH{X?lk)&XDE&3-q@mHv%x~WG zAr5||wJPNPCCfiwSO4lf8JnHWee$G~z5SNz#Nc%Tg87e+y)oFCT<4yCF&+KPadu9f z;@Vm$b)$T4|8*Uh`TkTjNqWQ9M2g)2d(4w3+~AO~uCnt)*tohqeO745$iB@t+p~k4 zM@;`?;m^|FTZF&9D=cJaX|e5YZDlbqVBT&xDjvO8>o_=kOZ1MV0WZ7js|GU+moq2l z>IRmFM7It9e70p<4T5-ZI}S%oj2L>$bt_5C_lvReiS4^{Xg6)D9r~-qzvbiFx#_9= z;X(J4X7Bo&ybc`|Q^dC9_U8=3LG6kN8C<(nQtz_6|TM%`scAff`nr_Wg<_d@XmuzyIal7ol#KJNv zqwFrDyPyC1mpiXzH}0D$@J?ROt4haSG4xoH%yQ(VX$IdI$;dK7P6DFBQj!5EodXm? zg7h$2deZyiDhfn;gRJ3!3|3!{=u|<)w*9J+;oQDDg9D)=1Q{mh7KqY88HD-3`oS+N z8|Ie-iLb6Vg>$T5gqsx2qlEMm1Bm6h6cXMP?G{yPo)_5z(5w< zVr6Ou@){X+^JQny3Wvk>iTngBc+C9fz*`jt@$cUdVttP#_MvhjyM{ty^(F4HDLO?3 z6kiA(Ax8{=RJIMQGP(X@of$|HfVjYXAnqY5Oq@>csJNkvC|oF#vggOtuR?H z3+wfA_1@(aBk2BcXBL3rvryBLX*lpAd1{B%;);)n3s2_5H{rs*22c=+B%HtZNU$v(d$R3C6=2+BTfE<^%Qdt29T5 z)Ph8yLIwpfSjHGgx$K-A(gdt}8dnFE2=UyIEdK*trvX)a6SNC=K2p&xaT1#h$Ecqv zgmr}JW@iv{g6zt?;TjA056++mCyeX5+GF1T{e5?`n9mV%PW}x>qP+619uu&e*wiT^9wER*+^oaO2mxUTrp82YU93TI6gR!q)>+WbND_b_wIVXFu-NKOb zA2DyxV>UTE(u=1T{;^E5VBrsEf0G>l&P>65Q%{!b>vNl2jhuEpY3O1@<2F?;?dO1z zwsc{xz$MK{F}uVZ4jtnv-gV7y=lX0q+K$~g;Ky`qi@|(u-uLhOzrWp^Gd?A?Jw!n) z#Q4i&9vQnz=ao0!y|Wp++8dJc$Km<&*OSlOa^3On9>3DU&0M?#0Vk4foqb1H2$_dF zQn2pD;Uc#MyEi3s>xcTxW_}Fjo-4V1H8s<9eu5AOu_+^;A6@3!ks!Ki^~i8uYh>D_ z%iS{C&yT**FU+;lv=4~&&s`hKuQRAC3l1D+S+HE`>q}?if;0V8?rV|xxz-t_#Mi6F zF8ikxShmEycjg-X&}iEg({IpY)m}_vIcLO8d3S#4Kvbo2^jh(jh1uBHQyzlRr8E@w zo!1D*2`ayH#Yqp+)CN<;zGCV^S!sW z^)ToZa2k4(B#_^whF#hI>v?DgjY2|Z1PpFYUsmrg=-0nuY-TZ6RQg@^RB>eq$C?9G z0~04g%+_rS#0kiIqjJl~EqJ@~T~4Q?pMN!-9j!OZcIsgp)r~vtHpAP(wNT)=ccSa^ z;XlC)5e9aQSsHtl{JrMnC@Z9vyhU4Mt}o%J(L40DY2Rq8PK=n-cskdsA)f3k8|U~d zRdGpm=LVz2leHaA_|>#(JAc8ts7nWDLIm;QoNz#UJ0%{)J^x%29g zbE#@VMxX8!e6~v-YkNDd#j1Qw^TzAX##{59vTmeU6rYK%3yR*0-aXSi zFK}VT?I*4p{5>)G@?|Enw_i?nxed_IgnSP<<(T5sdy?OEVJ2P1MRxTES@}UPM$rPp zJ&?QvQddnSLT@cD_uTf}xow{F<* zjxE|%mTUFqTc@0z6&2SwyA*iMTevMOOqTAO3~~E2qE*FAIdH)A_2m)f>$w*L7w`^z z7bXMeVq3c{^K4VWnIgUgdpZh{dhqVmT}nBX(X$LS z$Ns$Do(L$Utbf4)^L zJbY8xb)6D>j(=e~C4bmrp>Dz}aB3j037;v)bF;>0`SPi$-EDmH{0w|9HU05X3!M0n zu;9`3HHF+d5^}9)ZXZR#d*#c?Qnx!FnsXxgS6En>cZG&3d~Y}%^IX-t<1F(^9@Ww( zhrhn%+!UMjzpSpc(x^B2Q49>@Rh4w<}?ZA;s`Hb+Gieyvx>FbOJ}_@x26?CQ0K zml~QCl1(etsaCTzQM;o|)h%yG{hamn>qHNh3NtPo3}ZQrk@p`ygwSL4WM6ufC8Bo0 za_ulUw%F+0yAR>xzY9vh)xsk0e2u7Gpsl8UD2#7LQu*_-Ysj-1?L@v4LX3vyX9f|C z?7ZY2=1wBpCuT!lb^HW{1W~-1XmK(rRU(O~$iKP&Fid=vS{3B}h;H^X75&|)EnwUgm#rxr9c#}hYRVfHf|27k$agv*--VwX`+IIQ`YTn;67QeN84HWH?I!@E$|@!fF8hpOEoN$Y>)S$udFZE*1Npf%P-_jjw+5I!?6QvhLbe zC26!HEcNoYGfeBVS9^JV{qb|+DPkyORRWs*-z%4=%zaYjnV+twRX%7TJtLcOtG=Kv z{dHy4`_1pi;(5DP3-j77=$q^n6qTC{k-79Kx_^NjO1s=jbsT3$_PmMmJ~24wxtufI z`1Me}BsWyKLaU=SeguBjt7@&UNwA?p(Q*P$^Si$XvA;OMpsA;4K4mw}*~`JVzxn0) z`{(NyoERv8`081nwwHhRu($kEV(mzNZ^!4qjkdo+A9L>OJh6qx+;ky1xegcny$%DY znYW`)Ke}M9rFTcw!rS@>(VDkX&a!lLRx@>HsQ8_eTJ<+ocznY^l*8=Tru@I>q~UnxBh!4bPMLP2*jgJkk`E_TBb+O@en)uK6VecF3F9g!Z ztne8%{nB5*`E6xuPJ(Yt6_24-SN=`AgeljrUtK37?e-}5do2HExHI+_Ii)NX#vEVM zufDlS?YMk~uD9g5@w0U=1aV=+{KO=N94aG!G;-(Lx{JP|dIS8jj*Lg6oUaVK7@zt5 zTbgUT3Co9oAKn<5_Do2wS)<$6JVh~diyYND9knkR_cq84p~5R#rG4lKJs9{&^77Xu zFAO3tr6EdgrQpG3HJMpgau&3z=8#=@FXr1^E?@ZeXCoqe=d)g=&w)_ zu2cA2u5&kx&5DX`t7)5?&}bW6nD0w+rMLJDSlw-5(6763YlM9abbi z|8q)3Z3}2=2v7OJIJ{}8;sB~~Dy?J3Q;mf`@vax$(x6t@H#@|RCo$VRNJGKaB<*de z%&>dIaKx=A+Pc23hFMo#GFo+Tal`*ScgoD_LMtpY?-nWB-q7Xd&J$5-|=5AE)S-L{u}RyOK$!y`9-fau);& zE;x6{2t;JXW~i|wGi}?^V`|Fc2UF?=VUI%~2=1Rbo&OV3zAsr8hDf7TifQ}(7G>EK zZYNd_j$1PmZu$J%z!^gG8bsAS9Oj2U0JG(DJIOn+G@gZv;aH-!e?(vZ--(G*q<~O~ zO-PvE@P_$pyg)j{PirD%Z5UMIrZyVEKtS?3bes1*@^Qtnbimxdb1K+C8gWbkFulY; z*ns=rlyOHAk_3Fi!no7ZaI^u>gHu!qkSOb#HC~vbia&i)z4?nAAmCO)5f(+{*W^+n z5S7Jaz4a9VRG)nYp7C<(l&IDP5E_(-eUvtiCcs`23 z*czLgi)yNZmUt>6A;F3nn|SgbpIpejbP>!jx%(3Lf)|!2S#i8EI|_@^=a;+NI>wYY zZbFJEdHOhG*lOV)^~Z?*4i=Eu^EF{aT>$~;2b}O`W}qiY&^yBV9inOQE$xOc{h+Js z+XVPrsYVKf1tgaay1AWvZJDbtW>k})z{LFviGzB89DR$usk9ea%ierZtM;~h_`r?( ztsreqP9_pk(76=%?H8qxl4X!kTObzpBi>I=C2*y((c{(UrIwVHZJHb|UH-AmA@kD2 z%QGbGTh1z-3AnJXtl#~Y_0gj>p;Es$$lAG1{nQ2^)Lm~OCnXhCSLf{9C*kTW zA!}!go2%9Ua)gKoS6Si)@QHyn=i1)!WV^cTT+d@}(BptcQ|5*4gB27XpP~2P)B3B? zWSW{X`S_KWR7RG4`4e^Kna#w@rAwA9Q$Fx#_*dC#T3Xsu%_jR7B1EfA{mDO%;dH}_ zcR~E-lIIWU**i7GynfBClhRq#s5`IK^y*i4HIrI|Jw;mjR%6(+XAI{)ygnErHehL$ z1~hdg3-A55j*|7jF((_V#$L@&mX?3d5TT>6a`)U!JYLMo+UA4#x-z&u&zZTgeF zKb>88Sn;I&RRT9_2%GixH||!73CjOlpLDW!XdrLKYQdHMMe=Vhxy-%5GiY>l^nU$L zBoDvi=Lm%YZgixJZ+}(PzM7An22K0B%|CH}|{B3P}SC_l*YLDs|50Muyer4OG zaOq_>@7JtO?{Cf1x%0bRBY)JaVC!xh;niel0lu7_vsKc?kt_GQ6(SvqD#E?KXY?-l z^eKubrNH{x3c-T|AAXmorDnM7d=@jlYoNA49j$D>;074z3&Sa%a(ggz^9*b? z5wa(WewFbP3^X(I+LI^u zJ4*TZHi+~#%!`U7EuEVgRM6MocGKgGfp9N@iw5%KzL+v&+mv|H7hUlEJ-TVU z|HvADppl9Nu1!`mwwmVZF+s*437ELrmRnfJb&si#>NPpJCt`Q-CdcU6Jt~3{IDz%w z&eGnb+~Hz#FZ7th2nhqHbO5i5&+%tcW~C|m4$J0dX4=uze9n}w2~iKVqTZg5wW#-W*}c^1k|k4Q-qH78Hc>~}3K45k^p z4h|=>ZIW0oR+-ek<@^8G`s=u=_O|U8o`{090)n)F5&|OKAWA7I-K|K8f^;beh?Fd( zr4*3v?hYyGZUpJ>+UHp7e&73f_Gj~ldR+@9b50n)amI0c4~y%yZVJbL+{IAu)>l42 zjRd!dPknu0{|fV;Z>6a>bUF~a5W5URRZo5DNgVun3x$S4J(Unw6{Fed>nU_QP9Gb7 zTI*4Bn}ZHzJvQ>AKsf*=+JK`Ym-8I=Ior?0y3Lu)W^Bxw?R)0X8S6+acJ@Y5H6U;g zOr2<3hmP-qg08?o{qDV6GkeROV~119S~X|H4r?+w);e144`1uvK;ZpMw zqO!iY<^k2N3Dv#|-<)u>hqj~=di~SP=RRNi@AfJG<<4~_=BsJMoK7`o&i_Ar7~BAd zzeDfjdSC^yl{nC@{;T4J-VHoQ2+Wy~0mKu`NO%;&CM<#AQ*Vg4baLTclRrl8=LkY= z7L1uKFxX{10A8>$KzobBuuHh@Nkt%-L<&TZKm}#21MV^|qHKWMDG07RPoNHXXfeP9 zWz5cq&~u!BLvMqmAXW8xbk;1xh$|78lMoC9EGOTt4nZc+-6(n1%K%2jcx0i9WH3SI zBA`nM$|`E6O#f9w!P9X&^E}uPV$ERLL_cyQuO?lVzd_8JIFf%j^@GDfMXuyttd&EU z)xl)fOPzwzCe^Ssj^5ir)qJYNLUd`LZb`aq;#&VY#eD1Af%}l@;hpK8-6Q^`3A>)N zGP~32B}!RYSpV(r={ z;dG@V0a9YYI~a5u-20(QP^m#FbIlonV!=Qli1_C7CD<>LtLF~A_I&+10$}V67`Qh? z8=Vg)>qn}N86w!l0yk@G(egD%sM(;zf*gmKJ)lrOgpk{X#b4WNg{QQUEGX2_5Ccqp zx1+>rD-~EbYiT|Jmj3EcPBlIXH0R?{g2gN(?D#jo!dluHRcCPNLgINk?u-K3TR1~) zF)@icV9ZRE(k-M-BcC-NEyljptf`7joLJ30Yz=1_jS{fqgVF5URrL?z05r8=)pxrA zDFNsx2|6H)W5AJvQQs-Yzqe^{DM)NN)HFX_D~g|@90V)jtuaf+{MT_tr(KTIA~;-J z%tJX;AxrT%Byy*Nxef4N&%yY}3s2w*N+Ir_Bm=Q)eSLkfv&mF3P*89Vxa>apCYFGP zdShSQ6in66Vfx8W+%uOJrsa9Bdc{@Mn9*KEUXI%kd@L`Q+Z5)rS)Bh7^56-BWwq$| zl;`CY)XaJ*=3Mom?_;{&5=__i;x|n#FO3#EMPy_%-5DxhcBys7Q9xQ!(cu?Kx|;wi z?=7{V4yNY0hWf@I>a(_H3-(Q3GO!{K^CU=z97ql4IK=FQh$*0%kPme?$u9tLdxVhIJNP>gtf?ctPjnyN+Bg$w`1cF6Up z80^wYDyP`VFyIO=*40m`t&OC}j1Q-pyZY0aI$5RVQ?5ifq`o;QmMj&g{>w*? zif{;G4`{WYUwG1G<}ZhPxDgx_QR=|ins-m+reHct7S%pMJpZ44w6d~uUi`B|!mrr3 zEbfPo?Dp(ByXh`i{8 zZwlsTiq|@i5PM4G_&<4}g@>A{ljhJo9(0nC-=m?i$ROt5nwE_z3M#zzn9zVvmW$CPY?LA7*>&4ITWQA0O52t&#R#)kwL><(azI&E^c#_GEj{3@Tf6Zx= z)uYs2&~*R6HDp3K?sJ%Bu0>y}I%lVw>%+&4E0?i@7CalRrn3t!=jTHMjz@)>#Ae~> zs0K5P2EKt5e*xT|s%d%in@fo&FQ$3^9HJY(HW@SC=UZF}J@q=*ttFd`StI&>;i3FI z<)g%XUBCTxKbHH%Uhj;JYZRgnR@y>MyUgI$*3h86afqqCI0j3l3YdZQb1$Qz>w4AU%3b#R4wT3ug?-wmqstnU3tQ3H21q3OgG7Pys-In zr`q<RzBFTGTdR3@e~T2o4ao{;e@WGwZmW{;L1yNKOCAWUP#RnJlVIq zCFgm*e*t?nLG>JA_(fw2c{4N2xO70NyZ#~_iIFqCt$HlJw1LV&(_R0t?OM~O;++c z4ltB%*ZMsVD?LDBgZ8p>Bxq%2%)KA3B>}mZ0o&Hx7Qp=8I@@~kond{}$Om4zVZ+sP z&)COF4Ij`C^UYPhPc3J&aqc9&)WJZ}&`_*Ri4Y|vHJI#7U+@ycdQIzW)qr&Su-4#V znMc0nM>4J}TPvCF9BySA-mAv8wj3}+Oe~MMnDvuj)fpoCA>}OAVlY@V+bF`5AS;{s zc0(5idscCqbaeCPG$M$7;{W;xeI6YZg#a87cyf0U>_q3meV~*{Q;UXm^QOHp1d74P z_QnFz;R3aUOIbpKN)b#AFfs5!l#1XSdICYZQ?_6LKonj2&VOzJMCmb|dA08b+`ECWyr#!gFuUMQ)yL+f$iwnRfi@`Ev@vQ$#3E1 z+yQ?Ed-Wb6lG6?Qb#d0AX`ZL>oDo(uqNGH4nd{@FNy_PC{Km)~9#mnL04PGZyYMym z1O(^a9f-pTd4`}8rH$m9Jmx{(wqj}^T2!6wbbyg#C^{k`Ap|Tab5MC%iGe69*$-$G zPynEjlFp(mpp=SPT1XC0V5hv>)7u-mDS7jGB{Jbhj1*rql$E`ZjsD=^pw#Ejf<&P* zA7Q$&Ty!i9wa~cW4ftFzPre6L8rF^}C)r|m9B{E;CZ~N83~`(i@Rkm-Fy-XHr~c-R zhCZ2|t)weuTUV!P|%WTD{n3)TcT!hmm->!&iA8X*lke=6M5t~{EqdBE@HgfAz552Ll# zT!k)}MzJ?p?8xxy#l_+EsOYEr5eCKgiD}%qmBTJJDzAc0>eX+@bX5zhF>bORk)NlO zo6pZKFAg8dS7gZrViGjr=}Q;=*<=m#f4k78i%a(X@;twF-%sq@WAP6q)q{}+!Ohc` z)M$3Qz9Et#F8mOWj&)nG=!HqTY>KkUV20>XtC+7iU9Yv-U~U2nK2;2!ii-90TQD>* z$JxTq?>e5^Bf50G?q@0F6ufu2kIy+cqG7?V5#}-LGZpmJ!xjz~ef=S@+>>3sI)^#K2JkK1t|B)Tq1t}u;*YS z&SztiirbmS?S}Wn#F)FpKG9k~+j`g4Wv@VG??o1wVK$8~+IW77#=I`Rg)?&Sl0v;Q#r&Ycdp(6=+coe-6b>fE>Q-!Hf5URDbnajRg;t&%Q za4my{cx3-Vz^!}%^1O(XGPHSTc8J(JVRxO(YAhqnv9-}p3^99?31>R-uym+52Qc`v0!>^QnS3&+SCnp}vN9#7lGYO4GeQn#R95{VRQ1+4)+WecXsP&^^!!v$3 z-X`YhR>Lq6a+{vo38`uHd`}T?z?IbQM5EtT-)vocs>~wIC3m7%%dlh~{FjE1dsVSi ze8X!!zqom=BJ-Fv;?oULk;Krx;&%sOE(e|`2aQmj;`BXzDv2Rw?ne6HRY-reivUh= z;kvko$F+-N2(&CpNMOUKiN~?Ap@=I;O;v^vXMW+`z=N?Bxc691zdou2F*fx`iTq`` zf}S7#Wb48D*t)}l47-F>_Yj5N~Nph)C5rw@g%(9oRiNce8OPi2H2R(*x?#co?VLyCdruesw8O*24<+mA zvc6gDER1@XG%MN(-{xiK^|})4ejO1}_Z9uH=vY|@1gHGy>(I;082%ZmO8iDy_I>lA z>JYwZ%J-9%X2uILn^MD)1_nBmqQaPHXbv$|J)!|zN3JoM(gf1=#paXl+&4u;_71Q7 z*EIhhf5rt%(A6VAN@)G#%+0#L)r0b#{|od%j9jmN--)aL7D6S|GLQD02yoJG0fFbX z)6NK;)?;93OxXge-|1k~Y`agE9e|I?^=P7i+lY~6B|&WYURB`Odz@}n!NCZ}zP%i1 zaS^WolCxbsoTRU>kECk>IvFor)sxXlkqAgF;)0O7AQO7yxtW@ZiZ|$NO@IXCa@YZC zux6mn%tFG;$G@N2ZnLtozS|*p@%T;sNrDdIUrm4%?zCP!3^t)xaZA4T5D{YpY}}*0 zenmt_3_M!i!2dxl2iA{|Qy_`a;1f2o$rWPq1`ZX3MIj_!@sxFE4QZH<097f`*AAEh_^E4=!5ZS4kX7k~tU)?(dp zmBBQgRq0P_546Q_saASTbralqF_g`%JvBkzJ2O*%D&O5d28ZiVuHx@x*L~ZD47H@7 zZX(LHkJD!ioim#tsOpI6fnVj>AifhmfgT&raZzn6{WMyX?dY!-9Gg))tWXH_c`ZTImX^>~j-Pkm}&-L)vB3Vrl>9;D1 z*Fsn*75z;DBD?jS;8AC3oARB8XI*A0IAjVr)t}vIbS30ptUItrd^YTFzCz-V>UEs) z%ks_4kK1dL5$b5q{OsGCf!%7ORg3heVq^L%l`adS&ukZB_R2I)R;)Tg zWS{?N>OSnA>rHR}I*eI2(NeS>9Yf2Wa<#QKTcq@#=%){N*X!BT9aT6tiSL}J9vqvM z)>G%%=UbTuYwt&{SIbtMw%->yoo#FLYv5FSrmEo>D2kq|+dkxmk7e%J^Qrdq0a|$Y zSP3rFNd@h@-xS5D@K{*ZFWP?Fg-Y*=3bwxP^rLBcC90z-v)xD#MlMn!7A$sde&_6L z4Sr|(WXDXNsD%^4K4pci)b#16MLW3f;8VN(epL;2zF6LS7JXri;A{U;j`n;tGCXB6 zwfGf-p>YZyvi(eECC;9uspy?UM-&6Ihzy9Lh6BNk9S6^E$;ZHc< z>2PZM^I32?e`~+kdVkYh7mBA(Aw9jx2JXc)F5GX|P7iDSc*@ICnRkK6ybApf`92=#><_eU%uga8p|4z^1q zB3B55k#0o-DqILJyw}#z)g`H``z;g-@@XU(;b_YUIkS;GR9N`!MeKs<2!bW@jGIC3 zj`3Kv^cFYhMsctJ>Yn_^R$ybQ+Va01V~~j$8`*p!d5Zv&?|!GEqJj`WA?<8aHSU9p ziwkxi_{#CV-lel}hB+9LS9?q|l5YA>qc&(C6UO!i|n@_z-xyxBR2u0PSXRaHC(? zwdHUYS2FH|8e4b)u%RVdI6vEEQ`KNWLmB5^&#fNnNjwn|$>wm#DrInA8+qcrXep0{ z{dv>}`6|y#DP5k0%@H`>N0PLFS(hS6;1_ zhR43~BE{ZmON*buul5{rr;efeR^_VmC!5-?J)|Ber*v{%3_rDbb&?BWVOW!dzCFlwulOFXZL z+q_@`J|(6HCV~{Ij;?(_{Lp>zcC88~U^ce3Q!tcFlN~mg(OOH;o03Vjrpv~&JZ{$i z#l1GI*EwT>rz~&1M8TuDN=k8An@0_M&FzwsdXh8xqy-hF!&gnk8!5xu@Lp>&hSltk zVem}o*!iEyT{5M!g6Ztcf+Fo*Eq!I59|l7Q;}-K)ADlNOXD$`knsodU>MI%Jn`sC- zo44+X&_=upc|lq-*^ok2(UDOObFGppck&eHTQOC%I4*TuBg_7)IZ)Wti|JO1q)fK; z>*Qoqk=xsNP720-7Fy~vdGk`XGtx=huw6$ z!58OpdLq8Avq9B`9JZ1+GN#6;_XScub5;VM3;!tHS1rz$mAqY3#HgN~fEz*`9HchN zI{MTHX%ytOS#KE~R6fBKG)uRv`<8mx$f?47qR?A%*j1zP(Q8jR&UQGM!BR99)-P`q zlnRZ_J0U^x%WmRr)NkuNp~s94*7+4M^m3?iB>9{%W^KKP<32NEKG~$p#`toWN?g!Q zs{a%l^_RDe%DytPdTW<$Ko$FRH@B81IHae(SFpC9KE?ikc^xEZkKY7utXrEjz1quk zeK5E2+6M+IG2dF7n+fe!to(X=mzkJ*r)%Hnxu5XUvt}*Z#Z?IlD8vuMqd(9cLD$!3 zt6hs=!!a+AyS&Kpa#ww_Bn74Y)^6coGT-Z}-dELT-O2O<CxD2hF3ncvz_l_8MmA7+uf+s z*CS@xqj&{8awvaWR+43(qQ9xs)I`{e4EgJ z1H}ZmijiVVQbepN?6g&P{rSJBA|;`}lc1r4+gB17WYb?kVvS+c)OvaXW%E98K0y0p zusO398XAfuI1Q|{()`{vSGtT#m-b$V(-$zFBp^CX>K0N&7B;Ym3nac}&kLh9zues1 zJDc!WNXq5gEAS&x*o2<};`P7N0|{*aNa2M6Tb!btT*P8wc6PeB_b7T@Td2OvuD91m zZ*zm{45}M724bYJN*GE=5l6?YFc&qbiZb7F%TC!SxI(Uvv48EFz;&qUW!}B3SS-|J zZD{iMEk)0&%z8c1_Qct_zh?{s&(^X47j<+rVlFDSf&n%LHKo+oYuV>Qu>1teKw$foaCJzpTv?mlwE6jT*uactl52lot-k$fO1nYnhKslT z>v0Fnd-ulnmyefR$%s3`!1!~GGkQbl{l2r4CRU%MWW_nL*~lQ1j zo?S~-RoE~z8d(tbeXU_{!hS!4lRx#K;kUi=T|%QXCjoNJ1gyYSV zp!STX#ovK_K5-5#KKAZ_GLySH=5&muVc$g8V0xf59 z&1l`!NNKc=cI^Bg<)p`paaZCv)oiV4zTTHyx1$%*kgeKjB;Jc7M-kaYzpRLhy~R*X z9(;Ux`$5|L){=&j-t+#f+8;I5m*wwWNjbUA;c$yO8uw6CHG7g3SG4{zz;3~%L$$-l z-R}OaViu#B8w6EFuL-YUR6Jq5X_@M}_onF|#TLWgRU0wFFplvPEuSzIW*)Z9dp-DE zBD)sjg?uARC7iO$n?zf)dsJmE3m#$nKH)#eTyXi;Ci;6jFeX>OvdOSCch@F(XLNU$Zg}v6zO_fR zlIDb(M?>}R=5UD!$ZT&lZEU#$++>vFh)?#b?tfjK-oLy zOc)E$ zj+PdVqW77ZmK0KeE4%v9#|KL=yd3DJfTMR3f$)}rdNozm8K^xWe!E|2a%hM|B~=a> z43lI01qX04--1&N+oz;3yr`&1p;`{mzKGTi=JGC>>JgB`&i|GU@El|Aj4Y640D}f4 z+3=HZ(nEJC2F%&txBP~H5O zReLiOX6vygXGhye20G%|0+&@BCsHsYAQx~TgQaVS9LYLryMC7t3zp{SpvD8;g6!7W z6%%^Rh7+&*5i5xa5R)NNq7J}`SNmZqPXH1`1c*Rq?lLpHhCHWmHg5h~l%M_JFjUG}W=2j|eFFQ2=c_Iwlj9;q9Hj5zrhZ;>rt^GQE@E z7Q76wQG5CMNz>cGODN6)8Uaj3q`xqRKjU!R(2a_Ux{Zt@mg>EYyOyX$C~@O+b@*J{ zOpONy(yksu1j3hgQ$S9kp-f-+^YlM{9OGEq(%h`PTh4~zVe@SA#kEHdM_g4NKc)-~ zSDShjovC5RlTclq4dTV4n!Jqy<@a(V3RU_VYcpDq|5fRBjl1QW zy)pSyMJ%xtJ3c@-=+$4qG;{1^0y6cUp{YoQ81pr)^bwmBD6{ zEKO!T8_j9q)x9DOL_-QBX-t_?JKSEqxZfyD-&omG9iL&tB>BhF@k$+|c1DvsJmyoonO;T8x}Z^~;HiPeh?@Z=imKuzu!B;Pq``}&)lxTL>B`F^?-tF;k1Ow!q=I9pb6`Xpwz2C_Een zZ)4}CcHPoJDQRiXzLb;<)!Ohy6cs7A{TZwAJK5u?TKC#0mqCN-5GFsV-D=e7t{XV0 z*;)9dc)|3fC*b{XKu(oWh{!N$xC z8|L8NOY4*4WPGm|RtNfTU}AE$kqbN+f~?ocA?;JQgT<~Z1M9YxJVV0~>WA7{W<87F zg!pRRRqTrirGxL9TwdJOt&9l3B>4PXTbpg<@UM#J<>)^vIW*19*@q$$rozf5n2-LC zIq<*#T-ShOF#lCAoz-Z;H8r)d$TonBfF9HZA)p-`_|_tUwp1#hOTGyi9za|(-$7iV zK$NNZO!;cnqV39}erJ@DHuT>sT!FH}$DF9Ag38-Q&ZT^2%x%iIN?i!{XAGq6Lr zHnh`R#sE_-0xmtt!Nz0&#QnsXBXQAikYSF~(xHmIi-uEdYI}R3+=SZ%G8X`PRvAK; zGpGeQOu9-5BwB)qx&aUnh%Ffi%|LlhQ^D{24-d0fgyei8o+6lpQ$mLiYNY{YsII9H zW;v`Cn-Rq|%y2)-AQiG8A26)5w2QHU-7jry{M}I7!$TBGQ$NsTnY4YOz2UFz@kJbt z?_lJX4alp!q9Vl3d$k{V8{p}8#tD&fYMqihWfud;akYx2&N1~7>W*FBX_Qb5`%bs+@XE&Ks60`V)hj!jd1vRNwMw!wGlHfAbs6fBq|X6ak1rk{ zOlpE!KNsfZ6)Zo-IN9rGfg_YfKH_x7dp1f$NGU9*GOM#I~Qn(`T6)?|imIaHtBayVOORsB46%c3&?I_F-O^EhpR+a7DhS&ppfOzA_+CdRaGq6=%50k92} zAEMgUy>1Nr0hek=W_0u=3^K}b`sb^;v=-+)@?)t1*I=WD#@p?i%k@VxoPVqKb`y^b~nw{4S{NMtHroVPnZeaJ#a>Wj71cjC2#h(vfKRfbc z8c@BDLYZSZkLDq;58pLm)gYXMIO=P+>-~7@YQ%#lW>T;_{$SXDW61~3`9Uw-73S~FI$pak#e{IG1qP35vrvgGt|Gjl>qjGUS@EB z5Pn`+^Ne71-f{mJ9*$LF741vL<&d-%c@ChY7JnjUNTDh5WX8c^rlS+)uvABpu>Qpt zs$}$>Qa*IF={`ZfJ`8D?sly2o!UQ%??T0l`Z|p2Dtvy%tg}65-%AH%=^(r^!9BYL> z3=H(W?7~Eahks7R5n;f=S+7V|PJgbI{>t&xjLwDA`2Xu{jRnjrvMZA9IAL!hF^2Q2 z?bk0(WmBkFBVeh-_*bqBQawP;X=RH4{iouyXV^%!1`>nPGUwgYDNaKFtDrwU>ay|h zhOE&%B7jeE-iLbV(w0f2u(~3iQ){Fyw?8!(+1(IG*!iI*ecRM1q`zq@gv|DkY_B^WA^rf=^>&5SobO zsKSi*_vwQ;?QAJ4D^}6-Jr!iCJUCe4&4Y}$pRaQA@wLJ>j>G-rU7KkG8l=qJ0wrty z+j*(4U%%dFevFJ=%;5k9*<)(rBpV7o7ZX?u1E-?B{j89F`h}N^tKV7C3#J>(ki_PT zOGej*q_jf7VnW4XMVdP{q?5smy5Z70CRXTGjyCpntA=%#sTsb}4f!_u4qaNb_WM&E zftcB29wHbLTEC%s;S+wec-;7DJR!L-{?k1=O@XNTp+Q3Y_@NljMcy%YhbM={$qa-r z&=l7ne$yBse{PG%7VAyZ`Y1c&+#@>rYL7cVAxTN9OUIS>KH1Fir<+V|okgp4q3p(Z zpVM>hJLB=5F%X*b?QlDbw4NAFx!$}HK(iS&`fZ;OWokdvwSqnCqNQdd!c%-B=SRqo zhd*#zYXZ?xpPJ}8Pj(GulIIu}IVWJUbT?iO4W%g*Z8B$bQmi@Y%Ou0zeBKcqQ)Kgs zl>c;$w=zC4DPFVKgHYI&cZw!EB?T8*A8uBY((=>TzT@t9CYhvux_W+q?ej|}^sk0G zgYa(ur zP>G0Bt=5bInScjVvICF=zu9}msqY~`C@MNV(@4NBvdtA)QR`|qMDyrqd}ts|No}xV z*dHcPC)g-;b+D7Lu@6&U#xG2ia~QbY^cyWayu+hbnt%)Y%CbtGBufo2t&f{u3=YPG zpj%peJT;Ea`V-tQF(7)o#_Yee<^OHOzbkY=+smS!RR@%BB~8t=_Wub*$uq3Ka&&S+ z&i)9NMfRY97Jv1M2|!ES&H%PYO0SA{%f?M@pR^;jvs<9SMRw`Xi}GqJK)@pi#K0Bi zTUO^Yc8*RaL^tDJd|<>~Io?>pQgm>$7_Zp)Eog!KBRrluLFL0vjPWH$#M*>@MVzGiw$1F7Kl{1K~Dbb6jAX@SVN75RE*$GgLW7b1o~I|X@s5L z=hmEHL;4&{RYd}ck*K^=_+K>eg6a52QuREX4NiA>o4NTx}*+=#dg9u(ep|^ z2$X?2@C;OhF;xh@6r!#GW#+5;i#9IJJ_xem0N({jw!sBRrk;bWS$KnLZ_?A#eg8QL z2-de769VG#;G~D2VT+{z9erLt$Dmt|*k4DLXV3N`32<=-+#=fFXJm}^IDn-09rv)1 zqO_rj$@gPjb@hIkr+Rp7nJ~^M8vc-)O6cSwdQmOHRX9|g3Rwb>rz8zEJ=#Fa#b2AX zwAw;wkR&CGS%8PLTjDdM{ygbDB(L>fL;D(OIV{h@GFiV)MMcFk{Nem$5P&yh_V;*s zDPXsnsF*M&N@Z#Bb)<~rIoEVy&%yhY6eErz#{I|-x;f(pkDshOkaf4We|hVcVhQCS z8?j$XH5ti;+1-LoQrc}(Y-;^3adZ7^UwNKs3hBVg16)KTQ=X4{jQx4mV zhC1UrlNdNxK{LSdfvsqid{NH+vR4Y_MutmblnI0O%{!_b#*%5Fb zFP;F^ZC&nmDRD_TSii zyOzoS?dZTF%wEVzlb)|!GmQ7$=t!j&R^Lg4ypa^MGSLqA^2~G5y!F0!=OucqH-&#; zY7k6fqD-e*9_v5D`c`Ef6IG~g?{j7Pl9!Mmck$H@RACKXo`JtBvv3EI!_8tPE0$rg zVZX0v%0>I0&6L1bQB2W_k1GZTPtX^yHdH!x>`P~*JvW}lB;{F7x50fMZ9h(Z`2f<3 z?c=Ck9A(w{;_%g1SJHdzH>%!kHk~_&<~E(U|H#?s-d*LJCk}$ z)Hstv_i6&b!H{!zd(X_pEV1`nrQz*O4|&$3dZ<@(PrItA`7mT@v|{uHL@TfbP6q67zNK!0VJy)=@do=HpMY5Im$@%zTKmBsxJ3s~K0(&u}T@wTp<+$$RRY-_!1UfPzkRc|x@Q$kDVk0fWle~%)*b&Hmlmqa5+ z_k-?(VdhNr%+RQ)Pfbk{`~m_=o|hL8lFrxJ*%^4_zu=jh#Td|t!+{8Chw zRlf-bs`e9DV3Dxv2ORA8RU)-z>Xc-it<%LFd zB+m#P-cFulIO~u~>}B`2cGrP{448W&kqbPQL)W|MgDC~Bz#Xc@{SKtRJO`04D5kJ> zkWcA~#e@TvUQ%+PTtLWjJO;>acAI~~NvV|yB!4Ok4x(i=>dZ^5#3R6b!*w}FK-P9DGYOAm+=r_0MJsk)x4s$N6E*LnWevobPDZlh<;}@vXG< zEcaYhRuZGt1fNZ|zbeX0o3iq4jeh-(ulgo2%1su>y(!{qvl0t>m+iwzu{&bM|89_0 z4Cvr16ZzNgcgPjLxTW^Yd#Ms^90%@io>N3-GppxBqewkny;o81`-BB@sA_Vb`YCLD zcHYGE)|@yJ>HEcbzC^!Yel$*t9D@-t`&Y~7f7(C-rtC~Da%zyr7@)wCf7R8<{6$|t}FH%Sumq3RqC#`I4$h5 z@b4UlOReTs(!h)m=K%eI-g@3b7)MaILFBXa$7?mNF-6hgxYaEu1%XFhX;MOhw_wE#Y;Rq zJoiDVIlC~xal%zn%GWrMt_J~vdMK!-{GAT<*0&uxB6QM@U)0_uM1x^q=`+floCwYQ z@1*KkX(EaW@@)0sQGnbizBjX82j$a1Bde=&{t(64e3@VKb7OPr+x|}vXQ9eq*XeAe z`iqW!!KS=9FX|iK5i1&5T*}w4OV77CF`DN-VwTWsK4fQS$K3tRnFP8em`zF#S6U}+ zOs_4z9F=)Ww~`jW_SyU2g6IGEL+}<*&slX!kYdKm4S;zGo)=_rW=VYpYU-F5Xc#dF zNO%h>VTl5z1|iUbfr{>v_Mq)s6tJE5r3($qABlvePoXmEf}QB(6>V2dzx zug9Rl1iJ$E4wA&OV$n!`084}C`b`I1 zb#?W;Qc{N)umiq?ZTDYuCAipwDER0&ISKEFKYJ+85M~5x4#X4D+0{jn_TiIY`57V42hn{Z&|7C`9!*|=%K+V>0R;16Ytyv`n*tpFiKpy!Q&Urs5M7Vm z*y+DjJzFve;M!&?5R8VjEG(JU=m<@FWdTkZkfa8^=!mFqk#ej{DJ$c5+wJ1RD=7vk zz)E0Mb#=bmu`L8-;_QH8DIZ3k3U4PmdwNKdsUTCC7{UN{p}w7gMBTwrKLbeZ=+gJjD_d4p0ZC{DEtSpOr;Pvs4iVkd85piTyKYft`E4yR)h}9`Q zS49^3AGr6!E1c0v@++)US6il@doF=Mg*$=W;I+YKpk8gf*_+XiSFJ}A@3_V-!*T8P z?($;Hdasqn=6iuMG*FcXF$`kdML6If;^Fr_xcgA43^zx=i^FNka@dw4JCnuRlfU2n!54)rs5rD29+?a(uVoyRVGH5%B5pN1l2?xv zEMDKA{$wPtxqK+`G`OmLz39S0>8sX*iQv`!aivJwQ#8rAX}J_7oYY${!g*FV%k3WM z<>jt$}ZTa`mf3yb& zpAnPro}aO%bvsK>?#`y();>6TyDsY5{?83iHK(*Y1Jjq2KbSE*K1bx@_~>s9rkn&- z!pI2P|66$Yyw&*SOJB(Vk|ZdgyxgE_0W&ocvY?bX?q~wDC>J*nY5Sz4q}$$T&BB*> z)WS}QTwRb2iU4|6w& zyhk2jDAyna%*#)b9%k9Wlo0u_xU{meLeN~uXdglew%`ZvflNuH2#q_8QivD!hea{eI&n`yrvq6vciTF8dRjlAV~&Ji!d5m7TG-^| z>zKk2s3(Mhd)qJ>Ea@l=0*7HY_4I*}!liie-Sv^TrM=70_M%V*1}O~ef`WM1k{zPs zq3w5LdhV;O#@FKCo`LFc+{0%0L1V-q+^C+brGP9i?fFzp4O!`6ZeJ>$xL^A8Nm;oTsPeVLMBlaLy}&}9 z9$Ic@3JFBh->IvSG+Av6rk3YT`v!SMU)RTnuMzs+Zh3`8<*9V)`uQH+UczYC_2++o zRp@+zMWXnN&2Rl1k$06jCcG)%{c<{V@J)#;vH8_Gehp(#Pw%<+*#M#ZSbcv5h%4ss z?;0(gE^Yli2u%;i?GU5E3^8Q%t|_efhjQH(@u~(DN`~z+WrWKF4^Nw!xr1sCtB=eP zSBICZ^kYvVrSQARpz-Au&ZS$FLRH}hYG9a& zSq_vxm-%d=ud5m$@h%1TEZopo30rLiY;H|d3ST@@VmUKP~uZcHa6AG3nj z$ngC}`Q$t~Zzqv`CBta`yh6a>BngA!f%lIeCDR{+C~Qz^=jZJDodXN(`jiuk`Dk^B zydPA0mVI#R2>SXO5ovlAB`p=Uw>Lw+cW%9lAfc-(SG@zI%uSLwd%VzC`o=6$-I?fY zY*L6Axt(++`^I2L-qEKt z6Z2>mg8we7|NV1C6|PITfc;>?ME~G{!ozi#R0A8h75)-H(cJ7+X=`e_3gJ{hXglR` z1|S0>_(cMvVKhap2uZ5QKnK=xz*|nPw>>z36XDY%xGHGFcW&vzU0V|_;2DT=c0DYL zTLstT_8A0crcpr)QV#{n+tUN<>i)`i%U&R#sPFGr*?ia6)6szg^19(<=r)3GQJ&#Ckh1KW+;S^ z#UmVqbpu7)-IvH%J?#ZDYU=(b0NomN;(khcdfCyb%IR;wnFH8Gq{Yh1&*v~shOb%& zY7rIU-|ULzqo;o&#AF1AD^l`+NfO*TK)xyoj)@@$2qDhS3PAxx+D?@NtpXk#%!z;uAdZBE-d6=QkIldW4jP4 z^%1bEm<{n1&L*?E5W6h@SnX;?CXk^mcIk!*#p9ci5$=nrF>ol~N;d1f{+rpv#)e%+ zM-!!z>SVOkZHlS`2Ep4+eOj>6fKN#GGawXrnYt^+>3^S$Cnu%K@3z9m1sw&HZ-^Jd zfBdA9Td*gH!Wnp~@?r6uDVCPLBeQoI7U$PS-yuGoSLKaOLs=DQ>&fQ@@LYK zW97KzboM?=mV~kDXNF0T{w6b2Sa&oNa(GEQ)4v>s#-&=8>GcmWLlZ zIF z%qx(X7I0ec0Xa9^8bjU~Yag$c2jraXOoMZ5@T-W!OrL!1Yi3oI z?yj2|*ayoTH{%oMW&gWVt(1W>VjJx2#vstkD;ZjUr=O%W11}3P8NLN-dx;t%7Pp{> zJwsAblGAcZBxGm^m1)6(wB}@m1-$V8IZ$@^4=k zd|;Zdt9vvup4nUY(;;t7>}-}8&@wT%e!#lczWBn?-p$<_CucW#-|Tp z?~tWxINN~!rGrABWV4DWXk^67dxa!G@~A%n<5gcQ-!v>=`C_Sl(0Yf7JxEs_iQoV# z5Nc-0%+D-gJ*VAfdAghp)YHaO!t1ER(}R0PT`^8SnpHHwmNn%eRME@n@9%uHr(&%$ zkeB!1@35ld9j9nH3>_UE@u?znJ?;d>c)5BwU@iN07q2x8sXJwl+yLx@xFBtk^Xel@ z=i@(0;wKulpRL~MWu`x$3)i4aQZ^qCtloL{%KMK{Cc@&$$RMAZ{u-%)6U@lKK**{^ zcm?(4FIl^=!=_%Tk$ej7twe@3Ysz5;UZzrOu`~W$b@FDXfxL^P;8Q zZ{%dDUy0eW^d^xVbA-DfzNmWzH-8B2+GN&Bld3nS2Dz9ANisyz{LJq<2{sCN!kwT( zwDV^DueVgkPsT;s4W)m0r3hG#4(xlGmlmwiF%{Pph16F&ZjUPsN6k!(j2~F;RK~D4Y>Fi7_E;Bnw%ZRYeN9;x zOSxn{Uwx7LwkFT6cpy2%oy>19_Lz<6ugb!&n!8N5?#m0hy=)vm4$G*dsd3D`!CvDe z7lx0w{@rfPYEvrk_rZGDH(jr9--eclk&`n5EKFQn)?y90wbVXya~V=g&cHfhV#^*% z)|uB`2;wVhZQR)*XVtPzQ_H=lJ@#W48`aj4Sy%5(kY^rtaAJP)&eOZc1TXZPuGJeY zXgOEy50dPM;}$u2!hzu*AM;LE7r*A{`}S@!3?=eiiVu!xDAzuP(pS?6`5qVo>>#gb zfO=Ek`Cfrmt-mNJ{jvI{JzTSUcK}N=Q@Kfqn^EcM$@kIMq9;i7_%2DQ4d2w}%D9d6 zkHo~j#FJHh*Y#=7T8J92amH?M&vva{2$hX2xVjAAU~IwFKh668Ve38Mv2OeK@v9*b z3QZZMvWgV4i6qG;B#KI9lk8PW6rmf5G7{M<8JVH1LS{u)Mu@14$oL(n`}zK!-}nFe zd%d3fxrex}&vl;falDWBag4l5gF(>${yH3h4L#5#u1n~zx&bFKvWJ|Mk8T2mUGi&`U1t5(7;^%_Y3GUmBny7k z16$#BzPW6!-iB;s0}vYyeeiCOc%hR}_yrzE=D7yyL?2tBwV9QbH4B&;5 z9$Yw3?LK?aoV9Aik3F1>OTpx=(dOAq!fa;#UBBUM_&Y-}RXfou>d?ii*DWnMH(x63$TOBgL6VllBtA_g{S$0^2N3Rv{d zBvT+?e|o#b6t^V=c$5=Tn&w(sPG!3~9PsvT-*6#073GBATTa#adQtUfsAqr5{-IHM z)a#4d)2#VhgDGTzRpC1ua|g@0Kx zKWw8c>#7qcta`*Ui<$10Eez7}@v&HYIsA28b70`;Xb#pC3u; zEnjulrhM6xA>UA2vR9U{q_w>5VPAYs zeYJU8epgj?=GWNfoMoQ+kc*Wzdo)9nrO$7C{B`3Mj#Xifm@Q9Py zQ!Uw$mTDg8=?KkO^JUjV^N;u0yd-Z`1Zb6*hVZ<0GUBG~Oxuzb zYUizYHF;g%J@2lmoXuqaxR6c}oJr77il6_x_P(>)A4BVRXA^^+mA$$s!M0L6&Mvzf z1dCeQsh5~y0-3iwejtNQwSB+^e7;|eRpqVn&z+*VcQg6NYNh2OnL4_>*@_3lLU zSkTB(fr(d;=MV|oWvTp;J7r>cn_h6ZR-9AEv&G2_BPaMC>t4DPx{3m=8vQcQ)$#Oc z-^SYt>`J{KJ&V&iJLtwf+fj*jY@%35BSZ~YPy9*A;D^tPf>KkodXa74sLOUW570xJ z>XrSiXhBz_{_4MwJNprNT}OwHNThYl8RKo(Jvn_7Voy756#9l#$FDsR7m6+ab;p^U7Q3by zf+uqS9MAta^{~YG5OWCE_|em0S|PJdC-%e)|Lt5uO>JM^^B{`)@QP}(n<`iMenGQl1`w)K zwbhb!`VM-D-#ScL?Bx-&?lSLx(Y@IiY7{M11xP3vA_Vd{2Q~^9U?qxZT!Tv~cg;>JQ4{ z?gHPi$jFotodlC#OF_+ zjIks`ym^WH9#+P`cZse!ib9>ZTQYO2#bOXm7(j59=saTlZwfWnJRCt+d58WO8UjXa zPk#EOA9aNf%8sDb$MlVl-^YrOU9T{{Iqt}=2TCNh{Q3L$BB(1o0s>oAg8cm}k)mLK zVCn4ayq%ZGV9&_$g%L!9Y|0G23#V!U!x38^fhR+_q_<4$9wY@`nQqx|n930H;hd^5&O~Q`{5sH2N zFEG*XpdjA~;JCRO#6n3&6_77FH4F`@AbTrH<2K7xUZ=bcfZD*2E}VH*Q8xJ>$MWc5 zD6_%FP?>T=Al<&@LVj--8jBm#yYcu{<)lGaZY+poi&;@ta&ac1UnQig6^ zRu)sf^_E=%0@NQf2IbX-uC8)&+OP6t*#8!_#jFUHw<~7GoqE4ptO3d5xuJ8x{`QQh z535gYfTZNl0=Il6p{zg9u7`UQH}3p!^V`)(XQV(2UH%2Xi7^SEh#mf#*nY- zA-^`9P}K`uaww43fr|^3`~1ijBs7135WNGJ7|={Wj84i~BLJfgyPp@hQc^9bV`Idc zGLJ8*d6;{tr$daa{t7bb*r!8ln7OS_oIL5^ztmv$wz8515>Vw+BgClUwgLUZ;>0#A zpneo|FxQ(J8(E8o7Wz=P|3?>;&qD%<*5YgX#_6lM1yJ6WgSpjj^1n}o(?^EtH{8z5 zW4?|BT3 zQp|qmN}lXyYC~|(@0Pc=_PJb|VPu85$Hfle_P`a*%b4Y2Kd2G!5wv9+YTPK_7_Fv} zdJ_+cWbw^gPMg*F`Z$zJPS7>StL{}_viv$UqBFMGc>I&t*;tE${f1y@Ry1tp^P%1M zY6~`W6zXyflTLTAy??arFWHGyU+CgbUQV-C3Q(5wRHl(RU$CuwPf1@_;s%xX@iprE z-%ekwtWxPulQeht z>&j$soKigfD(}1smL65y?2T_t|G+Ea`95Z&aKX5`r{TzV8}?t88|gfb=yUhfb<-_g zq5AqYuak0Pp;Ze4S{q%1e*2v8W4^@26gpIEH=)J#<9x$v%BMp8*uS<1>!nm5D7__o z0QZuwyZGLjYiK9GuIh=oVe@E%_Z`>Qbxo{W655AceX!0yx^_x;APJ1edXXUd8T z0Obu?>+<%@;Ge0_CF>Tuf{F_qIW3EXyYy@t56RS&>iIpPhR*6-C0T1)US%zq^UqVU zvw=ISIXFR+e_pJ>RrSQl_vmFltE>j_N4GCt;0g3%+ZgLON4A~tw?VP%4~jYK%U$L= ze*EN?7ZY1{^KvmCKQmZA|5xJ0K!N1|mqpJl+8yI->@8Xphprk-Rjqwu=tK{UYjUK8 z3W}M6`A6hTUFbKzy5{aIvnJo`^9jn0(%$xnIpVe^0+D<~X)nEk!@$@=e?vEh<@v6 zTZhGCp)1WzIp5X=u#ew|C?ecBX%e)2fHoPHKDp1#t)-#)6v~ym)0x!m`OmQikSdje zHF{1}&v1dQP_;76^arB~ub^_I#VqS(D5SNWZu=I!?&_k2MiKjCbry5DIIw`1@i++`y$W>64!?qN1|5ralbs%#c+! z+WQHsxVyV0lwdQx*f0WcTt(sAzvYpjiR9&mu{&t>j7|3)Q&ikRbiId%JFJ`r>(2QF zxbq-x9!6!Zn?4HGQpRcuyA+S2^B+Z(m>)ry-Rhrx+>7l<%TC9gPCr`N^?CqWFU{ot zR&8H|#4ZVJ!n0=GF5l=N#DD8d&g$MlQ~Fni*bJ2#`{JUPbKY}#aeeWk;=wU zXg&`BnL>kEE!yM$tTr_xqbgZDk+=&1$lt4DXc&-lrSU67wlw&{=pX)u?*tZqdiz^! zpIIZ11`2=+i&d%S_*0OSLHu01R`|Iwf>YP@y#g)Yc2g?yyK~jTrjzBT(Fa2=mM*SwN$)+PsfS09_xDi zI2Gs;_qU(^%k@Bs=p6(HZ-PftFG#TkuG4%esi_Wie4A?NA%SEh3QWsI4&9@=JPlQcX=!M<;YwBjj|1t~3nH(P>JyLbNfEBgXezw4@)Z zY2i6jQxP6)6U4Kz)&J}d3ORH+U?bKL7R^3?dNkb1`#_1zy}%5F2k_*=?=7ERwXBPX z$o24f{`?^}P`8iLH;YmQ|x%7$(19|pdk)pZb;h=ilYHypWDV{lWaXc!pW zAMczXOSh_3A3wct)q4KCV(`(5Rt2qU(G-aCc6}B0Sn9~Xd}I3Id{k#0q{aLKo~)4F zMe~8jh3u)GpQcgtA35l?BjB;i=NH|QP$lXY zUgyM&#~{ln)9`cZ=WMJ4%RZ5cxOdF=Wv3!uvvH^3I?(5z$dpi+xqm?1{WDV40xgsV z+tz&>`P@QtBJjP>tu=p*l5E~@w-LkJnf~}JI9%`M!BRggx+wX9m>2=c?D3rS%-KKk z&PATztyJVBI(E+MWCrU`-ydsySp={G_V)Jq*~(xlX}Xjg{xg!17yHUieD&>#IcjPq zg66I&_gSs$yGVoDtg)7C+Zn4u>7-;+tIr(gBm<}3bbRBPspV7CNt$uaef%yrZ zM+U-Q*tHULXqwxuQPB?6cb4wIm&)?w$Xwe}O+iFZ>uo8@QK&d)|NXJsu}3=U#gT@s z`U_=;3S9US&h$*M7zbg#a^W|}GmS{SqV9dwrhCK%ippKc#_~&9I?I>mpoY1RBdA>6 zCaNL|Ma{z08~1vf55YewrLuct_Uw8%>{Wd{U29|@Y@lvux;96cd_KF!rOQkQ7Hl_T zt5SkCx7doS2%;!AJwaqPg#BIj#z#uqjz>S=&yNz z8R^xM8JZ(Uo{M?7u8MZK%n`_%eE-x_jlSWgORQGC&aKf0kgcGDeoMlLn?i~9$a70w zSPBY-qs8H0yE*r895KjH=eWWuAxwQZV&HC>eRF&Uv`*FyHTF=yDxQgyK-q0)Cr;n+ z`)|)|rPs3Lue)VM(r5CmSHp8+GG^JZdEtj%#wCk}pN}n|)KLsgKKBJX2+XFha~v=9 z$M9w}G)kt6gDjxjy!o9jV;RRe7A5`J{Pw)#Dao297&BoNB7f;hG)=|%rNe6>5aPw^(L#$jAq`sj(FLWHrWvgM)-E)#{+= zxAm^VBcUsS4dM)pxg3l0ufC49?IVrbcFA^XcoPU7dhoiwo(U2L{ZP`q`B*gkV#Qin z2t#k)YTjA#`E$f*x)2YK((lio&)h$qTK97>{23!t^OW?HUgHx)!T#&lhq0LYu6yU= z)(KDe1h^qKJrZl$#?G!QC>nmG|DcV`l`9)q#RRFT8QxccN!^D&5{?0^tnWW01_?pR zl$D0&-6SN%ZY%ATmy>IGmm&Z)@v%Cn{t9 zOjy9hf_E>|-Cfw@Rt^5qRLMww&h*EAgDEHLQ?Sx5aGP~0UYt34eWdx1-G0Fj>=PxT zn@vmv(zgcT$~VfYSC6(m>+3T~$PXwg+ETjcDCIeC#<%yMs*mCGvldYFcR^$`+MVn0T&ge{PkuDXj!q$p+Uw=v6`{@Y?I$r4^kkv5=wg^^Tr^{aNj)m^S*doI^bs4J0E#6=Z zwXHPD53`|NLM>hgL=T0>^-NI~9KT<8o^md}B427E;Ox+;0$GT)Li^9$8Z34dzlf>N zq2KA{!NPV2)=`A^Gn3K&)&&c(VmC?AGu?^m+ob+xGU^w)9#G-m8Cbd9u&|>~^l{>| z#`uijnyKY+qe>8ff;I2VK9#s6x3M_yB9}b3gVm(yI`fZiF&_jTEr#!|4=j0>%!j4# znoERY#bN}z*@AA*mZI{~o4@azSXdmN%rB#tyYTf|faQg<&hGl?E_yY!+`n1K!K{)G zzwq_^-un^t<6%}QF|2p0L7l47!xxBqpSl;zjW=%w*E2YpCx6&^>EgwEmA>CDR0obg zvL>PF_YoYr0|uJ|dY1BRQmakjrMfdVQ;${g>_gS7-?mcTgU=@fz~1qwzzMrE|ZF`Uh&Se8@as`7sWH$R?e6W~3CWzxUV9 zs@q-rw+r-*VGB1kBd@>~qGU8Xvhlw)|G$5yxuK@Eww$c>85r=@z@}zJR|k8ydya#h z!=Ij%v>s&dSH>FOfGJ*FT9{J|xF?Z=ofw4^Cy0k0eF@iorn?_nAHSx@jpRSrfa#%jh5gJqWTthc%h&&j~1qh&=i8P_ecEEb2(W^}hKEqiX-);D-~J zQb8td!v@@O=;|;rGecXRg`Wc}FTmZ1IUZ=gQU9uoVe2j)lPtt+R#{K1N*s56_b4L5 z1i0sRL1Rix86mIc-h?X;amIt@gG39~VFax@gdsk$b2mV{xS2sz`G~WL4Zp3NoSevn z>ET2AjcT8xJ)p`H{BAIE3IW?6lkq)xj;+@~-vWcbXT!YKrn?IIS;nho9a={rBN4bV zZ-;j)F_*2mG;ZKZU&O?E}p9;Tpo~n-Y%h& zAEb_F2EI|5cWD88SJ_GRgKvE{MAS40oG>xj0=M_9e9>EFWo(ek_0GvzDleDhbehc! zjyS^*>uCJfse4Cj+8jpnwlkIz&CPYeG3QrNuoid|DpB6I*!sR*`PjT$^eW8V$y#AN zSQUYq&sf+eOP(z^K%X=RnM~%I{a{jxRz*jl<7eMHjyLo3RsFrbfrzDFx?mf%z54^< zlKKUzRf5)8Q~RK@yqvD9NYeO|p!zN9oP!4ryg|O>t8weUFS?1nbW6TUWzQGyK{>E3RlKG)xugt-?j~R|*fMWW;H-SOudUEidKkaG95FuL*C)HXXC(I+iU)AKF3ihW(MO&Uv#-`N6M?qhx z-(moCZ12|cx^g5*^BPpf-jz84iTRBJ`Tl=V9vm$$!N?df4;a-sX$#1J1L zg1%EUxxAxp1K)0;kcy64Rs`BCmx_xzXH51rTMsouhclpon00@a|95ERFQD3fNVwV#qvR^7qG%;^ieO%4=AK zpg4VQx5gaC3{(ZsOZ@!#8us`4@Z7=jr(+*uxckPrI`jQ|$ZLh)OiQ~4m!FC+hP2zZ zothf%yYbfQ0W%HJmBE%~jl%p7F7*00DeaPGZ= zS|BSnS3wni>SV(WhaG9VFL$((TPJ`2UB}Ct+g~yES%|qMk^1H5S!T^nrsSP0E;mue z{jaZK8dPo<(cpXE%%vQ~`~m?HlM>Mt5r$?g0FvRjTsL3_OdG7?LE2V6b?O-SI6uC; zYFUx90Zkn&pVinewAv1}N6o*QnwmF*WagDV@Q%ix3=uSbP1&P}y}qgIb(kN3uL>R8 zIHy9O3(wV}Xt@*fER?#>c#ydFZ40I#Z$Lk~znzhB$72+iAobY(8aNrX%8RmV@7}dg zExY;Z)houhJ#mSjFtPAA@^#PJO-TtyJTdj zO-y93Hv2NvW2e9q&!m=!^gu7NC4O;XA?s2t(;mgCsn1!vSG^18Xpe6j6*YM28K&$5 zJIEOCQZLt;CwE@it6yy<)heI|{`nGxAl6g+k6C8Sl)M9G>t6C}6XiAc%MB1?wM_ef zm+Rx%aC!STGt)6a14A`6@j^!=9<`5Q1%F39_RRSft4o@qOc9Mp!_Fwf3ji<`Vin+@7;YP2QLJ?`zSfi1; zN`%5lc`zKlyxUuW?vdYd!3Ta*OE!FrGu4p82i6C-anY5PSLE6ojL-1g)w2TJ} zW_qyPYL2kRst8&oa6m$5)jIsb!?*qV{aa+@03VV; zqLj4stDb+`+fic>%85PzPBVE~XjQ#eou$O07?O1Q@dlhlLF+GKMEHPPA1^;n4RjeY ztt7VZj^iDmDqSd_`t!#bbn=yYWpYwbD-^e>YUPkZ&;&i~)?%SxSl56KeX%L-I4UN8 z#yytNC^_T|K#W9~n&&UuPK6QV?9F|N=8xPqWKe}mGURdMfdvZz_}UT7uk z78TVCal(5eBVNMHet?_lhN0^{yvGf{h%q3zlBEY#r#AB*%&Z!1%U&U$5?$picdX$8 z))XFrl^o0sogx=u6KyB=`*WQ#I6{GoX?38qlg8RtIl){10JKqC4Z%V}1oO5iarpjn zk>W6J?hpT^07e!ECaw#8OR`v$`t1VsHY+xeYZ5Q=c}g@^7G2MNLzV>~6?+^A;lga$ zAtI$$#o+Ikr2VJSOTbKKd2xaa=ob zJ5K#Gd6)aNrnFmhM%(`kRj2A4Se=F}UiT*JpSvdf%$4OjtEq)(SXK6A9b!E8{xSQ? zk6QUldP7CU>QmtYZ!r&Fnc|fqZbIH_SNJ<|iUc+VUIRnkqZmLOheTXZ`6;QWTuM}I z^a-_X$I zH%T~o(JuV0*H_8T5~h(&C~sHZPp zXkCf)^`*Yt7)|VT_b4briVM+vmyxEXNxV|D8a@Ktr4!)TKfp1lyjxe3qn$vTJ_ z-wQk!B@B&>7}?p0`BM3t_&&8){{vkdLXl4t;E`&ch}o@%V48#}F8n0QQ>d4%2Q}Fz zBqW59b03cOC@VY@v&`H0SjB9Qe*2P!qloRMYRHxyx<0JI@Q)Ip?}vDqWJ`cJEs~yVVl71@bGZPJ;H|Yciv@XW3%zD!Ue2p`WZCBlA1Uxf!=2J3mWrTrth{A zEaz~S;*tmggu+eJlxL#V9wUUQ-U z(Y0S5b&%@A#ok+|K9w#8a+TPcY}7yFFy^>tdV)_S{E~7bl%l))7arTpU|wPf;ESp~ zy(lohhkdP{-b^-Hf=D$`P+!^apxl57fS`r0=$;dBzE(9Kg9S9ck~}n%T5FOucI9i` ztx(?gD>am7VK7FkYieQKN0~kpf-hRnnk9`mSC6Ugv;8v|QxdV(=>1;XBjNIVo~|~< z|0;(fS~Kprb@wjKRslnBy&Y-R$x`3In%&dlVphnbURuM$8OI<1Ar|5UPFKpdW5-=Q zE2RW?qX!0e5Yj<~BOx8}W8J$5?2W)DlmzXDqcONmYyZs6l~+}*gW@oQFGvS{Lsbl% zc;)QjSWkm29SF6VPKAU)rxLN+%WFB0K7capn5T!xi^WfDz!IK=xdmB_L9f`o_!1xx zeyU=?_VRK&zofP{FKdK7a@akIFBq;2S**oM8w!$LLPFX$pVQI~U`LmOEWO0Ycn}YC z23^vJ;^~%u>g*(IE<}WeFh;O^5_Onx4v_?i*Lwh~_&;m1$n(i%g4W&pXz87+R;>b+ zZWo3_A}!F8AwsLrhj||d>;)(BpWyi~tLhJb{*bjm0qvvZvaD+bW z@(nqR5xieMgW4?TZccL}49u|2a}020?~=WuB2e27AD7=l**&c0B4nvV%H=SW2UBaw zsx#q=;78AVL)k!+u@Xs+>m0>S7nU?ZP$m zj*gcjM_{4HDltVtBe-ubm-$3376Af*4u~4$hlDVLAGvn1va8f%C)5r3mwjK}{O2F@ z+jl&J?lLb9wtL9(R*Fdq<^!wSxyvgMr@Zu^~m7#6k# z`!*UnI`OseY(+tz`qMk^vc0V>g)%UBH_M}lH?*$9otxV}&7iH??}6qO;uA2(?4S&RUBDe`Xcfpvx6(D3aSUKohUc>hY<&T1aL8S3K4kg*` zSAm@e)Xx;}0kp3vAnOufk&z;0MTcnrW?J(1MDuNa%B*KiQ+wI$SCT@NkON|XT`$c2 zChPs1w{A86WrI~+R4W$|+!X@?qaCpWf#R+T-hRR^&^jj8NpJ45Dm;OBGXzJz;<|WC zad2=1Y(G??1kkBJDM`b4aei&u`4|cX>O~+L(SL~BmaiWv zLzJxmN(x*a(BTYV5lK1TZ*$6A1ipN0$Q7a(TUo7kbJNz*<$C})Btfen>;3z9U^*vl zo$?PW7VtM-ehQ+$|;Wxu<6yaLKk$ zyBRfi8fP}G#Iq=+letX1rXouGsdCQ^Zx++;)2U)eiPBt!48fTwf7c>u-2EKg>srIr zq8oUN(=JWs{Zn40aG_P~-=t4IatVyaZ=*^sZ*FEnT=h1?PzEphHBOb-CJ9DO>UKCk zLj!?ELS72}ocyt4Cf3&0Z)%dBY4L3K3ks@6K%%A$#CRYlW%+$tTwI0HgX=Q*0T|j( z;sj&GlX2TNU(`RMf9OU*o+E}WSb}cKJTFUh!yy!5wNseHc}7O*t#|^c;+~z&wX6_0IfcZT@`M1OO)hltLXJgfw*N6WYNp{>(q8-C@AGaNHHg^9jIqdw`pz_#Qyyo(CcHv z)1~;veI-stxdU6}5}IY4`1sQm;c?#mYxnLzz|AyJ$1F*gw{*bAgi^AQD+&S|Z>}c2 zf`Y^1V2Ewe`C_r{2a7)xjjxGQt5$A2hK8G%ZSM_Z zg9{fp(V_}^Ex9}HJ*m(t2miEyouUkgZ660tW2sX@f(5Zg^undqCLI)cfI!F&0!S~n z29Yfh^=?9U_!wuO{aGA>A@Drl8gURF;37oWZaodnp>rDe-E6m^#X(RPL^pJ>Q6oem zMCdXd3h-(MsZK_9b#*kGX2j%3G2o7$!C5=IL!<-%kUFX&bm&kyh#*ZxPBwV;dlXJ6 zL`C@c`0O(Bvb}Cj4##$h>pYO6^acprM~Ex--vYZ|;J&~-uL6s*#HgLD0#{f2R&T;t z#S3LX9~%1~{icR!Y5(8f8ox_*^78Y)#k&Dlf{Hu?lsSs`h?m{)$iA)qM2LlOV3B72 z<-p>i6AmoS)cU8HMjg~9;IFSvXNQydriZ`J-o?D$49kz#AZa$k?%t# zV1AJr6&}9$pBW@q-D)=faxhs@`3ya6sS@CiEA%4Vq59cVn9y{-Vq@!xU`2aln`7LAMI&a_Nuoj_er35%v z{Au{c6SxE8V?SzZD@g4Mpu=@FSrSL2qH=*KAs-=3G2`TGQU?P!AglDyrka>ymv%J< zOM9$1hYgKd1tvwpW+xeRQ3QGk`wTKf8apk`x{US$3WEd`sLq)#OkMr-TK`4!D3+=+^7p5R%!6q1)Tn@Hh=t;djqbbIuRFZT|Bh~qgC2rA5xMai7>07j0 zgCFV=tz?})vt`Tof!?TggEFvb^X5x$l9I%MC;&!Wj!j8X4#@}24!3aUA->TA`1dl! zksG(15aQ-Ex4*zR=_?bs>8@ALEE+#qlfH z)#z*Gl&}Il!~1S;+nn`v*X(PYysyT&+C9y5ebynl8|gHbDu3jUkO#Q(rFjgBCpY-d z=EcIlFnFT3_XeCSaSIy6@Q|CBd3ktVV{LX-m~ukx_s==ppfg;=YnR>+aRs1Ys5bXp zJzx|A0N@V&&N~=(9ZAARXTA#1;U91*_7@0nh?sA_vysDk1qbO!1Qi3tCg+hOx)axT z?%b&oEfv9*pL-D>B_N=sJvC<`x>sCWjICeB+?U9TtTOKU6TKD4_k_Oz)Zp_^8O9x_ z9UWJ!CTd`t?9|ytxeyg4|MVzgKGtA?a<^YS6QObkA@3Ptp{<>r*|YWkmO;ZXpf*L& zU^;p7B>h7e)`4U2Oe~f>4$b|qP@yxhip93%ZKgn_iEIIU0&$K^`#AgYfW@4Cuma(C zcn)Tt2|7zyR4DqXeEh!$9B8#o$@JDvPbSU8Yg5CTPh`E$1z$n=@GZs*Q|YZZI?;|; zr;&6VdwqXr#VRge#CG%uWUj|~_zo(Vet`MjQV_0#=5eUi6+fj+l=4K%G~}VuUUGR% z@c!N;4cmDvFU}I+GagD+_t0sx9NcA|XH&Mg6~ofVssB4hP>8Axr=j9W))X-!dWE6) zRRZ)}CALe_;T<7pd^+~;5ukpC0KQMJSLHctPZ!^ta8X)2;dIv7Qx>s0J0&GnQxaGT z&THLzW;Xcem*VcyqSJmt5!Wu&ZqZ1c+w<=sB7beeCWJ3!@JFwwXaU%9_};!-QnFL& z0Zvf}g2B<{r9~Wu~NAmijM2^^q?`f|2%+Q=mU-1A z>!8Aa5x~Ltko2t;8`$#ln^v-SlUjC2Ivyx=nh>bjkI5+@Oi*=UYb8AniUAo9xw*N? zEs=U;_JKx5bC8j#_cG}hVQ05#2aoi@jM5s50Gi(!yYl!!qwXu`rUct^d)Pn! zc*DGrq!lQ{b7NQM%N^1V;H#;hh^)X(;Wo;(%8m|$D}}K51s!Y>R>7QHT9TJ4F*8u)Ue$|9#oSoLa)D_m&H{Jh==!eaN&AvMvV!SG%`PHc-J5#ETA%i znQ_OO*)md`OvO7Zt`y|0fnjY)w!Dd}Ni6Ajd|WKt=;l}RuJe%~2S7Ws9ZaM3B zT5svoQlqLf|MQrMv1vKSDP~#o^!VS|AnB3ayYDNVx+*`sN#VGwp3d5v6z^tlt>aBS zJ$7zS{ch|NPEcBZT)~z4xQ@K9hW2`qlJJOE9DmxrcSOkXo^-7HwrCk;fi=f%wU;hW zl(f@O$QMq>X84L;iFT>dIVd2|9w}mR9m5j;e_*;8T2@wu%FrfdA2;{!$a9TdWnSko zo@vK&^)zgzWUmXkC8|XUqYhP#milbUEJdLt83Gs8L#wEg3kvs{&~=m>k>!21!wB^4 z2pUf@R4sV#=HfCk7=FIpZK5O*BeXNYo77XjI^4ksi@}51Jp0KL*7S2F1}HniD^0o| zV_cjLTD>-Ux-j_D9j96KIxI0U5vfom7suMaxmR;;ets09W6i)!L`>^8JB)p5O3)l7 zo}VAZLusewaxT|T{d#xsu{pXM zWhEtBmpm4VhLc#7+9FH_>S}5P+uWu{6HxG<4W&J5#k6IMDa-j+*L<#+m?wiB)KUOB z4gAhr9{Om7w-zZ|mP^;1=KbLBI7Bh_p`7ztMWIZFI-ejPbgSS#N8g6an=KaGSoy`n zlNa-8ujkiz96Qo#?R2j$cUkx5!zb@r=TpTc9&4sCNp7@SL!J|!RN3k^YeX(q(dAxk zs?b6H{fKHN^wFcMs)Cf)%-`Em!?9eal8cq{I_q!jS3T?jnUNGc4>6DHL+^LwQ!{P9 z^As1$0z(U^FBC@}u`-v_qEJRTDg|57rfH8?_-x?g<;~_{q)^(G)zv?K1PvM3089qn zo0g)sjTE(s263PN@qPZtT-CRgGiW$MA3wIxWTB{o6#v@*UKD6QjfM~{gwdOUG;4In zL_e5;vJ?Sq*vwEz=aMFqlmLF;bJd-yj|Al~eQAZ}aBHmGntY2WwM4*x@hA~=37d-; zz$|6r$x&k0{hpa=$M?O2lKb@7=a%u@w*dikR-dxuaEe>u=x{&@k5Mhskn&2cdB zmJYYUH!K!m3?KD$x&Oh^Bz)|T&1-9GB`^XwewRWy&41{SCPJ`C8!qW{OySOku0LwU zOb*4i&6_7#Rrm)dCMK?_3Lt#P2|F`wSzdmTlXLpqIcrUpR?(+2?7CQca^-Ce|9Bp6 zPl`>gY%TuZ+|TX!U`JSF=Du$)rY|#29Xs~g4*(iCgQ14B> zC%&D0O^4tfl0zSU-(M|J8<0^OCer-M>nP4lKt)(2t|uh}g?Wl`d1ahJk~K2=JQilA zaOX!6JFGvVeNMs~F~~B$+3o&Y6|4n4du0mc1L)G)7a@v#m_de$(kh9Dz5_HesP$On z*R!l%y?Qe<^E1G7ykcSn^K0U?U_4aY*x0DkpCujZh+onK3y{s2tYF66^2B623&t^* z-#>ba1s8v&)vwF2cO!ZqDT`m(JFzLt-SK-z1i?wI0hyji_;%FUYdpA<4zIZO`#m^8vOn}74@b$ zc3*zY%p9o~k5Xg9U@%oTUCYriPh=I?V@e@hstC5&8{o_2ZgR z`~@K+N|FukM+z$6SFNHp-D!x38f43C zn6}c&!?)BlG?J5puN$sapq7JdqjUZG%XeSEg?M?TkzG8>P?G9KM;s!ldB@A-WJhc; zfs>#gy8d+i$n{Z_69&2FVMMPnRWGy9=-c2Rg175Q4$nVN;YW3{(wIUyuzAxaa0+;| zaps#M1t}E$UA(*}&%6Kh!R)DTX5JpahzKAohaX?gkjuU)?5!Xjs=7z!0F<^StePx9 zgnkLYJxa!-!&CrNZNi1eEYvh7+lT#V{SYi$S>PgjCUk9Bd1d7>jK10t+C|{NC%wGz zTR>HN8Oc2Zz<~!0UguHNH~`2B3lG<`zd+vj>t|P6lOA-te&@$%SG7J;#J*?-Lh5Ezth;FR%%}#* zKb&f~GHUC3^P2Lw?upgpQQ)1LU(ER|476-|q@~*Q2XN<(Jgdj=Zk|T+uN9;VKlYM# z!v-YfL;g%pQ;^(@O5M-&xc|P7dh8i)08T)E|1%V`7TC#mg%A1W?c4QB5`;SR($Gl7TjXjFTITXM}N!sH%Kf;ZAtw+ZV#eG|{Hn%_1s-*gkd>b2fbhd5)$$Zo+1boClO5}sZg%8aB%(%3ijD1^=)m5kFxsUJ+N&rUB2Uy3 zJ^zl2j(6raT7__exlO-a@F*xGVc6`YqR|ZFW6yKKqze#>UD>Lnr`N7s3&m|hcn=r1 zv9T#xn*To9p8gp3wYT=PWeu!PQAp@)!5u1@tk~d+2a66T4mjLOR@+%Zwls10B|H}j zZ~;=copYAMz$2`_YWFSjEHt!H3ZV#1K_xqnj@tpXX*kylm^^pnTnU0EZSvD&zA4b! zJWN;UgzuN4e_a1u?$){y2wtC-m-j|sQ2mYRtS%Zuh(><@CAFSHaR>_v3hMU!Yqf9h z-eDwHua78b(A;I9ngxL>vo7-#&e7TIy8@KfD|bwW+w{%@0j9f!)a5O!E_%$s)_N0gRuok}9&w?CM4{K0@R`;OjAUxzvx3M_#R4pg#1ORYLHdA#_eg_(E@EYhwo-|g1NVVa-mv@vU?qU658x~)nk zVEI7_j6i(uPQDy-EW(>ffI;|GfC<12qB|Wh{3Ow53eK@9K*xSfPp1#XQz*Tfq+CHD zc;12j^ve;|b(Gj6AcRFj0qP~h#am1)yv(`^bBQY4sx|9sDl03!Zzq}K$%q;yUK3+Q zO`9*sc*9p5p&!^v3OQb{P`#7PFlbLlX-A4>f3^Xih!}E*64?EkMzP~^v-6Vw1!|x zn5Qfy$x2E}N*G8DDzwL=sBc@ao%ZI{*CeJpCRb7Dtn@SBZ`DX|pd`J;jC~5Nexy0N zVp8hk=kqI5#pdOFwlx2k5tR&t`I08f%BhAT?PGI{l(r_6ha@>$<$lf8i{w)C^5rxPQB#gJ@GwrkWakq{issXv44KQd%0wgfpai?&5>p& zEqNOFYIQ-%aQ&X1Z+PSwpn$7@tf5|*!?Ob)Bi=+2t0M%nlF+`R7*S72OH(OZnoq@K zG7T4%7fxs6g;;_kwH}B2Di-SnJhrRXu|UC+qdtdZKm^g2D-t6ktt6?4oDM{%7Bt<~54FC3) zMA$_XNLv%+SAsLD*IL1VG7EfckB?Y=J#(Cr0cLI8|FpFD2Tnup$mcY~2z<^W zj)~e=4&R%eerst}`XEu#XHfTqJey-tmf>TYfsyb2pCv*x4@mk3{HH`os>DQF7}x-y zg2+)Y1GNneS$8H=Vz+_?6^zeHH5a)_KXhGCYJ9u`SW_)nzDR-n%1Lo?aqEvF=B-NP zx#60%VxI>?58ahR5f7F_Td|A1_Be*%D~IBjVVS29fe>&}QN|e#AyQ-zm;o)VC|gB9 z;eJle3SsQ*=kUfpDK%QnzwVl7c<@lgON;7lm3gAqz55$G;y^B+8@PR>t=(AcV2 z-Osre;Xk@dEK;V2n~uSwF^7loM&SUmn}~<5ofs=#NhD%(Jl~IqQjfoRb?-*Pb00JPNN1kjb=NT?>z+G7sY1wTsjU zB5lZ#8Fl+K^&Q&tR*8i2(L$0csSbh%-8Hm zXsE(1vWSd09g8erf$m_X{M<-S--cr{jUZ9i)6?U9J4^X3*J<=CQn%>!-vcB=WX21z z{~OtbNL(K+-Dwpfju^~WItwbpOCFlE_9V?c_qQLC=kQv{hTJGUZs5RE3)p8qL!6L{ zasPGK@yC~Ba@{W?WFhp+0v`Q9`Wm3S;Gz`1mjtwgtT;cG(|8@NBqG?EU?yN4Kh`ra ztTaIQ)OOr4iQLc2tBH~Wi&kk+r9%5m(I{~R6on4AaBQR)pOkLqIir!reXtkd6BYfa zlln3X$&aZgW)EE9pXYyHpqfx;|Owe@Em)y6C5+p|N}Q4KS=3K{fhH|3x)? zAia!YdYyzUdEe@_I1L7<5)2#KH`dP82{`EEH0V1fAM5<*_sg%>@-n_Xyrv{Wz7cCj{!r@76if-)QDGxLqHnE&a#U~9;;%S z{1KgFTcLx+2Br#gJizC6CMF;`zc`@0W|ySoSs)0_C;UgjUU4>*!UcaqmFMs3+*g37pLYWaS8RKuRcy?B>2uQ`E7P zP$ygpf>L8nt9)r`DY`V#Hs_x6Zm7`;zD&HYsmbD^8;!cMAZ2FGKxun(S*g!6dbxBH z-tC`l?hgU!Qlcx;y$`3CL_WL!(cU4m#@d0k*Jk$K*gUF<_H*@3_>ZaoHhHK)6E-j~ z=vrE)5!5mz{NcmVqKVQLOm`0e>-r-~>KrNOtYmmqNE4g^SQih70hmvGAlU;fn$(vs zTgMA=@~;9ciad1j4r(m!O2HIyUK|iwyndX8kYvZvR$jnG=+G#W&S-UGAtYx3%R|s~ zu0;glY&{XTCmo)P>owaG9$Q#h$F6-iLZ3{rcp~YfhENNW!h7!wkv_qQPsgWzCYX+r z61akW0>gN5c;-f5Ch-}cpH@32QNpBeVQF;xP3HiwJLz+~W zDH;b~T6OAT0N~&v(1){&OVWaKzJKtqj%DD^&71cgJea9G*Fn(;Cp8W*UXeDi$8$ZQ zsK9Y743nV`C{R>Id@G+~TA2UW3I)!sE^9q^_hRrYRHi?kb5jM0Cg6B7rS&wLJJI_v zDF0+}_S6d2wIJIr0Nnk#-CK*Sqo;qq++jZO&BTF@lC-VDLwdlh|NS`}0x9XoZKQ&} z(J?Hi`Ra&KV_RfAYr;_YSvG3tq}GLN=_l)&o_~Mks`0*&58p<$GKsv zuqg7d={>}meU-)`=v^FCfu5PUL-{8mG4U)jW+o?e@WJ6>ngKE*E!n$bbW@}%gEAhE zlvFX$PsKxYm4jJ;j)eW2Dbv~nSwOHx<^PH`(sKKqbquUYWN3kn7r+b}4iMNjG8g`P z^f<1d28z~PfZD2&!Y9W&a@~eI00FQ;S#Ei*(Tf160|S@DqZ-UrR$BO^nvWm#@LJk1 z^2r1|T4ZF88L9`olSzN&fLh924MnJi)BhWu+}%~|Q2>&hi8ZLOs3?cjh~D{GQ`Eo{r=HpYLxp>2u%LbzbNB-Y!UlpaLu#c1LFnJ7C;x z^3egVm+}%6rHrl^-GP1ZZ&1P6^oC0w*0G zn3d{SJxH-$jX0ml^saT%vd_ncIlYx6L-S`morfjaKQ%vmbmFOc4XqD!t_oH`@ZYni z@Jg*){Idx5pUmC!Ao6p0-h>k|&u_KTjJvXR*90Gr-cypSN^I=k95GUxPWex?nEqkA zChs=#bl>rXtKq4iKN?=vFl#&$3=HJKitA~naj9SIur&qoEDGY>Ey^oO(l{8p4rm)w z)~xXbODqM#R3w*XO>usBDxvNu4aLmUi9_#?6)P-M;!IrxfA89Lh^DO&*z#CTkzR>_ zl=2ez=%7grf7shO%Ioe_ChTFO{K~r+7}!t?(wR~1^Y}S3S@5n$8n&78L?gq3TBBDy zMn%6@1l^n!D|idYMOOXx%R-SCUF94pvSY(RK)5h8*(ENF6hvYZa`>^0_~6Q?Y4nWV zG@1seKRDYj7X^t$g#~&)(eohN_h1bUF3dBd?T8+B#~1AQQfOL=cC*LGdWu;owc3_i zs%79jGYaU{sk9(7k)FcJr*dto+mIncLJA5s%zU2a=SM;o>#}gI&v?Dt%dtOuyl#dk z=fea(pG6$abv_(X9E8#1>3Y*n9|C^f?#zz&=CZQ8EgOVhA3AK%;AeEibs32TXGPFC+9;0>XPtyk|s z32s$HTcF#mTWn`eFHP=7+|Sx~;@x<^ef)dMGS~nl`iDT#Xkr1t4JJd4u|313U!x+d zxbTF2(*HQV3CG#9lWwq6EIjgVxP7jytAln#1ZTbX%iW7VPyq^e%no)->I^p}_yl8P!fYz* z+m*5gi3NaVxqD#xeCiW_%kd>8vzoVTIfZf8ui*P;&Gu8zPUhO`b?cT+N3f6pgDzF~ zo9}!!ac*!!<7Endc-k$~&w}QmCc*Z;#SU4!+GjzB8!;i93sw+)Q@xzn^qQ?K3Lect^) z%>px;H@uJmsc3--Y%bj&e;m9h>A@c}6+d6maHO9EaG)d}?`&?eiM=h&SmEI_S?uu# zG4DQn$Y|K}RH&xuCo|ZdJin2Uq_PguPco5GF^+X+A2ye@X|i-8S1PzIRpO_&Dxg8- zb%nz^z@x_vrrJ2hRxaKdou02!l;(9Z<}u(16hOlFEB9V|IY~ed50cNlU``)D?+c6~ z`XtaXUpW6v)fM^_i(1UXrNt&Prv(SU(d>dDFB66|3|`!s6-Z~>Htm)?9=~CYE?9EP zNl{3CcexGvnFW)rIJ}th=e&D*$!N*@{UEqf;S}e7yf+Xs9VBiHj(R0D#kW!K;>C0x z%)^e_0EmHBCu(+xPnPv!kh75^Z&3=gzqlCf9vfIM^Yx&*epj-Yfgo|ueK1byIPcME zhkmZPM_3cbI8uar3k#}g6lghQO?+DSz%?6Trv22-BM^`fLJRM1=5w1&k-*uLv=`~M zc7_;6Qa3%yt>>YS5ncgtkTHnSg&isRE2K+6sz&JAdmUGONqAIS>Y%Zn>6P02lMVUi zR!qjS=u^9*Kd7r<>zj##6DB&`%zBftta+eEtH$ztq{EOYnnElCP6}K4<#B8sAR%^i zX>WU>51q*AANi4~{1uxgt4`DfU~4egD6Ucy{3;QVd5$MjQ9*k62GfeSLsWL++i#-; zoD2>7=G$*4fV2Fox2R^=yn_-e93@usX3b9C=;xAU9y=@bPD;-+-5C)$H%@S)?gUnf z1s_)02hSXebR2n2@%8P74AwkNc48haT{FGae{o9gvHB)`yGeT72=qiR^7Ead+~z5nm(piNYz*Gd@WMA_@a43`c>r08^Gt*}50>PrE%^4QmkHptcfG2IP%?EUf40 zPrrcVdh96UiT;TLm5VdM5iWvR2f&BezUyGCi~~9y3T`tI2tYb+58z&KsUQ@1RQ2cv z-zY8K3z@!To5naM&~e-dxapy9wNG-%&4C5C_N-`hq4M`tY$79W^^)0wdE*rj!0R3$ zPKwgd3pEUO$@0f5GzCqI4|yg-4{Qajnhoc`wz^%_av?;^=e(UbxTj z<<_l^052daZ@qJi|6=z8Yt(Ucs74_G1)h zj|;28*lM+IZw9JBAMSn($u2%q?J9r-W0$a$Ry{g)96JBhyURmD>elnSuHuyaJ9Xl$ zLBoia1MmoI9 z_9#VRvG)1^^Y2SXj8Cu{6!yGZm3Peh>#rwXk!gf4aM1%}T1tn>bQ$IiCCe5}tB3tbOaAAkkdSEx zUD#s=lX-g2VM(J%3bmudv^)$5H)fho=~i)CzJKo?_?d1?lsZ7%fpZN{SUkOLV?0-q z#Kv}6+Q9ER>h~xJihOS7(eof1KqguSjjM%*e%=S4nzU}>I^3~$qakUgUk-I1yDLKS zifAHi`6Y5=JXyU9^EV0S3+wVKaN6=GvjIe)`nh>KbmPw>obf%ZUAxvjsq=sVleq#i zPMA`Bl?9Dux~NZH*)aN-@gR5hc{l;%Aot_T;voRLF*L|RmKU0U=41#y6hafDBW76T z@@9fHYWs!D^N5@O=%IBA8V`;N?fkgbUTrZ-$>K^meVf+0faa;ifm(j$D7Xc*G|U7E zM;l^4^`J(mE*4d*wm3yHvl9BlyW`q?+R#Dm=k`Zqb-1Tvp`GiFOgAOPKBY%`cFR~#!+qOV zlzv?X8Q$^`Ik@ufWOU<#IibtBKX^W%l-V#)5>KbK3>U6%CY$`ud#j|_$&3*P8tGq^ z>;iQ<)u+@(;+6_>;t#WVY7mFUP%@eH=c20ESmp$A51F*EnBuMq!t5k7t~`$Kr!chX>D`YozEB5G?n9L&YW55U)xV>|I^q; zvQ0V-c4$F&N4o2)eH%>gph?a(bf_FNeE4emlmp{TKM$p6NN05adQ_Ng0!SA=frG?? z|L3WyDb$a|r;HIk&^QccM*f(;-Oyn6ma1iAoxrW=5iH%@j-m=wBJAMLAnvpVxbIJS zbe?Wh;2I|5YlD^tPz|~nrq)%AO>3G*L94>N0Z!mbZ%#F}R(T(7ll#`OIYswRy6q#J zM*4@+4Ybj`25VFTQJU>d?E!ORb(;H?!)T0|cmH_rOeUo6qgQl!fl85CU}uf~WWF;nFq~VH4R0`SK0!1ACKu$4bvsLjtkKqAxUV3e zopqZIDvYPg)zm3PiR^&@ErZc2qqXeauq8{5j&GnS6|vY?wqJ(cNaU0l=0uIUhQT3@ zWS8B>OYRg^NHZ!_18jBXu120*l=`=p;oLb~11J}mR%gr^n)z{rjm1gtrrQ4rFZz0S%&kF=)$&v)2VSApbm>?^BEYe7%a9=0R zQh`1{(1$u9^w(SZk~XL$ffnm=@4Isi1x3QLDbVnaLGTnrF2ulP zG`xcdz=Y#H5_wt*ysY&ZuoTf3-OK z8cC<1w>@csMP$GzJ~C-h5rQUIZUN3@Xq8vXr_`%2Djs8Vq-hx{U0I{I^7EG56!8SY zT@`=ya3$EufKEYr&-d@%9qsUALC{+Nq61-Jc3bL1Jm977V~R^C`B67~LI&zu*EWgTR+gRknDh2|?PclA1}24~wF%rhm+s&wd7D z@J3H+K>dy{yTk{IqagH{d~7HO_seBaU4Bb>P)qB?5|36GecRy zGORu)cPi=;SEix|(o7y?f3&OXrLY9f>TJY-f`!S6Th62Ntg2YDZo(ctMEKiP_h);{8;?xFp`k?LeY-$$7+vHvd&yX?RNYs8?K7S;TdHY47PTMEC7*sBqP~jPjL;erPKyPI*(0_mmbfP^=1mE{>E9KA3fU(#bxU!Tm zEZK;zgOWfqy5Uu!UNjEa!}n;f5NvdVcC<9o)E@np3ylPni=K4!7w+(C(W#CS>Kx~s zIa*}6v?AuEUxg5y_hE_Y`PQ>$g+)$0>I9#euRrg;H7ngCa{j$zt6t`g&#%!)JbS;x z3r?84uG3)7p>ODUZE1}oXSG_*R_ekMM}Fr=@rXdqzQg&=$EEvb9XvTZ@yLuI#m{st(Y*si+NN+$a#;K8^MTZLS5fyM zl+}R)F=o#%*$SqLdNa_(ViTNXn6#-7pE6(WugwBr zJz7iMY|_LqlMvfDU@;LRIi#ef+OU(H9Mn<}o-F`9SHp$b)oibEzqwn>0V>!=m|cy$ z(q`i2B_9;ad(>=!+!0?|`M}*W7aZ4Vu2k5PD=#GQ#mbF#o#FKPe7!PhSbL0KapegD z?PHJrntj%C_fNG?9a_%}IHBQGk|h<(YyDFNF(_SqAwT2jz*TmOJB?W4*aUPbZtT<@ zLtG9&e|2~FF9W<3nqm1e>j58gfQ=l*yVLVvMcjcjzgwq)(xOkJA=IdTnM`iN@vi$J zvn^{+1|!qT4_1}%H<~6lA^{VneHV2i-NMAx(`yFDmY=_TImp@w!zaGiU^>6xXeTFO zxT58drp?tW5mXLXvlQ%4=miExO{%0Wi^8ghkazlKMUNRv5dB8mC)}C0Zv`981!xa@ zXhhWl#d3_=7|TgYLqU_B+@B@=#A$(rnFr3V;gWM zQYIq1=55-1xmC}Tw|+-_CG@c9|3ZVBvR5+3Qk{d0Jfm=k|Fko8+1#ak0y$y?5cvu| zMm5#aio$5CQU*Pdj>kV*eh{DG825Rr3u#U6yl$mf4iEv5DF3ay>FC^IFtx@V1fXCe z3Y*VML<1%=OL#l82cFm(MVM_3aThK3XVWS45_5>tE_h5pKeQ#|&QoM(ul>3fD9*FA zY)^HDmRS8z``dSWE)7tu!@g_3vsSW6mCuiS+PAHYqgCR;+P|WY|MHVN!k;fIYkXfZ z)(YSMb9}?Cn{31S+11rOyKd)sc{Y4Gx92U$%*=S6%?q0mG!P9RJ{W!Or3xIAPK;m0gpwzvp%)tkF?nrmq}Pw(Sm?Df`+o*)wrq3YaaVK_I?T>uy z%oFpbpv*L|MS;kX!9LCFBidi;Qxvdxi;lJZ?q7QFC6{`_GMw^vy#iglTjH=J696fz5ni*v+E1fzmjVG8O#Qhj8lBZLoyYU1ZQfm~m|aYH|KAg7w~ zk_n=rJhQw~1at2uOAk4X9-TnQeQf4plg8&w+W+1Sr?%ch*IXh{wHApg3DV#bQv+k^m%HncR9O2fOJKXe| z=w^01d}=S~+XMc!s#?uJ006I=d448TTE{M33Q>oJ*)fobL&9{5MnrGvo)}$`Tkv0S zDmT~Gc12ei15QOKdw%iJq~y?u9U$E1)zd`Z9X8m9!)DB6*N+=8bD6JGf*;fqn&X+1 z%-tDF`niv3zPRg{Yo8q_j?gIh>v=^VbpLk(awi0x}ogk(QH2?JO+NaMQ{k$V4 zXOroAL{9JOJA&qB4~NzN*3T92>;n^#>JI+lafs-)GGDmx{n8^kZ7t~ z+zPrE^4aaXKJt03d*cu~`r8N&)p4SDS~@=(C>R%7Hs&>P0>r27qq>|u#0Cm7S$cw0 zPlr4t!|D)()1!(1zVZtU9IMb~NNqgGbRhjj1_=Z(8ND=p=|H{bmfK&jf57r%v5y?7>iX)8kjgvH{XsQPQMW+Fj_S&tgPZb(S^qE4_zI z9dYDviF#YS50^#P2GK$b+MkRXi)S}(n4_XQ?VNrqM_P{Xw~5knME&}e+OTOmHoQrA z%Q+X~?3-`%=vB}19#Zc63wqf4r#I}sjI|=pC_|U{TXe}X0$!<%%Pm4j)z279f6x-sZ%a`T& zMC-61=>++Fs)E0|!XtWe>~Kz&rkv#!w`b3I8jOtP;<@5bWSomMg=?a>y&hz# zywz0I{o!F@QQYq>1~9OS&8Z(B8D8Q)h?a6u(y6vzc{DJLUj0vq{ici!Pv1M#<_fYWT4m8zfoCqH(~B?UwN$%yps|{- z)3M`zwG%Cmd8&~vUPvq}&|RSu6KJ_*U>B%}1?PC0zAlJ+8$ca(Agky?Pb0q455Udt ziGG4E)IjY*(I)+$F0@y>NevV){Te#s`!7F^FM(#gdUa$#>%POq1xAVExrO@2P^=RT z=iUuP>THcaJa2$FIg|)d8-agp7czr_mf*k$(@qx3BcHO}@{=I#g!1FZJ$G=o zq&l|Gr_DIdwChq!!>rv~0FR_yo#d3@qsPZ;V6eL(Ot%|14-n(a(R1c%))knqLvQ}z1f5S)w8KR25uU&K7u>o@Z z^}X2Uz%lTP5pB@GSUeNU@Fr!P7%K|04~Lk3dR$v(f7)T6SNFN2MNE65gAt)N6GHl- z=JMI2Mgo5c(LgT~$?I{=womr9k@`k;=Qp*jdHaE=P%L&E*$i3TXk+Ru(>vnqlC8?H zs1lOjaQsrBm|W6f<0@_$(5Yr_|3)c@mz5~+BhO5Hg(8!cPE$xjpizrBahX8!wCGgJ z?>+)L3s~*GnPRCCMW($=9lxOSSOEky{zqpY$&Xa|EAsCm#stJJ82{jYy(LowiCnqp zR4_+b=-LJTve`Yar{+$GPTUp!*Nmjh@qXT2_T7!K35@mZdpP4iFP|3?$)SFb2N&i| z11I~KEr3(ptg@6#3E0FJt#d$dUVpsrGwv>WRx#(Iji;mMURwS5LRlh^TEa^(VR(LS z1Mu+cDxtVP!RhEn)lyD+a_@kU86(!W-FV^20@bo5%#twCs?V?2L9e?HFT~taN(b{h zh~|tEo07jUD34=T>#VD((7mfxDR21?G7xxp|h%kM21V-$zaL-=Rl6g z8WLVNAC~$IELs#+|G4}z4X>)_e;4DNzq76QVAhSl3ZJe&UT9RBll;@XliLo?7}zgX z<6yHKIsgOPGY`5wJ%=z@{fd*iB0M28{q_PfUPMjc`tXz!Ch&-Gv4{|OYek;M7*!6p zCU`v@JZ-wfknpP)GXJ?|LruOC)4wp4#OreAsSJy=hkP@$M??bv@;vbVNa~yeGB6PWb^f!NgUE2T= zEGVJNvEBlrq$9xmz&Ukt-ajh{(IsL6dK+788KPr9sXjhdZ-88TP*W=v9fYP}SUK zesGQgRMRmv4Ng0yy!{5qfqD~0LV{H?N33&tSO9?3!nM3h?9`Ux5@$2ozR&EJ8oE-n{^W#R1 zlmVTJl5IjmCY0E`c^vhX~RA;ni(1Ls6Y9L=j10yZui@ z#4ixUk1l*ndPuGYn6!SsdVbC`TCHZ_dGY@BQm9|xr5m0O)=@Au0)&Uz@$aB&U4(@& zSzcLc%iMR(+6BlR|Km)H4vLpK;LL&nWkpCUgsT>D2qR=9;eWGGzpV$!0*Ia6qLxjh z!`~Y+0s1Dow7EE0;DZRGuL6c~`|iHERhI4U*?HeZ%D-;^&ExkLAAS*a7a63g{^FaB zPjK|gN&H{es@VO57+ExFtk`5bMO|j({`}8jYqTABnCsgQijkfQd9>x&vFmd@ImPdY z+DzOrcmgVhv?*cJWi$gp=oQenPoMsO)>8@+m|>*Ll2a5}e9AIWkfsJ5z!;Y>Z_zgy zbr8@j`uzF1Nt1$gU+pVkm>xSCHg0T#*pT>=o;w4g^A*b-oId8yD5ew)n}ZzVg5RrV zU*JAH|4!ASdZk+NH2;r|CBui(%u)Ndw0^kd#;%@z^3W1?mmF-2Z{18x9{j;Hqr?r7 zI244Pfj$wNxK*b|k7=jv)s?(Evwag>D<8*MEZ}xD(d5PaTmx=rTwK!>MfcpOX{Wr# zbEwB+?(TS@*9Qz$Nyf#cbP2E2cUj9Xs3&90(vPoui_=UW!QjUiN`#HdJTU{lS(4F5 z^$w_V(677RH_@=6_0|)EAByPJF1p=e8SEo|$(_gHZx+3ETHdYu-3E-l)-%@VTR;Di z*LtS&e_rqDF-aH!^-FuFBt39s0;r#}UTn0x9CM^(su+%{WoCfnXBEiIoG76R1m3Pw z-jc#2wg^#u;F8c8H0V)LBC0DLW+ED98z$5jPo9iVmGT;F4n&jkduLVE2aYrvOh>i{ z@kP=G1TGQ%2YeIMrwyQdkCERP8$sSO=zaALYDD>??t-w1xO-l%g;;_nDu{dSMAlNunRYZybL(}yc~}M(4`}!sKg2bJIP&rsbT~|Y zsDH!Ci3-ku{efvXyIE2JoObQpd615Rp>`bNsq^}?(iRq3zB3M~sr9BVuV3!zF#rDU z+GjU^^ZrDB>p?eL2>g@zr0GE3t@D8!Gm^48Iu=gdSox@B?IRh$~o=5L5DH z`Gvef=L+PI49oU;!XMGK+^+Zr6aJ+Y-qp^#rfg=yo&&a4PCnf}hHldjZyy6jiFgIQ z*#!2XLz52toe}wn1V!Yw2x3i?Z%w~ZeqF=aE&Bt2#TKkRWf^FrBsz$-T6121%?$nK znmJ>TsOdDYEak=kLG0TNaKHFsc_3>N?m(X_>Nzaeeupq7)N@yUcM4Aim)9A5drN>aV?wipxmy#!CCCrXmclyhlfx8F!o;KPC+yhP* z)1YbRldrRp|0k(^lTOAchK}1{fHR_+BZWqbcPwYXVr`}BDGeE+kUB8;*2M_^FpMC7 zR=nJ~nMNfqP(07Y7PV2j`%kQWO_P?I1|yiBB;?&jI#-IHCN z_ypEnjqLbGXjX!QWtf5Ov8`4GRCB)(TdWFh#oHe9OgeJo-&P+q^#5f6%{zjFqMoR% zc`z1n4A;7py}d)Lx3UVwCn-D=vxv2wUBMy;7O;V5fwkZ(ESkAm?3Fk>_P#~&Y=jk_ zL{Bzss-Z-UfjShgI+6y-DsVhaRhPG?KmB|iXaT#wzw>gJRWNScOKqi~Eu8=`oGcoe zDEA$3bO&T}(lAUR&Y_^CP|e$iEFa&V!`SKS-#-eaAW~O~5$4(dh))Jpyj_9X$m(mC z=jwueTMiX8u{t_BR~;W>H-lqPg$7jM1=V#p%>(!jCyI-h{8H=zfW$S6ys2dBw~A%q z7P>Twoi*yI1R&%vak~}3`muY_z|Qm>UWZ-Ji75hM)!719tMl{E7jVa|3-+yABEB(e zmU|}5b##L|ioq>Na#7>&bQa@IpP3w^5R{;)YAOoZbb|1(h&H0Fh|F^x*VTC&ZPmTG zd4}oqLLy(y>Y8yyV83pjd8Oizx4H4Lcntsa3wk!(H4?AW`0)wpymf7>=?PXsFb$9S zGV3qCLB1?EbBd$inRD^a@AW)zgl1A2ZS1a>j#DC>8M3!**svs)j(6$vg?1jA1GsZQ z9;!+(adZJyElz?J9}P~Mf!$5qZLL?=lzrW4bR^^>{3}E2A?k;aL5D>bMl-N@N54ix za2raTS0;R1!yjsm>nR0JWa9MEJx15IJ`F_xT0e@0(la66krs~;gcdJzUdd!;?0=(v zYE4RGE|z)guhxC#Y=+N4pFGtyo7nz*?k~N?@miYN@6jtnZJNopD(C365aj_sI*2VS z(D(ZrF&$-_qMH=s9>44Sdwoywlnl0#97ex(Km;b)+1V=h%<%~K{72_MICDGkiuDZ} z4MFexRly47Jv-1h7GXV(fxlhqqbd-E3fUGrojE-1Nd8URzvm7wS^#TAc46g=w4Ou& zxcq|Z#~tqtoNT&vNolEMJwE5EPN@&x&CAv?I-*@mV`11MRddcdv?FpG+G-j7a3|K6~4>lQYkAS89R0`?LFuLM9&y$ac5MJ7Uzp zuZvCI--c7uGC$v#@gHU&ABiBFtJ}`hio2--VxvT9GE2ixH;;f0Af6r0NE9=+cC(E# zxB|x^!5TDq_P!PG{M$Au>vs$f{vmwF_-IGzrAE^Z6#StIk_fzN>s(9mw8l~<4HgbC zxTK^TT<0@Xbm`;({-~;$Yi&ikK24lncED`IocykDMTsoDFY89n18uT24xoAVZ&pxi z{`JF&Y*R59A&M5zRiBD?SG+mZGw~hy$5@ORlTL$C8d6;`0`m@Os*6hN%8BS_B#JKr zc+$l&BXZLs&u1bexN@bNp5FS)!}uOotevBz%R#qfpC>dmEKN6z5o5z+g&KIfUH47X z22Gl53x63BPzwAhId4FoCFd(@gSLqsOD1b&Ex*xs7=$I2&RLu=YRUA zuU-0JKuZ0I`E3ha#PMKbL#);Yybk=i%nQXALvyhx7GeOJv}{c`h|Le>QF{Jmws~u zB!+GvFD>LS?iCo=lh-|X|C2gVFJxH%hk3R+|NkEx7_p4=VP9lt=YdHNVkpL>;OymjV6S~ zskI@WyBS`I!ydvy6JvZJmrc(<(-nUREWS=&+R*w`y%dkh zM@EN#*S_^ce=5`YzcYHPT>|}C{oIg5j%X#L1B!L0+0Lhtw_6`$>h~Q}~C zFj6tFODsXuAgE#0-Q-EWQoT%q(1 zVmt)tv}5{OUhQvBmW(H@&^(k846JuJ+T%oc(zd^)Dm)3< zvSmN4BX^-+BD<&m8C&_hZ?^!PNFzcMLkjWvq975JBG|u$ir$(CmKwxYkN1Y~!kI=y zFQ87ReBdoXtm*P53-IGLl*2g84Font3;qs(TqYVl50RbI+L7Zu!LhZ5vWa7a*7Ny$ z`&1xS(GrOMOjOvy=|nv^wZw`ke_jsMgP(EMQex-a`K6bPM`#<;_)88%KTVph)?ve~Zzo;uoWBJG#bIv^K zc_+rY$u6ZJ$|16DdFAqrtc04c$7_f(odX0e!Az0CbbZpbC3>bHJ+v$X61Sik>79wm zXi}tqQw%!}0OVrg0NNG5aC{*^O9p6s%q=pZ@Y*?vdj(rZ-F!vyg})Xm z&TjnoOQZdKe?Xzd|?G)$hs(1l?8U6E~6$D`km7v_#g*b3Y&SYHY|kB1D{i zM6r?or%7-C&Z}j{_C!?|~{9r*=Bia#3#(X=qQwnj}U6@{ywcLHnc?+|frar5EvEMNg3*o;-5o*4~bAwA4rtN+(u` zsM)3>hz=}!=X*4#$jGPsxI|4##Uiq53qb5YJV0aGL3DT04odsMoDpaM%E#UeF@-Lc zx|P?TM^$|#CufQnq_{&a_w)wI&k~9Z8YiDFUiI#jhlV#>X!!X)o!2)*@9LKt zIr7%CvpnUEpWSVxxt66+f8_7wGsfBdKGsC%m=S}#`enQkk_!#$5RSusnJL$=i=Pco zVwMzw_&6^71~KoONsimgE)UHLQGzi}%ju7t))D|S$CE=|GD5j{ErMw$Nk4!Z6tMx| z)#4c|=eBtp#d!znMahvBL>xz%LmujFBwq(n3SFnq{$r(CDXfcLb_I7sF%K9A1KtUN zLtR}g*k2~yd?xqY+?zYMcYVL0y)#O#?fvQFB#jN-zDA7*@+Ub@tiR>2?tl2Oki*SF zO;9PiKWwvl0(kwxktH=)I6|TgNAQ2@x$xV>@m?erQk;1;R;4h`1X0vFGf1Y`^C5-x zWyO#~V#jEz#BzD1h-01cwk9&{x1sS!b<}iaCKVbD3}tj`1Ep6X-YNC4r?>l0*yAo{ zeR6m%TBAG7rt|x3-Z1oL4#}t~FN3ID5*a7wzaR;YT3tY&gAcbL_@!6Cy}k}5@Ge&+ zmE*VShKHU$n{M^evc1&*fUHg#BTTh-J>hisD>yfB9Pc)xKW(0it@H14{Ks+t80{WE zS9Q0^i}(78_a?GB)!6*t$pRhPmo8&>9YKAXF^>jl^Ym7}8(0quQS)6X*8j(xAr^&dbH}%x3 zKE>=?8AnA&h|sdbkG}@29aJnupp%An05Bkju&{V9^70aIl))B)P)K3vKC-^@sY~-V zZuCzgRj-s>E=*@a)==I>G?t(Eq8dCXi^{ZDMie!Q(;=~=|Aq0XYoz%J{V2m0qP9{R zzJtt#u9Da?g2emVoZ(KUyx_OKP}vc1<8!zu{n))5S1tw;4u;8vd_7xnPa+QZWOXSl zo;8FV&wIab4i;2ei^M}|DJki~2yEIk!D)bo5+i(}WU~Dq(V>C3eg@3X=Wmp{0vZrn zAWmR;#*tY__mkgW@1`yT(XgiX2iQbIM6i~&?Ija|1ax&m``KF{8xle&Zf&;n?*$qA zHXardVE8jn;dU~weFX>#JWOLJ4LXoFrCn{mVarEf%a+maH(3OHoo)KnERlgxEieI| ztpF-hHFDPWOB6Twc*YRbkD3paR$V?{*2L3`4Sb0G7yKA`-HSyIBx6d1kq_4#%O0aS z(Cq{;Y3A_s<;!rkQ;_plT=?;LG;+4|n!?THTsXklu15$;)Aao&Gb8Mo@o zJFNy%6Mm~viM!S?_Ql7Wh=bZ?44prvF8xSKU( ze4x=vn7iyD2=$O`B9mXyHI;MYopg4xNgY}Oi^|gncF#r|Hsa?+{H?xV9h&K%SU1x7 zi7TmkKc;y`@l1s7`5)i*;xHG1t0btPaf1|?q+&xSa*7g>tn*)#fhLKm2qd5@5T)a3 z#_^zmi(@2)QkK9zbcTH;F@}iP3=8-&OL$rF7rcs&e>j2H**?*bAOR8nahY{{L7B+z zV|{0CZy`&;`HTR8(m&`sZ6eI#d7tX+xFYEbu|1#(_=}#_Pre1KV-0pLM~1Y9@X9r< zA;A8O^c*Z{?T68+i3FPpZn=z??lGhDEp{ro~1V zJ|%rk*^<(>y^*3TI66px!y7mHm_pa$fLdk_YCJe+0zt=|k zif$3%nzrgoB@jS_f3VBN!k_t!QGqo`kYC3KdOB9;1MF|FZ|ZlUYp2|+nMLRO{09YM zHA+3_lRM-sudZAS9D)cp_YYcuejvC}}Oe^~StLtw4!Y*MdY zL7QpQrZGzHbZ&dXKWINaD$uLs5MRH3Bkf@hCb2e#=5RIRIjJN%%gb7@hTf_D{i3GF z-V?MXnPfEKbSc1OxRli&WQ1Ipge4hb1SDi)Pte~iOF%Iu(-t`44#$9CNTSJfw4lEd zXPB%6K{1yWQ&U4trC7F^e`=2~0LjaJ@E22@ZhL*FSQhD46>-sy&)&n&D0!S6C-sjl zLjH5iaUEP~)kWsQ>Kl|Wq(6(6OcbfMq<9GqbR0Je7_7dUn&eqHQNp8W|6bfrUK$Us z{aY0H$3gat90vsgp{9Rg^<2Mp?Wi0FlC!u=2;m=$qa+Sx`@Ok=jX0=a0}Zs;KFYDH3a>;deoe66 z!{mdY@Z+l=RhLyevFpkUY*Ei1yWRRi3{$ub+-!}~&LOhW)9lCbq z%9V4TUR;AfwO8h1oU?#Zq71bVC5dTTXKrg(?LXfu*1>{KutdV_m_I&ne-5#QO4Kyo zwI~n1s)XHaa{KNM4%SQB9c)QnjLT7;Uq#5d3EGB3ZcjfgF=+)G-%LOK{)pQm8T+GJ zc1d9ooSpF5u?Y&HeS6&veU)C_x-WLV5ZCra;DG-)lMeAHWK>Sk=pnGLSoA|e#H}1L zg@(%j)t6Nf8`Um$R)A2$+_W$mfs<=>5jvWK6g(FJ7I029T9InP-#ktiGoL<}vSEe~ zJVboK-yK=wAa$cf6pY!=A#WZRb2Mu8_EwmonGcscIcXt$z4NKoHn>um%Y>Xk%@_Dt z2zy~8zxIM&6J*;@{77lM8-dv0GZf z$KNw~yO4#T7K#tAo_w7AfRh3O$`I}ZQ-yuqW{H$gwc9t$%}*XS6#W5eG);V!`D zj+25)n?4!R(sEk=woM2L3t}jEYN{s(DXINs#87JNvKjn1f~XI2TM131gVa z$wqxAa?3&BAqF4`bVHf&YZN_#tw?;)2OB3#%*BJmOzRRNWq>)b+Rg(gj#y3{hdP?o zSeCEYxTP(eJh^p9U6+$A|Ma{03rQPF*{gpxB(dN7=3P8%oegO;>UO}gua)5H*XA&4 z{M^N_zPJpy-qkU^zEb8sx_a4eof9QS@_6_P{V0YF(F_CzJYW1KSJW+xLV0O=!tI5HgD zxB;8o=fmw05=;)fqn-GXS|Pq~S)EYX@9D(((jL$GEc}j|kb0;*h2Omt6`M0Zf^r(E zde02}iQ$2&l{|Q~%Lz9`%E2I;$!sa(M?_Ct0?!w0Pg~(xwm9F7){ad(nPbRf#~5?- z49*HMy@pqjM``NuQIS^TeM)E$rCypTsom<$ zE+YNqm=)^{E2u@sI?nzaT!Doa>SYgueK49xyqZ*2)u9F$l!atZ+#I68)E#nZ#S9ug z`=F&75vJ^ad0Qgp!A>*Z+v=cz9#GeCeQW`bWn9NCb%A zt&s>w=mvr|IF3EjyVCh*3mi*fI!GbZ^WZd)a)4H=TheY16Cm=#Gh(OEpai~b8$4Vl z1eDByOOFNioR-Bekzzo4!B3p%*0>JL zLMT8V@=CvBmA5!r`9)R(1uQQ%>2`z!*X06fL&bK&^}vY6fk3)Os-}kvTV-{&`!^AzX#gYBXFX&(mk`!*)bMv-r9*})TIXH^(%i__T`e%=E*a_b>eR9 z`0IztD!(B$#g1mh^($)%H)>HzkesAECQ6{Q_8!46Ui(@E|)2&Cg4+FQxcV9V;oA>C`ZN%7i_%AqmVGx2P|KNsHFmG5f; z83mu5A%-qVLgv_syu6=Jjfx2(oyMb3rcAasOYhmSFopy9b}^1(d4JmoSz|=Jm>a}M zMnTmR`&E|o{20gHE^nY%I_VlFMc-`{e~iIpD(AXeg5HAShbGd>FKnqq`9Zx)LF0F` zss!A+f5!KW!5+(Slr$UyW5sk*&Trw#?O3AU+9w`;MB?TKoceN&OLBMvrF7I+<1oO} zbP;Q=UHT*8O)AZ+WxRH6_RPM0`}P=JYkR5d>$2i?N@$b92P?A&^C2f)sex@XdiKLL zUOZ9oyNn56?tHsY6#Pik)vveKa+MdM*Z;lB3&y&uDELz`%>z*Gr)JCDYlNyTNV-WX zmG%-k@A%YvltD|*OcL;CJ34k9FyNkS-FWHAfS~Bf+?LL}-K&v+q{4sWMn<8X;R-jK z#89qEnpc#c~Y#N_*(uzWPSy7Tzu@X~!N0DyC@Xq_l1z#kyTMFh9ynA;v zKj;;0zRkyEF6fctjv&VFh!w3W!8i~pA4kKvh1yhrtVBN(?q2A5tiu*MidP*@aN^qB z6E1(e-tF*_kornT$t98mJ~giolQ=j2>cL=sHB);vYkv6J#8a~xPCDQi8W{9xsUS7o zZ)ZHR>QOtlxSI?kQ7Z~#MAH|#cIMW*aW7^C@0wU&!_s9yr+En*!w>y{r=w;CD32pi z5ESfQ{Lt;*ulXc&Q%IRkZpaDUR}vq%mUY_l&j&bH%5F7IWz~x6l|#DIxU_0lyM@fA z@OPe<_&X!YBu4_@oK?UOw)uO`s(cCC!RW$xW}ml{KLu)V`toGL*kbjB$boPEFScZK5JHPOTbbEEAQSOwR~)CIK{9Qf;&o4T(DC zzEo9q$T`?`;J~EvKL8Cd#ixf3t}P6!a63p-UoEl#>tK~g-%YPQQLaHy3TeT}Jbi_W z(Q~O!IeFL;cPg3R;6r8(6OP`ScWKtitr9395)x^grjq53mgRkXV}-0FWrP({T;VrW(FUAOVEU*l-zo;_Wb|x7g%d@z*m&UOQ-a*i;@5W@Fz&f3 z?LGGlc_HTbUYWX8sY6mHybGF(wNDKos$IcDD|m3X7=FgA-G>Rk4GMG-iH@1)*W{4W zA!{D&`Ba#od!pn>7r$_;xr?3C#iiE{OK;6f_X~eYPypzIj^v}bvH?K}`qg);NraQc z9Y!o&0yQX_Zb^EC-(zpc7MBK7E#izx67J?Cmj@uPv2S;jA6o!+YHwN`%yQu*%BoY# z&*-viywIZRwSV9=flFAxl2niW%Np~DRD*V?sGW099{t*cX4s(06!2Q85*}C54dxd+ zs)x^m0-mji);?NT)T<%z1^;3CM7lj3|TI6xg+~(6SlblQO4T}7h>8v#j z7pu@UN+2W^%Klz4&IMmy$aG@W12~zry!Uf|rD6k$zV!a0= zCOnHZa$XzJVg9PmUX$4LQ8wET`~Qo~-27VfeVT+@<+$c`-M2kpjiO6nZPJMcH#V=h z(C_O4$Z%)wlT(_u6OfMUl^71UUTCk8F$VETc~ShM51(I#xIlf2q9n%?+EkKsRRt~Z z-TRMK#lv@V6esF?+OD! zby1aa8I7>W(koMrRcWs@(ZVzf@W!}X@epDe^My>!Fg$pw#=|wFoWkkF{a#U6^ zajYJ2kkKKP(^DMmRA0&HA|y3;+0MNC;Bu0}V;?niu=PJ#gSv87xlhrK4 zh3XKm;Rjl33_Z5pnU}P>Y2y2vnCyNz&$$ z5BakELM5bmR?D=9Np#jjlocp7$_mfpEpEGP7 zyIhk%sh>$%CphVI)Su8{e&5P%jnI+**f|UQ=qBZMRf+k^xv;o4tq9s0i(UQ4gUdqa zSuXx3X;76ZjFF@z3X|NdrRO2CW6;FR@Nw;%=pdgDoe_Jr#>o-Ui2t06JXc7POhEfV zQ^5+nU)=`*DY_RsLik(}Vtx4Vp`am69xCLO)Kd&+O#9&^m|l8exAw!!$_!AK0t~j< z0E8z4Sw023L2~B+rHaEUC6|{WQ-1-Y`b(c5l*b!w=DN3O)k+VGc`38E?@YC zC_HQuN-MtMVl7Azmb97!U^i=4n2eR;amVQ|w{1`^E#e4}5Xc3;nj2)d$e*=gcHbvP#RJ!# zbJQK5m8zxnw?;&|goqI%qlSnrE}qEPhjj=LKmfSN(Q=)Gpd+AoOaTvWooJ1TWIxPSFN>(>!v+dGjtIMsU=00{3hu%#?kqmAJXO{ zy|c(KCE-$1Q;EGKj^j{JQ~Ai;0ZDr)H_0_@(cO~Fw_MW^avL81c_F^LJ1Tg?*!Syh zLO&$f)Ema%;h{v2I^ok1?O#I6x{6oF<~wsI&i!LbJJECE%8|Sn$2Tyda&_shv9se| z*78fdj_sQeJFa?N6B&$3m8$o9w!Lv2JZFKLogl(Fw%WD5__|d?3crR$90!A3`X?u> zR!wI6b!+VHJQ{6(W(Ob7=$@S`UXw3(<-J^V`9r##D!zG zSf~5&%_Q6aAaepCm&`}joWN~I)}A@@`MaC`5Bk=&TnZsn7?(XYz50u9^*FiSF?dPd z4rIOv4C!*!n_l4|+GecJ)Q{5FEK!TDGCHo5Azh)>tHs*zGHLSHyQwR_~+ z+Q22OJ)!HwkC~$JtD+k+y_Mb>mm0Kh9%!FjKVFD4BZJE9ke5U5vrO)ygC*>1QmWWH z^GsV~Z_QnY`n9>8W^A7YF~cQcM3G$=P=8w>VH)vpB9q%gDk? zdl@8FBFeb5rerBhPUNCsrBT%QeXAwipWd>69{1{Ti*z4U|C>#c6~7;vpGgn(YMOgz z=HzX1?GDa?fq6oB!*g?k%OkTsiVmjnz4|jQegg%2zd$adK%4ZNPV?*Q{J2(*8@&b( z{y6J2fecwx*G30wS#G2Sb7I4u2y&b(SK1&E;a@A&w^(io58hO-2R#)r!-_R;PcqET zd&Z($lK@4nA8R7y_Lra-uotgvU=bz$_Zo5)AM!Y1ghFo zJKkfVLG0Oujg;mWGjG&tcPjN#lB_Vku5Uo#AV>)o#uMcXAdK8UQ420u_Td@;f^QFt z#YoH~s3EN_AAd=97x_NTU~+ZU7plX#K}LW6#`9QQ?0OI~pa4*HX+gCGKic>-K+FBh zFkSY#brj!cYMz=2N|Nc$ zt~7(wII2Q5jFq$DRQ;TjUQO+rA_odbM3?g7Sp1{}(!sf5^7`_Iiq~qUn@FSr0}8}H zAf>6p_t!UbSQE3IHc|4-#PIyIT}S1uu~+g`(b^%WKAuT`Wk)4TQhIo z%bmN{GKC)@24l3$ii9Eg#4JU;FM5qRV6p3K->4o+zzAr>tb=mbFQRAPS-t}5E<0!! zC!_z|I5a?QM%PvxZRuSkrwsLw6x8*f!7u3RN6K)tRoA9b$2z(vo<5o_MW3#Gh|}Y7 za=i$rhMgD}ZlKnZyV^xt+&!g_Oo^rj1Dorhs*K zD`G86r74_NT^=aYp=_MwhZAb9)^XhII9YlEaUSxbb&ZX4kA`uJ?-}G9zGZ!hOKd%o zw$-v6smWe|?oZ3AmSHQF0fmOCRoYGz0$Vq4el};pl!}|Rt>h#|AgSBy*26k)v1`dO zt!=t&4g$R`gBV`LeLK%!!Em^Np9Dp(+@0vQkvi97g@-{of6Gp zAby3$oIll@|Ni^eHX(z2gDYDt?||@_Gx>xa68}B=URy?Q)xB1EZBLyawlhxfNXJX0s3SZQC?ogaOdqv^V#sc+RO%%qJEX!sp_3C|#8Vw1N0AD6^? zAZeENQ5r7$NgAyt9GuuAC$`h!?1OFkxwbN2d5o)LECn^@wC7@hC)?*U8c?Gva)bSA zDHUU&SLWHDy4KQf?Kw&X4Uy_E=dvxaPP^ro`I0|X0RAI)2v+6v$9Q~ZOHCts@0oi- zhG;^ew*U(!GS~iQ!Mq^Zf}(*p;&)D4d|V!n1$@5Hk*Ge6v|4!^Yl}LGNAkj?xni~{ zHX)iyWS?+sM=w(X&n?NdKTStIuT-qIWB{|942WpB9zxq0g<}7NokXaRt|Y5H(sJqt zGOl>=v6=jN80yv5qw9qJlt9RcfCf`~u)Z0)pEv5F82?U`VorY&f^loDCIg>$P63}1 z;OpubnDSx*%ADwl6|MCwTVyoRw0<^vaM*9k*}hvXyN--IU)w@rhm^VY8$^As7;E!{ zd)W5fI^J?7hSYC$_l1s}5;M#Food9Rgt&J?vHSIcxVfd6bc7niCf=J*GMawUKyqH8 zW7CO@f8i03Qz7V+mVfy6n~V4FLg}L|sY>;+4AP%4l2YVCRg8=?C5JX^(b?01zw! z{(tceZda5#EzuOUYA5`OPG$b3GsR1dwNW{>8eKcI`=8(tTv}gB#;r&fqxj3$xFAMR zFYA@rlY`0WgtTPJCN5c!{!71sw5{2E*C~ARga@ueoR%GQNID_+RWedcAFUj!S(KJ}mEF6i+<~K&5W-%q zl}{m5fdMxk1`k|((^AWL4Lm~5_LokMQmSe|njnU)=S+*mzA_(SFK4xG<&S49Sz*(Q`F9R<3iha{7`Kkk zkQMN*N3x7UG6JJtXaSQM8LX17wTSLd-rc8Zf)ZwAAV8Z){Z{7TF0=X zw16OV=NzH_>HYlws`#ZSG1w7R>Bvos9MUEX*JwFgY6D+7(*Tkr{ErO8%e~7OC7l5( zxfrp}-;Kspw!lt%)guIv#zkb|^(I}BrGr9Sa)3Phqn)aT?M-?v#9m#kzCD2-N97VS= zZ`YF2NGkL5}OUf>=E(Pt8qH9ix$7;mulUt)aQemO@3U{A5^6kmO4*y4}1S@!lX84_x`qhm3hKthy5oB zS>h`+*Db+b)Eig|pKh0ITY9R;bLcVC#&Fup6eR7MDD&xaSqw1A5;4PKcm)~vVTsc` zC%NNC*EVMELDs?;D1=$Hc!<{DaUOc=->@(@*Ozth*2}n@CsWGs-VL_1cYA`Pa4@)V z!xWT|!QbDV-FNZIl^p5)>>t+~MNbg;f$uFOZf<=7(MSSpvKx;(WPPKYtv=R$jM^{} zd$UJhxlS&4FZ3#-Bo+{DI$}o4r}v=^oITNxPNwIH@30Q7)y4>PH606NGeMyW0y5qLKHJ&9oe#FDNB?lODVDy zGig*rmh2^ykR?lm)cZN_nrHs+d%PXTbIdc39`(EL@Aq2H>paiPTogA!T9mw7zeMZ7 z48Q|ZN_#rCn0wsZcha6(!bRT^}E(KptE%a_;HahP+&o{4_(uuJIn{)_|0Advmx{t}f{`TAL3;5;GHDp#J|8l=&hN#Bj z%%^8TUI`oW%$hVw1dNtliOkHvlkKKYpZ;%(P=yI)PHa`JoX7a;@1qUbd_d)^;`kjq z7A|IF)i8R8&RVPm+ep3#(~sY6pZ?lSTlnx)TJ~WW%4M2Awm95zj^3WUS&}(1;%FQm zZoP|H3FY&pR|`#U#c6u>a{li+J02d!7)GmGbT%sFFL^No)3eKCCZ7dckIdlR6l{n! zsXHIJr*k}NrE@|DUkGWYd?9f%hoQf5eRgbXaf(yXyje3F7%`lRy=N-d{$u3Nh1NQj zG>x}dj})sQ&S{J9E0mkwh{lCuKDoU%9f6QA_Ag6MA!U+hO+= z;qbn>h1|hau5XI&Qp=QJ$LcY$jd`xm=(TYw@ky9td(h@8#O`1Xm|10FHG1^u#*G#M ze)|MMAq~I7k+&?uT1e0I=S%>?wMtTtw>te~f)o9}W>{SQPdT+mQELgB(`bqbG4;Z( z%3EGM>`oXyfC;WdGF}nfKlO${exP8+@v<9jlUgv^%x`?p>v0{ZOX zoBcJ+pmRM)Lp(MYF0xKDM5swBSlW_puuhM%YpQBt)~Wm25Fk2y%su*tL&9CCkc%rt zB<|^58VD`gPCB%Mwn6g#1XTJ3{f-vWhszb0D)zwn9xu0*$b5I&@vtUGkaME!JeVlT zy@<``@jS+FVwar1Bi!MX%;{Bn1GChV`6ciIEV)LLK9UpF;7vZ2ms<6`*M~deB zemL&E)lsgSz*3FDFo;OoIs^Uh{(;+4PgqrghW1tI(xGc2R#cC=#u8*JSYZ;N%TI}b zZb$Et5GI{1RjP!l;vahCer&YHT?fLZ9}iBcQ7ZzmY`>oFLpV}tB!gOFoTag!^=Tkg z1Qf=70AQUT%U6j#S{hl^SsP;kcs<$d!S(i3tpIV$63sgi4H3(c8!xq8W{*HO1NRPeI*~VKZ`9i~XTjt6AKh+xm>ffy6fhjXsm_@l2OyMxcJ}OR=0P z16yPqF zk<(FRT&1O|m2T(n;Tvhxi+Iiw5mM$i|gxlQ>PGL$X*oM z6JIql<@DSk{ zo>4XKE|i<)RmdW`O4ZSUu0N%BlPbmUkB4Qe@y}hBUVuu;nfKkxnPPo>XN+r{G<4j5 z8nq782J(EM4{TuH71j~Gubr@~ai*LhbWU`Jf!c})>NaKNmgk_KeF2h)mmA+Apn|Q$i0sFaDW(YFfJ%cZ0qW5tX3@CA6zo1i{pqDVBC8I(1`LpLf}4-!1?A^UpVy zh5b%55tcKcHtJwj{dh+5D>!&hfkl(Rz5+#Nr2}sJy8@ooLk26uDm5G^MV2kuB*vX; zOR6N2o-=*j8#i({S8iL9HsIkdMl|PL$C=C~uIdoh{K31MzQ>^C z$aH6Haymap&@3b>-mxjICnyzC<(F<=3!p-OVJ+8$9=jbh*A`~rv3v$=0O%K&Sr^Le zD>gL60O0MkEKFQuT!z&&rT9{?9$C{8X-;;TKDWn=*sQriyO+(IQxF6-&uv!CY;}*q z4hurWe)49iVf{Rn^uYtxXWs-C^*j*rVM6#U^IAoVrqrEf?)YVHo4dxJ^DQQ~ zLl`a?4LUho&BmOYHWG2X-1v3eKEn5rfgCB?_tLVD;}dHG#dB8%wDEGdOr1Xk3w$YKs;Q0 zDWaU27O35#nsYyt1ymuMVVG0ENKdIO5F<2B`%52uv=^8%rZF-htJ8IXT$ztHWTPRo z*F^fpcPulA%}DFkg2R$4M-M=B>2LS8J{|K&oK(&InFdvtMD8YOW9lP@20i=lS|i33 zT_{jY_>CEO8zIH$l$*y!y=}ngNa+(_clyj&j)D8tfo9&Nkx<#71D+a&SuO0WU7-P{ zI;)d1Zt&OCO8&^_4*)~#8EEozZ<_njhrM=`5H-{^RI4_zCi_Rfv0U00@64lXQ23W^ zP-xe{w@*g5`@K|KazL)#p8*!_)|Oigrod4yGjki16O^K^P4DjQ{xVI~;NX9<&p6zV zeZTd;%Fq^iE$LrUTKoHtUQYt)l%9?#5F0{!`X%$v@;;qh_j3p8t&RE6WfLz&+AC>P zY+yTC5BiwHPSBbpx{w=$jQ&yrdZq^en1oAxES0TUS=2YM_l3$5WNy;o*M1ArN!7jl z<)QHu`Kpk4d57>}E^w)aK})+1J+PJtF8UrZFrqA4^e$q`k$+XViyFrIrY1{4?Iezb z&UQsY<++_AS6Gm*NkJLtzK)M;PkC5*S+YX3SWtpRzPl?`sr|OIa8EZKp2cYq5K4BM$ z1&Ks5^cvNTK)`cshvUYG&Etdes9Vk3M0Zc59+j_EIA}|fu(~!&Oxc_0$uj7w!sQQf zgZVEFmtPL<{Wc(EQ;`KnNuV)fLW>n%nQBNjyWHM=#f z>w?y%Zz?*&c@r?ZYat(=zCs* z(LSU59m)-#HixQ_l=W%qp0E7INNcQJY9vbWMW}bZ+R>K*x=ChCm^;J=soC$7djXvw z^oB4w-rhGxA((rE1WoEba)Vtamwqkec_0N~im8Q62E;Xsx6iM^X&W&QZAH)c)`mGF zV&Bj1;l=aP_-;sn7!bK5_Mw%9UROq|CXN}QkZuLF6RzeI>*C^L~r(jGJh|iu*KUfzXB~QmKS16N? z0VNQeiui$2((o1zR3x+=C9{dVLxb?{rV|b@J$aD=uBK{{nxPP2|IfD3CaXajI#5>} zy@57Xt{^m)H7tv%Ai#(BFm=-1XHs%=zApV*IHek$`Cba!JI?4yG6BDoP?pdODpFVe-nF{SY3XB!yD(K*D zHa(*?H+NerQ)W>Sdk7H{$1q==9iPNeK~iEvu{U7qld@5f^mw-~kRjV{Yt~oePr~oC z2d8geZ58)ouEa!Qwh;8ops(;Arl8O|p})yHoC_=5UKW+tkI!)8M+E;Brm|Z!c8@O^ zPNHV0>d)V$xOFJqX<59P^t1V079HwuX({dGvEwb5@J_J!t!kuPBn2oF2q4=3vyGom-x4|s%#E+Nh0)Thmdi#eetfPUtnx}FueKw>J=NbVs{A0%*j74M$69y*&Wr9@|w z!w5U+CaPZcEVY}#nHnPP4EOyUe8%yxMt+*IQv?S=zc0H_abdAmPDx+|Lgf`fK+tYz z`C`zjFPTjNW_x{{~K37T_g8jXYTge^ZpiG!pRqwRqf z+t#~OE)w5C1_*}&QU=VS!@lzTlWLST@LlbL331|?mO&A<${$}r!b->u%agCljqfWI zK7QNNkf7{t{;3JGmxRQF6RcWmb!5BiCsXXQ(Z~HA=hQpVU9o?0%!e0s^|8xG-jBBn zUtVOfDaSrMCg{xZko;Ms{>=Jkvdg*oAIChu-E z_51o)QVx+UAFOKZPcS2ZOSRoZo~VA1PEBUR^gfYYYW2~-$RW;00RKJ!dMSNZbqH<# z;CcAzlH6~AX6<3Kzt5xMid+XeYH@f=mlK6jCZBL}%kG zqHhCn2`i{7BXM$BxZDA?U5L8DuP`V6^_k}I;~6X-N}af$oz>Z2?D|ykm|&l?t%cRi z-o+!b?53r6>tbs4F!;{HRb^&(U!X^Cbz$b+#{9ER+?iTyy^RAfrt3`6iQo15TI#l>yhiuW}}e5ieCi6*Wx@amE*oQ ztygc4_r}e&Nez^F$8QKqp_MRDfstO=Q^U|GA#5v zfaCTYW}D(V0|@MA6M*-L_nG71$*@`;xXsEG>rG{# zh2G&KDQeir0x_ejAoJqr?Bb_S8X58G^*S49_K8X^srJ z<7cz#o)a~}YBYG)bph%c$Ut`PmZSm@o0vq*gtZVPd#c;;iW<*wUPEr+k)ANIzV3V? zy`*n7{8vWIe0i^vf#8>`{bqiIJ7YqFWRjh_T7kTWsiBeoq0Yo!p5PJEo$^)jRm5t- zZI{P2+d1zeh2Z>oPEo%%N8pK@udndpvZn0E5J$Lmn&@a206DG%F2C6h_Dnz+7_p1F zx6#%U+u0dlJxI;7$)6-TTpTs|&d~6Ea-j6CEf$4lsGL{J4&UlCP?on7Kb5TK8ZUkz z9(D3lF|V^$u|EbQ_X#yI+^=bA-PxhV7DxdHuq*;6Wbvn#05VzwHTKNY$PpKWDjeP* zBBed!?$(O@oNG{^@STqsyx%$U^DjiGG7XE29tlb>5E;ib`!i0Fi_vfJ?WQJ&i2scl zbl5?F{sH>g_iVs%`7prXAZ2%nj=Z~1#b#miu0*#3(|@g3Wb%P>QNRCfvE|dvH7RE$ zg~)EccnDV@W&NRvRi5PZY~fzS>TLb4@kwj`SlYLx5(QtLvw(dJ>JH?9X@)f)57BGD zpQh~c5T}ULvh@-9TN%nX1Hft@!7GC_plvPBvdujWtf*WmittGkd5<++^d8${OWf6U zPoV)gq?CyYzy(AzV>(_tI>)hRxn45QV7Hw8m?G0b7znwwxA+@f@Gp(v4f=z=NTK6( zVHg4>d1Iql(Wp&2nLPk2p=&}zLur#e`>y`~<(bg06L+5$@Jt5WyJmdav>)k-anbNA z-!0pMTL9Apl;jg5C4rc}uLxW~WJHMnm4)W_T6?ftS!EG=vDS(M9alxhG$jw1Gvy?e z59LGGcGbn9Yux0UaOR#Vvn|!CKOo=#i;VaSf(dOk3D<6Q#wcbQs8=yv!^KK^u)EH} zya@;-2G3Q^nJ^u7s+XSx%Eg7SNpnUJ>$w;oL=Mf_HVq<^@}YmBU5>*&2Ykus9lPM2 zwaIONv;O?|CT*j}8$Jb@*R$Kh{PB#W;|il3Gzg2&Bi3Kq?_xX;DB!9_L_q61^b!k4 zC-L$P;Afm^S08euhTlTcAe?C#biy3WL$vxySBa9UlTV{Zc{Ya~N6F+ecrLY1i^yWs zQG*Pg0ma%AD((sAO#xS^WVD-C4Wh8|vE3*ybtfT|JwGwJv_W;rT;~MTIJE z&3(JKZsrmJ8#sEEocn^8$y&FZ$t+i>t$E?26Zu&h%G%R*ISZc>^wyu|8OYG_wAtDi z;$I!32qZTsXAYtvt(ER8u*knq?zZd|7H90*#MA9niO*>FXXU!*7RB>%GiMy?+mg?1 z(R*&=GZq)Oo%SnX)ooN#bevW?UU8eutN^5E0u{F%fjdz z+<3zU1XR{*Kwj=%-Cz1wL2f+!O}C1cnM)^aJ31w=b`3?huqj*4@%x+7^fx}c89YVM zq0d)E{LTw`5SqeSZvuC_bux)Vg&8em=g8nfIm6uDPfv>V!zEHu*S#+pNvw8&g)lI{ns5(nt*yRb{NqXbEURS!TW)NQ!b=2OU8bR zcVr2_dfC^7O{m5Y{O5%X;K!J6mh%`^G$o!-l@vrYMS0Epy*7>E7( zr{8nl&!?=0CElC7XDpmPjEgp@vOq9LieXkgCd#pByAD1S56Xw z;l>O-&x@WyO()QZyxU^S>K97~*MrU_CddtKI3UB;G$5n1ctj<(KTEAutwRyBDwIk( zBue{$4_zMP5hznB#>pqok2+F5<*InhkNBe?;mzP;|5k`-LdLd$Ac^gk4aqga{TtJ0 zhe-in#+;%?>e(Ndosr@-GSfloy(uD0mwuc#jDV|ZUqcTK^XYXvz78gQ^7{F~!ZUY)W$& z9u?VrhC@|f!H1b1vui0Tth{_L;o8m}JqQIU5hphe4TyZ1p5IIGO(yT2`n_1fM?9Js zMF0H^#rrk-?f9ZaQr4|szZ($2I1OA6lxOL#qe#^69xQw{`TVb6iRRs`T%>szO=Dx> zsCOxow|wCYnwOAyV({w?G#u30Wg8Z@s_1}5j;ajnw$JX46Yha3X_x?qH|0Y+lkX}g zui@y?Ns`doo#@5EtoP9Kf6SS9ji)WX8rP#e$=P%OXQzJwM7Q>oX^P_jJkPLcylE6H ze|%nxj0-oGkrL(UC3Awm{?JFC*fSC>t_}D{7TgYPFX@)t8TYljBq}1X5f{K8xOWSI06c1znHMO)a?}og`wY4aZmw5G!wk_3>pljd?JbA!q0=E z{8TDX6KI|^hw_)&FWUiX*Y~`JL1JNZnk^o^oA|9NZ&peL&xI+;Ci29A->T$n#uh0a zDZJ~t-7miy0*QrkMq&X(_ZcKauaAsibcdjYis1*fB&Rq31PRqFv1u1d=VPqjn07lP z`f5}xxFKu|%@Yc6!>GXQ8S4t-!R;V$wg(F`n#ANZ?ieDNQU*iPydE^HC3nw~`{?zN5@0d~ zf*d&s2PAIxIIUUzYV9AB4uWJsnCRm3{<`E~7IFn<#fDSwnW3}A2SBlwv{DO<$PTqI=7-=H7) zBip*x`V}1>*5@h-AT)MZSHZKpq8%jU!&6TXY_$fFQy=5Nj9X*_nruhr_Ar1gF0Sg} z2$s~^yQpcGucxO;IJas->B>mi(qX3Gf#oYY3f%cH(O&swH=Xd%d#CPa`l-=}07bsP zThb0<1oMYPtWV}UaxHQk8Iy|L0*uq~vA}#{&{JBy9TSV0ijV=CFAfi{cMx(M#Pe6# zmkK;Im~9M&Z|Iq~iuJsFcN!D%EU*!+ZBZEzk&hQK5)1!CP@ef zFYXTG1kR0@$OtEMzjNiSO2Odq2f-dBcN&T@_p^~lnhhYXT@iCc`YMK?ZPy(eGZWG( z_@@!ZxElHNp~ZCmom%|ZiPtSs=yu^B@~9xxvQsN#>tQ5i0(8m4n{QR9X>xD8__m|k z-TLe5(~9!M)RCilTiokpOTzAk|>F(KQ5MlK7dpGnMi)fMR~#WrrbpGkBOT%Pbdq=(B5j_b*9 zWpEFvGV&^J zn#WkI0)z4I)(Y69_=CuOye`;Lw>0=avt+B2#Mv7FMJgzyuSIG+GBw(l`kf?j_-xLb z0a~}ox;*csjOO{2g3RZX=G3NBm|yA~W0+XW@WBVhHab{HXpzgwWXV7PoXZHuiXX% zlTF^}dZygx8X^t}J`FP|6UcRi3DQbI9seKC5VBo^ic2 z6GqMWVk>N%qY@c%>hvn2HX203`5nsp?cwUk+k}l%+JK_P6j+sfh`7n$0GWLxCR<8I zTtu@*%iFxs=gtuU2JmKI0cUpI-4}C>eW)r~ij}VVgkQC|HBZpICg|!lCR{piF%AV{ zGp=}SzDT7?-`9`Bgp92?vB&k)eLYIjYQ|Pf=aR)yMnPV^UvX1z#+nWftU!XMg^9NV zHb!5<1Tpsy(4+j?ACemw2F${g5t^nJW`-!t<(4Q ztLt!&7s9VWpd_tY%anwJMNef7I@9lU^SV*jLh7Fx+TGfA>CVqHh;^QCqOzVe;G+lh zJ2av46T9}yUzuzMb_nfy(&yqswUPQ-w@+pgg6bIy22;7t5ka$j-EM8Y%Vc^`Fp{IL z;j1Kf;878?qSH^}7H4=>wH~`n-1LBefW)$^@>IGyE_m9&{soXst-pmIU4UKAvbVAJ z%|)*H#)6J$d!6=H0qEph(Hu!JC44%V{|9+?(e(Y651 ztN48%GujZV9z^ni;vheN?KbIHVZZar*OHY>$*jMPbTGhSJH%p|*eAjddFufZq2()7 z+`1{Az!6nEA=SK%c+4R)^jK7+zOSV6&G}J3luyVQ@#DQ~^8|ePc6TZO5SuJZoj86k zTbE~Dy|Aw%=&d})Ufj9SdLHh4l1Q(8_qdth)bq2 zx=RWZqCj(13#H+n38kIZ4wVz5mL?32ku*m=ntAHJA%4i0G;KA?4Ly&`mGg zTfXAWen*>iCvQAn%>3Yhy~BxTy^H7T4$M0|f6mMHytL$Qr~TiXJcpE+{<@PY2z}*U zRk>fi&4sH)U(o+adTV5#>M6d$`k)a=wJWA9TI`puqj=y8dx|+(B|M|SsiV_0|Oaot8 z9RmvfSj0jfbn#(VJyNG zmy3+;^%3x8@|%Ayl9d;(Ec*x3)+*ElTtH*IzCQ<~Rnh_Zw7zW|$%hi|fXO_NCvx|4EV>*y3yiLpvy&9n<-ygN$&+pm&8a^9^~BQ_^G8Df z_^Wropk97Y%}hDHcCN6Xucf>7#;akp3QgpQG7h^-jF(#-`aw9kA?O+2`FPakQaB_Bkx~Z@mn$CfrT$@_Ul+%xh z%kMchh-f2;@+#PLj5hxjZEZ+Rn8~WdTi^dWt8vu$$yUGgvn@SmyH{bq--D_?yXjYa z{j5`Nj(XW|5RzU z#tZ33_7(Ab^Jhj@ljhAYK{mWswLuW(Bih&=vv@5Si2y^RHv}*oOw^GboP97;ddfhx zn!9Z>H63L}V3c~=p5{C8Ym;|ZmLF-r9L%`9Ez&R$CRm9kw~=ZYUjtn;4J_+_Ztq`W z;;AZJ)%{N-#)(uv@xK<@660|&Y7ahLtgAd%X{PoTo^6?dY|}krBU{O{&4&T-LrTEAY3X4z4Go& zWaIPrpm}JST$YtKrPLK2t>(lD6P}SBBu_P{CI6eTWQ(`e@fR&nHY=N%)4S-&wtxKG z;J0GK_wO~CjXhtuS=lMW`wKrF!Uy)W{`0sK{|q?tchFzg9gL9BX}wR-`FbtDV2m%% z)TK+8wuNsegAuO?0bW3X`+qLqWQW`e(IPv!zy6JWe`soSA3^2tJ{2!fCJosfU8sWx zltQtl?OzblWP3_RV4^tGKoIHLaKWXdF^ykBw+fu+50^Oul0dhlWB8!UAO%p;D)y<# zW_EPTyJ8r6?{n`&uOuZI#=j3*4+BgQ--F*QZA4R(pF@6+NR}IXA#K*v->NxYS;+D$ zcGauz1r2{65l%kWq+f2a+;F)vQ)ro`{iRQwnE;Pk5+SWf!J@GUP6lXRgl34b&O06; zMOsw%Yv=-*SpZ$zY!Hgbjg;KvbgG@~jRKNtKKYq-l zYP@t1yMd2Auzos(qQJg6yLlB5IMxaXnymm^j17|Vib}hkcgd(n*)R>~AiB4g#jn;? z@GLz&SgqWLpAcRnVL+@_(%(Hn;5GtI@dk}^lWks4JYzAxQti~6SC$PawXbv+&pao; z0X6Qlvb_4;^nz0mJ$Nji{K>sP-N!HNNTZ$CH13gYqLCx?NGRwAqOcSW!~p-**1I4s zT3*QjrpQIl#4NG&fkKRf@Z#=)9QWi`Rq2ZT(E2#3xa2>RH`|xP+_1D)k?WCP-ftHP z%*YEK3>*)6dV1ztb>WUf`>^#Lf1%JEJr6^POy(B^3>GzX%%77@Zo7;#&oD+w$f`!( zFvd#s=gu&94PbE|HX1O7XSb#C_cJdkrosHYqNc$uk(r5wB1IaVZXlfX28Yhf$=Iw( z?UzW_<#6iSw@6p*l(L0}YqxAefx{b|?c1N77jW+U)HU`UFuNcc&^+3Shrp}>L0cSN+RS4s{4TL-*_uS`i zPrxkW>DE8IWpOFD9|$(Hb3S`--TL=@Tfg7MI^(X54Wbs8A@;-LrA*lohO3P@CSRLd zQgmv*QlRPHASiijn+gFXotG7Jw)$dL<7km}aT|)?SZjUhh(*EMk9jqQ&--NUDF>^s zOlNak)+Y4TJLk)6C5;;3t9rbcdnhw@)2Ln)3R2wMgV~gkBUu^;xP0O-kEHF`J>l5= zL}CRV^ND}<)S^v-E}S3f=Dl*vry2TEBIN5VsI($1DhG^KugL&Pj5FW{*Z2}ei)yVI zySTM!Ik*4s=@3Nh(@$!d1I2?%79B7mRMm{junj zuRI_}Ycz`T!1BHdXlCuw_F|Ih_3{5$-HjZB8Hri*&Eq@mD%MaWsz<(>eIOH5?By@< z`p8u<=EybL$xO&IacIt)9GRH(N3-)WVa9Qo8qcl`?rw`R3e|er*GWzk-NYHK=?i3o z=1=5m|7ulr_;bGliK3f)sy&OCxj&qAy2+Q%x37V0==ExK?9Jy?G=9zG*oL#ZA8k?R z#}Y#T|Ei#Y9#K3Is&nH9#|#C+hBs2MQUbc}Sh;>51-<4!ImJtO##E?M#b1Y@RjVro zi6r0mFvpfNFHNd_Fd8sD4>j`_`TO z_qy-o`m$u78F^vO#<(mG`__1Fg#8a56778ZAA_Jetn`Hv>Z`OY~bD^|Sx*cFE_?Saw8j3Q4>Q68{6?_5BSmiiPv zdDg3BCqnR+K+|FP{;3&5vWtsu~cJ@_#}n*D!5WGN_q+dj7`M_v-ZI zBQvMTIMHkP1_{AL_~tygzbs;E^1Hzw!3ZA#TEI6vA$S(j1S|r}5c2paG`qWU4kIg_ zXsMq?MmcSM`I#<6W_c(xTI$y>e@qNmnDxw7>`~r)dKnO|%$g#fF%o8IR4L{jV0Js2 z+1loASJHR&G6g)LidT>eavAjLHaU<#9|p%9f+;pg==5(*EM;1|o0@b1wCt(!Hk2M3 zu!BjmQ`XD-_PqPB8(wv~9>i=1>XpR?Xl1%S|Fg3*uW0ga6aAqn47uOF+!}8~M+JRj zTt2z9h?AioA_Wy8VX4tB%MK5Jc6woh!W*D_&Ux|;gEmPM4 zH|)%*cldJm_zp%~(7Vs+K_-_x=C|c{%RgZsmJ?c$M2-N>U>#qIb_q7HFwBm`%8tR9(CM# zAzfo6eVHF3Bu~o8el6i!u=b@HEYrh|STtrYWLzX(u$Cui+YD_s+Jj$zHTK2>8}kY- z_pT}~kfY&h2B>B!TD$=n2O?K_mUMVVQ0?xE@6i$__;9+YzbTTNX`I;l6(ecH3r~J2 z-_44&Qnr3kh{NnzZFYYF}T+XDjkt39{MT1|u3lyUh8XLqZkv`MBbP3CC zJSr(DDgW*~qp+Ck{j0qfC`po>mwom`+EBXYM@aTu~YdtpCHDU|Qjl_svawN@(2>5?%Az*P&E z7d$>qY8K($`w1f@?_Ikb^Y!W}7XgtqIn_chg%+lwF+TiNvd|K-hExY+S++h9qnkSm zJ=DOWYk#se4l7C`mN6-=JhN_7Q{@?UHDM5%^^ac1epo~?gN?BuuTg2%r$mvc?@ur; ztljzgRrfDsS?TM95#X_DLol9d_}mfD;1N_o+`D`>)t`MZBp-1Q1ra3j_Zl|1j#y+C zX|j5i@3ylpkBWbIwyAY=LWj)XD^|IvoIdq-NZx~CLuY3lEA_R<=3B|dy_=rN=oMf) zH9)W8KQr73J0g4=IsU>_^0nosjI*I1sf3I#zj4^_7)(=f#bKH{gAZwpw0j%~BNK%# za|fzHAgI0yt$wIJKuaJDkx&0@9ISHYE?IYoSm3>52c{)u%i8kl+D(Z6rbR@Flz9g27|774ZEl__MU9^D33e&; zGvgfODnaM-Ox}};BuaSQaGtS=O{9u|9bt|!6sh6BrqxcFPc3@C#f$*SD7C(N@YFMiKxv;Acb zA%i{e<7CwH;s1?Zia4!Gjv11(t)-QtpeNGfQ0%dSADHd4T9nc2@3hY+%A2~UhV09o z^L^H=fh(9f>16!59UZx6sb{lCzT8`_5Yz`%l0UXl^{+s55wvab%&?H-(FkK9U&1jTfw^3IVYV z6^L$(DOLnB;?n?5(NIo4B-F)UPAbR?-l8CEY8V?0-cKeM`~K1ILG>vW2P3z18+;h&62~ zDhrULlmV|No}>eUjqKkDs1h-;EjDt&4**Do{I`r=<~->avto(CwKMI1Y%!o`&0bHV zpN!5&n0{F4dzz4 zI_$CeCoRoz4U*Rt^?hUH80jZ@64>Ye2n);yit39K{fZk{mBCg-`2y0L&sIE3R7;dB zf5{<-h%xm`dp6pF7aDgbGa9ZrNHLkNmalvMHrJ-R2(NntgCRr*Pql zL|M5`cnvsPKM);h;K5;8kyrkoRbcI6bk zSn(b7sb^5p{I^!thut+0L5-$U+Hcma+kE}0 z=|#8;xF}uj=s~Y1ghg+^o%&JLlZ=vSF77-7I2lO^YM=-Z5|&G{28b1|iVA+`0aEBU z)R#7lZ8)8Ab`l5Ck=qXM*Rp)d{7i1BRz2nAF0%i^UM%^OYpsHJ`2PhO{}cYYH_jc{ zp;4pdu9M7?xK@6U$IRI2Vs}_0xbk&#lRuox+Pk!Eh)aXtiVtr%rf_x%#t>^J)$;!| znFG_Hj?W*EkfbIi`932%Z-|S2-`PBZkJhE!*}5HPTFBZ7ww=kluChJ2h0w%fkiDIr8PzeDYucnUftJPRJ9tR;?N3pVpic5ftFd_ zF-Ygkwu>2e%D4Alcf=GIk~d=p2QUYtPGL-1tk}o1MQ=QM{RDe2DP2WNNB9cAtiS-p z&D_PsMG_uO!ia`l9?R>1!c;0g1T*UDWs{h46bHVOSE0@gyGmR|W#fZ^lljwi2)yDR|X_2`B$J$jN+kd0?)6 zkNKNGTsR#aUuONjdi6DI+$0STN-9(O`oc9$gRCojh%KDE$9mm>y)}o~T{;rgx23Af zCao4f<(b;FqPaNmDqWlq0gF$U}QaITBYD2 zc#m$gTM~8WFlh%x@nh^fVzX^*{~-@fBRC89SZiLVe6{i%5%VIbe2{5`-{G@b<{T;? zby|M(EPLhn*}w@M+YRmvKgy2uU8@=!PP#XR>5L)d?)npCm&kEb94DUFe#_Ij`tM14 zDttjSahLo(9G;CNU=&f?chUQ8lB^QkHlJwE5?<}V$VCf3nd(l9d>S{Tnnc6Q$`Bzk zi?Sase|G=W@kh5ey?eDkQoPcC_eGKX3~ajZ>U^cq+GN|@JGrLCh+)m&CFh;#vph4o zDbFO+ti?4|V^$|gFwcJOw)_L6BJ=o`iTSc9{)Dfn!!nqH6Y3o|KKbNqT-A7;@e@Ox zQ~H~;idP?5FtEPCZL)TDwxypDSBQ?md;AFh9hK9R5^rD@DN5rI)on(7ngdz(w-+Dw zA@(BjV4RXNfX;C7=NBCeXZw~;bK^^tC=rD5jWGutc6d5?rnwUn%~J>|9**k0gMHz^Io&)m9)6^jm02IYXfJ|2#IM9@50U=8NUUA9Ugai&qn_6F;m^@0uGcW_+Va-j;7#j* zbiYZ#2zT$EOaBpQ7SNW&N7r&|LUJ+o>`GtE1DWH{=XG7mN)g6Y*J8bRz?(g)Za6zv z{$^75Ur*q2u8NTbK^%JgPyVuFbr=rao`bo`TEA^@0@G)_)tFqxA&dsvj3vLCGIKa# zadR7ok*ZaoH*8W{1YAZhFG6pR15D6FN_w7$Z|f2XDnP%gSwMTqK0qHJw2pdDZM7s@ z3CoV>xp2^R9NM7u5$i=ecd6Mcef{+w@sQNOinQ?`+W~#XNh+6oMx;5vMUwh!B4rHj zZyyZ8KEciLa1taax7<}7-ZXpgS5^Nw+?_e$YC6ZhBrO72ELu4+y^!A?ofRk=GIq58 z!ip^=?f74pIw{U+IdhN*HH<~%k53S(%TNbm-1F?(kaQNtM7UsCx?UZu6G(Hu3u%2r zD}1@YU+HXRk8t~(F4a;JgpD!FQ@d}cB^X;U%$Yb({E{~yjBTJKr*LPtMhofmmZY9(t41*)m;MG+UoLK(ArWPROuYuwX&{={P2Z%;%S<0PcJ-CY$EU zGzKRXV>>R=`ks_cAN#9=fhk?K!8s46h-8zV9HY7MfO;dTG;9`ko0nSHv!z&mi2s(# zz=A)QCoyPvU6kL0R9xJkk3L9cL%HGS-j6{kzwzWCE{nkR4uoN=s`7E0e*AOfBk-n| zlbY@|Y7mW!`=@aIexJ5Oq?oTVrwTaJ8E5bx>?r$dG>ZK6!S5ogrG#8>ebB0yM{dKz zYr9M-*Ti>@7tPBTa7Y|;sb?gHa8PFaSOxR{kBpKbOcf{0J zaES~rl{#IYZN;?ky0|d%*F;9g?KR~H@AzM-Iv<+3oU7V$)YUd~FkIXI{RxI-4_O?^ z_fb6d$;O}MTRGwjNe|lx}cxs)UtR!o;CGLTxY&}MXj9bY}wQPeTo*}gH&NWz2 z+$`=v{CoK|aqcCyWWpkiDG6k1ma07A-`=G%XXvIfWSANTIH&z_*g*l7Ta2i{%$=s- zu5-Sr`zyo13frS`gBMmubv1qDNFQPlzV4yxv50`;-1%kwLkJ8EQeW*0YvV%p?TPa^ zn36L1&ExcLs{>5CdaSscc%;t|=b`Pczc!ayS=8Q$I%kUVU(%v~TmF%r#v)>7>zxUZ z6O1o6JN5Fke;vz(EBgIUQEIkY5*!>4fHR#Rp8u^uo%-c~jUbh1>tleNWdJ7A_C-aO zXJ6#w?csMgAhT2d`YL~>R)SuM8s@fL7j^?dGsl#o$svJJTofMvPSCL0Z)p^+tM*0y zz=*nfzmlfMXtG*1XyETK>M`Q=IVn+vH0&bnxyE;xcE^U*h(nk;N~9N+#|^)Iq3!0v zo^MzIXU9#wDLRdMRNh8Qmlsr*S!#DBx_~!c`={@fcB3jY?L!N)DDzwi1q;r0ttnE9 zy}Zv+G196neBmYg_}&KpucDPwTSY79@#@S<77=(<)?S#m$~SBI2!2wM3}w_rB%g3m z3zGj*o(=3wt0vjo84T(qwAgYc_%5BI`>x$ZtU?eh$s!#5K70W< zqAf8&`n>V|xk=O9NuTd4dH+Zt+B|+lUTp~rY2}ModgxJwyaPm_KsVXbLr@kH=9_%u z)GSQ#6jqRMeTwl|_Q7%0n-7OJcK9W|x5ebuWnD79G~8s{a>J`_HGDRgW!23_wOJhN((sn2PKUDOJjFzLrD- zaFwA!leB%F_i5hp2PJa)ee-sHGG{Y$uz`+Tj*U&EmBj(HiB_X(IPLm4{`JRXM>#8E zE(h$|dDjQrM=A0Hd3^;dfyrFmGRa13RFoQR1&iqN< z^eSL9K*HwfH%7$?VVVG98793J{nk@NA`X9PKIu0Q0je|f1KB!|HU7NXYt*L?-F$iN zT0N=A!Cf&<%vtm_=+Q=L;THVSvAJjs343*Z>QE^A3_Qh{PD!ahP5s%CA|s0tdzbfP z_u~h^^^(*Z9-dWr7<5A#ndATCWTrkmiqiEd>qw5>0Fp6W{sxzDD-0R$1{r5sm=>sO z39eGlO~d{1GB64+zl{Hn{mEjr=RM7L`bpJUUpPB9ZrfZNPJCbJCUfp}sEORM_1Fun zxBqne%g-aqr`%syy3)w{uX* z7<*nTTA9jq>Nv*sFOc zH&RXhYYI3kqT|WpeyNaxi$&)+*+m>!1rpQs?G<_0Uil6^GQ1v(xVgi51-B&LPCdwF_v zBL3owFg-}7JdpT|V^UQU4@igq16fyML%$~O&uvfNoG(}$z^{cO+-(6-Rx>~8F@lz} zq%D7g&!YDp0S7F8so5)VXx5d9d68wK_7BanDm%SG;OGt?hUg&VRZc$0Kbx$GtNyXc zKhJBJ%q#u&z{BH^N;-Y*VrZVPVD)L|8ZVX-djuYyzfcO$(@K84h&04M=9uFa39k-^ znkh2zrmXMLv(v5LLJkmFwb(cq$E$8WOZj0rJ?W#5t=YI?L$+5jbNQyra1B~?%G-Gh zD0}50O%)U*DnQ)|rr51{ZN2lHa_<%0L&TIH1#@8SgWURxCkc@%)4j#}!Z;{4$$fC` z(Q9q_U>xk(m=aBkDc&(Xn1Vr{*RsBOc=L=>Kr=uR>e-VEvYLGUL!a{-{qCTKYLUGr zMCG|a;1Ib;RCb_+e6zUZlDN#kY-bZdckP^g%$XIbT|UCZH=Kc@t__d}1o%5^Yqd2J z!Grd-8~hAcSJ$xFPpw5mM(bgAikqWls(hNe&+uKkipJ#D&?PbSv&GBR3LqocFT!Pv z-x1Egr$;~bE|5p2g<%#e2pxnM+j&#$!PL)Rp;al<;9TQehsaB*3sarxRmMnVjzv~7&2R>KD@w< zG;DmC+zDACD{Y!(o${Ki%W=8u4F1eI*tOd%_?y#u@&B8;25gR}{&|;Ca1CxQC1r39amEYPo8<2rfZ%YJu8HTCg;hCT-oj z*;rc#5s!R<@azh9Cv6vfnk}%0aap&dsysA)O>yw$N0tG3pS|o&^O>HbhlM%YuKA{Q z59JO@*+qvwQn`q9Ag0R1pUFe<*dq{EZkDL)p7M5KJe8RUfPMFxo6Is+`6jn`aJuUF z!6@hyF>Y;t#-<`Pr1D8IPK72g7=-lh)$7(XE*K1IU#7Pm3TARI;&;&L6PXnMT2;52 z%WPA%NNmFK4=!$yYCgUwgujc_;VzV=JYMXAEDb@R)+sRGo^U>DclL)rguC|lY{YIk zqxwR^5Wax8*k^5>QK^eVCk8|QywD+d&&i5Q);g9+IeL9f$?MtsZEd=*>av$B>^bpb zzFN>SCXDn_k@WJ=wTey#U%Tq&%UjRjBkvUfa_Y=gGgdN`NstArC=gi3S;NI?{5onf zr&Z>Ba3{paf)`_U-!E#Yc0$aD92^24uPixx^6k$*SE^D{Sx|d(v>{^@$zk3hu z+XEpLTer(8;Pd?65K1i~W_3T@ulh%aG5;o-1Yi&)4aV&mTf4YE{=HyhYmtkY%?egs zUlS@1DL~Vr^pk|mDG*0LvCu%`8#Kk%gG%MNiX38L5#IxUe>iIWx_rYQcYo93>)7!Vh24PFXTLlRvRK>wv!(*J%JL zmjitbg;2(cfz++$HG!8{Kw%drADZGI3nYhw)TTZiUG`|{g@|kK>*pMn4Gk(u@me&B zg-uwc>a1>J4|`T;dcdj)we@n=L|Js}D4gZKSVSdB2F>@@I9(WI_S}oPhBYftRjoDF z4di2NAhM8TIK3$sE6*5Ye|moN*h5Maz)2MQGvmdM=3%7HYip{El%_e=(v=~}P{SGd z|NL#moU2m~fAM(!Wxj*Wu*gQAJm!|Qs4}$mir9b^e&23<`)g{giM`jjtSHj|`d{@2 z@7j^vY0;+#chldUIpA@B(Wm$Ge~9_6Vx=`c9TRFyjGkY3V($v$Dio?-Y*mwk&dJ@~ zdv&la+2ly|Zm*K!wj6pjGVEi?=w)}pl1BDq>$h1Tvd6mIosEqv4vCu$2i5^WtaZkf z-9;Epwj4?w>|D0!kLoKM$mncaqSdIPKOPxrUl04O{cUaRUraeo&Ap?IMe|)|tFpO_ zE>Mp9j|#LI!zdGK$hrnwdQ@n2Vq;a;lc&y)xYRs(rQ?>+kxL@(k7~{TSn9`hUCKe+ ztT<%*U&De}64L91h28wHdlgr#xU+BIE~k|%{{8ZCfG*LR5FhlD!?kPIUc)KCVWY?X zaqrrLLC%Dh;f@@jBQa;^_NN~|ebSE~esXqW=ze}ynVUN}pW&voA|mt$lrF}dd{dOq zR?oRe&X+F_U%qS+$R@#y-pc}}{T&zAvqz7@1W41~tCMcj{pOn$yx(t->)0htS+Sy@ zrN8xVPS1vsQ-)7{#cdycbWAZZ6K*6iifc9^q$ut6vZ!cJesLdfyP!?Snl^1Z-m*up zUd?wNKi&bs>nh^z*NmmZPEOAap%0x(yYzV1P8ya8113<$2K_+`4#}e?` zrDg`!8$Nsm_*!+8?Muv7{jl%*dIx=@ z$NWfxzVGqVr;Uf!SWwB*|7)ZfhmY&mr`Ppa?Ch&Vme_oKV8cpu zNxvp0T5q!+GG&jISu4_V$IhKCpMj$HW*RjxaK*^Ysi|Y0qXM-a8fde%lWW84&SCt| zD+0nW=w5$}ad^kI`-{EF%Gz}7fQKu>D)oi&0g;Axo-8&dqY-?T=Q+UF8u2+zLgFash|_muEEQjJjGtu%Z=MS zEUxyF6#+k`e%$o$@mLWspHa&gddwrC8wbT-ymYBIFA=8VKVS7Z9Q?;0WdKzEA}+51 zfBSQEt6Q~_*m~S-_z@H25>^R4?sNS5gfh^zoZDOkj&MCUg0r!wL;aG4Ib`+4);d3_ zZEU3#0abzy9lC(!%@N84wk8eFni4j0pl`R@2%kODipTWoRi6z%jV2%eC;I)tprE7Z zXP__Fy^@h-_k~#sZM1vTntnPOEBowSe{9+p_lEWAE!fs)?>YF1ndG-IG-y1;z_+`$ zzj-rjN9Q@Ez)#zO7ynLLeQ?S`$D>!s73VNF4AqWc(WwDbt&JNu{z4a<%0acDNNLlk zKEHm*{;QK(F;*Ig;LZ7$X{)1`k}pORHO-nf-PriMibL+hQ{e#AZ&$v2`E7|;wtro; z5XUl_QY-iq{L`cOqSy>`z2s1D&lZd?zQKTY!^rvbJ8?X0((g3t)e^~O4=4ruCLA4p zM5Clf{a@_5J@o>uNbuzYY=-jOvSo{LEZRN26V}wGiS1AJm3zakkKMXrYGBuiom;kC zn{nz`g&J!vI4|n?U~JFkx9Krm*zwf=rwT>mcXX;m&%UMcrgp)WOR6uqy!FYeq6-@& z{ng;$>ND0+y*wJlmz=%K&i{Y$BPRKzqQ#vh%LjL=?ztkM@wm-zbMo(#p4oK=uh+6SA}SRnQ)CWhF7tHDP{s^VsbsE>3Yn>7C?Y9CiKr+MLOOiBq}Ktk%$Zt-t`>a-+SNp{L%fmjx&6}&$IX1YpuPX_atb7E!}hE;21Hz{{+$J z2*wOEU=Dr_Czr-DR7gl&oey->>5`HX>)^mZbtL|#>EyFn_1P6td0}A-H}S=o9O^s5 z8hQ$&>FiOE+f4laQ0}(^rCE1Su%4bA2h##rG4vpniI0IlKK!=07BUf9IIA| zW@Ci6BXWq$vAe7mUTViavx3kT%Lc#}!qCz*ENfbSRvX28;I*kG1_Y*=9LkdI>gqC= zv^yLu9VG&d$k{|sZuHl$g!^v$m#(8oflZ5n$YDgmXP1>-j{BD&W&IwBsx)Gu_V*jQ zD%{j}o{_d!kIGVxzp;w&Siy4F81IJnG2t9rAxv)E>)hsp24^p-P*-v&opuakb}=Y= zr|`dc64|-T-Q6A6P51UI!+yvQlTg4eT)85(UH*>M=FJPB`G&xZu^fC|p4a9yaFo*f z9&mz$E9@+|oR#iGtt*)1|&QW z{(BjcsLjleA3t{C7!)K4+ulVDNCXtB%Jxk>Z0K~oAQhuz!RZNi?jP7FmAR7#I*KF! zYFw$T8YDVabfyGb+t@rw2#<`k-I!_?f#$l87`wIsic0s;(Ei>xPz3lj{n>-3-+m~- z;0jHcJbB~MZjFalPj7DoC=(}GX*~S=O8QH7NtePdSvmbf1Fqxr2M=t|B#^5o=kM_JyN2ZL_4eLGoG_A!MuszB63FE< zx3iPAQWQzQv=#6GZ)G!}h3*7FGOI;uHB}jW6#)!1OtXmw?eruea+@92OdW_$aT23U zlH^13l_Deis7x#@#+_hv__gS8iOmPrmiT;&G$O*TVO9Eds zjc9UMSUvd^aOr<*BWt(T{@m85PmL|}@7$?HA$SpobXzF~?ml`{mV|0_U2v_z$*lnsRPRU6y5=V(AL-xcc^ad%4lbuSLA(>_ncmS}tY1Nsm)$IBl z&^ji;eWG=ESYGkw9I)n8?3fq$&Z_6v~KGRtgy=z!%pDJ+Oa-7BEkvIY=V?vP+v&3 zJ>qrP$43dmT@?5dlx_>i7in^)e0@ajzaA}Zap);5_TwW>5+nuX-c4L^s>Z$=#r`l_ zoqOOFSMHT230sN#mgQM2LI+T`9$5DMRUmp$AI3l|<$cM)Vc6D{@Vc%}H&cw31jaf9 z4r9Ub@$tAoJG-@L{ZVZ^6WJ%u7qbXhi~8V)Kkxz>wvoAmmMqD+1OpUyYenFjo&zdr zN{HIF7KgpO0kL^AbH{oGB)yyurpN~UFkMI=S`%ZSd3@3wXq9{a{{4m;8X9{t#Rzh( zc(`txObfK4Vf4q_wxQAIAk}w=u{bMZtrq{V!wP{1WpC2rb!lt8y)1g_($vF)RR3XT zD(j)+5vs;q~KWXQ6EX zCJ+{z6NsbbY@L3y499+@1ElKcV(*}uN4380=36s z;f3OIDPqmTxAQpj^B;`kFv-qRg88ln1d6y)QO7wDKb=>Lw$iw+w78N? z)UP)8Ok?OeES`$DNZ@a7YFezVtsRGROD3F+bbShy4kQl+-DjibkG-zsG7-BCANu$A zOY+IJN zu1u7jgL+8w`FxNiYobXvqqQ6RWGg6?o3c?N==wYhAL)~X+!+pi4mACeFj)YbkUIBU zi<~x|O$XU$M=n>{e|mKG#W(MkH3?bF$h1Idi8QuF3o+nXoK$G|N~X6gg)yAl5)~$& zG*`6A*+|>9@lD~#$@b{5_lF0}x(az5m2fypTyz{PR}>Qyp_Cs#aYB3$YRi>427?R^ z@X9p^UJEMh+}m_uNkRJas^h@xbN!RkX4Hw}Ar!@hMbe9()YL3MXJezpwd0|HOkhVi zL+yhcK0kS41kkexM-DF|MFh-uvSQokFH8ge1i7#Q^hy-GmnlwbbxxiGQR4)C=kfFB z?(Li|#X1Sr1!SY}c6oeSNaYN8iHkH88(44tF&Fu6!Xd>2L;Xz~z&+gekxD)aH(~pg zwywv=$4HWkQn*1>W@>w)3C4>I*~pj-s=W%V=;}`vZORy@c0r3FML@!tzSF1&$ab?C zBk`@fIz&;7!Ve(wu5N)b&K57Zz3}k!24M=O$5&_x6x_eP2H~XtYOulow?x#}MMp{m z(m_Y(eTl%0CR|+?=ue^lBgR6tje+x*8{f+21iFM8kVVz9?=L=^9tnu_2_dV2jBx~` z%}`KM(ulE&!_c*bB(!C>EIYn;AVfV*CbbPt4mB|Z=E+;dv;gm4gP#52_Nv*{D6fLQ%xGa0 z9!&WQ3BtIFvwknNm(ssAP5Ya-Iw`0ylWQ~5+>!G*>$n~xSqjlfsd-}RK0HvkOFB41RmLID03&M1& z>dGpzFdcpeu97|$AtXZSk!D!(Yh9TWJq_Vk_~jmy_6TqFi0pY&&(gG_0rcw}KbE`3 zG&vS_gXG>-c1(PBvS)>6;IGGDRr743QNw=1c>>jN^j>eHq^L<*JXOK>ZaVXvE1DTe zHBo`Qu*Cq3auPZ9@eqm-DQSv*V}U6A*N_paX_^6!ZTYXh$zvW1q^^Z#W@hx5A_#8_ zYogJEq#eG#zC|aF1GS>2n@6p2I6|ZzE|9&w>(1lriN$sf<=gy4t(~i2ihI=1@JTNC z_fnvi)3PuMsuh=(@{q;jh(Q0|C~4-nQRJRV%_pOlS-Sy3lpef zGc-Z6M#nv7Y=T4y9uAA8oNLxZ!!u==cmF;ov_cIO_AW;Hd8aM2AK`GO?(S}(Y&3AQ zLqkJ)q1eJuSC6p*VoCsPSTwdLGzzSeEvnBu-GMdObajb^@%)66i&GOSph$CW?|XLwX5KR3K0jSLRF!v zWcc{{o=HtjMX8#trmAX>@s(gsB1l$8MHB|e))l=)FZ@6|-)*zhB?NeCNl35|2DmJ) z{}#3lyoQD?&Wa9j6Uy_&kfp=a1r;GdMqFpk$iUuBq==zy7D+hkPZn|J%nf!pN$gYlvJFOLEkYL z0+Y1Vn!I&I2n@H9iBFP?K^|erpwg%PK@)VLL){`oO#vi#n;8OD?>R5Vx~;4Kd^R%q zB7gKC1`e{ojHri$t*ozsTfHQ^FAB#O$g(1`(6$_++8Tmys_ZodVTO=P`0Q9hz2eqy zK|jO*|8!rgC<$hyA|@5UvCUotOo>pAEji5yKu`md)EOk*w^tj@Ei6I+8XS>hwgDWR z=p?}_8Ue~49b{0Xi~2E@{1HSk4~PUgrc;rTd{%=KM=9<$#z$g7<|6sMz_8hzN42soJB+!u*d=a&gA6e+&nq(T83Ze@d6p5Au0oS74 zmP6nqx&nGreG7|2!i+)`D$ORZM=YXF zKbSE8H?XmeE&PF<$2NhOe=G`voOY77mQ2i0w1AnbN z4W=Oq;S`I_VD1fBCH^3`tn=>Od+C}AsG|0!qvHm`cONbkBAW;L^dM$G{pwnVu%jA0 ztBFw;vG5MxfTa(16^(pT`tGoZFDxq3J$zUicqt5so5VXx*cir@*`t{2L(PL_yAW*B zm6v$FMbItqvQ7jYI}Ja4W}T&dl3Qy1Me#f zH+dNCon2i;R8&ebQCui_e|yOclky0&C`Am)t*TFd!p*P!{ZV+eE$mEJEb%G#pgEfx z*TeAc4e0vd2W=a6{JMvP3u?0VuyQ*&6!Ha{!FjDX>`%0mkPwu4DK#|)8<@~Vma&nM zqk@kt&zkBxz2SM~hF zM25_fk^xaexe=#T@&XLVl0IL&i^oFY^pY1aj`Vq}8RCKXVV zNL%rxf4zt=0w>dA;Gok@&J{d{i2HeZ)VcBpM^Y`mi*D!LGTht=@Nxoq32cVyq7+Bj7u=T@;XI}>?VGF8L+ z14+`brx+{GfV$3q`0ydo402lTA(h7fcs)sovR#^JbebTUJU?Q+!hEa82~k;|370QB z!Q&d2EW`>SbP=J5vkBg}%-+%WIxpoWhJcD#@!iw^S1TV|6TWQYl%tO#b8NTEoe zF(jnY($Z}tGAZ5?rr09ZO-~FpA`M)?T(Y&-#Qz)_sRb-9#Jn|(Drx$f1W#Qz?sFrO$iP*d z&1Bi{0FzRPK`@v{zH1C2jzEz&I1jCD(MjyRjPaS%^yT6X2{JM|I&6#Z`^ZD)fMkZYwl<)fK#g=sM|@hW>&rBp zoyDx9qcb0AFc3ug(Cu5dYMei%{UdWWKR^E_aFvYJ%D~-&Zk3C|LfZdi?FJlmf_7yy zVULCmdqi5B_M2}-vEch0BkG0KO%{8cd3OHVs<_x^W+77ht`@55Mf}69ad!iZ)hSBVL(k4;KJc9crq@w5JgpUIyN@;rZ1L}tS@0aekF4*VMaHWI2G63qfnS) z1w0J*0>Q)Gu*fSn0z(U81`|SzRKnV=yLpM41B$KMv2fcIq6f}q z3B~@~lF_>9Nuwy}n6FmTy(NFCy4D;i&Lt>C2>_kM8SV1uNKl1!f#0ogmj{UB+u%KB9Yg_IPFLTO>#=)9UT-UvkS*DjTl^Ql>K+#da<_)B2MQ&CS^{@bA#r z)x$%e-?PfKw3qnQK{51yGdil<(pb0o?AckfNlDtW8%M|Y?o1|OA}i7I-@3`bbG1a- zDh{RFA`73Flhrf-khy4fTle^JO*rACvjHj&zbRONk)4;MuKXc54eevcVf`YiiOIV7 z7Ec0OpvXl+gAf%Gx_-rKB~>s~h_Ewp$#(gT_@*xNg;OAi8AEXLYe``!!Db=4K}Mz? zWl{+0)E+#|W(IbfN93h2Mxd^%%kE6Lp5FhM?CHGN0GKv%cNH?i(=bo!B1wQ7DU(M* zXtCi|8%2DubIoZibgCXm2_6k!X_9tKQq@-jRQsH7dHS>tEoQ@lb3X;&R0Zo`^kT9C zJ;HHNxlubPDy~E-l!67;W%T5d3cFbJtga%12{354#k7cAyWQwb{J%hW{d$PppiV$)MD7u(ZLIp)w35W3dIt3EgP?6Wv<(v zej+mb8n#7UOS>2$7!XKIq2@1JrIZQp!XJcu=${v8D954*$F@GvDAQTynk75?kB_HT z1og{T7(`qr95 zbUa9pDbyYKEW1iQgmL!5`s|c>N`am{u4qDdF2K;zKL`ff%*=|*ByS0^vaVRs2x5dJ zjJn3gt!HlP4aQbg=|C9~&W2lcEwcPy$I2~4?N)#@L!7{}`;xR?Og+LcdVD5P{2h@^ zyEUJzj{j=idR$;T3ZXW;!}%alAYr0jTS6E*TIHh(;ZU>Dai-LQv=EA|lO>l*@p`B>Dili~U8v^$^l5Y;D&=siF*G z$qan`x(NH*ijT4wObN+TB-!@keCTc4w*5ou-zG?%i@>sAMiNeNwXl5t_;EFsvQSjI zNmG`RIrOi2Z_3R|_%L!lJdL!y1BM!kfa$SHu#%7rJiNUV6j1R~xiKa$ZXBHU?;!uB zpT2)*1GqDcnb>`wZSC5%amxF{%&nxbFVS8d1s>R_|K!P&{y5Vp*KSq(5o{8@1F_6C zcm3raGcPdv^U$K_+o#-ypv|(o*uO`yr)n|5fe^?dAg0@G0Oj z2|)BmmP-+sT%>KfqX*GN8{EX5mhI{-O-(ff8$zr41P7qu%K5r*OjDhH(oqy@O zzCmy~_ONbG4(y+cLYqW&OH1CfLPRA(W)WC^6<%UWA!UBGlpruE06Y*Ou@CcVTNF^q z;>K3v>y=rMPp-5E*`1g7?EeF2ciw&U7tZGG8*Epms$}09&qh@l85VQ1H=*LxA^G$CGkySR_oKxK!brp z_5K4tGL{8}TeRj4gtzyY-hT1^(ETnJ=i8@Ki5Ge|((2Q-?${fIhUbC=bZ8Ybi?Nl9d4sJZLAi|2dPMCjhu z`i|wH)d%Kr+n)5SU{(@9sLEUxdWAL8erzJ?zPH}kpScy|zSo#69r}aTnZ08@VO_yf z6!v2*yDBlP6~^ovSgp^2Yrd=Mq}IvV-v{2|$eypN!0#Xxd*IG%gcP2;j%%gG=+EKd z0`=|ts!j^fg*7_yk&2^pq;}_S;nd$-?rRDILjH5uTHv;ygf(dxbV<%h9K%lD7OL!t zFJ~&TkQEVK>1;VB-7Y)%u->r-NsFv6=-9i*B2J%nAQTBJtN%bQJ~>VCAmL^H2AM2R z{6uI$j8WjljgJ5|q~3T5xE%O#5)Fw3h#gQVO3H%D=^JF59k!!Cyw)TfR-;prB#Jg9m(QZm1M5Wt#W)1 z>Uyh{lpGir3$R#oCIpC>g|edt={?i=$kqc}l|C`<8}>|Ev&fHDZ6QvZ6QWG;TQlOP z?jCuSN6Zhd7ol=Sg-kA~-j07AEt$;+L$0{t<0)+@n*9DJr)cLBf&bJ(cAH_igOuoT z>{u;wuJ{QwU4{aT+&}c{`i|ls0~P6#BEa`3KJ%~U9y)L!0?imi-oo=K4dHsf)+6aQ zLbG#_=b=!*rPQJsCzmUc?q7I|NKQN!d(py^iQ`buL1P^8TTfa2<3m*}X8YW@f1lpEtFRO7Z9QISAZ~|@;wjHFR*cgT}+rxFZcj4`eWpAVun%{Ehsvo*&NqN*J z5$K$l^7*{J%(-jvODeRKNcgg>88e`7#m{p#HRt|=f0iRHDl2p0 zWT%q;@_^BG5QLeCP3T_9#Btp;=!c{dc>LtaJvFn8R`N{DqGu*DqJ+e4S2H8M{7?9< zSVTq?gfp_S0Q>N;y$y=jupE^*xJFWSCqcC%RVM1J`<*6?{>3~mP@TzRa=2{s7CqV$ zz#lnWRoBs+q0ImX$~d7Y1TLE#ZESdA*+Ow(cUwy>I0$j6^Z z(Tz$Ii2UoQI8E@DOBhw>M${4l~Z_Fl|C2qX~ zLXdLsS3+Y#*6MY05TUdqQlCX~Hm0nLUyP7-PmSeVpBk(!L_G(Zwfs;KqyiEhLX~q0 zwm30m-rb0m=XnChp2z8fMZTE|9TzGvEZs-2wC_)8sSsz(reAvp-Kt!Ru6$6TN#410 zrvQUXG0)4eACZ>;cW@XWPsXKIGX_5BaI*6g(C)jT>%N`kjZ|r|UXbf;_7`toEfDh$l`y-9H-&Lc>ecE1l~wXbKpOF4ABNYOZ%A zcf~FY3b>}0mN=QMA5hEQri;v0d7KxFxE6R^m$v*J>p7tA0b+=tpMq3YG@j$fkBc2C z>e$;mwijp_o_#`#{EL=rFbM(_PM6qjy#BV6)DO?w2F8dfLWB~yK5+?(U?7o36!@*6 z`{8IzaYEfpEb+Ov)Fr?Xs9zSDth4&-&rC1g1}7XM>?DjZ3jV|PsJC+z{zgHZtG1x@{uf;K5m?M$JqT=3kEyRp9h zM)2IM%?T(_wP>$YdwvC?Da=2jpmQxZ$jLRL-OXN}#~Es7$mPr2#POfa%eIJYC)bVb z)Z3RfbLvAOdD}m=1?nHXoLEk<=%=DE`93)KZ$rQlqgyX2UEcHjo?4!b+t>;8cu4}d zZ-7Y!M>U>;<;!p%wo^?gGjd6s)b8n5n=@*6=}1TqeK}*ez^t!7_YA>BT#TpgKBJf{ z@gE-(9Ow;S&X@SEkZk(MZHuC&B8b3g;b!IB9fSLHO!CgB-~U0*)NgG$+-O*Jqh2vJ zGg}S^neCs@bJ!e2eL#s90rKS&lAdy78FD%tP4u;cYq7_aPYjO4JR2trLx}4{NW74c zAokZ*vz46*HIN~30(^43M*BoqC`?#*gNCq{-UCRV!zj9(02a-&OXZn}iq?9N#M)`yY7D3c#_Ei}BtKpq|cRf=6RN_19Bq z5JZWkH#P+Rl7yQ*Sx%C^Rj9j{FJI2H(Unbm=9d*bd~f;Vv6(TG!hretX9!C0n+HA{ zz-6#CfknmdFs3;Yir2P&2;FaRCN*qEnl)M0kvJ{rLDwIEL(uMenXnV%vecr;6jurk z4u+l&+dK?Er!2Gez;OIsGZ$p2ABr&g=#tc8Exa_{Dcji|dDd<#%!kMn&SNu!pFR<9 zQEh#_ex~zaN`+;J-H+u-0-vM4RJ>Do(#VmzX>WX%O#e-5hlE(sOB2T$zYFdBbOpg7V{%6;9#J(d(?$XiVo}>w%9H}uyot`{4cf}cWN*bD4mg} zF1@7n2A*L?IZ6DYpgnji)rqa2680qL_qq*OGyeZwG8IS;uoA5x>;Q%oupy!=t1P|@ z9eg#q?0;hNKsNxSkF4E$5X$6B5QY~{=maAdsNd-g_}(BsM6=2L0sH^gkaYs$T`m{D zk(t>g3AR>ajM^$h`v8pBoqO#uEu4|O$c`PH+`$qe6eqf4uebf0v9U3!bon)WoBI!h zrKSoXmY=}{iRzk~8y)w^VtIJqHqRQA>(dM1bGF%q3XxU@QaoW=+E+$;$*WM4vDioSQR0Hmv%YZgVhGN0Uf*F=a zSY{q4McUZ}J@ZB^q4#*`K5wgBw0D|M!9q7_o`{3fGVXy*PCd5Fq@C{4FCVYh%DZos zUg}f#iCy4~@5}Uf193lT$|*Cx)%eNLGjjRUE$`(4jn^*!(@DQL7nT0?i2tzYN$obKq zKWj*D2e3#wiWzfB|7acyH*VA&=+iPL$BxvYcj+%_JVmlnCn|3LkBhBPU@n2%!qcfPYB>rZ$#D&;Iue;PwTha)2el!`qMTs{L`*2Q26=w z&lcXi_oerywToxlH!2OzdHP&D?$B8le65sOylgcN1~wK^2;6Pi>O-hZw)Lx)7+qy$xy`BZr@b|0UAR*CxrvKHMeA_%-GV}jG=iA77*c-l%@@wO9&x^WD*7gu&^+=3~q2JP_(Y27|AS#0G;EB(h^!Id5&fF z;b)O}aoQHvP(nur2;kmHIg$N$2~GtOX=yez#KECijeSLm2%Z>&oI>Q|rluxv?`L2e z0__|Dd!Jo29U7#opy%-gl0HyX4>F=E1|PH?k42FQM>sdytEwS(gLimTwwAoD7(YKh zF>_$DoB&Gw_`Raw->^sXa;NwsXE$liVCTVOTf$4dgCNZ^ToJ0*Hvrs(a` zE3<+vUO?)A*Kk{Ns_KI&zP0OmGtwV16g7>fq;qk+NC{Xu-@Qium@>GDZy6cl1+CSv zpL{g5AV5@0(bVk*?g`J%pR$Lqn z9gUa_CO6--XJdf1AtJKnuU}sF{68%j(3}V-0+vuL<%v%3zyK)=9l-#+W4|1kCBV+K z02n%-zXv`UfJR5KLNE!ofMo_LAwrn&w}Jc61wiWpeZBzgkJUPV>U0R*mjAAM0HyjH zsvGz0cuCbqAcVO3VLIH1w2P>y(88%DAvorQ4TQdOuNV9k_CWE3!6ilodaLgsavT%s z_8P6c2~&n6JpDFn&abkVwpUI;|DtquTi4Y~S;EjJ&ta_)drQT`hx}M-^#Da*78~e^ z6`>9QE4hsVlLDv`y?}r+#Kc=5El_n7Pn8A2xJFp>RZ*R(Xy60aU~cK<;ZY5TtLkNZ zR@2n&}?32 zjAYyOfmc{P=#5m_$xZa&RKZ($4B9sk&=b>l(qWNVg*y^d7Sjw@sWj z0>?#5czcN%27Mp+o0@5&4pd0mk!7kOUNd&b<+L7K-ViO?Mf~4f* zuSxg}#ITCp@(;icy(2h%ytbuf89M74;Cc~eec94X=pA(K5?}SAu+5p_dSVfvQ*$Q% zLFR+qJ1@1oTm$DewgSu=zPFbrmd`qR&kPn+J`O(OlKPH>RN}E zeA|j3|3eAyCM!cY86Ivj(hGd!bnMhB?ZsYoFU|eVP52okl`q5}9@`4cgKVL^Mf`8) z&fYI=dw)e?eTfYkhhc3k`}1lxHl$BOhR4YIy?fB5oqM`(9Xjug8~67?qoI8Z=z9@t z_2>gPo+8buQBg}wk~d{PGd*m#6>Wj!e~~#7Y6f5w^z`g8H3PV?Zcng@2xZ^Z)Ko(L zT-6M?mx)tgqX1!rFyNxWnY=+>7?>H99T(!`t*ngA%nZ;vbS^tf=4o^DBdh@nb94J5 z9jsTlxt8OFQq&-4Hf6&ZL;@vte;#p=klGzZc+Rp={3mPX<-wb)5`WD%Xhhns*vyU& z(G{rH0h*Q*k^T5@MTmum&u=$1)kikpyKi4BPC&+NlQj^vTdvX_6OpzWP#v37R91k% zuLhL-33V(-0quf~uxpY@3+Sik1XNNKLt;%YHo0L=U?h&3A{0Nk46zyT?Yge6>rjg3 zDzk~F7iW)W7TM;#?zdw~Z~qAM6grh0pzS|yZqB9m&w*G}u`+$5COz zsn4%{oQFvJv_vCu*XL<5fXI(mopb)^cv~;2e5J#W=Rb~Y+^xsiXX>A;^3n54kq&d< z#?H~B*-ID%HW_)pypuN5n)PCHMyOGZ8!5e!Lw@Zz#zM-_^ag9Y_i1v(j&e6QMegYN z;lVFw7i=7;Qn>X0L^pjn`UEt%aChi30eo(YLGfcs+$p0m*Ahnn1z`mRj)|cU`CjKh z#GshOQ3i~#gQ`ra>d}-5;==T+izbs=-hJ$tz*c*jE*tToJ9`NlBa{ZwJ#`~JJv8N< zWd?#qEp9iN&XqC)=VFq)0}&zS7v3O@-BDO_Ww-shhM1^DV@??{W1j=Cf5Fa>3$ef3j`bKW9 z$xEVkqrn8i6X`A5iZLNKcC2?XB3kK@L@<@a7LL1&l)5Ia66Oy=w__nfH2(5%Q@Zk9 z3{iVIK0JIuYB5zD%QZ{t1PgMwu#-Ba31gH_L0)KjPQ$;XvjAPqBg`4f8`RWzvEJipBu8Ui(&fP{ z2Uul1;15jpMb|6##PwI-J}=d0U^u;H0ptm=zwI;(Qf&pR!)tXB?*cpbu2*+Y+#+}v+@LE*G}9j}sWX zUGu}sD^~tk{(SaF_jNs%m?i6(d->>j_M&Fj;hhlNkZB9on{v^W9o|H7Ij*^hVXs4P zoc!1R2b_6f3j)qSWg??4B0oiU$ig>T4WLyN=Wca1F#C1jXApc&!FN%7EBkFZoXy&Y zOSi+$KfnkB*X zejxaQtxTZ7;{Y==mNK{)nN%ZTh|_$j0&Q^*^f@hD?_Ymxe{%fxlXD}Kr_nm=mP$Jc zuq1|TVYs?uiE8%#Kc0T(weM#;1i3YsIgei>k}E<5Ph}2a2xtvF&F;Qm-eu)CgguS5 zNZuK8XM~~}aIkw#f5`dw@48f@JDfMaZeiqf)zS~Rv35x!A(upnH49QbzPJc<3`MrcbHNDhq-JWbb;gWp2dr$Ajh}Zq3XRtjJIj#!<5>Poq9PhxvLnZ{L<*i3ipB z4pFiYQVGT+?HL1pHsliMbqvCXc8)d>O}#xfvgHYW_~i;*E03OT;%UX~=tn5Ac-`13 z&htsf8nqgJD!|#e&Mq#7Q;Him5_%nQa4|wz9Jq6(TVRz8-@bpp57Q60;NJyH!c3Ak zX0F}DU^_2j8Gr?bp2!H&A+~&saXI|1lp}r8F>eI&3g(hP7x<^an`TJ}bI86Z_YyVT z$v_cQ0iH9$6izs=iHJi)!7(YT%wZo!T;L_Jj?pc(+z5xEbP6CUE$2kOE@ZQ2*Nyp0 zn*eZhUy>}t>hCKsfjeX~hI3Da0rt=6(K@A=7O11x8aHC76_8G>$+e>H9~jD46Z zo}2z)o5*O^&mFwUC}iL~JqcOlWc-gB%Rv9fi>EzfZK}g9tM|T7S>kl7>CT7p2$sx0 zw-=BdGpz75`j_g1e!jRB+W)j!Wwa=CX}weUQKZef_OsQ{Pn%$G2JBe|h6)M4y4*R9 zMFuC7mY?)nz0>zY$(3@e3v0;h^mX303iDc6-9vNa-xwaj+Z7uBU$#g~z%}ka*pY*($I^K7UuV5&6e=Sw_ zn>_E$GUf%Wg0GmZW&?!yeU>ZvSW$T!gDcN(OgUO-E?dEY33s&`_6=3bH#h#XuVStQ zTfV2^2HVZ6Pd8hPzfy0^{>~YBT!LLx=UjMszE?$K#XJwl9dA6_*rv~LYU!S#PqYVn zYy@*AJ|E2=+nYCN=*s*0WMJvn()Ak5T{9&Pc=sBukF{T|8X~&t6EgoGF@+wiME9ALw+8PEj|g{*1NSDAY5k7U9Ra{ z+4eJUqt3+8s*>wsC5#O66hkosxo886XI9^5Czh55%d&74{d$;TzNAmBc}4u(A%?vg zm0hM!%cS&G|2VC*eD`y_J&Y|+F3H#tcQ))BQh0wM~WQzOP~=W@9) zr5#aJ0z*Ke+>-j7p%vn&cZe$&fB-#7;m)3Yy=Zwmed-j$_qQeanU@k1nJqZWPLZW< zNc?dBYx6h~Ynx{Py$G{l$h%?U$zsH*@Q%Y3&&ZS&APP?$`{F&=iBrADVd7xrZjm5R zEq#%vxA&v)#jA+Tp0qy!N>TaZ6BE%a!9)h55gLn)IMVnMQ&QFtDH`qKACbW0f|4Q9 z6Tl1d{G6c9TFhWkTnDKNt@B_kU#_f>+00J5UtqZ)PQbWIgqa)n?yba#*ibOEWRTb> z2AJ0(OrTkDHaWT8k<5??!Pwag?X!>9q0NJYTR^M4Y;@cJwF?k7zc=~C9t}*;Z0p`DaqPB zJm_P}&9H8J8hAKX-jxk;HR`NS`x!)7gI?C}{R_C1X=oW=< zf>SE0IFj)5C8xpL5f`u7+4d$iX4jLKYW_X(k#!A+`H%u?hL3{r9*%>{A}{R>UM{na zV}+~^*(iGF20E`dm(R^;KAUl<5UTcSFj}>I#$z@1NYbcIMAd84wC##dFBJS?kMeqQ zWIS5LW5#^ZF{`MbKjl_k_blAnW+nRiq%Y5v_ND!rM_pulX`s-}C z;0Rw%PG$ner_tajN5|hOP{0{j!tyCCq#l4l%8Ba#_9b~ne zi>@B+60q0`7$izDKF{=T56@(G*b09$u@q9jW3!@wX`^B89qE52=GUZiw&kFiyI~`V zv&EKI0O~>?GrGO6V3;}d;$70u$N3dgTDBP*>mEFq)!ei#qqQ1>>Gm|uY$9xUVz0xP zyqTS;iTVI&>7*$ug9j)F7=5enXu}u^HfqlqTzL9}X8wP7xb34UmIdrG)kAZQQrO@O z3=hVCM0N{ndw6^O^Lv%n4>4EK0fU+kjZtq#t7QMlDcbtEv|2s(%JJ?6#ea-Nq`z|W z{s|07pB54<_4N)%fni5cxGVVVR3(jpLAB_ka)J?aLvU-!=CYie=3NtC6h7a+yd~S_ z`qp!Tj3OHU^f<^?|}QJ6jQFlGx-O)+jAD~Reu;5I&&#lE9Pd!p}1tf z6%)zdk*3#2-5Kt)<5;`*iN5~X8=C_bbYrWiPBiXg_@|fM?^oo{(|Zm`B%O10?vkvI z(dCs-Nqb-F(c`fQ3SVvYZY@6V!zdO$%mz1_U$t7U5wpZWbYb_JuZyW+E4Qz`Bz$N; zUz&D6areQ^22#Fr`Ad_=<{sJC1*?-vft~3(z9bW>gsrIypG5yEJt-+rCiR@l_|;{; zf-n2cdg908ubZ=ueYKEeo;(v!tJ1vcXqTCX$30!2#Kj0y9x4iJ^gEuf6X$)BQ}|IZ z*6Fuyv3q@f=cJ9aO@OYO~Y zY@)z{~u`rmB@Cu{ul)rPGzW7z%Q) zuUr?#%V0m0btWSvtj%-v`fsCmAB6me)f1t-|2yO84{SVfF$ZB6AI6y=q$3MTDr|p; zQX4^}lYv{n3)d}w+Q^DfRFSP+n|Z+fKgc9DCg|>%o40wMOP0AN#7d@70=7^ngw_Dl zcEd*H$BYe48O{m}AmGQb=Lyn}Mz6-luOb5z(zJrMu=iI@z=PO%iHtphgIk^_x0fHR zoCt`FqeqW2q{)2GY@>iA+r4q!h7CEHB=E7GYWrFzH}?a!2)&r?z+8~r$uuiVD=Sa5 z)k8*q!#@pe06ig$U@o?Rf+daFOiWC}o$~ag-S!mj0KwzK3(#j416kSV>n(;;z!uQn zE`Xc?KqKAQqhI zc;n22&?T`r#vlyGAXGamw$U)OmRwxYw?Ry#P+~1cp`J)XtoFdP%w${msmW|V{ZaHXN}=I$+9N!Z{qIGxey$1poPtC*Sb`VQnoP>6{%u}YU@R_A^;T4y_FA-VY~1vc2a(&;q7 z4(AX_#`4)-X77QaWfCeZIXz!=Bx{g=mp=WVQ(1I-(a`K3w$MfXdtwC2AF7au_+y=R z&?;5W8Po7blb}0p2YP+boMKPc*F$`2o|)L?KV>Yz`KVzl>LM}zeJ7SNn2*sSH?BGA z^oRSzfaceZ4?{wS+-*(Y)*DCj&M7k0gdM5kr6}+A81hz*8^7K0J%4aGN-XPpV&t@3 z7>Cx4l$-7^zNhi1%dxp>&#szM6lLgsv%UR=jx5g-mzh;Rj|!Y%wP5-qs&4;dF6wm7 z<&2v;%G&%oa$4bA{gy6ZXkVUL*R(H;HX7UgJ-g=5oEGCOhh1DF+aAvI0&~7xm;JQL z9n^Zwgnx*;X7qYgse7pgQP_*FbEh}3)OP4Pc{M)@?u$iXdbO#gXT`C1}=|3sdYxC z@3d%6oTs<7BE$7f`71B`-Etfb(psZ5eM?|M*jBG}TD@ncDn{?N(6>($PTD_WX7=x| z?J)IY(41cT?&pVRpE9m>WseT{xNKi9d$?lftMR2nr#j|j7Og$`a%kwrXzhi$>r7QA zJvsF|wmT?9znOJEFs%P*lWM1`TpJ9|MX?;}wSpUGU0Cnm9gmxBHO3jgbEQEBrbouM zvRpVRzJh^W`K4gd>`+pmj+yTN!w5~4mD4qMVj#m7a3d485JYlEJ<)$bhJDo5)s?A= za#R$;p9K80tYHjXOQagu z)Kg|q^3>y;i0*FlIjtjD@WrSQNZ$`=^`d?V*~EZ|St4TX;mn&cu}tvC`yfv_OzBa^ zMoAXmND6`Fd@2g#_i_UXq_85yFk%}F7! z2XE%Qmggr2Qwi!lkY8JJ+MZm!V7m&oInd~sZfDdFH2b-86Fw~kzRMo?qm>3R)1c0= z(8IKmtNQu#6)?^yk%@u7YuQg77)ayd=5EJK-i?{#y|`9n1gG^0@b5gD{vM+&e?wGy zd&7nsBt{)M*D9iX@ap7*=f*i7$y$R&Z}?Yf-B^=3+Z&>Eqq zVCe2X93id~yY&Twdx{hF?s(E%)z(^-?;WNG(pmXuKT^WH%6*SL+M3f)^XfzW*QC~` z=^X6etY4XFPgZ|bWdOBXTjQQjyPvAbUm)$E8OkzE`^|I9DCEVi;O6Q5qI_2a#)G3b z{|Y~s?be=}FOABLwgP*Y-W|WnK3BJ^G*HcU%81XMuYXI|;-~2N+$W_SoB+pf?O+M%GYqr1= zmUS~ubIG<-ZJM7mIV(oqP)&b>brR%$C#hm8akuKM%$zufYdkYU_p1I77dMIO_l>{v zVrSN;Pkx{NoMGM5-DZ8DE&raCV?z~N$de!S#>J1uN*EZt=Cru9B7S#h{>T!)*hhPx zmw965`Oe_?UEv2g9Bx}X{G4!lA@#WB&i-s0`w^Yx|Ij9@>7U`9n#*)%?44tx#?E!^1^0GlQ$M>LNBY*3^9K2rnKIWcn-?8a^;0|>xD>>k6U zKxy0itPt3g^A}G~P1y=WBbM~JP*vjC^;;?Nj#z#aThw+y4-Nb0rgmbk;QRiuKTp#% z143~gNVm>xk4^z}tENRx^IxSH15?@fRoTgu4I(j>>HH}p10dcME+n(WUcOorybp$4 z5lR(;8W}z~`FnT1Cn~G6#7+r;QBh1Zy}>-FXg3+D4v_J8u5w++?gtHChYm@hErIYl zSK?r6(;?ofD;rtBlg9;OumT1!Y_M8S`S!s38|i5=rl1NUo*})9ZrxUO0_xtZv4X}E zT4$r8w&1i7I$P3ZNBVg7?cUu~Rwko2I6+!`Ne8MGA0q?tR+ne>jg6npDE@+a5=O|B zwt*+;sB*;MMKTDL{zz}H7dVQ`7TL*@QT9=V*>|9wm!s!F6>o)re*{agQ@2SrN>t22MlGtc{XLZt_?F%WLyiLFTuiC!5{%+`a7YiTv^v>S% zOYi>n+kH~pXmrEj^BPrOWbfpdC=HUtd9>@9a${xokA)SjZaXgTJbyO%#WD(uwW-E{ z!R9ddxJN4s{C2c=485;gC{X@Qj5_OC)5rh)=rp*BP3wR0b=;~6ppDS^)H9b!UW%{JrbBX5Khx6Sc^#`b2~2=%;h}wpFMeCBBX2b;FyNTF}Jjs%?6LQ zR#P*aQfFiZ?)Sa^u*Y(MqbK@;u#em>^dY26c;_7IeJ6vmFJQWW%C?8cZfIf>kbp&_ zCAzo%)}WMffaU2^?nZKW4Lwa z(EcTB?0WvYQ}Xs;eExZ;c$mTa@YqNOB0SDi3KKWJBt#~t3T<*S5}`bW+d_eyQ!$9* zo(yiK-|0x(Skm{Vp`}H#)JXUZuBXUOXhtIA`bb-9&PjM;&|X5udMK;7VqknR8OuT> zIS=c`O*Ux`UgP18vBPuJ}6%%kqpE_QFleY*1$*E3_vW z$RJLD{u?%L(|Y&s+ZTh0Yh8S@2~*Mg#s!(M-99uI%F?M6ZI6TQMBnwsyvCZX*I z9bC)Np0;K4=DKz%WPg#s)!6nhb_gpgzoyLQfYc;ikdR7{HLt8(*_yImS~}tWd~fdH zMc6OkKgvtUwB$ zHj4^|Iqc_fmHYH|5|7u4azKOsYDqeTZ0B7$&QLp zl8o$9XdvRQNJNQ{B9$VE(6GvgWVDE^?0J5#>*#r&`}@c5{Bhq;_ftCOeBR@_Ue{}U z?0ieWk48zU4LASh$Bau&{y|X_mL`YJPeFIMV(wZ7XUxvWZxjZN*< z-(J_09F)MAa=1OfV}|O|SVmuGf3VYunk7;Lubp})vbISqPUS6m?$Y#nc$wP+E_v2A zzVP_L@4Susew^Fu=Go}3%BYiPd%oXmz0|oHIW8t_#cbnr+x5ZEq0^S~72smi_GK-6 zi^69ggh|n2l!p*0+;$o2J`gC<_=HX*_Xtel4cFOUxp8By_9EL*F!(P6@54{{A)igXpN|g93J4lOT_axcGEgwATChH00To4lfVi;=c9P*}{lbBDut2h&zF zux=$5F!Z2gumM^71GyFG$&xuO_<%6t*AOhae1`B2?qfv{sxruV?e*V_c5k%5cQVPoj_}cGT zwcr|54?r)M_ro7(3BrlM`7okz7!<)_k&{k?7S`4)^`P>ME1WN|_6tAy%qPg3j(yn_q-;`#QKHIO;f_OX!-Ci#yHjyzrs%9zoD4f&vmx1{(#K> z%Taqof63|_?66&`en<9v_Jxc9)@P;bk6k!cJe<#17o*J4FaLkQ7*ikt4`4#fbop`X z5n;OgFkODZHnsr7X>cVS$HG6LKLU0TPx|rw`*B?*Ea*6QR26+~RXg^8z4ud1>?~}t zs6QZM^ei3FVO|52mh_qMjH+QlE7U<^Y2t0Xf4?(X3DXlgv$3Q&GhZW4PAjWGEQKTezuRG3qF>IUnhVR)Zo;sIDXRJLIXbiwPsAmPm8>sSzKBX?hA; zAL4x@5c_Zt4$@fE&u$NlS=*4GC|Jqh(vogt4y)g7f_mLfNtc;%&}UpHDys3p8!Bu( zjJ8F)O5W7$do~E4VR)gkKX|diUWyNVZ*e`oFm-rILd5_-|6?G#A>^%GbX1z~XS{Sj z?)htL$#*r-f1S#Yx;NZ>)}#JG9j)wa)(ZNHUNXbkOVr)3w0#Tbb$H-0qm7TJuj@v= zv@}D!B+W;6CzfW870XMr8uSVm{_>W_rkD zLj)vo}31tm4$l{F*5=ZS$yr{Ii@9Np#|l<2v#i92P8LsLsR&kPA*FRXqRi^>tR@s^M$ zrzWz4{r}G#wI8}sVrfA_u(!ou0vZd5C%}^1g0=04KxOia36lhB=nFkL3{7#&)Cg3Z zPmFH;c$W*Ir%SFc@j==KxtEG9bXZ7gihDaS%M?d4=ST5%>o9wu@QMITc<+eQ zLq+=8$g+q{Nl8gag;HrJk-8p3c!Y&Fos~5X(me1P^WZXYRJ;j}yeOptfX3y0`65Vz zQ!ELoNdFSm$=T_E`rPSIVZ`D$ki2d8jZLRPvwFj{1_nGy0%CH2Sh%piu4_MYhVj0k zK?q{3Z7>?c$-J_*++@KPLNWo#;^Oh!efJ7gWl84(0!Y;j{Vf33vZ5k3&`H7u%i#GMpz6o^=Gp}o zH1FCUkarA6%dbXm8{mF)TIUs z&9(n-oXkDhADBb4!cPsH6_Hn>^6dm`DoUp8)ed4|8E<5;vc$jOTM>UJ!3!u>@O>@s zR4e7X>o-;_Y(13b8E=(exHUB54y)$p+KeMSzxRC3ok;uQeAIsb=O|AleHVuzInUP_ z$#J)5jZV1Cd^7sN-s_(En}braZYeqOBmJBcVZFwkT6KMuB9|c(eaa+`gt^J3$>+YR|1WWLP zt|_>R%Z4wF+ZvDL3rxrD8?v(!3_wu=y6gCAGftU`H7DUd3RRB=YRz5n>RJr$>V^an zIZQE$luYi}_5*NtwfJyb6JA3*dRdntx?q`t0zW2u+sK$a299$kCO4|$PI3HbZ>LP0 z>PNt-ms}9x?Z=yr<1oUEv6JL>j7{O@wh3cYKREkn%7je4dDJo+v_rCWCQTPsdZQDm z1E0(j65_d;l@+B87xvLhjsAvUAlBB(j9Cp0;n+X>^0eLKpzQr;TyXQvTVT@ z#dHCFlXwT@uskLM7OgH&g3zwTbCJOUV|7@h1|yt}X^B!}Y&V7J=l~uFI0$(p835%2 z7ObHBrX<6tpg2>n?ldGXDdxFwD?aJ-6os56Y!Xf7bbN? zq&b_QBQzHbil~xiR4YrrsCG2F^X#oYBX;Y-#KeCdb!4_39v?co<&@(>?~&Fee}uyg z`&Wlg1-PnHxB8ZTj+>sY`C;D|ziXYg*|%Qdxh(L_Iq7!(T2m9s<2J@&}13g=Kh zyx)7qh4h$eg;LA(N`-{XJvoJ0`K78OVZA?`*viLRxXz`YI_7zFXyYUypLwer-Ge?z z1WOrb5wIt*2pV@IC3sHsZfhr?Hq##&t)n?cTN`i98_+)=0y89q7nFZNf-!-0NvOUF z=G(u2zjP)B>`TD_=L_2UcYbtra()DSWWOF+FM;OW1aBRy4YuO&4g%{OfB9CNd3$q# zNP;bb?5}AN7d}1!ys!)p0EGxd4j+EzbYbBUZybndRd{EdvFEIksS%6<{hMEfbLGq79V$`b*X3a zDKZmXw-+$acg0@pg8K_UOr;?b_A-JhlW9^;OS(|MBMM;f_jcXUI>}5$SO^(DITl?d zFApg%ne#TTAvFPBwm;mgNy@3qZLBkDp9ktT$Etr~lWUam#vzXbzUAE)s$~8G*G7#sQssLkZUE*Ui4wA4u7tS?7}Z zqQy2w*+fC^MNiGEiku(yO;%QoQzqXl^Zm}y(A9u<9n|%q*RSty!8N6TL>^wEEI<&G zQ^W!4(SNYj3aexcoN6!$?RlfD9Xp`au0t#|wwAv~HJf+`<@au~g+?z&a50Bkasf52kSt2NW3#M`1PAvHsgHFR%dy%DkvdeJQC8 z?IBLb4S#xjj|Ri+PZeBDI$@Ic0PFEjU%e`@z0*ludssz0Gm9(;<#+Nu+CuLvv zC@JBvOcV*e5QutKS>dxBHa~);HA`2r5dRi8}>C2~A5|kIvt=4rJa92}>uz%*5@W%L-5*2AO;EZ~!jUYtnW+NSmStgZWB)nE2m@ zwfv>ykJo$Coa--KEd+jVJJHc(&vtH)@9VWwSuA<_UUhGA$bISJRQ^oMA>4CB!@Oww zJtBrZ_3l>PrC_VJ& zM!WKF>91~6EDVb&ly!d3la`&>+!B1jsAX}`#pv6qRdU=}SfeG?OWQ_?QqeT)H7@#}!_w@I_2FTZgmJC{#lms&a1An46LUjdM zrNfxd5)t=O)WtX&QB)f2>X^J9m1XbI&%ftqkW3^&nCe41rlqBZsnemITXwyHVFpnX zF^ej()LTz7PZBi?3UZv5J&SYDn5Y2|8p9eWWdx_wTsTZd@2*)T4C8P*tG$KaynU;N zIwoF#X8lIWr0l6_*h~P45f=_zl1WfWfn`QVHR>4*kPL%hl|Ykf;uIVhW)XMu*Uhx;TvE#Z3ydLPv7$^fWe0h~|!i9_UDly|@ulIsf`_2j;;n zU@oorwPvz4m7HuEF4ah$H}XYctNEE3GN-b<5 z_%*|4_Ed?9^-ys_6yinun&2v%ujXGGSOEaQ6kOF7y%tIoYbCVgO2 z=1@@43(xy3zWDK?9qD7$P7`hm8%igQ44Ac%JdRKk*hvx=1}{gXMAK`q06H1VNKV4Au^Nec&HK?)+ecMP1~_m!yXwGN~VNJRgwQ-eQ8j5UCF| zE0T}Cud96jufzD4#3Hl+an-2Y{;Zq#52z;x8- zkb3{eq@|(3?8C|nBD-&HK2JU>swVVnsr3h9_Cg%#b; z!XY8<_HB)u766?89d8D`om^3A3yLn;g$2wQ*C2oJsdLF10=57Gl&BS{;9DB z$%kNwIr$3OrOTL0=ZozwvlgDT(H0?T87 zA;ITaRgWA5%g!Q|dWEgm+dpgUCOTSwvRvh?a~HPS5E&Y}wsBX9R_0VEhV@$aAkM_3 zFJO>zFir|2*fxmni76Vq*wd$ZXnfjIMc{<9+yb8N7>TZz#-iT$arg5|_fp}sSH?&G z0+w_F$zYs~DFxH*9uj07C}f>(pf@6OtFa&Jeew{kox-E#IUnLh!)&*QRU*P9lz8g6 z45_}u)xv&(#n4Hk$-7cXClBs&c_G4$HQ@Z7QFkriqU7UsShqNtUcnF8HnUv5+y3Ee z*moX)w-5Q7k?<^YIx1*Fd@1d|>h2`SJ4aYaVAhsz+9mzHy;mL~yFz#G-l|q%$}=!p znQ9>Kkd~f)pqH#p6P@vcUZE53zM4BW1ExG%Ei5det#F7lqt5%Q_K}JfX!6so&~)H6 zg#~pb(Z*}7V-CRl!654Sw6go_gustbo)}t&D_bx& zViY_0q1=fu)Ef68=TL1dLh$M*=UXcPqT`4Zx+Eoi0ZqAaqS?)lA>0vR!|iyg zW!$|ef_Go@zbCxUUn9!yTnPuK%0_JNPN3BKHF)CxeXH!!FxOK{P{fkoG-i>$^#Rd z$bXJ*+E|r#+W}euf9>IZN#%)3{nqm0E0yAwTgnPCAxf%cb zIsdj`0aYZbJbU(R`f|}fCaI~Z;-gF5h1GU^-;I)*gk?jcqwm@}IyyjbS;_Tag-FN) zaG?P@=o;7^J^C7tcy@A?*s6tD$0oF!XB9krh`ob@l`2 z4#?u*9AWlUn38hnSy{~A!VpD_fXz3kIb5>uA{x~ z5jFpF=@8*_`|{-rJYio6GlsU^P;lk!=v3}>{#9#Fhc6+s->}BE;6Sw0l<=VAt z7c=(ddoefc;aDg&M_XxlI>5Vgn896!iFCc{jvcefP5={;XC_yxew_q8zX+Yqt&hK0 zr7%HsIdY~re*vCBD_c&~OQW?ss+#zP%>$%zSq=aBF?82wWF>k|R)DDMO^o;L3jjja z)Hu=62)*}yDH)j;$Jf!h&3%d!GfHv?^%->a2>{wKn3aH&!`!QXA$5R1fBsb6O-9f8 zr{P2luRL0ffViK%M>YUEdFBFxnSOiOACnv|kt*BF?tuXwM3nxZw*i#`83oxz9L6TdN^96>9Yq!ty;FbGE8WJO1AJ5sw=>NwlWw7Qz-oR?F z`=*0FH)o`(a&9 zJ0p{y9R_T>$Bku9X#XQ`=ct!|d6RPCp>-^+Ue1QS>o-P!d)2;mtZ1dRuKCEv5;0Lt zwW@qpfpMQNuXsOyX+O|%)l2Yy8v3}?hhSjv>%;P~W-cFcsT$cFD8NF;&XJERJO^s6 z34#xQn|JEA7BYUp|3L1!xD&>``6xG=j&EXJKAW3E8xId$KgVNh2H z;z7`}XY%_Mfn%U&69TY41BQmoSKR3Yp2_&Uk*fy%C%8O=cFsDt=Z?z$-GT!XAD3T> zzMk4*oICc+CkO}nEqR7qr{MZ2 zERUqGJ?wpQJy1aS(%ITj_P}m`tyDGZ9lW*|BX!)L@CfcQWe;rHB^9Hx{`P(Cm(hOb zSXlI&6(%0^EJ|1#q+VGt*XEJfBiTsZ6Hj;!$u%}!>f_pIF*dfSC4_rdlD@?^#ySqM zh2Nhn4+SJ>Rt!?%aEHC?+bz>eU&;f)(e~+VA+k z`4qhE`a+uxj}>Y>hHXP98h?&`&rkeVT@Zik(Ttjb@6%YsdilDGWJ3-d+_UQIo>xu6 zKRHs(f;xAvk&j%Pxcx3Fj#+TMLFcey<-{_{;2%^CU#ZTxG0VJv9<>axhqaZR6k&fQIs=exDe^Bxrc(@9(eF0&-OdAl_GH zWjQxNdydOPi*XJEyl?pX#hp+S&*oXH0$?*->qqR*3#gS)D@R+!SFme_eOg4HBetg9 z)Rt_xDT4FVTmX3h1F;vaP`aT%*#d+R9`hq6=BSh5P6s^(4J|00a-5U7{J1F3*Ax6O zHzeCZDB1dF{A_;nOB8(2VoW0l35Q0LHxGWZ`*I-P0MuUt2BJ{Kldi4`EuxyQFh31MoT1UV6H!i>?o*8eR599FGmIJ&*lWTZ#n?hkB<}PNsN5*S zC^_wgnznXGhBw^z=vIP_?$nQ=&*5!BE!T^oud{1w-r+2xOBh$i>0dUnCtY0Ly1!v* zR5pHuStN=fRq7lHW^!@it6pFCnc-rv2cy!)*)w^d;=rUyGmOR4=jh#p1lIViLGy0B z-v}W1O<_*j(vRhEQ}rW@IFy3v&tL~bY==(%zi>F;&!-=$`?9%KA!dHR=aJgX?D4eT zo+^jZ_!p06{1|N0^-i0ZVYkq5JomlIhd&j{!*VOd1s99s<@Q8(lZ_Vd& z!`gwc>jly!4c$J`vtuk2?{~PLWn`}pvew?^HV`hW?o-;%xOTyM#tQuw*{LWHW$G1l z&v>lVGN~R3lCWscUAHx_O6;m!D8ywS7*FJ&eR>+DJDp`!j`2m)++5#5hR=@Js$S!} z!b7a>4~H3szKV>KGfg`b-P~SGHNIW^^_$@t0UT}NjZ5CU_wD1w(hNk8t<>TL6$~Gn0TO`AJ^{xW_*!rvgqL4mU{j2>Zqz*T@};5^ctTLtFGBJ)Zt zBrMSQUQ_gtfmIJe9oH$`=qalE{Fy8qNE3;q3*u944HlSr?#$>dX7jQ+6r(FlBPo31 zptbckuw7c)+QL$QwY8}M&+Kf-xd<7y6)Mj$A#&3)dxPJMcjN~`RxH8;WUQt5=mz}; zx+0~m3@Q29_)1AjzbV8l=1kb!@eK$yrH~OcWV8`$>+vZnYHQEqr}j2-i|{>$?@`as zD2J~g%!T&fl0&u&EFtR?tU0p+0Gm=xi7A^)HNH*CA_!3$VMoJeA>ZZ@CfW?aL2GO4 zfcNrl(xKb z-ue&6UW4R?z&=&>MvRRqo12?c2?w6r-1#KbpciY9NAy=F^D#Hlp#Klo>rwwj-L4)6 z2da&~-I7imHJLA`#2(|BHPxwMw`Wq>;;RDwj30*b24miQQ{z3oPo%0Pspwlu@Oa%g z>!TFO8M9=3hjo_&-sYMKZ$9tA5a(CEA0AvmBxd-RH( zw3%2z*`=5K3}+staoZnj7<{ZBlCwGgM{KM}tNHXP5VP^G+CC%m;bvv=RRH>uAo!5ty zFovbXHrS-GDcAe_>F+;B6_5cu!&h<8?KS_-o|!pjFBl)}WoA<#sYa(7QV1~6|CD`d zqc`k0tiS^h?^^7Q>XM2xW=8VxJM+tM3C-Ae3WcM<6S->aG=WieQ1hr zb+y64g9n?Sq=Q58>!tu|xR;hd$w$QzYzL;_+0)RNdG{_GX7OA8+j8?Yprrde+Py#S zmdc$yRD@yAK7Ln^DeZBf>zf2Pr@XsWG9Z-``gSo%|C``C9u(}^hO-CmxPgNP9em`Hp&dHP9Cx&)_%Q~v;#W*pPp%0JZvE$(yt_`QaY*4k8N=tl zyE*&L_*|P3zvtDhs^l$2@#d|JoVF!Kwz1ej2F>&}YOkw&CAgFG z!-1;p<`t5YTb=&tk5D9e%r}}r30!Xy4_YgtRfxg`l@Bh5s}v)F;Y|6*kIRygT|;3R z>~qGP2fG4`HwprM1Uazup;2mKm-nE;)oE#T#of8(=3u~u?BSnLAQ_x_NzeX!u)PdF1pSwA)x;eAnuhc-nC!ywR76VZxl`V$dCp*L>^W2k_2>Aw<- z#%H|7=XFR3a=5>RKv}w4Vqry^6Vf-qpUJzUgpnu3Me=cYoSR?K?TiMyKF4F9!EETd z5Us}!mII)pWyw_+h=Gh%oIV0YfBoUIIpgw(Wd-->>hGKvh%!+c{k;)H;n`$eAbPXP z_O%bAiy>%`T(uwQp&)c?(-Y#bv6e1dmM;eYZ|JJeFfRbZ(@C4J;5X+^wCBFHrKqh} zuU<`G9=Gq%lh0(xzu9X<-=L1(Tu zn?_f;0r>5*h49y>0VdyBUBC^6vc3KdP2PUJ9`-lQcfRK!z~U1yYCZMM$CMS zOr$Ci4u45Vk&g}mBhb($muQ0*;Ii2`z1?#5z_fcB=wy#FGv(}7TsiNxD}X&9|1@f; zCQRAVpY6Vzj|tas3n5_iC=U$XCfKY(B7U!!#>r1iIR%q=+UF;BY-fGSh;cqIPIbQTHCi@4#H3i zwbP6{OIZ+$9st4n-&&>LoAdTy=D`!%sB6I%L>v1sRCFfx5RR5J6*ivDbcev~2n^cUV8HubJKppcEG0-- z*WA1-#nsuF7|PV2V$rX}-r$g?O$9pTDuv4maA857;Ayoy4-;CH%KwCip9QES7v(h2 z*Vlx55`!2X{`Y?U_!XYsQ>T{QF}w6~a&jov1Jzw`T^6qjh#;AhSz?w{$uQ53g7dO)vr2v$bgAt{yZ=!KSEo4B&_}iXG-<=77j? z4KUxyKof3OWGT5w(OoJnhgr5VK?DHIRw(rlsHTLX@#_i2KLOUKP`%PT1Nrr;N&2(2 z=(tyZ16?w=pkUDdr>$`_4Qu6X^GdYuQnx#3=ov9CvSKWT}H8UpGV44ptJIh}!Gx1?d+89dkc z?Cg5EPOo*>>qA5}VbC4^dsKl1<0;g-8J`DsEctUVOf=8(O8Eh&*!+?4fZCrbPfXQ9 z)YZynPuRJ#%oE+tB0BRj%PN*t3mC)Zt77*|HM%p^C?c+m(WXCZQKi<9%|x$y>RGeY zCNEdoi*X0Guajq?W!JynFOvV!`9O`Y|5}O_X)VV$0SooT?6b0_CinOkF1EXr6N4Rz zg?Jn&&R2k$M1rT&KjpK%4CzhWz6HrX%?k@Oi$oECkgJPGgi){q_YWG1Ey(YEc@R%b zqq7K1ARIyn$8f0zd1^@U+7gsgp!rDQau~g#jXMMt4S*#!GKtRO$3ctfh6p@6%BDTO za^(swPk6^q`DGTYbqD;O@Za6h5Uaph*uVmMB9RkpIk{b#II+U$l+w9_S`gG{I^Qud zBdI=RJ(ia4zM6?0i{sl3fn;z7Ce1sRR*wSTzB(&T{VDVTJ~PxdDAJ`)OS6tiXho8@xx4{;td1;S$~X>fCJh zS+mqEE#qhI&nNWXlwjSshh}PFqB~~!i!jhuAKrP1+7d@))&j} zEmc?~cj7ki>YF7?HEr4*W0O7yCx|OVKI^an;CsM1_G6n$K*^K#@yTiBE|gcJ1^2 z4fuG=J^vDnLXl{P%2@|WAQwQ!PafrfR@t&iHaGSD`SS{sM#|(soQzyMo+tPQXk80n zPB~+RXxy!X==`Xb@$h(_@ec&hfHgCE=HU72u7^VaEy3V5Yt~d$Ru=rM_e;dHg&gTM zX5S+>xL}t+*Dyc7Lh&0q+N5P=mn~ZsUsi~Oa8A4Glsud~GekX{@g2ek^JH+XeF#1h ziwg95g$PO1nTOBC$B7#JA<~Gm2kPwbO{>5X>$iJB2&xvE8^VBs{4-Hvy;T>a<05c# zAW8`ehQwkdvuT8wiQDn<@pS{Hv4UpB@Y%`L5OuRLnZO*Y!AXP`Js%+f=sQj)B&?|Q z1n(FX-$TS93O(4~cfp1+@wJK(h4=E0}#?03OG#T9mbagk!Q1enf? z?))Jq8e7t)5>-{vlC3pmdYeA`NLh%=`h{va`ipMuQ&0UqIw_xUz4PB0M~FY0KR5UJ zf`3hPDZ8lG<1iC&8U29;_q`oAJ7O`8#69P_>}ea^*wzL{qQ`*egps_@jaQlv+J-22Edru zYb}q_!;tRJK?EbnO!P?4#-fWe#|Z8KT2EpEW9<+i0w`|aRP>L4mNA5N(0z$>&kA^F zwgHpV_z*K`VVcgf+g4gq?{eLby;S6yii_1`T(zs9*k{q8=UM)$t6Rs-FaMNLMTZ_pBha$wGC_DE zmI7G(hNoW|S#mfZ7fTOp^m`)(Bws?w%sNH5(fn71%ZcJ6wm1GBkEB&%f+XPD*O_;G5!ZzUHyv zT09zYkLvB&Br3$8OpgSzihom813=$@eU!SI-)n#Q3=U_;aO2jCf}9n3tfx(<@;(1; z^E>@rn-Iqu?;{e!mpf774V zLnytVj%9;6yV|yGMYC&QX0mMEIu0B_1D}5KV0SxY_prkF5{$3aQO}**dEk6oowEPi z!(3&JzCV~qB-3^TKsntYC-qJw_*OMn7+1qCt zRg{N#-JeqC2a!%YEyT(a43hCMro>mYrtICf z&k8lvRJ~IPVBesBM{jSPnv^sXTh~IYt%JDr?)~>t5_DEw25M7Kph2te>SG~=d-5rX zoe$!w?bYL7a~72`BtpKjlv_NE&$BF2KRJyhoJFlqOk+`HU{A%8>TT1M*Lhel3+eN* z|2^}}QhoPK*(6$_8EEr#NIBYF@8sE~>PakpV&_^H(E^=(9ZO42o=vHU7`}P`DSw2x zDeBWPeo*gZsXa}@NR&K4c3|+RCo_nt0JyZ&r~%m5Q4`^tg?5W#48WV%$%CI{$m5SGEp;@ z1M`S$-VOh0X=&!N!*`aK%%V2TePc(I%=|Py|rbl6|Uo$bpTUV+{ZboJL!B z?6~+l$H~-G5Lv;OK17hwJ)2Kf(t?2f(wC;4WeOte$pc`t?6FwNfE@FE(l_=zuwnE#~@?!vf z<7ZI4QUBakTgDachZt{L;^rOm4iEHS2-}t?4eqnS_lo$fr~-_P?Bc;@KmDsX8qFA; z0>lOT?eaXs8(j}Eg23FIe=XRZ_$RL$jK(#lTa`p&o%@^zXq(pvMvU8f%CgEr8BsJqKtX2S$_JW24lk zxUg}RFOwjhTj=O@#9-iugOl}24fF(|#9A)g_2lslsG(1e97SlLfEHkK=)6}?Z$4Nb zI(mBFHnhNkivCYri&Mk;bFqs73YW`ag2Z!;-g~`jmmPPQ=&j`BivgYZ+bUvFqrbLG z%kc*r{1zM_k)5+@&6;+nqaYJPBJYcJLq81dFKN(YX42Os!dHkomki7)2M0u2EMBJp zK^o2OhXf#&xjM6yF5NUyu}!`z3DTZcRJwaKcQ%6?UwKj54UH z$pU-E)I$CQ#lcb<8ODO&QueRqAsmtjwBcHkIsKZu!$e8c`D3(+9Evev(;@GXTh@3Z z5Kg!L`sG*+hKM8>n3zKtceP9dK-4of>;zl#Sz6z-+|4(3dhNPifI1-`#gl0pzN41G z0gd%ZsYStI)k5Kq24d6mXFczOpY@2j)01f>U%t3mwE1gIxzty+q(S?lo6owjN7>3W z$keDb|L=gJx1xI~>^Hbsd;6bY-bi2xOhKWusjI6SF&TvSC|EA}@7~#!r{m-OeT~PW zY=HVn1r3D&C+bFYT4V`oFFtmGsLm9Y)|e}7YHC{Qj!uj0VUUU(5%evv^&EyOBt!+( z&Uzi7Edudj8&EKV*kC6%URkm06~xK_I;XI>JQswT!(bQ}%~`@ek@ zM)^XfV7N~L{=-BK64itc)KWx!Q@~9z{S+x7tkrQWD<=u zOLI-SUh-3^oxYT5TM%|ADanm;!^Ykoxg4A*1gSepug1ZPijsah-XyigOv}It=63d% zM3G0tRNM#!=%;iH4O`O_kU~et5e15&uULkdvdJQH78r}<{PY!>nV^wGl;He7fR#)u z6&r#z;zGc6?kV@NC}+%=sih|Bk6r-1&jE==#D}dlL3fRrK!bbxI-L8ez-UI`1TTRo z@Uy`3i`_?QHV6_3?ww1(Xm+EnKt*sGIX2d*cuQpVfaZSsig@AgKYxlSE1Mf!MBRX{ zX9oFD@I_(u`h8EEWGwM1B^e&AcWD(lo}!G6vT@e34eCxcw8BHk+9Qm zy|S_}8gkd9^)T_lRUP#mk20bhoFyIjDvhOY9HW z=l&6y#X zqb`8F4kND8$O@YR0QNHy@YRci;&6kHW$z1478JG28@%`z`*amr}ocf&dc`G==Z>hBPK8Fz|)BdbCGB`^&+cHY(>TtE=Nczt~>} z`Z@+aRP{PGgPp`dp)M&27`}kYhG8Fi4-D1sLsGN()Sm)5@ilAwN)CUkRh$ZyTYP3_ z6jX z_p02qT$v~ic&yP8iS>xm-wX%z*|XhC|7ViHCk1L3`RYK~GxACrrKMiNPKwCHquCJn z%RjBWTnFEYAXw*Dq!DsbLf8duU}9WS-S`%^0;FUzF!(3K07V&CiJ^uDE5C_n#u?Pv`7DN(@>jL8j>=7F7B_<`Q z!eA0Cf%=wlUkvZ?;n=8cBN!0VT0AqF4o^;NZDKJ}YK{+4LRYUuCUXjXJS?E%P>qi} z;4i<@B8{&2wvUMBfWDSFOo@wAli8=I=SP}IK-#p@W45daS*{soA0yY~{%LLK%ImMkWqWEAF@LDAm~h9*+((5(OIjOHx8TG_avbjJ3IRQZd9C9;qPG^r@Q&oCHBW=xc#( zZMC!%A%Y^Ho0HDYZ>sT+r2v1>3qa>6i95U4x)z_R_#E7kDOVcfxn?NFk8ctv&HCHP z;F~tgahFCoC{9RAbSiNmy&JT9VOicqvgRCOM3I?TdW$`@B6QVI2G+8zmnP52b(nA? zT^rU3j@6(iLn}6ggkw6%r*;=*>p$i4e%7Jo#C!BZE(Tg`E$mg=TK99Qk+B|q%* zJAYbUbh^zgy&gX59=Izmc9$yCzaZ`#xHn|&*6H2z-P=H`uQ*Rd)` z`g`e*Ouoid!bAx@!<~a;b-l^m3L7e~K#q=L3*wSMDNek~AsG7~3aGB@a1nsLD?Sgj z9aHDe3sv~=>!jfWizyha|46CArHCUG65jtH-xO>Pk{>i6PSvn$ISV!C!__U?l#O`2 zc`Al*jV;+2aM2tK7{(mtnzV-G3FBrJjH%vHswp}oKUkQsZub{9n z{h>U41z>0xBunBzAZ(JXm@PF`k)JFgo#c@jp zaR8-Y$^YhO%;r;5(f4B)EfW{ty>q*WDrPPi-EP+ZhrLnQJLl+U8N?}ZfmKlIj4)AGOviV@w-uGY4yLgJybBkqhKasd$s@XXvj9jwGEpVO zp#3eop%@qpCY`$b_O%dkhK4IxgnMu@!KR~lLYzfKOKTRQq4R21{c+>q;XlFuy!&Yq zzappRNdZ2MdA=81)`-=m7E6gqNNx{ET585#m5~c2JEmz4o>yrcFEwWv1HRQP3TZ*} zxjFbEzK+&3UM1ErY8BW&;_#b}rexYbYu2u{f2kx(f0X$;wF0d-rjq!Kn;{}R*G@@t z$g9UV;^U9YQhb{*azmX%rB64C;)F@J-*unRaNI*`jD0F;S_ZsUl(W=jQm-rkV3d2u5A-@dzT0A9Wn=s2f0_BH{(G z9#a(lpfK`~q7NPFjefwCmS~~jCpVwB)DMEeuki9skDMGB-ZJPlR7LS(ZigWlHvJi= z`i6#J5H+z>*mwTO!ZnC~j7cF_9JG}QWd6ARcYz8FzIeQI8fhiyPE0XKPEi6A5`iI+ zHGOZaY6a)bGWIym$eK~OJF0clcUE_Sd04HNKYu!B;~>3fYpZ0b`Rs_Wv@29YYOuUi zI8Xo1zr5#`O6=})h2qa!S3y0sOy};IKQm<8mTRg3WCbk`-d@bkf0KNq6;WBE;^78Wk5UHoFhIkA(Y5}PIm#<%6 zt#I}?8-{bByu89h{((i_RN{PYhdfWCQ!BaZYmjUM1-CfT%yjfF+4-O@02|8}>HyW% z=UKJ8)DJ>!51jslHer^;h3INr-##02CPcJ()47OZfrlC!1}UJ3l>;FtD7@cN%pr>) zRu0`ZC%=nhL`FXu+;W_JERBO7%8H9$-i`5o+x~Fcd)rJ(gnM9=nm&So{*c*C_!h)~ zbF&QN1Y9?zl$nCZ{GZvh^)x19vd1K+(P3h!i=E>}U7Zn5aP0vvc^1&3xS>EgQeKPKFD#_-H3yT1c=n1IE59Q9TNdQp;1k8SHz)ZFt}ABK|4VG5{_d zj|8%_KobJzu>$z$63RO4fUHxp4L4ppO3Xdqt?g;z3T4o9H>sj_=_6 zDO79F-};g*CK&Yb6-!VZ05NsL@rETh+57)Ye+(E5y9lg}JTwH<&|u%G-?7|yfc9cs zvj|F4V%WmI4_;fpPIO0zYSM4UxLne;(}Zf5QM#+QuK<_?^aWo2tAG(4azbC2Pf~x3 zWsqrhJwV~ZgTt82d}|1JKd)lY%N2!Xx1y`7A^=O@Am)*Nc1`ZuE z^PzgV4d&1JSO>$dzMD%kcCR^BV+@pM+OL+Y^QW-9mzgQszV7K>hde`{b%)RW;{H|A z!cHwh@y4E$i|GqIAhx~8AG_3V_4d030Ptw=fV^=C1;OB8FTo92$o7=}1oPF(i>!MZ zdSRYW+miP{nSIV2!gVlvo|+jMvMg&p23s(Oc!_0)thRqj6oGc&T5-tf=K)bB_N>PD z-KaXKLtqSrFpgIPR;5gKm@)48+fvi;AEOL79DLsIfrt@+BwYR~twV>F!=C0c0%y^=(e0y@tz%7U@cCP5`b8|AzM-^jd-htW3iyl6%P$Th4ada^1Z+vCyUc@+=l3JZt8jlT(t zu{83_F;X&gS8NleG<5*qytwZZ6BAYT@4ud50C!4MN|qS3PQAimj#`5>RVh5k{>Ber zr_))=6L2v_#*{jO8jM?fXee?)7>F(fn2D#(e=+Fj1|(+q4o3VyBKaMy0vuYRGrE?zqo3lOMMjdWBcUP= zwOiQ3>bACGTZ8ysdvG&9cfK!t(Gibx88lr01>f+19@N^WOT*4%lgsZ#1lZ7>jj86} z0So4d1Z(0)oWlWpZRd*GG2E9HQhY@Ig6kL2a8v;G046&q-8pl)yCjBqXjAf$HIEV` zJ&XEp%=o~D_#LmRkxzd)M%BkLcd>>*(E`rDpI&%9f9G`m`zdT;5*82`gu#C42taji ze*X6Kv~%9kpq<^=$Vh02CDXk8eNZEyjxVpO`jLX_hG@PZKZv8LM9CAE zl~ubx`TqT_aAO5G=k8#pm1bCZef=e9YgNztr~UkS7?tejJ84}&HoB{g9aK|)I)0TJ z1LFgWtNz}T*i!q^VPTcNSu~OU{XpbZY+;Io;=HlUdSm9-DRf#}ih2tUy=B?@aUiUp zT43ko*UB!A9+)Wke_wG+3m^*s{doxE@Iab++gpyPqTnKof9$7hZXPP3Px?m!uWfP^ zA-Bw{TNltpf-kV*!v~QSi@39*krSbMXXNunHWc@u^eteHv5Rv=SFU_lh?xhPxCQgK z#^QAR+Q=Ahbi}wpiHj{7icazhu>^Ub{t;MM7*L@{;H6L(LVz|1^YR8X;vJAS$oFLD zYEjYYM2N=(ATw49A5jzx&&xq2hcmz?#cP^6sQ*|TZXn|Ja51{MD83dt|51}Q(B{GA z27GO5$GQP{}ZBhryE&WErTkwc<}Xf7L+=w@Z3&Wp*asuBGWIv7Fku*jpLeFof!QW zw}Ch&kG5qg(~hP)0hu4d!t0Z8woSWaB;SpHvur2y}&F3Rcm61m`?P&;JKijCgymE$z z@c5(!g#Y}Tob=U7?L603<(aWf;Hd7mTWDC2xUo@F|vdZp`=pwCCgMQ zL`IS=R7i_aC`*(z`_QC}r6Hk|Y{^o#5K)9kDXss<{mz_oe&6fg<(g~eoYd#@{=Am^ zer~ssUE5`rzlTIkJWqOSbLf$OzXH}E;oWWOtLt0&TF(PxptkN^_QyuFl-f)g?yc3l zx!3tJ4I?xqf;06D%%uGmYR&JU?-Wq3`IgRlTpG?x}dL+=7pes-_<<}N># z>$rYzvG%aDe)jeCl`36qoW$p4=S9%wude={$9ZM9FTF3{6FT?a{4+|{DcdQ$;-rdb zsd?jm4Hn)zl~LjLm~`0m81Ie(Cdz^paoau9gZuZZIQUkTG_G*hz%BuXkD_ImtLw3o z?0`*N_rqVUO;B}YTJ(yKXR=)Asrqwu20L`B(g#NbmXDhbT#Z2Enjj-Z8aRvNJh&&2 zWUC%N@O;&g6L_LJe_22W5!bsgyf_9uhEOR2!>pvImVGY#URqH7y|9|k9{AU+xEFkX za=TtjpN<^WQQhhN+1E=ZH>iGX)pxJa(ytLMmB%r|I-`y(2v^5r+Z(OVpB&}Pp`&Vg zxZNUS^VBV59QSNM;@ZDoRd$f=kfkA})n7JC)cVb}r%Xk!B7q|U1w#w<4B77L$|R+$ z=qCkIKY?<}nPm27hpfz1)^LHv8|}Mw>sD_4U*t4f{bKd`<);&Y3@D{$*z6zvYLT%V z=1RB;1=eqGAI|S5LRHBS;=Tu*VC9`4}J`CY0?9L**?u ztxLp^*x-~y)NGsY7Vcovs&`|lnD3qP-5~(Gy*Af}}RE}gc?s3ye9~Bru} zRhkIUED^|u?DB~h0W~Xz$}z57J^d0A7+Ebl^sT!7{_)D4t1WjQ`!H<_Gn$)S>2)PN}ZF`?1D1rp{a8Y5-Iv4-C2`P848%As;U~a>DA9#TGZDy z&4g`GzCwRNpary?BR+0wqgmqP9)Lz=b>){-d9LV+243$ruf33#kXuATu0Q!4X*R_4 ze$wesQ7_hJ`|LFJy>yb1tLWg`OCy3(bQfLp$vUCvZjsa0GtjZ4S)p@pAUW&c?cv|S zbzcui{N5C$OWwU;+=C{O6rtQ3u|)hUo!`+zhhAdm25#S{In3hrKYq-`_ZB~o0<97SOYD3EUKtxd=2foj&nhvLka!%cywX8A(X`I63tE)j(Cy0KQ-0YX*PdQblH(ZEgIZ>a@Xa=Fl}XLzWr ze&HxqgxatC)b|AiatF&XV8H^|XS=LLqk1w(NU95oI8yg-mm>Oakm_*1-+%8fIE=`S z#Jl#XBkD;M4RKGD``+`uo-7&sJ#<39oOtv0Z6oWxi^zCD4u(|Rq4=@MJ&-YkAF?FH zc&Q+%^x3E2V8~s~o;@Ze>vig(+v|T=3dLA z_a)G#?6$j)U+D{9Jw}`_8faJ601`sde{0J0YU+|-%N&B<`c^u*$^0FW*YL?n+EXcc z1OkLolHQD9#|pUROGE%r^YubGs~d71ilt<)lBfV`H-q-LmK+=$!ecb? zE@6#8c{O_6p9gNzw9-oSOk7U*?8|{a@lyS|ir4?`=|iueY{R|p6Th?W-ip7TXKiWI zrcJ%ht%srtkiH)m*3I`|xRSU#^RIzgO&f%?Y|||K%q;iuzx+9PLDXxFgmFs)o37e8 z-Q)bprXjyoP8UH*)Pb17b|vyijC!nZ^FXPB*DtCnjX4L9BSAXCG7V#0!nB8#|GndO54p3rB8vm$k0Gql?)n?*bJ#$<{^;DWxiH=C6 ziL1D{L-a|K9U%?%&yD=otmn1-MLb#U#jTtT~+T;n+Sl0(f4Qdf66{6 zvY}d_zn%iKudFA{4M)dyB^o+#z>C$*1%e0?v?433)V~1jjl{#eloJ)_)|Ju;%KJ9k z33@H+xED3mrDz$G}&D}kf&wf#D zvRR#XK|TYwrs=mS9XfLd_Zm|xaPe+JX*(_9;K91+0!IMD?RP(oLbzI z^HlCD{h>p3(erM9DEuUp#n95odmDeFQ?)oaeo7?G2n$?QKI9@BddaOJB+h%OPuq|9 z@A3K4iq1MKh!sM%4|6eUT*QGG;v=c3`XMo#T>{rERVanG=C)v0$0d*38argKjoEv4 z|M~4P?TnWEeD2%J6YE;;j`=h!{C0;y&E%HIF#Ixv(Ee7^+YKg@C>0Pg&yGA7 z!_VO0lNwmTGWG4bFX(*Wo!%+V2jaJXnpm}x##KyhD0wP&tCMK}hJ(cy$08-0CHI8F zl$&_H>LZ{!o-OL}z^pdDYk8z>zrm>sWS&)SJP3P_mK!}QU)PA}q{!vcrAtzU3Oo@N z)saP>`1yMp%Uv!q8OlTMjv(nw8MRj?$92?M2gW3Er2m?2VEArEu=iGfe-kc66>kwK zFcyBF7H@D0|Ah4aY zl$39)s|SxPoavRRcG^~}U0}X;sLC_<4_EK+SlA<|I3>4zyO{e@5@l>TSpB^-KMW4_ zxKeFn4;wkEgX9twcR#lMuK_{Abcx!8+|AG-E{4`_Zof>QF=KX9BTyhDbgWH-ikGv( zfX5kcXK=9~(PR5(LBNBt=mQW^^QiI+d;yZz!5Y})7^n|V}E?&H1kUTsJA}@d5 zwk60MKt}YZ%RW5H5&fyl?Afzq>1~z$v*GB3;~+cwZq3T@O%U?!YS}*l!#A{@=*Xr{ z4ZnF2$`E9gBCElfh^Gkhu1_90vYrg$q`wJ|N9YR2wAMDw$6sX97nY}s!Z$$QYJrqM zxohD-#0w!nXDlkOFWHt@7;SP z^!T@*R*Xb%_QIzk`a^jK6``OKN}rpNU%AuKye{}t()ahxO!_`GAJyT#_MFJ|^oJ;Z zuikUjTi@o<#1MIs0nIM0Qk0!KJP$t-S8V5xHv-P(X|ai5T^xP>@iUDO2SLTY(k!yw}$wp3h+Ffv6GY zk8%8SYANbxB-2`mh`jHlLT_*i(N!e3??O=Efy)##k{Up`_P(#MPbFXP*oKvVd5VWnct}BOI6**e<2>$(<2^L6#e* zx%ePx&;5SP6Y+_b`io8n=3E*NY*yVP$yXAK5spR?0uYijB6`lMIlQB>GKUJ$&P%aN zVBBikEYR}~G$5i)nTJBLre zrPrQ{#l~%k2wD6hnG~X@Iu7Cgi_o97&snA3GdzeB(ib#;dRIOPv zaRPZhtC-%m%Ba@DhKpd*lmV@pNIh8Ac2LtcEhFn@&A)y9_YYd_{9T7VSn}l5g;Vt= znqBN=dR4{n%!DOUu;2%})~=U)2uU-8H`iCU$*x_Gu|rLLHQaD#0rM$IJ3P(Hl1s%@x&^@e2+X+;p8cPvV9-=J)gqvGAjj3s)biQY&2GhwAU3WkxN43b^M}$E--2 z1esHTRO9r4RtZ~JoO1J^q=pf;49ne(2qc+E*Uu>WFWIc@_cz5 zpDmNY=)dZ*+(^lDBghH4SwM#C5=jk^`7CRX+wVo-$7bMHKF{kb-HjNesV;BKNuv+w zLdy&^j-=$4v>ncbHE|&Ryk$Wd*U`Ewj_`ZSGI`LC%ZO6GUKi2jUA=?A? zXJ;4Luja2A`&OHX@C})lBURsO3$aO?p#h@QXh0<72pEl54)9BUiE`8y66Ftz`3<~9g~PuOPV-msv69uN=I(69gh4H6B6IHZYQ z1g$dnHq##{n@nEw1+q zx;3va{c2=PbH@t) zxHjA>SNe$IqX)Dcxzc&UmQ7g~<%wU9{trpsqTrw==&GyTMz6H2#|U<1ime1hAjj#o ze-}ojK*VHq%BmM#Pf4+m;>?ue@b2Ts->vuEpMU16tIUqx|#Vh#U+x2b*}^?Kz&AKXPS6t}ga= zBnkDOs)ropvn)6{gn(;g_cHDLvWqN$SDgX=_a5)oy*sAf2Gn}0N6Nhbsi-e=8@EZu zP^ZS3vAn{e0cAl4D=p41iAO?08~&XkOS5R6FK&9Y8OlX&sE`{h(B6cZtv${|vQ!ns zw&r~AK2lUv89}~M4B)O1$o=2ERZ=u&IuM8o8cSI<>qp*dLUZK^=wKc_WpL?aiOq)d z64<^U*Q_xEzNCwV69x8dWIdYq5rs=jKpmN0w+O@2liL0L8*XKNJ>H zE9B6*MB~jxUROLki9WyeU(QOy!*lUVN}I~kwLt+VDPrUd1q6xtQ2v;sK_)mTSrnr_ zIf7J6m<7JObI{U1FS~p?id__wXk3KgG^p6WeWdo*vO!=-`@GD4910F- zz7DmO=YQGX;K29ZzkVNCHP`YE@*U*S+?`tR;rfk@uo0@KDfJ|NM z*u<-2D~blJ6?ZbnKi3rLqt?rgx%qNVdR8YAQNl%jvc3xK2aHO%H$CsO!)sI0A#LR} zg_?PPQnD@s0`Lqs0VI{_afri~49bqF@RyId_Dk|a_!K0F1WM3wCL>aUnVlk57~9}{ zZSat`^0r`H6egXp-&AgSF;}bNWudI{qf4<*qV98vQIm-e6@`iE2F85Qxf!sc>v`;h z_SNH}63~{SqLcBo7g*2$c4mjH^y$Gg0!DC*SiQRPrA&bAz^J5`+^0QlPp>JA{Q9eS zPv)k-Em>Db1mrYAd5-|Ul!2#F|Bp}H=Y+?Ku-$5Y+SD!<~kSxBgjXW&l;b(go#ut#laOf0xY`{SE1(YVcJkW|o!3mQ%7| zSP8#mqk?q5*Ml%jJl;lo{-BKC!_8AWXdU_c_8)Lqr}S^M*}85?~ zaPh9am)2B{kSOKYp@|FbizQnweS9KF_r7^{XPEAXxmlFDmm$I#JLr-5VcB5hvz`}T}t}4mG9JVO)2ezg-gT!3HQM(ncpT8yH&?hHP znG%JOnI~GSpR?=%gGom1jR*}1IpBC&STZvh(xy0HljWb4ccva^q>%B5-4+BOQZ<|e zha_=P2?^P=dm09uucD|xVmM)lT_CLKF{9o@+bn7w1R~=)GPUIK_b-!bPN@+)>W~W9x3Wu18=0vcflM^ytyO zXI)jjl4g>QLW3FcceAU`^(~PbOH2#LO-2{Gr7gnUh4t7#9Qlmg`sKt46qCG4 z5Oes?lFi?A2j7{peE;9KyANsf+fK8fu0x1*ZTZ@<%MUZT(Pt-b zBYLgs@BTQRt)0Amz7IEH4VDpWDHi6(Ip&EwfGT-SJ{g-QAD<+P6QRM$=WI&aJ*8q0 z5WdX;dkas`z7i%lZrn@d820z?@W03Ku~bu=|5Uieet^iLhvVjqyX~2PY)htJQT{o0 z8vK5$5L}Ee{Prn(BlyqU1keE?%0*4fG2&5YgX_54TF@_AyJ8Byl zz<-z>{Qf3A^IkgH0_#T{Sfg*Ha>-pHkowvmqzx_Vm7v}$?nf||W>Zt}WWg&0@}bv} zcs7&`xPZ|B(Ubbad60)idP0eI;TOfq5_P2ohr7T_e2gOE7iq~?z)W7Ni`!psbkxP(-7Hft6|&LIb6c*+LWoM&kzBTDVqZp%1XXR1iJ|F0 zV$L|_wtNDepH3~7W)$=huXjotcgM=I zFF%EO>Ee~7V0*l8BnH^X9aKbKJD)drn(*FUSKd&$7lZ~V22y_u(gdg}NRWAvSG5EB z5`aOLw!A|mHz}L-UKCa&D2U22Us^AH0|Sak(QV6Ab&%Z!wgeR6lv_{CizH#9P<@t2 zDV#ftid-668H5UrBR-(prrza0qYz_^Z95bYzjp2H)kjeOU$$EdjjLvsJ4+OUcxco` z#V!`l61F!BP)USEsjZ5(w0yS6t0*3EZvg-q7yQ=<5)r8sIOdCK|G=7Q!dhIehr=9o}wlZlzTVi9~PSsp53L_d?C4JqY_ieJvH} zG*Uf-LP9Vq4fzdo417)4L#{WeeI8GHbrP z8^mlbUOW7DX^1Rh{T=_QEqtc8_-U!N(elBdVI6h)^Y3NzD~|S_`QHK~-1cZpcN3F~ z4t=WHo|zsZpJsX&D=lbyLKimIu4IrPT}&KX^xthC>Px}_+ngA}mnYvX8Km@aNE0=6 zbQ*mxDIdJ@mBIPVG1!!r4s#G`h48bzfhcK*84t;KynqG34XnKNPE>LN9P+F0bZeI1w9fBL%eqoX=!+$0q#&I9Hq zvHpho<85r3h}HvX*a7OMY3_^@lkURn>m0#O1+=|M&3v)y+fxM}lN6-9yu417S4rk?}yF*M{vrYXRA#zt%* zitaU*djdP#f(0kh3wBdv($UfWx99jxzujp^gVEUVIqp9JN4&MUjNNC;_U)Zdu)ig2 z0{6+f37S zs$;Z#`NmDd&5af$f5Vh1Cv4t}feXI@Zq4&h*I2e%us%u)qQmad~>->BpbJ zO`(PlT)Oo9`>a?y`*&X%0&x)gKR{~SN?s=l#NB!n3*2Z-J*r*9teOdOU5WA&xkcQZ zx(T*Bd)1*yQ6OB1Rsm-r!ug(yLq{9k;>}6|hYVp8c zcm~VbRU9jegC(4I9wt!S=zV72*#DLgt*fA^zv;ZFffCggr5y*P>9KeyG1uOaw}C7U zeHipoos&V%YO0d2)Ay?Yx0CL*jMT_Z`c|*@)nf@Zc;{`mR^N)Bx6VH1$B=-&^nIxD z+Xy%^Z(cWP^d?Pukk@w~o$#GG#k6gsP{6)=^=d^^rM$i#LfWAioIs|pq2wt4P>0N{ zK=`YUaG|4j^w4~wB)HVyhee6aE23zA2quS{Z~t$PS7el+zOh#Daoy8DAmEj#ng|G* z?c3${oW9EnpN9~qGz+;TcbR5YA5I8Km==u!l0)MH3={;y-rZjqsb8u^pG~?7*^f!n z(89|onW5$RoyZ0B&_^96_5x;g%q~G6AxJiC6|sotB5##lw`LZC&TZm%N-!ahEN~__0`V3|Mk4RTgt`@4$U(*{yn2z z*^%=t=4sZy)$G1mnzu62R)5XhXCXo`bPWD>|JB;#GnO;~qj3Iv-`N+YM`5wy_l5eV z+en&oI-w|UQR_e+~E3^R`Ew_HYa$8KmZ4>h1c|qO(T7L9 zAD<>Sz9JGM= zyFQ=p(qj7bi2ZVh7srWjl-MKjjVgR)b1wsWYR)?&BO_xsi-Vyr+?qr^uefh(uyw!w zr!Le|lrYPW#}LqWccq$U$+>1H>UQb=Wc=e_j-=uK0ZXGR~d;%^r2+S$?!vS3k&gTQCh7&tf$wqW&if^A^;P4 z7fYiT0u$v?kShU{CUAtTUaAL((5dB>Anan8MOg76uL-0HxZBrG+;tq!&n^n0seJkU z#Ze5H*t2PU9~)-X(H{d&kW%@w{gF=P^N^3=X+%9Z0EwB4C%FoACn@+vsQpzt>qlLY zyOgJB+{OwNS&YR9X&4eDx5sXXt7%)m12!2A7@%HOT7sZdU~5tT?^bWzI7tF!TeRr6 zf1sSNr)wz=#aL4*7CPdzT#X_yxR|~zEL11fX33Hz#%@T0dNzH@k9RG*=W2wp7+1iZ zh-*%owK8+hdaA09&?cfmkjUl_9~MB~m&Tusjn$MeTaI@@k7a~}iOK4J{VX?O9xk)# z#-qHU&l4R^e?tX|77#<=680H0lN75#hT~WAq9n2oKI0`0*;7wH-WdpU`hR=5fD6w9 zuN6%j(a{4)O;~-=D&v6g%xu@JfuIYt&um*W1(=wF4A?H#sh$$o^!Vd4`QEb6N$Q%6 zK zh&}$|MI!+rq1E!rMyBW7yPWTB8xLSZql!s#T*p$0dgV;K@Bm-i2E4;{K>8Aj&g$+y zwjl6vikZ!Z^3k&cTMXXY-}$zsLvT@>onLz>4MTa zptZYy*8f)L1xMxomI)I~Q-EildcYgR6FgSbe=LTea*-=wE`j~({>v{8=f@xu0qNh_ zcW$%Q-rfinGH!CJNsC6WD*VFZXKZ`N{*hnAOmX5(eYL2Z`5+`9?A>Vt3l*gd*u5y* z6=$mDZsaB;Pc>#+)kfF$oN3(NbRWanFv-g(NW192(gWuk=@6!&!X;l+_Ldf;_&&gj z6e*$^7ziIF2ou+|_Lkt4a?JjSK1YGdA=#tn&m0a-2l-uD1NxaKVWJ9TY|bUc;!1%= zlgo(}K%&{H^ojw5MUsG#3dpUAsGl?0S?m*8;ypLd4WLq&8wEZAd%U_NH;x(=xFCd~ zA{2yjtjjCqyj?v7wefW`R0fi+PpgF2?=slj?5||k_*2=-sZP;8yxTluJ}K}vq9u0r z&}`b1HdOdEkr-S$r?Q9uT)EODTD5Eavq$k8`bNUVi=0lVpnPVBmhYV3c=m#tUd?xAf!#zFIAw#^c@6$vnoZF2~;N{1UA#5Eb zT>_N7<<1*@uQAlL{1L8c+J=HIeuql?&4wA~Au4@mZhdOhAfEb#2aFE+1Jt~s)D*`0 zYFTc#&reGqa?HH$l?7Wr|30tH+x72y8(zI!s_?=_UC(#g!{WN`mt5gCKJ(L)%lwD^ zZM&K)18pXwSinpwiUqc8r&X8#F-$7iKZ>z-uxF(Dk*RiN+PUdU;I6m#5FzA9i>0q4 zz%21z-Z{G-XgO$L{5hP})P{??At@^>e83l3&GMDMUCcd5sAkVTaOB8a|AHlh+Q}Fx z?ipZMW@|+`EQFV!Gmx4KPXDw$Bp32#NAX#%G|)*ABn#pwt=YsIgfHPaR6QcKkyc({ zQkT=GPwzQH+tr0mAZetJ0eni-in(B-LN;kXMKPbqDf=jsW_u8VjX}9gx66`}>vmi3u2JaSc7U+;9-mCw%ZtVu|@aDu0GBKwL8(LlF zEM5~Ji@11vCm*seXtOF(NEX|V;lAN}Q~05tg`-#L8H_gf8%q{+Z9f%PHBv=pX-KHv z%y>dKCrGHC!yF1S*ZK2DSXpg2oe8ESgQaAg3h4otm+0REL;Xw{B3#Oy>Dcn*7$-nL z{X(H1a4z`eU8EQ%()XF_DP*<0O8f0-wh2D~_}O!Y50oZg200T7z%G)+GiSD^-*2Yn zcSQwp#NfO%mp)z}3TxQ!l9wM|&ILmv$L=fY=Ec7(ecI00K4a6p{(EOw8{H+CXPkR% zPKdEX{-lcyl-s9+6Ut>Y6?bgu9+JZW5YX)sD)l$oTR@_$XI?3L}(qA9VT4W+p9(32S z$d!onYG`ehWLxu>*u6{`vFg9J9I6XV13O?pay$xZ?D=BcH0=D}DdcZ=yDJgz6K^9WMZPVWtOsCv76_8K?1Fc4L^~Y^G(gmOVQ0oe+yK%S`4_cc*PfJ+ z9135jXz-U#r5Yz;$^i;S-Kskc$qQ$l!Nl%S1F{CbOb+cb#I925O(DT-`tAs#9K5-;PlOW=}ey@p8x)b$4pF>gv+O z*TwJd1L|Kn<{WJCTJ>neYSrPx2ronSfj+xOW`hO|FhKrQ{#!&X4a}Yc`ZCci^5ef?L9(LH4c4G3M&%0d>qi3p{tO6fjh06%47! ztg#1no%ZrEyr>9RF+EY3Z0+4aGb9|lm?utxqykfl8_7Yc;4b*mqk08npmtNYxMHI;d$;^Za$^jgw=zSBg~*06>7 zzU0&efZtk7B~e-Nq9#R>TBz4QO|9L>Lch>_=tX z`hLhBx;+ee!zPHom*-PDl@2;$9glYZgyRGes;DeH*M_TSZtq%H<=fn6&mhI3=LC_-21f=9- z?J{Ok`5wHn-5%%WQ8*#I=G^(C-?^tDpsa*Drj^WXV6i#n<1%s#ThO5iA8!Y>&WeS3 zs$I7%HopnS+nz40&3)rDF$h0!paWT1Ip}jqi}lJm50Wl;8^wp8^|g0{jl89My+79U z8eMs3-v;P4l;N{LuwfN{n;Q+`RXQ`qT&DZjovy8h@V%}^ejXI$vW<$FU9j;ZD*YCG zbo0-YGd=0tIc=VPDuvvrk7D5*)d(h~F45My`F?=@uT@}gZ{nb*wC!%M%B}9@Us12^#F-bJi5K#a=or#ezOoSk&%t@ zzO)WQ?vP=Q`?DF#ziU$+ygPs2vz#lC9+K)hO?zRJ;?hzn+!w{q3@cw%n9zW;XO{xjIpnZRi1A=U z21%4WQn0BhV9b5`XwiH-RctOeI{f!?1MK?e&;zN5#?R&Cp^zvk51 z17Pz~W&u-|$mUcYTlvKJM@Dj+R;`8sPO(^;3ghR&w*wJf z!*BuiV$iEYMzR_J1r8ROfPe>OgA7X`K3SJx8)gi#3Rc_TY66&k5cLIuF{h7+e?>yP z_ARe9H&n(w0ZV&))jWEC1Mi{Z;K3DV>{HofismSDLj zU9eR1$V!5SQj^J0kbwiI1-KL&KnHYg^TFe^#KojXO*zr|_Y2uQCu+$PT-h9yMoL~z zb(Fw=pjig6^L>yvoE9JKtE-`*z?A?vlJ$5eO%{8{2y>qJ4uagp9nQWW5=zYu``;IQUy0)TK6V03qaQ0j<1gES`;`ba4HHfpjv?>5-(x z+<&(_u5iYYKdEW0SQpNV7HK?xUbI*JO*gzzhla?)#e*k#uqZpMV2azhCsy@W9loVo z>x=@=^UuyddKCD_w@^2&AlFy%b$;$YE#qF{FrPyJb-qUxzr`cXXA95G_@u#yZEoc8 zY~g_FlJm{8{d&d&f7lLO`@GTW-h)@%vr%D*CL=NHWn?7ETZBZ{+s%8j_(HZY1CJL3 z09(c6G{5>{8;Blt27eIO@=pvI1Hb$$&~yz08taODa{PGB@`?%v<2$ZK$Xz9?I>M+_ zdqYYjR#i?MRM`5*nO7(_A+btzYfW`PZ`am%(Xa_K$g)(bBKhn)dbJ}A96Y7vQ+SuM zr?|^*_+COgfCD%n%|+*oDN1KbTfNwf#%z{F+-Q)fC>*W^-Hugh5B)%Gxt`%WP2T0( za=F^Y?K+3NxjL(+++lUI;cS2oKV%MneB>Y@xnf3&JSp?ysq9rc_@5~Auo(Fc%7kWV z-MTeH{$8y;t$fSY;G`T0)R*w&wQDofE75z4H>vlkZwrRL7TJc1nVFfW{yjXt9IKN2 zCH`N~NCV^6wo7YX)Rfj7IC#)(fjduuyJ%}~P0fT`TqPbDEEvUS^&53L-AEy(@dA-^6 zt!@#odg4rKp6VAh6~19lD1C@J_+?_pbR%QqApWNs!|D|$i_pFYBjEc!!fOj+7_l~m zJ*laFQM2la>A4lvl>;x~K-2UbEawJP%@?7}`tZjS&7Tnl?>%UUxa#6xukMi|Ax~ zU>riYCJ2D}0~yaheSCcW_c)DwS5S~d8!Y%KKU8}YC6iFTVCe=A6GldD5EUZk=t^dv zQouDY=o#KvL|Vwz8386|=}DSC35+bd_?n7hr>XBy99Y3=^1gCqYA=eMjUT!E(OF(x za{4932dVf?;px}ie4~>QGv$N;wL}1X@u|xPzKtEhDhbY}qz~mHe>(;eT zp9%MLdB4O*2#}~aFq>ONPZ|AW+2E{=+|ru1OR0ZZks}x!4|HX1ZT*>*Fn*tQ2{$HQ zCsDf-OD+zgj^92k*iT-^q3Z>%4!zn1b@roVhocVlL=`QGoYjk?;beu$lvnK zn&4lbi4g@PlEJQPZqM0T{DS6{j#(6qdA=FHcn;db43upTIIwKy1?5ln4-9lhk0k9Q zx{2hK4i0(}e<69+`!qMWT1(p_;sM;knJZVV663<^@^?qYpljGR%|b&}1oSlV;TPv7 z0IWd0Nn)$>KUs6>jft_hy3JOU0htqyZ|f#8sv+AUQG`{(e(of3?UeU@jue^ja3~7H zYMRlF!6ZpHM#0f(J~Sq`ZJ$HCB<~8gCh79!HHU`M`ioD=Gj5*syEcn+j{dXo??-zm zYNNDQWDZ#G*Dw1~OTC8C+Pr{LG$T(yK$KoFEHH!|-56!#!hC)AHrBk+0^` z;S)e*P8%dM0#ZqslYAMvPm>4RO(GyT#F0u&J1+SKVV5{DlIEXrqCl-&HNwc2hJp`J zdXAQEh`Z$z7K422tvo2rv`(CeG&A`hKemwhj&}H28`sVW_aD+y?e)}iei&2Zp20sxX#$t@3mOHs}{La{$N{b$ckxyK@rad%|&>%kSx zTt3>6Qn22+r@tQmQl+2q0IB0CgqDKtN->m>n5ZW!02$h5Mi;e)R8AN)TtiIZ;KQ%L zshzTk8KS0@m_zlYrQ-lK#3XU4y4;P=G1jND%!5FLN_F(JE;gm_X{i_GUe`3@`dY*m+g?eb3bzdO;cvd;22D zGB@`PvHqkY|6=rIg!klL$F`q(w8u~nxLrl%;PiTX3{5TW-&9+-hR`N38wtcz9tDpC z#V7Y2e=t~F0SIa&mzTqIbjd_dz!H!~oj=S6>($C)f$@Kg=k79OQ^s2W7P}<* z<)FTivXPTf&1RHQJbY_FMafXQq4Lr2(4=E}zbUF18pt+?0NOIUd!iD#gjm zc|Yo7`kg!YjC48AxPL>=&|QLWZ~Psn5e58Bg_AS*gIUCXe_VEfqL43F6XgBOdypg^ z$!_=Xh;{M)qDSzfVxU|6rjevcgB4J`>29HzKv;E}Cn_FoSa=L26GF55*k~c5I|u_wb1uf!g7qmd~UO4)Q|aVAz9H>w0KI?F4;w9fd9eAO~#^g z!c~0jY?{luryt*%JW^x$^WChhE@uN1)%TKwIOua#=?jz#R`20x*f~#ws@4P!(0eCN!x4)52tlsjxM5G zr`lun+V}EuCvi;<^gLqc89$R-hpq~im+ExwHK)+FEx&Z%J*ErA!L2!`4}F4)^}EvE zo-$z2o`v_uUJ}>0d`0vStpIu}6AVBLOYpCbV9k0>E{ z;%WDm!$9F66Ww$77DrEO+BN|GxJsQbC`#qocVn=z46XcZwo5r=Z7)(lBO5SVKtT_8 zAZcKeglVo^{6m&(P;3CjdX4vo#N z3KqwUS=fYmXw*U+h~(FbXuV0AfS5t~%ekT+ot~@iQ&lpRG(D@uc6~oTUG-gJB_~aC z!~`wH$nM^p&{6cnZxYaMb^B&G4u;36{F2I(ERl}Bs31L6xD75H~;CGf|5T6Z$jQaD> zKa-!p`2L$bnc`wN6&C<>kmY~MV>F#bwu*0>83CGvl#{~YU2F^?$ zGUo*eG0IoQq%2~n>2OB(@861qw)W(t<3xQE)h}vl`-Z~Q9LR8$%lmwGjrWA_qzAkr z?n3DJoYzSeU(6@|#;7o%*El7D&JYlQHiXa6;*z7hl|DpL568yh(tHOiMmSXoEkyVL zvu#mzn#CdKY4OS`)x({+%x~4A0>lirTKd;4!WtOAA%iQ;2I&|kDVd6X*$eY_3pb(O z>q+4%lm=90t?wxnlX)Y47l+RjsC#gRD8?>OuN~aPz-LC7G{JNw(oaC7FPEDr`?NLp z6M}|L96>Z;GGa0J@*EZ5h|6fqWGsYC6ytFSk|W1=3U-srG>8L!ZrM`Op~ASThss6& zqx16WY7o+^a*^Gq<-?5np;*h2N!SwAOdXTgnJCyBT!-NV@FmJtyM0=1IBeNJFPSm{Pew7Xh12HBw?FcvFNG?<|=v% zNzO)&&UE{XVq&M&0Esv;ufALIM+pKCMKl7yxWckehLKQAHczY zRcTqV1dwesytH#kMH5;#k->+Z8Kfc+0|aW{l(El2DKoe~D7>yML(NZhK==~NQ)q$U zV9{pTxuMb#;QN5d_0OwrB^3@IJDaAUth%C*#FlQy)9JwUyo)sz;=-c1{KnIAkLM4% zaD(gA?Kxo4`+0h8dpg?ZU+?#tXTj63=)7D-9%0zn`KQ#CwTWi;-Qj>meN@uMa?&yI z39J*SvV{o0Y1R0nv{CiI{yQ0*O0?lA1J*N2uTv(8^yIt9@|W@58VPB&e6ZdG!?72meBRMX*y$GVtPYFtH4v4|F-xFqD6 zIR^PA(zil!bZBj>il!eeJsx)(39Z1+aAT)@3*}W113Og^TYNJU)oY>i$x;nd>)aU&|)o#xJ8A8aIA?Z>M17aM<=G zyP6IWjc;uox?;4AjTj;S+d;+Jeg1C;rJ*|f$Tp*0O-$P^E6a&BH#$?>V4@l4n6)m4 zuw~GX9COOKgw5ySVX)@fK0w2*Zr!?y?zX1Vla4ZN=cZgO5C#rzE1z_tWGXAarVGsg zlI>#CWhshBL2Kvps^#WXh*sB*$Ri=-8=ILW?s@%o*##2s{pbYJcHcUU3gAFo+=K-+ zkE>4^orNJ`JYzn$Ef!jv4JiRYD@2c(zS{c_IahpuM=MJS^*eccQ)7b@&w2;B2Jh)x za;4sYJ|nzz+skE}FlhTT2^R~p9BMWEc~{q;d*v^_+0H-6^p9Z_h`FS z@W~{Jt)(JQ+xa08(ZRSWQw)w1EFrGz3n*S{~VZm(%; zrEN3i{Ig+mT9qozO4QV(E}vvO>)sl5eAxUT^&3yh-^c|%@3z{PwVNk&Sl%QmD~gb` zmMsI2FG9|ZI!VhROzA@+j;F1u?9TDX*o97yw~aeIVR(!rNMCj}qJL+8z@jfACzmhV zL4cg#+6=Jh@pE}6dQ=Pu4Fx~hVqr8!v|OUrB3c+WK&wCweJ++$6eH9QH?_9} zRcmpW^>fR733&Xo8zm79eov*&$5qoYYQyVg7bG%9i1R0gRIQ5_pF$-HuR3+F^)2huKm3S=BUaGZ>0{*6 zj}~BVBEUm?>&)udz1#u-<#y%JM3esgTT0tQ?S3ecH73k|?7aI1cwDY;3(NhM8vj#b zW2n9kYMLQ|#l=gQE7Er0V%27WW+zQoeQP2fGUf-VaZoD|)9&^SrE8J}$w?9WoXQ0J zO5|`Hw<+uO8w}U@%p|NFhnjM2Aku5)2#iq1{#v1HCIvbZ{|At5=pYqvvaQ<_^fL^`U zkbXniH~KvgiaZn`ZJV?pdxv=^+SLo067Q^$VO&vF=rw2O^tbx5B8^&n-qe}D9vXPF zVadp8GZHyZgAAgB$1U7zai7Ve!-)`wf4dfO^UVnGYry*4^4I!RQ*#y>%R5esQdeo% zq{;dWSLu8BCq{9iT(#Fq-O5+SdQaoFKZX}_T>@HB9a^%n~xhS4pqiP<$^i9lF z-1G!gKGrR(Cm6ExI(qnwNpHAksQ<9PTKA8v`3+5+FmGM*bs?$X_wirQjiKG4TLI zI_2s1VlGr3h6;tR2xZh$ONL1)xeSE1bY%O8IiNc7CS?>@RaH&)V4=|%i6C~c1NYx4 z@X=z4fWZkZgJ%UDE1}vML6RpDwIcsck0sfUX>5}E1zxn~Mds*sL`E`F$bUh_7E0{4 zaCKnLVdWru-v=`U<+>kbjr9}$H=K#7p0O&hqF;0tQF=mFV>qpHyus@@bzlzKHKOJg!vx&~!5cw#) zj1#8&Tbj1eiS9I@YnP5iOU6%carHrEzRi7cBe2$}1km1H`{&_&e-NXYRFa4>VoJ(6 zvyd5COrE^m?rTvIUMRZpHOj(V!@q3aoW9fBK|Ut=N^GN}BZrKPIp?-zajL0ZVa{FX zjve($N05V< zQ!U;Zq8EPCUqt&DMO*3UP`3sqT?rxBB7ZxdviQe9wL6uwnTb@A5kBx_W`n}E!p^e0 zr5w+M5DK<97<`<|+qGj8)Ad*8oK92btpy$Vi9|dSNDdu7e1%qzMjT?jRf$(-p{F~+ z01{(Bj{_R8u|d1$vK?*PgTA6(lBgYstgCD?i;KR6FlqFC5{JOi&mfA>ul4z5M>K2m z_GBdsWAw25kkzNq9{ga8K(=kVSv5;i_kZ(`YSg=VY*49mM7&U3QnH~0sgLMldGEo`fSMz0j-g^`xv*=xgq}D%}))A}{q}b^KCJ3Sh`fy!y`|qDJI%~*( zPsd5lMmk^626d#Dk*S!l<3V+Q_nq5k7l-7_yu3i*Q(CkRy?QNL*IZeD%IpcTT+!hu z?To#BGZ>xL_mPkQhD+KR8HXnO#ZeMrMLR<@ySjAPIK0D#ZoIZmu4}MD8R(Jqa%E6X zsb?)p>3We0Pe$H9I_JJGcbF^PHRWX)7wv)4q7*usyd zEYS}Kh>ALJpblaur)UPAVMI>h*>*9yiSUd2qvLUTJZSHmM+jhO5D3MDMdXi3qp&0Hx zd>}w?m4inc)r%%5lHrw{zbY!6cG$B8vgp%Ah(nz={J`_8zxQETj%r)|lL{MG@)h_o zE)&b}X~|l2nZ#Ng!IJ_z(GkYF6tAp03{OjV~4XC;DQC>agCxb zvwFJl+|3^cVRzbO0dJ5$ODBN!9!t(B@8`2&m2lxcyXe)NvPu|sz|Op{${V-Upo)_~ z1x^=pGQ(YTpGY6Bw9KH9(r`FB?r&nnHgYwg>750?C;~nDAFL0L@(5g`&&6jf>VBNW zTYGo2U7r(pwVA2Wj-VsqtS-b+4amkA23u&Shb)JoK|;EU8lSqx+%TMgI&OL!!=ryM*ZYCHLk!5w!ms~{FaJS=RBFbd>0vUvC zG6~gcPlroA&Jq_P+F)oC=kDoRWIxPPZ_)(W|I*~v1C71~yl`9S?jHE`<6bRKe;C+J zxHXA1Y zQZ%u<>Y;+w9yhNx?TU$Oz}URThn-p`NLHLB0`oiHilIDhDROU-SeWG!r)P9MO}>!Rqu8rM4Y%qV>NwkymYG&-xH! zChB0Qa++OgGt4mB>)Km33y^V-IEv#t7^6wRUGazDolddA`Cq?&O#&|#8G~7$h~NPC zvssTHtDSgw!*7n9wFB{E&!chsQ9VxT((orv9~nae{EMaF6&GKAL7Vv4csi8HlUsmt zim{a`K*;*cqNm)fO@F=q6--BFlSuq8t4*YNy@{`rM5X_$@3d>KT1O}rgtzi3Qp{TU z63|gi)VveniQ>xZ);VMV{Sy;5tU?MjgN|@(nTkWM>`SU1)=k4qRseFi;N{P~8qg{B zArE6^JQ3adNiRUM;KYEN9O^hR&#q9vSr;l*_%k*`b2r*enHr!>5~wgyMR-W8lcS^K zunE3+2@hlyq?WjG@Mq*6&MM%#T~AFQ9d|v}r9)SB$p?g1NYm|!#C!GT{&iTFx^3H< zLsOIVGTXX%KFy*}_4m1?)LOsi$a-|fO}nNV+I&H~tFh&}=X0BlFpu+$U+Y{lEuKLB(6ga3*;ntdJN?eWwpaK@+h=H^@1sI(ult@*tT)r}5MP=g|LinHaH{qO0? zFs_%26P_0bAoW8yU~TZ7=60jQeWl+XLF4`O=_;H5GLXuodkz?UAxZC+Tv{}<)^yVw z(JoB6x%&u2Fras6mmra&4izFy#yq+LCLrg{wR&3S6Vr%`g}2y54G_Bw{#yhz(mJ}x zDs?|X5*Es_HnGFg3TYB#02CB;?2c{QR9R{Rd%=OX#Ve~@dXXjF?>rLNZQCTBjlQu{ zdGqGY#mWKq+4^3FOG(AdG=W7CAu?U9qjPuJUhYZi_Wuhv- zJM6lw0~927_m>xuCrV+O za4%x+3w1RPv=>sYe2I%Mbn>W~>Fh7OrH!$y=bt!twL zyqXQykPL0E;x_0ih4A4^-WCYIyFiMt@vw-RQCv|_*7;MnI_z!LYk$4|D za%6nP>sPNN6qyiluJVu#1)PvHask+Zb1Y5P@9+oggxt+Z^Y!dffrryisMZvA@foVj~@&~Pys88X~#XZp*9RwzxUX+3&Lgh}_bq~}e!_Jy9ol4JC z3oL|Pq973EuZ+8+vm2fHn~8>Ar8Fh-E~&guG9tU ziCuAyg1fK&Bi_?8DhwxI_e=OVt0d=e#tTQ2NrnX3Pi6ChKFu(gTzu zbzEAzHS+8YHZL+YG7El>eNCQBYR z4S$zj?KDGSc-U#UMn6&Dg%Uu&U+aEWnhZH z4D%11d&rYSFeZ*!Xr>9{$~_T#tU?No*h*cyJxy3jfcu}2v>7j6q~Bg9&(HOlNMLc+ z(y{aASg|Q=z?{V)&zfYDKmS9U%wrS6h@lrWGhmI+D}Y&RJOk)MuUPA;%ARL`9Y9U9lB$S9V&(6)SR3NTM$tF&I& zlFE?UHgj-z(fj+LF-Mz25^VqApsK>DPaZ+rtuZ1CoVc%csJy~LNya8Q?X4$zwM4VT zkd5ybNd&y^JD)Aavi>rKFN zT)RH*%(EmJ5|U(=p|D9Al8T~C$&h&`B=anUN-|_FM2JiYAtZ_H5Yl9fWJ;(=qw@XM zb!R{O{l2ebAMf!z@3W<@`@Gh<)_+=hK?xY9&?|_Qn28x9C}1DRpfH0KDpb&f3o-yA zZy;O0M0&#bhx*juR|1o1y zPn5E(NTi+?N0z-$h3^+*_npP;pPAj`#jMDxT(PU5-ek>c$Ad075*jeaHSFlH zX7Ap;b{|`(BqhBG{QaxV$JVx$D8qi2Y>)3~S3|?5Dr49ha3SWLE2$kSsq-DjeI<+do_3Z!&Po6g?3U(lL97Gj<4CC+Y&-Uq3zexEH-| z#BwOBIVgLPSO2=tRg83S2&~$>>wdW9QGw*H^sB+{aScElij%kTj=0+;FF>IBeq`p` z=Xd=n*NrTWv4WWptiY_5(J?8hMnBV=JgU_?XiVPexz!srEjicOx{pD@J*AR;w*ON% z_c*&7Ha2d*cTQc!X>_HDwpLL1{>KB2k`WLvz7cm}Y4D35Lp=N<8QsigD!N zkp9>6XFZ~OV5V6gllx-44WJbrF>ctLP;&!hsx(Ccrc*!OX0J1|28U$mU-mj+qL1Sl z3xdHJGZZFH&f`$gX=KEqYiPgA`uo8rGcL5$*Ed&sRr7Z<(Syn2ZGjiL>A1bIqBLDJ zn-mAhx1=K!#Wwtr1v@WO-wW}f;kyHW=@fhyLO994C8o2e`|oeTr_V!qijJ8^uv&Yq z%O{0Iy~mqviYh}^D5JdGu8>D2OX~?JfJ)judzf8}x<|9s4jna?9VB0{-5g>L+&c_zKH)&v!ZOK5TtyZ?c)mlpiGKL&@Rbg z{c3h&@X_4vzYZ<{&{DYJ4MV+;@{PQiYpq?@=?a%Bj-jC;UTH&T&N6G?yXreK0Y?lS zRHByWqILdO(tPqR`D!vRzl}UUD~{!$s>Z9&X7B1#69RIa^Y8_FT@sAm!m}dNp>X+# z6B$HC;n%5>wmB=bsMOR>XI6-T_1wNnq9=VTkCy)na*tI04Zhys*SOH-D`DY#^thtY z6zRJ|M?)R5-oo6{2gr4Y21Ju5`A)b)Sx&}fFI(R46u)_+TUV#o8sckO85@s^h1qN` zo)~tWixOcm04$^Rf*pPMW&94PIBe2R9>0EV7UV=Pfryu0BD3k-d*4{D!X27U3pwQs z7d=Gb{wP|jc%hV3XnCt*x^F0@Z3hm-zpj>NL8k}+XK;z7cIfe2O_e96J(tlWGV=F? zS}x2}s`!+HLlJDXZe4}Y4j#r}yk`i508#<)a?AC{AjqQY!7fj((R52Co7pH5jb~EQ zNODwIb=epYvFSX+WytNo&826Mq|YVa#y=bh8`E}!WdW$+{9d~<~i~zKi%oSmXzwZl0lO&te7Jq#%xp)c>{}gPL3vUaO5ub3B zEXt|H*%{bsMPn2VvWe#P@3PC&ZbZ`@arOXzax6k&&V4xBW9^AW5-&)BA&_;DouowJ zwc}b)Oh^F7GEDs+0RxBBBqnFYLjW+snWT|vm*5UW2P5oez6v~;6sRq=y}i=L4}7HC z*?8=GIkHK|EIJHWwF^UiE|h13C=XTV7;Up$`<<1PswAG2>E`G8g{WLNAfSGdfb9^VC zR47yQqIVYkmq#A-T=m$gAzUSFfQ(P?Y7OM))x>^u=_|k!vZOE?A8e!o%rj|ve);mH>#~N@M46l013n16Pi&vKI|%^6(6g#4@OUoT zR6&M?O(M~>9G#M=Q`D6=#mP5eLZ{4GcxT`v#Y)Gky{07HMgxMmRa%VdV2>!*wDyO!T4AvE9` zr{P=gumF>BW~R!( z#QErWH(+m(9V7H8dbZC?O|5(k6)7%_xmB?$*mn~x+K)7|1pS8E+M7VpIS@YKs++tX z$8B3zlyVx&^!U~in!_E=?A)pt6k~ zZGL4}vH*b-JXG56;HcS!GggSWATzT+N@JCz@j4 zHjxvn#-u8L#w1o~0T!UaxoryO&gCM?SdP*9(WiEGZ5O#IeT8l@8zjhU2sSb|YC`;~ zG!$awgdM!I+}+)b!Oc@tN6d%<_-h|vuI5zc{7iK^f|vR9m{Frd-2xjclRjl#m!1Qp zOu}!PadF=Xwh5S+6@C^5pI6V?h3IdzflYi)@X{IivVZ5uxpTv3^F9PcL^MI5MFrJ? z`a_k3*{8uQpj5VK06~<66k`DwdcPKG7!F_#!GCuQm8!-l5l- zySdP9=9Sv^++vy;efDU-!nP6LHgfv@Np!nwtvt$6i0fSU&H1FQv+QwIu4&D8XKsF} z*Ftnp`mL&0M8+=bZU~7CGe!DCpc7GCv;n!=jTGPqrjKgY$L32q`$(c$gzFP|ZKFm* zig{2q@i()+Fn3M;AgkF3ibNL0H@f3U-Qq60Q}R}x-DD_8H{$a(0v7#tWwHXfY4M{@ zYL))|q=rq5Y@gEHWa%1=ohL?)|2ELvXaeA&EY<;1Ymz|VvnS9E@NfT((~O#N`9|Np zJGfgQXHEk!ad%h6W!vWe&{P_cdr^ifuW)7ZOYBCE-Uy1y))3#`bTZAM#JNWXBhdfw zC{W$q{kL}=#U^MJFXJ`M8LV1M-WUCn*T2p}oul!TEI&r`!W--|bl${d2JS}cU^K;y zG-iES5m=ZSRki_@;=ScJ+eW_3F;?$=U`tjPZ|C(46heWZH9quCHy(9=4cfSroo~`7JUX3Bkfdh3E(&htLjb-B?g8R#~-Enxz8=I!$GzO=6~g!LErHTRbwC zP&q*Iu0c20al_0od?Q57!}p6brVTWFUZNO{X9k^l?Y*N;><+!lQvg{Vl?hGIcF5p z8hUEkF={U{Ipxs4w)VYBaeQ)oIo}H-%M@%AP6aH^=}%w-cLmW*(vAjeBmI%>nIsf} zAZ^dvwR0yM1gAn0Uf*M3WW}2<*|Sjf=N(QF?aCe9vpZ6opm~2^cqXx?G?R8E9_!XA zz-Y^slV5g&w9EjinsN>AE&M+%&%-OLY+;8uE!r@}Z*P%)ImJT!kZ z)O?2@c}{}!z@0?DpYQ#XM|Rh_bv#1wVhL z(>VC~BPoS|^D4S54UC~5^HM~h$9G24&&xN;$dovA!lQbwMla5F++U)&(w5afcvT@$ z42}9ctv0CB;)_oQyL#DDQ$8hIKdAX(V>R=&jIjz=kP+$&^CO#Q(aA4<)7E_#*;Oog#yPx5vM{nC9TI?L)3(o>Y+vE1C;Kfz-`nfG8a=rv_23Ye1D zYMh@-bEkB4y0P+Q&(Z2NYYw2xn`YV%VK7Ry?q{7)HO7ts9}p!FE+LIoP&~W@k_)oR zO}i28zQjhnfST?W%zzn`+vC@o%Krx71f_DFu>4wf3^JfY?z1gucGAZsw9cfQ2uR9) z+l^dd4+45_T~JYhJ7T>E$-v$T-~(CXsUn1|W5G_+d5-|B?w}Hc0(e+d#z#M58!(^T zgSd=z%vS5rxA>Jo1Bp;6pyym9r%?NmmV*a-9&J{?z8=9m{xzA%a@mgcme^K5kLwkb zs9?u4GPHlZsF`pSO4#FPR%!AV!+#&IT<{+5QsgUxXOjqnmO@uow;#7UJ4EcCe8bg?Ix{dZ zkYS0SvmFSnLH?|C#m!ZlTi+lIKlrh!S&g~A5em;TL&%>K__CZ!m6t#p!CrAFaBSZpO3E1Y^O%(;b{}!$qQ7hxG zo0e1t6Kv;fGZLlfGf`FVKZjPuXfb4htRL5?wy44yjFF5u2rH&r zS{$50M~d?jwrhCEmHmj}oqJ0_p2+C|B>74RK*}Bq944E9A!^n)U1Ry1$(l zFX<|Y;%i4W35{VW+eNYVw?+p=T>`~vz08#%Q_(=2%$!>o`4J?DH!zy0&;d8fpzw1@ zkh99ytg;3T%p_=vm9G6q*uLIY>-fR^djy6BM~oP@BCJj8#*T)jKLmODUiqaR+e^K? z%0`Y~(ndkNCU#}j%QzS`o@50xvt-x7Cr?03* zb9@~q#&Yp6bovddN01ong38Zk#3AW#`HGFtZfYXb_U)j9X558D-F)+k=?20uk|)F3 zFWGoF=s+a=x}yFTcN$_Vnd}H#{znNAOOuc**=;AORGyRT+B4Pno29QrTS)%8^ zkch|)@YNzkIZX?bE?6oLp(uN|=Q6?>V1TN_#ia1k*JHHAQD_}&O0a0uukSub36iOYbFUN=R>I>(()fKNBmEO74q0f@QKw7cDv}W1gp0CmixLC?Ha)2f z7Ekv12vfEHOHVKeNnIDqtK+g|e}TBiMlhs|AXFk6D8{06!jWVb>H5Xxvu(4Cd_jGnbQ9TFJD z)WJO2E81JR-h*3|3rfLb+ji_|K4;EXudb4x$i8YU1sico=0+kZOlUywfFwmC@LC7(j1CD6F6RFyxD2a3VsQviN3K z*VY=_cm7{5k}GL*xakC@8}FXkANKFx-cwtPN?i9T$B-6RQBID8f{(sv*ymt=ok;v~35}ON@43ED!#)k08Y@{gd{+K{Vs45C6)tPp z^>I_HpR5aWPm7-MxGtaRFEm z2qm&Tji`vdRiu}D?SaJ&oA;O8i+ltv`ns3tASW3-efs-acTqBM56b2nw66zlE;N7V z%_t~~9KR5>&22m-68VR4pOV-FKOO7Y z^G3%JhPu4%RBeu@_Ar-l$0wKmq=+JuWm1h`@MO?C-c4BagRqLS5C(hA>J27Z2Hm- zLrD9XT}EZbh(FX#ra6CGWC*~j_~g{osPiD%`x>m+wRD}}qb1<}2-JtS-zE4cZ`Jp| zW*7GT9G3F(^scyrUn@*uzi90Ofrcvv@Sxs;G&Imvq@=wh8xtCg6XKiTPDUi@t z%IM$iBPvm#H4>7}nT+>%{;mJ!mqDIh&Yv~ZI@U`4hZfN>5~;zwh)pK)QVeJV9n71FH5PBlbv2CtPajK9MMxcCmgKFb#hZgr(`;@Aix ziay$)vyR3x6S(U~rX{r@Jof0sz96lVkux38a4^iUq}yEqvPPTkU5yi6Cmjqsny}Yt zwjwJU;r-%kEP{+pVqpsnotxAZvhXILxj zXHYmqC@D8Ry7pGFS6{i(J02#4hId;u`XF6NF5%_THo;rbJVG-v^j4Hx^j(*3Z!Y(p zB@`i$EgDVPOKs9%3+->sqE+9f!W97%n@;Jx8W#Xy+Bx7G$4^+>|V zJ+N+gr{xyCNot$}%Mnt-7uH65~ z+idY%2W4)$R#T~K9cug>)v{Gmqk`wfih|THpv4ED;ZCdd4T3TU)vL^}Aio6$gwwd7 z=stPr%Tq!KWvJ_!5k4_0G*q3>*{{+fq+2kc62+i>Bd)w~+8BcE(Dx`IA$W;z&7=PF zncZUd!JF)CGYm^!W%tWnb3`#zW*MXGo)PmEiVh6`L&*>?-Hy$PkC`^wrQ0LzNkj6-$m$i3o?cmAq zzB#ub1bwlhBV~dh8hNd5^=_>Na!l1?X`1`5j27iFC;F>sUJIoLiubNf8_%eWHji^+ zoVZ!0A#qLYkFwvi=$)pBaL5@sy@YKiLNOsQ7FQsT`zW*u5<4+aMSm^JPDsP7hXz%5 z#>4&XruO!C7P}3`YeD`U#iI474c&`mKB54;EtsVH1Z*P=S9cbdb4uH7i7KgtEg)j> z^QTXHW#;?tb5*6-Hvj1`lfDh+KpiL~q>aA*B)&aWHRJmt@7NvGm1oCSeQGv84q21Q z$n0&k{tnEaJY>Jw>rkZu9bw`t%T2r{K0Dt;d*tFeH$S*V5qyW^$*#sf#x8v5n*B>1bUkMF9;`&81 z%dPi@x^Jk13d)CRPB;whI@xzLw*O*c68;#Z01BS1la*(xfubEKwOVywBe8POE=3xv@ zLX|A?3rpce&5go%FD0*z{p@rFLV{8mY0p~4_SUJ35#b1m0G@(e&Rl5OAC=}jvL#ri zMBtIWuY`rtD@V+Y%KG%_CK`qHyd>{f771*lq=n{P1>gA3;3R_Dz{Jek{KT?tPG%TP zFME>~)HeIIs4)E;iJ2^)e0$-MX$-+6f}N)@T1i-_G^k+z+YRaZxw$kOSvR~4t%^>+T|DH`uxA`ovrt~yguPbQ);!-vw=X5T4B|Kk zuC?99PaW6Sp5Q;$6E)GaaVN<}J!){@hDkjRqSx9)GdE(;7dC%ue7INOob;6 zKDOlXYYzJV$+HxM$hFQMa<1yiV}3{t>} zWFs?<)Hr3bH#h|gvcLDDnv%|y&#U_8z>>87>=C(Lb%hNjz!DTe(ttX!!)nHiV5JLP z!Y#mgZT8bN0*L?|A1fXW2X+hEPO=`D3Rub=i69^|xP!_;qE7iMrbqfQ%}8CUjr*#c)da^}b|#fZ zLvp|7>5f-&zy1^XJzxOsa_rl2@+ojNMcu@M2m>o_2wowlUY+p+31ntkUyz_ zps}r=3h>eqw?-N3Jw0t8Hc-pD&NDe8K$CKHqpN%OjF|}Bid2v4#}oewGNT=0r)1*` z@fW>ft$E^d9Eq;vFb?5lbsa|1xIOYC#Nl}s@ zFnxWux-FI4qbbdpY%9)T)|ZPPb$+W@K~JGrD5|kg7QIqgsN%edQxGHYIoH5ehk#j_ zXo)};k*S$k;EFn%chJMdIS!fhP735)>r=^ZDR{B!X1~@jd~aF=?k0j8qzQLM0W)Bt9&k{(BF@0BNgT~=maNFT z(#_zzj$k;85gC#iBOT6&piDz_CQKOJeqV}=Ts8FI519G4WaSQCvgGp7<_Jf(nif|l z0s+lVy*SB%yB?xm>EZU0Nkk1#zL^{rFoIXyf7UuN?+0#tP>D zvPSIt>hmuiANdsXzVIa+x5LMef83CPoL-^ZVOE-Uua~g1W%Y`(d4nnfpl#(uKgo@W zG3S;|J@08BU{okpxg-!dMz>=vgEIfY`6@vTTuAgy?NU_h4yqtn0Cbw4W};zJ zlTVZ9FKz{fA3cTiV0=uc%BM$-X?}~@c6HU&sR-)$M9FKpzfB_0f-hi{7fsLX%TOH%=W=|#)nMHby{=)UPgjU!nvMTQ-bV~T8Pt3y=TRp1LHroJE&&4L8);3J93 zD>k3M{At%UR}Z$7vS}qdmL}g-`|x?`2hnFHEWD+KSe|a{`DSgc51z$uClYJJO+Im~ z#Rvgxbu4=v5<86_UH>?(gshl@OKcu)M&6v12{7?Cy`DXV^AubSc=Wa- zrb~&@K4!1*OOMLupv}vpUOF31;hZbtp(Y8ZYN)Gw>Q;YSIt=M641u`6(kAg28GT^V z{*Rx|=r6`z8LGuvg^`$xF;{j#-dzmVEi8F?STfK&j^SkEV ztu(2EE=8-oR>A1jL1D-2LvqL`RJ>}y4M($qq9k*gcCs+L59jkgMkU;#VG|u4YXXt5 zsfuV3gIfTF!Z~t_xcr8f%Q5@?!%P^qH}bHJ4h*ue;8L?pIf(43Wt%pGMVQ2k3ezqa zBB6eYWq)L1AS~v~z~-w%ix&}t0X8`7e5h{Di;4Ytd33@owm7%XY_>4yTwL+@wPdok zoqb-<;AX9|i7#V7C%l}mzWLH$F52a-?$2<}nU$*cMTOMI4Lwr--0Y1OV(~~kh=Gu` z)m(u>(}c0j9P9`XUZD6e2h8Zjw+87LKEclV?VBqTvX3#1Jpeiohy3Gf!GCVvT$SXV zr%n1JRYJ6E8ckP~*`baAk7w=YgE~{aoE;JBh+~0u0YnhrE*87t*UPWp9@%QTj;J1H zOB2F&L`BXZub(0bjYPyy5tNzR>)jisEaHr)xwUK8-u!7y`&|x~W;fa0acsq7^IZ(r zJ>-1L8I^{e!4mjWxzztcStWWBnnJshMx7kj3oJgVntvYL^<*P$ne9?oV5*_1*~+Oc zPfNRzw(r9Dq!vAUGd^>AeoIEOl~sSlAL#HU+*6b(fo5QOL>(WZydzKs=oCiaKLTZA z8gj`(V^)`*o9V63l+b-VL+K+$lR(W3oJs#`Q-RVz*BwJlpG+!o#pAiDX@fWmb(tvN zX}{{)?bqWmjG{uhe?#+tFIydH-uS516P}D2IkGY~1==LrqVREm=m10Mvilr1q0KT@ zX}lc-$SEs!Viy`6q`^_`l_41m1&=J^5utNgn*63+j|^|S4Sj(kYDUpQmH%h&y#?jR za(2%iJ?`y|?Or>o(W``n+}O}$6$ltjP>vXYCH)H%R$5`W_d85NBzSsOX!Gd1eo)-Z ze=qc#3#!OC9$e$wIwjR%?(H$hZk}{A+rKQO4awd?dmNj5tLC(8fSkDMjM@9^Y=9I6{3Y_0J&}W1c zgU79t9qZG&`95!<5;+VDDELPI0bAj>G72zL*-ym{{x^kQlrl_RgYzWJfUPaiirw^E zE$@!ilUBLm3&Z7RGN+9QkQjM@hH|s9(IQ-WQ@%bA7ePgH;^nApd*eUY-DPx;K?_5P zD2;{M(bn$$hZX8fdyZaDk|-!z$j8#Xx-yc z{NyF&(U%T0ow;BN=tTvtLy_Zj-SxLT*^2=I0mx8M^7i_W+wIrHyqp!up)Nw**VSoE z0uu$ih5N6r7ahMg{afF^cJsxx51svZr(`)3c-2LH2QK9nkLdj;_onfiW(1T|AEz!D zuyXjbJDyF=jUw(;TRPdcdYcTE9Q6POC%cA!=sdKt0BckSP;}3Dis#JPaR2@CGpX4m z2aXxj2tM_;Kkiu=Q!k$#J(T13Pa?gO?2lt_vqxg)%JeH>%VMbM-Ftb(MBe@D(rT4& zkV_y3KZOt-*&+9an)o)jyLmT~pAiWOqj%>y1C8iknOw&ym$s1xCtCKqzP!3-nuUol04j!xv7JBK_sZ)I8yEdcX^aY0(&=B37U3NelQtogLur#CF(N@bB z(8J@LXea_C`iI1Ck<_N`$49W0@;?HPB+a=6?cf+8j83k@Sg zgTS9ubNy8ewPwF^Dv6~&Fi>4l$l0!nQD4<;bz3wZONE9ij}aJ~Its8zVg%5pl=s&1ITQke{33)w#O!q@uV9flY2rcimX*r|j)u}Cw=nDtmTPze{wk>;OEuR(Fk$s}n*PG;QwNnIwDYq&+NO77hq z4xSIwWIZY}SjPA9NZ$LQs37z7wj*kP$iOSx$+J%|$_EyrJ+t^;_G_y+B&bHLxvyc{ zqbmrw!YuN*JWo%!))1xn$MJgpez=KRGu3Pa zYDgP_2F8EJ#uj-~{|#j@T7;Aj){UD=*0)f6$vNJxGd_PXoLn>#8<>Kvm`$h6m@(i~ zvh8eVdFzAP5lK=}ss&@cEI*XMnkZ~g{3`Nuu@Ot|Z-&%v$3&O9$FBV@F>jznl57xg zxBu^;^O?P8tiIVim%V%hQSMHs ze=;4zZx-_JE}J{kUskHAnk4<7T8)9a#oPJRK21HPmonAdk3>PF7ce)3s+&%H{4HH0aKWQd$?;Bm-60 zGdj*-Hy93Cc%7cryMPdf%wqP83CHd~k@_zBhNrVHX{@5-L!i^&+4)56IImcXKSBIu z*V0(Px!rRI0J2SYZC{Hg&A*mXhWLu?O~ac`4VwJz3 z(i%#`%Ad(&93qz+i^a9%?@&G@ky-=uM4_il9vmXvYh#^qwkkW{Yy z3x=7WC+&$raPXKhd)~cS!KZ?|SPi23+jpuQ)vIN2_r^S+H?O-fwrb-vgL3R`NtNc#6Y%MVaj;dd=c;{qIsv{ zzg0O=9gD72{B^0WD>KFQM>dJ;o_)9(d1c^9;*V$&>+Ek*dTfUb;lqRyiq2YGHWKv1F&M zXG0B0UlKeALxK$CEPC+rd8J(I&h{?6&BB&Xm{5AFtBR2mwp`-6$z4czCkQhYzp>n0 zaKg8Xvwd`46MI1gy4PsiKLf?fz*Tr<*^gj{Q)JsP3UyiVVW^VfnczUmx0p3LPrTmt z+3EZxeE%W4JN9SRM^rP-vR5v6jT>fzuvf&wVuYvcNdmm{asodk)B}B6H<#ZKq|&Nk z+gNQiB0ve_->H*7rQ1Hqp;7HJaFE+Q-??D4^4fj+2_T|R$J@2523d@h(Eek`K0A>e z0rbPGv|e~6fB{Aw*#)yzy`T@DETZ=A?3G{ndnJZ%fQhCg7k3MW{*>%KJ39?#B_RDr zt9Hx@*r8TIP0h-@|41=#8MMP!!J-VwY?6hy*n#r1fd`q`v_qb_kntXX#*(W~sCK zdNb{xH@isbr`AIR5WuhzP@TOK!$cI1SHje+d7E{zuKRFA#8|}w*SYhzVZZm68|uDn z_yQj&TR~YxKmL1t8+||#pI}qv4EN}{W?ii5m!Y-at&r(6sNLp1(}Y1wa5AV~F~wqL z?x^XFs_u7{pt0z|?IwL4+Ns5)dK$}g2^1p*;z&Z<=0V?oPtAT97}UF9Rc_Q6^?p5n zJdHPL*S2o$b=_@yS=gA{4BZ^J<=dAF3$jMPx>``?mX%Ue%k|Z`MXMtUk{+72>DtG; zt3mtNzczJ$sOzz=`lbhu>lk%^_4rj@aoM)HDRDtvT`D(>DH#=hGXKZ2=A)zHY$t`K zEI&B7!65bokw1EW3zwMJ+pGyuAU21A+^FdG@rj9=WIu|oRNME-@BEyc#);VgIDCB? z%^T-g@|$G&!K4Ay_ecD3iPGfoOesqRReq~J2e{n#(JxZq}Z@^^@=r!@l0%4gud;sy;PdGPMT65W zG>?OX!~@`WCcVb5@3uF=F$?zx2Kx>^Fwi-y_b!Tao~My|uiO(0s@L*}+*w}Jx0Q^r zmqHU{b~AOL@6YXNyn3x#qy#d~}E+!^lQnE8My}3fBvQVV^_vhp4RX!RV{FXm%FAjV0TcEz+PbMVk zZ_+F0{U*2AccZ>B0HVhwRcl*toL*o?v zgIC&qizEGE=jp$J)7LSn$qqBy<0Tb;KuzJZY)iTr(d!6Iy8;8ANFHBJNs#(92uMec zfwIzzP$$+8zt}61Z&n!f;kzlSn4+R~+vIPaXrQlOS-8=8B(rcCj7!1GC0h-pkw%IC zwj?)d>bn`C&;T`Cx6~`1UMU|fTn4NYD9G;5G^Y4hEx6|x2}7k4IGx=ic4vv!?Yw8m z$-q8KWbGp%lf%0-i|mZnfhHq5-R+;x{2M)xs!c2;p0}=iMyvy`V%OlgtZzzoyw)HMo_918?zBZ@ z;c?68b?Y{Ne7E{G7yBQtmFie^ryN;}v*ERM<6c*qsJ-h6s6Q{d`s22$5UAj-Mg0$+ z__9SPiy7HfTgZqAG1#O>gvn%rjC%we6F+Q^!~b^3 zAo9CiSbT~4S^iPtnab zeUpvn^wj?p!|ba=wBTuUyHGEv#?F2FyaD{~8?<#I(F4Tiv2&qyM;S(+K;bZ_^W#iS zDr0BGWKbO@ot0uE8Lqo;?@X&Eb`vX&#z-Eg97KwYpdv^tAuxRar`wrzA=4+_ei& zaL$INh6D$Rp^{llyq@LKuRnNKU5{w-zoCYcFz55}rK?BbDHU7xgsTm>D%KxwliG~t z)PFW29M&SOktmDu9EhK0*z}N0BX2rByHl@Xb)}-#tvkAH4GJb|G(krtfc<>I585Cl zMId<-Jyr(u$p8oY{##l26TWhEZTehh(|$5{k zr1&aR`xdK8Od{6qx>lA)=)r|Y9;Ms&uB#E>?}nq+rP$(}JFIXmj`?3vWi|DmqPiol zGC7d^Lp#KqoZ9<0^9Xen%1^iCfvUEO6h*Z!!d^ zHKF9hkH?NLb@i>RW&pir;31*-7I|&_W_y0_BIgvwKe8Y#y94VJ>oc28bN>u{iS zR8OLwtI2!@M?9Tc16zqeY3p-V=gCHgdk0e=P801s!HU>)4U%C$5dfT=)Sjhu#}I(3 za?KZH)lyfNML!z#>j(Ag7UYdMUtUdWxboOB#L4j{@9v39ICvStUi)~LaOQH$=#hMI z=H}f!F=9ejg@GCnAr^#{ohz~4pn?q|0ulOwUz+_AO+dy7kN-Gcb)z-nZZ-#r7HpT3 zdWe;RDcJ;(@hZcg*Q%Qqs+pDMX`&RFzDl}mXen%_F@c%pM^CV91kecRaL7qf>{5gU zexOX8nbdo^7AWof0H4`i8lRh)7(B-GF8*rvxOzgihKwewQ+}I1eO5d;FJ6WRni?7v zKpnd&Isss#KeSQLazwH*elK+1yUDec5NgT2Fr2-FA((tkke;prW=A z3bjm*ynY?h;3&dAJ+_89o>@G!JOI+)k&HS{%e<_VPAs&Da@$W@-{{j0Yk6$WMl1WS z_xfbh2{$1Ki|i`6|K9I2neobY8(u>=u_~JN<8>Gu1}$7@1if_ZIn@^^r0N0NMMJL*BTFMrqgV+jji!q_<85{4)Hsib_Tpe)CUREX2#>5D6%+mnx zf7X}_1Z*w?ke(EVXs|v6VgtOTD>UMXkeSsP&sYwPP)xd;wR$im>v1^ugZguzErJ zkS0qMD@236?EPjD!3gPH2`-<-NXM^uc2sv;i?kgL2Wgb&?YOYbRus3>G9*oiHM^|2 zm^EO2eVd(GXGVX!Z)KjivHBYeAFV!`zy3Y1!xR$A`N5zP=df<$x2_v~iUz^CYFypa zR?j~sJE03uwt0}7p{~-$`AH8X+JJpSVHlpUw4%`QBWQ#Y@b#LhiHZLC~XI`3bzP$0BlAubTR;>@9dB7)07gZP4AM_xdln zn=QC&i;IhoUP`_faX{X3b8FdJ%+Wqa>;di)DXG*cW$mYUu0_Eh659?Pd>+|y@DODm z#&iYZAXNz7}m0g3!V+s{LMv96JUW*$E!VF&k$1I^sG zM3sJ>EA5BmPBUON=cIuio`3#NJOOe$^{ZDT^|dxmGRZDY5%vr2|I5n=P3=0D2kGfe z=k`J!BX)yh#|E8*Fnx;}1Ge$~^-Wg)Aa-NGB>wt*G5r`<&G+XYe_(z@OevI)Wq~Fw zJ5rxqvMRKeN3I?^pSqgxKnO~ZzK~b!>kEii>c`1ULqHJLm!AEK1qtNx`0R5tDe|>w zkTCjhbD@j5+9L{C9UtN}ClFNe89?XOh_^OM^RK3}^!ZfxqJDr4N3Y)9vb#bxz3UML z5@wb#@>b7z@+_UlckKS@@#ByERz+pJ_rUcz{Pfm_L+Y%I!yaltG&HNZtMeII8rwCH zG)ScxDDMhmDxYuSO%55&sEF3@^*G!- zZc-QBK7E=}GB;Qs^S`;COLYijYgyiLvvf@2J4(lYLp;ps4CBngi?)|DE6YE2N=$TBdqt1B`aaVpm->~(U$yt!9v>G)d64r2;gI&Iuo=Ug*iaZijtP5#swLjt z)!e)l@C53LH*{#OO}nXy%tZE;b=Dcu-FD%^i>GCUlh6?N1#qyi=sfY)itN3j@z+SW zZFeeJTzV2*0#+(R*2==ib6Od{*<+JJJrwN#z!4Qza$5JdkvZSoXx-(@Jw;BBeCG`p z02^24H%+AVHjMgJpRzlUyZvJm0hGJZY!&_p{Jj~b^erdoyeTLvFi8b^Y*`9sVq((3 zI^_EHSxM-4=Pj6uc@&upHk&uJ{#+#10Unk5k$@?YPh7e;6B9>7?=&~s0Vgd4uGp#{ zzj)Chxdm7s7XjiJ$LNJd{y;3me8;(vsg3ENH0I-Gx?T$EJc!4T6a-lCD{nCZA@^}F zRD663TI=hhM6%2Z;aQ@DoJ_=a%{BpP{D54x(^5CX9MU?E=$vHK2mpdQA5xkXnI%u1 z@9Ube2-t8strZV7$66CcXv>&)nIhKBFkNxy;iE4Q`FSt31HYBiF6@I`2lW z!lSiDA20lFsH<1(k>~HUI_*TE8?s4*uKnx5{}3hPj=#`KmY)NPlW_@OmI*4b{LBv` zzpyTee=mO@bU^u8P0z@P@hq0V!h8gt9W@!kk;%L+Z7Q3!Me)&hMUT**q;6TF4V9!l zrqI!s3Wu_sS=2?cjfg9`rh&eH6Sy(aO3#cbq6HGRn@kV;Uh;cXU|UCb+d$JgI9^@c zNokURZ(?s?%Zw$?j4Rgu8X5-FdJ2#uwe<<^c#(3V7h95*$ToOX>qJ*AGdwUuADTfH zW`6s-2lK3v(MY91HQEhcaC%O-qn0Yg+}v;qogIvlqyezNk98z16~E~Vc1D9JOZu?{ zs6KdLK55dM%5Au)IPEWiw?z~sqnOYq?6iBampO~-JcWp65h>u>sA#3M0f@APIp7T@ zyk;$auCeYoT)4!-2!CAf>DghY{n#ki@5QG$YB#D4G2fBGSjQwT6avpEWFk~9o-u8~ z0g=^cF&%W@2f`!r3@ZCA*ULkGa@h#u&ua`GP&%5kjV5k-IEE^yp!dFy=ofs+$fhB; zmcCTdu(;`0Wc*MHD<0kLv_mpmu_NM6*8LicT6N&e!wD_aqG<)BN6^vVWn9OGOd4-V za&eu??Yae5fbBGspc=W?Q`g3>KK`UD)cT--QI*11)zvRtRz74++-{Mnrku}JMb4w1 zHqY{D;j&yTQlHe9_ep7F>bOh(oL5;rim8gaw3cF;m{>_QS*gyb5+xBa$j%&6gb#@6 zsdMsA#Ur&41PftLza!~ir^u%h6j!cvkp-S_7T#F;wm<6##61pC+*xms`O!5zmtRGX z++&5urG$l`^-{D3Ojp7^H5h7XB)@Y@(w8AycDTT!;>is8aOFSQ7N z4Y>#dCY|5XVHw7PrukYttN0fU)xW1y08~+_^azMQV`yTc!~1P+Y3Wrlj%;jTbi`AZ zUpZ?ivD+$(6=R?LQ)krRYdw_178Dl^2+Xu3XOQkkw5Mao*4T6ZeUB6d!txwwm^xod zKBUEfV{eOVD(`C4CMYmKM*c+2@i$gB;R-*$8kxEznb4r{-a1@g+BMZdXk>F4r~=Cp z-ERT>i4ryIffg=pBm1i5D8=FK-co*CUGIj<4%g$z)FPgk~U@b}BU@4UNxPg^Jt z%E#wb?3`!kH!Mh<=?FlOI(a4ue4uxCdr91uSeK!WinROz=iC|--f)cgO`*vHI<{ZN746DbU-rk zA|XF19+dU#SrKX}s~Nf1T9omoadx(`hy_`$Cc)^A^-d4vp!qAAywjPvN}ianD6!f~9=r%IgEXXKdV_mCNCzVtsG^xmS~- zN6AyduJ}H~Qw|hqVz=GPASvT)%Ztw@ULV~H&Gh3LO)aMHY1Xd)L2D&XH)zZuyWVQb zX>som^Rvr~{U$icpn^ zG?6%Ogsrq%6gg2{nUi#zQOI9jRK^77+H9l9K$arwjm+)PWTAXPEC&W)i@HE>c@u_E zu_y2y?#TRom5%@znIT=)%`?~A%w@)tCwr@c216At0eWVFR9_XKAn5zeAv6#FPwn|1cf(vx_G=o=-|O)yd}jp?@lceblsu!AW` z&>N4D;%9}PIdcbOa|>BLQwa>Ld2GK!tJmA#f4FLHG>VHo1Mibq?-O3V>XeL@F*kXP zQ@V$`sU#1~ui))$F^=-fOvY7x%pX+|$!579&3-q*pj0zBAVTIkaJR8@Xd1EKw)pW# z@~|;S3MdvLYjq?SNGw1XmQcW!cjBB<(6+sLggff3PQko}_3EiHp&=2r4Ci`D8`!Lk zVadm<;3|^MweCqho70o+%=>rv=u$u7f!mQqYH@3u5C2itn4_0)^!UGLuzmlVFXr7S z48B@zcXaw5_X>zH)-MeS>-*5t@N<2Qgw15Mqf#J^qwR~?sjkWH4j1E-} zV>8S$vc72^*9Mf0Wy+HDTUGeoCcz`FJlo&M7Wwl!ZUSab%WyRG5^p#BRI&nz`jZH;%^GKUq1OYCmitNWR0N%jl$aZ_Ix~NiBsx&61F~Hx|VYVK6Kq5KGg7V`5BF`eb7Z~ zaht;Qtdp*_>-t#VZ5h<=!H`aq#;j{$lHC5-5t$hsc?>cm(|?TARIn_;l^~iPzj{^b z_;GQ>_$R5qcXLydJI-vQO3Y_R+%4}>%BRsvgb{}h^}Ez7+LDf*+YM<%6{Hvv3ZYRt zng1W*1KfC$$#6QGs!9uUQ{sm0bBn4bI3zk7$qM(06AzB0vjT6l7(IITx;RdLa8Kw1 z;tCH*qH-CSBreIHOByvfoVo|W*o(Hu!o#&xsjbSfJdUJ9pzMQY?%z_51N4w}N*?5b z=pI3K>}KSL{ioqDSK{UI_Gl>?`BMjlrrv)JiF#OH`BjWY%id@Hm<0KVM6hE=L_XR| zHW~AmNT&Hf^uUt>w3M!;0uniiS_&iH5MQ2M{)1oGHAbB{dp1_IHNaAZg@tVu;&V0Y zs#*b1VmKBFBfiV&rDUI-2o1dpy2Cj?JN#a5^0Q}|qceWh*7J9Prp}FP6Zi&4+@8|D zVojFy@^~RE93B0oNXFWSj4an>c`*ru;G$k2_8bn~`YesX;%d!tIcgFD$ZnD=P*3PFozzH<$z^TWfWs6Kg^{SHHLU+yKp{Wv6DxsWh8gKRBY|2F4(7}+HFiyA}FQcvkmva-vHC$CXN^*kH#jF{81ax@*U?8j_7Z!n5_b}Uyk>Ao1FK*fg zT#4up*(3tL-EZ{hhO#coKBPf;;QcaGUT=2{Gwn~a!Ip-~WE5mI8l(YrF9ET_OfawD zl5ejhk*(Gu`9x&Y!e0|!F?|)%by0qR8+j6XIqX_dtyZC9Aot;k4iDF6(0B-bNY5BU zqbJNwGGHXy2qdKhs}2D_qUyUyJ)_>Xv`hH=jEBx(2%cq z)W7YfPoF;{_2WeD-tz6+x76O3A#-lzU&a+LsiOR&=>L$))F#BE;cF${AmM1sO?oIo zh1uJ^dpB!D^73v25Qt6m)6ZhzGR>#5{zUYdG>d!+G7Y|_Clt%B;+OdT-M&lfggn4m zelsti4#$(MQ-F`Fg%VxSj3nfJ=MF+=nZMytox~?k1dhL4Cv4`I6C>MGs)DG?C;P4} ztjC?`X%W+I0Q4ywfM7S&o+J68>^%k{dB38KdFZ5fQ)L?a*_HY|GOV8g8nUSKpkl;OhTOHGt4*3O7La9tkBZnq(oUTm0)Q(IhON}M* zJ@kuKP=keIV>PowNrv;=K{i@C7Hk^~=rY&n?0jk4ld5}fh!lZW$W&JKH+70lr)gS9-e z(Y1Uy_3aMx_e+@JvSmdlN&%xK=K@!qL64`kcM&(YZe7I?T0kitD=rO@r0`3Z-WKL& zxCV4o6KxhwK*VFA@arK6?5$JC6Ckg)j}5&?8uV040!OUUwztL;RV zBB2%_xSX0vaNspX)35htk8uKbsf6-VB#%Jaq~Hxzctav8@6rfsS^ss3Gcx__p+mYG z{Kzh>!`>VONhZwX0+X0PWR3L%Qk3!I!3 z%?{{4cp<8*tM8d`*BFeK>~#+4U}BtZlzi($eym-|;ItS6f7j0?I$g#(%~fwsTvJf; zXY*B2mFC@OUOsI%`Xf=A(Hu)x*L&4o4eV@`x3#R^cCSyDr>wmAwDJ3xL1%d(8hl`n z&C{n2Bw|@h+5<>VId;xwf@NnGsH>|p^{@pr?lY6vADV|^xy#zPJYFopL0i@HyVBxa z+cBN)Crq`jvpS5sNb3^J{c%r93>C=4Bxh&Kld!M?VzgM5r@@#-Yi?93ST!kS8`^GC zQ^WKkEC_CJJ3-MEksJ&Q4xz2s;V-X7S0K?!oSw(dz4}fMl00d=geZON&k*QL8b{4w zdZKYi{scnISAud{p`K=~oj8_g*tAp^`%x2aRcT52 zU&@1WUd3J{ivi9{Hgz$KL-%&c?GJciS)Ua@m8#VhiLyw?PfIQyfuLdn6k<|+Ye?i+ ze^L*^k*y0%V6>Q#65}EPzDO<%vxr{tU`(Q^jcBN@VZ-2 z>3Y`z;e^OM~v99Zf3ABW5SaxUS;BUX9~CC%fXw2&-9s5uq4deYldGR zgQjm+9?$PCvwcwJ<_`VjTW{AhYG0v`=E`}6Gu5BZj*&^ns^ie(u@5Udofx$;gnf(7 z-DsGr*R5;e=GKMxt4%T?;ncrLilS~Zk)rIcJ)T(P_%S;xYaqXxA_hWvOcShfsi-db zE13=C!)zCu?skY{LS)B)xpT8?-4(4bF@f-3?}J}~LYadCN=TNh{lTDUABpaneAw2U zrMvO&-m$}3+J$;oRH7lHu`I*wJ?(G)ShN!-pp)mGN51ntLcG&HbBM72UU) z9MEeB*(h=2KlynDZ@`u>4x$rh&MZ=x9)iZeZLTiawIk*Ba5%6N5n#v14ck?*_oG&b(vV9?otIaMK7av|Xbo*lgvn0La214+%9QwS8l ze7T@<8+~o}PL=8=74#9w=g||ztE)Ly)2UvN(q1L0vxt5r#nQ{N>qHl=v>|zSFDJkH z6n1UrmVxFwwng1FG;4D`Ni~uu%zBc!P{b&}VIvnrjAuOwm;6Sh0o}-LAMoaPH|&Ij zv|%val}rQuN1#MY(_SmIwfr^CD}hpEdSC+^sTMbD-IbLqR6C)cLx=Gq=^jb&l1c%* zao*P*4O$K%aFu}7O}Vjm*0MB;m*^2xUh7`D?olB1Pe-W5XvHZVdS%#pFZ`axrmh6I&7d^nmCoDdL#jRh8E@`YzFi#c)UNZxck z*Dr6QNa|1j zvfQoilV0Cgg#dPxKSl z;PG5ZIHxge0?iONUeP5JjGM0jMRQp}x{(CNH@{mRB=cZVBZ?Q7^_RaqM^Pe*+n$|C^W5%HT@S#4Luy64sa$i;a*gtg-*!C=++v80z??NR?v_TiN zOtkb2K|oAKmuQ^b@a{<^{eMAn3ZyZ zp4<*6)NyFyEBV~P^8{UlUeTa)-EK7y(M>{SAx2Oam?H=LfY}2z5_V3z{9+yus-hB& z0%_$n*w{3vzVBL;8Dp>Uc&X>tV?~lw6-g4U-?2;au^YN5)WM9a$X>4hc-Ti4#30YJ z8#z+Hnke?3hR`&H@rrW;iI=Mw?4QZopm+O(E1$#EGiGireP4-+ zwdQ!=#d*&?(?3Qp$&ean(A4)y1^EwN{|{sD9oKW;#(&!@A(`1lc2;C%R76B2*=2-; zBs-E(6lI0TNMA`woFZ8n83{?0?8+)56;hvvuE{rltgI3MTZah(^c&-eW~ z-pBD;$A&A(D-DwBZVvo)9g(s+R{qHibDdPJHe4y3j!VHomp8SHnmZ^JwQS{gnS>6s z(@Cx1iqDfzs1?+}k?ohE0}PSR0c#aT5t$?JAEiHF`T|2WZ5@}^!lKLGQ z;5QY%uWQ#9QG>a#1jSPs_>HD*qiGFZ&MHdu)8w~@i@-BA6^F)^D+8kw8mzNxRP}yB z!k(CMMkb5yOz+NCs8|G*g1wV@)EUzs(t{1+_K2_vHyWutEr&>skrIr8!I&mZDkubU zy3N2EOIYuCQpN@%+u=X;hj0jkY+{^A77jVn!tfI zy|kpb@6xOt3v5@89a~qJbFq_Cozcjs_75k-51#0{lx3Ruu(crVFgV>)1{uhzKhEF^The+ggECp!5x|LvAx0LGG zbS^bDw4CPU#k1(1l>CG;w&?0_>A50Z`Q6fskQrTJ2)!h2!LgF0rqL!33c86kU+Y+9`6%LT@b<+sCvURP{iC})}lANHuNxoOqe=_v)r-G;SW8zd>-^t*2Hd^xVS{R1j*+z z+ULr0taRc5;+HC(^#J9*Q+|4M%@q+b06eDmv1u?&q@R(I^?@5i34-q=bBf&ONj`58 zxD(~Wg5}Dxn+b7|iRIOXd$`vn@s4~#iT@>FMVa&io^#Jg79-WJ1sa#gtDLarhOP}@ zG=%Q~mTh46+i|$lXRYSVPg>NMcwW}&{M$hVuICMw9iZYy;^O69Pa3URzy1co?(2%- zE@4%F5}BA(fA{*-+gs&DHt>;a*X(TUAc-h;4b1XqN&^l^fQ3Xag_VEXS{iO433d06 z2&zxk3cBVX5O{GWy(8l7?WGOxuw>N&%CRbnF8%xI!ylMb8dO5fM%gfP$Ke{OL3b0@ z=euqG-PO-c#=ZV$lu6iZi%&%7_?hjWU9Yo`Yv}kZhXv*c7_Dzs9f1L=^YK>#tK;uT zU+PwRzq)49xv1WEs^_isTOBKg^NyycQ*{OnYRMp{ptWUjC$KR1ys4*7e%ozjGP-!N zDe~b+S0&zY<5%FSw`ws`Lwc>_>EnhY$M&YBfobu0q}7_l5pI)7aaAUK(~T0VtSryfR(4NsX#Wv0P=SFE90Tpx>KQO@?Q z*>dFB1UE)}o&O%juxANnl;BLJJy1m zbM?z4hfVti=p+zWcApT!YXotZcoQr?A`n_1EDwp0pF7AtBxy;$!Q;nGpW2}eMW`mi zA-<m6R9z-qOl@7dID@Do2x*%cFP@D zh%{bd)!|bpo|CEOo*=A~y7uUyK`&*|A?bcX;NVv11psaLI`a97CI;*yL@_4>e1#7c z6JOYCMcwcr6UD1pqMA<5fiKJt7|n4AtgytexBFBvbplqyM;c`LgBs@$FJreOqL5&wGGdchEvCMoYou6up+W6REX`fv~%+ z_xRagLQhy=`>EyLn;F-^8QHjX%DJbusGK82Ip;A6O2|bDtyen6faA>4dQ!#PjyT(d1WRAUp%%F8iS0$v)_+zuuV#!(}zyRFc@TgdJf`e2l81NHmp~3Cos2) zQmV|4U@se3+1-dWN3k5CKKx*0z63~|)Q$6-#*q7qHZWrPiFkp?Sk^vl_3FzfTT-!6 z_BbS8TQU4ixaar}jE>{UElPAkK{}s2dTA?cTlH=S3h2TIMd}2%E9aJyrYL3vN$J z18p@wxkVXn@42(t!50gr)>BOGUbfp=@T?*RxDI>~%TmPG=3W!8K7+1gz9r2mPtlWV@%fL{xL#i=mGRT0-IT zFHY4HohQtQWY_*JE^auwQ~NF2g4*{!HGil{FYn{W+ly?NYMTy08F>|tfux|MoESg# zCAzfHNbLFDqW?@0g(i%nEDpMo`C|ZeKs=u~iDgY9vza1A{^dxqk+qNRr3LOnVttnF zHgJ?Q84X68QWX5qkuWR8D$_olCqKrq=_fu3bAu9=@yaTqY(*+!&i8%v^*uzAf>Yh5 zE}8=n-M_Kr+TdDeHKvVeL(ei(ox$BP@;uJ0Iv`sl*2q6J#3jpjEbM=La_ z>%oDmtd@6si`Y}y#ey*|I-!bpR?0#~Od+yff;j_a!y_Ho5XOPheG1qi0x%SQmPJ5c zxlgAoYSXXZsttZg6Xod86(>8!wK$ zf>XZ`$6XtuJVo*y0VA2NsQ2TPB!|fn|GDuWqBI7YL7LJ?CV7OKJOrQqX>`g8hqUaA zaPinwy>SjNo+S$~v;%&DtOGYlXeJR(t>`I9ZQAXNLF*Rw;KjtVa0v(kam7ZL-#!Eh z1U@>_;Uz3fcdP z@<8+Pb@{bdG)kP*yquENCsMy;5NCiwJZ#RHi8vMeI<4~7H5(b1e!n_k^7M8sHa=^T zeN>Vf=s`>KI<}L8WIlPTk^6J|ZrA1w$!Ke$f4Gmys!^pe6g`cOx48*OSVc>|+mYM^ zaD&b8^8~Q`e6cnEW%hEa>BX2GY*8iw#fxo{B0#TQ4Gpfdi((F`!Vx9AX-Kz7Rh4>l$aVOg|^;bIetlXNOj z!>|%SVHxikFX>PZ3O5ViKB+4|TewYxRfnWj{%o88pyC!88HG%eeq|P4yUgt!-w8U8 znxrVgVFgK@+yo^bt)O@eDBxyvUqL2*e)L?(^~IbXP70Ysq7NT3x@3+QfqQ_p&79c61+ZJRg!JW@1VSDskqCh?s5O>-AERi0*YBhSxA>aSDt ztt{)r#yymNV6*eMn)0L%(zBwbV_C|{H)&~g6}NmBhUOMWor3dMbH6;o2#r!O>|;ho z#f>*wwHCx|BqjuzDbhPvSNmdM z%W?{;VT}Lma{6gdL3>rxQAl#&J$Pz;SA08o@-zl)AZpopB#f+5R#7Sl0wk$vcr~qx zsk}_Mv5FJH;371=EN_ksustn-0mr`{OpSiys>VEw*>Q&><``4BLH1RImkm-roCfgR zcB^eC@}r|oEw8AN=U+RS(%QZ(so!fqn@&JoUcV`lU6|bdK1ePn^dAfUB&Ge=-PJEl z##t_lZn50IY??~=RzL4t+_v9CZOhnv?F#=QP%)!jz3OzT(KqdxB;;rrPno%9ht`A5 zRpn&mBUcpZ14n(Y^^*RF_=eY&>7MF@r~H|mDSC_r3zojjZ4c8#dK3R}H2F6QWchcy?|qOyZ-$}hRa(80=%CP^=oYO~ufXFv`1r?od1?7B*W}%9EWs7U z;DM+|Oot3v=aM0BkQErT5jNfAwR|0rskB4Q+H?1cZHa9S*7F@5H=xncOK#f}=LoIV zJ6OKROQ^JEl^=lu69%OcM#2v?YuC;ZBO6ZpmIGA%lq)o+u^%a&Fklz*acTvykssfe!w8w9%|>;+2e4-Q9u97TI_>q z?YVvAgrw)sy}?I72F-c{f&Gi-lZm5tN4}a4V>q9i+A3|2#fg%Bs37u}F3Y*9DC0aJt1z6utOpLQD`QEt3h9io-xg>=hGKe)^J5rSZrq=7r&tsymDTpy|nuIMNN9SwRz%Kgv z8T*+2&4K1(7yABUPdU@(^D&-^>_}9qU}F~-J|JQUT@ss8jYX5lfBW#thB{mGE%F{} z{!C6HC9(O}<(HE)T`tNvo}8TA8rQz0j0R*!|JWk&_zp&c138}o28&waRgwd^`DU9G z`o&ya9y7Rv4)!uN(bR_aBUX^*>DxvwI8#q<5=Qb$i*{Ld~DBekH zy+>^`$lk%uBm`>YC5F90f3kF%)ZUcZKkaV8Ak#NbbOT%l=7;7ixv~cNXV9s{^=mh) zi2>7+u`MF1q-$%?+s`>rYM4*%Cf@DFGvLmD4R^yC6+pMs!S_;vbO3xyF~ z^w23FGtrb z{9tP;%g1NUYB*xJ$x#BRIZKK>n*o5o#ggAIRg=}5ENuhnPhm(l=q#TVcEU_hc@pIW zmFKskvA;#d=t16DpXHd|R^+_7jhaZ7*rQm1^B#Kat#*oX#Ea2g0#L}&K|FPx9&*ON zI#Qxe(f)}>N|rU`_uzA=f>IKSo=snP3tGbO*cUTB6|ym2GXFqD7sa~xzIxERZCm@; zK?gXoqehIl9$50TeRI)kMhi>uZ<2tgXGak?OQ@4TH?V(_?IX)-;>edp>UC^-{}1S8 zf+8cWY|tm)p&@8!rvD3lk0jzC?Fy5no;25GkzzauRs8VdPlOy3no#mZl*=mM4P$Mt zNFfZrAI=#~tebE!m)BNZyDD>u^lx?Rx@Rxp9=rcSVki0^dgO4SYaes-6JC>??IK8A z7_L((^sAPU&Au@Tn`F|fWzH>u1Cw^w$HWiKu!oWa1865NW%Ff z1zl8*F6vXsHbv49EW5!;j*X2^pv27{5Ay-l2>I1C${o&cB7uWG(;hkN?nl$Y60g<^ zjlS`o%Tb?p$vVT{ME^C&kLabrgV3JiRiY|^Jl#nfX5xqAr~z`NU|En=uEb{%C0lw& zk|Q0WL^|>f&m#qdnTvWNy(Wm)k2Gh z32D5`?*e9nco8%sK_j+s;g@iP%K#?SlA*iIc`} z(T78dVa1=-Y7U#mRgz&-2A(bs3e zQ%xpLeAcRetDTRncm5r)jl@evsWzQQA6Eu{RnxKc>{iSjrxc2jN0vA@8Jzs{NZhxp zUWOXxgGyRwmo|gbdeF|vQdu}PCpSjSSORh(rs0s}oalT)g1ELQ$nw0ON8)PSJo@!n=A)Dk9Eqht} zTQoGV96av5nxkdZJ4===`L=cWuAJP!1f*l6L5a)$LAqQ?T4|AcN)r`EDN19O^_Hhiu4e)=d=)}HUgfwrm{+534R=R;A7FI0ZBs%^@OUJ0Z zAJdt+7e%{Qu25sp`rL`p6EgUU;}Y#YR%`ui>zBt;DLZ;km?bZtlN|{RDpRg;s-P=nc|D9( z)%v@3A!;uZ-6049I?-cu+dFQbN@53+R8Tt-OC%Erpr`6Cjihj`i~HZGQ^3V2_4DVyY3woGD(AH2}9-ZCT~b z^goyeQeYs-y9Z?@DB$VB(de0|&cfUxz;l^3i^E(~vTha2QsUuZt5#h+dAeLwD>D)| z#kTtgZNYNqf5~@^E~r3(XpBVlKe+*Y`k2XrJIM_YIVhByB7R!WLJoocB`%*LW>w6s z59%X3^%{&eKrtZE!m>X!yc;{Hw<6$$A4PAGa(|4I3|`9E1*8MlY5?4>o9_>`P~yjyM1`Lbth5kflw^^+msoP zdhYi^CugCidBYQ-pNdv=W{t&(tmd&NYJ9U3oGG}ituIH2lJ?=~_!&71+}|7K7#o*v zH8|8x`@^k5hcmryG`4lTT6>h7WE(OXiKc`0n5}t==}lD6Y;VHhiR9TYUjOr*-Phvg zoK?bY>PLWi)vUQ*6eV=A^{UJ`c@@|R+rc5LJl#gi4zNRXa}0taCx}0sR(j(9E+$QA4lJh#^@tzg?~^PVHJq?;|XY>>yiA{>+wUy1lh(J@o`L}z;U>Cnr> zpo1-_UvQU^HE##7D@0$IXSuu*NB3Z zh=FO^Wt$r&Po@wQykL=MzkXp;^ z9}955^T3uu{+$~xG*d8W^Lb?==bN`f)($*ZyJL&ZUTbH~_@{lJVZVn& zyfv;MC(8RsiX#ebZyFg%w*`lhXj}2epFR5{>S5O{+DNP5@%0Y;J@(`ils(0xnv&J9 zj)v1|6J{}!cPHq_ftqgJ++4TKmbYqg3211^U7%48yuMU^gDG$xheI58iHG;(8?OD^ zgu$7gC&TEb$!YG$N_3(AufGt`Er5Ex1`UFhDL#qb;uX737{U~nI{{_###1%=%~-LC z84o$KcxA0ulM#VEuLknEsok{$`oPrg^!GRTyt+D7P3RXe^GfkvS=*VdcP(@#qxG(5 zp%C^M-P#KbT29z&#O$*PAI6Lh4#3I^$p$NfXfOFjz077zk_0B0P`Td`*+~`OrNC3Z zhC}Oh=Vhdyl=K=d`(el`7l@z{WFDQA1V*Ad9l}Pw-Lwu4bEGe&q2=4mJny{_cb7C6 zB2Hn%Zgw`Y`9<&fv{CyGF%?~c;ZmG+5Jx00-}}SH0kIjuTsc2%q2Ws&N(V9J#cglp zpF$4tHpWsmI(FKGiz;FC&nn0>sjBv?fuMLE2{08GgW0U z(_`U71zl!?su8S}Le%zl8)j;H{{_=$la?(HYos2pAQ6*@v|dLXCKHE&aNF}tpDQbh zFRiw%XI^cCU5zS>1|OS6YRpK(<&E)zbkDljO*x!aE89_RxbP1TZ#R9^xw^4SwWd00 zUEdmgv>hp874CX}zoV+xZOm0^oUHay2Z6UJJ=|u;E-6rKGp_fq&uANf?3!_;gK&K9Aw?9g!B}#mltw6=zmFi6ol%z6gW^p_0k!6X%`*Cq+skxod_{ z#!MUXh3|QiXs&X=3?)^&ojryG1uj zJZQOY)XUbKra%x>Vea?trNLkniu>~UGh=Rb1WqyvF}>yfc=xe_JZXJH6r&<|E_YTl z3{?dUYSs&HCROn*PG)8}L#WfjHG!PW=HjA30V?Z$QK41@x@@=|h(tDzp&@TTp458Q ztu}K%EDk19r|;zB9Tm?J&4Vn~J8-q~wFM6h`P)(37|VHQY~O$}OB1G*^Ec?#WcbLo0VtgBRxb&RYV<4PZ)b0*BYFXKf`(A zn%|AtIR3**?U_c-*1mRi%Y~&m@NC$pc7L5csVo|-oJ<01J-y6{{{;EN_65lWIYC=g z#_X3-p#X+P|JncjWE|`;v|lbeX;GMjr=WzpiNa(~^82OS?-v;?hyxMb1IedhZ|I|; zl$3=U0uopB4p|4fED0g!AH4$35lsmW8s)Mno{4{7Qf;WwrrC8NN;rnZs&$v^U5$5) zbxAn)^#l`*TXUm{c0fO&px)l9Nv8MZ{QsJhEtYS7!|Obm@ZX(qWP+Pp{&pn+yD z#K(jrQ+}yZkHQOyd60Gu34z4cBMMmXB-1Xxpx_+?oSZJ%7eph3O76Xw%XRuB19DhxF*_!cp(N#sJn!=5DNpUt$YRmN zB>W`X?MiBI7tPe22d-dS+)9JO+euvY%Z0tZVNQOwppS_7h%*WDU`nTjlEaBj#wr4Q zAkDqhu*}UgI&fJL>3!Qlv?p~XZw=C7e21MbwCGY7h@uGV!#PI;&d$%e5ScgRx(n7v zKFsue|LNz`c$p+gBP^sFX+@Z;p2s}BbLY;y(t-JOxH7iB{!Bl|jq_rk`{CTj53iScIYr<+O*edaTrM?Ad|jGbV?W2@y+;=Fg5T$ zIiL%>JY)P2m&1P&u9DvSA3D^vYo~&628_4l54}1`PH6+3E!vzqF+LWx$nVUvgJ~gR z^njY`u^JR%2BpKQh<|ZiK?$!zo9tkAA~f{6Y#UJ|c^K@tu(UJY@LfYiev zn^&B;NCH$6dt>;bSVXoQ&iK*93J3I7V+oZJ%0&9)bx84^uihm|?d(M%ycltM4RCrw z$nTl!NiA@gEHUUJn8ImCF%jdn(=2o-N;@rnF0vyFPbviz34Qu$_u|fi`@S6AM2VBj zO&=L`soEWMZv+w_5xc> zAp%Z(_~CWupud52#TjPDrF@R=)7~}vX6}#lwa556CT8_+rM=m&sKx_yH*K2McYG?D zBbx#8=Xgw7+wxh<%*tc)zl`H}$U7SjiDR_y|Qix(8p$?ip4qL=DAuwfl z4B9-w$weKPV2|ce6wjRP&|;J{;+Y2{xL&b%5V}D3-~R40k6TXx&1cR%I<S6(W7A4YR!qtcEd3n_JDS#(v}qrL zay(@@$ET(DKJ&mO#S`0iL$l!~z19=@g$N|${Gcxqzel&>Y31unCH(vHT5+@H+e)96 za)c_E{v2B>IkZ*%Iwpe0%f1ou8KJ`~WFLbDp~rOO)azF;o+XK;zXuc4Y?UopmSWo< z-P;W(K?NTD&KI=mg{MpNuXSqd(42(=I)Gw2(MKP`rJ*c{%RgJ1Q(AltALi~Ch)-#s zGcK&vO`3=S>UQJCi-uly_7$Qi~n#!ZLU{e>Dq^y21533hoYOH7T}r|XQY3^ z?8+tKhWg7Kc&t3(=D+=$D^?WDKlJj0%SgMTDKP#2C$!;#=PvmwUSPLGSJwk1*l>I1 zs*JuU0j1#*W%1_fIi~V4z5nuME9%yF@MNaqfL+o5kdnzy@}G_v|JRxL?VB7i9BHk{ z5kyxjA#fPsJ4iV9oGsWLaDi46UDU5cRB-Uc0n616Jt`c9&Jzn=GFd#s@ph8X+7O+m z-+rhhUJ03gqCG<^$k6fZ(z4FH5)DR}n!HmIMf&U?O=cz;1hwQsRt+|j1Ta2P#Zc?t zsLXBjiLy5ntAxxZTu6LZKVvhq#qqdcfy`{A!Z)Hzq_}yjD*`*mA0q~QYnBSf@(Sg#=75!~Q-$+67~ zAIh-?n`On1^rW~4$t=!O>h^(nWqzE{A?dv)cq+KiTZ)|0eM)>9Sm^yC=_Pwjw6dzN zoZV&1dK1B8-DwZaij;Q#cUk21;OpT*8)@^HSVX@<6 z@#^_zn&!H78n^Z^5xpFh!Y#1!uNfLG9pC)m`4c)Yb(kDSuaKCPgX=cmr|I=%nOkVi ziViI{J52Vx?sf= zJd9W{EXR%0NkLmxRz`&hy0q|;w66Eacg6I3gBg)9PpX?_-LmWO+rw(*v zXZ>K&)ABjUaIOcxDiFBjwb_8!6&9{U@tI_wf?tnL9v9ICnRljRF5;320yc8cM)78Z zxS@@ZJ%~AQU35AqBciSUnBz;rpD*YmbL&5DE2LzPUs&FZkO?-%}MalBto#Wz5Vk`BO~1o{xDv=5}~p zkS-0KIa6B#KMC>t(v3cl)oH>PgodW2eTByu_q0x(IxW9k;2`Zn?+;NfxmM2;Uw|H= z&?~(z9uqP|KUAupbgmH*wJhU#%k*iL08oL#?Un>IlSf_kNz7qvTkNeEVxvY@DnA6= z`l<*dD&Xm$sTZw5^O`!PUG-H`$mH4V-pxh?lmH``3%3vziW*!(afsUGLyI(2L+s6~ zs+xY{%nwRs!rt>R*zc#7~Z7STYlR7JGX!VATr^rKA z86Xt!1s5JgGxT z$>v+$N!AB;`Xcfq@*h0Lv}_m9i84h_gD7!E$^up{XO_4Rl@49M1)fBn^IrrKN8hwU z(*eSLn^#%V--Z|k=rPE-x?WnaQ^kR*U8hd${rh6C*MAO3`3|cvu$apiauZ^iJaDdr z4(ZA5%1F)45xHWasO!tT%XFeT!;222;cHDzolve`Li$^;io_b>EJREy$|~T6M`>v} ziof%JM9Vm$B|=^rAupEJA+qE|h;~369&|;^6%pv^tAA=O`S{=l^u)8f{2PB{k9d;naV2{K zH?md2E95LXO>&Gy{IBpCV54_9u(b$5oqlP;6UI+T{&VEBfc%#|eH~KXv#$~0UvVP; zdi96KRm=mSg*u|BsnA>qi*!|MmozjJrR3bbx(K}MpmTkd|ujrc0OMo zWxy(b{(LW(XN2i%0s|{-~GQuf4X`sKnLTBkJUe4}^N=bF@4UG|9xWW~IH& z^1Z$%yvaN`)4u~TCO01!;*B|HW1J5rIhZi0P`dAr zY)2V(-vNw*?@Aqb0&^W3eUvj@@k;Tpm))k`+CfT|xAzS?b;Wf93$d9_5dD6K#=PPo zd&sqskda>}-$qkaMW#%$uoT%-V!i|)xiIzGm2PA!%04lqM4o;W+U|g&+ z+qxs4)K7v*#GoM%f!JB0Zj%{0NeJD9PhZEtAa&tYRfjF&s6+=c&D$$00ZcNT(8n8` z4Ew}7BBeuw5C7hfR?JF|E#vdygJg7xn4QG8$(U79h$j7j_{(kwkrkseA*g}bFHhxga6Bx0wl^xutdc&}ZSdjz7j`98O=N&y+4*U8S9OE^()6knn3lHa-{dhQ zpJk`COtln5E>F}$)^z$naLWRp&+x%gkk}5sui0r{jhGNgbimxemFJ}Sz@48&i_cc1 z{vKl;=>`L~m6c3ckisWC$qwB+=UO(PLikwF3b1ynI-@M3WrejDS1d&p2?#hY!=5)F zMFuSnu3+x}uk|v6wjSy6fxi2680+ANFZau>(1yVrXb;9;KXG znITw`NNfa4IzaSiabc=Djv&1c5ZV#vXVoVnE8v12_RG53SE^Jf#1M}N$(UvF{u4VZ8@5S@9 zwU%I3K|{(Q=P+;H&d>Z^eW8MvgZR&L|Hb=qIU)sKcngVApi6D=oeM}yu%BKEv%X|r zL!8fjT;i_Kf$hnE7T)p&Y9|Tak@R|cRw%|vMX7Ez7YqyEeuu9#b=Jsnwp=;?oFK!~yKKN*gkU4l%?Vjj%pk%@c+*2cY{ zz!?->)FaKB-7jKwsL8wO2%06tkx)aEyupXPZ4l29^&~NvzgKikT7Xw%_`(kAZ<~df+=lVy}G+ z(Z4KjKt|^Gx^rO)P6+Yk-JnNcuV--Z&aI#Eg2Ps^i)BO*-<`=%Q{OcutbYAB4LtY? zbA@acpje4dnR8BW-=|E!JaAQg36hxbJXmdNxc8>21jS*mx$=}PoHWc94T^*nw-#2 zmxpI^G-*1iM54v#@y>RWv;p?}5FdQ&+D@v}|7Rf*+ZkcU%N8OXT}_tQf^78)2q|t% zSrl`ORHnH&Ns-Q(O3(#`Q)~>fDnPM{?bHU>MAD50^4Mf%Ev!7Wrk~%7)>P8z#4p~s-yl<#*G zBd2)_bO~qxBMB_fYYYNgE^q=rW|aa2Hlhhh{0}es{kv?N=Lk~C>k-)b!T%$6Cy7 zU7P8`sw<)lF*Y^!D=K|keBg_BgjM+vZ%9@50Q( z&p#bHclPY%oJDERSFI>Gl*_u_Zx6mLFIYN%es~L?h2I+K=vJvZ)S}AJkv2P*>^$+g zVDpph#TH=u#uo;sj`u&9(=R_+-N>{3L3ZbjJ%;Urb@h?^*W^suwr!iBVRw9*-b`)s zc>*7iv$HcRn$B9>$N$06ZwnYJ4nn@Tg-T924 zSxNf_uIh=RY-J3?wT6az$6B71D3SZR&!{Ap0;X{^kz4QRvu2J7>;iHlZg0ZdwJj&} zH8V{0{AYnF%QBxW}wasWVw+ zZM=P|o+)CnH}0iwNx!cMT*q}s+;m&T9$nd7&tHtt+)_sJ!Sho+_SCuqkz_%X!#Vtt zJr=&uuc>JmLukbU&^fHp)}-~{20UwUnN`Nv`9Oq@XKW1(_n&X6djxI>%jA#;H; zf}*c0D;u;ctz%uEn0=`Gnkmz!P8BOW&ui3fABWt2{JAlQ2X;IWfoBb&S$%E~RnSK! z4bks%7rK58HNa+{&*W`Qr9507-8RKPBHq(T;v=NeY2Y_D_ zCExV#VC(7pbw;Zn2alBXO&$s+J4wJLnjfRpFV4+_Rl2JDXPUBV#~Gm|Ll@u(5Nvx3 zk{*0U`svPid&`?6(dvkYJV6t;+99td9Z7w^9O zGg?;!T{wz7UhL~J>mRId>wVbkJ5%^JgtUF~F}jK6lN}ZHgE^$%@WM)aF>Fax#|E{o z{P`S?Fk?se=R_zp4P5iRChdud5=YNZQ~#LO3$6|vUSlj1Zr`pdLci0!Q5kB@Ah<&) zK`RGz;D5i)9sF+fhP7hYX3(s)(A3oQF92@Ko?o0H*Jgmf^jXqaY!~QQiOa0tZgm!( zC{h<1zTguf94Uf_(=w;{)#T(A6N91d^}k+t5uf+4uR)knW_>&K{PNDGnvhI<>45$Z z`r5wm8Q^=!%Cya{TMq3THrnR0?&ub^L@CKhckRziWEfEAyk0VNteABC!`H91Npk?p zVX=HO&X<#ulkvx3d(Oc(bV@q{0y<3&{b#_%soUV#$(6YN`n6jH;b1IRpio2Bb?*1U z@F$=C!B7l1WX7R#{zN#3m>ZH*y%@?1-xOS}jx>G-h`-iXdN>3YXH36!79d?5&}h@H z$2$avBT1G&WVw(Gx+cBm_?X}?Ju-0K(2cilnIJd%ezuNA#R@Q5>oCU!MWlYo`*u0U zOetW8ZzJm8$(n|9vu4iBeLSk$$bSKTXW8-qmCJ38EceH=GcFgKlfhK3^E<7kbfaxD zshwihC3(dWZF%qV7I7W!vU)GTuM2O~UE6wEYgL@X*Y5AGE%~CH zr3G$-A2%H1cxp9cw28jOZma4t?PzO_Zk|%>WUa-qzAu&^JmIiD?0%BocQty|BnfiQ{P zuxijTeP%(VLZUOzRtAm0Bt|lhb;@ct8F)0t*e5`9J= z=%e0i&V%$KxyA?8M-c!wCbg8rlG28}*Ea8j&2pNjqtAa7&{khvX_3138E*>T2+jQH2DTkz$%BtyBH)c^e6$!cBM zTFNQ)l2~r>(j4i({H2?nER&|wd+&0Q*S%##@WmhT%;s79_w-qz{n+ z9HY!yusoBYy?f7PlXkdXYq&pUVfukNTBqzQ%rw*=@iS@Y(f1Wy<>ac4!&A~vRNg3C zi95VvyeP6S($mxXpJNvu-nrWYN@hIiHNZaA5#Vk=^i|S>G5oV_12*Mm1vYtTtKP6` z^aenqWN27U3$kd<*vE_N*P&@rI$g9-BCJuW(R=mk)mO-0o4T9aeiNdf#1ao=-LlA- zJuh@*7cO2@CuZpk@DNer77q9Ad&)Ji373~gm{;gvxP1H;Z<#Q&K0Urcu|?suFL0sP z-Qnaht}XszcXQ^M8d1o5wA;2diM8hO|2&7TUJ#g|7R(%}cJ9++`LFWJcr1TL`T<(G z153J+BYO6xM@>yu^Hv`_HnAeca(?~%%afOLg+9sjq2G)}i?%bO%;KV*Is)BU==Oo1 zrl!gL$T-24`TD`QGA_z|j!hsb_CtmA_wPM8Qn^hkXyo_XdV7ON*mzE=KUqmf*pBcsu^?)Rm=S6{F{HIy4^l-c+cdBO(-uI zEcgdmV8(|Jw>{tS^z6#FXDA12;uMDKvLw5W-&74brI9I!<-`WSeuF-41hugq^I(-Q zuWJ5CbZ9ZBS&w4R7`IScsmmKjN<=SN!F1yJecPPF?M&}quVKTgFxW$lw9g>BQ{)Qr zy1yW*uKTPzwQG*ku(XqQSlaE$lO3PTojZ3kjAv_7lQ!6!=vzxzaffc*8ovO%hG0=C zJy_Pf4b zHw>gFXFrh!6i0(YvkIrt0wA>&m3-n>?zPW4QdC(IL?^KD&(x_?5taK;#OM|^e5dkM z(N}IxSaP;(J2KrtH_M*dTBlIhOVopu!_R)Y<|3c2wqURi#z_D_HW_Q5=!VaO#=Y{c z8Nu8&ajb;XJ8|*yWyKJEC6b4zV*+bblDm1Zmm`ME;lMTe>osoqu3kRbEpyJws~ZK!(n zb>&V2xk{5N^f-_}{MDxN`?G7;u5OWTG}3r-GCpQ!t24@7&1|o&%A9#-qHwtf2fM%d z_%=T$GCDertF6bX@AkH@_%8Hsfqu+D+6uOJ_R|5@0%OMd{e0K(E4-rc#X}rWpGmE9 zc5&Gy_SumUgtChC5R`p;F`+=B^Ex*<`+7AU*A2pM$YAsm5lmWXNx|={fe!n``7JRf zZ{9dPX-x3}8KD!Yy{5aimX#38AZS4?$V7wV8dcV#*+3)>D*ic6nH|ms7R$3!e%&-? zZE0j=r0gMrmXF)UqQP6Yyfu5wd)&5f|NeE#B|(`_*ab{uMdQj}(d^sH^||ycO;RE2OCpiA=%m0Shb}lvS4w9B5{{I=gq@ zzI{jR{u)21q;M7q0X4JN|7~uZJY`y=&THLcsx=DRSWiQJmP>`f`_t zK37nloAG*vn{$_|Wju5Y)|$LH;)J@PU7u0IB)X1PyKZ`AL%|68DLQ*> zDb|aeoHnrxRXJiFkR(7rUG^RyV19=9Cn!$YA!32ZId<0Cf)JbVqW7!00W-oOXT?Or z7L%3(=AHNR(+n9KxQ$M>&ZVvM`L04&>^5s>b80Fk?5$XGw)~-E*a&}&MBB^H|B=^T z3?NvhXSiewkuu0lt6R*^UAN*<7u7;)U-;+CX?h#qmPP2;WMdJ~nfik6DqD#>MpgRg z6zWo79XhXIwP7ZS5#S>%!yLAJ`FZoe(YmrLxciciqxD7)F^Q?qS(LLk&Bmr$c*;FD z4<41V7lMl@b6rilM43Iky#B1Ce)7)CP%n2sRyMHB<=bb`rZ}THg6gS*SZo*`IZQ8w z&xPgns@9viO;jh;802)sr>;-f2jBKK(Y>S>LRm@PQeFYvYN{E*XU^p9bmNtWSQ*C2 zFB^$dD?-}qyYw3{;9XRz?M;{#3tL;g=f<9#1beo~dh)+zc+jIZueS;s)JaYsm9)w{1_NzmW9<7YfNOIzaEu-AX01V)? zqr;>e5CgEW(GID}$s_h{3Z1(rL~r2h6Ro5R=~Cgu=xMoUGAB2bgIRfK$7L0iMl8tW z=x&|(w2EiGX&tQ?R=myXG}aZhyO zkj~BE?G#9mE%z~V6>MHo|GK(Yeqq$B**@F#QOW+=2};V`)XAsA(J4XT2VB3Foz>Z z!#->}Z(c_bh0Umo4h-UXl9K&n(Lt}nW(Ap|AfbJ^$=Z9BJhq?bXkbFwiB3;6XPn|y z)K<;a@rx&Fe+$J5eL9PHDie}zXj?irYBChZW@H?aEC2m{JPgUE;YWA2?k5+CWsWE_ zKof-9Hb`5eGr4)*6tHfgdXO&9d3Z3s0I7sYVrbFKi2jEj&KW$8?^DF1k{X68v~tt! zUJkd&rjfqt>fi69cuPz}**h-sZ$1uUy8V??ij;BU)iwQ)@pL|s!+v=gBWQw(J}f?P z;J_JzpDjVwq<iY~}ff)-oeAQ&L)M3Z9TO1S3@(oVUmFGCOAa zw{Ln(6Q&V1#-~%aZd<`(GtoZBTGw3$8Gbf#(Ai1G*W=@BazS-Ec<^9bv!|N6zi9zSF^3 z>%)=Q1$HggTzn$i1ic(au=;_4I zsB&47p0x?3OCTsA5ep8zujdn^yH&(0wrKke!EJdcekxz^nO!>8m zFhD&ZVA)trx3;}SI2=8S24+=0in`@D^4wXRLP`zv@|K6~3iT-d+_?;gO{E)Tjgp4? zLhvE1WI>O{j2`V_6eLo1*MX*e8BY=Sa$uX@-A7-x_xXZ55-0%5a^NbzWhU3hj(Gd} zZFN)`pE-JPbmu}pW@UK^Z44sr_Gcoz*?{F=hb&%vCiF0u3;b2Vr&*SX%v+KzWSPKc zKr`3Ky3?u7eDeASX~zGS6kIbExGhHfBhK(y|Yz=l;WvMSxFC;b$gIO`$L6Dqp&U+LuY1@eEYb4aFfy zav5!M%YF}OW#t5t0BR>D;Y}Aijtd!rZkLu2Hs2@)ueM+(@K_iK+KwL4R-L?)Yu=ds9#Bm)*v>#!W(tH+WYcJRny!oT$jYBM|5qo#a5-6ySL zMB({ko7P#_UHfyC1V~dZn49{z@!Cjk9M_~^CpSG&s_|PrmQ!~A&w0c{H8jjxMWT3U z)cNzV2?^%Pl*^BC^MF-YxqeipaKTa_!m%ConZ+SrOhlK|{-eWvjkvP0{#N%iYBSVD zxr5^T!1G8+VX6Qj+$4I8m}ys>LHS}Cn0-Ou(mfm7UdN`h(y1$@vl|AhtuL5aBuWl* zgj8?Bi7nkxjsg!!9)pioZ^zb0>udN*TSa;`nS-|>m`&Um^`Mn+Fq&}Dv}4G=nSfXc zC}Jb7;h?AVuL89&du*B<@$GC}f3SC2#bHqdMuO`^B(~hxauHa%pp(Q0S_pMVjm%R3 zP3b}Yu!YQ+tDL?;H(3NM9Q#rG99Pz=QKMc=TBojEJy{s@b7csx6G8w8-h;yiwzQn=jwf z&^y~()upyjyo`R@2GOhF%)7y-obmN*ml;Od^W@-)5mnx8G0y==^_$mRBdNPJNt7&b zI}#ES5?6wERMM$`{rXk;pikCsuERF)2Egt{Vt8Wad5m$<_CDKSc;*Wm{jRvq^62mx zvMl#x4AuCiNKy`3*mHl|(GYk1(vL(TGaa*#m`f?= zL~EDa{ZSYGL3%Uk;b<`UeBbp8P=0#{Vk*JZr-Qi&{Y!=qP;%y!li}ev@qIJ=Zs(=b zY2A8=>Kwn{7@7ByaizbuXwDH~~2g^*VD5Op~s-}2GNHf@ASa-x0Drj7Kb z)}Xl}m&+`91!`qtcIZNk!`^=%3j)M%z!|@Zca^l68u@54W>F+iJuC}+P<;jQB_I&P zt(P#7G5YjNdsSfyTWG<6h%@(?jP!_SO+tfGCxA<#E2MjRq}Q?Idi(`8;d{EL>Ry5_(misC>)UvW-)vV^n;dzJR z@isu%u&oVw=hI2}Y`LrPKlKf}abvTZRCZt)B{=A74;au)j$Kt9kX*rGLA-&8!{gS{ zD0Z0?Sp~AUaxFRy8`4u#!`GV6lbg#U zCdQEdr^|~}cY^YfIBA}~^1asAuV34a9?9)t6?O})sO%Z?e$(p>-#1}(yl}3kvZK$P zldV{)Oh%hc)MTUMPYhGfbG*v9V6pwcs(Ad#xXEIl%s99CoQ#_j`T3Y%!5%6Me3m(Z z8w|8b}p+s~Ba{_aF8*JGOzb7);A^ zvblM}j=#1JD^KVzyP>_ThPFy0c4k#&LYgpASZNOtv4h3QPa;MGf;q&>Y5ud#p<|VW{=Mj(pM9l5K^9fYn!(v3VuaQrY6SWNd5@hN+kDA`qY*|+)0Fqacd&A4&*>|oLqaeV>MvjOTQLOG!(%uZw!*zCaU(u=p-!H$^yYO){ z@gg%q@uzraizcwqQ`E^2bId&`E(;45tl|KuaK) z=ROxP(hHTr!fJ`vUe6t*LKEv~vXCl@)$_Sev%zZzf<;Ispkx6hPr!~VJEZ_|Gh%Nu zEJ``Y_3`ZTiqlzBzX(Pknn{Xu2Hr4kEG<343gN^zZy#LJ1V+fHm{rZZHgTX{f%7hd z8o<~MVLxT)AbhMYoUP3s4RjOQ2Fl&& za?#aA+8xJ7?GGF%#dh@g15F40I(U31|0DET{K>IrFR?#`Uxp%MSe zcGy?L#d^jG+pR;YH?;4$KY6hnaW98`@WCc1{fN=b`bF%0;GWUC8!ulTP==OQg?t9! z1nK+!LtVqNRj($sMa+N{u~om}6gE^H6t;CZg)4+mgH%B(K4Izo$B*9=ddj+!D5?pE zAGzCSgci3sWPyKZAF!AQ2m2|5_0KEVN~GOKxt0GcCo^-iEGB1)VVEjehCPc6>6!{B zpJh6D18D`EFgJg2WzMHW6aQRVGac##;z8B2v36^1S6kYTq4-c`rcnU;vT14-5-tX8 z%@@?KTMR8_Ll*CC+HCfHkRn0Xnoe8BFlF7T$J)OuWl(|(tE?%jMZB^)3K4~@jar7I z?8{;wiYE9%y;iO2?%A^S5nWlu>)>y2#oFg&BxF~r>9**JUBvT)o%Z#+i?{I`h!;a& z%iR4IX4@RGfJ%}8Ay;0X0gr@MR9I+QaC0KW(LGi~iIxme$y!%dr>7r(V_Q&gq=K9y zm>p-mM?=%oZ)`*^+8~8A5Mi;rg^{y#zs$m8snF-LzkbaH&8=O_(%gY#y3g$$@+k@s z>b8>KXp-!XsZsXHXJ)of=Ezf-Ws=g@@|;3_9=GtNW0VS0_x$6ug)%u2X2^1$*pr}PLe+a?$Pjsdy8Cb^NwCY$}7jw>l?4SR{3@cYC z9pXk#V(T`RrGkM@o*eLq5@ZC9h`*>Eimx<805?JNOagCTn+>aj1!$(opO~0;YO|e( zq08v<@l3h1wXP_*QqRTp&G}!vKsQX_FcBXhClCWT{xSrvPG@&?%uQdDfa+bGxPXJu z-7L%UFuT3VRC$Y?naGKf} z6N=mT-|qo3>*3)sJ?Bjf$67N#^Zk2SWlJhmlirTjftj4s&48@`tokuW?s+&5qZpuQ zY4N#Txxm`7yZ+`dvZp2R9*=3w-dw_Bw_kK6uNqmxt}zkFjAgZh^%7@iWvy7?oE;*X zqfHh0Rx({RNjo0sa%=i2hqq?kx{agANLWl>2H)F=-44nK6~#JYA}s0*7;t&5`vPj4 zG-QT`)2t_SXdrpv^twfc#_nhZ8b<{#PpG6Sv3}{aw|^ymUaOaPGGXMkRqIl1wnG@Fb*c;W2=5+!mMW^N7^^Ytv@hW zHF4x>s4rv66P69)bip4nDG+gMwqwf(Epb1u$z31XhzkxfaD)h458H4%$fdv|Ygr`S z0F}n2w5IxM>6l~(!#S|WGma^pYHDg|RKURUj(|c#y&?S^lRKQ8c%~AaQsq`YUc-0O z5Wc;8xrR_?4F0}m(;{@P znAee%e)CqFz}MlY0G;gvDbDI<^*Kutw?2d;*6G;Mdn>wtB(L*Gz-4O5rW6X+8*^!q z2)|&TIK>AB(Go~+j^uD)=60{pK=_xL!U1s#Q>a>)UMF+4B3Sa;bbwDfojX_+K>s0J zbk@>K?cb!?_sn1c;1KMjp;yV{q`N4XXX84Eu|YKE7#!Lq)05_6G5ztZJ3mnO{LI0n zP04dK8UWWz1HHR@Qm)u@>`MQn3h=TJmZP%E$?yNtOB7zAqWA>Xy8m5oLFWlUS92kA zr34x7>8_Dv-0Jm7<7HD4GrQK)uuS{$s6+0@i4)3XU+Sy>9oBtK#*nh-mfhp#l+A$b z7i<<9&0CV{moq}gY~9BCnuh0Utgta|<3EyMEcq2`%7;Ub$F;BN$XSAmXADB_E&tZ? zB?aB>#6o2CaIHav#lrU1`;3MO$ms&%z2b#8Ygg9|DH-^mCEs&$vi1MN)_Z_s-S+?E zmj;z2qm)^QCK(|@NlC+KP$`m#kz{W|LQ-UBWnHoomwo+Tr@HU& z^E-~e7am-@wNQ2$u*Q1*2b7&nD-TUAX(V%z|y~0d8H~blbr_;?W{CGHCdT-lv^=5do%s zUpA0L4D4k@35%oHVZOlDmvJ$=bGG7* z*3Ro-+l!yPpy1L?KO_TfQi@LMDja{5dHYn0Hx#kGZ%#f7+Rv_CesRr^vCm0XmR%_` zAHcuhYi|gq#()53C%i;sgCel*q$!JVOlV zerP6BW?S3mORb1BI!B0eY?L@s{6${$;wL1C7xmcl!{}4w%lE{eBFzhyVyi(eFCKWr zS*{*?qU^?xONmV0TA2w*w#|Qj{6yr_YcN=@stB zVH)irNi{eQYb!7)aPv824aKPPP>d5l%CXb5QBSx_HXKwTKOYFk4~c}J)^8=Kypup_ zh>wdj`~q<*9Ra4haqCvkyTquWj6@Ol+yv=5fa5;DEE7<8F znuHq%;4I?!%m@wdOj0mjHx!4Kwds{pZF%W7-AP%fhUOAfr(`T{tzxIlx)NRiOF#8 z1I-DzHc<}NE?Brw0zP2Z%EMJ+NU|pPANXIbEH}CmM9FU~+g1Kv?l|BG&ERPZtOgBc z@WF048*BIh-zPRmm8@5mx3Nat#dpSy#$$f;;SXc^5gRUjv0g+C9+c z-9|o(jl~wKt4cqu_{LO~CuX}WE{5FgELYA9M3BSO)Lnddo>fwKM;<4X~pO!<6NOW#4|Nl~*U_Uie z%rW3VS%^LXpgU@@7*s&IFv}8=uD$*4s5W(cJJK#$yN7`d0$d*bV1*|nRK}beA1zW` znf?p_c7ReaZTbL8Ap&nI`~s?g;Rs8A|KJ8~%%#9j12GkJAe?Pp^^OH*FGMk(Ww;`; zfC%y;sSF#H;y9FTMkD}}ke98mfo=mEin!tC=sU$|qy(#qQvoov z(>F^%1O$8h%R(LC0yB@_lrZz&aANl@zz;(^(IzqrfasgCD4EpMLU5x-z_l%g|UKH?gq391c2kd z)tzyTgH5{%Vl+}WOaiZ!l?`HXH$uU;5=1DOxo8a>)z%0xgHU@LN&1KXVc8e}e-P$E z5)#$jN*D;BF7}K?`Y^Ok%QM2StY5B!Plt%U5G-qwPyodOaYzXYsuU6}OHCcLWEkaC zheoh@un>au7z@d*MNS$=Bi=GJ8O{DW1xpqmDPKS0f)58A$KYsb$m26&J|OaX z7~ud*VVEFhvgpM%s0Al#dqUWS$KLkNSu15s3fmBkv&xi&pODc_W2Ks^^b#Vbgv1{o zzT%*sPTX-a2|U+$@=^V_iE02w2V9Y!sYnXI2+`iz39)bV9YoTuAUM>iQ;&|IXONEs z1VdL&Tr|B5h+anu<&puc8uJ)x^;ZBq(c^Z(7w0E^w(8HwO_VUMSQEHbiV?I2x}0|d z9>-u}R`R&g7!xyzF9Nhp$Q=lXP-MO6kE@&?dliUpHI$(dEhda`h(z}-A%1VXP3Yy? zOd=G(wncOT$S%@>M>}?)qho%MJ}*^q_YT17u#6{7kmrQs`@4DyK?0U zp&`yCom6cinH6Na*op}PI5t|`+GB#1ZUUL3;+4q<6G5CteTC$$j+`2oJ53lY!~v+O zsND37^NyUKzF^|R!i}_%pI=K6Nl99U%&D*r7XtX2{+-0gff)jt23qL`w3>`v z2|Go#P8l%vJTR~SH3{uBy?|E)V0@^@!Jsnz(}%~uL_dGcoRKo{;q?`@y*^!KcF*^e zo!X}MPdpzE-?oHyHB6HDidIWXP9^|m?|YKe08;NyG$GDe=y#vy=FY{XnkStEL|`b* zz0U?{A7H8K%A%sq43Qy%`yd+zA8S5#Z6J#?AFKe#0hvM|B4=b8xe4+ixKTm1FSL`0 zUIWzQL+&&*ncaYx7^vygL(m-%D_O#B8U;C=J5bu{_c)fMhhem^oESDlR(xLR9OytI z{S8TEV?eO8_rI_*9w&IE@j^>ygUu#87$)6$IXfw7M0{c!_^>_l*rzi!Q+4(90wBPe zmo1V*((w@f%8l3wVN|_&w5-2@{6E1A%i||bXfcfA#ds%x)qt-e%rn^ru@#-eWKM=% zEZU5668Lbjk^l#R0uyR~PQryik?OblE8`8=b_lqS`jY^+8(Q~VQk zaPYfm(-to52(hfG`}~>Yy}_i)fQ-71W?|UBc8fjigp7|+0Ql|KQM0c1v17_f zGHZ}jLPNCm2s)|Ykr+m%U=fma#z+QiL&I9cs^C;4g9C^rRoWAvG#JpCqf^Va13GEiAMJ|(XJPK{hD0GPhMS& z%kSbuH*7HccyE)-xSR*vkYTZB5htk49_?e=RH@02XVN7h^;Q))P5d=(B_psHUo32c z+#A?k?uYip?0r@ic+-0Yr_A;IbJV-?+=}jLI(}N0b`*x!4u9Efe@ANm7gS5#ExQ1$ z8hQ+#k|4edS1{O1TCqP64?>k8pkpNV1Cgc-86k_z|onaOhTvW^!L={rbIigm1;y7!y(xi3J~w82VZ9xsiCy3gf1wIRwsu#)ianA<~>YviMEx z4?|6Z2_Z%&UIJq{w94d0K+yKvowPKpwj^i(;1lWb#fEd>9zv~Rhg@imsuZB{L&-QM zOtXJ~IfT~xA{?DaFcc!A ziL`#~?A4YCpbx^d)(Hp8YExn*gU;l;WB)eJ)vK{GXkI7BBR-)^Ph-AVKbF4CPB=Nj z;pkvkLc5a@(l^*eFdszcqO)TW&R^nt)WPnJh?hPUmc4lTqiJP8{Sa|H5Mqu%*>+ek zWyXk}f9iLi%da&ar)VzCEi`uy^{BY7f;Nl5O2s|;`>X~@E&cN9O<^p<{9Mog@Z>SnZhYz=H@m zg`7=h;=RB@f->a@(^W{{X?f!>uR1zB-1g5a)!)r9_FAHm8bDA?PR{I| zP7s*$xS^Mqm4rDy$Gc}B3cx!X$fscYlsH-I;G2UxrT&fKLhN^d2y5KnBG9*QU+g^D z^-$$C#%j2z*RpYWaj>z7AXb}@`T`jO_5>%x@g&V%Nx{}N^Y5eDf)@sf3z#iINtsLl zgC`zUN9WhqY;210q=PIJ-Ghqkh_bG0D2;2!;jZ!;ycM2F$OiDa==WU~IEfFPmyd75 zfj_XG>a4m*NCEZ^3~oeEjoTfNtNYZ{6n`nMN>aK6*-t1wZ4r&|4^;u=*brWet2le2 zBjDd<43lV>1gR-Pg^e$0NfJ&6b1}OxQPS(Ra9tY6wZCFOa9Lp8mTmbA)ebi(5S0cY zQjvE{15Yv)o!W-CW}yU9#eyv|V(< z6NOVu{$WoRFswM}DHq4~GuHcb!NO+pqg7QsN)A>#J0DO_5FLotG@+V6n%n#; zv$((THu{UCAKl1_3B~j8Gy1jfy307guHjl(SxB5Kpr~LMjb$ipzzfU9$@xBDD|rL7 z$i2j4k?juH=eTr~`d@H|zNghuwf0H#RMj~nvgm0UR++RS>0CK-U zV?``Fj=M0cRy%p-J|@{RzEcme5$um4s#PtVrtC;ukI_tzzoOei1)3?2pNq(pExLTgK&SE7ckHkk*ye$V{j zqE%A{1JcG<;ZikgPA86#_mC_ZChZD;LBfs=T*g+x=0?g`=Z7XSa;xfnWuQn;Y{y85#YhCCwE*s`U(`~2OZW%wK7 znIz=-jXrWLM+5=RCfFXBm1DmfVxdz7ahi`;nT!IkOwMmuxT2j=FRp@hmIVS*Vk{CS zGWGc>6P)FbvJlWbX?CSvQf+O9|=kKd9b5SW^3k2MhFe{Ubm#n4Zx?tOGdtdt~a{;?>H$%z10#9s&R@YG{8xROFyB$YFh z-e4k%#P8M7{2ae?(e9HcLt|UO5dd}*l$Jgoty%;f9V>1#q+e)l5f^vsjvT>Hc?9!r zKq?ajv>driE=V?5ckvwrB?QE0-*he-!sITosFz?V&A4Lus&=AXuT;yqb_pdk-Ar14 zuIsQLhqm3Op-UY?O#3e4Hd_jC|qv(*=TwkPE+rzx^x$%H*b0>3{o<>AI%Ma9g552XZHp@db3YTp9(F%BH91b>p1)r~!iNs}PQ z0Nn{<=KTVY9m!$_Jfk>{<09&lrqQ@bys%mXDs6llZzB4?FX@?#%*^u`2??kH zodl+ekq@KW0Ctki5=aRz)=v~@dq4jxzCip2@dpF=r@e?lA^I*j2CLVElzHN@ z?8J#+-|oc)1y?;=O1=PILvvsl(Wyt{JVQc`x6;N8hdM-C zcQ>vbOMyK3DaIj)bX155oLp}3qzVxVNRY~m!J4#VKyon;C_p3)K!M9kg#k*a>Zn@M-205HE!44K;0pF65@vCH@)xjC7OX z7=a^q<=7r&zIwCE@uMC0^P}J0>}RethiebtYMZf^PCxOZZr#vzzt0i0tFJsSPX!e( z3$4)Uy(iQ^OK_j+-ADflxqKOeFp?aQ#X~Mlxgi#?>cn%gHHXh`f2Wjw;!(w$1L4OG zUZX^uUYu?7kxBM7d&KDt_pS}b{<(~~FTXpz4QmKUYT+7Ka-moGSE^uS)OpN23&M1Q zj6d1VvMb&97#`}xx}yKG8pv%hvW=~JofZ%U0KT0_C=BKW8ULl7;Spc}LxPM8paTd8 zuuDqWFTd^Uy$CgB03fPwZp2#?P9dT4OOQDgZHw3(RrUF|ABJzpi;0Z~>ZKd_CKNNJGF@fACas;B)g$;(_(p=BUk2RVBaye3(fGI0n(P=eql9Zip>8YtCBN6fX z6p~G|a^<@DY?|-WgOm0Y$Wvv`_537DA0}{!&5Hkjmp<&~Jhy>s2zkK-GJm_)C++=x z8YHAYuna?_R|pwbBc{gbPwJ-37^3(vNHBqrV*e*PS0tp2L{$9CENdyFq&!$0 z>E(zOjDf|;1R_Qr{FuAL1zbQ~82pwJ+p@)pYJU9q@pHJNk+8>E3!3f?WO;1U^ez}> z9$Vy1=MK03tax;E=GViA6pFjM#zb|nhf}7qc8#E7&`?u1Vsm{xbZ7O7+K? z(}vMeWg7+x{Ciju^5)NR9|U@l=M_|p;J(5^7~Ae` zUse728Z;!xUoem^=t+YthyeTucOzzI#Vib$)xwrInIM<-pEp;>%Kj1KJQDtwwDY)y z+g{&!WM4(3vH)zhKp5?RoL`4O0cH<(fM8e-eUX1t(9qy;gM+N;)d!4$xsX77%ox~I z$*Wu$3domeG;zQeF^gwY;6+<{7jXTU2BAX+29<^bgo1cv32jo39%kSRTc{W=N=Ys* zfp4%UX0HhKjb+V9Pj3T7UyAn}Y~}hI1$lWA&j^vOkfh|%LTrhga9^DVR}>shy9TI} zC`F+#Rm?AoRmSj6=5I88com;#qXJDYLKY=6Zo@wguTe6hg2(Y9se&L2TFb|x@IwNr zu1*$16u$!fqfzJZOhmd)ZGuZ|-xw5R=fTqDeE(!wGrMLW=99WG_mavd4@6nbP)@Jk)nw3PD}8r>cv%gajKN3zFSrO1dy`y69Uk*c1gY+d%rYbl~k<$-Y9|ZrX{LQ#u|sVdwhqD1P(BM4D`RkZ2I59jb9;CerM) zO~A**<*O6o|OFA_zCO!?7+bA_A4R z`fcctdPi{e?K8)XpneTfeM3CV=_sBfWg!7w?dP8WfS;hoC!%iL7s{pja1YG#Rp=vV zCxUw-e88(4g-h$rr7C{n>%@gHzczM5F@XeC#|v(_z=$A{=w`z^uZDtRFU8S}cxJ&G z+=4{Wi*?hz)Y16w|)2?-jpT|GUxarK~ZJOlzEw&jmtt9NWl^gF>30hDl6 ziZf9l_dxGfmv1Cj?sV=TMh?HbD;egWRqk~bbcZ`NY`%;ebPd#IsEx*PMq0{i4_6hD zpHb1Bf5|v$=?@nr)9W|2oqw_W0P8H&IU>ocDxa)w|JHbMk57ZYZNMr%<1O;^Ulu;a z{^NNwS5hfDCk+?#H@~`ZnDN8#z!63-Uaj(rZ;pL<`k?%&&WEQ{<$1Plle#bN73~=t zetzPE{enN@T{iWDVh>?H)-#Q0($uptaX>mq`qnH*McHiVQBYcwJ85V#ec3&9r@;Mp_3 z%FjG~;te5p0B}!8A~{rELl=lV%|d)y#B?kD8;V#wk?r_*ieeF4ff@s^AdgGjk7sPE zE_F3w}w=Db| zfpuiR+Y_;noWrkwOx*WVAS3`?yBDktaWf%&J>m+629JZZeW$X-8|81~%_s6+C~e_} zX}C}hQ3*tNLnmjn{1&fAdwAE>Gmt%wth;~P}RH#d>u{{ z?IRcm@j()umMN~#iJ!?;L{9l>Nb)d2cZgI17EG-z^QQ65VAtm@&8k&<@dJdz1B{Kz z0956ezvVCsv`Hp4|^6za#%z4gO;ATl+07 z-)Ko8_#!UO{eOcAtgS~e`og6H*@jGjKxTUS?!PJsyp+E>tfQ5TlTMP_7u?S=Z{Pm9 z(+N>MPFHW-FV@;f;uPeTz&QnjBY{OvZeXrH`i%cryUrL2`Z@x`E*P3(-=x?p{vdV{%6k_}GyjWAePr0jde& zB#a^HNZt?0C^ch=EdhJ2(+>pAIb#2;G)|bv&)-cjR=6O9rO#z05?MC}G+cgF*+)Qm zmOmU4l1+omTg2YpgpKn@;Hl)9sH2MXTvPnZD|LY%%H#K3ytw6+4jwMUFT#o?x17eC z7sh2~u6m*Q5E#%>oh?w1-zW>oVPo(3aw%9_?@Hn1Xjg2k3Cche&YbzV-SnQ2an*_& zIXRh{{4sCdtXR8t|GoP=_C(zV^v`D9y(qt^v!PqmTvj<$HZC=L>BWp%lLYg|X9F!Z#um-qJT0x0s`5;`u(pmNMs#}jYrcpouhjvoXp*;L z#mrR?ne#{onc6Q_5Zb<<)$3P-n-6#3wsgB3_9Ht)uBX}*eBZZFR#|OPiQ}FIrYZ}{ z=f=-L9r27cJbIsNuX=lgF}h1Q+6P{vh77Q-X1Tv@V_AZV(ZwxWjhVx9ezNiQu&B}Q zq}`_Op3#5A-J~6US@qb7EvI?ppIWEx{du=qriy{j&4Hh=!z;p-4T9dj6*;aO26t@R zU_x=I#MMw~b=F(m4H3#)S8w?gDZY2l9>vXD^x8Yl`y?5)+`iq$+GRcCGL98e>`eAa zcPARR)`^R#fbj|oV{C6v2$e45fg5~OzL6_;*5P~RiGapNncgw^K>mT7o{Q)`uCM!g zw+o5BSeACRw$Q}PtE^_RsOXOD?CZY4v^`oYdoF=db39!%%CQD7;(uS&E2FhGLN>M- zx*OGf`<9M@M<=>$W6-ZBPAm>6{&l&Y&?MP0~jZ3XZ~1 zK7#<^Qwp)gv$3qe%LoApY5YNX^`@Hg^76k}w|hY9*Edg( zZyqUR7AK;4mDgBV2@DD;;XEumfyJa9peE5-HJGLL5g!@}e1c+C!&R+D6ObU-Mlt#4 zBySQCErie-mUwiS(60cC%%^bikU~NL@z#PSg50a2zXNv!*{N_1AoqD>_|L)Ad%dGf z7NW}k1crO>z`~D9oVZtrIt`mR{?pR30uEslAuci?(3o5g5&E^afOmIG4+}NdB>p(E<;Y?LD7YQ+AGB9Q z&ylkp)GKM1BXI?r{CvH+0N-MRDfs@KAMILfV;3)6$VYw?R)*$Q4OJPq`bZ)`n2{ZX z3ah>wBp^_qMAAbg;5moOd_BrlkR!D4@xv(}q{09RBgfy#*Q~2uQcu4oT7mB_w6kH> zemGiDCB1yPC!|3nzM>IXJ|v z#klfU=_9GcT{3zWEqSeh0<*o+;{#iE>pq-F_c+X05y5vudB#vII!1A%OjH(-U8$FA z{FsH~*qWYC_dB@q(tT@cBtt88!d!3d_p6OB8i_a9fC7#7q^3-#G=_o#x7Cd=G!sjH zHpJY1boqU-n=|*Nss|0t2>6B9+-ky&Zm}A^4Ekir`J|4|mt>P>H6O)hHhu0~^CW=p zd7k8Izr-M)kAoxA-xK(^=0~_?u6yTZV=hHmXI`}}yeT4}QF3Ic_x;5fHo1aBCTPjW-2L@ZM~C8V4!%ze-2H`iKI+TwnbGXGLo(y^@_kXWor>k{tG-NE z=#F?swd-k=PtQ^( zU6yV)<<-Y@_Wz!8qTHBaz9=`M{AgoUd|#5m>tlkpBPJEo`dfL=?9gofZt>vf{@Gck zsx0wrMTXJuxAq-6%d>*sw5s_)MnJ-frm@2}&u0&Jv>x%&On+B8%|Dj)O)sVEh}=Y& zH0PPkKF^x0@10p+RR5l`b-Kx3u76;9tF^)8Aic?McJ{rdBkxd$QE;Sy!{nVN9^cmZ zOn+nENf-I6&mI&uZ<@gJwY9Ht20H(2HSlHT`Y14IO}Qaq)I2;IsaX)8xc{1~HjWSdj#I*9N&4^0 z;~roUustMNx~9Td3SOxHQ^$A|tdk$fl!XQn9F+H4$q~OaNEPQZTR?_smaM1l!4gQc z(SX+)&usj47XO-fx&SZcm+@m-HF?hr=ns?Re9qwye2=yDZ!a>Vm1v@Z>n=Ns2=I1f zy|6>G<=F^<1#niDTis<`7h#=M1ujz5UgV>VenlfU9D#7&P_XXvPs5x5c^Hw85~KqW zM~{bU{pLt1F#<*nf=h{~j1(V`3|+FT(2+LqG&c5tiJfvC5o|IF5X-Bbf!ur;6=v!* zjHJYnN2Dh>g$aQRUMkoKR|zR0`M<0I56~nO(Robt&VDG~H*H#q^SK0Lq$=|=*M6{d z80{5+rXkg15q4gJ8&x0BJtm~jsS#6M&)fR25gQp45I&Gv0j57$0~XJn7+$e-sX*mR zY-UuFVge1sdl~9c3Z%Q%{f`V2z0$+irxCs4A zBn&Ca6}h7j&B7n|LQm@f;|q*-6%mR&XsBOJGU??k+bG`>&X&`D;z`(hTP12YB$K$& z{-2YW2w`kdZ;d?(2d?-_;9y38q2^6LE@&D-O0lcXkb72EmPy3S;PiVZglcw-)dd?h zr8cVt#l4hlC&wRN4XcS^{}+vOUFEAz=FU(8OTLu`ZNadWX`}YW&5gl!@XUhy2S<(v ze~nDg-?47fWmYP+(eY>6X?Ofm+;tjAaO{msYpuOlkTn#MUnujfB`=h;5HgCUERz|67K(W83QW?@j7{hlaGe$&YieCL{Vpb3!K3IBx zi+5?zmDd((AE!@!nClk%J`p)-Ni9yzyEypvTwPTaXbSbjp;2k>_|qeOni^4C(|R20 zbK_G`CCId_?dP+iY&)ar4Xl(i_4j>*@-%&F&krQ!#iTIDMRs?LKf9JaS)d}9-uNkH zr=j8YpgAY`Mm>23%36a}=SG4>Y4Urg+!wU>=P%X&?32E$MRtU<;>l{~al1>mr$gzT z4>Dzb((4Yzie3I$OEZ+vdqX!6Ti+RfvoxsRWI&}c!ubq6b4}RX?BYe5HOeQ(r_-*C zlo`Bun)Iu>%E~Tz^GE?t>F|`>2E*g;>W-Paz1f&+n*Tysaq%%Dwcy#+$*VG69I_Oh zRnnjS`gv9<^1@Ts#ARPMzZt#sqWW8NL;T3uC&n{*kr5?+;-isrsjpu%s|9gJWn`ds zj!DtV`O=HESHAI<%*_VRMZ~{6Y~S;QV?%3W7&k_RHV`MG8u-^V{+-XU5#XGBVL9i#Q)$}la$?%-3+0O);6iKS zlQ3m=q|^m<=DVwaa9=Y%PSsalLGaq*`&DEO{_o3sCCDo{@&OrUE_nwLAMvoPUCv12 zz|JSZ$?~V<+($2sKVk{&0$!yvgGXL>m+h36Oi4>41PHOFoc;az36@HL2-`7_{U_{` zPKVTHK3)z*hpGV?vNAH~A(bNaf*A8aR?mli5JDR2R8^9+(}8J{(0~6h19Vw}KvCkj z0&Wh+m4X7HF`<9sf$(2Q7cU$nMEj;QLm9wa*=)X!7asW=d;i9lYXECHd=b>uCOwKN{L6BRCSSb){ z4pDVzB`eoOctc7^Co+zYFt0CuCKpicNiuq3yP?lo03ORj4^QNKsHt(gcY*7_o~oO+1Q?iz!-qP2QQAGL#!fH>sU_dMr2?ruzE+ zSHVZmoIzX-ZAmZ_C1(H5j;@D7EBOcggH}i-7$g=(D5hx}v}>9z^>=)zrW}-Y;)#q%t}WxIv*zN0GZq$h z_KuEbjOy-vW82PSje5ONXqK^Hx)|P8|A4e(BHiR7-i<;q4eLY-Fxqt@@nVD$~uk7DSEYu30s( zlu37+yh4S^cIg=57t3gGH{&dfoE&kC8UE-J-f(weeDN|&z|kQ$qgZX+YtI+YHraeo z&j?6;zJhWi=?%XgoywD)Wg~Iiw!>lL&+D5HpIj5T#mQnJC4n#3W3h7RA}?y0GInIX zkJsY8#(%70?TP*>-&9aZFKE>;6fbW0z?YJ0(}lbxU5O|8nEy?DYD4 z+12t_O*BsXeYz~^4$l{T%bTe`rd(OvyGhA6w4ENhZ9x{D=5T?}X~OQAR+)scF$J^e zxl4&EXZJ~64bKjEnZ?@OV}6C2|4qQzLw4>G!vaxzMcbxU(**OPRDma|Km=;!H@?FR7Jct~=>5E|zq_u*rtH1)>GpGO4%^w5wV~V!Pw|Ic z9p4z}Kkj)tUd8;B&|A&seJutNMNN+tlE!j4o!y$)?HP9-3ZGf8Yrynj)wS#IY@;*A z+eatU^2%Fw_S-0aW9p#2OpRX4rJU!N+N}{QeK7FIZ?$}TO%vW4lY|}Z-zFs~tig6+ zSz9N5C_JD2_cOqoA-P*h}i4c$)qgjH#^Nt zp)6hM8mg?kBsX`3$<;xa%ASFJP-!0;waY8dZ_bE$W1@W^F^!%rgOPWjX;d|({-pTU zlPf)MP}J%hHff(~@PGB{Af9Bw>~HE_Mi5$=Ap^ka1_1{Y-R*l#OA!&h0!0T~?+5Y)Vxfn~tYrrn!Qt`kH zrpc#}uwrM8&ctX8B@D7+yutzk%AbZ$c*cqYrH6ogJ*?LdZzEg2v(tn4sUh2*ae^yD z(2)#_*>h93{AI|ZMY80RuC zApHLvgYFIIJ<{X~pKGE#3HB!_V8a)cVNCYU%d=MzPHtEZaUx04j4p_oUT_XP3jPvj z*8ClwBS?@MB*j1;t5F|^CA}T8PDCXBv~q?l7M^*q$u}VvBaV(2{SMLB$wd{Ni};-Y z1Qo6V8M6Q2LI2*y4Q&NhA*5*lPEN$~=fAuB)jn4e0FUorK$4@q{egQwpuAaYToLNt z)^-l8M3sn>zCJ$y;5P`8L(Z)_$6aU>*FwJwt|E7$gBecNGq*>{P!8n`QaLEZRkMKo z4k&A?B`zxMBXIj8P*W?;QUAIE8ouBnD}YYH^D!bz9vZm%frc>B$`gI|{Z4t;0S%23 z6e5df8q=ZTL*N2u;w&H;?wfz?8Rt??Gf<#XfKLnJ1JCzm-stIA0s6W*$4Qrf|I3sO zpb#QPW~>&`)1)mHoZK$jfJCNSZf~x^q&h&mBDa~+6Hx=5M!^f29vFS(6sS(e|L1aj zO1}sqj^$F#%e~Ib{g}+#vPBP`d&76BwhR^+-WRMIY}G97Dnl|<}pVeM9f6et(O8kRi3@fJ2z&)VKz8QpLIKS=iWP>N007P z_-6!WI=H@Wzj~%B(s0}AEizYk`Qv7xfPJfQ72>x3?(6-sDYYkuC*Yl8n&q(Ug)>-f=rr`q7}m{uk?#{Nk#sDhc=6pXGt8*itxUHs@gdaB<}9>?!Fr0to^VS8XZ! zGg)z#<8OyXlqyq1&NSHZ*p4d>ar81@O8t>JPe^9YHKDRfqR)xL>Njg&`b5JK%5J@5Oa#Ws z9_#V)FwXJCiM)hjId+a=eCF%osJUd1ElxSN+^2Fr?s8_LtduKYPMk4aeI_%9zrt1V zbj{pUu&!0lN0%tYtKkZ%SI?S^o}Swl633Bc*IMGm+@)D@^mL;qt8#^Rd_1Fpa}tV% z_C(JK%vrJr35b^*66f2k=ChVE9AI$fee}Ndn$5Nb6Iclir^k0i4S+WYR@T3K)Yf$By+AMQwRV=LH5wCJ?hy>THB!#HIBSJ`&iqXW zCqpv%s!@iN6lc4HVc{YoG2xND`o49hvbF7n^6VyJYDIX)dgQSplks+WT#s9aY~CbGs#(KLBB;SZxpB)DjUf>@y%1*Qoej!&RN+kDHvl{pgZsgI-UsAy z!b6!z?B{^i4ZjK?@MXwl$Z7sw=)=7bv_HoE#fulQI;bhuhB#G_dMiRCYX=nZ!UUwo zH`rcLP{07vZK9pk%@?8XLc%|)Iqd5Yy9|gpk3Ks@Xe^TP6VOYve{5s;#c0A_;ymQk zT7=9`;0;;lbOA1r@N*IsOJv7eR|hOoN0E_m9CXDgkZA}X{NnVNbCa-&9#FXp#b&51EFFYBn>G$>G2@eNZb@kz*Nh84-o4R>X!GQXe%}sE_=wr6+84< z4~NUKVV~eah+rcDt)P8S>d=?Ihn#Dam7F?`5+suHP$d8mi2EkQhM=oAyo9FV&^=@O zUo;w53gWIQ6cPrJVJ(WF@Yq&F5J01h$cUmF7kT}gTUz!Fz@_jalTM#9=ouN<UHa@-K_jDs!;bo-PUA%Wa-ca(%D%-zuCiHJ zng>tgAzO59vS=r--9YBp2mA%^_=Jca@*IC0d8A-wNYnGBkNLvgGcP4h&UBWkOj8RM z9DgW1zFC2iJ&`sNdYAY6i_xkVLwUFE{@fg;2w>zJ{i;}aj^o_q!W$l8tF2qZZ>+sC zZSjc*lYOhiRdV)y-r3afTOspGQeUgVjZ3$YNSZdl__*J!XPCy*zar0yHB$5ojceD) zNS1uXhh3+I-x;m;-u-%DlAa=da+}VF`;umJRSrwSs7;-rzH8?~Gu9t9dDx|+{0l;r ziv@LV*_+?(2r|tcX;sG|o3eJx!}8%mJ&xPym$S9vw_PcGCeA;b)vA~^w8k!bDsr@~ z*U-r&No7;%Avfh!3AwBWF~b4AT7|2YU!$$@R?us6JK0h;;vsQyp(sRSILgn$0l zZzJdCx^4B=C9XeQ?j78z@3@IG%Py~eXwvala^#a$x!+9h%h{T3Kvo}3YRmm=Gp%$5 zj-hO+_PFoNvw)9Lx{;r?Dp*WB1ddt>gE*+&?nS>e$U|{$^P&+A&+&Z`&Jf5@L8lXkMxU>C?T+7y;jYn2?tV)?D57c zz%B;9auksv-CN;_h7S@8NT*_SCgwJH&2FQ0LCiE8gax@z60CGQEo1;^S#=xI1D_uE7d`z$kp1t)VF02K2Gju9 zf;$rDsEH(r4uW;BB{!IQf{FbY!KNj|mWKIP@z-O%`9rj41BA!Wf1zB26=$hTsV3zq z04|>ue1$}62=L<*QEkmWpFaPph3g96E0BCdqJ#66CMQsh*eO^j9=y;u0~hGnjd2?x z&3rJtMLQxTm~>b{wyV}0;_ICZ6$aF>zNSR$?&sH6J4@`(Dhqr;{&I|6==haV`;5H`ws^~OFde`qaC`dVcM@rX3c3xC69nk z63{HfhN+p|if3iPy^l0 z@a&nQ<6w!~blURv{VXkdS3!5J3;x})S!%NjYSCgy>6Ig@WrI@0*tiX%(+#vR@D)*w#jT*q6jsI#ug zjNM{~5>pjJg>$(V2WtmM4qYzO6y5vUWx^>ecgWVQlzaQaS*MgljuRgXRS`?WSr2r%L?I6hq`q}AqqbeUSNRqe=B_*Y3N2pX+@X5cYSr)yASk}B- zHQT0tHF}-}K}7=d(80$ZZEI~U1@=$)pw!IuJY#?-%p{LgUEMEs5NSs^*b9n_FH*j; z5Nc5-1w$DQj){@1=y4tk)iJ7T&f2s~no9EW{1xJ2xW92)P-|0XXZdhIZ;W#3c%P%+ zl9~C*dY4A=Ywnr{2#JmpSKHk={3Ehz_wX{?{lluW9HsjzhCPzgaz5ueq`oV) z;#kdC^htj)<<03=>5gyOlA0>(>2F%?0(oXi9Da+8TR-}JzL%pk9X5mN&i1ja6U8iw zL0Jxi?Nxi?9taISdn0;P@0qsXpu$?}$1v?v$La5<9T<16x4c=$QF_b&7iaFEsccL_ z^rRnc?cod8);@n_|EsFH;7EW~vVf8g*O`~Qb=9h_99h0&;iS_u>!hO4_cug?l25)$ zx1zT9Ur@~!@95xnVJS#<9N5w15TC`o{JJ;IE;9J)_+ZY5Q9?Dh_YYg@zUnO=ioLgj zFH^`gZ|qx0t0t^KZyasjbCfE^JB;!Vel@!~sH(4Tlc%^Euf1jdS}pR7{`VED2C@_0 zZSW%K_FI5RCBGp+5jZX;n6hWbo9su5x!42*f+6G#^E&JZPdgu^aA0Ul0o0J#>UQ0{ zsa5FiP&Nc#EC&M!Vt#JHQ>Fjz;KD)^X_6o^kw~8sCsCZ|lj;NSpu@o5zmS$oQv|ri zi0r<31Gpf!E0#ONuSs0|$QhPPn#@QnW3I9g_cHJ_2vJ)1auuBBZ{BE;ikXG+_(tH( zDpC`0frrlI>kyt+T)aSGcA!V+!MgQqiU+}2-#hs1bLz>hNQZu~9j-gC2yj!r~eMkuc#>XbksN+xZ@BzoO2svRVqJ{rV64LB(wWef8 zvB;a1A?F8Ry}eOdzvWW#{Xl(+jd^Y!WI}!S4~H1?i)zG?H8XFMX|ZeFXJS$na1eiy z&LURj_fW%r{``4V@F@Ogn(;68og_=ufnUz}1tiV_&8&OGxWsL~z*33Zo){IjKZ?kv zYy#5~5fdYX6ah19emh-VE+zVmHH2Dcqc5PxlBj(l6V*NvrF9PI+MDELr+ZmQ=n91%hfoP)!IKjmZD%s)z)hgdFM3hvhcP{!(WeT^CZF_QG?a zP7YI-^X*yN`#(lIUJsRzw2uVEo6|ILEZynIE^)md%8YhishVp%2>fZ&>#+S+~jd7rTL7^S5T}@q&*E z6{~{yGeub!tyG$AA4<$9zd4;|9$dF#AKv>efj$PWqjT@Ayqi)?ELa53L0f0%I@5;_Wob=DINe#8K8mvbfdt5jPre)v{JG_olvp)8QvPkY_`eP{1PTjj z4*vA3HQo28x#;%8hoUt*xLHVfz$j{NLDL@Jd?M1-LRu1Dr}cPOlDHZ_bX_BgICv+em%lBV=o#TpT~ydF}J`Z`V#3Sf%JS# zt{O#e#Yp6v*CgL1>O`2#Nxap*y$}S>w``C^7RT_x0A|Y(@%bN#;N6>uV?>P$iRwUn z3Qnr^LP87CL7^B#+R2a*fESX863L4XDnc=cnB{SSA|4tM0;S+dE%8@?urf@71g2cQ zDh9PSj;ws^2tVK6jOzxTwWw+^wHaC+z+;BZjReUP3>^};Zl^2=qd~!vHkSWDbbQAo z;2K+x&|?9(YdesDLh4wM_(C#-XUoiF;C!S<=1d0#Q?o~l>5pDYQq3862zv_emR|4o z{m#bcjfLkPGuxh#@srJ>rIi`et`1h_*oBzgwduH!Y*ld1D8K%>ERW~mM0P&m+bh{) zx5wzKUHmvXyT`fB>|v|RJNJ16^C>V22nYo7?)>#gGGqpJ@?o@(Lycm7zS20EIBD(sU#FpIRoG}g zT_X@RCdg)hz66d!=xy#wXw%twd4F1(jXJf`6$g8I>Wbo~l;Dgl%W!|KSW?;5t>b6F zD)4$m(!NUipBXDm^5ACpYXQ7ZqoWhJ6vy*>%c!>^OaxGDq7F2q0SyR3aijL z9!723@nK}9bwpgXv*VLmsr(jyQ!85GUR90d*VjDE73FCaw}`UR5wb0~t?_(l7nkO? z>=I9S_MQ4!e93~3zXa&JmitpDu7oKSGB10!;O1=gLM_L<%og)zm$Qw0 z3n>~2UnZTyPJj3wW<%Y%b`zi8U0tu+cjnIW#i@<`9vxi5u%5y|p{T3wIsP&x(c!t% z{kygYMBQ3#J%fX#q`6P{lw@Wox^A-=psDGWRctK{IW+RVv_ETXF0*i;sjH(-NX>R; zb}&}^34QXi>D%_<_FcSkw>{8&c{@zc>+xfe&`R^X@#8u&wo|!y&l%YhzF=JbpRcOJ zQCj(k)g`S%fq|U;Fq>gsAa0yThdW+RdL;~8M*bnd08)Nb=7IznxH1A{(f)w@MlA5) z0Q({a7=%lqXdqSrol;y}fiU|)=G9&3Nd6$j-MH5f1O#)foc-`RPtk<82z7stQ-h!$ zR&P0|*5IG8E!+aH=AVLu65@LS*ApVKmQ9<6-b~UX=K-HnTD<^*9Etl-PA=UigiZTW zaY;!B@ZR%Tu3Vk!sI(u`i!Z<% zQA{FYL?Z!AQj1A6{Q#8xkM7RntGKYEkx_!thG?gWi^b6~^GEt<2=soake3L@?OWu- zih#KVB#O6XUc_w^2SgSo+D?2&6DWw+Iwo{>Yb6c?I4Vd#7GdJR>%|`D)B6eVwpYE9 z%W!OLU}|53W8>339>V~w@%a@2UcdWKBJ$x$Le%rak8Y9*3mUQAp$M6H^~Z-=gR!is zPhpYL($)2O|6Y|}S(NGn%?$|>Qa69^%SNmU&u6)pGr~MUL}+)XNT(0_f&25O-H}F4 z;(|%x))mYU^^ihikegIM`*mzEDXhW~{yw$r0_g(lQ5zJ55pjjvrG+<;28U0L3DTjx zFLSHv>bL-*pU;|18u@lyM=2ulds9 zIDY-A#%#jxADs)Y4@yu!)QbuTQI;@Irx#6IBnSnEBwN?H7!O|<&(1d8$eHIxb*ZhB zPEjaPf8ju#O)ELmu+ep(yK|<(Em<*L`=&U3>_S%O{!eQI7rP0xcCUG&SsHmI)3_>S z>amDD%Qapu>1M7yLeAQuBfVqmT{KqO3^F}!#%cdi#Mh*#6U!S&HRFzFVF0oCupZd9yH`#(-7xE!b`^(^Si>slm zRg{*mEknns6`j4q;jgaJFJ-6o_x0SjW-=J9IPt|LV>;uyoUYnVh4@I*tkn@6XJ1YZ zect2IoJm`el^>E9zi`2srZtT*3H~#rtfoDrII>w2UM^go=zZu6$vk`J z41(;7!wSybz>QgB>bmP-aD*(L=wiIFykdAGkl}&H7*^jUvT5JlB$2>?js*_z<*3jh zvG_7!M_!RoudkZQ#i+iSP`Wi0PMU_mhTx1iPa>h$?7SxYu5mE0?T&A(X#IdKxchdP zZ}%Px>GVnmK0xT{f7CowszwiU(Uc^|Z9s84>!#G&msOdbD_Q!`5T$e9JQa1YiUx`09?we;h$UV=H^DcML<4D`SAMrzREE*WJZy+A&5yyGzHKLg>Ytb znXr*c+157VbwKXS8^a=deF&y(4k9!4p8E;{02UCt*@DrBtXa?xbHwi!K1 zBtIDo%BBrSO@DDTj+kR%8#}oA1rs5v3NZ>GNgTlflCUo&*#7OgI*M7X1lVoXk5JM! z2dk|Q4w!%)VU_d6y|-k)lNu7KR;IEmm@@2#+uVONMiLJJTy#*UNzNy<1ua{;)EzVa zk5D}woz+l&T{v8;?5wP8V~xBY92vzpW-c74RhCv>(OkW=hlfX?O1AGuO@>2UBagwU zhWfbKloGR4+cMXP5?scgGfHEHjIyW_3moUHs2_hm@E%W&uQ}^B^SauME$>^VX@rB@ zxBXi_GMhH?>FN#If4XY@SYub+JFzc2)vkjaBV%uNc=4RV zrDfV@XC@r$N2=6pOaBey4erLR_)mF#k!)o*^y4ur@f~?3^-I8 zxmW(||HsyMz+?TkeP5bNLIWuyg-S>wdqj##NJuu7Y{|^36taq>VPsS!q(ZWHDk*yl zMUj=g-|wmJ`+eT$^ZGnbxBGwptIKu$&f`3e@9`bw`I-51Kh}DT#nm#-qD8QEYA9+t z!zET98G35B2Ol+S9xo;*DnH)gB{c0nzo-TO^WZ|?92?6PU3(FVHC^3x|KpT-mz8~v z{B{=E&B#DSQl-*h4Rj~4sLuh967HpTU8z)u=vP3G(uJlOS7~yPy`5blEQ1JH(r2YB zIm!taXxXK=L7QBNk&P>Kh9yuDgW4K=NH>7}2dwcEKa!|H_|n7})dCtwVoU-x=Wc_2 zoU#1I2UzSdGGYOl(TD?k71;Hk$%AU_ajLua!3Y*!DY`G(5a@#4=5@uISpjcu^JBpN z;4c-}*JtiSdNh*tL5u{p@XMlaS@a}NFM6fWWER8&{r zK%9)oL&wLLTsL|WW(c-FzicO7&2U{)L>0l8>tCzRaV-*@Ee7+4*w~3k@v!Jcskol@QrTnv+WVxC#9ujd)eY9*x0{)H5rTFE-x89qn(%ORJFi2ZR*Iw zv-#TfHnlp_>w0D;aheI~&-stn+vzlvmybUD#CT@^o3d)N{^7=_u1WC&-)t`W47OAW1C%Wj6Kyl~9B{`g?udZ8pFK z1IN$h;N~Md>tYho-jcZ!trdj(>Ec1;c)4mKt5V;6{Iq4;yUke280W&Xlkg9~Ef(us zGY&|Gfm!_WE@DPe|2ay^AHN&?|BuM5{`m{f4-tL?9E&JkF;@_0Vhin8Ip8g^tGK{~ zf&G#LW(jsPr;yM_Jo@(pHhmR(*<>l|_?sCxa*gpKp89>*d6}gLF};EfsrLXU03F+>RO5wuFc1ZkeqC_Ctf(jaB6nK%102+MkJeYpK%E_ zk@f;<-VX=I*t~coYFAt1%y|-{B4n0zLAgtmeQnlEv#J%~;8J#r1-KLjp7X-QOvWy(!-r2r%BQBY9>1IUdOuvBi*(F>Ro${dN9i$DFfVaJ#{U%RP3NomE`N=DRs^$6dz1 z_cIUw>~(%TJRau3Hacu%75;njq9^VB%6pFG&Q~P-hXOGv4+MC;h>tf}O!ZtC-K@Pg z_%l*O)w;)ion{u|eq3p_hj*()1Uh|RO3L#klZ<~Q9x*xh5O)V(A&7fOd{jLW>HmB4 z?eb`AY|IMvf#iv}e!&u;9!(nPG$bVnLYIU5k6V2pXdt5-q>vzuoJ;D)R#*@&8-&Oh z_jOEHmh@_UFXgVUGMb+mp90pvH8cR%jWp!jz`ErY4aceIko)(2;nHANbPz!Vum>Zp z(te&DoP9ZD8T762dNP}*Utr*2JY77r+Q>;#V|Mp2;cz}a%W2~p7F z6q2?RBk=m@DwIPiS;T1+Uzn(5x;P+!(?NFXW7Vo{O7&dQ;8bd zNJxiz%onFFEed(rHhI&ls-z~j^&0%{M#k06H5<7}lWUv|%R#az2Nd@mQVazJRxI~m zW3?u`Z9N5*4R40xHcPt*&~zxGRqdCSt-+bUGX z?n4Jg&R~18!if_b(R>ZGcZBgibu3@EwdeYk$>$Wx%9X+cK|_IxL1Wj|6TR9N%m#}- zhM&+*e{PbxJY2x(x1g)KoE+`gOvYlBt3S_c5OIpEnZ7hE1k@dI_A@wGe?}EBikEGw zX8GjK?zUU&gN1)cvL!jF74LXFFv`Qd^9T@~y(1ofZ0>QG#>?;z<6x@i_Oidp?4$e4GP^bvbUbBJbbYtxW4O%h7bLFPjP0eNR%O5_p zgiPNvIqIhwd+TD0{m-7tYDYzx`mbq;iF75Y1|DLoOv+Zg$*#=362|(OATnO+gB0=< z{ok)hc$_h8+eYp+g3J>7n#aWJ=uT`Uh%GQNDPD&b62AVC?U`-XM-Zw9i|ebnj*Ovf zBfHGu*^WiFXxKeN6?YAtWue&tWtGd%+G7$E*Pu)nlL&Vart*oA5t18?W#A$+jvO3Y zFWPM*_x1RCToy#U11ved9nxaHvl-uD&a@x2Imul8U_6w{=I}a}Lln9s+#RA0LH*`9 z*y;y+oebx>QXm8f0osG`v}H_8;Zbx)74%Z3XTza3)C|twFs^uwgScRku1ifr$YR8HGp517mIh|R=Qag!; zgBVqSZoCw2PlDR8g1OE?&G#1o?hC-J*$pM&5y0B0a@%GG93_a4yv+Qslcc|XQ6I#& zRvz1`BM4%MPuAFTb}q6ydL7FFYMdKx=yJBrAdQn4oIs{zJIIF33o+%uwjI&FO6mx0 zErYu+tH&a}+E6&`K>&g&o5SP=E*=+?qVG7YguMnasY*$yz5&~m8aq@s>uiHQG0;_* z518GKN7W*(FQ=oxQ&Ffj0{H}czR+6H-u4*pVpg8Lg+GQBpM%bq5k2tN@fa~t0c3H%&n zkp9eZu|qi*pUIJ?Ue?~97yUS>-%C$ATYlSTu zrX2aS?#81`d!PNg>Vg-0Z=?XSH{TvNSDMP+x=T*p>Xc8$N6FJ)`aEDq;keRVxssLT zk;BADf3$M4g5jH!TjQ=osq@Wm`So+?cKcM$eubHWCOzJaRQ3U>RFk&KmX<9$jz~{S zh-V*5F&h5kI(H@O<;x8D7TtHIeP3oH)@)>3Y_Is7B5Q4Z&SzqAATx`VMN3d9v`Iny&H;B0h$&LGTR@ zJ_56zH#%0c5|(9fxFYtESf~>26P?T)ef~^c>TpB;!e(4uWf1G$LH%uR4vLIFtd|5= zBc78{B_PNL|4RukwZnFT3?YO9ialq7#&gDUathUky3wBz>OaE2Lf+MxmlCQyYzW@p z4M-TpM1};g9T_X-3INKS!%b8xh;xP3XzHWE!Zl>#iH{-llf;{l73eBDw!QL)IQ#fX zx( z`q)RiCic)AzF*1u=rZwn##cfnal=FW5h@eGvL=b;_hoTCe-pega5{*EY`1{tG!4O6 zZr;54pF1-Ut`2CdN5JBx#}3&+^w|aqY+#6W%;HSyVi5>n&eBK*_JeF;r!(4dhWDs1>o$2@MU6_rQ#cf#m&N zV(2dp@`Z9W4(81^>+sztIxZ8#YEiqQm^@%XO7NOTk)6B>`u7w0>)uvCfSH>2BnP$l^N zQJb{+avA+=%hqn$=%D{?ANv&T1bs`dQ>&aw)IR9z5VS(_LQJY*zfl z-KnJw(`V_E+$Q4|iTAWK)Tx8M4LoZcseWB~)dDE!t5-lbU5VrOc-e&X-2TxsJX(SA zZHyTjH7D6iYta)#;F?I>y6_EWiI)%!;JU%XiRyS;6}{*6@e&1qi)-1nR^ z`R@Y124DUT89&s4+b`NQ+Qzk6J%7#uey)@Fn?;_J4=l`HW=P~ER-m09H}{z!kz&ES ze*E*{!+%-&s*MEem`L`)iU3YxOnT2=&xH$fQ{^&ch%O;cHs8xryT!l=yR#w89m43@V+TnQAw2MP0)^#@fF1}&hCAwGMAbdB*(JIVhi;ET>D z;rc~qZE|ni*>MDxVdX!_D{;RAavxOW;AfgC>CvwsX3Xf7@9f^9){8FA1o;3VI@tB2 z!9EA(Wo;gsHn*8y&j5?c`umH}vPm+NBMcB|4R}@L6szDV1?4@YHbp=8L#a?g1C;Nj z28%OVz@Eg%2yGp>VFB(#FdmXCpyd3%X#;2Ey612y0`9;62`0|ii~|O@i;U9>%&j~* z|Ls>}+dsk6iC^dB&AOP}s~G2aF_m6)R&zQ4VFy*mzF2}mK{ut*u1 zK{Uj(_=f&8k9k)TiqS{tlw&m~UFOBr*g2?|5&Q}cYfA382-i2*UVa9vKd4A+x3^&T zVR&GD#L649yjW?X;n!9t%yX#2_TFnx8zw_S`d)H!MIKcmjP5Q@u05i+03X%HYO^Y3iPO)_(fq|MY&e+pCI1J>6 z)6z{AJwIJsG9fcxv&Pi)Zbo)n=n6UHXMY_1=ySv6mVuVm!R>gGPt9uX@(^e*Dv~yL zXiZ8D^3KlIv|OAj?*Kt+++vQ7?NcGN-C^r{r<$tD$gq4-ns35o^}Fqtj17J4U01*J zGkA1qk3^216N{kGCboqXV~O!@iv&0AmNG`!z9S8)7hh~_*%%ThCZI0FBc1K{=h{P& z@@+HnEDt~7s^6#&0<7g>HChkp!BZm>U+MCGy?SBg@<)t@QkEUY+MJRpo%%F|_3Dt* zZt4QwRJZGY{s4XpZU^1DTy`?EVLxRmJ73~fPR{cS&Ar}huJ0TOmdR`HnHuf&e|fQ{ za{YkBb|d<-AX@qs&$(`{)~uR|*>un8@Kd`4oFY8ma9v-0#n*sc)Hbzk1P$Gq?>`&{ z=V4m1FSc!@jeenMYJ=?5A9Ah2WllY^FAcSOU zA(SmclpiW`VmN^mmY|aN>gc?$2(#Pc7xAtn@L~R4pFT5lRq(-9(sH2n_XH%0&ZNX5 zHTPy_hsRcAMS|qO7X%6lS*?(;Ik!I*JV;Izf1x?e7|KCPL0%PRR1)Vz=Q$-0UcswU zL2PA-Dc~?XGJ)vD`@n-r%|=~Qll$WA)YSXC3ew_%^y8Zs8!WA?OP#cz-@s5D99#oI8_GwOh^_9DIRW~H*N?|MK6vXi1PFh zc%dnw7%*&g(P=&Bj@%KOMG+-S&#|u#d(UR9o*dFj&x))wFHbX}14JN)M=U?I<8%Wa z80MYa>sL{zmks}_I@$H!fIluKBe8OTrGM{R4^bxe`EOez<~d*U#AfXAIJ2{;Oxhzk zf1*;#a{ieK3^wNNL*Lz6wXMaL_CltUf#VJJt3{voFa2++j&Ax5ng}gx5;-Z-89RL(IH?r`7r6jfco}#0<)YUI)3tbHj!QVN1yNcJ_wCHJ=bM@rU zP7MvuJXf@w>tv|t9+%42D@Z$;(p~a&t8pbyBWtMU`hLFz_nU_jWlbViZuPvo>=aFK zT%~b&;bgV#57{7X8viO)i87H`fo~q0*!^fKbE)efy zp1adXm;bEJ@8Da15K>1It;!$+-h1Va!+O zf8zkTH?VgH%YcD}Wlufsmcxfv!oC>RxXwv0Ui_}j8*t2iI}nPY8c-ZTw)QdOo6i3J z{(TSDN~yyxhsf!Gjse9` z=fB$x0^~KuC6E2?B@G>&Tj&TnJLStO*1$i5*axlKT2^roumNyKSqxGjqVm;4ur261 zX10acz^NUJh-BrlJMgFy(K+GU{{?ZWO@8ekhcCl+xQW_fUM9&+@g;WGEWdqSwjY_H zL@)z|9#MQ_Uii!d69^0VjW6;44WG8NueT2VcVGz(N6~l^&)h-41I(0+oXUaF~|Ha2#H9e2P}?yYAohiQj2AB|TDC_}QFaN~!2IjkHy;_vTV50` z?Y)k7@74QvU#MJL*qpv&c&6WnO49XH^B;eMR4G|Jr-XA_3Su_~!xgk|EIr&Co4_Ahbf%nClJlXT-#a3i{ znMCWBHw`@Sjo9S-xHIpIi`eqDk7Meu-6~>{{PbLQp>jjxUPp>-?RSnduRKpCs&_px zs46fpknr9E_dAa6#^t8byb9mrIlP1Z_j!9l(HjW)RVL}bT zMI(U+ZD%#87VMk&10HT&U#to~+3>!kZK#YfF;A7Yv5S6M{aSyhHD@C_E<3vm_ps31 zO`m_+?AgS|Ow(QbgE`CPC=dIR8;2*1-nxnvwpAVe)9TsDbc$ZGDD~8=2V)w|j@u3y z^ji5@t0}uGlFFhr2a1QcY-g6_rl^S1?g874cU=0GeNxH5fzNwhOy$=+cIdRb&%-6j z?uP+Rb>cD0V!A84;-__of8M8T+fR961<)-ibn;2c$5%d&5UY{`S85HnXGisK;VaZKSY@;jPd z9sLxsW!b@lw72kse}C@|cMJ*z4)SV{&Pfk;q*@=eG43)1y`0%?xKNk^%xH60w@i5m z7sGvVVP@z#zxQZ%_NNYU@Fp#zK;$pN0DeK1I0 zMo7D}kgegDvNO{=6axxCL=ZIl@|f-T(#q!&c9EzZ-Y}iSfV$HiTIcIcmSxafzxT6B z(88W{8_B5y9aF7boM+&+L`o;F627HaW&(kM#@1x0u#*>f3NS0uTN(3# zi;J`aFYr(TDty@(z{N|^EkqAV<6qkKMaJ|!J$jUNX#4gfF|5x3I$yGnXz?XF?(Xhe&;DB>k6S@LNLHJ~(g>Rl6#gC9zdB~O8NI-(he3!fz!;K|gT*Lh zJ_BJJzo7pIfX0N4vzETgN>VZ*q9@?40sRVMg@pHr%*SMYMS#GAYILV3!5+j)GDK&z z^vr4&D26~I+z5wZIZy!OQz42bh&tKX0Me6xqF!+qHpMoVa82TkNrT-dJdlshtY?#? zhH&y?&C?NWQ8fW0kw+vvxFW|DahU=+!^_azTV|2XsDOR|Hlvcx!?G`!Hg6`H45Hwe z7O(a}Vhss{hhgZQ@8X}}e@8@D1&g_`%cwnzwwlCAE)gFw%A=FLCwO|u0r~QTq#PrS z#7U5!)zoR$thw_B0ZoJ`cJgGEb!B6;u;1?Xc5Sl=t|aC%X0L zm&qsU=?hL@3>F`HpLv~El6wUwE2%Ch+liOUBqt|gTZRxz|IUHNMlTT|T&k9>qU=;u zDNye77oNW|HW^>Z{jF2}SDt1{K-o>&(U%SuPqi)jN2Wa%VR}G)=T19J3HS@FnG9dA zIeNm~tG0Hvl{L+qd1i-$1_n$K5}9B&u|b+KNugkY+A8VFelWP*#_7rv^`@4jc8v*{|PAD`L9sidTTifXqUh(a$M4+wsdqF*5OSDX$oM!Q1WAcK2q| zpZ%z77Ev4dF1WxbkD8^^zbK%*_l$zd+G`hNdxHYK1ICVYNu-I+H1JL>sQPEUV(9sx zSn-zob$5h9+d%1}(B?hq8^R|#R#F=GPKZQ5`aGzUGuLjpj;&_GqDc1pVE?tc{0DxO zi&LyD<_~S>kFuq(S6JA+2z|6FbzwKdf|1ptO6u+~eJ57tHv4kLo_VuvxIcVlMz86~ z9yUrgJ??(ulz+=aS#OwfatZHHRpHl;h;qNSCJIH40#!S5?ryX>=I563M1Q)jKXhwx zD%iiP|I3e$+?ZEfpU?B@Y`W?>Un=9I&a0ivfGGO;-kOOo zo?T2`e0crVtyPG`W2JbN8Y7mYsWGd@Zjo10p_?iHet^NkAKmO(R>{VV(H`RL_z@iU;5!^1C2M*?hcY)Yqm`f@s=*3T)Uf~X111bYph!MS4P)42@9L2{8t&wBBKfQRoog15-q-OF_B6Z$c;})99i*;ZOv;PHBcvj^Di#hq*^| zLnD2~ae$glugg0}lZeQD@8084swY*EPM>v&3wA;i8t=3cnvAJ0hN!0 zhJhp7_G|9`@PP%-Q^2t8$(-RsJN2=fU06t=UqQ8bbMWF&PEK=Pz5A4{L2J(R9ucc| zet?$tTet18u%Gi~$kTqwvCX0(h>)L>gJ@{S3(_r8ztxX6VKcOtgc4K8Gd zK6|!tV#t$f(ZYUJ#@zJ&KPe9lGuI{uO%6Mie(}4$?|eJ`y2yPAt3xwB7k6y-%yik# zKalT#>XA*%&eZXA+E%v+2B8c4%A}dTCO>$Z;&;_3;o$mtK~}nb5u{B-?6B;W)E9^Ok#U=&*)>GU8j}z zF82;dy2&_pWVoIp&&SuuPkrzt=Dh+8!ubp z7xR_FmK8tsF&vRO;T0~|Ft{-IL`I)Rq5DNyK!c)#OHaLMw#CTAv#QSKgOj%Xjmw!| z^Qvlnr5wAy`H;lPmiCz(bqkHD3eV3)iyj9V(W!}%e(zE851X>O(}6CdMI(dRX>CzS zJh+hNW*TbJF8$aCcclxqQ?_i_-tz_3nV-HdxPuwS4;hv2d6VsX6^_h)W6Br3-ep`a zhS=G*;hJdfZ*5NH4m0-bVak33ElfKE?e=+$_zO8tz5Hx+T=n)=kC~bg=gBs^$mYop zVa4>DzTb?V`_9a-w?AF-CQm|TiRTB9WwLy?*HFjaOsR7=2~0h*wxnf_GP_6w_5tT2TB68VO=v4dz?Ax9E^?m$0Q5B<~<9Y9=fmgx<) z@nP^L*e2;rkM_hOO$M+i$vn|d9qKg28FfPn6QZN&TaYCQ-VZzDs6H5#vZG8$h#G#jASQl2Cf5vOczg;{TsBl z@AA(tY$|AbrNC2S!akI-1~t{$m!v#yr%^ zh@KrweDSfsZDJcR^j}wg5&{+xRRLM&BDp8=hL4X4*aw8o{bUK_AU{D`nwq-7E7LW` zCbQ*5h0B|;7q$aMRf!LMDabbwge&2jaH7)F39*vhKf(>}38)H?L=`MbkEM97LgWL_ z4ne`RI+Fm1u|hvxl>%>gulCEwXJ1b^Ri<8igb+?xMt}J+85(vv=;zNsW7uh)zQ3O< zp}2*KvHsiKw9mc=eX43-sgpI=V4zn}a5nmW6n2Q&jxIes^WRqcq!|^{QVMQu{t8)g zsh$UWk`{XblR@h$3gAIS_k`x8`dG>(a^fr2ja)lhC*?YVVe8>#b0&&Rqtu7=LhOL3)78j@75Jp z8{S9?7jnDSO3#?&qw~=uBU0_OuFvWQW!2NVtFupd9r5Dj?4q@u?4RnNY2)B4x!iUHg696($7BN*;i}0E39fFyQe-`UH9!UeQD`= zQL)5*X1BL022WrV^Qelz>}dA5FMsIqQi=HQFRp9AE&;KW8Pa1$3&(6$3^5RB{FQRy z8wjZeYxAr`wz5gP`TEeYEB8V31*_MtO2d5x<>k+`^e(qXfX{*Jo^(`D8}yMxC2ZJ9 z2ZeuEis?4_%UshTv>+Ku#5#%$X|7-%5sn$53t;PA=B>boCItDKMR3>NTaWA44oQys zk5v?;im;hLhX75tssF|f3FK_yykb4kUsrz5H=!39>x{6xG0sNL0$LdQ0^&^K9% zxJ|Y>fuLjQKaZ;LH$?pe9ias-LJxzWcZ8jl5H%Nabj}?=j8ej9=Mbh6dqL#Ux&MUS z?G!-G!1~f;YPfS3A0Gr=;@Fz@y}Zvk{PXpDu%kLb4F(B^9#T+2z?mKsOJT{8UEjZZ z4flxQaK_f}-n919Ltz@wYCGIU1LKEKffG_w%dwY! zT<7RUb)2)01wbHZxnZ^7IU{BMhQ1SDCSe*PtqDAr(D+d8gq+;qQ5DCbK5bU4 zzojW)10tU{qV>nDjrptX!sAyPgZ%sFT4h-R*PeQ|%A~LMTS`EE@|)*JWQIOl>nNQ3 zC6RTuYvcFz(HzrJag|#*bB%n*2CLmg>dx8?40lKWcv~tTqdu9^y+g0^$RieL! zyaRh-Sp6{cgM>y7)ql$AkFne5G zDv{z`S+xN;Y(|-@OHwCX`rgv589)Asc{RR^o~^9yw3K7WEkURmpjxS+}kv zK`qV}3Bqaj!2TZTE-xlW4cBi6ywqU(y&SK^I?WRgH!*{0ZM#ynvQJ@WJ`{lzx+{ji2aAE6MBab&D= zpL(7_yJi0hLV;bsO_ZY^11DKE5-Zu^(eAES?qSfQgOoTjQqEp%o_=c_Ds`hTkL89Z z2AdD}V#4~_gwG7mxSzd73v>v%SD3IRy({)$Zm=UinB3p(HIxFv{)b*=*QQ${nREn! z0YeLndZdVkW3Axk=C*SHN(cV@;CJu1$to~S%1Vd+vba@v$4Mub;m0guVqp|7VT+q4 zjW1Ic=e~6Zr^dZ)+?&x`Mc4ml_GjRm4;h|4iB4HFl~M=lbtn|bj@Jk%R7Ci$3YqKY zdU-(7)kgSS9X0e;=TvHCL`NL%qiiRfP5j@?7=>UwqW@-0<4WtyIFNu*c9DQtT3U*G zBc#+oq6k`iL_xedx{c!O>I&`eF>8=Q(eLPu@TGsS&fEdC50U5Q4h)tQTL*_y=q4bf zr(C!Y6+s+16s%~dsSB{>NCSDMq9hO|f<)E|nsi0Q>O`%FNRh@ecPG>0v!_qtr@a|m z@7RR|u;Y#dvz0;=p!1uUo7-1O`M}eRO+`_`odm2x9bt>rZ<-^vQ6!WPDBS&kRm&j9 zAtNsWR2ltCV?42WBC=l?2%aO>hA*M$A&KmyrcthOypd2MQpfSVAc!kvBXREOPV##3 zl^YC6B;qnb<@AAR7+}V3l>06hx0B|OLPJh(w=n0j4o?y66JU+fH_@L;?!|;fM0Ao> zHk!%XAtB2`E57gR!JR_XhvZqtrCJ1*=*@UEx?xc{pEpz7+_OEdeqAUr-1lv@%oG)2 z>B4|s$p9=`Vh}>E-6d1v$|ss2o|%7tf5#R&GO+)4XR$rQ7K-Ru-Z}I`ZoAv$Q0!H? zzmy`NCLuG}@wv`R?!s*e!^~o$=heX^L3~t5R3PAx{ZHdApisdfl+kc;Vcs|ynQ35` zlaI=!Z6A)UKrHCTVC3Rx-AaG)VlUSD>SY>#c#&g=KW1A`G_#cJ0nF~TG5t~3uU&hr zx(fsyuOp-RfFq5DzZJHHxNw+x?9G`@`%@3(yr}N<@?8Cfp0_tw@zy8L!Zr~h5XqTk zm!%T|f>^ImD5^4&G*;FDm_N%)u)m?Zq;8jYJ+G+Q zwAk;YjqgB0h1~F(t_ZnW`%Be<6bcg^-_zv4J7OmmV}A7GM_^FP>^ft=^pVcr<)BW; zT$_;ItY+bIS?KJ5*_$)bN&wc6$t~_514k=+TU*xHlR+3p7tPr ziP<;cJ4J+kfn?_x9Kja{g@uhU4>$w9V%S*h@0s8)bv}6P5xmkt7Q3n5G|(=pM?NNmGdVRtXPm%jc`*w1lb9Z#B8@I+byTm zJkc?w!D_TM&%Gt)PKdQCr@7MTO7>6ZUmsUm zdRx#Q3fwq!6*t;Hg&!e{K(fTj=re~gXwnwl2JQn6e*X0!8-t^HmhKrD@?!H~^RWy< zeFC@T%??Tc=sG%s*crUWKoBwEJ+=jXV)vmh;L&N19@cs!@Ig5U%^xGbt}pcL1oiIJ z^}*z~9aA1?w2~K~Mc#-%dH4fwsv(#+=&G3d(V`Kb4v_F@usGSWal^PLBxLIw%0L=5 z#t!(;ioW=g5(Sw19nJFrTm|eiTsza66?oK*1}(S^a6*K1&{|S|QyyAWQlk)mW(uTG zL7|dB@`G*J#$e}cMfa`zgd7)_sp0L!y%yM(k`y=hDxhWq!>t>2k{QAFMfAoC+}LOM zqkCjzvLbXju#1-|^W5*6cvAyP@c)*chC(3T^F>6SCX^3sT|qfw-xU@d)$f??#jACk zU`~xMJ6G*x%zoR(z9^dX@ztbbO5fG#hm8%jS_h3gJauJV3cKyO&*uy3lqnpb=gJdi z^s@;JlTgerZZIOB*I?<%b4zDoW>+@H<98K7e-yWV*lM}U=1okQ@sZ@xAaQ@zbWMrh zhn!kRYX&qDDXq22KMyFsy?EEQVOM8&{I%fMYo%58IGcs3UXr86pGdx9RqPxb4nw(z zKECuJMt-1ngyqQ_wYkdy(;$s;zPMz8ecHG$TR6ex~dB~IZO?#9Iok8 z$l)PNFNo;$Etm8;Cr@Bkk72tERO%amCj1h7RuGBs1hM)kx5F_d zr9zZK3AY#t1~rR4yu3)w*$(5$%to7i%r9YoyJFR<31V^M58x5Zj9WN*zUlbAhUmcd zKaE|0OXn3B-v$>S+DpM<@%jGUlzH8%EyUtR7q6WV8gfK>dwcEENS3sIS!wCvYL7oc z|9K}a#lwEW0l9VXX(09!$gML~i{?44rPU2mqbRfh*biPsva9#an}6JvFk|y@+>H1i z88&Vt2uLSD=k+u^co*G9D-V(^WAG)A7gag=5JxBgZ|>p4H{$Ev)HKZ@C)m{P(#wj; zY&9$$LZqR@Bm{Q!i}JV!J&$Z)a8OlMr6-on#5xeou5m6%3vlTrOD~92_Tv6o=*M)0 zq(9&I9`uy>d3S}mn-uD!RqBoQWFwi{XgGJ(vr9UAVO}8}p_@Vd5zIeC-2$=HE&&0i zev;9rt<4-87k6%HRe2JVT@jqiN*r;0vEjF(Bb+`|5&!lrGg-r7{Mt<_SgNc0iRJo_*`U@9?-P|LoDCf@6nwh;64_M6feB?Vl zTFlxvANJ^}&0M4u{lJ{e1=ag%YmddzZa$p;^2~4Jj{DP{)U0g_7AqLWJ=|DHee2WL ztsqFhZ$kL%=9QZo&Z<=#${)T?doEB!V{YL5#^#Q^%$GE>>7_w^JknoWLX@eTB@(}l zWZz zAH=jm1S>FpF2u08(;XVILvZQF0-q=pzmtA#CRQ#`jS_2d zU?9Z%4o-7w1_nd>>a8=oF=3NzTS(P!-@fgI{f%D(aaIH^QW5J9sPy35y-g)T^eSq# zoLNRJGz5fMattSp=>P&fxR&7cVY5>rIY~+yR*wAv!lPuMsUB=mZI-QQMI5 z3LvHnH-h#nQ@>&{Slia_N4^0WHHh|64QAl=U}+JLUcv#z!a7WFyV_z6bXHh_G#hkJ0r+I+!1A;VpwWN{j1V(`ja)zxUmbWFfFkAU&w!6}x z%t&N63$c^mm?C(Q2A@Cz-DJxR3TXgaCnKNhvwpFz(M-N$qFvn4VR4uPp4$422 zQKY5S+{9R)AzEKmwK9LY@-lwt1jgA}-b}`*BefEDe`s|ZY2J`;+gH2w*rH49IzQj$ z$!so(Y)gYsE!y(Cr+2)w^vpeN zjM8<^wzV7XGla1ov$UJ2z`G^#EB`aiB$Aw3fKq;`Q-DG|g1}!VWG#NMzkO376HCGX4hH)A~;6M#DOw*=d za3S$n_r)Eaf2v5x2ar_?K^|AAiF*~sVB~SgUB3KgT?OXUD|k?sP#=vGq@^T*hdGTGsL?h=}>?5Gc=CaA;1MmaP;0zo+F(-m0I9RN)eVBG%etq#b(qa zu8)eH{et&RSHrRZ!@UMsGIll0xlVS5cTTTsksnXKntOx0xv+Qe&8AMD|le;r)e6>)!CQWbI%`Psn-v zbY6~C!=SG5+-zTE>w(-+hep%=$RK`oY~oz{dhfa$XxI`myY{SH`tu%Q}Oyxa7+N z%ZJ-g07h*h1}NchorsCK8oh{XcAOKJ;?lIULhf6~;LI21P`qHe8Q=f`hA04cR|N!I6WTbECD# zryOCxuD*W17OQ%nJ*l~v=H_zXo^da6 z^vwRiXG^SZd}eNMmnT|KbWjzyZyi;pUqM`Z@MalTB4UJyNO*#{V;ym~lHBNo=g-a4 zWH9SO--}jhu+tQp4=Lwi3OeVIS|k(+qpA3C*LOx7MFmJ>Cu;e6cPwr3?-RIGMuvm4 zj|(l4$9%s*G11tg$1sG}0gP(=^gMuiI)gpu;^HS@V|zFH;L?o<6!i`eTU`LYDCp>_ zZR}q>dbE|GS=i-Z_#RU|gx}5CmxP5cnmVFh!$i+mE{eoOa;1S`S8MK2wQ474tm6O_ zp(OCJg{+YwDm{0+*3N|ygw6&O4J+hAV&;h*2 zHcjzI<-uez{9sZ-H4^$9sQHW~4jkBo>*Zm9pBcyW(4!v-8Mf|6Ib)-GV$`~Z6ldPE z5?_o2gYGTeOXs~Up@N$u^XZz&=Z}UTg8;1P@vT;G&!gW7do4=RUnx?&GDA1SShK7f z`NG+%N0WSu=Ov*Vk}oM}Syui{*-UJJBpqGg(PPJoVM4K;y6DtEerzl;?|th&o1t9; zgFyGbrRJRM^ONeNYbHVQ16k@U43*$Zn1)o9Mkt|*bK$`N3*i7%HTh>X758sb6B;Mp z41Bbu1m44n;Rc|-4=}NDeT_De&@pkC&C)tLI+g-+&CJZOHH%%?i|j74(Lt7u1<0A5 z7PL}fn1~7cFd;eF;pgb&WI0y$l8HOGAl zQ1Z@y$@z=j82%6k`%^9#-&jCNC1Y!0o6tfP0V#OOB%sadf)$ z{V364G70DtN*9clIkwawHvwS_@RKvbmVnsIbr?Vq#97Jo0e^AWZl%SIn>XKpX~1f{ z{8F%pxZHtWgEGlL4Ak(rFIey7H1|<{LX0g)t{Rdoj?NHF2{-#*kPLv*JoQ#^KZWw` z_`8uRscuiAfZn@JNF@5>-M0#*Rsr2Vzuvo@_0V}P1X%1 z)?{Jtk5AMasA93Dh(y5V6;w!X!d>j0nkqmNZ2=kae_2}o@~J1)5h)%hc4UQ(u0!qB zz5^5uO-*A^rbRs$Bq3Y#hpEKJyT}ZQ!awJ}c+emqEGiHMS2I;Vki^I0`i6z#@7FrX z#Qqe9Q~q!;T{^&beG?|JDJT1E$ai)&_)!&v@oK5USPtk(LY;2Ut%NuP0>DdCUC9b> zAv`Yb;|BLZGkt9(jVh!)1jhP*9F-~#`_0C}Tnme7;w5Ta3AaJKRDxkH8lZPGWrSY^ ze)6UH4qXEJhY$CkUx9mC_h^-jle6c*&`>bEpkOxh_l850LN&-~KU@EyBcqUcguUaLHSfUPno8Spo)5i?Vh!8n}Mxcwo z0q8zC`34l%U(LMnD||^|aQUvw!~)>NKRzfuJ$*r`GU-hdP5_66Mlh+kiw;MT>|-$L zLoGOV6xMSKf5P*RatE=YkbuTWr~EBM(k&K4IJr`dKMQZYQP(DocuZ^Wdq z?<{vk=XeWC%@0hsW&%L2Co4%{1u;t_S2+J+4UM~G5MVq(zFe~J)06{U&~aRue2_TH%h8&4XP(fTOQ1s9lv>y&Be zrDC6lHn|MfAD08<*<{%bwN-!hEYzMrvORV@7#+YLsEFnyyuC`+2qLPX#f6~|lXW&Z zBdhSqm>xg8y!6hTjxN8|ZHZM3U%iArS8cJEe>Wk;Latk--QFqMzn1VdGs+&Tmz;4q z44Hda;GSpWxx6n-GF`dESz>!x`I@}X^P};)OZ5*=Ysw|42qMHC)&lfQ4pK|1^9>9n zTsbV%ng*q@d;p8$uRR^;$oK{)vxDma9(~B{mK;?92?%Hqc#l&J*96(wX`DW-)Q9^4 zBbzXaAbIZI<5ut|_l|i^!9!oHfzd_#VXvpm{bnl4-=BuuRCwA)MX4FaeX;8SbMLIB z&tDUZP(g-q$G>(SqHa?!O{^R}%UHe4g)8M9z)2fzB#wzet-yHRvXK<4E9 z=TR~V-A2rw8%`c{&vtOef;Q9@7c3o8yYRzcVI?@wPZj2)LFdEkCtfy;vpiUIVt5;N zronK6MrskDaOjEM+_32x(HdY;q91w4b|A zhiHVsq#zfV12p_OGkr;{4<~0Xy3LK0BW-j;3#`kFpqPoY_DHfRr`!t zx0boH+nb?X#kOYGk04kGEq&GUQeX2Tx^ciYaFq8t!X#6U(GW%tRgx}VsB4LdSC{IV z0*Vkgo@m3rY-YE^1_`9jW7Y`CiYW2HM`uP!f(te#WW05piAiQWXOFwh)8yuQUai`b^?=njkO>e7Lxew8H)Lu`qZ)n~s3 zd#r#X=z|SvHs`Vbv+0noFxBdSdl>P)#eQ+hBiuKE&V}R~o;kBSg9(Ifv{uvPKD|>y zU=sMCrb$72NF*nDq(LEH%eB#xrWoW0y?Jv0lU9aUJrF9OzOc>l2@lt*TCpTQl|D?r zf&eerr5y${)!zgd8!^2`3pwKI${EXcnP^&CvrbuFadqwDk(u^$ySnN&So|nfOb!k~ zG{NYdVRBvE(P41Y6dw-pInA)?;Lj>!rGW9B28%RcHN-1t@)aB*UK%aOx!@W{TWo2~&;l?~mEXf(Z7C3@t0IwVa zON(@!IYg=404-GTbOXvCIB}1@yY8ZZ&{9D3oKT=zXAcqAHMF&xTJGGrgUaBL*=FFG ziL_!Yl>7&fI%$HVv7bBkYL~vl#=H-S_m)cBYtEwO@{(^Zm6JxVuVK?6trzWN(Wp=x zEZqG2{d3ErWIThZX`8VSog53naWHU)$_#dnygQl31$P zygW7n%PpNSbh)5ukYr{U>?|YfTNscjn+xQQZ6He=_?TQuK+6l*!xqLXjI-`;Zp1a& z+s9`T>od**5CU^2+o3*Vv;b?l6u<^*eXg2{%6ej>1jxfjn%Fk;@DMZB+vxV9h=I=% z5eh-o%(&f-FFkQP$;G=}Na&QMJ*O6_$oQIuPb4wp14Rj!teK6@7cdubx8e=G)*wzU zjyub-14KL24HzcGUM%`MB)YI$H^Psjx<@RZlR?c%0=qWQfcZ0g)@>oVS`|!YUI^4I zC8S;;E3ArDar3H4&Yklhiq0Nm=z=6HSDug}wVXrYO%z&}}qHXL39MLR(PtQ z2ZE9@%sAgyRNSdW63uX5Qfg5{L+06`McgSsIiEE;Vn1`zbLLcDuZwbr$1ERF^<|hC z8^44t?v{;TW!0L@idS#hOr9|o+jh22S^=K0RzY%G%m}BqQju?9h6s2)<*vv#O_v=H ze_NgWCO7X-;YcmJmcv4$zIK!Eh~1$V(L&2~r+9E7Eq#r9V*d>-41`F}2p^(P6q$oK z&}8PrxFZ#EBJL}gZS>%a8=s(Gp`oQ!3=HPrU;kG}o*wrQ4r)TtM4JcOZm#;j9_Cz` z*kUErNotbEy0!y|8Nz6nkgq$Dk$m+lpSLhrqVJU$!!GSOoK4~U-784L8G>QA2wc0Jp+&mcjpTWSwhmrKbG%LFDWc!(9p` z4@qr8Ov}IVBkt+br==sPdhb7cK+MHK!aN0s5mjP{b2OC$^(L zb>?9KcqJsa5vtggX!HS)9D4c&1y`Q9lee|`_k`U2|IZ(nf6|lr1AISO0w7@iaKc~* zZw)EJQMh%hoJPC>fIV?~KPHgtO?IuzHjHx;ktFIRmoPm45ZPOC_oy^nhYq2iCI_P) z;jCmbCV`M-tu3%kDjg#o2bdD*tBm;&PY#!(!hez~g#)&r#_rg)iy9rj@c;lX=KZF= z0f|w8AjKeXk8O)0--$gBm?N}R{P;3*Xrl%cd-6xM!E>_{9G*v#dJkUpT zP1v%RcE6NxQN*cBy3Q{ddkj9I7k|yUgWD6l8Bn=6bG?5x>Bzve3A*C!u{oco2 z=9%aDzy9~D*Ua}jWA5(zy3Xr7kK?l)?eA2j6N89t&D_6#e-XkU>9TB8X=nV+f7TEl zXMTFHK3%}aZRLgTVqt-tjfu)|`TQlUGGXKZ>&U{#TPLi(ryMGqq;ezk*?(lW{M6O6 z5o_r1%8}LL;ltSnPo(>~9#y$8F-z&}xUKzEOUKk%>d-9n!Ig1QSH@4By{Txi-!TV$ z&n~~0-fX9s@e6SG!FoF^dS$T~g{3S#*tc3CC%$Pl&2TZ1(I$3hI{%K$){8^5RTY)J zbLU|e5Zi6D0x~gyuItsSw~BEOj?w;Ak=Z7~yJYTy?jkFz8PApkTOTPzA0j6+QuIS( zJIx$wQ|o~!jFC2)`FtOBim8n?Im!J9z~hP@??<*T4~8;j#F|@gmaUG*SI0Qi?rzU| zk2pWjGSSa5+8TsVboj!$sB36+bnXc4X z!<_2zojY_GNE=OhuTYnK520QX{}q)T?<#Rb(4ctHrF}pqDVsAjuc|YS1}XwS4q6?S z2NbI;;1-Ley3|!#1-l_LUJ1t)AO?VQaZA3^T8eTp8?d{t!GqJtmWW|Gxp& zsJ~J2VJ+3u0<}V6bdW_oW-5s#xhKXtVzggL;C1Q{VAX)fPCObtm7mTYHMo`OdY#_Hrx+kRDfS}#`e>wVv1KM$9mqpd0S@YkKC)(DsFrifmLAk?Lfne7LGY|E_q=^ zPY#{714bnJ{%Rs?x?o^zrE+Z`n$+$ zmWd5(F5S29H@Q-_6ECaG0FeFBKJ<}QcN5Y?|$it3^QVQg7hShv=B?lfUH}$qC{d6>khf6XcFVx zU-&(q_vXl&TJ)W5=rjFMPWTUC*TO4|fd9bte^@j1L8?I87y=O(i(mrLNqP%OiL|^` z+0DJc$Ef-Nf8U1vT!to#5t!3-AkGXs1N)868#in?L%SuZg51v`K0anG`nRuIxXIJ5 zK=eyyD0ND+7A^X1y2$r)%T5Rsbe%IrF~g)ILlT{OhRW1}$RhWpUQA!YD5fNe4hq!K zfBxD2T@n@))aHzPFf6%vEi^E|dqZ^&A-i-A-^@v}2;q6qX4^;Dy^0-d@=`t!>cla$ z?_Jo1+Swn@wuo|Ye*I5~M9G6Z>SXok(L;`vt5+>G?q}R@(6FJzq%QdRePN8llqoXG zDTfTioWdHcqP7tP61{sD2v-(4;fQ zqjK%id-jRzza|bVZ`C+)P^m82c4f{|?LmzTkWD1BC!Lx5pc9lEss`n~R9Yff>fCwQDdEG|#KaLnv$=FV zD7K_-kgH1ShPStqq;725^g$}mHod?K5wz`Q?H84^S8wVH@EJbhYug#*de0;<<)l>-@8hpYk{i@{Vv0Vy5iUL zkC9R1;NBABinm0XDby;kD1Lc-Re~vKpk*Qsa++jdG@!lVij2 z+X%~&MjZQk^Wn-yjvTq?-reoG?b;PdG9>h(i~ud!V`m_+4Fv~a+Tf%xcmII=ImfH; z5A5};Qew|g3(GMEhVVmkXf3N*vkah|^h}|rc@PFhpyns#pVQli*}Um*w)jmW zk!-eV)!DhP_e7ac!hWC#wC6;X;}6-rEdn-bn=Ry0aVX`jn1g?ZMPcB4Vl&H2j9Mu# zc9Xq{RO@+Cj5>!u}t7da*)_ z7v>anQ>@id)d4WakAV}hLihD}km7RwpEmwwd&*;*KZ}l?w)LcD)6#AM#hJx=%YJVr z<6u1(kkW-rE5#}1Twga(ABo%bqb)qR#hp6HqeDTKzm4p_&WnqoLlvD{c&+8*yWx4^b5`QIrepfQKH;ooU0I zi7`=6DTe9<_Yu+qKp+%o2`>HTJ@(lFoeF?%nkW#!8{GJKi1_$FL{# z4riWzoLqup;&=-=7tb-V@ZV}7e-5frk?TSCqG3&;&Xnu~u-Vw&187HROCYl>k+xFY zdj+BPP|r{z28~6rkqTK-9_d&X?)E&0tdgPy$y1B13?@KmdZ5zQvZR?})dE@`81`@@ z`Y};RqR=2(X4~X`^I|8;=qPP|4|1&4f*G+%7ak#H4cV}ytPNN(@t!9OO;XmFG0^Fc z7P?0(qM(Ffb3!t4Av^jM4RKV<-bDHK|8@B_gf$iRk)o{#8DTDSX?NV*vY;e?w0z?Z z9c)r1_oCKFG`&0sS3MYV0Cnm8D~20_Ek!q7Rf$>(rSgu_6^GfZh8`uAu&FC5F{#q$ z?I|+`3dGD1^ZaMazN0;hJl_;`WP`n0z0n?Oe0l1AMQYLUu)pfpbPMT~vnI0b%plku z?e{e=kLYA~H(nd0y_4fuiMW)PF-pm+lFsw`pFh%vsN7Ovf8e>Z5H>oZKnr^ujNI=? zyW_{yVkTR+>NLV-lQ{UPLjzBcsTmRV6cwepX-u@fVzz9}2v zA2`Z*AY?=fMcw&64onV0_s@<_4YqpCmsg_EP+2BqIPx0dW3nP5(L{Hrn!a${M16Go zHYn>@e(Fzod{fx=$@GruNU}VwGE;Is+?7wqK=W>W?JP6>Ov zViD=6Fx*GkZ#0*ka_GGBTl@<}U)Gh?3V>@IG53Y%ixvLhb+&U6b0Ll!i0aTTvh7p6 z6(SP&3cN0&LXD=SQ*Kp8GwV$bI{}=g$2MG6I8o!nqPjY|no2twywtKT9~_m}=e=CO z=fic*dA#Yx1yUca1xRDuNps=*LmDB&e+KyHy*fJ1sZ6_#-*VSJMh)eU@G3pdoe*FI zy@~tM!QGWaju|mx|As49uC%r5)t%&3+R&Cr0uV_76 zo>X0jvsm{cvEuFoU*gzc+}(vo#-+RIT7MG!zSFKAcW*W4+CRHD1oTCaBNeXP$5Rd` zEs}unP|%C}?dX_}7K3#UALBb@`&O=C;|Yvix30E~=D{2ANw;22Ia2X+HTOLxPXkML zJ#poY-m#q>`^9gv0?B#gh;lS_i#JnQc{BOx+|98r@E`!2VY+cH(o{gUkFFqRTkj#4}Hi7AUbFi{Vm4Ab_0T z`Yg(;$4YG<8Ki02K)jj{bzQ87ZP|~Gz|r^DQsJ9-Y&am>ySR@cWeA$#El{#%n{F1(Dt&KzT<{ zTxT?l+kw$XCmr(~{VI04S(Qbv{pT)pIP^H4pbCj|8;&qLm0P!;Ui?1>{k}erPX6Ka z$1LUdjX#}!mARS!XsD)To|S|b zDdt&K6NVEeJdRv}Pjuna_P{IXB^8rX*XG$y@!>c)-1AOgUL(@_?fxd%fsfopDS-HTRSoMJLc zz|vCb<%bU+MiFN8-vmzerFA*{M42yCnn=iyJ))s)etoZIT5lY`yO ziHUF&dfQ8Aq-$_7ZAOfc@Ls!)G&Irhk75I!x}x+CK@$x53cRI^$AMB#LpSUyiact> z;JqwFJY#7E`F-i+)$s?2?q+;vL&NJbRsn#&NHcSQ3xo_DXpOdJ%e1p#^|$TZnd&Ny zoj2mmX8eO|9xxQ-@kRUiE~OM}-OuCZq08*bSFc`8+j@MaIOV=oaHdkFNH$2s`_R2w z?V^t#f9Y)7KlSB_H+^>k{9SqZq&(YG%8$}N8=!EzXrU1BEe-j*Z^O+k9_S=xb@FU> zxcT}Ekw!g6`D^dD`qbg~s=R}yG?$i|D%ox6CJ<{G7=~X+@l5xfI}++mqGA+5TEbjU zp4?^6f>(1Q$9DQ&UJjL_Scj4qXd_~9lkN|)vSezEn0_xI_Tp*tQ`73k(P{&_`}wVT zrG~(4UFpK*gf+Zm@ct+g9%%s2ojcdOTt-XN-d42LIlS@^l?-9VVS2H^7~&d@u*`e_ z?EEw*ui#4fx1;-f_6N938wwGPaOT>122NE}6jnMaFv*fc!pbTKT3D=ov=SauQ8^3! zC9n4$EOz0qv3Qe#(_hOTqmTfVZQ3w>{XsQW8HdmpEX)W5SBcmNE$&01xZV512y?qz0*EUM4V$HqccO}VtA zFt+UA`}cDRn5_HrF<&frG~mL@%JLV(=8~0bHDksRyTFM%d75b8`(#2HZlb44-+Fv0 z*rZJK(Tf#{8I4?o<%&NMO|L9}`eNu@nLY4IDj{7>n{K-@b8S}VK0~*f8T)2^e&d3- zx>fjOjX^r~)@XYw9A<}SkDKUzEzp0~j6Y4&+pIrsq1!)pw|ag9^V$DXwvEI=3b64C zsU;cyu1oh#jIyF+Ci$=tL$3}rw7GK?%Q!dtzn1Z+5pS|4pu0gt8yreX#g3sfcYO%s z!8xAfI&a=p(1teI8#`BdzR(sF*AA}%sL+`WDeTC3B7vC|h8{Lx;&#YsVNLnoUB9+x zdKL%{fX91jd!Nsys3)f6kEc92pj=U#0*`m5!4yjZ;cYH4zU*kUj5&wfQ3&rqc}jA+ zcI2-ul||-?3;YiS?4wWhHpVs>akWkq?g=AH0@NtC!CK7fe| z7!=EV4CN)QK-J0;I3K|R>ra0vn;-#g$}$vJzn!-SJenBiglR2(dtkXLJ2YinJb#n~ zA7y{nLhOz~giNN(nM#u?z6sk~*RR**+8MkZ#R_|`1I4%_VQLWa9r~D=c6tx)`OHx| z!_x|i<>+VmarwWzK2}=vAU|rBPG?k!JBBI4|4}T#6@mc1Idyu^o&DUOZ$o#qSAJ(w zMNVrrZnj3d-Xed+lj=QD#i8Gb0VcUa=hXV1y0c8P&Z_a|v+cC<8q79ru|gr1q0*8* zntTzFd^Kp$U}BrMIGA|a5D;wJzPF?uD8v6r*>GDpCiM20$gu)$J4Ux~2Mh!yp0ZW6 z-^VMdRwU7ajk!&k%qZoOEL=0?y#v=zYvp$xQNrwvkMAc1!XSm9FyQi^pj13%BDwDR zwV>2n)S%!zq9{CGK85iglA%7^@EiSIN0jwFi$ChTx7Kj?>{eA&?;F@DTa`i-S z_Vei_wVWSc@F^TSaiW13B&AbbiUAN^A&Y)+#0<+j2BxNeFgtT+COiCQ*8aA_(gXK6 zkYx-XK%4D~AH}b6P;Be?NB@mvTfCH0^NGyp0YLox7KrG0T`9T2U7Vg`*D?Q*KGOSI z;j5i*v}&vZ8^y6Y!OybISwL$Jc>8%(~89@Y7^?+ZcBOd59pFUwO^OsvnGfDX#as~rbp^1RoRCcb|< z`9Xu5jWVb2*kT{uyQ3(#<>hV202e0c{jB`(>zn=Lx0ei7(yzK1t1M~vmrawQ+BF8( zn7QNZj2nOHFQ0V&+tklNins6QW%CUO?wm0!(dEK`aVP(|^UG{*x34=7E?-~Xu}hcM=e^SJAYOqr z0<^6KtfFS1ck#k=w{vlE=u9^cIVVsVs|xJx@QT zV!K_%!c*OIBRxGCPo7tst=W<6C5Y-rj+q370JAT~`Ej( zM>v{HQLv!-@VxVC!Asv?(Ut2fDptm%e(RWR+qgD3vSg@?IJP*=2YZ)Ha~B^x2V8~? zlC-u^@f&kR0D(KVDtt{zP|E}6zTGy9><6LGzv9Jh`}+3lClUymdEGPg(g6Yp7?WVYX@`lvitH5UMCj)^7^T z|F)p-en#{*f%rH5;6stc-gn61q&$``Go-6!@Qh=^GMz|Rdl8F?th%E{hPyp(S3u{X zb1A*Pm|Vzx~MNS+_Jvkem`B`E$SSH}kh8a|hM;S8(%d8e|A zvu@#1knd+nzEfM@Asm#mW2cdNkp5Kjf`J>K>vR4d9^HA&$-g((KUBSA_E>gIpX!E`q3fuC55ooSwVMvXh?~#)pm`t<$w@*AzGHa#wF}@4NT! zw|jiVCdvW#ToWWoV%}=mNK7wtW8FWN%daa|rm+X!TmuHR@@J8C`ZI88`6 z*wf^m%PverAa=ugR$L~tL4@4sGYFarnnFPUPpEmXchPLND~VbiZ1F=|Up>f;O|k>kmJHE-Mq!Zc3m$Yc?Yg zV@N@D7q>0i%tqzeMesu?0j)VjNdZ)o$AWGo-F?B-mLjWc;@yf~M5sjeILT6VI62N( zJd7ahPONwZH?Pm44!{3Spyrn(XIqRL7c$;6{yC=jdYE|FtswA3Stvh$Cx&?x3@Exok(cO+-*_v?c2su?mf7YU*R6U z4g-$3?|yy=t}IT0dTFz6&)ZSC$S6TXYymJ4<0{yPD})1Sdq!+#W2lozI4ziu6|&g6-tVj+S(*}HAamRDe;W4ue} zGE^d{zj@`;c)Gl;5iONo(v&igVR~_1I!+p6Jw4sx1da|uo&Id?xnv3G?}$aWgRM?; zh)xeZUTfft8G50|o2qA=0XEjpr8>XzdeIlDF}kP*h{d9!$u?;DNEG{tl`JHR(;%i zlw#+W$+N5+SaG5c5n5CZD#%sD7{`k3ftjms|-THao{8v-a7bB%jpFC_>!-}tQ zNjeA|0r-5}m~CANEJU#Pq=jr(X5J+>#-3!yP&>nwtTOL4$8s^N71(E9&(I^6DaS3X zt#9m@Tate37NMQn%r*jb3Uw+my<$>~2=9eFNmOyKSnf?-M;vSV!nd-&RH0w9ktKQi zW9X=Sc=(riG^+vGNCNEZeYxB%4?#KaX-XR;CdtG}w2pvuAM3rOrJ7$sU0HR>v z+^LXRVEVbqZ|?&=5KP3B&!^Eq2lY>v#??jwuSB!_a!`s1$)KmTqJ zzc@d&zh!u!(o4rlS_eG7I%o9uwNw34x}lY>+$Uc7A2nOH9B6A>U)WSkf4ffKFDR8O z%gy@x*0cQBj^qbeTJjrevRgGTz!r`sc@dp1H#{+;02(qpK7P`@QPx#kD77dMBY)+G zp0Iig)vPa*+9(v6sXDA}z-crz-14GV{#t_-?k%g%2;Sg2cZ7rl#)|uj{(%p-)v#qo z%i^Cu7XPHB%>VGAwZ{NL?hy)D^sHf^tNBrUd@yh4&Y#~93;GEIm)2r~WBR)#*glk? z7BxIp@((s`L0R)Atc@P6lHKKZ!Ib7=Lj1caTkdrLcI0|% z&YwR&{b5fN6QINzC>RkAAsy}k%9Zo@id<~Hh6or$P(dLnVTRzLev_ATNkt?kZy;0l z@NNk3p>LpDd@4EgglhJZkc3?vCL(4MFp{S?czEEHZY`+4{V8yypGNO&Y@n~J*$34d z0eGaBZe(Cn`ZL7h3^3h54O{%BX3%(G0F(~gx3dn;3$QaqQWk8FaYXS5wocXp1(*_w9b^5&F?!tw zw)13&-RQ16&1cI&W42++H2J%WWr4#Ur?$& z_d;^-s@^kW{N0BS{rOC}^va+|hEryDZ%G)ZzibezCwRM}b|Ay9Lw5P7sk4+9e9qft11cc3suB<8ozRR9yxGXt@WxWD(8Xwiu)ttaCCw(9F z4-rH|j6EreAsNSB5l|_JS$GTp1S>xHu3l-6uF0QUk3qnFd zq*@^vucLF{vaXc9h&ooWE=Bp^C%tiK&*2JQI(!mX1RQq8ix-AjAkH$ta?4EEv@0AC z{*!zCla?l9SQ9_}TEg5j$iJ%#71dJ$qi-K=ok&T($tt|j9G6q|i%08RwGO;pBktK~ zZr=V;Q}x&QT)2*8P<>nbqE8JJhuAid^6;-KC(rxq@C(y(Ms+gAkACp+%I#zO$2a-7 zC4Ks3N6+oGWlix)Y|B$hl_H!=ty;AVr?fy_L(71E;j>u{uXST0**$9WuH7-^%cvy5 z%6o=h3y+QcquZ{6AjhPWCmS+D@A&cKja?}fr8HI4rBwwgUa%FeFIw=6GU9iF&ao}D zM0mAp-ls=dS$<@uXp)YG^kt--K&CO64PEHcRl=1v|5WBrZh2?+z z*yb;X#xXSvUo9xu+L=D=W5|BGlm>h5*va zXl*g+j-lU+4i9&>87?aMb=k3;5I*F@Rd-VQdvG60BPW0X?}nsi&zL6aL9Ia@s6f|3 zb-~Z-EgpHmZ1r zp?Jns{Vm3Xp?U;*&kgx?&RJ?K%x!eg#QE%L;?*7)1Fh8(PtT2}8?V(Q=gGN~=CGVv zrupYeZirH{$KEhx!HvlJo}L=J-RqPLK-cCjwyl8tMQuKi9rLCxKQW zmjCLeMUgmlP4Rb32p0Fhp_04lA(oD)>uD0<3v)D78(8(Q*F;4MXM{r zCVUF*N}9)>Kc53xN;sXC2WLoII>%Y|U33pmm93cq&|oua{m+kQho$chSQ!}?*FvU7 z(}DWmq^E@gXxFb_Xdx6xer(9joq|T@^oHrKEHE$Qk(4Fu{^B61AW%s}^uAx}}`}HE* z_Y5-oo*%x2D%{ZC_;45!7$Oivu0Lif3TT|Qo$XJ=9Hj`sxe%52b{$I)AeZ?sV>PyC zxEJMTv_t!3IYvg>!RD(5`L!C4QzEgUB++tmT>}(MP5SjTDe=pTL>?JrJ1RWzdZnDC z(sxmw6qXbubw)f@`Ng+lr0qwd6xu;!%@~@ODCQG@`h^J#UJWKYW+WkaJI&k*Qz9?L z-ER*7S}}dNkCMUqAL13`bhrCG$|-*~yUXb1=G}Tu$S7=*x7orZXYA5`XTHp7ztghZ zdC_jeUS*2Hl|#>oE=t_9H*&$UL(o-Ie!Xh+9xS!BvKrcg~3|b9N`eXNPQ$cIjXn&q;AZH1$M0MyCl)mVk z@Sy>M0 z3HV8fL1p1}hxfjDpnvEczor?-aY5=iE1LUajwn3|djyc)aio0%CCZeSg}#;j*kISN zH5Lz0K}<}X1_%i*?t^*HXu^8P$n<$n&#SO;8yg&n`NZ`>DIc6E!e|CC?P(I!AGw5h zuSK8A?8Yl7q0cy9R|We+G#R=VR#&uKWMal--MhDj=US+AV}o{_7>O8^4@s!1`NzsP zOZq|-1@WOgznV^(G)W>}N=i#@-&28z%wxyU*Zs<$6z-+I20gx;o!w1(sK$T%p-&G* zT%w;W<)1eyhX-cIK2pk#v#~7ccD3kjXD-*a>mApH-T5ikTRlDP&5~UIOO9hb)aIKE zvb4~z`(xTnAEhO83lmqb;U@8N=`wfDoI#^UtGDg%y>4p!*nTn&2O=0`W$n8Fv+J)H z9g2#gThdVMl^V0odOcEE;Jfke-BE&fa*m^dnI7JjJvvwn9~3MZpymSRog!F47FEBg3e;Uin=s*lLYHD@Sa% zV?CpnJWrMXs`6)8WxL2<4=c;2z9Q>%Wc5?4t7*ytq&ui3Fw@baum60AKpEdzQ%X(< z`!7|V?cj}1KU%RmQMKt_^e@>}E&l~+e15XmF+JUyt7t5d?T>9eGHgH_*Q`_9jMTkw zJqaLy>`m?1{XR1WAlal-v4G?jVTjcMo=P!2a^#LtLr_f$FSR`mW9JDlftP>0X=S_R z3cIi@V;{fC%9zciHPee_ua;V%UZX}kKtG$S`q#qo(RxwWg}_z_3;wc;=Pyn4HQMC%UWVFqbf76_gZTO$V zDs^w=i?t3V7aoP}3&@{ezjR#A$ae`^!q%kSZ}xZeM{lYon$a4iiK_cfd>phTrK@K% z-RLG{8O|T9?NrNzCP_|mN`^ndgX$n;DneYd>-6YOA%vfG60QxTjf+vK2_^ZC8cB$2)fQ>KZ=qUB^$IYHb{c3GOE5b-%GD z6Z_Lfp!5sQlm{LIbJ~u8wBn*iCKLG!&>~uXapq*W{DmRN%(r54>pT1UGzGWcWyB+w@u+9_ z?yD-lzNiR2^7Rg2^0RtXX-1%Wnu8h>f+9ty84z^Hom%?s+Y7$VL(kdOcBsO!y{IL9 zH;lLpX6@a2^Yn*;|6*ccV!-;r$-`gqV35ku5D#@uZ*U+sYLMylL+q#MFjhaF*Vb6{ zWb`B$3`Onck{5%Em60gY1TBTdpzr&mim5e9NAM)Z@=NjY*U;o#ku*X@Q5`g$QxB~P z0QI3G1K8;$t7nx}$}f!sTY=Ooz9kL2t@7jh_m?Am%bFI>h6VJlE8q*zYk)IQt5gmJ zlaiG0KYkp@lvvwr>skX5!((I$AN}Qkq>70m?z&+6!s7n3JFQ=T--^~$|1GW)jad2B zz0v?W$#t`tYx3fLh2!*ey(dMyTM?t%ZB&(k+j(>KTwN27i*;)By5-T#^r;vb(i)cb ze)gy`q2c22CA(+d9h_cwjvPn4to9KR!`^xL#EJJuUFp^!7me7}Dg2goYn@TA8R6MH zZRru8*kFK~F*-n5y3Cm)j_zBtmWMwCc|^%ji=9}n6!7x*2B@k!E0Gd>%12|C9y->jUGH=uOnImNVVpL$)|O`{GawEd^nAwg5+t{ zs?|(8v9Hs9bqX0;>~+^I?nZTh0N11y{XN!iW}}O(OFWvkw&`0{0#-Uqm=Fkx19%`w z&ASJz=o>AzRk|&zTE)14o4Z$wJPoVT8tspRpU|<9cp(FP5Wig$zYKV0$y26wr;n$H z*|9DUH|c6gn-$GC3tWFOFkD^L)&n61XpVnO)>IGjLgyx-<+O7d_wMOtm6w%?=pFCO zo;wG&SyNGh{WQ85VT3nbfU>3=pc_OLw8X<>sQ2|mkP@`bDa#8Juea^$2!35tLIfc- zu06SOMM*dx+&crK-Z*UFin!aODG>whW|ASGpjHDn#eFs9VUR&bI)&lls|5-jz4TLO z!by!kefVHGa->?#TD4{l&D*aW)+6qzpza?wTDNWbu^@GE{1{o9;i=!XwJx?3;ZKll z!U-$c75~a#dST&-^L3Xds$L9l>}6VmL=2VE{;H>J>shYyYtg-Wh2|kRqtd3u4WODF0GqVHvAZ~pq#!)!&_ z*i(!-CnMBmvs8}XKhF}nxNL<0aq}K20b+tk4ST~bOnP1nTFwq337@Q!L?DkpfVh#4 zS!LzrQY1RT9#g;+t5V(eUQuSEXdJc>$VX5xAe5OL)$vSC!hEkRkp)rb&g~fP)c=UI zHY4_-1b)4>4~}Y>5D1%%Dt%~eqBNe(i@tbKL)G7TtZ6guQn3MjE65}@q!YStTBGWh z)mfg%>Slgf#m_!4_M^v*GyU-7^yzLmuQkA6ESUR(J{K!eO2I?9*x{pOqdqlZEp8); z6@)gi$ilt-O3FV$?}4Xv^z^QN859_>a_!~I13=4d=YO7c(!}GZ&z6;A#x#*3QQ6s! z={|%wfE@~tjz8AY*%2$rfL|r{7+nV5J1d+Hk#K?geQvsVL9~%;2r(x?v-Z_AyLA}biPfm?)jy=BO{2U6Dsu4_{dO><9@zV- z#=a8;t8e|76?(Nv#*GK1eLt6;RXr7`X6e*;$dU_3^?U6SJQByp%~?IbIKdAEE{c2lBEPh}yj;{NbH>+Dg2$^u zhjNIazUgZ=pEYrrBR zHFGvTyzR#8MX%^7zBB*h)snvcH`!90Kq!#cDeoA1l!VEEgsC7jh~KU)WD+~IhkE12 z+Ri63qESMzf?ImzdO=np`=Wjs>^FmYa_tEF1C>b^RQd5x zVeo-hi$b&QW!sfkNyLt!`fUGjAj%}3V=I9oIK*I~w28ZJtd|HDz-^8bkjYd@tI17q z>?{9WIDkiOaVe4sBX!Hgi(YAa<&Tq-iU^+I#kK-0*#-tUPF=n{e#WU$9U3!J-d##Ls_L`nSAhqKN~LLDg^!QN1Dd=zDK^Q0HQOszDb zzENI@dLb6;;bmVh3(iHBMsU(Da2thY{cwzBc=-LmI0HWL{S zNJ@?G?|OV-!~6`MoW5_9OWquyU>I*l@0GJ5)yKy;)!`6QC@7ZfF`3f@^uDftN{yGt zzIdv&HM>GtJ#$3QzI|G&%A&b&VcDY}Y=vDM6I#wS)iQ!Yio1iL3~CDv{V7uu6J|Gz zP>kzy4#nKEiRA5njW;e`?iG|uQzjpdL>Hkv*{D)Zn=%cR!+@f>`}`*+Kupa1dsaoo z=T8|%$_zTB)v{EPyZ{RX-;AH!P2|C-b|oE+4V!RkMTJnH?4C++(cfVcYQH>IZYsG2 z=w))Kf52*U_T6~>Hyk4|HGxQ~+lepsh^1xmJr@(%S4GB6jzIb|91)yj+rq-`TrMb) zd?Cq7TM+*|xQs{thux2QY*(wlz3|oIU=JyA>BC27LM_UEZK!z2jsXN`_mET1%iRZk zmRR@n^eitgkX36(mEiX5g3e|JYfwY`J}jIB+9LhMyNqVgCFHKEd3nq+A;fDCWL0ms zdGmG{1%2bf2@P<4?VYHmiLTuitL*SwcF~#bHgU(a-}S+S4{a5mnzk8*i*jeoSF7TZ zG)vuff9(Er|M<|x^`;X2qDU;2PqLGXA(S4p$mY$Pq2YnVx!Dpo3jmBdNmmKl8)Nxk zvi%V!UX}e8&REzaL~Z~KaGl>uONob9r6mrI)@a^bS;56?%hbhr3xr*10zK%D^8sYz z<)VvMy_sDjt*B2)hV?uHo1|fYqwKEY1dI8am);bEoVFyl&#-J~J$0&9H^+nkkvsL% zVnVlQ)Vg-Pr67{pqmc*q5z5;sim^lz!*TI!+|7ACbCIH1X!ffi;~ixn0lWy#R zMUSIQ(0sZW0e>#WHBq8gQz)sWsb^~w<$4~jGf60-vDcL^K&fcNFERX#ob3Lt-Ba<|K# zUKX;2F66A>3!PdNI(K#CqGQ^+#Vc(G4OnC?h=}Id>LAL)8PVY{)~7spex^iazS1De zLaT9cwc6Zc9IpvE8}V2g5H96@oj5Cz(El)JA?QfCxT~vMy+zGP2!^D z@ADkC2QV8*0bR{+B2BEf=JS*2uJvjn7~!X_`N}R?zZCoD|8Sgyf)PLLkkpG0SKLhu zha3|NNHeqjG8t$j%x?u6)NPFO#!GuW3wl$pR!E67q`(@WT)e{+(52J|Gr=s`j3EP*_}U zMA6<1+g=hM0-Mhf6+Cg?ye?d$4)(gz zVdFR+0%hd$Q4VAK*9=y9{hlmlzJ*EAq?gdplaFuI{%e7>2{cEUL$qI5ZSf`Z+y1VJ04Nm-gY-tz{od}Ri0GMQS<-dEk?WV-Eb0mR1=AFG} zX%cbkK>t>BgD_+3w0}BpVP<^;!AZ4zKG&e$_r4Y`yFF7ECv3dB#sSob4p+FXcV9)Kq@xuo~$sHh=q zp)^!e%S~IW)0)n8rT_)f7#EiIFn`;tO!A#sRuaxamgKC((l!CI-@UUMcvE3&CR-tE}9HgqSd z&}6`X>n|flM*|MQ07?-aNuXUdPQ~H3YT~U){rS5>sRR-twtPU3#s!IqiQnIZx`TE& z5Ot4SBw+%mj1=(FwtC>TMzS7ru}M%zx42d$X{2(rzg*DU@20DXrcC?kG5hw6FTDY) zYzax+|3M-!g>+X`9`f=PacKIM*J{JFl}CaTPb4FS<@bgj~8bR8d(^>G^Xd z^Y{kJRxSW`>gnXr7`nVY!Sa;LOJyYQ4%ZA`ijGRwCu`q7vH2&;cS;dJ)!KNerHf)( zUbagug;7%TA`FUD(mX9SaFYylcFx^uiua6hNp%=|vsxZ6+0u_#TAG&zwS4C4^x5E~ z<=P--dQM&WLtko4Oc}Ospu%F%z)Kws>+oSF@?r-oD`nRI<;sfXB%;MQ67(w`7f~%t zn|6?e+N$*1x7_l=B>x+)gER58Zh84~+MY2Pp}@T|w#?(>lRA-C7l<>q^xEwf3znG@pH%R<&(g)!+1{$U#ARDt^*qYck2# zTS8Q@)7wsAWF6El+4*bi($k(YH@hWW#m-FGa-i|B7NwmP=#JWnHr(sc53*HGO}^L! zU|C>&6RV{N0bW%8dZ8c7An$32*hC@Ab`FC?ib7kDm27t7&$+0RFs&F#^eYdYt^|l) zhyOw89HMY|q@JeDVE)VC*6hPS-;MMQHd3re{*rOB&;Ajz^q@1NpJiGzPlDgVBJ@k1 z_PL5Y@_qS`(DDh zGc}w^*NS+Y(21A*S<#1+8MGjmQzG~hO+x!HA7g{UTOFBN11528Avli&$s$U~wkF^o zeZ&i+?%lPsBvJ_!786bC22S>$WD3_(`raj5b-nM_o1mWwBhPGUGBv_~(MVAf>1xEB z6~Byn;tsr9p3fs!-uQ1hcd-t z*NxJ9OSw|A{Cs>8>fXl)aV*0HPhk*g57N9}x^#)=UHrWKwP)zmCrX7gDNMnoc*6Bn zRC%BY99V{<1Nd!m%5*-Dp$sFCV_X8iMSpdPPD~v0l!J62_NnXgqUa|W#uh{YME(E7 z6mgdd+BT3=?>GoXMK#8SM%lJUxIXqp{N1IIlaMokS1j~jQP@x0Z%1M{=eOxPlIP#46a5@3!{~qRtA7Xhl!a!=b zzwaM{Eq3gf%$<9+pwY*LuPNEZ=E1C6c~EI;z)+#Psw7JJ^XZVNL9B7O@jNepZPIyo ziIQm{6{UnKWE>tM4OE!Aw(b_VrC?+kR-+pCiaqi$oX~%?YaAiDc+*|n+;-mllZh6F zzCRi=p>ZR*v7%B+?@SVnl;&TVYjxm2+n8)fA}Qnr&6Fd}d2n%aE<|wl6eYQDoZ2^b z1gzWw?J3+9l|}I6UYC4+7_+fU%_g{$i2CSM!BjeBQKCxtHU2M~D%^?Ytdn9L!^gk! zh(9#;vQJ?r8ZC6ZRV$edS89c|$)9f*Z=OMI9lVe}g@w|XNqF|Ug?z7r2$$au>e{@a z!oc52bK~UUe~dadOXMPH1$GTeU;k3H>BhMu#CT#^l2y|8+4s{Y8g~7<%l(O+W$}r^ zrPC_EbV~?pwn^#M4fm32`dM=$jSm(OA%HGC|Fr9LC{sc><-d3DtDe}CvZZS9rp?}8 zjT$vNi0k7&OuPn?Jol0^SQJc^BoC{*aw>2Df+Xf(6b8opoTUhU@%99jk}$qk+?8Pe zvvmet;b{dh5&CPu4hwfnMD-zgtgG#E6RHU1=QQ+n!h5R-djNOC4Kh`?HzDL zA+`^T_J=-%m1U#(p!u^;2C)CXaYgxY=-vM&tYK?d*cLd!a6qgygU{wCqoZZAkYp8s zH`388?4h`hhR4Q4c3=F)P98ea;u+AJwY9thNoWdgn?YEpt)Jc88er!1dG>vny8)H9 z?p@5xT1!f>Bz91Y*u25eAq5|)V3MjQQP?{`ngS6ciO>cXimh9b2`9X|)S}=MB<8f6 z0>X0pK>yB>5H4aI85KF!_~_vmoq^|h->W#?^-5W3?TDSK(vI~)1tOf4Nv~evXV2Q7 zYSFy8&}M7_#p?w^t^-sooBqOiXCI55g<#a_4=bM)d_n^idD1;#Z@_ zwYa40czIzD^`J&wJz9JnyTobOnC^f7Tz0=l)hV|Pjnee#IyC4%@0o&Q3e?WAftBN3 z@)GcH?xai!ELMuwu196IPN8iY%ypsF@7uR;c7E)0w_f1=@cC;vzBfpD@|!(XQ=7e( zP5-dEO?bvjl7my|fMk^3o_I`Sgq0A6`Hw+Ji4~`ll)lJ=?M@LB+fd{g#2X;4pOnSf z2@jsbf0=UI+R}0wnj6FU!Jt_OFySaiIW%Nvl;EZ_h`&&f_`oxXQW*TkPh@wJsL#@V zIlB&5--sQkE-^}R<;UI*=Ue~`2u%V$OlNS#&{Qf1%B;DjleSbBrQ1AconU)yWsgMw1LQSK~pc^azCN_>Oh9}o%EW+iDR5iSRhWMUIzPga8~+x9fp zLP>xD{>;K$E1)+KfuE?BO8~6~mm%^{&;^~mCo#lv6`E=A;sDcQ_XB0lAqdoLLlr}` zGX63N1g0&ghmDfte~BIi*X~xMHmHp}8`iHsh{l(@dv)TFA=l0%?xJjwaG=VH6}D5N z9C9VksN_zYjc_=iD)guLL0YvEWuk_?R~S!Y`m)b`Jak7k_U@;>*8Wxp}wMt+DVT$~)Uta1`dXzzQ5d=xm4@g-#zOb;+S^<&@DFrpfH|gHJ`)8|Y z_t=Rd(n4w#6G4^~NEFIkX8fvGR8&-1LJ;*~`NC`gs&74d^tKtF5Qsnt%XY_kYDWVg z5j!xDP8Mw`o#4yGKnzn<_KT%O10!BG-cIKWb};JP6e2mlpWPndQX8^Y5<_Xh+R4gQ zOrH-Ks3C;lDrN`@88Z(N_mT{25VL^XqC;s9hOKQH8NILNfJ>X zxx$Gml-Clxj%H32c@RE2uA4~!wwd~fs9;+y`KcFdk?FX9j%*ilXQ~;>ULyX!ao~HMao{fUv=@zC$jzEch26A>D0 zW(HaeI_h*JIuaQtjj~F+No%!hYtH?2Y$mO(B1u)_gu4fZU5a{16-D2&mzZm$h>){#$&Z_ z)259;QKFH0{P>4e^hb~QYvz^1SW3UiQ@X2=p`|@9ON-_`oJ||jnZWxLY#@4zp@~+Z z3r8nWH^{1;{ovxBoEmI_0~aog8NZyI6mAK8mp%mQi|au2KzyVk;9N2?Qb$qYw00^R zwgK2X{v`G5G!~Q}gQ!k7MOwA>Pg#xR1z>c>qaWftrd%9#^2hO(=ppPl0A+RdUio;o z0J`3B6efO|oV>lJgK5j_@Ms;%Vcy^>6#C~6k9&wL@APT6N3o)=CHYd$Ync<-uU{(# z80IAZmG>lMzGqgHdoZy_Y*Sz}!CqX_i4!*fOHQZ9lfY&lpGUsi0XqFM4?^% zIeHhtPkBc5OnDrk?fz*RkX@hBjeY&_*|SX*!|pc>7zcwLKft$Q)GaZj||Fm4vGH zswoOfQ+}`I&$9h{EKOW+qs_X2l|B4heZPFE)ybTi-8;^6Svx_lFZU&bff=aXT4UBq zx|FepG6WLOh)gi@XO?%@QO^j)TLa!;L5Pbym|1Z8qT>5k$anT}mpMI4`f{X;#%0_% z{Vc>Nyfaw8UR~c=v@wax&3#Ao;b4CD$#-(3FM1C-4iQxhmWYR~3F%@8rcf5YY%ELU zCGwP6;ikoXIh7?gN<>BGTE0N4F(miU_=NZ-1qN7YLvisVKnr(i>y90I&L_5An_>J4 z;t7hbm2nHYJ8t*cJsk5v%OIB}K}dK!##{Z|hk_?>I+Zou9IbX!SC9B9|7z|n-NT6L zL1-#{2p9*ezP1d7B(7|0EQJDer^uR_v`7}E{24b2G2U;z*eJ)0;j_prC_F}Wa3BAb ziiuP}R+QW_v}nexS&J4g;?ToV`{e0UNozNR+n>*WD_jAxEeS}W)(if@86|mUsT}EI zv}E+Et{17AGRhDjl{}K*iy*jovYl|%CCpy2$<@_W;Sez;_eqk8u{F(aHUB>rrr@_m zof(o&848>-CUr^4C4Q~QACoFdqjB~&FA(hwWEj5RyoMBAdDnpMUm^X!=rM*tRxr*= z=D$uSuKaZ=?+%$e@=K_K+TQAW@3(rQwvze&b}M1-*kNamc+;QFPtO^NWJbnnl%RW> zWbA(6ZsBuKIkvxh*4a>C!lCstwFjmwpGN@|U{%xc9XmLu*pX9FYHu~Y#YpPDdXmzJ z2c>_%Q;t|)p58_#t?4F7QZU*0f69aM=iGdNj>Sbc{iVL&poNF zoD#tr1p%j0Cw5&=8d|nyaintK>t!J-D#8%jvzB)B7|u-4(}Q?@9{oTMBpeM!zP2M+ zcJ8F7*5Zt8dYw z#D*owEP4rg#poqeAA-bHDLT(b-R4B)aG>LAj)o7Wr{?;>I`&Sw#U4-%Lcs48R!Swv zfxh%#R!WFbvaRuC4q`4xEy6_BT7_@xsIR{5Ek<2|@3>eOJU;H7aT>h={Vij#vHS&h z)Jae05rNvb7|`_egtLBy53P(49ElzUWb0U7+sDvk=%z#jEK8ORzUTZW>jaZsqpkG) z-JYiigmOf(Dn73f7-I5PR6U8KnDmwLk6OYcLTG~ zDbt=k>DC-m$7Wx|*odELTvc>FMc!)B;~gNd<6{i$*EsHrDj% zbXPWZb|o_HrA#r}Mn@7s);SJ;Nh+xDlxF7`y5+L0QnVe>AlsG(@^^cQc@ zGWYxKt#fPCmkece7Dsv_#*oCE2+O>VJU$BbUtxCF0dWm}mU!w?OpC`fBK+YWY&YV_ z1qgJCI(Dos?%V6&ENO3DfkRiWl@UwW$aA5en*H&|E9^NIS*LNRVyM7OR0#CY?ngq# zqemT_HN{guJ+{JG?P~JdCxK@ArdR9M32M;j!g}0sx|-oWI~p9R{iSt3MOAv}_fp&y zALY@^WTIjz-ut24`B}~G?FN^>zb2;>uXbBej)>iX#0R27s}djRO}6gYQ=i0#;ejq6 zc-&?CXYanY7jFd2W;L(BWa-kx3|I6%NKW`suxC|P`YKPocjAGdy?n{+f$+gwK*{PNT=Hv=*E4|Ad$ak*8m66hG%QTN9tH zJ2dKnF9^YO+4W>!1awh^lE{Lz(+{Jc*=Od}q-W^-w8>Cu=-C1|W<+WN1<1YS+?V=p zJh5h$WM@`&P7%qW2UR>zD`uLqDqYT@0?Aoy!j_dH&^#zA)SFuTWupJlW&=nhvq{QT zqK~O&@?W&Wz(={4Ox-9xREj8{w7rr*3JRWr-cK<;2@(2$4(eV}$)ORT0^!v>m|#Xr&$V^ScpYeVrqR;M@}}ufFa`nO8)qAfY94-c{?Ou|KBm-t730T3Gme>3c;54u~aT zZ3+`p@dz#|(^uErcwO_eec@vsCY1?s$cLFxu&nof&%mId5T7?VmzjlTru9vkFjmmm zYeg>?f3!T1b8?K+nB9g|c_Wr4PQz8dtdHJMURRKmpk^N~NpPWM^hdXoZR!L$K%Zvx z`}!s6i-!8dJ=?3nmc+~87Xn*z{0w9g*RY50*z9pD@mQ=LdbD`jU|rop3T(#t@gsAC zWPUbhxxRGx7(rVlt_0YH0a3sZY)e)3=f853gAa<$^f}LOU~|klduM1q4%m%be!+Q4mc@*AO_;~RQQJ!*ytfvol>m-*rZwPgTv^0kipTLN2 zH5>NjtZ4OUt9IkX2Yv?OZI1`JxaZnl!xi|`p%}|!_hqn`1vxNPdPAMt7oWR{!jSY2 zAt!|KM2c$WO8Sbp>ZF68olDpGx%A;wXU1C9Q&rt*q~kRhsie#uk7O#0RE2`Hj@9#* zOcke?myAeonos@$sPA>r5W)(A?-!tJ6TqQ@!I(q6H#zdnU@qu~>F@!ezDWoQ;2K5M zmtChPpjV{aHrxGAKb4hQPO%d+;1{s_U^xyWdZcccYy09OHy4=iIA5xm40!dQMsI)h zkSo(Ju-a6T*JkL`4;@_2{R6yjcI042gQGRm8ohm!B!39y&tIZ2~Q-s;CGr30~Iv}AlP>KT8z^OR?!h%Cg zcBx2)+hqa?IQW8#lP%s*|DlWAB4_W}7Ic6;X&P%QD<3{~%=#2t)^yntB8pq1rNSD1 zFe-}O5f%@Vz^bcEf{@)YvH#l8kFhj2(a~F42>|4^+oAIc$D0^Q{zdFvW>782gmaKY zrT=K0-Shq0Vg3_YDxf0EeH$xLZZ)YsetB{+OC}-F%Mqt8*{rLuQK@+uJgL%bV@J6L z3V`5*JWzf6iG)}I0t7D{KYq99*F?yNFNG6Gl9zRU^@Zapl4MiNZr9MB2%wbb1*|S3 zE@XO>?bN<}G#Tk43_aPLo9e%tvZ(KV5!S)c>6MCLQ9|{`>KzkR0LxJVGos?-Tj5i= zqE+#^E;>tDj=-N+z=_4SDH|wR5^KzVy&GxqDTtor@YG$zsPaIo^^2p#&U^F5yZ1KF z22~mdpHK6CYa9|bQX6mT+I%Ag&nlZJO{0prCKLCImRWP5Q?1N^yCI3FkG+W z1~{~kq5qwdf_Hy2kGg`(*txU09MiXE@gp4lf8%h9l|o_mz;;nI@lz#j>J`ltd81)$ z2HQstNQT^yDV|xt`y3*YFF;qttTvlotTO6No9xBUQT+;5k695#*eD7r*&JDCkABG9 z6P6zr^RpC_-cR{_SSb_{53(2lhejtx3Z3Nj#el}CcHGncGgcxil6eKc>7?v#^!6-| zi;m2tSSR99d$|_<*P1&&`LB2+$P9=}&TV$r&!1meuu;j|=2_6PYs`LiFkK@{rV*UYDe{; zrT5#Nb2;}ljjhn->YlczLzvXUOXYBNEg3X#%hU1&=MM?0q__QSY2v8Pt8Aw~d$g;n zinsFn^ocTkVJ(#YYg1k=dv}RV5lfE_W0%w#!3HzA4mi06(F{(vK6>!bT~S2bA!Z`$ zF8tWrTWv4X--yB&p@&Xn-MV#C{>^E~AEvcRD6`?>y!#E(k?%3%6S`;`v+*RZn`>nZ z6BM4!u5s-eO30REZUKs1hc`1!93Fr4kTdMuxh0Q&uVSo(k>UTFk{teGD8GZa=XbAVB&!aK$NSH+BVAJ zNxj?uJ+@u)kg&_{g$WVE^g5^a%U6>7qzer*)Sm+Y~tXJ6UO%pw^1f3PCpW15KJ>0iYVUox8TzIQ`<2fx6 zY3IjZTy&I4Q%7Jb{+GuICDwmDPUnfK6TLZbzf%;MDDtfsRpOH@-zJT|av^RMol=Yy zB$i$IvN~ZhH4lyWwd#nQkuEOziin##Q-kN-TeISzedlS58z?qOqn{5N98oTpg~?0# zcVdIpX>l$^7KD%%pZ>vah2fI9`c2j@3zpDgUU~PymjG9>Jo`Q}IAA`^)>e!(p^Og= z8VJnC(&kTHq0hW_>DDafm=e=aO=Y4F@hyG(YXQQ5yAK|$r`n2%Kw6@RY0&P&Zk7E+ zqV-J15U5xUy&(z;z9@GJ6oc%fm$T}coJvcxpAE?*GlGv3gKX(~-jPseHYSTV zf`d~gfJ`Zds{;05OCWx`_pG!1Ios}_m<%pj5H_4LL-ERQ&yE5tPfA5HT_()y2G(Nw zn~)gf0=^*2FOC|(!~x@ zoyk3sJVh40t(D#p_5e>(@}RHnk_20{0w3Z zE^uDMAvC}|sNb2{rLfH( z`Gl%EpGU>}Z~Ui8?x)Tg0n}RK^jm%X^~<~q+9S#HVa~c55WUg5kEbhcf1^VGEO!AP ziGHTr(7TJ&HL5%D$0V1E7{s{TgJ((f1s)m z`d~RTQ8@L>U%VD=Cq7KjC(|p-OJ}ozP3TgiXcKlHyEL$R1QiGU-U{Q2gClpHv}sxE=9>CYWn|BrB`SLcg3VL+qT7&u@v(aq-0jL841%_yx_y9FSbJ zBAobPqD90Z08)uYK$12;bQYDAXwgJ!IW=-m4+bf∨hREzWlKt%}LeY$4|8W}PN| z{5;gqP>e{G_zhE%gf6pj7E-Np-MZgLt-uf zowqMPcjW?C6_$wq)I+r-;a&Gy?>g8dtMl26f*-8MY9Ht1=Z3FmcqIGpHeQkJFFHDg zRw^Z=8xnBw%OS^k$>@ck7_QR@YB!3)Jg=2@e}gTu753aGn|m|60TG?Wc~qE=j3U9j z_vQojX!t4 z>GOH7T(SY?i((on%kk#Dg5^1Akls!*CBe(!`lEvlTYa}`{<=j?8yY+-V!_B)W`IDU zn6=KI|4^6Spsz2PJf!4$3C!ChY{d8G;qGdO)#z1+}(io65B-j%$yTd!u zE-P}n)iymIl;M~EP9`DnfB~9(4o#jgApsLN%0XqMpLTm-A)5AP2pSdR8z!W(Cnb91 zYmodcac<9C%Xd~vgcZ2DB={bqp#5-~O+fZS#Hmvg6pN<5{?s~)=B@0zVH>+2A$?5L zrLtm`uMc*75{dle2a~mk45`K1fv*;b?U%Y(@*YJ93YH#gAv@e)BLD)TAQ`1jjUw*o z`InyREEbt3+!D=^qnM=if7q6hS!it@pPHRIHR1YoC+o2WGTsL7KsNq=gn~i0si3t+ ztZYhGmwf(*TUgM0DU~GWzRaP9f{~Hm!Iuffg8JoDPgwAG+-)iRWB~90ggA1kydL5a z3APLVVDsSuK!cj#KMTUVk`|1LyUJ(69Uh1gSg{=P+1O`ulAL6VP2iXk9Vt8BiVms^ zPleF8!?RBP{PR@~7vnBz1q7yMqahVVT+6hgQja(%<)GC1vE}UPD_h2TfY*FucJxA*`&a8GZGYU<`xoy(%UOi5$5zvZ~iLNyP<@jIEp7~4Ujc0|r z#MD3GP4NbWDHriEsyH|{F#{`uYs{m<(YaCeq9bCr%QRC^te~OP7)Gn#cKDL*sjK@Z z*`CIvHbibRoTk&M8O*id;-ZdYWFdb#s1iAlG#;VDmW~y^2Xm$|rhU=53VOqW&~fAL zr)U-j5i&dTQVSz)Ky-16 zwprAdp8$*)H`rWK3?&zFmqfI$`}lTO)!D{B!RAO-0cDi3HkW~q(^^bAS+a6^R6AxA z(o*^6Bqdoi-FUTY&!~k(!&hxjyk1*9esxaOP+zsbj-1U@d!*0St=E~F zN)v8Lwb{*;ZSXz$iZ2Ba;)>9YFM&WWW_tI0U^Xd;bdT*4t&p^S5(q1JIaksR=S8J6 zNGfSV`!a6w%d2&0 zWP`=KR8!%fsgW)9NUyVwu7FkE*x>cw)lYGj+0JIAO6; z3nk6mkb2@fDgl-kVo(+;X~@Np&pj50goXx2qwOvzBa7Nr&ps2ul|)f-OnNj zh(igV!)Gq|eeeY$n1s5)MNe6L!Lo@)z#{rCJRr=IOu*WY+s0@ndD``X07}MgpZxDo zeo$TN=hh_G4+AS9=@}5n3<$BiC+!`zz1-b!ynEMT=~x2sV4>xe{LTHEuADILXREr75%puy1o@ zZ-AIeT4GW=czqflaYHy?c+9-M`Y(eK3op(FL3RKo5se+~5QmXY=<}M}i!O%q*ns&- zgfg2kqaDVCyH`dlN5d5MrSO}GaHOI6?YGr8LwK_hbT%mV1r*~x7}af$Z|3mO)L~AW zvaW?jwhk~Lc#GUTiMbO!yX%;iRrfXjrf#L6yAHviIMAjdpdG|!=FdVIZN2rXbpiLD zVjd)G+SPaT#<_H9@FPJ_#4EacH$$vGSV+PGQc6i!h>7}0XV2Dt8}&kz?()7d;`$K* zyAdIb=wo9aB7Sk4%DcT%{c(=O^Na!@lz!^|-(kNEdgx++nl$!U`{_9N0I%}WVlxM# zuIIiDD&Nhdp{8nT`}Ovg)0A#RHR3st$>>$alg@8fyMOiPX$ekle|Fj8dow3|V#EEr@(>_DQczI{k_XxM6Gh38pgG+&y@7_wF@In zj!Ikw7t>GFe^GokEZ~I=eQ?QhM1|SbIpsAF(PgJ-{{~Jhat@^ z#m(CWWlcCnUWY-Bax~H3IZw znhdymL20Iz*EdtUV#Na=xzHt|VbAc46_IJ%mYcL~-LUeUM1%4bcQE0_nEFv3Om$=V zB3Y0=loXerAH?n04JhZOXcDC{?amk9JAPu<b`a1TdHj5r7;k3kVp3ts8X3ZTZEMGoYu4l@) z96^ry^x0)Qq*8}NiItcGGAe3(uy~!jze|z3t4efZ)25Ml?@yumf@bL4?BmsQX)swM zTZuq%ko|W1HW}%A`siuI*(lIc%a*T_;&Q)379p?zl1!M|@uGz*e!eD4LPz00eFD`! zx6({|)0VJ6B|3lsNu*LGddPWLA0A+AIC5j@$B!ar4+^0kpp~qA+qK$Rl5+7Cx9T!N zJ5pwMQ^)x3*ugxdIK<#vfKFtDN3hErW>jVS*((z_ct6y06<#lX(lz$+>R@y8+U_bwcIlfY-y5s`_bt^I zb!Y5&3D<_&04>bhscRiCDLb*T#>lFI12Bxan2XZ5MT<6uH)~`v4bIJa>CvNi9}PXK z;ZLx}f(6=5%_u0N0F__LL?pQp#vXO=Cz1%}1?)knML_tr3zT;qXOpJg_^D zIMjk>Qc?(Q6dBo3p^fOqU;&EObmf^0B1vTYu&2x&+@mMR)T1HpnmY{R;y2Vi4+LS| zcztlTj%X{;YDyW<6@ru-ug79C-B7s+aF4=6Qa%_KHRaUfNivKUcBb!M@faa9l5|sv zU?P_N{0UGZ(G_tLN->OtE_04`?yDxtf=1kXk43l(YB*3N3kk#PGEx}46i-eKv^=6g zz%MsR%yET>(_Fro<5sR*>AMR4_aAed$b!eKs8HY`X>66h@OP+i8K$~to5gkjEGe87 zO9m-=n3yYGZXYCOOODgMlFUk8jv%hH-0IkQDPa}rU5>DI(A;b=3u0`Jxe|~=-2t}P z>htS{4I7GIR4lkNXmu#hvB>?}v1;uxx*2p3%3!8lFr2PlMOltLV>pCv8@Lpkc}2;% z1Y=#pskheE(QGPV-+;oPp8GV{RqO0~i?GFSnv9_y4Zo|CJ${wpZzr21%uKY&t?Mk_ zoJ_TL&AJl6Vv2~0Dm-2QMId?*K4p>f5Wt*8T$wB}4%OC_u}4t*{ilmlf+LEYGZxf^ z;7GerA0nn`Lh$yeU1vnmV!S>78F^((hP$}q`hS1@?xM}uxnHn&xlL5OJp$c8Z~#-s z!EXjhswz>dri_L?S3>azM%3+mgMlovd0a!_P1uzrv#|}A*e<=j_W=+9{vzia{@9Jw z9tnN3VEmwYf>N}GhJ=u8esuZ=@0C2A$sIhsKkg5U`_f;@@tK>k?mhbn0s`%yPn6Sv9?jY zWAkRovyDIPzBuY!X^dUm0otlx%VL(F|4lVCN!2tZ)V%J%NIFe?rR;SMQR0N*+~K=~ zb00oDSgo_VM%vFH%m~vrW2t7B_^!gU!y~U#f)wL&AndlVpulYI+%CA?we6=xWqje> z(FU!5P1v?NfH*P(VmlHL$ZETZAe_JwwCKP0Decf+OUqy4H{-qMo|dFES_F7YQf&gv zsV5kV@@!|5V4v1cjcTnuxAIl^0IHR_RJ(iX`C>5cf z2yhYliCm;-p}OGrBehz=h`IjGBF*g~Y|?*bg!K#$!bdW}e=YRq)@wUM7D-Sf0Uah)MUAvt`53)keOUliCdLclrtoBR&X+X_8=gG6vN6Jt zu}hbpnYAXyRtS6!r}bxP|6Z`n0*Z{oae| zbCuIGGiL3#HB*Xhl?eKoUl0J}sGlEi{lzu;RQuK%aT|VdRoxb{^nl^}2ZjggkixV# z_aM??+afrDz{8Fg(8UIF*9*5d-;NXbLS4*_4}F2(P|P9EpSQg5xCLr8MY(jMhfRvz zIQ7QNK#-Vvj)QguT%xtLGFb8~Z&P_+g8~HsL1V11 zb8Oc<qE^X`lcgn44#OP+{h`Iu7n2^zG3<{^-O92}+G# z{}IcSF~KRM)`$ZIor2SNYx8B+>Wx#XmtFnKX1ZVOuCYf&{R9dunqUqst2`V+bz-`Q zr%llr2!5J5K+G0oSIHPcq#(LdGf^Qi1X*v*d)v_q3Qf71C5F6vQwJNcR*qZaZj+WB zVrvtE6DZ0%M9dCx;1GO-WF_m?GAg`&z>LVDuU=)iFsK=2du2=j4dGr&Ex8KB<|O3y z=$Z=0#*G`tJ}jVe?Mc_qcp--~M71{A^x{ljmv@8U2plKcl&iQNc-%jTozvNQ zB=Pm*TfE%TkUt~s13J|s+#h11eEQBi{}13@Ib27Kc*4j8WGy~CP1=c#n>Oi$F6Mj^ zuKqtYP4V%y0d`r;+NP*R7efxC5{099i-8T6{500wph;@*5LMhISbdKj>oX0cLvR2( z$n!DpI^mPUmg?QjO>pLL%^?FmKN%am%E$fB})Tbm+^ofYRX6vdxJh5A}-7sl4o zaze=L{d@OX-8t&Mjm8=i1&&JNx2twxC=p=mrmj2!gII2ilqLh%{3Olp(qr+aJ<@YG zYZaQR`_DYYo}Y$Nfl3%SB;`&DF}+DV=!g*^cYS5k2dD^3 z*R8g~QoKiGfm^IubIBf< zu)l6mla6F*OS*`?SJ~d&u1`Va$3$h6Jl?M{iAhK>b0q5gJj=~UOHS$H%GY6LXY%$1 zaOG4>PMDU_;A5|+5xS|YPF2%`7FodC;K3Gq1Wd7pJDoKmzE$8xTc+ z-Zju=DpRG$H!zsaVNeT_G1%nKZdo(e#Czyz?Akwj@SU*2IdZYbHjP@%6VwRk5!cEP zpPNEm6)ykftm;VJ%{|9N?4fK>zT%+u1~8`j29H{&b4G0t_lB4h*`##qYQ1U5I(6Y- zWtWjzGeB|~5fs+Pgr+n(90jRGUKC-tZDp>L9I1+T!_=7dUomAAUEEG)e{$P#{UqeC z+*SrT`Zfe(xT`r*S0^-Td_FEw{tVB-B;^J<$l9CzSOh-j4$DfIK}G+v6s<}6mha%V{p9_p_lp@huBDuEQc3c2J#y38} zMiJ9c7vprJw(Hr)rBRF!rnla;|67 zl^_64Jk7n{)$06(u+PDJ`tvY{q`Yxn$DUa- z3+~y#O5`kWmm9&{E03DrqWaOI(5DX*k-w}JilIdpcdR}8U{NB_m&80t2S>#m+g{$} zcxT{yZUNB;BE1;3;$UmPDO}Q;9*G1dXmMZJO{i2m zPP6HT7)!ytBBHKorcmc}Hyoj2a=$FL%Bg4gLS4hzw+fVXo$mjIp&jn}=%CB{ZoA+V zQd_51_xgRIo5Nny&mVTzwazZq;H(rllF`Kn30rp<-96>lj3~PRW-42Wh`?Nhhaq!6 z4LT);kh1gsJtu{x8i)%MnKL|=1SW4HDFCUZsCpgdo_YX3%h>3z0qoJeqi%xg%lYWW zHJ(e7g#@>46u$@2$ETTinggvV*=F?5y&T5|-GaQT3rI_pG<_tAV(fO*59yCaaR0!$ zV^^&zJ5YicRgyGtMClIisG*U%672=t(uUkW@Sy5o_#Az-`BNVspA{I2>7RBCk7qVJ zT-Jx#i1yMt<3cW7G|OdJX!N{m4HeOTYO9E-flE zjiz8{*ql1udXxHX*2*0F@cirC$I#52Z~8L-D5g9;r#PFIi`8<2s^ihRZNARO_zTTW zJ$OmCTI6o5c)h`@MoE-}%f+{eqYrb;B2aB$+NZx(yh;NV6)SVo>FU~Lkc)_E$<}kG z+c5xVeW{q8+}%5fs8Li{HMJ|gF2Tyg(l3gy{T#2H?+Ea~a4j&s_)B$Z7nYoZeYSx# z<}%Jl%+E|6_RL{E%mcBnLQdV7O%@tgX=a7qf?jM-XvGyvP}^cZ=%{JaEh4^A!P~cc z`ZP-`No-s(&(*g{*{Fm;Cc(ZgA>*5+bsurnvMwN^G~`hizN4>8JD2=!*UC<|5>sCP zX&>%4n07>M;)bdZKgmD!bs2%b9x6*P^CiZk(fgS)P=UFpDPIM#aHsiG4nuNDS{v5K z=+VZ;EkaZCR#n9{yMKz(&K-lISmGXXQ}fy|Lm}Nw+*GLP>wBVoy_bS+dXv+xv)G_6 zWqEJsFzJPsGHJ86Dm{{*+8CMJEhb4=IIlU&;>axtaY11vv=dW74^BFn5erH#o_9D1 zrlE9g(6eVq2g!U!Dsp;0~VW?D?|Q%<6oU_)m1pDLcQI?0suU0BN8EPx$f!3iAqS!LIFEQdN7pQ8Nap zBbJid=WM049zox_yBkT-O0SU3Im`J{HsUqNvwP_q4R9y&!k*~G7xoHI`2m?$j^rZ* zsoQkr<5RX{7a{cIYQa#1%*`aTRLBL6QenJYUP_rlBT?~czx`l+{hkjQH&>i$n3bJ< zZ4+1OF1;7Gkve*RerL!jH~w!hj~~LC2PIO2Fq-4rY14W=gcvAFpKKzuoO9roKK)TA z2^R(*JMiJwty{%^;`u56@@G#F7#vx44^$ zcZ>bE5;JaPaGL29cC^2re(0Emy<<18mWN$_nbrQxIJ3Kh8^>%y?%wFDpDw1w)^1kT z*vqyfbP*j@p(PXZCzae~RnLJS7Z1fKOe>P4IfZ zR?ej>p2Injphs)#j?>ybh#na1l;WgdvRJ(65TJyNsnzTXqDOUAtE0IFMyfV$Oe?@Z zR3I?^3#dZFzIzwhdc*K_T94{ZB$Sc}U+HONY%H_FdYYJowQRVc2Wc>Z%ffamcnsR(`6-D!vBaOl7EY0N{?qr-XvJ#~kOP$9-Wb0<~$s*Qp zeD|IqExdYlJ-GON^5GlZGqsfI59S7mYmK|&UK33mPM*7?22q@o6^0F6mt>OOdhqoc zHri}<7;4}nc~1^rE-p=ng{Bs8img49EKzUL?1lO>W8muMJT~aaWv zDNETJQ~RRWU5Wx+J>B?!McoLnnGq=Dc*+KN18``px6F+fW7J1MC32rCHj$7$5FqBF^ocz6U=b6kRFG%pe6*!i zXHqBazrz{OCB(*EF1e!KJ@ln#D4^dqhTW}R?ty-9p{C++38 zY~raI-ILM8igjHhuE&?W$d~t5wmN!r%`6H8gIhyOTSc63Yq~w5I(J8X zci~9Mf?(eQY3a<8-SN%2=Ro)=KlNNYd+{>T?!PW5*e=-8OEb{i2M@kF_26qTrLOCj z9W3f=+%>RbLG3`GcWPFC)fKOl+t2*>_>R|Q!Ab1Z3MRCG9gWdn^YQKXC~4*)(pcg& zayvVtN56i}1&>`U@_OLKNBx&>g)hBExCvQe7xZ4YMM|N zw!d ze&isQLD;|%dk1OpvfVgwQ8YxHK0Q&=^xO;M%5`(sZ6KqYE##ZS_ymBknjfRaGa8vh zV_G1G)Fs1u>FA70nx0N3kSUOwh-*WhyT10CV4zbOx%Xy3tE(~lyhUFS7yaywN=0Pl zbL9UTX)BM{`%!KDugo|6dXB0nyBe!}v!u%y&vR-UJ%3+YCpt`5&y!rqS)gXg@K;F{)gFtS+I(T4Ge}2 z+cp$_9(a}PA%20e@w>~#Co4Y3uvaVDda#3y!2W5S$LGQ7Yy;I6yUX^Q3w9xW3IF4d z+2>c}(^nJ|z+F_Un*Jy|yE~UxT+N|cOQ2+FW7zvPKLR_1$K!dx52hh($6_?aXl9E$ zo8%Q00Th}960WBA+KsXBRJz`lY)b=Egsnui`k}M+lP(sSKz+MKSr`(J$}5eq)H^s6zpG)SyU||it=)DYAkINb#YnX!?@I8+_H2^GHdESF1G-<)!?Zb zYh{)=-ITBzO*eLbNIe)paV3~<`yF;S=MBz&cPT50y_5Ka2Ts$r0a9*r7F|VFiOt4Y6P*k?r9FGre+TX||Ai;^Q=40bRi$OT z^g9uhIKqDVsutm?FT4u7XrxVVuX!ZN;ZT%>?XxbrePPLx!T{puy-(gb$GtLz17+UL z0oh|Gn0(Z2;x+^=xqJn;Y}G29u}Ms>@O-hgr(AIWG9oLT-nYI!m`RySt)?VImbe@9 zB$!&d>A@m6txc)UW+gSY)~YhMo|BljRh zZIJ3py)LQtVqN86nlS9QDjCR%T*y)Ih1Aq>q?pJm^+^j2--?HJ&t{juISpo=cWMT+ zgd#C})gTv{6i$?>loXiuCsCi{(-ubH0vd z{)L2mTctYb&xg*%y5n&LyMk~IY|OTc>6+rJJ~SWw`Rx#Iiv!mgces^$iF(HVGy(@4 zqVr+X6dX6GZ=XI~6Mn})s3j6u4u(f1NM>=;TfOfQdFD(a)Q9fd;R`yEdD!t1#Z^x6 zIg`KpaiCEgw@!|TO4PK`L8BUKv!UX*-{#1LA2qH)_q!c#dYKC(bz=CJ`@ZS{_kHvl za{&Ze2gLRIIbZAg%jG6={K{9VHGmLn@&-u1o`FH>vl1u;nSX_g9KSfmkIFI`J?^=91z20Y`AK;W_}iLN{fMmo+1A5zFBd=(FU+K3KE@ zhiQtfjZI5nmQ1hmxniGdeb4Dn65vrTOD#29!Zhz3Y7T{DKiFe{@CVGtM!b}K(6B=b z(bV!&_7==4?UDEpX;M?xbRnmc4&!-0ldS+OB9XL<`}&`Di&yRK9h(6o>7tKJ0yt}% zBXKTu4HsaLBtx71OFL+{=dhO`J3QY~mrIqY+;-WkT$wN>n&tM9YXux9bYj{2JwW;Lsq}fX@ z8>e%^sVf{F5FsvO!{&gKa9j3}%f8M{T3ICT&)#0_7+*pG-8mxO_*167G&NLGKw$rrv1u8E$ zd8y|ng#J+ntm;mHAgrk>nasUA&&eq_<~6-bkKVmi!3u3}h{#^^4uBuMB@Po!*@8gS z2%uTCVBPc=kA|M`yd?YEAbOR0o#4>b4@zAX=pjDmD+7d&bh|(WnRGpy{L}Z zzvgi^2w0IdNbVrAZ8qUliinN1Ju(uP zpUPkbJjZnb`>Q*|6>mt-2k}#OEx#$SHAxT{hiRQGm;PeH4fGjah;LChx%k9hq=J$%Le$B+AT49qmx zOV8eSuebrhil?CJ=4VUs{O-H9;-ev0Uy%h;rFuqA?@Kix*eYaAPz+>ZH4^2Rn$)IL zt^@Le-M-YO5<*46n7SJC7xv4bnO=yzuuz2Vk!U`}pzh7fMsZu&sd5_(65(pv8M_$% zsDR_YI~M{VxC8ih32$bRDhGhMz=imU_@ROd|Wj1 zp(s8lxpK0H&)l5T(memSCyWK zG%wvxBl_Rn)8Btzmre)mn#8u%GgLX2N2f@kC^bb*MUGm^Glol286lljyBIQOOSKOZ zcgPUP_JuF^-8fMmw%EOP!>XjH-EnqK4}?G5TA1=u1mPLC-3A82eXwbcYZ6qg>mK0@rRbGS0|?~?y8)b4GKpH z^l$arSvgrb8jlh^z|5*-z_cDsue;rygKvlm zkdyjg-MTFOB%K9#B{HSxykd6%OAGdUdc%UI05vgTj}FCHpE3r6vJgnVE+9cqO(BeEb`;ZjcEtK zoyCEJ&@yoC^KSqQgwJGH;Z2K~+@r@uJVT8v!``Gr8b5w~hmtA|5RstrzwO-kn~`(Z z<@7%=9|zJI-kv%SnKWk6mN@b_lmbG>J50?ugp<**j$nx&MPD$Sem34*S9=#kt`&a)*^MKDi(0++i%kjl;8Rdx{h)L3!XU< zG%cDWrUWV_{08Y3-^%^9t|Jw{EU+m$)* zjDBf$SL&2O&NP{rZ|j;&R+FJAlaapC&f&LWCHl@;^e=S@ojI--C| zcK2wORvH9W09Q(62xw=#_TH((fM4I8nvt@zsX)axHUU(_6?$wd8AX^f zTzN4*1nwfg4x_j6ao z*oue=7CX14g-i>j+=nMi(0k1{p|)>zVB!fYi5)M!%dp3zFy_?5SM3W;Xci>VRt2(K z^m)LFiEcFmhYoE^VYVqOEbCaF9G}qvHMY#Gn8u*tpC8yZ!M$tn?f+^|(zoC{mq;^W zASqvwp1h#gbj%EWvD`@VeM!R?S2eO)U`gA>rYwDfYivfD%!Gx&7A2pM9tR1h6{ufU zE&DyYprD`G)bRWi*Gm(UXz^q32E1uY10^|9;}9^gQWDF42Jzp;sksBKnqY5HHs%Js zU~yGsmXr6V{N8f7Di#*v!Xdj3UL4hc-lPa_nVIZXe~(=K_Q1vUy$!0lyL+J}rj)vH z%o{9PqJ5Ztbi%2$Cx9B0UI)p*?3D7kA|%d;ui0{S%=-vFiRLLfACv(*ZF@KPiDOo9 zYyIOJ-Pb3t{}I|jB)W)+_76y330M3Tb2%#e|D%quQG4;?1*4VP(7dN;+I(l)sqL`D z#@S156_gE!glG}!&`>=$BB&0o59pN{Es_y~MzgvMe3su~5C3=C;>foh%xC}Jt;K9_ zgSyxdkGYikbS$zsr%g+DyF2aF1DW+M1Yve|c1m;Q@7Tibka_5olbo*Y3J~PWDJp~0 zV1c5_DmW6Qg^-H^?Sd%|2j15|r)cN-8fpIRumemsscu81y(!T_qksih0YfP1Uo5qh z13Pu6QW*;5K|L8gg9u~EtCBe4B_BVq>Nvf#%zp#7(3JpxJ$+2&-Q4_!QeP{tyDNNU*|rL zAsV7MnBmreFhz1FKti=!jOtNv)0(W>&+8TbzssmVY+MxFO75o2Dr8MV+V3o%f$wPZ zLX9hm@T?B@yV`spr(Du6pYu(a#3y%iFulGhaAKy|Jx+`7L4v z-dt>^GH}kP70X?lE;|d5$d|`B>K`@|PzNU?>^@bdn1>{DM_qleAJ-26Rv@{-|HP(2 z>8)Hz5;1?OqlyMhMI$ zf0$Ko$KC1i>u_5^n>>1*X*BRzf!5gBMx8T+Wv0FL1TTRKuOjncru zoMgbq7!npAA3rno#qy#|G~0Dw9=*`eV$Qm{@vG)Cz=f6wCxD}i%gq8?ETCN1@;`}@ z7kvrcJ9({jfjv<#W9-gMGD#=%2EeZjYGOEPPOrb!s%USJ zFL;X{0|!n@qIWh(r{NKrYT*8r@iC%!;X(dcx+MhEOs2o=P;JtLsjhz(s@bKzi9bni zjFM3b;MO*35m7Nu->q)Caqtf{bJgP}?zGrqqu0Rb>AhN!it0W(aX_VUWl@}}?BR(w zmSreMU9{$N2eXGo%h#~=LngCn_x5(Fg{O|Gm? z0nV*8W$C82lCTP{ECOpK6b{@OI9R+;fg3hte6DdtqS$VG;^K%09FYG>ae@W?xI&AD zhL_LmhK;N&cAt)$paw{^nW=i`#PA`1v9Vzd2QK`58pJo8M@{+Rc^FH8O!{+zBpsa0 zuvz-wUuVD5?u{fQ#E7d<_{W6{^WE-(1_}KR`_h~IXm(jMIu8KU6-$PNJ~oj2&W5ZY zcGk6btG6?}u#e-HHQ6(y5C8QXAc|;O;Iz%Y|326${dxR4@ICL^>NOIOPaj4u+f*Z-t7O1K=;Qh8@vwl&wnWW{x0mrY#%%L+Cp2{n7g#G>1Y7s}*RJ`h~sbRA-cV5e>tFF|WrkZ^Tf{u%Md9sy} z?M9(DYczZ&jPZN0!PxPLc&p{>1hT&^;38NP2%mo)^BNVJWZrQR`OHp@s*yjVyPLot zyC@%m(eKOrFa=GL-9aa#08`#4^7xbh+k>sKWbj6gn5vLXWpj3vGtiQ?%Yd>16tu zsnPnmbl+2<$5)Bg?+gb5sqzh+o=+FRs{D)BqP#H+V#c43<7s#cJ2n1nf zp6Hle{-W>VOVBb=x_xr)7K3dEhr-^w(>LLCAkSs zgnZh#abq^z2p3zl%(avmTYbZ2{`9$_1J$#P|F+1wSy0x4`8R5&qpg>0Z=Sn~xkIQ& z{V$IY0?B4tmr3pWj%;G@-BTmYVEOCE-SvA`slP}LjFfxY+~9IcH8n^VEpE&)NJic6 zl0qqs=1u^=z1BnPip)1ZaF{YuQ;xLgydPs^h1RQ?qAtfgMJ_=ArOX66o1XuLYS-4@ z+`NIW65~!x$p#Kgq)FN>8+04xA@djHewUO2J)M9{JY<%_w0qr8gZ7D6A2MAx^n1Ko zG&x)(xep$-Q-NG1QgIV8%s)S<8c5cg^I^yp34J<#JaI9u*LlRKE|~1BEy!@6k?&y@OWNnd+f96u=SmgXDKg4^QSINPR z7~ULFiu7xeHa>Xp@}Ox7Z0AW!Pyq?E0wtadwS##LT1)MWTv+_5Ag)pjb!r>(fH6-_ zr>91$gI-SS6dr5|!9fl-%N``W$Ybs=|8zbvyM_Sd0nfQJT5ZLr(uM3+t30GJQu)Nj z#&&!#B#)wt7IFW5i67v!m!;W?%?tRMg9;C0PP+oQPSK#l3ip$xN%t*@$pD-hi&>lE zf~4z3HqPxsJ+W>p;^xd#lkW@X0YUZ6UhCkXxJNRP@^#qyZ8;no6l%RN>|L3X?wjU< ztJ22^Tw+$V!xpgk(Mp(C?|gfT7~t_i_TP|C^j*K=NuKIE9+Z!9uVkogjzxp)LlxaC zPMysAJV9=(PTgX6D==}$A0O{u8`R3zB}rYS&86Wx!#ebrR!P3TM`o@>!@EBBc+sRA z@;DwTb$ZR7xkgHbnu*j%nbxpLqI;<5geRJP1{|D3ac~RO#xKgQG<#bY&db5^%|ie< zTvj@kwCx6+Qh%?u1s7h#X((Zn-c`RJTR2J74#aRtM;O}Vc6aV(3NnObzOxDn3zKor z3sKW?cd8cBRU@~&7BC&j2Llle!{Gts4^K&v1JM$~p7L)K=*;WGyU9pW;71qZss)aJ zA|;6?3|uA`D)~IYHvdDY8_dDKcA&yY+EE!s-yL+L< z(%Vw~Xd9wP<*-Jl(hU{~bHK+Z-B&GNu1+H?S^@Dt@sF?cj`&_dj)2vw*CT^Tak?&} zXC(Lsggq#R+eSi~W$wbHNv&#Iwod!?`1HQTcSaUUVxml{k3lFTS`PpYd^Y+zS|*6} zq)^uxdU19D$rIO^cXDF7`R?A-vxTuGtz-ZWwo6HzSpl?6QN>u1S?p$7K?O~nH*d^o zd$tg&PUXm>)5-rCTP=rjTt>(tK+o9E?=MgL{xFfxLyz+L^Ms_R z&v6F)W92I?^uLA3h~h+e7xituk{kImos8fTpzbdOp8QdsT(A{EY_I;5=3AmDxcy@v zp5-WsU}h~c8exH;{9iSjpuIaS^hmt`DX#=AGz(F(roi*9AL?*9b1T1O@jQxKX9$Y% zs=2j!sTvPmJ|)GxPzz3HIdZwbDIhlm-YtEX@t!CO6rL%CD^w(mkAlg`dYnNehDvdI zVE=r`H!Q~Gq0IhP+tEjOHDNHhrEi4L+w%QTciOu@-$YkfYRu_g@&0n*Us+j$B9jvn z=W};WId$^mjO(>AmvRb6jb_TFQVRT%&!82nB*f8z6C&uFsL+*}miD2&Lnh7myE^Tj z;l{ig4Pf=O{_5@DY-;OacR^t&Tb}s%-8fltA9^-zc=orJk~HZu)^@2|_85IbQw7Bg z4~~NKW)Cv6wUzC_47a-z#*e=Wg$*^3d{>sW`Gw81&m^IU5{ZdS>jB84p1hGjoTMx9 zy_+*UJb~R|msxDSqJLpGrV#!#^i1e|2>Uh=oD$VRuEDjdSJxMMp{%=3MK0qjQF@XT z<<8}3=I9t5wOOHIcbe;PXD<)LWs#R-K zse<`{4Mq2K1{E9Am0yBW&)_}FH2v(&rM&#yLH)bCFWv+H_DMUkJD5#1b&RNz2j&FTcu_^$6?45;7 zghdYQLtiXPGtTUtJ^G~Oc8+WI3#<=bv9~fWl#@kdZA*qt$iskJRL}bWFzxL_jIzlk zGTa!0T7~Q=sH;9Vs}eqq>o#g#^$Dh1kBwR!N3+E8-``t(mp_6oW^?taJop^I7Gr-i zaU_2>wlNhiUrLII+-4Qz=U?MC-oekdzhFPPh{us;h^~~R^O;i(I|WcwUj21#^<=1? zRjV!o0ZgHBQ1&Jjkn1s3@Woi@mnCdD8T0_U`V#I~jj^Vw|=zeEK)%dW~-C z({0tY7ann$*SP)6PSq5Y2LxsG;}Q5ee^sx9dsAlozRQM~+R9K+QJVz=s(ZP!YH!RN zwo?g$4iApxaHj9IT(xSbao4*nLw+7^3xA6o?GrS$J;NidQ9xlSBa=;VnOWUBcD%-R zW-0!|nD_mDb46ch9OR__up=pMY)I5b6L!Fh4YTYXzWxS^RT)bs;ceVX(W5DXByo$h zY#{1b&jt&hv3RkpBZ#y<{oRZiGVk|U)0aaj9E3Z;Qsj7t{9Xs{vY>LQ_lEefu?3k1 z#IV}*pcXN)9?*x-+LE`ONlm*wu_3XQ!7Buu?8Zn4()QaFMiQfdS|rwvzr(;(A$nGh zAiJLbseCmi46{3srx{17Cx=K^;w^gROf#|W z0+NCgnz=;2Wn?5BWPUx^=d1XDKHpHWa(x9|vBg??omMfLQ9ueoK0O2B35(Y5o?}ZD zWJ8DPShk-({S7EJ@I`#-A6Ss6Z)u-&txy6hCLXY(IV zhW)g;d$-J{bs*3XYmNh%<;i$oJ;S?I;)a$B8`D|qbh*#cR=%Q}k~lz1%k2`g^Ym%+ zdL9ynVUXvb5#l+1^JWYsq$Dmpq_pI^=8yzA?_RU)V#uX%jkHtvX}*AM2t1=l8yF}} z4HK?}B7l&LJ#8OvcV3UV+P`~N>Wcv@c(;ugfWAy3BrxRaPvzBPY2GnE-H;Rn_}cNX zPzJ@wIY55TT4YOFKAkY7P}ky%P)BOLVlIO3#E7Ka?~YFUV+d>?k&wu z6Plg<{{JO6@;wH-Pmt6mfU9~HanR?nHD~D#n5RO@;0@fE%kUXKL^D)!l)*n$NV8v7U1FuURu8 zj|U^su4vxex-`?UnFD4mB4U~4*`igeZDsTsQg}ixy!dr(t*}}IBUmj06=Dg(q(&JZ z^*y0Ng(>O&t%Ij$d{;epj-(R@GGY14eR!IC|+Il0cR-U+N z>@w0L!o!zynTg16&)Za?A0RL2^Bm5=voL3U4{jkCsDJebsExwBZ~^jGPNpt!2#?lo zy!f1TVW7!U+juMUoMn46yA@x1YpE2Kq047>koCDR*EB=@$st!!aN7L5`ebLd%Uh$T zBzH~}5YyM?6L~{scwf4X@+f`g#*fVl7D6L4SL6@owFfW}BZ`1kBdn|ELx|bG5E<34 zQK~tJ$Vn%sEy=7de+`c-@=Uv^_JX>7(`H-or!-%et<89s7z_O6o)M?X^LMLrfN-zT zrbOezm3ti1OAdS{Rau+_CGsLoa zX!@3~Sh3Dv>(sJMng-AN`z3RgGW zo?#E7lUfCO_Mc!nuFbJ$@|G?(FKtf^DqRjJ-BtX=H~v6gNG2Ba9ElLAHVsb1pl&%z z<`pE8!#mEWim=+sZZ(&>B=Q0XI%jlxC%k?cu?&j?=w$e7VqObQQAdi*7eqVI?R*Tu zMJw{RhiB)G%v<~-NN2*7>C?4c$Dk_rM}_A=(WrSilt+^k$fD+QON?d|TKA^0J6LQ6 z4Y){ka^ekL>y!jiR9ds(MYu|n0*?htp3P!fZWR0ZJnAs?}0 zkOCTMah#DgA}Qvm?op1vxL9)>CqR5{%}CeGSU)6YFjTkp>~rx9uv_;I`ykwI@ICcZ zS23W{|L|QEZCuNierV%Fkt_feoWW-gjkI6kgtmOq$dq@dAIkD7?;=in=;1#P6eV(4 zk|U#Y5%j9Lxe7_C7A`K|NN2{X4Ix8KBogR9rk$oLk?E1* zDajDc*-!BvKgbN38K=Hea`t4J_0_9mf`h8vINT-4&}cR4-C6x5Pm^Pno2Sv1u|2L7 z;@A<5dF^|!*iDj*pxoIFYA)Mx)P1=Dat8j>L6L&AKx~M^>(LkN2fg zbQ#z@E7TfWeol#Q*d4y^(I(YHX0x>JoTy*BURd$ylV$S8@#C*zUJL7n%0^tqQ>Of- z&1`wTf>1lu$Lzom_I}9WAyQ6G+2J*r2MfNViXGr-x^Xa|5bXM=?U4>cl= zKCg-;DmWhVZzd~-w-m?dSt1*!;*E;IzQuh>M>(qeE4!Uu0~{V;4y*A5#XX>Fa9E0jo#+s&f(k4aR6w z;UQlOBqrDSsdc=JjJv9NSH5cR`SI}rTeBfwEq!#Z0c9y`%;6o`1884G^5--WE_>QF zKxql|rI586rLEl<^L^T{g&jfEzD7QhAMhPDFq&vWZq(RZyxgFw=V%aWp2wNfGv#_^ zyjfgop1wEA+Aq%UTE*GlI&X3{l{Fbq8QJF1?hY;N_f8cv5zyP0ywW$XK)D8V?mV(} z+S?y5JolS^XwWjPx|XTWC^=aAP<`MWW~)~ZBd6j0#YiDJskXwE)Ek}9t6y&v z+d!RfsF;$^^GtHP#to3VB+x`@uR1fMd7PaZzv`46a3#c7=;C1fYg=I9<(sE#nb{o}Q>159%aq0PN85j_Oxm%}2JwKfPAR zQeswCpf`cSP7;^K9g?C|mvwo>|5r!tOs+X8gGvjb0 zxh_X9<5-K9Esuglv7@45b+`Ds^le5e6Uy|>EZVQca9Y5p<}dTXI5cC%KL2LNMHa=< zw!)(zqin0XOf1zRN`SdK;-ydmoJl~FbepKpo>q2JyTVz<*H$l6N?giI=7)`q%1~x8 z<{I$9q(GCt%0mM6KE3G$%lxP2l7q*O9Ye$8C$3@A7E=zmd;%B^iXSPt7a_PZhE`;} zTWf8wcT43j=lYFqYGU~D`sG_B)!yo(^47ZP*aL$yHY)ec4BKXkwz%c}vK9%WbN5NZ zbnLv^AKzvS@n3uNia0sstJ7|5V1U-rHX*7WRUXT=BPMO`Dk_u(TJncN4s+{)ziD*p zw2gvndl^CmakK#~@uMae6%{QWb%oR##6v|ClhG&0CezSp@SiL7>eZ`a6IN`7GU6PY z#6s;&Q9Nwvlg&IHtJ7r+p=I}z=Mmtal3PaH3BoVC^DjPS zu~_F*1jHinHXky3%^E}0L0oWgd2((Ml}#K>#olg%Fy*r97_6(HPKdfX>{m|CVm=V& z0)-oZzEUbR9+!a`Lmr|+c};`6ZXV?;^B_~Lq=H_Ut}7aNzB_GVhZ0*tT4`uG zJ-JI8u^xsvn~5E;=Wrl1scl$c(gNb8j=MweHkKzXrc8*Lm zAy6aJ^jM8T3J0BzM)}Ejbz9W4yl9Rn&g->(BS8v ze^yWC=-=4=X3h7meLH)#^e!Fh5uQJA^sG$*dJQnC?DvfxS);@V-!0(l1CX2vwi-^n?Ehe1uGg zUQasKY2lH}I7Blu?P#FQ$idu0Zp47?rDmUxG+*;B(+hI)@B*vf2kjf@9;F1Vg+Ctm zWrarPUC}^mEWLfo%gBT+p!qjci<{R?bq&h8d9$h53)x7TumI%SkmCydeT}Z;QB&y( ze4TavH{AwmuU;Zo7q?&n*eUlIG=zgV;>e+@vj{iBr6*R6@RZ#c6A;w1jsFi_Zvxk2 zzW)Db&k`kTk+j(*k}a|%$};vPO;Ir+`;tOg6BWu9Awn3Dm@K2kTC!&;gi=w2loF-< zpVwXH%sKz>@AL3*98>vx?)!be-`DlpuE7O;uHtzR*`XMU1mYX$p$W7EdHIcOHH%0y z&?yN$;!vjmmD8Nf3pbKQ7jyFu-6JNPkPHn_$j^Vvf^`wb2p2Ju-rVj-fdwHw5y?d_ z82gw}b&#@CS`;&ogZD-Tn`h=7F1+bQs~)MdiTog1;Kz@rZTX-s=IwUafU``SKYvts zuYV`oX!tka3RE}&4!~#-kuI)GYUc#=IA9nW4br2qe;KhQUsM9>w^EB13|v zDB02H7ugPyF}$B%^pa3Kf)}nrTVHtAd&EoxMncFxC+u#c%Pgk5%Q%~@^lLJ9$t5RF z`w}%4S41WWNMF@sl~L%-!EC==Aqi(hooBcRmpzAu461+pJ~7SvgXqUQci#C}{QDrK zvG?0}3?>wWOx(*q%hG;Y$0o=WUo^nT6!e{XYuzRalbrE2GC?E?Ss>enk|i>g>ot?D zi0tC^L$>Hxzo-6Gyqq!-)ocN$(wH%0q@_qE_)ZxWLybzx3N`(i8Mj@{zLY$!%lys= zTc*xp8s~^I@+S3mhLi5!8_ZqadByCHwlwW?qbAgk$<`EQCYzIqx$JekHWwN&kyloF^-iM03L&sTj! z3@XW_^3i!`t!2@2R!Ip6Qqm@wA`nikZ)WfrJ~#Rb>@eO-}2IKi(p21-K`FHAt4Fy8jyQpY7YwwgM$YgV^b2G2|As|NYBV<#=>e! zwWY#hkPT6?LqVk2uWGH)9eR(n)^B*pnV9Gc@cLzC(-D(l%n~b0(gP$3A5e9@H;ta;2mgW6)7>Izz#WeKPsga_E zw-5HH0y2eDgy|nc!Wo$=%;8uKe~YLwtAAYAZE+%Rz@7Cs z5W^h{6?dn?`0 z$!#pZq~z7M_yqx&4n9_?o4@b&=%sujgS;QF-@B*#g=H7m;IJ$hB~7elvJ zwJ~fuRXwpCH3B0xt_J82L7mq+nY`CnX1GQ(`29tY#~zTQ8z|oMMbs4~@XwJWgA0~> z88Im&f#Niz)OH7FD*)uT($Y#O>Ip?2+`Wl*mbiJ@bTePgccDoa#RZew0ALBrqxjTG z$!76|Zge8DH*(VT{Is7Wh-&zUB&8hNH}cf%U$|-LZexklJm-1^+*aCq7Us1XXCd&h z-v81lBM@d?t`jTCb-5^yrr?(PnwPeP;!s#!&H(#tAmwx*R*{@Y@%hR-%r7Z;e}#m^ z$9@-OC4bFuI)!{%vc&pr-pfn1XwlGy-@j0F($e&7Cj!gq31wdhtGPhNeqU zsG8co2lZqvOUPgprxpRuvj&FjatRB@CL(Pqjx+ z+dp!MJtfS}%v$sZ<`h59`UwZNG|V$xpFVVD4@s~Cl3MTbq)zm%Mr-=F`Wi2)htQv6 zZnsndYvYrto*;_)VcxD&V-tejp!MUZ&9)iJXG2>J+L@G0B+qilI{`xV<_^vD2CSB*br+e@YQvqC~V_6qxCHHaxl^-aM``sJy@s9j3&4(WzbC`>>Fj{Bggb578S^k1I zfF{oyWS`SlQUsXT(JkY76+zs(mR$PnzufMji43)t;fm<1uDF_sE|wm9LqcYLZ#Tqr z>`KZE9fVlOkR?jf#;}K{aMB**VW(vKA(?J1N3EQFUGpG%P&u>Z3Ik0jD!H*cswUnR z>n4{Ja5l8Zbj}upMlwScu43mMnVlxn*jd36F64QyP3TO06B7-Qi%9h9#k~9ULX{cT zTxcUGn&4pS5rgA+FsIRHMW=DS#l&!7N$$?Wr%?W(deSp786D=lO%8tgt&9s~h7*?AzH#w34<52+ z9I~owRMI1U{zkSXbm{>1Q^3>wLlI9j@0#|uX#CZ%M&_iI4XmQ){jMK6`R0(?ABNTO zzPBastaL*|H@Q6#$a=)+CoR{iXVo8IX*yU{<|`Te5iZ0m3doLt37l zmhL6oADbtUvHP!AL^u?7r4c6CW{{oT-boS(_Td97RS6Bplnx(-Glw%Z1IL;Iwkf8~ zm}SKKoqgRbsrtkJf25VPmry9&HB4|RuNgx^EQXES7t_V~2&;xNG1;Ck)ZSM|onohV=#;k34_VGcb>sb|L1Fte~0l9o?#PxJq}CP^K< z9{G$B(S3FHi_`d3BR?(fjp4&!2s`P&x6x4!4#vq8=)j2xPiUjqO!-5MS)qPt5B4WH zJp}!ep>7x1>_no9oz~T?&j@gy*FcoW_8QKXG+TBRhA}%2|2hMp#T(xEnycBCUX2mE+ca# z$sQRah~0(9eE7_nX4*0D z+V$AgX3r}UN-Bvb(;hvBy@g^^V(YN321-kjdCJUD z`tA5P7h!Lh@bn-hrOojOjLU53nFTGNsLA_~S>=eScCIiwPQ3)1b#dgi9+9jp%!ITar@b13rI01LqvMb2<+ej-|I0(%VBg4EN5=G`yH5%~H0C(C zW&;2zfX`uXNg)u^0#UvNX-`e5cuoVF%P0y*f_>@&J}LHrMEfm3x`=}nNLRO-SIk!# zF4ed1zTgo|?(-ub6Mc#wh91>5#Vb!sL+ma!M7AKKDyAiO^`yus|Ll&i9G{-;MeQ5%D^xR^NaN1wS8gScBX;b8nCQ~3GK zUE=0Qb+GP)ZiyrC=eNj>H-kYyMPMeQA%*5BV^uP~b%?|zQI~=vu&GvuzID7iD=Gtc z0?pqE!BKNsU4$c~*^7A(@_U`bZTe8t?xHy<=|qaJBWl|Z5k(EH4vl`;V;SBKYOpV%xeyOf@?_{C?$4m3zUZZ^%aO`m~|8&*0vk7Q#GP-cOkOxBTze z2mAiGw=0DGv+m1x^Bo5~&2k+nh%F$FudXWY<$=g$i0LkP@r9jF>^A5=B+arC*AK9R z5@JJ?a94nr$WQZoq9#?g&KGlwsX)=6AeL3;AM*n$D$sv+&u0hxavGxTV?w~p{66{A^K#s=ND}t`#$+9UM-8m^ ziHWQRBL^l)K{**jbbz=x&z_9KKPC=RZd9amVnaclx^`U{ zs40*B2n&!w*bIn^4iZ?(-CnkDEZjCck2qgB80^jt(!3`%C+Dr4(nnf{=1Yhexog)+ z`=-tTj|%+G4>^dPlM5n(4N3YufrceC50{M+moXeno4>H|&8EXm zP(A_ux7~58ho_yMhsl2&NxKTo|E3Q@2!(WI`tTu#=8%a`EWom_A9hdbz<4UDJv-aA zmDkRScW-UQN>Dr3P5ypYI=UTNny1c8)V2;P;uu(e&%W4K0m zXQl1MZIzOypENwm;A#UM)e9~<3fmNVet4Bh4f_?9o|alOAo{`=NtjD&^yt&#D_zqi zOC)K#^C!30yC1C|>gBn<*1%U$rXII7+0hZ@q(llBk2cXg%p1XTI0tyCFAKU@+Oj zi7VGl2FzhJY9-qQrxKIwbjhxB#k9r|V;a=1y;?DQR><{+p!^NaSNEsP5_i8q&NShz zRoC<^hTjxNHTlsxim=<-Sgd{&c*r9sh4E+Xa35Ce_nKhH4!9t*SWN z#*g3Od_aD)$HAiUqZoIK{nDJW@a+|O+*+zN5n ziLYN!Fv(Sn*$lh{V-!r!vtgmkFIrD(PG#zV47k_R z8;CN03PgeAljrBm{%YCncdK<4=2hoE>#8fl=j!h9D$bVQBo>5MSd~vXN@~cBaZBsn zYPs6S#pc9^X4hwE&g)&FE@>)?Dq_%J@W`SodGUp_TqF_1m0e2<>0zfV)+Ar}7=(Ai zB`N7P+~$9vQ2qe#zm^e?Clm5;u27)CJzU>|#_xZ&QZ~9ib1j-CEk*#+6|4w5Pyd2> zpV%kGAoKZ3tyZva{o~gac7-aCCPT7no=c`b&kii}_ci_l)T;}BjtCm3a^aqd+-7>ExWiLGrsjb4OEU)&KomH7K3~)yRBqK2=#g39)(}ozH$Frgi zw~M&p$h?G2%xi;Z+nH26l7duq&FAOZTmLCa@0yvIm`Kn)1|)n;LYXpFiokl#AV#wR zc*&?5**byPw(V5{E=x#rL)P67Po*z@dG>^9C!aM>%Ywqh3gI53f}DrrXj%wp%Ne{5o2)Wq zn2tskFbXF7?=fd}RmEW^nB?*7tSns%QByIsA&<+p_r0GLkEUG;rz$4 zE)HaxGtp9~?164!KI)2tVW{KUG~W_C<2O_7j=n!;mt{}Cz3Ps9-Z#ywUH4K3i&`2V zs!XXbOWB&bwshYhvDoc-ND|}e&bD>KvXt z6V3znK0yj8L@SZyM0bhS9L(PNT3oxiNJx&IJV`ulmCq@84@eA>qM&P=Qse*k>>7>4 zLmZX7?EMLohT`Qr#Jgzl{jw1MV6oMd(D@Fp=J4by57EG^rKr(_j$% z_%*syX;VQkkYxUP?Xl-@5rG2-4#Fl4ie}>bw7@e4D*-|8VY{ifKgowg@)#dajMrVKe1c0^*oLwC6o$xCIW ztGecJEGXF5qs^vMt}D_KX2{pyaBcR!hSo&2{6{1)d?jgNaW40(gzk2oeAr@po%B$5 z!H-O*Gn(Oyd|bz@4OgSpq>CPw zT$RNm%$#`R>U!BapJ;KuIKa-S%qmWEsq2IA9FTDGq5jDbT z^kOmx_UzfQ;c3KD(EUryrQ|o{xQju8(!irANlq$!S#N50UvFq&aG9b5JW8ef?*<_F zVmS!t_3ZJCuDcDyH_tg%(EHIMzgVNRXU}R6Q2!RYYsL{jRj;-OFS{>_BP+zE#hi_5 z*8}IrHc)I$Dc62)%8l>|+gkejpoK%&ESr3CtKWKjg7IKwP3A<@D5XM{M6IN>pFe{2 zcz7f6jR>6Ae#flJPD#U>IVPMC8W;;@uRL_<(0`rt2xBV37E#Rx8tJJJkXciL7ID~f zjqU8LyHVmS`V|6-io!7l09nc{X8*22G5fvh!@8X3^`R@8X_e%ab>nhh#rx4QHSYut zACd4-OK30hWZPXvUrb_!snRcHB~iiyGez8Qp#l;u{Sxf=cyDLpT~JRPemC zUJ080h7(E(O`A`hs;`2@_|n%bz>l6)g!-a~2#jq{$SXan2E?+iL(f2wugI9hClej4 zJ$IF3IQ^F4Yb<#Za*%6%l-dh-DuOkGq7NTj@kMVeD~UgVst29|4tb-*)H3Kq{Ge&v zp6KKSg4DNbXU$)L)E9v80P(Fb9(*=AP}Sntj?2$>dM>Me@sWCx#ldkZsZMK8kIEK9 zo@{yEzf5>>go@@i)uJU*{_nZ8c-lGDe<8bCs6ci<18piGVo;}@zpJN3C@ z)Pw2DjpgD`Cjq^}6GATH#v+ExAScNO2uD=;+4k~}t~8}G;-q876BpBrMdf9<;bgQ| zVrvK~Av(zpW2?Phg5*fBX2zm51+Um<(z@>!NM>{!-0~Z?sFJB;>U>2>aH|CpR28AbBa z!os85IDFHnI6nU(l>YoP@Q%o5FXhF`+#hL{NZ)cFh^Ry+GQ8%Tk(4Hsi3Z7Y%(N?0 z(j{_0LUGt$4>9tfpeJzbRSNPpoqA~accMNM#S{Fi;m@#PHZ~HZ%qTR1l*{j7*##ZD z%G{^xGDU$w)H34XOJ4(8ZK9wA)Y9m@YuFeE+^w&0O;K*{4ji!(A!*<%Zv;qGE7A8~ z*Wh6gu?}kIWOTVCAApfAsZp~gvr&hhdXoC_YrB_e-Y%iQP6JL%2u-&(wz7}t)JbE6 zhSrg)mo(RGRlieypXVE`?6M*P;eA{-EqlfcRL&$jnaXW(#_oLDMr(Hjf36iLjI zjNg<*maLVF5ceu8w%<1uF!d@bBDfMo^bdD{1{$*Avt}@OhBoL!>iGujgX?vFxa|ri zRiq^c9b6$zeA(|5S5%==wTQAtbXy!EqvM^SX;i?edU_PLCFk6rh?1HiIQwFX?H&0v ziiLsn7sHjBiNU0jko211^)tWsxn~NH0)Qi^u{wg{q}D>|4C){T00N^UI_yD7+C{xRu1@JYA_;w z6qLx@bo=$&-{Z4Dfgt9GC!ByUVy^6Ia6%QY?#ATSY^dZx0ne;8115@3hyfB(Ifv7o z(kl(y%r$Z`-Ls(VnNR$-%5W^m3`Vmk`v7OZnh4BeJfiS5cZ8pjj*(O6hOKy{xwm)j z+^N0t6Xcz2pa6wV_wJKQ&-OoL@#4aDZ>etf*pQ)6IS(V^qIi3rovlS9-qm1vH|j*( z#f>c5h>8cMAMg8dC8#!M`oFUO#eyNn{fs6jCv<4ur$zmL5lwejP+-dAK`2bptT(hccBUkjXXW0RpZOSacIl0n604|TL4GoE?n_t%Er`r0q4^wOd= z!)G+S>D+Vex!TPojbowN>{D3SyfNh$!Kfh-rCmn^IARmw6I$J?KelJmu`GI zI?8i)!1uu>&n_;Ch&q+YPF`lVO{9ff)CG(Im8oQGisnZ{Q_fJOowJX~zORdz)<#`(I(rHKuh|1T|YX%EA5*X^cb9 z6Z@>lSgYK9IWp*@BEt4>YjjC-=iD}hu`6aP5dfxpqLiVPt(4)N4xVeB$d(i2CMYOq znlfaBF(WN82e1>OrZ(^$S>g4g^YPulnB13hTRqM!E-XAi|4xk?=mFosIz(x^ta^0!fnuaRkLM(Z3w zHK^p#OC%cIXxLw-4GUjpjlYR` z{bjY0CAhwdZcZd;;NC$xj4kjg{}Q61uC7aM92$8X-~?=Gu$$ZY!~LdRtF;%xuELPe7kwfGjm|uB7U}W3?vL1GS{v{B`XrYY5~C_F{k6EPot+JUjBgM?GyvEHPMk=+Wf0nN z1!XDyDINJG7)vsqJ@$C{h2#!K7Ahjjwy$$q)1;}CRKz1TffS;#*;XhP6~J8y;1e@^ z>mr1a;sQh}>=Q+|RY}2XJ}^l{G~E#9Pl$Bx4*y0+eI?14*>j#6d3&dW0%E6;Ayk-WFn@H&MV zD4$n;a~jK(8Q32msYUs>IRz}okAJeti5*_lo50kLBw=v(2aX%}_;XFs2NXD>+Y=;) ztzO2X;Wa7J{vuLxWK==s?p;wb-MKi(RQJS__A}|UWQtz&#tIZUeoLz@z$?YePYPPj z0uXbnqQeXH$$7#M3?*+Iq-=-XhQ&%Y>{h>hjF&S)v2v&OMi(UpkQO(w)PYnZLQixs zQ@-(NZ_h)ZCG0fS*bUNFB*awwXKfq3II!2pZ6lRB=}^{>9( zRDYb?*IO+&_|)uj&%xinc9qZL3t}wfu~k$1lp%D1K{w(EH+1T&B%1~3oGY9<_i%k& zSxm{1mp8#+)$oLk8Z)Ngp$Q1&`^U$@>)y-=sJK6C4OH13l~4mTkfK##V1_D@(mDY9 zzyfPMeo(W5=ZLttU@s*V_#r=c73C1vD|gl+``|R@t+U+fV1XA<^r0I~7BR%~#LkMc z4O{m(1=X%7b|$O51OF9C^3fA>;C~<7ws&tm{zWpUHb#LtF;{3@<-+9h&_)J+*$y$=c`;YX z)Dmsw;g@a!)4A3aFD)JP!PF$uv9TEdL(&?)tdf_#p!YufA6DoU;6$WZ<1u)pNj{im zp#?&+OriHdMP=n8kMeZVF<&M{?47;6H05ERaAHU6cJ3!)gZF-YS$&63{5djce!Tn5 z>nfYM==a_c3v#V;A`=3)5ba>9wE_ZMCCP(j8@Kpn0&`dIUDDU80wOO`gWoSLl~b#T zLCr$rvf;k@yScq!6Cf}|3>=?y;?F;?GW^w#zVc>MKOdKRe6TArCT#M&sw#jf`4Be! zk7Ojru4Vqu>h$!fU|^e~IA&;TzYp}6*cnC#Xv&!#Wa8xQw=sHR+UzeB#?q?2wJNOY zaHMK0gPIwxbJ~2Ge4$nQfF8U5EWT3E%&TqVXfuW2AGBejWWD=1&p3DK!Q1&#&bnC6 zT67`F#qG+0%+x7z(L)0yz>OY*Y)h|q%?Q`0c|qe<3lqk*riy8ew@%z3NGn^xc17Vw zaxw_{FS`L6M(wbL--e<`;fIvc>VJclH^XcZWw&)O-j~ ztv$VO-@0|0p)6udv9nC}mBLGsGJl=|+m;2$DHQW0lfo+lWjbePtJ4;ok!(!fSTCmZ z#ki6*^dUNFvSf$eO9_#r2|#)2UJS@M3I#LkC<^)%o+bsDFMR>ehng_bS- z0Tm)AAR47vzNN&J7Uq=UG!+QB<=f*4z+pHi2?z025|UF=g!K`g z7q66z1w(^7O+mM^_Q~x{uqf96O}b3vgMqeZRYrx$YzDEfu;((7(Kq-(L#){06HXjX zcGDcEr^n24bdG>SwWYe@+eXSS!~pDg5_^(@@a!n|-MA42W`pZX@u&FBjGmH~I}5-Y zC$(_1?KA@1#sv`!kBa1i3%6%jgWCFs)WOi~#GCJ!y4ODiF z(8v+$)iFBF=LHnZj+9o>WD@iQR1~GDQUIctZdT8GfQ3EUK`3v3 zK$0q@m;p|E6wP9cLit;9<>@Dp7XZ>8;92)$wYZKC&%h9ndUbp2hnI;=M`9t|SW2CN z(aJDyIgjYgQmNX{oEhUvBJnMS1z4Q)ZgJo_f}j4i8jB=A>&~6wcY;i+K_bdPH&#LV z(o3cOQCQ(U%X*8Il%r0BDo_P=)(`Z~l0)UyAoc3QcXzEpKp=*BBQYeyXf}Vu^TQj> z5R*4Nh0%yT*L(RpO=3HYTQnCmWvrH7!P0F;BL*^KPZD|9?W`SO*TrZrgGY(*%gk${ zSI`@DhT^o^S9#D-D5Jx~V%p-Zv!&@IxLjpsEg0%S!t}x&F3gwvz&pVWl?~H|F;)Bl z*D~{ciiCo>?^WK7iojXcrU&xlsoXx&-F7@a6dEd^mbgYWUAHrJ=?Y-@^`*R@%4)}{ zeO?nyP5qg8cb~eR^IR0I3V^$wwi-kC^*s>OlO_d0P)BdmRlQj+Meoj#jfcj!%vtJg z|8uaVY1WDzRgVjtlhig0$w9~>j=y^kc~ z3Oqy7_%MfTl8gz`&U9Fz$R3*9Wqq|;^_5W_=pUDlvQl;Oye}7lU zsl|3kKB>lgwQoA64zw4JF3}*^0)(T_bRI5xvItFW?G7_jQc@VCn_mXhoTSb(6L{{w zx62qmW{32c+k$p$gE)vuh`2yrH`ec3EgT^0=|kw`5Nv3oV|np7kZTlgj>@ywqPMzO z`0?&`Zt(thAN+G8KR!3CSCGue*WP97+iZFcT$eV2RX?Seg=T z2N;CFv>+W*Mf0LQTSnh7&Ys{3C-yGsY?6+mqhw23E%U9_YF3jRL&<{*^w)v8k{YKd z2B5ji%96#+$GZ361jPDCUzJSNd+}4^FpzgCk3L>GHP6~K`FgEcpA*+jZ4vIW;Yg~X zTZ>Q=!#fiSNa>)jfZoPtKg64l9YI4putX_quQMFE;z~OVjvP<#M3*ssyrKI&fFcT!d@4%_{Op0+g|?9O*XSh$br4Lx2=S%J(#0OpG&#R2 z%>lE@>%-=BOy1vb@yo?JORb!kkO*0{(lG zjHwL>awoV)0|a<&(Yf&$@P2xyKjjVEvH6h89Dmq)o0O42I1c;fRJ;iU&;>@gFy?d0?Pj$=tcO;=GW|K|07NC6aGgJf$%^!S&SHlqFt+ z^Is+@6|TWBD$hV50Gv7xfsE~a4!AG$s3=5w@0)wUsbP{Xb5liR3R@PfWSL+em0`;~ zDv3dx_Xl%Xxi1DGdFD{8%$*-RckU^_L?0h3#@ug)EMWxIY8f@m;(i&**P#Kf2VaIL5r6qeul-E=wx?!Nn=d4R44~S6|ml=`mpHd>QGNRYwY+N7Q0p)X$ zM0f}vC1T4VKbKF+%*>Q585Q-bdJM?=tsAPZ^)+U!M>86hDa!EYRVzfQNmt%~L1F_KIZCePyjCrKa9q^40-V zs0kRLRA$Ct0?9{|pRW~Kq>cYf7RNbfi0{4>{M;o+9kryrR_ z`#}M`_R3Usw3ez!qgt27x;WP%BuTbRub-ZeQ&N1FY`C_X9>R%+nHCqml*u9;K_aR5 zo60m;iJXD*h|5Rye?v4=bU#&E8DkkB7UbjkXTxyZ~Sa=c!%>6D2+Oi&!|2jY;M zTjzcg)f;fIDEKCIXy1My$3UC!Y_#r?>~%mzz8~qom0%Np9osoS-6sQ4Q3iKC{Evw_ zG!0_H38)m8m4-CZ=mwC=*Xx-~d?Sli)*YC)@cGPlP+R0Empe~9_AUD?O$SR7&R@Ek zG!zh|JjsV*p`jSPaQuDS6t_i>kPE;dBn>^1E`5!S~{dXdwqM}p(tW@l67RV7d& zV|G0awLS>BX*Rx2x$X4i&+%N~K?uua;tf}En0*{&QzWC`MJq+EbggBuPfZk#V!)$K z4lHtm!bEqsF0Jg7o70Sbkv$~hoVR|O-IpCkWfh^sSo1y~wJ6(|8>+4mvpzd_Ls8is zbF3mTDA~^!<@G18rTPGMucN4cv3bE{41VkUssn=ZcL2 zcc6L~<68|P%V)LlO0E+8NZ-vMboAh(4Nn_&R>`c^z0zuLvaXN0AjFB+Ul*iL{v$$0 z*;uXrqh-we3+4~nUNli8&H+6n$DFfEDtcs+EL18pE~L?K`|kwruQ<|Gfr$n2t9_Op zjL%HhL+>M?7Ov`H!)_nYVbIedmM%G~VtAWW{N)R4bgS4(i7XrRb!7m5^W_?@1z{ho zn8(&%vC#xEDAtFi8WxkNigKg|ic{(lk@R$#QBgq78 z_CxTA7!41)?4Bmivk@pf-USg@GH(fc)@n)yzXd&8TwvY|(u7E)gTsT()Ye??-et(4 z*x=x=FMn+J$r-`9EC3@6=8iklYOccl@#BrLmjJW@`}f~VYiDWd3g%Ro`Mk&ZY)mTK_W(vD}9-_Z1YgdLTzNX8JAaAD|;Kl{2)mSkUFm2vz#y{E%ERnh>n~9 zn@!{pW#@TnSnEq9ys)bftI*Z3z@9DCZRt{mBGrxS78@q50hd^_PDXUI*_P&3}~qKceY@qgZ`U&~JC;#hSy@Xq&*BTi_p`F=$~JOlQk zyqEk;oMCj@rAIPB(M{Yw*$v;@Gho|3`EEe^9Z)_MvASJ2@svzgQX&~(MkSZe_@9IW zI&l=$D6A9*H-#1JlkzhVn}*lTSmOfTalfZ$rq`7ol$&8)!eoou9Qq(B`L^D}*#!kz zCl<*noQ5$SJX2HztSBNopH+#lmRWO*4}>tKp~^MoAnHWNDg!z^Jv7khVgzB|R zn=j%%gC4hH8m_W~E?)fCii66hDCvAI<<+ar+!-JTUZ;3T3radhA-Jb&a5 z!RhaWmRKxpMuwq^6%Fldpe_3-4d1hQuAm#l6mG<3caSPFoc%)hVVPD7u<^=$hF-}> zdUO^=KfT;(+(u9BDwkgjZMm?XZQEu070k11dizp2!n+=KY4sw^ zg(_@TL{WU?j7a(S{|T8 z@9P!NmVimHGxmS4D!I+1kNjK*`YfiJfoo3^@UGK zXs%qXuGY+7XId+QqwBU1QaG;GKsHo~o53ZM9hBsY4IPYF1t}=8B;*T@GO#sa%SP7oEA<4J@{z|D#PTB{|w`B`os ze1p~hQQ{}vWY>^M6;hoOwj}h=_I&j=y3)xOEY2lF*bIWJNc;L z2-A|r2{bf=oxCibCs^cP(9!}RNhXr@%wv;JCVNJ4$~WhfcVVaopzK--1M$v_^s$^r z_#gpc)QSC>Ct%)sqs!yjfa5ga2HgPzWP@aE;N=K$o#$3IhDK20ps2!Y;QOA z)y%ni9(_1{z7b+1)4fIR=lT9|JCudGBCuOrU4@MVHuyTh8=Rp0S4b;A?zhD9g~6fs zvSU0R`Zc<$abr^6Eq)rivJYqZ@WlAdwKa_-7r^ln5>C`nA~U9i^7Z#8643W4x$OfK zQA9M8p=Ey zK*~0>gd-I3^l21K$Hqn1aJ#@s$vH2>S(?tKR1RE!XaDk8mCv*e7n|eU^6$kXA*Lbx zy6N+KmJ}r4sy#U)8ErlR51+gDek? zU@BF=(huFJXlkO(4)~w;Nae_wc4KYywES5uQcXg=S+68FowvK}P0IVly=6)t3F2$B z>^<>QfM61>!qAh_eOX4K2ZgJb^a5nkj(0Qj)A96OZdLwyLrQG@2bY>+-s3G#WtO_= zg++Y+?NLa1b(nSOmfYjAv%~ltfqw&y$o?)N!-4*t*nC;{>=FttkF2t5SixM%Kq>m4 z>bUgnUb3F3WX`XrpO$wD@u~U8-Z-s3Mm00D&W_xEzS!Z=CZCdFbsne3^z_OZ_iN02 zvxJvKlfe}|_%#3Ek%Ug7EggO2NP>U;4Rxz%joh!a2OcT7yO|xe01@Sdm&S4NCJ_6Q z#laJGEvSMl@)>#yX|`ZR^nEhW2b^sEGuNFhj7OTU)SP>M%8%zwNuIyvVoE_*wE~Ct zh}mEFioZH6#g#QR$Sk^s7{_oWbBvjLkic1V`9{%n_;$&a0ac1!dN5Qn6_PxCC&1i?Ht8O5 z{CI$g8vj3s#5=<|*xzWHf4#Koi~WGDjYfHu>{|y8q#x1#o&H-HHAt%yHgex4dLcN0 z3iih!0;*8f1dC64B{XgZWjr*DNM6R+1Rw)JJNrxY;m%i9X(L7yvE^Xk`Rk#JsaOIy zcITrO@hpDFAtTKYTUgd?bq%%M#IZ1wdgPekuG8zlNW99)?XIQfS$rw&HAIw@_lNU% zOhpdE$8W8Ab?bB%V0Sn$Fb3JtMI5Wl_APw$+eNDlP0=4&(ok?iZ`h*BmvSNcXL;!g ze*}NES;{7V*frRI0#;YmAuoQCh)@__8HqYUJ$o|N>_)Hoi z=a0RR0XS)eCJi_`X6o31Dms&v&yPJ-w{PbyF9Lj`1GkL19Oz;(ynfN^<*3UM7u0Eg zv6`Gu{)4;ki%|%n9e?&&9?<00h?YvP^QtjP2{>H^Wz)HTxkqhc1YvlRMyRdKfp zY}e>w@<8$IL<{T~1M@TE?8UU#i+k59`Ua3I>8hN843p22GhOu<^3~^vEn5iO|iyj33EafSf zg6B$}Q0|%~d?|;v19lrZ^_Mc>5KvoEA|2ZDjy;aY`hM(gYI^zI;(m|6YJ2v8Um}xWQygW@Cs0NC2Xkn6i{xOjo!eiXxI4iysT*`aDfiC2YJ>-_4`Y=VReai35ZSt$ALJu7e0aR z6E%F8zLR%8@M5BvFTaY86F&LFbHfpD2@+U;Hh;i>$ke|=ZKJMl^LbHp;^zjoA zD#}qfd^W@1HI0T#FwpgLbi6QfBydkhx54FQtP2O!b=x^}I-w$I3paF_WTcO=DFqvI z&TCY!GPW?*r@^Sq#UtA6Kexzhd}vd{ntw(*?zO1;IC_Zi5A{mHMGqH9cE^}854#xgt_$k_zZx9vUR(}Uju?i+_(l=rAPIdH zX3jAyo32BBBf(^B-pTuGVLcXm#P$k< zYYY1=WOMUwGj!XwbqUhcx3U!qdRWX+RF^m3;ppXdNxRnblIy`!c~>5$C$6&Z>nE{?q4 zr-$K8lIRQ?{(u+vSJI}#?qcB`BW2jD8=U1lkpiba9^byPB#7VZwZq!g)itiD4`?h& zR_$1F|CiF>R-bo89@eRjk(U`P@?(dKPj#2=Pxq?()JoRb8<9J3%WY62y~*F(*66tL z>$R*=t;$a+hi}BS-!f$OfLzN>rGb~4jVni?AYjfl( zXMvo7owbH2m8IpOK*j|tPWT~IV3W?LeyjkB5ORRKLz3xJaMFoW@}jh7vurA%j1`?Y z%{WXDzuWa3X;f*Q6I2u9NaWneNvA7OIPNxs1Y%@pn?RK8EfFo34kqg zU}oqZE36uE{ea9_L|Ec5BI*0`jw$PT>UEPP!^Eq6#r`QA2=1*R*Vv;$3nC{>nR5FE z0WUtU#-!apXm9s`QM8q>_$I7q(Pa1Oqm8$YhUPjb(Chrdv}IvH3vjeD$Q>S)mWI%P z%cXPqMwj8xb)p)XaVKassLLVXhqPW}F|LT=>f5(v=$9HveRbqWkLi(<3_ivN@2B0y zuHJ(|Oc2OY-y;@;!Nj9CLl?#fIAz=MKy2B8{>lyeUb4l4Up%MNigporI(OinC3 z23J{*g0%0gwrl%#B|4azkmoTi?FW%JpXldSphOU#kus*C6@!_8f%&^x>H?CH5!8g3g19=ykdc(hwSrb{-DbYaLtgcbMjHsTZ5-`j6PSdRBO| z>@6x$l~WijtBpxiGVff%XS!W@@V-K5G;Y?r1&3a~9(cgJ)g7hpEStc4qb3p`?1&L} zha~`hI}?uGrp<;P=@o+6q@cr3W=X}2+dyXZOpCIm?Iu_L#-W&)m7}(NLbE<*3D<)OQLH*Z$(c4398_F)w(J0^z(-F$P+&7Ij3C0-C;FG7#fqS+(> z8XCUF&g6I!6QOLTEJ5#X;Oqe%d7=bBqN4|956$BbOr7k)+7WO5P*4CGFxO`)RgPSBu4Gpm{UZB@OWO|#ENj; z`iYs45G`TeIAhk`e$y})-TOCUgz`~fEZS$IXZ!}+N~bC3#Ki|vt&z(khI2N^Y|b9P zcd9BBi9*3C>MLfj$l)#fuCA(d-hpYfm-{XWsbp~0@(H*zgj!id79*}U>%luK-$fk&;XrZ%o4Rj*f@-Jo-_+m%m+ zH|e)0xs0#cWbpTC3y(EObCVM&VB*+cEA;rM4{;jR0XO&Uxo3ZPeSuMhmg7nR*S=jl zNhTO>^k`TDu#0H<$gIv9Cb=9XpsW8dLO|?+)~Du4&Mn+} zU=16N1*(UCtA&M)M=D&V$B(GA3OsoH*dY1RrAxvH<58@|L&_ZV9L^(O5w@8%_ev7_ zS>idqe;3sV69QLC8VC8Y~$3ouh*58TiJL}DH~+# zqR3Z**c7;ZxT|ZpbLM))!+>x5t8s;5*2fmF`}%Ue$Ve+*@#mN*B;*vQ={2@e5x$X% zr`9u>AR~UdAm>mKezZOaTqO=$Fo87@r}S#@;5O4$ ztfwebDktsl0yfV+Alh^wByq%*zPz>wD-fsHRQh}IVjnRG4M^uG5v|H&Ryg5|&s)dj zd1hSSh}NZplt;)He3ooW6ji1~R0AY7Ka;H=dXSsHS~7iP;Q2Dom#*aqI%TLh6&dm( zmD9``6&bC#ZN#g^$4ubVoX!**g9wgN(ygp#(UsE826dUXX87@(!MsZ#U)lo|m0h>j zqs#;79t|bbPADOgaiCv>)@T>thS-8OfK?%>g`cB$av!Z;qY>}y*lNMci3HVmQ5XtYscV&jg zV$8)HrUn4Jw1`qjw2j)| ze^{;2l~JC|xYL-;#2YN`Z$p3b=-D63b6igMh8Q-}stbQHj~>nXIkTq!rDxB&OGp*N z{Fv_T4Yk1Z*mU4YvfRqzu~d=?MN&sINnS?KPE|dwedi70Lq3!W@t7~vIK~E9(UgJ) zWpYYj;2%9&gC=GJ3tT3%KYZxhv5D{DyS;W>8tmd}tK@~~>enBRz#$*vRo+Wxiy)sx zC97Rt>8GLA_oU70>5=6@ ze-_`k_B9(f$YziM~h zHG9;SG~y|cBG74Wlo{vJvDd4anBTccfYx21B>=^!ajJ-LSY`$R(5%M0?5=Osgs)HKEORqOoNM@zg%fhw^up#tC0NdD^u; zguu@*DXPA|Rku5dR=WT_B_Br~f6wpc9}&W$o8R6`=6M&%cq=@pbnnT?GSn-({7{XN z#Ed-WtkBuALAx1WSn*04M!hhV##pc=(bEgd-o3k8&vO-H7$o`(j7I`-sTL}bCDRoI zJ(_8qq?^el~oM>Bf+xTj6;#!aQQ^~q<(I$I+g$_ogs6@=;BSa0S;h%ll;>~RJ7uJ$ zlMPYiP^YO8vBW40-`9J*H<1HKcmW$4rE)u8n;)4dm@$(}o}NOkWa3|3M@?_jO z{d#3iGb>BWwU1oguDtu&|48})1)L1B`td6hML_UxijTVAH7cIf<}Jd_q)Rw^T#ZfzVBK_Wmt z>f#KJpX=zg3|4YD4yMc&$Bf0b6t=c&EG^t%y{E=8>WH@1{_m%|=}ynNHjn+5Q8zLf zp)Na2^9~cjm$ea^-8&c{1qle@o@o)kop>T7kt8~|ewITzdlzt79QHoq!9WVEsM~#N z!o*0yKVgEhLjS=*{ax82AELx!Ls`wZ*Wn7@rko%yhiRmtI-1Zg$jllRDL;0o*+?%^ z?J`Ap$y_o3OQD3=wkVJ)iq)9q(DaDogx)-5cX17}EzfLg07B6g@E%SRh(b4gYx_>w zQ%}nDje(YhRL1}GRSjD0bRDOr!a+bl&-o1O>%3pwl zKgA$V9Zk-7$7ECh;wli0JoVFx>|Q=DpC!{!yhG+K|K%Op^Hc_@TB#(1k31q*CXgM1u<%%4hZ*aUsM@gV3>odS)NAV_DA-ssf3I!PZ zHO*aqH#V>PNT$vNtro|9{{F#io_bol z$5l*M)*7buab#ggduqBtiE0NDKUp3Qv8@bZwm$-z$vz4$Q`6Nfby$LIAMipDgAJQ5 z6qt%vK|^{zRx9^ztLqtJ`)X90hS0=U>zq~I9*Ky|$rgLc*b`4^=gaC7Aq4hm)f?mv zRr(;1JGRBZSTagO`IHi&+kIyhNKesKur02+_Btmef>hC|SE|x(M=w1X%JDILzj<>} z>dodI24UNxf7xK(hXZ1MK6Q2b_F}X_*OeN<5vB6#%5j@V*`gH9Pq;Ok1TEd_^U4x9mLnB6wm~xSt{N!f zL82OqeEa*IcH`5qP1$%=ai)c8WdQy z8=!T^_hIWh6DvdOL(gCBX3H`Qo#uRodZ0hGJ33^`lH+eSD|5oK8Y3H$lvL<~gGY~M zAIU@kAz~1ooB0{E2J%W$6$?%Lf9(wU zs>eht3OUD<6p%nfyG*2mQahZdw~o$_AU`dCDtE3ZH ziV4F}U@AgU!tcH8~{bmFuYR z{>5y7l=d>jG}o}rpofv9<1@d@X4c9ceGK&ix7nVI$Ivd>GXSKKX)%4Jj0K-CBUxM! z)iDhN>_8eMVTjr&(nz%`R<`QQ%k=c`2~VELtQZan$tEJpKyhV>({-$SNNmanaB}MMno;&GY1_M; ztfQ@2b#%%T-`hpMAA6Ujb24jR{Pz=)x;7iyewt}52l}CPHlI}F4^A5s@A)DCyLQDn zGgt8yjV3lhFleMGhtQ`T$wcZQE0i;R|2EUggU_nUkukLuI9~TSHP2O7vtgTw?7Z;u z(}2F7+%~ui!@2`#Tlpt0Pwm14o~u*4aL221g$5P$b??JD#zn*S-+~D#XIv(N=x{!5 z*wU{=RUFu1+e2_FfJ_s9pq3btB$MPMLV7*9ET?%B z(GKYk*?fRtDTfgIhy+pAF*az>nMg@9MSVt#+;p}Ky+fvPU|f4?WGyXM_D$RR`TLTZ zcv2CtF$(LpHYx;{bu_3P{0m2f<{8PCRle2&S&K@GzE*=O2q0D1BM%Rc68|L^?L_oU zwTkw|hrUR*eb#@4#VBtAB0eb>FJ-G9ouNETV*;hGyoB z^O)n~B-~zzU*<1cm&oIU&Vv9+zlgDx_C^GyGkR8ooNBeBUs7Up?uA%v0ZL_}4viE? zlq7hu;D?46A*~Z){%DBjt=yWHrtgK>67)9t&s!9N z`oZyz1Gcw{uXW75r9wMdHU1^LYq;(1cg1&<{LMqjS8$pPg(P#0U~Mkg8OK#I#^WXG zC0ExEUBJ~UUKgRo5ze(zQpm{CqnDa-cCtBC6NLkcP{hm}c9{MGJrLy~s^{A(-Rvz| zn9pkS2dAoXVb`3ls|on5rx!nCbSTY}MzoOAXW zMQQwc;sAZ3{n$oPzRPKMf>RIVLQ&sY^@J@TM#{WFquq4+0E*{4j7Djz|e^$F5{T_2~$aA4C=k4 zeYQ=EK94{kQzE?M;^KzSB&fF&#{tKiB!$XXCweSGBqOH|m8q;J9{T3wCd|L!QS_&? z=5iIu_&`|VY*Sd5U?9%)%lGg9*43eb3g=`M6pUy?2pzh`gyxzJnIX=P%^>S`*JH#F ze(0%n+qbs~bFQue;Aq80PQj<2j23qRl?a#_VT6>5xvV{DvPdH|EW(`#Rp65{bchdv zZ{AJfkHdJ1$)mj;bZfcr^siEjet)#Z^Sj;Ph!h3+VsaQLcp{b8Du?owM%8C^>DS*s zut_b?xj6$*Tq`Rv24-E;z0Muqfff?DWGcD z+@^cyHW*jO^MV3oISs+aErOAsxZ=OGhMo?W1x>2P9=;I9o1FmHbj@d;WQ36!j8|4bG$2l6hA$ca8ga{qJY4 z@oZC(oM;McGJn|;dRL%5(g$9zDnqg-iaSuiS0xvL5d`KzwY(_`L{tz1yovM5zVYMp z0_n1lh-2-2pmbRzMx%=ht(O>Y1YE-YeGm(NIEBcUy=$=h)xCkDd`x)YmYiW-%H5+xl;So zHhS*z5FhLpq#k-je|CepK_5>S&d-~mrQ_jXxKQt}phoYj8QyO$=ebLx%qOxhhE99v z;^sI%bupK7-kYx~BbK(&D_b^JF0Y2aCua_4X9AbgzPD9{Rh9#bB>z^wM0b{^4UsL$ zAb22P$Gh(qW4fvU|0kROv#%n%6#I|l){r2_FD1RE8p+01S&k~SDHY8C<68kV9b*Wx zrRR_YM?W+}%{60}gJCgNFPxZEhl*l0t^f$+PlwP*Cva=n-0)jDaO`th-?6Alcv+3KC` z%ax<2s`qkJx%{cnX2i?R7xfX{x8o20X|oFNF@=pB`Uc!g_y3Qr|A5E3fBS%O5-PGP zG9qaYA`)dwlF}yGDx^X(LQb+X8?qA>2?+_AnUzYiQ$|Ae-s61UA9daT`+q-=mtR*u zuW*j<_xK#gdmW3B4d4k0mxngve^hwhD!j0rLi>qmSrU+s&WQx&EwZWj3*5##&z1BX z1s8))0u@R8rP+z$6K}kWcK>`dnQcf)5WEK1I<}@4v5-?ei77+K2f$t+y?mF!;}8uX zp~V#%H`-a^l)GHJotT2AtrJggC*i2z$2rVl@q{*+tg9R8LbKt(3(wIa15P~<4`>{q z<)?57-&M^Q?*Y9B2;;*v3cU|865<{d+k$V2bDsH=wFHZ*l{2{&eFh&!y@e_jXWlExpv@FCNZSQQN^{rcH!}pPrjoxv_rDu8+cB@AxFZYfp^VekuMSn?=3*m z5(CN$45!nUQAZ!sDLj7S-XKKa4BE$B+jVXL_!P0j1gE=K^>nXi4H~fj^gLkNxL0vh z`2sM)%!Nk=C^&i2~l)Hqd-T$wxNk5#;Oi_G?EED+_7h zqRXG|LR=xT8*L6R`%_pqU->LgzH5dC$YI1b^uJ9M?qLYx66pq2MNe(b37E{ zuZG}Wqs4rZ)At}?;Jezm@cfr>2=MOA01b?v81`-3G+}vDRE|#0eaHkR51Ky#3To%gleuxVH z<=x;#UCg_wsabme!%~7?x&16j1H%L>APU&SnsEmi5(kC`03Hm7pHE+rBrCWNkz}9KY%{4 z*F0GL(9$@H2fvbZTtw1Grj=Xh85**zgg~fXR8^@&CxYzM7!v=E!=KE+dUnkiKE)qm zz9#?$sqawXd;YsvvDM)lf1k;I@^bzCusHHRY0jo*X2j?PZ7M^qUiV)kH<39)dP4?5 z;LvTTQ;QDJ*ceRR9?<-?Ln2FL0RLBATKUMIm;s5p)b@98j?cD1C85~6rs#F2Mx&{I zh+ZM8oO}7uP%H-6!S({d)|Y8fWuYHvJ_$ISPzp&m!UDuLIqZRcumP%gvIdgO<;_U= zBOEJQEkYjw8qZTrIQ`3wRkkoh-+9~3$}!9ak?8%=k_U;dI|@q`4)iR?$(LH@rly5Z zzbr0QzK<6d3xvx+(Jcn+qr#FMryy3vHa*EC85hvre6YC#cL-kP4}Tk1B(a~kQeRW^ zKD2EAc%Cg`hMJr8AeTV~Y*hrbj>EJ(C5)zDzt;NpUfPc^k=DEW*ZS{U;_`d_%1hj@ zy~ol>``=Ok95nout#J$xDS#w^C|Q-{Jpv@~CZ=92DCk?| ziM_YOvn^D967P(%Q}P!1FoYq&9d3v_WQ3k?1712ZStOn9b(tp$(6oyco*m)ApatLr zA_*m_Fvwx|av-5=M!Vo%aaaRDjd(Kv&C^X(A|-@CR`YzG&ZOleOPdg8MYILb!%QpU ziQI)E13PCxYx^up^A~pvp#H1uhBV)->B`>!74^tYivx!IYT`I0drXNG+PXgiN%GjH zL`Hr*$b%pYPzpSyMDf$BzOGtI3sMKkkBVp-hy)e$`Jx1sc(6&QbDN$C;Ox7Vj*K5L zs)A)Y*_-nL8*5cz{Bt3zZPDwNpdD(vKL1uF{co}@33xzx8_Uth1NB5Bjo+05g~0a-Tm3krqtWp>pXv(L(QCXVIKX5)sj@AT7sn zFwYA=l$d2RFzVr`TMR1!E8t)aAiW{ogcU2%HjSfYg;dZ?IkNp*;xjM<1Ok}FZ3%X8 zE@z!QSXq)CPqbCA{Uc8w<+Vb-e&D>ewte8Ts%0GA@nvO8CBCl{UM*?Y{t}|{FYl5M zsTl`Gs}+jpiF_+|Vt;;u#EjuX_3Zz;X6t)VxT}KNdWG>4v->lEBSu^S3Ay`pX%w=m z3~d1O2JqYfEnoMa=N^3w`r3a{Ik>C;Cn{&rG5S{zFwe6I!-W?$eaW5Pz(Ok z17L-+Vbqm=+?wP~!Mp$P%0q-cSps{31`yb0!29K`IG;gmm11IG?8DlfcSIl^8oDnj z8sPWB%`{-sui#75{% zRuM1k!=2}D;}vF?GC9fukZZ{PDX##_>&1%}nQQPlvth*#G|o>F5)OZ(P7NC)`N9x^ z=0tyk-A?nIaA*-83}V)&`N#z#wv-2#hj7;nAzG0Zp@|*{we7`=iRDicvU5YlhWbxj zu*+3kd^HjD0YMeIIWJ|?vszJ65d!p$q=Q9{FX>>R8r&EiLHZJJKR+m4UOBZvE$LEw zxfJS9vbJj+$UD&)%|+TRCg`9mDM82(L7EpuF&$AeFHOF|B3D;;7EY@758lsWPc9xJ zYMc^TGL9FCkTlM|^8Os7v2(l2pDUe*abA>=3!3B|E2j{)IruINBC|KD=fH z{Uw~MkZ>p@pREXDBeO3gz9pr^5=6cclK1yyk5id2Fjo9HLK4n%x-dGQTWf|lUg+|0 zWvU|EQbd}xf`B*b2X9OKrg9NzZAUiT;%YL#azz=ADI7O{Mc3E6Z%7}`8ChxIWv zl7&oA8Igr0gEt;YAnp)Pmq@XR+ui|A5{^}?I+l-fqVIlX-F1vCAH}W5)eD5Ip@1TX zOnn!W#|o&E^zcX)ZBdK8ETq)`dRaW?-%OHaNWO+8dI@F(HuHZs3)VRUmET4JctWt2 zpp>Nb!On142J~M6JknFLHmjGjupm|21-W$QP$fX}avz>Os0KWa#ozzb{Q|C1Rji9n z$PI)$>d+yt2IpE?7Ukdb4_4472+FFi99Fph(+9Gw1Ft3u+nh&j4w>Dk+PV}L6$O=0 zk)n(PTgYX)p~u(c19?Y^AOCFg^sP_u@%9X^7_I2Fc*Sbruy`Tg#Q@HQaz(gt@D_T~ z!hcs!Dj;OZ;=x~!DMpV&D94CG0!aN|av2~KQi%V@z^|cE`Ix^2TZXU=g`^I5F)t=e z1o7e4U|VXanlA$21Tqx@)j)lDFP|8OFQpR>Td&}iFQ6-OdE-i@oizsMj(oC3lhWpA z^}V793jxyUoq=jY6u7=UXrf-FV%Z&b6~VaoBSizy12&FhhbVX2=t>?3ux!PEg-Q?3 zu47?D6skLh&YNzypYCo&BSc1wkRxPsQNVlpI^3$i>In0nef2^t*)vZAFTpaH0+3u=dzEn<-d0Exjp zZ6G0J7v8DZ%=qf=#Vk?;_bWl$rDd1%lkC(5vkCdoHe(!Q>wGY{-tv=6*L=cu!6VtC;5 z>|$MY(cgS}4G>9KEkMLNbZ3Dv#JO$+Y4P!`_NP|kd&!6YE=W5-*MP0$G)Iv|h$=2x zVvk1UWgKt7m(V0(t>nF(Z(?gN8!pD=mIz^U#P(W?9C}T*rQ{6*u0;j)N%q&1jXj{6q+qlAY%YdN z_sVT=lDclALe#KYy$dF665g$1O|NjFlV0#*z^AyTvf%t9Xv)~qh!F9AWQzb zrNc-nX+L)FT+e=d`nogc-)D_KZ*KYj3AZyc$gW?+$b50JohY6YdmZ|I%4Yub@`Ko)o-dvPQZL;gR- z0q$pNS^YIc9b>(v<=CGSkiaaiFC*y)7#+GZmp_jLH%7=uqT6ebGyArY99pQ&#gTm^ z(imPalB0ZwEH5Friym*H?l{E?rM>FdSmXu%;G3ZUIg|kd2-%D;XuNlP)G=;1u}zy&pcJj z5&*sKLy#C%1rUH1pc3?%r+lW898W3#<*@yU1F!yDt{X+LnnEf8>yFQ4ahPdl2uas z6)qG~en~(Ambcy_aepYR(BdrjxE-;2)d!?`CSku3YRbLr1GqK!fkMQcTw-H=kz_Iu zg#mI8AIyCrWbwjWmmJnf9w#7e0g4F8l_GL7I4@jrRTPm{=EB%ZJe~gmWT3ww;S^{K zAleFkeRT6GFtjf;bZ=KEiCIG*e3c?<^}=y=}7+G zrcD<^@8@OXOa%$0##t)^qcXa;Y#K;eS7w7K0x!O7^Zf+m){R?t4{D%s(mRwl?)Vqk2Q=#>D-%;OB+QSiyDzAz6eGF{QCRSx zsU~4{gAs^3`1+5*eue^utD;b3kg2z~muw2R*e&vHk|%%Bgf^QTDY5bLn4E;_Tf(ATP7c|f)JBp{Di*hHpK9EkZeYq z-uPp1ttst{CfZ0yoa9OO^e+|-s28@n{3XL6hoYsy_5Tl7WCi}>tP9H>Tpb>G6F1OA?%!fnlCX>PE9ZIiL$LM{yOPQi?g=M199yX5Ui2W%%!7%ogk(p#gxs5vu`i@IRTuGu9|tM6inA3|Pw*-~xtd z=EaOOw3gTlhBE;Ufq&?iXVWg-2%O$Td?Jq)lrUx}5MkdwRzjqKfg51hV%fd0d+iX( zlP19y3JR`IQ$S&Xl_4RYP%APA-#O3A{^1#30*Ek!YS09w;#LUb$U6@LcwuguL@J;{ zZ|ru&az_L<0aH-IGQ#&z9Xuk7rg5zfVU3wuoyqicS2Q%scYvIH2)%X%8QmmT9d7Ir zZ5ta2vZ;;iEJin5mWCV*GT~udS=5o>I^O)U{NR%&9I_xHH@-B8{K_Q~Y7l;t4}YFqUjmhxu0?O5FNeQ*^8Ks9=fQ9x zN`Y);p6BefufhYw(cENFy*SU2tJ%R0j}MEQH~BtyKuGVKH;{9aboWF5CqnJsTlkyt zzT*?mh72XpWvFqAUBzY(A~!&%ZI6-E_iGW2nz62LxccQwzj=NJ`ZUm zL6iKAu&{>M6lbgA`STHr_^q7l&5+9w#Qth#)Bjp31qvP*15s<^$|Nu!csbbK zxGe_?&`SNoyHJFLcI;q6M@#_ujbvb*=|>$L0`UoMo)>1HpeyPNI)IGGJxioC7!-li z1q&wR?&LgD@Lg-pdVOr_VbsXH$!07)tM?wR&ozH+4m zAk;>}KLSo&uGaCV`Ku(lB1otJGkOy1A*QzftcRF&HGpX@CGj~#{FO?fI@=#87+4^z zE%F+%Bj*?@a0se*j_od7aEomD=*S3ijJ!%pl&6aj0rlH*-(n`Fq`}UIHFKL*Vo6MnU(*SZUFYPqmF1t0f;UJ1{k1k`30L^E0S#>zV53h^dvffB2)2? zL4Ka$SVRG2=>lyua5_8Z0MfHq2xFo}Z)cPPMtw1r7Ec027W?BtdA^ywCjDP^#m~Nn zfdniN0eYxtONHd0JlMa;a#e*E z=J2-Mtq4g(bkd|zPTEMIcOdIMQFi~$Yt|%ZS(p2HdJ4bJ-2b@t#pC)}qn;W|^)L>W z;E($y7oI2g7q*w5t*LokY}Bn{mx)`MCnhFe^+>fb{@K*j&_xmvX8WU{#V;+5Q!Ck) zM&9z32s(+~)|WQkzU|~Z3K96*p{Z0zORhdo?3H)Sp%V^aR8ZhiR++*4xdhaL(pZ!| zwiRJ-8H}5cvC?G8rcL|rhOXUdmt;8n`!LhOOnpw)^=F(N9oTAVS=c?-vBd0n5Ucii zs!7-?2N3|5$F_)UuFe^?h^K5?pmtJsGw!L|Tu|MGzT-LNibuE48r@woY^&wvvslWW zvaP+}ybyOGq}uW(?g2gyvT)>pL^W|-SN zPZ>nEsN2bH>vyXA{{GFT9?cJDh5rxsi}l~W`HSRdd7X;iKK$n{eR8tQpYCa!_L11w zepQXYASeHdjxtxeK&v4G?mND`RW%pfK2P+zw^JutIDZXNof%vtDgK?)HWX@)zHSj7 zH?p%xq$jGokJixF$P@9=rN8B-= zC0od@WSqlLle&Y)Iw^Hy)ex)!B#Jy#J&tzv)X9?|qd&6eJywEN3MQtFOhS6-dkKP0 zgsvaZOl-U^9Yt)RaE8emP*7~pDKTVxca&Qd|E}WwdwHl-8*uQ5LI4)3?F7s2`v?eC zegWS{Zt>9@sPBKqiJXd@!f-hK_c3-2i-JjGM$jKvGzbz(VA^u{9)1DhLYzGZ(Qx39 zhPK#xR0Kyb&k2l?Ua+|Qf}5ITq2SqXoJZZxy!NvQj2lt0gQr${#xrkTHu>}Q19JuB z%l$LylO-O=qKdrmg;1#f4eJOK01k&-NFH9Yk(tN?cj4fNx=(D=y>jA={jepPBviO! zk6|xVJo?Vln@HR}Vh;g?E^)qj3p?^aiN$H9<46I-k6<&ou^hQD&qH88(V6<82K$g@ zSA4QgYr&;@D7Hm8{s3Ipc0C)heR5;7%yj^-NiZaq-l>03J%(Zo{(Th@s<~El@O&Gu zQE}s{&!Tgd%1qNv&UKzgK?}qI;Ef=k86WRQOkcO&<$R&6{YPYWCGzn>LLIa*5#hOV zOuR|ARX9B*o`x;%%!ZTkMf)fcM)S2u%%7AbN9SEW6{>EO-pj!<|AN|QM+A^OW>vf} z<#OXx)4QS3Q%}-I_4BX37R9oI_7qB<7j>b<7ykMR5d%*TGrTNjQ+8Bg-k6w_1|iBrsmI^H03$cJ5y&r&`sQ$AE$Gg)ZKRNhpGILBa=G>wyc+jNFvmQGC7!bH;8Sn zE?ea-wV#r6n>rWGv@t_-{+t|^y;#mW$gr&2UYu=RiOmP;S<~0`S9rLr+3{=GV&Nz! zr!e1Ci@knS6wBN!B|30!PWneq{l>=Uj_X%#c1OTN+%Nq(UuIJVE86${6OW&k+%8gF zwSox>e(|<}pPL){3$BIEzNnA0>!40k20*yzUn`j%jgA)-6}8uj))yuPiZW9z1<$mk zd~ALCei_Sx-z|bw)~q5jN=gZ9XxKP90=doaUp!W$ADGNlKOt9COSkEd!-v6f?_q_| zoU3AD^{lcK%_EX8oooa$10KZ0r`r{ZR^1=d8Y?#o6{pziQ_Jt&9ZKJ~_8O~!-Ob0R ztk@Yhw1!N&hKSVD(A;*rH9Sd?Q|eS!f&*$L9*z7J>U>Kjj)oLJA5X9Kb0IBbHq>dk z%JPC~$#Y7PotnD3EOUd@*)vY#wMtsa^OXW~^R+nqmrrR>&G?Iovd(dR==v0<2D;M< z2i5O@Uu(3_=L~C5Z=AnEU6{O&NqVDktYnvE7Tno^AU+bV5GIXe=pYetgc&^^nv`+? zel6XeUzCO&Q#0#57+J$h&2M;kT>4QUk|(TnZn|7>nmgO-2|u-qdUKUYrL2-&rgfpH z+B28P$gOh%)I-9Zm4WLlKbdNtzEI{kxX$xK&3Ku9_4n`oYj&_8B+Ij&H7;|6l^WzP zQji-Mz{0rDUog)dv3FCT&*%vt@jCR;Dta1 zl*$-y@AYhw!B1#T=vIIOk)a(P+KZ}WyPCK?umD)5ppYg!zVpJg2^O1u2*p532qF-p zBnkz&`C{8UALjXCO~>nOQ8DaP{0ku01yo^coSI}oIj-w*xN(IH-x@fD;Y4$$@bwY7L;HX`1N)x4OxK& z=aw-5hvVC`&9eDO-XBB}F{DvDg9LHVTv+TFBFfF~Zwsme=_(KA^k`@>u3?u6vPIP6 zE^N%q?@h()K-OSFt{s9f3BaYd3ku|*;w&ZG=@7679`i_x9FuQH$LuVYN^d>>L|OYe zhgnQPpFrCU$FCRcipb||hiO{QP>*i-AxU}{%VZ9fsGbm|M8&ML^ub1B&X*Pe4e@)= zq2*U<*R{Cv-hWj`ZOsL2tUlCi+Wxik(*A{B%C3^;yg+!yw10P&COGA&WL-G zJiWG#mLnH79eb2iViJ9{ltLZ!jX7H|AoKGylflzIjbzZ2)DiyqAVXUw_R)o51x8$HWh)sqLH^tKMUo z?flc_eWix~kmariYVkSEI zAJ6v9*K<|PCCy8;f4BTYwOhj~EL;4LDlWj&A8|C4Vw=g{GDi`PdD6J>(aI)pXTxlY z@CIvLvAuJ%=Z?<>)(i4TH!q}5od@_3b86VcV;POG{!8{sv-V$0{#R8yI>%4jI$03? z330&Nc4fe`Eo49PwBwnv^nOPK+0#WHIOkLBRc~0A^xRNgbqs-gSXZ+zV8Yf*Yjpg~ zJAL*O{Xn-@KVn)jPtq)B7OCc|V`#A94%r5`0vFuJlR|FLU;_e?}D@EqTQY zmZQ-ni&${g)z*p>@V}@sC6NMP@qvj`-dm52zbW8KPoE?w#MoF?^T7ir(63XdlOaR( z12js{1vboh1hw7qUGS%pe=D?U5-ZgE>tnSZ%~m<`EfaDA^T_aQ@9K%m>Enc7Wx zyMEeFfsN)#6|3~ml))x_o8;(b9j#E7#nzKD7by9)=0Z97?~wjN#gc4eau(w>R>v45 zdhN%)4@PIv(m+-7mk1|+MtmklqZ(td69^l|=V2LmaQ8beQ=iq*@q<PrKP7|^N#nr?k6^Q}gOB>V07)R;(~7KX2Boe9Rz{MJD4f}qKu98ThDR;&p=_l9glEk$zk8aGkQ%aEfHs)YR5g+p8O{Uymiz+LvRD z11%P1@7z%Nb7*P&{r4v6++3Dd6h)WVK*Ypb?wKYqT6mgZGc6heT%C_Fof&FtIUK1%ZJ zi@N4C(|s;WC#BJ6auaIBrv5_tm6s|5iav$i-M#k4SJMa2UtTN!a6cF+7WKShn0+1b zpoZrsEu59$T;theqOxJv(ZJz$H@4Nk=`x$#%2aq)tWe))^Wx->Two(RIzP{HW?EP% zovumb`dM(R$?xUMX3c=R#grM1Or+4!(LiV}#>^lWwrXWyrCh;|g##~6ui!h7#c5u@ zc|_@f!k<0_UAz8CejWaHC{yl{N*8rsC5?;A?p<*^{STE)t*3Fxj$Jufc_y}cq$2d& zoOU0Ts`qY!`exaRi`1iE^iES7Zi`R{m>qH`QQn+9t6V<#wm%MB>7H2c;VHIbASzL@ zk~$@$Y2UYe^>$j`1;dHslpOIDHWP2zV{Xmw19-Y&gQAxU&1YW!mCJ8R^4{4i(Fm~8 z>-dS`-b*iVe89Hsmr7o{qHyx$`~4r3$HW{bWfdKB zl&(-m{#*CZ&#-Dzhn3peC*$sqR zb*Lq4>h0$f-lB0<^<@Fb63F(NrwGYTGqO(XI8-nAt1_uU^@aWl1t!&oy zI%Jhhp`pGk{kd0-!@G_4UhBYWuaij+I89?Ro12^TQ^ww8_9r*LZ5%zy-VeKQG)r+B$!LmDLe)7^$$Rb;vT#rKbn!d|^RiT}{TOze0T#5O;h0 zv0tUMlwL804WD&&btL{6xp3kS+kXAJ4N*gJMMa_w<-!(8+deI@O9~szu9%=FB*QH< zmPWtB*%g;{e?rC{nc0C#Cqim%PZ&rBR(|z)^JrQ+24;EWtKJvhx>X)E^9L%O)(*vG zg6=f_^%n;Q2B4ldin|E$c1uf(cg6;HHpRD(;%@08=hN@>D)R5x{{11!kKjILzI|-u zCxTP~gM#TGC6!_aR|O1MMAY{fB@!U_LCBG^ba*mvlkbg+&dw%rswAP2G|g~rp02nL zMK-WXV!=fOiF!Ay5bK&6{%D zoh>vx+z9GCVpLGxaB(gN5BXhH6+0A=l1BZHbU8UW@zeRq2=5Uqy6^y19#F;bck`5Y z-$oj}3l1*pUJKv{N){9U4>aPk6kVvj{VU{YXrO99Zlm!xjOT>3^>6&<9v{C2>}lAI zW5)v6&8w%y^E4i+ZiGj})8GGE>VxHiPoZPbmvdW!m73q~uIayBL7OzCqqY12mG6_iP5t8Mk{thD`Z?n*hz z?ZYDHzNI%7##hJLAnbU4_r*J`eSNPPWvJttQqFUt43ya>iTBhi>^HljD(nZc2iR6b zCwqNuxy4oL)FtC6y9kUSf^Y0dIOZZH!q^Puhjai*LH*)T@rX)j~ zto?5K2+4AZ*&Ip9u67FwQ)$vU=d}kKRvvgX5yC$@qHI6ik-m>2yGq`;aN4dp+kJl4 zq-V}9G<@W1O7tE@Mfc|~AL4je8Wke(^xi$~xtU%W8X7OU+}VNkPil(S4*WR5Q2%Iu zy+Hbe`=s2r=lriE{{^M;>0TfZO$zDv`G2rrUZPPbTdqD+A>TLC zy9Nu9meMHmt0_#fX^utz$ootYy7{)sDwJ&mwujymTKTzwCwhlwdbx)GoY-{VdA8pb zcIDL8>%mbg7N#p>u+m-?k*5+{SIe83t$Vmr%45IXL!Kuee{;tL7zpqO{wM_Is5R{sKHZe2v5mfSAk4ZT$F0TJ%NPv;q zHUJ1rc)?VUBo-!|3S>SxkajWNYCvbROPOt^T{E}4hAcM&FYo83#2AG;tG0YIgyJq} z+)m;}`A$vEBaoOBpYIbFSf^X~I)|>1x**B$SnI~o>_ zmMgNz2RrdZ#SjMN*5R7jNSR$p$;qNO{u~Dv?;nOmV4fs>-qEhdO@5i6vM#hj0sJx_ zd6o?>H!(3G+YS|DzAlpxDg$55bpHJL(`UEO9z^a!AVU$Ud(z8AL`3T3+r=CBQT1Zw zn3!w@im?eysP=zqAgP9afe5Cks9@65(<8w`n1#KN?sJwZcm@swvT)eI*w{_Mjlr*PTWdcucQmkXV7l70a{**6P5i=Jc4Q-2~{!I$e^yV5bnY6$QQ4ldub zXAg1q0M za4+a|LvP%?DFs5j6(d2envEh}s-znyPOt-7HoO!k!{3))^8Gu{5?2}yANR1c+S)64 zd3lBE?y!x#M(rdV)|t4i$L~h3)Y$Vl$3I)!84_OUuq7LOBBwQX1S{=4^;qGhi&mV= z?@DUvb$XtKIsMIytB=dmiZ7fJFT2*(`azDxUZ>E#!gX69-6~CCZeDy)qYMnU?D8JYSb{Bv@lC-6m1&xp%gdz?Ng5 zTth-SJ&$0`c;EMNzI*pR0(LY${$psU{C&l$cRzpbUYBuZl9fg)JIx|}y2VjMIsW?1 z1H#P|xz!mNC!OYokJ8B7WUO_Z+}zfm$(ZpnP51ca%TluQuTyD1G&b@@Q)HKncCM>W z^b)utJ$W!R^xme-wuA+iLP*8EKyM3waW6_4sb;(u@*?Sb?ZFs*6 zP<|Y`deyi*c(^^Ut;@vXfx}v*!-v<6Hda=Cv0u|&botqJV@=nfvq$p7+{#lAs(-h8 z%*G_kyIfdv-qkhX8I5l~pMQpjNznYOVeQ{N551N}t)n^O_?b2CNnl*;&iL@dj2gWG zW`mxWKbdzJ1a!!06lyR9#vI5L{aTT&KNvf?&tiyP!@@xPo9zw0&F(<~Yt{q@x-kZe z{85kUL1sB$yy&1lmB({zOfBhreZvWHu{0VlmS+adxh^aTI|SvcXPZsBoT+9{XLOhJ zkKTgEj3oV|9Z<&Rlsc!qd-ty0du1&H3*RzVd4nN`8xJH~_*@j++?Or4I#~HA;8R{J zgOXVv4J{4LVP%EWTcUq}SP%TJv#>CKc%P;G%`b3;_}KSnEPNI=zWlL#Zrb&#o}K2i zx6yWk*Ah>8$&R;pM#Ur^6t^F*b=E3aprwJM#0!9m;j8o~0ki+ri7`OpZGC-y@MINZ z+YrJPZ!5+abfl)G-G26L6KQeiMqhiXZ!U(0^loG%GjjW?lfgd)?B06 z3A06+$erm)9OKx8OV#uA_Jj8J`+#`A)MuSSVRO5+GGjTN9 zi69Z7&!2D2DD2a~(yF$RjtV$VwmXdXc!1Kq(Oc&~vz-*Rr`6VWcIB|#1;OVH8E5R% zr|uH^C^jgHNRbdkB%P;WvFAIoLnPyB4utGy;YN9fHdi%6&^_o2q%$k_2RK7P{PUh9d4{+6-+rUn0dk`|pC zwk7x-u8eG?!y&F4=cKf4M`AJgQI%XVuV3>oq2>DWrSng11v?$XvUgp*d$pb_#$W>^ zHwTB>-TOj{D0Q8kPWz<8V<;4^vEHsvOLeaL9tzyfw&sk>uV31y&)zXKeCHK!((b%! z_3!64*)tB<*Y+Ule!RvYi-<_XrTWq?>(t9<^mC^>pUF{GK*1Pg<($u$9(@1hWzqC# zVA$AW1;w$!!Ec?{&*Tl4(O~G`_5Omt!s!h{Ok=`Nl9E`k31)+c29&l84UJ*3=OxYk zxZCNERJ^e{&E->)&KQvw*3oXi zA73}^sZ+(wWdZ{Fx4l@81VxJ|D!DP!F|H6byQkq>@uNWBFE&|K#QLe#Jg8SXDdLe3 zyqxbqQY-)OWf5m13wSSYdY`N5{jo^9=4?%Ii6YH;E~Xo-Rvy{@W{JI*)k_0++?{{5 zw02X*Zqcsdv!=^iTov7@GH8prMLo7wu8Mh+#(#FA({kl2uXxvhXJt1gFG%l@>F5u- zloFVAKsYX6Vp*?d6g^Fi??&$fk+)NK%w%ZEhE%?LkQpu)7^b~qcUy)5-_#TX&Eo@_ z?r6W+4`k#VqiC3%IN#{vmqY)2XOhQLy%oGBOjFF3u6a2bsfm{?r?*{vud(gW(VOd5 zUoZSv#azTcVaa-Zw^mnQTIA*5n;*u$c*^-pds}4g)vtE9S@-DgJoS9n{b-RQh)1^W zOgeb(oXjUoS=Q-_gUtbdnli^HX%X(MlOO4yl!RS=K5czhTwm?3XUW#TBGR9c{2f*p znWEzKpx`s~9!rLXUMZ@2?{FA$+v4wU zd-GQ}arRf8y&rMj|IZ)APP>wJD}$?7muP61FQu1SzrJuG%rP7B;44Z>R&S6iVAybX zEiLUtsGQu_hT3mE=MIapRJ%*{s^ek0IyJTelqP^8TVUp0o6OT|kW9o26t=mQ61w zr2DiqV%$MdpAVkn=8}-$Teft?GLMJ&)vHg%v(N^Xo7cX*Zk;$hWVl>RZ2O55RW$s8 zD0+LQrW+04vrX=m%c&}_lnb=TOylKwYCqdwq3+ODv_5x2n`(PLTJ-Qnu^(@P5h}c* zGyL$@*Cj!M*M^3xEOy_kdtI9#T%D!UJMnqq!Naowcu{`;KJ8ZCURYhtV_fd=?nA=W z%I9KYJkgb_IqPEMRF;^SFfC>HZv5xC-H?4hul<#;>ov#vd);wktmioV>ZUbcv!lpU z-TK4-(2U>RdriNt*V59$xT%z+FsrY9k+$$>j4qY|D)Guixs+a5mUYL*mV!Iu+duAJ?-#AiE z9Dm##{j;cO2{hA+^76BCO`D@mAzd8-du2}{RVG1%2!PFxmLfI30TB>dW~o}Zc{gN% z*-B8s7gg(x4c`sqXliPjJPrR3jUoS9wX3A#I`M$Ry@ND1K`6%%B2(S)XnRP-zYP(H zAYQ+5<5IW_+!D;{`I-XB_g6BYWhg9^M+O=Zd}dwmSb;QVt((A7NN^fdzKGT0g!)T& zL^qur+HZFh>^Zbdu;0J9%>W5v@JyE?UKzO6iIk>ujSrLcb+c!cb-VPIeKFC^z11d6XbFdl- z`X}+;Z{E!Km#cjbz^?_6Mgl|eR_&32PYU+EQ;s6OY>GF?_86$@06x8nv(DCm?v4gZ zkL1vkPv;R?3{x(Yhv8u)3zuC;TC-2hxP26dp|uiJJ9~ER-yiW^_w+t#X_(xC4ohAg z-Qphf2A;(5@1cDuWap!B?4*=5Idv`K_4i)t9vTt-AfP`LoNTQhWwA zTQ!_!`%T?F-*)S0-h#?E4#wa!Nvd>7Zxozz=aR8%wR<;k$^SfpyXE8VuN8{3#R zZTikwa{Y#Qd4vT^Va_1SZGV4f5euJEG4f#Ls7_pFW^1%ZvDMV*-EH}~@st=8u1lYH z?7CS0xw29(-ez_-eKa_@w)bo|=aoi|^6vvXG$$lfvtRV}e{MznyM4RtODD>kw1{KB z_TGj0!Sm#V6$_hLFuYV&ew&^atmg?!yUcHidXs}(ZTi%=KGt1gf}9mKoukt_>Jgp3 z76t44U1%8dJ~|`bf-%cMf{|voRjlO0o!v-^P18Q}Fn_c9AJ5UQJz+N-95&(T71Nq_ zb?v*AmX&3`QW=6ChLKU;$-{g0$i`_%&J_nIpKl);sW1t9tvHPCy#KqF)9lsN{4y^t zbWpmVHLsPf(M^u*xh$H$AdM&*S$V^l&X&M}0vXx)8*4dt^(0t1U%Ir(>6WBv=d*;-J$wN0S== z?Zm07#}d(BD_Q8`#tVl^^9m)+WdGc7R^t|)b7tL^X)S2Z@rs8g%ZX))m!nb7enHEL z0iLMWY3+Bc<|6jL_t|kGqFk~gW>8r`og&A~`6h1P&|q(tskK_=Ey3#N{aM7iK7yOa~ht;U)3r86>rL({Bi0`PyV{(=Zth_mfzrdO)aT# zh11{dS8{MItTonruCKYbMlC4f@m#>$aVyVc-;2H-ll)!&O3}^kbCsjFbf%1#YV5T-|+L8aoBjR(VX3gvuY3T zpsz|^4dbbnI+?7S?IW}66GBoSO4j60xvOt4k8=Bv^+tvzm|lWsv|-U(aw|HsZMbPo5gO^x!L71lJE98iX$3CM_v z$div%Reb3tvu$pyiXfC4uKaxTJb}IaN#IN4iii8Wm3t? z)1Bh!fa%b? zP7E9^!u%Q(^f+%7XzX{N!knij3%p{{d*{_voj6_KyM`cRJ~&NaxKCT%4OSW~M3p5gB;^u4}$f)T})1MZ(mC?@9d;o>JUq4FJOi!gy#J)WGeB7Mz z$+WWh;EE1|#xy;^zg3pk(~d`^`0KnZd~?+bN|CJ_HwBx0Y6m8i+05foFu#lW z(j}*;in{u_X)|Mx@AI9q7~-DXXXEqLPv(&p=IKf+}ni z@0UFCzxXM^l6UWDK_-=iiW-x25dz9u?mh!$5&;xQ1O-qMAv8n6!co?_8y3&>$D%Hl zfKg{A`hB6iYy)3^dA{%926mFy1_=a7-G%y7w`cl#wE+To-})=4p?wnL;^Iou&R9V< zy5Mp$f26vx2+v>Ce#k%_R$nLQS^wdK@2eHJ2miwSnlwyDLvn{LZLdjfE=>1BGlP^A zM6DJgD&X8GA@(7Y08~`nKeojLlGH~-$AEmN0JcRCfNt*`%+0DBIWF4~cpNYF zT6)P%&ve?E@2cLW%(XLcipbJ?D8`&J1Zq&!LJ^W(nA&{Z`rzlzTVu~TqAC2Yvy-)S zwJ)?Y2V`&GCQGoC{b^HZIwf9ej3;) zihh4!1W6#;f6@zw^~9jYrEhwAb)n)P;wv7saGp7{610+1sOAbuZo%E8^ zxF!F>+=63L*>t8wSKD-#266{3VA7Y!Y5w+28cQ^HG4tm#VF;71m)w@HI?vJK-FXXL zthyPk=;*3%jgKDqRpe2Ad9<%jJ?49$c2j=+>1roo;kS$fH7dXdP_{Q;HGEf&3ecqU zGH%>k`D1`aK)^1!8xPeww6aI|R05l^E;>!{zFD(ePw%sad>L99$7^L7=8e+z-R*%E z2CaqburGX_QL4K6x&Q%>pD{0*Cwk+_JKF&`PXX`MGnKdSYQ?wDED>*jxHE zE=-gk#Uj2M>xZ@XUf{?sJ8(T=iH~ObaP@6tdrMg*<40dMj##c(&~oOHvI?JEZsx{3 zc`U9y3p{o$k5IY{Fb!+3PqC7#QAcxdCkP;gb>Bk+Lcdr z=hxlPJ1b$KcXHcyCi9w+)0{Q(FRwS(WjLK;qV1DO3cK+!nJ;9icGqMx!Ewkpv-Td;R2%^verV-bIfpAJT;d8xmcZE+s_B&6CCT)yyC%5%_L zXnRVzfJju4kHx^_H2HZ)O6-d)P7`OpGK(X}@YiYM;LVzuvL$8pUj%0jt$c5miqVgZ z9$YJSfeR&99LgAHY)m4}J)7fXkF9P<@EH$sBXoo~viXSXAChs>y1jfAOY5Ji(@`JA z<0gKb?H9zmGCy;*_~9-q?G$Y3t9~kl|4y#H4%yAJD`~1)>+efv6!>wcvldYK zn~e4!2iA4|O^IQlOFwo1-F2p%-jZ%uOxHR3{_dlp;ZzButIQ%|VgjzN5lFPOtNniH zOm|Mofuptq>V;opa;(N*behWp3d5)%>M$m|y4Su1V1QeUl3iP{<`=q?sJZ!?j$M;DsK0e+&cY;)L)pGd!jGmdjl0iXNmgD5?OI;3zn1?5v5+A%+mT7@W^aF4jTeiZz3tu9Us?ZbX#|RQmB078EAF* z5$QzOmMu*(OGt>4i#v5_bZqRxGwbWu|2QQVRaM;$zV^LSW0_zw*{zB70^*LjYkW%~ z#2YLGs=wHUcyq|@YoM==xAQ`G%)@g5MwjyZTZe;Gc?9zNTcQcS_v+G7%XIoEsnKQD zoqX`cjCgQzaWNg3)UhK>oulKa)0#(uB6U1@C9e;+Dpb%;VFM-0!XWkK9y1okwfo=j zmIaL8j)+)u*?bew@cN;{ ze`TPserCdENl3-JVz1JM0y+9;&up-PZpT!)90Tu_d5ylG5$@qU7hlbW=nv2dYdX@@ zd^ziEe&&pXL5a7F{nRV>u(B>|QQIUg75@?U#KcWFcSc5(z3*&?7qT&F{*@Ha5N3e^ zSI}%gOWlKmmw&A2wE{O0uZ-g`$Qkfw~rge2Ag-4ObyS5U8F6n zt6P04ZVITkU}WScL5{;gcTf17&m3Xn4r9IA&+zc!W^fwxsj2?!x`cHSsT`RViHj?67#>H&L9#6riitbR&g z$P;j~q(2lGM@us^W>w_JEz{a=OqZ(VSyCbfDT~&$i2fI>DfT*XK9Sk;Ykc6UxTB*U zDJi?MbXdSeqkn+mm6vXSe_zlNpnA7vFTMK6u`Da;*3fYKcGYM8_4E@Hmw-2#r!GiH z82PQsVXdn2_$2P=xaU)7)jDuEhQ`LXi;A=(rS7}-*7Gt>f38?%IQHk3x6e)XpKlxg zypAG*9@KnRny+~|yDn8uxJ5_H?BiR%D_L<@mZg)^@-n|huY32TFKJH>ya^S3!nkBf zpP^Bal7j<1%Q`g}7o=Ru+P^*6#%ML(qc}9J6V)?b#@n{<`qN><4!lc4%rabBoe%{rvJ;>Z3zcM_xOY&3nv*qumM(9}ThCx;~5I?lCX1cPd!sL;A9hg7@F2 z4hqz--`(+~%QWSZ+TKTKRIJ7d`ucxYRj|waKV+SCRMhMG_6Mb;Q6!`#6#)_H1_KdH zxL%gYjJS8m7XLTWEvz&EKsZ|FDn7B}PW0?WxG zh8+pLaY{Twt>>Y-Q?V+zvE8rzO^tjDU?g$-*p0hI2;I=4u~RhC)=;CeXBAdu;f!)} zu)LHMYn31*^@*rVc0c5t1$unxX`O=ZgIXdsjcBs{!eO_AaHwVsFnYY{UkfSec<@f; zCTFP@aG@qq{x!^Wx@|;}se(df7>ev4PpjnqF}mf6M=;n%kVxelcPVT!c;z2aR-Nv9 zn?+NigDi(;1UsWHkz|j27;K13c?AdL3%#3ZC`_*xaCcZK!4N}8`(&+Q*8WMHu3r$< zt*6hYMwqLa^5|AoKQ!n$D+ayJvmTE99b0VcVRMy&;M>?wy4qIv#!uDUuICuqcJBNL zH=4FtFjh0#`*`Wc&!-Huot+19!o9S}hNBr?(!FOdj8&kvAcf_eEPH<*|6Qv*>|zu8 zyP#fBRCJA#tQ89+b_jw`CINv3G9~@^kpKni5IWv^Pbe;`8#?nGaIdwsWq;qA^%yhO zDh+~NqGDWON@X1uN=WEo&6X_Lt5}F$is3fTVX`sgiGq| z4}-q7b)~0zymI&MvB9Tbe?zU=kVZ9MJi>bN-zTa?O%uaxw#BzABd&$@uE7k9cpK5VwWQO=RVCV88aN5$pQ(E0O;^3;*)v+(fO`mm)e+ zQp?>XgPamU?d=AMfaJQux9d3jC=11-jv&d_&f-E*p+hvH6?R zri}hUM0UgGl?II?2ae`q1H)6b4_sIC9_Bj788lA(o^Y5%7J6Ug-6Aehsl8%*_(&!N zbBV~Sp8Vq6Tisf^cac3K?3G6MK^waK>A6l66NuOgd<1N$m}SX<7wwv|b{xNyZKS~C1m7%yAqBq3So zz+|)w0AR*!`RyqZ~J_@!rXYin(mt z?V4TbzgO#Ga}jzXVfzfDiu6zhD3q&s3*S2jsax_0GY4_H`2!NiqA0OMf( zZXMWpLjkGV)KoLV(=86i_p9P!5ista#YAy1rrGG3`WFQKTTcBbZtd(OE~wci07|1D zum()3pu2~HrvYuxOc^p5GjCrB4~Bg&obD%)^K^N^S6)Z}CkE_~#u<1r_88Xc0G$HxZ>AqdBaigyOdALw(148W8(j(hjaJg*2E#?4-OeaB)H-wOcUJ+>fpAs0qM|flkd>L^thAAVr zl!H3QQDR|X#ft!DFrY%%K5G8oVBp^;iE?pq0Zct1qD});3n+x2s;Xx4ZGu7TH%RE! zVR!+eIO^MGWG`Pj(uTZ4zAPhsbY$dj0>`Zv_{fw7G!doZpz%e}Lx-5n2oT172gL}! zz@QEcVIDnNnHPbItG5fJ!{D`OfIl916%2JBLdS;Ok*g3|)G$jewNK2Pqy$(E;CX5E zLk!|gtFhZGgI`t&0D$@WH4fx)pt6){irK)-i%^dHK0P$B%5&?ml9fM#mozO2OX#79x8FF(q|@VL&7g8fHLXg#I|T3zce7 zOX<}+mn2g_6X*_fgExOSP*6k9t7Kn-0z6`R8VX4$kYPbbopmoMBt4ylOGj6iBwi7h zA>y;~1=u!?0;C+$N2d(yfXM+EOeg8EzlA3JE6ite>ef&|{jo7>%FBg}T~YWOcJAp~ z@i?MOn)!VjFRug6%DzvZJ7wA7!O>#&8oC+#ao=u>Acc_#kqf-*kp z61%GrW@8a{K)tNf_%}4{2~4|9VD_*<{bDj((8~RX=YqY6D>_;-DgPLl?Z{tv_wL|F zcYzD%Ws-oF_Y~h6-9Q{@RVi|Rq68Q#?lxql|f$|!bhne zWktU2W~-cR?G2~v+Z4Ec%Y(H&asaDqTIBy-hrPYLhyoq}+n*DRho*nkd1+;3$<=Cb z%6j8o$j;JaN9Agd7IYd5sz7rHLzbK7P^{PdG2Fo_kvbDU_xX{8VINd~-wyYpW>(rr zh_)N>(kk5esi3xptV{Zpl??T=HKL6vFY?UH1b;4@aZ13sZt^cP%S{0K)xn}cYue3} z?Bb<|um&a1_^c+){#+m3<`5jAN5b)W;XK3nZw>xGh22YvSiIoifw(U@E$x<`aJr|? zBe(fx4Q8YpiIaVH&90Z(Jl5|ksOh`Ex|zoR_U(23t{~J+`wY{dTnx;#UUe_dVWv6Y zJJ)qOCvacSZuoeY11Tmm>kJxMYqnmw`_C_?M0^{!(K8IUI>K;3nY`ipyLn2H0dBF3 zPkAr;BqdklQ-9gi3ZPOvKb{ExX`;0WU=gi~d{Cf0N`+{~%gWZ~A@G8@r8gZyZ+=n0 zj+66RRwGxzhvw!GLQQSFo{A*YO#kDc*-anDzL&o1RiYOtZwNPF3sT}&z~HZCW^dSk zFyBt{=q0w~(VF1tPRcmS7rFo-F1f&7l-R21(dg(PBDh2P8JlTG#ScsVUgpt1zE~5< zOL*GxvQBq2gkyMQ*x#K;@#Kfyov~d%Qnfa`7VdKy>$Nr(YUV;k+`cs@ZZ|3RwHPfXT?`*(%y!8blW`W*b-B*&ytCaktdA5od|*Q%k^QaA<@?i^=fstYevK)U zO=2e|jbEcqVpg-S*zVSs$FDb`9d0f%Hj>?HPl>kq72*|{sQ#9IQ%@G88Wg6g7T5DN zqFv5iCJTZS( zf4iZ4&!5$WS9A1Xp>(}IY^?-lxevGZU}vsbIz8~;z-rA<+NPAqzSASWe|}XkTCYI2 zt}v*!tKFrfVypGZkIRK6{LTMtwltLgtaSdqqOU#85Wz2X_dM#wMG=ydMfRyFp-ZpW zhp$f_R(>0tX28h`BTfAX(oKx|Cgo~mtn%9$Rx}DHt@wOgcid@>avI_-`?C%fHp7#1OdIvWMO@~?+t%b8kw*hM9r}Mb#`s}0&FALX0jp^nP55tqg zH-#m)4bVs3K7o+SU*BnU5|wq&CmAKgbJd5vxXla75_s#C%hHfy&S7yOXVN9B?PpdCZ0QY* z8c&m5*4;@^wrB;)mhj1L#d>=*&YwSy8&pw=?)FJXW0oi2E;G_MNUQ9WPSmNsSOh|p zfBsQGbTUA15Eh9y`i7zkrC)swPoQjkQ;_f7$v>qjDM^po5B`k2YMqy6x2%bID9>qS zaN&gF@*sRhcx8Ll#i7w3DYMER^Xf-FakGur;1hQn>5UIuCUh~hymcdlj9!lPC@_>5 zQBVwgDRndZY813B2~oEfLCm(a>r$esHc^+|TF+;+KesYS2P?M9(~+uD^lHlIWxBML z)q?x;@DB^rrFC_*am&@tN{w-RlcDPT>r?KQOvpdJWhv(s9=31wKbA5lD!UPVE{HTy z_J*pr%h~ZF8O+TZ2gX?7sq&e2@v)w`op?bFIFujOyfWCJIlR7L@`h9rzxf|F31$k5 z)1;jzE6}~}H zEU%O&4YFmZ=4aQf{O#teX2D^QrdH4;;d@`spW;=bfPJLtChfhMRq^Tx0HV;G<*b|Q5q zf_;gD;TuwKF2Ww7rT$Dpc@@JO%e6HZ@);>tC6=R1qtE`dlEuXrwV(cqnpLkBq_n9a zhpXJLq=W$MCdMzvUi^u@C_+baFMPLYwlU0V>wk>|F0H4;e+5*s5(xG?0?B$xfu_o8{gMV z|8>BBr30xdCWaeR&w|Vs=Oq9fWVDwBfX>&oUW;Bne>!qDxwYw&e38k(Aj+Y8uopdMY+I0=U* z3R)t6Bn}E@9)~*@VV?Q%Tx&SJkrF6_5U48vWlL_{O}GTYDnM6~d+e{tc-7u&Q3L~n zpv_b$Y*7357?|%FT3d(12G0m$iGhz`lsR4LfCni!_{8gf)OET=LGgr)dm#+(OP)&0 zK{XimK-@JmJ8RH1MlM_R3RG4Q0;6hm=+G?@G&3_3UO=||4=m~+te`0aBD9#J6&e;T zBjomSfmh`h7DfzAxHxkW5fLc-`9NSCk%@7&ucQO~ljG)neDhMk-8Hqeth^Tz5sB~8 z`uFutpRq^)#1WaEm%u*{7GcS?jeeQ;8e~3dPgOHL3n38k($=Uci@tTf7iu9{g1*RZ z{sj?Z{wdS)gH+YZn%}Q!Y;0&JYAWT8zFDG>hUs9ZyK#IR7xhGqquDgYoPkbL@9tdz zSWk1ZPga++gfk@JfEcf|2HHhoPglxU{yK40va!#P5Yo|v?TVl?AYewqAm1r zx!Tuv{mF9{8JOm~xm}&8OA4~75BpwdPz~=fhEn~(u@5jQ#UstUxeT*YIL>zPiIhpr zP6P7tu1&cbF-e|z2DBW=1y+EF%~-wS=9f{gfwnemT0uofO!4a|aa!GwyZV&$(Gd_x z?=VT!zIx#E$AEgMAO$9^>+~trPQ?u_(_sxe-FOxs&bHgVZd%4LMtMW;S95xZ!^&1v zZXhE=Tz9~J(WrIfzMPy9edBlS-_vInQzVC++E?=}H+!%^l~Goy4HgwRoHk&t z2@jU~I3-VB@#R{mm33F^RBeAEaop+BQyiqUiAshM8N5t+0#7hovkbys?hKc(1)0+* z#9Y`dI*{76e7de91CRJ3Y)7bi(u0y(j=?f_Rs7g(WK3|V{L7b^3!Kv*wzu7p%^NBV z>-lhABDSCJ@SJ_1;~!fBcuC`S-(QooB2eayZi8Kff3Ycl;ftTwU_BHn!X6la57?*l zaZNPIW%4v}L3js&@Y@b6(&xaKPBm|3cMq^Vz5Wnd{7LK0}_=|jr zY1xhTU!v{MpyD#`*%=zl86Bw-NPcjJ=52sj#9KQ@W)}M|eb~(eQHZ)+{J5xV^=PZ{ zrsr{yyYFIV-H9NXImmZA>_k6DxA(c!ARvZ`Q-I%B0a z7W_f~=S9Q0JB%pbil6($bQ^EpfeLE*!=H=qU#Q3=rt3`wDo;?CBsQY;m(N-ZM!qM{ z2^Xn`$ENjrSEquNZmoQ1)bx}=9|OFxLA@jFK5BNO{ZlpMnkNzQ=d|o(e#vDJ z!myg@low5mM5Erd2`ulD;+(NvtpVd7|yE zid|ldXZFYNj9V$4R*F5QugWETWJ3u0+sVBYD@h6@AE_Tss;ip`pC)Q08uqHC&(i0v zi*-qvGi z6jEpzYp)f~I88iW{`mE$PlxT7U}Oxz9&T_{46Rlz?yon4H)fW5iV>8CO#ADJT^anh zuG?+DsOnl>CK$`fyLTgIwuRW3tnKW?CUoeCXYechMeOh2Co~U+9f=94a1GYSKq}EA z^=-5{B*N~*odoO-Oz(a`TO2VkV7~@y6==*pYL)MS&QJ{3rv98Scn#vawR!N8lgA>M zvtfcap`5Ry(#{vaTOWGIt#Vt_ZnEl)=4K)|WC1z<93{PIH2`JP!p=k!>pCLM43J&tRdT4LgR=z%!N9^MaMOvnZb@TS zm%6SBmIX)e!|94{*I1hLApBqb;}99k@k;%M91QTl+psmL_HJq11vud1Fjjo?&SL7T zQX<3GaE-tzwPZ^B%5yd~d?=(B%gkKCI6+GKNbQlxf;c{^+Jz$v?O_OtEcj6JE!))5 zgmp{OV)65LVnTD&@aBEaXd5%iU%rvk)O6!I{U-(|XX5DI-OJMKQRVIK|IHi36TxVS zq^gXJS=i*<=I3t&!BCOsfxWPp*lH~apoBpZv%5M>is)}(l=2&HmU$%c4E{eKgc>1# z1papI{L}SbxUrCUOArkse9>cYb-N# zL6FYLJ(yR(g{&EKU}OTQ4_JY^r{IEC=mta|0Eh3jbn02L$M#F=oBAkZloH+=;k1dT zgj5gvV5a{l$mHQ%w(pXG=%K#D&QRalx+@{`3cl~Z!>hl~HP{Tq=rtlW`#2!?Mb`2HY+-h!uWK@^z;t4!GTo`=frN(}q2QI@EmNbr2A za|r8x)0r}%r7Krs!oY z-3u=#nX^HKrmkK%uh2hY7Z5p=ouA(V(rl+LWMCB<8tR-&{#Ojm9AK6ya1nmNwb0=Z zjkzp_FW~*6p}8#V$ocxgP)uq<((qnzhxJ`OoHzlR#%V|j!%bO2$maC7|^uk_9Qc(`gZA@Mb zJAhVCy?g^JiUb->7;4$#?4H2+BWD)n2DL%@{hgq>S>~CEP4ezF?#twe&rnD8F_G>&K(cU=I zH`?LHB_8whIo-ikP`Kh#+!*9B14kh&aJ7n!Pe9c8;%kfa5G+k!PKn1~&Q*?zBvTtY zJ!xeuy0T+6@h9?vb|>22aSQ`9vfsV;4HVr$7|o4)?ZoLG zSY&R&I!zIvsWV`k%vPi6?ZxSEX`A)L^PBOgs5VRYWL+b*sh1#A%dbIM zu9j^^(JyaI$}Tv-+U};%W*q8#Ljw=ueB@r>;{4K(uhWLw+pH6l|9r9(tJlIW^Nvv} z|NC-(gr?po`zfa0Y{8-2dI&t~G2I~`b2VzCU}Vo*FCTsX#LN#=(L;uH;Oz1~Qqou; zkEYhry_ z%)mo(4^_8Z(&Vw)=IPqHt0b(&eZ`Vj2#f$XiOJOwD{8ic)ZCLj&XMs%fyBi@wdmF!SmuBOhgdVH?4S0_xel%# zR$Ovpg)(ku&1n0r)q?qkd2>?NrC%@ncSxi${(MI7_eP3X(wgl_=@n#jlHNby)fze$u+(xw|GjCK4guw6JM#^TmCd{!7i}xN zG+e-_>vQ5XQJF5D3|@sYR~_X0JeE=^(*TLI(WdVDYM*I}JG^_h_Ec~`NIJCGu8Y!t zs)+Nlp#$ULwdW+vV%5Jz;~TL)Bz4X%m|S3Z?51=*mJ4njIYd@;_1$dO<-pye&7zLQ z{SmlOVQQ4S$eRELtgqoIW63J#)&zgT+!jLN(zu6a^U85Qnm)!ufp{QS$D_X4SQBwj zE;v}_QM`FGAd>X&jS@VI0F^-}^CdTTAb|sj3lJ%W9b`0O>I19-i1orl46*=+(FgkZ z9|*b(2k3zjIBBRw9XYF9HuIfkju(QP4IxMm$vHw296(?6dr(ael+r#ebzVodpkhI6 zDm6891h3ueKMWzUy{_VDe(Or@aB$XM$#@AXmZS)d#KafR{qN|43a+fcP0mo3i(7K9I=0H&vP#XUw~^4@OO?L=gQGg-2p0-mI_C!7esY+$gIF8#TRtZcho z#=n(RQ%}h~{IWu_R6Qv)Q{~tzG`;BiHUicYpR>w(FYXwDpfZJi*=H-7jm%cp%Bm!*B7P3bx*w9iO?1)LRH7*M7YKk<_uM z$^jSz5mS0_u==@oOWYKfPjKSn3qF3FQ~LF*CUDv(1;xln%kHBThD}J7{8BLP#e)aj zJbGOweIZnsfY{iuq2u-D;bD@=YA*lUdw0$AcPbrdKn{pIdphG!R{P3Us1-UxMsMnn ziV8Hfgy!fEEfqGvj6(*(`!KS}DvkN2-ceYpyea(pY%_TB%D-@h31`BWor?}o8!`(! zm@ph-T+I2-k|o-h^4eqe0;8g=>C0nz)o_2-fsxkOcqL3YyEX0FbI+cf*eK29m^Pjp zxy}A~&6+!bK~nHJutTBJdd}(jNGN=EqDHK?u-15&qSvVCv6|-xZhwI#_nMmAA`grM zShMV4INTvQ`VQ%iyTLm)!M6kkgjEWPrJ05o!Ogc&C=f@1JE||s6Kt1?ajBj(d!Pl* z%Q3qY9Zf9kq~NyUIUj}v+j@cGfPYptImE>uehz1c24$Jz+tFNeE9doRB;IfvGK&uW zgjN}c@V(!r)8r%G`iY=u%J#uCN)HzWRM3UG7p=F>99{Qo zKOg9ZSXDlwal4-AG6pNGK8@6zTBxE{FwhjaP8dD?J2l>W#$M}`_;uF>vprF~&Rx96 zK@MIc+Gx>V&v5>+StxT)Y9YI+EW%l?o&7CLhg+%x@v^*SKk||byJMt##@x1TdHo8R z!;L)jc6oD_lI52{ZEw?yf(7UkCH=P6Z;S3Xm%o<~yhp3%p!tghWu&4#qLz%ZIvIo* zbv;85E%>6W+v^IH`UKFW)oOaCuDgye*6 z+2L4C<@$X>yHYCn&YX$;%Hl$&I9w+D%PKjCD5 zN^XCCMH&3))Cv3ev7U+7hFIa@^ta9W$8e_6L`eY`qoB^cEADPK2Hx6?mzI_V4zHoX=gJ#fQ2ejAD^bK zz*a5&b9EP(6n2p5PC~Q$x^-TLlj5V#*(n0LL2M#OD(c@v31|fotAp3k{_Bts!V2w^ zBefWa?*fcJI3}AxwTVQ@&*v1>eTN&%Ze`#aFuGabB{D2J?WTEYWi^Of0fEQg=UZE> z^w@?v2h@`)@?o^)B5U9o2hUA9e@5$n3>SJxYJv1qX%HAcI4W_CBA~~C!NX% zr|kN^c-vx$KC24hw`BZ_20K2_b^uO6w307Z*^mj*#?{N959_ z%hhf$>J<&U0K(S{H&01MBntN*wuk!WS(0TdAWj8_1}>yD(WE&CP=gTvIshh`6HIJu zB53$cdP;+_Ak_}E#=3rqiL{ng>Ni@zxxwjroB7^Ay^l{QD7OXjz=>S#vKa{es%Lr0 zz-tMIUmTUCCSce>RuQJm0yl3oc!vR7mJ?=-s1izz#F(oH@bz0g6&^j>i2U7G^-t#F z@<0DvbRT~BI-NyKSLNaSHo0stI|M|?KeQC_HhB&i$B{o@tUvkB;M$u%+Iu1){{H-R zA!5pdCZ}lBHfVwZNik*)pR6Z-7)wflR5dAfeh1#&PiSm0T}j7+gL3RqEbRAaMjU{W zkQAFFu7RoH7ID9Q`O~Mido@&gYdDuKotzUdH$6Zm6b~L0%*;mw8Eokb#U&Q&nQdBNxc;#fflMi*%;DjOMfPZsALs;HJCL^Ls(gvz<4KTD7JN8 zfB&Ua*ny@Tj~e7neB`qj(ABOT^ZdM=n`h583IYOh;IFA!@S^+Cy`agvL&?b$>OQX zz7C0WGQ?3N1M_t^%`(1Tl2f{SUO~C5cFb{gvCohJ^`l1XheCJaVu3@gqkvf?**TeZ z%;Rj#FbaZyY__UOUawSMG)u8EmdC*n0C0#&U)7Xkw0PF~R8fuv zTq$B={C^I|ZY^V`szcA?{Zm?YS)9wUwo4^CYUc_}EMXAjKZ7=IOfWV4-WW`t42J(J zk5ZEH2ni+mH@9Y$>BQsuN-9)y3e-q+(`}BTv$DSGLVHAu&2j6W+G(ZawVZx?&^Ir= zFR0xVTIeLy_c9f8HuLej#_{=zT0UpNpsr_8>`+Z$-yrb)-4eRVbpSsj4N*pAHq+rs ziGs{*cX#Z7IEMJ{0ijTiSmwbTcOG^1>c~;|KvvcIB50&@Eh93~C9ip`hLYmtSq(n1 zT@=AS#{)lS5IDuDS2v4G1)AYwR$*sEPDdY&+49(Dg&|f!<4SHwS4DxNDlAb-KU}Di z%u&|Oft7$_{Ks);yphGXfBsw?7+7MgsbOPa7|L~2RV|JA@ZZ@g-wwD&gn0lF8JA&4 zb!ErZ^@~(Nt=Ewt+$JU^u~Z4E7O)YWtd_$)!d65P&=v!%njjX-DQd+~$p&*32q z!#<+FCy<3SBXd7c6e2MT@Wf{!#IT9C{`l3&5eC^)!gV1FUcL~2l2B?sS?-w?n9!LO zo|DCQdz1A(CJ5#TRmFMa>{^MU2mhYxf}_Kei=)_Fv#_V(@D z@ElI$>Io*f!OhJF5;K_Zu0q&UUmORdKqU%VeFrGp6V;(@>Vfr7=8=7wGKw&NED3{* zEXb}q@8h#)7Y?D-$aMk3!ak3hU-8cXBn%FN!pz^U;^Jq=J2wRc2ES66b3;p2v)lJ@ zYtK01F(=Fkp+AO8l{O2IMWBKE=(0JLYYzHA1i{XF@lDJpd5>8jfdb01qhq9~AA!WZ zdX-OQ8A{eC3Djmg`0`SZ3ZOF?K$Lq6a4xQ6q_#Z6Mg@x+Jz9&l!Epw_>ST2VLT-*l;YqaE{@2zzP>v~l3s&gk^ui2 zw1N$ai8mXd9pytIO5@&MA_0Mm;^6a161!C7wj=lB1pzQHshOF@>|g~+EnK)aZrz_5 zl1u2%Gu(u}(7j;cO$W8BQjwuX*5+EFp^lg!qmbmws1SN>3i;584DsMPr zO4yrAUSo$Gj?{zGHKnDgjYF%8rk|fn4o<(~>yA%YVIi|6C!6U}+{oNBNO^2;e+Qqy z;?!n)r+Bzpx7J-`<+8ABsqX97rI8BgnVm|_9WmiJ8K#+{%xkF!O@@YP$DI#WGOp}U z7%SgQk#Jvj5%0^mnMY?IZSI;pF@%CTI| z=|Ph2>io8_?SARtHQtEs*Tn_&+flTP;*_S!E2TU}#m-;91q`0I-Cv|e`9eq2h2dp> zY0ZS&x$WPcGM={elnpdXOe^x&DeQ3+xpB;%R0iEF9q=S_BDHUtIq<7+xJiTRVfshE zF~^k8)6Ql$WsdMZujLO``(GTfv-_v4;x&;!-?v_${x~1ro5#GCHvXFcrJYDO{+3^0 zlmGvuQD~Ge%pI4=vl(V;S#tP9w(A4ow&pa=D;|1y12bqO3Y z{gz$_2Un3*7BbNWRW~AKRtsT%_oI|29d0Qui_Nlx!JMOc7(`>Cz*MeYID-uUz?(+* zND=kWY@@`?viW4&n2S=?6zLax=b=0=lQs%H7%t;Ld;cQyWKpX8_wkNj=Z#Nd{QPKv zeDz#CriegXv_Mg1Wz0R5n=b9lo>%_W~e{ZBGuJl0@6w z-QAIRY>WQq&)PJWPfv6l^W{Lx0|ZHk{Ss0O7}LCPBcamAZTHK$qkT5SBLE6)*stFQ z>Greb{?fxeW;&!_1IS$*``moF*w%q3uz9`@4qk?<39n}) z6T~;*xD5to5E9Iqnu=m$V{<3vQCC$B1QQ<2=Wg{YDk*^$N8=3USo(&2BMkZ`CV`Mq zG!OS+$1PZM$;-<>p&bc^dD`&RXazWE?!qYT{`@iU@xg@t)aoXstRqlYU;i(soA0j> z2N*^OG!vSfsqL9Dz-@P4t3O@`vC1oJ+m_f-* z4I4MOTVHACOHN6{pbVh;Z;5TgxP^*#mC2xw0`5~(hD#->#5P$pSSdPzM zmEv|4XPB6rd>0+vtk~Yw1;qGza?3%t+g9T?WBXtX?0on0g|O$Jn>p*6dKWHSfZ=Ws zmgU`lU-iy@x7`<3=Z|7zuKO=lR+iQ_J~F>n&C;}uHiads26ttmn(kB?%10^I#jdy3 zD7(1Wtv%jx>yI59kFIA&Mi7Y;ctxu4bcLJ`l;Gn4B9!oNCFVi#<{XbQK<$Vst;s%m zCH?JnRMe~}u9T;G=wJ>n#94{g#^x0ad(IbV>In%=p5t4EjV3Pygb=Zxz;|c@`Jf59 zsCeFBfRi`$3i>rvx?2+tLEw(L>*gl0&?ONF`j|Al|!LB7%+9FyLU`btukE;2kwV^*t`C#qBynQX~g zAEqjeUQ0#Os+Mku5~5bJo2>G}8wOOkRXKYRre~*HO|u&za_=FT&y{3yp(%{t(N= z>o)e$nOjo4$&{k))-q#2n)u^@_gY~|XRLd-5WJ6?pj1!Xbj#?udDXqf-m`n0?KaJr zk$1Ut9hzOc9N+@m!q)!58b)uKaFqADlla^30^07?-?0qo`Bhj8Lu(H5yS)?(#cxCy zPdI2#-ShL`dcAx)xu|8hKJub)jNRDW(BzX=Va*tUv4(8g-D*b#i}gLf+B;1w6PlciDQ2kIhdu8bzD zj!>W<5mQWF^Z)eF13|&5=2dg`?s=z{!e9@7+J`ah$B%O;z|vp1#m@yEpz(#TdCK9v zdwaKPV^nI-`Ub*(&jzX4Ej_UaK_k#zTvF9K^-opP%MLZDE8~Cc?Xwd*`^ho1FgJ$D zNf?&0gTvhlPZ7fTR*xL?c#+(}$NoRRr*LbBhZoF8>hfe@J_kF{$w-Be(k>w(p&$UF zZcOuH!O{Kc&5F0R;dlSMyp2}#w8+}-=-A=L@3JTEWhEsfo3$=9V$Q5fUU?&zG1pev zF~Q)aK!h^d*1zYHA#yMuuwj%aGKSNN*KDGy_t&@{6Y}#_nrjr zh|*G>&HqD{G*!H~MPof(7nY+|VBu-k+th>&3PL$EGtP>M7QuYrszFEe7cWn|5}AdZR>!3*e4WvmRY#2W`Q>X1;274!Ucz;xYw5}qZ2}h z-@VURdmax-FyK)Ekq2#&D%>AGiaRGJ=pH?K1bMgyndo1?zBM*B{u>$)p4m;lYZdk1 zrTX{6>2rs{qokzd*6-gkPEGut{nVpdf|C;;9^qbgZ?Qkv&ZlbJZVCw@0eqHI zz^i`p{JB5$c+Q7*qZi!9@UhQ9<~122F)=n~WPo8GwHPQLEHN($h=>4HC`U$~ z7JB|UfLcI#VgQ0t#}BN(XrTRiE#a~MCCp0zpbQYM^*Wq5qZ1S#2!Mr#=BMIA5Wc{@ z6{?=o)!N@5<@6X{VWXxoM;@i|eX#X2QHB^8_EpZ0lFNz)!TaN?lvKTvG33wARxTzR zwVY%7-yI&gKeWIQGdO6tP@MU#;Sh$#BGk3{V^?MhGVgs?NK9O8}2`$4=9?2KKRigM-C^G))G4(NZ^n2#t)!fA0uw zCAgQE$YuDd#)6Qf{gj(Jv+m8CPa}o1SW(F$#dwm3X12xcp|r!qdY*jeHtT(GP;g&e z9?GkZfHPsx;{=Z+qp+-h8(rIgE}L$oBP7IvX-ar%Dz~ET3aG>>b-gPi#@xqu)D1?y zj<%9#DThaMb<43^PP_w(oWa&!DgVf!=5`}#FDyn2_|0M0GdoMzQ|oaw2+kK~L7U`I zjpFcgW<-=n?YtO7#5!h`6xg)r0Bkvw^rr8;l}-Tzvjer{1@hm)5bDR0}e9 ziEo_Vk&AMQr)J`G(kVSWC@5u3;zU&`UNw^3!9K*PSXhl`L}6zhavOcLr@v<1EIFXs z2i8;j+x5!%3d)&okx$^Q=61$FQZ9|BtxXkDrLbOqtsLcX-M^*-x8$Q_s8S%u=%tpC=<%5WQ8adV#w&vnq#xApKkIG^%#`k3nqyxiza+7dgl)1`qtxCP(B8kE zpzI`dyXj=5zoQ4Uzn;gkF}%9M8Yj9(map@CQyEayU^~NgZ~$HN`}$1~slI({;gtVS zt#`^wK%9Z{hG%xig19S1>x%2cyss&wv-8N_8 z?;REoS#EvxeK1{@gvZD@E*NWRWYp$K`KAea#JnnNxGJP5FH;VI=I3Lx{Yt_IF_j3K zLpom6WH=4IT|J+{D=+A2B8c4JxRswjPEBP}OcK2;D7doD z0=+;G_zL$oA;WJ1i2(oCRVctSO!FH$hRVu@&d#v-%qN^k?lK(O3wltl0~$K5r?@w_ zq=W)6AJ7H#2-m{~nu=E!_z3gNWYnkeVo2!?8)ZZ)Pe{e{h1d}r1*rgt-VA0=u)yCg z`HjH@!BQH^+BkEV#DNR)Bl-V|&&Q7+zkknpgllYS3PSmX>26kdCi3Z-esGUu@!tNw zrT_|6YG9HG04*booaG)83kz!kAanJ^m4mndNLSxr#g6y@AZ79UyPU0&5wrq^PL#e-Y~~VlK8%B6O5hRdJz#WElt~>x+J2Y&;7_Un`!^ z&_%+}1-GHO$ESZ=gFARu8!AS|Z2ITU-rSj^v@g#!|8@@??ds|!);E1zFfslbf}hQA z{-_e2^uH)b95ZzVw4T7e;WLMoDpXU2ooHv}1Guxq!EzmPgbBftw$#_<;xcBkkbC(U z^`#)zaJjGlOSZ*?8bC*9zcby?=)07bmTzmP;Op2kMSkeG0rt}tcBw4L15{ygn$ zVb<&vm?1M{&wkl3dOqj@SAmt7zX73#;mj^WaQ!Z|`MU$o?#pTU*;or@nYO!M@1^{f z#HU`sVn_+WTO8#rsKycSshi!dh^sI`1=f)TfJM)hLdW;*t1GvIn+i*W{EG{?4Aw-CZFNS5hI{e9p=G3&aaKXf@QUl;wD7YIVaytQiZn{`Qe^}wTcA(zy?lfY4_5@Jq9qZG6|JmnVRCBrjq|;{|o&l&JnYb zS08JpuosuM>6OIO8X8_PJ!3DTUL%YTQ9#5fP>SNzpGo?aZqSRpmwBF-iyg=R*{kbF zNrHG>P0ulW$rHRf;CBjQxu39EV+(Tw2LuJNh>B{?d&1eNm?%gr zBO}upGrzeR0ZU<|y~xGV3JD2;k+BJqy#iG-Zl1?|*jqzp0OH>{K8d!PyZ8pXxw-jo z64+%5iZ8_KIRyZTLg)R;J!$$`uW5x4Kd zrqjPkVYj|+b z{O5lV?7uf!2FsSKI3y4d*1GS>g0Bcx%ZRNC^4h+jqkq0Y;%ec0lS0n~$Non+#>w!P zButT4g_w{Bg$MdML3)YhhekgtFa(US2XYK%Kl4{|xiZY%xW zdr(ut7ybGf1(PpQIth0QSo-H4{F`12sf#n9(*8stTSfU!9@%RbXWCV21IAD}2>mQX*qVxih@OaQ%kP*s&Ef_r9X4-AO{zkEC>5t4mQ-H=O!AX?fABD#W-lITH^ zVbU}v;Q<4jh3=FHh_)$8mH}MW6$Sj(LemzfqTD{INPgVMk6B=hXfr~7pDonQhTp^i<_>zCb#5fLd8gN;(7I#5h@ zuqehtYHMk@@*bxy%*~aN7CO$#vVZ-W9l!hiwG6C4v^fIydFn+L7TRUc zEB_1J)-e3Jn>ggLrhWy{)a+I{MHO18)KkWKe2oygUs`u?@DVgF8>c6Rs-moS$^)VB zxCH7(P{6gI=jR`4smtm>IDz$4hjAZt(T%OGwhax5m*_gYBR|Bb`kkJP_9rlKSx;&b zr4{aBWhZhSee;WlEUr!_V<~s#2G6daS@)hm7VeG^vG(C?(D2s5BDtbFm^b2XHdHt7 zQw`(@8*jtH+hPd?GeI1_m31taD+M4o=u zGAtHt(K7iRd17_X%`s#`o^CFcGHthtZ#AUGpHkoMuOIcq$yzh2jSXL<kS_IPNFl_ecAZ>|{i2AGHU)SPEE(9Xtox58F9`~v_*(l>sDq!zKa@EA?iaMI z(>t-g{>iahELZNm-fz%7(egf86^1f>j^nqvbGUE3j5dbiSvKv-6Am_R*wij67};=H zVS)j-J$m%U8?(y27%l(e&w9B!yM(p2&<9Bgs?1FAb1m6#j$C$np+yWbwxwQOYKbvI z6u@wRn8E`PCJx+_@({ z4Si6OLF9}y1hQ&fK7Q;ZMHOQFCHJLo9^i)ZlnS0FNVibGYaSEcAmB@y9tDGMpEDvJE;bvT z5FaDDq}ok*bl1|;jf{D~zBRyD3zBdAwDUvpB9oKD*T>6^phCAe;k>}S-VS77n4~nr z7Uu7y9xTqy&>PIo^i&$X6SrDhUq>Pw>4a^4k$eClccJv)u=rb(A zQ7Zk?i;ur-XrajH^A3zTybxzr%K7v3aALRn6oXyR#Kbt{KzTLsy>ei}6}#2zS&(w^ zT|o=>XSRcV&*IDTADy9^0bj5GNiBaZ$Au;j(9|RsZ$$kZ+{VT!VmzkZRwt)6o3=L2 zxz3ey-`)Lg@fi^Na*d(&*VwuHF$PoInaBvy+JuT@U0oE5-QD-kPU*!8>sBz?R~+k# zclJt+C24m)WQ48Qr11|tE|uKdd-gu2jP7L}OJRA`m-r9s>D*Se-#45)7AFGA^z?*} z3h|!iUw=mWt@m;zPGL8p+k+~JnW)?HfNYiB-1}(^M2r$oubw!yaKmWJ3#N?YCiYSi zed*ir-TMrxw!60`P*c#?(6~fn7^WI{$l_KQI;N<>=cv`S`}q~STT*S}v}x6zgI`Yl z_95E}%+pf=wR)}GK7JWU6^{A-bswsV>Zgz2-va-oyz!!yrm7ac#|U3aPEND@qoG>e z?kj^wG;-ElgB~kmnh*Fzq6rAKB!zHrzeXGd*6BsOy)F%F`zQe4kl!4f72a8TXT2Dh z-x6mf0B(p-`f=j=Q&Fs}EH3qFFGok_ll*b%QY+k^DarH#xEXnEU>Neb-@e?sXlHMI zHG}Q*sWK+<>sR+u4=bzd*H@gzro~pg!69IZjmNc#c3FGk99d&K%SS-)zO@@>9_3mR z3GSL@1^Ht~biD}<9I4|6JLb>y^dg|QokJ3YpzUY6Xg(zb5P+LLr=EW$CL|=d5v~1k z1#DPYYy01!?{n1MhpeQ9;-H|*H*Snr)x`1`&ptRdBEf^n&Qp*MTg$r+y!9CEVg<`G zR1YV$xw&=m6EBr^ARYszE>?rdM+Ci(_Ot;-{7QSOaxBH+|8eyeU{SqW+xQ>|s34_u ziHZu6A{_>ZN{W<7gP=4@C^afdh?JyK0xBRaodY7$pweB^9Yan1*Y>>Ud;i}z*Y9}A zbp-b8+0XN=b+0>~3^tv2c0T&}#(!JYBB*heN2^$=+~>(NFoYkuS1*7spQAWA?X0V>VCoFemL5|<6PfEIiwIs9CpR?Cz7&PAYL|^ zA0k?8I}<^#RMkVdMDZLXZ&tlu^;}#R1{8b#4#M7>BYEDz!6C<008uf}qeAEAzU&@= zj{e41TKM`;3Gfid#WJ&(IMJQ|;e1xlc6%MJAuxxGkU;wUw zG|_wrd_MQ?-8;kjxPP$q+j{Ub2XS5lvT?1f9Tb@*cw`kL$z@@69=>p#pwhXGRo$b$JB ztmu1yc)QY54@@i0&dyrGZ_BCMfz6zOogK`6Mf-}OCWar8a3IccHQ%%bOccIgfeiRlk@MWyllR~tY`onq*M0l= z@nvcK&z~zBMb5mEPe8O8io(%1&s+mPu&ZqdmW13>$8tv@$TOLlr~s}AI3Sg2%xpY0 ze#@D4aclFz^e8WULBsyH^^=HkbI-Fu$p0|v?0E}`tei^8ou6=EdO1<=pkD~0F)hjQ4W`prWz4+p8)Ef{@8+z!RPn~N7S&9U;d-6y=|%JvauhE%S!v@qI{vB!q4 zD?XG0==_S{&zZlbjXws*_{+RfSdMq|(!H9{x^z+ATkb_VW1WI&lds7BjpLuD`Up-s zP75BRxbu~$Ka^}yJIgbl#CXE%ZQ{J!5wjrDCSDx3Aap%>pOS&p7%ID^&s!KvkM-1n5BPHNJ!?umk%5!!x7 z1@f{6Dg^Qe-(kx|)GPCrbIc#v9?ftHIM%Z&7vbCSWM2DgeXHM4=2#ahU|8S7qt!cd z#CX(nHaMu+Su7l!XhCA}etUoGy=;sSl$$fR+jfs@EPY)fSUugdO4?+Ly7Fp$(zf6FgN=7xO?29q-So49OoaBiV=Ua49985z&JVl7bpW|DZs^;} z%%h>&%7<%DUM*G>fsG>;`-eHMrHkd1pqhfA>wRj9WmT%A2kKb>HX2tyWk8sGdBzog zrrNF|L4<2#$hczVPIJVmWdgpPZFGoaZ-4e5yAri5X%E&GNEn@=({+E{73n+;TLqD$ z6finjZurC_f4;B635mCeZg`!M;f`==y;lX5<$5Lmd=)^9AtWpvdp>`PF8bq(Tl84v z{hzh7xg60p199YakYOGWeFE$E4%p;?PSycW0n9>vFJcFPDn1iu7681I|5C*v-T;De zs|27PrK6)mGP#l330$aQY|1l*2(iAuJ|$Et$Y>leEIN~)@$Y7Gngrw?%GQKK0DI((6WB&iSQ)16cRc0te zos*UJ1jv>#L?hoTw!T8ftUr7dy(P(8S{^tG7nRtwYg1nDY|nuY`Mr?v6L3XfWE$3= z(hU2p(hKl9Byj^_^m-TEY7rj5>P1xC`n|B`h+I zu%N)-_eO?K)Hz(;m?_(pJ;WMNy|1*n#?Q|qr`WDwb@tPj*wdHpR^)=QFV<+SB`O>$ z?zT^H!YfDac87TvJX!xJ-u&iDT`%kW`=)72F#qHyfRHlcCQU!X6|8hIxRJKDexUQn z$%b@Ss@GHGNhFin>gOeRa9=RovhCXG|HcnV4QDQz`2`$VwtsN8?){~$o=NwCr4=Y`bt2B3PCFDg_4W9T@5Es(t-m?XC~IwqqMHvwe@+O67gw& zP|2MS@`EfM9$!%GU5CL?BLDSSnh^g!&&_3}!MOd4aTg%Adl)nAC-7iQE-7rNH9#`% zkwdJ94K?c*n(rsQXBk?zKA;Ehfw-V`HZa%J7J%1gcj@$r%oqmRI ztN+IG6tjGZJAM7Ei_gT(E}O?oT?)-)nq!jc2jNPVcT7f0&}J(Sz)!5t*FUvXPdkEq zONNgR(})MtXHMx~o6d((HtqBEKDB-d1{tMb6a7y6&hO1Nx|41&FyP&9zb*9+H$)%ixq_2T0so+})30q><+AhDT1@qpg zt1=I~hplP7w!Rlu`MnY_~x_5e2yi-o=b|nFW@6MgWLc&E4 zLMDd~<>aUvKC&X3Hg^zMwdwo*dM=irn$xESP&Lr(W< zK>@u)k#dt=m+$L`QcyP#iLE#`lawX5XezpB5oX-M5Lm-kzaRPUMwfgUI_`mT*VBOK zQUxp5a6$J;2af)-U|$a=8U3?Zu%nv#9A|nBHsX8TGr)G^mJiCz%#_Se&z=UPxTHkI z+?L?gUI+T)( zp>q~CseHxbw?_tb`ZbuWVJ$$`buH*OOf`U44z1?pcu7b0i$c9kX-pxFNaUa&0c$W) zhd$fP94PahaI9YBwVFyX-_*WL@X z6(BKCaU9@60tkrseNi!sa6Mm0aYQ=Y5Tqr5@tq}ZXC*T)&qiYx4y`nm5MFw>WY@Zu z?iN5XKtO7x0aOl29Y@)WFKg;?|2)us-Yi)#{!qFR7aRMh2x*V!pP4ZTvuw z##w&;KZXBe2K?s~6wGzQ!$2v`r{Q}xr7R2Pc#e19gPgR!zigwvO8raSRAUA&;{pq7 zO|9wBvJX!RcEY9Biuh>jk6Mj|Wg5-mIuU2^ftLTCnMkADnsHxt-Uo-FPc(n+A)+co zWXJBw`;|L>CP{u=1)%HwzB{h^&&~d+&QA7BhbR6iK2;loe!NEsRYR?>Js)pnb-i}BPp8QpZA1WIz@ zA2>SfTCxUzhAxv8AaF$grq(>J>hN>N;VGB>Z{Peo_BKW#d=z|u19<|(u2z5r&<1!` zc;#yi6SseIkegd|@3~Zcz5SNOOrADN7t=ZGIA}1u2(c&N8p|$z&x$I3kXJO-CbBkf5|D~kf*~Qy#K_bSz(WKo*x0d|;!@LUNn_V% zyPj9e{5sWTEzykcoQfSq|E7vi)Ngm8W@!12?2wjZt-ZLX&(L-_hlEo)Js|oLMqcRm z>+v)&3*=w;L@U0GYq}h?RNTLXclL}-7&y>WYkVJ5UMQG4D8CT7A5<0J^2s}F`(;Vg zY#3%|Rlj}i&$7Tx+P$jfn+#O%=sh>YqLK@|P&u@$PjMq6V$H#Kx#w)~j-n7p=U>ps zsr6tbilu+-lA-T5!xMShdvqZQ9Z#Nhlk1*1&odI(`Fkbtk&WX^zbK#h?e{5ZAje%U zPPdo2vb5WHs*CE|pC(Ktd6qKnn|Ls1Q|;U5*Jju)nzybRZIw|5Hq2M795fs zY{lflL$JIokQ8g5U;5$OH`%1sCn@FV6kkp~f_JiZ%`VmRa%nx`ZGnuuUF1f63yi~aYg)lgL{?jd^3(V^%rB5Zs$I5{~%!NL{FbUoF3u6 z-1qKcU-#c*b3tU}#)d1uIV2@`GFQRu)d#EhsdHwj6_PBVL#nGh;8u$hQEHHNDxQ@i zXQy&O3=sSd7S2KGOQn-FyCw$IYGAk3!63}2gxjQ==q0)(1Ki1QrP@IUVq&75J~xEu zUNp}1tI75n`O$3%4gtLeN_)1&p#KgpO(WscZ#yh(#7nz;`uP&8^Ll4Zt zkHVBP=Mn7FhP6I2ijeA6P1qTJI{BJK)TW(vdwGxvj4i$LucvB%1mh+;9z|tHu?K#| z>oo}T0j)C^qI3h$7DkrLz+f~QzIEi(-+LfoDu5JN#1%rE4aY_fgG1nb7e4&W7wZ3x z>5nOJ*n9Jepm|w1Kq&BHe8^x3R~9<(v%m{c(X^MyVH%cB(wKls!SdKesnr}zfM16%%W zj-1%)ZEDVj^95$H1t1*;OuytS#-sbnqK1DQ{IZ3o(7-{X09tc3VC;XERD#hc!X*Sm zp21EvXcdm-Fm8o6a8CDaBi`o2!^1aA_&Qu4KmHEz8DtsT_HDzod3m6aJS!{f##gpe zUHmYHgqweJ#cKkn>=4Zy8)=7Ke(dJ<18lhNb%O^#*qL8hd&sbPA`7kz`l)|Jo7L6T zu(N8hNOqj`$L!((*XJaEHnqYrE%5SfL0?EU< zbt~6s42%uiQ_!5AYs@d(W{SO%7#N-fg&TONCn*evm-r61M$BNq_weF<4|dPhOsZz> zvoWn3xV2qycel8CruE}sw!v;-O3Kaz8(7^I9C9_Scn2CNI$oIy^(Ap&OyYNj6~OOq z@x|OZtsM(R16>J-SG)uco#R?3j~gBrp=_ z&LW~B67Gir^!0ulz^?i3rhB+U9ee#4CkReXr=~;GywBM;Uw~rW2fsm^W9S<8{^ztS zxL>Ci3s0{(hyS}zreJ?<)1~;PDLjFW8_XSD_a8II3M)Xso94rID_h$Dh}>)Xq?OWH z7cYjR=Nb1wnSm=Ma617oKKz!oFk=vu$?fk|VQEIGKZn{)Q8VcS&}kL!8|p~;9yl~M z@r+-ASwYA%CdS*Mu2W=C4Y#Ntqjhfti%j-U!d&(6iJdytIzBnmmU}F7G#Lpf9vNZx zKc@?~ac^A#Jbh{#$4vmk^ELUS@bGp}{D?Kyw%is(W z2k}gt>yj;ZJ#NiR%+NP^56mKwAS4uh$OGUniXzD~w6qYf? zx+fG*R-#ZgB;WIU{GoARAmd%p}Kk_|y@NtcTXOu{NdfG)MqV2>0QBzvnsyDpi z=<7rMoFhbhbr3=%qQYR7 z&{{7_NqV-p(N|)Ba<2*_=jcpDk;zq?tu(Q)@G>YUkV%O+hyF6TI{G1Fb~zVp-{=`O z*ucfSm8nY!RhJ~!ds7|H9(a6a*j^xp6GOzl((I71X-8c&+Frjj`76~2znzg`UR1JD zy<0W@*CPH6m-|(oDvVYhzN@}F1IYK*jtzq<*`=l-)~w)^4+~h%B$LX zYSr-}u<8E$_cMECV97!d6xIx}Xs9Gn?$&POd_Uw7-)d(kL9=A;?cMz)VQ(sOtuDN^ zS=9VsJO7T^_-1=bN|)900ND+Fu;9OA3B>==)6z~*(t_7 zvyr^N?$5*NDZR_ZJp1~xeX?Ks;ArJy%{8r^A^S)O6eB)6brq$buNZ_r$SBIRpNrqV zgT)FBl&(mA(Igy2?={S5cIg?|dPwiWJolp}`Zy&2!fVm@@#DwqXeggN7PR#v9?qW=CR(7+}MyW>r8&z^ueZvp}>(gRKDbAQ6~| z(o(EP+j4AxQ$HPS1K#}_QKs?-)Q0~lgmuk&u#?Q$xx8~yA9f2 z_#FjITbW|OX|;`v1TM& zlDj}yamlvxhZg$aE;5J%9x+$mefX?_-ZX(BE@50?vJOf*H!B_F-v`3PQ4H9x)g&U$ z@!+8n@>D=}o>li3jbBUHuIiQ9SsMvS*{~SG-rvv4%4d>6QV4$MRn&COaw#AplOb-# zVJ4;;LlQqGdgW}*b)A;#cctk;W^|zM8`C_Im?z*riX9^a`&`8;HGwUE9^dEe(vO}phqO8ENiahT3GRQ0H=g%8HfOckn zeVoXmo84=Pi~v16m@lwlRV^&SwxWU2ww^qpdh_!i$)<}y7v}rCa8F)A;Z%&QW{0B$ zyPRP{&_Oz+@^FI6qb&ujfD%lIobbWID@nY;;>vT5Qio?pL@?FIY>H04MER^=6@bMp zFx$!Xmf^GyjLTjg959bm+zlDYWqgx#tQ~ z0i?*#O-`m*41RL~sot?*3drzeH1mz2P5(`irTfy1C{D)X69K*?WzK{H`{OH8WZ{OxTcO_o@EIkG{ z8l9oobS~x0WS`J!n2ezB{uwMf3vzODSi?X6w!HYYU~Ew`q-`|xeA|s&Gx_4H7VCQZ ze3jC{zBUoMtJa?=rJP2TCd+KKgaeKfPPD~=-JK|{gCyDhehJ_TEWmfc42aqHU6GJ9 zig>4k@5!+%VX7s1aevVw4JigMR=OL2l3jAPo$XaTtB?dci^S!75h-RNJw4)yN@j9QR$Zc85Kd%gNJ!|6PZ+w81U# zR#eLDR#$$5o$tW=Rh2QGm%Qd($;QirQOC^8&Vuz)Q|6u8Uv_%H^11e;=p^O!)8D=c zxNOfWNJy3rq4oL5h}#*;$}0N`8#2WGlqTfHlsZ-^Ent%sDh4o0z+0iTz&}Or;j$;E z_*6|yj@?${aDj^uHa@SS(LuO%D=Lc0qkN^`l;Z{5?!6V~Zq!H$>gkUlT2`eoXcegV z&NtKMv|WoOUb&++VQf{7{9#+riYIThlX{skhaB#KlxP*#50(AIXivr}9`qIhC;)Cg|en-CT*k#-7- zOmHw#(@MJXY_5|N-=LKE=x%<#v+Kir<7f1@9}I_oBjLMR42v_j_>0Y|pPv8oXLZ~h z7Fmtvu#@fY1!HJeeA+*haJjn+WoC)NnR9(_Mx@Jfw*5uW+nzf3okBwTx&~6yJ1YX# zHnHH^anwz3d@x+`_BB(}T;K8iIE-;)oY~kF9UW>$M#{P_Er^ePGc?3};$-h+Ys^s? zGz{({eyo+oOx~#D&rXh)o7L=z`q1~f;YnIOe%8m?cb?`RqWy_WajqpFHJiWe9`l+c!5#D3{nvOFC5m;^ZoM%) z?E31N*V$!^9atezrOjuTRyyuv{+9YUl^=ys@{24u&24m&vT-q@LW!=5L^ve8(q6y* zsD6E2FEy+4?MRQ#U(<{3dmL(BBJ6(t3UVVh>o?6FURSVa&L%@8j=GU&nT@}U!>~Wj z2po3|)Hnj}A-}mCKHK-qa}-RQby;+gs-C|PlchM`U3Y!%XhjrFXm1B3C0^^Su#}LJ z$B!PUKAkR6YJaY~vpt@5DxY1I7)^FAhBMWbT$Fc&W3^V`<_x2R{^j&}ufwaKV$(|Y zC+CKLxe^Tz@@q0KVS|3qBljTLEX3WJ>&Czy>QxvUilyXGNdW(d^c|G1Q;Dzd`)h9+ zG3<1&w~Bba9S-00cgl<-5J*=>UZ!?k{@oc;%`!(^VLf3TGfev8_uuLo0D%?w9bWsO*Nbc&>FeMxNe{ zNR+3yY8oRMiznnl#n;pr@&G~!EDbM8;@dE9m)@x>#HjN(_T@ei^TWq#;!HwWCqo)T z0xg@ua8Xx3XK*0Zntm>Om_>s`fuRFf27M3X^3*mqrmHZ1V}|D=BOTtkD~*zN^LbSj(?w-bIJRK4ugv7@ewcLOxbC2^|77+>>A;OOFA&>&*KVi2wWpW!TwFpML6pwYp{19 zz_o*+q7w$af92ieB9Li;XtmrI=MlJc^Ro^De!;B%d3d;1hWuI)oZFBFB6EDsj)Wnt z$xAxdzW%HRlC=mbIF=$m7zK=0J^4N2P)J&SdHA}leE;N9_V;1ua1i2~khzeYWyLq2 zT)fX5w)d3F*!VB#TYhXzAC~q#@H;mSHxVoi28frmw)j#)(0D6K;$G8RN$l6JHi0`m zS;M?>WeUXM>ix54&d|(@4!6cMxGw54C=#}b?jht|VAf;P|&va8W^li$ChfG3JX8FBxi~qaz}oLkwU~ zU*C{Du=o+3!KtVc1yxWSzXJ3wkYoSw)hplvOG8P%?WF*J!hbLw8^+_K1{@@#VH^XR zwf;PXWx8cBG0N4qN6)*-f;PA~Av`Eks>tx7w8t{5u!$XMXc+vwBIDzmyF?iEK!*$o zEu4yTmo8mmpr_}x6o3+#*fu!G29($HBIp6LLSX%a>VxkK6vvBP(YMcjNZ{FFyZl~V z3+l)J^YW4f*d;UbJe29FpFUAQ>>?6rbCFps!5vUlWJU+?4Iyo~y1K$QnUk!MALjv; zFp@sQ$jFFz$3ibqa&HdQFW>>GCF-^PP(4MFYjstN`~KR-ty^aqkHDqMUY25Ds9^=6 z88Y}P_t_T@3~zvJ1emhRC6K6tys5~@skb4T$|9ZC;uc6WPEL+nChp+C5-g6j1yX!h z)zUgArl-k6Z%n%!my(l9NWq0u)6gjU`PJ5#7~{7q6c5O=bdQ2dZy@3Q?}vX0f0ZdF zh%z(K4wVk8el3F8+cEGPk!QL2uQ)gP+I_<$S>_3E7?0T-=o&w!O%dy^&CF=*Ef+0~ zKTA-;ThJUjG_roM6u8W^YV#oGv|u*=g3^&@Yc3x3dlw~HL+PlL-n`lLo+^bk2l1q0 zUo>#+QqE&alZJY|Z^M9Rzz`EY4qRQ_?!`7pLi zvd-Uv^G5tEEEJ9&D=M0RN}bW z_0p==X(neWiQ=@>)Y*#CJ9=0L@*gJ9;4JkErquF1Z;2u=v}m{poWT7a_oYRnh&Y3- z9xUv-viSYZ;>R;C_$d}!i=rcd9fMwSMvJgK?=<~$|MX?|v&`qKBfY6w$=Nt;-#dvd)6twPdB>%SYFSq z{XG`jdgkVVpTw7HMMg8i)Z3cX9DVt%CqjGshCR`eyh;tv{|w)V9+UJaU;TUewCD^r zQfqYf8y){-5fA5eEB7J=&&C?rKzGVl-bzn?cG^5DJSA0fq9RL+$oV#r`K<~`o=kvl zZf?j=E$VPt=ZVR__i4xbBX2(6du`HP&O9K!XE2YxgSjSB+uC=7_;QSPBRz-#ys7hF)8;MM4?3AUmU2B zW2y3-^BCOnGdNJ)F{mS0yw(hw~}XJk&$$W5B>SG&vs~Moao5enIh7dtgvh5 zX`h|jFfB?)XX`L5+3(lBb8*|NsjF>%eii4p|6BDIa1Rm)@x7LC1IvC|aA|ezkoc<> zjXTI8nQ&*VNNcDQsqETf-F@2kqEdcL^E5}c@aD~8%7@9{=mZ%1rFK^+ zVr{+N^yzbSCSHI=t+hrmyO#emfVLW46B`Z9Kdo5a;^)^-LWuH~!Bb`ZtkBXe2c}|eRyqam-RoL38T6%gt zOBd891(I_nCy!px&zVo$k{(wv+b1Z70H7E2X0u_V+c#$zAarL? zYh)EJ<0Ko~%+9XI-KfJHk|T8iv{FMkOK@M36^<2nt`(uByd%3BzY26;O3Hw63Hvf9 zaRowEe-qESddtGe5D_OlR%RGO+)=NoD*m#h0{eaP%`jnq|5~3v(KV(O1I2=@mbP|8!|u)FrK-xx+YY({ zC%Dhi&68wKkkW8r~)YMn5TtV4%CI&;k%cqPCxkM(vo*M66m$fm4%w9Xl zisOS!0Vp~~6XW6g4&;(F1NE}?Pl<@of#MY2&BLNj;7qzei1b@7IcBIFme*^`RnL_j zM!~4H+4`Gsi-xlDcoJj|0@Qfz+O+~8B}w9lBO}H>+rL$jQU+-K%C~QAgT*j(Q*yl2 zVOx#J2pP(gPrBRHFoOZz@kvBb0Fa-%p$7;iB_t&+!A>C2`a@;{4TmI6aBwh8&tKKp z6dD$|FXf*md<46YpX-?8%gS z&`B`l_NuA9pPUUMT(25#a!Q){*f}t$L#o1CL+?2fq{ZkxSF2wm{h^=V*K>Is(0!o^ z{YSKt9{NsxPlcn}5s#mPFX@d^BP_qaLiy}XT7v`s;H1~1M_v*V@|@))e>#47*Qz?g zt(fjV`jgH?e&)Qq6*_88IPp)b#!fUTh50V-$Zlf$2agXz>o%B`bX3+GcZ`{)9zqOd z6U*^;0Oz@ADi0!vi%qsE9WRXt%Tc10RdUnolV=ACryu6l;u-!5s=z7v%dNG*=!;m4aXPobaI|3!z8_&FfW7m#km}S zA>p0^^0Mrb)Dq;C)u4l_?Z?6I{iaK)<|XSn-&(XYm}} z5CF@so^4%2ssz~3*RqC5#NG7BxLxL25+O6AmM8kQj7>XrJ&EYdr=t_Av}Hf9LB!7~ zL-%w%cc%sSY=ZbIucd_?qVBd_4z>kZC7O$t$UTr=v^`m&d2Fl_yTFtZVgblxo*9}e z#pm6t=H_!+cv@6Lp#6Fonj^5i{c=$F8a^sQQ&T+0Eahzoi^0@P-1W(2{NH2R?4gWm zaRC-kwr$|XJx32KpwX9wjh~(zF6~h9sa_3Exq!rsQjK{#OTg7xE&N$u+fLL>lBn1K z)C518tioC)aY$HeKuPUP-E$Y`>g}=(mo4AT@x}57412hm5HXKO6w#|%V{?MW)~xSD zQ6eOVi(~u6(?^Q7{_rF#$K8C+wx+D$T9u@+GV`*ZBu$=AaHYI)7W!L2gKXKNxjR|u zXJ6mSKD{xWKWDYbko99Djq?gI~C;PI!x=Za!e73G1FLQR=Mw zO8Rg-rm09vlUd?BOX>!B@}AL*rjD}u(I0Xt9rfSQ|G*!uRu8F_NpErUa>|AbN~NtX z5B?Q%t!m87gSW{Ftl}!TKuiB=x2Nbfd1cLG`mOqK@KRx-khlq3g^XW>W5@xDG5ss1z9;=Drq`mN4Aetpf*R|XV zPzec`YK^&kQMx>cmHm%O}i z^dO3_xfFER$jFFCS%&s`M4AgfUg@WrNDYOSRu_wxhKL=wb2fLHhVT&1Q?6=BhjCwS z1rAqd6QFyk+VcBV#kW7#Ep+EktPzEe3+Uled?xB%z~i|q`k?hh^URbnlVD=|&tn_V4|ux*0H5UOFCD8n&X<;VhWZfV z6Nkaih@2M@aW0a-c##G!k~@HgjqjRl+D<6<$^c8VgXDByZt@^iQczH)67=tG4_Qv0 zEMeP*uY-mFR5#trclUa4!;fvnDQRSA*KyxhmSFv1K-LqHVJ+{QcZN&Fr9)Cq=dzH3S`*Y1AY!j{ZU+89K`fBWfO~S z1Y#PeolOxN$k_uMEg3GHWbkDH5jgbhF#b{FS_Dg01X$^@E4kScv>JYT>wJyr_^e^J zk~cp2QApz<_^N)05OQN$+Ieoe|4>jSTHyToy1^pbDy)q2SoL6T7ZiB#B7`f?r^(4s zT)lE5fK18>+**HA6b(YDd+yb%^3yO$tgm0u$~lne&o`J@?7a<4oYdOd!|=odQpdnN z3uaCdF77K&-!)MovlEyojw;ou-DG_Z-C9=+%pMN|iaj+}BOBkri~!Dd(IUWVB53Ak z`cTQ;-%8k@Vrq9&%U=%-M+mDCfqNlro09m4iy`nymnnd%+IIplExw-$7w7_*~Uf& z09km-UuS0@x_Z@#6_)b#Kre;aTprot*Mg~4 zJuop4gaE)Ez;Is|712PELrvp$-L04hO0~q0+qt>=#?^)aKvB}w|FDcpAtPVKsPgei zXeC7#pOzdZ_zuIb(A6IUY$rSEWA9gb?$yxdn;PMGf@ciG(2UfYyngvo76w{??^D7Z zf9fTBT7HBhT@`mNOE-wC=q@<(K4-gBONQFp&(b!=Ylf$YmVf%hot5=Kl$HICi6ev> zoiH^Orl!UI2x3qiRdlUs>3Op?rv~tAVC!wB9DOn;r=t{B?1wJiVF$Yvu4FBZu>qreszYTL_d9;4Z{DL{j_As-nNxKLwaz*5 z1}$IEQ``Q>u5r{&!R|K4FJ1e9?6_lY7Gu?#E#|~RNl}z^EE&T||Wr4;0Bnk3Krb8*4cay9}hXW(ffHwraXL<7(lWc9) zrwhn*05?pbiY4z)%%*>@d#LmjGohHVZYQJYMfg}l7~)}M5W6t{x^zG7c2Vg0wyYag zdY4djy=`ojr%cy*zATgyh&P>Wt4~brRdsab*MC;?g84;e_ND8eb$EHNzym_dciLBZ zU)_uM!U6|0-6+&vAUdcikVlh2=-96cOm=H4f1<43tF$yP7(c?zZbvECb9{e?2swfW{dH?d#ikCtPjn_5RaR zWt1qOt4NW9Sr+yhTVoH~oE@spu4Hmn<1~1o-qHEZsRJhVq2Hzdh9?|ikY{`r&HxJp0Rdv_7wfV9HGULrw1<4I$@Sf%E^AI_w> z*wA%WU2JtUTSevd5mVDn>f;_w(HCBH;kg$aV3Xk$S|JiU;2?z68ZRl7d3VdLY2$`g zubCglWP@$ve1gUmqYvDG+JTxNAY+cuFXd>Y9X_g*Bxmq4n5pa#3U-mzG2h`y&R4wD zNaDlo+qV-s=fV*)IFN!zL`0xszIu0A$Y=NI3j!=sm=fsb8hJfi&tgOxO~X0AkyLum zz!ts>62-Hw-n}d6f;4c5rY`;WLnOcMsh8IqrkucX&o%Day{Th?LqCFVmc%PA(`|}PEH9AAglxUJj!4z9)2Ky z*2^52In8aL{e>DMsNd8Zx?JQ8iTnNV2UKFVfkWYNditUB{DXq#A0ZrUuoxH`2AIA9 zVyaZsk$?O@uhCTKdgDaQZh%|{rUA+2dEw!7z>!8q5#{C8d(s@pKkP~`DLILl147ij zY+|%&`fu}}Z`r8n=*q^u0S`ECjPV<-)r8C_gIcoja*L6Bv+!X<< zz-B`;mNSRHHim7e8yV%mUd5!&e{5tl$z^wS_}9Z|%ce9<@Er*M_)!amf~ocCb8{N! zPl!j}Wk6M9`xwA7@G`2d(*4rL&TzW2rm}Jm4nK@Bsmu3sKaN@tF*`FMg{dGZ4A>MQ z&WVUjxwWlL4*x9X_G`ciPJttDXJ_Z=yY~lQPF?RiEs{?u{+^;Ah}uEap#zbbDZ!zm zL2uvU<_^Oe4|&kWhx~?ruhAKps7bp_K#aoIArnPN&4MgQCNT0EO`Hb5VWm*n@})UNf>$VT{fgx(f`P&ZG`Pv7CBbXrX|_)QRC5@w9ZlUy=OF}< zKZeaD_zFD*ey0rcy2biiKeLe0E1qGe8XWvEfPg%HD?wLd*QuK*(Clas6J5Qn-L!pacK{V0r1*##k?#*7rdqX`{0 z`B}OG(AUbPjrwaQ3I4}I-+9aA60F*`IFUf`z=M+afF-Ksm6j6W#Etadsfr%UhG)*5 z3yJ=UwtMsl$;cbR-@j;9%Q-nY$@iHr&DNsO5CYE40DSC9&C}1`fLY`9^hwsSo=eLk z6-@nkCPig|G;A-SZ+HVs5vpXmZ{NNxuYgSmC^?j?uP7*km#U=0fVp$q%ww1yVJ2kw z9&1DX0zia=z?wRNIq1z-R#!H81X~Y?58s^cB7h1VHr6+rBmbU^p52zlZ_cNR)9Qdv0PrO-u zqvoq?VZpVzX@^1y8XUi_OyTVCan&7|Zr0YUC=}GSmJ2##y#HxDdUzx*UxpcFI@j2% zdGN)89UigWhJ~^Zt*F(?PzfS*wX?Th!Gc-^riychg_eiGAV5X^d>%YFh^OIBeZxiO zW1zM?JM=0CtPj~kDd9h`hAQh%Nci9L4nffU>ro+%o5sV=;9d^?tB;>NS%JAaC2Y12 zON;^oZ+~7a?pIMJ+=s0kuw5jyWEc8MQ- zIE2Agm%*y&BP}2($QA24^?)yJOy#odf^+Rv zQ`6+@CZBaIK(H&XdPCth9I9ZgW8*wGH<$kT^I>qcs zTb-e9AeqIOmdWlgZNCfQ>5Ce!%l4v)dT;XH_MbhP7d=ce<0C6hO`dwa@M4vSRbXRR zLIF6btl3?nBRO`Z%`DvA`tgOI7mqT@-07O^3E64L&^$?og64Kz3f&Nv-s0rlEpq?|!5PFTgz zkW|gTvb2Pl>l@el44SsJw!%Q>keh5$Pzz^8c{x~SQ$U!@&W8@0>w0=8X(;_0Akk*0 zqSUd(=+1vqY62@EiqcJ%3y;Tt>|XfTYnS@_A#coPB?p6ooPYgu#9%EgEv-Ym#3&Td z5?+aDU(YPhr=l`iMn)fMV!`+R!rd?HTaiIQHzD}w#lPbca?O<0ufy3c2Wc;X9^U;| z|2+l&{>Vhb=@c*LDGX6NF9Z$pc!2-}?)fWu_kpDXj&exKnwXeaP-y6k^(WY%dxU&^ zdN8`=P|snHU4mW0`%oDH>dY92tm7W?8$wc0)Glc#%y9lJd4(0v`#Y+Cv2Y zK%@#<2%2}5OrQnIj9YmVW4Y3e-ww~jA{&?pP6F>KHkR4DRhvxbX{P6$ za2Un;|CX2TA`pIH!%?`urj-umDGH&?uK6jd{?*;Ct1P2CMfygDqQ4|^LJh5u)s`%{ z^vlG7MC$xLN5#MNGxYxE}vg1?Jk!XJT#v|h&VONolTqM532FjXKu!)6+hH~Z2 z!4nRmOH05~5`Dsh4x&L<2xbZ%hfuR!JTGC47=+Z&$g46M8XovaB2rSuS2uw%2IM(9 zKR>_G#GEJb>iiC)l_edoM&UpLNkS7mO)9tf^8x}W@e5k$$K^+S;h2MA*-4O~2DQ|3 zM72=(*H5ei7z#k&vq?v&F#gk%UCZdcn%xUE1fW44*B;Mu^MmFa1J9+0Q}Uu9qNHF}DXDo;9uT;{&SoGejY0%^90A1@D2I1uAa2j@P( zjqu{RN{>u%6Qx}bfk2ll?_A9P$JLjBQ@OWatAq*}8e}RG4MLH5$XKLgNTVSn$(YQc zP)MbsQ79@zNg@#v<;W3ANkT|uNRlDKx9+Xp|NDOX>gsgPL3=;X@41Jy*1c|n{;Y&? z_Swf1$DVO>8z(=27|!>H81^z%fBONKAli*S=GS!?KNxlLQSH@8oEZf}{5a4XQ&ZBu z4*FqNZfd0uCSdQ;MM8|BYVT5dx+Hd4mEHK#({n2OVBOG%HI{x$HMyt>{0OPX4Rt`N zSTv}8(@f{5Vr(U(klt-@i|6#jYLfJwSuC)e6kWE|?*TSGzTWHf2=X~Rw^kr&ft6q` zaPD6D*xqbd(A$iBZ$Sp43mJyxTK@hph5&B9a1HmMTvXId*zNSES&bW%>eSZbZcuLs zD7{}x@$vfB)|0-QERPwspAe7auunU>Aol1*&WVHB)21`CF)>f$V%o^GQ7mX&TI{U; zG>KH^Eo=4jCp|CfUq8R;hWYP7kxN=DF5S;qKox@VVzGrPO!%kkT40iezYob{>WPU- zS9g$4v0D4HS?!zD`PUo@RAx;->v4}+sl$MY>D9g2%Nws*AIMBhea9aAwx7lRaTa^U zgs14ofNeIV0WR&{2fsfz+H(EzjhUO**@x}p-L88w=44(&rRyB`-${mS^4DKfhkqFj zi;iE>7$?&fx4i$^;QEhqiCXY1xP4okysXH@)5F6fteb;RhACQnvlix(R7AOr6U?^P z1qc1Q<1k0UjvZa88z36CSr6DwAu zIbVcF3A%^j(JHg9A{KT&rnoJv1(vQE`}XaFd8rwqAs~)u2mtWF2AHU7a$zp2%T1Ng zj5F8^?=Q7kJbI%(BQP+2OG!SrQH)Q{feVYELlH(H?Dz!lECwQR51>?ex#pHFoKO=` zLh0;uSbo8P`#i3+aS)`hGN5AcgFj)qd!xSo0dFTs9A-=)PL^O4n|PY0{PjAubMTg_ zv(!GpPR@iM1Y>U}Cnup)!y2^UEy=tqmJP0hqyV-pUMa}W3`nEg4%z4%lJ9q#G$-lg zy?Z%z0s;bH!CBBOT^nA(A zp$ObsZAe-%;=a;GS#Rzzs;VA6!tylh zta?Sg`QL(ASXo#^(T!P>9T-*w=^DP3g+3yz{}o`7@B`r+*wwrW~?b%5ES@*<1(g)y_*Ib*^I1i37O2gesNRFowHhZ&pz8M za{N&idy~{!AC-zvn3IU9+W_7NF-z$2`)Tyg*!inYL<}gH(#+8>DdEbAy%Xr(K=G*Ff z+ucrWWakz!?lZ^OMnj^qweG>q0n*?do0{QU5Tp%jmpm|EbC&T%;fQ{ytr)8FLOC5*Re1V##} zd^`LyXZnn{iUEGU$xe}-JNsor^I#t^gG_iKk$sqx6Vx-4skpcpgM>Vsf+^3R_;xT| z>yGKDstiao&CwpmQeUeZxw#(7pI1ZHu{coimT8V|K{Mo5HXEj9`4Xkfv2G9xN0LyI zHMXOlDnkhU^ySOD%JMvTojiT8uj0OIv-+OLho0}P$Iwpo`}g5!$$sV^+g^5BAPjG# zwoap`#2uUR*o9Sdjo7W1Trg;XDi`);*slA`($xY*jw2U2RZl zt@rVX?;FcBtSy8^-EknZ9EX&W?Rz+-q@;E{u$u*?zA2RJ{sDj|d^+O1L6J+@C?`&A z=3<7|5!;n3S2#WKaptcwU!thE$KqupFsLjs^rab7mCi}^-k#g;o zJDUWyn!bC#qQNm`tAiyLQ=Vq)_Vi2bWi;th^UAda^3B;G6gb{p&;K zHg4BAO)VGCYTTOPQdNx{8*otL+NJGp4m8~HbJ8$S(}oDtbTSTtVrA#u<8j5+wFf5U z7i=0^*c6e+qoJK5`}WUo=>(7CbI`oqHm@AJJg3d|t@pk4<{BuecpHlfR|zOIQxbhZFZg|!?% z&;~&NJyD~h9oJtymmZwt(73~ z4_tqi)%NUNcWLkdA_3hkdZ?!-nORs8)6@TLl3KZvXAK9QzbbT@LLbnhfXK#46rJis zr`pkYL-FXnD7F~b#mbfB*p2u8)Jgu)&N`dSC977=K?e9fJiGx*+F&z=1w}d=31%HX zW|FFN=+Ihd3NY^6V37=c9RP~;V5DfRD(HHbE&8DA4W>X4O<*aUi`Cg9+gp5EP>{fq zQ5TIHgY;KW;EpzNUB)&Y?k3gt!2_Y8%)odeV@{S{2Sx~6-o5a!1eX$D;F7%ylSo#! zm@f&Bh~So9Vypqp(|zmW>44!AZ{94ml$Mf$8rAA{4?hL&E)5qNhjMkAVBkvH}$Xq=Vt9erL*p=3;Nt z{T-6yf$74SX{ZkTJrWl_st()=C#~aG;Kr39r4}C(S$4|%dBXaa8hb(}|M9Wt;rq#6IDDzzBB%HLr7+Q< z_8S&CRC8?Lhy}v5{Ir`qcS@p_$q(w!iL*c0d$*96@22;WKvLnGu zJ`w&702-Rs2FXhs#R%R)w@H19*0XO^5W$xn^TnAY%Xs9wE z1ijf1z?PPkMKaLzpxNlK`L<@kiYR!A+x@)&ayV1W?#Qo;E-&@S{F`l{tu-qwEWE7d zFAvXS_rb1h$Tvzq^8?Lb@;}s5A41D$kywDNeIFY7_N5fxSZh^VSEu0_hvbB}yaaAW zrne1`?YjHzgdP4T42@>Nt%#e4Cm5qF^cYS76kv*;{Qgma_qq^8p*dAmeuH8FPYfYH==Jj{_!%9u(M8qz{wl)djeKWSYJV^X#_k7!BV0ujI28&oyKC_vu;z~Dv z+89x!z=%C-Hi9E7Wb>q#0D{Hg8RNYnZ3~+cKpVz#&_!tWnik30O*6U%hO&g2P zNUho^#)I00j&W#0f|Y^<`s=Az*Acu`B7FcVH(01!KV|wiPGupi4;rr-?m=pO|H#!U z*5f!zG9cKxMOQD&L5LPB?|dj_GnTvICUWB=+h_0EE60!UJjKuAprXsJUePf8^Jl1e zTDAtDGjt4!4g$t=MWz%~5IdoE%GrrR;S6kRA2&v1MEKk`iZ)m6kWc-w#ZN5euFeO64(4B8x zKCf{;AX?=J(5dCo67C?Gye^eJf4=w2zgIq%y^&2#NdZ~cZJoKJXC3^h7`X9|A3SRo zMO1>JdwOgJQmR9iBvwI}mzO_jj!TJL#LjYLsD8h*c)S%HeDzNAFEhC_QD(Qc1-=)W z&dztW127N9r|NTmV`}(l0ukt{zOQeHBW9yw)X~#J^V^xf0=);IxG!7BYN7&5ZSNP2 zb|yuE8Fz7&Shj309E3&1d8~+Th38nT}$mHmGB9P&v^=W*(W@iI|5M8x}pKVyF z+U}aWXTC{ow)!u<9r%|cjx$*o(0}&h#u_IyfanyrOpSsHtMSv(d-TZrzauC=k}J{L z0i$xecJFSGMCV05m>BXKZMW`#(HP>?`@(?y5+=vv$B&cyn)$swv#@W8o7XD=ZW>s- z#Aw3+A_^jW+%QkmHGC%v%3|c#FESTpL{RjIQwMWu6d~&5$&<-jRKMrKDYLe=b_qTB zrpxCcayk=j8aI>|-ubnwyE_wwdf0gT4A338Z3MM4Mb_@;5LV@bv;b@T9@VXtmBk3M zB)2J!UvRa+qWv%$6b`|7%jmcUDfwfT{K=gPT z`8li|`Z}zQhdJ4sH7Kf239IESB7(y`_mMCY>^)eK&(E0{8;M*nAPFThm?(Pcz4d4lsDMKqr!C&bT&ZKB z3pWv{4N2I?ta-l<1^OJ!KY&7wM8mxm-XB+hZ4Cv^kPvou2GHzDg!0>HRB$-_BihTaoi9RhH>O?|$rxkBp7b!w3|DVoFhU3mZ~Cetj>AzLk1<`Ef9pfyaBFwcjXKGjH1Z!h#n*t<=<-TjS@`gz&6BDR+zIBfCXv0u#t=SIA1!108Ymw>QXGtg>vN~#%-W<@* zKU-Pm3ZL@EGCa_H9-W)mnLvS17Dvso29&w5herR$`$ms14c=if^{d*m>9d$l%-UKHYa8Vadj7o4kpo z*ph>Gp$y$|Z#EFUo)6h`(#-F+daq8Df(YUSF;J;hfXqQ)@rZEOL`ECjVw*EZ+uZzO z!9{2qP>1(x4KBeXG~)jKN8ovuuxS-dK6&dP$JrL67U+C5eDrqSf>nnO?moM_AvpMi z`Hq>Hyn{w>?FH^s+V5F&RV11T5trTF`x)0yr-gl`es> z0gveS!NK{`JEYO)5hwQ6QHVte%T13KTi@3Js0mUR^>h5eF_21t>%90jzD6jWxE7_Z zSS?WFP2wT!MobO!EP-riv@ zGqBy|frWGp_XsD1nkCgJK8%ZtgXk=LWW)zG;LDc4Oja~&xFVrQIEbSlHN5s=CcR5% z<%QXhlauK*TC`K|-kFcB#Uo3f2o(~z6D(4X4cAXiehdvca)prNuDc8H`9+r{#yf!t zquiFYKdNNB4=^sN1uO#w-^vHbMx+b?XJ$bU0?7J7ouN2Gstpw7M_)!P;ALS1SuA0_ zSKtNg{^re_);~U%i;6Gp%E#XUTZv-%K`20$ii_)o!ktM%Qu5!HJ-r}Osy}{Y2O*84 z$%UiI*$W66-QTk}Z%nVPR}0lk(u4zN!T3n_G|?^rX3e9;cDgRkbH3GH-7B;6`5B$4 z%-K&H^UOcqJ@~w)RnOtVTJ}M=7dW2y)0wXZ6bOG#S3EC^Q)aC2-RmNNNgU4~UM&-s z5L-JBfNTJ=b@u#uQg*lafriK7;ghvzcTExblMoZt+u_Wqc({`bEb#{mVW7*Ur8-S# zn>g>9M-qmFf<5KVod^Gg`3i=xaPZ(%Qq6!RxSSAELuqOI{N#L5iEe)k^Wzr) zMn*J6r1nv|OQ^>I$)>({}cR8cD`)fPN?GDYw9&P5h$dT;QP8920xR}c0d?Tg*# zn9UDI?@|P3-Hm!;!kEIx(l8{6r$i}F;lPyg2}w|XSW!^{Rt6h(BpGI5@ta}7`!Hen z38QhHdK_!AsvYDuV(B>!zWey7C#jUu)6*M0Iap?3Q*spfm?IpAM@EJ~(XAjMjuYS1 z*{{EG9slP1?#KE08-?Y-XM!~OIa~*HuXCI;C^GA0Tv!O=2k)D>yUPO2eIy+p=ANJ zRiwe=c>{%u+4k-57*ZM91($O&N}y=5c2UQ@PDhY-bh)sj=@c|D{^R(Ru(#0MxG@~$ z?`S-B4)4WA5$7(%d>n;SY!Ttg+Z`O116N2lU&Zv)K8rpI_Fsxh@FqNZqyY6OK$ik@ zGc%037~vc{9d=R`?HXShyE=F0z89w*WgT}r*_7Sru$I}mS7u#Hdf_Z#OkDXX zEq&dvh<{=Heg=l)XZd>prZaVFIi6U&F9#{l-t_9@B$#e)&r%3IKrKR3=ae6}t zCm1nS+~(Q64rnY%q(>A5Atix<>L6OgY#vN#ibrdLGeyqEJ!fLlEFqMeyov=1V(SvFRCCCD$MR=C0ZW06* z#kqsh%%4|nJqAn>plMy<`SUy&o+YZXcSv6@wleqdS01vu*9>71k&@zsfbH6zi--Wq zc5?heADWt)jP_XJ1EMT~bMA8-TT=4c$AKP&7FLLI0J<(ky@lMy(g55;6q=9lQc;0= zQ|RIqLvhN~sKj*vpv{|oeB%mzrv;HSn~w}|0DKngf(+JKwqB{G4Z}FW>khuNMs(&3 zkOoKd{cCapS!`gpH1`@2%)7lfL5tF)1J$4kBI{cD+xOuoe)xp!U^UJXzk`L_yJw~+%I3YCM zsDebWHGshsK{BOlxU8@&s(x{i{Cb@NBkeQ}g|y0CgFXh-)&Ghkh4#Amoyz-z@A+7ZvaEqs8FpIjH{5CRzc*Nnbwch=N(eaSDM(2AAF@ z1EWZANqjn`{+xI$@pfHoC6@Y<$Fp`Lpckk@cHMn9fjat=|IyLo2o6MMf+A&*nOHD8 z2R+Yk|LcLyKvx--kk1T>|_l}1S++b#xm>BT9fp8V*h@c~yTa>{u8zWL4&0=RK z*O+4@Ub=Vhm1Ra+S_x@vXbucB6{Vp%0hjQ&qdeG#v9lm|z8PY&vb^f*>UB zMnTqN2rMRfOWcDx>RdrQrtioMZsX)CxQ=;lQr!eEQLL5NaxEPA1rXwtg8=A76cp;3 zUX_<0tF#Y6slR4h6&F^xjok|#BL<^yoQ;GD4#;1+?v#&7U|G~bF z+xXEr{cn9ibbC4rq1%BbYY~t(2ag#&Nlyw2)?n=qoTE;L2>xw4j1z2S_=vF!(@$Bv z4mX;FgbZNOuAn(&njb0kV-;LKbYp~Va$+3wQl(g~Jleyw2L1fywfkmcjfg`Anuiv| zDM5#OtiCeb<4GZ!vx6}hLd_M5g0Un#Kb1=)jZCq*6dUxyilza|L3f2;a7V)LfD|-u zY(4y;TalVw}Awylpm1#;CI8xo33`J5C+MCW2+(M z2!RfY;wt<|?ItDhvhM6_!r(ff?%YPhcqwVB|>ZCLApttlG~4NG&~+j6b?HV||OHZ<%iEE?>q z;#Zr9cHKP^5RmEBy9+XxS-7G@d1$Q`S{*sI-O_g^C?ipEv95eoRn_}8&u2Tmzdvn3 z+#M%`c8=~i|MasfGd532$l(sz5dXotyJ#ej3XaE3}{7*0+D?J2Dwm+3f$!g+Qq;u~W0X9lmae#87gz7T@7AIPpj?Zh?siy&&=N}5 z!e1|L6nb=Qfm>cTNWFzCc244U)4GKtzx&H@=N{G>T~j>v>C-3qVPEc=H;`}5wF(%V zpKe~JR83<+V1b$>^aIW#F9Qny=~qy2F)r>jJ`q@HcAQlt(4m1^6knhN@o>*mxEzk9 z3$Wldwy! zl6nA6AYQ__FN{~B`U+Hww9$0)=FRyPP*j|aiDABA@V20MbU7d;Dh(iYQ}p<9s^kI0 zwl`8!$)@h8;k)5`UooBqP1_7yhzkYgV5>08MBz|WJgHAh>QYH_bMs#7Y0x8CS|GW@6U3vuW~!ME1d>3>b?Mn?}l)#^G2Y{9GN{V~jD$XDQzpqmY) z#tw?VyMblN@pI64(nYLF#bW2FvOt>A^y1;g_9BpL+i%h(QmbpL+wvjy^OOJ^*9tgb zzOSF34U}G_ruY*85p}ZjUC`eU+(g}FZHh7Ql`!+PUIw=%ma}2ZmBtCi<}_x&kD4G; z%td_y=?A&gfvhbUzj&8n6@_Kq@D%@wMSr8OXn@(m2-15z><}Lle}#`Dc(EB`vBlV0 zV;T5sAKqbj%=x^lsKd}2H=Jr#0=14j6cKZBbUKE8P~(2;&M4A@D!srZtDe+^i1ski zT*HXNSgUmgEi}}P&nzh#p>M!Ik!uN*@}-BzLw4PL0T*xBAM5#pr+|Y(FKH{TBa0Yr zsLjV`-i!Jf+%4xI$qlc_hP73gCM;O6fCcI@*N)GhVGnr)1lS@7=g~9^zOtonfvTQgl}j}=@*mDkzXDbM@Zgf!F;o^WEtIf|$HZv5rcm|13o5!!V% zHa6kl7=|mBFJJEFGl{|<;*$R^9}tZCf2&78n0oW}ZR9{+u!GJQaR{ujN9JP(x957C z1;k${DIV3ie9S!t5NEi&55ZPPttO-`4@x|L{Do}bU6I%TKV1=C+|y{cHXeZ#+zPXrh@neptIy3bVw--krX73aKaHlUx#0M z1t7a4b(3}$DhX{=AhDi6R?h5xO z1hU(=ul^th2M0sC=hMa!c-uRK4AEcT)CZrGSqBD}pkmyuQ)B9h`$+?WwDRaNr#!@k z&@o2^ZK&-){%gU39fGzew8o+5p+`Z%PJ{&-*Js4);gzHTZA^awtC;qf*&S+uc-Y{|Pe^D*qWc4P{&_s<0PjR5L@yl%CInKv znK)}gLk+7LZK4QN{RA)IoPq>9>|sM+za=V2y0b_BAPw&lnAcoHV6;GS){ou1L+=w}1cA zb?e^VxLxBw+NCC0{Wv|Sd%&FF`UBB8U-AG-7@|i#FpPs5eFY!{F=#sw=Fx7`(xIU| z%M3g)D&JqO%J|-QPk`DnoI(VH`*mPeP9rrU$ZM7&K_x(x0Sz3wDxt2D6C5RY+mGia z1r|fn0DKWge+XNLmPJSO_D$K-z;8`zSVg&}ymPsiP*TrKsPkKa74qQ4PT84}iEWLEzs;d}w?H zHh9(bl0zzzCKmtgF14$TL$QlK;uE$Updj%tK>R|ni8DoL~idPDc1rV5DlY=HQ3VhZrYGAyyGuitrXYwdoJ z%>;?Mc$XkEiP4MgtozRw})4yudc#)>;_PIJq+|^(m=$i3b`e-DGmIVMXuC${agi%VOVH1@kli@&t27D8=hw9a(F>hwt+u2NLRVf_8 zZ*S$HwLGy}H^^2H%JH6cC~SA{o`b>70?R09yZ|$V9lIOhARNZUl<9q6(}uQnfRI0!gwl9H69Bvv=$Hx8p`av5twT6#LxN(5tWcLhGG zaaC>8{v`Z6*d{hQ!hc`!FdrNG1qO0>Zq}z(5Pk!=-l2C@xGg zTbW6n*uH%&I2r%li9u78Nu3yG!Y{`Zt^z$qqjHZQ*WXZwasm+tA_FT4gi>e$NG1od z3>R~8VGl}G;>}54sHmb6dHH-wAFP0V0RQ#_%2hrUaQ^Dm*{AhF@YcEC49Lz%00!7-@cOD_+$_%&Fq zSaJ6HHc=1^YtJY#aPk1Y4fCjj>JmSc$_x?L%gw4an{{g2yV8z%i3=S$kM}~7GYySq zOK`^krQI)I-<0G%}p7j9uukq zSJ{o7GY~`90~5~}h>*ngL=Y>_5!5hFvOl8^H-4Z>qbx0>@FXU^pqr5bc|!amAUEVD zAUuhQiSp6cc@a9`zX3jw7pVP^-=t5EUqBS44SyKu}KV1#oHg93smCq9*}i58S@t| zv_)++RnYq0G+6cAE zNZj7m*3#G4TyP9sZ;B~YZAyl!b05ZCd$e0NMtj5YEv-qCtI z&zPUgQ(cHCuYq6&*!&MwhCxct9q|bXQBfKbd?>#FJ&VG$vl%0>$)MUlZxsMK1LYBf z2jlv{k5L?;F`bq+*dizn2%Q*&`-j>MaJ5ceKAX~qNy!~KPK)GSKC(3|_1Gz>>cfTG zN}^0C5oHc{ZY6rRqblK19lEDXM1LrBF1J~5jc)@LU-&BMqQS7b9m5h=4BxG2pr^p^8m5M-sy4vN zCM-m7+EK(vTqzVda}^XGeVYSNV(KLpjOPLyhG=%CR|?9(Qn=cMifErW>?-4zi2jEL zdQq@)dcr^++oT3lgQVf2pfUc{h;zn!7OYXm)cup4=L%&JdQDAj`{wjtch%WH7=bDd zqFByxQDB$#5{U^YyUCOVwWA=5xxY1JWyIg1sAdB#d4{yBs0N$7;BsLU_=6WoZVEwI zoDejjgc_%}7iY*8s&!FDV!DD|0~*TLg7)CXGC02fJGlsr+aYR4VM7Hj&{(5A3V1#t zi$$sr)wPK$Kv>{*=up=eyPp>1Y!A|ra~=MU?(lvM+gb8{AaiQho97IhgDlc!20Yx<| z@ezWXM76!-6@--ZUD>mNvtvAwrx{@zt{%3om<6yYKfjF70cwK)nKU#2DeWs9yB8i& zI$X11@r@@>J~qW;=``l`;XI+HIW_&c-xa$5-(A~_c-9y0V{qu@Mz({-mj~) zmqjSU_<)djm!q?Dz`b4tRBW(1!GzYm0&&dtK!7Bpd1*^>+vx!Ote;wt(}2VyIgN?Q z>lQzJJkl!Aq)WsS1Q*6DrHYq%8iXVzV05a{2G~DFxfQq63j+#pMz0(ilkq86K`H^q~h3`K!HSEOxD2&b`T4YsKY zf{*<~@|MfrbHO_7@8YGYK=U+}UQgpHV=JpDWCN8atisT$lClW}u5Z>lV~OxHXJ9CB zr@#VV0c;&L@Sl0W4QPiMNBN9+01(`+KAirN>+i&)^-JaGa{hV6l|V#^rbiPI5)A-t zW(X@U7v6@68Wi$^YJT&Kjg3j3K)OzbJfZKUgPih1=qNBUm9IvBO%u9!5aj|iBC>7U zg_1*>T!ak6G{Ocj2%n0VWM7n+XxA~&{t$L?^YfnscSbzeUSBYc(4&*)m7Py!{mq!4 zlEiKutdk16iur$dL4Im&6*XDel|ftv85e;C(zCTqDzHRis8SzehioetQxin_^3I<> zGpQyV&&!8{nCIA?<5axcwcz=$WqAO9O5i@zUoMeAm5HSW^!#vecwChfco$|;R)N26 zl>`Vf$v_!R+J>pK-n9tH?JS-34rsa^v$e#dBQ6^pA&}7-HD6Z!$Bf}c{HI^snESM? zZZ^8L7`dF*!v~h0p7_K|`)WD&{!C5^x$~5EVb$P`pA&(&1^_Wd;<@sE4=zQ_YnC zQS|ZT>)p}8ill$nd=A>o!bkp@e|ojvEo;?qRY``*e_9XdZ9sHF!eaCr#=CYoGT5{K z6bBl@i=8r{OpuS=922o0AJ1SO#Q4wh7=j4)3s+u!5Z2Co0lT&R@mF!y{_BK2hBPI5E^thejkb z{%0z#V_}y8%rnV996hyt#!krvbbm&gh6`DNuW`TOj-ZAQZtZ9quv+BTFA|&zhQzb2 z-?8Jeb^{0^h+0YC0o6X<4TK+EZgKgLky%Yzf~AAlqi=Ua=ksk>B}a17XL&pBE^Tu8otn7$O5Sv}-sidZ6P!c_QjzsAQ4 zSpPUQbONacY0*9d)<|LHk0GT)EUx+RVRVl#5L!rFjv_GmL%~;!i5F4@AXOt_S7kfn zjCT1W&E2ysIElwL5ht}|5NJOZ-}(S&{BA$%|4uQ!3^V#iU5mySEQq{Sc?g$6Ok7-4 zO6p9A^AHbCBrV@2$`)ZqSyJWDXEsO}&~5#+)ab-0mp0-X_H4Ukpom!t6*h^F#gisJ z?Htun9rbBa=Z3=q1_wPlc|rRiTg-{u(orq_?Csm^H_JV0w|~W-7bqV{{NJW;dy7E z&c&EA{7TMVyLJlL)gG|rQzG%$*hq{^kVqViFxu80X*3_T+G^?-FM$AEQLerA3OiN2 zIIq;?LS36#84yj_9t@&POidvHCQ0MD*F4P|E?BC?<(T7)OVu{s^-9_bjP+8|u!zs5Uh5)BovD`v^DY+)K;TWrd; z$$$XSwklo?f(YY`#RE5!BT;?BiiP~Uh~=V!7=Im?G~YQk=pIoghof~4%Y$W@d4^?W zKWq5J%gRa)5N4chhcQNE4Y{M}ZJvZZQe{V{%@8bn0}5iWySW0Eu^@mj)jk?;AIY)* z<<+x0>)ADXWOS+s^!^0#cT66z1Ae9V-IqFQ!-5Vw8fY`wQhmBvjGv6A9p9 zonyHXa3rR;pr_{ydCo*WxHW*zQIZXf9=r`ij`A$Nvp1U_0CYqrQ1OMArxbOi80i3t z{_8JbYB~UafqJK*gn*b`cXNi1`&27vCDp74{rh55;zDu+Lwo*ECHw@GJVbm##uah) zR7wzaS8!mmS!~#Zi5B>@AZ@4rQ;t(Rs-C@z17#$1X$=QaDB|h;)6EHML^amd{*TKs zI9rfoX!Xhd2S^d=a8RV>cfZ5vDL_R<2;qse0z(X7?$B%8nndBx{bS376JpE^K^VgV zlLcL#dspoWRMfXBm2nF&iuCNPh_;OF3wFSP+~?^u1Mwk&8lXrF9Vi=~)6mXC^vUjb9%T4?W66cPnK;Vq(eSG4w6wuyf%a#odCAGy3Ziup)gA5Q{Y2Q8A zR?i)w&&9kaZEYKBh^vyp@7==-d8e({+b8Q%cX`d*=J`tt;>O0k*%*f?K+&H83@Hd) zKcDo`ZoUc&TrSqFf9n~RhsF?Z7r$_57%pSI(Fb8#LWAs3Ye}5;H3tBVWUov5V)5_y zHCdRZF)``uXqrWrBa81(xY=C{r&mv?VDRcE3mP=zfUp6a^th=Zz0<8l3uRZPOJ6~+ zE(L-)TT5r*OCRNb#PdrPCDI3Kj8}ls5!l!Y?KNB|-(pXw9=W)1zVkH5e7dx_SXGViavHPzLhWRyY-#X}%h0k<%v zjeIYWC;+4!&s`f}0>E|77h$8$j))b4Zc9YSc(SFnHJD&oh=Yc9LCg(|SJ7=6Wi`z6 zX?c2P7g(T&CZvfMxo%K3Dj`Gmf0YlSU6%M+_@8(nj$-f!3OlHYAh`Z$Zvn^)cntk; zJQ^sKOP~OuK?FGBAlSgx5TsYwhqmbRL(J&UDOS>oGguD*A$qh^0_;0khp735s7kyCOSAP5G)HPMi0_@ns@0tRn6xa!&3pp+Sjeo2*6O9^xe8CjK^~( zDq`9;ayBlGF)9iFMdCFe>*Jy#Ing|g=s(g$IvhBVSywbZv>TV0_-#z-gT$UJO zaA9u!t-AjC;56Z8(2%)FxM%_VH3&4NwiJMy%c%IAGYi&TfB5y+i}PrCWxF^|0e-`(p%$>%)? z+Q^`&FmP`$3Kw(Xf>Lec$B#X{9%?>dZLtM;i!VbSoQ=y$ZtoB-3%({jCa(ZS!K_@xB8@gnyp4P6&U6>Lm<~;3F05Y_N7X7?Auv?>5zG6kEA+Hp3u-n7`oAqpKIxQsU5 ztloVBgTErHR#oh+$LJ0vNHB{{a|~Jl?-3O zOPw^-Uv587PR+>J5D=gS9MCoc(`tAZ%7)dq-a0n0Gn3m-D<*TO`=Rdp;>C-cb#rFV zeg>f=bwf=(y7UOqYuv!uc{LQ1P=a3bx^ysm7rBF>F>JhcDo@+ckR1>wN@dQpw~iSL z;Wig3UHBB9nn>)rh9frU;NcI#5nIW^1bqWd8o{hp?9kf8j-?k2FcXQ<=np?5$Slk< zqc29CoAlBjK#2(VE-Y5(uIeM)AM?ify!vBs(F6ml=6ByYvCsjCGStHBF&ay%p7LWS zP%-Bfw}td>+qR8ny#d`b>>0Xz0c2?Br>5%P6k~jvsoz>L;7M4Bnb|y`kaPiX?o@sT z1;MbP4RjpI)(JyE;eo3Mgh-MT}tu6>e*g9eV0!x_Z z6o+*7+__@_ghj>0Pc*MVC4UzFPBiL>?c}WFFBY~B1$jc?=!)A$31qtBkzVd{7$rHH z?z$WDxcp=qbyG*ji2TQXR7jUVo5E04c(qLK?|T&zt30r-U*o)F#d+_F9Oj%SN7uw| zzrWMj?R_1Wfby)+vnOtrW8h89CG-whVpZLd3s%-5vp4}O%H4;{C8yr(hZfDKQZ1*Va7`H6K%_UU@68$WL zjh}atY!oq{JY-BNT^_Z8MHzwT7>4ylW#T$w0T>X70AM~9ylE(Mk}b09^l0wW9QloB zw8J=W+___5K8&ppfRIMi!a5f&pEn!90!qcLcP?DI1SOBQ^~!#bW?WocNAKN}N;O9u z1EF{psy*w7hZE_O_wq@HRyXCi+`4yBbBz4=3LH9lMqBRoHKvohM!#QA zTD{h=LZ*H)&QN zSGc1j$cZUfgObPb1@W#QeCC5FLsg>#JSklmlc@lYpq)LiemmN+#Y^y*9!h%kY76ZS zL0&CV4D|pa4E=!)B)9@wPzW$y4kqr3f@DEQ-Dz>L=A|}p{6Je7;mpG~cW0LUUfgMt zZqzh4OTPtFNnI*!DnUk=&O70U7)wmEXZJ<3eE(r7@;=5Vu+=()*{1>JtvkDh(5uF@D%IEJ#>2KWWUb_p^;>Y}1X$#6}=s*%#?&td!<{h+Uf zGi(Mfpti2=Ot6{U*h?$(^)t}_5ajHaj%XBVuyZ6U`h^Q|-q3=?_`sHYNwCc| z$5{YNNUn$K#P36cg=l}Fca9Af!~zn-IrcQ1y&)t_Xne=QcGjSE z!);_glGv}`0@xr?o8$?b<&z!rjX!*t{J{$Y`E0T7bvqWfE6x3gUi5%WUDkFm@I78* zu7R$osu0{NSPsK|Kt2bG^wSS=a^z8FCP8${8P;~ZqZ02Jf+U<=H(O;YZRnNQPXU&+ z0JgSm;bm>Dtt1(Q(&w=P->iM|u|!k-92$Dr1!da%#~u=Bysz68k@dU=T$z)KCawI# z0P^1lZ0lmZA^Iey4W8Q^t9KF}h!5YN(Gn6P5e=!YsDgrb>E^Wv0w_&SV^(dLb*ZTS zh7GI}UlS(u%^T2=p)O`7l=P}9F=1#pH+7TaA&y7ubXAHu1!Fqh86Yk1!kP57atgm8yc?<0#sER)gL>oo(KpyNfYWS4Ue*!UGahI2ehZfo# ziwG=;j~Ax7WIW?OH9qHn=8zpD`aMq%T%plr&d$zgmo9LW2EHgOW5scOfu53CELISR zl=`pFyjX6#X&sIpFzx=w23Or+KG$C|S0nd`u-p|x6dPkV120~6kVEVdRCHs4n&E*T z$gw8>KD87x!L8~SQB*UAm)zX8-tmQ7J~k9Ag?#v_ola6oiyL{pl7=V!UX25`#4V9rGyegT|E#AvZIN%jmKNj$UOU*STNIb{?ZBZoDf4#BZ;1@$;uy=L z80YY@pNBC)@38jd57VTF*^})OD8LU!{4k9?BXb9h7_eAw0fAK%q=16#n~^8EAhKs# z*!DGYa-q2Ot*9y=_*q+Dm%Cg62UlwunnE_Xb;xXjs+MS79AF5#0Zt4(WBoOM_Dqb* zM*;}L(8hncnDDq-TQfnEPa^XTPtJVl?v6yt+KXD~$nt|cYxz$T=mimN*wBzyYEwf4 z%9fGo;G@2Ue*C#P`HcwFq~KjIIbu@T=l94OfDALmmmZSF=qb)(eNo$fHSPtGy6+aQ zS)j&UH7crFAo1+$;Z}5Yxo$1f1v$ei`jtGs(0+-JZf6|?sd2<@L|VK_mfCc3Z_~*Y zL_+~)_m(G=Xt^@@bZyn&4{ksR8y_F1P{RxfSVxexb3M?na{9(Y2%fDZL$)B$_7<+U{oMg z9bc>08fwhJO(A101X)-mq&77-N1@9hXD{%47oUa}y#n}xg-MLM$qO|lGNUGjISuv1 z-VLa6BdMBKU+a#DgOrG!uuxe zSFKuAQ(qqecK28+j-slSo!w2RMvSB8H*#J$DZq?(J?$u*2~+ExF5u_SpNF1^1*5GD^?dw+k^9C*?*~?4l^Xe} zV)%s2AwV--@_+<25-lcKSE080>~R&3%uzJbU`nXN1_N2!zGw_Z2M)D0EUF>ezjHk% zz5`7v9^=gP1p)$*@VC;82oKChaZgqb_;DMotP;#%;`pq2Vo2f1B zj~@iD?1}dfOS}{b3kEPPH$73DoH%jfQ~NsK4RQjEUU092!^S&r>*|2dFPd%|A^Y2# zib-}(POZB`zkk{5WV}RlvD~fkYTeJo=5OZy!0;xfW-mB;FQ9ZAK8RRGQcG%-`q(uopErkMf zOG@kiBSR9u`|X z-01!OiLWfq*-Xd`aQeuPAE#0JEm9pMHKEAJxT-Hwt-2ymZ{f{?HVI~&Ubc|pqu78n z>kY`o;u*gI=RP<+GvNy6G~lb5`*=x>q8vnJKyZEz4{yo_K@EImkM&AC%2f(S+Fsqb zA%aK9rN;;i|0FbB5Cy&O^-6_x6+W?~4D2QTzG_N8VTv2iDZoZ5elWsGQURwWA7y9E zm@zfUDuRNV8cAUB+PAJg@lV4yk1xO#?(OY;a8*KW6H;EJ#_yK7_)aOR{&9cn-O&)? zX-fi;NzM_#2{pEJai5+6_6B?9Cup`LDLviLEB)?Wj-auI=I<{AT(RSN8G!#_6TcG} zl*L$|rTM_%pf2ry>Ow#@^e1GIZzPkqB%E9OH1?km48(deAOODTnRE4ID5k3 zcP6Kj+k@Mz^DSm1CUKoUIPN8}rFQ6(aT$}m&X{J$^X>`?A7U3Tbuz>p(c5AY_-W_S4c$|r7d0zFZ&K0>At523pPlE-Ow-;<&Bz1A zD92x!@|)_|7rb~lVf%n8w}O9l*uc<`-mYEz3#2XCf!QdC6WGFd@oQY$B0)lJOo`kG z3Jsiv4~@7%MhKRcPOi(8fNYZkB0NQlk)Uqnp(=tTEWD}7lD=m4aV_hYb!EZ)Beh&wSbL>q-IP*Y% ze=$HDoQyD>iM`gJtXP>;1UYFM6$kHhq`R{v(U?0jn+nlo{vW!p56soDHw6)!n+bibzNzX((Bca z^gNASGa=({$=H)J>!urL-zmn?dgN1E6P2{BQCj014EglE(lzW^PA42Q;yvByvH>*; z;3HT=$=ytYe#Wdx18C`#_h zT4?7AOrv@ZIRR6{=FQ5`Jl35M z$zYnqz#LkbMWoU}mQ;O%!&+r&pXFQYCkluj?_^&9H=qcY$-tjKznVf?c1AHp6e2e8 z*_eBGKPWEh7XZ#^q$jJ&E*o8r2t;CAarP3+Tf!CxuGiO(c(?@)B{&n$XW~Ee@1JP5 znwW_?@HL7Z0gyY>2`>Ko+I$4|-@x=xHarE7`F&GE=&E_Msgni>0!xz*p1)sKXuZZN=8==%H84 zyGA$cIm_yp=n+{~y3P@8%aloITw^!CyN~0-^!o#|Rk&+hZ=3x79dIE~?SjEFj&i@i zkWUCEm!?@jwJ3e${de6D4GlHeUF z?E_mht49$BU`nU7SN?3o>C?}lTK+!LZ<~sF|8+o=5(JPs6FdAx;}xDrJhRD!cv1e-Y24M30AJocuyif=|k?_S!2P){;x2hFg@QZOkcVm;t-U{{+@7A*uEgOz*Cn4zOtzHFVV|#ujA>vKu&IUj4B+bRch*6x1n=G*`CfNXDM)+6{1peg{8l_* z8yxC<4X$&nCsFucb!j;F{>QU;B*|%t$Q1 zxxf0D?c2tbuZ!jh9e2?3nEubfhlx4ue~gmI3Fw+e)^ON^Kg6Q zzXNeR$AS%XsP5dCFXQH(B?~+PhSCJ-qI8i~l}4CD9@lkrTnUU&fcHQB+G!2QkZJ4# zn5&*@JkmZUai7-J)m4@|B;pCD?Ezp#``;L8l%HvQ?7<2RV$oP*pZLFD)F438!|u?B zdp}iIa}l|S`E#}{dY^w;zT@rh89e9@OVVyAI;GQCSy@XQi`~SDEg*3M8 zMOiC^A`zF6h^7fic0$M&6@^ftEFohF5kewsND&c3LK0CCp-3s(`#;YW?)l#L@A1DL z^O%{(-0qg^dVkLQoY!_{-CFYU#S5qR|Fj^&yDEJQq762l{+Y6GH_t+8OOqJ}1Tu_i z{6i+y9QPmn#cBe=s_W)nEt+~tdZQm_VLA!CE1&s$4@Vceo}hN|K^wZ}cj6M5d57l> z+IJJtBRmv!%#C{BsJa(;zgiVq8c4grAsJi8=h>lJAc;$6g)Jbc$!C$|A-tTH96od9 z>)jvvfeA{OEjb#!RH5hm z?fodAk%z(8;jS6&>kU_A{EfD=(I0=T^Yd#E^8G`C7X00!Lp8ushB!H$aXe}^Y0|4B zW&IPf3c?G%y441v`YWl5U*gi3ei-0{r zJJ`3&)!IYvwA0-4bykhM_Z$1OGLY}VUo7&dUWHcF)z#Im)>6`=hI{7Qll0t@lO~@@ z2p&bOoHLa8g3do9Ou$r9h1KG6HsJfaaI_2Sp#G>YwAuOg<*!NZLGC*heW2} zGeAl}aBJecwU0>+PI~gB`nlw8S(^|JY+F^~B9D;6!i50^3Iqg*ON5UoG#{WwVH3jm zQBqRoViHWMAp`QC`^{0)ayyR^C3!^ZbecGkB5`W!OEP;#1Rn>);UghPcyz_1se~nB zx1=Y%oZn~f!48#0SowF!S;J~s7370TGZ^Q+_#)P4`T4v1 zf;0OQAcoiAYSXyCh~+nEE)_FhF(~z8nZV{8k@-HdK%c~-$8fJWn z&Oo|udcUn)&R{0!60b^fRYGCUm3KdUZGXgnJsJ?m2+1Cyl)u6Lr~9J}T3r%p_z%;F zNTfu(2@F&y))p2Pw$st6jWb1HNW3&hrh`h0T$(B?jEq2mBknX~nkM!uZbMl0Xyo^z zxiby%P9cPI8dfTP=i4+TGw&b~#qXW_Y~sQj_cv)}dAoPtKiW&6wS^ufd$X6HWJn$q zmZa6a`zVgIh^7e?+Gp%ybpkk1gtTo}ey#dr+`&acOraN99uhX`>aJ8R@7UqIrAmV4 zfXS+j^eC*_tZ&iLhW8VouxqGS&OC#^RigUx&B-0L3k=Z8Q-TGYp&`F_C-!(RCeU*iMN`3M0 z*h`EQbF^mSzl(G9$EDYwm0Y@T;ShKYg)Lw%ivO8|#-VtsJ&D$MBwj>0#_XUQ?BP*S zKVDi)NsZso1gJu{wgRK zQNUv)I6A>e3I~Vd(d+awdB!jR-|bW6nwL?RW3`rHrM`ukBI?%t`}NTsm>nK_VnCK) zY2nwsQy(j_vyh`UCt(mFU@{iFbCiWTZQZ&ot7448V1eXQ&=)?w!Wm;uE44riWw3!* zsoab>^*LBc^z>EboCQ7kIAh+)#4erwrN2+e5Ty{X7$;J8O8_eY&vT(T86}|{6~4sz zvxYDl`~f9Yosrg(JIszF_vqED?k82-4(^u>GAH9LqDXvWtmNrC<YEU7J9Q|w`h!pr>^#eHiUoMRDWnZDK^ZxI!&2<_ycyH2vI=81P zt!sG1AE?MgIYK6RXZ;ol@sKCno`pApuS2VFSqvY<6x^On56lgHCI(%*)(RD3v)$(1 zNCKaD>C&acmo7DfG+o=+hg91d>j$@gSrUTJoy1GZk#FR8_x=sWIUl;P<=a@Fly@VN z+FkFGbL--qs&$?@9aRDizOD5VWEGUR_SF_^G{!roY~JSmA!%d%{HCqf*!vGYw3?+~ zRAEu2DoZf%z#PM*b5ke9H&k2M1j-4~vY>X^^ERwo7yYZ%R6|qCi+1Yp_t}iiJ|S1H zwiKTeAYczeLl6kqeUUQLIOJw3CrXA%H8r>aG4;rTUEP*rs|{SZHWC^@=Cq`z4icrFlu|z9fU2}9s1Pd)@5`?bzB4X! z=E==A*~p1OgHOQ@M>KT%FImFl)b zo?Eka?YCs|50IUvI3SYSeZGF(M5F{XUb1(ZQy6_@8ukv*bz7UCu-YN3MRmH3vS<4s zVp4eL_0p*^ixhr387ApJ_FXlLRR#J1+9*Y$WAy-FgAPO~*J;{R8y*8vQSkRb3aGTV z;%$Z0PHxjs@#Xv1^dSH|_e?HTmly>e%iit8^ar&yjIu?e@q$oCh4L z7B(+2r_*5FA_1mH!6`u>Kz+q{19E2Jsu@@?^o-70hiOJcLLg}A5=yR4uJG|0v6ney z(~odkE50_YXm|264+RsN5;4nt`wP`tMoljNl2bm~uH7&6gnw46V%VyIv1@)ER3&gS zrMH*-)uoBB;kcams)G)Vx!jpKcAL@~4pP1C8T~?G!;18Qbu0|rdsn7ZASu_#z04tp z@e|O`M9-Wa4#^+$5}e=1#ob`vL3Md3)~%i~O;trjNr*wt%qS2%7VNO`j*Sj0Lr$H# zh4r|HzJ9xtSp}{Fj`Q87tyOdJmUO0p11H;vzHnIERW+!wrXj|YQ6!^ns_zxkJ`qts zdH%71xy#2je#2)VNZ!nf12tB2kxfC*aZP7PMlIT%8@wOE7tqX?fAc7h_jf(8UlMSX zumL*<*sr!J~wk<#e^wA2@uJPu0}M> zZ0UN#v_LVfbCpw@zuqp@Z(D96U994;pL&Y)wKaM<{8cC4CTK}?Zl1r7_i*h;6MdYz zdxfs(iHPH8h^UE){fUMR8U(@qqU=W&;0je9`K$zq+o7XJYhgSO4+}}kf&}JZQh;5~ z`FYULIQJq zkP&RK_IRew)+~g%t1#Q~vXex&|V=p~_1L5x?z0atv@Ye8|aJZj^*< zkYXtN0)8g4Z{1pj;$1b4H$B#7-n^5Jp-EXaJGQ70vO?r@20a?0X+WA!u_qI%=iA}qWkIoG9 z4(!EljFo6*3RxpbpBU^)sS?Q@Zx>LD-IT#lT(k%d%yS^W!L~?{L<|eEExoMf9YDVt zSf|$`iyJN*8n9pqgY5t5TO*%wAnIsJ9?qP+l`YV6l2h(`-W^4|QHk=q?&dw)1 z-`~)ru!=YZ zX3h+@UT)Y5zjGb5QG6Wj^wT%}T%!snA21i}LIi-!aN7DU)FS+CL(SlmnI*=I9oyrL z5s55LhOGor=ws&6B4+HCH-XP$8YR|t}!Kfq9#AQ=m?&ani0dD{N(_esF z@%N%@0{Lny$kzaUeWjAYyd%1o|GZSum2VW)5TC!CHW_33Fl;3u5+gZ2UZP9ut{ zu@UEnGW&1#>|@qBU%tf3e9MRxJ)>t`i{S=~nEJ03Jex>%T{ME+U{?lM_rmnr;0{i^ zVw+DuO;1ASSFo)R8#YAB+P$t_N80J{vBs$<;zt!#)woYcHU*O|%sFt^SHz}4K`1M= z5;1Vf1cN+Rqr64bp+i$2n>6fGYG0W3@T{grPH}dF3>)Xe7iW(_X{+&m^t^%nid*0P z`+dW#m7LQVLPwje={@7w5ojZ{xha15H>NNtqik|72`T8|8a#&nVxVrm}};gJPs zW{EV!ZW1%??C>!3oJp=8kcX8?==ZPBVhC~^6hO`!#^ILb)RJukMIc{Fr-tbn4+9HA zp2UmZ-TB{|#xQ7O@{k7&9eUgQ&M|5Z)B{^fI}^QQ(38R>F*m6m6$9w$b^1HwH$&p6 z*|qCDyVp;CvR7= zR_qB;=SHWddf2&FbjY(tns@x#_@r zvbD}CZjB+uw#0R_4X&2==Pv#?qh9Td*LOh>V%Nc^@T9+}^&`?V$#ZNipEi zPD#DK3SfTMCt4zDTfcJmD^)P+jYrub2NAV`gkK|<2yus2!c2*j$fthh4KO7Fe%8up zN6Ls*8Gwdp`W+olMK3jIK{863EHH|279f!*?OfNo+hp0y@sN;h^6vT1MTvFz;_zAybp9W;qL1r>eop37J1_w|+kXoF-sx53Eoj>Ghp-`T&y&9z}WW#r*BIC!7taN60; zhACMAhWkew^z92xHlfe+F~^?7ZCHZo6}@m?)h3#nePMd(&IXg_>A3ZclYkD8@?u<} z@Ay&kw^m_K6@-*>)l*|49M@GTUDq`r|nD;Q2U$g&A3tH|Ta(Q*x98 zHp#uRbAX)Zrn|Q~yn)+|L6{?Xl`1MK!wU#w(*l|o&ijdrF5HHw7V!NCtZIX+`L^K6 z%0ekSIQUVT@hvP0v_O7@NB`$EG-){#C3TYFuP~M$ECO=!n581 zNciX3e);NF17X*PFJIm}Hs15~MRB%a;Su%y)lZk~1h#oRihra$*SAs{wH>eFsw5}$-)~s)%u_YPo~w=CUM64A*5G0ov^{(F97(r}ueD5d z;TN|Sv_WVI$&(w=8xj{!ImEz6lw=WAtE(k+qV*6-t=xhj`^^H6RU$<24Y{UO<*AZa zkeWQelp6$JT5ZVfyF+Mz*yP_MM;grrM?MbLTE?(|Y~~$s0xU)wva=mmxcG+cnu|3f zBVz{Fr1=HP$Mp1@>m^cY_h#={^Vzd?Bu0lK6)>I5PbF)%eY^b}^`LPdeO8nwrlwZd zPJlxgX-OW>Ao>l!=u>((sG zAKFbhj?#9EzGl^RJ~C1HLJdpI-BU(db zd_V)iwwz0O$A7$b+mdu~&QqqLbTW!}4VYZD3TIb=Zf%EoFyf3ONm)elkOa|18VWA6 z_WC==GhlrGA$c}PE9zNv*Z_jX{vMF_V1X2bKsl;eJYU1DcJ8E^%^Zh36ioN+;LO$4 zOWG_?TmGcF{?KDhTYJenu6-u=!d?@;Y(j@VI^#VzsYa#+yJ`jn{8?pp>r-m2J&m`b zCI+LJ+3*2XS>C7(Oa6W^>-?olQP12^EYWpHnDz3U*E_}85T0%aE{56~dgk!=Osb9w zRk)4u3W={6-)$Gi8fOZeST~kmvzlRn#qV4zgC6J+#Ustlymh{F-ey`ANrUDoUQ)fm zo<^1C#agt=Ofq-yjOJ6%3^tiR4%Mef2z3Cz9oeeI4-!)d`y1 zxnswgjA@UrP$6j_+Fa-%s^t}IEasNSTBs{D1+vR_B8NyQ#P~Qf>C7pclhFoJFn+%B z=W+09*tAH5VGI+AB)WHUYwt(EBap?|scY9E+!V5lYsU_QFRnSvEgRh2pLeMO*o)b} zp84Ar!gT;*QAZ@bmqMVMiR7S}FowK)K#p+(UVD3|AhD@OLS>K9fySNPt~aa#<0~bW zE=jYQOzPOKRcE5wiZrAMJ~q z{J(5&Y}7QHnObG>1qMo4y+d1LCGlTKSh_F`L9tuP7Bgg2_ugGM2Y4iuB{tH)J9Z6f zW1^=g*#-mzNaNoKr-gJM?j5PojC)7MbYfNN%%qDGCr%V|gnpBRuyKjKSEroNRq3bD zX&;m{v-RR>6>mpp+^W^CPxQp4t)xosIe1M`TOgsJ14SV_y}yq;e8o-L?4U(@v6EE? z7Tj{*y+*qM!(f^+s6xEpR*_LrZF$-&wX}~t+B9#zBJ@Yj=g&i#M9H-b-orv_qG1e< zAnA#ZU--1#j_&g&^E$0BDBAMmKhj*dq$N3D)=n>6*k=M4c;(AC1EN5dP7+Uz5CYVb zoh6H>58tzu1#^Ug(p#5xwVAE%|&?`Bg9$3r7md}D^^@q`ky2Bq`Pff&m9qx zQ6794O@(8$;}bIhXi-d8@Ol9V9h<3#hB}uT#Y({#Q477BxEccZvlG;j#MwJ zN3O~sB!>*OMRd42Z&HVfS9A-x`#E(ZcPXS+_@3=#UwIyb?#;<252 zQ0}Qyp$&EBx~4U1z0TYHf@y81S*uM$_F|*tvTI|Yy~lxw`|8|` z&_XsB&)Nek{(_1`TRJ#`uN8zJ(B|Y{C#0H4Za-RJbW!;eMJGknL-?A9@su}B9i3qx zw8znbXnn^ax)ikdezB%maj(+Gblng$TsLuY&sgc*I`_H`=eEoO)h(qN4tDEYBnB~X zXN0wOr&Fh;=IrhWMdZTG7%>OTdz34CTrawpzv%tz(d3__>_oFadbAnrnBe_**rlFg z{?g5CprRu999WOcP-vnt!#!}f1R)u>p<$Cmk)*BA=7FQHq88UiJ)mGyS#e3p%*^MG zeM+&}iK|tBFs`w0E{3!oBhZ_W-2rAx@~BRK+4%62hU4*Ddl0J%iYL13Zf+*6sz|gC zjUw!^wh7;+Qa{xUCf>g_jTS43Vo;108P`k*JG02uhMuNoB4FuoNzPYUss_&TV&S#TmtHWhy3+ zu6laITwSjqh#=G7lpFDM`ZHV^tlGhqAqMA(u)8Sz^0RiH&re@EY0HP^3WIvBMyHJ* z)o??rL5q)X`M@Ym<9DOezW6Weq%dxCf^sg8k91ZsEL8LL{xI~~&U&RU*G+M-Hpg{j zKf%Kz$`KYWiDe;}GU=DG6bUUC5Ehoxse_KDhDYFh>m@8D$j`raxu`SI<)Y;oK0K{@ zv;UmfG$+F`#G5hXTy>1CS$ZulpJ`ZnHT&6GTOJdAfJrDq#j=2onPw5BtVYKrAY0wZ z>o*rOW!Acvgp35l^x6}qg7OlNik$ERSH<)pwq#|UU4h46+!N>Rxj~q{ zy3=0#=jGdkMjp`}HTBw3{k^nClEy$IsnxDs^e7QG+h^l9M*)54y#1A>ojGwG^|#GF zXB&EXWl>Lh=yWGTGh>~eJsYELUlTWDF}BR0xcgWmD~Gy!-tj%Pb##7BbB`mhBWQq@ z9%M5Ttt40@meM?9;AK*jqx=sjJ`v58Pj^8)+Y(eh2~%K?VWj^?Ce-b4_P1+~($$xU_l&<0@pJwGa)WIFK>-qAB~g%Dp4_=`0gCaO9U-89 zy@Jjj{n^DMKY&-IkG(~AV%BPIDnGWMq~xK=6vKs$tJUZOi(}TglpA$R%WQwxz!_Rg z8hfyRk&dnSnzrV?5$L{<65=(cH_G!+r&40)ks*yHXB&!zJjcnvCH$I1r%JF#kM@%j zkbjREGp5~1QYK_47lH+sqfeh&6PPU&F%{e@l96um%G{b_6;{e%h8yH!eUj*F#9=M| z*T8m3KuN-5Ni(dyE7_Lx%Y`HAb z$#6~4)E6?S@v5=%>9pXZI>*={#awl!e-oEl+ym&V#Gpj|wTfhh7M*!k z(b3Xz!o{Num~eOtBr|gaBofoYR09n@TKY`0&+$VQ1~2%fWsRz=cbGCIZGEQ5J&=dO zyuYp0`*-n-nt}|S*vgyupN6G0w`CtCk09u0#)Uq44zw%Sq;BBMCe*bS_hFHTqHh~oCl*p6gL{(kTM`5r=b^=Ja4fXy0Sw2Rfc5$E#3O_4eA3AUb z8moaY66##qle5Y^LjDueP$!Q)hMjSzZ+!;43HLN)_;Bw^9GKey>!r85jc-FN}>V4%G$YUznFGb znRdy=RAj7`FprK@uIAj^%_iS{ud;7|TT>u5rMq=$;D``wTBq6`MfWGqND7Z9OPB)9 zCt1g0n7R*50->+Iq%TewfOnMYX>#5+RE}*=-U%J`I&s*#p-o`yFI^F5u;lrLk8abp~m9y)b)J9b^uUmZi{`iJ(b6Uu`$zNPA zbNG{v(xDDNzUp1;<2kKei`e++zd0_xe$InBjaz%({R|!;;$yM5*9Ynx_73#DY-Wr^ zUgo=&U0$gVD6xg>Q}pHd1H!#@Hf&;-DbfTK8;0G)uBLo_=-8!87bo)4=o2@uFs27& z_IQ8P&mcrp{s1t@c6KIWgqH<8=)S${es6_5f}%QjR{|jf73HZFZVPM%8$1;KDV|#i zF*iC2@=NqRi*p>UQV&#e!82B=O;7k)K@G+-{HIrxRFn@)u2T5Z15;_9TuK4JX=`np zwX|PpS!%@qx2H@?R#z$81mScHr8PT)9ZO=@f4_#;V8Vg+JIHqA;D#QCoCw6#tom*Z zk+ta0!OXJ%@brAOtjM`K96aYWOe3Fd<6s0%3G;wh36<0Ki|QN0ydbPJJO7 zKnYH{c)PD>nxmkOx4}6mU~a`I-)-E>!cm0V0&mHI2hOWwK2&`u(6B_T8Luj~w!u1Z zBgdn{O$x8MnIi0yY~h07JU{wpQ=#4X*#AmA7L{jKlpY`#>9H$644Ed)c6+CU0`(FN z6ms6C%g(}Y2P(ShC+Kxua5nwe6?GBKS1;0%YQU4(u4No%GImf{_PX5UAqR6 ztKISuW;4UIt=_SANX(hVX+r9G#pK@nM%fpM7iFyPQAr$8zV_@!mi`q}Lpo2sR-2@( z0wH}kyeTa=JG7#uzQH5Ch!iy91V=98NFs)-3dBEhS2wlYjR3I6N4oYd|9&ZI#09iY z5{WL*3^1+x2x6bTP7y3Vv;cfqIiD~MmP9iWEYG(A1t z3RC-a2Bk+UemIV~k4xZ)Ew?D#znGQt_Qea^ zUnH6`=43~IeDZRNFWkDNK#*C_Tw&k>s{TtuhfBvA42P1rat*q?B_54~k_>=yxxzue z>s!s|ii7TmMt-vn_GryV<0sN>kW)qKWyrIHgf$>&YT`*L?B6N!5xu+Y8^?os?-Ngw zM64;jB3eO7>{8l7q}iNF_Vx|DVtRD`@m^*p6s|%6&{6Hbf9|RO8vC$-0qT8A1FI#h zuzK;>>%b=^gV47mXEdl8sq*a)<~EJdniBVd^(?hWiqApJlM5EhZ$CVtRR&+=z|$isSUM{eY+i1|H1zL z&5#6S zDJ>a^6UY9!C~s&%D<`Vb7X#FYjo9g`sk|BnxM@1rjlfzjhgd9A=zyh_t(>bYDMNUP z5?+>s?RBO(koBj^vV4}i&T+s>r{@1@1k^^E(7vIMy~PhKtI?SF;TyNCSitjzt!NQ0 z9uAX`z1iz^G=s+WUAgk6Vt*HP#mEVM8nJ-Bvh+XaVuMa z+$%~_B^{XEHM<5>PbQaw)Xy|nccEAiFP_;4=Zc==%BiM}8?Qyfn>V}vw!N}e1297= zzM&%*DoJJ`esj5?tHA*qGhg6C7&>Q8h;xvX$o(%_DR8&Ik^g?L;-zKcbP zPLF7aYW@i;dK(&g!@nIoeR{M_!#Z_DG!1>!sZ$HJ$kSl4a2~d{6GLranUoTVj-wHc z%Mt$B!$LDB|0qoV@rQTprtpg<`%YxR5qQ@y`s-zNy_&#NC4VTT3wI4ityP|5tIkb^ zCzPb^S>Mv2vDyItQ$N<2$G!Im3Sdi(WW+%v@f;ty@?e&Kt)+-K2_699i7-A`)RTiO z0UW^g!F!Ol(Zs3mlDDbDYV)AuV2O`8R>R!1_L1+X$Oc@F}pf=9&Txq;QG!yD6M6b1Oy zJLDT<(m!P%sR4r~sXSN$vKEg9vy<6|WqQ@gJ{=4|yZQ)?nsp8)_9aYbg3*NKsiQDIzlM68Ds2g_mcUC-1e-{1o7dxzwx&;S@Ym~*tY}RZ`BC} zlTaIrVQ(J@M#gAS@^|gh(r-@au2HTuRp*Y}38v4xjmUJwi2<(p!|&!qyan(WJa+6h zkvQ$$5x*Y5`R=oroX(AWYyUN=y5ZO-TJ#5Xor5mc?x)vrTJ>u~Nd|#CSaf&f(kgPJ zYfM;tyy5AIj!qjJ024ImROQIJVuZH_hvxiGDqi&~fvQ=bzI<^H{n184<0k!eE_Uz< zD5=K$wycvIQfoFURpi(*C!=F>NjKuQW?j^Hpd0MuDb0Q-hd5Mg2{>zSFN>gurQ9t_ zHuK}p`G2`Fg@>rJRiNxG!upha7JLZMQKTf;uZCaejS!X%gYee5P)ouiK;*5Tp&8_{ z48;f!Mk_jR-8)Py7#kweA2oOR6@hah(3|TJW?;f(11jh zh47RaTT&pzz7xuFkeWw%;~Fn4%-R1dt0}#gB<1~j<@ZA0SFqH1`SQcS!Iu`Mx=*cm z&l4_+K0tdB-tfzz*&xx7-4^1rNcz0G!E(3NPN%d!GKx zdw*P-(N5Ed$BA~gAt;;x64uN6?KgBUHAI33e|6cTZbDb9{}|3xi8He*qYOy<4SA%c zzkhZ;im4!d6xZRsY3lKGDpSZrpuW_=p7BZ&<7Y;#gm{pYGY$2G8ck>##BD&+;D>RU zl)Hyt%i`!BA0;OneSGhsQsHhwv#`yXM^FMWN{jq;{ zq^0y|4yAdO;bPuqO+SR*+&pOEj^z!!0kt}$@b82=D(ytrOOey?nMFSqZV&;bvOm|D2<4qd$XeSKy~;#`?|aj&n` z-&a{{>AHIh^9SLgw=%6a&O(~j^`K;X{mEIQwOxa88xLL`R3?eU>q;EnwURGabv?(w znhE&8V6Q})`=cJjSNLGmRqq-fb8_y;0x4VDPm6*1r|`#E3-$NQK50h|M9vkhnCYK? zJ|V*r%kQC+Cr3V0Pk;BWV}U}+Xg%6J0-+X?`y&;pFFrd9qA!v8`!;)f-#y1JljJMV z)|~&8rMnn7L$_^M^TM%@eaUaEPP7(9B_$(0v)c}EU5C*QzSVcd0?rfl&kJ#aCch)S zF2DYiEyikUkr&hBZ5FS{FEn7WjRYtmwvGk#$0VoO|I5;T7(&lcw^hQEvVfDSF(me`lOKf!)_;R=qSN#*!M=f$Umm;jz1_-c& z2;iKv>1X8<<1TaQ?)9MwExd^H!~yA;vlXff_+mPx7fsDkE_zue!e-C_7X8eZgAScm zST=9_62n%K)AzY`Rbl5P=Z-WlZMuUp8z5YG&VXm+;K-~%M~7RTet=3qJ&=%<%w4Gk zcl^_Ts4zbvc#qDCH_$y_kSrR6TCIcp8zrph@u5nC0UvKZY++g_H@#FD4P89WKB&49 znBCO9r^DFSM(rrZ+nhG3POEiQZIsRb#4`Om?8wa0G9Ifs+EHczrnhz{YuBtf!joiD zsFdeBMrO1ZV26(n@c{#AKA{VRh0+P34oUZJ`TTdJM#5blbV<`fw5PU(FESMrmCt0! zLj9n+-GCPOF@7hvVTkxoa$D->2pVzBYEpugiL8J~(xfejA7YS9bd`iT1Ni9Pp{}zC zhdZo8yB3jmhYwJrMv@vKhO=?Ym;d1R;7)^o`n1^6fH)!`WX3lnWZ6CkQ!P#=PNUAC z4)C9g2muY-no8JfHo|3KQYS-#2tEAJk9aB-ynhg$LLdlJC@-1E;7;LOf_xB?)`3b4fAMwdVFUNdfLblWwqB^Oi4(i_m;73xzUagi)26-IJ#$4~ zhL59@@z-xl_Q-mz1Glv_YSXJ>zpg>xABeh+dfJBN2L_mj+ z$sMZ*;<47#v#9iOa~USVi)VgQW7_e0{(L<7sg_fEHjd*8fOfcpaDC=75wT**!2UO; zr=|vuJvHthzJt|geoQW$ZJcs`PlcJ8=>0|B?{47{-*$^Kx`~IU)ZOxCfl5E8uOoCE zl~bC-K2n^p0_(iq(s1p8JwHZBf&MVN9t~V-JNft6v(>lsAN!LkV$4c}RRY**jbA8!kphV5w_co6lBErDev zMa0rFGis3%evsBBDGeJ3+72Ng!)qTrAe{o~_IwYtv zm5lKfI7qfQ#-so+m+tlWw(#=QX504iEoIu_Gj0BH+h+2 zy0;jY8-1BPGSo&3e}EE&xKR?Ee*%#1>w~_jcjt;0rEeq+g2qf`QF@Ym>Qpyn^qBnE zXUl&8$RX|B_Kc0LZ6sw#gy_FsK|T(IKcO-jinX^})BAdXS;TuYhH($o{g^68Q*1>Q zd3qI^C(Dv5iYF%@8)zyL3Gq@Qm#+BeneE=BA{3d>kqOBN-2@MUUfH-6i;BDXD9b?4Aalz#^8dKQW= zPvJ7jnXXwNN0~@)GEuant68VwoT|6)@?EG+^?R@!oJmi z6<)oGj*o(l&d%`Cx9`tkMYjKl>^USoq{zduyhQF8O} zup+J*W(;P(2HNn3gTWOeQiI4MyU8;vO@buD7Vv(GomWnVSb?5d@TSVs&9W8EM4WVz zs!gvO9Xi2V`!Ii4R<2>2*S*662%*qHhpNUood3A=ikh1y9NN@Ad&z5xZ}{H5$$v%r z6%01c@||K?!*>QbuY?s15*ub$UENsnVw8+no-y!Hc)Vkbn=KEscFX;vb*?tZE7foQ z>!|Y2pmFaQ4x#A;SUpbM{M4!KSpYia($xkj;0rNKL6UdQTdOKE#w}aC(3RzAI=ysQw#S6Wt9SDcb{DIgHUUkP#_ae?qa+T1#FR3!;j zzjcH^5pdGPu{hzRn~Td-fpkGFcWm}Hn}|(af+Cl(wX?Xmn81E5-5u>>Jht@R`=EKN zR{NIr8ceVx96>PoA@vns3^JTFFNL=p)hEfmu4YhX@8s;eq1mx7pH#=`-k4AVK=O+jnkr9hq(7N`?fGwA}&pOMlc`@O%iP zzrW4NouU678hZ4MMg5SYA&VqCo{9N zwIDCz-eINbR3?Ea7fVmtI%bahqAuX3PFUzV86z#(I|$SYzg*4HH#F>&$oTxVDD%K{ zG&809%24J9ASrn&GRGI5mOT27Zquf)zt>cq%*#36EGRJ(8 zD7(*^ZXzO&Tu8NuSDK!?`u6lT_Len(rKN&`hRnM35|3^-)lAnur|!?aASD{@(ytOt zxBfdex>G0b)QR3^C~OAut4bk~lK%A?0vkBWY2?7P+N@-uWo*yy%B~i%bpdB-*a#V=VJgJQx zd&EKP0isyivat_3TGo)9=Xx;OuFw%Z@}PYJJy*E;b9hC0lcX!wy#v; zQgeTTEdU2n(r~eF%Y5u$APM`TSOo7l92GUW-+Rs!U!MQ7Cd3@sfl;64$pp>&5N^x9 zAVi8cQ**twS7Yn{8OZAePWp+NL6j0CJjZ3~dE2kytrOgVTTeK*6SB!#ROfiAJaqlS zQH*g2>@Rv)5NQbq#d?}<`Yj^FARrom38O-+-z<+RX)O&`ux8NNH(Xp4qU2R~Kms~+ z=uij4Itp-=TAtaq^L`0K7Du5;OLi@ctf{#?;#G=Q|M_JS-T1+1xowHs>4L$wyKQ8B z1X$272+cL3aL~dJy}O+M{!-Aj#VsSZ)XQD-8c9wvQm#gMXOgRdPO=%G7#F_lwCUH zQizvCSxu>uWWf1hy5j`rfjTUc9bm(TdEGyY;+C2WW*#xHxvv{>hMz1~*rV|$!~K-r zn@jl$#In8+bE?8)%Nxh1=4)9q>}6Z>_^X^SjU^*xdLjBs^S zRAy*3N~i-cqEy}xD}(_7@o;9(qJn;8P zA&0sS8R@qfUMKlcY2+?ka8>s;-sFlBkBTtIZ;^-%#nif>s8|vB4~$zS{S-TV%Ov^< zR=BB_wy-$-W`Y1{+X{wm9{Q6aULgWQX#*eI9vYLDL$HEKlXd@~c6u4H?3WJq$TEz% zLY*i@47YygHvC=$@vj5w(32icOwUlo?40|$DHC)+i;js?MQ^oS#0?q^%Vk#k@VP8$ zsvnkc-P$AFcH<)xCQZ^UtLDW^ULFmUnB`4+5q5EpdzNct5y=K>2O=!M_?X_(-hRs) zaGO^AdeQrq#x8jZo5@TE6kxONz0@Ks5&H;zCJLBIp*BhpkDc8<6fo+WZcn?x!!pS| zlkSN1%|QpP1GD%KvTpiR(>87VwwM$v!_2y9bOkQE1D6Jl9WxTT`g z$yJ!;NqSY=KJ17*Pdq1&kxZrMRFvoFBwlz*DJ%Upu;SdQ?+41Cc&tTKxWhXo z+0RrympQL2I$nL1v)^`|pvm#Wf}?Bq@@#Z!taP!ws;#Kp)T0QSwZOYp(mys!RSXpr zRE;I3ug(9doBkDw>RVpk+Dd{+w2Mg9fqA(I%~p>87(g2@Axz3$0KY?+T6)fT{E6-g zmoA=cm8-AHZ+?>%OsHE}`f;^K>z|br1P8Iqj3OGIi>q?B~#NL{M)+Ja+Znp%U&3B5o${lBg;%zLSg?TbiZra)N71h$Z$|zuRqg z_y&5&h&9c8PY{(yYGc*AB<4l_Gwa)K<()4i+RK-2))kHh7ey$Dw$8kzD%U!Am1eJA ze>(M~FvO$0^g*xkJArB2_?W)77ye5NmN0`Ie%rQhUkil5wfTpr5h$3LhwL7l$f4Mnj`3PF!e(-7IJqsVibll4f!e z2`AXVLZ=%udQK&Q2@R&QgtmdwlIE4NE7yjb+tuts$|5RoFH8%Hp+n2wxQQD9LTFo} zy#Ks~ey$5N7jNOhk&s-8l|i6n`1(0Ah^3A>Aqjt(f_ z9Gv98b?a&jA6R@wT!3g0I0!9@fAkE47t;E{a|aXys6n50Xh*2bpX8}Pv68i{RD4@l zY`#2ZtxA+VbP}PqeiUU>pAhh19!hgyQu zDrX%(Yv=T`!a*YDUL-Z)?t@>BoNNUOh0O@6YBWC`&^3dX#gMBpj6f#?& zr}(cpJX*ixgQt^Tm0rQ2!#Jyyb-`pR{p^ZCgH})6VA0s;5%wZ98_I6dG&}PFADMhG zWID8b^6U7jxirN^z#$dw2gPZr(P<6=QG}A|7?R;RND+iROKvnA2I!{4v&VA|Xj`Gj zUWNO(*w{4v8Iq`sUXnbEYaT=My9&1;CV8c#5#im8*|W8j!9+>l2%CIUZt4_66X>+_^`|B8j1y@Y3aZ4E_InmQZ)L4`sPEXKcM+- z*~c-Ow`0+l6eC)cNALC_v=OXDV$7+#I*F?%(H1u4p^^l}W25&EMslF!)kMUNPELg% zq2;21>ku1Y+A+0}{7;(C174bYKp|U`LM&k)4dhgR0`dl$09^%uKNL-RSPWssXC&4R z-GQJ?(uRpW#3Qx!O!ia^VI&px!3)GYYCMdR;R22$$`lwr-2$qMKrGc7t8J^p5Gc2M zvB0+HX8A@dra>vnwrm;`E8dd@R+9$)h3ueiUDJ^xH(Ae{KYy7^;68`*M*BvWj(_F* z%tYesGTPe~XBVEm(sJUlnuFrzWl7ET-}nnu`1HQUJg0MU8?;+4Y}9C5X1$~n`u(3z zT&luvN}D27qJS|c(9Q^IrBpZ~l>%>qK!ak#!-kJ@BaiXO-f-~APP#a-n7WQ9ei~b{ zL@i=3$S=|y?pga4euEE*(uylg;{5*=B@%dyOXL5&W=hR>IA)lxUF*YE@Ih^FjS<`v z^(RuD>I-wuM%ETh9}=GASu1IE8NQ==_ha+tsk>~atf5$y?8HS%31p>S)$ek9?QQ)d zPwBi#zVFy)i>sxAi3jIz{sBfW6C^UYdWPFdupcUG+F7My!KDG<6*@+=qa!Y`{SO5~ zU6Gh~*f9E-|Nj6A2FZ} zF`?Wf#U-n!Lhmd{8V$DCCQcA(DEsb__O+Lc;iKQWNod2x`7x6#ipNQ$W&tShR0bB# zquOpi_mgmgZmh=&`TymJ&@BF&Ih-G*2UrZRH2O^pxEi@8 z8^AwBCyopYvXtG9C}FD-@8v;`BZ56qcY9H(<&N~P;#E#FM;|I@lfo7Z#j+iPpb`iA&8 zoFX}yTWJUlDPJ>uI_dIqaHZR7SS?}Xsfm{wz)?~Ww$24!FkJefk#qL*szlr)Y^Ynm zzSvZ9%YLMeyg)#(I4dAW(sQ<|KV=A6gzzoEm8($}0MSJF7*Gbm{v_5%xwg{4zq(vk8omGCJ9Ten znwFZc9v=p`v}DDL`@Wb-PE6L)8cZilh-4`uM?~y@#CFM&Ufj}L$=7<6Tv4d+k4cZl z=|Y^rPPLqxI{)%ssg*g4Fa67x4VAw1ix;muTPpDNbCJv(pa$_~rMK>PsL0c~-`D## zp;Xby;;A&Mq<_nOh_&(&%AK);v3GiAtGByV2UZF{evVr8Q4O)%*c|4aK!LG_~tyi z(?I;h2t>4PLQ;}W|6{akfUZMmp#*aUgn&jh_2J+TqszNPsH;q47{m$Gg5(Cn3`KhBV-DcH{u?!{MT>7+> z)ytIvYVGE{L8n5l7hYMJabpIFxf7mQw{F>j^kaT4kO2j62Y5wrF**jCoyzT`l@i#^ z|IZ{Rr3XDV^+0ONITrL9#_pxfImA(h2De|nlw1sl}?dA%peBJFuu#C9HFMCv! zd*4AGe3s}-Mve$0q#d@L+B>z|u3XLI95eTVc>$8`($5P$#1jdxU@~j+K=#_mq6q+l zxqM;@Iuisece1k@kFut4NJ2dV6+6E**9|$2f9~G`f97V2QGl)i-~j5 z?Ts=LX+Twx;rxD?y=N^I=M5)??7JXbS3(lg(hlB_fbHYMMGPoXIOz_Ejd3#UgDbGt z$`v2JrUx0gur6;Z+h4>GjO>n0B&nGmvh#Vq@!>k5ptBAy1H`Vu;dK zBr@|}gujJfSyq(81;+V2qqA2TEe^0wA0b=40|Gs2td`VtaPk|_KJ|UgwDdbB6cSl} zW>tx^?3AZ8@BD;1ZUfP}lE`-Sn=xHF+sCRi#vge>Pa=aUVjvE2m}D&o*?~@VMuGSw zDL5&qs~sBerFQ_!KC%qI68aDC_>nH*s3!-7SevKn0F@w_(dWm$5tlN%aKJinLAb^8 z%?HP3I`vz0>PpLJ?InnmoYYf826{xf=;&54FERVb5_H^uQPV3eOpG zIfMArsaJ2YhrZ?-`={z$GBuUC=gqIXl-%cS6K=fE^y=={+m@Y@*mI&H-CJ`kN&1ki zQmat<3NmR}%FHCZM5SU9VbWC?@jwPE=er(O10xdWid=5>zvDs_db1hrQ7!)`*L=;$ zfp2>O48tnlk;PI_vq&J$9@|Y2A&E1(1HQy~ZF-z>7=I=KR-*rubaLK?Ne|~ZI^L2U z?Wn)x>rrT{Z3o#!{#IvYY3?0Ewz_gbx#y;!I>WJZRj0XH-QG-)2)yKMJ71l2VjTR0 zY%-@6yJK%1G`SH$0n1ZYHJj{S)Di6DNHO?WBXsP_q}$91)Ceiea;C$hvcyV(q$s`M zP2GdsyQ%Iaq1lv*;_#9uVJgj- zU?|o8j7a+GR)<;!SUz~ll!FHq86l{2mD^E(aeej#nx>q2e&*J3^~!o2+kAPpfUJ8C z?y^ig6ttwu=mb+>^flF@!$YohHdUyT1w9DKK0>V-zRxRHu;2^|AKy)@!R3;Ugw4p* zbmGMJRj7~0D810a|>?QM`j2mf|zxQaY=KCiorDQuM7H-xaK+5+zj-afOka=#@DBt@= zG+-+T$3pI#OT0*Qe7xl#E3m#p6ixP#Fk+@dQEHhic~8tcT}joC-otlLAcgfcbV^&D z{p~8tD8Y4e(W(3ww!QAuvmb}8XdnGBe(*7h(YlcZ2L3p zjAuv&PK#ktM;N*o;_t9C;Zz_PqH4=NUo{4*nbPf%e;mHbh@d8c)G8?l*~%{ArPZEX zKER;EGchx2FJq!6n(!%I@(eBvkm*eJLik(QA8tYiEDKd}`*6pp{4%mdjV4IXYRIZF zPao?b-$_{5NAI87hfGSF?DRI?RGH8ui7m=n+pph@m_0QH?Y#A*XMjJJriBe1)ko?iLN-#Q%)Rqjl2YT*>A#r3<`j(W`V%KiA> ztd^pUCWMu|aRjBzY?#^*b0o9N^I`63dcM?KM=yu~3%90bVy zxfmw-)vR4xi{KAuj{tA`e`sMqy!hp}3VMybpi=O8__6vTMC&2t_$z^*of2xuFcKfdkJGT<;0DUI*L5X^zxwo3HVd|6>;3&w_u`o+><(s> zTzy||6H%sfQ)mTFMwAT)3RL#f5d5gDBD-w%n2A5G>y<}!Y`?M5U#r(u{nO&CZ~fp>RYdlh{eEAs*$a%eW31M0J78=lm z=iuu+%Z-B%rO$;PkwtO@zODgGKNuW5)F5YZge>cji*v$+2}%QAWMoHL6oY=?G&g|@ zfH@W{5%R4hwZc!*W)X_M)u^wSu0Rea`+~{B@cH8DNTRQ7QlM^A`0J`SR$YiDAExX! z_v*Smz)A4~D>joCvN39X?F2pmj7D9rBt@b;6|*?sr6XO*&uonwv(EW^l1NrmctxxH zy=%a-(A_tSWk^A6G=0Iq0QN>>Ppb14(BQJ&=^o$QJNw7U78_jUPic2VPcgNN#;9(x z3e)*U$7PM+I_#?WB{`>m^SLp`B`(+xv5G=5WZlWH0+sqssjvChipQ_(Z`vQA=IxMS z}^29Ph_4*KYGI1_)p$)V~buaL@ zspy4x9Gd4RX=!|+mlA#!r2Denmq=%6+(pWR%)qo?P{I&x|Mp}sp(T(d&+;3>s$+E> zLY5irh1cj~Ezb>5W5(g5kD-lRCtgtI-D(m0Hr1q8iwq-NiX$|rIVyq?681E_ADLDd z?bTxm@45MZdXpSoo_s;=0DCETvhe8BpDQPbLpCqZ>Eq*$V4*d^;1N#bIZcD>^NyR* z*tfUFgWbuBov-~Yw1=3R*W-4VV=E?C(vOyI=6Ep_k3lPsK!NAZx2#a1$YDFR=s3gn zp4 zET7>WFnt+-rp6!{k`p>sa}G?|rdX~qQnaysURL-RTOLk0hzqItkEmyZ@=x38HXA)K z%PinQ=Vj#m9=_r@dY}6ZVWR0x_FCD74FZQgdMXvJxG4`$H)|d&N7|; zd&!DgLK;owZey38+LnQv(I-QL9Xfyh>HJ=QR-yP{_s2ci8I-^aOk55ttk&8L!Z$82 z)Hspg(7}AL)_ObLVL$^CB#lTt4UiJQXV7$^@d+LfTP5tfL}Ju(S11-9Vo?bX_3D`P zMu{vqzjF1qTNl|sPrCr?(R`r$HJ#~=w<$>B&J{%(atH>1!IAvGBuaxHq{pH+3aPl0 zd;YD$n}yBd4hL-A73M&|Ua>Ep?qM=qc5)IpD@=V*j)4rNBE*1GGT-fIPiAl|z38iF zX_n^ZH=)K0Q>&{*)J4cjz8fRoIzml~c2BIYPglov!eI2rT-s3)JL>P8T@bZPD59~A zHRj*5bsL+Qtg}o-Q{(N4ekEg`z1S3@F4ZGC!FcvA1BiQ4H-vm#Bq@)Iv8y1^Py*Rm7DLkFTBSth{{C(gGcEG zh;014X@Jeickf$hZKhr0td1Q&-Y`v5M~^{Zs`QlXVN#7(9#;9A=iPUU`9S~3j6bk^ z;^iuZWp>%uT)xR_NL>>P12f;zXZ8$t0ebFvu1mp)IAOWU_rjA1L3)QSSRk>mtdQ9E7_LVl|AOC(C zSh2ioI#jXv|9;|c31-Q9^O)FZL1ptPQxd&>%zQ1L0S#C3KMfkFs_w)ZX8;~3>NS`m zC*)*kfG)S5&aNmnAtB$ILM{n@hQNjx<8NNJ(`AXzy&Al+XwR&Nn5~dSCf^)>O$T{m3?pd?DQnrq@g^gK&FyyaJ{13Eg^G0 zaGGrJW^Li{-3yjV`VIT%g=9X&^6*6;9<`T2(DTY@mB>gCCSkN$3Sn)@+FRzJ!u{L5 z?K>M8O*pSTjIoaZtKRRwXCa%61sGhn2qF+iYx}91@t-^%InZ`ff^{q7r#ic6W*u%a7PnA1=K9o`2PkE1hdneCXSxuX?Sp6>HQ;(!6pcL16 zlmtroRZhu1e6jET6#avdk!7KQFF!7==ka~!+zT(}d+Pd{#k_wp^(ZKiWV|g})Fa!u zr0W*RIOqBUy(=>MEtxd9i#nFnO&=QYs+qu;*nT`2vg2!MX~vOL!`9qtkU`AljUF&m zG$eEbcxSa$NUQTn|J=JLNqqmj_eeRlFCf5c;Qz!` zQ<8ld42fiDLdjksRFovyGWJfE3`tB9l8UknIg}8QP_jmesP6kY#ee?y-t+puuDSoe zX*uWnd%n-}*&deFp0^noE^)D*n}1p~G8sM)nu6>6Ues_{ z)G`aLHwj1Dt6ou-o`wexNMggrEQ=ASl6l(*#jklcqJh%hl;7kh z6Iz)zMVMfKXOW2(6&2M@xWa__M6YsT{pD&Rz^{(oWsj zl^Ub%Uz^Q5G9@l2fJ~WohZ_ZX{d9UQokK0A0^Z?83gj`@jG>N5#x+k_7|4_)=aij#l?C26RFjVV?-&|$mJc?xU>c{9U* zbeB%?bnrIQR+)w9ztG&X-f~!ge{CN+Cj9c;9$h^?%nZv$zCQP#;u%{ybM5cC@o7TZ zV+I&*O4W^%ymhD~m0D-v-&wV{zbroghD2WAVUV4c^Q&*s<&z@Jvh#y0LEk@Vf0^zs z0*$jR*Yb%+9~f%+7!W@`3+kf1g_ zR*Z|Zn|9c01U(xw;f?_-O81|yt(yPw+n0qx9RVe{E~L2-5hi%Qk*$s5UBUCc{zJEz zXcRNZh3OK(gjb&Q|9XfgCLHxkNT}d9SbLk%Tv`wOY1LN2a{*RMDs#XV>H3$8m@+c5 zNeYAZZ9hy#n{6w+Y7@V!82aP);~&=U<+rFD_r&8b!VT1yH9vK%Ij}PGi{N!%z$*#h z;klRTE2TatBk1Uu$cjTsg_|AIFElP2Ndz567Ms--ugfwvk8_}WSdTX7obD#lS!ukb zx<}Urc=U#xU60d*+vCjD=+v!jN9THPhhpbtoFmKF#^3cG)mu!e;oI#)cRtm@rqEKr0JB@ioMsNlwGb#j> zgMo?3+7-)bl#anD|E8;p28931SHyXAP&u1~ zUN40AOnUsdHOvt=9V%^ZIvTF$OHGy4f$U`c0(Dnca2{zaE`0ki42j>c9j-;PB9q>g z)ohmj7fuQKEYW7Q=LI$MjHkp&S>_-rJ;@$mJIu8n*8cfH-1A&zq_vKY_;vjFHhud} zNgBTw>9Gcx1Cwjw+tf#=taIo?OYRFs@#MKZHG6EidgJKkafZ z{umo()zKEs>WQgE{NdQ5OIEhx!iSfY|GlXTm>Lo5V$*s2^-I87A9^u)hpC*d3lY9a zI=YL?UrcFN?#%3VpH15iE6>jxC5y2+W*l!+<(22T!#0;&>3WLrLj0bfI*L@{Wav9; zORm)Exy^Z#>SCQ50~Ya8Vu-14%E>e>CL?whU9&U{y)G+igt8%32E41u4a?=98}hD9 zVox|;yLCnu+FA{$vgqA>2;9H9&$7$Tj-5OEAt9!4)I(!F#!8vfk*kB8e>w9mD9^;G zC&4P5I62tT&kD*K8rm)$I!u;mNNS2{UUtX9H)cdnr_pxJ^Pz$Y{>GLIJMqaWX*aN0 z{VW*SjvFt!7RMbSN6YnLFLsD)h}s5O_s|{JPO;pN+rrPiBCVOyBImz)j@s$3=I6Zr z=sI_J-Dfewr9zKM8dgwE&b9fnL%yX?hI)1RtM9zANQpfUYOJ;!uAv^`w2O0rSMh_W zv?lUluvCzV6+o#HR$_Quel%|h+os5rVue(MK1o6kb`JwD3s;>8DQzP47pO{QGrNpM zOESVBPUG&^|5wt~5}G=MhYhK}O&*OeUBeTAmqs#rO`8r^c(*4L96j|V37TZ_X*xov z6LT_6CC?bt=UrJJ9%_vGgW1nH`W&A-RXsI*OWRE-3mNW^p&#HU(dOeAzG>Ddx9->NVP380I z)s=v*+uo-}!wEBkN5ZloJ(b0z0+7+E%#XXYYXc(Br+AR1j}c)74_N>XjiHB`gqc5; zI||A9pgI?*&cK;dlnjO2RFs~;OUcXy`&X&U+4cpz*UCdYX=uuT*VflPYwzJK)V zLZGuuohfBey&J{{oSg8sxt$_;+V?i2a9gE`S*ymMN4@I^W1t=6Jat9=-Sl+p87Nxu!m5ie0PgL*{y6GneQeS_NwNHvRzd0MZ4@4dQ<%AbY5;zfC-{D!l31l3(<^D;$A<7x)|dkyL2ghvF*PPmZU` zoD64Cv&EFY)+XHJRi0N-zP2>DT#>PH%z6m+$n7eM@7755e&TPx1;7j6c=ON>rKwzB zD6s-IYimt|eYGtAY^c|wg>|v$o=N+CU7~M9NAY<1M=fY1znxSLj%DrNS2SCYiSqzH zRp{dJ5<-cP<;$M~74Kg2+$3|;`W3QFt`Oav?T#xS^(|L!;G2$U=H=y;`?&ngt!jGJXv0nVR4e%ze%YG~W{ z#dktkfBZ;Dz4>C;mskPgs=|u{$sIzcOJB15Uq_^O!r4DJs zIKX$AJi^I9Xol!^n63~IYE&Yc{+aY;L`}wa-+xE>)BHThpr|L=8O0u34fHN7u;}?D zy3!}$hd2;QMGC0vAv;A;Qx3lKKVJ_xN5FyR$S5WFeQ0Rj>E3;+$t6T)kBz7;S%TT4 zMu7A9Q}^eJw2zR=hzkhWmzLXe*t-#f=Kk!8HjBI0zW`EIV;O*VjJkbjn9kX>Q#zU= znCiRa^+hINGMNq2S%DEP*B04aOaa14tfaWh@>Mk|;kCbYzh%{E9yxM$vn z$30dpA*!7>d7-i%83%P}ublkbSQ_cy&y|(e$AZhTUp8QmfA%mo7;u_9ZNQyJs#DJM z9!pLy+lVPChdlhds?OfHb* zU`5g1l$-ZEN~h3urd>pXy?>V|&))eYGhDx!R{M3vS*vO$&W4_|?6gOVFyLWF8@{;&X{gF0`@Blty*@HLDyX@D$C+!8nz!@We z8w1?=%sG%0KLzXe*ExO%nT}K|ww-%Mrkj#2o@|3v!)kR7wo{%wIW|Hn@`7oXvWn^oGfV~)LTD=y_z-5l;wZzG!sxDd zbUfe@gH_u$Z3e0dioHE<9r_JJ{fam8-hO|TvDAl5ng+5v3;coR!1`6Y_+u;*MU+O|uNVuP9R^G^M2Q$Wvw zDsyRhts;*e)fBPG6;8}pBP@2k*yC5nM+urv3u)jRBF(iyCD@(I=>$g!hQHNvixyK$HQqT+<=DEYL z28aGw9yYG-#0~LRvmO_`Zf|s0_tmBoy22OXn;bKi_1KlebkYXF zH`*vB0L>m?K3s2(3y!W8Q)rFF@*MVw#Jqmqp*t6yRn|PONTxNnvYwFW`Yezw{~5N73c`v4;}Md3qBq%eL~)j=xKL0Y-?jl|>e z<7K`apd0DD>1RNeW1(Pgg}2A60zzo|MEPPy$iwlD_B4Iy9Su&*Crg(dK^Zao*`zB@?~O_Yv66}@xT5`5_|ZzZN`o# zRf&lADMYe@I^r#oBstvZ;FUCpJDCIxO-yQYmaQVuJ6D1d=b<2YfHh^r<1Mk(z2d`r z+GfjD#bZX+M(RhcgPsQ0|IN(B!=zwG##MWkvD9hG*oINRRSF-vWnQ1%UC>VF@gCC) z5|8AauRBO_drEhQpd)(O_n;@d|C!qpW*GJjmau7O`|{)5L425s7K)hmar-YfEsBbZik`T6e{BIOJ35enpU`P5ukQNkk4f~ppwgFtT`3=VJzJBJ zK#Pg&Ohz?@+fDX@PB$AEmFaarWT!QND6qzTwyemj@Ne3C8hidodvUlzSq}U}bkF;8=?%-#DGJ4zGRV^@F_U8R+TrFo{)} z?ENuQxmgL1qqEjAHq^TyhD|a^#*8>PHs@&61uMe36-R7Bfx~3mxiZ@mS_o-QODeXG zNsIf`Bw5H!OuK&p#E$5Nm~)JYVGma^3CoS)*php|yY#TP-ncpLj|i?nZl|3>Q}@}e z;!VQP9am0C3ORn^5zX>myqJMmNykJP236`Si=LWYf4<<`*My3%Q!LX~jB7A*q1$K! zy~F{{qEgJ?{vutP`3(1VQ+Sh=W9HI8T+8}8I^*SquA0Pi7}fT4vWo~7=Fa&iI+`Kq z$;kos-N`7f?t?LDD6}%&*c-`DQ=Pex61Cfv&0(P9Sh3pwcJanm5uk)QOQ^{321|+A4o2riO%h za(#q-Qp&;#|5Ah6=h_({HUbi}il;wdDsDMW(}3Efb()!-s1Nniu5pK!$YU=Gi?Wq5 zlRD{0gzAr%o`?7f@wN;n|BIkI+EyU%>hz;o(cF)cV}Ev$29TO-s(H_zPpNfQbc|fcYbo;@h=Jo<6So2) zu8bOWz9bP#fZ5D%mAdE}$khw6oiGq?6pfPQltD^h@dU2BNqlvH2EI-!pr3(^+VgO_J5tP_M_-FJnQ$X$IB9-+fV22(goRp-C#=e~#iAY- z7RfJ{pL-Y2ZHI1Sh%k{KKYfCX{)v1@{sR@qcOVevW43o|Ho=l8ihktGqg$U&vOWNF zmebQq0pD8wNP+lJjUEIa4xiZ7d2KsADY5p3G4`jCvkDt+yNP84dOYyP!?v>{yS0F4 zH~ORfV~v31>>(3+TpHIMNdOuBpsGK|NMI@J+%&h|IU#V3Fmd3(t_%Nbg!8YRcD=tg zjck*GE}jMs{b&@tiWNnf>J-=~cgN(~`v(P6n;^n<_fx_%w+GVS2%w%Ib zbxwi2iM-4|K-(OfdakL=XNYZNKD&Lp+mmQ+Kh}Bs*d0tlL1@O7bed_ep2B#lyth=s zUk1^z4PwR?Co)-Tg%F6bP7=}sNI-W#9h1iBqZv9z#s;1iyfp&_f$WK*m_|nH%bZa> z4GbM(O+nb#VW(&*9#@nvmE>zgq4w9eIXC`5I2S#zoGqT}(dJ7_i~bO0ll_@Z6=jXe zECG8Z3>!%sP>-)CnNYIEm`teotqu;7=bdp)^0uuapDd}Dtp*Y#Wf{;&xo%GuDh!aXM-M~kF%j0WZ5BCU?mATGFPwke<+Z?|8+OUlca%-GOI9c<&(ICZM zqnhs>?W=t||4aZ~<(MlR5k`KtH#BsqOdSQoRR*2K4mqX&QuNIzRM;ll<*#+VzI9tH zq#1xo;}c03b9tFz@2%Qnvt#~&~HT%IXGqd?ZnYG}2Faho!bbVxz zDHoXde`%^4^xB70M|ifZtgPg#TL#h^RE*^Ze7a{v3n4nIB<(t5Az)99XV4ax~T#$k!)b^<9>svaR8ks}dMh8aV}= zR*g+uJH=?sVNmRI;u)6HYPz*fal+|a75z+A*C!ua@^H>ewfJlQnXHFCMP}1{zjzL< z^j~D_BqYjt398=J#rVaLzcxLd83KwM)b@Tzhf5J2P5|p3*xnoVyz2Se(xtH{28AY0 zU&ZEhT+R6=CoLr82p(S2T_V_XCKAV&i})}b_EfwkyhJh;>09r;`809BQu+xnoK^Sk z-HW{@4@l+f-OX92asW9iio8p>+eHBDr8kyTQ+k zmll48UNk>)??Q$j`u;&I^B8>&3K-!2=H*Ye-C#v}5sw2B^za2ID4I9}2;<+@%tV*y z71wSCUKOh{LUsaSvs<6s>#rM(evF^U=}!hV$TTrmrvrVTx%CTgPV=%f!zrE4FJ)!g zkC*ca$jz#?!BlP51e&s$?f~7eDL?XfASE(OoZwfW0oHz&7A>m94*-kb0;TctK zNejj>@s_Z(m>F})R|i>oMJ$|b3xs?5=-d-Yn$*gPQL>3z4C^~PcIdE%^{K2-6rSLV zf3=H%)!4mLunX0}Je7`g3ipN?3J$~z^%umoMA9SDbkous3>xcp>}UqGH*|;V=Y{Bb zh*{?swq6W*%Hy%{(Mkio1+3_T#%!3vTrLJ6FnZr3{W=oY#iN^ahK#4PX8Lz49#~Pe z8JjZq9;iNb-pPwAGRAVvU#<0Ia(Jycjnz^gx%Xo)9o;Wy#OacKWb3_4@4r8mrG~n) z#~qs-(0g+B4yLDMpUIJH8QOX9eRl$Eax72u&I}AXo4v%bh_@3mE}-YTpW4i=@qqO9 zCn+giB%7{p-(6Zy4^)Ru;{k#~;DO?z%gcnayyqmwcbm`LmxVYb%1s}XD zrr}aI=P#!dddnd7{grCo^$*U)ZmXkl^l z6j@RIIKKfYWft1TdLs#Wsd7w8HH~$BwC0hkZmT=Uvgs~!8GdAlLiHu)g=do1PaGS$ zz}guTBPZ7q0JDQkTCxLb1GrfZW0W$E_D`~Hc3653U>N)f#(#{Rf!@!YNf9)1CS?KF zPA>>J<|`3$G3?rI-Q>JXFcsS1^#o83Et>L-X^ei>+Rkph?@OC?cvmIqn8eaXqEW!K zc?s@I3!Fydj@C59Fk*!rG-!2%Y3Owk=c2c?g|AK7%qW@6aR=^EkN8Nawj!>*eCZM! ze6+O0`0zA#QNY-kCZN&3^s!#umbXO7|y`spR&L=!%}BE@;G-S;2UEW-tJ5oYEx508ua1WRZtNdU?!a6s?59#)#V|FQxMZWT)V)qJ$f{;Gqz}iyeWA`^qHwg z-zEgKSIvKh#EU*$)~}(5u^BovAge1<2w9}YJ+!^cG}E&dt8gix&9Q>qjM%<&=X&~A z+pYPn>C@%904U@AZ{+$|TSPcJck023efU~K)qZP82q_GHf?bJ53HD8fwj@9dLXx`3 zP-RHL6Q>%ZmUS_iQ#rh6LuSrAptx!ieCd*cvYi8XWEB}7--?K$i~O_4(o9P8 zYE|O_k?{2e3r;GI_&XmXCR~EeB1cT(x(AP`9HyoaEsDTY-UNoJkvJp?&(1G_v05`Z z60NhA1&_i+y&Ge3!RzTv)a!h2ce5*xk4eEZ^#6Wr7pB|wzME3}NoK*#%+|vdMH>ff z&{{S5>WDBtu4f@vXG+vEf@7Y0zq7pWge6oieX8v5ygDnhNw{Wp5-RVtOQ-^#D6$Ka z`hGvGsx~6rCcz8u$*ntga(hOhkN}($s@uU~Zln>yjDG!E$|g`*P0ce9#;_veCVusy zi}vI$7M2^ZL?T-y%GGr0SzmN}^*om-%gK-zzGu;rvn8i+xbxv`46yD%8#)NT4ImL_ zCaW7DNTa+<8_n$298uu=WubfhA=S7OQ2&n{k1bAu(Kw$}+sY$KOs#g%xwB_MI|GpO z1PF%bu@M*`=>I5WQuZ>Qi{3|6ECMt$7DAX*51z+N!&NP+NC_6<05Z<$lk{Z%S(q;P zUTqR>Ml@3{Wyh1hMA^Z2rO?NXo;vl-lHqXOl9$K4BoXo?5o{%^R9pIJF6#9z2sH>@ zcyr?H5NGLpKtDVP^&?ZY$Yvv0m&?|kG>Mu1e9oz%#Q&u2v($aGu#Cn61ef1raMH}=H0kN2T`7g7 z0Vp?0s{A@%tHu{W3@fkC_u~mLT z(s=WS>5CqW&YTx2#)M7Fvgak=F>Rq!BmQ7zuAZC!IcKI??VzrtlI}QMoID~qYhKHN ziXxL2f0Q*$4g86V@fyhh8y0kFTJ~G4 zYb#+#ZPK6`)^nrMu-A=21rDuQhw;{D@1DAHT>tN+w5vZfp3fg>{kUH+t2K3zAC~U- zk*5LF2|{t6XzFdPz78v!saf4J5kPIw5#&&lWhAHp6dDwFr-Wg=lmY0Z`j$hlOy`> z87Y8L)4=CJZf~a%)WDso$m}i~+V0QZS6CKOdk-1oSeQi9&@&H78G}{BEhZy2w`q>z zsU6Zga2vY1!4Z$pY>B{*InafNnB+RS8ODy?JUTY(4Hq<-80!+JH`I4Kr?CP>fGEt6 z?FTw^hhE9|Tc&D}xVJ87)J^IzH2oN|wq>-EXf};;{AwO+?Tjz1d;L79<@a7z{=9OE zzb;%Qjhhpa3AsoFs**mwu+sEz#kDu<=57I=0Nfr66`MV(;>MsJdg{r42uB(OTZmVn zu5gwSnY+biN!F`q%yCHy&ZQ`+-cVbs7PuZ4b=;%Af1~Rb$Z~5zG2b5`Y908oY>9f~ zm5k|S^t5TgqFEz&YUd_}xe<5wI#~%Wn<>Z~9{rAtY8gHQ6Q5uCJ1x-HI;rl3~{Qz>nj**WM&~oqP3rQ&rW+HRxjBQ1w;w2#RZmxVw;xEkAA;WUd!`q8u}{I;W6u`n#Ex;lDC78+#K#EC7!|)gifj@1 z1;&ZGF+BHSO6829D>_&fV!fTUNof-hwLQpO>@XGMB0%EMLm<*AuP9OED^XwvHH#+*Oj zk0E5inEsw_PzdzF5_tab1z4ZNp?Q3KQP`)cc1Rr@7#mt!I{e*hK7*0yrbni=!{bSF z$>ob`Lu2u-8*6de<`GesL&!}Q}-%>7U$5T}6@6YqfoGcOJpfPzQfVu~1|5XpVVcBB^Y&`?<1 z@m_eVeAhaipqaGP#3@SD<^1A;UpbY@&vl|&yV9wljh#W^1aFATdXb3F{3v%cu|vwq z6#D_lT(lQgY$RMm#1uPr_ybRnjrA2Mo{kT>z51a5rPBHHDK4P+>`yJxs6h7Juat|{h522}J zw?E(y#8=^3%yc=O{g*mONpyyd9GOD*=*M<@8`^N7Sfk<3d&4ZtzG2b+OENp*sK&#e zH>a~yy3^S09+@{Mb6&XWo#UJ?55De}LG6S8rvdHsV?NFJD`jGp;7iY8Y#kUa)bYNb z+n+E+b~a&YQgLZ=fd9iQ)s6)=80g=?!qAZ6HcpK^UW0!H- zkmsm6$BYXDnVdcQa8~W9Wp2LSt*F{eT_h!`>1Z@!aAJU;Y#|wU`Fs0lev*DrA}ok) zp*?fo6?%21P^_dMUb(S4z&V{k5n)FYf+l5kg36v73qS|ru;RVvDypg+R))m77&~Oy z9t{4%D<|*dPo(JcU++g~Id3G3(qYxna&M95^h9Fw47=BUOGsoNpJnm5TD6;Hi^_sIYrzkFiA1&P_ z=BGdY-D>0q^9%{>%u2*TRpIFA>E>!dIFFs;KZ5XBEtIzhxCB>W%#LFPl6&Yz2U1`JVX@;K`p`gn}r@d^Hlq!r7$NH`T- zlFmU+^&MyPTh8Xnk7@i)%6NrM_bP|#yBD;{f~GU$sVek*(19XiPmA5?4(3p+_EC`# z^eZ^6TK`MffPSOYK??u4`~AuxR(3Ttz13n^_&4{k#7+{quzI`AS3}c_Q>3?6`20l< zozUkrPolWJmMgwW00Y^{^t0cL(GF9X?eWN?Gn6ci-(675%#%IviV@GcSv+f6nu-{ghqAoO@2D)5R}BfzqqK z|1|5d2AYb4cweX}NMow4oYwF;!qY3)uIb&*ToKpK09SPc3sO+xvpHqVv`eGG*S??O zriinl(_zQLWFH@14|Sug)gMC>7EqO&#nEkn^L~im`53ikQ&JfPY=;dSOa%h`9W!Bq zm`9h)r6ZT%Gz=SUAPd;5!h6^J@5Ok3vWK7peZAyd#9i-?qm?Qn3<0wA2WSN^jL2PL zo!BDl58vg4OhCfkNYfHqaQJG31dI-jJp)TRiNu_?-q@~3FS1+3BO@avFs$#LmyKUA z?}@OS;Rnij`@3*OxGzf%X@BPv)*#&03nmirQxYqgJ;&C`ffOogyHK@ZWQP_q&g2j@ zOUUsV;NJ@z>E%RsN`BPnQ(5ZgL3YayLQ}EhW_Q+>xj(sBT{XE^HHq0Kt;%-iXz0dG zND_cge$HLL{%6#u3g){ejT#wFPKyQzN-~$;tq$Xq_?e|j@l7q)pQDORWiaYc9W%c(7Zc62JC45q9Yj%&@tmbQza%5v-bS4zR=&0B$qC(d5bPhdB) zJbce$cFfw}kNrLiXFB`3XoOZPH)U%EGnDzUH4pA7z02xZR z0bsARPMR}kPGN4aRpw@NXkhWxfqjl{aPh^K6G1 z7=rs>4RiR|Be$}?278UIh^X`95tRpZZ3EmqZr!?zuo%OujPP>lA^lW6 zo=#+$aQPXIMH5HVVOibCHLw!D@qY#zKssGiS<%*a##l5&P1n!N}Yqyil$Now}M%mVW3=cZbkaO*;}v9 zq+AO~B`r8mCZf<^I8_h3O*%;PY!r}HyW?h`^IyEgUGChh#iQ9S@^0sgN%NThgz)&x zfV@XNg}kV1<`8*ly>pK)bkV8^Bv|T1fw+Z3{vPP*4ZtM~(}D#bPJFJ%_k^ReA=Viq zU;A_v?Y=<>&3)eIzk20$G!Nl<($h9why=kosZr?{j)A$8Xb$yDpYXzUJPkyU)bGU> z8HJ#iSf6G1Gm6K4?#M}%(O8jL2b)k8cBuslTC@Bz(@({fgk4Teb&}-WrO*w{B>xU7z^vz=!({^?*4{;74j&ebQa~<^_Ev}95ZT~Y zQ8uJZzoy3)XyFDiJL5MayiX;JOc9t&P*LLZ1%)4pj#am2MDRxDA@VrXE3zi&85p+B1|o?$P!1icPWf#lloi_3T* z+Zyp)=I>zmknrvkCmJHmu;%HjXwg<_@d3?%>|Y%(%N!YSZ61BK#e7@-EzeToH_`Vt zy@4LU5$k4EhH{N660Q>{4v+}eFJ?Gh#X=r4T;-x**lKrBp%Jil9l2skm||5a!}1IFV>8~5)TRP#BGScu@60SDtbrOm6-l@`0!!zc89<1YoZ-g zpXox#o=5F|g~s^p)GWM_3%NC{otPL79XjNAs_E6yEaZ~oi@nVH*1ZqMuGfwyz@{ep z`Y@BgR%vJF`D?!4UTHCluMu@mpkeK57l)Y+cD1VZY2uQL0&=YLUmIQ5vetq|HTG{h z^x6;syAdCc`C ziOwqLONj4$S#*)$ywH?I;RvL&Z83ILzI=`|D|6X3p7;6dNd_)Cz_}QfduPsv!jCvKYtdRQZL#i0CyV}~${2^e4!Jvw?QGu7& zpwmFXWT)2Jbf;cB&yAi$p3?VUME-m?j0Ovr45Bj8%Z(W$OL{Gy^k_r*#3qnK%PE`6 zIlLlHpcBFMtD2PVJ+4X}_EK_s5!v&crQ(zjp&JS#RaZ)|iKXtv}X*DchS^;Htq^z}_Fb*fQ; z^!Ve)r$_Wn5X&7+UtDCFB{94p)bAe~8|a;82LS5%vY@ylbjNZn02@4BmzSBD2G zRo?QlL@55BJg!Ig@7>JhpuKwbdne32_whicdQ3YVgIUxp3*>-A-z6^En}N*&Dbd`U z#WTW>mygJLXZ0hk3Eq`>jyCLuu6!w@Q$Vo!{RzL2HQ;sn&60Kw^O^1`bb3QMm9qX4 zcg&bam4c9!&y`AfQ{3I7BR&;pP9vjU>(}vmiE=&3%y#tXb(z^jYU;mX!&YWC?XnOY zLtM8*)sDS=m@0GL^zYNOw!FcmZE&lfcYb_s^Y!PFe+F+~Y|Vm=LADGBhCz5LWB~)G zA?!4tkSw=N8qb?G_#H!uq)C|Vl?Zq^TgR&3sPYfJ;W$7r5?R3yBbKXpKZ&FB$DKn$ z|4oj(i$>h}*aNKIbcKF}Xsk})U)E@(Jv53pmM8WS4DK|K;RN1^*p{cdfr+6qe2^i}^7p}WezyEq5hS=VG7vdadq-YB+XMarHxfHVA!{ z@py5&5j%Zfzkb~{dZ;ouxB(X?F;0f*=zPdrVja?tFr-I4O;@yHMxFb%xLeHR&T4Zy z(MW^~%`hwfVI?&GSvG;v8%0D!G@I{m_>M$V(0r(I;crBr5H5J(A{KN--fK*vlJJi* zN*g-I)!P+(STxzfc|PEVlc6N|*`W}zfDncWh?=tMqihhwlg&!pH!=YhNln|xt*F_* zF&dC%IAX!u4`}tY1%s|2EB*Np+EA?N->-$n8D(W|cB*3+*6t-n`u{1ogA+**mEcDLCP>%x#qKLmOV3?5-b@0ulU1&|x{goB#9#Avvk!l$jVaV^n7ighw8t7AIY zbk#*xxU_cAA234q&Ub%M#T8n&RI2J^&WnuGVDq(kKED>!MH-gW*{4aI#_F=wSf|;bp>~-h^jE=`r<@(fp#R58*#xZE+SBuFej3!FI26UL>%w~> z880(tC=V;|n9b_;r^xipd-RCA0c4azR?8&-CSwAf6mA_5voaPCPt(|ijt{6{i)ng8 zFWTJD)^1si0F%Z{o!Yq|0I@1@LIjgmX(Ipybr-fH5Q~1T5Q(g#J+fp+J>J%|{Kv26 z#Vp8?c%knvrK$sj13&9!p0qJ=Z_K|$yH^ihQU0yBP=g4>+QPult`pz>ffQ1`NXT+( zgd(PV*IJ#R-(u^RpICiYlKYRLyws@@*(d7QDm*7nZr|s zw!@wc3qCfT9)$c=E_xakgZqW<3-u%zXoRsfLA^oiX+Il6Z z!KKv&BYPdJKUlGO^Yv8^KMz#=wJe#+6x4|~i8(@d-hqsL97eX5pFDXIQvYeAt_wu zPnJWr%J%6ePo_(zo?e)!ZYjGocy;N|tRB&V(9aPZ4FslB_NPoosIbXCw(3mbq9*m5 zXt`F#RYE;rLHvZdlNdr_?pS#Vb7$Z=1(SgC!g}HQVqz9w^jm0iYGSd16g3R-mD&b9DwtJ6~fk3R^^A@+V0;Eul%XE z;$M<;0US7aILr+m9Qcf_X#?7KaOxJaih@2Eei=ic5sgf^HFlOPuZ=e0A?>wP?}^zI zqcHr7h|NV1j{GEO8uu{7!Vpsemi}~p!&tEV+2n*xkwnmY#cLHql zjx`aNLBh8Wnv#5TJg;0|yzb?noumsUOUkN@Izdd(L;r8Hvru*OMKnnN9Z#HMGH4xw z;WuLgoM#T@*Czbk?D(-`%ttL7<=uh|0~J)1tfvx*KQL+h;572Wxe}DccL70oM-iv& zo$%mx&619fk@IfR95{4XgQgokhE~pU#mU{@((NgV?2!3PYIHOuVD=-C0Bb}xkFr_5 zKUUym|7?G{L{Mu|7J?}i6ZP|0C&G+BgJ}biRFVC~F*YV!$PgrtZ92`6P?$sR+5afa z{yQH%u1ZT9{p+vB>BvSPI{a?LIrOUxBkH zpWm7E5!s|7YXw0OoZm7$BNY{@>V0_uF?-^w=8Pw~qckLdjwz^Q<{~{4CYOQCczKWkSfmBSo}>94L>Krjvm{ zzsR@U33^_%^jy@`f(3Exc$08-t~QA9nmlUa^uhofv`8K6R8q1oDk@4S4-4d(jPQWAm ze7E&d8WY*~Dfy9_np@f%Yiq3)oVW0@?!8#I)+AN3nq_;s?1N}bOY;z>|W!^ z2%CjPPyjaC!6$AXN?6S&N$LatJ~RyETkMun=p7skXM3qI%1cZcMMvpLQjZLb(ljut zFAW7EMqKc-6O%IZhA^to8*TY0TD1ydMppyzknpxUu61Y2LjK2NB)lh?Kl`KpVNK&h z`Xcku>Ui*w{>+!-M_-9MEwSUO`P~*AZzC{8h!JZ79GBLZUGUJY$WW|CAZ=VlKzgIM6*hD5)o}7dayCoT3);F0B7^W!FXQ+iw9W0Co^-LK@24^DD z3b0-hnyanCMJ&Q^nqH_TvMCf-du@ z8{3)plbsx0yuWB7v|3ASs>)u5D-m5%CJ19)H9SRYa)3<-3Wvn%er9wQZD!b8I3jon z4sC7P9z;Q#iCULz3)$cbH4>JM1d;6d>8PxdN{xy>F?kT-ABY`OGX@?;h(!p&&1Uy^ zp`}csmF8bG{VTYIMSVgZavqrD>#ap|g7xw5w-c3DhW(lBLsgDqCe4^%2IyA)PMNo#rENnW1gtC?5ukCfyFlbh!T zza(*5L~Uhlt56|uaUe4!K^S$!7s#*qoqVL5u%AE z?f(sZ)2NKrlOWWju6!40yPlkvCol7jNJm%=h$<`hYTisEORD3<<1kW!*BR(_qUfpp zEBrfjYLqCY?-84(-(v&~*#P(QgIJFSmqDm92VEvl*VG4ZmJk`epBq8I z_A6Hg^6Y)R9grU*0^*kJh#8}!3~2`uDkAf$5MMzioESN1;e*7(f_#?;1tX?1>@EVo zCEn3?2d#KgQS>X0tc_y8khnJFLucma)M#0Mo!M-pab)^3Aqw5&x?-`yd;yLOIg%pk z%qq1&)wAaEj6q;fUuXU?94trrV;KXb3w9Vc&WDLwT!!D=a$8BHGN)Z!;tTcpE@WE{ zQ9C#s(&qMb0+E(5zYL~-RGumE)%=|=p`lo-a&DYLbBv_=Pv3xk?dY}5@4hOAJ`FFQ zyq8Ng=(NKvgAU> z!mKw-7_GusATYi`y?TqF2P zX%R?Xeb_0u^W*I8rxUzk)7jFpFn%@w9mu%7V)ys1NdDxJU(Ll@O}wGR9La8p2Nw%} z=OgC4Ut~fSnHpv{)DFv}>NYOR-2E!yxAxlcdojN7IU{fn0(z}(89t27$P*^4Wd<fAeT1q8no9x0W5khOFXIj!)i4q2!2@l(G&0j9};A%qy5d&CSiHL+jw4 z-9*0-k1ei?*pksdt7(s^*yx}$shLk#7?_)UDJ_+y%(9Ygdd~`{G$p_ak`D&Si41eQ zkP``pA{&v}Z5q)Uu_5zzk2O9%2Q^FTuxnZxZ$`;6{jn# ze&1!tnSJU2YKXXs9Ez(ujU2hLar7VlGSQ*774gvjGtue&jAnr`QYVEOk=YOfvd$sB zJw*E3ju_FT01~9OPog1bR^R?%u)p(slE~BxJ%}t&w~GACs#|*b7OYnLy`o3e@1Xue zqpFYMK#Cd%7ETa7QkAlz`Ksi_UxKnFb;KXtC0qmO`LIp9{ns8X-?jRK{&?k5aoO^} zlH*J%mQF$B`mLN|vt$DKS0t}xXn;*GdR#yT8k-U0(+G>_+8_m_DO{0d*~?L~sT1L{ zLiYTSY12Sf93UQ>-`QTNZHLv0iYZhEe2HQq9M9^@GZLA3_m6IGWM^x%hk1H=>C3~P z(bHR(TmV1O$4;U)q(8cH=+4U%E*TwAwAe*RkP93?z)9W-uQcA(MCt)<0!1-%gh=oa zI}%x9pl8pW!uMNu|66c;DOw^)_af?XEe`=e4>@rnRvSEb?lIpZYe`J6k4Tn`tT_ue zR4RSM1L%(c)umIlt|15VMl!Pt&`O8?#MY6lGh8 zC?oJ^I?#Up%P*-TZgZQ6$$$(UjjMn5VR|{z%uHklg)%IBe21{n?G=I7r$kN=!IIV0I^4SD=f$_E-TM*+d9g(#(AgV(*ktUHZGPD8iOeI{Em5gl1S z&Vqx00yI2)o`K~MJ1F^PEr(g_?{v{+}QbL(zE8@XxVFiHA&=4p>9a5jIqdz z%>OPg%}iL|(>se?IRJU}f60Ge^)mE>WM*&`soA)3AJ?UJAWittA%PBfw_(Qg?S?9n zhgPvqW8Qd|r&VZhO=GGH8o>SrSzq@yJ3esYJf}uv;sb=ENW7N((m#I%A7f<#<0z20 z!q9|owRpcJKMC?uazTjcId=p9scc7<-pxC?VbFB!0kW}+3Ll}5o>Brqf?DdGl6*1{ zw?G#Fsn2)zmHX2928<0_4(T9SsE_XE>6xx5#k0n&HH5%o$xHSpFM^aH1-W;M!TEUT zWRUZVhKElmf0eK?pjcOsXwkGfVFdO566)Q&IKjOl*Rs4OE~TVpG$!Oup7lkyS)6d! z2qW^_imwxQd)|yEQr@9my9q(RgoRi16=XnVdvW7X^`+lVCN1MSBm2KwyYPuuE#N(B z>F&bnJ6aYPB9$~SSWTh?^>g&3N&ZA3P1F}NFYF|j*T&LW=AM12p{z-!(bg5Dj~ZGz zPIc+BzQP0*9#IIO;BEe@Q>RX4Oha7_jhuvCRiCE$i$8D7lJS|)@YeGO5cDe$jg{4! zbzAr$uk-EYd6g-dtJmt$ze3rhiL=v4s30DfsmVz8)_oY07UNS!dP z*{UN*We1vbKKWg$clD2EbWk%9RSy&c*HDU`M6c5!LzIPqw1Pv>Te|nXTh;rsN(j+A zs(%S_TIS^?7yp3|AIe5vd^cyMhll%7EoI_?2}hYVWuLFu9H1cQ_kF?tBORppmITA2 zCdmpPw}Ui1Xx@Eg!&GMElyzbp9UD}ifJ6`#YYGMXciNQBp38hioCoujurj_P1^Y!i z{wOt>&6y3vq4_=Ff+H(=N*6+J@7RjH1zZE2 zVrIY@8Vnud4-y0@sB&(3ripWr$Gy897yo3S)XqV!PByt+<;+#>iam`}K|?u3<_r42 zd7Yedc+ZL6)4V%z=VvcyemAvFn*p0f;JJheht6f{maD6BPY<>nSb28KUJu%sAJ6%7 zbTX~9!!uBhX;gl0Y=>=V(jEHI%GGY$o2Nxys`tpRN3&Q`MhOeUt=FGJD9FA$j@M5b zsJzmLRDr2Sqg`QMq5= zCFo`lS6r@j;e!3lbxc%;1(FA$YLvn>BGEhlyN>dUC!+@7x$ZSR`0I7)TWLpRkYd|F z`?lIcU-Ip#mg~*-0#GqWnM`e??7APVGZJ~CeDAFKh#HX}Q{Dui_*|}{cZH5?TF07=N1+hcYoeUR-;O#GILxbNx^0^QURE_zRHsz320&UPLbvz zi`b+q1CUty3K$JUynCkhRWD}8i#!t6F|;MJIuTKPt_Mus%CB!MNFMhupdbOEJ@aa# z9pot=2zwRH_V!ua0iIWM3W&wM%U(@`cJzBd!)wNTp)pP?Uh*U8%946v?8gVc)l6Fp zeVB>R6iJib9$s7WA?@ObN7T_#wVza27&mq~nCBfTH*Cm`e~{+Ic9KI9A8Hb)mGJA* zB$vF2!;~S4=f=sYjcRBgc{(eYELhoeB}O*t5Sv4y`kCThMi$B*fKSu==mqjSu;lXJ ziUsQto8B36A>7*`02L4Zd3RY2Brh8+1jBNJpFX|h;ujK97?|1^7$iv`Lw*oqdvz>4 zlu4F7C=PP!_J2Kpq0a5wyMBldY7?P{WIB4}x^&k*KHveR`G#VFVzZh5fdw=}G`1s} zp{#tW7S%j|Zp`YE#z>;SHLn4k&cC@@lB{IDcUprmxT5@VMGKi}=P{*KN(EZjp@drb$2%q%fg3W(~$+ zK_5|P_de`c_(hUR?3C&!RBV-^oOw$sx7_IQbM>}Br(F2@wlAz-KmoW(GP4=l^A8M~ z-%4La$tKebUA`-1jF-7YZcDF;|9i|M9Ew?5#Q8nPqX?9gr-LZt8e!!nGbR=MrE|h&tE8^A=v{r#kSlxyslNmyL`6^f=AtO zHkY_#)@dZS-t!mke`_9MaPc~KwBueean8Fb@#D8|*EsyK%v+O)rNKut&h`has`H~D zhRkOB683|d5dJkyzZH;t@%#1djxYZC^Wl!&{?0o@CQslRM-Q7XLU*pr4dms%pH}h8 z`}vw>hQ17(T$YL!brKjlg&)8Iy=z|vR2_hE3yGTyaFIj`7LX_cu54jNhB{4L$&#&G zo^tWZQoTwi98#Fpt@nwp%l!Q72QivJOG%T~`%U9*>h~OK3={8Ovzyq$h%s#Qztmjw z_+Xj8*1_qJx>}JFO~)qPolGt?ZD~Il83cbxtN1p`yl2m!b7eyle6yUpDHJH8A~!LZ zqXDlTEcr34yOe!9*}H7a>zV7ch`&;;dbMgp6@$j3x5iT7N4$+BmB{BxMw0>YuuX`VI;BULWsyC7N7WV>()1{k1Cd`i~EKQ`a97W9_v?eDc#ilut_6Tmmz^ya{GOau+Fm-3eT z0l%uT5;ohKzeT-C`-EPhU-OV`c8+UpY}tDa9;(Lf*5m)2An> zmxAz^-u5#HE(BH~8H)8d)N|x#)+@AUo%#~F_Z%G?v3^Lvbu);A@pX`;P)KE!A*Ox$ zOir(Pw<4bXS4DG$2L<`MOxsiJ<^6j>a zK`gp`J=9RI7G)6_A|fT`rGLNc=)KUcRtaRe^Xnpn>vM=C7^*2E)W)(s1gYk*t=s)+ z+KD!-CR*w!;u?WGI9h!f(+j7SU-wSr{0No3^UkYcnPfzi&2Ee>2^0Nob|6GvUj8Ld zrsw>X+9=5+DpEH0ad(?IS{bBiG9~iQvYl7u=1cmO>?npAX8m3qJ%q?(@me-edLI|J zy6iGVui19Rib|yGDG<8yEGH00GUl}&2ckx?zlFAePXsB3rli=(8P>fv7|oEi6PO}+ zxH{j-IlnmJ6N_nYK^OB=2I#0pha4H)Mzg*;1jJt%`@pdvdT{P5WA)6V>>BhqMXn~b z9&zr_pObQSn)Z8ldMHBg+PD-HNWYgiQTuyU&5F^(OTxn0v!&Zo@!9^fqiaps6hcaE zUXpgaA<@`Gu|GI==lKFWx98^F${>9|^g5dz9#_TK$u^ZQpFa;|RU=f|z5Up@Yfw^r zK9xwBlb7c>003^yi=?*I*)2LtynlpdJX1>uF9@gl`AJRTRB770m(4{;h6VFc&jSZ@ z5^2h1B$A-J|1}b+=5GNzfACvz zg(o|3z(_X2Ff&;2(`B%+#Z2O@4dW34epdgykY=}4r9#8Il18+?S6ff~^CP~^5dHeLCM z)+Kt@4ZYpH+^vKl|Hi{0DK=_o?QDpo&xzIs8L010M(7a3AtDF8GV^3xkEtAINUJA6 zrOLBdRS^Vi;Nfd<^_32eI;5m(#}XGq#|g)poh*5pBL-CX^`t&JbgwWU!awx7e@mlo z-Bl77`YE_fcZ5l!8Z~P&E({)|zo(OWW_roYgx+OyRfCY?<|L)WJc5%ucV|QYkHwG9 zxD~7@Yu&~m_|HRyiPxNaH2$D^w0j`xfM6IUstoD?qEEJNH|E25lSK>D`g%UR91+x8Y;UIrHsswRrWX`l%ydYCnGW<4I)~Ul+^!u-8tj; z{r~^>6R_~<#m`M8Yo_4+Vjicqm84RryGCu4vRb8 z!v-2drxs4kd{nPFC~5_z2BNZYwM1AUxC-f?tV4^ERDRL%|8Y0I5k0?@nV=o6mkpqZ z9nnbYdsO_fpjI_O0GLaMN$L+M7XKN)r;l+M-#PYUg2a(;^=zH|Gx;px}G+!y6PGMD{1$3%7OienGIue4hJyewDkplfoyh4cQ(jb3_iynJA8qVU>!<9o&b z@vOZ!<&l-5FXJLumM9mQu6w5d2*o2(7OgkGa4mZ9PuZGz-aMb$fJgE7hxyvKR#V%W z8{*!SgO9)W@Z{=&N&;qMSmAiZ{FY)+2SvgNi_mo^=~($PgHxF}nWS|lhproUWRfP@ zToAuI3{4WMFbE~&N--QlceYhtb06OXn2<;d&GVIh1syEGeITa_;o$4Ow4F<)qu1A$42rx@V7oIQiMzM#c_@u;9BYO2C!O%I!&EkM5_ znonD%#VPTE%>qdbYfn(ZTcH(lLPye#ZS!A;_}uW2u0<;ndQ%H+uNZ_DkuZ|}4C){~2$iFKIx>;T&E-@4 z`_09M+KYzPHu%REh#oLi+2OFg(+IA#jo=3(j$05m>vR1y(VcypGH8+v z(Y=Z4t6!1$f+r7_Dj>8hJDr@oh0}`Q6FR~3-_DY~2$*k51^)xwdF^B6z5)_~kbDoh5BQ{}V z($TAr)Y3*lOvq@E>tX)<#?F#r{_jg!dJD8mLPZLnOIf<;iI2E($H6T*uPZ-%>1gDs zh7@vFDq^D$_kuzA)^C(V5~qnHrvc=qsb^~&Z~*|Yz9;fJLX@#mwsj&OY2OncI?{h3 zbIJNv<=!&2x_jk_BM&N)LBYao94_h4cDOp?WTyKX`IIAzcC{4@qO<|@e_S7pjc(x= zE}yB}@oPaZO*4~onP)9#KVN3l_{*1_L(qdX(FCj&%@V&KFCa>Mc0!k5DQj}rvv8m=YG7Y%hP1F1#t(FoBECn4kVAlK6JI|mKR)*?|W@^ z=ItAXs;k{Q6?~OtIIBNaSe2S68TI^jjLDqr-T7oTl%6880bRsswdqr*61Zv}&0lI_ z$WDYY3H;c;+7k8h_bYT=Z<;bLbmLK@>d9tvo->fE6|M!ZqJLvfq9^Li0w7*d2ln_} z&#t;}M2l~a&&`S?#kPiS{qS8%b*4z^mu`k3muw*pzL4AFeqBY7qQGamE59F3BqogH-wV35}!<%_N`)=Ay zS$JDEoh-}y=w-pwpzr$m*fGM@F4)F5q`i*J8N|{3l~V=+CkqoIkTtNz%PL_?RA(O0 zL4yWmU8CBr*SuC*SOSrp^v$?TExg(y&pWx}Iu7oQhxY3x-buM3O+ZfUbXzHMsYf@L zyJtPQHf+uE%k4@#u8PcV)}-5!*FEh0i$-q`9$P?wUB6-M2@**f(U-)XWYxvsEL{PD73+#IWszX3E_0W6*xJ-zGDcPJM%>#?2&23!$ZC51zrlGi1qw zR;^nn+&cZU(kd&7Xbar2&e4a@-WH=opnF*Bcd@x@WTa~6QE#41w~6P|zmdN9E15@l z8W^5-HvZ5+iKNhrWWxbn5AMj6BW6VqpTOge0bn2 z*BNHw^n|h{KJ)hM_p8j5l`iVYIzQX|s*nb7XUv;$x8?!3-QpdPY*PD2%259R5XSCRzI!V?=~ZV~SzM-|vHDtTx>m{*Jp((OWf)4Rp_sCG_w z8!Xf3=aw~TKMb&UCnl1jfQm$o0!vHm4{{-iW=uJd+1^I)5rHx#s-FpRoA4Dd? zzRPvAWvbFJviWPxfmx~?f1EBC0bSpBem(%rim03BNGimVb#hwYMC4gc=El? zOK*NJG*0&lSYKgu+;dO5DhE9xsB0sO>L#3$kWzb7J=pCHaseZ)xhbvi5BfQ*f#E z&H=OD`j6L&~P~*JrF)I7)01p*q1^lkOgT^|A%^sPogpuO_H` zm=5ut@TVNqL#wK7-GMmj*ajK{__|!Lc=h!4YuRSi?vZ?a|Ga*TL9ol|YpZ9gjL>@0 zq)BV>anc>0Twd2WhBv&7TnV@%O+7tTD&|H`d*dXIR@u@j*XnnNO;WEyph=(_mSI5| z0dEfcrPqF&*ufb6k%Tp6U>jkfRVEqd@Gzzlu1}Q6y79j6=g|LLbSrg`JY(Tv0ObNu za(sGuRYkPJ7*s(*x0?Ef4Z;4^m`!{HY+8C zf9cuUZ-$X)r9L3IWaGAbTx!Gl7I+ovH136jm$I)1i)p^^dG`PLLA(rp!sxn{o$Ln1 zD~VOQkLi@ym^qi9D^)J@`T$nFN8c&*&m_0qNKN_1yL^8zW!f~hV@hB9vC6#}4G419 zbh|_=mqkY!Hx3?BSJnL1yaiW}Jh%0}rQ>8z8KHkxVb~;W~$}^A3uFEL8VFLpC2n_>xZ;K%|$bacBS>$yM&ia7G6Pyr-fOhrTMQ)VJnHhAov|n6W@-Z%eoH94xu-fit2c z4yj^2JeFv;y6#B&FoTATiLdjBQ|rRvm86HgJ$u>U6Au@NRGJ6&p1^Dsr{PU1ivzsk zq9&)b<}>52P2RKRef_epCg~occ<9$JrqyHZ!a`qi@?ekTayJ}+;Ff&(HXwRuq9_b& zY>+lMCc?#5|M04|G0i_;m3O|G`d@Vi|9snuHy>7b@nybZr~Bv6_!6ofEPvwAU%X2E z8r%>i7Q?mjSVtdYRQ<(>AWy#t=+2HKHa8B`Yo)q!G*F?qzu;e2x>|C;nL_1y=i74| z)X}^;jo9L{3@;7xN^OBoBr^Zg+r0Y38n62F?BQF>B|ImR00zUqvr{`&!*E6*aU%pB zN>(BsZlg$HSc`YU-X!Wg>CbtWXH4#~%Z0r; zY@>ai|M4IkJMytrr*uhLqWLo0wae#jM-TpXtQ6Pv)qk7USUS^oElT}qu-l8rFOibY z!{EEs*KHaXrDxfV3{wZMA8B3^@U*vi;Ayu0yg9nzWTxJn6X_th6;ZC{GvmA~?kYIi z1;Z>FoSWgK@SKNraO^+p!pWO;mkfw*sqL(NJfg*~*~9g;V#dk6p7Rc8jjU1aS-Lgn2BBZBDTqAJ<;&OpJbi~2 zIyxrsF$Zv`taRlTQ6p{8Gj!xHWP3)@3)(?3VqJ6|e8oOVG9;3>JG^P8T#Huh=X%e_ z#gNj%v_sJJJx0h`RY6sJAOe%(gbqvZ?(un_DIMl_n)2RA27dWvo%Q+Pl99CW7m*BH zFQJbLg}qWcdE$gE5~0s^dG_);F^inb>0h?!eAH%$p?iC3-t5mm-#~v8&2Zwb&&;~o6irOu<9Qj5uW$My0~#j+7nhG`ufECt z@@@Ye8`f&_7%x+#aNyH1JGg^m)~aPo*=C;g3`C&t=9i{u)3=7Zy$1IYXStIa#pY0D zj!xgMd9Jrutj2bauKt|vkx~Kn_*k-vu-+ldu+0{U<AEMDtb2U*%T(Z6g z>qzgG`f)S2Z+@8Or|yf4T~ALf&^K|sZ9dEQul0AjEt(MH^ZYVbG!;MycxJJYP=9Bh z*1zT7VepD>*dF@W#x-z^9+6t#)x~9Y(VN?}Y!_r&Cv=pt1}PUyx&hu^UB9B)fR7r9 zLoZ^%yF4Y0&HnFrj)}{D2D0RBfXn6kL^SlUcbjQbPd{;aw|BnCKCT{WjMq0Hkp}Jj z&CdFUO`=Z$VXXpXn;gP*LpOM|H=g^_npWRQ;Uo(t} z7*(&;!wrU~bh2KLMz?nLT(1VTNgfm&>w_;tEQfP6PGGtQOj~2fXY!?}I9USV+qC7= zQ$O}_f$UiHFSK&`@0Jmtu=5R7R1(pkh`JhT4B7*#`AYKgr5VxNz(ups$;rgp6XY*E6pc zmTv zS19~B_|iivM!{jB);n=m(d={o`?7&=fN&Fn@hV!(G^1^8^=eaQ#2C7W?^))|5U;B8 z#F=(E^x#5n9Ly89J}sNozF?ob2RV;pCA}5RSKC3($Hx1*ji&pzx5P3-qjt5*P#yYd zuTI*l((YkPFAY{2x!_p39+=2*SHD)RZz!3yN{f^elDDLxaWmz}8mcxXrcSrtLO-eVr?da%bN%SE@|OI0h};PWtSDZD=@mp=yv|0ulD2WT}(F4;_WEc5~x) zkNiIPgz5a=v$eEhik_@Eqwtl<8&j-jFk(m77rq1H@94F9eCd?O+fZIKO3^t(`d7;0JuM<`f( z@n99iK{Uw9>@I#UCX$?o`l{~B$pf;#E@AF;mBC{Bpkjp7UN3a*}*ido?*$rb* zvWO)(#l#&A_NYfgyoK8MWhL?Son`(BifF%M$Bv8S1adzcK&#Q%Ousb!{y5#Jjoel3 zCYKU&w~CymNoulj3(TBbC-`pk+?e7a4w0CqBDcN0^P^dtyWeHkkV({p|8#}VdXT_0 z`RV%BAMfbAQ4QRA$h-OE{`ub=KPCpf>jkjE;)9s9wCM?~-^^)BWm(2vGz1!kD=uy{ zKRT*YuU<_y)N7I#zrv(T8{=k@yp#0Kl3K|F**h{Cwe!Gt*S}MySnuZiPltGSZw>b^ zJ@JYs7GdoEASv9dt`1z!%B6zS%z$#)Xh#gpSkr=D zxvvE5<+T&>94=e^kl*GvMI}4;pTs-G$tzjGtod_V0bDRBNry~C3aLfnQu}QuxT%}; zlw}`|j_34Aax{Qe?Ifq}xNsGqfWYX5&LKmT8naG<2gI^4=)Jm>9$GdAzPNofb0We| z`#5*}sB@YPYRe_U6`Ay|aCLk_f-Meg@@;8KLJRWPb0urrp*&5k=r#G$#oRB)ZaT4; za zwi9k4CpC1bhkksT?^yj?2@!syP*drLu*{eR^4fNxA%K*UA-W}d&YjJffFrx~R3l@* zF6x@TAh*4XiOIRXdU1^!ZT^;2(-X#+H&DsD#OKT>J#xu_`d@cl4L$Z;Yjw*|&rUBg zTBI~B-8AZK9R6xalSnEi>QQV8>8S*2h4|c_9RpZN=;hn`Au^saXNmCFz~E+^#|-9k zFV2XQ5V$(@Qnb`Pcc1h+1Z^eJO^%ONmSukY_;G!X;!89!sWXdoBT@4J|D88A6FO@wZl$>@!sKADB zpyCfzSI#$GLi!l3>yGZK=7rFf2v=zpZI^ER)m@8VQ7$dgo0>*hr$Czy{4C-s0jCn> zs$=)cRw>VxOHqVK#DspxfFpRS!?pCs(gwk5k>otwPu#N`r54#+KV7Ja&NCVzG}Gte%Ek}cUkNRc$)%t zl4y?z9W{XI5;!KVioV);B2^HsFuoYns)%nWc~97U!6qlg00wu*6P&#b>Yv7<*mEiV+oxWXZcW{+) z!59;dpy09VN^Z~G+URTZ4u41@9^FQg@;+(;(hI*G?>HN?px3r@=n))q45##vT_k{? zWFy|ZJ19&x=2(3U1A z^UduI6C3?dlAejP*-(MIa5irUAW1|YZ#`}%sa3pp`|%@FZx|RFG9CINPrcZ+L2bUa zIvZ{mr5dU-n=ib$Ktb;746SJ$D~|AQL9F>@;~R!xN|p)cZYc;rmuxz#y+Qv6-ih3e zNCWe%o?FQR_TF?NX@hmOAXO^{rx6$pp9q>q4Oz7w?}JNQlUUa#yZ#~Y!%fw=;O(a-rh1< z%PA+&wM!zpF%H-kiW_@zMx1@u+TO%tFX73MLf)j{k7ED#e=)d4H+@0V>0#`bU{;7JLfeXzlv8m1BET6B8Bf2DOn*%30=o z8x|Qv%-n|UOhS+0Z=&{W_enQL18Kkf+zpk}r5`ft_n?>DvO93py4ikiGq ztAAO)76ws`wI3w7OiZL0adTo0xmTA%^4^!2Cl&ZU6@zn+KNLM0r+Ygk<$RmTrDh1G zfh0ze_V!p0Lj%%d4%YIE8e9(MhzbQ7)}Pj6*S9C$P$FtOtN$f7Bb@{5p=QO#Dn zcmB3IFs$%;_38zDVV*1-_<1ZP`j?C{zijY(Jn~}|%uda_l|K#dVkAHlBg(Javt8at zf=+?9Y2ocbK=x?dFTUx1v>VkwZdFgWaA%`7A9$z9ysakdHQ!{#`w6;ie>B$}&CYC; zcbww&W8){NEPNvgps)kl5s2|`mcQH28nxZsTa}8*OLCRqcRuXFluG;;6mRbleYNsn zBxE5fko>(5&k&7W0F;Qna->!!Z|@nc`LTDInd`tqho4VN-g&C}(#w2F+vY*;Qi@K3 zDp3n2Sr&TB6gOxnccLt30ka#J4P~+Eds^BSxPuN$Z+7CcP9wU)sH!+1=mm9!utxi_ zsh63_s>%;l|8C*V5dUrFag7`457rsJJ?osPspUMx&%45Qai2mndAduxZqjV9KE0Cy zv?!qa?%jVyR?(8*2IsFy8e3=oKc&;{{X7fY)bc(HJoY2~^rqPRbs%6Wum!(uc0v<2 zkqzqKBqJF(fVp;0-?@d>-QnBB?6!Hj0^d**5}Ri>x$fUdb~mP{7)zHYR2zCh7T-?rxqTjwrc|^7F4| zY$lCFp~$c-q+EQf`d%1c40BzCq*HPFYTl~+8J5d~!K^Hs+ljK+{L0bmeL0gBD{Pv> z21aJ;nh#`Pt1R=XzIyEI{*20cAt52Op`owsFFzMR?97=n_5YgM7_}GES04na-sfJ9 z&uJ=j2k%Cd$o+B14SelVhaNmguU%>D{~1$g9bnYsSj_ZeSPhKxW=ep+ztq0OGJ}4a z2UyfgtFa?hw0C460%9d`%C0ZZZTi})+Cy}t{V59wOYB65x^!vq0V7Jf4B+ZLR3EnJ z(W_THDTZAl*7foF#>%rD_S-fhc+ygksfM32s()laI(G2~PV_}L2$&O+P}Zu~wMo@) z`WI6glEybKj$PkrjfRLsQ}?%wjT(|By|`+r-hV5a*6&+O%3>-zMHhW>xtYMH(i8G@ ziUsqp?7Ds4l0u}U1m4Shl|_m=?Usr`3J?oSF?mUw0|Lx+yLQnEqGfKzI8x?Rn6TQ6 zU15?C@!xS2I_lq|wi>cgHsHJw1 zhVVbmv%!ueWs0qPF60V(my78ZpSLhhO;-3DUS_a^#wQ~8ke50w_ii9()xbqoL``$X z+9bknd`mL()MDKD&nxux+yVP#1q(#&xB+uNZGla|>o1h))r@07cUzsaj@oiQx?-#v z)KTX2TNS(LK*u!Y($IIxQYIt)`hj0nl=#gh36NthKz1P?h?E=v^4{SoB*MWp26k_i z1gHT_lm!@RbuOtLlr?F-2&wL2DY!0oKE_vPH;)!{9H^+5Pgv&czFu=SU_wwN_%@=J zG;8-JiYMUXEfILIIK#qXa<`V(y!Jn$%f49@+h|g_`cf4&<|$#ySrBX8*UN>O2p}pL zUC+yCk0Aoe`ID67gBCrhO_=wR*Eic?&uy6~0x5E6=%mf;vftKAzqsN@d0#|7k(#>N zuiLli`4deX*(tp`$w-c%xkl%j61O0tUH*XRHOJT89(^|Eld*J%c~9SUk|y-uK4ft{mLXG|{3-6kUalO?Au{z-c{@KZe9hwf6f|^yHkK!>$ez;ul41%qhQEt@t?5*!v=^~?7b5n`e!xrJA5QzVHd5fX% zzpp?8jw?a!2rRu3@Eep^Ql+3MJ0n$sEBWwr-wu;HNixf^NzuswHvDEq-s!$wwX_EH z@7M3m-N4Ssq3+d5X@_b;=8Emxw_A@NmHI`}ZvYL)IzV2F`s>?F=e1dD?6u0omc>GAsPVO7}8a_?7tsqkwlXrB~wpbFIQql&Qi6NUu)b&lVkos5*H_kj; z5KsDR!3Q?M>|Qgmze8#_IdOtXb42ezDCqic*s!5^px2jy{FlG~J>`ozN-|>YwA?K2 zw;ocpCemDH4abiq zhDyapUhh#ovhHd{QtOJPOP0iA=L&fG^r^=A)05sH2oaRgFQ_@REWge&9%inCddRL^ z-?)Ol_jGBs4En!*hC6*=zR&iz}UO@1n7ktq2KIOzMngUzxx#y0KFY zRncuy9c|6xUPU*6K#olKvA-7P1c`!?^PuH?7K^d0yUdyqJ|#<|DpTWHTLhl}zM3om zl_zL0ksuJli6HDGu7?B#h}D?n&E3bvDYyeon770}s zK1GX0D#hSS*+|HRt1BM3uX;W{lgafn2^2C%Mn2zq?L0OCD`lKD7 zTRity=7$g6X$7^)mRG-+DT1HegR9dk3}>Lfm7pxs{KBe&LN;1Tx5h+s!K>QSiL}<) zkTDYzOiGT?hRt5;`thh)7p+j8ODIm9pd@>uu8 z#MI!%PfrD^erOc+?n8&V`U|1YFv1}C6+z}7SB74Dh+|V361y!h)|4JoB7JG%&V$TkTm{lIp0wkvh=hdZX-rkJ;gqvzO$MnlV4M!B_Osx%%kVnlGRYzxI#NI!C*Y}k)!BwtH=m2~|Q zzpdyQfeY-%kMcmw@XM-#^j|JN|FoiKjRJm6fN^T#6JRIqnu%XxnYU$vngc`CSG#jw zM~qJ2l5MKiLqC{$u((wL0ftwcgk;CK;1>txmzKrf#_B zko5&}Z!%PT%U|l%2#3~FnR*$TfRqn>&-Fndm3?sa>&(Hke>_n1Z5iF=p1j)s|Dt-6 z8kBizv15cD&aA=uwSG+w>RT@vwSSxL!f>z0VI%^N@hiS<-8*XC8f1~8m$v}yzSBte zW=sd{UXEvAzNoPy=up8XHC)K;8Nq;0k-6W*j;7y%gnat88>Xy)LagCcQ_;M=$B{b+ ziBMeZ#NJTFE42P1TmYcs-l&FGxvt@3lFYUhgROLRAad7zLMDeq!@+PQ8IR~QPk*7|(OP;%oDP!KHU;8rZq)CZAt~2?!SrT3U+^r92hk(TF@I}>` z_gX8mzHze)2wG2;cBb!?MVfmpKt|E8vfY?S=OG>O980nnjj31D=B0B4dqy%NvhbBW znR4SPm@QD`M3gJ?cahF~@dQD^SREQ`47E(K|1p~Li!mdoWO>dx6ly{WEAY{+K6gDl zDvJ{(={Heu=RXz%A|7}RULvic|K!P&CC3B~;{Y9y1WI@x`?}hzY&DH{U+qwx(&C6w zdtMy!R6#kMH}hIR!$qT=tmUFIUuvG*QNi!OH^pExzCrP*#cOpgHD1y!EPt)$>TaQq z7D{{6-fim_RqwUK@#OPHb_tC|HU$lT#tLCAJP0h5t{+JWDLMrjT zY|oNLNLAX-NbsuA+)%Y_D81p0mHG9lg5SThFoP6nwHj?M194@~Ee+O^yYsgsk-HEW zu)7_3Fj{HFB_-t2?d++XK}O(V&E8!go&SBUIB&e6o&>?lAp%L{B$Dp%;B6#1N?2e? z>eNLuwUz6hpIhc^tK~|XdJM8v*&yBbIIaMhyO14-p_>j6@mZ4_R@Wac7Hwfcx-Z&7 zWN4xc>w8=~F9beZ67zy)0%&rIx=z;0OJ>}XBE^_Q2L(eBGWhrd?c?IUERY^qE0wP1lq@neLLk zZ6H@s54-Ep64y@^8Rg;eqfUz6-tjPrMzjy0?T@Ezc?f z&no&ajcDF3Fxp3@I3a1@fR+P)&+#`YnPFSUK+kS_y(6y;%dU)P7ad703Kr+Zi#W-% z>0^JlS>yKo!V*LFbF$3%93jFsFwgfMs2;uOTTvz{Zg)7)>DN1L^Lf9JYnon=$#;C; z52px@x->`vkCrSs`&hYT^g;)R;RL6U&lf{Yv(1NixsOv#p8C+EK54GgNhp3NcjecX;HeQD_vViJUu0!aD9ofOn#923U`O5w-kz&BOWg&oTEz zBztmVJfWS-NP^{t-_l{)>+@`%krfo|Fa#wRI|3xs@{W428X1#j9hr1t{$3yTy95oR zn-Kx6xFM%Z$ywnKJH<(5tIIZtQ1JFn-oEpFF-0hw0HL6L_)Eb_64KU}4CBg|`aWQB z+bc^mnj-ZaWX;hF(UG&z^M7xzj_&RUaMEH`{CyBfa-_% zXn5OtSbr|gyZVw(p{78O88F>N4Vf6rU=u)`wM(gwaM8@Ml)MAYellpGd|2WyK<;Iw zUWR@gBHr)XwQJUre3Tkk^oOAW%LI1?KxY>M*H5@IjO#?Hq=*}Vdlf$A&g?#8D#Ryz zedQ8n4CMKugUO?M+3Lo2G%WSpJHmhu|VoGaHJQ5d!Sw za%OqiW-WWM@V=8Y_VLGI6BT4yoB5zeG*iC#(;V=Z(ecFQh`A(hiz%Ljs@-a$WnwQ0 z_F)m_s;D@&9haI0y^+Px9Y+M1(}zdeNf!%mRjWe>aGY~2dpghJ;gWDiaPPR8c6Obk zD?g9BS@}76(&@mxm1Bsoz z;A*GP;IH#U9MJwf0A)u3`e<4ybLkEjvs@zDomiE-ZH-S(vx1|se;UthT=r%A@i|{O zp<+NhHelYLZN+Wg)!WQWjKksqaN58EsQQFhwdqo`Tqm^Z=Y&BBK$QKCTjBuW6V_Ywv}g2u#=xO8 z1$7If%0lcP?t`nnS@LL{(prh1l?;MAncLUAf+-XU^n!l-tnYu66?~FPM-e*UFI%JS z?a5s8^-!as*A-bb>`j(F)38x>c}lta*1Pu^B-m74GQ&(3Br2x)^@o-Q-Uwsiktjq> z9d3P#*OmNy!AiB#n2TDlS0P@^q!M@YKNw|I64D_iVF^DNIWm4kN+~)ZvQow1{IrAB zqZr}&z^X0OVN!(8AP3A58@6-Yk>X|C;B zn6J)1y};PqSi%AT~85MKyiT z$AZqVraR`=T)F9ZBU>;;dvme&4j_a}+5-^}y|w3o3U~)m1W=Mmrk(9sGKeKBlRjBw z9w1h4SjM^Wi9@TDNbEU7%X3;#mrrCkrh5Vnkoa*rIc3wz!5t;VfagU78LEbR!RnJj zcQfLE=WIqG_d3*2F_XWi(l{#(=P? z@xH!mNsP}zO%e2hG84FaxAet@BDO@VB|=+~bm#0zvX7O>-k-GUxq9XrsAd$)W(oC? zuEY+K6;3CSA-8;xQlz$-nVDEBoqHGu=`pm144I|IdSToH?I4z=$AG@0 zyf+hvBmB&6J@XW%#lWqFw^kc_96EBp>uP0mz3YsYiWbH#fKq2%posc1F-+>+;M$`H zzKiXKZ@oj;sdZgVFZecX~<>8BAFtlECpQ=>sDv zfn6*|uaiQ+29!Y}T1xE+x2SPf5r>Yg@rj*1OMP_7TjcKMhO)>q{lGXjK+zR|G5y00 zMUg7#Qeb8$2xXN3gGTtA<`?Ib$Pn}U22HNA-T1NX>@Z$r+Y*hP81{6)-D zuvAI8n=xa?`FWo0u~lM1iG_l^ZV-ZAl`gt>1z^e#f6WF$+|TLPKK_Li!@YZDM9Uio z%Za|JalH`%3P1vava~*b^Nz)>W#24~LJbMNQWkn~==`!#1@)ftldR`SO*QzcA~R`>3vK>w*3{Az+d<7oQe{MS$= z86_~`VECF;64?Lv!Z)(NN$1cStq(U4k&6io_5&$f0k^=La|I!nXxL7s6(X;Yk!^f<4q4$MksSN(R4t>MtJX-t zZQnkxq(72e-2Z`(bN#T5s=>%VKI#gqagfQd(5u>=Ie$yUPg+v0e$*%#9%Tphn>KD7 z$F7s*0!SdEW5C!P)G zHtdGu$F;?hRD3w!4LVE__RUNg{P^K88H>b!!;KI$*sD75wBV|L%lViSMb3v2kK5zn zQ5Kxm6+hA|$KX_2Mgd*fohiuv{8nw+$kLDPH&;mt0iXdx|_Y2UE| z(y-VC+-mT_f^{~F`tB{t^wx--_uc7MSGu^HrB7!yXxK3C;-63~x4C0oNS&G;8^JnpdT3A>x4k?aj((#Chso)0>7noRC zz3SX%i>n$y7xIlw-rjfEKwURU>NhOEDOE!}0Kjloy%(Q&;vh<9xY*5QE{G&-ob=TM z&>)F7ejg-CRmJ)0_Uh_J6T2;@m@_qtm!aH^Dr-<(`NhZIzaZkzSDhNvmW6IGD1&+x z)UCV_Qx>E>?ph2o`BHZt6p^lXEB7e-yL3%4N!6gn<6DcJ-6zJrZ%ywS_o}|T)|jSO z9bd)L>vi=fW6P6dz;Uk|aU4shGl3>B{KpqF$_V=TzlZcHQmvLPTgvUsu~RV|$S}e^ z#?6wuZ*J;RN%P z$kU}1=N+E8Stq9C{6!d(EP&+STIFg0e~M`M(@LGZ4RYgBQ);*N8)-ojUjmdT7O^f+ zo*}zHHxaCjaUd3gfoC+Aw80U3WM)OESF$L(L6SLR?NK_Fo50|6d)6f2HVHU308Z&< zRmDf(ncnxune^LkS!3pDmPluVZO8xRUkg_=std;LYE?lds2%yQ5 zpRd)p!7)kc-#X+LN28*ck{qD&Z*yc{%umzTDHD8vp(-@uiR;k^Y zhxNsla=(iKo#7_S1$VTf=TtR3mwomA-2o9ol`e?c6Z+*g)B8U)bXnoaNs3mf&J;A5 zvDcegh83=To)CPuWZBG!+AT)h)a#zwp(><@d{F$n-3Km{gN9#k+KG(;EI#>PpQmJ+ z>b2vn+7ic`8C&=Qysa*R6Y%p1p6;T_jV$HQb5UVYv^>3?fC@1&aHElksw|^K< zXJqwK=u;^5vYTNiZ=%z&;5EwB~*oXBcCPvb%YVV(p%GM6&Uk!b~4HIT2qg%^Fb~St}Pw_%0 z@!uEWLi~0T2D2=4z5T(YbA09!_s$@deO)hA=WTATD9^HH5ZrjG4C=a2EMKwtv z(`r|Lwh4;79f8g8;})gP!=0z|MGGrhn+Ut7jBB9Z^`Ik?_iU}V1v*+*aUq^xJ@|x0 zxJY+U216>fd!+hs1Av?etFJB3^*>g?W;F~Jfs0xM&T{680>lE7)-J^|Z%ey`R;RD_ zM^{Rptl5RRIVOq6LXveH{Ku>`4Al}g`Q{JZ*i%zavqq#Wm6o*LDKn!VEqUa!^f8Q( zCK9MSNrg6}Eksdwe^$z0z|SVf6K~?SW#+jUx_)X}D}F-jmZ=d(FFonHS}o5mbj=g8yf-{kG7H;jG++cF zN^G2L#z$n^i=YfHh{DGc9wX4Zjg`4qIH;#2dEynxhp#4}^$fMP!kf zmT3Sad$;#AfLF(()Ci3=MeC03265*ye)_?Z-d7w~$ZaPkia`@>>-o?CocTY~du|vE zB|}Xh#E=X0rb<5pDWy1eenKw(rJAiiTquV$kDVh(4;TzdSG1`O8n~R2HuVWtp2Dnq=7~cw#rVeCLLH@_a_4wTOlsziMgz ze3szb1Kocn8C<%2nM^qi7i&z4h|%>xk-F`=XFZ;$q>eAfjD-uGUK}P~O8wk5j~qM{ z&kwM(u`_~DnC#wT6q%(aIYY4$Nd&bdss)f#r^dyUlNG|b-z}@HvBCM;!LEpR84>cz z!DYsN@04U;GPGi>^{uP(kPVZlelp%q2@hQxzO*}ZNACS;FU^+@Avt~G!NJ=8jNYE5 zr)`YMyy8}`cU@ta#5EE5hYk<~w5@gJ>aQjbj`HIKzipLR5PeOP1Sd6yxNlz$Rrd`4 zamT2#PDTeAOaFN#jcKNgx{0*3Hay&ciwC6dqH>|kRV5$P^7~f%GLX@u*RIV_BTRM< zzolTR>k<|Z$mac%#tPuBMh-ZLUxkVD2svqJf6F!X@ZVOjRZFGK<*u_AUua`n+aU9}TS?Ac(KgcjiwQX&P>E0z z6P2zWZW01YV9&A4?;v_Y_{nlgJ5hQ)ViJY-C$szTQ3*T%?hz&cwkT^}3via~f0JRT z0e>gM5K@nu(eY8~g^W2?Jln$vy#hVaPC-yr8w^{lI$#IiZXk2uRZk805WQp;HPWj0 z*msPivc(mt$eI+c3dco!=@K1F316$xCeal|<$=GZc56v{r4Gr?*wJHt4^!%97po3nE^qr_jxJxeDM)6Fye$YD9#@i~kfUH(8;Q=Zj#5 zB3#31u{&3}etlF4XtT(_DKA&|EwGuwf;L$R!(bd#Bb^&M_&p*e?E9CJ(lEg)!7Bz z%mRSz1oUwmy@7X_m@DfMa5pC4SrWrLdbV~!7G6{N?cu`4IdHVt&vON->{@!;o9KHa zbVeLJgOF)%mQ9~b8)zcddV6~l$@ShE;8-#gB#oJQ!fz$*?xM8bz$1#Ftc@n;+F*hs z$TIV>nbEE@Wm4SvOggh{GhxNKl`O=gL!2q<=NXaC$HzsdU^ZroKE zXJ4IN0g* z9=Ia*f#Z%L?IZVCA5D0ssk^o5=k@2kFU_(6Hj}_+S)XE?4-bw=CV|<;vg4DvxTU8d z|Ca`wVf9Sms~c^YJ>GRO&EI`rML$1zwIHc^g=WbXnj;t3c!u zh3snbwn~1-bWU2b{*e@Vk)T@Z?*W%np-=cSh1yex82wv{Z+c!G@|p1hgPQ(LNOw z6^>;lbeczr+=$hV0{zAId2K~ z8~*I7_+5Ba$eJhOZN<99ya|hz?`*BCM0YIW18!qwAkht=z$j~-LX=C}d*8|ISG{}D z(dL1cbSp(n^0c0{nfHcMt{xAf^@wM|oTzB$hq!KDr2L>Wh}5TyTQ(M7zJ8VU(^Gml zzA*i4ar(&~g7bmoia{1a8CjeSxNc}~W$lW@hscw_cPL`*3-eRdTefaJ*1sb*G5AJa zYi!IMwR`kvDEA^Ln06^hc|A#O@yjq5@P$a+gLyoE$Y&h6I|x7{W4XP=O@1UmKNgw{ zxkY|ZZYQnwWAM|+`S(XysOSE5F+KDdF)TiPx+VME&>_p~mt`E~VxdAyWJY&YL;uYa zZ%DjXyvZ2(aAv!rlQs#hXxf1TpXF?)v#)2<$m|C#8Wv5lUA(;w z^4d;u-#-k9z6vA4YM(B@_gupz!-M+val$7j5&xr9@6sQT79!mEtmpOW-aMsjD{2Ae zDUke~d6x}~ZpwAstJm4&`>pskE9#Iqi}+qq<7-s-YybM|uanVNh|*ZFAnclH+AD8V zBsy0VH@x{Rv2{ST&=ip4LWs6s>t3<2LGFXv(FLtRa8LxbNWAidGfmVD+AgJLjWB9- zlVBz(Nzd{hnZ7{}rW;ja5wLg-Y9>iHFunAe@KJb%q>*(H4-f0nvu7OY+qjDmfoz{0 zC+;9Rl=mOt+4*DzN?Xux3w|R1nzspo&J850A#ttft(EK|MY_;LgfgT{D7^GJ?IjB5 zm{&!ESOlaJaNjOvy5C%IX_Hylf0a#RhF4B*`50^-s%!_uWXmvzpG9R^uk(l3zBwj z30nkENan}YW7R);gNqRBJR5M~Kz-xX(O~m%5!qav6B8u`kG5r4_!QEp!g7(X)~5Of zr*d;$1cmc7OdJFk#G;Yar*v!Jb?5RguL#1G*c~;0jWR2iBSVFZxK&j+#-xk)&(R|2 zg-{PMm{@m1gv@*6`V*WVZ2pn5Em~ z!Gy?<4p48ENVQX>IdRzq8;&-6PW%aYl5V)u#pPO|1zeel;$gw9wD4F|Mwt0gA%YRQ zl-Y{9|1}6K0+TNLms1)jW;b#wsKNsNQp2=rJS?RPDbgqtDJ*%;M<&wD(bO>wwOixG zbzbMuKS)vy!;?PZmaK16#9BR`r9>kTOPjxysWUj|@_)xZ@uCnSd@GTyJ2CR*rg{CC zj%oJd0Mw;K0^H+}-YrFfju=j8veJL8RGFJ;xi@i6eC(pE1Mnq|gVrLZpje~2SR-g# zu+Z&W)eUDV{#*GcHLntV<~G{it&EBUrhGq=%jN}JB)C{Vhr{FPY9Cwo?x($7|0@d8 zA%!K4Tj5GX6Zu!5)cy%L~-x-y=g5|S4sfb zf|$K(dD|k*qs2$cM7`+W$Q#wj9rOG;9>3o@eZi%}H{%-)v2jXHnd2RGrFqj4?bH(b zZuL|5|MSm3)m=}a)vA1X>&Jfk9pX&JKoT}OBxM(0h4><RFb|2IJk>uNm1 zuX2=i^_1t3wJ7}xQq7#o20>?RrtS}YvK+1^as$|{u>D#3NDX9Xr|T_$pL)tQH$jaJ zOA%o^>Vi&abq`(LL^QC9YRs_x(KGT*FZzxs`7F?k`DJrI+R^X+4+b1sa0xtJiNu zhfRYUqqt)_mX72(e-^PQ_*iR%DN5D=@jzCQ zu+8*NZ~vH9fsIg-J%v5ox%F--^M?fBQx{D=KyXD}_x*|5O=fK|2Tm;gY0fpQNC#}A zBE->~XqVEN)I${U>0xeHE$4whpQE^cxtdZn)I2vbel%qswC(lVZ!LfncVKq7<~3m# zb4t(OFPUJT>GpEzfb3B_-qt`9@%3D6;~RMW1cCcQq;=e(=(OI4|Ng1AW5kSQm!GfS zGVRy)LpnN-Zr^8p)D)l3Q|&h1(rG`gcV^6L_jo&G0^e@j`hG)wf1C@q^8rKmX$nd? ztIe&o=g%(hWCiCm;t7C&E)65sR{+d&|cv^F>G_#)q6y{3p8kX03Y zUhRs`nZ-!8(aq#90=9@e`p@DQwNOX=C`hrmL$>c*<87r{rgY--(q_V7l=7&!T0BpIh6@cxW4zC-VtDG86s9Y#ailN} z#NdK7toZ4yrA}vO<_&}$5ps=Vw8GT@WCVK=+~sJzBBAG)+JvUr{b^7wC2@{3%rRTr zj~$icCIlnE#et`4dw<5WjD2D#S=zLH%c^&WQsK$vnt9${#5LRs61@<}>T);)IR z)v_6B(r+0}5h@AsL^D~b7?^=I^1X=Mj{T?#=EA&sz9NLDz+(pSJJDBIRkvV4w|{y< z#N@09|0jjV2*W_4nE6?^`?eS%we9Phu9jgm;_uaY(s!5*H5QE;BS%7t=hxpT7Y3g{ z&I0U|u&r&Q{u8#9YBzr&u1UH6X3or8F`jqEUnM7I_z{;+i@?PqhfocKLYJ(aMUwXe zx(-cgl3GUNpa`(tto~8*L$kmoF6)G{`Vi(2~dF^kc4mOl#@+@3Xc;}1gI=|WORQAmK&SFMh{iw!8$EU@4?eOP9 z*0AX}n6%JWvPepxHd_BKqJ0*@EPQle_jZ9F$%x3LgZF!UsO;}s;<%eZ+BvH`h!ozI zJbx)*JmjtQee%A2&4!kn1jPeb(jBawb~^t_Pt-_L$xBc~fPU@eUJ#i}8bRB3lMUN# z%2LU?<@Gsf&CRbb^iM9Act75&Z?@P(;1QyN&JSd=Xkz18eq8@6w;}*$FS}5UKM$1G zop>Ouwi3?Y8FUiE-kEqL9@6c9`1y;dADxV5R2w?Jt-i4NRZe{8;`Hhd6;)HJ3Oamy zr(x!o?bY4oZ1R`(r)L%282K{3xI*RK91~lUt9?uVn0UF>+)bm>!xxTpzg+jVXGoDs z+U#dtI`*q&?VPydTYl_$r;`bdmG*pjXL4+*Phmu^W8b|_-^Uux%zN=*)yWnPx~(c> z`!&#Ltz@;L*%XJlSI?zAP5otrY3ICq`yake|3#_jYpKbyV;Z3en`2+5T-J%+raq_J zLX-TnUHhd)+_t?EJNe_fVFz1Zf3LZ*X|Y%6qE&y)SoYn)Ft5kHhYuY*ejh}2dm%`!$!*w*gs*=g@rKsDtBq^903`fQQhG37YJ$Ob*T0Y{GB zVPmtVWpw4>)K`_|qs-n#q(vv5JY-!@$>7WQF8&%e+RH=Nojr5rmpk5G-TVVOx^w87 z2I@ULdeCU8$?({D4Qg-S-P(7DPWgPFyRRbCPB*KW&HhT6yy@cY9$TAyt*6v))~B$= z5uu?^w(J|w0cXqm*S*5mnJ#5qQH7WBJqZnN!bh|T(C)%6E zGFBA^u0aP#Sa@w@I`K2fumYgZeEa087xp3QW7Sd*MkMZpc-cdB@#({d z>p)YbNCi~7vDt_dzeu5oL>l-sf7Ouaio9P%(pp>`5HjAfW~62SzyD#6_4!1Jd}-bK z>z)ik8y?26aa)nDO3f1z14&{yZD-A1p!7?ZGkA7^W(eN4cAKhrAd=`iEY9b6r1_YDeVwYg5=Q} zUKx1*@pY%{da{WY$FqW*kmj(|fKwsJt$nl$*qx9zqJ^(~H8E$DzsC8P;%uW&kLd z8uvBy*IyUKo|q6ESFGz9xHe|kYa93W48Bhi4r*#7%y{8*r?5@dz2b)Yi`PGP+BMU6 zPxyvD)w#Y8wl!Gs{L9mp1K${>eF`+lEmH3Ix^mgNi><;bOSSV%@Wa8v?(6p0RcO zmSqoDXU%-tw8KQl=&fJe$4#<%{+S#KGz;_IHS1aZePO`LSNFz6=vO(Zo;!Fd?{kW_ zQn#rM-@Q`ql!UVVn&N!#@TNF`uSP? zs#|Ie)SA}nV3Xl}&o3gVVu<^1s{QU32HIpeEFacvWBq8aGv^%R_vjru93DRTOcTn~ zrAq@fT$gZL1Q_I+TYIeRIbexfNy-6_?~YlAsy_5|c6N~G@6Dz*Hfo!^s&e}&*Iijy zwraF@lTDkt=DcmUJ$Sx}bx5tWbNyo+tBX^jG%J@}J@=s@E(5(4E1oGSRe#o9IO#O$ z8ND;yo0Z=D=F)N6+4h@KjBaf{bEe1QS8jE6qjEwDjeXzOcZjUR=*6RNk9uQklWB7n z7pZ78exX*|;pLJ(f4rQ%;CrU|OUJ}v(Nm8rH+y$}tmumA^~+!Vp=@%}^imJMac z6PPfFa_f%>cL^jG8PkljPaUl?U`=SpuDO0WA%Wk5ZZ3XMvu(@jtB3reaH1feIDgTJ%xUYG}=&NT{H(k$9dgw12j}KyV2-0DW=GNyGmw0gTTGqOv!pg}Q)^ zxQ4@sO7>KRUINJnyg#43$c)tK|A())j;ktd_lGxvf|7!CgJRJwEg%?_B8_w?ogxj= zQlcUa0s_(?(%njmAPt+4mQJbtyOuN0dCvLcef`WhGdi-@UhH*W@h#|k1GT#;AdMys zHxBHy5Cnu6{MogD4Y0_2K#7=XDK_-eY};@Opl=NK`w7(GewBE7h43dbYpI6-;`M+sI0qvYC` zQy=KRAV>~5E0LsGs7fj~o|H5)?jn7@^zGZeA2a)x5Yy5)aM=n3MH={uAWADbVLPeM zd+D$vfK3@RqpuK2YVUeq!XPP-+`&;9OgD?cj|ABs>>`fDhf7(14U35xIgt~N8uAbj z2M*x!1`B{w4B~16yW4ZHq%c)j_zetNBw}vL?yk$_`$|f3Zf;dpj@_BcDG{L4g0${e zr4V<9jDP@zeKuiX84~rXA;m0U63?QLUg6(U?h2qxTaI6#8WROIb4X&-05|v2g@B5l z;O1#5kp96Kc*Bu*2x{URj}#Orpd+E16W=ZE?1*ovk3?;ZTF#PCEuIVv(SIGaK10`( z-b?$*p{_DQyuQk79t(vl%%fA84LPRF{#AofdM-X)#J`($)y3Q}^Kc2(8x z>Rk-Ig0Z~tn89;lzrDML#$(%CDSh+#3Tv-*{<_q3U4rK6e&l5N_Qp|w@Zt1XyvT#j zl_JarI_toR+^j~BVH-2wSZFBHt$ysiq~9?3%=_0iwiL9AS!KF%r^%!yr;`2L&&%9T zINf#4>nmQt$)Bm!rD{lwo-yJxU=wv_)MHV}P_ z%^dMQS(m4lH3NzvN^`o95}x#Y_~_L0O*`ALHA$Bu-tMN9)kmU6y&L|nxp0UDYF|Hj z{o=LW)wp0y=u>ELy!#H z0$W$G`S~%oiA)yY`i4wnb0%pd=CZ8 z|>W91>6*TM-g-m05Sxl5O&|o>tF9c7^mPy35nu}IN!#`ytn=(`Yr9Ab0;BgD0^u(;d~(D^1}*@pN0Ot%P6$g^a7>B-Y9WFd z<}pGL7Q!WkwV+9-oa}%;$bDy!K>K#&enL`1XG$vU)es70msTMNOaS;ZRhV;wy9lI0 zAWL1cgg1%c#9`UMhO#EAWZluah1PBF@s_xmO&CWKSTS(-AjsnTq{VG5Un~t}_1kuM zMOt*jyIdn1Te~eE|0tIep|kTf?2cY-*EZB;7FV5R%tJ?t3YZnoGZx$)-?{5gxE&N( zPszS2$p5(X!}5Ll8(f1;o59_(9IHQ$u=VN~s6?3QqKsvxX=kuYgGgNn%ycEy0>(7F zk8Pc7y#WXiLs6b3M%Tr9NhPj;*y>mN3(VN>t)Ob7)qW zFg)i>PlRig^!y%&@DJkVt1Bde?5KDtu3?c??sxn%TTu?N>rQ7-qX{C#tKRhO%jlk# z;gPl7qu&o!mW@rI?R2j+XN zl=6~0$3uEY46tdKoh-Q#4UuTFtQ^+dXU^29Jehb^>u$?(bs?yC*utVqv*ti2QKa;O zahsS&cWjv;EvN3M)xG`iw5O-*$|0E31oX$RvHroykFBh@-*ZK_?X3#g9X-E!dQ^|U z*{RRgK_43Ih9_$GcL94ST>8itFCUlX%a1_&Af^u^Gg2~HCqeQ-^!e)Tk;Dnu5kY9j zMYWgBL{49BFLKK)@?)=>33s_LeL3Bba=(efQhm?W?th!OJ_?CIfE^|hfw22^VF+l7 z+uJe_Zj1E$a+;dMZ@nHw@&hkXw|Z|js0C1akaJ1{i8N9?>n8YTA(7OGct~>vxOLzh z?gyrI;h|k*QDdJ+tN^!c0f>vS$`G-9MQmbW)2#Fxj<988_|U)f^Fv4ied*p-8$jRr zC-sX&XoNRT;R&bRHaCEM$WVa{A7In+Z!0&na@5uG&pGHI4(A&|vZ!ft3FS?9mN&}TglTHJ2{s0p}Ts$)2BxeLl57@C@a%cXJ z;kI$PI}0e&UuJMrf|6@=c(1BGU1&6jj ztgQjpMyPAR?|?=QY8JTbLT`nFLj+tGWDN`q*3V^16xQigT~7tg$j~3wG_P(L-oWJ# z;b6idHW2iPcrTs$Hh}zyxFaBA9YpuR_rm1-dTM-1@5e89aGwEAD){gwx+2LPamp<>HE zH@FIdA_)mpsUX6KwKf362ni6fZ~i*jz&QzyP5bAY&=P~LFxvw8bfyAOnk<%YHvmI| z*F=IMX$D-rXw!?ko%mnjxvNRqpWfR?7M2k{ynRGxpSSmzWt&lz=UJUo{nsx7#;^gS zmpGbx$Ujteo5W`o^WuWcRe$EEuhC@UE;4HPM7HcWsHc840QZirIZ15p|4@Yn%Hzv4 z)7n5R77C6&&5aVA%lW~6k4AP^cFHo95-|mrB)pO%YrKAcX}qh=;P;R(?ZckkB!*|U zaK?V=E_L0X_WnhF%!b$Wk78bg z-XlcS75vIgpDvP-5sapBF>xo0=`4^o5}9&E_Rb%}-sHJhVIEzFNy+2p#~ynlrg=Ip zDD_MZ;$`F$*aaK|Vd0KUou4n%3%0PS4M%pe~$c z!bACE{#->>6Ta-eFwE}Leb`(-mQQkOPLOa`+FaD1i0`{$;P;Vpnj#DYFe~han7V-uxa;Pk!Mnl6aYt!ZipvHi$o_qEt@<5Y+4mR=2WQtEB zS;2n~ctZf)e%1s}>?W8-;Ys2|$dW_gp^q5+0Rj06t;9GNa9v=!LDq+lZ!9G2Lc=!T z;tF%>*VoK%*sT7?T@%(dii#7Ka$s%-yeouyRRst-Wc2{<4~CLKbwP-w19-Kq1N8{$ zkW5&9)k4>U2&4hw-f@7JX4&TkCn;bprKQidd%Xp<*)+iF?!S}qfsO>=l~j7)6c{PI zF$ULy8SzGU%HFC6fIYy{5`5Sk>X$Z|;0V15w;>f$cAydb8~%tmV5tHkAqmZJRx%!> zRDA#bH}Ow*gbYFo@ZtJE!u~HI3~4TBc%WB&BWd{e z|Nd~W1@0l#1UVqiApYLaUcyxTDuEIW2?c}*uL`~w7a=Yg2w}i2c(W~p=d*2=jGXL{ zcmaloq%S@nVSM`BMU;)n?T8voI&X`ekAhxsSyv2ZL@*w4DdB>++Q46heZeC zU7a9kEv={+u37l$Q$w=2dy&-$@5pH)2*?q;Rd8H*a|l~G$cOtOAs}Q6BM1_$WCy-- z#e?JH2#d)O;uns!Zp-fAK_vsu=?u8LPyyQvOgnGE&J-?!tQ~fyBsh($UK#+dK7g>W z{ttEOKbK#NN~bve^20AeZ==n=6=oUyI4*czk8>{ad~2?Eyq{$K7X5EQTq_D(qT?ag zyu*6FgT!G zW0)m=tM`nj$BXHHMQbOne~+mZ>7WBv&*tW#@7F}^@XM5{g_HM%&c_NDu1Ln9Nb2n_ z1E#TglCsrebqCkhaDIghh`l-QV*ebz_*lzJ6ptzln%~%YXxxh7BYY1mHxF+z)b~lN z+j~Pp+zZ|Jh)=X#9u}2f66#UMMy08;a61_ZM|jvxhZOw$**jg1mLT$LBtRV;)hjnn zH_uf#=en<;orVBmc-g&Y{9b613e(^S_RXhDDLt=@xwy#dFcM_Oj2ruV(Hd=O8Lcnt zPght`K&)Na+-#32@~YiVX|4XEOA%#g_GQM)s0iJtw9Io@Bl_C9Ei zESw!IOhIM*sMTw}NO)N_i>Rr>KDELrn}|)7w#=&XrSl5Iv13T^Ay?@*7IYb)i!q1i z=IH?Zb>IKUY2EVd?Z09OSScl+-zSN5oaN``eZ*V#H=!fa>IP5;5gCtgK~9dej!r`V zOjZHX2ZDuD=q!k^b-_mt)>g~K3r<(y>4);23S>kmxJD-k?g`jnKe2yVgNI7(AfTGK}IOF>uoo;VDKQ zbC@EHR)MpJVDXV0no+$s<_O z>d6M7MD}{6+wKhr(793yo!!6Ws#} z7ziUEp9L=B8)IPm1%VGK(4h`o-W`hn$0Y+{6s9P`rjZ%Gk#LAD?ka$z8C;c|ynN6{ z21?2x@|)^A-&}`*Qj9=aT3T^`XAE$;%v4Icq0gNraUY_;%!gn*Br@=`l$Cz~w>j|a zRGc}>^mMAjh~uS8RQ7ZC#T7CI@y>kcfA;$IhvrzG6U*9~kCV0Lim z2FElWpSc{fpdba%MP^)~(FRKnDLcCjSpL&uQsDXh{5eNOZ;~Lr7_xF{9>mBQ4;T5^ z=>IiN5=`iZx!b>Ssv!Ws^i#gI{DaW5FdZNb3fa{_!-EZ}7!StC&S?_6<;Udr@2@qO zSjgBF3CTqgHSR3=r9PIJv_v1RYl-zpxI6GBfMh?9l$^Z#sZALZyMo^@6Y;i4V)vXC z3i6*bO=rpA2Eba(Nra;<9RH37C=Vv(-@W5!;pz>}$)Wi4_=2q5A=pw~^AD7m^lTAK znC-i|ixo&4{;9Ih$4r$LU{yStU!hU=E^h4DpQvvqS*z>bQfH264#H)6ruAE<-t7A) z5(T&CI9o25qGZ{=(*(_pZ5$kb%j(^6S5ey!JGz7@CXp*=Hu2?Wrc&^${?Lo*zWo*k z_zQo+=v0v?2}wcghU^I@rS^y@bh!hAdzvgq?Y3SUa^|JMx{*-Z8MZ;H2fC_4rX}bB z^qSdQMXm%zqgOPFVPK9-a)$&fnzJ@0m9Jp{%}8cepe1Dv;JlTD{ za-pG14pS_7pARP(osvz^>w4vk&24c}y-`I?o;;nUllH0x+c3V*v-h!3rB@U`ea-jt zE6>o|9?}p@7ptZy%}{)5eaW}g%-7Zsk5-7U;_R5O`Eac#yN&+2j?b57H9l9ei=Cw0 z<6g>GalYqPy5%hx6cB}fQk7^vVC0?nhw~zDHXTaky;eF;5_3wB7FUoVB2ScLqa-;8|TbSU_1f>YizfMiDfeAnv120=+sbjb+z+>p|ZsMJM0a-2Ki zB!i{WH{LgXA3zD(nAx5G;u~VRU#_TXeN)LJW6?6f@@wPE@^+R#COz0O#oMk2Gp*8q z-4LU=ojyS($^JuO?>edeyjqLS_wF4SLc&`>fBZD~C5Dqc?bF@eV}tH^CIZ^b>dD>X z?NkuD%ECi}g_6LCVfBhrrU}$0(KG6M@Z70y+J8GRV4+*{sY0Siv!SH-`MQSy$>$Qw zsMjhH9ru|>M=uwZFR(H-(4@_i_$WyJPsezCwGVI z1_v&K&i~3aP8(TgXZ&F+3M;EH9+O~!re=BrlYt}W1l*r>66@%~JI3jFN=lhjye%~8 za<_Jc*XPYHu8n5{MT{^fSLgfajR5d1O-xkaexAn%|8SqCbh3tf07I-Ps-YY`*><=) z+WWx&b2rh>p#i|Cn*rJS+P>+^_6(5qE~%@lL#U-%8=Pj1eSHrX zJYe58wfTZ1v=w!UtG}eA1l;yElF7A^i232MvGmWMA5l|63+dzI6O?I~5fyb2ieE8^ z5Zk1l0NWR_;4w5CEQulaC*f7`mqr4INEjI-0r1FQ0%V*U)EFShmE>QrL9S{i5xxM> zSf}9J3w!&`1Z=i$-??)Zq)iWD@(0clpe0T>ZL=REaZI3s;h6#ShT6~ZmauR`Tbr4R z7nAJY_x5%68YE?cd*i_9#d$z|5Jk%JR49ow-qIB)-5_;9qA}muLkJr}r58?IbFR1u z(kWwO5cOe#-`M|B z8lvK~qaFdvq&a;gdUNaj{S2hv1yvU7B$%3vP&D=RMUBcz)t1zPK>!rZAADx;eu?vY z{BSx$ZDM@rV_&nvy)(Ap=92=uPDNQ(Y}8-pu;Zk6!f0 zMvo}ZwHuGtv%bS4##MU>(hupeiGm1~ds7~&e0V{z5$PS)Dv7yO*Lh4!dk4%6URX_f z;F6O|hd(<$&f6&T4sZT!>Z`qNSX`$Pdh^=T?^ML7w^ys-@yvMgzEE^#wiCj{C`lOg z3k18r28U)PZ?zBUE5kF(lkB;7smxY+Rb9)3_g0y0eNeB&11UMV@HQ|LYm&3$~b zo^D|BvAQtZdTh&&dqVBo2 zmX>etg3Pr`Y59b~p|jv7b1G$;v1jj7Q_E50;mV}vEq+E_y>{EWw9I<99y8IFmC}~} z@#8&f26J=O<0JHJ@pJ2f&B8DrU*EMVMK3S$Sr@&kolBVt_MqrE3#`UB zZM#ioB^zCQGMj%E-WpnVY9vfPOn6+-Ug%~QB41LBNh$8wXfID$9X#H*eLpkcZJYYI z`BsNdZ|mYGRM!g$f*Q^q<%y`2$5`320((C!eb4$CE9a>Lt_#<%bj9XB^e9a$3f(i-+(va=Z4v^q>ot+yUSv%ot!1%goH%7` z^Ak68%+rY4jUR+k(F!K2y^B11ywe}XSx8npz5J8K#5{NT89wD7;vS5(uB ziv_({R)Y9_eW%Fdgo5lKj#^s`e=x(vJ$Cj>wdQ-ZuItP(oyCsk zBBv+o!X{=G1+%bO2EHM6gv%r~p{jV#mH6c!;X<6VP}9Gfbggt$ImfSrURC-v zC5fpFsi@#Bbsde1qc&L6<*GS#!;uSjSpIC~lXkj09VE0d;qr|HPkSKK-7S7Lw=fez zpL|z_xCF~^7(MK;q5Mm;t4pG>Ub)wNz3Wo5)q&TQxRfneYn=3Aci9kp8zS^N2lw?d zk{;gxvzDt<+r4Bes{J{F6TQ6`7(KWd9>gqrui^NlrJegdndHkV?*Ks{G#G9w2!Vx=)@Q zRhqy{eN7MVoHGIj7zC();8kKGVFE5SGAlp#GBY#d=Ha>R=~+9<4~++6I>OGLCvVCI ze=PWAZLN4>Qqn{1oaJ9EV07W-g}u+r2UT|h>Wh_?m4UHwda0u95!&xd2#g!D-~_vd zt}pz1p(Q1%WdToZDK{&=gxEPb;lSPxkm@$@=4u`)w-t?o5-olBD$}w;<(hwL>NM(I z;NNf1|N4nn6Jm_Otm6`__3l872jZv%9PBTp*~IrDHW_S6U&4P1i;5!jMPPT& z-rmm;wwvEx6T;A0A!g01f#(%+~|u-A%aAV`|*XAj2q+4 zuRZfjOi~x3?)pQdMr^zV+S|fX$A0nk^3kDr3jH~vto1h6o9tE%zekRuk_$X((X~3^ zOa1$X>yv*pD~eXU403Y(WBC?Pq+~apGH-HNHznBWHQLr^z{g|JhEYTG`A48h^F6hE zYmvwamu0PU1NssBKU@7pC9{^@4?fP(@lmy`DKv)^k}?fFl8aJ#8H-kpa!$ut@s1}{ z&7I7){P|vxr%B0c(5-vnSFb{hR|d)K`*X{+5!ggg!SH2j2dg!8BI2K<-Lr`@CSmR?(IIyTZg=Y8lHaa7C-A5 z$sUH@u(qYYx++rOZXdd~JE?WYhTh9_lYQA)x7X1&dbWB_zIbQ$8m_i=*mjgL6A40SQ~#kt;=VWuuE-JjXLo~>^g5Rk*B1NeaBx?I)}`Kd&JCA7_=!Gj z3h*nbr%hD}TU<;RcD|u(UKJm&?s2@S_7QVrMcu|4^DM+Cw|pml7w-}<=b zfF%XfnERed6SRmA(-W%{^!3qTE=UX}d5`j+ZEe$n6~aZ2{U?|8Jc_-K`p82*0<_~e zT(zqg^GrwQ3ee#w6%|>aGL7s9^h;g41PTSe3XjUWmhr!TYqRS-rz^LKp3R8a(N9^g zXUyGpm6AVyeonJugImysE@Hhd^mob885bAFFsf;Y6eQo>mpr}O9DI?;^E12j*9Piw zTk?Mg2y6>Dh&#^toPawYIV=*VwoB_9uu&Qs8ZbJ@Q*++T$Tm0 ziLV=P$f3A}gfxh{ex90JTN4UauhUSF{hd^|Dx$ubfP|v~F6*br)|pFw3m`6_264gJ zAie{MhAVKuGn|EU8KyI1F-FA67zVtZwGVQfF|2SDQ*!ElKTKAEHVgzUNL>pJeNgx= z2`P-@0fB+M1=D~DkX#6OQbr74inrpaB^fBR{b1UZs}2SPca{1XQ`B7*b@j}_K6qP| zpE+dR$AaaH6>(xZ1E4a`>I8$(SRtS2gzp0-0VO{#03>WBFfj)|)F-O}!%Dx0KuCQ!afj2Q? z-dO<3VcYtX%t6NDgalepJnnth#Q=kX8-jGeZb6Y6K8r!XWku4=f%u87DI$Gtkk(e$ zj6kkux`^Aoj1q4xrWBiE@%#RFdvCWp-cmbrQx+AY3B?>REMUjgw+x`3TDAduOsV~&a-sC?PfBqb3ofKx8wu~j{g zNWnR>VR$$;%G}o0Tv3*V8ZLl^HT@hr1-Es)Xr`iLPi8tT$mAYM zOuV~q_!<~~w+efS!3xAkXLUS|$k7qoYLs7wBjwxM32xX-!M#PH%55Dll6+X~RA3@$ z09~RbwHHR5FhBp*YKIZl%HY|XS(O#BjqUBE92}(Y-V0uz6&LGjoO_O`c7vn#EItvd zc~xJ5&N&pSSTj(-QptGd=~dxd&$ztdtbNP2s($($ zRa7R}^yUf{9#PrZgl=gI3xU10ZdtYH8Twvm!()Z*DM?Ayn?^@9bp{DE3NFp2N6s)& zbWSZ>1l|etK$dDR#E5?Uz!@%PCH(pG)p7q1oEJ+O_#*qBQI89pN2RJ5^#eO~)}=Y$ z`q@4Ueod`sw^8b^%OlP=%%Hz?NHpj{zAKUCu8<&%1%#rKGd{KJeFJxYq@jp2}vMv+Z_ z9%rYwjVdYU>6@nDl5PGV%nJ?8>e$AHV0=Rv+K$f_uVXYL4lkrmn%Y=W%x<*Xqb?0) z2du*ZKD1r;WVE+ABxDZR)1rL*uo+XlM6%l<#jdW7@Mz!sdUJB}d~lc(L^ZlkR3t@5 zdwC{lYw@hSUft~qy6xbQ@!975wdJ7NY@s#r%cRuvUAI8W5*lsK#wG@Y3udwLi*j;w zCZ<1))+ZG%kzenzy>llVJ}g2rGw+*+H0r&GpvwZ&H1S~BN19sHhSSpe{(+-@y$b+q zoev^1<@+JT{{?y0E8NSqsoUp1^El_wvMD+Qz>^0r2M5z@SQf*q{CzOfv#Tq@gx6r? z^K6Az?In0f9W$?HDmnYXy1BG89~ul|O3q6+`s)vuuP7ugzIsg}TD!G=Jwz619=JOi zjId_gs|`l^T`DTBEx(7Y)nzty$e5Pq7tGmTMeF}^xCM{@%$BJdM#)10jJQsNgH?XZ z#Kj2=m&n@}?#{)O8(16~^sdy@uyfla^1Xa%WoT?nK!JY^dQEt$D_#_nY)R&Rl01z) z`1|13Sl@QU8>yt*^X-f^v`;4{Z3! ztWLvX&DWi3Tyr>QZpRQ3PK=LJO#`b6{AHR!N50?zrWIjDcP|4i9t)27DlLl>u=EA} z4+W47)Z0wW%|SqahKQ*4$S94gSd$%~I25w$TV7EMzYg)U7XVR%fILD2XdGm#VN;5D z^0=%|n(Eq$(1HRM*ixvgSF^m>>5*#6pay#J=g*%Pm6XV+yv}I^PkR%B9<_%LeF1mO z$7Z$c|Nh+v>bHq$HjSWtr7oS;62RRZ-A8A<3#D zC70oM4yi(0n}AVU43?Lsi7MA*p+K^*vQ{V77Z+bdf+lQiY+|0lEDnPHh^!Q^D&?`i zcPug#Of*33O+gxxwoMqP5D2B+0Jyu@y1ej`;;m*ShL)E2IyyR~f)~N`{u&q8dbHrh zk&&F9TOws4i^H5VhPGtW%ge9QSk!PS00R|ygmgiEwBT2e(krj}HiwlrRRyE;``;$- zzgE~L42i{q>PNxcncsV08#UUP~zy9XekmD2X=$NP&aN7 z6%{oGrZzV>_u{VUUIz4{Y2Ypr5RilQ3GAj3?|NXa6GD4REj&~}K3k7bOVR*b1pJnz z!)RLbhUze#x(QKRJ>u;B4 zTkR^Xj*qtq^0dgh?(VnIvf3X#Ka0%DU-+@0lJ0cqv%NL`ne!GJiLk6zBdqm0(R0Y( za&leF31G9ad9c}T0y?#`cQYKu>-B?%^s4EX00R zcjvJ7@EVM*@{Du0Z&;(t=VJNqy-b%+0qN2_&*G2%vp`+cnc5VtRgdVAIOZlo!4f`w zvk`Z;ar)CFN9~?A4k=kDxEx-*M0s;f6^w4q!mbdYXqC>D$^~f%oQ%Ojjf~Lo3t(kt zzH*^uU|3!nvmSBK_=EZKY;9w&>~SFGz?cCA(6_WgQk4d^H%z=6J6{&}*2V3ZXnEu4 zQd9Sa%WWk*JoBmg(O&NSv%`hCmd!)@S7%Ck*(|%ae`;{?Ml`!_*KCg2IvQA8!;PK} zA}ccMeg@`rjHuTMI+2um!eEZPu%fm{EKz|Krw}1D84Pb&(I=I8yTWt5mIru5jBnoD zz`C|~;(ct)9td?YHsx-nr&W%xlk3Wpuci(MkuHt>j7n$FO|N-8+wcvCPxir!(r`l0I^`UfX~4vl*0r?qYubq^z^YZ52!&zFZ3IN145$0}sx2U~Y*CCI$^J zD%hG?hK2OvFgryEbyO}qS}?mA~odg1ZM3;aL7+-P27-gE19Uzf)E#d}ME zov?T&^w|bYBVCHqVVg?J@%vY*nA)?b*RM>4H*T|^RxRAy5J}n}&Cus)%ILn{QIvX} z7#~d?`h6tcyF`~$-bVTsr*&<*Mw~K+>*DTQH_9hC;B5u=D|Wq{Orf} zN~g=Y*+t7U<9#~MSMLKoL6%ytTCk$iMWC&%%li8d7T-9nq5W0*7;MzDXJdg=t4hyV6NR&4m($XsZ+FR)T!JR<*>(rfm^Yj1Awjyw3G>(i^_HLO& z)b?j1iQ&cLhp}w1BZh@(SVaWAqouKh#RZu8P!Amiy zQ(nAy0j^Fy7b$`81A&i8+oon_M&RfTE_`YU^8g2cdM2kdc{f4YR%L0F-!F9A=5@X9doc&3Mo z^9u;rYg9qPq(RW>0olnQeoU3%X5>jaN`ssP^_8P{&}rcc3JSgefg<8+T~YCr;hG3| zltSW|IM~<`rWURE{=SR0fS5L*a#7oZ{tGnb-2D6lVr0yb;LpN^#5Gu;6R}=ZO@Tty z{^ZcvLRI-IY7_iCKyu!=uuZ*r#uo)Fv^%DzbExRB*OQHaf&e`h8VT+hU;%>#)~Ozb zEgw39`jb6c&?r%D5)lz0rdMHMhvq7KlP$1)CnwL?n`nSd#6zg%QW-2J2DL|P0$()r zUBv{IlEZ$8TszvD^F5djT0)`X6zWCRb!=Ndq)FSOZ4DY4vsE9$J7lKm-})m4ei`iS zx{JAC?wAwJ+L)h1b${5PTgx%0!knx>hGL$_vkDtGVNa0@fq`_jo%P1@G@QEE9tTPu zS@8n&b)K7d#zLXq-uPPGvBN9g#}5 zOL;d>^ybabs5;4VzR32rIKDnAA*F4;EZp7s1Pv99tB(Q~y2%Vj`7gq%|4R=U)lP-i*d*qlO93E;0Ww}Y zJYQ%1dNYtJc0RSlkg@){o< zrUL1Zt3inteMfStfMk@DUE+}ltaR-djKZV!70#liPbyR6!g-Sah>$){&e_kPm>lO;kohM+z0Up^UA7>slu!Y5 zw@-TDNm8@1@o7T(MUi$L>KK>mGfNr*k@i~g89C+KVZu{7R5OQbP9F50c$8jBx1Y%o zYA=s5J$noJ+`&geop#IhhGd^PI?n>$c`4wgS3y ze6+j9U+5FMn})Ljr$U6}ug`ATXnbJLm0J-rjmvp9PvjA-eOY*C+WB~+>wF92S7|jb zaaj)1V1Lqq_x#gql^(w_?E)T^i%e^1_SLQOEe?X>157yXO7EqpP z`8%>fjBsw}76qL5^AfF<0|RY%j;Az}b7F_2_vQ|cw{b3Cy;=6ND=t~HPWPnJPJg84 zPr;TqQBZ(@<;$wX_MV783hZ3UF8W!k4vuk44i3vZ@wVK-p{h@^{QOp_hDI8tn4UI* zwV^8Y-qz=arT#uU7a@%YmQTO7W-9V+45;RKSr+PZwVs|FMm`iXe2Kre+OIpbv^bEh z`x{*-Vw0s{-~4C$0SU>6+Cn#cd>QD2@jMQu{W&|C-+nW+v4I%FWa$?b>23U$L+6Y~ zcz=pF_od$gLc%kJv4lsexnOWr{%$%|#+rnzQTvA+nEZ z;CwJ0ZuE`p&q|+~GtGYgzIVZJWpE^xuRT|1zYlYye;G z14B%>%F4=QzeXx12tHfQV*@7@#G3<_?hQtS1?RTmfBBkj1IZ6zD}}Nw zjUasI&m<3pwc@cbB$4IYZ;T<_s);IRJ4eT6C;-!|-N#=avAs3KoSpzsx<0{p8|s!= zAV}S*r-4)j#A{*&L--{EJh+1}`+*BWc4-Jg_eWOTpu64sq=@A7f>akVOGLbf4}QH> zrm|*1*u4@-}?UFQn}dK(t?A8QX{Fm zaP2vS6cZ02ClJvzBML8EVg^|VaNP~~gSR+NnI9hBjf#eMd=A&e#Rb`>L+va2^5t;? zfOtrn@ZXvm9v%Sb3Pzgrs|hb+46y1x7{_Pc^zGihl#{<$1hhTHCf`WK{``77wHb?z z+iAj^DC$B~k2w{kp{Zh6j3@bV1Y|jwX=uP=l6waB_M6~Hwk%Eon*s8m=G!3g9gW3- zR3$)!9~X-2TReN#2tle55RfBRlavG&7}{rzjVG|55Sz`<&udoLF|#V&h&F=zJy6Kw z`Wx0`V18|6`l30I1h!u*iRKek24QsBu0DTyIyw{Jqp;MmVObbpeEcGYbKwY$VQHMZ z5qORB+S7(_Z!6t63c0?SH~>FWnVJZgKyBfvA`8?a0~B2Kbea3S5ZU^E3&UfaxKUmo z{@Wp0Lyy|GLCO*#<{TdwztZcBztsj|NKL4#Vx7in4tD~sC9*0%_MzSiQ(@EdxC>~VOd*<;Um_!lEhpF)OY|@aB z<&(7GVJf~d_eXDxj+SZ6ZDD#3`$NgG)nEOac)4HsYGX&oUGsQCTIqL~n0NfCA3r`c zu2M{RLp{yt6-y)|Q_%JWWdMCarec3zaqb|a*WJPRI@X&vSs@KVHtAU)L2={9dpWk- zwv+7r@#>dpYt{)<4#u8tIOWO9kTJ~H6^I1gYvsL6Uyq)htHR+$SI;eKdecwTYvoW3 zjj|8kQr0uNMM`wN2R~C#F7{-|PW-`$_*-?{=D`VF^LSU=Fn`}-3(u=`T|Z(AD%F27 z1_0&uL4vB??~(+P@*6!vR-E|rdmbAcgw)%m0-^RZoj7!Vgx79+2fAqiWU-^F<^_eJ zKbdxN&ZBBCw@2>I*@lkR$J9MldTBgKQ1FZN)Or`F9X8t&Vo1=2;#*iQn$XWI%)?HCLF5=zGR@vh1ym#72Eyv|{Vw54i<}2ELOU{nK*W z+`;}LL%%}Lw!->J8Oz+|lZ6<4rKQiw+pFdM1{OIYj7vI{&r-^3Bl0_e6m$)3)0p0<0nmWXj3%c?V z8IfyN?v#{#ZJuk{s;+LmV^3^s=8K9{l7?(2(&`wI zW$RjW&_Bn|)?EjCdq{(%0*4Ye@pYgNfqIv3#;I0#_3O`{A8aN1ZtbEOo{a;`9F5B#RGC$ z#PO{qq)LB~RQj&|>&`h4L&<neP$9oB$e+k;Z;>twS0>rv=wpXx_7>$F(o783LPEY)YKG8US3`pn7JJtuXqj+ zHV;(C$oR}mHb6bk?XOMr3s1DQol_Ru!~tmdsN4YgA0B=#ExiNDMm7_VoN*B54MZb1 z^gY})e5$2IO~fGfY49E~Di&y^VEnMLwe_ftnu^K*R)I(q46^QA0{fW&AZUQCO+a$w zMR-MGcx(6+6cpgcpwV{a%9V$>o$x{p3)|CkbFCHfmbQprSmRU5K1yqr(Xl%OP6>EG z%+%D@hO{LmYEV1lxVul#6Cq#9Pi-OMhbVEC%>Nmykhk^3uw(=e4^Ilj-Y}fltqgL2 z+g~bp8gS$*urX}`br1nvdM0MCHF4i=YGK5yP8=VWIa`h(aT7a7)gUx22KfL4$I zGRP3P8$3FH7KDJsB(jKw_;v&+27}N;w$F@Ds+`Wlv>jFfV`CWI$gb3)zgPj#hnkw2 zrq6pC%e!0OoxxGf>v&ThDJ)dHCSOs}Z(g^@#nC$|r4naOhI01Z2m%RL80!!&J_aDl~AGEAE4SakgR1h`hL(?zR$|;H3^JxP$&Rdx+ss_HO9LMq@Sgegw$z)Zq{&{Dl-&ObULde|koeqERnp{w#;8XYdqM=N^)rl3mZ1J9|}MXv1c+h%9#E7q_PlU%D$~9heETn=yLB0 z?{BY#8#K9U==lg88oZ&@GVzXbKEZ^uDR@`MZ#)d6*G|ltv}Hwo5eR)ZMvCGsptyY_ z3Jay{XT`J9dT}Gd4ksbKq9^(zzTI?|36v4KEZz`0VCx3>MP5lfcCJ4nO#F2Y-KSJ> z&`IBN>_z;jm;nCTC6|^H5sV2qe>2dqZQIadtoZ5;2Mhbf=9REHb}#QlJ?Hlka(A3P z8Rn@Sc#M*XIjt+xHTOERrOaM^o)4SVdOH!!q%owfA(c`yQsuvEs}NF=J83J zySwlNodO2u-O~?&q>FLQth^3vbbaVE z4S0j7wObVmK*Sw4tx@R6=!9Qq5Xj59F|kP3_K3wgjRzfg!0rL>w*vzs;{#c_DzJ<> zmbl)(v%~kE|M-v%UQ+?VrjDS(yLG;$V! zH$yn%ocJ{Wvtaf0$rFc->sOl$ z#6@tlkn!-S;pGJHMsQL*m(si5Z@MoC4=}hc&H$VyZJfT0wF%uf9A`wt#F1D4`TeqI ziRp}rjt+nzoKoM*BE0I;KDET3TRvA}*PAS9Ga_#NcEXX6wG z^N&}Smmxhgs7pc3PZ?&if3U#MKn6-RTo+pWXI1u5^4pvPn7Y$~>|p4CD6Dzf03vb_ zeR+khg@rsafPA^Gnu(Z7l7u{F+yqTBMXq}O8Pqt0&m%0O0|j<)AcKPD-?m2$R|16v z-YlHu1`5@nKm_><1gK#PCgclpAy?~U3~Njex&gduHK-@7ct&e_@2V+?GG!Zz$qL`aa!KSyh9Y-aw9UexMe z2J$~G1=J@0*K424%G%(I^T2cfuJ|k<(#Xijpy^kC{CEn4B}jciZ0Mlh7#=H~GA>a^Q*c6m*4}<_-j!}-c-UZdw6p?r{iDSLKYqLfV>1JAghXVe1O$g> zc5>3ee}@yWS4NUm(pY}_RFj6_AcP_iTtfUfug{gsAI0(20j9XZEJpI2IAnVPr9{+ZRac=)pye*Jx$#dw6EdR-M_= z$Hl@Lv~%LW?dn?6<3vA14jZB0BYJKz*1%br!(?awd}>3{*dn`s(jLpBB$8F`XwS1s z!h7a>CIN7I8#_C5^~*~@3c}1>i@{* zT+9A=rOR(uqV>JEX;;!TM8M;zo~Nh6S(p~3@`l+*fekldqQv;w@m8wvW^`JLc--TN z3EL-)RVikT108oc@*lG_pL*-b)CM{hRj9$3v)0sm{S)U?6zU+-v2jCdQv61VV1$=C zcgauNE_sYS=Us_te5uW9podIRw1_d5oxjq7F(qw1HC}r9GQ7Sg;7*ofNygjj^EEVB zF{y&XyYF+9Gy2gJa*MjQR8-pxi$Yipeaf?Tl1k2!)1*)RMhbTn(PU<2IqSYFHwC=$ z{~ulN0giPazH#4DNs%w8`2`8l};6E#T1i{!h99@aP3ejRyOWk#7&z~3R+ z^}W>0tp$5YpOlMcgF&}cmWt$pdyeKPxch2$oI88zW7R{PHpDn`GD@w&{`d#w*(8@^ ztZBIqLMCkW@)@_|IBP2_ZE8JZ+)-2F@NOG%j!^CUe50uOf<^Ta-yifth4@t}PutJ= z@smrb8IwcmBvQ2-EGM2Les(Rz7!5WYFHRLW@_nyK4(+d)g`t|}V+AY@5u8Qax7$^O zj-l@n7x#Z7(Lo{Y-Af`pFdv#|HP=Ie9HXho;EIobfH1MD_A$!%o01|^$4>F~!v`s; z-@(C`Q#qJP*Q2P???(;Z!OdvUp2@C@(&S!8$ksDvT*ajUI*Sk6x(B@&`eK#anTEzl zJNC?IRr@@g80T^IRcLI1?giK4EgQ~mBehwCxBI-h{v)zO>cYazz~rOzb~%PD`z6{x zmfPl9ChOUDt{)!0zP`~KtO4X7KVIIlC2`bO7t*gTbZn$E+S=zsxtw0oj8gV;zit zkDkjNJ`Wo_L36tqwNlIzT8($74q=_6qmL5SCtLzsY!m^+`wH7!dwaB$Db$Q0)Iy=5 zM^#tf8k>FAUJpy4f420y-+&_G-Q4%!;lpPrHuk&a*S|yNtvGOqITv`wf>-9o$G`59 zKTUS4Wz?Q(y%6US@(P002IhEK1`#2Gc^~*1>nkGH(rzMs4`*ZTLMy&CZFmC+-DOXC z0DUby37g(ubpvd`vHWB293HKV6~wDuWqXwL_TvUYfZncN{Ymooz+J6KQaK2TXvE+Q z2F;zw=4iXWi*@hw8N*$;DUu*a6*t(`2~8q`Eshp3ow`q$Fqnmc^Er$R4CupN{qNE7 z-~TZmI?YT2$T&dNMYFHhZaY zyP{G4W-%J$kIzKtJ}~yW4FL+_Tnwi+%!ZAVP?HiAXmNYd;o(OPe+)LjQ%CsuJw=3q zUiA0t53c$9+(&3N=vB{<@VcEz^qI@%UXnYJkpQ6)lqG-uMs#TPJo9PS##ca;nl^@* z->a%_lwgnsBh3~#5{a;x^_vZJJ1U7Yy|JazB>7@6LY{PK@0B7{t7`;%%=ANFce5TQi z_|~>ct-GH(!lp>(>KHA*_7j9OZEo4i>o86aenF+4qE0&Ut8s+>8l&4}93OnB|qQW=Wl1e)3NbM@7*NTz}Q$-DeHODm4p#i){4HDSox06 zq}IGr?L1o+AJsY%ktLr*UXt{DChT~e%#pL_FYt2(%h{$Z|B~Hcx+J;1jBn49^}>qk zvdQmja_ak@vhGsTzFRIW>$_zkozv82i0O3%r(e(&N2{-{s84>FD=8>EyOc-ISG(hGVw)y@WkXzPEC+%wOdVVrth)lt%hf@Lxgs@l#`MM18U0 zIjK00W5gVclhxwh3+i$QXSr7hERZw*>K0UnFMs+J-JNwJ;QSVPy1MhC{{};nCY_y~z#}~sQpLb~y z@@2>(9i|VxlmQp%r!M`r3}MnUdWuy!VcAJ!^jFvL5r3wrRItKk+YV73tZ|f7o5(xG7 zGbk~!L9T@EZ&5no%9Sgq-Zk0$w{ao*;mZMS11fXzR$m2x=_{C?3rHS(F zO^;uOYLW;-p$|ZmWmwp#RpA+Y{*czo^un0I`m#^1}%n4(m2U=0?!=y3ONDc z?Z9n=M4q@k z`8_7aNNLqXS2tqL?w$~4j4qM!4J(zGmkvyk65-!UGnO5B%u0qgs%5Z@JhW>J_$Ryk zVW_O>(&*9KL0ye>9VPF7Dp7JVIT|9bc`+OraC9 z^;=T){bSt^y2`c2u;3NKrkz3}XZu1ZMcl1KrjQj&(fPdxc)KVy@7XRe zdWmj z^3+&aZ>1GI#9>)h)U#8`g`;iq2SgW-JUe$`$$5VM0hdPX2p46}+A7_<({EkBecMX% zSj@{@_><3a-#|O9+WO}sNzJG@U1svOnCEZ$D`a=sr{64HJmp>L5@$R%M?D=j!*?%~>McB)d1#RrOSV^{NO zw-(!{F~Utor>4BlpvR&qC)*J@PlURzdbsm-QMn+m?mOz^a#G{MV(aR^ODOiTc=e0*NNj!@2| z2&de2_<)2&D6h2)ph)*yGaRuYCiJ_k-aC_2JR>3w63{KMeN}5QSGWx&7tXp(0kYnC zD04f`3GfOEI!8qvh35RV8D<7(+an_*5sIAoz$d0qWBeVWV7L_n9zw%C`z&gF|8eCF$D+4#}@-1k~c#A4)^NuiEAyrk{D^k3s zcAB*g$Yx@kVk*+MmT?SIedxik8Gfc1y{IJw?mT?9MAAoO9#M!NDj@tQ{Dn5a8G}PZ zGlm?*iSp3(lMpTL7%0nSX*))Y*&oGRe&*-52i{%!5jL*l$7k-Zck9gF6iF)eFVCBq z@h-1eSb40^p``4@laoJ=YMA_|y(4Zf|DB_ncqG{T6k%cmOos51VZjQK5ZT@uT*=e?aGQJm}&msic<5!8sVc$-8@4o-| zQ2>w_q#NQ-4juYCQP^pkXxe#he8KKpaWU^F_zgcm-~~cAMNlPk7_o-Hoeu;+YMtdb z(D4tq=Q1F-p3C}lh*##Gj`)$5bQ-L@J*~wYlj~1|D1P7dn9LZWCdro=Gk!6G(|SI-@UsT`UTu61Nq|dvxOi1M}2c) zOK%UClq`DRymR|QRMc%Hf2mO!T^-|iJ!N#bE=x7d!g@qlAio0zV?!m{`dON3rA8$4!v^B z6f&%>`xZMik!?m5*?1uQE2j+7S8`!=+SKyp=TQLLygm2;uxem1euGCPEc9*g^NSd1`^R43jXliBU8WS@_ z^7q;?nTUNJ5llvSMM_S7;>T>CU;q{r6x8h?9J8!{$xJiVL_|daG@^xsh$2ki4V`y{ z&>3aOmeLuHf*}=#7HeQ&piJca`ST@z)KF>BbAGPcwqBrQ5_WBMCR)Gl;Oo-<{{93{ zxjS5(s9sbWF!SKD?Q`j)VJ%`miKI>RwKMwk;4(NTCGkQ$MEz1`hE)-(;I#W~9=Pcj z)YN>DE+>XJ%p5?#i`YyaY;INzWSLUm&rBqrhMs;%P~X zGs5RbR-?fsW8>p!u;`81sHmu38?&PPTBNz-VVGK4V!YtE{^rI#gJ#?awbsW`_ujpV zEdXdxUA_2U#r@yf>%9#lQ-HDN7ZiuV!}~^=}-Nl3FK3lobwTb7nohx1O#3$ z-nTzPEcD@aX0HMy3e?9iqkXKoPl|7|!ZtiQItm8I0YJFvYo{hB#}R3N8#G#!e#Wrt zJ16I%pnh}G{IJ~R%Vw*~HYF*j)i9eLKnC~e_HMx=Cu{`pFbKpsUe+UAiqv;BVAR3u zpbe|8>W~d=+=LDaM+AHO_wQS<1$i1AtRCubpGbUGM9eW$$q0V&L^yFf2*bLV8hGn!FfZ`!S7iq;^XhOsY(Io zU@x4L46ZjAkba#Q^-JGA1!?|O<OmgC`h@(Y-n;dA z28Pku8mAocn7X?6%*E*HGGfxNHLYJPkfh?#;h?;-s$0H6`}XAcOlF4-6CA|3eIpqt zy)z=a(xc{rKEH@`&a9$d$W3Ry9P635TzeESz4Of<&+U#-{D4e=>q)!sTH3)qd7o3K)ybTtvw*^ zD;UpdZ=sjJk!0p-U;FhgsjAKUR}v`Kyhy^GRsF{Xl2k!6Dz3Fm0b~mJjgT!{zo1+^3T8i%u)V!KXog!>{^4?E za3cll-_AxW`NaR;x8AY9nz6I9JMmtXgy%Bj>yIDqw0b%^MfyB-3*9`%D8zJ~5b}dL z_q8==h++j*z;j;>S`N_gR%)!x(1sFIMi(SmLXPhNmbAF0r6wAwVrU+~5XtBKzc3ZU z;Bshflp)RE0RSE%pHk8c9c)JX``v&|M$_UW{_UT3Qx^`1V<*L2+dMEfWNR7Lc0KXyL`32KO zL}L>IAgo$HRUOB-@8kLezYk`R*+kcc7f!_xMcl@6F1JAn>#21zGa}0iB4ts{3Dz>9 zvu9*v_M#!Jm%x(li5}g0_H` zf(t4PguGoCQUfQo1>0a8AIDlGGqSQigoT}Xd;h?x7Iu96qo>t<{Dpz-wg2%6|F^oz zXeOnkNWz19SU$uPRTNAm**I$XY;>mRFSlY_D~)Ojoa~?e{-)|fhet%`&Kms>m&bKq zP^A>QT}*@s)HQ?f{WEOZ+fy3Q4uc>;1eg+SD{c<`*GjElM|9WX~aO z@N`$_yJ@(P1`kCIxyq$;%v4yW!fW!PGDtWaqZ9hjkPt5-;23gQ!zn@@dh)?D-gasd zAQJ28HTC9)xdVX5(+8`G7kK;a&Gnubl9g?;V5hisjj+WQO$)` zqtl!*R<5aSw9NKgVTF_BSDzHFaQE&pQnXo)ax*aK)YTs?Nkqn-OU%&KlIedW!$Ea< zyPqp^zTjobU7pF-i;cc+Poby#F+0}xaAsjc8eM(;J<`{g?7G$ihl*3erOmN^DT6ie zY4***g7;IR{qsRtH7AXghmzBO^f(j=TW!?$E6cvpS{-PV;+_A|Kh6JbLsRTrr~@=D z0Iq#0f7tUoTJ%f#4x_}A_e!Rv&Tjh_=b`rt^B3Fp=L3AzF564he&rh4t^Os`Z2}H; zZyx`RzU9keHv@VHhidX_AN|>HI$ag27kP=(z;{rrqjKe~$n5XSN|{_#Nu#AVVcLhn zi=>YdUNxWw1WvrLza0o^stW7gO!Z7O z3R@j>Qztj}d2neaoAZI8_E7sw#zzoPAKT7sfKlK`_cH9=3wjC|7FbmLP8}T%h3Lm1CKWZL5H~Zpx@W8G8Z*=Nbms<@=Cf3U0IkfdZ z^jF_%J*NG7;#<1=_zLe$z12g9E_<%o+Y56R0XzDq|JMc4zA>3&e%dW*wq=RV&aD1i zls|1B8^elTS{F4cT8p*7`^=lMRNnh%Oi$Bbldb(MCi+pH_sn5V&R*W1$ysji{a?4~ z>4Zl&x)t*&DsR&^TC&bzPf60Aow{|SfeOS7=Dwu^W6{p%%mNJX9ssNQfbY@4XKlY- zMcWzrCkf&G+9{E7v!RA#8CioG93yjAR^E4;SYib6@)d|MUE3d${rCQ+VS#{<2%!TC z%O-3Ah^@LVQHNzE1%+^Bq9KHjNb*+Dp!P-Fe-TF(t;4+uTzWJoy5v&sl25TigXnB1 zbvWc-8_OBqJ&u_kyNJmA`;{X_*yZ8vA}QxNxJeUINrJ2LIpopc`xW(PKCAVp0jR?W zKjPW{(h|aG7gXZH#>RlE6YD+Qif4IG;g27bn8%ihpn-z}Ih8C6)UWQMv-VpyZouCq z(l_wNHZ?WXqWY<8VnW00-Ds3ga7LkD2bjyywCfw?-`-xsccWchPr*DZDkyj#Dl0{l zV2WR`r=x{t$K2K^5a*1{$3=%XF*JI5dc;oZl;!Xlb@gDryr?M3mNa9~hsytq#xI4R zB|i5N3@&&@Wsw?u9vd)R;vvg$G&f5k82o)s&bwdSA28l+F=oS~xT>H~q`ymB zUmwcGyL~$$g7o=uM=RO3!I74jYwfz7WmmEFzfjH64FynB2NrCkPrZ<1Ox@t^% zn7Lx(;&>fa?Fi*QJ;&bXNhTfQv6}fI*!X!zbQX=6_R5Kq{;aNuu8XJc{_L^4cyZ68 z9tkiBA6&JstQL8G@nm0WqH9tGDwIZm|!v1}#($f@yvjGDxRM*7hxsMw^CG-_y z^CgKBlam{~OBM2BAj>s1<2_>=ph`1jxqa)G3sjGDEevn(1O)ZChwN`*H)K|8V&LV())@a)yO>IAE6X3Y;ih%MoGD$VhdKoaRxEZ z_x}psDSMDl*M0*@?sB#jjm_di=1Z3p$4vZ>Gi-P)Jo`BW+zcPu5b?HfP5%}JOG~>q zZ^nPN#zqEhPs;*}fF6>%XXlD-*baS+etuH1Zip?narNMl@BXxT7BL$jlbBImb$YvK zaza{CvU(`XE^p9|dv|pDP))>#51HixIJWZg{J)o7e`j*4VJ`cFyei!3pozThRMw=% zt^BRm$=%IkQ%mPBx({Z|j`NJA_v}2kNHHTvOaW15Y!1LS1bU)Ly{5U0a>Y2j|OKCSpMs!a3m^2^ud1jT(r1Dp!M+aRcGm z*)eI2izPT-1XY>HHi8}_2g%47GN(hqQr!tf*th;);hh5ml>2U248%j6xIEwO{oQEn;@NE&Ah&e5 z!1A8kzrX0>upLXE+%5#_8sZW+vd2Saf&-jBY$t6^dWo0CjnxyrKg|BRJ?_5DB;pgN zoE4>@)aPr|cnKr0Eak7<{*6Q@&+8vo&s11uc&j8zP{zp zTH-Pa@KtbKnj3oE`I4Q<;!}E3Qo(rVPE!#N9qA(*XGbqfU%N(L6>|LQC#2VF<@$F_ zSZ`cheN*S^>w6SJjMtqCpQmpHkW7~|qFI2e)|p>aoiY49&8y=2rOb=7o--Kmfmv!L z)jn1sE%VKH=b^2S9{JIm$n)TR$CKCL$&Q342oASdUy6yOnM!So>0ybwG0a_Q%#tBi>)AF59R~wNYhIrPdq04Hi_4 zZg6t}gSU@bE|2W&7Mb=k{l>3(nY`L3X6}K7fx!+i`292tPK^eT{)l>JuH59> zd-^;R-N1$8J9kv#_FlRr;nwF4!3<$w5aX96z`qA1dq74kW8@w^)>ZbC36Ygmc(iUNeQ2)iQC|G_G#0J&n`f zKYAhtcm3%=4(@olgookMF{83rNTQ_LsMS?n2_4NE<=m0NGpg)Lla42+r_lLn{H~d& zARBl*R~q6MnJ*Zmf96vVV`Z55?~UbCx;>~8*{bG1^`zae+_!)KR%Y+ddpj1o6A;!S zb8x3=rg}#jZkv4j`>gste!(ya_hw?4#Pz-BLE>WPac#IZ@H0;z>|P4wVYP{Aj->4l z7$_02PN;sq_n)4{r^m403H*DnOiEVo@W8ASZ2$59{lSdp~pSe{le3e@XG+TON>?v z3dqp-?hjpG?}G*)i(42VMTjJ*WXZ(3?KV_msIg;xuLzaF-?PxY5{?>73mZi2p;#uq z+NDb#>*{KPrxP6fn5D13yrP1NSf8wmYd^5*=3wVN+LoP!lq)J0@f_s0@Slr> zm>cLq#J!&oEMrE(WKP`5opynE=nMuc&D2t*siG~3FIA{dB*@QjQ%zi8CZIiq{0-Mw3Y&?C=P zzU|&*WW-)qm$HF`mx{UEpPJ0Is6kX75}wB^Ga(ua(iavwN78kOO@eZx3&EOSS@h93 zdW!TWFgi~*F2Q6d)lLwz@)FF!L~O3}eqVyDw0mW)^lMSkxuMgb1+X~SDI_dQdzwB8 z4c&vV3o=rNov$0ysot5KKlWccI#@u%EK3a6*^{AGB;HcB!g{QXW^K+vnf2w75{5-U zz;mWwG3YF3c-dRde9pB^9{<7Nt93tq>1JcK#AlwuZF}bji&$&s@ELt3w@ch%Y^L{&$vK8%_HeRuG zb#`)B740K2*x4=pzTdMlkYgXosjw`SU`lasUiV6RZ$SB9Les3!I5fJH5S6yaAV8^G zVey5<=Ef)C{$7>ZI{k`P2N}yYlUlt`I72LEx2>(uJG*b&0pI9$;U&*HdltLCd(5+I zQ~Uoqgm6;LtHmyKP-Rd^Sm(Plby&#~w!=*OVR2~79(}Z!Om}$^M32A|(vxhxDIT+) zMDD(9TcJ?ehLrpFTHjPpjF$SbO`cp?G|ypCF5A*#qwjF|h~M|br8v&-IiEx&Y+O!i z%{Z_Z@DD!ntQ(aPdA%w9KwFCf&Vqpq!wMw|O<9%fz^ISg-u5_8$9Ndw@T3!_xigqL znrfpv_w@cpi8Y;_+z&IPR#&Wd$G)pCrzDl!b@7~fLK)y3;AD|M70Rb89Itlm-Yyy% z^S^y2{Wlhxm2L=iTf5Gjw98;(V`F4Fk}xW2#zp~M{K{bQzVl1EnYp@uWu&2qhdIRj zb9A-;RM;HzRNvB(p*@&Hh*)xcE`Ow|tIH2#)eRSpD-Q?0a5pjGQuao?jl8dF4~b+s zQufb1(>vRr9{}r$%9n$r{N)D}By5Ap?zbHqo0t%~^T_hNdZeH#nBIEyWZT>=8ygFR!CckbPcj^-RK1m*w|U#f<7Jc6skU>`eshPP zbXjxYDxM;Io8Mmge1lRW*Tb>BOiEeBhr(u(o$*cLq&)@$X+sl4I z6Pya{txZjC(5UA2jvU2o526I#E@+nMPJCyPB8=22`n3OJq}DfLU_6mX$x5DpR)}Id z<5#N=dshe-Fw%Q5fRK(vvu-J34hR9&irPO;K+QfB+ltD{&ap9XVg`)7Jfw`WwLe}m z#`O|UX>}?5EyzNdIOB8;<35Z_iP%#@-1K;%XCFYtrFi3vnf>`n2Iu|(XsCc@Z)0Bm zH-DhIf*zO9B4brnHh&~HI5YF4KeswD|L_ofNt9ydop6E7BbC`~Y1TN^fcS=T9|;2S zMo6kmGo!s-F_%YCZRF4e1h6DpDdx}EShZUKuII=DgweE}qkJ8{dkB_Btz@Op^J^V}1jTK%VFbIH!m3pzF*w@#mgK30}>`-U{6D8{tz!PAj+y+6L zlpRhhf&&QlvExxXRHOu(pa^*Qwqd(W5L$uCc1Cd zvT?A(5$a}vrJ$$? zsm_s$e?=|1D3X#YCSSjPt4Dvv{TIHRJMI#g5(-wtRTVk?mLANtZEK-eEQ76|ukb{kwMJjw;k~jG zh1>dAeWOCcbH~p`HcZ>QZ)$kLrOZ+);q59}`e=8mo7v>htuh|hQmLk0%|n661(}$2 zGcL=Yk~nN|xrOP&Udq`V+damYJyu%3zIkqw;63b@V_WVwGIv#>HIl#bN5ZZHJEN<{ z*!L|NT|V`A;cDVZ4|{3Q&wjJiEu9&TOuhI`rQ_cAZST!=+v^lq2#Y&#Zg0Go+Z*5qAX>%VN3OS8PA!aY~lD(r!0s3y$nK;i(=H&smj@w+bz+` zA6u)gHaVlKo9tCbGkSJR3`W_fFJDF{{1$G{@LYV<*8L|Bh*Au*UU%k2K3^U&i5}86 zPo}FrZ62*TdT6s#qb3cYHAT0$DQ+Z)I|l}aq=xSoYreSPV4j>&ZkxHX#BVmL=BQzF zLgmjkTiZOoWp9UXgRRv)4^C>>^spU#6+fnq#jiFLGG2@lA>r(T$ORz$Ex+m_I`|z! zF3B8GUNyN_0qXb_f_}*?H5~8z-}Twel9&@>)30HGbc)o+e|7csBWJILoC)i2x<{~8 zR1W*TkByZ}u)$RyFu+Q;^0KlTqxSY4jEu3fXQhxOk$*ftVSZsj8*fU_48ThnPg#13 zigNtrRla`xDV*BGn>@p6ETqJI12#txZfQIkI0VgVLBZLTO}y)s$!2=o`ZP9%>;5%< zz3oT`x(UUHLupJxo(D{ew~-qKT4q1F#R2w!a%eYSz%9{d@S^zRAtJ?y_qh+`qOBazI_%~NM75#0E_ymqB?(5Y*}nxJ>Vz!xnvf~yRn zzes&fs3so5VB(lBk$nDHL`3^Q6H#{QD^Qyd#vz#IPFTQ^P`H)jGsV#N_qmrP!Ji^9 zD9Ba?e+9u%AB1S}A&W3s>pe3tT~~4Z_d>0g@b6l>_wRcVmPtfqUjU?J(ss{fVk(Z? zh22cOvdDwKTcA)_N(l4LsOs497UrAp-}Rqdg1_E7ZFW^d^YMZ%vMQBy^1C2WCWKrV zAR`TURmFk-D8{vaztai`@K9WOCt&cZ>Jd3>gbBeTtqc@=BVzWUIfd}4Ln9{b&Fn05 zu!cvNH6k4%E^RD2A^p4aEj=|=8?{$PzELcUi`A(fpd4Q0b4aPd5YJw5I|&&7+o=z| z!J9cnaA9dVcDfUEnZ>*prysQ2noylSUE$Yhl6Udj(@n03t9RxDQl)eFx1A^R3I(?& z_frspW8-FQR~LTk`F8%jvw;)|Zsq($HhP#SpGy`r{u# z&80g(=8*5PxG0ocHgn)^<>WcV*P(oNE02y!y=ZpyT>@-u$RgLu|7MD^l z9wmN=#lRB6?cUO2ef?Xo> zT%UQzm)QOHR?7`Q=Tt7Kv=Q8Jqw14t-+&31&^F`B+xP4T7>@}so`74NBk$6gnh8eT zcY1HgH*jPcRMr-hZtav`JXRz;r`y%~>EL&PcM|(!oSSCLZ0h_3IlBvc7rs^%ozrQ5 zn^fF1)3E3JL64a#{;u0I>9w7LANq8@*mz@>6_fReWxi3c&R@t-E+|yZF|;xCoKg6e z*zGe<;w8DJU4uUlrjM-Yan*3|GnBjaTJ^<9g2vU=8>Ba@0ar~k0%9esOYCy8$3(pK9S<$Ob-F7V32v+bLdVv7t^V&=@1ME&l zqM|;Dd+^>2)!C^6!xAcaskbLIxd(lFPWR`!zIeggZf}*s_YTvzljhsQGE`O=8ygN6 zJUNOo_;sxW@9@ zPy)EKw$gD#SLFERC+@qf*jdjmW&DB@pUCQX!BB1RP8>{pMn=UIg`9=nJqJ0I4dt>y zYPb<6`tkG$%i^z3oV>*Y!>EAn*Qt8v1c0ToY)cIr!U~WXZOyIYQ14Qt%{g|KOsgeL zae4MubV&+qkhrzeAX3V*5F5$q0GOHX+=&$L0yf|* za+-#Rhc_oG;sMF5!RbCI64m_2ptI;YI%0_5C9sI*KFOmfe4j^B9Bm-s68+A+d(2D zkuFM}Ra94}h)f~pKu8G=CQlIHJ(8@H%orap?^l9B%lCj~aA;4*6K1hM zCnz_x49kcxj_7DrB#x4iaP|y0@$MUH2tIrHGAF4xthZT2H1N+?$4zgKgCqHVn~W|l(5|0Y(y8f%f6*px4^ z4cvd+#KgwD+)TgqRwh6C68QS@s?Fgq$wQ|2DAEBzr6)F1+Y5jCkOOBwIN1ENDe@8b zTDz?dG`3GXJ@=Ec8o@>@fu%yL>zlfae1fCgVQ?~wJ6lXlbQ-wi0d)D=&_IVfAJSaE zm&Zt7y0ixgBTh00H<2nrM(R29lR5DN1+LLGmzWrHTw8ZmyfqJNOgDFc7NMvrcxwEw zuj|i-!_I2k)p8A=e3N8eUOAMQ9&nah{>h6M{Ftqoji^$-dbPW|B2I<2Nv{WCz`XN+ zoEV6Ikod7n)_)!f$qg@FuzmZs8FJwLVZUgWH@_saHGM2Bp=->;ZIb)k^UZC$ z%<%(H^&S=cYTWtY$WL$~; znlA86j_v$;rESQPx@!G->w~w08+T=Di|>d^<7we$yY%HFE?3N4 zmrQNx=*(D;-2c{Qz7TozzTzigKK`#=bEOu)KNsdSO1~Gm{N#&VvADxT!0w25KhB#* zP#}r%0B!lzK-RDL5PhS)oPFlwso#*-*bq4OOWx#MbEjR=ZE}j$JuPt^r|;i9P;!DH zdasG0?lXLKGjq+uq%&9R+!DgQ58QF#P;>eHRPXCqozxw3-Q_;EF^nWFSe<+>-Y>Js zD(w7{w48i&-eBY+oIkAh{r=1~o+#w6AS3Hm*D}=c^p~QY+*$sTR-vNFp|Ef1_P1Sm zs#DCVA|$KoKK2ty8%RH!50;E2s6PV~ZKJ1^GbEd&u(Vjpw~d#s1{T<*51(qLq}?Xa z7aFSQ9?@NqG&u;)-pH5mM=PV23_{f|&bb!@2A;BsT{X+!>t{1FZ?j?mih^VH5e~Yy zlUFraB?p~1yzWr8cc|=4qMcb48K=rrouZ-}^DtmIX9i{t$zvuNHq?x91wJsl3z~~} zLqD&&zZsuz`$~K5|mkyC0_fpVa9F-UO``X~Z)?0?trvQplb#?K%yWM#xop7p z0x&&z1pZp;!(3t+&!z;m_6}@FM4(wjdTo zFZ4x!xMJNS4~8V?X#Y#efu@`zKC0@fPR zpwv&i{QNpp2jksSQw1@#d<{{6`#=}4qR7Y4#r0KX%!Yi!h8_F%eWnTkDF8<{Mu3m{ ze(fThxZu@<7mZ{8f{>0dP5%yMoQ^no08f=5xMuVYi_->JsudQT&3EmDOn?}oVh2O; z6;xCr1%n+O$)L#OLB8#qz`$7@%Jm2My{HVN(|;b)YbBuVz=uvmfp07gF5^mX9u467B$v%yoU}GzYRZf`m=FKvIAl*Ra;+H1j99w*)Tk zNoB(-?NQ?S*c<>E0HfHW*T3?@hzw0|uVYiqwc$%31ZS|+UM=wp2rTLi&gXu10S&P`ta-qFJE*P1S(a>Y=iJ}Daj^vc>l;}u- zuh2pq2c6g>0m8afAvaSXgN~2L<0imz5Ktw4`Lh4XtJ!>Hfu)W~FW8CWIgzu!xr5IQ zV{t9`#zf4zHWDDcTD%g3VgUE=J^1Btod6&4y8xB%;k-dKBr@{y$|dDJ4=_EULIyV= zN#_Eb3U=|mfo}Bx6ks6T7}jpsumM(s@lpzwScs6ka3_MIxjCZD8c9^a3*Jmao*5Z?h%~$} zUku4e2=i(B(i5-I(`}lfX)A0T=oqCicK|t$Pj2s+f(C%jfa^H*jH;J-Y}^YN9d@2x z8a@gua-odVHii(?)MsJJuMUPd_9hP(63=QX@#?ic#}NS=T?=V{9{3*|`!aokfR49NGbI zF+?WYzSfa1d5eZ3-0H^)qZFc{>5Y78c8_W!KR>xzZr1K(J(jPagpo*&VVi=4KI#W^ zfFz?=C0T^@E2F@?yF6RGC@Fpo-!PWNsYhRZi z*V?%C<}HdEMebx=uB0WI)0UdY<1(38V)8sB7x#?l{4nV1 zH*&6D3-$_Lv!Z`d@3F!S=kCJAJ*s{5xd%y89gK1On%j|FkmvO+K0V<8XPP9v=BvY3 z?~@z5$UQ8sS(Dow6uyuds&-i6IljHY3WZ&7CB;}STT3x=t=S!0&!6Oc_C4o@XN6H< z>GhuRCY9MAcdVUHS-u^c&VP2Sib?a(nMJ*u(US~U{X(0I-;tLlhE^2?4_z8+v|8IA zTbXNHpJ64xl#xUFGhpDQ=@9(ls6y*ZRr-f>ZtFfm16lUHL+KKOM)lF!0L8PAO#jMk zf4q2cGk5x2=tqhETnhB2fX=&lh<@PGL$+y4+Q-{1LY1XYASy%F@6!a6j@()#D32`$ z>~P^!_VAlCzMVvszZT-s;UKRdyRs}|wqIi;RvC4Hgt>l+DrD+J=jYbZ&~CI9AH4-`l0c)R^v zpoXr))qN-Se)uHyD?J&$cI3d=XPt(NI4tDfB^08F65MLQ^=4>A@7Q z2CwM-hHDsZsWd({+>bNW;+5IYBQcqdo}PDeChjNaaHnoBAEQL=_GCP{voSu3nGj`A zD15%+oSA8p|N4Ex|2XxN^}N@e`k)&VPJJz{u*nIvdQr0I_avhCl>Y zb5F7?IL07vX#OAFZ{%y#X}E(`Lr*PY%#6AR9Q1I<({-(eBUHBU#O?QAym)c{XL&g5 zAi2Ya|JU1U+}R0SwA5z-h!iBXv<_IG9$^`tC49!(fCk}MKgS)w*`565OQKHccCK4# zR1hB9r`h4y_^VB%A3&Xpt21XezMvCmK->unI8MHYdsf~pf?E)|m1P--sYKbe{CyY) zIdSHet+h0#SaE;6C@cG1!_wckqXZWnt>jDrV;}QxKsJ&-8UL ztc0!*CcCb>CY>MAXg?7 zMd=;wQijoI^cKOR=h5@pcO#zBZ$&eO62r)V29<7qkXhEuuz5v}9XEz@|Kd(Veg!u2O z6@$W!%6Ieq&&uIw5=?Xd6yQr?|N=Ft|de~d)KeS z#1!Zvs|%Ey-QS*iqs!ghI_`QQ^d7<;jFdXvc|$`f z62!d_OBF@sbRx|m(|iHh2-%GRwZ67A;-Rj-9O2^t4$Y4jiQF;Fc)Z~W!aRlUZ7Dc4 z=80UT4sUOx`}f@tu7Fp5@9I=&P=IiuXl!a|Jpmj3uDOHJEkE;>1qQM9C07Jz12 zg!6`+m$%sGlHeMA{m*YE=Zu$eYO7A^B;RbxU7|z|`f2eM0p=Gk26ZkjEUI=n(Ubs% z_w{~2Ky@D&1D^@gHG%9c5wkfZA;D=kUSQ3oF1m4Hq1k(_s@+=WAacaU=khhr-nbE0 zB0ykvyS+JXm+hW|QYZ`oQOc`pp#Sm+k1c8h?wB0c{?S9elJVqUb3PeKOxS8_YU;Wm znbfeKspn>BM^CI(9U@kih-qBAbvJdc|M4AL2tfjG*Xm+p{$N{VKicIC+qrnYVHmPu z=N%F>F~MZG5NSi?QHs%?45uu)K+ybcqT4TvoWv41!|2T*>n(*hn*uxqA93Ven2xoJ z5sE`4rDn~#K(zTF2cZNb!A#5=Gug?t-`P`0)0<|+MVibDHiW?Ztf?^K* z{-O`UmLooYZsB61B4Mztl4H%gnTl@wdmqw{_Cs009V0eX!7p>=%H!rp@99=}V|&Y|hp2235O%=28DcQJjunIG(z>9bB3nDMAcl#013Z6T`{X$KK$R z{hzk zWl9PqQ^pKQBuO%tAxfwyN)b}D4Ixuz3MI*$j8VyuA@lrS&+bg$_x<RcsjcOAzNHvb^l;BPcuqAOil2Ul?agLp zX605;jAQ$4=w#Z}>Da!;b33d*3!QcS{Y_=VMjx35$@lLcUik4rT2u4YlcXfxj~_)R zr(YX?65zkJcFono0}?J}TB^$BvUY=pj5HlV9;&4UbIsmm)6=D;m(9%YMZJ2pX3riQ z8X6cU$}2jXykYQQTKm&@)40z$H|=nnTd>#FT{B;>V9O@D`J|-{{RfLh zD1gdtuUq67#C%ZxP02)x%PM|1&8wk?U$zG*3`^7$oJthjx$B6Are@Il_ga%v*#c5u zn9)?K$Mbots2v=PGm%t zO#b@fc&h1aUlA*L&@Bkx6vM8A5u`Vyfn!V+q;9#D*a-z(Zs->uDbY(TufWzD7%X5 zPtrdA$Xnt&vra}vx!SrlF(YGzu-bi)R5(zId-qZ1yBGZVOK&an)SCI3EUOr^rY!JY zM`?i6ZVAT&$&N6Ln-3FP_hy^!Q6a`azO5Kum45h8m0{)l_~2NUU6GoDA?4(osS%Fy z%ESXlkMiu_|A=*Mf1o#=LC|~~?1=+AIy|D&$C@X6IkzR)Wku@Bk8YLgw<=o=97|9z zS{OrTR~JTb!YVC}fJGq#V(vqI*iJSS| zEM}3mCS2U67(>QpV=(>16ES?CE^j7*KZn!w0WGbDLlWDGEDoizn8gP!jD(FDAEdrf zrP{W(KtOzF?UUdEs;Q+_Z0011-5B%v6t6BkB*YUm_tg-@YzB2z1{X$^8P)L>6>8nHkJru5=$ zy58(;x2$jYsfq2+h$l71QrHz^G_)p$hapnqo}HV^k)~y3)i6puTPp+!nR1uXuRR+D z1uvRm>IM8UWw#ckezJsr-Z<1p>!=VLWb)A0_TIokg{y|ip<~zwab^El_6@(dK}oD5 zf)svR04(fMT&FJ5v*Ba>=hYO7H;uV=;(UZM9&tHcF95MpV8;$t%m(IJn3lFh=jHKY z^yYQKcyQz_-L-2iYkmEc&*H&CFaz|Y*$Pm#BqzsOAz<>5J!Aak~61DrljL}|*u zpK5NuI9y@GM^6){xtN=0fqr1X^=Q9}yZc7Pptdam6TeP*Li{fpR_=bjE%yf0K;>2i zSnkZ*o3D@ehc$l&89}XBQ@G>O&rj!FJ_gB%79QE|HZgyK-J{l~${gZK{)J)wEc>OO zU-jgq*W^!GY{H(QPc~IYf-I_9h4f$8XiF((<^*eBb2MsOO5=BnxtJ3EW78v~^;0SC zdLM#DHKmqFN56l+T3*5J>izpGEq9ByPIrc0Vojq*q_3~9wf(lo$9!Re zo9`%uu;HGJ0bVTNRl@NL9RV;*+-LE*o%gQ6xs?vUP%-f#bR1|&d0>C>VsWn1h?rGf zoUoqXaTYq90XFP$%GW$+Yg-0q0T{5gJijrgjK5Fj*sPmU=>fhT&&$u}L$o-E%Lbc+ z9O;$*~Od%7u3R+d+LRwm-5KE@5e_Vt1iznv(Si$ ziWbBFfowzzZ!;B)C87f?oDTW*fR+a1$zkF)==EFuQ8UH2K^lSaBI?pIIFI&Zxp@jO zkZ%%Z9OjLlWInRIT$zbiO0(#ayfh{Dt+zz`?7$fS4I(YAPKlFp^KEKlR}?zE{Y+alvqqkv3lM-nMnC+qV{wIWT5^12eW*Eb4A#;pLqx@4ljk zP4HweaGO#7mU6t0Whi8$=*J5fM)a?8Dvk;o_mnaC4zcz&h`tjk=$m8qB zMCZsSpJ*>nPa?cP=4n$_!0DBDyDz|iB{g0cFNJ_%TYYnLb8J91(-K-tl0&(@&VbEH z6E}i@J!qq1AL5Y$|9f>?W#qJ}sa;Ra#`yb4Fc=3YYHweN|2QXd zNs8AHVnyr*6HZmIgY0{_T8ZD}?M)7@y%&r#y9ERVdl^BE5&d9K8ixiu@m+z`k>*+(4 z*n{}*aiK6+@T9S?iA_%ao3Xxufq7R(hWP)voU$Gs9v>p4dElgq?6L?U(QdFUZ(GnF zE+znlg$ScNtm;HI2nh)a2t4s+o7lEz&z{tjprD{b2?Z+h-<0u#cespO69*Y+dhRDa zZjFa-2*Xf2h+GC0!|1!l%<=KduETFRVa$*^9j)Ns@#|M2>0P)1xn6P0C*{IY!b5WWHoq!A|8maCmXv$$GAVr*)gwiF5Q|U)j@y)6*J$NU);??V>i}xT&d$XiL!TGs z2kk$nbmd5wYbA1H1xfe7t;QI{m?g;Pa2<%hjIat9T*V^IaZUhlSW}_WRn~MQX}hKE zMDh2E_SSt89{K!qa_VBN4i~k?f)K~D-Hwzr}W;nopZTxUWhqX8@ z%Ocjj#utmyg`B|6(?RYR!b3tfSoO@5{@wQhpLF)?F5e?Cc*l92Fber^T};SbUW zvsTAg+St5>tO)X=>}?WD2H1g8hlH?$jJ#F3om}W|FJI|xMLI)1SywNwA&<0x1nO0E z6%}cjROZ{$*NTj<78&2Jm)^hsX+X9Vk$B}U3P46)s|YV7h^hOQw(Z?}q2mClP2tca z{Rn#n{35QuGb_gXK#Sh^-Bm?pIk@4)>3TMy-Qn<=$NXn3{n-Sq&BOv4hpTN{_W~+x z8J*1YyqJ~|J*ED|Y$x`BCgSS9MXT++fsurbXZ2P)PQPKme74*RpywvQ75XJfirDyA4eC04Psg5xOruGqdY20r!%XYyH3sBjrw1 zj2%LyWMx-H z&qXGdh`Hbe0QkSWL(W9H3_Xio@7w*58yC>k?USwdelUQ`hky zz8hGS<1`{P{;S#7Y2?SSOwUrXuoNi$OE_%AH#9U{hlSN?=(*>9f8mn6o_1yhz9Cty zq+6w_<|Sa9#avzWpHB7mLgwwrlU(4w(#NOWfcf$6O>`BZN}E}AcLW3oy(#m!h5ESD zhx0lXMO^UqaZ(wvvEeQMu%*Ok;i1>u%%HM^tSslO>_XKB`c;m}a2JWjdOq&{rHH9Ex0c;K@JtlSc1UPcm9PpRL^da9@ID&&%j0YoC*| zZJhFNXlZ`)EGnG*srK};=acyXuYr{`D}?&iyQge^^gM}%#!rjU*!r!Kc3;Qf*X3%$ z9J8;iCbQ$dK0jGfp-W%y7NAQmIB>}I0gxh)Me8qLzErAO2q9)!92`Ed_dcACfNKg- z#LD5Xdm)RH=E2NZD*f9k$O{(HE)ftH*D5lT9~C%y^ypr<>D@@HpQ@{e3%f8yMz;5X zrx7n$MUrm828PAx1TpKjz1i%GDQU##Z8@ zPuTtj=_g+oj@q2MrO2U|JwdCv%IB|i1kc#vFYED>`A^`DL+BB%ux3#q?8{K@p(1(J zXFokTX$+4x>_iRG%z;lLYzfVID2~EpQLAp+l1R7lHRs#&x7Kf6%}8)J5}h)MeA`W=Y7GA9?G@$0@PQn=f_uXe5(NKw&7rD< zv#|ooK-draK%e>T+qWl}POj*(ZO;#YsW9|{Z16363n^(wPfyeLl)yM_PZ1xVn9vqk zhJNA&IvQdhgc7&}N7>r_zJD|b>9zW~S6Gp65fn}*x9jW!iAf@g9{EKd!lKj#1yhCP zSkQQg;JpZdg(Khc6p~u!W5+hwPB5~tvBjvX2$qpt7i#Fd>{N!eoT+JyrmeO08()5R zKX`Etw;eyOitQkX1{P_G)^OfOaebziaS+x=SaAiYUgiLToo3a$cXF7mEDhpsU!Oe_ z{pOFauy&IE9V_@akcGpjpmdOs6l>+QY8qA)eg z${O)G)tpcB#k^y3fXL#R@z&COxAvrW-+FqOjEuJ4Qe*@xYn)QPV49s}`O;}b-`o3& z2y5o46>E0Vc?ukujg%EY(@#7$KSdqlIA$R`QBZ)2l&Fw!{y9`eFLidwE}^NbU!$S- zg{f~syf0j))w`iVDmJ>>tz~y$N)?@XFX|*y<;m>L*Z=bQqwq4QlDt{N;y#^n&Snt_bXs+2Bw@XjG_Pku{!v297rhIKS zO{>C6w)E9cGK$Q0C9kG2O5}E57#VWVSYQ5@ZIPUmrnZ)H8D?-2%feZM@7^WbCeP}h zI^~Zs_`Ry8smL8<&U2#8MJ?m!6$E%JX1;PrKw~@E@(kC5Z{NQcole<5l!d}?{l<;4 zW#S_kutz>RhaJPP%HPy%-E5VPR`6JcoHiZ?=1BPZT408<#B0$KFFCD=(NS;1WF>X= z+ppw8A%@Wu6}KZdeDEUP2+(C<#(m*EXq|m@(=6=LG>{=>O0CifSxZJ&6SOFmu2?nt zqbkxiO(QW63SB~rK4Adkw8zSc?C^|6w`QuqGqCzs- z0b)G^X#Dur?c0O=l~*CgTwdtIj=lAG$JF{W+SeYb49bCope$i$!Anw)NB?eR6$Tcp z8?Dlj+~iPMqU!(eOL+}4Ga-oqt|2UWllxQPo>yTSj#b>yJU`lNw-{}bvZ^Xv37?j< zJ3=tEp2uGLxL|a?91BUwrscQS?ZsL(bT`-0FE{NF3SLHVdMM0ZQ{mAlvp5V*WGNld2PI#8v~^)VWdd50`)s>=oKPR z0;yN5n>=BJ7~h6$iS!Hu-1LS?+8r$nJ*T9PHX=ti9aR&ZeCMnr8pLFU(?e&Kb zGqwdT`SFq0FM#LF`n{iHa$?4$tkP-jsa&m}Zgm;PEq(aVdV+)YY8aOX{Ohe%A92;3 zV08(B<23*j4SoGB^%x)?P)}Y5GD!R!mg|{Zrr%ji#w27nGN_;T~~KmTGOmfx2=^G04X-;Dc05R=3ar^XwGw~=hEx*%?tqG z{C=9iPV@rizY*S3Dx6@92el>*W|{k`jpgVo+ysJlpkyqw_}J962C71gRmQwdUs?L$*j2nKwwjrCPH!&>JXaAE)RnnTBr-L(wYnbvs1&joLaC~z>^E?%aWs6{4X z;Ol|AKsM&$ldb)|OIw2-vmfgzQ!_K0Ltr--;f>z~qm9naPU0qodZu~QM|{toub9>> zMsGrP$TRWn*NJ+Sp(?~iDuM71{;Rio!hDr_9peEH!1>>6$%=!{&Qf@yW+TLS*=1rI z9+W5YtlVa<}gfVG@gvI!!x!HW| zYl&}Hc6WrA%Gl6~`jV286qsED>pHSWgFVH*`xsdyFZDR-)Wr!004yXp$jlBM(n)Op z75M608sEhGdcb!T#6Is5z;dE8Rsl9Hm*J7UuAxtVOdDSWOgd!6xW<2rC z+oY!^N?DajODgsBV%k=9t#MJbPC~~bwkXKStx!?nuKsN4`Bt#FOo=hCjW<}x$9SOS z+LiI9BK5HjcM}y1vx=a4iX8^DzV2r9p+qGPjJRWX_AESNZUP6{6jm++)4?4VpYnTp z4xFwN*+uuYdEynSLU0tvr3?$Yzm5(~*dcJ|P1=Vk>zbP%g?SVDuH}qf4(Ns3lv(&Q zDt04{u(C>qChIsJ5ZRXwd)0L>HJXWn2RXQq9iN2oU<$gU%eTag|A3ba>gXL z8|8y`yyWXi6t{09+}# zi(Fivy?Usje|kHm{w6+A(-H%GPq5H=RWX&9n`;(s-*ZU{^TKl-l^4jdR6EG7fK@kS z^`+&xb7>E><26a900$6NZON{zO#YJ?97!XOxpZr`v19)LNJc?s)Pb+|g+QF+_#b_J z{*m*e&($e8nS)*K(IP}M2D^r=$`*)xiCNm%6@nkPMW>{&dd&41nWuM0xU7Ua>bM9V z(w*PL0Uh4DXZO3$UI9X?R3zW}JWUwNu7eGn)x6TIkKI_QpsgW(Gzh8~>!=}ZLnnG* zc*~!WbpP)Qm59YI^u%`_KMsJp!I!V^+c&B)rjU?SU1Hp#ROA#dr(gm0Y%croCWe9X(%Zxf>@^T0}(T<%1LPnqg87q7Y1f zb~#pe6qih9Vb;DS*QwUxJhB;=tb6X__&C}47cYvH_}x%pY3aHlf9mABPMIbyx(Ej{ zsEk#$U}Ii9J=O>BxU)f7D|<*eab@U9f;f7A_^J<&h5AhgAQO^7?9lqPzhn}Sw$X7a zanE3eD!tM0gxH-qzi7XO*{TI(XkSeiz*Rf&m+4Qi(!9E?Gpr?-ShE9Sgv7|Q>7t`0 z?R|bBI~{s2ua%u2GJLAE$6SRgkoS~Hs68J3+=Zo_)aSRf!7TMjRyM6;_&)WPx@*qC zW4;A`);fzcGj!FPEQ)*{UZ%Mkl(zg%#4*32$?>l}*6l~50bH&`#%&V9%$F_}J*D=I zh}fV}7TLV{(8-hcG%G%SEXDZM4lDrxgfg}rcAe!PKD+@0!*}HtkUtdGIJ&?FoBukx zuhP;iy%g*!4ga$`_+hf5qjVS1|<#_5Amg z)g4ZHR_WZB$1mhGa$Q#sIPNwM5P*bHos}KU3rmPJ6X_n^y>R$u5s?Ehtd8)QcLMA^ zVY!W*Y|^Y@RXKRtJDL?$RI?LyRlJ|437DU2I@iT)bD%ENtw2~%NJ#Mo8m7Z}EI8Dy z^=#EsSb&>O-g{l-GEW-lL_u&INuS}g5&rj7KWt!Cjlv zP4O`?w2(w%?;?I59cT$8bU(g4wdea_d0mcbl^ZgY<+1C4h7gh!fTkF(+1mYASb)K% zO8$Ud-W8Y>5{~6;?XykeX$uhRh)=xf9drKhaLM}1J#;G zASdCFN_3vYwG_nHa-oeERpDMr9Jcdke;S6Irpv12-GKh-+3!Pk)_qTEanu2Z( zmzh9Ay?7GXqhNqkYMhKa>X=7&017JTmQR{OC`|vNw>A_|# zc2!0BI(yQCLx+S&M};uGy}d=k-shtxGw%Z3g6D{2bSO%Gd_A>YCq1nmHn0F}iZd2n zOrsq@bB{(xAvTzWbxc-t(S?&E&MWWS#l48SM2F{X@lT~S2MF@K#DfczYVXb;E9=rv z-3$EovKMGZ{^L8!uxhA0` ztjC4g+1r;~T|_Ge!X@4U#1<+8n~(KV5MVhc*oRkIkR6%1x%yfYZl9nn_3nz%ZIJufs1o1;!DdH* zr{FIn&a80FJeHwYV)MvvnoW*{+(BGx)HoFbhnHUc|3C||S`-2HV_Vu zEMlQQ(G-L(7p3cN_fOsX>e2xF_$vC8t{iArmanVbzgu*Er79k8T_v3cyF%4XgPYPjaecDY3N)CG6>IuwVfwN{u}SbS+28gkQ1T#X51!kxFY zwz`PnsNm(`i~JeB;x))5;}o5U=@uxM8I~_kZF+j-#?gFe;^HQ}vVL|p zHZO2~K$X&bgYmbkYg6?S%dtiV*T^{iOPjYe7UTjLGa5cehdcz=)!+p2zr_jig=jd3 zVT>0Q6LUyw0vn*PTxzDj_!FLn8tzV4fJEDE8y#7v-)1PFj}~d-&)=T~lkG9r=<#(X z^J)PKW?WKrv6F-KO>Dz+*lrMO;3D`hsWs{+GyQWtmgH)4S3TH8c28-GQ9tEB|Ilb( zcT*53UI6LQ^PD-Dn^mIiM`{2n;eL$82js8h+wX@=bhr!j^cJIh{UY*J@dEoNl5vOw zqX8FIn1U4+!!|DvI?tLLqp_5s5Oz|8P^4@J#f*I|{SDnh(QTE0G04JnyH5Gg!J@9N zt{v!pEYtPqYgi@4@#Xyfb{I0pHx}n7hrSZp5GojfHIffeD!GnUD-p>+eMxcg?fdt~ z>twqCxe`;b@DbsS8%?mDR?XqOk{0AMK{C_IYj6%U~y7C=brc2N7x!iH!EaRml zE2iOw(=Ri3T}tXz$~OUsw+vrL=$=EX(Sl`UtOnuR%3;KcCt6DOP7sb%3%D$Ftou|C z(cG!Q6Hn65_QVR?@bD-3@DJc%6H+b2ut3%Y8S-u3p(T-;)IBK4;w@5iU|Q}4{Sp_` zlC!}{7)&gaa>R8(Zv~uMh=Q6LpxppD0_(H7C7b8FSrnQ!onE%`S}%guHfFy zaIPVt2noNsp+Ttr9H!#%r6m~Vaii(HX$Iu%xTj)_9Qp}_Y~lLfBO^P^-?PVQilTNy zKtDvO7ZeduZWuWQ*$PG`=ueCp%A)I?g8t%i-wkxg*QpXGbQohd&#jY}&$|Y|*?M-a z?+Fg-e_o>(JO#*l9vn*I?|J_GHca@nn-4-Ylm(G25Qrsc|6!yoAS2WE!@jvK4%;RE z!G#^L-;{U=tp`;HpI)e+SE~vI_lF@?5N`rv#DKFeX8rl)`M!6vkXKkYeCLG=1z@A5 ztyN0$ZoVq9#i+OmD*&h+D^bdSKqF%}FTj z5M)09n^8GbO^qfzGLZLU)K|TZ^kr+LtP?!G*khis5)QI*v-8i-QXg~OxsyDVA_+ES z-(O$e9hRU{7A5@cSHi(t$*Xs`>rLiZx(`i)y_AjoaxjXuH8rQxs zkYf?K*C_2!JWqaOOTQVn2Twmb~%=A?vV!&j6-&7WvvWQzkoq4ZB6o&OG3Gq@RZ<3k<0rnX!%u$QMd%Vc*RaL= z`t>hk+5Z3lu%gx}aF>4c=u!W)zImYspR;tw{KLpvz@fL1SVM`L%h02NTE!9M}avcA((U* z9*{de0ja8L&JUc7$Q5?4rdzZyZoBYtq)#h$ngf&t zaClj#m(+RcBUQGm-JAykh2YO+-xLQ?|2W7||oW8*}8WHQnvxo`dHwaB8$xxbgA>WrM8mH!q-+|}^JE=}d*O)N%f zqQ5UPj;AHxjR)Lc=2Q+8ZN9M6%E^^@GGAi3yA$S@_gf2?Nbj$cW^pw0*VNtcaQ|Dk zm~<y+` zK<@+Ev8JT>mD~+xQ=^B4dSpf)U-su{>^qvJ-x!#4l(&QDa+xrV-{>V0R}Z;8iaJR# z8wm&qxH3{HVUn&FtA{HZ;q>DH-?1;d{*UT5*11rdRYN%V9U){eML8Yi-#G&VC2XNe zNlD4O>~(e_eU&FKnXSE{9bpIyoRDtjr&>n)sC#&?!%8X#}kXxbLt;srq8Sd7cejTPqy1p-QU&tuQM@IS-kGoX(pLvTUYN?lUs zW=3Ul_Uzf^015y5y0{Z+4|X5+^LC z+D~?_M=fesS$i?zcICLfi`^w8OVg|LNgaf#4|`rn82gYc@&!ZR70mJz*6~*YmeOX*lV! zt)5QfUfh6P*4cjW+eWjUoGY^@Uy0m=z5zYoE8L;c0cx+6&?*fpd2~xDiQ_n`(p%kY1rpx@o%shqlfL zLH8?qLq(tjp(A7r0lo6qU%y&QNHi?xLzxXB^$kb_q9_?70Gr6xM<9I~0!<1s!{j~)zD6(W*7$Y4_+*gt*Cjvv7HEIpmWHg|Xl^tYPf5ZR#(dQRe*pgEvZ z*@da9)T%ESP=*P^UiuSv@ScGjQl-P4^YqD)A>d5v7^|Cj2&L2O{~L=Z;rw{GtE0ns z_NS8U@&DDl-RWATzzWVEi-F}#Oror5Ym-34aG`eK+JEKc=|0xg;5#A1s?nj;cTN;6 zNY9wz&TK-nf|&K!;)@DX;A~=OF>z!M##CQ&`ry^yUcirv_Fg^VpaGf>9A3|+A&dBC zhiy#YZlfz)29)rxFyJNiEn*Rfy2Qk{!jZ0?0Kjw&aFoBlzXt`0rV!WIGVr!g!6*PK zO&CA~fWo!7tn7L7E=)wY!o(A$_rnB8-LKVCv}MO zg3#Uo;a7tG-e|)(5bmfR6p{N>g&>N@#PA^`X)tlcKMqr4POlaU3rzYs=NY~L6i2F5 zAP*FZ#|4v^8Cb+cVGAl%*a4^lp*MBKYxMTmieQ~N=#NkE4OAGJhci2K#!+$*4KKbC zOl?1G+(0dF$i!_(&HhJ9s0{6Y9BO^fgOuH-F7f5(1GJ)3lN zgxF5bpI(k58dB+oq=M?C2*#c>bsn~;NXour$rqvp%-M8be21ZPOO_p#4LN=t!W(Ah z(5(m1mEZhxxFNo#M&981@bGYg1@!q}c^Kh`>Q!YA5=xKcz8S&KP~PS}CLLQ#L&G!m zRn({Q&p*n985RSpA`Ci0QWsf)4Zzvd)HJk=lY=7)qw~1fTp0g^)^CNQ;{;!&J`oZu zrK1~)ScAG(m4yxqTTUZP5XV-~S0=l;&C9p~6tRhl2D6lKSKIG}GDOMw7xC*sh}le` zr=uelFqh}}{K`|}HAT@baQvYzUt9ljrOCSRbupMcvab>?gZaT#lGwX0OXx;&%oP-T zG(lvRL7xXN&f}12sup?Ioq__(7!#2q+qT`WmzI_u#$+A21xgpu=A!-UEMb=QhDNiO zTxAG~alo%gKV>BBGW>MykXqu=bnkg2?Xns@zxp;VRt*UraqF=DGwoDeX>>`h78aVV z!y+{9(EImA!%FjkQfrWlFu1_v9rE>if~OwMmgt(pxc$uOL%*tw$RqAG^c1-{>+lEX z_1zP;B}?5#-aUXd(IS82b6@*f2EPeMxpzHYBBEIW)VKZRT(VrE*5$vcQv!Pb0(Rm;o3@lsf{;sy)P+|Ut*qTkt!U%}ORUA=PNeeX z5yd`jn~RNdS9iCt&4<{=zNp$B=uRrzgK)MZYDn)39V5(w93&%AFOvH2?b}!?&UPmc zM-UCx8ga(Gi`(0NMm~JF9GnpSUFkgF$fto}rhdV3xpVvW6LmpJAWPu*vE!rE;ygSn z$lQUZXxkOca|BXwi15wbt4F+{$umrbl1otV`=k9%tB6#)6n7I(8X9Ejm1#}4&;kNZ z(S@7KIf}u@!S3#*kBwd;l;3)0xXk*&RBsWS<3HfoMzFC zXh0@{x}2sx0Vd>*eB75h^HNik0|*cH%Bd6B#^K+xtYUk?EnF1fZKO+|9}e=scojM+ zs=i2YCVRns2r@`xXjG|Kn|}PE^y+B9YtioO3l##!N@jj=$C790c0;j;hWaq)4m%xL z`cd=MS4FR7(KRMnWXa}-;?nQb= z264zh1hTN-Gjl{n|44Ns}OZaa7GvV zCpfRW8RU2pvz4py_eHjB39$k)lOY*;?D-*ebr#Y-kGqj7{soy#LIaJn*mqF|Z9cY? zese(FkCsUQv8WX?0(UN7Y$2)zu-Wwly1yYKHg;>XeCeS|oTW#I#hMv%RO!@U5E zfx+aFsH>1x{`;Ed{C8{#POE>$mXusvaw=0GcS8<&gF_5fn{<09r7oYVI!yHGnB#A> zsRD8676g718@vMEypbp4?WjdFB%dWDF!8&Mvi>z2ApkQBaa(L~hpGdSTh|Sb=Tpz6 z)`yXPYs1MfH;O3;@k~JxrcQhYSC0ZB3s?jGoC!_pch6OlpiVf!aGN0Fr=}-kj2vqa ze1P0-*tl_1*llMVwuwE=H`! z>IIRz6bZ5#46Z6)S8!ml+fEHrp0 zP3aG7YK~JS;O}J+r_H;7Hu>EBGzW39CeK!tGH9PAW?$t75rI;iFhgfol9nR%P?47zbc?&8H%>vb+9-U$jS8H4~mCJ@gf_#G6M)s|v_Y*ioay~Fb!Cx6TYkPMl`#QM$3 zN7(EGbszpX>cGyK9ISH(XSanCi=2Rsy~{WDegP;2QpsdwjU7rSvJr^h9+%IA7v}-Q zVa;bC-38P2$xomTA>?-5h?JDn&~#sNiB-BvI+$}-07;r+91gx8BOnG?u508i?JRiJ zmQ$ji$cFuiX4v65Dy!M4Dv$g11k3ot&{7}iKmRb>`9K{-F@i4_({*1yv~Yx*2D^K{ zDR-t)q8%fCdAkjvBLWg}cA^`^c9}XV9sKSm$Zr({TLh#@iUsKLXs&{mCwk4)E;6)a z_HNa<+NwLPVA_r7;o;3T$p{HFpiA12u%9&jnT)--s@d> zRwFZE1#E&j{E7YBIu^&ju!;S0uAc)NsE=**4G!Aa61HR7uZlidp=_2EIcGpG^izL_ z&6@+hHdjHqm+TwN^+FufeauKX;h|w@7>GOunrZA>8A=~0(ARC+6o2Y`2ZovP>P^Hz z#fcNal}ijTK_$-PD7y^z1wogPif9LYNAY{zyo!tqF(D z`AafPOAcj@$rBe0dioSVV*SOK-6YDPzXzqSE&ua5sUMd7Bl;N33=r-RIEm0F`^kt#PgPp%!sz3If!0iQkh?dQoIRJ3KyulJk_PV@Kn4S{kO5y;e?PHT zrlX_l7^#vCtt2xnXl{YkNV7uM!wPK{p8Q%Ug(_qQQJxb+epwH{etC9vovy_+DIh)6gc>sj`h|vFfyJHd5{PJU>A6%)Q5lu8*zx!lV0@P2*k{`~!!5pp~M8Id)a&zB=9&R&d z`7v^($rN?!7esvwIb}7kkxc1;y29>hZ&1BdWep%Evk}ZWI09}qK zXkQLhr8+la5{btHMb!unX1x{Js0OkB%Gy5|oc!osaMsAJxw*Mw*l-ZvR$WaHcGQAR ziUx`?!zmwacXJaihcK<`!x?8zI*dq~h; zNQ%Gr+(wvZrX(lB)jbE}|JJ*@@nw=H2xIeWX05iMHUX(!A2lokB@90mSW@39e)lap zCgu+K284nb!uV)wY8q45>m0#wXQbycr0V`E2)qel-#6mqnrLaXn307EJJ8~luPU#KLehdySyy*ruKM%DxmglWS z@KOLH2WNuwwZRC*7r?`YM<@YN#ppi-=Hq5&n4U(FNi0es;eiUGIX5{UDBWMZ7?csD z7h^#fMzIH2j(S3~QF1*2AcO1c6&xIVa9R9w6mkSn$2m`Zje&OTWWon{rkb1*%+&Fmz=h!Zj&MO9Z`F1uiK#`o0h6pBTxk9wIPZuv(5a00&(WYA)vV~-{^CAz6x8S9uZVhvOrU#Jw;2i*N z0_yDnl)q#N;GfUmXYnqidr%8~hQx%Ha}_j-yWy_1Ok2A#Vg;!~E}ul61VY}lB;_&K zLn1jLYNFc--I!e{U;S&*RkCGD>hZBCd1ebSF5-bo#W9f=Nn|}uhsb(?hzHQy9x;_; z8S*!pU5r5d-Um zsB_-vAQYO>n0>31Fpnfs*0m_q-qVxN`tK`xDN<|}#6o0l7f!55?#JVaFog{43JP|n zxI7@ldX?;gKDUdrbEh_{4Yuz>IGh!hy9FU%0^fVB$=dyA3R0$BSU~QLI7eb9(3a;y zloy zFz)F%e;0vSZ6i&=%@JS4x9RGnsDB+U@Wu4x4)WlDoS@RQ>MtC2*&1h^Cc^8>K*RFw zMJdVvrqN3m!+gpVG~S@!vx5*%ckaQN2vSQPjQ8t5#I8V0wotc;Y~SwDVTZg6GuXAb z7R(Ybl~pC`UVQyoi1(1_Z-QPd8!&iyn(Q_N5|qb(wbT2dCiwAi1@hc|bX^3FL}xH? zu?>tiZZse^GiG5ja^qr!?;+o9#>}7P68a+onBaEIy!G4z@z#~T#7qdj*3~aTjR4Mv z{)DuSfdQ|~)Hk-e`ucahsKKy+62Js)$TFPkx)q4f^S?t23W=uxum+6wC{QIg(#f!g zu+5)(mN1?cImd+Pjv6Wd+YBVa%qT`OBN#9r4l)dtC!!so00x>N_;Xea+|dOxi0B<- zOdjfgUe4^0ppj5%Xvp{F%NIUpYfPhK13W7y=R|q9T;$*A1ML!~LJ+7CqPtL48(}#e z=6@_hCq7(8uU8D(q&_BH%zh$26% z(n1$vVutOxLyM8c*i{r;kN%VTQ(r@z@cVlO98T1I?qZo}HKtM|qF=w((^6AaeS@sX zu2P>v(7?U+KW-(g#?mD8bt8sCf)_K6v9x9HPpQ{f&wus0!Ok)eRfJRM{d(Uf-nZMp^^m9xdRzC!GS91IW2m6u zLEunoAZ$DFiJ;t+KY6mAHV7M?N+3K!Bl=V9s4V$4P>MXj{I{j$%cdt3B&BWLjtJxE zc4~WAxVRVs6iq`%*r3xUl@`z}QfVp1R9hG(Y7xE+)UInDepzo9MnCREL=Z`OxFjke zqUPAYU;mofdr7PgM8+4Qpc^vz^hEAG-Si?HOS1Qr0RBl(>j5H!l6rFpq&MaB*crAN zqh*Ip(PPf`pONhnBy#^`&<4bg271f8R)iCCbQ7*1fS2O-xKvOK zDPe0Z;jz^Io@Ts!|Ifnn6}TxrcQqV)trC%{32Kk*um3}sJzw6K<$Op{Fu(WrZu7+()r#&M*&vJZ2kH={UBZLaM$XguzZI+}(|Ul>Kw+u<<{7Oo48y3mu6)*mx(%Z1iEUn0;4 z*mF!S<+MyD2cvEz(iKtxy?Yn`mvn_yLmT1dkLkuD5LR<8E;F?fVH8bKl<>;ommFje z1GB0{fu+G9OB50BQg+xf8jPD0o)Eyo&#z`U!e(kH))8&NnCyLR#UJGFPo=XuJEIRs znZq-XP^0kjxNijxOm@0D2t5V6>bNl&4A9|aNWkU}_!^9h7856iDDo4>PeNx!M$4@T z_6I#K74r!h##mezOpt2Acz&xpPP(hTncyG)d4q)x#Q7r7Y=mZ|6091s*e@0c#qr^; z!@)u(D02Td!5S^>04@}iMHhT+a2A=R#X;2*&I_T*<@`2kC>yjv+`Bk-GWB zJ|Ka)kW%1N9!I>r<7NZnAQ)?e{2R^syHQdiGiUC;?C4mQjDZswkL0r&8mvU~5@4XO zKa|rxmsI6p6xiW5Z=m9*}VDm?j5=({}DgJ6%B!#m52&Nq2l-%j!r zNz~LeGSDb4%`(3Dl1G1co;_`%lEdF)9XwFs64->fAzqsp!k}}o>#f_g8?K5&dx42> z5Ee!!v64IB+23ccn$U)j17lVWQ$3hF=)dTW8Gy5u% zh=-G@EOh?$76YVSfYQ0W3<|DD;hTa46vm<^y%%rG%1ptKQwgPBCt~G!jdtl%JJfpa zP-ATl^P6VWV6Xo4i3S%2<`9XA=@nDpAD$ZeeQfiCVbu61Hc3CXnenfnbZf2v=*Ixo z84jW?{_jhTl9;%Hc)(r0JkC6R8@&P<&nGNWZeCvOG-ir~F+8Bs1*3lA=+Juq0jR=o zz%d|oE`Uteb$jE-3anodIVIGndtiV_pHJgQsK-x_>$S?)&0x{MX<&_TugH9W1**gg z0)?^*^08Mu+C2OmXfxMSyXvXm-vL@37g^I(`4DYxhCanan99#;Ixyedf_b!%3v41cOLOx^Yg`zGv z1My(VDcih_l(`=g1=j#6AG{HF0;syUk*&Ja64pBkfkD2%%^P)F7#veoZB(k_b5BD@i^F-$KLi;wr-g3f+$ zw;?wkLrx$RojUBuz#$wuF{#BHo#IPPPI0mrlfjXc;ui|;iLXjL9*>GpfQ%GiQt#!V zi-^w5M5ui0)~%)6W;s!W;E9b`Ab1EaC30DeDlJ7;5VuFoO|-*7L26Y?`^-V;!(80} zIG4&EtWB7$D40ExZ0L#t0n8#TT2^s;=y1#Mw`Z(v8E_<@|1A)KI`3;EoS9$gB~trl zXDy^NWTOp~Iryad`uqFiu@&G=!{Z$E-OmhkDdZ?JjLWl(NfW&tkdD}YQxV?ie_pAa zC@gTC$YX)=)0NMcVcA9woU*Da#>30VIc%C$^O!z?jBv@(3{G$A@b)T%2-Q5D$ddt0 z|HO4irFsWbnb?o9tw07-Ok|!28cHWwA*2pGb__@}rqG&WxYUPR$72#ioby3ZngZ_c%mFte(}*x?j84u~r%Z@MM^+$?rPuCS zgk(Ov7kJQ2J8FRY1JlUWwxcnZLWFDT>b#-V;zX`EeiTzNt*;$m3UN|wO|>KH3p*^& z1eLZDxXraPdhk0NH`1eR%w6+`+$g^Whyc|bEO<$iOrfM9$h5ZFi2qy;J4L<3jEbon zFj71RUmnm7fNk5q=VBrp)A&Jm{g1E8!7g8khid;n`$ijNU81#wECTYA}B;<+06O*Zhh)R8EQ?Dg>T} zc|${wIR~f-~2Jrg);9sz}#vmGLNirbMtt2nsfxhs6>qju`0F3}f zf;+*LrjE(wbbkNt1Gx6=7KX+1zuM;mqy^;VN974TM%43{t8Spdfp7Ck%;6kPM{7o; zU)c0g=9OW26DDS8Nnkfv^DFyCoCld$Lp274g3S*^KM>~2`S~Lwm=(Vrz@Jg;33QO+ z_RumPI(<5Nas1D-2I!jsI%LFG0Zc;b>7P1qS?H!9xNjd66z_U5da09hc`GP3c9n*G z4=OFJv9OgmDj31<7-MQgH)>`ym^-o(l-AQ>;_QjhWTFef5SJYo^s|p{-@c0cv>DU3 z#OpW)S%=%+CZx@@wq6P&hwhUHaOwo*lB^Q^$4?>>A{-Q~^JDIH2i}kP)BWXx)}yS( zo4HmoH5z`e189sdU1bznnp31uJ0Z?Xtawm&3sR=wn*#8asD?y1sxu|{9!oha2gZx* zdZoh-KNPTW5gaT97!EQE!Fw66=(eW-@iHW}ySq#ZPRPKMk7_phx5DxeW|MFTSH8p1 zOJ-4ECDa;nBZy(EUCBT*z#D+z^^u4|VwSuL*FpefG5Psz79Y#XmZH>wG-GheIPVeU zRjKCw)^~`Yk3>R7RGBCUEeYj_4!86f!5_$O1VRKn#rsMPB4(tG$wr~21j@)f|Cp*s zc5Wl%9eR4A0nY)*QAdBMZakUL{jOT#71jJ-vxwe}+YDZNL-l@kEf<$x}% zvbG=KKMr6EZ?mPK^~WEQJ6c_IW>Z* zhy<<{6cl*k-S&6kouKgyDMe`vR?g!Ex)4dH-y2|-wIyK8SsF@%C-|e!*k269S114$ zBm}f1jfoo?6ZO9ulRf{VF_EAfo>x<)Cdz5(88pj zs5N(UR}fDw1p4<@)_2#(63Ph$9MT+&<(T)MM<0sA=hU&{Z{Hk%f=NjqExeMF#6vw+ zL2HN%3^P9PlHdWL^Tnx`MhO8F`5IaVM@fFby%=si@Vs{H!cKBPB>tfmS}ZhpY5dTF z05E{6mqWQx0tk8T%s@`69?(cc$Yk~CGJ9ZVu2bJb;RKcHtfRxFm8eG#aNg%G?uGPY zQ1hK>RWiZ#?$foAu+uvbMM^j6LGTT*M|!oJZ?`22My%OY z3MzdCZWJ|L5(QHM)Z_Y3^NXPR97FC%TuX_<$!1mSJz&>Fe}TCURLH)optmqW%ps0k zh}61ne@LFGqT~X&uM#^j;JOZy$y$^h7)J!=HJjr@nf4^p@=`wXdNU#;Vr&0h~7vbY@SU3WwjIHwP)%t1~Aj$$t*Kj z_=7L}FA3y(@7}k6T_N40&udL!d7JwYOqe13zhx#nXmx*`k)2tCuZ%;*OcrXdx0nLo z5I`?iSO82A^Aft*>X$DJb;v!Ea!}!B2HXy;GZz5N3AsepHICs#y7}$pYDH&((xAp2 zBnvv<6u3OLjM|52$kv;s8haexbzc0k2(OPat5@zLG2vp`h+}SNs?G7;BQbEH^-rC~ zw~oc6aq6PIFx{}afKe{0_9O;UwZK3{WIr;WAjXTY8869I0GTje4M{NsnTU(i=NFL34 ztds}vfeDlKkb%b}hjPONnR7wqbGsCtEW)x=%dkz0U3Kgyurn+GF0}b8PWX*=F+0P9rRdmVeotcam7q;|BX1{T+@QDDZL0zw^qF7q|rj3$8b8)>izqQ<0mtGK+uR^0LBdA7l55|WVmSExNoig*$8LdYq)YC-a zg3eXk9@;n{Z3NI|S62MSg~A<@NMgSeo(;5w7;Aj8+%3(2i|-xKzL#z@m&5b;2~t5U ztgUUGA=eQ<-$E+)|3EMGE`Lv`)RUoak9k)g&&)zx2hE?jG?v0~gl#p$y%swWJfJ!^C7=7sY9@E%Y{M0a0QR0K>$ zNn1nX8pY&TVGrR+1)F;?NcGV90LLRCmyG{Nz?P_2j^eDUR&K!zgDSV40DQ}KTkC@6Uvl=g^= zcmB$3zushh-->9C0W#nyUIBK9wT$Ar{PZ+LY(gkj1_pHz7#mjy&tQm*sO!)6fBkyR z2*7l%)=_|?ig2?bYch0yVa#X8Oz^v}4&(9&X@$mOPyUm8d2!e$WrE9B5`7qc(tE3g z>EwmwiCSp?{^#rXO-xBaxrP!6>PU~FN%R)P?HiiM)BG?(eg|iv4hR%O5^4L<>k@7U zSt7_y21yiLhTyJUX%#b<78SyBx@!Lj6LwNmpK0h5iq4Py|B9T<{vTcM0nhc`|NmzP z{oVeb+qqp=*L8InpU><4d_A9!^^7s-=+uy;aOkCo2}T(5C`N|P2c3^{y)1wRIUhMh zX)dzTz&=ZM#>Mtj*oJ7ToYP=!gzae5LVsw~+~|p7{dKE!3F=o|4gYPO#6i3((jH>p zmKow#7h?gY&U_N9(_w4=TL1FmvOq(}^P9Ya-*W$dCt`MtxHjC%F1xKMDemqPKRCd~ zV&!=ew%jdFw)4L+&rUS>eC+5UlxvxCnYAZpN(u92+oc|qICSo5hc{2OINPEykBJ3 zKrm(cUcm&*27uPD$e*Q6kg6mHkk5T|p?xggeKGfaYI%zxOTt$v@DtAm@BM$wK(Wle zp&+u-{GBiJrm>`*yOC<9lbu{G%u#TfFqhBTJ#%|oZN%~+3|hWj=fJGb7&t%lbo;qo zp@+D6_wK0uLv=4iTxp=W+}14(+A*}xe;U8(|E_-+gZ`<1v&W}soSt=K$3%x61<6HT zBd;MwdPn1W{-sp60|#Dq`Y}0|UPi_@rm1&4;VOn^$CBoPRROFv1Md3U^p4Eu!mQIE zYJ2-}Go2qA%5=h{I$Jy-d9xHH6yL)68qhsv8a!l};V!9r@% zP*;W-X8_e<1U*rCe*XjyCGscK$%PyA>m{*R;qqH2Wzwf6b=TFc4V;iW`By!NEy;C4 z_r7Rtr%s(v{kNw20E$T+{E6F6+%7&G-JN93S}9buMm7L!Xl8cZ`TqU82qNh7oC2z4 zIGf)%=}?vt9xifGg|34nc2crX|4zlUxU{jdYsB?Uz`?}U{O9F2>A-;ldCc4Qmdm;3 zq-bA_X|II-0mYwGM9!E?c7+z#fC^s>HXjTAFXv0rj*v%RXGB zH`3D5RwRxyNeO2=th6aTdR+RS7N5njLwoDomHfdCAeGD#=dY5KKz~bQ__6W&{vxxc zlk8vV%XE8oX;O%*gUYDsWf7R2t1Ui3avZ658}N;upGCg5U@3&uNHH`zv~y>Z@?y72 zgNKVRR4>vvCA4rgs;EE-vE#l9ji@B#QZ9($Xg8-|`u#A-I{NmCYh{GIA*cmhmS$auK@9+}KnWOLk`@RlE~^%h!h>3L`#?Vr{)CY_TUHcj-j?)E!7 z;Gm_K-}|)m{rg57NP@?LCiL72jftwW7pV z&Gi*mA5(31tC%#r=>URzxc=Pc?#?#x$JsXa8Tb5% zW%Gg>2B@f}*CF)O8JclQ`l7MZMsfPrdZHVU5FC_o<=FjAZkA^za|Zq5elFY5)9}#G zYn^%fEX$c$M3Q>sybN#^sJs%Ce1j&tyo(uO+F1Y#TMl|No zuCHFrei?W&pG=_Co-50Bb)1?p3`!mV4Lz{pygs>vfst`E4@R^61t@7oTNszKuxmXu z`#^Vz839dc_|V>W0YUPxF{&X$n)C?&@Q|_>i3-_kQOUnEzji9OXv&m2zi&&n5tP?i zd|~E^N8}2NHAG*7cO$WL&z^V5`{{Y|SJCBIBBX-g@Ji2695+-ZJ%7G5Am7>15!-Hi z^Z*LqdCJOPnB9lsp6hF-ShsWP|4z-LJ6D0iyY4_FqMSSNr3Eh1O`B?hUJtQvJX#5? z>n7=e41kCsDp%Zxbcr58d?ph_{gZ{)`hA3_LDN{5Hh1l-eD4r(0lf#N=F0i{qXlRb zER|ReSJ`F1$yujnkv5s^3s{7ZGmqERT8pJd@M*Yx_k|`d)_f46;*b!9Y)SVcx*mj4 zW9&1xsi;|x`5MdSpi{r_;nN$Hk z{QGJlaryN4(8;+vr7^MAr-tv|ntKQwD1Q3MTiaA!tC?kof8FDSwNL83w>NEN_SU9O zT5UC>-)ann*z1^Wc{DrTXR&zex#)U`6^RvpR_+%rTo@i676F1IaR#(+7IWtOy72+* zPJlnqi@ZcWcX<_dm0(8}%CWSFh}y!aLkAD?0M|xW<4jP;^L7JO{XQVg83i&6&X)SI z9PgL@%D09kB_#z4=sLTAb z-E-1WxM=RByO8rD%l`TX_gT_JA*bVG|MzRs1Axy}EKgr%NQNjM@Tr6arWPj4mUWhu z6!2-+l=>RcZa44-S~e9XsK&3F~9c zR8EA4-;_9K^yV{ND8DmeHVo|B(r5U6CtnG%$-7xkgd9)dHh1pXqYjy8Mh*bN2lp>d z{+Jo0myC4e{R2U{CCLBDXw)?=2COy1X zCQAE@XIYTIx3!jGdV0#>A^L4#F|a6p3!$9O5Okeq?04=Ruit`r^es%35kKET$W&Qs1f2RnnU>&9xQnNWtc+ z;;{;N8V!Z4-~b`stwP&*ypyc*AxqcLHU9f$w}G#4{#Je`8|yy57E0#}?0-m_}=ue(G%Sw><4hS1_3aSz?WoBfR+*d6f5$k zF6F4}oO@Z(qUvolFZ-YQe<*qd8^jSy8`Macq%%2On3oYTbUwiO;IUnG-8 zWk@+&ZPLD8eXxa3Da}V>i?xy%lUJi3W#c!E>3yY7B){p z`Oog(y^sr+BaDzD++)RoZ*+LakL%)BIJX*XPF`8Yz?E2X8PA8KcABzs*)qT1A5R5m zVmJ({)8BP8QAtRCSJ4@Tf1(AT8Ul85ea<@lO!nwlthD2Q5|R)d*PO?`?7oD$-MD@G zIEA(`Dq&81BAR=*>6B!Sc~dK8A`5u-4p&aY4hn+opCkUqo7e`K zzk=+IJuwN8w#kNMUBb89(R5>youl=NL=xF7O6ihkTlQM1s@53RGBB&{ zun=iEiri+zz%}fUbBGXY@&2N189yxzAudC7RNY6#vbvPrVt&7jVa2@p&E^Ys+_Alm zQ*ZFT-CtwZvhIfM57!c7i&gMR`*bCCuy4zNMXbyOW+XWd0))tguPSpu` zI8H^?g3O*;F{U3^B)(UcAwC|pW=;NrL)VD@q;!+Ey8Efih7F81s_N>xzd!j%plZ$; zYkuw%+>L@i@u+^bGuGE{$~-MkWc9(G|GyxCno{C@!NQ2u{B~j6c{2mWsPkM?C`cH(35_Ee)LtTCDvr{uI zmkN%)wM_FBZ#FEqlHUuHLB6sUs+VOl3AQ3{})y2BN;7_qo?8;e^#Pz^x2zsKS2$$qwbLz?# z@sl)VW?-dZ!gA?ZC9b~V%C=v~MBp$>>q{7KTl6izuNlv5EUO5sb(X=w!HRSJfxjwt zH@rHs!IVuDU%EH8rg!L0acbJSb)(2xjYU>R=$t-o1r*qaDB1AW zbYU(Pm3xn^np9){+RnA-9EVQ#EWJM3ywiY_j;EzX8QlO&#EC)$IO<|~E#5ogTi^7> z3q1IXrCWztnaNCdG5hW1Fvn`cz)3Peu9yEvc)jEJVbQj5-ASBoQBhGnElr$8v|D^! zdnUvxz9gI?>k#7MJt@~O8t0imAiRbO^4BLzN;H44H1zg#A}>-FVUm>L;Mjfg>1hjy zbRn#I(uVH>be%uU?<++6#fvHbO>il_%>SauQSQD_lJ4nJ$HRO?5NplMa6VVLWL^3; zK>+YXC?dq1uJCDG|M@BC{48p<@K@qAOIafV=gc9MGyisf%>{vFC)BcK!$=xn-QZv+ zok5Bag=kX}>6)||r0XM&I8nQ+$vp7S%dgr0gkZ5{Z*u8NN~dIgWn0I*dF>?WX453=bC^5F$-j#VJ##u65i2Sb^Qd zlUq{qiB}ERv717X5Em$p-~YNmLt@4uJtr)OF8I=Z(GQ91Y*JxN^nlV38fQh+BMGQa z*A8mKW1OFos@(dJp7cASZCq5yO-kOv&f?u0%&R6$y=!iF$;=ZD9?ViWcr9N&?-DTB zZ8Y8CMW_PW4jed>O=IVX9(>_sBb-0(%)QDB>$$NrR%k1qzjJV4;emzHH`kiE$~)+U zy*hU7mFuny9$7E1*g@{M4omkmkz!`DaOK*s&A(iE>e=$xM-P>bI@hkec<%lvwbj|a z50qnGzKv{<4Gb#^ubG=IiSCd2`Kc9&cSC8QlksSaM3W_(TC7Ndmgia)O$ZHhyMHtx zn8q-u_2R*4)Uxkrsa6ul)IaM)SQw9U!h%Du7WS6DWnEkYC1CXjico59_w;@>bSBBS zl#Ei>6>D3vdjaxYg~43QOdLFK@Dan!s`7hoy}$7Me&QB}a_qRuG)9ktfE2vNwp}<| z+GMK)YWAB8&6gm0gno#pfGl$RMoa8by{L-6&!Q@10tdkDF_KBg8orN43MG>QVl9-a zP1|?BxoXeu-Az}mItzZ5<$><9eN`Oeq|^UT?D~;l0xLVplAq3d6a&jYRd~-7iTJp7 z_3Fz-7gmE73zdR%aX3do&hf;FN*7+(t~qso;czU<{5KLFa{IOkI{Oi)zs<1;Yo=Q0 z5TYm9>ir|y2rvaKp+2^7E}avZ=`GL~kf*kqoxLyWrYPYbai_bI)u!a%$&sN zvQvkviUW58_sK4z@`G%aF1;b)>rnOrwBv+xJ$1t0-MIFSLm@}f-I#XxS%*GBI z2mCR)scNAMjQ-x8ijH|@gMXBs=K}`&bo>&NT>IG8m215AybEcb_j;&#qr5M3G%E&U z*SuLz8^UqYH0vr)bU3(fXUaMPYXTLFyERX!<=cw26KrTNE%=0!LGk)!3WS2amuMUG zA9&0`exO(tp*F9topB__ARA4iI7AU%^l3Xma~sx%j+KI38us ztY$xp8N?w-suKP>%T^(GI#&? zHCEPnAsA`vLt?EgHE~E-gH^X(bz=1C(_}Y7q{>u>K~lMA&xjTGM=r;2USzPCJ7@uYx9yH~6D3BFX&C*?!sHueF6~F|7)nMSZM^f}S`^h6K`(4aR^@Qn z9y^226*Oda2>V{qALJ=B$}BFfCvI^?;h&9Ah$DdpN1=@H{x1zJ`Q_P0S!YGUe^U z6@%*P#XwnmTzH%WClRjl#2xyCLgAd<$FX1)4M&fhkMpC6qP|MPa&78}f93U52M2i_z|i z2-zLwVW`npFAN*-RA3z*_ZMcL3$DNUrV=No$mB^Y=YlL%@h~}xOu(%o)2+GYKUG`t zr!Dwa^xo_eUI#s)b)y6r7?f^*!tgVDgnTWtQjEc$im0rh!zkci~>Xo`9@YGm)uR3EKls6 zIgL1F(M=_n`oovB>($F*Vwbz3)nXJpm0V%WAbA@vVeLPl=lj8X2}!PzxuQF!bHWwh z4_~_(scku;)9ESVW{^~wi6I6K2-iiHr6mg8=;F6(fu6zOP$M{^eS0-n^;>T<`_?)- zq5WQj#bq@8<|na>y53_;V^+utYaN|VbI*+#F!L|ZpsBxy9BZ1?XXNKMy*fQq?y>yt zvi>ovKh(we6cM(S;7}f5AbM3b@2hhVjZp?yf!k_Y9pl{w4+qX|K+n4lLvz)uNOw## znv{>VuA(&NAAY1*hy1XBkKewX8E{a1u>%IQKrirSSFmTPH{+gc^uq3)Z6^|Rv8iEg zJ+zyXYGHo$jNOgCm@g;lN~bjZh!;gp6PYNYU}-Y=oo$AOwPpo!Ko ze5LliU|#S2PNzEEnyb%_tQLqS__<4IUmKPq&zzJ}5nGvCnWK>#Ds%QfucU-mc=cer zKYqs^wQJ9wg_jepIaCNa;Lnbd`R^R@G^db2NBnNxa0wg|^&q+wh2EV#G8Wb_6S8Jm z?`cq1I2oQUP+Asl^^s8cz+Sb;h$&JYX%g-KXmK19n<&$t+g>uuXW)>OyOKcxSsI2; z1@&3cU@|-CN4M5IYNC0DBirOZFaUaY18RG{E?p+2Fo~aKEpSO$+4cs{#u>Vnw;*9^ z>(k|XZOGDuY>G#3T)X3%Zphk10{q^A!KIU+c4rdW>x7%=n z$z(0B2UV5vT?{Fjnw|8Mf}*%hCI8!3Qp{p2QY)MNb6%$yyRQnmG*WS13*XKi(!$;_ zUk2#W>U&e#TpOo&UmN_jJhI)#d9}0*)_EfDn&X*g7w=u;?UUsxA;<19 z#vu}`(;?*BkCN{Ti0Pz0VC6F6qJp(jo^r^#Cq`h+ zuV+tuF{DP`L`&lRbXmMM5KQyyk4!Q1i!dC=_LVDV(}Tfz4WvTS$B`WP*D^`s#DioO zJ#+p_2by;Py6F};DYd}D>vYJr5~et6O((4(uBXDFgk^y`kN5md{CaECp=M;$pm zz(PVa6Q#>NW840%L`~lqcZ%KbBwb1P?u{uX8G6BAz1EjfPH8JtaX%kR3d`D_S6H}+ z1|uBQRt7coNYQ@1z8<8gpnv~MnD5Ux?TU>IjgIBdX0MzF^t~>T{8vSucxb4@tikz) zh-!DV=v9+9b2E$NClAL=o5SftGBYg&Y+OVx!@RCe=awe$vcc~ADIc$s!wzji0yFa6 zuLpN}It7tbgvb?l*45;LxEz36cA{lfMo)NN2fk`G{gsbJnUEUkY8$P`Ky&= z>@+j0tHdIx`g=ot-%-&X-%Xw1n#b-d%aiy}-*5Y3Hrij;a6?H`;$y{T zrW~<`1>}~Nt(k!9!&MHh7WEUWlmTvu)+BzD)foRLPoA{ZemK?2stMdFu)}Eog0YF# zDA@V4?27A$I*;O7Rivc27OhqF%STIan;X))BxW(G>}U!REw&vj_VQs+ zG<7@SR#_o@6N}I596uJ&kB9>y?ZC?IzVbN1&)7#iSCS3iv_`CnQ6WJm%X0>lT{Vz7KaX=qZ2($s!%v;_d(U%j*d+HmBKiLQ47v zNi-k1;|8E4p;Y3n=J6x!_SUDFK*kwU&A8E@ykNruQ=K`8sTEQ z()h?$E#cxQgy|U1K<*|#WS2C+@^AcBLgazY!LQf;ql3;8frwZfQJ=mlw;(?fJ2DH% z`hhv_plbhE(l}`xgpXjta^`7^quydNlBCR)vA_DKM#gMJYJ%s;15Vs@4jOG9yIXIy zvwS4Dn$MR}-|7r(qGCIYuWLd3qP)td2-qyxj!6b?R8hgH-JK`@S)Ua{1$JacseeE< zCxH2&L9-@?e3Qf$+L8od@cHh`-X2tJ4c$Bc>tfz%PGu*34Wq|#L)4J$ z6@#yfVpF%MBiV;zZmI9I%wLymNfl1J_lUG$!G=Zig`NRP;=1Zg2WT`?u09b+^bg{? z#aAZBoE_8#t}FJAtwYwF_Fshs=^sAC)twz;^?(||aK)gdb84DRs3RkT>_!$@cVZ1; zDkMi2q~D_3CbzKwZ=&2pvW8xQw&4WTcLxSvz2&<*PLLdg+8wf8`v)}^MI=+#oj(Yz9DmAeX8>Y9_X_~(xD*h zoXrDOO}W?Uux31O6u=L0J7+@9@G%Qbu(WT(vCo{x_-_I4F>`)Zkdvx{JYpD+0%HHytG!s7tAX*3CZyK2O$jI?WZPANrL|XsKo^Iyy zIECP`Q9}MI%(N(eYO1L!@gudPomV>Y-MDbGRgO-TyzKCw{#3Rq zd0E!fJ^j|sEsk@#IHP3JzI*$#ET^ouIifP$erMA9ef#%&(JFOajxtXsNX4@pqCNk@ zi(Tk80YQ;4#^#Yk;{l+6xZb_qBUmPRSzO+&+W_+VP?Ic{jE;1;IB)&>^$-DiTGYK& ztLG(m_v<6`4068mTRxxZ;dY8hDj^}^u{5AeLUd&~ZJ|ErghpXKmIWRB6gB;mg78FZ zf{%6bm5{M??Fy0f*POhz=NNxcEwIxfJo&ZEKjh{V%^v?3)-CMxCUXDPvPo-J=5QC2N<}F*!?Y&=J5pw?KFSVmjmw&W^8D|>X$nCJg z{>}DZnQj%GfP6oGCqP6ds@3lQIi{ZX&d5?6$f0`+h^;8cqNy1eX2UoJamYA!7m{85 z*Znz|3*?kie?8CTFu1ipyl*Oj_X3ZEh}LBx)?Yp>Dg zpu$|pjwFq>aQBil`g*}1^}s{M3!9dX{UC7C+J)Z{fJ&L z>nEw8XN-dl$T&v><4T0fz{14!7+2EDKN%pYvHP8YnC-Euf!U8_;6P;eV?DI5>`S>a zw6>CL6(rrpa9z;(mR(}o%Hw`;ds1ATplu~yip=wr%#hH)&Q}X3uaDN88r=-kSH*zQZ;v5Ovj`w7dc`%Qddg1raC%1 zFMx`a|Htdsw<`y8+siBDtt`BS3+709$9Lg6K?QvCY` z)g|{aC6~4{W&1|Y(oJCNBBL5S_*#M^`h;$lSDK=;J-Ba3CylbBNXFkcXa zy#oO5K_nU*CVJ(n{6_9>^#$AG$&S3;*vTu)hmd7Db$olFlNC$ZSgaG$GbA#yaBpxd zw}7vf(JeY0^}~}RQWK(^nmf0Du9h?yD}2hcXAeGLLR~GgkAJ=HeM8l*{s9Ji*qE5Z zWB(XVt|VS>f&(_fTe03LgeNs2hjo}hQT0e9f!CkMIA(a`fM=G*AtqTFI$HtEk$>Jn z&(8&BNp{=ISb{i6!)8Hn7cwIxzMd!viQ**yqo#i|zYCcCZ(@vG{qx77gGSENth7~> zc=M&d0#lZTn<`U({Md{VFyV8qzno~T^;{js`EQ|$9SaX{fQrWDs!GLYx9zk%T||&S zgGAp{<3fE30QX$o!)_H(A-+R^qY0(DNmCQQbxm794k5``F+D~-Ky3V zI*pcpH8v7Q%jwhWU3kGQrlcFrjt@7T)Id$8ScPmfn z^pkE?yBls{L7X^ePWM#VFT;`tt=c(lu>%PuhfM}iL&erqzT-R?H4zS3T9&NCY@&F1 zrK7+E!i>K)eT;ub{Gdc*tF`kyb;r;Nn=D$?K^7CG8W|c^UCZ6HpFsECE*|aKdZX?b zri1=w2L~X|`qM0*A4Kqd-WoR0bZjf@-3pJ>y2(I;Rwel$UM@yki2|!}KkOO_P7+9I zWHjf@o9m8h{9k3Bxj_IQPhuWvujtO&j`6olOh||WPdm%D$QAtoOoqqbxznY;T{pN5 zI0%5nI-*p}o(Z@s1~XQ&S$6Hw<5S0IlavLPbLJ#PSP$WXH9y03OUH|;EpYswD?WQk zyuD>{UfxE6z_mvn^D|O4%lcW|ggEuu|5se08vG_#O^ME=b_Yktw{PFda#-n9%ZhW` zlWBlgZI0D!;3_DKbm;PGfyEQPvDU%~CIz*GtM>Cd8U{@*yz!=HUdK{%&f%?*^@YgU z@zQ;^e}qG3b!CMM&+P)rA+4Sq@jjDJKE*=QyKbWv8>LzJ6@%(w*y`22Bsg9s!?9yB zT3cFLN>wH~PMvHT?I?#5!_g4Vv^Pq9VeJ8N{`BO!0hF4vMtFni{8=7z;y%TnY{)!sN5 zr8|q%Cf^IUjsO0BB<*uptLM7!o*WxF!}dnjP|h~gVPb`Jf?<0nE`=|ldqa0ueG(2E`|sTAvi$f2Fyq*+s8 zw?UA^lzkg41ArCcBbjc5Q|0Hou?2iBfCfK~m9suRZaMuhh~Gg+QgzlH2|*Oi``yPe zFSJw#YIFH;zracAAoV&Uaz8wgqX_k;L`;MELQ$K_G9PZH?BpS`o>f-ClGLdwWXgmIKJ(u$zL3roh1#U9JVw|(jDMwV z_p~y{u9cVWp)D#UnJphaO-=nEOV6tT_&RrkyW6mEJpb2M!7-}iZ+hW49fxa@Lo$ZzoGPV zsCWKE*1Gtx`VJFK0}$GR`5cR0y?R}7{dLunjx-MUdhI`*B=*F6H8NP+{z1vXriJCv zyA&A1?S8v<{(1VhU(s_cQl7LM)Bx@L_JVa=15U1~Ai#~&^81v(F65RgvzRt}UA z$dNFR{muVwvm@G#X$FKisLhk9bLPAOnuC1vki|kIjTMH+Czu$?Mo1oJL5RCG9~bIK zqbJB{g9Z&YR1&-srYTwu+4&35Z?*=_9qF@o8JvF(IubCZs}N7Ji!`b0MPf zbyq`k*$$4YbdP|o zEiY`R6_^&7nBPv0@uZJHI&GhjpPVyz-Iu`maj%H-5(5Oo%zs{fMxwVj9XD>fq1ELT zZHOJz1u_+{FG5aIv4|Zgk4{D32#Bd;Y#eOE8nOyW%poKHHswvdko8+)&e}9-AOX|_ zYBUnq;Jb6dOh`t!=eqPG@xS(aWkt&Ar3`+t>sjLZt5j0TF^)Ffwc}ot{o2QMlu$P{ z1;u3rP|(IHMGiCuL{bj3pms|rqD%wB!u9&}QNvO-Ed>jkC<$Yxf1i#w{5DscdOx0C zQDNleHFew9gsifwr7sF4rjyHNQ?uwO0RQr`*ve}mMLZoky1FJX8IwWAD@%MUC#JX} zskoJpFm8O?n$^SJOv%&kR(3yPRmSRLAfG0W=UKS*+)@9Dx1xhKQMwYOfZhjQeCXZ$ z_`bf4inc12tg%@y{G^F3R~4F)R^k$`L3A zM?nZPuSDfl<@ht`x1k_WZF#;rH~;HWRk>X~G~k78#oJPmf z9|x!HDl_%?Sp8g9A6tvuWaMz4jn7@scKymwMlQ7I7@6vYk6W>#hd?tnvx&?hYFeN~ z42C8QgWyw69}l4F4_J$e#nADGiAAe^%hlzgu5CAwDkouXG<8BV(TJMG+TaBg%b znt?kCo(#~fAHNfT;(XK$-ddJQ>wjc45bLZo+kon?Rf4meceFBS!b8ifz4_7N-8r-@ z&mm=yD9h>%{dtKp+#zyP??++^iCua{Q*_G+UKl~6c*ZKaODgi!?~Kszeif(X7FPDR zmhTOfBNC~oX&hNb?D@D#%iD}LX%Y^Aigb3AG#}1+?`puyccoeWUQnT{kV-^We z#28R+U3es6)ZF9e``G~ASiZ{IIKTY$u2~_&uT5yyW-9^qQQ=ctWaM;IKpLvQ_HNHo zQe133{bwd^O7ZU~-GztIFJoWjA0_Txop)k9BV-t-<2E6~f=PDf+U{{C3w6j^Tqezs zvS>_DO>}-4m2-mU<3Rvh%AfRuFN&kgIr^95qY@m5=f=|ND|xvD333Zj1Je5 za|+S93%P2UaQ}X~b8l+%N!q00C!aF!O%+gi0k-;@lT}rt59H?jhirB_7A#2a{lZLV z9WY+{9>%~C!D(l&lm7XC7YCicyd?FI5^dKuvakd7-;8JFB$={Ykz(2Bt)%~Tht%j1 zLGvh)!}wI8xI75(_|bNfPWGB_X|cXk&@Rgvj6k>ExyJQ`6Y1q#WQ*PA9fOy*)s;|y zKC0z5VxQ)rQ=>;~zH5)?!M*69L_5KlEO}>VU)cVz1R0e*${@XBJ<`&8|5&xHoxOC1 z^jwl}1eF(M1ttKhDbYcYKpKH(GcBX+>M-bio??B~zH{n;ft~=C$Nv1YZgFl}fDIZN z@&*m~R`(*+l|Uo6fn9$9y|Md>qJr*VkWf}gkh2e<2u51w7ZlVrRVoQ)K=<&smQqF< z&!cyzYpUJR$%VFp^5y+2n8Zm1vKb!M^;^q+On~jLoiFt0iM{dI$l6!p<{WE|L7rE| z*bZwW)vC4+&wa7C`v8mAx*YcxNv~I#?`WX@%F>c7Nx^Ab9udBfwV-R#tql0;w`f)+ zJ2>7gH?uJd%b0r^M?s)3Rf^oWi#d)Fm*gp`HBkCsKwlKrl{w`#j~e3{^{ zi{UJ8Vek8jOgJcHvB}+7XmpY}C2d1~<6uL_kTA@qvVknfl%e3wB68ZO{Xb^>F)3X$ zwxmv3CEdyNS8S_ew*UB@7O|JZH%sXtoE|3!*Jspb2YO{L`cnhfZ=Bg+s^^$dhlGSwjwABo{}N$Ey*W&}QrqC(dOqY?1U&Wb!Vo?g(cmgxPy zw%X}=@GZTETwtIkW{S`dI3)0rui88Ad7+-4&m4i`)IvcqW8tH9YcTd%AJ(&?l1nlx zG~M339ZB^=b)5X*EKqfrp6l6ANh0(a^got0Q<4aZ&!LCA#JGWoit?D2r2#}oN+iW% zKB(Z2=d|Y%a?9Y~g@Wr?5xJFuD-l2q#}_OX8>yg3AF%R@!lR~|1sVq4r;q38yk+m) zkLJoT*Qa-7F_zzEXE#snO@@{xiaghl*!Nxg$bqsOV9;gu>X0jA$b3D2K`nm5#FJB6 zoK9_+Ao2BpgF5~CO+AArryZIanRzhB>^prKbZry?F#x5t#`s^oK9cuNQa_k~N$@?S z_P1PMpaEx&Ej#UzO3GM|RQBh_9-II~)PS4QomK$tx_!AIK{Ir}J|lq|mi9gHt;FB6 z^(4$QvKN4A>M(F6aq0MKDCJc$pBI~fG?+^G3(r2@yiCzj@Ov9J)IQ!`l{9==o%8#v zOZOvb$b{BH4X37g%EOTqbFnZ`Vwg-GA4eTa{2v?)yF_E zbnl(J>$A$DMqI6nw|b9fnE?$mra*VguHCvx$`RhoF5LnybOoj&O-NZKr%uKBKs&H| zj(Jco^xPp)QNJPNb|3pk zGeuxhGaf?OC4tn#t~+jdzGOx&kA8M!aGGrdenW`89z86Fo0+)8ZFkb)!Gx(CY-APL zNx&%@OYh+NwEyrYoHkR*a$j=UWlpr0hlj`f*MI$0bBuq9L(BgRd=}auoTpo`Zm|B? z`EJ~uWY#qK6Kf}WowlSfnFkww{_&CqOm9Nv3rpaps0Ou9Endo3vcR@N~4dG$IL+ zf<^-W%spYC27Ah*6K@s4Xi+78iOBsKyPpfEc88+@z53s))nP*85SAYgJb2DsYsd!0 zyZl1KAwa=r4k0nruT1HON=p)t0+nO46<;vg5CnzMo238ySJi}HJ8%8%Xl%)xlXvf)$!McZS z2*`nRY4N}>tUgCkL;15%6Ra^kT_w|?47Li~H5ma9D8dtE1J_n4?P2g_A3${r(z#2Q zTS$x4i~SE1;UlA=40vsS8|W>q93eISx;07?>h6@l0NVfBopNT1JBXDLgrK3{SH~H3 zXatTOY0iDtp(g6Vn1v~&y)ULF;xjg3Pg`AO;#;~@k7l6!au{kdffI_A5QXco7Sqf~ z)nE503)NyuJo=k=#erSU^q`OMy5r04AggTzIPbd{g7E`i&wV&+0h(F zM9%*WjXEEYsq$(=h5qB18+hrM)yUE|1$*Kr7J6$&w|!=*%-L+aYJ)>XtAmV5~zo1m-S zyE$<3#dXE+E0QtFY5Z$`-#ijo1kUD1`Ix8?(u2`h6qSkv)U3ixMYJwEOVVNPrFPK&hi|en3HT}@_Q#T7~R-AfiCe=_7T1rPwh1cr5weYN0li?<} zUg);{i!8cNBfJg1nu0>D_2T5 z{ZUcvjE%Jji4c!WoB#DLN!xKb(~t0vY%4uOu+jq$8bcQJ(qji3H&ZxhI5#uerO>kZO$lioHQ<5R=m%{(;qs znlHSz0LFqDbO*4 z;^63K($KP4Nk>m3`b0aDrD+yvulCw-VRDn*7P%5|SHtn0f*B;`$nm5dm3dv8+ytP1 zDo4#^Rn^Y%Y({hqHt5?|-2H@9szIsK%N|89rWLSfPyO-vi-Ti3AwAu25BgCOe+l9JN>zyH*~9QE_jg8K*o)UlGyLkW-^PzIsHkdF5za-3`+x z`ZnHN-t@@R`uAxm+<`G<*0wv1BAk2YEemr9knn zeYDYW`b|a4@CMlQM0O)p9cu7_nA9<}(c5W(WtNZ@pl|5Y<>zqyU_YEPw71<9L9jvn z35Tiiu4nOIlP6CWF8Jnsr(iH3VJjgL=x@4w5`hMs{VA*H1NBtYR4shbN66BGf{Hg$ zYwAI#ZB`7eG;KzmaK=;Jwcv@RZ_n6Zm7-=bd2+4Y_m5|7U!N5HL6cdDi;91|C3E4m z^7o4dA(x9xICbJMY4Qgnp1a8EL1+K{N=fE@n0Ic5dSs@`hc71)@hQTF=`ZP+1a&2S zF0|I$8b=TTihKYpPVi+0jkdje`#Xm;^NZt7CG4*!Cr_i0lRdbw-HV+M$&47cuiH(= zX0cP3mzSHRirfuhkgTDj)KYl6wuN3+1K5*I5?JKqs|dB9PcaijmgA4Rt8+(Pzgn{2 z11wlW}ZM&%=kcC07sw{(W&k=f;#ZA<3#;>M1q9IH7f`tIkV~&7soh0B+&1rSadtC9X zvK6|_Uf|LATEpxo^V7R)=uBFEY4r1{6Fq!-x#rnvT!{Jx^(2}>TD%95)OVTs5WzL` zzpwhjw*TuGmsY6sX!JIuuLbZ6)Sp+AT2Z!PvXI~MvRHX!+DTQsW}pkQdgaVzn#LxX zy@MVx>*ii`e6CVm?6nNnw@rn}rCB{b;0U7B8$@%xis_;Au4k*rHaz$6zE$RKs?DU< z6U8|Yyj4cjJwPMZ0(H zV)c^;7g+qKNkT&K5z4-U^-fL=Wn7JZgJ4S3@s67-z!-%6-~nlVko5#%OJwlw(`T|& zn4$i_>Pl9ZAA7%k(GTnQik0Q zH%(G$*r0(TM24oO%d?(JLtR{q#J?pj4bHO%9ors%#dcyz$^#sySDE|}nJ<=v1~8CY z2)j1i86d|5acXnchFS0l+~t0b+IGJIzX~=~nL72@U7Ul|QKZ1nSW^6?vg6lzcpyiQ zA3tS7+O(-tB_eFbCENX1=9-u!{zj7nv#sNcZbsjw)ZX3e?RZv?&bN)6h1`?S$S!(G zIg`Vu&_n8V@4i<+amNcgtvSJx$B^WM@bBd4ymK}@0?nGDZrRY^z; z6j9u_O@mS*U%8gd*U8?U{j&1Mn)z?qS}*Nav~nn4S+EZgR>pU{f^!(KqBi&a*Wmkp z5Z1y0i=S`8gh$E|BEu1l|Kf)&V1i%v`up&!dc7k{6F8VvHw~5%CkQ|1=&GABvc|MfJ2`EvxM(NHI0#=Q1sE4g{$OL z1GK(cW(G<}oe@;R;NDTq1N=MICynU341y)B6fS!h8^qs*j;@vFTGkzHOhzdmxTJB) zG9d&F?`vWwiTK1_kMe|j-TDj|q^RZuA6G3rm}&6J#@5z^@BvhzbudI^M^l~!x#d2y zK0`gg0tIwaIKv#}fQU+xJ`;>c;(KnurHdEqb5H!|RfJdWj}|$FrN$S|d|C@xCfj>N zTOt9HO!Qzv4vxP=tgxnWCkJr>f=)n4K*oz}=IexL0yNxKP%)w{n{#zftn!zoRBicX z1?&Y7_;a#(5ISjCtzE3FKv*r$Ov}b9tvZBgN(C3A3Au^1G3&DHMMHW>)FeDO&8ZOW zySX1Z#p!qo`1V~tv1tAYf;Zar0U0ri5d=QRnbEEJ_7E*X?dTly3J?jyb~XEkSehnzm7hF0F2S|EAblVR0D0 z)@aA)8>{>V8SfsmwzFQKF732k^obJ$?rfj3a~w; zyOAml*)AKXP{UW6>0w5yNPl~|Rt#+@-Q+*7Y?t663lS44kA2StXQ_`FYe(yW&9E?F zdc49C2cE@2b6(L;i(GKgB4Lh58I|CJ5n4;lO+Pf!lxC2T}r z7sr59lD{SEH=sCb!Zs#7N=3Ud{)P_RHKI~QN3uFBIFHczxdP9xTsb{u)Q%R4tTyNC znq;xcq6|6o@h>mp%ozcEBhXH=S0;uOW)2#yncoMMQs5z8Orch8kh$bWS*byfi~}}m zA3;=vOu`dUZTW}8-SSmE=lN^{-iSjCtPu&OD@38o zpFhu4`PyZe3>lW^$S_Fe#w`GAT9^VhXa zi)lmaF(qXQ3GD*-k2Ot{oH^cpzy14PeAMcyB;Xw_`kJquRDYp$lnQvz*yU=%VF6s? zIn*$y?b^2STOhW2#>84DpPmS7%Up?56thW8D8oubx$CR5|!IdT0@z==GSWJstX-1O`XKalYU0blCptnXs5PY0Xn! z^=YZ|NuFL@grbf1YTbaOZS_oeCv_fHv>p2q&3Wz(8U12E$7#wWyPh+E0Y$qI|Xy9L>$NOcKFI891ZvZ9zISWPb->9z@u`n}ZEBGl3Us(+(Yesx= zh~~-uQUpjupPv>KYu*VJT)bB1g)LpVbYj?DB1$GPzt1o5MxwuD6ZlEf|9CUd* z^gK*z8Cpk87HTrE$Xd~m=G^?qKfh=0+8ho=On0yfzaM5{`ecx310l|On`)i`ydS`k=rA-HUdN zGRbH<+`h;9PIm%|*UAZM9ayaxNx7{4^zsM$WxFAHGLD-C^ldlsUs0?kf@l_8yCOa$^PF3v2Q8%-2yA8eyusQ&E-<`R!b z<2V1^4fl15fElrmOByF7wMVYkNK;dR>CK|_4z-TMhAhx2pl#GT&Y$Rw7*uQSesLJt z1VQ~q0(a5M*}WlKlrug90B?`$uGA1Cru+v+GW#Cs<>Pfp*75T0N9RFb*01<&Ck| zM_IET39_f5S&FohxM<;yOqbi%z*(9<281I32U;wJzd-q^R`JOp*e7KUdd&JH6* zN#|Otm3i-6+DNUa7y6ICt909e{v?rFZ=crHdQ-EcoaE(G);pZlE`Jl>SNl_DCQ9)( z3Yx&UJZ}U@UAz5PXEX2sBYT1s&p7=|(@K-N!DY+m_`Fy3b{V=46&rF^Rk8p!_`QgL z*rhoxx&b5uGvifSDsB@0?F^_FkBH1|fi5Y4$e#H8tiPeanNv;EM zV#fEp^tf|K5X8GZbCMliB^#71-KE!lHTw0SaQ^Ti-I{bM;c5U4;teWT{I%J6>iCB$Djw=s+ zQ9Ara<`$ET@sbE7b6eEq$>^}9Eb#!fmQIOya+bKn_x{u>il(Z05 zad*z9MHKC$v|jtR-o~y^eHWv5h!(C~T(y@Ib01CTH{xoin%(1SHP}#+?jg3D-L$Oq z38wSr9sRd09Hs71SJxOrv#R~oB5!gliPILE&^L{gAmKXZVu=6w>m3K2H0j+vBKGPw zIGlFcbn2_^i7QY|{a!Guv2ly{ca3Jqht#_Dwf<6;@AY_js{a?q8NSm_7Q|_)sk-oo zEzv4D=X%}I{MIt^%@8-98QG$P_~m{iN)TBbjr80uyH$HkX45+{tYj60gq4+*b*(DO`3w3)wwGHI zG+KT8r(g|KE$DJ?&eeZjB~qPDuYoKt^m$fZ=V&Wmrj{SBdTWEQrb~X7tg^MCx`F(B z*&UPsXfmHxd;s?Q_puF>C|k`yM`eDg+($RNLXGj7yUcL|X}O#jmeVy*IJ^jjmUlFe2TIYfZg+N;mzpObxF(Edcfu-2m) zhMqiQw<`lwqkm(DM$>6#vks~M1LJ9(@Gy$q2c>KaX-aTpg=$5xWk5h3h)RXVdS~nV zYmbB$Na;q-P&~8*NAv#7Giy>C&`wXCz!ev_GZ!Gu}>N3w-nL zv$n9p_I@NdCpSqwHc1&Ly%5+D*r#hYup$(0d9mw3)T5u*M?75EyIuG0jRE3& zDr8YWK>}Z*1o_ul4qL+BqX+fY>Say-+e|L`(Shh-M5y49k|D(=U4+s@M`YVp}>}`8N5jcWvfj)xfdc z#`y1|4Z`efe_%Tq5wy&Q)~!+ISH_*yz2r@JMZ<1 zWJeC1NIaN|`dZGM{K7yo9Udo6@X>CZ z@3F1yf*NIEpkb!7gCY&^;N$r7i`7#cJn;!ck00hzugAWA3&svHzvi~WXXIvz+k3gh z5RX|7Vfwjs>lR)?ja0gw58SRJpkn}LH&Y)6-fnl9UUM`Wa_%mx^*->MKq9>ye=MXr zz51t}@N*$f2>Z7(Q_CLVMJeSx^$bbSCM01ECmFY@HdFa5EGL$Rw-4o54Ho)vH%k^WIz9Zvp(kpVunpJ>($5q+w0$i5xf{rffA~ zQBkNNjU*EY;z=`bd}}^eg299%r^HQq_%S0xT(~>|TTY%ld1MQZ&}>YnbIAyh3sq#- zM7)G3SU#inkcP6_^DghfE?uwUBZNhqJ>x_oL)WSthmUu2icGNi9XWp7#G<&k*duAF z36B6ypGI{@gf{++InkvQ>>1Yi)eipc+%%Ou_*!z9C&3r@W9rjPiwx=uUwE>X49Cy;xPiS|s6Vtu8u4w@r7 zBKX66mRbfHIrpdd)Rq8fVhZ%<-KtG^@jM1d?$0R?)2=zFK#&N7z(8L64_&)%sLdq5 zkv?2P-=lhrX(kG7gp|-M;jdB~&}4{c>QKc#6@UqFy+MVWCZ`EE`~T}vR035VwQO1V ze7887EJ%j=BAMC^GWLlH31L}$;-8ET5LU|L3roTNG!_71G2l}>hBGvx5b@8ncJTY~ zSJ;%%4U^I_9tvUN^myf5tU1m2k|q_0%*D<$=9_u!Z;} zM9?z+-~d(Fk0Ryjt$0v0N9g_*ELad)>nbo{Qk$;^6lo}b?@>|>U*Eo7`S}Yprv#ef ztzI`U7qpBDF&9l)^)yIe0ocLn@)CAo+uI+Yu-<~Y#h2c_ZJWXlkSQgs9|;?<9_4^K z_nqtEA$rCHh4YAH7hIl7p`mL`Xnc~;bsxOGbGdQ7UrMkG-OI452AXD47Klx@pr@#G z*7^ps(en>QEX|lCP1M3KKBg~>c<}UTWU_OCLTH?x{{2oO`}F?(+LkTk6*%e`V*8 z%AbjnaPa8On>N}m1*!subZj3I^M%<#7MB<{;bi-|l`nwHdkEOs@WP93zK05}Mi3au z>gdkC1n{~Rp0sW-KE;88Cz!F6^$kDncS;vNQRFa`zbgT2NB2qud+Rl{aWsCOdHo)0SogT4l*=ORGm_JbavmPXI>&E|mggptK zIzsFk&Mqo4#-PiFZqNscYPGpsaS;3E|?*$ujqp~I

)3rEAAr@U<_yBP4JIrGR_H>3b_=#~XNlSH%t!2$tF%>{{c-xEpNbtllbDY2(eg zHvk6ZXzStfDy&PVR{8UAW%%>aZs4;lCyh~*-Z$p{x{55o*X?Hla40O!eCyD~=Vr)q zA%$v#T^Zl@WvbM#zstE@aM8%KofPY~{+<&&@BRGs%^`MjL9f%~0*u(!NR=cuqV{;} zYDj-EabjL{>WQbzMG-6gg^Eh%uMIl>aF{4!K2Y_%otQm1bnWys3Xg|6&pu2G%!fS3 z>H|=vzzz(-uamcWUbV*rPFnNX1mhGQI0-tzx?7Vq%31)#Bn!}3od;ac`}fDQGg1i=D#@mB zvl}9tWL&yr*2R@eC26P-8b->!Kg8cKa^>ueqpxp4K8WE-&7<+|0F|deG_V zCVw`lv$FD%m{)>M|FfnBjt@TBJxM!IYcRs=NYjXExJJwfZ_8HldD9cily`If2bgGZTP|+d6}&L+v33R+K+RX?9zKcG z)Ktp`va2U@@bu+~Ma0(R<3s2oe`nl(tWp#@jmjpw;W2wU4_wNY3{qnR;#2etqt*hW zy(VZM@+0&Wn2a!TtS=z|$a5FpZ1}g2oh355FWyJ7lC*>EA_K6ABx>-D-+AyLN@hq~ zhAOd-ZbMGt?t|ty7dP|!hY-^WtL$K4VfRouHO%<$UBUFw9LHYf=g52lK0zBKZo^`K zgmS)rXBYhSbJqOR@&MCO(&t-Co5xAQBRexl#+!8jCXgu%#fZ7ReM!YE`(CXi(~A-I z>a9(__teXm)10J%z&1yot1Nq@F%ztr6l8*gQZ)Zi!7S4$ey?R{Xeb(T;dLs8ScGD! z1Q)480)L^A+-fL#WPgfP!vRZIU00up2N735;S>Ra2@F1-RT{elk1m6NV06Q?p%Vx{ zS`!}r%-+PXd=|grm_wxy*kkJeS>Cf=C+?QYSxSp*|#!qwM6B2E@v2k;bTq0Y) z39CWhV{FeO9QyN>K;Wro`C0qU-?_WB3|?6>6CW?siT?6cB>R~mhYyrKe8JKsuE&8P z<8F=WZvPGYw%^6{ZEEd0X)PaWxy%_Y79ee?{=3$=?@XCz?&fx3B~x4SMX!aYPlPZ@tEs7xXZNd-76QQMdH25T ziH9+f0x7a}$!kkEdbGR9;dus2Y`Bs$8}LJkFMzD|QYXbU$KD6?DE*t9F4@2_{CAd@ zAX-`zVX-;IuFqVdHWx3Rl8JjvByw`JCA#_ciN6OAs7$fs2tKb~z7gi314bV01`QhI zvFx$tkWND$A`5cu`gE<*BY*DV%g-zc{Gnj=jQnp|ePrx?zJpSA`1cRq&lz31sDIsp zIZu>T00e{qeaW%UCY2p)h-B1u>2lq&q5t(Dx^~H}AO&u_J+)1Ct(p;zlilZpNk zB%bdrd0tV~oLA65B0E4J_4x|_@fFyw;v=k*_+jgpS@I&{d8%WW1!<)qC&Jr|UcfZI!vIY>by zE>vsN2&+D27#_B;@)!y7@UO%zrQ)KgcMzcbHSY@VU{IPc^<0K2_1xfFUS;^t z((C!XeD?Q)1I%!E{R-86=CXz7y$=;9f52xURS)k*HC=q;XvyJ)78X|%H1alhg+1M) zpxSmF`XAMn^U~3_fs5&(IrDGVj=Mw%Y0`H;Yxh`wI;Ff8nhtZyLg8yvyy9qHxjsrrj%#r;YHZwg& z-m74ab>Ng}E^+Qhn!d=NBzaYZqhv7@Rft~4jt6gK)#r1QwH(OQ^>ka`c}AyzcQNAp zdFvCH2dnW2dcNEg?tTWD###+K?Xv$}Ve}AMPffEn^_dvGJ=ImjH+MdQYUHK@( zt5Yxa!~lXhB?JSZ9ChTG$M0ku3G~JFZ|bQwaN_K{4<4)+H32%WfGV&j*}q|ac+>$r z;}T8e`0n!u+WuY$R(Y!)8n#j&SdX|q6nL_}clQS(wAg6YizUi};bt%$i+++fX}Ae# zE8>s~49pm7638vX8&9iWM-c)l2XW4vIp=1Prvrl zu5l6{*5d6!2Gi$~=FFxk*oI8aY;=7P%RAE0uq0L8kHK+UFUs8nmh2fR!FQyj3b4!t z(P&(`ySxBQQ*L{4b1T9CGW-9hNj3ah=ArW-&wQrG1d9%O&Gb^URJzwsct-k_4N}qH9xpsy+y0>z3!26^$a^;Pj_GSWP`j4H?uZ(63_>pJ|wgmx>t0_WiYxSyHbxtmR)Q5k& z@u-oRzhF!kxySI|g}Y#NHWf6Q*##WD@J#QZ8r`7Yq|w~kckF*#vSb8TV2w8ZLt>UY`DR?n00VhirjfGFLPNc%Z)E=_VsJdV|76Kq*?mLj6okvs}5WS!)CrK z@QWDmoQp4f0yE*~P;1qyGErw~-1J!Xj7h}tL^75{kkr`c%Aap`9j-VGmB_Zm%>*0{ z_Y}b!BwfUm&E!C}gU$On6K#iNB)+$4rFZHmACjzu{Q1~VvwlQ718qG)btX}&j*KLP zjO)^p4rKqF#191eq6cE*@mRQq|6J^DMS(GkdsF_?pBaXBZqX`?QjVTUrE+k z3)hwPZ7*=AIKX)~KP83}oGd{^3<;~JANP8r5MI`NV4p;nfE{<=>?AKIOwkaX!YII) z9XaBDuz<0PS4t3+Oaiw^abf(NJ zWfiUBpfrSS(>zn86UM6TI{G$)mLRgVegFQ`PRva5Y6t8CpRBp_rlah*!-Z4%>B&-b ztNLPD72ASD)baa^5;ADA7WqHY3-cjdZ~^^i&&F_tlA-h^T^_RuNY(W?bYdV%R*L^4 z>7%~!4TfKYm_cb1JpRQ8=U{Aol)NSEt;A#i^17T4Q~v8psrl!EifYeUq;Pg=nAi}h zu(*$Txn!{uTO)GS9xmuhqlr5d8lg7At^jfy!F-&?>r6C6s&cR}t=6|wQ=Nbs*E9?2 zz_K!-xDcX)P%~&hE!qut{+v$_(&ZYe?hA%Z)Ukq@&H!q^221Yn{BkYGmF}-%C^B61 zvV)AlOK9NrsLcgZ7Qu)3P~GtmpI!V&wnelzGCFf3Fb>5%cYk!^FA_9|XPd>g+P5qd zeT8H9`trMMi!LlI+)^SsT1I5vlTN)tt(g;XtW0{{aWd&oyuzaYRKQmfF##vpym|BM zag%jY}qo(bt%a(_ipEt8y9nN{^qvBA|BMUtsQs$$l?~z35A!u|i%+Z|f^uXUyzCk+eUn#SI3Y!$6eVP;c4d>+<6{Q_6+KZNU`SS~Iu(O43ppJQ&S{biB+EiXDbr$Lq==9a^b1qY&pGILt#j5regu>B zUS2!}v02$OHT7CbC}-b+=e6Q~8EJ05dfshT@3JW+nQ;vtb(;2s;(N@uf0y-q`Le!mV5bq|!ZBdo1ttcFBN`Wn6@yjN4K`ozL_NjNsc zc?87TiP?}QK;$ExozI+iu*fWsaGbJ>Rs6mLaxh0C+zBhF2CxYZ)j0GDmO_Qc= z=+}MrtY$&3rCKdaTNO80I_*nO1BPQoRBEjCtk1U3yuW*5hf2(hgQZhExl^YuRV?FN zwx>D+jV$i;_vcuQ^M5GYJy%Lf5jHl~lPo`HoMahY;yh#;G$E5bb4+;0(ai)^mt8rk zt&lp4YC$~k;}`IrRVfqWf_L{hVC&4=Y2^6HYDkt1E$b6GZ5e2+>~Wd+g`kM4*YgJ( z;>&W7y$Zknx_tLoY7(Ej~_TtclOG%b&+R5azt7#ZqUfm5HfbL#aL>y%ONiu znL*__8kL62Nfoe3@|vj~2(txB;&!~i{p|lT1blrP)MN;B>2e4m&fg=Z;y<`+j1GRc z^!ul!0exT4V-`9O7&NdyvV0Ibnq6A)N{<*GM2c*h#xlZA%ILKgnW2b&%Gi?Dn!75X z>POtI`QhuW0h^x_{w!z=za&1P6T2x+&%bq*oyQML##tzRd*44rvXw!u733pbZYr!x z#Jw`5UyGmMR|0!zs)#q5cJ#i0lOr~^!m{;jGGaY*K&ae7DD4pIueP@@+GK+h2RTl_ zZc-AwKJM31tPSZ_hJdQ#t}HJ8eCDZ!a=vKAz?=-hOQ#aR+SKx^@3U=g*YXhi87% z=s#&Co99!p5C}(u>pSJCMAM1=I8+2MlnHqw9v}#4(vaD-(|33H2 zg}*N%#eeX=Pv&)z_VHOUn%ma9(~>qOxNRM~XRg4S(^s-6MV2^4+ha-ILWVVhFEKGe zZ6OI+x1n+1==M%6rTIK|MQN2ptz*iehd9l07HLQ_2+(*S3q0k~q1g}kOfk-rojfO3es*GW;}EMcdSy7wb37|%ir^So@U{}5 zE6Lp8(_)fzU`aA`PB+u3v~ZjTn`vX0E~fpVF&rv)7V@gA{hzJ$f%%n@cFrJ2hVb4* z1)~<3bOQ2Bwi9u1B-H_ROI0ISP`tHFycb-)fAfrHJL0QX^_hbZl3bWdzoF1P&GJDw zhl-@d+x^pi{#llC&FSVovqPPyL6aW^biNg{Vk+S6sC{7uty=%t|NE>CFP14URH~(< zG}^*^3a7ES=R-He^tJT0%knmjiU%7iwbyHQIO0;Bb4Q=_niTWqPDTfKzP1BaLDpn> zZKjvuB5k!P)p8k#mA3R+TQ-PS1UD4#+ycbxLP3l%^-dqVLlbR zUQl47X}Cg3bN!Qb>l1Z|OvKpYKz+9E5vrNbcdG_rw+_?hDrbwBk?a=uV_R2+KNsgp zDtFw4fd$L>$b~>+S2NUxP45*f;iGx9CE3hj*)k2u$Jo0L_Y4A;=1JaIZHTa|HjTf7 zUas7fdIUK#oZH5xd)KbmyH5&7k(ov8aSHa(3s20_RLd<;p52d|gP=DuvbG$lFy&Co z27k@5sUgMx?Lm%%#x%FxG$J{z69Iu-op2JBcjZ!k;dtbXgc?N}gm(LV=z+lQ z2{S#G49fy-Tyf#hi=QFAUmQP|6P$(-*__;$3e$5GNe-LPn|Nq0IoAJC$9a49NG1RJ z^#x=#IOO$mgi&&|7x&|Ir+<%6O#DhGjmJwpu zfb?rli6mv9@=F28syqj(_BDcP`T}eKt)OB_qgW^b6xPTDx;$LmxX*TTwyI}R4u-n9 zx^_urB6j(lU|8%g{obR+r5|1IhxxH((tsTgb?P8ntJ(1&3(aH~-(ld9B&md$SmL%^E_R|T=eHZizl*LUC9~iA`!~h@7fN^l5-G_M)uuEf9cqNe zgDFr$Tl<9Tao)rxBwv)8{0h27@L6DZQuNT&^S=uJ+zL%p%?yikiM!MT2ZYSsgfodhH z2Qe-~xa*5Jk|vq##p+4(k<~Li{O^yf&`km$h#Dm0&IbNOP*8=bA89wH{@?rUsV>X7 zyU+;ibf(uhCSYWD*M`UOZh7A1WQ!+@U{ROdjhi&NM&JRbqn>_;Vr@_+QKw0Oo4l%& zY1&ZTUc2(N20G~*8ygEoDgxDV&{<$|MUt3+n`;4d3Q7M`ql4lmuF1J*H+X6ddLd#> z#c~&*i`BpJGyGW;v9mzc|0FWka~w~;XNm*M9{4F@K0f7|=5|4MG);IBxR*Knr66IX zDcLmQUQY#JD5FWuSJ1)?ov5OX4MOkk;S=W5WI0=NT>Y4S{ErA$?)+n_q01 zA7CREBmz#L9CAOn^N-VX=hFQMXmY3KGV7&z8uPmuWye{G<4mdl_8tWG=EpR`0 z;K1{|K(qDH(a~7iGBqJfRRL22D&~tUhO;ivaoEi7^_2L%2nck!=%JQT;Q1}A(ZFz- z_>TWRY{R+*uFZn-?p-+4tD{9lvZioLLCLla>dE6wJp1JS@2Dj!g=4GR?a-usH%xJgV72V*dYzWpKst$}Tb7D+>DT25tEGg{W^*$8o=43Y z%3$j(77NHDW6t^ryuQ@Nnn)E>q^1Fm{4Y>wt`3cv^gdL-jX)|fDmdS&fjK|}lDXt( z&yrZ;pa@Rd`m{~iAQN?z!>*O@+H0GxRCND{wEnSA=ii8%({u^nh&|n|>6q*rX=x2G z`KHn5V*@gg@egG4hCQk_$M^Z81u7T#gc`|lSn%w!h5lfqepo$P#G%g-gvv^yt-H05 zry){aYBXNJA&3HVcmG52%j?XqcwzN*zJ=Xv4vN_MB*2?uoMQSjNkVjqp{g?EZFJ&E+!sgc6lMaqYVv9Nr-z8;oGc3h( zAZ(}O^dpFvlKIX*SMBg~-&Pa)MLEfzj`W5Bo*fM(W##4T@B@>x_HsHs$-QhJSJS0j zLJ|6zGtU?-x>;D@-yyJ#H(q(YD^V36);<{CvnFeHd7U5eLDvVySuY;&;eet)2yuEf zTBH8wi_4$fQUMo`K#TF?v%f5RH#~!U61mB^a!)X%>lyf_F14200%Wr zVu>{L`i4>t_Pa5&Ot3xA?MJrk^jjljOZEa$>(ioLB{*>Ry&`A1R{CGsH)}Z85lvua zRY}at=?oe&T>>UG8u#iUZS^~(A9HyyhhkzvO2p8{OAW#0OS-Mtgs%swXc;Nv77DnO z`1uPuN+39D zYPcWY`-^_Iktl6|Lz|kkyc(BI1`na79Sp@TG^&~#m1Nl6oV(G}t9AZ#$n zUrQ&Q0%9^`8EA++XCcnX(IBk4yan}37*31;o%uwsB3gpan$P#c<-d#dTHLl)lIaIZ z?SR&FT$u4MbNc8NH|BgdfBQj}N>VgM0I{?9J~WTr$=XmmEf_#2E6E;aaIK>%-4|81Z~;!y!oCtQj=@G5;x*~m4wr2Cv4bZTI=Fz z&(ZtuP|uV{?4RgpmGz-PD8DesHdwzQ4hLBTmiz8?bp57n{eHc%I%}gbEZLx|0CV;y9B6-)aA)NF)^^ti|Iv+jTrw_IymjNLB|E zAUyIOUU&*3OAJ;FL{aT3cFY>aO&`HPfALu8Bp6Wrcn@ALH+hgd@9B zfiQ-dOTx~_dlT+FeAp$6!rbro$JZ)uV{%#!Om{Bn>+yMJ=j7Tal2X9g|F7kUIRT2V zSgRYSaAOO+a_DW>AN!f69=%_gzjZ=RrCXcF$1AI6^?ZEHSUb$GZD9O}b*k&Dt~~j< z0~_!7TCR~(OYBmS)8;Qat9$g+&oVtt3!eEXV4{1^jTifWnOq4gBgd})!Q$OiVA%-) z5_lE)=twh0^dDbAiMwviXDvb#UNl&P`t`4JW$}R7Gw4CU-aq4(Rv=Cwk4AM5=fFu< z1|jUUc(I=SW=SbQ#s&wpmK;k|`=;{T!6ioBo!xTRj2e_C8LrGZxnNxJFbUB{AA{Iu z#tB-H7vi3PDt^*AW44qZCV|-HB~#rdh_8qmEZH2;Gix-OOY>gFGyB${ zw-fujpPu|k?x8~&LHZ3lZ*bG_@!K0%S2yl}dYk&6FMV3QoWsXqnp07~MDjki?A-Y< zFI*u8Q`}tF!|2w;67%}?Yl8ylT3JFR#{06~6I5ThSBZNeWuDu?jBFS%7;uRd=nJS# z#&g%Eg?(Bzq1d}78cI~=ELlh{d5fv3w@tcIX(gnx|Ji+}?R)i7<5L+9AZ`;6x&!ox zzM%_K5f{I7n=%y%b+{a{JJ%Yx)M-uf!v{5N)MzNTBnY`8yib~_DBjEBnod)($)t_y zub7qFQ!SXTnl~5%H>f<$&J2ABUMKzn_}N;*v#mL)**O~f?krSm+qOpqwR6Lq33dN^ zR~7#(-^L?Pu|9hLy~D3n=0t1ozw(1!rbU)u6;N2F-W6`1bbDlxyg=zqV=ew1S`HV> z@tK1|F~W7)I(nj5i?E5*Ba5yFtvRif0sKFY`Ek+m^+aIZym<{ zcI8#p(rrY~)upOSt;>u1b)XbWY9Pic`|$BY@~?B5`qDKG4b~s-3COq8)KfCtW4VB2 z*V!@@%asfk5}v^O_i$kE@_PIffP=$PiPHJxyYtZGBSIBdo*te~XCj0Uhre?c^PVG& zvS3!8?~@v;KN?AmSEv^FaWQ-tX>GbZCB^_7G7q=kp7AHPll+a0l2SYf!yqq=TXnUE zc76g0@OiU-7jPAQtX{CnN5>RDr4==+Mi1?|@FT_S_RFcu+k_bz}I#p@}YRO#^bRD)CYro&8F0kf0(FGJz9RyU6vbgS9tuJ8Y|pn;_VkTE2coJse4Be2eG|Y z?j@?Cj7EIjurDbt7T3()!7G~bUI=_dPhGimb75Z*RtY3Domqt-S;H>scV>UuFNt0> zGjjGBx|bIZMAejrq(mqQGP2wE>$fA(%EFT+d79FNyPo0lk74KMQlj9XX`=K|zoGd3~>of0V<-1BtK03V`*SWX<=#9JQJYA8}v$Rn+gYyx}wTJ#}9y_hY*q;-V%$(WuUyR~8S4dBE|E|?gYAZUHwrRYvom9ZXh{IP zYLUdU9fh3A!U^9`u-;khpuO2LwX0cTD?As_ETqNx$H7!F>EGCS*I>>_=yTl~Chv7qq zuGyHL-~*h)F3zTy@u*ItUtt_qq4-1XIr@x7QL#zeL<@J%ntMfc-`z8R$J|rUX^Dk} zkuEo*ESTQ6&-@s&c$X4lh14fb0H_O&O@6VZ>*(+0b`!2qdQ&bYzfkjNuU#?xfB(%Z{hseakfR&h)+U5bI&6XOo`nHyNQZ^=i;UF6IqYUr;&ee=#1fpt zkIV}q`!>mjW`B5oD8d@KxJHjB4}cV#>UfKlK4SE0KoBIu}jCOZw?&XDth1?OgLz1j40+YAPf=*47Ur zs+UDi=@tvUxAN1Y1t%YUTlp=E1g_qMr%3K-slP$gPZ}8u| zc?7V+0I&Dqi7EFgEW_l~_WX0&x|{n3y^dM9eK7ZAAKzlN!u`R6>&tN~uO0VP5l?*G zWABmJrGdz!x9{eq1F zA&}Z;-zxW~MHwhb#4=@vvR-G*!BZnIMNK_A9G9q=Vs<}yvmQxx+dt9O6ZCcWc!A5H zP*A%aJ&x3o^sscrVP&1C$tM%OGZl+9x`K&=*%ZBBjV-{oXN(OroMF;K76O&df6QBnLs!+bFyvCI^Mm! zn*`-PS``F>J%ha42##u7@bLDKv8lXgMmy08r$1bB?#8xjUogYWCT+O7CQGL#X1Ghy z2C_=>3Nn9m&^7WweIndCFTN( z5pD+ac}aDQ#VHy+8PXg+7EWY8@#sET$N|Rh=_kuMX=d`LFMB_oqEgApH2$=B7onoF zJj?GJWTjSp`2q%}Nev!GIGD9&-CD?Z*Jm%0oty3^gH=rFK&0}r^rMM+m@Y3#;E-x-2Anu@ zWR0%!ta4AD(b!b(qA2~$8L4YOzgS}1ptS#r>);IaW7q1=-q`TN)*q*}SLh&I89%Xu*ARJr=y8p(q1qan4ECK&nD&- zX#u&RC91B;X#Y_LNMU4EG)e`j(mclWY8Ym0^_$6sk$lf>!G$Tp$rA81V!?vw1EwIH zYyhg%tguJtcm-XMz-X{{qhHa-%w=czu3hOSvk#@Eq^0G>Wz9;44UzhjO7A0LHPt^d zwg{jHfy{?8;uz5e^D@Frp~aL47GkwL+4}~1?Drs)6&-`Gb9Tt)&3DFjTMu>S1>@Fi&RdhhEidcu zh-h@L{_Fu?Ce*zoK(kW?q@Ez<%a>HfXHM9@ZJUUJgQpXhC~RVw>ib6gVG>nMU&N4g zbN=bHz6?HMh367kBc9WYj0x)xSm=jfX~^xL^jSq#9Z@4kzaqIxb70bbqHyh^L9amw|(t07`o!UO~&v{S-${2Bem0d z*?_i!sLQg)z_|K8Qd>mvM0{DS3k8G`eGcxLJ~)I|U!T)U5L7CFqEmqBO`Vdh*}VB$ z?p&k3eHG1aOqD5;ACpByUZX4zYOhk>^7wTBgR!e0I>q=Gcr}|CK1s=1tBvv|+2lrJ zR=>}Ww&Bos54)0UiBX#up#RMuK4$kItKUm3w%v}t;Z}1GEce)YKg{mj6WIbGBJZ|L5zJ~gCGe>L?x&-Yn%5!40WQGjB} z6}S1yQ0#iMCd6bVXJrw-bBHt)Y<_(}JR+}*IeLFgDpyC8z7HAiD^JM!Fg8a`KW}q_ z?;Qa&sY}N0vJ9Ew5I^&OhQsk z#TGnxpVGd&>FTkRh*)7j+orKzNLR02yEX@@T%RaT@Q8g@y1jd=vqxERf(h*UtJj@8 zd8@P$L#I#Qz0v<7ve*VBj0V5GvMBC-!8fJm8qa-N4?8npW^&(cB+brmdPlF*s<#D- z13c)j1MIC)w^rw)zdI2ZS=^%Y(Kpt6)G}YLIVs*8SpK)eOkRR1u|ty{^4z<>&u->- zUY!@%?t<$?h1s#{3okEtXocgcu>AyfGEuTHW2ls32EEb1Z@b2wd~|We?Bl0TYY47< zK z6GT9tP9IE}Occl*>`!{hC(5dvU!E&gh(4X@mM$EOipoV;nR}jw2vJljvaNhYA+ED+ zOh++0FucSP5x$N{uGW*K7$2k<=XLIXr;}T?&4ji07p+UT3;sC#U$+R?@u*4U&8W-d z_FqH$^?0Iw(gy%0LK*ZqxbF&|;2c5$fVTCS#T=*wLWsG<7wijPa;^>H9#$v8jp@rt zhLa$#I(^IDQQG`}h$$)xvv=|8st)ZGb8t#GSuwyclt+CY;PI&k_96YOhi_ck8oR2A zxp4mcP9%9hJZ)h#io!23>oU3VO#hzTR=#g2lN%9lOm1I|;ZLDG+!MSu1O}$@xDFXR(_{dT4^Qw7)*HFdTMxid>PiJvD7K*d$z+{%eqdvuDY>JwzZOrcPFUmTJ_XGu`q>mA-@&}(Z906>kSuzhLGJle zrzWKu8FnJ07=9(UvdE){k;AyY3$5V${ge2(iX)u>jLreT!3g}t5^y^t-U{`kXSZCsg#?*L;jvrt5=5B>Y~i-h_{ zGNNwJhsTLO^gKh>vD^I{Uk=iAt9o}6z<*cppK~-xG5}D9qi!RV9MCy($cqVQnF+A+UxQBS zML!Cm=J8r*h>=<-_!ARW@N_COf_}R(EaGD&>AOF=$IM~ZRwb)5Xo!XNA^4BqAEj-5)mO+H;HBPz2x~A_Xz8L$tBpIRN9eVs#tQSQdztWzimsZ30 zC@h)o<;5Z2PII?3;@Hwinx8ToN4T1Veu+7xSFd*mFV(-Z6+|SK>b~;Z=K>-~T9ohz zkygn%B)f5)12K1x3mw#g-5;9v!r7l&shv|sYfxrwP4g{{)n>3)`Xqn!T9A-<6GaeS~{s)bL(bp{8UzG%~{bX<_{>b^5S2y=TP0`2!fdQsU7BN$PwTi{UxS%Bt~Q-#`!mrDl3=o|+hGddJmw49 zq<n3kHi~%Ob`b80%7%6 zUP+ijwU{Vl@DXA?$fk!pA-kE@4U+^w8n&_MBGaHM5mwP5Hj!v`-s|8`&u#c_ZxUmUV z=96ig>mPudlnH54W0J*~{~iUJ*BMhIv0KZ4lm~n#i@ekN3Z(Gm2Mqo z)QWw838<&Ul7xBDxT>8jv;&WoRTs3OQQIucenWaLeZ<|ndWX}v@=>QVDi;OCY~8uj z53^$OUhJT#XQ{SxlZzp!8Jj?!6YSKZnxT`zT%*>29h9fOtl z59;N!&uKlL>vw9KPDnrbfT?>uM9PMtx62M3YC&g^)>cC@^?yv|a&o=&84)qE{f zD?v=oGnHXWdCl|QG4x@gk!Id%qiSM~Cyo{7oYTJMY6PGgH!XxO@u?}rENNm_5?6am zDQUsU7iwq+-gS@HU#RrOf1oUnf4a7z5?!af&^}jmrKjk(;C5S(y~$6To2%7rYvSIV z0dV`kT`O2DeT^77AM%@Ize)2FLg=7IL~hb)+)iEFp?Z43V?klj7k6yar6qd=Ab#Jk zlRrt=;NaMMMQeE{8TfvcSPP%+Qi=&jd*O7!H6lb4Y-FDI)va{lkR}~H)eY|fRlG`@ zfJmbBzeHb9LYS%X)&?Pyz59@##wLp9lFK!5qB@tT_ovHQ3_Pf3*j_r9(khuk4h{}F zojVn`L);=!N(Z6dFX!5vSV>z*vLQ0ce^&7enMJB+aslJRoVfxY`V%T-}YXW zQt$ITn^kSHW=O@1thI2rxz9u06EO;|Vq8MO*0y|Tdc~6s%r6&d#S5BNb?vZLn`XmS zO6V;aOxN2*j}<8i&+<`|AyT9jU#B1$w1u6Fii$EQ=sa%cDw_T!r{3>v-l!fjNQr*o z!jCOwYM(Z(^T3E3c73kAaBwd9j7x@4B!Bpo(Iz0I;A@;}2Gjuqc7~g1sNWg8Y*47+ zMTp7REeCuClG9K}C*0I>gi$Y;?7796hX;68ov{{o18Kd0QFh~AC2|r4e3%D>CiO_r zOLVTQYtYNe>AUt~NRj^Zro zgnOg5O&jpD*o_*N*cC{bZrd*~P}4rpLr=S6C@1ygllXHxK8Fa&=Tpgsng4(g#T}&d zdQ~}$kNkF-bZOR`UZD%0NX{J~u&g~=qw8v8kI+_lK9nIW&SuO=gZXz=-JP%lw2BxT z<0>Pv%O6b`<8u|)Y1x$qIymQA0l0B{?6=E#LKf$t znpFJ1UX?EzNV+ecl}Q@vWtJ9vYic{$}B8ha3FQ zhb9=goR9r-cRV33>Fr)FVE4GV>PL+p9VFbxk1rKHbO!j^)p+&e_XeUHpaErserIag zuNSu#7b$N0#nkipTj}8-d~jAj+~AzB*2}Pc-vsrzG}wKh_^M&5ei zAiD1LMwwY*ul>cXzsjsa(n;tW6lh=+{vOGXwD!2znOBc(u*;UYNE!I+D^fe(VE|)$ z$GB+KEdAmMywk!lsCO6DU)YMZ8 zP%wBpB1a)vCg|$5W*id%20)3R$dW$|pPSy7d@+>dxQlC%yR-pF)?riEJ4(@3UBuOb z~RdEPmwx zw9<`10m&8v_r7&h4Cu&wuy+#!l`@_2LcI1ea*)T;C#uAQuxj!_wj5g?bM$C3JSdxc zd)oHV)9M0MM}H1=(D4COM#=J+<+g%iqCz>Jy#ye9SF`0492|OqUD+6fSf8GL95GR! zsK^{$<=T=1kNo-jl=H!=DK}eK)nA|9h9+dKvUiIe*XqCfX)zBz(tK3U6-x366nob- zzdZc(u+ml-7>ZX9wklQ5a0acSBfDO6?iCF~eRyO+Z@9KIZGUQ^1bhS2(TRwrrg6J_ zj!GO2X2Sf9B}MdPLO!xAmN|aSqs1+CnvSd}{SbHV9Iu{YRB;Dxavcy-lZ#8zj%GVG zta)m684Pvd*DRy&*@DRg3I3znqGNG;9+rf3ase1Vz(^&;D_$;3z_jt)?L~JF^20Nn zAG0fENA|n=Cd21PDwbfP;Es0(KPOdw)O&RUnJv?r357 zLxdEypP(~oq|c45WKS=dUwytaK_Xa~xSF74%Ev$$UsC=Losqbc!6V(82Y{PA%E%&{ zoy8joR(-yG+AKM&$YNI!J?G-~J1??y0cg`COm`=$StiiCY5x8fW6*gCr+J_yO(p{6 zPh6VSKa*GWR(*Y3RA>K zjhy2`)aj{q9N+Hb*?2`m9Bs3ENS95H4txFx^NbzGxq0p|AmYx#{LSp|`SDf98uins z((DaP^5p7b+=K{1pBovt_0Cqt7d{@nM`y;V)kQq!Ct@Lm{NeaikkCn4v6RulLxnC^2rm?@Cv+VY{v_9zEk2YFAIng(|N+8D+7c2?Q&Ac%hS-ZE| z>gsNB(QI>Dw1^<#pRh;(07wr!V!yeENBjVx=c4v~EB?hqJ&-K*!&!fcgv{C?DE;4; zdwmqat(i`T9-NfEZGB-1ri_h9EjUi=32>uainD?G7HA)8*;&NiP>TD8`3m#SuX$Bv zP#gllbav@8gQDYBU790fUrig{yLC`C4UhQG$c21CTL<#?>3kHU=WOqsE`qpWf%uo%9l~NTriK4Ak~{C3F^!&B(gz z+S1Zfio;lEx2|$!)PaVb;wP7Op$;1;zxmVVhHvUK9SW*Ld^Ufn)itj!$Z>TpIPS(M zH1+uOl=Wl({`iU=8JVO=kbd;SCEn!K*RNl(>8|90eE-uJ!+Vb&`9lemIh;squpG=c z@#C`Y%^L{~4m_c?iAR_puAvuMs~FII=8~woN}y_cx~i9-f|AdrGYiS2DeTSS#$C-+ zdPuM$kI*DX-enn}SuoN^HdtMPb}92Ut2TfCw(5TV8f8%9$yhW8@ z89J!rX7n2V{c~JJL1D5V#92bHhz`k*cz9x=k(LxsXe?zFl{fp>S|sr%%l!-#7+jU$ z_CXxIyCyM41CdKp06G=&fm#P-LqR(?GL0b`cb#&dPVSz?AI947aGWn#V>Mik}G0`k_w~{s`DWLT*J-GC;fklD4zF zzb}T>LTNQ zS)$#(l-2tKjVqk=_1t&dT-fezhYLD-+>ubEf1wP9sxt=7JvI0xD1-in`IRr?_V!n6 zQ}`r$_B6G+%4?L)J?ZI2&oA@}>7dPG5|6M*Mw7wH{JrCHPX5vI7B6 ztO?Uw)#iX}SELODZWGxv=2*iut6sgDA6Vz&v8ndw!e*$|r4QbB;zU#6>UlHw-x;M? zSwX(V+Ud7;^wrZ_`qFujqbN^E6}|D9Kag#%Tg^)R-G8AG6dyh%Js?i@O#vKI<~s-|yfO9`gvjcK|I z3+y#oJUdxl;S)5XL-xngqFc%S7fUF#6GRkS*W9(Vpk>`!Ae9VecQtTEOQ5?Vzzw7D zDl!X}e6;-QB9edl+`VVNm7Siyjv1PHnqFU24>Y;p4M;SVfLqo-dGqP~gBK3Z0g_}P zIs>>rzkq$oLk>^*W6R#c&I>Zawtw7ITWpwgW(Je>BEPX6RO@ZRho&Nkp-{Ugq^c{B za^dFqHJed3oto{pP)VFtn2EtT9HGO~FD?tYu?`M7Q9W~9vqoQTcfo+@O2prqbls}B`14n@x3s==a0QetF%8!d_ldAZG zRT8Ey8Pit%>`vMiP7}K{m)1gnqF7naq~l$!PGhjG_oK_bSx$Lu#<`{>DVooivBx^* z_;E?K6G#(bXKM{M#Lh-|lfs*D%@Oqt-xSb7P042W&Bpu)f5wg0_aLR`9K64%qiuY) z!Q^II*TbSa=a$hEFGuRbTzezTj~kHr72rq)*(UYL1~~QaB+B|{qE&CLtg;%fw^SxA zS)+nlgMazF)5}JZa4Kf{m?S_Rxtxh(Bf@L0ys|~ko_lxKj|X-O^l_+DyY@R|AEyIX z7iu!yi4mA7L4FIe(u?loJ;2l=JSr+aiN%0oihnw7uU|s=(0xPDUP32`99~2o)iU7~ zMTa9tZ6gpt*N$#Kao!a?#)Afxho4o+fY<+ROp8YK=4XVdz;O`8FyX(VIM`Zly2HL zur-?#XC-Pefd*eh3{+jHWOOzjD(=z9(f-%>s5E|erFvoM3q?%*D*kQrQZ+fOnPyX- z42q3CFj~gFQyznRw;JYb>XfENfR8HYBq$SEM%kT_I_!R&5Ji&5kFWe&{Aj#zlC}e5 z9_U8XPT`n`{pej;ky&_1uyI}UA+Y0VwPr%(n7prsnlav4Qr3Z%cv0NNkzx4jXYJKh z#(XbZ@1HweFn%RBH>1vy&7<{wz%ZUu8FH-;Psug>mTOpbtJCB;79Lu`oidZmsX{RKKsf zecMcD?|G6x&^|Gy9IAlL6le|R5n0I#+h#b97_{?EUl$@IjFg%|iU zMX@RF*-ZMpGI9@D;P%J1*CE*nZW^GwYoEN?kts_;=!@!*Xd7i`^rxuQX%K@mB|i;V zPcWOPDeoE5Q%JBh_*{L9>}m=6htaTORbtM~mrgGvnd{G-9D@gvjz%m}>&eRl!3p1b zfA>hwNCVQqh4M@4l#Cd*uF8rm{kUh}KAmIFOi<@Pbtg`q?8A}SV9Qw{x~lSE*l7+n zxITs(?PXj=BV1fQ8D5&Uc-e*6m$Ni_G|lkT_V?4iXFtW&)~;9e&to7y=tCx&@$}8N z3|9DY%2fwz8|^=ESb=Vf4NkvWpV=7+k< z?Iv*D4@BN9#NY1j!-sxmUyCL$-w_-@km1|D99l*4Cl~g)&b{;$3UH zWHuZ27LKbeN4lC=TdzFktFgcPWbz-m z^9j!Uc+=w#ZXNp7R%}#;%=yx#y-KGwy3ZK^Mk3KYU`j%h7^lOmEb8#CYY5K7jra9+ zU+8fK#$aMm#|9gb%bUmRFxCbR#QXzv_n|<^WGlyp((d@RYWZkMNhgjvS;5zQT8R=+ z$SPRg(FBFIMG4iCGy3b85q11^l^GZtP;|Q1h{gm|VmenwsC_@&r#Pd~vq`*d>m;%E9!NOyXKKEUI{@rhmbt-eW_$gZ{Ro*JJs{8p< z_%I8E8iI1;{uo_M=*tKcMdL(QQ1f+t53+1oFU%{u=ElqSJ1ahXNM>N!5*Btld2``9 z+GM5z6zm+`ntD6^_=CJvCul0Qn{Z;2?oN}cyGRuTs?XbXByd8z&H^y&C2Nj;xmJ{LFoSbknKk|{hIlUHD;)C7Tj zFBwJ}@JMM2a?$n%U*!8BQe6BGyMmWzP^tvr<=z`@0>mi?6x2g;KvA{fsLgXUy}>L1 zl@?%LvCM^YDT_uA^y7CXgw_gnJ^ziMCt!gAvcNUz>yxB)fq{Xb2?aDM!Y>W@bawfo zaPMyy>?RzQb*Kt4<^MaBE55jK4+Bw2i*D?xpAa6dBdONN2$q;E@Z8gQPifz)rTJYW zEj})o9#{K9hhb{3GCa@8=T?|;zK$#bB4qQ#{kYZVT7XCBDRntwpWJ=opbPu`co=M5 zmA$k3jvpU?OotsNP^+TpijvPv@fG>)1OP6a*DyfzkPl$jGTCAR|1zMpiV<@&W(p?l z;Z9DctR+RY?QM1l@dZ@&v9HaDebp zHPTr5u2Bv}S(WOMe9^;aL5#KCGKX;&?{N)Co(+Ovnt!cf(`U)z`nazw=cE z+MDP+!2-%pL_Ux6!u-+_)dD>_Jp6ZN_1D!{Og0kkeLiYk{hnly0c5CabQ&iz0Kt99 zQQ}Pww1z(l0KhM^W$>pu!@}El206P@^W4)tBpM+^9OFKBw|dNDd|X zE0TNw9%ZcsBk0Sblf zr|K}TM5Hq+Avfx=A9r$ODIeBI>dyeLs<6i86F3PRuw--=#;-VZCqbFPsW|w_)jwOe z)>+%LV)vg#rTzzarB-gc{70qk+Rq+oZ-D|Ryn~I~DF(UcPK~>qI-4%^-0TbUe@}Ma z-?H1U-+y`9s#&u&G@J~u{(VuIifaVj(^^uNHWvLTys&#_DDaa-3&lraP~fv>&AZ3; zF>d$udcC(q?I5#Xk6f5247J(~oQZLEE}Vb&n zyd1HRBa2863Few6>3)R;KvN0t|L8toW#vihqsTwNe|n3KWUGoXe~4@bkV}-)lm3+K zTm%@sxO|sHj<+yU8V{s}{3W*%u#QYg^Eq=OezXtd)#hKC2%b^b{P~yGJdmGg{%Q;w z?TZJ>H;-M5Uu5m^RQ#hgQV7&JDzV?N#{EBHW8!R&s*^W02RZ~Tj9|>nHOUvCAAp~3 zH#j*n{3FTzVykC#5hVngcIfM>enxJI5N+)7Q(1EEWBy|rfMs9nOjKnK7GVx_vtnlj z?I@CoV0$QCq2`DkkSqfM#=p4x)o(>J1F|7QB7hZ~zWX+!Abpq)FrOdeA_ zfsq5;3x-`^RuoD*iX9=!&d@+lueNkhh_6M(9ZNpadI{;^54on$*^H`Vj;Kh2p@=Uc zd}Zdba0tW~FJ35ki=wI>F)@ZxSHpJMg9JsTv7k7j4%1SGd z^SIv6IWyrzUM(+ovn73Y^&-Ap$T@54op#k9O*D(W34@geN342512J1OeKZlGVCdv#%cudI4%vzhSP`OR#<9Ua^^AijFA0yc$RM(=Bg$Q@Lp` z{_ZasRtlXjYJa(euVj<1vT&^8e3!vadM`@(w`2On(GoFCZ;-w^N7g@zqtP-e3hPZili5Y+!?=$<{E?vq~hWJ(nrzy{XQ>dXL@FsyCXb zs)lrJIPv=dgw$Z%qDUmpWhCM>x~Qy|ahy}xwg%d~j*CP_J7)gZha(msIYsqh4l8!R z6eL+3ycz)u)sPFaho&~0$2TD!%A-BBoblPn74XBjp0PZ#Rwo8z0e!_I&aWQ( z3Mfg^>-l4J5v`h<8e6&d8tnG|Kw7vupFoT5Q+3<7H;Zn&}d$Ia7U1V;ygQxjI@jl z5x6FE|6hYuS-5cFy8#g;MdMF}rMz+3@V#^^Me2tP-q&|oY+rR1SAP13i5~q4Re9T6 zcD4y|g1YwBJjci@qez`X*~dz046M=#4snw&Qo?l#DC)6_^kKGASCvku7 z=;<^4)4!Rx3ZatLT(6eu#+_J`$t;Oh>;ce0NhZZCE!;w0d%&qHaVBO=9<1g;@A zug~R4eQhif!C_SKFLLXsixNnI8)|T?jO#Qx=A(tN7mxBk%iU%@zUTX^sprm4>R&C8 zjpJh=PLHQBg!4=<)W|n^*=z^?5bhSX^9nvA6CYH9NB6=7#0loY?FLUL;DG@D4gknk zQW)3xMDaXm0kip2M3Jbc_XXExmTyIl6-{))s|aU3chkU%S7?^f5MTcJxSNT^C17iM zd!YhZ?$xzpdJzd`1C|$WJ0L&^ax~u38{3Y^WhgkMS@k;~yJv&>-e{DsE;+aCZDaS} zn9iihBuhc}!P6!EY55TU=;)&50$#I6ga<*G#+;DySh^^bnrK(DTaVUjFcVhYvx~XH? z-{It`@<*B)cUGNtuvQx3`&qORk%7ry3GhZw0wa2;iB0~ z!|yNZD>n&12;|hvV>%>of~pTk!TXP%W0V2T0}|#$j|rtz)LZsoq9B-qn~`eLM_=FF zk@gzrneg_E8rR$bbX|L8(J*#=9@&OZYf3b}rs5VdgM~SD%G99_JwrLo`?s$U?op zd!^gpJfk$xYu*~~4H>-!9F0`HQEg`&pGv}&Y%5hLGTqJjM(O!cGOOD&L8E~?Chf_> zUPZ-T$cvvB{AP5>}lHMcBJ!oZ6O4e~tycDQ0*?WGUT`X>zJDMJ!9pAoR0@TW5 zL3HqOwqnFcrNis^^>Lud%1X7_E6eIh_K_ILk#OLu%bbB-vy6>4^{JUxm=-%OiV|es>gIhvYRx}U92;K+C7Iw=7?ln03YPsLetrm(wCzJ5J&d+*Z0Pz zU%xKv9vI?Mb|=3OHv1h8QDWh)?>zAqR9n73T=wMHX_mnBCG&4fzk?wDeLp%BhF@~t zTG)5&*b$!k5F=#g|6}V+;Bw5{_MhDhQ4CRp6k`n&*(&L_M7GS(5K&o1O_r>o?A3&b zAzM+|CZ-7~iiApIX|j|hrA1MaRNCwRJ@1(1dH?rk-shS3nbC4xzvVoS^H_TH7<-}1 zcqSnm=3hMA;DdEnh?7z4pinh?(`hjhTbqO43hGu>ueB40Ye zOdihw`pA8W;!AdhcLHb3J9;JsC8hOtj_0lYbv|ms7O7`!nI0?_RwFpb@fjEI``($& z<6=o<5WU*QfwNZ%!0}b;?dP|fIs^ViN1y3#Nix$BdBtT_6)_~hplEt;onLnDye))FB+I}#$4{M# z%Q_51C1;m57A#q?QAsS?Ky=!OE zqNchUSJO@4hoK9@W{FvkU2m6j0L)wuji4Tn%wcO{^8F~^|E-0`UpM-xRQzyxZq0B; z8}?)J2Xx!w-ws4dJ?UamwhFH#!1Ub9*JGD&tS1|X9VspFZFh3J|8I4J`*37vLUN4Q zP;tRZTxr-WNMCiJ-+lSy^$`IyOasQ*jZM`?=Ewv}Y9@|MDK>R=^VVa%R=+KU@%PWU zd{xfnBbRD?XsIC&iJZ}P(n(9xkd%7PMQUSikIqOng}rGTy7pd^#*JlK%Kdq;TSxV4 z=OpV0JXy)I*U+B}>9A?*PX%KF$L=CX%Y=LIucV}I$PW>0n(A$Fd5IlW>Ga zthPAUPD3M$e?}P+0v8MD&Vr$yz2l%m{K=Sf2a2$sP%SC#950(J{pIblq4pMv>^a|j z9jmWo#tRSQ$o-e^JtaFBLcfuV_(x)AOy4-`{N?!Tvf}^#d^8sNaq?mV%>GNZ2;H0x zKsG1Kl%dZkiF-;h=CFm}FIz_B&rB3ThB_W}IRMT}Mozr{K1Ap;>t*s(67=+`l~8#F z^njSWLtGG|fHvgB1<8=^8Rq1sS@%KKn>PHXUxYTc3Hji?g9i>|l7T7gV2arV6U?s_ zfMZBkF;48_=veo4o0iU7l<_GDh5e-dA0D~2Wx9t6K<$G-4Ve7mB~OP4yVk1|9f{bz zGehcZP}Q$h$3cxL*WMgYLxgwIXQjuB@#~)=|;+;Xop8S zZdc0=*eqBd_p2iN{)K>5ndxGdsuQp%V~zxBu+D=*8+CR{+}hg{uy1&3!}o(%&9m^0 zPMvo35=0LHnl)cZ7!uEwMK>ePp1P&}^Z13317NJIyfUwe3RWpXg2!vBqN2w%8#%E# zl|=9xnzK4m$lFSOea*Y{VE?QYSEe%NA=c2GY`^gAqYs`8bG?nq)dU5N(NTg@tKj{{^Xczs5bd=P!sHEE=_WB^b zZkj{cHuM2oC{ zA`G>!4FCoZ(W6vCRFSbRC>C&{f|bz{cE)*tRR{{Ap0_;@0xSwjzQ6(*6Jq?^6t}0v z4i0UlCJ{0-D;lszuqKpeN^}d^)&h&tYDSA6q^Ybfxu%h~UU+24=rUFg*INZB4!!ly z724Lbf|^cfLV5D(otNjG{JtJZ^G9zRE5m9)^f6z|t}4DUyksIuqtVMZj)>bC8Sv{~s$2|0u!#_y9dT|SFdmXM(ZFskci>i-Q7QVhv^^f=|cPkmjNdQ_V4 zF}}U}@J_Aj2P;el>pfV=fnjS^XMEnD4;=C##xZh zc#A$(;%+Dl2FEWzz73+@0iyaqm?}ty`MGee2uYCVw3M-ftn#G0=%N$=hS^57)RW2% z_*;QMvh(v>5Q@R3Q>v)h&-Lx&$C7K)#L=Qnz~+K=E;`yKN=))fA{%-Ow-9PR!EahN z1;Y97a8PyLlrBCA*YrJIcEgkq7tXA#B#dVSRK~aRk7pxWcWL)G_x^V20=(`%lok3{ zy?J<7F3z+R1_+>dw1-$MX1*<3i{*$BQZ5EM8&ZLyb z2U#ot{7TkN@rDQVF7--Vo~BO)dKH$N9r&wIgii+ft%QihKYbKv3=Iw&y*WD+AH_p)AKZo$W@*AVDT`~>fO49d5I3tM73)nPqB*Z!R=eMxKX|=?U4*r0^UzUk$b1} z7FnB>a~)1=)fFx@D!1L^@#fh#Dz%S%2Q)7K_VwK^j0RGH|8-`$bgAS{akFO4Fmp3aX)9d1Kv4?ijdiqdL*E*fNF~)Pn~Rql+Y2 zblvfiuZ0-=bqj?8!Yf6#`)BtH zbxOa!i=_mAt~xUe$We?&;iR5xcFHnQ#E+8P2?N`FBN+f9*y+_yebr2%=mW)}^YXve z1CLGYh>C}vfkbo=9tWn^%di3Q9>#N)WN>^oQ`_F~5=ke&gH!lyOaFm(O@UQMEUrtJ zVu!=|n0@P(Dl!#wa_TSK3g+6TGaaJA1tdTT-ebc4-wNTMN*Ci(U7hrHrLtg_RlJXa znO--JHGVr3bB*-cg7lj>fC%FPS{~-xNAd``5H%}E3{yN zI{ps}G-KIN>ED3uZ)Bx+1iX=uK12Wqr@75ozESJb%u6@#-x}u`b=a~&U+wlD)8@OP z<72t%=m)7a-n^9D0z^kACbn<3+vI`_SnfF7)B4S9$K38~cG_GLuU6WrB>+lm3&K*e zk-L5Q7{xgbk*07hbFPCD`)L)mcA&pKs-KL9ddoh{W@^Xd+QLZk&_fJ*9hT{de?F`Qxj#WXAr@xc0<|AHIh^E z7z!7Slk_x)eCP}AuDD`dp6gb-NZ_|2TyW@zr^E@>Nk0dsfQUGUCNBw-2YAFZ^4UPR zM(43*kTAH+wTxv!qfRi~ev@0Wa0&rcJ8+U#nhE|FhPyMO`fhY}3!wklB92hyY^SZd zsy6d_Iz8cD@^LMCj@}Ze(XQl8lJ)fAD$5Gz2W&oxl!9qzk(GC)FUK*XLOS;+OIyYC z2sqfic|$4TK#*eIu5`JF0ZDflY{DH23Op#oLJ;8xfzd0#jR+lzIc^hTH|UUPKAt*0 z{LY72Au~lPc$r#PK|7N90)hvWb8-D#dN?XGtqN`0k6=?54%YU7jX)_V{ zua2tDSxU1`LR#F2gl;S1%g3NCFVFPSpE`SXpR??Cvn#f4-5QH=l?fZS+xoJv%)rCqV2}vL%MB+l{_&scW&8ljfswOi)7EZ%XPoy{4SZgMv8p z{Fc4>s~e+$X`L|t&J3x|3n?FQfJw?^MEdOV+DVpGte=UDKJSVVY}0+b63kgAck=E$ zqCNNWLDuCeccsJOXS0+q2m8dzZoqy3J{1)NZT$V2X7Uhrq#OS9dmg*iQ|2G<1I*nqCvvs<5(Z$*7mP8gA9=##%}MwY zP|aRZ+Hp`9e$=FblrUugn$6zcQ@Zrj4M;=d>dq|y!EW6>zObmN5uFaw4ODuq0ovYr zyK+Jv-%F%MW!oxSgIr09Z_;0Xbv-MA?FMg>ospE^Af4fEd3$v3qu%$^-dkvF*YbR6bZnIn;D1}J+le>+qFz2oln~7CFPu{RRa+*}Hb8~O)SCgQe zCWe%iLi7mw>Ito&Fzz%q{r#5K9l`(s@k+Ygz*0M^ELc@rW&{^7*C-*7LX4^YHg`Jc z>QrwDSg-0(JrG-@D0$RF{d3C|S$zco_cp5Pk713Hx(e}%3TnFL?0Qw#f(ZuY5EH~d z5QwH&*=f>}TshQ1eXj7Yl;0yFw&zcF`z;AQyg3>-buFjxDvgjXIQh`!NYL9!%V*VE#REFZIRRom`?9iT`|c(1+9XxApuwOS70dInLa(Y z>YVHEG~|*&KW?0RkRPT$Jh~Mon@^*tx1|UG&GfliZ*aBe=+VB@2Yp-(JPqbHmNgb+ zEL3YMufL*JUwBba{_bKT;gNll>AJ-ISSs`_>U(EONMNhZ!*zG#i|ASwnZE9R;$Ol! z;)2K=`h`1~R9uDV8i}(R0tHzfA-q{ck@669)BY+}XL-P=3~( zoU}Nv{Pp^V0AP))dWQ}-?l8Z$_m=~6`o9j$ zR0K}cuaCu=2@@fW-tCt5|NAACu7NTobbL-pJOF`!0SNK7NQqDIIAh2XqQJzI6917E z-@dyG^;4bf^aoS)wy-m{<5v@F2sycYs_sgu1ng^Yy=dYo+*wuO%)2aPx2?OUY*s+TXXQ&Or!R* z+E|^>uCD4s#n*55qlXX09jl&~R#fu*G%c3^NtA^FX&;q>&WV*eE*C_M?13cX&VMp8 z0*obp0BGAY3%|PRCpI;|ZrJQG{xLli-%BJta4srh9bj@m#@o#lWK{5zPt!){{Z4%% z0y8i(9~TsB3>0udqI!laHPkt$CG+{_ryo6gJ(!vX9wInjVHP1%TG}JmZsL4hRq?zZ zv;_iSr03BMTy*yL-{W}bQgok{INXrv+TmsDpz$sylR?UWohY17ep0($$g_^1M&-LD5Vx64&^rLO0k%THQSMpUDUlrorcCg?&wV zzR@@D^`+HUL&Gk8)!uy~9EOq>C7SJK#5U`G!`;7~uz8d2PH#TNwV{R7g^m0WIF-z3 zba;5G6qz4~$6TLK?ecwH`cX?YX$Bu=bIDd0C&-s3jap-EbVr98)8BF0X%`IRO43y0^J#YhnRnjPkg{WTeU1qa}l_H)D{x zFo|xY9Wz# zyweEz;q3Q9;T3bUm2duXjV^pY?;^S@>+2hSv^)aCMo{kQeBTzhehIqG760jC3NoUu z-+kA%>DCji_ram#=zJQvMxVrstbsUM4Op5s5v z-nckLqm=r7Gr4~*po2(}LQ)H_+>5>g ztdE>^*S@)UYx}9)2h)%4VpO>#Z`rNPWiiJK(wCm55DmxfLae7wemq60R z!rFv|g~u)PFMqfOW|-74gM>VPE}hO$&_3`GWN7-z`PXh9-7A1ynl2UB=MgoWHPDDI z+>=Rxa;7+F2!lEDE*W|W^LOzd=sbOj9vyLOn{?y!zdQ$x-h0+ZXw(Zg85q2I_in?& z{|}+;ns3jtD~&h7emLZ#mXtWH_ph|wrr4IzFi?Ohz173R*DJl%(LP(_9)~-qDIE~j zBrT|RI{G}?C+<1i?&jiJ#&qLXa@0$I<*0`9)KaLUFv4JLj{N-!)+12MrSC&y;qKt> z8Ra`8IF`#(X=7l#p~JCbZpPoIA#;$Yi_a>P0=TRTEGDSXJ9vpnU&$z$U!o4CKk?~x zsEou`A=HSPfXiUK1Us}sX3vfGNBq94{ZP2cs4YC*wa@R3nr%WcB)WFU6Wy&cwja9@O@&a7XacgkXB=pv}lf9(77aD)FC`^Y zDpY9=0eioEyioy)gA7JJdp8^?^JN|-`rzf6ijX;HMk}r%2l-*-@w*r&I1UM9V^G1J z^nZ~;9QmgH=pSut(zVA=R_Om}KS-I_793pc+k(0VU#DGyA^%-W1A`#fF+O&~z<(uxVJH}CO$6`R! zr8c9BuD@P%Y4Yr9YnHIf9266^HR*M8EURql^4*qy=c<$UP^i7cjy`q}h5Ucd+sXwt zt`&N?^xQ7m(xksY+V%cq-5FfE#5S&x#UacXK}5^X$ryuyfUzZD}a;XcJ5MZ?dc6yEIk zu{WlKB#BOydUQe_9O4^n<`Xl57fi@G2EUg(L419@ub@|#j0K~dlF^?j=MyAHYi(Uk znbO+*NjRNt5j1ae*SS!hqa-dKN-L7I(nTyaXdGLEKH1dXbt8oCj0HWD#0jAO82aGO zof~*K^so!aEr$3h!5EoOp$55?t#OO5ndsutwYiizY>Bkj)uIcP8u6?Pw7dxGlpS_I z4Eb=}MQ2?vm3v;v-TK}1H%a7$c$|svwXNqGP3}w7=Y_!|R0oV0**aVo!*zV~izct| zNFC@W*+D}8Ig8x<)v+~cn=Q}o2dew>X>fxxS3+0CF=v>`!h0Q%6^nb)#-o^?G=%3{ z?_meue0AitaegLHhwA54NHR69tCbEyGYTLLgw(SC!9{iB;0+zxFb7)1Q={rB@AdT}})6jxOgm3LKfXo$M~NUK`qMu=-^pOyf?@cso< z^3uPqG=(b?pG#Q0^@ZtQ*yNj3XSDWtUg%mj;?p>_B-XRFc&0pa+`;t_YfHpnUZ|f3 zv=;z-a~lQBz5mM6YHyj;W|`c~Smu%SrBNFwr~v!~ENX#;#cjwsq%sJ~c~@{2cwnh_ zFMEqr8^>0FK_6kQa9h6qYK-E7bL30$bz&j$qdk8;7{W64hjI9BSA2Kmju+0dsA@aa zH%*wq3TUJ3^R-f;;t9$g$Q;8`uD-Ml#gmja<@c)QKz2lk_VG+lU0o65Rxh2XpUsz4 zlEB(ylhWF_eBNK@=8(H1&QN+`9OTMmEGz|1B3{0S;Z$q4@~!qi$9>s=@D9;)XThY% zT#I3E`I8V=J=WwS<`qgk;Nyto-xm?2I}hh#yg6nl1^spDb!By>%YRFonU=bF&g*`Q zor@w|bcU4&e`D2N!{h0w>j%Z5EFkS~W4&WJtu*fKwsHE@QzyV?;fkS2sTrg9giLt< zZK56>qgV}Ty=1#c^~8X#=Rcf*fS9`&BB4LQg1lgbF_}n0e=(fjJ>X!#j9{=g{)!b| zB0Suo$Aj7JTmHHhYlnzOjty#<*uz}7H2Pa(RIbfURMqxy;aYDY^0O}ul(=L7Mr5B@ ztH>5Ql+618Dq?4G^>Hf7S;^2^R5<;`e)^fp1&mY0J62wdEX=rV3Pp)Ttx4R~v+*B+ zHmf@#MV|{S-&7?0@!-*ePIUvD4+!3Mu5+tqhLiLHa&e2h?dL(!)@Tm|feO}Y!Rjdr zGqXmt5m(=N)=y6VbASez-gOGUm9cF666wuPb_>wJ+**Q+kzw5PS3V=s)=mVO)S640 zG#93LEJ6#QrP;et``SA}m(HHu`O*4iJ&%PIE*hets2IN2VM#9utI*+w*pa?Dr|L56BbKqi3}E`s*zq z%a7iE;{YC6-SU$jG6R_~EWB8;D5T?==oWTcHx#Y5Y<}5j@s6-viw?9YJ&C!~im{Em zJx(0)fb)}2VUXI@$gN8BbKUtqVXZoytq1J$`zdFtWd_|YGi1J)Xe`cdLA7F~53v|+ z-vEMETfdafE;&+YS1lOcJXDc^#z10_j$aE1KU1x7 zQ&@bX|DEo6s+ix#GCINH%P2>5EsU1Pg=Mt#u|1>Y{f-?Psem*K^QD(Mzv(wC>Z=iVC8R zORuKa)s~5=Q@l7JD0$D=mYu<;?C)R|rN_uWt_++%*H1B(`tk{A5DY9t%%7S}TI!13 zYNv?l(R7(3M7|VzJ0VW7@NA-Hzz2pGB%XRO+}65Xa(~H?9iZEX#I?r8)gR2>j~T6K z!-P}Wb0JIeAK~iA3{*g9X4RELJFAy|gN%u+kPJX(ZD+nC$kQzzdd1;$8bU0fM3PLK z$L<&xugXu3Nl&*qW#1?fc8-kNR<2$>A&*J}2u`vtEE4X_rg^Z_ z)1r%yI9ACg93vpYb;u!Nl+T-g9P$#p4WU3g&-wsu##nW_$*U1*qL+QQ^bPaT&ZyKmX>M{i@R_Z?n-Bt$}P zEaq$HfRU>!Q=-R;=tVk}`cG4Qi<*k(55_KY9$@1#A3``dij)9n9LV)Kc`}yTGx$m6 zMXk20+^JVFI>@FTf~)zc(Ob`^y8&73@;5GYaCN&=zW7Lg- zqhs|QaW`B!v8u-x*kcP*Z~Qhs5C1Tr8|4%cb-Ix^KTma$x7+^5EcOA^BoaEIzRCt8 zq1x9rwPP%+yrRonD5FOB4VrqnRfW6BJ{&rikXW2YWCXhP7v(Kc5ARdll#ZXWjjQ2l z3vQ5S|52eengCsc8=~KCCd(iEYahA!PzaP>!j_Ea3C@}~hnhv?mAZ&@0ZOC*Ac5N& z&1BRd=_qhbJO_izaTTi&!T~SnvA@_Da`r;vt$5}@^QFvK($x(r+@e0s8~jD`HK0=b z_3Z7sw0l4ie*g#j8UEEm#0wQUS}Y9nIrL|+8+Kp<)$7>}E|l#U<*bd;Ai41-&STpV zNgeXp8@>c`nWU*fk}}`O@K(B8i?fw8xbwttLxyw^v1In$T@ZSHxGs|U+NG|ji`a-9 zHY^B~l;VWBlWC}8>GT#@lx)Bb@)vYNURht@qaGh8Bl-oo1U2%AujGDKrW@fJ{fz^= z4AYF|Hu3MKJDJ{)j_8nE#YI;gmtkO}_c{e^AAMp^y_chE!>>>DTpbHpR42eSk`3>daw8rL|8+qJn8~(JDux#ZM!a2WBr? z3-d!+Kk(B&-pJ6tZVQ*=@~FSvP-39mR$1f{fZc0_S+@kS?cwb#!B4&rC`t_RJ06%+oN?{o!Q7R@rD*}m9OB&C{1enpp;-kNYo+Yh%X=2sq9U0Uca}{`Cu_(GLc)b#wv$)1XV9CDl&ym z$MvKX(?Ck3Xa&&Q@}b)Bw5yg`>+{E z3oGd3GD~(}Z~ATKshJV*AV4vdZm<$jRVME6#{16W&P1EBKyc=*F^}AjKnMVHQm;A0 zp?;xf0ER0#K0MSQ)GzX7TmF;3^g^O>)1bqUaEQW}f>?k4Op9K?zmiptT}@jm z&jo0L=1F3L#9$Ze4SXxJp40UU5R3FhC`GV#InME)vmgKdm=H^3CI8I0|J_^NfZX-~ zc!CbVG`c}u4d+Vt&>Cp_93VLCiGx3V+g6j)o>3NS*A7i{tJ^4pD=~Sxhi4Fyt8Vq3 z*u2)yC(0lfAVImlS(hTHp$Q(Or_~A8ZV?s<^QEi`4sM$-R;C+@RYf+8DLGuJ@Fav{ zxW+To(|KF>^5UY_(DlWn2kzf4@L6G8eXI(^y+gF~z`YTtzfw1+jLoDl&MXIhq)HZ| zz$q7FSoP3-)ue+^YkVTT?kMEegFF4NYnVl!BNvPlh|}YfuOwbk%(UVlhJ|%!&+l#? zEk4GsPdM>(cG?!no`~E7&xk$!0cz&prF20XMT7`$N$*-UlA|74Lgewk9rK`EiXX{l!!>XEw zQv+;-XvZig145{_Gc*<`7!BY#-!aKml#E77Kw}cR;BNcSWd1#dj#>DF0=z&e%+Agp z6__N$C7&yFcJwkCK%1`<-B{`0G8k;us?(mRlxs-g+``*1NR-&c<;z1M3@>XTZR>Gy zXHruC=vL8INf!m<9>$;4=i=Y5m_2)TxL-6fKlsIB`HlHrkK;YB-DQxujFE?VKOoPE z2l{4gwAIOL4?2k`N93H+m?88n2#t3~^}>;~9J(vio7NE8GZQs+^aCu)LEiNYOh|$(rYpykiya)Y<06r@d6|0%U2|=?mY2M=Tm! zMj$(PewFZoxyC#Q-@^R10_0%&a!Y(t*~zlJ84MqX(n{@WTfnRqF~A_^CW3a>OtQ5N z$j)6bd$txMG&L=)J~n4^zv07H{xH2!tD% z0}nyCcty_Mb!x8yZ|806YCT}Yva4~eOy7@E%$?9+UXEJ7QymPZe#}+UG!jOA@#EEM zC($cibu_0&mFE67HZX<_CmerMwb~6w)^M{3KTK6*EW^KaDLRFME0Rr42kp}Lxm#b* z9y*7u{{A)?80zG$XEBPaB@^D>m{g5&cD{&F)0Cy-EBaC#gF(<{((0YEN&#|K+WB*4 zoS1QOj!$JY&8sa?${g(*!^9DvCc(n{^3ehsi}n(bj=8X$%6v^-b-?0H5g# zG;bj@4_d=(ric@CUVlpFi6LcS+7>LIvJGK)$(SgMDGGLjmD`1$naged>)mulpG$~3 z*!+BSmppTY(L_w)WQKxelnh=uxS{5Z{}i!N6ecoUV{o$f=uruvr|k?)sqF>#@aYxj z(?Q~u55MvKq|pj}3tSYy?nUJ*s$jkfOv{A9&{xojYT?@~xya&Vi!tw9g|&4j*Wx4T z^Ps;);Dc@dU`ZJ0`t=)PK)EXK`|GuNJ7jhYLd*Ku^Ozy6Ff5gmF2_9&mxReGmx>nr zC-nX5NbxY^i4q@R9=9#~XIK&Q!e)uNL}(7=K3V*AwN;H=jgEnRd=9yi62BDESU0=? z#|BO#(DkuDk_7b7H|+;`N>s7Zk}7Z5b{sIEgVCdkx#@-U^t)?^`@=D%RZwylvpb7f z8!XPX8*(J%oT3Pk%9vC3H3uzE+%O1ipVYuuPVfVh%P!XG>9lE5`E97d5Ve5~jO%HH z0j4`6C(5m=+VFno{sxzd%{IV=di>Y>u@l`}Y`&hh=FN0bo?^@mS<^ENrimN|3>$lB zZ5O;)`I#%`L*nPf(p)`k>A%9T1_2q4!7h-*9!>|mhuh@C2BX3tnAp1uTAr6Kt_VEp zBbp~(Ur1jL9sx6@UMPt&c&*L0%I(q&G9O}nzF*-AV~d`o|M$HMBMr?JHQt~goQHy- zdxmk>ODr1J$UXx*QasjCpR|Hy#}D`Ip-8Y3M)EkdHh~*;s13S>nG$EQ&M5t_zr+yQ zB{oMq1Xbo4XaeNZ4aCm+;fHq0R0W*OG~ZAt*}p@TGhsA|K42Ei9KXr`Y@K|_;#zXs z&$Y^bvxG{2p=dYvSAVrmj__8fF(Kv1_>qSuFApr6`(>SVpXLpPAWNIn9h$mmfYDUI z&{;4(Et=Jg!`REfH9e4^OF$oZyQ(fqvXJYQy?abLyp6Y$!9k~;!qifTtD~``<#wy6 z+xIWy?p^RO0%OVjq>>rB)Z6LQe#g^9XG;js6l6hUlYdw)roElyJ49i2a{AK9c? zGdsS%gdWEprg4Ba`+@I(;%16+#Z$1za?fJ913ucsDH6xAR@f5CP$>!)%n*NN&W2B@ zjNTHzVgz8bwYDXKInLG9SJv1qUi>Y+OQUOZ_00wq4(xlQSz!Z@jtap*bLecht1PfB-UL)yXd_2(v=WfK%7gLqi#ojdP>M=J3F zlJ!SY3xvF6=N4Nwar@&;JRuJo5Jny`^G31YA1}E)!!jL8O$@AY7<#z_H9BufpMWnP$ zp;zGQ&?_7f0>1TGnV`OkOz-~_9HLy}1PEY~(GBWcBTr11gn{k@~*~5pPMh z_Zzu3D}hO!HyVcjs27qZhR!bU0Q3hF*A%^Y{9qP|DWXLx z9>I1+#pKgw+8RmghA%0G1M)cWr9R{)sk5MCLU*K{8g%kq9b|MC;|{84^E^hX;viKv zf6pQ4WR!d-o-C1CC?S>J45Jruo59JdTTzK4$OI)&Ja~VTi_iKd-@SV`6e)J!A1^fi zdi&_OZP)Y}8;c+o<-zU4Xo{)Eix5GyE-&==@S8%Ttu#nx71N?cWv-gW(MplfeRrQe zeHK_0R|6>M6{hfyQ(V@Zs25n89rQO9QgP3o@pc6|t0o1{&{?GD>XaOw^`^Vl^Eq2m z_j*+7nf%)DklCsZZFAyN{#34Q{bljZS+W79#{ZRPr~X;#VpmKT>L0(QIv)9nln}%eep=3vzS)#*${+^OXy*lkbk+L!tL6rUEvEI9=Pk2+bD{tOaJGXv?QV&qKX%wSmWv|%{knhvl zMv1T^J;KCV1;cB~Eq^5_>1TtL+&ZKP59*O%(3RbPRioSk&W?5cw3L|Uaon&6#UmSTY1FFumYuO9XHFZV8f`N2eMGNL9yfMm*31f?k?LRg z`~Aq4PQN@)uc=8rkg8Q`F=a|?ChnfFA9XlB#m!{$q>e9}DR05uz9Rq(=baVyR2Fa(v_>s;Wv+y|ZIF*D1Td3^@U zx*^{3KE?Ly`-|4+zV27yxmTL_^^tV@{Bg+PNY$zmm88g9Q*?FJzKH8wo zfQQ|l-GQXsLu;d2Zb)3{pOg1rzL?%CsW`65na~Y8yHr2vaxmIr*1^W&F+JzuL8F z)9|E^A)Yv3t@D^}tgS8&jrT>s&UuAC{Xvum0ADUI;m;fMWdyOd=yCfEXLj0nIfaKBx2#1Ord&7n*z%yc_3PQ1O82EWo zKC|k*j#%T~dc9(~`VguigW<;}U#O|h#m6G>S#XYG@u9MnW+D<{N`co87Ho z^E3GGbC&a+=6vnF;BGpai??*`iXuqpfAo-)Ax1eLo;eN?CJg6<2wH%HuGBV2YA$Bj z=*y(@#9c>d0z1>ehj>6S!i-?}HT};F>}awiB8mK{px=qMMx~(~2<)uXX)ehg@MLTx z>)}GvS2EwB>W2cX%~c+^-@vkQ)26muIuasGs>SU}xjP1Q7{FY#<0DmL#xH2`Hb4Mn zjoQ}0Z^9jCG>%j|-+C+_ikU5sSK06NL0!aNL7eF#9pBV%Zh04Rw?8(JrQ>clX~LQSPw+j%Vu|FBfV}{$Wi_PKou>Ba01^ zdN=92>tn3$)&u@tGW$sSqkBYEo{w_1&JBF|{&rpOy!1?42GNzZ;rr(`UOe`bj+e}H zVGins-M+1mkf*2!+|}zpkMD}V3l0YKGNr@(px%-I*1z0LVPHY5H|;oFqlucUN=r(j zibBD+Sq^%tBIKYzYSwkwx?r&WS5!F#7ZarHr+AGY^P}hDJ1f_$fo@&Y1)%5lEI>b* zh&<5fuuw=1kEWQc>kEVAt(1`y4(|tg*?`Zpr-V(uz%4@VG$q7XQc0Ox)Vl5ibPiNy zt@b78!mrf%xSC6?{H{%{juUYU)$Vm4x|BcD#nG1bZMinh>sZug+|3L|)61VysdK{# z&wQOaM@}O|Pyc+z{)T3tBQO(NI0fgDR{jJE1M!^_hT9J~=lwVz3}q z0ZhE3oQyr|t6*h~y=cLLT^>yn71x=v}hfx817!uK0bk4jC zG1#y!C!MS)Wb>sPfaGQ75ROHT2W+=HzHgp@j|zEr`3OcIb$K>5eJUENw&iLNZL*jL z#&J5yV2ITdl^lwtpHNi1Uht4*);40^kJrYyzr-Z^*tb8s6%U^6IW}N2yl6#ajoc&3aT?S9xNW>F6$Ch9&KWU>869&=%C0|s z(U`s|B^J3Gtu$DiEfJMpgP4olBm2I%C2()VIE6!!_!E?7aW*Dw`p4Q9y~9?}E#;^O zhznbCVjj#h7Bt2&FkVM<2J`c*J%8k%WxaSm)-OJeTD!&yBXm6SHescJiqlyO{huid zMcy0dy3cM^{4{i`5Ms2{cX`{ybDS&J_O31_ehw;a_c}5_Q2I{T+bfF*(3W^(DkhVf zlD6y!H1gX8_M>ojqk!jN7pr)xzA;l0mu_^RN*6STc1*xT84e9RJCYY9usyYe`5XrP zoVnH$of-@FDV>jS!XQf9;&|Bp%l!tY#%{D`FEO}~>xSvQEg5O^1h_*NqSm3qw{}-! z4uUbEsuwHDFrK-DVKQ95O!>HYD58(PSU7)@V%Wc%p~++EJm5~p7iVv zr#z}UJP~Zt#l_{g&;WIXD2)d}7HdS?j%_LW@>lu!eP~s4LZ<2TG*(%24s0i}5B~o9{r{TAXTM$@$r0f_{q*FRg~d@h0T6azk5g+k z0|MN^nlqTn86-{;Lvpd^GTXTEn*HWq(R_4Pv$i_dWa5KqC=bd-I#Rsm==?jjq4KD;Tg;WlgZC(;W(%kS&DUQ2 zHrK6cX`Lz%itW$z@z~!Ss&~%hlCIMAN;rM$)E8B<&Qd~OK9nJ?^?4p$eXy<#-{WIr zj`S5E5ul~o-kW{f3Yke$^J`A~LmDw-{QgzGM|t54oL=)XJ)Ir7PB&w2%|?cu``F z0HSS?@TN>$-xE#h`o0Zqpu$xn5^kD7RH5BHIpMiUa9!?t*4^FRj>AS`Q}nbn>jUum z(Rc-2$50v@9@IEhKNY3)X7k}Cr{qJ({t{W&uj!IF*tw0Bkx9Qadz$lr|>M3 zn$vZCtCJ$1SANSW)|XGdFEx>sg-DW^ko@6Ud4F;TON z6VxCHWPXQ_C->+u<(%>Fk&v!8$5kf`j~N1N+#MeqJ0C$foa>I6_R5L(h7H2t0I?o> z4yRYT7vVH>s!%FJe8>4BKP416o+lHpT8*ebe65PnfyQ$Dfu#d(Pfys+1t3XIls7=Z$dHD%>oRQ-j|4Z8Llw7Q4R@!*csT~ z#z;-(PtuJH!Z@nvu6}e^sG6+)F74|4h+Enu8(|4Ge9xZEsDB5Y3`p-ocPEYM6q6d= zLv>HZLGyZ5E7)G6=wmiC_(?&jELvyCD2)ALU0AT7Z+M=S-v~}=Os+6GpcsluRk5kT zp?(`uGrCG~x=Mw$En{(~gZua2c)PMyRaHaoeBBouAD9jzuKl%FQZVfafjLS8S}H|p zW8=R6_$BJr&re%<pmImD;YXGE^gzQ%! z?A1U9=zeYyh&Bb{o&yKs67RE)i1A?X$zOE+?eky0 zd}-_44@r>R`7*kBldkFv>dv)23f{QY^xnU6|F=D|67PihSN5KShC=&34%MWxp=7vK zK6i`9j$Jt8%1{VDHroPd0m1>9e0tx&j-_LmjPl5DAb=B6&%`*nkAMHYKSp`JCUzbk z_C7QNpxPcU=FFJ`W69Lg(x?7dpHdk62?c)AlPC2EJkP`sQzDqn{F?$E|A~5PaS41T zxg9y*pr`=8M%J!Cs-&#xL-rz=#Yph5+pOy zz3ij-b393gw~`fa(W_ay_`k6|yeq|Fh9BcZYegdjkV0cGM^LF3J&ra?Oq-duMkOAc zi@RxNO<8)ph*>H+y+imWtVP1Sv5t*dL{5rJ=@Qh#Gtbd+%@db5YMbMhp|ATTEgp>I zCgC!czQ}RtD)grZq_oA+4M&TFsx+)(l?7(r6HN1ToeLelOM5^gERfIIqJVf`?Bj&z z;9ZsR(_-I}-j14GJVx9fay0nume`B~BibZsB?UL`yR#(Le(Qn!Qdy}<9=E1A$kiKV zb?r%m!~=&ov-eo5Xr61ODF5yC?m=CJ=|p~z`J+(QU3XZtN`=ke$fKrsz=`)8E2XN_ z6!MVBR(k5O*JJU1A$f2R*vA+$0+6YSGxF}q z3|dOdLx1@0qUS=oN%#ovHKW}*sBvJXVHMSc?fJ+^rP(JeI6b>TE@JB|vRjrVmPiM8 z&bhksE7qs93+X-8)Dr~yq}hbR*~Wq6&pKj{hsW4$)UjutEGa1|@d$%bQrn6AJ=4IH zd_C2Pn6d2qiDTf9#{JLd`hk01|GK4!GQ1e{9vRr7)-3Cx|He!f`@VhIcwgBa0v(uu zh=GsT&GYr8!4#Uu@MGFltLd0oMAGs=*>~sSMhNa|HFr@IK}=Mux?#-l?;RU4Q}hoZ zj9BK$2n~m7ajsCxrj9fkHFKthX!syRbz{UtAnKy&HUabd;yED}R8Wq024MSd#mwq@ z&8OMoiOF>2(^a>-;u;0mF)~jWy^=(1rlUEYOSe3G-;_V>;#-;ft1{ukKsFQdp`>ca zG0t6EPFYoS-!OfA^6sm+Z?O1M${Y^yQy~_^sK2YyIvXcXl&r!s)a}XGnVTv=q{#wq zDN{mRjG(vpltob}$dOE~bAR3dG?x~su1U}Dq}_po&IK(AmmP?ky;)hBFLFDxK`vv} z8V6}^+|%Jq%$SQ!47IMZl)sr~@|DV|9hLL^%v*PP@*&bsd8m{O|9(X?CBZPp?($G^ z;56v%d%k`t5A6C`PcGhygWd{hN=&vCifzC;Up`H}V{K!EC(T^UI3E9sP~reN8k{`# zu~)`R$%+fUBy0r*d8{Jy^QQ;+b-HtS5A9SFc`6;UF9rTcmy+@4h&p7e{&N3{-{8E5 zZ&x<@boEW?vx3zk17R-r&Upjc138NJPk61?lS(E`%uIrU#<%}o6bX_o2QP!Ekr!)U zut+LRT+uGspW|?17{d>^bwccmu^t{fW&ys5%Bp}i4-&I-VkeF)A;M*Waq(k>i1Woe zLfGC*ldBFV7ncp<_xTVo^;^cL)FwtbD0&41YJKOZN3PaKBa@z)73jG0FliFj(-dg{|E1 z)#n%PezpQU7Vjc-m2z0u1OnXw$H>13{%VF;Db?wx7J0<8(5N(GrY0_%DT8`y#&L~O z0TEl;Sg6KEMn;*w;4>`dsg{;o=xlU|WQq%F)D!Ut9Zd0=5T>Z~=4M?ynvxdMTHNf# zhz0=0_9aIql4B_afgj0f=#Ink4c43F1l@eS!m|`GK!qC&a@-PG z_5|xldMC|sy@xN0Oz5$uC0q}~*fX)e46I!d-(e~MirDT10XKd)ZBp}8BmcNaA^LSy8N3^ir! zxoPx?4tuipon{=$J##uyYh9y;4ga8tf#RXMaR4MM>fe~U@-B5|7e-A+j|yp_;E=?CBG?6jl?OVX`Q#6bp zoA8^5TxKkQ=hCRuxlqE|_qYDmhCQyS23`faDuUo=j@tW&sef*LmrquPUDX$2Noq3C z7L;%Kdd3uvd6N`oe?lVWMiw6cbMu6WWj~OuFnO}1rch(rI~;JN{)I6lNiLRkZvwsL z7M|FHN!nB!8)?hs59_Bc5|P>F;EzB4vU8-J#;e4qU`=DqVU@$z;I!!Y>_QU;`X9Gx>WHli zMJC=;w+}Nt7T+!9$)LX18bfmtHJTUr*IaZP@-ERG>tmtm>V{#z6up@WkTSXSy~Zg( zSVOmB!$CbXH=t0uDRPpF=Iej{ddnu!?bANrNQ)HKI;6t(;o!fupO)6G9<|%(@DFQB zUs+Gm=sluyNZO(CP3@{5TXi@)E=WJ+fT#|23Xl7Ys8=6oU)#hksIWg-5ZYop;&&w7ogM8 zP43s;>!~k`lR*oJt4;jUr4qtA0)1ZWVMgNs^FgLFfyd}eo4#AMYb=e9#NNXuk)XV| zef#!NZspD2_}w7&r0-z1_!@5(IoVx&K6Dlmc?8CD+O7+E^ zXEKjj42=JM1G}jwy$N`-MA%Ie(LpQ-tu9Qy5C*QuLW;!1O15pBfAr*Q%%vMvKPEpTD}EXf$zz{G zzJw^>!|u14-yO0b9Cg`H*Ze*VqP7Fvfp))ku~`qJ0n!q$a*&?pNFIYwKv)bT<3I#p z@*35jTY2H%+6*?fXny--v)2(;DXL@Uonbz&qB4KDUi-0?w&$V`I~dE$*gG?$=OtOV zbN|d(5t#HmIyQpxuG%!$*^INKL*1k*i?Ea){vhB&3ysEy+ErD^J7$rxsvK^rl<;#xut+y+4EXOO9$$Pw&9YHQ6g zn83q8V5MY+-J-=~D_Kju+{{#MkFT%bnxQkStw+8~pDt_+3h01<(dT|3AOJ=Y{O~K# zk^yD`!a|c%Q(MBMvioiGqVN}Fc{E@wlxTMkQvhsbz^61T=aoc$wMAvs?9MI^y145s z8TozNg`+;G?jO;9m!+v@lbmstwzhkAIP?_EV((1Lg!=HOrsHS>Q-asf5H)zUqLJ#1 zW;z?LIW+VIoPrH0207+gze#ljb3q7a|8P($H^TE3Up8#bONRMzPI9%}N>6u_GhMxO z5-b$pX)HPfQnm`W>Z`4Km5S=s%HdV3s);;eGI;_dR6Jra)@1$TV9$W0)zf^N{y$S&rnm-B-R%pgO5#a29<*ISMo)M1+?G8-M=@m?B<#FUSs4q*FBh< zs}KU|Ig=yxLz`wlbAY>hdnazXh8pGX&(h-R4FQY9{$c!Rh3?52!FvxJ*nw}h`3`B;Lvsfk1i;uv%t~03K05~+8j5-`78D$$ z^xgbAzf<^TaDkAIecYv)%#4FwC5 zE)uyr8W_D~oc|@D2PWbJhxAA65@LQ&U^Mmd*(>5Z~%ra^=n*$S$aPC)v&kQ6)4AZAeKtel4*ll(?V)&_Zb@7}#j30?2vA}1*oqSEjC z;ZX@&96I!=gNFGVFXoew>$uqe$}Pu@CUjZGP17HD?(x7!!r!TEfEZAR$?Oe47%yexyV-S#}mV?emMqlZ$Dyu%D) zeEZ)eIt|ni<9dZ6?YFYg@4wV5sZoxT6~MLBmwEa5OeLG~|IeG(6{~Oix!Ypqb@;(q z=S$1Xf6vXXs8HN^v|jax-j}a*4am@|=a^H{ijM;2if-9VqY$(R%zP=PZe9B6kx;h* zZ(EMvO&KGD1qQd4HjK45ZB*3>$QaKh2H8&3z^ukS2)>s0ccjapY@?&yCB$iRJ5F6v zl;{Bb&?~XxkxQPdYTH z34fArxaC%cT0W43^a-J`NXo0K?)vW8@g15@)@>G{7s~i={ocK|fRnRp>@5r%1q$;3xw%VeD^AJEH}7DeZax{!VYU zLR}w#`!D9<^DtMkJ&&ddM3#d5D!ivvM6F8SgH4T@BqD}Ca}+^8| zDv#`H7|=0+SMCMNog)SM(uNr{*C9oS>TdB9k5n7P$f%Tu4#*-2kqNbzzyv5 zp<9$)Sqj+wz~96hrQSRe*YDiBhhQPK&}rUVT*bHy#WW!gIkf5M(Kic+Q|64%f?FmS zHwp;w90RVl!q6$83!+hw;TByg765~EsZfB{K`_LO>~mw8u$KBk#@D@nB~ktdaqIYeuZF$=BX(KnX6mBfs3M)scF35aB_|tE(TkC|PARO|7wRlb`ki z?d^M@Z9RB)QP9P4r_g0&RzLCW_`BIT|HILR(iSD13HZ`!?p`>%uKhhuTf;}G1ut`7 zau^5;jaj$tc?bz$Mstlac6~XX#$IMfBZv`wZ=Ul=Mpkqr!1~z&y6I@nf@ATwyBG(y zqhtNKckyXZFdv)v_ix|AU~UJ#rdvwe9Xe1tO=fx&lw-4Z%?5*zUXvv$DGi@{%_a3I z4eF|rL<3Q1(gmxh^ig=we=Rm#`;zjB*1AO!@D^b}to@knX|-nfv2sE}3G6&PKl(;9 z)`{@SWrl%i`nh)BeAD%xl@9aenUjNq8FDUB90+Td(>tskaOCth{7wmT2lvc;CScyv zqaIX@NZ6>TVz_H)5$S)sulNr_l22I10mqB;4b^uw2Gm2A1nk9SGX_Hxc&cA{#b$eqkd>rs2+3V2eVGG;{GOm@S-D0q-mwSt99!rhp zLby7mWertQJRq~nUINdH|0=;GzvAMgD%?4KR-4rR{rXXyZJIE6!wXx?bpK}ZfTSv! zJo*}K+UPuUqL=NJaAVofPBH_badJeHZMkb3LX;GWr9sbf_K$$8 zb|$kRcMjX`3o%i)EEOHZ#E-s5N6Eh`AKjjyN=yYZC?K>Ae`&M{&Z*RS^z~vn--cR0 zVG2AE)&XD{P(eFU#7H5orN#QKTvZL5K=O7VG1!v)mJK4%QQw+OBY7FfTN z3SA^Btbhz114Z^KByqUJT(7HLAG$EqN`g50xB6~Nd3m{5kqG}Jdl9y-aWa9VNTDrr zH;Lzpng(o*#awIwf+D^rvLh?yQLe86E9Cr)7Dw*)NG2{3#HFtta)*y&uW5NQWn*}R zhln^r>?TW&ip<>^J!)(32FqL=_PoJ5rEy#R zvr??cds*$XOUH9@keDh+G#6eTvQDYBus(n1oDK>tzE)zkNUukBS;Z_>ulozXL#xo^ zYz4m`#Nwl^%@l#peFxf!c1->>6`GI1_GW&+w_8OvtPJtRT7fc))U7K6l9mzE846zV zdEQvOc&H8Dee4tpeDP>FwSP*w7Y3@69-1{!p~5eEFrskT`C)BX&GUf|B}p;6xRqDz z7GkX$2q^_$M5)Fnk`Bx4R8puyb5O?vnQr*>Y>`|gr?f_H={m18?}?ZJMI97T3J~_5 zJ$t&Mfz+J9_~@WxSxE_#n0tZ7si0vKAK0Mh;V#=hFzIgT^`?*LHF2NON%0VwH1d^c z)2C+*JU|)G#s1N+g}#JOOXj#1&w~v(k+Bwm#oq-gTqO6Xk?l61tf^|nL$mn_0pw*C8R<2q#A6eCfx>P-# zk23)An5GG*Tdpdl@(rp+(p_Yh;44VHehFj5E5VM!A&lBP`E>Ca0>bzu681ns8N2m* zj)Dz9i;eDd@`mcPVd}gor9TbBs@N-PJ@)d$>x$bfW>I*Y}Eb4mKl32_v+ zVz|@SIzmmHVwf*Hbc7Bh+)ATlcbTfJUj1oL?uiUZX_XV6Bb+7IJ(+ifL+WlRC(%vD zZ8JE!VBta%V7{Wh&~RWX2JRCNTl0DD#x`j;;^Ui9Xo&Tcc%4+jQWLx6?lLD@sWA9n zv|n7MKaTq&BBFzA0-8|zqF1c%%+-af)ZCAxrObmL$lWq9^FbJd0J)+6q5>XxnUeya zyiL(%>-VOmw4n_=p0s#wizki`T1hy6SC(;rQVFBmO*om?S!;tMX1lXE0;r zC$wV^|D$gUNP55+MMnS33sapETj((&l=4qR^IB8c0F~w^ExGpjOR_dp{QCEPPCO>{ zJGWq>f2=-@BBBnd<1M zSY2J&OVD@cNW|!rK1J~8|J2%9iFj0iUK;vXATnJ$hz(nrr-C-z_x~9c#j9tTua&Vd z9A={K*9gO_a6Xlz^lF)_mBFGO9(QJ!hJCpFWXW~}K`nn`lV7sz{c9#;N!*xjLQ~Hf zV=!dMgu}g~cK`RX8lPk4YjLXEk$=xDAKL&hz~c9rRhZ)|l?jC{8YQ8!0ta;ign>@9 z@%?OaWZ8FQ6M@^dlj30NRJBc~%dEQ?Z+IHmAfMA4(8A!U{#K903fdvjTq9*$CYkh{ z0EAp$n8HMYW$ONUCQm>0Yp;IlrQ%>oXgXSkSLg_+w3Sm!&}eBQWE~F}ux$Jw_$?U{ z(cRhd%B zUMXcs_K=haDJ@z=p|mI&V<;qBG9`&n*~z}|`}yC`QS#(cnY<_P zBR;q__-Xs0$U*Ap#qSs$8oWID6Narm-Ps&*xIh=1yM3ZY3*g;LrY)ZE9e7()Z)u z<=NUXk=M1wO@cN3d$~o=HS5Elg3c!)$3f{^Sf}r1Dtr%}=vxir^Zdk~Ic@KEtv~~! z2V5VGOWY0cVUv;+S)y%=pVzmR{C;8L%K z6M(a-25cSyqD`x_LZ?kKI%0MJ)X4hw>_8%wR(M12ZUzi*;hsXL9vTWokI^KHR;&h$ zA&47RIvDJ0$xTQMz~&l9tQ2hQRn}#S6M(uQ1q&vF0}WIi?(y3&5%7xCUj}8PRAB3{ z+cq2%mM@4+E-s#q!Ul~zhBm-qeqLY6rEfZ2U_B)#3VkkTVyX z5m*nf>n!HrWcCe}J~=}`c0(3yI6CU1u|jA}0>~EL9l-Tx!2_}izd`UAlr7NXE9lF3 z`C4Xy$DxT@4`O9I0Z)jr zhab)Qq@)h*jHgNymS>=|ghYvewhp|dM?}C?3c55{Fp_eLg;*{?_lEaJK?C#Ey6-_$ zVO_4w3aBAJ7GO$>rw3PI-dLMZxfbS@Ajb)^Ac-&)7cvSlFNoMkYP9>5=z#z4{~c0x z6aks8E1z_B+Wz7JVgEyms`*x*2xQO=;29erA2I{8brSf1c}(yLnCwAu=mSY#15o6u zxFF}>6(xL}?3o?P+aGN|4}zfq_v4(%HR~?kKCz^u(W@c4ukOSYhTZtFD1pBOL?3zW zd3<>tVeZiMWG^?N#ccHG<@cpdQ&9ap{#vh51$qI(Q-TiLMttx>DzH<-*8~fnFSlL`CZ4L$DTOs_`d=zm zz9;%EFc&yd@L7Q;@HuojS^@lRSOLbm8=`pt!-%;5xqgky70DIZMh=H+qm z1`jJcvMNf&3FElrXoE{S;U7xG_UI!K9)@NLlUb~6Vh2CEV8X(6bCka$$MG%O$m z0c8fZ;0=ImyP~F6XG4xiM8Qgs-O%8JkHJH_nqY@%7ldIVgYq@EBNT-qEH}=czHgLn zDX&|+q{;o+VMPGbSpZdc>giR>LXx2f`2JGMewVn{&lkU*{%n`mW_Ho7a)bBcbIy;f zTr)@EyUE|m6^%*qqaYVXO`&ssrn-V>2kW?TU9jvW8}+XRZ>;`S`oSV!DL4g%_1IvL z57Ntd00{mHaA>6AE?SwLj|Z=?;mcVsP7y3V3!%3_3CV}f%-SFBG@ehYv zCmEc&OYeCA#*ta}407Z5f5=<#=m#!YsOD^P%xasa~33#g7^= z%M}<91?7-vi;STbNlJ990i08GY`vM}4p;=!7wItc?3p+AJl%-i6S#Tyehj6EQnD2~ zyoycHg^-E=QHC*T_j&wA!wbT~`L3V|D~2r+24N;^?nw1W43ES=k5%G%Q;1yD%*<>= zxHa4V_aYZ+^f`-4MiT>ZP+@!#q*e`16EvA^J!*BY!5rkr3h)-b?ECIL(3To3z_+CR z>#o~OWz$<62_!xSA<2ZS5b%8ABCz$SU!x$1l%Ua4vqZ@Sgj>|2PQx>$3lM`y_SP0h zTnZ>6u6@vC4-4BuTRcqkQ6)zHl!FGrCB))JayRbWnTZMoq|A`9weL1yY(T4-sNsk_ zhzVDAn1Q~IL>98;Rq*yF5;(qgTF85;Vw_8`b{QniFeZx;={SiINk&618jA+7#B0r8 z@JJ=YJ1`afhreE-_w~HOk^OnLyAh-c-eN>*YPtDw{AN@OAjWi7uH~ay^?h3E36pP` zDqHw8eJX0x(2M8XxN^;3Uk}E5$Z>A*5}FDE*(O;_)86WieXQzYEZeDI;$)7zAnMA; zlY8I(+Fj3R1W+Q{0xkWP7$VEy3`Dto{y*BCcXt3(RRcVM+^*eTMNLfv^!KwEmyi@f ztH>DVGR|?_u5+w_iGt+~fH*VDUVFwoU?MZN**M!J#<%FEmA z0w$W!9`~s{JZDfqN?>G&Z^1f>1=d>6w;2_^PrPR;7|{lW$Y&h4w0r{svOfs{P&=^J ztTdW3nqR>$WfEh1G#_S=8xJ3T_#qc%n9&MU;^@PEtHd#Q%>6st>*LFS zHl6!DMjQeNH-!5qjcmfx} zo`D)j5U=2Yj0pvK)s?pO^Q$$$*n{d4G9nHR&S-G}y5F?#>Ij&B)SOP7mmwD6=vv8~ zfEeBI+=K0uOZe)7RU|e!je#fC1YpK27bF4__-~k}t;ZH`2xR6tIyg|Z-8*M%OiqsX>({T;2TJyVG(gct!3hZ| zo1qq=$O@3d{1Q+>Q`9kiUoj5WGA5)1P>uh0)cGLx;~Y!FOh}qa?t{B`v%x>%!LFsd zBSKcg5UmY!=Uc%k-`c(>+I3cGLfXCg;DcS&pyLt-1(DH)Y_qsc#jDs<(k@K9SfF!$ z=*qGW{+}W%nzK-J=zeQmbzlLOb?!e=yT!$KdUtsVool*wI*lcD?`8ir|MV*PS{uy5 z73GEU({Sl)IXEZ)1kCzRb{K%y~zohnE@n3qfDot-&j z1}xB!caG6O2pC!EK$f%HfbW9AVdPJQh6jaBlHXJLBjE7dy(mB54<&1gf#&B+0E3Jjl%m6H^2KAJNrI=whXmaNwm~a|a z85Pvz9{p>i386gNWCO%BLY(VaQUVld5b9J(RUQZ(3rXq}L;61D`RZ}C$=)7&}4Eu$>1Rs90rW!izRPUQ6 zks_HLv6@S%gqECgY?ZWzHzTi8$&*z!1fFWWP3U*s`B$zgcTU26rmAKB`3B=aZEr%# zp?n53Z#D)B14>YDkGPOupQZK%2vte7hVh^KrD%vsK%{p~^{~W-De(Ck7m z)ekF$rkW^eRhw z&jRhU{Cq7n`mCpz#2Hv%AgLu8zBkX$xt}z2*494X$A*_nB|~4gW=#ZOPS{789gHsA zT(5-;#DyzX1pGd)-vo-Yi~QWy@;O%lt~50^UM>^vlmoX0gw18o;XH(s76v~i;2dFE z?H!8dwGKizBCEsW!M4A3AAFHrKitA5e}jI2s2#{7j_Xpo@&TiOW5<@!FbJYb%H(+) zjwTF@dtRElp^#3ygr5_L{k*wYnZZCAO_vi|W}DFP##@+{VRA;EWsb=QW6*|IAwDA$ zt{2Xpb6sx2?F5%4PKPDMXllTfn2pDpy}aZQ#-w=k+0`k5H%MA2)~E+XL==+f zQD)r=H$n+0XLk;gan6D!$^qyFuC9Y>e4DZ z4}kALg6TTpMBH^XkU}Ka@2)J6Q@^t#ilv6K$4Z+KA8!-t4{r4 z_GuYUZS?xnXV}fh6h1AP3U)WWT%?|}vli?j=u4FsjIMH%9L#^}ct7FqnT!{8J9fY5 z^*?nBDJgfFPbHx)5a)<&$=HL5Bie&YI4S%Rh%WIpep)!#(U*-orG9n#9gu0A2Siv4 z^wRy3@GSNiqg0CQUBD_mfaje(1A4*~c6JF_S-luS`@yC~?A+%^?+?fKwG09~oe<=Q zoX&&1VY9dN)!ZpaRnc?0d)Zf86io!itZ>Hl;I+cUC<$5>77u%st2yFROaDAH(_?(- z2!{T?eYlq4ezt%<5lT;|GkO6miAc(^&xBBKuuwu|3HA2JydR%8u2@Rq&YpUTRyb80 z{XObV*r?j^Sm6`&?s*M5vx-7FA`3WJa$$Tk%&R*L#55WNMQWd??zEaXL(CI;gJoM7Y*|1~Paw^0GK4>H__gV<^!=HW}6 zTX%z+fvJ@cCFtRT!m)d3lde`im`OxV1($O&fNw_P_m!6%oSe*2yh1;h4%iCFjj@Xg zIvrf%TiU&+ok3%HdbBGCSd?gIiN+jJ(I5&hp^+vTXShPAL(_lq;z7I$&ge3HTtG>% z7XOf28xIO3h+65mB%nX3o2j@7tCyq@m;h<2RAST9f?#EYsV>BYNYY7teMxI|oZq;L zSuX#sFOiV|TB+Z6>H=lWeIUNs8eOOvTjG|&B{;jp5|w6WN!s{a9ET$2m}rowqx9+V zYcAuJbH$hmT7kPXc3qygS#yVHW$TM|dIet>s_j4C+NHenY{HCyuHF7zo}A1!7Jr#@ z>h-cobj(_s>|0=+1i5igqn*)8k%n877;57}AIfh^*hi$>H@~0Um7nYXR!na5_{;}& zcMYxIG@sfKE9CCqx(ygC79-1bnJ7vqb^+Kz?DcE03;AL5P;vm88ju^@0^koqjSoH) zXkwaVup)?U$1`^>+LtA9#R!W*&xzE-;lN96j>)92g=;`=f-J_+M?W6Ih>({zV_+FZ znOLZFR_sgZz_KBnkH1~08jpLt^MZO64j9d&M;{x&W0=l{74v4IkdY>`?3Ep0H)3K9 z@1{9cWSRzyjK2h2SkukiQ?jzMNaD?+agwnFON0|irN~dZmDa)y#`Y5Um555Wckf~~ z(5^^M=WrZQ1!j=cx7BN5r~KC-A56;M*daai<*4Ab|%+Qj5vH&FrMqR-q4MH6~ ziGevwI@&;F?JR$()8g_oRgsu0F!&}`ElU5htA>kLFOp~uLj8jwI`W*_6^@nQ(_FadSm!;~wJ%mH!P`JV$!vr8iq}s+rxp->qY08d zX{L$uoeKYtVXDiEAp(5?Bn&wRUTGb?E{}18p_EG;IU278JsS*)7A{@7x!ywNUzX5- z+lT76Fz_^s^Zkwy0P5CjvOKovUML3vZ#noC;##l zWtZADYlr&cW2jTUe)YciQKV@9d`N|>gOzGG-D7Y^?3P#`n<1dCU^ZIB@yRxo6(}O1 z)H&Oe7O+fVQ&mn;uPZi$-EhkR#+iE@#y>YjPpiXZ4cjPDiK{*(Q^!mzMfcEs0Oe{Srl9(v53r*~sF#L|l0S z{vQnd);l>R=D@yB5`Yrh(=RjNjJKv4)A_MsAwaCq1hD*QQI?@$1jn3TbkWu6=*DeE zZ~OYz{RLMhno;_1fO4`5=&wQi;lK~MjoYBaricj;vO?tVP9ZCm2nf@$GU{sJo@MiE zYb3%gkL(-BQ+_wzc&Y*Fdm1EBl`dGw4S{iUc~bk(#@&Ft8~bnw;^3Szb?WAMrxL%n zVQ9hmS3Jm+$3g`PVCqatUf9 z!}%D4xb1h91j%4!wen)UI-(73`0i+QdHM^~%3wxQty zetKnXM~HS1puyQkCJA8==tc>gW-w-l2!kg60sq{H4q|`G7x_{+X0pbk?bELs_4VH0 zEv_GZRyVyRi}lfDdg)F$ddk00cGK?B`K+bY-O_3&zICzF3>V0rU@B9E&3+!?JuMUn ziDxK#e6jv7)j&Zf?7E0xLY(LBq0g(yCLF^^=xqPl_#SsoV*(&#;%or95-|q`T#@2( zQ{D4*j@d5YE4bX?E&=ZeaK6Wehr!(f+JR*ivLYC)yyZtf&IMHji$wa^My-X~sJbxY zW2swl*U+E`EUMg_0qkc%h6f}D02i=@lsRFr0B`|;HO0LV#;O)USquGIuEZ9+3u5U_ zR6Bh}vpyoxFFKl~6u{%CJ8??6DbOHa<_*zOY_wS(b3)*uo8_Snh1odjGdUT!vA=`{ z1YL-wr3RLR#rAVXv=ML_eEeOLeF&RtG6a5&8KR z_u!Q~cl>eFs0}=O`ZN=Mp>&JVsD^k9uS?Ji?sE2K` zko=cUd4m!!wr-uUF>(f9y?T{3jGk5&08p;s9*J%dkS&4}RwEl@3dk$RJqKoRa-QGw z^nBi7J}Xo~nU1MJ6&)@k>l!@op@=Yti_QN^kA-G~la@13ziGsM+XHUy>ZjW0I&8xG z))`vM8!WJFbd12jrbm4&5v-NMmFA_B9?m&q@{E@q|Aa>~Is5b^Xz|OcfL6#r8Blr>_~Dbjwcx{XMp@kph$32gl|`1=tXgOck+hYsCC6l|rHUT!GWGsmLkP%iPiTKo|tSIQk zkzNQH-xFgn2?QH_(tG4Epyk3l$3PpP&dQ})fFU3rx47~E)iw@F7UzO!y6*k~0q4Xx zY?H&EU;A6>>*>W-b5ol|Kn$SyP3J;#8q`d@^aZsVEh>YFk zL&Xf&Djxu`ainjC*W!d$_5bb%ao0*F+^wlsmK24B3ReCWA%3;I?W>6c?s z(a{Uxast#A_<1(Ifvshw#}AykpWfp?;YZt- zQC3-N*RCbL1JWRAKuV~;?iMXH4)}v~Fi&>w+qYEkDGctznWIOjHv{%bFgpz@zrr(i z8iGO#tWu{f6tcuQO{t+Iwu8Brp96|OOinT(qQi_+Zc$YABJgP_OA0}%GGu{9yeLWs z`tkBZem)oBlYoV?$io>fbX{mNkY!843-%3A4v?P2+R4!o(tLJ!t3gyGtI18Adf*|X zckyFg&Z##c^FE$tnOC1)gt~%&^|-hK^A)IZ0cqC==|>vs=v+veUA{Us%}Gb%`g)CX zu|gm)ZZ7}f2z$cy@p(hN1#MbKf?5(UWKIn^@bHuSPL4C-x|7uVY`DCGc-h+ubmJjUxIKD$UX=NY6r0}|Fx78i1`nDN zrTf~efGk9IGhkYOVh73;!SfzMUmRVEPVJ$k@E|io!H*-4gl~$TN!MNBk$yPH4VLKeCjkSq+muqi^C+Occov*d-gh1*N)KX3rWIHp7}y&HBoj)@t$#{6`O z3myXb%kOub$LCf>o^%Z1g-ce%UvOz2c8sJor63pCAbcXR2<|l z@ZC7hieVUP{SZSJK;nie&U!Jh3ngeF+`V5AZ9Hhhr`dCTRT` ztwpwj7aA)YOY@EF-S-K<+W)Gcu-~km)GE!##XBM|5lXUB_$XF&o_6`B$89 zY@965!STf$f3tD2N2SE$@&%6il{V`$udX@e^sU-)(lJq~$?JPpy&YV5`gG@+)3F5| z<_#_a$O|I}krOALMPbYO^CN6KN818xnf1bj9Eb^I6+AdVkrAmyp@y>!l(6*6<%7UB zwH6u!&;ho10fv;$0cbB^EHVS5E0Irk>M#g~DFF-{u<^5BA*^1$!L=W7dK0D$uU>&y zwe01seG_U!N-28uzNEzIjS)X=aS1{M^Bz!{R52Xk{3sYbMf|Hjz)pB&nnSMwix#s~ z>;U!APByaP4A+E5MO1<1<=|jakjCx7YyuT__mL$~vTVVEqOF^Fp~uV(yR^HpFKmDv zv@tt*KMvjCOYZ!7?M~M@6IVU{UGS*| zq5~3c7bgoY!p-Q0H4F%S(PKDsW9ZNwSkG+fbA_#_aQxxC^{`b(cLTSqvWC|f2vYo! zv{8!hbiPFtNRKW#g?YyJ5f@_7i7pJz6RrR+aw%}f^>6`hJM?AX$#mF@Eac&oMiBs} zgW1FwI5sL|5(8z$Ch+kT$g&aRVDYV+i`J6sPKC=fh++_CcRk?X>(hedx)Xnx6GPXn z(~lEl}E^C7|QQd@D|2aQ-*o zVca`>?+S(L{&_-PNT=^e$yqUtSl)YkmvTS6x{j0etX-Z~4=iOxe5UdwzhCKePOq(E zXvX}ZH2VVmuhp^-xjviKyYamk4BCB9zt2)Y*g4d5ZJ%4veybCE!yOm`>sHS99DaY1UnKVBE4IbVy^}6}+;cDbd$p`IhelbE zZOq`_TfUp8yHwi^fByP8(Efl;WT)-YSNl)Ml*y=wb2Kf_G5vTUcjwV-j5E89=T-HV zB&zh6k9i(^+sQhN7Z|5$XG<~&1+^A=` zkFQpy)q?n!PuOyp4cp4rjZB#hL)vh?qjLn-eL1%={GnN^d|`(NXYWXXj<{A@Mq0iy*;98J%9n79;9O0Hg zz*E`=bda3<*srMb<|e|yAJ)G*eKjW7wYJdmVa?W!F9c};c*D?o+Jalahh^0VM?^3F zf1O1N#9GR^MV!CsL!tQ>xHy?G_>fY-w!qwpqy|GmIMgsfp(!5j?JGbTaDc#3eZsQ= zI}e3|IQTkfyU4SF#KRWim`f6L06^>_{&zJd;UXO;;PF6VFmk}YCxz%T+M@mON_Cli z6)%Wh^mk#C77lw0>rq>zvG8Ej2QZM8?(6I8)f8L84>TOXH&w8~5-|s+gri9GZ>G&= z5p3h&6GmGmFfyUjDSuKgTz9AyjA$P1XOYYuBGRMI1LtdR>=FJr`eWWd-Fbs`q9Jhl zotxOwRmg0ZrF=(4__1p2)vLm1Eq~nC~4F-3(>Q za@yGc>2B$42;wsRk@hjGY}?XUht`PIJ38Mp2P9hOARVGf1fgW78WI$A6N8jApkOQqzDc>cX$Kc# z+~Betj(zK0!D1PFQLf4ObK9~RGkLhA^04>6IVFG@CQMIHc5j0?sGU~`-!uq_A> z?BgV5R%5b9E<7YUvGDp%Ss|+K{GClt);#MzRv&-%-Y}`@pBIU zyUCj68G^YxI=BM07w#64h`9d)S{D?Tr<}Hwu)&u^Y3sxEeH}_@1!yu4Q6n)@;3W|O z25P>$u&^){;@A>1(3(NxP?rjNN-8RNj2?0G5Fs2c>H~_g-{DM;uLAZ$lON3OXK8Z7 zE>K<>vY)DQr^@q~pFUZ~``X8WYo-6yll$DCf#(YomMHVX=h2{xSh;29vZNfja~qz@O#$Qf(1e z&mBT6>uA>b9w_oJeecg?Ol|zw%Z55SsxpF{vQKC-cn@nym}*O&@)G*1Z9)q8=J!rlr)s-}qaB=jwS6iv|SN2H7Dyc1;+;wXa zs?|@0ZCYPawddURn4J7&QlhZpsAt3a$9iVPBYZ}~x9tb`%_}`qW(?R@=4&o8mdx|l zwmdHO*Xv`aoJ#culI9*7IxPQ(qx1X?{Ey}uTiJiHsr6tNwd{2_ex!f2a&&K$u$zBe zN5^yBV|GVPH-++Xsz-96D(-4L#>tvkNkU>}^k+S!6C7PJ8T6Z{HPqbMbktOD7cFpH58QUzfTOl?vP$B}ak&SDbV z!|^;F*Q6Jq1QSTG$!>tufh=9TX{HZPSA57s`~gaBMTh2V2FsT%%YfdhG|3Cw{w}Ug z5T}6qZ=p#S%rW1wBusuru_2?O7^llU1nDuts}6z)S@XA>o{OvA;e!j(;#>T(0&qdf z^RN1ixgIfY4lq+F55paQ5VJS0L~;bqOyS7Y!r}?bs2N^BO4C zyZn3|s8&&Gc!R=@Q;Y`nuG=x7TMs@dMjm9FLB%l;OhA&l{YPf`AoeRu4W4-%4Ks&H zz$<))lX-NK#A0WovZEsnyW`rJivCZM0~u#Ro*j4k>-9je8I2EtEd(sdJL&gG0Tl~z1% z>HOITZ}wHvvwl6(G0)Ho*Rtb+$L3jY&-V7eUfR|1G1J;*nAuHBUSto=nGh-U>YfXlTW_dH~8KW1usXCS8Rr=HZ}ejr#S5uH8BoCHF0WX5yKe>jRZ@eyC^kwV{gJ`mLpUU z{1&kP{QK~o?WD8fU3l0O7Q-7&0%d;a|9C;A3JA~OEwKCpPL3>_X#mz}E%Gj}B_02p z-vnE0*Ot_dj?$(hv!T&JJ3?>)(0X~f$4=U=B`*XK0jZuK^T@$0z;2-ZV30e-SAYf$ z<_iys0@%tunzh+aI|PSu4GM(VvV0hiLhs`Y0El?Z&-SdF(&TGI31DOz35Rwh{`vL6 zdxkV8YjR;UKCPr+V_zjWy%$z&6X>XZN&6%pAcej-@7}#Gs#_g8Ye9jmt+Z(xCe8i^V-qSBW zZbz0g7>k*w`Hn8Hg$q$x_Lh+MfvL>2V%<7Vpxu85vq5e&O?1A7 z)d_yrS9cYALJk-(MH>cQGRK*x=FT2bA9G(e<@oT0?3=6B4nMeMwrf4TEm!Zo3ahq{ z$4>4&wJy7@&vPx;Qh&3NGN+6bf#ZWC=Lf5eLy>djW~kU7G*mhov`=K@M*Zz<^?ZYn zb5Tk%%#qHnLX%f$ zlMb2 zp#J`TO5-30$es+sWyIxU4}F#rM4Sw#UAs}}b9ufZ)b8CNVk@XY9!ok>Rs%pTUWi#Ht^+yAt0CY}Z`#Cx=eY_= zlVqZq&3+IhCo+FP5>)8+BWaf^QO<(R@UH>$`SX5cz}ymSZ?{n*crSP_2pO{*=|91e zlwy`)du5W$50s?f-kD-R9h}d=!2l*Y17Hxy4{(C=wX}ej;?+D0#4twu;dYyIW@%?l`lqp3Y?5lkTS0LTmeMea3ElpnWHs{aQ!P{i{L|;;C z(83+!*CURnpjr|Ci{$|9D;Xn?+%*k}yjK!YA55^#{G-BfdO+5DYu9G@1%QqMPfs}Z zE`S08ab$}~>DTX&Tuu`Yj1@W1lA;KP>a^<>z;AUwSZY(K9BhkfF~EV-&9wk<2Pr2A zKs!O>@Sd4UZS{VMj;r7Z(0x+`beB*!27}^MV56h;rwKfeC{S7pRFg@I<^=Vn1;z)IZ2&!xP1*Yb`NV1d)r zpJ3vD-Pb_2sc~B&A%nb0j1Mm41bMb3>;(tx-wEY87~Ga+($JEH+ofv2ZW;7qj1(Y2 zcaN<78*%|^d>=pV64;`vn*rZ-7cQmLT^&|_uBjC*6*&`6oKxqwJG1Phj=&!mWe}Y= z*w&$9f1+l2c%=T)%U#Am?@7=Xbilf<@XWmD;{%g)|BZ;{5|(m(iEluDo_HFRmPW&MC z5Z}I4OMVnc5+!k<>W0UA7A8foi;(&njRKlvOYjaScT+e(ew-`jx&e?G9fLaVAT)w& zdOPXamt%22e#k5UqhtszYHosqh3`UU0RUt{h2z|KDq`=9*|Xnb<8}4qtHb5VAA9~1 z81<$A@IlF`*>XuEG8^d#=fOE+9fX+snu3}D)l6i`vl1nEsgcDS%Q6G#E8YSJbumL{ zc?!-T|L$6Eq$}=ZL+O$iqhp}2vixAui8c)1&Va|mka2K8uJpWVs*0*7(ly81&G+c* z6FIhPaEzZpQEJfX*`_XWUHRYGuF$``?BuJu?W3Bbu)69 zUG-ZAmdZThW6oD7;P8xWmvY;Zy>*6W*Y)OT!4CNpvySnlGNUU?&b=&%9aDLr>-BCr z`?VkH?jAqN*`romx-K&EvSj|<<61k@MhO+pIlV^DnvR5+BWGoEg;T86Exf6W9;PJF2s+^Kx7mTqBVh$$HGj{Kri(@tW z7q18tfp;KM{Crga@m@*&JF%&95Rpa+_e9~F0B@T93%r>+ReasLJ0-V+08;CD zSy2tRrfp~seowtqMrTRtrINdZ)*$I%*dyN$f*3e(CxeOx6<>qa|DoQ*!VYM~*LhRC z@+$n^Jq1UAD&64y;DJHAeAl!x0EZC#Tc$!^PM0Ef1!n(Klgr!VVl%5Xz#oQCoVeR` zdqqWQJgZkOnaFA)Q3@!TNVk=POl1ZlcD_afF2;k%f}ZAQX6USua2~D90#H^u;U?`` zgtqr0F8bb~P%vW`!ZM6fUTF3hgI|`2V(@Ehzect`Kcx)roKN zLz6$h_%Hu~k!O2%!)}0_Z4|7Eoy*-QBNcRm^?x32LiI;keAw-*tp&y@rDtb>;&c`( z!!)^F9|Ih)(h{+0a4I_v|b|FgZ2PWY4}_@->~n{vzqY8;0?of zcWe)w+g1~vS>t_AKdoW_(nbzZ{&N?lKm_smR0IYx4u3du|ki%WrA7m$VZxLgTJT%5*W%VN?8I8lO=x<1im?@>>U`#9D^EGKRdmj$p=7UYqC$O!^ zG8;YV|2ltvJdbwmchdjD?HxX6Q5~}^t&y>7h~EdSve+mHr_pS!ttrmsoH;WS!#d*U z0CNzA=LbH34*SX@1LI-1#3po#8n(8b+=~}U2pPe9fRGDpJz{VYi*^JABXjH-a!CZU zZR?`KPt&A??h|*weWc%|2pk_@go&l}zC;Z-h}_~&4g2Mh(}nrvMo<%eA}Up1{f4(Z z{X&fy(caK{%M?S3j^XYU><3(NC%1oH5{>)MG?SfVd%+5K>AA}b8M|1|KME%ZvRg$2 zR3JM8V_ExudI|3kK+cv+s(`{h%1 zP^$%98Kt4eHKAMRRyzweJNVU>3t^d;tU4^7*0+4DtR;Of^)x7^wGXX*$u zeNaxs8rcV-I*it{!YQ)BKt;g7)5rd&#YZKcyRI9Rjk-{vS zv5rF9)-U`jK{MulX$K@X zD8Uxk`M04azA}o5HQD{?>+6Gkld%6M`{suD*G9x;fj;R&&%4rzj9ZjKAoPWV+faHh z8XHJ$;J3;0MuSZt9Hn}=q{lbNpCp0-CLd&%V1O~jTMR)_u}YZB3C)13st@L6U~etk zS47HE3?DM_iM+|5j@p#s3efXxX8VruISMNymPcS10cNlTV0lO*bh9VT&$lt~zo3u6 zc{9$}Bw)%28~ywDCru&sJ4p_1!(s)Fwu<~P&G|hsg!4e^odbV@Gv~4#Bqgly z%|hvHZ!o{5z}ekT)?$sq2+$831P@0H*ay1CmL4M$OxH=-i>`1doM}r=}J+>c9B=yRo8m+vzR?Pe6f2sfAX5I?md<=*{Yo zyw3~27B}h?M{(Wx@v>v;LG?8Lu#pOvakpldnQFHJ-c8pI9W-96=a5)no&v61Vo*qR zf0vv<#8_Ot&0&*_vPO>|_q@bQjwS)1BkLnbJzQO*m|^gsM8OgO)#I(tF1OV^)kwaC zo+_>%ItjXRRv-CzO-A77WyF)Z3%;wwg2w@e>=L+rlAqCm-> zd#||>rpthE!#C~n3JY5}3Sv&J36k}!3RskKf`EnkBzT2~J_xb^JVnTMfcymj;rm46 zB0`Vm`<>1P%>uqBb-4`If1aAR*2Vfj-K=Ph2&72p>`L0ekN zRCi-8Sn%a#!}PSu8Bd}@w+K!b=b6HB`JIvu+vEVY?ej-U6Wv}#yKUcbVP8pnMPG%> zk!|@+>!PER_Zqj>dY_oJXy(>6O1H%(P2N03{G7y-VEuodL_PU7z~yt|Z`~o;zdc9>j>VvsO=e>Qf5{65yIrr@C4`E= z@}{c?2NySDGT>d%uUwBC6x7%(pp`U+gE|H5*1i*XmgJaI)#WVNhwNEXOimBBU-b(N zgvtJCFK_~C@b`kUGk=hhJ1pWCdgQxJlL{UTIV!^AB z3+p0z4+T)R9a<95DR^K`YyRXTh?DSu!3pO_wvw|1<+NV!%rEE?+LmLbDANrFA_PeB zh52moEWHRLnJ$IZsq;`Jp>fBck;6Lkawl`)kt^vL`2Y$$F*Y5pu)#f8%2Rk3jp#Fh>LI&}QQK=21QlmWf zs@r8hJ2hYS$(r1BU%-x-7tx$nX>jT!2pcuMwBin|_=EKj7NOKejr83reEt15nA?-& zry}c{4wBsL?#CQKz_{)_Vu^$E*>_vM@enp}0m?O;1AYoH96-K+o!xWXRrq$c8PXAI z8kjjle-!Lz16Pa77}`wG*IWTb%}^VxVk0i|PocXBf&YkBSj;CT(M)nVb>?V%X-h+w zGmN@}P-MOj4FI6M7Zw|oti5&pUTk;JV`v6Ypn=p^Df0`NlY(+GTWI?hEr!h}{tFw> z^}B4WtdNi810$guVZKo4`6nE4lL+hI_~JLKbJyKD*(+VR7(AR>6bl6GJ*pmg(MWJW zxBBkdlLm58rSUnElb`~n-vuh;?LjBtj|Qs_vIGbaQkVh#8z}4xE;Eh6SIYi$e*<~R zQ4|YkDn>JH!Dt7Kn+GQu>*)fAz}vj_nYV8xaCOp%LTeMHhCjxI*qC5!j2%aX2b2;N zBW7Wwg3k~koTT1{Sp%v~o9>m|sj_H%QGbGLRE6&m><%;(q%~*wnzIVOP44ONav_p9 z5#cCZ=NE+>JR6IxC1ZrhU~}BeI~?Z-!(MEXDcHm}fT5-OL55141_Xh{!2YxEtIKN4 zb`crCN{Ep-2L!0VW#>_#-}~ZmahD; z1{aA7>l&8Xjzv$ibS|60;GR-d)e&np_`zWCe53J6?dI*N zcHvubZcLc)JloEHwml%-q_JB6U;`EbZZ~sOWM+A_UF3>5*k7kR`SJ0#__5tt-=uv?A_I?AHcMchzfD`*XU>g8YDMdkGk!0Wg z-bO_6B*l`#>{l^+tU^Avol@4Du;kpVio4VB=6g$d33%_|GY{>({3s+)Kz1^!j^KeS zG;&A8LWxFH9SW)%ky;#+2SUFA@HQjX9OgF_gH}ALk8!=FBjgsoX`Rk0W@AKTgVO^dv{KbxKIG^NgPBIUSsaSs z9Wgi@&Cn!LjwpMP8!P{g@H5Xbg>TS8Uzlb}=9`05K%gQV z>4?e%uily@U@~XLdS4FMEePjBe+ULS&zw0$O^IdMPaJzQa|cIOWXoZO#)+eRD%M=rmi^cwx_9K! zbdki)4D*Z&QzBk>=Vq+fR{wmP;+Nf%Gx|Oa54|ZEj#7$z{LsyQa`+D;ubl-kv?hKXL}X9L~U4gJLj&SMge* zQzcDI@$-Y<8*WR}??Ny3JGlEuSA(w!E_*k84Q&$`3}nFKaRYz?Zm}+BfD-s45cML}pdHB3 z0f>JH-XBGY#L!SzH=Ykj>)xwZByVNm`N0wQ-dAi-+Nq37tOfEvBd8yhtN7jM zGxX9ysWdh{A9MKgh|uZXn6i0C7P`i)Z}F-;`PQxfk-kQhAWB_g-yDHXqtRgBN9q7) zaLwsp)*7V-$re>TVLRofBVmrL@X=IZC&m^I~w%$}eSz~uwCWk`ibD!-th92g>F_bkV>Y)A!88#p4p3!APL6@LDVqU@se z-FU=_agdOY_-5#Fbv+itCJDQcqLfw~d^Jb;?(^@jr3+dFR9)o(Q$g=1H8cuCni-WI z@LGgr9-1~mzTi-t6y*K_ZZ2u>9iVn31~E3k&Sl20XYcZ)b{uLY60n-(L1#;cbe@d| zTf*)4t-O6YM09=so-YR%J2brR-3|tZhM^Y7xFj14YI=AbKB62&DhJu~fw_w*gUbTQ zOlg@kU_5FvJZn=A)NPgJUs}2>)5A>JHN~-Sq z(6=h*oSBzkVYT>CeEShG*!F z6=LHtATOF!XPi%oBYenc3*302eU>Wm4f_m>B;?m=2Z^lQAQ$WpNk4RPlwMG5D9w+n zLUQJ*TY2@;#_JrE@8V~54!g{pKAnfZUQQ^GyrxV{OcJ+SXvyCMh-F&$?h5+IOECLt z6M}yS&;0o-Xusk9qy0u~2MBoL+QEHB{1Y)3@fXNTs(Bsu9kK5LbjCTt^%T<^mX-Xb zC$$!Cprv4_gpDQ#)n$Yi>Ql|gf=%lSZrK2)s`I_04KLO|GgO3axv=S`cV z+kzI<+AxzdOm&?LYymWEEvOQ&{nLJD-CgWof>jYBPD6g8U;~xtQGN)!V$b2i@Kwi( zmQX_Fz}~`pkV2A+JUQgK4|Z0=s`ihd6l=5sR-`p(VYgw1uYtB9!u<1x*JKh_uzF94rn679lh}9ri=#YkA zunKoEZ7VDA>7kv@!6^$?omTA1rAwzTyRJKGm|NQg*B61KN6Qo69&|#ao1K;p9Y zgS-X~gT>Y#KO^zbF3*EQ_r}k6W~;a?cnwxA`f}I1NM+@t7$)5Z%}q)2n@)wsro@g_ z{G3=@xH~zVq)*Is<%}f^4#WVGCcq7 z^P+QH(_3$BHRqSA>AsZAU<4g_GqiH5bBoR5)GikVSJBb~VW)!*4jCm&9D34wq`J9c zWYaC7p3gZ4zxG*WzF7bGO=nY_&9x1p_s-kAnEkBJk+H+0;*pc8i-^Z+T<5s_u@|%9 z7*IKQn%%oPFu&j`a9-#DiHwm>x*j6qpfJ#m)PrIF&zE-)nD+qBpo4g}eP9}_m@qxX zY^C{!1-?RlC-E6jP8sp#GBi$~R@npo0*Lt)T-xrJ_CNC>N$Bc1Qzrd;|7mi+Dqg)C zxxQdjLh8PkeDGtZzQU7*Ot|OlYMz1~65U`Cdgt^+UC2h5GMbqQ-;+mc5oTQjD}xMT zYc&=ujB5wi3@*ULK*UwT+K|FHzTr)2hsGst4^msfbL#GcdazFrfb_8hA>^LLV7Mx- zq_mHaO?hspXch0KRg4?~7(-J6t)iZ=Ha{fFtzaD8kyV&yA=hWHLkmn6yho%Cyy|m< z)_LL;G2Zs|e2~}TnKPEnu~ExkglRZ*=YE*JZ$5^R30h=WS6o~AM#+qan_UsK`)0yRDVyx$*%fDdfZe%2zdXo553S!feEZ9e!#}Y1x6~zzCS7**gdD?edCl)4kmF@!zKt!Ow z+ubLcy9^&SY|W;V6&6P9aTNgNum|VCKwZSlyz+RPP;(mYNt#uY4@>jR83V~XQyM2StENc4>!M89LC9{~UEDw%VRtO%nYjO{pLqR4&Mr3>Wh{ISC zc$#sE$ucMruBTMCII5m>e#=QNkXHP*V>CW8VL{--HB$uLja@&=`t#bE z6>gK)Pspg9_o6l}G{+`w->hfVU-;E5osy~|`#jbKuUa#?%REERV`Ot-&h>)>m*o`L zGo#b?)$X?aOY})c2^-@BmvC}#>D4>dHAN+9Yq*||r9CvwIFOlgX(rxwQRtMq@FhrU zQrdkfxncP#v%_Wf`R>eJ51dsPv+nQ)UN-0uULqutD5g04#X83F$)8FTD)3o@C_e*$X%_mtx5#oKvq{aTU;g=bkZx%U z1rv>=v8_|BTiF??{D3N$0dO#!MUIU|6pn}kmxx=j?%?Br8%*$xqSwsS%j@s|^o5q1 zX$*t`ssP|!o97{NF`>;Lmjl^Z!VqT{7e3t;BrxRR=6;jw;AbQ;A--d!R4+&LD{wx_ z1HnCe0>u!mIx5y6Q{L3SK<%^u<|K5ngzxpU{ImtRgqT@n#o>LS7+dAqJCTh$rVAJz zlc(;ln*k8i*`ptcC}D9~1ReT%+_7LHys8eV^1y-y_XcJmse%z8y=UYrS9w8o(c{k*e^$YxalA08f|nJ*Utvq$?yxgNy#qUl&wx;GFRgd~Eu zMl!MhG|4!HFpj9|!)Q+*h%C)7kvPr`yf3&{Yx0?#+H}&KK~hGH5EQ}Z_UApQwKM^- zEeG;JfN4^F;Ij~mVCUnMlC4eC@npzj0Rl`K-Cd|HNmvJ~7(9zkfaNGekdu|PSB}SF z5y)Uz3t8yJN#s#XanHxUo&`hU&cJa_Gi)J)G!Iq$k%Wt=8*n#3-R6y-Fwbi-!l0YX z14WsGIuvy40Y_EP1G`?{ek2N%bEBPIoT5i()ub|Or`S}H){F2DLGlaiQD2nNalU1{!g^VGDD@lX7l?OF_`TZ>4itB;O4 zM(0pv+u0#1a`t_bR~65TKBwro-q_W$M^7x%Z~mCp@MN8X;>ba@8)FWO&PJrz)ccAq zxYRt-$2jvnM^xN>Gq2C~-tiHa-809J&u>5YNF%wNlTq5m)$n3GEkEnszAmGjiev-7 z1xNk&(fOd?kp$d{bimZ$O{q?6@iG4_oM zjx=DxXHBJv3jSGvNVnL${VV1HRDt7`P7IU*xiyBkbfJ#`p{D8Y4u4BA+R+$OOL;=5uV21JnsGIzjUkp#Y&0S?-0YI*O+&nJ(?UArWT@pu2-{F}4izSW&4 z*YWgf8+d5U&t2oumuB)&!Nz%5rp3?s)vc`$OMqCgc%bglsx(kqB@|c_{`EhJHS(Qb zVCV7HaOdR-$Qo;(!oZs$`qkLAD|qYU;b}M=p~f-=T1@_{=v6So;DqZ{thuTP7nn1! zA|~M^ab++BCJz$H#S+bQYU1V-P&=)Nv%(g~1Efj0vPk&>tfB$t0uc#8RjJs!g>5~0 z8rrNF|2hn_Ow>*+BGyM@-Hv9>^d!e27Rgff%`}3K#6g6q67j zR_Xx;SuDS1phA=)(IG36g%gMd7Nj&WLp}&88*xU=-)Zj+w2&O7fO*_AuveT<6an%O z0X_^>l-}M4QBlN*tAR;+Bo+d0_bW0*_NPKeUSRw<`vU0uVPCrT2WFdj;Xo$VzP^5) z!)efKV2;KG3zLeAo4kFYkQz?${&%dgqr=#NDyImM2YPW{FA@Rtn5KsZX2yBGoXvQl zpwyX%n9b5_0<-Q4OxjO|Kh#fjM ztKImt98Ygsxaj>V{iSCkNADFdhxdpdwh$;DEjb#TGyAT<`9o=Ud^e<}&Cp~V(-n$q zy1LL;ms4|>50AL)_^09FH%IN$zgb=wGF`Z=+p56QwR`M|!aoi57j>tn|KoY{Y%4Qg zhFfJa-ogCQ@V$1XTPFGB2W&7{&p7p3t!7}P(Pui!62_Sap4SVSloqF453q9U*uYJY zmCfrxK_9V|qnW)smwAre%^i*{pZ9e$PD2+dr0Q%upHHuRhxM|JZsD zxSsd-|351nqU=JF$le)IiHwpcJCxBe%84kE5i-gsGO{W%Dn;UStdiZyN;Hg;3Tcp# z(){1Acg{KA@AvnA-+XSj&*yv&dcWVV=kvO*$MqOX8mqVVN?OwIOQXb-=MuexY|jFd z`3_%IWNR^EirKi}oZ>5W-?i^${3fwk>DIhGeY@1Q4DHCjY?7 zGIu(XsIyOwmAq8U!A}9oZez}aq$cJEkcKc@n|CpPtD{U`{oQ)FW>ay%gx%?Stu&GC z0W29`L)T8dRT^_#YIZFGRo-y?a*!dM70pEll+z6Ah=tQ8Q;Ae&@gs1vqV$ zhQyL)`R9=%%Dt%0Vw6Kmp~#dF_0)t31Wd*ohfa}%SDA4snmXkiMyhNi8$XLC><7fJ z6PM(>PB3R!FM)mP*mmU^UF3XhX$06PbCcQoZ-p)CKHb!DFI9K_Yop@bR_0B0(++S1 z(|YZuO$~%O(|NrbZ{dGT?HVI`w%Td;s2<7=ElFLVbJ2-{Mvjrltbh@fQ{e7?NC@JY05>(PD_w8!Qm||yi+S|SbXC8b|0ix zI138&?Se`AahxL{hZ|3xyzi7IlK_6oO%WfTu^Ry`P`t}!aSWq&@1ynEpc-&b;HfX* zlS0aN@%^b=k6G`C=^?Zu%h|Jgrf}B*yUdHS?gvaDC>b*r@)pVA_0+@i6VN>S`9!0w zVE4OnucZnG0f!bOK&*zhLOP!RD;4PcZw(r3#4ITOdw$u{lERF_eT8e`H;})cS+rV7 zX|tA$75IPk*WLG9qkZQ?Q`z_}yYcgW{*fkM-eq{R%r!puVa5BT0s2*GEDGx?ukbgi z>E7PlcCFwsGpVy{OuXsZcYX7-6Bj#N$iCflE97yd9UDSNCiaD6XA7H!&YGAege-%y&qBMNFR4huG{+E){ z@ZPl!?r0U?Prv#}+aG6c45+S$_caY3Fs+HllZhVb z)y1`s4Qj0g!&?2JR_i>WA-c67OW;a6NUwte&wJ zc4sFkaa8+MSU109rDS|_V)g2JnyLHxOatS4c9f0!^sX&ul%=X4s;UZ^US?z6YgLPY zZeueKe{HSlcSEOPkE(4G=C4?t`0avL1J3JMi5z?r^E!8Pg29RGx-shyvx$Y5Uz9xE z_Eu(TvFmrnHq?dq?rcHs{_CZD5I%;uwKL!Of9^1Jg$^XO#$4<+7P4omGaCMpH0um< z5$kDD5fO_2Tw|0~z_`Y^j1t0g?}%0dtkPSx{=mL!6I^xV@^+g~_+cOZOGN@4JtGwt zx_eil18}TvwY{?B=^+&JoKl^AJETPbIGPkX1p)U^l$qkdgYyCt?SX_!YMzT)ha7(tR)=LPK8IaR$ic$dpgWmiu8#|YS@^7S`2 zara`agkJ03<+oQR3lLIp>$i2T=1eNPv%X;)JnP zBSxKWDr%M`FgETpRCU%vu}nuZ;-~MLi=it)9Y9v@ao<^zMvtxw%;XN1@a^-R^~D?# zqz+|?;7^^#?0UO?tI5Ew#5)N~i3#B@vP#QIlis>K5)&iz9Yk=qkzBk1-rV!v5u;Vm;u6r^8I?cul1i7QYGN0*Vq6 zQSI_63?A0pzxQ-h+_>%4T&B`^5+XIEHy|F{lJd0J-|h6--dz*<)z08PJ=WZ9*Y5L`%_w~L; zOV{NVg+8CZ=f=Q>iJ=LxdwXc3v;Gd1*p=Y&RgJP$jKvn~7DC}hdEW_$V$Y8PC+y6u`cv$@Nh zR=FJ;S*y+Dk$pG(oK(8KZNSybsC#^6@yOzk zs}1{9RR2u6s~0-1O>3_}wB0*;Prdc&>+1F*e;GbhV`ugJ+Cf&)ecl>6R8|jYUebHY z+j$mx*1dkT)@(aizuhH6vpe#KJT^5rc`0Wek^}-4CD-!T*;m*E+z_NX z*p$PilETTpf{k)e4$C*@XK?i6@1mAV!M3;O43Xo3%VeE>@kzM}KX%e5vqT9ZQ~Rsl zo!htXj=n}gf1i^Rcl7#_n}j|bg6~URxp(B={9CA$6nP^3eV^kr1|#?aRcZC`*dH?v z?BD;EeCs&=lTlb^w_(rG`iHwUY6ceQ2 zLg~rK2n=-jX9KTRJddjDTeU%{fy(-y!^82}Jb;FjTA8_Oe9mi|7&kp3JDBNRV35v) zoN$}v)pDkO>~_bs@^zm^syxRrlO!x|3jzN4APZy*7Sjj&P|4t>lW{_BE&0#wS7ON6 zqGii@Kw^>~C02an%#!q2J!6~Zk(%|L_atN%t}F9+{nl{c1otD0fG#e;&K9?ONzxL; zfopNI)yt@2J3c|`0J={;W?2oy-=L5ad!fQ~k)4pEbqWR~3S4F(A;^U@>u1SL!R~Zp z?nB9@5Dn+g4{hVu{tVnu)-ce$YG}r&X2rOEcN6!=A?H!*-J5R=;d580e(Sbv^NTB9 zrNS?%VRxK+Udc5MpnM1Vg4^IWDrW67>U{lo%VovB{_pmir50UF$h_F1xAoxT=WaC_b-dNBepTa+sWy!AIXmk~#+2vVc2rpZIiujh;9+Vu@Q|({TTe%; z|Dn?4${fv)73)-+<@jYE$)%IsQw1B?yQeiIqZ_L;&E$->t7XFNvM~D4)^Gx85|Rxp=}!E`XgR z&MRI?u?P`w-yVtMa`^MMW~DJb7|( zoiYsM8L*1XZj1m~FPy^)iU+fK94g^bEQa4VlJ_-m9B&H<8eNO2tx25eU0Xj~Oumoe z?ENsQ*}(9K7k8wse{o~|^&x}WG%ii3{%Q2^5FD*yelMLhFxw<6!p^!Qh|-#TcGE=F zx`?qFKynad#O6;OUl3?26NxI&N$PGLnYh!`%ky2w2DL?&PJ?P#VjYcF=ATQR3rD!L zH2Te}CME`t9WQ5d?I5-DO(OBUh{)tS$fnQOMVz5$M|8Y7-Ld2qPxwx_Hg)T}{xPRn z&}YoRWC@Ka!rD)5`^~Qldh%kK-Z(fLSz&rE8%CL1I>lNL=yGE-gdUcHQ5 zkBIf)Ex-q3anUZUpTCH&u?Q0X&IN!Ngu+rHiS12A$PLC4O1nKegbZZ@{%6z)r$<31CQUw-m7MvOI`>_8spG>I&9Z&=w5fI+*1^T7Pzm_iXsf`;o@?b^PQ%X{5u;YAPH#L4cE2AvFb49?l479*mEH+lO)pzbH zlap9u`zZVlE8oHxM)T)!FP`y`wDQ#y36BxfiMGh2-_-qB;|8)(Vo`A zEl-Aw&03!2qC!Vgnd$HPW)m%Jzs~Wuu=NoE7I|cigP(UxNdP&3HB&2U+jx?_Adx#s zYbrj)@WsV`sqns@0J)$ggrR#)t|C>nyp}8Tz)NLx-VY>X1QPJ0sRKu^An50 z%j3t7zXN5z9-DP_GL`S;d)3^tU#%Xt-zaC}n%|iC4?01gzxEB*P=W~9xB8p&nmdzX zow8p|{WIfgPuIp?-p#-NXibYR7<#I$d#@$ShdF;#g;u=UNsw-NeQe1tLBzs~5&}k} z<4fJZU&!gNkI7M1-Majp-(UTQjg0T|=W2V=laQ#W8+E+aMD}T{>cn{p5RYB;FIFtF zqGwm@8IOQT2v*X0r#mS`(yf@7cs6q0x_vtm1((+)qm%r%HCEq@-x@81R}|++{96Dj z__(f0{Q55jUt&^{PRcB6>n{NJSy+00JMxI|ZLP^fH?Vx6O}Y09iJuq+qfu6%J_df}<`wGq?rwMKr6MrP#Kb>w@j#g^$y=TQONeJtY_eFvJZsKKqm`nf zT!%;S-ebsyJ$p8`*9(5~?`k!#n*x(1FJl)^sHr_B-LyrSm3}c@*>Fj^3ZI|Y&4|}r zczF2p1-)7+GLeMB!1*47A_Q2$ZD>E9Ys=6nkRI-WrYN!upiZy_USTe7Q0<0nl<@_= zl^9KQ>y;RRQeZ*j}MPxHPWkE)Dc=(6VR@JAC~W_!85s@4c4tp*#FM${T{qSUB*Ywhmu_vy{n>pnWTI{9piz99#`eEVs1)Vg7e z^{tGWMQonyYCk=Sa-!17sYM?4L7%OJq_@lV;I0d$RnH%d`}%drrxjzrJWJK+aShEw z3%S2nt@>+*!?I;@^vPmFu%h-*1#rU+*m`wEX+M;`q)jE|R`IM#us7tJn6`>3WIwbL zREsAsUVP>dI4$fRkxzRM9_%uzf6xt4_trS>b8}sp;i~VSb$fP#BkH5^IhU=^;nZYo zYU(uu+YyLn-1VG}y5NR9c0)rO7pwmgn&0(ugfsNy*ia8(vi+N-|J*qek zV|m+NtQLsM*v*9>k4r^}>u&V{%Gf2h(2a7wI=gGvt^^KlQ{;LEnA_!HaRGNwVepQx zQ_T>PQdFf&1dOrj5JfndWe4%ohowuoieFTld$wjlxh$=@kQ)?Ia)VD^5??x)_Aa2o zuTM)m$nlwz@hzc}`KqeL7y@pPU~u#O3e!9u<#+9R^yhiULSvM*wFZ?)$m2|Lu5Oio zW6zF^UA!x}ol)^f1 z*JS&Q!H3PoMn+iqY$9>}cqh>p2r9CX2}&FXhrWF&dyD#y7P-mU=W^vozgPaL)?DgJ zY1Ym@q($}-xs*_va5>J(ai(|0Bn#$7_@nq}mP1*zItqp>$I7plk5&VTXYuY})ceWv zWJ?Ct!kCjbscC7A2^0(6x^3GI_a6;vAMw{r8kQCUrLJ)K$2kZcaS6CBmoK21B#b@~&z@Q#+ajkMx|jm3TU{ssPRe6K02#ssSZSYRP{u z9)G#ls&?I$9T=vpmUujQ!?za0Tb8`@?kFq{S{LXi@n=%=p^TP~zC>#fj@Fg{ zwJ}LXbjYw7q1jORuD!2-n(_;BYt>5mfLU^{9NoMf5@iH>;QI2r=dp&SbW1%Gnout;gBk za1J$K_18Yt+M$(|8Px{(YMLt=&yZm}-C~+}!!zKc*XmcTZ~c!){CqjeR+$N$q6weh z)ov)gD}39_gic8wu+>qrP!9H-sXbXcZx@Y-s4{uZDMhU{!&38QGr>ZRo8gLIGxF)~ z*!9n3qGofSxzfj!dd5?(s;c^QCx9h+lP|8W)YTlmS}U|1ei0Nk5h|L;U-Q8tmh~uB z!Z}d{j?FFmo_k2op;fu%xK^+UfLlh3XZUBbdQ{pD(qbcV#!e@gYd8yh)z_L#Y7D#^ zur!!__Ot8HufG#hI@_vkf4c_q)OS5#cGyZSG3*rwLd3b=OaNZ^62>12Svmdu>|0Fm zJ-tCpzT~)9)(REYf9BTjyC4V)jO1$?8*qSyhG?HYsBxdUb>e3I zC8A6D~7A!ELJ6Hry4p(fQ*L~TvD>NhmUZ&h%LLiIRabeTL zMBxG1jR!@+q+TmDGCB;W*=#WQn|b~e7MmZv=;jrVIpT$%w^h|FV;A*p6K#(M8!!58 z1LZI#r3;5lDw6xF=8_??UM>CQ|DRkTfdrJ858WEnAjpaN@edz==C z_z-VSFT>c1t*JNQu>{0u`o!hWIQn=S-+8^*im#2NW9RP10Pvg5Ds7EDR5&3tLwpwF z6v~QC+`IShPaHq4#GorCwl5t_4W4kzGE4fF-`J#Evc(;_DkXAXx0xk`WXNfk6b&M9 z*jqaVa(6;Q=ff65H7bw#zGrjo25Lk)WsebyVtBsuW3aXsCbjUza8&u5J6-sj0|yQq z7Z50nPd?}9>Z)>q^UvfP@`H@Q$w47Zmu8*~x$MPm%r*ILhR=XbY%I;y^6TmAKM#ed zHm(2purLp_AvO_($?YCLeyljMgoQ@+>D;*;j2Imzxn$BscWgf9P@%T2&h;50-$851t ziMr^T`d!h52Hi64<*Fn#7nZK!Qcpb9p6}TDZDY4=(XGMCi*KKEnyww`HfCvM4}2`U zAL!ervF`*j9+LykISkb3+9JZeRuZgSLAn?XR3vh}DIF&i3aVc0kE=BUIH=8LZaP36 z^NW){UE*&PNskD3Rw&|z77KI_xfYIb#QGYmMimOU-MBFWmy2yWhCwz7FhPWTnvr*W)t6BQ2wVb#`A&VP1VFDiZlP?#Wa6m;IQKk*kvGlMa7(i;B ziW@0mHhR6USeR6rGTexhT&P4+AaJqQ^ZSq+#>D`&VGsFNP$mG$#9L?V%9SgF27imlI5O>4^@YU$g6AW%6 zyh4H_3Bjk%*El4A06*9to+Fw&F~EbW$Rpdi9Ak+u?9t}2HWkPHLYH8hmgtiw8vd+H z=d|y;$z9=m#vfcr+dh{3UJ7$f;u3`n*_0Q%P^oVEd(xY1`^39j~i;W_rg5$HyECY?@WMPW5=^B0ag`Op+>F zU`n!LyB5!D(B}nzl)Sb5Eqd9m?6erE z*XZ7cICYBYP3!Dle%XDWxo2yPm<2V5PJGHw$;)O>Wu|Z z90`P-X6|XPgiL8Ct$Y0b9X5VY3Q4PyH6k1-njfoB;p~eQWS1`Y9eOX&4%&qCvg8Ng z6loRwZC9-kix+nl-T`xoVQZ_)tYb+c7{RLG)OnVISSNxuObeJBb2qz4T&8ZZTS0ss z{lkZ+pD}JbCU70wt1P5&4_j3*0!GFubXdYOf{bcG3*^T=RZ-;QzCX>UmJ(I$CSTa^ zX33Q}$O^b%_Q4379Z4FzWn&a%_*oFqsZ$RO!5u}>(Jcsi^2eKKbr77a)gpl%u*f1} z-6udzX&`JOhUsaj2+16ATpcA5W>Dm2_HR-bZB;3bu=gU7_wiGATM%^P#686${(wjP z%`@*Y!6oGoT#Pk3H>+5$T3BNDEp_>Md3fnUR{q=bJPT_{Zjeuj^<2J+JNthd5#jc1 z^|?u1g>}XO6ikdM!alf>-j9KhPB@#uuJIpMU041ESsst_vq39wp?4FH+0m|Iprs;n#2*$Q4C>222kqu z>3OcRSn%lPbjs^Q7RNun1{@JN*!9?}r6e=5ou-|21VO`bK0ZF?m>am+ zK$Yn|=bDbUgo@HETWImL>a^xVAj1yQp8|?QIbSpDF6`Szr}_L@=cWnV$v|X#GTA=H zO`D84SO6~?75zCl=(?LWQw)P8|NpO+4R^-K|76n2rxulORKmP#${8Jcf7gN+{o-dw zDfLCM{H8i>Vz&ckLb+vUK7xcjE(#j;`rGehx`e}fmRSh)GI*Heihlo?7@e`pQ@I7Q zaUN$Oms0_Ia`9*CWyG+1WKzT?!UyD+V>9N=p93dx18^^QLThO?JuG=O^kh+=O}t=3 zytyn?bh>nr*gr~@ZzZ-pbiQ|4r%9OcWHXIjd_!;CtJjOw5@xFN=cv|gH)OdIpD5aZ z_ea`$j1zM%8YOW!X7w|_l8t8r!XW#WoQgw^1X`|ee3$h;^g2P2zAXv}X*%B2bnheo z*;A%$VN@=;n(>;;45^K0_M+;v$p~btF=jczx z-@z2bl03pY_ECnemtHQl(enu|tbUCrAtY_4h`Z;quVmj>H~ zLzxszIz^1H@mfP(mbH+mo7DX49fzvu(CgzgcfjuJba!|6itW{3Iw0(@Gas`)@$u2Y zPG8;{N0>IpUQP%SL_fsu-{l#S!<{s<{~Aj+NSLa%SC>_=K@p>`X_uoGpvZV;zmr{7 z1`It=uKH2CthJ6{O5D5I!t4RNosIhCWq^|EC4?e0Pa;H}7p$ zyG3?(bBT!)iI|%nyFu8Q5vN+gLCL82F!_koeV86K`-37SI z9X^GaAM6m!0}b2D%Gm6*FE{=)J~kF!dEawZz=9XIqnx02c08&cXi~=8bFaxpPqg0* zyjZo%<;*Z@mWL|*js$0zjW+4C01?@e=Uq%c?d)@71OYd6F|N#51+SvUx1(jQNYAKj z&sF8KE{4`lOgUB4dN9k5%q|^ji_i7Wv#;E>*aU1%0_X~w*=Ib z*SoKf_+0G#E@}sB3}J$;8DH#4XlL%pBsM4~guZ z;iX^_OYtb&K?t`tFr?@(CQNwm(pFoeFVlsbcSq23}*T2(rtdY2=xRq%#uVjb>KM+CrqG@!w7n@QMN+{@>0R5)&ew4Qf2ol!~D& zO@9@6K{*4s$?GaBo4KuG$4HKQw*}poi6cWKIZzB#yT&ZI?0ybmr181+X- z2S+?lxTtS)oiq>>Kfi4adOg6Z8mqcTg9h(7dTjO6cN<@Rs2tioo`JaY<4;|eIK~w{ z7-W(+wN0y#(JPKC?{#bGj*mL8k5~VdT|r&zvItNpy5Q$@rWGpdI8#%F1O^&OgUwG* z9=;ZqJL|-x5n;(A!;*1gDm|dtsX=YFDmYC96RMQ&Tu`+7TIX+JE3mkAbo~5mvm1@Q zvWM|S=m-(7E|srdARU3cV!YT{R#bdTz(AtV*&Q=#5oTiw`iq8$XN5YWYHDuQFk;D( z+l^$?)HvQVB$(cSUV@CX#uzp<%O7-f0-hFj{l$n7H>%!#VdHb$-TwFiTs(imxoOg* z%|4Z%d*AelvJc-fyVDb=PML&Pa+?t3TU!x+Np*miAvr>G0gBiq&re@ENMRx`J~DrF z3aYgvDZ~=GLmOmlwT9;=T&J~?wf#zlc#Q&mi@#9hZ^7zEq2c92ExY6$*ofRCLGh*A z``(>t;4%s6y-Q38pgDf}bUo3~$AFnMWo!EZ{_e(*5v>!Q-Ia&haKD*KRJRmv+%y~-V`wYr7OfJnrL=o-uZTowMlL8U`%Bm znIY{D-)~X8ZlDlcTAx1(WHA5n5j|`%c3yb{qOTAxkjVy$ezTM8ZWp6|>9NYC$2)d*=}}wHY}aoo^QU;WI-y_V1Yu%g_1lTy z=_94F53+HM4r&R&v?5MRTf_PO_S_47KPNBi+wbV(A#UHhN`ut%x!mWp*N`k+S&d>~ zPjd)@I4vyMVeMaUSaiRSPk+Y+P=~tV=oVA8b%DpNn+rwKCNCy$S>uo_S?hXw`6YK2 zFz`gNpHQ#pNji|{n_ou8;#1-m-NvvN|RaT#>+8lnN9JyVE_%yDS zo^n81qZq7#4>#L8oB0TfKN*j43yS_SgAWQYSrRzeyhiJKcq%u>v77ahfC&CirYb8@ z)aB_Mvs>d{j!|9`T2x7ri*RG3s<0Xey7BP;Z!kXC3vAL)+9dIinP^n`u6TetcC@Y; z>-~&;JUp36U`M@;YGQ#K6*!yG+_EI5j_u-9Vb3!wZtSJj+e4bVAumLd+dZ`MvumtJ zU`ANPx8$~PB*k^oJC39!`d!zDS0Us}t|%M7)t?$QZhV*BjjjG7U#$kEh1Hd@<5Tj= zKA!aHYe%L{{E(+p|DQdB@)gw_iJK8FBw_)m~$i9WrwU^tFjo z+`x%PGavAMLHVBKIjys6T_)CUCaF<58Q|;dz9)|5Vva+oAKl74uZgn9iPs4)_i^Pg z{W$BV_h`TD@@wIlpNNdKvi7MFk4MSeB(@$l0@BUj;wU z7336{&TN~VNz!zw0^7H9d$vQ0H9y@x>KAhYI{NGMsG|pu9z7~fg!D0yA6VE@VtJ=G z7Z2Rg$R#FEZ(-hbaCDXc%$2$+0utiSQ?$O1%LAf(@Kr6%+}?q6s+IEk)hjXH&xX5_ zc~{+J;9KX4%NNg|mj*==ru&dItK!DZ1a!y->ck9j2l7wo(6jgOKbvgYfl>6JouWrh3uxTMQ``Z7SE{Qv6>m{zNz4_ zchAkzQca%t(K}nkA39$ybXE$C1We0SFC|KXk!<^F=S`tC28BLm5j}ekOohky6a%ezYdY9J~*6Tz*zVM=#*(y=OYGK9&p^LlV6x z0}rfUC0-A^V7aXS%h`RWUR{3XtV+Ww!!|9$N1N?Ywmx9>S=a?&BF(oxeI4c`6QZS0 zk0q<-r`}n3b%CmyivF^uDkH2meu&Z(Lk{s*h7FnibK3e(ucGQ~IPo{_TlPfvvVEss zrh_grAR)AN5B33|18Y;%hN$P~vJHsJh6=+S%tNZ$`r~g68)mLOv*8RHqw2DV>h6a>c4Z3#S?j#))ud`uCd8>r z#Qf+W=Hda?zjG(;ybbi&8C9P$bei9iq%0^edqkz7CLhKzJe1lA_`%VV(Ko0nk8e&C zibK3f^u!>w%K}C13-bH6dN`RA5Qz-Xa@SaZsVQ8Uhn|pBd&SEoka0wmk6Ai6;R5~_>ex==&I0Xzncn2s2g)QkAEzXH5;AAm^~CYS2OTeY%TIw&MY~G&tJ9- zAo>%+Ia88+*RP+DC4-(YHjA%S%Qz;*I4Z#J=I_q^nubsP80`LSu;g-lT6yI0Lil8L zm@;k3`RO7TCkFM7nZ_bNqJHd%JG-qq-N$B_vxIdWxrD!1MGw8OL%zA;ehCC*nOb1gj5`ti^G`;1ZU zG_HK%V}Djsvg3tTa@nqoeF!#%SnK6Ilz~mb?h-aI|NQzNEA;TsyZrj!D$(A7=e{!Y zWb55yq!CgK&<;!O6&Cj{f{Rijqqb>r*4&j+IuO{2_jdh^&v>W;_m+BtOL72svB6t<2*|E~4oO)&ur$4JLw(G>#E4v@fP5HF{ONZr z;6TJym&b0mejFbT;N>8^N7gl7*-$|$-rH8O!dHB%Hi60Z_;87hWMY8lqQU+assCX` zi~H)k2OQjPOb9!KK#y-Ob z5wOG)&1!l`&YR9bwNB6yGZnA?e5`~Dg;w(>jVZ!}*AH_;`Up7u1Bu>LTOv%jU7wY2 zDvOjHc@S&7t!&C!xJsH3;b?#F*`<6E&`q|%^pw8WAdkbL^-)-@q9I8eJZ{l%oNEthTB3!o{MH>E|{Xw=u9T)CdqZYWyovd!1~GmpR|k(G96 z0v@yy4bv@o-)x?kX<@}znSH-$l+;Hq2TzcBH%;Gr*yrrsQ>A9`N?1fH47)NG^)A

2lrkID>Ll56FG3vWmAPjeXb9$`6BMjh8MB7fMe^EL+GsG`zar zm;+W-6;|5_>z`M8!4ld2;wtyu1W%nC4gRC`Cx&5N-I`<2!&_W&i885qmU(XD08lzd zQJ}=Ppknwjn+FRoN&1;6Wg{ZoGb7RZNKGT$aU(AqGwkmM;6uSa%qi=xSB_yd%MP*l zn|#KcHO9m~x;j^7xmNfl=HrzunetA>Im@benGgzC>N1PGixL~H@dtYv)A-n4w~9b9 zaa*2TKHyLQO6f>Mo!a^28_^vhelfK|4% zReUX@kF2ykfIaC!9&olqED{wxeQ2m@c%YHHCfB0z{P{skuVNL%FNg-Vx&1(j6%Tz& z#ZWt;;@daaG3U0&Hebe|%P%Sspy;`2jegzfE8{p4-COpVAx#rl$W1^hqCq5ZXFR5s zvK9w&7V_0Vjomf<%WOQ4R=#TP-%KLuMY*NwW{hyhLoTyS7wCE_Uym20)3J9 zL@1V;14~Vr2;>YEk9CW&>Az}7JFGq>PC}FbLN!VFVKRKJO2sehDVJFjx3GYF(X^=G z{(tN^w`p-(&)T|tHeuHe61#$k;1f-oy>eX?t+YV zZzYI=*4eFc7{0~k{KHXarc~O(M;2OJ1&Encz*8j>lqP#+2qwS>pOc48NJ@u~s9+4e z{!2wn*d*))p2R4btaod4h8@w1cnmr_fltMjq7*fZFokcYrcS@_oW1~pu1K2KthqYl z`r4A-T#kL6vP{H}`x0OhKo^%<#}v`?C`h^wW)ThxQRAKynm#{Grcs5?y=BWapLUX` zO23p@SXlk)r@qY{40v)3R@Zry{nx75{a$}L9`BVpE7DuJQ`q>MeHOOuee@3?(Od7v z83e6g|IVASjhDfQf9OH#9j#l!CQ|hH8Xc{tUJjg0!_Ctu5fQ>(S+dY=sz>P6Dd6g~ z`eIYwS>-a9c`4AQ5NZ}&oBl7W&gSR8rzh$;lk%F~e+-({bVF-@J`X@gJV;i*k3&gU zR2}!chH8`^cc!)C;nB|P!=NM&FShCzo>yZ1DM^fA!@p+bv;SvSmYqG!N@it~J~@aH zkqZNTdg&3wKqd++$lw|*=ws8XQCz93(ye^BT$ag)tb`s&v?Qz1&#XYd8n&mPLJ{?Jha012U=T=y# zE1D(wBH_k(U)O)B$;sJhQFuD?<-X(EV+; zGbbURy~87~$|4n(9R{nT*b44B29(B(Du}4~z6wzE{y|HI+4z2RK&3pl^1pk|xS+(J zd!Spd;!|v4xn1gS)|!*ILGCkywSNsip{&Z;gD~5%Uq3aT3Ewr%mELmxel3*y+5N4$ zX5_QuEK9#6j_oiHcwf?o^SNP(rl68T4m=~%vMl&`lL{A02ZnSEz~na+m;am%H}aRu zpJ~dxp&KfOAT(2L*|M|j6{fM!AJ4*$H|ppvQ==JI<;k*V@Z$4Gg%>Rhv%_6 zM!D|B8Ykm|-GM8I+1x2wJsg|qcgH$a9>ZwBVM*NYmsEZ~_i07ki4EpTlak(f5}hqoJkATspPw`J@*%8?@BaTz1Cqb$jIf`D(ml zJ>wC`)q#eq9_-u3EnRWPsA81Q?|EUM&0B<>U{3)Sy=xz5!#mkUgazNSkJ?eaepP<> z!ppDUdsmu)KY=7&$t^DPqYXU$&se-zm6{=8R&u?$tTg=cZlymLoA!VK`wC}D|1yiE zzwpBVFo<-wDc_E822b{mHQQyA^oXdask5cag0*oHfJf6OXLrK6#Kk&0SE?(7H=xwc zqgJ@(RaKV5bl-kzkbBiS8juNROp-EW*k`%=?X0e@?gBgtD*QVKZ9tTr>0TL8-T834 zKLBqJ4Rd9LW;(wh3_CzX9q-D@%A!>F%G834F3eZFfzoBwpJGkz-G3}{-zFW05Q``$ z1`~0aSNM^Lov${uihD*Y=S`RY&MRosxzNyt^bXZMf8mOOfx`ic^{GA*QetUi(>t^e z>=Yaa_IGQkGLTV%!IFq2@6-hAv0y>JG&QZS>cZ&e&yqAgCYpe?k7U)4^9UTdAM^X? znfhoN-1af5`lweu2i__i?c3!eu1P7O!56za=WFglq(;a*rTf%ABoy9Tp`2(zhk|lp=wkItVcAUr&`eiXH@;TT;KsAMiQqBtE&oq zUoCSle{EQ-zLUGA(7k%C=_jIRM>#uxZ`5Hrg%UE2*q6?spWu2%6IAe5&a3L5M%BS~ zPxgvM3KE60j9X_Rjonvjr?JxOe%WhsdlyBa`I zv|5QC7E)TwnpEETpQr$o!nEU`wjV!#7<&(jHDhI;nVu0>uCx(d4G!mVft|JUE(ciH zfrjZtt`dv90E452pFBOs4FbCJbwaHr?&O4DDwJkEc$fYI@~sM*qIk;lqIGX$$??}; z(+ak(e$_S+vIX@@LALOOiu&Ksp`wcaYk#KnIY&++tabdtn@wR0 z+xBd7In8PD#<0JxT`06{G%~8Y`HrLoY|gr)Zj)EuIgP=|0e!w{3k z_jT?DXbR$q!IxP0-j4rw;GP2q4~n?8W)!CrohZ(;8VilVVvs#F0vuUK~PY6q9CqY-iA6$2!#)Qiomemn19z2jQj|XQc ztd$rH7jRqk?{c!rj&(b-1tw-cJM9-|&vswOd?=+*ls>q>e=nnF*-lgAl?|a>f;;}P zxjlryX=Fs|_%3GWLqgsMt33U~smh`Ed65sV;3p_J3V%&<4&&jsCAdwlukAe|3{aT^CtofudP9eWVf|-7fbkXkJ+hRrK zDG)bwzhCvAK+DBc<2JI|=pXbC0xht7Ob_UTUXo8G>q>T%X5MuW0e|KCy{F7d{8(T; zad6_%i$l_eSiRX{Yd-DKuCMnz=c;6QD0+b6Q57u$`5qT<-0?K*MBSSY>ocTG3{|>a zY*5|mHEbl1VJ@9hesU)&9EehI8M* zet4UcQ}V!v=A-6FWu-W!MRtixxQ-wLLWkAK-T$(>yF6SJ~k8*Pp6AcU+68P0@^5&3z=)k(8v zHM;b&F*|9V0hvp*^CNC&F853=4Q$E9*!x+r^NI5;v2EHGXpuNdw*Ws*)d9WIA^t|b zGRS?ErkJQoV4p(2pOMLgvlO0jhjm|S!4R^O)8ocm!$Nz~oH@P3X;LRbj)mL4ktarQ z(F^A_g;tr0&_zJqJRjf=s1)~{rW^UyG`G6)gI81_WvocU!u@srcN4Y%T%N~CD79+Z zu;HfvDL|!^{U_=SG7Qh2?e@#**w=o^5@Xb|r_L}%5m1cc`mj~v+@NIpe$2ZNT2VYV z=1G;$cDQEY;c6(k(i{^e(h=ZB$WIm)AM~1O=MCm3`Xzi|9fImj87FSv1&Cc`sYf?9 zm&rgVE=V@ay+V7)hTk#$H?DO&}&!L z%wqT8%jh^968k%9J|F&`e(lw2iEW1*CGW7XFxl7gMN^y*c#_;Qv7KvyUpig{LA4oOyVkrF%07 zMWS*M4J7%UGnP3zDgv^BHu2$@c7F?5M7qc^Xrr@ASr>C{99!)RO|Ut?5h8}C%$t1~S69~ON~f;q;l`jnahygAwtI*u2~FrVv-%w% zw~S#${O688ko@n1nSlfMFG7WLipn5$IQ@Yz?J5m?d8fo1PM8IrPb(%0VM zYaL<}i(l}0EkG4!P-YmK9d&H_ zoH}0bER$2bFZsNll$hr`?6x(gix>N&05rX^5XRK5F(>V$`fM~_xxNq2 zlbH5#DJuLjLFwYeJ z_y;?F;2AFK0vE#Uee>qY)-zt7A>+Th#n`32)H%nyfld|kZ^i7B>&2drpLZ*cMc+p| z7rR^|LeUD#PSou z;fe{-HumS^WyhcA+L@PpkiHP`n@zuG6m5W%8!Fv?BIs}wWrAtU+@#1Z;A-f9#`m|T z;w-5+P0YK3amwPFdvF?%aS4<9_l4Z9h z;=%hBw-l(@s%tK3tQB=8M8z1Z)-B49?f!3pacm#ge(| zKCi@fa|TofM#8m2)q$5(?08%u|JO9D{0ss&a$?`-T%*an$;u3{nZU7D4*cL(ug*kv z#Pa&y{GuaM-i{_`K~ZCbsy!n$*ZDslNzPIA`ZefX&Ey~g{S1@*cT$?mPT$PjwYC%l*?%)0 z$|*_;#8xx@N{^B;H4EPotFUCTrJOQlzu40N+Sb_9M5H&bHKM?4^V+~D`1c(N25`-NmX#wqqC=m(r(mnTCXYoNJK7Cz;g+Uj%@#4A; zKc`D>`sEot#f3wp;t;^pn_Eqsh1`buSYi1qCl+?rznbl}EAa@FHGtiQ7hIl<{&f{g zE_F4qp5w=lk7#GaUj!(G&+z>zu~yv=aLN_#D?M;-248kezxLi`9cFWMgc%MIuj$L9 z?ZK>BpBEg6nAm5b>zd^^UhO#Qvda0~^&{hKnjSgU)p5oA*@v18INA7V=z$eurAN>X zvgo?y{F}$)<>srr&&FnHH+x?(*VtHMX{Hsl&UW3X?XaOapP-uFFwVrSq}7u=N06z~ zu-LLNIJfSTt*kb8*iedhE8IyeFFxxz^|N$cd383^pZt2cG%L_{3T`!i!+wmXApElF zupN?O9~FIKpW?_U_e4L6BK0loH-C%p&umaA*b88RuUjTRT>4og2Vzr3iC%*cAXjWFeAf#M0C>_kZK`FhXS)Ukxs6gTJa)2 zUP}IL(1@5Hy=#Kg|aKaRkQ@1{cMcmjNZ`UmD9}SMsY%3q5 zcJAEBTm>YMJ%?XE0gz$a??94J6X!wV&D~4VYB}Bf{;vUFd??g;K-TKsp_L-j5xYq* zplS!a`sv%BzIxSNxE+R7r3=LM<JVV&ZUnRw--9)-o1$qjixtO zbJ_mvnU&n~w2A(or-x3FVGUofTURrFN6NSbLRk|w#lCI?$A|!h-ezK7xAOo$!Y*f` z|N1Hi+NONwh`9oJky2s;r~5jiUU77FHVVVv?1SPzWv1ZR)2>cF;&ZRDH6z_mQ( z3J>I5=N)=*e=2@ljvXy_mGPd+gK<{KsIDsU9eRxVjmpOdr+oW+zuw%7J-9t=WZ@KM150_x+lgeXL?7;~MdP{=R@IR#wf# z-dD`Gqfpz5wouZu`pmcT;ZB3o54d}?htU~!unOK_vP1q1cm2=dYN5JxKCG?(jEbVY zVll-1DXS8fcCKllsqSN$%W3NpB!CN@)lQ+x>|xIG1-r#$tw=d{{+dVEf+e=VPljW_)d%V$l^CQ;Yi)6L`> zq_My&!ZV7hf^%O>dNtPW9oYXwmw&#lp!$^5Qvyj)hPuh~`^{QUt+)}bn>2!$MP(qmhn=+CjWA87#6pyM%Ue|8_kAlVCr8@kcL;5af zcjdmBO@})ze8wjBpF2deBq0*8Ub?MYx6aOd?7-ECcIfF4S4|10Ihu#V&eyGZq9E`8 zdFh1Xzyb3>7%o1V+yb2Du*0da%Bx8+=F?hxPyaJ3GP$#Biy+5&E)50Gn$>DptGi{l z7Qf9ICYPtl`g*qhLI+WqYk#`-JhztiOW!<>*UE)NQ;^BOhkZAE+J_J@D+obzw~FqdVfgTuao zem8c3A>&UlIL4;>b|iKMV@_s{bW;30Nj(^wm!m<3Hx&H5QrOk#v60}p)pSVCt#;O9 ze~Xh@WU3+l-+cTE-m6Y0?V$5k|9B;9Vdr-58tZfdcjvFm z>WU$ym?_7XwS+Y}g0rFU^|&_J@8H30_|j+6yUsrcEFk77BAZ#zKj>Dk&Yed;t7AQ7 zN>cD=Q}5lo*MBxdhfWE3SXR-7H738$E89DDU5M+8*_y{9{>obDKj(8-6N8|$kGB>U z4*s_R8mik?Zlt|wLmm=nhxK0T-zS~C|KXsb_eq{zq*|TuCb07atM$g~)xRF;I%b1CiXY+^0)F*sp!!QACw$y18qo^4-doM4`+}OG#0_eBIS{mfLVmq@6)g zQN;&3oKzS-=^V9*C^>6z9XHDFhsO+%>ZP4m&P^zZIA7nzTj&}~1@E^&nO@+2y}1JL z&iZ%EZo%JTQ{KFJ^GIB!`J(^c$>M_%kvRGT%!PmH&Q)QV;{0(}jr7thm<80mm6h3YhYG-Cz=K%4>0`uY~cswLzTsW3O@MKN` zeeshXpirD*nG$XjqrvGpm~`pVB@yOnb? zyYZhVG`n{(*Pz8~RkvlEp0?9EQE!my#rTDnj(*BmTN1mrFlA|cm+PB!M$|sBb)kNn z20B~nE_EB6-2a8|Y^S|TqkdLK8ZA`U*%4U~c`m2Y&42okzyTTtI^I2ZKRMPZMI`RH zMSd@MIr+Yw$*nK|aOo6YL_h+j2qgaX$1L^9l}7m4mLv|2gKhX||!fcA2cIaxOhA8iIMR*jg|Os!r=V0_0%C>-y7 zaMsUX@5pt4si@P+^W{ff_|0{-)}$YPybu)8%Bn5DhdR+?yr%t7`B0&xXJ*b?%EtH$ zjJ}l?4b}A41g`&~?wE0-*AHx8JmJkeXwA>InNLs5F%&x!PX0FVPDKFjk$sRS*Hsk7 z&9?=V5YGG0_HE>FSnj!Iqg^WZ4e)+} z4-yPuiG%(4hn+y)sB9R?pvuJe=N4WWjUrO>2Vk=$mW|&( zryngWr*d?C3s$g|xwkB4n>xNMu?cy)6Z8k|yvtgcKer zF#@*8`Qa8LYf5r0+^eEzG;P|{Z|M`L@L01b9XLC0S*t;Z%|E`1>cRK>|M7Jma6Rwu z-#0^MwveQhEo3%~L`2FCq0Bf|M$&MsN+q(#DMHFh2oV_#agtHW5lLA^QQGT%Uf(#s z-~WH#kNf-ZIKPooeLvs#xUSc_G+s&_fvgQB{eaK*lOG>AazvTVQP|!)A(Z$71#1$C z7zS}6HGGoKn0Jzlj<0DH$sDWkE_()PXxO<3&F1T?>Hk{dF#fXz+}bQIaJ-F{EOoY< zVkeI^SUNGPVo~UfS_8F5BG_3n08;(|L*kizr>h%!@6&&}ZE>JKNBet_OQdd(5zVAW z0?aUVhVPfcUAv`4ZBI|n2gG~#1r41Y;h-X6_nVTdRFi4?KnWxS*s&MRqmxVo_`A{sayzS@CL`4~!9U0I7X>f~W;2lQ?{&Byl%lF#b zwyjz=Hz4L^xcy>c;XNyu{UJ`zYx>n1&_-393XR5hwOJ{v-(z67`Q4>3M<1A z%~qYb+PYrOw39EpJnZN;b|71%ICIUAn5zxOB;1(fqNZSQ?G{Arqz*Ptygz#B6!&L3 z7CArP^xW_6o~8YFw(%bm!cujeV4uUg4A@Alz?N6bV}v#Phr;98#W}-H1qMDkK(PZu;OlHUBw}2;LY>3p^?GlSTL-T@v!sh%_5%t#Zs!pDfA9-=RR8~dxJg=Gji2s zF9&^XpWPm}_&-s(Se|dSKy~G1k>OBB2S;sUs zJF7f9xi0r@N6LcMb0aAUY0q`nex7uAFQ4|%E1Le4K9Cz+*o z_<1vnOQq0eSiE}#14PbaDS&<1>7!Y6F8w*e-_Tj~BJhGjZqWRGVs?TD|C4N{E|BFQ zt8yAl>At^FoQJzw9{^GSjr+X6RqyWB^bdO-VciG!^=cf7f_NfR(NCW1Vgltxs$+im z$}ZJWox|6zy=oSxvzuUP-IZ_kovJE|ur8|WtN9H6h`mOx=t!52s8Bkl`ZH`1&<#$FSN_Q&hJ5jYW5NsMAy2#4tZPEli%+2Id4J9qfsm^+9DDXO zW}A})phK(i4!e3AIl8}+yhS%`RPc7-pKBR~O)K*tH|($^7E1R2%shjh=p+ccyKHX0 zbZNJ=76W&WJ5z5{{<&2r16C{xQwf_ArdIRT-&1c`T6;DKtZ#gFz9e=ufNmaGnoaj9SxATAs}dUW(;qpp~8hyFG3`BkfL zwq?h5PhgW8w?yaFk%Xgzse=oTlQ|(2A2;^O%mwnrMx>R>lzALKrCwY?W{3NYz8tPh z-}HjDH=65?6I=QYh7`cY3k?n}Px1sdMG{mB9HZlJt6|;l{OZ_a0_qJ7D-XB1gGqf1 zY6KQU^grASedpb+adpL;(!)q!Gt2=~M8J#up3sfhjatTunl$hfsx7!+ zn=cb*A;>+bF93zH@x9OY5{^mYvh+Lv#Ukppun%~)#HqDO*z0c-Y@SV%i&Mkyp8=}z zwx5jkQwWjBTDu}7x#7`xB^mwg=H@p3<0w#Sz?fD2Cn^3mf4?;kLDQM$h{q#arSHv6 zMHdsg7P0Ucak}JX`@dQkc(Rs+{Z(`qM4;1MfJKxV6rxbP!5zr=dP-ga$D2m??q~P3 zXrW@@ha?8`@KcUrNVIp=1EPFwfqQYm3Pp&mpgD-q$J?vcy<ssAhFSUC5Y$TT}{( z$WyKs?KB<%(Zbe{tZ*&=2|~=#@`@ufW^F;y{f2$Sui~5h0#f!kMeXcl@1Q+lR1YNe z=53VOJ9IcJuQsv{X#h1~%EZ$zzW8PR5V>@$QDx_uH$;C#wA8!!xR%YR$wlx$daZ(aj>vn|LxwD9 zw^ey&E1xqkV(_VJY?(WYSnekwUy3VoUHcqf8BVp)dlgiL`L_hu%KkEps4`+>K6=%9 zir_W=_W1mF$K319QvYa7Nr8)r9I5|QQu_M(H8{+xU$s7A)Rw@Q6q$0&a3fp3hehAu z;NW|g7e02%d6TZM65tB|2U9JPY6ug5+_-R>Lzv5RC&eLRkMC;ZHSBz$YgYaIY$1#V z_eicTVrK}j(J(yihs?=m&cb($c<&XlY7@^q;Br=$@b7+hjkqj0ret1c+K&-imFCl= zrJmbTD~G=-QLhQvH|V^1gE!QsHJl0B+dFtA%?6ZgVFd{I$7x$WQ~`Q(E>&>7z2hsoHr z$PAi=XaW@;9+f<+pb?k=1l&X6Q`nBg4S^Vynsj(L`W|1jK=KTEa1Z_fGdfdDyK@29 zgudg}T_>T+h^6?mClLZC$~!#gI=fS6|MR2X4<0-avyKFooqvN?oVslkAB}(d%JG`5 z2!iQU{;rbx)vG-#+rnOht3hhLd=0g?9g0jNtV4=Z>d{<} z<$CZoftj;+{yj;;cLK99PQiZlL4Kt?aErS1wT8)a#rMKN{;&z&ncl@hSKws zYIsb4V{-i~D`O()_rsd>Uz60qK z(p-2i9{FKQ%kfg|$b?}UlF=MfGp@f{{`vK}Oy<|^cB9BmPocTB1RV{FmYx)aQy&dS0l`!ygK34_O+_$;<&o~T( z;Gw&b^&~q=-0%aIT)gQ}ydn2NTE){V{ilat%2=<}ea>6$PG-h6HfOh*cs{~%SGR6^ zQimOs11YFxw{0xR)|g1qeM+`V9Q}Y+DaraqY>LRm-1u0o(Ztu@&k&(Z$6Lb!dge$f%5GMOBwT5VG z=;~S=lR>VOo=AXdu|TU|pMMCi#**`_02H?6+w{X_HBV>)mdkiCn#~~kAh%%`N2Y{$ zb2u`rE%uYkbvG&IvXUCIyy4ot_w9E7`~`$sgPxkYC~p{v$WQrTqzr=Y_C9v;OP=Fx zIff}3HP?aub4ea2=#NyCaIg}=dbxhSx0`$}NgVB{6l23$bUS{2S`RrU0EJ8qNHIcW z@5uW1VvCkM>;qr!HCsf2USDJ~z_E<{Rap7*nI~5Cjt9TCprBDNE^b z(N%#3X?J%z<2@Pl)?rkP@q}6P=XVy8qY&GGq;Zbh^+UPVohw6EU46PMFw(Y}#fwul z`kva~q;_K~osy0FmMj8RFN9o*YNGj=@4Qcb)^ff9C zp7ldf#+oBg*1c!AKn09IqVPJOxs--RWvqs%j(kR_{+?yjlSyA%N>9nFDH2!%KrH6e(v`7~#M zkR%CeYkTU2Bzq!si2FEu+PScaX}xMzRnD1777|%ArUuA(`LQ5LHW%=nC4EL*u> zQxXyq+|=vOeo8oq-%f0VfQK0gqlDiQ*vH1D#c$98awaZ z-9M>VCp`@b?^9onY!6XdWzpR8pQC|>NS|ASe8 zr`U^ISlW1G!M=&FuP&{57E0`?|*nM z@Ag|0V{`Kw-zS{t_;FktYcmhyZAa3os=-{}kxRaITqall*w)h@FAFW0c3P5mtQsYI zy!mdmf?U!%MkR6PPF?lqfDY1nZ+-pkfR_Kr6AWN)+2&@y!)E-eyS2vKuO=oCnvf`q z?1C8)4!VQNH3Jr*7)1VYcg6MbiVGzzebw_c1Kk9Yw{W$;MyDtr2ENR=-lSjd$_Oaop$yoTc z47Ep|wj}-5)4nqnMa&@o!m-c#V*__Be$a^qy~j(*aFzP#&`x)&o74$C=iZrpIekzK zSx29rpY}8hc0WRprz(+FO^^iW^zQBQHI!`*JUW`8T<~?O+UhFLCVzLlErg#~l5n|} z7pyqu~`H?iw1w_!9ao~c2>6}G$94*y+ z#++v(NBoFsk1Envx1GiJ!K>fbfiUY*NS=#JWxjvCpB1h!CRjvvg1-8PQnHHW0Ri97 zvJ{vZxp6Bbu!V8pH6ItAClBVw!N~H_mStbeT^}W193#yxauvnhA{yTscP}Vwug{!% zKW9a0Sy3}R?VHW&TAS@MzNO=xYQgG|#k~D#hxM34SxAodj&;f_oEqpm6^${TvT{po z<4x~;Xj?dn_%a1G}!4E~&>>n!Ud~}<+nHzQiBMDqjVA$@& zw~pDpKzkm_y}Bi7ZdK{@xKB6n#U1CzQO0Nle3Wv8ioQh|cDDPJ+`mFLjema4N${X( z^WCV@SXcXfWkmfazs0Qq+QDbf-p7xt9eg|g$uQArvw6ZCSUq%%!l<08q&SdSDg91s z*hyXuD^F%`Q5*^WQ@Xp_*gK$ zSsf1ibR2ej6Q_1>W$jY3aP8L=3l?}w?8f5TgVJz`1tWE5Rpt9RC?8_GiOyT9taeB7 z64Q4q7(2Q#Y1{LY&Q4YE6-Z=c9NmEIv;;&NcZtQ-)WbT4X>`ARsJkiv-;heeL&w_z z^l||zKP-VqVbwTdm@M`x-~vEZIdbgShg~{FB7lwCbRKinJ9_&jW3QDGq#RlHCRfw{ z1cQb*umAQ>+@Z5yy8qX*cC9FK%~*1xW9<2O+fHf$AMJLJ@H}y0ZG343^T{bFCja={ zqb(%@%-fPF%9>+>R$qBuwO}YUuA|Bm<$aEo`Hn}ZKVd7oF!^B3Z0Y1C8m<*Jm95Ky)8)4b#y-Y(R5R?>K=apT6L{*7b?$qZJ@ z;MQSGJM7&+w+BC>Tx_}PXrg-2JW>x;FMolPI?W{ADB`dT;|^-q9ow?B)Lg2O8FFbe zD(e~8mB~afOyhg^S6t_#?MM{k&V)k*%j0JF!CJ|W=|gxkU?_m2D4hGkucbezj04&IxGGkM0{O6$bSG$ztDLW&BQwcsnL z7QD)#A@*HKJ~V6K;a+(G&Z&OWreoHv3!5>Fb5+LGiBnJ6MA|j1JW(Vn8I~4)>tV}j zV%Hm{{ZyY=r?pbpXao%3xN4#?qS7b~6PuIl{@6D}rReno|LbxH@zQZHh^Tpn2{Hoo zFLl>xd(6;Tl`x%yi3l?uY|JsP89>bs4a5oEgqDdoOilR=AVVW#KYjctY9~BxQujbf z(1oZXaN4sEvjj0wlE#CiH-qA%S)CDU%ut0;g}d?j!4eGAdAW0#Q*mF>9hpvmOh0|l zdL+-8JpBp#tuDl%a(hZsge{N{V0=s@e~l4&4aEh>)z?icOjFc+{TDZpRM~K!Mm1Qp z(6+lsD_Ya*LTlp-(=zr?x4x(C@3aMk-lKr3OP!ciE~4FiW*eMFI3qx?Q_1BS{RuGU z7g4}PkPQeuYvWb}etrq=s=PODO0GTc)P~?PG0Y^kg2y15>*vp(+LadpqDKON{i;bX zSM5UDgu^aiG#zw51>lWm&#bl1#|mxiVu@@?)hY3i!nJ!56p zOqr{kp1#sTik_h8p;I&ORt`%tH^exhwrrBu<`BcGGpCp$G`LZR!B2C4fDCcz8}5By z>jKgcPB{zr zKbV|7T5&}n*;qB$sWg6}N(luzK|F}eLJd+~9n69=7btRghB(}|R`5TB@MPLoIa8~m z-j3Q_vv#qc1Zb-M4d5O6$LoQS4L53OYO2%en4Gi_ZreECjUNLhvta8*6bb&(p?*_Y zZrltwu0imv`>1=67fiSKV)6V3r1c1-nwseCzCh+8#s!?@d?}WjHf=G-OOSYag^~HH z-L>m`3ebJJd|j^%NAG>gsVEl?^iWQgUy!Pb-mWrx4LVNBc4S07TR-kHsObf=YIUkf zI7ACBZv2k&YPn6V(t27*a)z5x`3bu2AT5S_$n=Nw^|zv}024&zz%NWRQ`mCsSTqEq z*RLOldZZ%7mV@4r6B;l@y0+D|nyVPVygZQ6mq1&{GtN!#Ba^m}d3m=vui{LYfy_QU z#kADj%nl@^?cu(iM?7hYmlGtVXqy{0XmF3_YY+52HT9^Wc1vOH*5|(UhHxwOD#%Ui zBwbsARYTkAJbqlcrjUbQ9CE@xEf_b|mAF^Q&Qc)NC@4=-sO4?bP}abzC=(!~T>!Ya zZZ9l0WD=gV1__(&YET-;>4QebFKadplmncfRv3J2dkT+)V|^yEthUjW?X19GpoSgc zpsvM>sT5n7!>VqzP-{G)s?BlteN~a2E&6&iot%EU(89g()Hu@EWNp8)tluB^rXU5mJd0OI#eF@$8L#C9R&Z~;v&#xxOL}h8V(&GZKlSX{p zu+QQWW;_Ic(BeCHU1ZQ0ArQw4Vm4`CXvRid+4vk_LZtJi3}aSc3S-B*Q^vY)@l4Lf z`LtOPG;*-~6|<_YifI5Zs9{q;v`bmss*TWk4J7g5v&2Tq9r;i0R%$M5}Dl+X*O98QBsYFL_f4)nRlT+Fnpo^mJ1YvZeL>t|e;7RvAmEI5Q7CzhoaeKN*6 z#6DQPqE1K#k5yh_%b|>X%Q}+tze6!SgVL@$+%n-eZ8=JOOJS;>UYIqAIiGd?v6~80 z#-+v4=!}?c)^!tF0X9NhK~wi2(_c}WRS)$tp;wd|wUscrYeBg~-j>6A_iiyTQn;&CeJq%seFLBzoCyan87E;=(gLv-A@fkh#PzFb?zY+EU{SD)@ousA-enU zrAckDigw*=y1duM7daUJkUW}Lj%VCT!#)w}=GaCZHDFRfhHkydd7*YOEtg3R4?C6x z17?pn^TPB9>7BuERstzOz^{oFJGL*mXuSmcK}b-1R8q?RB<>{uj~5(9KTS>E&O9;4 z|J(HvXChrM&7q>}wK_muP@d7LYV+d{&6f#zgN!FCJ{(4kIEhxfKY4k0s0{kpuWchS z%Xxc$Shqvn*WcfV&#=@fw{6Fc-n@0WoB|h+ST0vL;)|UXLaxw{DXRZ@LP*_8tC-kt zg9}9=Ncu@pyao+~-YWHBK7H_36*c_Mzj;^0( zz{e+cVK(@e5Ypwf!Y<|(wX=5Gs3hCIEu9Rs zefQV8`*~pc*a?j?$`uu6z);KjmFwRrmU`=K8iX;u)=Old-mFnxLmMqUUAu<(xfflT z*2@yt+&7YKUf$etIjUj2F4ZxCn;hD~6l zkvRWwBv}33W<8~kzf+z^!h%a@Jlj7!;OevhIvq_N4ujSO79Z$nx<2 z!H$@Rk^nqlA0rr5EFP~d{yx9MkTu4ZpqOey*M3$HW)y*EdwT?}AQ*DKW*591w$am*<355(}-hq7j5um605$x?dTv_9aV6a(F*2&&wyDM{`>!$$eO6%dgg6T|()CHys zJshIK8+L!9`ZKPt=*XeaHQ0wqd2|>D5g@M;9s&v>$_2F2+PNUCC&2=0``}A>pcD(QIzYk2N5E_?_Ud?IKsdOvs)zCqg9gB)oJJS)WfqjjatmvD0NI; z9v^j6gi(vb7a)&ipk~xdjCisu89qh_mZRra-AT`2K_ZSV*;1%%k z+r1$wO|RAaG(B+)Kef8D;-A$4szZLKW(IjcSQV#LRm_c3dRC2-_(D2|4fx$8^*%V` z#gYLI6$Qx*PV8gbPs4ErHhk%Lskz^?tbGgyaxQU%!|nCqC#!aYqZv}y)*h4h z>f;FM%enZDBstnkE-gH;Q7R{$iqmB*su9I z#HFAI4DrkCh=LER27gEYVqv)ny*4v@)h7o8o{7nbK5%q@-|69gPJLDMhECq5+W((1 z8fFE>>uO2&>i#*c<^3yP2P#G?#v`LI{%qMUXGn$lJ`7`HJYVYkFe$3)xka;nEyL2! zT~d}0{Ba1$?fZJ*+>?9l%cSc>H9Ka-fjMo&`iZVL%h|6C2ADZ9O|g{`TE|WuH?Z4A zE5t!d5Mh$-!ync$Ud37muILIviIji002o`Ds-gEfZaF7hF|5{q%n4uCKI&NJffxp3 z15(=JgfccuMeEkBV@7N7E`mnT?8=$PC+3ni9*SA?mm)DTdI5cAvGpcXint@QHe~FA zwTC2Pyvid&%iVPbiZbYEg1x!+U?`Y%dE)p^ujD(^OJjMjwSyorz&Yn$PP@c0YI zT_xHbcu3fi;2uN}Aj#)2-QBI7O5H2e)9<+_Y>aeuRfK9Txy=+ky(55m##vZ0%rEO3iW*@6@Ih!6@HV8{DXM=}ovR*o#P$WbYBfG5hjc<= zn!0?hIj1Sw60MSo4#cx@T;@4;x2VkwtYT`@`jXZ!o6h>Sqd}f7?+lZ3rZ zyK21p9M=8stY+Tz^^#90f?FJ$Z~y*%&`!B565l(|ndEIdEp~5HDrt`qO1*Z39hjIm zgpsXmfv&;5m^?$lW4>-8 zF50kS%DtgoC5nM6Bnod}OA&t5{aSP>He-Z^#hwLQdrPGMsS}3m?5M^!bFSmPa+&+Q z5p2E;AG)zIKQJMM{zX!G+gb9%WZaw~qtHzMz=|dG8RmTM2Z*Kg+eL5pCcbsZXbh!W zEiL__Xbc89BD#kkk$yp5$|Ty>rs#)Bf|)?}sMrE)CjPQR?`d>#Zse{$cZx@T zl|Xvv9W5jO78ZBKprmc?w=a=3>Be+ti$L&z>A!qAWbydlPznRirRnwClrv<>fFma4 z^=s~@sezHp05fojrIKI~>pB&G?;76Kg~MN5{~^gef}5heMsffClb&5O`djEbDx)p2 z89&LsVVIBmmxFEPX- z!;9N@w(?)kPBua#lyVQ!o*6e@01PE=+LYW^SG0*~Dp#9@+^li?fyWufx%fOt;vw>j zNS&&E;G5JM`pGohGJSpXUsW?Npv}QnVG|i1S*SE_sdv5hD^edAoGFT{*J97t0KIX4 z=QQ)a(RcqZcI(K1?dtWnq;{73AZT@~^&gW-jA(5q|mJta|T;UJgQo`DcsN^v%gyKRQ816Ae zjsZYU2n4Ny>DC+T3VYoRHQAm%Op&N=fv=aeM~g{cpg~&9UlZ$*)6e6*=2q798uPBY z7WqjLS1SPz4}OtiBF^5eEZs_-4g&WK?C#DW_H%&JRN7z4$_F-~#a_gbA-}FGQa9qr zR4i5@t*rLOc?0xODF1QkQ_DIvS#gH#*YaDJ++JZZ9n_k$AG6+jr8$nb{Ti+xID)-N z?fn45g>wwX+g+T7`Btq|^i=8O(C(hZ3v~W;V4w|t)V%Hd636q4BCF=)-CBaG1mt~h zAK`%;Hm$0Z_+jxzFj`$+0qWDIPOS@Va<~o7`L9<-yuDI=^VsxP_5M%iyIJf)VuY^{ z;!}~ysS$7>lX_odoTmaAXN6RN5KthWXIGeSDLf%s@ro5cD~==U2EJ*}F)-H5Z275e z#>Q2hnIg*QRg&j-yjYRrJnJSJQxe+AbiqWg5-X6b3)5*oAW_vwBIXE$h<2f))&)A+ zY#4jx--ZI>`a4(8qc%Yebk47zpAoM-I5qY1UqM5jZP)30WmHS6FGpgS$tq7gaBSx~ zHm4Pd?sn^^S$Er-l)f?VPw1=~eeFhFy0pwIE!1u0r8=8CRepD44XktAMRz60!}?pd zC;G@#LLSqkGqD4VVK6teeyP5+_O zW&e%_RR`h1*c1D0{eu%0?>0e_&K>C{S%v}}6a8-(yc;fP=MV&6vmgdFu;CEh> z$=Rj|U+of5Pk2J;F0O@Tsq?tBT27%ECK5|#4EVQyU-LN2T5RV)vew+pSAW6B0>{wB z32GEt#a^7K^AV~|urX|;QrC@Q`W?H_#)!;Nqq0{CNB*vLmJ2lR50qe()NRUGt})>3 zis97p_u<|xX3Wqjl#x&P+Z0W*oIPqxuxI5r;?7|UB-1xLLN@x0?p-GwiOMwz zenz@Jgt6(xHu|6OGRML&GB)7(Ll*z)!5wDYTHg!6q>B`XW`7NBCEp28&QnH&xXVyo zJD3<+nUN$21Ns{QvDHU~u$lZX^o-jh1xM&Uple!|PfmWEv^so)s7-?%O^7fOn z_jfo$)CUszwcHXTF3-VK1h&z-jH;mcLm$^}Wt)nKN`Fs1Y-5BLo*9~Fy*x74oxAVR z>w5Qhd4?r3H0&G`$IUajrE}Hu`NLmmzbV$@f4k$f21jeVMSnjNKt4)- z$?_#WE^tEoS6z=UTD;g6^v%}_4-8oh4=0?YPzd7DR;%RvKZfM28jSq(|Jn)UOPLg! z`m@;Y1rmOVyx{?(M9O!?#>U!muKU3R?({`Cm(fq zR3*S=Ux)AWezRZCUffXFlE-}2|tTOqxw!f zupFl8KOD-{YzL_7(qKvjz2`+uN(ey$m|=M8EkaUm&+XYsyL*D=>;S-?GQBKk;R4bl z=`((H#Q5ig6shhAAO~WZozrJjcf0yBKM`C`GkCO#i4Av_XnVR%tq_XX-?cLQRBOOw z3jdMk3L+0O)BYCh)cA{fRFe5}<_&UbKh_Im`KQ|I`D?~>mc!p}P0Nw$ zIf>NvoT+D!y1BI(cW=;Eo^oJ~@6!?n8wc(8(f`P?<{Q5aF31I-ybJN;PZFkMj{;Ix zA2g`QL+109XHAdXVp;Ne`GoE0{U~MT4Lmx3;8AIT4?dpyDWCBi*t1jx`kz1FJAX8G zOm6BH^i-PFHRTwe75f_T@y73e`~gurVft=_W>%HoID%4cSYZp2?m*W}(LnfoWBSwE z&omi78*z|R&=U8#9g&y+YUs4iqjt>W8ct0)%wejU$^yFt!7HLJFRiX!`OIsJ+I;~( z>F4p^t?JwplitC~8jl>g#Vc4XzWNeC^FV6ere3Em(z8(dGsgi6tXjIv%#P^XUaB)- zQsFkgO%}vQu6Zg8=>RTmbm-y%w?ChJ#cBq?vYKohZx>k{-AA$y5jz>j-$TA8AY!Ny z%b%TTB2sF3P3Z(In(E(hwX5~E2kG@6mSkjPpqP$YG(h!VT;MFbj;{H;*GZb7d86n=zMZPcLI?hO?OHB$8F|NQExk5NwME-b@6}qy>wJ4fQrkG z`5h+Z0ii<0o!-B@g@ox1$yyN73iGDv8^Xk^J;wdC&iT6)Y1P`yIuQ9Jl z;C1rr#p?oZC~eAjPOhgF@Nt=qWgSQD0Z-m;Q#{@H?IY24^!OZ< zKohKQjn!MU!OP{VE>;)0cZ5a%2faYw`S-UicASV@45md?R6Yh5rK-TY+uy(~6^aHLF%6`WLF%2)rV>|y-Yc#rD7 zwg3;$gBJ3&0&=J9X}-4oyXJMz{l_Ka_`dYHNS=`V zptx6Q9pk;wBj`uVmMzuACa^NC@PF9^8XtII7xtckC0miiCW5u>6n{|n{b%p_9sRtN zy@ZL8fAw*$6{%wzVxXoReDn?agJhuoh|G2<`1ZOD2lWoiU_zhk`m!G=H-!NW-T*~R zhO6CyDsL=l1YUg0cd>e^$2SC9O9F3GMuQ`-m{d(+2Hz<~px zdA?j3^2}MpGXS)HCFK*4UgCupZnt{Xi(2i=Y=k0!7&g#mn@66gKjS5zw2ctORlOm# zU@c6xPV7LQcR(zlE+AI5nQUeLppUsG2Lnw|{GNFsf538F3zH+$Shd5`YI9@9-6N}0 zu4p7671o_tn-Vwl?E1>Nm2W|C0+eK=4A;@!-z_dRr?k`3;;I-tfjIe2-7(HiLPgfB zxoqZ9>j5eJK|%FN3Y&Ir>T*;kcxJHD64!KN$iZhzNFE!$V1Z`Hy~GDX$N)-m#`lR(sZ}{M82bYo3SaBN#9`~QwgEGep<4H6iF+2|9-=8zaI!pEZWmRd z5=Zs3Yrstm=`Y7)sn@!1|FlB%R=+_5K!&=$%MbAR)ExZD{j_6E*zd>QV z^Br>`*0X7!efPe6m^Bip|CvrJhjWX2j-2{PTWuDJr9>TfYU+}xMszF~hjmCJ7zaT0 z+PHlaQczdyM5gNaS7bLPf-+OH_0h=2M*espw{qT#sU3uKfNHkEv(oeiooei+*h!-v_<~wTvEss4Gs~-(tDI^AVXLY&ZxH-{jdt zX`OXau-0(g$S4%tUxpa~p-~vpO70M{802vlf2~pFkEU_=dsFblaFowV)mx3Bk`a9? zo{HQ*%&OBy=hGKaVO9&=C}ewQO{quYq7bmRZsLWl15Ybc!WWJ+F}^nsMYM;Oyepg)0At~k)C&qBA|JM5(Uv^wo9l?UN zbLa4$XWF%Ht#tBPeF$Lwx%J*=ehnl^HBND2$jff1Kr|F|J1bDyOBN;HKj0&&P4mzj z%eb{uhQu*R_y=hJ5g5KLs2{+U)p{Sy4vp=qrPY}rd>E;Wik-%ZI^5afY@sbwPD-jA z6e^B*w(wJ~gU#oM%;i^tjtV$?cATYDch(y}w7OGiu~)cbOmUo(_j8M-Fd%ZMveAqF zY?PLAO!lwpf|9FGWO+Yv#g8-j6H(e0a{~G~VtgI#@3@F$SAq$AtRo`3M{3us?9Y_`7lsU; znK}Gr$14A9_n?LCUJvXL?2&ipr6SW>udZ_EW*er+f48_2)+bW=jOQ{xCw2T@SqGc# zSZ!(^c{r0=L_{E=Auqms-|Js~>aC{;X6N&aka#p-2Av_Dbh{}HRi#G)8Ah?Pm9Q1G z{d7uMPHGrp;Q^{mx+mZe!Rpox#pxuwL}YQnGUE#UYLLAE+jfT4j(LTOt>|^R_r}}S z;U|qO2=60ttD4tJQOhw(+47}=Ts=KPD0Yz{K>rq#>i|Vci@J>`HE=~=zM1kkKuT@_ zw}hgaHONitOUw2%H_iWOICPiZ>fd_vFzQB0B?~n6eyj-N_Bd+)CN1dPwQJYjewwD! z5@qgYVkVDP&jU&8Vla|8ehH!iUNg-COAzFmzMap~xYry+tCFf~`UfejOVaM#Rv`O_ zGLhQy%xvaz(vlglZg}f2tjq&nP(>;1?&fA@QQThLeiu4+JJpVnvo*8lfz9zABE1D+ z`r8w&M&_|!hG4sEyVMMJa+v=2UJCfwv)V?mH_`Fu=RbL4)}y*;SWr_w@iXvbt#cgd z7p5?Z4vj<`s(aF*Rse6d{2MV4&us@F*}V^IaJN9sNxTe#2)`18i69s-Pao`8Vne$R zb(bAsXt<}tzYXs6GB3tYK^T&Ygg7>_x%eIsJmkIFfQxIMWY4%elvdaTExjo@O(E93 zW}cL}ZhRrW(7K3A<{j#V{fXFLc2P!oB#(Blg_lW22}5rf+QJ_lLcK{$-omj;4*F{4 zIU6GPmcKp11ir7!7E-VMFQhB+`@}ZGF7-((s_3Q?HalbQ{bz$l{5st=W_GJHqFH8> z5puKUg$_xbvHJ9EMP>hT*&N&5L5(+NWsoo^G_L6%)mBy4+gF6h0p5MDbcn-C!a%Z| z-+ug?E|5_|)$l#{D7h)s^c7pP?%p`fYiACReGl2pWJgIsvz&WrUQNiw>7P3AqZ8Ns z&QVc_`cv+3mzo_~d6vtJ=1-GZQ-;?9r=9z-i}aKUyHSg}R_fC+)kF2xAH!rLq z&&7J|cznoI``QH0Rc8)W?aE?YrYUHSsyEBN{y(m1O}V`(dR?77$}S^;g{LiH?B7Nk)Yr;G^_h5_QVBrI&b~lBUm1r(5xHavo=D zdJly|mQyEVL?tZv?Zo<+-saGfJomI=7FR5pU4?}vgO9PA%Kg>S-a6_R$Nw+B6qd@k zEPfsNSHJ}1fL~Mkb172|=2$jlzMGUWTw8}N4Td4GO)xBL#M1FC#TX#NiY9jtIKYHV z8)R+F;wWlI1J;o=r-u8-B^BROlYw(_RaNvc4`8ttn$_t*yalIFt1^V6_qetfnk8O( z(mHvb`JLRMCqR}b3>bZUgVP}~8uD6jvYFbT427y^%l#v%Yf^*PD{#v=)1CG2rB2jc z=aLE+_$M*ro)s{d5y(jqQPzy3006n&LbB%OV+?L`}puaAvN zb&nD0$FLK60Gkwv`^;43(?*f>=lAd57gyeb)=~Mr_A4JM1p3w?({nzXpF%;LWOst; zlZC6Q(qp2Bc(_|#qa^(LPgp~`=%2&h0*Q*F)8yo@Mz-&o|4$K(tAAE>TFcm=) z4w-%3EMB0+kbEtLhcIfw&0`Z6BV8s~3mwRc=j&WgUgVcNK6}C6>8m7k+0e7GxJc<0 zfaVV%UI<%|=2~SSKH5S|e&0p7J%9e}wOav_-V(>D{;GzB9TxWid~Gu=e|Vh2LYQC5*%ejn|5d611=U6I_75QQlJ#o6r7#^q~EsygSMo zj;(3yi$91Yxi&Ce~@(AusnMo|tk(CIYW(JT)m}kT2=A;$Xx>)9}Ky*JB%a z=*&Q@C;8(PXPYv0f!sS_583|t^Ji_p5HMn}cHh+THEW1l2+6U-+)jGqNkykiAQ8n) z!QiHb_opAfZ=m}5Fq%LS{&B_nI?{oU#YXy@cWZ_|GOlzF+#^4 zS^lBVtjrEEo~QLv_7oK_8)uw&`Szg>V3rXV5q1H1BTl}tX`npix~Io43`=pjWvMvg zpCs-ZyVkSuh5QWM4c_cMGs~j{C6GMXDrm{ieKaV1%)1XBB<(;nIc(p>L#NZ^iE9f@ zJ5WoFcf2>SljE7iqMrny>9r<665#aa{2PFx$UqrYo}XbCpPsFOUJ*iesR)%SI6fuo z=UEHxFsGt78xVfihv-YVfwPG3yN3h7rHRzCc4HX@l)|_N1S$zLc)B9HL9Z1jY8>}0 z;B^U8xV(EkqaX!wwYPYr4J;2t=Nlgm&P1{iRfMrUPpr2708vj%-qtN&p}oPdPB-CNfxXVY(kB1;-G z-Cb%Mc?D_RjOUAn60z;x55MDy=`d_FqnAD5ooF%9h9i9SV}BbF|d*I~t_RE)YNNr^OwLi-u0X zXgf+AT4PP(=kNvvaJk|qxi;c(Bv0sJ3$I$Bfs)M3g7r<@>+4Ix;+922&U~cfoDu1A z`&4UUFSW)WP#vtQ0P?%sDim}Zzpf^)Kp@=(uD5@mM}>N-ja>@P&5gV1axJp# zhnRJ{=8uPpJz zF-nP z9EvF}8>BY;!QH!W`L(>OT$!kBs{K{lLLw^_VS^p|Bb6jVPjk{U`G{4wsn?e=MZa^b zT8OaWn{c|uQ%W5(=4})I<9$q)?Up_2^(FTe zogSd6bBsjOB1q<)joh)8_ta8H+zsFW-GdFP8I1@tk0XPsgK>#EP#oJ(C)tX0T<*}g zu5*;Oo)~?n6S51+{Cd>5+5HC!P_AZh#D15%&T(PGk(GH?036toa8wPoi>Vj|%;c}J zQBm*y+VFOdu593i98Z|WE7kwoOv@KIL|PGer;vhbJn^odsR%n<-J9jTJoMlCWGS4c z=&*ExD?X1gV$2V}JNkXFkKGAf?+K#AVd+rSP$g-+g@uX~CV<)6);=rW+|*;v2B=k2 zG1nloxd+rLW^`QZBS(+ko@PteBl-!FL7_-Otl8hFpJrAq8`lf}?5c#nO-J zA!mb=Xb`~XlS@wzJ%>e@TRlC0=vjh8o{#S%nV|i&P7X&5CPD)OpEL^)jm^hm6`xmn zAEx;`*$m_D;!AQzC8kK`4%#dFmr^e}PGyO*`KLPH%PnT-r|C>9Cg3)5tR2{$64NQn zJP-kS3hcNwHbZ6EFEwhf$;VqwyLG9s)Zj+eo%psYVZ-gT`kt9Q!0L;Gw`jkyE{$B$ z5t-d^%!HoK;Y?4}p3Pwdg85@VQ%vg~JCZGb@D-mC)XxvpNprMt#qoTkm`^Rl3!6m@8sxl&u z)SjU23g*f>LPwFdA}K+lY@X7Kz88*huOdRhf+)z{`D6hSDT`*&v;A6%r~zfm3b=A< z;zTE$FasZfh%^XV5Fw^j0H57_xNP4esSQQ5zlzdZ>H*EL4QbA8NUu>$k_j|xR)^`~ zjqB1QAx1 zhE%;@Y_8#GNBtGWX{QzT`STK)uuF|D`F77GuZA8L!6b7M;Z9LvNRTIG2bZU)i0if| z?nL}g+;1Eq1)IzmDgapRO>EOW{QAPwa!_R+!hzTmsi2$|Do2AXq*aiP27hqEAr$R! zUxF=hNYA93WhjvkV~rHU!6CEJNctllXeL7HAF-WeCV*upL#M%vBIs-$nzsfnL~7k` zvj3&SrBbUQ9dZIAlk2MJsm*b(l|M9Ey|-7|Mo=_$IsUuD;J>Ms-k0RPHRIO zIoXdjuA@_<{L+}70Rs{$PjfI9GY&zkMOzTT&{gU8OY5CdqHQDH1(Gy+5oA$`69_EE z$l-M4HyjM*RFcHAYFoG>)Vbm#6YYopdnC{JbiXGZ4mAxVu?SKWsTF}NQPQhha_YAq z!U|HUdoH%u3Rp|DMU7TGMcq{#hTdI??p6E_S`@&8I* z;&Q>v2}wGdf+FVuzfCV`5vR$TpaVDkxQ6gSqQmOb&M~`?)KvBvEeaYL$=S&Co8&6F zX`Hfavl1JOJAHYNWpk1TkuhUXVa8Y7m^)p=mc9_lDSUvX&#qm9RI0RS@uI#d6BrrD z2G`FFyZFUDam0h8j3(2(rSkShUxLDEQ>2(9S@(VafpW8|=^rwh9Dj(x_c&qRk)~Vl zOAgWRwBvdd%V=E=XI91uYEPIX_IgnJ3DXf-5`m>rQl(S_!Wp}^`kf8MK-?m$T`e&# zxe@uhzDCymHzy08S4I)AWJNR|mm5z|fh|((eqEKlx*u$F&i?W0gp%UyQN}HgiP|x@ z!`AwGAq^GFcGaZ+h+K~B=ryRv?oA0LnGB4yy7AfcK`P0h{!SClT^nzw_K{sv?Q zex{P+57HK9zLXz+(yD#?hJuD5ZG5GDUb;WMg;0Oow$SfFY( z*K@V0Ovn=lpW`9~Fb`-Vhz-u(kV-5N`5MN+OTRoGJzz5gLIOl~iKSEVVJ#C$H)svn zX9O&X@&%Z_Pw?uAV=`csc)Vy=46-A<+@NB{DwHcAY%BAUUt?7C7Kw2BYiIPw0_aLa z-aZaq-(PKw4Y`HsB3#Z3*~4BbzrchA&c>Ey`~p#62a42=39o*;k^FyY%N2JVcgE*0 zss1eh)3x$7RBb@EH>$+X|E<-H%Vx35#~J0q9v80==!k_qkwXfV?9FxKEjz)}Dm7Ns zMEtDW6%Ec{m^bQUVEP{^wkJ=X#8bVKK$Xt=`Vo7Yvw9^~QIhl#@SGWHsG(6y6jifl z&#tZA#TRolr?v@~kaQpp@*~$}O{v@!IFt=5AzDD+-});yEa*~d?#XtLoT}7O+B4_M zz&wNt3N`Ykfzrm&Rdfrc(XgS&nw_xgb4`@L(kV&#a|pR901LyEG+ts^7icYRb#=Kg zYSb9#^Dw27oMU5zWCeS~H7}3N)bxsEFP2R` z^rD;U{O+fJsdH72jdXB`wvlsI*{W_*ZzsP^=F;?C-V|m(z-oK4>zZ8*!z#;-^pS(; zW%D}@+T`8U!o#;5%jn@JCtrL)btTrM9y-|{+?GJJD(%{Jo7_*st{VoW?}Yn}wS!b1Nfe7hSaR<@ ziXHJCr(TBq-(x5%j zFPuRk#YXQ!kH<0nA-k1I-D9to%Oa)I&R0-fn*sie ztP?JRer_9)c8E!pRJ{XhHkTjsZ0o^vRo6z4#}vCy2^_^s{Otta9cCv<@I& zmUl6nF`#KbUI5@XEqy#Z?om&wv}vQQ zSiI9N&Tc^TWFa({7Yg{%vL0Lh$JB6cO84@QgCf&wT{1S#Oh)j#nrPBcBb%uDjXD|5 z{&(^Ixxsbfw*QI^Z;&FPlJ!^jj;c7Yd-vV4`qenbmZyXaQ*%XC{z78~pJ;-C@%yXA z{^|Gr_ycVifmH+{~2d&iV3yU?Bb+Ef2+>+?p-Uq7; ztT{O(XTMR~zBM4typQc3Ye*?UTGCxag%!Js$i3i0{!rCsiuNyv@zdm7j1G&Q9E&S* zj8BpKkNMNVGzk)mH;Va(X2E{moN3ZAhqrE6gDCD;`ve25F#&-m7&;G*OKeYmyKI~h zV!H^l`c@R$G;*RW3ZgDnTsm(~*y?-M_3g|ncOUnO^g=oE@IW`3DUiVCWdJHwxm6XB zi-ydd)U3`|-RbGGmwppFP-&Gxn@&4DOjYpD|5ge1RZhQmc(yYOrGTJe4cCj83M?a_IRA=*Z|loith8Ry9q#luPjWdV$?oWfOdAcm!B{%yIv2E}32x##xjFk$ zmpOk}cen0c%SRmajNk)X>H+4l6||$e5RVVdcS$8IxQ;uE*{L|)4cxuWGEDTccTZRb zXEroh$H7c|Fn1S!h+9crvEgBx_^5hdXnql#{Qjw_sYv?x;S85?LsRr2<_zf@EZEOGT?wquu-Dp|+1^zMkdGnHRX6wrH+x~9lAYqUxB`9RBq;9&D zk{#KMwCEJVH4Vy6|!9-=t-vnTJj@&U^6=wO`9v zZRCY@GRtYNIeIA{q&@$7a|3`U4{xgoZ{}dT@y|7ikk4l&mo5>YuQmg$0u{e&} z(P)o=2z9s0|0(2u8P>K3#F{8{N)x7XNpnwhE!HyJ@L}Y4Y!~s7P$|I^|Mzi&)p}qSeA{h$u{0!(C$& z7yQ3ZgYdLk24S3?;zjk{bgCwtwhEH2484#=XBn#7W7_%6grOx;59uy_eri#XYWm6v z?a`uObn2o>NaGY^Zz2VSnWGnpeK2b=&O4d8PGjNpej^&~N-Sdc8O;&~ET;m|*U;hP zO$Ud$XIUeZe3P!}v*en(7T2cZj{XI;)qB;?!1HyhiOX+T)Le0GqH)5=n)BAn!D^Se zOF?-gs<*uVVERm1y*i)g^a_c7Ig5Lo{&($**P{c*n=QGUFbnQdVM;jKoE}0Fhn331 zgO*Br{`E=%ID=M!#G+7+f(}=!I801U3lcV$KnI&h zXjC)?fnIV?1P^umse@EW=il*ZzZWwK1|vlXqhgeY()HJKa_!x=Z4=#t;zZsy?OfyG zYZS{`!>%XQ_I;$$y$h4<^h9>N>WPku!E+J8(Jqd)<7dkxf#QEg7*Fs+0w+3kDq#u^ z*bXedBz*s5vr{g4x7h7|!~pRqmak*_8!auH5~9Vw+6+GcBI9&QCLeuCuMmj~6BY)p zosEpGSshGB^Oz&x68ShMF}MXq0OW7|cF!&B*8duhgHzn8R5&$!}V&$=$x&9c&x5<57{LLJFBUspS^+@00s;-l>>^H88*9DecGn?mcr< zU(6dc+)0%D-j5btna?qz5Jn=57Pr~Zby=Dc(5_);$7g7tdrR(+U@E6AXMVCzLYTr4g4p5ODUpRNeXfbRO-ZxA zO5HXvnCR2S*-6;0xTPdjLRm?s*}z{%yTEHoC`)zUw+?9;rRu1!ELqj@2YM5|#m3_>6{4 zo5qR^K$hTM6_q4!L>iP0RL~5!ElaUwQT&?XfgjMmVyK2A2}GB0=y8_Lwv<~|g2KoS z3Z2G~D7p{-O`G4a){*S(1Iu$JDB*#Nve~Zz24HG?J4z>q6W@hQ1R$^Bp zxma*UDh)^3NpNQqWOpy??Gm+jQjXYZ!$fQOw;Zzx(ID~DueeUkrJTMTZuU(bCpsGB zC1?KmpxS9G4nWCXd3Am`0JZ&n_Uv(imov#M;+<7eHRogOEB?Zf&Kgz+sSwB*)P(jF z0YP(5?_W2`c025dM45olv8sc^o-vn3?+Xx_N*9IK&nqzLD7goeuYGYFv&=b9Y4gvo zy6+=|<@-M1&f`pexUaN&T)8saT!!7@ON2K5a`GoaU55yLgPu*dfaXjwhr!nAcpRi6 zYItKF#0}3EdgVre6I99$Vz%~C3Y&n93L%b1LI!D=OQjy8fqxsc?zak;qqr>6BE-0Z zi@chz#iO=QeD*a41?BIJJT=icOhA61Lbzl=qz1zUJ{7I8odfrfd_SlQcXn>ce*IGn zJ?TW$yd)iksAZFQ6xKUS(}0sc&Hu-ir+)l=Lurb*846o!vR+{;Sw1@FwUE6^61{fk zmKEIX8d>b2{bEl!CVe9w=kDp}tn;ntQ;%tZKcug5DoIS>e3!u8lN+5b`K|gwn`Dng z-Jw>$Z*;`MqO?<|nC?n)a&%iJ$`kG7F)*(%^K1L^^{8Jv{o(qCekb{*eczG#@3rbq ztm;+!s|e|cN3b`;kXszo`^EGc@xy2*^zqrb;{d#0hj+LN=K`Jpu=I7Zf%%iMIJy5u z+#(1L$5?+Fox;IHnIFtKP#H>ZqxT`>=~@pakw0=f5(@sW!}MapPqt`YwI1(`p$5*Q z&LoYmymB>l8`j&HQ)~ox&#$O2gcE5hon_}ROXtMwrtLI9bBV*tt#}^W|nV@K)XXlS)c0R*gH9463WD-IJv&oeY^Hu6aCwpl`0;ahd2&uRmhcdMKlEwiD(*AuYB z!5Cr{;?j}XKX_{}^qNco{GczyG0RY(y)-$R;x)mWb7NM|zScvSfXRL8=XNls=p5^l+Ty@)~N&W`5Kex_ZN}${Ib<5Z*cUOJ&r;v zKG>pho@_n2%Pz5m_%zMw=ubz9|4D(JyYvTN&xz%{=jXNBgD@1|xhZV6v{N9}rcRU4 z7fLELgGWSa3E^J~NF^hI%a=^ogJ-YGEDUhHsZ1Cv?b{DMZ%ow3H73|wp75exxMa0? z^2InJvhH!``?f_M7g;pV!Fqaj#Emy7Zun{l0>oA%qSrdsyjAtrWzXXEQirOMn249=Yda%7_4;d(ZlQL zd~r5bBxm?(f4Ll^EQYPxf_|ugmbeG$1Ty%Gpt4CF8hZZVoST3B;MFUIN85ECxM(bME{9 zNVE2dFS`=A(Uca0aSoLYF;P5}?1+rsiK=*B-%6`gr=WNV<%6*ES_g4vvamNHQ#Wn+ z;L3AGM!r?VPbSv^+lU&*Q7bnlHbm}Q^Vak*8!MdZK5>&=*Ahnxa z1aeUAQ1VXx@%mqvg5T%TQ3S4?&HS614gU5+@C(G}` zynOTDW80)se)iO>EY*wH3iG4p<5nPLy!;zQ0viXMp^f}m($}h5EeoeCwd3tm;@kdeFrMG)udcx#=hfhZb zh1(#H_~9TIjSlxkli_D%zbn3B3xB)RgL)=U60kv~1MWbsRNBn{DmnIa@EEGgZ)H%Q z!Y(Gl@R4rW()ISlP{}SJWB^nos`_rc(xaw~pE0A&wtPrN_-Or!JPoJ|=e_J=#>)nBta8V=_o1L0GkVvo!o?tpUNKSU&fgHwtQt` zT4!xtMdb(~o}|i8oz>GDhhR+gcIW@0YG$&r$bBi|NLKH+o(g-ELUosDjD7pSRD?C= zpDLlsC@0B}9C>>VX7rKF6JT?LiQ5Aj(sG**9g*K;Bhx81ZbnpU|9^n6qwtjs`i9O? z+=3g zToQH!(DgRvMD-}w$j#p!+ zwv))D8huU-ZSr)u!3zsxxt(ambffWf3YpS8q8C|MuP_-4*u^tot?udHtL}^UH(I<#>ix4 zzDCMs)C3GI&}uDNp32MrP6PBdnN47NF50`;Q>tBK-C+$HB(}YN( zMq2Mj_97$f1RclUX;8!rE@Bd1HI5( zWwm#_``^ivCrCECq#nsy(wybVle0B})=eFdh^*xM0qjC$O1umI7V#FYEa9Q1qUg{x z8Y@F635LL5+jCoL;O;|y*V?$O@F$|P=ea}k_`Z)J9~hwQfJ2d&K^=sw6Wf7}m*}W! zr7caC5FWgEYNAMuGIV;ZkCcDR`O)CtT6|77%Xu|PkTdB(HqnD;xK{sid|!FF8gvG| z;bFZlJ2crDyIY~2KVYZ+Y<>;-0P4638e8CYCvwR>d{8>Q9CBimx>ruWEj6lwf3S3f zKL_I{lsDYw_}qP834jt9pNjODmCkw|w$B~bv^_)oSCFL-6tU~54>iBK%;?^nqog_xAFf{@}yHZ>jfxir}Bd z3%S-Tr12`R9=)TNW{rJz)#IN}E74jqjn=zjmwu7hNFdUm0_{;tb`*FpwTeQ!5!RbS z3HQ)(d^o*(b7h8U77NemCaA^r>KnqYGeTf?I7&fekm4)-a9`UC3&oLEB}k-2x2xGf zr#8|Kp{e_)28!DRgSpae@4ipKitO9wB$Z>vqSL?s)Y~>v*~>)Gr2p^|Ll{%IKr+f# z(jx_JobFwkuY=ON>XLF+*OgIUKeWGJ+Or%$o1{?+b*4hwt*GVm)zKF3my%P{AG0A)oQhM+YR!!+`BPh^vAsg?w5nydh7MsJEymXW82j)*S*;M`{A`? z>R*{wqdkA`{9PBb`}bn{kLc@=bEmLinp5PPvh?`jlhVad($1X(K6hzD(k?oO4EMO{t&SDT zxCqn7=%}}2E`1^;d-?Q;Ng*hT2nsYiXS*Q6)Ns2Qbw2IFB5igIZhLS3Qz2Sa@c!TiuK5~37R4pj(W2`s(`Ni`z zu;!aHa+pmY3=?i3cr<>z*5*Wtc);)vax$G^p$LU>a~ZJ;_YV%dFjgyPst0#JKJdHK zK0P|tPZi7GW$$_$N|H+@}4a$sv9)0ZQE-OYfo|5t5vI(JYh2x%iQ94_5}|*z9W7ayADzC zA#GvT78%c^LF!2(j-&e(i~3K+LBRR7MPz9Q!h&Z(HpJ9|Dk+r4_@ypj$GsSG zs@150O`PK^AR$S>*2^D7?GxY6X5DP>?8z;m!d@XW_0_u9nfl1aS1t zDIe3X&x*TkZk$-PO08b2zO}dpwhye6au`P@7GI50QYuP|CWa)5ss>%^ZoM6_rMq-l zqapx7na50ae8^I+d@@4WBh}^)b`sC(sg|m&M`R_1O5nmjNW6^qUAQD;>^JuOvLTOp zzwA=S!U9(lE~@m7#qGdA#hnPSGG`@J`S%MO)NRn_8xlXw~s)x4Wn3 zs0*$mxP#>aB9Bz29F&8@aGf>T_}}uBS%S}5?ttxPw)_}*rVD`us_k6nm&STKP1Zd z!|#9%yWQ*Vg9n&S)gTd|$J7Z=_w$hIP-IzJu0y(HHALrlH_14scO}tTuRuc7*ljE? zF9zlm^JUx}wM*SK7#wHc*l?Kg^ZowjR{9{q9BqmH({g}L!u2pT((X#}z(M1{+x2h= zeBrPlU8*Yfz-;@FCG3-sJuiY*@d`^03#>FCy39mqz!6^;9IRt`u&8i?2Osol zfpoav9rQU5q-uocXH9}_&Ws<)zNCqh2$(?|4w`=4Ap+SSq!x)XpPyPyTd&RxT zW%lpi*nhs&UfrNt9krC2lWxqg?exdaj^8qOjDxHWl(YODJ8JPs5B;rYi_gU64#l;8 z=SV7EYat#2tr^4|2z~S(8|q4;0A&QUDP^#|G+n_PhBc}tsLXsrKSI{-S2R>*L^9{^J46z%QHfj1d2UEs zB+xZRJtMMZP~kZe1+a~o41l6cxPRY#;lhs^ekt#lVcDTvbIl-l012>|g{0?=KzvrHtB~i*P@b-YH=COP8od6mQiBMn*OKa_yN3UQ{ z2tcg{4R*I{lhT~QS8`%_NqUmWic)$<R-!j)^7SgiViqiZ~b`9KORfXvkEyq@N(;8`5gAa=3oExw?Egj+4I{>9B;OMF_+JA z&TWfXY&5j?$92@xo6WYcx1Q@6Jib6xoBt#s77S=t^gH9gKEnlsjIDzq#ZtI^&z`3n zC>%s}15d{ra^KaRe`qxr%>R|9Oa}HXyAKx(?hMT(Ac0 zuc6uZRgByzPaI(AT(1O2`u5#v^h_gVDP}Yh>W~IHJO!OTkfpLKIxJ~{gc0h6d+?p6 z|Fz58TUTp_{&^B9TNuvY^K(lyVjUr6Q`Rv&4w)qBieQcBXH>SNK!#a`(3YCpRMc9d#&sq70#=XVITuBU1-F~q1C#V4 ze%6hX@8!H(YWzv zAwxz?y0dO^~$oGd^55m$=wM})qMR^h0@dgsxwaz5Af zIy!6FR{si@?NS@wx%|h}p z_%$dstkQ)95w!7ba)u?=y;}9^3EQ^vd$l4Raub8%tBh&g#m|GLIbqMHcTZq0I6T;A zU>8|I8XxP|(a#Q@z=#RrV3YKBsWkSR@FqlZ0Refdp5xojwOic6NWM!q^J>Y%q z+Mon?X>MrMe+;B?LNmlo$j|>N^*1J?w^jQ;;4z}Oooem$cW7;iLWQ(QSE`<}h>l!D z47}~@u?Fm@r3r14SpuKzgHFFSDp6N)82-#yJR%!MnRwpDL*qNm8Cl2@^$eiQR-R{< zHaxSq6kcETOI7Ygj{{r7J_3ysrq>WzZ~lNlnsf1Qju}u(ai}Nhid@zwS1S|HR)%zu)y*zli?Jq^qD!bIP`*DYx(w}$Qg0G z-KCdU_1Bc{`Z3m9x->7krmIs@{dki-k4#&)YvpMuCC$8G^N=v#%Wveshb8r!Io{{P zTg84nCG8#$gU{bpiW49Bi-Rao)SR(;U|NzY(r&;2+mD?x+soq~yE1CdE0;lo)VT9E zaznM;VPXans|eop&qEt;b+9WT=c?A4K~a8^(T)=NiJV2`W*~65*rw5on0#wE?rG6v z#LZY1fIM*U=ld0EFLaEy{g{@BHOgz4c0%vM6*XS;?U<(g#WEHVR?EgL2CN(=W1l5q z2`o&t9C&mjOqoceNa=B{4)Q2ypy%ycNH8TyDSjNVk|F9QHR6m4 zA1`%CIjUBl<-*UWfpopoi!+8;egO9|G0N`XcvHAOv@8kx`mA2PT5Ll2adSpnm?!sS z2Zw^JK6eq&!eb4;iQzUxfwh;OMO!=3>7IFSHc46%Q2=#ulnCu4NwLh)#1yhQvEV|s z<80gc)vj0X=BOT5);rkX-6(j&0Z<}xar@$ZPre#k;dWJu3Ns`d{<|h(fI#sY1)o97ei4jLNTK4Kfx`?zJ zQx{+01hA*RL>m(5hg}w%)3FEoW%PQCCKEC2?VRt_CO(<$sKJEx&5oFUmJS!#z`vQM z=Cn^)35^V@iD*~C0;3_K>tGRnl;0zHTN{j-i33p=nij47j$U$~?H3UiXqS@7B8n?n zL@8e>Zv5IOPm120sFg#94o!M%&OVs3Pp}|xC*csbBgU!TA$M`_W~H^MjNeyKnlIIL zbf)%i-9WqtDM0?;x<~^^DrJ9@TfH{nkC;L!OVf*4!j3aBt%jcA^bX)0}7YNSJeBZqr7MRG81-f_!T0Z-n5mt zG-!RQUI*^VdeGU;qG=yzR$eGlDKv=S_y37kDY=QbG78i=Dm{2u-B?Uk#b2WTf)}wZS(t85rf&Db{7+I` z3>vU6W_+t6&RkJb0#^!fN*ebDMHn!=v{O5)*Sq%b-`uq8BvR2V6{Up484566n2SR@ zox?h|(|^8M;$kA+-f~+a=yTWH8n%^YMARibzKaJO)Oro9rIC`hjCtg0CIkMOB>E&_188Z^D?o$2jxrW2NZ&yT$mn*4FtA>O%ynIIf=l1j`2WM=3 zRMO+>;wtLfBRhSvTU}WAJ1-XcMiq`7SIpqWi+f_vH9FDr!@6dWg6*9LS0%%sv0ZWK&qc0elTeD_Ohf14r-otav|y@H#sso+~VW9Jt91IO&|;hlY2`3XS*A%MTzonH>2V!~P0 z_KKS?^UFF`wk!ea`k4S)8Zwr*K(gtMP3%Xe6bD)kAXzO3+EkXWWUEEgOSqFB^L*ia zQa4yf6tu`;g9DZE;qqbGvSN^H(0?BjG1LVv!FCglXThD5*%}aWsI@ksFLti)_4Zm@ zIq=UZyj%GBE?dBGBsPb!EZ^gFM_~o(xfx5EHFwjlZixauz{c7vgfIz{JToWZ&Yf!t z5gY0a7H>KEt!2X+Vf{>hdGYrsD&oX@GJADJu0@wAyk;gRa z_Go_nW9B_eoiAQPcGXWQ^;By;amjF(-=s2N8^Ubc8#H3dnK`v-!KdJgFRdn$xHom* zN~J`rhM$v40=_$5w`}eYo&M?5r)|fooX6@Dctl&b4i(|N6_?AR%D(A?>Cq1lqk)mM z=cA9_Q=&4riYUZ)6&2k1RW*EVUN6rGg{h!pV9=DgKr#0*ujq=dmI-`Ea)+@8yhUla z(>v^~?B6LauCXs;XfU3-J~@P)aPaZ$TUpL<(O{ckl`_&WyH0}! zZ^x9IN8?Ofi-Lb7v=H9uA$DSl`^kp+_=_ptv-AhIxn&jhS9vCnI&X%L$hE}g=US2S zw(`PtyKcO(nmySR{oMohYvg^F&gM>Kw5Uyc$ne+M_1`T2F+FpPTeGv47G83c4Pa&P z%(Sbd%lHO>G*LFU?cM8x#)^p+ogfn}A9+>osjIa|BwypXo{c0QtE^qg`wg5nz$I63 z;oMs^91Z2xynFY{0O~QY$*@B#o35X6)C>C3^?nZZmYSZ>knNmyo7}l3e ztn`2XPOMwEzOAD2&9-silymd5sF%IwKOqkE?98URJ$p8y7${@w*zZrLRSb^kb#<;D zuVf$m$auz45brU5y)dpt=&jSavrp<`+Lk*#D3RYw(<|f}nIcY(k}27J(6EBPS)t+G z_%^EZB)F*Jc*&tke&SG9Da?NLFF6iqo@vE{z>*WwaQL5*GaIK|!}((|Dx9eFGrx`k zDxRR2fY17XUKk*%NvRkyvvyO&b>*QyPx?$^Tx->;wT%=*)z|hv8om{NdP1!Cr9B#X zv+T^5E$fRddc=VsWnOKi1%!b*y(Mpj{g0g!L&cp`^Z>iN&%gv(Cwv?}T}+ePC0xve z1&_P~9GsV+zvGN2LC2uP0kN<+e&yQQ+`tB+4gFNe#fwv{$3a>GAk@6sdLlz8!=I37 z$X?6o#ZeGx2EI~{+LrJ4(URD7!8e&V{i0~zR924o#b$o0+{O60;_C64hk5aW?XK58 ztPlfOLC!*7&|kr~z4GE@e@XJ7OM02nZ3EH;q200EYq`T}C<@1Qz~*#|u3M0^)TY{d zmKx79+%GNsNJok96M=rh($c~OvPrO?8E*4RtdrkV%!F=zwfI$C%MTUbgDTdtVh;NC z^uXs+mc`Us(ec8Vjye^`CUor?=~3;R;}5%uz!B|?-3xGt;x_Nk(lz?RNawT3?TAMY zj_>`x$;G@!sh^83#PJ!i!t_tR?WgUB<57Z!_Z)yXu`hz{HzJ$2^%g|tUq`-~$-YI> zQ26a6kXwIutFpkHWdrEVqnPMZ_9v|7P;m>GadQ#G@Fz{;KGXyA(-&U2*QRtYoemyS zqxOwGmLFyk#$UWzlWEg2RHTkvir0TT>$0V#uLVuDtV|WZ2_4s1SlK$=x{a~cL^2^f zeH;*)JAhYU0>QmHB3qCxJY5r;PlATp%;>G_MUcn1&oRgnP-pxP*8Cc4trO~B44+L|k;n_N_30(tT$XC2#_bOZ_umNhONM+cH#&mX=$x-7nR8F$$j>1ina%PaleW=UO0E#@(iUn!kML;JSE#^kuaqo0^iBA%#AHRJ z79EzwDc0>}-?AQptdSNJaMG#2N`5_Rx3Op*k#OpIjS_JOE0>_Tv{WRfR#X)kwZzWw z|4Q_4e9T>P;9=y~Cm6$D3DZil-7@0kr8#n}GSi!^iTfn>Pv+*H5htIn*PFb_>dmWr zBXz!*shacny$-{TJ0uqG=Z%!WETNNk6U!k))icMPJ9KFRO)H!fm(eyf9|M5dx=rX( z0b^B;v&s=jf*yM(yqOxF;>b?u#+=^QVKQQS%GdI3s&(MICqIw3%L0IFA#6<1N23rh z@Z-`X$LPOu3jTV(jQ?xzcCKRaL8BfVhm@I2IAAV?Q*M4-jS65E5;|JmN`xKI<|V`3 z4QljFTV^iP@1KWKFiO^_Us(n3K+s@Lh`%fUN`!$^n;E(6fp#Di-iP?!ntXX|Gk*Lw zOz#1?saat*$8GzYS$Sk()dFQX1oQWk^|g#!Ks4Xhi{fcu4k9lCRNZKa2H@V6fGeFi zjF!dzRa3To4-O7&&I&;3viWyJocB_daFM4l>)J;pKRtlQeUFFhSN=sS+fSu53fnGN z77i6Ruabr!^uAM11|@s55~_{q!1iSR%g>10p$y#tUGWaQ4c81Bh#Y>}ilCMA!F@%! zMq7RJ_HE_t+YpsWEmat`MlI^?F>q=I)n{}2@OLMD{VtF@8| z!Bpl=81`&-{_ngwmwzm3Bjdmx4h{~zmcAWbem}V?HB35DIuG3f_4XEra9YCZN*pk- z2)>CBR<8hzJ_MOo?<2t*!~v0gl1Sb|rr9#C$Wa8YNR zHx|sBr9_&xs*HVVK2=gfX}W@p&)<4`^UH}_rY`zA!B2zLdY)L&Z;R`x%}R$*KB}Fu ztHDN92`{rA@3Y|Tx9IoXnAIIdUw_y7$a=grcmpXynOie=u`c-JSu{}aS<2?_EmYO| ztdVwgZkh0bwoF_$k;IBzd5@Ww=@5*lN58EQ$z6IJ5zqSPv1H9M^+Rt&bO@>d$X8fQ zty!}syKOI}q**<9UTNYDc_}rIXFr6=0ar(=j6Fl9zvq}2MhOI;RMvU7`AF}aSL<_r z%s>>B@qC!AT3gwKf}p|<2Z7X*rh*JY#Z3w`IJe?gG0Hz z{34~U)JgA7lu3zg4ah@|twL|#e>{8oOyy?%zWd)uE1NhzE~R7nddp}P?_rT?5wq zj1bOC3$jS?0yf*G>S3TA9zJ*OoJAJvD3nHxr*Y$Z4Od>vh-MUaL9~V9zudS>mpVdd z1kMV4?@mdmO-T*>$A*8S;m5TDDVS=Q7-#U zD<#gEwJXD$ZGW%Di%+WpPgo>%Paau6@Ip)guhaEyq84={FeRk!vjQeNzK04&-ysgV zu)yrS#FjESe81y1zPPC-^PizP{^KNka7OmB5LC8avHf6vJCr4`|M6 zynPY7X#HvD>k&`r?~2On%EjTh)fhh^v=Eb6Rz;ogYk3vLH^gtPCOf=F&6>T1-cl~B zNSuVO#5;uu870s}lgqZZ-VVQ8_SXR+7K8mm3V;WW&u=xTCy)YcjNmr(Qr8=YMu4JfnFWmFT+`M&* zC3?v-QVZn;gpxqNff1S9CRB9UmwD8_7r*w?iu7Au;l8yePGf@8Z9Dmv=IC=k-1lK; z9>4jqC-Bdz zyTLSQiQFHj*;_?!-cDvQ7=Imd)L*8HuIl(?j7;-Lc$jde?J&3od8G8A>3MgHE%?c4 z7aG9Z%WI*u_yswWd(6B{$E%}o28TTh?aKdXu<`Z!#0cwflG&=Rz24SPQTWfSo8a?( zEod%l%Wiw8rHGmHo}PmheO+5vf%UkYo0T$4tfT|1L|Bk4uP5ZFl+$Cu(&1-*mAP&l zr5a@&94}7M$w^KT_kAGnk`u5xNpekU)v0rQeWr3NWOg+x`?_OJ{Tr+n?2A8V2%o4yW7#D2)Afm<$ZKSDj_m{s(W9v?9#j(Uc^# zDGkT>rNMopoG z&^%%BBmg|F0L+>H{8Cf)Grk%{m=s&H8j;^jbxD8hJ zxZvy@+1+qZftYo3aQ=058EnNvI8knW?ry#4bkrPwIhuT`+g2y|c1Qg1VdXU~mGrb9 zW=X1(9>jHny)B-y$xYnksXRd%EYcJWp(h=0+eW)7&_Q{fk0E` zH*%@mX)Z^eYs|76*C)oW4YF2gs2lWT{GdB)BIcdTmzHc~wh^%ykS%s4`OtV>*v5Xx zYgh@x(n8-4s0LCnByg7JZAMGki6OF}*u-jupr1yRgeOGa(ssv-C{@mnAp+Ah#foOtv-AG$e8EV zS;9kl{s;VzuEWj0Xvjf;a?o|}NsT5=)*%FL-*1n$`Qz%K%oNQ(759PHU=(4eJj+x2 zLyHDR1zBsXX*5azQJvtdH_HY%9U6;Yv3j=Ea-n7{-6QvEOKjMc5MI$up!*S=4mM>#7&5qd$H;SN#p z5ssUH3l`HX^X%WU#ZO-7SyvYwbH&3-C{!C8Y8ST1L2YS0VS;AvtJVD1*m1DF0--_s zIPOc5+22pW9K0qKaJ5u83<# z6XFuHb?~K0pp=WQmekm_tKxFW3HXOx$9#)lq1u=)Pr&E}T_Dr3`J_ogwKVf3B2j2MYY(@{SlFgxoSi-jL-TTZ~tsdzgZ~oo&*2@m>&668-Qj2|V z+B(VhRFG;!9>#JZ-AzC@tB9!n&x;F3_PX0TP&2c!dP;O7-^Pog>uwxdR$Lp)G6{kb zC-;UsOlDB;si&(U0kKecy}8HLZKFTF=M1s=>EtlvOHE95aL2wB`Z(5QDbV*>yV286 zd?KD3w#WV7&ZdD9I4%>w1#$vrRw8)CYG?N&_UOEhp%OA7wVUukyLgt%TL2Uod(+|* zzOy4CloHp=9p{ESH7wHpW9T_EE1WxiVn*-wkg4VSjD|{sbt$9lP|*DNDcsiBX|Ixt zOi5S?D-_RIE^8937>$?Zrdfd|M(4n3OS|HBVfYmx=k)%y4jCx*!rc=I0>ea#jT+yR zIxUb@w{8);@;7eSfKkQ>Tdt0*D%;abYx^3@0}NR;hU46A|*n0LHf7B-P6Fq|%V)Ll@MF|0)L5t6~x9$neA2^Vt22-NDkJ^n! z&;hxFeL7oo?BvhPDG_qQld20?N(iC;pz-5N5X|geRpfBUO->lj{$)?{n$o*+(6)P7 z)$%KCB2f?CVH?ieYg3%zTDy424sUnMUuL=H^wh0p=J8sS2uV;+lNI(S4v_V+iZ#FF zArylR-ns-#Cr03Kp;|uL{){38`ACQXmL5oFTJ)T?YIJz39?CY$yOMiG?gKZr!GOs~ z1;la>l)b+X`Ne#}I1s`s{i|!6hB#>nnHsO^^cqgrcwd!KVt?w}1q4@-3N4)yj0pbF zC>2LvC|}#GMTFNoE!|LFvW{nENT1%SqUx_Hn$Ef}^N*Uayj>Po0Kz_`hC~kj7kBj=i~d z(thPa;KVn4J8ZFvRd4k{#N<*%qqJ!pgy-W_tC7Jh$Sl`ccpDmf|VnG0&i+R403ynoK%qs9>hgMWzBZ* zdk;Yqj@jL;2FFundj6FDDbF)|HR$7(*8Ll|dhn=d(BTbK2zuK#+%{iZjDo7ngI%Ka z%uj$NPCoPD&Sd_zK$bEsh)~bO)yA4~)1I;8evru*aZ8}tV`b~IMWUmD29-7M)aOrV z*Z9qJ3o(bwLlJn-UyoA+m$`W<;6_~O^G}Uhw0M`V&r>Js+BF;cq*m+qFTu0dO^5W32ooZB7>=6UBXK-n=lT zM!$bPSYS&zXKnY*w_Yue3fxz#e*LCw#{Pl>todw>S!R$($0cu=5R#MfQBi{el+{3A zWD~bJmb!(!CQ&}@j>MLumKgF;$8LHeG86#+r*W!xm@`U4$|7?!$yzH{e%R4my;Z9& z%m{s3^;azg-c>({m6&2T&PoA;yHnf~Vci8vhG^)Y%v;#=kB`%3p=HhIYDR~PR7VKL zr0Jlw)ldS6WF-dvVHa}56+dQS03cS44b|c?!hN7B{rj%YQ_FUP>q#Hkv15GBMeA^L z;T6uhrpD){eT?eBBrg22*cZ*Unl@c8f%8yKDf^2kIEiKzTM!{UaX86ium`W2qMe@| z5okxAB~SfkB=< z)vZh%>O6iKJ?ZfD$$O>Xe&6sfV4D2-V)b9lUUV8ZDW+o4nUUrNGwn~;m@sYqkdoHg z*Em?O7=taSBW*h5pOW9u*rm5nVd0GgU%Z%>_?mIy}YvCm@sgd)5``1FdmGtB|Hfv=6IRtr{>wMRZsK z1L7t~^5UGw-6CJ}Vcnvb3U+;9i{lO@NLOsYpT_!{T>9mlnRS<4|GP!2TdUT~n~plc zuDMiaF<7RY$BQ43Wa+Z2WHjSaXV%-cxyX90KQ{2JV1JRRz?CdI zF)L3k8wWv7+!q>2_bx5_6`oW9>i$PPrjAa_ITaOkz7Ib$Srs3?hjBAeH5Ot zD5G?vnVPTxQ|^cFw^+~Sv8eYc<7p8=mtaDbUoXv;>d3B`Me#E31t$L=4@S$MObW81 z{=}G5Zk6Rn_zBV`0J=@V>Rw1ONlHJ>!x7gRx;tDP5zSI)P4Xls7c0sT&e!yY2eYdK z2a1Jb|D_hzsq^~oA(56ldUQgo8iN}JUOKjh4Uur}!B&SvOe!^OYt=34pAWJiCX_n2 z*HLU%WI_D?(MG3^jn&$6@%(dPQBl~1c}Z1h-QSot_RIx`Bq!!IsAq zya~SjoHCF@U;K1F+l^jB86sas(!P2LyoxGO7~UJ^T%}5m$t?-z{X6S!#eoR** zPbAe%0QZU$D>C!}zt4k{$T#&4YxfQ3$#zpux}vSb_QYQ}_$BLU4Jn7yb8FV9q0m)+ zGV|3o|j1Fh&5MCrEoK59|;B)z@K#>cA-|$6{g{LLB030R$3;#>lJIL=& zdEM)8V2lD%AR1a=t;?q*5_AJXU513b{%f=KSkI3$!b)b%25XxnzsQ5#(& z`V65GdISZ4#3M)UyUAq1PvsTk%WPI=XXZR1C~$0`apyuPYrZ_~rN;3hGDS)rr41Z5 zv*KH7^a@XIh7T$BnNMd#uNVvOSL4Klj0fj4Vjly=!;l!6XWk#-%wL@DS8>f4E*jF> zRTd%P_x9K{1S`X_S2`5`Z>V+tikxY-*E%U5n477Rl>>jE1|H7~9tp94eC9!_E%Ki? zN)Gpx2i3}hhzMVpW8nL#IV;6V0OJ3;4I94oYndWMaFN^!qd#ZdCBrv8Sp5FGYj~>o zB50_m_(ScJb?2SL#G)2;@tKCa9n-Te-`-#V*t`bSOurBxWl69kX=pktf#ru7HYK$} zAo1l8BE|g(j&5%~r+!pgHfSu=Vq;@%V-_Aun%4-N3}z<|lE{La`A=|=eWO@2=3c-i zj9Mtp!RiKmna*#}INtQcHm0ZN_mv_H+iu3P_!gVyMR2vs(<|Hr zs5SqRIS-M{053^CH5)UJk~aa0alP0%0MIBmLI#bbjB9$)T%q>tV5d8f<{M~<{!|he zIJArk+y!Viw9xz1LDMZ``%7i_#I3ez{yy+vH6ESCOVP z+Fnfpf+aKwlcmuKfzLTu&aZCPwv1r$#*?(hc=d=$01%={>E69i88T#dXC3aYJz8dX zMHW7}Qzai_ z?%Z&ncJCS`L|>BLXoTl~#8)>~^gLE&c*F{HDOOdJQxDx()nRatWn1C4yR+?v!4SPO zI}hqz9-K~-ygeWn0E^?!gnbJE)02!ICc)$t=EvvCWG}qbfQnD%BJeOuch^9GkKSe< zj{wLvhaWeP4S2d;O89gZ=D)zE)8M5xPgh!kFMVi4L=M9zcV#Q+i!iNY^R~G|C{fuc z;G}?4LS9bC6i0`4FE7E`LNY2eW?hd(FnMoO5AgQZf~+I>9^IR3+4Lja3z8GU)}h{e z`Q0-oPMPA(n%DNrfbbvKOt^P8THqVRP8c_vKwkl1(-;TGP7}^-xc6qsVYIotwFvCy zH%x5ls*Xzsflg5LA;ELfV+}3r=uzrWkeZfqJ8>~Y$}Yg63)jyGYL&p@-eus7-?Jyj z`OCvmf=T=m_wte~|EGe4SF3&{&Er=&d5tdz;5vMFk1%vD@b z5>#>h#D@N#f*>U2VddC*7I-7X2U93g2}Y{2ozh@p82FbiXXD zw2BFp{zmea(zB`pVaSI-2V9)yK%mKHEJ2W=KHZq$?YNN$Vqr07-cgTy%?k4xr&PUB zqnEE46c``DONxmXM|V!)q23Fo4yB?o_cL9f+sT8{^NTq5Q&uzZWLb>krX7%s@Jhw6 z+%(4-XA$wm6l;5yu9AZaa^8JHZx@b$7!_}L^uCExWeM;I#9={&lNB;Nmwu0SW??7^ z4I>?a2R2@xwK8Mr?qT+)s16~v;y0R z%e%Qv&1V?!C|uCIuWKRkTb=kzZS+EQ>#jSNuvpGd^YMuO9VCjTAaPkPCrmIMuHnen zjJ5K192&g#V$|^V@%yW2no`e%IYd2nd;Y!X_bOGZN>VR53Q+v|{{DA0$LLFyt_n$y zxH_071Z41=p&u74^Tu`L*o#V zb4gWbm||JE`+ra)c@nAlf8!M*-XX%NV5aAR0(27RkY}}23f$lgx;PwjsmJM7F6N@e z4*BUM)W=*dBOgbA4+|?t;tw1!OF90Dc~w3b1cr!)OqU-F0~T%zg-9G1t;4@sjSQ}k ztT%WP9mA0&2se!fbZ6&@uaT{r;2@>>+u0{M7ZHqMJ6EGY>pLb*!|X0C&1xXvC*%je zD4repTS5Y)^!o<|`El*zwQO!pF-G(`AU(1)9HIr*pEq9@x&3u$aEpG651~k$!7`Xv zvrb_?CCwAK;B(-9i$VX{)rcbzZLE-thW0KglU1K**`>5k7_NgQ<%-JWCEJ;StoUx) zIy!dY+4z{R??uJeOI(hL&S1=t?22Qv>UN2Gp6lVU921&(3VFXcM!|y;M*XMxDLy`Z zq5B^Z)vE6*kIU$!UJ^GiqE6(6+CvW=U(w;fWDW;&ap8dQ$n)oLtmyOG{gtOav9c3} zAn%^`{xKm*hj7xs{;gq^DpefYcS#v4fmomg`r*+?e`GHD?R5R|Jd#oycuqil#cl-3 zLQGKx)vpw`dm`msw8y~26?0a9- z2LvWXX(Q35Lk}>kNloc0kdorvefw(RQ>S^4j`}LxI_>Y+tecaNo94vnB z#HU0I0%swNs2*{HWpRMc5*;I!(2$NqsZy~R~zfprEt|Bg#>cncX*lqhU{91Hs*i{*kFA*v=<5H$FB15tH`r)5|pdN-xaF-UE1$9JsIpZX< z=%&1$-Owz2pfJgXHD+#gH9;w8hVJ3o@so!H)+&lLt|8Zxj9-=s&mHtlO|{U~ zby22{x7xD5;2Ul#gJI|XC7~lQw`t-TRtpI^8+rYBU!-L97qLAaQ`2%A!jkH74&OOxeg6*^U$CW6SO1@fqxGia)2BE`x7gMFYBjy!eUGqn zNHJ`!zB;<}a9Jg!k&@kCyO1y%m|2G{(xL4OjVIq{$LF)VI6(tRjH>Kf@O=eO{gip$y!Y1 zk{;k%^zlJap}8G*2g+w!BTHt3ak-bDFN_H>+l31^CAjawu~}Ab7awYtK{*h!B}t$$ z^>~`IQrzWuiBlJz)8(=IGI{vc*1{{(B_#LgVYZ=^+2hmbUj_F@1>9oP*s-_4yAqwzKtTSZ9t^E4_r`!)0Wc1jtA-x2d!7b`>9Sxf`YgQ%6 z^!Px(7yk%(3ns)p{;FdN6Pm=1SVqv&OEoqcs;d(`fXsUQSL%x_hNqN`0kRm-29=fi ztMm@XF+Xc59zw%CD(Clh>L&1PG8mCrmtAv6wUnj??!1_HOT$FP{QSP2>^{G$h zD7wd6j9t3Xgiy?4q0?J0_|i^p7PbaRG*@n6l~Etk9-hIw+wHA@r+Wi@#>n!6uhEM^ zR5*{oHFeuXxM$b1rMgvFpD34F zPd6;pQnW}3aE#pbcE}qaw)J`Eh}(WfqpmkUY;L1bM-8MjcA|APl~kj1zb=4 z7PGoyba8fSq;`#ZEyjoVw5UYKzb3zy$Cr&FbYcG;pEUP!5y~LpUptlNn1EQreSLq| zap?U;@~_8_#~DfgJT)017=mE20cU)2Unr^&o#U%zmc#HW>Ut_LI zMC_e(;HvK|70oRzeWq+C6dJL?y<16P*QubS&)o#+GQKUFL5M70F0Bh%f^|q@;zlKm zQr+mi>$5d}iQ>6ZW9?x4%hkb)4!HfT#uF(P%e_0|@>HAaYgaGP7U=$9hI&Fl70I@0 zBF2kw`<(KLs=K_qJQJ#~FO%8i0ZPA`?p$V3PZlHB7G5yROJg-G<1t&5TJBBAh@{vZpEi0wZG7@Ym_$ zwz~1}vU9(q&o0L8qg>JT~ZG-`j>m|)ObgBII+01QjSmb z=O+#{(u-26A2HE$k2=BrZn0>xjA0~^ngd-)-x1m_Cj zi_)WJnp^9b0T4qpWu1IeS(;ry8|f8a=4%lm9XI8^=w>UUh8-QUF=FzLTJVpdpDQ&*{7SpGwH?Q4W?Yq# zAV1&fO{*zrZ4irx-m0wkfA8qg=7*vbM;x(VgI6IPPS{~~tX<&d-@M8o!aDA5{}UX( zXGm-OsCPbPKQtph7XrcGSQnyQyQXN|9#+_YZ1rXwTy+&L!+V zOvHC@BlpmfHnwJ{%+D`M--`X4r3t%8Olx+he=3rx0TdO$$U7QB9lpfPDSTihPhsK9 zLU`)yj9*CN1PJ}yDxk;NZu#S34 z`xzlsdUa`c@u68*1>l-9)RqAr_0-;Bl>#%P!4tlSle;;cWE+++?pURDsyX@>9N|WS)iKTyB^tb+KJU60?eil7zFAE3#7tN!41~ z2sJ@ZvS?(*w{wf#&PVUf$*)jw`?{+26{fXLZF_%b2M%i4fmFx$%=cUl6ero15 zc*0vX0qIuU#h=2eRHBmk1isY#iI-OT{>ts3`m!4O`SStqnh}Os62Q4>>3<%Dae^XA z3*WNEP%}WD{5VkcSnY7O% zN7TPgc7zzS`Ypd)>>5@M7fOIq`fM$?(E{YsGJXsnRB%^(hsBU;_iooMj6*^h6*LNS z25stvP(qN|Xvw8}LtNh!!+gKjWPQ07I8uj(S}#fpt}Z^9MUk@;f4E9;gGz&tT00_M zdR6%H$m0?m?nD0p*_Dgmqo>&)P?Pj+&yBFA$ti;L{yJZu7Gda$73b5md(Kj;(NHV* zQ9Y^D5Vduq$C;L7#uScmOUusDyVK>5WQ)G6N;~(OvNqLrV?rCCVm^@0QutMLg&Ohi zn|*%tz`+TiaMsNmp;!jxy5vLGSsfZOWv*^ zYZ61q9zH?ErSrUmH>J!`mLD*~{&QQqp~!VJ2mT+oRI$&Qo_zEd@Bv}?5OzA9cd2+>w)uhUH)?XY`_|)vZDv>B5%#B#cjui*a z4QZbc$M6sQn0mAHjswqJkE_L1BrI+>xmZ{%q9zX50>zDkJ&H?MDf zjC)%UWH7+9+&u>a2^&DcwCtx-$lksh~ zUJNU%a8Szg=&zpQ{lD%g(ci?YpS@+JUw$vsfW|3XnXDx$OEG4C!9uT|mhqmrrSj$& zoTik~oQ}!mlXw4oz2@{z+gq)NLXYdt-PcdeaW#TZQ5#9lgx+6-J#j{i}`o7YeHfhna6biS{z_$I^m68lPKQTsUdyh1nr zSFHmxRCPZoPX>Hiaj_jX%<}Ef7QPmhsA|@32x;t16-%x3it-}KTYz-k+_~uE48^9$ z*J16NHDb>IEOc+r7DlCk?10&F$pDS_)XKjf&ja(WtZ(~_7Cp>4oS7kJ)K^>$rgwku z*+DFD*$$NSnJ|vEw5Z)^YQ0_s)ODHwF~!yq6gNyyK~A}NrK`ZE6>k3jC{yx z+Sa*gia=n>b>`BUS1arSc??~PJLAK&iO0vMjJ_kb;PEumM9Gl+j`xYZ8OK0~cZY1g zE$m^aR*d_G4--z58DaFxK0GamNjzIVsq?8_HmSeKWfDGSidu*0mu|*#=DwJ${~S6l zLs@EOPl>tV`zf+%Na)itmI+}=wd;<5RbZG=>v3^mRcE#0_s4=3<0I3`%ZnRA9Ow$4 zFT8Nk(q(yc*2YXQP58Bgc(p6Go_#o&gYL!8vz_yq5;C{mx{J7OucRv~KV&+~iimS<54y|G<#TtVgsHJtJ~*`FzJt)LWLQX7OSXA%8xj>dpw!X5 zIiZar%kUxH5^S>C{RUKAAX!*x?*9>G==Ba5PQ3kcZZvH-Kn1PL0mc5^O3nh>7@mv% zc6=8X4cWDQUb1HWNk(<3-@h1=Vh=;H@NJ}o^iy}9c$fLBNWI3&@0&q>=}kubyRNKt z5=p?U_QVsE8J$lz&Ml2%ob8=Cb$JBwY+ddRqRn9~enplW-Gb*Z2(oUo1mW}GimCnt z1}a()vYt-+?%THoZ^6nbIrU`~6g(EolHb*UTOumJA6I>J`ZY3iJP6NzM=X+>4IdKE z#Nzbp6!qQjV!^i$$W&bRlGOSE7`NpW=FN#%U)oPfeGWd*3^Ms{e}8s44n`gUho-@j z|H(5=Kt5OF;x9S0{?s#D@npdROgy`$_yK=%0fF2elE_DOUe_Ky?gtEqlK4zD-xRZ> zIOy3BP3#mys1*5uytY72QfQIcBgp>EQO~ijwX%$iHq^cm)wz4~4mXA*$P05nyp`dy zRJvC9^`&;B-j=hSuC>$AwmQ<{$+xHHTOWEcqLa6|lfEvX%1u7srtY^3QqxBQ2M{a% zAjjyL>};tKEdioRzFtFnCA4pg7A@4)S93-{Nu94C8yTp^xU+gemm1)leZK2&|$ZgEk}&L zKM~qf6Et_ug>_TVBfd6ZN4;3$)x=OinoFTeJJn~zNGOtxl)~38b6H>zzH0IDBkl8n z&DOPTmb`!Bnbgqjw#NY(X)|Yi&nzA?S(+}WAVssOsNGc^(&syFIAnMm_pUyZ$afaI zBnk>+%ix(dgcS?jFgfnNl|<}*_^{)so!(+t_WsVx`RaHKxf1|x5$sAOsD{Rtgv{7y|8oJSRhM|4GA)PID1>vWYD zo-&k+&H2v1+CCMP?<`Tte+1&TEG=^UUS0iu$LV={r`YgFWRQ@|5xI~%2SJ%Z!EDwy zV5^hxqYQcS-e`>+DG?OHqMUw3Q3jHX!gY>yrb7=hN=WgEm*qQMyotzo$16xA_*52n zi3WxR+>>0uDQA-yXi6y4=T(CW8(?RX8;p5DqgXUc$A-7$vlyh=eZ`OK8KP3L#K+9k zlR!T-twBSUrL>aCXGSlll8vVV?1pucG#j3s%zgk||fgFg> zoFlnKifT$ZWPA3E3utej|7sV5{Pyo@(+O*LZyw%ym~C*ECCyu^mK$1a{<+SPy5Dz~ z_FJdzwS=kWm+cFeqjd}{`ft2#_id!Q?Uun4#piJ6T>>)GjIP_+y*?D!kvTYmJqS)o5ZTYMJZxh6<8$667$DZX$vL z%CC8E)(vFCl-?8588b50Mzem)R<&u!gv0I50#!LbSRUTJ zZ{K*EC)0Q>c=7!?K>5<$NP7aU)6>bq7ZbR@2#SZKMrdk|7ey{(b?gV^5CY+z)~Sie zILq;sIE@*#@}r5HPamyn_D~7nQUt~fuaS1Qs34pPsBELGIW@n06JR_3>q1&PNc-(* z&t1FT9}}(Gsd2xvTa*D)-EkG|HqtPz9n&c_3i3UPjNRO=cZW0ud7F3HJQ&*MhV+<5 z+ONVGC>lU21m@)|H39?>b=yhOZ_E_L^3-GZlMS2B2Gign+02iwS8|B=>^Gu#Qm1ds z77Zi#?yDSE4c-MIQ3b*uGsx#CL>^=Rw0ZOT1vRKw?>B}RcGSH++dJ8Pem$B%4l6TJ zpph{rpz-5_DJ6KfS89st6Pz(!$bZK^wccRjT5C zSFPYFy>~w3dUjE#A0P?D6 zYLEHG(h=hf&t!~vBsWXONVHF7dB@~Kue_D})b>KeoJQfPqdg|foLFzbnXx^NxYxhm zxgKC1APF$ft6w^YK-I(Db??*X)8XAARGe5RDJha`XGo83fys^lSVTOglT)0d+D z=P;dMD>e$CmiIpyY1H9(7qzBkbIQctkP{cuBq;DYL%0;XLTE~!kDBFzzqHtMa6us* z2fhUuBW^+OL!Ng4+6{t|JgVd=!vQ8Y$Ej^^i69iH>5eO7uTV$|g5(${v^xca9xf>L z4DbXyr$b>Ve--=Ew5iA>_#u~q9d=wM8rr~(<`S46AIB~1&xg*Raq5VSqmr7(43va; zQljbc_=EPrvic&+kTr!d(ZsUOypUv3j*Fwn(edhv+CF{QX(2V=gz8UaI*^zdh=6%#Bva)^=p`J{YE@=M$P^S-o0R@P3}aR6$W2WzuBoJ2N zBG<1UwV5j&cQ&PyjBhGK*Q>*bnjWSSknYd){n7JLF<2-C#jM)}>8({&_ALE)$uJZQ zN|xA4d@pVv{DLQq{q?zXz=ex%`B3J*JG%9Ri_r;XI!BmbgexWyLM14fLqeZRZgx-Yj2v}?$os#Gt61U9owOwh@4oXyKyfW~~1*7de++PwAq zR;jJ#I@dr{_4(=FLcIXW|LeLK$bcntok`tl@+Ub9ACE8@9s(#7GCeN0K$0>$YqH?Lo9XJ;i(84j51$xw~m znaO^X+RW_eIk1A-%FX;kkM&FcD)*xpPvbMbJQQ4#Z2=cSY>p#*ymYVjyCFdZGsg3O zE0Dm2=t{TDyC7cTS#-8E*|h_`w#1cyF?oebFa~E}pO``~&eH0A1_+6-WSeqZ?wL`n*apg_o`f(D8ITQlt2*N|_8Ia`nY zxKM1f|C3nrBUG5dNjUWF+utZ}m38o_;_IdlrOlF>H+e z*dlsH2ep&Sos~unr+opjQn#1tOv(m|xGk4Na>%5`gnr%bC8k->aARF9GJLri z3#Mz|B$ZI$7;qqV@-OQq|0~)rFJ7Rv&b~$fa5@PAAd~Um`#c)rY$oJooDy6Hw_LKu z7lIKHRwxpV7H{{CK#H`_yB5bl6lw#41+6nChIEtAWEX6-t(rLDfeSpA_(bE%=3; zYI74Pf5p941e{60s;gzk(p%J&7%3x+Ka6FgE^=)pKf}6Hl)=HwMVyXJ_;q_W#(_VV zr;V_Pb~H8n?qi;g5Li9^|G1ObF39E(DoQ`?@$u|&5)@NaGI!s>Nu@phfE*#uFf8C= zlu6Tr5PkUZR^d9n=c?9%iXDM^7{MziZ>}`wUz1N*kk*}lgqcf0kzill+Q$X)0)qmn z<)rm$)o3T}sR&f}Xt#61a4BJax9$w|@XfCg zBC6TQ$VWd;a^yu#CUS)d)wL#`-|u(QASAo;H8tvrK5{sbvObbKM2JC;7j1I^LdKJ2 z=lk_qguV6w%bpi+ub~O&<;&b#s{Zeak$eD|NlbN3hKGmRTCMcmmM>aMyiJ!5hnsK7 zdom)z#_?BH5|mYbUdJ8J(TgLAasB$`ZS8XY>eY7q-S2@%T-$x`?p>)hlU=+mAJLZx z{&h>rKN>?sW8Q0~1e>u?Q=U;{W@>tzFA1CSN^(C!LZ--#4mT~?E5FY`PH;zAdn`f~ zuiYC6R)n+UljcDc# zRhd17;05FF7tOYbXU^>}s)*uVeT`Gv+Ca{t%tuNKM_qt|vVZ)lHG;ku1c+294hvCb zsoS^kjG_eWJ4t_6DG36Z$ckUwwo_Ldj3Ic5ojQ{H1^T4F5}7>N|8Pt3-pueLwoSsb z74Km)x3W6x|h+VLXhDNAvZS zeGOOZ1r{vr;AUp(K}RQRiSG!xvg~aXcrt-S2M5$u5l1E9kN2TNS?|^uB|JjXMI>x2 z+JiUCDjqYki}u+$Z^;4(K6#T+sjc7H6uBVxZRV0r2Hr6BkfNu+6U`ueI??1ssC)qT z6^qjT3k_6M_#fV5w^}&y*;T2}<$%NKqh`a{a|d=OV(NmvD2Nn-$Fd1VB( z@$b1uTU4yB{26Fr+NBlGq-h2$Aiy>KB2;`_h;#NTuW#zGN?Ym6rw3|~l!e2Lu1B0jL=MWny>-JFXZkrnl#V_P8|1Gnk zelre*FP~nt6l#jj!3CR~F}J;P`{vSP|KE#TqnS@7n2*ub`O`q*slXnackW!}{B2;y zECFb1^3(rEywJVy(VLZBwY3}IeB~cfF@~gkn#<0^Mo}}6amZlV$Wf+d4$%N^5&bC2 zl54vivL4s+AGb@;IWGosZ-dp!s2oLF&f8>@b9&ClA6fC%0Ds9_qZb_V%+{&p<%ewO zp>WX<3HNL!;!Io-dDNhD{AQr4dHmhs})*JRj;l$*2WGor-?<+WHS`vqZ-# zLW50hrV}gxdbUd8c}h)9HIK0OugTDST0GzXBK}4533*o0RmW>kPKEXFq{xD{Lhytd z2p*Z2rpN}-2vE!v>x1Eewle_!txU<4m-xOL+WXtKVe9^q=>Q3$T7fLyk&Gf?c@p^7tno`@owuGs3oJt>%=kgkxp+U6{6+$pc!l-W zTFL?;>^?6Il2PEY{!gh#n_X-eSI$iKYnJReec^S zBUzSL;raLNbKFV6thMm`DSrhuiJQ=mVTA!hEVp{GBE#bNTb(w{K!g?-EFxM+RxUwP z@=^*CKJdj}YQ3!A+}bMXyorl2j3elaYkAK5Z^&+XHyN*VV_AVgMju)7cF}`^-WWD=45YApZj1MTf{?1=IH zX^VcHg>HZg0e)8&2!vomrFh_)SC?=>7GC z%nT(KC3zKK5}^>(vhMNRm}o;oR%1?>{z2`V)E7FFvbD1_j1ma$k`j1c1*Jz2IY-Eo z2nkI3t^_87OEWmAN_DduzfsJ|RyKvAEQ}WzFGTvq9wJ2RW?r72w>z$gu7##y`k-^oz&1T4P+{cA zmpNsd^;(NFn!gZf3hQ*l$OZz;!TwsIi)TwjsMc(v=FqDv`fv~ng2brc6)OBfWm!Sx zr)MVUvgKv>PQ>>k`gf*HWbq#b{s5`GNsHAG1Ysb0b}r4j3|Gb}$c_`l>MmA9UV3pb z^1BPZIp>!JH8C~F!jycVwH1dWPW0iGulQA&Ph@`)^jLB3KJ|r)taQ#&5f~D;9 zV++?QWbx1R{xhNZW=y``IqFhx3QCOD-dL5Vk~*htEfox{7$^={t+=`d2AO$khS^bN zj<{uj*Akthp=)zy((vchxfywaC^?@?Ryzrc)BWQu>W#husvL@5%**&iGb-c1n@R?E zh_^GSAD91R>(^Q&ZM8@L?A=qFM2>L`y_pkmwX zKkp8ouc_%9Cq1*1(@x-{6bObXZoz~JFn3ITAY)2=%*?lW#VFol=!Z+n=K5#0!TWRi0~PVNmE1LR7xS{!92@j!{VBpdQBa1ACm^#i zVF8LVZ3PpPWA)8k!TkrIk!k}G^$8Na18~#6fi)WoL(^?hin-5SrU>Zt8Lm3cKlD}n zcFCnX*B>FK%u}K1B4g;NGnwD)%LQq4;K4DrtqE^iC_cE17#<=XZBB%Z%=|asuhuIl zs46TcB}I;*^LeAjgFidU6e)g-h~a=LI}C);s-J&(%&taZ`Zt#XC&fdz5bCT{5a&!j)6 ztnE5@F~jiqvm;*P?=sPSKg;)7-GO#+EHU^Cu3t)^>W6 zvs)hbdD*qJU)0K&p4vfC^hfnPLs&&3aTZ)FiJM7S6>8>-sWd1dA*REYu9QBL; z{qm3(rFUvSeT512B$Y&1dhH#)2%e}W$Kdu; zS6`i(S8p>Vqpn3q(+R{gBdf4bN}us7nNIJ1T>Bg;n1r)jA3CY;5#YAoz4uZK5IS2f zKJTIi_3Q8GXqUE}@4WBoNAn<0^}l9yH{NCUlncsKqWEw!dMC+(x*aZP>u*5;zPE6e z$<Cz-9?d-u-qO|auiwNW|=yYeu(?Aw!y z_ewO5q{l0$N;Huk-`uh`k`Ri+vjqbDCH5v;0l&Ks92h!Q-_N)vK}p@fAbsJ(^mHje zQ~>vka)P``G}xLEn#BUiQ&|SDHb~R|B?p1XwLROOspxl@70b^c=I-?npjuG`#uW`$ zk-L&k0w=?;Rd81%zSX&C6_YmzLAyI$6B6#QZ9(n>xPm_c@QeizC@s(LLY<5Eg0#or z(Qo%ElZOnHSl{D+llp>&t28@VCk|YCBqC13}nTDrYF#Tz2_0# z>87T&@KAHs_^nuA4pGRCcJ=tM)d7q)*435eUe4DdBS&-^v8EyKCy}EwORAzdqCwg$u;2V;=$71Amu>fZApw2nw?-{mg`&ia*ycY^m-e5Pi+ho zH{O}KpCSh>`_A=9imF*{y-j0G=kpOX_^Xg%p zgz=a*YnFD>*~y1D0T6#3EMb0ojvwz^LX|r{^t{VCFLUih%jT0J`Rk;{hu`bc|I%*` zql5dOQdJH6$!0{ou04R=^yU`a;iOm_e5?Eg+v47PPK6sE!=7>FkH6%R zpB8DKyMy3JwplNWz0q=!iP%jej1N^d*JE3ZmL1@u1oDC0yqlDo@boO`D*toAq3hMJ zzpmRr`FlA{8a=v}tO=~9;qPQXIc>bR?1o6HTx=IF-tf$}_0jm>mt0vv(aWW7;o)qX z-~z^^uE!_wr6hN0co9`CpVCTIWR z=lG5?>T9%YSzoDV00B47W=tdO%JEa^y#*7;Wwd9(qWi$7X(Tk9sqzk~SC1xUb-Hv= zuk*O@_trmd=R`rWq~u#$ot*ZX40b&PYt9Eq22Jr*Y*5>~;P1zi*HxiGn}i*gCc-`? zF4t6g%TKQO72&Xcsu2m(3+8Rn&NKoS5QyF{K{3tQeuNS!c;nt>qOGKg0JVWbWXFWMCU-6IC}@y!iq@pGpcLIg1T*7(|0!a;M$9caP(B5HRMMt;@bYtImal+@tk{ zuS2sKs-C-~A>!x4y8b!=%9VGty}kY3dtzwieSkosZeWcY+#{y=*Cm&)FzZTj7}E|b zs=m!&wk0S{4>xYu!h>)HNBz3_45h>B(YKI zC7ti#Qd*AVtX z%%D#)>dxX(17H#niCWZIk~R5$xnYtr1`1TjOtJm->-k=m2cyTp^(xbo_a{ayHV6~d zK7kfL;)0r8yJ4!x5|#9IOemB?Bwe%$kOkWO95;?a;qBGMVWb+7WAboQ%oFX$4qteQ z&un&Zk`^zBUC{7OeZRdKynkZ78fRowg~QM=bC>i|`3P?y+C^Pp89QQa_7>*nE$;5? zzgl_9pP>sW__L^y*^6gOoTaZ@wbOO&vAOd&KV<+AU-rVK^0nwX{@EnRBl^Mbm$%DA z1uMCD=~CwmyNdF1(Gpx5O?MtRMA4|#vx-c^hC%3*-xB#!H zR!HU>HCg=?Z^r|v>BYAknhuSnTk+|#0x3{j{pzPau)eiw(*`ktqw~})6VDhDKtD|T zZAo;)70SKG_M?d7<9Mzkt%s3#=%aHzHuNGJOGqno4X_<6|UG@$XVZ_Xzo$hj$0la-IIlNG1T8~y) z_+m?EoM;nAR}G2@xWrZM5dZ*P-vo+2DQ=kY*JoN->}2tk5-Ky+KGGli0s3&vgwN^I zMjRwQi$2wwFPa*vd3!#)mR5D_XBBw^6Z9+kq|bG&3sZ*FO3tqbV#=dEEIxnkVL6y# zyINJ?i6~3>R%t?5xZlDa%*6N$G{QugiqcU3!DnDn(a=|MQnL;|e1EmY`tAp;E_|NT zPyN)Dw_Ej3dVDUG*ie3j{r2CDCvZ?yopj4!qlMR|=k+jJ)5UP`#D{rpHm!8qKYJt{ z(0B;De2nJblKTSX5#N;Bkl)X*B{~=Sg);Joxw7=3Yj=l*^|CA?MN)#i0VQ9qVr0`T zxYM#7!6ImT?_HeaJIl(-i=Vg&-Ugqq%r1`s{7A!$lY$hHV@|2M5Ot6(UBO{CsXsLd z1%SHG3xT=;9Z0Z_>fD8`pU<%l4}c;z{Zs`mO3q3!AE~K>D=-*33(?-{@iE%GCCZ}g zsZUFbilhrv6o>~y2wxI48lyjVa;gPKIq#st$%?GO#~c?3rzy^u#MOg!bgU(TW#yYq z4`#D1jN8@W?%kb%@k}y`{o^qSL|K^Yn171gD`o2pY#)sIS ze)Lf=#rz*1;}HH7%3>Z1*({s<#W!!eF~Sl$)Y-MFc$TpqqIsF3Xt-dC(!+Tn^APAu zL_qDZ5j8S@!W6S}ax7!tVW)HXx@OCkE%*2Ab`Iu?q?2nL7VOg(XPzI3Gnyn8%d-pC zycAs$X0+6ZA%7x;Ao5B=0BkOuW#QtDmQV_xuOulCu)fpquvyr0uunQaRz{T8+khS% zl(;dyI$TI9teJ8bXr>1w=oU2*MaktRT;-yo$Ov=cvM0mDD`Ak_*xZmipLZFyvMY1e z+tsPnN5h6$EG|;K6V``DyPI~_3!kMS@5Ga|R=1AlFIP@b>O-1Ny%1%-U48C@^l5tn z^*ob;2ArSUu=ca3^#s@!&d2G~2w(^)(;x{{l~CW%qAP7UOav^ZtnID6MZiDZsNCg> zD1ENgg})yB37H!vdpkzu6t4rwpN|dm*yQw091jvHc*!^VYZq! zZOTWsTrQ87F-28R4@?c3vE!%*^2e;=uUZ|`z2rBsWz%d>O(K&b8(5e~)Zcw8D=YDs zCk|o?my~xR1=lEsSz_q)beio591?0CEUKrakAFqw>0LUAe*t2t@A5|r6|VNz`T5^Q zWlF*#H?2k(Fb7qGdaK9r=lNx4XMZMc!?;c5yCscPO)J(Eyd6tL&eTsp@o!MxC}>Oh zZusT2gXG9e!&Kk4Z$1uJ1*vGH|iFAV5>!_X%;eR!)Aczk>`EOa~#Y{`I>g!AOPtb9} z(~35M<2)BXjucq_$%2-cn0QrBi={Yq`skn;k$T%E{48BsbZtX{S;kgYRY_?98lk`U z>~xN|4SOrqT?7^@obPDFz&Y>l+qrxsl5p4R@Qq~oA7aJ~Xi|K*&6rB+a-_?lVCHTwlcd;LE)M%CXIllb~ z8Ns7P$x>`dB8O&)#Pmzt(rz?75Blx%zf^s*^nEvV+kB4x=r6O|C68C|1|g;m9WKTy z)Om@2Kkqzs>D8G{?WSy|jpKgP=_fjTwA}nUWVONeLXmzUKhRw_1(Xa~S67l<;3=J6 z?;2=&6h6V){SV5>Ct+vlw)vbl*Cv?G^3-TQzs8Wj?!;1Y6S2|9R9wnGWJi!3u8TQE7 zgMo+5cZUlKD=rL0HL*PM{L1(>Jkv-yFG>A&8_&BocIwiBHMoOjkvcnc#K#`IJYf?Z zR{UO0)}S@gcp0}GI&|zabAqGdl?QxE8<@I zQKgb2d6t_BXBiWT(bTECPhWgy8%m>rqLN`jqVKchk%Q~;!XF!ZOET2dT3==3Q;ta!6&;?)eo z_9n^FeeJ)U=FgXmh69z(vXUTmKm)sQa?(1B>(1?^C(6?DxZ)jit#lOC#`RNM|D9!=I6kgdq4YTEHk^CJ~ogrD#TY`u6*qvzgMn=hYPTdIG^!&z&fp=ZIsjnjG_ ztyZbZYZk3H1-SzsgA7ps{jW2VOAJ=Zv5iCh@uw~Jh+!dIrDIF2Y6WT9o5kdUU}ws_ z^aX>Y5CNFDhA&J%8cKjkMvUy@)V=eZRB+V8R98uxeiylmi-`xT2EgXl&7y`XW%n2w6SGb( zBbwk+k45k*0}(YTl*FAlb3RManps8ux9Szk+W^2Se+L|g zjb!ggiHyP?L(D|@UHv+jiAO@+G0QCH#bs?+xZ5!%@l0%D%Yj<9h^2T!BnuqC75eRl z1hJ9I;d8p=R^$FwD0mBc+&R|wE>@a2wE zuMtndR&L>k+7Ub?h)*|T(gdA}xH~nnU<-ehCkvm3jRdr3Y8eB=Gd{$MRf^&CEn=W4 zYAwlE7udT`pNIa*XUE%wzZAswWOTHJnHxgaCv1`I4`KQWGhq{1A#kiWTYHgHmmPwC zC_wBsXpqm|0WwhM0VmXg=D*9^ma6|L2tGm12=eM)AuLq%fispceeqpu#xK=*$AVRS z|LybDTn6GzQB)qk^-C*WF_h;j!bZTixKju$0YVNYO2#s07fI=X3(-N`WGNzA=RY8P z`2hVVRa6n7+mz5AJiS+qXHA*1Yy6`wI5#)n+<@K(#>=8`PW*Hey{!mp9WzGAvAkf> zB549pxLJiDGbq(ow_U^&CTa#_Ter%0T?p>%I$gdOfA(;Q!`kWIp6BqFUh=4k3+Vh_5$X@_V#rG?p!sCrcGTWMz9oxJ8DR-^D!8OaJSH zOKuNI(frJH`1R9Qra<`DE2m3R?Gn0aX~k+MuUNYDjwD=8w8b^0E?itmO!SQ1hI4Z) z8t?ZA)Ew}?Fq2usJo^QTamn_~@)^%;{@G>wymNHCrqAY>i5vg0-1K;OR<+tvCJu4N z3dZ<;ff)-sAi#J&5`O!~>e}hL6r5U1>YdLKb^s#(U%qRAwVhQSag^5wA~))@hOa96 zFeUsQ5T+0d<92uYSg_=u+fhzMTH-p3`+X{gdr{acW1$iwpCu+TEqTUUhra&p)sbf( z*I;h6miQJXs$0p^!B^t?I)iOBQUz>vRb|4&e>Ye!mL%NuH+ZX`iAIis1g0!_-__YT{X= zNkO4B$xx`~ea4!as>fWQAAQN@^z|DcnV%B(;^5pi4Q`L z^B{K^u+o7nfOcsnP%P8TvM{3Ml`b8%Ne&{r8sa``XE9L}0N2q5-lk*6)|0@bloWGg z7o;W__X1^Cu(aF-L_EW1PS6Yim-QpnwxI&y(d<`cD3S z9ISV*KUsW-3CiWj8=%b5^j$v}U8FB-gP-L$sg7IFw3ON(nd()h{hEJ6JM)3dfvp~a z!C$AO52(B3YJj-ERh@hd7dIX0rP!bKHWZF%;CglDRT zpUnnK%`y|hdN8o9?(W^OKfvg5_U`HH9uUmm6$Ovya}kyRHbF6buy6SkU}QJ89mp_l73`%jgLXUx!LEt>?$$yVll z^e7g_olr#pJNloUWM8FfRk>1gbN6c~jAX9alqpSRC5zK~(8(ZMDViye|F2r}8$&mXE0!T@Vm4_iQUZ%bVBth42&R2R$oT8> zv`S}CBu2hOuhv7`YnO`>mmF8LWDe~q8kJGO**O}2{6UH27T)(l$7CWkg!eQLo_ThG zq%E>FO-$NbYXia?Q40+^WBW>^Nd$cu_Z>XA6236sv8V{)mti+Kp%;oD}&r6HVr?V@?@_XJD4Lx9DEtP;B z%bt7ZA6`3_ui02ti+}yQL3Ij`mLjXy@LYprV$G=b>I`bIdc4yDF5TIo)>}{OpnHQt61xZ!=kl0~ zuv@|s%Oh|kAT(69#&<%HRhpWJsz@LZCiz&O?!VNOA)qSrR>T2n`#CFKkhwNC-5J&D zNrdBB*LgbQPfjBXSNi@?k!$V!2`%r~R2D_7b`u1kNZ3u2)b$D_3Q3G9^GM1DTqw>;5;E^Tj48syAUb3cxYc0 z$-N`NN?cI~{lcrOt3hPzD>BdeqQ#P0bnx|2Dq4eDbH**O!FsGDeiiDM&fUt-iyRlkB@3UE8^$~W0wJj_s?kEpxqCI0N<#l5U zqwwWl`Fq?-hF!%C{rDa5iYc`%mxPvv)*WkCVd+sNbY9OISijkOr8FBjoEdWs_>VxIV zf31^+NKvgK%>YpZX-lc6$+zx0c<{auZ?3q9--);tgF8snrgEv)ulV$fNQFNc6ems> zdiFm&L||W76+Dg`6%}ZK!NJA%%Mn!nC(ZFmoeHVdgf0&?;J~LtL-K!(Bu!Q#qJy~b z-+?=2#D3;u3nuWjwLCs-45!YXJ^R9j-%NCI7)ZO|(HsBMdr{Sr0`kupUqpjEP3>E# z>Wv_}LiDZxeDQffvQzVIAk$sV9U(Tn6Q@qyS}2M^1}2P=1ug>$4+~yIA^O@Eo3*GN zC#r0WqIP_-yh4{3d7W;9@tb)_O$dfFwnspCIrQIO`cT9lQUHzcVBf@fEK1_RhY7Zf zCE|7fnqGjD2YYs^C6TqTl2zshrKum~x@y%xvg%qI=Y}QE8-oYJec(&_UjS~QQWyF{(rg6eTY!txrJ=|w_`|3=X3>U*CXIy5^aVa>4VwLgbUF$rhn z;)s_}rO=?8wy&-W5ivfY3UjR#0= zfv0_Sr4v^dS+!9+@7}r7RZs6)yuriVH6s`ulnfr*w`EG&S}chUpDL6Vcj*PmdqIbU za>Y0-{wVH;J&DqVisV_bbjzDUulOE6zy3k29+QucXpMUq3z51Ch=@24Bt~%UR>N8* zx=&$J@oETF0cVG4W7mf#nPDy9;`V(f%p z{0bXCWXKRWS-Js7*T7q5W=}KfpJ_%+-i`SKt`2tXCph|)Iic3|Fm>FiW~KXlr=^Lc z<*A&&*$1t>SGQxZS-XjuAYEa`8K%!N}L&5ULWaFj|EHS#Ok8eeKoE*iGzhwPFgtmmy&5J%{lA6H97hQqsiin&2QZePqFw*74(-ErT$$TN@lO-5^w#gFaL;8krx74 zF+HB+J}n6|mvoI=@Bd!6Gp~Q9sdeES7gEz&Jdae^|&`vTH=|5llbp zi3Ep`P!0~XcBC%|3fKjex{nFv;#Zy(yNawysDBmk2s033?ZeGJ554(u%`GyrDH|Eg z*mT7$^LBjS!zgQO!r{%L>~^}D@LQxDh>acRTv>1r`C0QNbljB1uW+qHwWHO5Q*DHR zk%B0s6mJJfcn$UOGq%kzS`$DxgvF)jGX-X~P3*DC?&8a%Yn`%7OA!R2=TaJGYodjU z()d$0z{*q-jj{jXehYN&;&c7~MW9APp9E-b$g_eu3U5x*gZ4^g+3USa4$|aKO4q(bDa5gM3u^h|O#HWblTaOp zSZ!(3a>{6N*@`A$+`pY5>Q*Zo)!yAEW##Jwt)qX^S~skbO81dj0~1ede!973Co~hE z(Z2YHhCZ>$be<-d9km(ah@4x0#r;Q*ViDVY=WLB_8QV$yiA9%`>Z^@woeA&D-$o6Y zaiN_hstaspxz`ig3=mdO2#}(s6(6pJRzS34FYf$-FSFh7)u$P(?jPr|`LDdT3v^sF zXElxszB)FJ?eXd9>5aF``<~~=n}y(ti|w7gE?b*8iO|WobR@;lBdeCQ_Hr8IvT|ro zI?ZUB%pa^T`;x^1LWDH`Ksrpan-wmhT-WkvfK zr2QZxSvIIFyaBATg&j3==PtU#FfM%9q;E6`6@WOd?p&u<8X}4e<^D+|n6f}WsBeiZ z{PUf6UhJWlh9i_31?-yHy@hI+AC`PSbZQ*zqz z%Qhu~IRAk@z; z__et(!3pm08y1VxtZov2)yUVTWG_`rO&S!?UK9T|Ki# z*A3e5c+Rf5b)9R!co)zha2xSa^y<9zrbPODIJ`)b#^+qjp8^7c?p>7&_pUi}w>0~P z-^=82(k3E4e5$e7tqZ{6S)En7x7oo)6kDM`8{ z$Ks!vJ-tUONgZT~kZAw-RTL`E?t_ug8g$ep&~7Y|1>YC)xZWXkm?@0%xZCjjhz;7l zBI*=St$fJGjy-hmFVD$Xgi0cT=CjJNul123TK-PF2(YdW%D049KW zu|l>`6DbstxICG(JA;FKPKDh74w&7+zD8Z3E-(6P_W5Wl4O`0h;2;~YGG-6b;gMqI zFSveSf)>XGvOqN$gS#n6pQ&4Yyxw&gM%Nbi{u%g zA(EELvQ?q!si}^9Wk-MTwTyf7;T8X*98_PoQb~mq@91bfwXO;ujE9~2@^EM%MM|pe zX23t;ycH>jlCH98;u#+>=?Z%=X`&6y%^EjtdKVzHd?_HN=}16EvEQObfNQ<|ajC1x zPS6o{-k%6wSUe8f&`w*1Ve?ChusxzAQQ4A;)8Sf_sXx5w}u8NN#Tjlf^0F(FT;F9F3XFB zAdl*EWE&} zCa4Ohq?A`7SP<0njw|;B`mZzYM^;eHf_B7`pgIP`s9|q67nYVD1m7CmLA7=3_m{_F zfev)4xv<1!D6PT6fPMoziZxA6dRfGRWlA3((!cZu-9wfvUVMX$)&d;TS9k6r{si&E!EUWBV_Z9f!5&C85p(*{ zq0PkfGVt*F)~#DB3bNyMp=qOGZl>g|Pc(-Tdrc~tL)7vGj1T~IKOdip^Q>^2wAy@m zGTtY}E^BK-PxFoF;s-HUJ|izSp#;T3b3B$t`Mf#;oTUL!R$b6$jbx=#Y-96Be#q z-#ZHvr{RBqewZ7Iw`51|JQoKyKXdheCF;rinFcb9MBYwoJS=cVFL~4i8QF=t%bjPd zTlem_7xpu!>l_UO;$Hm=CK)^zmQv)Y5!QB|V=V-iU!D(PYL(n$tVS;a33@arj6fd8 z-)nwNbWp2SchVPb_i(;|5Vec*=Z!x|LEIol8Fp+FQXp|x@wO2vB(T@_Urx#Y|Je2X zn1^!r!K2P$uDnV#g}%Tu>`{fj;_cx@zTN|oFOps*saCA%V-(3E!k{@B@>`pFA_6F2 zN)%1nl#AuAN6mta0Dgrp1kk$@Seh-Oc1(4AeiO1ybPMXD0Gc~vrn=QR94|P=8Qos6 z&A+-flK%skMegkdM&D-6n&r;g9$B@X5FPRDu_dlPo~&~pXqXX1LEN^py8Yg+MpFlVwLBHxpeL!^~y;Kz$<`8GxzvGF@bzZF( zk&6qN-0`;3%*sWOtv56nYMp}+4fIIHm7f0ue1P$sfn_Tma** zK5W?Y-tDujn0tgnDEy{g1Lm;9JNcw7A5YP6%?@8#>3pz(Yo0qD50fW1QNAq*`coce zOyglIyhuEV=%@k%@8>$o+DFiNb@g~&{k~$S`0w_Pi03z`Z0Gy?)OIK;&^JG`UHdXm z4X}!Rb??X$y_yshWe`eQTFmA0iVC~m>Ts7_CJm*4z(4vuc7(~);HPg1l^1-EnG{Ia z;HvY7WyC_`iB+V6gVFsfHRzc{7ofC1gjj{+N=Q9%9u}OX0?+~AcI}AcWNjl!BpLzY z#wnwcw?k5UDVo{a=QBZ^Gr(c3D3mtJm@Ya2Ci^?C;^=X!TKnIZru%q<%Z6{miXlOWHDPWDJ5ZF%Vd^*Z%wxty6As z4eDOE{@QcEmFX&PZ-?poJzTHEmeX>QvFG2wfC!$MTf8K! z6{Lq-#mjV!mf`M7ZBlA-^2?<{Sw7>*o1tvBn@3Z9s6$U15kpn;z!>DH#_-z@*_KwB$kRYh+A=4&4YM|3_(3s?Qc zcPewjn>Wsj{q7%REAj*<+AeM)iO<2WUF0OTAxwAMkJ_cZ3Ve`ey`$Mg&{AY2si$*F z##w@oR=QBHHHHVmo%vu#)qgj5E3$=epj^9q9b(k2D%q{TRUr}Orvrw4ihYBe-2(0v z{(t|2srBgLk^cmIJT8l6aO+)bRS8WIg9hgGM127at=y`HJWXp@C_QJLcvnep;1>P2U7zi z5ukh%`2|2biU##@gx)gmcgeQkRR=6qRs$JQvCu`KKX*qLfpr4#{)(1c!g^p`$%`iA z3fha~a@_!USs`3FOi z^*x+H!s@CuMal^GjYaTe#t=@hNAjsy9reiDOENU>95G)8HK<#6zOpkLV8+&sQT;QH zSWB~`EHpql3nyxjvMr#e5uO)zIb+WqJ8V}qzRen0-=Y7e4)iYZ zbtgp~JILTu3~uYJA!2Wz+2KbtK9cVYacJj!qk~DwTFD2>tqXo#biprurndY}t~i z;J@00U!N>#6;)rR$6Y(AbgDehF4=g=Jh)75O@Ob1p{SYUeUn5yQ3;PYF0M;>vb!l; zerhSoJU@#VT)`5>aJmGROC7;%PV&Xz;JZLrja8<_e%Z2a{<-83W23}*Q?@rYN?&<# z{1kahd`~t{y>{M2{$t{9_UL1mMtr(Hr}5nGD-H&^JTC7!Akv0g=6(9?{L2aYc8AN} z_C)a{8965UMk}_<_b~aRJI*5dlVMmS!^`jO8rPk4oX0`awUre7x|OVhLcx>b_2AIV^LH_x(=pqF z%#n24rAxQa|49Q1L)jmB*;8DS{_)}Bl1MrcHC7m9u0GGLxNXpLX|Cny6}1uXPGQ~dw(i%fzb=WZZHda_F5$2XC$Cun(#>mcs1*N} zWkyCuj@J)oAHZpM)i^nKS0s5xWg|6JO@9G>lD<#lTobpiEi0=cy5c}skD#YOJX5yc z3c7sTh?@345vQtQW)6?GU%l*PJ#`DT)Whw{aA4#gN||Fb1OvB}J+lfA`{vEW11TII z|JQvdBgc~8!_4JuL%$NC+tMiczP`3^F zI(G5ECmF{}^A$%>6=tZsi3&R2NPVt<%LdEmtctwz^Q*zr=V~^by5-loe=c^^t*kPS z)p&(C!A1(f3|3(FZqsM>1~5o3zo;QEKsFHJYnK$S(<6i?%>Bse>1Jk|*rdU&B0CQ~ zJXvIIGiD@?$-H*($dL|`*ht=p;q(=&(6B2^AkRbnlUM&Wp5vZbAnd`nZ{M9VXVgy} zxM3TnXwhOUAx#iIg@+MEqmrRN2@5B~jiEIcM&S_1H2#a<_-iijvS-HmXX0gMZz_L0 zkZ(S$5gQd+hm}z(c1xG`q1SIW7jk$r8$b#1 zIskV0a~h~cHS5`{*DX{f_0B)*6;#R{gI*`&@>HW{fy5|?BLTYOXzKKn{EtP2O;qc~GJ;Dm&-p7RTEdU-19dy{o+oDGLukBAw1!h$ zp31#qWTc|v8K!!;d`OSLW&1`p@*Y3@!Sa&@pIo<;4=;#HubP@S49Hqts++%L4JYGX znuDq5bovq5iFB2M}uh~@61quN)W;J=xS$A51Jv7$=dx}dIv&|U8eq_=`OHZayV@oy>Fl*2!Vm?hPo-q5DzFOamN5#4?dXBibxIt$Nk1=G> zY!W`##igpdACaxFGkV92srao z0xOaT*5x>;`DOZYCI>rT6X$+N{PwGM`k1N|H_NFCp-B+odbofmfJ_jL8t5{R@VtoC znt7>tns{pkte2=h3H32rHU*$vguB3gAj~@Jm!ja~X6I5an~+m-;J|^mJZ$?OCfD7- zXD*_qd6p}D1A}pN-QollEKSF0veHr#zjk=^CdOuEIE5~2q%!^p+&{@%IHN9)59F9YY*N# zZ-Ze_VcZa8MBOzv7<}>E;ODdC9=o3E6MxAm*-U;Rzm_w7*)3fAeEsD%N#kA_PS5!e z(rTK^>Y>M+4&F)OD81LF=(7)L-1iKsH-fcz`8r@7m2$}O2`|ip=UlE2Oru&MfhlSE zuB8(?o=$>Uo7T2|;y)#?2<{%vyZkHMf7_DIn{n!2Jkj(;3cN%Hm zBv1d;&maj~dP`+E*jRi? z$BrH0BTcwpYI+Xf*Et0f_9>JF55-g}z)?vYw-4-@$`4HIOoA#cT~CoLD1W8&LOn`A8Ei|~ zaH52yQC_*M zhd3pERQ46Ls6*CY0G4Da;hh2YR?8Y8Gib!>Ga2(5sNhrB$ro-IJkwyMz4~ei>R`kk zb7|X+kAZs9n+6(AfwN^S>QmOnEuf6;eZ0eoRGeR&{S2^hzZe^d zBiRLP^Y^|BE+hVHR|h;E#R;o>@(7^(PV_^a7!jSms;E2 zbNlw2h`z2Sj2#({c$F;EUq5LpxeomZd-h-qr9DfyJYP8%P4T5`2D`?8O!?2Jl5;V=Alm$Q+lX0*;#mRM^{=H0 z2_tC<uL)8R_RSheAKUOU>`fl~-Wet5^1E{w?a(fJ5PX zm)evWod9)=4*cPW0=TdRq;~MaD|Md)I70d83!ul-DP*s^(rBmOz7^NCvTabEMhDBY zUttd*tsVE0D3(6UB$PvL*7X>pV?S4~H)Zvb00(n4u8gA6UY*Lw^@%jNUcr+%c4L0f z%9cjoO;+0WZicHLM)6%89lQPEH(WMqas2DBsLqi@6ydFpsTgC)tAp<+TufhzbBN+7 zl6n}F3;B5iqmSmc7Kpn4j0oe*p|DD3D>xFA)=k|-GfnB&Qv^}gnhC&D3%|nl2 z_$hqH5>F4W|L-PV*fZeew?%xEaMA=rf}v{ER!_GPa~^2qN(>t}SNqSU%YIAKVoxxR zuf6s9oln#-Of7_jJ$N^S?qKPceb%7k!nZ1n4vRZFQL6O;>L@6ZkX*)F(xOW7)VNX$ z2>fV?9aUgUC0&!ot;t@U<0S^_drgMHa~*dRLW1{yG-cStZeT}03uj2OWO_L2t6fi_hj(So88 z`5e@G*a@?a2RC58VNE)bfQ|vfJeDc2+jYl$qa0OYOU{h@hp_gcU;zAL$*eLD53ca$ zk)90zr|A^}aaBr<2p~m?Fn4g#L#`@0Jux`C^3Wg(|0TT>h*)0RC>?Vkg%OH`gn z4r(8*+7oxUW+)ez|Iz*X*)hhs$?;EFh`hW|`(L6YcQhp<7<^W(YC!ztaM2VNU-Qgp zXx}mbqWf>Op_W&_ijXNBJP^T#`Rgot{GJ+o=w^>=vc_R9tbq8o?^S8g#Lj;b;m^*v z5n$pjq=z#5=Q^puVbZ`pBzCey7yrw|m;LH-+SrvY0eOXGpf+zuGkKmSB=*AM+OywD z2`*H8OQEmQ5?Ye0;ER0&_;?;mjqO=|FjipL8%{IdOoj*neoJPORGZ2Wfc}V7v7eaQ zVNc~rAOI*#5U)ZslLrvsGW6bsiSU;shB~aBdb209*J?YgAci1(9rwhG0UCBCQaHv> z^l6#c4&-2`41>_NHKY~8uuGp3w5=eJohIL@XQw%k;M%cIlw+MtFhx(Uf9gp?)FQ`XFgS^)12 zGMvLv0y6>CqAAeW3^{m^7;+7P3-ughI?=-{S}iE2VGS^ZjJrw z>MP2q8-~}BefI`5I`Hv0V{yP9hlTNqyD@hS)?{OnYMtbLtea^P6VnbA<%7tFhi!o2 zpPM9VKT3wW+O z3ond;!#OY&q>Yn9Z!GETH)0%zw!d&j zz31y_)D0&m1+ZuF2OVC2(+{IPV)CCDXYBtrD*6*cP9#_>FBro7Km#}MafH8>riGBZ z>u7&e(TEzj45XR7yw)=#9IFu+Z)PC6R8Vq4Jb~#1r^plL(&@;{T!&%Q#WwuPy?l zBMW9fNyu*BzHQ*g%*whAwk*Ulp!*^pm=ewRuphNuE6$#Z_u&fZ9iheC!w2k_ABsvh zV~2oZ1VSwV`KV3(H%5Evq=z zb~eUyHo1V?=ctz7NV3uX{WX7NdLo~T9*oL;;5(_Aw6Dsj%=)g=1pgW;B4O~lm9bINp#1|PO zB+-%@Z3-Z+NE--SBGLa-%ZF|VDoeZ*|6&V5Lb)v*1jK*62!McUqB|?UO?qu6`gL5n zAHd(pdJa5g|09y$VAxDqHE#yf_==&gaOf+xT<|KwCjohx=54qPKj~;cmvJK>q>X=} z1xc9epF0B`>S1&~a0(%76x}#LvxFScIU3YVwU~zcC(zMs(%9{C82ihQhp|6s8zduc6Ga5sy+#+OGC+kU8kLZ`B2i>KvMQ<+SyjtdQV3#!dwZs`b} zm993N6KESrbT1bVkqr215Xp2YnEdg?JVpJr!P74%kbZT}A&l|~ zUvw2nG4IBMIgBPdH#gT-$kLd$p|BBT!&|KzN5o$Rhv((-1Hdlk34w3Ej(PDxAmt5_>d}m>vP8sEAi!qgKRm#p5gQ zINStpi*eJ)LZ@QanJg@Bh5GZ3V@t-_XO~R|Z(17ZHq4|vgJ4Fk8s`SyfWEA&7-t7G z#YEzc;&`>+F9*$v6*u0c!;h{nIxfzgKHZE<*lOz+mFyb8+ZCP~Xj9*U{LFZ?HoFBo+|+`UWG#RNCHEEz)Cpfk)yE*zMN5 zBu&(6sEMH8PnKyhG7@icoiJAg|Cx`E1;mQ0T^#JvjkuZm!EYB%lBu?J?c%`oZ~=(& z39uh^v$5VH=c4~e98yuSNU_|T+h&k>RDRtu0Wiq_vwy@Z@txdYjM!n{c@b1oj|p2i z6_hfQZ;(9^zQ02Zt9&@4){QJh2}+s`V-;dS+t~Cfl#f81Izg6woAkv~oe7k0cxONw z^|Vl0375-O96!89zlUyQyoa|1_HEQ|eY180xrV{h>yX)$Rokw>&eZ%hX(gq9C-pnx z#LYm(4Js`)lD@I(*kpM3A6Rh&X2BSQ;6!}26xW>_^tB1vV}N&vFQ8F2cR%cHLED8c zIbT8K!7;p?$W`42b<)uZ14BoPHb$_fLN0aUU@*LRRQ^spN$*bTkLnro*spHey=Iz# ztBO*|f;?81UK6CfBr0*5~)FUZS|6`NA^Ee_U7n z{QL?~VdaB=mr&gJ$vO`?7`?zHQT-|FNnl!Ze<0>Ms1#2I77891?&U42%bMx2q*_KZ zXuG!7YeVqT(Ws4f^PJr2)mSBPi0pm=^x#KN;d*=iKn`Ct{xOB=;WNy!rYnFe)r7`@1S-P1+F;BMDTk6O4c^DvhQ)A**XU43q~B z;1@{z#mmadT7WHCs^}W}+*hEw5upG*p9bNkpc_DyD*-}5WEobA<0eTvsn9&H;kmzd z{E(29y?Bybi{sc$kBoa*JTYQ3BN<1aN~jyhI%R`xdn z?dJRVj5!-L{k?k;3apYv@8|!|a+N(vi0A}W_|`&UVv4KKHILVu31sf-Do85NLe}(G zly!&>N)Yys0PBP;&!CM#Hun`QzZrZR5Rvdn&@72ha+|ri`O|`5RvSp_53UG>a{JAj zxBmbBMesyL-kBY+W^ z7JHH!H_#ZuQS`M^*Ci?Z=5^d@e>s*U_V;!2Qls{>H|L;BVy$Mq&@o%sR>zCzC zk|9C?i1MVRWuOR1x1@$HDT*0_D4r}t000}<^$ULuVdUjaU6aPu|JZ6DR_lOu99)Q} zf?aiB%@On;uW)NEwF~(O`O(jNL2PhME-{vtlA=+nF?$XZE(_Y+Vh7aMEFEJq1RDc_ z>=~Ir`HRdpMt`y!`HD7vtAWA&3c||GoVyk!Xj$eYtw4Rr?z@2F#ICSeYJ118FGJ%( zO&88h#z#XK#J~qWDTltD7reY~Sp67lwa2F^98j`THPlm4nLMeV1)=c}CNCQv1vkQqc3OSu5@$q&qjvbe8RPE(Uqd9^MXAo#m z-u27|9T-N}yLTB5hqu;jL3^aK$5M7N=Hm%20*+Pe;g%_3WW%Jh`UPZcal%#FbgMiB zJ08-5?F&TH4gk?Z?dg{TUWKXzvR$ZpKO^FzwcEky*TqCQ+gE($S2qo z*%Uh)-&_~>ujA5jP$I}zV>5i1Cx z$+y4?0`WkY02kpTipgL|b|dfO9|r{iludlEJ?}f6*npsuZ?}kM@*TKt0b<*K{9K<# zKlwt#tb(EC)IQs%QRy<)=hC%SlbVh3FMFMH46K_X1u*vjVBGOT;iD%uZbq?6Ve_NM~Yo7#F(_q91O=|rU8yBz> z(U}&6Nh`u=-vhoO$Zy;Ai$dbC`Q$U~5HXW;9+m~1ClZ4Yu2mW-AM}dHuuIgX+G5vr zzvw8@AK?#<+tW|O>#HATUJ7J_B8!tqYum6{DNhK&q)RxYk;~wxu#61_IsmpfIWVHToedk|Ly+%8xM)@$Y*Pn$7+OVL+v-XYBT5`Sa)6liS2WnEnQM@ zOoH@(P^cKoGpZ4P&NZyNyy*gln|i2URKPb>U6ZsG19^X)mjr2tW1<;*T4nlZ`)J2{ z?HajV#zjWfMZ8D!k{b{U5IY2b#zPbYp%VNd> z0$>hagw>FP%h99UluXCLQ9q=Ei^%-q5@70peAayd5VtE^8dfKNbqDD#eK(;ch9|P! zi+@}npkJj(Kw?-v$EB)AA1!=)iz@y)3i)4o(Dv63#u?t{_mCJ1%S8{@lLxKR~M%mSDcB7v0<%^>Un>`?}owa6G_3n7eK|dGg-f zb~I+V)&$4|PQT6lvr!)YJ_0IZhV1GLuVWAMc7Gy> z0&e43gpGDbZ?nIdg;@cvA{V}#xly`#)27F06=5JI-<^6qif}r4JGEA0DV||C6uRSP zEhc+DK-fC?^-N`=v2>tqJ3}xa#S_+$YSB0wX0qVPwf1tu3Uyhi?PM%_`gYtedjM0G zFsI&yILx{9*Hd5uR4UZIE2JP047?vORWUe4J$TR@@ty-r8Ej?OMU}kX0!Zb-mLjm| zpmW-}b7!S*U)wh>fTK-0ZuHVTL|q|R0opzHbWtlPL2_^oB0d`-IR|v0%b$K;sxrl= z3R*6x*c@Y(2VpQ<$UBsU8xZgUO-_3V7v!qg^Y7V!J4w9-CL-cz(oin;4tCyoBeRP*{a!0wiYRoFDTj z@hrb>rOsr_0H*NuQZ2{MEzmJ3Q;lwzgOL-w(lNo9O!(S*cza)i?Z!V}Ni!h08i%>H zZc!4r*!#;VRgS6C;gOoehhhkXKIU%k*l5j|D{Rz7fzA<2jh2;|&a@=dVY5NJnK^R; zCCQMzB!}icvQ+DBJ+?y=ND#3xUh7r=wGNzeRZ39mmg^T33y5goFfY*#?H zfpLmKV2c~_2oVW17l1*1-wHd*ufQEhvO9b#L7gb# zKU-+3QUhRbJNso-`wBdKJbuf(=Uaot69uDVS$g(X-%mcWNlzizE}KrAfxwUy z9x-Ud+rTz0;Y$~`m4v!uOZblOTpuNIlgF0A*X&E~=vKeO^w#{=>L*q|KRS#KRgp{V ze?t0*k+3$Q{qm+L~TIG4-fc%0z&Ar zg92g++-6iQl0+sn$imVX{hLA^Sn#yTJk{O`cGLjO$5nLb1b%E5mwyauA-5Cn z-iQ-(@tr{qIwx~zdz0(YzTAXF3&uUQ_4@Rf{O)IDEQYOszr5`^_#SS-#7NQ{cnzqS zT=;v|tWTp8G^zRDa|VQ>pnpL0Y9A`r9NY%C?%ud|?GpC&g>W@*!$nK*J?<*ae1i1x zzjsReHM0ZpsLkxFhMds#FT{jY%1|(p_Nl1}&^Ae0TB1vt3>p42UJtr)a`auV#GZ79 z%v9dI866pB$TO~88(-XBCd=9f2?<<7gbu^ZOY)z?h9k8Nrx8S1bj@0KIFQehYaqqL zFf^bY&@iTBU5ss06Eb^mJ6`bkdK0u*$I*c%SQuU{J@wZ%jj_9Qm;3IQP=eBSu5!JVShM6ON>A^@o9!^Vy~%uggi6dEF+ za6-k7RSPiBLw@>x%&ZZ%L|G%f$APc!_=Qy+Fv~5B1dcLiLTBl!|MZj77#s&GMu6z` zcZ*zypDp@UwScvtX)wsNQ>O4YYvA`s$8Vi}*alhy=;mF~w{CmC`ye(L2zu34yQAcy zNL$e!V*ntxjHr{I5~cb@AM!&`<`Ey!LVau zni!I*x-#%qlixHDoDU6>a01Fhtz)PnVF)z~nX2ynGM%CHkpQ#|@bt!0Lxpy3u!KI< z5pq9L2-t%cOsyo^@@-jLjXQy2)+32$+|UG{>KPY)%t^qZKt61>$sjl$5uTLJ7!Zx3 ziRm(*guII)Dn-G7rtD8xB5<8eDFTt4!c4atRkHbbe7y}&{XfHHh)xat1W*dH5(DB3 zxXQ!F2QF)U4E;nON)RabiP%0Tv(5*J5C&?T`1wB=%3gr~vWI#Qk;))D3-I}6x)!<) z>=fa20QwO)^6=vsmr&Fxb)0E8vPsv394`+Z)}T${&j#e3!W1`TZ^1Ng558CEuj%ej zOgOlrlRC3bMFCbwyh-iH2zMZjL3ele0+|lq2hv|q|DYDg`)i1LmQPC#YTcG6%15B_ z)W4eQeh;_Fe|jDw!uCP~h`p5~8vqf4`jZAU2kB2%fOY=uZC&M(;L5 zOEp10Dh-~=D4yRx$|BI<0nH20P#} z#*a0ohL^hK4}5#Hr`JkhV0RJMZdG6Lc?fW*d1Nri@Rx0a5E&V~+y2a(^kPE!aAgEw zz78U6w6fblG5C#_(~ClSltzB4YNcuyt$1?c)Ph6O6sBd)G*0@ktPcXNtUzAK#qT5jS8uRJcrq|pa^L|5n;vo&2|to>fk1CjaVqf0qeqXZup-ig zXHRC_M-W8Fy#-*CVqqx{jX48<8pUf)JvDf@35LQHDbzr_9NLn+dv|$U>$+#pI|S2! zy+gq?03bt|0k(IKt(1^pL_z>}Nv?-(`vp1;tO90CNcg1w2isr>FZmUNSsy|c_XwLS z8!KKFlI5jEkF1&Q*yg?F@X&7@FKEaqfR{p2Qs}Wt5XW~{-YkMs z_PDpM*MoUUy0nydauPQ7bLQkg&%A;`k+=-X*IbMi^M?K!cGCa+_bc+p|AhQu0&P-Z zV^31Xbd(EVCC?dU2Tca*V&rqdacy)!Dd1ZA@yL1`TR>E@rKn`Y#aM5lqK0;eOcg*g z8{OBGq#g&`aw#yqKua%aO(YXCDML0Wl|914!`;*P*cCE>bkeQ`P|9fw)yselmq<%r zH}(!F4~E$lrV`WgOz@9iqDo>9Cl2Qva-YKS#gFstBut8HY3ro)=+Bc0Hi+6BJOIGP zHM+g(xV32V8qXITm#Xv)4+4>9q$J13+bmo(4mrH*iGE!VqbzqICRawFKp>}>uEFAG z_~|mnMu!5vPHYPOw2ilEG4H+p%-8xSE|`D|S4X+I7A-$Fm!7Fn9_OY5VS}{yg#(M4 zky-A*lMTTV37aYBQnuUw($}4`e1Sj!2%xW`z8G+S(obhUUZ0_+R`3yo_Ld zNPYcsfq$U@lHqtm=*T(@fB5e8m@a}w&Z*#$Mb0=mo=QZ#c5jR^G^WzDL50i1v1r`g zXWRx$>Rb=-pj5pYu5pW>(Jq_xbeD7(mvkwx7+Fqq0XvK+MG_|4y8-3L(@=^b1fI45 z_Lz8oV+)fO61`339Wn|TyLPN(%IJMWpup>!SJXl3;R#|+!#w%jN^!8#9?0{TV4O6H zXC@!SLeJOV0MsX*`Lc!)MZZmF3D-${@V6+H-_iu_Diil8-JuStP0r&}z!hd2o!>MHL z?WmE6A@eDSv+=?Lc9?VVJgyCLfrIb0Y41_A5oB0Lb_`x#^*m`Riia`t;e=>Fu~Dc8 z7eE@$G12$0UYNx7zi#2!?!3Nca%VWLwe<|l48ELnqJNMql40LiePwrBFjjYfLsrC${E*33Mm}zN z!Uzl!wwLYQvujrlq=WL#AwA^}e;c37EGWjE{jOzUq_Tav&V*aZ%KIQA`O5))#8;IP z_BQ&K-^<_FYsAF5=lw`KA(gc1{S^UOvGUqymFM>z`tw4&hCC!f`BV0N zBadnb!!GME2&4E_q8>l~&}K_WAFP5DmlNqY&KJ`IOdA+@WVTr!mZ4Z~8*+7wp$Su1 z<(@N%p&MNB+@YsqSR-2)FbKoxB;DnlX7nfXrHhP8u`2*s1U@i`vu^ltu^lkwhiO{h^Y<&% zk`a22psP9KW)MWY7&oI|BLRBcHR7zRW6MN7v|vJdC1GQ~0PPl~1+|o8{DOM-)ya_% zfM{m}sazUeZ+AXChY8yagSR&?;Mv&^+hZBe-~poLDDHa<)*r(}zNOr+Wt43ICO_ps zl)tow&c^O04^b6Rci08v!DBKYnHmEG!8d}DsgFyadLsTQ#s%C%j_9}m9SthINYQ4| zxRZ?W)l_%y&V6?Jllt3J=K&W}gAhAsA(C2%@VDgqm|jPAedL=y`OdLJ>$!8MPW25{-Euh9Bvb3wo|T+#lJt$mt$p3| z^-&kEOyZn+L16y+BjK+GMqZxnKDs=db6UT>B45+eu)g)y@6%U3u&QqA>b-gfF|OvI zkZGh@6`oXrJp_SZm`;k-~=7#%3syKhsnkEi|UckLUTD zye=9Xyi+mc?_v0O%jxU3g6ltgI?Z59trp=3z5i((-eO)D7sA4sY*VMTYuTHxXU_iS z%_^A1A{X2<|18h0+E~$Pt9Xl_WkYQViQ%5b9Z;^L(XoH%1ZOL1JTZCcwqiFxP*z@I zLW0C-Yw_rr*PI^mXCS?SUl2zD47hFKCNQI$``x>D3|7yw{z-6HW&PR>hyn2<_D(wd_ z)B^&T$IJVXM_F52n;dCy)$+aZHncS|G0CG(dbl%+r zio1%k9jBPDkAs!;-St6xqc_{y_Io5qitzj*o@+qBJac)(HLudjsBp!1zM76Ud?sAA z>ug~dM8dHdYpJuE__9cFHx@|aiYk!}muANuqpN^ld@8J5u`rym#6duSXE_YCvj;yX z2qkI{*KvV<@n_#w6+sF$p_tDd-bpj@Z%Ao?=~auiM7;B5%3&D#2`VY^<1bZsKh(7v zN^DY!2nxEE>&;^p096Ba^LX%oJQd~>(*u)lQ(E-o)NLnE3X_!+Zm}S1oPnc|&Oiqu z@4t}w63A0zXDHqjPkepIL~R`$9F{-5nh+PqoIN(uQBqbW;#mv;6R!x~gLuGd?$8VE z)J3OD{y({Px4}q&<)2up!L#ECrGjvkiJw1AxiBcy45&T@q00LD_~apAqOG@HafAE8 z?b~OLesUR9o#Gr^Q*Em9=-vA7L&EkIHM_rvjAqn!tdX?t;Vu^5Y+7>gvzvB&rJ!%k zkiO{Fd)w?+M@1BC3m5S3nb~W_Lobq)cj2WADfZ{_;&^ztUE1q@KH&M@O0%|05mjy? zD^GTjJimHV1s$KX~A?!gDW&LvE{A=XEO%3Jzs3A`?$+o^~D z2{cykitAv6$f$~6CLo)uU@;>5D|-CR0LxMx_!uR%^@QnsrK6~ z;)>T5+irZSFfFQWX=!l-vBEGV#i36eeRhLI$@XOW6^}kX&()6WtK{(~g@KpAxj%gQ z^79Faa?lWNA#KMKL3FwW*cRXj*6FUh6DGS3XECdcReH^>xwt8yb=KhHRpROD?R7RS z7XI`UYx#bd;;mfI&(F>6fwbc$!K~`buMfbhgpRzFjLfb&EiEm2I_n1(?pJaeXbb{P zcU}SdDqC#K6>hTo2w{9BjlN}k`&LcW174p3!HdG@>*2LG+7w~0N{YN7{BU{ zg1n2N{*Mmu+6GfU#lC2yg+o!t_m=S*jQ_{$wS4iO&05}D>(75y0Sy^^n5D=zffEPr zgqbsE>g<^c{Til%deF!?3M^6=kM(4a9(mnp!D_HYDg!n11g_qIqbT#$PJLCA!aB$R z^mP^RBhoDe^+S~>iN;6y-8~po;6SEWw=8J28RAI(+|mPckd1J;!fm?bk2N z@>++mRe2A(x3d~Zbr+s|p}m)-w&@JriS|Zol-c^PGM0jdmyBAhUVJd+2oibp>654r z_xoOhnLXFft~>bZ%f4wxH&j{#lPx4cgcaBdVx!&vcBl*{1$cLA>+4;$M}8d%m{aJk zX8t+{2&Z$;BkkH+zb6S9e<)^b#_&<1^1vGel1HIXU}adW`ZizW02>8s%yL5`qj`F? zfdD8Mcz3dClm#bh}Xh6%n%n13&3$F=5r<3a?*; zZP|SI!?P3U!su^zCBSkXl5lB;#_qh&1y7GqBrO5$iGrpB8YcV^6BAQSN&wFe%Q|yJ zG8;^q&0b=Owqrg)u;}L;H26U-2NePvVFUlQID!K9HrR*JRuor>W?5mQ1=;c;ks(+g z5USxN*b4nElYa3qu=5MyJiYvCp+3Tdv3sxk)7)Q@S5)Lx(;3)|m!cp2qi`aAavgx+ zd~ZAq8;p&Ob0DYmMCIF|flx#Yo~AKof;=(yw=2i|#`^sz<~_pO@Ggc=6B83%y?8__ zf>i?B8l5HLnLWL|IDMD&XR)%LdOBM`CgofQgTKzWAIstECz#eA=Fx0-ZlXjlCOIGo z-J!ewJhxxjbqvzUBNY62UCxk-Vs zx4S6zp%k}q_|c(x2xBC8tUo1fzq0ZqI&V7=A4Bx@%?4RNQ5`5~#L&!)3;MvhSXtx% z8wVUG^bMPSYROKYKJ6N^LZJz-q4PkI_D<(z+V($%fgGmwq|iF`H|rRcv%9jRCDy_H4bz#Hto}G-tnqsh-+ROUp*y1Aa>HbXHAU{n^VO&7 zzZSdw@aylQ>khe{|DI0yVO`zT&vw4|W7wRjZWEcCJAV=tRTZgK&uljAt2T8-6*UUv z?J<~PFWwpHUO4IX?sv_xahIM(%zp2-^TkKW;Vx(I1#EXsG2 z!%dBsB9uD}JTtU^ic1Q($vSZJv17MnL!pDANRS(P3w^da!ggMK+G*?u$WzzDROdG0 z5iw=#(1rR+Q9#iBM`1d!6#5WwD=WnNd3ojF@@41Y+3>)QU~H_)Q=n_tOGc0U66+7h zbJuv>VrEttft7K-E5Fz3~i?`54jMU9SzE zM#RUn09m7R6c=~7#IV&oplCFpP#?L*?02lJWqyHyTJK;Lc?t@=&LZtmwRsLmnYA-L z?_oHV5^{kiF7wUF$?1F=Hu9Ew+kd{8o$%LUoxwPt7h*jjnE=TG@5l?7iAuAps7z_b zzFZ}uM!P^f5yE8e@c@|Z}FbBmsM2_m@@Oz^6*b9(iiy5JHaEYep|o4 z|84SF7Tr&xw|2Xs`AW-evl%+i8--S$jvjb%LR$OxOX}b#;;aO4dtUdn<3-=FW}ytjXx|zrT#-lK@&_mf5oa zMcNH}jvagw;g2NSw(&b%0tTXoUS{8l8BHq@g3WfOGKHOx2w-`BF2~QcR{~(4M4vN) zK@5|=Di@k8r#&FZ#bmp56t?a_Y^1|4V<#l22?raIy}=$(nM`grdPWLE97MFgFur^F zvvow%*}*}M{XPWfpl*RA|Sc3RN4l{?Tn{*-Okx}4=xHUY>3$X9~Ih-^`6=ny*(<&k1z7x8g*AZ z5=d{2G&_if6jA+Fw6U#CEXEyFEl_gm7mKKLk#gw{5s4rEj%aJESzv}i$G6AyKq1y4jEgu*rwvxbAd9u1q>3r zELY*;Pl=^(JwsVR>mV!#SOmgVzSa-l!yrfF4%Xcu3kEkm#Nexn7l+aTK#r&*hqu&H zIXSsbr~+V2J`>9C?=3G@153OB!uTS(EcEoExI3DgUtp~JO2>x(>NH(8MeC z*iJyFpIkf%_ln@p;&mS~8@`$pTe@#TD4lWgdiM8zGE{oIugR_@=@i<--J(HDKW@9c zJ!}S7^Vg8czuwxE9$ReqY{Q3s{;_q(c+Yl^@YF66dL@mudu6KKOB)d4t7zm=L8HO5I-O3snf@Y1s~c&I5%<3ETUzvKB22jftS{GO9k# zGkoo6t1KuZbL2@o{?cZ=h!A5BWNa_>g6Rd;AzKzFKYtzxwi$+(UY)}#n$mtib1==a zwOE{w?;As*4w$Y9+1}wtN!mj)ciw?2yXExhn1-7;EI6eBTQ$Ktaa36POBq-JP}{uD zjjGQh{Mq*_p?Lwwp94Frdm@qOGT@S@#s%reV3Lbh9y!|Y z=z7+E+rz!(Z^J$a%Zq<6fNz*>NOZen6{5+2RY$t7VU)HFC_D@=1r1k>G+4s=xtF2Uu2NCxLebhN@NJz1$I}xPZCP z4M4jRy#IN&r%o-Sgc{=UqAlfndsZLK6cvTrckblzO<^Jl5q`6JgqON%i1+EZX%d41 zeMz@$hnkVm3aTyT+<#I)hTEPG(i$0KcSLh8TC{TE5!h<}v|_2KF-?EBehGvdkEU9(sy;B?TVd;gBQy|~VtJlSv2Ujyde;YymxQQ6C&bLpMhO1wm_|GmL{gZ)?Y&{8}>*LW4^>7jSf`! z8^Ob{V+U$N7TQ?aO9{9n^HD7%BQyXquAc_J=Qd&MK<3F1-50xl!SS1mvj@<~!_RNV zhaRm2GXKBE4_6W!~LlI(NCR`SoqQ_RIX zlV{n9nT}p&;Qb20XaPimiYRlvzrX)GdddK6kRt%S0Z*O}G}3*I5YHfHVf;Y>=1#}r z3aW#)m$4pvsxWy0@($66?ryfJ&z?Qoe%lhbddluAUq}@;-T_o*IUV-~`x!=>FwG$l z3{Vnig~9rMp$@Qfz$PqBR$M}&PiGy0X#n@u*g$6UD< z>Qqg_9mBj5510W=blFdN0nP+au~#R|GoK@)1h=0gf;T}K6abS0B%D_By-P7u;B13p zO$!IGEx47O<+J4kET*8pLZh+Eeg(`I3)g{SOx`OFD*&>ZAC>R;@L|FH`IkZWlR009L|=(ISv{l|s<>nfLZB&sAJa zAZ12M3O8_@UfZyHa8!m3@sW>H9^S#=0uEG7Q;Gj~OdGi8;SlJNh^Yp8TMY7e)&Ys{ z3^oIi!}e*0{+LMm(>|Z+$U$0N8N0b{#K21G_;-$GpYY|{Lc2?pBLY}ok3MRb3jg6e zXT$BcY*Syz9Cd9rU+TdQ`fYJan4ZdJ zyG0xJtn0sQ#I}I{e8sF2Q=DUOd}ZM_EsMyjm=t~By!M0XQTKnbvDL0!G9;0$>Half z!L!eI!pk18&1-Sr|8s-ihzo0P*J>&@KC6HXPE}o|OdXI)P8gnv+|*~^g6njj$d&wL z)3Ki2Y&G-oNI`bU-brO+{pGb&r_N+MAHRi3f6Yd>_AS#dFWy&l^0(US@Eb+phhK*e z`nhC|)@#S9v1osG-4f-Wde1*1DryS(lYJ;Dk^Da7TpBj=CG2H!agTl(sp^2$rH}N} z0R?DL?k90mgx=I!f&L~mC75BkgXF*e%4>!H( z(lV-$hP(pVL^u&_bC@blvlw{F<;uohQV+Hg`Xm?~yoAV2XEj1yI%9h0Ls@`CO>vxK z!9NP~*h}`6Rag6hJHO*&aQg{d@Qj|TzJ7*`BnnI9B;CZLv_db;eOiUPiAwCTrv{x4 zTkwDKbWT-1|97VOW&ZPYRsDNaR$;6`K%p5&`=4}k!nAr{%t85OC9bea`wnSIKm}a$ zy&wo+J*m>|n2NY0FhPIJqqrH+dBfG1Lv|>6xZS4K4&1@IqYxMr1H6bRF_gbKzvO94 zYpaThNnAznBZoxwxZ(^moM4!a;*#1CC63iy){*;7TR}A^Ba5B7V*^TKr_hBsEC()X zu?&5O%W#1U!0QIp(Wyuq8J{X9uBztsFx>#D6SDE<=l{`|G^dtUR(ez5$o1S&z$lBU zSOEy_CtU;>l6P;dV!|pmz~?b_fk#Ef#PR@-&I0b{sR|AVE&}ednmZ_5mCy;Ja6vun zQ1={92gIyb#5aKQ5z5~TJ zRue#v?9~CYpgGCI;H^>i4%;>f^e~T|S=Sx}u2W0m41%(=K;=Qik<*7R0H-uYf~*;u zq6tA+wab_jfFxEm0~J5a*6RUasV!zmY{$A)t44!+I$+fm5V{nrrUGo|r|Z|8dbq*= zkRG4EQ_qjVm%78}s(|U)xX8p;3yGKuQEM$lO%7*@e`85%#X~yZ7J5=){SwDrQ zm>Ih>pL$-kskiU4q1jodVkV}g_AaXiSI>_r;;R{IF}r`JWr%mO|7z*U9b?XR*`{4W z`H>>Q{X$&RnpWyO?H+bM9)E4AqV(iX<;(Ows+IRu){OXBraaG1sTt_$8(YluBwHf4 zv!*dJ>3zc!@g6kjXk2qw0+EM0?+(ZuPAqWRf=6K=RowEeKt|OiDRrxbP1&% zU#9csVeW~Z>IkXi>C2Q-FJNmS+xng_dB45r>J5VPo@(t;{+32O_k;MK}F*h@;IRE0O)*VhyIHNua z4h3)1JGrQL?9ZCn5@_hown^MvBeUSqs0Gi(NDT)&#nxYbeUFDa#69ZAEKOE2ea~0z zGaru9C=E(VtK1rQNVocU;OcWNQ(w<_mGkR~oyq(|h1rkKpRPZt?&S(0OpG4fXD$F^ z^?UfUxK_}%-G!f@A8;TK{F5zO%poGLZ7T1Y<+IJpsj3mMH1<7<1D2JLPxSTc!oC>$ z)z#I}vV1SA$AHX(?rsgS>SaS%_kN`OK{Qh^(LLMJsX#Q26vu%lwk}AQ#Qh|Hru`8} z{=g`Ri$kahtfT^ZczJojjguf40_S0_(WF%D;VI>4TxCU>9Ts2mVd4aK5)mCIM2^Zu zW{>I8M3)9}ESnyhVWJul$38XJ!y|MGeo1&~0$GIZ*O8Y;Foa@lk^;PD=0#)4_fhyQ z&bE|^by~wU7s7VI3BI|vH&PtVwu~$lu-C#ZiS=bI4Qg8QDAW|1Lsrx~g;hB)-k1Ca zW6urXUc|k7K=A=3;BmP(8pga1w$$C%sM6WGiz?~ z$thn;hcC~wq@)OR4#mlE55bBhU`teqyyShg7&m>*w;OMRM~Vf5@H<{S>nj+Sefx*@ zN^!nLTs4f=$#z7j={VS)v=Jq5qFQ>qLPm#x9J}X9kKYd?hb`UB0+FLc zQ>HpOI3N29T?7{8jPmH>Cx9rJJSotx)2h_6$@i|ovpYyD%Jbqfhsg$dBwUqKKq>Gq znvonMRkA(0#PG3x;XL>wwIwRd$5cjY0&G9XQxafP*T(x&0W?|QG4(^(xgRo`V@H4& zkTDf5xot*9zaG1^IcP1v&TwJ^pYjzjCE*sb=<@Ytfr{JPuZd~S?Afn?Szp9xx@nF3 z2gf!(q=yUTA3+s6D9+e!ge_cvU%q)oX3Q*+w<&^n?fbNiEc70jdVem(w z*-sts^txKWQ=PJ|X>0V;zAVAwH$OQaH@SosB#q?xRo#y&yjaiQSm*4^ZZ_KT%DJuP z?9&Buqr*a97il!ryBu|mm|rv8cB~<6jn1dB&?%RVnjN(`r%ozxcHoUx%AOVxRFHd` z>#S0NNB*02yVY;6-o8*Vz%|FyY&6U=W#B-eUh?S}$rED|5>egu%vu9VmO~jGv1aYx zbC<~sJ~PogJ;!FOYFb0sI(&1v8Ad)2iwioZq9f!V&EEBTpjNBnn>z1uS?vpZpEB=Q ztFSiPBKnK3#At(TdlNT*`X2mj+GFpI9sD7q^W%P^s#9!a(V_l7343!smJL*xv7dT3AkFQvDNn64pC$(x$z#4`mz#)+7{*30zse* zj-kZBz`#TlQVO2=DD?xf2{-~1qVUahEe%YApmr%syhchAtpZP+2H64a0BxzqFa)yxdbAcl+h1?KEmYs@nd)2!nFA+U6NUwo6EYo>ppKIS?6aWeh5a^_^{t9FZg(&Ivo z@4S}N7iG=?05>!?-hBM{@i4QfgDcRcfodhZOEt0m^;E_MR%-1Ng^(LkYU$ed3LvATn#LfIQ42cA9|wVf07X3LrZolpE?--p7Rj z9&Yo{?7`X)k9KEu6dGodn|=&`bpL+-z3I3-+Kz>+llOUj>f}vP9E*y0thLn+Dp%(8UPzph681zzr~`6a+!^F*ug^ z{a6ilItc?b01!i<0cMr}YcTCDfT@5G%IQhkZ@WNHSU@T`kiJ%8KR{LuouB|<{!dkK z$3Q^>GOP-%kI9^PHI;VI+D<)`(3EEj(;C3sSDI_j|{?Q@Z!yzo?cA7K3M!3As$K6(UG~WNj+qZ_^JIj^p6Sf+MES%Q#X1{Yf zU!SFSe=Wawk6lgPm(Trms(W7;*sTsyRnc7%uvU+4an&LIeaF9EJYKM~ME#rfyV^Lt zdrarcN6kZvMlb56{TP)@RzG3dHkcuH!GrSk!)nZ#R%qZw@sqk9HJJw-LYPta&%I3W=o&U znAQ5MF{Pl#ogLcT{zFYHEa@EIM*C&2BpmdTyfxS3&SKyF3$M{d#iII%dg;PS?Mhr{Dz=gaur2u~`zWGVM2yli>c zucOxY#45&AkLZ{rD=%uk+}YLh!Dp7+o8uSU{;aE%b>x4SAK&7Byk&QPZ+1wIdsz0& z)C{v4r;x#J5nQ!%m^^l~L;}AW??YMP{^)pTuhw>&bJYUHbn(hlFSgSJ} zT|4C&pqF6zr_4DF3nl^k_hQGMOvUMggu<$56yZyJ>Fu>5!tFM&-2oJ%`|19knKNzE zIBeRN08)kQi3q>maFamo*4yW+1lKJ%cua?|J}?(U(=@<{7&Mu8^0}Ub^#>$Q^9!*4 zqRe94jtW7g2e4bDs};Ld!rY)P*k1On&4!>sG$8r3^g}IET-YxtM6u%P%{0XA+Ub|*wWUMCRAaueT zj|3J&u21Z_#G|V++Y3ySqu`~0zLY6Vycd2wjBSC}R-lFNp&vsBgWBV-XK_fa{u<3rqHeAAW@A zDj0KW{?thz69@dzz>Nq_)UWa^gKPQ?Xz;|_-*6Q;NAR2(Xqx5`!Y1;?wALZ;szbjbx;#6LZ`yNjW%L!BwD}k~ zjwGMbd1}@3yX(*X6IEkt=LFsmNja0=;CKJ~3nk61y;%-S=W#)pw_VoVeQdq@lPc}e z6h5Z2!%;1=orTOy=Ur|d_D;{qOX&DMctO{^l|Mdja4gjG?y(XD(~ie-`_E=gE@z$J zmr`AIMf^zZpY|D<)sqFC9yf+@Em?8QKFgxSJw|pky+w9pKrDDT=bd7Qwn)&4%8aA> zS?gFj4_Nst{Wuo^2BG#?+c7|hlbGW9m*hJGvYkh_h@;RnwW3SwrRRx z9i|P7Sv@`0nqr;7c``LgA^u!)a7RbP?^mt^VS<6Ki34Fj1ZC9+6;E^)fBc=Jq+k1W zw9Z-KJ*!;00_(*C8-M-TZ~HUh)QfMbgVkm)(*rvjwL>@6=EXa{G5Z|xTUR*$efx3M zmwMUhEcb#Yw~T&UbvJYQ$eplM(P^MR`l-6~a=eWTVLBg}oqgeSOTdyNbKmzFuai+? z#i+?%^7?+a)nw|rfMj096b(;*vZ3F^Hoezt5QrJHa`HBrIkns$>U-{kItteQEXw-1 zG_{-~CNu!T{?z_-toSf~xE>agLi#ZF#vngw&r`B#uF~YJnD*XG) z9o4b1t1u~<9M#f5KF@jl1GT95vX1?acSR+F;YYzaXqI0PXQfB?WqV`*QfxG`oq6Gq z8nFA?T&ww~L*SG_i+8kca2r||IeSb`PcKpDNo z_`R`l8iq@Jzn-7#Feqb7B@!4@fXTg278@_qEEzT6YPc7jHn4u&6LWAAM&lzOfc+C%SND`1uOPEcE$Dxzk@pLwdqZMvw|aAA z#3^vTDe)wuT?>yALuVgWvY}8bz(S42NHBao%*j#g$3w6ix$e9xOBgu^q9I(H0%qMt ze+$V`?hlQscWJibx7{A%?<<4{BoBV#ox&0^%LI5lu|U=b0wI5G76T6&{|^N(2n+Ik zc~|DA8w(g28RkM+>jwBGd4faNQDBI#4rvU}IVhK05D77xkO+$PeXv`?jv`vDF ziR^8RlG4t!oRuY|0YRa_v9XG|x%$Mb@E*w}JU?;4fq_dEiXJB1*4q=KUm?4XsFxJO zCoOHQ(sMBUCFWzsPKGQb0T_rJEC2fS3dZos#I3?h7<#8Xua*iQpcXPP)gQfyP=GTv zS-fOdQDiOHeTMFTEQj$^9Pmzh{=AMfr054`SY<5=H&%=Q@?H!2f_P?T2I@8@M1Txe z5@BFQRfO^(xxrko$aIiFHbtg#8?Q?D12}Lqw(I~B0QE34E^h-cz%jEm4BI0AN_WEdm0fmmrHxTKProMibx3SF+Z$Sb)g}E(@AO^e zS%)--d{up#&5tK-^|T%n%M*ePDM+I2!*6HM4}~|NCTy$vQtSRDSK41LP|rW z2o*g-8dgTK(jX<0l~Gpq-v9HW_xZi=-_gx`jtMBg??d0pceodZa6ag6ag{&+V5LG@Uu*7QU+YM* zcJ&5tozgPE`eDaRcci%U0WaAJ1wJX!KhxX-j-ecPPJIAzs!Yw@CP1Pb` z0slnV+`(@(<9bBQ?F5MsF^8c7eO8IXmmQLg;1J32Sb~gV<_+ zAhaTyg7>z(sT}*;kMbYaAt(v_w;fn*5)~eIs@#K-wlEmt5 z$b8QA9uzEmCPiP23zICE9|Bje|8(7O^gS0!=VG|-FM#@P9hFX`*qf&$ABMLj7%(|D zpfyowA?$&RmiNuW(Zx}RAIEuMI@mi_91Q^#ln6+8wgpRS);x%L=L*I!VRnDR#)4OH zVC}p4^iKvc4`2>%g6~MB&e~Q8%|!&7N~}0dp8{8a9oQQTYI3(Rt;=1(t`MgRjI3*+ zq4OWaTr)(x@sz9zIf=tB<57yhvlfEDp7FD_b?y`xfZB-_ZjbT;VW&9w;vqh6mrJP2 zK4W7c&<&QRB$I8oc{1%E96E)_d3TaDj#UB#qjilc^qh03uDG7{;lo_WF(5Z;`D%&f z4=knro5rFbW8(X|yv^<;5%6QkU_cgl`*+9>{In|lYo?V-BzrhBl z1o%eXaRn4NwDnDWp?6Nf*ROJC##U(hc(jv0@9`hJWm&$EABO?3+KVLS2PfP)1^}NZQu(a-**CsaoP~72G#o6t5PfY?qDNF z1Zug30&P)Wu*O5L=qI-js%Q0GS~L0dfdrGuo2jWVl++z(ShG+>N#X%f^Oc(Wten6h z&cllgo$a#b{JjYJA+s#F2B?viM{4mw9BPt#qjxMAI0_gawN%g$u}RaM-4e@B(aFU$ z-~3!0JWf(ifYWSHx?EHg5mda`Vg@auD5ESm?!%fi!|iH5=oEB-KzpaT&lFFC)z-(j zCcumbfi_2P+qT^Ey`NjordrRBOMT9LL*IGbCZ)Vo5!K}iL+7>}*}Wj{&nnezhGU#b zCu24(9q4qmksax?@j-Fi5cSP{!xHOqxY)CB`yI}T*0#!DU+nEoUdTEW zjeapIQq{f@mfzjH+tE<&pSgUwHos%)+2vPv`X(Lh_j{6j?kU&r(n9$qD!;xBIGUxo ze!D%;FY;$7jEBJ(5I>OB%q0}e7F*_nQTwx8)%{(}X`=%#!dVxk_M}&BF}TvN=C|2! z)P!$TSvs)FP(0YA&uDkPLVOKNRQg2Gsxyz~#Y_wrq_1M} zNRs9zPpTgk6}f|mDF`oXEf)fz{5I%lZ{SjxGoS)253?L~gkbYRVl>=aotU1!1avKA zCa`oGcRy$^yM&f_qrkF=MtHu!(rX2tkd`wx>pu6%I{`DG;U{0=1vS=ken4M|dyjT* zihGYMU3D94VeXU?@Fl2f*nnnNRJ<-qLh}h4$X}LD;1UI=xaulQWf-7818ZxnKsdx- zPJT`pT-ZHOH*B1*7hHnnKDn*7C(l*wU4&(cLu+gju{|}$R}gU&|3r)+5Kq!rq(t}* zK_WzT%m$v9dmvHDIdoXKMHu}b{3Rgv(Qtj0YpMj7bTD_mKa4@CS#NX12qRo1UIe-r zpc29EX4k1N->RZ_#(X1evn3!nmRtPN(DJN0<`%8&YsCu7v-4(Y9?!0 z#I;774IujMf9mPr&}_XMb1qAJO=jT8fS~k->mPn$NkW=H+%w?h;BSDHB4s?_=M%jN zXNJc9MImCGeeKkYI+;B<>Gl%q5w_NpeUq?8F?Qw99qs(x%nYVMAO;E>dEK+0>g92# z0{#dGg>R*36~#tZBagNcfGu(ObD#5Jgp=3>uywRfMxKFS3| z$OTQ=1wqS!26%+6444Iq9o#1Jz|fIsJ}{kjjpUY_0KWWELr7G-y$bxwQHAu!z-%?P7UW| zDmR7c*5>Xg*nH&EZTTwku}6~8?_%8RHYQyO&N69ju@=7m;p&i6&F`k!KAlQ+FIP$? zVNF8CEUF?k8{lSLQMk?cqD*6RBNq38Qz>Tm62zaBRhW;at+TO_MS(oKQ`7gW;K!Yn zC!R<4yGfn(RkGSr`t`KT%u4x#E3T+(`Kf+P)@D0a@_H+qT;Xc#y4A{5)T_2-T$rJFWE=nDjiOxtRA=Vj8QyCeUlF}wM^oO6@KN0* zj0cGiJ8Z}0*2M6Kc@rvb-cZkLI(5p91L~vV@6ubWK9<2q5k3YR{sQ#VmIN1KG%U0Y}&`pc?o$yujetCHU;%OpfAp55HNU=80w4 zgXV?>jOtYL1wlKsu;s082$RC6&M8xY!7fq898M4sZu=>@w|thOz}? z8*)4S&jtnyT!Nh@RRqJ}>ylA4$bVDQY{WHcy+A0guCB(WyM40t$~A>&Dd@v3HjZcS zm!IM;I>*KhIqFclPypszqKvK?_#TuaeZMdR=fsa?sn`oPkAl3Kp1$L#LffXoLrG?m$uv+8mn+ELOgkA}20PHN^ zm`kGx;#P}HFzQ%)A+BsaSmDUf@I|;7sDhJ(K2XUwJ4&VOMm83Du^bym$05$d6?i=m zZL>WOv3-*QGMdTpQAGRZ;JdSw*vZ@(*h9gW|F3zQYPDcwGRMkoZP(D9M-d^20oGPE~9CjTSL zM8gauFmOz~31tm{pgaV#L`o+Y7wmaFLZpjlHNYx2*w9Ji&l57md%hd0Z29rQBk=g- z#p^BDFSqcW6a@|({lHB;QZO}P5X@BuUt@~yfN=ROaOokrH|wxA517;q*!>~>#Ug8X z43PC1LJ3#_PR*y0s^qmYP>;y&8%Z=;T z)2ElRYaM>uez~tX@F`&bwF5AbXp2!U`JQpVi0@`x>$#9Gt!#zm>QR^8NIdL_?@HiQ z-s6z4fT8Uc6IYbPZ~mBjjYY>Mzk2K&(&Z(b4h^-jOc;UNGedo(uh~xHMTE^!t?C#nVTnC#?w=zy*%>>XMO`jBb*Pt=jj>!hSuDrznG!qTrZGMDeo;0ZA?a?L zW9EOW4;QY|qaSXWfHhT=7GKS`V)NoF$K~@Bo1R=P;snsp1yPOf5!~EJTnfHk??IOO zkj8?7b#D=rtDsbQB5m%>neGRT(w_<2=3ReI11SzmCX8gp1iTf`b5iZ0wae%dv^4aLi z2z9}}z-GAjgk&iI4P--1LtFj+s>D?x;RsRK$b-DB4wHoQaRm-g4-?U)cc0?uO7Kmw z$s{>CFgix6p;Vuf9J)Qy@^^G}6uwqgZP*CjFpq>Gd7YUgRTz}Qp&j1n?9I*1rz)Yv zRn4XlY$&cg{#ji%cL!k}7QmkF(~ey@@UJo$(x3k-J{VoZj=dO4ID$p+`h$bU=9ZQ> zcNKo+2Nwnsrh^Cp<~(xhxnctrgL8qp@R8{lOawmrpz(yYl z*C0ZaGO@D?;n{`e3#LumGyGoZiJ982g_kS2Y?PMvJB$x&{P^+Xb7`p;wxp8CYs89I zd}DKqRY|U|{0Sipfpzf0tV;&|4nactLXP5DsM*W$Hvg|W$$TYf`RS#qvT40ZDdJpp(AZsFhR5rvL5py*hLz;wslu9Y{c6BKMFQAZLe77)R{E6ZuXH|jk9+P1`hWhOq8sweQI~-mk z6lxv~q39bN4me2rV`L1+Ui)YW6hS_63qe2dKBqOad*0NtVN#IL|Ho!%{#tl?01$&M zN9<-c8CC;8cn*#C?Vo`xgl5Oj2B}rJ%4pTFeb=sDW3Od9&COSCw5s$WzZ19Hc;)1>0%5` z&%pp*DdzQecHLX{9y>3TPph_yzNUe49J;9(sG^rf-Y*6$Lv?lbzaF&JQ#KB>=7|{a zR$k?uJwDjrKL8MNnOdJ~4dW+&x3Pf7twTiy-g<|>_|*vtQg;iLYz za!wWz;g;Q4zrzS^kG6K_u94>0lwCJ^=q$eZ_S|;wpmYFCy|#izY1Wa#vh$PVg%~Wyi~wSXiie7Gw@2 zz7!>OLFPQ!vD_4s%T2A%epPrOS^@YQdNMXtme7hncBqgL-5Ucc!4g>D{Z(^wJVt*F zNjC8MPesL&%$e=Dnsm3Lv-2`|l)z`!^dpiGfoXpsr@PdW=lw{ncmUL#l*Y+nVPW5I z$2h~~yS8;{--pODPMWq-Qh1e=v^J$6ABNQclhj2;Ef*uInE(06E~>W>#@qlTefc>K zGHdvLJ*M63G{jB|V8wd9L>QVSdFF_Q;hYv--DhmgFj8>JHTmv^mK{EpB1TGO3cv+Ha5xX9M z%#JiQ9qDZ<#?#F1yc)DajmfRJg1h0LC)CXpZcrTK=H>OBlr$h8c#_wK6)JS}Mr479 z)7#il#@>iVC#l+LF6g4l+qQWwN-94Evo9V@)gI8%I<3pTc9cwO@V6*wQ}Xv&nJz$g zb@%P7l_2g6>#qmM52q8r_@5>@lVrB*!+3_qc!c7YCCYg^XU`0TaRLOMi18441^ki)B8t$vu|Oa-Gn8D;n)yW7Ke8EPLEN z^&!T0xzhD{0?z3lmCuS`Cy#ISe#Wpv)*JDrm<+S0?PLAzZ69@k8 zdWLrt&bz+#kbq#!e77tGmqAAfZHa*^O3qh zr*u%3SY>e$;JpwAdJPmO8wQxmTW2aMVg!PinfN{EF7lR9PAWoAFUcz1T0; zKY%d?IZGB2$SK1Wt0u)2*IAw7ij>8JDNJKBZ>A_7LKwKby8n*g0o6YhKOTB{b{K7A zJYgsT{igmyR))8z9xgrfoIq(aqw~Re9nfs?xXIkV!o$p7d=+{>Qs4*Ay_*UH%X%EM zvnPwU;C3xGh6f2@i1>CVJxXX;1KiAKS94-gMku!y)LJN_1Ju4Y`o0Cb0Okh&<&aQs z8fyOK)T=|#oC9&FY^=cXSDM<<^?XlIi>w_I;JXEY)DRH^M0EmfB1SH?^q};wj1QfGe+{Z&z9M@Ptc?A>;4?QmQc0BbIoci&oG~b@XhyBodxRqrl zP11Q;A-v{10$CCqy71I`95X4+8-FyZY#;3iytk=26(~@rPHk=w?O)%eyASv`SI$g&X3Lxvd&Q| zqX(a~*SKh3HcZ@yYE&*3+B(g=?U_cN;h~$}>zlWR3}sCOzmMs#6?(G4wPs^$ z$kxFwL5YFn!Vl^*C1U67kZA5uKeDaDWEE^=s9VG{7QIiU)oeff zEU+#Ty3*Rx^0}m>A)z1}dUra~AeCbYZ{QEklMCFjK%0~Tw-v==7LE+H4d5e-(EX1y zwN=>@%&DF`I5LlZ$W$~msjFPI$Z&;A3IcV>G{x~d&C@whu(BY zBE_OnAA>D3HZF@`d*Tt=_yX+JY{X(xPy<~6)Dx1md1rrrzp))AvD-92Lx^W1EC|^R z>LQd#;piaHthzcC5$p+&1{ybpL3|WuUJ$473$*nMhhptoT|v#iAv6nEZ~nDq>YuBt z+n*bP^n#r(om~*JWUrFcz4#4*IM8~2{_&&bxj29Ua?XLjD$DxGMIfKyq@dC=a|I_H z#-Tdr!rd$33vQ5dWHCaGvXT8oJ;)u06YD@w$1mv(8~TPe0_lJmQr(2FZp=Aeg;Y)P z;MZM-?0o@X>Y^PZksg`Mlmekf{uK4rvI>etukfJY#@oGk)5eYDSS>D2ut{+?(mD_k zfzv6GhuMJ=V6eE~sa3wzLf9}N4MJXLHwJsfbLTdYG@rf%>InIw z9f5;cIBoZjaUP8bJ%?rrnZfJ=7OJbR7eHAw>}KFjMlB7q0UW{G;8L;qjU11Q7YR0U zFdOf=wjHid2qwObXbgPwFu3wSR6>@Oy`^?ddATob`x_d_U=jl+TC)qv;Y}t*swMpJ z9ON-{n5Of1R~FX#HgGB|Xx#Lmb?a3?tlT(O6Y=`x^I6} zzY2i+*^l$Jn|K0)b$^T$S+vwlToa9!W*e`u)GaG24E%bcy&`eLK!{D}#4*1kuXKHR zB?5Cn#N@cSc%Fn{nVT=@EY?wO#f|#RA;ZP_<)=K7YCgBQN0x3kYOU(peEkN0m>1nB zOhmYQUq~^go=H9(~ZZ&q5$u{X$Uv+F+j2(X6Jb>}G_S%tw6%fOsB*QJ< zABt{cU>pJ;H1L@HdVdo@NRa7rAd2KvWu4ZQ_sr_43=sT#$SYibq60RAlsL$C?3W=L z>p{eUP4#bia+FG0*F$g32drWDQaVlWUt8w^7Nz&!%I&gAgf|J{_R5l;oT-=nfI z09sH0`0&g_D+P@|%`JY2$vxaLmqV@w8e%futAsmuejh++pfh}>Vaw?yJ0CY&2hI^| zheA6{=v*T4GhCOg%J10ZbLF)a_aAu*fJ_uL+mNj800sAMPVht2%Ow7~0NT0cRi(kU z($hbgCcxzP0cew$V*2~N@a*uQd1<}Ar(0uB00vxW9Z9zM{ilY}df~ z1A{w-u^hn^6!cgDS}tlsJUy3UyNdpgy&mtT_th2fVJKX zoLj8Z-PM(KdEuIMuES?RA*+LkM}DjqNQHs`8VX9RV_E;G_B}ctG!t;+;6|?P(iA)7 z?GV;SFe?l|2IWTB0T0nBMI)x4;d1bKJK!}k#=xg`U@c%^cqB7Zvf+dwrli%nMWl31 z>;1A*e9V}XYFMYgzQ+O^9SY$TY5W|zAMT^DoAo8m7swJ`B=aU4D0oGfJ-8%A zWJtkI&S$@^kINeb2|Q%&4DzpN`4m|G+}+F}HN#-z&W^r$d;z^+#OD| z1TE+1Gy8NS5O6wck4p%zZ!VDgCT4lxwJ@90$?4wow_&c;(NWDsr6=0c8y^M7^Q}*r zckxAQd3Y6Tpp?e!b{Ann_#!w0YrJro;)C@W%!bgh<@{pG5XC60q^GABUs2wc9`=+M zjfL{woIRik;L%XpV&@KDLCmFC^&Xg9b)kBM?wn)b7z>IN7B^6ma@5oHVfW_^rP3fW zIcr{6DCo?&qxP+qw62!jb$cDaneB`7a4{62IcTp=_XAlM4SxPn<=^2JsQD%<}uE zapNk-vqNMZ^r>0LY}dNzA_M~V-+)rG{0#7_n13hA12Dcl`}hBPu8B-Btl)70b}1g4 ztjR@!xB<~a*TZ6v>j=Rtm^u)~p`RH1;TOQd3xQfS5~uBD?oZh>ihqdgYEQ^rKge_U zuo$XeCW?2GkUiLs_uUHE)zf zY((qO?U|B=3EVL~f`iLAtu$0st|7)sS(yn*>4aFRLd)`y$TI;uuV1YO2`Ztz(FDiW z^oaPBk_M{d7vslg`1$%?^M_Cr{7SeI%Mr6BMMaDALdd7zZmxltb^iU3*8Z;!D0Px| zC~}UYJsM;hP(}{E-XZ-KjTg|Cou+^HI8CWgg`-gyNgyXrTnAHj64BN$$)l9}g0u=E zH~CREeHqt$CirK<#oQQyJ+!z%OdSdzBi!4nmk@d< zAi%(`Tpb=$WZeUT?X)86x*T8RXVEB=s|WIu+iXAy+{3-XE8t>B(f zafOGEZw`(^LFFzq)I}xBKHd*A6uy4ZMk1y)%?<_&(2h6^MLVS%>{seYABPFUXT9RS z#ozXR?KnxqqfWc@xh6i2>~~afSo`weVy*Ap(_Vj?W*hRVtnF50{WB<$ykkMx9x3xnZ~OR_E*9;ujf(W_J4 z`E-Z2w$w=CB{HovO)v&5g52vh!1HBq*M1E_>j7ETtk+dw&CLR_WGeTr=gt*jx6&ViIqR@eV=S!-9})(*t#35sgoQ^d~EEFIV`FqLZ|A%4`- z0?rrg_Oz{X*CAfZ$KNcX@$Pv0u$WGM^cl`PUYkI>(+s_S?HYluAQen`&_PEMJa@W4 zRCxF_O!cN+3Gh3|7?%vPp6wX{;g(=L$)uqMLJ8+N=%Q)#Gi0aRHaisK5y}Uw_Lm2I zd%YaI2qGs83=Ak@`oc$1Jcd2W4kiD^BiN%OPKgspa+u=;9E+hW9JG~iN5)=Gd ze@FjTXr1oK4$j5%r2TT4FtXu-makC*EEn^Q$j@%=ClKfXT=J8xR?*~xiCK$i6%@(f zfU#YEk79}M+N#hIvW$5chgF<^(oR# z7*AU54P4>K2%3$^hAPaV1bet%0}BPn2j)HIKW^6=(TIZ><8Exx#@<3C=`WvX~t{U%pC0%5^{KB0yx9xs4Q$f>t|s zv8*CwebU|$4|LmCmrQykLkW^2d=smUpk(j^s$F_^&tMs-z7fI`4o zPn7p9Bcsn>1DO%*+v+fUz%)q-1}}I)=1n=!c_rayN$V-xN^3x%cpxc?9jvL+sYP?- zQ0vJ9r>d7rYjiGnU3vFzd)mbK$RK0@x1At3Vq6qVh_htusF z%YF}QusJ2n)|@sJ#k&2lWNKMOmHeL4){T*0&OsqOjj{4mtY>!nCFL!rZKdVX;=K*6 zm~Z6O%;kvmFcxcRS$50*{rN}kvVC`ojyjCU-IjX#sLYBCHmYrbys zcl|vk|Ec~|^8T_BHHN9^rSOcW3mMwC4*Nm03`KKUMPHxIidCWMM74`Jo+Twh2S8QT z3T}y>WYh;oAJ-pln5JBRK;WgY^N`%$m0Y9~hK!6Z6^0_H4rC{32~r62X!7(EK;yBM zuiN$w_#nWlpt%Wh+u&(}%yx2x5t&&H>IDTqf(!%q**Y{yS9D{jD>#k3E=lm`HlVYK z$}ezL=ff)vWC%@HEwoy1!8{Ua1ujar8vqd!^`d1H_+UFa*af|+nx^J_^jUM^`C6U_ zmKCmk*rMfKoH-ZH3^_|(r#*sa8X@{f7xk6I5*ZFt_1G^R*O>(ZHGB>JDs5@dXQWmK zgJo>ir+qgb*jM51D0oc%W=>u?C+*MZ*032K4&D49IoimOdTG4H-m$j&(T!>s5H_ zn4p0dD9}J^JN$RS$$+J1cJ2AeF)T)ye90`9oyA zdKof?+%ou-AfTp16u=8h%uk%)sK-x2kr-V+PR1|YxUrO!Z}PB_0eY_XWxFU?A(#rg z0J@OD6*perphNn2tuP0{WI`h{4rrHKTLo-%L2J8t^ULf&G+eoP`c=DA5tTS-c=9%J zpqtM|$GNRbA!AY?L2|)d=(IO`Do+FYOc=wl^?jCm$x-Lu4=ty?yUBnZYkp|zkS5LY zbcL{xkm6I%sh){bNqU~_A#Z_2i{7OK_z1vipArLtQ?hZ0%HmJ%qgRmm@IcxRK-AZB zfc*jW5bmERR73g#1DjIjQ`x+e%qc0pg9NXOGlK51Q<|%4uq75qHe>tM-_}?x6v;3V7Dc-o+Jeu zF8Pe{`7~jJi+cY$coxB)m7>Ns$Cr7N9WBCK!bh{#T8z7u4D6V zGx{{Wp>k;c>Mc^tcNr4552yZikQvFf++}!+k)k~Qz2ar*ozs2!5&3Z)8zHZC8}8e` zxs3^f6wGC2@t(ZSEN1^oclr3K6CEyZZxtQ;@@2*%zq<@Z-_h82TTeipU{mz6=-D8n ztFcsu^KQVBStUcVT^n786j;tD*qYjO4{XY@fk;koI4o4Q%_|9qe})=RcXMx7RM%b$Fm5GakkcTn0~CzK&~L4l|p?gbqG)x)4P)NItb{k-VSl*|=~ZpxguGSm0jyi5LJ7;O;C!8=4QCE&&!KCI_R}+y35VcKo;sd#dIaT>SV) z=cgJsMG5mJI|M+6^95vmz#uCah60D~GDs%KFb4RDTu>Ck;)oaH*lk8=5_QRr^6*K@ z0?13U0RYf?>rZ59Uk8~9mVm-**6j4(06P7f3)A7rxWo?@`q4(X2kE@xfHE;KpU}*T zehUnlNWeuKLt@LuFmR9sK!?V{IWr?7BSApMuUykT!cH0Brq>}IF-R?fEkx$rYV0?1 z8t3B48C*gW;?R-_V3fkK@NM3pur-3Vtu3 z;I$pdz^Wc=H&U2AiaN)(DMbM68D7{;9bG?^|CeBc8l72%Qq`VYQ4w;ETSX6OL$%w8 z<)6wFPV$bUo)$u@@5wgHFl*Om>h8nqnv9f>*Z%mYiG41&eLfzzjxO1^TBT2Z(IH zlRW}6RJx!k*q+$@B?NSS+Soz+_i!H+0@P>O(r5>E6IN)f?_7lk06A|1z`5D2w4fhT zt@#&o>7WiPf%_|`78sMK33?HSi_q_td&Y+vA3$ta%LdYkZv@9Zg_( z@7jJ?u{{d5JTMy|bO1jHxj}*h;hfh09dr}C3D}9t0PP~M6AEXQCuSjv5gB{oP`H<;Rgg`Qh20F{u)em5+dqOoIbS!oWWaK85d=j4r`-W<6vDeh96} zyXpfKE=wYjg)UjNl3vQ;i(cwC4Z(}y{}aP!TjsmG4A#fDfCMC<3w z21SJ}i#Z~hM()8G@AUWk+W+W>NkAR;(d4LQQh53I3ySx(!Uo1R=_L9pu&53H%v|h} zlHc~&dV5}Tk=5maA#eA0%4c`1yBIO*RGA@RD!QZ5prHDIPp*RVXwQq#WV&_#{2vmHzPT?1v|f2jP|ykT$iLL_?aMcPG(w7NFTEn@V?hEcj>J2LX4WRfu6yiUw?gO zmUWd=`*VpU&=$Tj{N6jSX8i8z(W;U$yd1G}zv~bC_i1{A%+8VN#Y_xJHt-MOnOpNS z)LD1V#q(pCY4f70Ji`1 z<6tv1B{VyN(uGd@;rBS)@-P~j?lgLV{ATECap|a*1EdSEmhD*IdF0!X9V)0C+gEHP zRwwFWSLtEvEV$r$@b(*FzlX#Xp;Zo~$w$jL^X86USs$5eFhY=l{=6xB;aeZJjX>3* zh~?!iaf8cPYC1lBpba<)yIa!YVz65tXpjWdlFBDY%~|mJB|iIwUCv2Z>65#;uI}Xo zw3W-2E~R}-moX41=}MpyAkYH@nWlK4_(ie#r7UB1p6dlAt!|6wdndal!6*fsW&9$o zPEkj*-zWkzP9W|BOE{Pr)zlATbfZSoDc$X_)Ybu&jfP8X1f2900OFD<^1s|=;fPRL z8Dbew_x;$5TKF3h+neRZ840vMXtS_(?9b?Or{JzTyMiWdvqm4@m9$6Rp{$(|7*p^p z0onzU^Qd_x%Hs7Ctio7hO^RzlNFm#iHY5 zyV?=#<%98NQrig%AmTvfG!>#el;JmAHXOA^bFVz_^NM?b1p$d;8C;*)*6Gvoj;3h)m9xLtB z$z1UGE@>CEzo3|?{&~{bWV?a^^<6C#Xo^^qfiedt@_Nl}c;D8~1olSXjJ8uAmnZ5|+r^FT@gm9vPmyy2{}# zDXn+HTTCz;_Pe0EU%h+xGK4slB)SF8N(x)7^U5DIsuV>5p3^9b!ln;IuvE*?Z>Y-! z-$B><=bx4*jV})*r`)f&XfIlucWuNxuH7j#qjSGJ-jX_PcLc^5g@jko(&uC!e&v zX{$f_Zn$53zH7Bdhz{MgT{qLR+#~V zFK7ARsU2RDoxXGb_r9{I+Lv>$7BG~W{YkFO4 zLb*cOeYPAHt^3$$P_*#XKuA~DR_}yb_d>NlgP#SgkG%L{GvdIz!}jE}{>;KN2Nj_7 z$XHr!ZvBkGm_K%Me#G93SUJ``w`{K+`Q9_YGd#Lz&Hg!JLqlAD#*BETF)aIzxr~Qh zIntx>Nk#j{%lIpwO!LVh$G&L)JimO?m1hS25~EJ-&=OUgd?ciUW&-yFRgZRgg0Ck5 zzBGl`RpYx-XrBYbmGcM&V1REZY*$akfY6Ivf)nPH(*X1=Nt{T?Lfe#}9u3jjcs*aC@WWU6vQ!h!yoD zi7na{aE{LJoX!MZEEZS-D6_N=VqZ|_WNP;ejK_aHyCAnDPUL@kc44!i(r&MVfF491 zk^6@6EBTf4w?H6__JduZ{m?tu(Dq7$M2w!20~~*JsLJd!pvQ&(s?SbXMu+u(lMC0v zqlyJX!_;;G)fQam)4~D~9BN=uf+UOyENCFM3ZI5#>fnq)RK#XRsixA;Kp%L9QLn)M zXBAL&l4sty!Nq1aMM(_AV0{5B1W++ACjh`Q0V00yeCtwBLZC+n2Gr_WKVh6yF$MTJ z=?fSf-xvZyPjjAg?GE-CU4yQ0AV3>FW#ToJmk3Q+k%t^uSRNn|{e*REHnv|m$wITp zIuFp_I<)(2$F@s5;M;{8_a)d#NRsIf(YelSIbJO!d;nh_Ju*5xX#3b!gj*oah~M^G zX(<>A5pYt~jCx@-Vr^letfRw8tzRA_^9~=qdF*$uR1f5#C0fX*V<0p*y}S~Fu{KmK z@|yprV=q8|`gdMNcPpHBhTh@Heda9$+rna^A=QupUpP`@=(qelVUA6K zGIm%{-tNHw3$g8Sh;W02EG+Km;ZF^HXXPvcV#h2D$~#DGeEtqD;Vvy4xdKZbA# zb9pu(o^{sMX-lzj@FEWx8fS1bJNN%lLcoUE7?2MrJTHJGg#VnSa`4M_TpyHzK(-QQ zZiB=*M;<*E^etCg_De5DDltZrelU|h%$*FUVPaz55+e?aOgPXg;woww>46%ZIs(#7 z75IfpRCP4JR+tVh5f=~7Ys-xN(BI&cJ%E>^|4rO=@ns6%#!Jo&6i$Cw8vShOKu?!9 zoBPE_Rm_ZJ+sBLT3vZ~z?pLnw@Vv5O?jKzih9Z1f>L;A0IggJseOzl`JM@7|&iz>T z+;4S#MxXK<-CgFD+VNwm9&FDH%Py`l%)ZUL2d31s0y6pLhj9K*)GWFA^1(k3rsX;B zX1^rb*e3XR+n1;2T=zO!w%GMVEvS^=bNRt`qv#E6KQh$==aPxc(5?yfubV_7N9~rx zjqA-|z<9d0CrnVKfBHKh1_IECy|G(KOgN0e^q@6x|5aZ)qT!4GLFrIB@Ik$Bgq4_H?^ZYJ~01gx@TPn zep>zP*!o9{6pR94HTGT1u7qpTvJNv|_#rUPFL%G;73Ss|TjuKgqb}@Gc(>;C%i{Z| zF`%ibx3^xbSlMveKa_j+#JaU+XYe;ZMz1zkE&jDy?}z2~+kD0K<6Jwo{LYl(9dUQ$ zWf)r})FmCBZwibeM{}OX-nHts=UCc9*QkH1%8Oe(zLFJQkm*`OkK=hgDnvhMJA^~U zb1rGxM0YpaqRe&kCF_;L_Z951HqZSOx!n9o^W7IuiAY*WBWhjy1pP8HvBq6XL$6b$v?#9j(4s@H0m zzX4w!Vg?F~U-aSqdu1o5VcWx%NF%>MoMm#N0id^AmoyZt}T{AYHB5EPTm^8(2 zb8@5TNQiEIK<|9brGWVWwUX?>XyXI0)q9`qCAbU{6M#sExwp|_68{Z~2kY&>;OM{1 z8%qPQ;^F&8v;_tmzv3-|~FoiRZyi;-3niSz@^Mgku%`*`tBweF_C zwoO060u5ChgZaO%U6-IsEX#l#3VcoRC%G)=p=eH!)U%I_gM~_l+v`1e$FVsruC5l( zLkoW}MoCrrEiJW^Gvli!+lIftEz{82z<~h+@W*6|SX8VpoMTVlQCW%>eJ>l%!3Dfg zpx3jqE3u2P>P~w$?m6o`ut`D~f5%rr!7>-)0^3pANB5e6Mn%9mk=|{f&?H^PyH8PC z>A*PuYOBESJq^TNKiz2|fMGc+cT9Jn4Y83x22S0F4&B|kR*a)h48$q$&9*5iO#_{& zz1!6nq64SCe;!6%-0`SeJ5gGnN;UR2l%b4$v`8yHF(Sh3NB{z$yVG2YqvasXg#Y3@ zI_pK#zFFjwGl@wyGaek^(DRKbO&0yWLFmcXsODsOq@IKXC@WZ*L=v`JwIR2^6-Tv?rsbEjv98cyVUf|iJzjK z%P)AsYGcNb_3_%5_G9Y%c%&mzA0*_rMFG!9OA@V(GBRq_sfP37<6A6a^}^RZM$W#z z@Id>0*Y8a6oQw%qjB zm&f%D=xT@*i&d^jQeZ&9P3hBkAkKr0rawN&`7rd>jT<&|4~2*AJbW zF}ziYURIAs3H^g;wL?AYq#x*a1*{UmnOcxfpBn2(bf0p!ZypD`2YdS!rhgwO3Bk)OQqJS>fJS`4z{&NeLOwCS zD4z!%;d5EO4uOhO8#h(YJpjZKbW`yStF(lJKm1?Y90@ACKyjD?&}D4cri1;cVJXPa zhj;^j8u2XOu0Cm9dL|fZpGXpp>klUN1VffE5%lc-kL6R@$UOq91 zH2)`?4KdK3%1fbsdSlBcqyn+1XlOkgg7F-X??VLCmU#>^^n~IpAxf^#u0_ zY~_LgN_Vib`l=yygV=Il*J{+^0O30BrJ#4DxFO;Nq7z?qnmg-o{X@Q(gFaa91}`g@ zf`S4ml}Vun4<#Z2f}<*sWNBrU2jJEQXNw?SkcQ#uAW&TPZUHOV^#aD<;j~)z$HXIe zv;k^bfOwWr**j&1Upb%bTTKqMfaGYrG;oUo!@~z40vvAUgHI{&K1hJFI#H@~flvfs zo|9~!+U1)PGi&G09%HX;Efh&8I_JiG8_sFNp=Adl1oUL&A;h6AK{~4!LW(X+f)N4m z_6SH1Smz3S2nc1vZ5T8#it}RC*2WEoI`HK$*MmGV>tg4=uw^CPSwnv~P-rs;hnMSb z)%J%)qP9-uJRQG$$imX@pjWY@EP7z4*yKaujnXfyak`aI;~6tj~In& zGEIE&I60-P?YdOtwc)yz@WL6_KWzL^!mX>%LVqk_7DXe7SzJu{HYem)LAu&yTY=t> z*9*>X6MQIdGZNQkrq`dZBY8qY_R54fkLSUgr@uSN8^voETwAx_o0mT*EV<~WyF9Cj z%``$x(H`rUj(DGtZ2UC)cAv$LOn3W+Ia>_JydAgSGxUL{;gv+EvXfiJvS0Uy4Yee@ zwK=299P~bE^D`{&LARz#U`5jck7qx}3T`K_%rF{>dl#b~)vQ#LpR2%MIl#Q*CX>WO z5zo4F`-KVAcvN=s=h`#!w02Vd{~kpa=*51lO&cl@L@l1~5jt9WLYT)3PY zG@@+^J5^eq?-?sR5WA=De$Cq4l(r-ryAjLVhqoFe&MqD~(p9uq(y@7^^!`_UBa5UR z{))lZ7hD=IJdfdT@?bu~+yy~fP>!K<+Xhlc9X$-g{=~WFNn4ApS`Y2A6K&^TeY!?_ z+MT^KSMHzaicf7l?#ZXU^1;4;9_)<)<9sROYIy6z$a&}ItzX%o=xob*iks88*n<&$>1L!HZTi_{h4U%FG&=I4<`HTJuqOja$keOM8X z>?2!%&JC1D3RjSjfE=~yRdT8*9$D;R$;LL~n?==CNe;PKtXT&(J8C46cm&Ii(Te3n z8jXao2MCPH;gHr`Av|ofeuJ6sRNNi*`@v{vGABkdBO@bK{I{v8)uj(q^DlTcP5K$! zbTBI;dmFhVAHL{}+ME+n@zodio+3>5)Wq>PkaJ@nhn8n~JeJ_M%nfkk&F05!Ejbk!8Sy5hc6Z)i25b zV+eNtocUOM4VD+V{YK_2@_p~ioSTCiWMMvK2`Ae`vQR4TzaM-VU>*W4#wH#r&%ljl zu7{w{2y&WrUo#)uEkj6KOmM;0={?z3XFbtxJ!G#^^h*7t?}fO$r<4s9G9SoH@zry` zKl0v_cRjQK4JX;vUJL3TeEd~94Hx1vQBf9vVCjV_{E%nD1_udL*UEer^j*Wt2y7j> z9DO;VbMO|cnVQxK9aqnWu!o=0t!Z?B3U>gaBZ6)Jw&S;#5aO=j)ksNr;;m#-iD^vU zDji;%xZx=HMZXm3g;a{s?t5m}veyG7m<&J<3SySpj~olam_XT@tnHxjU-Nz*(YOux z%y?VP`0fSH@ScDrXX~wc%1|!L9DgO!?`Jg11ey3)by-G`E|;|Z{CyVFBjTRUoJg|^ z+dTgB#V;SNHs`Q3e<9)>0vnQZ}0t99KVtwQWdx zoILwiQcH2Qnb+P&=Z$VNr%Gl~NY=iQt$p77n};h}8AzuJAB=2U6SgYz_)&|Yi5CMk zwYv?5essFW=-p~CT<*X2t8@NiFRQOxMJ~)>(%8QJvDy0NI-$05ncbm{vd1kz+CQeO zyg=wByU=F#ySu%$=Uvil(qmp$kgoCkhug$$53hhrC!Qyo)r7SkI-pnc>eT?N&7OnK zvCNgSN7XE!EG+%U-8EKixvKCMsrj+5S13Jk>McsW$11L`V5RJ6vv#qrXvB5t?Uwv3 zD)A-f&CYG9N;Wjd|6t{Ns+l8l^4gD$^Lgd=GMyQ}^>k<2oUavmJC;la1;4$4*f*sX z_NmIa7OG5_3m8GCUtKD*EwEWaT=A#}%R*MG5*fK@TPN*~-Uo%hE{I+-jGZm8$v8<^ zGxDOiJTqJqMbuVbjn);C+&@=W4=Yf z?PBh4s}|Yyq-rbpuF7>%DFi&jXRrT`pV_)a#KnpJw)^$EIoEEW z%8YjhXZmpa!=h1)nN2{ex+27Ju3xm#YK%a$_CGwU;Q|SV^uA`O!`uIB(8Gy5PmLC5r;{Ct>K3j?4jG zera=dtnm34w5nbb1}6&~ICR)q$9Kh-)ekj>kyQ%#Z+@VhUC!GN-VikCO=nY$bNgU{ z(uH=B`tE&=p&DHCCI<5rv=1F>X=7U=!-czL_nEaY-EUNw;Kkc{*!jz=@Qk(p(-=2bxU@%3d@W&(PToH^McFGXHpHtWv1_k-#6&`G zz>;LUKl2$sZ@T+$%U3$FKvbB?X@#2U-edkHlJu&r+*NQ%@#9a@Vc;WEUtk(HGBoWr zt!}fru_@UmD(|vWAd|AxU5ALP(`H|rBQmbFQ+v(Te{_YmMPAw~P?^~X)P^{#)(VTc zZ^k}-nnN~1KYkqAlWtj&czdh8m7W-%1X_vFP8IN9oKJ#S3a_iVBD)serPOj{7_w+G zb$m})MkdOnlV)~iU{<<#kqH<`--Z0%_9}kZZw8HeeMFqS3HBZlcQvDkOUlPxgU+ht z43t1%DT56MLQOdAN^wTD&c3jRi5nIarG*_8l4dBQdT-3_59=Th%&~*-t^^>-;Bv4=F@b-QC48ICv1YtRg(k3 z8DXOhqAr2g1wue>khu@^?Af7Mw~$OpcfYfqeAqC@>dwV4aBjz5WR6#H;|>-HtDpcY z6g%$^zZQHuC$oeo)j2t*IhZUmhv6=x$XQPgT3i{$aZWALlexb(;LpHGiuMA!X&xn~{0P1%_$?rT; z>5oS4-;b4>@fW}+0%E7yBRm?L-YJ3yLbYdUuajJ(s zo0CH$zxMP`0Cn84&T|-qXyP(ozJrjhl4LA=P0@_W_u+Dnu&&8iAR@d9Zwv+>O19eU zK0d0_@x-z$Zprf(FN&Y&h`#&PE%ZoajuBRG=qW*2{xLnUx$5;M1cV|3{kzbL$kS%) z?+_T3*?K8ICx^5Z=4EmBXB0$BVk3?91#L!(^x_TE$yNdjs@XxFacFMI+L7BY_xMwE z7Pn>2{bL>Urs z4cm2f^S8*DR2^@HU5D5hOn zgqoGp%YUE68A|LA=TBuxmMl(|WfI%SvrCs}*F;av`Vjt5Z-tlTLDRRFu6$6oI=pOU zc-iV}|Ljp-Wu40JEMbs(37Z1co@%s!^7Ru(Dsl&3>Kq4E#TCWhzhQ?AuaLSM4kXJFh=-S1PHn5M0fUu8-$z; zQfUt#ZhfJ2fE?sLaioTUjcZx>=xg)UpdjHxGoCLW>iw!^15aRU_Z%ONMO)51JbMB4 zsT?Ngjb1?<=sZ5@Sw*rv z^Dstv+`IDD7a9|gyKP|0kXf(dH7^UtBV2ps#e4n&zp*R;eZpHS9YkhA2{9SXc%obc z?GV|OLk+vf0JtLdkko?}-Q@xFO&;ek0Nd7w7Zs#k?Jq^A$Ws`#%FhB24#9EC$7%#E z-2Uw)tke6rAq%_P-~{W!99+n0t`NlmWK-yO?7vx3|E#>fb9$I^ZrdT%=$k$oU!lL4Pwu^2w!osdY>D|rsqi5=Oi9EJ7sMfX_yEHp2 ztv|koHD2QW`NXlaxE_atm&&kuMQzjgw^N;!^Yql@+ZDkbomcCXQ0QAHBigQKu<}Jp z?KPQwi>}fW`FTf-s@mBZ^NKlq_Rq9>djl_>;KZOhYPYo|*pf}a{NKbC?}PpofX;8J zp~tw*9KhsLaWlR1;n!O<6oI%ktz+?=IkcxV;(q)AwePcYt-4_TYe2zfKgd-DxZ%VY;OF5 zYxvBluS4G(!3{O9O5FgglSEE%^G1l^3}4^762TulVJ}dNXWVU*Y=FPNDwzEE0u$Rq zyvP~Fb2^?UBI*EF=;Vc?6A>aGD0vKyPBQ1Cc`@gqWU6C5mPB4Zv=l@8^wvyue9|R& zHh``-C?qZx0$NQ){m{3g+RUv4KSJ?ovoW$att(8BgeP*g}HbQ_O4tVk^1c6s^5pa20SgPRvxJij@c$9nvEM z1NMKuUx3k-!wAk^J`5E^Xx)C{EEZfKkCjv$SjjaQI58M_Y+?jv-2Z;8yjOXj6V!!m zJ;F>|M3wRN4k1uVK`P={Edrv6iJYW>gm-Yv)Ke8z7Yu=)(1o_u#{yhV61^+C#p>c5 zS=@msd;nymLnXig{9xV3qQLGchPM^?T?8;9ED$Yw^&nHiJGB%#F=meo9j@#f-l9I^ zbE7_MMLMtcU%X`j#D4WBcUqK?4lFpBCFWa;6qI6;CriaZ3*C9upAVNbFQ(|Br)$X< znRH@&0csIuU*Il?4p)h^t2OUIMLvRCuIY1l_H@CHGbzOB!c6bi0$oNfylU5!+j3C^ zRIm{hq=;c4=aLpBD>c^d>1ph8=nT#r-PXI+6pEBZ(&pON^*hiILTZtcByqi zxgYvgTg$z$Uu`&XmZ{-PM)lH&-zUxPa(kTB|2o;jg!=DFo-mM`DxuvmrIXKOgp8>F zE6sgdl-)goPIJTuEt6Wo!Yr{IX`XD1`96K z$+3G&2cgQnvUtlyptNsmPY=qifIkeHc(jc>KZ~BfdFxi)buIY|Uz>YNp&lSf2~OdI zV<3-Mtj2JM4hX6oY^c48&3m0a-eyB7i<1fpr8Ok3Coblwep1=9hl2zZhmso&0RiVj zq&VnGW&hKAs<$V;uE^ZAi=I#H{^&W6N@9+P=Ju;GzStQBc zyJ1wyO12I{j*(G_(zGifdt}7hC^M^*Xvv7r(Mgn*N-9TtESHK|ral(y<=4#HQ>-^}MFRpeZ z*+S$uN(D}s2ky2VgSF$;k#TBCd>L)s8hY$W~kN_01c-2o$|L1h%9QL@kUqMR=0DPF1Z{JAKOS4TWexhuJ|k~ z{;G2E=1OfPI;UO{xrCF+wf}9`#szWSSXrR`mrhx!9a!|uNxv|t{6~*?{r51bHAxL> zd|F#2f+8S69rmF!QmoL;${UV>Ksk-86*-a;0d;vRBQJ}~tk;s~J-sJb4uryrg0j!S zeRP|~!FdTQ2&GZwN!c;|=25*S$S2vw(3~g3`Snp1@xtQjru|(Lsb(;qATjaKQn`z^ zcG^2iHhV(#WIP-16^h6f?4YBwb=@j_7^rRZ35XoEXUt$GbqOM(J9oV&{{`Lc$d^w} zQTVXIbmHwX$)l&y=0lymMr%JhNc;)>LEX+qA_`cT^w$OBGl8Lh!XzU7$2ZzG%iQ*ijfgBP5A3SD?!cmoh}?wwgI-kwk+BcI zSJmc-`rZa z?olOj*%evTCXS8gG(Vv)`tT;HS@sELV8od8Gzk}dVZ5*VM)K9_PE<4JszTaCeT}3Y zEy_y0*D*Y|9P|0r?vxy60c@Z)a#Y z%dbKVDK&%F_emAd3zFxQB0*nB=6!pl-tv&UckXO_cesvi*{|~{dq&yhW9;p2gUqeGk?@XgK?lC^2_lloDL<0o5#ZWw38B94f z;Wz~tANd=%#>U$)gxnTa4VuW>U(U?#*?h;LL#@6WG}UuOkitNzU^&{dkfa4+v`UO} zol_(0^^#N={0)52ZqT6Vig~vLjvMiZ&C~J)dUGchy6joM zILqVn$lMo?J+%t!^maZy`yY=6s`o-zHU*^}FqgWjA`y84;B8!Bs`vctTW zXj}W@lp*6MWo$eYbU-IyN7aM9)G`~L8}2!<>DWp;T_7mN4@%*4+pG#40Lr(qbH>?kj;hUF&r>7B{6vMRMEhKAPgIq zKO5XcG8TdKJ%Kkb{4oHEsus-Vu)l-3VT3mLd5<)@R-dP60-K@M;lt%}qG6{m^3T+# zSwlyVLKo6*|KHUQ@6+qc=@ZGff5@ zNj{Zm+lz6Yvp#+OJb6@&n{<`a@p^-*v5)qutd~vW3_xU_V>gy+ zzr2&PK`8>66AXA_rYrYI}-Kf^-#-7yYuP9qrW0G4OYBtd%|VK?u>R{Gt!^^m3Gg&V^YjJ_oc<3a(;AtrLj%( z%iiv}W{LLWBIk(^EzIDa7cq*RDY-0 zdtM#cr9P(VtXInhG;A|?BdtJjNqOxm2*4TR+tSao>}mbRGES`kDl7}lAJTpQ-#ZZ_ z2oiZNvHu!Pz-0Qi8hm_;AK&7Pe7n!Q(YGj%r(yUuOdZRY_o|rW08mQ|pE)(>OGDVv zG4e~+^YDb$ZLDs74zp0IdvQ1B=J$^J*fH=qzSq^|P$itF!tzf7*|}?+09b1y>7Zby zbsx(crDl1F9g^vYt<@6+wg$E_+rvp^Zh#;G<22+#Gq|MHFvJu1@4xWMkbBzU$;-Y`)T&1Q#lRDxIsn~Y40Cb{!M;Q$Br%oKJag*5~y2cSM7gD0I=tQwu>Ya zOQiD~`Jwf8lzk5{x@8;eTrBahb5oRsq4F=?qQb#Vmhx>?;Gz&O;G$@LBxfh!2^yWj z^PW{X`94FSEh6|sQg5tU3~g-|ZESo?_)u9Vtb~*mCO!k7iNSFAKS1`Eobzn``?=5s ziZCc*-^l=OVo@KpHgcB;!EKtY(AT4TJPs#=>ZQSi;fkCUES7tlk>QiyCv-onqSjWe z-qW?veqmp~wgD&XL8f+ss`9W$rZXzs;~Of{G_8ZB{hrWY=r$}FX{td~>AwuI>tT82 zv?IWaD6BPoM^2hb)DO?k#O&l1Hi<9ir zHx6o>VVh*_`OK(pkU?YfmUek#Hcn|VxVWL58cV00b<-|Y>sIlMtlj%!=JoOMYrpMn z5V+b&v#g4W`SQ+#ZgezOxXjhrmGH)UvNUR>5rvoH*%8asoB?%<3Fwq!7H52<-H6R? zX>jM28%iro{~hMWlk8t!lEG4r#-~+`h&L4dE0MhoBGb9B?}31zN?L#T^DSUOa*Xg8 zbVoX0WT4ARbf|&bKo*BwC!ECD&+uh3U_OweV7sjzHBz(m$ygC;&xlVaC}e?#hEaUA zA*cIN@@q`;3`FZqgguE&Yix9{2K!UfZ?~XvlhH4#E1n zbJ?XmoXRouA7s!qD>*gf3%g1$MtEF3&G{&SZQ7?(4|X6$lp!?7b{+x%#Gmk#TfDGyMPSDt4 zy}!oT28+N7m|rasnuqT&{__`;SoB2cQ^9fG2m&Q@lS1Is>JVm!D|cFJ(=}5#8yxBM`876epz`2MZVE zo0k%bGv&#L6;9U=4N|jw^x)5^A>q3V4cR)({nP8jBI$OveU0wgP0 zujT$7&X$Irv;?TAPna9!=vQFK+GkG%1jr_-SHg&8YHIq{Cgli>0uoyk*5}T-C{~_2 zc@h-g=?`|f%)Z^Y-%cO`8{E>WuL7@1w@zRKUmKv9uMoIMu1xT}#>x6?rV`*Hwo^8I z4ePCJn0CFy+bP=uEVJN`O+uC>>!5J=#pn1r`KE?Qe_DKkgAyu&&wD9C4$%fZ*WwcheZY6g;;0h2~H2bsV8 zmA{YI?O_2$4q|clruj3hn8b5b+Yz=L&sX8w-4@2d*5yB*flZEeUkYC1EFvbF-SASX z6jA_B0h7*(8xd4ITWv+JefI%P@~rnI5gCD-=8yU z&inJv>%=dka~9CXGC}QAYT=w;)&ziD?(DcpRyXcYu-} zMS6I+XKk!Q(?JHqgzqVzW)i1CsjIGI*3=0-6T+$^neK;f4tB^~%NUx*Q}CUB(B)Blj9dkYGaZoG@RuI*8J=vk$ z&mH>u;DVz&^(LoFRHP90NxoaQL?_urc6)Kh-+n=`l_Ux??!3%)B9%@oWq7D-51O8i zY>afHwGbY+vQ`%x=+aT4Ff!&w4-k!ZUdMJk1_uFdJ$~uZqiiPvd;sg#QU`Mu@8}m{RAt$d*Dj`YRNT6**qHKy4-?KYS_K4%-%?@n!(u^U{^fJPKe`o4$?yHq? z!%j^w_}=>G9c0Mu)ZgEIpx4%|8yT1Wi%dK%>eKTY9`o09z-o!JoIiA+v!9~ikB$91 zd5v+wvVZ>UeQ-SC!8^W~e*ErVay5pDcPcZZ}j`al3tm>~mEH;Tnx1Jcz z?L3I%%S;Ks$yK#Jolos9NRUY|ID#y6Y>YBcTl;GqNq9wH-yuX)I8#u1f{5df_ugeW zI-MQ>&5ty3tc{Adp;vwcpe}hKLC)h8}eMXU7`Sj9WFQkStUjyZh&FqMjjv6 zs0PZI`m6zTFL;~zDzpzd8!9~F!v^2*2O{E_y5{$&x|UAf=I#BRbHZ!ofqdkGowKOw z>4GJI?9C0F9d_$aT{W<^#Qq5Xo=1yhL$dU)(nBo4)53qJmNuCDXPdsAx9uD{ z5YV#HaaV4hW3CghlvyPzNUT)8MrF@4Qn$6L2f}D@6qS-RV-?B`B0F#Gc1<`r<=mW} z&BI`bxA|o3m?x5Int`J5In}C2J5JJBVB*J0d6bo)7MJF1h-+}T3SHCxf(N#;>Vl%{ zV|#qC^(IDXBKDtOTeLfFy_58ljJJ<;d=8@NpsC_svAg_14&Io$B=({1fPfJm35mKS@N1Ve|e{N`kvFB}D4%u$m@vy4QzYjnHb z6Qn>CU76NZ(y!=~0#ia)F3>Ojd^K$URvjDV>PIhr7X9zxTNxoa1}ErmFq_In@&go5 zI&3h!vUM?FGA&>=1QwawTctK6TdV#{h5`o0O(P~-#z)E|q|=v)aEei>}NZzV&Wzu;BSx=lOc8MS7^Pnsj)^FASym?S*Tlx-K#yiYn z#(A$^&8Xj^xACqq3Hn(tuGW^WAb6~J07#a zC699V)`K1HFSy;d3b%1{G<4`zsuR69EV}khx8nYA?Iu3OsiSpw9|uc9C7`3IUJ~vM z6D1LpiU;;Bx78x-y4GBqqT@Ux@w>seqIgo-?t=7J+%smMw#t^>EShe zC%x&Ty+h0dS^jp5tu&f1E$APSU>3K9vv8>J}03nFaN zoypAN`+dmOpD-r!W*>Inb!r>xQlk^O*kqJ<25D$Rv-4+=diYj!5GU?SBkF=*;ms##}%l+NQ}~v-l7@YZ@|8cf)~I zy`$yRBe_KNM0}d_j~-+RDw1+SaKiod!j< za4Vm41A*rbm0NHkox}Yh(BrTNE93z}#Bcb?>LDB9=oM5)Z7xRcnkGM~ zfhW9#$h||$2G$${YA6{)mRT`IWxeFecE7-KdI=R$ytH%wb|}Wm+GfG>Hskhfig9M` zN340|yngrQZ_L*8lM|z3aIkHi^$G2i&x_Yl)8O0Np8UD`aOaTQAK#2_ZF^W{*}M;m z_#xA`4V>(hV((I}+tLxHy8&!ZUH46ZX6~?in7cJ@}2}gVr zk?GvB1az&~J*NK}#4fKH3Nf3F#H*WX19!T)ev9=&mwz|SU?32ir4n{I-OMN)4j?NMDk=q&)W-TuuDM3X^>S7 zTb?1se5S`Yl#yH1B-eYy3W$*mxedhZPk2o4TZrD>9qmTISah)@t#ATeMl>+3n_-O4 zwrzKj6o@Ry@=8Y-^4V8k-~|>@l1Kit5UUH9l5X4VY+bRZ&bH;Y_v_pUX`h=hOk8Oc zvQ07Rxlb#D3?Pxz;%Bmz=h~oW$?sQ$9{ur(ECU3xbVZ77Hy~y*ugjV~Yxkm4Awd() z1oIA^Umwz^l1;qZv;vFDtOWnVj;H5Bd?U=SahtLS_q7M>1_o34`;#TUEQl;s#%df~|0CmnJIu-yZgH zdVg>M*1Zm9g|pEWpvSvIW65(@s;N#4Ayc-3A+hT85^In`I!L3EoDG47peb854St-7 zv*2{k#jj1MafKei6k>;|%|CMnB#^K=KqmJy_I!Vi1Y2a;R~tQ;Df$`qWwU^|(c5`a zAI72=T`V0|=KSEBLTr=HLG7)(OZpHs%j;s&p`pvn;(G=!F&f5(DTm`;ncjQ$?VHQm z2ScA@wtZWMQ+ywB>?|o!`A}>EEQpgHJ89i=@;|yEJ

B1_*nWaavvgkrguX*+Ho-J1m*sMUZ*+?5Ig`M0D^30r;J)Yx>=bq)o~4KCHi{ zG12VXG!L=XnFflkskYUH{$_Q(hwo>^0C~8Cz}B>25|czc`NKHfj~KVv=YG^y2h z3yg+G$)*hWX-cDIIG2TJsnkfgvAlARm{QcTPcsH}t^%M|g_62y`0+aKJ*#-CYTbkE zGzn&DfEkCxVL5DuyWhT# z^Y=xQY>-z3AR4%wn^YVe^Dj?3nyHVFv$E^r#|99!i#T3;#lY6jZ#wos1ys#L%hxv8 zN_bV#-<^B5NsfhY^t%;vo?V=2Sno>mUYH*GS^n*>0sau+KQnvzTB-e&Q2nR_ILamw z!zT+Lz-TMT0fT}WL*IYsJAeaTaWINgT@FPuDr}IhOWAU5!p2rwG^}5+RhwP@akC;^ zTdZg1}Va(oY4)qHyp(O1;ROS#oPPUQ=dKg;6Nm~37swS)!}<-ciFW~&z-jVrrSN_ zAsdmGXm{>BYQ2L)kjQ+w2}piZIMc?A}Z{6&dAO`6#-*(3zODW$o2K^03iLH z*pOo}`${c$%SGB%7M_|scgMI5^9;IE7Lx%;UY@dT{ajf%9fp)oSWD;p`1LYa3!kI)^H--QQ zM-m%*xB`*LRUKrcrq=HQ)~UI`ODPfTo1_xXs)vDcDW>S&<@xO}#yFv`K&0^_wQem7aal~8=hLj4^T&Q2DL`J;L>MNLTG8HI zeDdMa>|50?+L+C`z*{J%1Gl)Avz-kM0*cs2Cq}AXEEv^o7PW)EX9I$HqnrcGE$u0E zgO7CgdZ4$5(hnZ0CPoQ~65)Rt$)V|BW?Il0X9U}8=vYni7{z#2d+0<27-*`>mx8wZ zu@#%J(%$6ke51w#SI(%=j?aRuH4=va5Cd9``ntx1>}bOune74U?Zy%|E^4eFgT5y~ zVU@u?MZfz|Q06Q_eMrHoxnKCzXws_Z(!;*r`7AG|PPMWXH(D?IvT&SnlfEh~a{f*+ z-{EU5-lJibqd2?*)8q)SPfYx@urFwSuquY2|d7U(?S?gop%%jcgns|9M z!WAe!w5z$hInL6QQrgY;j5%My<}^*5oMI(_b$OlzY?!7f!_jN&OcvM@# ze(?+K64l1nO%r7isG=x5)3+V!HX8jP-`;k(?cz2Tqeex;a_%^IutjA6lbndUNcBDMtC(`!Qh&nvwL(Hn{L^ z5BI8@hO1>p!UJ!rf@&1d3%HA4;9gnoKZ-4wn+dx`pq89`el)(2wrts~xr;0uqw*pP z%Pf!b?DANb5j@SmUV2NoakFQ5-TbCa>sFf5=f`#VA|PiNU5# z1_Q&co@$n_dE&ZSN3`bMRQ!qCc*Ke_X(bqriB4Iu;Sgei; zBED~fsD*xt{Sp{E9{cRoySmb1sJ3FZ`IIh~0vE5ZM13DVsNr#QQbh}Ej@fob;F73y zL#9j`q(33%c(cy4I>h8mvCQ;4rt+b|9BW~Iob-<^k!A9GUJ2PHN6~tO;yTmg_?!=&^WG()=0Fs5b>OVx@?8Me*Jot zfK%fEL?qLLPl6G^U$T?AFQ9ImIiR|MwJPF`Hz)Z`f zpf>>bSB<@K^YJV(SEhjq8`=msOEhcYtl?fOLuXWY`BL&PMchu?F*tw3uP9i%oA71f z{BaDj$20ZOqqV-VI;(ShDihxN*z{Z?B4=D1w*SQvQv#eI;Wacek`H|PVmg!0_vaO2 z5b;a58RVYVW!aAk$Cp$#Ua|iXNlX9T51L#Wq=q<;VIR%&rddI3cpX8jl3{FyeIm`* zqvS+jJ_zW>LlG0NPNXiOx$?H9YrD4ntj%b~VD0-zipnL_Q4U{n|ji z1M>;>PINBA>^4mj?_GpS1hIEjU1!yGSfenP&TyoWF?v{oU?2GUZVH2@fkmQgMrm7e z@#+OaVqjNLwwVF_o+V)neTQ2_5Ffv?Nk=e~c~Cr+1d8GREZzzqqQRt9|3%(kU4RQ5 zK1=|2w{*j0-F%?^F`a{SYa+7c=RoK#>ln&(4EFvS&CqkhySLG6vSqFHCj?g`(C>v9 z(PVzlW~HH4WUh0A?o&O>UJ{BMYZ5T9>HNQZ_6}Aiq;DGZJ8<=dS>xH_!_OT9G@JdA zuHMjleTT^>qV8YQySB}{&I|Ll>)W;!bl$?hSt8C|=?y{0=>U`9K4jB@9md790He}Q zJoh?d5=&?Sc_-&0v2!1=$FlQYvE+bVywcfid30Y)y6C)>3Wk&_L68*C@$&s%E^kPy z4!JXoWmh4(*d>S%K28a@P?yin&kIKE{CUw=RvZvy@JBOe5I?FM#z~pE5?AygUIu6S zXXX?4m_B{_NXLz;s`j^K6~zZFpm|XIZpa1Rz!X|f6o;4 zWe8~-U$4&9Ry2zONuzeN@Hlpa#(no0+}H9bqkfn1%Y*ZKesv!EqR)=2D=GcWTNXQoCP zi5?yDzHJ8`m2Py0a5d4`|4S}buU&S zOU*mK3LlbqRGH&DE}Smw+ZbjPL@!GN+uX?Ab(-r1dO;gHN=2MZ4AR4s-e1$emvNt< zcR8n)Zp^TJQy!1LjsHgSh7WlFQzso_0}ptMM$Y03){&YSasCDQ#awKQo}4!LnqH7{ zF-Fa&+iURh`;o&2H!wZ@%hPuF_w@in_(t2Bqfr?!`lM@j~JV|C$a(wFR1Yc>MDUEphMJjWIW4=1cK4A6g&aIsQQXBQhY0fCb(Rdoo(HgswF&hSSiX7f!KC-Z)CZawA(ju0qYZO z^PUo0BT6Us1DGobj!bK+fGlU`0^RbF3uoGO`*svIKq(zHuyPRl*UYtOMr=>S{O8>Y zfcpltFYr#UOVty!v7BTdRZ+bW(|^z)VF)u5&ZtLPSPjH63*3wMHyX9XR>zMCb-7sH z9c|Hg1xp#o(h)yapMC5vLodloQ<%Za2QU5P%&`H>aCn(A;DNM5e;)t%I;?v5Dy(kP70_!qe_%_DAj9D zAPd^4cf%|00;E9=_WzxSgR}EqQXxrBk~|S-XXpEd7Pb#$b0~p(#}ta_Y$w(#_geO9 zV8j;=f?~dMA;Rk=nf4A`XF)?EQxPm)GlkjvN0>`E+KGC??+XjcZ?RMI*sS zTMx94N*d6-zEDvnF|rK3e+G@WUbk*=9&0{+BF({RK<15*+(E3me6j}RJ>@tnEAC*Q z+B`-3knI2Vvdj$i5IokDkvMgmoz_&~G#NiuRwNkcA1}@bap0LljkZ^&2B+UZ8jfSn zg^$`+kmIF?u-{TdmlL~bX}~ZnH^YPpySN#K14`8t_q61ttiBl@Z$n&zOxWk9_6VGG zRxSp~dlKZ2?zAY%ps5&Xd7=@@*B;bnQ%fS-z$TGB7j6X|k~C}tARB5ts8jgN#Y);$ z@F>@NLTj-kSvS=4Rhqr@9hl~FT0Hrbcc9OLhVXpV+>=lL#p2%2Ng)=KgkD0a+z3ej zyFL(p^@x(Mqb<9!l2Eo7`QLu@U$Ijo7mdQ#Kf%9@_cxV)1!cf(uxA;Km77_c6pmFh(6S1mG-0_hS~K|uW`sLWo+L6VIShvnceU(&jHNG1U1Yths;&Ch=Me7eI&pNu z|E+V{_<6n_m&pI+RnY(Lit8O_fF&z80*_XB{S8;mqOR}Smy~4+ok?(Sh{0#m{vO<& z#8(i68v;f#VAx#eCiGV#)vLYvl{#wk4K? zk8Ea3on@pBT-4Xkw7B!$&hs*I8J*8jO=stQOiYP@fwJ=`{)1lDR4ZfkO|raeDf1oC zm{jAo`|;z)+P7Q^Im;fc3c<6MQ&2@MzWtBFbYC&9+IdJYNm57lAD{WC~D*!esra_=3Gp}13k*ZLX0>8SQW>?>8(q*`5sReJ$5emvW?YT{9_Gf=}gYEfT%|SEz8Gq=~Kw9V}wE>~KM6IB&z^ zhR1GTTt&}o9L$!@KCgqiBrhAFj7oW$gyEz+nB#XdU&l2y!dz~&KI*B3_|(F@dv*8t zE_Fu@GEx^onIfS5Z$w3KSD5J~Ce`!zB?#%=SjEojRQ>i(6p#A!>V0zA{FU5pxQ(g8 z?aoSYNm}$K!YTLYwb-$Lzb7}~nqo{p=u`X4p#OKm1m?ARl-70BFZOwJ70sW%^*4r&kO&EsX16bZUJf(QSKj9sDOVr{LeJah>HMJ^M7M;~PzBF94_WJ63EJpb@Jn`vb)v zZ`rMT_l9sJ?wM9fA<y^zH5*l%>ycN{Gf9H3%tDAeKtnAW zcaGw)-qqHU&(o0VW|+sYAWMp?PFF1!Ll(UiWhk&a@duPka1YPY@VB(UlETk=u)q|2 zet{bitJt??cqz7$ zIiQ}tC53VBfgA5O`?1)rk=mvqM@-l4ziA)doztT2>`XXojWZ+UF;cVqd(M>`m#$uK z*x~8DjQppXO-~$H$l~fYHara1<58v$u)B#csK^+MALz*H z(*>@bQ1m)`V#+lh+=R6=qYhN5TBVA)@vuoNCQqFzk>VBDEGAJ4hob=Ag33Z8wJX-| z+S;m~M5>A|CFE6MI2*8*0fLpshI8savMOYH|etuC|uzS`-R)7vOd!x5)9;2M4Sn z6&?|Q+EpP#zH3Zj*QcapoQ_}EGHY?L6(@0H>2L@5#q?hZ6z(NX^;-|@IXlDgW-_HX zZ`nVq9Itjim)LU0H_qeGpl2Aa9)EBkZ#*Mzg|=;c|0U1Gp}D3QbjM`exb6RQq4;Sb zE{1+YF&`gR$D`FdLvLhHV+WuZh0VY6aI zpitl^5t49Id5T#2#*LTRy(p%)Jyl1pUcH)7jPc(%(7uwxMrO(aRC2=Ya@M+i41Y3y z1z40T2*-7gRR&tx#B%}VT=)45&CiiGcMJYpeda`?el1$8 zjd);N-|o~Vt4{j|H>`4uc|bg^Oak4c%*r-b(=l87Y7M+V{*}o8l$ssc=%uo0_%3}B z!@Uu)G_gV@+XGIWl1)|2&y#oTHW-Yc1sY1?UM?k-mPdHAh?ff*@W;3IxC5_WPX^}S zYh^by;0dBVuWA>tYS2x6pWcY8tufEA$^7}Jd5l^*zK*Jjx+^IO)0|{3)!jQP&o~26 zOPP;2IYELi`h_I535=Yun-CmMl!6oGYFqG>Z)RY?cCY9uJfKw7cR%w&mOWm~mIlPi z{ta$`W*%KQ>v1VRDV6wR^5N0SYbVd!Bb)y25PCmCTplH5KMT^LiTU3vXP-DQ7~cE# z&>p{1n;VW5g%*o1uGEHe%pu8i@}%OTIFEl~Uv@eVXb!COhLWyuMs-V8U1ufHq41z{ zmFb}%)uYE*rl#2-_F#et5>>2uo~cU%-Ae`v(sb`Hvpo;ykwj8Y;w%{5Qi4poZD!!% z(_PNHB=R8xUEna8+n-+GvbVyMC38F4qR3CZ2+C0tcgCbidun{hM2@GlAC;vdBoLaw zH(X-}kwZcr;H31wOvXC*Q*Fpni@k%INb<=O3!}WE>7?$7PHaHtnqPWsBEH1D_^jw; z%<7+rU2$TNd(#~Mq@QDroe_S>$kHr3tu+_QQ0RURo2>N+KiS*GGfbQO&|O?fm&dLh zv+(ni0Tq=uK!*Y`x$kU8bAu(_BTA0#8+KyP90#i=&N`->@60oRBEnP?HLth#M*vTN zj^jG&_kQI~RT@#&AKg?omIi5vV?$ujp>E;XZ&fj-315t;zn$B}zbY1!Ehn75cFF-! zEkIQNNN)WAdRc5f2xCls3g^eF9pp)lCL;e1vj-g+`S@OltZ${+>*udQT}b8%*#{DT zz;gC&o7Hc%oCvYbmpF0lT0*uHs*zsQ^zJ7|P&;G&;L~VAfJEKyL2k*tLP~{YrCHv% z+-gi24suT^$CyZcVIs}ZkyIn)xtUx$YdnTS@X#wL8IT8Fn&%nQ7_^Trtomej+~g+8 zY?umRiQTsW&Ai4=_d_Z7Z)ZsV?qR0J$7Q*)B*=w8aCicD{IRPmEe1mz3_8OHM{MXn z{s#_lP5hR=bocJ2JK+qL>#hOwU_wj%Jzh|{g`cnBC5{=!uZW~@oJD`y;C>O~F1?v? zH|B`X{Fm#?tJBTo8{1GhntF{dgxdN0=0VM4i9X17G6@7C5m;Oo2l6GE-Oh)c3ZSMzZ7F5Rcr?yHXe-|A7Ph8GmqzciHh*TCSQefE$= zauqZBPx@qUMzCTb1Mmv5&#hixOOxpE zW5=Qv^e&SC%makp4Y-M>OfjEx%O|akjucbW=BECK zVxKNdwGB43&wVgn!PW#!vgqdfTKRare$&~wiIZfHptEjwwzMR}NcG^FAqXsgO)k@? zx2SfxFWUW+jyB!HXQ@^s>Xtq1pt!X9d!{9;HItK6Ju}Lb-}cYHbc;f*=^hk6VO{L^RB{T)vRs~9wH-@*Ud#Hmv~Ma2o%JIM|jLRw#H z#JJI0q-|vPq+*cx5FP8ijU>L&ME55el%F5${Ld!6+z2cH|AClvc;!dd$ zq2vFMCh;e34Cxq+R;9wGHLrKgdE;tS(b3g)7EKmtOS?fG=MquFfaw*I8t&qVi$c7; z7XQFsiiKk7KVAPz&pxgxJ!>_gD#S=Vi3CJRa*QV_8b~>~K1Y+C90OhQyB;35;6%)P z<<^;2Of2sVY!b9zf%o9zFlG`@g;3otA3NGh;nm+kv2>OwF=;d}ljbDbnmF2xbr!$C z9Fqf@!RJ3*nFRqf%q{k*V7VN4C zp1(riLzB44w4I`PlC#>c-lh;xsdhbj#3i9Pj6xmvX{}L+#M&(9UA2o8`7#I|Lf)%d zo#$K*t?mcnSk?`?yvxkSKRd9EzY&TFt_g!*J)bqhb8)|I7Vq2JE_xYbIyorYB>+2j4+qVY+`@Yt>Xq6 zFt|a;R8f6!=zKOTKC=#CeN_pdE7d$P;Y?45%<&>s;4!PRwnD#T<}ibk))QOhjrF&cz#QHHZd~&>8c69`Qv>mtOc4 zUxXlIZOKEl^0=vh!I4pgeY@c8HT*8>A}P6cMj#H6CGUD5HvHGiB0Xi0n&Y7d^HiQ6 z_QLvD!6x$fJ;*Rd+h=v{i%N$6gzLQ72#Is2k6cKVLg28`_zaD$SJ^v0oaf;#A%Q3u zL<>@x3)amS=?TI_NsmSG)Xy27$b@(*v7561y>`d8J0;u~-?Ds8Y(6eDFg-OP4R5w& ziRyS5`K%9um``>6rc7?^bDACO@VXT8Z|>!732BI-&#?yzx17m7R4y*&mqQ`%A?Hw< z!dE@c&}R-23WayVPk&`#u9LpfeK7Ne&RI#M2bmmZy}Oi4L{MatetghWu)PJRp0t;* zP7cWe%PfwZF_OD#&PQ+zVXaF{vY;maK``xo-_-f%?|%5YdF_1q4v$a&p5@!AN6+bJ zR#8pYP?KkvR99K{?8L3u{Vj~kj62V}zo^Y7P4At%#@*v!M@wcp);&uDjNxSD%^lxS z?Wmz!OUX<~te}B6AN%vz$ccnG3$qs(ct6{TuE+S=J)FeeP1mrAljO;#`dGiI27E#$0wZe zYr7nIhbE(W6ONF?ShkQl0cg?-20@9w$#JMjoVgrC5`8~9x&jp=#D3SnTlt_h*@ez) zU)Sl?#63&K+-p(JXasB9Zsq|Xoyb1(Q9J+3g4q9OPp4tlue~#e$?8StxNf?-wFD|R za&fK3-3REw<1Hp#_u@4BR*RwhDhQUA{x8FAT!Z%Mg$cbQp)L7ODy_^KdQp%wl}YEV zR=vo|uee5F_#Ag%yxD3M-;aHomG-WO{8qtx?5xqN8kMCR6|VdjwGwzRbV$cHDMgKR zcXyh-Qk%GF+8_V`m5R=d(DGw$`Gq*&XI=v<{O}~6oBr60LwY+RioXnzP#{}O%;F3i z=qaXtomE&6KEDcm67Ww(it%G6dUj~vp)*)ce`cC(!iE+h(q$e&mi155)ALborlV02 zwvqcZ)rK^4$zlEeG0`U|%A{u>?Y8^U@R>9 z>`|?XFwGnSmWQoZ8=wQ5FHS-XM+#>`({YGiJ68TBt|DpDI2fLX#W1}B2kC{miDfXm z2(=5mTBl>r_${o#@D9wU%6;AGp?dh-C7aQfGl%xU1Qj{-3V?(T}a%itHSf^BPVxb=D| z0-&)wt}g=nQzFCy+~=aZpG%e@$m&mq@#?g$(k*pu<`Xw<(xfH~b(=+;ze8-Ln8D&_ zhOVedRd1Fk=QZuNr{t#lz1+}jUKR@xL_f#3Vzy(8*~8Siqo@no47P1875hvY)u`~R z!=0d>^Iz?eOls+k9ZLck0)EGbiN!x-qaI%lcRDzt@J&5C>+bv09V3fp-jCbhj=>U= z_TJ52`ztboaQhGaCn$a)@`|vr$`IF6)5{Mvw#<}tR2*-PmRbDPHUxbe|Lb6Q=}r5B zZ&2bbRPWE|8*J6hPnB6617zczHCII4m1kYv;Xfq=3uxm-KK87gtt}U_6upL$zt3Bo z#v+f`r=RoF>YS;pvjK(`sy^%W#>Nhj6h|p>%<4HWMx?nTr6fdhID$oaYVm;-BCQp} zBba6X8vzo|8Bbkl8_@pD^sdcB=}vgpf~}u4M%{k5B29A0aYeb_16lI~Ks@$aG(+FF zD?a4+QfW#6-n1JIXW}BC0EH4cbm`c9LbUl;ohT(Hd7@4c;?B02iuKQE@>04{*^Z1J zuIkOri2k(7a%vB3U7lhjL6akCR5U;2_G>txY)ys-ss#L=RQ&-n>6p; zuJx2f;)}v1_UHMPbHBw}vHwQZ=zv$)Ti6oUM&z3*wu%jhKj*@J*dPuW zNZV4~cTsADfh0~)0)0NB%#u~pC{`31+Ra4GFGr1uo;}h(^)r-@H!U+=7Rk{48+4(`?#@)setDvNRZDrvUUr}e2P8+C6Sa&58 z_%`|Ex#P#vD&oTrAO4(tQpCHMz7DGcm;A}<7Ehy+VUv_As35Cw|Nj2k_3}6EF7Fe( zJMIxgZ5H|+75K=tTyxe1=6|v-_glHV(|OKs*XdVkz|t#zfck`*59TH{|A?4B2Ch1N zqSjB{zW5noT$(@1kz)SRmgYG+TorP0*b!nD@wgHx>P zKV8&z|H3&-5l4v3e;LoJ=<*Od6E=ECe*;gTC=w6FzQ2&i))-7Jc3&4ic@nxht7SxK z{vS{vKiXuoV+2x>tt_pOVUZJqT;2NGu$QGYiEQBMd{!CmG5_J~H-#L^1RZ5WXg`Sf ziUbBk8u`Wv%Nma9r+VxVa~tyl8-jC(q|n~Jefw6q7();iOM6+aMrp)+ zmD_@>doidH)0yg_d~6h#=CbAlD&MjjnP`D@H{{zJG-YM^*TB z0tfa3Xo>MXB?2o8OBj7F9Y{6xy_cRVk#b7=(~qSEzR5!z`|Ux5JDe5J{5lcuRQ@^0 zBZL{BuQ?_wbnH(9s2Uk0x7Ms_pm&vnZOxKhO(n+~}a_h%Bt^ zn^C!XLHFZd|IfHnZN+#zHWY{1%5^fo(!jY0o_*3`{lB8;`u-A=Dkclbou}wQ@a6 zBy@*Pv)%pyZWsCc*%&TUB4~RUPc~tk(2eYGKxOj=;!H!rL>F(EjzX0u$b`;rguD8m z^o*$9q&e;1^7{TUy0=U10>)ON%BIHh-Oyg(@3Xrp4Nk@!RZ_FPD;$0-SJGxC);m-T zlsA_r60A^>nhOIYEP1UB#~)l3{@*+cKLA^O#X=G{4Ri(p`X)xpO39&`K{&H>8b3rz z1b@7*`}g*;x&X?;&x)8+M5zsKCU-w8fHHJbNsFPo*zGH0bhVJAtP9&P+2UL!h{qiL zMMbEhsB+cwvV(9I;Q-wAKS@5_*N*J=Sm-K%MY~AcDiymnn*2Z=PF+)m&hIXWD@gUt z(LufisAeJ|KFYN1k{J%#Dh&jCN0T02ic9na<>%kQBiai4H2Z-R5|!N#So!;T>AC~z z?RGzKeJ{(JNbjk1R)?4JCWd|ePZtX}H^uBhm{dVbHlU_{t~spI5E<69g;i57;t!xiwu zfJv{KB#n|=pwyji)Y%KNq~yR|i>|~S7=C;x-~{+!luPjXJ3w8M>p<8>+yR(yq^oM| z4KA5I=PjbF5$^60fz6sW-Na7Mnt7?jU&?#p&30eIY4$^a7ni|mthc4sS zbz^T>kEMdP&JRgK zlx~LA?V44$3Ugmba%yDAKMh9cy`pcFaIX}1^cCD8*7oZXW8h7JB8(5MB&~?6vew%Y z;CbipS=Y>`at~y*P}}J%8FPS$DP^LPzz+65D68c_D0Z>RGdr#!W1wl(L6LkSv7Xlf z#U=EiO|}U8#vW?6W3gJlV5;?N=;2LCS@}=-24x|*EJsjE(ygw*SEMtsZj4D+O9X(~ z0t9KFGSKn>C=|K_y{=vFNDChkll_eGSaPc1@eExzEcd9S{@~r;}+Bq^#KNv_4_tNKl!N<%hHG}tjT_XYFKb$HyhC# zDPAJoeFV(`Kez4nHjl%q58ZAFC1v#G=-TE@pP7hVK@qj{x5SX){v#XN=WK79dT{B4 zk?m}sZqDr6{OPDJ5AJ*y(HII2!`*kN06j^L(2ctW^acCnT+HcVxlPER7M-`gI1<=$3*#TgdY=Xpyh19>5$yW777gxa>-D=v;Px)Y5i=#*e0H7et zi4&jv*Nw!m$|Nq0&i0!woiXOt85i&*yBBP#V96}nv7gyQQip_j8oAHcyE!~q?P z`Hu5NLy*S-5o;EfItAejdFXGXB{Ea9W=9Ds7xxD%34J~a7(`*&;GB8|)vSal2le4_ zN=iz(XM!GDFr(5B7K-r0l|g{DzYoR-*$z!iSZQTaAkx4N8lA_#cwKj)F4CQ@?Ivu$ zo7}iEd@XkCZ%jdxmn~mux~{z1?3t~m;qrA}M`sm3uQO(;`UW$T9F0_uM<BBCFV$u4XEkPYdy zbm{UFCu+1?t`-CIg0gd3IObxFuTGPYNLS3W;-p{}I1!`r z8+U^@z6)|ae!>LpvpClt(uHDm+^^RncpfIR3d#JohKy*0u>?g9)!yh z4=}n-@lFJezm&4#*RH*4`A}P7Q3SV+|Fn-jL4oBNPgV*u40H?ITP+3$d~gT3Vu_CY z`jjWzba>uLeYs!ct#|F0c5IEclHrhdO8Kh!w~};5w4~@HbTfD7VfA6VDECO=B`?FI zGW4_}2`b@na@ZPm_>}dqhKY!MeZ@eiIaM*AOX@WZp_Fe9Kt-YlvItBf`vO)n6e589xf17UO_7A8*2V*(<>5xv1)$8|~5TmT#^G$X8{&lXm@_h?- z+S_2+`N&xtR@}^>wTS)3;SLDpxw7JaSmYi?noOu#0K z>CQZ9Dty_STT3p>iuw&|uK30?BMR5WiD&mZRx9B%$U(MfXutEYpzoYCZaj;U0{KBy zG_bbNx!;C9a{JBN*nL9f1O6B-j0kHimkRmv;%We}pxy!Mq0$f%>7Kk`31}!E?Y_26 zgB^I=Zb5a1y9HZq10UWm0eWm}svnZXvS_N{S~QD3Zwu9CuP zO-lZ_FRf3+jR|csu116vkslMZIX1pXVAjVfqg!{N`pEg{t$t`P(z5L!l z7Rv$^ZsfK|6pJM))fL;c6wPpZD-_EFi|7PfxzxyFJoj%mB;gTEE>)kxmqm|dmV^i+;3}0IR<((QH!#_&+6kfSk<$v80xi(n5 z?#+v&^F#IOoE?kP0fF|yD~l&QkpQJ|mjTq%_uukrh`*K(PM_gkv;Tkvuc#(;i+6zg zO14uo(Kk0jY`?+uMBNpGrDKX&5R!yQrG7=9j^G|U0V^W;PvTY;tXy2|@cnZ|_L}rp z){}e8Mo3#2PDG20e#-40QBL#!IV?+*eh#NKJ;B^W#jV_2%oR}xyjB0ygidpMYpXP) zqnFtg+|JOZz3J8{@{^+Hr!?xP%7@{^8Yz=tmR+8Jj3T&3tV)|>=Wl?YTU78YJHJ)S zmM7`X|J#>Go=LPV8)ti5VjsBdakrh!ogeY0bJQ-yxDXY-dRR8Do{pOr=-}nFj@8RKhKhE!* z`u6#}$928dWgaeLCnU<{SZu$yl}cA$Z2C52yVsYm?dD#ObI~7o{C$os3r0_^qu=|b z3l_@%L^{s~^B&)C<-3sT)?MolA6aeYsC0weu{BHA~IzB2G!9 z(gw)`=lKdSr^*6fVyxFwzkA; zBaRP}oM~lNGMYT-Q2$Z0zkW}Vw_(tMkQOV9W<0i+Tp<}pAhVZFB-msjjBlZ@uMY|n z*>UOBRKB!ygxbfv$F9TF)=+9GQ z%TM#2wu#Z#`&E#-mRoS~&UD4UkjRSjBw5P?<(OAie?<5+gUj02LJ6*_ z@94>Uf=33!k0N(ss|=)N5Gr;Q4fJvXn?kCG#!Zp(ZgtTL9EtC%A8qvYw=yIgJ*Cd_ z<(BcDL8ovqs7P1E)##N=laiMWYgT;Z@AxD^Sfa6i^|jEu&E!9s#26cO>1`J#h)~Gx z=&WJV3wVNY0Dq65*#GEL_0E5kT;_TQdSSF5nyd|vB3A|OrFLrg^3Qpj|2?qiKaO<9 z)AGU3FYvvT6f&cqRJclc;LUzoAAY>`8of%uJ1?snn#9u-W|sZA zD&WcJ8DA5xm$l#9DZ--WmU%82FLUdjIG{VCQk7xD?zY}F3Kl)%N*WL{w|KI6>d=d2(cwSUieJSR8&}jS z{PN1xI?_p@kxrI|{+K@u6v~O-e^8*!!aXPoPA5sTg3!Bmt&N@*I06n%8U2E4HZHPP z&z>3}*CH9JudSVSw6?eQHWWo{Hwnp%f9}%X*8tlGiOB*U=X*@FOvE*;+K%b+MZISm>$ zYP3;SO3Ril`#(qlqLzvC2G4Qt zh?KN?)F>&jz8=ZGah)%`fA47%`f;#Xyye!hHHHK<{t>3+xf0tp)J^K78ZmqzTzgZ) zHyM*}j+?j3NK-F+K=TG`{FV$>KbE;f{T`Y5UObD4n3JRY){!^s^2?od3;08Q?hr$t z4?J~e&~5!df+0lxGUZFP3eb*-myZ`im|55k!6JN|$ZxV;lJXDbQf1^AyxDj+#IxmP0L0ir^`apOjV zxhm?e^;!6ErK#`b13RmMW}^pBp4`s>i<>(>X={4?;=9iBfNZlfiK z`=DY0ma@EghJ4(WSDUfnq3h@5^kise*|Zv}+l31zeA@S-^CwG>Hfr4Xu8U8#i}*01 zF0K|q`rkP>>sITBUXjNUG&pvBadhw%H8nK~@^q6++$o$B=!Aj+u7VQY8f+QXp0v7H z9+{<#D0R!cQN4lkG5oW;a%YoNzLr`jqKE^Bc&n!qLOmun%nIr(Gbz!z*&3=t$Wee1 zg6=uyq^ELp#;N?b@jV$RQjx5Y1~JnvEL__=(W&x?C4h^tb8q{GHFz)o|h9qi0gESxDBAA#r4uL*%1*Scm?~X*9J2 z1y&_;np3{r-AA<}>_2>`Ytyk~#tfk&s#K|x+aG>q=ck>08R5`YIQ3p9NNQp}-}+6P zq90qL?`1N4)R)4{RzOxUEcwFov6V4OV4gY~f${KGC;Nt;Gs^T0K5q436TiR9 zrRlA2PE9Wj5!y{$~)XxK2t^P#ka+Cy+wfi8@jI59QB*>l}GqKqq`=t2z00u*5j!wG} zFp@ltvy%y=zWjREueK>`*RCz`|JeW+?B<6LM_ELJdOPJMShRWQrM(k>458fVYeI%@ za>TmoG_UmBxvn()7Wi5DMC&Ynym4${_UH*uNvsYQrUnKzWd1T6=Rdx_#0d5ZKM+z% zf(jnnHTw>vWF7E|oeno-VPCsW9nFNnd|xsM&+7;Jwd2@TO24DXXPI(wag!IDVE%T* z(FV=J2V)oNoabUfgjPK=!8mW?_jA<9SaafvHi1ce3WuKsg&G24udbnyDhIu!ItVlk z)@zb?K)fNW{xK&bO^ib zus(i~u%L>DZIl55HKm@?+c|kSyjI5Uqi@r8bJV+3Om5%z)5TgtCvLxaGT#Ie2~NqZ z8x=FZ+Y!()tEhb(yBM&|tEzKA+<`-gbZNfRii}NKoAtB`&Q0sps&z|(YEMmQ+kC;# zY_Zv~s7g14?726WCh;cCsncNr^$go{yt11l10psnGjo~HwFqUMGT&EJnQU~8oyMK4 z9ECIOLrRKh9GR2`&c0k-m;b!*5FbdAjCFfA0;Ab)Xxj)KVw<}{a!Jkhl&LDJrg5h4qnj(fSMx8i)(`x})du}fn*hCoR5PNN5zwQd3` zq?F`*3p43U@N(M3%E`8>h$Sq)+FS1=_qy>^hfbnZdi`4wY@A78Z{+n%hfZTstx}9x%@Fz}0r7h~MFJBXZIA zsn^U%gW4);(XML8 zm8CWV;`cAy-5rc{W1|BV0uu9)88fnEy@5=ZUGP^^5`FFk=ietwi5|f`3_X-y|Ne== z`7O%;MK|*+)2uyCaYAd?m8qTUH*6^Eb6yLae&vt1-rPvu9|jH_XlK68H)vw}+Wdlu z2h0YnSsLvx=eQMq6(s*^0>Pjxy(FF-A(3GlXj|=4xHECyV3b9{vw_H^S0&joF<8Zv zG-+bOpuvOFbIas6nrMv~PGr>7QhoX}I0C?AW39f*yTVJ?0oyGj`i5w@;zh@+*TI`P zcnj>3I~!em5cMyG}YAf6<>Gi1r8CZ&+qir%cUh}_V=!v zvuL4mdkl$>x5?{4;LPdai`!2FG86-C;LA9VQU?i*-K}0&3rz@IHtjbE0&y(Y7xoPN z^UtRpayD(+bhYjmmw1e&06bCMO%)CkrcU)1&p_Zyj3KsmsUwxI?Dt>pXRln*_9)E> zFb{*gt^oW~XXRMFy9%7(2Ur9L1!dkZnKf%xpbMG#u?M{LXVPT>3OAAg4VwVlf+hP_}QwituNK=WE+I;x?jw$q@8o=tl~A&Il!04K~ZZjv*N+ZhUL1>>OAHryQA zq=7RBq-b2_&fsVJzF=R?mRVb5ro>HL7zhIbTY!x{W4Xy(sxxe}7v@K3UEpGH_*7e_ zUZp;XW@*TAhHW-ukLw}kmZRTUd3Ee;vcZ{O{Njo-;gOmf_lRbU#AQ$}7kBhUQz^N2 zfebbM-qoYDL_e(VzDS&r)g^U3rhiH1U`j66@48~w=M=L1g+vC2O2G!x#Dy2NJ2%%^C0K^Tr-xauP3}h1jfylH0uq_rG98TZ zGOv`5uw>=(Fc>Kx%?@uHoy4Q9g?egPzfoUzP&&ye)!K)lFaYYpZ4FP3&M0fbWf5Bdb=a zQt&R_gVTLqbV*ezypWfiXHURSYo6Ehm`~>9CXqt0Kb$_M_iE9i0$q@*+IMIcAM#5igkwSTa|J@VNyn+@_5ucDfLJy-yn4Ycq|o=X0r=*<%f%1o)MR3e(H(|6Tw5F zJQ*bTY)MjK+p5*8A7!=&7yTe7r^~+GTW5D>nj@f+%&H^gJH?P4h~=m!S?OO_+1|d} zPojnRi5vizx^ZW6Aq3LRHkhj~QU$`PCBSh>N5}v%XaW;kWH>WSMGP?CBf$P75vqQJ z1|JfN4Xa8$#$L`Uc=C_~NYc|_(pK-p+JyGq^G*LBTq8G7;;-IW*|09ir1HElwMEPA z1tG-F!1)V^{6}yS=|>L_^~>L2N~pb?^I46LlXVvlGt2$+<|O;bXNEhYr5IONccq86sh;iZn{=Jpe4U-*+VqL-YE+UMnMa_I`99R$R-Hl$(euf@ zr`V!Xx%X;7?O_ZIfowG1MpT-3PpD}6|Me|~ry_3biRZL5G}gBnx%WLhxY+raZ58XA zU=z;d_2lFPc|}^Fiem!$Z>0=(cKa2G|4bYi8A(YUbY64aOI9T(0XHa`7{>!``}Ao{ zI8g$T53O~6P#7wnNm_xw$!3}`=eNT`c?IHD?%keDW4>HA($l4gIN~KY#$)jpa#)^{ zL@eM=yMs3g+?RPzpp3g|DbhGYWigvyc=D1AaT!{`Qr0VK%b;^+5cN7ZD{KmXzdLa9 za__p8Y!;F+FLjjixypQ=Nyy&0LqL^{PEBFHHGm@&=YT3rFtNA~uQ>kprzC z4tw}@MIgM|kfVx;T8gPtzwRl#b?cUSo+zfdO5S$4<14wS62X?ALP3iZA!6Pg9duiUc@8CgmP|$m5QV6`uT0EB>@9J1!BG@7ne+bUNYgR zY#%wdLHNeb2&FTon_dUBThymtc2#m)QRi`5-cNBNHVXn{)0#CEnlJb?9Ud4&u9j|p zw=cb(OH>yyBMn&{`wWcFG9gZRHIQ^8@lQzL4ezejk#V>!J9QeZQAnYWDP|aO6<6KV4R1Fko$7-1yz6?k|^k|9qUDHvcY77kB z-y+~0Y4S4nirz~rhDR!c1VS}0B7OTHb3Qdq@>F8Fy!$1o(G-dlSaQQq&|VU+s&EN- zRo$mL?dLk?YVevZspT9@EIr~Y)^aBG&r1O`_oWg=dfaQpX|{=(u8C7-1-3c6KsQ$A zaLD*vW8=sh_OMaqFwMdGg~?O0@IHIbYK?AqF)k%_fcU@ zxC8?Wy74SQjND4|a1TAb)BS#Lns-EQINL;b&TD)P{WKF&%urigjnQpht40l{Q~67f z*K8mg%A#D_yvVDpnazCOZ~66s8tbEN7uEF@z zg1ntwMR@hkSKtBy3G2HTBv-eHmUE%^QvdZ$q8h6r0*^j!J4p;#*`6RvvFq`3Y$ZJ2a` zT|wBpc$;v!LHr4QVyvK?ePg3efIf2S4WuZo*Q1qnglm3hDXBrSFDc~}_>@IT{ZQMV z*T@MX!JWN&*{2Qz*NeQ7e~5dFShA?;bwLWr@)8UA5+85EG?ji=I8>JLK&0pz$JkH5Pu`cpJcH z3u5O8Da=_iI}RvH1W8=`J|D0>+yJ+$SOF=D{OfYp{DmG^0z@gUd}>*;Q|~U2!B4mW z(bKs!tyiZ`49Ncz=4e#2u8`H5(}+APmEevLKE}xYJpOsdgO3ZJ1XfkmS39Cf!1vL!{m;JcHp=@vzWp9?^e^nm#LEy< z*EHLn+SuaWlZvhWC<~ijqV2&#b(6o6mx*uHvCoU6KO-OAvz`bI1Sf&o9={orTh*I1 z`I>W}i(We!rNiS2DaAnj1hl3myYb&&E*C}0%eoFJ|yvcc(D!+O@tETT(pj z9XaRy3Zt-j4;m?9*Ri4ecn&yf(44d z7=yl^b5!K3e+;E$%Y1*MF_Dz@`hXS(R&%5zA^%pi6}qtqRb>h=pP;Z$k2rLC^i`4= z#^H|jCf>Qn0jzik?k!UE=E%Od4b{L}0Y1;}(_dmw5r>U9{xtUYzOefjnIcMLRD03R zbw`^$CB8ybDWYBO)k`IF->cJkhlQAAv+uPH3MsYNYG|wB?d4DG3|oVJp`eM0`(_%^ zWWgMSzP2cuvEN*#=cq%OO&xLS*)jcNqTuO+TwxFcNi43p6UQL?Ai)Oxl57PntypbG z8+%z9^q^OYAZvIO=l-{*LIS@Q^Sp%zwW~4=wu!SfCNS_WP%mLXAUr2GRIgvZ0;&AZ zr!-V}lrtY@a%xBBJ)bUw{5*5c$46Ts*C0AOoUT)44H7c(r9quHm^Z{Mw+mtd9^OqR zBQk#}z-HOk149-0N;4llyfO!i)%W$!ECSAOH9 z>brZc-?(wa_CxtJq%zTsf5C1JKZlj93xu}X--4widf}r_41752#U$~CX&rG6*$dE; zOEXnmjGaJ>o&0InulBrrc}x_1n?Eph#pS=Y!{trk}6ov=W3S$a$xzuMeq)(iuiNY-n`M^!-6c zvqu9*1kcRfocHeFD3Bl#iD_z9C^PP9gJ0XK-S*d#fKX#32?j}}-;qa zC5j+r_J_lVTcAidOqb^FP~!=ej$GJ^Yt-E!YTJSa5XmTmIK)+eSTi^-}T1=VJ_@5SBM<=~^ zsF*qLu3}{%_xQ0~eFZJ}{D=G3av_W;Z^0GMWE(NHz-xB-)pdM0GH`$~xxJ8Xzq=!0 zzvN+@bKe-5CY?##rT6b=mb{7nrK6#?Vf|D9=1$C(C%AHPFVk?mf|sVunNp4Jyn>Rz zV-uvdvoo>(`+)%G$c5GMYD&;RXagl{f0|Bn+tNGBA~lm z-(!F1Xb;0q-R<$ublxaETe4eBcI@9VQowl@#%4a7m=8ZB@4{hLmw_vq`~iLx(hEMXqk;xAmke4x6GRf z{q@b5Ina=XAJQ%@d{T*sg`XoIDdstR$m!4~vn+v!{B&1Er%+doL6r*n34x(**wOEC z>wCJf|0UXO`Qbu0^*CLi!NNl_C+7ps?mya)MYzHpW|#cOaS` z52cEqL9h7_8w0yuu*w;qG7%rEY6zBZ3iJb=runE*TYP)ue7hH@`#zxWHSj;X4JPb) znP-4zsRF*v<<~g2@ESFu<4vu!fOmCTi>D3~*0isE723}wzlZq(bssfx3I(17f>B|% z=I-w9hw6rKEUvQ(T@_B1+Us?6L%ARNgQJmkBZ)Z|{E}LUtD52iYkBLts=?Sni(WCe zoJi;7G?`wFU~DKj3#)xcw0H!SSh^GMQ{qpmRH-s-yQZNz5kSi5FCSiE=o7m&L(SqR zl9tNgA8rW?7ihI_w#JqYzujfj86e(@^?8g zXz}8X66_0kl{spI{%v41f%zTNcE@JX&&|TMwbiD}8p__qiIGjX@$KfwX#`3A>qGH* zt|^X#w^pqxU+156Bk#xktcgChPvza9Yl!9uU^t;|UvV#shC7B=e2%fiQZBTvo^36e zYyyym2|G{wYURh?CWE3>PYyI|us^8Yz|e(Bn!8b3NNvfw3v}WfMvExpe_bAx(UHm{ z3UObytgAr7b{7&yfmgr-i=P9E$u0#{Mksjzvo>W-%Y!~yLgrawD2oqL=$f9O4<=$2 z2Crm7z2V3>Zd0#5{&U{Q(9e?WGdh4MHRzG7%;dqoDRd<=ghk5hl!w}ru}eTORq(vQ zpEW2oFil418Y2XT!52dSqo$ZcGDIQ?MsvI9l|p17uf1>C0GZT-;4tK=SCQKmpW36CklB8$?3->&eEoDXyk$+-46%x7^E&M>Gm z&^({19I7?`sdLNTbZ%5q6=ih!oOu53Ht7DwElGOJm!VjQ90Kkt`NGl15GfZf#B z)%`1f?7dKFR>|KDaToRnuIPUnhbwLCwXIXT_U#P!{C?N8%Brx+5k3B!25~v7uScUy z>jd6PWy??Cy)C?0R#!qExc|(1VBl#xBVjWg8#M; zKUVFkL&2w}3Wr{8x>b4SR&Zh$JOkc0vrQ{1K>b#cU2*Q*Iit8=EL1l{41wYgrm?g5 z-1k&gFtJFmZ*LoIHikp+gec-_FY42y8~i--$)hCiX?RH5C57NgcgtC`S_&6V1B_sR zPG_d5?7h4)Ctu;i#IL8>lK>-RxmlMN4n6v0ri4?#-<&rXgN7A>GBZ?`8E%frewxFS zceD*GTz(!bL#x)Uk5%l>`4_lx1uUx@!fxH(fha1J)c5S!t;?zJpha&;N~X;-3!{oY zcyOfpjt6X838O;osxT%mURb?_3xfo6vj6e6FWpf@5jVBaYZ2aU)fGM9`ou%u>mY%n zUoO6yOOIHw0#c|1^dJ*@Kg?@O@Az-TmT2d+3-=zV{5@|f&H(eWDm1QJyEcKXj)vS5 zv_Z;yVr*_U>IYAH(aS~U3WEO;eSCE<$r?(F6_+5|ALY#79yZ3hq25lBn)GDH^#?(n z|Gp>=xQ!OAl9)$)X0sK=Y^dJ2v4LW;JnlcZ=TIN$va;(dQAU{V)~#C)+tpZIVIm_j zNlx+e^Sewb+aq+6frcaxOFw}vYJQk$M?-fJF9UvIz5Nw$>vUq!GtqYX%=BuctGf-{r0Dy#w!~Vpwcr?*>7+k7TAf95$8nu+U&k~J z*go?xYE~)vYSgQ@G_$zhp)rCb3lb2(Btvjwos!bjBq0+4iLVr)^m1anCz21k*6jfS z8EN#h+6X~hPxZBFHpb2UxW?p@Wud|PdOLkabiU>Ku9bSw_K}Cwl$S+i*%a=oP0{5$ zR9@1%Q)?!qnRyKx4_$F}2~1w0o%$TK95QS{v9@9bSlT)C6g6trJeBiS6e=7!pVNkd zXA;WqY57<0=LfD*na;u6X3(KSPM<`Pv>ZD|;zV(;uvpXs6}SmEBESgv3+W<)oL3If zo%tS{=&mn?1`mZRluUJk{D&8nG5y7?^lZ1t`8zQ%Q;H8l4ke?AXag_{8a?9ds*23a zhIi~9(Rd?UiaKl~Uu zphA!&9!8r7I@9W!*2K z5)B!A1^AW4d?zXI)C~ri*CgF}SzgMA519m|kCOZ7-(UVLzty64obgKz6b}6c^WzIl ztja4V_#bm~)rPov>()W8ONIrB_325{9V=*a`eshswCuv;Dh1#CRjel?WY!w!BJcuqK>ZfK zD3LN`Chb<>OZ2->5Y`!e9x-|oSBj_(2?cR-N!4uHbPKjw@!ag}^#0dxw(ktVYkgki9g>~(4ZherL>R_1vPF-!FYv3VY9?ThVMVGK#g3i*6zsS)pJoC0eTF+X7SD2Gxhu zSAVh@ya(NKA;M=EA+r%z+pEPZZO7h;ax@q}xE5CdGvWC&?d-~5 zuO^f!wuJvxI$Y@|y3*|NBumS0VMEI1aok3T?n63&`tPsib&k({2cQFtxgu}y;K5li zPvif0cW?j}d_5Up`HQrZnTTYbwuv{RmgRR49%2S_D|cAf=vAs*S;^>yJV`L?^RI)J z#AZ&RF8^E@!UphW1240Ah%EI{`n{W=g$R4KSY$bZ#|8wHA&shBSy+0cQLXg#@1Xuj z*oyjN#n%QRf@Oq;?+dyU0)#73ZdopfHxmw#SE3sofX7Nife82Zw&>A~7$N*yN6wsi zxa162I1X&fqeXv2l2rh;QS;;kw_#JEp(Mp(pvm8BC+ly2M|Nd~8u&wTg2`;>2Pr^cF<1uE{J zJ3P2b;5s*pyAz3z0Jgc@>|5}&ubB`RFHY6xH4B!`8K`N?jlQ?@jQf=b+>K#@Qam~p zWFCpl;u*^z2vW#%m(MRNX^Q#u=)#tLDhNO-_uG-aXp7~jU;wM6q@@_b%ZhM>ES>wI zbtsDF1Hh2JkV3n{3S6#ZL7K493Ik>ps2$%~)waw1*NR|9Y zW?N8DFeLm`ZO+L#96(a0h~qkW+2yUPm^guio#nfuML(%9LBL{xa?uwyT+&&Svtd=D zb8Op#m+LE16r`e&l795qF*6OsEdnQU_q0;XkJoq`?a^Fh6fryjAI2>z1xM{5)T=Rp zYFXZ#*|RefT{DL7+>CyfYnWDqjH|sTe|BoLxpl;~Gsx8;rW6x`6YRobQ%Zh~GeX@( zbN?Us(v*Ea|HK7?jnsV8ibdhmP@aV~@=t3`?i0AxiagG3(SPg;5EcDDU#a_)^PuWTs`K1@Fs3^H;%xOy6+~q=B8GEn(UW=YH}NH%s=n_ zfqs=LX$V7ZTHg!4=8i;~tm~NXWgQTYNX4sU91HgeVLvVW{`cC)xvJ)1ZlMLCdLV!9 zf`O=Y@eA!rs(dg^;vm35L)@XBWFfoG`vE&5k>7u7*rIbS)0gf^9vJ?vgEKp(P0f}g z9%j^US>`WLJ>YFGdEr?C7)o(gjgn?BGRf%z)_e$y!$WJ zooKTV{hW#f&^nPvEN99Qp+OP5zOkVG|cIpVh{AEpNW_$u;Q8{**J9hL` z1mC5&r8a|8W|Y+xrMRk`{CltXzxuwiB&n>}!#v}BG^P8(SAQN~KXpe78IjCZ(MT}o zoOeC0uV-K|Y@*~(*4NSL@ao9J3i5DBQL@sIc|DlG>T77UF28j|9Yvv~8N=4Ka{j0p zq6x_vi`N2~ujETko%*UrXq`HBl(EIAX`n8O^alnS1)-Cr&Y*6~d)Px$1R{(?YQA7- z9dXdW_ip+EDa={W4LI<^!Y4Nu6v_%>AF*-m=1`Q&+&Pj&Kc08fzDX^8PGwzF7czYeVy6EiSCvCDn#?4X+ z4NrIQd%tzW5aq(zX}R^5_tr}f6f>6(qmL;J!;TT^SYzn+o2w2k-nVbx9Q<2KHJ(b@ zU-nd|1oB)-##bHyzO{)_5l(HaZKg3VNoY^$+fpu*|ICVtt5s@mZ!hz}pXjALempKR z=dXUOm@bAE{~`JpWit2&qW=(wzgPwFeQ2Ybnz}b-IE-g()!wW&ZtSMe5p_`M4cs_7 z;MrHs%XO+*sB|$%FknG|8dWSgS`u|#-5(zpLxLO#kn9}v)^@epRy<6i)&9N?`dVXE znHR%vgQB20;(QO80!1Qhpl*xNW_BBeiv6XBXxA6@T zVR-|tZ2lRG!oVqIYw!Bandh3uCiJ;Q6G9n6T1&9spdf?h^M};AF*{83z`A8oVuZ!| ziNeURApZ=vg^a&Q$$!cwm_<=gqfVVAkDIz3r>_Ikr5WB}fX}jECt%adZ@PuJcL( zYc~J2IE|@In9Z$SbT^UcV=;2(#he-|^a6*j+tWSrO(!+Az8>`?&Qvc65n{kPRgn7D zo$uPsd~isu@#)3mPYyEMx~6~Y3YeD$e!bQEI7gnu`~da&d_Z)3gUYt(Y4ue+uGzFt zfiEFUwj!a8?`L6i0Grk}PVb1-b^Gtc`^7a}uwvlPUDaPz7{GHE)$8+9^}AC|2^h(- zVJuk)xr~P4HX5*O*|J0S@}kO6Ap8&WeyvevHv(8Gl2EEC#UvX~mg<;{b#m&1ST8u{ z&K>*Y=6Dop5w9nNExYgYayOyi5sgCZ`SwutFM+Yrjncd}jA2&fS{kVlyXvbK8lqBx zl99<%c5{vn*?JQ@UV)Wb)CHQRlDtq4%@`pX3BQnkTYdO!jt|PgkW&6N88KM1N46=8 z`fuOg&w`D9rLqzJHa^8pA2+Uks(nQKqGJ3Qf(Wb}V1ZQJ;*$k=_qQUTFJO-u&y8r^ zurq$TvXX!@&)7@jlMkpqxV$s@${s4BK(x{`enC-{q*+9xyKjh=2e z_fn{;TOpn$KO;#iCYCdvrRn@pS1zPDdjcwDd{L+VGT7?tv!Uz24oV~dl?S&8*mp|D zy$h_ERI?eGI*CCKPP+bR7*!j{JvTRZf&Z!PyLYdrYZ`F$NdbyOeQLMT5@)_)^Iivw z{iaDyINwxVTl;m3{4QYZpjXEILuD!|A@1?1&ztYz2ncXUMAN~qnH;<%vzVK|nK0Hd)EAl0r!O6Ne7s@7uX|-eQkeeA-4&cfnj00^ zvkndcDzjaPJElqZpXUP-7KNnL3toOd_uGr3s08lKJga?cdiFP&N1~VA$uF}Y#7KEZ z$A1CzTfMQpW&1@<59T)5;qj$OGcWI&E&H3e9S6A=X}_amxIrR@J<<3cR|`dmRX zr6or7BNp;wxZA0(r_ij?k`n~>b-6K42pSJFk;-I;Q*+4JZI0iu8WQwG@tEEdYuEqi z{_k#YNvqWY3uM{~8ONC21w;V=MP_rxX5sk6@#Gg%=6(zsMd)$}@wDuVXQ!PNv-F@r zm4KTm;c79Lx7ywPP!!7_#KpyOw!LdKyi%E)e$P2~Y6H0sSqJeYUw3!4?ftMujT+ZU zEJ~WGqo%G%`NT(MUNhMLU&_{Uf7e;-#-^tREt1ikN_Q}t!)nZHz9MYhj?m*LPs+H6 zkp3QxoUqkOG!3o%tHq6I*^?IcI(qTq@-zELdn_-k?h()UDScuENHsz05fvIPExGz& zZui@_Z!;UOB~UXA2DX%zEw9ZOIrwP-^I0W~GOdf{<@zkHJ^!UN)LSJ3*MQGc)7lb| z0`~Z{jhPiSC{{Xj7%yk4^i&vJtgq;QpuXwjk{e|39Pcl|w7z+oayyHGbrXE*uA?7f)t#7R^)J9CYM#PR?2*paYM#q;5r<6 zXx^aGlW@mCRI#)@@9q#va$?LOAAXc;p1(wh%Ukik$Iw`zrfBy^|7^0KbQvnL9VABk zNW9kmZj~XV_Zjpkv&WiB#WzZ$7IH69;x=gT^gS%qs?ayD`cfX<;jL$$Ib6fI$|0{B+bb4Ky)+- zV5gU;+5MQ-B*M73Y(&EjjZWE#!nTWvFUiyY7U%{WX*z?R8ESIwa&D1B)0M>NH%{!^ zZu9PI&^_1uv;6uv+0MY=I$5~hvHPxyw1xIw+=1B(_)qz4BxqiKQy#;RyYKYqR#yOi zV0-m_R^!q>GnQGvLu9-?clFgG$PjU_M*g{@c~sL)w<@aKg3pa(XxuI8+S@F|o2BYW zq+v4u)s{PC-1(s5MpY`(&)8L8v1j*gr+MxuQjGR9!Vwp#p1%G@ok_jBNgxFV3Cmv? zFqh}dm80(Ouk3ypr%muzW~7w|kFuR?hAA(Q|4>~WnH-EE(0AJ6#os2_>Y)~tkxN4^ ztZYS*k9Q`eNM(u9>SA@91^w=~ddV%IzPh5-?tlPa`bNuZZiNxW`_dqz+P0Ol%5v#Q z_gIF+BW{<#ZsbJ1nkQOShI$RvNm{I+0%BFkQXlfu+Uo)ZVtg z^t@jeWFsMSF01u;vvGL2A8;^!z|)L-bg=4A~1p0fW4XQTNuDI)kx@WD| zCOo*$NlY-hC*`3c60VQB{Zp-_ukFI7$mmSX$?4HPbs>_vGCCmi)5y`gRC&4LN%!6G zjP*ikG6=F?r=-PNgmts=6a`M91VRc@EqvOH&Ee^Unr6=i8h}@9uCxiAj+ivuX1dk)BaP#P8BM%Rw$Zky4Jt)>vg)TF{Dv?sKE1 zFG8BZLQKebbT_C_wC*y@DdWjjpiU6w-JRQsF#@!)gm~{pG4e|t8q-!WB)2;s4J?8` z45CPni}NwL*7=6@y}8{bL$Iedf1U&cG{T=?(>=o*Nym#D$Xf3uL}o2*yYGQIR(Yd8 z(x1$laVFd*5><|^D3yv((G9Y<|MS1qCK1zdiq@{?8EguDT)RnkvzuFnsr@dlZt57S zvnL_5L#BBRrB0uPS}ExC=>{{dqTKqDSLBi;Sk{-RSm}MZr;_irdPXSTR$;1kYjUs7 z3!Abm>w`HulOvZe7x=xXhV2R2%NJ^PC>S{PC!3`oVwsa5{t7hfG55Hr?r4Df|Ftdh zE6@|yS9QA5w|D;O&jZIeOg}qWq~E9`SLq11i>~&3Q7Nio=C6;D@qbGe7i1W?ka;yy1bOm+g22i!SOW z3+q!8Kgg8mWy{W4GU?k8^e)X!po!U4fCLm>avt<^DJa_7n`p@RYao@@S5YclD=T$HVty%1@)B6?mWx%vBB`I- zcj%DCe4S>*8*8RZ7!+MK2B*L~5-BO~m0QO^w|R440-WoR?<0WLE2iaepU?P()4k+_ zR`h#UrUpH_dosY?hkC=pgE5t0p?l>B=cOt;kg4(hnmKn`j;q581^NFSy%4)@-8$_@ zwz{+*_)As}c#&d#HBY^1(~d^U0l-WuQrocf#rww(-dtGJ&@M6ZYubwiw&?25Jy|Z}6`ElM0nvHIUvw#2q;Sbq9Tj$rQXjqTDW*M8#FyNASTN#YX zt3#Ar)U`*pK2i#?`dwZ5NJ*-5g&;G|9J%`c6{V1O|7tfVG{m;CB+}r40ug5R}# zbF3gBB%tlgnNb?eApq#*pS{nI{(XG|iANTGI`}Bt;bPwhckf;YJ)7|h3?Q@Dx5c6*Lq zzSGjNy;*#f+f~|~J2$6KCCzJ{B(>!RF}P;chgTc`i;GdUXi$t5WOl)uq*DE9 z*%gxGbvlJr8?r+zBi6w`c{N&Os>cvBv1~W(+W-soKzkDI3O*fNiY~U5PkF>u=IhmB zNM?E+M5A+4blhT_kr6(?S)uRb42x_~s8#&CoPOZSU*+K+tFcCoMckA&|CbMNQQQ|NXbg5i&_V3T`zw3tN@6df%q@ z&w8NEv1kT_Yeo(OJm{Ipwko7+>(&ipP7bCG<=fNWBdCqs4nwA{o{73(77E&BeQq!4 zP+Pla4en_7Nt@xEU(y)2!qPeijV{dk zD@<8dyXgA0sIe}7xKgP5WV3@>rHk**UO%K(mb87Zm$vW9Srzvu^r~&&T+3X*8)0>) zYseHnnx^CpYU=)Pd-fp-Mq->{E~5^0HeCLHi3lMWdg1nBi-{B8ODel~O3;x=X8VQ3 zub$fGBUA)PXvm`#?NZhO8VGqa=r-4mi0OTv^5)oN%y=){^HC z>$k(S+rdZwVFRSuw|4R8*GD$gLUmJ+nY#36Ei-Az(QAnY6vxlEUX8T1eIyr%E55

WSOXo`jl`&HAS>63C zmFhn7FqGnxYrFmYyy*+C6p6qI-4`J{2P!dN2u$9l(dLhStHDDbX3_TZO^{Cr@N|)2KfvAe12*?li@~eYO8vve>{id;4%= zD9Q?a%iilY&wVO#tat{-zOd0_$3d*qr$#lIq|EO5p_@bB(Y`(io(xqiSt!5W1@N=N_Zw-rc!n zlv8$1@J+r7!T<6ja$Tb$7o4hr-ljmypBvDk!nPPP4gIl$FclcQW#Y3jCiIpRb zW|wz#fjA_)-W!8{CzNoRR!KrRa)%ixGwS-$PGzj^YPgm#z^*pRg}x{-SqAH z=mupnCS|eNawIWQJ`q1l^oC8x-#VcpXkH;V3Li<|2zJ8dz*vH&|9dQ3=7vk%bduEvPwF0mK#8YTd8t+@TR9JW7@DP zrf&}!THSF`Pi?gAarU%g@|z@3<4b^4ms#ceTUiM6*X-Z1KmW>c!?C`k6IgDrB&uMy zu=1PUdEaImJ=G;*8_eZ=w-rDvVu7m(^M8unnFloRnweSu@gOqNbdwAk;kxTiY%Ic{ zCbBxmpUPEiG#mSQg%KkFwh}kjKX23q;A6?h;DGq#r?z!FttJ7iu;{~mQ&|5oD3Osl zSaPuaj#Kzvgo;N-IdPoS+3PSzENwD*tL~=38cT6=@*5_73&`TQkl_`9GJ^#XnfI43 zU#dl|Vj*5DO6%|>V+?Jqzz3i0d;Eza679v+A|m?8+{wECHoz_0dtuog{5gktW{&$hYVYsSQeDyP z(GsnVy=I;MZ2e`P*u}V~H??g(K@>H#nQlK~m%hGGlVDesHfXbTNhLy9-kciW1NmMS z4aR$p9lMz26)+Qc0X+%rY-N3WC*=}0W^ZY_Z!tead+v=bh_@t(C@ucfNxMK@RKC=v z&}3^jujCWD9aDuF#t_hGvMqgu(I_N+vn+~gePtQocgg6R(=1P|cI>mYHD5&{gAyB!;l z4QihD${Aew>hA;MeCb3y>B1#+q#AfKj?!+l8y$Lai76)*F}SMG=h@fW%4F-%L+DS8 z9Mf9QnrzmEphL2Hu8Lf7K4>m&(#S(U9aNh`WF>K1PE9<$&pmDmA&k3sUuQ%$azVC5m=U%^wRK%n`ZrNMZ5YFfB}YRf%$(@#3FUg=@ni6w3VXC9hL|8kwN2LQS&p zXHpKynsVoOiSAfJ&*N>lKZ^11zKn1b}#9#~|0znBUL(W<$lTCkFGJjykj+RLRYHzlSJCuz8K`e_B*hsGy1-`aU888e2f8 z-!%Nsl?UP{H8AkE~ak@zF6`0-K3MrN@yyOxKM zV1IEu&;deMl1Kn3rZgs8q0SF(E=dw2BINCzX%`oJ6B4gyWHjw>mg9~8+dh?@!dx0a zhSD>L;O&x2Q!*hIJv(tk22IQxvh{`>6#ydZ^qh zF?HHBRO>A}Ye9u4Mowy|j&y`gVH0lEca((xmmI`VLxYOurIpVrQm4+(fAqo1$IPq& zdKp?d(Xx|1^~adcJwhd=TD0u&T<-PS-Xp8!Z;ywca+(DWdm^dUGZu$UY?gktIk3Bw z1cGZz>{u40>TJNKw_1f&*%Ncn;n{ikijw=>%!Wle5i*R zO4akrFBEg$xI=tqBNvI~7}m+XmfBo+HX*zHCb}AX%v~3B$~j*>DG(f3)|3r zZjFugn%6x({_7ssy~IU|d$DRys#A`cC7H7$@ivPuipt78&5(*!9ADIMPYy@W>lpSH zC;qf)+BD-P+azHdQ1V0EOq;G$i$X8Y@9yV+<#F0n7L~xT;aF>KXpNC(m?~;HF>G9( zu1%ZsA?V#>0!);@rP*jN!WQZHC5XbJBbp8g=cL#^K5~nfwkFd=WnPkyrf*(ujz6i$ zaV-A1grZxa;rjW|ssni^xN0BW;~mO~ISbQn-PCy@dpoz(GX+rRRs07Ki;w)&1dq-e z236cTq?L_L`0F{6$%#Uqfx|;o?J(>l7g6NSkvK=vXNC;zWMp(e=Za}8Z}|Tk;yG-w zRE|FX*t8(!q4X-!7*g5#gD<8qs~g3lNjyw!7ty1!(YT zDY7rSm+UAXeSzg7f7bko)(}2&WCPDFzW6yH;W*JrG(fZrud2T?Y$gK-s7MJDGs~h2 z6b~7iy50KSOqsnG=8|V4@|8%@KN~sPc=~(JrQt*LaU>{cm_`8ND%045-t-*F5d9(U z2z_K~i%Zu36o+z_-f`&EB+!`15T`zEGo*Q^>hSSg9} zzPJ9$9|kJ;v45WDlPwJkKYg|yW|rFK*7Sv4r?efdSnxaCm>Bf#YePn2l0JM zu1mJdws>Qv?MjZ5#QVf%fqQY>2lDx>B9dfS7`LCIBLJdER7kY6_>t8SoZ!|C&+6}d z5_J_wYB|V(vo;<+e7IWNCR*TjB>M`iC!QTAm-Nnbmy8Pbp6xW~78mwU7+GNG=dNxg_j$J5%kQp)XEZD)?W*PfZP21)WI?q? zoEP6%LJ-6amF~ek=Ofbxe)X|WSz(kVRE)A=TT`*Jv_}#`rDq+ny118&G*LQek@|h! zSYYNz{FQP~D1+LJf2NN=pK42|Atoo?grSf~Boh%hr3P{rcnQ91aNpkf>n1{t5;I&(C!0E(3HmX=$fZV#V6 zJwiiC$~-z)ov9!ark!g{&qr}=#=`iK7*+hxmRk+e+eY@_?buj{)t!f@fhLJ%i6$j> zIh`E%ykr#;=eYwQ>jp(8|HY>r}^Cyl%9Z5iBagn0^9w%Z&59B$U@>rOe ze0*U*#R}#1#_PlRy_mz!Kl~D5i)My4QA|74qIwVS0x%|#hIy0PYq}S|NqX+O?q+ zsgso0xw`e`hC904s}T9TWz(81EAD(-MxqZ2qBYaYL9a`(zF-u4D08QndA_6+3gd;?8u%*0WZ@os{?t|gbuA^!-#ouNedu8<2_LAc#X*lo~Bll~n8}{N9 zOi>{~&G&9bYrue;bgkmVz(4+$?8_AL0=#p9j}6Bgz}^uPD@C^>5I69LWsMFuaZ%cKTT>o*llPRqX={WVZw5lH(Qysx9^WWwd_8WsQmkCPI zUbCZ1^FpVd{I4~Cg#98@2@QTi0+X_38A!DQY%eu0wi&Z5UbApC#0o1Dqid?-Lil@> z{<8`B6J4ePdQ$uZHfq-FT}N9zDS&23P|caCDL;*5^rdnNZS+&{D6vti1&mphmU zCsH|>V500iSF~;&MwUt3A7}{?Pp+)@GrKjMO9+rvLXs`&Y^-^-fy;VSYFRS&*sX9cr~&& zlc(t@tDM$t{NA5^r;1R!oBnWR+sR1=gO5Kd>uuI1@I>351Gcp})atNe#Q85j(DKT} z9RsLRrGh#m@m)%+Wu%ijc)Ugo=2l?+q;ZSJ_h3NcvXX-3^hNhm?tEnFf+Y<`L7$d) z=Nx8sT7#{OkW+%h{X5-0j{uLT!7={Eh4F(jfszDl#;cZmFq!(K%VCZI`l(yM7bJk( zxua)jxF>Ypy-PY$S7l2d%2=Kb)*_C!9HOkzZ@ z1Y;`}Llb^^dVI5)$eFsVE-m~l2B^HYqxN$JR2#ORfqRU1=Mu}fv z0}Zr$Dr5$B?bw*J1G!X6h!VVH@Doo)OMU$zf&b;oD{TXPf|_i39wPD56w^}A^yBw} z%OyLCj(}m*TN^EZ95flysZp(eEp3T_HcKC*yhA#79y3wv2zfh`gJCT=&JxU>o%^!D zJ&}wmjQJ+?nrlWJ->KhCA|@#M;A=&6G300#v~#s;fs`KNp3KM6PC{JOGrEB^_A`3}Sv z%Oq?wF>1 zuqC2MB&WofTV*(#zKN_gsoBAd{F-_;MV~Iz8vLef|0HFVQ`X{N+2r=%k+281$7=f5 znhXeYWc7Zv^RG9KnA~{PZ~4?N#w?k*UX1lLf%_HFK3?p%n&~7gTCL7Ys1~>9KwGR; z5Q}h$*HfJEaiGUYbu0ph!C-5uU;4b%6V>%g+gT1XU?2CX?HD= zDrJjK@493&^nb*3jG!=>POxj)ZeKnZx1vB09u$*0{)^Ukw+H>eF@g03|MH3sV$+reNPRx(Booj#Ja##4t6RvxT;ButbyiRiq99wO6^6 zpYjhG*%Kxf<$7tM$;Vc(X-sbNnY%!1_;>DM(uTBR=()03B8Fdj zyZ~0h%69YnsPVA=!KqDOSt6G zuiI5031$a=8#m|11JIFWG>{5BnZ->+Ro>IW<{GPzs!ybheBMh^+M#5fgB*37frjIi zSeA^W0Vq21_ULcT2Mic+=+hZrQVapP8-uiZq#1J1OJE?;QXg-LzO>{Zl_)JPaM1hZ zk;0*&--06r#ndkq_>iUU>5lBZj!X{ zEOh%ku4baHO6#zaRfJh)>e_MW(F!te)jK|t4^M6so1Lz4CD96cX|6to8$0!fHl!iv z<)Xi=^g9%)m5py~$yxt&LdH9Q2hRBAob&031QJtH|NARMU4F zdbM(vPPypIBGkD@_wLI+WRDKo7&SiBnOE^7z1giN-CyQ^zNLKaCR;}>tbJ8WwFG+X znOuExqQT&^o?Tve4|7?o)@1KW?*R+f^fd3!jl-oPCU|pM&$9bamT1N>=uPteeCh_G ziPeHk`muS%sU~uYAhBBh{k5TQih#ePM{lqS4SiNpp73n&91|5V{_!z6&YsNKB}GxW zuo11cYS*qNHik7^u*+BSc1j6)Pi80J59}#o(#x$IgP8QEx;!g%r`2`0PZ=jz$2m6h82|Il?Na5d*| z-*1e4OW8$|#2|aNscff^H6oI-Gg(s&?FLbj>`N%h5~&yyN?C_AmoWV9J!(H?D0sc<_QcM94yyuoJ$dmY^@J3Yl)HFw-K9$8&%k-{0Wcg zHIXQ0h`fKBze>d%UXsr1{{i#*xmDy4cNsP;2Fz3C{;*+yi*74}=O#7ZL#y!Jk=gUX z3N3w=kW`^7&K;^HlDwTcbi}$|Yfis@qrOC6*S^QcTXhP|)A}WDGpsjiBnLECv#A3H zDJ&Au6^m9$wInAG5ORq|;8-+CJP>%9CQKPkPb|VvCdPKua5_DDygT_e`tKF9qV4yk zZ9FW=%`{KWe_rNBxj~`tAeQACkFsEAh}~7wcKwdcZqq~)9P?+CS2RL7XGU~)E6(-C zI>RBek>MOTzyNP5&9xBk4nK4boDiPEy!fj1taeQX`}%Zf$)Gql`3JF0BSm~=9O$0x zR_loUwNh){&)xYkH2}vH1+TZ6nwh=EylUlNV^EDQi<3dUo%&LhSLw<5E zn50H(2b(p;p`oFLl<*E+CsIXBR>cqE^c2x+Fx2B!lg$+-|2TQfS2~m1czIesSa9K3 z{}tWvQLpt`9&s^hY7g6GpJ+NWSw)eaDu3GasWrV+Pv+aNeLs*9KbARO_=uIy2YvbU zVovPlutbhhRZxhG%4YwX?VYiDbyHq7Tn|MAQZi!iUph9ZTUUoG872kaD&Yi-C);~R zs62lXJ}Dqan1)n(__>>dc3=Iu znNTViwLz)i@j*M|@olKEDm}=xAiAFFH0rW=$a4D`YRnHhX=}N3f36@ zCe8N!yys_#uM&g#F@G@a2`V!Ci|Ja)U<#D?ZH%)TRF@4KPVHa+Y83D_c)NJGTe#OY zy_;SfI1!Dc0ohq>xg^UnSe~8lvhG{RS*?3luHjM+)tQktjb@M2t7GB^ePY3HI6$W8Puvu+Q9^ zqBw*bEK<}N-Gkj>(R|w0OqFq7F#B3^92_vw$Fi2S{WLaVeF<#nl=t-G$>yiUWDMUO zSOPZv`AL$+NPBCW7#~UdLEqQ7&5(0_ul*9t< zBP@DJ?_yll?ZJZ~e7FERxDVPqWE>qE`@B~XK=?>b5x!Mwi2(z4yXi1axd`1#q*)Nd zx4mY0NW%&rTkHpXPAtI%wK#VMD^Is?-+X*CEqi|X@OMiHnxLvh&?;&wjXa!Ajh-~g zKVdn4ON@Pk&b;c%=X{5Wl49u^HIjpGfVN-;9lX@j9cizG^Pw!fe1Se2)mlsB3APVo zVu&?va}A#CBge67{P5e7QPH78Pm3jw1y_|orau=QQu&T#%qxjtXNv=fC^?3gET&g=Y%Q3qZEv8wgU%2qp4y?u|dR7Kb?(EFP{=7Ch3s>4d8aok1hJ%$7_HkvP#Gn98t2 zj1>*6e_M0so3-Nzz#3uaF&Rzz+|5-OjM((xL*)<$pN@1wP5uSj^?%Y-%*Xa2=YjY){PIA34`6Ibv@uc%{`%+%o3G@kB)BlYR? z^BwV=Wz0Mt$W^24me6cCC9SS=psdF!9(se%y{i&HK$(nbB#^JzKf$9}W}dbSNKxg% zj6ar8xdTs|EWfX8_P+fR)wzS`C%E4J|diIvMKKWGsNrCh& z(P-7>5k=4O5fX_HZ~ojH{oo6;dXsupm+4LdYSDhH(uZ<{JinZ1*pjs(){hwXrHMFQ z@-$(Y#{XqQ8%GwbI!xx*gC#6Wy!V;5;~o6cWylb~?`L?sqy-(P5D|_rJ!H2(CJyJp zGMx8+=eBjC{%y3$+fV_5DRnklzk%iG7ql9QMb^=EFGt}DDVK;)d`^$3YJ}Bl>%%0K z5H>J9%B6?_5@DPcADH||XP0M~50hpoMquBYx3*mPc1)XlCD(ky3Lit2xH>^^<6ifV zN@zn`?~|Pn%k$L%ee(I@RrNkuK07hl+vnYx=ic>R?YlI8XX2vWZ(bq9pmoMZU`#6n zze+^x^=maXp0U!(^Kh2{_}cUj1wUO=@DXtXQ?S%uF}NhS2tR6c8+s&Um)@%9TZ_;r zE-;UQx}K=@gPSP^sbU?(Kxjm9ARmBEw{G?5UQ8bFq{1>`YLMU2jA}|d0k%v(sE^ej zx}H7F@yvmDOU0|;Xgkv+Ca4=bmFpFMSs@|;+Cc&9ptF!F`gi+ivEWsu?jdNIPmG3* zUhO?HWgp>X$U&`JX`#P^iGgYdw^(5Cc~BBp$#Iq zpX%yvv|}!itjF|a1xzvSHjO)6=@%9yYRcS%xI2xqfk+SJ%pnGnW&n3g2ADJ#%CO%y zZQh19JrmOlhh`b0!wNlZ`vkG6Om!;E-7N6vk#>$F^QwO|o+?CKjas(s-Gql^{>YRF zM&2z5k^Z7(MTWq@Im+?n!x>z3VzGrwi-iqi2oiw=gN03zgBFx)eru0Y)6PiG%^f%R zna31WtcS%pKo?ETKW3(IhfXCuVO>q@tW3$P4OHY}xxke_IxA?NN z^kH8F^+3j1=u|Y7A07fJ)V*qI^3GYPSMM5jXl8&d;A*?cQNz^7Bb!eFA51-1EKc4*PX&m;>9Kx z9wTL_CMvj<5=_7aiobl{A-+2HYUpGjUTlWj5&iu6^8-O*OoNAp(wOGaGUalYI|^9) zt-~&{zm@O-&bM#3LCD>=lDo_(%EkjsL9(+T?g)Z3=@*r`ub|(TF94~`13Yujk2)ki zH*hH3R{tvKloRF=(L4O(FT1A*A`9bMc_abCVG8cYPSMrV`#s@r zLjxuZ&j1TB7JPA+^LZ0hYf9eVGjUgux(SKc6@TcrQVv7K3@@sDID-RF4vw$+>;R>g zz{)gyoloo`&<8$`-vYTaVx4~4!V!;iGromkC$v6KYeQryWYLte1cAr4qBJ)CbN9&b-GpN)4<0NEC9NSl^0fv{N6*n}<+GEid2+xi1q61bkXj`MH}~|36Z0_f z5EM0g1Y)1YO`4cCaBA;whQKcBL1#cMD9M~bIvl?Y5BPLc9vms!`Kigt>)*U=Wj0OiRnnt$WQRTP9`!!wwNc zO3Cry+D%eSR6abNy^J5J_qJuDm zkOir0e-3=u0c8Q?t$G!p-7_3q+xe?^S2K~2zoEcYgj)#^0E+%XcTIi*-Zl^mMV_X^ERJ%nW@6JaN`7Zg)lYu#j{}Vi^3(I?XnX96)gsnRTW=bH!7zQ zo?8F*+Z>In-i=m#*(gk4nNVN%%28&kfx%PX$Gy0 zy6>;{+ITX-#j}{Z`LewjXi{348uaP2W9#CU8oL$|d4i>zH{G7^YqplGuhg}mf$_B} zAK0j?awZv#KVvB{qD-mTKCtl&t@^s2y2a54z`XkDhwEybjjVTYmn;4#E+j}$VDWN~ zX7#_9GUGoT_MWJp?53fsz$ZZ0^=Z|;Zyp~rALCVsbG~@W>nNGAAn4|sK3=fz*G_DY z9t})n42u}1sW5HUPT74|*bW%leM=XNNIfzoGM*uKX;r?LT~yWdp!BWGnd%z9?VS~~ zU6CACdY|9qD@nIyoW)5M)x~+qptK8bmmHFC?f6#hH<%vPJ@%x>$;tI72ZT0=%u69g zFF<~EnbhzCBbIIV(@(UcVArLV=kaPM$O>rGQeUTc?*~~|$f^YBK_uIIn>w;J;J6~! zYcHokcxMQeNqq$81nrf-zbODD0_dboiM~GfqCkZq1_%X6yX{tt!<%*-+JZtVdrNbL z=Q;2v+AgF!zcA{M?1{na8#Qi>jIoX?mj#wJEqZyzK3|{8hY}ReXg4|eO5euY0s<0P z3|wj5M%B*0l3Q_9Doqf-Khd&f%L~gM>0Zm2sIaB>ZQpU|ni~d6UmaZB^P3GGNOd9B zZ?LQ9UZ~~zDCcv{WH?QZ{eke#nwlF)6Fg8EzscuaH)wXGoL*_34%2sB8HH=j30|I32dW?jB47~Q{a)8j42jUK&Zn*P8&@AJyO*cDAo z+_Pvv;>0>#zSFeVslV}kd_B#>JKjJ1wb^dBeT)8n>@wnk-r_dxOax{Bc7PEgZZ%)-1*h>YFu379YU5}6C1-2OC$A}g1hY=D+qH5# zUo&q~=%a#l9i?pe_GQsh8MraL;X2A5fWWPreZyk@c<`W=N1x_u*d<9ox}LeBxLAyn zp!jJ<_(4fQn+L87e4WhDu(ZsCn(qf%QQFNTCLY$7wPvDt8YmKJ2fH!S5Kk2bgW9p{ zs*Hi2MB>1)64RiE;TZ873Cl61l8{`Q$^z%|G13u{Xdz58ttWZK;vmnz z8Tg@)f<{;%bVOJD88VC1hJs^@O`p}jHZ_>!;bAOkG!z||FUUk02316Eg$QY7+VA2L z6K29#fWi9NL~RMoes=t!jJG7%RK!y7!pveTWvI71C2ywI7JB!VSnN`Sdzgb!a8@mlg_L9CRCSKTW z0_!p*DL8-mhhUet?Q6-jCX)duvm;RrK~cC3B8#oT%Ct6+!s*9L90usRTGP5931iU4 zHzHE656ygodbDax1PUgc!iwc_G6SBp^B_4;6e@BkKyAtzEF>=qS<(zUat)Sbsw$u` z(_ue+C~dNDAP6)UnZGSaloXB`ttyF_Q=Oas#m=SfuC8AIXH5AL%K^f)EK; z84(M!5hI#cXqsE6covc{2xBjBq(Wf;4;^!iwLcyUKXD@acu9vrW&j9egTV#dN(@ez z<=N9gPcqj?*3yCt>sP4n;1h>v;+k1A=?Nlk-iI$$Selt_T=G=G*~4|DYWH1MPTI`$5|AaD+R?L9 z3yob?9D}&O(acYb6C-v?edY&)Mfrey zOt*LMbBPy!YsD+VE}NY1OKGzphLU}earRTZ*r(w)^#13Jznb)o+hD>tIdLJPRj@-L zhli9?up)GQrSCHFb9_jMZYA+AZBlGuC7w39ox?*h9Y5YTd;|y_XZ!g_MJcJNSwyK{ z4ErkSkVAFqq-{^}4?5s;&q}|;#zzCUEj4e!zSvc>r_sxFf0){Qb;!2yZI)J+*;d1F zVM}6c-gNK04cAm5w5&Y741>!0%{pvGzv7b%`X|1<9_@F%+a+1skg_k|nNz~@l&TDo z)~bcf_1N^1;hjQK9S<(OcbooL_)3Mrr~{CkwedT7Q!+5#ISrR`tx2OsKW+S7tZNu=gFN;y zFmTptf|C?I%&vn6dyJCwcE0R=bMx{&&(JCa^j$Dd4Qj^UCKL67Nxa ziP+dh%p6j7eaN@_Lw?_Nz2LX61?paKvkDW7U5sEq<$?E@z+{$q1!Sbu zZ}fsDpUZjET|gWq(kidB{Q$&;V#~t$rR_Qiex~Fryj<7(6|0qL-ljA+ZQigj59n#! ztE;SM&fJ3gsHo@s@i(U}mFxhMhsj5%xkYXduWi*r(avBPE4{lSx-eeJv-KA>hxE+Tu5OEh@M#xI@An_eW-IRwL-hff z)$4+ld*t5I6~&W#$6MB)U>v-=rA5)n8GjVGUTzq+Z|#{-wbGXX9%v*;wBu!>rdZY+kddaZZ3^SP^CXendsAI# zTs{<%{_n8QnQ$5HAZXC2;=@JJBQcl=c*4Q8e4omKi?c5vOK4fJPc(ogvlby}!_-)r zTm2~IlijRice^x<(qx&X9v$HY zsib|#^UaM~l0L0#!5%szN({EFsn=`z4fmqpW8e-=YVOCI)te0Z_1n4plk&OL{t`Aj7Q~D0@m22DEn8lbF@92N zt20qi2?RfQLs;3C+c&PD4s}=8Nj*LXR-`J@mvdC1H7vNhYXGnF)?^_fI@CN_krZ@@Y5Xm)GNWtxk0I=X zp+wH8zP!Bml%*%tM#8P7v)ewic=aLnCp;%#d*T@xwfE#u3(W4n`bDg_p!Ak<{((6W z5XXN_yevt`Ec-_(vN)AjM_>^*mNFXy-dF+Cc$1VPxP{$f=c(cnDB;3>{qg*VwFJl* zh41pIoN{dR7Sk8EG(x70>*Ot;`a55o|Ig_wB{(7V(~esCProzx%)%BL#?jUN?N8io zFrm}!lZ@t+GL{}DP#_tGggg&1^`y)i3lTjA=BeOJU7-V8j4u2lD#m8M@zxe3Z&dCtlIzaZw3Cp`_r}voD z=RlT>n}(dj6!50ZZvghLmh;r^DEYYj*x1vNk542T%b9k%^YW$ z2Nw*4KQZGMgB#!n1-JwH*-NIC_}9K0#$GA>dugzX^Pgf6%sgXlv3QelsjJ;6)6}}u zK3vOq2q)R0Wev@edPFDpR~|V*2g#KfmD@sDuSQI|5Up)z+owVB_LdeCbiOaJzr02f zzHi!ChadX(O<;!qqPZ6$Ir7jVJq1ujM`)MO)pY^?_h>!=DknHJM#e5EvL!GYazfI=j(yxffgu47cJ1&rsXVGjfU|`!qr! zFHG~Er37G=kYexg&bULflggpB%_KZTh7TtYxh8#K%2c}J^AHsu+DDuGW@ly9-T6bp z?)z`V5sI|Euk&hdy3U)YlRtwo-@ji`&wpLqMEay$mR@HVHIM6sEP2C&k<9lnWJF3! z5k>N7M8gtTz2S`9oj(RNNtPEkc(7YaA7r@-Ps^ro%@=${k7zF*`r#u~+q*GMYU!;ATcl=PbXm(E&w z&!c;>D_^g3b4??j5tTj!-*sB38QS$TVK8&Hz*%AT+W(k zBc)r%7^yTgq6cTZIQM&W<0QY)RLY@$&i`)Q>drl#o3bi3X)5Ya#RbHDdN?FQA?lQA z8iqCi2y#I3!Q6W96zfm&jI_*jVGP&NJl^0A?-omA_F5vJ#*Opo8|r4tloixsGtnNd zl{M;WuxTZtb@b?s%IavsbS!veL&K`kPH-OH2}TMFJ)i0*xg$MNDBS|h1mC#V1jPUJ zs3nEw;FZtCgP?zZ4SpKSN!eyQh^;=L@1rH_O9VlKzXa8D`EIROwY7)QY=Q4KwSinI z+^^%utD`S@`&icpX%-FUv121G58QvlR_clVR?pn*+Gh#b1nG*r1`md7I;OCphD8&G z;7Va?Nk-6hiKCO8U}#_3Z(%)*Ryr{<41kSEASUdQzmS;5Z;I8vq_KVsq6OyGSDU|& zT{qQg&6#fM(QTj=2rds zWvaU87BGWZcw>#brAW5H4o);{90-wtL^*PMqxaLBXjc8U+EAn9^4$+`U5PiaZVA;^ zsJk6IK1lk**T>q2d5ak2M+EKPzXi(m1g0iF?-Bu+aEQ=`s!MA^uU%`3mkGcD_M>s@o}3 z#**ezH!%>2^(W~dI4MG0g@(c|;X3o(UACVIuF6bHQB1QvkDP=_pQ{+hdUHV8d|BQ1 z!2Lpu>_fPxZM}>>M>Ht8#I&I$hgy-PW2AL-W9-@26q6P_+o-0p$Ui6_HiYnE!#ksH z$f)|XE;6d5=hg4;Gjo1=b78WIuHMGE$ylFl6{n=OwkXnVsr|M2AlWGzhQ1pEhr2sQ zkIrqUP}r7C%jx%9vuWcyjX6P4(QD?pHFMTGT?ZFiz^gQwXqr3}go3J#ll;q2K^M?4 z@#M3p{FOSDIN9|hyi71}>igPtE(29zXbElPOQ1~ZByy4@rTMhP_KiU4sgus5WDOXi zhOkF4y(E*0-mMzwXbmoAf=>ayaq}5&jEh*ql6Y=<_*42o&&u@> zPDgIE%JJWsw(YN#UPovJmEoNGTBT%#8;p{ST{r^MEwMVgGD1#W?>ztw+TZ3)KQpN7SB)3AJjqgY77_F zcjSx)D8I?MLYwJvgk0^wuE+$b7++JOYA5`NH7TyMmku5_^ttDi{^MWzIB2S_5Ew%K zfxYO5Ij;D|HI@#t@M>y~BKcQ>bp&wdBXPRgsFfO@dq0QQikI%HL6Ks2;)4gjC+_KP z8Z9w`qU!VUu^vuQE#w;Rh*e^*a@$9vYw1gmCKx~}g&l8~L3a`JN_G=g3o9uVI8h^& zxxxTB(s@6+O~{oB`ViVK#CyYKI{^O$BV{ZIE2?_M<+R}8!8U!yf)}NAnOm~@ig^t- z5Df^K{c8zgAp3wHEpK!NA!DHN4UszU<%LI@d$5WUr`jH2hbS>ifhGH#jVJfWXWO>j zJv0p43Pyw|SLbZ;yGP%{63fcj^2Ji==ZTZ&(?B6HKXPpdR180#tUO)a)oukFirge) z1+9kazIdO4(3+hhilY}b@5e7&)$V z=@Amkbw^;d_|^2VS?BRiw9EtbKxOo2$-cw=IZGH_ z+kLtC6ADX|C=)_+;_wZI2KnYc)VCGM?jXW771wKe#h%W4bHGk%MuT1Fx*_3f;y=I7 z@I%INZpZQO4n8&6d>wF#M?HIYqo9{D${9z9zC-A(9nFUHI=N$%B6-mgr-r5#f2o~W zc!j}Fb8*2Iq`jlprLc< zk-CijO)x}M#Q-m^mFFX770w)Ax!11A<#Cc(xWMQ$XYNm%#k@f%5)?pu&b@oGFgB45tvNJP+lCr+*$Aoxv^Dc7SR<$ZWLS5?qW!uJgCFNt`$A^ zZA-GMy)u-)>4{5z-|Ll+Q#f2ovpUz@OZx8pE^$+&F!ZtK{uiRC`p8p)Er3ai>LX8A z`%=!rF-(AcpN>H%8Hsb|?q?hh!pK;th#-*dvtycO%)hq*7aF>?~ko9@t2OVrUk4P538 zMh(d4KVhf6C81^&uTiQhclWsbO@;%31q(_GPUAynM>oa!7(Tk4)#we0!b?G_G_I-!u{<|wiN*!fUq{qlF z5auRdh6i;T&hk9j(R*f1#*v+}v~n&=zdk8JlDcaHGj{|@ySyn1ZP!6b!`X_G1qjFJ104%q#H zAC6kG8W)a~mC4%qECYUe+M)o)V!g1{ zw=Ht&JADizUx{vsO7&}=Yu~73ODXCsrgK~suP-pv3-H)_yCz?2W{+0l%5mz^vUK~B zAyfp;EB^1l4ibwcdO>=N-zFUtwOO+jJLW)d(HC1Kb31MK~2 zyzuI6xF<1r5&0B$>a-X!`Wt=jH4yuKXs);4x9VJ9d6-A)J!ARa$jOTJMh^9xgFKG-&JJ9$M-yB~#^i=x72r z!vzS)qTt1xV9bIAAt10C#_cIL*PihC-)J;%uq~o^UMx7!b zIgia*`H@Rrt9s`@wYthB$gvm2Eu6V$iiyU`Wvx^J%teO1H4I~T@;K=|jHrGvFiD*G z)3=|mSI!vJ0n)M6b&;YBSsk>Q9}*@Y55-A~l$4tIvCID0`vZ{ZWvQm_?Ez5G9XRlb z;83?|cijWAXxh7f|F-5iFe}IuEcv677s{Z(2mfz8J+T1ij>xF_++|dYd6(1vi_?~G z+FD-P_2jfh!XE74|8|!L&G)`-T8{r^@Vfj;3x>?`_muqfn1M@@f(+)vQlK=D>Q+dI z672=Q*L?4`abS*=^0XEdaE6J2R8QUQrK{-CO1g@$=&LS8`qUg*mjd!>DGDo#k}e-K z_Tvp3odIk`n1l+{V(Qd_d&`*~%Ha)@0gv>jV;i06CWc+qMf8N)uBS$G6bdf{9#`4m z^p?Z%4nfKG;xluMxPn_v@W9_s$usQaTDmj>E;F*b{iR`m)s20;89aKyQV*J zQf0I``FVAcz?M}nwxOZ*%VpN0^oUxdF3u&A2@zrqnk9(|pIjbI{D*Mx5BVcT_%1lq zq~t%1&_bgA=v|pKw@MpPs+v2%!N#_34;J0rb#=elh&G6unk^5#Yp{3)Oa@wfsdbU%nN;{-d9o5Wk#{*X-l z8Q}?bB(xcH4?6hRX0Zs-eAkY)#r`QQY+14mTyvRC;FS6I>(vjsgmm9S6~P3RCm-Wa zWh%_-`kO{=TqIQzUjeX*eiEazkmt46r7uU}^0g8~0(CB8#&V`Ua%DjI^QBB0iajfr zg5KG9h&G%}dxY9-c#;D(3t%KtOj>qfS2Uk<!!DkL_cA3b_sin?r;-XT=Ab>%4+X4 z`7O+lvGMmzd*0=aV8*!~Q56k#^$K2x*jqBk*x`t+0#D4H z`(k>M$LdMHSLJnKx}K_Np%+Zr62-_Pln)m&em+6RDtv}GOv2v|z}k?Qp@`FeG6U-7*w=k&V6b=AQ*17gwn#rNA- zjUb0z9Irc66&JqpoPx`GDL4D4LhI8y^Qwx?vuBQ|sz_jY>}hxI)`K0#cNUc>CHsbs z|AlIua?DpeTXk;#@cJ6Nk|qwS8@hC_b#6O#XMAD>m{z}L@K|Ehj{ayBVyT#ducu6U zP>gMTvC1nunn6F5uob%+__S@+N@jU@se^CePeh#2rH7q-#)%2CWN)o6IF?qi?1tB> z9*6tQuaj(FM;SnPsVe9F{WQwLabS8)>8g@ws%b?;uRn%?J;Of)S>C0KQHZ4afvPX$pnyv72)O0*Ag~+cx8B_O&{TN%bjCL|b z1{WV|tJSgNcA9DdvG8&(wis=HMhYUJ5u{kk7zz<3;4#z9c)BE_f?1(ur2&KE(9fgX zNlb@!KDbObBDDca)x{>9+`82g_sq~+^Tcu!?YI#|L6S2JH$6c{xk2wY7@_5ip(xsuyWI!b)?`s6P{;zwk9GsWw<+YvZd1(Kg&f4%fh-xk{1 zFT=`cI!Q75r0lQ@1H9bdRYa#aPj+{XX}aIF=6Zu|K7oBBrfso`HPe}OCy7#pgQsjy zqlfdmYe#fNa0Cf815jR32Gd(@V+?3`Es`};O>UEFr7quw`o~X7v zyUYcUfa27Y`#p5gHCPQ&nR4KVFv!X>)Qc=Y5~<*06q3Ud{GL}AYBECf^%Frk8$5aAe9Z|M~B^c>|q-XEXWhuYJ| z$RtP1@^^CYDBoG8bcG_2w72OzT5-*(IX{N*8E*h zPGJv^s_97)mOX2uxL(emYm^+hn0tVEyddzPNuB>*-F%M58Oxr4Nfu9*4O$glkw*?m z?laXLru;S897d1~W@)t@Q*2$|cQYz_Fi`W9u9qdy1@L@F(ErFl#)iWCfwu^C%di=@ z5zMC5%JnZ)$=`p!B3r<~A%Bu?NOxy>=k%Psk`UKv(=<6(pMQAzRBpU_bWTNYC&`t7 zM@)c?RWHsh?i%SBs3_tN?Cl_az%gsJ>)1B6ZE z0aXDL)eWhgqu__4hR}%Dh8v_bGSt$_D^3VKRs72MPcLd{}!-0Ug6}`hFdg6SPN1W7oR5*WBBSNM%OK z?O;Fb@V}$-S$Uz!0jg0ccv$n>R$aPgNB3;DqoXfZU%`?dw12O;?-*Vv0L2puxfx&6 z_Kg)m486hlD3}mn9<|I@=G8A&tu8Qvbi*75o5eMT+U8@D2Z*zis0Xxs)keCz8xiKx z*=4%AvZmA6HG;tM)|gah=7wALlv+g`VmQ*H^*Ldb)+PEP5CX#oY1^sa&P5cLuweDi z8ek9d@p<$feLf4rL*?bJtU<3|kWQu$c@V(KKS4Q(qdD}ZX_#tR@o=^06W!*4_=5@E zu+yK;Q8Q+`F75gh_VS}l8=!t_xVW5<*8gk)^_5I7BQiQ(>f5+sLn3dz5%r;4nJ2(~i*3rjz_7F3~8cdfH0Ad{AG7$>QT|jfr z%o>LhE&QJ83{E=0TNv0{`>SDFSov_1#9PqB#o#cw60Fc;k&%-vt$MVQLJOf^T6X)& z8KT5MpuxI`@kbggtYj{QH*U=XP|LPnK>`dT$&gsapHRlI+0wNbkinxbrB}lOz{Ze7 z872|OI2~=h;&3!s00pGy?wp>)kNM>VD8s zV#`$k`RLHM3=qU`M^VusiB#VUcBxzQlV3HsSiCEhQV}q?*_P-xF@xm%NnU$z^V{>c zW#4F63XN+Dh`}&(-cb`gA-tgD_ zi~waResW;FGhNF3PvN^v8;54xjl{36<7eZh67)o6{EbNQ~CDelqyG z1)l9x$g7s?Vp7p|cS}wj2$GCOw=xV&5zBDup|$at&OKTPT_uaYhoQq3AiV5o?s5EQ z6b#{jY#35cC_1sUrw|lapObcrUXo^8B8SsvapsAbxU{rX-{r&Gw^-UeyR=dZAnV0l z@ovHO60Sx@gXQ|*cm zphRG>_iJirzrd%!wZez{25i`d=KP4tjAfXgTbthO8k$pq(sB)rkChkR1nuxW?pm{B zdW(V=LghHO{7aUmhH==!WooCQ)iZMQh%tXSh6T21SpIDMKZ*#`-U?%~u-wUP)Z z!yvyv3xC{6!-pOTfwZH*4E1o087z{=Wq;|&dC_y9`NqxsiLE*fk&a7EM1jXQcczx1 z6{2kk@sz+3jXt}AiCrbbs6M?|`dutqB>4`b!A9`=m0vDN9P^6c){?M+A)*kZosRp2 zrCh+$3hqW){mm3RQfpF*ooHlc>qyV`;bnYU3Sxm$cu9~n4dx5!voTfa@6rV3;!LE$ zv)b2>f57GQbq=Di@QVCXCq!NrKIeOE_AGFvB43MQ2<{si9zF`^k*V=xVyO$T2O|eQ zdg4YM^rsZm>sN3+EWAF9;q2yrZU*%og*zFGeftX8Z-7JrT*k4u*wvV-vo|=A_-4?S zbo(>{g95#}tK$mmLq6S{=u??i1!iivCowzPPy=%hCeX-NS!B`|JT!+hPV`>p*NYcq zw1RkLulY%l5BVsqQ3XN$aoyzKCcoIj$0%Jg6;S2T3I4h%hZ+<3M14ahc3|2p?qG2X zgd!MaY%7el9hecseC(oDxzwW!8Hu>zHQH^c&P*PPw*@SAwW>!iv)+$H7t^49kvJimnKa7g#=Kg;{?qqk1%C>#e8z$t$w;r4;= z`fNVF|4!x_i4$UDsda2#Qcr4rpyOM#GuD-p0{$Hfwt_ywC{1Dq;QmHp|3BWoRQ_GaSM1Ql7z}Y$H>!8 z=!3k=S2ttiop71yTF#vL8jN%l9Eb3yq}CvxoP$ImxdZg8`O?6Vvyt{pr7oG*XxgsJ zVroq~(q^f9xFY_#?&YQ8>aAZ>WWwY~{59ZXQIG3}G^k>lHPY17JA4EKXHvXOZ{diD z*tFAM;N01To_hefEI~+}J3V%ea%k4rxY_9^jSfGA6jg_B_Ii2IZTp$wGjg9z(Zjw$ zlINJAEv&HdBJAbcSF*b>=26S9`NrR$a1e-@2_u4sjc3pHw33Wo4DuyDtg|vVg!|_<3sM}yapD|n z3;9e&PJ)brWe)l7jA?)1Kb&hYLn!-eIhW)n@iG`R=*t=HaPAP$g)uY-X|rflWf<9G z-yf%4x9`t&aC7^bo6>}bGEzBQuCxPRlIxZz3kzc|IU$qKKc8sy_Ip?#UAumdEnSNg zd^JTiAVrReEexLhedabsOmvpkskd!hJA@>*9=+{{)6XbJ?$1+S-)U>7pn}jc42Igo z0Kw`}QWP$KgUa2AdKn|ANqENbF%7t6@_nmU&5roKSJd9yZ(mvDDU#RQdpB)@ZN2W! zl2>w4THNo-7|p(&^3*hbdK7MwNA&B$wwfBzYnlw-dVJE(b#2eBpEGl28_K&+4zPyj z^LhB&Dn@1FTk=N%oIVj?s63Skp295hF6X~+q~sGM#4|xW2q`VZM>=yOJ<^CQm8YEK zsAWy60WE;;JqS{_%R6%i<^sC=%bYqiXxvo9_D)EYKa!fPEw$uef6=Qagw2dnb@#j{hqZKe-oHM_ZrV zQKhJrQ==ZelJcqZp8Id6CFp;<19&0(!1p%+TE>(MF$6KCV5Q(p5FF~GuWFkj*}R+y zj0?gROM}v<&lXZqqS-CH1oR;elj@E*-A(rl=feUv#40!S1ms(|&7|k~fuv2qB$&&V zF~FMlkGmOt8uu6Mm#!lJ8SeC|gT4Lvyws?LAbp^(Uxr3n_Eg^2+D2C_l1c>p0H=m{ zuq~I)jWZ_8Bx4FA6KRO~ZJkyVb@j1$s!|HMcRtrXi~&ELw`s|D--&~oIEzNi8REO! zFrF4nns%n9Ay{j@=nupc<(23sc%7K0>k`Ja5n*=C&MCE zAAs_cF4=;R%R;!_^8p4LHfqEV=E_Lqw&8EnDn%~2_;&}RfQkAND<@7juU@^C`UbPSa z>n@XonD#CMl{oO6%abiDfR=Y4>SVM4d%Fxvo2j#Bzt5c#IH(cA5O+z~&>h=4FOaN{5R} zrZuKJ|DVvP;k6O9Y!G=cU%r@>Ww&G&7h4X^&{TFeYyUNHnozyMo+8iWkP9do+M>({ znubz8EkbH&bh$BPiqIF&T`pq+X^_QO|A^1KOw2aL8tk(1R;j88Tn(05VA2UxyyA7i zNJ~35waS?HCz=MfxUj^apJfFq_PQshmHqCxsIB#d%QuRKEnrqKK;NWcV*d^z!a3wLvXRr$YnRO+6Iq*PXVXhHvUDZN=NAQ;n}0DXM6%@k^(~b z!o{(xHfk|4!OB3a^Gqh{`~Pnx1A{+NGC+*^e0Qa;Zlh<6+yS2W{J#v?f8S+DwPobxmn+xe)9e-jd`XpvH49K zlU+wrf?WLXED*~{dgLBw#Zs9v3igI(Hw0bw5}Y8BKZ-uZ1U zHVKOxB?`cxTlS&GDAGpZ7@%Dq_+-`Pg;+vKh*a7^+o%K~-Erfje27TIUv(!?vrLL5 zVn=#I<)pY$xc`WKj6CP=z<6HN%-#9rcGK@!Rs$KiA##348|nPKW!LQ#;EhZ*^hFa>U;!yPLsoCjl>MZ0oqP0}ADArXCL%r4{(bxSxMbSU= zGNY9f^8=8P)6?d8gX{kc`JIURVr;JSXzf z8`uImbJvJ=&Xq{{E7L67<&kY^W!Xj@$!~iL1X6&cngt<4JAn&Y;3mMZ@}WT2#?4UJ#KjrWC-V)EeA?o0#J|%rrp&}E8LXby z)pY|OE_HHzRqwvOQE{h1mEUr|WD3$yQurT_>}O@`a`H}f#U(w8f!qA>xm8fl#T3y$Z~`8_1veXnlng#S z@z6&6%!k-t!;bPbJ2A2v%7LW0*H2pLIP}6>>w1kNc9^}<33=LOX6TJ0ljl$El|5}N zYyz&!wF7QGxvCPqBrj`w-@dTwV^8H(akTAo?_+Z|dv#uGgPa23aF&yfD`N~CA{40{ z0I%U)3)RY_)2^`bP}Y0fUF*mtKE=rcMyM#(w6xNGPplO5Q&`K~YyS_OLf9MHJaOY~ zu5IK)X?0or%vtUWuzlKjKjtq#pRky4=wD2`YFXKAnr9!s)=#2f`wJy)E#pE@ui(`X z`xLHIV8UPcGnX%LuE;zB^n0(4id(C(>o;Zt)D4Uy^3KXzzJZJojYiLErDSs@pV=b=hIgLg z1Wh4mQ^nmW(2k~lNCj8NRe&WMo8GaP!!O`_Vbj2oNCk!nz~j0)IybmD0#PD=M^c?o zQe8SMe^5Tj4&|iCLjO^QR-c*~?lM(%yUQV_QI-zPQ59Gumi2L;w{7+vXD_J@;+=W6 zo4&q0pw}km3WIU*8kIWfmW`V zGYtg;*4?f>dy1Z=Hl&FjDPkVy+}}%@z4N8G(xNPPp^8FrNKL!Gpy3>#wpvGkYd+6= zz9AnFYzXcAs)fvbg%pg&QErUWC6>R`RBk`gf1#P5$CO2s_{tcRXnd_jbYuHs`Ro>T z#m(-%v>l1XkIZ*u&ShEIvjfG z($g6rAn-*(o>wgtG6jeh6}o%-8Jb{bh?UvXVlb7k9D^=N`Q+*xdB(DQunsH+x{XeP z@ZgPT-9V_#TgioLQ56ob-5$y51PZ!r>? z>%maYt}556b|kIgk_Oez^l+KF*5IaKQhTL;egXf}5o6Rn;hiD122-fA_7kwM7>m0; zSt+1r;n?zk;$Y@7{NRMB)G@djqq!S!=s~ZdmeF-FB!hLAa&Bf`*9vUOe!e<4_gTRI zbj~o0wtg6_{_kz^#Zm$aC_1caerj%a;%z?YLh0xGeNDFpP&-po9CVxCBD#)h)ch#6 z;pzyw9iKH3-d9pt?-|xx$vH0~A;jW@X;0Bywxtg`bA8Oz|Ix{-qS6l!omZ%jK z#c=nI(@omCb^L6CY0S+AoqfK3)K<~Alf%mRXSaZW#FjHJVV+_Z`nH@}>HN|pZL5rV z{a(?HuS&L2n`-YkFC}{ItX!A6KWbDRnEAu)xXGD5mTvhay0R5^UEkAa;|(k5cYN7S zTNaUEb>O~C(Xubwe(bQLo&PkZXov-B{mek&=MC!cqRmU#F}-cBv|v(sZTq<7MdB6> zQGcc-5+((B4N{J*!hCk@0*GPDGOoq<)isa&ht=K>-x*az0ggPPIbTAwN&T~Yl#E9U z5Om*sGIkb>Jc%(Sxc<-J3+&A8%-0MIRQ-ggJ%MpM7+PV)Df2bcv15x$8b|j4Z6*)r zM;1qp(rJvrFI?8WLpmzkBMpu3lz|=$1+DCGy#NKyJ~dvU7Xu{g&On%--#SIC(-;*f z89m)Xx5xu-(Aj+h_m!0aPQOXUB}bI}>$HD*4f<Vy z@u!2ME5t^oXP-V_ue`l-9PHF|t6UA@sDE;rOBZ8{ys36}>Z*I;sZk2cgmm^=?~H$o zs5$fI>7<^leKYd-?sLOAkehAgrGgQCoPHTMfXNr@#yPE*SZm|T5UM!iGtRyw%J)ht zNO)?hLjF~(bgMhIXz|MXi$c#3qNg0JA|@GL@_?Jq-c}Sl2!I%|{iE2CiWlXFD0W;; zT~7%AguFAjulJBCsBNI$T>w0bU7E?nu_}B|UpZ06!9gaS)T)hkmqm$Z675!qTgi7i zxFyhsk3kEqNDSP;SO7*{y^DYMK3jqemqF*(-2Cs*Hd3wLB$kg{?+&uk{VFf79RJVp z3?W7WDmoY*z&AoptLTiH>JlcDD++9^B5fpe#-<9srn15Q!#Qd0gicRw!bcu0I&-Q^ zNSo2!8i*i4%OL-Iieq3GQ~mV|XUNuIwW$64W=57%;qA6FT4+T7Tosbi?r@9V&f61k zp_S|4buK*0SFEDn(bN0Ed)yY;4AL6BU5TB`JE)i>xDG}Onkh1kp{^9x|Ho5r-6V(M z!omco#^t)FvXLrC)*T?mBi)#aXsq^V0l9LHt|nv5aJsw+3Ox&OOoupH5g~7{w!{+k z09d(l_;&cA|MOXHxMI!rBh4{Tr@zIM`$~CKeJZ6F{~0%DG*Eisz?z=^=FJsVW-^1W7X6dT`I&a)LL`7C|d9?rN>m(rj?aBBXUj!Fp#`g zE3C4TgbiWiF*xov5lFNI$?Ckne_b4T4d5I*%I+rJS9hTmRN3h34G+6-AMLuhno@uw z$qFk+Xq|6T)ps3$+rXHPs9#+9VsVCjzUgpcMb|?U>#HORX2~U%O`0~%PHMwLQ+=VJ zV=AcNatj|J0qA*8C*4x$L7o%ccbD$n$66A`(2vfkE51mdm;OIkNh$};e_2Uw+T2Yt z@KtIA@Edx~+6sv^3+4K6q9P@7WKIeF?YCB_pe0>x#ig;q9w(l>-+AQ7c1VsuAva(p z4U3ijY*l*cJ?AU1A_YW2XQIBx0zNT#k{qXZsT4(Q`z1GJ!p23_y_h3OE~{IBo>*70 zKi~ndKPK}xB!!wGjfF?;cDL%>OEqk7nZho+fFHxBln)8oaY?f|--cW-n-uz|V;JUA zc$BsU$+lW4Zv9+in{qgc?h0w!;^R6V*!19m4&jfHY4*%?uRKZRk+}GV-^Ve96MNVx z^9`}WV&FDhm2($!h5>a{lf~zMENgYx=iU4|*#qJZwfbeGx7X@RL5L~QMwwf>iR>

kx3G_%2@jmO-;Qjm!I z(dk?c)M_Fq1>zG}b(zOg8RFIxC=X8#^*!tlFdqaM?O?>!j87%(>WxCzvv8aa zn*C4jO+D$?e81;3I5z0kYHh~g2CTlKcRdG}G0Zb1K{9B6onq#c|Hy4s)f7|(6)@mn z@sUKe%%v?A*=@yhgNq#Y_WhMBYDv&KtqtXZg16A+Y&-?7Pe8x{VeGlgr8XCv8Hw4 z_Lre>!|`1e=IYmZAFHcFn@=xj3-&^PDRXM=4@4bMigk3`z6%)GXj z;8Pm6UKYB$x1Am{mq-p7=xXPhE2ex9(~Q8xMW3Gs`p{dc^v^J&5P5zc!#tleW^G!N zA&f5t=b?G+WdRLL_w#pwc&yDIOM01v;fnL&eBikCnK!wl+SrKNj-icAlC zQ*3@golHlheN*wgiM3N{(SlvIb?vX|cdH*dRxem7OR&h(YZ-T8nrcm~x%T^(m1dsr z1n~UH_vSeJ`kntYpf_f?n-Z{N$6X9A9l;*H0TX$*Z{IGYYVA5`P;#n;Z(Jl#9||pD z4eL@Yd$z{$%o&>u9(z}WT^}o61&Je;q)THu5~VvK5-%9k9c`-8uLMujAA^v#AD$9$ z=_`hI!hDWyVgNBRM@nl7}uolKf{;^0nJ=d0W|NMX0 zIuo#*^RNG#G4{1Wgplk@GD>K%WM7&rAt{tCYlt>7wn`G&Ly@6KV#tz`C9)?}BBi1z zv}vQ||2lV>XP)^#U9P!)KkMCne?Omd&ilNV+W(vd^}~S|XiZLA8|}B~>js5Y!D6+m zNSab**{FW~cd=6EYrZ;!6A%G&q34+jsQ6pRsqU^c_LfcmxsvxcjqTI{jqJuneeYg23ru+L8~`%*7SiLCLS4fct|UprMP5ilA!L=MCcqMD29h# zOcK0jr@lhYBgAbQwB*=p`@k2ZVTBR+LARBfT0?YqP@(d3ACe`40~GcT?^1#dr-In! z-uN+TjeoiUqdT)TtJGZ`54nmq{misrpt0U(SX#<*m(|@~ByZP8=}`k6)LmwNBJ(wu>4x(B(bJUx;FMEsHg&8&*-c^EZrdPX zcIeG7X9g>9rpSfEryfj-t6H%of~rVRCVs!ZI*nLJ{|UPVzYZ^+(CzmSsffR%JlXww zn(pD+S1iU&NySMcWMV5}ARX)1JnJHiDK2T*u+Fw7ZMR{_aK(4zeU=mS#VlL37^L(q zRZ#8oD`I7t+;OA3xenAD$V>q^C}))7Ca_uTX-17wfVE+~cA1mq1S9!{!Uxh`osJ5e zt)#C0bmEjT${-bYXvqw2=niE6fMXIv=K2mlMGc1r0dfHgbvYABfW*g?$m_;TqtSHK zk{d!tedPkgWi%&#xl?{z5Tymn*>f9UQi}NhoVd*nqcvDrS?)MJN)icVCyVy0DydpomC#e%>J)te zc?9)XGU5J+3mkVLc0;*$$~u9@Go-=R zu~PV5$>Of~a}TES)wqnZ&z(NC5na6aKeoz@Tz!lgYO~0HNcWuf!Zyqy!)suv^Ienq z)G^M~xf|r^I_E|v23_1#6P;zVIm_|ji@Ro@9|q^Y zb|H$z&!qnF)J0x3%1_NM6qfG_tK*=3!RJ<|Td9t_)h&^2cemPk3npuBnpPRPXlzZI z8-~te!zw{<*t*U;0UEsh-Y*?#e-OHDF$*qQY#DG$;k`cCe%wzswM7LJ!{_8{DymwR z-}LZPY0TXY%glE`uo2{W;-FiA&DIB+%=hJ96=56*L#D1=ozVu1OXo|~ zA%sx21K?aI-z4nH_ejV3>}xh+K}YKY{gqV-X9u3Yg8q8%Ucog(ogO$S2ms!6Py}U1 zs0iW{8`V^x8ovB?A*qsnj_kZTgohsff}9vl5H;kT3qYPLVpmtX{B=iUB!ji(=I_ zy!zzu;P^W&2gd<vi}w{Sz#maZ`gM7>C)f5s=pcj zTmL36cqJQx=}|&n#TR7FpZ5bnr9pcncg1u)z&9Aj5yo>Q(XIfs|Mp>q0!|xSTGnZ| zD*9X5R>gF1Z4u8+hxXbFcL!EH{3|YKf7rp*hCab2D7$m$z6UlmMQvfG#ZhYv2d=2| ze|QEce=+?8pAA3_>&cJ17xt8m2N!#K<$go{q^<5d9F~?_ z|6>LfzlLuJkCE6TIL%>FZsM0^I;qvn3*Dmb1+P}#?K;k@+3>2?{~VKUD!&ce zHTX9^Rr>pqDaTv>XjIxsMJaBNT6@(kduLQto&Q>6;1E+?vPSb%QNhR;4?@?@W&wKx+psW%b#)~xKr&<^yOFV5W1;^|Nr z5gTo2hb*Kz#lqIM=is%UKW3~kHombAA6@?3#5L!$DrDe)`En{aS%+?P-`}-hlb(qC zXv=j@{?O7qeXcNigpJj{8i0d)$IVyKJe6>N(SUip4h0JhnwunhGuZvBwmf?9;1)E+ zkzxCnF23Ug60+jQugUcebk=Z}P8zrmaVT1H@UWtXV>%Pg+CRs}>bhz6l5W4ZDr`Gclz1 zFd))8BanwAhNDd5EVw5Fw_}}9&ABi`-EkS72XYh;d6Es)p^K)~Pmh2EU#R9itx=V0Ds7H_gfX^z9(am7Kxc~ z{M1}5`N;6BbIs!Gz8I<{q{JAmYU3hNzP!J`+tRUd!8!a;(-|$#RpO@MndR?#!*uof zZ*ijoF1ppI7IYl^0Efh;Fg`J-2rnMH6a5VyKG5h@{JyY!476_@!}=B0=COwsO8;f- zt?};@a;1D#o5wiJJoa?IZ}m}UT<_0neZjCKFhaUI%xllse@$)!M{I3T@R>8`NnAp~ z4+B>aYh-22fBql3*fyjGr<>8jP9^J19qMg<`dv)=TFhQT$Q{#Ld1;1s-nYxv*evmQ zx^a{*1&ws@p29v+xbxT^U_HP+$UBQ0Mak@b8X6sdxo!b6>i2zGcOxk?_$sFzTIeSf zLTC^$VGwFiQ81S|PFiuDS)DW#jc-DV>FIi)oD|1&u?uJANvmQ{|#NZ14Wgr$G zclA{`iM=zkw`vL=Pt{b|H90PV4YE9}Uh3#?qd_PBXM%qPf8*b(JN3mfniPHdc>zx_ zzA+hio=0eV)74EIH;xvc&b52peVgzSpdfLGLPI@ZXi!e7&yRsv-bq<96;e9)lr*RV zj&l>>S5l)W01vZORaB+2wE?E~LFnx{0M*sXA}cbLoMU{anM9uV-WXl8_X z;xv&ngI`xetdu`K{w4uSvZY`j=48?A$lkqM@XlIg(xQCf(IWZSH+1-xlh};;c8fUJ z%zOcsq4=aGPu4ioi~u0M$HMFkvWJYB}9M9&1UT(CGpR!Dq`K3pD=5MPIkp* z;5m(M-SBjO5!lQU!>78jt5M7frvFoc<<`})S>BopEw2~YMi^=-jm<6F((#*D!kG4( z`NpB$-~Y{|8=irD#i{hV9B-X;a+ORhIzvLS#N2D%gA+q4cy(7_L(MuHY=w6#$mjJw z+l7Y2^8=9Q;CiW`*RM0~|-G6Uthbm?WULKb=Wh*j?0sjT()h*z=LGGxy{Zcly>{G;+E$R|BCTUJ3HICk23D@k5KMeyxS8;;*1; z5yg5iUgWb|wBC`WCUUDCB26-AY7ohDHt*B(lL4=BWAOO+@nhZCqIWZAW4WN{to2R! zb}KcD<8MZos-Ac9FfrbzYS~tS5xjoM*Jh~ z6-2{?7%N78jo42ES|?mi^Ne$Oviu5yn^Lz$ri#Q3B7Wl(ulmBgI^+--7YU$6rO7&hv%UxmSiZ?9JaPna{^9VErO3-}P1e^Hjx1p@> z>nP_=ryBqhpXo^a=a>LHI4+|dkW^J>`qfrj7ew>$v-?wYw{82)sRhO_K8qjA>I}Qd z0`dag@n~YYMOvw@e{f;X0oqhC5H0tw-q&PgtQq8UXVsHP3!4QC(r;P&z6o36#;!ap zC^R1PWs%(m*h`j;#2JG$gL?7bJFF5$hJo2wr$jiqxZd+-rt&uYK4#7Ggqq-@y0O!K z660ik#}!ZQ$R{|Yo*y3vAX5^ng(gqG{k3Y`vR4bvZrjr>OhsqOsy4e8)E`!Ie}AE~ z!WiZLESuY#b6pfyP@IZDpH{s`Y<*ib@DbqnKxiltaRvc!BL*SEqi6Y?P5&wl$?|$e zs;a7@`wBvg5YL6<4k|Jnf{3P;=@cGC>AaF#DaBTYW3B&~+6y(j5DyqtyfgS*@QN2V zLt9H%S!6&*fYvwk($Jg@2o*pPhXNM++5V!Xv|CGpm&B!p`=wn|w>MY8FL(!@g<3@| z$k)fGCP=5AWoX%|TmuXEAtEfpOx|=hL>l;10+V)cV}M`zT=B(ujo}B4bUjgT z>>fD#CqYn5d&yHPTy2RCOWa;SA(G>BNxi@(1Km@AaPHV;5kw?jxU7wJ zXyFJc&5yL!u(khpdJ*xG2sgflap|1%IMylyaf)P@1e+Y}_X|RQ#^sgdarYPrQ`%9Ib9pCx5m6G%Jex2dy);_#|2?{N2ZIcwadV3SOP($ z>v?7!3qf(ZT`@Ia0U=n{(je#7ft|)G0`h+OaaSxzxU?H^tn#j|oZ5Adq(n(}7qZ+b z#wD}3=7h}$o`o--fu#z@62kgrQnuJJ(Rf22HYQXR1Zu^4el)Q%p^fYl0ZYP3nd^>C zO0~#?60q|6uuo%=L`KKN06U0BzoDV}FTYrHd^?$}U~xHX-@S1yu7?`0(%)$B6kq0U z0Armc_f3&!3)xYQfPp#E6ge?8@(SkzfC#VG@7;Xwk-I_Ol0}73wh!yF=q9kQ4_vC; zsHz)5_Y#%J8B41PFbqzuF~;IIiCd=U&%q>VS?#iWFqIGq{%ILHdg;=$IpGx09hNYwyliq3 z%$nu`;O2tcxh1lcliLoRvES)uTmJ7Xo$0|@u16NGX>I73`7?_nmnFyckKAFM?bD6k9njjqEv7T*nd zZq8K-YGC^^sBY{FeQ@c@rSVKDTlao*9Au4&E zvFScdVR}wz)V3!JRN^ITjJsKytJv``r*gbK!2L}7@XQK`Jp8!B`_!~hs_0ph$z z?l77~X;mpNCc6r5EJg5of)aL^t7fUbV|0 zYrOlg!PS>^K!VUzWO@E^{b9B(X;5Hoi`wQj-(7{=WyZw?O$Y%SY0s0TYU%I`*rreS zHg9ulKuN{DE+(eQM|QghZ50xX_#k9^qkjReQ1Bdo`C2ke5(9BYg$!)ljt0`qEK|H@ zDZbV2n$>Y&G>_`eQRFvjHP{4KylxG<*FaDFq?}^?j2isQOQ&U+TRbKP|C;GPcKWYD zBvRRlr`e~%NbbV^BFNA_d+NEgE*efka7>O;zg@kCq__HCE;{YoKhaToe}GCLl$UGk zpBligV@#ZiB_1c}xx^Y{i3DBcDji$r&}lvQVUS6JVM8|fIfpmC-~D6(cFJ{`&8v$D zYJiTKntwf5t^eRqLz-oAkBVC0y9*=$f1!=MQ8y(9r379W{RhM^NRJ)xyBCt_#ED)( z>e#hGu*N)f?B2)M0gBh9uE(~pov8tV(2ZL4-Z<`L!NUxECFdb)3COLWagNBp*EU%t z$wZt}f$T?PfE=vL7=c&K^;@gIBweg$RGzmvxC-agck+Sf>0|O21B=|E0<75@QeG3k zK<#nMnx2g^{j=fqAF*RK3XFvR`?>kHF*>Q&6-E_KEe<_v_w|V4e-})7q;=rdQhPxH zf|8yu**oKvjzxBBlfM8GQBWI=Y&5&Z%Gx=?5N1xQOJNhJc|j?`)3qiX>c7=(n(0M6 zcw}D(uh6st^QjTGw%ICA#8#Z+1hVYZF86g zVmT?a%*knYY>bV+@-_)Fg~{TTyYaP{N)+udP9qZgqe%;4tIkaaGyk7?-<^gHyPfv! zo76;JxHJoB(*rXStg+vu-KurWEEuEAZ8GZqGxFOHmih9_n{H>G5Ld0hPk}x%#B%}N zh5Eq#&e&V#lZi2I*Y>XqFU-h;TY5XKG*|5#m#H;QV%76LwYHHWuc%v>E^`4p6hlm{ zn>9{!(~bJ~SB4jQc?*$Ej}}}!&SDjkv;7zrmB$zTo_0_0JgU#6!0HsSyIvt+6Ro9=)-8$-jD|XF_o3AZ-;(dhvlRePJU)OfSe_dz< zzB3o|*FGx;eXY>B@>Nlrf}TRc`9T!u4BEU=_4+>u~p{jLZwcfHV5p| z5Fa+t#%9^-2Z_vznIEgwW01Y~?%G&YzRo;qg8^&j_o5;xNZ10}&z%2c(a;;VI^1*Zb|a zcW}PizhC9JIdhP6ac?q_KWC2~IWjHeN{a1Dzk6rcJ-%t%viti%d+2^pk?d$Xx4xOq z=Y@@BurXbtsp5NQcAmWT#v6j3f4v#C`k}fjb|ePRW1WpV4mjqxbio0X88RPyXZ(P< z@O6ah&`gdw@4)hbeVO!{abH-p8}Y;pLn`6IAKCG zZ3l^!UH_-@1hW8PrtbkmV0IYexVZI@b7@WBro4JCt_)t@a)lCrmtJOhA^IOVE^HeI zW>oHYW8;)nY~UDU!@rBd5eC!AM*M88%8Q1s+=zRzCs(Qb}s4SWc5hJYt&}E_BtX7N55k@ z?v>MQWNkPvEO(wpEPyCbnS^`{t;G0S)I@@pwgYd$cCZ%GiHEK|9I8aI086+ z(Ej)d1Luabt}M4Nf(@PrA?flhquL^I7Y@EHK5xtOl2pLm1g)OEo({vP;**JCWW5KO zD>X6}wA>MI?;Ul}-Q{WXv%{T!G!JSSF;L-e_HpdLA+&Z!d;UFd-3G;4(>F3pWz(~P z;f}q3!%2r$(j!Q>I&kRSaaTo-+QkK4Z=STTUAOT~+8~G3^>k)|mE0wexySd{35x~@ zE1|iX+V|)}YzO86Um;yP6_ZalHUZ~@!AHNI{(ay;XfcqGWOoK5*ue6HS+KSQW)OG^ zKjg|U9zoMs3X!y{$8vcK6sjxvOop-;W_C@s@Du!tJ?bv%8aRtc8CfTK&HRc*0jn4+ zPQ>MeqgN2NC0ZbY|38O4b9Jwp0o&t6vopY{dZ>Qp@VcUmV%8UIMwA?Hh&rsN7yXELq zn}9a^vSfg-*?RXna{tNKZgxtA{#CtSpI8QRr*QRYsUuf4%ZlF^859|09Gz`&CS~FY zGo*3Te-#ndRRdHTxQ3giI>Hs0js?qaEDIY6=l6%xM6!Lz$-3sM!@pCu7oHO&ZZor1 z_)gcyoZ^0tfI|#EYqDQ@Ecq6R(>k8)zp;T?*%o#Rv(2{Dh^+4Qe|_q2La)*ONK4efc+6I{%1ft_qYYc?Tzgl-28eFG?3VK8aQzJR2%{1?Wc&r?aakJ zHCn>$oKn?lp=cBdqVorBIT9j&I+YoL8wq;x~p>j#G16oc~~}bZ%yu%hvn;-?lf5F3i96$Ne&OVRW$B z_i5lY=z>&WclW$6MGceBp4o%XiRr~e!>L=_v=_gtXb!y>SI~8ZK`dz@9j?90N6#Q- z@XfCe$tGyZIwEbN`pcyiH2ZjQxCJff*BZv_rn-H8y>VsH8Y~$luQ3DUaC6-zxA6_6 zruz+Ew|WZ1NiO>am{{dsJ*H9`8yi3YXUtSDKQ!EUVpp0%-~kv=>M?BFx6>a9zcWTg zdvhU8l$X13Soi_`_{6z$X}QzIi~R0g0tEyiK+(rl-9ULyH$s}kk#)tf9jItz@t>al zk1ddK3BU|nK3e~4wt%n>e2Yp#s4QbiTV13HK)Y+hF=<&d%$<#vssUaU8Zcm6_dCXe#K6UcQ7Md z(NRLBRi5mr77behhVb$qqlq6=h4`JWevtyIc^Z!V5ySOrj5^-4XSKuCLr7)RxB3km z7MR}N|L9)^5HpxP`>oELfZtha8!g#vVbWknR7sq{+7c)j(tD6Kxy~?k*{?8?2?qIH zHe8AQi9d*uqPY)I7CH)ZPA$X@0*{U48doHnm2@)I08aDSTx4R|VmliwKH=W}O`UpLa6lk3t9OjJFF^}bkaugNK{x=@lDXF7Xf zYF^dRo=<`j+YD8Ub5x7XNs>I>KiA`BS26p{ymogEWkr^G*zdHpR?&F$p)u&0xHD;`IZUGF}!u3i|j*V>=RrX}Xs~I6%lV~7l);oL1nG+pk zDu1y`$(o7Ldqiv<=Iaj5ccB&WDSvm}%sC~I2CYMWZ+FMM@`B~)Wv5?W=^beAT@ZZOM= zIv_&=V|=c1Uc@qW?? zGmrtgo^4Qpv?*PAkYc0G?@H%_5zp>fCk_yvHXpm}4>r^#{( zEu9TrN}E%nAUht9DWZhSrVpXq*~H2xvjNnl?PF}xFVv-vze5w^IA^eu4&-*=B3;3# z{_Y@?52|vJ#rP8dSwDBNo&gmv8!lbTO)Sdy3AZhMe}`U`r*_FK^kD$aqhhwk9jx z^EMBB%x1ZD4&-kvXVbhZT7X%6ksj{l1Uo(w_YBUaojN()15+GE+=C)nzm_bI?kvCJ zS;%BhjEosS>)Y+XqpdLW$8N^oqTsPpVR*x)O@c)IN6 z4x>ZU&CDJiT}#{YL(&hok$UcF;K6aNv{ZO2NJ$$t{o+jF1m|6T3anyBeoG6%@Nlil2`Y#EtvD?q*N%S?29u?*BwwC3wU>l+@*q z>&(P&fi#Lcd<#}H7V`P;B3G3;3V)67-S1?6;c1i1syF^2DDC*{#1dkGz4{{gT=83= z*EZyHy}mAnU*qpfyYtxGjdW;h{6u7NF-thl6cjS=yNy^-Ku1B-cALdha)RI(Ab~?q zPJXcK&^o5?W*pOf(R{~w_u8%)8u&uT!Id|YCcU$P!E9xJx<&$~UcRe1T`5E=+xVVi zKSQMY^VXc87lu@1p3D8w8;XLhP{c$~6|B7n3dPv>Qk3-9<{1oNP$8&g4l2UaVh2^5qYmKKjJX3vOKB}gO$QAlw%=__lZ&IQmA+<&za&t&-#FX63 zk*IMBvRtWO!`##gq=-H#kEdhJb z^n2&P9}rS0`aSR8VSxsG?}>yNm&4JMCQXVg7Fd`=0vhptZmTf-*u=^T9&!PXK2Xm>t?H7nITxXKo&=v-}%&5Ty?E71dizb6t6kaqd|;pbCFjE_d2wDj~+@{ zp#R43SZp2>qDW?@Po@P6NpW)$=4uM$G}fx_J9pwb-J-31*^eL2omR69YgO%>ssO;V zb@lHqxlxrom0Oj{!jA`4&b~QCPMn{MmJr6Pzp`z(KgIY=PncT>ygFNtdOxwo2LG?I zmvGM}yOvGC{Kp(*HS*R+LNBMsOkARYkwUeY)Z6vWoI@Hr__q)UqXgKDv#r7Q-=Zi=cT=68@G; z3gJtL)A0e*XvyVgC1%hbPf+=+Uj~_eKK*V!V5#gz6vD&xT)eFfoYn-dZ@&sug6;dw zJ}I=~kNu50sCkXrb~YmB!zyvizUo|ZPBR4yKt{#rUaq;q=^+mQNqp-mr>DZ?83Y*r zhQqUg>mDoHfe(t-FuB}fo0oIRn;Ir|;t)Qx&HQr;c6Kn@iDA7*Ay@Z1`sR=q>M0VI z!hC3V?&bI6KQ?tp+i<#KPzj&!Wek=YnZ%Y36#ykZY0o(qluai2GH75UC_4~I4 zowlE`SQSV}=e#XtJ@VJaX1+)!CD^4rKZXmUxitqrhLP4R_y;kMh3nh(aBSO#v<@V{<@z_SkqpY z3EoVOa81VRQS0@RuRkTd5VB+nsISUc26;smgDp*h{^FI?85GKjrW83i+s_1jj7 zGYEksJQqZyezO&ZkrsaFm4xX45*YjCw7BpKt0&9XDT&Zv5$N#ggomKTX$n7 zA|UcayR6CjO%P?$dd*`_{Jn>%ve7s zWx@qay3uKd@xB&UWy6j2 zoKpns4_Os^qWCPhJzS%pJ5q4kyW4zP2Itmgzic4xqj)PQYLAB1O1*Y?5o|*5lC*Rb zUy)7A@r@_Hya}2yum-fin0uwxZ!@qNq&LxQ$r>X8hN?E+!{$-`HQkzV0Z;lJNHNXi z8WPKH=2yiT$Ua9Ld|Tb6Lrr&IQh&|JAcX`N{lnO^EhQy)bcgj~F|4i2;J2f-_Ha2j z_uEo+Tm^DEA~YF@M7y;qm9L3fQ`dLs_;e3J)SOr=J(JnA?;m`7p{{a%k%^KMR#FY1jW`i_&8CFP7^J*K3DR+3;@oudV}lyu1#iWwVYWI_Gsi?!1G_WT)kFT>5vecxATn{3*&so%Sqzsj6%^C{b>MOFk2-Rs_fwu)vvU>=YB|g>_d5D%{`tYL(vRLh{LQw`shB#) zN1R_#6{(>6gxQe#TbjL3PEb(OxI3Bf~Dy(lIPoWAYq zaWNt5dhU%ofC$l!7~gCt*yEp9=m^_mIqlt5hx`D&TzOWpqOs=5`-b){9kNuqi?k-l z%zzH!Uhr~|gK0Ji=M-hnzGyaum-*`8qeTO514Z`ky(2V=T~7}>Iha*V@x43byd5O> z(b3{YJz6y6Di)aF<&nf4bbW;-U=fNhQx)-im9#l!308@NzI3R+#)>jD(QvJ;#>|uL z7Hrvfxv4Hs<5*#*I!#u=X{_^U&pfk)iIdr zGJbzvJ84pEO{qlag}kA!vwA4Gb_D_h;)Nk1%vDuQJmdH1RA8VOyP`x~Ijb%9?4x$J zOA4sc=e|PO(?D4Hs5mp0p-;pM;2FHA4hs`k(Ts3Wra?AYdCg*Z4ggj(7T;VLHuoo4 z()}qk2{UK*@WAb54lG4fFME6Zp6gIvs*MC9BjJt>t)}H5tJ{S8QEq5U_C^OzJiLug zQP;us%n;0VB#mkEXaj6Xm>?1t`wI&P*@$u$z`RuYl^Dy3~9(u zMP~N5=n>>IqzS23fmT~ObUnNY9AT0WeYo8e@KMN=xw``IO(W5mb?w4EsEW|=xK8|Fi{2Y^)P9z7a3MW*!fPHZO`sO)Nm&G7q*7sSw`Edbjt z?Z=gE%TVve)T!{XaH4MWzMIe2*5k1m%ll5MDAaZg55(%?*s)`l&f*el(;o?bCvF;0 z-ags4W2(`EmVkrHmq=%BxL%OS9{<1xFaHpYFs)aoLA4UFzG98NY4QWlcOF5Gr;&(q z-lg!5wK)EVg!)N(nZ8F@fB9sIitBO#cOm|UXu_c~340Kvh<*=50f<9fW|_p+ADsE9xHR7ypeH`#+TWBN$!me- zp8F%(WVp`F?){aFdV&-gaI?nRCFbs{~+c!-6H0hS2%lS4uczKI}tKn%QU$ahV znHd=U#^BBe&JI+YyaQ3?b&$Q!>kU!v_XUrGiq!G=!y60-IiF~1pLmb*mOAvY(y$s- zy-wBi5RILJEqHvZ5a(duz_)K=vcfLA$Vo~IZO7vHdR@$Wx56HR(pE1vVj{`1+BAjY zgTP_(ndRs>i1xxws0|h*?aDRvDv!t3-7gpw7E&|%w)^+*4_&+Va!lePt@Pa7W;mwq zWS+!iOlSB#-hrPvfkm%&9fJfW-pQzpZSL=^dHNzg)Yv*}>ar8TgTl&h3y-8o9YKl| zVVbX5o@rB5%w{`hG-7O)@D_%d))Lu;0JVp;8R*#`vK~hg*yV!cP?VqDI;_zVi5!jG{O*sphZ7CsnJ*-E)`~yHwkZ32Ta7{xt z6AU9WPMzflb*ZjKwOLWH*seZGTefWjz?K7j)`RTfjI5zvXL*k$M1}J%P^4rMz-)W4 zSB}!|6@%I}m3e}p1FCf9>J`CpizHl@^F(DW?LH!vC{4*0k^JDx;yen;O9SW8ZI9m{ zU}C2clU^<(eE9L^&OOoQ3Y2}CCWL0Jq_m=bEfxwf^n!;!l$~z^RdEwiY%a{9%86JK zlzli5M=_Hz3DY##sMC}&$BJg|iCEQS>66&88#_3ui<2yUT9Xw2>`ZaTxUv4aUgDww zHMvo<1J_p)oiMI)q7VjHfS!4w&YN|hd`AcD#;Bt+0}a%AGGM^XNVgz_jUAEtiY0xo zSZAH+zc(o<*e9s(rUBy~FY2)6)ljdW)o4b;+m^brh#XfrT{r+4GCy0v1@!F%I!Y_lu-TrUDOd0$bT=7iL?d zrVvL1exmql3U9&$97+VkU!R-_dci#00rr9@f*jg%$p}$|&Nr{Hu+VNB669)1gs6S- z4SN#~-XwW(Y`X@l%8o*bV=fvm&K3JzD*Ki@X+edCRBZ}9P2MK@yroN*?r{y7cQHw* z3oKagEW>hTo>&Ewg8AM|ebz-tzLp_`0b)o~G%o50+Q21IoCbNB54d`gjY*4JB# zM~TP+n;l+mqth(^IiC+tZ_@kJw%s%W7J|yMp6w)9(P4P}xXAU9!xr+UW~Q_gTq&aX z^^*P|>nUc)>vTUpeDGixaa3?&(^lz}cbtR6%;sAicXoRc?x3g3PRfQT_=`y#ueYO~ zlNP;ktP{zrppDS zZX*8Abe_Xis$ycR;nx@=iFjXk_)!LJjihNH~imNvRVUH^`dxVxVVjX8E`6^He1sUMvI~@W@SCvNHTa zp7BQpf4K<049#}t5n6_}pw3`&^i7mmU%dkv&Km#w2*pN3#;4u(`?ku(%KVGxH+SF2 zr?<=}cr@B#`Q-OkkXpEU_Y8OVzR!?n5rRx&(d}`c$7vlRw zAsTqLg*9;CNQ8k$i=j~h2CQ0A5MUO3f%qR72aa8+NHSfph236IP^G^P0;Z7B@b%7# zk68uk#x_ifd$w)K?RK0O*1B`qJU+SMaJ7>~UsdUjxQNDATKcuZR7ZXlEy^D~XPzJS z^0{Esd~WJmLu+l&du&6iMW_7Djz`h>}F+rlu^Stw-&O)t#Kf2>PQUlegs=D30 zd2`hLs~e!GZs9{ZE_BE~Y}I)CM$)y+zb{2jzSJJk_~-GMjhlyn_AZCj<{>xuTqXnZ zX>e_~yAmcD=0M3I;a^W(Kn|298m{+$1PYni(9p6PtCQ5WB9jzpm&BkwO90Zo3i#}E`?ZcH?2=5-+oSff@a=|( zgyFMhVdSuVNV}%jUTo-ea&m)6ORc6yCzT{$BLK%N_(QI#Jm-Ab7E9RA*eS6;f&3jo zozx9o+}Wdyp9dZ&3=cRKHV@&wd~{pAzM!BW8n8P|Ddmh*q&+W3(CbmA3N9WQEJ3Vr zAd`KQ+}zxR0Wd(hJFe;OmcmHn+a{(f$#JhQk zxUN&+lT_dJ2m`cp`nHJQywP>7CH_y_!GJS(pO-IT2SBA6MT0UcKIb-=jzn6aG*~(7 zFn&4=Wb^Mo9Kx(AtVf31vFTVm-2zr@TfX8Ux*QBO7kmEk!RE(04U_Vz4R@?i-I$WLAtbvxG-J0#i`y>(%@o*a-0Ckn2hzq3+n2l; zm(Lj$8s+_xUY7UW`AR$mEhD-=Wmk{~*{dC+hy82(F$ADwVYNFN-+H*K+fjRkz90X- z*Nq!N;TXCBN#5msGB`JS^k{43^&yq1lQI6I+w@#Hw3bN;RQ4R>_L5iQ}Gh^#-Y-x(4lNW4fNaybLUZ_YD|o_Rn3`%UUiS zKL$*shV*k-61sObGD^*8v&H^oyls~v?7epDD8>FkD27-EbnZMtS=6~$1EPc0m?D4L zc;cRp^&Lv{I$@E780bq@!nd9+8cHfLLI_NCtuF?(SL_15y#8Qz=OkQ8d6vJvoptK5 z?+u|JK_O)za9VD%mBk^Zl_zJq7!8=uvnk|+H6M4yB<`+(8q`k4j+sJ)?gi3F6b-~O z7EG;mY^PqVxWOL36jA|>g&jSFRSU<-Hm{O#PPY5g;$598#M6S-4o5tGVX8CcC&sq6 zXY?cig=hh*z}&Zd0_AY5e4-H_TXpOnRBUk1)-shKo;-dwJfWmSTz>y`gc7oE%#WY+ z`BC?nipjB2-tOkXlWwU^=wC-;%n6?%E#-*{FdNayA;YG6k7Sl!_-5q&FrA7K>sYk~ zFFM-r(RH^6M@JoP*`mcey8l2;ZUH^AQX`%^(*enR7;=-IzgcKxu^Lt4#W4#HmoFd) zoPmRyB6k-ap_f_jb!e~z$md>CPsE%fN@166spkOtM~fR;XXYCH19 z70JdtKu!Rj3vYaZirF>ngkqrkIzs_l((RYqs_Xqyp6B?D&vHHm=>8CnvUpovZ?7h5 z|K8@8>SLO7>aqD{;Zt#)`Md(U^JKsV7-RM3#Dhi9g_c02vVvuDmldpSBra%T9;0Z7 zg?OB#{_6zel&qC{Lc2veBTsW?=52g&7hH6lUj-z6rR!{F?eal9vW9pbu#JJwde$sB zCT(%Zf)PhfjX~s!1-rt9-J>dZaitCaSY^{1^gCcwA-6#Th;8XEEN;Y3Q%)6RLt!2< zhWeBWnShHn4kxQ>*t@%gg2GDoHwPd~Pu;wcx z&o?|hvt`h#zgy~uE#J_y&b=#*hUTt}GW%iKPr5lNLKjoIDCi6t4cx}x^bc)uvnc(r zvXZGmk6+3ItbTj>Ftg-%8p{Ve+)V7$aqbYeeRNj#DZ~$@!~c;=)5Q)wvVd5F2Z!G( zQ!d-il31F7zB_!GBWQ^$*Vc9{uz|5~B3`xU`wt(m{AdY{ zaed}8F{Xw_)CmIXIu9k&%@D;{W!IwIrX;Gu0X@`z$u;_mN)%dmSdgOE*(M2u{TE<5 z-Ncl8-KadRnNMvK3>CI(#CEdkZme1M`PK(B9BY8z z(bASWN-RFCTq|_GrbcObh-3#M5jAkjghTS|=orCOB!}-F1siYm%H%g^;%z0*o#Lc8 z9Q|Dk8BWk%wL%=UX4Vq%sE3G6@7JdPk`&2T@%7zKt71^7E80rw_@*5H6GFFmIbpPX zE8{z%fBit8kTZ8!*tHqYGmJZy717H=n1jIf)a9@?a% z@12*G6?{OE9x_F}$?l(whN(R-05T8GonrPS`o@k`$y)YChkb)|KDIb}OLI!cO#?zQ z97SA>W8jJA5vEDiRwWJf&z-lfc&mKvMMn+We9IiQO}f84$(^?wa!*ETRpk{;64(Yb zZl`zN@&_1>^s$&>L~5!t-*Bk%hg;j4VP31CF%`C>$AngPBCe7BMJ{Sq*t8Z779#J1 z6@UBG*jh5d&{=mJGUQE$zzO5EBDb);g(X#{H$>x1AKu0f4_nHa(mniSnWWh>UP-O@ zj2bFb74y1KRbcp`P~!)R@nhlP;ZRZcjRWy*MMhpb`}NCU_lZ6byy-2Tbsi53t20DrmvFZzbrAEqCP9yy`qAb^(EeKP-vHq0Ztd*d_kYEXTV4NZWuyf z2eC=u1^9HG%|ea3v~(>w$rJrL*5}~k?-bjPZ@W+NniGGX8W&<#U6iWn4CSA0<*;E@ ze3|?EW+~Bam9v*!rqjfHRzsF-lHKvCl^Br7WRx@j5|Oz+yA`;u6veT#w*;YeP%@(-Dr8o&(PYmR9%_X6gOm^V)YP9nAe zUrsbWi5!HdVhihuD`5vLooL1_I;2krDcH~z2Pk!UNQNLOm~1<%_O%9u)6C4ua)X#Qu(aqb zx+W-|nF(3|se?We54E}Y)azi!{UZT0^Dw)LUo6qU@3^`DH3A9F$_1kJlr_EVW|)(< z;f@RI^lv>j{N%3Bk1WO^Be7BN%8zz+mNvb-H85dVpxK|Q59p!c+xEO zG1C=m+V)s@8xvI>^M-EE8UnQlY(lR3y6a1x4P(h$zajegpl^mzB#S0J1$2~acg+F= zaEZ=?-vBYFXV2`Cl=T=_)%17$5Ba@WnGIH@pf-#j|2Bc|v>zXnq~%#U{#?iw?&oG? zQX3~d;0OzgBfpS3&Su&}RyKG$AmIohJr{;cLJ26xPqnKYl_! zH7kqmvXKqJ?ya<{GOEht`OvQ{sfG)s=<#;!;*ge`oA+G&bRXPpNU07P4&9(+BN?gQ zxXj!APkKE37;n4w%d3W!?4vVD|OJW?2)J_V9 zJ!aIJHES+@PmYGi#ycYm8BLi~8Gcb zzT4a^`{l(<`dYbBqRY4qAwgd?PmTJ|nkng+!Lp{RI~UHf>k*~Aw5ds-eZLNM`O9e3 z<+wXmZ^M5$NqJJzsnhUQ+jy8le1+UpU)|I*IIGKm7kztS4y?ge&&Hoc5)Y)PM z)E;J+r?v0>g7jY$cf22?r|rwix_SKg@zzEg^g&jWY#SPS`1XV+#xpM&#mnnFsP&E; zN!q|po&Kz{1cnT{%Qz3qMV!a0&lgLY$YR34B)xbvXN(43(u9TP5D~lam)eqNu_)~I zHkUSv?~)_2LkJgmgVm##YP|$uF8(dIVi&jvjf8V5mZoxrc`O;A)_cbJTD-~odtBVV z7{RdQr)0r@xVfoLBN(Z4gz0!{$n&7bCb2X=bi>`mhQ`MI%McBB$xS0PxCIN`y&x90 z2=#~1>fFt=F!8lF$ZQdOgHR8=)VfRooQIF} zcKK*-4>*8-^$lRYyX9{&SrQooH)16VPrH{GWRtudpHa} zoCC3c#rP@u;6SxWtaHTPz~@xwo6WW^6$^FfSlN*ScWZ%9i`@&P03V0hjl9+1_Q_4f zNHSvN$gOW!%iTOSY_s&kx(Dj9E%`@&QYV4S_0EvN37tJnAW&^Zp!v#P$aHEb(+wA{eD z$ITNAoWV%}GhAoyEra$IHC;MU2?#?a+kbvS0ZcKWJec?}W zv86aSV=*R!&&%-juzG=DRI^;FwzJ!T%+f;aImQVX#|;%Yj+@UO0TcaEN{8rND!$%G94_pE1SXcR6C31(~ z?l7qrH|OqVd%^oyQA;KOuNf?XTp7Ce=AfRIk;|_Db|c)r%R85x02x@;%zn+jN-QZK z#Yl7W*MJDo(b02Av=zTAyS04)qIJm-?0L;F%LCHKW9%?^g}XZap6U$42YF;8hTyUK z-MiO^)-0(r6As1u)A7(%CqLcyv4%Kz?8f+7=W8}U%H#12*xO@@+9j>{Nl#)gU=_QG z$%x*~#IVfxxr%US7{)`2Dzsp80$O>jX3w-P#W8TwpaI6b9G=*_*Q$|($jedaZ_H+F z?d;uaj{SfNHJgB}xZswp-|E&{f4f4hybz)=UlHRw4RGsarPvtRWUg#bTDb35hrS(H z+O3$}bEDNauZ`wO`bIM%HdGAev(fO7T$(0*H~pQoRQfrtCQbD=Z9Hh*E|k^RX>2_+85e+s=S^vni)03}en)zkkXUvl%nog(Jv}32%WXQ5yvx zhZGV}KteDd3QO47@#qbqz!;Lqu{~EhTY*X6& zZR;qvPu%DZV^d=Ms^O5{vNN4(1rm#?*l%*`hDE{3P%hW{ty#&s%GsJd=^*u@+ zrQK7h4=iA+d2@eEn->2r68<;9vFv~#`(TvEM~GrvU3okZKx0OGw6vlf0yS9p^zfay zoz*_t{nrdWI_vW%oejl`OHAMbHtKZit+L7|{9VS1IT{;DO)pPz9uR}Ze*OLso=^2) z2$IXdIVM{pVwymaof3xYw-HsMxeDb+6k0%2co-Z4D8GuCBTLkbszCl`@G1I>nM$)`8(ktD)d)MbG$f@wRL4h2VmuzH5d6p+IEo(>+dEO`D*3vR1#24?A?;;Zz#D zZ&lz3d4r7AyKl){*;1|-rQMI!r@y*Tx%`2GhH9B_p4j|SgJ<`Tj*3J$$ziK9HZ|Sd zItN#8aH!|+A`yLp_vV=OR%zefOFChv$Q!r3hrO5hqg%9SeLTRUBqHvcqjd>1(a*2L z8nFXWPn4u1JU2RZ?|whfQKv>^vb){G$^rkQF<~v9XSdckWHUE_T}EAT>sf4Gh%eVo zK@~V238#+2z2BtFpCEEuswi*cbBA4AutCr+Hm=RwSi-R~ZB2)JGa^>d%6 z0>|S@C4wT`NJZll={)_2cbnhqN=zutSAZ%-Lq2VVc!=Zr?O`R5`12!hP0`%=cs1vh zXA@@u(~BAA7KHY8I*9;^zIW_sy;@mR4CzZRJ6n4`#46G&~1SG)f0NQhyp>_O&*f+LqO;P=9A#oK{)EC#iF;n~qX7S6h@zcjx7sPb9KuGiKr}cWGJ(b-}tM zqM!`@A_S1TK%1J9NDNXEAQ;vL#dhON6pysDJF^0`A|0>W)j!H~iKfIfJM@+_HN9%vowVlG z8#ZhZKm>!D1dfhCvX$Qgjc$Ed<#7RfkT&p2LW|VJ{0D5J+f&BNJOG}=?|tdMXD<8GNno z=3z^pw`|?|t=oH;>GfTmB;d>Yq7gFodf2ld5Ht?m?Dp zV-&S<-i@U}V~RR6!jxsUMY1zXxdYD2XiIu2MwpN{dwkuimrwasWCW;1T){{MFrH%s z^RtVNlKr60AAfG}PEGBp?OItrSf@yKOab>;@!FAsgIHU=mFI_RSBeS?N3g?}LjhTl za)gBmks6XgKZ^v&8PoyPC~A9$IJ$B1+2^!^B1T^1`+Qv<_11{`76B3P38VeDdVrAed$Bl*mefbxczKZhmvYpf)`ySQx zXQ$1ycN5M{{5pV#&Ffa(_M@&e*)_z=i8KUBM%%_hTRQ=@1EPS$UOx_C_CtUc)};HR zt?70d*qACDKiEMV0zi>Yh9fgR9hT})7ng|MgJ|&Q(6ae#3JSb?@7_G&=~I zGxb0apazNOt$JcbMFnxCGeaZ6lQ9m7hAmTiq$U|BsLmsr9I@~5Xo5*R1M1!$Jp1?P zU?v^W`;*4_T1XMi2AZ%xu|eo`asM!Ge2DP}Ue@obsW}G=jDjvy)uIKuhc9px?auwt zJx}~fbW%Y9^l(-dBkQM}L{E^DN7B8m16l&hy)bM#OgJztKgWX`W|Luo{fawOX(>nb zHVUvdw7MK&IJX*?ge#7#0Yd6US~fd6iT`Tt^eqg`a53aa%GbnORLb+q-ww7rXBM1q zR8?rC?;Pv_iJ~2iSTB!56S@i$PhWK-C$oWLc-Qm?$c8L~Jp`X~l5P=wj<^3X9{uDm zPHHSqxHlwE$?}0B0xANoE*rH@LWT>q;?{Hd`en$Pk7AKYwCR&C1_8plK|765chz+( z!H0pCJ8No22bWE`6Ms4IHdsy^4AdK;vqsbF%7^EWcPz%$N*XBHx+mt$zyEZdtZp$i zM^KTttWWh|vaeI{wn6{DEkaSsbaL9s#o+}Pe@o4$N(n1a2dHhaXSgDx?no^M|~AoRqk9e0N2ty*R7tMK&nlxGI2__%b^Scy{6hAB8*-7QF)6?hs84x7 ziO`5Bnv9WB38|DqQ4}hY%tMI?DGj0`ln6;BLnviPDJgRznTlkdr~9tW_uX^ub3Hxh zocn!Uw!Qzq^;_#*?@+%)sYCzcbgaNm!;bP#b%ubflnx#f1(4Edy@ z3w}7g$1?N^W?Iq~=QsJ%R!CEf>*_3XIWG`b*bs{;N&t4I4%W!yd>&BYUVg9skpcb! zx~Ob%11)}AoG5!zf{S#D1Jr`G2nJ_s%s;%JZco9FhIEL@)%JrXJb-=k#Er^N%m9!z zg4U5YZ-8s}Vqnv}`3SP45^?-iENCh;2dztS2=l})saPM-_jYsPPz-!tecam1Z-8Y; z5i#Rk9mT}MyC?bEC*%rJ^=dbv-G)vb@>Regm;)pYX^F#C-v};4k4C0)z<^-L;0nRmatr0J21n10?q*lO)ujyAsr@V#OH92S-}IB zv3Lh;V?EG$G&zZ$5-g+vIA&_aO(NPp$PL?=P7sg>1Kw`VI0dgdjQTqJ_AT{!)%kt} zjCCzC`W`f`fY}zwD>eQgCzeLE_&(AgS+`XySdV_$E_E1Uzog7DFmSC#25qesQG`L- zjsb?|{oh2M!TQ8WKhQX;V!*!Mnx_2SW%k@j!ir9o5f44hZ}3&fF%&i>ef->K6UWMg%Z2kG`7cNW%)4IeWfF5F8 zoN#kGIH#>VaWRd2Fce5o;L$Njb?6d@x#UV8K*zNhg03fdznF-N!PKYln8=DK=PNl} zwvCHQx(k$6GYH!RJB|}(rf&ZdWl=qGv`pzHhy32HDL0clcy z3mL4u-2UINP^W-2Ksf<@$yiUmf4>RKAfhQ;y{ePop@GF66!wVOoC-~Pd3pIXgG0DY zm|vYx@V!Adtc77ux)cVlh;h(jk`M|pjnb!{tXhtqhrFyY>*Fj{BjunY`PGQ#oZG9- zIM7%!Mc>6lF;U;U_vX3@@;u}R=!kVcdbtx=2t0*=?5!Xw-UB@MpF@hn;YiQQnX_hH z$T~C|z7Tj-Q!!cV^-`~019=*DtQSZ~MB_u1>9hO^N0NUv;f)ZvU*C8Fe;Z%sZ-`Ta z;_TZ6a1l;6EWqIkyW(;m0v_0ew~Wax^-iz-9pe3P3&d8D^3^n&pRV7EDGR4Yb~T`b zzP6e?KnHmGGSI7`_XH&nE*>Y%c49*dA@xG?{$F4FaVwNNL>&fX15yF;J;`ZAP_XrF zL5LET+O!pykoZ_Ph{+SqPvtx#{7^sNkBTB!?~@j2`txw;Ztsx@!7;+<+}iu>DR_yn zve57tm6YINgTtG|`dC!TS|}Ugi?Nl?%FsW%PqdGhv9c%`9Hq1e5WsRACahRW2S@)^ zp5P&Z55`aIZG`_@tor*}-ZXb8S;t1S?#6HOIoxHKqdGP6-lYub{Iw#(d-xW_s}@=* zMfX>Yt5qv@O%F}eXP%p@@5H9}|JpX`j&tJSZhZXDliE|v!(DDPI8nJ3xfz1xYt6tYQB*2DnTqHOWC?X+{%Q+VUIe*+>%Dz z%rbp;81Dj8$wQ6^Ko**v&prIma6ZQvyHg(cz;4dy^o6^}xV!;We^`CCBym!Eim&{K zrXQy>$fg;ahO5lZt!H9yaPnfaCr&gZlEhD5_N?&)uy*_QALrIlVW5~@ItIIqXh=e9 zmXj*Q`2n5Z7*>hQGQnIaLl!}-Yn`vUd-jOQHG)@wrLGFLtbfirdGz7=1JUvP`02QS z9I;J}I3vZo|FyNkl|xP=2jcevSNLDU9$y3sWmr|;;m>Dw4UpZ^q9eto>GHY+i`82? z1u@pyw(Q07*KlUe4z4!<)h>KP;u$00z zr(aW8QzL{mjVgGb{L&lEcR-Y)4x1PpzItneaBT_$Aaq5`b3+NC!v?6B0@EC$sjKXyLA)s9sEA8uIu)IMoSX2Qp;;9j&ofNrZP? zX^m{)gsA%t2*`C?lm>O;+r}p5PihgI+p}1cMnf|ZzzeFo_@}pUPWT!&`n)utX7;!{ z4^I;!$X4h*?#?&C`^aqL$`NsT5ewWZiJ08r1p-gwIl}5U#cK&<@O!XtIKy5f_sfi^ zekd92&~#5ty~adq%->&%y|d5u*;eK>bJxz%OSo7|%#NRZ7r5jUvVw|cY)Ltw8_&k+ z3FS>#AF-aI{vU6oNzp7>2k@ZCrIs*iMSuY?{MU<$8-<-y9d&rN&DZN|y0{Vb0geWx zob21$;f8jfB>cfOJt6$j)P=h;-N|^59S#Ouy7UI6heQx{<6G>@ zC>Z3TO6;%$cJo z0SP+DZJKXX#;xOf17s#pBFIWG&>rd9z!4`ykfUq*u?*KJJ%k!C6CRq6KCwXq->111 zOE(I@%}E8I>N?V5)YUVI>uLbH_~}@Db2ovO-v9BGE1YxiM83V|nDX$p4Mb63iQ69? z!!4MoQg#i#lKCwn=(zN`f+Khfo1T z1~s+z^{jY6bhsN3Jc?*BfNH|4S9d|hy>u>ZQOIQ-v~Z!LTmA6_VUNm*&o8jgJ}j!} z7&$hxt~qP+m{TWnuuwDwO&~kbW9-uDs`1~zv%rY;q(zGtPYH7yO|4HPtIbQY zkx+?dR9*hcBk<)zt75*7erB%<|17y*vw5SB7ri>g?1%LQJhi*Nk8^Va?X~s&PyG#g zPt1VWXZLhhZJ4%p?B^re76Xm^S%qptc^_qLc`(57gKi`e7BO?kNePjYIjYgo`QXba zW9kB}FXt9^%V6Mmf-yXQ^-uu1c&MVff;W?$b{@vTkR6$t=Wz3ah-8h0aZbJ^sXSn zczWlyc+R;ILNFe2MnYy_+H0}}0$ZNR`|KfIB$-(KOSX2HgD&U+CVnP;&k~x&IFltJ zj~-$+Nt6!CF{XP3{}y%`fV8$f*o5ugeT^Vwvi^Is%z;Jpe;Ou+m`ShMgKEQsrwu#D zLL2eF=*XPF-UnZzt1m(FlaynOdjYFO^UhI;kH5RPm?!bJ;2j`DsX;_*FL`8jBM1Pv zf2c!7Nr6uYSi*Qy7Aq=-_H0NJg%R+b^&VKw!JQy29%~>$+#IKh5`3sl06nxc{*&T{ z4g9p)L<50?@*U$2jmXIfN6|2XXtpJzFCOwDNXDAa?##jJwFp3kW83ynaQW#&KqF&p zi5)NBnq%4+RA8=4+x7wU6>$7%*Z)o{SvL@R#iWf4Z0`Oyl~djAAOm>kxS~eP=rtM<mhTq5wTZPqRgv! zWuee8__}rTg}B`>EWKNGe9bKNn)WUeT7e^p$B9Wkg0L3r3RC59ZTk^v zpY-1o6e&@11qNLRGfeQ)KtQ@sAt^&pk*E3#lfy(5#Ix!Bl%-MMhi$? z?Qq^$gvI(Cz1WglYfmi%qA8AB<}2Dvq(O*(QdL#$z`cLHM;7n)OU1cSN$lQ_m9xrxl>`kbNtIZF0QO8qx4Yk#Zn22 z2x3ha`za{&ol$Vmy+7{2qhn)eqY2}xuJIx#B1P8H%xTvzY_psKIXU#b8uP~+q%jPg z-_3^fNi@n-{;kgN#e|N%hQ?1U{Zzp@0<|Z@kh|lI0$MUcqb)?};ypribQnxUMns&< zctkbq&}#-U1i)tUzb+{IqOzNM$TyfyKzZf>)e{@>pS`0i+uyc?6Qu&^4f-$v+yP{% zhkcq@2RH^~rGv95b9ApA5VtoVq%Zb~DKSqU9c;le;Y3E?$M^5w0C1Ib5PV$LLL0M1 zOP5Ah4+WY*0YaI-*$x;?;J@$%!$;$KKKu@EagkcMjFEygu;v+-58e6uMgO}6v+T}e zasS7BCtc3X^g+qagP#(zZ=Bz@dtJF>XsDEOyz;B*ZruUhTb|r(y3MSn5#sOHFqxFDy=Tip-0-ht&i~t=JpkC)Y`@il5Fi$L1us7^W z4}8T1vjvJ&kD1ebLZ)C4L<{^AfB7Wu-N{OGGj_uC31ePE+bMj^5sq}=;F$&Poq}O` z8T8!bhLnHs7%TwD*bXaHtX^JQ%Dps@-z$LY_eAA$CIwAdS$Xjhid3L4snftkEGwRF zo^u=I8;biT)lH{A%09VmLCPJ7;0W^Z;1Uv&!l5Wc(d&S#@_{){`1eIfpkvB(1}ovH zdH?U4ai51P%oPpAo)JS`v=_As$nh@n4bkVst_7`mK9+a3FFBqnk~v{S1W8=i@aKpd zY`FW0Ef@f^js?>B#^u%u99kB5*iSzk*@Rz%d7~H(w_Ve*89jn&t;aw# z!!i*CY{D`zi7U>BT6A-H5H%r`N%$6Ai=`{TcWg?tK)nfd$W?4tHBiU1PDY~`?4MLe z1miH!Xbps`p{aK_WsqWnP*?XSWXb$50qf5F|{Okc)~$x&yY%e`OkO=tdaU(@_5_Z_v)Yd1%d8l zCR#FRS53@`rmHB!6cm|MkU9F_Gk(Rd0Jan(Gev~wB7!`8Z=Y@2R2pb?`APyA{=}8L=bodnBLMIT?JtT@o6yKM8uHin}QwgK#$*{M^ zA3);M+1XEgG1|!dZSgx|D?@EFwn)CxpwSbF4Bd{5sk&NmKUN1s^uo#`CEkDBkl~te zY$Mpo6snP$nrh4paae73*>69deHZdy!mZ2O1gNPW$+Ih&sMuGWJ0~=B)|p5ihuJpq zZFE}3O>&nW{|rjBZrg^`5P%T>#kGNiY~c>r>!|sEs~Ey97rSfhMO9od74FxMc~XK6LvpajZsIk&{Q*R55b#*^)*TjQCT=HKCkE^ z7{#`^SU;m}NU`FFr9RaV$-^(!eH+Wo=!WBKN)@1s`mEw@qfqQF^Q`6U5XBJtdb3SKOx0_WMH|aM*sMYnNS<3f(;#U zW;l*HCPydfOh5lx?*bKVwZg=|JHaiu%v*yL()361hW>glJ>TE;DkI-VQij7V;wN05 ztRn@Z&cI%m*k0(e1N+Ic1AwBs!w!*C`AExv?xEgzB{mM}Fw!C@N_ur6DVs3X5D7Z8 z`XoDSenAoY42XB(y}fS%rIG9jb8A4$@*6WJ;vdTI1n6JDz7>(WLV)4AQWnCs1>^L5ga$wYjz>04?YTeAU!s5owYrT}J-wE7td? zWlnhKGI`M*4(oB)^AWo=eu^9ygqC8(TXp=^6&xd&7W{LP2EL;5280PSQBR#`&md{R z4DZwBIxOc$D0zfy(_zGInF;qr%UecK*cM?RSu@cz#slfj!>lkT}( znU?Wpo6@|(Ipchs__^mFUQT5mDAB=sN8xR)l1$~B?RnX4!zp$jQV(@u0_bm! zO%P4YRm|=OW0TM&ovzj};LeAz*!1sBQLCnzl7N)JeQ0hIQJEnF41VgEiHpDlhv8(7 zvleC65(Sj$Fltc=3l0ESR!B0)S83A%ONVd$X;C;q-2PLeRMZ7z2%`}NuY%@uvIX}M z4qYBjF2d=KzQcis)igvL5XR4@iOO05-6s$|clF2)Yg7L`=v-hOvKa4_q~3R>uYmDX z$Je0%-Y~f3HAOLhV;Bl_Lm5$rEgX{-xI}4S2H9IdWb0dT(%6Et*JI3=^W3l z-o5O`w*NF$92iJ?N9Pq^#m>ykjfL{Fo8(&(enUP-_MI4c#5yKC&SZ|#-bzyk$P-S( zMXbp*($Oc7GBdJdwtS2asiDR(4r>3*DY9$)3YB2Wx1Wg3=T``MQ2G(e@~W45pMF6KwIG z>b02D34rU?$1W@zhU^~h?(V=Gc42ctN!~fB_<;#%`}AL#JvshWoG45G8i$m)U|Zsk zS<$BpOS<9CpjUB68+*uPi)o8JE+SQTtHSU~t~oa*Y!Q~e1E1!bLCT)po&uhGo|G>z zthg4!5ozdec-Zw9SK~!x_L{Fuz(GV=xcHCuIDJuP-Z>|Yk|39?!>Fq5(9xhBK)nNw!rX;Fuz^Ip64_#TtK$N)}_Yg0pVr@y~{ z4!RG49_s|W;gF-H=8XH^epYv}jPSyOlZHsV<6<3NJlAC9fH9%Q{6t+_$axH81(0Ib zc+Hys#y8O%4+_x{K&7cYt5VGkkQ&~f3rOf=?5d>wAt)@2dPZRdh3|D~`A6OmE-`Oj z!vrpI^Z()!Kz%U*9C=rejLLoW#toKn7ZL;%ISMX9rTh%DgNc9?IG@4~_tXVHD&6?f zzld)}RP8_ytqISh{Gho&uJ0n4hw`d?DYzMg<%=-mK-*Y%`&4EY)~wkMuq`44hPb5> zgXGbO&4}5BV7U{UjLd@L{wS2>mH{JKnHNv>zKVB85))Fe1O(A!7rx^>z68??w90aWhrZ zBBwF*cGt6;zbv8#MxQJhNH>D;l{r|DZbUU@y|JyK>rQB~n*4lZSX#((L1`A}12+TX z6H}Q8MRZasL0<3;>0PiLG3Z23155L69~{2l(5V_7cEFNQq3;EzLdY7h=~UjgZ{J-Z z-+JBKp);Z3V1A{ z4OY0y)?Rv~y{olzaI{`PSYOQ1b0AzVMMz{AQ}H-6TXU9?7_(7QxcE`{wOD%pS<#+F z5(1oGRx%i4izK|H8a=Hg1jjzbNT*DF-8lE2Ub5dC4p6Cxe1R>a@k-PY(`TurIrDmm z3I$k$tia}IX=s32mJ1ej@v#8S;@KF$$toKuYbZn6{n$+c0=OvtZMWgLJ{uCeB{)DH z5Or=V735$g&pI#~DO?bn+-u9eMs0~Kq5wi-laSjl9BNYzemVaD%Mi*)0)UqVg%Aw3 zSLnV_lWN2QgzRV$AXK;c4#p`O_COeYy=gD?l_b$33Q3RLl}UVogm zVoM@0rlZlH!g$?qI?Fh55yBtPUXeDh#N4G;5dvFaSNMZX;?(xHo{FRzhosSeFlnOc zOcKS3qAYbRFSE;mO%aa|l|Y@82F_+^I6+dQR%vRj5=$l29N6Ouw7*}G02zD!t;1hS zL5f6T-S#_XM~~8kO2TR~Ua*}9zh06~gdzjSM2<;$C=-SnB%%p2W>nrrAy zM(fFM3-6w?uT(+2CDFL@%Wh|}Z+ah-_4=-yG!>U)o?s19vKspk;@l}QYM|8wniF+e+R8{LG zL@;FRjDMGxj>+;hBit9Cq;R|meLo7 zLcP1~Ilv(V49Sf(8LsN|Bk?ekz%hG`As^$=zUUH(ha=GV<^tq{M2pgi^D6<~VE8@< z<$>%#*%6cy@{XjSs{B_TAhnA?CkV{oaaktWfMGTvshW0R3@m)rGB zjvSCxe_ph>tT8+Exxl-#=J#ebRETQc)p@AiG+QF=!ZEIRUD*LhEa*!#>}D4Sgx7du z7caF-;XU@*S7Gb3ABQ^UF_e$pX3R3G+TQ3Fo!HTKhktwOcb1H_26(5y{{oBKn+MgL z!I0AD)hjN85*26YiCvqahBN9jtW$7Al_M1ubzI;o1$qxBmNH)O&8wpemKqIsi_FJG zy@Vy3U|#eZ-z7^yz0QI@vxB%-aQ{kL9%wFs#7xFzNz|2HDnw&2;dgsdu~vnk$MFj8Vk_m~;EcZ18`irKJNMFl9pC z%jqvB(kTpWU@%Al!7{?2>_+U0f&IQewzhy$xA^Jp-tFz5iWU_MAd){O*1qjR)OBP^ z3LlgbTqejtvHaZA{X^CSA8_#;*bu=!sQ7oK>&x#P@YRz8sVUC^daQc+Nk$m#ff**~ zeuY?xEcE6BLY3(44#BiwKy?B1Nxgmhe(?Zg2@>XZ)JMz_lf#XxpFUOmsHiwP)-jY7 zO2@kS-3G8{;T}&pM%RPQq#pU$JK~xx8OZ2ECx`;^19Oq_j>sbTRO-i({SXr(jW9)$ zF4Jm<*|NgVH~_VQqKmQuBDI$yFX*y0n<9S8YW&)~Es z75^2W$YN;Xa$1f8`k_@0p8<6^MbN<_`Sx;~kdg+S^v!%HLlfl7N7G z;E5x0^tTJfC%UjeAHZ$1d_R$nK^w)^x1`=MtQ7D0MXYfS(G7@vP`ZKkL3xV-yCwZ0 zaBp-8iImFZUxr&Q`bjj)x)7mU@%60-;pgcZ@H&a-hW_iaYh@;A_-foFfKh`2C3>i0EL6TmaT& z5JgOE%{0E9;oA3rx8j#sOqp1?rIWwi*=WZ>C!OAMgfGm0YMDlf7jC1@z-@kvi+Ikw z{D{WX3YZh^ktjNNTalcOD()dNa)G&GZ^}ZvFQjb5kejK=yiiyi+kZ@gvg&Wg~&H&{`54GP#8d8c%{C`1nD2?=L9FQ%&}1?S65uO+c27UV6-QP7qz2DAMh0-6p2&-a3&qB zUv;-pJ!UuAd#`ioywCR%2s(I{g?+}b{J#pzru=mH4Fkei#W#zo5*{p;zxNF!!|OBP zf~gO$G?H4a|5N$TCUGHKZt z5w#763l}4R&7`!9aZ4Dco_OKKhe>I2drvIITtftnx3_t9Vs@hkp+5k7KY?U$G`*(;FC?f;Nq`5m@?bk zw(I!so(Lj48dTR!bB*>Pawuq;qWJWh_i{r*mD`t$9}%tbY~RXqNZr8h{9a1?Bia|} zJg#7ew;AT#RmR!VajR0oOdhI&LiG!jIE=gaKvuy-#uFTU7++m3d=`=doR|Cz(2dc^ z<1{*pvG!DTHWnOLu~mTwN}55CRmA$hXG$`c9Lizeco;@zJ=dftS7*I%9sc1Foj79J=bWm0PdqMW zWC73A@T|@1y3of|~@Si+J&`gs#Es5*mf*`a55Khq%bcI9e5-f8fga~xNxhRPl zALY{J%LC(H9+V{261l?SJ4&NT#+?=X^g$_zs|R16WydlDEw}9f3MZF2G98u=CG~9f z^o)jVK(mN8y%rbaxOd^@a zC~(nTNOJBa=H{PVrm?dK;~d{X>BO$Z1`E)u28RUf3{~*V!~BU7(%vQGm*^O5NubQm zz%U=kFX?ijeG;e<*Wg) zO9uK&8Mnc~h1e9jfQVeaG!~z{aXQG}5OQMlKpg-e_t-&V`SMLr#_rQ&W$4H6{c^7* z|B#CyKI0@)JhvG8@P~Qo;o&`Q1Qm!aXz0d-k3C$~^=CF9-KDYgN3R1Gl--UQM*xW> zmw&bPSO01pB1w-5m+V?uftjUIhZoB}2D_06W^n-Dn5r&n5_NQS!6+mK)cyNvb54~# z7KCJ>SVZxy8+13v!Ha?PCNRw#uX{t8+JCkrcwHD?e%7I1AjAL~uN6l0F6*fNzdkkn z>V+z(py=;FjA4aGAeaS>V4}A3|{o8V1x%(P=MAuK&9o)p7|itodVin;h~AG z=V#b(5PAc^hYu<{sM)3_g|uF{v&#YwR$OBKL}dg)1eZaOv~C-W zl7b-JA+OMUi|2R(smASleoW`um}4I1d|Cz>1>=e)f~!5Y*)2V*@zE1~pLxH6%pFrQ0xj8<8#ly0tzNy_Vx%1J2dV1UB@6@~*#5Jn zkQ<$}FF-1XgUH<54Bkna=V17OUz}|f*Y|8ZESc;Z8U5&ZkFUk2H^bLgv6yFb-QSac zm$^0IN@6%~ncyYFakzd~Q4@U_)oxP`T>YClrhzF z0zyS8G&FB5K%6VbMb)RSuoxwH0-O)}VYdqvHc#{jKrM;di!jk-j~_rSt=fO6P%)st z33vMav$rZjxZw?^_12kuVWoH*2@X(?EMjH}b0x_`=3V~usPDJKhZy+0h|`^!T+4!K z{}&WB=&&dMxX9KH;vNEoR&r?ZCYHj9o!UHkgtWfq90^=|hyh30F{ZRzxE~JK1_CJk8YW!K!NM`W=AHjlnbdCm&$`YD-5H9GW08q)VR# z#DAu(+JFQO(T`C%bPgU|Ngp{n)j&}Vz3kt-lRkHyS%`w4~jy~Z( z@#XDz-GU&@K?|eWxsa0qcTivQ`2}&tP>5ja?pgm90u1>{X($A^uwnZjF7;O>qNZs#2QM%n*M zh6q6yL3-QMnGy_au>d&1xhnOONB6Re z6G-`fZgLP*q`96}hk$ENNWcb!jb|l9Ievv*m+&*mXad7Fh~t9j2Kwwji8J7HPF4jJ z8VNo!u_w?32$}6jLJS#g91|bD!DlDmyBn(Wc}~l5LtHEjuz*kq47%AgUqBOvg`w6{ zS2Ubt-1`O#P|<`_Aumy|0dCOUg(7M!4;Cu;rRzbgZ@!kP-h*#XIp!#Di&w1J?q_Cd zYK(pY6!Mc9nPpqRu0`3Z#W-sG2ZIh`uVw);{^?(QKmz?J)XzlMVg9S6Q5(XeiJ?!U z-&B?8k{X+jHAl6=Hhu}jf;-=~!F{OguO?Y$0e|Q!NXr8Nki2@^`RB{KXX*jMQ)1kE zUg_g^dSgMu*&_!8-zBdXc4JMA(webo*TYTVj7KT68&8p)UW&w4kgM^fzbL6{1{0i; zJ-TP7%HO%7Q&rpt%om;(nj0CBGG|iw=yVGKZ;mN`+Q+XOR*ZGTPp+3=s}F% zkPKpU-8fi(d%KR|A&7@fvLbjBn|v~M+i_bQe-S`OI86@7vw^P^+rzKRQZ+%MC+i4sUJ$13(^V}XPQ7|25Sb$#_c2%^OS>QCO#Ro&w4fCGfwwkt%R6R5=cxVeuyDFcg z;>d$3k3S4odm+RWPz)hfzetvLjOWNW3wyuucD+AyH$u--lJk%J%o3*WiSo1@x&U7I z3L@*`GT%n(!~6cS$KOmcf!Pr}tbSrEm+ouS#X`LF#~VF+aUgxDY?F5a0*MT__Vp2$E$e6|Y4t z-X3DpP+yP3%XwZqxy1yohjd2(?(>lb9pQhonZ^nWx#_uY-V(0;8pLK4C-83KOI-eS zB7?%~n@!W+*q-2z3jDrEAi#T#o7=)gC`q4v91r=Mn&@>WiHZ3X?<%pp`7ZZ$=Q0pD zaAS&6pc2?cT^U4|oy-7eifv+6>-vLGNuplNbaSh@yQlg(mb!RR_@gllZ40bEEkw}N zv0U@R<6gUbVzATEvpf!DHNXSfx0(6zra8Rx=A{}Kec2bb z#MIREC$=WN{qjG3s;a7Z|8mFeK~T5oLC8FRJY#o8+ll=dNJ5MQ#IGs7P*na`|^Ng~UF8&a<5EwkfixNm>iW-^w? zt79C4?C$AZc~^p7niL4QgzsL~kr#7Y=g!F6)`>Nw+u!Ova3C;nVsA>W+3awxDPirK zxqcmO-CM>q1jAAb-8lCh7?PTi@W>CcZc-Efb0bvAeb1wL$7*WaRcu~{x2g3-gt66pf15n^>e|oYv4aPv zG1evbNOB}(o;T)H5{Rf|H7OmNrIL3+F0lW*?*8(jvNiiMrB1&Lklg2+ucW2?b7h;$ zm~8gzrEa1*{V$HIqidk}JfOykUB(x~hU=HGk3MlN3{?o6EDo6w{3TxPW#OrYYcws4 zfG}ymorpXF$(LNKbJ3$%oCISukb(Qa03jbeQ$5BCM1P~hg#_{~R%kACkz>LGh#SIc ztDs7{N(3P@f8>AbDx)8el>*?eu>Hzq0Iv`&IUkFj32Ute3b-K_7}|(rS(I>!{Y1Wy z#NVG+DH{om0_CwF7~vI@n?#9w;|-@Os>IkyCe{Lb#az@8@LlZ8}8h>7aOitzl;H^g6=7W3yXgwzt zK4Yich7Oc2nay=o4@S&@W_>=UhOTLG9CYvZn$F1yIqdZ{zhF68+lIxkNQ5{n`o>*Q zC(IfQ1k&ECL>6I3H4%{9cx>6naeK7T2Dh<3H?g*{%yF6Z@jDIWA-JoES%$$oupdnr zl`p2peMHoKoN#EXJP=C8^g)EKZtAqb-dTSd5ne(wFVnNT_V>Z>gLq*dyaTr4{n0sRASDKkD;cyE>shhgm(qMcNmJeU-C@Lk%%s@DTJCe4||uJnq3!?>ub! zF%NeV8O!b7oziDU(B^8hoxX$5K44n~w%B)94h+i_Hvps_-1LpWN11sMu#EA+Qz_sw zRGP5m*26AGAkjp~dRtTTsyg+>S>oBlFtgTbU3NKohtFs_^uLc4x{a;|ew{r4?~mC+ zW5cdKuQ`=?$6&*(VNAvj0zG~EKU$=Nh>LvJiemw+jDLKtK7riZ9&0r6z9rxqOl(OJ z{mc-*A3KJWOxz(Xl@x4)(fwp-Vdciyb(|k%* ztRB4iJ=f}oo6#HpNKdV*hpDBPo(DYbSoPI?t=r3Vy{57nxx-~)cQ@>R~S* ziuFx#+8LlX`YJ~|x;`To*TzkEJlW#gErU}G@3e&%l*-BXsiON55qo4@3Ky7+ST zoyiNoC&1N{qDtO zDIV9xLRo5O{NBf}Z{6PZ_+D!6C9$^d5t}aurVMO2h<*v6cP%mY|Hf)B4ILOI=c=od{=)}#em}q}NQqA^k}J(9+Z1tb4HTYp z=grfY`85a&8Qk0Z($2)Nu5M8X-a**|NSi13(woB)lTRe{z`Z*Q#0gR>0c^1sFH%{F z@eJZh5l|{Tz?R15sr+adXebDXI0U*EzS*+jkr7r!9ez3NOGp`2P*5N_1iKpQ53mfj zc4wMG@rNQhg!rOsWMxykNz{xpO=(ujro~rxh^tyztpb@!8piRA6W8>&Z{H5(*<{e8 z(L{7OJ^8yvP7aoq#7JQg{NPwwQdcXR-+ejHI4!mRRl-mS+ZUR18(Y^AoFE&rQ--_tOXB9sXmlBSx#Ibcqb|WN++-Y;sZ0hP>+kWmb zsAI{wWwyI@bv;5t_+bU3=jY($wEdC|cBE)^u`%nZs6)*a5$St(M8@99>Fu{~0=Q^S z3c@l~g(smA8mCk3|meAH#%e9l6lAOWu4OB$m3Q-mxB-9JK-Rl10Sz=CIPy@^a z`_Z@xI%mvhK!-ludrZRFb7a(pu!p4jXSHx?yNL1Z*O@fKFdIB2`%l zTx^J<7V;?BX|;EDz5xm;HdaP=4#G@M3JQT>68sHT;?1Q!!H9am?gay*^j-r4g+GQv z#Jz1DO}V)bri-9}Wtq2TPCGi3dG_TX}?(EN>o7FUZBTqcr z@cD?1V)dV$-P*aQl=F|ehwb7}Dyfsm-g0hs+Y5=;+d0-7k%cxsvMaT5&=KHXKKY#E z{0gomuGh{9Y!TuWzPjQF-&!etlfcE{9POJ_AFyz`uH1Zl=e&ic{8sg2hBiZ0p7ZDR zcfPuN?xQ)cN#%t_enzuBt{hXsgLdmui*?f(Y5pCijMsO+<}lpLMr-B-b$Mp@PhqTN zmw2+~Is5agD~?XT^k*=_K>0$#x74#@t&x^hFSVDykX5O^m9mtl(mLz&RQs&MyoVkq zzuC9%VxT3rxlk%kB8$89GZC(ZLf-FxIx>_SYvp$wT+p(YW9?<$H`QOxUfm@=WtZIL zYu#+;-4)y%CY3z!H(t4>RH%+OchL@BhLrzIxmZE}Jv@9f&2a@VEF#Zb`Ep3^vyzS7 zg2AEKr5|IOt-|>69evx2&b4=RozKWQ*SKMYA4`(`t)t4iwuQXvVWpX`GA`WCn<{J4N zT5>a9ALo72cFHX~OtA7o`ih4B+M1^fMtB-SRN||!Xr&OxMfab#)vMMU%&HAYaC$h} zB|fKV%VP8MW=m2k2M28T&7Mj?BTPUxL8$#LZ`s^SW|?hx@UZ@^MMr78609PS^MszT z2pyG~4}vKxq9pR)g7*wQ5ba-}Z<;2`D`J;_M248EZvAhCw2h3Ua-|H@y|M2`_gd*u zp&*E+7*VAZyiD;(XK=i~)44tSD-Q$M$}Ax*9q5a^vY)|!tk#V zbItRSFWcG0MH&kjZ1E;BK37&=!paqHMEC=^QDdg0qQ>^^Y#4lVG`0by1cXZAm%q=z zAlNe?Kc99r9@N-?1!*6-Z}_xRCiQu zQ0@3nr&g)X=ODudy7bCXY&StVXKeCaU<`V-P_dv7WHUAYl_6AfXqcTOmx)Ahsl|gC z4tO#zh?`62)wTR1pjKYQDl6CDNnYst$z+83?b<~_H7QA}&TNs8kod`$ak+2)oH^%d zv#h4J2nK=JZ2;{&=zN!XJg?$=bab2x2;c-Ex@v>^?5WS6KZlUcE_MIPo08u$B&H3; zL9*H0(jwNP+O3ArsK}zNa)`s2(^B3S$D+$T<)P}Dn*R4C@RH}wtq(q;{vbnXN&-vJi6FIPN@yC0tArHV9#F{P2%I+6s1Qb!EL1vs?*J>hCnYZbAfG^fE z85`X6Ohg#oCL#!|O99B-sjvt)#B}lLlU;{9w%|?v^gs*jIz$e5`}%e%n`&*53|cHL z%>}wtbWDs2NPCd)S2PSEwB83C1_;h*zK5yqtqr!z%eD($z=PV}c>9zHP#HX3NN@_I za}6Be2cC0s8w9H=Z@rWtobzgv$_$|?*g(vKL@+C`KI-^*EcuSEEDZ=TzMyOZ@t`xD;Im_BL7S$P2#xiO*1h6?u)y_E-fey1Iy*RU;cHF3)ijXgu_s%rFF57ptxaY* zYyDH6Hg)8z9$O__-TzT0TikPd?}5?LHAmdWW)F>(iB+xBpB|jv#Kxm3%j4V?%l2e{ zE1Dytd2%C#1Gff;j|I8tjarId9vR#q!;xe#HQgqSZ!OO}(46mmeAK&jU{W zEXYe{FecUVOIHW!OPtNOS#52-(r{oXr@d4@bmdQ+9jZUNZCLpSC}L@+#VQt>}1PBb9r<=}cf| z&MEQQf+fjUdWUW^7~54>>>SBsuNthY+c8oxYo&wJ%=$}tn>RF`2zIY$uWN4zyPtK~ zZj;r_bwj^TSPu1kF;w`QS!$H<7JC?CXQB^*)YQ=>edClxRn*y#uGjMRM^hvOaRYDo zd#&rGs$>vtg|RpQFDaDo6dEjyo6LhDBXjz7r|{{E;0od>&|V3nv9v1|UgZ5kmMMqM zYDqx^t2B_W)m|5%#Pns)o5nzc3j)jN33kdI+VK>%?}qIffQ7RIcl`n1fqaTB z%CB<;zxLZZI2@*E<<(Iq%)Q^%y2A))iS6$m$6Fs87Z%o8trO*j!fHMOi?Pn);^k$9 z)r6bd*uqatloZeHx?@TDglj`(ZTttnEJ47i?D7abxylNUODfoznVH0=yf1~`5IhVN ziUl2>h-VZ;nG^NYM78k&S%#AcuNjm=gI#N)0eY^~Bi`QXd_qD(co#H;Pr~FMI5P9E zEe?j!67fcmdnJv+#xEcsmXiu2F>1-iF7^P70g@33_6><9tMc-;tSl%oj9!5R+~pZafaB>VPEKhKCQK`at7mM1@}`192E@`05%4Kp}f*&wJuj~ktk zU^m1kB_)L$$r=iQ?=NKeHN)bCn7r!6dQIogqkne}TmPD$pP#l{8au+ZNl|RvaTM>z zd3XU5&1Cb3b1%;KQU!-OgxMSc;WPnmAl_)pDY_SroPzjdhPk=9wyy5tW#_9EiRQCu z0lnNTX)e@>#mW{ID*#@tj7cs!37Zc4h8ucb-Rb8*vV`%MF`S}Oocbg|HUa8h-H-_d zDxhFx_>dB11B?F0ho#@)-6WSWYO(<-TcH<_v{+7?*x=w`8t5qDBxI(W=(T*3Z2^F; z13tVe_`Yl=G`0jqk*P9ftv}qKWX{0Vfvl^_lTQf|3poWMA8_C>zmRs1)3iJTKNM6} z$RiB=V4}m^p+f})t?)si$BQvYpew0^Z9L4fxa!EM35&q`)k{;Qa3Nu}!21nXF&375 z3YAGW51ZXBOC`g$9AoPiGXW69$aduWK;<{9&jZq)US7QwskcKyp!54==>Kd=W%UXK zrYL@hT#F}o%{Kg5FXGtO1PTs)xvhDs_mgS)*sKz3bBI^{*PmjyD&-;jeV)56Cy%kq@nY3-zD|pr-aotszCU?PpR1D-=ZJK=SC^HX1aK|JQ;N>@Z8i>KS}0v>MRq}&Bg_B?-V~q0#m^Y-flVM zIrw8qSc1A$@in$O`Ud(DZ|{q3NdBuj+dY1)eSx^--$A>h6;8TWonB0x$60wv>%LF{ zCIZ*hGd@(e4E9-7RV^*3zT+M1)Ha2&@Zj>zz7?f)HGAEfTgncDYYvOVgQ=%kl9G}X zgxaUN?5Pq(uLNmRpo%25)bm#bME~MNAmpSwdT7ICrl5IPJAbG$HWL+sFVh70! z<(5N%$ZevjsY%!!ytJkT7p!j63B7^Cj$JC$*cY(zNZnx!>&bPF-Al=S9k^KMsb_4b zQx|~LM4UrPgQJ%N0m)M>U0rT0Z@l2|O)-?95LaCpxf?!OcV+`J{w|}CrwwyVI^mXp z?M4<#eCkqS!s-Z(Q#8Tp6NPSbyBm1sPz5uvaN<0&#YGdoV=#C@QwtN2(0x-MV9UZZ zuUTkoYpW0nvG(`|2yH9l*nt!+bWsPu+EePnx5ih|y`(e0cL9mGR2B)m07ui%sa2NK z3|xctGQqP775Wye;b#X(A^aO;5Hb${Aw&I4(R2Vd;H{EzrtwNh1Y-BbpNHo+E5R#v zwuY8%SVSZCc!96F+8PEPc(A!(5yaPG4{z#YtFGi9fmt3r9T;=Cg$j+5D=g_8!I!7# z*xa?TC6%#V_)xIUxd*z>=`auqpl-$=D3C`lh)s$#cu zjDNpS_zXy(sIs!T)cN{ZLqkIt6a{Khv%MjAz<`;uZd)A2!V+VKs}+pBD{~jIxI-ZE zltK2)S?K7{`Q?D~CkA$s2hA)n+*J+sw-n4%1Alj?UvdCReZZikO(x$g7ddeeFdjkM zMgKfIPYu~rRy*J$QnU(`C1TvaIUXya7hc)&aJU(E%bt=Y6*Du4{KyLr!M@K@sFLsbfu35X5a>PA) z?dsrbu;s)F37U`g!3+z8nv3Y~Gs+USW4tJbs27`##krv5V5J+tsn~x=En#!Vb247R zbqBS(cOn@XBH2M<;YR$Ee?SS~hM<2h3}|&e5g!#5#k-%AeZ*Fw2-pY~gjzj!YBs_7jW9lkR_qEO* zE}_4jGgTCCrOPi&9_>#*vM@i7s$1`^0#%0qo9Ugo_m6HUlaF24G(U>XK59XXnV_50 z`N)xb+W88Bc}XsPho31feEzIfs!4d=`&U|RCc|%Tog5p*lO2_?(Bl$ofYjunX$~ei zT2XdR)$=X5t!&#|815M*K9^W!y~B#ST2$SR|Bfs>3_$>(Qz93B{P?lZIWu#?nq*|y z0IKqC#pH~GX#D>AUn7sf5D>Y+^7SJElK3YOh)J79HEM~%*L$l8-QwllKI%n@@_=U# z7e#DdSn#I;P-Y#H=icBNvqt_>&%#{&DlyoO5E%`8 z7S-AJ#}YtSG{&^!9T3oTezGXZNKoF0P-4&<0J{R{HC3#v+8HPfXtd;I*RHL@gGly| z+r#i$VS0-D`HsrQBA?K^fKd1CJIX@1MKy)0e0hFL+R#$@rb| zx~z;BBDjgI6fAz~PZv7dP1w8???-(=4|ma*SL#wCp}g3@CMmOW6cgh$x@y|Zp%R8rDB>Y*WjlBe!7b+Nd(`j;0KmN>?= zg3-pWv)C6#hxq6(}Xh6`78=AJ3G7aIZ4F7L>B#e>TRoq8VN@QEWxfo zYM+Z32byX(tE-DZC)8CzL$vzr;2qPj#K&meo3dT0~LFXy6 za8qmvUhRVAHy&vT(aOIk0h~e*@N13=x}mDiTe0^L7$tAd3Vwi189_h5ImyTywFyP- zYeObzr|`Qt7ZxUj6iZ5}5?w*ArMRa+V7al zu-I{KT)@Ofyx-ZZ`@mc|MhnF6>|?#vn*?Ys@a+3s>yv^(>4z7*KOLL)y=kZZj0G=V zcn%&tJ)b9Cu_E4v)!gOks*T$U;(rV-CM2>n(M7YN%f84nihsqU?EtFk?0=G6#tSyZZNmMZTxNqs6$80KR3Z4xc$}W&wGWSuIov>@l zuYPAeo|3m;$|oni>T_=bpqgmEait{VTF~#oVMWu&^S{+87IEL_y?$+1U3ll;(O7pK zpR6k@j#zD|G#p#*+rp)!I@Il>7-{Arc+%Yb4H}p$xPWfnys7wprptzV;#eA^w^%CW zHk{@LyAi;L?EUWZV3Q6;rZEl{o&>Hs^ho8YHcP5qhfV+?XFs-6?K|)&x;w~P@m^x0 zj$atiG#qs}&@3$YTLDn9;Ldt(U_8{;fTSlnf|CMOBw6JlamN=3W1;AVcd)J`{ww*U zf~|QFvz0Q|<%*;ICUB>oM**IP4&7swd60Dv$DCLxc>QU2tbVn$_(B#@y?9KEh81=_ z7@Coq;=_>{3~Y4sP*uW99RCh2>%0RP9lw(>obegxBXsaoKrMvxhOG1H92X<=x^Pq0 zVOAowkNb-m7f`Dv&_f@e+K5GTS|MoB0hGE0N`oU;Z$r3K>%pn`Lw1=6e+T&Ivq05B zBM_04w8u{g-_*#tg9Xi`oN?ZB7U(>%0>F$t+k^&cs!?Q4kh&d03Xrv*Y8&p~AACf` zMMR`o2?e5)9}+5sTkPmBp5gtSK5Ld5Btr0D$Vmaw5@|rbHc7nA&8-IyU^vCtN>{|+ z)u21B{DN^4E9HPPIeLz#D}PH{iSIY8SS2`KprAaff{DQW@9Rwy_YDqQO))=yTm_)1 zxrN1dDpCiW6)V>!>5pXJLTcis+O(TYbckK?uCJ(DcxYgZq_$q{ICU^eYjAUJf@ zgMxm|8&EzMY&f&^pheW(pZNyk^^t*QoF^6A|p`0F{ZhSbaDm10ompKg1*^ z&v$ISfQZ(Z;s0ir0b>P691Xe6{_Pl`S*0?%=j2{)O^y0s1&0agQ`PdczyW0uvYTlb z4ATypWW>bGAe}8%HQq=vShRlqeAJ^xr@Ja55NVNM*<$&bJ6ozBqo4@64NFtm0Mha_ z>WvTvQ4S^;f`v?KaRjak%HTyf^)@tOoWE=AP$$tfzQOf{JMU`uci=Fn@hDPWyVm!= z-|gh&R0oYl4v0s~_j`~B2k2ip5OX-Xd;+3PZmi7;V3JYSbZfnhDUiZI8iJt&o&61$ zBS6ee5Mu=7$hJ}j)PN?+HXkQ1O} zm;5%DQ^7l;%-)|HuU7IF?JiwRL|~2gx;vpS+n!b}?|2aqHdBYY>cFwXJ5C=77@AwN z=2CLx60bK(yIap1CTbjh_wvhG>zMAH55DE~Bt>_8X)-Ab3(aX9OPZZ{EzEUOHOkWT z08X`|)(VV8bz2qH3!4WA2+fnwluNwZX>1%cv{}pfj+y5E#l6ENEOA3!71N6c6>USi zj}!@x?$i%CQQq_6nSF=br!Drwui3q&%aUJrKiO%iW%#HiC`&9+*KyRKkFDqG!fVA{ zEKU!$O-g#j8`4ra?dt*l15WEA+!R_{tTZ**A0JFTk4Z22(%e3;P2s=3hQ;g?l2!sXc9C%~Op@*F&FBP$*13N=}Mjcqa zU7BJ5Ul(X{+OLHTPesDe+xdzcF=`R-5JW(K(+3xMMo^cLQdo&rINcI@}D zVP-MWGDV_^{M-dnsCsdC-xjSZ>A+} z^@d^yvkS4z87F)V1Hk8jz@w_a0$JMhH7yWl5QfC(VTb1{jV&gGg_O6l5ZaPsRuuDA z?cAkF7a!_@pPrsx9j?th99sNpwdktMU^Ecu*?{v3K!}vIO!Vt7R4iBL#*lX$ra=Yx zHx#`qa*%(rWafSp-=4K#CP6_8`Gv1EY?H7;;%Oww8{aGgNyqOn=GF!n2;>AqkT-^| zDTrJZ~Ni7ndl@mM}#!o3$BuHvTvRYh&BLfJ-N zaLKU}$vKOy*PWea0d>I}j1JhWTL6S|izcltG_F%~U}pV~6!yfdij-MTwc)t7JcbYwSKcRpQrr$1=N znz9!XI|za5ll&#^Xeyff(Rb7EEQyFEo||HJbuZUy)~}i+@!Z7Y(8{M|3~4^_g9WX8!T7xk)s>YLZ)SPi+M)0jTtW4amyV%iu~8^kbx`~77j0L`yfah z5~zO?ZJKt{{M|t-*NN#6E*k*(MXE0sd>}B$uQ@zD*Ma`o4wl z{JiF%jRe%uk&*WGH^L0h!;H)@VGEqRLqbCShpq2`$FgtVzhyp^)u2>}iXUep&@`+v!~=uaAAKS8yR~Z+l&KAT16(Q0jW4mg!M8$ z`J91rF$ceF3+V|J_b;@>PgwLQ2Zx4+PPx6YzL-7}csX6S*_0hCP(D7F*QKS0u&7o^ zv}Zrq>q-zUR_-g?U*^JG2j$jPkPtSwR)QObPX+F(I|Fyz{m8z5=-d|D^J>avNv7>C zj*!}sE^)aR+OP7+06gH>pkPyKd(0S5p@$}l7MjBSR66iE6a&Z$fpv&slqg7J#5V(`+&fA^ARqu2 zc)UD@hIt7FQehZHfb6kge`O+!6*9yX{UZ&PtvPQpGLYttEqg{&C%!6~C z-1wuSpYi$4s~^11=iDRYFW6)uE@(10Z3FIf=gUh$Xchsz1vmQ1^;RzOYAc-0*du57 zN_A7)g7496eFh%+wGaBDWwL62#ESM=(7e9MA#?iXJ-?QnLUr|f7C$d@n`@bj7jc)N zyw}iJp`^G|#LbpbJbNXudNGf)D>2pu-CDQEUA6CjAp3?yhab|@Z1#o^cpJ+!HT}c- z%V?Rxq;9%)Oy^1p<<78Ho$Zc^R!nGQq1Z^%tufbpyzAe~@4`MF4xH*?kIYr`e^C|c z_r0L6@aVqtM~COFce}c+95advpiep8Yg^EML?q&DU#q--m50l!W0IFSZ_?8U=A8=s zy8kYPf)z!>_V&m(TR6UHhhFp4sb3NNdq~q{$jGRFL@xYZ+1YVVRgR`#E4Uf{un_{h9=~pf zh{z#l=j<|((phvtxauS{X{*mOToY0WbI0rS@^l~j#~18`791x`ScxiDo3;?zBZ0it z=l=LkbRr;(puW)qCQ!oP^`zBuPbczf2pC1uH%LokCY^;_f-DiCz3&etwLP4vF zOsQf7B6-KKFX5)~>R4!!H5))Xlf>9q8feBHFWqM0i@S~Cam`TvtOz;r%Om&vbs<4} zd&tBjj&b)!Ct@%GYd;s4x9f87y`v|5&u51wi3s1G7-(zzqq0oRPRdLoParx(cW2a) zS*nfp4cfP<5tXh zkQA)A1%Zf)_P)LqMn*=Nq5rLX23AqCWcIYEQ~;wLcJrWp*v^tgD|VaZFmg078F&a@ zOUhx=49=hEcCHY*dt^c}+qfDz6C?!TMHnui&bRy2TaO$dli)VN z272gz@rTZ6iEVND`P|6W6c>q;YE8l&do$U36p@UfSm(o6mSrc z>@{Go5&2tH?-83oeg0RgeGgoNK@UKKLPQHf-wj_1=p5}E>!EVR6hYE%K=Ob+r8jJN zM%;t_ahZsA6z{1N2P1J_e3Nms&-eakb2l_sS+Fb8O4PZA5jA+KfaEviE-uVs(k;Th zx78iJA`x#9#w(Dyl|8K^_>I)NADovh=^Qw9yhXM~4t8zV9U>Z?^v@X*9bBwP0!>i{JU z4L01A#X8U?z0tj#S%&8VBl^+aL2R?pi?sJ}zvKNj?P6kK@%vO1fJUOa#fIDcHjv$O z;?9ZNxNyPnL-t|p>}3w}Z`Bkwx~k^PyxPqzS}x;On|B^$oz;7wn={}T@wr!5!GV|l z`lVr`0$u;Kn3e-Gg)(m>R2g3}9j7tNJ}p?0^gdhN`s}Z_bcxg7)=W(~%Z@zy52By% z$wc|E_fF_O(POvz!=3jUt)xn3Vy+_|EnCCcw$lR(bOI*SWd2alaG&77pC+?%<_J-NP;uZ+I4+qJIq4-Cv_ zXZ>Gg?%gsEy$aYNg83JUUx0KyttY@Yet!33O_fEf{+$k)(L>L;dX~M3@?_1EeEHmM z3mv67ZzUK>#K0SxaAeMGc=c%Jk}dIu>X@4zv#@lVb9F@H2S@X#MuSzG00R!CZr4w+z49S^X@*mn9N4FIf(*=m;rr#t|52}^f|~%L#I?MYaFo} z0f!o=WiW) zg8acBFkrb!4@rN++jTk376^*-*yLmxNjf2$k>qLg5yV24%F#OZ@K@nV#uofmWCVz0 zU?o3{zV~ce+bZk~IJmi4(7Tk!h2kEM+-n<0B~h=JuJTRQ5ms-2-JeBQ?_LJ$_t(qp z2lVvxO5W=OMMpJ*=G)9ePyNBehkxbU8o_v?0k^B(xB>z-$DggF84E?1H#|N*P`i%X zzles)+7Mo7vh5S98+lOpx3J|2<-%eIlWcL)^9)q9E<0y=x#i?fSN(qZn78}mWD3d| z*fff(2nh|(<`fx#LyUob;fxTW@BV4Gk63bp!{Z{jn=6rg#@^m+AnO6#KPO&@yi-&D zXmd5>7acl5RrnQ0>WdtPgX=jNWvo}qNbFlSLb=bGn|FX{X>&%Y1S>gwqOyEgvc?ImlX`ggz6<*~wWpJ6~TLUn&r!MCwKo z%{-VQp;?lG(IwbY!DYxeExpzVk>y~6TgmzZ#f?u$SWU{^k>ke)N3t+u-HC_@8mO#q z^Lw3~EHH78mx;uG5bmM&1saQl`oHsrQuxU|{qimOp`vyf5=|~wE!Afe z-0oOhxY$%y*1F<>!cdFA-Xl{}uI;`?&y2Ad*2a1o;Pq8H+EL3naTm-)rRsnkz%^hAw-UTXP8*mhnSz^;gAS&rwhG8t>{E12(2MBX5Qp4&R@m8*0omf>MY|r71_mS;F)XsT)`v*o zyCJngrzRB(2HY)V@r4-`SO@azLc7thz?4)8le1UG&;%T~pejJ01}Dghp-%*I!(U&I zrxcEoBXH6TNCi@*pX~Dq@L}~UNZdY3E}g+l>4|~z$z`kus7>0JPBdx6@=0G3 zkd>a^j_s9Z&{_lqLiWyD=7V4+09#T-)42|hCI~A)1;cFFH`Vtz zAQ72B022>Ic@1%O#wfpSRlrY(9ZFfuWW1`z=xPSJD5W(0K3MH+S~7jQT;Vw ze|&OsE%yI7kf|0HqNp{Hd8V$DTDOt}Sd$FZ4?hR%*-r`Gb%Nguc4TujNl8g0P_~l< zc3U(*k5z^@DV9TF_4f$90{d)P?hUI~^G|=$Bl-__2agc40NCABc#Q43Tz0e*&T@hn zr+i(7)Z&TheM>`vg|VRQlKon>@vm~#{QSbu720(h9yZHWes8#O&s=Y^n`Kl{^;DDG zw`iV_kOHompkdb+rz3NPS;8YqXeh!VI|Z{0uQdq-QG2JuOBg$ zDsq>v&tmh@A({N&4xJX8VdPhK{a`5P`G@Hq8_kn=>Jw&H%l@P=JggnLGAVT=o;Fv^ zy>^|Nkt749e!%U!#3F-`$TJ-p%X-OK;zC za2Ckr{ZZh_L%5EnrvE-^MN%EA(qUk{=*?L$*kH(<0zuFO3rw&H3Cs&sCbk72OOntJ z_*hf*V3Ed5MDUN(K6-CF>>*z0WZP|p)`4*;>$APWDwBUHKYZx5CvIS!R10V}#7upkQIKl5P|PF$y{;n;@3i$9PFH=b8Hbhml^1_d;G>f6H05fI~z;aly| zlvA>Us%p@az1p+pX;%w44xC4iDJU$geD-WC6-Z-cJq&Y*Cr3ED>8bBi6u0^9+pHNFG(>De5b|_gQ2OXo8w}mSvX(>TK4B>;7V{M&<1OyZ7#X9Lw zq%CZI2YGvUaVxSBgL|j~RXT8Oe2zaK)6gO_GgE>Yu;>XkbmB&@Z=vh_9He+q|dC9n^m)G-io(EOiOSN?>^YlNTH!nRFoApTNB1WLNmrf zfn=wzLmXy9fv0?n`=mj#XaBR}Vr4u7MwXl$9Ph0%Cv{SG?AfFBH5rNydwcs(&r?w; zsoDGUEne*g?m1(sFq9%-nCG$<3((&cZ*Zwz7*#F>?kUfwEDBwu-jvL=(T^_435n57 zYxB@hz?-Q4YUz_{UB$VbNA~yhvNKL@yca)yXSh`iLm}IDbcmT9l=h4Q+xHbhinII-nhoaS@dj95b?3as z6VYF}5;OqRyeeD=I~wGl2DwYA%M@QfnA*H?Ku$EKjA!wC45OpOY1WjI)AI*vBD43< z?5^L&JE{MV!{W3v##V)lYWn%cG<{wINEg2Lt7ZtM0lSYz<5WrxyZ6{ z_>1(nEFt|g@MMaR1Ik0Ne$aZL>QPq5b4rFwAV+}Xcwl&oq<|%&DrymyMMEPp-n{RY zJl$ieL4Km*U0b$XkNNx;?^4AhS%hA)-R4cmm>t3-;BiE<-P+MnHwfe;N3Hz5jy zAlWwDO9{e}lAzIA-=u(Gn0 z7L6xF3u-4d{A*xf=gs?<1elhA3}LYR3I>pIf_0se$$+3uUd2`f3%f60zIa4!pc2=* zRG=|~4OpITM5$FKRtxB&R5L15Zw}Pe)uD?@rc-1aYBInIVTtwiLppPRnY)|Y-U5FV zqG;Xjj)1(Kf=32(2o$ZFt_vGNV%>&-ngd8WhSbVwpjHW^`Y=gH1Mio>49<)2 z&9Ox!8<>QlQxBHoTO+u?o0|;s$tj2e?Sb!J8Hs@+flHublUa|bu@D(g9HxB0xDgw) z*-Li!Cg_&HhoGrld%r^jo)^D}=C! zKpSd0@5~BTevlO?b`T(jrprJo0@FcU&2@lLiqKx=$R>|TX1sm-_N-X_TjZ&J)e>9- zF0#$+6qd7(q_4hA`L+rRv-c*c3Q1a%P3{<&@#Dney6-WO)cbJezJR~T?`M8Nne@-S za=OPOQ=iQSSyFD1~&fD$b?Yh`}a<0PW^JMEjCWb+-HC%KUU zJ6s28C|n7`(6=gVYMU=MHBVN3SKew?*wTCbY3$nU^WyG%%`^pr^My79(r_qn{{rh9%I4Z8gr&W?+>U&Ve6mR3sB>*le9NL9^{%Rey4 z)MnCcJLR3GX1zf7X+|fd{d?B=d1u6!4(^kUUYr(Y@pC!KlCx=XuBu?{3fC&AN}z&+ zNtuSwUHr47F16W1Pb6Zpw_6^{2{e|+P6~G63g8$UNb&E=z+nwQ3QvB&g8f)$z5ib) ze5wFsj>62vjTP4(>z3e9xtSy40sZI-pteLd)*Z&QJ|k2(5_UpFAXvD+Roe!M$$NlL z#7m9S(23^SwQJlTuM9&L8Hs$qZCHVVB#p7@#ON#?1quv+qe?j7z(n^$hcdi4rwD}& zDO3<}+f6spTuxaFJ{G~eF!;;d+KdZWJ&dSSFc?Ygf}K{x0}n?PYMpoc@W62-oh7`F*FQR;vZDPh=Whl($XNICxpkd@*=poNMs!RO4m}UUsUD< zp?*>6sSbNLViFRu1@B8Qit{r#rpV68wSrcJN~VWUi8M@rmji7PG{aHfn{`!^BYi(w z49u`$YY<$t2FwIBN<>joY+?X|{VHxM!@61{PE!aGwX7i0XNIOW4J>UR4Xg~tuL|#x zVEb=zqT2FJV|pf9}yWL7ZcW&!wQF#zlyQrtXb0uHY_kTAn+Sm5hv*~zv&`|Ce*o{jjMq@ zZGD)MVjUwZg14;>n5PuAuh?bNz@4L8A$jlu?9OVoXAT4fwFeUaU_EmHW>w^(VpWS3 zOjds%WV>k$Q_;N(K&S%nQ5qMT&b<8I0C33 zIm8dZq(piyfq4ocGIfM}_Kc-`WlbC8FU#;qY$w!Xj5f&@wJ*b$z{`4FojG9Sk@*ioxwlOjILGipOO! zIHkx?4@cq>g15uCxplzSz`!&C9TPtHp;bKRh?;Bw4_;=s=&!CW!t9v8eGfx2l4JD? zT3M-uihVpOb~1#@&a0+`DKaPwe{c^qVxztKSW0;OvX8Kc$9fIqLZLC?0+g65>5T7TM)Jc&?7SDlBtv;DbMd@^zxUmY5v)Em-5 zwtd-vKXH40UWo#2y_;sl9tE3>dAWGEHIb<@Umpb~qUMDH;vu^6{9uh0*FUyy`3}fhuuI0b5PxP5wNHW6% zXI(scfR~yROyM$}wf*i#5o<@w+1c5Dlar6jEMV&oiOKN9#Nc@L+dkm0dE9?igMEOe z37nfvxU)@i;i^yIB{1n%`W~oGuwOOq55{LMns{POQWA+)0L~^O?fyLPo0^(@8pjL3 zRid&Uz)wbNHYDBy1_CRN8lpu5*i__OlM@RACsakUBLpm)XZP6fcB4;g&!m0bU;~Dz>iv9{mKp4WYGTIZvSw7zNe!FNnTr)93PZ z+%ovV-#`p!VM|-y5R7@DGs>~0-EoJqvH_7rW2ucPZn>h9%)~70G)QFFF#ZRUc$v@o zi2@Q7EG4CFxnU&(hDT#yP~3>w3UKM@?DRpIgj(Q6Du~p$iwP5vW>x zTd=S2{$T8wr0}yM8?(JpqEPOefMFzFu#{ z_JI3|VI1sJG*Pgzg1YDSo17ct8N5Sj2$9892!go7?7D1y7Le^j4)I$>HAR`?WZ5_g z;ikl=n%Y{G(jCh#AEbX_jZGUdm><;#$$=300O}FSTR?zJ4?u1M4ht)zCh-23b`co* zR+~b>PNj3ug4eN}0XVu|e|_kq7K>P{3HsILTtk1rK3E-5)n#x=XH0D`w6KY;j_d`pP&0g8{boa^dx)5_u;KY^G- z8qp!SNecX!IMgW1Zdj0v_twfTRFbNdm8E0pC{q>=wQeCo>S5MklKA$m%aMhFtC}#T zx{x!wXacMRp|UkR+p|GN3){DG>4ub$_#Zgx*>Kxlm8TKQ$QbsX(~&~)^VTKItHk-C zw45KUExqqr3i1QAg7V7j{gr1;zmJ*Pj5%$lv=tYaACTI)?7NtvRX4iwjSd2LgnU(v zZ0^K3`!XMKVm5;I{ZHy*`J(Pih? zN^YiGes?2_p5BR?U?t~uvZrQ?=P!NavOQb2DYI%ZQGP7{w(OUtTae6{8dO0)}3I5Di6r?ZLF%%uy<5$s-HXx*oRR0$?^>- zClQtKe!GqENV2-jg3)-y;-rhKt0_4>k00lrNTDGYmNx0}

$rzz-`#m)(wW*>*a z#a|N#%6BH?%FEzev>~{HM z@gKIL+4=MKYHaM)Y-9u_GTnhn6IU|oy?wi2wdWzgY?)i2)w9b6;uwZJ5^ic+>kosK zpc7ePA@LqBXuk6~uN*ctZ5v!K;MNU}FKvAXBb!1+xOOIqF2Q?%`(! z>LT#R&${d?lhh;2Rha28RsZ_+OQgG+d?`rs{SN)ho>@6i`7k1avtAiA39e#k&NY^c zi%~N-v4X;*dQN`}Mc6RcshHyv*))=(d@??_8SX;6KznCBr~Ky%j5mNug{{lfP3q7I zn6PD6-@Ppk^mpP`B*u<;6n4tw7eY)0ut9x=4L^ZuAivR{K{+S6H-+_%ZI{~E2)6@* zp0haU(6W#w8@cRFKe>j{wn4U6gb|Fa@W6s6SJy6^afMfi0_Ipqgw|XV!eo}!KL(?I zH>3{13bYuautSzqsUj6cC*DW7xiTs-K=o1MgykZX2DtL!h6XJUFg$;-@K1bo#2Jdt z3ii(2%Y82m>{Gul=Pr%@d&_jjn>WVbx;PExO}dOPDqs^pI6GLE1f|y1X^|LL!uGp! z^xo)TC5~O&iT*RG2598N%juFP%Et#Oyaa**u!d*u_sFC$NHU~!<&33g#l}B-cIYb$ zGxNYmV4Tf9OgRle(?1qcBHJVC$y{gu+NXyJDxbz z67Z8p?eL%Cj1`BD`>&?&Sv%F9c^_k&uV*Be!^72J6j)W|%@lk-P&j_rBJuS*An>6r za@VeU$}xgC<)=T=b#49lGxM$|ciVb1Cy6NgfK+YzW6MunyFPS@15ct5Ue-uQ%CEw* z*1iid%Q@HkkJ^kCD}8E}WjhA1bS?3GfXoE_ty`rprIue@5g;1NGQat%~(=kqQ?0C`#bi<0=t2XUN+b?8(b zh}BTQ>xHI+bI$Yp^Rf#nq&$9LUf?RnfXIL;ZhIZXdO;8}n&N)p{$$w(jS|%f23jLR zTa7q)6T!$N>CPCIaiq`7&1INa9E#lr<{;UPCKa^UX8Fd% zw7wINUIvQ_kYChJ#y|ZRQyGblEqdFq4h7}50H#d>qM%s?*B_3JAX+?fQsVbJ`V@Wp zMnEZ)H#?;*CAbi6h8K+HI_j1cqX!|&MnD!IV^H24vq9b}+9~uV5Ih^YTqL}TEMgUqZpOH~UmgrJ zbcJOlL7XKd?Q3e2EUA#NkB5-u#sHc-R&ym~32~f9FcMa2Hwh}~C?suDycDeG&OPG( zvgcPao*NhySxCHHlROXzG5H189|q=Vw;Th7?!ZFj2Xyqa)Rg73r&p|n*S}1 z)UZi7T=2CFiqtikqE>VN#M!qh(@>{pQ_6KudJLGI!RVXdN9H>2Xj~o;9ahTD-?{lZrDQ z=>YN-Ho^ucC1}j49UbjQ<6XwcKfuP$zUQH=>mIaD<&qo>G+@*NVZ-vd10P};nk%By zo|%~unG~gNCpHs7GRK+4T#h%+zTHY@@Spd`zVuFN8YySSd;_%+LOXaY- zt@Q3zDj@c@{poPO@+HyqAe*)!k)o5W8S(jQ)#H@oi@TT!nPIA<`+4=(b3QP-oL%^J_|Z(&hILX> z%hBm+Fv8@`{{7d68z18!&C_63^4KXIQsBVOmaNz@X8|J~21=IuYo>kYZ?MaJWHsz9 z<52=tA0`*VPQBgFpCz)Fc|_gpEGSp_^F*WgIE&JeH=cSe4m9bZ4{muTZT#uTV`MpY za!2au`Q6R=p^Xn$+0IF=57WU$;ZRlog30;)*rH!43W(cq$mzx1FUOb<|9P^D?)s&+ zfrJAIB{yL*_ao==VS|#+652ys8BFVDY^}O`Zay74R?@t}1zs9-G!2WH5u|_yt}uB# zF?ql~X)SBo$`1uIOx~y7Wo4g$TasYqo^hwaYcuzA{^d*IU|bj!XIU&d4WA``bGvok zeD3X?L-ejew6#x}oorsLd=^*cb7$FmNu4>ZlQF-Lh2nZ0sVMyaZDj{0SFc?>nS@}nfBP$_sQ278oU)cZp z#9`oupMl7VLtt|L_39+zEoAiGn5gHN;aSCHXF?4WLbLYIW7p(jt6s@W3fv znB)krdj-@zj#T{|INSkCBKKF;t#V>odU_;c1CqmfA5h_#tsqQrLJZpoR&TK{UZ@1u z*Va;rD6k2?dE(zg&bx{hcDv79AsD8mo-hYN*+B=LFvfDRCY&JmIglhl4hBi*AF3i9 zDRH-jy$PR#$UWI*cfgwlIR15CUzL1T@GwlyCQtAgU{6**<_(@3Mo2G+%ET}t;&*T6 z=(#9b^Ru(_+xgI?00x(5i1J&vt|8&l4j{cmi~iXpHS5j8hYuOu+&7~qefDfyd4qp6 zI`RLKZZ|=>RWeQ7=G0>|VVU6w=z$!AD#*|?Mwz))kz;5YfzRD969*z`zpxZp6(eQM zO`OnT69X6ysS_i(gSaYqH4ID`Mit=OM9)Q4yy8YfTe?(!VIE2aC{N@dY61i?Iuf{x z*d?7bb7ojdwa0#n-;b%Cm}L_8S$wM~oJU*KuhjgXzMjBO2=}J%4y~ZHG?#V$^fqWg zNezq`%A!|PfI=6@9|k1;7TO*c4%`nkBVQ45)TJiCtP1gr@aG73cr|^&)nV>W2%9zW=Fno|uF?D;M zVh3COpSWmBiL(WSmQ>Dt>!P->u;XpFLW zoqpLoXZ21Nnxx}2N$OMe?R5Rex8_EBZtz-(mWydBbARnEnp=v0J>8z(x;mYbd4M5= zIC%0%J>7Wm>8GELUwf{+O_co6JPtKn0EVZWT%KNQOdQ_4!;9z0k;xag`B3AS2(_=} zkBIm&#wmxo*`5~~UDmSb%zn3o*(A5t@^HfR2(S9FTT+J8iyZuCkFVL7QO0TK?si&T z#_`7+ra8+puKt;{w0NNl`yKOQ*WdRai*ZsrEM5{WV19WnA$zK;C_H}qB{~X9A=b8A z!Qane*=CU#CepQDGPXO=!TXfLnd56e+P!?+tiI`Y$UwmvKdQW;(F$zeM{~rIC^Y6< zSBSb}Y(cHb;llCdYxGloI}||5?$p^EcZ|6JBI=(=YI_b>vaM6Kg?m%r;RwoNx z?9Peb9+KS6o}&jX!$!k04U`d$OkG2OR!CTmxPxVRey9VZ4aLq}Wfia#G;IGVnMyyT zGhCY@;BD#WSC%qQz`4*+(3$#cv*U-De|2k7yH&KU0eWQ)j_!HQ1ybl(f0+6lCDQ4$ zMGXDTIoiS4+EXW*g2jaZuC{osgjB0$X@WQ+d*klP>$F}_G+}ALNCtcmE8B*K7G|H_ z2F!gHBqE{@wfd2D4OWZh0iqu!_|1vQKB^3)E zUFo=mN^-C>c=IxS`0(K!K!+UtU?MvuBwO??vbQ!>xBYLV4RG5ja895j3lmUg56nh< zY{o*j3@s(`rl(wvnKd;v1v8F>X`&h`#%lcOE?QFj#d+)jEB!xExB;kfs4N>KR*)9< z5c)DQ(NR?lak7L754|D@r$8_7ijqpm#lX|imj@B~R(VGekiRKV)c>RDf@gr?+G;Ff zTF^)0Epaw&GV;gwW-*1NSX`CEhy__^FCB%bB90YT(mDHj2Y#4=HvczCC(H@%0y>Wq z@{VZ35xV;hX=2%#`9OiehRVt)l%0--T223lb0YWU(l#VwW#BeLA1j>L&P1OEewcPK#3r8YyG$% zhHQ+)Vjgo&H8nNF6&VhB29?#=R)B|%P!$%G0wP(!KJ8so&d??L+X>*6U{`B$j6Us* zE`Q>ZzXCiCh*(6N?~)KoMqA!wOA+oGa+IqW0cXr#n7Q<6X~xkI^4Hq_8`vljp@IzT z3;XI$%rrpWMPGr^3uMM=S5X<3v*d!R%g<|yFOryd#K>U+d(qU5Qn`dl;+J0gje>#4 zwtjseaHf1UcO4a2d$`PWN;^U!h?SRjOpH-x8PZA~zn!tYoBO@)OYSgYR1;eI*m$$>ruE4N$*D)gf)QM@wA z`Y9e9JUWpPCN?XDZ$_{2lb<%Hwc5C&IO^NMT^x0V`mM{3J(fBUH!tRVpJjL5ws)cp zjG#Sy{ldrH*Ev)>kj-C8*Us5$g;s7lYxJ}1x)pscKRoBWEz~^Eu8b>{cV4CcMNodO zT&nN(xB(5^&wOGZbC6ANVzYXr=qHJQOfzxg68~>GKdeh1q_WZR4jtXcEgSFmZJ}$$ zm0pUdnvpC+g{fpLu0P{D4zmuEx@yhKh4sd%ZRsk|^mh~&oO?WRyN_>KuY&cbCUKgImKX-#*N*#S z2C8yJce=TY&w1B=$T`W288acr)uCGhnl5 zaW{(?H| z2jS)xQ-59qk}Ir7kv&mRv49duNTYz@x!OQ&L4#F*RS~+PCl!+>=$e2azzC>))*#Lh zTZtl&or{O&;a;Pmr3JVHd+Sg<01;L94M2YTFFU!Q9LyUuwRoZ0+ALejU94uq;`0m9YTN_|Ae2pW?g_s?&buA~FT4qnKW6yn+_wRg3w&q0<_-Fv&VIjEEcB7?2 z*dx%&pAKh*0MDm2!sF3kPf*Ed@q|LkdN_rCf!G%!w#-lrCjVmsH}OJKsB#|;gtJdg zPaJ$~U#;w-3vmS&9vJw;i$OmNz{=TW+jf-0Q9*o?5Se5p=7@uShbj6&s}$SSjzQv0 z&lFx>(>KJw)Oz!-pH0>j+G@%N{ms5@;FPOwo+KO5Gy$7 zX>%&1HR&LQKn!38*e|k7Pr-YZ>JeolJ^M;1tDJ`X68|dy^ z^FCTmpTDy|zpmA|eT^5x!_~}D7r#1x)?=iwZDjc%%)Ix)72e{U^@VcFm8S&RlC|Pq z#B#%{LfPIvP2eQsmkCetR!8b#%+9=1T#+qWKZ%%N2UuL+Z}>G}zvRy7r=4e74;y@x zXecT>I)-QDMciMGN!06}6^uW4%$Z+!Jn2=_#^KQu??u%@;zu?ItlRXbX5YlUZ`&UW zXD(mMdeQiT#kJ4^PVax`cI`1+bH}i#GcGgz#oD#bm#Z8B5@c{N=GGjYVwg78G}-mW z)Q4C51Ea)qjx=WI&*G&d{)Nw{o$tyA3f9+qma)lhUt@T&*6Y{qHEREils?Ex7CnGj zlekX;y0=f*Yi)F2%#@)Hab~=eTbY&-7bWar#W*l^!$Qzn_`klFS_$d(-LOX@ z9ztl#G1?V)d>|o>_%ZXHgx?YW?IaDsVWY)9&#O5N%b`#U#<*B&W*LlQ2{Jcw z;#6&^Q-^1|$9$iWDu=T!o>>l2!+yeNju#+;5*Q#CJ7{XIY4KciXN8(WVr~FaF!A_x zMbHFtI`YBq?L?DSZ0n-8W!;tI&2@qv!FlV-cg?(c|)pY_PCj z{-4Fs=@?87(_Liwi!Ie-ykKquqktxw7>DnRr z-;xSsfykou)v>s@pc4X@9iRW!F@6eheIea^D-WA2KiWnl=8=_r$RXQHmX`hDV~h+8 zM@tELnQ!mjNY5O^L}O`&xr7t!M-g{Wdfu%N&80j&*>mkvygYMEnY1GOQlA3!~HMWCKA;D2s|DkxjEtW zBt5rm&(}fyjf>qYDPC$KMT;{dl1?xDE4TD{>UvTj$AZ(^ly$3>Lh*0j&|jw>T`uNn znKra6nCE$Ms_L(pP0B!x5oqtnNUS0@O zge|Fr*#d3}?L8%H?AqF3ge)Q-1fsAhZa40aFDH?z3E51u6&74xMeVnK4F2%=p8tGb zOZ&QpLf$TC^bXhsF84iPmqV00`)99M_Kmy1MEslC=%z1Px3+KHN)PqA7o>?;Z=Mrx zytSc!?-vjuFh^-4-!|{~H#E(ztgglJM z_d?eIC?uyp29Gf$&V_-Glxc|-RDn5?SOP=$rp||d$6$l>W14He#9wa zCC9Kh!n3$}`*zl z=KV6d*d#iLYpC%3&23puWF$60wkx0WTs|^c`O>=XC14DLjgj2zkL$uEM(R zaY^iRnXpnP_F-F7jplo`=RAL}hhpAsrssLs&>0v=%FPWcC07y7$};lv*qpx7eL7CF z?bw6s*fPM@9j=aX?Nye0ii~&tX`+VB+6pVr{sOwkcz-prcFO`Qmy&Sq z+n@E{4ih*`#Q@6U;J@1BK0i12cR{fl8*17+oAKp<$Hu;TH5@ogx(?8*H`vN^{(mhC zhR6EyGrk(1f*%Npxq*Bq(xRwY#XNudlpgJIYDM3d zODTDNAYG2DBo}?ggGo%uQMmRl3IGYVQF@;ZgF3#+4pe@?W8saT{WW_;aLAv3^ zsc5F-jMqzbnXVq%o4`|e<}{1W!`X2;;J-IP_x{uOAA)Xc*gv!7D-8t`&(zT2J+=y~swodrPMg}^Nzt3t0-Fn)Ku|UZ>K~!c@xezu-sgS=`Q{@z zt@A7(&6`+4u?41Jlh@BLdYG^Yh^RV87Do7kvbU;-5pnY9xByH@V5f^bh%SfjiEk6K zpR@hmckHgo#6t(b08}^pyhC?>%V29$e&L=okm#}D`}M1~=znblRTx=P0tgj2cr_Lu zFF1!J(&FvgC+%3F;|Vl{QK(zWaoo-|te`I~aj?6-c0=-q*#+kPg|zqCPaFo?yd-8M zo;zk{KTk!KA*q!3g}u=ocs%yvanc>y5n?S|OinqfBNjhjTx?0t6?~c1M69y3 zCQH(lv@G331eoA!hZncF^z8)Q$XO7~q+S&man1fiZjefNn183yd_TZNgp*Q~itgo@Xa+uGXBj^%BmmS5(LONV4y<6JQn z$o7B#RvfSlAeR3j)mt5X`t{odJRxi(v5uCJC13!!{#*xV!La^;qIcoTy|E-)i#6#a zRpZmO9lTT{kLj(LMz%zwm?5Sai)AM~eYY!?&hI7P2#jCql{{Fat!HIlzrLIp;THw- zG7&kLExcMIsP+Q5S5YviC{Ud7uxY#482I4_pIl1T|W(IVtYkyTn z1p%D{0TDSjV~IFv;q3ajq8n=r;uZoT1#-#MQ2{z}01^w7Eg}&WBfwGMuv4ydKfCv0 za};(1MA8FT3lCjf+_D(=iS@X_)&cjLDp&XL_VV(Qt8mN$^J8i}#dD`~o(Ece1CRO* zdYisTHELk#mS64nwysdV^jtjKgTD4eA)mBf_D8ipz0$3J3A{nP*?ZVUv_9`9jAB&h zg{`aK&YIt)K5o}@%q2SK`0cCqPyCC>xp=ob!>j`f2c9LB+{`O@8-Rp6u?Z54@#nPZ z%>EL&@T+eqvx^g}x{j`2=x^J0y?Dx1A_gn8xxRZvHs71%_9QJF6z8z!k>Sh_JmkIg z1lOn9i!N8yuvjL6ZWvZP$}hkZhA`11?B!L6(h2tDf$p%XLiAaU*RSdgP%t3KI7`*- z^8-oHlqM7o5oR+v5-lBa@+z9DzD`P%vHBjgiXXz36_hJmC@~FM24AX!@}5=o?GM;f zFth4FT9^gQEyS&HNr~YVOx-)4l*zjcx6$Bh!Yxb~rjk&a_=kp8{WV6OgNqzwGmqrl zb;U0{Gl0$%4L2(o;4o`fmc>L45>&?ornl!a9>oypE%}LJx37doQd8)({JFoZa#=B4 zt4z;XDKMJ@Qel&oJN@gS{*l0d57hY9=`pae;#7@Jo?Hc2roFE1>py^SItF67&8R@Ynre_cY=k2ES&o=r)vY!3hU5E zuZk=)ck%9Oyh;4L303EU>&>n`yksvq17?TCg$vgjEjgsf0oVjrK0OVaSJ?3X9k?Bv zw1nlkCzr9OpJCw%5t6&c+Ld?fl&2SPbN1sH78N>yT zu}fRUL;i$S^T(<#Wgj(wj8tN#QcP$d`9c@dMK%yUFtaT5v#@yG1r(of)Pt_xUbqf)&``7_?*XtY6CSa>A~5W(LjpWpO5t2l-O$kRQ;t9V841W( z0yfh{W>1|w318mAi~rZAqgh zW9-X~l}BYCJv;gCtn0HeU>0+=M)PayQ>f1#AJ>6%?=>Q+v~cCF&6x>FdtRu*SZ8Pj$$$CmD#>z_@i~{oT0h5*cd;x?! z;1?HSqSFKbe;3RTSXck}AzTW$nur@{KL493En4@(3Z*Mj&V2?#N&W}COyCc6n5XNe zqR&@h=cbSY3?2MAqnzfJp99I?We2{UHZoFs5F3jSNvNi;&f04W-NlNGc)%{5i}+`k zU3P<2XnHI9+6VcC5^bq)J%TBdQbai1{E!L-f;Cvs{eK@talr8Hm>02awCTYEP6*Zz zeebjNvS5N;BY54!YYqx};`$45mSGIyx_Mz5hkg_OxytH@ptN;GpDMyl4F;ASPLi) z2O)m#Z}t}q=r+zSX(VZn{$FWdN@rhK-aSeWR&*nnI;x4LJS*Z?vorI+^zuDsh!O`Z zJ`J7P=~>Ms-ZaCIlI)a<;a)%zWJBZQGVoRCVBTN?J|*hkt>R9Q1QEi`{Gww}ng9OM zA-$I)&Rt)*EWI}7Ai`tFf2~0G4l~R+#h2*Zp)>O>KZoW(+-y0%$FfOhIxJT3vzuX= z1l>sUV8Y24gx*fhLd=C;;v2IuYbP7#-9SN=w;lKhK6s)|>M|VAJST?AagHtpc4pV5 zV=upC;S7(4#&5QJH+gH2uJ&_h{m{ct^YS08bZ2>77Hz2{9w(|f;Y$Ns~a$K{HpJg=Tu^^MNC zNHs}0sFCH8(hBK<1lAjWN@imc4;*>3NHLfGGv8FlPtAtvJ!Z~SS*G3}r-uVkpFBAXQTDQx+3wbtW- z{r;#*8dvTQW58Ic-6;K?qh=AAuDvipg8d9QN3+F+r5Gh)%U=rnqHHI~n=w(9L*s`f z&D%$xQs5zjtUTOAPQ1JMdR0+h0eiUQ8UBcNU`Ko>(_a|6q+~NM@8i%ks(%h&oCD^% zg4yOFXs<*|am){Yi&0-*)Rbcp_ql981fvVp~)B; zm*(h305iOod&8YQu(l-}ivtGu=7vXIW}JWE`@s+QV{CNrWduVNFCeKS6X7P1oGYDm zzWjftF)`QI^73&Zf#=UDs}5X4#`PoRTL$=X!8}#pgO@&eC1165%XbTs=Pf{{_*`n|*z8sA&zrl| z#E9BDb5dQOErgid(6lSLo^Vs2O1QuOZE%&&ZVu7K=J%WJ3!^ReI-ftl##d7%7g>Sv zg;ZY81sMcnMfd3M&`Nv<`{?L>&s$#W2v5~}&TC`1zmWq0g;nLDPS0jm!0cOd6p+!jj-R%}2 zV1vadUa3d4;a*=hkJ}zF<}J)hxKA*ARltHix`BH+Mkc_WTOR5rIRswqhd_#Sk`_83 zTP#k6E(&gcIR~acZ0HJo&M+TC_c!(Dn>zdww+C7`zV_`~4cqvTU0`Z}GvHq#EdWyr zCvabpu+|y-DImTe{r?K`nV2`gj-MC+V&QZ%kZJky4sw=xAaa15NiaIPB67PI+9%?Y z0D0NChJ%DA3IP&zHbC1$s)mCR#soYZCZHMP&vQl69mY=!0b^15mEyPux&>UWjmTxA zjlAEDZk-5@Wt>+z_>j(0Qk{;w(hHYX?Ev1XOcfFklsV6>&OXP@^V6#iHCfK*Oia&_5W+1j$o}~ z+WXZSSXDt4OF$HKkKYHFn_q7pJhk=s3p{JYI?ps%`DI=iGF-<_R6WT3$gR!HWe4sJFs;} z_zfaH^ipmbfcJ<|WmZPO5tc=5nFaGZA02ASrwA`V<6{`V);7&o2Cy5iiKUn;gMm?FBJz?*FEdq6y@cyTRx;s{XZ`}DW zXzWB4H%7SxH`6qSQCUiHEL;};zS+0D!_E$!EynZJgUuqShvc`Rh882M3fW4Biy_ETCy&fFEU2o3&%gW*(xI1KEp67DQ4f zvPe)$?C_QE%F5i(b&Qm78Us3v_0dOJ_l$t`a;cIaCFz*F! zPZBEN^h$pG@dr*6kmSAUejIH}bYj~3{bB|Xp|g<(9Lwnx+q#{oH;L8U`Qyx7mdDkfD=JHL z(8YI6ojG}=poLh{llcf|?B1xsJYaAI+*n)wuFrjI56Ui_N`fx)@bJuyMigvZMejzh z_zt`eo6b^Z6@4J==z@UAjcJ%Gdx4mN8omrhN~N%s8-Y+9E71m@`^<&}aB>{nNNg;V z^UWuaAwmo;ampWm*0U9jPcYT9f*tGV3ioBSv_uFD)8O&m>9y3uh)S;~T+y7qz6(Z< zlTc?uu^v{ikLwI3FDm5WOPa*`?n=3;uVwYZSS4*oK_`r3#KmR(*?#wEqJH)&09kVd zw*#U-;LZi`ON_mC?gX>lv!lCVkc6xLO#xvjI;TWFF&N`pLC*nV(CQ{U#PW;NRn3OH z@Ev}FPWTm!X5rEyY7KlIZW7uxI7pwj7A9i9Z;4yVgN@hu=Gv# zBe4gAKrC5xbLHd6Tb!0(JS@(!P{b+^&(SBuW)(XCS2+G42TJ&p88ul}!@!{VXYRWe zD0A2e^RfqWX&&*C)qlIxaI7(FFZwoG4%&h2E|U8Ee$3B*ASuV~N={k}iH^GLX0gf< zA!Uh;Bm@~VN>l?2U;&b9094}-zKM1u8N--)wJAoqsWu;0c*pb>lA>(E3vYcA`5DSB z8&(Xbp@qf%X2&b1&PU5HWL`1T3v<1@QsOHaFfI%qx%|Jy?-`u9I)}Wv)@T zjKJiyjCQ^C##h2U*;PJRd?3s~>*+b=HEQS1z*7_cG#e1Z@w~afvsKid4=y{ozm(B7 z+8(E4zs(8>Fwy42?=ZI%2KphOhuIpaj&3-yhA<`wObyBgb9BPDo-fbUdCr-c#llST z^6FimvYM@6XtNwGW5Q%wCGsNS)Ra-~Dnyq6=Qe^4pgSGqQayI81Vmz(acCZ?yFx6^ z^XEoYf&K{f!a@f)U2N}c)`!=@=)Hs{Y?gWCN;x9eH>9PkyxqnaGGKV@57*#)CG`T$ z>5Q1Iv7p^s+X)Xzk-K$|wK^PrwI6k;KOmIFFfwt_g8N>3>*9RtW+w0NsY};7o?YU# zyc{zX1XzAH#$bn1Y*CWKO1`*P-zwva;s*b@JerR2(@W`p(e>VOJ@0S)KMo-?m64qe z+2bgpAu6)+CM!u~mYvbgii(Ulag-vX2wB-=C4G!IN)#ng$*5FHD&PC{j`KO+-}m$D z)?a7TyVvVEuIsU`O^so>8=yFcd&?#3y!D|kCoW!W`J*h}L?cEbt`IbI``uA%YLGBW zC`s%T+Xz>eRt+E^a@Ur6%D|{aX-6VsmSnW!49YayqoP6k!*)lxS8R|T<20E_CG(Q{ z>u%nonxl0P!?c;A*mW!v8=8iu{dcO*nJQ56X?!4I>OPb?8ymOvXV4Q9{8YkqZ+%)d zOv5}n>&RBiz@(2(mVt+Et=^mzj%?%U`wsQiMHr)n2LGM?ewm@vC-+(Aco z@fNfKF*Q;TxRmoZ2%g*eT0PJRy@$dl=w>Hgt@v_3_vNLt{(?F9rf$9eZYWdOSh2GX zo-EhjG+&jzhki18%w@0zgBIJBFE+Qnj^#SG@$Jllb6rX*3vEvJWL6Op6Z1b>HY|D8 zP~|t2X{o3Ed^JQ&MwQ}HZsb;cgPv#NXboH2UM)nQgl5y+lXl9~U^oHIzI!7W9?7WL zp{Ay5!$Z-#$yPEpwv(;Y?d9SDIGPQeJGOfmP>}L{{S?QvI@6;Q z2rb3USDV{%Yyn#gB}8N5$l;k}8nh&CPXGaWTrI;3up>|gL&J=PV18mz|27K#JCutA z2px4xDPEFtvaJLYJ$Udye{9NE=v+aC!Jl0NDDi-L9qw)KXj(XUHXip$(RKlhsO*-9 zA4+g5UON8Tl2{CDV1AAL$cVee2Th#M+ngZ9(PviMFtj^NQ`BJip4Y?UDvH2m zC$2W1=fCe_(RTN~fq~Y`n>d7x9KOmzwf3UdKWycQ<(042`c}BQkMi-~yT5UoX4& z2VE9t9ni&6$R&ww?V2YIG|_`3Yo8q7Pj&T~9uAx!7nf$P3pZZ)yboR%)5F#`O#^l0 z$e|?IaR1#Gt(5mGS{k|aCwB0z&y2m9RO16(3OaBvPsukRh(hwXcI@4ox-nsIHYqHq zdak6@eu(Y90nt!e+r3pR13OqY_dKm;}CRonok_(0B^>4<{~I&$#O2tGg!|P@E5bk=U`6 z=Lm26%)ZxDb9~GL1V$TJS8nAmoARG6y=2>~0tjjMEISEcd3&ldHSK`YW@~q8%$`PJ~S>aAlBGqnlA%d6cfe0!Js{YdhgdG>`ykv7x5 zD*AOnu+u$z7?7vPxCk3*^4`z4*3Ps~cd5q(aE?ArIMajR0lhn#vdmkC+sM6 zK-^6!ANnqRJ%qanIq=bdW*fJ$LYN5jo1Oa;!;`diEls=(@ouH|nVPn9PAl4zoA7v< zaJ6@@ta!hUQI73kW4ax3+RFoyq6gH&|1xgBI4K_{nf;pNUD(9R8ql z`sIVT`);%9G-LRfZw~LaQ%%(?yYFNk5<4Tn`S;|d7cZ?e(0dU1)PBgBc9j!~eYZ8- zYV=sSeSB~2u=pC^B}CaEcU-u{RP)ixCR>B&`}qz{*so#KdG?*iFEM0?ohDhBBP-tN zqfFO8)-Z8Ii8sqi+~m5Bi6%b`RT{S3KX(7zrsirS_r*gSYOY*=Q+aUp>^bJ}Y>m%? zsokvG-epX?U`&pH*hj`6dyT(rD+jksHZqPM(^*}FS6uTKmYsPl@E%xP%o55}ol{&16+(XNdFw0@|c)zLX;7jAj1nQV-n43Fad1bt9TEAx@ z@kN~?3&N8fxsD^q@C%E-ei!kFdg6kFmfq)^s#Q``IzZ#;_UiRjYcoP-I?Tt1N43xl zR8tIucA*6doit@i|a)!ph@g8y}fUET9_ z*59nb_}>L9zpr#sS-&>g)GzW$)we|N@W{w}Ps1RQc>{-Lg};_TF~nBb$-ccU&QGY0 zKgZM3DW)yI@=B)FEF+4^+z4kP#fRzg%`hUy-Dc&=`=y|kCvyQ~I3c?eX#RbpQU3{iB6IC%LB4=03u?!dW`$HemvT6 z*6HoV7rN)cgJPF8Ro;?c4-(YqQBfFA_v>Fy|F#Lnz+(EapgG5FVpN$?H$F+YO|?!I zWj&pyT`Ya# z^*!!iWSW`yJArA#atterpSDD7z*0^fIK9BVxZ)g-jWP6PhFz!@I85WQ=ix$naku`T z4EhUTKc)o7ScMty8MsV))~s2n16#$^;m-08kGS;evqfnzTg<62>vZ)N9cVg z@lB))*UOmegu~3I2H~zm#t4RRv-MMDb*8&e?6Z^`laiYEn6id*?-)&$t{D})41>Xm zRif3p|NN$*?w>F1pE~~}vSjaytoEF>M*kGiobci0jDeh1SW%QT!77u3vwEx9087#D ze*X6cWO)~CpVJB}hW|rhaN*M{mwU2h&b~PQ{7`qay@U2W>Ybt#O=^c9VbfP%V^PT` zO!88ak|NuF_oICv+Kf?$5oi+4V;6Ii*>+!}+t4y_L&lG)Qh)ca$gQ_9I9+o>olRT; zf}~jGN$#*2{Z5bs`HQ@xP7};WM>b`r?j^&q6nq8lL*UWxT!kP!q>pj|8 zUCXF8)%t629~rExrk6X(yyx}HFAB_RVsVLWapledXWK)meJQkRnAjfAC>edi6+ zW4+qaJ}8nTm5ft2YFDHyRk=j&a1NU)wlVjSgv}khb{&#>5Y;=mQ4!%mSX!vcoo6|B zlK4CBcph4jK|5$9yoLM0X%~g`=9AdcyVE&J?$_#+(=qG-*z zO^Xbq%13&&HCQ{)k%mIoKQw#Ql@Ff=tUi7z<<0QbndZcoH?3W7U5$NXa|}+VRd-jb zwy1mh4IM^jEg7VwnYaaTy^RhTsK()!9qX#C+A>f_s5Ho6Hq^buL=@BWpT)2i$b5>x zIrGf`IFm1hJvjk!NZ+ETJl7#>ozvr8PMVonPo4YyoC9J+-2(w(|O-?qxB=Ok<5eQOH$Y$~k~yt4gB3{SoXNsYz(OdEX3IIt58@a#PBx zh7MsYCVh-X>lIjlV;7EyR$f=Ns&`Z8&KskHmHXVc)-GLtL^bj~XkTd7Mn^V;W4xC( zBUF|56(*&~fnSpcXa%4+^QauD@s^DFg59wd<_dT4Mj+RN6PYPBqEc2*$_QeAO zH8YK+K((7V5~(}Pji{fc_;D-!d}4gR@NzaXTBvp`662J)o*~R}fO6b{9T||!xXJf~ zgGQQ-+Z)^Frv_5ziI1esKI1SlTcbIg@2w$pAUj$FCsM#jd1HJEbej7-oC z;eTMRm;0+5`pe&rVDR8rx~8L~R5I~tIKv_n3(?fue|ekmQs?Sle0&*kAtd4A3;FfE zf!oBbOXj4F=b0dGuceu_(4b{G2S#&Ha5oivJe!Eg;>^UeP3v_3brCHNi|^UW>2ub% z)3aQjIL0)sU2|Or21b1|^!j_ssP;25T6Sr#uJ?!!C9p|Y>~7K$$-s?zGT?4gwWH=i zbGQIW^l(dEU-#knm%Sh=kvs5>qJSDDJ6}_Ie{1v;gjg^lM>)#a_7iC{G;BmQ#+q>V zoSS;uVYtlbMh9|uc0i7k+Hskwz`vW^m3@{RK{hqPUZLS%(I9pD{deL#LxWpEgZebtT~f$XrVx+K$y%3vJ+xo=k*?(8!O+M5d z$@!@TU)d6Al=VaJ*EedmG4ZlhjAHn!RHwPQxfav2NRkwldib+z?yGq{1MI6Ze9vqd zby%q+d0;tH9E zw+PRa0mqa^2~EMq;zM_yb@11gaiX1uZ!HTcfM@purq>=kxV6oRNj@8HMgw@$?0qp% zE84;7)2U0Bh~&m zD2b1O9_aQ*Cfks1y0|#ZEw8H#ZvNT-^p*lC;8MFKRnM*MqLQ`o_8{ir2nl_7xmM-1 zpOBGby9;TJBA(Xg{yY0k%lVS+&ntw#g>P^OgRjZnNmRrxiGvm04M0PApzq#aa7t;6 zb#$Y@U#$$*pA=2zJ^2k{CmkRI(Nuo}AYye&Jal+FCZqF0Ro;~+n3)Ak1RRk27_Qq#Og_?O3v0@|CMw9zaEHVJr;FH?5R*g7x zNTM(q4>XCU<}7H_o1Iq1Yjr9!?sOyE%MuLu#H5m1W_^cxcJTr42fQmB(<)2nQvv(o)UqrK z+FJJjQW@--8=w34;mxWEzP>hA1eXDj*vHt+M1}yC5j`fnH4?m9Z3nF&>3D72EK{EN z#fy^KuBKM!Ph?c~M9=!C?&<5$2YJPxJX4=guFEWteX}uR0>Z_tA*r{1ert{L3#lnZ zP#e)6TPuN+l6s!!10WEF-Ya5C*-~I9dC7uh4}D6u$!TCdt5}|ku$L37#rNQGq~4x3 z*o_RF3a?^$C8EQ`yS8i~E*ZoJO>5u$r=@vK@07W8pvJ@ZEA*e2#u(V|=a&5kmYZ>* z9a9b6pZICbq(6|OKO*D8k{R5vqvFigif~E}h?Og2B91(LFWIG=K5jh2B~$D@d#ZqI zL>ERyepoR`c6M&kQIKO_-=w1XW5wkMq5(s|aD$9&Gk|qL=MEosg*rNNWI5DPj4!b- z#0)4VP0H5$iXy;)#Q6$?VW#=qYgFYD2@`s8<4i!Z9sBkL@DGdfueWEQH0V-vx@Y$q z_BYnlq!`se6`~4PAytwpG zK}Y|Pm!o#n8n)N;M((O7xf-o+y4ICjmwVT&lk;#x0e8Z?q`E}aO1bZpyw%i9L;rHz z@L7{`9*4ZnT9V;W)QFPkU}WXnE+(cnHhV|<*&q(EkMq~GYkd?uS|)OjwRG09htTB7 zee7?;6Od@VVNuD)c6D>1L<6Ies8Rof72C+eYS>!8AvVYC?BP8nOk2P=L|Zu*$41;{ zcyM0|Im9A*-#-8Z*5%COrTn*!>o|DXZr{kh%GL!8aPft-oMof#%Kou#^MCIjD=nwI zZvk!9Yp+$0uN)FIN5)#pWSzv;sh3yhZ)K?#t<+QVsdFJM0K}9WdpDa`jSjYEqEQFj z{?!#3&fg&`R;=i*qr||pzchgiBQiSf)?kqtFyV##+k>oASg*X=JaD(TeG1Lq7LW-)M#3;hJQKYQgI6qj)Iqte)so*Wzl7(TedD1&!(s7- zX`h~39ZGhb*s^hXjctM(4e_eT`eAq@$GOsi# z9J6DBk_vK&Ojz^e)5ZDI9N>80kl?fa={N7h3BM}b2FXJKJ7kg=z`sTbRwiazM~><8 zqG?T&v7?db^;BXj&db&WfdC_?Z%^Ts<7`M@kZ4reW=Xq+H7x@RaCc|&r%JWrx)lR9 zp1{q#pgkGLIYPZ~mlJ+S-n-l?XNWjBj}Ef^DVK`gOJ( zf3{uwhMSM?@MhR5%JF*ZUh^+nYYE?TX;q&dTPAa+;J4Fl{)f9+#8U}<;kAq#S22~# z28a*ncW5=$vz?Tn{10!tKx=c$6rO_-ukuhrzu{|SJ0Q?7Uj(Wni5DZ~dYC}!zVrk! z7j3CEL{#go@W_4kjge6$NK3ZKtQUBkUDe zjtFBTqLBavXH-~J_b*8rR1G{2|3;Tl15@yF!%+Lt!*IhO=ooXk2|9WCB9KoG%e_zE zFFcjMbl|!-UhXF*HK4Vhj}GXj4HtEO{K5TSrL8Yae+r=Vn9HT4RvpNzuda&@rCwRR zgg~ZXle+w@TQ_7x0TJQm%zS%;X7Y9UF6y!3vJl_l59gl!wrH#wQ2Q)n87V=porA4v zP5YE%Mk{ig0(E@WlL}U(hg{U-I<``zf%>s4%}CJ&6HI@732d4A$_$8W(5jF3N=>E> zfej?kf$RL(>J#_4LJhxvJS#6}L{<71x^hVo#G}D|Du6c9U)2Py97fVHG$0&5KnQhjLds4zG?JQ=N)LU{1_1Wr<$ra?$U}&*z*NKUhUg@qEuLKhxOR zy+YHctXy)HTfFQ=qSrvn~^4MDX{%KtpZnM#&N4M_Mpe@XVOwy%@yxQ5sI1phP9L+%v z^+sc7@0!z!F*4G5e%QDFyIk5-a}`r@LKwmWr>PTkgIW8BMo zvD9HJ2DPOW>rhWv(#sA31`)rfFyT%Fg%H!Kl=pwWVyGxrqoCrMgNK`GTtr}i2vr0I z2v1_CruZ{MV(#3|z!);K9>sHd%j$CISQ<%Uvd!N+_1BUl?|0w1j=d&LoalqA0otN_ z-ntHwom}(E{I7t2rwOq$|CD3$!=PESX3Cz~_|mJz!ics&OYsjby~p2Ix0p77l-tpY z?}FNydyT*gc0KhBO~h(26EO?>}R&UCU1`JS+5qi$u<4=_%ysmRCoDi#YB77 zk^Y5RIjkh!WJY7_>Oo@j`*TBjy<+hAP4}<59XM1fKCh0QkR-ZPLX)F*=`Iy>vm}yf z#2^iYZr|lJp5N}X7T{OTFMk1G!F`{wjfBbZ*}8YHR-$b^7=`K%Cia0J8a(_GNR=$8 zHs&d}Q2Ri2#VIo7?s9k+GJf4XU}f&(n!T@=Y0Fw3VR2|H0ZmZLS`eLump*^Ug8D?u z96;``X>^X=;D^#8QJlP?O-8#tIuq;HI>fOPE2CctI#_8 zyMZX!d3~FMP$Vkw1CXUm>n#WtpW`kJ({VJjI>bacsx)bDd?)kv zCJ~q>l|G?gknFlaD{^1!N%QxMF>UYE@2@_KQv1W-q(@)kaIO`gW4ZLotET2po@(WR z9UG*C7l5MYRbJ}_8wc%;Jb;?MC?kH_tJJK@pe@`rRq12ROmp~U_y>nn-yln=7Tvt> z>g!+IQ1iB636K&9lh<;)xOPO$`NIkNmoaKjwz7@ zTI1`4d7utSvNz6}rW|~D`9Jq7&u$%oWe9mm1Q;I6U|7_p*y zgAfu@+-87vQOqN&{V3xRnm%F3$@)t8%JdJTy(TkLL_U;%>Jsno(YbhzISGRO!)P7Lck)|L@CcXLAO znKrFh?y0TX29#YGI}XUJcU2m<@r52(Iz~@*8jnOK^im$Y2{oC)&wmW|hhXzIojgtc z1Wp~OxGV&wB_|tzLi&Y-gltc1VHz|igb-u;V*buATKO+8q4(UgmI6cbZ=ZRqmGrX*ov9iqC(SE_(?2m_(bHX*|z6{)Avak2rL#vu< zhxgf^Tq8Dbcf)-_?Hdk=;~?D;9Q@%unMSX*#oNtY2qkj}Dn=;rne)@)A$T$+0By&7ygA&c62dNFkPK|<)lB6958 zT+1o`{Hit67l+|N*Ri}MV`>oo@IzM)YAZTIih(cL!VhUCoM|{Uh%UAx#HCSsY04Dm zljfP`O>3KDT{hn4SrWY5YeGDC%#15)ii1)$@+ZLk^L^1dXKP~09V_(1sTY>Wz*jyG zn0{_a`J@rmm&c(M;44bt(H;5FV=$eSwKz%p(8j=s$goB`2R@|~%e6jVdw;p8k+@CK zvl+f#-VFAWm>0n&LcJ)yrAZUbjBfFN4|4{Ao!EY2WC9Flxw8io)UqSh=sRjpNM7|0r?JLTyDyqoAx*GE}#u*q8Vs} z_`8*C0XTefBj{{8j$wRC>jY+c^g^^XQZdTX@+K)Q&mvb2jh=pGBO3Jaicw?7D(91< z@@$@S*5KvyY2$p{8=rhP=D=akbBzt!&a&74a;aMIcdvc_Gd=D*Zc|3h!i{o_xm278754=o!Nk6QdIJ?X(4zzY`MUvK6>uj* z>-`z17#KLG6~gnoWZ`I*$k4vG_jYtUtlE?P$i*!E2V>E$-R0?7Y;EH=FVuD8&IQxN za%DX`=T({fS4_|`^KYJNJ z!9}h1&8I5{O~3ZDnsTr?5j5acvpRhg8t9^U^+qQ?fB!Q;)Xb-+P%qVI;d9oQL?{~y z?OLMhNr|nRSeSa4^F^?wZ|!QUDY|dhsn6DpN9cL&t|&(luGUUU%(!)HsF-s7Q$?=I z_KFe~&f^mD1WB~CGU3AcQ6Bw6^~9v8S>0&gi@s<4ushwMI4b(OKl@EnU0F4y>R1k? zXQ8Knh=)W3hZ_f=xcfsqQ7rnoZb^7bzU>W@yNWw8w8wo3LFY=SmT5X(&kNK!+P2b< zT9S|jWzNF-oad8&DA)_*2J6QhpoX>QcnS5QOvGw^+-`r36`Uz;7=5Am@Z?}!JCDM+ zHG79yvdgMgGk%3}Sjkbw3;xz=*s7nI5C7D%09n%G+6-E8&*=d=Z+sJGbLSobI;=GVKC6)*F+;-v<>`t>HG zUmMUs5&5`A%RS-okt;eC54vtq9hc%*KcJ%QWU}(`v5wyNY^2BJ@du@P+MZ%!pt`f? z%k41+BD4f{aw$iM6I&M7yVml4<0iaVsPFOh#k|Oqlg9eKXZIt0zFj{%=}0?c4_|P! z1ILd2X97{;IKulvT>B>sMQx>tdn~H#($|Yfas~W_AXu1Y-_asyj_u_Y^`-OXC^*=@ zMccLk99{=j=cn6N2=%;fZ;$(|PGmBBDd1dR2A<9~4!yOxL%mRtNso5_6|JxnVoTMN zJ952`97$8%Bq~`oDmnE~O7gZ*qepL$)|y{k6)e7LZS7dN6S?y=WAM=)PzsPiCl;}P zc?FWvk}@UMtFn)0d9g*$5y{O#Jwe@PUPv>FA*xTTlpWmsdKBJH$f;} z5RAKUga-Q_seJ#}4mcQ&8YMb4t7P1iCcSJI5f(HrF|?$(E_yrCWsmOBJtg!sQVr{0 z%<}bkyeb-R#7s@rh+FbcIaH(^kbAQV4b-;njm88_3wHMEbE>fN!m(Y{P&u-9Rzq-6~7#%j%z%9MH!6SI7#B@r5@UlVi zK-7M?13JQw;d%^EKOc%xqb6A}%*lj5CzcQXJb+)K8K`%hj0UcsxF#kQul2ryb9L3Wr!NS&=JR(JK+HvaZj9aET<0wi}3TkiRx9(=Z0)I;g+^&*1P z{3p2d$?e}Vje00Byv$HiDoCFit##?Z>W<3N_+HlN)9UW|hSt(%-wWQO7O=K!NXwUP z$PW=oOHya6&@E`_Z4R4Nt;|n163Qd;=ut%yDqhj84-sH?%dID{#S%~2y|e3GM%fXC z05#ODa|0uZ%)@(-_v)=XkTo26X6ZE_$X}`MXypzt?)Rp%yf^ynB8zr%RZr@CzaLeR z-0lwXjZu$AJ99V|KKLWC0~b9tsFLuAgY9^A`bg-HNGmCbtf~v5fCVwBo%dxV^*u4^ zez2vkJ@J!DhCC~H@BniTB@tyQo9OyTBT9{xPatxtnc@y+8!L`37f0T3*>3M|ySLkz zZSvK9EKu)ryjGJdk4A)wlfG%6)(?^%cCA-SOy!U2cGOnwMR6(_S zqJyGCek#h1zG(kTsI1VejN748A0H6t65K#yqYisJ_3G7l?%AoQEhl46%+aiF=@cW- zEa4WXr03c-4vs!;KgH1mB8CG9nF{GsPxHREUlLNQf~H1scn>vX?Q?p5Q*p3dx&KM)(JULw{HB6HTv6 zUbeO&Ku#|fW{1=Z*C#X9pL#@;l}^VwD~RZ^x$&~eD2>rSXW7Vfa`4GUp~1U$Nv(up zXie$r0Z$N+kpGt#v-)8tyb+RrkMyK%Kc^tHM8buGsT7LpPW}yY^yW*w?I=LU1nXyV z+SoaO?%$16985KIc6Bw7M83Frt%qcp{BR}dDX901lV#r{U}va=I$*w6wCd(${;^l& zt%s_UpWagU@$I!tZO?2!(}f%MuhLMSdExsH?pQzbnKCO&}A}^9RGsA&@y9 zsiX$8A7*8kn?TkIVJA3uScA3jKca5}_Ip=)y$+|PENkG}6E}-74zzFm6(QkR~d-y+bS%7pMZKh zOaLOlQgPT(^m+aD;kfeX&wdf|;fcj`0Q4pa)&08fx}B6jM~|+>qtZX(=?nXe%%x?v z_jMn$gm296-)-+r`}wE1{F1h|U0zk{{bTZJ%Xe2@iykAD&{VI3J3$zfxOpdQn5?Gi zQ>`mfRUY=W&%rJG&Z^DJjn|tDjLeYPA>2Y3d!rutxm>MttJ1~nz*`P|Ec?V7RYAznIX zUj4s}G`T&+o8(@)INHd)#(vUaU`&DGFnvi(isL;h!iLo zg@kaq73UV-iJyVFPxQt~z-^M*(f^`dBzmJMHtQ=_w-2{ySdrMx&t}4|{Wr_LKFQsw z5tm_7{Y3W;=~VZ2!}+oa!fU(^nqBeUqrQiC2={W;{yGIwRSAz5d^@nj`4=o6u20Ht z@u#%OocpV(-N@KZbqXKL{YHg;%ksCRxg61e?bkZ58oQ@YO*oYaU95f2zE~*8Dl6@S z^Hu&tnO#roo^&^1U2u>4k~%7mBG@VmlYe;&Z2GokZ75tQ3q;xWIU_b-iD(PU1&rTK zr-8?r9>W54e8cN0G-!rB+g+G_{P%4UlG#|T+;y!ubUMiR`gK|NrH`$V&Ix_ZxyUz; z%>M6JmUV*EBoPIyEfkJ)TF1WpO7>eVXgHEn zO#Uf_$T6NCypB<@vymfVZXC8igw)ySUE%s5<#=j$3=S&)V>7c7@O-%r2DT~D=d??1 zvhnxIPMtda!^g#b-ot>O!K-c@1KDg}RZ!Rb6WXZk%cE*Y@+!KEu8vklB|?)VmdY4} zq0fWYLAr0UKr z=-#tuoVY$p>n11D{Gzh!H9I}6ch{~r*^|qtFQWTm!GKWZ*l*WOCn}6p!%4cejlmne zXTq3-7^tu7aBjg7Y7++vE`QM(GxlO#adB}MV=vfZ*z*ZV(tGejz8l7#nRMs#x^6Fz zFV!>Yuc(*Tti@W(pq6(m&V5R(QGFlSALc&L|3emoVollGyx%koF0Iv7Q#e`vQO_l2 zcz9}GslFv8R+g02!`c0$y6L!b{UiF`({mqcMF;Qr*)48J3j4FxG$29lrLCOZZb~Bk zdU-bhnCMw&9{+CHjF>iL4mHEn7YNi7pyfk{bK#rLEh$tsP(vNuzyC%~*};Tu%05Xf zBygDX!aSQ}P&*MMMmt?w!^;bbQ1NZoNpQN#Q4JkB?J{5Y6x~pU9 z_%*BRRiy;Cu4z+4{^2koATH4|>n}`CLp&nAw-PS`0UgfuefH#^aLM$ogwx`?A`8=a z;q`6Gbyz%`h9nL@)uG5nt@_m_{bW4)Kt&lK>D_LcXb=F@u9?E}w4?&F)j zc=37fB6s=~It1l4F?TvboPQVw4KJDmnonhDYh8jEaxPwnkt<1-WN+wcY%Zp%q+@@wgNad{{HcgxQu+z0SV&{NdqC&xWzO;VdlZH%suId3h~A zEHsQoGn#p;4p#$3ROC(PMk>GIK$5JPsVy5`Xro6*!6deY3FKBw|M-`FgmWm#D3W=A`Yd~EOWeSWc}{~Px$)d!E& ze)h)Pe)38bYoOxpd<)cyap0UdUX`HOv(RtiV2j6BegYxpi$HF3Q)8l#b;MIB7T|Sr zUG%()vRy#1ZimB{u0WxIU;sD^og-wv>#(8Sk3~k_fVN&*k~c9d3osCdw;L6D>cFW> zF;fg&^|7V=Uy^!l}=tOp* z2+Xl>x(3ks*$ekPaA}7w_a;PfnEo#+J6Z{>{+q_m`!=Q(h3Sd%Kc*mRWnc!_mD1`F zlb(`k)vQH}x*{s%%cH6qS#t*8y8+8)UY!?^vp7h>&O+&b5)szfs>>FQ7BFFZ^8>_B}* zendz(^QWbGB5Ys3tV|qMf&-S0;^H`kSY2%2nL(%ih*cfX&9U`GRW+l(^?Eg%(9gj> zsnDQWBoJx*Z$f+X=EKR)L-WHD79cQOWHYy+TbK9ATh)rG1x%_6o5W?u00`aT;fBvT z_H5a=FLPwl^2&@iTBz>1D-MtEraBxv;RXe%W=yD&yHY6ducyfwO?odIWfbZ#5@p1O9R4Wg z2Y=qsX&BArqCtM8n+KS76MpQ{R~>%O^Ml|{sRv7fD*B2E>((EFJ-KIFEFD|hM?z~o zNP;#_QE!h?oA)25SL|m^dWH{Yxb%l5pU)PIU_X(W)CUW1bmedNf_b$R*_2D;^Fvjs z-js%ok`FD4qn-AR?Ls}amBPs#l*C39rT_*_BE!DPGC`$TfA{^vkzEL%q1GKSV#NHj zJrKn+YQf*#wdQ^>dl&w$u(DD~+IxO+*}?F+FuBXBW_|3jbpPMUUosZ>-LhP`q50Fn zceOip9{NeQ`UK13F;+J4@Ad@Y2^cME&!A^~dLG z(O19t%Y88P{)2>Bxtuki!%Y?Td-@97yuI?A^3kQjrwX1%f8`iEFCz%V8`(lrD z-T*Tpt*fy?GYxyF3S~*9#9iH-Xe~ZH4``;TS*KdubTjI!%d4=~ev6-Kjvu6bY*e6* zEp9m6?05?1uFp|jx5n<8o4~9{fnFCqQ%9_lCH@SoE5p33>Tp*O1TP6Q1hqR|rR0<@ zoJphBBXZL47)ltFSd8;H7DeGiKbG2K^X<$V4By&!E&^i zaiQ9*af_)rc}5XpQ%*=&n=spP>NiU7#zR-cERNppkz(zm`7mv&5syPVUdwsxsJj)HZs^K$QS<@b6f=cCeS=j` zXjF-PS;tJ@MTdwI5dfSf_3GWEkPlz^vu`1)EvFQ^jW%;)0(E*iSBMaye?R3WxFK?g z>xpB`-gc2m-Pgz{ zhA;1EQ2n~pVl*uSMF6YRz+T-OU7qE<`6n(SllkxyDZTUmcj^X zW_eY*Ix5vHs!CE*Jg()e_>`(i>_C%pBTZ^%(f>mkBsx7{pgIN|xa$bj1P@6!6c+l+ zm*wWuqf@f577w+sun3-m2=FFd-`kQU`ImA+p*SY3-?s9SZ5bJN3fsb_YU`KpAM@$) zUWMN+i@Bp~A6dD@e01&nsA^ZSn^~v(XO6F`$2!suw?A{=#;@piJ*AV}pmOKIJ};U@ zt=h7EIZistw;2j|1~u;>0APIhvz2*CUD>Ahql^&?HPq@q!pBTws|0#6ct_U}ou~x2 zyQ-VE2DX+noNA^a6|Nb$-8eo>DwzjrsV;N?zrii3R5FH>=AdmEWFVD`ik|``i&-tuC+++2>*8HU) zljeK&zox^KFd>Hfy0PN-x*^ZFmFd}C{{Hdt0-SIgnSbJxX&C9_z;V4#*I!1#fuOwA zlZZ+*)Wj7X8S}?J6f<-$CB=XMUJwl0`a#ZvNcI2`MOaMukE6^`Ceg3j&Bq#8zs;oT z`BQ>j$Y3pNQq4Q7#o!_ekaM3YVRO&A>c2mB<-lxb7Lt(8=PW0Ccz6WI=2uS?;ZGYL zr~~1`MI;WqY3_fk99v7i$Zct(nMH!06qR5IPTKzBgjZSL@%ZOD+;7quq^KNVH})H} z`t$Me-S}%A$54%nSQ)@uvjn!#z3%QYE)&Uf=B^F z-SUp=-xtQ3K$G#O^^el>{PcROw0A$hzvY$uK?nT$onJLN{M4+Z1$}z{(R|;VT3Pdu zGxD_~_1|KP&bvoOo$eVsQUu-|=P7IcWkbVFlCcev)B6XQfME=)Oa}jUJK{K<{A)dW z!<>mfm9!BUUokm9`>%Xrk zzneFBKuGh^qPI{r7aiS??-^osIm)`rgevYzGEnc9YKlEO=CopH>rm2wQll7Qe|^R?XV5k4U-==&F1wfZdUkU9X4ZT3Y2Jol zC8pofH53=20I38?O6rBRF4sA6Z9HdUebE}FUmN`6VG7|MiJ_`AGscPa|1{E+)C)+n z1CmxGvrnXJSSf7SADuf7_rc6!Il+ zMY(pDBTz)Eoa-Bs0$KyXK)yt;9`D3Y>9gSPLz7&4fux9|72%vmx^Imd@T9^UvePdn zh+EC)Pbj@vwDn5|U0c$80qGobAMW)*3PD~Dl?X4XT3d1Q0v1zhU&N7xXkoZSQ@)p9 z>)Y>sk_9sox?SHn}OL`n%NdMwP!r7M_@!Bj5fjEb@4?X~|apxVX;K@x(!Bd(7A?eFtAPDlTr_ zt<)5)xt%$#&G$UUUD20Dw!NYt_Dy0wl0T)KKwOcW@y9o6X5p*MdlGB6xp~hwo}QGj z9V00gC6j%9-T$mC<3((vbu@?^FqaFzmRE@SvL98R)f%G2mtwYN&cXO!tNjGGl(P^- zW(w;hFx`TRqQrxM$`W4Sy_{hY?q7RQ&>%#6R&C-jbm>EakH+8bPtOs$A(@_%*P)Dr zF&mEH$GYCL(%b((jjeCSLF>Mn_bh^_*9Ci}?5j@ABx^Fre`MZwJYYcM3u~tM( z>NN7X+m&Vmy4x-a3eXK%>9_et@z&G6&DIsoQ{tacm-ARB@7-@q0}d&5 zLOWU>j{Dor(lNfCB^9R9pQS!jnA?z$HWcvJaVbh%q5odg;iP)`;vc{cb;}ab9}D2f zlv6nVGXn;s4*^T|Pq;~2Cu_w7|7Sch@l>j<@s+_T)U z3u}kcZa7E9Z?pauzrMohD8dyQ;JiwsMUr*`dQ{tVe%5S0xk>s4S4b9;UT-_bgfp=_ zJ>^Ryi8QjZ8jhfY2yi5|q_x!h$L2#&&dc~w%}ky)>Y_a9*#JPKVewev-e-Mi-QZd9 ze^`jR{aNIcA1!&AB}uF#udiprOrukxh==ith~&+Sy+l?bl}dP1k@p5^AX8Lt0yD ztbhsI@MTyVJv14{arw-|*Er{;Aus9BXZH}J402i$XoE%ft{@=D~FgPMXYZL2SS5Ny{ z#<3xmF;fyJ`bO&Me5mu_eVa{pbWi3~pZJ-UocK?b)oXa4v&!myt#>EvdZ%*3+V~^{ zO^I>XaGdPzuKK3im_ydvt8nx7>qmzN%@KfuvOFNXSG`)$>6>WQL?)C#T)YqiT^(sE zB=mn^;f&v_&vO}8omfcoE;cdWfJP^CbZFMe?IP`{HCJe{R?7VTE&I)`hq&Nrws$G_ zx(q4=J35jcyg-tc7$ryPbYX$M^6IBEQ|7w#yjZga&HNsmNrP7A9U3aK_zXZyOuEUA zCFl4tbEiub(m&A|zT38yI;VEw`u(@w&W@bufnAG9s8BYv;GG`rOcqF9=qieyW0ju- zXw-y8mix><70GlTuL>8XBl?1Hm7tSN&EMUc00lkelzmO2iKZE0fm6bsWuqOct

z z-ix%g-THx+AR+APhytS*;Z;gOolO-X^K~0pu21Koy^L{*Xhs8|sQ%AHFA(&D-!lfBE;k5H+r+N>vW1#)#hY`y8Cm zW6<5@`PF@_s)?3C%eiNo$=Oq9kJB8z=f%SgOXAGXyAQ^m&JDd$f5ZvK9sGppeu<%f zcWe!t06V87r75{3NdiFrnEIAm{{e=?Y!DUMi0+d>9xn6|8s;cqYr*N3q$bs`!C%~G z@ymfkGuDX5<~jgY)dO>f;oesq-^sorga| z2`X2Gj7bR8aSagWi2uf=^2DzDXmCDUvR$0D znOWwCP3^sIP&N1wpJr~!})0k4k5b|{$8paZB@hnCeb|dBLdeyC@AQzf!ZMnX22T~ zUAnl2fER+mb*M+UhOc!NY8J>KA;DNPjY3-kao|T2khAknB0E*K-Tin}%%u_{J?2@B z0-Eb(M3JNYevng!QlKw2@|r0QuYY2*li4Dlv30HruJ95%K@#}5qC@vSKH7eTr$w!b zEgMEm)T)t{I^1DtHtx30#ahaGho5wK}0PCCfc9}dWPO*u_}XqUwYk_w0ZpxLep=EbrX%?Ozlku=PUrYq?>#58 z53{?0p(#1O{3$4cZqoux9BpoE#51FM7fix8jUYAAztl}8FG}{> z#MoXGNRNKmo4v~lGFc*uzPyfHnOODz#%yYTsgHzbC8{b=(}y}p4uv^Y2I&9>b$o* z>@y|^BaiYBw%8|S3OAmJs6k7QZXf+#EGXv}1*f>}#)u{|jAA#>sCi_LDxcPeRl`zB zr`G%HnoXrsGeuU(sI9j)H13N8F17laytUq9@uNo_l&8an=*7FTCe3-Baf&yVJ1E}# zbzHAP%6(UA?%Nz?k&$E2@>Awd>Z(Wsj+6!tDWpb8WTY+*eccY7^%V0(_ymQmm^SUh zQS=C}S;A(Js{RS+#Knotmh?flqLuP*Xuh6iv+`B{6VtDoT@;1tj|a#Q;Wa?EVL3#2 z%Rm7D0%f4G2-_pSiP)F=>{~_U8BjD~PT;@=MKeos6Fo}KqqZz}zG=*I{04OT9!^&M zfnjwSIKwv-dQEWX6WjN!`%Ca2GBOZfyJ5r3-WpR}h#Y#A75TlCs@uum4Q`$fK*ZrDS%`JX2y3YFtY>mV$tNCC@+Zxw3qDb(CcxtZjwD^ zQ{vQ&8w$HgJ-HE!|M->RM(wdeoV92=ON+_=T92^)Xm#Sc?p5ksHu%q@U&1b5zu0r1 z3Ec$B>goMi+DmzZ^C{MIqvU&)4vzD5U~0Bfa`N`Wl~;>_KBLKS{_yFF?b=ui>T&K$ zAphcXc(>(+rvJg)OBzXMDcRu?5iTim4BWF|Aj6=WJoKS={$ElH=yE3TJSK-d>S`mL z7VP{nB5;E=h8)Ru4pN???-Z37<*BMQ^_H}{bi$!!FV!)Fh~l5>tLBy1wbMS|dPv*-te_I&D7UU-RE$iMN*|JeYFh{KHQYu^`pqLj5S3fVf{i zbhLDm1(-FvvIZj*{ufURb9GpVTFMvJQ9;QwuYd8oakErej}TF}fIAfvCS(?&B2Z@|3GBq2n9gBl)aC+GADgc{h8UK5rg%>n*ieFQJ!P?h#@~SmO`&H zWlBa4do_mSsw?5lY*@oeKqiqkiP~DeYM4BF+PhaTW^1f}wYcrDZ(Bc{C|ZD!jY!T{ ze17>?V%s0v_}!?iHq$^PfA`o&3=4(#0m9o%OFU`c1Q4X}M3?kIFI^%I(T*aC=yvG4 z2?5`qiuSe6D<0x>pyx z-I$&!RRpO!6O;VQ;t$tVa#?OBpZ57)9MQ`^y}2Z{@`k%QBG9Tu_Qm1xpwRx>Vikm_ zo~|JDQ!HVWpc6-0WXgOCv^ZPB^?%069X7L0VL_b2A+RebsJu;nN_2Y}Wt?eHSQ|8L$4Bm2Tn@puH# z=SSKpa41jRSI|5JowaXn~qWG zt?C8-`0$Q7Qzs3U39`UF<;8iwJ>Q2Inv*~1#s1rG+SR85qPxvn-^6&$!?{ICVr3yV z)M(-2jVrIbQu;GrT|OKcPsW@Ce$)Mv>-8H@g|r(`Q2ymqXisRp{~WK7LKT4UHnA%n zV?Xw9*^tuqCYj5rZ|BV;+(r0r@cZ&MR;0Hl9i4y%myj?0vN}xGDKS0W7@oidUyO7DQXRB%Iy*=6lF~7qb!5CL08E^k(eWbvvW>IV>kGI$JwS|^Il=^n(c6-D5&~+ zAyz1u2{OA!#)D-jL(=jvlIDJCSP6|70!|!e6>6;s_BLwX?GHp(MCyW z=&2`<(fG~#lO6v1U*TV|i^YF`%>>wU1w|BbKjM%ye`Vk2wFI~&%pNZcuiCdSzmo9C zRZmJfSC+~d!RMmwkl61Bey=~gy0Mg=TcoB=jkkb~!#A?V<9lOL9ED z*5~A{7=+@jE|gz?C~s1|?WLfGI99M72?aTxhahWn!5~TU?ao>34QTn-Vp1ZfT%s0dkm3Hwo;K&|?Hjam zJk0<;Fi8kYF}uM0JelHw&ML3=!BccYTX1(aMT0b9Gjdh5&=S@?3vn*NIyAo6wbmvU za%|4R=y^i_=!8HWnIpgnVcg*>zf+4{G3a}W|FTQy)4;yhKmV@U_Mebh zTDw_Ptddu!-2}7KqBQ-G!K)uREu@WQz{u4M@7`zAK215l@UQT%*vvAK$9(ya(veX5 zw96~I6TSQ4{}Me9z0KM9ztSPis!%u+YGbI3=titBAuPR9wr$oe|8I}IE*fghjsph#AG+Q=uIIdq`^Q)#M3yOQLiV+6Eh6g>5fNF! zpdutoQbY<9V_%|FgOo}}Xd!EMSxUB4q9{s|lFI!&pKoU7x_|ethd-`|Yw%s(pYuNF zbzVy|@itC0(rA@}gQ!f#tXy1YRAK&Kr?4K5d+;Qo-!1@_s+sJkMvarN|Ms0b7ZM;` z9QDx7L5L@G z>?TULO@22dU%YIod>F9}`peylo6innU|l{^PcrT>5jHiw+}Lr-R|Ljq;E1b$AiR$0 zgBgmwgBK~j#YXo&u1I;|m^Z~K;`)jA|9`&sb@dzT`+)d^q-)lVV11c6-{kwR0Gv0g z0dR6zfqfOQj$Gp-h%iTI!{kt`)XJb)i7;whJU5sO3}vPIJh<;GO}$-8X=zK|_HaL8 zOY`x7aySp(3?}!ZM_(C~9~Z>$%8%Vx^kX3PpB?TOEeN9SB)_)DeS;>%WznN1W3;y_ zBvCrW?UH;Ys)o5z;_Kp)- zj{Hj!&D5F((Y?Wc2=&6wqTAsU{FO8#URON3%A&g)R>kSh|9C70BOCm-RLKwA$7O#W zYGsAr7DB5`fUg;IXk27a#w=bSjDT?PX`Y6bphXv6^$5QMosB(;WP|aCwcFPlLwmxX z!>M!dg*BZ4i6s2NH0rppTZI+I)b65Cp(&V1NIrcZSj%V9#EBb#qZ^%bOq;A2LO~d2 z{MKhegUHh^UTs3%K8<$7oY6`j@SBRiueu@LSZN&yy(A;2hCWh94C$-hw*R)Y4JgR< z5q1TWBg_`79rM5+159M1#Tv+KEL+1T=r{{O6$|_ks2>9T@S9^ecAC&9Dn#BiT*ocX z#VLcDjqQk*#J|8eP3OKazA&VE<29s^`VGN02Utd9!{;U z&>W@#i#dT%W`=B6zs6Vv(#(o|Xvo+hP;>%6F(GYbKoHj!;~W0O{fH>MVf6r?UA!Al z<54?jkq-USOkxY8twF9esuz(`CU`dW_AY~kE2fZ@4yp&guQK;-P-)%$&$@Wcqi^>v ztT^v_f9U0Gmhw}gmmLjM2yYOpkJ95KXn`)pCMBZ9*!7aR)vjjwQuiP+`97g*Em)DYE1*d!YkM-K0t5Sq;ZrV^O#FmkaXXqU6>!` zF1*v56aiE;Dw|X&qA|KPDh^&@0upJy7Sw&r!)n5z!-)&uhq#1N92ao6D9|_2h#EL`FXI~H z-d05Eh6P4R47y^z&JClUySAE{1^-?vWB}EW`4k?a`+zER`BY6J)=KhX^G~4~gFWEl zGynJBZ4q#Wq-N%e67Iz4*Xb$bUmNX-W|f{)fxHyc+7dsJJ`;kUH7%{H=8s9|KXg?4>}64~Xyy!z7HyXwBIz zCzAN&#DRGA*fXR)Nm5Dl0kx=u05z_lNP<2G8n4WoPT0xNqn2NEMY=5}C0mCZE=K=G zyI{yCyend@(Al0w2EDr8xfU`ytj122M_bz1_&|rqBuvLstWvgNn8_rJC�N_q0QL zEnvP%QuyG73s0yfVTeGYW!focy(2+1gdm3|=PbPFcU+qQ!&acjV%^}S5$@xDSnTi&<3dc=w}_u+E!59#+nICHg}4%*x{A0OdT9f zN^sV@rCeQCc^oddd7YWkb-#(b@fVjLUAKVc72GhX!1l<+PEzLoY&%q(*>vnMMK2JN z8y1FNlZLD=>;lQnmaEX=S?^&VO@0)bTy?RSt5p0_3keg!w$Duz*ue8lB`)X_V604h9kVI0IpMjy~}m|9h1$k;#ksfV|V=A-wC2F*5G>x*%)&01fM~C(t9_ zEy(-r;h!Mpw#@EZzHh2BIOyT5vXTOrI?Yoywh1)Ab@!*<1<~Y!PcMP^>LC{WrB?lV zx&Gg_f3=k811Ywm?b_`0t6@FOT@vElcmz8EZYE>Hd3GCJL==<-rm4mKPL&xWC@?E8(SifQmc8~2Y=s*T@tx`1?OS4= zIV6xdIIHIpk1A2(_%4$$HFF}FN>gH%cM?a=y2I5&VDHeX$Y)pjvTI*V5IF%uxN^1i zpPeFTNNkR{B-U;dsSe}8J7&Y*g1k1e1Q4v8o*U~inyxl{80-htrHESu(~$|cR9>Rf z`d|VMGvRV)+K-_AR~gT>A}TwD6?A3sx$mw&vPMDTQFg$`k7Et2IL3W({~xRDi3zSn zzH1h}THNseW0rk)z`d54L#8egn@|i`Czicv%^=1`J{*TEosis0BU#&58dad`%%(f^ z_VFpYlk)7;>2nICSWw?deOn(y%f4*BSfrlI@UWs{gMY*W>r+k0Bm! z1N_Kd{hBECB;XaSHjt)D)%*{pbHb;(iYo33TCDOS1r<`PA-&2=&1qaP))OI3%J*8F zUTM0JD_V{ofv4)IolD9~on;+N;ZXPMQo9<4Dn7uXl8NS!Keh*4R5?I6J%fvgUJ_=^ znKNe$#yl}(tVXbgtag85ajcY0=`21^-9oD^Uc#tL^`EdwmZaKoWot8+M>hVRXXp^- zZKhde*+4V)!6`eGDL63wxX9*-qUCU{`~DwvXwyIF5W=CxB5!@YDIKiR5Kq(2$0m$J zm@A$&3}%BZu#qtmUsT1&@{b{@l~diYFt9yP(xp$wZE3I^B|M8fZI?VKO5V*n`3UTAxJ2tG@Yb^}2Sw27!|1*`%Ze@S;#iU=D474P#)A zzoQu_O<*+)n63bpUO=+et{3^>U-HvT92`aUqPAQbW<9-i${cW=s?ui5*dJr8jz8w3 ztxxN7^X^?U5Xd2mH7t0BHUW-WR0fy`K|1PQH;g z%?l=HRO+E8k9lpYNHO)W?WL^~xWqQfTJaQE-BIQv5}s!o)Qon{Z7OQW;0q`lVd^;+ zL~UlZS4pW|UCJA{fz~_Rme-Mw_!4eOjZ5uibrL z==Bu0zT6Ps8vg$G0Ea9y`dSe*$;_KFg}ff&lHqDEw{>zpMOCsAc(}$-ocK6BnzfD4 zqtPEf4Npq}9~F5$Q+*o)gR?%5^$){2q?C}~nWw)Ok0LQWV`T2E`^&}huhUo(y$F?< zX~{Jys%U9w^WoaXK@Fx2YNb(Q-J$vqn;)_pPtjKn{MH3;X6>J<=;bui&6^BZ`l-8# zuC9*V$)z8Ib{tMV0H{$(&;x1&kv>co*<|cGSL5;Ogzaf$8o<#Gw^Wbtp8&)p*UTS zttce6PqhxK38ZAj`@ZC#22JntFWiE%4+{wyt!zd_argM~wB@)H6o_$pujAcmYI))p zK6K{H&m+ac6AGbADG$4_fA}^6#B?#eSWFiS+;0Al>7s^(9=vc`XkkF;&(wD6I|8Kj ze_U1$9jX^LId@+!?@WF?Lo4m2{oL3qp29P{N#k#6ltmbt$Ai^d^l!-FA{8qcVS!u* zOZw+5g54I{y56+y=t(NcL5jO|MW*v%^W!TVbOMh!sq{^0p~jV-I-b%rYmNpz1)OHM z0)i0scdlwS0{#lRN+LE!0bkr6=0hYx-Q!jr`%dj7| zH5f?@Mn54ry?V3LSDiV2h1v(o_@VF+WL|pz^%wh@tES$Js2ggZGL0gH9}i2>a`mf= zWR>BSK%}E)5d=yRetGZ^CndgGX*;fJaZ6pbj!Hx1`8`YdkSsnux_PWPnA-DAvqlyQ4BxF%SsppDF0RA0P&|tXBhE> zY^5zZm1X98=K*xI`|0%n>ed8G9Iifi$5m!}{`u#n;+6cjR}Sfp`iB+K^4#(F4=#bA@~WocPGHZ}bGWJqJUnLG(*KQ^Xe^@{srkS#Kl%1E5@ zG6<|%fN3CPyN5*$pu6+(T}o*Jgb&wmHCh}xd-aM>UM6}5jjVyt#9oqF!XwT9N7?-A z*`7a_Y%1NOS-@SzbI~PSN!t{H!YJ>g98UvV-RN{C=AOC(pxDX~Z>b=;bdID62D~VC2$QswO z-gtUp%{eP%@9**7IHD%am)>*=zeb z)&EYt5PcmB9mJpzz_Esi-zCV2X4G#?cD~gdtM&Om8klK5?=a2wfyvYexc>4J`1?Z+ zf2LyZpjX>lS9H#u{9LNx$Tv|umxsmy0TJ6Gu=1gm=6=_lZxxDa`3R_93FNrGuWKy{a0r z5kp!0{v3BLrgiGlWrMJwL1BNlA;h;P(=NrHUwkEX-)Xz@b?L?Z6!?m%(9k(!jm~w- z{yxXFms)p^HhL=ISLlzwSc7rmzfINAsBsvG$hIganI6nC39nbkIu8J3f1mkXj2wI=ze1)faO z7U$O}5jI`2hLzZxoBeash|$TeUB4_d-`Mqzv@1~lu-UTh3d2=2T9C*l(bgv|lPxo? zjA#LHbf}r=gN2V+;X;W%^}?S>oTf6s!RF0{_$&pF9O)(xa*xnSSD|~bbPfOPP3}4! z1DOc<28qsayTpnDne`SInmHl;=(KJ6;`7@4PD9KxX8-gb0aN2AY3_fVN<2O461_3| zt=A?qvvIxk`_x=pl3{mAR@K5`9M*03Vvoh)pbEmDwm&_dw@`pJX4#JqsSB4kw(>Ks z7}5FG>5VJrOsm=TZ?9yB6(gTs!AxrB;E`?W)Oj}QSnT5$UmVuDMP%K5V$|n-tC%&X zYc=Bk9Mh=YM*RETn-?T*J zf;kI;wBWNmq7!NIspv{7q#FDiE)fy_``==r%Vxla%>a+hGL?6G;Wt939~~^-Fe&$D zRPcO_>ZZxSSC=vemw!E!_6k*PgWO>oyhl#s>lV{N$#AxshC%>yi$Sg*fe1!Arwadz zJ!i{{;OEY-sMC`QHN8-8qE4|B&-KqyuBms{*ehpVZ#q3QW<@6q-XTidptt4l#j}@Z z4W{JdWgzF=;nDT)9hLvoJNWqb)O@$&)*JFfj}Groe~e}eMnrb{m%yJVWo|e8T$`5p zxJBnJX2kMl7F=R8%&(6ty4(o!iJG3$YdUfpn1V+S+{=pn8g0;C!*dq|^9=g1&)V6p zdQ?oc5f~UAsJnFj46~luW?~4RHI%9UvV%dbwySTHxEc#H7(3qLP?5J|?@tWM!!~G4 zK4JGk{UZRkp$XH6Z+1?ROTE^{EieQ@S72mk;}a7z5X}1cJ)7lU>umPCuVbOQ2RQ+Je~H?dPM7SMp#XchVmQsb=s4>6&0zi+|O}vv%d$M z&AyafC_ec}k>X}9;_nn&Sg(pOKO$75DEU;%68ObKu|9?s3sz89AiQ(M9?1iH5z}O_sg$jEIC3pV12K43!HSg zI4ujsB}7~<-?|@X$mW(&r&N9oRR^Zz{^_4`L?>`@E}gYt5|o|1QoosmIKlB_r%+?p zEzQ7<&%1!x096(cQgznFT=&BX)yf5{Os#lpTU(u0?qy4H?>-x5#)nP|}pN z(gtZc^|n2;&9oc6u5ft9`xuqJ3u-=S#YuzpS9k*J$iUxMU;jjcFl@1Z*xhg9#0PL; z1l;73oa|#FsVKB`1_8mTtddXRzapl#_X_0@a;8c6{4~6v`*N@ZKZmaD-*09IpHXLQgQ}yR9UpbPqUzZW^ox{&j>-a z?bNM%_wF|r%6SPM#00H)x*h%IdU4&2mM2hRep{P&07@VMaBEx)i@MUQ;M-K)u)Za5 z>q?{Jee^HPNE~3C6a*8OjIwm9gj*D3DthmT)E^z3^HG!8SE@tRSg7J7dk2?K@4oM}sYq@gzc+(v zFy1&`U&}`vJQh8|Y4P^GNYz0OegQY;vZR@R!Bo=?x1ZrJXthl@EoeDSnZrV!uA&m$0^twjn5+9XSOC6ZWBdxLgL!*-_!-?H%)kx+5gZkvsp= z@`wLK|93)loYOhZfvY)qb#EV%IN5$3bx_4~rfIGsVJjsiCx1&8aRf;DXGde591T}v zNtE-V1O);o-V@^m*^2R@u~XQj7w}d5Ip~W}GgLnb!o*n@{>tn@D-$ydW&s2X)p6!n zlHjCJ^qx0*qhI;AE}H`Kr5rQ4>H^@D+G%FOxD=;5u? zUF}7|@C;O}wNjBLUn=o!t-}NvR{*bT%g^!<@9h-ea!UXu< z;ECDn51$XNOEQ#XfI&EJ{HXdF7JZZIxlGd5pzzWuNa%FWwT#$_>jcHF0FU84SJ#Vv%x6DdtW8I3!7(udtI`)>xJBkP- zjg4pm=5@bzI9lWK!NwY`TSpvh02zN1w4N2^Va}i5jKU)mM7) zq{AmKO!wKj>8#254mvrTd)wAq_4`PVC%uB32yF#3*kfAozkg+qYwXj=a~~?Z3~?3m zy(I1!bI@1EJ=@}uM>j`F^Rjbbbdv0D$1my=X|NP7(CypRp5*h##rOs_&BR5umwACx zat(NUwQ{MAdg#F>@rc%S<7n2bG_WZo~FzYpm@hQ@V%+E@_1jc*{ zL_NNMRypHVM}LQu7JIPkiesN263NhggaGNoKBS+tJcI5OpN3KI=VP>um{kj@;uZ(G z1tA3O79N=RLH|6}&S9H2)r?ms!s|}kQ4@|wEb6PZM?L0XKW^=fZ+MsTV`Kl%Gcj!}-p>ai}Wnk$MLcIi0W)U3jX&4W;jtomO+m9ae* z1~jm1UdYXj@f9m2rie|-^Bmg)(tGfrL!}*}P$y$Nu$;hi6{&IKPsKmrLJ4^zUVD(V z93J;F3yuK7t1KnyNLpH-2mTdW4Z{h)-oaalHE5#ZjBZEd{~7}8XyK=btWuJ+QZL|cDE^gLDAviXHOfOlWxini&x zZS?Wn>IdiS=N_g^*U;~wp^+NPa$HcO(P?&3V&JGc;sDxL0J@HT5v}%f`-p|)RLIZB zUjp@H7pCa^exYLqcU!PvL2Zh6^_EtY{l4CDhoxX_sPB=nzK!&GaYl~C8=uJdD`Qk_Uiw6&cAPFXLDw%n)DA!BoMO3KFi{?9Z zWsOKFtXq=i3~nfT!7<0epC#Vzc&nt$uG@#ASF!$M%#8fHC3ttNb^Y>@{8#bQl+Q!& z^!qnwUm2-5^D7#+|JB{ts>Z(UJy_A@y_@$S!*O^A({!)nv%}un-mb!CsRtMR!p3^_ zY|byK3Jv9x@A)qehBlCf==sLL#dqE=%V6(M>`hp22_1*j7(nHEUZY(V_rt+1V$7Qw z6P4^+J1_o+8+w*L=^GOmvoP|gf*D5D)`M^mGlVgS>0BMV#S=nlRUrmh*7Z2k*WqbT zSdQ9s7j4H#eZFi7(~V;(fgTakT;Ms=u1lBD7ir_AsAb&BE|fTW1e2q4Z&up?$wL-F z&m3g$iGc*tmgE|nmFBx+h)Vhi&e0nNbWg%MEw5T?+Wj;n_fM>4p}FTtT-t$yx}#dU zDZMx%21@x+{rq#2V)r=>w5BJY{cZJCw*At2#t*x(i^LZgtoUeF_=P)>4aE#G^u2by znq?o;Di-bQ_7#EJqAr~~3xjo!EZI!Tu9{$FH9D4;ukiu9#%tm)@`VJeg3y`eqp!-F z;wK~M;>8(J1NJdhupSHbJ$z;ao=~SvtqK#0qg5w-WEaN^MNI(!A0-0U;W^!ROwv+G ze(mV*8mL8%(lyyKp{4Mdpiwkk(S}`pyR3#oH~YosVuu#D@B$Ov@XTQozpmR_Sl@9g zY}&GA%e{r+n{`GZS&$5AWw84D^yyFUg0|zYW-p!z-g>9|_U()LWN_9nmL9d{nVbD4 z#@1k0wDlfti*y3>=(B5zPy$}T!v_x@WJU9J-n@T50TWC5ek!;&#{etfX3%#Y`A^+;Yt;bBm8ru3!TTiX7U$LkC-A8B(Oi`%grj9hH)~d{Rk#;%Kvsc3qxrT>WdW zvKA`Oh%0-|7f1W{R@!jq=NEm|`@dOkFAlB_V@CIxW8HeO$FF%@I^szOYhv=R+CMMB ziJt!4AYv@gEp4(d(o>Qn0xde{gwquuo(wB>6PI!bn$gjxHpy5!?@rU{bSkrqx^+`W zlx0S6{foG5iP=|$=MS@Uo$paDNJ+{S5?II9&XE$g$83q7xN0sNlExjzTpHTI=$iW) zzG;mt26@@w-^eTwB5eZ^X+j*Rc}u?LYIO3{Iz1w&0Y#)qv9d}DurDHx&cJuZYFtUEJrkrd68XbHE@;Q@!$Wa^lkMylDyDhKU|q;ZA|Ofjaht^^`t4`^{*pvt>DV z@Q;f8AK|6ZWjy|~Y2UPt(V|S}?iHopo{KCA9I?Cn*`j!gmma=j&yU>6SOPv*<83Jj z33tB8EVQys1)Bz)w&D%{`0DthCayexgb^!(O-)L7OL7>J6`R=dFi~kjz7K$?r`4fd z;8(_r?)!d&)fs#5B0|226L&p{%pZOHG05G|Dn5lsrL@+%r9bcj0p(9{9D5WGvvjUz zuNe_3dZGidD26JB$sW5>%R(J3EiI2_<=3v6CA-!RO};%q!_2HI6BplhI=mE$%_2Yh zp3Qp_Oay!&Xr5V}C=$n*PwMk|RX>?qudEL zX_op7CV~tC$xnrN&wk(O$|`TB4)`du=nX?^=;kq#iNdYbs`RS0uUz=?3ZchfGI=<7 z)t)Re&_?T%f@(hO2waA8YQD(6A$DWZ>5Y4i48u_xRrt zlfK=1>q3A_i>G|ie5q2n`A6P8h{A$RauNT2CfU)jJ}%x+ms8OoDGCWX|<;o{r6 zZ-&d}h%TF-BcM=dUUc4M^5*xS61D9REeRGHx})4OxdtdX?1yMjlqhGt8~1L2OrAVw z#iX;HpxBbbX>q`V=EpPh8|ehrcOeO3DZc+lO@M^RyqhG2cn24gTIl4k2|neIP0AZ* z*Hg`qQ5fRTd3<_tBs4HkCw;!=M=~6NtT*wZVI;im&oQd~UU@3vnXtu;gAfxEIkP;+ z3jSEKp2ZEY91!!Hj$^xo?;4TR2XGl9D;kjarVCzvToH8Q#Hie-D~~>YFBvP5Q+7W& zopu`h`u@aYi;n~8**`gHk(ybf3Iw0)yT%zm3wQ3RTySf1a&s{RnN+Cbp)bBpYLK7Z ze4KQ=y(`GGll2`0>x;Q-rov$yx?+U^VuLyxE;kI^7~njAoTnro(~+ybOU%4Fm>+7s zqIkq%5K_iC(>Zg3W)kr}n;d9Nx8Ef>y$oJ-r8G3^rOBV(1I=p9dY|6^%l5aqqFVq< zH-Y&3%@1)sUv#sZ@sqdS(a&&qs=?!}t<~D|`Y)M4dd1`>)o~|70IlYpNcb=5K%kGi zTkh`+{>YVHp3n>%8u()BjVx9|WDvC?L*$7BcBjHLx8EI-Eu15RMb9@foz*zm03cJA z%CPFa!h785jUm-~bW@iy3rdm!t0?Odbi3lkt&YIhcKwaZ$5{y1Kxo!@+A9n&ikVpW8W@WDTqntueMK zd8HhZsP|$X!|4yzS_3rFi-yMsWO@Eg>7>*DUAEZgXAZMSD!ntb?e9IC^Q;Xo5cr;(K@p(E|`9Ph&3 zmz0l?$nSZv4G=E+od40oOjFC)zX7rnez7VUu-N`?ZjoKf@8O_F?Diks{oToTE&08> zZe8==qj?roJkq7ejy+arf&}N`6ty(0ikuF0*=>oIfPLt`=_VNKa8N_`(!A5G2H&uw zY>1|mcq;mfbF0JgSrB*4*;rrXx7LtLoCZ3QC6)osO5E4*jap8QHxXw^8pWDrE$w(5 zu|&)$a#-AN%M_;*TCH|Wb8c*LV_4mRC(rk&y19dCuAb`McODqsE_Q;lM|o(I>hS4x z&>em28*OwRz1$K|FfxB$(!iQgSMezGq9yONmOZ(U3$}KyB}6gM`+JUM4@b1V&fGS% zwRcQdX{k$`*a~P@#X|`ZObu2R3-+JYW|c{p z1Gr9c;bHEBC-d%$lf?_eRt>~AgZ}7TOu+$kKDca>8Pr_00`z9?!y{0c`=VZOFZE)l z0)fl^Pf^qb=hD3eaFA7_AJ=kA>!fiFs{_wBLtVfx)Ge+bmCvCw%QPr3bwNmWM>2bs zz3FI@QSGdmrlryxQ=b%LrAarseQz1&{H7r6UfUyi=ve5DYkM3S2(mX-+GAk|Te~mx z)~ko%c+;6~-;O@&=XkiyX2mVO&+%|54e;s zMXa!AGEdhhzdyhBJ}~BX!gLg{91yT*&xbdodMGy|3nSrM#;yMQ(K+idVWe!@{5+p& zsf13|a*lvJ(aYKcya8qe5BltK0hJsBcF2=uCs)U}D>m^0kil!Ivhlrh@YdYdaYi}I zP@*Kzs4Hu?d>P%`N*7-IA+E!Jt%tXaV#(Y&zr3X^w}|ebo7sjvNdFewjwEgtsQlh$ z7V$Y>n5lF;A^6g>0m{+hRvESPxuLttE$K35pD}NZp10aw3%lbR$SWV@xYTB50SHtN zHR{e(0n+o@CC$D~8Oj#{RtxknR!| zYIW5zBNtYZM041U0rjTgM!30q_=(^_ugCrfScz6kdKZYMRdW$7NUI+ECf6M`tr!$> zhgZ#ojwWZbYt!Ez`1>YFlT4d&$arl`vXdl3p(hgPy{9x<`c@2dP-mE^<3+b9USm*W z)OX5d;pwN?zDZMnD79tGWNT|J7UrOJRN#i%+S+f<$otb=D7=5%$foNXpujN%N7a)2 zylx~SR%mWB^&z2|@7 zi@wdbma0e4s3LCo{n5lfYCT|?E>J_@?7*)lc-E18_)sY3@kyhSn=yF!@J*X;$rYVj zbmij;WkOqTcQ0j_x)qaxO36p?=79Q1@sEK&N-pZA{>`>ca_$@#-3Es0N`fay=yOG< zmU9_K{YE}rWpb0)v-^j)a)rjHl&#D%Lk*f7=RNE0<|;?-ZkD#&8m7&UuWbSsM}&vh zXSaQW237`)tY`)c8Q7@1F>hN5aT$74Kk((pIRl1ns%Oe~4TWdO{*#NiJBj3KaJ~oE zt`3RLe_;8QS~PHH*UUTiQRl7ilvmrVx2op#Ukgeu^?&y*MrH7NWg{ztsky4Cfz)K5 z=N(SQT8DzS>D=q8-NLIPGHz7+MvFLeIOk=%H0NxQ$iBBb_CdG z)jmDQGN~mnilAivb{{T&&$|loYQU{Q-q=ZrK;bnCwW_)NDT!#{Fva%uO3A&BxAW_7hr<}M#75hMV3>BCbwdTm#LZUMs|iHv+0@r1Y< zaZO{8(b5%cMGX~U(CO2SnAufLg2g56CHcenXrKr^f?tDWvh+j^&}PYAN2H2#z(W;} z$%u{KBj-?XHHS?q{N`G9`!EYLmNb$}DbI?1>bDm&ck0I9U=@YI#w2H1g#0ml&ZX>v zZ?1ppcCwsVQ|%Y}-v3!3ejFHo0aPxcXrGr}F!VBIgzH;ad1*z}#VP7?#h2qI0jEz- zc;5r%H={sOPk9pgsp`JA^~vz?`;HsJQOK?xhmy&(xjtW`Hzn`kOxxkHwA_Wp3gFsiQp;%o-#WglyS!2 z=Egqh=7@5~+^K&HQ{Tpt;)MMvs^s}D@C~~L^=I^_UTJJe~hgIvbA?{uk`4Eg4D%2W5+>N z^!f44|A)m`V><4Etz6!UJv0mTDI#={$_vls&v`i~ z_@>}1D|RkGI)*oB1yPq=L);-5yRSF$`i*{@GpC(w40+ezl#St7-%V@`m+ms*462xyOSs=URTa-#U za}ggmB&ND0=PKARZ^yp&s5_K;A=@l6-mbT0G>=8t*Fn~~?=stuQoG-5mkvh#8`{)X zP-s&Yj1B%-nqQ0<-ZF}PGM;5K6uus#yButbH>DDXF@5^UFJQyl!Cimi6Bg6hrGxF$ zf>>%huQv2EpE>RLzH#4Jc$~m`-f{qa1%R&<(hZV!9Z6rdUN@QX+tGqI!hL^z`goP9D+yK=3 z6pI}s4?fC`STu&!eo1L=gv{IV)-EmsMd)8rQgXyFD(z_rAWiPKU5L<2$0_%7h>^n$nxzEWTvexr3Tb zL7cUy850JK8 z7!<(!i(ARjtO0~iY}a;{O5c-Fy`5TBU@IN{p9%`v8%T%pE!r{eKpv`(4~Dp4=uTJt z&t|2yiIvoWoMYh4>ljVDh{fnG|FY_GVStteZet@L0xZz)GCK%4S0UEa$L3kfgbf0N z?O$J*B;1NZXSVIo;nce3 z^VuGmp5|cCu3NXU3+cllf4KP{RpuogC5>SkKR);P;gR*iK-S>zUYv{AbRy3ub@2_V z_M~TE+L6oNj{cI9{M*&Jped$%=Rt3fJxg?Ip22riG((GRy$-gi4NepCyMgbE7Ca3& z9$AmbFJJ{og2XS!AD__5PmS#-_L3Fbj>3BYzS!EAw`{8y%;o5G*&$P_r3w1N54>z<#Q^gUb!5GiL$pek$F=0+}OdQZ;Z1Fhr5l5gkr`)N40+-mC8 z%&WalLU4KFC|)7o_p{~8dKx!Tu7JSFe5A$QErU+?Dsyb!Mn`S$n*~*SH25uDeVE4m zj3w!GCzE}+b{ar9iU+wPNft0zvhyZolA(^wNx^b&1;<8{U?~)lU<#m^2h)8Gf>AXW=SyLTl9l7Vt7r^-JCV{Mw5%SH zqW$R*%}_gBJ816eY|UpY8aNVWn^^YJHPwu(vkaNDtSVQepSML|Gq5A)YR0wm5q(I6K(IRWq~B@_n5QIpE4CsuVSn`!n1$p&J-oi%-> zK~B!impAwFG$B>|cy5#xHh%eCBp(E1yA3yC30|=$wZ6y3e;*_jx>2e&yuNd2Z5RmSijw8ktFu~eL{ea|@uwdh$ws>UM=x8jzIWfORn)P*f9?9^ z!Zek+gsF4PqYd4@x+r|7yVa)E8{W~$v@kCnkn%F^#cnq-&4;?<>^o|_4b{N4yeOr= zbIxu%L%>RM*Sa#mAGhe8vsfA!cHbchA_r^#+ujAnvwKW{wWp2(8w2}Emt0U00hnxn z8sORgS*lIkDsht^Mw;`bQZ`{?60^8xBOSgKd8{e(8cTL|u7Yf`HU5mEyz_o6$GG+X z^?<|ckP97FFP%a$4*>N~bWi7AKs3;m-=a-EC_K+jpTyOVU#G?HiY{h;Jw2uC?V2_#t)Y5C_#Pi# zi3?tKW!KkWH1l~N|bhA|EIeuX-2Y1*ZX`F&1`I6d)G)6`Oi{ z>9jdJdiHyQ1+{Ht@WE#ei~n|%!`>2s&OzaYa&v|zL|oB4R;6l{Dx=IyEn#3u%m!8l zGlIA*Xc*aDmB{r)101h^JSZrLjMJT`F1Tac50i;Xe(C-4&S3yBdMThxA>SHK4&4LT zt!K-6Qaq%gv6}Ike6w-G13Rq4GA=I1`1{DoR#skUl#?KLdfu`VUaI^Q%^DaTwbEGuqx2S-O-+c+b_|N?RMxSu zv4->HA-h{e{4sLlCG`$5p6tD6AzTcJgyA@cDyr3fOTC@C;t0%hInieX20KXH>u%j_ zsre0P6O+oa#Vw|*nDzRjoBS%RAvZW+%D`Dhy?SXt@)Nm`_pYdMG~_g$KFn;@Pl|yZ zoT}#V8v41Z?cigilfJH zOG&^uGT18XvzuM~^5{2NE4~H-@IOB*HWHvFy)VgybI6H04cD`-@+Fj zoFMyrSQJav%HUB<*HH%(H!pbQ(W~Q-oJa}JnptgqRw)zU;TI2DBv@3BSEz?64vkT630d zpPOK7wsTZ&b4^{^am-?^oxJ zV1%L)xAeVG$77v1DB+rUb#4aH2D~gaQ37K{63TXzi@gp=!Pa13%o2aI7(c#>*Hw*E z#&d5ryz6h$8uTeZ{mBL!A~X3jjz#gCnPdqR7m_L#qYcF(eM?u)FJ4@~UDEDDlT2EI zGL0_nDj#@HSO--QF`nU82xvFalInqDTve5jKzfu}J%9Emn+9)v-oqwFlkWSmBtF)l zFkS+-fVgl(E1#pnA+8^q40J7N>qJ3JJ0bcn=E-`_(*wQnB;_nCQiRd=uky!?-T7C2 z6XQk(flA?Ko!YRv>B4*87bWMtJ#;^*|A@ohXYkeO1%jj}$w{xdES~%XTs-S4n#Hi^ zhnPZxjXS=Fa#u|SEw*o8!=p|0x1~}U&CHgs6h{Dh#61;EZ3ST7cKd@D0r8h$BrvJa z4c3d2m4*0@6}gU%*b4y&(0c!~XEbbdwmTuk9AbW9c6fW~k&44jI~AY!?rJRUHS&i^ zN)Ce@Xa5WRsT)vZG1{`5Va6RjKHM>K z3h)b?Cz`32y&{q~2hW#S3;>oT7)q~OGo|l7rV`FIIuPfrX|;f>F4PQ4Tl8A)3&$qjz+D@=Ng;HEfD5}ibQ^fpMRBL=R4{FwJ zgzp90AzpSZxCIf58KyDp6Dg#NQO8c69xAsusoT$=KVO$k8!4HMh=r!>?u1;k4z38Z zaZpUm^9WXA>$o)_f3^D}mD~HBU*%4x5?_Q8LQfnUF{fv9309CiD+dEc)K3@2DNn)c z!f&6*v8%&ld3y@dUjVcc241g!k9)Lr##6mLc?K!#DH*#u~{rO`H8v79cY<5kxR2K9EpeB$7?eUzH;Lk zYO%PAq+ayGDNTSx3tc*Fs(@^E=&)aW7;$CA2uARqr0Xe#OToP!{Xsr7Pxtxd?|0h0 z1nIrLrHMxD@D6g1=>m9M$@X`zPjHcyFITLDuFT2nvCz(L2!>ZJ>W|oWck{c{*mHKq zY^#Rh%;R!5WuNzzUDSHO52-?sn+QZ|S>PUZk zr+x^+zdt0`K`DqO-G|O}FO%nbrO>@lkln@d@Bo=_@pxTqij~}B# z@FL7{WB;j%Z;MXk zZX!PFD{b`U2{UJI2jHz<-)pEoSxxLyJTU4n%V^*}KADiHjVDi@Yn>{isw}-pkfCnmIHq8*M zM1C|k1COE?ndIv<;IKJ2z#zR8xfuRur!LhZ=6zib6~}7jC&m0KG<0GqFIrbDP-h2T z;&0)oZG+zxk^swf3vbkJHvyI|I#c&%6s&>bJsmAKEyXHk&*Er+0-4xu7v{uVJvF(V!jo=fu7v<-W3Np0ZKS& z|0PvutE(oC!nzSMRo~Q!8OuGjr)|;hyBd$Xtfo(Gdk!36olB4?KS@{LakAG|o@}b} zxQL`9=P!P2;KP!12Q<~=HE%4Pox3nbn{~ zYKxIE)E)m8Cmz3FB02=7NS3ybzysgleoX2uypXdooyKB*Am)A)(~aUyZczdsaX2CZ zz>ab;8-px=i8X>sL|}FqMIxnyj(N{y8yk_zv~RCyzdCi=>_1h`gy)}K??~vLrT}N&l%W|-4(vhT_hjg>NgG)Fc3&Ypy}zGKuX@K?Rt0L#t&VM%~%D)?2#Au zE>9E!eukS-vXKnLK?SG5S9yD##vR(Awlk}oEht z25iBZs)`h$*ZT&F;gw`0{Sp`wib*EtVCoESq><{y;Pv_w~8G#)XH(PFhU+c~Dhr88=;QS=@YG z8A-sxYvT2=o}#FaR;zfWJSrY6KLqpQy5ZbtbW^s1SOvdo|q|0%!`;$fvl7yGi<>P^v35$c3u!(-Q(+-oQ21Z{R_S0{J)HotCQbPhl zq$b$WNy`5tid)8e(}D8F9_@VbGcy*H8M$Y9)2Xq|Km)|+5^g_>eA(N!D)3YCM1w)P z>&sADN>6H1{(hVY=w>5{R%?otH?m)e)sE;;+r#eaF7}qQSpfw6fulucIYUzkA+mei zXp!AYANw;*IsYTmRhT4Stf#vK{NKlUQS$RO6&IeIZx~>Jo4(nS4tC%fn~u zxYO9K%09euyllVS5n1(OIF_y#`%fr@^b5VdL8`D+xsJCesGEEGa00MP-awHcF{>t}(k zoCk8d>ASfdDpmpyi%SnlQa$61R{T7u#f*(s>Bu=ofSRW6oWr17hBPNGWPv>l3*oU0))swjPRCl~1$TraN^7)p&L&*E|$P zF13~{Y+t9TVUu+`b}TGxrF>;M8j@bZu2iqQ(&giM4LcxwS>wvpP%vs&2VCE+f)L-D zr6M1ssbMUu>I>c@c~xO^(i-o&Rh<%V2VUrSa>3TP&E_N)OP984fj+m@Y z4%Md@05GOmZp9+l&S{*dc>DuD^la`C>ov({mv?dZ$w3xx*6uaabQ|MQzej^{gHP^= z=%KVp!MX3}hMNIzbJcGCh%dnml`p~Zu_HGRUmV|@J+?O}YhyOJmZj{gD}%&gDXoUf zDC}2Yf{+9TDaH1E;6Q5{ftCRLaX%`)D;2m>vK(Y-Gh(N%Y+__x0XJ`E>Vm$u6et0` z9G6wEbUuY1(QeiSqz5-@SW%S^~4llP8}`d%B*{ zq9&yeyG<5^J<@AhlTj*5JWRD}zL(0QH-TY^O|KwjrKP3yZMuLnsmP%i&R{&%V41a2 z#AefE@cCN$giVj`t<&rVkq1v&V&o0S&A#vutK>OY9=Daw0Vb);K9ig}`J*tEWDqn~ zW0YzNhtxmuc#Cr%R-Sos;D*z!Eq=q4#ucS3EYdSs%}3Z@-aqGimak(VDxLR* z^QE@)<2h*ijUa~yZo7bwv^F#NAdgNPLR<{THZ#4eZ{tEzwrs>d$n@%wtX6PPxncLlIh@h_OJ z>I_+iGMY0iQx#jMtGPvIS1%S{HP&z_P>`2x)SJD86H&=V54)Hs`UECJTUG_8(P^S0 zT^iQZ%Kk+~3?sn8@rN{itC_I0$FHiVav6oxNcnsgUdi0`>AojR>5`p?;D=v7ay9g7 zV~s`W>TQgMtzBy*Mr?yMhs|>!h6}1`a?eR~#laUI4kS6dt?Ef{r;}!#dd)atN-Eq>x#h6ZP}T<9C^M5<5f;lCo#dqDmVhN zl0qQ?*%}^Xv0u#flz*)}%s+2)#@M33Vj4>0Z(LlUG8B7UW7 zdZKLNG=`xjSo;0zB4}{b)&L4C@-fJvuosoJ#Doc}{%*~u@X#n2f@KJ%7yecaFhOK_ zo}Jr?R)gEFPf1Q_M4M3)E|nsX{RG{`tA7*9@L3*G{?1P2NJ3`8IMC`4=r=^Mj#MzO zYOPitle0R&=f7t)eWmL;XbTbpk8Q@Xd-TT)d#R5zUxXB_)lj|}9kuqOcFNW5@W6Tg z-{C-gSi|siLS!EOH=3u>qhDwJErrEQGg7R_#ZRytL`h1MR)5t5$8Qnvotz%s zh{gbUCAZ9qf4D}Nf-&~Cz}rJ?2MBk7JQ@}AY~}gQy+w`RLMBX_(w5oyWOVRP~Ll>?(27`(_G}FdkFmmtX=D<+BR@*3DRRqtutl&=FOMm zRu}eUZtDK#RZZT$JjR$ARn29LkwF_cYv`drAR3A;iofw#vXhu?Kl^)rJ3!Now>X;v z^WprlXsAmgQf^0e+}gC;mh;-SN&$^aPDx?9>T)k*$?^q#Cg8KdLQRFkN)V7o8`YX2 z@FtC_vE1t+k@6PkK%U65!5O42JRP(dK59`Y1{fk6?FMgM)PF9=M3aM5YjnH{2Jl4s zafYq*|1Qs>&L@pX3{1&w`>&i&#f2E#u=@RI)tW2Elm-#zCX^mb+a~w(Uf6&URA^$m z!Uy^ISCa701E*HlkZdas1^mwo$gA{aQ!mI@ktVuO~JHS^X+# zTg1R{k?~}ei2A3D$?!p`dB#l@&+W&HxEfkpIg{Yw0}XiB!`gUVU60gJ0l9^Rj#c8a zZUMDZgwMd;RntE)<-I*3TG)wtPO?GBHf#9MhVBf#RSP2*&1OAjY}6gK$j@Z?yYVth zF_&pLM~oBaP%^k^rq8$#-V1(7 zyQ)MEsz3o+!m1Qw1#UE-ouGgc0L&&jB$6cuC2eq!+2NJ&0fbFYCtfud8(`@^ zF~u2rsOvK3{Qoz>;n&o-)s4jr9(hZ(C(K_wz+Q|^?F7pWpg_8rP^ba~DW8 zuzLzv3T#U1C%<5YEMK63@@pNJp&%n%^=xKakLfV6nXaq{w-~}&5!u_?a93{-{O{gi==KH)uU}|R zh3%^-Psdvqx=oAv>gRsWKI%i^MScEpUT!!?9D)M(JY3B290yD}axT35*o1jb%IU6f zgqar|amnCkb`VI9FlSF_tr3UB{riJ00)G@HTg?3bZSM>vS_dp=C@E*+??Jr!p)e` ziZLj3Y|or!_p;6UI%0Qg8jZ8sp|nHZ%)9qDx6v`-OBOyZ=$2c`>Zww4;M6lt@pli| zeX3BdQdZC0KQ&IKJ)0;KDddpo?A*mY!fzC)jXP!ANW(e0;6hNZj+w9A?<)sXBWmA& zxbsE_uD-pR^INj4Zta@2T-T(y{Uy6z3tD>iJ^hpX3Nb)wOzYKj8@C*5rvYf1)o~Hk zYAph2*UzKqy)1ewdR}pEg#kM$bDUv8bI8{rgW~%Y-S}e&ObzijDj;ihR+4UXBmqp{h6;Xs|d$5DfJLx z^l7ezK6Va_0mAYSkTbZ;9vpcI!*<)+n^d+qp&!6c94>Wr?_llUFAyS!NqGuYcvxq} z+=3aN=&(U~4zj{Ey7<=PQ&KqXd3(&O%rTpMQ~jD3wG`edJ@$B$>?F!#pA zoxfK@_kq#mDuSZ0G_=@Y7DkJ&bg5lt)wM)Q_GsW&8^nPu4*7*u&7``0e~+Bfuw~18 z--Mx@x(-)C#2VOKyt5MJI#P($`Ewd1W zm2k#-l@(1AMpCc^-kqEGr&&P)5Et%fr}@5XI*-Af1l&E{rr0w5q8po$^AXZ`agpgC z?t16+leTgxdb`9Z!_vbF_50r5XL?^;**9zMlA+iAJ}{!M;trfN2doG?7mwkab^O`3 z3+Khah(^SVUQOAZ za;QAo)Yj7dFoQW!0>*H=TUSJXBfdE-?^b&6+JRIKx}j)CHf`FpE*nRsS@4&niE(#q zs9`Y$OJ30uFh(_8U?;1*o{p~!X@=DwDQ~X^_&zU52=0^daQ}(#)AflJ77zAxZy^P{ zAnyFc%NAzR9G34T&l25Mq|~5Q(CaNo3eH7?@yxJ$2^SesJCvO0^3@$(Y&Xa6w5n$!<4Lzlto!is4ty+->@O0BG<` z$(@vd=;#zIUfuTX7w(t$_*`h)NPdd7{f*Wbpe;CQ8ko)r`i}-*?R@xx^Q`xv6+kW+ zZU4-10M-)sivQ-7UGZiTfElux#-}8iotqbS6^&>)wRD4)u-cNsuSoEb?jXTu+EOI= zMBAB#w=+-Lz*8M0N@nxJL%=C~v%k`1agBid;N=hwQfHaE*iw0)3~ zoUz&4V99Cgam5H`GI*_{MXV}XvH!!^o5%H-w}1b3*+sUJB}*By7b1yDmXNH;nr+HX zgciF*_O-GVDO6)jmXZ=`jIp;6DQlEely?1|$N7nwxxV)wzaAdfecj({hV*%#@AEib zYu`N=8cT();^MCI6e1{>NhsHGgm&Vx%GJ_B8`nayX)ZVG*$BKS0*^9JM;9}0UYK#C zO4q{;XY*X~CXnYUx>gWR90_z{p$EUNQETlbrdVl7Np&xebf~=NJ!Y%jT&@JZE`N+T z@z?h_M)?ee(vvI7{LjDIK_ff33ims-kzLn`(TTsK7I+jNf6D1o1R~;p zj_ET1zw{?8D^3-5&-w^i+`3DzKn%|DJc3^3e&1$}z^Dwn*+u|FK z9|{R=-t$+1E@P#hvo`4Rp0Nkj7TAefp0J{<_S6MiNJcQ1mLKs{hpt;pO|ochU1W_^@fWNKMDb3iQ2Tj9XDHF<7CrKJe9 z-dzd4a4@rYOFL#B6>xg>-gTVH1sqUotqP-OR9N?aY@Xk{o*uay z-SOsauPQl7sZ*ip>LU>ADdF(4vX{2{cnZ0s2*h)z%5$PWfN1%Lc^iy> zLn5byWRdRdF7ZZ!^Wz5kZQ#mJljK{Rbzzg?qz2z`&dg%amr#jGt$D9Y?*v&xzB?*O z46bZXEHbvny2#oEB6V?bQQODQarE0s2X75$^V_@u$46%F{*U{#EWL)bvN zu8Dp*`ZW!iljVt|w5ad-p$I?yxqAKn{K8-1H8R%m#k>v+6GGii&-D9jb#M+CEx}ap zzZNNI)T21%wf|U^&W*X#JxS%fFV#}awc!VTMgf)?W@LF#Ak0lF&VYY>^mk_02APzv zj1YqVqfT7AX7{n1-Ia!X+ez*3f_~&e{+^BHZIMIN4qf67d*#DUPUnRo#V}>>kBOp< zfwp^krA@o>xmCox>c?g0MZU~QZ9OP7>E=(H+Z^6exoasS2scJXDR}g=i*cb$tvPS^ zjn?Vat5@EQz@hLu>y)c?kV<*=kTKQEVos}D*xYz|qELyqGCy(z%i$F9a)v z^Pd4QKGVH5c)Tm# z+a||dza2@nn$uuFP9>nln_6oFhhPd99510@Jn|vAWF(X0a9sf;K}v|kd#0p2us(^> z?qofSO6!Q<@Uw_YkER03sqms9AmyRztpy~>2D)+Fqv3ZENfK+Urs z^NS;nJvky7na1tqThq``Mg1|jjyhxWtM<{g&&DD*li77&2=NrxkEz&IJh1L%N$%ra zzkfl1?)dv;Cm16|DCpaO8G;nSCJf5;R||#Z5amY^01C#hO&!^2(b0 zU)Yl?Q-L}0$?f(reeGMUFbWwm$ZXH(nrWwB)%u{l@JGC`Q#+%64@ddQ&&cf5kzNN{ zEx42<8E1b!*zc@vQzHPEWz3HD!$g@6q}Xq`&TA|GCngt6Dd7b>s>a3=}4M# ze-ct2F=Q(t65f!qXre8%*VYg~pIxf(_cm-=@^NPRM9W=Ni4?b{f>P;UG6Du1kRNZO z_{u_7z$gk#vry5?V9!6ioH*w2Xy&3FUaAh-8W+k1;-wIl=QV==L+%ey`&iWgC6b2SkT?FnInVH|NG=Q1dUR*~N;PGI)w%am$}&-6cWOUc zT6Ld(y~E!h4zV42vik~r4&b3ECz#N5jE$%Io2A_4&cQM@{lvTH&i^E*?imReM)Lsk zsIP;|m$!GQXwPnktt8}If3 zInh;-&m1P6#TxmwLo22p~IB=OR89XtM`O`1Bm%AlA6=PuKsx{`z@E6Wi z+*o4U&n2v`vr5v90Z`xhZDIKT#@am-ZNl)>xfdlK?FeFu)9sHNjABm;DV&`fa*tFL2>6nyn&q86f{@H>FJFDhr_x6AEXF=LF zkHDKD(;_{)T`#QCYS&N$`iI5AC$TQ2#zWNK-S<_4q=qx7NZYx$jq67A*pe!&i?L~a z`6v&a9V$(>7-wD(45Z|~r^Dwn?x!x`u9ME=V=vyCK3R4AVL z?!aC^3|v6azs_g)+_`gS@o~J^zg7NLf!YvQ)^>JvBT}T*m6^8C<@(C$aWlG~_?ZSQ z$|~?#6%XKW92DX_VFyH>{WXc?0r}gzbscFIZ{-R?(0A$UO^;#N)+6zlqS4>jSo^_% zuXPz?nq+tYLniXJ6ZNj1c;0uF7w+PzyNNnPY~ZC|P0D+Dcf0*Nz!Hghj1bn{m1HzB z7!??H9K^sphrR8c3UlKxDx(4j?0&LYCj_@kPtCl#Ch}PJGtrNjv2cJ`;9z9n;TIlV zwb*6n5a=>84v)Av`4B%?67cxaNvTNRKAlPq-tN%L&Lk?hd@o8rdqIS72PRH?A%A4M#8@=y9kEUeATY^8K%(!CIIb{2h7z=r+eW*m!e()DV;v) z`gEA3T@4-_Jb6Ro3>sCx6Blnc&y7#(d;N?eOffS(xo>0cXRZmps;8*f*RY+YX4SHD zkM17cwQ?mTJ^&Hw8%l#Xh2ni2^^sVc(ogeDh7dHmB`Ew5bH6>hEw?vpP&e^>g_@G^ ziMaX}^hpD5{2p{SCQVx#JiyBSkqv zZ#_VxBZ|@)J&%x7{h5E`7QGC@Bru-iQ=ChsHS}CV5kXtZ8ph`CYrHlwl-FtxS;GLE z*F;X+$ie0-Wh)7*Nc==&{XW+@*;Shq7CZpxUS%>c^zqWV&WYGFw*mu5cq&;H2M(N8 zV6EXOsklxWjI&1)ew$la;2+J=D2@TQ`=#&p(6i*Caw~{PMe9t}D=BFc{rOY(yeo(* zHUhPAPY+ORrBl43uK9i zx0n<-Rtt}5k((gP7*XvW+qc)Tj){n{eX^-ul7pPa_&Gm~EJ8`bCJxfQ`}=t(HTnPO zeBtML{n~`Box`kFnH=#5Jh#8e$=oIDAK$A8Hm+;->aVPoZyLB4ZghCEyMj!;IWFbF z&piFjb(+oM50)FZKx`_+h>Z7|Gk5N5Hn)wGS7i_<)>SfiasAev^slb169c<#M!7Cy z0*Ql<*5woRBE5!W?loB2)8YQ2%MwqcWadE8OP?7z?gSkx+3Py%DJ#MhK;T)a13w_y z!}|16#-0ovlAx|#Yx#2t3k=SB>J`r;p-Kmld}oQasTI&FFGZ6Ct#Uk;TZq$VK#DW*Pw!)Jc!tFJ-)mBm%ego#orHYmjue zSien3C$!@lEu@JD=?&#AL9x>BjB(bxV7PLfvs|JalwT?z`QG-wYL|tmdcFyWdT&_b zviLYG{(qc^g*mUpiP(_2+q9=&4(YJ}O%vSTPrc*5JBvqAMDWY>KI)s*xvDT&$l>MZ zxA6OOj2F^B3r_nG<)u~nkh>KQ!h370;e;4&P{;Yzz>pFZ>3sAIckWIy?D=x4b!c?V z)@J9d)9aPr31h>#rJ4W1b#?2m$NHV#QBT`QW3+Mp{w+azzId^ewkMG%M-|@gM%Q4s zdNxibz?d<>7NxPx6dzL;;!efaCVoUz7vtV=FH4}-hHovU(giR9)o*Er2&T!kr>y+` zuv;k-t|amq#2xGMC9_jO5n4L`vn7y&B(3y22$3VcW~+o}2wWN4trKNa=+Ae}w?3!Ijt|Hs>B7 zgxAjuj<0?E0riSK`DRa7cN|r19c(SEX5r$!miEB%6{3PxIR*2U^U*1iYyxc*Np`78 z`q=?b$Bk?Kg&L^ZVMBY`_t3d>O0xC%@>~Ld@L8rD9zsLU%*evLIL(~R{na<`RXpNL zOOmTo-C*6KPxHEkl5#R?#hG<%^0{#)->scCN^7I^z;SFx3tfsh=!_;Zv`NCdh+g0q z&v8)I*jz59-t&53T+otg>xV&(Y@wncy+ZIq^cuDsm;_k-DEIXa#?LO)33&%omk zLR*pWkNQzZS9hyVOUFNXT|)N3_Abn~SjXN&mq255f$wL#>G-ue&+aZ8n!bMiIRe)u z`RShxO+5@h7dc-d83Mb-d3bI}o#iZY%nSdI8btBnNu#-F0qvO{r8J8`x-EBvY~Yr9 zg_Ota>+?yiAyjr~^{LN%BUdm11LJpb;^YTF(arTSadKr2XqC@W1UKMJG>@VrpU5a`REcw(cVG`b4 zk#!y(`9chudcIV&N*gN5_e&1f7&*jgTNL5@p%wf4|22nu5Oj?yW|vpT=jB6ARpy%o zSCf2oSgb|=l5y+KF4^kw`;Ju)QZ1ZpN|Nrjqw%Pg^_puvb=0dAxG5;T->8Ro_T+r* zC(kZIorkYtzEA0j?9jSFPyLh)!zR>`;jX+S^<2mG*aCJm)(;vlEv0zKg7TQqbMH?7 z-Uyf`d2>d9D99)#35*2<6nV^dt_LOIQeB)2}kc16q;Qgay4O(ZZFTyn~l5Jx` zY@{g_e?+rJS+W0g&FRmGB�H_RK#!9v(Y`2SjYLTyeHjXB)9L;Y@=-8zhl5dk+3w z;F`iF7A1goLYMVBE2&p2qMLY_tOlR@1XLJ&Z znrC#P!Ah|@LSc$6#V;g}BI#GW_I`j2l`2UD5g#{L@!7Z!D~tw+qq6_X1$zDnkUDeq z7xDx0(=2({&3zkZ8uS6kH}raOZD{zh#oXAp$TKz&%1&dz|1&YzR=>}#lb@+nGyT6l z#kH-rd3;#JYQ<0;!o=W46#C4Z@qcayBYpnBiDs&B+TWE>7#1Qs!aQDtgzpFckiZ*b{pX2d5V9bU~{uV2G|U(vLl^7m7B z@*TBv)94uG{kx{`u!9%A57+n1cphclT-|zY2g|P4j0RRME_ZK~$dM$M=u|w(t-Bb2 z9h&Tlu{+@toK7%O&DJhsK+-7je2QPWe(QnfHDM8iNkgthmcyP5{$IogXB-x!MX#rL zYjs4&_f3hj14_dSR5??^=#b7{xnbUbJ%Vu5jm$4}suqd=OLG3D!hlCcv;C?5|CP_y z*Y<=!`Wman70nCw1ivGOny;d6MJZYpwa)99OnCvBsax!A&%3yST0;frlKOd?0Ff$D zB7?HagF~`5>6@5L&wsX`+ThgEd#P>6x)LNIzR5pX4%n{76&=aOq%N*Ss0p`SM^*Ba zmV12}ExcIDl@8fbC+3Rg{gz?ZWHi3XI5n=o0ccZjD;YO*GFosW>v^92!ME*+8?pBk z`Kf~~mDBrwc?-?-oW|-B)k)Q!KcMf&6~zn3y|4xggm6P5psKhSg1V+p8-=&X*o1sy z7CiBA6RNBQ2ad?=*I`%;GANeY@jrI()YA_{-*)s3zGNjE($L1nMoPaJA;^Zbc2qeT z72>V-Q`ZiS%St~2KCSE4K0{b0pS?jn1^A}$kk&Omkan|bEdU{M_ z>!dweQ#}S=s=oNzx7yWAD^(J?3xwC0`|w#eLf!H^yIBqkwx!hF!GAY*jyR_OD$uA- zDwJrk`F@mqJb|1j)eW`k?rF)l0(*KJ+-ck!DBfEPE7`@)s*dF%SCg0OT3XvdmhI6o zo8B3*=XTagC3#e0G1st)y(bT}?T{^-44W5S#Rz|Caq3QB^mR@=)VR+o#F@30Voj{Q znW2I)Ngau}TNn_kWB`*8pP&MV)vN#it4}N)ApPIT`dL#I!lL+TdEwozl)Sa)NFnzKu{vTp;vPjdu- z$?Vnug&aFBkr+pQ@0WBmk+i>$jzCCx)XW9-f*Xsv zy52|H{EGeJ>;>W?#C4FCi4)MTJ2*~`NUIQ-p3UXvlqey)eY*m+h!Z0q9Z6XN6vsJN z90rz8zw7gY{a3U{h%P6!lfM3Km{0486VoU9phlrL^v=~^==;sg_hZhezt#Wld$Zoh zl^R9fA=eKL*D&LpAUhR8j!QGS^tlD0djEv8Z}_q}cdS!8!)XQOuA@T5RKUeah} zMWU~t+Ub4mUNv^$qL6f7yHKSBga6TQ5ym~|sPY=i%xkq;`>BVxi7UyPEFbOJ<%HqS zS!6LP_!lM#?&cGkYo)-EWqx55r{#P@xS(^NAHGfQF7IY@GQkaR=;;XCE%}gE*Vs;p z0jSRIRdt}n!8vxca!KButE+4D@`{eF#vL2$Udda5oqG_4EeI#7CPIH~!&CC5;}kF6 zcer8fxriYE)SP!dnlItLxpIPlA#c?Jg71uA@aG;_07VSE@HrWVMv%Ni=m z)Ph3sjic8seZJQt@aRu-AuzWRiPRNQ<|g@w*hLSxVNR0wx++E{&`HV2R#oU84RXrM z1tBPVce3ICD-Y63Yb=fT94C2HwN^kW(mTWvX1+S?G%)h^)PIJ9d06PHN=e2s-ub0JC?+ zRiy>kf#uJw%n}Gx)p=W?DtGZcKD43ryrTnFHv&r~($#>Rpnu$t?flj*{g=61jR+8x zt1Lk(a2xK=Y@9!vu}37WAP0#V$J(uG-?5_(`Wa3(onD_Ge&qvKvZjlS@|=NJE3&4Q zbXC>FtVVESRu`4_ai(MHy&5f-S;XG5CRgkFh@ow#8C9u-weAM;%kd*5ki5CPM;9@da~uVakWfm7qr;`wWSA zc#x{439%` z6K?ivbV2^-*#GH7Y|Yglu2^-=8}SBFqPg`0o{{%1wU4FecOzWvL>abJV4aPXyOm(> zTiGv$Y^^^N-rVgC|K;}H4kKpt){N+)3u3&S6m&`XRH_xFX60(yFY1q`iZPXVficCP zhlQwx0|eYa3j}v&WozU~)r_hTjmv zm~j;19%iFt;o~M`QEYmvJji>3-r25Z;~)ol?4bb@C01FIl{5#^HUR&ola$V%U)vrB z$Z?l1wfN}9y`iSX87`9d{Ryk?I-bry8p@dlZ)C0P92e|m5Y?cUFS~UGxDovA-rQb< zEHglFu57I#0mM*;w?do3<-<&oli)yx@`hImuNUOqOnpZ`b)%!Bjx+|x4ETeOGh~Ws zd}X#P<~UKSk&MFR0c-~HGI-ReQ4s~Vu3A6eZyM?49ge0yB+d0@Q#bf;Z!H;4QpaaTjY(YO*NycLVAWlod9TbQDsTQ4n zr{1lFXnY(vpmVN3m>~QqBRgIQN;>PtNNS@!tAuY~tE}ID8oL-C(t^$$lT7mV!=#cn z2Qv3YX1@DW2SUIqxM#bTdu+$!{ob&Fiq1vZ%(&uBCu!PapY;gw*;Rz{9)Hj*$Cq7R z$`Emjy+5^9g~CcPDlABiYn{%aBT_I=Y%!|91fAML(Go!0@K|o;Y{O3vXf-+_)c?uH z-N%mASEb*9f1E)2J9W3@x5lLKLcXx@xnaW7pDI%R&*6W8Wo$=U`AeC+uwn7)I)4T< z%t0pP|1K+(jB5ezsHckP-I>EJSuHSpH}qzH(fBS!+eoA>KN1sInvz;} zRPbo9hNsE|^0OB@dwy`QU5DBaW6;>VTS;_Ra`SFv7cL50+wY8jSoyBV{tXtt4ljr+ zPuy|M^5w*Bw|n+)yJ^tVnCm`AcMq!4wr!VTQ}zZ7d$QtcWWm6K(CjAzBO@6q>s>L)K`^`1=(4ePPW?o#d@ol}(y$gPw>*oG8&MtOu<>$X!{GJ+`-f#4Km;3Yc z>NkJB_rCAVl8xJ9Hu~vd?R~O9q{&NpaViGRQ7MNrKU++N;eVl~=s0aR<904LcXA3# zUL4}|h4r0`r_YzSr6W`8;hd9p!bA)b)rGtwT~azF95bYzM&;mkBI7ssK$pA!DhvBM8WUbbAvp|dSghgWkXLxYF0TZwQtQ*54F4O zS(2@8oq0gJ_2zz4uYMDOI880Sra=*d+bREjbEM|5*vFmU>E&9k8QRvbX&d7Sw_b#7 zD}38p`tDCjHykXrS|wqzx%PVRTFXj)#k60-2y2bfvTXMx7b`BOSCV$vWBgqVUXmU| zzNNVh*{zdEJ4FG!g0+<426uH`rkIEE*tZ{EupcT}lej1A7d}P9G~=)Rov@|WHy!J_ z!qYE}vFQeHz$%Moby2Ir^Xw_6>iNAhcJKZy4;gs{X$Z3Ok3He{*+FU}fdG<$8feY3 zBzU<9xx9#~lz22oz-yHSH1P2F%y$hnz;G>{>inI|`R4sY4k!;%z$GSC3^Z+YSr`45 zv>Vn4Lrm3sI@XL5&ghP={?=ICy3TT_F(|cRS@?Z~yWPZlZ> z9%3->go3NBl}z;#|*pY2Rw9GuVHIcw2k+hL9U1HHd(Up=@JbstvVNk%x zQV7MVm@oG~(sCd>n0VIX_jK9-Q{9xOIVoPwJ$|&&CngYo^lE#z>(N+S4e~*&#l_Bi zVbsCj(gqzC3|K?Jd++cYGX|ZweBjON*WKCHco6q(Z#8BkK4<$MYWrL+eo6?NNNmB< zp0#fwNMNWC_&hRQb}k@r3oIcbV19J-FCMvYr_3?d*01^yuPm{0eqc35N+AuS-xWd2z2bwzP$xyuFu?lA;rikA^6b3rCGke-nIMw$wO=jaQ)*Pe?LT zz_-ll<(jk@ol!;*vsb3p8mg4OVSD3OE-rVoZ(ww}QxcUx*(Rj)5z*)WNX=R!LHVER z8AqM8_kClrW~@f_Ri9s2_xZ4lgJN{Fa-T1bJNO|pbA*vq0&KLj*B}|5JS`cir3Wlt zDoEYGr@PlED|pV!OjvDig9z060;#%vx0IvIg$?eqgs^L?6e9lNm_W6SI<(WNU5`{g zC6ceyHl;YlEv3XQW<2qVn6m>Ib)u3}nsT~L8|Lk_iuXIp7zdyn_WszAO1##nNHHh! zC9jr++068p6chL$<4yMU0LQrJqK1|N-2w*DvqP*k5Cp%%frVw@b=7nUK5s$!+aRJ) z4Wg!mMoe4Z`CZsO?L!`VpPWW`#_kxkZ4FLOzWmhe$5hE`&(p}4`m4F-`V*#}tK=S} z>lfR-(&3a2>e`L`Dp#V}AhO`)lLey~16RrOTWOu3wmd<>Mu|9&uz~bBb)?hp)r!3{ z5`(qSGW*+~O=M)5MWDE(M8q>VURd&KFxd~8jd$qW=Va%y=a}SNV^N!h>>3ERt);pC zGigL7RB4T_Qvrr7Ni(%;H}Z#unRJZ4l7l+`VH*NLG*Gv<9R&-#UZn*UhkNC?QlJ1&b3ajI=GxJ;KPJ_JCEywN_RGZR`XG(P&JM9bMzc@#}>^Oaj zhB=d2pwEhwl)&u;8w9AazEVOeRW4TMpfU-H=Wk`|x|6gHc_Sc_9qY(-PY&U_dvoh# zESG;_2~>@Iz~rwc#j|&diShnaU7fR(qRRcOT+wd&Vi#` z;vJ0-*YhmQ4|VrCeE9J0x2OM9B&E+US=p@33QK}q4zlq}^rz|#^!v$zm(?pl_R9Fh zJ;3<2=M;5a$S=+L{-bkwE@9@pwrh_W^P^TnQRjF6du}fY7(_$tyS?p}%D4&vW@b^E z>!!6nWsac_Wsa74lsEhRd1s-S*&2bRr1}bRWYh&FIkBlS2nW_z?Vy2S*s9HeX~d** z`8knNse#qC&8*GHC}MaPa-H~8P`0^>59%asmRpG!G_%nH$U~yALX;Nw#F1Et@ZBJ+ zb$MZ21G~UT5HJ3cLvRHKNHVB=YFtI8v}MPSC1K<>FYKiFmo8tekX?OXeAVk&`n@13 z$;B=A%iV6Ys*jLsn=g$Z26U!UDHdbS_?7dkWt6zz5yehHYi4vRm&~mfTKM-!TY3V{ zrIqt2n^@=r6Dngg&Qmn;;igF`R^d2GuC&$Vtg3JJVmg}vLg^62 zkKY;GXyZy(b888q;oh*a5&Ie$!cG53=Y063@uZa(Cq*scJ{Phob7Q|WW!&q8sD{GZh)UyHfMSkz~D3A91dN1W8FjRFq< z+#vY;`7U}%JpB#&>TYlhwQ;1EQr1H}<}umJ_k@Fv&Uvb_zvJ@7jdlDYwoDq?{E^|P zY*p5&hH2g5CsK@6E6yxE&kg#k#}{5|-?O86BWvdmN1JZkHaXXA-%dy9bnytk$qfu6 zTvvR^jQ@ql%Zpmy)3Ff-5r^gAgFZT=PlN`EGLagj#BSp{zeXX4B2==vY(|e5ag)OJ zhOB-zJ!`LG5(exRFKJ|-)D<5eS4T*=<)tR=2r9k@=;ifn z4G#pf48BWq(o88-s4;-__9oy*H$`Oblj`9y*wR`9TT~4~`W`JfynG@;l^im&+w|yR zy;Y^jGc)PZ^4*fYGF!Y~c2(0!pSW7KOaxjGkzM-^;Uvve2L13A!?Mr-9WVPdkLf+J z8=ztJ1Acm91~<^sB8#{+>65F|>`_$-?>AS?=4MiQ zuEeKsH)kwOar=3EZJ(F@hW0q+FwyHAU};Mpcs%q@Qd4tvW+VpXe_W^zhsBnN7{#HS zm(DHT-Wykzv7^u8t?g~grkMMBd$&vBNiBGi_6)Mtkd{njXYuh~7VV~C3tsWzq3LnF zkf+)XYs4NRj?|{}&tK2CzPXKh7$W@O^2^s=PRL4K$g9%Db(ne(m@R&3Gm`jICoz*t zwo=HdQD*nvGh7s0nsIL%!rdGS?+kr5?)KGNM#M)N_b=65JLc}vIbb>{udt~5t(?>v zAWlmys=6ijYT_52RHQuXc}wcn&c3b)20WbyPQGy{>6aOyUs7ir_6Y1c#YJQ8s!`_) zhTO(Q{{{fxVS2-ArKRa6Qua8oymYx;l-pG8Ose_NA-(*>B$0)o33hv832|s5nHW2E z@6I6G1|;uDaR@DB-fNI7(>xcwZQI*3{E)H#nN6Ga()9tp#colhJq0%Qn;oyoRe~*2 zex#KbXAo2`#TF#My1BQFa z^VBwXf49vfJC$0cllBrRc?qOkG6Wq=fHWo66}+%j;ZtemL@>?znxiFj1oD@L@#kf` zBZ+V+;*T&Bc5jhKDFgCiw6N?v4({9mw9@QyTd%C29=0u%qnqq9Fm*_QR?o!4j!TQ~ zC*qn?YItY%)=YR4UWTGGuXL++odg$4xJO&tv(Kb9-~`l^Uef51BjXo5oO59kwdApD zU&`+qIJxXi>+)y#71IWa&gyQ}Z=)1y#ci^V!1 zh^JPbD^#;G6cbil%(&=_a6`65>B-hx71I1jGuZQ1`!jXhx1axh2r}^d+fGbi_LL_}wB_%=Hb1$r znOXkv6ZSoI{Bn9n4ym%{)U6NLo#abrlj$q%oyrN;-!OFVh1Vs^G{QSX`Zu(0ez*5F z=aY_&X5B8-rU8j&{~vOy&Mftj@Yw{mQlv^o=qyT>n@|{uIFv!2bpX#QF*$8dD(y)`dwRS}nUA!Ch z64Fy=raGnUO*j*_BokvZfI8pv_Noh3r*<<%>=DKdyQaiT+wt!r;72>#Kb$}p6kjz? z?sahuG#x(_6?~(uh!1`vwgYL+Bf|}4?Cw(mq3&~fYqI2T!{o&oaPsdO;w8osvl}P$ zYD;L{{KO|~KAt}G5csj<^ylxENj2@@{6FW$aw+bdVw_}Tn6LYsnm zq#kBHK))`j{^DIPU>!u>q_*`YUOj2YV|Ksi*4e9rrw+ti!qtXM!Ab{F;O$?ia3eKz z$@kLLY8#Jj*d3d1@8Oa0muH{Xw?ppyT;XPpac=TuJu+(MWhSWjk|?)VZV*#PmFiNP4AYw zefko=DQe>!@Z<>L#kvIie z@8Wv`m~%@O>#tclQQld2E@98M_C>xXe`RK#RuU$p!q`a2m><8TDTpKmTpLuK04#j? z=o0E~W1{*n6aBXQt?_qTJ8B!Ph(?@Jc(pf3+wU?IU0FILYgI;kEylvfLrkU}oC9)i z-YY3TM7}99^%`!f`QT6o9S41`e0roBg)Q#hJm-1p_k+u80{7Wt_qej;)rREONPs2P z>BkB8tBDd9%cK&q3eslZ`oWyhQ<~viz5jKLMu++C<-DN@^BatqPBJ=xUj!WVvf!6; zR~y7RmH4_V0Rk+XN_|{VB#G?&I}VSXnwZ~^_X17YR<0zyD0K|73y=?QXpq>F(?Pd?OAU(g z^l24Ibl>LSiLHiNY`GJ-szO9s$>rT3c|_wk+!|Wza7w>gcdvKbcsI_xpT~#t!(KK1 z{_MM{FmS8SfZik)OJvB@wA0{rFssqrG`s0p;vJR$0zR6RP~kLAB7wrktAGD_w+sZ~O{8#aw_A3{dbBrB<7)!;GU10vIq_ivgr0vqWEf*`3} ziMz8Z9w&;h^L@d1PjKZ+`IKc(o6JO@5GJt7{48z|>`~U#zd!mGd0+AP$dCD5yjHZr=5o+w(|w zG`v^|hHRZJKO}ClCV-UWfcxP-t>iZ8e2`*S)Fzq*aGCSGI?W07n{XxIiB$_{E_YFw1Z6>Y7xs)ZE&4BgUCpj*GodQ)o3_y@E)QGyyy=EEgAUHAWU|16ceTFErK#L= zGykTvqj=HfDQDWtx6fCD(PG$MhqNftzYsE=IM==HQO4z8Srm1_-fFc=O38tsj~eE` z9yFN<@6E3>Po8~nL@Q;j!L&jXM{4dyfum!0=Dw5+XupTh=8y6ih}pt@TvGUaUhyP& z+j60IpLNTl<8%SyH)M;vlACyX*^zn2ZusvSs9DUx(|Gjx`lSVbW{$uG`@^@Lkg})` zO$NQbPcP(&<@HTngWHEVzJq@r3vyb#kMH|q5IuY(N}ZlNdV7SJ?t!f~znk~m%q*YR z7Q_yzU~%@li~|z+EmiTu{3pMV9JL)bc4NqNhy{EalGf{RZ+>s$3mUT_pz`AJ;g?&% zwT9*!MvPyhOWEh*RE@RW2X>zJV?(UWti45%$v7-s6me}*u4}(*zrP&jq1V!^R&<@I z`yR%=_-)uh&WU8PCtTAJUj6ccwzbTy$sU+YE~|0c;E%)%!Ey*0o|;4wSPSy2(JA;R?yzX* zmgnD^x`y6sSNcV8kR>NH8aAnAc5+x}LY_(=YE88MO8+vGSvB^x(4tn$LQy_RH~KE) zZ_O9iZ>Qr>i9d|}oU6A(n%7NPHhKGK?>E1oOl%%Hgsvdzs30e)kUx7?bJZ${2BQUr zT4afmPOy9b{#ykh{Nvt77_<<|tdqz7AZf8sv-LK#`5+Ng8*Vcsj&?6@4B#K1Oq^@? zWpDqiF+Gz->?y|2>5*U_601mCg!ldf9)YI|31;5LW*Af^q7xooSrKb5raCTmebt)r z-#J*G>BrWNsz5)u*zCYR+7AQ#AT7W+Y+>nk8Toa^wR;8ky}RN$7m*&_G(qYo|8v*eBxSyxDcSUv{YvY zw^KdG#PhjR+@BIE#(S_i4bn4!D?9tcv`n(NVXOjlCzZxe=&|0wWn0^3_%~ zHwVM))lAPmG2y+Po{nF!S)Cf!zE7$>%k29ACA!Y9q?g{cHJp|9L=~@cg|zKBYgnN)cFFJF(v8igIadaOw7NgSOylFF1wtK=uZtry+-f4f;gsCj(6{K4-GC*C#k^=09)GGNr(Wm_mblR~t1je0DAmH=)M2 zfVpvRE{yyBi3gcSPmbuh?aC3o9X=-G#uhubE5qZxt?up|`k&&TB zX-Jtm31mdZQkdj`;LHsiHaW-%4Rk-$UT44U+qVy`>Eb~RtFZ}=02n@(fnKk)Mi z6hx+*YjiG0LJ~FD@D^gyD&r=vDfbcAex~sm?kb5UgcBdkg{QX+KYQcpuK7(^ElVYo zr#rJbHbjdb6UM7hl}l?OJo%#dxh)t$V}>NPR`q}MR2idfo57SkaR1(ze{g~$ z2hN$y)P@8Qv0_tAE!P#gMDJvVYpC})i>cX+{1PXEKG+~1krs?mGw^dSFCb+d-~OxW zvq;;NV3-jZ^VKUT=8dIloAvUXD?AsHu<=pKtSDo|OxXQ@@qSUA?0@fa&VYHAfr00f zaaD?4GxH4{Uh0Mo*w6+hpKva6db5zs?9&=ISAn6uvwf18A<+-IWoij#r zOaO8M&(^;0X3yQgG$5x~Il`t`4&AR?TLxdV-RW~+nle#G?yO}EOE>JtT=1T`AW)97 zj>Dt-P1pX`p(T@{ZCeBSJg+bolMPS>?jgOHwa@p%?Mx2hlY z=Wv*&0W^-SioMz4k$!EP+pBM-!!+UnO+DFv#mvi3uXfjI*F3P#oS>UBeG7ITAtfQ)k zDhX*I7s17Y2V0Y#=V&D}AHt&4^to39R2gI*+f8p_{QM4fd-&osT8|mb*+%m6odG-C z+O4oHLs~*IIa)i3S_pWQbd3O6>F;D=3Ib=6hFm_E1$kk#k4&zD?{QcdYW2Fn;RP;6*<=2U3iQc;V?s0Er8*D#p z%~1C?jNFDk1|)73C43GZYHl7DUhwMf-Mi9W8Wr|eZTx@yYN#!D|t z4$Qr6)SwMUvE8L<9eybzLY<^LescwxxKT8A2?Z#AHG5TZF1s-9*3jIGbLFt=oIP$D z)W1ZsYCJq_eHVMCJUuuum^HEX$y$?_I_spwzs-aPf%#G+G-#|NOd-DNI5bQNYj^$f^{Wuh{&YduWT!;*5x;BHA+_|C z5z-6o%lS-mX7^@7-NDP!Afs4@`)ke@-e>AU5}AC@-GvV7QJ`ZZg9DhnC4PWQcikJ7 zT?yL^UzOeJ_3RZU{Ndar@F%(m@v0na40k5g2E>Z?38DJg`si?rgCFksW?|%>=A9Lu?)vK=tv|@{tF@dw;b)Ad0Ju`{>_pisTX&FfL z{{2Q%h^#}D(@^UGv zFZ%1gUPQ#IDT+5=-}6Trps=QeBPRPzeEYaMDNm*QJsLc3$D-?#X_ zpyIae=;?eq85S2y&a@Pi7 zd6uAodojO3+nEBFB3s}=L00ki;+GzH3qVaDLXvfyicx>8oDadElCx={0SynZ=Qqh2 zZ)@vWkm^3&ri5^q2KVloGDJI(+nlO7oPU;FFec`2B%7rb*QXj%f_f7Zl2&}G_*Do#9z0C3Oz4tYyoesFB@=TFwH1es^Sb}&7Y!)wmCH)hHdLgId6ZnghP zH@(N|AA@qj`_Am}#a7v?@9`nC`S2m1OFh06q!y~^M!_&x)VH5~J=RX)gCx*!_T0H= zl|lmmV{#u6Wyfo}6sKtSO?kX=G#p%|bqTX?L1*MMc|Y6D7XfAf3^IpL32pR5ejR}*s*?z z>^ll_#J;Y@e*{Qo82O(fW$@VWYr@&_!mPx*fs*C;w=N63ljKBj7Gx~?i8DdWB;r{F zl3w&>c|&z`Z(cvUq=T4or6;w9d?WZ?JqdJ3>0+ii@XSR>iEOB<2il*O3nE~dH zY`Ce$QRSQ#MsD`_(AE_=35(C>WXas34e`kdN8X%@nCJe{C-F7wX=1tSSgnI-C#oY6 z?W4%>l5$XsMV}hBuh161U0@UbRD2;n?$e}0DC#~mkbZ%GP%zGIjdxdJV~G9R!G0dQ zxC^{L(SQS3H#U5+W3TJWWDDa=_Ts`DzKx|Aw2!V%zkWwT!-n}keRFf@p}&+RY^mFe zb}fFraAS2@gkxS$_u%ZM9ax8v;FuePnQxoB5Xfsq6l0{ONa z=rSoQE;jbH)J^yK%YW-K?lt7TrZy#ycDZeWvG};(V_oY-zSDv0Qr{LJuor~G_hGa5 zEc=rE!lk!vyD&*oLINZ^<6+dqwUN`rXaU~gUCf(?n2pdA1DjdfLyFJ9z5-Bc5uMAR zYR@F_BD#Cs@#DtLfbW1T9~qgx0uBN78uRrvH{7D~ladNEY$3vjmvUm)>dT4U#BTIIVu@;|Y2N7jCrBX>NJ>x{KTZkI`t_kf zmEe;woc+bjo+aun88z2{^h`WU`1`oqo^o+_@9= zJ5xYkUB4#J-Op)mJ-SGCE_2_uO&eSM;q?SfnS5>Tw6RgG% zD?#|y;;eo1`FkK@upOEP!;g>28-x~B_AK(iqjLq$A-7PtpG$QmkP7(f=SQ z^{B!+pZb)ai&z@Fb{%aNyamHU>+d-=?`_#%8tLs3=qBKx!PlMB*L>{2LpJOu)@pXD z8GN*#pN=Y+1=+p07_Kdj?7>P_YWRVP{Pnye@$-4>3wm(MP@FMWI@x)0WR8$!07*2TFD6aoa-LVTadD@!I+WN_7b{lbsOz!Wha z!C4urMrX_RNOBS7nMe2XKUt3eRsd<1)E&BwLoTgiT!I6FFVY$X-$NB)92Ip-9?`3^!#eom~mFW@~(i`-X+Vm2RJw6iFh23 zUY>Nn^WQdJZCfqO1&}a?b;lr#yjpEo4WkZz5u|?aVuvZR8dV{#3$Xd6HsR1U1bO1) zthzHUF3z4wlMIfbmK}FWS+)eH_clNSF>%`J&3&qmdCJQL&(zvzt233_FyJ!)zS`1x zrbI?6TyUK_Z2w*splWKIiy6-J;32|UsXfsS*23Pp~_Gzp_2g~!otrEQm{rEhgjUin3%uZfn` z?!&{jVT(zeMKwD_DC$&`GpNOS4$BoS^2~s*ENfGkoN;1!^9GXlv@x8d3E@$ zW51&>^gDIREXxFR^24)>|5~2(e0dr34!HN;4!nm)*Yw%Qo+y7{{8ijf#!~M1hL0P6 z9IEX8viLVbbF#*c9>xm+1CGfarDDSV#T+!Owy39h{7BB7C$oNF89#M<2=oT|Pk}=O zh{L1!IDQ^NMK*wH2J-4RI7VN}yyLDlplyN_+y*=--3ut#`YA7e#WY<9`BT?JcS93` zBT_z}aH*g00)z%kKz{W+68~mUS%m(if<}$d3-h<)wV*tCJ5M~=M*9$e>!UwFx}fT7 zwl&^db?e=&9S+@9-uD-pt{lDP_TR3mr29XQn{L}QS^-_r7QF}zydAF9BFz`S67XCnx}^7oXy0Ic2i>CL2Q;tCA!FF5GxPy&cvS@i{YVnnDG4Et)!CX zwUL*$6bliZx#Sv>7r6y_EC>7UXhm$47&jn)g9_H3fM4U&7j5#(8ISm`iF~AV-pO|5fgFkBYu1F&69R}O+hECgB8DEv`gH@mN&T(65vt+)ta5IblWdVU< zfHmVwxPW8%L!Ag%(YP+@SwLmyq_OY>KZ_r7M|LX9H0~QJa2MToT6Hlb+-FvwEm%!} zA4$%htF&g_*XmHain%rMX#c|(*klsK1P#c5`aDu4E!(oA)`sZg{WKo^6j*ZZnj+eD z35;8|P+xYqiG-Vot`DQWO*RGsG_|)4J~A!qz3${$1yq-%>DNV03wkmcAow64zSLeP9t4({$uY)v({cTb8Sf;Os;roY+ zHvLA$mNIE>_xo^pi8z{f=H=z>e((2Z81GSj^p=V=wY)T)1^Z!-CMixT>(-sn`{rT! zVk46#VUA#(zQ@ohp+nN?+`03uJ8o=++XdQU{PZrCjn{u@gQ-2$6XMdp&SWQ+6>0Y5 zZ(0oW684QBq;FB*y2#4$!dByY9*of`dbyza5CI2mRVk^naq@{#OM$X*u%XelEo-m;)XRn5 z8RN#yPq*fZ$Z>^?k!V;aeUa-$x>p-8E-* zCYlK)Qe`BQ=KY~?+G%2O0(_7E&TFE_tXpb7i70jJQ^2t3TyWYS*uTHG!?F2Op(0r3 zRTcppXTCN=;y37d5?_V*l{2UqocCV{P$5nKqZQ!%J-GhyIxLf9H#s^`ytpC3!8OgE zaa|||M+bgik!jHU)7@US_;&oUGlWq*d-icW%j^!!n8!?{cPKD{J(%lJxf?srIM<)t zpZuL=W0(Z~s{=dg7}`+1dTL5Pb@rcL-E^KVSJYkoL7r7ner8a58 zKJ_}ShKWl-zBaer*l5S96y>wh^E=fh$-R64)px;(2*lYDm#$FG;ZYEz^w)bi|UU; zCMH*|Sl~-Q#ze{U`puh--|dJ65Mx&9wCOA<;<*>}s~D(e^LF0Dl82cXFEtD7FRJAy zn0j**mE%kBuEnq`uaGz@)t9~(ky+N(d**JW0JGMvxF>Iwa>>)D#ffhCak!0rW=Kq^K6pxp#-ooDu^lv{k<$L1SQ;pCL-NI^C{C*Sjlo zo~Gr&IXvI+!XYmWdh(W38PHTU?@pGBx?C$HfmXe7?EANDe)nO|p(`xHbQJE5&f5qrY)%>L@SeClr5n-0C?l ztAkr^Ui8v(gvR}zRyuQ?Mh$k^e`3lO_p;3C(hnAYUC-0oCr4%U4IJ9ra||C^G=p-2 z8`=M&OO!qIu;*td`~;tp!$#QjYF-!Rbqz^FIXX71D=e8HnoQc!xrnr8ijxU} zs!G92KTvu_Oe)rYx3X|snoF5guH863hY=x!Y1tk7QE#6s5(7xe!@`50>A1r zS|D$z^Lpp5PD1zf4HUmRLTd+ipa%43#}$O>2_(LRl!v88tXe#vEbmEibp`$6Kl~Pb7c9*BG`f)_(8)!LA`*D$TDo7r6TbddN1^T}z$=o}#chs|)Hf<+Sv9#ulw0ZE? zUolJ`>g2qR?k?Y;+Das?A}T}&?v3y2u#wX*TupBZnlrxVI&)I1C`kTS=K*btO&q-c z>tb(Qt?XN)wmP-vsZK0@WYxNN0xjZ=rxW_2Da|LxA0ithV{!XC*(gS_8(%PVOKYEl32B2O zAoIxz+eTRqn2c0Z2gTjK{fYx@hD6l+-D(JgOE`^5CYhEcW=_)Ul6vrNbv9a(R*5Go z_?rMQFj1!e?C|m4{J5O#xn6@Vkv)P?dq(#*#>Od&_l025z25=lEu;er{7=~4OS?|| z;tv4#J^qhLa%c6mjnl)AmpgrWsty<(k!^4I1AQhR<~G8I4c?O2qj^TGdk4FiVv7lQ zd1>JkDf#s9@=E9=;GFerUih-u!sL}oe2-KL{Nb3?HShuKA&uFb)!uRIi@P4G(`uLQ z{0FJNY!nsDX~J`cHdA*Kv=@6{9Br%ueB+DMZ+J^YNe#APHY6vhdZ|M7>%MNG2j8se zP!NR#?(C>-)`t`N{UrA1VhNkNQ0{gnkS}*?UrL|&>iOOP_vle(H@-UC*6sMmuC02T z;@%wlgp9jpv3s_7i5oH7|v8%YrG8`t7IQ)mp1{rrsvgScy1==9~d5VCMq}*hk#77$U|MB%6@L0F)|2Pdv2uVW9C@KmeGg(nLqNGSN z(ja6-6v|FkB}FMSLXlKflvPPKg^W~IR){kCz0d1XclY=I{rpB;M?nsMA?(2 z*i9}@Pi;0iZkwLr;<0||Vf%_F0X%W+#aFFF-? zED$7Jfmw1L{e{B==x2y|!MHpYERaSwx~Jo=BMCZDTEr~nbHskIY?k#YksOYD$i3#G zL`W$<0)PO5b2ry{QrVb@2|gY@tfCt1(s0(nVl+6A7QVhhVooP!;PM7B?_DE3tEOUc z-OeqbzNLO3N-^U_egQ_0+~EXb!|X3`+>sx#I!CvGN`;^yI-FCz^n{KNj%&l7{)6rD zl|Z_V?t)qtQNQvSU-He>*m*Hf4X3Z56xLe2`*$xYzKb}_^vrZkZ6s~cyYKcrk-SU1 z`Oa?Z*$Ao!cw_oQuIR)^KrrzWV^z<^Vd%tK@NSYc_MTuu?M8JCfRu=kp;jy1CsHD^ zzrH2sT)W<|Vh9BtYX2f<@G~q40pHTQE)mjfeR0 z{0I(=3qU0{NAlgY2jv%6Q&R?tPmrh6U_rlp*xJz#^aI@VB^ZeC`8R^^mVrGdNxCA2 z2X#?;&fOMkF0!N?w zwzbYK@u;wqlzhxwCCX$^XyCImFj|-5*uI!w zb>cPW*2H!Eag9Z=*Uzfj=3UoYUtfn{E7RxmN=i#9>KUknk*Z<#`E^9p)X%{_{BG;C zqZY*1B&$B`07I5CQ2jGAR3NIG{vYw}fbtA(|7VCaW*wr+6weMxb29!BjKc-Tc=J!b zLz_~c@87zCY&`4D73#!kKU}f+bJdsJSY-Ldwe-tYmCweHC9cYm56i8ddn?+Vn>J2b2tj|7T zo{DmVScWJvaf$X{75RoG2^L~>BzCR5YlIsDR6v-;7>@aEC1%OQg@6ieQJf2i?mG(Y zFVcfWR|cR7GVgS+E>x#7T)7}0uzfr-mVqZ03(`>VIGgfD9Dpf_drZ5|iU`CVb)5O> zoQ7#7B4T!{W&(Ad!oq3Jumf*K8O#g+)(8YQpHg92fV+zs+z*l_im88VCf>F%G|j|L zFF8dNX=b+(45|JUPXN(DSt((wP&oa0GwZ4k`J*S08%^x%3Ue}1yU5ZWQh})PAAtcX zJCRS;xzy9Es(Y{X=KSG#xu*y2fsYb5>^Mg`8;I6ghgY<)=HDZ!*M*mgYES;BhwA_W zOASDuQ+n6c<;-Ghkj;gQ|FWPqxZ@OV>hfDDxvHZCCk7P9&kiS=XRI(cxRrYd zgO~{05gX?z=W^U?snc9*;$Auok$pULPrCO37SDeh81P6@RmCi*fbG~n{9VWR!K@)- z?VZxkhKLkLP3>3t9gsy_s8JORhh3}$({fDCvxO$Ou*Vs!g%Q#loj}dN#qA5Fv3tLW z$Gu5@7S1TK*q8HZsf7>XO*-)S7vQrgd)|CG;v!DX=c-U z!%rZ260R+UM|bSlF|ww_b+pv}VFzk5sNTbnia*6y_Xua*HC+$H~5$f1c z0RJCiP=80+`>Cc8XP#p>GjX(r2&Ud$2j@Z@W@-U5G5@G_YFrk?n6`w{;I<+2fcPU} z@%Gz&Lv3t61(7GA?)mu&xW$oOmaEtF8oWka;RlJ_sScRS%x&;Ci82d|0jqRW^hxY) zOVr;&qIp1FeIbdL>}Wf9_%?u4Sl8sezuf|O>_e_IMV0i9NOKV)IC}}6QiKW)hV{NF zjBe%l3o^m8dq8=yZF*Q$d!P`BSx-+;%!){5%ofIbOymTV%t$0hs+pOY6}k6u?_ya1 zCyhzC-mO@%!sip-K(bK-XTb_baQyqw*6(*5xgBi`uZ_=k#j?PrgW}NvGZ8ukkLm-e zRi5v}J(#tSQ9*7Tv>O-{PxKMb2pq(r6CfK>gbo=<^}>A6WcLQ60a&MaESb+`(68Ic z1_Y0CTSp#VqQA84m_sE73#vl)KTaEGHm_s#^##z6(Y=r(O_mh{g9lvxfqsN3IEX$w za8X`rdfGog_`n|X1tM;e`+nn|I~ZRVq)OtI$Hzilfmf4FmAi~nhx=CfChzOH%o%)Q z)kjtVzX|P&P%;vIF|2aR7cuh$Aok8zX1Tfq7{H;P)AO1ARnzV3l83?Lg1Zy3UO(Q4 zCgM8sbio*o-}~_8$I&0BI*Mp{FyVBlEXLoF^b_oFSx0^`EATFbc;pglfc80%DtlUU zvbMzra3S1=2X+C^@es@pIVi$Gd@pc)Ej;Da7QGmpdowezdhMM*s z`1FG=WM$Ib3NMWjOh2)>T7MTrunb|vCMpW*{eJS?f_Q)Rst*vXIdlmx9P6(eP%isY za*UvXjcTSY{x{zuprDw@;b%Mmo%T&3C>-yivS!Rd2n3OL13c|Jw>#1Of(ZWXKF**S zmhu2@!-KN#*I;U-X<;$FgI0*F#{Rd*^@LgG6w6K`0EHmzb>tg=}=qMQH*m{w`~6Sr=+8e zY_PJCa^&A&wRUJllWew{2OtxJL9>H$D0GtcpCa%)8jeM}b;|r(a#X;Qi!~ z%vYy=oJ@6Yb8$0QZpRWW55TfKIB_dZiO9;X#C?0TWV%q-RmO8x0l_iYI3s)q(xLkG ztRJw=CIkqS{InEYLc~rBAgtWlAad82kPQ#0pOg&`Cm7y%a#U}o{a|NjCwao4GFU-B zH>bF{5b`R3YX@pB3@3lNy%^0%61PTF5g$ZtAKtTAf zO%iJCBjg?g2!C$|I+yA1PYOT#;eMPzS4y!tPl3^fk}7<>|79-)jO_yp-1ZGlDRx9~ znp^y!spXS3nsWHkSFR)H6?y<_P-XfD?4~k3X56xG!xfXbc0@TsS4h+$Q0-=e4=N64 zAy{s~feuPQY~{hww|fbBtmhDyN&PpZOhNrhG4YxSSuOcptXuN*CCA_ZVSO~G2KS8v zp>R9Vhi^xGc)CZ?C9q0uhk%Xz6_@~pnI;8pW~f5O-do@NKqB@B+o#49uD;l|Dr;MG z4$wbQulBlm4Y1|`i2rm21M5OW2Qh&))_F>%K8FnJa5HMZNspwFk6+Y`g~v@*Z3)8z zOyv{rM|y5>rOpXj8J5u{if=5&5*5`V)j2+ngWF+a=eLo>1$tsPg6Xs>bhek5950BM z3;+QFJS{!+U#F(Cn@m)ZuyWoYvkOb6v(Yz zau{F(&Q}pOmh(b{goG@JN(emCsoLql6kQ>Bca4`-=Ydlcao&KkjkbIl zAPo3^fblK>@iiahCxGiSfLxK+rW;F2Q*fYMvDyAna4~@QvJ`CE z4AmxL0!9!fSs2v}vjo#@BWJ?5kD;t~IM_Tu5)DUM8|v-cN8B(Z2e9Rligv`l3o)Ny zn_y-@ryK0NkV|6#2I;^ri5b}FQ@liR;KA@!{Gj## z|B>|l|Jo)VS`l{*Y-kK+&31}HS?@1_H|h9+{CIJ6aN^^_oe!aFJfRg5B$&_``%^ex zcaMU~l!Cyu_9sICm_~bwBi7LWRcDFKK$9%1q9icx?zgLmC1ZZT&+o^rlF7AkHXzTD zUrmqk^|-1Hr6IKum0419n2!0A5>CY~o& zv9YXPej~^P!Sm!oUMwSxBus>xDwG5Y*WQw`pNFfbvHt11bFQO^vIIr)6uNzFMd+>3 z2&f?mTIbb^pg3VxmZ0L{Pkl(J}Uc zs!?dXuD3y{1>3B9Gr*B7io9qkIy@yxF8ziL^|SCss>Yf7cm2t#PG}hj_p?nM+BWEi zz7p6E^}J605XE6_I7R|?pfPOz4I6)l2J}4oU2ERDtYf>uF1!-Gd@JaqIHwe2xfJ*Z zz**yUcHi6i3D-y`Nzp{XY_6~!y{cUgXW;`DMFqoanZS)ItL!`cr~D9PP?dd?+`_z< z@fJFKqKBJIoj2HWVRP;vMl&RyKk}q#&%Y5 zw;C6wF)A0k?`!}(H}9NXoe$1^+Yi`XtmSX){5h<${?=3suvT0a-p8_X`mWCR4iq7hI|)*z;9*3I$2T@7^K1)bp0!wPyI<0Do)n){kJ%|yV8`#G(CV}DNN zFX0!|gzM4F48D#JF)GFKdtf&<#E2=GPeZisgq-|k8qFrzD3g|{a%Ux4Cg}NgUdGii zk9VtO+WMx;mbUMjqmXmn^jDqj8GzM@LkeCwq z=GTZXS0u|#;*o`h^mrdu@D$HM0%Rl+1qTd-zP1-CbqIE&(D*tGGy05CM&oZqZ5ebVP1s%cw#Nx7N3etmL3Vx|PZwwptl(A=a> z-GsPn-odP+Ot8YZ^Bv0u@MqqtkKjbTBZu{N@i=6KfZx(K8IsBm^EPyA+KS@htVBYL zAd5{o;HAQWg`&-GCRRx>ZBC(#Raw8CTS5Un5c+~RrBskV^eVd0Q3U4ymWM74ELLXss`BhUXyJaU+#5J_SNLGYq=8 zX^0ONsN5ph9uQ?LZl!^(jiCOJRW3=?;ageri>lzKjYc$b|Ni~^(MQ128@7pa2X@+O zA_Ue^qvMT29w8yE=d*rw>LYV=ThLyB0tru)CNz;l39mTZ(=no0I!6iUjPW*kf zq9hk%k<6#Vv5mw#qTHUaBH$-@K`(b!v)ABFL9dhv4=iY`3+-`JD|^$K)hFxaFBWh$O9@c`6gGo$NYCoo zNrc2T(SwB6Fo|5Vl$IU#AaVhyTL}P8(ith4WLZHlxwR`;^(@eWBIW16Uh?2K-1{=c z#;W*|7zabfpREK03y-kDzxzZFh>Gsp+MV0d=V8)v052GLP%c~hc4=$-u6cA* zDi$TLXAuq&C0i|}VCH)ZdYEFu?L?EaG8%eZ#LvBAStrO~M?p~Mba}1y;)d6_XC}*9 zgLZ^NL-V&K+;)o#YZQiw>Gy|{4xD%b#Q?XKbVgTCejCbMO9)=8U1El(wMaE?mL^2PrI0gO0Q>-rWuK$5# ztqk!_7I%74Nq<1R;9?qopBPR?MEFwuU(tfl$ztm9g-KH|_5({{ubK|MsTt@o(_j4p zgjLcFv*@60*PTKe3T|huc+FDoHlmM)XeKIm4LuQV)F369=rE*Ofs+syglr`Yp~z9} zA^8aTU)u_}0dDI~fru+iLWlmE#`F%sx<~zO$T^=^ z>3CXfx6;=Gcc#vM|N7n=?cCIik^<~PTt9_tsEn6W)>oMrWB)hZ9U3zBGw>v}u@Rly z^urN$&_uNJB}HY5#X<2Ai4?F^Tn_5%Uxh=rQTktdet2j}jS|*09DTSYgc{N&zevp=oz81A6x_Er>5F2wjWzC?X#I@~F-<5`b> zQgR#>$^y+2-Ier~U}up5 zjt^J@k{y#UssWL;I64UxgM35|T5D^oR~J|z*C~0#^sw8$HE>y`Wp+`#TGwPq}>Mis}laT-}F~7U#pa zu6K+ZT|Kc>L0U)BZ{1T|V>(!DM2e5B!r#8GFz+ET!Z6`b%N#%3h#10Jbbq9cMIzB& zYiz|Qdv|-j9vT9-0+eAQQjC&4j?I4wq`*#Ed^phl%Sh#qTNf_4_>Yem*|!#VZik8$ zwG?&>e`=}6X)J3K;_;8`YSmySbGUMo$ zpL16~1up1unZA?hFMBuB;%q2!LZ0`ks;ZnkUljE%*?0pvG+up6SPO*gxl?N$wOHrqk{B{94K*SJ%s3zeIQG1h;?r9t)$-+T0NA zF1&A?6PH4S7AS`5sner(k`CrLtR*`u3d)Ga(V=FF9@q~-B_~D8>QeR!0nZXL7jkcV zc8;42m>4qVhq%DDoe(o{{~Y&L7e>MWZlNyJ&8VMqC;FTUs}%4Pip5_0vsJvI<#xs_ z3&@sAV8^9^cSGUnm4s!ABI)M24q=Y)wj;c0u^TccLWKz|N*X56{IMkR=z?Lzb5LMd z;UGU}iVb2~Mb`*m^VLu`K^+Q9RxiPwOwV*cl7Sv{Db`i7G8({3*>}YP=&DShbkEy1luo68{>p0Sk z9TtgZf%NMWlLM1J7T2JlfW&B&j_52rpEtM<=?0M)Tf*l<&7m!abb*+kIKki$2$lF8 zB7lpTtZ@{Cpq#{!1)@IRwQHW<0{H)x!9Q#>H1MT%z->rMHic*!iXwmy;Y(Nof;qhe zsylqbAqwWfOwU5VBEWA)JWEj9p|}TQO^Vnrw608?XxS#KvY398XEy&AhXVo4OT<|~jd4X{+Ck}#Gtd1{-V3cx5cxTFR?}!7shL`p#oKzX0 z0MwI8z$oT-p*~H{Xe$1~p+;g3$F25=gxKIb`uVzEeQ*}{c|TY#<-p;SaOtQfGyxM2 zud9ii{WJC?Y=LrqsFjU0k$OR$r+#Dz6f!Jq*aq|Ug%QSX+|TSLigov0B-aa`n{V>q zI^j?(q8fW?VC9HcW@g>K(vp%6cx+tk2|hKaX|%Tbt;byq&LEC3VSG#x{w#uAT)lVu z*vs&c$$nb;+}E1;MKR~T8{)#=`_SJ74$8z@4`UW>IVxNb28HKGBz$EIDHA<>X|(qK zqoY%XcAo#|F@c&HsOOn*bY1|qniujrvFN3s3T|1Pc}7eWpi65PIg%HXSeqcDtFVg2 zH-$h~K*VY70uZ=Rgu()!fj?lXnl?Bd_}#$V!YAQPp&8zBXd|F|DxwO3s4AGL?L#m^ z*@7aAXD={G8q*hgTOA?}trr9>cqN?NQoa*AOtdR{h1+Q<0;#RGW6|8L=o?9l6KSkL za<7>=M>u3id1xL2`HQ?BGxIQT!Lfs)d}yej?NXW*=qLcIZtkOAPJiFN6I6K#0#HRN zw4a@ZSq)|fIMTqjv%$in4L5nc%$!PfUI=#iXjNBZ%f_dJ=B(Zg)@Hxu&0i*oZ59%y z+bxL72g{P2w(<{(&z^i3y7{y0={~4R=T0qzDTI%jYk@c~KA=>?ZAXAwvIEM)mfU*q zNnR>CGS~0#kL84jvr65-{O+nit*!XeBi;7Hz-f(={&ZmFBoPr=98)A?R86m5URqC6 zfB*hs3J#^vdXrvRZS2$P;x)tV^r+Lz7>im$?>-kER#Yb!dR*h1J8CcTGAJ)bJ;*8- z95Md5r*a{Z{Ic$mErE3UF|CW&56Ndg1_2hV&|)~vf~!JU9RO(vJmZkV+m{Q5xg0jo z(Hl=^<(>T@0op=J3Lb8fc!4WWJr_PgL?u~WPnt(aI}hE3SP;1T3(T-2B|c@MuVD#% zAte!#y8itPeA)?+fyJPC(FoWWM42ZYEn4!*D2c#T$j6E@f+}Sg{K1hgvR0XB-f<24 z<6&8F^N2OVhrlc)dG6CLEPU9J5ZWoG{Cau&T>R^V8sq7~;>!w`1swN}Q0Y_oV0cFC z9amEN9Eg=w^w?3Gi2)zA$m0EnF%&39WOIeHQT#n?GhV+VRFj=w(JfH;jgUAjvVlv& zOIwD8nFvP42j7|K<`X#>=KsDG8@w>ynSlZZz-R9hqjZw$mc6PL>o) zsjxs~=2`+Ml~_*~`cn!GDK_EHP+y_rBSh%RT;9_KDU0yksYaoC2k~Sz@q&xv=C$3L zzfB5DkV~B6{3YZzrJpo=i_L55jCA^HDJyoMbSp12{%dwdY@{Z*-K%r7bEKby6VH=+ zNy;`r_m^_Fq@@h0$| z_*S838+xav48bdOYv86xj`D*FVW#<3Z7iUBEJPEA1aByVP^$nX|07ungODM`GhR(@ zWbz2lsi&uMU(SCf7OQDNQ@vGyqd5KO+3zDtgyxcogP?lN5;OwBTel{*Zy*B&_qGCycaO(E3~ z>A3m_K6oVvGw8%91mY0ICMA{G8MCY76`couN^s9mODBoy1W1Ll9`zH5DhU<6wTIQ$ z55gieO}HmlpUTHWO+-+@B_M+0N`=e|gZ~S35B-YW&^JAY84p}fI9?o4)87vzI0#@% z?$-5g<$$3gs8#VOw|@|)INf{q@9+3SSi#!mLvqzDd~C@5PX(PG!O?t2d?x8lYi5u# z(Pig36@ApKjl>%uO6XDuCi7%*^fn_T&l-F?D;r`PwC(KEkd%3OBzxsR@UlLX1I-*A^_;p;kgz7Ps_D2G^hJM;aSuMR0 zJDCv-g;O)e$TT9;D^`Uhp|F0zLlHvh6J}+Nv&%^E|t;mrIe-y-RHJ_rzl-pfRpam?`FSjR@}eZ1aV%R zSheVSF4Zj-y6$&be*E}BjKpzYEPK=s3Ix%4EomMf`(pt0Z_ruzAdqU3$(QsPL4<(B z&&|`jLU1F1>J!l~33C~FIcYjA}0^BTa}w>&IE!Y>&h!<)%8j? zKFGfS?g+fZlYd}3Bq=z?TYPK3Tz}E7FeP7|2B&V7&Dmb(kT)s0em@=2Tz_iv8uD8B6?HpBETgx zw>aaynN?4iv@l3XsIK?;cIEbP)-lK}9|5_={SRFTt{1QZ>4Sb8b^d_45x>0#&Q}sI zGPQADC@yH10$zeCJBjQUdgAI%;7|ovyOQ$Eu$t<>)7*`fJ8WY~y6o={{u@*uhMPKf zn|YWus-CGcpBZwQ5JwJVd2k)2M8bw*Sai7$t6in*0lz9UJP&)|5)38yl-EcaZc^fR4=Ib zVz{9{*d}8w{I1Yb2@gA?;kB7E?RCpYs2~KWqe-`qo<KMkFmD^&@Y9-z%85Y(o~RK2E$Sng1Bh0w0Tte(R1A1-!( zjs*rUSsfW+AvOYm!^q5GoX{(C1JT}t;KxJl>b%RzlbEn#k&4GVKBc^*#1tOpX3^0B zMt8y9>B2p1bn)!<6G%n|Z?KGgVZwI@+lCT@_b)9xmx>jP%2}O65^F`V_#25Vveh>cZG*Xci@uKcT3UNvpCq~84O zg1EtVB(89`+ZSIgsyIoj#%G!=qF(vKTleBOm2YP|^Yv6zkp#~SO~i*L_Q)FQ9pn4G`n+<7XnPP=Lz4NL!h;k0ce5(7A)%V=4a>R zLwIE4QWODaLW@CX2X;*t>Gt!_Y{7yYQxnoI;h3)qw-ADXBJ770Qb5wcLujjZYH5`K zbR3!N3Uz$AO>avMbg5*YfF%{xNE=MI2U2qtsOLO2j*Yqm#l8vz+s0&TbkrmTMO0KY z__jYhIbH$KBBmGG1HLOvNJK6nkg5e^=)$B~82j|;f%Ng;dgTU6Qw=XxVgm@BIvFiS zUJ8WvAP-9OcdT_YkSIlyGoP1JrzRNZa^hmL@|^STCx>(HBRLzO8QhpI%#FzBCX%=M z0Di!eiqQ$5)H;bpjI3t_;PaR`VZsfeFxQDolj~qk_wR$)-Q7hjeQnZDXZ68WwF6FA zANzH!K$}3qA-)Z9;G6jFErrxKJ>tp(2fmu)`l0{K6p-wUBT3ce*(~jIS3rsnHBX+p{~Z60!qc?#uGh9)r$wc_gW?(p z1nSM?_ox^6z1d6<_>0OP&<7A9ab-n&-|wp|T)Jlm z+Go>CwZ4dK+*nMUO;YBx+jV3b0#chl`c^(F(3zYujP>bZfXKIRDFnP-0Gu1~X zAnXVMg3!>CY*jj*-P+ov(9>0e!W;G}_7}-npBdXrVN=1zQ5BkAaQWEXfNl`WJJ31@ zCuZ1apaLKvZsgv}&)jqYbam)gSWvN2C+H5rksjn_jH$9Ey#Yp>{^vw=^X5&CQN%P; zKU7G3HK~e!1+H0H!xeQ!-x_`^L73?Qvw8qzPuH426rkk&D2NCUelcg}q(23mg)mD3 z;li?N?}yv)!9ocJERae#!-T&D-zy)on4hg8+*iQ8spphghLOQObW=%LZNVR-9S`9l z=Q)u9#Nc@svO(H}4|6Wm$L(=@R4beec~ z(-UcdY~%mUa?d*dvMmuLz+5htICUd>=y&GdIM(KV*cG=7_*Hync-I$#3SHT@u@fF} zKxnY;Lu}x$HbiMxG|hgYpx zbCPdpUc3;LC?(V96rZr8R-QSV&B1bgLh4<~NdKqg>7jW$D`y68f-)Sm zo40Z@y7s3(p*n6~JVFwnut92ghE4Nm<<@ZQhAlQ&-Gc`W$KiA15r-9VQ>ywgo_O57 zk9{G*g3&i2MsB7IEI49y2q)`JSL5TMfg*v-QPI2bAdosNuw~2M@*`@oE>IH_J|nnY z9j+)>;z+ARkhZ@xr_|0fXJm;<3BK3kHpg6GJ- z$X)~Jmlex3mXEq_1-vBSSL8LX=d?Y(jf^nA3Vp8(*ynyY-Lal9KfNw_;_`d}{y#q_ zepqX;HBbn3QyZ#5jBet3^_A1_{F5hJ^*2OR|CpL8ZfJ-YbY6xpbP9{o5paRMy3$(0 z&*BuU3D@LdS&lE^;8}ZI@&N2K_Z_@V8aQB;d>lcm*jMr;#CM(H?Opa+Ryru?6=0{Hl1d`wV&3Fd-NeU){A9OzhB% zjEwI~FR`d-Mw=OO$lYQ#O6M?2Z~T~*?vZX_9i9~{uveH}cSD3q(}6e% zbUwRrY~1sjXB8cvPpeDJVCUkcLb=a$DDTe8nTwrvcD~_rZYuDs%5U(Ofu+@1$a)L=Bu;Hw)kMea%@K@nJC_bT`0X~Aog znMcO!5@SoZAY(DwTsqKL&ZnkY%WG)7{1DuI<|ZuqfWcrQ}sa*CGP3|!I6nQcJsZ%`un8GpQ*I2!S36~gTfmWQ?aXL2| zqNbC`;vC!-;mq~{s%#r9!m5dxz5YWOY#E0dL`qp!lw3O zu|U#Ipz#a(_VT4BK-t@GS1smq5sm4_%YES5!it9wI=twXpNf`GTibPID(HZSpBn%9 z10o!`EmPcFjrBjZSCVtOD&=Tx|2Q65dEheXehU{YFnxYu;VBmv7fKC~)mq#8oJa&>T>MYty{*M?piEelw)|JD2Y2FgxMRfvizugUhEa_>`3)L zXY@>rixbP-%VEAbraOF}P09LfX2XUZ8vd$-th5r$hW7GBE4t<6*g>r1?Ck6eVUFJi zWoHK%#*c-s>`HWnBq&x_7zZk+WdJ-MxO*-CdE!Q{3q~wD6z|LpKrl#QY1+^Zvj~J? z6SaNIS^KhAuNLEfX~2Z9*X#xJm$%NVDJJD`ct&jsu!sP|AN&mP6SHI$)~ul+?S!eR zsl3JxqYK?GPsR294L2{1Bq(<&*4EK2LE)q z?|+VFKrs#x-fW=9OAcX`l9luslHTfST_0axK9tVE!9k}Fc+G4bN55!;n*BEJB&e~R zif)fA=ApFJiV%A=xWSm~5xw|xII=3Uz7IuRk#nL4qurpG=oMfn8Bl7zNNoo_AI13; zrcAMgAiv?tdqyf`SV$*|tIXNSjOx`ZR|fj8_IP)mm36MXlNzsCfsH%v0hRcg?db-p z_%pud(JmD^KeLXmuK5wW);i73&YrTjk3D9zn9t|Y`j~D?*~WtI%P>@-U68Tcbk7}3 zEYxr?EyiPxsHrXB^_Vo%6l7bZjvsA8`hPu|b^QO9(39ginP%GQc0*z)s^OaTW)Jp7 zUb|+F!9uzvZr}qY@Pj!;zt8I>4p!^N1J`^FQhjyqdK;u>_nnMDAEn{{LD z{SC4D8$`FY^zGje5mxbe)%WWiwhs4o{k_jf%^lJ%?_1(Ukq(cjFN8y3++ADBai!;T zS?*`XGPptYX)5DFsU0_Kd%_INXoh-yeH#_$7IAvB)P7PtUjJtu0$vK~GpvplfN7M?aNn+etW{{8!+KY44ViP-^kC1`-@ zEpg@2FI}qP|j)2d;~j_O;>LzoM8iT?(VUL^n9N-v8`OW7zPMIQ(pLWvBt=sAOCP^ zu^3rs+`M%w6RL|KGvNQ#R@p?fF2(T=y?sDf7z;73>ZyA5ineOhN%TC6L;GG~5pJvc zUMJ>@e^+M}BY*u3>bkltFm3mdzz?pcM~eo|$ts%*@P&4ibk!(I0M-qx=xjxDJ3;qAPDddKRK!x0Wq1<31kkpjkh^ z_j~W=6a!HqPaZf|MkQ|jhaQ1VQ=PX8zlf2R2nX7m?435&5N-81$sNx&3L1R-uw{yg zPL+F>!CIkeNxRMF-_M5c;M-Yw|5u+Knpd~S>bHxBPT(ar*>Chu%VBC+7(B-4xiM%9 zDGuaOOWeRD!WLv5uD>UUYuT-jVScjzaH_diuyzUfN2d+)orvik0i4hQDl~JDD7eA! z;j(`>@gxyvn&e<`Snf(>Ll`gHTHQ63kEwCkJA3Y0_O&^H695O)lDJD|oFq+55sR8{ zV@`6b!1ogtG|#`1mnYBb+Cd8?g!D#}S`W;(BUoe@Dj08w*hK;E(Q7bYy2Mg`59~W^ z?nN&bx4y-(B9Jzu+N6rVhLszF1_IzVRdf82A!_I%av$Kf@1=ECnKhNTL5vOXM3#LS zCe*^!N-{|T~TL0iIn>D~$KlEKY6eTwIp9xuKcwB9PQx-!BN zDlcd5p~X7vIg_pa9{x$kTwPsd)~!K*u?DOm;yDxBUQq#)$l(kBZGOYPBV~waxhV~X z=-I~panYE^nt~TXLzkkxQ16E2@lYi>t8C~$Pz($Ow;kgw`*@}kdPV_h=|fosdrN*t89)b;de=>{7+44ql&vj=Zoxl*TiZpgi&a#zAWPOb-<7#;RGJ4c$G7TT)euS~7a_AGvOI;o%L z-k@6AOrFsG<#fivjeK{TKlPl_T(s;Xzp~Qmg}J#EYdGF}OsYI+>cXA39nZA9#xZGP z$G>t*8DIN{M);zUu$|Ozc!SKrF&S9Rk|Tt^j2HuOSb~4`9MCBbJL?G>VxSJ~gve_( zepb@QLua3t^n6qSe6I_@G0;GlS;?RpLBE0yo9TvD;|BXHk6lPw_WOsWaaxIc%z!tB z?Mh_6crh%5n3$u9PMf&?m|W99AB_(kYLS(wg$>p|qWd5_F>N^>4h}zjQ+_eAj9c*n z2^M6f0nRDj|}RC-|X|JLFaLDu#;`b+0)+*O>|cq2f0^Jby3@$lmPe3{la zYisuqx*G{&=;}!$%94o&;rZyPI2X^-yQ$ zd8m5MUrTXcP&G;x3sB#Pye>5tC+{J;w5>88a7+izV3T>NbrWfHA^+RHeS2!DR&y1= z3?g0%dPqpL@LFKL7!VfXZE1)FRu|Tc(ud#SG`+xrR?M`>EAI=R#h$zH4@U5Vnvs!F zfip>-fQ{OV;^O$3c>)RZJI?SDhy)PrN!wrWkRk~kMW~^)HUBmXk3_Fo>Y=2fKZwm6 zipI=Fvh?JX3STZKG&_7f+}*_Vqar&|_1bha7W zHhz_vme%-6cSUIs+sXDp!LiMUP3k_Kc=Y)7F1qfje@=}ial&e|zHuJU{)vN!Jqzd< zeCN+U$>(Rbh9k&Vhd=z?^hs+RcdI=!NzC!xm8&eqdV*!#Bv;U6xZ3YdGu%@89796$ zMfk{4TkQAmx!^`hLsK}jPIb>^b{ZCrGV|Wr?^SyHR)=dYNZhm7SA{ll&w3}9@9k}` zj!5`_;+`D(yhop_vRrGquar#QxLj=&`yH9AdD#s+GLmw;QVLfeqW4X`QaV{?wCwGW zrbb^4PjX$wo%HJBga`81+{d!Sdt}$8o%7I=-_oc3QfGt8PTS>=K4*umPdqickN;8V z=GL?F-?Ja($)2W3PM27-IeS6r`-tRpX|L`x!p4@v%=W6fE_TALH@R*H%>R^lt>f$C zetZ=*Ll3Djqh*sy-z4aq#!G^@kIBir_%dZ(DD|Q{r@}w>*QJ-Irv^WM-IZMTT6bTX zx=jS_o}2}P8;(snfA_L`Dz6q7a-*VDd&7&Yg8Z-dI;`y)g8*W0UC&2K6_P}U1)7OCJ{Q?yV|FNRYlCs zAYV@_McY@54Gwsl6r4TC=cZzG^wnj>19{&WXiV#qO-o7@KXn9X?ky>` zw0Lt}Tg7un*%kVwE-t2O6G{2`8_xa$NtE|ZS+;dza&mR?@ih4r8#lTGIdBNFcajrm zOv|~@?zy`>@t;!R^(pE$={q@RTkN)oTVMNe{Y~4s_HUm@Tqi#4rSZxT+j`{aY|1`Q zp_FASB7?K7A9RfMf?T>UKDU{EFx1ie#+E$OmdA!PJ)X6Cz2oaj%8}Ligkj;rhK@5u zEeFrPy`@v9%iBCAhqx1p|bsa&r4WVTw>7YPk-UHG12Xa!OG|Ia$>b# zIB7cX?-?6&P)W#R)cKgC=-U1Hv+tSFFN9s!@#FhJbKJTYyuDinrkbNQ1yzn5Noc6Z zLxTcswekZKn~5-qoF~W;V}!4YiYLHSaw}txDlfL7)`-*ZIpoGb1W0Ua>`hBUg7kzc ze4|IPN-wi82DyM3UN^X-L_9{k_JKxqcfv?A2&?YKK}l79=@#; zCFbHp*9S96b99g({$+69Z|Wz6QJ}e`bn2$)Ghe(a=?~%o8GqqEhR$L>_|qBp9D8dC zk7Ny%@#e=f@9>0fYsoTfQs~O^(ja zrNFUHq34bj5fx>EZ;~2zqCA}FAHtGABZLD?OtmML_auf1<@&sy)FBwHlSmTaD)5Eh zC5#nE5q7p@d$vYQtol4Fo64XNelB8;U41(`y1(c+dNvE)cgO7uA3wgvCiHK?#~E04 zL}DeD$SiDoB47G-p8>~Am4Y|WBYxHJN>Zgg4;Yi_mp=fBE=HC0@bDnxe{LY_04+~~U#-6UZ8dz%nG;i+(Zv1tED8?gmoE(F0>$bKw(!chs zB!BUVw^WP~Zy!D}0LK?+7L6*%gm}L?GOyyHic6}}`V{)mZCw6wam{Xbfkd-74W?yq z4!x~~2 zu(R$_)9wf5owet+f*uO(*HO7kOM7~EeEuCS|NUz_KMqEV^`?h1tz`8ce&%s~(pbr- z+$n9E)rNKIwa~I*bJwXQ|BSIwL*T=jz3&yhCms9!wg z@xt*uIqRJ(66Mm_%bIh4)Yn^nKCNM}wzKG>+OTnX z(K5pd?LDWbpS0Iz*d1O+WBPOKEvKwR!t4n#zO_&n5T|rv2LYX>|p8D@@Kgui^E) z`HkjrM4p+UppVnZsqdDiv(u}R(YP5kwk)*$(j&SMZ_V0-H)#A6e_I9<(GU2}jaOUI=eaOBepz#IQZ#74` z`?%F&7&GsEmovKV-o29#vKZ%=luUM3ZnY41vDNpkOZ~<8yFnSG)iGyvJM*fh;-3&J z0sndl#rs$LVjG?F7nNvMOm1A;)7$Ga7^8D4{PEZGkBP@V>I!i=*{I(1w4=jRGqLvc z)HV%Jl~Nm)%H;Gyicqh^`Fg(;0W5Yj$`&NyFbEqjN9cF4@q>PxG$LWGu zGmPDBC;V8lj$d5|?49_e?y#8Z48mxnbwk3Ji{a@atcb`^kAPdnOJ%wKOvK)%sA#(@ zO0<|9;KhTRj}qT2EN`oOMOJ?$Ds(JG#fh&FmsDSj5^q5hR+AkNNteKkfVgB}q(8uT z%#cH9cT5TTC~*vP5Wa?S=?P4vA&~DRlrTVK4T|+QEI(Qm!=$97M1e);X2p*(jKxL3 z*q=Rn*3;fT{m=}$Hb5$Hnj)&fR63>py#@w;SmAYg+{((@fpz+~tus(-7wNj0v5Ee#}qzD1~7n1y1X3s-7;`sR&X*4>adJ*0W@l(Xq5^J2= zV2%|xF^-gT?)y|kznE_#sO5IWLYizNCbXBGT#Q0KVc(Bz+<95&Zv3a6b3ZfX_B3Av zD7LqE@$TKb34vG`fkc>@3_c9l1iT~g1j#spjjfLa8nMHN4^Lv(RaLbVpe-(=Q>Rbc zX14>1FByR*icltkB=D+}HwNNuj?cIr#@gnSiHB~?O%ZW3UQm(*IpA9nA|r-+>LA58 zv4zq(+B|-4C6?jZmogx{;okDs342)Q&AZ7L(b%wBCS!B13*t0gg47}rk!uqBYqRH% zpy3vJ}jx#KUp7P)tSBrf z2Y59^I8^_g@J{bZ(LWxle(dA(k?q=NL_ORDn1A-=y3eo)i5usCVhQYj-0Nc7zV+D| z#6vct_O%~}Cw2yJlIN-DY7R6u z4y`JExwb(3hwp5#j{8iiM%d{MOZ}u@`@7EQ^M3y|5vP8mRh_{%?@Gotw@LZ$a%s=X z0&IRMG`N;B-|28$Fa2vqhT(psl+lyNuf0B2wr@Y!RrI1BVB1;n!W~Coq z+QISQ8JEyl;Y3@;>42|;EGr@fcvO8*CKDsM4mY*zRaTZdTGbZf<8y(o-_mm5$Lyw% z&Zff;x+dCJ`|Pr8&$8eD;2PB7Hc#7Lx`jmM3No`papvFj!R^fnad8zN? zAFFEa?TdtI_f8H=6B7ql*ZN!B3l_90RlSC*M)km#H}6BvO-bn06`irQ?vUZFh%E1l z4}JT>^!TJ3Yt{S}n>qM5ZCd2my+i826N_81b2FYMw?p_vvFhFD&wDUjt&cjDs2{q~ z(oLW9X_>~lRhRR$d}3mBdU`)E7&3Uacm?PbfP@8`tgKNV2fitW)F=M4B0!Bl)!(GS zN(wrSKup>4!V(T!EjZ~2?+$2*k3{eeZ^WQYXeA1wZiK7dI6 zphQ6rKtf7LFoX2n&c42jB+v^ijjL*3c|QVeAqW*7A(b1?O^YBnxJX#cY>N-g69R@t z{K>Hx1=6Uo=kjN5m5?q{%%N#l7%rOcq;)#djFXPI|3fuEvZCPv*V^YRa7)J@;=oRP zL(Z+XJZ@t>%2O&1RZzdlxDJ~ZlzTyKU50_b=epOQf%wx>TU%RF?vNq zVk*E*(T}n)7JF@dBFTgWBZ!E@!>7vWuz6w@4A)!}u}k#wN2_cjSNE-oy);xbX0F6; zK{=%M%KF0C@e^+VDuZCZiAh_+2}d5GOYZW3Zvpw^6U9|EanJLXmX@QJjqvNNLWXY% zl$oM~owP-e1$LtOCileEx&lS~b;39{LDSQtRY2vkgQP_IrKc zai?HIda$J-dQ-^$ng{LCP=nrp&jgbVYaAAA8qB3V=lo$N%FvFv%**f)-~I0E^J%_4 zh*+~RMYm^2DSk;(fgHWV3d4e~;~TB?uIrh(q@|}kZuS~jz3$O*uBMNUnX?T79kM3E zd!0wNSBaYq)1UHFKBn^av$?VC)x}rN&#xMhdAMPvM9zZv)kAf)x2ySWR^~aB%j`|E!*GYNr?(^R+SQ!UZsm?aA(R@i5 z*4Tgef)?NNcS2mY-ol%+Iq7bt>@193uVeRY!`W+jnwgWSBlMwq+3zQ$%G-Y3)bPzM zt1*+@0IWetA!}*wE^pm0t3F=5Uwol+`KNilsf^iDJJ^pI@OL|~etU4utDF1!9VVT( zdt}^S58Uhjl6tbaWJuTM)18y12lK2N*0$!KXtUM1#dW^tcp=BR)hAX*M7UjAnFU|3 zhFeX*{mt6)h1E2sRMssM5KU){-gkvLOS5Osu8b3PQpY8~$mO{l!&d(mHq{&@=5j)0C2am7mtDW@ zv~k_L@xwHlS|TN{t~d`wi5_ZR0$Dtrt*wp?*Txg8+RyHF@#GVE!zAyrEdYrG6VG1E ze40MPWMy?Jc>hV-kS^q$PGz`_TgGZ_iZuQDj)ie08-2vGzfQ>|qVzd8KS< zn0}}(N$Vu_aSNNe{KWV|_<6QV^ty4n6zosb?XVE_6n%5-wPER1hAJ;=a+SsleIhaLCp1O~?K{4H{dz}p zrz6(g$P!{iX~kQlVGqzDHS#W`f5RX$@XN?>!r?FDcye%9UN~S!^o9f$lzLNIx)8jU z1jl)lc8~lKwgXEM_-rhP;AWwzEsG@d0z^0nlT4TaFw&_C?4<=Jj%#PV!qPQtfh?Sy z!h*r{OiaEivB&G$@g*u7(Cotr_71qP!t!S);RM%1i|u*QOMLlv8$q;zE?>nHJ58{m zSlQV>6(vXQ#;*zZeDZVTVUd`e&oaz~vp+XJG7_a@=INnyta3NB=sZ0=+1cx*RIsga zJe)teot}?LPnezK6_c~vt@-hPm)0=%WyNB{-@aXlj9h{5&Udw`u`v$7G1jVF!F2e< zqJg8yX25=9PBW!{#oizN>)(Y|Ipw4w8}?s(HrB9cAUq6VZZT6D*mo_a;jpQt^?LHe zC9f1!-lugb5PM?E04-CH1L`$lk^ub*SxgoIdX+OD4--~g z6X~_oL&mHxJel9Wae>)SAl#0h>}=_M@~ctVdwOK7PMoRn7$+UJ@@zZH|FM2Gs&L?1 zb`NAQl~tnukE*YLsw!RoJ_r)hAR#Fsp`=Kc5`safbccx2-64W>h#(;)A_CIVB@HT4 zA|M@yZlwErwlnwM|F;%1%QZ5@K4-uCd473mAon2fFq^17;;Wzs#T zM!nO>wcodM$!%@>3pxM=(_&KGGd4caQ}l7b_B&s6_UTL?%=>^z04M^0#Mq!B47h-{ zB5DRFG5xv|u8#86s4C2k&p;r{sH;;@cj0|jygcuq@5$X*=S3o3p_|e7tx8Rph{c2t zy`%N@qLL-Lb?8y#yR~uLrKyMx9DxV1+Cv{UlT5nf=T)^ooJZ};S7z{KyuX9ijPEx= zC$%>?N}!$_KhNPMZYfgm(4r5Elc4u!G!$d?YCpB*x%|Ffy6OFi&Q$yjeckrc9vqe= zx<0C{id!)TL@1Iw)Z0l0zT(@nOEh%HJP#JA?#UlOavWDJ% z7BBR`Mq!{(cjRuVy}n?7IB1D_Tj5h0@8C@6$iolkNwHWvd zB!evnmmKgS=xIa2w~o!o*kL|2?&~XJ;4Mf5go8Y);b4mfaswaaY4N8wF7oPC;=g>U zlYTZ8RD-TrXKQGcfAV16pZNCVX8YbWjKP>Fm(}VJ6)Dj$;qhSW-gS@x{q%V2akAHC zFUO1xUr|d$yn}|i*{36(xcSd)vyPcgi;i_Z-nVbg-S;3%=koT_>~cf!8n?2t3Dl^C zVPL9df4UA>R;)(i+HxUd>r#LG%Too$RX-JJ`txtTyVxkk&Dw1kQQp00Nt@!*UK1nx z7i-03S`9X)&Wkn4(q+jD-gpAig;WC0;ij5GdApGA$*|M zgm^Sr4#?+}gIP&G$td7Ah|2h&QtEF$WuYO}_lVmWo!F+J5rn~a`{)U+r*slnvA{jc zMb?%AXeIDl3j_I^K_qxpX~?#ZG+qIh1S72N4p|56{U8R3#3N?|W#C73!`IB#$QcF4 zGI;n~L45b>wTXAhb#SUeJnC(5)gtB^iAPy@Z67lUJt4IUEb&pSti{AQY8sc>1G9%7s5H2_)< zbcb?3Hy_&kksyvUd)f8)djH78sBtI#S(i)JoerS9z!($rZDv6=kBNT|KTV zfoCO-&(Wk>%t!m<_732ba8EMgcN#YoJcsj^>8AtR&%{X$e@sLUU&VA|_`NkNDsfVu z(|Jb9-P3E*{Y4E+$)l=g!debjua078KxI50Bj!s1{m~uXZH4U*$~yz=M>oyYD3*)n zI~-A{fv-&NzSy^US0Am5oP2cltNDJZQL;9DX$7zNSM7xOnI{dItM2?$%TI5g zL+d_R|Dgmd58erj0=DT`bq+~LUSULK_4gus^bG>W zQvouvAFIm8%ZBC~E#!4m=nSjgbyj$rh`n_@+1s^0K1rEc;j-MRR2<>p?DebXuJDNG zVIT^dH7I`ifH)wa(zpRd;j#0_PtBq$=F5{#vphrFl&!)XGO|CQ8J~Z7Dm+dNF~oF3 zlOiU?xG`+eH^!^p_}vDziOlAq64ELgs8Z==*bHyy}W zTI63S+gz8c{C=5SykOG{GFXv7Q#dfn6ZdNcvCf~r#a1KDQv}j+#Iht51oNZmc&+;? z9ch&&Y;-sf0amOSNCitZ*k%z{38rX%P|Ev*;!8&mWe2=)Ba#^fz+Z6dv`geGkr(&b z5b1a^B%+7mlt42S5e!W=j2~EMe3+sYkt%1Xzo7-p6e{*2TkT*M2auE=^lONo8Up#~ zE^7m(?I9fcVgW_#GpXL^n$D&aUym>%S>?WtB*PV7-gbUQ*Hhae}uX+ z^Z7Ac|-Zc)O)4fEM6i=o7T89_vG6IS@?+S3tyf{MaENDTx-Mb^{)h z0#xdxA4D?E2Z6hY$PjejsxHtkw7ohKzXmYMSN-S4)R5?YnTJQ21BpLC>cDbJH`~67 zT@8_Y0kafv-P2&&rEG@1J{0uIf5G7=tO2DVAkPc4R)PFAx6~|0SDCNgO6f&b*=kvR z{f}b)i#fontmp}%sg2#;K#*7dXNvs4Qq^~$s{aS$3?Fdf zr9aHcD_}yTPvRg!{9WakC7%dXU#Z!)%B|p3%6PcFS(cbwDQaOs z5FFM4Rf`l?uWoPbxwbvWI9(VjaUK@y?8t?dhJ&vixq;tWm_&Y4WGDpP%ybw%O25q# zmZ9Jn9sR)XJu|n2SblS;qaGQGLW4>D;8JSiEs z@9XQZTT8~Ss!}HjyM%>PV*=d{Y*#RZX_miKT&XWBRSP0#|MM!%FyixdNePK$5*iDG zZk=m>tDQ-+BEAOxFJ927Y0~z0QrqyhjmtaFx+b`|e%w!OI~&vEs7`lryBVPWtL z9}kY)m6r=ph(Xsno;QMa*W@FVaUPq6s6N1;Cg9T3 zSHdMWpPFhM6p!qE8kt*};MdsDFqEiZ$A|ys)2aPRtNl&oNZia!W;IQLSKpq$B7u>J zS&`KQgrBfX2DiqgZT%5>Y*w88>DGQ^TB22CVGB;A3I0OtT$6a=3N&jm--)Gyr|aP*@>rL?Pj#SK*D6dRo7-@ zL`oI!|KrDn*w|l9z&#~UOB2v7OuRRpDJ{JYujy^$bV`ppJRcwAHDwYKdZ&HF1N6nr z3>M&QOjKZ?rzkM&R#1+_;#v-UjxrCY)vS>@PTHLKm-yzEMi;YSkAG=)NF) zsZC-$tML#-8w9=|aEJ;Nu~3h%P%>Z3Au;atQ92y|S$lQHfb+KVGj0MgmSoA<4MInS zr;LWdD|&Nu6rSE))-^W0(ne1n`|oT$413wl%7l&Uk!baA z96z#AD1rovkYMaC_oQ5r8nT0&E7=@*7{DG(t-t8h8e4^cP(HvYWM@N~5$pr!m}G5DFz43M9V3i`-1G-ESY!L#3gHrrW>oh@lZ zB5^wY>>qD!fsyomXSMWcf{22HX#JaieoZ;)l&J~@X1VTtMN_s0;TWhCzVpuJ~c zkWKvcfYf+t=DT+^S<17JvV~1S;gPM@rB$Bi>W_|%TOwghi+b?_%r3uVu2M7(YWguq z*7jBuULd=(Wn!6&c2Tg%b?>Jv)nOr`|9%^eEO*%nIPSy)3 zEp>n|W^}`0V>^}w4$tPm^GzMgYu-;ZvjnF1HzgjMslTt(!qL>^JU;0h+nX17n8B5=(hc|MvEhPaTjsDyuFJdl3%aV4g=g* z&mpUVIu(dg2z4}q96`uQaf>lF>R4VeeX&Vq8p`~dtRvWkIP@Ej)Q;wcf+hcP|^e^ASsK(OpzmITl@Bz5lJ zMGX2&5S7p2j)x@D%LoP{n}+uppN^B&yN8vJzcJ+Ld!9$}S*Y<%Pcz6UMPL-x`w&$OPsIZ@m1_H8p+ z=y`WLi;>|AnYQ-avio|i02W{svbS%G8n{jI?XGpAI&{8F@K6dQ@MqwEIkez{2MTS9 zk0X~MLzL@;zN74uxwHm*39v)=A!-T^u2P=?r^%bXc{+l4-iKP{N&9^&Ir}3M z%9qhmEo&9{L@uqu6XDI_m-KbGcY8C0rxzYLI_$QFd75;-<>cTQnC|wKdcmz*AX{i| z9Zn9Vte1iqB;Ur>in3(T+1ArXP59jN*RcFd)5X{DHM((sCguUP2;aNj#UUNa&!4Y? zxT6K~Sd;juxlGe;aMEQcoH2(x*1)*c$d{(`hEeo7BaC6gN95dj+K!j*#jjBu#`9K$ zw>LLqf}-@Mb$-)yjji{5YZwiv_erPD9L1x98{f-IZhT|Um_%ok(hxPwtz$d~ksR*w z!s_+wsd8?WHJ$d6^c{`j>&Y#f=qin6YShq3b-AAi5d8^cg0O;kZ&0`J z5xc9zv7IaQz+z4=a@yLx&9CzjRM68CY6~B7(!D=tW-(HJ1CDmMnFmHSbg^y*G~84H z=@PZXzL7ahQE_I7j&3)P#`a6%a{=?bt%R?$%o00lfV9$GNS)_utJ&HrtRKfR8gF-Z zb9-m08HK%E!iAp^4PAD0k&INg@(a$+KB;aj40Hyh%SLSMSXWxV8Py{tVu)qBb`LAH zLfL_fa%*8crM1!?+s#90T)AVZ-{G-Y;)robq|kID4q?yYbMK?am0sHCVivpBCwoiS z!*CdS(7S_7_t_@*?K#QFKJ+Z-qc0FW;WrIr@bMvRciMc91+@+IEL?^*r-o_LyZ|me z1QkiQf~_r4-^WD~QscyBPD%anlOr>Dt1>9?%=6p7$2d9E=0)#JxljJQ!XL@-CjCRp z?%AXC$=>>UQWk}`lPhpeBL1B~xrciPyQi7=!d}{^d3J%Z#PeNtaO)>^c@ftFn&gOW zbryxg$(_L(rW?O98=r->nk+ccNgZ*KN}sw>T)9%Vb1D=aeCvKVW2%oBjf&Jz`Aojowqx?JuqVp8qfx78q#a6?{4H*M53Xrim;(};%m2Wb+0wpA-%Rl zq>g($jz^*-ZC#ISD<8=o)#~17BU-QbyrZpPT1@80C_DxjMe+iy8{Gp zb=;E|W3XUo+|}Lv+vcxFyD)X#hEKqJhY46rkMn9ZeZ!7)wHTGlXv0f-;k=KV!(F1kkGt?5 zCl6{7UgOPfG(5@&vu2GK^LFshAUnqI zV9%?VvJ{nBXKEXM%Utfy>HYMHPL+^sW5P5Qi)%{`wm`-)RhZJp)O?>@;iU4ZDgW8? zl{$ZUuIilES(dj@&`GrfLp@Ja1;-X_P0KZNLwViy60dhhTQ;5J9~hy3?^0$4k$Z#_ zwFb9)H1Rey7l`yPnu$~okNPBs#nU}Hm1ro~TzTVLh9xc~>i;8bV1zK_en0|e3jfY5 zhU#zo&goyg&V!}mgg`xt7s!qh5(t&=q3mYw@^_`=CfPd9yIA*$!0p_M&r8?lhnZMe za&W6Sp%IhVsY%l?LS8GOrqhzM{FiINZrYh1G)JWryDL*rFTB_rrwwdpEUZttq&xpZ zTl>vU>b;5THfM3&!7Dl)(JZ1(<~eEsi2}6slov^-{QNLa><>#1YCYhN_W~w4soh*5 z@-cpzzrQ{y?a016*a4YTb7Fg*{urz(x4P4uWJ_e@RJ%JsImiYgHn;t|Gc#%!sJZT_ zCf@q{72<4>Xi~~@vZ|^9bVb`|w|k!SwruQFQ*Sh1*s;pg>Xd0n~+aUIJaxpP)&OEpghNm9CkD<&e_Ln-b6 zUuRHpxVzKb&KT9Szp2#G>A9-IDP&U{3J<=?>DDHXdCX>%X?q~tHjFQ#S!AsS7K33v z-WbqrFLvK@UT`WeL=)0d2Kb&jV%-kMEpQdQQitxF@7Snj$wU;(J^mSYJ)T6M2tcr{ucXI>qBI|t{(2KBOYqm z!a~mmfAE#G=eHPwUf6;!ovGt9;ur^E@C9Mv3YYL#PFNtgg#t{rzxGIoSDJ=cAtciY z9H}4BCe+wqk)lWV?*nriVEi*3t+bD|i~8SGb_)#t zJta(?i#NFWn-0z**u=21H@@6^$3m1eP zXq9f^qJR@Y1oALgBe@qK4Ht2~%D6SRT9j*ZCxoaYJ`ExrHDRN$a#~yI{jmNp*Ahq< zHodBczV93Hsct}K9t$!Wrg|3G#(eZ$(IR@%IgG4HNzK}E?Y8xqQyWuOD`Vm~{x}sU zUp|!&1T6OXHaGff?+l1~YD5?}`!q$f;DZ2O8Ymn%1e=5>SO>(FZEPqQyzImW@{>Y? z7}v(JqcJ%-TOkA_>=oZoTxBKQ_HqRgBLLhei@i$a=X)IFOq!IMvShsK{DZ4Dj@#Z+L#v zr~3q_hlA)HHrg*Sw;%am67XqG6j;JueiOR@wh(sk%MJoJC4q%qeS@=r%0MKbu>OnI z`o@w43&f>jy?z~vVCyx*>k(1ikRmtoGGL9c%Dv>=dVSh+d$bJgecUmIWlcxuIADh& z0K6Td?(V`iP}FTqoJF<^bhn^TLRN~6KQldzZwzfkMu&5A*IOQHP}M(t4#@PB(Zk+_#u{-P`&}wZwCMHVB z)x&c2d+kc92otz^AEL?!GlzAs>(AB#-qK*R+}2{3s*hTw-YXNuQmX0|6p%Zj^1V!dlRBu!WE;_o}ixH=vCO%DrU|c*8=Di z%Kh4lU2WN1)>8>zHXZt&&BS#7A{@BJo!=V7nA&!@!wITw zo@!UyVb5e7Vn{Hpod93`*=Emr{X|2RBTY*y_sz#2%ws-w^AW&3jR0*b-5(=7g|Af_ zgqkk@7;FNyYQmrSxmAUZKe^n8n9Ox6G~L3Z^kj*+Z2s;O2_83SiI2^U{LyM}7Bv|G&ldGBlL^$S3t&Dqt%Q3*4s()pDN`!$} zvJ?~4Q-u}E0D8x9j|9mEeJ-{cQwx@JH7s}&9$syEJ~}$}RmW;aU2j$tuX36Y@4URc z7%S^Vqu)iB!^Ff=dZ&|1RTvp-&`W*oR5(Nlg|7OQ?{2}71pDtQ!vv`gdiaB4i_y^M zajh?-$t6xBLP_{Yo0h~eZMP7$uk-=smoh#0%tq)+(?Kt)JWQ{Vi+Wv$llJig1#-25}Q*m*y|=J_0E zZiAQs1wC@vM^|mNOb0^=u_@DheHiAuo)x*S(2*hFd9a>uY8GCtlRvmpX zPi)-x(Gpk_Aw7Mw&O^ez7u(y{@4wx?W;0&;#n--uR``3d)sR%4VcHApktdD$(T;vY zB^4~!azFR3*E`$V4i+SVFho=o%0ee7(bw7d6 zkEX5~AACwMKQKc=5JbD)l#zL%-ux7LX?~Ym(}Gfk`t|KrFPX{5-lqr)1;2?l zp`*QhBY5(AsOtuLh6`XsB3_uMBFkr$gG6U_KEMK|HCu3nLZ0h^}ng{PE*= za1js<75*m*LTWw~Xc0h^Tvj=e79H&eonX_D+w50T;)wYOtZGO61pdJ30eB#IGY$gp zA?mLEJQxp%T;@wkcJ=0g68>j!MFx(a#he!}fT@l}LPC!NEkXp$+&^;-3^f5d`M1^8 zPeR-fcXI$4+oEF#wYIe>lXlRYR4WN9-1uu%F%GX zcb6+>(#_LzLK9CHzR#inqbPj z#KUtv_Dw%Cm~uucvNCG*h*6auWf%LhY+I`wqd%Jq9VtB22T_`%du>2&og*rC+YWT=9Yo12Iu>C3#OP5e#a-~1oSjm=H{F> zqa5uYl?DR>x&SP{pZNJa9$wVttKV~TN^ff1E z$i(?=^rMM$87Z7-mbNVM;_FkeL4A;40kiJ`m$VaTUnt-$5OyU4Y!379{~x&8g@vF7h!pkYS^c%n5dgP z@s~NiR(GEBe4RA>9yD{?Pm?>1i;IgBMC?uK246%mbhhF!vNk-OLHG?E#+4T~W3`vx zgdz;=&hjHwJu4p{(JPr)-O|3e^~cx8EW}{R!Ue|c(@WM(*e~}FIP;h8WyWn5aRq6Y zl;d?Upg7);-A@|0udfal|BT>3wSA`6SbLGHR=)4qdDlzDwMQ)Y zuGLVuoDF;g;6(#n$3^4Qe7(f*oJZoub3b_U$!x*h7D#wB#QdyQh6&QGjH$1)k^ryw zYvApx6#bV00ba=>(KvClV6ic!fzY3hFVw+-=KG#bIKPUI5`q#7U+{0>q@CX_oP1rLuk{G_KTcHvZVB@=h@jw zZAP;j;d!bq8zbiC zIimc+J8Rkm+jH-G`k;U+hnv&7&5OIOx2~SQd+(B7HR~<;m+pt^>deQxN#qQ^wUp&9 z7!t(Chq>f;?=53{xZ4MZKb~6cyu~KK%L2Uf4I0wx0<91U067J(l1AL7T|CVm2cv$~ z32*tHRp2Q^y*(4Q0e8pljds)P81C*SdUaO=d!L?8fHU%)?^)r-cHcuFng8<4Ch5y# zBcMf`Gh5#31~@0q{K0(JVZ~t-{k2;@K4m0*+lxNXec=zNXYMusU|h{2uBerjYB4`Q z9#syjEb}jpX|Cdtn8;WY!@kwX*U%C&MJ>RF1>`=QswUm31axAt+`oJmGg+XDf!h0U ze?VL*CZsu*Q`c@qYwYaXkano3BQrp((lZYHfPjI4X7E*v={WZ6jGA#vaD<|w8#NHo z1LCVA`gJ;?(q{*JSFaIn>}pIl;?%%mYsm5D;fJjFsjRez(J9uc;J*{3|4h3n=L6eCtbQ1ZFWz9*uLx&VzSH*(BLwo-%Cl@AUK|VlzZJDxY+rAG(#$T}%7zKQDYg zXtY(VU^jHkV7@$SY?r>_d1AzTDqh%bb0#LlEqIraR&loCu1)n#iim24rWf*GI3Ybz z>%^~2WK||J*2gf~ly=^HE$h2Nz0PwE9~ zLawX7`oDfr3&?AcvCH#5x`h%c+OrbpjC;?Fa)(QCc5$TAnuG{?2=z%N0{r z@~k99e7S4BEz-KCHaJU#3Ie|;UT!I)0QJW&F;`#)nL&R)B%nQdZawl7W~nguEuVIO zVj*7hUp4EgV5J$vK>7pY_0=hiIuxJDrU~Z zoiDGQ=uUejI+%oV?#Y&??h(y>Q=3*SRO${_U%8uy&RPZ;IX*n&d+7THCis;feYDjU z=FA~!Y@`>aGO{#C(ksss9`{_os1+9k?qE0rYquZz)=X=aExq0lg&!5&9)x$2U(8iL zxH*?7a31#k4nArvH;?c9mKs!G&ApWqdzrV8rv4=E$%FdO52L1mK_*d-xe4@k zU`D0}QwG$o&CG+)P+JfT5yh2He!eOfv2p!So~&t|4gg39xi7^4cSEf=V&16pJi5th zVz>B^RLHtI@--Pf@`SD3(XcA{lI!Wp3l59mwXwRXMb02l5wT}o2qH4WTjuS?M7Xeb zm#Mh)hilY*Om!=rtv3dIE}-6Lcq2b5Oa|$Bd6WS1g5UOjk?+cg^wA0(a0FoY7_jgF zFqHP<%h=U($jc%$r%tYY9?y0@W@%0r%OU{A2d_=PQKIo|!e3va6g@C`3uI zj;h$*5Ef3{s24~CVT<~OW-EQnXWxpUl-GN5blR1 zGB_3>U@sBs#CEa&Ge>_3#|hdtF;$*?Eyvl-1bf4YthwV30dLhoVWA#@;Y+O7lu~Lv z(V&?II~`80M*e4y&C)6W(3oOG*V3^Sz)k%E48`D%%L@eL6H-DUWo#TH8f+_|#)HW} zV1~9LE(ec1&-Uag99nT)<1ma zjrJd{fxREWu8udFi77+e5T+O@pto}@@fJuSF`)!k1Cpi>p|6UHil}bg)JnZtH`2jE z!*+14^gyYuh1V<(YkUXN9Yg?QJ_sHWyi=VXqQa0EXE4(Mn0pRH+;k+IRcRoCgYY2q zWTQXj(UvZ>C-a%M5r}pDPi|O+4*&|t$3dqt+CcqJb{<0JL&5Ql>l9mVQ0WVS7!9D6 z#z2Ap_+KU;ICPk8?hNMRF>y~roDrb9EK*Vi95oOZ56~?s&9F-ZH>?oP=pmWcUynz@ z+FIZL8WINxd70=J7G?k+KiA`FhrG;l{^@9L^3{ck&gKe-D0YoP6)p6suY%paxiC6}Ia1y!S7;E2_hiD;mil!4EONSomluWl-m$aa zm$u8oV{7paG^&SniF#zPcGU<$GeumFi(p&!=JwTMVr&UHhP+VE`O= z2vtL_vRzC)-%534-1)XctJ;Yd^{#q2xLaB}s9TD*?^Chu&}-~SG)ROS2J!{?P2`GN zH)cD6ru$U75icohuhCY(N8_c}TWYFUjfxO@AAQPO^4YjW zf%ts^jzvA!G?Wb8L^p%5P&1?WY~K`;au#}SBA?5|6t}8;wo^EVRcUWejAwQ)Zlw_> zWE85n_>{6)XBz1*021Rz6!qX0Y`FZ$c5h;0LPK}PcqVcltZm<~VZu1wJT>!J)b?z* zq7iNZ0Um=J6^&oORtF*WB-;y9rVn@}PwM=J7A2%sRF zQ(7l^?#H(#Mo%kYoADbJ_Zs=;R=?`n5|--#^2!)UagA*Acq<5$eLwp;K;)qScYX|Q z1Rfu|U|h!)s2gBraQ_h0S~Q=!92Rf57PW~M#=qk=0S`JzE|3D<2IN0)N6O>DLl>*BKh>-3$@@96@Z->34`&F7*r~v0mGgw z@hVL2Z&ZaGpHO*ibyj-B98gjY7=fg$vPyJ_&Ii~6spE~(p7rHi82krD2I|keIehk~ z$Wna{QSh{)Ylo8i;i!LwZckU>xl5Fa1G+1i8~XC6Pd1u$!f7imQSm%Qp*(l9Nhv6T z`?5L!`Us!!NeXH7=iXC?FY%3iZ_4*fU_RjMSH7k`M~;sw>sA7Rf|fk!c*ETbCkjzO z4Rf`=6(r|3yUX#PFl7iUNb`;hP>~i~$Fa1$^w`X~-4F8sbsv6lT!~#RK2(=Ut$_LW z&Aoc_rpMEbTLfg}b^I=QQT?U1-s5HWqVn>pJzX0c>E^nH8#+7x_>5QDH;mU^PZl>f z7;qB99w`?!Ui_&Oyz*7u>(7Ua65a1yZOIwjZbpgQE=JHjYx$_8tf(Svo8jQt3hPOR zrRMTcXksGs%*?O+b<;L#K9hjraS<)bN_*Mxm6dX(PhKkt?&B>j)SC^M*jfgm!s(^4 z&|m6sk@78CQEAqGL6zcgM?hOCS67(-bxfxKCyQ)Uia0qb^;hlwn!avn@hOc@X0ZMY z7OX;`)SR)DRKs-ZP3=qmy!CnFKFNIPxGCaEd-ae?L>3~qU+aA&7IsQ2{ORT9G}^Jz z9+YWVwsjP+UFuz?)Rbk;d8x$(xkXIW5?YbOQhXC=!9@4hUOsVOoXgD7;Y>^k82zIp zeu7I?fMIPdZX~N>g5iazW4Y=2_RdN)4Nae==;` zJ^LK3IOgiao90A_z4;#Qd)c+h)9E}r?X>xF_jqF6QRkwHrFb#Zd3VmaBc*76Oj3c& zgROf@kE>LKmsWk-CU>hPh%Y`-yZj+5N?j5_?Zs6z$BAiL8(+hMnK$1PZAQg)5h2<$ zC&T2^ovS`)3pYgK6z(WfxOlrzI*u1hO;|?#UR8&|I`2cZms|=qR>QU5yQhaP0CI=> zU*maSK8%a{y)&eZpzo~LN7{96*WuMjNwRF$99$*sYud@ok?}mz$s_k&t+>4}anO3Y zAkFf#P0Lw}JSgqV<08e|>ODr!aH7U=VeICy?x8u6{!FptArE@xt{uksiHC895hoJ~ zQS+G&{7+7u?K03eT0;mq!C=xT>Amr_J!8tgy|}l<{K$COb5n6V4G+bkIcPL!ZeBEb z5-s!*8|1TOQsPyLQRt>IH=X&>+;CC7a zJh|&apCLnq*TQmn_k_`lkIuH1)UxcX+CGD>#N1ZF!GTnxptLK@Bx-L$1%;|Tc)|tQ zImVRpM!fYBsQK<(e#-%usoy@Wp{zerMkvs2FSYN_4~TC4H|Y$<9!SMF z6sD0iHYNnLooQS8w9?EbYe0)q`gp^shR)XKDBrzSyQCUEm94R(-Z&$-g-dG z^{UmlwuwQG!03|@kj*hN$P8uyt%|h^zn0o2M(TI!3Gp49g(%UH#7|{1)rU) zlL2*s)8I9FdU})v&hGHrS*y4ROVD(*&VHG{c+Jh~c%SchyAM-ix$|h9xSb_xrD?n_ zaevak&sF)O!QIS@AbyYrz1EFt*NT(8LF99yS%%zZqV+jA_~sif)c2PSARZpr5N>H`TtRKsx}zgthZ^>2)5#JrfiNYI`C*_S zOGfI*>PD)!P@Bc@dYv%cdY$Wqv;83VRN%^@h@~YW;QszOEM21gqbk%+ z64K2&KC9-y9BRpG=V66ecZMkGFT@tk`ZkO0jO#A}aZBzmJA#0c$b&7{%P=gV?B>F0 ze9t^Vp~ySZj)t@}PN;*Egm$7j*dc97x)6y8Y@tQ)I)TIEv{IvradA;aYqe17@aaYP zM=hx#>Z4S#)Mq1Rw0vs)Mg|N0%;W5_V5nNa+Us~*!(xiJ31lZspCHMzww#0;EISk z*M^mSKtt%8JaoGLO-W0U_GN&o<#*V^Ys?be=%$zWkoECS{&RvZ_?ZHU86&5fgJhZc z4X5==pOPZ=0@KsgNl(BO1a+nKU;%vGPc2eafm8$_QBo!5QK}#N77>;LaauEbrfyuB zkz#{z52)NhY*UE6$;anQ>jt=ftj4M-z~1Brc7B-g7r`li59AX7+%U?)21En}5I=t0 z&lQ8eg3J+Y7{ic+kjm=vF3hli8y{dH0>5N4Sj6)JLEnG`p(DxQa((|}6kIuh;h3Mt zfygIhgLy96D+4}xNccrs=V7zB{n122M<*2WRi_Rx!5j{O9~V;YX1+g?33$auE9E5u zR|u7&%pSW66M07*cZMZ=@t=+G-L{wdJKk>|0fF#0m1;Q9b?Fx#WK#&?T z2VTJK>Fc)Lt1tyw=6{Xyp@_Tb6r%qZc04?~(ja;K;Z<(h_Ue;ijQ5)z$NBym7L9RW=u zARW$CuZaRNO8%Z`&TF!c$GYR=?eh}Oi7^wEh zS(EliTF?*P%jmNuN)=|W8imh2+*Q{mq=FM>r(bJj)cuj-#(pc+JO&Cb=-+sgq&|JH z>6cqoMW%?`o$I%{2!u?Qz*OBHSrKr; zHPh=qTQQ+f7;Hs9#?wYWi9|NTZ2>j4X9^)^cfW!mnVBPdGhhS8$blV9QEeF1c6l!q{DUxWxcqq?0mA2DnX$*_0>v2!S` zy2Iz2BpVVNpg3)dQ5haD;MVy?u3N6xzFwa+k~~&> zAclhY5RT)$`ttaog{RWBC%ZL) zKlQ^VFFiUCh2GU%>06t*g$(D<6jGT&}0XBbyQ=q*D;)G}qMM?{PFg#nk>uoxHbjLJT!$How)~Ez3p@1Dk;~F&B!B6c%lr?bavqrvjn2h`KIi+j3A__h(lrs^Q+`1DAZ!z=CjRC`!Vchj zfFG85kJ$KV7iLW5NNOo5Q$BZx=j!^<;V|0~HMXmJiGX1xv|@q?qvP|WNY|&+FkYI? z!{yA%ouzL(%XbOrI6h~$e+Hy#DKh`~L_3_u%*swGtQ9)!xT;;uX!8aiN_1bo{xjf` z06|f|Q&c=vlmcUZYbeEqr)6`_b3II)?=zH0ChRMzDO#IaU&=a{xyw-TD3VY=eY$a! z`-%EPX1`Llp>)hxu_GN$B--ePkOXh;r^l!dzTuc6qMPtIA~Q3Kw_A#J_RhJu(BFX< z&zrQj109g2J4Zk%#o=#w47=gn9Z$SK#>oA4tAsl?o%?;a>dYvHRA# zg!^?y09&L9Xy+^*tZSGbc)EvEPes5ErSN14%2jU{^PWNtv>kQLMrQ57B)YNrZ z-z!;QRPC;sONm`mPB_^!-am-QfcZ9&N^sxho6UW%0$wp1A|2QSeuy*xIoR{FZ#U(5hyE!NPKV<4x#6bRxLXo*nw0xDXBkAey z_~diSc&37~+f#pGDhdG_q!2`O&b8eoPtRJWsv0||-NA05M()0^HOYH+XN!l&Z)sB{uV2b6dMo-UIKlv)$Kc9q6L6(jwb%7{iN1Yr&qpG9*BYiJcc@(0xkU z_n@Z0-O-@Z=g~(yWgkYi`KfYmh7}r;0-+XFM zpI%m|B(+-7kk8jgxaN!P{>EjiaS2SaJI$*eYeSCY z^`{>C3yeUNdhY)4)8ol;rE_&@!I+G&}exeky>oqe2V6| zWVubs(Byi(*^dNOxJJ;IS`gIx72eT^>Em~;#_Cf>UoVZywsWX7Zy_2WZcR(pCL8^J z*Lf0|mKM-2J%gT&^eQFvs0iHsjQp4u`@|$>VZBOOsuIKLK$;n2p z^s`1B7ODYdg>)c~zY-JPd>!g^834YsA^~%1->UZVrq6eHW@XPXk zm}rB(4#&A4jZiS~Yx>#CmqR{A8wOMq6jxYrYhnxv+mU$}iuHC0Lq`PyiReS2N)q=3 z!1SqR-Su0~e!3u`m(ZdE4NN0+4A7|XtnVZADE*mYf;Jh%+l}OgXlo~`7Jy66xEWgl zDo$fbrlbU*$O*}AhOlE9ZI~pdr>7CyKG?~=jv?8^Fd{aCyWJi_NvR3qh^;3o?J%Fcr}sWP`>Tma_ZFnOF(UA?@aX?fy94gCdl1Y9u1H~zJt5&6nSG0|z`5oGp)8Xk z1-*+9$i{Nz%6U#sP814KMqmK_Hvt;_gRkzc7sQ|7F|Dfb#G69J6pU$ws&Vo0K@bxN z+QdFTq{gRM0isxu$g1>uXuE_S%Y|P#&mvBpoFHo1?Rc{tG;?BVYQF?Wp{L?yf3BwK zRMShq?wySSV2-_Y^Cn7;Il5KU9g<)qE0nH48Ykz)IKs*Wr%p>;=t%jOV)=`F%R*fZ zB$NV0+vHe;QhoFYGD}Q*p`_T)EGZ7@SU(4Uu5?*_tI7rmo-o$GxqBXtsDFZ?EHr~i z9J~GRB*veCGI+og#sdw1>({h8w>JDKO22}ok}fsBfC)yiT{JPAPU4-!Pl*# zd&jT>9$|_BbtbJ?FtXAnm)F#g!}iI_;PI3l1ztZ(Fn959?Al0RpkZTm(0VcTuZt6w zsGR}43l?r)#;0Motv@vr?>*!mt9Cb?`3f2GvnZ5u8c&h5mN;0u+>3l!D_!lQr+v?o zu%ff4j#)9W_t7Ej9;N`Apc*PB*pShbFcK35P6$IUie2>Z`~GucuEtQQ*DZ`xSe6XEj#_6g!m;nQI=4X{Wz%dV76#A~V$P!mFnV z<8BD*ZFx$fnBcj-kIn_IQ1Aj+$r>e|hqu>7h{_Li{0T2ACgKNqE{fp{mij1;-#^Y0oSV& z{EDOk+qB-!m>7QEb0`j8*XDb?lE&)n;+~9=^fg#0YO&I520re<+WuRomVvCx=vZ3F$W5*63@QSrnj<#0XK3>TjGmLia2U@LXZEf)7y=!} z*nUFSM8bXNk(fI%1gab;r~=2uKdyAtE5%4BbPA^!Lo} z*>nEib+NmOx{Nc-`#y2szq`#BOJKO7&)GUwQ_v-bk*i0zkC?tXOhHVph>HEMcMwGZ zJQPeIMSkkp>xeVo@ft`Lr?K)nzZp`xFz4fA-elu(8H7eW^!;L+Fv=FwU_DK{PJM6Er~%69=pzh*CVGVY--9AN3{J- zG{m($Wa1MW`Ntu&ZJMmaFdY8n?jRE!W@qi9zet##c>ml-MO&D2rm!`(bOBiDKK$o} z49G{Ur24I^{#ygD(NpKcj>)HJLE~!C;jh+m>Uo>$3#=_=KFm{VHcoBJ$QbOJ!GO`b z{w=OF=k5gamw2d3)3+RGga4#6u}mah|L``2t$@7j4x+wLn0>9>FXRff@L8wSp8#G@ z;T{rflNwZ|t@sU3XZyeS+|lV3FY_vIdhYKi&w!1HTMdOmT>}#@8OY&!0*e=088&@VUz6i*n2XVDvqnHfq+xpJ#5=45@@+UZ{Kbb@|3J)Mbw#u&v@x!+5_ z^OYwLOwKrx6K88X-EH+F6=xQI;RllGdJ6zw>(t=Dg?s+s@#ABCXi)!bJ$9%n_ShdU zamyQx1m?gW)cH!;B>-TAwwmVmQ6_tv*BVEI$`CN}A@p80$Q$51p#&)g7U5tVKNs!v zwDpf)-!AIa8t*k<_4>9lFc{Ce-oUCx0>Kqus6(gyiQmAnT#!;yLczl${Y}&5QnwhY zFeazWaaH3wf`#m_lSMH@nqS&*p$EV)k~>rGm=K|N-%hRK{^gF?vD&Vok9!_JKgweS zByC}F1@G2IqIxn9_A#1pBTa5Fp8d;VR&51M*C*#=Z1WZ*+CFFifTu;QV}iJxXA~2&?xDk~tuV3pk13 zW?=uwAm$Cv>Ga3z{v<;0q?q9AegUO1X2|%cNd?@`USI;#s`b2viRZNazSne^J_|@N zfMWt7M%xzLR}dq25!mk(P9f4aFsLD5@B&i52P0;dBCKJ2D03BZ7isxz5s(~OD&UC; z@`NZvy6*oG)y8j=;a>tZ0|~j2cVrtlSN?CLED-_DYOL$AbGd z7uQyj=gS{sV>Fmj5@ZBG0v)<;2%)bSVmzduJa0M=h@gm+xd~My>`b+9!~hBEV2(GM zL}D%Vg$D0<{(w0h84U6MS6W4cbwMr|ynsdJV8djEh>+ACQ`X9rl@(^W<3Wc~H@BE~ zMj(ZNeJC*uO8h9D0sEOqtfq=w1Ii5 zc{zXXHZFrF2_~%X+!x9^ekrK{TtO%Mw$6kF=cMi=HdF4ybv6YNqeQCSnA^!~H*R9F z-|3m2YxfMf)@NIF3NbgglPx`uyuG_H+9Gf&T|%sl#a^zNd=>qwyc90g;vPX4=4kJ= z!q$UamoSGt^E_(M-}JrCHBfAL{2%#XA^9(jlsd=|6Y2u@UlFC2?3Yy%A)xzoPE zpAOjm<%j@nDqeg0BMBeFcjh)g+zzHZXa;cvE&}oijD}n>$d-~Yr90qYtOcbwLh<|@ z$mrbPSvhv=wxqvlb8)AO48g*Et&l!+GNIg|ir3qd>cRzT`LdP8#gHjI)wzZEtBtxf zqeV^|jh6^PKu&oC{U(Og%ND;7M#smB`Dtd49lWiIyIb{^dutHBps)_3?`$YBBeWR< z_lEN+@$sJW^v~!@qmrEC#00($eeR-&r!ApjL!VEls@=3{

S?(71)4NC&fWRiANr zmOIfbA-m6^dHnTx#C66X`sXze?n0au$SB&v7gLLrP)kLe)>#HaH`5s&vNN&=ew zvu=}v-KLSBsFc&3E?^6A5)dcx*(|<)FNH@~+Ldd6>ONtQUiry*_ouT?=2%5J0tCGg zA7AM(7ZpFs&bu_X{iZGWt={zdZF0 zvgel_`DdMXu`FY{_1cc8&Oi(H4SI`{;S??)JO}LWOD;TeIL2Msy{{z}7nCpQzd{!T zlO0``makd`rDx}uXe;@9SC1YGkN(-G;@t2*lG}J)qnn6}NWV3Q^jO<}DiRaA(yNgw zX8iQF%qzLEm(C2=Ub7cs@t=6z4=bo582sVKW1sz7t@Z}X2TrOp83kF_Oq-~>v>QL_ zG+y_Lg~cNDHmg~FXlvYltn8%D=IrH-gQ(}M_qQPoR2w~VyE~VEe>aY7(p2P}Jqa&# zU-IRbioDSNG=WcBM|!#Lr7BKV1LRt4kJjf65Rh|4!(8*5;d#^Rz;mbP#+Sl?q`gy@ zb!B$#xZCZbHVH1$#)Ns9@hCs#x^xL$LmkSW z+g}S80d<`Kr21z?+DR z8JM_x%^tT1v8j<vDDk>w^X?E}%rUgH@H&3I^MPY{s5W!us(_L1uq^sxJ9ML@i2`Je+ zDgde5fTHIK0;a~QTQCRDcd3A6SxO9vyLi-rt~h6O=j68FC?&La)ZefXEiD?f0v~Sd ziUCfesw*Mhy!}IKxDyW1ZjekfUWYZ{%ia*?`Up)KN(|svNLc^n+1Lw z#>AlENth)H>QXR{b_zW0P8#4=sya;+j1K?sSK{yU-M$`YbS&l#p6umJgp~JiS+9QA zleU4@YJ-ENWL15!!5B)Gow(p~tT~GH6f3$NYcVuM6&ZY)A z12y1~BzRS9@ykoQe=nv8V`>P@=B6fO*RJgUwuOD+!qi-U5pL)DOMrO-{8dIl0Vz-o z8$O^>FAZ&+m6uCvpuX!jMSS?cv?FTC5gzX(QtKfe3uxu$wptU| z@lalXZC}i4=>8)-E(Zu0fOy^jkOl(nuFZhv3IaUxu*k;-^z~c2+13eUTZM}d;O8JA zG0kOT5(9;W+t6(9p(~_^IIoYt14*Re$)ByaAtC9~?J~g-#)VmW!uIrK6IjtXEe2UZ zfchy`^xr22z=mxQOF#JRqs%SJgl zIS?utSY1dlye7;6?=lgQw6%5q zZ=YimFuS8z$YMh05@ea7h;kS6`(Mcc3QBEQ4g3vT?0AJq-}yH@;l+zrJ@W5Ges**~ zfftsmy873TpRd4>gYl*}{GFkQ5A&aX`iqQn`HIh<`$3ALIwr^ql4+0)=udg+Qy>k^ zqoaoc$6i6mVSxa)#Cj^CATDOT0$z;*v7%O&6Ts2i+D z-AO8%x3FrJ;kL!-?mUiYc+)R{{e%cOU|_PW&$mke8d<8YSe^80&WoqPXT!s-hFb?n z+N|p8GP4Gr#y(~11K~Jf0a^yni0iw_BDoYFSSIafq>?rd{cr_G=z^pSl7hfrfzsrM zVZ=Yh1z3te>Ley$s96D$Rg4N3MuV3t9x;)ANjug%-nZBbQLk(8M;|j>yfjdn(_x$i%^aF9@%hmnFH$;Rb7Nc$9Lu?96qoR!GiQyM({L_@~Xx<`q_Q==mH0pNF6Bv`IuNL{$$%A}1(Q7r0K4(CD zdXP20bAvJU;U8L9uVu3w=RcEAFg~o4m0_ue0s+}H2AO@@V|8PTr7!MEplU-ma;;Gt zo5G6EQ-i|8i~P8Ljjs4`M7;yyIc3@NvOxhrPp@lf7tK70Y6QX*;itv(zsmyHi~hRC z(;7}}Vf%T(^4>~9?CH_Kgitk@*DxyW6%VdNCI;Qpt}2pwdcWHy$s{87w!~r=A&QTPsBELQDEFR^CWHT|M*T>w zxg@xMz}ur0ocVc8Gh!l^zTVl!J=8@HPPMr?dKaV{EEO}08N zA}wN};(Y7~!D{)EXt4#n<@p<)?A7W-{#|%+jrNH)9m8lRXU+IW=k@AxC>xX8lm-WN z-HfZ@d$ppU-?bH%#zvMeGH2EbBYL9N#1>ZY?sJ&39(jaWxF!#h35kjlq8W~-{feuY zSSzN5{cKLcyrbIG#9UoZm$C)|^Y! zb(hldu){Z1hU?qYo57p)NgUl+oEuFpuv_D>iBqu=4GUxleG{C8-j{_Y@@w(pd9e%1 z&g%04L%kUFn#ipfs#YBAvWC{H8#N{dLUz?3_;an|PvV*761dG?CNNvYa|_7}|JCif zA@aRp49C*4UhknSmciwlzbjD;{kH=y%BWp$Jven=4oq51^4Ohs%&PKOUw7l2MbcC8 ziqR%``{0uwmLd>legs*yPa0nHJ&xNwFH7&gkK<-gXiad<=e+A$`0~-75@)BV`%Lro zs09ayVo9yf1(SBgImW*@(OR6s)7?I`dJzStWs`gdz0d6aEQ%GRZu!);CwsfoUA~N!mjvswsK9Qq zIy3vziFX-e%*k%XB;X_Yi<=ra)qe@O7#pB9W`}-xpFHPF%1KU+QeWK2Tw2*l4l9{P z)PJ~zEgRoW3nSpo9gA~hXHO3fJ^g;fQ%$k(lFWRymoe!ul%9QKo%q1%_(J?(cL+g~ zM+Ix@VdeGcyT6CC-z3u2DU0Ub&{*Fv?!0fMOF_K?Z)lG6tY@KveKbzrzldXS39%Nd7mY^a)(EGNQEfKf23G)Ye4l5!Wa08EM#~2 z`Gwuj?|FMy;#_gamvg3cayl9cd+{}MCxVd17{c>RYs|vJcwJ}S`F-N&!kcZiG6^^2 z&|+!sN{qvNGS6H7;IQStMgVsz_zKX%pVt^YC!g7pu=<}7elsD>`66M)rXCX!&;y$?qCqM%O>mYGg z*K&1G*0{rCQW-9{OPhQc0H}7bso#c(Y#h<#c^-P>%*m+Uhy%C>OP=8GAuPXOOI3EoA_2Yw*@+t$S;Yt=U(!dfz|M-Re{Oih0?$h-oKHVOV1J-5`| z0fx=rFuibgcBTl&tc#ZOtjEj!fCEH52gx5aU|xCo_ANXFlM4Hci&ZWr(+hH!i;BJ~ zYEQha6ud>%RG6opNi;`IHh00rWft23Nk+8rwU$Ngh0y)rbq^G9~`Thf>f zp}xL8dR}INl=ZoEA5_l6-`={x^72R6nL|Ulzpms12X}#0fuY8+V%dEf?~n(d7M7OYf;?zuP5qj$Z1E}ue&Q&{ z#rhF-q^O#`Hr(?=&k}YfMus=1ruD;%iZVAQKcKXC%B7en;jK~jTtjzf?@5}zqLLB> zSCuUy=MHQS^79jae)hA~>>C(UeH>Rrpm~-bx2-wMoxIMs`+Vqp0Tmt`%*Mnd%#b6B z`0=y%l$q$-OrP`^azlq zk{oQP!dRKu-+pxnWsJhTRH3&Z+vHbRp~YUVl`no=S5>81HAG6ky%l+Z1sf3%;HrPL zyUg5qaCVGJtF5I#z=qefdO!Z`!FWZ*fa^xho~8Rp(U$9ah2MX9NYW?^ZfXMR7r_v9 z-PpCde9f=Wu!i90g-W|53CS7~T1yi})sF7M;y$t8syY0KQT{)CmkKO~I=Y;vW;+!7 zc?p?B4W3+l9rQ=z+j{mBCaDlYsfCtIi-S@ld(YBs66SB$0+!kvY$*lB2U6braTZ`d z`w+Hv?-l|fbeo2o_jsb%f!i}IFMY(jNqgy{(CxltCx`w1z2vk3&TKZ-dd+k(ACp$= z_tZ~B)^&SBFH=&X3G1*H(8z*~s_+^HX31PLjYVxQIxcG?Zk&+IzCp})vqc#mC%j|vQ1mzB*#GR@Q#5YAUy~R`=qWMIs$|m|cp;t6hWfmtWen?q zF1>N2dnoA}nspjkHq9i>G^7qT0u_UsA>s15e%k)~n!nw5o7r)LlNG90w$@=zechP5-`nd*c74FydZ5|e#TFMUX8|!aZHrQ_Q`mo}J4ssS$YPf_> zeBQb3Ft;$cl#%vDR*dgm*naDL$1P)@%nB=U0f7O8tcA&n{2KZYpu+8HwI?JA2Q$H0 zfHQkbsT=ah@*9VxWqG+z9lmDSc++AsN=nk7nw`BYqW$BVaGuwJMC#SEeIyr{k=PN4 zesebP6$dRQGrTUfFyyq=>)TkJs8WJ`Dc#AYBz1{K>)cUQaF=NU0iJEc+K4{CD<|&5 zKCac#Qh$P<1B1L^Aap`YSPZrLwiU0Hn`(QVUA3Osu;1CSM7#~B{6rg@BP&9Q9pBZK z-aFWRBQNJ(O!O^svZVrt@ZKcL_g}xlCM_}Wfk|_@L)L{`xAx#Mnp$$x_GMSBtVrWs zoG_97xqud0Uw0;XrzMx~_V)uG%2VtfIKP^n}aUjlW$<|~7f|fCYWJq7a3zTSB z0BHq=eTLBtG^`r&e^Do z{W&48yS%E(#VaMfEwcR;WG0B&+uIXlZXJ4J8e7RAq73=9f1ej;^YJR@O z*UFUYU;5t`PeaU z>%>#@faSmS1f~mbCKIM7zTk(r;CEJk)yc^TZekWTwm?ke3X}MwM}gufZFJ5*`BC@p z-yas%)YRO6aOnb8fCM9`OY|+qYH&X?2JFhiR^*YmxSx&=@b}&jTKJ?0<-G7AVc_7{ z!7&GjoWxNP|hz zzM5o(qy;qP5d$Be(D3MX$_rjX98HxBG0wRraUTYa;svU~Zvum<<+AFOTH0vA7cL$l zd6WcADYdo08~N*D-+j%ZbW%S*4VW_Mj}#f5ulgLw2Z{DgPF8wRm7J_gNO&*VYR85b-G5JX(E+?mF1+kOX_3*VTsuAwohf}7JCL9Up%<)wmWO$$9#O zi$`N7CTu?Se~KKP%Ubr)JKyyhbovK)2=nuLIQbo-*BH$hk7ht}Vat@ceHtJ5tJCUgqO#LLDCNG)*sNX%6|EZ1J9L5ub7~ea)rW-EY90 zY#rzmlQf}92HrB!!G4y#-iC&x&D~w@oZ5&8X@u%&+HxZ95N-=Ta8}k5iy)NF9UFYZv1T)P;)c9Ayd!>J^YQQN4Brk7?$T51}kHYMzS5<}& z4mvE|*BPYi8!{ZaS`hW^O@F+*J{GkO>cd6NLSCtxdzN-GO|vj*J#kq<8t%T!30<{H zAy|eNWW(}!lKj}IO1b}fmk{4Y$79Y`^dXqjves>t7s1InPJAc+w17^;*i6Bx^la-x zP|)(KzkpG~s;OqgokhQxmoIKM1zZ*yW#h9?XDG9bGmNOSRKE#^bOL+qZ(a6Yrdeat^xZh4DS%bgTt)jq6ONC3!8=Cjwvlg7$cbIPMZ9EyN+@qh{&O}EvVI{d4N2nKm`jXr2 zqa`;U#Ak#2UO3%2HdejSKo%oxjr_a4?66-%L!A*-5^dNAQ&u!Phe8N z2k+eb3W-~w&fl79z)tIf(K5yaj~CeZ_>2`|1ixuj`~}HW5{7B{(*h43T=h6uZ<%Qc zZkV@cICPLXXq&eEZaXdEc`)GulTo9c?nEJ6nD2sYZuQb*8Da`5s<+^Mp9Y&Szsovp zzxnAEdkD;i&a?&2ZA`}3Q75_q&d{y7D1{j~(_tcd`fsm{$5`_Y7y*CppUE$!m?8GSzWxu25FlXJ zN-Sv76j>tT;@*K#27yTKeypc=9m2=pGg>!%Sn*uVICux zY}B|gR4??q;NT|6svLH-v%?6v8|t-dE{C@@y}on>h=BofbFXU5U;-b}G5?9rhJK<> z9FHqFH}~H7_&BKZnij~($lyw9d;gqN;;oY{Kke|?m}Kg2r>FfVBL8;*P0~98F#YMZ zwGi*qqe3@mMM1~+&wZq?+nO|Q9Sm+4XfyHg@prdHL_`=#@P=a*SulA+dnhTqWzaQ@ z$UJ@8+ku(KLqL!05jyf|hPxRDN5RXd80>9&#qO>nbgyEjrp?G=Fusx4-eyRn00gJ?&iO zeOV{P^jKcrscMgf4Ba-MEWkfpqMLmUPUMe2GYhF!R(6MOJtK-S|9QQ{(|fT*xvWNK zsf;)ppNiX*1Zojn8k(ILLzh17x~lxJjt{TyPwY>xoh*4NIW81s=c&Z+YO1M^c&sNQ zcMq_a&Y1-T;|~&2>NvQ^6{5;YwTu^7eVPmd^L5XnZU=*|;OWU>HzmwHdAX@i{n;zD zLIK-{AsJ4(-XOK9;lVGc;T=!u9RDy@B+!Q28k3rkFkRp7DM|^)|5RFdK0rUMp02tzL#+0%Ln3HZg8WP^WwSRtk zKOyOO6mOX3n(WDQfy$BTioIgnIk6#IuMIELQL2w#ze`A1T^-2s(e1qjO%vZy=Qwfm zgI87VVb{65TtmaYn~!MRq_2#~lOT!I{3YFE(JCD2v67_G;hpa@QSWTIw`p2hmyLtS zljR%hDAxVHrY=`6{!!)Qw3Ub$+0v9u({foiw^-Pcu?Raoz%hTiiiN=b>%y-yqF+Xa z-480hSq1O=VRZS<^1Kl$OZ#5>TbZGOvl7oNd7vKuG=pdP6D#NMiUBjw zN#OwP1egmuEUX_(HP~-9lPBi*-ps>oV!ZcP%=aFfWGg zelxDvx$u~tK$(h<3+99_HV$ND`2mOvIaK@69&&3r>+0UmZ@NASUSe)Vc>4P5#w`Z# z^F=2u&#HQ378Xa?#DbPq%{so;>iAT-e%wDCgdhv@dqf~!l}+mt5kXHcnPLkGP1wDb zNR@_7L9w7LQSLLfuZfVS{iX2v#I%>GuHgu8|D=a@c171@0Sxhe#h22&dl!h)SFS}Q z4F7|v`k=f#>*JXAk1Y2dY>56@Zyr#olWQ0m!D$LgaHhX+rbJ>r`4NGD*=0`7JQW}N zZ&URn6(IWkQ)#kd$jBH8HcZWA?{bjg>)Y6zwQNi>!;}kVKbDpAaE(Pf_hg zOg;RcyDSD;ogi4>0m%^ClsY6eJ^i9V;U_O1?vk^JHAy)Cpy>f<7WQvPZ4DMO2~Z5d zRF5h0ZN3m`ZwCw~FTvFDw|Rn~7F1sRu;T*Qq`^EUptmGLx6a$v&dyIZbJY|~ftMh@ z@;fvU;s3N9wvLX?kO{w=3DGYPQx!o*d`qOZNmD{*2p`l!(6%J717HQjM|o7F(2<10 z`<@J6+vMW{(fYrvn}7ZiYShWyL5eK>s~vv&RQ&@3;!Kg4_6IYIWMhL@_0rhl`y60P zKUY_O2N!6e8_efmn78(fg{*OCD0cnv^#I}fpf-oAWo6#3LQDAU&n)x7n3{Ty80Y8M z*w~gC%nj$B3$+SNVB4lV0mowpE6dN(7$vrW86lxLAPPzQ2YL6pe@!DgVjml{y!WFL z)xy)&o$Y?C@ag-+Qi`9|bCwWdb5#5S_GdvMB&zk5{3#o*0#C+St#m4gE4)Z97syH%TK{k4aFyde8Z&n8h0oH0quEFMy`9q>7h$|EU#eGoK6CHbxmgqseS{$jOxvI7GmIod(GkL4PnBS5;NzJCggl$X|#io@}`w0gF?y!0~>{|fxljm z4-_toD@*Fxv^qamVU=u1#OI~sxkF{m%*w#0ds|8lUwWkn7vgxGdG0osZc(~^pgUO?JOM@E`qJ47+FB002G1} ztLDiOiv4_N#gmOc`L&g&XUnqlSHIM;52!eOvo;iK>O7!ZDC_?8b3nyM=wtgw02nv}j za&dNL;IA6D*_n&z!SgB+!5^agc%{JRq1mWUW@gNEE8%6$vk?>A1oEJq z%GE97ok|UFqMo6?nCPgeGv_T9^iH07-5Ia*+K*R#u;&4CPwFQ@9x75&`tQXLEQf?V zUWT+sxx~{xQ?)n~5ql~W8A>>`bgmF$F{!gM_X<08le8=RPYliDQlSTBx?Fq1T^~<> z;xcFrR&`O)p#xE;)-25O1QKeWXsQ^jtef790gQZ>x#c%qU|_{+2KEK5`uTY0np^y8 z=US;>Bzm4c@%6s;Z~0Af1!_Tm$gG6;%275NTH5hTq)tA-#f7-z#Sup@ubKv55-@AS zj-yb*F`Won4ag3zCDMbA6ce*PQ0icveHmOY zQ3|5(*fh%?X7uvNWMyW)ZkvT%n~ItlX7656D6*eKF;9a;(lQBhGHbn#H=%^?b$|Hd z2889-zbC(EyVylXE9Cr3uRd^Q#=!IZ^Z>IiVV3`lr#ZInW%$614sPNIMoALj{N7W| zhx~_NQ_KFkIE7Tjso_Gya@f9S%ErBh`>NN*k^!X5?=dO6U%oWz#lr}Rf?ew^+yVEb z{!fh4(OuExaHyYQy8;t3$eqW4Z^2jt`U?1f{=f=J{{}OhfBoUZU;Eyqkj)9Sul@Py zX>qIcfL*<$q$JEtWKu*Qb}yJPP!bXny_UZZa@hG8Y)&;mS!=KZ!z4}!rP_4Oa|s#H zgm88Y!58ASe$%jAhYqFgWcx~z>&)dS+%>R$|_!Nl-LoBy&VuS+1$_|&XK2B zG8uQ}|0}u;*r0y`05Mz_kZxxEPzz?^8`lgUu3DS`9*(SOcw$VFZoExn?V z_maa0WVHa^|M2$--g^x;0@CvGJPBv(Rm*C~{8MrJ+}utE{>pE%lisPuKRcY}MTAFv zCR;uqB@9`8I}4>T(VuNqXC4}&iH?q&6jf91!^=?OQ>zXNI=db{IeBz2>b*a5lU0-A z&=gAp?)zd81IcPtI$nnCiCz!UD~L5TdTZD)JQXg;m+yaXYk0e~w<}0oU-~nZ7t<@8 z|67r7yy~EYJdGsGTO? z*|g%Ka(hO9)g0_bFcBX#d*#43~)ev{HZSAxApBSAFI!0hLV(r!Lp0F>_0q#}nDPmacP# zpTEusF3AT`ee;_o zIU46k5EqjvR{P^EsWj>-}h1)$bz^-@Q+*%HX>l zn}QABCNKVRuXZ?JO|@f(Jk-r!B#3Nmb`H+H#7)?8!`8Ov*}|+s%__7%4T;)}R?Gu* zD$$F9d_Hh)?)LP;;QrA_qy?pR&f)$Z3tDP@ed4(KdzIPb8$oL=L>Rr-ulGH>1g-n9 zHSc2X9Gmohl+intTc9+&JJ-%cD^Nt`%oY};xw1ay;_>CnZnp^JZ;Rm;B8PN&2?_I( z@0Ha2Ps42x1LtfSoIJ$z!luPqP)!eA?C*qQr~v{)?#HdaxdJA*SH{Lg%kXx@yti4n z(T(JcoltjShVg*y)8B!aw`t-E6CoiXztajCtWfnGE1{8xF?!&SA6-5&*P#IIh~;R5 zayPYS34Yyehxw1EsBzclCy8;pq?nKu9i61fT0tQp8W;#*un`8Y0(L#&i1EiElz_wt zj0haWbIWod0R{1WBWBnEiSN4`Hj|;B(>(y*+yI9#DJ5kPFf+93yeCrp{eJ#@dUCW{ z08;j5SeuiQk-f?=eOC&d4m^}u_>6m-9`Pe-X@GvZ**c~tH2wFkf^mZ#Kkc8RBd9hh zAy5z(0XD~C|JyJb0a`Fp`PtQ+7x+0ZewIZE2?^c!-^#q8fTLW?I_s1pfTV*4q?}4h zN}!zZP{Rz}!IQuxJQlA`2O82272JU}0UOF}XTuZ!7h0bMSYC zoh~%HbGW&)rw5HVmqvAW_BlCDW@T&3hm}fjE3iMMA*Z_1MlL6npk-3 zmhceveMlwTG0)-L;^ON3BPl7pxgU|tLp{HL`yRQUHYC+l%QBcoU^e9z_6`n(hHXUt zwW%143s_ux9z1~Ie8*=|He=LdDOUiSHlVxG2JQp4hU>T(AOBT$zO`?a$FcuIO7iwy z9O{s_o2D*FW5|1>Y&*Xbg^rAFFbT^gAqz|^LVn#Dvv)f>7HS$8peQYUig>7>N+1GL z8GtI_+8ys(`I|vJGG>}5EaEX#Np>yg{G5#9=XDZ(^*hvw3wGp`L*ltc_4E57v;Ci_ zY{1@#jR3f>*W8hN$NqqJbF*vlhSB|7U@<4Xc4%t#4jadniKBo55fN{$aefn__?+>v zon4}iNV712hHV|^B-w4;V+KFZC&y&!mQ}Hxr(9R9-jra~ab={?^|ZSR%JOT@qa1bP zkeYHTIPYKs>j`)tM*ZX!6Q$y;Pn*~ez-$p`+jd~=9=-sw6e8fWs@3TrQG0t%%_oP6 zJ9SQFlhvYS{SWqkEaXJ-B1}0gl1A-M7d!gzHhB*VeMyyn_r*hrmrk@1Wh#(N!l%tj zzHMz+{mOjJDsqT>W?g-vR1!WN(2W1L=5eJco;e*aEsjbS?&w!fz;-Ov?=(%=bCucEi%73;?F zT^+gwrF6f+-T`xB={MR0x>aEgn~HQloxcp>sfNrIWX5X>&X!Lu|!02PYkaKh)$ zwtj9Suj6qW3x||V9a_!oPkzHV5AZGQ1 zOJ zKpsZqyt1-nV5zVOvsoytfi$LL?nHtFo)E@Hg^@WyPaYb>SQ8EvG$@{IXPPhj6S2TS z4y;k&lf+Wf=KjywMAEAbn^8lkLNOT?I_}H2PPW2*nqjof1!IkiI5^Z=E|zlI+H}2r zeVBm>W+o1jw{uJhKLwa580`dUSdC%>wsh4FCJ(@9+TDgEjL5h+vy8WJR)1>j=;)a8 zo&Y!b>P5p_AnV46j@H(8g|sn}Ch+cG%y}{T^K$^n7nnmdEUvn&E2^oPW|+bPvyzr} zSIFi$3t8UB|1Tjz!X1Yw2pbgOyBjD5xRzQz+$L9F);>!wPIGRh2E#`Ct%2LQ`Pkf1fLjq)vsXkFkm*acPc5vfJvwTjn?Ws z$j0;=USC2Jy&Bc(I8NoKm9O%Wz|5Mxq5)c1W;I5TWEBjhiVs?;ZW$ zB57qRd4>vNzmyG@_*H2N+A(#f+qZ=+$I^&apJILN>xFC~4IAfEUS7zair=q0xTT^( z2!RsK=|#XoTFD3^rd-Kz6J~D>z4EHhq@mjAjz0rCP=S9SCQ!2}GS6uu+{IN2l z??x{R!rp`u7EDxHTJ9d7^_a`yS1CIo_agPFKn2edeYfTW?tiFPfPjDp#7L-a>Os;U z2^k_V&Br7M!z3Nt{?uR*2mock47?#Q!!XS74U+?xvo>6DfvYo7`B0H><;aPl*44Mi zUca-fs#TEu{ey2PAK#op?;9^-J>3iL| zeqY5YgG_v(UX5eo$dKhgFK>)LZ7;iK(Gvb{%NX@610O~Sj@j#FU`=gol5`>NnXJjA zdxk21{{G}-6i!@7fJIYYh=@`SU0XBfofs{AvNKDde9$)$Et}a4BMi%<_S?9P?d@fu z0>7f*cRh=JKe7vaW44zrCM&W1;>7J9(y3EcjcqMR z&bD6qayZ{IQB}; z1#-t2;wtkm9HHYuW?kRVAZ_E`fnw@I+Vpi9RvK3QmfTs`Gel96{kYDv<+(e^cIpuL zofG-8V_P%vzXeeQ5$3#ub7yNdT#UMgQG$hx+EHfgIYHCIRAa@g!7uUyyB~5(P5U_& z&9AX&9_IF3z|h>NaP)pV_Cz!tqI7NRXD^g#5m~y?rum>Fc=yd`eP&{FpspTOyQ!Sv)Y5T*zUl&ISZ)N}z$lQkYce*Kd2^b}^6{RH}e$M>Y3R!Ts? z;{FE8#&MGz+?yEa9_VhQ7yi$wnSN~V==cv#jtPOd8;uXQ(LW;$;BEUp%`nfRNGU8N zg)Q5UE^GIc*tS`W69eYaUQVwNxDqmp)B^T5AqMFUAp3YXK$xdZq!QPKD8v~%MZ#5B5B#vH`aR6P#+?x13{VUGO{Q@OsRIBYqvT@?)$%hQn_4qd#RTqdFgc~2 zO5_3Te@v~CyEnkzY|Vp%1NY|5Kgj2m?_PFHwK3Sta`;>uFs|6P7uZMl?L~?hO5?4q zW#_ejV(^w}dP{00rrST9-MY|OxnB)sFC@LteAKKJ{y_I}8_4;PFbFvaeP6bmV?c7W zHePYnzG7T&6lm;_XjQJ`@vV8*4rUzKz#c(jAuy5v)VX)|9NPu+38(` zT-`IsylMN&6$k#aGuPQrsq6u{v@hN#TPn2t-lSbOcr z;R8W}1*#(F1uuHWAo%jt*Q1~OON=M0y=7!A zSBR^!vx#tt7k~t}$-8*xUT)j^M5SS%-Ry;_74xB1+CfUv>39A8_6YbkGs|*p!&up> ztEmG5N<{JQBTi0sv{}e14LkN@1TX>!z|&v%{4;HY0B+ud_(NY{<5Q2^2UyXMbbc7* zBS#jT9B;kRhuomyp%ODJ_tjZ2=5aZFWdM&E zGsM`yTh>95g?(vYp!(asPnyHxf<4w!KCHt}V8|m|*J=@h-Ziy#a+R|-;a;w zUiXRZijPax<+oF)4|9wx1;0icvfOs@2q3)5bo}%(;`*tq=)wugcJt3T@2Y}OWLE4t z-?;sl+xEVP`|{>Y$+a_m2mY&)#TFItsT=BRFcv zEP9T>kSVoaeq5!zc1?*^ENRGEz=HZg zD}#;>QN+`Pqcniiynv10l=E0@-68WMo48+zG=p*a`+^UQ0~#kLhz15i^{A$TIjn)& z(5fNy&APKIEZnT`^@#K0>2WdOJ%K8^$h#=yzKdn=;B;CHH)mgTl((~T`}t|$o63!~jOOJ*AeVy;*Rw{}9x+vcGsoCTqk1WmviC*=85)*T|z#&DUxGs#Pdar3qY;p!GBW zR@9XJSZ&XjVnSMe61c`%lSDjrsZ+X>#fV{KUTC=d`k^ifH8nu^+iq|@cX=QQ1vXq# z;YJ-Xd!s=jc+2LVq6`qmCnYOOKtv}P0OwuJGG;pExJ06)tjuLIDg5WpAGJG526vbu z@t{-SUZ1Q91}8<@*RM})DqrTdfRqLFlJCKUsSg8?envHpFG@MKCx7fVK|5=w$dt57nN5x$~b8iib){%8OS)K=~i5 zh0M*@x%Wi0p1dsu-6+OAVuOIz<$VI5jlrMIY0#^J13?o~{_J*_p4WKb_JKMyB0_Ke zM`g^z>-AtK4b}BNyc8W>8C0;ky85>RRk!xDNIuZ83}Ix%NUaw{isR|40!_Z* z^vdHen9~6EWk`Wfe;zYv`>C-S_T{m4&v)F|8mePvs|6d2E+RdYq4~*~P(EqB*;XOwM?i-@g{%_^-Kb5l zen}T|whrP4$T}beJpU}hH&1gze}9cr_;BuFZmZGzDLp<|R3rA0yHgFcV2yfO5@rSP19m2d>|2?o2NtXqwtb+w)qjc+#aRnStWeG8)5+sk*b#4>umX;E(c~Y`#H6L#zbMbpebYO|1NX?!4 ziabCF<9g6Wp}Fxd1wuT&ILq&Onc!$+B3DI zvvMMaS+%Bj95zf$j3i-sYi*^}Y((Ckmpb1hG;Th^4vD$sTd45p#<5SE7MMP?PHxd% z9V9R>^>i{5`B0W#Y$jwvBc(ubjV;$)v8gp=cC~m_`T1{4pLO!N*HetO%-twj+&8G@ z2*z4l7o?<CYC()%Obg!EZKYB4h7GrhwqTuNRAw%_0+U1~R?@DaMkM^9_xCaKZ)<)--5a&lj zPEh5R3D#5$$uH-d*T!?0*L z6K}KS(I)%kA8Uud-)?Mvm**4I90A@a?}IA->hm)wGjAtrHTCB(75_i7z5_1je{cUz zRN6*+$qo&q(pDNqN;EW8NV{lnnzXDGsgRbIq@=X85T(-AmbCX?|Le=|cb@b7pYyt% zmvc1k@fq*;^}a@giLO-=&IsS4m66R+9j}abV{H?wClwdhA+arvwMn=X8y9{Dlcf_KxD@3%Tcz?$Gn}`B4se5q(OpKeD$+tKhru9JwjAng=#?*2Mx9z(qEOn#-KnBmx_gC{ z0+!_fx#daUo{@dwsm^kpIXm2G<32uM)i#R!%rlv0s;X`XZc@HK6!GanV+f603nw>uBcUlNKoVujyh)5Pjh9+~LU z+#C|D=tOjQVr$BgSX114B#n|7YFzutymJpnW4pSJ&QaX+$ESzh7W%A(Ea2)R;*57F zg|bTS=^hyPlYF*mQyRmS_NeKxu^Y&c_?l3(CQV1fGAi%ssS-7n+@SMBYJbH!Rj!J< zcuIcmCQyY4t_~sW{~WGFu-(t)|tT-#~Yw!*x8(-=12OcU~EY_K^%5}ufVWQ(gR_;K=hI26%V7ew%0an;XF{{wK~PKJY=}`0TG?5e=vARw8G{( z+!%N~=Xnv);?+egRYcPH3l`3=la|hN@tW@zkzSo=AuLmn8&M1l3X$Xnj)q>S^9jQ_ z>^o4FjQa0?8iX^<&9=8Tz2LbMfIgtK$;nBK69Pfa0UHTo^Fc1btHSJYH5bIi*Y#Em z6xO`lC*ZkI&VA&Fe`TZj5!?A!{G&f=>D;-uzBqRNLB*4dFXF;d+bXPoHgg-g|G`vrZ0I=ygWKM}!(SPmukpOb@I(W#cs^IU2+AR%f+(Kyui5|5-ts zAD<)Xy$X+%2@Z$d(Qa>cPFCW%SjAKGJV&o%P;y3tPAp%V=~J7oAwwQ@K-DpnAtI}z zJI#0co7_I~S%diXKlSz-O&Oa8l-S4)HTmz@yspJ+J?V_Q!xk;a%k2ERyy@ktf?P`S z(W6f8KhD(x6F;o88g(SVu4gDJ>A)e$UsunZZ3|rSDhl3W*jZNN=H>wf52!=gXN9@#DhtFGJ4&GVFaj4xaW zhdz`eDb;di;c~pX54wn(l{_*?fi!agYW4?^uwH?)8U!+JNfstNXSEW`c-EZhixxuC zN35r>dLfG7m1h6I8S#9p%~!=^-JPusGH#%I!Tlf?`Iknf-38y9V%YKgN^J}c0EW!(keneiVVytc zQ3c1XuE^%)v7sSfkT=Dm3O(}I1u>66M%_K3prJv|?KVsTk2=!c;}|$YcX;zmDlmVj zHcHmkAs5=_lFsJ!F548TSY>-!%umQ(EcQgu$)f{hhDBwHJ7VD9aAKKm5 z_iZ$jrRZz&zjX_h%g)})FePB=xG%|s=@glm4pWuS?0fLJVXp=(|Fz=eo*gs%9iv-H zVgjxq{JV4ShXI+5A~%m+d|CA@jZfauC0RrZf-?zTGO1FNJ(NAt(e# zZM_0l4V&n^YkkxJ>n-~}x(dXr-#TnsP1tiQQd-(wyY>Ne!cAOUbfL*w_w&@io*?c4 z9_iEw;s5~}QdU{1%Xg?wBUev=Xdi8gR+z878@<{0>HZ$x&9?wK+m!#N8t_2pKsbF9 z>R1?3Ldw$^1)D{??hhLPH=7U>0Xn*D^%rI8&XU}UA?$w5&Ds!tk^wilkx;JE3;QED zYYP%bCPBiO>mWZ!>jQYD8x$8so9ebfS%Sc5yL+BRFB- zO_8bng=qIxUZ%hstD&Jm`ILq0(=TAb110$ft5Dm!KCIFrky-9zT#v0-q_l@P1XX3f zkPx(U*RI-?J-EF?%FD}nckS|1ITf$Im6>__-n}1Pk6pM0fhtjnaY;CC6V+;Z7oiwe zl2NpZ_~l>UJg)mP#_6nH*%yx3=I~>Cx9ae1hc$TU!?}_pMU>J13$EuC2P+6Lh3)n0 z^&Ic#y0aB;9e+M&ZYgYYhsRIu25hSemUsx6Z9xGO&^p)MYog=jM^Lp3J?8Az zJemCISJ-fYB1WAgCO&x=v$|-B!xLRMth+SeGqF5gYHz;798>BQyZ$p9M%z=$<|o>S zpT1j_x#a6tdOyEIgo+#-o94-!9LZEe8LrvcGzJebs6y5jYIeR~o@5iEh3UiAjT_q- zIfjBx+`oxHDhd`B-S15)uwg1jZvm_7^~QGf#len_`qwR=*Is>PR=;~IGkbb32C9{q zFhQHqxhvD2#cO5g>W} z^ID?W6V;KMj3*+UB?WS9VuS2@bGkbUP6A{=9^;|H`JUaV14~{Rt#fvh@Gk9snvk#w zqII}hR02d++ZVN19DTH2+@qaheJ&UbN6`@ovU3~(thzEM*qc9AC*)+!YHa+zyW5E2 z(dk=Db1AkH#+$6$&Dmug!1q@i8s=-$QZ4Sa%d~0(mo1A4@|h~r&O%GTxdCj4<%G;1 z@VI+$LB#{&1TD_WuH^+e=;&eEcQwu-l!wO`5f`@@WZu(~;JaFxYu9!V1DS-=n@UPc zxfK)~#B8^5G-aG>UH~eDm?QD!Wm?L~T?uYeU+)qe@>{o5iiRQtm`u8gW5+RfA*L*j zj*>r$4E^vKw^*&a%0BMYTk%L=B==~k5VRV@Lm&K%ckX8XPGk{Nz=;6Tz+_&J7Eko^MzLxy%k4#mR)q0 zm#<^6R}>J*dgT3k%*lY;&yFTaYHjD<~&?9P{p zDk}RAvu$EYRCIJaH++bVrPHGy8;hTli;P}L)bsOgrVz-*IaTAcf;kdD_)tpz>=#`b zWoDkl+NKY@tzKHAO2f^Nl=^dtE2?BV)%O{bon9a?nefwR+L&!$=LpQPeGyUgXW zE#oVPmCioYC0DHK8V)m4?(k$-!SLYXQ1$R?BW!G+dxwYD~%Ep*YK|5@h&U0V05gu{k7(73}&~TV4a8mBL+_oQn zXW)!_PyCX>0XaMW)tQR%yPkoDj2*vBe;kbti;FcBT{ir=uV6Z~NRQO{F|#?fvvS9k zldmqUY3XvOGjyg(-F#fi{u@;wZ0SI@JbkNAAnHj(q4=&5HK zha)F{0CTQAYoc|$dotq@p7D(u?zRBvEV_6GHgd-Xz5c=eBKo6Ws@z*udlqq{Y97Tm zx??6@96>gX!u(s({7VYM1iA-fPSZZwb~C?tV+b`9wROY}41KTqs@ zE7PG})#<9GEi-2099=iO_u9a?qr1zTQXq`n#bF2anC7;_$&4r8>DY8^w4mg@ax!p% zX28o*_o(BA*Oy41D@SNZu;c)Myn&K3ir(vth4lQz1M26h>;}(Elb$}6_1_aCww{AS zVEwv1#4)51-*yh>6MkS>FOS$nkVuj-1C<7ZI1y=m8;L(?k$NU7DS>O!pBwEa zOb&eLz$)-l*2B9=I~1ulu@6%0*~6h6PyM4{e>o5%HO+*!xeb*0J<|j6wvIVgZ#^@$ z{Y_(zh7#@AL@$zd1&^!Up&1@=*(<0;07jfrkAohww-@Ny*%gY}U)|*VUO-&d<#=?v z$VTDlv6@F){K+Y=O-~{bX4{h|t!v!et`8hC^JCUPh!VzWs$gzjf3U4QNFnf6OW>#b z3eY+;!lwjzf475%Y*+Hun^p%tQ%lNpb+At?tB=0mdcnn2yvwO5sI+Lc!CvoPl&()? zWQca=c53|oE?e&>Ps{=Rg`}qLBNZ==?I953)3@i+-YP=DMyY(l%sii9>1^60gJf7L zq)L&~QxXm&c+ceaX?R1~i5c{*xz3^=*6rxIln>}SQjwmjC85U`$cb)6DMm1k?2{oF zmzIUQ3M4KzMN-tjpDfjf8+%4Z0tr*E)nyO0lxzT#7?Evs$AB2x8YNk88aNBU;ExOj z!pOarFkM+03(!eZ$x6I_?R9xa0gaI7vg3mX-{YPg3jscc5V-e1Y~*ln(V+LowSjLZ zE=_gy&$hk($HydopF$zvxoAd#i1(XyYti!VYaZ3h8!Wo2@l5{+-@ za`Rey%2`Y%CMva5Zm$d^Qg%0IN|I#l?6`V*MCA9{-wX@$=_nlicz4(MT$7Dkd)6MB zjG~yUYKf;4zMsm_wp$OUt){Zog_;TN_s+5@nY(tb??NivcvF$cO0_|O4qVF;JZ8qb z<(k{3?nY?&fO6ApktJJiHyO%xy?dc(YP9iv^IRYM9CCH;8$PXf_DGkPiN6YBD{>2~i+kEXe?Af!2;FQSw z)0W`d&T44Hix#|oy&0pg3TSPFJQnSJN;s8kOl)j098EC7IE)}^S$7s%sU<-7h_6Et zwsH_C95K-2RW}xZE)eFU-C*NU)MFqgbp6-qvhwoU(%k+{#@bEHw2xG{oyUY?X%3C|f->4lxMqwPWY zPGf^A67KUtKd&8yyi}>lWh1XEe9jPa^~@50YY&^@vkY`LIHQeMOF&<)WDRO=T0>W_&;tfB$xo;FVfP|T@zOs zHRVzj5X^(m<05OjH#e$kATIu(g9Qv;Vx>?|UiF4Ei$U1dJs5G;>7LinFq-=HH7O;f zzKQ`xbgwU075CS_Qoz5l*uLW&Zau<7v~?~n^($fJr{K`ouF*q2)nySw%&_vcQ>*rf+K5l9Lg$UFh3}2*C_+Zr$LpxL&qiws~pxvPoy* zFnz;938hI%H;Kpnn=lE)wXWZ_dbltw-n@_IVt~e*B1_@YvWE4NS@=x zo;-2b9-XeN2pLf9pP0H1nslhY4nLT52SHW&!&b@eSo#I|!7DE>dWnL|3jqVcv@0dS z$n3Pp%`=}~4Br$qLME)Ab+4)cNm~lW#%57#iLy^imKXY9JQ4ipRFcC!CYG-OM?yn1 zv*XMCZK7Kk2 zpS<>92M1)jQry_szM#D|f+khF);n5a*>AfUgmG(jbxWx>F6!HJ4yN{GD@fXIg8 z4I%|szS<)L!&+^$Kh=C|Lkiub#5t=AQu}BwTRp6O`N*+K`~LHUO-^U}daIL{HLJ$w z^@0tz_#_LK4bUZZFoWCjKG%-k@qKt;FpWyTVaUu&N0}ksbD$Nk4&7xEI zBi@P^#lPG5WV|lnX!G)Cv#sqGi!vH?>_33}wr7p-H0j=d8Y{uQ)}}qYJhVZ$F~!zn z&EsOx#Ow6JQ0HH5Y_Kl(I?W3$tmI0cu%+jW@C&H>wd@pk-9D^SB)ZG>9@Qxs)}~SM zakUegOebHzY~`|YCLsCzjGH1!Tk7hZh`W(S9;;nIwXDW6sc)wsQy{AIAo~Q+abkA zlx>e5g%4-u9K?q0!Nl;5Zog>f>CzV`B*wdr)#qB6XEV37G`L>=(XJ9!TZbJ)6k;DX z^jeuSs6;XbszhDk6>I*Qy2!zCglK^FsOA)eVQ?kWm=Q~_s#;$WyPv;dEiEW$>d1b9 zQ@@9EJ_TUw&mEBrutfk%pLtQ5M>~dedCAH5FSdMn<*(=o3}(jy5%G0 z`JPVz(DPph3xh-tYo+jqgKy~^`^4MWss?ed5=qGd2A1NZ$tmZ$^T*WbKHI(HMB=+P z%-DB!cuIV_n0Ih?)}2$P5IIF7W$kNRN*bCXU2QfYO>1Lbs%3Y0rJdJS8ojc3bn=8Z zvj{SG7p)a+Afe^_cvp|}sB!Z{W7En~WG*2uWWYn3-Lq!h{eF-0uxBtD>)4>3GrSF! zIa7(cZ$H}(X%3AzBO#Np zLto~MpGLK< z$>Sy?#0AHMw2M;=xv7&@fsx?MBm<`y2~?rTNY?!dig_oYAVYO?=fR8ot_#SK zLy}BSW+iuAi6hm(&XvWP(jdXYKoIi*OPu;^j2h z{k`)+0)GQt7BMX~Prnd+^62Oggp>R=g%{B0*?aH=N(RC;+}``Y2(y}j;PCuAdb<-- z+R?#{;2A0gn?YasG|jl&Tvkxj@xXWz^RTOLxx-KT0m=ILOGN58IHlmf!zQS#P`vf5 z1aH|laAfV)Ry`cm)Yx+%K z*8tTRn97R`-my|{%o4ZFxLq96FUlfEF+Ft_)_`n$%o{V?#xeidUV9D}ELP^8_ z1>;PERt$X^dyyOhE%vti0}RR{MjzHA{>MJIBQHM&Salj{Mn{_8yBs#R zu$YR4f#gDT^;0;Y*y?fUn*X24B?giRekcQTzwkKEC)NCXe3NkBZJLi`} zJ9+rFiX(!AiJ+@mwmt~W6ECe^=7Rn z`1@2S{^lB^SU)en?bU{`N^&@pCAi>J|aW_3dTIllhgfl&~EilopIW1`GLyO=p=8Gw63Z#d_3q zlO|{uopRFyhNG3Zke!*1L?YQbj#o=L3eO99(3Kg0gT6bvZJ+CprDvzSx1e68?_(OiJ z{^{yb`eR~=i^g^_;QWtfFCK^RiY3P_)7LA@Oesu027hy0`oTKmymNFH^V8qf8@_*T ztWhNumwctS%PJMZ^KicKH0^nIrxDDvG}D7WXc@+nNI6zj#UK2_3i;W3e_sU=_lgOG zq|Lbpx%z8Zp62G#i-<_BUoU3zBUe@uF^m2THj5$2mN{SA+f~*^0VF$)J_!>n%)Lb{ z{oA((X+O^|NgxlL!^bq1%Thgh``!KO=d?_~-xG76C$sKMmj&jt^>!#%nD)twPiY<= z7zn%GPmKi{O14gTMLLwVJD_uUH}ih2$4&m5pw2#e{R07AJ^C^)rHumue#AZ5P5vg} z`DG2txM_YVWT3LGn5tnoXyJKkU^wRlDe}mE+I2T}?7Ww7eYsiN5a#G#)BDdhDkv#& z5Pc1dXtTyAe#O~2WS3@|gc;Su93VQ7AMg1NwldQRXU~0!YGM#9l6+I&dnjQ+ysDez zwWKQ-;aM0T5sqFcl*fzKrods7p+~s!n`ozxj6Cj(1IG!b^Lvp{hTw@A2jnoSc8|Oj z>;WLIW@wXZRbH{!b6<7$#li0vpGfQxQ4g{+bU(NO`kF3q{z6u(r<^%YKi)}Xx^Ke& z8L8nkb)6H|WCX8qca6oy>;1FxiWT+qRfYoSe}&+9pp#~p0M3Mo1lA-yy) zi7T;(Sroxo=H-1>DNuF2zq%WmGglW}*oYKovA2?HmBXbTvPmyjHkKdCTooSmK3yiJg;oz&o~=|`gg zPd4btAuE4wDC2}h{%!Q)&5G0u*SAKA#FLYDpQb8%cfG2`%)#O06O8w7u499ugmLj@02|D%|0rC8}n5_Ac$GPvv{}z4XyZQ-B6#x5{Qr?hh@k z{ocR(%#YBi{-{MS{Q_}z4$*4kL=DY-gmpE3@Xu>Z%edtoElG0n?XZ#@`@yHAqqE_2 zRUdqzZLhtCu_I(?MgGtE@tCw=y7oxBsR zA99;|1BPX=F!GCy-GUUUNSr)5sHU6Zo0uqde7n}^NXDymb036g-{=rwL>Qq9vxk(pol+T6 zUKmY(e$%JNSYU8a?Aui_+iS*MeenuvcbLPz?QG{89D-N1z{{7pYJ0`hQpj|5DXOb! zedYMw-NkJ_cK^^55V5*;u^#;yCpULq&He*FpFh7DwCKvs$tgBI(`Q^v5`dvaaqk52 z=$~qttEj7(H1Lw5(MpF~!1L#NnI215<1V+ojH>DPDeayP93n&ao9kIHFsG3G9mbRp z9CNcuwem=}#KXv>@@=!}k?(wWV|D!5Z*fjOPB_*z^2D;cyMH}ye`ktZ z`?h9@bNU+O)b;z-^wk6W^Aa4IVQ4k#Sxpr@>z}1GW+|F>;2!bQFNISBMYw2E6A%@) z!+q(U{I`sySp`krbcNaiYEY6(&`4lbHBTiYmfI?&UZ4}gypY_NbRgC5^h;8wKc%T4 zWgx{u@!@apt;k8j=KP**N%g4}nkGp*PfxIU>+O0B$ zzRxd|t}JN;1}|^a;V0MC<))^IMRw0jtPj`WEBk@kP>>{-hPPo-O&ixIV*TDH+afH} z%->ETYUR%(E23z#f- z;C2hEG{Uy4cK-+Sn$`TVq59A~RFAp*4Rp$>o%IEiPrDoc^g$5%4n&k6jru_+yjZnc2 zJXYM3l5}SWU$WWK%T!BJSmlkVC@!23jPUCsZK)2sefts>?J_+ko!9O>%vroUx2HWE zAR}zbh{@80N1PbXth=$rpMFN+{Q1gNQ$IPs7cY#7f3r&4Y+x{p9IJ}?i5s>~W6}La z{IJ#N#^4@Q!e{AcHelr98<3CNj5Is@EBf5^$X`TczAjf7`TG3kZ9f{AyRCCU^MJ9n>!3?MZ!Yu|jX)4HzueWUN&m|__k|5Gc66qMQGxaWf9(T`pxXuv zUW0!k;n@y@lJ69!0@)CRI^Maq%5pM3J?jZ9z;Myj5tg=8W>w(O5`DY~7bj;hn77)3 zPpyCAz!r0zPCDjUklPr)Wvk1HKYjIz zkwe7NTQlYI6y^})A7})NTUr+G4iN1ZffZnXTGxjJW5|zlA8Rr;kO;!W`z7m}3PWQS|6X$JhJP=)*Vm3m zt0HTs$F!e#DVR@2g&JQU}AUKiZL(Al=tZP`md7Ga?D;O zf{O3oAH!jsxp<;d@4Ysn_HxKcAn72|cT3tssV9P^BPUBU*Ap)!b^QhphWOV5^RrPl2|a!+#vu>eb0_E<)1@jB*nVZF zQ`PQVSJH$Nj+}%{v0KhF4bJ5Xu z#6XfaWRB(Kg2nCah4nHK+M3Sut}wCzO5Rl+eIzjWg*3S|M z5iI)q_i#;B2|8t7>)S*szOPDO_K^R2D(9wws`*BG?|+M`fsF{%I{xgKW60(?3wGYK z-$Jz8L;T6OzrP%Rs_D1IZ-!B9{9ET&dTdqv?SO06=7AP?K5zRIpWgER=g&*DY=?Xs z?=!=%bn})icb_~-os(UY#|i)>>6&rD5moKE#4~=t;~=GLamRiG@vH08p4j8s+L6cr ziv9|<84Zg7A|2aWBvGxPHZ*F_b9j}Nm1S)2JMF0_Kv*v+UAnZBNLGdo=zvNS{7T(s z4}cNp0|7ORO)o&fcLHCC=_BEdeDD5!yMel0K4%KC>1=^X>-`hn3`gn=vDx1zfHuLV z#CCDoEUwazoRYo_`+wlZ$$t+m>>hEpfYKZJ`4T9Z4qj3|m$(lo86ji;yTMPKxK-=; z{vi-Zn3mj)>6n&Hi;m{{YlqFq$w`Yka(rR><3`>zJ4Z)FIk|P!F%1r{q4y%cWNNxY zQu4>`yTry<0ab9cv#2IvYHEr%{A72Z0+)O+>%n!9=7GtV(UExx{9?o_2JeY=zn2dC z4`?=N5TdX*{G>?6)2AeuSZD^{MgkKv(29_dkPeT~Ywwc#G}cZ$$(+ue;&xvC&TsY``0pLyvDbPz{(NnHmvJ zmu)ka!102=_VULWF!~CF^-E!|S{K2$m73u1?-dkms+yaft@s?izq?D8Je|m%dSSx{ zu>?Nav**s42w6cwiDe;<+#Q^tFNmb#|Ju_29!FGVJ`1O5wskL@==|`f8Y}6YXx5st z*RB7;Z&NKS5MdGS`F3h@dd>$re`d3Mn<}Tyc)66zBoe|~oF_lsh+cFl6E>~>T`}&m zwz3|AwD6M~-sxG7@AHwk*l$&E5kb>oS$)|aU&h}S?Sfmv(l>K9_tgcJ<<7NofU&L~ zpFJi&9oZYMQrXT)FJz(^*kC+>ciRU=a@6chyr=eK3r|jn24Oj(GFA|9GOe*EfTyl` z`t{{a5MI4CKK^1yS&z;ipDE2$*1(U-c}+>8dG7BJM9fYa|GdAnt52cn^NC8SA59Yr zdheG;S@$c+FNzQHKmK<^n#n8pusiMNxNFc5$bNarNA&Kf{PSyU;&4y)g7>2{KIFyb<1ObXhk>U>fhDQbzqdJN)-X`u)_(xR7NCm-` zM~F5~D*<&HjN25@ktL`^QNTlDze3yFkCzU?P@Ms$EJBf2p!&tr+b$pwI(zrOE--C^ zF}i)njux`+@RPfs6}ay%em4a+XIV}0?Ph09f z{H~3&gNPHqN@NWEF%=C>TqqS50tYx1-tZ>Ig^@JAzNJND`%83 z6nBg9I>@{jSFqErleYdL=IiHo;pbs^xEZt}O>FxNQEy49fWSZ)VPI1ixT~tFrs`x< zpE+~pPb&+cUYs~DF{;3zVW@A)SrS>NEHKq*+!%c4%zxUhzeD5Nxv)cN1(Q}^RTr7L z!ziU=8MwnL3tLaVeR9RxKEF`kP`E1Yr;?$=w87L=cWA?RrGIxEy`LNz zvF}>nWeqIU22ycbx3~Je*y)l|)cOA7-7bdl(a&B8a#cR;_#-FG6kX?s*6f1>tDG=V z#Eu$=Yi{;c+6X}wlka=i9ILF_92wSD=jcSFtaPYQ*YdNU(jw@nI%(F+4QVL4o^~>fga0Q^~W?nlm1-UG?qSJE5+*)sxtBQ`N+t6R&E?+CU;PFt`z0 zU-u|`UA@$DyLR?IY8YH-<9{~Q2LAYW5h~L-?BgHRJhHvuHQDA3?MaOs-}h!KKiqrg zz**Vf#Yztnxt>c7cUj7llg()04My7oC(f#t_b`Bg9}2Oko*^V=&zUE(Iy#SQ2e23b z+k}>L4~~UmcmmTb8r*wG{}nNeS2uzM7vWz8!FC!e?|>z7d7`95a6*(54DWpX{kIE* z3Jh+k)4*nB+`RcF<~}_r7B8|Y0s!R@B~luQU9@lCZR^3YUD&ijUaRiGoJTi9$R_9L zm~-qA21TiGrw-5Ikj*J(Tl|}}Z*_H*m~SDRD3y`zKL8j`@*_DpIUt}D z8<-@E@I4fwgxmP1zDdWS+N;8r3juI;{U;Q{$EUler_OARl2__RNC+L+^uD-dU%6l~ zq^zP6Z-HRHI=TsGNm_PcD$v)BrU)^oyYk7OydW|OAgiiA}4se5&oC4nkUU`=p} zAd%2mqZ{-Mj~jf1itFdKceN36&x9UprZiYJF3k6;2$Jr6YTYQC_U4NWasA3$x{UBx zcud!76ZCHchCz_JEA_$_auBDGiZbc))M5)vx#5B(gz8C1M|Io1vB zf)_2SN+C3RrA@293*n`hQ};L910SbgNLbZ+-1f?d8A8p=BSPZsx3;(-zTmp0Yeg_4 zerOsC>3SRI$J`J0a~ zHMbYM+_Z3riPeJgxVTEPTu9yaBh}0eJDSTWYPlU*yR2y>xTTUhBaj0E3*(EOg&vj$ zU`dr3XBt#)^4M4^grv?A2rFhNm|X&^`nCh@gnf$-sw1ZD!#b!&iEK=rG)3q|qB73! z%LE;bI43u&v#h2i+K+t6=6Xn={!!_qZ%VTKT5WZpPz1EV1V$Ft_bC# zf!*f*i*z8<(&xMBFKBB;?hzduBXR#ZZ4^+3KFNdHaYe4kfWZ})vfx~)YxcT z>Q9FnYtJTD)~kPZA=1iTLkR0xFg6=D?N$Ni<`|8C7Rvyt$TtL-mrx+k2w@R(XaHkFwPvMsFh zFLPgJkEn{5mR1fEa+|dV7;GlKDQ<}S&pthhDg$-&Qem3p z^^6NT?Dfhu=MiUfLljis>(zi7wiQlVVuOe%0o;Ft5nvTMMM z0hAULqUZOe8Mp{U_y`s~xmn`lae!Ig^xmfjfkAD-h$7v&`CZ2wEkiEF${vq0%v-?d z=)K4aIWB!)U!KV}93Rpv_BWa0{v1?>EH|9)11N5c8O=W5h?Pk&b2n|*re@Erpm zvO9au@v0_#_#{XFTxx`|--^MI(p+F*Kd^jT9B>6>!muMlFPO?2?ttL9E+YG)s_L+~ z6cO41)FvaN&wBH3d{hD`BMFiM@`H$vpG;+{GFO; z$NoKt8ZmuaHl92o%V6_^aJwolNYaX@>7k{4_EmCIY}b~qX5qEPq5O+(l$c0pf$ z%*EwsREFB#S1wam4(R7Sc*w@7r|xU;kK`8#@4vntMHG1?!-g$ZTfT(LXTv(T{J}X zZm%jZa2E$%CXO&4vM3gON?KY~P(17F)1_&8T|5k7SWp8Io@MU@+y&}bq2H?C;Tw

rS{-?+D_egFqj~nB;tfSPw z$Ac!U(!|hRy7RXiv6<2@^nH!4=(`4BI<>%=YM^$sBYe9|=eSPm1HSD9Xte+M_jlMv z*g(XMr&VP}?f2nn&B`bacx1EO^(blM*fmlEOZ^x3#o8hrc|N6HFv7ew^_(#R$PiTH zG^%NsA?Hyr;qh4c=S^&!+b&a!4kr&pAB(Hqz9+GD+ZsdzU40P&jvlg7dO(A>>>hrl z?XkAp?|i^Ove5U{1CGIb(j_WH92D z1T}-9O=ce9bT+bM@zprEx9JDzKwWoT2;J0S$qR9qQPzE+{%yKsTc-NPe| z^+FSLi^OX8qhSd~dNguj5YS?bw#;Mpm)1Xb8a})~!PjfHPs4#SVuiQe-gK`T0bhRe z^UfGhMt|KeC|Fq=r*se%<9q=1D&LpIq-}gk)Wiyd9207u? zZ&3!mh=qZj0TI;zuK>G=9tE__1&6M^$+emGLpa1=#^Vm%Pkx$D1zk5!(M zt@pG;S3=>F@^AY~Y(iIQY@f2SvTf+ZNZ;T%{oi3^kG|i4m6#I9Rfv>C=#+>E6*V={ z>k*5ekr4$_+nE(Zs!V?b;}C`9(W6H#4b;^)!xWH$IOr9zO3ETZ)hf6Vjh}!#(I|1b z9S86mQ}y$ii4N7sh?lTEdWC4Y1KO`hxD<1YF)FnnLO9_fv2WkL=J$9;d0Ih?b%}4@ z2OuF_m+yb?Mqh36h$=sPF*Z1apazMD@izPtbF(w$G2Ts_QD@BQO1 z{+E*t_$6j#Pdj_55Im~xlw9n>6!#dI?fi=u{)iL1x@6~ka99l-MgfVFGJftf4HSXm zL=^e>J+$4G9G3Yr(l8gC`W-2PzWxdpAD#mJr37n&)&5mhq=)PB#u zzBw8H+Oc^AZpO$3x|ESD>7IRB)T^bnb#O&9g75o0UtFu}+IkSQ?5af?I|FW)*YtjS z#tHe_cG#xCtPN$YTCy{~VJ>s~W z$7Xx2hs#zkQ_pLs3e-(~Z`{Y9C!KuW-y~quBx8UPYenm^|Exg$XNt+k%oWJhU(nr2 z8L*Fh^Yw1cvD64Df9JN+v;c!AJ4ep#3n#ofur!N*(Xky`5BTzpftvnK7yx57U&pcS z={pRszeFA+M=*vy=Y_dxTeWcXD+TT>EG)z_!`K^LxxEN(L>{l@P3S~i7cg3QPUc9J zxgDNUbEEo%=M)eDiu$9R^oqf(H;`IV0t07i?AY^$hG7Ir5nRY`<{^f(4Aj&gpz+5p1)Vmu5!}i#3uxreKUgsJyX>$-ot*|GM86_u}^=+XU9`@&}mX03@P1zS30M zcdh#$eqmCtQ=Kjrtozc)TUQsu+*8Tt z5p?XOfDW_d4Z)X$d7Eb7ZAN1ifHO)bPf|yP-nvx+8|73MM%*Tugn$~ym8JN;ebk^m zh6W~V5|rY9Gu;8GtdFo*IhI8A?(b;g zMFb1_LL6-fTDC0mj1V-e0Ow&wy8n@BOy+M>;-|OYg{K?58|?577r3JcC&m?W&%T}f zTL9SeDW&5 zyguP`pCvy7o;GGk(#$zS#Cc)ZtBIC zvpDVJpEksEJJ?j^Bia2iaY5?nsGqb%~yow2*IHjms^ zV0m1DY?1@&ESU%B`_Bb6%v4LnA?b4?b4u9s0S?iQ64-DY&@{9j6^FL%LaKCKzAdZ& zvrR4kB2&}m5XkF2+-qB;xKVuL?q5Bm-(y#T&aR`JmuS((6||86?qL2e@)d&1$tuCp zuW3#qFZXSw{&VtMw;vL32WpR4HsWZ?7kfRbnta|ELC??}x-I}GgC(m~hQ28k8kLEa zxlZ}y^RHb19}s$!A7HqHOZh^UDP2@DJr2o1L10M9$X9=tntAT;3Y*|U*{E>lZN zN#zT@j_HFpt(f!pwyo5DCBi`2p&5#tb^7mxBLzemZ3Ujm@TuMR9g+lWY;69K>`+pc ze*GFEp1obF5%uHUd-pyhU~|DeLV?f>2@=`HiX$Bb97xeIs>e#Jky(4sv}3RXCoPD) z-b7t#MtFe(L>yoZy(9e-C@S$}q8y=;f>Vxhfj26UAbrHe0PDF}4|)!Qsl)%z%2wSG zQT1n8<9VV=4bu|^e4mja2>b-e$oOwOkTb*2c$7EcuJTUpBF}=3t~Q_;gtaetw$>9h zgkmV&u&6pAfR^nhE`6Ql>!{K0jQSd}0#B{!S~b9!faq%hRQ`ki6+~;MKumW-C(U0m z4i;C*W(g0qQX15f&p$O{UE<~rOPuSOT2SodIQ3TR*VzPMe6EeqljV8id-McEo%y!j z)~x?p`RI2#x;aiRF7R`xC@Cpt3o@rmBch|DHD;wSoxOja?~Oex?pXcMDza6vvY=|I zf11EJFr_1{y3rAjYYtZ+mrd~L> zgmw4~gUw!efTCMb9c>8^b*tB$m(ue0H_Igjr7uidG|shMM!cdtY?TUsfY+9?b&w85`o`Q+5kYBEIiGodlLc zB8D7rP{4ZpXFhy>--wz%em>)^a$rkt}_I@A)Xai3!jEz}dk_W+a_p26nTmPlxxAoU+z_INpnPy2nM# z4W?w&6QYRng#E~#69harw*LKDIdV&DT{D#gH;07Vq#+ouLpp;fRAYxJ$d2J z2@bG}aNWd!eXy^sxw$JjF$Tc=KQ&6`)-+Avp5%YdR+I77vx#Qet@JgjrChW8K-m8b zbQvVMRmVK4?5oUvJrp%1+U*<_)IJK>k$?MXv;!qo+J)-LZ-^2EXDn(oK>ADOkF1Y= z?dT{}8tE-8Ja?7 zDCFWwG)}+x+s|R{KG=uL2KI7CY_DCz8}K5FOU#ypPOz>jC@ie<$d>HqqPe-1%aN{NGJSlmOH07-SNMW6FycFXXwZTiV)opjn5nM@dNul46FY ztlqzHlYq8TKri=c#9P-Hb8{!;e_1Q0H1t3nAxp$Niee*tiIHl%eb&JX#;$yP8;Pii zNM;1F?K<`FTP8cN2MK1*^Sxo9Tzvc`g;+*32d|CKW8*tOJ4%$+A|rT;h(~C+jjs0Z z5%G81ybwif?EgxVcX4@F>v;rWtS02J+6LE;YW-SGN34>PM*q_4T8g=r|I0hF6zWFG z8xZeHQBJm=?b7DJEoYL%-WtDc^>Ce^9dM`?_iA@ojA`qi%~Z8;+OO2Mk?E`*iR=;O zv8@WSwu;KTWHq*J^i>Og<~Y*HvyvqczRy2cqI4>=(bU1CV@yBafa38^4L2`o17s0@ z9Gcn3&!5J0FD~D;KicVyMN@iiE0dPqGr;aSEf&()yf4$!L%+TYB_n+czwj^~@Y*1e zNtcqZC(!QFAI&ai`&8uz~b7LCANu2ZnKdj$M za))8h{OG-OJ*+ZrdU{%5tzf#`CUii^&4)}je`v^VAcnVK7|)DED&9>Wq`HOp8%f{W z>_V&Vg~Tu)BaQFb%L!&(dOs7`^6G{fUp*0K(_j7&lb#MoTvaH-e)!E{xK{@x&gyi- zwFEPBU~R3hjC{7=Eu+bT($N0vk*43b_ov^fA9%xS)bB9)^F+aXPmnlPsh*UKj3J!3 zGVml-K7D)0&dxq9vo*_{LM`D=|KDJ1&1>%&yykRi}Z!?e{bKpd`72`Ipz}x<1ZmzcV+ctlD z;sxSAd9N22U4c)08k?&cn@dSeJ&h7XMH;A$Iw&?jrxx%yudV%YH`Q-7fV$I&1sudp zpQ#fo?J@mLp1~o%W*6ycoGxx-&`7sNB_<{gH5IL9>j^}JB#T3H>Ub5=D;tBjSeTg1 z(A1t(Rej6H<~{owq}P$Qw}K)(hpz-*`4VYlvYm8Q23Nl;sv+a+IMcjVn!K_PZuJC1U{^$TgUH7Tc60-Rb$##I9fhn|GDqlC!yeGF$d3tj(42(_pVPBa^Te;QaRUc ztNl!E?`%v>~hc2K$DD&~-siA%kbpJh!yv7O z{FUE$kKj$D3DWHFo6aG-D|o%Qz#2ip*j_JupBv`TJSwr$`e4~!?X(q1PET)avnfi$ zcx&f0mcKA$<{|7yZg-mf^KrhR!MjVf{=B;S_M%R~qR!Z9sg*1Q@=UrGOfI-*BE_bK z;?|ApZ#|YA`3(yt(;e$oM9q*aBeH89Y3Dg$+o*zS?6P&O-zlIE%i=rRKRMH6IP?{Q z%lXrVpocrPJ4Con|28^zh%|z5!w_sndCQI>FE6S2E}z*A>%1M$pFR5w`Cu`<5JpEKNle9dG6=G{I*-id;ZjqMzL_f+d2E{J}A1# zU$5@t+jH??nMqMitD=&UnyYK}Q@sRye17IQBDASGo~#~}l&WENCj(yn^ni-?%DA+q7+ghBN;^ey_gcr}`C3HD zpB9aW;oNYksX;-QR7F=es5)Mkj#pf; z%h<@s3}!nfRC7blb@OBBbT*VRozN24u(5KOHQhPCaG`-l8Nb62pha#%oPR_k78~!B;9hxDhCE2vHCs$V?xZ zX}6199EvyxsE1Z*uu4j*fdvk(<@K~ zR!LD~Q#8|j`SnW`hSN7>eGb0W)yiBIRk1l2Aq;4_@)i#zWmOZR z3&bO9{Cei`$-3$Qn>o2|@sT!NYS|$s(m?J71k@QNETYIAUg{}7*@>}{e&81j^uS^4 zJn3-ID(y7B`Mm|jcvvcYApR~&2H6*Mr^*N|GB18!J=Cy-a-o~n&nh*{YP2sPAI6zw z9(jT<$=|sve|}du$5MiNGE#I&<|B&2ivA0xC!4w0MMn~)<_EOh)6gqJ{;WXYKu47# zu#-zs8IQlv6SB*3do2}9NW@r?aE?xmj;$xuY4v-wY7ob{QyMj&j3somr}p^F?vqF7 zPpqVh`WdzdIW(1Fu59Vh&~uE8&+69>JUK(c7g(=3`dJM6WSszRmqq6a`|rR1?3J(g zUj*|RhfEUO$Ez^1(Br^;J*Q|;{1N;x=Grn-RrU1~$jH;nl2xh8!PyiAOS9&BSqVlS z&eyvycriW=jHq#HV#t#-V;Qesy}He6z2ymwnzAh)MKS^RY{}zOze!gLl2k)gOWwSM zvd*8>>8`Y9FZvilOMRXCznH3qmw;PijAQv25Lw;r-zjyM-rJ*XO2K=zFfiMtjWRLKTsIMkfnSCczrWTl6s&|Nw#&;!8BhGv zw@V+c*LTTS7I)bv^B)uJJz$r$vWG4+a)siqZdV_=E$Dt80%j=`IfqYA2K#u(?_R2~ zmiutZw+ly_`4ZBk=3h|W-sT^3!$tbFqQhIh*{yAO`<=e!t!gm?=0j#K%cE2)JVeIj zp^GlP#o`Jha11fKnVpq6ZWCz3E+7LrZCFW3NoNcUP6eMs!yor=xy@w;Udo;>_XO4o+?( z%fFFn6N8#u)Y=>U`0){#zg3SOJ<7XY-oZ%Ak6=Rg?%sWAW@>6WJj8O_4!X)<7v)(r z8Cj7xaRSI}VjZSKQ80YGs&+D%E45)_K#~5?f&Wi{ltVmHVa}-RpjdWTyNv{*U>WqH z8MepJnygqPH`u@WZR5Tw_o-WW)4ybYk&2*5CeJPUDQ0AO3B}D(;#F*m(1@0_)AYd*8~(Ion7E z?czFc{p_nx7dNuvH6;z(fx;LvRDOF*4)tt+NIOc|f=64Hv~E3;a$oQS;_1R!_EuuOuFw7Wc`kd8I@ruOb=X+Tj{qZl zkTR+FI%NgOy19Bh-HZXl42XwznpF_g>C1)z%wn9YL>o3-t+Tqc+H38e@Kw09^XxuP z!~=Dv)?2l(LKn>cZKf04XNCbrvG|$e5A!=7ly>edgA4ZXJS;LA zt$52hix*zPPE*Y6*jGMs4yT&7b_|or+!#{OPK`k?$Bc+?)ULY)Ev8prwpk%x`Z2#wr6cwn(9tUcSr2v|0G(xezh(|4 zG;VqbTG~;g=NuRR^!riXr87+C7bPvfaqxVA5t^`doStnkOWkm4FvM}(et&C!`nSDW ztafTJ`+;9yy2D)Kr8f?ykTQgsYF^gEhii+2q%?5NuCbYoR{d|~PM#<#7jRH}^|9s{#YFu2fnoM5ojs2F&;J&pSuT4e)G4LB0mLjSmHVXdQ%BC~K_0UJ@wheqm8jHIOis(DF*3Sal#Zb!*fi2VAURZ3;v+F#{9+BX614?6s8A zE07trXS4?2qBr7a=Gk$^UHXb-B080jcFGY<3^sNixnUrmbe-BG$w6Fc-`x=H(Ci{E z)>HP#!<9|fx6z&NZWZ=zqX$;HYQy`wMp&4qzTEqR5W$+#n%ers?19=Y8S>%R>Z%=bhc}h7Zf@vM_KU6En9&8R>(VdT+*?dfu)|6ubV{U_+2p;k;>oZ6=4H+kP*7)ARoW z&Lf{goBhnNf7S0*w;kU!wDFsqag}gxr0JOR<7X{NQuCGn;_>WAa2un8ras3YcgbQe zVC<}#AX6En+9rT&6oilVUD_M4)8INTY%bwmC${I(*HGeC22Pz<<*`U0o_uP`Q5CfF zCJRvR(OS#FyR>v*4y%Cu%JS&!cRE#-l_sBF-;D1rN!27qbk&BdQbDT;>9^FPWk+tt@jw&#gf7sQt2h^6|>0ozwC|3&p|1Xc&%S zn{GZM;7>Gk9J0X}W4#kXwB6LUQgu9jS%e1s^3cUrtbg8O}^a_oUJYm-` zy)37<%P^{y!Cpnf%poQgvez*Q_aL76c?u~bm`6xCBy02YTG+Hu4&V8s&n48*am5rc z!k7F}5fLf-okXfIllQ^x^0EqvF_FAh>0yn#K=O>Wx{Vv_bbTujFt+5T$VClQY3L(x zyXnZ_8Z%7r0V1Y8(qW}>T7DZi*bM67iEhOZ1kJt@@o^7J|t z*YC^Noi!}l!HKVp4$Y=R-#(~3d<`aGZ- zWLLpBAl0=Km7r3zM(0s%te5S3Ll>7U^;pfYASs89UHYip0TMQUir<<%i@0qMPnfLC zSO$u<55DTMr$2dee6j{&Yyae;`LG)Ie)N?c7RamOzw~a#ij^zt?*2|Gt%WFf2roJ;BJE)w{3p%K%rZZ15)!I{8F8Stp`F^HKf%_28-dy#?Lzqe zF?P)gPx;BW{qAPf!8K6}l7ALNK!Qy^IDeM?y}T1H2WviGkl<|19H1ItPP)VPf|QR9 zqp^#qF5v1aK@#Eg>PTLThw~Xw6{J{G@ATY zjrdB`Hs5|*TU#}Z#i*Dm);k`SVIuE)z4bw);3x%yn!>?-rxG?Tz45Y2IP=RcusF9S znjLl$tp@~$s~mbG3R=nlxQ0XC$m=^pLIWHN!NhTYvtQqws}R@Y@f}!&c#Vnc4!E|y zs{(@k6@AjdT%JDg%}nwZf{!r`=-@g`NRxMx7ZTEsg7Gw zAR(P{%+ezR1<>5M+u6?an3!^)YL77XYC>#YfuAcxcEMy}8##nLcp4*6c_Vc@Nzf%!4$ z!$24;O^$nm)8E@?E;Ebj2LaHSc*Q=E9a%TqvB5BD9k{+YY512>!00E1g<^?fb41!z@Dl!A5N6bQJ*0)NS` zf>nyjk{-C=%U5~>MM6lIc?w7NXq|wF$!#N6aNL(KUkIGD+*6+6JYWU-L>XN75vTuf zn9(*w0V>-!iwI$tg0h34%H@G3cztOY-Xf!HuWXun;3EH+21s5Wd@}0lW&tH2HkbDo z9s{zI2@w{b$36gz9PVV`c9b!IlIaQ|sQ{k?4AVbFiYfsLL_x$|}C6Bkh0`duLO!Cs# zGdg2noGNg=R9&|t!E_hNj_wc>*n8F4Y%ir3Gs-z#$(sA7mYYX?`g;B0qQ%mG<=cR16y^*eaO zKKY&g`0Y}i>}m?t*3U3HJ6kX8cv}jAciu?l_UPh$nOa*yY)WL-J!|Kad>eW{TiVKj zi_#|6&DWsnvg_m&w)pL+50E4luVk`cEpr1U#CX>!9G;I zjmi%nLo@Mu2N&BrZ?^Ba?(GO@<;aMJ9sO+P=r%XpqCwV{Et@whv_;Zr zW{@-T>^%2dj>gn8m)nXx4+7OY%{|^oZ3|4>bQD{`B)*N8;BHV*v2#Y_zor4hz=p#y zVQLCVNuBnV8Qv5kK0|KqH}cEiEZQ(a#=aP1#7tAV!So_nBPJB3^y~2;B$;T&pX@Wi znuSI}3=~R=xtN2*+Tq3?*C?PO&HVvmpX6nqAHS_4AKeK0 zZzH-#kKP1_%b}Z9EJT7sv{e)o5~ajSBM*lAjhk!yBRQS}=5J^ZUwMN3$Jd|EH$tgP z36-+d^vhfLl{_?edx!9*%Gc#QU34obrc1-PULP%EWb?m1cD5zV>kP_WU6P@q1waee zZ@2U}?cSYSthamV4U-C*68Rs>7YM`lD2lXa-HU}AEKXJ5_|llE0UXSaCx=hU@%wqD z2RokAPMIvg=FdVp*b7e~EQH?3V>D6SLMgVxfr`^i+}{kFQ+akxr<#pTB61{}vAr}Y z2k(u>p467il2IIV9setY6$RXm(cbI%WgC}w}7uZeBzh&?(^uQffSfFtV)s7!u zhaL?ipIUF1lRegdIt!Q9*)T9ylCld7HF-k?;J>&tJ5~om%xaDTpL%`!TVP8ExV5+$il}T)Un>4h;g0Yzpy?Dq&Sh-8TrRZ*i5N z$k`G+vnP8b9%omlaR(QGHi7~7|2E5gVM)?9RFAD|7eY+Efv?ah=>5Qv`$ZeTJc~D2 zIqdJMbi%Q0Oq_9Kc%(*c%c4NgD(>4^onq99f8I_}EW17In0$Sl$7M3h&(KuGFsOR% z-e^f*eT6IMe-_2Ye)&ExtYOQBdnU%M{Xb7+J}`x>DL~TBEbwU)niT^nN@PmwS|lK& zv)ky+ad~<9z^5{G)!~o!qm%+^^myj$>jzD?l4S-r(|y>U-VKQ3Gbnp4NP{lWB%b^zEo$*C8IBF zRgWzN|G&%$eGQxd0}CoRMC~roDO8ctx^eA7$Q%mtfh-0vBvj4JV&GNEj^Cg{tq zJkW6tY$tXA2VmEEW+1Bw(R?pB*opw%-ngMB4iiEC=QV5A41W@KD>P1wi$JZxIuY~Y z{RJIq%$`7P(r|JtP!l&0Zk=4}{8^QDtPJ`*%VWP$EUhtkqMO`5XPZ_Z*F4^zhPFNR zu29_jyv;?u1!UuY;r%YviDm_01%SdPUmjx^IhGeQv_FmVsoB{vf?>o??{wwLEk$L0 zEaNg<>jQ1W*!Vz>cm2!JsdvDla2CZxrYMOZhzI|?Q&GYJw54)CcbVXVM}lXQlZSPg zRc+qht5RbNTF%W8k{VI4g5U}(z&|W?^7r>=qTy8GgW~nUyo>z@Vi7DWHfY8nHm)7n zYxwS+T}@ak4us=Z&~n2wzgJ*xr#U64=HkS>uK35vQgbL&$G>bVL@Mr^qC?|Lj1)kt zsRFp<=0V2O2>E6W?AEi}mY1-OR>g+%e@{@&Q&c-S0x!XH4oAN3Ny*m?rI;99vDni(QmhOLpa@ z*t@hd?c-9dn}btA{QaY$Tk1f}%g2ep)xol6Fv+h5s@IDjIapf~rQVGz=%( zS1z6-83wmWfv&3P`B#ep`qp4QF6{@_8tnMlKfNwztUf1k$>QnbCO20N1(TG%pK^!i zM!GHa4=UPE0l%82RStXe$K7sl#bLnhfZ>>FbsD=xlf7_GWyji8Y#91u9U8CSoZl@* zIRbHlKW&ZpbeC-%Dl@6<7G6|$C-?pBv#Wog)uYFhaJ_^Ap`vFxecSS}-JdqbK_532Sv^plq zms8=a<2~PSv7r+UqU`MgaojN)n%pH;zhYq6f9C@q{%?Y!w zT+tGzjEsCJW7l}~;xj(;=;Ck58RL2V`gID5B=E6_F(go!;9L^eSW57KGhQY*Q>p-M z*nz)`D8S`roe(foLqZ2rNmWGeVQ?k@u|cUEVg8$+w@N>j0M63VS$n=sm$$CBV`p+F}f0`X)$CPRzh&WbN9^yYlwz(ny@f z&KtOLoId6@(L5mthEVkfPE8R3>8< zaTmw?lkB^+U69^n!??Abc?6m#dR)yMG}}c^Xb1+q7%s=U9dcTnW>ZoDOygvNDW#dd z7Aa0I@N6ya5*@fi6$Zm~{p`!}fdZS3pX>-xzUYJAJ>v`9iwwAASZpKneZU@DBj1`Y zzqzMYVjv(W;EVTPOPE@;i1ZqZuX=&dH zIY~Le0U>A|d}#G}53Va|avib62zwddO_j8R^s60gP%6X=G;As5Auu4O*R^=d#$)~pT#60D{D;2J zz+0MzH3|RibcqXPR900*KYgkL9TphB{&YBH@>=aBgUCiRRo^zjQ?bG)4N=?urqE2- zFaUZGEgVOzSS1S!r*6@GVejG*ND4vU)uN`>-7?gIrt;>&Kgmv^9F(S>u&3aWGyDM@ ze)wU(O^YM;I~248EZ*t1>@ROsu^FqInRToQaoa9kEe-UFuNa6@99}A^_hT!fDmX1C zsc3sLe$(!7hmM^ z=&IMiO^6*(4oV4L<%1Lf`q2pJ#X{^8Z{tar;ol0L?3pPm8S4RgAs7tCyey&@!|DgI zmQ`Hs)H8Npi0fVC(K$h+L4&NeZt-zjs{xzLMj|CF12zJg4e*weWITjmajAxLTbM^B zdM`O9t{gqgh}(p91KS0QRE3&`uAxCpJi!^TWhgr}=0Qklm$yQr_V#=^B8!|DPMA0e z2}i7bi>J65pgu4x?x_P9ihFEicN2X}2hhe}pqEnhZAB;5)b35e(m<|a)k;F{+zI1= zeoeJ7hbOZo-=U?n!IOoWEodN$(*W_R9LS(#p&D=Xc%5jHf-9B-(Xbt)CG`(}Z_~z~ z3R?1qPMYDQN;`{iDh3fMj(O42(#ro6TVgM%bNEnLXL3!TZLs z4RIXX7Gw5>mN*4G_cc9v`Ri7YP+&M=xb2_>X#-k^-bNB*Zn^e>0|m{(;Nl4|&7D^7 zlJAH~8dB@VvKewdeFF*fQ~aODPJ>7svO2Ix=hZj|k6!Ol^>D>4&WkB=YpjZ`t$lHS z*zY;^Gvf+dF19hlk8!^_%8)vm)?2P9Y>F8<(GhS3KTWpzVFs?KoD3Y{y;N`2&1HpAj83a6N8ca9Fg_L>13P) ze1A-$>HUQ^(w$Lq0~tFA9ZtSNZKLUQRa4XGQP&)Nt->=fw&OlN{{k3&N)_5Db@4z5tJ9} z`>@r7!uR;iZm#4N)DK;Kw=p{l8PIK53vn(ebxq4u_J#qZtsV7}>Mq52O4g34FZX_R z=)cbFc(6m;Ao+Cej>=32Yc!X1gr=3vFJx^4Pl|?L4WR!6c{OXG4yhkyv+^LMi@^rE zT`UB1FS+s6;Lp%@&H;pWYqAQyw^4L1Gf8>NIy?Z|>#vGIM(YI$<{P^R0ZKZHw$=@%>d8nW1>+es|FehR*kkJXjO>uBS`A*k^^DK=}#x> zs?o@QU@hYO=xw^69nMf1E9PU-h9UNz^UKL!o?Wgye^ztbX5V`Jasi%?2s@K1s*H|V|az`1?;+)|i<*!qSpXQ#8* z;n*1(MxvTFNPr~uEwK!sVVs{|Nfmmv|P{s9P-L1ViQTUOgL4w49@eAv@ zAVEb}GXVt-0Lwi2OIt0<|MyQtY(&oHR{@PC8zuMZ9nXT#n5dCgcep(Y#+5?NVz8sY zb2U(-EsxC!Wne%gCJ&;LE+2=Oi`vDgA;VnPLoR##BmiqXfld29k$CCNm3tg5E6Lju zBj&l{VClfw>!+Wv4gT#oIIsjj<8YGb3@!g{Tn>67(Q^q(XHs&q)|Z$dVB&2s%-tMG zE-EUjPys%#-nf1yrIZRq1_TAzCe&gky)S&mb5$-onjR-2aj45F(qS)oQ+gX4=s_G1lC&Yy6|2>2H_Fw|%Ao$WXQcySNNtuMlNUB~u-dNqZgvN}31yQycd9oAgF7z%X>3d65VVrh*(QkOmv135!V988+rTrKE6M(A^OH z2Se<%Mgw4Z8UCR-1QgZ5^wWEW$3u_yh)4%da+=<7_^CG=i}tEb zr0Ks-jc=in#Bts{$ruyU{^?Jnz~G2S#wiPB4jZ$Yl~uQC0=Eoe{Q(4;LVonp41nH1 zqXS$F`t1UGT;S`w1Zu_N!xz$IGvkRxE5xVX05&suwg^)j&m)Q zL;g;!7mwLGLpsBLSQ>^citI*U*Kg607}A+N(ygkl?xzG!n4%p5wMN?`?-ZlrfNLXv zLv3x>8JBj^d&Ze%IDi~Wo{!i0EKnx!Mco4sRNpm)4d%1a|C^*J?qYp?O-K3Y-ofPb z`JRv3BkE@2HAxgx7cZoLi?ZQ7K9RC39Zp%c@2;6Cx#s=$<+haswdL6{?pMb!INnFe zW)puJ9&cjZq$%TRx`E92ph28|jvk=Od98>+D~nL@CGUsW*ZmKn7ANBtgoKm` z9WvbCWv;p_g$Cx|6a=dcg9C^(+VifW&C)H!K_f6X*@?SWtz1d$bP#!yN{7*pz@#al z2GhCyl`(@8Q7c?eF)q2&+bHuc`o?5+BV-78XUQ(-3S%EXKN^nxvK_Dcqrm}^_Xr)SO33E%##e;K#s&{F% z0K$FB5uA{~k)x4b%S|X!c;4o9nkkZ37om}KOh_0Rb-hgkAS4H5XY~9qoS?|fO@86FKXZ#@!2lnn?U?*$nkh}VVp4G}`MZaq))#(ah@zOR zgvRyl9ql;jOD9J!&Z{l@!}MqlmuGiz-fc@uBCF26?5JZ2&;Tzc5iS#)k5A^ZM7ymp zwhs)7w*qI_XalFhXF?+gT)Gp)klk4W#5`Mjmn&`qMsGlBcQJ<1^_{yI%=JPFFm;FwA@KO|q>MW8PH=G3oEw?A?r@+J4F=Yd2Rrr=rx66y?+zow<-ht1 z#)O8)f?{#3(&bBYjy^H|xVX6JNDB%c#w%iQ;#}w~?Tm|&c-2~JN$PQpu0x|Cade(0 zWs;r#>JHKiDQ93O9WuQw_(64d48DR4e6J3fd5wESRh3!RWc zsN2A!tEHs{QP`yyy#cmX6(XnEIV<==rUB;mmSnc}@Ms15p|YU)(V0P-kU9~~4&mP7 zo1jH+bekA^U09g7c@5E;O9AOZ6kCMcy4$53Arxl!Eux}T*U!rpz=s8F0^is_UK~-k z9_80Nj8+6m;JHWNp4tdYOY1@t+FeU=UkR@;J#vF62Y+s5hV%3IdFaJ2uVpBAig7Ytir+*;CR9R)`H{JBj7aAS*fXswmTBq98M0PdsXtJ#B;tH zPhfakxA->M6mUeuI(76CG;k*#RJA^=))wh5BLui>kXE1f8GX@d^jiVsB&G&iD9_+b zCPLG^D%bJekgZZu^|XVHMHprn7v8@}?-Y%6vnbgC4;?g6*anXnVf%Ih%wf=r_Au6; zmh-93l%iWr79uzSf7pr)ehLXF)Wcx4IT#=0EywWEU zY;~ce9ftz*Wpmmchx;cT5M55WRm#BkVS}Cme z@IsIG`YGFyXpTcz{lD^$CQHpcL@F0JT3W`DQZv=#l8eE~gX`(?;-);=XO|V7(`{Kd z+(~gycBX2k?oyo|vDRV4;`5zEN3Lq`j@cml@OBOjYAA}&SN4&zhrPS*SzY*Mc=|2Y zr|JZKH{}9&VYFd)^lUt}=;19a7BCDb$&P+Kz}bSxe0Vo7kaRLX{H-z=N;4gz??6P7 z28JdGP*OYq#I>WIYN)AwzZC(cNHhc#Fo~wv?n=3bR}(^?dhnS9&s`Uui(APrK&GhTX@^f)$>avVF#FX4T=t zPl3KY2x{T>A;@5e^-)nc>(uqLDW>+*&jF17A34neyjSExwOS_uTH@ou+>Nl>9$Udg zB&oy5@mG95VB;qgQA(`^q|kNQF3uL@$JZbIuxeY(TQA})fSWev>C?UO-4`>cMM7y$ z&g?9`LK0#)ArlGMF$kkW1aYao@J4npYV>YR`(Vp~-%o$(YWxDBa!ASgyDhAgIcu03 zF$E+1E6Bbn9pgw~9Tu_J6SF8{1lxfim;qoV-3f0Jd_4T5zS7R)&y2&2WP%@g7)@}Q z6ZSOXc_V6oBXibcJXPap%IpEU=DXXL?yicipUXa_ zEXga$*ob)rn|udk0k=&9=DFnV5P>ZZqk~s*O6fVmcEg&7n*X7L@IGX$!pa)x6DInA z77dBnAMaUd7;eE?q;6i}-oKn{4(8sZYi@I0Mc|VmTNQpg%)N2AlTVsVk{umOg|7>P zyTy(j0TJajJt)bc^`!y3f1I5y)1Cn_F|pm?HMutX@9787MT5)^NG{k)2-Rx9ll$sP z*@&Ti{KQP5LP|h{f!L9TjE537`@y(I;AEd;z)MR=S0qhPD5y}?f6rd4>XMzgi$ne8_mNn@oN%?ar>o((o zK#yVCS`uW#hFG={J)78X1>x?K+bU)A;TmZ!RPL((G0T+dqNs8(qvgD9Tk~3aMg52hQGql63!GzQb|gV$YI&DiyHZ1)Q$k9fzxU)O0dn5bTF>h3v&~x|!-YZ@(NtKAi>R!lZ=Oo2JH5yejm0y8pl%)mBTV8RP?IVRxk?aWQZ z?=NPcchfP?Z3T6YZbtG=z>p&eBKN&Smo4;Fv9lMfd|5Tl)}K(rvZ$7XP@Pj)7fI=X z+_I$uTN-&YrtG$`IOB^(*a84~Kddjtx(3T6eps<2yC7z?=>?3W{7uFPm2C=^jEaLZ zvwAIbJ8Rgq$cgdr{YC{7e%utk;PHV750l9CwQ$+MW)L z92hcM$F(1|;L;Uhu~x%1g&I9zYymdQIPT>qb?P!k2V%=;F~elx1WIpk;I}q;(i>K6 zRc`~La8Kj6WFz^iW0ngGPxAH#l3K$ZOoL@NYT8gxt`wFFhp3@aLuT`N1kA=(P_ z_zZbuO(>NRE}wkKnFFlC>;m8j-$Yw+W0_!pT|5p1JTB0U9Dxf4l~l6DXqN6Fl)m80 z-wV@c^JM8I@VgG4U1{_ih1UfGG4>=wj)HTnQ@^OozWzasGfo4?L*L!tU&;$$9~4}* zkgE|zG6s<@9k{Z~lg^oen-wX#xL3Mb0I3n_b;52#iYQm)G*5kvVi=i@5b^1=QAAY8 z_%W98p1~i{HC%Ah52Ryl>9HsX=^8TbApIT?12G%vG&%NuHU(P|HP*9JNfo$}+K$!# zc_Mv+lRHs^q5ZU8g18m#>M77&lw!={&w>^`(GlBt-!1*oa5X&viaQO80J?fyIqLq0 z4rE<`fRbn#P=mC!%1ZswL9=A9|0u>wL+G4q9n8DU+5u~ zHxfB%NWBOi7q-+Es%Y+xFBqcT<5s%4>Bv4bc|xRQPOY#Cr0$cRGz5lTwb2ZEzicJ> zK-?|+)vG(ke0UCZEu0y8=;1=s)C7;~pCan{BDGux-_I4&d#Z`5#3XKTSm^ER}tk%SQi}EJerB8$0N5;E;VrSSa?{S0>1A2!rwGQBby1QH~fIUym!Q5A1r-;laa zyb(Y}Y#4CX?ZYOQB)FI&2PFXmVb&;Du&^AVpxA-Yj?nN1eDW5L&<+V^_3Fx3#k1C z8VFlwp*^b<;~j*dT$q&OaHy03r^~8N7!B(sO^KtV36f)Cg0x{1Qy$cw#gfKKN}e2w z2PX6IGQWf~{+!5(DJNkYAQm!X8H}Nz6BVD3D{5~5Y>vdRfi?fEP9s85|`Q`s@eIl6~MdW6$6kA;Uf7q=z|l4`XY72LS*OwNP>>*97{2 zhoyLl53!S$?BU!2YR3j zP>Dy(LeoR%zsH$-Aj5*gm)J?K>7mvW=L^>HwAbc7Bv0(Ffvt}2Wk%Q{f^s$sZ)F`)EpBgq>jE*cs95ZJok zVt6mLC%HMB)-EjgJS|}Vfdt5DEc29y--(kTcSG80ueBA)@6zi1Q3QX@?@I2Lub7ti z&{4-b0BieJv|BK0ND1^>dt$N#E>82)F>iwwi(C|k{Zoz(_{b9tMvDC2z*_YJi~_yz?tI*!jaHvZ0+S_<=_oLVBJYzM4SOo zUtfR@DJB{*|4HCCpa!39CuYF%NJZ%UeEO$wP(?IJv*t7Z4nZW;2O5!%p(jiOW_}36 zQVK&<0M+5lh8G8lBe`lyl?KBlX%#DvPYb1~nlKZ>oD@1)3ZXiuL1qP))ebI#pHCW6 zflNWcWqtK!R|{yD3QPn%8kz9qPytfaqix>)()^j^S?DFy&<@^?z!iJ#fUcA@`>I=e zNmkv(Oeq#SNlqgXKQQ7dM8yQa?2vVjgPuF$>K&qF2y7;NJf!G9xtro8eX-BG6V+@|BU}ftY|3`R{kzxXT+z!wx5(YM%9Wz%T zc(N$G#1P5w`q)`dpFzmWNOIj-k$8{X_v((#k#Ts2P;3!hPtb8G1}+pOU6)ewZYY7i zoGXwx`EH-OQXe_8{fptGfSd1j?uYH11w-zy5Waclc5fB1dYBxj7&bPc_ITyAzra?< z1_iOPi!?m(Mye`3y9rDRaRqgv>nV&75D_#v430WuAZ71kWBZ|h*%sq2rmCgIX}~Cf z|I!z&l{oIuVBvb&7y{C@!2q~^_YL{lI-jbF`5PhPiGa#NsJlzBZYc$b zC?IF!{hTa&BKi7t@4zJ20X2_RudFZl8JxUM1XJ$mQgS0+e9);-KUC7RCaWHp0B#KS zi+_IwAA$U)bkPNob>{5F#ZwE&`A-a1pd+ga3f6M>bD4 z*={^FI_{@t$!VZ^NuDDL{=SNdghx=3f$s zhCR?&Fda{tv-0I_6aKK;2GpM{BrF!8bMebV_Fg#AN}&|PFdAZ#cXP!&Hu41Y?(1B< z!F>atnsyCAe=)glY=|;geH{b2DDsJ&|F+vXZRgw6j@3h}g3v55}0c@YPwHxg0#?-SS^f-(1e=vrcQ= zsk{|f9;WCjycR0XZ}72-K$NpGM;(=h5O#>^^{4rlB7B0xI1qzpg!o-I;`xS-;e#39 zdzv%VlF+CCKrsuq9|*)- zjn@wd-ee7?l@lj8C>r*dVo41|+kon|K+>>%o9Aq-=##7Ac4Em!8NpBEJ}r>cnIs_yfFL`~1ROB5(ceiibNq6s$*PzU)V%n;c8p_g&{k4}ON@w5@V2qX=`I^>vo z7UTM+SZwRH(`h(z4(KVI#H9Z`Ra-gXAeu8X|DU)d^#2;Y!`W7gP2de}$AD7}$KVo; z=?qZSnB5R;U_(WQC4dzCl~`y255nw1rVFR6#1qGlhayY@H45>cK*vaTqk(eWP-$Eo z5DAx8!*v4g#RfWxrr#E7m?A#ER$*ettmGM=F?ZM}!@vzeo|m-?}@wAG$f);j@Y6*2B5O zp?wkfhf*G``DYDtJQ~j9!PEnbd)xB$mY%L zW-5Y}U~FDfIyQ>AR~={ELS{;U!2w;$1_%1Ro?iJ2PXJ%;)dpIA(_z8$yie0>9@-|p zp^~6r2=+N*nxn&m7Fm8W6I38fX9w=jeYTlML?9+bL@X4r*hq6!<4(YwLqaJFK0`f$ z=>b3VJo5{p8dg^EC~)|mGDpJ{mSaw#AP@gE6nFOf@q4!uE#73EQvvnUu8kI=dg&5@_1m?$6+HUi|}r5UDHCwMgOfoMW!8akvS9vBlN}G zX(Btcf{whPU!}FRvHy%>0zf$NV|>!YL#3bNdQ?ft^6JahZU_{eFrOhb>l}j^Z7|Id zm3DX6^l(9jQ4)G2w(&(Yp*1*SzTWRdFodKm%_+BFQ3z1T@P~r#Ss2hw;1Xm-tqR0mcv^PkE$h6N9|5<1aRxqyw}O$s zk_$na)3p(pVY1#_3iU5yCc=w$@2krvk_e3?9ZQ&e@xnm$w1mur?REEvU=){?Y$l4h zB)=bazC;|Bmd}M|#Tkb)vesNjz$F(DKx8}An(URe`>KbwYhoR^FsQQnOfAVWZrAx_ zXTHCgYYJ};U8`=c!^Z(F+A4v;n3*bgqA=D%;tfjN7l-r#5#2!JEE#10?+1a$AVId3 zY(k0`jIM}8xB|e%zm$|9Bsic1RYoXbp{s=R!_nTAR!RWLtcLO>37K{+ z9R$MQ7H@d=_pYGIDc5O172e%GfVcK_ytb5-cxWI<<^z^Hg}NM)dV($wRTDkP*G>qR zpjAVxhJVNX(%zxV%ywE}B?@ByBKwsP@xlCDfrAM}oe#e+9D)QRgtQE>tVQcJPC3F~ zQamv+p}IxGgf$6ED>S#t@?yuDf~AR4@ajwB)F1w)k?u^AAJ!E-o3ai*Mpe+YRuc-s@r5l}vzvkXG#tEuA?=jY+vYG(w#1q;`@f76{b8 z@O(D0>HpAxi+B)d{_@idN6;LLhOJ5`)!RLlcO-UhK8B>dW8a0=-koC=%xm5>L* zCscyl2moy(@nkZz`^}q=&N!_*pqyr%(KwtH zoYv@~?U`?A=Wjr>3+Q-Km{EQw7HoZE84Jk?IHyw?B-N@gX)VH^LcMg@{4b||x&e`9 zD7S{)a@s5&Buz$B%KFxyeA+V^VAkyXY_r6*vr4n}@Ri548k(L_k7+zXM5(0ng3E!B zABu`A$$E=*;S_+sM<3!J#k!vQv;+*cRfzoGm^JQAc&|k=gX`k^D?-LkK_ONwDNwo` zt)D2FKr<}X$bnN3(K>IqX9ZPa@S;QFlV3$3Ka|0uU^$B9T3EO2I@K*3(s#CdS`+IQ zEQw^QBMaP8d))rRfaWM88;+1`!RWO_+2uMqxNyQQ4wq1q!4}4^Lx0|hD7ciRhi#>$D1zl7duy_!=&fFeeEp!jelcXbOg~@Ery782n1f#j@TB_@l53MH9#hB&UY# zP@zgX;8ly`@QuojfAh9k(4#yl_)6Qkf`<+b97g|mh=~OI04X(wtvoV^8IC${4wTu3 z(LDND97M(wXp2%j!N8n?m*4eAEcdhzTJA7P9VdGe;G=>-T11Uct9X+RpKc(egB*1% zqOiMBN?Y_)y{(Ue1G66-Vay0k?l3c7l{G%xIgCpX4SETt0S-q=5`z_3Vf7uWPr(@k zmJ3TYUid1WdeL;0wXCLE+=_V|h{dOiiH>V2RQYDNi7d#0?mkH+_6L%u(A|!s5(GaV zOi8}rlWvz5y?Rv&r7Ov*5G%%&S3m87QKwas|JuoQfZzKtzOB-v`X%0u7La>&SbkW^ zi=4(gx6IDHVct56;DwID68i#u_aI^sod^-fBvR#g8IB`It)@S{@7yBk=J>ncK3!klsy2yUH zOyx1a6m{M`7bC2me#)zGo|hd64iMB1*Af1bz%VYRVg*IQ*O z5xWiKBGiLe4KcZDm3uPQlU_^WP>_#+DCO2uu;<{8i#B{*^Rh7O7W?l$j5A<8aYl$p z6^Z4roID^F1Jm@?PnBRprV~ZdmI$2D-ru3gI~5Bew}3+TOkGn0lx_yR^{=^ zVF-qK(iT==+DTB{o)Mu|(m0hoah0Ua6N%;G-!WdtPpX!Rig3?^C(@s1vfIk8cHXCi z&Z^IG%i$P{*^20S7a<7qb%VN8E{ag894RZzK>HJj{BzPi!`c{>Jn1By!g*}TZS@j1sst!w~+qi%3d>eQb720*=`1BY#r8=*sa z@ZF(cxz|_oKinq@S}vM2&Ca^(YgPMwn8a+tn^G0qN{ftaM-rTxkr6Vfh)o_Dhk&!n zTOk2Bh?Gr>?}vJ#lA#)bmvFNf*mDAPH<`80E%{6UI}+lwlEc9w<>3YQ{2=o3jw2&b zush$MjF4DO!ON2kfj9^p8Z>&R>*oKiBRjEGXx&BN((G=2<~@t3)7#Y1r9cxAqnyDjAD? zT+$7^ndAvW()e044*VhkT&21;E}@kA-@{!uxPii1rwfqReO4=iyOqg`3b2Imwac6= z7SBkD!!mpV3G)9I7hwFwRh4zC;O2S;%Oe|>xV+B*2Xm0(INixhQ=w`4H8{J~>}mE) zrWgWk{#V1TSkeXk+4y&q6+(2P5@Qqjw+gByHgNZ?@WAAsFDBw?+_j`fP)JLcV$?dq zG!gLex0zScI~`BIK-M1utUv*ZISq6dGoDjQXCT)_NGE+qw{!G*5;;RELVEvlG+b>` zD^g7EAXC_(&{h{^d9 zy*%^LjAw86te1AO@p6i|z)do_-79Q0i?xJSk=L^{x=Q3A`+Hge^=a@wqLlR={vDN# zSDhIn9>vJ)v2;XGiN1c8uJ_U9`$q*LsXn=W z>5VFqK8M@-;1i-4_fCgU)^4-Jd728L{1f+vmIB#sF(Viu)HA}I%L+mtfy5*v9`JSl zlxDhyFv)5GsoKRGdzD1G0rvrB=U1wzsHh6<<#yO$czqa%=18$_mLZXh*gO@yVIwA6 zCE0>-zl6Q=G{x(lh3ZTqRwm`VxHQN}=yu!143b?k#X>RJ7-CCKVBCxq0|wW4A(tEDWjo#;C?w3F{F;{q^C!k$s}DTeLb+oBM!{&| zReTGfY0%xiB?l6>!E^3=2!x~taS~&^ypxx-xUU35{_7P2GO|2|apB!Be+Jv!`HEdU znXlQ$Rw~Hi*-TuuLi{|)#Z?8*LT90Ckd<(=S`iq`W3bFj@M(+`Tf06Y2n6fobX3Lj z^b;ByH6QS2DC5&A#K`6eG0N9H>nSO|gfRyYcT{tg-Qd^vB!q7$h#bf*n0F`b)Wj}> z57xJ7YoNdXW?&=8o{e=jtq+8{;-@bG*Mr?_j>C3mk<>QQB6%UaW#ZvHiaqZG|L}TN z&s1^=-@{z9rw4?{SqxRAi#~E)I|&MOK&z3QFhsjIeW+11!XQ5>cSBM%v26g4vi$BOw+r}O8O~1SpwPR`sx*_) z!Ea)b`)$G15j`Hqzu*Em5$Tys(yFNl52ztbww9KEiYz!m5|rteXD=B*^1EPt?J~ z!rmW~8Tz;k(NlkhxkC3%;s#q;3r!3m!-)ex$>3y$;?|5HuG}X(S73!hV6!x&ewZhb zo--QEp}^z>0ec;*_z&yX#X8W=#bCsy;$SaGaJKLJkZA#&gy?}iNb5_FS`JZyDST_g zn07vxkzJTrlLD7AI0;iE@5>?>B1boLobDaGw~zw@P?vyho)vLJ<)<**{VxiRqG*S0 z_%+5$l9bPZRl$G7;Nkl>-t+5ai9o!DW-!Ojt2y3>eTHbHAXZf>bpJn$y$5)Xcl*bEqpI4H z8byVmjVdLyBuIo5wf72KBn1iGO|S=cRwQB9r1M;|>1c1jNwZ!cPuFO_u>`s9@v|RqYIkE5z6MUQ+S47@?{{`#` zNKR_a*B;sJBFAq~6}i&{)G$VsaDnu8Xj`RRhp8YsRLAcrTh*ZhV++P~{V?iIrI$@2 zTQNpbEm1)A=e`&+WKYcywz;?%^rW1*^V|*uN3{COah^{{j@^r?ABPN-*^1rTj-?OM zH59bVjM~8Dap?VuJNm@16iuA;^vt@oD?Ifs8yO%RUQezA-BoUxE{m?&&+uaI9yhU} zt1Zj!;_W&{OEGT>NSNBkC%MJW{lr_4S9|<)Y&i|lC87CP#N3+rf-4egA_!xim>!|5 zp=_@Pjh#D`7{A(4bVvgxP~z0-ro4 zp*E(rQwOi$hV;)`p8;Qd0Sr7;59sKL-bs-nC}dIb1rEEVHQ(9Ud3Td0Zm9oGMBK*u zRnW|8c6n?nE;Um8fX_b>t>&?j>wV#yteHK`|06*qm8tPti4WTR;7ZYl;-0y(s$HMW z7jn&9l>`3*?MI8p$b1e~HJM>_A%72du9UsJQXt&_il$Nv{X#aDI7w|}wh)E1xR1rf zHAQEvMt-d2Qq&K7+1#6}gIBd2GGparI6C~+&54POZv+OV62`+$6Vx3K9&kVO82nOe zTn7yu8k(1p-^L^9>~Ft~m&{B|lb@7GOGgee+crV)g|B(Du>zUov;xc_Q+nv_M z9XnX2E7#*j3^hyp$W`rR%wBe3Re zh$@YqBc>pex2>CbgzZ=Db=7R!5oy)Y$g2g9IpKc2!$=Ff^UC{tFeJPsT}+|`K;b-Z z+*cj{l_o!I)eeENb8`OpD6GTv6u@L%kQJTFeaFMzqIQU4WZ3`pSx0_O97cc?_iHD_ z?R`wgj61;0hm&0Pm~DP>1H9%#%?Ra{kvFI(4H)3J`qJbN!owsjS0^u~x88Edp6YPS`I##&4Uu(DXND9WZfTcD5VND;e5kFj22}Un1oCvip z&|s;BHp+YsBMbA5WnUI!gH^V1d`p?ohA*Tpk6D&B*;AjL*6g|OXiiBX@s6n+Z%Z34 zrH<>iNMS6l{NsbAI;+LK0+RU?TZ<;75{}tx4u@UnoRf!>2qT9`CMq$n!0rbSW5^IC z3xd{OP|F#T4#uG2vp`mi)LDY)VF2Oi$k2(-EQ3)=wagZvf7rVL|JfHz5)_b;6Y4bCFfQ^gJWw8zG&GdR&`)cSnprQ%gbWG181HDz!!de3lnNM?~ z_vL$W+r`C4*5@V$I&Lde{(CSqis^ub;)Oly^lR8 ze?69DmSUayVrF^RA{ohg0f)`}J?;~k0^poq*8ai!43D2YSBBc_3>YE9xBn&H7K21~ zxE5hGK*2G`3X2gvT`SHp{O=c<`L7}Ov**w64ErVHa1TeB zmZfRYfvZ)#niMjPcA(!~L%-fXHTM&J3TfSGRpg-zzPrnH785S4-%c`_$~qujZQhbA zxu+XH(>dt#3uT;2|M>4(IyKJm%_5-ve?=?P*PA<~PwhtKJbhERl1?E9t0QZ7=5GRYNrN?{OI#?0m}pgK z`NEEIo`GQ}y=EPye_jUioMuna7*sOgD8w%cQsFT%wTi5?wWUd#9gSM#RA#1glgdkBXV|W+Pp;$jeQaGTB zrNRwSUYMEL+PFny_cK|Do&Qn}Uw$CpkbFE=%e-&Ff>ZpYsZEO?hO@D}ZA9oscYq*BWV!-wl7RZ!HA6pu9(JE617U9vveg;1Ym!AkU(ZPUIbPmjH z_E?b7hQfB;YE-zfWc!lTae5A;k!O56e7HDt2PeahN}OaPE}<-cyJa!58L1|Ep9&l z9@$@R*85Q9lSg6Cv|(uf&7Q@KDJm{i%>wYt=|YvLjVXazp#S9NQAKS5SkD0+q*O%eMH{OBxTxY|2$o^UbYN2d*NAcL=gY8_G6o^NeqtEm= ztYG@U30ToA-g%;Dx-r&N8j{aLGp$yIXB;f+8Jv&7qF5UboO=s1d^XI}?~b0?+oSK) z^!4ymYdal+x_PWjp|V8!N$U)iW$ao$4w{soc{cnr{OoZLu*&^%I-w^AQDXHnj)%?V2w@`Es~r`RWMU}paU(yT z0MZ@+&t(kSh0$YQ6@4M6Zit?gD=4D|KUDco7&bHW*;_88Z%MCE5+K z!)a5H8%3yjNl6$u3nK^kCtkL(Rl-gYL2hAHl!Lz9wtV6a(algkh{q@6W$LQL*iS`S zD%}cA>(|$Z<@EhV@%8mUU(u9sj9U8>3tpgK`>o{ouVPCG^IM;8|^Vq0$ck%X%jU z^4|5w7m#F6#c%;izj$~?B$Y4zCp2s_`;wEAwb%RRnF#};bhAdg z4`2(kJD3^V-mTtR7T48S!JiK*3X_z_FMiVZxR~0r&x}wtP-=U6qWMOwPb@*4%WGW% z=+u;n0Goi?<&BAz&FwZ*nif0SgFMk_OjdkV1%}|?fz2uc8o8z`h)7~y@fOH#pi$Or@7W1uqLaOxmfJg7IE zu)Aqm4ghnOY+-j26K!s-mS-a$M>C*t+IEpj?jdgdY1O~+DF7|hZJTXdtA<=??wFKW z4W^z1T(4R+Xar^i65PvaU$OO3jD5ifY$EmS+a3c-?vN8>zv|ryuSZ7u%f=)B;|3vm zYIX*Gio#3KE-pAz?Um^5YU?-@XKE#-Wa};SQU$ideyg&N>|GWcA!cNtTPd*QPf4R5 zdHBqIay@0MIFn@^*L`TB({s=H@bGzR4g%#DltSjfcubhDStF?NBRR(VhdG;<5Ky_~3U$-2s8yuKTR&vz5K;Ej4ES!_tovu5V_; zaDhnGW$ZX{G=hxcFR_6F$*87O{CN3XK;5F9Gb39WZJ6?^_=5GC)6QjX(7t#CgLUHD zs9rXOZ7jS$H+@rM)AEJU;S1LE|7pSEd7Zj`VmimDQ>%mN9p3x2)2DNDFGj`J;1I>7fWhbLQ*~%1n5W0l#!l)9_qaSchllZjh?ZL+zE_1zqp*?m_&+8YC!QO)zl56 zdI=)UOZ@Qq0y>ClNY)a^`2bsf>ly$3h&D~aa|G+L_clOY{Jpd~knHyqtF(nOgIAjc zov;gzNz6yC#JB@DO8-q+38H=SZljs~lRw%^g(-BNB0Siv*u5TpmE92(M^awIhX9jO zV@m(uM+@>#=ON@{;c^KPq2ECH>gAI+8E&MEW2ANWYHVywL`@8Sp8Z=XI#aPUV8>0( z&v(@!b+Q@hc23q%<)t%cKfzl$u=7ql_OCA$I-SsfpRcxPkLDWFTskC0)Xs=rkA8R^ zsOr?PYa>z)cK!6z8;PF4F7U<<@3!fva@2q?VCy?rBZn(8%0xf=u2)XM@WSBosAn}r zOWwMK-6||K`*U<~Ih>HuWaExAlv_#ZzZqqaj1f#_3@MOk^qcHr+82A#uX=kq_5n{6 zn*iX*g4!S2nnz4GPQ_)tAwHcLJBAR@~9sj?oQPmF>1iV*MU9D_qJAD)I7fbeA$DF8Uwm}{-op; z{jbNz$!|SAT9n&0J-^mx!B@X~x~lZ|-g=hesZ)OX-ho4gt9f}ZBY%)In|A#T;u&d_ zGGwT%_WZh`;lwP-Kvd+*@F!+~8;14Q`1@y05RnK-D=r>YGHX^7Q%&xGG>S{`$PH`` zC74~4bFiH@UOR%Xv;?W#T+Cr38u&vbUXH4z^6BbC`dJ|bVDFutbtEQ64nZj**nX_1 zWJ)#54`0VdrY}^S!c)Y#(74Gw^?;XG>h1O|C{ocC1TR=({2r4-zcm=pgK7d8f05b9 zp+lQxFXF1pI6Hc?MO#(m;bxsf*d{hR0f4SJf7y@5jE!7Op-+sIlt7A>OO+iWjd)It z4BZSvYESlQn8J`ySl_jPPs`WQT!zJFmZpQKuUOuf@B28qVUD+EU!I6tHpNl3@2t!) z`&VyFFMl*TAPNrI?aC;11Ax{?MEYWHgPOA%=)^g<;Ww!kg@&}5V=0ck#Ae2W2(C}*3}J7m>wj9Q z?0(`@;f|VpL#vYZKMr#g7P+Fa;;&)$&)-zXt*9<*w=pw4y_geZ#>+>Oys*PL3#5b( zilh55ZuGC#y@Rb%r@vkr-TgzA;{c5gx98@6TpPkaQKftEX|;uYI{nhIdRh42ETA%WOpwPKJ1A$D8V4ljLNt5txO7Ma=Zy zym>P#xy_rapv+X43)t+K?r8>QW@d^Ddj=l2L6AW-yF3xAFrl_!$MQ?Mr2nnF@u9S` z)*rO@Unpr*qKXtB58)QPlE68BP~cMA*zZ|l5j9pb`CcT;0wS<>H9o2`22`9O%=Q0E z?=O5`;sfMbkvGgGw%8ybENrd>Q78ErNN~CMfG7AVx6WTI+R466PxSD0qj22NfgGfD zJ%4_XJl3y<4o&bc^~JbBpdj$Cfm1Q$kv+6SXy#qlp(?v$;;0Yza&IE0jzSbxJBM0y zU^)wni!H+f_bYqN4Wh~r+avM$;-%D_-}<)rs4$1K&);$A;KA(hGn$AmNDuY)i*;r$ z=pVpLn$}2COJ~v-ZQT6)Qq<*)Wgi*tZuW`_%r2;^F>z_!R*P3u-2>&8Oz`58P#X~> z*TBP&$YH(HB|qNs?BnSjv3_YGvGus)dUDpBgl05mk;b<1Y@hZgFI~EHBVh+~e(46q zMp6WE68ps!N?NfZ-CC2_&12EdiJyzS#E~I&jEk7N2|Qnte%JR0m4_CYNN+P;bPbZq zz++-uHRH}REq&W51!Q)(e!;+bUV9RXpw+Mkbap}BBms{xm0h?Evtv;+-7@8cz9!lK>p_vVj7ni?{_6Rrr@VOS2Rw=RG1tmM|mi+5<6Cp0y_R^Jh{ zUKuz5ezs@B`SHcd-lOdLhG12=nzXh@XBMd9c87<5E1ZXGKU~Oe1pJf$!hr(@1nl46 z(9p^oTQiH=FK=^Vp`9_q4P;9|VaaXzvjvdCSJFIh)LGIe@jyxki`5^E0$BaYo6Yot z;d#OzHs9(4*aEM7{rDlhHr!uMa+F)8v^4BqLmH1jtfW)NzG{DR;d};02fcnj(_Q7F z11WvSxHtVOE|&_;H#*PRd(XL)MN&dAR|n*MQi3l7UL*5qU>F}h4VT0`YQM00HCw)y z&~RT5{QB!OWJ8~L^RhR*%1lr!_KMp7j~S6)v~j3z=J@25&4Ptpz5PayGY&mnxZDD2 z~w(@oNzZ`z|(HK$p!IAnIw0 z(97-M#zB(*N+`GVNqM!&<2)`7pE-a1r5D?YGE&=zP0%nr#nvKD%yA#x4OizMbR9|l z!RW%)~}?)@BC|TB;R;Q~%`LHb|1EQjXJ=Py6I%W1fN?%ZbkpQbZjlVo073eenrER?g?`s(+t_?a4gaf&c>npj)h>~NyqIIsBwc{|Ai#S&4DMM-Qm=dG4bvx$bN`n!rFHrm;`6{w z7p=2@lYcwg+d7onk~H0EGuPOkHsfvMJ^K=kbZJnbDay`vTR1CMvougmstM=U!NJ?Q< z&z?Oyb6jJ=xbSNmyuQYFLTDhuAg*jSX;DhLR<{e-i5|fC|J0lIrWog+=a1*8;33u#MgRu^eOj&!rb;mDq zJPa6@ubgrzQ)T8PN{Ns|YNq6+rJu^<2 z;d?)SCpt;9$HsF}?@0cQ&8EH+(dvpO{g1p)46zOu%}rW_Dsi2)^@FcNFDaqT(oqT? z`RwUa2J}PhU;e$~GN`Qma)*eBh#E{Js5m4hVWT}vGzqz4T4ZrCA7&S^a}Ad?;;nb% z03d+7;w&x3g972uav78roLnQ*TaL|Aj1dxR0bb-ioOyiIT&g0U;|fEDoAjuVsMIZQ z3i_mHW%UT`fRGlJ`VYJHPKt-!`#)oAGMAM}jfq4y^ZX2XU|36Bi)*9G%i|zaw!{!; zx{Fd+$I$~j{QE|E{gl-rO8e%V2jm6d~OZG*N4$BvKU`1`Ls6 z@@9cNFIBaj^y}bL1sVLqS%&}o`Gri$_T1KgBwl1IZ|uq%LrpAs*n4)SEQ55#$O9od zPFAZ5?&+EYqgo0SqX%Lq85WOvJg_V$T^hU+MJ5q z`5jE`&)!r7G!?y$oIVk|4)C1MzEi!rZM-UPoY6emhPxdzie8u8tn&L<)%&Se*@8U7 zuL85ZZE^D!eHJI0rkCz|hLOTD7&5Ybs7cf`NmEw1OTr+N2Bdnhk%OoOd1$2yubVwX zlIzqRo5@OOII;&8Sc>M9VSaYsRz~sQe2VyjLFU>%Q;M+*=NuMlpc0ns{t)i?v`S_P z53lQPvAh3$_jSz7$6Px55eSwr=uuuPT<&B9z4wJjZ~HLDxxqo%N#x#UzmQQUfTw+w z*wR7Vv#rtPt{7GbajXFBD>!b^zloKID+EWs{(egNCEgp^>8;0Fr1Ui;a6|9Q@vfr< zcP6ggXUwK=Lxi}+OrYeq-8z`v?RyWBx$w9KA@{VDfHo0YNI0c{nN4*~jmv$mINg;< z85RV8U$swdrPBhYC+7;79_((o)W2phkwV+=KGQFKb8p%7aT$Z+PH*6%O>u$(<-P9qu>(=ENe>IGofn*RZ)*Nddb(w<3M{yuz|I zW_UO5%)zyrR-Tpp z4343dP%EhoPB^;^?ce_f(dF8lF%pUvQRl$WYn}uvLDW$<+qJBG-+b3;w=`$K>@fumAOJ^8E=2 zCx!7V4%rb{6AAu#h~cm-DdtbVWxKO1Y!e!4^$e^3srHOoo7nU1@P_8Z3~v=GbeqP- zMWYzY%7y8ze-xCH^Lz`bE5DvlvXdt7eVQ^+r1Sx(e+s#QX-s5uvKZ8LYK_)1&$ za>qD`s6hmXxG`VV z;rd&U_O+h5saAKRvepKffAjGdw#3`7_4M|Re){H+v@YHj>NhJgMUvJbgT&;(xOalI zTl3%isKMmt@G_GVg7dWmNa%g}pWf(ZDHD~6;t$sNdq>@^amA}sJ^$uKfcf+6`9E`v z`hEsJ!CPbOxA~u`rg?->Lt-~KM7Kql3ib0;@_2)kJl>fRrQ?Dgf!H>KJV~3B@X7n% zqfwj9H#BF}s#S_Hj|3i5f-A;6s>o2YTcuHl3PV~IeV0_bd89m&m^mGH`zUwRcU}63 zV=nIb19gwQv1oVIeckp6CwIuYGaN%i2X*fR7uW-$*WdL2@=9> z2l54FJHVI^=KQq$`>$)(IW1e@)wW{(4lx~D+_m?PoPgWroOR0@l`ynjwXq4tIh`1? zbX$Jw*@kt@;5m)OV~5cqLyLCW?|bY`W914BR-yDuA1~0R-kI^P=h5nk=P?8STj>-8 zc!EDVv8Ax}?J@)@DR_*ntm0~mVH8EM(jxt&k`eaY|H!Q?2)b^u6zeQyVa~FpAvYE+ zllIvjcT8Q^hY#OuE{r@aX5Cv9qij{m%mV~M@Fm+PxZa}#12XC-bAs@sD%9%590g(Z zGGeE%y2&m$jTeQ~clF84Zx&+p(XtiZ5tJS*klu4~?#GTRaJZ9^f*_k>@5EaqT+@|U zoEKdI?U_G+e>2+SKuQb*t4J_MN?Olb`-hR$7$FLCO2+tACpFpk#chb4*iX@N8P+{K z`Q*uyBn@FYJARTDUY)sqR>^`@xm%(&RAy$j=jVHNsOsA&Up=zN&PJ=9% zQnn`nP)nFZllJsZ{OAO1{zfJbLi-g!;tw52OT^>OqJtH23M8_Gl{zP<$Ij z&6&!kSf%6uuR2MU2;=xoQjen6a-QJSaZ`Msgvo=`?E?^veWv_Tpqoh&lQlj#*05cC zPuO!WetaUeq4^L3+bUnA6jO>tZg@70-C^j7OrCchC6>66_X9jvrG(WUKbv{p%LF;K z(T@0&qo!Z4N>wv~`raNr*x6LJM~_pH$`QQDp_fsnL;TSB#!mwjAvCPaJo_XiJPiSf zO`k22L<7y!+V}9`07=f{V7$1d^|AS8W{TV>V3yCWK#d;5j0DOs>ABlDE)F129j7I5 zIR081BQLco`b36^LT;?n5BOTmpd}UAE~!)uS;b`vxVwuBiY5jV??SaFG zt$1I>1SQEON#-QvRt+3EGP+6u2;@fgZI#Rv5Yhf25~o%|c^Tf{6T#-$vvaW9hJOGp z_cy|<7MY0`rcAjhf^N9xtKyDB^D08m@2Pl{D3bk7`ohpWrfXHr-ycvdG9F>yC!F+1 z8=w9Gq8Ais-F+Kkj$I)AU}-|**CDm2xV?6S zoxH*R1knWM4hJse9f`hVx};ON^Atl7(XfEOK+Cx}!9~^mN%I!{MDPk34oki2J+GC= zW%c?}5?u<%blypT&?SaJ#O*4HyL)?&9NDzu6nUtPDzisa(vcV{FC@FEIs`cbeWV$} z3v~mc?~6#|MecmHUd5y1{J-?7|4cD;pDjc>|HNM1;=faFE%n^G5jAhF%(om?|7%C1 z%>8ReRi8XD+T`yYO;}N0?Ef=Cu^?c)vj>hI{mh_o#WJ|GC_tBnH~7c4`%cM$8M`@V z=1z|Y1E-P$K>ati?)Vu|ROwIcwElYkpbkS?OUV_1h#r`aU51EZtf#g0kEivK%8i;Y zzpQR611s?CG{&iYNkc8O%HLGjn_{j1rCLh-RO-!juj?F&dWhdxoKq>JfMeyzTZ_PnvX`z^yGpMyw9cgVF{&!`zU%SWF$tv;HGHpe&hgX*PDg%tXn)i6$TeG>8FuMoASz`NQT5?acYF=OBs6tbGE zKBfn6Z4%Q;DRwrG+h$vcsh5!ENf+*8jE#e2=!x*8@gW!NpKCBD{p`q&i~O1T=>F{iMpA84^v87xvOKH@X?k(b4M>vx-S5n3*V$? zQ@%D+dKi!rqjeVk&+gFu^svANYHBzDgr9B)mGuiz&0uV#JXApi2a)J=b3xHpVs`e3 zLEhF^R7AhIR>V#|ch0(N+bJ?~R1&?O!{$tMnKhU z;gnWbo^F*Gm1T0y9HFoh+DwwJb}zrsjr|HEFWxfzS#E}p&P3oMsJ=a-w||RdV)mF9 z3QKZEMm@nY!@>6T_3B(MyFA%Bzb|u9z6I83DqQEBRa#}XKn#P{~~};jGd?( zou_D2kCA~Dn|Nm>fsh(W7GGiaofA1()v(K@bLUDczpr%Ee?|-k8fHI-iWff~ICkug z%ZJq#RAfBNRDj~n?vJJbo=R=UKrH)!z+gqD{PXn6zy&6nVFn+nA`Op7rSO8I@;M-szTBIwHT(GIB5Wce{45+c|i>2MWfvcgxn zPEw-h_ZnqqBmgp!WAAcL9P>Cs(|C+Zry`gshNO>X)=FItF)%lO4fnW|(-{8(KPWf|TVPQ8i zF))Cj&P-(Y9vFEYr+ln>!695N7_z;_hfh|t%ge=IC2GU7J26eqyu;*=zQgh>+0;!vNd268lmV_!`IWsy-#WZJCL3Mqs%Miop{)Ll2 z|J8X@k1xD-Ajnk2EsB)N&o5BD_Udp^P(EHu=>CMHv2fw1##{cr*3R{5o704&4HSdGtcadP9U{-2y!kth7 z3j}3)+O1Q%P#N-$szb=P-Z#B`?%eH)tU~B3&86L}vp$bXZCxj|=^e}#DF(=VIbB?U zPi4E2IRuGwOxngyE;&~HPvd!+v`t;xxUW~u-Ijrz?e8nXt}8Te?LBtd=L>6n7mWJd zbAu{wtnrB7&dl6>ddf*@A~BKq3Oh4ddw1^adTkDqJkQ<+zeyzl8o((`hyg+L|JmI? z9Oa3g=lk8=`akTdIE`VsL{`n2)iIx#+s5^q3X&fy8Gp9V+oU|Eu!t19k^C!$k5`x$ zZeZ$PdT6O_aajKF@ytOnQrnHDc~izX18CKfF$b{!;C6BIvrrWYi>~_K@Z>*iObwt} z$ZF6#W9b8p!l-XwIadL@!1G`sFnbX-cvbxYAD_IWb*=uYMC7R}#jqZ_!~}QHX}n;2 zm(!bNVkY8yVclkUD!Kq(YUsSEr?}xL<|9!7zAn=kYAHO0Fo(if;O1ZI^#KMZH+HUO zmlH!2iM%@Y<8e$do+mjanbE5ohIsiYt4rD1{v83wG>O_tB*15Hv3kU0b?D3 zQ^n^iJ(R?7hN#n()ibIO*3dMOYKIVH@)nYT%*K_*Xhg=j?2m>=bU&PL`F zvMVli3MS;6Fwm@`y!escMXfEmi{P2T*b_@}%6dTNvv=nc!w(!hDD)60$9pBS&I+IC zH+7Re*Dg8a{yI0G36;(wZ(XZHlYZ&3f{Y@Ic<2yCRAav+R#jC=k{kl2QmV3lef-@X z9a=V`r%g@i$@^#X{IncDf7jdVz3xc-dTj zplM#Yb167zrf?ajy`NRd)HqGyP&c#7q6JmUgXk*K3$=O2tMpLo8PyvPtwHFbxW%QG z*)MgmiCrX0^|YU7!9;CwEff;K`VdgHWY<|sUGmE7Rw?fY58^0N%4Itzjv0p^7#7gR)T(h}H{gbIK&@``=X+amd#C^tyBn8k;~b{_17>!{{AAE|Tw}cuR`OA#x17B}XMlr}N^;ja?I; zBuxHH)w1RFUfe%TpwgCQ8;KtGDuqY{0k4(DaHq8LE#Rs3{o^OI0H_HHgzpIEkZNz)#ThM*#L)Lvel zrKX2serDapm-_VGGM!ZW1e3~f)wNWmlf~pe3gTbZ7^$5JBvD4^`V$}zf%$#PkGYwS zhne`pioFV@Raq1qQX%gBTUG;*(UoE+dcM$a%h*K@qk6q!dm&kN)fb(29(PjEig67u z@3mX(@5-Q5iYc53BW@*1z|jus9S0N_y2J7h$c+R)bZbe#9raAi_@7{ z7zq0dJW)NMsjX+?GDyJ?eY$A;C5hn|i~*52gf ze%V&V^+SqlM(@NaB{bvmaZ!VWfw}!RYYC|FDLK?#fiSa zeo*I{7Zs;|#5s0sN+%{H5^nZY=be9~Pu|m0iE}XArt*eQ zpsv){PIkDArI*D2}T}lkFEDT9e&dc|dl~HNT$q&2^o3;)8KlT1{D|b$G(K&oZVL>)0L| z?Txl$HXZ~2O$48CM_2UE-=Ax5lU#Q694@~NnEGxR@?H z3+vvE;M`IK5hAaf+lR#?z$E{tn9=kihxIGgA`qu2zt0fr=JrpfhzTG34I}COQIppm zXJ{wFN9g$_Pv7EQS?i&-QMOJswSkY^lN|b)JIm1f141&}Y#wh?e6%2}au70PLLlN!sa3KC)c5Fe7YePQ5y6kh3po63P4MvQ_FjxDi;B}AkL;X%)P{A zmZNdzxi|?oDFVINo!IW<9{wakKDas2qXABaSP3`V7qC0oT)7A`N#DROB$c>AfD6Q( z#(7G|$eGO7sBNEZ9am~>=5(+krl4a+d7E+yj^_Pn&v2`N1YbqV$A;X31Di?3FqH?s&s)SIYz{JN{C zid6bruUl3WyUTAqbXJ+I3mzLfWABYxm7UJXSEsB>RJPw5dPAC3jJI<~z22^An*s~M zW=31n7)nzmW4+2ai5EjWPJfs`=y zaQP0J-2!C?k=c~Uo-aR=8L_Y~0p(AA{>4roeWz&>m=1#m#GZesiYi5BuRQ7R?~yqEC#^Cu zwk3yUr338VEiZR$`RiJ5LlU7N*F~!pM538tM!z%hbjrywO>e4%?_`~1GQXPOjN2m; zvp5gTeWEXB9;;%y^zib>MU8BMV0z;3lU7(*f+E5uR&U(6W^h#H;oG;z$0ea#8UpbL zN|&%=ru$zF7;x+6YP0KfQIV|btj#Ed`TZfcpFDJMoslo%UGq|Gm8}n#ny+5{XY)vs zW_-x?VL8uUU9InhS(QZMPM9#kw1J6kCUErao^Adb%vR3H!Y|2=wFp_Z+^^SbKlUG( zUg~&-*C#5R{XAF1u`_Czw6D~20$85 zI}^fw(fO6cnh4b+C@5$veNj5zi9x{NBVXq*WfiTrVhHy_Gf^;2fc!pn`_j`mILzE? z(#2c%Si|rZXEURXQr0%0DTCZ9>Bo%4ZKge_0d`RwN{OraKcFZxgg|DO8)6qvZd=_| zYybFQLR#oA@I8kHF-yVQad5aJA zCzNWEcIS1qF+(nAmrBNIf zl(2@$m8)f3O}A?Kyiohh%n5RB zo$`#b9SB?3wQ>rBV%Kp#rkJ9DryLJ){d83{Aj5wB8y3TRr=CLhS$#Vo1& za4)&4>c zvg=0c3P=5#2)~9IIF0C?nWn|bAObZZIAU2!_|YPp2SjZlSR4A*`j&5WBcp7y8+Q(V zE!vkG&)HKmu(FS+y0U(6#f|X6Zk$Fq!OmA;7-@1i%1HQh40#J6koils+^{Qj=hU0x z4F1{3mh{MJIj0}$M@i5v!tkg)elikPe;mJu{vZu; zl$Eim8{LQ)#86Dtys6ljFrSvx+8xE41-LK8lo$AwD6!w2^EmR3JIA^+t`j(+YqxHL zn5YuMb+_hz1g3;ChNE`f9jKu-W-&6T6~XE7ZkYh( zg|+fGLoX&Gng@t+x+sZ9SO|m9>yMMOZ{Hu3=|i$y=RN+wGw=PBnzv>8%RFIWdF;J@ zXQ13>(QkVn=oZJHpCo%-e9rRpvnQQqmK^?d<;DX#=dd*u(^i4s0%3+vAn|y}$jr~z z{lXj0=uF6nrVXr~Um1A8v#^=A-659mh$7BPfU>&7(wJUcj`7K)%ARph6} zC^7wLhb-s+77_qX&VsRzV({pls>&mLXbv85JSKvkE-bzb<(uR8?Fo!}mwMguehq{? z3DM=``NNVBdsYM(M%Y}+JF>NGo0b*1i$^{k&%hOa{y>P97zb>K{qc{EaqknHLcKia zs|RIB$JHtjgsQbtC~1G$VsfjD#?Udv@^MpiPR!%+$EM z=cYD%8+miNYvcY_SJ!rVz2>XD8_7-bSoOOF2FU;Ve*_1%vQMx<(@qEukH+;(bvwNJ zE<%ppr>9G#sDd9eWR&33IIys?kJ?72fFA|IvVp^WE8RHQq(9z4P$*N+tRCgfX)19E zTZnEc&}%E3s(1sy;nhrQH3esS!9+x!8__WL5aSl*vjRqsrL%22Ey8woTqfG%|GhUV;J#*7w&R?$P&hqfXy_J z;m%_5EP!q955Z5fOY7bePYI>mUZV@%(epuctZxI0P=UQW(kw2W-PH4^ie$x+R5{$! z%U;yG*tj)hyNvnd@f%2D56ATCn0ugQ3;=4ikjWACu%@xpx@>9cHpDkAb8&-5gQv${ z;Gu3yL*KNy8}5ibY}TviYL^TaPdBsN5i{|1hBq4Hf=-29_n`pRrCt2xnB#Dt$1Wjp zYD(Hz?2nf^>Tjn94h;pEoyRY5>)w9B-8y1meh71Wvi@c&-Yf(3l4!UKT<5DSj?!G3 zh_@c+i)r4lk0nh;?I6NBOffX|J$%M3pr@mMT=AO$Z9Hb`B+a`2Y*RDQJf+R3pw)}R zl+w}g-cRQzUADjQ>dk=GMqbAp>@U8*CLpKx$-7FQ%0ZcRiFZwqr8{l5?{k#6F)F=V z!hdRx4$M99Vc|c>6iZ;V?oyk@!=j8CsCjiXK$fNG7NGKuXD(1mDrgFKpQF&0&#<2Vj}2b*bx_!dOi{;g2VIdw~Nqt_1zXsPY}3sB0HdTx2M z>f!ZN=?MEY6ip7AU94hrQTEM`-(Tv~o8CY7O}^Tw+4^@4?W4mp=fkxr@oOyhLtwjd z)u60miZ_|A&`B1EpG1pk-~4a=8vr(241TI%Yp$c84r}D!htfVuO%XS!+yP~yuq5jF z=IX)7Lg0uu)?Xjx_OFcQW#iab8IJftAF&?Gy9OI4557^$r)U}_bv62#g%GsuYpD84 z!|&=u3|yF5&7oQZ;Q!x55sIMbWIA?LGfM=}u|Fc!#7yMW?-W)3;;d24WuHoJix40UT+d zr*d2h30XD}A|1Q9aNxv&bcku4CggL3^Rs83JXcufGwo+_R}QuFu-TxVYEZ^HddHQ zisG4xMx}Bp3Xjn=vDi^I@;65-erxGuLY$e-mGCr(5%LJRM^VPv>%sxfVSXT`50pMK zjZV{5ioNr3z3grkNCvCDqB zJNcx%7~A-A)n!@-dtI#!_Z5WK#AJ}#gBcRqpCguDSvvwY5g(!E;_KX;SkBzs|1wgV zQrv6JOl@fJdg6r*7Wb*rGj<=k$jr>l)R2KLJ2+({m^YOMGwDfRD5BjQ&%PUVzKXnLKnka)&GUZ>*8<6ix`=bT3dXryGKLVQuuiRmWN zn3h}5|MvJy5UoZ8nqNLV&$yK%r?9MA)YH?@-ln(|+v{ib}GI`OQUu43j6I;Up9Yzc}-X!o9q9HIffa4Y`YoOdaGzk)TL+3^7^Ke zM+QY!vnJZ~*D0qGGo52iW!5BFS_+gaz?^e@Zokp`OTX%xiq}88n3k~^DF}XVA#>bGNqF zd0o?dnkaV&Yxhn}&KI6zKKDyDpWWBL1KSzx7_fd)OdL&z)FTp#hgnk)3 z9hfm?EFB<-jiiv!sd*dT;xN6bx&8r6X|yN<2DFqd{tZG=bAeF%#8Mol)W+7|c~K6j zQQuW3W9y#jOcZ>ysvdMsXpZJUn!0ol>({T2l+?o{V+1&nT}03BE{=b#JZl}?_MW>He_ z7VzMQe|p%eYNx+F1n(*JmCvmeOS&a|Cim;1voiCAOK?plVZAE-#PTaM%zpJF6-$Sy zumb(YuWen(ta@_DO3bA#ZcI~%hzUe!1q_Joa*Z?2KUfZYm^T7Wo z#bsgvNM=Og(`V1t$w8@S-9uwj3ofs%3$(_h=sf$2U`?VAc#5~^jO}SuNtCrn1OnCG zWVj`)2mOtRV2LT{Z=He1tZt;U2#E|-AuNZ!bOaC?ll8!}s8!%^FqSVUVFo42R-l)> zkUns0c<+D|oR0!AV=I7_mu zGDkl!>{#GrWgk2IKl;4ny$xx!8{jsA8z4RRdPkT-NQ5jM!{myB8k6f)c}TCuY?Seo zNAF%>G|oHttxEOA&AzzGb$0QOZ8X=lDH{DxT8D<0sit~1<94?UO#&r_av}3sC_Wc= zPC97<6PYcix3Q$3zNxQDM{ewmsU7{CtJ$8%CyGUWxuhsOF?vb+dhPTs(D9 zvwXR^azh-bq}VZ8V~V8yOfgX$Fu0&Byd)%<)VjoHq6`xMCtoE$*PstCJ$SU_4hTnx z3g4usRlG-Sw-4S^hHo^fRgRR`xd*-y*G{EX6!OIg*N2CeSuIzL6_l_j+!2$L)0+q7 zNjlN9)^Y&5z#3jv#XP0~$Hw6r^{w_Dkn0vXo_*AohRiL``b}=WWHx{Y9OoxTKKk*C zkPM(Yp|o7u;n)#!mGtS|yop2+n~}9cT6r8{!+F=R6MsSHC4q;*%>T~@7VaD8!xt&x zoD{CV1e)V&_3LD<6&ikN9FSPc3iVakuaC1{#9H3I;GF($$ezZP9j#Sak9q5*d*;jO z>>|Pl)P_m*O{@lkZNmFPVJ!`JM77ZWvL3)hGX(D^*m? z>iex>;fGs4hrQu3D>Jho^-aIgX13Q~hpZHkixy6S^b;C0oAA@zOHqIEo0;>)F;PG@ zG1T?ic6FISK-lW+f;})RLuOTRWpx|a zUL?Q=yegsrOAZmuP^49`i|JvwDeVI~&12mrk4ZCzRO}CoAZ@p z231#==eI9R%6Pk=l9x=MI`wf=Kb6tpjkuZ1b^#pb83<7uVnfvQ%IiYVsDAHs)J(#5 z2E8z{{ns;QhVc1Wa9Vey)KX_vc=rymeUREFUB+9v_bzKM?Y`zTsMd6<=QOKZNV-3vDN1He z5q?KYx?49nN==-=mR!y;5D*MPM`sbe zl;*kC!FJDGdSSHY+4RyA5tQUfO#EEf7_5ygZ%gvWBUTL%zhmCy0yg&P3XA)C6 z9hJSa6uA;0Fby0?%o6fra=?&%yOs`aIlEOU?)YmULmnw-YKS}B&LY~Sivt(_lZ6}0 zSga^7UA!ir89dVg(%;5$yLk%Xk;XNBPNbF zn9fieRrCDtjCD}@u@T;5H)?azCTOg?qGIq0eVc7^^JrDt0o1{_SEWtL)Bnkxscl?=f}5W=DeKqzxd^x=_-?~9C8FhWXN_fze=c0meXM{lD$Okei8 zPKR$7J6^myVdy^7mNb$uuuza4hP3TN6ETI6F*Z474p*+|%JQmH6?GL&Fi^s(cl1Xd zdM{<2#^|nrCY{L$?45$jBdHcUsCDe;_E8k;^Wa;q9CEWQo$*M4NB?-vpdlSmg7u8| z{Np(g_@mW+mROVoO-@S3b&+L@v&=ZLF3QtmohbE+Y48;ydKXd4w)Y6 zwmi`xU*)(91GYTZsTYsG(GsUJk3IiZ?jF8n7H0dG3 z`+}nEs9yd|h}t6`+WpL#S9|~6c7s`^vbbOY5Bw|%jhYC!ixm_|uwF{0@Bv*toQ)v` zuDN>%(+%zFF>@`|9#Sky=20>3qev;V-FI`6g-V3`X|wnuW$jQoCdZbY`8N z>nv<9be&2TXnos)z)$&5r0DE%Pb`5!X1=Q1VYhqxhdK zB@B8*$RU7%TUxF&6baTe9s24Zi9SSa zP>z9iW58Z-gQ*thm^Dp zolkix=7Eq-bIO(-7oh~@uFJhTJFeR3+jBO%O4h%gHK^+f29CXn-7K1XxOU#!U1`+n zd|XEDxS_L4e)mJ*vvz?|3Fq(QFZ4rvztQL7tUl40Jj$B|Pz+~cZz9A>HAzv+@gw3H zM2qi8R~O^L-l7HOt~Ne4XC|D6&rpwz*Ei!*D3wH`L93}r!0k9MDGE`o)%~ZUnhKLV zO}LcgX(rGB2+Nfq(76#BK2<-CiQ^o1y78&;g=?*o1fCQCi?-7(!+{{NAZY~|F8Lf$ zFkLR*Da}9a0T5c-wRPhaJpF1#_>gy9?65_=tLiDhje+f_nq}{8P;~ zYS?!gJ@JJyqu7+IJ@Mb#_(X@+Hrx1E*2ct2AbaI#8_26kwpEX|plB4DEOVt>>5?l8 z-U3lVf^|wImQaZrD_15*VmFR;%Vi=YzoYRFt1pkv4*>&ZKqJpU`S_fM5yyJ6ei+x0 zlenNy(_*nU`*J$SS#eWCLR0tX%G;c4rvx{=GDqbyB5ee?^$B(9PCzr&6$ll)4f;iy zApV({SgvQKbvj{-&2n755oLX5jCsGyUq7`=daXOQ^G+b^feL8a|Ci?_3>bUAT>oG! zbGm;aQ&*OGUP&??o!Lx8_AohX)+S_nkFx*m%Na+m=HKALjk-IZjhh?V`z-0N9=6sM zH?rftGDv4J@#(wcMn_Ni%@5w^C(Tc;7(LK^SQ{1AeF2AS<~a}27;ZTWs)Tr{Sc-3A zNbK{T)Sl(yZ_m_#9oaLm4&D=Z)i8}SCF8{?MVWew^D4qMj*}d?uuHd&?{^u0!dF;9 zbdehU&Vhl1(angy)bHC*J`h7tSY;Lw4Xn<^^2bo8>5u-*OW^Y;B4wc`#$;cC`~Zv; z3^G1w`0xk^qH@(AyPw3S`O~i#{gJ--Wb3Ka8FEnYU8r+#up~ia)EwA8$=25P_4aa= zvQeo`@0+c7V!n?QD<1gtnM(@IYjvI~c6sK}rp76sa*&paR79EZ7Q84z?mmYVic}v_O{|C9#G{Fyzlu>*?tEjacW!llXy&Ov}gdec0pB!q61lOovpg z6lG@?+zw!MB{Vfu7Fag@t77on1C4tZM&Jhp#kVEQ0|ynE;)_e;_b2bi zEJKMXV#17zaxhF?S$bdJMq@kNF@MdtN2M7Z8opJRR4Om+9XBc@Geq@+=A$F;d;ECm zgu5+ii@Pv__{1A4X%IzUue9RGZ$3?NM{|nX@pwfE@zF;gi4_}WD&Ag?9YLt5)a|_s zr>D|;IuYw71Zu0^V}6{Y7>8c_$?aA3eNoDarI(kJFG@sFr(j}h={(^>T6)d(m*xsT zVMPx;?TOzq8WPA*YV|7t_1`gK2!Xb3#;Y+kwq7xt=8FAYk%I{hLjbVFLqxTKfdJES z>lxcan5N=AHc!ba#NhP2m1GrCkwe6p!^D^cG2kuBu>a+Yrn_^8xzfKq!Awkcq5Ary z(GlWXB%ae@o<68f1mEG*Y#)klW6UoquTw%BU{Otd8RvcM*m>dh%i&I1h(7$&R3MI3 zRiad(n?=vxXtn}W~ME8hqW*kbWA$RrT;`7qQh+t zLKP&n^+;l08u(3Uq+tRLyL%B>8+1TY?@`4NLiVG+&U$9hm~wpW=Z#UhmQy{7C=**kq41fw+v_eURonJ zSU72t$;~pHGeJ>Nm^z%b?qmS#nZ~L42$@vC0>sHi$ny8!C}xC>izLmqJx^ zY_!9f?H_-(azc2>3d(BHI9XaAYdY#i#e57G!;!x{`Hujj!d-xrB@EKSu3fvzK^XVc zYx>J8TIwgDN?9`)@Q&5&DcTcXmy&^OJ-J5)ZBnc?KhtFXnq6?Q+T^v1?SQf+02c~b zp5#b-#wp2VYQB8so(50SVhNVf)=S{WS6^+Tk#*#Q+rL4@ldco**CQ$M!)qmSGfYh08teUu^pi7g=yzr>Mw7*74I*aXoUghD zvUVlu34$+nyk2R9z6a81jM7Vh@*`wSDWvq_WzU5EpVzzhw)a}=`49Jf zO*a$uPIj{WTOsF2Lfiov9gzvLx)E8_o1o@A`Krjj5Y>UNt^3WKQua>6Yj16Lj^jx_ z?X2(AF8*d%ag!B8GIgIJS*c>^m2mIfTa%{axw%lvZU zDBB1^^SlCEogM@{y#T|KhU*LxmAoc^i6E@Y5ZQ4R`aTIkW*#&7eZYL=+e7_me`CdmYD1sX}Vyu#fArU=CN2>^=T=9?N%VoRvEiHM6T#i599s(twr0rWYcfJKA zs&H^0EuMg{1v8wPMLN1j8k--_I+m0@5K!lJLT#4_fQWc+FGZcx>Q)bYGxnh)_sfx< z;4ti%8x^v>-$t+oIZ0g0L}ej)7Bd<##El^__v(W>stIn-R-o#>+^url@Cg?4Lmw0K$99o?DGb^KmPhej2-JyANs67Wrh6gp0%BnD*hH zBFcantM%za^P!r8@F~|}fNch4l3In!<~n#^hw2O$HMwECu7Cu9_rV;aw4Ih9u?oS& zdI)QOe{e&8HZEdHwB}$i2Bx>n!f~A8jl1#f-nwz)kkCOjHI1iO5oCKDToXCfDXj=q z!#4~NpK;!`@%dD;ijtxU%l1+1Y-V3z3nf!7K0xv&Z1dnU$oE!S!fb_^6sMtf-f<2P zBmqQ(>yF;sufU@Vl>OK zqZB<0p!5cA*@J+pQRPiXv>(d5e^S!7J`pA*I+S6bawHpHVzE5@*@zuKxsp%%a|Y&2 z?$eOGGn0Yzm^Z9hg}_V*ZYG*YXu8G2ZeX*X{c+Dv&JY85@6+)Q-p)qp$WXUJ5)Dxp z#_&d}$1kJC8SOhn4_I&0zby-syao#&;g5A4@>*Ka^FoO%<9aau_sJ$aZij+zuAg_; zCqPZ|+JgQfo?QV&!&Kc^#tB?YblgS+V470D68Gx9iCN;DStsiBr|{g!ef0M8IpdQ% zpciN!a8e=k;8#aDDe}-}NkJEE0d4H6%JY2?F_jG{R*Sy);UqTf9@$_=0&V$v5r%6Nay~2izZlB{m!!~LHH(|J~QD?u$d)GIk^Zjp{T=A z_i5@Xu)+wWfA(5E7)~CuZzQCs$LV*s4Y3T6py>1zw z=S&hFDOl9*uQyAyER<*o8!CM?xHvfK3P^*i|2pvuqSo13u6=fJv9 zmbc_E-&+Rmn%}-Y@tj%w?x1opp)9#me#ODvU`i8*F;RdRwJ=wZ?TTi%mya9Q5!|f= z7K*&F3l}Dgv3eyG=Yai75$hf;l`kCo&pD*L0HAYDbWdLSln(m#Z{mD zambbuvg{G(KdKW!Q9EeZAO;^!MQjD+x@Vz*mtq=Ik?ik>fgi?wg7yfxlKfGirTStq zrIGWo#B6ZE@NAFf?lfuV{yC@zsEGpz2*k%3i}~l**nUZq9#R3oX9O?3jv>N(kdp>u z0yA%6BarjTwn~wYM>-vF^bMHJ~n3xNB+6L>dSK}Sv5_VSX!Ipr-&q86zE zurbHfbYPiCLoyn5pMviIvb8u+W$R{hJpOXUgbhzNzp;fW)!@PDvuw{f&m!M{jKNSw z)HJ7a?ZYQ+q2KzB8Hjc+@sUBo`m5l%?)xUI2gDZt&X>bRyGEW3oHR_}WOnaOMluc2 z`E%TeuW#QUQ;8}80&#KXKqYm-uO1#X;1JJga>iIb!Ts0)FwK`@;)mn%$ZDXbu!oah zj}lofw4?hX&jrwK^uE-ZW3{}5Wb+#&HF93pzxt!_qkD`@9zNFxih#zKe*#X zvj>v5|6|{Mbwo7OAdd&88~joH_oie!K~&&|j9v0Bfc%JzjOo;hbVcSFm89IDB8pE; ze^WE$=)o~C6p(5&9=C~cCfYa}UM8`A>P8yU3_`>3`>(5zq>y?DX1*=!Ro}HhZP!_g z+3sNS;dG^86{~6efPSQW5v7zaGKt0hW?^|a7?nb%_C{RhE}b*iU)fF<`Q@g`gGjX+ zh3r$L78IX4s<4@WXMt9*t7LBB+0I=Wj(b|hL_|i)LGZ2)GsBYBZvZz>a((sO(TZFT zA|UY>N_SzJi?V#8g~1x2i>_H}qw&^pIDq1U&Pis*31Bx0@gdQ25v>omEBogrm}F^P ze$3YuJ!F>$1l-*dp0POZZ!_F0&-oLW{q>}cpZ?kBClva!Rvd9S$MjQgA7JvPrvby5 z(R}G&-PtCPnI);g+zZn}ZovkfLW=i+uC=UGsQ-Q`&6NTCKKb3&TeyF5UT=i9IV=&T zsbZq)TSg84rUMX&T;l1@mJdUZMNrAK9cipcWxGAqvK#Dj8Yn*+%2M0~AeW}g?0~S2 z&Y~oYK_FQVJTdNf(mR5Kg8l*51{Gc0Z$VPyia*&0O#4)|J9+!bV$=N-{~P+V&lQ~3 zzPGMbFKxyxF2QDoCjXDG9wnQxC5<|#kJHO_afr-^tVt9>KEfxhhT@#&wF(uJ@h2BZFS zZ*4b(0skNx0wI%3baQSvk#9Mth7N+Q^J}JaV%4YvyzSQ&?eg#3D>tke0-q^CeR!{mel zsAboK8@WO;z4~GIw(ZB^wIh=PZn_0XwZh+^RIAgcD+W`od}El)j4NIQm9-Ml#o?G; z;N%^sOqy; zu5wFQ4Yb&Bsm>fg8c7b8X|V_}D?HN9-)I=FHv+hIcB9XyY=bPEw1+mKDx?Tz@_{iT z$s0?NYR}~77#S5EKYsCK$DX3&^{ESleUjl$&7e z$s=i>mq(-gmfo?dE%t)L=xWM_!R2-rLBGJjGj` z%)9m0Z!8R^ft57;=VVNb^ZHqFJya@Fhl^V4&?eZ^+|cskRW)*CHlnCA z+MjqI1M*8JKMJE>sl&o3xnH1pMH2jRyB<``6&9|c&d8(Nx3IdDb`pIIERMfhD`raM zwyY^f+hCl*{CXOqvMu`Wh?X@N6&Z2PXU<+&)+nabaRskM2Fe!l+Qayfyx)pu*Ta^_ zExF4OQmvM^EYo~2{7z)hRB`lJ3#rxUct+vDj7p^FKj^>!OCM`Qr&xi}KGv>Td_$M( zXZO^!Z?LqKimC-DM4nlkK7+64##kS;Gx2>}&qLMo<7-1RjqjLas(urg3Fda?)ss~m z5*i2t-Z_HeSi01I$%W?|muX0@IWr@78VmOspg|^UyW!r9r0LF%j@0CHFFw$ExQ*X@ zsvw4~Q$q`KKn(ET@)*%-0`%h|1bbIfu``Y56Rehfq&)BUY$z&1#_KZ+2 zCw;B2+6)cX?WN%z7;{j$5ZfEf%LB827{0RA6OH!Zo;1+c{ zyk}{{CSGK8_K1W`%cv^x`Q^&Ykkf4(Px}y+k?cbLJWQZ~Q{V;Fk1P1{dY~7jNgE{N z?0b^kJEnOmkL)Q#?;F$f$k@Svzdu~sZJi$DGE5 z)x(SZg64Is;{7&uZ~&FO8Z!=!;uqS}Q{?gSxMob}e@uEVJkz|)jZ8!`3ON|c;MGH- z0jlvbr71ITf`RlU?a2#>Uy)&_R51e1xe~|^X;9Ij_Vu_7od5O{@U-m0EPE!7ZPB!k z4Gql&nQavda^fpHM>f7bKmK#f;@9G5inW>aw20u`6# z^tJMa#iy&Tx9VR@VwR>481)bKWaPz&W^8!381jx%-I~EV0Lq9u_tn>=4bbuvP0Ivw zd-t?fpB#SsBG?~A^E(P{i-i|GLIH|GpV7xhbW)lwcuOX98*09v*j#i-LfKEg%B4$W z$TH7K@bdE!N-q`+!7Ekz97xP&<3X1qSWCf1IX`(G7{{pYh8>+MdE= zLE2;pf|4*XnQ!YSOPT1w7*@ z(R7nniyi3N|DtMd=YtIb1^qLlihTUYEiEs&a{_ctU%h(v(Zs%%ZHpLx@(Nzr8H3h< zK`rF25`#bHriP@u7A>1!zol+d(d^`nXZw-uz`>&Z7FU9&B&%AasC|}%CM=i6ftvPu|P7LFD*1G??IR*t? z$#&g`-hsq=l?#D$WPi_}lc6H4^$?fCHJTB4!5dn>W`KydrV4nkH8Q%4J*w8Gzx!`l z#4rl3sJiQV6`1O3ZC(rq?Q1PB&U2H^*18roLpvNU)j(JGSs z1P2=3{^O5aj266@xiQ^28eY1K%y&W%rBBJlV)FU|`)CB?v9b|>|7sA?X;UZhx8ptt z#^q`4`pa?Xa<$(IbuXmY6Y%!;g-JQ9#6Z=R2O*cJ5rc@W48uJj;!z=3)+KFo|Idn+ z3jvdHM8XHlGWKgZHMHQU4KH57|gdZa_b&O zQPbE4u;5nh`3*)k71|Jnua0@RoRzIMkUY1czQ2kGOGYy$YXh7d1| zf~2l_o7Oh8ml(w|dcq+FeB)6>)GQXqOuH0v9RQ zj+w^>Fm{{GthhSVSJ$}CDl9u(m&u$ph;rd3N&?10*zkke*~^ilkwW7@!xD*kqaq7- z<7gI~BG#agva1>8NVq?DXeqJ_-Jxs(;m2cUht>PDq6J|_P3qA<=yEq4R0ID&)z+8ckYR& zw(F#Mmn@H;`t8u6x1MwJZw@KCzv7PDdeX`_&F{Gt9#$(SZT?c* zbDHQ_8UB&FPA>{HbG80q%VQi%)6@JRXeL=!WMqDP!g%wCnF=%v;>{t4M>RMyZ45Q zstkoSp{_kfmtHl22?^i9u75%EsTTr)r4VEVfj>9ZUyG3T5GwW1)s+bnaNYeuMKjmg zZ7lv>GBRvohu4mZ%x`7vg?-lQZKaXdum~1`h#{xpkruUM!>+0>$DPt3v?wt#NE%B zu69k6VB4V*&P8+1GMAXY@%3jnYh3uyxAMte>_L+?!TXE<6e~zpG@ygedpHMSWI1ql zsd#odB55b0B)HTT$eiiz?E(-f`2xV6x@gg&59eV-{D6HeI@fM28p(h|hj@oYOcY>V zFZ$vWM(eSp4JO)d3{ZH&sSD_f3`_giaZ^E}uW+fag^7>!$O5ctAmy)9 z-qTjylQlv^Gk3h+aF0p)DLsa|mL*9odqbYxkdX3cK771U#=7aub7zfslsHyQ>R@2} z`b=Xnzb85AHz(TywWdW}K4ckiGX|DC_y=$2w6OtWhS-v_RZC0<)H!} zyFwpO@u7<5d>rMsQ5ZBlM+F@+S*z-St#0%YS%b>^p6VE6IMLm!_j|_~M53!-3uXxi z4-ePs5{2L;3}tH^(hJllkL63nIelKsQ_pUTD zICJK_#SKS~drStU2pk7qGIPLI;M zf&#Oi?>(U|Lo8+Nk0MEej8SN)&78n|dpg z@6b^e)rCoE`-`UN_c79o`QKqaXt8tpveGuGTc!%<$E=}nZXg;HsL)G9)o+C>w^g<9 zxuY6Jwo8HU6$RaDJb}+&e%35MtkT-{Bem_zkk}w=U|-I>Qy_%JHGDiQq2p)IjynBh zIDXRK056-QGJ@W3$kl{d5Tyh{Gl1Ku=1j3&u4{?b5kO(0C&QS}6M_Q=MSdXAD%}vO z-}iodu}tDXyQgD)di(IUSxHW(Lv)2C?6wv<^uImI{1MH$C#t@@S~qYQyS8sqInq5c=USV6KpcsegXJx<}Gdcv_7hsd*!X)TE??lm|h8WNB6bd0v zD$qJa5iEfT-B?gh;QIY8aTHGDR_P1%z>oI@XUxO)?qwnQg@tway*|#|am)1;FC@$Y zK!30%b5C=OVD_|;Lt(!S!eYW9Ic(y(ZE{ak0ReVq-f!x|x*b*`p{GT)wld?phDl2tV%-KWQByz1EB^IB6Bpz<2(dLbIIj7wuu~cVFjU^$x-#!W_2lpoK*|mm+s&01b2mvy_)v?S za(?6-Rw4F58QlJ`^;nHHSg^E3liPRS{et3H^vvIjGVoj-{33hdz;f%JR@v%t4-SCy znPqE!^)@W31B`FxgF{~wE$qRg@wb2vV$-qA5l&r|y(er_(nQ`N(6Lsuc=QbG5e`1w zJYiiTVp(Vk&QfH-ro@~&g@}Q@Q|0P~X9f9z-ud3={oPBOe(m!{51&@liLgPOp112g zGLJ)o+mna)Hg@9pBdPno+MQqIqiy{Zw5KI|#yfq0ds67qi6ehv2n}GAz6f z6ebU?Q$%bsT8J6^tBppjn5B=EWtQAI&=|0{bcxI?(@0Nub>iDjANf;m;Pp&eYaY0{=;QUo5n4;q@b5)8 zwi!Ua}ir+hXigqlp_9^%Qf9$k-$|dVq7uOKK~M7}mf7p5GAEhp3REv};VrGc?`dM99#w2gTYS7!6}vfR=C#}|zjJp2 zm+Punr_2~K?bQ!$ZD#$tOUa?gzqpj2`~SAi2==esXzGBXmAz)?nl%gcx8-#e2kNgE z8cu`T(X`$+!{<*$o+7SuF&xR5wk4tgI3`7Dc(;dD7eq>M~_qtXu0+ExH(RI=qqzT`@&usBnfa4E8yIox|>1FE3B@$Ps zPpiJSPmmyplrHQtnc1_S#Oaovg65-+rcpU{@$vx4-~0Zs2=INowW>{FFf50o@z~J^ z(f&Q8%kas`$k=nxQ1>DQx9lfd{=<}C@28B#>ip}q8h3+s-?#NRb2gdyd_2BP87c1m z>)a1O*7LpFr+<%S>$XKbV(Zqe3mK)`N|ojBPDs{3Rnz0*Q@gqFf}~J!S)OD(yJ9T+ zjPO`Kn#HaA)Zy8qM-LF^aOnH@z5WqtVQhJMMa9Fon15f-&@o3@EJEE4$(Dn2x&-=w zdm`b1f$`(2IqqX{%q&AqAR488Koe;6!`f8lQstZle3-Q9zBp{^98Q^iK6Fd9P4|~8 zoh;iDmCiSM&ri9g(x-A(F*5{~zWfjpwuJe-rB_xZB>d|%hFQRDk8duUraNyq!aWN+ z4Q^Vw)Y@LO$-9R$>^e_?V^`=2nZr7A!=+9iu~zyakfbV@-5v=v+t;>q(~=APsI-p2a3o!| zEWM+nV=qKNDi~?;ws(c%hp#_wpwhB{ts|=eozX-f4?6hfN61a1H_dKaVPqs9G<0L+ z)F-O420{C)&!v`I1am}KLQ_W$TefTo-gMPeLD+WNw%odo2X!6N3l>NiC#FeF zT1B|H^bnvd9_bA zNBeq3FrWvM<}rT0l)}|;V3NwJ2wBKeEXVNqPiR5%OZMc%*N*6}d~2K${Q2B0_t*Vm z`?TGPqD|`{PQL+E0_XBF3MI*<<(QYJi{%)rBh3S8&%D@@Bbd~ycDJI`I`zS4w})T@ z&K27^_W<3O$!$NgQwsM8d zS(2byo&I%qHa`62bMdT0rwU4@vny@pMSWS-uKR7x2eaAS_1Jl)bbDqsbGD$s8d)^! z*H?>~wpssT*Urn~?RDI%nKX1~-hpGgG(zBnPG_oqQ1$4Oj~~1i7=n#Uwdk#FB^YL! zIBCwkFGG=yGj2useaKEe0F%;(Ki3|Ahl)ff=ujluDi}oEaHvp{gsHmeJA!#apu$`t zls|rrzSJkwJe3^x*==v{OSsO8@kg&|c_C2wJ2J}Bye%vw$$v=Ih)59UHjHkxI00!gCj`J4>ht^~ zSbHXpKK!KwGbu9mh#i9DXB0FMd{^+DpVN`{C7$|KnfI@+vD0SGEQ({E)8H?qqS>|K z*}b3U9FsM`YE!$12|^Zi7ow5uKAIx#e}eq`{(gebb}M$99+s#7V&0(#;}xq+H6+z= zPOg#@95^QftL}GZWN5tIrdwksFT|`J%=Jo~F=Iv@q=fprci&uGq#2{|3g$8fx2(-GWLxfW6m+aoH zij<l*>Agctsc*uj!oxQHQk@>I=D1B>sa9> z&c@V^$3NM7WseNsl{_)2Y4xqO*;CM_y!t73ViZx+#~>}hF1L{n@&hg{B%);Tyv#{W z0xb1W=3q$j#50wo+s3l^)9&{@wf)+RoKW!!$xLHKZv1 z;>;jFoR(Io1Xv;;Q%x1ls5yxPCgQpE+EHgoQ;U#Z4tj)C#KmqfS>{vn`2DVTm}r@g zh7-5wh#9irDVu3x?Tl6R^_bzGd%HxixzER&}F=vVPL?roZbA3*f&J!cJVu)7eB3D{&z zc%eT^$zN@gFQR_=65IbHmQ#gkjVfSH{U@^atVLM@l^{PTjk2=giyu(C0!)mS<-cGC za**8Ghb?IhNQ1e(#r0TUvJsI`UPdZv%(J34`_JozKRk`SKT3S`p(jcC1y701^SH%) zmz%V)Sr1c?L?7gb?=ibrU76J|-C&|$jx@-hlzB-tZzddGqF?BIvdGpC$0l=fR)kI2 z5F8v_cYZnB4k&`@ExU2fHJr9K8H|LGh{#gbN#o6LaA;^Bjx@Id~t^E?(@JQ+aL~HKLq-mDL5|=p;H@dVm{-nG|l$bk~Mf)Hh1n^D(_^c zPfvSNp_q6gwQc|OSxb|BrW^~b+I+Jtq@hP{T5&;ICv)q^H>+Y8gt(L+(-p)gR<$J; zoPD|fq7A=*yV$f>b;=bdw!4L#5kc`0Rolx4{w7fi{n1`brX0IU&sK6{WGZZ&7I$~+ znWVpIrRxR00y-|8-rf_#?s@t9a>KrtGBh%hT;MSl3pRwa<2xK!UrN^-PSw_a>+5EF z3FU`lZI%>D{7_9ud{scn5d!Fcf@K1cmwd5`#=Wl#Ry7# zn$k{mThI!D$jxQWNPWkMR~wEVJ&J)xW{~_SM`7yl88dsBT7j_zl(z$frGT?u_ zH;2PTrSGU=t$OwggkIM09v1LkbhbfROrd$@Q}A{{URqn>c)8M3;nQbAgZQ#BVfRWZa(lDUUz|_G0Cb7v zr2=|ciT8OZ9zD>HEjW2uJ0s?uU9}gv8_G5XWApO!#f}~0_?%c7qK6ky;-xAn$7)i-Kirbcc;Avv%Np96~=5jyYF`N*)d#dcYM zqjhz^@n9?D5qO=3IHuu{uk%fpX^I(okcY4w6b)nacKoua$GrwVp6W++brF<7{CcS< zA=jT@uxC7(dsDiQOF9-;SOG8F8Aq`fhF$>}T$fh~F_sEO!(_$GJmtv)de}gB@rKPy zTI=z-zk@ou?TreP$ross6mTC+4I&tm+u9Aw1pgd|0{K_)q-|bZSf-9JlLRH066PAgo?Yq)~~kIQc)409R{rv zcGv-Ym^##sfLfTA0g4XIexJ3cai7d{thzl4U7>&tJ{%79j~~Wfx|W!j3m$0~pz`=S z0|SFI3-&1SjT0};DT<$sr%d6l(}Y9rR={J_QE3z2vF#r`E!J$1H%3NNf0H)_Ka0g2 ziqhge1-MXRetcnQ=sXYamDiorqV7IYpg%JMD%cK4oX4XQG*iUJrT6SpI87BY_lGbj z9gmF-N2?1vqqL;+FSd!FblkR*7#xTA0aSeu1^tmcq176a1(@68P~+NyCOQb?*ThgP zj>9dBJF*VYx_}LVI7lOU7TZ~>1ongen`=yZiVVH|;BR73?ivhm@P{-axt|VC=&9+d z{_|o7AT0k2LI#@a*KbTVrZTbDk>ih|$mi!(pO;SB{is*bWQm?>${AdT&wqLD3^s-z zS6$QN1i(n5Yv#46JE}#6ehmoJRMBHmc?8~xWs?5d5#!#c@Bh}gW$n$Bm62O<5$D1n zTHNq;D&|k(KwQ5b7y^hW`W4JfM8v}edys$oZYtP-hyZ}HJ< z)i-#w-f3epb zAQJ~^n7V6&i*W$gF4oGvW@C5j?3mG1fw3|HSMnn&<60sjnM4_E9*DE-6V(B5+l&)` z|H&u6!n%W8bQ_rZ8x6%8hruh>aLt+mOn{l2Rd97Y&SLVkwE!};#;l4hrZ=pHH0)ewQc>)R8IQ zxfiJ_1TO&5S%jGv-)q{vuO{||_jfGfM+Y@^^JbRNK%O*I!Mvi)5^SVPcYD*nQg1Od9lh150`qVR#TbQ(3Wu}X7 zTY*y@K=-GRRpGf76Vz3A1VKyFiw^C18A=j=(1W5U1Lf6#Kig29bfD)>N=#&A01o)0 zQPbUT^wh*_!^r??S|KiEr;_&WI!fg-JwK9X;>3wx5X^NET{0-{XS$JFB^U1HhhG}} zp+Ft(0U`?ll@uQKiOnrcwXZwX^?@-q&YBdW)HB__>))AB+umdOfN_f#CcF;@}Q0 z7Lei0F)~WUdkBp62NMfm+_$h!MP96CWrQpy4?k%5nu7ySkvaWdy?jZe^lx7J{Q0MM zYw=ZG!1{y_I2ct!U@8KrM1h-NxB;y>wzItCcpD@mWne|;RpNoGAiZSRIz5ijq=yCO zW$`EOwoN}S;OIAM%>Ef{6PXHiZb{fycxm?xZo|y>=(sv&Yy8NW`i0UmO}p3CJ>7d? z`MDP=nxP1J7>Pci>?dx=?=bM9` znaSpxwxdAlZeKf#eFtX*z_lP~BNj(~3hWp%jPD)FnGavb&s({1qZmQpco@1;?O>oG zDwqHd0A0byWk=EDM3ljS+QVr;5~)laE1?!LCWK!=AWaa59wCXS1A)%ep2Ia`=B2ns zt7Q|w#B=gNEH14wxaiv1%)fAK-`!*GCWR^r z409UV{#{$%8Lu^(L}Jmkkh7H$hmQRqekO=>5xH!2@bvn#8{1Dc;;m!h@Ij3XFq^_# ze8E>b97ul?iuYebUB~KMHihFQF{-6G9N;KqFv?2!7=K= zV0ty4Wi}d-->e=M;_adX{VpypfM?8*WDRPiw+}~h zRX)*c=s1=~TuKCliT`Hb#4$7{ck{1*MIQa@ zW5~Y!+L5yGj-5a6Xr;=IOuTk22O1nSFd`~)K{8N{hq}}ixJ+}m7l25=t$IFM$YB^V zZe0z<2l!e7X+-wL8IR)gugvyYgrzH{$T!65ij{GAgmGYW&g66f|i7`lZY^3+eU3f%oz2Z1)UQ z`8G8icR(MU{rdWRNM_IZv8mJ!AWsKn1s(tqPbOVG3ybqE7Xky%k)o**5$@$vl&yG> zuV#;a`LEYdU+}*7Fm`a{;U#9p)~W9>kZCEpzJo;^rNIRl=Kx}^p(GZB$g5{*X35~v zbF7{}$XmVb)@ukLtkN+AS>ty34WFGgj%pH8QGuM9I0?ZK;J#VQ@NOKpnJ{x~TdQiX zXb`d1!eVS}3DjoS)9(D-cP=JoJYtFFVFw|D*tzniwfA-2 zyivW?rnfUj0<6>X7O4+e`YPjs{(73msE4fZ@{*@d z^B@Wv_O^5xZyXw3q%lv1LDsz!=YQX@%$Wz< zqzcgQEmCJKoq*7tLEH+ReE8xCtd?H1XbKc#U-sVzeY>;E9mE)lbUFs%K6!yf7lzWO z7h%c)Y>KI?DA2~xqiP$IjFoiGau()Zmx5Q5lXrS(E`F^1=6*mUf^#e3x1{8GW17`4A2VSd2CJaQWG@ zgEh6GFhP}YME&i$M2MBG@n`VVS%o;gVYm-QR|%jn%m0;t!14bDb-sFyaOXzcugnS8 z_M=z~C)~ZWA1+qq6BIr#P_^^IGjp!vqjKVQ4$1Idw;z!FRKtYlsg$E-s4HD$sbPzDvslK z=s_J_)HbZ19IG4at7nUgh>T$1lp&clDN@UteM|&-a~dmkbzAEZ8CgIx<4E0vYQP$8 z58q{(Y0v_5+u2;ErbAzQF(oCXtU~SVs^x};M-a_%?oF${wY5xxKPpbF<5V<*@y)Vf zVsrTn7k2P&KFw4E?&|@t=cCFC#Fn$vnmTm|)r?O*smgEsT9Y(-%S!KgL)W$V#ORlv z)zB5JZ;)f}i70E7=YB?&oJ$@7ukUgHPxRBxb538Bm*)a2yQ_xZQ7txD> z=qQpSPn_@vCYBFmCG|376S!}k^Ymw8QJZFY&4=0&IA1I@P50mhm7g=Gq6|+r40aB& zlR@!2dc7Bs50Xr4M>@WQKbT0Z$c34~jHtQ^0tOFnml{$@PC}@GL5ZYYx z>;6>|>yj>^jI_RBQN>RNax$T)z{;ETq4sXRbq;!o`3=Elt;kn=fWzswH?GfA%-c({qUfhBQq+$mv*2V4$Wyz-+2imx59)L%mV|EIJqkkl0D?rVa4`l9ptd zsrI3B2_~E42tQa_`Ye40Yqe=kq!T#34#%&2wEBjX_jO%a)2I5&xCdE~m|WW2 zr#xKzs7Y5=FaLVUNtxXeyU^v&w&TebJv;tI^?h~m9bpiyTA}&oRKYwlHoVYgl{8xv z6GMYA+h+p9z>`$QwrJwpM^y&_bNKIhyKh-KD;k1h=$8)W<>?MPA)p6=Bc!cM! z7eRt|v)Dp2;Z8fi`2Q;pfrg$4wrt`$764?^2>h@+wYgE2L}i zU17H=@sE6m6}SYc7AekZQK=|+$4=3%j)TMu;+Dx0YL3r;dSpr=UE`Ob_nIZTDnQKM zLreLR=*#|QpT0TTd;Y9AypK3gFPO+Ul9G=LZn-tW@k=LH7YSU^`Ws;<`a?a!vkup{St_iCU^6K2J#@0?7jEIKkxt8m4G z_=LFn_sx2>CpzIWND)<4)j@IKv11Q@%mbA69~%I1q#YmfEkvT|(ukMH+}j}oUiDSr zyuKP|by8yBFoDD&hMl0%GT}pW=8(K!GxqAH7V%oGm@q;vqec1a0UPEjm61A8C1y8{ zyW8Al_Solse`kOBg-T!`=bp$2*^A5W<3u+ zDr(bT-*oTH7-P3M?5=m%=g-T4IS@_+EiH;SJw#>w-9``!gQx%Rl|4a`Q z)bRr~wJxQgyTiMIGLlR51|xvI((#VRa!GUK4u_&h-hSXn?!nEyA4JZ-(G5Uo95~Jc zrSt!x__@i5KOVM@Wdhm$QBrbns=~d$O|!MN6&1w88->RC3m8j%lb*8c-orC%pp*UU z#csp&xqIX+C4fpBIg%k_bFb87XALeww6V}mmxi^Q;Obo~bh+kMT4b61;Xx3^J2%y{ z;toibiNwk#$DTP8NbgeL3n~=Lm+o0TRj#qHkH%m zP5_B zQFCXbi<*)>v3&yh5xT)*hMg;P*9`5J<#xhB1($ZKPhmbfTh0BZMqu>eG^U;XxBg?E zjT@iHHs`Q`zKv5w?Sn@28UXV7a;Qj2=7gJo^Al_egD9e1I+^5&>X!tw#NBk_qgtcP zL7DL~bARV7Bv~B7SEm92ZYu2Hwsv;n$3hyj5v6B^#!v&_b7=9hu zbW~7VfLLn4%zL2Z^PBWJSb`9dmmMq$47of&s1cB2Y@@hu;ni{ObD)0uqcAw>?K-*k zyo&R3E)fZl3(ZukytOSniZbCjn?jIKDRDBe(HsCjxGxP1yv(aSIdi=-(oVePbQJ$>-rO$^l)3w` zi>m)^()2EM$=jw1147g(rhF;+bb-O3O~k$*W2=hriX=`ukqD0oaWJrq&vO)Jn35j}|2 znq&Q*ua2qDYL250gOw*0kx+rD_#SL9b3FOi*jeSgvcqE=Dx|N^vUL+;ueq=}Vv^xU zxd&5s8#36P47*3=TJ;X@$Bkco08)b2FUxCr7VGLxP*VlLp=G(V?pPP0aik?6&dCXw z4UqYRX#K5AK{%y?To05mrwp0lV}u_#)*reQ1_^oxWE>Z6-dtz0nMj)MHYcEWJJrL$ z@QFYVFYN9`=!lAq{^_VktD?=}l_Bg7w?KT|Mg7o+q&S6#Sxwa409svmNFHLRwRt{6 zq!m3)2qoODRGS{-wk6Dj3m2D}uAE#eCOrPq9UI)X1Q-8STU|N->ZOWO%3$rT4`}0i zU%O8@9pjP5j~|@U`))*UOLSlxS|kEe(Oa~7$GWS@&6_s~RrkG+D zgi4*5itCGM*61x7%|Qv!Zb9ZwUL`&j$qD}5!()F3`P+3U_{@9|J-QbG%+#xvX3#ja zLtqyBJ!^X=lUVIPjYa8w(&i!|ty>l|HG~J}-YxBj?artvVG2#jtiME^s;T z;U%^6QUfB$4YQXpQ-< z`2cI6IC1Z!R*V4Qyi4QPtzGL&6CcxcRxVjGlvJ#;va+IQLO8RbUqinehvV2<0X-1_ zR7POFT)#A9t^6TmAdWsVUMk2`m-o8|Vt`%itzZxDoxdy@pj*o3>@a}O7vkgPo946S zt!Wjb-#{a+sWCO}P(lJD`&%5jHM!=^&f+s{dBrT_ZKlt|t03wKkRAzKi!HCO1$9eW zcy_I*Pr#szNl(+Yek^2+@rf`s5F|HG4sssoT#az^4w&!Yz7I|joHAtsK`BVinkh1S zfqRpX+RnF0ooKlkwH~DSs7Gqo8BKzjzc?&{wap!(oR;dl>zgr+ z>I33eE-bAb+AHmEA$j)BKs+kS&17G1dhl-TY?_%np7#6}S zY%%7bH3q5p)wYLYGGIJf-6%{rdF1=1H30nwetq)k!~Vk%Btid9B$H=X%Td$1=g#SM zh;P+H?#tLg9qNg#VnoOQUY=G!{vN;(?!n!25|^rG$;an#fjjqU2TmyLtv=za8*DWw zBv_MkN&n40uE8nLOn2Re-k+;xTK9Xbmtj9EaUANcA|u7V+iWOgzmg>3tvh|j7)0Er zTxsQP&m>{A$xlHP!+VHHOZ%dR_@XO%59fHJ@QFz_6az_uX3JzuV<6n6f6Ce;xRQ}> zNy>n!Pj=^hhmy7f%BmdPM#sY9_kpFsq(VQyX&^HoT8i=d*Wl9S%kH~C383mH?GAPW ztTxNEW=&;4_WOtzi{EN(ZnOd%$HCN&GGN(dEwWa*LOc|I4pD2Va-Mv@B zI^1~%ihWiC5H6Z%jPoEFy3+92U*E3YFiVD6g#AoGbS%kFxo#0y6_5Vh=WPR-I}Wb_ zE-gB`W~SQbYuB&Cu~g{OsK!DHvz*-ObGA@I-iJt*P$PjcA;_U8I}Z#NeA6#Jy8`Jo z#eAo9r9*PsyFgc0rPa{qrywz?n?e7gv)b_=G?`tA-VOZ%724n|MwFYW?u@@U%Fh?d zdWZ#nO9p%@G;4=NutnH?OdI45ZtkD?a=End6QUXqHSrXYA&#EWTJ&2M#TH);3xx;>xJm>?}FWFdEt4NW_WGZMm`_oy&g{m+Kz?BA~mAo(cP_?;eC z)bIoHQZC|BTsKK?M@4(Ibue@OnF;EtG`ZYXx%Ij_>1))2f?B7+LndCaCc|YIjgBj$ zO^YiGrCcCK+#q5XC=576@PmdjE-NeRbp<>>=zy23Su-Ae)$tQ2hB3gh`A(|(4*a}b zZV34=wH8^0hAN*>pA$#{3R^T6OlIpPs-L&v59VDA^(Yrh&fN3VI(65kl77$# z<6&d!XD9YGE?dM%-EgjIrXz)pMA;FtpPTSK=Og;DD0Wfl4m5WcHHf!1($evc}d95kZIBe7Ig#azVBWGB_G7Xv|WF%!I8W-kg7}+5yOyRJ2B=% z)jFdiwqC5Y{>?_oKG%ce=Qc&z=G4XT%9>^L4`I#%mg`w~^Fqg?3dF0_=={v&T#uch zOfwuhW8o;#Bv#uyAQ@0j4cI?G+Qug&JVZn6f||ESTM1w-bSV1k)1XS!>+xqrb zUxE0qP0K8!{^uHOoS%I64&zTc@_QZT(oSJ)Y@pZ~HTRh_+i`|9C4i-8u>?Safmg#1 zYyGR6``$wAPUs(kK6bphp?DJNE2=q6JU^W>Id#{6Z!JS|()SQ*OoUxKLZJBYq}NO1np z6GgUe-_|q_+6kyKfZOTU5Wfe-4It|hapGc^(af^4WufASG0-V!(RBc&Mw1|r`0~T+ zOHZE}CUR~+(L6&|7K&zu*Ff@e_EmlXJxU2k+MKQ3q!|r`VIi8U@0lZM&9Kys(bOXf z1G9rSm>>$m^xj?-hLF2i3C^DCzxHS|-s#S>*VOo1{|m~)OTta`1x zZOK3toc!;nRx?SZ(c+{keRJmBO&c@gd(aSECXxb4?`E|Sx9|K_zmsU~ReC~ilq@oQ zsi1rR27}NI3*nC)x1#JmG{JQglh(Ir`KndqUWA@;3%Z6NoZV0?9fiPcnN+NXiwvgA z4`B%{&=6zwWO)5hrIVE|R;einfFNBJ@C_J4Lh%j#1tot@6Vg-p8Gc0b7?k7kL5%sg z5s;rm81$m$7EpIl0t2&w=ocPxC0||CXV87N9okE}>glxM1h)%c>_&OykP)|WJu(@l zLM{rF53`8$kU=svcnV7N+WueN;9FkVdu_UGikZ(h_IKLjPHSqyK}N`WwR8I7qWD`R z?4nu0oju<^`@?jbLsEeWcc3qV^?pfnIJz#lP3I~PT=&fXyzt>JDK2tc)!}a+AuA@8 zsYf*r-lMB!qZdR}W{VUR$I-T63c2|;_dP|3Mh7N8a&GeoR;XilC$nek9>p!4Vp0gaLc-?Bf6zd_?&d*potP_ehRsuf<*>D zL%A|iU4eW*Fbs5N;!OAtdvmi==%kR>0d?Q?)7{_Ie<(FN%Y(#(E%p;eILa)_9=KA+ z;YyuhGUWAQz&LE~d@T}>oOKjg=stml;4p<1J;2dL};)1jq_LtfWwQ_jj|=+0 z_&Pi=Mj1L`a&>ErD*^)`I5|O-ziR=z(|mVpfcm!e6?Up;84>&dR@~iNkWzM|_;$sCW!Y9R?jPPTo>WM6AIlB*P_OGd#PM%yvIX2IN>+_rlEBSeSS5_9J=6 zvwr<_XsO9GX*N$bCeHs*nSJZA>xQM0gfM`RLVk<-3=cSJIKrbr&JYyW^I< zOwdNu;5jrlw-cF0e(>f$qZRl2RcZu2SVwwSEo_j}ejnp4oT5w?|Mwbo?HKim0ZFeYt$RN)Gt zCpeY&?VAt!h!Nf`!f!BI?ajxZz~%=8C}7MQ?&-@ky@dK4D8nv*UxBHBw8+YhU@oaC zT=K#iInWIM-0^lRmw!HpE#M!Kuf194Vah}C>XcKkP?lcVUl4reR5VZe(tk8ZGnKp1a;8heDOiZw6Z3sgRCoup(! zvJB=LL3&aM=$DRII9}H$Qjdf_^dSN>tZ{6OhJFs0Tq^W)k9L1-7emO?LmYlkvp?54?Bh?ooK+0py@;a^9sX(-eki3>DaS#!g-+ z%0spah=KOI`U#BBGVBo! z8Wp$SvpK()BYyJ&GC>{bxHK!}?+?5&p`yMnUlmLs?#`B4*4xbN9L3RSUgZ+Xv#gbt5QT?4kdb~?YgO5mV0$nVB@9ap zE(YDiin21Gd*E=nWG+g)c!RG6?HCUl3iI>FPoH+oL(B(s%G?-)E$MIGwDicxsIK_` zJBadwkk{=6qo6?SW;)wSi zfvcAXo=ZUZ0pO{ZeU82GqBDkH1f8;V}?69Yro-V4?aw z4mZxL$FVKBO@Kt3fvc}7@=Y#^yTJR2ijy4x(V z43g(nKb?wZemd}wmJfuLDEm?Cwm@;4|K=V&e*8Bt z2jVNi&ax--M+4*B<7RibVVX;Hvm_iIm}?sddFlnsp|EyDs2h=^v9FjV#<>fjsWloj z0f&%oD6gJGDnfgsEWfb@TyvFwEW;xLt6w0oJ{iW>5ZB=LsSaG^BD4hfTtBF5N0(7#+wY+hA`uv z`ly}CP=6~W>68Y-o(0DZ6jCcqY9X+Or--qumuuk}T4C*)rrDn|M)*Rk*T18H8g>Q_ z?>j2)3XuVPA2XZkClJ*GsmyW4TiTSwwNw%Yy)<%bjZ3cg%0jc<$pM1j%vZ34u(_ws znR9)0%Cgqn|22zi0_XTb{*g>37}Pd-wXXrD0Y;h%1r7i+&S7ot=cQlcU1`nk`j9uO z#_tlDkdVGzaCrGq`70y+d_lv(vR;Sr@&^+V)5cJyIUgXx-zzp7U;%^qgYagd5I=FcDhZcR>s#~L*U zb~IRvqpllgQtyra^R_oa!HCASkj9a_J!8AGG{3U^sJ}|gwBZ{g0UEk-Gdo5f4&fS% z{v(H}z~yCKKerG6dg%U#B>b!m);YII4sij2QIZAnkR~A4D-j+-iIV~N8m`61KR`g9 zFAh)3nz{}vz)Adi&RR=L*MVok3$Wn>@HJ`sl|>IR;YUyvl64p*xU|8#TPd4ScmesZ zIlq%7MD~!~cilpp`G8H025}|o8BtvDEBt=mvbcAF~AylT2WDei;?5)mu z|Le1?);Z^Wt8MS!^W67+O?QYF5g42pwAz4yb;v5v*l1q=r=&QDd{ zI!8IQ>A{k{IYMRk2iMNVER4J~IWYSWC#4^B{0*TA)%Ec38iV$Eo7a-$7rbDw2h6a6 zh@AYwNf6OsTk1m0$;3H`DsEN*-8}{>Vh0A#rg9eRFXrJ2#;eJnfW#hGFj;wqD>xn? zBg_yBNuQTNrZOPPgd|tUxxn`N5uPtBTm1|yN551Qe-ozjxdDg14+wCB&ELb#cxf5H zYh1>vFs{eIroOxREC!taBG=hN`@Q-q^UtM0Qvrie1hG2;DT1s}>(z!lr`8gZ{zvS# zP>m6tCxh;PTG!>k3`vL`(haDJ=;_F%fIRLJ#-$QenBlc-Z6!fD5QiV7wbEn|lhH~U z8S497$knmrAsFM`T;e();(hwv`}jbnq+bhGSy6!f&4WGDW8xQCmy>w1RZM^W;w_vA zUlhX9j8OE;-6ha`l65_AZ>dEOJ>Mn;0U?|~H51^tneK{X@D3JaY@RF-_|Nmp4`&ey z3;txHwB*SUS@?Q7*a`FpBoqLYkJgNv?teM%uSv3xX z0Ji4)FD*lK!+b!m)pg|BrFohjf*-0s+HeU7HC`rQeJ&jFa6W($FF;1K=Y2dj>wXVi z0)h%3(ikp6i4r0`Csm<$GS48`{MAc!-N)vFCD%Qv`g@l03!jsg*&V=G^FXL9G>=c1 zo;{RY&K16B6YT-6HM&ygGUOs5vGht^d6)IOhHJ|fltzk{LYy9s=Bln}{F@+fL?!}< zF1OFegMdqg4YP$=A24p(ZvsGA><;gLhIi>cQbEA1}Lg5JLZBq zAV25HR0x|AaH2{;vuHrraJ`-5+m4xjZPms=%vORIRz3PR265K8)ZcOVW>#?l7rn3j zV;&J*SN~}!TBYQSVAF00lrTk@k_-m|vukQ2Pf+&ZG+zlHa)L9Wj$-}}cD4tNc6s|3>@}&s;WFXKWB(O~}*YGEA(!ZjU1v6S8 z1fcC`YbnSN3jJ?l;!t5#B&Z~mA>05V#;T_#*5QbhRIOcc7q-*4aT=m{9h_n8BG7FC zb|l%~yrRo?BKbAb0z?ebJBU{RFK^O*>=U+>!%(PwvHa=34G(ATa$LeR^2RO0vaiJ# zscF^Of``}KqzkJ_#9V3h#Gyc1Tm}+G1aFg!0@U@jD zpf$~-Xn@Z``cD*<0FLtWf!h%!1TQZ%*ihsnjSga1pksVW4mXV1=gk4Gly7WkSo=Nq zJ(>uF*PVe>j;aX6>R!zPeAD=D5=DnJKHFAa&gy3*#0m1?&oHxbTEmu80^wQthEKGo zusH_yD2TL>nY%<`1b!N8sGpj9a>_^Q*R}*4p1S0cqjG-rKy89_+3@%ZrS*Mt?jC-f z=$UY4<{SMsQeig9oE6b7Jv{vT^VN_t=ELs7Gr!ZM7gpX<3xfe~F^;Hy)r!@tw<2+3 zzLdRs&WH2lHb(1W)`u4bA&L?@&7&{|RAToVKZt0s;Pq4_JL1CLM-=HJefEd)Gb_*( zBM6k%6l48#9pO*#000swe(uJdhQSVgVi==tB@3Qr6IjO#R$<_spa|X~nrxO_$zMY{ z>LXT!zHcwxJ1Rj_l1)#V7vza9I;(GB5C8&rSOCd?U9ipxE!tXJs?miWx#RB1=KUTX zN@NwqfkGet0?z>*gt9pv0D}fmIsU%am^T&R?Q%T0onD;qD|ZJ3+eA|Yg2XNtbrMDs zK7^Z-=2*WMmn;Mt6vTN_C=jUvTo$BrLU`Q46@(v@!t91J)g-BhVvPQafTKAd`bLCs*eaMXqKvr!Tx_*mOP>z#o&Qd<1=VmuUKuM?tk z43LYg)(NG{C2v2IN$kS@{Eg=Sntko;F!9iX9Y$Kc_dkT10#AALE2tO|J)U7%WC{P? zr>DIP(39LpPKG(@%CW|tb>@@R0W2}dj6;;j%n@|mU^q!s4wL|0J#aDuuhps@U!ZM+ zNzotoO(J?$O zx;VLt!##*Qjel=WyTba{|M(5ie{SI&>s06*p1z5`@VEJpIp`#JKkr1F<1*Z11ZZVG z6pic~O|E@iq*v!ff?YA+?Ag0Fv5uIFpl}zw5t=+Sn{o+U0tG!ce^ohAn-F3I1&N>c ze%N+By_dLf6F)xOb?q=PU=d=8M&ri1V{5iHm@{Kz(%sYSjE^zh;peQz#ye88Xr1x4 zOWecjn?Q(HsLZq|%x2+m%xfzwtuDcUme~2wHhqKgq~4_CA-J%HN{SIk?C zgm^T*emO+tLg59tUxFM8(g+iKKcLHeY$@g#>2D4Lq-%{yk>D#%R#JcO?NHJ{ZASTX zfjl*11be;?p+lf$65yHsh%l8d>`+wl+osPA(H9-L0D7GG=+UF1Rd1wIzq$Bsc2fdkq2@F*Ae*P>1)ER$yg;{FnP z^T!t#Vn9yB$;%(Nz=CDoOIlbsz9a{L!&w>-XN^3ZvRxg>o$=11~6gD=*RWHBcZhTkD~<%xgCgQ1YadD z&CjWL6|@N0B}Dw8cIA43rCx33*$IGtFkx)*^)1YF#Ha@wB-(=#pj~JzUtV~K7_QuQ zD^s;alY!D1o$B*z72Y44Be}kh?-I{^e{Xhr72|)I)S?;|MIVODzE7=O@>uYKfW}2k zf`->Nzxv02Mc=eLm>FUT!bhdl{hJs!VBzkIN$o=h2?cjweVy;`4$+}+rxI)--y)e3 zAQ@?mKEQGYfbJ#pdL<#FzjLhcS1c}MlHDxta@>qC^@#i=T`{Zfr38^Y+jPYSL>qB$ zp{3Khm7)bK6;Lh8E>%{luj5%nBB40z7*&#{mMm8C7XY( z!nWpE?Q-ThBpTd#?!q3!O!IF+e;x>c=#yxfFWd35@in(ioil@a!D-%6xip8x0$|Hb16yBp<$l5mYm(iY7 zi_;&q`~Z!Z)tim>(HvZPmu(a}&KAFK>YD&X3-DX>;K74AB6$G*;3Z9uHOPHN5?e>| zinfqgT0!J>!^G%xNG73jt3J2Gr^lRR*r@AhZ@&*Iu+8Da{8!R@p;m!#_~5IGUwBZ8 z;S&aRrT*WTrawG7jQCqP1H|k=X9ku;dHcGSqHszjdx+r&`u3KeXnOK){^N<_O$JDi zu}gTuUY`!Cy=sJ;8B^k{TC*Wb6l^_Ge~DW2?^kjP8C{U!Y}^~YL-O~Jvl`mOp_hU% zV05l?(TkIGxN9v7zq_drO9Y&Y2^cjv=EaBCEY5v~)B-%nl2$qS(7)n|3n*IZ=8DJZ z1(XQ6=onwLPBpi>%-tjrs{1FgWST?@i*FA8pD^04kGcAoc2Iv49vzbB)3+ZRKbHRB zg9ENp67V_j5rIX&)3Yyc@EJpk-WZ>TKz$ZMr_NY-EH0Xv8X!Qy@E_L~o@40|G$};m z0SQ$DnVWtfb7JZ8<*#oX(zYR{nEyIgX-Q6IMT?N6cn0_3Uy7=xTN@BHIQN7~(wGqQ zd;W+A01%=%g+BA|Bew#<+vJYLrhqh%E5y+S$?O_QzN4@O`R?R9>!bOlAt*dzv$=Mb zHVf3}FhV%D&ZpI>1U^fH{rNKD^wAT);dN2$iZd@w_GxE4nB0BuerKPPGWYIo&d8$X zk(d|flGn;dpPt#U=2?Z51TF<@@&N<>4}${I$*Yy*)nXO&Fp{-MxB?uGuo)*0!3;yn z1TjXXfuMYf#mi_?n3nfW0Lr0pUpz{fmHr8V|GbN0|#8?bf!#O0Ax)2Sf$kMs6X#fF}6WOf*>Y5pYg^s&>ZYGx6-^X-K4`;A>_&3)MJCGDf^G$Bm(a}oP z0TN{1t1T{wsY;PQ!bK{tius8C2!rBdT+e13_mC?5nA*@WA9CLteNUeK665 z0tGPN#3;*!-)2tP)6sYK}V z2dfg3lIG27K%g``c63qlQ=l; z11it`l{Y&^N5T;g44l{Wv@c%&g}vSEF!QHfkx}LIZu>o0!e#F9D+FWyto}hjJ|*>^ zhDlO|w6!z1nOBZAkC!cYSgY8)-j`Q7Vv<66q`gvdwJ6m&p)m)K<--o~8nrTk8!cZ- z`%X@>QpE0T-8k*>(Y5bVWd5zVMY{txJ4C-0$M|&LZ^oC<(4Ji0t`(SQF)tmxyh9kV zr2@!?n@Rcj{phTd6qk;Qly@$q& z*miKX-G-N}GnP#YOrWiLWJ3Dk@^;EmV5E5bUL|fE$V1 zNSy!P_8#ZE4;SSw&XcT@aEa2>uWKPwO;J>oOkIhT@Rx&Y6j;~{~bMbwuIg%X|^HDKFwG34+rV?Qi| zirb8jTw0b=*U)8$hJkOx@nvN3V;$D)MPS(sT5$dw=aYw!y+j~GY!+GyT4-|`8Xlh? zFXodos_O7Ei3XC=bs%&G)!sb_(MvAHYWz@F{?Dontee2QLelemi!|DM;Ug#clbF9w zzqYCtIieJ8x#lM87?*fY6lRt{qCqDa0U(O`ve%O^M;ByG0jU%OGy0`SgSr;2J(9<_ z;;MH6#z`WYiQ&x^mkGh%h{O_`iOH6bx8LfY-J)9=No`4}kKTB44Um#E1fN5Pdu9Mp z{l8cFha@v#2?B6bfDyoWV8BVhy@KWvF+qTH=NRS=GQLD5f=`anU_ywGpdd3P0{?e3 z051`4rl@$es@UcRs{T`KyC=COE}fOa86wDUhxCjXCkg!Hzu=8D`fH+LAy6wKUl1lA z5|V(MGJpD6*2#ZQSF-(ICu2Tj0|Oti#$e~f(^fkYC$V0Lu657%D2ftu?vSBwKn4J| z(nGY_)k9)f@y>)V*dl8{wt#%>^8i*6_@0CFH4iN{DmWzfJs?&R!b!VIIQ5TQx^Uq% z>UZrD)mfC~1IOwxNGO=hJ~8#df@eRt04#(%fb%-YZ}68=@+ zxq>z!=3&lBT^t%BJH#`GF8r?EgM~MumPG10&d)Rsc7CudBp|9X@A~50teAP6)@#1= zU2r&a=-Z?h?FQD)-u3TnzlY0MKm%`8)6Sjy{BzEX-q?OA2vOhdm?$tJI^^tNlhSqP z!R}dR@>ml(+yx^t!S0AE4-_jfQr%0yNN7HPveANAe=rzPJL_v}KVv}_%iY~^i=r#; z#(EEQ6sktlJAIF_v#x#i-z6MqJogepwV-vC4}L4aB5)H{w0g+ ztyILDmB678IqU&!BZv=@OvFIxh@o<@RWf%4;53pu4$CtMks?@x^ZInvdvHA>@1Ph% zk~~~8K$}j7nf;4p@|W4>v=KEbXy@D}_)nnvV?_J?_PLF=c6OD`niohehE@<~m&gXl z=7^4+&Ck;j0tEOHI>mLTop@75=`PBO3exC(aBz%t!nO>cJ4BR7j8br$Je=^1&Ia;N zyOD!zg3w>ka=Vm2eGdqcHeUB>N4D&MDIG$jowA6;R2h2nO4rFtm=%M8a`<^boY>TeRu3SmhFZ6F%h2M!p@dj*JN;kHT z_I1*(^+7Au_}FDBQB^9`CvIgYP1d$uyOd^wNnc0(5o-@D!72Y&tbL8+tt9KN@Z!(4 z>$&GIkC90!9)I8!bfqz_VN+i*Rml<>W}PFA7mjw;kI*09*aa4^>78A!f7E>|w|Phy z%2-}i9GdVMsR1Od7+gr*x0^PmjzjhyK}0SgvT>iMYXhjZL0l!9xyL;u_;>4b|IL}t zC^7pv_6_X!7$ROjcJDr?@*ieOuV_Q0QxD?igWuS!nTWhs3`qcqV?kQ^c@=fo4Dcq$ zvY2!71H@>*YI5)ew$Kn&KG?V35~t?;GsNOgh%*>2-}>FFy$h;}m}S71@sLOe!dN&` zNtz#~nv2oVVYs4~k~I2<^dFb}x(>tvxh4tSaSY@RAEZj1862F7#gGD!9w-AW5lOaz zT%C?H5r8c8{n)2+6kU>Q_wmaLd}c%fNH-k;V1CnSNA-jQ2)J`Gl1PG)IoMSK_8|iE z3s{Pb{gu%@nMen5n?fEd=qE98@fW1Etwu16xyJw5LQ_bd(%xQBvJz(Ud{3QR{sa=*j%ok4A^qv}B?dWNbUcs(kbE*k4P~W4gN_?6 zruR07egSaAZ98^oTUki}2EJFfTvyA*MS(8QA|~IV@hG(cvbZ=p&o>Jn`D{l4N0-YY zdHwlhgu>ekA&-!P5(Z6TJ0x?g4S9F9xwoSl5iYSD%{(3W>qN z;j%wG_sj3f`-j+bh5t`nnQcfy2tFEgm))5+QFBPQLCt;C)f6E19HA;Y^?b)sUO|0} zTX70d@b3M~Y@H7&v<7GfJ{0sA2MKR_cFJsDIGA5f23g_w;#S-Lps(Dmo4MFThq7I1HL%g#K(+B4ZgYx5mB^kMY7} zZ?1U^fF=>MO#T)(YsAFCI{q97;K}$tWwMGWtuEEOMF#POKw7cxL!0{zw#!ZUZ+cE7 zD}TPsfzQxg=YKXOvGsy(3tc>EUt+q72PP16@C}@~-lf_Xw}ZD2L*`nm6H<=bHBY|95{i?4xsufXRrb6rTyFHK9cy z<2xzp#JgDPhQg|fV9TRQwLwb`$YhnCdEhk`aH+aFIsA#RMDX1bOJ}Ok;#I$u={o7! z4}+NubnHNB&uV3-H5*B;!P6sQ`fiDHa+kP=?9LWfT*>gP^s)HE7iaOVwO8IT7;`dA zc(jq(h;i}z?_C>~31_OW$e3YLxiMh2q5t5@%|`kZp6HO;^m)HdCC^VE$s9j!p>Oyl z?q_1s?S;9A-S00qYx^F1{*%5OvsL+wui=FKMZdeub$2v_dY8Sr*Lb0RE$5TOSPogO zXd8CJ`i^f4L}h;GUg$ZzEa}4q)ql>g?lypv@bn(8xU)_zPtO6pBdPa&G0Yc|Dpil( z#lhl?zD&$rcvISgvXqkkF1s-W*sy=?U1`0rth!HvQ^~Gez#oW{E<(vxDOynNo}#sF z&Xb*vdp;6QFlK%HusxNEke(p8!15-VY*qmz!`ffSUX^?19NHYr`r_%yp~u}z^jURr z+6WXyay!78Z^epp$EWB85ndt(PA?EntDmi_bDyARBzTUpQ_Cv=tj?qT%^|MD&I?l4 zilk_wZp+lDWh;toHWqvkvF=G{!3!oZeVnf`j6jJcZzvgMwR zbUtB!J0kYgGj%X)NXc%>r1e&ydWV z%Ie*}UcVJRucMWkQQf;`(%W?Z!Lu1NF5I1-tpVJJ-)m>-@7g%*$@R{9lUdG#y-5=~ zx6XoK3nOP5bzk^O*T)5hu^*CjFS|uw#`Y;fAjLYi`Tl>B7JrWgD}US5fL(v}XiS#! ztVdQIVb;=()A|n(O>+AwWTYb+%G`HNCt7ff6U=yW>M-S;12<_2b1uXo^~3}ObO#h) zMTi0h5Ui~t$|CdW0cHIoiqHoUK_Kbl2>ar@TIJ-=Wn_~TiEf0O?Q79tsw>eLAQmyy zwX>3V->_M|HoW}7d17`0!AF8}i7kMLe9dV0F0JkFe~qX==!(b&wTEU1eb(g z7+N*{H8#w|Py!$z!vioqaCya^aXK{)3mY*|L$rtkMUU$v!9rM?BYCPXAwdGE07s*3 zQL+&uZYR5RX<*#ky2BkgB;^^R7OqEzkFgWKY1)9JttE>zm3-ybsC` zeGvfTe{^=thTQq21^1uZ3F%}>sWO;vVWX@Mc=37eB*`q4jo4?Cz3*?g8B!#RPum8QfbzVBsEMnnBZs(BVuO0wAlD>nXUn-pPQ6e{t36w z+g}y`Vif$DMNemIqR!(!iHYU=t2I6>yl4J7Y7za!n;ZU${f7=kB%5j-uaSCu`Bp-5 zrNPZi20Ty)Mg=nj|Xu$4@x< zOh;QMXR*ztL^$OKOZiUkxx7~BI#cb((>zVYbNci!jIOg{`9lr8qw zXbw+Jm|QL5)9c3BGSTAk^ruD6XZ|*a`eEGx->PB^o+QGtKR6*mqWt(-zluk-`1EiI z6EiT4F-MX#=jCnxv)ZNtY(Q&F^t9GW0g3GP*L&W%n^|m1PTHecJp09 ztdoTV1T29jvG{`c=JhE^vzq-%o(tuyb=SN2{0$lqpcIb{c8uR-;MC_rHwj>Y74u^) zP0bAyHtN^S>nLTgykZf90uscbQuO`_mEzFk)Xu0x?D_UbqmMGgLm}sgo-&MeD1d@B zIn2gl4LqTJS&tso2Z8_j8wyA}f&K82WQ!hlYCI3BZJ`ZvL^gKquzDPa{*LUAefhq) zRuJnf^e?8@V+cx#wlDOt6Nm{;C|i1P;R9sO;akmWy|%8_Xle8EDPDX(|! ztwVD#%)MLvA+%7}c$w+=Nfr7q=PCBr$gq-m#qU=+chqNw6=FUeH}8#Rq(7|fzGi>bqA&lBBQ?WZS)temF-teaJ{Hz$o5nC{!}(F;{}`)UqQVG z{*o*F-Mb+vva#ehGZlKJb+Bt0&djw@Q|~HFXZ9*68ey{z%Vu zZ@-wpPZulaADmQGJnVjA#%bhssx&2S^Q#M6)yhPo{MVmlTrzDI^YJ9t7Uh);l;K>< z^r^E`W6_bk gJ6=jz={@8c`4i}Q9fu%BQ9I_YdUJ}qq;s5srISS>3tZ+f=_-H<; zlj*6}8DYuGx}J`EmH6T_0j8VN&Eme+&;}uV)H6zuf^EwrM(Zg&wxl}u5VVfV%)PZfm;JCKd>Vt`SKf- z;nAX*Qv|m`f3SJIP%crZ%Q<+u;60vAQfK3bXF zE-yk%sJKaEh2+A|9~oO%SXgGw#%Ov>uL(Zeg<21Rsx#~e6wnBo=vrsMU7JeKzUog}N}Lm>)jUJ0OVb7`JG6G8ZR5ZO8`nj|Q9s zApe=OAx>*XO{ax>+@{CQ-1XscSf|N>#C-Pu7^Yq?K4rd2VojxI?~8Q@CXsP-ON&ZP z)l5m2MOp75dEY)Fk_nDvzX!qmmi7?M9klB22mPTTB|ij!G9uj|`6|#}03xFA*&Zz^ z!K4h66@+#)-)}Q5kPtvTU4TT|gWg>RdCqiGvIyy696+UD7Sp?DT6CRu*)*9T^^|N@ zfMnO)oO@>G`;@1fp~)?|-x~LhUp=0__>!L38fIM^`3+@CZ%&_Quznt2aLn^@vs#Us z=gWb@oPO1z%5>e>yTy-><%J(}2t4}yhRvk~Ocyo3EDO*}(c560sZyk2w@64&Y{4?& zGo3m02WzMM_il}xv)rgo-P`^?NwgAL|ATWhsYTl0UP;-Br8*^oZEPWrn?JwK@LKH< zu3mfM$_L(5<eN;>Gb%r)P;A~+io%J}T{$gpUw+Tq8Z6Yh zukWSPwPTvrR|P4z*=r<5XEp}D?VLCx)ayQ;-B%RR@#UMKLEjy}r~v(;N#$to0o5bf z1I@0K(_KmOzxzh6H;tUjGuU}_?zHu^PKw{akjT0f{N>X{De_yo50@$%KDpmNz}{ZJ zqQEeEuzAPE$v}PaPPr!rXIh^rAm(%U>)R(d&MGZWw9lv_D_l_SG*s!k=p*NXV~mn}YgyuM@|Q=2*K6 zYc`5#{I;21TIFZ^ai_yrsnZ?b7{$daXY83)&Hcg~J3mljWTt!L$aIxy3QzQ6|G0EN z1{Lce-nBE6PuY@X?Jpe`l~-S|edgpsiT1B73+MmV^ci2h)rDF<759X9+HsDgobb=G z@4rrTbDFIZ=D>FBaBvwN=nTYJSlyIBTmXB-f@zG}WlhDo^Cu$(I?7Kx3)8z3pp)ot zmJw{0ajn_+@yw5X_7NweT!$nQM)u+^grfF#e7vUd07N)Mlq+|;nt`Oe0HiZL2B;d| z;LJ7t$#tNJ5$$}}cC|pn`SY*PNynWSS|!n=0(Rj(lHU&$`9AyLf)|0jtuh{dF{%@U zW$4~S0qOrPv&PX1Akeg&TqYM_ygy&V>K1-eiXc1X(oT&Qw0`x5sVZYxs04N;Xxv6n zS1K@T(jo}|90Xd6iG<8q(G|fML>@QyE5&W)n1_A^2O; zKYlo;V|RjWj%mD2Z8iBL%XkWXu*63m>S9JNPR=*jl=i!~XgV!@1WV*XhGt`Sa72^M zL7GDF1(EN)nhy^4j;lDBrG=ku!G3sDgZlG2*4z4CmkX?=*?iaq_4xg5LMc7vd0x-Fu*0) z@d;TBwj=_9UAl2&OxB7$k+5Bpp%OwBalR`~a3=Q>m>)pSirKyA`B9KV5>^iY zrn&TlQ^PaZTUYy;?RkCIWwUUp#-FjtMpnOM_7&$EguP3-8ZmpNE@WxeBs+!i)$~@7 z`)b5H*}my1{t&R+>z|A>t+5?G z^}Te|d+z#zj%&45zUsm%)}_xLiZN|TUN`69to z9CU%OBBuBvL5A#1hMSy!oj~FF)>|ZT6NTkN@G^Aw+C{!o#2v_ZxjGRQp15kEk`(9N zFy}cE{}7_)!xyf@_?}bWJ1jKxC6wB<#9^3i!o$NKxPUbz#&l%T7K7D_Z|Gg3r>AD> z;IQhTSNC#^@Rq1<(lRm&H++Mdh3pU2fkg+W+hwixzaYY*HP}i;mV-Iw?N9APm@(N^ z+G)(enG1r8C>78+ET!3DfGoI`O=bv$6NAnB09Tl)(SRsL@1(M?HfRvi=O(8TBQ=p7 zP>3o7W)r|^&~9jN-8$sSpi8s{Br|}BGL9X~Fm!Toc!d^<2;)FX$naiJ*?~sELH!U$ z2ke);eF(2|LxwR+xDGhR)pZBtt_RSEWbzy*uN)fpf!19ddo0x+%zW;RG{1`L4~{W> zSKD22)X2PBM;*T&D@bs0Zz3n}h$aX5Eb_XBm1TNx_rb&43As`c5)_QV#&u-ttd=s|O```Y0IU^F8*2*K^#jC%+KFI!Tn1UOX8aCRyW5`ck z#B3@tuCllI;f=`#>%r>C(K!?p`bx)@&#xHqi37Zy2>R-M>q3n&J)mN^jdH8_FctQSH_VB-iy^_$V8mJ3Lk& zky1yM)81^3K%}!k{tl7FY3rTf$Def#D-&cby`QDF4DgeZoLHRMqBBO zUxOch>tWCIe`l0pD05?nr0H6LaPi#vrEV(14fDhjc!*vY@5O_rXYN+(1mqH1xex~R z2FEng!t+u{veX8=H84~P82FK$S|Gd`G47Bke=!vmf$HjNzKtjD8zR#N5%2U3Mi-(Q zdXLqN$L^AXa5Eeya=YSHy)&r6N7bLcw(Q^z`|iZYGl1jVlH-~(f00lK zto3}XOpJgK@uhF#M#9BmJW-F^Z5$aER-9^-#<66{ojq%MbS2!WtdU06C%!j#CQjcD zS$JL41K*v^#%&A5lAU)gPoXU1tGj<=ooBRu;JiepDC6(vX2T|*2_7*vxlVfw zRkZS4)v~nfMGNc>rS&V@wr%3J@cOOwi+6-a;&Z_y&x6~G<-4!8WvF;w*}yutWccEP zB|@uTze`EU$agxj?K|%o-^91FBFZNW)+DU5OwLhepkxZIdPzH+ht*oyRejwT-;xx4 zbvvZS0-_Y<9MO|4d!wDU;ETSqWh7~y_Yx6$j-by$|i9Bq$OqKnfD^fp1towe(uk`&l+;_`_$qI9SPZ$*{_H8 zZ(X)6Z|W9>swc?l6z(wS%l%J7^pWt-{b#nLwL<-%=Bj-0*lsN=yca>4D@`)X)W6}- zaR>`Hw?BEZiB-QGn#tL7$t)7NHPr&4q;O5>KxL_1b#yBx)f;P%M0~e81(uiOAP+ZK zoHffZmKSa&W_e=Gfw)-Q8Z^xdq=JStHtxeGseeR)kM#z!zv1%DKK4=R!#B(4&0+EI z&O|~CP9n@h&VO-TLG=p|Bf=m&OY%gx2G0-Hk-v$+4o_lXb;c%1LR-UL=p*u~-T^15Q3 zpaGDv&GL;ulimdfGm*qpWbMzPbPanY-M#ygnkxYtD}I=^Z-&nh%`lH#8$K`{Aj#06 zd0>=m%)w4+UaXYJIIxN8nUeCeR}FlIzKKaGqx`w#?Fu({*xA`#m|(Bc_;V~J5;Cz( z4RE*8#{+UEIa6nqfA)kL$}HI-c{!awyrgu*T}3!6ep!M?C!&cuHD)$^$#x|<^f?OeJ;^!;@wm znu~vccJP>rC%i>S&jAEMWK=04JUm!(w+g1r036SQ9onG~qm+QpscvlKTcdRHrwx2l zfxRDS_{POAm`rf}x|ZdBA|EmmS`a>#gyYz~PKH6((rnq8qj#b-q!V6#-Fim#G<7$n z&Tl-gglp67`NtTOc~0{^6Ug>bv|{s|*(slV@lfri#C9Pe`ddR)&CH}5Vrm!7kMEHH*mw1HB(JBjY#x_@%z)gxUg{5Yzk#j0?QJDkIrEKU zy~}D1zk83yeBXGDK_T+l?w*h!<5<--`sz}WQiM|f%rfa;Kd(ZAktAj3cBQ`%cLn}+VNB>QmJa(KqrABr#l7ELL z2?9S8X<6EBNq*6bW@ScMQAdj{tW=on;!|Ht(}~=B8W->2yfeF*X+{} zVFNjfO&JG)g`YP+ffs{hf#MrIGPSPclt?6@%(26g;ykKWh^`NZkDNSoyYhUj!V^#S zG{6}`3^X)d3zQvq@})xwX#jTo*2!=Dj9doZWUo#i-{LI5-L4M5~Rud7>uv72enM_NExh3xq?-WZ?aH#<`~3?LcWjNPZ9o zL*%oTQHf>)nIYVI+I@X31C6j(<~sK2N+k3nRK%XB9JTceFZX~hAVHgF3~&@3P#q;D znG_Th-V3>Z?)Mq``SbYSxKgC7DHZkC)Yb-(09h!p4R+o!5Qmxv<231(EiEnkMow}n zcH-Vb9-?=dmz`M#T<3CM@~df=-!jtKzyP#1=@!(v(64CTvu_sJrQNqBaq$K2yITxy z_!OnwmQH99)XtT1&U-S~?ib^LvLA=$Ag7ekX+6qIMVTP`fxLZz0@)JFLD z#jo7Vr}naARDO;j?NYB;-&)luj#zGyWN*{ut@?2ZL$Yj%QD~;A797 z(e<}qlx+|$dhNzk~UkUgwd%0NNqzf@V-q?@nrrQJF! z0JvCgbi+{9dt?={+OymWY0`(!jEwUzu)Rd0ooX9|NjMZYH2l8VV9TKxq$H@g zSP}oXc=VV8?sop(r~sH%FBcV3V&zH}uV)8e*dQvl7!Fwu+&X0Zy#{m{GAzcuio5UC z#A8<{tg50EF%D*|>45Wv1Ezm`?>bWM`gE@UX}8}vw-gE}YDf<8ee@{$%8;&iRcEdw z)=^1X1)<;|)p(E%vj28U$^v*OzWF>Af&T<&wH+ncAR{?BB&?fD|2ndLY*5w?{w1yw zq&l(Y16wKGRHBcuM?N`_DVHNx(E*9^dx<5fShC+xG4`X(A_Ol-xu0>(Afby`G>hA5MR~yDiU!(?z zfR#TiBBF#iczHx=*Pkw8gMa}#gEqX74ZSMy_rJ4yoT2D0ETg&U;lXgbX95*{5tvvH zFmQ$!-mZx+LxpBg-*Cxm52eipQ+3mDrS+lcWJd<@& zX5HxClzDtFbAb$P{K?lj*ZN)_&*xwD0aE{Tr4E>d=?1+PF!OUb|9hWFAaAv9eCcxb z@XbH(ufKAj(7UZaWp$SI=OFb5)&c&vzuVtXP-P1asrRe8bwy{?hIe~N#qUTrWiN52 zfrjFbKXPk(6yKdadMD1*q&U%Ur2pjm?;PWHKa;d6NK+@8K9|AvEre0Q#UEDd{iuQS#R%i;5b;1I{fugq%U^X3Kr5U*{uouQa8r~7h(0ileN zQ-*u{jDv~^f)YGiPw^ts_Z8|?d>y>(-N+*F9M1e>pN{&}DaSq!Nx((gn=K4f51gpSQkBzEx>hogRf|a#3r=VaM8Vo)6$kL@1 zYH(_j0kF93bXBK)&A6bTAbQ9U%ny2sxDnyHz)G4FFzh#W-1V4zntTL$y_dmkf&iWQ z*3@TNXds+1NMkp>ye)utAlrIAS%tht2?HnDIhSQ(khhe1&JFRAG1^kJM=#O-l-;# z%`1PNhMo+_<@*8#*_rDYv_&v!(0;4l9t&T;no*aq`MD}vP1@Z5DY|a9`QyRG)%=Ri z%Acm5{9Ip9U&Yw$K41xu4T5kMOeF;8TO_KU-Y7A|Zo$#iyk*N4qD+8AGsaNWrF|Zt zg?X{N?jbqkmK5@eiLyK2y%)ZLI}@NAZv)BzmAI?kL5hH7a3ev!AP(q_7gD|u?1Knr zx#<>wIAMpDr~C*LS-_7L1;f}PbZ=v8cWW<(0Qct2b0{0+k%px1dN_2gczj3-2(cNX zI9yqv_l!vL$b<*`E=h^NQGMW|&+USwS18draZbJr{Trf?a1Tax3li}Nhumc4#DuLGud-B2gt~hoA?cg*dNcP- zsMm1~>kUaDp_srP5C^XJOOD3xK&Q#f!cvUw1vb_wG{6@>)uzwiwsYsz;c_UYhQ`NE zEl-duw!4ybyyy56Zm14tziLn7amU*fv+GDs#uJ6Cyi@h(d$vJkZMW{F%^Kalw&`N` zz!%T$zn%+;!Xn2(%vAgsZH?owA|0tx+L+HY(Y@N|CBEiW#MUEQIt&WePj+VhYDK*4h-A`yffeC32q7Z7+(;-~`&{5XQ!Kno z6-9<1!s<5RFF#%2DhPJrP|gQ@?}7Cobp!4PuD_Y7X!tS8BiGC8^=>yka!3^6TXp+x49EgA5GJAbecvWVqh1{Ptb?4}J{xEw z;pMwrhwFyG9XrH&_JDzWpx$sqfGjJ3vM?FZ_1TA&18WiBo^hP zxo#qm*`w^P!)_0QtCS#nL*^QyE)v&D(@H-WPNp9(MG1GAF?EFcewplJD zS@=iB*GDX2)!XxWg?45X#udC^*?oB+yA(y@^UU4>^u3Pa{-_-|A2ja@GfC$XT#p~CJ>ii!i-Y)55&-jv6 zw{6?@{I+JGGd}$kk6&Fh!xA;u3@wRT4IWqW{A0_*8<9L%OPRxU0wI1`c_XtEk>zCb z&C#PPVBEhmfhLv+Hu20`JaJaLf}J{OVIWLt{XRw}*<(%XrX9r-jLEt$qTKVD?8{q^ z7>0lsw;g_C&cdfh`XU(n-<&Z^E(YL#9?P!r`EP<30+LoO9#+`EY5B_-S-yPvLd4_0 zZ~h$&B&dOaoXGEiTIdoLNg2fe-;wany+cR4Rt>OzA`VXvrWfIzJ9nZy?3Fi~T+7Fb z<+fr>i=L_ue87%WnEVJ6mhpk38b=kt|NC}tnX3zdsNm!nbhP(3Z-rn6L=n=u#iBdx zbHHa-2oUBPA2{H2sXHK>@buwMuY+0XP_Ee$*ihW!z+?@Z%U##frFRzi_4FZmW`vf} zNNfK0QE7SKp5&BATXYY}7;_`dU#Z(r~E|3pK7-SxjK}JJPZWlR( z!ljOR*2ss#tomSrEEz+hi296}L{B^`Q{cn?K{YhGIAAy}4~$Jr8t%HnlZ>#=`1JH0 z#*ug}m|9@B0`UdJ7Kuz_1$WObM9Tz40D0p}a6^$81mr)jyg-k-&QHZ$NF<0qJ1@&9 z)>fb6TYE}Lz`COGM4E^qAmfb;^Hh9$Z+UQW`+He%3>O~C>7X)%RHt!HUgxpiv$XoQ zPJsU0@KF0rbmSaB&~5!c8C1KUp9=aYcXZDxMNSD8ZT5!$(%_aRF*Z|%d0eWewXr9I~bv?bMj2PX$XO97g@6S>zddrk`fg7rDM z6wR6D4`S4eg-bixPXv7Vhg>BooXM-w=TokjXE_d^DtD^aVr_}zdrk(zz%%Abl=}W33^XpULx$i}B)x3%B6&dFug7 zXzTml*?IWzav~o8)%TH?C}z!>Q#qCn#h+v&@p8QBlNI|sHR}UAz3lwD7rA%rJDL{3 zwS`kTqDA8o^c2apku$%+C;b`h83x`|C|`)WczW;tehWK!hx{dN3c<#Q_Z*!!Z_2Z1 zz14jq&u-_GrHl8N?qjFiH%~rYP|dRWL^*Gw#kg_d`jb7(jyA1Ja=*H)2Y^Y22jp`) z<2H-gcyh5w zmqcD`z45w+)`X-Rz<_>@idKg#t^=cY!hdEjE-#XWOB~^c!}hF6IC^^YIpWaUzrOu5m9)ecQc?xJOTw^V&%I{Mx4y5L-I7UIwJ6C z$RX4b@6x`_fJKFv2;b)gl^4-@7Q7s2n}KL+K~$AiX=G}84#FWQ*qNR;k1nV|8P7Gd z%r!%!)0kHHsM#O#c7!b})U0 zbqpuE?_7uAW(mo5GY_w}(?;R~X&O?H)j(z;nkrO0cVc2fDfT1kae6u{-hc;QHa5#} z!2TXKfCb|d`<3@VEl<+Ip(VF*fK`XA#+a!Tqy03LP0y4m`xDt4R`$3w>$AADFYYq* zDg1F;?DTOH%GXRvWk-LDwhU!ZNAc1Qfuvx2OST3HgVHR)*-z5_)F+p4+9698X6B%T z=#_REGws&%SI}cOX>L|-VL#=@2?i)oZ>sA}Hxv&s)2X*#yB>L~SDe`ztMhL4OQW`A8A2BNK3vn|W5WhTG)iLLZwze$MJfUu_9!C5#**1vFf^qMP?Hu#4_mway zrlzKp(`aHy0fUWFlY4*RL1-%(V28TWk_K}l&;rDV;+yOSyj}g{z2hC2bbUHiTVDWD zm)L*586goBQHmrS31j)aLt50dTElTgE_9bzhP(v|6D?fR{n#<^ps=bluyvrbb>5$O zNo^mZ#c4aa93BB#yHHsE)j-7~4dn}ZiWE?{Jd-iy2(TF?LNcTYU8^IJ%P@YfxDukW z8MdkPteS36(wT5hC5#v$8<+(4*|3!pO5wwU!!d%J@E#bX7y>xi3V95G{-k`wkb$a2 zgo)6i`S=EB3lo2n+ca<>5+Gg@KG|z;zfORt8Xc61@lDe_mMggDXVWyci33DW@;tqA*YFWX2`_qz5Pgb6GHS!$Y z6;_{efhYcq;%`TyYh?SgmY|lkQjfkTggT!JVTr9ktU$-{`E8%i@=_=x0RrF0>Q=m+ z8GmdW31#8FnHkl2S>>(mPpmI5y)b-0rP8rz0i|)Uckg}M<8_QOwRd}{@|J=_fVradONXA}-Rn%U@|rxC+au~bt0gte8fbC!C(%XUU&jMs#ayXhN8KxLsJBTMcz9-elvXquw`MJ6 zi+0>%6R-f+0p)q&Xr<*C>&k@7BBw6?EcC33uWwm4Tw1iAFW`CiI`it9jf;ARy+~#b z47a6Vaa_KXaelC!yE`?5x6e%qyIHi*T_AZWsR<~7JUSG{y_~~7=p|3%VB%A@879UE zfp8rUO>s4TO%8%eJRPc3CBo94`RpRv826N8PtJh^fy_(*M*c6Bv6~mQisY5xD1cc~ z#SnM61IY4NS7Icl^6;3LmiFU%q&z|BLxA_|!4yd2R)QJ3k z7U(A9{|#AD+?rNr1&t(V)i6UePg{|@rYs{aLDj^|LeV@t5D##U2T`~GoasS4>PsS@ zBw5%L*cgek7a?ho4nuN}<#aq%9e{O!Qy1u3#tR#<>@6O_>%Gl&xLZ`@=myIyU_ngWyrNrf^O zg8xirO?up0U>rup!uN;1twgZ+K70S`L-PNf&SaMGHX#^E$`vmQu?%mhU}mw@n4!VF zw+$8qn<;oHeXGtTZzcIr#3K5@cN>w4L9}g`YewJ){oQy;1QA7ZZj#Cvs0oqO3tH6h zwv71r_??|2lT*^NvRe==oG!9t2_yE*;2VZ&=(ca%;474DJO>JiL{k9^x{!VUzC39( zjnj<$ZD_Yl(lVz_ulkr)JJxwy+W(3#;WAV)h&1YDOL+88psSGxWz+eCjP{Rtx-Y8y z3VWw{@0<7RdI9aEjsHQwV?&)N8s~1dR=j$WLB0C>*Tw z^qmt}EoQ!P|C8AuCegxNw3^u@wlll)*3ji@Ftnk;dc2?54n7^4ShQ*^da&$P4ZF;6 zqEv3fRpqw!7A>KvcA0C_H8%U!a&3;#7@qGHpD9lbmm4g9%fUWKB&9G?1) z`WQJLb@=tt|5*Nw1{Dd0!wsnn6!?#IMq=E4iC(Ymne?2Ot8=E)Z=-<4Z;jSD6sR11 zeiS$F8@E_{@rH=3suDWsAH^)|$J!WaE`W38!EUxT)WDGwLiKj*h$A1}Dal9%9=Y#P z+ot_!W$#ti8eyI2k^e{6mw;2*e%+tul9DE+5Dn5mDwK#MN|Y%@h73u@${3-^(11b_ zkuftB8VsovO-QCBRFWy=HAeW>K1c7{_g(-0xz5$k`%66MdG34Pd#}CL+TkY^?!PL_ zRgrLtWK$~)0ign(C}5s1tH5JJ&luF>btY5Y_e-yhF7>8uXi=GbX5~e7S2*E10UP0* zKmYcjtd9J)UL(|cz~DI+qG-h#_!Q&mBDIpX$SmgUT?Mt>6Pn;o7PT;$xF&snf-LqL zO&P{Kt(kyNKFev9M*{wV8WQahPlTHW6n8(JzNH<1q$>=br|vkQicV7C(0?^YzLJ@bQ&Pu(Llv5@3@dXvIn~A*`rFVaAZgG<0B@>2+ z&b{U@51_RI>nl&{RI79{m%*+Wt>8SMxpJ|b_`}p92G7b_kG3Fy>ds&jT+sHpA6Y_p3N?u!gzS% z&eZ)g=Wjdm{>g#_MGc=tZY8y^ewlCtk0<3#MG?pP>Jd0`_h|yy(PX}8@+jk zW3-CbiQDy zr2{U5j7&SQw%~=U(`McRst&o_*tuu2pRVSqKeBU;>8UuY>E?d!$Jb$_3nwL%k1trW_}%U9%Wj4Y z{6CY`Cgups)VaHGDVI9SKVKB7UOPJKKjas1W&N7RDot+OD2@wzb7p_*%$dZ9f2iR% z>h^0jd)W1h8Oid^`5Ey-R?lNyJyz!F3D)klP&ICm*?Xq=r4LuuD%UMrXC`T`B3Um~ z{V&Kq3=Q1s+Y?geu}|i|XyjMB(+#yDF)+a0i>1Sx1p=0DfqN7W2<7O+s;kW1K`jZu z?1eyU<|$Wgz==kDL~uR=%mHOnvZa5r@fz>YhdTX#(V_ibm#GhkQ7i!QHDI^Zs|8pe z%`WmQ$06@YiiC&Ix5Fj%IN0*!eS|JH+w#~tIh@Csr$ix(2FZcW%)fyBwJ%6fPZ1Ta z`1_wFM-2QYTkKy1kq~eR(WJfO(5{nEiS8=|&}0$FwqmJ3>mY@`&rilBbapwBn$X1b zzE$0Eo7;fjS!{cTW0MW2z|Tz6(Kqu^3|V?n0J$gLuvZz!@V;;XRV$^}$Our+Z8k`l z(9-KnZqwwf4L$Q=V=#RcH8kd6obT(-Z4z{mRCf#`E0|O^i?cRPJ>k$sZg4?jLdse+ z*ZqlGY9)~2Fh|hGmE-JA9zBKtu+8$QVWhAubN4^}x0!d&J~Jeg7iV$RT=vP}-xOTw zSH$PNf|?oCEIcyzp(kOxB&s77EEQ-pcfB0c0j~w1)p-;mus+8VAX7z{!y`$G(OEJJ zgOcYVxBfTD=3ca@#?Bhn$AI3h*w-NSF%%DtHNAUa)|@%{o2&N5!@m?A^X+)9FAtzx zLM_CRa=k5ziHr7{z^LsDTSP?hrA79y;E-l?YUf@_#o!7zMtv`gV&sN75JXt9{ zS-$0vc;c^{e|)zkeQq~yA47k5fRKJ9!_<3n(BRd@}0k=D*yjy=&OT@3qap{nBnX||BWc0r3iJ7fe zjW3P3jjwaD;7Yu-h{4#F=bacexf*>xy_dt>s~Wj>_Ai`WmAHnn?vUa>^cM1x=#^p~V5fnX2pgF655d z&hM{0e7w^6b8_!X-)>1)D@hv{r8TvgkbT1S{6((E!-o&s}R<(Lj z9>)5PG`WvV!ESu9p+qWw(&mJGsOG^=m_lkGWceYk#oD^MmkBAQaD!9&(dYk$3zbe7 z?CRN2R&(9H{uNWl1NOQC_b`CO6Bvk2 z7-vDXd{mSG2!vW;Nx&t+=>^4LF=NS^HS9pvU^4^58cZQ%OCiR zW@yI%l6tu55LXaX1Z}7*VM*6)hvRq_r@eS|Dd5`Wy?i+;f2v+b3GEZinHER}itc+A z0w{g%a_OU;A>e}lqMxMu+d-nC4*Be1EA*?%JzZun@AJnmMjxh|e@^Uw z(lk@5@3+L>>e!mIjJIy9hwc{2SpKs*-*Dlqb3A!lJ?>dl@Ha?CPi_ceS19gRlp8u9 zFjGNf!D;_D{UYCag0?hYDBjSWt-WAVB*z}BM`xm@&x;qbI9t`Y{axCRufnS`*>;cA zzk2zy`^gvfB6{VgDR_mFVKMxpC=3aCzt?P1xdR@2j zk$n$@wJR~5Ilc{pMXyV{)xq5xTMK8N+PZxuvllMBaQQO3iHQm0VYcWlRHP6A;SCNb z=Ts@Tj?Kp$IR_A}J_EjGbQHfp_wnlYb+Iuq5E3~wH@#nke3OoLG!-^E{fw~u0l5(d z#egfoDKWAA8K`0tt|=`q>YBfU6Xa-cHE%t`o&bh>zO{9z>I^DgIzCx1`!!f)a2c_a zJWL9_`yv-#rz)0lGdJWk9Bm5GYFV|qqxva=X`AJs==UdvF>t$k*miUEoxV&;m3hA# zCa2g=ug#(G(XYP>O&{jg9)N+s4~OhzUXZ%bzFfLyjdG+3?pO*Ji^`5@x;qnw>VUOb zJZ6~CgZC1R6a=O3uTucMW@^i<22wXhVumm%cjP4e(P;@GG|K7oNpSl>*>&N^nBBOv zVzh|&8!=wpM2Gw@=D#G3TL)Wnwtd^I`_lH|AyM;nYqb0fMV-%E>{C)48%i6+yK`1{ z{=NgOC#mMIgkcKM82Nkm^;Nh zg}Y!UkCBGJA>LW%elFOq`24!%VAE2Pz%mZW?=eUo=x_0px_>)k+^DE~boppqb_u_$ zvVZx5`uJ_tn7+M#QzW{fgd`R8o+%!%n;u{}5+rr=gA)&97S#Cl9Zu8CRZ7A#R4r`I zw(D-@PSpMRXn5U@MYBQ|`#xS;^sf2g=~IrP0$P42k8+0fbs?G`2eNx}t1GvDTL0C; z;onU%on2g)I_mQb?oU0h6#LB2{kO6FjvYd<@$UzN(7^52a)L*b+Uz800vFl?Y|Gm= z)4>KL@qk0AJ|Oox=eD)LBPRve^ZX4Xfi-v~H1thTl6U-@Ea%=;WR2tL>1kS62)jG& zbknDYnm<%c_wK;B3EDPfg{h%Y_Cpj@OhTa{f`3#BW#*sP>9q&%FOiWkXoar2(|vRZ zYy?%|xnKo?=!=mgzvaa1$!dcMGe%T|(^MWcy`fNt8w@J)Wd(hD9d>3aJKFF(9oY+po zy;YAo1dvzmL=}L3y*jKYVN_xnROJpW4}6n%wyo{XK$Llm>B1n{iJGht73D-B(KHyw zT*cCt1!q0))e#Y{a8mv7d4j~1j$acQM>%!a{Ys5hGiEy5m(=!OV&#V3l3Z!ltk#eD z6v@m8yYu=DI5}GGN6dbA2j8^?p7Gj-?L0{p*5Mfw7@Fqe((8-e5^c377d%;i z$5rs6i=Nxf8vBE-e){VeSg+wRz*x6u#NK)@+AL(>*KiDrh&dwf#`;MhK|jY}D9p<| zTsZaYyVc?s`6R3tgI@qa$vB#KM}$BaDaIXJA62()*2FcK-;QJr3(vn5j^e}!A{P!! z$elFD!5!@Q7t}|j0z?Zfc&wpAzy4{Qb-r#$7j9=X|LM9_kE7p`sid?7+%oxE;*nA5 zUGlXx$S2-`A4}MM?S*4#)u_G2c2Wb$4{>A);@d{3k~9 zO-V^1E`6X*L$K6rBQ01|K&P|2Mc%x5L-TKuk6{84`gqXb!RH9xB@tjm0o<9y0Lt)$ z{1YT4vzE$S<2?e8YAin#i|Ll*hA9^XK2ZRNd?1T+m3LyF_@9N6o9Mm2gQGF(wj@7K zg7@*rAN5N71~@0ARZS<`1lvctMSV4x-Rlb-EPCoOSkrtp5;v4VN?$C3HLXY_T5uLH zQ^%wiyBz52j!4ZICZS#i>Eh&b+afp0%kL$Q8~I?Wsa;J*X>|t7vUyn9Y|9aITMyvO zn##eUi1uhKRZBZ|Oro~`4+odfrspSf^|22;e!F4mdw@)z+1%XBYwxoi^Q1uNg<}lw zdeCmx(ZY}n+cO4pmUF(w8ceZkP|uuF7ZGo_xK&=@^p+W0KOOyPxcBa9Bx9E^ODrjU zx%m(6p~r5Zbv!fnsBL+d;9NYhG4*ZRKm48DtJU*5U&=RM)|oUXQ^iVqDYLxMwtau~ zgNRPGY!O7()58_`7_zF<7KCou`2|N=^#11KAD{+lEl;3H4>a`;6*8rj4A5t*R|_{2 z3l;Uj;m|+|%_JfwTHQjm&PeryJD35B^oEINfz&~+5KirxRiF>WA9~hd^Eyz720|br zFRdUrkH%P{V0k@BLlu6=L@>aUxN;Zx5LxL%cn@k$e0z_xG+7g%c}^q|C=L_(|Aub)lM zJ4_6d2JoRg@6OZ^h$?h)>M(%87>RR%{!P;8)ki!=>3}!jMlaAV5)#gUyYewVR9spK z&m4V?W0*j|aCF}f!RHv7&(9yJxb}S=Cwf&V>+gb<2kW>5aMaII`GWyyUw@wwUMVIv z4|wI3LCCUZ?YJt_DIfFDD=h`-&%dPgsBpQtJ5Y$Q5PU%SEqx8v%s|905f?TVc0N-%s5jq=6p(vpA*c$bn4kO(pMyj?_Hj5(=zXp>X36 z;w_t6QpNk5{Vwuq@&GEBRlip`KHZXwMxn*HCp4cO(s1_~kkz0^e);)xc@%;z4vxCr zzmS4LpgI9mLQb2e#wkvkq>(6Xs~lG8bXVy&#>_uu;j zuA8B-e1YgDr72?l*?;mF!N2HJ5&OGm)Clg=d(TMOcFucKgRx)t$=?HCM^9Y}d9RQl z-tpoo4^Nt2t<>Qu40v*w=G{oK3^pCe9k*E5c3Ufl^NNM2Lg0vEE*55|NTumd zu+8#>F88JQf(fT#+t`458rL_uz>kmPno=y-P%7+`CtwBB5LSxvG+6khD0l<|0}Xzn zpyg6||9`Yl>Xj(K&tW?7o4T@EMDEA^xt+%iwFIIRA3oTG`Hxr3!Q+-V+&Af21 zLRJe(Fm})c2DTwlZds8V4X8}>(mu3Qi2;vh^2B5Pt~+4>kO}pMY>JlSx45#ky`Tl7 zd$)=@R#=}2`jGI;47cp13YJ*p$~!rB6IWeNmKj*=e9{a-th72Y^>;UjStLYoq=c7{VrP! zdmEc)jxGs8R!`jD*01c%>+i7L$rn)0&-Z3u=hOY8Zi``lwJWnT?Z$MA6%rY@I}B6B z7T-8NIo7zx9F`J+;;9U(qHc}!#OeON5FY?bA`);jBL^LIn4-diJ|Gh5g(wCupk91! zj;RO97~8;ppu7IK<1F*Rid?qTrA$ut2hSThGHA)#r+tW{Sp zl-euk9EdX<$PbNfKI))5YvSd_n`dLClD@ggb1VYj)~J6#B6Ej$QdU5SPw@qKH;d9% zfp}?`D;gWOhJPFZe8YA-s`4KvDIyij*9G6sv%b@&`y>=oV7j9rL#(2Z5Vl%Hjf2dryP+IhpuUkTC zxHETX@%zGzdJnTjqb*aFg!eydnig?8gL9mLD)$?7Hm{GiWjG8Skms>FI=VAPu{F@Q zGzq}g?=C<0zzXv6hB^MKK=1f|SXWUM4j!^eFVcjZk-;`nGJ}gsNi4N2wHCoDkZgvg zo{dmV^mRn;YFTrC2Ctu>$(rTMCm~J~WPwJ5S|N%#ssU+G5GImeISzq?+ixd`O(#h5 z-11Xx!rv2v2wiQ^wuF7lJ)bi~JYvEz-riG*ZA6i7H?`+~H!I4h|Jyal zyv|&qr%aFt^8=WOu=_C|{xbjBo3!bq+*dT%O@W{3#;?F9!CZS{;w% zRoxvF9GJ*@lhKh5B%GvxzbWIcuT=#v8u&_Kt1&5X<+i_u-GE4v=I6_?o$b56Ka)`g zMX8_wzG6#I*dc>LzX2)_o{BMNXV%W&iO-r&c8f`o6PD5Q{X}awKI+vCiJ~ zLXMEY%Uq+!XA8_x=HW>xOj?zwSlt_PCF`&(`}~pGi7;xB-n@0@>Jp>75n%H@JlTC) z6IDw-7{8I~T|?W;Z-9r4zSX=_zq<-SUbxnZ$U&!W7uc4_wB(+iudP9@Gj>=eIG}F? z_Ew^z^;;5RVD=v@tb|Q)3t)3S4>?7nu{26yoI)T#!?CRpSOV#Lsxcs#(p2)_CYx>& zQKQMFfEn5wl(X=2)R4~3G;gDRG zAgZdq;L1;D0Uw81=XCSz#1AqW8n&`p*554wry*<0A0}_HCE_Xe9S`#fdqK z@W9Ot&}w8ZQL-8)BZ1jh-^1$@Evdtql-pvoRi-ZrRT%O!p&*M)9U8)gQ9fGK+EOdI zU229_*m;!Pu50h8yv zRbapP{w<9o=1xBn=8Ec`aJ)seKa`Bd6Qj1-;Jqz%e9|no?6_&MqW+S(iw)T zkW`L%vE3GE4MRai6Tbk8EpW)zOWePIf5V@I7L*$#v<66MMIH|b(0K)mnO(g&Z>ctR zP&0Pik53NeOe7LrlQp0z0cG~AFyBW3wP`)TOAv-2fw}|#YK*OXUC`Fr`V%^JW0VCn zjTWCf64|U3ny@rT0*#C%0LTFZc$72+@RXsh12^*v;51D7d5ZkZ$#B{qObt?7qL6`I zOXJiM9MGeNlN1|Z>qjVC^bSD80MhbipKPP z4&GDfV}?+-QycA4CKWA{Xixe%!R~QAMz}uyOfaJ+p=P6XJuujoM!`Z`FS_(I^x@xi z4kC7kA-I4kB7l8}b0V-6I}ij7)?7Gs;MP`>c{S|{*vm*W4)bk5E`R`xH!Bk3{#rS^ zp(~(`!DT!!IMJ#y&m`a-P4vp!$@uy5@-aE5SAGQ=QOF~#d2;;P`ZLW!ft!avoXQ?| z+*zz3)Mo*2l2=G)_Sl(NYA`iwFa zwest5f4w5uV4CR%);OW8`8CY1z7vTODan~VFkB_2O3WTuRvuMGz+r+a&|L3Ab3_g; z$G}8aO96kTY-<&DldQ4Lgx?bCAk-JFtS?HwgNas{!T=K&B8EVZ-ddnDl#xKC&nYL zEbY*$)-^y<@ho9q15dN<_@^*#4=ss?hzdDqQU%^b+%?PmICE!m4!Q-zd4yhtV<8PB z#2DgNn#3td{H=;uyLd6QbD-M8|h6uoH;t2H!9I|f|Z$Z4tvyh zdX=@X9~BO2yXuPJ&UDZ-eawh=Ti6FNF%B^UV^rc%)-O=M0y$rbz;4pJFG-7ET3Rns zvwR*EcT^D!BE27Ywbw1`5WZUxlq2W_cIfMeC#R|>kRhTm>Lw=g6;%n)9k~i1UDjH| z7Z^BU9@M?aN8zu48D09)clyNfXiIa!Np#@}Ut_8<0{-g@TlR%^mf3bK$Ixc~No zj!jKk?s9XEQptI@S9lnju{T}+69JZZ`+dB2k@ZkMe&gUuQtNCmVF*MO&rjI*Wy-?4 zN`(=55DXKdwj}Aen!CVZF>&#)`5ZFx7<`)?jn5=z!i9S^Tc zP)hLT1a^v^2PDFqzI`BP+gJX}aY=iJk7EbIX_L&yK^s}T#zOAV>Il2V0u0$GHJ=vO zXYWMh-kdzjj9sX`f|p#3inp7}jLA{qpQ9}@SK1C<^W7$2-qs^}^p5kf41oJ0n26vu zI#eH>m4$iPZB-nYexwD9qzdrJX$=uaE*$7HHYCNJImw||)B_mFhoNW!Ng{BdfCZBJ z%8Gz&r!E07mnwfu3P9)Y02&&^&fN*crWlGhgeF4_N&>pa-+*J53JQ>c*Z^(F%C)v* zte6KJWRN^dY)NH7n#$NSlg~v4l>J#vSdYm#(SLV1=51ec%>8a03?_`w48pPzXw3q; z@-w@Bc#jsmVuWCso~SC&x(Xi+3?5cX7KcD-d3ld~lM*J5Af`TY`{; zf^`A-iEF($9I1#O@y=``+p=iPJ8eKttbQx>CU^=zMMnP-`8m{S;4*-^cnY=Jm@psZ zXg3gyFejYiQBei(9?bs**?1XzDgIcv^*T#S*pnZJ^Gg7=B+7eZ4qY9U5t^&6$tA=m z5h~BzFNH9#gZ;@$bhP46dU5d0yfFc+j}x}GC?FzLSK(*0u7th_fswV48XFrCjPBV7 z)|8=@K|Zz3c3Cqw%P?2t?-3qH32el#u* z_rhM563=($n=ky_g--HUH1*FEHe5A_da{0~uAd2s->d9)lKSw-RuH%;*Lr*|tb~g| zJWh1+A@mX#h2|=whi_VFY-B_q1}$dC6YHb0FY%-H;;lc`t@5(|Og}0*DlSOkqLb3` z?flq>!T<08;*&;qIiP~9`2b3K;EcAnqRX@t4=v||P?#DzhXh(L0`FB(AtV9;5a3_9 zlV~3*Z2Xgi<;#}Yy(!Ww!I)CY&c5vKgIbHm3qaf9;oqxI9Z|4-`FrtPsSSAl;K(V& ziLN&WBeO}w;tl}}B*_s}Ix@HIgJ~1TgD<0wy40_IOV}Q#Ts?Ol3P{cg$VY79fM{%> z$j0p)(3%8s@Q^{j&_u|e$iP{0Al=mG$EYBK%oI?Ec`4*iJHPkqmm+1xlC4{LX?*}o zo2yhIiIxCu1*pX33P7|Y**f3?=9wgeIU^*42ys=ObQ%Yh)X;hymge{Y9#_6%o05>Si;`SLG=^XV$8<0kL(t!2>$(5XH8~*9;rWwKs zB2&kYw5E*^KnsYiBjWB|PLh(tE^+UPUqDsQgX+kNAgHM(X6n?AyHHb)GnVNgLYbsQ zXc74hJp(Q9(FSO0KL_Rgj!o_x?sVrPuo|0YeBH?=9)hw;ir%s=t@$Xp7fiK6vFSlu zW52BMnAI?@Rfy(9s&CHG&zB0l-s4f~eU-lgC$i49zxGO7s{u-M%*B;ODsv5|g8OK{ zyBF;@kWS>$nj_8nGDlcs(sbUKt=LMC)$1_VMtKFk9|4ucrs`8n-B3jgNM?4dZH`g& zwWc$rDSoJ764;dE@oz7rU$Z`De5BQSB4cU8jNlNHv)iK8C1=vN;AKC4+bREfb*vAw z@_E7uuDgsCUf$MV-{LjM}?JGi9(a7cTvUq@$^oE3d{5$FW zSdRsC2Vl20jdXto0Y04lhC%X@I}v&#WZmCNR!;MH1}Z(3~=A&YQ0!q zUTFxDonut!jefE%F~m%;*OO>-()PrJfiYi>l-QB{wCxW2@VNdW_P_xl=R4(z@t2Uk zl5sL_q)9I+1L05178{j%8!I_);Dv}u=B?k~1igKIwN}jHEZxpp_t&osB9-^U$+Y?k zJ;kg5;$NW-1oNof%^FdI<3UJ*9BYiTMRNC$qJXq|oE^KfcTIZ%-HPh3 zrsS!3cG7fG_VfesJ=IgH2>aN3gCHwWM+%%N4}W_D5U?rPBP)|XF)vcHC<)mtI0CBR zG_6N46$LPVd4Z_h854PVX+>z}m>G)kS_bLpF(Fk^S65e1aA_9kh-^ssUtkyWoqS7H z@2Hoo{aMDN^hs;c<9-58iwlP8?xfoEHwGJ=Cy!eo096?X8!$9yfsy!&Z*nF*n&`!V z2Zm!Mrtjyc%u0A{U~dr>Q_m{W9yUQF@l8nBrNx2HGWcPJ@V8K^93s9VDHjK^ZVx;> z7Kzg%{y`18i_9uNPWi@J0Aj>+VZJ2gpgv?~9hUdSB>F!04(Q!LfMU6Md9_VGcLpgN zc4x0FtTdh>zd4*47&tel@1w=d07d_2&*U(BNuf2V!u)r>%OQ)!U|h z4tvwSp&GMWbUND&MZGJ#jl5IMP;($v^Df_?R6*q$yuPMFPgg39RZ51~a`S~3gF-)l zt$2>445+bfpfqwWm}QmiGTvBoJ;z2Er8$xFo!_>7siLi`xF3K zaeHsk+}}{wPy1l|2-*qFfu5eTOB6&xh}0Fp<4dPB7KiQ&Xn1gXOKxV3pQKQzHuP^f zCf)zZ7M#2M$a@gTtm|J$z!Il+e2Yi)75t)n$1` zUr&)>`=l3V0efdS^eDe{MF?FyQ*kG1uf1W4EVI+GGYAc(y{=4JQ&jltMvkwlH|88! zon0s;5+%sbSg_FMN%^ul?A*mef8xy1iCte79;4lghNt}5C~t$CNph!4?g4)B(P6k< zpEu>&vqjTXy8(ax46XmIa6jTXyg$gPiB`}#)CrZTdW%jR*8!$Rgvb{X<}JQ7>=-T) zug!tU)c-1Wgdy8JCHwwo&v~{u&?F*$W7vTbsST@y%X^tXD+9S76R-k_$+WPcq2Vr! z2dtYFFH+wHc2q=o6n~kmm{w&BVh9=SZsb-G6#UQq#u+p<()^Q=yCw|T2ha{;KlhYn zJb-=$5zjC`@tWqJb5VIm?lzD%{>@<_rtgw(ng$wgyQjZ$pR9g>x zc@&iBYV3?V(rSem)rBYg%G|wM!W~WL7#zv1?FbcKm~*pxm<$*2-GAzwRy`C|V0*Ki z)plB>b~rftshFWnKUU^k`Iw6o>04$DJ%w^p@VziTH`Q{_GzKkLoWNQu=h^QgHZH)G zaz4!GgEs0mKo^0}==@<)91m=Sl4(XewC(#YGN;rBia%{^DljVFvOsLSY2UQ% z3d!MVXPDoDNVx>23a30+#Jc#f9QWAC*E&wW7e(gi<8+R@o`*Xcr`2UxCV#~SP&Y`c zYKW5=7LsrjXq6+*CFxZmIMCvQ{S;bwZ>F8w`}s@ZRtBstHbN4Twlh2`N&qIw3DwhQ zM*m4|%ZoWsC%=JV7*^E}Sv@_y^aeTw(abR1$h~Fr!csjPcND^bHt;?22KRj~FtZB) zl7DvoH#?+N3(yB#Y4~26Q-O{AhLyeWfQUhQq(QXuWJZ6LL?X+^qVa)@TmHDb29O)n-C^GoEbVfic}!+QhC%I^ ziR1t2JzMC<=Lg}WZ`%>TH>FEKa3DqAy{1k5m|bLuwjMgD=0Sdq4%uq@`ZLGuD!i&L zy3U3tKGI(tVdyBIe?e0TFvY~&>l>|TdA)<@Bu=MY;y#{U7iTwDU`LN%`PY*cn*=h< zW25nl=%u!0mb}{BYcsbo|3SHiW!65wHhX<$#>4$zHLU=FksHP@y)uVuZUG_@J**-j z%shwxqnvQ~EfCT02%Wl4Edo6uXLMu;E}B6VG!y}Zb2}YhAhAeyiNkHy(H?7L z2p-D-FZMcM=g~k%xH= z8Z-B{htq)TOa>u|bxsX4;wwHq_mvyQXg^RFJ<8qT*S`)>A`US%8!a$^key+w*PB+* z01Kd8-Q-RYfj5CA@(Pg7dn5==Eiy2HZ}XjK=|AiNHbha4$9Ka8R?w zwmwE*olf$nP2Ic1Antuo=R6(6RGyJ83kwU37+XPv&CAg35l;)^*XExnN6K^V-D7$@ zgK|~m%dub`gS_&I~ENjR*c>V6q`v&nx}O zf7zK>u#S>B0$B+~Wyc*qnDwU2VV`^)Ui3uUyEm!20|aUC z{v|;-oC<26tXxz!8Kt}QTZ17jzC{?eGQNv1kU<*QAb*3tnMEYuV_V)|L^LTh>5N~f_U?5IkN`LN- zAN6^B%-+}sfTd)&VXgIP94vA??b|AMfo^vxZzwyjL5f{KozBOt1D_`(q*a)Aag6@t z_>%`YOF?aehX*5y?DSz#&GP&sdhj8s8YCzWWJJNN4pQ>g$IUy_LS zy@^Px_9F+fN5B~9938RIZl`%3aL%Ckp9_Pih3ErDU$K7(`YbVvi>W}0GQ~%OZ7-%g zhUg#RP*c2m<)%#*Ub$Vq{Kmk;8fox5 zqiyOR)8GkWjBH1;EF!?)YDBg+=<8uL+nZJ#Jcx@gVCy|ksZ#nd*vWW^Ovhu)RUtFq zbw`SCI*d9&){lJW37moC3c%B+o(AlDJ|j zJc7V%2lImCbBEkNRN|vRw5q#&0GSe`;J|3I9oqeU^l;rcugJu_-ook#kRhg>Q2rOt z1O2i0;wAob7y>qW*nrlLcRUvLRKLOrab>?=VyVdurks>kIr^{8vccHG%GDLtkND09 z9|5Wpe<~AR@v*&|qLP65+Bg|+9uJ?D@%^iKJmPnEPn5TOAof)!h0C_=nc@4N zVz$h0L->+M__}s$EPS1pS@|`nDRGvjS?3_|B=5?5A2AJ( zTrYt0mp0Iohq!G&m6g@Y1Io(m zXs~P-Kgf^pvVHCj8UzMoEO}#NQ2>?qzkcx9`ThYcsUQn+ zuH@Z^@ia<>%y7_p6=(cuGLB_d!NVPnzZ^G7SB^EYb`PVme+r50J~SeF#UTv^&ad(^ zWC5|aFehQag&0C1`wkwwgsdIPKycn*x`3f2mSs1)QgP@<0e$Ac{pPZiXkjL&v9u z>IUHSfN~fs8`bgV%dyYkYG0D! zqo3xU=(imN6>wyXLEM2-Tr<4^cb&j$T6nKsd4ns%L0~x2%hw|XCsmD%1WZf9ec;pn z^CWAm1U)8uA(VZ2L$#f%`}Yf0pg-J&9>T3lH$mRR-usN^+SLjgR03CO;)%L~_d#Z` ztU(@NW1!Bm+6v{=P9>$Y#9TXY0RD0`;|nyW@iaS)up1ntoC_CLe_vdlgTYa=A+!q^ zB(q*?>r`e{g?eoO+V&&b?d=chk&kGNxg^?8V#(>4UeV$;yu9dZO`V)QvSJm=%)dn7 z5?-uD#7#X~I4(XL1LG(I1<(a7spAk7W2C4mFM0cxhTP60{Aw4DNE+J-VzPEF%zm9v zzC~7?IJXZ$g3TV4DB(b9XGZe|YTS+iBLXm3u%O2nbLPHWQbt*d0E~+9I6xN6vouWPOblgnFJk zeBeIz_%(`ufhb;))qDIlS5A#JR$WAg5kiQ61sy?T0ChuFz8rHnrBJ&I#yrOP%_N4$ z4@tr*9EjYCB*=e1XIAM0m~e}-tt>A;1e+kz$Swq*nMTau*bi_1gyvmx6p#Xtc+uQN z@{J)$SX^H`O=2EGlRzm990B~KR$yiTYByA?VO%UnU*@`>1a3_sxEzNwPj-Jd1i6>G zPOTvbfzmW1X4o*_^9wp0V9hcK!PGydF%)TO?oqZ#WB;Rt0T$)I^qz~0t0BE)>pAqy zDwZx?de^*yP6U~*Cb|AFc=ka*6f_9FJV8~-%$jYaL1tuo-@`m`$$HwV{y>lA;oy%~ zwR!v@?F=7U7jM@y|7fNTj$FJkT(!l)cGyKF8g8g8x9xCpNuxd}L04fGMXDAlps`EX z)CCTLIZyICCfG>_FfA7JEVOIQGrwNjuYU$rA+aAh?e`(&fwCG@oc%9d93Wl=-Ai;^N{aBg__9X+F2>Hp|?y)5OPB^L9R;acu4U z9v|6e@pAqpmjiH;BKLYA&@k;_T{!+MM{dhJEMY302+7G8_u;hBNxAxIOzYD|x|4T} zy~_6|Xbx1DxJ?FRcxY8rF+X!6D@#8^-7qv}1SR1SKaqGlNC%*k129Q5m7*R$)_2H( zIt63m&Y~9T8_;wX!&Q;YR%-}9(i0^u4k|U9zMA@x`CtR8k7BKUJw+lIG{Ru(#uA0G z+ND=2b%($BOwayLGs9#UA`O!iry$B{?@w4QF8dxKyYq#0<3SFwcIytUe_6Qz%_uG+ zRp`QFdmhmDyKn(VkLP78urE^aG#8p z*MtPtE9kusc)X#MpYJQEl?In~nZ%eP)DdK(f7 zMYS?Gh<+_aVO#L?-(a#&>T{sB8Q?+%hgX9~{S z*y*;T3Js&mq0YQCur!|+Va^MvigI|?amiX1=S(Iug3~}|0C&0!`9#~z&JTB^Q6#>! zFyA;0rvlb~=(p})tv7K}y#xL$h9nOGbmIw0=H=rP_Tpe}E0US@T2rs+#dfUF{dBxCTwk9w7`Z=d%O@C_hD`!A;(ldh5mwA^5n& zl`Z4}q2<~OqoY^aCg3t!E-mnM2iHK%5Gya$`wB)yt@Ge3P8ynfdrxc1;~DYT)VE#3WgL(uT=`3#+a zd8@UWp*8od#P$Kob$Gj|tUAbV7&*EeQ;aYL1KrX|=bk1#_66J{CwCU>p<$3i>gQ>E z+8WiX2V+siwgRFq9rWGO@eUDEMR{Tbc=%3t9}jh8n@O9Oufzg<`&=FuA=_=dz!QCr z-)?z$BX0Dx?e29uC1ap*IKBGYu}wyvwbp)Z!%oLIrs2C!d!9g{io$v?c2}uBes?(T zX5;toA;IiLFdex%@=s9S^PZ7jgVB8`JXb6q|L#39!L*cm)G&AiP!q{gwtA>?P!5m@-f@dCafF0kr3=z>xW-F3ua6%hksftHeV?a zpszDahqj?u$559A5T9L|u!bTdUcp~+KL&}~)vav_VvYUiaDQkp8zba8fJh5WCK1hz z3wKrmJSic>`OeNG-w>GnJWTX#=%ROLzd^S>X4WJCfR~VcR!PAI#rly9Og;$#wfe#Z zMurxI(eJ^_j_##62cM)lbLf|UpFa5e55^+P9@1^UbmvuOC!a^7sx4)OKeWQAXP%Q|)Op82yMc7Fe8f z5nJf(oIB&*h`vAe>uNyoSZ?regg4D$Q`3KLtifvEV4u1o+0i#BDJCTS$?1Z?Yer?| zsV(NAmK(OP(amKHjA~!b+H-FreXSeN8_w3gIze>HWJbn;;d}mKb+rXQ7R1Lv$t5$= z<%e>lv<@>)?BMshKo|}4d97Cl9Es+e^cw7q%EZo7-XZCu$=bpeK%4txGP zqGcrk5&mdg#CPm4d%o(fT^FQy_u4@tm8xH*1ep%_is(dLyLBrK-zKh@CkJsz1?Jn$ zOR*9UW$tY8MT-Q+o9Tk{at58;GP7E!`c=ZEs&i!Vt7Rv!R*5J`Lb~_QA5il5v+r? zIvFdi4N_6QwB9Y`)qr?<*DL%|jkz#rS~%U{PRcl)?cx zdFQedMOAy0l>9;P!W%UNO#V<^!qkF$$!1(mYS5gEVk>kGsvdP$`heA6LBB(1R zBqmmYvnN%4+3Zp@3GHFy7VHMJkUC26JO~a-2dXxbhp-jqq{juz4Wo5yp5H(%M*Wg*FI_pknQVPtOW$%w~fL zmI)#dx~&6Ie#jL3;Nq5odnRs#Q=0io(E~WCj|LtN60#TgKv1~v zaHc?W+e738bRdysgv4FE$cEPCSR`rcu4t2l6$C6$C1HqYDSmK5oYlhdXHFA?yRawR z>FYH6A0JAQpy?!GzJWQSOlcNYl5=yUm$EW#FJ0Yr3pAI^!ZDp@TQ0`M=JD3oRU4`D zSaR~P&Tj}u$fqERJP|J>!L+$9tjAr3S!?eRA``G(0@NcPIrrB(W;%A(nYls`ZH^Y~ z_eta52>=w!MT=zB%9Y+NadAwF$LqImHwrGm6ZFT)l5y(SKM$*B`j^M<=2Agl0SD4~ zlZV{McPLF^trrZ4SoCcWcR1kh#R9+DTXV2*gWs-WcRhJu=ir`w7bXOav(;Ya(-E9F zZnMzuRA%fr{c4S;;ZL`Pbh9_!`RC)EkyAf!R*T}344%AoUumz=gyHYWsA>RzLF%cNvlrOT$;q2B8hGx2u*F~9* z8PbUNA3=4aaCt!_5TMuQnc}9p4B0vRu;x+bRG#e4UiPTgn&Jy8bf25-@dvkUI|ILu z7-TdRo6-29c_lOWw%+uC{UYAjEA_NaG(Lz9B-D_l&S(^6 zHTZ79GK`3w<{_s8yDwji0w~+Ndv_VI)bdolEdl=H;N;(o{+`Z#%NxubkY)!0|GyV^ zWP|AfPb9&}O;{NUsshLnlRY>2ifrDj4IGGe1OQw#4L>kiuzrtij`WW75nP>_7gO+K zCByC8_hggWS|OEbT9sd!q2$o~OQo}Jc{9KMnse%oW|PHIQ_K#YA7^!MpRJ2+_mSOq zfM4KD7tx7}iD`b+6yC6IV#E#1gre)Cv@ZB@{+D;ge$uxwRF_r?S_ze?Sd5&pec3$Z zj{70ePT@WEa~rnh-Me4MF+!&0ugk!I9;=TTLZ%ZQ@_}exd$K8v8}{b#i%{M<_4#`C z?J#{WM}kJdNs=$X-#-dTTFdxTNZtYvb#Efr(!(Ma&e9f_8XD@#D~anZ+g+~?7L z2*g@ZZTVH=@GirxL>ii*j6H^|;Cf+0DWF0~%rw*5@S8Jj>h&pHh~OV?5L%e}Ky_sdSQD?CxE7s(fN%hgrudzN>Z^7xK-^pMO`ag{Vl*xTusN zLsLcQ>s1H`p0mj-|A#6Z!sSo~)uLC8bKV*m z`04NQ=t6*>TaIVZdb0%g6^*hAsGpIH>dLR2`@ZRxdn&?rA_hqo7lP|yh9T9hScMXw z>F~zuVJex1<@VRPi00y18$eJkAvlY%Q4|1D6(;8xIp+;lqy=8RYKS;tPu7A^z9ulw z;n%P+#4z-lb}gK4fEsm*n(ly&RT?5@5oM2qokkzh6Y#<2OhrkfVBwV?>HfRWeWNJt z^tbcbJmyZFNTH#?W0WsVh=4 z(Oe{G7Fbok-8`Lf`VJBszHI1)h+8D=$En2KC2-Z7Xz>8TR^GOGP$Px_j4YBB6jsc> zV207CA9GzCKk4E!sX(Wr{0h4%HVi60RvwDxmw#0os-O$pUz;C;>bg0sxEZ3O*RNiw zfk!zZ25ZqwZ`C=@T-Rjf9cS)Tt1)!tQfqG?tz92@-ArsO9Zm6lVdlMeGo68q<1?>o zwdI~wN;hW6UfLieqvHB8HkfC^3_RQAgMBqN(af8M5uVkS;&U{{J>*91kN&`Qbo`A5 zVOvYt{rU|Xil$XwhCW^i;|)BYeHoZkg0@Y_rsGlQXB*3{XUN*xV_F|bWqW|zrrAD* z^pSguLozmI!{iCV@YpZ|EI$Y~L)$5pWxH^PsK8_`8RqW7VO`-ZF97Hz92BsqTpThq z57KahD&t*!3pj7R_$D$&W8?1O#r6M{9*UTU8gf{cw4_gsBJ7AjL z*gu3foFAeL3=$o`bPlRq#Tu!#4#G0=o!&Mb=FIEUY?)ccG*HFjVNjS7tZ!<18=SeM zz`1>Wa>I1Hpb|CDxPM^HARyB>`bLaYZJg3r;60#S6!rkW5d%-#zWLo#R%MHIrE}NF zgM@(+&?v_pExRqWc5Tsl6SG8A+xQTxZ~Ku5-cMgp2# zr^09A@ZgNCFkHJu>-N2m-n2;0Twa!R+wqNR_S~%;%v2*nRGU42-P@VWMY2}yq}2iU ztq1Z{!pAYV`Y&y@s#tMh(u%bn2)L9ultJWr4}6p(C zf|T&Hf3r`wu+UcscXSw=*FVmk*1}*E>8)E(j*pcA!bC;p8Z>{*aENEM+zx%Hwos)2 zu_FWAg@-Lxod6~zUA*qUkJE#H zZerE}pVl-;dUvz|#%sZ{7Zv3^ef`Ng_7srY(S7x45nQpNwN#OBA1J4rpJIguXlQE2 zpwLMGugKaO%3f|6w;W6HzVw2|g`SZRO#M3bxEUiDp}|V2;vSw(mlReU3>{L9QDr-V zaCy~@#E81B`l+(s(bvv=e}45)>E5i^q}-*Y$TaAc!972kf9oUWQYk-LClM>Te-5*v zU0@ZE5>k^LRxSGO35nC@&7ZH(hU@{BJ~GDnauGKYfR={ZJWe3{${rkNJs*tyXc5Pl zsA!1D7zYQ>Ua^%TtF3{`TMdl)G>lEl^gLQV^{IPu>{tX^IM#IUP37d*~u6t+Bgwk05aXnhWUG zEl=iqTNWIC(omD)*x|%*&cDiAgw#)Qz%Ce!lmiG8VEEg^W-oZ;;qiuzuz$3bi+n-q zMQZM*AW=&NPV@m4ZpfDWbv+|Poo$yy9(%V4E?d@oRgu@QRsh&Ckb^sziXHp$;2*gh9*T6>lr&vJ7p7)O+jPK@*mI3`lEw-^Sq3iiVG*#^2UVFj_BXPf z$l-doX1PwRkly=><ud?#`^!h;&_EWZW&pFq=7ec)3MRwhyfm zw27mjn42^U+MzI|&(Sssx8Wu1_AnYyZK1U5qga0F{% zz|N7XD{o5x&YHm^nL!z-PQ=b-Q7C}q4&JxCkbOpfu%_e|!a^HR%PU2u8~FKWm8@e2 zZ$Tcu4_c@N!~AwBDZK}RBnDwDcp>5^tqn8$3($p~c=}f_YGVCV~o7Fnr_KrZou4;BsIyyk?Q#Wl*+jB~|?RK%e2XsR^pBk9Um~mD3P8 zLqJjU(rXCG#r!2nteFg6)Jfo_AHm`*x?#BBWQ?3EtJ)7`k(kc7%N1qjAeY$W)Pe%tX9M{|k#xT-s!j(K^J& z%uKpu0hrvlVd=oW@lMvvHOI$b#RNwAen1Y;?MHyS6pZ0y@1giHc~GOGWcnCzAyu1D zjV87=+yL@u0aVBta!5^Z(AsgR<79$n>ZH%Y|KaOR;Bw5{{{IUz%#3X;V<&5tEHxss zl(8?_Dv=V|E=87PiIg#8-`A8BB`K9uR3yf}L}{~C)>7G(RMP)_oLAxZJkRUj%WH1; z+;fk*&hL4AkK?l)4OI!1!PM;Oz6DJ4`Urta_tM(PF~RSji_=^RQY;?eYZzSn8X)Ty zxoB{$hh6V4pD>?sRf!Z)#wh6&7(*?VEIFmX9+D^&$+iZy7u!!WzhoJV7>@VKC(O*k z!kz|;N}}ho>GEMFtuget@B;=l1`1fPtZ$+igq>#sqy_^3^9z-53WZO5#e#H*q)2Q` z>Uoit1tQd(n^4PVT)(m}9|HqEmz0VJN0=!Go~cUp@!MHZZX641>v>CF>Dcm+7W6t? zR&+7y#Mf!ucqeK#ksoD&wF|(nGn&Tf&g9B?^jYc&MPo3}{mc1CSm6YkmqU;tm-7KU0)}e2n1TFjx9d(c$T68HBnKfLI!+A) zl$(C$blaYsVk=h+>#1}*T(?uQzp!4QhjYHV*6)LMu@n@jt$3K0>zA;<4vqENc0E2m zg+|utVoihZj9wR$<;$aqGCSg4*oMjE7@|;jLE6{C)5}BLIH+}E1`|^c*mJ2XW4JSj zvs+9WR8%y=H#Km#g;OlI`0~C7Uy`j@aAOalHK!joZ{BrZ(Dh~~rtocKB4R3j|4H`W z?RfaQr`{iKsabRvTbFP3DsR=~bhxdpvtbb9PWi+$#JUvBoQsE5ob|uYx%sb`tk_j# z4a~*Y@WA88lPQX3lZpzfw4k{}etq%lX!Y-Fr@buunpqZv9_}?G?MUmG#d`d)S+d_6 zSow4_R{811m8o z@8$e-httB!pbOr=`}jgS383^j)QQ00;4PkD?crjT-Msmp_d>wPBxIn|oxvEAaRkf^ zn;2q0iyw6?&x6q?7lBETWB)E)OrTWK5E);)(}I=fFJAEJcz$}5z9RAJ;@1z%L0tFT zd|`KXs`7EGRF(%wjFuv%7mNNph|{V-W;1YjNUe%^lYDHKG>Z(i!KSnYE8fOhWM>~` zbwDCVFRygJYs=pOB`RPv@BYCW7;VIe@fsKnTs{SC%+%nED+(mx#tH~2)rW{q`k+#~ zNNpvoV^AJtYz_CPTmeXPm$=?0K7^3`SMW!)MVUr7hAO4#Y*)YE56=ea63`<`99 znV+5!?&jC7q`acLXWUu!sk@MrQcGy0Q|K7Yky9tVy?bPnv@)i(8_%ghV<*%<(#U67 z0;O_w^m!BId)vgW&5rd*H5xgpR$prM6K|rp*pdEBAQI+wQHy@gFTs$^hYT6grhS#J z@p%Pyb(LMeDLKR_jiRlNKv9`Ra0C2u@y@EBP>(Rr`H;ck%d|fosq$ z{=-@PJ-A-IddjW$rD>G~=3<8mwu$Q>;+pAFn6r1XgI+6O9Q6oDDU#&JR9lt8$Xz<-#6a6xF%FIA0 zlP`zXHQx5?__s%-eCnzP5RgGUxx;9X`1yRHEuebLWSRL|k;YVn&CcPK= zImDGN={aNQE=Q0T;Mn1C_DPr@6*0e_z^bvgz&v2atKyX};u4>B%0|rYq%T!5j`PhNW%G=lg(i z(7J?hADfI?yEh9UAQ{(?Z)8$VGR);XX?Ek8{Eh%Hb_;QZ*0dv+QM)aL&15*YEm)xf zz{K-<+abP$XA1uHy4w}D*CM+VXhr7d0Q;m=U? z%PEKSK>9dc%gMnL`Vj$grJ~XR^awG` zo*(_9fda>l7!h+guX#UbGw`|zqe6k|ol)HE-nny8JXG-KFg`4MKy*X87q5@BuW1wl z^f)C$Mpw16PsYxKJbI6K8g{VRm&bx@lv^4#3y&h%Y4x zM1cNLRaIF4ZM(2YHV;NZUO?Q#`$;vX9_ISYNne_|rO&edfGoB9b(ch@EWt`}$QbVN>{z60gvXAk=%mV|3Ze#wZYh+ghN!0!rD{M@#L) zCh801h6v;ARP~OY4vzw4U$G_GPLqv_u8$FVr}zCh*Eq)JKhol8++}RkTv5-t0faa? zJ7xYOD}4N-p|E+ManN{V{zb!83Nk1796@e;@&?xmh})^a@^y4JIklkI4ScN=l3M(q zps~6amM!%tp1IIpb2HVvY^%TSOuT&XKKma!T?w_G5+}^x{g|W|30|+8&$?>dhM2ay z)w`Yb8QY=$>zXb4P6WyC&Y@W=VkA+^Ag|FY74icCl?Y=(A{hyW1W$P56YNtcm$#32R&qRl2w4B&4E^q<_6 zQRdp~B}0dzTF;F=6wR#eTK2Z=`rERSe*HzvkUR`wYN#QF z7p!?fw?+!;Nc1d=f2zDXDyH|`vc0=^x2;g4*w;Elkx;45TQLp`LXO_sR@CGP)=Sf8 zj_(vPW8~h>iIO4^ysl z3DBp9sb_Lyj51JHl$sNinp&ciq|}`Kx;;y)_k0l+oXX0-wL59YRL1F@g_*846#wNB zqfx;?e9x=@B{^y^fZV80Bhc%8mOUMSu zhqVoJ_TbtMi28*aFr>77zn%>EycvjPFlNi-MnTVd#{ZO%`%R?y=6?85w7%1*?K|Z> zuyKnhP(y!jw68MQu{zrSISoxBT96m-ml5K-_`~zz|NeC-=-g;z6|`0zPCvw>5{$IH zUHkSn@4ane79RtfsvAsZj~`uiVW6YR*yUCF84HXR4iQVDt>li82Iu_0NM7i{XmI4z zj7(h>S9aKbg6B@8WlAqx4)}guXw+);iWN%nR6F$l`guCiK|TL8^aAlHHUVw?%+%Lq z(cVzzjYMan3TzrsCY-ywRoBZ?yK%9iuWUR?x0r5xo7t zf#YZjPZTyDTN}BUm>q~q%+}k^nayHjO%mj*-v8}SoXHN-2+MC<< z=qxH)S6O}Gc!Qkw)=57=dHO3F`1XF30&@UZe_&c8=ORx>QWmdQQCS5;9UOd(V^DZW)e6X9_b%lTK=BEy#9(E2*ODBUKc1pTW`Twv=Te zNVXABk#z04+fALWT~l!dOs}=~%vkUKF8W+;k=%QAc9maPY6sB~D4_3B1Y>!yfqF{m z;q2|9;f__$3!5kkGE0K%Ady;1*Tna|10d427z$}y-w*2z7r&f3{$yPBbr+GAZI&yG zhmLL%0*Dq;!#qB6C~W7w4!r&d)7zjtF6TTr?7+0lN|YYzLlE}HL3!*TkpvYfJw7ve zuPD$O#@T6W^=xF5k1EYJf!90yy-hBxSGVr1tu^H_s6xV7QkBafxqi88n;t#PiDuBx z>%G7l#H#eF>zARo;-V>XR$UBzc**95KU^o^-T~-l_?0U{Q_wlB6@ta6QNrI4BxE{5Kx>%jW2DD9WIis&PH`hXo}O8b5mMa@nXQaXm-=@K zxIFY`{kg?A7gf^#dr(mRg2Eb+OWS%h2N_gC1uG2Gaaolk7T)p0;F9V`;8M+<4p8oBnm%`xtl{z3IY7St>rkGr*QNcx`>#RH`?G{VI|e$zi3#rT=I7aNM87H@=o&2 z1-&7p{KWk)&4%VJ!ig)*GHQYl$BbFwb7%~kkOgEATtQ6-R8I>>TTuG6Vcb}hl%q!d ztjdzQqV@G%HIY5d|0V?a`EfPa&a}gWH%WVxGzOR%SU{OVcBaJZ)qtD}Wiiwc3SQ01vbP3akcCjV^O=A?FYW)0NFUrb)r6Rign=b#3N2 zO{^Z*Clsb>XQeM*Z7v+%pWZ#C>*ZNX>dPrnd32$nJDaC;L0f#H4E-!>d&#)2-)BsQ z`?2G@e?q;mqnxSrUs}=2L*FJKVCf^%ufwiP(VJ(JKfG)bhAqg zjBm)zY3-<=3GKaC)=fGC>zIGON^Sj|vfE6fBtC2qQVI+N#Uwz9$TYx*w=}I0RT_n8Oxe-x$qebq$Lx-vtQDDD=mLxyptV5pq==pvH2fH^ zxRdSAE#XcUu~P+&Amdd6w%JJWUvHRqmAkywi^VHU*^VaO9NzSRQo#$)2P z{QL$9NJHzw4NU8TG-FY;6~tL0)I*fwhK4d_ME%9}9anbQ^gs@}XQgNA4}7 z%jfHS%E?9z4;CwfUX(+Bo!``X$2tk66s{LDJ+SJF2XYivej`1;>kn0F7t!_w^VoC$ z+omH@SJab9erE51mXP{Hvh|B{A5|pd1C832MZL}=Q_v@nj{#!P3IuY>w7qMWV|tvC zN_980){XnHYjgq@*(+wG!bCoa~0{s`Gs*Lm0^J^5-9pMc`n*OWQ=Q{OZWJKw2!(+wtP*+5SU@ zK}KhrJ4CMjKRDmQb&af#@r153Iju{-ZMZ~Lvm36UvM=6c^^#WLKL>XQn^hTizNpLV z{XeuXt)`jmU}f6(5`rC7y=1O@_%!~~B28KGI?}yR&63rFtBP}m+^_a?@I&eQAS8z5 zRls5>Y}-=CnNwTNs(GrdOr7#Dj!&r7@@JJ#kKfi^d?1=`2$HC1QI)vst%-Cp?w*AZ z=VOuW5WulD$GcO@#-hB&_6_vQU-`ujP(_6u5Q?|w;nf>2j&07osHN)qeDuiV2#1^z zCwj)M@Kioz-CdWX-d!3a-}iYZ)5e=$ju&RFPXI&kB>nHb?DRQ%=hj# z@Y_sbKsa4G(XRl$s*kT9u@?K(yiKXl-NiO#d9Gnwf5y$VV?BZ2p-|2%LJxH0>Nk$x7`?#HgDcsb#!%~$YO~}c6|8&%2hPaqMWom$YcXYcyEjwbu4~~u zte23GBPeg z1hpPJJi~YP=-+Ki+AH~XRaQU$;PmM+$vI+F`%Zs@>A7i#j`SR461Upq(d091eH%jb zFJ3|hI>5E=9qYK|y+C=9KgV0Z@_NA1e9K8g7fa%v*(C7!NBgd+wP*9 zXt;FjEoApX4By$^mrG0s4-awW$&9jojDQBBA7p79``NrNe^sBv>><*w>(AaN_a;V4 zf9iRaxDZLB+Hydd+4#6k>;Cp{K3Q(VX-cfQ0T923W^`cRK|S*#XrS*;t@xbe1d=ukx0GhQQ~`#5R%AMy}^j3QsQ zJ<_&O&;{{;iq|Wx)?%5l5U0bBaf?o3zLzc)%7&2Os1#^ualTgP<+shl>GnliS9?m;01i-D+!r1j$(%F)pauS-C zo@mcT7NOKEKvHzIfC(rN6=LMe_)|pRn1lP{%d)sI^+ZNm7?PAT=@3SyC~@hAx;JaDdC5s0Rr%z*?-ft^x^0+y zbB!6v=YA3B0E(~&?o;S)L__xee!C{P$^Il~+`e+g@lgCcZ>Kxj0PWiSR z`7s1oFX;hxGNO5}1(M0OVR996Z1o9c#Bp~At)+(tV*+`{6;S{2x$fQjBXbo3x`KmF zfz(dbtE3uRyZsIO$X64id)04j>@u;fNn9&yuZQkSvy@&`6~6&WF=LZ-)NVt}>TTs# z@iv|JkIhw-4LgrFC_OxXa=VAeXAK=tZ*1C-Mr7CcDfmKbRaz3_3b8A>A7uQ4lu154 zKXb_Sv6@~qmr-(|BmFmb{arACdmScqMZ`adz4IFtD=*lS+)V%}f1;E0k$a0wl_Ndp zXB@}U9km=NQM{3e3Ai+MTeVuSoY7eR2}s6P@)%^5wJ!)}4vPD|DK|>Tn}yq(I8 zUggDWZgix(?{4cwyn>}>|!S;0Va?JSu z^KuMyRtmYQ$?7}VNhBgQ`FMX8V4_HJ=_o$_eaQU1bFA3AN4?EtsNPK<0|IR_c+#Z( z5^V`;e>MN(hY$J~3fr|?cnu|Fun9bgSjCZHMnwbXn0WX1a#>r+Y6oHt9FofU>CbPx z`mN~s=G^*Xl1j#F8Wc^B!i*D*rr2#{FN=Dqf49jtZ=O}7tEylh7O^FFMn#&O+FuTw zXx|Yd`xPmS#%&&sop`L@Iehpp8J6L+^A6m(eLEGc3dX)n=};`V(b+1MgO@7$TdK-N!{Rl6u`qJ07VO3YxzZ4XkYL~VXR+@~@+qPEDH4V8e} zq)YMR7}i#Z-8In=LcoYf1&n%5L;;H+CIIii3O71qI>1K`OBeec2^-SFErLW3%x^}P zwJUg$I;zK4=}$Dgw3ly4bj+IpbL{y4{Q$Nhz(&P!Ypdo8b*s+Chg?FugP?DxOFTzh zNXbc6L*YXZ_Is(lz?y9Uti&zWmR3W4Oq2mza+bbK7=SP+fL>H_pWdoVUub*zHz}+R zUE#s_S*CC4;$~0n$6eL=N3MwWt7`Nr+BXW1;``{{eOIKp%56fZcC$p%437C3&IKyz zV!ULd$+l0|#-5C~t3Xcle`r`DJO1`he({r!nltl$%KjT+?a}$0T&pND8Df4R2T`<9 z1v470l|sSCd^MjPCIIV)pwkUL7uLX62*$FQvJurA7MiKm9xT!1QchhC97S z$NL55umVhU{rn5>lzq9ga9vBpJmRrY9LxdqGDvX>iVuRrmET#EAEY@CXQZ{O9JNXa zFBe9k#2CB_FVxlw)*adic!(aMQ`fE&+SZNEvS(Kx&h{bf zZ;|3t;Y&5KxuhHMbXoM)J=Nm3R`&ThXYav)YG?d!-iTEuXO-3NAH*#!1#N8R+)Gp)J2gBb^Lx)e}5j3=pPUi{cw<2yY%vo>sQBHYj;c2 zR4&?eTP`i4%pm>-RCO@$cshmVoE8Zx+ptCQJya4RCFj|?WwmNn3w^W39&mL_ zOfjH{xZXEiZhfZ0%-rxcm+FbE2Fiuyv*(+FaSGI10UW8B_-E;aBP;O3+Rb{6B~MTn z3cNdFEHS<-T_WbAexL45A7NtBqa$_+-J58kBkfWGqY9cbkBSlv0i(%?sy;nH3pg;u zZ)HsQZLOQ z4g-zv?0lG=9IS#ApamCe#-RErivCv{1fKdk3NCp(-~lpl5?Hpk8YsfIcI`j5&~$IA zguAi4>|#xt?mVVn!EPGa`7{N(X(9}-)hOrT=}@teF#AT0=ZFYbgb$}_lH|F_MT?

Ztvt3ef?28KQ>ETJTI(1dSrndsErU{&D6_m2|!C@kcQ?Cc;0!uVM= z+qmfzvTZRB0(M&}&`&XggdIb%dL=E}YOxL&Iau8*I~4N2JSJ9NI82^&*R(xRts&UO z6%K~u8%uvEc{dC$^<=_bcLZ7LB325?uBE-n)1E3H1xs6`1BUa>NL^a=m&x zYCGI^IGxAEWQu|&*J)n&*Z0GR$`p`;;bwrWHz%b;l+5Qdnc7@%4?T!+vcHwJ)QHU5 z1gs_Ewpg{~(C|%zTFZ+mR~Tgg7?+RZ%$t~zFr3>{?B+fHNAtO>nut_Eusif%LslS+ zP2?EgxbUCK@(7%r8IcDAB2L|(Tg+aeHX8EXq7#T9`hrfKJ8#Yri3JX3p@N5B57|b_ z%1ipI4HX91&>|LID;O_y5!uunszx83-;V$tk)KE7W8&2G^{(9IJWX4{?LKb@wAMsW*3KJpbxM_{#q{ap~t^QQ+MF92!|)0OB%u&kB-VS|QHPK~5hun6+e zO6a5>^v;oUlCnG;JyDSe?uuJ=TTVs8*%7e&J;x*h;FxjT@Ah)c5hAytLlzZ;A-6z< zbS9;!c>GQX*d7wo@-lIa=Q*XIV%=6iAONE2&dnk!`9#v@h=LaQ!Z%jpr7%&8;wOmR z)sZ5_1Vvux$V7gcao_eYY!EoF5FdO6`4yHg`hwBJErVKfRfx2L^|}Gbi*SrS$H5ythLUga}C$+{zUn~&lH6&MyP$|1SWtz?Va~fXgPTkSuaG6w{2MRjv!449)0oHZ-&AhPXr6c{ zEa`IjiE>xol75j{ykqa)(NF>wN9pvW9pcbBB{seE@#9B91I2{@jWCGHz!IuxNw}60 zr;HxG74UsdExoDYEF!euTY~@Hy*r-V4o!5&OX3>}VkY2jm4R!cNQ=Q(lW0kcUZas`(6-uIC4zoBaoxcTQsa@*pwzroN{utD2+?of6l-?=y&Jw+S<)61>hhInQBr43T$B5WcnRgF6 zVLSc$MCEB3MYt%Ke)rIKopm>-FE(LdSTgI!$6%{1ph+($)o5chW%RTsf#`%;!D?S~UVGf?~6S*xk zEJ9}8x^?SHb&a%;a?V-umdPZ^6VGn`Qw!NScun25op7+-|UvR3NiAgl<(`T|Q1QXS!9+_4D#|x@-#p=1p z%TO;oI5~6VqZo@b=^^m*foKG@k5o}PGABw(jYjn=5{$Ii$_;_5MD4>bP2Lah7wu@% zmi<(!u)cc+bSVlM)of)AT-0s*J?b}b`JA&?l&0o!HSx{PQLfSChdG#RznpsYq4rO1 z4IGwis}VV7=D2EI-=bBQ%XPZ5V0*!*Gwk@_w{&b&!;S!YLukWMaUyW$9%Vdjnp7KR zjE9fX);ha@XXgX`6nV{aHmPFOnY_FDi&LC>>3HuKDxf%5X@~pay-4Yu8D!CIi_8fFvBjp>lZr)B@0o2sv(ck z`0BoC_gS|XtlX~DC4_I$ixFgo==yu3mL)y?7~q9=JJ8)NU3fqkpY-w;xNi6K48qoR z>2>YjRG%MyoDDb+`*e#yn{!$Av~z$V*GcdIywb#3iXe};LbrV2wQE`%U!qtrud~^` z`On5W$=Zp5y<`7;G3;?hJ>?m+z5YX*$LqZ-E&Tt!;e!(RynfM~4bA~8)-qsi?hJIm z!(5-rD_LB0$MyEw2xFabxbN^mA7c}_B8!v?+2+}NBU1@>;>VKK_)Qb;Z_zS16A#y6 ze>{vBPpZ!89T&wROFOb_d?nR_IijM%tC;u$1z-0FQrfWcXB<}Ovo(FE2!7;eq>30(wG8H)ek-GxO9*`ne6f0$`#e1L8^U~ zMHnrqrSvsJhuYcYIt{Dwc>Gt;(tfjClAmoe3V{F@v91fRJL-vYE+T3{*WZ6}Z(g#I zwE(1K3yIkxt>h5&8#6+M5Kk4+i2TqrK<@@gd6LQ(SGEcU&9#uXAdk;2(HV$Lh+K(G z)Z_A`<+UedPp?#lFuZciq53kNtLKVfj7)L7OwY0aKY}Iw1ja_%o2=`)H5d)cS*xM^ z-8EL34yO@Rea|jz-+o=mS(R!ZSp^fCg(f6^unhNAsV_SW8*_J!^}}mIpeSWXl{9H{ z^`(^*nU8*qq?0)xe}o5Nsz+l3zaYn^4oxoX`4(Fts{|Hfo5ks5JUs}ho7ndyoH!rH zfmD_}Od&}Swak^zzRvyL3j2pW_3+2OA)EvtSHmRN=R1Zj#ti8$Jev|I$X|22nhzZ|N9Pe^VDKa=}(f$eV z8~0FsnfT2oP{BS5sFQ;9I-l96L;c?_=X;Glimh+QtPpG}fVq_st_yQ{8s z>cFUlfbAmJQwY z4mSmzz~Dq@eEv;zr@Em8wwljXMPZ^92(^_wK#5obsT_WtR%AMxME)UgmE}WP1_og0 zQoNl4R0+oPg5NE2&umNXTi4`u030HLhF_XML!hhoeQOa5Fm`;f(?KyIxM@cvEjFF7 zk@_NEx?Y#nwfb1eZ;h+-mN!^ASv%BdaZ$A@#?_NDcKA%{_vgd~n}aA&($y}H%}|Fj z-8iENDmwjBL~Diw`0-h{99OfVsvZ7-o^RfR>yiBav;yRm z>3DH?zu&8>5)653BrNIT+L}%dgQh}p3mr_-%Ur*8vn~)8fb3WSX{LCVPDIt^L=6is z(!gh`gw5P`Ibwh9?Ou#WERhCqHnfQDRq^bN9s7Imd@-7RxYh=EZb2}P0wvlg{y3<6 zkmQtsfN8f2iepR$IW9BnkYBjMxgxFLl9>g80(eCs-vvd65eH_W9R^rJGU3~XDnB78 za&^5b!4EMaf$cX8CUEUIx);igdO6Z@#Yjm^syO5xMV-+4<)`Fkr)&w5*!f*l)BA_x zqOy3&H!fi~im^;`UWqysm|L&$JCyP-jff|=?LXUFqXnEhrGqMM4_1iBYij4;pY44) z9}#grZ&nbKl;-ZDv`EOI{1Z=8?@{X{V>jNRw2MlAsL`W7kADm$v$TBBGHq$EW)+yA zmsjvV|NJ9jPM@NHfSP$JTwe7YitS%jidcEQ=ca>EdZzR0MjI);WaOl;1~G z9(p@Xol^LRP!dB5zZ?7qVgVBi+;r!;<5dy;nMo~AOkEACw&MC#cCF-5>uI{|2n@L? za4={5Qa;|RQ150{jf27~dPjl^o(G2RLaz_jrwJose%ld$ zd9}Sbyu64Xu-w+vU^HPUJ-hv$aEWUPTfL0i$R`5A9cn%7B%FAKu;#(6>D_0_i^&vn zgdst44i-bAx_`yhwU`(~=F0WvgxEElvaFSVp-9~6I`rEA?v_U`5xW^z z>nnWj#yVu=i89Zyu#)5>qN~{L<(2;Vf=V?GtU{)yv^>z6O~#Jhlh+f8a+j5m+xpyreRd!-N=SgQy45GB5P)BfD*v0#3yhuTATWhn%2CdN`D-+iOTh3mYIZn zq3cAc^u3;RDYy2!XB!WXOkLhdrs;*sP1N8mf?~cbyM3oNRj%Q2ewJ^;&sPQncoaTq zj7cNZ!gH3?*EjWc`LrL6n^ux1P96B*;W%kdv4jRM)?bRx8WdDYrzF=spJ6fvKZ-;f zur?qGI&*K#;v8G?*o2HGv`%Dw>8`C8vvT>OZ82qc~rBt^hiysuRy(Mz=(Q#j1bSvsxTU`m+jxu!z zXNKRO>&&`KC#Cm#l`A41PRSMm);ARIb1iC!Wu~5~wJSpplpj((&EUlJQxLOPaiVP9 znxCJKR4<}4v+V!0*9K^DNG4C#iOY0NB+310Zu#j@>;fVOFWni-)lx17r#ZcHuJqGn z;sD$F)8h3Jamg#szqMbmcQK7NRVx?}A(WiP8?-?o`?xaa@bc;PqsG-{aTNB0&)j^#fB{|~=^PRMnT;Mj8rD0v zJ(_V51QHuX?nds)QN;a~N6IXm+Frg{lu0#2YH+!s4YADT5!s8P^OH@-ylIdUFGe|bpi|;1>2;95JB2ovgw_4-?b@^cRB1NMk*4|00~`7dXH63jFzNz`-cP- zK8wsTF;^T)>%K#HQFltKX?_k>6H^I0SylAHeZ?pM1yQn-%9(L#hVfa{y~f_4SP^f2 z;Luu-K2%Q<@I#D%(?yN-&mf*&n1Jn!g{Jr?B<;*jtZ@XgNY|UTM&lh8eJ3Cp*8X$s z0@%A(n?(TK0nr)MC@>@}09}2*ip~#q8aXDJnC?g63f`;U^W8H*i2jAUcCi*p2y@h% zzik}Mo+mFoNjCr&v}gbRIZN8=n@U73`YI50!{5$3p^lE^z2;?Y>u}pH zvJ>cqB6{Z{or9Mgv6cF`yGg1DIkDJY&5uDrM~``OdtFt+aV|jSTs74c3-eab)0V;F z)bGKrlqpqw?t>q2Yn{o@IqKc1{%E65m1gG8J!_D)O)b6(uoBl4?9kLXSoXL~K}xhZmvq^@#|eG3@2g9fary5Buu z4pBMVuRa?Gzl7nI&p9^a8F?%;x8#t9?dWA02^=_)47SawLT(`kX0hyEWQXuw5LrqH zUakjh*{q>)$0;e_qh;RkEIpc6qyu;DL<%O}2I`nzX$76p@s9-yC2*3RFHI`g)BnyI ztC~C_wB-O1@bbDOM`7#Cxxqijp8eh zQL|mO=~dL`EN5&(JJ{QjM>HDy|HNX zCzOJzRciO0YJ1q9O|q8Xu{@{T-TyraimTR1Pj8$C%aepwl$*#h7B{b3#UzPly#O=y z5PQxvMF~x)33+6=p7|^mDqvp7osW)~LFXc6*rS-)%Ck1Ls=h@5t9+^Fu*CB)SI^~q zhqLd8l(IH<&Ij*sf?o-jrsIlM_oJgpC*6WUFJXJvFMUg@iG5U}G87O073%E~8Y+Bw zEgeFnC4<{+R%;!KVU@{-X^l{GNykehUJ%+4bXj1aiql}W&>lBQ$d;Tlr|SQ-c7Bo2 z`{%!!&9C8;6W)HJe&LlpsS{`ApBFvf z>E%UqZBz|Hahe&plfqb`Jd(euo?#LgZJhfGZ-BsCb9N=gL(sw6yQtf&FPy0W$T&9?{D z69hy9Y|i0=50!MqfO2t-Vz|wbpMWo`4Hbs4oGq|hvipWx$>+W7ZISqsRVzn+WaK_N z*&+HeC%=gY5(3GC$Af8*#ts5v{wMYHB*xStt6V96*oo${06UC_d%*Hv{{73)r8q+7 zCN>wLkw6zLxndfP1bANfl`35x{2Qm^&p9TVT6fn6?`+tx*17^VnZDJ_a}#JriRI+SxPj13bcNQ- zR$ebpOcHd|?aCs_TR^zRSh>)V`7IS$+vL*z2a$OOfemqhn$k_s)`Z{RO?XH-)g*B< zHD^zscH~^AkaJ&?5+Th%dJxGxfS!JyC8}U9V}4ELTR1`gc#-xrwz@ z9!-{79)1(Amw$I;H^ELyWN5)PqsKKVXoyB zT!{zsQUnGdX^$Skd_)i~7qzCVgfH=EqEj*-O+?uJbM*$7=2j)Fu&J!Gd1O_E7y)1< z!GJ|4IB!|o($|p}$@U0l8reV4Bw;SM8Q6{;x0xXcn<#9UfKk{i#tPR#h|GE#H36P( z5-KIv8(vtG&?%x~B*G1$74CoAXT$R@#lS$y2bKzQTv8B@{hszVHP{pFmp@Qyz46uu zD!O5EtmfLVHivuVSF| z6O@Sj#e_#xUm~+d^KOeDjvkGkMrZ?#q)(|qkisV|OV1^#7DRrJdWh=OV0ie7SvOt` z(=AVb-g1S|yw-}E`~2dgfqz?mo_DeH<|=OfGbU`S;WW%78Esw!uelmd3${5w#hx#)BB^~w)ya`HXePL1}Lj5`4UHiW%gLg ze7AFs*`pD0b@5L_6(ul3@YlGeEDQR-!(BtHrCY{`rpA^o8EIU1c`HZZd6k6(*xdFs z_0@d#ZraaLcZsse)_G0Uns@13SRjBrNmk$U$daFxH z?jJVHiOTYzhAdgiKa*KHnuaYg16-JYs#`tN9(op&MAVWqu{1x%H&IeFST)%{ z>8ro1>rX1Hfa!LB&suWlakxgL6~tV1QMb=eQk8rhbhhrxJ9F{@@r){u4J{fE_qf}` zdMEFKpG>rRA1m(H?uQTk+zTpN)-+trhh;p9776|9`N(%ArIz8dwn)~7!E9_80CEy3 zV-a!hEd>Ed>FK8qx{c+2_^G*oG?LbIzI0acZdXnUL-N2oCp(n`azn~K(Yw>ym5>NOBc@<3McJ$XQI%UcAwQR z9k1UZE){OEnUsa`#P9(wOZEsiM9S=kn(bP!c*!|Ju!1QOYl71;cc zM{c(qPV396vcbTA>w~(TYH+k;m}M}$rXDM2ZBFh(LefNOFZfB+wo;X0+^^<4=f6Az z75VO9SC5Da)jS)??77$#uv=pA$x6{_R_kg$xA+p-2jo4HY~QJ%^$0H#`1YS>5w0MH zJ5as-2>lW)sYVEpZ)glulA*fB%7|s%KOQAGid~4{SixrO$chTfw<{DJr(j8F9}|Pw zk})rzx!aIwdb#1I3bpAouD->({0E0VK5b~A>au$9-Dd`~I`w(+w&|*FQ_fE0m9UPu zeeGHT!`8(M?*dMfpz)HIdrKjuxWz(xV05;*^+!$m z#hgdlzm!KKPA<^?g)Z5d&_jmoKVE~TNFXfjsTgKdHGR*XwGvuS!h;XoA<0vG4*T|{ z;d8JzC5nxY*B-rl!+rt5I4hLM8f5X;zBNUXimRC5vQv>_Nc013v}%hzXmIiKU1Jm} z$U(Ld^j!F}{ak&oz@WhTI21TQJ91mXA0w&nEs~f zE=x%ZyFWLVwSpSE43BE}&j0*(sy#4xe!WvxWTcp(iUZSzoo;}n4zZ{qn`7JABeV$3 z-f@Nfnnb}8nJa;?&;BdE%GQ%(=Aka-6FjjS@)AlS?3Y(%%`oA`M$10zh>lr$rXGJM^jgb@g{>6XUyQoQrr(r6(shtfr+h>3$z=!?L8dQJxp_IHvEA9 zSXq5LVxjYYJYr$hXXBY7@*{*7cE}21K!hNQ?)XQ+C)|@kC@U-_t$emoN#inqH|;m{ zof5AmiEszlq$3GhEvwa$oikEe|JB;IxmoY=P!n-kc?PrDhLpk+Ym}6i610z&_KIT% zHai7CxJwOyxJ&5`tmVJrP4v*%6@^*ZS{`1a7Za)r04evt>HBj<)+T5X%FFM8!#j~x z1xvy$nB9Xs3WcC>8T+ax*nPojkpk!HTY!Tw0428wAf^kYBB3=_R?U(Kyh|4PUm@)~ zPXHi)>snUgp27N~Ljah-*mpDE^g$F=4i51W-S#%Jp+%baaDwdKp(kC|rB>2eMia3@ zrAmliGJE#w)w7Nz7hQ;Is?p%Q>a`>1Y*dUh>zKDO2<7L+Ax@J9~~lgn{rNIh)&0A29B!HDD+D_7#6cc^yW!DkmDIm>-q|8f&-8xBU&|BCAIC`g#Q9&9y0qxiKC?WRF)zr3RGB2lzR^!V zU|v}HK966Mg;)L_t2e9Ft7(Qbq4=M<)vW11WiX~`!s_@mOW0}|*nsKmPrQwGEn5|A z8QCes70z4CST#SnczDuHGO&U1Udm^CPbhFyJnlV}l@_4OT?CFV3Y?krA+Knvrm)Gd zr-Sa}CVVb3fq(??VFrEFLklFl8M>VxcZ|AKh}K8VhhWT2I=W4!{e>=IES9GotQA!z z2P+shRHDCX@QF&A4TGg^F`g6NoKzEo^cmw%DkG5`v*4@c?V$TFP93{-+jsTZhFact zuqfj5L5e&SaZUGzOH*w&B*d8biB%vt{5lM~8*QH3}U2(p+HX1 zdMwIKxUbT0@E|PRE4YbRFxecIAv|%JqhFouEiv}oA=nq2(2M8K-I2SvpkWjnmZ04t zmIF%J(+`wUeaOzvPK*XD!$Sv2=jPGhB}OP=V8-(W00x*HTK{O{yK!S}@3MB?yR*8u zyKM4ept)IcPUUg-kE=HetS2SKb^8h5(YDQPO>%COD1fNj-uUlIuk%NEHveYs&51CY z0dpE<*W9$8e)x(jgB$+WWbGj%SzY59g?E4s=4e>!!~g2l7$~`%i$A@(uinO}mL$Ly z47t)Vd4i%zzXFJNDsNHu%Y3YwC{&|WUzVjWc{s1;*HVv71dPXi`EXOubLhSA%UU+b zJF<*QE5>z_TtkTaHV~%{Bx^2mr|ica_4}aB1K2-?+v^BcV?!<6r;`O3lB!j8mgYk6 zse>M~6lvN7lGY;j5rAX!Nq)~x+^>;zz{29(2wVEdFM7nNBN)ocpqD!4a z115ovhYNti_2=F@Q8gODJ2g+zk{a*JwVu0WxT$T$;gU7B=R$(`C(xlyo2N`h{AlMF zOEedZUH-Gzk4gBuYgZ+?D)Q%!wyI)lm=m76fd5IbU9D@eh|7 zNsXevwqw52E6M-*Sa4Okut_?WJB*3l3Vj~%Nq{XMLd<_smC}h+QB7MnWd;GvF z@He|Br)LVNSk)kC=vqU+5B-&f&gaF+Mn+S1D27(`$^Qc;h{})mFW;AC>nvUvTHdA2 zlMc0eSP`P%f}73%1(23ytaBuF;#n3U3Hc~TkfBXUt5LzZrg>uobij+5R~+kkvZDRS zk$Zu5`Gs}^Y-G^+%f%duJUiHtxY(SIf;a>$cx@w zzSM)(eKq>@rmtaOr_7SFqn(bmHE-PV*Wv%R`lITAK|j@>@%!)le;%-X_}H<@OI|vx zdvorhWno6amxALqnPF$x0MwPxdQfBw??RHU4E-1>S_YEr)T?IWKQAFHONbJ#o4Pru z`2PL-68b|}Ehbeg0mSc+`k7mJzBb7_A-p0U#U)dj87w31%>Cpp&;0u7wEv3>k;JCT zMWgB9CDGK>)G;NS`A(!HR10b9iNZqrt=f>-M4}-8_FerX6N7=RYgW|Q4s9X((H0eD zYMqX;OIsA{c>!(UlLdnw94Q?+nf$rX#^pejDt;^W_rHIxF3}^$D2>oiW1tQ6^cpX4 z5^RLXfx*S#G%4WwvKYjI2!6Bb4ZE5FoSDT6D30`aZqJ^1^keM|X?R(o8HoNOAO6L8)w?#dh_cN>F_&OvVnvTQOkxjcX2tG-5Dy;_NsptpE!}2gHehA~x{tAM ztvdFMk8ZUkjl5aW9N+w*4DOj^GghqcMy@SY`gKl6j=UT7Xt#Gi0wZ#yY$w+YGRRow z%f96F?e7!*mbeUdGmYD{Xn{#3lYvA#(^lmp1WWDRpFKBl`2L9vbU4cEB!j~=j#HXU z^52l_3H62Yh2mQum`(@}#oe%8r%J@bm#5#N88DpxHDY3C={||#JOZm#uWseEqkK}* zk72o4dVjhAs8aX97`LF1R=(6;5OT71;_tuz9@eE}$KhmX9fvxlu=*@iHmDKSZ1!6A zy?|b^amFW%PY0SS2Uq0A##HQwi(v+?*+qJ6AB6_zu1#5STEK((t|gOYI^S8XpiQ${ zG@@g_$rXuRJL~TV3M_hKMs<1@S)5wB7PLvT$c#*B{(yk!W~haU8eap$=O;NkTc&w* zLa0di?Hr>t;tT(Ly%c%7=4%VyV4;H~J=)BL4cjq!A!ly=`m>d+I*@<=j#b$H_2HZq zYb1ZXWXX1&a<-BA(r7D@HB1^>g@=0$BNfZBK`#bCO2Ku-t{_ z*-WQXC!+Jty>TSJoL>Hl{?{qh62^|UI-atw{Y}@e#wxY{ET@sCY5G$}DyQ61d(*<4 z&dPUcyvw|K4f_j+s(+|fWLnEODSQ3tmm56q?y2uObJNeR(zC>^axr@?k}i7HCaqgo zRto9plhcup_U@69v`N#ih^G|Z2qjBwq9ZSc37z_uKzVccUd3jS-`8XS5)m;>BApK? zmLP53PhkTR-qE2*4xDU-=$!$2E{-L*J+-iO9SD0TNO$^_>{7;=Kp|I%>Eh4(CO1lm zVV-^|8FPWun?VxkEPIsmXi?FD+++`+0*3x z`}h9Oi$lwbyGeMM9cuOjB3B>1oRqKt^&w^qcp$bw@Qbcxkx4dvInpuw;cJbye#eqIWmn})PsrPoZlu72JOX3;54>@9 z*!#ytq5`%DB0qb2>#W(bz=D_1&xpUkTwV~_$RN}LV!Wo>3`!A6Foab{J-tM6@%SA2 zsj?(ejL&55zBLg9kJmgHu!NN%s)UmiTW|wHN{4s=^WVu1+V_zqy*diO+5N7xTcy9L zLg0m*pAFmpTWKqiqpi7q>T1r8rpwcn?^V5@5%5Hia>Vu9R2vZ(dBs}RFn^v@7&U(J z$XR#)(6T|H!uHDkc403-=I0=itXZdau(8C>z)UQp$he`ilpq~VJe>RcrVdd9(3gzE zMgD8fxjUyt?}dtBoD<;*4AYV$sNZhieK7`3z3_KHrY{exn5~hmkT8iPvB|-QuTurl z0#VIghQvxqhJ&Og*(XglSs5-K#>+FFF3Rg_Tjp|Eyr8e%&hF$k?aLp5gC<>6>l*j83H7mBE!I(meOP4 z^bo&Half_kg?i{pGS)N<*aeRu7 z@B6o76d9|}p-G$VC)H6#ys1sFA3vWFJ$iSwTU#ru_N8=*_it>S#QPJHTOuATiRitH zs>7N0b@$%A6Ef>0j!1Sgk%WK1mMt5Oxuvq5aD4^mLa4&zXnTd- z8A{x3Bi-})X@8!Ngj!WjD|u6*EZsJoWyLhVT!LC5j+#R6Q;BahSHeM(5C2UEpIQNA z`hsuD##mn=_>xE~vxRe+B|>)kfhHe|R$_?~YrJ3)b_Xe&qDR=V3WT;`lg*dI5fkEZ-kIvbOy7}S5v%9Up&j~iYTF~FloR_(M;o|rihQLE(o!=CELWtN4PX0@c! zAQIOYPVaVoT^8jKX&Jv&6OK&X2{HxuUXIYV0*(Fod*@6!Te&T`urI=i}x(-5^Oa|cFoG;;F6nH=jY>lv=RO69?U1A>* zV-*M{p0AqQqh=z_1b$F3ozUA$awfWTxtoyyw@SFSQVSsv#IEd5+gk?ZdyeUx0za6@*Cwe3ork;XzMqgS5eqyb~F^)E$*SlTRI1z5ed3gQGaGi^M7*n-}qHsj~TI4~vD;I3o~9ZU)xOd-+) zSW5IF8?WH2J>w_T&Z(q7v}9ypWU6vCsr5Bi)6Q~(Mu0^F-~RmvD?fpLj6~60yi`+V zl29=E#6&1ITv>U;j6_BPI}S#Y*QSN#p^>2-$pVu}#LCpT3Y@p3x8AC1XVi$^7Lx2? zTZ3FaMyW}hj>EZd;F$SL36kODKywyw-Cgb6Q;!Zw^m+%zBEtHUmyCBx&DsDbhp96( z@wYEA-#lMCQhzedAWq9otTlDLKb=i=A^$VTvWq7Zig*cu4~BK&Y05hZI4Di>6Ul5t zS-L4OY+;>*80=jlzvsA*D*)+{NQ==&!^bT`DWcdQ4%-pDqz*bTaQ#i9ZJ9>EwfV(L z&opSS&>SlWJ38VWdw1+eY+crZA5;v4MJva}vjgJP+rpB(nF|Zn?sa$XfO$vb-zka- z`C2;?+#Y18<)T^$T+2SXejT&vOBvf3lrOi`89Z;xGfU=~b)o4dIBZ{vT@N`qlsZ}6 zH_WJO*U@F4=9gWWJ=wUP=w5~Tg~T3LAnXqas^rg$t*~aR3R~gq5#w=KOV0za(qok_ zCnvtV2(izgz`0Fuh?8z8tE$Hh8M5~OIXXlIt#DBcJ5&0Y`S0}D&c#$5DdonMTk~@ERfrX2;VM;!sVzp27A>#1OJc$wV!7 zBw1T#H6^f}A`oB`w@T7~M6z$tO!GJr{t|PG9BZ@Jq!wB*flHk27Gz~F1vv)GYaLnP zK-Bv~6)xBd6jscFCx4ms1NUW%-Z+;1`95C;S?HVf7s$7;&qICt_Vfq^+Z2q_a@w>f zJ-phpX&Ne>UovSKdYL$ck3eRPubH-4GodGS4K68N)ni?|(eTuuNQYTZwe48xvd&$* zz6sRI<`Q@2yY;mbrAX&9=~KoSDZPdKDC)Ts35k;V>rmYMH|o<53n(dp=(G5oLinjx zAV&%uW+bj>xOZ_Ic7>v0z5Q0soo_B@${7bHoUFr;O*ho`08XetyH$ zIp_OX?)!bem!HGNs*hB$1jc{k=86jId3^o@w!JP8sW-9pl%Sg;q_$;#YDqOR?wVpz zC=~XH-N=>%ar~-ZLlrw%=1q8Vnp<02=dBRFv-87v`=a|l#qXryU-r^Z8o1!#;KAOE zS`%`%4pAsf?b{#tPGQoS0JuQxMF^{5j3W(yDU6eq#sLqRejkSpEVu4eCE1VMlgbmY8d3mq?9SzzGuB*kVE z&m$tHXOygD5mkh(QY^7B?-V{XR;xc~&(fLe&H0i%^W+XS?be8L+u(t>xA(%G^H~|< z)^gCX!dFe>`MXMl$goTj4Z264ZtPP2j#s2sAp9JrDZYRF46QZ#BY*PW4__WHrAcy^ zN8pYkxx!Cl90XyP(F`Y*TGrzLe{O$A*h+h*ovWxjZB zPTMZwcQls2?6` z4-`=4*O}8qY9jhW+q^3#l%Mz0}TT*et?#oGu)40qe%uFb2CA0?yl8GxH9Gt5{>W&4eN#y{= zk|$&&8Ni8O9=-9a=2?i>AY*{3YKa+>RuUpUM5N4$v(7v)@~4wFQL-j>oSmT?kNTd; z7r$uI-j_Ugl+o_zUdCJ0=H8Cv1JJt3rxP=)JOn?FFXc_^ep|CPFR9(CUlrB_>09cr zco@qsP`aeSkPI#m1|$wL%)v7P<+Q=rWc#;D7EO~XpZeUnPPOTZkl0&9SqRE47-b5E zj|hE^(#T23jRc;avE{y3rzm3$A7Xn8AvT7A9du~(xh{;|U+bs(@9PHH9Sj~jI{nebh&Y4*5x~L+z^!7H*FH7 zC=WGxOjf+TS96^-B+6Q4^vngsus`*$8kG)BU7U7!q?b1%PISTui9}ny_EXEmA_+s4 z;Qc-!Zw6=}HXe(8+Dy~FF2ERNuNHb>v$iE-67YF1mTDhfwi#?}){X42mVdtXD62kTS}#`ONJm~BOoJ0NBTOQAX&B|lyCf}yc=v{Eleu;ME$S)A(j9o=2PB2m;U8$ENyH3CM^ zePXos|gP$Hq%>wftQQeBtr25BXoWI zTPu(rs7p?KS~|~48d;-v_*onokUO4wX>Sji^bT0PI`6N7>!a&I6?F&pZYv8Vro+44 z0-;g15pJEx=<<`2k_t7kuaO!72D*2Ow=6Qf4@Fc{ilmR7I+O2F81 z$TFX|NDi<)7*om&$O2iSSZ`K*MOm_Elp``_8U8C$4|N+h?9nAQOs&5zmX^Koh~_*A z7ga7*z9c%WDSuNWM-#_;>;GAV@)2g7z;|=ar>B4Qak3k4Du%e+$?00<_!5wtYNv^%G6xMfkJ_|9!F25+kLVGj>(x!>7kQU`ghL7!t< z`aaNMb__0cw5fT`;@8K#o_P;dZMmkEzxMTlodKrhK3$j%0P{!CFpQNz@hpIx$G_CM zdhj{Di`0GMq7Y8q^+KGGIDUjtDAN}STvKXof;a~}T(qwCHI0dAT|{)sRXi$_-Frm5 z6qT-`UlLj9V#+c)Qzmyf`z>CFE1w0CbK+9J6ohYZl2|+7PDU#o4boU569oi58H`Am6YOY zYyn7Afgc)7jk!0)ugF?FF9oXzkuf&v%e$zAD!!gX$lN$C%NF9EiTN6EGayx8*tMmu zPcPI^W<+rwUz0q{DLWxE7-gqSp8{_O*sO(e7a_HW<(pg^#IN8HB8fOaMV3Mz`}qQg zes`6pNP)713Hu1;wAp_ZY^Fce4OlPj$~8zx)|8>Z`}ex598@rX82GT_k6<~xw4i;Mg29#b2K zqjHwQlG)L1P~ChB4ZQ#?5LX6S@1Ms7X`#YlqGbOHZky3gaIXo9i;#u)9ewZ>gN>n8LH}L zZaM9Xy*rMBMj#qQoX>xqSW{I`S}Pgu(J(d*XLC#vg(Fgi&`Si16Yc~RRz^7)cUj$t z@cQHDdT)NokQ2k`z$Iz0I0aH=bl%u>M4b==7;>3&3K`sscIvsn#SVoj?#(>#?&>L- zt|Sb7LPrE`797 z(TbroqZcgpk%%T+kxN{eoxEN&P!uF`g;D+h9UVClSjS;O&pYdvzO;)36bc$QaM>}- z=W*>1b$WYWSzM{7F9$GM<7=|V`hl;OSXJLSqZQf!Ps?9=T@F2eYyBgmvd{o$Q$Pex zxDIiWEy(P~DK%;IjZ9n-qPjlOz(>GQw1Yh8n^V|`LZucYjR?R$mTrMSI*t*4CKpLK z2a^$2qoXQTRdz)4Nqc(F{9R;cag);!GfHrXT98#ub#ArvpwaFn-4{Gcu$bZgdCu%6 z_cSGOQT*E|Av0-;nwM$(tTPRGMNG*ZaM6)mNgSIGUKx2{o6w%Ys2!O{QtztBF3UcR}sjW2OFU>vT)I*XC%I0!y;Z8qP#LBC_oB<#hyS^7I&D9EC`LPmd7ud-`@ylk}RWot+-_EZfJL zHXdK`e$b@fVVR2{BTSAIm|=4#{t~p7i+NN@zkdBr8no-Zy<%7Z=BKPPL5{dT0>no# z;|Qe`xWPQo_&&3^6~xh6`T^ylyF-K81v)oS3%RXxLu}Ff(yFCL{d;!3CcwT>klE3# z!fEM`HKX1n3xYsYYi9NY|MR{G0U)rLF{MInW zvQ@3Zl8Y_=t@A#wo#*--hO)Q!=Z`-&we{Cypp;ILQ)9FvKA>!KX;L@p9yEQ7jtjyx zvYNh4=XeN3xkT7)H~X9UOC2dNA{{%(Cn^^J7RFm6goAnDuXYhx$;SQ(JW$0rujab# z*3`nH;DUD$K*i)4nnQ;O8Dp1l7V}ncfXZzB?Zb`sy=dmRiPP|O3Kb5M_ci#uP{>^& z=pK9fi@XPw>TMGNf}eTy8l!VI3-fNkN8gWkRIyX&^+fGM=xD2OC5Tsqpqx~&v}cLO zN^ow}I8(F*k~|O_8|$`mFDpng8~<`Y-SHyUsW-<=+|mW*Y+li2h8-UFz8%UMZN?GX zS!VGl^E-c`?$8T(wdtE6maHNm&F#zRxSg)=A%_JdKukrj;ecQ=}@T8&d{=jVsVPHlgPuG#Hw%wf@e<=e*ajao2?wwld z9`m|#d@0VRVKj|1poV19AlY5vtP?jDCO-rLs#H&|Q<=(1hG8U!*(qQ(I3OJ7EF798 zfo8xfxuj6d$GZ(ONNxPbh4tT-{Jrzc5!I8iJ92bNGm~3wido{_QC@V=XQ43FQ$T=J zt-al%GoLjP`DE(v$**^Z-kMi?%lj!@9h_UIneS=E$=r_%f6Mz%4lBV+MSn7Of!&4v zH8TEZ?M;`;18%0NNs_@$E08fRn-005>sfDNV+l!2x9ilL2Xv{D{cNESX;=Q+$oV>N zu)H=Vd?#HFGvjL$m-?i^vz#2fiI&f!IvDB^eD4VG0!i9B5CBvgzlPA?<-GVP*h9kD2GnJU>$mACb&P zz?vv|KTt8-3N+Bn_a;ph_GYxN%LzTY38k-x+S)5L)1pDmb@oZ$rb5P)+wI<~4}SN( zlK_%tAJcuTwYx8Sjn;-Cjep1X7~^y$Ju3kr|ZhKfu-?G15D;%5TO zqhdvQ1~#cxy_ncRV+!R6eF>r>W)a1*#x3rMNFh#_R_YV&w1P?^oZOE=9AZ_HO3_PE zP?lCE_{PTS+8>1#;oo~^)uKW?X<~2#6WA!d!1`d3Bj=CsS}JDz)K!J;2c>>pz5KwK z?vt#f=l9>+b)T85nOot#vC4u^eLn@|(#`tHYYp>wkYX|KuPjMK)=6zU@zm>=sqsEt z4ZghWN7vFd?S$I?hq~Z)*r3!UHg3k_(!c`5ErBOv?2XhAt2SP0%YM5N#UbC0Ej3P& zkuwWis>6ua7&oxp@c=jm@59Ho42b`IG@GD&*Q3&1mM}S*lScQ7Q`6GR0@xwjc($ zoJ7c+!g&kmu^7uemvBV!70j2Dnv6rSp|1G)G>B{x5o#TDlna~cXg&y?BEK>zETr|A z&IL~jQR-mf4bi7=2(JWDYH#FbStAMo5wVNeb|&E!+N2DLJAl+zK~Fh}AeT^x3hHW2 z(ZopvSf*AksV_3b7m$)|01F)=nzz+S-xDGJib2<|8E?#5_vnFtg>uQs{s$G@w?FHR zoG@v3;i#4AGfuZKELyC_k`@$Sj=|)%gxIaF1vU3@Wjm#XjN9U0O=m2oU;9bjmass_ z9`CEHhRY#2ZM-3>^i8ea+&=i1P_q?8J1V~`UU{$_X6X%p6Y9lrmhPxGRipucw88ia zg_U9{l|gB*aE^4TLfHC+w2xa?)_b>3YBM@BOFKKe4u`811VSmnt`8B~p210cLSh=) zsKXeHR=ItAqqx??6T=%tw9W3GnDO`q0#Z8pZo4-{{6^`r$>KKvtDRs%(q12NvnKu5W%q`>_;dc(Nc*LH+ivrF9HdCiFR?>{CrLD0Bw9}~SJhP}JgL~-ug=y_+J*vV0S z%qqBvSy_;bk!hhNMaNEIgK@dBrlnX?`5pex=>Vc5sT)0d9QFM&6(nmXv2Mh*Q%XwjHs@ZBAeDTb>yIzMOux43GQGFsh?!?xi-X5UjARIx zS~YgUru5V87VK;}xBv5Z1LofiVxB&#bn)WFVjF*uZq4j{(B~tC{tC53aR@kPgS^f9 znBAX}4%)A|re?GR6Ox?}xg_V$KoNldS*Hb8WVZ-+n6W@h5pNVxiRn7FUrU4W@|B zDYhN^M;}n%5Zn(WoMCYWjyuMYn?%DZr2FH=pb5fJ0Tq zO&55yleTu>yb=v&MARdMEgPL$y8q=a>X6;vId5#Jfe;i?U<$cdlNK&h4}~UbV_|r z$%qB z$i8ctzj1Ns9K^t>BmvGPch`t3eWnJ&nt6LtPiMLs*K0f2oZF^PpJu=97bQQ95To%v zvp6mB|D4-rZ{U*>w-RbfN%hO*(S_5$cjzdJ<6#uoeE zB6Oy;=yw&?NjUXaWNG;W{Kd)`{%wpXdyrTbKqV-%=8CU^#%jKVOr}gqu9UuP*!u>y z;F7>KN<7d?Nz{aL%`haiYRn9#Dbk!@QJtt# zPs7R)cUa|aLz^1`P;maYw+Sh2+4^vx)K;IMhcIZbJD9^&_JbU+;^$H1NYMo0b+VX$!Z?IyS0ozvWe;7+-u52=kD@RD!TtR0nPKO%a zuwp_cBBdE?N_LGnFnM1R2(_(D<3M!3Cy}KVO9Hf5W)=5`U?K;$T~sHIB3Dp!F}=a*Q6Wn58w;2X97o)+mB*2!0v+i6XzdQi$i-Gt z9{Hahx3X)&zVl~C$qV(&O|rAO@iVg+es7;wI3iJRe`H^r^z9vfJv_VK)#CweNA95F zIx3_ElQ>Y8f~`Yd_wL%r|J3&xRHZnyQ&kOL>Ta0X5W5)Bvo-lT*+X~{683H{i|V$;Qt@A$2EoW7FbeEr%r@lLuZ0!Rqd zu20Ubz0oe60h^42DHZ=C@moWNu*KCZ%o+fyKe>@gt0tr(inN15ISNzozUo0!QDP}p z+0@nj^nN?1g6jMI~!X_H8pMADMf|Ig-R zKgq5UO3=P0*$c%a2@fKI+O$KQz6nQ*)KdkY8 z|4>r3#4Er!5SqxMC!XkZg&W_`8Gm9pnc`2}!+JyW9E9M)RDt6-JujYK7u)Ix)Ib=x z9$I`+T@|T}FC7Y4O*)07^58(Zd*uOD&|10`Uz|oEM{U0^{}H&oc!&8n0bgG8Wf?GO zfBU0W(Y6UFSF~LI1OY{Dz%Q!nUjetfb?tf^X|TrU!}0FlX|gj0@7}qyqvC-d+mP-K z5~UUGz8F&=0GK{~4Z@=-J=DS@tB|iD7@AsKjC2#%X`<$E{ z;q{JSAj|YEX9a{8u~yC(4_h1;`p~hxZ)@OggJjwL=KbF)O&BhhS^#^S)5XdyB15@; zLd^ko?9LqTkJ&8O$Z4qX)EUl<-C&nJf4740IFnv^5)UJb2(kT>cI`4wT{oQK@%bO~ z`4j7^3{tf6v(EI}X_xa^;UJ}+rt^O?%utrUL#N+oj?~=S#rIU--@qo5J-c5H{eqC; z*8N!L5yJ>9hYOiFI#b#&W%S%>gv)Yy_9X9`#0lF%Vs^SVO8p!7FHT|{4cv#?_WDHHWe0Sj)cTUd1k-KLIrYA zFQDugEruidR%~Tg<~KExrlKFCDIB$C z%}Hw%WG*GCAuLGnRH>SAi%djTWRJB_CFfR10(V?o7#EL*K{uw9(WAMT!u|&~b`;k^ z#zdsT|7yO5R=5DKF~)FLO0p@KIRm$UtLS4XwRIkFv=@shTyl%uA=1OrYdG8q+=4vv zzWCdLcZyZuEV0*MO61p_IwkcIl3h^7FiUWmnA}fsf8U!gwS9X2W=AaZOSZKkaFH7* zP;MZVZtH_Xv#VrYQ%$Sj~N>mz?T)P}y?mgZ7KzV6rohl9|kH1>%7PP8$ zr_245Yh05C=j+$6Mh0klg{^RMUh}M%a9%Ue(|0HXBO(Yi(ayv@cAyL|s3?M)mwj5B zo}U4Ko)f(k#ejur>F8>>gDkmmAmMrIUs@y$V5n`+MP483%%C?if$(e z*=*}@m#+flirW^K!dwRMrxmRcjx8N=S1vh6Q)?)G%+l`fQhiGIC zi&X&RGJ8=jE~dO$$ONJ_vuh4>Eif;6^*~M;4cs<&XTR}-oAiG^hp7WTsb7RmM9d~Y{W+ZZQ^m}l4a>}) z{?tUloacrRq4Um<>(3T z-T8J4#Gl$fNUyeKa^Y>uuK$FdYFQ-1Az7=L0)9Z_&PGLYRu9kgp zqmzou+!INNGx}dTo~e<>%cJyvqiXGpz1lpvTn4T#`It!8-Hpkr8nFSon4eY@B8nt_ z+apbrdIE*4vz?amLGzEvSsGT1pWNPG>%2>1n7ainCPLW~4uo|G0r$lVAx|?fBkU}- zul#|`7bW((K*&^~6Zc)W!BWV-E7v4l6>B79r(nwc}`j@ zh1B6`c_4Oh$mpc&jI`#q_e^xTG)?!!s%djN_J0+Sh-nC%Lm_F$F-|DJx_8#Wwl>`2 z#n-z>ixID*jhRDNib^4KFk1FI#_q*zz=FwZzxGoLC5%~Mv`~G7gDmcwe~j9KcwHv$ zCIc?xefNR_&lK_k^Gub4JD-dMg94gm4FZFfU~{Tpf;EDkib);pd2$Qi%<&d1QD(P{ zy$1-oBVXs*K%o+ch=Y!j1EU1{K>`}c!`g=u=3>}=al1wA!pX7iY~b?3k6%!Bi6Z^% zSsNBcfP|%>0aI>|QTH2^0nv!b^^*Av-@fGse%rVu%d;lN(?=wHSM7X7NxRnYNkmE$ zI7@s!vQTI;aM(bPXj-k~Sv}luwj&qN%x_Skf&33Ij+`c%!5&k7q3{Z4&YI7Mo1;G_ zwj1etS-9*r&By=c1#~^BKx70ZQ$nTJ`B;!SffH4s=2OJQx1$&$2J$xLuiX{N8$tTU z04(a=I!d&l+afS*(L#7gaa28SgFLwH5Q!c9zFZ(4?OW4KLps5w97jlMjnR4b9x`#~ ztGtNViw}mreu~T9zx5AKU%u?OM9^qXzoMCa+)uaeD8CP~81$yji$}9sG^-V`uezbx zZ!zN=EM0i;Lw!Q$qOKkq5H%r9LKT$}Zms0U!XlV|@3_oF{rjP47KX>s~&?Q||6y8`NyEZEh?Mj7#p1C=R91U2gbA%Q&nn`dvq-6Jejjwie3emkfV zBX~Cjkd)0C^VhDucKy>5H^p-nj?M8TuClQQ$80f0fT$sLuzaP)<5Na&G=~pstRqM5 zZQ)OTA?$K5d|_b&zW-8ww#zf9AN@J0aR|{tl&MyBotiJaurlr$=fMZ(0_>v zQN|DCLS1-Tm_3ke_2FER=mls&i4`-aIpWTN|TeQJR~r zVOntJ>!TXekIpITs*1N>(Du2^nuJrAwlYZJ&W1TI% zbYq(<$rNWHH26H6V@xau^^RB;sDby zNyUn22T&$?FH~&4$?wSg9h*EO5QIRCQc6F!uVT|5W1+3u*YmU#aWyhdeLaY|JzFg|mj zn(xkh>iHJxe4S|H=nIG5?3rN7VXsRuE|iy@1*_W4m|Vwu=N|tmD*u?V&hxt;UYDX? zQpxK`{^`D6*(@3vxT8m2_ult+{h$)Ry}rQ#{rt1nCkxmH9OV;s-)z#bPRN%{wRD%m zV$R?pwXs+wH@T0`EK{u~1>!ag&ajQbvK4gy+oC#=iny;)8dXj>2j}IC!z3TFNBg@6 zjS=e_{{5f=`>6~CfRaJTByr9&_Z2HqtRp#(3)+ZJ7?+utWW=BMtO-*&&ZFF2D6f=F zuI+Fv*R{J;tK>~~`SI2bDtI)>#Nx=FhsvxOPKin@T^tnwIx* zdcwB(C<1inT;J)IT=PjOxA9daXLW`;z9DOAJ2Yl;VDYHx>6QOhAtMV31VraDiyhuh zjqdZPZ5~tomcU0En<;QPVoTFyryhlvd#>a*Y*@1L!LW}H56a*Z&c+}XYY@|(D-XM8 z!!#cD_FOe_#uR~B)+|;cQng5cQu9ankBg_1kRz4T$_A>J*gSs6lrgepvYU{yZ`-zC zDwxwk;TSPu#ITMFSK7MeccK!#lmQwbxQSoF*{{dT4!i2^w7Z{!6lOq=qysZwO+BJ) z>(rqlIlHd3=RxP!nAt>n*Ab7%bJd&JbS((jGke))CsXPK5meBVnx2A2EH0Ey%nl5~ z$0e~vlu_C7g8wca@nFB#% zXd&FXJ2u%1%|K#6=I{7P5YOsW#cdWUnMt0#3;i&+su}zjadk1OrMnsS7S8FW;&VgA zSvXX?Lx+*f9ZdKbPkvM1CaJ-DnSdjGbu?c9hzNV(cR<^qr>BWd9E{&xh=;T~oMf=t zqqcJUg@@x<;5E-ZvHW;h?%#C{0kn}I^9v)ie+v^z4Kn<%552-q7C4-)vCk~f>`dH6 z8<~}JofR`eA5|*KZ{qR_YbMubWm#Ftz_bD>OxeHX-q+{jVOP5rWQ?UanlAbqxt>G`G zmCdAF<|8S`@KLNsk2^Zk1tJg@AbOq*QCm@($ORw_gMN3=pmCW`11(Z89*2!0W%Tj_ z+L0>xRnn^NHr^DOm5LxL4C@l>tvqJI!oT_8kBXxk5}x_RA3tIw9=^q*>+ye*oimbZ zc<%0))Gqq-u?-mMU=1f@;VW(=xDDqX-hqxQAQzJ&dB zCA6DbYdiktl5oQ4M+lQm()&x%=ESVPBysMI8#e?p;=P};$OhdyR{}tx0qfNnpJXOu zA0(n_M3(I5Cgn^ZuDtl7F7j2Y66eWyruJ z$mm!@umbtA-VClCw*1ZGQVxj@adK4yMLk=?@ivh|NMZbz^Znj`DPw-F9Xi5^*g&+9 z)bjr-MZMkqL#~ntKFHFHA=8giMg6m%ds_X5S1&i1yl9bTX56J}l(U*PbX2m_sp2B+ z3+&uAs~Fo*{D{n#`e{O`d;8arL$SOaYi(?toU&hJQxJ?HyzWFN(~$|_8K84S0TRLc z*Tu!gt$LNUefIH7bpBBF+k$R&U&59#Shuja?erFh9sgC0Z#!~Q5jHU@Q@Cs6tHrj# z4MdK1{&jPUH@MogTk&M=nC#OWH#Uh3y#*}*u=j3NND~B?d2zDK4Ss-U0%*F7>p6hs zK_X$>`Qz{dvRvJZ@exJ7yLU%m_9L{>3@KDAL`YO|9IPcDT>8&}qigUb`yB(V+e{sl zOO_sN1=METb}ScUpPaTaCwIW7oGOMBa^L-AVcXi&dc=NaejCDBW z_4WRu(1N@M)l*;A_^>ej;gE%9OtNlnT{4}?I4qLSVXf~;_EqgB+6W7i_kbso`EfYR zNi5m2a1Ns=$`H&O`6_pX4@dpY6;gnZvEpnr|94~u(9v+)+8UOS>6nQsr$fLmv7?|d zqGI`ngR-rK<^bot5+W`@l=U_$L|IN^JhEWHf}1O74#bO59G5qF<%Iqe+XFf@==#C> zn)<8n&4Rmm2tSLM;r*0%r?UQorcejK*U@@{DS!`1rZ3aA9&4fTLg zBJr-J-sPVffaYKA0}tEKYk$#Cu}gVt%usp$m6A#mlf}k)a2@yA^aS*#s%!4`d~RY& z&O5P?+#7ER-ahBd9%`fHI{Hzj>GT4F3Gxt8J*639hCk z<#oa;k!&Z0UczoIS@P`%u_T&RuT{NP$}Jw}nAxnYQ_MYKu}b1nz1i3&M8SL{kHB_y}1#;POKW#p5c!M`R`ORrC>hLOR+BV?we z@Ir+a38*0Wxzpk%$EN*EJ;!O$1q~E)h3#(0iD;ZkaODl7@7C6yKM;zdxQmEenFT1) z5P)!eK}6^3?^mA+Fa;A>Zdho_hh-Eha~{W`d^@mp8OR8?9{w3pgpyw1w2aOZWx506 z2xz32>jy$VQJhnR$Uo%GX`heY@*&Wr9hD`(p`Sq`)F>Qp-sR*6JSm%@(%JOLSp?)V z12$SfDIpGE8^cpGK*kkqlOXCe9apmqENCe0$CeXqVf@4J#a?n_AI)ddBU44n3(3vV zpl1jK@eG|5NG@n-1-%LX)Fr>d^yKV(eD_$Dj3!4iHKx{D#rvgAr#lnUuceL6>d|?y zo-H|<=r}u!V<=^TZ`MfZ<$w^gMXflA_+F!75X&pIUy~S+ zlymagtZHjLi7|~5yqjT2$v2mufIHz%ZZFSWd@J`%R$Y~Dv!tGP3hpdhZL16h9ZGv# z$vo?$6vqzDJ~aOVG3GiLu7=^0fXJWoJQ9bGq~+Z2UYc+3k^T`CY6CeaIFjL2sr50AG71FQJ;guZ{JeOol8)m zc=qa*5;D7<7}oQVnV3g13sJyJ)GZPrXT6``4asd~P802NT8zH>+1E`;bXd&UE5x*U z>Dp9GnJa$eaAl z%}QKFm2m(^BMGG8APcbvyXEjZ@uEcEv->gUM}y@`+U_nnoOGcKeUh!<^E`Gd)S<2K z8L=Nt4h^3eXogTMJ+bEE7j_vl=MAkUU{)l&R;nC@X0D3Yr#}3CTviwky1JIQ20lR0 z!2Yi!EcpB1@A3LC4_rDKc}ACh!Q-sre$Je*RkG9=B{N1ZJ=<`u*@Z{m*BZC5>XrAC z=baw6w1zF85Zd^e|Fu_iC)XPV}^Oxs9NO=_Lve1%j%eIcVT@xz#dTXuQV!C zLo7-D+|}L65z1%svn%XU|8P1iqi1-7! z50l<4H)lL@J|Qg!`ilYf6SJ8eH>>}b_1;c>m>dl^q;;OTeDRR^4vS@HULGk2CVI2~X%cY`ExKf}4+SN#{cTh@)LX zsYK|gS&xq-y@Y0J+{V<0aKNypvIWr7>)ObA2x6&26MQ4M;Ou)qXcR%Outkj4mo@~i z5j^|ayB$n4LomY*yX+hyC=P6TF^JEUm^Y@tKpadmZP*~`5Jd^o0QzBX2sX;_93ShT zMxV;uB|RKPM&6tOG4p9DL~TJ6Ho$qH`Z+!sj9y%5dbdGEwBj(`DZ`l8vW?fKDh|W{ zb836=N*H-rAL2&_l02TWhCr#kns)Lz^1 zcdsw%p{a2IG}M3ml#%1t5U^%Xs8H*DSUilk%oE1m3@pt|f-G#WM!(MfSN$vLXSQ3KxRcI$v zseOURm3|46MBKt|#?yRL5%kC0R^%0ptJW(7o)z&Sv8WH=i9x=XP}m=NV)6PvHE5SX zfK{;L_kn_zD$Ij3P$IbwR$6h+Hx153arRyA<*vjjq=1L`kaVYn-FdH`ZdLvl#pB)C z^-UNQinJTpGt7;lA7N8?C1}M_V@t!g@Z|=Te-{O*4rUmymSj*|Oiw?FL;3S1C$)6? z_4CQ=UU{NLiJC}4E8@0J;JfX2E9ff9Ex)=B8jNz>%fDogEo;3}KAUi}&OkiL;&mef z4*VdMpWhA{l$a1G?TnWQ4-nW(V`%<}@|^q+$*x!MM9)yk9!2n9flpIyp4m&zlr-G5 z&K1)S*iUo{4C;Md{=KS%4wfv5g1|~DN>Rj}lcygJbEaHY~AHso+vscbA6t0iK)xhnBF3&ccaIin28@K$fmbAP15Gb}3V+8T-on|@Q zmy4S>lyoyNn1MuOa|D03M%?WOtxbK&hj<3MvPqn<4e6}3JB?66U?hAQSG+#6DDveI zV|Z=Ze9yw+)`b@u3-5!~+QLGzKqN4M0hS`qIL^bEM=dD&*?l>p^B2Qdh8n`-*veQ` z93kjUqg^nxl8Jsg6p+im2df{LLvxn9$S_3Nk1NY`aS%p%K_rW+r0_cfMT|!Uyk5E1 ze;bO_|LmsJlhM!n{NtvN<9p0GDa;^=+0K z%K`u5kgx5gh_WJs@)+EY$Kiz039m{9oEZx5g||gpEF?kY%FLwU`Tq$pRLNuUUlm%I zjPk*hlyWbOpQT<-o8`y3ol0CJkQqYeGkXD{7OurxC2!M*ghi(wDNz0N>1{29PWC#W zpVgavM@~#pd6ws84LIDrfBRdNT#(%yWy^LeN14s`w1RET1LMQI6kA8n{QDiJxO$w> zyKb~;e|$qo-jd!#n$Pz8Y?^3?#@1$5S=vKb;^Gf}ceX0)C6>Nm55q$OVP=R%3j)9* zX+;UXN9jB3igZx;D+sGxyW_}_0(jGf9@I145mFH|k3@Yy=PA{uCE^GQ%_Gd3YfzdL z_q?jAR6`aC&+M?s%BbM`oAe>dZJhG~xqxTw^H6`*faM4G(h>2T;Pz_~%2; z$@9j$G93x(+@LE$u{8+IWPvp}FYZT*n=FO{H`@8^-ya2A`JYAvMZCs9QLL?$GwXHO zs^(Ga2|dSW5Ce36QPy_E0WoD>WoO~giuWkz5Di;cDU_9PmJTuZ4sH_5g=(pnqB=V4 zQQBD=i?N^!==H?jBFqg$QdEnQMXWRyjxcM*v1rrh)}}isy@I$d6%W3S(q8%-Tg0Oczvo%F&pDG<)$@+?65CDAw~U0ACSSgDevp@a4y*e|$@z_5-HQLh zSoTTh*$;Abk4$TQ=>kSVJ)ZNnXA$NAlF5(YqQq*>ig}ei>qc*C(JJ)44HN z^B8$Om^^L=7lKsUf}TX^gB}VLq=eDBN;ol9RDjN zu`$oywVe-QNuE>$mBexDLidyk!&Zl%zcY2cy}Z&zX-Pfj-vV6F1jdxu;bC2+XWANf zKLHz?n3ct%^x-}0!8-V3=%-0k7%RzCPg)i3umForI+UPdTL}-;9%0^Hld@<8j98{$eZ`f zjha6h<1c1VM<`u7#8r~VCOh0=r^?`F%%hgMXkr;7fm7>bijxZC6qy*xIHf>{5&+xC zS^7y&#&|A`O%hzd{6nfXQ?qH0|Ggy8F$TyH;h(=;`XD~G-#+Cqf&^?7Q)jhXcA-tn zkvkOobHCmPMxU);CFISFb>QTArKt7sEM;nkGl?id(EQ&W3Rg_Nj7?z~OF+k%1+zVF z=$vo>D+dYkECpBXCLAWk4h;HJdY3nkxI`5O?b*Q(4$Z``M>YH*;7djwVGCP^`Eb;Qvq2<|Y!FnmsJ0E*O|Qkm=XB zNwEdjY+J$mm#-(se@NldH8}tGfs(o1NDsUM-_ku~l6~-%<>O8EmKJbS|?c(xvtW>kI4 zQt_y*lc@aZM_vG}mTdcwCuS|Q4U%#{>k6!jBd{R4gF){ww{4v^qk-*)LG@LW4cuK_ z64Dw!tBh>l*W=2e7!75K#gAYsEMFX_w=cZ0hfUSo<3<{fx=t;7&}sU+;B$({7Tz@Z z7yyd>)U*D>+`%KIPh=Pkp-5{-5#e-z zl#t;IZ;C3(M*J~bf$B(njR>iazgX3siTWYYyn$AlxWYM=q&{J9%&s?=#(-UAWzJ>d zF3b^;dpWGZdNtNQ*+1-Zj{8wi6S*r;K41`_AwC35Ufth^ig1S!|{DN54bSU z&f>2=!+OH_g!i8>siOAt5G6ZcVRpNIcYp_Bs`fvAx4X)Q(fNmY+&A0Z;a77eT?_oW zAJ9qk0z=qMH?It|(gJ9PfPiRQ#7fLhKMhkCJiJRPo>Zh=GFu5Nt#F8>V={iAOf%#D zvA?x9T>O?`v?TCX|JTeilPX4`M;#GMLmZZ5j;}UBC+oZ-oQpHY5dcj*0Dh#k96fsT zr9;+yab;W+L1VKK!W?Wv3%&Ux>}Z1R12vxxfdI&1&7 zK4hR*=(2GZc^Q8kDLq>;9#q2@7==%Nx4t@48_{z}yc>`ZsjJkI#@@p|O!2w9ZDqw_ z`JL5Bg_iqTg5N|O0z)|7_J21BXbprK|Ho_xN3IO1)Wg`l5sv{eBwu;_c*m(eP1l_d zvh^r)#U@NYLcfCEuV85U>;&X0656&e8c^d11KwaU0ia^%td{#Odz(XrnSJx2C-wg2KzASn;!KU$}*y!|gNia1gKz%y+ctIOY z{N%=LZF6me-UgcdC=NE6F`ASRl_vFqj`P#TukhGJyTF9u@VCXN<0k4LubgdxRxuHqxpv>j@J7=O?){khJB&|GwQmugtjfpH&y9$?+v(qgg*&ff5d=Y0TrG`KB z0e&Ns6vf}>$l?mQiZsl#nCGNHc2)Rgaufy~0D2U6FeRIHthxz+9T0r)c1Jln8P-G= zm3_`?Fk|w8z%vK<`?Q0_4k=4fD9G33&OhK^IwOs{9Nl%e`_kZ{Xx)M=;@j&C_Zf1y zOP|;G^}5lXCf^Ow zltn@Jr`$ zJYfldn)cAHC--cmh>|T_2JaL2)%sz|&niQgMFkmMBQe1SD0ST`>B2VOZ@&m3Gf3|T5~|EBoOM`v|s*h00$B@?i_6YY-3@u>T>;(oiEs+rhz z`%3P%=P~0;r)U3qdD_Vt%FH3Pc*^DpbJd6YqP7>-A4s|NCH7+BfFVDp20wZl>$GB3 zl54NWd>3oVl1IRuWRtQbgKzn_0%vCl>AaQo5qc|f1f>Hz(A>eDn`SRfH};2h4K7$aKXotYodZmJ@e+?%j;4Z7Eh=kE zE&tywj#$oviD42~p}Obn#zwjh0TyT1oD$2d!85XmIv1J+2h*(d_OTAU7AaiB1u!P{ zW&_$Pk-u=l^{>3po~)uQlBpTDM29%y3x+Y#7Vj^Pq0nDAhLkgC^tX>mkx>;3=m!Z@ zpE$YHb^yrVJ61(^WQCZz=aBtm{n5#CM<01dO4n{^Ti?DUmJf;ea4NfOBE}E(O5;IE zcS@KEmWfV-CjrC@6Nxd*+tNztL5UZ_Y?`CgAVQx(gbdoanf(9yUTRQh%QBTBJf-5& zyOhxxrahlgz&33%Ut=-5)w5izpqKy;Io(iA(* zL05zX$t|9}IBP=%lGr+8zL>F}R8QashSM+0=lF3uSv7EfC>`_A4#y*k&< zP8E5L`&LP6$Ohg|d)}Ib4mq(3)ZFl6IaM6Sa7clblqH(P9p3LYUgAt#xYwA0!WEa2 z2Yw!YlvZo@gQ(KwR>-9!0)D<;^Vte%kjJ6BwW1g|i4%7j{C*dh6fRcPoB8?ubGfCp zB%S>QAqygmip?l#(_x%UFhupDK4q9GNqClEefCEgW{Aj~&8Vb90G1@H4Fv3BW{twg zW28(tPqY|m?GbwY;Hvn?sccOqZiU8jiSlw-vzUYQI8?KiJ^!P+%p#OGaAFtybg zVY|tg8>!o6`!layYOEU$FL}Jwxl)w0fAo>P^bIsUZk)~q98hbV0qHiWbEd-F4p&A0 zn1m<+enq4t>XqYk(DO1QEL@#mq+@8wOwbr+GZ4B<9==9G!b?aP9N{jVKb6^W>*(_S zBh5?*Q@0{rZ0))1@_&qsj8bJvOUI{70uz|R179S~Fa5NRd9QFPz|76mf~YG#fq~I1 z%R*oX*R9hcEy?Pd%uRauPd)Ujv)6qoF6kXph%Ut1q_Br2z|1o?nSa?87TX} z{w^(D5|@yx-G4OKCw(!Pgjmm{9SaPo7DJkxNP=IL;Ah01@Apr)JsTcF!(aD=)cx5f^fl(B~&q^_QYB2G1DwWb}g zC#&QtoF0v$&dPS?AN3AIm2~BL`N3AZ=q=Vz`K-Y0-@uLJ441Pe3E}<`azpdeB8Pn! z=x}V=gmdZqk0b~x%&xepNWutfNRrKA+QI1Gr{!mG82`gXRy>Re`iz@D($ zQRWpzO~ZAM0+p9lPy4ccX4!y{{0Y%B0{4Hs_UZG&DwBWm3VHgft;WmP8*Xg}Li+Ke~k+K8(Yh?Oac>6 zNm2n1z@w13H7*h_jRqK;gMu)_&=->}p{Tp|P(CA-Ml$bT=CfDV*MdU&pN(?t>Pl(U zo8wGs#+KktH%qVgyiLMkR%g*9&SxMy;HphH@|%3R&ke=hyLab%EvCRY_*k=!swd;i zLzgaH3LCUq3PZ@Rv}1dWbg194@D{{T^5gmiS9PcfG6}5M0jKx5cK&Z5@9=&`=a$h9 zU<;xiOp9JGBBEe!N)t@4XzPTZ1HEfoaoL;6%of!?rs6a{+@rbr&8ZA;w7_8!H;OAq zBZa~&cCL(l^kHbQcg&e?r-$eorZ0Js+MZUfhrM(5hDIH9m%f;+6yM$B7)F5OSss|& zCBm6|MOxy5%o{QXV$_3|`iPhgmt>p~zT@X>{z8gzcsKNH&^@(xN?4OqYJr5t&P==8 zZ3oqN9e-?IX8z9>)i2nOt?IwaJiy}PvWXVXn_fjEX7#R%??T}G5uO4;Ssq>mS z*5o$oI{Fj~4;_xW%*UmO7QZcJ?Y$wAmB*@NTZfJ?Tf&A_-pToT;{bQkx%L${i+k_b zffapDkT^Q(Y+H&@7?{clHycc>w6%{K2nY;JP{Jf&lp}KNEIN9*O66$tWRfTurd$-6 z+{BEyE{DB?Tcl$XKN)Gv5s_0#mZE9UOn5CNuOCg56jPArlKjJgV-~d0UgKew$M+MF z^%m?#p|&~qiN2=>sZ*EIKIRillF3P~6(^SwJ573Q?P~rlfR&X{q!t2a&i&%+Y?Rg% zV4-4e6PMsJ80v|u1hXW zi~h8VS)FK{Dr$3%45xTc*Q>W~A(A;|O2ueJu_*3nMhgI*5(pNa9cTO5W5$!8kUkV} zFlX3TWg|`oAvzgqJqX!?c)mV9J@x6Lf%d04LBSwRZ$5i>u4K_k8?k~ELt1F|Mr)W2 zQB|dL@J+kZ&L`{oU3V&ghs$)2uA92_^1cF}xvaxyUZ%+K@(t09< zjK&4!gkM&dbBBaEp44{v3t3K(5M{W9UtQQN4p>yseTAq)AmUzfvVR?~kbprX?3fD+ z30$)N{-F*Nl`|{nCRP<{m5Vw~pgb+bz(1H$s)ESnalCnp7IT-qEL;LsD-rA4zoA{A zb2vmd0R6>52Ws`f+@^sXR5H^8Tf`?b94upO+e98Ke{&un>}u!wzieXsU4X$Qt8Nfj z5(Nc*6OvH21MbdvK}9ikw<;c7$`o5(DcaayxAo}<x zn}kJ2CvmLwCeXchj8@nFdV0?&0bpjV9lLM*p$kwwxBL5FuU#W-2Ew3O49D)qIkmv9 zx@F?xoSNp{n$fiP?Xh)Oob?t8!zXAh)?a+uFo#oI=fcH{gTm`IYIOUTAj>1im~(E{ zG-yX}lJRjbyvE@r{s4yWh;&3gVdK(8M5Uc*7r#|D79 z&42JuJz$~>tdmA<+VmJ6I;!HUxbNE!AFdpo8Y8uFAIPL$T$scFkN0bJ+p9_SDwao@ z(K;=NnbKqW?iw@Sde5;ul9^gqBERyNK6|zG`XsyY&u=WxU$kUNU(8B+ZtcAIq|Mdq z*DnC8cDl8BgKf>*w{JT?XXXlI*Jamuep1cB`|5LYxH!4wr**=;{5Oj2Z8x4*+tagA zJ-fmCZbi;NHl^eBe=o0CwQ3eMZ%Yh5Jpb8v(#y+hrmhCROl)jk>eY|$*W9ZHecJ_& zCzWdap~n22-Yx6$Ys1#9>&G0-&q^9m`6fq=8adL~xE@-fm(9(>&Ba=3#`VQbN2MGI z4ZRZCre({kZ({QgS(=;wf#mY~0{zY^KeC(PHS6`rb)S66pM1{Yp6 z>aeHT3Cj1+2xlvXo2gWP{Ph>f1+}RlHf-8-;nCpMo3Gf1r)z1s%`=Z1-010{9fRxf zz2s{^o3@kGudUj6ZJ(K|w>4j^->uDn{vD!k_jDaUYTfwY=%1roo_>1Jw0K1z-+eV) z)OB_THm9ae4mW#&lfqEG$z-OgSLieQKmxOUG9jf?&z|+SZ{H4$xY;b6Pgtb6eM%Z9 zc$DkcZ#Q(qojaD^30JQg9i4WfK5Mh_>@mH3K)kCHm*~R1i zrf1i#Q|BV{x@UTW%$=ONb7uZUch}^XBb9%HnMzP44Hl(D1AdJZ0ZKl0_$ z!6ufL>;3#%UY*^y$FzUyK7RaoAhuo8PVrYS+8z6inrCQslSV4T!|5k?!R{Nir&wpp zn04^)*cw$G3?@O1KJTD=!9jP~!20FzM5_U^0uWSBUAE6|#fpJ-Yp#o5e7Z)X%CFlu zlAbOS%0s)XV^P*|n_SK`EHPy+IBidKd$&C>v1h?X#<0)%9n4zU&TV|hj2fl8;nA5@ zEnt8auh3*RyaL`x9}2IHeLKh^3O<0(%ZMFcERJN-a2mqqW%~LA-L&P5tb&~#duavx zT14*3n=XHsCA@l|Su^#^8Fyi9d^Zd`nYyE9! zEV4E?uReU+xBemK-JZUA)5}$3-;QgujV4UE$bEWw3XTGpy=_ClJ=0#-eAu8+kqup8 z8n>N6@Wj2w7Xy>+FXH~Qm@27$q3BV;!bTpLTz8Ru@o{vqO{oq3qMSQfwq_BeC{Bf& zq^@3gS$Yc^egY}j!mmq8Q}cT5FIBcc5kRHVdmX79`B_7 zV6=jt8bw;pLZmm4d(W4Q+vj$IL%Z1R zOnm;*gq}Ct&a@lfsaO81NnmgsdyQ{!d$R>co15?aV``(@Hw+Kn{zX@}DIJg9&Dnk1 zYirjueBhTuIWsakD+ijD^QdxjEY&ft+WsMp>h*Lx^EeVJPp9i49!MER zjUC%z@L&xXl?|&`;Rp2A*z)X7YHI2NgWrD%I(oD&b#goyoa&d}y&KkCSLN8TV;qCr z2v06}{&xgke8{y<3hC?S$hAg;1B`0bsNVDx+m9M8TXr$g9+gIccM%YkH~jD!*VIM8 zuFq{d%Jl=~cM+YH&DquW4&KVjYJtvY)a=h|QKm&+%Wb?wrn7H6N)`0*p= z&!6vG*eb(%U$xyW8a1jC7diAzWA$Hsa`9f>jmnK|)VJ@Rx;g7&Hx!vXjSp|Na?7%C zGp!w?Do?(G)qh)AQM9%9x%N=Io9_rCqgr6$A%m9AC1I{s-MXWoY_^;*Y0_B8S$|Bu zJz0;*VSVp#c4+OG;I*dK)|-?C_AT&jNy%l#;bhDWc)RiTgkFONwVHUi31ZhpUd$Ei zciZ(d2*b`UF=spYT>BfE4ggh9RMf=HGtRDV1s#vKv!nMAmWtbMEFMq~ZeTw?p*ePN zbX;;IIxOrQz6rlW>a~OHmg=C}c+U8Y^4Wb`jUh0Ivg%=iYaM!~k5eY!w2yfT#Dg1e zP3#ps+2?gjQy!^{lMNdjUKkl0U#2RZJauZXl_4@yh-y}q26WJw|LJ7%8iH58OA+sbIku4*#vMNMJ5-KV}_TE$~DTVBntdQ*W zd%o$M&-efMdpPHDI&^d2_jSFl*Xy~iL}=Xa$xK=Ah&=IstKRUX@xY)UpVCr=q2b}` zW;E!GtgPEEXes^q91s|2yRZP)M04T??6Jp9!x>Iv2GX&k;Y_Gx#=r}jUtgjvDlVql zYAC?@&|W51uf-(eSj5{h?0afDx})ai!p~G6kdw^K%@4u2609tUPjEY~joMaLyTJ(F zeehrd39W=*xe-GyqEJH1`Tp)N)E;Gw`2J{Kb9L}GBDSAjQ(5VV9#Q`eNK5M^s0KX2 zwOTikZAgypVmA>{g*L&jQwocQP7B98>c|lwt|}K)RW}mq!O(`S#DgdBV7LCvEn%fu zkCh^h@d&w5{9LW80Z zQ5BV5%-1F@7xyYMP=pOr5ZeQQ1QAbSO?w_=a2Mp7)#$if_dge5q!azG{wcY7cz7H; zeq8ay2^z1Q^|ykR6Z4P_=_q}F<@|>aAM%{Wc(AyrNE$D{Vv*l0H#%f#DIy>!IF~E1 z_P*oa4Gc(K@dxr%P#tk%o1|rPS-nT#BOg9$Bz!vrf!94AL3dXII1><3!!nbMylRzT zj+?+_ab_Kh*o;STc>4M^6KIaZ7cRuAv_r3-0T!hs3@s2Nqh{C!Z!h$R@}sBInoLoh zIFL34$`3ueTf6^B293+yi%l-fjC9)_&TZHCCe6{N>>$oFd^9#iXl*~V_TLm)r>GU= zj|DM$Sm=xF>veS4Me#gbU}*bZ60=WsG*`U;xgBHS^4nFTNiiS%(DP0o_pBF`kT~;t z;aAA^Aa(T70U2;16(L`i$?WKnmRlQ`uPraSNJvQqiSwMU)HXF0#B}Hda4JkD@`Nug zKr(`{f`|45!*OFM*wB1P!@j1sh|;F9Gcp?TAf*RWu>k?pA{Po8N5yqT*hX4&HzAja zzVCO*oL&$r*L>)?m(c3$x^wTIH$Zb2ednge##`|9a}uKl2A}SMf$f^ll-3y*`m_Z zLJ5k;h$b-Z!k=H4o_!eVf=N;dZ}w zRU0vYfxxLnV^iG>BtuhAFQ}~S5RR|-p||2we+W#*4cCoyQ_xnv$FzJ6&}vprj%y=8 zGGhqHWy3x_Mx2}ko<43leYe{KUl)#iXPhzn89z%?L&FQtH$XhGApoupQ?R*fJ>iAG z(9jx4-vXe%?g9$O!~9L|#w-K2v`*eNeqiV{xMhP_ftBc5h-wahd??sBO!J+IAHvta zMxU2~nkEsD`Zt)fd5c87ash;w{6FC_7YCR5T7vMwyY=u{%VW#7pOZ)D-v6VQFci zNCvpw)zu|GnD$L(Hf`H>0nSj1RJGZwELus5Ya4y*&_ZPTS~nOle<;r&aQAVKJiV*c zbD!CwxxE{AFcvcuYx&LHHQ?Y)9&8G{iSv%`<=5MIXQo=+?S&*H?90X`U7#IgMTzkK zV7Bg)qj3WaRT*gG0A6m&!zSpLd+Sl5jZpROS+ zEf!3f1n#TcI+wWB&3eM7WR4Qc8nZ^EP=>q!+CxNQ!mXQFQeqHnSmew|w1>xz-Pv;h zTWAQ*hB5-{d-yOI7HB@$`BRtQcJ@GmP0VeKJUpxGaJ#JR7G%vIg}e;3v@d4+FiEhB z099)MnB2X8pE5*Q@c+!hjlTZ=rl`|-*9Ss53SB0Juz))*GsiNb{1{8dv=d{=rIzk% z$6_6GS!mY(ah0=|;UC-gZ=}@)1sJ zbW}*0+gby>Ror87R&8d(#*J0*u7n|?iV0+kNt;Es36>}B^;m~PyFJZ?xuGkeX}J9z zMvv@lR+(6}1Q>Yp^+N$JC@CwOU_8T+0#1+^QqJk=wI&i`bj|bU12Huq++lH%fs2dQ zr<~+d?$y<0v}|+a->R>)H6Gwr55^AbjK`1pQgkxB<2sp&z2K+MwL=n`;psnG=9ZRK zfFc?`ed=Nh1IhxVZ9}&1ZcQ^hNaJQK4v<(~m<)Lw(3Lgk*$Uopci;9Vcy$B#`GE50 z$)&^a;O;I5>4u&xdOW-vqhD4|S?I;Of5WgcrJwqB!z=s0E7T|OKL9a61_Qz!)Wl;7 z`}f?Bv3>c1)w}UHGcCWISOH`&k z|G~XlyRnn;M3S9wjGhS{80JVVG9m{{ms;X&qp8J7HkK=xu5WwbuPvN5Vq0h2gF z)PUm3?lQPuY$t!5dXJ0Y!i7$GgBlRlTRS<>-@hOx8eCL_>P@oh6hzh;C#(tgq%+Hr z;tD*a%u>=R&*98WUq1JHoK{$$tvE_$P1I{4&ts2^S0rxo!0c;oWyNZ$UK7B?IF^V_crYu$2V>Q40L0VDwqS(}qfvB@si7;8ctp0Kr%47Hz!l{-UjlB()>T!elljFC|qj8Se?xG^v#tjW+7|P4x9li-sK&~^O zAHo1olFNWnco-4N1lEhjQf>%2w8V5tjC6nDo9>E1dq>@*2J<`7(E%70n@1hDD0sA4 z$=g33<;?sLBL#$D=88e~w~vzq+;+P4-K6N+2}QT(f1}J$=(!!W_snfnH-7FZ+`Erw zfAbTyz+@HW`RC$NT46?`WgZm7V2gGYGbG;B%ZGmL6A`g|vs-d6Zl=~RbUd}CZ@^y0 zcJ%vx;x?0>_vXn%KcfrK`m>1|h}fdv(%-`ll+hMPU|4xE=?>}}bW@5izn-|mtvU7< z_?dnltoQ3MQ?T~#iK0aid^uWuVzw~$69wGHiEHmGFd$dy{6(zMTOEQ5F`66C#z6)6 zCun94aT5xnAmZ35{7UQXMV#kA+I2Ea)&ter`Yyt}3)2biKVrlbMGPkRm5ej{(> zV{9Q{#S@`}u(k#wL%+5G`+rNyR&E6a1+?Vra6mAb0ufTe@Jj5Y_bDsLwk1Hs4<;G* zx;d?*T1oA@^Y!VA4MeSFVqew0Hc0c=P5m)&TOtMc6al=$Lu4YdZyz&}x}Tw@w6v4x zB=rpp=wm)mX6v^1dtiryCohMFo|wkO_fV5V5J8dI5)4K4qsjmSa$pTKz7Wtjn z?W6f||68oC_2?*J62TU72oeRP+1=gEbm^$X4fvq)B7MBQx4ye&ZiFX{hLZd1*RR$S z0Fy1-3swvGdpu6^KF}V`g-)V4E%o&At{~evTHalzR(BP;X`lsaRE?`$s>YrmtMN`m z#4W5@kobq9A^bqu&2^@6v%5t>>cxmAcduGUdhBtvS| zEhz86hu{~Via>?Mvn60ySbE-=?k$|3cO7ZVXU4L0?vK9lq2Os}^E>2G9)Vxs0!AT`_@7cSTY3~Eo#Xaf_6cP9D zPn+vM(t2KncPbPDjKYwklwJVA9zT7$9>+@mf^37>OrNJ0zyfi^?QDH-?u^GLb|}sU zvd5#IbzpPwx|Uzu3k=*<#IBVQ2NDKL7B)6=^94DyiRcIeVxpq7J>^;cJtP1}SKxIt z+WVl*1T9rHR32O31wfcG{m(C)_JMo(hdd$1C1yhO3++89-pvILyKq)05ip~s3Wee! zw6;iu>D44iE>H@gXlwA1AkWOga!X;kR{_}xo5|O$BOvQ<7|QXS2=s$z0RbXBMEB1^ zbF{SYtEJP_$x^fn`yY9(HJ|3`nkQmOYwJ?e=BUf_@gLL*SirU=S?eR&lF*2fkq$f1 zybt?xxG2m*Lv?X?R2>mLH1a*07P^`xSHt-D_@(cjo)7k4z28dl3nM4!iz0hNdPzk&S*}LY8rA7f8?X@oPv72+ROt!V{(kjGt(i zF^TkxA0PetQX73r22}GNF;DIXP*TOJ`D0aP5XkuZAN>hub}}+E3x+;rqaXvEKs21w ze`fnWQhR%W^29nPdP&lEAz`UCUN9uY7vShh=PG@&x-N%&VQPh*tFU@;X=wu5{NI!3 z66Cu;4;dPcvC-1d+!NZI;DO-M351vygp85CcW8G&+XZU!U{^y5QN&D>sp6^!dXA79kHR?63?9RU3hSw;Owpt{ zk9XbVD8%`T=SlhW9&fDX7=~?Q9I<(xDoS8BYkBtq1zTXu7+~|Fm>4A4sJXm8WfEH?qLlUN6>cNd_Z8Czfi~}kB zv>`6GGO<|a_{4(m8GU6U8Ivl?qg4uo98@@v(4J|_xt3Fj4I2w4e3dF6tU;m3XyUIS z;BPekll!g<3~3X5IIYeD)n^8yqUg+UQ53{^`LUHGzFoU$%`!IwKa**+aER+WspJ!e zU9<~Vs-Zdp_64jCoXi+1UrbnZbaYto!aH{oGe0U#*9clCg%uzBw zvnx+(LSHtX5M4@_~9T+vOXr z`*oL>br=66g;U8=;d~(InA9u3s}ha|-MN4NTUygyw0UM!O{QoO?s_I}oRj)4@7XY) zEWVI>F>oKJ{y*G%sJ+YP4*y-qpSo`6tyLq5Cpi0p=^BmBcyZ2Hq~_@wD~5dC;h_8H z9`)y)VuEM+s1hU1rnYYUcQ3|xd_Vh$PpjC z;xB_$uq$6%n6|?Au>;uZAN~*Qk$IvaoDTW*NJp_=@YklMUQCA3`>(Qj<XnV9g$uYrEKMN$<@SI$>CJ}^K+|vBgh`bd4R!r*p@j0@ zmZqF)le@yquEYs}T8yfD_uf5n60xrcxZvEmAC4WrRN*v*#raNnI4){3msrE~y#Vb) zEm!M|(dzF98FXXK7;-_a>D>TN91t8FA-pT08yD(=H zFOQo&7n zgJPf6U=>Fz-3iZdAzJPgv%d`$L{q;&KwNoaQKB` z7xV%fFew7CgGsEGz{T&}Vc4+mhB*3cWn*K0NWZFe0AB4eEFs344f}AGCJcb26~pcc zMnD$Slx29B_Mr=V*x`B(K%+p^Vqp2JwHT!zvb-=5SY23-dCn68x_Wx;-vI>^i=U8^ zz~pfY+^5H{qko%^KldP2TkA2bW5B_pv9EGOOj!*GnuhP+X%NzDTqBp%G#vkB5bLjc zwB`4gm~$`t2QDd9Mi`{kVkEq_vg|H?|G|SmV(CV+GSWE_MHo>6!9dt4)~}x`NW*nG z=%)!#A-#3fq$n?s1HRPFIhHNthK8N&A0Grzvg0&A_sjeyH>WIMHiE!{Fm4^mQj!bS z>b0p2<6GOu?$`|7RIGji<0D}@;zRGfM#^I$cRSGieOjRMw21uDwjbkUUbw)?7`?&g ztx6KAaC_uM#tj^_PAGp5i4|(JV~js?!Ar)lvy_dD9xuREOXd9B? zq$5-n@G+c-AO>qj#$wBVVq%DM9CrldRf8i1@x^Cw^3#}$v}}_GCbOGgES(h7`gH#S za1zqOPu0BE%5AbJQDc8M89j;PJo>>=e@E`)vVpDwA60xe8_QI=zVz$!-U{^t zMw@D6Q1}%T6vz)tNl0YX&)++NNH<_@Rdx_v@yOqhHu9M_pFMu^0*2__9SV(ZlvKzJcwLHM?9yL9Z)SfDF-w=R$!WpGsVOWyQrv8(~+H__oXM%S`I z0!s+)&LHA$XLD<7wq6uNg7#BY4ZhWar`<3K4QEAx!>|}JqaCoT>fIN*#-ngUqM`Hp z;)d0cf{9K9QYXN@{QLK_!Z}ri)zod5Z+J}C&@c>VRX18k;+qMu;NAVTR|SmMy}Ewy zYLlA;5jvdw(IR^~*+!88r^%=Ku*X=Y|CP@si|jjf2cBOOJ0smc&Vf%PWKTB=7=6uQPk5mOnkw$`x!%Lb76L`2XID7FC$U6{)FU;R9u z`uH&kfaSFyWV3c>HyQNa5L0qbA=rS$qE1_8I%VKFI5I0S3%B>+*&)nZ65Z6+cLbpN z2y*{xh*!hG<=RLzfuISt>^r`?yw^R;$zg-`cRYIX47fjhB_Jq>D_#Ge(O18?8cq(2 zh868`ppiH>(fsq^2S@h94qBK0BCd_fI~>UWZjmKL;CLTZLZo^3ztbJb(=smarR1V` z&YZ}vLdo}5J>-G0jFK~W=K6TUu8ytk=_p1 z{B-@es;UxjI`l5pIv6Oikw2JPFuZyDwj5nL3d)csJ2!U-F_rQ09l})hB>v3K45aH4 zJt~e6re5>~v$)!YSdZnMorajpymE7;h}{RtMbKYRcJZf*g11_2!|EB2`-}xV7JGSn zK~d$|_0#C@nW6oG&UoTygCOWt7H~0(hQS7$*iMK8uSAz>i|Zo$`ud!+@TGE8elchw zfX@YlkNx}h(Fx$q!Rbw?GuCDOTysA?kV4bGrT2BcQ*=u;}zxxhA3R( z!OP2=E1n6q0VBu*LY#C>DEuu0p6e>L7XB8Dv5>4&97>}z?jA`;<&cn+ygdO@lF%Qn zLL{uLQ-=uxU}B{C98yCxg&Ns$bb#kX;~t0*_QMPmc=^lBC2qH7hDSO*OGqFGAA}|W ziS!E#l}2I!2KXc;*)ZI15&XBnD=AefU%$Q&aQ7Hn_}hy*r5GMY-7e8zn4;Te)XczgTO~r4dvVR8XS@1C?zIP zaXkzGNN7$CDoCfuS)4d~3B+d!=1YtkxOIUJD3uQYT>^qgPz|IsFf!u9LE!~ycNb1% zu?MO=R)jzeGZsNoL?Z(Ntl!o*5r0df`7EprIGz830D3ADP?~}iWlTLsTWm0vGiWap z)*POvmelLKQQ+P!3>}0p0m5H#*dctuDjbRDFzEFOG#|(52+z`{s^6KNmlVR#b`&9( zjDUlHpI1ZZS&n4sPymnu;^Gs&WBvh8aOAhYbGL2ZM8|_y3pJ9Op1ua%Iw@=StBa5c zL&JzgjnjZ<>MV*nfI6r_$;=NruFgjO{*yKaYb_Zu7u};!#JfcGsjY2<%`nTH(Zs}L z>$}_L&M*Im^=o#5`-EcWFhT|O^q*>V!_s*eGPu8s5M?|Ai!f|u9hnhODxGvAvH{vv zozf7MHy<$4y?uRW5(~b2cNuZ_^el?m7)YFX!Oj~5oY=bkMOm@i{^S&iBMe^YVsb+2 z^p(vS$ryW@>R8N_|$~g2YqH_B&sv{?;$rB%qNnd$>_2@*gMI)P3e0B@{gN+)45#zL9 z1%+$ajRTkO?2?YFHu|E`LT-AFmGhlmm)(Z(4d3{U?W7kgp2RGCYco1po|<#rulOrj zCxcv%ke{O){qUemo*&~7RWol=af_i!q4sK)0xL_Spj?Q=y<&2^kyRvIr^@qbN5n)y^8ZTYs{vv92_ z(6RoHd*Ku=u~`2@*_D5{ewYfKWZR3Jvc{1+GP^l1niNGwv!qmlDr6CrK4u{X=(-%@ zrg6k+1o^Z+(k9BorERIWY^58VG5*9rNVd_+M-sjQ-MDRaYOs|D`VF z$rB1f2uDbekhsK)1Bc3|oKMkA$;V$gv0Trl*cayykuVYrpMvqf+%7YCkZUQbK zW&)52RZtp=f&Up~BKmiEMm9D-#2s!TP)k&CS@~fIZ^~>uCb@{Q5|Lv)6$k%l5n2nx zk@^$L6I|R!h^Jn%|0k_Qh7mFp%nm)(Iv%lmwE;yh)&=G0C-mr*2H}@LNZcz9^bOh$+_?&i?8Au-MH;0bNmmc8>jP>y1p;gf+e1O7n!q8&0qHMoI~C(` z^`z@@me-RPZI0L1*Va-2u!lsRl7!KtVim}Sm^%S4G2GIv9`DWe5k(Krlw_`_BMuS^ z9Xvj9Hfp?F@a(z%U*Sk#M?l54zJYQzXG#_Tum+LLBb$GNDhQo53hu7c&prIt3~%zT(tO| zOuM0>d%MDe0NiZ5SwpxlAZS7E_H9lwlIKG1zKJ@+XAUh+Sn(^fVNDYgwda$Q#hjXj zcI+UP&VLoL>}Z>Fc(k`g!L15^#w`Uc4pSkd2c#h*uK3tMn!rl zMpfdRi~N9xvFm&H%d%!xL+uXPyB$*B$eayu8@R1upTso%?PHo1b>L)WkyEYl=a+(f z;ZrLz3RC^NrT?s4sPq|Ay>?)Iu1x-6zb6kgb(W&OFmm)yw$0d%6)|2`DDnBcu%CgV z-Zk4Ga7WrU;|Z1LpXTguj2eAx5g3`BylygAsZiuS`?kCIU3bum-8qrcmEoty{$U*a z_JddZm!kW5T9Q$NKXdwQVqp99aQZ=8_fA3m3-OMSKv)gQ=7d}eQ`no!_ zT)v*V{<4P?)q~o}hVK`L(@*vNJS4JEA1-cZb|&`#`Jur|&WDShxf8P|0&Bw1;B`nq)*Pz$E5}&kHp)@Q9UU%?M)N1nuNAB1$;KC3g>~pr%uNeHTA~( z65XE^E>tGgaj$gLybmgW=_yV}cDH_Ie&pSlr}M`3Qix>^H#k)_T>6jC{ep8_<_Ct~ z95_ww|Diuay&~dT%JSUCTia%xnlD>j*j2>ju_-g(Dz-3g$#N$Wj0dKqll&-m2?%(s z3wnBaL27@?xaf)BD=azo3oC1orp zZTmwtcC|+$RZWHpRd2(O3hy3WbQzyM*X^@OYkV?J{fLfE)4Nj7 z#;RVFrsNXIucfaQ7O+Ci+pHhEN$7F+sohb=`j080o1|_imgSAYIMD_7Y84@F&uxt> zMC9Zh`S&Me?DDsOBB}!7QQeFY=iljUO|u5Oz4)ph3iNfe2N1N^H3y~uI(HRzlyck}T@_k+Q7ytQ+g z&FuODX%2uLvRnZWybdY`cxo;T8x$duB)|pK0K#jwW^t&0R1HvtASmoUi@81rGm273 z37t_DP)%Dnw{lB^$jYEF$7zCYy9{#%aJ7n~&(;X80P|)h3bNH<;r!y<$h75+WD)AL z26MnA?UH~s7~Kk9CDz{rD!4q_xl9Kd&O`*8|H0@WC z0$~C`|Llhc8adCp=Yp0ZL&37gyY6eSV_4(u_OEp;(^m)SZ@P9oT3cE`pNFmHNpJFL*lDZCKl% zS6yD2V@dhIotDvSN8{}LoMYwg*}&A5%bIBmx&MsL4h^u6k89aRjar&f?eTQrZAl6m zU$(g~AW{xFWb%QAPjgp{u9z3n_f0p*R;mA*li8LVmSuS4C)W#=o~!_#vDOFgyj?9< z1~)fG|5ISqdd8}}Kkb*G-5EZiG_|3u_9a@Ok>#|`^6#Ig3RIqMvY2S$jXnHb&opDG zzMzEuHy(6ksrBTw1BHes-<#QXbsP{>t`Ajq{@gYDXqUAKM<2~*eKj&~QXgQ{yrUW1 zIaXFV@qbDdoE%*?Uy5Fs&Qn{cuM@srDC%o>?LBXdTPlm;@`$W%jAd+2zGwNZdv5z@ zyLk;1)5~)^ufKScR(sjgo{8>4krV;TdKS4b#4HcUZWSG0lzU$E)wwrBvpFYqbm{uS zwA-w|XY{&Do8o3fON*$2s`>9L%r~&8@hoR#+{aWM8AMX7fdrnMzvoF4Boh=vJCdI`966t<-}Dgtp9*I3Jm2{ z(ID0E5%(tzuMo&X#QI2q<8^!qP>TZ>rip=i-1F&}gp%Arl+>}u1yYs*C&JpZU%c5R!Zho9120KKul068!8wwb)(#cR6dV5;Ue>7_O+hr?UIhJc;LEg5 z>6|h~v@_OLLTVeUUth8f2%Na7oTzl^>E;IyBBkBF1q@l7RB4~y5tBKY&q%-AyZ>UW zFhd>9(=CYuHGF&avv7*CTAcjHcE+CHA~+<^?FB>pghzd(k9sbJ2N)oHbkQ_A#A(8`=CA!7 zr#6^gf4fL#WT}5!hWZ12zp5*Y0VnBvZ31}gd^e_kaBO6eivQz5r`mGXdPaf6y!dOt z7Uo+$BRLK{zxTcqv(~JwH7QS}30tJK(JmXjrWilsygSu9roU(O){gG_NK+&GR+lfI z&3&I{j7Hztb!mLl%&+uf8}p{lR&7?vu1^Yv{uu0wODW&dZlqSHxpm^@p^)broI?8( zON!PXY90PDT$&#cCa*V9QR|>RlBzbc7T)`#rm}!otc>K}6zASmZ8dF>8G>a&ifCi`Tt~yVox`5!K(?*1mKf-HvY8ZTHqma4 z4o+1ZBxFYskrIs-`yt2hLnLiL2CTD$d}meSt$gG@0tk>BN%$AUYfLSSE?&M&m`-6k z;c>t$tCMTZO-?Su<9t+!fUpV0KGHa*WnFj)LBphGfT2_{8OelibYb#Olc+6FG`(aZLClKs&Flc@u2 z4NgLDV3plqI(V!Q5=KDH@KqC5Ur^@5k5^S)cy$s=wo#LVAA|;o_yj4^8e1*Gb~3G2 z_PAnV1I*a~4}w}V`!5f((D*(%ppMW3s5BACz-~$qgt)L^TYX4CaxMX=pq+0XwPL16 z5h5ZJU|D9mtgKXd76N#p;M;*QjhjvbWsYhKqWj{-i+Nv(Ya5Aj3QZPY)Z{-K65^jF z`Z}OZzH#G`O@bw-2e>F`niv3|5cxs?znhvW0e>|AC{-|gC^2c?X0^arcqh1uEU34aBV>Y9*5x$n5B zxxOZD@$LB=ixU?}WyGxZ;_Qazg8eP27p#9#k-Q9-_i_VeU?m+o6Oqd?Mhn0b=`E|K zuH`gssUKEb_wQ};l!mCPlPe-=w z>Qi|oE1F?rye)rr)J7ZNZ#N1+4h^0vl`<~jqICJ{W^;gt|EAs*)sq|TmdKoj? zgR@<^b+Yphv-UDJO=m4V(v2B)9={-7BN{j?8CAHQwmUul`@;E1Oq30GznO74F890S z4%iHRzpJynJv@GL)VcWP{A)A)4hLR~*cX-a?-pH0R>q>b?Q*yi<;!J?COQXaEd0VX9UFWWj|MC=3{L|c}#;bSo z!Y*r`_r6Y#Uwx6(I`)wBCW`pg8^8Cxld+2OOw=#ht^KCUjwby9xA4d3$|Dr@`{Q^1 z9&R{eb)__J$D?{<^~LJ)DU#Qh(&CVPr8Y59-=vg8YuttlM-MRQlWRq7@5+v=A9rpS zSu`RqL6LU0&maA+_oaQOY=ORaLuO*!@UpAGf+{0L6iL*Ru48_{Hz2N0KKEBon3k=r z8-3`~FdZKJS4Bzb$=bLP1G>m(?)grIC(D92-#vDGgW|WDfdX~;{Gk0;?-hLLo*t!X zE4i>^wGFRyR5lgNu{;2te*@{xTQ36o0v`74NYmS^vn!Et0(*NeF*2m zHF8Nkx002C6_GM5*sZz44agiTdpLGQE+i0GwxI+mrbyDoMG+2 zb6{w`vkRrA-Yd(UY2>7xyFSde)bHt#K zEnxwv3!)Eng&>Qapk%=h{%~Zp!T3$cMUlHjYmr?L#6y{ctI5E{0p1J*Gx4h%vixY| z!QWcd(;7n4ODLBAnm6N%irP|3^7A+28;>GO0xoSbPfyP)Oxx}S5MXo4c^`x+^b!I( zVc~J}a0GoIDCWttXC_EafX6mhc+KTUArCK!jH9}Gp3Lb0sH)pm9-+Kn3{gqaiAuh0Fy6gI(3Lhqmp*6y4) z!I=L%k9&BlvHlQ%)z$;cH_~QaNKX^mT}TURapTiA+0Ex(X}yJ`!S%^AKgZ_j4^p*d zCbqbnU9GHt$U|hl-qQo7elwDnZfaSE>G0kAY+sG_iVb`f7L7N*c)5G_M~ptHtaI^{ zOo!`Lk-pzI3p>6*LxONY0zV?d(=8kPGCG;+?>h)w-K?s5&f&Yi){fe%X(46!=Vz0? z)yGgzDwUOfKWDwCXS<3$4ZS~6c!oKmmnMp5G*@a~$r$RT)31ZQk3ZXc(kUK#vZ-Ss zY};&DE`5AWBLyk%^{_mu@1NNc*Pi9|%ryIURsZhORxzrMQyWdUa@HcPe)o*jrV2Ah z`{p0#=cMXvCpBK_JesI6FPfg*9GjzVzz`I&LK>OP%&C7i`6`NkMt6s@YMA}i%f~hx zy;IJkVt(x6C#wq4Y{RCWpd!R4#vXzvuhCM<_#ftlieRKdkUk zc&9=|;gM=97`BFe^m&>D%g&iFR!->jOKOdK>~B6@yl)tJhn<%b_ne{WD@7mgD{)Ty;<7x0S-=$^k7balG1}zFSu2tG_0Z z@bvXNSE#SVw3ZsW+VL!pSF$Yj5lC}wDC^^_tj%}tc3k)|K2hsb-AYELi@5#Qm5!bh z?CcoIcHZ|~8m>m3K;xbTo_#{pyq-lVb&^Z{PNjuDZEda@i@BZ$X%&_R`{-Lg#&Y=} z$P$`er>UP~G^BvrziWJ`#*W1QbiKfzt$VX{p|B-gH+DA3^Bsva(y_#VpwLN1rk*+1 zzon91dYEdB2GV5r&N}fv=BM#hQIh||M zxtb!h>JP6}x3v`~aL$o&NU#SY)f1O*I zoiJU*tv@xmWn9dC-u_Mh1w-dU5^ZSB2vrK9ghcIx1#+-K2nHft;ud}2x)8ExU?}?^ zlm0}2njGR$h#3iVBiBwP52_D#i~&e&ZPSLaiOKEw6?fFrxyYx+_}tX z3IWd{gd=3>rV@pHssnu&Q8n*F)~bOF9nee=TO}9|w_HC7eKg2)MjSv%GjN58IsMWH zu+WV@+1k^mv(RZ1AY!Qc`)1sfjs)oM@jv&{fveYouz&>k35be{2J^^qB6@(51TPts z!Dw{}(hR90rW?aPAMYXmH{t-VhbSR~KyU6C1mgr@ z{4VMUHz)n=b17@d{gjGrm*yP^54d0n?6xpo8LZXLD^P$y>O~kR<%XbS%0M6={HBiS z+gDfFR3vVqLyKGu8feq$JO6y2k`YvxEq7l0<>Ykdu4c!j!r#ByAmMS(SWNPi*?i@S zX8VsnV=E=p{+kAX-oRDJs1o)Au81n@nPFOZjxb^=T3f&3sC%PPZf#@uE>KOLMaqS3Y4|D!a{ z+r9Jz6-l<_98J_oFZ~XuVTuc#E60jbC$x;`)RXL6K@2RqHClUGg}_66iF zRTiiJu7BoOsQ%kqC-@<&uS!(w57WGy8GdEenMGcwo_9(Qmp@pM$`b3gnLoekt9@ty z1HpZX_@8PEMUR=aOPa4%emAL|a^F;KSCxE7WKKQYAi4hD!u~g<4WLT<71R#{CarU4 z*?GWVsl|TrN30;ptK$8Mh=FhXHrs5ESG8{bT8~kry8ISAV}0WH=ZkS>LQrZW+61UqB)wryzNgs7V(|7hjKg9;Q6s$dWuzd*oU($)C=< zGBoYi{W1ntvTS|Iki(S4TJ$zwXJWMIW3|5ZA7XdS$GxV^7<{kgf2Z_>tcJ)-b z(wW`3W1W~8r2$v^2gMLRC`8J4RW&Kq??6`6-{#Tp{(Uxa1K7og=nm}vEq1iyPy`wy zhYkMKC=Li433k|HwvyLQoDEgb_}HE(Q#d+sdGkil+B^6BGkL4sx)%9Twllvj05QBB ze*e=+^k)3X!wU=yeDnT2!Osxlnr}65-XVoB&taGi9_TPc#M^pn|9zSHkF6%Wt;W0k zpwGQLjBEF#e;A8Fz4SL}W->sIo7bY#fnEYbFc4y_3M-TMmXd;Ua@_El-rFM%IWpol zeBiHa{FbAh=L~H;T#CmC)vtC*NeNojqo$^b9Qz)qQo~OewGe}(KX~SGWNb?MvZDrs zJ)zR*g|^2V2DZ{RDaI)|@L5nbt=W$J{MJFO3S-^Yty!l`h$J~!Knc!D?!ZU*ubY+DoeXj?QxSZ`sp^VrKnQ#3ov5e*&C6mOL{8A;iURTCC6HZPbdJ}Xqf+4L zOkDC9_nCO}Mi>dr9mSiBGqW$$Z;h%ON8QU;aaU1UFq$y&iEe#<`lwF!&J2}>5?lc0 z4YdSO$ai~;87vg76I7uz8vXvnqyA}v3Z*mR2kz6OMZ-|B6{Jjv|G+-L2huc zon@PJf#EMArY9ian8icI$kqP}_<3OV1#2`%N@bN1x1}1{`csm9Qs0d%a}qzw%(9zo z-b22hrpkHO-L99)MPZy4M{5o9PLIJR6AwoKJk&$4W5K9iyLPR#znh6d;=6uwwv1-$ zkW%W`K%qifD0y{9_DCGm|7o_R>AJ=6nIBxYlERXAlFpt@<^DxRvfJF18c}(2n}6;J zdfvz_H#ygnXVXwvu#BIr;~%@IK~3ig84rhohb%OSf;C9Bf>8+8>fOd>M+}kJfcT;= z$?_>?i)>EV!<9~)s9l#DkUZe+454F!4YpphDO4(E56PzZ{r4gC)YA(iXbGxgkIBqgrZBVtqmtM;u}{`q_t z+&t{OFvt1h$Md1!I2N|2o`R_zcT9PsIVC)nME28)hY+zP1qAK|@KGa37K%_F#KjC| zEsmJ%@wt2AsW-gk_yp%{<8;P@3Dr$2Ne)O`jR_z<=h^Z2H6I=F*5F5qU>ob^%i7+>O z|5kW#ild9^u!?RIHjFz&+a{O#|IgL>%`fH2PkgaKBl521o{s6sFGf98={%!S zo*t^XR(lLY7;V%aZShMNOXC^X@9s(w$}37i(l0#st#|1W5p(4@(tH!S=(%|xo++J; z)E7y+5F?RXQ{F106Hy(}zRcmhv4lGK_n@Ys;N_#;^Cc?o_glZ*@NQ4@zmg(>BG##2 ze{MRRO-*G%cC+ARAJ)=)HYmU=5I(*OBPGh4TklRnQ`5rrm zL9zl#*!e~Mlg)QElKU=XIkNAQ=Q!(Y)yb^jE~3DAK!zH=xQ}|BR&{=c$?QUB@MGcZ^cv8#jrvR(1^Cf>3kAX@M%qOPj75UrCqG zR;Qq(t(?8XlRZiIH7IvInYmR?`L08)q6Y?Vd=1##lv$+w@pc8qOW@#Y+0NGP^RkQG z{yU(jwIa2) z&t$!VlUuu-^Eio+qeIQZ7d3N(G;TyV?Knb|VLHB@QRp-#068{sZi^8rz9Ds>PK4f2 zRh392C2~}9U9oSFxzSeY$%~Gx3??OnTpW`@Z9~39C6w1h;0DnKcy}!uUjU5&5wI`( z-Gp6$2pqh0a%}@j$WnVF5sosmWn)k^Iia(3U~$T)T$z;s6aAk6`-{WQ$P+_ZgFA?f z3`}rU^#NPxsW89#!?$=W1`;39Q_*5%2v}mRzf1%!{Ax#z-;1n3#)fnj%)dm5(bv~+ zJ33s#2uhiVR|OI7g`|RBsPqYLKjTsz+b!QIOLAj@d!dADRbk0hXlT>_gxPpXWLc+* ztR*sNwmD)OkQtHY&BWUv!L0ejThdG-$1b%d1t{a6x&6bXh*ry%Ga zkUwMi&#eFhB3s4!lJ8+MRLH>B{6n$$4GE5(X9Acf5#g{dj?nAoyX;kn|6q1KsIK1* zdW^$c64x@J*CjrFtXL#xrfL*6a}vRVC${+)hfhh7x<1D4oN%fr5mt!YJUHQJCr_xHmZ@-&^-rK$v-^VNN~Ghb?+ZS=aMQ$Kbo*y*Llg+`Bu zYPt`ad-Y{bKQI*Spbwn%(Vgj;bcs-ydgv=OJ7dwA6qo6DqJH?BwXrDJZmVh_Lu;Sh*N!?vpLE9Mr3W5qvlPPxHWY^voeT%wnc0@TjJqGP z;pB)*ZT%zOIQcPIcb-S$f=oMh@V|Ly;kL3c+gqrUUe?uMXl>2CN8h~k)vUjQ>foP} zvG%Jg9ggoJ#(4%rBB?gDm0sqN+LKqi5X!|Kd@|~UtAl4t%h}oaVgEeaMDm%jrpK>7 z-Ui*O|LPmUv4LhX_9MCp356*^+$7?lBweq}MANE&VfELic8_ z`^vVRa^;@b=kRsNB4LQvuINfV9Gpv2V&U6(rI_%UAyyCRE<$S53n{R88O#h{+mM_k ztzqr>cyS?4wC-xf4${|DwKKUg?LF$DQ`d@_fM!)powzj=q4Dn0=dCA}kWt1$sPu>} z*9@~Ik;y5{B3nA;_tl~rC$)^M0*rQq$QpcT|C6suOE8|26DcxSZrzUMHfn10h^#bl z8{Plr;*kM4Rj6jr+03?&c~HUkScClY+Xza!b?er#)i|9gDah#vHwREO!WDn^Y^0t$ zaU`O6z;kItK;;t?cZE+23=-+;|I2_FozAaeMnDoqyk#%Q(v1<w1GEQeP6Z;vQ?AKpMCA`K9v^;7-VfO|gQ?T6wsJ_t z@g_7>KT?8F56*N#CXGHTh=}DD+`s>9!4t?6ZzG^~m$tSx!lDi(4YzC!s)q`de5`au zo;b~6-4O3UW+TXHW0XfR4%zJow-R>Z$y;l7;suBqEmVZ`>20{T(|7 z8!n#*&QAmso_sUM`lI2E4o*&K<0sc%uDK7 z@&mrv9j}Hf9`Vr+*blHQXXq0r*L&hsK;4XIpfO~3&^^6qy7wzMe4Qjb3)~Z+*?>uMbpZe z2Uvd&H^-k5cAcl(wMnP*o}1hciG*Vs59OzML=D%)$ZdrwZn+h5ruM1g0dfRUsr_zj zlQ`brUS#&tGBUF1_m$TC6ZK8)bCIKFFIDd7a(R2qUu9%6tsl0YKEnNQ?)g#fz6EUE zH23uSy|SNTk6S+CtmBDP8T8d{HctK7`YJ7X&(%)*N0P(YU%Pq|W4m@-x-13lNPAs` zcV8dJi?=S4%o)rX(z4)#-wENqMW4JGi%lQ7>5XUpc%IxU#xOJ<5v+O>RBjr|W7A!6 zpA%P5mQfuyi6odewSS0jZJPs#{w_f9_OG1YOKp7<+gd$(tad~?9p7DU(?F=sjI5m& z<1@Y8$($INWwK<@%8BsWL>=bOt{ zpO2qERXNAKo~5CZ^at}?Qw@aqZz4aRl_g`5dEbXScy4%Efnp512G+|TH%>0fr<*v;LApA*H`(}~`1MoG?Md+6P zugV)M+3erTp%njIUasisTL0k{xa6n-WI}pYGwvL7cJV5~uN;m5#*w(vyZe&`05!5I zg-1c~sRj4{b|d4hWIX`BUW;{RxiLZrly@Q^H|7PVb!fib1|Ui!6QSaHP_5xU{&{|S zVUFK?AuZiaIJRQMqSUv)uBOIb);~w^tD)k!Uj%WVMLs1XHWO^j3=D-}@W6)`zBwit z`z;3w#Gc>yCim6ivRj_|;|SrSFMf>X^gU=Cw;JS0w$*$-q$U@8MsD)!km|XHDxK9* zdo$91Hq>7&{NCcKU8~g?5Rm8kHK{b+qJE?*HB%;fRMp^5&iaetJC{)|a+em}<59U9 znpkkzzO}!2Ns5%~QSD@IQ==EV`ikq0i@ zXKj5xf!7x(%dZOWpJxpHNRxJgKDg4W@GC>g`L-$_8pm}@lV85sFnVd1a6@@&S6qMG zn`4rF=0zE$-{-O%#>~<$m1xfVF=E#fKkyCY0gzD!gCdpJ^n#oMPK@GC%Q{gGDCF&*KxGk*f!W%b`_>!ybJTsV^{iC!Ryz3LchQZ%OG5*ML<|T?z3WY5R=SF9(f4hx-zy;CD>f$+Py>9pr5ia2ioJ33MN&5u zRZ};9+H#=BTqr=iO8URT(O>n3(RKJt2}^J6je>0RVbyU9-j$3@iyb$%-xqcQ%Ye>U zJ1(9ROBN!A$2qp2>nq|XZ(!YWXu<2RqC^nFU98H8K)rE`OBVQDXiZ2+%9AovFuaJP z*Om59jJpv{?}h)?M<&{$OYc%b_64Q`z11fe6oAp+qJg*};@gM2Ssz^paY{i`av8~E z?WEfDH|6@maUv$VU{+rGdF3qvxL|We1g^O4qfZXn>|DITIJEzdAMe%!T(f&0oz_*` zy6!(?@;%Sg;%`=sJ8s^E6Zva|G$$Ex3{aPF*Riq-!jzUl!H;7(Hf*W2I|Nji!mfl;*GU~s8tIXQ3tk6Tz zVFAwz^BTpx#u;&f*prb}SzH`lX=%jTcQTkqqH)j*2nY~8IE8of=4FfHj~B?m;9JPP z|K9?#1=3VV(_yLH%ZALTEhGUEmOaABM!LQWc0KY{d-055$dgg;tVkg}CLw?|6F=jl zanGL7661126)T>{nh-G=%OCW7WIwoQoz~Ufo5*4bp^sylrGY_B->Bo$pP!m_J)(He z2luT|!+8SKjS^hqQhFc$uEakSuPlXPX_N^AU5aG%DC zH@L;LvU`gSG^DHZm4yx+d}2Sa#`(5F^W=Q~+A~}DdoS`)qHrIBVap;L@8VkT9qe17 zW@&Dj&p(toeC_d6e3)4$yKG+Mtjo1Y)2N~T+JW}7uKH6adet6&+;r)zl=8mn&$((h zZk%HkF+6jtXl^ZFk)dg3|Mot&^PYiwV-X(t_&9FT!0l9ce~{+0tFetHf(9IK=cdg%NB>C6w}_BtoSwW4 zMTap~P2!Z6AeGlc3pFKxt60&&!B=yx#bfAvQ*uTI!)SB6o-?Z;`U;V)7cAkX`BqsVu zt;^bf*GiA&vCLI0I@_sIdwo6$(82Bdq^WJXdD02d`tyCE@0Xuh;c@*5e=tqcv3)tG z`_4saTt9fczBBRRp|X_{-drH#AgHdjd49IJqN-~1C+r_qhMqbo7sNQ}C%!(kFD^AJ ztNWG?HMi7jiem>UDh%}BKI;!^@D-f#yT&6{#gLbN^EOOjuu@mrdFJ%#RxHoMHa}en zv#151k2X3LV^I#Gm})PwmyWybGo>RhzoZFy4zMVyH+P=}))ED;1>=esKC#P1uP>F5 z{q^6!V+EO5@hc$y4IWH{97G8G@B=1vRakw6Nm?yo^Zl5Vhs|>S{OkaUXZRW_5wJG$ zo6=FNpfC!N>VzLo!e$U4lp&5C4DqP}iIUvzL{)!e*6rVt`jGcR(82$@%eQtneqd8O zxlmL7*XZl_@uzA-^OimK@XjT&A*cz6ulVlWEYQOje)aKjJb|J&Gcg3Bz~lAVHeF2c z$bDzCfn_uk16#&W@|K{>2M2Uv{R=Y!vRzwnVO}ox4#Fv4?BjsBGa8?uR^i#C!N4K`Nk|qlFFTlj%vhvR`^wg=!*gW_HokaBoNrHxg9XZ_RKkO0}y&vg8 z29$*lMpdXy7ZmI}7p+=!QZ@$DZ%}1hgL9$BLlxR3 z^3VQtK021tTa(-lNk8`x_Gd)GVv?qHXZiBWjdk}Qh>ag*DxXfiTWe;&!~V_Yz%oTf zQ=4mb5j+RQVL3Nbw|w9##@&v9$@ZP$`t4x3Lf7X5C1(`fNRG&bQC4WE_k z=MrOGW)V=4KjAfr|q8x~8we}|dz7Sw{`gEs7J)rZO09QWz6?Wadaek25 zCaZCv*k)mHrHVFe`yW#vpZ$Wt#VyII?{s4i%C8u`!3$%nZ)WzlLTg)Zlx}<1vqsN` z;g#EbW#<~bS>(_yr(4Y%WiArP`-btvz1HF_9`a0azxEif+k;=@Sog%`&Bss9J=fXh zosr;Dx$R<3i$RE5y!ymYfZHAHrP*u^IoqC>zrVFTDlUq)%wOS&YJOsB)=8tZwiA9( zuB`5jEu_;G19zzPz-L;iz-%-Y{5VA+HCpIrfV$ilNi66EEgJhCpQ4X9DP%1STNik? zxV5a^#7{BrdIN2Y9-<^yM{sO>6cNFM5{RTKrC@DgMC+LSd__a*3HGPV7iF?Oi&q)i{4sfiA0uD7cRt5AFVm66tB(?J2`SA0W&2LIB&Iu zGQ>d-C$~I);47s1VW#@e1~w-89mGWcYnc6K*|0|-h&TXL5iby~_~-Xqm=a6wbnTO^;wX^zwZ6mWQE_A7fYS{p3lLipddb)dP@mGOR%p zg{5tt8NUid5D*>R zL5znG&!JmCNSVejoA!(6!D=( z4pjh5=e_aP@$yB-Zb(}IsEnhIhpfJ_R*->)l9@RQNk>7EVq>3chUJ;7^@bze2RphX zD3sg2p*ma3^AMy!Jb4jDUKT5aI4G>{Mkg$`@7uS2&)NDcY08y$1$bqxA1%!69^F2a zUfYnGx!X)!u&1Y?=3;mE+HZF?Wf$ByzYO;{g`GOgT;Hi94CC$XdfAhKoE4O}PFbrH=dJhO zH(9>(xQ5h3+2-`{xBMyPGV@GIBZ^Jq`IQNUv_E&+LF8#M))PkduP=DE5BJhg7+H6@ z1R4f>X0=|gCa-^l{^JeiEiJ>cGJa@un!EHB=KQQd9TL-<9HA;((^c&kFngFjd{9SW zLzSZdWK$IPyOO#SH@h3hj)rmYHIUFrdz09(T%_-Vve>mA*_hpk= zy8ZX>=MOXMl*cM$M!Kb@Chku3PR{=Gr^li`Y~QKPO=F{{+w*5L9!d917dLNO1pMEJ zznm-}*T-88v@{Q^E*Za)47|KlueY{^rpN6B&1cIu zoAN9CgFfz}tK4?C<9Pv{tXSby1|3Q1&k#(1eJoaD9=#>Df-b1IBZaP@93wPb|lp8$j-~r!~>yAZ{#5!(NtUBTI!Ik*NSxxc?wcx)3MwKjruYk_VQ1U-tHEt`Sy(h$7igXB1PB#vvPW+)yUp~zHKLn>F-mRaJj5Wsqd4# zHpmO!wsa42TZv`YvSrKu3nsC>*!lIyP&y>o^_DhRR{gXXug@5;$}nf!Dbmtifh_f> zN&;6TwNGxJ?zcJc;22+%U8bi{eU`QmQL%!VJ`xuSWj;0nO|1>gAlBZ&ahq+QNP|6~ z)TjBGNE@AN5p-_zV^)0v0;SS*U7e*2l3YR=nu?b{++i+gPkc&o=P=j=(c+lI*y*w^02_`m6`!S0_vQaKpDWg#A`kme9XY;!Z z!0D3=H&=P05c}>&Qe%EISjT>r`K8M#>gpkZ5jr>CykS4!KRXK_uzU9m?`-63INED> zCfUwSC{m_zHbmOt_3Bi|!K`e%q*bG%KP>8{XCwAGUgZ^i++VX|o3JI)Z*4RBNP`MG z-_n;ci!BYVm5m7&wkLM{|pn}Q`7-AXB z34sCMKi3F`nO=$}@okS&PdMSo8*@FT@^`}*%wjiq2lvu**g^3D`-r=Dzlm>qTlt#` z3JwW-gKdz-G+f4^(U3=YgM0dT&LY8%)DUp&>g%(0c~<_LZV)KZhw>Ts(oUDo1Ogj4 zH@|^6LzRH7XA`Tf8&)xrcx@tThb>VL8xQC@#78MJXMk-VSUWKhiAZxs�=?pE)ss zQDvykG+8c5!Jw=^*=fZ$Apky~ep%2bgG<9GBWa_Z;LARjpYQXWRGz9{_;BZlRF3*8%IAHi zA8yNqrjxQ~jCbHDhGA z|LvinWCV2R&$LN4$lMc5zqNtqhTfr?qGOgQ8T61uKUCnPw0!sHQr2W+c3|&v@5dhk z5_M%M6!nhkO^YgyKS9n;n>PO_M+Wbzva|MEby~7zS`H7{vn=aV}Fo{ z0GGW#`@3K31{>3}Utz!6Gj&_5IYP-aHK}}R<4!^nMs*RY2QjZfreTg_6=!R}J7b=d zhT80+|78syJmkUZG$vK}JTg79s3u8CUN7opP;PyaZlq`aOvymY#N5Ozj3N9zR@B&K z>mqG*HmYJPW*08kQIic1ci!r?jw}gQ8thg*A8bzHuv<7nV86ViV9s>RxCIo8LYXiY@K4=%N26;60aBBgM+UisT`boUti6x7k@ zZ+_BSaP<8nII56D9^(8WC#!;(M?D1bk@(SziqyuV$B%mhu zx#zvEg@^ciWXpwSwl1iPC+nW*s{rf;9D>ytamYrG3Bt`{#wh?6V(+?;w40N|gzu;RpJCl@CMvL86N*suOQ;y>8E zym;(F-YfDPe>XqDQ}dx{CUUOwvhw3~op$rN;NE$jn`CzTT-ZF@_~%5uZm!nta+#7b zpAz1sEqNA)l&wYh6_&^R(XZDI_T|`}EAdx+JDJ(MDkwX{$^8{`2+6p^q3@v#vQ&LDkSpQ}*)d z3(5T%2TZ~lDr*7?At+u#K^N230m0XqxrxbSn>(TFZxZ%iPUFLIB2*2?Nb_OzQ30DTbX$xR+s$q-l}ZZ)>Q{a zJRg}m%@oj+Q~AkFrP5d-H~NO?z164J(Jy<(P7`ycBC){VuU%9yJM_5tK!0fNSgvbm z)(I}b-Tr>R8$Umjs3^EfU*UU@!!}yyS$SA9&rSoTy-Tjk8EtNi>~$3Cos=;uYbgAF zr}KUnytgzB6w0C%lh5pVbnVYk`lb9p5Jq;iK=B99_+fgEy>03p)&B;@B-G7)vYEO&Kb=&?teb60zG+X;TyZM=} zB2|sCKC4vwrpryHN6gG9onL?7J|4(-REgFVvup5J(rlEshqM%C@Vx@&~Fnw?Lr zqQDwyzw3~%M3h>lY>{EgPlPyZygWA7?C_k-OPMmF-g80zw9$;lV3P0il%wM--^SF>dp7VAB#5J1!4fAJAqu3koW&{j(gw^TcF^ zP;B8*n3@OwL5Pz0w{AUBNJ6MKKSi%XqKDwa{wey9x1PAK<;xQI50cPm{r5NZ5vzA{ zyTfIsG(r)CQlk4u5+|`uhq{y)38bfJ?j&?&^1m=a#i~Tggk4;ysy^MzYy0dLM=9_Q zyK2f4pc;!ekPU{jzC!GENVn3TPpe3zX_b~gpzbomQ^U&3i&rYEjSRl%Y+y8&@S)hp z=dogJMHEIf0M+%y%Tr&!CXtSV0}T9*Pr;7~3<_%h!_LNbETl6K10yQ6S$+EPtA1f{ zR)+$O45|~8lSf1D#<@UHEF>0N(Vc#x>LHpxL{;#?cp8gEiaQLr(JdP75Ei!l@gGPH zo+XgW0A}D?si>9*g}P=>{~P z;Dhb&a4&`8HdQImkgw~3O6nb@n}3Dk-q?T8@EN!4{G^q3=3fxXWL|0hiA&fcZCV53 zl|tKKt21nGVxE1v6MaacH8wy2d8FJYr3^M=^mR0Ac3t5%YbyTZWw?)=M~rs*1`*yR z)|soAiek(8@gwP-{~qO7vZRXQ4l*Hu7%{TP6d}A8DKzah`AxmARqStgh)uNfD7NN~ zGtp4Kd|9<#+-$8z(qr(l+8Sg&8QSO#G)-$}^wXxBhj#91N;4HvQ!BLkt+s*EGM=CB z)A~L(@L^|G{vq$bB9=`SKh2mvUwjwDllyo|Fl3+k|XfE z$Km>Yf0!=1IxaWLXuP2; zbZ@qRbhhu+WN=aVd^zj(eN%UQ-TGy8s^T)HTTBdPwvPtrEcH*$qcsR$zScEtiC=zQ zpMl9SPAkJo#t$P5hb$#zGxpAzubvz00lV z3d!tQao4x53^Vd+WhtLbHpzs|vQ8cEyK_H2oTYNdi{e@QMZKc=1MPnjb6C6gzirqh5S%d12qr<)a!KF`GU3q5X&pgs^#p`vLa`%koVVzu7Z-z_V8(t_hpR~OCZSab|5^?c&r&&sxU!z@+JYdTX zLSnGU*^&}iwIfX!Y@zd?pxiPn3%m91-Mfd(DAbp}MaiL1LvOaO8r(yA08To&jz{nR z`AC9u5Oe)A)8=`qA;2t`X*;sb##(R9Gor29hKhwYItr^ZGMT;9%@^Xx0HWkNBZ)tg zaAo+GYs5UEeuqYQz2p^M#1dZVsoI8A7dE@>^KXuyKfjTz&B%YlN1(eZK8SjG@vM4s z3A{<9`iZR9ew>|EVSLrezq%5LBxSn{4ULDX#2_1% z)G?4vtl6@O-y{#Q7|1y+trgy};}A^5fUZXlZRL+)U%PSRt{yC0qf=5?h?kjw;}c-o zB)tGWhRcF}*B`x6Qc^MmS(^b`Qf+ttBm4_dgz{Hi8;wPeh4_570a6x)j=H-sTf$v2 zpY(y~&+$o|5#G06>v!egMdM9Bl6D9IfK7~D@6>yidX%Rd7OBmt4#Y(!=shD zx~w;DfI&iQrR^C>1=}YdiLm%h+El0+uc!w)@l+R|YPmdEU z8+f}kbh09M*NS<^4QH6UqKy)mXj)}nJvQ4~l2kCdY0`M2IWoX%VEp6t0AX4CW?>~P z)n{dIwpl9KzponX=IR)#uDP;nFHdgKCqBtKX0}+y$unQ9+C#YQ5|`%jHFZDY=$(;p zjQBk-PWQlT^mEDfr4;SYZ#`Z}kAK5MzToIxAn}>g`YG2l+Lb0HtF~sI^16}8tiM}k zD%Zh4qsCBL<=_gQ&8vLg<=O~P+~Ggjr8%x&@r^bvbavpown-AhO1A@RB9kO3292@J z4~7r)^LDi?y~@ev7uU~+@6s2mJ$b%?GF$EaO?R6&FD2DYx~fm+{n{HMQDW+-LW%ryKY-(wCL-E*Uw5Diq%?K)SHLZ z+i_lJk@+TIdyv}2JFzSw`ebp1rny3^A47wbYRk-q;! zZ`JGOR>RX?v-tr-+3)T$KZmhT{6cn7;F9>AB*k1>wo0hrZr7@+p8XKxFWrm=jnJt zc>NA_>#tMed?({Sno5X`?q5ampgkO|5-Ea@&wp06~&m^B*w%j2$ujvNhZL1!2me;Sp&`^BYMz3U-|0;26Wp;bp z?W{66Qppvz|1_d8v*n%-9|F$BF8q*xQ@arGEtsl9!4m$;`qyx?U{3A>q-6{pQL1P1 z{iWIjBVam>q^#SI_Ga_$uVWSy+rqUkrf6=~;fmz4&0B6p8I`X#tGDZVG=1Vkzx%DV z?J1VcvEy^8MoQn(dbfX#Y-T)qbYqnI7bL=aOFQ+qw|(5xWsr51-nl<-vHsL8c`2#< z@35)z|G9CAMUB>#$!HcYKk@kXyDJdZbn$e^-P8K=ZvxiL7K(PWwJiU*SLjOg86%kU zz&^!NFI9TV<*FnX`LhV1kXu1%s}LhVj4sjd3XzyB!sv&K8eedBeoKgtR1Gjq;?dLg zf!^r&>Sbhz0OAYWHdy5Ruz;Z8CShUbef##6X@Qc9c)A7)okTp=xXVx@ldvfqB&6h3 z{rGYF(+-%SVa+nqSI>?^gy1;>0{0?d%|=4jXqM8pz^Wl8KZR$La!2n*H?H@xpui>YG&RRBhb-uW>uWwd9=+bJ{-g}Q9 z-^N+7kDUbu!Pr=R0stg%LV;Q(1#27Zdsl~d6@U36f3YJmJjBnZE(1mw@F=e+^!4)c z!mJYN8P8qu`>JN1^Fn^q)Rd**Zny`3OgNoLg6*I!T$l_h<~g%b!*l ze@inxbG2{Ec#S2}vH?lKK^^u$kmXR=e1Bz5WA4GYvKXNR+6x4V`@_p>ugxN%d-0>f zw-(1VLR=1~A-ZYgPwlA~Ds|)uD7~t(n>SNqHdC7u0Oi7U3FynF1jWOL7bFdT|9(Au z7_WZaYiJ70YF{tMs!YfLvCu6`SP zQdC;_WAl8(&8jLQL8!=$5)WNE-d`;>X)HBo98+x)K28O6e0!`iV_>*TVg~mL9dk`v zM!qz&+%@%V=gd{X!ogy_)7R?mZ=;|1t#S%$lwb{@X^3)^MN;#cU&blJc`tDu;`E4s{3_v zRK#p@eC?H=+_Q)7H@uWDUMnl1!XcT>sgPT}VueiJOoNLXyS^2T=2)<(^HA&|k@3} zyh_vA^4B%9naSN%XV%kDSSM}cJ|~<0Xuni`mi_s&H!NaLneW=P)J`ho)UMnVJZfQ` zllOgVkD{&Vq>TX@@2nXvft;Z~PIYT(scqk1ItitnG5;jIqLpU{lcbvvmpILX<)bm? zIkQ^L4jd0MmaKG(ye;f{Wp&ym+6I}^KK0KV=7ms5cTSWZ`S6_j_D+1{5Gv_FUi}If zEj>$3R{xS>b~c&4?0|(f7ZXo}$H++X&|qt9WbI$;rKoOWh0{wYJUqfM-`TKsimk!u0kv$alz!e|%gKv9STRTi~=V}r&_PzJ})0Itb^DbOSVZoQYczptL z!2u>#kVN>6q}PwjGvm*%gg+N<$lb=@cnK8E%>A~h1MR6akYRp2R5A+;4+rrQhAW^^ zvf)}Ndmiy)$?z+xsngDT#T1aV~@D z^y%gkt4)#ch^;@wiIHygEJv}|j>?;>KdSAg(7rc)OH2{SI>-?NBY0&?#ZV*ClL;7N zYHW0Fg-@geq4-`)YW|aJJ9V62VIaVZl>=#3 z_fdEyL4g^gWo#Trz3jt%+05)}rX39h!UqD)T$9x~PlAoKl#yf&zGTmsmM^HpkX=0( z?p~0}rOBhvy|uoy?Z?a9>q+Mx?jbVOxOs$|b6kFUPUu3={uYmLe9OGAF=-;4 zLdfI51(aCo6-C*O2w~Ug*&U}DzD@;1U)z+sgCHf@&Fcybtwb|(#ZUSH4d~u0#MG$x zL3G-|(XXPlw|T4n?w{1@9+?-D)7Lg^+`qid!@P2+^x?S)!{Osq_r=9C&GkH)fpjPa z7i^5oEPvZ^WoEeM_k>}|uCVF9;t9Tw^DK!!Zo)Z{EJk0HRr*0jw&dbA_JqSz`OF1L zLCddQ$z1hmf63nQ2f&_U^MgGn?)w-S=D0;yO{Tf6*wqwLJ^H9VpNW3<;|8eE_m4W+ z^`8J-e7Q#2NEQt|I@@Hab9zsNU2_~bg!*#4>z^~IIwo^`+Q0Y3mhoh_q@&lKADl66 z7yUE!-a-37*oy4x(ecbv^K{D7HP`7>=&Z8?jB=TlzH9q<3+sOWcNT6Vj-r>22~u-9 z=AY}okAX+bI3ldWi;N=A+fL^VY1X`yBNvU!^i7G((r?ilt$M1KKxL$O631^>L!hP3ER;uf@Bb&iQjE?@Bz})q87xeEz0f;_2Lz7K>%`hw>Jw-0lCx|FWcQ z_~@q*wrg*jMbdI9I;!w7+9)M=c3Lmf)xX~RZqo_w=`or4CG5tsOD_`56N=@Lke*R2 zsL65)X}>h~-p_$BlRV70>hiqLAwF6wPOjmqfq)Im>8FPfs5LpPCyt8k(=jS4AMo;9rI~ z(FWC{TvX3>T#+Q?V3GAKpmhqm@YE{l5;~O(3sDz8HU-nMxT5Zq>K(Q`kV`KYd!1u3<#{vzsW(*U@ zG{Iy#$x_q<*v{Teot7v`sHEoiG!Y> zrZRe*=)d&WL``T2a3mgo#esOQ`&b^yQG@k7duDV^B3Y;QuWNomJ(Nb@NxKpxM$tJF ze1WBD-)fHh&ZI-2ZNg`EP$ZuXKVL*bi!X@AgopJ;?Ck3B^$d}`1BQLQ%$506G?f&+ zDd83R-@^KZM{@(arbohE;1z^m97~fWoiklmxpTZ0!lkA*0^VFzlYFMF_0ZCi4R~z|Ne7e1s$V{Sn?w%%*k7_j$Ptl;%UAu zpP2D>Y?9O*4j((Gs{iiRTCdL~i5?#>D0(w#A!5wngO72X`t39K!WUsru|WEM4R?Wm z4Oxr^u@9;_H+1|F%Os;leEUgGmfpm|mMe@GX4R9XmPn|KB2=Lb;R%l)f2u~N8uQ=#elMg1{*SR zZr;2}n^6zLR1l6>5ryid-852~x^OpEN?k%6w%~y`98^_2fMC`J76yOu)Ui zW3w|egQ=0T%7xUb2w24u$4Eq4$2cOH8xLfQmNIFfbYzh!wsZv(PL-8H2GCuFAPNy} z7FiII54DNPNTJr}veFEcyCs+65~o>!Rc$*MK+3Bm04*%w2!x=*rSLv4;kDmI=Rhd*j6l;@Mn?Xal5#v0hvf=m1I7tdlw%XHQ8IO zhqCY}@ymz_j*kOrY|f_)sqqC{Y`IJf{&K$fZ25{`E%8It6~-*uYsg(ax-qeI{ho_- z;E{L!;ao)vMAqP)JI$JBj%kSIegz>&Na;)VS3L;XfcYF=j$J%=d~^=LrwC9vl8E>O z1&hvRXeIr8Yqg1pMQ~jm3wh-L1rK4QLkDue6d<#t$X56g5bS3OA3O{kJA|0OIZm7o2Ace! z93-s!^I2D0Q?`-YFGd`$bJ+JHY{}h!~lXYkt<_1Ha5`Y`MSGP=ufez zp8GrEpbsn<-93Rl?thksO&oNTKThEZN9Vt(3lCH0a|cFm$`?fLUVJ8wF89?|zzt*JBWB=}5ZzmyOP@0L zY}#IGM~df}3$OUv<)r1S7*?&Si86Y{!Ei!kKKwA2?|ghTnVFeobx1OVx%KxR_6V5$ z5xoUcWqS62)+7Q$uvMJmFRI-lsUvXCT2=X61*)$i=)uBm;f06a4Ss)rO5)^2T8FV1 zQhu~&IVu7&aqNCz1@8xc{;T~BXBSrd9k-S&SwiAqNI3zq=a7p_uJMBHmfPGu+Ewe_ z;k$8VXoJZ(=t$%r#*(Q8+jr2;Z}8H^EW{sxsf>(AzFoUY+u6((nvobqSVh8!t3dC{ z-1zUYt7J`sp%%Mr47G6}YX;W^8A!1oqxN0b7E(moHWob{FRH7nTN^TMur3Z+xKf{E zt52Rcc%P|uzrL^{!9l)Iom%w}YbRKS_^>(bfS32@;qxfI2zk7LV+T{&VKBlrg6_I$ z%ZV_$4at0hj!6+ZpP#33$!;=P`5#B~E(K|%eK6OMb9I#gnH`_XTPkhF4RE_IT7!p4 zIjq4;i>roobe+Au(^>f&|Gl03F>E$iCM`6?D;XI-VjdU3ReIdvZ-W@c)ALQ!;j{6h zuTebSTt+??1JxEiRnBB*C}!fjTX7+(POGb_-Pm<5Jqo&!OJ-R59Z^zhcd=e5apvzl zR)~EwIZekr?$Bw6b&%4-KhVU7r>0*x{Jqz|>me^4$*v%Q3&AN^sVRr=ukP!55UU7X z>r*vl#a-w*XQ>mr5@DrN4OnVHSXr2a<`drZiKNYM9zejp$)l8eEW zu`BG(XB*JzNg&8ukXnN3NBSE%;Z%c+dA*Y5WGTvTVub6k_#(33u3T1#tuSwfhFPp3 zAx0xH8o=f8*b1o(L}onkKThpPK|g}#B@kWy3#bu@OT>+U@>Ie2P5ghC!4c~dc*S9} zQBl0)D{X0kfRQlE)sRr7F{jZUFOrfFxrkOnHPO95o)< zyVhd|YUO({9nLJJiC8~f3RpwgK_YFrZgKdwiR}rHSK1ZOJZZ3R}VZlVQ0UP z$Amx>pWC^6osE42cQYGNgD7T;g9^tFvyUuT+ z+s)xe_Pc~`3M&x$wKBTu>g!-rf&>H_qf$MQ77{H3Us}H45&>mJa*-ZO(lEe&zPs`{ zGdaR>idI~MzLjWv5ul>q_TkpT&A%2Urh=dX5=U#4kCq$;xyZB+5?aTSwhg?|h3hy{`%mdw zoEHQiCFVWQvKarF8tsKrj>wBhaQ3kyRq2=0+US)SX-{lE-mwi}>He$T?o1o^M}cJa zA5&^^Bt|TecWE6K`>Ys%!`wDaOijVsao;(l*C(s|7m~vtV3bi$=nQDfRbB8p;q~#L z#v^QC`;HxlT~s1_z3Lto4RETINiU-yDGjInVFDJSmGPYFC$;Xz+k(F!(211hXar{ErpK00_jFxQ;sVEKwopb%@SV5+*_k^L%Zpbb@=Jd zXU*nFEMBOASO4#k-?_u&O`E3GCI${OHATKP+a&5z#4?z6=N-Fq`>&iL&x5K!En!BEgGU+MVF0<*hYpZe?$z%t4MqID9cw3K$0yL|NB ztqW<%swD^)H%#3?>rr4%u20j#%DprWy#ITNmj4;Hp|AuRk$aN#y(mIPeK1lcVS8`g z=~*0dAguz+r4<)VnHDCwwzjq>D-To6k0~qPiPJ@+ij5bc>fMiXKwCnAQ5f`7?+``N z3v)fPC?gv?+z<3?*Q6ci@KdOg;P6``z!%er8}A_g#fofuOG<#N(Wql?Dh?dqCdJV2 z-@dtnM&1HXR0{V-;ecDWuAg{*2*OV&Tcgy5B6j_E-v8YReJQvkYpQX`dBMW?81myI zeht}nsHHf{+0+s1k<|*K4T(AWn8!s zj@>0b02s>zAAGTOE%|b6)Ak*W&8Z?*7o=q7ou&9Uof{jzj#yC0b>KI`n^1x-4pbDl zFz$M+Xp(0!myetM)VU@fmHdqP2Yr@pDpRpFAwSQg%ie2huW3!#zLu}%I_5m|DaFYMzv!b-539iCy`ue{C;T7(*7S>PyFRal$SG#zsNr2D@3ESe#6tl zaT3gOc|JHsHc3jZhw!AV4x;(MyLWl@6FTvJV=GMhvz=@;H3^{7kVy&%*qSGs_*G&d z&Be3`jjYbqKsQuElE4Bp7N?P3FXb)o(Q}jFMl6l@_mbdJm?I!L`(G_3&I70d#BLHt zsP+~(`0pg7CR~cd90cX|$o>1DZ9qXn>?*M=AWA`avO?r{xE^FuF3BsDuJXttw5cGS zGUNMhtyxQM(cJYYrbyx%;$DD} z9OKVEJ4ygs5U~J9Z%TH{3oSR^2y|h0Q|<6A%8y0q;RFCUuSx?~FyuDAWj3tq){*$f zLiN(hR3l;>K@6qfD0@~|Naz#UFJry2jF$Eo``tLk?^rI+Rr*rb1R9~36Amqwbz&z)NC4C{ z$R02Oc#Ye!IK&Bdr7z(=zxVxcZq*tbq&2#3ei&wbdmwWn(Po(T1dgE*sG7SU9BhDc zN@j))N=J`kGqN`NJqR_HZs<(tKHA87Z`QOzH}?>u-~e|ATG zWM*LRu{5!M9dGF53*Wokeaztjf``PnZ2ORZmE9*!`X2oa-Xj}t{P%nt_SSMqs$Mv# z;YBayb12Dg)j=@v?)X#{o-1C-9*VV{67iRb8!J07fS2u947D6u3&L9LDf%pIx zVKL>UT^6~Nf6d;|Qqln2_VqnHikFY{s_=GRBVZbVRtu&%T5l*Xl9Po=%8;(EE;(IU zR80|s2DsrSAb_Zpl!{B*P}!rkUoU2|j+~rFj&wW%v%#N7)*mjWx97Os)UI9?C3egq zAtA8qB`<4VI2@NXcS3=FTpUc~@0LB5=Tu_U;-jif4wG&*6`0pkbgR@gWCTDh-juKl z>}=@C0culAD^bswWoJ%~j8DSM2*V@4{&s#7r60+<0Syh>Ms9;G1#lP)^Sq^_tg6Za zJ1#m98WN^ql@IIZg>$#qr!M05ZHAfAWH;7oaLaUuu5qezqR5`q(71`C1aLP#Gin0F z02*$Ls{9?cid3`aWurHA${_1Vd-Bztw!p4Y$F6F0|n;--;<*GMW4 z7d9WW-HL0r#EcyT;{?hL5X{-CPs0>ISm$f663{-aZ?LPq8yHCAdpPH0I+CZK91XhG zxaf&_Gy%5DNG3T1DO&@S!bR6k;w?&3y~>-VM0TJE`^=$xd$|(;Ft@ zjEjj=A(_<@?@ z&^0_1c|*fzabxf!{T9pi-46RTP;pO0NYUGSKMD#8+O1!>EUwyw{yXYMB_Jz2?-q1E zF^*`Co1BM_WTq(HeuNgrmAtlFRzR@2}T_Xd{Ja`Eaa%|p8R2|fgZ5WF`UO<0A5Ssc1KPcSd zV`YQY;_*GKK)~D8oL;|FizkCI)f~rO7uo-*ngf;Zu1$STJXIfejrG+-rj;5uHa9)6 zY3EL5vQ~Qtg2RQ`snN1Lj8ej`Lo2sFNa~&HL6#9}f77qWvLH0$r^TR&Un8)|MN6#%+cBCS!a+DgcTQxRa&B!)xoh+ckFypuO-> z7ew0s{kF%{dn;mBVAy_tIMYX}0gsKu@EEy~`U|Q7X5w9Va|uBVs2bH4GGP#1ho_bR zTV=Z-9tub5>#%dpz5_ZHiDx2A?f`{-yMI zxRpc)*@MRpBJj4H++2ZUW6nTmH8ZURpz977!*n6h?7SBD+d}If^W(E&YAZoI9sX|q zW0zg>_8&3Nb#}@s=0uDj?i%>xK_Bv-och^*T1lTk4HUW3auAAVwi65T^W=Ltp|p~4 zLJ47kB3^4hzOSSb{+npLN$*nr$23iA&IRV)+{G3d?UruSq1{|pV0J}FGx#=gQc8yc ztMuk^^}*m`Wd8XsAHc655muH3*W3KrKFh;tBm)qg4Vd5#sE48A=JIHN;)H@rb|an& z&p?q>oFe@6b0NyjO$H-qbNGdXPGlX#-X^v74$kPsg9@(ac~uU~xa$5NUvC0d^V&v@ zZ}ZfosDzMYPAHTnDU@bul9C}rWvY}W3X!BjluBE%n}j5lib9e~np8+Cr9m`kQvKJx zcR26&|E_OeuJ=0cIb`qOex7?+_gd>lh{I3?f@?A8BvI@n%vp|W#C|1h1>B0YS?Mf6 z@H(b0Hay<>gip2e)$1#Z^-N8hT6Bh|I8u|7_5gQ!ftFg8o-G|@jIsHn5Dly?(5=!I z1|aT_F95wJwEsztC;Kdj+v6wIlPP z#*$XS1@>ISO~?G}v&Ww*nKOKvyen72aXtR^g}E<$r7DlJ&Qy-`)jw0Dr`R|(?(5Db zt>Jp0wgvT<%a3Q{$ft7&qVKC=vB7;~vzHiklerk%RpO#|ZrShRGZWIBbKPTSnDSOsfDtPPMeOBs4A&iY~Lq zhenWQ7czk~U+nqb;3w*<=-T(={S-WF|8abyMz;73bOD#>~UO#AU6$zho@lTt0n`p>87P&vuaqokc0n6+#y!hFj zIW!ZsWx*g|6shAsc1#dRmq$g9Nk9#vr{u2q-hQ$6Dek>_7BCpAZz+W64RSSpXy$cx z!a77>eWH6zJ$RUfrN&J(8&d1uA;tH0_8(7Mh{VjMr4B92_9@$jcJ)pdNLn{xaFOQQ-YlILYHPy`@kuZPDP*3W zic;e|Kmd(^oZTxdF}H#^I5qDYsx5fQg0QEH)$Ej_w&`!*?pqpbbPIm>27bQ2FPLgR zFkcc!z#$_#bXjUiV9)KqF3l$+NHt@A9!IaB;3Wp8A0HTWv{IdDrI`#Y3E&L7ZU!Hn=ri5Kl^7qWo?Hi znY3snkC05(!G@Y?*3~5wr@eIOT$ivzW`3;eYjl$SSFY5z!~z%ZyrXp5G~Tvo1|Gqy zSFe;&g6A0nxF(SzPLg0gDu!))_RJ#_W(?x>6fn+0k=Fe_WK?jlE{NaU>(6lcbBdTT z$X3ufGH9^nAP?3B;-5seUCjcaX~p%5T?+);_8hX7XUS=NCs`bbb)1G zg*p=P1;n7?>P!k@YZcx)PDcSe53++Q{m>@@&yOhnhIZ*~#QJd=$!stJUtiLCs;N!H zAJF7c$=xkhv0SYrOdiWQ@GtCtp&fEVz-o8AW*&`}aVMPKB)dq-HE*8UDZ1&cKBd+# z@a3QpmkGKLZaaJ`JF;;Dy=mYwODI87DrDoFGvI06y{4C)2 zs7XBmmKGfc%#5u_KHzIyn7*0^bPkB!!sf$k20uwydLJJUIWLf4gKWs07x6d3t~!Hy zbF?a^iVN^l^(IkaBjStN#x^8vITk0_w=@Lg)g$*YZx`(gGwEn=|03Rmr?eE?-p3++ z;3%C*bQt{YL|*yhu{u{$cyiHENUJgUep``wq8n-wkVEt1kIW10ALV+UXPkG zeSw^#`5y=PV1}ZLvGmo@Q0`kr-WVgTi_%xWe2Hc6+}Fp8#cX=~V%G8SX_$=S1euh< z0EhtRo5U-ZF9ZFQcU0o$<|a=E$ozO1y0N9`txzj_0?5P^F2dl52sRP0U(82Kpya@b z6Doz{4^>j;ftyRtvq$TYxc#1a+h`F^KNd$Gj@E$z7?jWa)a_VXSX*1)!+sPjhe>6J zn7+<$H`W3H#Y=ooVg)%0%O-4gdk1v~F>*wcZB_0iH{|EQ|5J2yY;WK2u9q-1iO1Wv zUSO)|xdC*E)phopE=r#t*0O3mND*Q4dC z!3^X_dFJ5OW15LzXagj;>V;Y$EdSuDT3+{_Q6#^=ipkBK^=CF+JMUdNavk4~{^}u7 zj4e(6Ojp6Cn!kAQB={j`6&D+$nB9r6h27Ct7fjP@P9%!vTE~J49oST(Mh3x1)yDV5 z&UJu#I1_HPRpw1Dm~ts&nl(IS6cEVncHx{4&`ctsz$EYn$My!5xO?)huDesL4t}|L z38b8dkf8QvVsJn-3F@g*utgS=zM3QHPw}--jz6bQ_j497Rhspp#m;+ALj@=aKAq+G z!3C@c$tUA`G7nm@3bQG?Jp7T?3f6BNT#no_Z4yRcD>!j(-FSn=|Duw(Kv;%;;J>5e znSV09m;15NUVpiXda-XSswV~RbcHk^>Y`W*nENijD)urf5q+Rn%q(wr!FP#r{(Z9D zECQ>s#B~k8lTiSx6Y6`20pOz?d#0P!)B! zc9j~@s5NmGEb#!k^#A^#qFP;na6sj;>VsM=3Xp7j0pkiPlq-{BRljk*Y?1^#$0yS>Gg}e^FoT8+>CRmN z3rw18V8#7YV_u7y(=~6_iv_!%D$P5sT#zz`^9NsfzVf=0!1u?Vk7=Y)z2n8*C(b3? zW}*SN-um0%h^9}+`U?pfO$!3HC0cRQFl_+i-X}wY{Ru9I4I5$F17yd?E}xDsfsO*@oEp$B`SwIwzUd^2 z6qg(2;!CJVHNTPp}W8hVd0Zg!%Xx}&IB(#+#i^i(gMl@;7MU* z2!d=BB+VXs;EKrgqd%t}h^pdNUpCCA@|R14EzK6z7C=eJa2kmlYq3xx`%S<{VZV|_ z{sG9YUb^IghWKX>kKW?WpEGrl+pQnvy+(vJaj zK#ejFIR2F|Ox*rbMMZ@;OkQWsJaUm@hco7Uaw-BA#jGu6pjUyHguJ-8_@>KoUA-+< zGK(}FqIcoEPmlOC(IsKyrN5=I>VIaq*0D2AdDy+aux9H{kJ~OBYro#Kk?wz!j~KpV2QWjww$;Aa3>&3m*N$!Y<;fWjtt@E9N{I z0vTMlr#RG>i4`}xFiyomzz`|7R$?w~X#{=@lTrapvi1&uAwXd1?%k1IbRk&rmvVOt zOYLB!I<{Q|x3%VLehBk0iY@%L#JweUEeuIi)z8TXn#OOiwqDfN-R6h?NOkS0*^0sa z@ZmaG`EUZ{k;FNwCa!+#1z3iohUUP5`@c5{`lvO%F9ekU3%)UW2yOi~SiB2lLs7>W zl)X)bFyR7NNz9u9Q+(;ff<+Ncd4jc{v_ByJz_6d~007{H-J$>JY~F+sH!?1Wh^nwP zpk;6vt6*`7^7{3jiRS1qQKJ(NSiXBh=M0+Qy0<$Uh%v9y@?w7eQ@EO)&E>odT?-3> z1~HA8GGz)j#y>RakBN?6r=`V>0U<+P&M=b35|98)JpCofbs0V&`_=h{oC`6^*2_mh zfqjv6_AgZ9FMYHX+CFL&1JRp5D&o->tau^j%nY8hEJbwWNCaovgz)DR^oY%YEtE(u zLH9A{p+1XJ?;n^j8Cp(?2_@w~uTe>0o9~y@amyAM$19)Cm1LLNbB*)+&4zG#^z$fO|NVP%KzkzPQ%RyLSs(k7LgQSgm;zc6$fwnwww5SgZzy6d2c*@m-^Y8J_t=yT|XM2H?NxOcLz;{Ue#Wdi*(IF zd^4FeVZt-uHm;J&TB@SWNLZlLvJw*vV}I%lb?WFw8}Q#2W0n=b!)aCDkR0MTiZK2=$pq!#{Df z$nn?Bf>YZegMEMe{prT7?-uR3CenQN_|PxelosGMXvlf=T9HG1A@3$k8FEm*%+u&Ph=`j-h1lgIPts)I{aO1w~S*P_uSX4Bfcx(lSBg2XlFLI7EM<04KC zgG|D)P-KVHnxX#PM~yIS)z7gN{8u&G;?wEHL*+8=yOR2~Abee5=F=_k&v9#!KojhvREbS&FJNAn` zJjVqi?{q!ax>x?_Bri)xcsH)4RSCpq{m=Ph$t+&7=6PQRLyY3&<1IoOY>V~ z69{q&$f!(A=CTlP*0@tMik$C7w-r-P7{V}}jg5r~pLKV4KNpZ>YqUt~{T;hYa$_g! zr7c%)@KKy|$;#P+tLc>p!+L`?%mCPsJ7Xl1vV?T>LDR5jm_4HRnR_utqA zk^&%~pU`}$@QvO(05A{4Zji&hyuN*yuHY<~RB(Mv}P=uOE;vYfe3#J^*=h;lT z*2?+^Pq$&3@yM?FA_0h?&`}>PvA};^b4j$oC)e)nz~#s`fE61n;9J$|a6{2mgxtvR zuZl9=-H|!Zy+dS4azpvW0hMh|IKRQ^+Py;k|EHtnPBD|TKZtA2ERTU6cg=}kORyH} zMiSaS@+L#~OmFhYGE6BUS%ZS#zU;w+2X7Jf*f|WAF>~d@`|G;)*iczdWff`g(d-nY zVy1!emfRbWq;yg}R7Ev_KlfqFz`<47RUv`F{QeF4*fz*6h!~>j4xp~iRZ^OTSsg9m za&;Xz+C6vtNH7ji>0SV99jx8;>qGm?z8%98Ik=@e=+C8hIuD$L6vs?cQYIJFE34)k znV2kWI*T=mm%U8W0s{DH+_vZY1DFR{q8gWX(7~+<29>3yI&&5YrKe5e;(CtTJ6Tjz za@aCJ>^|<#=jMb=!Tgy#3{V1%A0L_{E#j-1Ynmx#SKtT({(zp;Kdt;G33=Df;ho$# z$JZ*pKP^D|Dl?(2ORNbuvovT2f#tPq4_r2UAO$ zHPn+sYus>`lg8*+jJi{5{j+fBccD$6OJKfLHj2SRz&3*Jqh8#FVYE$O|9odXXc)1* zW@u>)xDiS!hYsDNNo&U1b$sxE7)A4msZ*yeOIn7R3}#uJaTd8N{oeM62Gh6}P(>El zMOg22Lk3kEMmM}fTtV(?vnG81*tdYhaA26=aczdQ6Xo^_A`fCslmWufTC2h^?$f7y zq;aFtd4SGR6OZ(^{Uz90%TN3|Nc_40&OtoqiYC_+t7u{VrM z-{Mmd`g?MuZTjlS)vH$RMA9ly`+_02CrUu*-P1m1px~i-I`D-=2PkM@KDWBL7Y!T; zBPTI!4!V!uiGK;>jiz}SSTugM9CsKAns)mbC;`mH>MyVeXX~x#2DC<47%uV)6CgV8 zGhEkF6GTq;*C<<6F2b}Ja}4A=wi*HO;bIZMJp>E^xTa;`?JMW_Psd5aYSW{lK`D5~ z`%Z0VfcTJwkU&0D*eV2f?M;^vqZ><)zz+YqCL3kMlq?#hN%XGOKA6RFv=PA)2T(p@ z3s2K~N;NjLI4ISj_551L?`sZJyL51)qUS{(piyc{dHlB3q{ve2w%WUwwvm!PqEf+C zJ9mfFO|X2~U7Pa&J}tgkT8j7?3vrnD(Sd`aG#O#1sYaZJlEyEY?d z=LjW~T^m4&f^7oOWUL7p`J-_#1_cXvy<~9$0tMgbnv{NWD1q37M(qrE38R!c`Uot} z1y^pE|J@Xq0-J9{_wPsA44?&I^Fq{A6Ev{dl~sMUP^-9E4?f~l$=yfnRIXAK5+w!C>i|Z6k2-QgNE`@^A2;kbm9xB)(Jj>%;5V#Lux$oUgUWKUWbSV|SWs z<(C1zhK70emxCTazr~h!baZrNgPbr9qT^r7ft}`{q`Kh>XpDWq5t!SZU|NIgd45?TKOP~002z7vrbP@x5XkO1S6W>lj?@rd+5p}-TS zi!jZ1gWk|L(*q`TC2T#TAdZn~d|5@@(`u} z7YG9jGBC1rYYzh|rSiAtc+0)>b9n`vN7 z%p#)X9sADDK$wCT$w+wooTD(6WJ6{cNR?=`Rqao!oWx~)jri59x37j$KeJWOZbG;hu zw4wUUmE4h+E_dNuqvvI*HbeW9`;n^Pf7+;@z_Dguj(471T?lg1Nj$l=^UJ1=9ziy) zn9Y1p)WPJ8oSva=3tA32DBdq=U8;Yk%@GSIz@Hfbi5^1!xh8}IFFZtm)#4go%^{&e zcZ)Ihm$>aIN5(+=02jWi7)Rh4I00P5%~yqP74}WoLoIpk<(PF(L0_O0MC4U~3u#&SVL6B$0efWG9uhUi;$JYtfR8Bvfd_GC>7m*4ck<%8EYsNuuH83MDoXnYEC6u z7#W>~Bx3fQIqN8f1w)2_kog(KLxCxRP@1t6Z7o6p4@d21=+f25=j1dP(%|5IuCJe` zry^E};jgOC@1(1>1FEXkr-OooU{m0Q4tWFvcT{8Z=7{pnW`!Jd z3iL3~@LBVH=*`k=B;j>mP>V24!K@{ip((`(tQuY8B@-5Vi$<;jopuLQ<3Ft%lW*q!}&pT&au_*#`>V296Y4n3Zk zh834ozT&~N4-#>A!!mL@d$ncA#u~3F!_$Ap-D;#jsKex3FP0W}sCbpk@ z+KSOAns8SIN;BaP2@VfkJApUcAk~mlIy6aOtQPbEAi(^0Blyq5X%ko4j)aG3&z@n9 z@)N*Q(Q&v;uCn_C48eH^dVt>_HkFLVE+o(l5&u3YTUki}IaI4f6-Fy7u)XmN@2ra$ zJg7w1)jK2FY4DBasJDYKbouxT{rBi~m*o160{28^LcgR%8p^--`5l;eT7{E8OHL8` z9&40vz93Xo(M1?v>{XDB{7?A%`{SEZ-oHOO5wka6(p`W_a)9I!P*uzJLwdn~d%qLd zydcHBFtTu+mwJ~zCGmox-x(ioxY$xf?3=>CS}1eGHB?l^XClBL6EN}oA;gk`4Lfc# zswbGec>t{f9w~jmV$YsE3l=ZV26K`@WKkTP0-UETH9iVSc(Fb%#2^cwSrt4yPZ|ee z)O;DFuPB7MAX5~Dqt#hS(Xp}Ffcv1|T^!>G2TRZ&FKv)*FwWsa!r6>xG!1sv(2klWV>RyG+TLf|YAOOlKpC$EYEt;Y3Ym<9`FiKQ2ZBjH z{|&0YVYvia{zib39vfFrMIA+7`XQ${&`oGYgyEX@wauzFuMub9qew{o( z%64n}_8eg9a+^wNh;=@iBtl_{YAvHXy^Ut3oN8?n&J#FX@I5FdM-%Ye_Weeqj$ZBs zW)x^Y+_l9u>dv6K$2h$R<&)WyRpY5c1#+${BbOg5S zKiZPQc8E*(Mz3pXEClIj432`ei0{j&Uv<|Yh#__@+V?e>>|$6r8dv9N8I}(ohW`_( zZc}$aDN3tk5yV0VK)AzVRh3&sH1sn~1ffbV6F&V=(m+wbR9%{YHWdvB4b0-o?AnTf z-eeKyg-ENjXDerUslIdWYrAl#P+eeh(ndu3h=>SC{nP;Ad2E~%*+j0@*rNy6>eyId zX01;OH^-Jjg@v)+9PGoz8-3zHRW7|Mr2*@JF_rScVht+duv>Gi*OLfq06gRZ8A^!vtBz*^+hmo26tuq`3b@J8=Qi>aeo+-O}*_H%z(kPY15`l3$C94`a+sd z)g**X%chcO;+}~ye-ZT@BXay1-VYb)>Y?saJAV8)I4joMG1UXb_Z;R~X_~M5jR7B_ z<>i|-t2*l&Du1;Jr(kEu1+erHYPIBnvP1hHAxV+X*M8~Xy}wZG&0j(jceJqpJ!}1F z?R$D+88{$(7|MR8TxbbL_IVU)XS+7Kp!`ck=L5pH|QKF-xAcAr_&oC`4bO7nsk`_rw zi0SO**o+}e_%wksX#D8lG<c69XakLzY{%+N;vfm2KAcht_1KwVwHj+bREF6osBa9=CglF_X~cIb z`;Lum>-F^f6aSh)ZIUDE9HyRnMn?15LTyy@fP*mo3(!V}rinufG0uTeVX`|}uX#5J zlm-UKCsg61OT3o<6%{^&{cQL&H1Y6c!$0RGp8Sho3w6$#Mj7ZuBXo~{x}l)j*$(~O zCGzSA+jk5gQbR4Z4Xx z`lPxTAiOkM!rCj=_u5)dikD(yD$L!jd5W$IcMURnlHL*tOnr{EW=XI5Ob%1m`_L@ zI9(L1oE2NNkHl&ySZhy*1$wh&kVEjDs5&@2Vu4zJZ+9*~K>`AAe4)2->HBsE^;bbK zzEZ+G;cl>rJz%L^?D@$ns%?(vsZ&x;?Yl9?e0XRDv+NGQYLv_e+l?eLO+&Z|cl|XDq@WuEe~7{eH+x`{jH%8cmy8I99gmi{N$oSJ=TkInEoXG+Mk5_jZy${8!Tr>lq^SwC;w zyV`-tZEe#gO-dm!hArsHBwgV*sz49r} zzp6c)SdfXtNVm58ENw~)+o*31ZVZ`Y2}Ux!kG zt&EhbDjt~Myy%Jsr>0g7IPADx3w7Lv3&L~s@R_?w$N^?(L`KFZlj(4m2L>{mNN@RM zK98~Z+J205dT3?_xC?n{8rN71zw{m|i!wklCN2&T2%<6N<$*N}Rbp!k03cclMHFr8 z5TqHj(YR}zGoxo>(yQO!-raqcASW>U9yVtVFUmt(k55DVAM%!CVZ#g?C!h*i2zW+X z^#BhnR7|!3+)h3A4Fb7#RrEI$D!^!GeQ#^?gGzO-l$35T>NHgw1C;RSSXvxJ&(q)r zGRmWGsjQjopYs^)6*X~y)%6t6Gh*U<3T3uxZsi?0pPBAy=5#uktC|PG*8S)PRq}6$ zG1pR|q=r|CSMyehmRviJhGlk}RJHEQdDwI}OPMtIggyHdV1=&@#K&b_??D63DMaj&Vqv39iktvV5zRXr9l4!Zio#rYx`O zCU_Hw${+VY2fjD{;ln97G0y=yL{6d{+iHr#mw|P`WG;&8PkIK=PrV0do4%lTgJ^I@ zKww~RZy;*|N=(27>%mPq1nVx+g#jL7+7OctAQ;hHAf$xQkIlW45Oa3I1Oy9#+2g*) zHs4={4X(`=zb3y&& zIGn;?r%sIH*@6ogO>7L@FlHsDl$WkuTY!7v9EySv+ESJvO?E}v4bTChN+#_DTX{z6 z9JZ7Qsmb~l9`spUm24uR;VGkR%LXY*){o<%>O{beB~gIOi(bBX!B#2#y6P}+_SdWG z%_o(Y$yzT7Q*snS2P+VHHM`p9=vcy72=xpA>e=;2@XZR%y(?wuzRnZSR^!xQ;aZ5fpMik6GBx6!uv+JJ83hik9>HK2UWt&4H6b#9S`+sEfqQYrtQOdA- zoSs*{9Dox9!u!1z;8H~^n41ogr>G)%EtJSgvHti`}Y zVE$PKb?T;hALMZ(ZBBxAjvD}&d;3e&c?962_EH}<4tRA1|JlB23Z}MSR+OcmaKwUO zpN1guhdLI1i?4u&fr+^6|54qr)K`GB7rmZyGSH(2#7&&bGX>sEp5Er3KbI>uPLJ9D zvmpr&qpEuF3>d3z4`<<%1NEpytyTz2++xe#_WJ5?Dd^ew0Y$?5t_FQ9GZo`da!dJ) z4SNCS$dw29_}P5{N07SEjl+O>Q`T;LC{;gOzH}_*M93bUSkbjX6_CmB9;qjKms!(5 zm^#$is0T=Db&3`u617j$rd1qy=FQTA!#g7Nza~5-7UN)&E&kPpK(=@U+3oN%Q+6Ka z{nb@&;dAji3;bQUwiqc!r@#h9?MdAtVwr3r2?4y^R{miPE*yAf;2iIqLg_a$my zlw;6w{?&xkz&%QYV?lu9bHWG{4;`g}DdYwKPI*>ZTZRwjRT$_LHD3l4h2^M~6}<43 z;&uBmYJ!0Oe7Lg`&|&-O$wDf~*;Z1#P=b<`RMoB|Q_ioA5}>Elps-&!>oIH z_}!f%QEL6O=V=x*3!UH~1I2cKQfqZy-X!YRPJjjo48Nn2>fEIJKxOV(dhWAaz<`W0 z2j>RN^#xk@?wDn`Oapb9zm)8#k!06L=x#z z&2H8BEm)&U%>54^K1d|1YiOvT>VRRHDXKoF6|tog|1{IC9J;>#f+-|)Cc1tFViPDn z;LGW6WTU7|Bgh(gNCKuBzKbSmP>mDn@~X7hXEoqvwrmcHUW%R~s} zmuO!?D~_VKK$PO0XN-~YrLZs_+~RdBnB@r=rn&cI&r(eaA86T2-rmIFkWu3s4Lu;+ z;ZaK~e{<_%%$sQs06;7!+ZzPA;FTc&o+QOgg>pY`BErTU6&o)mW`tl5F33hurdgs>rg)() zae6*d@6>EW#q9dA7XYZxicGeeQtXI5c+3zwjRVogHitO2n4s>HUfR(c!b-{ml!unW zOrqBnJ?~DK-M}$U7;1oRYi3SP*XG0sc9({V~l!kJ)s3Cv<^mF%mvGZ^lX&8~&Hxoh^%ltzwFLWO~8fNGE8kGk5 zNXRRw(WDkW^1Q^V!=cCl6(JMyX4NE`!eiDz2oh5WN#`sA5J#X2EF`nU4U#9el>&;$ zBPqZCx+yRp9ULCnv!J(Eg3@v;iLCX>R0?H;TtAdl>r739z0P3@aLNN|6D-ed21d~a zv^xo{ROAGdb|qs0QCgz;C(Z6Iz?vlLJ?<{UzVl98@C(FLQEc_<@;wIu$zwIt&C$_V zq-AFiQQ5*~7==uZyt+$m=Rak0nEJl~n3xWhkvxyc+JJL2H{WDaivg;R;(7RFsT@{#19h`z%1N*_cgi7xc(` z{W=TJg%+W}`q?M2hEHK5=3p2TVlf~aZ<(P39x>%vOiH96UV*=%G6klq-oX%t0(1$6 zB|}k^U6N_k#Q`($z&Skq{FVLakzIy3ktJvqqz`7P$JC`qbmL?Ggv-c}#Ici|xZr;T zeTo3-qf9Z*5SlQX83b5#M3nx7ko>Q4D3&6C7VzJ0mj^)#k&B#yP+x_1L8(OATs)6} z-y+IPBQ41wxPncm)&*S}HAPIJ8Z;|N&9?v%gHUuv=syI<@jN*lLSrC6cn*9t)Mxw& zC#N|mTu~|$E>DwJ{b=*%?mvANp!l*YLyrqJ0B|nTx9>2-O3=nkZ0@-?YbbECRjV19Mp{!?O z8$C^zZj@&SG`#JN(Y3*<15a;O{@dR6jA7YxyUU82Z;SQVe-{5!d42IziI&Ct=pd@^ zU=|3e&-s*KGXyZ+eXt{3E%0RKA{2u?&v@O)8JvNZy$5xjS|MnIL~+S%Z;${%6N)J^ z;b?C^qNf0EJ(+MA#=>ei8MWe1s65XJ>J{yiHU`9j)WUHctcyX1glt9raaBJt93qJa z<~8ho8sUzpV~BHOd|rHU2tn0Dbu0Kw4JyeP0q0wJ{#Tn=YJKIeHW8f%I-k8KQipbv zoa_KM`x+)a#sCLv5!0D4hL@Y4Fjve^|Bq%73_*0clu(kaVh!U-37@EhEyIzwZ);q8 zb(AhQDi+CxcgQNVGa)SG_NsnA*SfG@{i!d>-Un7#m}Q zoFVxN5=M)K$Q(zGN5s8BJZF#EF`^*61@yLUiZr3d5fZRIaDc!Xq)YE6Fllfp)X)k7 ztSLN__VU7oab0YY{7S2r{Sn)4vNb~N1XU<5Koh{x?4N={-{4cy(9_?a`G^bf_f}95aZ+lJ6hpE#8cOb_$$^K{ zwjElAyF;f2X7g<2sy`9MhOMYSNM2(8b)pu=-0_r`YP=5oM~9t$n9+H^5sOLR48=wA zAvtE*3#-Iue;eXB!`PDh;aExrwB>y8hFeZn0JqlqE#!zqomPO{pNYPl6&H7`49AlV z&p|~^<#aVlV)ub&3$=u;=(w*bx~4)>#!7-AA9(}Swm3GGOxS#NiNBQp-o*JS1e>N{wJQ6?@;#xu_t~Qla%&=a zR8#{3@;qe0HEV0@(&(;&eUhJIUt`j;ook1?;HS6LVut{1J+n%b%^Aln;sty$vy6RI zFv_N00@_o(bo70ca!ag}RC7fE+x%a+E9gj^AHi{f^tj1()edV#MMY^XBUofaM2sB@swWJucD*5qu238^8B%FDS4+M694{qf}r8q5_Rua0<5~HS&}2XOJd9` z6j7V`AZ$N*Fd(Cb2#M7fMKUz7I2PRN(oBG`=v_3!y6 z07#VCUG6Wyhy$h)sY`A?K*;!w5tCH>vPm!qpy#*};6~e(fG}|L zWtRQ$gclxR5;2RfXuJtin9jb{(~4pPLC6G!|3BViSaGa@uZ)ecVMj>NbrsE9MYB1m z8lDTlcuf2M*|VtE5P$f=9v`CJoN}yoHENfg;?1h389fGY_P;RN{fUJdV1h*+0TmSc zmR?1LL2rf1XfxYDp*01718SE^X&)tHVzKZVXft_#%;OxDBS^a^*zV^>t%|8C2DVU?)h9H!m= zD=U_K=l9;x)~4uhZEr85IM}mm&5Vmims)x;RfWH<-MR=}Vra7aQGvYMw{P!+C~#@O zX}}QxUa~L|(6g{u7L${cvkOCMQp3zU5y@hWM-9NAo`52Pd|9Z~L@NxQT_R01tgMK| zXPbkK0Ob05tnJgx71!f%Dw4F(yaL65jiD-9(V@amt{XQxee53`db7{KO5Dqfr6N-F zY|13A+t;i#?`z#(o<9A?nlR9v=hmqv#nnNYF%F_EzDl037>zF5$OMh>5O}@)w-IM#ZzT)62*-&k8_3^8?ytEVy z7%%Pu4RmV(wCwwK#mM~rg2@c8ZK=niQH0Kl`DmLAzQ471 zCok0;&0uW?+;cf3WU*h)T)&)K@G0mW7|d}KJ)0eYjx_4doyEZPz)L;v>l=UaIo=+F zaq~8+`u+IIFt6h6f{@G&z(5SF3)Q0EtS%Hm!R{riQH4 zML%BA^JCnoeTPJH-pizI3E*e8RyRAL0J?>DCLd+sX9dq45k;5B;`@=_kw6Ug71SU3 z)}S<$0#ok;G6D?M0HnGDXa>L&S=qJrSmRAF7V+I0Uv8O>+gXZlLMt6)qkT~0Kd8#A zZd;0PF!*XnNGuwT#Y#$@>cV-)@;aqVg=Xb=Ox-8oUb<-hrMU zHBFz=(+;lMwsgK~ypep-$mQ{vJD+Jn z9`>E3^eI(&a%4=525#FsFeS6TV$>A@K*;pc#N%=fSha)swO13~`vEYI%f^gklD8V9}F`#q)`9dgul9#C?s#mn=l5tATo%6pk7V3*Tex^}F{Z*- zLPKY+Dt~0NO)FSh9ogX2Zu;Pd+<{$|H-x*3mVscIep~Z86AX??`ZfJMpASZ~yN-z6 z#ZZ23{q5L3+aPiLGrfLnI(WuX*zhd-|1enG^(SvCrs~Fpe3&uj^N8TJx1?|F+Vw)M zBD^s(JUl!Bk+^%|9cO7&>RaLI2=09+GQ#z*@A()EQAs5w3rcW>4J*6Sj(uZ-s1xp+BGHi@-i6SQ}EaT@DyWuRM*{~{dL#Qozf7JcePd-IrIk(Xz@s^ zB0yNU^+xf`SaVbPX5WEoD6vIkk3{0ZZ%M1s-8~;3IjAM{j|pS0R)%C}(XQQUJzaldVI>s~t*D*vVl0un4RfdwU0Y~&7q@xmn)D4PY* z8B`2A*5Qsf1pzWZ^{k1b#2wOHotX)WXKtzk{c!v9mco@8u4$N7=G{xdoaw;to}X90 zXZp_X-Wq0P@$rY^?~@DpO=LgrTMJf9ws^lVhYWvB$S(qTTg872gX3XDX|H}->A z5s|8QYeOH#*99_57=H7XEz^NtVOVr`HMT7nf%loNr%-?6O3c7wh8s5S8 zO;(5%ARa!nE{)Osh|@YyH(%+#)MUo&cQByNtNpnEaGj{%#$?-%&ddgN(9Wt_d3?dC z_~9r;?};G@AFxwsNak)?CM#BzEEwj4B@BI09Ai7_^eto~*3i?}E=2F5X z=naoS#&|CUxw#2B)j$7N&)yX`Z6DWk9jHkWV~302a#{7IW<3AS$IEobvm@z8FiKXQ zv20wa6RrIr`kvfGkMZpXHR8;*bdJYO->D$#r`Yq^Q$(}t5N=K}#>@7YvbpxuoV}d_ zE4>(`3Bh@zJYK;dI~Ju^EDQij3g5gbffCL3+s+Bdy6E$QS9#$1yv5do>^ag{=($t) zOazE``;iurc_xJtKy@D3U|Pq)B^)B z=>|{t#Ic@8&_U(R@eVyM8TZVcoKi~f&r(N)f+m0en$Uv&GN3M_V%Q~hN6V!o% z&g8?m-NA!Pe?QEestaK0)_Uf;GpJwy50( z#lafW8Os60I%^=A=PzD-G<)ig-QYyU;gg@W4$6zl9O}@vEX4oX=6Y8n^$QPOa2|@&D>omE;~JE{Z@`QO?n_TjS-8Max*Oh z&`O7Sy}JQEp4vIxZOB`9FqZS$-Y^pzBg~z@&GXS8!O&GgD7E>Rx)omucL@A#OEB)Z z5*%#3j(tuZD`i$EKvdj}{Aqa<}mmywWk=cQi>xK^KbZd9?zC~2;!-TcGn@` z2Zm4`<9J4wlo)JByDy!L?H>5Nz9Auu&vP&ll-~7{3&WD!w(hfi;C}^W*SrO_cS*lV zlS5k9yUok4yEVY7$Qn!gf&n2Y)sA2&t3pKERCMbspplyhl8xXS-*&=Ziak)SDD_A) zJkD{XUj+UdXUN*@F)*VCfr&N!wqxJ~hsrzYFMIOi6yohod=ozJ%9cIbp^I{o3Xz(Q zUQyGHry!8!)UCy=yWKX`i19P0(?{1WTobAdCcP@o3r`maJjc+e*tb@eX69p2{p( z-arFpa(Qf0J*pMclp|56)^*Hz2$9HJ3>TJYjA1AbeD3I_!1uNxqPpH8O$NNRdE(;6 zm$>MMe@|dCRe(^ejNclNQNc?Dpr(&$$uiG=7@+pI-+Zw&?DwnpvT&<-Fnz-~o*68B z2f>2T7QppE1}=k!bV=@L)JftuwC|XoXqDMxWn~T8TdHgSqx~6=9~+b53X=V8BXg(I zzQ;W_f)`l==Q{hA_j=gDRfJogwMzJITwDqMiH;POO=@Ds_n@j(aNgY$T-myaK~K!v zT+bFZOp2Ym4zd1EDEfuJ5jQ*TA~$_{5Akp4Rmw4F*b2&A6E2PxQ6}1q81^by5^)Ug zN!Jx8_{xv z^cvrBY1EMh)6YbJnaFc#G^6AiOgxhI7jLkseCXeqKc@liCV$IZJ(G&;^477Q>p*Z= zFn|6=aHIXjW$Zqg*(R@*7Flg;7Fa!z-ZXpb8f4_$hX6f4TUTitCAUPFNA)DesxQ9vQlK$)+Xt#FA&apT-oYGhJOKk;S( zq!Oh8^EMxYNNI}_=y7OUN>&!i@6uaQ;&oSRXVQG76~D4Zd89^e3p9?MOMeCz?0>4< z38$xySSn`}+0F=3V#udFUJ&xBe7y$T85lG=VAsAYeEYlJzh_@IzHi#$it%u* zcwcg&{)$lvMu|TBBc57-rO|AZ*>~leX$#&&9B3Tdu1!>qlyx6`g2GKCW+u;BptghO z4*afOudRKN>Ja>qkUqd_Wx(=fy?O1;_w0!pEF9`py8Wrp?bk}ww$GWW439GfQd}QX zjw0U7m(ECYYdwrU+nCB-PLK&+W%F9$6#LtdKZ6^20ATcObM${Q@bN$ZMJ{4ZqT$0# z_y`zuV`XbX`mf;A=X3+nwqr~O+v7b=^sn(pYEu_87}eRJ5Iux+^j>wML)vzrg@$-; zOEuVmtQ)*i78MronF@f>C_| z^LPc~-n4kLloc(9`Y^a|dLd--P~vhVqDvFxjMDYka8iN;g!Xkw-WZ%v*ALhnh*`Oj zgM&j-2TD<@dxKZonN!{Ch{?E3b;fqg@Y33L0SdVTkFprtk0ha%<*14<vbGE~Qx%0f#nG7qRhe5+B%ID= ze4dUsSO%+qOZTRDa}k0b{v62 zu6lQ-orxG4$Q()@c{1~^e_su+I}6tbr;EWzggF643QpyVK%#T3@FGquRle9{b=Z%6 zWEtIXUXCHK4|zT!?4xZtWQoY>R+cLm_RkQkv<}#B9B#<3jhCs%W1wsZtVVMH+OmCU zZyr>|RdV;!|CWg1#G? z%p9Yv@J?~LsX*1d*KUP#=AgSJVu*pU@jkr={nPP90)y`YyQ=Ki+(Rf)DKAb69HCq% z3km`-h?!-{&ZEK*&aLqU-!?~gBaKl6&m)w2lk=pwfI4WRh%h97y(f4q=uu^$n}X-u zB>s{9t!O_hF+)Jxb*wrEXkb%RkhwmFPrmh1T#0C^~Ia6e`)@ zjwwF;+AxiMAN>kye>QsZDoS->6%`8|mGR~AKukd5q30XjjcpE#Wn@Zm$;FDu?H^lC zyqchm+lr)`B&9b19Y7@ftcr&oy;~jLN{j`pGEQ7D?Z=_a!LEd@@av4hQq>3~K+|jE z86M+eSu9<|7Tjz6f~%55X7$kf!rDV?@MTLRS1nDhitp?^hQnK;yp#O~#WqU`Q)hYC zEM;et-wauHO@=0?WL{=pX@cenZ7uwr`1okEtg{fQ9_}k*-s^f#4L>@MJ@}D`3d+0j z_r8T$B=lG*_fk-2;!h=Q=w|;F9`gaI#DmoVr|YZ|9Sx0l2zI5B>H-KDt1cUk={mc` z(V*M0X%wTmLVO_dgup!24kr!<0AA*ti6&SVFCEAN`LOxK*WK_Tak%7N!G&#-x0zMHektpsS&A{`7)8ih zfjz|XzBAVvB5@@xSjcxBKuk3HW#r?{mB80QuFe)cip6!Iims;ET3e2KXUg>H{ZAXD z|C}8T>}FZ}DWkAqi$4vI>VyEm(#y0N2()1peA4y>>J;%qwRv?y#sD2 z(sOHndc8~^_th|`tvjSl6Vqtiat(ym*RNio>o#<8`Ry5w+72<%xiK~&;~k0-&2${o z2RRIey1ts)X;1~U6n+KAxWniyin_J!2U<&_1BvLK7MeW;m@PlL1p>EVrNn^%0PH;o zXa7vOZ(zjs_0;;+clT=zpu^t>_b}@cmcLrA?lz#VulG%fkDYg;bQFWP__p3derB&c zJGSsSJwCuWQ6-Cwz8hNWD2B9*WKBEK?y8~VZCLeOSY01f7{HPH569T8%Co~)zEAo< znI;hF2=0-vX}y(@(WgNylOeblpCGt9R&12AD(rsl0B6M()g4Xl5Gz4!fco1pelKd# zV6Z<(g91QJ*B$e}BWWM4(YVL&MF=`>3g(OqR##BziwUQam+m1spnhu1^ z%x?6LS7;NDVS$=%Xh_I`?bnpg^SKlU3?4#R$V1wt|sW~u9i2KV)3M%rOHj` z?oR?gKc3hyg5f=y=j?hM4OA->U-t-~p|2do7bkg9{Q8JCKB91!@EH(B?;=an%pf2` z))?$MH0Cr9P*+z6r$}aCDZ6^uaUFlO_5JKm)xNVuGH9{%_xQHZ6YuST z72(Av3oRkNXP_NYC2Ikn!b2qi<%Y>ME=D9ShBde*5vfm}m|^s@6?0tx!?Oe69~069 zx*v>|;a0jqa9)fB73omM#$)9vaEev=Hqfx3LJ`4~CS2i`_F=iu+o;7&E|12TD%&bC z8dbaXrza_4?t@P1)lb^G2+zTl+UR^2vuor|!XW}8!7kdfd-nnbh3%-3>G!udobYvB z#(wK1D6duk%z1t6aU?)2|K zX^d(HO-3<(j0QFXychBI(LGtzcL3s}kqZUIsf_U$hZYC@S`p4q_E$N_?{UE69(0?I zDqgs7!?q(fF~jDRrUc=PN%{r%~UKQe0nIOWJ$tM zNcieq+tl`>bkP~;-7Xn~N?&9Zn!w-bt5jNn&tYRvg7WOy>KK1iCfkMMQvZC&9Hs0@ zPF_IEzWFT6km^!`_a)e3CP%Ir;B3giJ&yctBg~;vq^3YCO#Zs3J^+g6h!kbeikCoX zfR!ockh|Y@3NO5)qu8qo`6*#SmjeQJ>gnJ7in3>-ak59h{+PCqu^E{w8V@lQ$^v%A+WdXt2)^qmA#Nkt z6!+giXRXljW~Jcm7z>R2C@TQWgES}jD-DbA9j5m+#a1^D+I-%Bgic9wmnof+;K0k5 z!8jCG=x<33$mn&2)zvLdq(RbBc$cmVFJ3vZKBzNT?ws&Ee--M%Z+YL|vV55T6%?w7EGSUY< z)KDyeJ$M&G%3$oBfedEpCiZvDYbAUJrk6~FyfgV-EfrhPaqZf!Icm!EJx0kA_vM_^ zJ~S<0=Mz>BzRO7Ll=z+#Bv{h#3cW7Z~4>$P$ZJt$MVCQ=b%D}Glo3B&&!6QRc;pGf9NfExm&fgt{<<6pbG{J9~YLn8j+X{{IC73Z;$z9S3 zK<&sJ>Pr}Uhb^Dd=+^15U7m7WDq9Mj<-COQ10ssyZ71z#^WBr{TaW(;5XJm&Sjk-b zzLTFmCPV%lU@&eG=}lC+bwL6h12zZ?g%<|BjA)c-?8A})jWv{smoH!5y?p^=GNJ-N z3oXc}CK3?X`DSII@wJ#i%qR)hn2P=cCH}RqcmU!!%3vGi)$G{DW8C)1eyn16AyXWM zl_*XYpp;ll-jH{MrB<&t`iJesIQOJ}0dBMX$#FdxyPhVP zu9J;g0t{H~9HV3_M1sppF-HU(6LuS?X{qVMP5At`-K?!+tO{;K1NS^| z3MuIsJ`h1QAF-S2;9YzxG%oL*Nk96gHjOruZ9|dwMOUrnU?FC!Lh?`TW=(RI@hvW@ zst5e*As1`3r3~{wyu^go&fz2^?}ml!N}0~fj`rzMxn+@8F06IxcD;~YqYJ1%djksN zdDBaI0qM~>DW%YlFBZLuakB~Irs8poOi^MbA&l!5N^zm**2f>)f_QX5#t-chZgvuA zOvTRx5G3_+mr$>n#5&t?-KOqY|JY2h> z_yZDteMhe}Bvmh7zBIke*Mk;cy0ki7XA0VTEITqq)Y^&@I?wf}PBCikx}V4K>vCx5 z(uD4s1HPk9{(pSE2Ut{B*S|e#qOtc9?7eruhy|k9L206*qF52^hy@Wy6npPN6cELs zTfl+>w%8Svp;!=&u`q~eWJE#WyVn^sdH?TweOx@bd?I0H&e>=0wSH~+unc#z?XX9T z+UXGE97My#aYT>Yxo6K4Llem<;j7s=+BBItW!s&nf#R!T&T`cnd-EK{<&yXx1ay};Q2V4r8HG1pfO7dpRyZJjcTASKZ{>&z} zJH5H4((FxkHe-vQt5sIJ=GJ%IvtUY?+H4P?Nfh0k5iPWWMZRuOKWhAm%S#rn95m?n zl6QF}qia;^^J}?_ySS%}133r9JXPDTNe#<@fs<)m!*vz}mZWp_E#pxS5Xx6rdmYG$ zkB8LrkIpU%&Q|@E=ke~`YvF;7)33A+4-4A~4bzwnBq26yNK<#g&QY=Y{IgG^kn7O=;(fs+7~n#=?!TS8s*oGc8Ur{V>6zx*Q-PZ?d=`Jq-`gy3JH2n(^=$)QnT0!N`4Ju45EhwdlZlz&N+{j z4hAD6frkKoFpA=kt~}~X-0?vHj$Nfd(e#ZzGv~r5M|TzqQh*qrH@=REEF)%6djF2O}&0`{Zs2BJ(Al>fvnX##jvD|y^HM_*_K zT1z^^nnJI{DO{=)D6l;{c36iW5aPL9g^JGLjX_U&>|+oyTkh}Ca+=p{-a!8_*{FE} z3-Q9pYaFlc1F(IV4=utaYnb2VrkaZ!J2!K5w_bnbpKG^hd66t_Moqv9o^lKDnYFrv zeho2LT`eu;bRF$t-EyCVZI&X`03ur1P z^MH?=+of@Za>qvtOMs}xYBF#QC|?-wut7-qS{>QA=Y_E)aCb!Cw5d+2H?7~Ulxz(M z5ZT8+Yw0{DH}`!18(=G)L7SZNhktcDH|cxT8+V)Yjqx<>2|=-oDVh`tGp}4ifh{k^ zS~&R2qCpS(ByP%_zsxyi?n4{oeny{^GDh{j>y;kDZ>i6o&CoQ5y9U*pZQV)b4nB84 zpuKpKWi~x{Rzz`z)X+X7_AU>lGfL-;+qRwQ0D!dvO3;%yWf_{#MVnI< z$(8D}tJ(->(Dks%-smgRWk#uUAt>1UHkG}VOMy7M5$i2ib(A^ zvZa06ym_<3Xl2t>8H9bq+Mi0{gUCi`fIbDgBn%{um_>B~au+SbBaZi|?=$=#!WO{F z-_!dTPQ9qATGyx=>89z>Er07gEQ|T&a}c3OyE5cv@^%_y`9wh`Y;UD6yua3zout+Q zQXcle>+4HeN)5~3%$}VHXwMG;QRsIQfE8PdYK`|Oi}p|g$WspIa338uH$%iAwAI1~ z+$QEfcH#5OR!!^IKkEHupSRP3(4XukOb7*Sr1aabuo8g8nzM_hD{b=uvLmTc{yDzt zVH-iiJV;bTfzx;8Mk!LA$Ai*D^4}`E7BG}w&*?&bb{)AkZf?4+SMJS%2|&Ht%?%xX zF=}JD$lY(*QY zFAW#VA|Al`&4Jm6M5U;y31=*lG7=8vX|MnJ=NwiqB7J=JvPI0$&wh+Iq34T0xZXOrQ3d~^N{b4Uj|JvRgcRwO^UlV z|IF>|CAVHK_PrLbi^sCwu2r7`#!A3@JYE7UR5UPzEy#ehW;!&7O*fU#V>VcKe^~Qu z&~@c5nif4w(x|OlSG6tY{08}KI8LH<-r*-x{W@kd7mQMp*00qO47T)BwrJ0OI4aeJA2_WVrK z%?-@1+=mR<^{DO8tCt6H0C7&D-d^W{o-Yk|@9On8YO=qzsFjS8Pecc{S=mcm0HtFS zroQ+lQ%pxW6kq)5@x~1sLQ}i&DzTAj^>DJ&44V%BIvF(sYyQP=T{17MNYRi&`VJM z14dY-1;|>$OV6A=yEAMuexXhsJFYnVnpQFN31yl|6~4(oi9Kmc5(A$)GtrgC!Bd(e6xZ zOY0Ki#p4A?7Y!NVuv;Ocn!OYHBf)b89&FAvmc(9$lSEWryZ%JsKa&r3k18q}Sl-Ms zNAt_jx{um^V~FzZ&hlJ##!`PZe_D7la8l#`N2T-WG5twp?T$LL-8TBe*(8Sq6s-h%jj_T6h%J$Lb4_Y$kx%jSm$t;qipJr9F>^cWv3iB zXU=D%A;A8e^e(8rsGMS3IAzxT=?B@TVS5G~Pgr(;@nqBptbxD3%Vf{DHa6Z{!Kv8} z>MlkMVJaoz_ji}(b?ep*<`i)uS`2|w)XO!>M+giWFHK#OzIDRp8T;dMjpn>VPHQ&! z42|GHQr+w*W*khvKCg*xx`mBUm_mG6#0QjqeeLq(?1>>844To>LacOZ-~PG=#g=r9 zdlVU(p&|Ps0kXy2ZSznMK3~w3jtt_iQ@3vR1SaX}A$IIy^Q2U^BpNZkX;RcY*iQl! zs6xFLTmZQWNXa~{G>EN-Za&_Dm3d$7R za-V&$(EgB*tRjcz%E=8o6-Y>@f~WG~)hR-t!?$#9RL!rDZdwitfN&0r;fv!VMm9gx z{WP_}_;+_lZ%OTRXqA;xdhSSaa#t=L1fjLf&&g@7Gv)TX%!{`VG2U?BrC!sS?z`8K zqwgxw-;vuN2 zgQy6ha9;QewXV8=QDP!PVf%FjY@|y#KIdtVYI24MK0?WlFlhFsooSna;>tD)^?d50 zZ#%{JCGyk@NAtRKnii}s`eG5dnc#jWOr<3#iH8O3KcOK}SVrft@Bn(LOU`uAmrUwt zIiy52#i6w`;ao+Aird2vr~EZTWgx-y*+L8zQXElww5wFH+;TOA6YT#N&jhRPJo@yi zr+bXOg09;Lzuu~^AFcxld#V>GSXbxh^l-}V@%w94_syI?juwjRFgQN1gibl4cTXI-Mon)n{R78yBW`$r}DS=X`BbIyM6uIwOhGM*D)z4tVxa2Ym@_LI8nXF8r)nOJ(jLv*6- zfL%wjibipZD0u1br)vq@91rIho;r9MCrmKu2WoIhji9L8yz*O1ot(Cq%QwC&SD%HG z!XXuQ(3?Fp|5oSd4uAoBcMX`-qSkgrSzCR}%AA!eS+@4qz1m>V;2TYCzPr2bOzUpl zy0wamH=M!Wf_g^B5P*$s!lii%VB8isTlk>pmrIm~BQW8ykn=ED+PjAaOjOw7+CFQ$ z!OxhMyfC#R*jQV=Q4cdh{7=d`^Wj^G5{Ir`AhWI(njVzXyw}m7CN=#As(&yUauGg`bv_JEBo>Qm(RXFR3 z2BzD&tY}VH>o6*_L~z?`jHBZrgsCAs?SQIzSyMQDn zEb6}Qiqo&X%uj|d>m2*m=5y9fRkD>u&UUr&mT3`Nm348^)~R>3E5}WgdPH?G_?HP* zQPmGU5a~h}<7)>4n{{;e{+LAXQ2eS-;ttV?dgQ+e674lTu67@Iq3Chr82%fY1=D@^ zV83w?Hes<{&>uV?|HdMot>;Z8)6c(&@1rZVl20nI7p9J2wpKr5mtuPhi_Kj2D6-f! z86kt%&K;x2>8ZWBcInY-wR%UFG5%$;d{)luzyA6w<3eaN$f`6ps_+5o4-;7y>4Y() zk}K+fs!G1Jr*7RES^V0&q88Q^R@>{>qpH!4@&>WS6=sO&{(dpWcSZs?aP7QI^IT`#%3nI5U58##%$JSQ-eG zYghJ7v&k|HM~E0^#)K|SK(4@5{sTnu!^PKR=YAItl$>!#zGCS)YRo}pJuqUmmuU1C zpQAs!etD){j>uS0B%q7-%(_l~f-OD0Ei#s_0PFIlV|Ma7a#=lW^C*JG5sDRA`G>$u z=MiF-ykBR!--CW{EJ#?FkyIj+&Mh7S;^d@0jSuLUcVYV>!-h@VALjHpe&rd89)mQ3 z3{SNi{l^W{#+v=#{WqW^pp@Hv=~ylXgAPKaAbGS01f>xLqs=#$jDap^9yc}MFqyks zkvQwtP=lOb8i{4qNn3!ZBl^NW8)0J1{m4agAq6$agyW+0ea+H~Mn_EU^r*H9?p;8y z(B^ctA|YojB8Epv^8c&EUTz2s1?-;68FuB>+8Ry6AUzgGHq5$x?r@dJb2ig(fT`UV z#LPQ#Z^Q|T4*7Z(dgAA`mqL|-2Ib`AKWxKVpK1Lf*-^Fq$ZNlAbL>2RPwm+Z3@TgM zcd`F##cO0Rz$1o-DzHTCMm(4bbdm^BBr@rjOH5qb4ue9H$4II10}SWvr1r4yEPI)@ z4z>JWS1ZqZdSL?Ynz>J5MEQWC0z^>ZIWPce@tocJhFx>N&7LY=zlA0To#Gb^<2XkJ zoqEE08%QMPk~th_PTspbZv)v-mKw0dPn$mBsvg!jG>PMw&e^qavvJR!)~3bTW9m0; zIuE5-^@p^o4TTu`X4GoXu6(V^7zT6;NFCmfL{d@45b>GC@*b+k3#rwAD+M7*Z2E_u61m_l28mcp zk?!pC^9vJK31FA(4|9YDCGn^Ke@f;xm3meijPWNGUdCe8`OGNUPZUFetcK5?1L1&2 z3Tx!=c0NyV4W(z)9F zbvXY@H_;{lsmp))EOgO$DsAPDPV<)+n=J0vtHzffeHJv0pBy#Y_AkwYv1htpF*!JT zK*1BYM{u+X^~sEMJ3?_`xqJ%Smgy=gHTMiuRIw=fL@9>6#kO;$o#w-axl#jha+YAG z3$rRs8Dq+jzeo+ag9b^wuGPffF9lM%fnS6-9T?C(-yPL6SYN%L_BXUJP;UO%^R;*E z;k5WytCn1QJX>5SWv5Geln_lKFbO|$X*Om8XO6+Flw9X6)14i2}1Dyf=v0I-7rlapXjJfH#n2FIn$ z9=-kY!S&|eT@tgJYghJf)^*ZZE5-@hQKEFQSYLP}dx@V*K+ zdpI=yiKvPfr~ik43C=j@c8mjgx)udJ9Qg!RUrYzQWjkI{mWVEiQ|Zbp!lfe7>+MMl zmy4pABVD*&TIFtwVZoIMAgMY)M6*C-@cw9qjtNb*Rt2WzFL<|GP8R*aJZ8^-PnH7Y zFbx8_r@2R$KCfBP$0Wr4!r)UT#!(;pGzDPGS(aFIP_UamNEM}^Mv-o1%3mnP(N3Lu z=~6It2wimTTH461torO-V+Qur(as%}$EL=*V0*cV{cBw5ff0wcmgQnOB`dmN$4ZNM zOkq~OL0xVPcF<;AEU*U|esM_|( z)wl~T4~}f$8aB^;a?ZBj(zidQeg(J*Ea7|?%vayfvfeH~bsJcPhaD*jwZOWyw_k>p zm7MMAnvTj|Choyd(>V{?)cE6v_Zb9uM@S%yme9O8H@3HI(C%`vCsnXm&qA54Xoxn8 z%-&3l8Y=U>tV(BGFitD$6Cwec>>w460OeduKfriyerRCeMWsT8X-TCmflf)N(r9f- zzI)KdPOC-!VSe^Vqw`PS`9*a2)Kk%-X?p(`1bvdU;rGS!@^=gFaVW!i+3$xeH=HuL zHdr(=Qjn*@oST+@NvdDB?sROG>HwA@K<*%(z;sR0jT_0pT&XX6NYGEjoQ=78V-|>V|Wzo*N&7CM`QH3UcWfg}-mw__!UT9B560c*vg= zWPM~amR^HRJ*uqNXz>71MK(DNy>+rfazP%^+8M;H%u9C$UMhLDpk%1Aai0qBRU#i@ z-yh@-P3A>l+&`y+w=$(|9KzF)qwcy+5f$9OO5p|KS7w)3tA#ohu>r>nC>>r=Esvi5 zac7OxSXuf-4c^)bnHFyj1jA_lll_TZ<#t>gB;yDftK*{8Wv2{dybo}(&+t$FSzm{; z!h<~6Mdi6f1b>iA#KN<5lC0y-y?dwpPE%vvfC86ckMvKS=XPULmgLLFxu?r|ia0dG z9DAZ-2}xl068Z{Xmvt}AvSWF>T_+pAE>)>Z9dA1y!8H;2t*3Z>+(h0R@2+@v(dtKw z;>B+BKK|`l$B@xmrq_FYd}7m=zd0^{{$ol|d&)#MU2C*(Vye~IE?#_+GeB?MTXT;G z1Po1R*uG7SN_##9gTVe|#Bl0QN0$__ki_)jPG|6QzQ5 zH-&={0(WRs?N(mFxPDTMiui(ak^LCc1NL6KuE&08H3e#yxUq1;0p z>|*g_VAQBFWBR1>^q_yV!yVjZUkVTV7GRZHmS^oLNPB7Pl~ z3Zsy7y+j)WVW?RY@BinV84glrVl+M)QXvow zJ3;~zn?%)68U4)xC!Tomn`J@!0+1ZF_bPey`gKlmLDs&kyqY?y00sMR+cs5MDRF=5 zSBgTM+V^{R@Z%THjcXonV=pg*yJ1crV|c}OE`67j`={5B+liIR5LuZlpBn3~{GL}$ z?Je>h2o45hM?d-k6d?X{$blODeDorfKocIrLL$2t6i$kvBo&<$b+Mrxa6dC_<3UvF zYNbq)o`=8Cxbqj{uE7H|C|+iPCk-r$pw6Ue?A3L`$u@7@2)pQD`NuETN&P===B$`P zKIR1agc=fe;aytby}$`P+-e_^Nl`Bkb##1MH#6(!h7FIg%ap;N)KX8HvzoxXNTD^8 zW|YT#a6F2Wy8)&FgWH$uzvEwzs?6uIx2RSrFIDFGq_hw4tdr(l<_OX+2s&9v*8fv4mF_po2Q?3kut_-^7eDp>H_w6K z?$Y+wDQ8y9D9GxfKT=ko7$wLvd6@gjAZa1aIl9cJno*-hr69uqWE%TIP74>(3!sLv z!HO%Y;G#*L`EB7$n5eDaAga-17SGEon%@dkDU7EfTh6?D$BrCvMk(x)5^}e|qp2#3 zW=~KLm@?e5lX9oaqN8q+W@v6beeu)Gg^lGA{bOh!>Ita}cmHHI_!F~GwI~>WN+o8m z@MU-Jo4;A9nmHpkrofx4ewT7=8$n0Yom$!9DFbaH8RxOjgQ>t;pJ(;V2c)KH$I5%51KSYKlbN$0QH2ZsFg%#$1l78*=QJE9{~` zo^1yYrY0O1H$LJIn?&lzXQ6KG&0Rg#nMXAg{#B*z>R-F^u`2pPtm5p{m*?ZGCe5L5 zUG8FHwDta)r5o(8FIOjiec2uGwsI*zT)FuC5LU1&1ve}kz3uBKSx@=tV39kU12fD3JaM9Qu4gVNDdwkw`H}->* z7f5zX27{o@x6AbeT@YD&gU{v1Y~XqVv&PVRA5T1xfH3$7Ktwog+oUVXr&1LVGZ4j? z&(hrjvWbtk`PXNaCfTa2a+r4%p7r%rpO7u))J3w+VOG^S1O`M>qpZ~e3A&Vg{UR0J z=VC8tAqUX<%KpS3>_;ASXQfsD9fbD@fY(!3@}YUDHLP{o54>#cwzYm@c*7WR+j$Pm zVoBVcWvfT5WJ%z1MV(!?By{O~E_ZNf+vsm8HzR)t9QavKT23jeOh~BsFR%Q07iXZR zgRw!ZZEDgV;3=Ls*R(a|X=~}pk&^g7eeDUyOrbkMy0Ee{;{!nFFtjL@a~O?djUUb) zI_<~MXrQh87^eycpFhIl4ic=Vjya7fFU%9-6| z_}aBm?w&Dd(VtIhf!W~LwVv)J{aeVJQ6)s*FnO*=gfuIGTCeMR!JwFJ$yk&bp53%g zPW~%D_BWeTv+e?6MOrPOGyJxUN(53NJW(93@Dx_1l&2UMgk5mWFBk?QA$|;LVjzfv zO8Bt(xFLlQx#rkdHzRxxsGi52VNKYPpl`_-q{9t}{=&J-#(MYL6d zxzBl;qm#4($#o%XIFegAes?{!DeiB_6))>cDT07A1=8(69cYhP|`AZeU zluzHn0%2IJcc~Y)+x8lM+8>cvt97f7?p3)6KI)Dp!)g_|8~pvcr248S<3M3%oOwJr zpbtK8^c5mfYk<`g+>Jvj7-ci9#BIZ6L{p)K*wWnekmZpN<3ay9ZfC|NrQmYFz^5|r z#Y09ZAVxAAL}%J$BQgK(4ZqK}-8}mo9Un$@dG1E+q;q0Z8XU~yjMvCW7b$Pw>}m(J zDaQi5T(r315Fm$Minv?B%1wZEqnwK{MPzFGJr>04_9~j8A5_(^ zb0C5J^-6hO+a|ZGL0_60$se#y%3Uh9@fp3HRc7(C8%d@<_(fhn&w*3Mugc5Ijl1!D zVVN=4bL!o-A=}!~HL`DrU=VrJ&&GKRe=%hH*7EWN0j{IH?+BA`O`4ry4cn(>pm!DIl zRqbc-#hS539gw)|l{{F-j0!^KaCBY>V}x`l@eTgykP_GEcrhwYpc+~APiaY3DQP{` zQ`r#4QGIo>&>R;92ra8b*5t9WSd~|Tt^izIq-s(M(UtxS;>*OuiKApg)vVwU5DftR zS3~1IWi`z2c4lYq_C0E{7_Wk~ue$jSj_sIJxeOK$yw>+yJc;NJV|qtRlWP^cbW(TQ z$3ySCdst7$vKh0gXeoe2AB~U(4Aq(ilHuG4J|+-Jlx42II{57!uvd^*@&%VPn33aWED+!v~(z)5Ak~8EO!CBL%vMyp{9S)Qk)5o zf!`@9X^^r$T8xmrgLS)S*RHx+Zu}_hbGuWrV+8fO_I#nFOuKs+>c+nSkS9#`|&S` zO!lpDGK%dVp_TY-!Bx3J7`X=KuV52(=SiPTA0!D7L?Q|`Uol;jv?|RqLC5*8EaTZa z)l@GOKUUG6=Vuk3M`hGyNw>FP`>uRW7Nj%i_Y(=h9~GQ(B-Wu0LypgD`q@XTl6ao`M;rGN6U#Or67RJWB8B ztA)2IzGC{dLO{!5>xHPH*@nwNfPuEaq~Wym?62?Cyc{13Fy zu4xgKbG9MnAv;HXKmFd_yQ#vZOno1nmqZ?cP6~~dErW(mX{@p;)o&ht`pu)DC~|AD z>>Q-^%D2hEpy=@M#HNhuYf^7qO_~>H>Hw4F@3Ewa`w|tqXdu>lmt3?3RE-j6Fd>17 zd3h2!=8W~iMJ(6HKKhXx*3QqHfr_YCj)l6)Kg)vuFZv z7!Cn8&Pri#3Pz3A-XjW(LGMP9)GL0$6F0#-TC@^gfyg+c+^jlqr_JGgT^^o1>U&eA zX)o>>UVZTQ;J!Gs>UI#=e9U^ERxE@?leaCXv5@a^k<1qaQ%^?|a`~0$`30?6w5Ulp zX`%z)F#-s+`Cckxi(<-cv4&vNO#7iiv&+y21E)mfJdI1t>#TYbt|vWSIl9ROG+X^Y zBpLsLlDuqzLLpjHSf%0-7@cjWng?UJ?g6ry9j{=@_)IIER!@u zep7;@C0Tgh!;^n%Tya%Xa%nme;yXErZu&c;ns*Q2^mwBm z+o2b8Nc4)w>K}eEk0wg0O)CBL#k~T(M2SH@S`aYklY*Xj=aa=-`Cl!@PJ#=HO9~Rv zKULF-!VDS;lZ4q|oDE8VX^;sSJkqm63J4Q2fX>TG>_GU49{mrtlc( zU;GMT2gHs-Gcx36t;#@v$0*X)XpxaP6AGmO>C2Blcwn=l{~>AX)*hPiSk4hW1O^I5 za=_xi-dt}VlkT;CBJ5!ZctKB{>-1<6hi=CgnM?W-uW-DF^%b|GPxFMV1ELM_0_YW& zEcK?y#jVaj6FYN|cj{U_Yuj?_BLilNtUM=oa{W)p6{oR z*!PomP#iuxk7d_Nw`lt2O;SwW6K=h90`N{Ydul{F6++FLg70)|2sj~EO07fRw9!hM&w?&VZz zABu{(=lU=YWZ^U2->&1FW*-OwWo|wmNsNSMWGANA>MKH>+{eenS6GrzA`Q6&z9|8l z=QDp_V^$2$;XU>Z#&I0Qn|rLUeJR#z2Y4k!Kr4^yTsNEC|{l{+}JqP=vT ze=P4EzpYSGTxAQ9J3o<3u!NGr9IMRmaoyuTJ#qz6*d_K0x${Wb{R`Qr{CdsYsEBrZ zY24*>r*|0qpIwDw?-0~Bpw}%06sPt*^QfqEbBYuxF!kvTvUIRzyS7kdhyBm3jLsfu zBX}W?cv|=J%flpjnMey0yHmsLa^u`yZ)JG5TCB2(R8rFR(bOoRAeo{iN1u0pNzKUo zy94a-V2$G!mBk2cd(Gt$8kPOrvRI5KxF6_@;nUtPH-5cadn0!-4d=zyE2HN8G4yEr zZ+N)F%>O8}oHOp#AM1Q~&rVThX%F;yWNh>LNL+mOOF5i~e6?zd_*Mk70-=SJ_gPDs zGXN>fsPU!PGLnG_CQgse(Rql8QK?5IRjT*wtsG^x)1#}ZK9?Y_2SDeW2IEamB^{@6 z1jhuP#D5!6`~Yk3nmC=~ah}I)PxiyGr3oD8id;t@uUQDYD;83*6;V=#t+HU*Nc3Gp z$K}6OaFNFXiza)!SHbzL&#m)=a$MPHGZ8r&5#XBM6wKh;^ykhAUE}V zDj=cR5aKNdcNs^?Fn0d@s8B|`8F_tOu6uL;3dsex-^B;SfS>pcSc!IvS4s#lYY|Ai zX_+XTb_0LiiGb^f0H_S$%Y$}ozc}loyr~Gc4}T2-U%#pug)xql0nQ0A5$RHY|FJ4L zVOy*QkfaXBcKxz>^67bg*W0|S>2gUFR!tUE3w-C+cW&YR-ZwY8WkkZ1@MRXc z_?zIukQg%O1tl*Q)T^Y@T}to_0QL+`sbmoU{&H23>{ff>@Bb?ekbrg4l_8r=K7N6} zZ-?`u)ac}3E)8bHA4pEx)J2|oiI8=P2`4F}1v`1$_4em(t0PI=WG)`cRtS&*TQ z%KP|*)^d()84u^OWrQeO21?kp?8G3Sq7me^QtY`Cr-3ANgzT1IxcG3<-ZqI|#bO*X zZt}w+f!BB+0#V_&+Al(2_UV-AygvZaZcPVzj>k}NnB4Hb4|BWpQi|rlmMDD0ZjAG_ zjmP$r)Y zieZj~cpTxW#`ap4;=xUSvQ8-h>`~$ij~*gHRJ2}SEGk{JrI#0YLQk+6b^*RmmHdUJ z#bF5F5QtZF^)07T5B)zB!A=F&m9;S09VUYZuw~5bk7sy-gP{sL8XKn(M!surcJGakg?lDD95vpeeK)i` z?_T&~3ibM0QGQciTJZ=|9|b=D8E+lyApvc!;*v&LL>WoYq}!GYQIB)Q5R*@gz#0$e zVv)1SOxjm>RPXn5l3Z|H;7nd>r8J&i@&yt9c3|?_v6PPcZr3*`NGsS%iR85{@YfUn zcitvS;DdeLhDt)4wD~lzrgXVU0}B3ND16kvij7_aLclMN&>|H)4BFmUOMaXkWv&o)nz7KlHvCZe-^|qe zu%y9=qUIUX36T$Ex{zUqfvf#qN$NTuOa$$o^L=a4WQD;M3>m=-yQ~FRm(->UE8K*o z-w8p~1qrd<-3FMK17%CQB8Lo5M!OLs|Bdv$F@LtLhrIhSq8{hz6bVZ)(QTb1PRkCV zD`nF0*u^wC4UW$F1=cVI+?V8Mzd~{N@I3Zi-U#=AfV+dj6g? zUw*Io4_HEApj?nOIk^d^C?OQBJ9qsTRNolGd^u{t-^QVgj9@diwIg($qRvLLHHzqN>Qzn zDiKAck)Dw=3lbb$E~JDKl7xke1}iLy)xk}#D1y&Cvd)}@FrB>JTeD96N4+D`>Ka$d zH?zg_!Njv@vAp%migg_q|Ie+?#3fmGZC^o~$SxxRK)@8K#q)#iAR)n`nDJGSwfIzg z#sez(5TONd8exdUiR4YLwTfT>$-o*Mi>4+pW zM^C;4wtVWg{|C7QMVxU)0xn>m=0CouoeuzzBiGTPd2#1ss|?mn0g>*=Ga@^B2S8`rPKMCkt=47b*VpP zS)Zu6sbPCCaz}G+`+C4rjOAcIcmHRV3&ST;xC@vO3Em2_GNlw}l@h-jBBe>|=%Q2^iG3bMqh{04cR)W`tg>pUcgNc12qMv;IFm!2I_VK2KP&w+AY@t}(nE{@M)2^0Gb z4>Q1a^V*X{QTNN4~_PQ==*S#_wA%4{%9MXU(vMH$}> z{j4xCm|@Bg^bcaKkhG)G2dnru=G+idPgK%jN9THVkJgTS5qrtDkS)cmYvcPSj#~~- z7^Uxj&i*%JwR(ycKV-t^=Z!CvGh3C8tGZpxs2Dbt$$DjWP-q{xeMPm#rgEH3MWV`~ z*Lv%MgM&qNLhLnFjD0~-Ab66WsrNwtZWYwF@VEM96B`_Njy?fjG??SJgXj}6RS+DJ zTdO~7DPCW6hi}++;rk(}h3_X+F1KO@gY|OhAmb6ChA$mR|0<9x0DRxwO=NBOcnIN6 z|7EDFSlvMA-idX8GFu&kf{ki)ot9%nvfOA?t;Wg8*4aKnzeQa4{yOYkO?7**`-58M zJdHFHM>7(rfzd9{BpEiZv?MR&scc5@%Qx6isXE7yZU|CQhJbLea1;$-Xu!XDx?gvv zNGYlAj&EApfIWLlblV3~i(w2)&3NREocnudJv|k4-A>;P+Gws1gblFEOFpKE`^3`Q}?{}Gdfcf|eY*24p#r${KgBCfJ|JOgeJJtFfyKOwvRbjzA z)amp&#bj*!Rk4fe`0z516-B76Df8uU&Ra7kObg;Lph@~3qX|$Z&VcQ;)LDEs zaD5#$5LpCgA6@4cNqRSu?F)wwYv0B8S3cGI85F4EqxKb7bO4)6lV7R^rZqA3nnX=r zJ(ZqIIK#ddelpw7nN_SEm)J}9bj9nY?Ch5@ggg?wo?y4}K)RX*fLBE9+`QDk`glHi zY2lR8+nOQ#wJinJ~gda$uo-ohT$pJjog>wm`TIvB`X>s1XW+JrcgheBmi zV40G`U?bj3EiF-y$9Mk6&J^%y4$cHPJ7XQzPgYug%DxG8LB>o(zW7}^EP%2kTk8tt zC0eZ|Iu9xJ5kBgHLTWhamSIY|jI5}M_->0w;Bnk>?ruv{&vqlW96lGNEC=Tdc87(3 zDvk-2CT?>2?U?qaU$;+IIjbF=pKG>jmWNO8pA1SGiTc3L1IE^dnnvtwB$7*rqC5PS zV!Y$pNA9RFn~gwF1*9u}g>2L(!LLg-Q)-$O*Qr(!S3GCw;LdS>n?V-}{pkT>i2zqeB`M*g+wB?$d=~mCDA{IOw(Gm-xr&TyBS|Ft;O)Sn` zAZ2VAb1{Pfy~a2EX-uXNTyr0uJf0Lm5|`|MZQHlUCuCnFMl-*6p<=xdw{T8*jxY1T z|2I+BV`uJX4i<4sqdVAm#gL!!sVHe78t^?Qtwdvjh)vC`>|;<+lvV1;IT7SRWG)8I z7&2U65THb|7BtL5%@|HnNmWqdPl3t$$Q}n~jl+JA(zP>^h8p8>S?%!|7?>UF=>G2+L>} zCcGCiCNxl^(HtxBP-#OT$ds()r^glX2Sxt(`48iqng6gZP0^AA(+IJl54((5I*{86 zE{38k?BxuzE6=FYq}e6W>C)8WkFFm%dUOs*I*5F6&mDB&8L8PvYYg6E74h)N8K_5w2E(oozk-0D_@joE*y)S|1POZ4d zHyDK!q$kuUAy}A}UWui^Bzm~)Ut=RV0(zCXwhG;gZ`Fp{^AkP$xM+rpa}>ohO`@R` z>!oC*xj2-5l6ewz&ysByk&*HviU%?=MQP1T_}&F#aAjb3#BDkHYgtQHra1mZ-o4D^ zpmf6N)9$yR(;mBT)RGVLk3x_bPHkFey5Wh*tI_)eicl~FAxvSyHc7UZoKi9Fm-!c61_1yl7}?2xqWzCZ?yb0m4Vw5kFn~B%8jz5u zG~TGRoFouGwamrqtE9+KuEvsMe2c0R73fj$I^`^cm3h6)yh1*L<{TkpJt${He4$`v z5&^+R*IzIWRAJt4U(<>UHG+JH0t6eQXay>4kji)bCF)Zl_Xc+qQ$KobP~PzktC_7v z#1p8@N_uMg^@=H$7KhzxFM0mHZNKDHXwfy;LASBxtIRTOS$hl;e8Y!vaubg{0$CiCoQ8g@K$XMs4}+#b|z;`gxUO>Lu^G#(PFZ4XBsxh2Wus3kwq0Jhr|%*2M?sZ7V$D1hTNZ%9csJS zV0I8=cR;`b?XV$3#Azj1G1y`MLr*xS^fcZHu!?<0&h*v5B((Y3aE`ywFL>5+ZL+ zVN?nv3Pzxcl~WQ)c?d-|BZJ}ex$B4Sh+byepH(P74ir+brbbLSRhHe8c9RsqVGTEq zEm+pAqE$dfo4ao^#%EW)UuIehym$QP9Sbk(19{Ya<`;e~y!}uaP;^kYy4whsR}qh^ zc1SGLcIJe*t0@Atyw&oJ%Bm8Fb_et%k`n4A?gX+QP^ug&^kAnG(im%JsHq+sejS?7 zAlZrwp+6v!-+%F}8QJ=|y$|3P*y6cKQORe|BrNO3NHcAQYUV;4bA-1IXkhA%?!ntu60(8%54~Bz)aKo&M&a1XA~q&% z5-O|JMdEp5$%(R!7UqOxoV@IE8o!m^4d~BuBF{WMN%1fh(XUKy=(qHaW*f-@Vtfa_ zwPc``Hn2=c?6h(!-ZD68>9uy@DeIt+O#Kr8HeA?jt5jSa;>E``V$KiMJF6+}@v1A~ z-9KJHw3F;p4OK?fu3s{hB+HA-9&R4~%N~6tp=s>Dc%*y&(sQENy65qYhFq9c5ByU>b8J(3$sA0=XPP8B3~s2BO8CkxIFA5M$w%f>_7a#%0z~C_Q96rjJ*r{dpU5*&)!&UOfi9Jk zY<3yDvKaJDKfEw6IcT+rFG(J)YE#aASaOe;uYXORxlb9iP+|LyT9Khm>&%=T-$ouH zRXgL>bw>Bn3m9~RhO|(d{?W;9=JwmQRSQkYF!p*dV8RLOrQyN-lj+=;wRcEK6BE8| z?rt&**U-|4#hLga+eD-WvK%ILOSfGWPotrru8!o7%DgosL<3XP%WH{wU{H&Mq39!H z+jyFjpVJ!EumVKPG$_b9@w~4}lR#x9!E|vPII$lE?iL9sy`{1gxZvMnGJY)o8!;Ia ziCEWmR@W$>YJCrMGGgt3wjUf%4Jh^E^a{VKVyCgailb3InyB)L3*u)|2D;y;AjyY{ z_0@8<-*84VV$@|Y8wItKI|BudpJM?KMQ(laME0!s(`-rT(^}inSxx{5Kh;gb0>apq zkD>?q7mNzieyI6!J+fi2A?Yj@F6h9|O>Ih|j|7QGWM2RwK(52+=)Nt;LZ(kV07{9h zT2E$#(M-mI|0{!fMCYe8by7OmfN=@rL!@>Me^-1|A)^n5Q5RQ*n8$p!3mda2CFX3o zcSd3>P&ij^3GBQ+_u4dvV{n9^s=-);XN>>A(Qx;*GzA10{R5 z^5;NDeJ+NL2!jvyF>d{9roSe)dMlX1e2Ry(CGY={c^Y)gQaU7r$HYi^^8gv7rNZ%{ zS%H@wqtkI=>z%$inszrITytSYWo>``Lilg=R>wxw9GVsf!7l$>Rz4;J%l>*rRsX{P zfdXQ)(8sx-`HdPn=UX1+n2MS}L}2o_?9^wOZHUZJX1q|kFxyc1B%aM7ZOQrE8(M%) z)P?p!Q10!23|MK8lBJK%>z`c#;ymNuS~Waq>g`AiQ#}>NT?s&Vm^(LaeAWQxk2w;! z%xoP4|0(Iy|F7|ts%O>&5nvw~te1FlqEH-BHsH*$0)o^Rl!(bbm!VQy2|$4P=Sof2 z)?`pc_xSvsLl6zkkjSvHGX4XJegx%@!~PB@IYqj@?L8j#|11Hv!Gx4!)_mCyQxg`<6;MR2NC0DT)S=B(_HY)LW#8<7YthUYz- z$gDsip@k!6dbm|7P`hZFW+fdew)38~4ak}60`y-~CleNV@WN0Ij>&cRS;v~C5$0m) zB^ou9#BnvgEIlHHB0o93CTiy=4T!UtfMi*qWBBjEK+xj|h?$cB7=cFn0bqn7O}2u` z)KkBPC6YC#_AV_EVIj6PgHQW$jma!Gnd%^E*cfzDy{wdl@_xL)^+0Ah{C&_M##(3!z$24gG+fo&5Mjab+<%Q5sFAtQa{ks&a`f&X#m0~azd=7X#0BI zl^jpV99pfvjt6+$`j6)wn>KfpDiCQ-7;lrBn!QB|aJ}t$PI1!ezhA#*`0o^shMg!a z=E|rrjIp$?BfyxR(QlQvQekh+Dh(t6Di@#=+ObfI88w_wC1ZHR3q*P&E1;l=2PVA-zb=X z$Jq7(x-P0i3H&j&-46F1%9VSv?LP+hz>?09RkSsay<1_fsKLn^!kN)4vf-yRk2&lS_KW;|36?6!niW0lzn)Ws{zJ@`KE1yd< zit^s5@Ac0rD7D*ZcXUmy7G^e;cerhBw}0;Ri^eynRvX>4|9}(6X6pc9@*>>sO@95v z$jvL|^}Qtp2j@S|9p2;8{?j9_neW==d*i;($Z~CJRT(~ZXQ#JW&6|xozx~F})0c~4 z(@s76GVuKAfhVoi@d3sC*8-_>_Ul11*Vs61kkyU1Z)$>@vuCyPOUaHHTl9SYmFGSB z8LI*-SM?nv$^;OxwuxC&T%VA}!IhVUkchO9Rd>rd-(#1jcTm*QEY5q=@udicM(KaD zqyC7toP@?tj?m)wJ@3$3xFB_H7g9diL0s%2Ry^ArPmI*GFHKB+ep2wUNt2qqOWNOa zFocR>&)No$1)G%tcz(UfR1zl`J|w^0)vCYLD|9BD&0|pF?I|S>u9|Zz)!oZL(=$j} z`=_nIRY5B(`e$lfXj&-9R5u5HFwp%RW(*l?&%DircWH2wO5P^z-oAb6v|g&nRLnK` zj6WKt5%bj!qsq;$h!u=I%s~eX;6g4*yEbfJb9nMay+^_1UhrwCsGKMx;E?9nDD2wu z{+z8MQpf(^O50%55bDAQz(bZJD;m~pc_lfn^UK217e2-9Q@uYM*WP@NWBT$5T^gJU zRu*OQj0v01&wj7`S!L+cddR@sJJE*yDyj?{+uBSwIy^XKN;NOAFDVN27xIgL;*QMu zeR|ecJ)3`KK~@?X1vTKYxu!Q`s31hsbIK2<5K5)`YV9I||1nyUM;5^DT)A=Z=^m0)i(jzr zyNO*Ikb+%Yw`w%-QVin`#?lMe>PJpdPc_f^xhmj2L2v9eA}2h0l;bC?TpcMQyV|nHd&W>TLTdwjvVe#i! zOYdA~mF1N4jr{k1vapNU;@(#(BGuXEo5(+Z*i!ziGP_-H=}VpZh5H-*psE=joD!A3 z@=VWuQ1%$7YkGekY(9@ij0q*`5lE>DME)Xdb67;gW_1DQjhxWT8;M^PA&yvhk=n5> z2d{m@^A&w&TRFF%*R~{5G&-HYz0N=)B}K(9+fLP5IvwE|n20Qew6Nmis3=cOufSk{ z9@T^))^>CeMdF5(d+O47Jx%VZDWSJC22=+Rd%?}OLLhI16kky%toWo*`6zn?;ga4mkMV34hZsuEWFBxoKYT@!hw$8sgDGEh_U+pJ zK7q;VViMtg579ttNT#cCd6!>>ca!X&+OdMAh*hZXZ@gx*GaNK>p0aR!fB%U&11?@U zKE8`s#3*;qQt%s_P2(p6zrex-ton3-NXVwv&3n(7X9{v0s|9aQCfdw$_zxwqz)Z%-}u|!@ONrfx7GX>z6utG zcM2RdB_(T3GnL=Y#tA+f!^@A{I_J^8MYguKGJA>l*maU5FRZQh^W|B$RLNJ-Y>noX zgwtX2M8l{l7&_>|Qvj`Jo=Q#n`!U+_p&(JX0*9VmiZuk%eF$x3#J+966*6-HJxkt0 z__WB9nti6u4TB7y)NPzOy;0i#YTlg@Mj3gzP8jVrHWqHLbHQtgw|tB}qUh`zoB3)9$DeN}_U?`Ovmw z1{}rBvc{G2_yIJEq>IYteTgolweixk%Rd}OzWK0+f!TNu4-fN1DtyjjNQ>SbYv0+4 zd0M33q6KFaU)JH&mrQHPGqSO@wIthzFiO|d3y$3z59B$FDG%+XN7WxJ`}iw1&$8u% zujYf-Ge`97)CR070f$Cp4Oq6D7tjZPFdu2{?9OthMnKew=*_$dG>Z&{QvR5vUihJC8B|-|B>$Zyx@7-$}Ovw_m zn>@Lxbsp1xKm%Kcr#f_R&gU7Qcj4n!s*TIBf~Ej$&vXi|_)1wet^JZ-pI?AvWPj75 z-;{N;5BiGLb2c-x)XD9aCS7KfQ~4xJ{5l8uCXp3kZWqP`4D=unZ_oDa$v1BLq#Yv< z8OkZ0+T-1dJ4{M81peh3hkW>5*)M&H!lI(m7I=O1Uq7T0g|1r2+9%tnDiK}ehfgW) z^jAWOTwdV6y2M?N92B%2S4m0RX@?qoTt4xPL^e!39g((n4TerYEXnP%7`e8PDdZ)u zOTPvHSUrCPb~~Mt*AM-+N@e*Q;kBJRmmM?|zR)7N@vp_?#iYwKs0K8Tdk^CW&D^Z9 zMoMgM?E=^7WDuoIT=3yeB~?oCx4o_u&%S7%+O=~TNL3t+yfFJ+?$6@)CBSk2?kXyu zkk$a3Fqd2R`?p%vr}PcPwH=}?N(#B7+wwSms5r((>h|y2xif~X5FR4!$PBaF3nOkE z%l)#a=D=F2cfE|d+FyCx@2)=m#q5+*&v%E)DHxl|Y8!=JUeaE5M7tSF>#3ZV)(MU% zJe4rQ(yMN3RA*F03|H+jX_Td9DkW`jyMU%Dhj2IuE?Spqwgw}_fksY@uSZeIDhPea zGOQgdG5Mdbo9@k=AH8qe{RiYTqeEaslgY%%(@iJ+9Tj@gA@V;*Otbp+|IDLV>i--4 zAN!88xLQjRYGMAMdvqqRZ*GfnhQEuNLy%T>se>9Iv%??RVWtl^WoY-=n91yMAH%tu@^YU{Lx?=Vi6Y(yP|kMx;a}|q z(3j9jR&Q+`cSA&WEYLn3e@%A3;wW@#lLvQt) zZuPL_hISL~26wlqsM>zZWl*Poe%$ky%cFgZyEfP)89cPrT_)WoIEjLbJ$ssC57os1 z)L3&k(S@?5Y&+{kLLg=ph|N=u?f#aWJ$mR=cKj=`B2Y)a1$yrJ*T+76fK2h~+Y|Cq z@V2Pf>(q49?Wry@@mR_Ksa6^vRZ`HkQ>Tq1vJi=iF_~Wy&cF(hpk1x#`(W9oSMwj9 z90CF%W$J`RRh*~FSv|(P&{jfpD2o}VvEu4-urgM~63#dbWWUnZZNEFKcgL>W92M#s z@#|xO?LvOF zeMWDB8DyvG#D7FT+-(xOB`Az$0=kdMc-6XOjBbRB&(wdIOVekC4{RCiu) zb#wM`Xp*B*(AoFv{XRvVRxW7Z!}F8ur!W0M<($@SPiluhQ}esO{6uu7+zm+Qv!}gF zy(l&QjIkge;x=MWu@KVVcWP8^ZBCZw$1OP453P80z~jLFe((igso9j$&l&l9eK)KV zq38oYe7RdkNkjj)&#ifCD!D2RHkH*Vu`Z@f$U&Y}r;kc?cR2fp)Qb{_f`V#Kdd`+c zNl8i4PXv?H@+SLOz^v&r1Iy8tNrI4wk;PdFv^jR%xM59X91emyX$qvUC9d^zabxA9 z?iwFyFXL%}b*iLOiOLLky2^W3XQ&+FxQ%KyrP?1qiuj9gF2@ck%aKZz5*TO(0?Z7p zBj(OksEkAzEW{%ogbj@zBV*#Ek$9YU1B`5!E%>~oAl{qTd*;ZG$|WFaWP}ZCnwO$L zxt#0dl~|1f>qX+eojY5753U3U>XY$Wj!VNM@Bq?i9Zr)NCVyjIS5*tN`j^8CVjTec zoNfGGJx5E2%+~#CVW*vjAg27-mZA`qk9{uto3yvIS*}}TLI4+GPpXSiONIOFKM;>K zcNEAuIE|@h@kVs2X&Du?2QppmKZ*L_up-oANW!z#Q&fSe+GjCy=BF=z81W=nSv1eA zjfyAFQx0^qQ%gf53NKZn>d0O^Vaql5#RrB`YB*W#eD~={Mz%P2;yDwpg(^k)5@qQ8~pks(H)~kZF*B_^{w0jWo^necY$-hV148mwL0*s6`Nqwmu1I4@|wrmWKK zp@On=ol63u@Q~?tzg4U={gn3T{0$CEPuR^g3aXT-G7I0;NS9f#p;2SgP0A%)-h`06 zYNrOvy)X(WMAPTHtwm{6Kp@iG*O`|YsX5r&2?-C8(;u7qs=NU&T8!fw4O>;Yyo>1`ej?` z=bGrsdp}`kisNUePitk^7B7-`v_RVH=jFtG2}|><$0d}CmRfzII?2D!7-stP=~!Q-InAd4W9t4S zFExB0ZrmBu4u=R141xsI;%>eQ5X}Rn{yF1vD(6_vMllu1h!%rUG-yOU{?VhS8wyp5F?!L^!Z3jG3&$CF$TB(SWeL^HW8Z%hIVM4SBaPmy0b~yA%-99l9}sOk5u*QN?86 zz*>~&FG=S>3Dv%&k-SY-N2_MZgt8`sxa%10sQKdeO_@4z_W6~>Wu!XPQ+GT$z6(SW z#qIN@%SzulSXfl6*@G6R;x2Y;$-^UINHF_jRvqf~^rB$KAvHowD!4yC$N%6w6?2d5 zW$fnQkH#E%sV1+`-ALMwWUv#YB9T^WjwqipX0)tKZvPU%jD>8x(+&DAK7uI9lHl96 zt|=i~NHz?kZeHUmXq~JRiys3!jv!qPbH|n9xOd$Hb-ZJl3k`NE+*i+~pWaTOT9<+n zfU{lOtHFy+S_gC|)~8p;%Dk$>YG_J{w&%|8%g@XUKy^`;gGn)nPvY4MEQqUhvz9#a;C%Mk0uQmY2 zsT}H)7CLXUW!1R0&M%Ml`ZBoYz-DH*UmqXIzfjNHeD|&Jo$PF`OHV1M5j#i@N6*9B zs=y&s1jB1Lv|ePup29@B*|0l0S@c@o;ARjWM?1@%qs%mvi5h)SL)##a8lA zf^1un?JBS;ms-NTh_PsOjWL8VPFE^QRV(sY27_6QtP;}_MUslrvdYLxT4pFD`***dFXz0^ z=l1>oyPezn*6qAcdX2~Pab4GaU3TbnX}M{(7ZQbl!Bd#tH=l`R2e2}kb~iV9!qj=j zx@;Fo58&1VUS|B;)wne`S_~SnOB8AZKn?9K*XyrGxAf+jr)y~7vn^W)&oaf2v?11Q znb;rGF=HviVS$r_xI2%8Lw`fo0D9-oN8j*GSE4zgqLVx1N&yxz*RNu4Jw86}EYt>C zxJ6J({cfoKRF!09YvPt^R^vQVMh^oDLeQ6ptNoQt7N`Rji(&+=5ptImwL9!lNeN#oWSDK6lZnC_N(Px@hFH4pA_{Hs2m|XFds+#7OA^{f5SBqy&iFvtIODnL2md3-7fWQMOG46;Y)Esb#vwaa`~0;2MlTD@HTpnx-^}v&1N0^|=-RlMnv;c>vHuP=$3lS1M+H?Th>8)aqe8a= zI0rXiZvlC z3SDiH21G`!;=4;OW4DL3Iqm>!El;e}I-#A5j*7pzF!8bKx`rdKcKELafkVF}{zgDh zK?j3^vtqvHXq%+=845hqp3nN&^ycMx!*rQ)G$^9@*aj(A0o5Um22^mcYl+`6N5Nzp zVzNo4jp-l}Y3g<Vo%00| z5fKQyjfqhHJav*u1_@vrat)!r#g3Ddo^aH8YaB?3#qGWvV_tXWDm+p2k|YQ?rkEf~ zymH#b;;=;3BKRSj5elg&EdGw+fc4S;98Ih z9^Z5kuIa}-GbT@Tqkt<21^fp|&ImsE3NW)yEce+31I`$~&QNatkUYNYE5QlY==jTt zBK!ICr@tFOWE!;qUsLnBhaaO0^gGylk5L$(D}Aj1aQ^8>auL)`-#S#8{&@DzlO+LF z4S)3ab)Nf4nal4#?U>Knv8I48e$z`y%Od8Zi+Of?zVF^}KIU0sWh(4ea6J~AU5o#) zY~So{7pi8vKCF@Cul<+$f})x}1B#9XP^tWS-b^1#L}yG6#NR5_U%Vwo-|g$KXRUU| z-!=F?^Z?)cKi~8@M-2L8Z^W16e1QZS5aJ+q%C3Ho@(Kf_qM}tmm!Q|La0@p8DESD; z2O9lfGOyLj3>Rx0&WTzD?f*gq=oJ8G+YRx>L1e!^_8v?F2okHrv=|YD37t+*(-B8&k;Ft+7M>5vu`zLE7c3WeT2xz&|Ys6t)B$vf}WECP&G2=+t zQ@y73@)KE^qQ&2b+K8v1xalECC{VJ9LO+8%%M&wvRDeihgK%mksFZ6XxdQ&yNt9-M zOpby@_pF&ODT6Fh3dCWLp7hN!$Q0S9sdXGAJ0z-x$s*+ghYq!WdUno@mf-Dq%;LQ> zdEq8prc#)=DGMG0#5**sHhfNSPr~eBOeR>|(U6BX{$?8^t(r*A4%$5dt%4~nYQp*e z33$A~sYvMKJE%Y)6bh%0Eht>%Y$oGPNX^CMiaAv`J31`v(7uII)2Lg8VN#+5n2Yg)(M% zbefXkZs^lc9DEwN>b|=Gbw^Mb+4dC`&Cri>4Cdg|j~M_9ew8-X>0o5O1{3>U-KDJK zw~^=T4*~Gn(qr4ms7;}dXkpYUP(M=0;D*xzx{dW%6;}cQ89?s!ZSA0~B++_@%e|HK ztd=QwAc~3rkW^VUyg`!(#}ZLq(0zV1_sw(?Kd`m#r+D#&i80(ewx6qHCJW|3lzm!L z$4nutirk+OG24_|fjsnH_-NYj-8Iz$Z6tVwS8-aIIJj6w zV76$qr_;r?0i!zK&~>cXpe9Nx{+)61uL8;BN^uQw3?LLGrrn~|C4lStCa^GZEFntK zqL#!ND!#Xw*6-*{Q2YnnKir?;=7;d8nv@mTnL&Z&!pov6=~}QfLqr9|_-KKvK~SoI?6!-jC;t2pu5Ast@TOr~s{``2dkuJj-8B_FC4w zLw#;}Z}QL&DBr~yya|~>68`R>IPe2_FUV0pU00xBi-bMeZVo-ZMx&V))I*5H=UngU zfRgLI3|~AYHv1<4fQB%~yLlB$%9>h$5o!T~r1+rWqSH#6d>t(fjdCn>4cD8*vI-&8 z6$pLeg9mc4_CPxUxR9Wk_uR8MrYLqFpq+u}5Y$r+t@OCh4+8z+QaHGE*Rr_Qri{3j z7TCQ`mXe*}?JFfAAv4^^HVd_dHFPlN1m*u_YIh{x1eVD%Egy@K+S)f4ev}cU4y~8( z5Ly6`hM91r9@r<&I(`&Ew(vOs954CSGhovPFB5|9Sz-olO^Lt=cfenC9Ba`1gKJ|g7;SS4?O(-ELm;rQy}fiW+A zooq;!G@j+O!S4N?Z)c0-rrvB%b;XF+IE66NYzr`KyBmk?{H_(wo1Swbiyc}O|&HKs9r{+WA0XZG7xw7;h(_7G&P-_3nm#lCJ#MJYi zTdrUzfx!h(Bg_lJBAfM4G;WaZ0k0<@eC#kp=!kIu{G!-bp(iX{*l@L^c2?%#$&Elc zBQN58u@byM4&kN(K=ux|@p8610;U4$r2GSQjSC5dsosTK`8O)%k3-+{EfllBbdWetRYJ)+tKBfg-fKRlL4cgR=`xKBC0J}Hb z3&BuU($>YxK@(HYRoYY%?1XEIZ-b*UruLg(q0 zic=Hj0yF}2Kccj8??fz+u{on+m3LkoHdZn;I@qezz{}nx26%)18X4+RA6UUY)vpee?*s zv$wj?VaYC6ED5a=zOhqW6IlvtwYKyOjWVpFNtSs1YRGj_A1d&p=>ttfUk+C4p?Q~L zWNCA9<$6psEnEm_L|*0Ga)fGo4D}6YpNHHFjvpv|2)}b|z}|MytdzJ6>H}AgCq`_B zS4OchZM=IZkdW^;Z{CQM0Acj@y>#(n4EYuzP9YU6Enkp??II4z`C0{jT$EZnho|7w zKIQ%X@Sx!IN#W}z8y~*RtPex5{_6qo+J%%*5W)CT<*o#SS|(Ij5Kjjb+(r&;(> zwuS!OSPc+x;t4C$2p!Uy-%)yKc)?8GjEf6L_?_Ox&`{ot0TXm4Ks@R}@lgCLhH%PI zEqsrM3<66s@fxQZT>kTuAK+iKcuzgBA!;9)wPSujjZ1RgHYg_Ix;#Y?0g2sWw5$~O zwaWFfNlXR=25X=6`w&%8lo4eem6cNc4QNKYK>A75R1|M&Re%p^Bw})e z42|h;fcO>9*ncM?Vj;D$RQ3BXnlp!pA$@U(kgA3WJQ*u9jcij0Q^U7F(VG7pA`I8$ zk6zz+QqR^79}Jm0`%idVT50FbuStC?tmUJk>2>BftAB6S*1PpT+UjI#$6>R~Hl#T3@@W6LWAL#ii-{Bdxd{Hx8hg^Tpc_)C0kNg5B1UlU-YEOs(fR#L)K z6Y5)MUB6pimW-*xSv!)It{|zeX*wPagwgaN4`$;p-(N@^ zgh})rg*9b#cLB-0MLVX*9C}p%A0$USh*k`@73h7OVdXXXOhNBBY&69rp(9l&fL%=d&)Fc^_`vywPt@)ZOU8=Nz3-hY*GInotFdS}3x)C2WFF@V~JQs*ctNwz{ zC>HId7zSQk?M)`oNQ25;z}!n{ix6Fy;iv?w4eauPoLt5GGO(t|8jpM>O5@V!pq2nl zFoY~7)IUGD18JlCPp{{34~g-B zYf)D5vX?elNjPM5FMT^Ex2TuuvvL6(q}3Qci~J+N9N}r3%U3LH%!+sf8M;Y((lQZ| z$l&J+SVl;qEc|4lHY|p7`<_*OT#b`e)@GXLM(rTBE+k9KfEhN70kf_? zaEhkAI5ysvOTl9xuL=>Sd#`;4uT}xrw`gLZ_YBgboW-yTC}0Q*-nan0qCF625=Qzg z#$$8&G z1BK1$+B9N|i}O2}od)B88H$~LcIre1{-a`L(K&BP5Qu~$ooFn*hJL^rTF2v3&85-R zIPL`_V1-8-8b_|3IkV6LiFx>ULYi2gspyD^-9_4)1BR!F}wek z%UNux&h(ElmAB`+md;_rC&wrL7e3+m7e3Jd`Avp7;J1FZ*x&aSCNFRtACF*|!F34D zkw0^-08F%0_gdmtfCOt)D|omTjJlhRblGyMw_ojb%b()@;rM3`gfS>po!}@a!;@os z%@yTqp_1qD{$@Ps%AYaA!me|XmU<{J(Pkv_^dEo}069Hl#$QzkJQ>%ToXvR9>L?d|?G|Gum6cu*>Q|squQqqr#9TphR4ho?kze=o0}?1jq755m z=v+^MbJ$|6&jPnWK^DZMRv#b#MNNzgq7XqP->b@N1ae~q8Q|GK2Xm*o-9dQr+Ci5! zjfP9^r_qHCxfDZHEG{NSrH6e{U}40tQ8Pdg?SI!l;Ce3gz(?;bY&pIPLp2qbqRH(& z6Y_H>SjiC2jULEv2nC8TXXv9}P>YymmJH#XYQ?pkw8h>T85B>LgYgLW0N4XDdHLKb zBk&Z|b>Y;c{WwM6ud-rdN=gMCV8ZVn;0{9nM(eVGs#%E=(KmAV$}5|QQIlV+xI)on zUYG5=>gHD^6G+U8ku=5>^oEl;C@hl;{iR_EFXoY2KGTTrs!U#u{7c*v9T+*teYezx z)+14VS!D2Z@7{)`Nq{6!aoDlCSte$E+6k>%WmqDM+CHv(c6`(17|xe#j4)PmJz)an z)d>s?tX|~s@OoTa(a;HfsAnjx01kv1)l`<;ze@wWQP99M!IHWIsvjnCV`vr9oF0PI zC9e^>;EES7Sy%ys>t-u%06k4jP3WZIFNZu-2Ys`V9U~|4R(I?dUK5mwm@pvruYtOm z+~L~zTbUl{E)-HS6ybZdnn_9H8G?2HQn=T1df3c6qX2S1a}&DKIF=cBv+ONA2X1l- zY+T&QxVNy&gne6lOGn#HKbc6kBIBg;)??!6l-TN5+*hH+&!>V#VOb7949GO8CZxuO zm{Ckv_<7teR(A_+x#v^zRHzq#;DPq8`#`k)F@3>cqsZC0Yvrf2GWQs0CJR$iom97cnSyLlNRc&({K(WrnDx0219<(KES1W4>xre&OWO6w(D?jl`ruDL+n-~jsW-yDHU@x#d;O-435o5kNi z(EHV^R~@|oLNAneNkLWd+dF^ov%pvayy0pUHz0ljvf(7)U)4BX`amQSz0kMZ_@);hbsP7290nc^-!$bGr2bx1T|&?qfJW-eb;*OV+Jt8 z3TX2RJd!Juh~CgqQBm2fHaP-t0um{3AJacQ15ds3EhOg8j^G<%>zf%}JTzx^oa*m> zNBKR#C)oZuP|0A^rcJ7O=oRSV6mTy{MuD_~=8l_85G}}dw48v8&D-)ta}Vt|$M~J( zq<~XKeq#6tYH)3A)dj{*9VvMA#-4DJa-izlV0;x~@6L`n0f}N9!W9K{B9YDqxqv-T zQ3-vFY@U6|7B9QT-l?2RJ=i10>JJ5M#-%j8rvARsN(Mj2^l@e0ObHdxE|h$QCPvhm!=!6zxdK)w33(7x#9k+ z2PnIczIzjk*aq{As=JAh zWup@VvL{2saKLJkL(3f26`gZ7L4GkOc?jF;?x$-oJVuP?aRq*lpD#r)hMi5g5B8GC zCW2~SvZU@QzbG?2@sIFTY0Vm0lhU|nFD(xSvQNdEeUgN%G4+LIwcw8ZV6VRU%^JiF zJaZRTG}v4#t6BWVOdm>XL1qIUMO|9Z!gwfZItL=~7{l2%5=pvO8MrzXd)}-I;9jr> zQI25%{TZv3t16NsK%5l%}od42KJLs5k4Hj#rwR4_obF9xAan{g>EH0*Wdi&o~1 zT7h0G!&2>JN<_59-!Yh+Kez&LBE#$U0*PHAFE3vYV4})m_?g8T^=Vema7T^~M32et z6NN^mD(>X2Cr}7dNW-g@h5j zkR0?X4!W_vNb053E~dY>zgt6;9z#>121|ibfoywN7q?%sbSOYR)DrRK%b%U50?pF) zBJz>MUR4qGl!%X;&?t=4$_KApxMFxrC zW+kHKE~Y4NM)27G8GQNetj^-z#Ke+{L9IOnaV%gch5uffg40Iw3i}Ta7cgKoR_y#T zYHY`nq+W{o0+!tr0Z@&)6Ckc4B~-AT01r?;Fok)7-j(Ku=?DTs8^qr}Y-a61oCl@U zLHq#Zr&vxrtvOQpg~W-Yxs!jVZ?}$VhU|(<}Q>2ghXso{qGDm zrR&j98ay#iN3cA=8dIOJu*!B_d~a~;v}1b6dQLeJ0QVv>V7kD6x zUklcp4<7vuj*a^=T4+7T?vA@=0Lk#A5OAZ~V3UQ7u3NZFo8hkrjm1X`QXHLeqBK-q zh5%}#oflJ)k#Yy5#OToaLzBQyx}k`lXq=RJwEC58Z;`F*uDJ*54nCeuZml=x4a0tm zj=Qy38oZx7li3d&E3kE>XSw*#$`3nuS=#x!=uOMz6U&v-kVgS%7?+8D^KW3_5MqW8 zMcn|5Nbb0$!ftn)CcL3K!_dF&cFqBSJpNV16aU-ui0n3iChy+3aV0q+p^SEmF!#1R z_FT|YidB`kKFBqJED7!`LWIx;C@jGZSN4WhbYoNnoNIFJGu|A*N*Hg-vY#M$3!+%O zIEub90sQUEe;E`%Z9{^F&(x5f9##w|!2+wyAI|T@P(4y1Lfj`T_YoEQB zO1J$xDE9F9Y|9#Zxo_BUo)Pm|dKb-Mx2t_E+xJdj@rP6{9oe_%PDj;q;|xn0q=l-@*jVME4CE}r-^nxmwJsl3A=qhgER8)#M?hKVp-(51f=2rop^U6 zk}z`CV=0oFbQf^xo0u%r(L%-u?mysxeJ`3ZhQ$M3U!>EH+%XbDjHW_vA?1Wg6a){= zQ*94B_rM(k@dquBvz_LxF^t|A7QyN`}MEMDBy|qBgzB6IgrSs}<>ZB$_&~_Ys~m{{%dS zgnEpNVOas~I3sj(00dv!1}h<*aGaBI*H$02%9H^D1tA=M*j8K|rMMda5GqqdQrn)b z7N)s399~C`KANb!y-ZPX+uhtkv0BT2CY|3Ca%=l^;PcuD?N>n@?_~WRcZy`PQvf|n*0Qk;QcZ!#VR98RVROb)A7vcy%0R? z0Qd#Eb%&$?2{A=GPPrY+ zWu&PwMEVAf{Q2yA{dwq8mjUA(nu(l;D{f1 z+;dt*5IhmqR;_-hqlYvM5-&+$ucV+7RP^4$w3T2G1GSJ1$^DD@(O5C8An-(@>hbb4+*BuR^4WM{PS|b5dlUQXq4jtp5HVAyiPD6hrKy zeva|nUGYZhr>-@^OMn-H!Ne`DTH!n%@I)ABxMvM6Mm{Ed(3dN1Jieq@4gTw=P|eN# zRq+4#DIPkY-iyh}MZ65QnhSKZm#&inE~Q-vGxWxtVRMfd@^b-?pc!dl^P918*w=-=TINq4#-&Ei-g zCZ<8$0H|(oyfR=fE9Aa`SHU+J_j3XM`+Jn5OFs3t z%fRU1;4&(hP&`Vw$s*^?^iD+en*-?(PA$VV(%#%NiDiG{VxT?Xzye{MKQmt*%s%c) zx>af73cmdn2ML=xW=~dafahQ$s=bk|-T3%E19G5w2Isc$ z$Q6kzmR#qMb^XFI-%@>|$sDNiYZm-?wd13)GX6B~RB>RBrgKq4@Jq|i)$=Cj+zfvS zmQ-jtOvfFIcXu^&_^9AuQDBlPg0q~urAa_^8|8ZmPHo-+S;6pFWw-{ zbb$^nlR?1NO@#T2lt3Tted`G*OI$Y;WT0|gJM{=S0&vf^&IbwNsTdN~D+qoA4Ri&;;^&S*Sr70=CiYjD*Fc!W(aqv4 zu0t4-NQj-S1d%Yx0A^pS)^T}B^@GyLh=?S*WYO6Mr+gi(O!vHrFW=D%9!nBZvwLQ# z(?|e^e+OrpR)`{1w(Rpt!PaAPI2Z+~c>V*2PhxIV)Oizze#vIHx zL*iBZAoEjxAKGkzj^U&7Z(tZ zaTizXF}N;$>P`)!86K{iv``sC03{&PHc5a4Gpk|eO(2*Ocf;k%!S2fe?ua3ydlSWZ zI%%CqY>%-uFdgKUH!m5xh--m{1>Q(3E_&#H1R4;EIw9{Vc6rno&0X`?vwb@0=6Mxp zbd0 z)Chvcr?h!iHNF+;gRka*LtLYW7$;0`i#nUXeTxX6nz{+)1jRE87Px^CFO&N9=mEO& z#~#s!F*N=>JogkX+pVk0SP7_A*g)&a5!rjD562QB;?Al`6KDdcec1SB_aAq98z{BVdW;RdNaw` zx216SCuL|tkU`Q&LSptk4Sm*nOs!Yo;7%oc+Q_1ilQY5nXNc3jTmxMuX#lBy;U`Xu z5`pXs(>p1N;<}e=2xL!q1$2*RJx7z!_jdd0m+&sIGiuum z*?MKHwR-FChUH06biZ*IUC%ZKqt6JiS3#ark!7) zl~VASIv&Q4BC*S2PF}l-+Debc%zN_C5|I+2Dd6mc%vC^}g5f@0^EWc*vez1&hx-%Xa_9jv{z0 zB%aFI)_-U_rL&}#ME7axMq1VF&y)FgwA++tIf$hJ?znmsdVi2vUdFzDz7DNurSe#9 zyJ{VVWf=JY2spZH=O-n->UhtErxacsrX)6u2;8G4$#J4d4%RE!K%)hAhr4flst(L# zy@iDX1*Od&hPv;)q+S4J)fYq%v}P$~JR`NzU$ev03K^Aj@nC6%mcZRd768mQ-%d~To_ZVe0%VFy>bpS7Q|r#d zqnJam8hvCGumNXo8@4Tb#PzA8OR^_VFR}q8z#xUkSSI+i#^N>OS=Bpz4H3YOa~paV4ndI zFK`y`yPr&3t617y_J-3cg=>FpP5!lhsOu!|Sdx^zw3?Ut5WJ099W#ykF{Ki`W ztN;DF$THmSyD{ch;Qc?%!lz}8CW^e8?eY|d7vT?mQWBK>xa-4kL98tBKiRzwU>0^Z z(YniI&ym3&j~DHM0Cx_yI;u8$4y97#1Lv_x5$#W=J&YLF$jC0R zWSC|;qnb1c(9zTA25iqHi1N+wbT*9w?5Fg@qdCANAFqTO7#0JL{+^#dckDP-C5&wX z#NSf?|HGjaZ5gz2|5*Qccx1eG#;=RTD#abNWsntNy+>F*+zmsL=_yk7GCD4WYd#zN*bCQ^sKt z8U&#uygX5Wh-Ah!&vz;ApAFtQ+He%Qz{Y{uY$t?r+|FzzET#MouLP9KDqBj~u;jx` zZTE(HpVA`X5{TPZ_2Dq1S z6tcRGXHkdq%<-l@#%QC*ML89?td~BY#NL>T(yWNAKWqXHx0B&FVs79}Qvg+{ZtvOoa_f^#J{lB>$LQc0*MOW z1A`zb60j;KuNKJ-*Gl|7ovsMckQdW>WW!=i8M7U{7tQn`zXL!5Hco(Kkd566m~HTo z;bN@B30STHgXA_XEkm{*40zT;Mkp%Lh>3`xB7bDYRv=;IxF@g-9{fOjaQ-Glu3JaF zLE$E8kV$DMey4?TSfPK`1eSyWiWiB4U|PLZWdzqJl@b`v{;7q4w#XWSuGlt!wpJs6 z;Zj3MRU&JlAMgq9FY}$xxVF5mpXsr?)2sWOH*dl}H`ksqWNYA;$;7lfOkMOAaVZj5 zOUS{txm>?T|2N`E>n>HS`1H3coRhQ%68p^ma+H9c5)VPpvwfEk1+qT6)bLOvj@O#= zgTuQPs^S8NvmO7Sq50j_p9G6Ujopr_No8y4z2QRxBpsN5M#`-if?e-h&%in&X^iG=LNws@Q9D00;t6a~lR{CTm=8uWkghHZ zKa}(?z_n?Ii3i#1(prhG!#xQJFPxhushWBW&}1b>0s8O-i2gBs>!5x4Gk{VES>GZo zGO0{=HqIx-uJ>cZz;(Q2CQ%W(AYTZa$=HoPr^y6}HCVqr=AxO~V49(&zIfkI6wZ-^ z(qtoDt8jqWEgteG746uOZ$0#3E1MBO$zE9e%@3ifZyh!uZcv2tYv zlG@n8gBYF&%=Az}O$2cTJ-Y#PuD7daTSv~pMBeT?NQ$l`Di5T_dRrYYKm58X9H5jQ=Tc^%L2g{qc@L%v?SpPR7 zBdF2p>>FACZ#ndXkMoA*1NTi%5FhL7FK1#@Nq?M-w!0z6N#v6=sv*(j`_;S-pm+YS zy3q3O6~0BRh+voq#Nm;_E9CCOK3h=lW=JJFp3u{avqsS|7>#(+AR%u>MgV1-wM&zt zDS`%IB(-m)bWOZZtSaD^^u!}hV>zSwz`=kP7j7GWh{|!>BaK3oNGEh(3C`z>K9`e? z5G|NB;4HvgYX!yVZL>H;&>P{Rt3nt4Gz-`djp{^okV|4dgdec=gWxb`=Y+OZyd>3f z5VD#vNjoY*D~5NnZuK7~66`i*g^Y{=F=X>8`v8Cv##52pYd5v|A+kE(vi=uyo>(Sa z_^+W-g#OkR!$eRVX2us}^261`-)4%FiiMn&xZ;+3^nZB%kC1sY?$j?Q077YtfVH!J zC(LB|ZltwkB%0iXg_BJE1B8*)$t_LW=P7*Nrr|Z2*zih<0KO@ob4JhwNoh+47Nhy7 z0u}RqK8e|s&;2obY9Gph4xXYk>0g)z+mR3UZG?-Z2UwqD>jjxcvlDl$Xz-`|1E&R%Yq&ad|O0W z{^$_cXp?VwZ@iJ(mMzdUkpJ^UMsGB8kq-=;l&T3$W8uxI~A8AGzM-uy1}p% zKvMExPt*VW!*86)tyCuZ>GS7Gtk*C711Bji2iVaP z4bUr)lp-f06j2etOaW4`%UbbfI5I%6ASVyemT2cR!M|~&6aR*E*7}rhVK_~@A$-X| z0-I194nD>5gcnU@SzUT$QN0J)e^%Ar^ROV1Y#DF$@>@1)iI9tOAS})p4o&v|n+-gL z_#2io1m@w5`s}+~9od2p3}$^$vZtPs;BaWB~Z3d>Qi8 zQ7`CN-5$^N%^Gt~zaw*Ug~qGJp|R1AeoQ?u+&s^#@l319V19h~)!Tg;CUF*=gZ59RBb*Qq5q2lN z?l8KAlv{R_KJ$L=kZWLu_7qll*FWlp!K<=PfD0e{$rYiy>MfD)dePPIXNy~H5NnvQ z(&LNUYfRDJ-vmKH5EF<0=h|md$emxX4ngY(b7j>dcAw%aa3LVy!TK6h!G%^J*rB0c zR?W_zlz3i#L!AK@YVo;}K@j<~RvNgVdLf*bin(0 zUs+dFsLGWN&_TfS_jL^3OWPo)PJj55Y)uq0qFjLR3QUxj0qNTYKoq|dH|;JwdkdB5 z>cJlL|9?rOtnNL4wkp2cd|#Ciy#8@$iPZ;Fmvh1{7b?B)m4gGbEMkrhi^HFfENH+Z zRC8V3Gu>U&mL+ON&WHM}V8_nhFDEB#7X0>#B@saq+%YF@^(r$d`I()6u3hG^B+g0X zf-O4#z${SzsQfZ{U~VotcVCB{RWL^)digOpzZNC4ei|6k3Jv|a;@ zcoR>jpDoaTg3xF|pMNxA7O6YH-Fgnbi*BP+g=eO4V&PlY$Oc*#Oxq3tU^6jKGatii zOtv{(N*DMQlZdx_@WB`lJDF_&36Km8gm~$s+w=~-EgeuaXng^$`+_VSXi68TZ=Bp+ zh_9#O-$-m!AV33$5WaY8E4<vQy>NIBJvMXVRmf0thG@GiDlX5h`?il zY9Jv6OamOU7ppx?q(S%K{P4V`zgl@2wBk(8qC{)%p{oDpMBhLSLud``+dv60=8$&; zaJSI*e_L#P{{>f7(8J#Dik_E0R1<^M-sT{%NpJG}M?ucPpkO8^T3ru0!3xKYjuIQX8&U?6l>qCc^^tOb|(%ZZ5<#s?1{vQUfAJst#y5-y*7H>2q5f8^(`J ziCKqA4JdmCm9Fuy+FGO^(iBG{2(F#Ev!@DOpbcDP9K^UqQ8?83$g*$aglXuI!KPN? zuP?|#ZZuSyt?NE`jR>M_Ed8 zZuhluj+=sVQC|xBU>9L9Z)~6eJ!rM>>8)bMh^&VDs40S6e+=kX9|>j=ZvY|}ijIKE z39o}){&0V_U>4+ZP6HCxir>lMye|{OQWp|oP-(GtQkKSEnAmCK9Zdyj z|0GxM;b}&;45ENLp>^VUN1yHEg1bg5$@7e(eFJex8bQeFqrct^68`IDr>ZlvLAp{D zIflxuLA2nC&G+KGI%A8Ni~M$niLA241gSx9#hry#(gx7Qx5*Q|)ld>}ba>gp!^H)_ zqDi1DR5f%l4uU0HL?ht2XJ1;hJb@(AX$R&09}WK~kX2$MxSifF7lQ=?68^n>qO4*A z`FJy@Tw&4tx9JU2LSCgJQ+`ImW%-ipXRevGEq)!5NN3)whOwnZ{_Fx4r4$yNQij`5Tq_az$W>alKnu_-c)OAs0>uI^3UUN+hA>z?3Rc zkF2UHFd&uq7DaYUV_9+Hm}4PC8eDC_+I-pp1ke&Em{5K`4p2j3XJTd%)j4zWsMMt~ ztm8985e*40?4p4!^%4LLl)cJ4cUM%g)T&Q~sN?t-BupdB5*SD%&m3hVT`UFwhll&9 z9S!)|_X#8J1ek+wp&U7ap};GRn~y*K@bC)krA;8&2B?m>Tw1_GNK<0<<0i)hh$GN@ z%w%)}xD+w85ZYwN(7V9jzz;FT^$C|qhz0wR&9k#}nH>=3z?`TN9~~Z`8hmC(B#!0H zpdcm1{-!=R%^eKlqV*hk)-Y5z^?RezqyA75I8>I<$>Jmk9FKzUlhQ2_wH34s0W$w5 z$_EvSiSqFlamCgW>=KlW+d=iAI7lD#0V!ss#BFHG|g!m9uDU)YYpg zq;AG5LK$O8ooCDZ*Y3NP(glLgnxdUFG$h&4PEzN=KUfpMY&)`zR*h5mGfeqd#lmXS zvPs!|E?1W*GG% zfCt%!qVQLka$&PIjD-+rnAjFv%^ly|U4|uJPqWA|3kZ`a3OkwIo1bCwQRZnzy7<#s ztH}Xcn|)+|>K3`}Fv~%#N@1L(3O(n+Yat5qqdW3w$z=qFataOn#A<%VStHPq@`Kr0 zB{fdJg1AiPK~Z>so`QY|p+(iyaDmW~Y2`S`lHx*H(=jkG$hN@p5^$@O){PZD3(ZVS zOjH}Y`mq~88dYMP@0hkk_3EUfoD~pVyA~dUuKQ=?AsK(Sm0@@z}eU-#H=<8 zEYpNq1QH~%*4%!7=$0-*Bd|WbdgyV_AcqeQ0z8D2c&uQ3p*4fhBB!4bGat zgwhD{ZWnAY;HuY(6^~Hp7aeF&z^Zv56iC5(VkZx)?9(n3I64#6i}Ps- z9TflhRqU1CRwcCTjnq7dOaHJ8Lkd`E$*#YU?FDYYj3+jDL04H$(8BLI=810?{<8dA zwpP_x`nprfVCI>9=}gW9Yso*DcLEna^|I^jSS%E3-|#chOdcW68Wr!f;$Jo$|Ebx` zQGo5=*Gr-#40J9T`U>8R%a*b;Iy?C5upw9Urq8$+LN5rc#-27GJDGEC{r1T(FE<>O z>&nyFZ*QOT>-%d%`%XUNl|gmaViNV%?c$SKxcu(@6cgE+Ef1VtuW&HVN^wcvcX?mM zb=yVGos|YMxqX*P4E1$xBpdsPuTBcSpQ2WBT6B4)tC*A3*{d_k`+{cE&x6F@L$7{dPF8QI-@Jhn)>v!>I zh3bi$Eo0YCjaTTc-XZ#jYwk&0aEYZIKCk;gL@Z0|Xx{J; zr_byYcxf)9d<{8YFIU>@2bN!}%y56p{s`O$Z=H4ON{IcPTU{t3|CRr}`C+~GL)jCPqFzVyyoR7?z6L>D`_PDs~a6D{1A!r#jCg>7v9epRyS zZR@-H;EE5yX}g=#PcUpd6f_Ui0v@DHnB=DL^F+R(uI|k|_nuwa+Sic)8+^5+;_Xe3 zuMbN5@18xo@OxbtR)@0+QaR|tvZCWn&GC{y{NJA@LKII9 zJutNJYba_zB5O#lLwHFVmQT@iND1br4${rkbMgOX8ymJ??+2C94(fzr25AjZ!EtY* zB5A||Q%}2^vHB-1D{(9M0f<&_qM*{6Yg#6fQl*~jN4zG>+nCU8D15MPb6XG+FljG6 z2*dEJvz47lHG-~!l9!lp4G41gVX_NY1;7#YwG!o{APZ|wVYvoGseBj($L(u~A#P_m z3MMjLKYKIDq|r0MNmn#dCy?zf*q_^VodI!38%13^?nR2Z#|(!pH*6CuI9gzl9tlox z>Lua!Bc=E<)Ex#ZIC^wMVG^O+rNqh@8O_&OM9ZRQ&nNAY)yZ*v3DU>vqp{nb6!g?^ zx~ZuAO*iNAWeS5298l5@l7=MDE$hEI`(FphPt1~dMRXHpI1?Bpb8R7$pxaJxZg<07 z;YZ%r&nj=7JK7bo{tsQ}GHuNl+Qo{avwfH064&q*ToAM}Xl3ZwupVzsif(XatZ3xT z5~Ul5d2>1pp2?2htw}vvEAeK%&0c&@gZBaq(&I$0<(Z{IVK>)sTJZ%RgQc?1ZV@%%l7CVp!f|g(Xbeg%Nz<=iR{~ulnTrdikv9wf9}yR*xO^A1xho%CwV7esJuB zx~y!Y;^Q8RvG(~R_s3*@J!*dB6t6b+bG3S}pVHM`d-pb+yZfu`Bi^n0UPJP;pWanSLVM~|tji&@ubuNIHgSYSX;aD}Y-A(yW$ z$&HqYEjGrf)ypznt*ch6+dQsV#obrG?tW&gp?dITvt8}d7v@I_j~%}#{BWbPL3@Fg zYR+vXtvXLv(}6?h+7|~Nt&#qcV;?biSMcoivnM_dj2`@E5@Mbnyi9YTb&2N4T|r!_ zZ=YN6mA8Jco|BL`GM4@NBd6P_fSOXpSfjU6g2(I9mXB>c;;SEwO)TbH%Om|8Kdtx1 zUt9`Z1;?93b|34D?Bdl}AeS4-JKX=nng9Mse^J)wYdk@5G_v(aKB_za?QxP*ZBMya zNac^b6*6LK@5WC0tCh6tH_PS^d{c&I5qy~|_FA%E3h5iy);8{;QMU)5_3 zcqJF8(em+a;rgjo6~pFtYCFf495jn|So6o*C@!tmj}bknck_-62S;=2VlvDs(K}KX z&*u=W{VU9Hmvj8BlEBIw*WHReh6NQ)JK8+fX*iFKCH)xa`|5C~w|HLIuFo75ZkqUm zZlhi5VopXy)1t=w>`hmdUvRzSyw@UXSl?Ak`F!Xy<5b`J&WpZW{mEWq)3kIIyguAg zbYCnkdb=$*G5u0(>}-w&f0h-Vk$zzN{iht`6;b$Y``aBR@)ob*z*BBrdGEwmjW&%H zywLaxCua^G$#O&mEE)MC;_Q`L*>`uNNc@ic-_Pt1*B{>^Eq!TBG3vxExDzx|oTEmD zdiuP^8mB==qweYTy=!^U9G?9M!m>w33|D1us~P`b&lHw|0-WlI2oZ%m0oZAi8yGfV zE4Rq{CMMQD%2xyVALui5>VfZ#njxqlaD&X{F^rX>mxMhKWc~jN8;1yk!p4Vw_2*CJ zh-pGMPZk}lYN|ko3oFW|DHAVM@9C<~xd94M1@kqfP*z~nx%`%y<-5BvVn|^u=2JMn z##A0752gexF`%^3;%k_U@M_3r!3bj!J>`qZlT;TO(D|vqxk2IugurU;CK2fk?|Ex%n z-s2)!CUF~b^)on=A%lJVBiDNX zG4Jzs`Th#~E1dXShy6I`V*l7!!c&zlkHFkxiK^%3IlLY03xi{BhwNfar$6iK9HPrC zM!#}SnRs2UX#2LL-g#aboB2!mMtYRAE-w3C-&#Ivb&BB@pZt>Q$rA-Ve)(ONa~i#8 zKH??v+9o;wS#XlLO=g=wf6LQH52x|(HfybZnV$BN|1j=icLz@Q0}WF=MpygInOX0y zVko+&<bWX znECtE;1_{MRW)W&?vHM@*j)KARAcBiQmt2cST$b#Rd3~3*iEypYY&@`qn9-p70B&- z>a`%Xe3skxpbHM82JNGDo>RJhbUm|}TTz7@IW9)UX*4~aKU{FmZ0nJ+4GH_BEgp0o zk^bP#>E*UbtiLgPfWtKIV9#->e(#PQRc0u;qir{pvgCdVdApYbhRch^U_= zU*D6RBrokYw)&B7XX|dRzKnBep{-@LJrTV^P6E?s4{pZGc4>d(F{-E^x(k1mLc;E- zql*3ag%+-SEuodOak%}v_^;r`{_LMi(;xi~E0D|j9$`MzpLq6yvqEoCLQ+EcG_M@Z zsLSgG2Bn)Hl~lHk*#;)9XdnIXEAQ`~`+ZL=?B2aM9JHxwRIGhEbhCOz?-L7qz4FRC z?)*-lkDg&Zr2&g5gM5tzTnh1f=hyxcD9Rc-rMH%UO`Y_dJ1w=+k*{7KhU)icRFS^n z_Gr(Aso2%$m6(0&d2E3npPYNut()7EE3OxeC;{%JbBVAtUu-#e%>%3EU#cf)+1gIY( ziosR@@LhHA(4lI%Ti8+vMXNRjUnaQ{is2up*f-zw=Yay0(V}h|P2O4B@O}IE9B+}e zcaqof zsL+4bjc57-?3p`uS3DsoDdueQDe-|)-iKqM(>1rfxcF*BFl4Cx!{=v~mtdr}8H$z= zdwz>kQYOo__Tc@WeHDuV;q}?@q(+_5d6n4R{*fd4XlF(=Pj|wDLU}2|=y(W!Q4pF9uE4ak)+?i(K6v(#znqIq@#b@kEh zKYgOQPLz~OybQYCd$svQ^R>@L`5T|r7>FJSm&=;s=t^-XvNhuV!|&1b!C;J0nm6Y6g+o)rMv-qOj_u`&59p7-LP0Q>ms4BUv`%sBC_}7&!-!|G_G*2`m$}o zzNtR3uf6viAGB?{Ew8O}K*iTWJaXRU~y(MQ=5a~ssR#58Xh z+3#dG@ZEaZ_hiMtOs@wCc^+F9%ky_kd7^g2Zl5ng3q)TfuvitlOuD~DPW$nxsB}q^ zh4u}5TYU|Vs(g>hbA8kj8bVrR4FdhmCzNF;Ya3_>ESlDJquW!K^%$?~mY4jQ`|Fja zW!>uxeDP?^tNCKc@Z*|acuj$lm!@}o;#kimZD%Q;^x8(-F?~*JSdWgrsb2Gu+S(GO zYwE9r^0gX#{R&oS&pKD`+xvPT+qvG`D@EcjJ&T($?H9F=2`mtl^Hq!YwJ0p87W8ZQ z>&`=)W|?$lS5fItGalyseQcHN%(vTq)Vv-lN(Ca{*ePg`PtVDiIL^?N02F1p4S ze$5HWT+O%J(|zUzb+yZCzc`gUx*d;CYu|HUmfk=$*TKiVfz{dTEE-{>F3P+=OSDBl&w!WRpE6GWB^!P{ZYYubTjRdaq@8FxBuH0Y5@A|rH z=jXOgDIJNGAGivhw@PLNDgh&#IV)i6Hu1vp8&egI@A*4EeqN}AJ10`wo`bQvI_n(E z``N@4i!R^#`r4v*#hS|foie9SZ-^;pRaUO8&r9}x(tTU&;LXMDRn{4n0W&sk>a<_~we4TsZXNaJrl!9jw!xrMWDE!HQ!pXR zcWhBTEU|X2ZpEqj3-QczG>woSG7_}VM8p*-^58DD^+wxtfV?}W=QPXh^3=sWks zHp@gOH{{hZKTi%5lWgfIUYv#{>@#@vzHFowNMZGI+wm^r_ zGc1#1dZDaLyQR;qIWTwVbLl*hrAuRru>Jf!oExv*dyuZdJ%7Haj`-50t*CnEYDY&$ z|6G0qd%+-z@CSf(w#_7!7ezCXMlR0TkO#%lxH4@m*+T}mz#b$iyI`* zMc-3^55VjmC>(CMCQk3D`knLX4F5r=>7r6K7acMM#{|YUuBrL)yV6o_+W$w@cYss< zzhNH~%Ff=akTN5iY^iV%4O>x2C?h*tb`-KgvUm25jD+OK-igS{p67kO{QmEIz2|au z87JrXem|e*^E~%+-}h64lFllgkq|6(f}2r-9ejI=<`3u zckYDMdkmz%pPYJ-mz-?nzBNR3`5x8EkAX*@rc;jo@PILUKxw&Yvq;{Z-rVj{aNe<8 zO~Jk(MUhj_`iF*!-l~}ji-|&q^Bc-OD}N|_jocf%6G5@F?Zt}kufV_HwPVMX?BT?C z`E4LxluAUn_ZPD^f1($0SGZ}w{FK<;V@zqEj+mNGbm9JkhquA4`e1ZbfPhBfDB2wR z2zL9_*MdMI1t)sCP|=iIxnkG>tUGZ>;FYAMEI+ny`%7x4RNMxJ8b1E;x#ac(BFE<# zG5ks!fv=a_2yQri-ivst$18S$=5V5SUX=Z7#>$e;-JLCCjYhLiI4CM!iSZpPe1(|< zhFU|p@tz}BfxLr}Q>Pr)C^pOP{@qzfN>`JKp2-U{>LtpoSARKd82knM^s;Lxh2{dfx7Q%dl}O1<-fg4`J!=z{fSGxt}rS7ZmrsADA{KzKiK+I zI%eqr7Lp-aZ}r1ciK=IYQpf_n-nux$m-;M2CO7fUbN$o;l#ch7y zN%HSc0Q)`i6p_YjiFQU}D7IO`U>NNtyq`Bg+`Q^v(6jVoRQ=$EN?NJR16wAGBUhdn zUgadPrNW`tC$c-7^pM^bXkM(j^zK(*?Do{rEH0|iJKnH(QciVXXk~SxaJoHn0EHPR zxF;a`_Q0OWEYKhR&SzYV#)s29e}4uxU*>c@JsDPLfk~Lj-2T#;BI_F`Ur<;UQSx(e zwV~5FU9gTA^^j_kd3@^>;Pd+*UR)2!{*+e0txGAqs53|fo<#6ts}Lif|fM7 zQ2CzXy|&-HHwBup{1YVBUCnb9mfkc(UUCd$0oCK0tqxb!pm_k7E0oTi`lDgwjHzUqV!vgtQncn3k<2tMeE7Wh?NBi_E@K z5skJ8Y@@vIS5OU>-@?n?oVteLH~i~zwENC=9T9YcqsOK#ws&m?Ka{m$Xy^zyYr$Uo z6{eAL!jqfeza#E>AdYIZCHT=tO?Z~!Rl|{^LsDC4_z_inJayV|5QXXBr^omloU9G= zs{&pxn7np7FId)CYI?cv&SE67UN|VOXlls%roY8Ax&P~kX=6zU3c452Z}kOWqA(ym zpvO8piJ+1Sh=|av+du2$4UNcY0NK;_K(HWaKi%>(=mru*m`8!zS7h140v1;NRWj`C zI8YP)R=`zS`UXF+gE4nhG&Db}46}93psJx0wW1PXxW$;s$#ML!f>{I2;0VdHX8>>% z85m7TQoX8fE25wfMGoR3AoHS^V64QPkCu`qMPTzwxx+u2MRzWD4};ng&XBP=Zt^m2 z=omIa4`9kJC^IoUD5w!Mo2mU)7u31xfJ=Oy>u2&d1v$C5*tm=0=l{jMkmjjtrl#yL zBzGMI>LqvE&Jq%O8x;l`6{>nYgFIduRGt!{6}@}T+P%|Td)VumrTS|z;+sfB2MOJ*X!MHN~ALC|>wOE%2I z5m0{!f^SLq8W(a2(lnqSjt?9qhxmPcu4q#@X6XbQ9r?)9+OmrVO|wcka=*TkPA4I| z=_JlDzLd>PK>q$V8^$nLjf{GOh1Vr}mtM(D&$O8Hk32~M^}ze#)(dHW)}E`=@9oyG zvp0tIJUO}299FNZ^@-tRLmpOhM@thLiaTvCXWyuY*SXD^wnu)K+~+qO#$)vmIfI(C z6OksI5o?g}_rV3Ca=xC)ME#39pLpuK>z9{#A>LncS+26}9>(s_vRORyW%ZpDjGHh$#RrO8ti={?FjszL% zxRm;b#2~STIFeZUq(ZU?>)RRjm7kETwDzFm3Z&*UC5G90CgV?46|xIFh&4Rk;G*vA zW!ldRag}t>Xc+X7u@~2OZ*AgjGrpN?yV-MOMi}rdEce;IH~rXmbDO2jaq$==`Q1o5 z7dChFP+ZqmJ=Oz)zTnjC@I>tB$QKurtgQ7Nrvy9#mMZM1bid15dI#a3--|!=`D0C| zI}Zgd9zM0qKVNfhuD)h$mE<&=_$j|H`!l=S+)Y;+9pz1+iW$NR6W5!vbPtyu{&8fv zv}CrlQqny@QtBx~tbY5@*Ws4~)KCEc8PSht<>f2e5k_8mcxUtId<$vr{;9)dPEJAB z7Xp@2QnT`P5ysDwMk7N%ZwOnu^|H2_v`n)^4y^5^L$KT}JEE)lWVv*-X#4)-kLe#3 zvNy0Iq%v&_jDIU;BIe+d;<)V8RQ_InJb(4m@_xtIecxH^6>I91*GnfF%J)Bh2Y z;*7IP9$(q-Zv8`+KXMN#`UTpDG;-#gDNs2#tc$RR!*fxA4U9&HQX~)$aZqn1$Dcv@ zY07X)=uRs z6Z&j2bw?4gVJhZzBP@;^Sxqk_*x3oOb68?9a8G{l6y{!OWwTh^=J6i9BgJ*A z>N_Obq)c%U?d{d9A*>l6q#KQ~t{p?mLw&578hi!YSXvl0om1rSPyh*a$Eh-Rx^{LE za6lj)1T6sC0kH{>mv;f{6?T)W8B`KeGc%kyV8qM-t^a(ASR|*EtOQ2KRM~)0a}WjR zf&m@d#fxWv*2`XC+(0V-$asQ?T&4?@h5;}pTuA~ioR#7!$ zMs4VmtT^aHx-PE}jPyq5fB0|;$T2yVWE^R&z}RDaE}4`CMwRDp(Q&-?GvIfocLTF7 zJ~TtxM_{6n0sm~8>HbkVOz?(a9vi_dH#qe8|4V-7*&(Bij3fu&y;H#JZ$oV14*puo zHG}pg5K`yniHU~ZHrLfj95Al;L82C@MVi41Gz7#X2jG&!!^1NfXO3jSl^*+75dwYA z?*6_xfHUVVaB#fV1Ji^ozSvH~-<^>@Y=nbl*0hj|v*+aM@?O4QPHbxG*z*VSO$-GP z>WILiUjnH2%8u}!V0_Lb5R?2Bn!flR=jmDNFx2QXTs3GNU(C5gb@nnCJ>cPE>Szdi z(|P%vKEz;53RBN{X{1{^fle+>Y+U; z7=p4MFBr?eJ-IhoN$R)wnmVG(DA@|{_R*d>BOfs~_l0&JW_3p5M%?zdZn5qhd~mueTYg_5S8#A#aNxj|MvH{&D?0tetq8Ms}chH z0dCh)7sOf9N#0Qtc}%o5aFNCj7-DRL=awV#EEv0=)-Ty?LmZWjPG=l_`jrvy{pL>; z>M|E&!QboFn9WT(#@)fWo(Kn9YR*IL`cj$Ad>q2>ZZb}+{_5e7Cu4@X)p(VT>VCBS zMO!cQ;43UBKHy1q3pQ5lEj6Hl&3|60TAt8SuCL%=hnh z(uzF$?6=5)VM@TAQj1$jl;#X!D(*0`a!=qUcnU?s>=HM`uTxER-p0wBigfugIR%Mkp2=gUWFZQ z8rkt6Oj1u-`83R&DF)8!0pR0pDcA!`cMvLyd+do2^d9EgF9B z>olXa%lsDV5uonwPV~m-;1$6;!0@~v7=sRAF_(K|k0_ud~OQmPwXVcLM7X~e} zT9SmXT)t&c=yNYZfnSl#nB?u3y~!^wti*(NzW2r}*DNp+*|Wm%RddLi>>Zj#UzU0X zo*gOq|MJT&jJ=-Tael|au@`u_e?@7D*7$E<)*(aR)5V`sMR!;<+N@684)8ttGtBrW05%!e5BQGzKzbNA`IDdEDu?Z zl|y#}WD1z7b2;Tl$CsKx%faFCu2%_2?=P8TnG!LGx!CV~9`^K6Y+ULKOL$&`4J{zq zI2&VFope|Ic#=|>_j)yG)TN@J5xv}t>%MF%w}xWB)s zs^8b1)pI-N*?Z^^$J<51Qc*{@)9#_qIw45rf{)Q?Km5=$raD#=8JP2WSlLb`e|{|M z5OeRso<_LmFY&6d;XK@g6>HUP0GCOPWcSy(0>)>B(^IER)ArnxC{EZ5|z9nB% zP*^(eyuETsYa(%$D>@r}cXaGccV{gBY_nCqTU0bC&lUL#55-3;N~gdRkc4Go!Z;-U z3ZrM3qxh-wTpuMzD7jTccUaVvlzxD-UNbP8D5%aKVU6wWg}??3mEL=2rG9E^Dh|l- zSr#G;#UO_hXOe#Gj~putH0RFwfCCf-u)p)4nazMc-~C{%4cRNS4i~IcF;=3G{QJVB zA|hz0=dDXcgS!#{`_6YNY?-0Cd|vJP$?E#04s`;kHPha8JopS5!3A|Cr6gr=^iW;` z!suv?vu2uz(s}tSe4;aEIt zlOPK?KKZgZzyEm-!WseuDy%*D*SXX6)YBKdy-lE#HUA@nNH?dgXKpPWMyf#FM+*Y? zCLqoSBUKslE0}*R&MFKWGFYpM?b8)3Xe9}#kbmaB%^}V33taKji2hGqUkb)#9 zKT#w<1*jl7WIyNnsn4DU_oYw}t3LLDm{}%RIur^{(xl+0_I!wjh9&@7Od{fzoCuNz zHC1#kFHH}mOReaH|k;HfnI&@`4fBIfAeTGH%(&$_D zE>eR_S7-M3-?*%(2wly5Gov1LhMFe%(odsPR&V0Y{dpW5{1L8y;V?prGMcR?l{4qphu<@4c|Cw>SiC4IcP< zXe8w%`8iECnw%n7WJm%%tJd5s5$!*QI^+8;BKJKoR;K~UdiRM8<+3-RVP>84NknFp zQ@DU8Xc!6{ncg)N+oDR;dWpgX)q$q48bd?;fNSZwNv9TiXE1Hty4=-r3o}!5Tvn=E zq&9Y}J*YK%>l_AWWBBrsc!XEqy7Xw_RNGPS{;dh zcDgEL*l{d&YORcWIw3cIXJJki9fKavnz?a3HHNcRwK+lp<#PJxIGN_&s0Sv!$DB+~ zOliaB-~)cdQN6_lv4c3gYq|~ZRW@s|Y6XL)rdnAH*xpAEQa;X}U74E5n!#>37ZR_V15G`1b<> z2aN9C3L_?C!N~aaJ`%ag+v$x{wlekS&+MIA7qODkONM(d+}zSZ-jz0tlY#a;v_XlPKLj_%N1sRzBO zBJh>7-bO{C=RS}h-&?uYowEqEjSK>6X=|&osTi^mclbpJntq)JmXASeY{ zfxhd9A3Mi&zretdVW%2qm}!w&pz#hC1j>mq(6e&q@!(79U? zR0r9m2u5E7ihe-!-cMmjfH+9Wb!)?Y_EMSZEX&#{oUQ`F~3ql z20jg~-?Vp-Rl(`~9k|lL+VDQg@{kC1v^$50MLNqtL~aV|meqg%UbE33zfJUFM|*oC zr1x4dlPd>K`f2D@UsF>HQtHT(p*V%JtHP);N53|^r0pJ~{mOT!U#;-dXMT}dxiZ-r z)Cc`L_JTDGtQj%Wog*3INtR0Wz4-EzzUM{56kxcB?Hx=PeZjLylWcOX(GJ_LvCbd&NHo z-?1!H`2TsB=A_i)T`hIB2~W{~-9amqj4G#NrPNSKO2~NR>u|?4Hw;hqjNuUU+ z!G`CKfhJQ_-^gm|1HQbFERwN}Z}(0Nbr(>P7m&?>XME7#Fa6=9_M zBDJ{bC=6>zCAYn>XLz~RI6$?{CSYK#n&f^Mf7fXJxC z#89woi21CcK7b_s+{gct_G+jAAe?YYOEZCWgf|bIdDl!uJ)3^^ z?cTs{>Q-js1J#!uL4CU_F1fSwv$^sDawG?1!vlM~nw(^Bn0$Rd(J_5cwhtq@=pOFq z;vRl-3nRwXm^c060{eScJ<*!bzIyr;6m^3ABbk+zP9xQh7w=T?XWXT~ zal@Kn83P?&swX88Y=pPQ=g|GLW!z>LV;tt+t?vA6(?z@8jNoGFX>M?8>QIf)Va@pX zXPX|CCDOGM^f79?EQdZmX2C*w%dSgD=i$rAtbDf}(W4{%$1&gMb+fWTQ*-l8SS2~c7_3e^E6hB#<m(g|WZPb;%}zI7{l^^>g9wuQ9bZ zso&!7i)@#@FPQkEx}u3%CAKpptjLZSPhWA=%f;jEH)br3E$Ftob!&{f1NMr_!?jf{ zb7XL?%)XoM?YlTAry6fyh={vlP$)ZReKSlJ;kWoclBub&$mnN1Ral@23Bl-59xBI) zIU}QYDziNNG!0J!1AcnA1qWq$0FCoAK;Febs|O*n)FRO1k#&pI$yxnYv4@}T@j>$- z5D4qRM{kG9pD-GEY~G%K?k>j$D<`cjBqBl;l0ZAyHsC6OOoBZJ^SHx68<+~j`b4|X zEau?w@Z%E>_8=1oZa22iui6!FL%q!jI2@c%t&!}C?{yv8`A*hDnT(g07t+p_J6QEC zMJ7ogg==bR>NzgXwD3Gws)G$)V3f&-0)^Zpp!;+%Ag84xC@UY<(EF^c_93X7-wKkT z*Q;u3%t3}YSo`7%oW+Tb0Q3V)rmL@S4x*lOTsDkQD?|Sm1eV?JI&`tS&;DD4dk=&m z5aL1WI0)M5xF{Gw0VX#9be-{S@&JvpI#dLo(+g0Z zC}B!~i2#-zb}*7L7My};h-fZfgG+o~P38FgVXBch3%&lDJb`o6)cz3Zn!A=d2F|30 zXMbI2?9#r1;Akc#H<$zOU90-?vh*}pBA+0;Qm&Gw9Et0m)nw{)9B1}!%|#)uo58)Q z%Lo)#G3w(sOYHn4FJw{Ek=UWG(obtV(d|H=o8{Vi0Tpig_JxHN55AYw(1+OmvwyR`!+$ zvu*aaB&z7%q9^e?;rLy@Gp+{pwYXp(ky+%iFEh9_egA>Tu-&Q&J}JvL@661z%>mhh zW`5Md^jbzn;jm1>xu>w({L-7Sedg?gj+LOm!0KM0CuDi7uB{>cA!M_F1Wj@g-xmiO zK%Zgq#(V9m`iO;1*PC&I@*)ySa9-47kK6!M4fU-?aLO!shA@0Mcoo4Ptr$p$R8&+! z<=|Ce3R>$RFg-v9fh;ZYp|v^oE<$()@$h@tKb01yhDJ?sUD}Vu00cChD0%jhFTW+=&BwAGJW5DtYLZSW_bf zNLk{O^j;oQhLhLK35kuR1wBw0%vi&7aglWby)C^R)6n3R zFo#?PaJ2fRog!*Yvj_dc-!I^b-U9>lCh&U$?dx<2pSU<33?1+(TruZrIe+{H7rbE~ z!EuM2=&?^gRTwAslqE);|LNW*p%G>^zEZXfOz1at>yQF@#ibkd9RUYSjN!AooL_bP zjp`X%cC%8^?HS=)y3vpgI_Yx^8U~|Nm$Qlnqx4Lh#vF)*FXb1lU%x2fPaL$Ro0?F; zU|5G))s`DuJjwGZD59S4oRK{VX^!GXMhtVaU)4L!U8{4lT-?2!oij>d7$yGE(eOut z4;DJs*4z>$?`-%SZicxnwXwpU$@o~_aIZ50{Z4&}zrH6=^j0mF^+m17@{duh(aYty z?UAUsqb;^ki3Hyb)_+m)y52ymO7py0%FI2$&AII)q~{?>e|EM{%3EVrZ~snqnYxQ~ z(_)svkT9ip?pRBMoFprW2+M*6bEni`-31QOcddrfy9q;&9RF~N*k60U&)F8!`4Dh} z7a?Xa7PqApUoo})dHmmk?C76gzU-PV{v2xWDQpG6T-;Nd$g=wvn~dRsL2lls>F?~K zXNexhU74HO-yABn?1jMx-0uX(n-vo)a=zj|eR@l#4t7! zQDisuSyIyu@sokUE-=0OcOK$$1{oPNpmh@-AZ^gJZ~;h^U@(|4Ql9}ax7mWyivrMghWL4T6>M$!KQ}iY zGoBzqDqz4+{o0Ec618;+Vg+$(NZd!jE7&(MYJg|h3Ak9u!2^w$q&+u)Z`_vyIv%{{ zIdif=MBnP=D#I{&5CEf4Ks5q?B_KTgKhD#tI8DfDAZn>;Y7(H@XC3_f{n>?u&q1mj z6dLMR<2I7aiR6HhkOeoI2nJcfYKMyuqy&*?I2J)|?q_H>_4SqBx?)h}a2m+b&$7W? zg9L;kPx~&zsgj?Xmq!edOaZJ8I?Exln1X4XR&Iir<%Q!nh<3X`h)c>O?hP#9N}EN1 zyTNOrH@j^tI#(f@M4uJImeE2Sk{(RNq7}%6YR~$_k^bYA;*jvhzrOcg-i$t#{F*hB z8G7(J%^}{WqQTRng`e2!KI1CBq0X>SZ6U zx=H{&&iCt%ZfWLLbyg8~Z{O?t;E{X7YG2Nk$%+XGmk%81{k({|@^~K~96l8tF)1jQ z9J#0Yc*QF}7XOTnd8D8yK1%DhnZm*l)A^-aZDZ?I-JaY0zoyKlnTd@tiaA-Y`*tz1 zmg)Nh=9^e`tmMM)?0_55McgPLDPLET*EL}GZ9ZCcw%_r^GboVM)SV3 zMlL%$K%6oGhGCv@iqHDodg5V$p-!s4fYkK#U!KdCTOch+ie3>7Y-n&1-m0y&@(Zk3 z@?(OfA&iTQdt84RHyC}YY=!4V0R;|nL&a3BpslEm2XR*{Nso@U8`8mN`mUVo2G4P* z_DGmL*>bm1wSdDIH9u6!4aHXoln%k=_w)6b#O+XE4c`VEl!AeQ7mlx-5LojF2np_m zbZ~?zgW1UqFyvx_%8wLbD8d*}a&mGudF_*9#FYU=HpGzMC@3rYmpTFMtM zOd^Gb1?7_(xQd}FoWHx1h%!o^X_Wr!^Zy;DlCE_9za*wgmzNqgJ^)UqQFh`?aP^cd zEVzzOFyHxuiam%E02*;hNihQL9S?O1A78TP4fq31&jAi}J{v&bMDWniMe07Veq^&= zIL`}ct}fDHU0PY0_*fW)T$lpG8iNI}Mrwj2cla^^pc&Rx5LUp!h2vfNf+}p-7SKiQ zZl44KM@aybz+9^lZ1W7tVUXWhpiNFrC}^IkQEW_dAOe6i((p;NBxxBQO`vx!nU&q9 zJPF5hbz(wHT1^GRSqWpxVZ1VhfMr>R=h!w^jW;w>#JBb4G_!};?LT)n>vIN0V+ON0JickrihD`})bPSg_mMBzkUC?_5ZX5L1_&H?8D-J*H<46JtwDFD0-nU&Opt z^Z^hfXn07BReDVbZ^Nj&Q+d17q~fZrT__u2dxO5z)KPj{YAYw=gyrC}d2PfDtNZp&K><$yIW%ikl*S<)jOg$xk4GLu1KgU%M*ZSt>7vL%h zJsp>Z^gnh9poyKor9I%ThI8j~*nYTg&~SK43hVEllnA76;96JTEem6&n_yhKA9#Gc zudbeP|9$f$z%j?FXTYH~y>d94EW&}eh@MofX@F|DBB^r!#su{ks;3D2Bq0!z|j zlP27QY1)QHOp4_+=)YB=m-m-_xwQhX)p!OYbBI)ptfTtR5)%4 zTH!#wGzjkSSzNK5LS9Giu{{#OLCF>%jZSmYIl^K;fVB~%*-o*0;5jYizO#C@6Pjxn z2FS+1(t8!=hgF5<16X{g`uRnbFos#-Wxtb!m=4Z1z3PI(>M^y!`|7A)D4$PfH`WJCEMf?ZR%S!sNv7|{DdE3NX6vaj zN1a@*pe&7pgTRx|B@?Hbn-gQ1NpYl{My)ZP!NJaqf3JADFLC&0YR)xUJTt63x}K$5 zhDXRms_}hYr}xlzZGNyhCOux${ad^2%kIvrA$X7Feu-K&?CYO3DJV!X!S>E*&B(Ym z(A!d_e$25CSsIsoW4K~JoM1GQ{@wiM(yzb%+npbX`bb-dN$rjJZvvD0`hFZn(n9M= z0f;QlaXn^$^9ex4q@@^h{Ma+&?<60%jaI7lS}zrw2XL$HsFB7KzkK-;;9RSynL|vE z=T+3Y;vF%q(}hLw)RVXu%w!cD`#@Cq8QiurmW3y}T~!Zg1m z%8x3ls^-u>O1lT$Vvz1PozXikWq@u@k0k!U%fkcM*=~rR5En_sB4qan!Mfz!`SWQ3 zu(nOWlKsx7M-&GK2L|OtKHkt6ocT{$_?%c{Y?isSrwC0&SmbeG?@hIMjs z>}3NW9XxDNN)Lvgu+%Ba%G%r2(BQdJlbo!?+yhK=WEJyJO$VE-Of*D*S6DjCze#(L zKbJ|gee|4y@4tAq18)>w9ig_vN}qx4(E=(UUs#4_fB+DZkUZHiGX*dWiX42P#ruLq z8y!p^#D+l_p#k%Hc_?ZAI`JbJ5?{}hoeAs){*+!Yb3)$y=3ix{jpyK7PgAxiS&=OY zM0f&t!n1gDbFi$1oq03i0YnQczN-l&;3U@q^c}7!)rS5U{(JxR2TXR!EmM*P0X=LsG4 zV!k@us{Tat{JMYIm%exQko0XNyReVbHq6gO%+1uU-TXj$cpysnlQ(%!Id>N2Q--Yl z$#^y9(uvUv>tQJ@J#p4_9|*{zZo z@j6;p9$X40ix=#87esq zGk%f1(FTD#C?cW_yy?RMCum$e-u*S7%CdFIhVn1^+LvVqYSz`llYK7wGl79w}F7IEA&W|4UXd0>;x?*kqsrHx-O^ehZ2un;SlNDVMU<#F{< zh*Td)&$z#E9e{$tl7}IkRcO#Zw!!IeAGSFZ@;VS+z-~i-j;;d)CAAwrZan$tc~XvH zeu8p*4W_GeaC58b>SycTfq5(d0bD{0TRrh%V@v<+^K|8{xuTVXb|;@*QgD2hDXurh zf#~Xow=*L(v+Y?$7>Po$GFJ5};#5iMJ%UF(kK%_VSo9`|qDh=`}FON(7E`SPIo z?L@Lq|LDQ>f;!x>UTgQ5QjhgGoC*yK8K)E}%M$w;t${@asBDH?b8+vE7c}TX3o&HbkzjyA5+%LulE>wF7rtK3*jRa#I@u`(>CD@JF zIyRa<-S>YO*u6qpR_^q#g4s)&dv6RaGNUu@bvR(!-3dU!Bz7OmFS1J~D!M7B(iXv} zv~o9BZz;FbSViSUdZ8xysnJUGFPpZOomb@(5K$o~k{iqK^;`>ixwDp`Lhe16G7B5&v>fAlcY2sOU?&kT&mU_Jqs;k<9GPU@2WKCE{0C5jk3p|qeFm>L zAJ76<9blCY9nOntDq`2#blu!`3C`p3Wjui61?D*!Fu@A z#gn9Y{xLZvr7s{`!B)_4QGWPvjxj_<=?P>~@|b!^?xA+eIHRcagjZVHKty9=%8m_u zBZ0uqR|rQSLU%xDqDI=tY5VQiOZY=-XHunGs9L%@u!JpY3%))m1QjYwk_3(YO0Q?b5-j zM4of7V~_6Y>!U<`HWPX-K{_@L9FN#yM#%i~U6T;-#d*SmpvF+zYZ{CkgkUZ~y&fe6 zI`HkAH~uhOFRyzy9ig}Y)~uM`s}>o0yxOxIWn5NKU`hSOmteX~4mHFb<;^uRQB+Ayv8<%dVTLVC*FpO4Mg+ z8%k@mFcsnq=Td)i{hh&nqG*oibmWr(;!j0CkG@&e?zrzRlf!)>7RQKJ0=^Nnu4jJD zn~_U-zNjdd)@W&Wb*mQBO#95dR*?T@VEvITVSn~}o2O#`1-hX5Ga}}H?jdUd9)Aw^ z8^{!OV`57Wmuq5$+_ttW9WtEE?$nIE^d-0=_xCh z4^x#>Q}g>bX)UD=wPk=$ zOYN?&(6pQxnMkxNr{s5?10H?TSve;;jluW!1FVR|{gX(NTqsXX_U_3Ae57W?LCbF@LfBk`ls zA*R8qbrzb-gLmhIxNm+p@6R#n-fB2nyqI_Q8M(iIe?*tIj(PL)*yA|SMYr1Z?zN8- zaVdA|T;*EB$NX06Ye>?IkL15zO~3<6qy1?0?dd$e`)^TVVr%<*kl!UFU637TZ8+K_ zx7>%;b0|yQ8M%W<{S{fL7HvjdC=2UdvxWxXT*(*U;=%_c3mDo*RvVy40g^~iXE1Sc za&nxMb3xW`JMkjNMLi7W(;@aWveL8GwRqpqNjY6cOrDffRcma-61oOUEy<7}Y@m`U z_qfJk6`7F0h~V-FZO24ea1AUbt;&7zWL!PQEya+o~}2YMh(s6rcNXq$qgBb}V5^u*#x*wBGV z%S}KU0QizU2NEQNdjda+7GJLnuO1$%yL;vm z4}@5o%vGk6-r$dJQcb@W8q45|TZS;es)UU042=;KZf`XgLIkn&I9OGvC8KfL{0wJ! ztACZbQV|Uv-q`mxO8A#zABHe|vPk-rr0DgNEP++g`g9pDd zJ++HhhsFjSV{n$IJhbG2>Z_Y-o)N})q<7uUBXUC zC~c7{DHWo2{N^VChV8OZwbwhnoY+nCaO>>zwz%7M7dl3gv&wicV^hxTY-D(3xtsWO z*|a4$9C*(d8>oVM9i)T4HL>_UY>m2)<%?5stFkY&*WD%#tlPvV$9LZR@_8bnwW2`+ zr>6hqD+SJ<*glF=z+N4R6g;Km)^sb=r^_s`_**EnVcM%Ukw1cjwe}53(ggF}4&}<) zbr#h7wC_Oz6XJeFF3{d)#IaUj1190CSE0aqBUBBi)4KCWNjFBnuI*rFrbF$D~o%{lJ)oNp&Bn0V(NBmb$}`o z&0T8$@4;1IJq3`FHNTf{A|zV(j*Vr2r;XMd`Xov9MaJLV7vET<4HC6JXSbj$0UgHK zJg}N&_8}kqE8T!S@Ky|D(Gd1pw-5>-J!Ifi4KSZplKK2unk}|dOUtCCKlik_ z`=Iwfhbyh&vnH+_XGuPRNpek1O_808$pgSRPG<38Gl3=zXius7+rk(F!8I45 z?Ug-6K(=yFJ#;$D^TDVU0+<>}0+W2Q#xpfppFDZ-G12gU%8AYigR z>hzs*8-eA~i}pWrbMinh#Ju`@r4V$gnr$t?Xu*V~?&{ds>Tz>2R+fPEx!9?yxpGd? zGAdN$o7=$x^_9I@HXyt4HG-y&aBBpX6gZ?{1Jd=^P63b#Yt;YIiqp+PKg0l)Q#n z#yOqefZY)O!J~kv>`2%8*43ZZ?o>v5zv$i9d)m;ulT$NmY^At>Rxp$`+~gm7-}<6~ z@eG+4pGH9q&eT*Vox@XFq^av9I^$w%tIi}_iz;mJ9Nay!xH?ko<}B&7V!I>cI_hTV zS&zopesj}vETax2vz)e*DI}6x7!V-e-|F&J&u9ev*B&Op4BA{pd4rDjR#;WjVorqi z#kS?KXT=s>^3YYnMUkJH-1x(kD4W#xcjg>mgWtjN0q_w7+@kE)Occj?qwWv>!mk#f z_3EsVvzHG2{V71h>7-rk5*{GhiL`D6zXxC$kN|O)RW=aX@gF-<>}@OdU5f zGdr)Yt`tED+XakG9+iOroP7=2!>7O1;$P*!COK(6?{xCtMGWMqNr8CLxx z3V|GA@JWgZvX_Xum$@>*^nCY4-OkzHS1=&m1DAV8WMm(apa@$K_gX7Y^`# zFvBpjy7(SzXaD>f%J}K+HkPPh91`Tj-c%X;5QRDgf)1#2&$j(;CCUCkyN z$qj1C&oN#dZTa2h9Qpbm>dP0=)oo`*@G^>%5(C?eEd{{SpfU3Jao96s)|57Zqk;55 zx!2M9Q=J0QYe)O~k*~E629IRi_RNPr@!zSh<%WB`u#`FAHZ@!}S8Qu2y!H1w#5Xda zEaOFYYz@w^O6j>dMBl_enDYvEx51XDMOJh-vcSGJ+Oa$PI^x0f&hTP;oguM&R$<}U zHFFN&2KBVDJZrCA4q9Lh#bY&4S%_yEd{Pk2)!y}x=lm2;#Wm~~GOC)K3Mgoj>YRixAj)L|W40+h*e*3zn zw4o+xr-BEkOx5BTeCLmv^524E5?#Zn5G^Q+!vOu%dZ%;ysd2nPK+&@2yeu#7VagKk zpOp;AwlNj>X)7`CCdJQW&pDV>{FI4LxR>8JSWMxvROZ*lqLZ&Tx!#k~^iI@c4UAX8 z@`YY7VUbnsFT_^AGf|-SX1QYNlwUECzH)bJip6F2ji=)lOd58ky85I#OfgV6>fOD) z!8{?Mark|w4*KLsywx}DUOxxkgP-3igX-rULp^52s7FU`(eSd>j(tT|w*}yiBsW7w zQZ%x7^u^Ixe>&c>deTyEapLbHCr4U2cRW;s zS&I9=GdH=SOFf}Mrlg_*fd7Xs2L$Rs&t>%eIp)YN4v>K%i^}8dohwiaopT54;&~V! z^6_RoZjuH!(pZkT$)yMl0R(%25WytnM11@9bWhTgtBeRm3j7)nsS6#9`Q`p|!dXk> zp8oSE7-1+wBdL2BTn!?Vlfz+T@-dr(3Krw!qzMqQ#I0dW+w9_*&E}Sk)bq1i1MVl1ax>JxO zin(GF6V(&tQUZN^tYY52OV#U~T3ie$F!cCd8{pyVK>ARi-}gK1$9o7cyBO8@6{?Q!Pf4fIXx_HA*Q38r!Cf zEZbUJ=kn*v|1R^Aj=1zKvg4uMx>mE;C?53RCYjH7O@AdLriL;hH0K9HT<&EpD=Kic zB7eWF0I13%nUOhbpP@&}t^K$zy6-4>3AK1;o^*n2gbZON$_3E$90mu*Si}}2b`R!$34+HVnom@e8Y=z#tlx8U zfuQBOmi{;et9jfB5AU!loBq`i4Sk8rOr(LGPS8NSsPpkXR1;x)p#ec3U}l0HWE{}a z!q>|!>jtt84#K#fo}S)uKap9cSwPF2sRbbkMuMnUvposm=lr zBxS(#HY=L)6b^P>4vUxGJD&$bk_bHZx84X+qZglvGa*WL2pb0FF!LT60=R64&2$oH z0vWfLKu+D(ur=Y?v&eCK{BYDG4a5V7?)Gorlwg|yK#Ddr9LqC@Z^;)9NO{3Ye_tHf z$k)F-5T1fzQt~jaJgXkW3oz|WE#_5FmN_Et0e)KI5)3p4!Zn)D=>Ym1z;MwJStfR= zR9ge0eyXm6EX1LOILLm&!M?W1f>NNn0GtDWn;#y0oK~c1+yEJ!oF~(-N0fh{!v8Fi~ zHLj+i;ZzRYtF^6>r-~WcWeb~-F19%x9lQWqP-Dgs%*sfGg^)Jf3Wl>jC!Fy=4<)fu{A zppmEz>pQssXiNLue-HnT*WK66U6j(~NDH6~Q#pYCmH(Kn^G}%c5)Lm6)S#f@&oBbn zZmyfz(hXcF;$%Il{epvr10aP2br{UbnJIzc&Yp1+&nyBH{0?F2D<>!+N)Hd#dpN-= zwEDlQW{5M3M8yAM4aK_L4l!y=w5<7Y5~OycpDE&T+dlWC=x z-geXmnO$?7mUHFBS?xBUkDNRfR%5gY{+Cn}_4m`%ZW}E%R`#Yhr}kxExQP+LS3LdW z2Sl9Cx0~18-S_W4*X5gguaMTNw$Dy>oY}lAt-hWZAW3;k%8gmp6*F0TFfu_JsNTO9 zT@pbjgbN&k?(*J8O4{1zXjQ=At_|jN)8*-1IzG}?28;249|`BN(@#K5fs)`2{4h&C z2?=`8`!vDS%i+sckXtOn?BnBeTzi9F7ut_Sk8mVV?I27;g!Nz|WDZ-8*%mFGmgx^o zOd!I%OjQh=23uE{O5pe>IzB!a27xfa>$E}w0LCDFTk`**u!;zW5$rL%)6zIBmq0~_ z5Zh=&*weV|ot=@w-hTb}Ez2bk^e_dgZcssT+=$oXGr%@-qLP=gv1RI+@NF{kQxAL3XVgnAos;V!P-XHohJbZ zxUlSmBmz;lfqf2x7bn4q2*wTR5!D&39GoE`nH_x0kw!WRAqAvQM>C2 z&w!)SF-mZ7c;BMC309HKU=a>rf@!Wu*ZT6^AF1`UptIkDmh)#=XHm(?%O$r82h72M zdv$d+^8c#+5YIf&c0f;Sw&cmd&KjMJtH^_rEm1ES(J%l7&}OY`qgHfJNFr+3tRhaK z^jR>IPVw|JpP$dqWKLNlncVTCmr_gp|F%M_mL~*!YiiCmHC4B5kg^tUKooE15hrIkopriM_}jh zUYD1Gm6Z&o#X}VblOqt8V+hU4V8v5b*bU@VRG8mG$g;?K2Ba$lvVjI$IFyhfG=8VS z_az+mFVvj$KpgWkDvag1NFjP2^j|R4N?t_;6a>;i&|i<0g%w}vxJU!DGJIfVD(3d9 zN5UOK9+DhjpaQF93g(tO{sb|J9Yn}b7MXj;FchLOla1)^>{Nty0iv`9jamMQhawNp z|6%Jq;A-ys|9|YgWkj+^8KFX?tgK3DAeBv>gi=XJT(-!FC=H=0p@n2cT}DeKNz69b}G(V;w>_ zdW<-62>)4KJ@KnUWJClLxtk&qv9}at3%IBI??sp|rMa}`Y1-%aS5*Bfrj#uy%?W4q zJNB>FeAoiFN6?KK3M3lNzQCO$-`w`N@tH11*T%kDY_8yVpp92_@6zU;nhiF!F}q@J z`Pcmpy_!3`ANAnhSTwCqr(XVj>UmmWjh@ApXj;22(K$YJFTJjXBUjHKym|BUQw7Qs z)Rq?AsQFQ>JOUZCrK`5w-J^C&E3UEQRsDTW{nIO7D}K~4ovHNKo)6&~R-d;~H*B-v zmS`~NDjv{Y%bR0$B{oK#ro}oWNh;|!0 ze%kb(#%^*WEk!)z2ZGbo`t0Z+fq0dDmB(D@8B>0ITQjek1IH(O|U=pDZiTiU?kw(@|func=(c z4waL#Q;pj@cOE6FZS-7iV2;w7Syyu$>c0#h(fMTW=G$}n)c=7mEp!EtNfN?uYjH+Q(^^wl;o$lua!XA{3{m9qyYO|8hWpeQ}kS{lA=hGF~0 zqJR4Dv^3`@lP6EUxH3%Y2hJ(iq zGVNkNPt`}zP2awK`wEhRLT=M`y*%l8nT_euihvjkl9!zxZytFM-diK!tR*UarS^ea zzMr&d{b@&wPHGIiZPBU2%zmn>t-z8GJly_)%@Q5=ziMj-%z82w%rBYx<*9N6HkP4W z(NX@1Kyf|kWHUA0f#4vc0Q=CauAsm??z=@r%5F(7eioONJ#;t$X;rs9@*4yDGEU7@ z88mO8fp350Sc4C~7fA!XZ!t=w^T<@F*SNwmLiKjtg0Qi@drQFRwi|}peZRy-0_$sd zP+8|7as9Q`*(gS!lw0AJ^asz**CU7$g;rn#V=Lr(h`#Au*+2FEhnX$5d%Wp5`rteB zsKD3y=H<_ewiA_r8_%-o1gHyZV)W)Pef8`Rl#W z&D&Jkl-tZv@Y3qljPE{vvy%G^q&y(R3qn)OaTG1!gF{?5!=_VPsHdWHf#zMXba(

@PMb%5eKa~F2!({FNt?!CIi?mw zMn(#@K1_=xG?09$tUM;X&{SGm1SOwHhFKMRK|dRy2A>m3D+jk}8ZaQ>$(;U$-H3%PjMl8ys6Plz&CmHO#B2QHI?4lqqvjK)oxh>{5H z*G<2y>v`_>;&pYM6%;f|=Tp!aJbZWxZ`nZ7t5+{MXbMqhvXN2yH;w^;bEf7z7Nlfv z@6N`sIvdJjY3^96sX$G2?mr!YXTyY9sE=tX`SJ7T66#0@p`=Yj&-ZZWM=M9;{&!*e zeC&mLf{e;=uoW35c0jHLTTd0=5Z|2fLVHTA|5YItvj!B)@=T{gOhVY z^zlNi&8}FjOfgMrHo>fAxLnIb#5{4EMontpIDE>Vs_wj54;p=ZcJv_L5}a+{^XDow z>7F+F`{n!hO_N@H@6K1MH zH$qd=YC@^%Pe-XdmP9Ct0iNTi?)ZEvkS@l>Y|ot!0PG$IXx4YgoxAajxR zebx8we;^O(jFSZY4pO45YaONnOL%xk_@1zS)sR@10D+g&95vMHnkn?AoP5aYoYN1h zQFQ)QI`8IoAU`#Q!%mE_6wB<>q<5}VqsuUh9YAn#D^OC?;|_V{3%+gh_ZSA<6)o5%MWn&k_!rI_Que-7k7>{ zesz9qi*~&xtb6u^DfObN7Wh@RxyGT#$tVOhcxB82(<)j)$x6lzGHyn1)REbb#-1{1 z6XXBz{^M+eGl!PUz0x+RR((P!dgQ*#H;&`z zmDt0ilN=b$Bjw9@868k}f%j}#+3nGuhIa4O`^?l3F%ls%~rYGLocR8)A2fFp!f zy6aUhHU;N1p7MT+h>~z~s3U7{{A`Nd2)+iV4(um0a^8r&M{+-t!w_}{#zf_nU2?0G z;(Fw-xL$-U1gPCwH|2-6o8+aJ)PIg8N#xpM6w;N)j7cz`p+7Ze95YZiSC@ohHM-<> z8Us@?P+YFf!0BEaUtaV#q?X_LQ(ob7dJY3?d-dFUg_xn zTlQf6NKOJaCe#JQV0Ga&1*S%uwqu#$)4Hb5Yi4G9Y1vkDg~uxA=T7eH(|ht8V3a9^^`kt$qJRiQ=nu4*n>&Im3fL&!MORlMG5Sv}aKpNBk=Hj_uw(TFvC<4+myts*KK0h@roVESLa( ze|__J&CO|Xfwdo3;UDcQMiMrO8uxQVwf?9~QS7$pugdB!1jvEjf14_%Wjs**(D@f7 zAP9?i1<{bBC-sfA0u%x2uPZrshm7`|RB(!{(?~QxrJ{J+j(+tZ>WOgIS+WP26JP zH0ly1AFfUd9UUF{)_=RQ*T+2Cx(&XW(t(IF=}qfgd6Lu>`M})c4l;*sWMcA{2=FMp zI`{V{p*tp$7?Xah*C#H zd>yafR~iAj@fx%vJ)X8DA!X>^c3RF}`xnXuc2>VoQEtQ#*gr8p(LL##iWK+n!1Rwkxz?|&{JJ=QW ziSm?iKX5eQIg1Xm#c>w?1$as8jc*kQXW|@|%07s)Q7cg$!y5V*6c-l@w9j?pSHx}( zq9WN;=tS{7mLwFEeH8U{fs+m#7;+$6ej<%GEl();nl)Yd^x3m_7ktBU^;mno0sRb1 zsGQGmzxF{va^5;g(Jje3Sa@-&T)TC=C^p*tgtOgZJUW$P6UF5IC;u_Q*CT) z_OBF%+(3|u=o}-SvyA78v=S=5;3ogxXYM;V-5nqd%Xk2n#SGCaU;0Z+eVj z3yak{OPF`sHR8<)g>nSCkLCQ)`ej?2{7>Q-wruWWt)|Kv}M@c6PmT zOIhsJ%sXomiiIx7#((<5tbFJaXK{eZ@#xpyK4s$(_SazO4d6=-FihQQoXVcG5xL;H zb?bTzltU}Q-#>u!fj`HDX$l!1<%yPJQ?nq339dWwcDlvn$qiR#1Pl|opaD;X_-0ht z_p(L56`8z&Ot5qjW?jT)ukW_-u}^ZVyo37ipr5X{|9E=Ez|{J?T^Si3KFP&JlM;#u z)bQX>%fAz-otq2P+)%#Vd@0NRetWbuaPb6btvTtz)imB`iI_lm?#;W~DW_mvPLS5M#g#f=6Ksl9*mCWy7e zR!Mc}o&$<*1}}gQ+ZTWC?Acq(M+q`7-J2Z;G=?Z+B_&@Jr{-*ZX+ip%a(QOd!ZG4# zN*2dFgB1sjYmHySV z&E#n7B*w=O^bxe|5yM;BeCpS{q08qN2aq$_U#fDfCG`?9A7%eaA{gl~LO>$SvFlz+ z8c(`znTVJ_%;okydZ-%9W82_(dTB2~?LHe4a&5IAy%c($-TF*YaqDPi-ppl%Y2TO! zD%9HA8c5cuWHg_0PGR&J?N_Id%vO@8&>W2!qei_7?RWmqiSZqrbZ2{-t%5wR8~U+y zXSWfBmDtBAo?vZn1JuO}7PW6I8|*N`y0efSh3B;I#EBE;55*}tLdYk&I3b7l z*pmPY4m_V(FtMA=4Km4@;SZE5u~Y*h*osFC>tHvI4(_gzSKG3c%L?;5HpQbZO&xaM zYDn^BmRim6Be^q!NY2lULvL4fzn$vp4%JFsXF{Y*6pi24A+f;pL`k6yodWmM1 zmY$-6R3R6;vxE3jlxg;8T|?d#hV6?LU#se2&#l{_fp7nf5#!fBRE1 zA@doJh-e|}IUA-omGH-|9FZ)r)c7`fQ`2ZqI0ToKxa%~$v-F9yz_VA$l+6f?3dZ)4 zecKK*?NJuGvhiC;Oq9{!FU%~wm6tb91nq}~&hwQ`RTus%3p2LU;?mH=`+4<-@_YqU z0e#e9{vCrZBZOQTEpA#>Rh3G zi*FG@_&N`rGiQ!W9E*59bz?{4M96NQG%1-@Tn(e^oo=j#as&!5z~ybX^+%(RE{QSm zTnxk?atnS_%M&e+VMs1ta{apWZc0kL)EXOid@cf8BE>*zZ^c5CQ-sXJ0lj|z#&{04=zUSqqFS~*IbjsHsD=(YdfD6i*s zkDHdNyZi%3weJwueP?Ue&_JCxJ%9X|?>A;`3LA6EzFoU^P^oG=dCxirKtjSC$>bOz zpotu3#O__YC|n(zF>~f^F6^`d%(S@tgFEkw_Y{SNsivW1s8q6h;Gmt?d@f8JBP29j zU0q{gH%f=^zqg&p;x&HaqD7SR_ZCD&;HMpW^my2*RBoFBfLI6o09}5yZtSTzIxUx> zTYwc0KKSNOJn$V^H^*(0f(=U%bd%whQCT)$@cX`js-1F$4CJD%wHls!XJm1y;A9RS z9F{hE3zsl>Z^`>dC;y^XJAPwZnU}$V?aZ=~hj(w^?!Dn>H}rsBBc^82sm<2)89LPS zeR3z`MCP!6xz}}3ZylXQrteoBq6zK{_~hrLs(`TX>El1=Y&uv3r<18PEsPC7RBY14xdl)}*S<2$)R~&jK$S!Jozj?Fn1qTse1%dF&wdmRJ8fs%x#&MjkQs4IVVm=l;t`;~BCV(CbKJ z-Snd?dx%-b#)A$7*(2gg5K zt!Hd}c&hE5{G0V@)R*tyyX2~3@I@D`s%m7~XdwfxHZV+@ zh=0G7;0Fd@G!MH+OD;lmCJRB0+-Fs!s$(5q?+6=+!VQHSw0aXP^u(=pdiy{D7EN0) zjNoe+2zU}B-GBW+ctsxDl60^3YX@jN+M&tMzGL!8 zK5lpN+Dn%Ll&YoFwlLi7bI+VPBeXy2yQ+ijMHsSr_3DKwG9bn6d~k4IahaR`bvnJA zD0uYBN4kp_FYYm@U+^GfCJN!ki?IWh-zn9!6EKaka$r1grR;S?P4R#$GX|Du`uS^c zq39%c_|>|D#sktwQ8`{!0FEW{i#Lu}1r!^1z2;M5@cpP&&>uv}j2yi-67NFjW^8KO zKSK4`EM{|DG)QdAtdQ)%Vo)+IcvtOwu@pD0{29ZgoDIf@#AX4!mP@xNd^15sAB z3LVflq5z94f^Xwt6u?I$Rdmde#!bc8Y57p=qR6@>-DUp#Zr1KUSNOOyp;indv@Dh5 zLX++EM2OY_eH#(s!ZEXqr-rcf{>{{wWGib$-9PPmJSs*y$44wmAGb~GglEfE z@vBUOQ_=@4*?ab7$e~wPhX?HW{-!EVzG%4rpZ6P2dKsEk|NDx&2j80(Z$3R!=AjQP zxorWNai-FlDOWap?j@vWjqR~D#NWHc`3#E@RS03N&CCWw2p1^y^eD@aQu{l@X+w8f zfmt~AO22oapkXxXaO*RQF{`=#whD28UWvRGawoO(*7jtuhEVeU0_dGcr0v@tN(0c_(D{ z(_I*}BW|#9AkVr40CRXv`iYT^P{Cha>B&yai9pc`6D~gQSr;mO0&Nl0i2AawJvqfo z%D!FG=CzWD3JsbtPFm^Xi;T%vA5PO4d&H#8@TRl-59{u+_mx)b)=tGgy4Qb`zNI-= zEb3K!JH$0#G-hW`2dW(^?us`)X_`ZD?a;HQ=l1OlSC!MdfKPefo5gkOOA;Y$I6kqN zpb>_iJBKFk;>Txak~Mh5tQmDuOok*d>*aXWa^K&w!!(>c2m|ebHUYdg-Po7?C7>aQ zKXnut)W~H`8FhcI#jL3YFekf;;)|GSMviW5Z9P1H4U)p&KQ0ixm47PyMqgr9$-YEz zx(&MNs2c`a*j{{rcS?1jLs<*H$*K|nDOp8N^0bf4-QgH=^$-%+2c zw416fsqsaD5+`lSThO&}17?i~%|?Qzg8r#z^aq+mE*eyC`!oVzPlLMxx2Y|@C6VZ& zxgc))`>ge^l|{)A2t_ntNp8Vu5*1t;Pak~iw)ewcXx#z?7c~!dTZ?DwFAL6(iJZ*c z-96X^k{5Az@#UhNaFdawE9PmUW1%Ca`fK5aqSL{_cUBbeOd?};r;ILMb9RbyC3PR? z@C_x8n+WMfy)%b+^E4ix~3JC_|*VLKSyvXzK6%;xRY9a*!;o5m#{i z+<%!yd$e}zju~z=X=={#AJ+=Ruu&li1zL0}4qh3g7spBV?EO+SaYSjGvW^#8nyUnf z2B=18k_m2?GdU{d#bszQ%@yT=zI`KnV3i5+TcMB6;b!GK*^j_bU>FXQMFX=XlLPw( zP0i`ghjJ;VdG*p>M;RYvH-E);S7w}Y6+}*YVyr`7WH?)#^c<8Yx4F&(X^#WI=zJh> z2ycnERTC)rQ2+(;-eT`*0V}{6{5pRktkXVE&(3lC_wWC?d62X}iQcsRU6ydj(|6w9 z>V%mwMO*Oo-MvxO+K=W0M(UlplzU-+{Zd)%`Y)YM3Fhk1v13i-DLJbHUenZTnYyAw z&3F6fH@5V+m7Jl{w_m>w)OqPTqr8YDP(rDFpWyVjY+>-{<_$z;8fmIS1JMl{WM1FA zGJeD$T6Tb3hu7#Mk3&XkU%6WBb17>j$ho3RxSsqfX59oY7pH*2*Yx&{PmOhG2>FD1 zTg<9CqS0v90H4m~gQXwauAUve9+|f7hDJI^ZGy&VRa4vL3OB z?n+HjIV}U|7NiPPSP%U3kK?LZ4m0{EdN4OS%Mtfpk(88Nky;RP3a7?zK;&Rg{Gb5? z`f!diCVXp6X+Aq*|9kZ6jl)gFyhLF%*kJ+DuN!-}=;JXWh_WY9?8`0hAtshxPS*oA z2L&&7?AE&LHeWAt0?(j8CJ+`OcPtiLFtFBdYfp&;)MFV#i#ifV9PiSzXKNr_i}r(2 zsG|n}3sE|@bIT@p0Worb=QE5IrZ4Qmg%tj;k558+|5PzIMDwJ6l%sb2`sKLckU&WG zQ^u8eq^+EStfd^x6GtTG+4GkUur?oOXTuz9D>;OUSerIXWZ6)6B{5NUEDn|d7mH_y zNAuZhRp6H6?+=qAsrjW1^zGEb^@ne(38!L~3gM*pKSZ^FwkK)i9zH$oH4}=dNBMm1 zBzgJvtvJvm4f@-c%b(2tA$m(h6sV6Kc^EG!WI$(8nm0~aSRc#XmS+1F* z7ko`CbOd{8(@tU0BAmJG6}J9x3mv5e`hUKszTH|MN41_EjC&x5bW!WYywYw;>Bw1J z0&g-dCsLnAhon3d7}>40nC~R|Dn)@nj0qmN<#CCVD*zu==d33qWgntAOh-pqfbxch zhW-W1mM#@6wmTUa(Fcxp$U2HHR0`=oMY$-q8uXwVaS)sgrS;FFgxX0{?9hk^na1BB zLvOh0&i(&o+d$E}UR@e#>2WBmlLg~h*b@J8L3qVDLBT0K0crU!JVs&;4{#7I_es#N ze>lH@%?GW{F+M3NNszJ=e$~)|Zr)inv3}h0I+I3sMe;Tb7G1{Q2 z&g@)IqsFIA2L%7!+cEw2^3}JR_Gq2{$u^;|_+&t~_NugmnDkB8YAPzO6qbK<>HSxW z_`L()FLz82I-=XVzjusjP3_BUjTgh8Ej|>hPNQi4+nxD*{7ZN0VD8IWVa ziI%!k*3rg1%<@X6so)MMC@Zfu?kkB~!Bgf_bYJy6!b~+SP^ak5RNZ7}OjwjvqPdP` z{yo;OBs9>R1m(buz=v-ihX%TrcCV?aDJufWva8URt%q*y(r&x(=x-gbdi(mSurVJ$ ze%!5PK7am+Q--1M6*_fFUg@ga$;8a;9GvUtWhXySL5RSTbvf72Bc+@~&f>=AtE+k{ zDn<+naGeZ$)|j;3cFj%qT6Ondu625iBsq5UNL$3OKFP?W>u!XtS+M^%4pidvb&ndE zn}t$iU$eM0wX3KkB-77aMIx~8Ri|Jye-wV!F~JKT0RpN{ydcu9Ap{dlFK zDgRBolBb0o?F~m<)@mwIck>!IIVWc0q1=&*X9<;Uaa#eG(pF}Z>z&@Sr}@KC4J3+z zB{ zFF!V2`|R>LYFlR*{g(T56B8lkiBAasuFNFw)gPOPwP26P3Vj~F-MC`;@()Xn2efJ3 z+B^Bxq3jax>=M(>o3GRNX4hw=N40QO;@pHkK@2eM)HoAOzqDXeN#+%Z1at6&m7WXZ zSBF#biME?fec*}cG~3u(5|#&y7)_T@7;&FzLP5Fs0zf48#-qf?hb^T3V{v);Ua00P zA~@d$ffKE$yTUOCO`Dc3`dzu`nP+F4<*;BYLUn5In{Il0Y#!t3R~h#Wdd0AjT;c^YbMuuE z)AjVE=veXhA`VxDfBOCT=C6l0!xqG>)NtG^K24IT#NH=p zXc0xuzhA9`sXY@)AjdE6P&W?+<^}}g^8BPkOKv!`je3nG(kx_Cqlh{-rMM&(+WGJ6 zrR`>etDW$$kt8gkb#f-|J#JIgnrLf={{*;|4jJcf*y@P5V!DM7)6#NDT)qAM?IRdC zWJ?kt%?>#*?fw_F^cSkR)j5-|e%4XY)n1t}^XRG5r5*3Zt(85tM18epvRdoNDYD;r zq?~?i{~=>g%+*P4CDOxIk8f7&IUenQuWzp>+zBTNUKvk;u3;2~K#w1$-$Y{L2?0A2 zT%^R^XStt-T?pB^8SUoI-mQ5@S9MaE{2N;mp)pz7>!_=nqM>U9{cC3c|d}%BmMlpbX-UPgZ9H46v^DKbx);uBnaw zU3pVKZfkYu*17unkNPRo>cLCB7f*8yc$HB!_(h~uVWPbb+|tu8Ed-XsgAyk zcixpq7fzbAP596340_ZrAPzJMq^--qMjk1j;Q$|d__jLBojSexmTISG<=?efuv>qD z|78PMIF~#g5N*vR$3UFp^!~e(j~?Z)lcp$pL^NyGtPJthSJ!4YTHiO))bDZ2&pI7+ z{~CF_LqxB%Mo|ar&FchpKZl!zUt~`&ao;s(_?y8oXR8miY#{L(KJf49Bb+>Awe7K%9mPUHjn_*^wReyw%bc;vw2(f~dtIV`O4aa3syU z;6H7zM7=Y^Gcv*Rw2L9&eqe%aoLUEue8JYv{`rc3P_}naz(utVnPchD6?;bOM?mMf0lV@C?;4W2#OcROsG4oXQhz zTLAHlD%Gn`_RAo>drfLoV$MD&?Iy{zxz}X9 zMV0Z+-Md9@qqXs$v!O1;$p z6Ip2a^Q$H_ABEHb?>P|;9iY2`zwJ}FCSmXpkKLG^)57`lM|m9T>s?OyoTb!M22?d` z=FBW~uycV=uC;3^9@KR(sdEmY5{L3-xc#%s)9mwOUrlK15$Jv7NHCeurW9_W39%&9 zMyx_m2uts;<@{i@;q2KlYHc!IX*4fIJ1>t&$(ihSPSW-3mH-Wp=`kd?3!U!fMJ*4&jV-!)QFMT?R#IWXaN z^dHi{|5U@!>~H%H9C+*2-zWpvx1h}WD_oGUt-tG@G-j{QGU$+E*4lLi7p%sooftWV znAiDKONmFb^|UW?$R0R(ZLcWYxQmY7aQ@NdXREuM6F$!4nzQ%|I4u!Vy7BDl?6wH@ zyj!;`{y;!8GrI2iFppiI{bKuGZ4fA7dMvTs44>6fO~I=QsH?6*N*$F7eyEt}g) z+};CEyw4gr?`<4{W4n9!F7CDqt~Q!%@gOCU`QreG6xD4GHEpBV zf9uxYT87QV!*EQ!l=E;p_7MgrM@@rXa(QqEk6l5*!IOXg4E$}IZ@L`)GDU2Kh3O?gYjVbQ?WeNMoHpN9{M zC`%&gJ(JDF=pTLfmg#_xDb>ODu8i?2=8oUJI~8xn58&zp61OfL8@VEswJ48KdHRBd zR*OzHl}o=%4Rr0tljAZ}<(1jm!fSB>?$x$+h7Bb-vv)1AAxq6i4}0DSl!eeNI!nT4 zKS>tsTucEX!qF}&*^X!;s8vV0 z(+MWUzQFK}sNFhzCJRl&*$yD9A!71*3)$1@>?V1kh!uu|bCvQ}+yFzyZEf^&5X)+a z)_ZPpa`+)dg27PpEcH@B`93c%jwe;t zBlMpWAc)vinI!EouyQosd`{ZGXZWI8cZs@aX{x5%oCt1d#CU&08p(IpBuFHcb8-2i zFd$1D^DeKLjlZ2uO|46RaTg_a0{aEZ{vN?Z6Kk>0=t2#hv2(ZcgvhByXU%Sgbo59m z899BAo8~H|b@B~V@A*G*&fD6UD({7#;zbUkb+~~`QvNus@PS?OoJ$&^UH7B7l!^q7 zOJZ=FC^MGU#=ukAr>8XIS@s(w@~)YSMea*zuI2J^eq1dXwZW;e#xx%~pK7gZFl5At zJanexI0t5?rl#XVK~)q-uk<92&sZh%Tw$pftdk=6}t}g%Lgudz}KKjf2s3%tvWS0;Kdmr3vyzrPPDZ=d;1Z%#eW5~JFP9% zv8x{awvv5$qpr`_B!F8SlVyto2xyp`MZ$;`d*h^~9Sut3+oqK+_lFYDHDvF@C$4JMUvr!EXN0;88fP#;l0|o4}!#PLLA#=KqJY^)VJ^6iB#ws!z)*3pA0r2U!Qs>VvtVC}d!*-h7|8miAM*Z8-gE>zd!))eO7F_lPtlV8c>A^MeCWlIhmEnJgF=7tzb zFpl2O4KAKJBP@Dt{q;34ZtNB+KDL&;`Y~rtyxjK{u9>%b^?5RzTv$T{L14(TAZYyf znr#Q2t3FwYtP@*stUr#50*OdKm}w%-3T{$HY-oc|4XJ6|Kb*u{I0w#IP|0AfD>m2r z=+WZbvi!J6iWEa<5Vp=qbg#W8COmTHT`TpxdPULD|80RB5qRa27)K@8f2_lu$T7Gs za`A&1Guk|Qw&`5{33D6g){-vIKB#y{85PkL_zHC5fY)U-`EHC;lkTxb?a z@7UET)I&}aeO};GhD@07eEV7PlpcYjxOUko%ZRG-Ng@Z%30O@wTX`~Vr3oJrN6+aT zg0A(xYD=~^Y_woU2b-aEi5L{~&fz4h1RBC!4&vbnq*TU^oew%Vs*?El483yNl{qH= zR8?&aDsIxOnO{n-eOuY}kr<(gQ&k)bbCQw|Vn~_rtaY-hS(7%K;9t;LBG0DE9hh*-Vminbw+N^DpYcIL-ae z@-0Zk%>A&%HBkW-h*#dKAGFVk011R4VX)X`Nlp@mpAczQ*Adngr3bo?)`J)L+Ktpi z`=(`?i3n(JEiJR!+46f_r1R6iv=e2vS|XW$FRxVmdoJKthaKEs_He);okez*j*gct zVr@=yTiv8s2oru!pdmTn4*;~xOFIP3QYC<%{Bd|hFrXYbN_SF(NwdiW8PNbxbf(P@ z1Dwr))2}5Gv447T66xh*F#XP*IxRCWl6cYbs7Fb_6B`~vpz?uK#d^zz%+W63>)*mb$c*PvaRvuCXaBYYv5>{YPFcS<%_drm}Lg3{~j& zUh`~j-nUBqnsxV@#3;*RONpd(gx}ZNL6c?QO(Olf+51mJLOlygBtaoln?9|Sz8@U9 zOnnx`rc5A%6%k2v8DP`7BNBB54k8W4n)d%xO(FMuLd|4F%(K$r-FPpf*#IGga@@Cf z(lNJV3d)7)!V3-r&P#8isFJMaL_TX@bi;~lAve86%a)JvtU?rvHisMs+4JZ+qw`RS z1W0b|t3{kT5kq2I!ah@kx_}z7Yl_IEz`}>>Hx3F(Li9$YfS$^mK90m#+yb>N-au?5 z(gBfUxdB{>xTnKLc|zA%rBk5qKqb1<0PGZz-kmNnDD z_{P=3QZopcuz-{@TmTDmnLQ=kHc(E)`SaCfyO82i)OL&IFZcILGyfC+#?pLQ{wyZd zlsGUzg!$m>m4!fRaZF1!DTfz%PTXAnRHGbU@EA~NLNS554VdhcCJlY};B+Yw+_h`; ziR=PsscO*HJf^Lg5fuI72~hO)vr+E?+g{J^i&&bfqTmIY9h zXFpj@JGvfvFhSchQ#+zISdglTg=>seBsRZ@fn%Xeo(l=_ErrcPLF&6F)rLx)-|v6F zS=(9S@fM>Vi>i@Amo9FV9Oe*|(!SJ)@ZL~22&*6pG$ z-ig}j<9KWyfow|;x53YgF04A*uXAatvy*JkBsn_&6ZO7dy5{5)%Mlkv)T=lrc1=jz z4h<-z#6~QfvDaWOXXzZCinIHeWFWG<=Esi@=X|W^!lqIhBy+5JsN}~oaP#e(H>L$d z4~Y<)LLspOvn*?6JugQhnyf-%wvz{ zqCODbt8?eG;8l8bvgNv4(Z)0J*Ow_7PuC=r>C_eL{P61~TLtx&AI4@y6_7WlomUFd^!^Cq8DWgjNk}rf4$oY>42ltWLVKH#qQm^ThLcqa&mS_ z;`+_B5G?V3+B7WoLDT@BZ{7>$t{^I^oUeb#E5KIyg0AwX0rDG2sb_TS{Px%5Ih`d@2`}c_+J=W72bE!(32P|H^d`r) z{rS(^Yv|_1)qeP3lSFe5p@snQI9=uJg$%7DO^Lo&diJwsUJo`6C-_vYoz2V3F)?wJ z$NY7+)=hu!^y&EmiudMZGfKcwyfzn!`k8yJn@Ks~oe>^VWH2`hXv8Nj|J)rB!w>8m zfSQ*;5WMm9rO;3`az$I43_NLWV?kSetW8#5%-hLQU<$X=@X#F4ndoSNZ!D_&r>jXh z{P-xQ0NZJ($~M$Pf&MgHQ;7UJl8i>?*F655T2!X0E>&6GDI z{yV{>7TnQ{$ZP1VCXxe1UQvCZCY@*8xgm~7t#EBtxXB<9i#aPHOgpov_JzpP5e9jk zyWi6@hrFHQ+Om2;A)TNjmSz)hf4@?aQ#Xj=Adj}jqb8i6m2g^N-nX|Z=PzwaE(^={c0kc8>1-nHbS!C2Yv!W2|5yeYA0Fy?84TW6%5-ig1vQ0Yuo0l>K+ z^K_>)Q_Hnk*{F#`*FXIH`5gQ|kx!hohCCs!*h^=)-+|p7d9TbLFlMtmmcoGnl@O3#Ld1%(x&SEyYNMaVTJth7d zUX08Ya5d0`g?DpZgHf%gZ2eWNY-V9m8ZOFGqIjgHkQqw_#SUQf8kuZ3aC0SbrvNoW z+{o+eGJgNm3CR~^&W1l^-#=(3L{8zpC}`py+1Ph+OJ4p(KZ5`ii0tbD1!N5Sj5JEb z|I-UAih6#fkXv%YjL*n+GmvZ;_%g}8u4&VzQUGFbz+8y&qS`cPE$lA5h?CWH)=-qa zE-l?@IoxMnw1JE%afM^6tK?0YgNY{g4h+=y_hy1~V>&mZb zcF!V8*MOm})pG%91ZWrBnJSS!{ulHC9CrhUB>qv?#@C5%6#d^_e=k9gN(9BLR(IP~ zSM|HDDCFLkMQ$Yn2zqm7n#*?6qxx>Si(sPE$F2qjX2w~`Q9CeMeZyJRClQrV4&Npb8mh0NR#mJH8BUL!FhCN6d74@4o>f_qK#-sRkso z&fOIx5`jYjThGH8%v+0U5dcf;Kv}k;K90m!z@wGP<=PA5q~{R)Rb>hk;C$!lqh`le z4y*}5r6Q&OuCO_+zRN*1iEiyJX|3#A1U-puRh9S7X-#Y+kLkIZkBUm08%)M8i0pu2 z;B$md8JR=a_U%_7%@~aV@~A|4$ByutXuYa5n%SOsCBp5UO-gu2k9;E79D-11VkGR( zWBMQUYe{efPESis7ST7E^>X9660ZjM#TKMVd+tCsO3U z{93@%14NkT5i(%hxCQt#u<^vztVvKE zKHL!KS)7U1$wnYV6wTz<7|qqBW?JSX?;uawWgUP)ABxf$VE#+%c{!>$M3HHRcfp6# zJpVU2o_9-$;+SN$SfGUHG67lpj$U;TF8qL*7?HBIhdzAxp!KkY#G^kM(ku#7X56Fj zPi1A1$Xtd&U=)ZHFS~BLPBYoj1{ZCeM5bp88|N)6)q#9ka;7JDilR$87wLGC&!-+wPmmz8Xg#C`V7TgWJzU)SvV+C zNsdMzg6XaHBUw~P&myxe8!JD}22kQ2GqQ4d1QoP7R8cRAOMIRqA&~lTGV?k1bO?xC zNakr-9b2kh0hhp(L@RL2Ma{(xlGQ5k#v;eJP9oH`g=&jFy&D}M-X~9%l!%aJbx%3) z@TDINB{p(Gv_Sor94~DHGaO!m9NwkM@TR!fq}ASSHKLj+aA=nQWGV4eEW#thgmd-7 zUS3=xPVmx*n(As{Nu{|F$3l-EJ$i;CT%i70^pAJ$+0(MCaRZ6LDXce4U}r+;Xls}( z!p)#3A+L&hIAioh1)73b^oy#`*?P%GP0Y__jCdCjyRY1$!G!C&%6{)Q>@iv{soExv zfJAc3;`|SJ)Fd^l`64MCGDL>;&M&`H8+9=AECnf%No%ct=NalkxlCX{95`8-O#=wU zm*F!LfB#yaJ+RLait14WzEX~nyckrfCy)zl3InNRtGWNW@^}}0aFHBnGPzKXDO~M8 z0+i1Vix`+GX%L7K%3vj0uO|tNMOz{c=vr(tvFy+p!qQ6&rc(`u_`b5*(*+wNHX7G0 zysvVT#*M=OpygFT_(E-Zjy<$KLAM{VcW>I(u9_FSB$C4qqkyviXafVUdI06nq4p%g zg*gm#JVD4L%Fqa_o^&I9aPCfo!z*YKvW5cv%!OJqb1o4&@qrvDK1XT=mF1rsshHa= zQY&XX>C7V(KmoSiI%$cYbz|Rr&ZzDXwgvDQV=SxLPQ08(71wSwlIYsQj>y6k5!7fz2Y}w0%+Wr_!~fZR5WEUV@4ip^|K5#o zjFh&@3Cg*;$IPzA{bO-sR-|XmoTeUuMiwh z%AusHpr~k(@JhDoEktlD^g6ik8>QH@$9&}#0#8JzE@EQM$I&L`^lF(dT^g*hO_CE< zRkU>xVo+Z_F=~-eFc+L?E>wg>GFH3p*$Duj`$WGO_QEsB9o?5)3R|~;)PgyzIO;fbhNc#?0~3G94#Kh#u3XGM}}*a zyC$ME1axqmIyWVzpdmzWxr7iERX2YN zKq%w|%j13MP!Jsi`x$UU&N~QeO9RajI4bA4q>;gQer5iK#Z~bs@v~IaKh+jXTD@%0 z#**6!JQgY}IhL6yc~oVagu1~rTk*a@Uy+G2LSa-?faa9bWiLvf z$*-tHRr&;)D%5uQ6ccNMqR^XFq;CRKJn#5!lk_KnVrr$}tJ3 zCS0Tzv)p$fQAplb*^)>U+dwe_MB;~n&3uAIsv-Y|UX^ZjFZeWc|G43TPYb8zZ{59fb%U4h6>~=I>Ct+a zqhNQ=zj({kZ{)IFB;`^-9$gb3Kfg$16*BFPwkQ3r{igFfMsSp7$=ok|g9!o~=7L1T z7g10Y^=?u`;ePEwssz&D^65D>g<$D>8sUY7fd_H8a7}da4zbXgR{&2-%M*$;ts?dH z^>bQ{xPB|eRdOAU!9uL-7tdM(KoBED5#76lG&Jf&vw!%Ao4+29Zp#4_WU((*5~Fsa zx;-a4YyU&21Nb3O$oTwfT6$}(`^u1(9n z-r_1%A|XOg=0np!4ZGt!y^X+`o{M4_7`JO~pzG@vl-p{j-&`X&F8~*-CeTC)o9?{% zsD=_>P$3GMhUG9MBD+#=g2;CN)w=nI1Zhiho{UyhD>qur`}*x$wy?!8<{&y~ymE8D zdVEn@f>u)xz18LgYH@A9@oVvKc>3OW{9UZD3xS`!dJ~1#^7Bc5Y}S|^7HFKxsra@)qqE+%3d$S>icC;l!E} zfu46U6s1l=FPvhIgO(%?@{c}^Y@*s~Mn$9i6uM?M1Va#yl-@ha?qXA=39d#$RWbbg zo1=3gaHRmvMvLCQe_zJTJ4j%t7+W%@RYX<3NAUMIYs=S$abx3jArGQXH$cJ4>#N`mjXrv;d|Yiyc@75+3atgWC(&~kmIT+ z(ako8e~VO287P2t5;2k}j?=L9r#LQ(e*XtR)@8sHgTd$5l^tz0PUXV#P3DrgK~M8S zx@zgE&RnB*=LA%i=nend&|E$n!H$yprYmWr(D z)vs2(0M|}pLgvt+El3jvh7uWLoU?IC5#?j-p6J8BH^q;YK~%{<0(Dn}QK&-eXoYev zM7$(eEHboTqP|qX%OdroAQ?Ul8c|%H^-+N*-IwG{hll$FOFf}j)MLr`yA0&!5h@1O zRSaTBHc8P@oQE?Ws^8>g4fjc$>XUVAe(wgbO{%Ld>vC3}uY2^TL_hey$(t%8M*M!3 zipribR$s=d?$&L7tG!v7bjVCb?)O-Dm#$qyA^dGy<<%gl3llK(3ArmH=A2NQgpfha z{pct#jo($jYqy5vi(2HGl3Bl}sbzeV)d+NtdILcYn0-8q zY|pe^t5(@m6CriB+JEHqz$sb}a}D>F{(z;`~jc_E8QEYSFz* z*YlGu-6qvVM51jj}rRsCH_m8w4`Zkn&${1J@cfq)2Wz0gQbN9ysqO-KCB9c%r z1_etP+(9BKdUk2qZWLF_%|1WB8iVKPljwi5A8(vl0bW(YcY(j@0~;r)np|E{L+VZ5 z@v!KTZa+tgkC2hhCBl<73c)PidyR_OorJ@@UI<5%#rjprN`zM z4HuITgzNnNnV~Xd$k#uSrG=mqJtZWht?ym;)O+_zj?J^a#Kx5Z-IX%o^4m2RvGkvc?HB(JE2`5n&uIPJ`le@xNnw%ixAycV1hap+NUPw4U`bpc!HXnqXT z70zldtaoo42-iee7DiNv8!~x9-hs0FLLA&Zv-zjb`FwlYs)?@u<(Y0FTdwpx{cjtC zZl@)q-CYOC;#oqZ1ug^TjA^NI9v)w%b=gXH-`wHPOS@%m=yC2kW}A}FKRH?(KAj&F zxcM;vfn;++@4p4S##TgIi|c}JSQk%HRNbR^YHcb9)>Ka-;{yp!u?ec0n(?w*#+4OE z7Rc2*FmO@%DQe2J(O8NYSMXE^@eUG?ls-5gv47!05RB%oC*&`sk+_pmik|@vB-HM; zWs4s2W+z(SIwMH1BD>9a3L$I5%^H8f*nzjHMUYsHa7ASKpYndMDYCuuaJvu`q>J*$ zH;B>SsiDBHljH1QPJ(Y~ov*hXL|VjA;bM3Oxbrg}A=B0Bx~nnO$uIf0TuGHVlsS!* z*-34|3SaeH^yK)BiB54OCJBL=wgncO79u0_qUo#QmL8}7v7SowWIWL;%;7g?YYXC| z1pPf=uTA59s)f}RlP|PXd0rLRTB33OWS~{X22Z1lBQi4NrakEUPYW4@0=lbY)whw; znV>sZMWfHqE-z{|tWzd@7Rf&j>RXOzmjK9sGW-DG%p)Y7V)DZYC^;;1T4rpRzf|v?ndk)}$h(-7m z!E3h#JiCNmau`XKEd**vdHn+Ygr$4AifXthAPFUyaKVvh;mxK_-Qj#&k~3Z;l;{g- z7o*f@g$5%@TAZ4K$8-^6(C(M4)_*3KjWaJGm$=&!*A}f>)nvWk4{&s6Nm(4D#JS<( z_acNpiLz}l;l+$21TUfvMqLHKa0m_&?^~s9ICJJ+(b|H$opl6E%lpKM!J*khLbJo6 zZE9Mr4g%Xb1D0`8iYb=C#&KKoBIhTb{&$^G^_V(prd^4LV($n=@jsv;j=W5b{hGnZJJR)>Fmf zHhelI3f&@D?$lZL0QIMA0mXy%aW!URw7#zPu4zY=ESqL=;zD(_f8o1Ycy|$7p%_9} zp`cv`*m<+;;(~~VHXf;Rrkv&_qfVCtF-f@rz2zn{^V` zQ20lxobPUr;RDncP_FmYwD%Q}8o}wtVK05jGr__l3HQWNA6wK&y(0}TqP2>z7s^d6 zp$*M?JxeMB9Rz^|Yz4QrViF~Gpb%I3o&_I;3dmEF@enl$j)DZ%Nl>*G8xd=pJuu@3 zZ-UcBPFVoD6I6C5pC-W07>kbU9d(JjC)_`{0DutgPU&15mh{f9d$J{wwDIP+60`Et z|GpaXSS#fj-Y3IWy}u zFKish?eWf6WgCt67KuGkhCwAsij*$cc9k_>cV{FZv!gRtEjk)!Wbc~FQg2X6oFhj7g*4=}- zRX9C~$7j+=Oj4_U z9*V99#!8-#Uz<>ni*!4wv_5CU`IIbIQVhWKC)Iv0U|qiD2xu63l7n>G9H{FRRkEP3VX2*f=^ zD;ZfY%xK(z#l^+jN_q{MSqdm42&TKMt>(|28!X0EkoLM)s>*iLoZS==ASeHS>0xVx zi^^P%4S_PZLe^8WhKLVU35c%+zKg>gFw$@1He01d3$yDQ+vZ#LBh4if?BsUH!y3+yR(E!C2ldm@m4uOuCC?f5W(?0@b7{Wl;P0@mzt}GL0sO zYMVT>Y8Ylq1mpJgURP(NK|~c*&}?o{i~wgAJV5F@ri9Br(Iei&nZ+-sHg49;j%u># zGYEAlPHAIxWAg%NR|2OG99ZPLpT3-p30VxjkC$~lBW}~%0 zbrTD#w9RQlBi$;y`l(a<_Puu}_rW#O00r@Q{bzfnakNXw4gt+cmh#dOc@Wh3C)gNS zcOwbV0xzAqD<55IrYMaOHani;`<^|nPfw7)5B~oNW?E&h#bVf8rb-kCv#IdLmI4{R zFK_4>M9Qjp>gv3Z$e#)#_c_Y{YpsvgFASz>tv=<}+ZVclw_vmU%X0h9D4EtoVDfIE z^Sff?3dXSo$5gfLhqTh8@(yo6ILPkOc74_~`GCa3( zwqWCJt=p`m<8Fz2Hii)s{@L+rMP1Ejv2TL!+3H-Q13Li03Al5~Ew?~l%Lnkx`~6#R zNDHr-i4l8(;|yt(YU~1US;ez70fecH$w6k6n?vIfdm#e&bb8Fo2AMzLak^?ocnc7O zwOEKu3@G52!LTR*{Ub#0!!Y(%lyd+F{fXhTuUxr;Vpip2!0A{-t+Snv2zCsqN|rEC z{OPrWav$fR)e(-0=mz0r9XU&sTu5J@9BG0O>Z_vi>E>QRtr87k&}3&+uQ5qQXH*Uw zf(tikdij~Tf#?vGwfZ3SB%1WS1yYUc{wg@_U4~nJsC+b&vKApn-Ha-|6{}YHr9f)= z=j+#vABr1*O@agrTXe$`?u2y`xtA@-_=I-gJCdOL*sLs$(y^mQmqmzXbz!bVDAHn= zg|U(Kxu`KoI?9c7w_L1TdDvp-&+wDVc^+cpA#;9vsv$S$Iak}hs)IQke&w+&PV3R< zhUS3XF)!zAd2)T@dclf?nNgL~w&&O&f`dylH6%g*kFWm#&-s4)|M3e&jaG-GYD9-s z1TBdb617JpBocz8wj}lzGN{p3QEC%nMHaDAJJH6TA&C$p)XbHdaY_kADV z|j_)1r$lFas!-n>e8!Wy0E0p)SN%K}kft&f3#X(^)@g&oyej9y!V3+qN~R zUHkY!S5uS~Fn-mQ%a(PKQ)Yeeo9VFPUAUXq2VtUXTlmrqD@E|i;SEWq6c2TO#;|{4;Dvq8^pZ|Y*w0o=XGGhR!Vzki{P6&bxv%; z){!f7BeTc66W(%^MboDCZrgV8UrUBEUc|S0jT(hrPGUK%Ar%Fb32>(K1>RlnBa`WtWYts`1tWu@ z8{tET)~KTc#k>!oT>f>Gb-n14JRMuN?!O`U%>m>L(Ph2*)^JMv6rlmi!b|`}ZT#ra zKbI`oNI=2>2%S_!7FjL6)@Mo456<>l2SyyqZN5i5`-w68qInSE>8PoJn0-OA?7sf~ z`%BH_Z>(1?WW#5ApTTbfMsW3d&wSo7*10j1lm)*Y9t!`-OF?h+%d!CElG)bH=!S&a zCcHjB_GP+%>^oBm-EPg?zJjB4-^o=?V(lJzogX2U1O_{a0;YE2Pl-o24548)^i@~l z{5=hrg>E}n)>*yNwN>`{rFKK|4_o{6+k}wHPfvg+Kn4#hg7!Y)&*0syu`&Fa4%vQDg70}Vd+J_gYN)J9sYxUpO2PXatjBC0gVqon~na&*}|b~>wUN{z{PgU@5Ns@18c^b=40`2 z1!R&Dm!`-HS))|!;zN}5tZZ8kT3k^YL2A^+#CeXr7dZ^2`` z$`7&g8i(l<;x0O3@QJ6pmuEn{WdKaDl)nq7i9FCwYTCBQ9|pzjeMIJJ zxN;9xVrkkN&gY>0pL++=~&u?%pYKZatQ3*kBGpl2j5b%Bw~-#rjGn@9wSHW^9x(5M%Q^|13NLX+N?pOY z`B#QNjLN=2!-nUig<1e&gZhD1DAlcXP{Do>I=Ofje**~&<83qzV4$ggjqZ;jAlInV zFlij~?@1;RGPPJtLfX>n4;HBn2^o$-4tyn`Nc?7MRvjkB46z62R=+FGaI}$Yih67! zAAMT|7YyNioJ=m6p4R25IoWkFyut?K&VrA;5#8WBsx6WocTgR2kXphm`O(UmmX?+3bnqrzZ2PrQO;ef|tGUso#5Q z=$m+tqUzn`oEEa8-N9v^V^}vkZya6GE@#7>fXx)oQd2`G-;IPUbAi=ahT#*?-SqbS zopB=>a)OA_KQ0g%60p!nMO3g)inOQZed3JpfV7%$?-y|8@D~%Z<9?@lH#RkC>&?J5 zF>v%ksGUxC9TLOtmQBjT^n;y`lm|l4h!-fBT)=gBAj4 zB8eWmA)@v8N7wdjqd91E7jGD7ijR0S3b&n<5)|3nFbTNW0)rLfYcpX`%#8gKxF{;2-;wsz*3|MD$;GmO+Lr+m3`$k>s%|*5S zlJoBs%9nTV)oYNk5UxIFA)Evjkd%%Je~lR#uk7LIbH?toANk_mX~Qd2+O--9z$vuj@|J=y zB+n{U_ONw!LkJ|@y9nyQ!aaC#8lM}*&^ zXV8I6sl~QS>-O$ou;CTZo%4T|#?M$`*R-jd;B=_2{ZGe>(zpQUTlf#MP_|nRUU?BAk;V|-Bv#g# zP8p?4NjtgZX=+llzkHNWf!fUfN<%uscLzdQj%F#e9Hu2juf#ZjmU3Le#BYaHnP1GRy?LX^By^M_4Cuavzqx6QI@O0#Xh`>;24~;rT z_y}iZHNj#jR*P@z-nuS-MwBn{D3f^E2R|}CgXI+W9;e-iC@`Aio0^%-)wm4!9LL@D zg0W9f!gWTe*5fYQ`Hf2{OApR+)c`O4eCsmWel58~VPFN*EK=K2qeSlBy*qqJ(Gm|^ z!cv6b=p)Sm)54Fuey}#+?!xA>Qbtn_fQ!8@RTB}UT7l%1K0%QxTLQAS@7v4X)q=no z-E(s*mq2(Q!>>dZWD=Pd*Y;L4`C-6ZZQa=WKTGv4Lx<|E%S=->_#&5Sa*I~2)@;bc z=0P!{+JlPWnVITd{tYpLUl}b)ie^k{0Ni9mjIp-}nY9D&z4mz2>1%yM<2}RsAHQ+- z5qk60+vNcM_G+= zjEi%53>GMhH<4hz;&;yh2sdLaMj#WB91rdsjAx7Q5d-M;-Pd1lT=cmf-bd|uB z#RG49WoGJR2gcte%_;XePoS>#{L)od19PYf6eG$0H`uY39O%h1Y$*STTLX+?&sX2+ zbN7{ZMq}wiy^vV7N$;Y9d9DQ?AIjZ)49U(2wwK(J2aO1=d7T=|nvTpO$|dTvG{3Dh;(t=>gI zjFlCClL5py=Joh5M}iXEijtMq`i{f@y&Z*K?uBLawC;_+s9bp@rGNMuo-Thf_(g#Y zK{I4%`^$ST!vwPzcgTOG{_RElwQJYH$4{CxY5UyS0Dou~xEI+nm18zhk$Lb3DUhZ@!mY-L87e z&A1gzVZHPuuKv))f5#Vn$@((i;Hn?xej|hy!;OJwy}>iL$>2_}byIlEG*GR}zkK@b zUT)%=p>E#ZJJ4A8l3YNU6P;*)afzRP@<|xSp)|q*<7iz+o4>HOHZv>FmF}Fm;}L`> zAefD?1vruPR>24|>++s4>ry{sEA+#YKoG@;)h20mjqI)3hIp@toja4<`%2U85N zuGf)(IBG!$@VnTo?2C?R49>%s-+S});)O72?wmOhgc*pT+wye%bY~YA1POv-0|E|m zt3R2sj#CV1@YC5>C>80^jQ{ygs-!|7Bg6Zvqqh-i0pR6{hQ z=S7+y5MLw2;!)h9&y~iir`!0WHJ@Z0g{aX9CmCejSt3I_u<2b z*VwA}Q#G8jUFrqtdEg8W=c#^&NZrhuJoR`=|35x)8tu>4)p#wP0DRYMGq21K-+=t2 zlhJx7p9s~5N?P?gKi?loUU)ieitN2)<3za7f5DdYh?9{!CgvQU$fh4%l5as`>4Xw} z&tAOf(zfk`hjA>#nz>nUj4>@IJ~)+;QE;QZoPvNXPu3O{v`*WfgRY=_X*T%Shb$Zy zslP0jyEj(%sGJmI9B>>)It;YhaxY`zkxzi;eA=RfUG<3cKWp{mcs&E?fb~+H~&Rxy~0<`t$1syU)N} zXdvEWE}Du*n3o4!tktKVK6g6#eZ}^Ivf$L?80A{{`eN`EfZpwxIJybr0~6btlYa~K zCTHLesj8xoZ(ONcBBV3Yo2{aixFN#>VC!^?^MBLw@|oYX%8w>JE^oalB#*R_9(3Qy zbZbWfX!Qb6pd5K}R_94bcpVA|RX;?o`bUvWNLewR=I97|*_BglBU`e>)F!&1%CT5#DE(Npmz1x<1eK=NnJ{lfcM`n?1E*J48kG)mjtp@{jS80Fog z?~EX(IpE)4>%L@WikI>TRf6L%k3ct4X=V%pbkV%@kA}#;@L}+qZDLud9bmlrLz=|p zh8FC@;3RSNyyqGH8>SZtoyF-bC8450(fRxca5FsMr=P-*d-kJxNC{9BeDcLGDIY7IS$l02F41JM0!*R^KN79DRw2teY$Ki+dYUuNsbRa}4d4izT2 z?BmRo$Or8sSg4u7=jz@~OW|CkEs)X41K~-9j zXniX%=ceBm6+cL7!I-m=u)TTA7mWmEb#HwVA@bAUw}(7-42WE}Ze4M3*yL?sHp-Wb zR4D+p-48ct;AQ%% ztxPEQoEJ01)7lj+>z@u#5GvIJx5eegLC-4`E`Iym@ujSF;7LIg`DRE? z)6L@{Wm}JZ?odDEgzv-cgRz4Gj~Kt2xxUJGDf)g9)k425Rr&LmKP{MUa%yk&`Mu3O zmvFy2eMGLJCv~rL8~{S};lnq$HcSP3)t1l|6U^adr!aj$ z*A4oDh{oI-6hSE4%Y6{$?aTQ?J=`M>S!UnQ%N+fd8uodsN^4fE7~ZR@DfRW86Du4C zM7(`ClB!Be9;V6d2=IyyIpDjRc~|wbS;@{o6JlY&Yd0$*Gl4wmmp3L}^rf~}I3yxz zn@24;V5C!*3SU?zevKn<-sB|s!{!~F_Wa*(Lnr3X=a$OUIqL=ZZ8!APAD!FMcn&MF z9)Vsji+u9%p#m%w2rzj??PBrCjy*YLOnphBdVKScr(CvlU6It`TZl;a5Vy;=sQ0x# zYDy&*Me5Sp-3=r~ClLs~Cv<|{->wgL-1n@vedlF~#~KWnYjx1ubv$vm#xEzk|K4F^ zxn((^?u`FBXTyi4kFNfb`-zQd%p7zsqeS;Sg3D%SkQR0A@f2(oiT-z&Fc%|sZVb1= zHoi6pQbbWY^+#)pLyS;OpW%~42jlWp5=kCm;xy;FW9P#09$`ihu8mG{6>ebPmb47v zz+W-~oatuWQHw5VKcl*$4Eh2o1eEbFtbCJ}b7Izs=mtMJ!c8;$)+)5GoGT=}Z#jyc-?d0a6obu=fg zT=t*erg-|!9d)!U@kR6UA}2R>j2;qZO&MRWWy^ks<4WI)yT|=dIXjcdqXJ6Q&s`ig zqU4_DM~}%9mwLw6?I?L;3i80qkdq#xzs$)3e5LDDsEHm1J=_0}9HTh7f;qcYj*6tD z*~M|1(oBW7L}((606=`X+zRK7wxBbGnYnVRrD1&GLS_;p(F8>BrUM96BQJ&eLVG1YZP`oPlgZ zdF4W%Q*lNt&_HTnpm&|IUiIVjt=tdO4W*&{9`W&`;;t=QclJBC;a!(YM&vPUg?r}A zo(EcWF0J0z`1MgmPiNydPDcc4e$%!?g~s>$@o)WZ{F3{V^|dj_*xWIYNz+fD$Wh~u zRBVuPZh{7|n(e4MA7>1?`;Vwt*QX<7T()kV_>fA8Dz`b3 z$%=AEd`L9hOrQ4wbSEu1IYH1V5(ghF+iR}v`Qh^8Dm(&?5Iocc?_)|?@LsQ6MbMoDMw6>dS%@RLo1J|9mWsLWgs6 z%9@-{1F#<6;RAix=JmZlRVCM!K~u)Gk=-Z2D_x-d(girJjc3EI&_q{2#rpk8R1&gM zf$B7D@}owlfg5wJug_nyc2)hvr1ihAIqO6v zbLL-L)=b1%g)?b{2ohbmyx9JKK*$k`cLB-&BtmMezIdCn=?^yf6^xt5+SoaLF5H)Y z4mg2@nX7LJ*S;(>4;h-<3t$qF^VLpT@DmtP=`;i|rwDJA z&lqIz_3P)=i?>kKaIC!Vg3W4i970z6P5ouYAvThSV02PIscf+~ z%s)**S*Fa4*jwOMI(^qWj+#wMy4#oo*LAzsopAPMtEvB2tyJdNyKXDQTyb$F>wSSG zM>ZU_bl&l@8B=Yl#&eXeJ%LGvl%3rjeNCqQj`UaZP0pJ2pzNnQd1McUgLHSn9!7!I zhkdc~d%f>YUy}tW@s&CZ4%6wDi4!L59e3wPBDp*9&+5875eC=^ea2Kp8JlWkTT-c2PmKQ>-M&*6))6N8HBdXZ=%=6MZyY}+d zjHj#H<}fAa5fYWVYrai7rn4Mx!K>eyY4LJ_Pr>Z$O-`kptAl>PQyTMMYxWlG)cw69zJ}S zSXg)WWrpL;%IdgXGgEFkgzJ}Rs*ae2n1IgNc6pgmZ0-s;!M&14cReDjWC@dBM^02OcSF2BF_rJ@Vhzs=WCWyX+-d z%d7p=T_UaG-ZD*VLd=ZZ63%fV6lfy_Rk#lrFv?6yK^Fdwr5I@jo=`znb@{n-J!E); z<|7%C14ve!R;>=oisKIeSZox|w0v1mQvy|a8Y!)JUqA3fk2Gx10kjqf;m1U24FHlU zQ9JK8kIxImZR9JMng(s#-r7X9eFA3K@^{&qtNbJb(p^ej{&pP8@CZ{R!_$KcP6nr> z2L^Lxa&JHzM@HI^+Rj^?aI<(rI`;%qC{h=|u#)(CbAGFyl+V0AzC498WI^{Qb<2dj z-F7f8sFpPKak&A#sCWrkM;NF@X zd{Lun)e!NA-y+UlN|FONpB((X96zyHeR)oxp@f37)}~p;-q^p96e}086~k&TsB2od znqEf{9L;Y{Yu|oe#U->XY3t~_>5Y`6&Oo(-xoEsAf9k5HCIB}u{B_!sI%~x&} zjg~M)EQWtbWhT4xnr2<;W&&_Za^7#=GQbEP=>A77S(mc>j7m%{GB4mpIAwT z0>^m;A< z-;vigre+_M`#Roieg8;gld8?R>4bB;D`ragC9BwE^E=OY_=R^ z=5E6kPs4er(PlmHg};GEqo9-uPN7HM?=q2U&U`kDHErAF!>~rW#`)5UIA;Pn!(R9% znh>ElObULteI@i^Z-4zB#jv$arOH+oU0c{^Q6Q01s4?wbKd4#-TpDJ=4OPcY|)KqGa!`82Mhu(fH#Vchapx!XF$j;xFX&2Q-h-QIG3i_c5@XQ2Lqp!H<-`0_6U z9;_Zw*~aEc>cul>F0*mNsoQq8YB2&*!}g4-K$POUNI7`0103#8gWEfDTCaDYGa zOp}ObP{N0sF=qnB0E^GBjdlUQTz!<4DDIJ-DY7l)4%MT?mQ^|Nq`W7VWuL}qD7|Dd z%6Qy8m0sG=4MRW3C??CA*dJ953Sd#|Lhl#%e0zzIVU0+th7JhWK3+2RNz}oITX|fNuvJoYYlpnsMfHb5mn2$sU%( zkfp8G9#UU6T<+C(29d`gm1lZnI5r6xkKW$df z>$5(7)Ga7@y$dP%lm~bTg<5X4>LW{n^(q1v4CrC!FKV};5m3^nVx3Vsy(@FFf46Dl zH6=gTG2KibMzAEM%T6!wHC-c7iVt7Y#AHJ*dXt5WcDNsRslLhIXf`y8{YAWNR*u2( zthv!qo;aR)hY?dY+v}Newd{h!fv!!H*40AjyF8WN*^j3jB26GAAvjpwVqRKsp{F=0 z#neL{f+JgwJ`H8`27RI*Y>AW`nknbY?%qLxSLxGMf3&hh*dP*^R%R4AJ}{oS2;^vV9BFV?OKF7oUH!|F#49aO<&@!1*cocQl-CIKz< zB3?#`Sr|x4B_c=tYGf!Z86$ z&+_d_v8nhuJx&|~ii82;YS_&{!W|o-o4{w6EaXYFttkC&J$md#BiI0ik0#2kf6i$E zAQ;W`x2zva&5BA^L0)Lr8wSk2;(@3ulU=0YA z$AfmJ&3xktnnv!Gepn6wO+p8*I#(+22w#9om-9_}Fe6!=O6s9JmS3JIOdCpTnGF8a zW;#=L72q+h1#rq14wWa4Wn7;U)1pY&@W~(7iewLgcFsI<=)oUo@~Y@ z5hw5?b2gQ!LqkW$1izZ;K!BmNA+mrfuXDwdS00C;C z+0C!aWU;sBnEZE7(bqOr-Ab8`Qtxl~3M52ASKhmu{jAz=(B+DD5%eroLE)=?38|D; zc3{)MiWoa{W-W4qOF={C2ilsKpYe8wY(#kl*%x(VlTAgW;IK-epPKyIgIBL;*l!c< ztlJSP{58#dmvU=%JRLMZ&~tSwtNU@CO7RXS9=fU8Z7-IP*o^k*M^%h6h-_+6yp%Dl zk$1dFxjaVT$F}tWQR$PHB{d5@SKn&t(o#pKR_t1LcenS%Zos0}ON`>n>E}y!^_@m1 z?)RbcNXYmE)b4Tu4xI;@ujc!1G5u;%AqS~lT@m%IhBLfz0JVf(_Uc~J9&J}kP_Onzr`Hk>Sshaga}TBi)+JDtF}6!3(rfim-3z~R~W5JzDD)He(>u?nI1T3*zQ_U2#NYERe8y;79 z03bd!PKw`#=!i=1kVHPK$;JcYyR;tA#sL<(0^3c1YgbxzbGp2?Q5EOF>e# zosFacZDzo4-o8b7dGTYof5CoIxs9^yoAAz(ej%bo5IVrjYy}U{2D!cEFA;;C3XrHF zipFr4(ypEag~}#HQk##s16r@c54MDdngcyr;msN~YV=pF`9v22 z%aIGIFr+mBsN8$;7L@nEQPY3vJ8ZeDntJ*l=QvI#?fSSkw;+gMi1|NN9j5IB3FKcF zM+&c5qb$yRo0Hg~hYnR78gSLpS`>aDVH(Xym<4XPm|Z2quHcvUBQ%R=*ff=EEka|` zn8&zjXbuN`o8%Upx@DRNzB*pfQ_Qeg4=ILJWWW9uAg1__HEp&TfVL5 z{xK(Msna97IgL90_TlWx?SAWKRe5sC^K)Dmd57Aj<>a?~m=_eE<{64G>-QmFR`{{r zrb(rj-)i0E(UIC4w|%WF)^VrlVkHS z%jLHGr_|$s{nI0^Y`rg7yTIXwgZrHl4)rSTp^$n6<~-~%3#A4PCLO|Vp<`CfN#N46 zY}#~Bj1n{{$K~x%jRDD_kbr=jYW>Ew&|qZqB2@q>{pJmd;16-YK}4)0#0S{VsKjbC=l63s=wGdX&$jQA?_wC$DShc9oq zRmBw5HR~7C~l}>D;=nio6_!vO~XjjJfH2M7s4F=B{4cHR10$U(Bxh4=$M@1GEmUh^l zL^p~OPGhj_Y##ug*>T#s$mzvj-mpi}7B=13AoP%O&%FWx0*rFD$ld{91$_~D|f)fF%vbvM&{r03WuAz4JuM`2S=U%24G3M^l~ zd=Cp&1gN&HCpP%ZY`cN=)FTe&OWk_?GY_+Ukw&DG6Xl~d7T#m^ruQMbPUq!5=h8bh z_RY6&t}NR~o|4wZlo{c%$+-7ZUK|2MBwojIqQv}n9>W~nk2o5aCgoT<~dBxuwxq_1;R_lScT)SwjB zz)}TZw<&m*7!+Gb$;II7=YFRJ;-xevLptEefrqd$J15jlZ3B&ZD=2?7%xS`pfBsx5 zqL8h{jbout_RoEGhsW^ZdN_0X-)RTD6cx)%lA)sR~4AIKl+E_?9 zeilWt7y%i|@MI6fMsbxr2qI5g`+1A^%`P5u0{&t9^lWd++b)M_gK z+wKEZ;@y%R(S*kn!F>frNd;^|JRj5_NbG+Z1bSpF%s;|3TVq%T4NY|MJD*yMq>T?v zuA9fN7xE4ES?RgY!lKeE%=*Jhd$aC5*yy!##aIkrNr~s%TiN_X)}!A}HltVsR&PMR z(R>oX%RP83Wb?A3_ns;>0XS*nXt;J~k&N8twYsu6{=hK@1JomSAl)P-D1sgLP*u7i z0e?g511v~P4ceuI)zx|E%3m^1QSlEnY`;@~4<;Nvm6%FR^Xx{$!~v`uWcm^BWP8#7 zMEb)OZ39iyxHapoLM~i4Xn-hHZW2Jn89ypDPmYSzuy&3Cz=d_9e;h|1AQia3Xibc(!<`d0=D7TcE{3UZn&}d47;j`-%AP4zR%hj=MLO;keWtywQ02hLS-OoO#)sG|$ZG_nS&CXCQJ41&7xKWJ7R zXHr5uVq{$tAD8W}M&ERFDoWgq;`dO`8(D+mV)l<8LU2QPgICig|8VH4sL;*Suvcym z?^*iV=y${1ojyvm`c^-~6q<>I z;|vpuI88NH9D)<|6eb%2uYrFhA|7Zyn~lDk{aV9slBejW*7u|<7uzNSRpz8Tix2Ba zAfc*N#G-U3J7;q@WBQRqJCf4x{+Nk6kvBN{AQ$yL@Ogw*u~Q#07!?!i`NN)zo&KkT zDMs=JKm9=9gug)YH`qclq{I_mM9l(mTlAtIGlwGkb;E9V8#6mhhVspw7WSA7KZc+= z)G-)ZmJ(r1%Gi}E%>ZYBT9NemdsgYZIXy6c^UgQFIHM#JB2;fts6~C zY;`M*TS@M;?E?<<4t|rp!NR1(Xw}JupE+3?)T`&pp%2#4!$Po>0$j?M zk7|?c@U9#P|Gu{=WX)YEA-Q=A)^A5YjwM3c59+RB$++D`JgW-CV7Sm4vkbQ`+( zW)z;!G7l$ocmMZ(&?lyXRiDMWJbe0t=U2v8GZkRqF8bX#lKqo5+x}D8=4kDYO;oaQnRyd6I;Jlz1>@p=_%+Lo%teb=2WY^E&!#nSk~{*O1#x9LrT;Cy}B z5pO}(DEbD+-PZ;+S7WjgzzUkJYhyq4>H6aJOyw|$+{PihF|Sm~ojq6(%GHvv4+$b~ z`@y*{C503C%vBK8@@sxjNM1wU7M*Uh)I~OJqG%6P)Gb#Tg{0K&G3!O5)QLcetJ7^3 z&%Ze7=wNaOQ(grefob}yp7U#>kDASRH7a3Z7jReY94b{(Z;PHnuCHXi8iNrBlB&K{ zUDDPlFVdA!v(jp$4Pj)Rr}y-Z(l;7WICx3;gOShTA6unPrJ++n}>bvdi{oP-kUYtK>{60YM zRc7y4Pi;K9$c9E%G(Wqb35`9QCVKzX!%ePkUfedS)55C#mYt~z*{$o2f7<$RL$J3- zJaBD?<5hE|uKI(t=%gT?g~)MrEs2#JQ@1AOGF`L^w*j*yzxBU}h-Cu9@#>s2^3Y$0RzLaJ>zO3W(OOO01IX&lDi;(ok ze~0(vpJ5i_dI0?v*etsWeqsn&f<*}d4;obx$7oy(CJ~#!m%TX4*8To~i(kLocNb>W3=APUSD z3db$du-mJ%%4~So!_D*|7eYlPoIsvMB$KY0p+kpS%Bmv^sj5nzr>+yu$A{lYl~cqe z#GQ{^H~9Oi)lKzJ)XkjKa~F~cz8?UANNW>(tF_f#&H@97dCz8OQ*Y^ zNVf_)5^#G~S^B64yA@}$oUm&$5vYbt&$3a*DB8*Tps{5lLwPAE#Z*D9~4PwX&IZyM!VzVcCYov9t8vH2a3g zt(%1DS=Fo=`|{<>-Cm@YC<`3{qm|tIAd4&G&@*#75R<^L>pTMK=<+#hR_e@}`)klT zWGGaajItmxqODto%-aOWQPDp`b!Pl(IF;*08cz2G$TK+&{CaI#1Ljb#ZCh`yqowf! z&%I>(K-01$E;a%Z*;gh!RE z%Et$c%>sD)!;UYkru=ZoDC?uwoonRrwC|M;`r{OQj^jK5dia%6&OR~e>WVuPT1{J3 z$tm@P)fe^udok$Rv}sjxB0XchK~Wp}=g({_ukt^}DUZ01TkQ1HK}ThCqWl~-TBzmS zlliuvL7d0H$I4Tax>P!Wh^K=G50=AR@)vsUph7xbgD5Q;ovP}s8}@6Ppxi-n2kO5( zaGemZ?((v;ZC>}+&_1W-#dEBj<7m6;k=B8`uc1(Nw|uId@aG?z{B}IGwl3_6;>Zvk zR5dNF{Mq33zY`Xew5gWWkSg;uzM1{|+9HeqQ^MW#b@ctonqLmR#4=muhdq}KXn(%$ zteR+ysEK0udIC}N&W{b+9Q=A^@Ff;UDo88VANOg5Z1oc<2yeeV;f*CWFMGq=7zKg= zugbc6*n6y1{O6{evyZ3vYhIuxY1>#^hbzSc+HB2F%fG5yw{zPK7qyc9BF@MuZ_18+ z^kW*;gS9YR;J9;Wt3I54%l*@;=A1o9e#+&MW)mobj8GR^K)}8otZ+AB9nWoc(%c{Wgbt)lFy_L9k5p z-LMs$YkZd@J>tNz7JX-O+KwK1q8fCgcSbVryC&u$y%|qLE^FWG2)M-4fo8pX-Sp6) zKq9b}h#6GKbqZ*jG~?-!Z4BnpmA)LMau~}cOHM{s$ARkuh&~B^o1dW=EdcmF6MxjF zwY1}~R$Us!HPeTyftZ7zmV?B6_tjTnF^{ufOS?+nE{{Y#VRgk4j#{q^djtjVid%E4 zNkxJ1RBe|2rYGQ|(uW}CD6WI45YO=}v0(h&3`jWjNFM2ctm5i}RS~7510#$e!jPVc8^XU~V@v zC;4%Ipy&RpUcdXN7!?wi_|HEY-U0s~LL-cJN=Eb7GuyiPNK$bjG^=@Cqu@@N%2yYD zr}V*uczO?Cf|G-W%$!woVtu+YTJ{E^_X>(iw~*l8ySAY<*Dt?ckNx6g8C!iY%>U$N z4{v<^`iG|XK*^AD@UE?AaOx8=ayGrX8`H@CqdjqjzwHqR!6*99Fr?G3fO$4>J3`_X zDCC(77k0de(a(-oik&p`RFiHfQG9}FE)`YE8>Fr92zD~9@MD0hTzG0p?d`9`W<1PR z+{xG@mKEZ^{vBLL1cb?TIy4%4b5In=us7HU|NDZ!cmKYmK~j4A&~H9|$;z0pUMFzK)7N7bB|$;1F;lGqT)7LY z?lv!-D?fAS$JxoWQ7u-xotqe^r)4tL8?vX^jp8({+SQcEE1z1ebtq}7Uq7Zv|Mh#y zo=K+;9Q?QQjeR)8SUIcxkOACVg_fWjz9O5@$v) Q!VJD)YI+J+XsST&9E^xYK5%Vn^kN17mArz`{QEdG z;T^j^QMvzaP?ectQ~`;~d@72q5mu-fjYQA(hy!Br!=ZhD(do|jxwkYdNoEYD$vr{@ zYEZv0)S9^0bK9OD^Pf+1;F3<}idW#G9IU#SG+4^a^oieHOi_QNWu+>Z7GT(|4cxQ< zUSRBo-8>EmXMN1U125=QxPpvqO=JR+%+9KY5o7(#7hMzb4JynvLjyQK^s@8&1K2rX z8tg;muQ&YPinE&!)jrp~lqu)#m|s2j4Yc{6!hC1HVopD7_qB@c?|RK^(CVtja!l^f zW9Pud`|Mm^sB562!ILw&9R-u~V}MoT(6+)*4m6(Uyna94WugUFX_SxVs1{g%4NL3> zhQN^3D2K)@r;y+eq0H$_V)Pb4v2(H-qekLeOWJAus+1YUXE;LHM+~`(Q#ltBVoDfI zn=Pr6l&K0gr2{S9Rh31G9UrOr&ESSe?f~~D-~DIw7b() ze8WCd4Mp;6D{{ahZOEt~!BMG+*0rN&a?^)Tp*8E+6^g&r%v@!EJ#GHf83a$DI%I6# z`9BTv#B!5cK%TOMto{-xA&Qki32ADh2)dgucKRm_z4pBGaTKM$GhTFUqB%fa?NgPK!%)Ef48^Hk zG4WaJFV+Eq)vnzm{KJ$<)Uizmc7FMK=78oOG!%B>cW6MP1xpKl;PqW1aTT$#L{jR#+9#7I^GJm=@^sb>|BXOqH9AM5{UI@qn`e2xWC+y75Kok}Xc zMj3t{p^U#0{N`b>?AEE2a#C$gKXR-7%(_lD0U5Q`>r+d*Pi;}%0BrBx-CoY4V|+r$ z#CNZnGeSL=rcURy?vKje!oh81)@?N!vo#IyuG9=J*u{n_s@yR>vWxcB`_eo+d+LJi zi%h1t^tp!|+=@n)g^YcVqq+8}TIQNv{ILB6D^HXP`R)5!3tcLDCVq9OmM^Dm_#4$2 zq?uaRYCo{ot?%JR#v206vl>nS>1LlQ`6w_`{^)9O<8*FJh`8d8#8Td>j9LrzvZK}) zt7!p7m$&|V6NLu1p^|U;*f(d-F~wL@j?#;nJrnYlLyvhrv8U!wRoD*2ef{Rm_XU*F zQjKs!J{1)b60-ZRiw{LjFbMW&d7za)2NewIFbzV!*hymbX_YDw;6SVV)QE2_aeGoL z=f~*|ZDXmtg)?Tf3@6b~wLL}fJAftR;3#wnMtMxka#>4e^2tpdA|mL6oU* z_c7n{#p$|)F?RQ#-De5)t8qS}8Rx@>l2yh575J@jnG?PrwT_uGbvYzzy)CNdH=nKBS6fBNa2v|w~=;v^};@fIz?9^7|3 zwWDqUC<+Ut z6PiM1vRH5Zr%)}2+}pIXm}vS7Xc_5cc zbIifer?@lcAQ_d-+0Ypu=C*8f!`2XEmir>RMG^05a@zg=dXBP;svNWB(QnRR6B(J- zHMXq({_h~^O;@F30ON8THTn3DHC`4DK|s=-waT4mG9?UvX(H!nlTKdaUU3D6%v}oR`)MXwYpmZ(+a!lX*48*xGdVy`_Ir1*h|$*Q`0# zZ^Y1{&%XS)@3G8DYtknV8gxTVKWVz`p(d$;Z8yZ{{#yY^XTii`P9SI(9VFz%Wp}a| zC5@9~u-n`-unrP6kizE&nU|MX^G%q_4EU4Yuo_l2Hr^Wxp#Ve;a^9tK`$ri{YVv-J z100Ts^Zfpr+GfpG9Z7u^zGhGr%~pV>8`~=?O7?ZX&Ye1)kvX}H$vmGrG*fycg=;)v zqy~>eWxjp2v%7hYC~aU`TmX2Nre)csUWW|O6JOY_%#)LI57qYQObX5&hlg(Zqb5_A z!L=kH;P}gi^7v(cliG7+1a`7r@`-iWr{f2yB-X(2-xe;kOJxw#`C3i(kL7Ik;K4fE zsTLcZ#zCW~a(Dc`$!w1^g}*ECjb^ma`iBJa4Wubt`k76L{+bwKgSW6rGc}Z+Ie*<^su1(HxEa0m^;`k?&w>8apIjM*-JrL z65{z8{%p7#f9iws90Rihj!0N&C=l4KNIoxYa*yz%hv9DTMGqs&!;oAyh&{ZCem|Nw z{M)a;LVqBlR!Fx|ev!PhNdu(>ZGX%VQKVwy3qPBDi-LYH5K%&&02jjHDty*wF@AZs z)*{wxnOt+AdF6K0tGxc$-hyaH38J`7ch@s2=ad0Z1>&3O+Y9q>7)B}5nnGANs@uQM zGyQ@hWiRWwKOok_7MH zn5JhQlYJn4Ln?qBx*Crf&t1zdpV~v~n0pnd~RnN}evgYq#{w2+ZtL{_6{1k`zm%WS|#`XDoxF$Y8qk>umS#z>oH+>iDR&iFol~ zNEk7v?$p*AU8ADgFu2f;KN6lmmgCSW_b>L!4n!A_OiN<)R!U`NGFK#Yl{Pdj*q_%Q z@A!9WDfbPM9Z@!K7H@YmTS?IEEKA3P|FUcPh9_~`ns{|8Bj$$>vpFMrJ zqV8DVFS4?D+N3ysbtp8hx@*fu84Z@495K1PDWaM~{Md`pdPwZ)jDit?MGESO&9fid zJNw>r`l#fAS|!6~epE7{jmrnug9{ukF$7tssrz*{L7z_TYozf%5cs#o&p=Hy4~nSg zoc>pba+9#eybi?v`rB`p+1>yoeo;P*f&gh8GPn(Y;nBhD4doijrU2EHYgxHsg)T9} zaz^T_bhhPCm4Zb@uJjJ*N_KMnscX5wb)a40Y&182ZI~NoZa@^t72q6)+CWIO?&v>6 zWoI0k>Oyn5TYDS(18{V>7aD?&z9D(GwQ3!U^&pqM-O?z0o>WZXoo%Kci+xIwX)MHU ztL7AI*{r5wfP>~Z?_Qx&CBfn3i`Z>+9YR=?O5cEcOFB1Wr5<^e?oZUnJoVUUkZ@(W zk8}hVX@!+Z3Hz9s`yz)uDENfw1OCQaW3LMUms2Jo{^^N-C)E64d1h~$A^S^&{lI(` zNc9)QD9pl=B6iBk)S4krn`$`~4&QT4IARnolLeREL(AGs zb7IoKALoY$HM)nah%lu=p))TQJ(!4@^oBZVXSVg}heuPEX*lnHhDJnc5EK=!NK}NQ zDMT$q$zw@3?+E}?f98#84mOaj3Rm_zJZ;ylnc2>TnAy_g(B{q&4*(t(;`4;%BwD<{ zteHxJvUsuoSC{5=Q;~g&Qo_sieJZIC$_$`qGZIFOio$P)tikB9DVV-bWvVhiRRh$% zEaTriY{J(2E4^xzFqtb~DBjDtnaigX)tTm15kDQOvu}~B$+MHGV6ka!vjdaoY?|HR zjnjA*8OX@sD2Ch~+P!(BO(y?;#+Dzokq?GJAXRNKE-Z2dQ~=>gH=4D)>*#-D|1N|l zPxgIvhGs=BQ}vE#)yRbn2Ipp@$r%wzJ8JOKh>45?8wE-~(hQwHg@z&u+Mik!)cHkL zPQVeN_F^}SsOugEqJI>#x}^1yB2Ty{0CYmEFiSf>=0zyY5)V}UT81u#JwbXQ&WMz; z88xH5JssQsNh!b(ZXofa?uusvla1j}cwctgKYKk>N4fB}s7kzC9_r3|R({p6J%X=9 zBb_p0)382xi-=W31mPA=N<1`@J`@=F8uELvbH|75sDd`5G1zjD*54NdOh>juX@Sw8JuVy&sdu8Pf#MIEtnO{-Ebrw#t# zi_WG~!@Df5+bJTl>FsUJ-_+%<*O0HsDFEll&^-pXN~z1usu$kMpTY=HT=UeP8`^FM zr!2hfFAPa-wEQWKP}OZ2h*;;GP7=Rg2&@nvK&R%xc~y`nrLare*P@ZK*KVodRLF^n zer<~?bObl<$ip)1souqA;~q@Nq>wFmEUo%c7j8ubu|&z@pky=#58veKs83eFl~>ao zgg(v1)wGPu?ey@bjt9mATG0AiuII#_H_#LJBitS%p%a3hzLruSrVj+m_dOswNs*Hr zV?E+D1{WQjb?qMq5}ZWx$QH^fjp)i-8T|SSqc{_Gg37HJJ7I9kJM3g^aQ@miFSj{n z;s#Gf4WV4}=|>vAEa$JtFwmNpGd{kJT|EU3s$sW2h4GGy(Bd(Y=pViAsetw3ywVR$ z+~(s`T-qc|A8ov{42T3H*G76c;i~BEbjpB<9&FSmAw`@<-L2z;ijMP$w7x$V*`-tZ z|N7~ysyDwm2glXWzSY+~_{SVeI#Elf?u=U&cf>oEIGg_cF1c31qSBvNJ#6%bizmM? zZHo9Dj)8XT(Y~uES)a65=WEU!NPs#DZojF|=DWpWHGi`hX@3Sic-WKKnS=27>v`)tdKmI$* z;>Ta7NyMQvMFn-*_AIWw$G50%Ky-k|V~UHh@F_@CJ9drg%txaOe!TC|2>KY|pi}|x zrAKysxYD6e21j9V(%am9ZPhN9mBLoQ> zbBgHU`89(gIPG3r6r6y!{neqy27;G+g?53cIR_J$=)SYl@%d;5LppY&bygtXYpvKjd z-RYyMTYL)5p7(n?_2$%-1)Z0a%;_8L_*F>NZ;^j6JAS8k(CcAvD9aC>=ljHw!9r8M zYNEl~{D7FlOqppI$+;MncsQ9^mWLt(Y2Gp7B(e?ae}nxx<-(~`JL{EmO+YZdDjoP} zQ?1l(ba{fqC?*Ucp%U&OyRB;%H6c#Ucd2!Os%diE3aKzX`&OCw`9?E8L6L?56cy5N zsQ}ZHwSWLe(uYd>23o>cG1j0IMnX$4$y?r`B{N<%%w8FGS(01@X;R-u-U2vP%Q0_I z-(}~jf>^wVxyQ7ij%!=+tbIauG&dBcc61-`Q@+ZeYA!r@-Owh^vs)gU3I}T9RwvVZ#?6n;(Vsqwy^l#luVbLG>UW18!AU!Y?M<$oLzMd*aaRv-`Lb?cl1{0hI=rK=3t z6rTl~WA(CK3z35;DM$9#pb9m*Wi)cgBT8@}jiHC{ca>U|vh;(#RO0)?^)q@B6Ns9S zm=}7K8FB%VJA1^hr-8852_Sc?a7RlN>9g|ZIkj1`YLyU5H(QAj-rdl=>e~~m?L3+j zJby>y1xXCwb<&-|+H$s#|E#PU{8>Be6&Kom;C~x6!Cb{@l7w4lVH`FIrkmltzX86W(38}^!FN0W!5L; z0i$RJqD$dXc|2iuO#a zmwJ{XJMWv=W1DY$hN+zGc;>rT^<$3>d6#tA_t>lCwZ`i=r3Bs_@?$fHMkyTLQKz5n zcb&ZWxliSsASU-jak_=4Q)VE~>iu1Q=fY(4*hXOgT@jbH>5;PgKf23Rh1%lO;&?vk z6oE_`_nK&W7rn!^KPVD%i~IrGE&)sYKy-5I%LfjSRJ~S@m7cX zn5Y@Mc4+55k841ZzSkCnE>edW9QBx+XY3~OQ^VF*`1oUGhB@Oz-jT^&pes9+8%0GL zGW$iZZKst9dtg|Ih>0-n;;Lb9%9s1zvHcxUTfGbE>Jas5==53%3^n0`ulpYVUc)6_ zN?2wjvkWREGs4-%TJRK z&qgm+XKX5-88jtV(W>B5r|&*rWu}Nsf>POzkQPR&ZP&An3o{`ca%@Ah8yauh>Bz8m zo88Iz(w<(w%sV-k*%3}w)oi|qTEFnepn${;8>v0R|2ljBGUs}I%Z#v~FQpMbuj{%y zN3>GlQ31FW(u!Q>%LKoS0|HE8RzQK74k?+6wt=1T*3aCT)XI)*2HArRjcCi3Eopl; zDgzRin%e5`DA)OqbT5v?fOo&AA7-g2e%_x#Xowp+X1ZojYr>~dCXRx56xsbxsX%kofR657j-hRaphJI}Wn0H;==GCgm z2LwQjy7mJRI|YyY0ddSZUR^o){v=D#C$goQJNNax&`hGBlPF#KKIxFi7fcr9$)ULO zSxwiAWR@6Cbd*?{69yiUJKBvlgl%hf*Pzz1f`%Q9>MVwceSree9NIyF5;uWWU<)jsOKvhXAVtEL z1ni)+Il5T5Bo_28k<%Crs}?XnT`-@)X0;7Gs7eIl;>?|K#o!4;qbcvVpvTOF%}PU8 z631joO@|U2P|(x$&znVAkqYLEWV%diy7*l;M$9uWAnh^XA`zG}e>pv#h&J`e+c)__ zQhYKrGHLw39l(UWigKv^(4B1qy&K8p6ci}EI#_RmnG35#RBT@Urg3>Me7Bnmj~j|k zPS~fJHlKg9wqoG&_3{BNAL;(%s)HTNdR}`|7F?1b)0Kva5I0M+Jho6{4YKJclbPpy*mnc~c=ib&yO{YV-0B zjRWUJ=GnG2GrLoR^ne7aGBcVIs$sYCVRba4mq0^>H>mU>q-@E>hfPQS33;tfjoL7U zQZSXg?V`D0JAY*TpcEZ@@Wjr*{3%rb&d~Lekk()$N~6B}6~;zFv<3HJ46X!8B~ zG7*KN<~7^ValaNuW)4`bo|_S%l`uba)^C!%@+Bhw!que35&8q79MZauxp#MV7Ih!| zA`Z-3KCuEbx7GHfEIq-#b+N^A02{-zD_6dk7JUDVmFhGNMn@#80OVRqHAVITT|Bqu zx*o5P5EvsFkiiH580*B(F|7a?l}IHOP-ZW~ctH9hV6P_|K}RR1l?Z!~_h!{qMh29( zomMXF#mwLmJyxFWGxlUS-r06Pm+|HP2&&L%U8~96lpRg7NXJb5Sc=UW3gTt$ukpja zs7X;Be+|*x1Zj^oDi#5HiiSI9qD@GlP1E=f_CFh6NXwWTstNXT2o`3#1Yeeb3Q?+t z_%YQxYG2xhtlb<}$Ss+_o8dn8(x$mm8IiKRU}R&p&qVpK5jXit4sc^%?9;W)Iqb9eTb7KR$HgQDY&k8Zqsg=9hSNO=Dupxq}u9X z0sNIFSaze@(ftgD1h=vnlE;7ie|)_MSk8SLKYS@SCqC!i1Nqg`8e!f@k`}x1`d%SPQaX)uYhg;Wm{eHjid7htjzIYLi zPrC>AmXjlTrfRZn5%u72>TzE)rb=9#+^Mp^Yp#G6!$BG`V7e)TC{@Nj$TD}FCwB?N zvF`23izk%xI@PO5^^qI_6l4l<=ywYK>HH2S-6 zM9bc~^~X?-9Li)#3&I`#b^ArE0yIE?lBUU@MeGhT00gS;&=@_!6HV5PliEZQ;i$U^ z2o6JHGjJ}14Y7!aVUqz%;FvA;l{&MLsf&gppo`X|(RslU!%#`Q8OTPe_5)PF-*`qg^oyhDG56w@CJHB_ zd%=lwQQ?3@MwjJz0-)awxcky<@gjGJ-1qAId#uB~N_!~8qEeSTg3Rf$S;iWRq8stgI|&+T5cbT-)~ zl9?L)AQF#A<#t?X+Vhv`P9#7vS}QexcT`y-=+!HLuiM4&h~aBe70(FC8AD%Kq|AU> z74#!nSPBxLDMO<(g&HLG-*w)lT1fybd~hzHN-igPDqnLao4=fnU;CUY9IX{F#w}ZnsR;HixmUAas&Q8U3si?VWq&1RZn2r( zgW7PzD<)FKW2`{pNg@Ub)5EPiJw}$Ep}`_V61pKCWIBcse+Q0ds_alMNX%(<^ch8H zK+zh9k>^enaADn&-c1weR%8axIuY_VtV(Ps;h_oRdZc0pc zfLG^H^+9zh8{}6z!#EpBz^c!mPbrB)@kKsdNUH?$$uB@1ze?a>%=Lao&>a$@73Hf! z3L4P)T+=hdjtbG`Gb;1#^pxdTiig-GgQ;gsN4M9tK6%dzG+esu5Mi4d*^Fz{K zbpt&Hv~nVae(}OEN*dw^Y}$4*7{As1XMFlu*DwhM?7k~}l07|JM4Ippcw0@AWqFHw z9T&^H%YGe7HlbY01}XD@SeF-TWXp*!=^+6 z#RKw|!7jjut{egtMG2QFvp8}#!wf4vWsfGn^7pCSyG7{7#HN0)-$X|a7o*3x=+wZD z?cO4-6N0yQkR)OPRiSF|db_RMLpcRe8;F&kpvEU%*Z;VD@!}P1JR|DQs~PyaXu z3;&=f{%!xp>dx)HDqU33#PHj zVFc3~#OLzx>=8T(nj{%2#0l9NZq4Y5Eb`l;9zv-=%ODa=`ZzQs6FoMW4P>kbwNL9* zO((w-UTLK;nIHJgbDMcw-V(Q zic*w!mHrDcd5`TKqjIX0iUafkuN8*$?2qs%Zw%;00>XhEqFb4x@S}t}qfc^|Faz38 z^qWMH94A;QK@*y&L23-C7J)6I7^Y!X87UDF*e>jnsB!%t>TIlY|+7K*u}fUTO7zU;-A3ePG|I zZtmLTuV(9ZlDj7PDU@eadDd12zKB@#_9bJA?;5^O zel$;B!;mC&KZqd7U^9kobJ%=4#_Wl`6~f35trsI=bA-ciVDBIoy$vFL** zn(c_W1UM1wjwq1JXn{wupZU4CV1E@#B9yt=Ki7SI-)!bFZB5h)?B1?+^#*x%O2#-#BP!Iom z#6^tPkClm18YXDQa0V&j2EO}+q8thogB$P~#4VRd%ZI+K9*aDS*JJKR8TUy(c;En! z3~hxtiSJpYSdDUEV-X$txSoKV2)sAyh8FZG2(&{tC(Zj887oSKIeD;A;AN5tB93SS za_|6@PBv}O#LfMD2S`V4D2k!wyQGWHLp+^C0OIuT1oH4ChfTck?dkP6cAhyBtfu{Z zq!KBNGrfrk&Se8+^&U=sBQR$I`9>Z@sVuyhB`1Fnnpo1_;BAid5WOgQsQwSJ8v~#_ zfLz28?&Ju&NfmG=Qm0NWpMBS>?_J4+K=5|(RWvLo*B6#Tt78a)2EB*%JbF>_ z*hPVeosG3_KvGd8N$jgq19t2pp$Jff4>coPLSQADQ4*Ixk~&})ahfAKKr-AyXU)ye zPwn!#K&BJXQhe6%{ppDT-TtL#aTbw;SyCg=Mb<1fe%ziYG-x6tA$bkP;ff|t!s=Ze zxepGIgb!|Sw=WxS`R}AMu1Lp1w@hSjMMS~;;N*e@>Nli?N^nZeO+zL?48Na=tn#nr zWaDF?XaE@fjnL;&QHqgh3W0iwnVf{)flmI(F+WHL);RYk2JA|>B_KNeA9#bYh#=V1 z0iJ*mWM2os1t9Ow%Q~W4F?14&8kxb+!R@O&0fr5{By|CC%=qMd{j$GtZaolk5$!ki z+MoS@3mqANN|4TwdhH2LhN~s zwIR6i%6g^PY@ zOhUUz2o65PYRlAgvlGEXq6Ppb$PhG;+%2TjKY=?)mShj+7M4@g z`2h(oS@Oq_A5Aw8R8iCioJz!;?jssU+{d8DX2wW?BUK-@^j$-W_)UL&;162FGXZ&-|$bZRw zm7gzLkzxWcC1cxX)AEn1Ox;aKeuoP#AM_kG+xo@>dghDqQ!fM|0Nk++qa4Yn~3^58S$%%#%1K%8ooHM6S{~%q?KUCki}_{&v9HMEm-_B79&9xr+ijUE_L@>`Gq6!fM-(e{>gH9 zQ-l)CUB-q%Ask@~Sj+w2T;E>vPS3UJ$S^{%0O3Hnk;nf7j5EF`;I2I^fMaG0<7QN& z5S|V2E?+K3{}Pcz)+*U5m@_|ZOo$bGbSBBRL;56*7;s3e&2<|1x5;CVq4-cGEiElc z>A*h~Z!RW#V&z$8$GXMD_jjdGiIPQ?1W!^<#fEKkornH6 zY6mAAWxMz43ft>rCC7~`w@p&bDJYgawzqWO;z`G82KN&s*ofucYb0Gn05%JohEfcc zr>Ru~yRf&Mm{!q@$c%wv0s{T+`w{$Bc~6u)vU|yYpWf}J+xMRki5U4{U?+0Lu}q!o z342MXZ=@Wbfy;_2l!SQ4i>M$`!o02%afSR>*bz_Rj@w64*XA|_)5@pfW@xAo0~>~- zjAy&qjp{jCjCbh{LSwv`&H2`vB#iz=!j(kfzQ;6g$iqebR3z3~01-ex@m<-tBH}TT zK450_0r?sQc$J7QU)lX(q3W37Tq5RHP>cwues_F^L|8)_MLY;Zm`HTY1RlZ&*aer3 z)wECsBopY9NVlvcy6Ed2%M5qpcupKCNa2Kp5?nEbry1@u5O_qJ1es7+4vH+noSfRI zq$Xt^Z!hUWJ`E{!#GOV;4}kAW3I~Gq6Ffx;4`=T&!oT=-=Jfw#)wQ2)W{Nn<#u{}} z-KA2&MJNRS{&V9>#7Fo0vhxa0v5r_7T+o3^l)AkH0pY5!1{@D%U<;8{Xuvv4eiL~1 z$Q-B#Na}*EIKi#~K~Xaou*DZ^nD0`LA{WhX&{h2NPZ!ZufD6~n#e2sAFh=Gb5R{c1 zj1pdG)$nnI6IB&~KG8bNrR;$V7*Qx=2+;2WjH2aD07j$7*b+X;dxS(VWAcfhM*{=W zZ?~~SwoYzg1x0;e#*;+{P6~F0^{;6v^GN|m`?Tqw$ZYZp;MYMq&oAX%WEc|EEBE9n zQgm<=Wdlnh%pld(4s>ET(|JvBKTa)qV2^|mCWkP|Xd*ZiwC|E=MtN*r+M+Cb|4rF* z1V&08Yx2&Ffc!y>Iay|(lD!k$9g%Sp+lnuC6=;J}d`nF83JA@anOF{hrc)3BF%>E} zEQp#Oo{z5TYNThtLZ}8JdD9MpRl3xozmg_OGvE(0V;KDiz>G&cVK zXTCqPe;c=;|5b2iAU30t0V~!TbwAUy+8n&;;0}CSvEGDh!zOtLenWb!T&W=Yt}#LL zo^j1+o+(m3caEN_kuwI~MNL>hYXNykt0C_yvUD7uBmz05!t8+`Vp;BPuv~kn`|Q=X z@7N(pSa}j61;8YC9*;fI5P|woL%<3d{YJh%ZHVVN91A8~PWw!Jq*W2L5F%`Z+-iP4lxI;yu`8?n&e$;0b$B zK$82Nk`{fhH?UnnsR5G^oaXYVa)<(Opr-G&iNq;A^9t&2I38;u1o^vg)>zVDFr9LxCHIW^tI@sT)5lIRhmnEOO&zGecC@rhz>MJ z_Yzs|O$%CdM*^f{&%3l4yZDg1XH>VmyXoVe06lfK4(8J`GRjn!0BUk|#!4yq>pKv^ zj4?EfmD~k!Jq?+ufRipn5g9+kFyhAIP-YSX2LN=LF$}UKc7~gqdpcBsJW=W~|BLUs z0LOjqJkj|ROEr3`^swc5JD}W!bh?t<0uah+OC6nvAm|w}_TX4%NWR3N)Mg{L z_wSw*C>sEGQ9qdqU1U@YXYyB|9nYPSrgU>4D1?nI79)QTRu+^K^I6kezYWogp;Qrp z;xl3EED=mEWOo-4%h@#ANjuNBaU|d9Qbf#=G-}MSt;pP(SKDEJhO2uwR7 zM=Ma1t@x^DDP`~P>+^qQ&T8r`Tc~d-_ SE7oqOl08-=34v)%@>o&POaJ&j!|HcJ zXF_4RqFT>DN#)-uPcsw0A@kR#OHmfZD}{uJYFWZc(T=kDHEUquA$}yR)r)FEK0G0< zv2>f?i<~ie`Un6)?8y(ZAP1t^$uVYVSMUgvy(+kt2FH$51T=8vF#m=#V^=fak3BX4 zI0t&>CYq`dI*m1X4SyAqZ=ii5yA_oBaORVw6Y4__1b^C#?g;1|0X@oWFliqDwXWa^ zZ&kxJP1<(4ff3djG|Mx*^YYlp07RvSDzsbF!$vrh2tAZ+KO;;@q`W|!uzRD6KaYrr z7MXv^3IND8dx++l;Aa0{34Ej#{WpORp+e|yqZGU8Q+!}rcR`(&?efm)y_el zNeHUT6=GYddvhpaH8BDscjPa@3Pmtk(2l6P+`Z7_DxyUdH`XOM9J%I9OnOn_;o&`G z%h%PbUHqlgYi^G|j5xLbZtBdPheDJpqyS*^S4eRYvx@&=6MvFsL!Ow-5`tWjJvcaF zoYsqzmmk3b$TN(jDVl)v5-?h_o8p5wXexpLk?V!66#0Vx!zYe+T>{>7n^Fns#KTNi ztX*4)*XZNxdnL+YQM(Nb016@bFmay4ucM89hqcOyXo`r~b8H`Fk`pZo*)z{>-^oe3 zD6+YpK%vkv$)KC$o$z}$Okg-Hts|k=WCc8xc*F8As3AzMFVxDx_nS^n;T8=?4VkGd z!krClN&QiTDHJkPot*j+Rtaqt+=j^VGDh#<3Rda z0>pTVeeI}4<_oLNWK(;#Jwg9AZQ1&;z64crQr{P)`>De}?2du2;yvwTEE#4KtC*Is z`Z0zLQZg$QqP~$`rHEBg>Lb}$_^5f$Tx1I>+x4O@{AQ}!RA`ga#*A$4@k!;@vB=+a z^F2dnz_L93J#t#?^t~MglR*o`j~aGDm$9tY%W`4MMZw=Uer?z^D6UH%aC|Ga>%JT3 zXcyZGegbr!Kqcvd2&VLG1;wTNJt!+6PCt~-!bXnl-%E)#PE|P=RS!8+^OOR*QMsyc zAMJ4~81fvU!o4h{bwy|u8;JP&dEAp&ym9aJz9rATBDo0aECK+6gD!5^iFN`jGf1@9 z#nkEs2V$0!Fu6h2Y3gMCQr-fk48Z-1Xw6kfi1PHn!n~Q12?lNq5eYjPDdI#$k~xrL z9EF2C*MQ3^n00n6-dW|`xpNv0WQ7i%k-Hw<%YnUNs}WJ?IysU<4NRd?`@zLtiXuSL zN3mq<84%+rahPVhQ4cZjSDXY$j9&l}K(nTr!E5ir9-J{mq=fZCBy;`q=g-{P_p};c zNE{CyBh%Tu(3C-#BXt^Aa8Q$C+v6~IjsEm_o zwi}0QIS{iAjEonKapyHX;yjz|=x}ex)vB0M9J&<@racHIDGJdTPT(?1t{rV;GI3gn z*s&+!YJ`rqU`?xJ^7)esqSxb^$^#5kpCxzXEVyfqaXSZk2ukrQ^?T8r{`syWqpO8E zL(S5tV(CII`^eY%iMP%i!LQnCReL5-1ezhyf33PY=gC*YM$(r zw0?hBd$0U^w;xX;nvmztZpt3Tr*O>5sTm@*h9oTMf>-ewvAdH+2O7RnMg)!UvGvjJ zRj__6T~o;3uy>3*w<7wsV*GGJBR4CLle2I#i=uBq*mp|62bybWT=sB(jsyAp=63J> z6-i%IX}!zP^rxF|!@W#E1 zaT;npNoFgMNP5RC@Lqx+l9PoT`7(hZRr_Qc zS^%a&nh7x>(V|vNG+f^SF0?ggu|`PzQ-@@D%qOzR&^(YxM#6@uVN^#=j)rQ0+{ZXz zka`%6gIpUsivULhzK{7%o?q~%p#bkrJ%N?g=g}kiaWMCL14>l)q$?>Y0ml}~od%s> ziPrke6#JM4m6c*-%v0Y%u_`5>`n8$_%K=^nd=6+jn#FdLyG7Xr%`kp@G8Qf@Mv1ct zr~Hgtty+;{apJ}t4iMo_718uLW zbmS(C9K?YGCzPTpw%9$xsvfLwxTKdb(=k4MqprOwkjvZhzDP0m`0ftp9FgQJ5^2@D z+I{SO)Vg_7n@mrwb?v=B9Ta~r-@0&lUFGhNoNgr`RsMDDDPh9eqiaz>y!I9JV+v~SoY2pCQCFdFfDlYSSv-bB^96Tf zKe`lgSf(I)4fN{gj)S`25W_%J$z*R9#_Rn@cT%1bS^~5a*-=UUB<|L)JpxU-aBC3X z^9F#?=)S2dW0T0Gn6JTHy%Cxs z=a7O*O;u1@GIoNRWe;Px!jp_5Og3ci3P4EwcGGTQ_*IdwLPAAq`o`K=Sjq3CxI|gw z>?RF~vM2C79Q7=;^ufirWKdK9_ki~I^v~4l`tqMYi*wMXwg^^bi4f=?%X0}yf0__C z5ar>142F&sMlmh=qOzbf^MJC6>t?Q-&_5tqoyX^!`SYA=&r8Dz)8X@pKGbrrtoSn3~@O8bDCFEvQsd( zOQVlPUhsFO^R(s>r(b_27A#D!{Nj-*sg*wOyIjKLnQpgFd47Ej_1#j&^2<0>{t73~ ztY^Pc{Z^Gp=&^PTZ}sZmmNQP%CneYzTlbC1C(0xLKFVf2_b8xT0N0ARB5%z&@?f!w z2e@Zqkq`uN2Dvdyj;ET=mpLEQX0LR6-NYXBOyjr0<;Oq*Pv2kpKH8)F^-fND0SloG zTUs9#s5%x|p>PtVYWe%Lyb7 zQ^Lv9nDvQ+4{#-719b`i2~+k^-A)8w0m-29rcjWU{zGnzLXw#ZSQU!K=anS?2M!`N zz}5+Tn_aknAlvERr7G5=oeJ=pe}t@B1`*#khW=Zw;4)o65FaX;i+HKctOqW-15Bds zubPJXdJl5<$qFWtwK(Ii(soWfBddOXkeKxhUkB47sEgSx6W z(xuo(SCiv%B%O=@UewB8jL}NukK;cO`rjeg^!E@f-zG0?wxRpf<;OS0=MDs`9@t^! zy|OAe?A$WpYpO->CGh~;kM~A`CV5WqQv!7XFw^k;rmN#2eRV4IOc{Tr*YVT?+U)zk z)qUIBmGU*Dj?epHr23PKPdBXBS5pcxu4SndrajuW(tz*+E!9gyv}MqnAV$#6}N8xE|; z8=VLK%fDFPuO8{OOJgg6K3ON4NL<(`hv#}dsI777J)1BzJ0w^DX)mS0krBC641GIG{0oqbHY z;Bw3KLXKT+_hy%s9eU)JgVxv8&yV^YtnQ3%vfEn=o^4=@@nV_EP;E{TS|=ar?-sMS~#p*38%wA&cL2SqTMw7dlqlC41e9IB!|#qUcG%SbAx9x`i47kw}ogF zK6(3DB0%#F-EPj_y7W4~T*9Mp}3Ljf_C#J7!l5hQqt9s)w(LS?dM7Z8K7>^4foHW1~Q z*GM8I5zV6IBkdi+hsMX|EIP%16XiYFiW<=&RAI8^2D3820{NOOsV1CZ-jGIpx<2f~ z^S7Ro9pTCkaSW0<7Gm~6YH%qykPn8`EMevEI8UeopB%?JYCCZaqm z4n&khy;oFJBvAJlf;I#lrE>n^I&Zlb13sOKIvziV&-y5RYtD~)a=ul?RQ3JC%Nx%= zL15xDL~p4w-8!>|K{%9j-T1Yz})+@4UySS{3KiUjt zJ}8eSk}oI#Zhd9jx{~NK*}cPKoZ#ulIR(kc%O(??CoV#WFgQ*mX+Wb$(IL!pg>y$K z8$q0j4}fH)0=f2lLK6`Q;~qdEvf_y-{D2{;!IX710ImU2Cy5R71U3l3uWA`c&Z>yZ zXrjLhHr+JPlrTV#<1cK(mm5TDC4-uUB`pSkzN2HTk!=G+@$8RbJDdH}A}S?Cpd{eN#KFR$c%oQt76t*G{oZoHtffulu~}?B5Ym@vq5*khREN&yTnq+@&1F+s1t4y zvO)9D0R1`rm?uecQ-7YH3oQQGS9f=K3J&u?knHb;tnNp?wNXeuaBy|9m2t*_1N~n* zp85+iC=E2qRz#^>40ZJGz4pSoFC(m{dc^0cQq1Si+y2>c2TRj!@7`OkT0D_v&WSS9 zpeXf8w5IP~YwHJr8&|UQ^g1D~?QyAx{z?nYV-gsoZ=8E`h;K3hy%)bgxfmmT+K zK1FleVEy%5B2HQw)UZuBuW86u367RF>(P0BHuR+FU8&`JqwLZ&Z+QD`&(qmcvwC1t z&u~jXakk1foA=Us3i(El>(%$KWb#Q^@Nez!qX}ip49UU=XrE=oxFzbNj_d|@Hhrrk z-H+PcKiUgYpIsN#y=&vSp=zI`Y^{KDy`z(uov2PjhHtd+ive!_MEOv@RGPpgmQu?j z?TW4x?vzqJ(|`VjrB$0&$|}qI%8K$)5{52Z0r#S=X6`;Iz!3jzz#{jIR%nI=pS0gq zJMM41PcnanG!IJLEjr@!sNxip*Guj_9U@u^6a79{zCOoD>L1>!pNi`T#e-Kx`t|m? zbB}{`-vkODL@;;!H<=e73TP{`rc#5xtlgq{@P)35<2^OE0KwHs#%lNCv{4k>RXsN` z!6vj+z;x@hYy6k5d#lF#ScII#qzen#1x$MpSUY6YAT-R}v1iZFm6`Dy+UZZiV|d`O z2|RPg=&eq+|7@${B9!cGH4;S)(V|H3k|@15u9b1ef~^~QDgB_zd*M54mRgcBVzC?Rr?8B7U`l0$Z-wpjR2xRpUNK{DCVfi_o64~(>hATO}wcL z*eErGF%|&2N!-UIKjFA=5D*R4FN_h+QvgwzkiDORZI(Bh1eKB4T;jYZ8wshBAABVA zB%lhTOeFR%0C@hG%|Wq}0{=i(I098WJ8gaL4Qdw7C~(WNPza7}37A&`(jJ0SRS7SG z4CUxR$>M@#%T-Hhvc^>4c5XT-4QDusc%A7OvZd~k1KsE3h~BaSpk^e_*nf4lK)Ex= zoEMrLo&^}6T6JLTq%_@^zS8dt)4CQ3btXnN!;ZM0cb%SYTF4BVpKGF=i9xPnAWAdo^KPM?uiT+ns0Vm($SKlX)UOygju0)gc+-asRwl+t! zU_jt-z-zyQ!d>3ZTVs4T8{R!|773W;x0mf2{SyFnri^uZousr|r;l0&NR-Zy=4KAv z+oNvtbBzxCDj>&>fnusonv2m0MAMR(Fb>%AYhx4sFY@*yIA(SJ(b27OI5J)NPoI|U zalz8|+2--m&)ck6p9Iqx-cY@!|Jysq`+jk{lbqQ@4TY*#GxOVxDx)L$dUNCA=PgfF zf1MIsdqb$gM&DuWf{M+u*}UA2TRt60-3z^Yj2En@YztGOZ2!>iEj6lK$V`oy3utev zOjCJ#INL@tph>(tb#BKt7Dy5Bpcsvx&J?%ln7@$}sPCx{xoE&P=6!VQUChqWJP ze(ag>fBxk?>qITBQEh3h@sRk97!~F5WF&=n%?agB_-8cWT@;CvW zS}1)giqG-7QsGpnc$$gp&eyC*VxO}6{yg^3B`7uLgvGH)Yu=-w_$;^w#u>!VuiwZV zXS#j;So3J!G#{qf5xvVrpWe}(6F6%)(B4t z-_s@;CStk}Y>7rnS;^ygW>}$Q<*lNW8Klh5Tu;mxG{Zh+F{HnMzCpX%}QwmJ8w6MvxADnwpzb%WE>N3*`b3rNJfGP=j0deen`Cme$(p`Rv#uSjdcf@viF zF?0-@PXsU7NkSv^Hiu0URRUnfnNs2b3;sjQD!_56>EtLHPIDjv$v#pt5fbaOvjipi zIB){81P812G_*6Hi{C21i1O^olQ+a4k0_DIKX>qVNz7GZzYX$50*o*qU5;5RPY2|L z*eKOiVM@Y+3U%KVdE%blf^ysej~`n!88}juACzBxwXM2+IEb>z=l@)iIQDlz3}C8% zvyCFy&{mPcgUB}}N9<{Ks`g(J%m=CP|8EX-)Gsb~ocbiDir}K~yOV_t)K%`t(+EU1 z&E31=S)E+PFMKZP+^Ouk!Hv92L@=2<508wXz`g-)F*Qbn+xXPT3g~}DmP7WApcnmg z>@LMyUiY$0{ZhBW)$hI0cBiJ3J44Na!~4bL5}Ck8agCUl)^WuIsSDz$mbmTRu7-}p5iizpdjSf{74 z;nos`oS=@D_oc#yrCF?Amw)f+&`NvcVcMYD>dL^kO*pfE#O}+NyXr|-IR2E>tqqK` zn-Ox>RErloe(^W=wZ@xt_Srx5GqzFQzrX74q2V2Se&A4sV|Jta3mu_53L%GEK6)J5 z+}-J-9rDAV#9wl>cS`hJ;2Gml%c|J3A0KgUF(1BYA6xDvwIo`#;X{N%z@q@;hV<&* zUduHs3GDn?<||Uom@72+#dFs!%r$j<&t1^Gc=>syM*Ly>Y^E~N)b;VXP8%i$m<5d* z?_F6-bH;M-VEeG+ui5#l7aMqEQk_mJY>exb`Q&wi!C%08S75^<_kkvn+uww$B2$ig zmm8b4j9hY1P>qPheE#LN{E^Qx?Rzs>ZJH+90|mxhXKV|?J&i;Dn5}Uwb+^hDdnn4G zEos3v<1u`Et@TZF_L&8{nyABG%eSY@b@8ZXJ8od{n6j!WSM9X0=wZ7la$T7H<0V0% ziI~h~lQ&vSKMv$FWLH@(P$&#-*9XTu>4I|}mv@fW-KSIIXle4CXfE};vDvsrnq@7G zX<^h*e7bO5APsHx1sN{1w8B2|w?AFkUy%%AgdK*y0uHWMvbnIv3 zA!VN~P@=ODTW7>_9{9v>jT&*#3UC{$nVk7%a%8$);Mwnr+<3lI{3m;hw|vexKQUjU zX;T_z&>rn3KVNQdIi64yy*GSYQm&o2jdad~f?INN+XTeu?rA>s^FEvF<0hVSV+B6z zrOQvuSvik4A7iqg(779*n?vsvHg4Dxul42Aot5VYyTUD0lc(N7etfXA+KhL+H=Qmv zWxQzzE>@;Z(}{SU==XLbeLH!>_iMIpZq(${`Yd^4l(o0sPqAlUcVMYIe~x?6?=N*S zJ;I{d8))g`3UU`5V@u2I7Utxplfs( zxER=%HGIabyyxr-_Le`gJu?^1%6u!B8u1GoKFdEA7k(HE%?icFt}W?&TGiUl(DojRJD_u=>2CL}Z+ zSyx)#`B+{4?)keayW`p}YZ?V4>|OggE+)@UdZYPcPo}OgiMN?F?AB|8esUbXw%N2P zF8x+QQql$t{Ur&qcp1rbIJ1u%8F2$3m3D1iA*7bB6Qp>@6)1kPLJz+FTB1`|y@dU-wSK zxVSi90pm?@9-B`N>Lp%h7SD8;;Dc|8iH*$*yi6WY&Ehj?%^l1bMla=i%>J$mKZ)vJ2|S+{g|8wK2P z7rM2J?o&)m%$;p37(I+QR^0OUPE0&VW{6pc*H4{|X89%&Uu^*6gDtkK{>T&*F8BxF z3;Buy6l-XX4d99D>gt=Z-oYs3fxG*)6Wl(>LluvODlVa5>B}8tL!JnE@053vvX9jZ zOt^%U)NRspF zwv-5ZYcyD2i(xGfGm$=dvgFrN1%+3(fBfX{+{=*Q+0-)LM?;Q77M4pC7vA&z$9JsL zD-Y}1)MwS#y@~=$Lh!qH)AetZxuwkCU61|Bu);h+s5%4AqB|bzQ~jUQQMi59|0!K7 zJ<)Ia-Dqu^lPL4~)JCir-T0he+(ob{DM!x4HQ(Lupp8CWQtHR)8Ip#L0 zC!BJY7kNip3^voQppl1K3D1!E^eD~fz<_b3<+)cfMRXimxKAW61h0SPz_&)!KJCQL zr`z`L=jf^Whl@Xc9p#a)uXk#rkZOC9R+qX2k zH>``Ujg*zr+>@i8!dzIhG0gYdx!dpZ=jF%7Z-)e|6N+6@=v^Fq?(<2boAeKszy9(y zQK~qU=jEi>lG_R{+0iuFyY770^=;#A1&OML<2$xGu8@t~M0pgbc!#4=?``vy(x-%8-y1A9 z63-srltDjyV)9*{eO>*qt|6my@E?bN9((*SkYqeQZGWO=+X=nPET-%Z?)7e!9lnP` zHD?E_^s*+~cPyP4ic9Sa^4WOh-E^SA?k|xp^*Z<66}Xp5*&o;AKcQfKbRE68M5$}{ zfV4_F{T65ERRh;vtxQjFTw8G7U)N)8uIR1eb-X(c?B|k-4({$-macg3_UO9$Bg2e$ z#rp~bE(y&Jjw@unx983}u66CDU4ixdz%mi{ugfWu6Xwz%uWWpo-KU_SeD~Q|jWnD6 zknvUzo`Bpv9{P8s^#fNVd-ts-?G~uoFK9Bb7h}a!>TuPt)$+^&D;n>@$7~j!c zG;yxEFn;GTp16hSs=&l9^@*Og@8XA@+(I(hKDg20@~l0x>u41Bjn5A1w^49xpK1NN zldQkT)NEDC+!W7)>29Bvjjazl7DSm$=f@@vwr^^7Z*FWHTzUO&wkMk;SMZKh8iv*c ziv`Q)R~wwhDy?iC>e}R-qZW=%z2RE>$*YbgIpcCuTYKoo`t%m}2cs^hb{tn!6id05 z?kVkEJoHufQ|m-7M}5QuG!oK-m1@`^+t|C+COR#vY9o`;%5mS4$xmk9+j zX~r(`+|=i)xTgR&>Xu|4VNb7vGIi4^1MowCCM2}@jSIOW^+;U zTFi86)Vh-J30OHkX^R$M@VFt$w-JdUqM3y z{Xn*1I$&Mqhc=BKc=WC2_p0q|)?HvVR-wmtzBEwcyKKmvd$hSN%qJbEUlZ4+F}K>%yRV ztov<(660zJgNU(C}KBTR}tP>U|JY1(lVR)~A`Q%;n~*1X#Vk z8zsx{j_L7z_3EKwFJD^n zXr)0|Te;wN-SSCWo!Kt^`C0Z&R>*T~ne<*b^3Z257}5a2ru?L<+V5^$eXEiE?AZ(K z;ht^NnC+75SVCE{{GsDmQ}p5PjQMOD zD~I#V!tQbWy{mtv4&+3NINGZrXWEc_lbz9FsMnOb@x1P z^!Tg^RPQ7ESeMOCSJT3SUcmcjs(?d(pWcT}TA9yjgQpKp4lJO_M6P%KfHMcJu<%IA!eo)&WOuY})>zg4&1+~B74Hv{<05llr(o7?3c{+k zzE#|xGpR44V;`xO7&uVb<25btHn z8kP|y9s}#QZC-YRW0T92GuuiB=8>89U0FQ#F=y46+PBWu4IJ;{Df+-59diDG9fx|srjyO3H8*1BDlInOkfP-Mh;L;Q<9Szl zxNW@g#Oy8q0Ux(wmb3F>%X9QhFOPl7W%v3uVA|aIqMAYx@>&9T%4^}*AzI5z1wlih zeKwq%m)%)AUXYdhz~yQ9)IkOVonEg%j+aF~+LaSc%$J|FbidlL#?LUFx+4w#CuB=9enFTAB4ksBJ%9S(8zlN%>+fLPeF`1s0v9Mki*OyKASp81d z!qH!?*IAO^FhAkHs9)gx30xK}W$V;1tNp_@w{4iKh3eM!4Y)r3`sO?DPRbLZPZxLZ zt++4~RLa$*%t~pV_mOt5dwP3GS=AgHcQJQCnE1N@758K2 zHq%fvY*r7pnaORN&|67CvMt|*uAeLJh`x6e?H9~F>OseFU$$K%F{5Q>U{^uA$J}o^ zjt6;rcSB-;y5 z{(gGo_a9X*PELlFmi}MK#ydY0G?e_A>NBG=`139B$c;`;3hB6Z?D+jgjX!?*$mz3t z3;hK=7(Gu;_9ni!ZS%U4*tIYfk znGKJ=Vz4QEaA8+kvE6;QYeg!2pB^yo65;5-&sREp*wI!0JlUg_XsM&aPQ?3TvntEC zuNxNl<*WFh%xqJ$t<+IWO(lUllb3jNj~#iX@DM+}U~pzX{!GMy&Ai>B>9W`qC}rA_n7N*IRL*KTV_&UZiQ1?Mut9IAHD|- z>9=m>5za;M!3VhD(Ig+clB~Jq;wj0#TX+w)wnuajoq!052+1yfvXT{FpwYho1>fb4 zvyxm)piotMM13VC15X}1wxlXX!*zJrGST+RmEG9s`v5Ukdf?~LT?h`r6^4_03S;A~ z+}zwsM@g{wt&p<(%5@eemH8$_Qb4q5!AH<0Yp%`KXk#THZ9#{1KdT)RmrI@T2N;6{(Re~ z)H^p(rkVWm`GzggpB`_I3>&UbZy1{DXnQSxinrSyvloP4oj(d1y0ZqF_&?~*TQ(}c z63gCe-T!i>Zc}aiKWn#aa+#YkJi#5y-o&{$!)PS)gec^lH#^yeqhH&29!(tXyN&u5 zC3e5oZZWN{Y0R+lIASeh!tFfzWA|)3e}SH%Va;Jm*0`qb&Do(VN>gqerdr-H6vlhmQ}>~iZK z(9Ald8GLCf%4{` zCT(0_t{L}4VEr=7OMOAc*BKwsd{ksoy(z@rpg(IIyIzckar<*g|KhIH;YZ!-14CzKT0v)NrQYfMnN4H0LixKTO!*5(7UCR5&s@}(Stz+M8u+ttg9GCW zueVAjGe`HfggcJCO`X>JoK@>Bhu8DIGh}~Q!ZE{-EoE)p*Age*%4ZGtZ5Z9mwA3S} z|Gn+jhTeCRW(JJ))pZ+$b0g>Hrskcsk6p6xiJa{n(BC)Y<@BiXebX|^0AHN%oNfW* z&+72T6yxnHXvXK~*3moJZxnADXA~1*lzQ^F_QXJP$&@q0m#@q&RKc-D@SN*elbO@Ij#kAwEDw>U({PmG+Y|Z0 zc~&|mvOYJLLEoSw#8|CbvuC2W>y|;$AOGh;PnB+gWpk*rWi4@BV^6}%OUR{b| z%b&~X4NI3DZNuX9sfRw#jfTdmtMNmuar?t1Qs1D@E-bt!XTj;fhE*PWQY=EO1|~ZC zTAb$%k2KnD&!`ewzw^$8ug|-}5>$=z2ArM4sxsnpYKumNfL{v794YOu^|_subw&wE z19m2^RP@1Jh~c5me+R)i4@oY`w7r9tceaE@ZEYF)<+VkUx1`|Q?MQCvtgS8gRtTO< zY?P_5xpVg}n#DREYdAfY3qLK5tZ0+hWD(2NaChuJAi`65t3l3&p2o#3RH#=;&3WGK zXYnH?jgeqyAS*X-elt@Pz5Zv7@a|n08vfzoSp%HspeN#5^`UQ)vy$0*=n5lQVZa0> zksyQ-Mt+MgRfzTRsXNf}pq)D3*_4}Gh=$x8a`&6_Qypp7J07k*u;b16arF1t7ix=* zdHW$FJ{9^ra1%7&dibj+A!0|v|Lyu5_1xp#wrv{)Ea>I;uy2JRv|oyaN#QJ`XJIGA zK^i=qoJ*Fw=Hq8S+Ti0VDaLk3U^x-jAAN0S^eo8?hy=_OayWT|hvaPlM_fC1e$uan zMCLtGTsM13ZMi7*@8N2_;$`_`|1xz%7CFSUvIbS|}n*TDg5z zt=`1rBD3?Lr|^*@!I(LG4_#eHaJE_OODV(4yVZ_ z_=nSJvW0G|u=N^jIQV{rht8z$GhqIbVgOK96+5_$wqV&J;zsO&+2!l|p9%c{8z7Ci zxcI*>vqEg;Z5J@%7ka9mo|27a;RuFQI84^_6S#BZK4}_?&dvAC%jri=%vrqIeGKNN zcPLH0$nEvkE1+S0Ie-)E7vUy9!QmwgW=$02ma z2^n+Fb3NQ^mbZl4U7DPqInC(VGjPy3kD{A3s>rW2^_9OZRue0S*u-YdC9|#UM+fsh zKNcuEt(E1gV%Xpc$RAMzO+IRMWbw zhVRO35ao`ysS)%Jf0K|_tjvBSF(CP>V2uk)1Aq^0v%q7C1trB-1+N>&zOX&+^`w54 zl3o2*$L2o+%4T(Kx;?9zb?KC*T2qAgvBvi7HT>k@%IQ-nT6JHU?^te^UYMnnqS%`0 zahu$^&+XaGk(P0(r4_miceFGT8+00-QbWt_jdVCmH|Oj~XcWS!_rv(Od*sU-LTf|q zHYQia4miy0@Q``1GG02M*e`Xpyua?u!2`=!Les27DwX^9b1}MhjJ~ti@CvPFc{j8W z_@QR4N0|C(Qk?p<_xsC17ACBT^jcYL`>Yl2MBZ%9a~0}wn$V_UANYWTrccLJ%WUMh znA#G{y2VY!P1}o?ZdgbfSnqifcjv0!-fn09{CqxwJNOloinRR1h;uIKB5TFwZp3s= zXq$w|4Xk2jzN^s-B;v?{4?VceQi;_iXacC}YI3RqB^Q<~Ame?_Ue( z`NwV)UVnXVFfn!hGUuC5!-+3i)8%(%RTON@7U|u_l(}!dZX@l=8~QJ$rS!0=6(GHq zfuEn>vkO$=uQZG8a8F$O(Q*)H$%a+S;4}10ZF1TIrAcfuJX`dKul;g?@umeBcWl5m z@@5rw?SpOg)DD5ba278qJlNmvk;BP$c%cE7jx>kVKAY0h6)Yll-en=u-ofQv*Pg9a zDB88o;{t$(O=$YEl?;tENYA+I+(%&eykXFA8$fhY&q0OT@f*U4XJH58jS8VA->)w@ zM=J#w{vtR<^=x|qKMK%lQfjPX*|=9&v${+N!hTYLaGb;mpY3;5&uGl9FqY-TDci zg6GTxFWp;?eHSQ@?f{8!)BLl8+WNhz-MnV`wYrqG(2`1>x&QH6?N|PZe6+rto2tb{ zetYtd8<`&kMtw`wm_l(ctk#ri-!6Tvj(;rQl|d$cmmEFXQv=r# z8NZ)}QTCLFM0=l@)yYIO(Ue z>!0>&&5l1MTJ>pb8hoD?8C7hhKIa*dW)bmc^ASsp4S(jZrm6Hqz0VM}M*c@QB{Z~Fvxn`7{8PDs zMjd1?q~@oDyvh3Z(0Huo9p{%(`{Cb=lIrOL4zs>51GV+6Z(idov0Gzi0xH0uId58= z7MN*ccFD{Mt&Br&@>6C+tz)trGxNG%>ezCwQrw%;uBOvvU>{q(#iH4z9jhw={rF>} zL*!$B&6%N<6h$RZi_t4313@ozx=mz@|CU}hsgWS<#W@218^Guxxxz-}f`$z<&x-3( z??S#+?NOjx*Se$N_C7n|ui&FRC2q=}{dVB$0(L3-?o*W{!gOPN@ z>_;tkAbZJl)B2^*o}08aU!uOizy~PA2zI;=D$IB7#rZ@S^r*ee&8Y)Rk41zo<9cI+ z1Pj7c4b>XQ6GuvNRl4(n7zy0ztxqq1bHa}L?u>-g)YR5st1i!TUtjm6q*~!8=!Cu$ z6wq?}6!ReXPhiWX6O!H|6BB}&;tf|0aUDEZj`PIo=9$o+FzITD1ytI7%${vnIZ+6&`7KzcUqka|Ze^7m#deHwu=B@u zt&AHhaYz(m*D=FQ$gjaY?t%;EQBY8m-mBOzWvN>qd@m^}S?zNn}u#5OIj&#+oedE}=t;>;~Ay%K#COZ2oEhp13QzEHTF zp|8%wzyK`=R5>GCMShiyPPZ2FM0XgQPV{el9US~eN8PwlkQ^!xwzB-Xkp1@U&EM^n z^{Yj#jF_XZ_f(f{IiN9Swd)v#PunOh=%?@I|BI^c0LQX@|9_N_$Ozf9VI^eGl$Gqg zGDG$zWJIz@WeY|2PO?{48Bz9LA)Cz1|9R8$=YKbACo0#uDo8 zU7xfCXC)=Id}r#DXU#?gHxw?H=2+JsZq_)n;LXksW6YBY^wu>9ggNg!Rci4&=-QRs zPRp*uy;zi!3_w6xT!Ql(dqEm#`YWvbYj8ZN4?+u8<+C^6x{66Mt?Fn09mkCu0<+LW zS$blIcbS=L&Pg@Pa)m4pEhk)l5<18gs@2pO&R;9bS)pd=tb;vHCH9@3o_guZvbC*x ze-wB1SlRX3vC+w2<`Z@jsIh>4t2&Gt0>GfOne$CIp4M((Tv@x6T|MXS?QW!3!m3YC z+&EsEakB9GZLs!|aCMBe`^s;_fXG^r&X0#TqvtfH6Jzn{XESFCrO{=N`tHiQncWQu z+nD~XRAPPc99(%*C8@L@Gjp8@&NeF?L&KFOCfZG{^Korq{5KBv_-OeChg5BXCeF5j z2pX8`%&3%5y?DXbI8rq=5nfx%?M%MubyE(zIA{F#4>3hX3$NQ?7I(0gU*TBQ*qOiY zVa0_FHDsdm=f913ymGA;{`Y*ZKXliQ=RUh(m0D^4@1VUJZt#0=e_`3bX2{hI*c^(M zX6sKi7K%~Ge}MijF@*B|v#52)$RUBG%W|-S!KRxJw0=x6Z{LRMEby17mCm@vk*D9g zJ^3sggOuyaPpMr)>2N>AC_iA{JT^0HY|S*Wc?z>+ix6F1U3PG?vjX)v(H&Bp!AfTq zJ-1onue1HalUvuIE&=VuJs3H_GRY|a@uUgxl4855F9Mi^r(>1Wa4GoxK#UCx4Q&EC z6);cFmn;D6gMca^9z@p91|=0iUwdmSCYV<=02&V{8v#5Co!$zsBLR>O`vZpxnNhg7 zxRic6+YG~yZb-Mv1(#e-&MXt;0S!>2Oc*8A5GWz>z_)uXO0(a-e(ejHNPPVKL+mW{ zuiIzWH#Qp5q_~qFW~8Un@s+PQ$;$Qq7#vA-wRohYwY%t0NPF2y~2USX(*aB zOD%{YT~R-I2arLjN#@scb?nb&CL)3 z5*3-FJ3q6&l17K}>8YC=5{iWKfry5zK8qfG>)nchjG`iZkS0NflK`r|p#crHxUKo6 zrAx({qjy=X6~K;1YpA*T0w>wQZ~D~F68PwliZBfdev%$aDk=`}nnAyyC5pOrc8{Qe z(Gvnt!F*n>_{*0Bv#!+C>U^7A9bR6Tt^Iat5m?f#OSW{UB3t;8rR({OQns+C=OG2R zt)u(;yTLq40>h(&qB&vF+VH%jc{9#F&uivt3V}}#Ae=RL_8&IL_phRB!RIRIF znbhj2k^Oe^*JICrXRsO!L z;La`pqR{JQhw^v7RuPqV;4|FrZIH0MkJNZdpf;UAMcQTQSih5y8Iz9`Dgm8 zA>QPl5|@KIDmPag4ics;oul1D7JO-GMf2wlCI`mG7KNTwR^GctBF3&>=u_SM8e`u6 zPthuQz?bo7gO99lRg{^84d1AWAL#Vd41LE~I47U26?mQgslWE_SgdsOI<~V!f6_VB z-PJ3tfj7y*&Qa^Sz0kSY`?XTV$9GHlNn2^R+M?V<@$i$jPo*u6t5@UoFt@foG0(TO zDco+>=vtF%P-rCrREXc3I=uEx;%Ay}^~FdhW?a`UlSw590vU(4_e=v~`X*VfwW+Dh zA)4;)*+XmLEmUtP^z?ac=Nb&M{u<=P&)bWL>_{J(r@Ttm>(6q_|M9kUz2DS&^>RcQ z)7GBt1=Ad!9PkeF{fu7Q7DAYrLB-1W%%uNG9cBKC?6WH@*p5k zzK5p%zzrGstT)Q3xa};j+PT`+TF(9}C03q)nqvOb3<^hYoDLq@bF+RMFd{O^&kjTgh?Zhu&^){ z5)=v3Ltg+Qior-KM*}cq0&;SMSTqTZ0^=?2wXf2J12bSn5*QNF2u3p9ARCsxR{a@F z)u9_wl}RTA=Fk^-Oxa@4*I&DSoenZiU&a$CIf7=Z2_jqG^{BbNzKM0NK))^=(B=`o zJM`d;@da83*;(MKQ<&cG-4N!NUU)N5`jrP{_{CuBhtPAfa&i`M1Ox>sCcICCkuAj4 zbxF|m->jM5A_85k9!&TYb*)73hhS3Co&K;LZ&XUWm&BZlJWr?WGN^G(?U9$Q^BHGn zXXjS3AJkyr<%Ca6eBs^x@I8oS=(XU6Pgw1S?7gD+^oqhhj>^h};MI*nIaE-BgjO+I z6+R(5$?9qsV&8=fLdkjgxS;vU&B^%!E-8o)ONcqNMsd8h)NhlLtRu-z_Y$Q80MTDS z8>jV_0&0+juAiNq-xqh9j0kjfiN3by9*B$|9wwM0Ji{EdGV)jUVPEday_vJTEFm%1 zZqp&~F=KUR2@7@aUTtaR(M_$A3>Wlj>d3T2nwFfHacUJ)qUHmC6 zL$SN4oyq(or;rWaTw^X$duIh1xq=x#L#P@AVl!Uz*b7}mi7L3)Tqv%qsq!*vGG99vH|0XvM*mM%jN+I^6L_*!azyGaB8X%kHgI&ODm}{ zfaYF)!9vJNdaS!v66U{JLF)e_<^6d(OBlN4TF(6FGdMb zh3l17U+D-$$Kc_E-f@g^c z^xGbs;FqA%YARY~KZ60HM|>)(v0p9{^aP0f^7poTQ)_GDw0jLT1&>udpnU{HL|a_h zSCFTtB*;J*)D2=z$YI9~z@upa4Q6KsMVRDUVGj}@7eE2b3`%Z`G})jmVFbT~ z!5;d1$f(57&Ub4i2h@PiZW0AP9f2D&t*{iSt1<4LE@*Lp{hGjk7%AU_!P^4|f-WMZ zhix;B#Z~XK#WHYm{#+*ae)dcV(Mz5vTp7|dFra1AueD;%OivfXNB80l(hgKMAAjW_ zA>kj!`TizI+Krooi8brIgdr$IcH_&Jd(R*TShM;m$Kj#H!b zyP{g(xl{vVwoAeb+l~ z)w-;#DMzdRtgNE3r|4>G7$pwI{ja#46D_IG^6Qed1zodMN3Gfqw9L)rXc!$lhE&Qj z3xB53)_iJkH@Ogv3*T#`^M<9fD_T|U_hKGEkbExJlFy^AB`c;?OP|No5bW6!G`pkt zTE)@(_}zS^PLh<)dru>e*F)T^y03Ma{O2;Q({iHgG_Lc{s$1WetSKh`$Z^xUgXeFP zz(9YA+_1KBt)wqyxtlz#%EdzCZVV)R4{1Ynfy^e=PuhwW?mp>V-6) zFC7VHiZ8VLpU-j(?XP`?*S=fc?^5P=E~>4C>MA9rMXjxg$*{nZgrvLCZ$uPr_R}X86~zFN$9?B5h6(yJ}M(zFl%mC z@(x?uc!l=?AYE5O@MjC|e#)e(CU;}N^kc-4c9R%TilSd5ya@JreqdoboX1cCw+jP1JHDQto}7&ikS3v{1omsW zTghAlZjyWe+=@U8F z*v{3pOi5ClYUIs%H+6NMQ8B?%YJGkE>h&*|G^<>0foqA}2F!8u^lHdq&~O*D4=aRc zNE2l`bl*=$uKKriq^620rip?-#ln5kA8Z*j<0(i$BaB7;#$KuJkS7-z|~ zXc+=Z%1~HeD4!os!~wa&6JO z+XC*ov21K|$908?iI3pe&4FT^@fsG6lZAs0L>-ao*Z)ugj%I%M(6xZd-#Iwy1AXd? zoo(12T2$Q6k@4$EO01GvfuEkG_~G#Z>1W0-Tt{sMk9#a((E39BfQx?Vv)FOdu1w~S z0i_a`h(ZN_;nv!{_*#yUUsslN^}2Y>EyMm6pv{d)S+C{hDf;Dm3tjRX$3o#hlpy6R z5ln5l06jfslmexO*N%6WmDrE8Kkl)&G}_+%UEuqWZx!#^|AF^_?s(Gk@ile*hXSSh zH_+33%)I!!7kPl#GTobyAW}f3OHBHa{oEMP0n2h(9+xg}wwJW64`l1ynvQUEiAjv6 zaFs2vC<;Hil47P9dF7WwP+$Agw`FCXjnQ9CnOP`Ybs29L{;lN~Xj{s@SiNsS^HwnS zB5JVafZD;4WpY-nMi@9D2D_^s8I`wd2SWyg&UW%Fm1!-Ff2%fCzU?7Xa}%vp=$|qv zhhma4?40!?Rm}HOTG>fg(4hII`}yd(4VwewwdEJBB7If zgRDsZ(M!h&X7-<5Xu@N^s-K4k1_}uA^74Wc^?ArF<7nP;7Y2(XUr_CkaccF@d`6gn z{EjzpFwVnSBD;~3!<_f1iUcBseIgVNJQ#t*4IjyRKwm(J7GRoz4aS82Kc0Ar@WG4^ z24->WhUXC9!c8Y})!Ueun83v#1`>M<^yzjGU=tz$b`^U-bpT7oAn=!JI7OmP;igi3 zIG({!7g*pgnOwlh2l`A*7nl5z21bI?P_W~9`t)fWE(wCb!HT6Yl@JF?yq%pLQ26Ae zir`_9MQf-oP8F+cF2W~fVv-IAXk~dt#T3N4)HZ!UoRy&-_@CoF>^*?_s`sVG? z#frX;H8!U8#VClYQo!^TQYu zA5{6sW9sQ~@6KkVM{AFP@Rt}C?{@#jmTKyU50&42F9@&C#$F+RhGS@VktP+er`vJ9@>ru zn|{S%yxpy-U)Eh+@biJ5*&_r_yAd$`^K%u-hehxWhm{gK>K#`dVVOc^{Nm+j1v9xdiBHa||P}X?JezlrxHryWV^17OgbrMM_O5cO2S1m*sCY;^E0EB8LB)fycirw02Jzkq;%FiuMW0KHsYUAKTSe8;o{*hDq{kkft@zPS;? z)~D9iY`1B>V4hg+ws`|=W+EVzix84`2)qtAq>xSkwi9+zoLl>PU|Ia`M;so`Ftz9v zK;AxS-vYK;FqmEG$Zi6l2cv)A?rz2Zjs9y9hA*`LFA#<=CI-edScbzHe+TtWf#!D4 zr>H3Us2J_tXKkgHL%975K#bZF5U%OXaT!U-VpIrh=AJ*Eo+kI22+Ka$6F4_K;O;K8 z+tI;E8n#ksyAI#cvy|gNF+!cO5Bp^Tkhkmy;MQ2GKAdsmHu=DXFX* zDi>yAU(6Mif-(1RbHlI)6`e4#S;unVj4oLv=~Wrg{a!XtX^hWFlB@_rA-K9WW4{ zVAtqb3jGkXwAA_iLHO0WI4rfbgb6G&-fKp?Cy$=k=>e7Nh=qH}U@#S{e26s_RvCY3#=PXX?`QNGwl%+H=!r4+c zlEBlb9_=B%?-^zHYLiIWV~+l+aaIK3?OmtlU^@-?jxS0rUA^j@ZO(&{%AvP*A4aMS zeWcRD=a!UDM@w9bHn=at!P%By74!-r%Y&ek@EH(qi~#IGM6St|)<9bUYn1m8wPCt( z2E(?bI-Zg47Imv@YBs=k&KOch6&Z<0dcYbHNr?h6+Leq8 z=Ss)|P}8s8)@uScrk5}~vY&uXPF(!)b!flFdZfaq-Rqh%+=jpSa*8 z0D7)rb_V)$If546x=>+;anddBMaiUVReb#TQ!-xhJ`BqN*RtC1CKAIh0;{*TBA(>F z-NzUV|KZB+0Jc-wj8~hJT*AR=Dz~4piXtF*wE`$JGC78-($rp1P!Pevp!{_17&_(y zZVep&aYAmK2)Cvh{0lWdKE5a!P8kHgMAGR$Us9j}W8Mxx9!g<%A_R#NIb6S=r(KE- z*+)_`GT3nBCUCDEKHF^Afkr}CcX+k_op^6C(3hqU*6Iqk$1@R%5>(x#B98yg>1{#p zV<8|oqNV+KLbBGMPXWiVsmtxvNV?;U0m(FBJ7r2+5}xR%e+38wA;_gO8RZ@Z*M*ic_=3;0{Yi5g`biz zdU`1HDKMn{vmuY%9fuEp9cjzL(}x+aJp6W*Gd~AVxz}9-z1wS3XvH6n?I{mEp3>5O zX;iJPPTN0tW}?x#D`4~qJKV-aeU>(+UugaY*V;h!H(&QY-3Y~-zkoVjYEk>4XRhm< z@VLC$E@K5gkWno}yVDS2pJkY2D24I%0`T37z|9Cx~-XL5z@f4HBLPB?tFds2ISg(I1? zOwPi{5s)qicv$`cLYeii0{_!jWT{iTC?j}~C4tRTkx}AC&y}g%Ppi*8Ki<+$CxWjw zo|Uuch1fMpdt*Kw+$bqYua*)1Ja)TbxT+Zxo%IpDF6q$=#XB|}@RwrAFGrYxX(6ke zO$1BDUzd$HWNoCZEA>uM#%ED*7lul)6e#T1v7s<;R2^|tmC6KZw5NL z=jWqO%T_HzFhj!+v7~O`CdCN^XCy;O@m^pzzl8`vQxb4$ObGi(mzo0BBk|FW4h{*P zMR)0QOo3UVUXWE(@W%2ZZnkPLzvJ9H2&*RP0Xwm95R-3=ik@9Mf>IqwzeJ{IS{*=# z0Vma$;`G_ZZ1kz&-=1(8fj^WqL{x>a5Md+hvbGi%ufIQ;i3r)Zbhad(xo7*X(h(fdz__soX=;c$4AEMkbfA0l ztz%jxo8-a=rG$^wuzLnp>gMEV#=jF?_h-m4ixI39u*@w+nH@X*avb zvl$zeodr5v4%D#;zn|DTt?6x2vsu;np>0o_2Dg!W{i$yQ|AH)lTKG9SfTE4SpX8AaG$u@VuQcO{j#{nr-bR6P2e-!(h$3dm3qL1? zUcE{t<;gQWS%*_qvx2ih^)imrmtWuBhjsirO|xbuAGQD6f}yj6@`}cT>E*$YF><&P zEWCUmcKP=-tDku9`eE;%YxM7OOK{%!Qm#vWpitkkYqk^h*c*IEN4RnS2*C8Vq|H3R%|pq@+(sZZk0)cL754e+2|imUj?pxwzFb5D5R|>YN@~-un7ciI-(Dd{yWvB2{<1BXz_+1 zHC=psJR%cF&;f@sEvRrj~kcD;qd)&L3%(4)ELcg^%l;-d?> zZ()J0fZC!41&>oWn`F zsF`6O0Vx1n1v8o#XYY2{V=i)phYMA4zmc}AEDmVmv@At0WBU^oR{tDPm6!h7Shbd% z=g6<7<_=F5*cp`=j@Y=)&0qY5WoIW2=dKZ6*#u&%y)W(E-25Zzp2)}qM=MA-|B%jI zkRto)_gY3>WTfq4UYU^u(c8DbGX9L%(nvWt7$$A+u2cnhpByCgxj}6~EA2(jKGoo7 zgebsr=a1qgH@A)=d15qKzu0;UepMX@X&eR5HSS6(onOZYWf2@;e6n?4t30PVaio?^ z+n(3e`8i(q$hS6pg$DI0T44EPo>)bT{tAP;cAT2^?msS>qFO;?zi2Xtoa3gecp2$Z z*HYDA!2~_C36_tr84z)3UogAvq4IaIY(ObWO?^$MwYDNWRcbcl59}mDQrj=yO>4VDlXZpc=+h2TFx2Xoy^DE7F z4rU^}KOK(sYh&OQ{FQBQcgCe|EVHTcV~|qT5y)3}Ja9of{@azi-Oul(8wQg3xVD~5 zSO3C7KeZK6NHgon5FD(o$Vj@X7wD_mF>~qcbaU35e);bNfxqa3F)#i~c2ji$+g^ko z^;5Rqi=IM6b}H+tDwYl!(%r51pU>Y*TBoTIVlT_Rlqwu;7o#)&PCj1&A05oDCKEs0 zYnnCh=CSd=VL^#@S66S_VGrTK0Mhpz?`7;tNl&QD@O$R6PR%nuCur{Up&d7%C0fVnYJ z9|Q&mgO>+LQlZ8O1KTkIXv6d0(jkwdbb#>88Hun<1|jHk&)X^Wa;nGfYEc>QT0JLZ zN%DNdCqUh0j@BNveDhCe#DfTpJRBwkEIfc?gRon{*IGyRA?Rtr&Eb8Y#eXAGc=VoS z&`AN`!|%yuaxN^ekNgCfwfJW`P-Co3*xGk$L!192RRRke`=gwZb*S_;-`Loz7&xRp z^T{5UAtL+oxZ6BcM*{aHFpl&oD=X`>2KRkh5fN$x3jljdTaeSV4pzI}K{^gd%OwqS z#*5N~E?nKk)z$Wo2VmW-HF^3fYd-zO{PUBn7`VeHCR7y$bNQI@0oWZ#|o}Ci*GZl z75xUI2kdQ(l|yE)6NW^-ii+5-3z9IowNwIdHg}y+_}c@CjKQz7N_I|`dma~24(SN! z*8eHCkfUbx#H!4FmrOx<`wwSLt^FR_>Z7)g?&C4^uYx)LUIhkt+cfjDlR3kZ?d>~h zN_UrCdF}oLpsREUcVE0exBDbabp#hcH2@N7y_c{)T53gLBoQ1O+mj9KjNfU#m+IN= z_;{pN{@VTmpTl}jZh85(e7>_Xo2k*5tHR& z(~v1Xd}EIllI>=z;d0%0B;|Yk6E4v%mY6^Sp7OcBmAM|u_+`B5{VwexTzea3H|~??NU}93PJS?G${#YQQn)0ZvrC4Y0IXguY3n4^>1u%gmFhbLVNDfazc_KaqL$r{a*qGLrJ<>Ig4_03 znAe|_St@>^>j7J_vy5th6tO$hME6Z4#z47KdXv~4?V!&D2r>;X?n>M+;=k7!AG`Ofp_z_JUOT*chhx8WE(BUw0$08Ilp z?c(47Uj(@W;HjD*2TZY6gqQ)jSqOA*V`Ib978N*r1sNJ73sXS44~(|hp~R6$f?a~1 zpPvjGh(-u`oB|_hKiSg`YnTmye;jbyff(64&D0{#0s*~7A{S4mZ;{g1e)r{Gh3WZ| z;74^{RM4h8*JS++-ZV)%FbxM;{r{LP_AsUn(Ug$5&~>Hk3N#=uYifkSX40JJt_6Sx zh?o+b)HlY(k$Me2EH=US-~OobFRg@^mlsSTe8a$~;3F(G}V1RqxD>xv){5wOO= z8^tcbOuhgbgnMuSz<@kD8uGO6gNqLg!I65-g{_7SC~eSSfk(x*K{;G4P%iuWEFBP9UAus~uuBXiD|k0zMw2NNj`O+j>s!3OAmKm&|@ ziM!mh3N=tLJJ}MZi*@hF?chMr2x#QX6X0Q0WItnVHRgVmj_$jiU1a%qmnb#0VvPqLPl*2Qh{M>o=8}}d;7Qnh6u-`Z_LEz$|92NECRlL?q3SI~M<7YQuQx`3K z@aA6f;t_YWI6X+Rk%~D&T87!@2O}cop6pSWVK-OC+TY!Mo|#Dq*fJ=fp(Q3AD4d@L zvhkBr3{*WdD?HzhHg?zNL|dop=GP=Q(i~8Cjq=F9&EM&^mPV=8R8LNtV)}jeNtGg& zu8I%Aw2v_8y+pbzlS0yNxpk?=Zu$UyR3s@p!pughFzEd{uA@IKqAItWS-%g8_>BEy z;Zcsc=J7CXq-iYwv{IlY%SvEaUt$<$k$Dozt~>04>5F>8vaR`dm~OC&dGc2_*T|09 z(EvNTr}a{&Rr%HCyFsoL zFxGMNCW9hQU85)IXZ?I7-Np-Y*lG_we%OeIs*CltQaA`Q{por5%I+5(_RoiM+beH> z#cdxe28wU5omh1?w^F}9^0c>J6J9<{*)0Eb#75teH92h(5&W9@0~mF+HOrP5|9ml! zHZ=y4h#idY%i(WCuf1E&oSA99Aa&^j^Cw{SmzK|Gq@&~-9xc_6*M#$lV-DWx)~H6`S$KvJsxL6 zp+J|K{%z@j>CNVk-r>!auEzGeR@)_x_8zMZPw?4y6&mThmR;+!h-)I9IB|VZSdwTu z$4fZSSWE62qS^gXyr)udA4FfT<+I*1^`N2V4yC-U{?12h#~yt!@2Q-|^A2o)f*+jT zwX$2+yj;mo)=rXM#>NVZ)6?>n5lR|w2lSU@3Lfv2DFMZ&DjPTG!z$uk%?L+AMBUK} zCu(we2uEBZ|4aA#V^V%jd@(V3SPbt!sfH{sV}mpwjBtH}64TJBcMc+IZ;inepzFYmwY?A0g7S~?kHtiHZ) z*)3FQVL3Hf7E(3vPK?%{dB}*1J;#s=|1kaaReS7pQwAypfaOq^;>!s&x+t&2V#QKr! zLb_mvC&+|pKu4dt1IVc%X#V%+5^diNEXi#^x7pa$g$IX&LJf|GhX;Z^fkQp#k`?RC zo0uT2Xa-r61=X_u>HJH82E>uod($E1P78#PW&=fRX9jrm;^{xzxBxEu{t(%Emwu}B zJiUb`NF3DjG@m0DYDjQ!()&xkK5&MCcLEwv78)bz?#ohTo)WnT*dd8RxAqwsOmQ-h z@*40qjN-W6F8oqgpg5I3-AK*W1(P?Zx8DN`gr_4o%Mpw$IS=$QGBfYvzu=jOXH)WE zcqRQ#kx@}i%_LY&UA_MfW+^uu2);0Z1t$6Xo(0(rNT>!DD;@%D>*@|pvHd&8OXYy+ z1T6A8Em^i$W566DWrox^37pv=+^!rlNMUavF@|df=@PkNA1&&lWUj;5Djc#z&ZC?J zsLsq|KuiZD57Uimd21Lf!M_E-vqj4~mIcEc7|gu@PE|=IHFZDU2ViEZT9lF_WEZ!J zSyUCIE;Ydga+_+J{BV zys%tO?s)y3MGqw_xp?Z&vgOVICM+F`<(15=a~@k?4^Ogw{21S!qa_SlKiYi*Cq#tz z@h0d~(NG5LSYO*XZ7q)v?ecW_FD?z-j5pfOPe~a$*=RNY-QS6OvMGAikPDlg6rEBq z1GoZ4_57ppx(i91XoI~6^;&GeE$m00TMLfEeZNm0U4?x z6ah5TEmHxd{c3z|MbkYS>)6>R@jgF=Hs|jiHd;G&8qKX4DMo3%#Izs!TAdWpF#_I` zR28h)Y3f@!j+I})3W_GI0ne7|xxC7lh7Ef-KV$EF}*44KUFj0toWdozZxW43F)@(1E zM|~p{D&u56M<>-r3AOYfE3gUY5n+h3;;R9gdWjS!eXk}PDq_VJX9MgNR5g#6Ys9(NB;@L*?Q=w;?zABkN z39iP5m1}?QN;#V2J-;#T;YU#u3JSI`1<|lOAvIgdV+VL=^`Y(L099C5k`Xw%$jpJFNk@k-os!C)5CD@U+1I;PfNQ1{6KO8v+J|$kTD)=S;1x zo=xIsKugN!x&KK^7Fd5U87LJ|xfgcx%{NG+^M$?H=-cb-PuFU8*V|YM4gUQ429M_B z=jZI#L>?yjAv1FZrUfLs!}Gs>y(=#-zy9+k{W%nfgNc#u#jWD6!3w-AWONVaptOXg zDQG&UZARd5u&?hqWOF{<_5RI3VAp%%lz4P-FulKR0w-1wsi+94m)GUU_Ur4Lj`eW5 z?;T7?(J?K!$tDnCi{S%RweP<%e`u4yTe9)*Ez+O+iw_S)iIb3Q& z%GX`^-JLo5PJzCDIKXHC%iPDupTtU69yT~AF1F~ge1)~wlk$?AF9J%~iDMped;9kF z5dE$DUCFC{;lrzGVj+Ih?3)8p-9V`O8w=lfDrC9Z;gZwbfW;E)^ zyJfWc<=6-shT5rZ^WuZGPPmEm=-9UVU7gArCo86YAIJ`nbXx9B!$9>f`kw!Q7nO2B zlu^@9>FNp>axnf#_!N5F)7a7GXX++__?C?&sxG=ub+#;~gS9VqcAG~iG8mpcrEMZ?@6uo;pfd_Z zj1YsM(#8GkdeX+wV+q{ZwB)tEyuaEH3!~=!>&1sB{3I&l=1)_I zhr2Ls3zCA_&M%I~jGj+o-#lqH7!qw2W41ihWTX2b^3-*nHPq`pFXhjf9FYjh;^;X?9u(<=tYsg)H2 z7g>@Z6dam>X!9lDS}9iVSetx$()t;PD%*smG#Va>xSwr!D zomO=)jj*l#ixENH=hQntIq3tU_QaE$Hy2Srk`IMpOk$f+F=^fLcHyZhU|yg=8Ppgl zj7&@%(J(`K4j2Rlc-(S$f@8_ojt;KqOofziFj*R6XRyeD%o$L&e7BmohyvXWRkf%j^~e0PGP8=~f!s^C;wY zgxi*ayH4J$zBxoo`T^5DJ-zEWIXHF>uc4Ah6ZWVC;{iPB^l8(sD*mV_)hTF(K4iwy zf!7T*MyyhHMIfXS!Y5y}7(q`dx?M_i@q(IZv2UpIhl!5+=3>`D(H=SUU~1X~Z>^wU zePMBUd0yetorfDgK3Qk{X`hh27mMbV@h9!qJ+Sh=`@rVH2keWZhg1YSUuhiDx3XiZ zX8atXdt^h^e^UwC-RbiG5N4y%`Tn;OfA4@J*mGP=lEm}S5G=>!qrWzcsmyIb~bm{Hng&i_{KIRU(ashR(~iV`BtF3qv1FW;3H{x7{zxY%chOqE4Gu3dd!t$4ns9}Cnx1zwle*H$lUh_;&Qs2hWZkJEOZfq{bt z2p5qf52u#RBYd#%;CaU{ZaZigSQM!U-x5wb**%e;@jlt5N71r&5P$o2o7=(*kXl_` zp4YGcyo%Kb`r|-FNd=10VyUO)f?(MDPEly=c(9*~Tv-0C^nC{hy0$i__3curu4L-K zpp7zTi_Yl?Z=n#;qXi)ph=IxQXljiTGWy8vrp3vrMOY=hYBBWmWVRM&{GXpcTD17i zOddw<<)xpRm9@{?0y|zk99a9=!~ZDki&(?ET102}D4=S)AB+eNDIbHNH@-Q{*M~5l z;DXxAMbR)c69MDSlyR&ygR~EpbR=lu${ds?+}^7$gyk$bCvRts0*AEj2Ea7JhD3Gh zi6nmxj`frK*DOqSWt=qQRbOnNT@G~6C{lr6WKjmQ4n(^1`0-;}QPByaI=iWJh^&X2 zdYrZ0-q{)HCyOlln4!x_n1((GwxlLVLkEVzqB#Vq!H2*VK-7R9MbE`W1mvPth`YPB zh_r{0zT6VUq|%L$eIGv%UwL)!eLXn2azG3oh*{}j3X_W1h9EW@kkoAeCLH-cpOYY6 zm6FbNPzOSIF34(Ih*$s^g*a7qNNJ`pl@#x!V`j#NutDDj0~87h-K~B>GC)Ulu|}u^bM*)&XB5#3PfCN=jdn|Ewr=dZQy>3 zB&55nP$O*)oUp$v56O2X#q=2&_n=UOL0I1VJ>90Z`nlYbgT)CaSHUWv=Qqz|&WkVc z@S4`Ao_EG;=YqDtpe>5+vwWk4Ly$?`uT?a8JG`PXp8abRHa6=k{nZn;*6Gv(czw@e z4;1kH#F=gSO+IlK)LypR#w~Rwr}7OcJMS)M_vFtZz*X4;jwlqi6qzJjghfGba+3-$dr}@e8LB@a_ z{|$g!aR3KJJ1<~EqYjv*Y%*o=w>N?lBFb#B2GdQErnaslpQ`|I{o8r`^I>(&Kh+I7 zN?GUTsTd5obgC4^(&!rg=mpppEvhc4co9EAW4q8hlJSRy*7(tWm|0(&|i6ruUWZ z{m84QG@_WpJ!zm#wi^B0DhXRDQPBjyNa4}T5ec<8kVL-;rFxGKlgJN>jO-9*pQl^# zlj}Dr3M? zh(aPFT7kUlX3PF646Ki*K_rynfb>Zvbc^NSYTwxg5I)$7CHRGeTtaN>tpOlq`{ zJ=i)xTxjhV`uhRo1jEn4dkcJbsAIsxOj=G(0>pe!>f~Ew0*4xMP8z_S47|3piTfd8>`F`QB_BTw zCK<(B8*#-0SUsWb&CP(%gC7H_cY$b?F#t|{bYPk10t4&Qk5an-dz<$X0VcnrOt748 z^pwRq3D+3|>OfO5;&9Wbv^Ls70jgA^EN_C@Me5O%R$ozK_rR3=TM^Fu{@TBA{Zo^$!Y&FD*tP!xZzSA5A zW~a6j;lL3-6{G!+3S^c(u!K;7z|6E{4_fJ|g#~kj;jTRa1Y?A(8v@|(6?O=wrFbmf z4>V$c6$`{D6M_1wD`gr)f!G)V~Zai8Fg-S1>1fR@dK8lar zZVbDPvA50M>x`A64V zem_t=gi;HW1BB}*PG9`b2tB`RWP}31Y%mYF(+sms4uHJ(_VxzZ@6fwzLvFU6y}g*d zeK8ZFiGrgXj7Czwx5N9boZV|xln=-?SW6;*87_rXANgc_Z-zxGbi@5GT}8<22*WJV z1sas7*jQswgy9DXOwgiEUlF_}L9jIuQ&*4C$^&D0P-lALetsan#f2E6z!s<7)EY|W z_x^r9O?|8t3zTemLcnwZT?id41t`$q6ULm-mu08IE z%I!MtTl&=gv+Zf*!V|&7n1*IIfg<}VPKcifteNpca=W~tgojwj7T1aE2H@TI+exzL z8Q3nynYJfsOPVbORDb+{U3IcXkNwkf--PTy*%tcEq#t`42S$gvp4Leir37JPcy)l0 ze)vEHStQc3iT%WoEfRU$`;rz5nTl3AKR~8AQL#(1n~=E|?xW0P**_YT>@?fP07DWw z6@L0J=SxhT%Ya9G_F>)Lqu=*mH4d8F`d$-(WjD>>pRG31H&gf1qn>OuuF!X2+24y~ zuqC<@932aW4i{E?C2*brsH;gFE zB*hi(1I`Ir20rJ-D5jXmNFQ)+=6QItN4%F0o*z7GL{!D9TX-6 zd5zT6FMr~yoy^hoJo0VtJbw6??M?|!mrw~VDl$4 zBn{)n^IRD$K9AI=K%`Hw)`DUR;q*eqhgnlQHx&Zc3UVlnfcZ)}nRxoEET94E&egaL zJKhvz)s+K&hGb0G9Ur=ZG6wn1-;hxn#(6z;M;fM?UZa!H42kUpuCj=VP)$R;&cG_T6`OnFVJa9 zs;Wjon9^c3@N{9gC-nYx5b!E z2JZh`^}PnLJyn391)2H3qF$y%5`;A#UgQ7w{MN-9Jo zB-%|vLP8e#OViW7aI&X?F&xSV*6#=KmdIDXpRdjUUUf!^S#ltV0Gj`+xHv2n)DplC zyax(;Uq3__hZirqL$nqgXmty`V-k6+bDU`q5KlIN>mp%L6Gr^mDO>7|;fDs=2I$d! zn?DgSNE*QT(K+0eCVS&^;Ox8xy=dKzhf}8P#b(zDIp35N zyW0wNm)-AfI+Z-w){p`fkO*25-U<6#pxOfBbQ3sv7_gJdZoqAH4{k|tEa=={2Tclm zII{xIi1g?wBA!SQhgjgqL6y}4KLh$_bMhCFfnT+utw2miFz~2DVSk*4>_rl1Fyla0 zg1Drl5daI&X_ts)oZc7tW{|gnNQ&FqECfV_L5>23qlw!B^aQ1BK+ytA<7w#KGBlwp z0ka;fIAW!F;B-D#msd~-`0?adaZDB&;K+&pkcd(u08QZCgQ*hr!On^wEmns4g9i@~ zOcKE={qGqzvYws6;7rJQX1~To9;WrIEs6>3%!-xN#l`cNz?Nw6llhyW%^4Ispei5+ zebp6Fc1J;^a0c+$ZGIPE2*4@8AS6VI0(VuQRy_yk6mCkoTeonv^FKGZfD9}W?9yOy zp?t*Y!CzEVm6;emeSLl3fzbcFZ zDrD!CsCi*^L49356wB}7;yPMgaqeJetQErJkePOTya^*vBj^blAfyN8Ig(-6O>gg|LwtL1%>OWX*C>P&)7+BEnbw8 zs>oNl!NM;iI8Z{Z?2n!-_C1jDYUrzWdOD3Zkmf4o12HEtKfi4{$J8<;LEpI5d+_s$ z7Nu|k$VdCifp<+U#3%_cEEoBCN-C4}bwzsSwkyW1IUsLD7ZQ4U6r>#>jb2_cWQbTg z|39|A1fJ@(YkO#J{;Nl(Q;b^wHl06)Lcjet{gn58Q5K4b8nK#>hI$$Ug@L9ER zF;68WB^q#9z2NCKQXl|hRO^wGqYpd@(fK67>*d(pI2*RcrnroZ3}s8FQ>VtR@mYpr zjzIv0H&)CR=f|Bzix+zWV(9I*wX=Jf;nW2l?dYE-eW~+^wd0(FiOrJ(SN`w~`Bx(9 z=1k&nPro;7MS9N~GYz*2>-`By;_bKO7LMxLB_t^oDhW?;E6F=s!9RZ7E=oPH38Q-S%T%cAW^DxX|~(%4E0-3pS{2!QUljoJX6ng7}4wbQRPO?sh%BNhDAh74RZN)NvAbAdst-N zbPPNSx&6EI>o%HShHINlY`|uciIr_a{i8uk`NMY#S;L*_nVOzncT4-=*Zwujo9AH; z*%OmO-URa^2H}9W7~pXD$;Tac-j`7K4?18>1bbU5W?}KHuNy;V0`oJb-aHHo3&Bm| z%9g1Z)aRXckS)_2usi#SA;`tWjjF~-HyOjf ztA^Y~u5%NL#%-8kCy?oAUT4*+?WNB&al6WkS@KNJ;cpAFVMGP`3Ms#K>lRI<0!FVE zZf|V7NP{f$^78)cB&+Y+cP1`tU^)nuFu(1~pLwp!no864LAsYpjb9t7S8eziv>>^h z4sRILJr+kJ`>~uFP5^HUrR-QyBH6g06J&O`FFHQLxYi+$u(3Ie!7w0;T-z{PmW}`r z4wSHaxrrDFNFdSWoi{rGHVSn_I!?*|i*^N~yx<~(t$G!EfpOOle%l(_n*dqhCFV)74Ww@*PsKmvVI$V6V z!SWLmS^p`WTXjt>t#4qD$Y*zD$B`I&xIQ*OznC;-O7aN@_MsI~`fCzRbeY8`vp)Cx ziOp_5Jqpar5|oxg)%%c zP?7B)SmBE6FKC>?i|7b(=%Ca`hL|zdk&3&A8-D%44{vbM56CX0H zmP^>VXHOo$7!7pfq3d$+@V}s8u?c{>ZCiXT@=^w^krMtGD z`FrY9wKL=S_~OOaJbbrVPr8f*n4I@kuZo{6eQB!Lw!pu>y;M`i0dGUChiP;QyLMgH zaUpgRQ{p{^yFho6iBQqyQ{pjP!{P2Je3Si#Qn5H^xUtlAt*~&ky)zvvEabeyLz* z`yZwrUE3mA(-+Xt!#`b3=)e^$OO`A_%fEVF7y1StL`T2XJcu!K6w&-?zH=$+gebfZex8)h7gQ_|An!RV_oVIYN_2|HUoqQPPMx4s^XC#cp?0ClK_0#PP z%xVeF_FXDHAN+v?pu!Rw%MbovbpwzzG{+U2Hl--uc0$#;M9$^oayi68*rdW0Z!5EZ z$5LNN5hx^W@E<@l^lUuh@UA^cpjG>~S!A|SO#M-A}j`irs@CNvq95mVL z($eQB4yZFjLucijuW*gAG$umYsgrSvIVgDRTSfMWcx!7b8}uJx^8Q4=@VB3rP3;aB zE>sZ>SdUn$fOH)xV_q)Y-CS01ev!KQ2NsiNp5mz&egln%YE;kl&4{B@ZN9 z1?s#lE1QqVEExCXFZB!B2M%1sy54{I@TD*xW|sky;U3z`wPI$X#1{RovQ{<5GpQXN z=|#Ymc>Bl=G$7o;6q9?uOPGmZxG4M*_=E6-I~#8mEuF2iC8~_gOzz&D==Y({iFHzof3|#rHZO* zF2>>f>ldhtV8H3f=AAS9oLmC_7wdvZ}Y@pEs+c5h1Fp-?_ z#98{ypDpW!2sl*7Kf1mp*m*2UWD~`3m)Z@%@%!hKF&a?K()Ta*DG3nC0cc3R+4Blw z8KQNP1SaoURz3Ynm1PO;0>VWz=PUmf`g3%xv3+qfk|9#ENzN9))&&P0F_D!+;_a1- z5%uV4`Er5ZO+j7#x{eG+`(efjrYt0@9xjchwQwDD(5NgdZfay= ze2A?e9M-LiQ?|q`S@0@%M!=*kUEbhZVH;3M{OA(*k`k@GR9BkR15Yvc-akp#p69>ysO_b?Ox)yN zyY5`tZqpXci;dGHe?{%pG=3Q^s`mKi;}^405v7CgptA*$$Nw-!;?bCV9#Q?N*LRk- z7A)CtcoJk3wLmH3%lJxdRmq^6Lg2$J%o-o-@)09oIx*Iz4x40?L+e76uy4cUAt4yL z^{*mRJ7)TBU4H!|@uYDGVb0x+r@@_nrT-jPuwksw+h(XeG*25TccHX{uyb#Fq1qct zmk|ahY;2xk^=v@gYP7;oMMC58Y}Jqy%8^AyT5 zmSA{zc$Mg)keNH#mIEOnOP^_#moYvOP2uV0NG?Dkh% zIp-My1ctJq8(or31GDkhty_~Hy#IJe56>9+iU$v2@;K{s2K@i7(5>+NKioIdT_Lan zipgV)DAkS2z6#SA_H|tk8Z+?<3IBWQ1c)*=1Kr0wk3n3cA$5bw%i=jsPR>4wp4Xh= zbz=v8wtB;JX}dpF6g0iHC5xW)AKY+4kQ>%gM7U6$f-~1rsYJ!wo3TGIP4+HqfAf&P;d})UnSs{eWFDQdMgV(#6u*E+ z&|hUqFvbMuJF1u>jr6n$=Yy&$OiZn$T~LJfeSn^SaGx$EFf<8F=yuN32}EKA(gT&G zp98L`7+e0&5KBAs;pZipSL{_+H@du3&Sl}c%(@EXKFgU|cgg?KzDTB!{K%=>QnPTZaTx`TcsDH?asS;eCiAFkZ|{ig zgPK(<|N8sb_PP5vF1^xp5tGRlAMt-yDRFh~no4B{hu@nAkJQ-4pZpehFXbg?iABz# z^@W^Vel-ECF!PoFOo|AJy`$4HC1`iN%n84Z6Oj2Ianu*|d{&tD(}=U;6kL z59~8>#KNrJ2@JeEblV`fcSnb=v~T;gAF_Okq8B6B2vaVPL8Q&cB+GjEl>Nm_ui}7; zn4KT!EF67@rV1YMNIg`w%{7_nXrvHTeiR#94srCmom9P@6l#S~e}Df|S$p>HzXhvc zG!U%Hz`*7+aJ`^EL}2q~9q?L1feN07=r($(hgib=^;{+TP;a4wzWekyeYDPtGJM`( zL>^+i*Wg=jUgl18M}F54%~^i^v_pU8%4D04BODC-7e&8&dJX9Z7>vXv5_JJ5*RsDE zgP|a%vA3x_Uno~F_V)pU$vLhUrzf_Yd#7$T|Luey43Ae)iHVxlnLW2L$~WWAXR8Wq zCLce)1M7m0Ej&+dZmjV~pTWUFG^6V@1K(x3pmB=far-OE>!wxcs|2)hB)WQ(DVG| zT+_~Z9ZJH6XMtCda{`(rqs8P&lMWP5r{}?7YE{G;RsctLd#fx~+Ofk7cyBs4chciU zjM6wjd=Z`a?NFG)WJE>PAQ}Nh_$e0dHKb1T&oFsN-;;5Q=bX8OW%S@YKBrGWKolhI z&4Ft8bIaFPzVsCCUMl!{+|ls*_3QT4JdbY|)PKNtHg`R^8t;htbD91CMng-ure+9$1@cQfV(t-xsDXc}v7KvCktoMPW&xe=r z@x?;#dW1SqIDg`o>j3Cx;JVVBdzzE^fXDc~_+A8>A5HmeHQZn1uC-;$X-Utk zSIc{|dS&|u2lYO3<-GWWsjuk#?^t@_+_@&h(H=wGw`yu66kOjA_z(e=s>AWy%S*g6 zj*riIdHgtn>f%$DYx+sdnlvrEBbN7k2a5|kDRhtguqK~(p3j<9tD15b7)B=#u3$eG zg_3<*T47*$RA}eCHAy&z>q=!BrUvU^n)?S(3HEibxHb6Z&V2ybhBoBqAtn*)&KouQ zW)Fovii*mt#4jp~)L1h^Pv^yYE_}9vCOcVnztp*DRXRh74~?w zCP!4YWntvi>!-D}IT&B`phIng-D!g{dfqA!Lz^u*guU|~fWl}2unz+y+6&vB1{VF# zVO_CijS?!oypYE>0+``m};RNAW%wS)ytU&!^?0C0jv4_gA0m<440DF#l}*OQO}!s z8O4WX`1nKA+g|+lS><~1IBgQW2VSK~36pPq^3f6WE>mR4%t=@E|29=SPU{)1X0R5Y zw-Ucg=jpp@;Q~R-V>ghFHRFQPK^wx-@XiNa>*h5XPG;~%WV%Y=%{*>DtNRJTW!op* z$_1v><;TPc6POf8_+$<^q<=nhp5Z?Oy8&R^45PRNMMPqH{2o7E@3my}KJ3JXQ#0`< zyy7p`C!$g**`W!dW#HK?w_vzyW(06J9zaLTJzQj;SqD&b3`4MaBqr$IL;ub(SS@R8 zpe4F>Kj7EVp32&R2^_(kV6}nFjp96ue>V*FoZ}0#%5>NlJ zC>GWBIb9nqmgfXokQcD#A_0Xa>OzbNLve9&`Z<_TtnyJ<(Zc4we0e~FtEU}fatF|I zopHq7-F-N2v?sgoPOk`l0TS%0@Df^EaQLs=9OOF(feqJg*q}c1!Z!SYab`LfSM-Hf zIWMFgTK#ZEA5KZe2LoK>bkFs-P7m$?4l$wiDRu#OVFvrCps+Br9*mm*X4^0+hgO!l z7$a%(EyGBWcb_V_IT)pxZ>}Df7syu80l1hdbqy;C(qAz`?RY{IUnU!#Ui8ri zj5L0PURU8ptg^=rA!%tnT2Fyf7@8t(^n8*hFNY&pkt>h~GL8fkhVerthf1IPcz$tw zb(*~jntlv-S9-e-9Z}rA{Qz>U960PG7$Q10A*zi2~a_dhBAaG$s`5=;$^OxQyRc$y3ffxhrbN8{>r=2ug+gjEHozM~*&D@XYV^^~W&ZT~I_kPK=L-Ck!98 zqD9J^!KbFC<8@nK_-Ps^BXQw6d#`=zj~P+8YLNY79F8)kLP<_fVdHA2yf7dRQ(#v zb+B>%c4>{U#pFa`GWk_nTlee1{f&rKM6-K+n< zJK9UJuy`-bl8=DW9P{woi;=d3ElLxeWcWVgGk~2rAr*)}X<#e1Woql|-(52ELht;g zOA#Qn^1L__@I+z`X6j-un8sqNBB8#mTMw+5L7#|kzllj0P4O&|{eENK(xtdStl<%O zARh6Fokb}!=SLbevt+auaCq?KSlx6RIuJe7roVVBAhDD!UM0tT96!(u)wfTo=6||v zV1mW%Vd)?C8%Q?c;AAM&C0bTVwAE(U`t&rHsQ~`ORh)}P@J+;wV0Nkp5zay#W^rOF zK2PvFkltpHKoCO?#B>1_Qoe;MQ=U`EU@F0rQ4S<)4?~l+>}qlG{V13&U%hITau`3G zL+olPj-`^5jJx5k1bBm;(d2_Q{9=UktQM{o`0Hxjw{P9J>o_5Dez6DkeHf<1Ip$H$ z2IV4RF^M?~V8|Kwx2sm}ar-s*+=UB3&0hO;bnc*!{r=s%OEj#KtHeRlNy)@Q9+SC93GR#0y z3JY>aQLF*a^Eog71uxcXYHF?wFKKf9j77L#14d^nrnCYz?K^QI9#qkb%%u?;AYXJ- zPlV#%y8OW0yAT}ye5+)=VlG-ue_w>dT^L&Rgs_3Pw|50hT=QqoHbOM+ts9doF_%gUc!2DIxO5wahlZFi#1f`owoG3hKdDfMtLKASK%9iNO4s&Z zg=Ij*Y|>2|~`vP?xV#SYPmJG$X=MU0qyLNpjSeI>Kv?AE)c>3po+;wh7`B{UX zJ02k}@`!Ieg4TuY*Rq*I=FUq}^~QV;Mkw%k(6Mv9U3*&^{a~KoZ!C zBQI3$g5rDt#p;PoOYq06HY*bt1``?nnKe(g_gWj@qX>wHzYR6-st4^eUmg|D;bJw0d6&O~2G@(+M zH8pti2Ci+e8N6R$=yD%6C|&Pq*z~yOg9*zK4>xv$!UK>W_lZOfGrs_wfs(P|q;c>i z{2aQ)`85w5Xey9ZPaFf1rlVF=4PI9K#P^+Uwe0O3yeQ44?tG2=s4GIIH4kzw6027O4+m4_ zyb%l7ObX(Iz^tiLi*Z^ud)J(Cl;In8AFUlV9az-&t6-gwfWVvQS|WRJDJRe!U@;-% zz`z(V6jR*zfl8p-q8h{>sEe0W3vKoQmr_lDE#>+Hw+)FyHURJQ?#viu%0RdPoHR$2 zle5KVEBZdquO&0s`Wx1!^QUd0NYl4tBb@lVuAVNVxq;UJJstor<0VZtb#<1&6-hHI^*Cjnhb-Or}*uU)sUr~%JA zC=Tkl^Duq@D2%QIB!&dIr%Woy&o5$4kjp;sHQd!)3bx-*q}stMMf^^pQM)|iY&l4b zG6=O2TX1ST>QS7=p&Gao4~lsUDu!gx5f_r+acjc`tkN^NxJvi94`}eDaPsbiR}Lw3 zq`-%b;0_-SQAEjFcQSi05RoFI4rEpYB18H7tq&SNXTW9Bgf*12#U#68$b3hWt3<}C zL`L&Ayj6|cpk>MCmCF9Ir=F(SH62zJU_>Y(O=CD_3G7R%sj&Z<6#9pv{6&4E1&+R$ zf24DY#&7G<-_}H5M9g0Krp-Y}Qc{}`5-w?V2STc}z!}gUt6A%hZ<+_ue3sXHO<>PP z!VFN!U4LM=^?)W$IM2p6r5vaQklvPVI-X-nv%CG_{OVmbL3QVU021>Q8z8N}XFV2a zSjQ~R8!goGrRR^A#|4|G!TT0(REde@$l0oK^F>+I{>Q*djLs*<1w29L8o(pRnBH>= ztC3HSj^0gi&#E@dGQX~_uIGcwI)RldAJ$0I8?kh_UKB&npdc0nlc#BoRkAtJLGTDH zmY5hZ7c&GtWu8h#^Xqc&JdKtO5cr8IS~62f`Hou4Z#=B z(LR?U^B5w)@bBboapNMHt}FONma5BwCl3@Dj!>tMQyvJm3e0j5ru+%zG4{lX6B%_l z!;fePvcy&muyhU`H8H4R``;1@pm#ysS)8?yJ~<;o18YZ*7m_Pb%b?}Pvt>y^$^ zpqoY>fqyb94&E|=o^%%p`Rfl+=CLq4lIT|0KcdjAFvB0*Rd6USKGp^R0Yh0=2n0@b z0GQI(0Hl0?BR+WVXo5%y$d`{`K;?NlLHG8{m)jAx3Z&tldHR^U*oCfoc!gWkc*M78 z90sUG=tUEl8Pa-KDn?vC3JKW?+vHx5vx+DM@lr6@^CbO};*B^ns#rWtlt9ZP}88;Nf;PI%%c*tr(n1yN`|>FB$(=3X)G5fHZcC+r7O<-qrw zd%imaNY2J4%tW}&bs9Sb{t>X!5BJ2mix0o!pED-}O(4+6^0B4%%HvM3C=4%u?o$bZ zFF-;SEHqEjnKH^jaLYR9!N*W8RwQ%8V3|xJ=w)M5Qz0=i4YhE>y>b;2?MmqJ2M$Dd z{(gKD$o2>iM!AfcC_*20e-h-dpj6C6r#cN=i-$=|U&4pP{+o%ajytm}`SwAT#@ zZ#Bi{VC*)lx*JXPx37D@FQ*Rc(c{N^F^^UY{DnYA@%^*(cn)jR`!;)l;s)UcEGW7&VS|#-C27#Gy>s}%8d@|pAD)(Wp z>_`C{;@lh=PV3nSP)tz~#E7aBD0d>C;Py4Y%s#Ik7xn==!{wRRa4-Wl$Vb?9Y;bIg zb|Y7uRk{9>J@PS-!k8}0n7p9R%aAQIsj<%d<;B@Z0R}dy`h>#9B6ECvyypI%FN;@K zwK)J7sR0R&S2`>fOP0CAl^AHuyFs0&7?cXphZYpI;J*usvkw`h1pJL^YIb%ui58Hb z%5b8a^Y5J5W%*W^z!6jLzw#+P`$HV4b6UT#D_;aZ<+U+CSxvdi=W7pI9i7flKkZQ1 z#v9YgNH5#ZV63l9v3Y<2-zNFvfK=jj171}AfDn-nt{|En^eviMxIfT7Qz)>5hHLrp zF$NFoB)x;ppykb)AWd}SPL!Njte^VXYN@QVCiLXcV=wyL=Pgdy7t^U+1k+&$L1z3UycViF(t{V%Vg>16ECH&Ub#{_%e`-#TlOSg52J;U zcifDn8pJ>shlYg_WFGjp9%@IdKj`>%@{GmzBlF+9c@J;W{(}eE6d@Cx!%4?VUX4XS z#+NNXq$uaMQ+F-)8Gi+oOJOq2|V z1^0P$0ehd>f5$1-#lWJbyPhj?S0)K(sC45a7I~y;?rSR#D&oJ$y>i=^74Fw^ttxBBfbz552y#nTL0TY5ZXACyl`22CQXPC z1D*OQ{bU^S`AXk{{Vf}Spb1(*#VgHe!y6ROL8+)jY*CN5cMoEq9w8w6P#`OCl+A%B zYc?2D+z0853O7-h_$&0VWP#Wpf3d_qI;;_Zo*Ih%& z1@3m7m1TT!3*2P0HOOrsQUIf-=|6XzIa?vY7T2&rW~f;cge=16K4de4_x=Wi;P<2b z@}qe^BKj$Nf#-pad2bjkM9dzqWN~Q0`13bsqLvwtyr9+ayb1uRY>vo8<1-v9^cR;a z%Am0Bw;msx`&;$`TAR}AAWcLJv#L#HUtYX86~dH~<)_3aSkZyb^2!PG_n#u_zff)W zHG>(ap2q(hd=HC_*sR({?|>KUAAP=UQL#R+b##v)+Nyf2kWmnmQp&G=_k1RF*gEVQ z=Z3(Oni^fb?j@X@oXxn9XTODo#aTP3>19*+U4mh#)~G?{q8cP;;E(!H@AsSEYK^R*t_!=EHH1Jq-AE-HqxR zuU;uClyt!%g?gewT1{283Z6@-Nr|c5-C(;1Ie`q zRFvMwsj8n*JtcuRBv?g?aSK2^-99a1HkORq>+!Umdq1D7KarZD10V%!O9a?j$PNmh zrEPy)p&V^3T7)j^SY!pA(&=8x@WB#wuJM~6+%wqx#E99^Knklt`a^>SRnY_qTCR3m zyAn_VQHEt%uBR|pxNO^zAu#}NnAko&IdY*$AJ$QJfCW!^W~_phA(0fYP!lDwfVY=d z8Ty$(9$KnyVDy;$S4s{1zfu>xP&jiiz87bEn<_w<_me>jE-#DQ% zI(7oXqXP9qqSa?9jq<0KdQ>%D&(3x~YX|&s9v!^jx{h1v>iW|JgUYtwoK190gN@Sx z;Rs3)_t1dCA&RJ9R7k}6zau^dySoW_CFvdX#h#`TC1gph?UOu@(@(oW8Efjz9?fiW zHo&rrgPLu(#Y4kCcB7+SNHOOm28G+~?)`rMN z0qo=waM$37!jb!IiSw8xKLlH){JcuHbhI03nN4* z{?a5h-Aw1CtW@S8ks4^qHQ#mr{bm4Kh#zfBp~yEAQX>W=?9roLxY2y?Sp7px#fenD zK3CLPx2|3L`@)N5ApOxnfSYS*h3R(!?8Si)?thEGriBqEXGqr^tI3!eP_N7IsOo<@ z?smeXgj+}$+Ahjz1y?D(y^-8^LctDtn3$TKNO4Y@POKvPCH!H%^j&r^2O+yn${~GU z_>4zHKwa7#Xc#K7Y=pPipBruqFC{zydzc>sWt=_))I%c2aRAqQ;;bDSnTd&^^H5e= z>evPB2}#@>Ji3Jon-f7v`YL4*tw@o)OxCsBLXRKomMz-?`@yM$m32SYLTo{rupP(RX-n>_@bW10B?MEm;mU)P3Qh8uthrzEheRYAQ*?< zh#73rQK8H3`|H`3H5rv_Iv`>oB|hsKQV_d6@7-n|M#QX{S=nR7PaVgok z?8|%$>I<=+%b;1R`ZF&T8q*UhO$Yhv#V>!*TlTk~gvG~Tq?HmE@bVP-z?~igiXbH? zLcrt4OH_&|k|6?@!<L0fyE#0_;FU ziIoI*7l6X|ZMN*`)i5J#5=RGWBBK)TCJv5zh8~B=HWVUwqPGG9S_%PYA*xxxnsICf zH@CQ#ET1=UbY&-pxA5P8E1>jcdqy$= z82^GVK~d4HhCzTj6k<49&7EIg+{hXk4SMmlGYf)!Zg2Djx$4{;^8wXMm(_EkZMYF5yT2f%<}z!5m58z z&Mi44OGlQlFtiq*S&KGq2XS)b>q^=N+^?YEBN9?(!ZUXLF+4m>R4=R*+uhvmwoKf6 z68}2Mebfz-6Di~WqwU^ZhsdN5I9}*l=c9e;5yr{slupe#o95CVPvi9SDYfFj>q5cZ z{BK=KTL3OUQiv=80SD^WxgX1S>paWPY|OM^)c_yjN9-Uek3m|*D$)d1|>IKb3oMzI2Fluf&Str5liXeG37x_!}Yk~ zbJwusIJMd*%fnjsbW=yOrrRxWR>75xOBX*>zB=`foSU{wr%9vMxN*0z1K2- zi62!|Qj*lCD9lvBofq$6j@VrZ1(j$-GF|M21EmyAok?pWAY7pBF`^0W1+$h0>=v&$50PS`IX2(^;x%CKX{D#5b zkZoq&s-mu;@qo-UO?Jq@u-K@Sm6gpJk^?OVmco}JFF6VC+tk4o7Xr1IcsbKp5Gl^T zD@~o`*fM%ECYRsB7YRl8`M_J%donr@{S$-dGlcf1a}SHIFtl>mHQ7Vyhv`i>Kb|NB z`QJM9z_B6U9Qe!@fUzAO#P^z{#`=^VFqo(yW*ho&$0_UE=GzC?M@2Che=EmVTKI1? zzHC*Ua-WE1H$*IZ94kO2%g3Ss05`xRVU>3GgZcOl5&~(Fd4QeUuRTPwUIa2Nams`X zg8CtKH{xtlQimVjXv;Q6;eK=hFaZ^bwj>FxFhJ7v>YLZEi;+K>7)2fZnBjKFgPwx0YMGAx2(bU@MctF(4@-- zvMWd@1-`Geu=vLI-r9#k?*S9A7o*wO>W74-KP;ri;4QVLcP9vJmQm2|h;M)`_|f-^ zjL8Ew2MNm(r=RdY#PrH0UF9ULg91+g$V^(#y}RDLa1o0fI2lwC;gZ)qNV5UwV4;Eu zhdV2{WZX}LUf%C-csCtCbk>35j7RpHckjyK&4q7QyTt`ICMFG#&0Z_&aUN7NCg3L? z$(xCqxx#rjDn_ycVrUNpnYh!7kv)Ppzb+^!sj|+Fad#U#Z~;It0`m&b9NFgJTfN{Mhbu+dS_8sbe5HTh zt1#4!C8(;|MH6<%|K*jFLBuCR726dLubeZkvWL{P1{OaS*VWqC;_}w`Q~XmIec`KD zo{O~BzxZ#k-VpH9JBsH|5?0)0F5^79^?ts=-S|m+-M0=~I;Wnx@_b?|xgv8~2p(lP zB?zd%5P|y9I9al_D0-YdlIz#lBO8kDSP2&URQD~0HI zC0mJuH3D!(m;%?4^;!4~NMa}^G=C~kVZ5%!zF=*_^p*-B<4iKek!6keo!EIEgAU_t zll0R{i-GsvZV%BIgbZAWvp$%JPNR+#lx1YGhJ88A62f&FE?PJ#gRl>Ho}adrH5(_2 z+*E|#f8p2{-jme>a{|*F?Q=$90D=|WD~zhg|5tdBn=Y7|Ct>p#rB4vsNK=7eC0>sP zm{K&#MWC!f@Q#FpeE}2A>z1BGzXGLW{B~n+gPl76ocYJFqx_b2vRFdDbWazU*92xiZxxC9I?F+wwydKz!OrBL`@OWF? zCv^4ZRHpWod98z@>L$m$P%kAi-hAb7zOVAI z)#9(|4Ax4ZOwjc9=x5Q8v-r3^G(23=G058LOSdG9j~bY!bhdZwqzb?XMeG!wscr*+ zOdM&kc-SQCtL!~^FcyZHebC!)<<&*@_(9Gi0b=Hb2W*R6umCvQ_hh3gD;*jhNyQis zaC-&MwHWQ%APp$L?$_2nl>pNn3scgd2>Q$8MBsEf;4OrdSY7tyh|;*vwb;cK`4D0Cnn26v^kgD<%%^iG zEEN(JEyq=%pwlh#m(ILEo@W$aOBm|}6ZOnm~r%#*qB$&O&n1-mSNILr0 zl>07_#8Z4&kqX#5XiSCjK07A>6noshU=N|#{w_Nu+?Jl)j7edC^UvjBk*WQrOWfXQaa zNXVJh!L9)Lf~f70WAFST+8!vo8IH!YzsFV|-|#*76Z;=;7m+rDk1ke-S^SNlncqY7 zi-k-3$?;7dQuCn`E8kpwtnC6`Sy>bGz3JrV^_nNY$O)!1XD?1t3x`n;L0~>5cvMR zGhwdd&f4mWaY)l4t69emA3od-e~md@;%QD@VUlrhs6(Y7^E)|=pPy$xC+0fzQo*!+ zVYd#1DGZpjP+!4_(8JkgGrN#aHSS%z7ED7zu5h&LQI-7G^($)^LUbuKnhxB0maGT) z5EjXr3Q5VUaU{S&|0VKC*)Y!YJ@`e6;zBYq8FJY^gST(qd;om*2%WH|R%D=%cCa{W zoyG=eg`bhFXuG89i)z%Q*_#ueO~RD;MBP|i??TVJ#s+GBOdjC`MMVKb2{e*G`;f>4 z_Gliu6X6r_cl~~!il$Q&6jVN}y4-S?{Vf@N{{K@t>x!Bl^ODQZ6jsxmz}hLHoUkv% zXCB`n(i$@ez4g|**QVRt=Ji$74slC})eZ^)5E z0m1?%3-byZRfn`smH{6hA5JUI;82W~`+(dCsAABP@!3k7dXh-RKX*?2xHNv4DNaT2 z3;vN{%oVmKMm7^OE9|rU)_A1(qFIYRjZa~aydiO5;uCbwXJe*2>3t`0e=Q4oy|7p!++#=b4!I@&|E+oD!LAI3a%0{m2F{uC z_AQz~HLGaC9r38vR<3?NJ`W+oT(`CHPy{m>hFKUgzXs9#ARs8X3r7E>EWlg-z9oCD zQdK;&1)aZJp-V!fkte2ault{JJ9zdN;n`ywf>Gd;F7xU5aO{;+{L>ls`PaIedeU`^ zWp!aGiE1G*zY9bu(peJ523Tt$s)M;SV>=fYx0~L^vqblYCaMxJn@+ZmTa4vqxP*nhFg~rcY zK|vjbAiW|1h6xfF;$LG|e_e`kta=|gS)Z~}XK#0MN>x-X-be3*bo!_Bs}QYNK%vB< z!RjU1cm6i}aWNO*ltV96$3_td+GG;~a*il;duoHj(14##vB7|8U}8lVFRwdYoq0+d z>`<66c$g3gh0pMupnJdzANK(u=T7t*M*rKW2AB0;n+=oh<>`r0DWAG_e#Ta#zpGvh`U47!cYfr0 zaj0XwbCJRQZar5_sU1{VAT0tEkd1wNh_|B@xq)E@!des{*Cz3X6A$1Lba1S5cH z!LdQj1|DdHnq7B__OBTra5p-|auap5jvjrOg)%*1)4JkGt?0PG`Rl!k&KnX_w!`B< z2qJmhk@EZM*|TSRVCcGHyDn!N-Mpn0tKYN~=)UEG{1A|0@jehcL`gT?Q&c7<9yidw z?j19p@IA*eB!c_XD2FNysV}fLiG%3`kgpqm^b)F_WBI`J;_f3Z>}}1ZT80yoc=XMj zj!occsYByg(eA{xFv@|Kxlz*OQ_`e~;!EbiQA-# zGjnIhCVo7`_Gx2a@qOy-qs!D>YBHRt`}AxtcwLf&*<>?yK0)Eae<1F-hTb^>&gPTU zM*_W|Tq+3Xoj@aw3>!46>*z$no70TC3)bHxK1d>Z1RBw)YWBqeKhBzt8$ctVauxdk zmzqNzM; z8IDg*E?KxRVzkeD^l|K8x=dIs%_~F+e3**D@ek#kE5MIFF{6_~3dwvSEx&kZp`>2Z zZh~n$je0rAkuEJ=#df5lzu@wfD~S<{{No|{QJ+wa4N_s!@vmEFEaoVWf--<_;{#(2 zSOrLkEWKF2JK_4jiDs9e> zkDr-}Y*pqZrp#;cnt;Oc&Rj-cL|gyMSI>F)9^`CDAjw+SmoZH2@=(&mpHdoxU(p_VzdAh|Z%-55A>Th!nY9z6?PG88o@t5V260L3f< zxhBnRI8Cw&b_nv#fQ4zf%#jif>7PV9Wb2w%ka5jb$#F#Ci6AoqK4BW%PZOx4)X@af z64SM-)*tj4cA7=?6qSz_66l!K1*-ZKZ|r%j9bfGc+Gx-QGjHKSmim}U!2c>*YCwFy z^FgiFl-h7l1JamyD3%2FzDCTg(gd^>@P>VczB(#Vss#(sp6-Q^AYwLw$p=>+8)Dlv z{@v8#)nsdH3!ispTzvegIDrO-bVLylGQ!vodv2h(S`5gDKdaWP87=9kMM^CJZH8=n zgpe0OE9eTcLkp(S%0}#0RrP|EXDtWXf!2YtnfwIDE4A~_s_C91_F_(z0VlM3Q|g( zjj3waLGQ6S0CUs}Y)r3%PbQALA5`2-U_c^5jEZc5S<|Myr{_Se!|)@D*8!_%FKK(>wg* z|9DpD_@h-j4zS}+Jeip+v^Yy2K-WV5qHT>0Sb&~VjXgRar?ULXhJks+{_#OW-`7x2 zUBv9pBox=gI$zJL+fWa$vcI_bZ4Ad87tIyw;1j3NjcDcni`=fLV%SIj>xtBEjLK<7 zr5uLwsruAnz$?P{=JI?YJ9M0NIKq%do>P?luZq@lVVou?TW3S0c7Q=Zb%giZFTT=& z*RQTvtc3~m8pK3ymcx5N%0n3Vu)=lh$Qoi#4B);p>=V^g;4N@DWy#itC=UBf$CV>p zg9lpIy)k?4f*i@^t5?V_+dZ-BACR-OF>snM{x9mk`k#32vfJh#ArXm>WO8yuCP70l zaiw9HXJ>%Ek%Wgg(Yl9!q3Zcac^``vf!iI(aHaQwZEwpeL=8jC_Iw_maw%NTX$|r8 zUf~Rm(#0><(}N%yo?3Ex{6LJz{(L!?^l%)gk2&PgEnWz>wid+hnkN`+&=QyZj#F1* z1&%?FJT7;HVZPUd1aqVn5sS%8YapCr{O-&;{GS#m^Qgg#nb>eNDsH`TT=s}m@7eq6 z7?ODz={%BrxP0iiL>E1Vi{+2I(3n;jsZ$EgM^y(6CgMk4aluT4&gGm3w&|a#&6uv3 z&UY$bWM2;l+*y!&8Fi*Xu397r6N5xu%v9rP!}!8yqH+b(AQ1`>(81L}si(EWeKZD| z>;pA)7Tz3cPlkP1tR5ad>J%%D(UZjDr7ReuGjt4Iklpl$OT+w_lx0tB1R2!SWzKHFXr;QB7dUxA$HV0-?6O*&(9)TJuF?^ik*((Y3XiFC=!&BsrKnW2ReMq zH`GwI`)@dM9;(h!e29W1Z03@~=JcM$3+>vgb!vy(Pm0?$?yYRgN>&R3Dd~=}L2#5s zUeNu{!)VVnNr*&+?Tjg#-7w<90R6VX4^NhC)a*(PjmK~5ejX^Ehj;XPgIWOdY8#fG zlHqejGZ6^Y#POE+ zdf$W2%BZlyh7h-$M~8du;dsOoV}EG&UN%I<=r zxdAAh5r9+Cr$vp!#-yPnari$)nxOswknto2LQ5cs((QaN5 zgX=@c?2HtpUr|P%AUGG{yn^^Z6yh3u^X5Io{)!-{s`$j?^3c>!N*dd9-$d-s92scY zWi~e)yg4c=+h z2oQ)>+zh(0JM}_30zECFFUUXi_A26DnD?Ncd%A)~^-(Cqe<&#d-qwb#Q@(f*d+Hd7 zd(zP(Pu99>Q|6&$0VULVfGvLn8>fI10vYu^;oc$CLzrM3j6DEiAjKLqBg`CH6`jaG z#rPdapR1Ic=g&QAvx33mbx!&=2%tnSm&zE?vA@9`wn<}esL(2Lio@>ByOh1&&C_I? z`EP>XW+5wtMZ;DMGCSGgW;1pA`HQR21rk&dqkvS0m3vhtz1f$bOW1p8HH-%9EwuJ^ z@7F;b#HMxb{hsJ@5lODHgc2sQ1)&3jqe3NX9Jirq?pC2m*XAEEcu~Tjvrgkt&QR^j zqB~346SjOo@gkhQW9pX?gzX)`8Ir~r2ay=YRXhP4LjO=;L4JOG1b1;F_+|fBbST9+ z%y3Ixc!x`aG?0^6JstxUI&e==qaeL_XQA##kpqnP)(3;624!YUWvE^PPXsj-nSQqu zDt2(GZMKnFX~asD%q`3pHsDDP+fSJl2APuWbHG zWgt7}rcht>A=DM7?U*81+@Ai9lkufvLYqYB_Q_P+uwP!+J~xktT84l2bYCUu6Z!3~ ze>0d9@E6UdU+koM6O&B|g@&~PYf9l(dGT{(c_Rja`rifCff~m+8%pDQq^uB^^--hKZbp zQ|`cu)Q}xic!?c^SjVVm0vqg3Jec_L5h9(9&;lSd*+XK{XopS<_7k7$jsIcRZIqa* zb!?8BA>v(VqOp7T6_`w*+BJxEbU@DP0QnLJQl13^)ByCYICg!88NEbiLjcLIGqMz* z2i_M3>_p@&npPz0As>{%N6c;(L8Iadm^Db~Nu+WFpQOR|p7`sc*n=@=DCVnfGmH9r za{rSu=GfA?*p#zHSIDlJRkjKG^F?0()*Lq!)IaY~E1XQ`!PQX8Nyfnb6F?kG2OVUj z9u+64co(yyie6m6<|J!HenI~oiVwjyboNmLFwtBI(q-T+en^joClEP=0ibjS701zF%TdZvg${gLla<me1RkfMv1fb+@h4XY_tVin*rhFeYL zOR|TnvYQ}O7NgY;&XB*;hew`P(l}#sVvvglJZZW6Xs47 z541);oT9E>OQr1Cag&b^!ZyQuxwv>)d=c)lD95ICEf*LL$g+9HT|JfI&y%%h?8tqo zBDP8U5PdtmdR8Y_hvpj2rvS=#QT2n$=a4dq($RZ)6BxzU#=Ca!#3M&hRsQT*xvb1S zUEN10SqFvFh(7+8s-?5-af)Wsz;QrrDP<1k_6YZ&bs}K;h~F~N!`T?V%^Sa&)YZvyX#IE}f~ns|q;qO*Ww*<0wf&7Y z+(uNzH8$wDgHU2(uq^l-If-eI3A&wvQsI>0#;CgT7CDr^|JyI4q0yBS)PXJYKQqv7 z$By+lb7K?{8g-Qbp}X(2V9*Zazt!&U<@J;O(SElaP%_)RvxpmV8^{1`p3XhqdJa!^ zRvO20(s5PVmy&G~zFqI%P&-e%gT5wSR4*ioN0s8dcII#?SH-J{uUU0j@eM&BDlU;9 zFqx(z0LiBb?$I)EVAhpIG1P?xI-Yu<(3dA!1Rp+0JD{8pjdeb-kyiUEkZPpnqwhTo z6!Kd5=b46B9hhDZAe_MtOrw@!1f~a9f}m}N9*A+YkvK2MP~Vb6W2pv+E(`Z3(HRHQ1mpEJot*k*PhpnW*km;}M}cp!jyhiv z8|9vBvmuoiQg)4Ld3L1x=m?(e#@<>ERed|VL>dZn>sHyJphZBa5E6x!E&F;e@3;BV zkk{@ZLi$jy_IR z9bny})%pxxU&NLHeR+9^l!}%fKRN<>KOBY7P&$c~21-G-Zm=?9)?sKx9V@z^NSvL8{gVL^8nJ`JqG5rej{uX{m-l2cr6czy2s$;Hl+YhF4emet-dHEG zYD&VN7==O58*%EXF}r|<6vDxG(5ZetFmnCrni8wzD8NG^3*oQ6Pb~ujHN9Ch0fG8$ zVQEh7=tX^40;MGpldXn-NM^t9aO;(J#RPr~N%4c=Q&+;~h5YBh%Ymdrc3(N$6+5mJ zLrgPs&02~{Z~OQ6C^T&2y9H2p(B-G%<|S7LrLy!igW(DfP>Q#MgrG2O2gTT@PVCc_enoDK{zEqBIU3$XUe_@R;AyzZ5kT)>vqYu1=U$3gq` z({Z~=&l=bW05EQ#?b?JOD=P1KNT5by3`fV>jP@=`?2P@{Zo|T?e{O%nH=~xM-Ib#w zwWE3<7=SGNK>(qf#oPy<( zrNeg*b_o(@PW)@n%L^>B1+p0j;HlXOi)!`vk3(%nb542mzNOh3$G}cI9v#-3Vy?e@ z#$yf)nLtEurfUF<3ChE_t848k@;MHNfJL2=@EX)3a+9;^zyTsZ6>g&WA2^0aHtoi2 z9l0*RZM27dKui!BmE}?>_+lqTjdg&`6;jq1u5&1dK?B#|&gHJA`k^NV${V|dHDE?q zXy`F$bWn^si%1!!0TKcLdH%4H3Y$%08YWrt^wE%qyDK>Oa)<$geW+s-tkSqkc@Hgd z3Wj9RtbyBAZDW`gPKioCauYL(qtWqBAWDla;R`~~25~c(L*U39pZf(y6n+Wx)$=g7 zs+@qog6e*0-1t*YJ(*Av3U8J)m13K^l!shd#dlGmY>t*-#`D6__2pN`?s zjuz??`*R*9jeynUw?m3z50RlW4gbu6($)@m=Rk|Y6q*@leeh%&Kbe6)C29O;#0W31 z?kq0~hhIkKq1zDNz%<*2tL(R#u`VMR=SHs-lR(#VZNre&eAsC5WodGpiZeu!@&5T) z!+hp(w;K=J*e`two+i>PUfEb z|2)rmxnHlDxmBn0`!3h@*{(m5;%fQ)`sVgDmC>yC)VP2@$ze-wc#b`+Zy{5d0uxe@ z>(~pX4I(CvGu&r);dvHN&B_3e% zKy5Bu@S~6}m34dZZHtn#UFY3*#ZTL%W5>J4M?~L1VvvAvoba%@_ix;|A(RJ)%CmB4 zIet4eqnP;m%t4^y#bB4%PZezUlq=XjdjD~Pyyq$0m}deTPw^5-RlwMz-*H!7MUeSNRuGT9!w ze?oi91BUDB^>yU88@xpl#nRxUH9crF?h0k6i3~YQ!Ix4>mRGqpP;} ze7S5{cTRcW)_7F73@wgB3`a`PXx~X2L?DNU*8+K8r)*K(xfacDVj_)BoF7%CR5=)e z=7iKzBlqeI|A2X9#`$NIU`HO)`9`kzJrM8td^ka2o0Uu%BbVnI7saCc;#nXT zQ>-?I1f2Rx1Ge1Mjq?$^f}e1K>4-Gr_$?#eg2#P4vES+(B*KLW78dD}INPaMk~U@M zYE{_j&y64wLYamsSft21!3N7Wb&joYQ5d{|za7uhr%MkiA}w+hH-Ez#tP~H~_Io56 zbvxtsj!(4RqWCX?4qktOp6rWKUCx=I^XH$WH|vHpw2%1x#XT$nBVFhIn`Q=*Few2f z?AS--?hPBZf%Pmp8Quws@=bYq^4#MuGr~bCT{$!jI2Fa`4eJVzmFkMHH3K$N9|LcC ziw}lxaeVCC=TF<)nI;1<%VV~C#)U*!u#=#|iT9jc98XR`(rWc^8XRIbb8H2u=7I+cJ-C$0Q|XpSJt!A| zJNAd4Q+(l*CfVMfM;;Uanh;7i!YPv^YM8^yybR>azNFyGH1px>5lF}clMrq)hkhI( z)TXz@U$Fmwkp!L(^S4HWBzl3lb1B_2q~bg9*$~+w&Yi1jj)Vs(DN&mH+_sZg+9+R7 z`upGZ#ArVM(V+n(UrDloGZdAG-92~jJ_d@EoOpQ?0yj#fPM|Vr+4*`$FF8&q*Su$b zDm}e_LZ)w*$h2K5pA!dObQs^V?I~-+@*ITiM`Iih4E#muSSab5SpF3pTGUzZOGVqr zxyyG<2pF~K3Gr`Z&`Up~oiLfq9GYA#N>k(SevnwJ08iEo*n<7?47Q56MxX% zA(P}t5E}q{n;OI*M^X7fGHMKY=!7J;Y4dG40K9zhFH=p8Q8KvOx3@(IphS@U>h3_O zt5C6r2P_*Qu^+Kl4u@_*2xMnR{`M15a61fo^m%1wj9$vA6)uOWNlZNOQ10%Ov{n$p z^}09daw%y=_WBAw2j@4z)t38yG+7d&Vlvct% zD**-F_}1FO3NWcSkX7n*9*uLSZrw~p*VC_(nKu{ANy;9;lYM#v>En!oHK$K>%~AC<#TN&cfnT^(=0x?3?Yc z=`@$iq>g>W3n6MF%2%a*gu0$`{N2)ndRqLFYGlX{3-xuw+^*j4Xbe%R>}9wa%5#8b zQ)brZ9^8u9gwx%-bVZ4PHyQE9TULz_gS8_dCey*EW2<;SU1 zS>FF8O7M!54DMZ%?dSyqSB>p3GH_m(eoOHMjseP3awMlu{~QBklPkv7?%f|)6Ot25NE&w9IttLc z^KbrNp}9IWAH1f2^ZFIH$Kx})fCe`EITejz;k~iHkd!H_Vlq>XP^i6c8BpFf;v(87 zJ_Wr?9)3SHBmF97?4X^HtBrL3qnCk>rvU@Z8|NTz*T@&*BtRHSF-ge+!X$-Q@8aoZ zDuWFYZ8G`X&3PUP7cS@y`u4C-Qf$pX4wWabtis-GFHEzTm;C=1sbK6fi#@e(m?2mf zpk9fU1VhO+%fDa*muhk#Ch~ zh@25yJwUe?Pab*k{ys_+_1~LqOG4SplyDKR9}Pkk8g-<{c-Otfud87eY|#8ufLpqD z%SpdtGB=_vfB0=|2o_lcP| zd-g|SNacJI;PyDNS%=Y~kTfa<5|v3Ny(!x6UM;b`1OXj?q>UZ{L`f299Qw9wK#kzD5F^RB<0CSX$bnU|1R+ z)BKA(Bv~prk_`NC;&+>Vg=*Hq|0>Y5(}#oeB-PSx)4uK!aJOUo_Rzo@am)FB>2kMG zP{k(kJ4*8S=^G`6fF=arQAC*|&F^3gvRW>6dF>KmNf}I29g#npY1#Lb6jEc_9A-{* zGqmPRb8m7T?w%%Ia!zBD_@v~!9%)mp>poC6&APdrc4+oOU~J+QJ-xhwsMH62opPM( zOry?QooYCcniyqgVrG5=&W>we+PwVKHRMt(@jD;B^lMYnoEBbQ^=_v8hot4;u;?80 zJ*n7;!3=)|4brzXJ0TL)V8Dl9h6ZyK#N@g!MzAyZN69krzH{}2v%axc2#@T)hKKYu zKVM*ia^crbnX!PhU>s|k7>LyKg$q%H3@*(z=Wrb@_Ho3dUF~Z6bf`gN0LUbG5_zQN zaOs|)>a$4C3_&s)$nuZ{qm&p^dE*9uiZ+8q_u4ImA1_}bNpZIrSYVFSgDXeqP-h83 z6g{MfmX8A7JPo4_;Lfz5{vVDCi7eQgy4`#9xD;5!%#cfxt`!BAKjl-B?kbe>?C_7G zPDCA0P8syKn-ym!3@Y?Afeiv?kpZAF9z{Lwllm=ZtIQ zGha@Ak&yh4WJA-DfbH;@d%e6mr)SQKlUrUh%z-oOa&i`wDV1^C={Xx6zL~eI#U^a0l{o*SY<# zrPB|UT1T}qBf_Wd>mW1uWbHHDIa8>Fh16k%zK;Rc_ve3NK$3-ncnRt2e94_Gq1pG~ zpdp~zK~oqyE~f~427|pEY+vqZ*@}gT{6$wIfWA1F+}svrRyF&93_BNXqBx_ZWTAHd zwCo|>xVTA0<1Wq$Q4{@c&ihtk&Vo_K3mUI@pd?v~)`%X*Xd%bf+vJjwd&IK?u_M)s zES>#4HNh^HkLgG6U7jhwygW$Lpv$@s>BO*E!iD24>9I2{yLmH7JY_v3V3A!@rUdk> zW^4p;PM5hQr-VXaH47>62A!8Z&m_=BDc_!at zfAhnFUfMc}FcMP_OK$%Ch5fumN~%fRGSvRblYBxuJ z*ZiYO8+63(z2#-)qUs#=BxLKYgp4LFXNz9Xj*i7A?TGUj2^}HA;cB6je^OJ5bm?Mo zxEPjMK(y!nYiMCPc`P9sf1W&#P~5?sWjqVkLq2CkUCn#K4R* zS9LR8v*xN;6!9^Oexrr{K4V>~^8aq|o6PX@Bk=<_&)wKr7O`9iZ_hJw>DX`M@Pqs(j zEHud1{CQiJPkQk7OQ4TWv?pP@Q{t=I>Vx)6l7^_d2QL$eZm(HY6Q^W|ks2r@BK9cN zFkI-rLIDl!?ZIRO)CO-H3cj|hedwa(DgUW@u~NFYD(aIAGr$)DSx$3vDpCn@JC@b! z8|MzFDpmuL2lIoMhG$32bpzOy!;ad3eWhpgK%M9X3meK=?-Dyp=IH=dyjXCts_pkx zgJ!K;{&C{QMHXkWE4Y6ULygnT%KvDLO#eH;#!CmC^|f0z@Arjgx%5=&_W3-dw6EHJ z!g|B@6;#pd8T}`TsSfwKMFOx#T^`r%`6WU~nKE?azY_z0u7t1#%vyaT|^{DYZK19qa1e-ilYyloB2Un-q{ z%3u4^>}7_Ukin~J6x3&KH<6WEog4I@88a+aWyTDCKGR-4RlYbHh~Yp;FIo=H^yF`` zltCxC5}MS>B_SGBow*sabbs$Z?Bm<4^L9k~V9C+6T+++*SKRj&moB^f zDM{sB7tHMksz?Fx^n)1(ENGdWB9Q;x7AU~%igv(b2%-srd3ge;MaI^Z(2EFVC&`KE z8oMUd^!c4lE+H6_$t4QARoPaUuDxZ>dm_cOzfhWJ^QjgFra;1~v<{XBSTVFHZ!T{} z47zhDFp0wP<__}Zf>?cr>XHJvVdAZ|`ZzylYvBrJ_|MdfJ^RIZvfh5G3bn?-P*?%@ zA}TWJ(7`R`t`8bW(N=N!ARd`9bEte&Mt@wu49_X@%2T8`%Hn9i;`op9)&D9Lo(Om= zR+=1KY5P#|=C(kXiTN5D!pzBRz8yJtLe3LB$Ml~hA zwrv@U4#Y}U3>D1LsLi1@YS*3s0U(7h?c(IJtJWYebj)8g1HVvfoQV3d>aW^nsASHK ze({6MowwENncYD>{F6Q1^7Yac`MJE@{WFcMHYPMyMUStxvLsKLG?#xN zkcH^;fnA;ewJ4fAxphEHMJkQ;Bps1uJO%AL;`{C}$lDgMmn79EG}n6%9IzcH74I4; zLPzY>mdJC*^N6OUf@YmC+|S)XS`AhKb>r0+EKjY3JZGPi#UBRLLSw*lSlYc(SUruN zxJv9cpE+{cDl2CCmG}z{7Gm8L?nW7ZgV%SXSPF>DGzRb?&1vR+%$emgm7>=R(}^O} zB<+=-I_jtxs^wIfChD4n`MNeH9^NKj_sQjkH)lSi&ACqR6 zC)nN?Jqg1}CjKh}G1=K88$6fjy>ZwdJ=+r{h`!}tNF8H`64lbaMW=w|3b##{wztzx zT=blVu(<@TRfu^7eJKbn*A91J>XADHKjLWW2sp(Ww=gm?s&wJ1p`>GIL`SjH*nnbU zC@h{D9&r+FN#OUHQ=HB|f1<52Yc(RY-uo${RpF)m-^OpRBZtmub$jz+nkr7=cTRmV zcvnGXv`}1UjD$|O&|gx#C8P{>UFKgdfO}$#BG%vLzJ#F>zSG1{p>9r*$tzqO`AKFl zJF!=}mqk?0NpY8PMWQ!9hXX)3TO9TU$Se?70r52nqKLC?51%Qr4leKq-b}RmCnqoh5E#-g!^AiDUhE=1{l3YN?H-}3uj7Xpwx`4k5ge>$(svQ)@Lyy&R-2WDTLbw#a9VO`0$<#fLL&!DH;e5pQW8DY0X?@ZOp;mWHZRgt&v3QDP=Qsm2993yCgW5}qu9 z1LWG~LtT7{&YpGtg+?zWA(`+uS?lE!%(zGAlS?7~;PXoFuYmFrKmw}G;`9oTLuin* zSY-HCA6!Zg0-nlhkfq3-K1gqkBce2!tz~kM#9j)-#^P4%Vr_qV-^cN1v%L&daT8W- zuNi+~%#g%KSipF8H{6F&5RFGHZbR-Gj<93HJ`4aWxBdI+M2CKS=yUN|+ zpGZ78Bv3Pegc>abXw1c{6$FvtErGaTlc7?Tu91G-d917<5X#p)fT#ZmElc11W;pX( z{|g=2o{}<7~@ zl=;+o_yD4--m~@OgVhXg@*T1L32cN!^L57sDc_&HvgQNb#0d$7K{D1vcj$BuSHd%P zYh1mY4;*kL8dD;75S1Jf_R;;kLv}Qm{-xUkPCWTkU6zGkV>L+3x8_Xp+K)_A92fG~ zIc9n6+~W{67Tnu^ZCjvaKf#g)UkER1{tm`c6r&@NDVK}pt_*l`BL7j-E!!tn(5=?b zI3>M7NZ1TF0Yl{FQv(6cRhx2+ohsjK+UI9e=heF0si`VA&omK(zj{-lZY@&mhZn8aB+Q?2smv?J=#p{oz(TY@mte$u`|pUWWAA zT!rEbqJs#^JAm>~<}gv7?29t2b&Pj634N(l1C&*Hb0xay?RLF#@l(nLB(jp?I&cSj zQmkSl42)V10G$_9Cc8gltmKrC=TisbPl?%k!CyN~sm^e%Gn=rnfv&8{y5#v&GE)++ zph;$F3?4j1%GiN()Bzuf&AmuOm%)LI3_B8~r$N4YjznG{=3a$L%ZU}8YoFwHgUa*J z8MQb}k{H0mm60gjp{`l4@4bxsFemMQgel%A1~Uu!w9>idr7Eh_AM>&!eSP)6El)9Q z?AfPb>lzcd<{XX?g@nriR>iqnym!*GB418H! zM-`{bGZlliP}xuS@7Z&_levG#)~%yA%r<&DZtB$F@|Kx{de@z|mox3Dk2GOpByL&O z>(nZet(=bV5BH>`cgR_@((l#FLl*wNt1afHoo&MNVk8lho2w=*8H~h)^YX%II2CcXKn&bOMvvt6#P2=-{yr* z5nU1`q^!9}C!7mpA|EH}t#g|;*KTf*Wu23QzRpzi=AuKJ#{@vh_7UZUWHWc@u(8S- zEQ$OW4sLM>V?gGI?GFknX}VZuLbFH+sB0r54TxzkIl{TzbL}mg#{>k5-L%M#;z_*8 zRF5-kdaF{N5%S&A&^0OX(Srx(;6!L!AJ(d(+a{%KGNrM_k|n*GTUT5)lk=WBl!as+ zLKw}>VeSK{ImNcSYu7IQ*w0d&v(q^mcbGO(YL|L3^q-k)s-hD=0Uk1Md|OqRVQ6UR zl*ly{`}?g2vCI)HwvDxon*ps1Ra%%p&_zs*O1i{c3p_0vq1rdxLPghWiqM8?t9PfC z8Rppj9nP;X#-bzo5xy6HShPCoMCZgz&;9!|%M}9cJ82>a{xcBH*WKNHWKpPm%-eTU z2Jiy_y3GRA>{74(OWMAD?q2O(p3{o=_Zx%U%J%^>XYDXQ1Q=a{z;E8XIU?%F5afiq z%%?p+D=8s$gW2J3K@&H6Pj8^owK}$P>yf|e^&MDMeg;gGD*S~8tfL?8uB|@yw$Ngg zVFU6a8x>k;{nDzlcj%I_wJWG}T?A4f>LM?|u+$jh)A9%Gugu-+0s?@J7)K(LM71^$R`**t)lF?|W_^BmD6wUpZ6j7W| zPCc-hS!Y_RD?Ab84WmY;({PX#T{zvk5eW_WZM#tLK!@&6unXDjSKvP*++t{>8@VTg z<@Csj-lVgs)AL6u%G2a9achE*?Yz=DvX*QO#x$L1pPVrM{4ABx`fn!T)uEJn%Sy|C z$8IUYFI!@#N$a#(@16S*ncAAdr772kSl6mmOY#eD{%xVGNtxe_INO$b8w1aZqDqx2 zi7A1bP<_p1U!^z$nNTYr+}*ojZwK%xh;~85VooOfFn8)WL4aM4{suP{gj?glftt~m zgpA^AzJ>DL#F~EYMaP_7UAX``fOA3>SwYKz&u$JgdYe;LoSlihJMug2e>^_xMvW?b z7>JCq-=d^R0k4rUifZWh-ygPI1>aVkpY)u`wI>HL+ z-re_00d70>>GSPKKj)~!)Jpehk_QZ!%m<1)x849Dmg5$qU63D%tJLOjqzr3;*wFE- zs3i_sPK+)*aVn}+gHk{~*{FcP8FX_ny%}vfM3Oc&WliLy#4Z`9qbjNDO!w{?K6llQ zJ-2Ln4b+GbI4<}*n_je0esN9=?LQth?G%3K_b^q?LgP0pV=`JST*(!bg&$F>HZFxs zOE=gw(PepzPoBJRVTe?D5{8<Ej5*-7$XAo(Y}dGjXk5SkR9LWxJX09F_EQm$MM{DY0E8?}U3 zE6R8KvC3uHl^;+DBIIN}*y_uLz(Oy6$a#b?Zv3kh4xNu_95&vKVeGhT)M! z7>+!3c_kaz;e3nj!N?Md04Ta6&k?T3Uz8a8t z!65}-u{rIZ`QpVn?Z*6ByyE=zkSb|^J={@2m4B;k`mNKC|Ed@2-BMvb8FjwlR>6nA z=EyI~E2rfmiz|6)=Z8z_Q8lv~lahYDZe-0&Po^L!j$ck(*F14(Q;_MLxN@EoO8nA@ zq+RpqY{fjpO69Hb-P}i#)WEy@$mbXJc}P@~tuh%tky=6OoFPM0F^6Y16KAqWH1X-Y zIwD?cz@+&s=o0kIs=#WT)ru{iQ`x#3m?hH3n%+xQb?!p>$($i;@0dsKo6}@|!v)*j z>V?!c%(oj)_n_lnQ7e+Zn$IRVQCqsC?OJne+WNeaJOe?fdFxS5z+(!)K60dYBc-m2 z>nMt6@>lYM0n8QGH<2X9CsnDhIZ&m32NH74LmHwCMR|3xZ>L`uyYa*WL}bdSzWisL zC?+PA54`2TDnh6AKF_H%LyB*@qXcknk3Gja@kl)$y#LyghZetn!G6~NaY&qhB;^df z8?^2wAE8kxvi!&3EF$)?lwo?P<~eH?jPLEUBoGA}h=Cx@%r+E&dapt9lZ+3DERYQ zB&%KrhD$OOQ-+Ka+XP;V#@erQ%KW+Y=i?a=_|`fCS%&CxJ@nGAN-u43`&A$3=M^I4N71zz+J>^MtjXWwuCsW*uNC7(9 zHP78$CDw8ndKPz9%H~$C9nqxO?*s_s8q@97z)aju@*ZTAaQaqoC~U5RLp9oSY{1FW zrzcX@auCSn&XA9l;n~HE*_aN%fm=j!OSrcX6z5iNC|hUe2-tP0%E?Joju#mTCB`hz z+P2T_?Cm8@REk657F;v!UK_KF=_};nhlPbXjoPyL#Qo0+;qz)*q|eQ?P}Q*xOg^4? z zp=OBsadwOMf$Fx7h;YQruax^Fm&v48_TI8}O2Ma9g}WXIi9Py2J(vPp(y#m`s1`dQ z=MfZN7MXex=21c;lyk5{CGCd}uHPhMx%j2jcO*D))x;kaz;{aM=cQxIDg_^`hZJJkI|IhJadrxL2@r6mXI<4(DsvePiln8WWo$ap zsFJlnoKaCNin_RMhk)kbwQ_<5Uzcswse!x0EA+-GRb7#^6x|!^+BP`Bj)y(}Idk49 z*FV_@`1#pFmFVbj-1B6{izp+MtXV6k^bT?Ym^;B+37A_^c}DYrv>@P>*9gj~Kz->|TPwWs9_W2r&V3jd*_vnh5apIgsl(nMsm zdcw13MG2C2C#3K2;j-N)L40s@0Q;at?8G^il-HL&P55k(Pzt6yr8{&OKNu1@?f4%Z zLRT67sxo=ftGAK)z+p>&l!sE*I}8wX<5PJ_uhNQq-*oGWP|pgNREzaKx2vcd(Wp_F zcmi2uuUDtFz9wl)T6og1-PYgWrlon1vDxPNE0lwHL9fYq|ldikC z4HGDU#OT*iS2zCVY*pk+_HsmVR=8^@ISEL(#&9Kg?Vte5pwamfdgJO+IGhlkU}so$l+BbFEtBMNo_%-+q)BLU2%GXh(>$ z2=@(5jOz-dW`bP;arw=Tjv)M4C=Jb_zjWEMRv+~#WWjkOec3e&Cf4KNb&}g#tSeUP zMD-D*tNg0Hva9Xlz7Gb}=Zp^y4E%UHjmcN=(8DS1C{Ok!FLKLmGcKDz@uJiPyMqQT zxK`8WO^tJtb4OpY9bP%-p^{&Qs_Q6^J!^+tD7*^B%n7wB?d5BfWK!8Z{QRh*aZ_q- ztac`jhp>b>k5kUfSQ|NDTWb&zP%Apv16%H8o#J}r=>+zftP58sXAk7lfNS#Ye2#EE ziu00=hfg}_sYB=#8MDJ7ap2Z6)d3M@Nil$4e9`?GgO|Cf{$#pWHd?Bkxzwmbtde$s(6=AhF$r5*PlQXGercK+_$6uWy(At-PyVqZ>Y60Gxx%L^TSS9(se;S zpT!M5FWHKy`gr0gOQx@L;4YLXa#3edGqTqbhmJnb?LUXPzE1z^@3*~oDBt1QbtdjF zw-7D145$@6=RiwDo;8L(Ie6VJZ((DbmC#mMTVXV)yCf3MI^@EI36g#RyV82=EgPFF zueKs2fOL^7tx|{<74vNt&aWFa@Zso=eulLtYE&g-MmT|075PJpP!QLfh;B&;(r7ux3^{(yv z`;JzXwAN9%NiQ!x2gdT07D=~(t{1oJo}+DKf!KbyYNeK-KbnlFA|Of$3iKFXAv$6J z{B9&MI{JCCtwAz%+E>)x0s9uU5(vlG*nOmsTdmwm<FO7SMxM9m(KTkUEo5RSjCe zw2GD$(LiV8ppX#BE&W$mrN94&(}~>Gk2(9rPhh-<*cTF}=>GTv0@^H=Ce1w?Rkxf< zf^$6G57~{ktK`WcYtDd&fD=bc-?4bk6 z=-O2o*+kFmH(?bs#?@77HK7H0s787}QbxV53{Tcvsxx+QuFC8i(C4cQ-Jp1NkrV5C zZ>wnJBA)}UD2jyF1H0$8Xx=;+imD@sa?DiLL1X}6Tj`gbZk~6(C7yK$7)+#jbg=2H zvJR%(ad0c18g=wB81zY$dmO)Bno1#4PcBlgAn>cWN{cY+QW}UiLnl5Uj#U=p`f^S6cpRGR*drz${*GHDf6GB zHybsqP%l=b(tPaJ#lqrH*;4BnXu#Vz)B8@K3Q@TBb9QBrm zfR)m+kTxdB4)>r2*754CP7H1G%v{TTZLDY~Q|@ zlv~7ifYq68I%u7HqP25y|EYd<^`@S!*wcDf%yG*r!@IO~cg~VujwHh6lzNLTm4I6) z^Oo-|F6Ui4*j9Pg-|Q~g-m3a427n;ir3>NQ;n|VFRIb)BHfP1qhq!ppmV1$}z(6mM zysC6k;%J!v)cB&_IavMJdyO zVnqCc_mMKFCz8m{9Xm=H(-Ih($4p*^rpl8i-M#yc%T9b$VdRX7L0O!tr=RMP6DZeZ z{vO*V^aElx)9AYWAGvIOMVD;J6Q;-m9w?Z+_GoTN=Y`AnCYKIVIo?!G|2qBtf%yAx zT@sMstt@5A@F%A;-@n{%J|CRUc6rr{bswkR>?wOW$7s=|kMlv8NS#daX*wuEvf9uj z1O;`D!WWb*4az;cP%lkc_s|0a9Sv~Dfdh*#ma=hV!0<24n@^>Si?*iZL(UsSjhOu9 z5TaqYo^>(1cn8`dODoISY*;X8tPC>|(D(W&qe_c8*kEi9;=hlhDuL|g=8M|KnV z&=S^|nox7aYk5FYiGP&QSSO$AA>1E?W08U+L-N-Uc5;ZX(CEs`%XtLkJZS`&lSejO zg*jw!qU!QfYDawAuJ*4b}DsS2|nn!HnbC!yF(|s-zALPgr z!#^7=GncZH1v-i32b*i=D*p!$AI3}g2lei&?^8o20;!L8I#?PaE4X9GC>Cga;s(3lvL3sd`@2LluBfHYFuM(FCNddW^ z=RDY4``6=C7G8~NI^Lxm$qP>@eWd>W^k~@x1+lECeS~eGk^ z$YsMTPs&?5`ofS;r&eihY|vGkHO_v9JU4ljT=H(bs9S%_*|O&^SY>F1JUeDG7gBEw z7ZC7}_%5wo@bt^TF5{C~GEPGh^`=hEw9E!D(=qjGPaQbEw|ujnmNdPNWhbW0_GhCYkVgr%My?oW`M;(J|b+kmRkW*aC^tO!Z zgViXveqDJXhYt>VbBIkkL9FU%a1`!WowU2V#d z#h#abu3guzpBl7hKN<}+n=ru-fh74b(Q0j400|O9oyVKhK1`&Sfc0qTDHh;osHTC@ zMexdkpv{?3uhZjS=Il+dd$G~xY^N^nMYDJ34oU0SVPmxeKk@pQ4bswWEN|(;LI&w| zzK}L|BLJ_@W2fGOCydqE;MKAcn-M=|kPKOD+}N#j+6-Xv^{q;bph0wW!ox-1PNsph zta-RE?4Urq)|c+VXEL|=ArXMk0r21{(zH;yGj=Z%dZvIxUy%X8s;s!P(c9P8w=a|r z4bzfXzBpA|gopsq--AV9G}hqD2IDQNH4+2k5RtdfdLd>r1Y`7hak>kTQ;RNIjjnQ@ z$PER(xhTVyX`vE7M6}IGrEQcQEl@n(-Fi>CRL_P`>LmDSm(W6 zt*Y~kYEY;WRM}BZWcui0y}fOF+;{+`huDtn`JAVSjF);MI~yUIP|X~4r5QX+u_y60 z1=H&q3}cA{=?iT0cFeN8ES-M+Cc?q-2mBXv*oQQH2|hLMf^5hhnmF-;WxYCe#N?q> zqVF1AO2|N*AoGM$=~XQZmt;^ncWQl@X1AL)9#8ueG5J`s-gEN`My0OobUr_xLE1GGUNwvTf+?T0M8yoPE(otB#idrbD18NWm(L$xSOSwQtST1hVj#%qW6Fe z({-i!kt9%=7h@^@e**Zftjb=bxp1OyNXhdL)+}}Luf1<_gXbi}mW0A|LwXk^IEyLu z$ZG#&m2a6jDcinml99>nC+LDi=9g*gFUQP(S%E0MpO^v z=Z~BB*~7U3LZW{tHyI_lGQ3nd!lQ32{y%GtNJFRrF7-|v;rO#H7B2jZsDi9eC^tg< z_;Jf(f89B4a)_nd;HiN%^JG!Gn`giPAw&MC=gq+=4trdl*U$eF#?NrnOxeXqZT2cCH{ z;7IRIoJef+7TosO*yE{Q9l}*+A}z1mGKjwH0oKFHst=$DUi4dC_di(i$`$<AO6susBJs@Bt901Cf^-6g3& z*uXM^#W)LBw|~n?_f7Ut76P8DKQGts3#!JIZaNe-GlRm9$&|=+`>t<$DaCBcLUeDU zfwB(an#6U`?`@9?x}&&ld+81gEWD<9ZZJNmH9kpLv;DCZBBge#w?MTVb!v4DocQ!_vcqy_7Fi9)=k zx}rjp1|#_V>*@5pl7F+S{xkpKZwb`kF%UC&n!fbpxpQAnt83J)JCTY`Jm8D_hb3-v zTE*PfF5UMYXj#dqQ9s{VelLetmCIOp#q7FO@crp1&A+I2UHEEqw)_6qKm05h(OH+V zz0Y==dUMiRODrD{JT*?C9aSTI_g^qVrY0*Lkl+jd{OqKp9uege_`^uPTbznRQpxay zvs(13RjXEN;?4*XID5pSg3)wtGT;&En<2d%oX3FQR;3EUR(&Q6E--o_zQat?)Al81 z<|T+*0rAs|n-f&+=d*eBopJ-!EirUZ&)or$6i)pTt~+rrMv|HxAs~vBYl(EhGSJF8 z94J?;Bhs+nKw&6IeX)10*wMgL%68)Mqj; z6XOE}PqtGe!__z6pZivc4UhZ@M`mLN- zQ)L8xX?DMBu*mUrYaBcg8uX=E%?oF3s{y+U|b|Fyki}-Dtp6T}c|^S%~aPs*prCC=;P6G7{BC(VY3C#RH9X zpM7?cC-f&Ul7xC9ksEn<;{kg+JDKMx{T_oU%=uxnL#LRI=E{r9xq16`5b+IC?WtuT zD5!{J&+F(^iX*53GCw}Cd)KajTH|pLv*B-tP}=Le6kSjs#)5C7;uwd1rCfmlVBfFbzInU&oOk|-L{sivAiV%J z834>pZkYcn&K7}#XzN5$&Z3$gOMxPTXlUXA%qFd`T<5+=jbryuF3%ia!?_z-llsG} zW}Mt$CNPY=e11Let>hQ-&qg$aezrQ#&j%U5t3?(l*< zw}=B@STdp-B@rsY&rg-p7|S<3K!*6EWfRlG^G&0c3E_vZux0f?i>t~xE2n{(HzPVAeqWHCm6lm=jr(_@ezaMc)oNIrlTwC zPCIt2Y-GoLuXZb^am;(YY$YE6 z%cJG~KkOdvfHAxOevj6Q?R6ChzuqmF3Ic#nQ05ie@dhVwx5u+P8+jyq3#rEM4dA)nLHH`0=WO(eU@;O_O*P zJVH^QZCq)gp3rv?n_f+t1OVE+R**|RYil6zM7^c3OP?tOWMmN{KO39gKC92OIbNI( zNICymhBS)gwx&&2t(?}8uP5DmTbOzM35bepSE(IDA)%g57X0{zKIvRKGIa@3hZ;>M zDiRnE1kMg%hUg*PPHp5DO0|;wWssgP!R%jYEe;tWW#il-aWw<)MN&J@bAe zyF=Zwd2{>m4O3s!>DtnoxY`@638A=^gypx!?mC@mE7?ADtb5zDdOQ7t{jejyHLtbo zH&%wUGbk^*7?YVI`W?yuLd4>E+DA;G9+LuIzMLX)I%IybK+0jM4HKdxg!Lh@nf~xJ z)#SaN+YoNB4(Sp3LqC_dt6@Rex&;$WKi5CGG3~Dj?=KhU4^Sl;pNTo1ra!Fn-MM}L z5o-?Ex}?Y0mGUw0((RIcq`TS7&_r6qxY;)=PwpG#TuT9dzYfg)ti1A{#Y zt^*vfN|1Y8c_l*mQ`}fk)5O$JK=V4DWbAM{Tma>L)qRo{c!r>-c&qj&+;3Y`rL~DW z){a^a&(T)|9Xw*Wb?)B1tEhD2a^jyo%aGAS7UReN9ZY9Fo||6(=r>g(S}r99Hf2bx zjHaNJFg2;7yU2~?IR3Bq_DludV7hMYtGgINfr36;7T<9z83~Rg0y_&dq7{nv0d1%kot>16Brurf;X~z0k?Z+*VgID6q^V>E zSxR1C7$M##iL?_aLoZibJ0jE0c>Ba@of|a2SkbY56XT}?hX+j`=JT_cP)*+EaC+2F zDELK}lV3i5Hs2`n{>9d=o?55N zvr&I7I4!?+s0^zBLU0Y`Q*HmmQ-L)SX0u~Yj!x;jUbKNQ5- zA`6Z00f3yIo-WlA_e^$?7jH(c2DX0P&Ff<;*!HfQ5)}W;{yjZlT=ne>n)~0G`qe;b z?^LQf>;GHHv%|cK#gwCy=L+h(RHwt{E}_HE&_}8q1Eo^HZ(f{G?x=g|H!S~7&)PEU zSO%X8S!jHdlTtEuKCkiH6^$L&1#A|fw0s_cY@b>zB zwwxT!Fh||IV5uh}Mf0x~DZQY8OKCyq%u>wy`UBQF$@Q`F0yuDR{RyWv^g zHVNya3~nxZ$xbXPj{4!;#I#!EEfU3!Vmw9oS@NbHvj0EW_c1qwxPU6xZ|+XcnwTa_ zYKz3fsFF>yO+w&vJ46)`3I*M4D+K;$Knhz_iYxk=EP@skrwYNMJoyg=GD3wr&)T^h zmPEk0pWP_)*HI7ZyKoA_E^WeTIA=(2)%NYsd!0M)k?oGA5qgiASEB1Ecd#b}HJ-JI zH*($sUYzV>0KmIyZRRjiZNUy4XjHqP>W!W-M!wLvV88{*wZa>*FbLuUKNW4$!_bi) zU?BbG>`kBT8p`h0Z9B+jy>7owhl*#&(}5`M35VNk?qdHxPCN&`~fTGMCi<3nrtpWH$JiQECGVAcI%LGQvB~}x}@1RBtr4@$qK}Km% z$wV!5TfY2)1e9|nJag!&DyUm-beDlPXbe$lL)sHWDJ!~befl}C!^oxW(FD7OVp6un z3?^G0q$K4>S-uRNOQ=dDFQOdbeXhzOtf5}?+`j28NUEB2TkV?H#lrH(ibeNi%mraA zsJB+%B|Trj)oIf{S=M95i6kon*l4;RRb)EqY8i)Q9Rg`CbBLawX<)P-jP3+Ap%Ay4 z^8r0NB^Mj-q1f?$d~++?X}u7!7Xw7}(Ct|grv!S|VgCb*05x^11giYfu%Qi)+;lkS zM)^L+JTx6$PrY+A12O0f4cx?f6+yZ=XrrF#>+T2^7&eD!*9xjS~Bn>^XM z{9LcI`Ih6Z$5!P}-s{w=F^vNQZ`&)I-e%n?uapR3091}2>G6fQE?oGr1$VW~JraV1 zJ03kFr?f7)A`zmqO3-g@Hnw<_3CjzGqcd0iwNiSPfGtKAHK|>($T0k zzZC8KQBaS{sqm!)LXpPtuvTSVGhBWLiEE&5l7A&?qXY1+@!>i{ICgSNzQqPpew5$s zqs9{gAt42cf$^J0!McThj_TaUhK_-c$c!`hr3ter$gUMaS<=z@K={S%)aH z3>ArfUN$9yTx%&)ocF{d@8nL}oK~r)Da%LnIk~yH`uXd!+jCNI!oyL`Ygu3S4-0Me zzUdvGg^b=QTYW~zkOfZ1mU*sQnB<`;|BY_g^vb%hM`7axP6+V3?R#l#V}X;rPPOvC z-LZF^ocSv4D;y}2XRpSMYcGR1@YP|=Fa!rYy5{Q^D8`ZuPFC?VV-J9Rl!spDC>$L| zFi9p{9r0Sf6rZ#ZJzNzXAOVLAQH4d&6-Y)XwQJPd0Z5;;2CoUjzM27>u5w?^oqs~; zrIbUiN%S0o{fQjslK5Gp$}*!$?4S-OU|chyyg-QmnjyLjJ%=MGp!fw3IMB5S!=)zK#+eaci8qJ2Q=>fzR}f;`RZ(wttk*AJAaD0#Iv_MUb^f48+b>}=0D-K;wQP`m5G_Zc*#C2bkUX|&aY9*EcG(!EEvu;IFT3-_ttf(|I*nTA+XB_jLuhwEk|F!ZeRFx0H!!jr zj!zLOa!mPiDco*aan%yv;}air|0VdD{mO=_I={PH$id!lNTr8Aw|LU-|1RM-86NEx zFBbtE{yn?KHcuaP>ZQ;>yDt-%{Mzk@i*na7dPD9y6O*@v&9T+yG0Qw*^4%`@J$gjs z?!o;wv;3Sikj@F3vv{FZkzD^G-2iU4jN*#y4reMm6n{qXj#{CjeCOH& zKk|2JtUy(I!q#5}{z$(RRT`4=t>CBI6dvF=7M=ufBk0Ua5T1ksPwsw4kq_{tcmM6f zia{RtKaB3)xik!_=yn3~=D&81&5pEzoj{rvJFSnZ&L#|&phzp2*P?MiF#rjI4XvGR zf5t5t%9Qv-?LKq2Nm!&!Titp8ivV+X%-rA~GnhEf>KCYt8-m$i>3kYr9kbj_AJ79+ zY-rS&fgsp5tIrr}%i@bd|3xQIa@))`gwB;9Cxf@4Id?%Kg;UU=z7V_-nf6pDf~>>; zG3IM`agAsxV|*?4kABy8V0D%Gd5&>QD^?nlQc}mzBf8QgECbNz zE}wrpx@J6cIM{q~Ipmuu4hk8VBHDJ@=*%(^Hjr~@NiLrJ}PI+BJM-2(?p2 zEH0 z-1wxn3Gjy8q~zB}IT5NO{VX?*rX@0ZOO~Q9?hrc&)ho$4d33sr$v}AGQRT8qw?Cwa z+>ex`EAA}g+766Fucb2EM^y0* zD{N0B5=7=C7Q6)Ad5VTWG209PQ;>X3>sN0Ui)y%cuLde($yNnnwoW@Z6`n{C*wh(K zhD(L(zIkCezi2HxOwK6OLeWR@loS6=R#lc|J$eeFyd=zFOi!P+!0>T0OS3S+m5JKH zba^@*hAg9HzQ{OqIfK({VJk&yQxH+d#~7yuH%oag2W!+~;`bTNqb`9!Qty{%wNIGv z%d&=uo`;U(g9CyP$I6N#TE_q~f)4JgX^I2lpr(Xh&UNtlh*kNh4`egMV`}&m7r|AYcj~}RXoK;mC8NKssuXl(9LPM(77v2k3{2s*jQ2PB!8S zc2(DQIbyL0@lT$u$@l}5b>2T^0b%o+ytO*gQ$7J+wqFOY*qb5vkJJMHMs@B=T4k#> zIcrm@HlsvS?ASyZ+ng)5WR@n(KSs5wu6bY?=~Vrv&i|?`!qi7NThKK7aknwU#D7c_ zaqOfI$S@DEbSVH2Q|o{|+tDYi=+~)W3ad%U{e|$wu%y^=Q{RL`^)6)7(MNvZ=@yMm$8V4u<1b0#nvWI$HdgYRs$lR=Q?H6u|)0;gp zF}A-ne(Jdg9PvKd+w7YA5AHVfr$|7R)O72Y*1kcSXRp#7AflnE8V8O?qW(?){o`%q z?;1t)*94P%v=lu(C+_o)O&nmPo|225VbXdhbIs9sV2ko=E8i=u6-5nTg$LY}^Q$t} z#QZ0I#){zrKW6whM+1HIkSnv2PO^!C^UyDeiU7e5p@4}W9d!mj7Ub5TVbk01PK-ut({@_WRr>c`aP$SG%YR*xs2UP z?znl9Gt#Ru8Fd&Br#Q>Q71^;N*Sk& zKZkvBf@Beydi%>!=~>e@EV8tZBrlWs_Vng3+wO9xk^K;O<;qLpTy(|Qfk1iQV%6g2 zz?tP?lY;K;-P?gDmy&pvKx_&k8UjsEfE-ux;=A8wzOAJU#6=RVB)p(<6q7OS{_RJZ zd7>~8G@oHm6q#8?UR=B7Xg)VhQV*5Ma?J|Jc5UlE!hXW5o_X`{>0g&$ll{KtzaU)& z2+SVy?t%Tv`b6)aYLcF|^Va=EbB@>3_+&*Mg08JJLnbkTf0~ZYOJ%8E5MyqWCPfKC z&_JJ@Q7JiW2JRe8GC*KOlg~v8OaOP+v48(CLH3d<>SY`+n<%=3$woLmeW1p2Z`6)K&pP6YbPfH95p`;yD!{0) zpW8jl68cz#K#E?lv(HQ-1NNpI$``sR_v2#+4Mg@RY zY5Zu7iX(U2vKhy5r(tWQ_shx28M%ulTX#9PqD<1_NtR@Q0f~}{7)yE7B>u_~Ec}Q_ zCc`$})dEQ(aiHhKJ za~evm$Mm_%R&9lSM>GM1*v-^3@FHzD@e7#tIB;YEh%o3 z2?2w_g0m`ANFg?Xfo{carTtpE5iXIyN%3vSv3uA_Mcxko9HQ@nhC;@%Oxg(uzMG}3 z+~VOWxANN}D3phM^Uj?IcA}%7gFT3}MKtoIiG9|pzuUZ~>9m{PQ zXgk6K65GS;q2A`2VU)wic4Mli&KzK2c4GVK-m71HZfqNWs>%H0?S>>zt$gsOE2F?l z?3E9+= z9(yzR0_zAGS%DZ8$)kKPpCxi#Q|Sak%2wK+?r-Ih|G-IsIGnCVP&)W}t9HzMgnKZA_J&_DZ6Mmwidz7G&(hvp=UX;|I8##6Z-mY=u#~&A}&^46A;k3X0 z{1ep4D@Zs+9Q~-l!jHT{-K|TuKvqH}qe1+a{}H*Ek|=552zJZSJXTZ3)~!<&LHEv= zpPya#@S~AS2NkoVz^!pIZjzj^r27hb%QjiwW2&K`6P)D~_LN0I)Na}vyL9L~W!Z^g zq4_^{H#DkL-!{~CX>s+?o$n&F<=IG~BI+9jvgY4(=i7JR|8dQdnB=jQRR+IIKVGr# ziJ^Yd=>=L+vWovfQsP|D!OG+k)moBV!QKe8g6RN=o zI*Pc_qeca$%_fS5x?TdE@yy|i{&X>o@cj97a3W(nHPL>M?Da4=cC%Av$R(ujZ-PbQ zZdh=w-t+#|0_UMzNi5Ee%C_`o8ZIKw8FXt)q~1eJ`^ml@X_VX!bntO!u9kj(C4Et7 z0Kty|Pv_gTRjqmptWJU(D$omKQW}jlI0j>wQ>fG&Su-fiW%9dqH+B$~w>Rg>eT4I6$H9XHCpp0z0kOtF5Pe40-rv1Inm zjp?7n0O@CROQN^|O=UC&_DdSba>Wgu8yR`-MT^03yY$!__$20{+GtioyGtJnckCWr zy-v@cva1}kN!uKDPEbTr;HIC}EcQ2BZLz8qqpuzN|GjRZ(~18_*L%lP-T(jN2Mv+c zB0|xkLM6#;7$rL*8KEKBWzUq6ktC^PMA;*iY!aoCRrZ#m?2^5H_owQ5e?Ncx+`M~V zTsqe|&g=DfJ|6e=I32L~#@l25B~?$8(AZ=-HE=LX^`zlmybdzhFPe{V$r0HZ$8jQ# zel>v&pn#qTgqxh7pRZhm#1Zma#u^`FiB?~~4qGF!^We8kDEa2*X7uo^tnq}B2ghP| z1rNfW4_js*RFtF1L^O++hm@LX37m;!DH@Q{!UzAFwWY#RkaLjA=Q0Qgn*fcAhxtD@ z78e%O;K^RVJ3oP#8In6r{579@uZWQt*{An0AH$DSJ~i4)FzKl}ysSrL_#?_H-1F%V zVGAP!U|~VPAcrkc9unyE6Shk9YMQ=*3Kt8z5%=MJA#!J6n{*3J?Z+GYzFR zu%TZQCzy}L-(dytS02m{c{o{Uc?;F+Y8@MY(&GRVh) zCzBhVsHRhPJBWjuAo@OV@euD#MN&UV@Z=AI)as;lU<2^lp+g$cnxDnx5@m&qq0A*8 z)mQMoSM6NaKKa7m+c=Uy{I-HT516AIw>EM_C5KLXTF z9~rS)9U@7#Fq6bQyn}9IaL9iL%V~If5x;d&!NmeWRk=Cq-0jEkkCPZxob8F^jz=2) zhZmT($z4~ub(6tyW#xKkxv>^Ndy$JDiU65HOp{~=+Xa`B^mI)Sc9lU>FD#9fE!8kY z4PAntw_vI#oSb|(^|N*VM&M+E-6q8NArTJxD(c%LiHPjUY2bqNhb(>5-$SgunM~PSoLEQV_i$l;^8ai-jE?wlodYy8|5xW|jj|utXX1u9L1R+fDFZmYL-1=STT#O-r1SNun z!qdzHc27ftP83xw5u?K1Wt=8kKbRKiI3{Xm@k^eUryAkZ@oE%KHNNwjCIw&qdC$4{ zVYJs=T>g!32-C~faTQ$7ttK7(Kz_TVT+JaxF6_Hf`5OZ{XNu8)q{wp&$_#>iAn!b3 zp4*|_#@>)eHExSr`>Jo91s(>W9hAd)=I)>}${aESo3PQP2)ci*s^W#}=;Vd{O^fAMcWA5rkKR0f_y}MaFW{5bI*iD*V<)V_3-S9jR`rSS)XNuV3jdt&7ZyS^|{~U6Q09qmhj$ zg6c^8KKzeK7>uV)fdJNHekDuD?%x|IMz3`VVsjFrGrP`N0+>6Pt~Ue?*+m9fe1{Rk{zyCu|x#70>HaWG4>#x z$~6dbplpzIW2bgucLqj)i{N$@w~{8s!)G(53>3gx=sYZ2qOf3!4AHQ1jd4K93!MNt z7Xhw8Zs*c+8CDIlOr6Bo*yR;Ot_E4zdI}+Qgm;~)BLfD2dNM~u62=hoyWziJTvQrUfHh=yHe#;I^n&yk2O@m(#p7dI(y_V~Yri_Q? zE>AO47PAxn8^d{MQlHrf7LkMx`Cm!DW$kjx0R&;(erb+LrXpC0sJ$a(n2 z`TtI!2cJJbkgO;_I^@2J7v{7doUAD|#}wybK(i#?X;?}-JT0&iWsyc;XE!x^Hr$KX zKsL`6N^UmBAVEaG2))He^bhP;N6Baa0nld_xBVC>7lJ<$w=wVt!XXq`BNzbuABtFN zUtb^EQCJ26HYVUb-|3B}mObE~Y49ZwphflXpR_K0%rWyA_4UB`7V3bw+y|tVs7T5Y z<4%Ij4a5<%COy+sMNq{J(B>08kn12}#E}X^t|8a2Jw2vmaUudni0X-K>Na~zPoDWH zP<@r$4wTcqVNKYJjKKp&I*BO zszR1cvIKw1ukA(6v9A}TB$Q0HDTT}Wt$3FB#+`e2u}CJ}%fxfOdUiX8e}u;*V{&!Q z)X-G=!Li;%#?ySAWQ)|{K_2S=gk};p>$+R=4F3^NCy_T3L;q&qB4iPNTq8Q}imxz% zEaLtHVG*JV?VYnvjb?Taf!jvXzm1gGCUJ z2;@9R7+f5`SFjiYqQjO*lzxstcv%^JVk1A}iPUPs(M5uO-G=qh<@`n%5m5}gVK7eu zkTl(lew`?d1i~xM1FQJ?KX1qn6?bTFw}XdZDEliiHxMU1FxE=r!s(m=(OnNQG4?X_ zF7nFoOxD2#`4>8!1L{En1qj=r#f=6V74D2WKV{N%^ecu%l!XuJ?MUGx`yWmw*F8Ra zup8{j0hl^k8yKN+7j*kckLGnZ(K8Z9R zZ8aB{0CAIn?yl&7(W|fv!wHd*E>46*eU8x_dP`u>(_|Y2mTbaLMz%rN7IIr`1I7ga z40Q&ig;K2k!qZLG&J1lySNhL;o!-ANQ{))R)#kR$b@WW=8djqdE(v-+H`%@MX|Cfh z703%6&WVeD+8kRKcKi-8pV3vPlrC-gt?P62&63?=tH@1Hp;Y`|qru1LBf7yB8reju zP@!y;pm#Y>Zjr?N+;;I$!@t1ACr+ZcKjHw;)g=b~4FOqkgFIaHMMQ?(CY}VaJuTxC z-)oK_2Jobbi3!OFojHMbo?zEdRu*<202V?zAd+oH)TRgZqFl%CW4ZkI|5t}lp#uHh z{HXN8=l>^f=wLU&CdUjG>MWmE0jKBRx37-$Gyla*5t9KJiHn$Mm;gw6p{T`yexD-2 z>evpS0wK@8i=q*!!9d-*w9t|HL%UGy9YRqIK`tlBy^D!?o!$<@iIB`992|XAOu%Se zi5n(;&`OkK-rv7R8(0BAJrjF&a@ywmUqEq3WU_eAh?ZISB>F3yIfdOQXy||@TqA(x z|4}26U^vtOxXK)ez}#e##QWml23#>&*Nr7?2u8w=QAnZ2fRvJAMi?k8z{ZRS1`3aZ zcR&eP7T}>jOCFR?9}AREd!~bMnLatM)8y({$ZPE>URC%~bp^;{>FsPqZ;x;n(w9VEpklBpOkg;5_X6U(m4kcX~*L4rI1 zMgS)1FlYU?5?l_Z+&ZMjimall?j$Ea`e8h9u6YrfddTpEfmUNGtXrh5(xKyAlrRw- zyT?jY7?7uwJ=NHEWLc01zDyEahw-0?Qgw-DnDp(mlGo1b8sv%85A}=O8W_xNT2?Y{XRWxOly2-{g{9T(92JGV*r_p<`qe;aks@ z2}*?6QXyABgZ$r$2;f5}@bcC$GppjXNkf>R?zv2O)A`>F8xJO4gvn1`mSAT2H*?ao-?=Sv|h@RCgIP0R_oawA`|NfTSkKL!^siqAU zE>$1NNkjBZ*6OKWHet;6s%pBa;jHwbVW(m~+%`R?ZsicI^CD1H-SnX+FE#gMbEHnX zP5QdzPMt(QOU;Dbd^ztjRaJ**p3~mjGcKjzM;){cJ{-|HrJnlmxGaC}skCRF3Uw!Q z&!Jgg#gd>c|G4>asav{xR5O2;jR5P9`fog;tS8^=4XRnmkOxUJ*LfJDvu~bZnIL!J zqX~nV)`od*j>r>=U*;Cv^<9#685EqQA0Gx$?Mp-c`WYfB7jx*oqvi%K`1zV-cMDzh zK1=RTEU-;)!vRi+9D)}j9b*q+kj9c10VXCU*lRp|-lO$~U(4{lkAxpk zsKdU0VM8HekHSuEj6f&T6(@?DScA{iiu<^ZXenH} zlJi&&6?W}O=Zc%PX7JvALqYXQ!|_i(Jl+Oui(dRE%E8sojD3!a+nqgjJ0|wp*ypJ6smjqEZQl6-N^kbFd2ecX zuY1lVIMIK3+KyiJM*-q4LE0(nY}Yr+sZ<=(EGt|wab>R0Pse<|TkRZI(UWjix^LF* z6EdE62YKr36|@-cMe;a%3`Q0)rZ=b#h4P$>s%*&o?D#<{A{{@far)cOh|xZ0ZxshRATIss(Y!v{mQK3X(==(%y$g?ob(w zSa4mH`?C7gkG9&GAIi^k!h0Wk7;^~zc$M`0gF;n7_r%3BzIvQ9TfR3f95@)f>&UP} zs^-Fr(*Uz_hgYVG%k{@6M-~h>9J}3ci(k_F@L*8SLXnDeLCT8N^^Hr{efbWdq>txta9-?w~5<&nTWd`?c-of^>3SOonmG57wgZ z$%Z9E?uOD~$ud4n9kpF02gF<^<8E$vuXgBAX!=6b{6&xXh6VO39Sut<66^%GBO?wG zo0y`rC$dBcPU;i(=dfXIRjY-4#IXVQsd9s}&ztVt9dC1&UTmc&(sA@ULCHYO#6(q( zZD`~DMbJ^g6f=JusC+%hxwFsPW-ilG5>(&@BQzs!DDv}&KsbS}z^xadHDm!fp5N0& z_dS<$idu&XZqz^(AZ@=f#w2BDH)^#26a=O8v3FN=ljzO@-9h~0?&xX9e7Ig^k9gu4R`>DZ*rqV#vo5W-U zjOh4n;7mlP0eS+*8DiCM9yp2+8H82-b+ev{DAX@-BB}EM&ZdX)Jc(+Yw96hHzCEfD zG1P0#f(wwLO2~u6I%M5!{;no7(31lE?UK z=SP;Je`P>9%1;>FP_pAEz{qhqbp1BX0*nP5m13zf0+oaR_f^ zp`*>7DGfuuYP8!#Ude}w)cpP&1Y@4VJop#DuOp2u@;nyPoVU)*6KNro2)NeDwGb_Z zsvnHx9T+8V{2;&D=!jkIfWRVIJ#ekQ+saE}7e;495-_mT zwHkQrmCjdGjz|Q&Otex^RsYR2QAZD|r~shg0SBD8fGbR3ZCMOiA?*`V4<6iGX(B~P z0z_&P0V)Ab8*ukFN-W7<4styBjZv_nOQumSz;K|l*%y4zSDk-O_WhwhE7ZZMbmInY zG2G)|%r&8&BU+{gd-!H>GngRP@>$ENxW%;@-7m2-gBU5(>oJv*lzhl0t6NbEE<*4N z?dh*%LIC3sB&iqM*s4oWArfA^s7+r-QY_eomk~=Eq{Eu0`-=>p0%IYe#aNz2W|;?j zaem`w#5$uSgty;*Oi<5r=cej^K!flz&~;h><&B8@k?g`pn#lj9*$x#}a!@Gek#$M} zl4FRK(RFd-3IoSmw&|@T6buL~`PCB>*9GM<05hfUeq8y0{5v8oR~M1=HryWniQH+q zs=BEUKjbas#kx7J{9$t}RUoXEq@L;cG}a#AKZ={JPOS9paM7}@60eeeMDW;dj91UI z)c8IW)l6xe5tEr}*%IYThJ97D><=)Mqcbe5x#pIG@ zjj}yUXwI>Df1pdc<0_`oJ8`~TZ?ta0WT?)lVB*8pxxr7z{LI!*M>G`JZB^M9Iy5u; zVc6h`%f05fH++>WPu|_#M7NSb)jagHSHp)ZKg&6C-Nw@wqlBw3GFQtCQlOT4e99URUHjO(>7GpOu06kbtqz2j&z*8JO6vS_rXH^Pdfh< zZo&p;H0llo8p8$C_x&vH%$NqJ9n+SfAi~PC6+UU@NZ8*B1Xv2v~$lu6ZN;AlTxZu z6ptK;F(YByC>X62PfD5XAO3|sV8j|v^uKX4^CW@}Q7D5QMIGd}2Ft#v|3miYYJ&Tq zcU=n&Dxnk!o)66fi3@3n!l+G1Xh1*p&<(8F(E58gi^e3MfYX3-2{?^2a&#B%tP)wH z1#Mv+baEt}MRF0yZQYF1iU>l%9XLMn zkmCWoiw96wt~4myb^hGBzw4#gz`gI0T>s zOn^Rfw25#=2V-&jKEbItRR^{HJ}mkZ`A%~N(h@p|oR5+datsY5v5E(rTgDY*x$y9q zIuJ9(qn_TC{}0X1%Egs-@pIhnd1^{^G*S-;rV4%{Z*wB7rjRTCL9X%q?~(j&sw0eyc&gmERP{qo$I~a-COy@8voGaT$Gxu_PIpe6dL+4j$C|Y! znx=}k)K4$r6S+5djDs%uP0ZDwk4!nZqQ|t3YknwXe&u?D_G#Y_I~D~xrvJng)f-13 zS6TcpS(2vjIvA{*6>v2n`_gH#>WzDla4y{df`}m}WY*(45@&OXryS z+j>RL`SPV>t}n|u(=RbSyEx*1u}`Y@(c>4Qr_AfcBTwHx{itr?g*+>xOQxaHMg1ot zvpvTPM*1I3%-S~6r1}lF9yZS{OUlhZStecJQk7m1QO_AW_Pay+k;S!;H4)775xtQe z_DVbEt;6{JlC~Tx6Vd-7aECUA_7j;T6(|8&Ce9SjYY(o_jC z**bW5^`oT)Di+EL{+}-m2RU!QKis1>%hjGW`~dBv^3mJycw>&E{7yb*n5-4mfVkyH+_SG7(X^jT zmetEGMUGE*=)(<(?o(kjOm|o*s_s0XU~r)#BIctObY}$gCQ0!L`?H@ShY1F8fYq=Q zd`-sE4+4n@=8t$1L5tC0^6r^5Bkm~%u@V=_iYM>^9X)=xul)Y5My^J31e%)i`>Rly zRHK~}#(#!{%8?E5#Cr4aGM4#4>u?OlqFSg#;I=kA3`VnB*JW;}86rFu&r_oD#X)_K z+`a%=iQ9hdnps=L2CxN<8|ky>R<3 zj9(7-jJbwAR<}ZarAl7L>yjd8fkh%tSQt01PJxqc#7|EW+Kq!7Oo;FW@?Fg0pi1ZZ zV4EiR>>oy4SZT4}MSNV-;LdU;BD*5)o8Jr1?K=#ubuZ?=>ZrUDa_Fqd!Ht`4U$?0| zTyxv(_{o<$HPN#yx64yHlTYyV5b8`oQ$T<4Myf-WEWnH*= zw%$1zr*pgt>?xeX2VM08-+TLSVaV40L~+mDuv?$M)FG)ert`ojRujE74%YXUGxpSX zyQLl*u=0(h4Yp9&xzhHkTxCBSNU6n-nk>;(53t8%@xz3&SKG|U=b)?Nq>1}S2gfCc zjh5RsidfjO-Cp)5}F_DzdNJs@50pZXfXwspUJ@!qEkTb-qTkMGmyX`rHL2+-xvR_b0aoqCZM^j6R7{UswO-hBB_Z4HJY z@BOs&4^W3IdB{-bFU;#IhQBOmnXdYn@)~xDEfkNSAa=yAuGx8d+%|0b-frF{MbmW- z0h}DwhArpHy+0U2cszW_ zK0D!VFr=Q3clst|7H1$O%!X8>AT2404mT^2XGDHq`enBjT<6EOY4M#=TZq%(-wmt=`dVW!c0jCb8q*s3Gn^+ZT61mbF%Vuy28!s_RV=pZ zxW)(=(^}-t4K(RehRKbMwAqfMTX8N60r*vnlvuCuME$YJ4PW1cBE$t9fhTXMz2H)M zp3Br8K(h4v_wQ%npbvJ2WgTl8#?(4nRP`ylRyv=zt^4OW--# zw0}RVSOlY3MA{q4kb2x>SbKw{m@mJ+#m$T#i-wj{<+PT>@P+@EF!;Ke89TOd@uKv3*zl)0|h5q}Z`AI{#)$|pvU%&cP zPZ(b>D%$%t;HSLVz{ZRt{h!%6c8S_5_$+5KRn@w1_)IjtH^b=b#tbSWpPuOWTA2^{2yo>q`p1-zo9Q>dDf43|uGbELyzd-P%XIuA8*pjIxXFjvjl-o^RFFWSTsKbn~{d?zzQPprQB#v!&no0#v1(Nj@0 zmyY2X481H0)7?~({Y|ZLp=<|*5-xktP;QI2BCXlwsM7s%cI6p)M^KxVFpeI^_bD5t zQ}Zic?*5#q*Vj@Ze~qu-$oeby{sG(dy`#IW-#ir5e-QGEr zT->>PUNtWHYj5Y|7*DJ4X|+>LwA;3=r>53r=I9Tcbl^VvM2~xNHoQakb%eN5RaTh6 z&9d6W;Ud4eAdR2@6?T8*m+6etr@P#9CRf6mt0FkOb1s)y{Wf8wK=Q}l zTBvvP-o2L{{`r$eE@9nH8-D!wmTi~H1~tEfLjv=vFMjLy9_}3YDC_F$%T)%qO7!O+ zx1ypf(~qmD$U8WQ$^{KI_a8Ih4L&*oT~x7Y<`=ZR<4(_;0w5JKg3gbYj&9ZN!#w08 zW?oQ8h#8s3bU59ukaB&m;=IowBnu9`5%xmHZ&xbb6QWY*3mITm>(~-}lw97pc{|~Q z7ImE#?(6U8Y{uWf7Rhx`IG4d=B!r|1{T5uI3G!f0mqS>0ZKH6Y$_wqCqK>~dlO?g% zG`#E^Q)uE90*HJ2XVF_=2zKtBl176&J(cG1f*+ah#Q?Y48Q zTo%l;DW-iD7uQX=FwVe{P~AT^KKw^)=bJ_rBsJya2$vI#q{(&_ItDGjAzU0n z?17A9t8Od-5m1JV&YTfG7M3v%e50_iaAa)k4ZcT3!E#k-8ph=@8pKbdLLHQqT?+BR z3W}Ar^&u0J?Jsn8;nT&{^}q!b-CRva#{=*M1YkV#amv&Q1hu$PG<)V(ntBzWL|Gjj zcDTxs?An^>iYbN#hdv6b=mA^Fd{OAtHE!XlK*Om zZ{(W%5|zc0j6WKmb}@@sC@!ZcMsWpvj@LSrzP}3z&^z%xc~{r+;v2DYQ(eREQC`PF zn)QVwx5Y2(>f&xLk5^886|laUv3qLZxp4mE7enV$%x=x0-0~W7u}cvu7d8^)-YR3B zn)qhiuq1yZYT>~8DoNJ5CmkxpT6OeL#l z?cG|7JJBeVrnwiL5nLF5yNv8wTKTZ;&dm?AVm{-s>*P0Hxo+w27T~nE(&esSvO)V1 zS6SoavS0PNhL!y*j^2~bF3Gl=pWa|GP)<#`9yDGuA*D+>ac#ZNydCHM@eZH&XD()_ zbP8GSoC^Dy`nDl5kM)=JKwJ3sY`vO3yC-rtgXwvNdIN*4ehxGpTdC<~D>~J+>v80} zs~MrA>nNMmS9r!0>^8ODyfuxjepgxG25#SvtWK43{^9rDJ;qcbEFkyli7 z@`gd{Vg67fxGON*WV_D1ENDNw#}MQ)IpPuP@!!q65Tfwp9#%}m#Ku}fy3(3wUuBrr zWcC&_$E^ns{4hfE3O&6v@$!bRQ@95I8_?C{HUY8x1(1YLIH$CDegBxKRVM{S;MTo+ zzUaS%tfkOP*(O6eV+e)pVh1CzWy?|o*mx8b75N6UFswu+@C^u%Ie$K`Uvs9+epsIz52{Yd+o<1cQw;jN>KXn{EkF`ZD z{rLC^rM(gCG-xT-3LgAZY#HHk?L^UFv}F%m1~K+d)^}TD^e>2nXCM~*`<0th>vi?? zG9X{b&a;<_U(RAnBF&C8wd7qspqt>ndI=TKnliLBr_DZ_2s@II-+qUV&QrJY zNwBZ)_ZmDmQdM`o70AuWF+>c?$j_fFe0%y*yuFvT7P@jk{Q|0fuvQ9Ormw&M5nbI4 zc(%@>ujk86S6gMhz0Prt;ax-4M;0ZTp!ZuzowB&QpQrKEO*JV~nVcP)LiRLxu<3^G z*~vZN{jblFYIkChQ4tGO^gM8=(sxvS_j0<$mKT4hiHp+t{Hszr+Xd>4D<2Bpiqx+; zAI`B~LCChx4o;b!E(z9f1eLYn1 zC;b_s7RoQSNiQdL4xbx4V#NnOpU*TLwiM`U*1A9Va_{<`N1JU~`Q*9I z%o&{Lb-7~@-#z4a+~92ItA#x34HU7_tsM&2(jWdTL4Qfe;^OR{E!3 zisFnWTSpjYQqxUKF*}7j_Qbb)|qQHXXHL~Eg!de?OCkymD1I` zhel*!ax~{ zL9_Gn{MW3vEj1JAhA8~#@#9C%pFi(kWv0{p?VA^zznDiyxhjp5cApAXZ}@D9>D08C$=6kz9mEYwH*U+`an8tRUnhQxrOx7;K5AYqaU6S>v{t%TB$$n- zbLc zG&l3yYRaZ&zxh(QU$%7)roK(mEGb=i`pWs|Yw6bw$l2H=7F1R!-N4{(^!2Dgg2tZK zB7;D^yrC|%*|R@X8afA3zQ632mvNv__;+xqY1YwFZk0cwWw5=J>8h%F;?w4kROjWs z+_UAHs2ntuNm-|jPNuXHG;*NS@_yT-+ejxxz8F?ws= zqR0HKaM|>h)6V6YPtdgVWo)3ldeBkzT|?26p>e+a`S$lo%g~SfDye>d$ZGt`VOK|* zp@-%2`ld~IIRi6CRtaW;Z!`5N_=;q0DImlz~GT${k3BHw&t z!<}!nSUcRWIQcv3=9h2wy_?Vt#TyN$km~WC>BNRUm)EUAS8RmxJ#`;lxm}`qNv5Vi zK_Xi};K8aRSzJX)udIWWwJdG3#@#1e3g?{fANISIeWQynvmc#S`kpQ|Q(^^N5E zbiU|(?9!-w$6B{EDvxHR8i7oAZ<9g)TuCdm3_c8Xd%HKIrk z3vu?}x>NgmTEgCftv=<(Q2iv9Oky^T8|?>T%D~sE{~st8V@$pd@pqo<>4_c0X!-M`>bw` z%^Q7eUcU)Z!519w>ja-#@Pxc4H@aCzAtwfa@o5~E*tNC-(n_9f3JChZ#~Aeu8BeI- zS8_?PZBu^u3Lc*gFEMI^Tt|q=a= z(#g2ENFaRR&Gl*A6}7u`b#;XR=K`1fpdS_y5r{jNAQNosYhE$0UAr_;j^^4U$9x?d zn>}#X?=sSmQSt8VJkwgxB2g&X zXgA6@1MEoq+1VA43Y60S8L7UDFFEuI9IYxA1gT16!;qV@tV}_rX{Hgh6xr?zg+~f> zQr|*ax!a^VBt+Wl`?0e^Y-fboIr~H}IG;&QN_vgz1(9{L(P=ZYolpYOdBx29dX3+Y ztHr352=vmL}Tn|lN2PA!%y=Syu*hg9}ExRhP*7zrlpu5p+STOBOq4$yb$p@ZXfMh~aIp3K_7P5Gqqt+^HGbv z^^6ndS#x8q6TvT%9c!-Na^S`*(Qxn)RsIn1Ok~}Yrn<(-*wqOi)2~(VpQ&=!dllp@ z&Cn}S>3@Q|?Ql(-g7X0z7s|=`s^9B+YsjyuEWPqe`y1+q)7h7+N@ic|@p-nuEJ!)1 zX(*ZI9&tnb!lI+P3QF(WtJj5& z&J&4u3~RR+=?8V4e)7aAf2zltwj=*~lExmh)@;fSEyl|x6aEGPWs#B-PwfT|J(Nhi zrtyJlNJb`z7rDX6-2>kItl`Py#~YDRZLre}OWiU+ez+hJ6-uKF#P&PkfmlezlaLf& zn7{VtKH!Q730Vzm#&ske5!8CuDwT%%dLv-pi0yxiO_28|Z-a>dlA`_mJXYR1ctD;W zlbxL{8;-O$GH`$@26}d)>uc>;RTFtoE)gKux_NdCwzjr^nl=!^w-lc6$jQ|%yg$1K z{Rl4~-+TSA+qXR-IsP@v+jkJIks~>eA1^^)L~w6k-_nUTSqXM_cHlg|rC+{&y$-P5 z5HMA?bC#pB-$vR=M507D^zTM}W#Qet*pirnVcUmi)bdCG4HHzIYxsD0*b`7)$ZkL@ z>7^#Q4ZnaZl{93{&2E(WJqZSE!-qZhR}KyBx}0DA>J~~gWL}xmry5{PC66rNrhqz5 zm8ju|UTn#-Wk#>BuV7w61;Ps>me-%+2V^;Z8gk4-$BwOl;gODlj?d#{vKOdCpi21# z1yjw6GQu8EBX=PRVpkt84-bK;Z}tx!S#s#0o0}AHBn*9w*NUsVq%icsbnhD!B(JT_ z)+;Kp^$_!|(-+=8xxMVGTL?|~cQsndP~APN^tablS(k2|P6R;RIrB?@^B)-LsQs48 zwd<{(=d-lsc`i}w*hD2{b8B_nQj^Zjz0OqswQXs){5dI$M;4@sxnX7Ei#e(fulGxL|1wZHG>}q#vL!&5jmp2q zPjbPQg;K3*T>3-BqAM{f;8IP!X-3z(j`Wstto(p9Z^BiCc`2V!ACdyvax#jT2lqN zx=@aa2mAZw_bff@txV?`QXZ>Po1tSSRLiD@`7(U@bL)Pqmvg=Jem~!Oa^G}pz2$r- zr^d5yl_Zz0VDX#5EH#^==0fwK33;9?G80>9llW?Qzm3p1H#HscDmI>LG3(H_d_6BM zVvr0x>Sy2b(P-7od9|y(EnsnPZ1M|$vGG8VUeXsOBCucnt`Kaf@9`Ct_ddFqKl>Oy9ww0Gxi zs9Rkl#0Q4^+MTQ0_@9T#e`G52>rlL=!GCd7Y@okC0ILGF+VQtH=!-8aD7?6;8ewuc zLmjS|7F8xBnU(@_T1D9yOc^ecW~K8=X_+1lkaQmEnm^L}X?DsmweaouUD&Vbl9Jd5 zzfE+glzs>;(J{x2c?3X=NQ!`dCntT` z^d?lduhA}W+6Q5_Cv^MHu6lZ$2sXJ6z**=A=i-&ECJ4P&@kHT-B&_@_F#CGFF|uxS z>#gKGd+wYe=yEowEms?`)U4hlA+e!jrhS4yR$$Q}-KsattAn!DpVb;ZYk<-Wm^-}` z>+_-*@X%@tk5sn{!FKF2H`Xwo3&|K#ag1vd?i%0nO>oAdt5wI2RTwYp@9Vo28tS#_ zhz>WmvWyZ2+D)MfhR>TQ!duJ)w{JfPnd!S=mK*Y^C&llaKK&xtcUs!(@KxkhpMhKE zsIszMquuPJ_4hHS&0DwHY@%DdcUNgzTkpQP;lLNw_>;*QaEIRf$s41;g_gQ-x^Kth z_;~SJP}GFeqoQgH8TImQMgn4&>iDl;Uz|OujxsOVj1mM8j(JI0s~`#ua%AM;%s5(u zS0&kmIS93?YjDtYRt=Tcrf>S5eH4Z(-OjqAk{bHG$ryzv5l$e%^z;WlN;@dFSppAoYSZ@tgWb2J2!g z%WWIV)8(&5_KJ=P9;&q2ye#jPLCFNozZOm=$kV?5zwXZK)bkhGIX2pP=K7B3M3ES^ zE=~91gAv5Nn9q)9op_|qkbhdaCF??TVY`{&!jS!muAVqk`zYzTi(+=eh0CwKyOyPN z@=>>(9y9YdFYEFcF*7YKi-W%&nK?UtXlA>?)rgmP3bp1 zE<|6-bFFfeq~eq3j*io2q8x2@Wj)bQc8K3K|4<#D{4=%F%^Qw?36Eu7GdQR1YUfp$ z%VMxF|EyqVu!2y39gAKc?}FWddHc3`yOwvM>H(+Mc{wf!(k!T zl-<_te%zqWWI_LlT4$|1pXhX@l5>QHz*YY+_H`j^y-i~a+7tG719kJG)8o85Z*(~R z=9pC!O4?Vt^(!;w?s4Vn(^Ffa+XVXea~4<~Cgt!spyie37Rv13dSZ(|7dkUr}@ydJmi+__HBtTveXm|0nuKT+p1Ix9056&1w|tX@?^ z<7_$-QQpFpFB9OGFnnVBnvsbKd$z3LsHmWkk@2#a)ngaf1K&O}6Jpy)OFhiW!f;SM zaSbfpSD!!M3-M~_(?R4e2(hiy&Ue_&`nQQtV+)iUXw6v-&*vjI5CvBD_N;vTJnX)h zJ%L_Xvn>1JZ|7nPivd-kgE)HZ*u@NUEXLB#G+i6T ziu<_jV_n@^NOF7w1D8^SMMQWoO)km-7=f1U8tQoQ_KeU*US68@>y;e2*3vD7?Z1|Y*PDW-fpyGT9R#sU9pk^@t0d;-eE_^t-E)<(X;wE9yZ63Ps_w4 zPf8I3$FgnP;)xRAIUO}9yQBOU4|-*EPiV>n#lAaJ+`im)?&C(J6_oXeOTD|R) z+LF^5%P=)kJpBBY>E>(u-rt)kxn3sV0$h=sDEL)#F zis*B1-FqZB>LepYNP{CdT;0z%gNo67=wZckcR?+M-RCcs(eK;-@GIN%&8xOanJ8$e zYN`f(8ZZU+W5=KNvQ_MX@k0B1*{eUxW4Mxw(f}I~L@Z&4<FkX!9(I>bZJ8cg>Ho$X_KMhqqn7E;{i$kfd*2pmbvrbA(R(G#_6##o(hJ!;66Xph z3Rp;~>GsBq@d!F}35StahSZK1FY;js*yRi2PjR<0w$Rhjp_p->f%oH{l&jbpIO`@V~NZOMgjtHgS3Gq1L2gv>nO86s2cyKawzet{ zu3od|I$9yD3w~Xf={&ld=`3E5T_{Vh-^v7$CerC}XtJe>CM1`=F`|iED$3h@;(fo3 zAnFp^(BSn)LkbUfh;+)G;?1?!ToDtwjY{Z4*@_c$`>bvsf5Z46-S+*)Hdb-Uzqzi@ zxu=q$Un6g7w1o46R{A>Q9z}B|nc6J90v)pjw_jfmZeR0Ed2qk69PLZXs_%(U#ojbm z7{5OvKWUewS6Q)YX5vsyyVcmqtk2DAM_V_x=4n$X?~d>4ua!Eob(vjKTlKL+jNfK| z+sFhkSS2`5-fQkEw6f8(OBy!(eAw#CwKYC&v%HexnUrvn>8KbVrnUcXlXVN#LtD?ypkbwDALd`Mmim zPorg$ZX<2)nm)hZ=xCy}*I4N}IrH>q|DKa>QBsmVMt6+&J4EQr%{nyABg><3!nx z#*y)dCnw5t)V`?Rj2OHp)MuSmVp(9jYVBGEipQ|4<&v#ReY3-bybKKI^T(TaJ5Qy% zM=WIudg(4L*%dYaV+k~ZkscyWFSy|tN-SV6vDeE{|Ee0fUob!L?#-Smrv9rY=)}_; zM|(C&OC#pGBK1UK+El(U(71Sk5OXSO)g|CzeFK(4PEz zNjcq>;VA}6!Lok+4GcY-u&mfH4+kpDrMJLK$45F5-uKHA1({k|P~!;ae3fn8ZG{qZ zaSn@C_OFM$n;I0tNOfY>3eS1hi~c+Y*S;dV(<$I(YUZ4y&{*grTJNJzXlZ>$+@ z_Cs9IlzILk->29Y#r$@n!ml*L%_haN7{iXABmIY!*3yp46RWo6thrGv!1iPC`g7T3 zt;xG@aIR8|Uummlq!YdD+QaO#d;aw#YW7`iBL51A4K!9&n#w9yR?0iMTDu=!=_o?pBw`qBm9nET^x zZsTKne8g}cqjAvfu;)G|KK$A=azZEtAtBn z%VuDHXkW2G`}oZuWmkm9tel@jCFqs5F)WTKcP?Ecbo)hxh(tIpJ!wW_jUJjWq1125_4cv8CBOvS$M zLsKqmS_Le}BzC&0nlI>oOYu;-msc__U%<#Mddmx8e9$G%S3(vPC#5cJK2@|ZLH+Dm z5*1~%*T=9f$qcieq=CkpIfQHZ!O*IG+A&nK7KVdjd<44yFNav zJ+J9;w(n(Tsfoozpk3&4fUFLLM9_H-*BuKRTchRHtsuI)k+oTdwd&pUQU6EQq$QgKtxpmURq zjO&f8to;arxeC-~gQU|M*aLPo6LJkPxNor8LdbRc_es|Ht)yxwK?i`ewt;gHL?ZpI zJ25_T)oy<9?Wreu#l+Z=MmC-cF@h0N^%W_KwKu_>`PiU10MyMsvxz}l;mIoX#9G6v zE6|xhbo0{_9son+eyJwcA;4tY&bvL1TY#ER4m_^{p01OfV{4JNe(}DjiA8|e8w?Sg zeH#30@hXguu&~(t{0*qv$f(WD zJ)B&JPaa|6H2~i?h#!-wEvrQXHd-Z(&A!1$3KX}NAji}njAbwnheOKA9slFdd}iIu zt!-^(!QjCr&FJp#URXKqL0zyuo__3Ia@LjBdkL(eBHBM?PwqOb`;`=qjTgKcH!-kX ztF@63Zwzc$-Z%O}fbJu0?Z#UT%m3Af7>jK~{5SuhEw#Oq({|=Rw{vqdBqaw@Jd%>$ zM$#rW$jG_X8iseCVSOl7@IgN<&2IL-q@;L}sEbzf_!_ELUglZW)-4f?EdrBWY^4>9 z=2?MBAvALdQ|NkaQ1EK7lQYlfkkc>*QDl1z<_Q=T22nmUj zkZ38p60*t&6|%Cj63Uht;rF_Dp6B!Z>vta=j>A*W&3#?hd7kh0>-`#e#pB-3W*0NA z>z@v?F?c)g;ewgFx#8&&YUl7J@vm{*tpNkYMN-|lM|TE&)SJ}k{y;Np(Op)eK(gm} z?47nOH9XCqYrhb6ZuVtdCS_)!IIXR0XYNG&nO#aa^+7}afF^3PdM=_=!zQ4@qGHzg z!F#!JcUD28qNyp_P5ewd%`RHd$;eRf%c;JfoP$ax?TdVbz(ko z<;2Sve?J49%9|)C-%eYyr(c~fQfuix^gJ@rxTO)-#)Cmivg39 zz!+n17RReA%rw12LpNYS-G`@30l`W|#W>6oeU>Fb!y(`WBj2@0V4W^V%w2AM)bY}; zG{7qCA`7KNMYm%fjttmxv>~w?p}`NBL1H5~clLl@iE%l`B{Us2Vr;OK@C=6N=`1t z>2X{{Bfp@a45)##bB*~jtYw5PG(I^bE3UhSwWgnb-6%7+MHHF`2xw+@0)DK(=z9{o z8iYSr3%$^5$s#YEsEiDzV~EWx*!^?vRP{c_N~IR$aFDC3t4Ahdc>NQTFbOCb6%tS( z5z7v2U;G7YE9?VIbM!fe+%jMKz{-a}bux@gy3FE0M+7Hoy|J+|9WQ1+$r-nf^G!@o z6EsDzs>w*8MT7W**}35F-@l1AME<9QzcG+(s5ppu5J}_t8LtpBk%pe$FO5uqPek_<6coCx?HvylIo^eUutCx;GZDIy7lE-Jh?xZA5)I7pCk;ly zS9xv%pEfF5A2;4iOwSwFi8B8vpKB@FS*3SPG;+*MIzOmSCNiK4v% zsc*}~7(@F?{gfZ|Jv@0wDB*j-}|RNvSgpcTiBl)_?<0Dxc>7pPeOZ1QpV!0 zXwSLAL!zF8g&NMwg&Hou*&Q8SJiTc-htbOa4RR}OQ&{hnLr2oMr`h(NGl5C^bm;1e z{@S_MDyOq-@WsnJQ?L7e)$CvT&U7gM*5esTUXhAIbuq)U|1h@ z%R81k(;bQY23GGspPXi2CMHtytEIi`d0)h&;4!^0P%_w8cnJ^Hm@ph~K|Yb#XQm6n z9Y(WekOY2T|Fz~@XD7i>^@JBrN_u)u%V2-MDF}AN5SPi9IAD2s=|eDN0XL71i75y& zSA}PdlGb+C*5zQ25V-x}nWobQ@9tp<ybjDCMwlL**2?uiM=7}H< z0=3*lh4)N$LDCEZaX7E%T`&J`vad4Rea4a~=(|B3=>?e|rcoZ9p#6c^!A-ygaf-b1 zKn#$XVx9?#o&uaN+~ny(R3FtJ$!zKPl%N;j7fcxr%^wKsp{*}3Y#tiAKbV3NBo&I7D z!WyRh*dpZ_zzuU);qD)}Lc&f&K^*%z`umTb#~U|sK-naWsE93WU!(2n&L1RttTk8? z6^u}vbZwxO2y0|)YzUl;NxrJP6f`upuIUNT#q|96(ey}eGaa2M?AunAO2)?H8kzML z1Y#5JJZ_2(;`pgkN|u%{$`1R#K^)5#4ykQ0K?KV=z@T;~t%u5^it9unzt7I|v1f&% zS#_FluXor+s0d8;eJRxv44(ONMqpnu5#L#AxIORmh0-=&nHQ(tguC($k#VZ*dAeR| zN4r9G;f~ddM0}CCnfUeJ9w5^HI_1fj_TiZthHL`^$}Yd3wzhkN91RnmH!gnt>Qamq zYxOX{^aPE$SEEy9_1TSK1uy(%I1}0|MzuT|6$X8^`sU-cE!pj!!!N?6`0%$}l_|H} zOJ}=7YAk-bY5nNS%UhzM`TOTZ(a1!F*;6Y5=Nn3X8Zf{z_*;0c zMpMT7+t|U&&LR!!qB}FUQuLr6-lE_5=kY}#3iBiBeY1lRihJKXtq5;-cvU9V8Prfc z`Db8kd`sI7nkLs-$9Rdd}$_ZJL1(u!Zq6mu1a-_w^=4-_;Cj*2=tSu0%)M;YaC*RI&c92%(!e1BX` za6O5T4|1)zspighKk)h6M~S?>#@XKwx`dFYt!@#Eqi6Q=Rtr6;O8NiMt^ri$mqXdNCo6N%Gn3v|Xz!t^~tFrKz zd=s(!P^Lp)P*=w!%*s3I<@){uD0GC>r}$$_3)3pE7B*Q~6PW*-9+$E zQc_c`vTY#DB!(Z4wFHAzu_6<9-%$ZkuXUN_b+e@sssS+KmQ011rA823+1}yvsjiMz zKb;ZAm^eeh*t2|R8yXIfe^*!;sIe7TrrKg!`@>Ph~iiww_hJO2` z;)|d=cSLjNrmr}*&ZhLe+d?_@XmtJ_Mej4iuFB@K8}F!4_KExNpZcw@re$~1+k3EM zcH2f@$;^oI_?Ba{Lk~Y!?voG5X17o`5Hk44+ zaY_%ku2;UAM4)IXRjH#neJ(-1yAK38>U^&|d7$!{oWw<+vEzL2UK*<_m-=Pwyztdc z!YHxjuQ%oENm;|8NBb?sE2ka{3x4JMekqs89wN2k*g9XfT@n z?N~b5SH~mfq7k$>`!uL~`0!uOltUjs=51X^r>PG9JE7Z{=ndj3HM{I>2ugfr&-gg4 zdL80;+Z!<{p9GP}9OQPI6B_7Jvj7OE>1LB3)En&PU%R`SbGHHJ!rePLIhlaCFb0>j z&P9djG_VCKUqbW81GCcjBO$?@*HEk>Q%@Kkit&f9(nG?rzkM4MlQ}X&2(TI_?z+`F zawiVhVp~K%0d=FCb}uR#!?pp%qE4)RbjpUEe0)71F3mcco9{qQ2Bv}h!~uKG-abnq z|Mlx_VPRqRj*gb*w8_h-9@R@kOTP+cprby_$;s&kKd#lHI4kRmksVmE~DhQ`gQcPh_)W-sYoVLp{@kh1lK=AM8)GK{u>AGfTAwVRJW0KKzs=aI;^MxNX8y#- z-^0Yi+dgq}?q!#G%f3T-x=2aj5vRMLwzgPuRu<>AYaHtLQ!{K8m&G$Za7rve&Y5lU zo_wSux6bI22rD7QC(6iM)#pb?md;N?8wxlj0v=V#+1Y=b7rKf(T3=nFC2GItXZD%} ztliQd-@p4oC`W)tP^%pYEx&_N69J@yb}i&OW&mj2fqW;gI|Z?lP;U#v+nHV3WeZOA z{q4xFAfQ}kNk=}yDmUO3-t)(iCt9Y+JD>@Djd$Wf(+ueylQ6_2HF(0|D?>Bh8x<9W zx}4ugL($6sW&x0vlpS@!5FEYNR^h$8lb&-%E(!|%$0N6D8V`<)+$0)ym{=Zx8GTXL z8rvIVKxS!|e;06lRIRm`Ip%#raWynEFss^zT}P9%oo!imSPf!2n^ux&I9sZ$jZhMMbrhhk&<+2r&gEHEX~> z>OGP##IvjL*5k)c`2@FI15-vc{jjKebnQQiw_~L=xDdSC(Ncb2! zQ&4R7z5EDVV78F*yo)HL3?VGerJ3s}l77`;uDSdC?6{3sQo`L+<*aYT-EPW_J*eA2 z`oR|i>{|Y-|GI!f2ez*&$2vL-bbB;t!daS19=A8*Q??vv`Nd##JT z^W0?gHalxGN*OImG59!K+%D@qwVT!Q)a=`xD^g_Ui7X4pg<&bs%hY@AcA6#W$k%2w z0CgUZi;GY0jpE(i&L$}yx2wHNeecM0acs;cDO-vo(%&}Ja_+^bi$pZm3GH=FCdx`i z=|v;kzj=jn4LrG1W_kI6ML)~W2Frp8&1IVYX@>j>uZl7w&Yub?xz_n%I>s{K59a0X zahRWQyvaG*y&TTVBG)xpR9pW@vNbEIPstsAl)sX2Gk%^U#?ESLG#D0*I%7qkv!fWO zO%O0U-&xn)%>`{_8T=?s4vLl-7kVvoj8rPr0P0PbtU7#=fstY6^l?F$Dz$w_M59 zEpC_^Vc~$8bNsMnP9O99KQ%S~HI*l&uxOkFJ_mc5PSXyc(Me!CBh1lTR(bW4MEt<) zd_UtYPV6kij1Pl#N(ASP5%cN&A1IL>&HR% z&U!O7b%j%%%m#u>unJNYf3B&a#+nKawfe}?(%KpfpdK*-1PupQ8ahCZS#u%gTal61 zpwuAw9!&yqn`*7oC8#(WyWOWR@$kCS8|K-Q!4%L=Q|`)74HOIUEwz>BbMzxYiNt1e z{`~oO)?%)C5vi#H7y#m;(*mLn?;ld~w z*xHj7!>10QtfF!{%lr9k2yYQfj0)>5J#+HR!%b)DNebWY-23@V+4oR)E>W_qzCyxc z7yetqfwXd}h+Zj^svo<%gP^V+uc}!pbYJm-BVCc!IcZK&^&4WgF-XigC z*$@`Tabw3y9u3Se%o8?>&vsUV2sqn1nK4!r)az6xZJRszV96saXj9V0;{EN4>YLx! zEa}0OsQ$*Vad+hpQQ^^9D=~fhrBqdqwnJ6xF&;~}S!`1_7}!_ac9?X;yolvQ!MNGb z+t$i4CXy|Om#^W9H>Y3vuRlpmRg>Sl8Ir>0e%)OzNPdw;v)=W7ls84iVZDuV4dH_l z(f-oGB_ai^TqO<^*M|n2=AS2ql?4n+6$|EGnlH3x_ZZwo>KS;ykFQvmp4GM4eCAI& z5U{HJw&<$#7g?Mz{(w}%q+s!Io~B>J2GN8eV{8c*pEByqi6tDH%iLC~VLl?X+|-w! zr5Bg!Kxe*A%DJ#MFK_gXg5=AvL~;9jOwiUR9nNJW*+p3|C^&%C>Wx(}h~tOE|K z``UsI`9}a!FK3oq`9%-weW5EBbxS2Y*FiESD74_aINYQ1M&nv_{@^~N!h2B%R%gnP zaX!%sDKA$fOnyMs-Mw$$l~!9J<}CPB{%i6R6BDyawIQHl$YF_LItEqWK@|b!OJno- z;vrbD?3X-xL=NB-h%gI1x4z$^Ba&EmZB>!ezg;!Bm6>$+(RWc6o@$rNU$rhG_6h|68 zs$i9cN&tVrD3-jLYaQBVt?c3=hFyqEBrJ-CiVBj69JylYvFq^vprarId{_45WQ@M` z$>Yc0L2W1L{gZB^ALQ~u=+M7GbwW)|%`w&oT?po1grG!3B>L)*qQ<3{-|dZLURu_0 zi0>LY#)_)`=Md4Ld!}zT*j^aa(m@`--OVM7JZG1{?N4l|_o<4W%(Xmy$RGO3z>ls= z?Ktxf3h|wF=ddfUzLJam60=qtAbcv#&6N>@(9so)zWFoVc!qE1_O?&MEtN-}+|IQS z_xAK%T9olPh`M#w#94mp#L_rdQ1ilhY87^AF%rWrk(K!I5}(Z;v)`;W7m`+T%zKMw_a$ocZd%rYwfSRp36Wd zRXUfwD8~9$jI7l14I!hK&k{9e`hF=$#)Uniw8_Lak}qY)PQ9}d1$$~hygAG3=xGu+ z`pG$G{^!h&luYZ_E&CTHbybZx8fSb<%)fs$g`p6M^uhg5@qDfmh=hhiW!A4>>&sa> zP@KOpu=(SAIpe_RNi%P{XJ4(T>%6@(0xK}?JE95V1Ihy7#ZHepXTxT8vHOfaE!ubI z#H}oQ`&65nFq_Ku`;cVV4_ky6jr@3_`1$57?G6<_dR0Rvs{<`}S+nMR1mDlrIO(Si zXSEjpJ;-3y!uh&nKw4VBexm1~>iunej^{^DKmVayQv7H9YVT39H`e)`W3tk&T@@h= zZy(Tv1qYK-&g{kcRjAaEb)e)gJ^X8Vm71M!;%R3ZQRjb~^qC}583hKX6kND;3H1&~RvY`&m78qhG_Nkz4VV=xdL75yE>sHaIy&B@u`7$SHlBzQ z#E{=51*TdAHJDM*pq00dkQHfOLWKj>x*77sbPHVedoNEV_T{t~l>rUi%Eq>ti77^H z#{5;AhK7bOVmVFmgZmN0t2M$FiXtZ91$hGA$b&^es14GrN|9B= zeTU{tKw1Pmwkc3zR6u~y`(Ecwhb|2hLFW(b5YM`GXaCF;u z-pU3cokEu3z{}Ta?X51PuU$Rj64KMkT>u`l_S&Fzxv^)0$w(p~v-oYdWt>gNSkFvL zM8_u_LGu*{KIFE$)ZDjNoCBjK&DaO`bN>eUmanV;kv3%e_=r6ypZqmm97mO3_{$}( zW|v9-$EEogA4&hhLVb_81$PG7Wr^*5A8h0`BRPeIyE-}Ha6P)T-(=IRa!&p3wf6iYSHqy)G`1IPf45Yd3yymRN<`a@Y{6KG8s}2F?BX2yMl8qDycGJh~)wWJt5UzQpQ=!gy+&t=f z;D>of$%AV(4I)R*_3~A2wyTq$-(6xG@+Y&-D+CgF#R`Q%<#Q@8Qzs|mj@`PN|HX)j zU7E@v#I%$Cq|OB+!Se2w5e3lvn5jqA2O!e@cKBW(t&z-@s3b`}$M0-AcmCh2Q%x}0=a>CH`xf7)2Q=^ii_@iiE! zwO?hb>?&#-5`C2M&9|;gDtw5ysL1w5GSz&*+uZiKyNM49!Dm z%3q(P&;|vI+`hX{OA?fKR6=rxHPu5iJHqBY8R{@GedT40@zs7Rm$AYQ%Z>vjpCMxq zNf6hfGj<9Iro4>|vd?kibx9dXNn{A7wT+z4mILX%5;B5wd?yP4sDXs`AtYvP*YXO6 zMGxi(lS|`1TxHjD{-#x#W%rbaFwFP((_)Bm;AjA(rpKP2F#{TjxMXkN$Q{St+>g&n zOWT6EBGGbboH@gHG#YQ$qyrTEbAohBQY{qTm@x7xDT)YTd@m6V5a3o^oPe5;qPi~% zm!OyR7K(_uI9|Us+Fc#6bR77;7NG{TDV)mVJu;EotQYck(+gMhk_S zayR<M6U9xoKmnukMXO3_jfh((b_UO$F&*7XTX^{M zx<{0fA#aV=hp6wlW75RUPy8pVAUE#z)9YCh8P@JOzIxHi&48Wo&n_0lzd>Fwrl;xo zVepn>?#aq9Z)OZAt32j^n||HwcwK2lXyEKD{lJt2uhup)mC_NJL$0^?XbB2yQRM(^YeXI?@3HLd2k(!ozCu~+H&PR zgoF}q6J#Wl#n~;;Q`4H7+Sfl@cAn@T(4bNk+OLnd9a$(*jQo zzb|yf8j=DUXLTb0cI@{0a@bU%m#^#wbX8CE>$O`98ATvHHT ziAnE+n1;R+s9L{?Uy{`_Y%%oF?FUB|1}U=*;^7^6pB~h zmvytK393+ukx}RDiaxKyGo{4?XG->2j%*X5U1=8iJwJ2Ryxlg7(k|INKDhC=OSe+P zSi$IWA?@anx%V$t9()~`tq?;7RJf;J`oCp*SXG*Z}(e)sHQ8>7iN*ErRnVMW*sd@H&|tB_;< z{Vm;8neYWe>jH<97qrqAMz0huewSbDo}b;raqz%)llf?q=(_sS7M*pE>TeVlN}*He z{T%D5yHD)zLJNcR(&AOEzm0MUzST{K$rShA>FJM&xbi$Aw~I9}x64vmQdM4_!CkCjo2&)n5eflEI z-rlUycUkS{C)UMFHogv?NYm55S0ffLU5VI*VD~c6ED7fExBN!Wu)0s5%Agw)@|d;8 zP?`na$OIJ>5q=xv(*NX)fzt`4?L8zM2iLD|yL9zLRyrcfh2;cOJ|t!fXjS z=EB&o{Sh))R{eamE)AO~ua1t+KE0O|FmHQ@2GJhw0*p5BSzeW;zyR%@s1+k>s2qqOwJ$O%PxRpW1&MC>LIW zefB6Nr7Fo5@IK)s$Is2nssSDuF%K&<%a&-0z(AaU%bB!x4f(HhhQ^Lr*eu|9q>SFQyNqpjaTszg7)w6Aq1dot z2VihW=Ga4g0SjISqJTo1Wej zFBug91BVU^f%o0ZVhpL;)K<^rHj3Hk2eTYC3O7iH`1FITphMPs?i)PJ-yx zEhDFTzylzgMruC_3AKlqk7Z`2*sB&EHVdEy6_H}2Y&CKmLv+C=_IQbyR;gI<3xJB3UZx%|kIcDdL1;(<{QA;n;*omZ=G+=-YY|cZubfj zpO%Sj_7pjaQo%MFxfj7{tyt~kwzz; z%PpPS3y~)E5tY0qDg0{Q_ZoT~N}f1so0N4pW&KK2f7s)4`rth{>#6zNZy%5a9H~3^ zELP+4wgf5VtG4M&?oFH_XZ$V?n@7%nNKrdwVn65fA&!m7v@t^MVUODB5j5u+Ty+&Q zVi{~p&925PL&G&Ne;d={x^rjrfTf98$E`cf4vA;ahQS%YTGLpoVPU5CT&m77(45KS z7V7u3TAz}ugYOP}_yzHfc%htBrJzk8LY73ee9CnoHPk-84V-`PuGi^0fF5REzE z*1D^aCz(GyGxNaL48u@_XA?8Kii-Vm=Zc(%83`|K%<%|*DkNjPibQj7hGHeuyKu-g(bhQT<{exy~{!_eJbif{{u83m-cUOIrtp*|hy|NGYyws-pF z*zA(DQ#n2=Kcdr4-F`4u6?P{-Pi#YL44+xVL%%z3`Jmvmu5Pfp+|o9TGhnY50I~@rzI`IyA9KQ<8_?$wvXJsW*#Ar4Zs>jB z>~!w03>h~M4;eT&aFpE1h-H5KaBXcZIO@s}?vUQCk=wA#6XaBkC-yO(3|bSsXuhDTU#)u4?)0X07aB;j8sGSN6ZTV z+rfE3@<)_aXhZ}pVeAC4vbuVmQSfc4GL!6H6I+?F7MJ+G`xe}X^w+b0Y3{Sl*|=H3 z@L{BDVHatJ{f_Dli39xQS0$6WtjiMi66U(QZGX3@sa zWPa*nb^k-|Ae%U@q92M#hgYq6!W=U*1B^Dx;nNk}E3#}Td!1X2&qOl`8tVIVEl=R(0vVt<*5M+Y8!0JYn}VUZ*H{8`slSxbm<7-^JE&J5m`%cE)NsW}yD|I@C;!F=Z z5#d)jT zq7Mpz7;nHnQ)r%M_GbdO^yTk2OoL-N{TdS=^Y$MXV{MPkO18gE)O%(ZubY(@PPi;O zPiplo#wC_!yV`yKI94E(3EavZUhWi8BipsTgZ4?Ai6P2M@O zxY>o1tQNK|oaTv~oAfsL#~%~C;d^GUl99e%s<52u^Se8|)tGmshm1{&R`JZ@+Ie+* z#dk!9zl^^x*0Hc)nPuI!HO-K%@rk*{_@DQ*%BHX#B)kbD;|C!{ng1^eeWcb-|#qeTZ1K)omXJw+@)mVaQd3nOQVfMDf!=(k)%q zds|^956fyGj{6<4C7lN07WK!VIMDbIX6i^c5!O?oZ!vs?*GKSHYC4%}N?OjFgzfgZ zT-(6+CRr^yXU?qhZ6QVeKh~d3F+?YecmT(gsOs|sW9lJp0>LS`0b>+v7$Yb@s!#H6 z%`$Zxh`q#p$1z(rxfrhe4M`)TojHGn_=^8=@ZO*C9-I9?{@qG-N=o=>(}s}nfVYUn z=T9b56N}QzPwm+&IhaC$;~6K(DioT3gX-#f?~HBa~5 zyVP7=y)jiM^Ex)$Juiz9cTNy82%Ai0Wy*bbXPv&2`^Lqw{WFM!{$M%cTjkH8n%0=p zK~heOS5re$+LrVgWNT7B+!oRa$P)^3GVV#fWtNu7YVOsXV)2^0s3CZI?+#0;s%mww zm;$XEy6W4_iq1^9eE(g6&$0GWKZt|}zgBtBDOO*Rd1BTQ0by*&d1`#Np>O)v*;yOx zBk{5cL1-A7nwWdX$8TnvFVV3{|BMVr0C*7clTba;Vpr-h(NTTC9?;XXoGamZOm?1o zeP)r%7E;d-zGH@+=Wc!S5dTwk?VHqNZB}X9)=*288R&Qw43&1UT7})(b6%rhPw)1T zO(nA!HdwLdW}nd&1U{8k9?sMlufdwVjfzr z#Kn$lY_e4k>qvbRVP$a%gnc+07lLk_aqNd z)Yhun9#`ah1Js)^;-PT{)yEC~Y>PI(D~WL+VZSy6^Tc3uOE>;~P@J#Bj0ga|!@~40 zZHAyd+ctf^FS;4b!{OYOMtaQ^*iuKr0|aX)S68U#oRN_c!IOpbT^Z}2h zo5wX#pAghR)M{JYi@K6z5yAt}ZGb_SxKq+r%_zC0U+Of2<1fyp!V`0+nPQV3l=6hy z2L5ZIHiQAU%p5xgwC-}mUYcwRmy(a4J|#8&8bSEL(6J%H)_0W@2%n=oD_;7urlvfw z_}%3H9^nB{GJ$;jzNJOyan%CM)QAX%HIa<;=VzBg_0yGARkxrRfu8qj=eWPMpD(P4 zB#h6Km;(6RMd(SV$wj~@xE|e-FMAC6Bx1Uedgi4|tPD5U^anm96;~MVKlZqQ9=wBz zv!CXd|BL@fN$jzv{I?LXj0p{;SBFib&C>ey`DUZ-_iTs9Pq<9jHnZI<<7*%UxB!AF84-ss(;NfTb#pz>;72whYAbthxOd# zWDy+d>c6*KxpwJcXshXAdYHn#+igdod^MEgqj{9dPO5O00K%iAJyR?F$mc6IxwC%^ zvCHyg`){T1kjXyzgnm9Czl{HL(Yb};V4Yh6%KY=(!t>RKX;btJ;%UEs&wEFXn<|1T zn6NiRvpf4XKAxQ*nG+*n;g=5%xldc@?qg(^^=wUA#x9Kl9OLZA|2z+LnjYs5vLHMD zCPuWxCXBG?z+g`%dx!lToqQPp=n-dZ{cvNTF*B3sicaCb3I8o{+H+teF{P=k zP07U-ehrgrZ0eYW>+Y)wiNaq9gOdWVV8{!GR5)UGMO7Qkbp|SeuwUBI!S=63G!8Hi zFED8dV==;)i;#N$6N+Od4U!Ka0*)aZ`@*;gk@&T)CBJ-0A17wxXO!uh0>NqJjqL*5 zF^3YwHI2Wvwzhu%J2UJ3dukmuVR2~eHM1RkM5kF?Nr{|xhQUT!+K4V%!qX=u4$%4h z{QSGh%1ZBu)HiP!am20JMAIZD=03yEfCmUTx0ECaX=w$*V0C$U`Id@@T*-bqYSp&3 zHZX#N%I1$g{IxQPM}N{M-wvn%l)C^{h#;JtoE(m^9WkE>ISVx9L?qV1*fb{IC&x7H z-k2Jzaex|U7Ibu4k%1I1aPWxEr4QeQbnAyDOi3)l=2}X)Aha(-t(!JPhPYK;hl*-)J$#h2=z`3E@$W6sSd-sO&m$CkRW9 zz`zhD16o!&@54y%3g?m=(^w(MgMu-@LyX`!wE9ZfK?p4X`JkNxwXB!1x*i$w1g+cb z@BAc?tzFS7PgbigDR7`LE!|4ESwgi0D6BsD6#5@R9*FEt6V!Wz<2R2MKFH91AWLw5 z)d8RJLZ-C$UkPx$?;VLv*#LXrQrv^yVg;D`g3+x6&!vA^&bGE0MR*dqAVS;@2B!l- z)mX1cSX|z`9=xy)flk(~S!9@{pzV6F>h4SEwTOwsO=cdmfYmaB?=BQZaI~vWss&RN z3+TFHX?~kxlM@=utjmqNnwpy>o2ez#tbw^4v$S zU%y^;Ya_bF78BxA#XuPXE|y*0`X<0ns3X@H8webDckz{TYlKl|B?9XRAt&IM=Za`+ zq1xEW%6bEwnc_s;C?dN6HVmbfIZ{m<;FgKlSbG;2MigC-5T^z9CQ)?3ei=L|`i=OB zggP9K=)y7Y7<%N>H!eC3n`BYp;cK${@ilJpgz|RAptro=!dl)Je>}CPtV-VUCi~;- z)q8v76E?Qx>_Ul_a?s_k&lYFe19!IY2s&y_JDs zhnmS+0ocR?%{VM!@Ldn)Y@iZeF9XTO{?G3V7zM4_OIM+l{yjVU0V6sbaP~+D#KnJY z;e>hn`Sa&ZVEcYg%!XBjHH-?40%Qs(x{1Rcldub!gmp7CMg(Xo&Xg_W+-6J0Q2poch$`=M;<(u&!QUioQ;wUHh2CEqxMq4R7 z5!($iF|t0A|5XFnrLO6VerfQpIctL_;oyhOIVmH9hE3d7@zkj>1$eV>MKo_&pl`?n z9PY5^ft=QxrmK=*8GwmwfmgN-$WZr0F4ObT4-7&&kbF%(UE3U_hgr!-7he&^YHL=j z1QwyX>vYi4J=I+R5_kjzHj}vTs6hNjm9A~&lhQoIxuh4IvO>%?X&?+rmW8h)>J0WT zbJH{KGK8oW2S1ee@a93_7~!ZaPwamWpwwv5oYurl%N(3mvfUPb@f<%saZRP!bF-jy0V}~YfuM|f`}Uf3l|BoN z*;E@H7QL^-=mxP)Nhd)+{AkbHLS>GO(*>o?(AzUI-ug;X}04l8ULHXUNb_O|@@ z&#H1l3#y7o&6bnDKON`;Ttskdp*m)N(_t+Jek%-NSWZ7vBj&_BF-|JTqSW+ZU|{ek zycd3a0%KeWp$uStDc-!9;-doQ7EG*0iI^zRT6XN+t3vSAo)TkUM|pWN`0_s13MYGV zO^F!%o;$~PJNT#O?rodW4D#v015rS_b=eyzT91OVhTV6`C2dVvBl@b%nqcXJ45Q!$ zlL&GmECMYjIeyNmII$e&f`Rez2&l_wepW<>JP=uZ@K#k`N#QsOL8&9GC;Yf2poMZQ zMZn)y7M49)NW@W8^a2m(2wZy+Ur8p?DTk!qYedJ13oSiTdGQq?`$WAloBarqbm+?P zhe)C;DDa3_-ly2)gLenTt_K6}{ckGD%X^@pVYxLWhdLT^9HQDpGN_1~foW|#f@K=SfN zQWvV5uhAFrT)Jz3MpClax&7~$s}6U}NqBxz+)=syV(*TjA=Ulubq~@je3l2#)A8DT zPww$W#U;GAsblLh53 zE^6>U$r^Od%DA*f9;2cLU+}qwl+BOaZR$v^N)?2s5fvv8@#c=#WU37 z5lp!LPi$GcIZ9v{K|X%`_;st8#TRbkum+mTt4I*Ln{q+hBKYCl+$6#nftxDj#S7pb z{~85oYSuq5hO;WhVb{?!t|MWBhVEsQmWzw4wu3jB5z7imv;jyk$!Qh4_?da1ZkEZB zu<>mD^s3LF6#)VvMsjm#MMJMF2-O^FR0f5{=#L{6{bnjgEf(z+UfpbZ?C~!$gpr zs{c`*E?;e?ig&TEWc^xnr24{))_j+eSh&Hk zo37scHiiiBo5Rt04J|cc!>OHWe!Cc|^^wj3rkDf9%Gh!sWnasud3z335Rrd{qR7Kg zON*X}<|N!^$af4q!#se<`#W&y;>Bc8`ve6Ad6ku#=1{knp=mW0F2g_ocRHAPyXN@f z7;hmct-MM{6_@T{HK8Qvg{lWnfV4F+^n|eup-Kuo%A*xPJb;Q#QQ*!x%#P>| z6mgq*7&Id;V=Re(sG+Yvq%jWXD9C~o5G%8dg-+(h|Hc6t=U`g_kbwr#nr~Vew{H); z^x!Jx-a9G*Rm81&a2VwxfCLo0sPM>15AoYeCDbkxi4WXV9J7e~_6MGij)L5RRbU%E zy;C$@g z__#`nbJpY^S@-R>dY#;0q0}#KoO)Ax`q`%pRdTl%=O?tPd-|KEn=4OHV&v(#r+deL zRiI|&{|$aLXQ`jU>Me*wT_i_pS|PlvW||8Pdm zeV0p>eI!iyt_52hanhVvJ(c4yP$vAC;r?3vKmv4Kf(DdXG^<~7ig6E`7mIpc5E-F> z?jVw5Wmo<>5r!jF8#mJZlQ_mZ^xBT;Fzis@*FQRTZXYQwCMIaE+lLG2?+4Jv^N;MH zlgVzf6#bt@wBfeq zgBnV3!2!o$eL3m!jpFSIGX{bN%g@jMj}v9`n6|NT1em50O$7NAB2nnLz|2Z1Im8;) z`qo(hDevqi3=U@{RJ&*P zo?ShkUEx~m2|89)ZLMv#O>b%7lDBF)Hx-_0%w+aLldA#uwq98v5!lm-+%%k^+)>r; zJX%A0fJwRQ?NHnHljq8}_+ajzlOnixu z=RF)B_#cQAOt}p#^2+ENvZ8LcgDoE3Qt?d;n;1Ax084c7vRI6WN#wYE6=h2h63Dci$B)A3U+@-m=tAJu5~=MrQKsIlB;MI?eHuDk`Ojml*9w-;N$vzR))qH4X-`1clGc zR0Qx0p}8PNDN1YNrq1e%uZYM&*l}K4E%J7qy3dE$;im4`nu{0`=Tx|fpkXGooQPZ4 ziZP4(KBP^|PbIF5Xv{t0TB5??2G${Mfy*c>jzfyLngPNxXL7nx9qwWJYZrx*5sM=^ znHLJDqt6=UpEV|dNAYQiKwuUZSFx4&Mx`dXUo-ARiUuL|NQqH-LP$L6*w_TL&dX#1 zj`^ppg_T&)DBO{7bBMM!C1v;SJ0CIJbiZG*tH;7y#`Y{TO1ogrgWt{t=D!!bby9NW z@w$HwRThihG{^yND# zq8(B|pcOkmp5m#9b1*UsLNz18-DWLD%xa)lrI~^IYXw-J6&qi~V9$cMdNT_P3(bsW z4lBao$x$dM-uUN4=R6Am$Lx4_1R4bGT!JNo{}+6lU;R~p(qf>N!)gRD({=ISF>uV0 zhiDE(m1-A${xk(tXl{$6xuzmw{NvP{HjOyPQ1JiFBfa;>9G(t6q=_`PaXHd1M%NXY8y}}PdT;efYdcx zJm}neSF@@?Q-_NM46@zTyuAKxQZZgv8$}aFcsBt$tsNY{#F{QPQk~(S(Y4QIw_?+# z^ic9TSTshd?!~wNdlf13Qu%Sa5YfMDFTMwmz-pe)3g)1stM_F{i8S|$4r(>UNl2}W z<>EG8G-o2jJmL%=Oak`A4_3UHHmHaD-kQA*MqV^C*zgHI^N5Iue^K4~M+f6-Bs^yK z2^tkr#npT@D?n$7azz*hB)hJldVc`JQ1D`jNPwR+3!ql#<(Ysp3~rV0nU9Zgs3@u@ zqqOS7Q%2*Qs-1rH%$aCH)u4_&cHC!08jsV6^CRW3QnafCem0(sb^@$qTA&@7=kgtQ zg4VGM#s)>hsO4~|q!xkM>z4?+KV}$;>j>^G5e%`g;2z`-P;V^|f$ZO%B|-pWLjV-z zWtSZc4Glp`&2%DD=N%%b7{;EH>v66PYSiW5QCup+2Ab`&0$VnCiDZ^6zYV6-#7zPm z{b!|Zu`VyKR-tpH_~LaMa*E?esVHYQVHNPDnmBh1y}5aQkHg8AMX%=iywn_PvKb<5TLnt7Ni#h!Df1>FkLUA$w>yZmh>WiffXx!3i(rvs{Rd^a`3QnU5~sHtny` z2_b~__BIw4y_KEyJ!3i^uNN${e}A+}%OLNuv)~x{GNMMeX=>1U7isPF@M;!I#P5)Q zzV+Q@)tTe**1%w9VjUUD`Gu9D?^D+;xo;^+q?&egY&mGBPBXHwJQweBQ;?UJXUwr= zw#_{!pq=&weY)yZQ2L-BNx`NWdLIIG63P1}&?%d5d74<4wDIF#lKjzprv3c+lhT%w zPb(K{Ck5Nu&unpf`>ng&)IjcbK!zTu%MS5nps0 zKdwmiUy*9gFqF6MZqh@wnTU-IuQr-$w~$B=+dA@{p5d&p`{+LX zGfD5&MbSSsPFU!#Q8nWMKFvu-m$ZH}G>y2Oxi?1CUzk5r*-X1`wlm(Esf#utFkSFh=sivHjE~smWM)o*DnUR{ur~Q3X{Z56uQLoq$LEn#_!Xm@ z{nkyM39?@L*rCn(#mc*f_WpCz zNlgNArp%tV|9L1x%qxF?%;~%Myjc{torQ_(#WQ^ehJ7TG_LSRU{nr+eK#o-ZJU$U6 zLRe|&i^d}ve!t#JMRB_>k$!O4n!u9rbYERbB#SW>6_wXOC^2>zyEc`spVI*AdOYQ9 zc`T%+2#m0;ARfyR%UsBi^;)n%U^GA&i+2}Eq@EKj*OO@F{%QcC>DnhEa>lM`<}QH3 z{nnHsGT0QGu_+Z10O#->9iwdA?|Fk_Pd(hcx$vdGf2W6ySM+Iu%1X75lwlP$A@IB; z?xpea?-&FWzJ9vZ_u_NxKmeDUaL%db*IaD;Q5_{}@{VK$GZzxcZ67wLGy3}X_Y5Pa z)yVtLg|<-_;z4+^rR)pJW-zYRVjUqbfeW7c#*G`lZd<1@xD36=9t_TONVLv< z(~01eiH?qb3|)|8y8R)?C8IlEjZ2a|84;0hl*E11lJw~ z-;4e2MSIhW-Rp)}Nnb`jk8R5c|CU7}9eJc-V30;DJAVZGR@diV`=1EOJqv50z#3L% zIEyeIFtToM)w~THB?o3Lfc*7V7ZgbYguJ(p@ld4X<;`E4g89(@CrdL1Ou4s4hHLg2B zu@n;Ez1x_;+Vx>Kg+ezBc+C3E-#7hja2Zpb=5%ScyPLoI;K8%V78A`}n`&IcK}w7> zKe2E|P*6Rv>CHg@m-5LtM-nSW-56Ju+EkzV*0FI{&YqmLyY!j4>~*OhJ~W=oCFs8>Vo$H?GU3+>5i>uoIu| z-o4uiP9-beWfxl}(4KW}dONo18z=WSRJdQUCwg7BTFc?SgdEx=VRu~i8X!%8e?K~? z&XOX1&alB5swQ}r`}+74|9o155Kp23bvNM8(kNdUrU)EflD@~|Y z>W^{aEE!So9|Z>>m6V)BtE|m@!76YGtmsn@)0G;BUwiutfYT@|>IQ`(urpL;Ufix| zZo>%aP#{cW5Qa1ophn&M#AjZGqwW+OS?MADC|x?JLF=3~^qpgw=+=m6KTGa4*2P&}&bL>Pi#Y)4bo#u2Y7{Eu%26zmd zv3AzpE)p6dc7Mn(3c7SDbLt{}{bh6zqiovx#JqLX&AVu}pZ;d8wJlfJcmf)F@08>3 zZoqBuVyE4+cWA)-Rq*CL&~Je%GdhRZ8{ig4JkfPNvqMJaYK(3xv{~`xT28-5czJ$M#vC zU1sFQ#rFnlZU}h2++uaM3kw)Ht9$qBnZt+u#Yt{?Wlr{bGh_u9yDDE66Lj~UDLQp; zMc0j;TKW7j$>x5(%UQFHe^@INsz>BYdfCN~H^Yq3bGnODdC4*D(ctb-&RUO-t zu_!%8-D}fYQM@1FqwkES_o@D~UryfY%r?(`Zk%_(zgHx-qfv99kI5Qsak zZsmSQqA+jaloaPweP8$*o3@%EK+B7H?>7gym^_Sx+qqvWi+hA@DzZzwm9afv$LZ0Y z-p=OsLwE>2lgD+LMHI@0A6WPv;ffYY5M{oXEeQ^NV#3veFyd=4B1+cZz;ujhd3X0XZ#@W{iGeQPfB%wwSv*>&670omZ>f;Dxx}2C@k(WrA@(3be90nsCn|M z?K9q0;&rBu%7Jo)Uqp*epL`Oy@0MQ1lQ)i+xG3fnD){V&T=`Umxw&@Tt})z!vaPXx zCnx>|jkeR_(AvC9mcD#Nbf_Q}XImMc9Y1wv^cqvMBgQ6e^rrRk{xoR!_z>@o&5NzD zFs=`23G#|WLVRPLHDdXY?AC`U#Es*eq9nudg-1@GUU>F0 zHNd`0R$2482Py-d{Q-2Zm#y18D){czJ6%B>Mk8A;h6w&>K9RS!B@tO1O6 zc>82t`6LaEJJ+@KZglRZJvI$6+}5zW;x|YmG_;{mnB?IU&11j+eg<>Tl+)9^+>>}# zZgb@No!HHfzE@wcaA79%q$Cm__57rJBhL>&Fp}-bpFSxq^yGF|mrEor3}=fZZ}_^6al0Ay>soe) z!uiLP+QuLWPxA6SRh{#?WXYBz$_ns>>SsPB18#(d5gOIF7QbMi+Y{R_Hqc3ws4Ri}3^&qHaTb_r_b)k@*EFNVvm3wJ&rz}1DjB3WH^ZUT{@P|2N=Wh_VG z7Mfm`WBvNUK3k}O4={;4&FGy+Q>{lQmFJ!Ynktmb6Hcw@t7UaqsHd_{trA`5@uYZo z9JGZysS`A@f1_w_ZUEedfL^_L>@h<-&RiPqesg4>+qEv=s5OE`QbRhL|LR?|6UmiMiPnmuFZj8 zq5;zBY3cT}=gz%TvMsq}+enri)e?1x!^1umqxn(B7rxoEX3Q~l=vYr80w;f?VdTJP zZydLwBKHdjC_`oGul4tCkeQ#Hs!RQd2RETrBUKJP-jRIdsEL(BS4G#)hfZJ0YOj_= zu6(&oS^qzpKI>O8tAb|jd`h3^X1?KTuXWS9_m;n@sAz*0A)G?n>mn#t+`Bw$!c;`y zvKu2uj7XzW3qlm&^%ihe65+vwgoN2C$~SPhJ+8~C8Ln%InxWNd26fv9#Yi5zdwjgK zVLpSX{DD0%O)jAgQaKr&|4Nk%30{5sRuntt3+}v#LNBl(8Kx-<;KaI9EB!R!=O`M9 z@(1MISJ6ewH;?Q5ruF^k=NXNUXEe^@VHMhdQPC$I+Y{=xw-OzHDK&lJ8=sj!XkwO2 zDoAuiS`q}*?kAC2t$TN0C@^&D&1-K_$i}NWmQX*N^3~s&d{j!oH9J(Hxoqx%{^=Y( z>!+0c%S7U9fdM*}LMwB^O=}K-xrfr%k_k#;ch!U;20RbQYHUyb$l?<`eg1r-G74xq z!1hSfFB+`VFLuIj9zK3N@W&t94jncuh@$^u{ws%vy)3Rt%S2aQz&*%^KfHf$DPJo) z)N6K%)kI~{FE!J>)6+gA?s}9k7W9IsvtsMki=YdCrRu1?cv7&{<;XJS(pxxx8CZ97 zvt5stl~1h6?Q&yW$QUVdcqJMKRBBwxs;=;&yK~rZaN@!^-ycHcoHy>JW`99VD=7`=QY{hllyo3SOOT+34pChu{3Qn;+-{UT$W zpU&Rts;c?d_j44g_nSVrHZA5GRUX(#B~MksqFcspD>;}QZW>M-Y-AgmD{5OlqXFXA zv)g&d7Pb`0hmRa-LHZ>0KYpY8u9C~BM-Cb^v$+Bw9zLEvL35Sl!HeI{3;m$prcIk_ zY4*mCPxt($=#aN}IN$?EKA2YFOj7=B!5#dz$^ix49eS8`v7RQTU9vzgkdN z``ClcINGR@i=_fFp#$o3q({r<5}D=Jkq3ZRt=W-VC-v{8rHm+<`+%uHAm@RXgY*@~ zUBwDE5cpn@zUn0M6Zr{8@bYq_G=5{Q2+k{sa-i!4ez+d-m_YCbesNkCUSV({A1UK(JnS=KRm=OMBk5 zp4zgmZsq^8pNo2^HY;Jqsj(hevlTvlBl6ERnU-6;&&|E^DV-HUR#VsacTATqTlO@C z9aIgDt}$#FrBzpr;lt-rXe>7~^HX0ufByYTmKonh=Z|h_>Qt~$n&MdD0}87Voo`}2 zoe;#4&oWv(?dGm875ejOo=DCCbY+45{?EQkLZqxdbm$&^qEN(dKpq_iI{N?)+hw1f z-8l|Vj<+>6(Gh*qwzU8_I%bfOd!O#V*T(%O*zzT=Z&iY~t_|T$Lm?U||GNtHkauN^ z?~eYxdk4_ir(B<-s8~m(VH~^dJYmW@xuKy!i=lX`u3Tjyu2(6V1((`K`~0oPL6cP7 z>SrflW$ZT!&yzIHmK?#l#{c9aSIedX9BtLkXAlv1wcx<&ABs5n_5BCx!tF&Ku_KR=}kVhXwSZVIa^ly z>@K*nWfsu+xKO`g^8_)&w=(mh*@6X5%a$&cdrG95gS!}5VvRh#rIg#<>&?orqTu_U z!8I#g2Ut9NscpGDX|uy8DiN`rTGn5&_)l}^rq>^9{*?Q}dv`jrqOrIuwX{Qke#ETK zS6AD&Q3Ky{h7vT6wmqe8o#F4kyDr09J9tkCP3fg=CHIXxcL8*oi}L9j)ox2lulDot z(c|P~vm$Dfia3S(r=~39)LTj!mjKqv|1pMrTVFgL>$xmYo=gTp+S>pGG?o&6jpLnT z)Hm2q_Id57tJd&H#npf|Sf`(^W5lq9$JYsZ&rOk;P48zkC0$A{!A=DnTpIdcA)2hK zOS1~MaMLwcX_y*6_(ZXLC5|)PLUo3jUnAS-{Axng2risyUg?Y#%a-|Z^Tt@I&Y#*p zyK6|umIYOhb$oLc-f*->(lx)AT)nFa^dtSA1(LH2MuV?C23T#Np77n{?s))`AQTT+ z!dlcbdhDu)_NiLIA6}6ji}SnOFqk>>y9pB~0^>i~8nble%G$3|dm=*&$l%v`5M0W- z$j1MuhRWwJh;;}$KR$f&fPVdgXN}jR19y+NOP%G%A%< zLzMB{^Ya_y4xc_9Ml&~`#QVgHcuMG`l`tRW0K<4 zfPi!7EFf=%w3%a$Bjsr0Cj-!+m*yliMmg+J6b}-%f!mr%`hM9Df9%*@)3IV*XJe^l z^ue$5BG&(a1aU($&}|4PlQNBOEpU;eg)6i`b@d-tSC(4KD?d zVYlAb{}C;T@zw32Ac@u~pu&rgkUZWonxF2xmaR$|Uc;@(p~Ia{(>rpO%Li*~AGNl{ z5FCy}PDV|1fOS!P+*;>1v)i?Aul==(N0Ki9rUrN=IO7Vlr#EsEx^?ZkEPQR85H&yy zZ`67$4)HlVXTg}>=O3nMyDJ)3|KzQI-sFp(J^!@UZdqkFy9<4bn%eeh{;gi*w!as} zT;6z$E??n!?M=rsrM_FH34iW{jQSE!&7(X#VOz?5jPb+r-8cIAL;|gyNeHNm)W8U7TaXbFHeLEecjTDhbj!t(@DMOMfe?ZDJHrUJUyW+Ngm9Zls z&v)ptUzVp({V2stYya$sii*(iuobJ35m|At} z_EGzH$Pd-)j?nL{R64Y<+0bW84Gh+KjvYDjJ`!qS4VaqI7&YqdS$~B=G@+lm*#F+C zsb!LDsl6y;7ba0HSW=8k*HSzc&fseDB((TBLisAZVjgPMGahYATl!2tWW!#g+8}CckFon2v_LYFY2S``wSp$ zQ=TSO)(fsP^^alcsF#)k`8~TPK$bxVK@3Tr$1eh|aAlDZbkEu43Qx}#(v-AtxTY*; zG1|Dl*Gx<6LLcUJD?L5Em)08>9$((IoslI@Cljo)f{swm-Y6Q&5B0pnAk6;&YMwk{ z>)4l60SeYr% zhoLc4sW~4UCV)RKBh*9LE+bi_J%)Uwb2v*K?C>gY86#lT*%yyp}9NsJVN^)fd zBhw3RE-Pw0eLR2YmovMC4f;sh*0$fC*h1KTKF?~v%i&jtj5*Q4t!~4y%#p`EHqy7x z7NU)VP=#}uV`Na!NFl9fyuZEp$@3iNM)z`h2H8nm?*7-fE_S<-G7BsR+STi_G6iG? zNK?OY#)lJ9$Pu;gJRa{FdmO9`%522Yp|io{HXQ1p(BFIGQ5qWDtgAch229$l2UnpV z+SOJXSU}7HY(QA}!aWwP6nYJP86~A_Eo<13JFd&C2Nz~>l+WHwaq~Yzhso)yIuO1l z<}8qix~}d+Y4CmnK2t49a5bf0w938cM{L>4CGl%CG{%lyf!?6qCPi^N!Y6kCd|)!d z_c6=Y?A4KD$L3JLcx6TT$fh?zv6D@1Uut5qDH-fPc_(E3Fe+TcX~ET3hfSWmim}B< zvCDi;{EO!Oollz+uq=4FIycK_uYuZ-%}7``P7|Uy;sDSW6+OSs*@dsbgvp4E)eP#N zr&QQC-D7>9%bx83^@ASI#2%c2zB*&)10ati=%!{^mz2HkFjg$S393-Enm?p?XRatax5C6 z{*&^$$Cw#Y!y(O^U+&&1wXYW$w*1aBYVnvJTOa8vGJUiE=+i!q46vcN0u_l6f zU7YztdrM?py|gYaVt>-fnraLbfKg?j?Mw;03cXpeU~2^MA;<1iC+?)YQ8)4rF;tqhVyI(U@8mvR|6q6j64Z?U%}?R#AmaRANi**SU{hSXQA+Rvz$s+g0^HvZ%r3ZqgQ^Vy*x_8{XfFqctp>sJ1z{CR1$AlE=O z7D^`G`L`i(9zw%{&LDpGZr8OH4NI)U0(2aX#b-~sW}yXl?>ljCyNRtplX-+r&rj`G zDocg|LlxKTT>N~5L5KK+Gd9S=8ef+;9`GKk_m51%kxSt{H7yVB;WWQVn$c*;Q!m_N zA7|Ui&2B2HCIPvDdW_WmEtQ*b24|z@-D#IB%*P5-mm_7j6q!Fml36gQ?w`76=y@h4 zJg4ot+`rpCx|Wrrr%qjsso8}cHZ%xL4X#bzFSlRa6Y6{ijQNs7*>bczMrxma>SLMV z|7P;1Jc^3rV7&iwHJm`>@aTcF*RAtWHt)TAC%s=@?{o3u1kvOu)ehUQnxLuto@+14 zo_r8l2YzePyFn2RENOa$8cOIHA`F-BIoxkkyZXS>Bi2bNg2PwLeEj#U>ZQT!_~({a zh0d0zAOAW0$8oNw?uB0xgl5&fajwA{Y`=l(6k%-N%((+yhJc!;B-H-h`R1)#Bc+es z+UrG5xc5WbXry!BBu8(`A4ad-wa>UjWmBYZ+q({IYF(lNY)7BzGll@E3hdJD(mU5uL>DqU~SGXEo5(3zwlR__}CwdqmZK;h5 zM-f)(EFCWYq@8w>26i#0vFbqQPMyHNh6ZF<${FWEt{jHS;vUnxb5@Am4g+l&41Pg#J;99B>B$(d*VER4 z-6>U#5rIjmvXkvhf*aKgP+D{m-tsZLRWh=*%v;g8uUYVsDP$Z&V^}CrW)LYZE0r0= z`OJ@H#qHYRP+n+qkeWw`yODHEV~#DLRtyRY+x?=FEia9&u+u|sKiV8f+ly?2UqayX zifdd|+z;)VNaqUTcI6!YK#m30E`1nN>EX1yr8|JXm+k-w(5!@E$XsA3g~JCiveC}> zC{^eN^{WZGRH;bLWg- z`%cW{sBbE~mV0S(Xw|Fh<=!suPSiopA{rOU#Hfd&bRHQs^USOc3{%a4@K{2pWLfxX zyEQ2J6auxx&sJ^PYhJDVNukIL;2&QAuK4Q*2*HZ(D&o!1iosA88dIj^RaFXWM`tDc zE;CKnES-TN38U(tyWI>-zvR+X|GGw}U3-OTI&pvW$_PFBlVz4Au6FaW+6r7e{ry)Y zfQ#(i(Kes(Ei`xY1qlhEQGH7g9rftmeS@KfLVuo6Rb$tND7g#zbV(r2YF&J?pOrON zhof;h$f01Mt+z^XD9xNV(fqCfH*mERa>LDkIA>6#YWunk7%~qij4Ju4G>30So`Kkja9d2 zSm^sf$Qn!2efJSoB*W#tdN;hTF!9jHkdXAf{f15m8Phs(W8Z_?YsRK)6nv32;Wr0a zGv&+uP|CP2)4MBcI8ch}#@yCDm+cfpUw_dZy{m;U`kG0rXfXHxLcNoXA>K3S+ZUqH z&!B=rYw#^kjF;XlW|Qw@mK16i|9*~2tO=HfUs2}^?LecTOc4T@0+pr<)3YhDdIb+t zN%1?B-75bQvD06W?9jO*jgB5Xc$S&*Rdj_@5EO>PhBzg0i0cAe>xQ>$+jixHP7!ON zJIVQtDciceMBin|CkxeRr|CxULjQAPs@intEAXWo!S`#oa1R8y)e@{vJm1m_#37QJ zzi{EDX2`czBs)+vT4ut>ASkz!qTn_~OBD(q16$DF>F9VxcrDPeq}}(}6e8gvXhwzk?fevpBnvQ-lFuY|+wTbp zO&eSnBN)tfyi~j7!dne1#JoNof9kl^Vr|YR?WRmx{1N%HaQ>Ix_t`Sf=27_B`uSuQ zy+e+<2UPB_RSw)2|y&lZNcO6R0j%;B5jR28UhP5TF3VWHd z7pmUgC;j$6JpbSQ7I{w(1s@9XzjbX@G;#CvuGf=?4j;Z+sd%%9=R6?YBHH*8?Y=(9 zQyhNm&QllX8mGMpez51Ti}K8dsB?sx$saUdzqq>}DX8F{jZ}P0e=wL~BY$kRuoVg+Du|j6QQIgkX#AE8=@let;W+ zgo)o-o}z)JNYF$ENjmPV%06LuC~W)72WQ|J8jHt^h=(JU1rQ;Sv0lcZQ>Q{bW;8xH zEcFJFCOYu5v~Aqa_$BZ1_Q0t{0jSL_VEneo_`rewlm8yDgDzTh+!U5|(Hm3MWcL+r z0r}N}rG)*}D?pu|h9m{KrDZahrvIW-LAx93T8KbrkqiIPVPGKMW?qyLCERylM1g?b z)JJ`UhDHXLt#5-{GAfL9u#S71tkJ1s#|wdd-EQ3)*QKE>p)r`pptB1-cqZqnY`)f# z|3BrY+gt=tX8+}v2AVnNq#s^V_w4Q3Fv{ziK#SPMbk zQ%+5-`Td?ZNJ$ynm@N_R5PDbT=l%PQ2*)A=06Nt*+~G6*Wr}*%ANj)Tr?u^B-96;@ zEoVf_-TX@YGqtp(hft^<`*ECYx7KcT2AiLr%yjw<-G@SNfOY=mPaSscoR zm1Je2-cqW0Spf+Jp6mw>K{~hJ1tJXCR1|MQh>1EddDpuTQ6NZ%iUPD#w{Dkx)G7VO zqMVW)L)jXa3-vBSWNF?Wbg6KWD31{4+6MF$NF+hwS4%#joKWtPjb>(u6F?0pGOEiy zJRhBWCpWW|UdE?AdYnTIK!tR#>cr9YedDO;s?{!2<|vpBf*}b7>*`A-ua>i+ngnA5UPrusqS-1 z>f%NYxR&A&q9{#bkIet1a>c$11mrziwujqZ^Zyo!xZQgCgMGV^AK8`Ux z=i_Tyid@2vb}Ne#L~j7iF&#r#)a|-|sFhWB(9DuCq$__)j)$cMEo)BAip2RirKM4l zFGaqb_&(sY(|o`HjPeZH=(KU$yS~p}@FGzMk)>aJm(&*67Uh_#rp0gQD^xFk4(_jq zAWbf2k-{sIJt0~NQ?hOr%9d^jsImtYE_%4_8ey*P0!! zNyEBW+?T_}s+3;=(5M597R^B5YECa@agalU?~tworYr@<`bQqDx=)x9ih~Je0KA@< zc7z{kRX>6imTh2Lc+E1IyQj{x*MksC(7dLtq&3jT2lvf^HV`7v%APqxiiD1Ynt1zN zPKXra#JG(E`t$I8IwFguP&jnwDF_1vh5zfKgeJ%(oM@VvB}!yeD;3j&KLalTGRo$< zcdHaKK^Z){Yw?Gvv=-F8>eLlHU>uHUEd(&5coE_OUj1#+K=ByK^1%@f|@{p zW{#)8cz$}vlU392?rmq~9k+C9rg9ynXqZk{|JaFK^ zGp*Y@aA#WtcVWO2Qr|JDzhEVGHRX2AotZ<5ZM25A_Hqg!IR~(ibTj|3kXCW~>IQ)` zX8}uzfMwniYE8i|(#ctcyR^o!F=#b*HF-Jbh%5@FN``B9*gldi>Jr8)rG!#x;1*V8 z1H$78C02#0%=LXfzRM;Gi3!o`QH<)ZIf0;B$9~GbJNK+PhRT=T;B^zb{IK|T%Wsdb z85NZIJZFKj`IuAv`{=h1W5XzC8tR|VcKbojfHs%BP zoRVzFZvMI>Mtlv0x`L3`B0z}9<2yuwFGJ6N_^wjeW}t4IY|;}`6h4m*A3J7^2nFo@ zr;=-b?+~sC^luq8S)1rKio_j~Hc#y+1b5n_!_{PIP|Y#69cn*&`}S?s^BGOKhe@w1 zsJ<^yaZCc1Gp0SSIY74B(&b523(}qYXlt=0hy+&6` z_jnWrX|#nv_A7m_Tv;hBXGvvm>U)r@Wh@`#WW{ALc2U?*!zUCRjTu%14B`{rtYb>o{^${JC*-flLx4mszkG_SqSyx^em zRY_IOS)1Q|s@SdRdn+_(llRwJSqINewtp7L8SfR%F{ysqtg4)7ChfG9RS@S2fD&s8 zUvQfj%#Ql&f(^+tkvgndeG*|zWCs;L$L#Q5ueRrS*3fToA%dnwnveO1cvU@dSPdj+ zl#&Z)I4%_Ap1DH2d^FdAZ8+!rp#8fy7$HM!>kS9{Sk|(TM-Cl&JV20Pp6BI=&(_72 zn`+$W#5BCzI3qIRGo`wr`hQK5DKf(W;$Bd(plBKY6a}|aiQlmv{rlS}530{2@TC{f z%|-+fb#Kma`)>U$1b+CAf{8{*h-o7=<{^N^H;49cOR)Sn=%5Q}1RaofeNJ{T1uM;% zzUx?583`1KN+O05qH??Ck6^BuxCDth%E-zx6O)DqsgG&hh@ZEcC8dJ~eP?~U^lOwn z(!Ro#9^XowW|G^QAq(R4a=HmJpaYS5-}HbKb0hMaBnjFUjS(XPPv^C%PI5>v))>=! zzgb2r%^_ar5Fs4?^qC!Dp!uaY@%cuFmL#EYU+i|n|Icx*CK@~Iy?#(A%o~U9Sh}pT zC}FWm|4BZyEw}=p(I#CSJ2PuvDJ{JJo_x`6(|TD`;`P#wJOHJHo?nJ1OyZofFw!_C zGeE{Tr7NScKs$&~NzCYye{AyHhG`5La&9?=@;(0v3eOO_H4U~e717cBSMQ$Ds@7p$ zAh0SLlV%LF+72oe)L?=cyP4bMrwJmW7o0wqrIyabHC*7-8S>&!4$7<)v8q~Bn)cM5heNdTBKtnNriKF?tDMX8!?^+ zC?;6N!69seW}K=;d5k8dx^AQ8FH{@h|%V!_xIPmWuE?MU+5{#1k zn$W@7_(Z<1yUzS?SDLidi`?*a-P<`o_uB5U^e69v(kb$Bc)6M1HSg2j`&i2s34HqJ zsjZt*$3%f_*IrwD`HNGv-<(ii9kK2^tWic=Q3sT+M%+{A+T;TM;Ah1js4l9m z7Bwt_RrV6XQ8`)EbKPiysVj46D_Ap~1s{it+GV14ZJ9jNEqp1RBzW$gP=!w;`BhFj z745jE*rHGmn`La{DQT|aeuMOjX3svHL%m-u;(6Fdo(yM0$l} zixDG7I-Pwk50U3jQ^7B`^UYm{6V3ouP_!CSV#^S#5eh}dnHT1iH~hxc3zVB05F7Nd z)ysNYjDzqli29F7%nCCCL(0nM6^fJ>55CWtpxgrd*PREymam-&_NaU{JzcxZy>RMk zZn$ZsaLq_{`l>Xerb(s;?6Un9^dny4UP=^Yx+h1(f^Tm1>(sStRK$rW z(E@?_@i=BUZcYHo4;aWU^&;e@2J1aq>Iq#+O11->|8&rK63P^Qxaqb7bzavok6pVZ zATS8u_^0^WaNV`2U;H>9s?|EAh9S}tlum_Ppf{MewgPF|9rxk&Re$?vICFu$Wi=o5 z=-u1;_Q9b_ouSZB6E}8yZb_g^`xcR-awgjL;kc)?b~%i4g{Zene9me@$&?^d@P7!9 zDHx+N?zKZi;P*jwl1{zP`myr^NcLtV~A%e~HovR`T8GX+7 zt8u4QTW71E4@#fg`OLL$wGEJ>4FyFFr?5*vepow(KrJjZZ9?Hok&mH+ zq~lvD$eMAax%-a((3_?Wr7KQGetAe{f@N+i3AQ|w1_}+EqAL)?LZl={Fa+vwvP?_H zg29nCLlfrQVbE&vOMu&x{sN_a93u4Wr$c|agQE?0@fGYj9`XRi9WORf_D;q zA_IBiCW3!r*uGotNjz#UVl^!>jQ$-9Z6oj`(H$W0x|5n3e!5@lY}D=6;RP94yRs0M zD8d&#YV^zHih2Q|972 zXgWDt2JTm&*q!}K&i579rg)jBEl6-J@&bGtz2=I6j7V!ouC=s0KmEhYi5Q?H9geol zn8O6|{6nTaln_M43o0JQ{f42Je*9e}f5UTv*)Qmp3s5K~w&s7AphM*VzxyGn>Q(*< z$@7kmj%~CVWq2dAI5H2G@eQf2We2vay$qt-n=Fc-zSqrH={7P?Ta(Zfr!Ic^x89G` zTqULeN`4wry9EEw3t^usR?2V+Tga(6CU4%MVU}nZdR>x_SbB?A3eMmgqh4xiVLC<5 zhd#I{bU(W>?h@8~tYz9^CN+d%RL&9+vx8s`N;&<7TbLWDW>ReRgTO~vZs`NqEF&a` z#t@e*k-^&hBG90ZAz+Yj7)pk*z~PG8Ts}s#ONg9pe4?h<=2u|KCYm8BUC*6+!gz>{ zw!_N@hN3@2^#O^&c*wfAmf&69O%b^G(5dzF4|KaI@?ho5=L9NK6aAJBP7%+zNSte! z!OdlA5Y29>KYtQYmU4P@-2Tfj`O2fswu?*ZqN@2dXW>%N7wp8(Rg4eoFv2cYVv(kY zqtBiDN=8PJNvnJ8-=6_{-=oK$`p4w$-0X;h1X63-4wW?v0hF!z%BnceE4yZnbJDmj zRR7C33dE3I#p6ZDIAvQPf&1P!z6! zZgrt`g<^~-BvvB&LM+w+CW|sp{EF%Zc254J=crmX1~C2zOPj zU)-|LaG@z^wR!0D%VDu5M+L%Y}zLtlk!1%vH`-S-&b~&8j5)4SEpyN%GVyF zGbtrvTn;m615gJ}n~j>Tx?xj0BCjT&P&u2hU7kn9=Dq6!DYr<=Ej;4x-Hi+n0N?ym zl|dM4cwd?0OSlm>jJ|Enm>l#&6axTznZYZ@!YpOk&+^&wZw&9XU!m%)w@*L(-A_w9 zyY53y)bdLC2?FTIpDn+u7_iHJzC*?KS6v$w3fKGy?LSoe_O?pCY+De-mU2{*v-@0Y zJ3#ttZktAWOKkC5@w`Y&X9)~A{X)b{7urciI`%JJy7b#*M)}33#Lcb6QA44Mxy2Ge z_L;WXS?mnTA*>-j722C8DlEjwgqiJZ+D`GZNZf$_LW_l9j@qD<@sKcuc z96EGJ%cX7`|IW>ULXTF*0qYTCd!=q{(a8~43v87md&O_qr_u~TZTLGsys||V-Wkxp zHdgvoNMirH;F{Epv_lsIdn!z8m=r18Rv>qoWszGu_sa@LFN%a>l}w~-L4j&W7cW8s zae+e8^Bj4oG-ye*992VBpJ!%C?+)!t&K&%X>w)rOh{rN8ZS;)-Hyw=~X*Bb%sF+T(Tji2*laX0*%)DALybH%DC7w`brF(RXB6V0yrc zrRzd%=z)Y1t?^IdBM+Fo^{Z6f%7ucNDY|Z=rbYC&l=?~MD%hsXcn_Mjcjbe-PE_5V zGhE{-tE9Z#SqZ5X3Qb2&JAz($4>ECH?(-}x0>!UKOkTM7VdSV$nkj@~3yP=Gf~(P| zUEZzQvv;p?CW;bod~l$8WkU<5Hf3|MTTpPy{?SK|9!1dg7N%e<8iB5bg13n>H=* zE>r>V?z(AYFW}~pXmNJEdpSOZ5nsuy;QXHT^(#vyD!Ui4Xb#n-%Z##P>^D-K-#t4= z;aot?d5Q0B{9f#SKH`(o_)n`31ZqMp<>blmRku$Tankd1OcF1&By8NQ!%CdzW(RBVW;N3HXo7#UZsw zn86GY!1U{PXPTSEy9u2}I#d_>HoR20Eek6QUOs20-$5DsRHj_MoyD+~QL<>f5{r6t ze-fbmN4J78;fU51wZA@CH)D2(tj0dWCQOh~6@kumrUJDxQSI;M=C(Hkct-Vjfo%@z zp(XM9odukP$mOve?BJcF9?`BI2dP|k>X<9L3mTCkA(aIcG;j~84} zU7V+(So{R0Tc9edr>NPNQwQ{m|I}?59xy)3G4eH=d8qFB))?Fv zI5S7OE4=s4{?J;g+BPdsZEx+oEGFOAWO7vMK84TTcyz8Z}OW{Q$vk@Zu4|b@!mkT^{DM&oe1^s z9*$zZNz7x&f1|9j9O;Ei1R@3QQCCXRQ%{fHcVm5v25rMpAZfnE$f;97`DtTwWDgG* zPyouGsLGT|I?01^HNSzn!_docpPC`zpBP} z2~Zx0Xd)@Tecre(#0DY#$PdNZ%RO^o4Hf;F5 z%I3@CLi0DjQrr-Z-6;J}%WqYqKV+%Jc^Q7bXjtpS8@WRozEGdPzVY2aUGtA9b${&M z#Ez%iKHpZVM2=vAOKH)@DWOabYwkJy`;yX>%IoXS_KrZES`VAVKmUC>UVW7)&Tphb%uizjlFl*ohO#=&=tjf z--)-b&(UkZQazKR<=SJEV-0mV370ANH779LkwYG#k2Ooi-a}AZP^M;cFqL(E_A-08 zKDK<-L79P}SWFh6ZWSe&%z>AW-8l`TKlvKs8FcvH*<~hlQ1ckC2s|MA|Np1@RH3^(hdiB8=QfJbV3~Fy7^IagF$`t zuL~S-wzyfGT(l9MUm;Q=5LYn<;9E0j2H_{%FJmQ7pC?`4h~$#sU@hFby2L_x1)r_4XrZ$ z2m~6_rWH(8gI>nn_}lfqvGz@kF7ju_91G4~LxUk@zGM#N)QG|v6rSjNtOo!3!5&GK z4^q3MKW`@Kgw!W4At0Bbkf`ksRLDLC^hYRA)#*}?Ka;Kjc@5q$y4!Vp&NGd+U`$Vq zfV-POTpev*ec5n0Z|Ik}4*cWm=Qivv))l&`jIxgdBXGs%t<|C_k9*5x}K9b*zCHX4PLSnw@Ja`a^* zxg`*g7f5r4@Oj`QrgvNk;Ro8~wbJZIViF?_c#&nRR=rQ$0F0%@IbpVUf2AE-Wjg^L zF}2+Bvb;Q8Vv7KN5j~5&;uCg;qO&X9{b5tPb`7t}Uby+c02?umvO2P(KMzpg8Jj1+ z4wF*4uAqGP#^lBl@5Xsb{HV3tEj;;5MJWtZuCCIl6Q&ArL(rlty*`T*9ug?z`&(|v zh6WN?61*jnEMmHidwrC%qvokM_NnBMv}4pd>YaN3kuh9+eB4Yfx4ax-SRey{gbV9U zE62`jQHQ>NsI0qB>p@v?F#gf=$XrNPevRl3QR1+$=UN5|s4@LFFiCO-8Isuc6gny5 zUo70c=W;~jWW_-3+kO2D(;qGtLRnscGuG(KBoeRU-?zDUF>pI|bFHG-u46=g=Fl&S z&m@uf(Y-u8OinnVJ&g#hzZ&NSv8geV8B3Hz*=2vXi~qA^TdA10@O;&0?0kEUo^Rxh zaVfpOsa5ejLJgWQqh13BY%=tUwa4Gs9gQ{wsZ^B&ks#1W8FYXLLB*|bs zv4f+q33k|2pklJPwdRhVwjS@E9F~#Y-G`@5_6yp)OAJ<9UWcCDQ0>^lgxG>Edc6R^r)cdX=8W4>0JFX zxAMCrIT|21_l2$oXm4-(i9+v&1+q7|Q0wGHr*+RWzbm0w>^a#fT4qGNi~e2KUEm~9 z>)k)ho?eyNK5zygC$+kNh$v{Z6=_%SL{e%U5HG0jxLP2i9;j+JDJwtCjZ`}_+A8OD zAj)KzObdN=#dKjVF-pd>d z8rHJRUo)uvtwTG7MW)*GnsX@ZIXDYf(NPn5%VK-ccl{}YbG>_SOa@T7VZNyd8v^u0U7zigw&w5Gn?|A|gI3zA83Tszoe5 zcx11rRw305H7nQj{>6XfkRg9+j?vgK$U#v%2syB)j@{|l?XRqoyWU9S*=?&Ro~g6~ z@`(`~fdbktnIg3%UstSZ(ZHjVRWD~!*ni=Pw_kZpZj)F<@Uc^!+P@p_7^^d%B{6E7 zK@vemddAouS*X*isGLkp(V78oy-Lb2!lNI3xxfd~Akqh$^|{r3#zr!hXwqn_vSUpr zgMV{u!3P>HWPLa6xoynq;pNN4&k_K3YR}l!k9OC7_wf+ww)${GMTu3A#O$KFy)$wS z^p;Y{5M`uMaBH9+(3hoj;cuxVzvxOk>13;gd#7? zQ>V);YMP9x<=1>+!OzFW=1EY^^br1=XXO7t%SdQ8z6NQ^= z_ECnNi_@_#vLd|$z)g56@>^hz6=iGnu=zwU?Z1W1wm5z-gW8wHZyVSpF8d)xoR(do zu679A4`$O#(B@F9mC7Cz7Z}nfZO*P4j@MA0@FlfyU;<8A$g01s+*KcSg8RLyfYjE? z^Vw|mdUP5$pAXZUi+$Gmu<%C2X~%lrS4ISAj38N+HH zb#TFrCk-x+Z~H7ogoRK>BZm$1A5$jF83NSj#1+`$GGX_OrQv_Y^-O6{WAZp1ND4jg zrUwy!9Ht~)K+1^H_E0Jpi)X?oRYBC1;@*;nBpnrnoo9kPi@Fzs@sIji`cqo>+4`+~ z9&u}nD552~5|yhlISLNBd#z&CvSmj^`G^SC5;#(TvBkEftrUqhqF`lMeg$NS&VmIC zGRnde?(N-~xFI6{mHa7g%X^U`8y|Qh`O)T;54u=5LYDY%fw~93&ei)=e-RFGmh}zx z9TjeQ=&WRB0W|mc+NlffGYcY0XVh(5U-x90su^TnB6NfVT36gMdm#Box)cvdw^#%L zA}bJ^Cjh-$yH+cz88lR?7cU3iK7_LS|LKLvya$MeXp-qrlkeTzeJ)&TToJ7!ZYWbV zKuo~}5Y3u+nE&i4E8cznFPwq$+0+yf0x_i-6ptzFk#aBPzLm+CFd;HB&oeHctRA|2 z&4{v}K7v`LN4>K?)=afWZ9i+(=}qMMyPL!SS=HTOhF9#Iwz}y#-TI8vqtwQkY>&=V zng;;m_pw3jS!X>GJ?+h6FsTiNFKNPMapmcC_R<>TQdcGVcb>z-_dp*qG{f|hakKSt z@d7461j_@auo9@a=CP{?R%QAZPXf`ej<)Ei=$c9l!F^WNQKq3NHSfII?sXsr3_&0a zknXR6X+%IUdlTWhpMYM76-p`tWG<3UTWl8u?98AN7s_esn9-xNm_XfW>TVtY431}? z!LlsZv4_2n9J!g=+VkXpiCK$R^C2beU<^S(J^l<>$xr++xvHSC(PlnT%Pp39WEr@| ztJr8d6CBtTWJ$l94zQ&hmwDcnRS#jlpV9*_Jw1E5GfQ4;Zqq;r5t!DqXrLWm(30MEQyG z*ts{XOxUi9R#F9`yA$i06sd*S0Kv4qy|k^@P!E32UWVg6@fP6guhdkTY>HG4q$v_} zM97XRSXNw~D@9a>GujWqEP;ounZ-XD4g4X0OehDkjf6fDO3svLyo>PXqLhFrJWpfZ zOS7HA$F{00XED+}8Ay-I4KukahQQiCc2Jwey$OO3A35?gMb?%#U>?H45rqlk2pP1n zajTXu?=_>d#a4p7{$rYv{hnk%AvyrIs<9^X)J_yVvow6i);i6T%O6Cpt!HG9hz7yv z@rwj4OYR%#rPaE4zYOEShaNfVsq^bAq}=9!dNXDmjcsp`pp>s69op}yds?PZG2mu6 z0aoOHYa11sqdE2zG=_8sTv9S3I=>+;oa+w_pgZWA27XYBVdG$Ok-RPCg(+%DV^BH( z6MkV;aBuLmjbOmMoeY%DBk3F`VBp3o#`?5Y#@nwpTySdbo)xt#D>=5Q!rKVPjIa+` zv1^xv9(dGB$nWNt=AegBE&p76q59{J_jRK0nCaFVC@kLE9IU%le`U6}m<>s3#vtgy zA>dJ1=8D`n^oPCRb}Iyt!6UA&_y-m2Ctjs6G%mhB@9%!Gv+H^`Gr~HNDXBLxZ6eVF zrA>>OqO&2jv2dFr2!L1#jVdlizaSTWfLzU?$ouxubVUl+r6_wkQMtjDs;n7NO&=;O z5u}tagE<oH!!#4;_-;X`yhK^3S_&?p9s`M}~RDwa2u{B{b2>4!CDp z1sW3l@-x(@LXvnODEg>@r$$5$NgRQ7$*y`Y#yMgKO6RIQ6u?LC)DMy|eJd*oV7ZYF?Yrc`u=?h#3|Z98Z-FkL2p87dO3-Ml4aYferx*Dmtg!WGwt6V1Zy7f5?nZMAsO z`6-twT+ilwgwPbapLll@R*#pX&1bE;IsE)dn&00eg+YHB`&HW!y2VMO<9qk&<&Tb~ z6ezgxCI2O?H&1D|&1?eEENCK`#Fxp+!Vp9U;)W;}c73TRl8av{Kcs~|On!Qf1EZMf zv}hlvGlFg*#>#w0ZMGQ+{KPY+b_I6hQCTD7WXMn?v6|Od8LPr;I|Xc^@{_5yjB6+- z0nKUbHCt*hW&ngYDZUNHK3m{-fShZ&><%H-gsfZ9r9wHk$4_H(k1om2`Gp=l8&&RT z+xD6vN^vZH0Gte)cfSrB%8i|xW5$k+4(zC?1!ah}mTLB zrHSrI=B1RbP6A^iCDHOz)Sl!TbQf;#f@;=&Dglfgz{Jx!n!ps9>4p8;kgWP4CX$_O zNz4UT)_kh%r7w*wwP@~f`YeUwwA=bVGPaHI`3SNJ=<|KM#a#*9UMxqDPQ00GlCf&( z(sN9bjikC!V)7JN2x~WlhErM@lB7f*KW_=}j&O5I7dVNsaqU0Y$&C|Caf-%!UQyw| z0lj+<56CESiP)>|{7207`M9>|*6S%M)??!`#dM|LX~Ty*J_;`a`~456&I-_iEMw>M zl&-Dyy4TFl`6Q}qbyIq(k%58FjE0iMb+6~@P+6ukaJ9S2p1oJ}(fHQ?A2pLz<|MYQ zNaA?{GOcMX-vkRF(cbZj2T7H~#5P-kmMnB91gMC!(Z2ki(B#qdtMo01P$>|_=LHQ< z(|1X7F9LtSki2ddo!{@uAg;|BqocDz93>a#O4*N9>weW%@s{9WGVi*hKNGQFu+MFR z_L}=4TjR2ZruE5dU8I zBqJU19yqXsQ}u8mie}nyvGdWP+dnRlkr=c+2-(6^6;5FxY!5QS%amzq9vCHtO2TKb z@L~k(OG4`nswkHcu$h({8ykmqbwWKs|3E=DSsxWT|NgT72*t`o9ug8#!u)+SQNCoW zz}P(JS6|b=NNl2Bv!=3P4_f;^RJ2vNmCLB9OdBy%;TS2%ix`LYrR%t2qHU=sy_eh!-aWV*Ouuh0he5n} zY4nE7lbT071{N+{=4HDGwKrV)G(G0~`9tS+SoP0kPibu*{15kM zoq{VX{En6-8KfRMdZszmr@9MGK)H z11P{okYN&K`IVUS5OuuL99OC~aj_IZ{>YKZ*Uc5qfHHF7&6=`lR@=5+>NRR*qShgW z>ON(=tG`M`Z}kP!q0%P} z8#a;&L5SG{`K?vV`|O=Q#q?Z#veFPh|Eq1k%?jV{@x)kGR9=Z5<`dPPc9#(Zc{wW3 z!{+*ZQ!vtAZU?aVCU0-83~!3wFSLewb_fja*~oS~Izww8O7=?O2hguA~u>Pcty z6rO%GlwwDP(&fI2+993>lWNKn22x(pR(u19DaS8T5hHEDU$SBv?Ry4&&$Y)$&f*&0 z<%?52`yql6fHPDsPDu!r>O{m=)~S_42wJ=HYn2DoKU9aihQj=ksjwwmwdy=Wz$JW& zsAr;1V|PhnPzP?E1Cru0)MuyUpC`hQfssY#Z}QFvPl5 z)^KEUU?@tIlu)c(EXEy&d*TyPR!f86R5Fw>n3G@AxSC)hbDYfqLm74&3*%GKm?L%~ zOz?}r*XOZznO^&6AE~Q%`dQG;KlIDezAX46CR?mbm1%%Q87-es@MYS^Tt`RL+VNvyC%;u zOdKqj2~G7YlOGc(vc-T2odoV8nkS`f<-W_6z%v^%mTUG^aWXZ94M^1MrQd-|PAD6A zlPu<}<;_k!lXONe;ofKb_ceh$x7wNTD@JSAo>uQSZk-!Pnd?Imk6{gF&Fx8GUfNGd z_;PLFq|~m76O_0qL1ec6T@oop7p2Q`Yu`FtCPcuw9G%e3r+`m~k9`%+(AoFfumf<<&P0^qT=lP_CHK%B)nOtq|lEtcK_UxOp zyv2lA#0~%fLtU;Ol`&g!7Vt<*OLJ<7^w|k5Q1oz&D?oW#+X1mAVPT)OTJxb}%Lb`6 z{8v$!813NxiNQ9XS?`aCH=$em?gO3}54l?R%pu%Obp!&%+Outp0H z3F%U?rKwm4#v!^cI-6%HqH5p)o~ONJX4iQxnq1LeArp6uWKW+Lhy2UjGYQZ_8@KSe z2eCjh2?8>Q;cOSa392410)@KRv@KDj{V1>i1LyczWRnW@?divtQyM{NCAtjb*#-^; zBOSizy)zh+?T#6^G=>tyG-^dkZ) ziA$WPkj!j_j1f-KIqPj8X9{B0>@*c5N^#T2&;a;wiOJ;u$Jd*H<(#j7{9oCL%piNr z*q5wXCLv=DMM@<@vOdaAA#E6pZR{pBDqB=4qY~1Bu}?(Oh?L1zNvM?8|LgviGv{3Y z|L^}?=Q`JzLC^DiKg)f;w;TL#@t)ELEJP7jQGhYRqNLyfnXPTo+*SZ~SDhmJV#^0m z@ZvO(SkwUn^7mH6Ho7(i`vdN<)o&BOGfAJ-!lw#ssS-7;OlzPwV}VKJ(t{VJb3SxS ze^t+X_sGD>saPGL;&{rgJ0Jv@fxMoWf%W02)ii`r$ocSC($i=F z*52ekmV3xy=6C+Ts-Ch0ldoB)q!c9yM}QPC(&~A2pfiBKYBn*R`;T*U;q9(|-3AX% zIS|LPkXuaD=neQbxRa%rRPz$>&}^3&JS_9+GAWo&F1MWwnqY3u`7^sg>sTldzyy;b z?3R;m@={9`fp7}JwITKbERL)Zy(~vTPFVq*5jHFBmU6nV)&6S=l}`KT8Aq}$IK2Q6 zF<0|H6se_~ij)d{5G0O_>T!n9<}cs8vDVr3B6#J5;7H=ks6XOYRt+EV_%{U8^}zq0 z1L-ou@UP3u&@EPae6AR37}PU8KW;?x1L3v4H|e~x$+y4!KB>;!_>3;C7H{mAwlq5W zP08nH@6+@DiV1hmnLZ#V$~|d7@cQ?;@1G`}k8Y=YqusL2rd_5t=IyM%Z~XT?_B7u! z#cJA-H#+H|pI2-!*=J_r68AW0eqP@27d3x5mi>Ep(b(fYhrd1fsCduV5%<~oqGvqp z5Ek><8(G5NZ{JYPZXi$yPH-f~hJMN82t33gH2px2vy_aBFl~rAFqvE+v8y7*wQZ`> zlZpR4N=b2qT%rZv^M3H8j*QEZlgGuyaDPHgO{56s7;O^g_>*vZsOTk$Mq zvq_u;xP)uue`%Kbwn}tejE0n)l)^|vi-o(HgGG_98=&QV4?ajtWOh%Hh5C`sK7Dx& zeVNF<%%%52xFzw{Y~H5$&zHIl-|M@mdX`*_1>n*%&UF)>MC)cz*v$!3Dc?jxwBuV4 z+sm`UhJSh4bpFDHKH1bGDD>**w^8cV6aOVlnOiQh9~s56uNQkXiFH#RnnbkT}8S;g4kwwmV4GfVNzc zOn2-Xc;wP0LoRevlnQ*$*6^L#IXUf1haflswje)8B6J`ap)%Th*N=%Ezui4J!)l?E zwsQ8T`(CpX-nU-DKUcks2h67x>MdbU@-q6_zNJfL+03-i&o=E~3r**EjSBm`eaXJf zqwp(9IH4(bdaNzb5vfkG9&Nrh$42HF^U8Aey=KqQtl?fH^r1Q1f>&tD|f{7H%vhrM>+Nw^ABd5y7$! zx!sO!OF{%|1xUcy!~2$PD0)R5&!b7WOE+GKti{LSS^8MVsWWB-OHN4uKjSW~^e8{_!DQCnxlI7r%Wzqw=gFTlqp)M@OO*KOir zIY`9>0L^detrnA7n>jY==A?sx0$yjNPT+h(pf;qlUEDPQeC3p1LX#8^tW3 z16|DOVbTyLjnn62BqC||k2~zTF`N3TrE&g8Q8ux*Vx6lN=}!%t)-cUM&lkSvkL{8H zB8eMav|8UQYEcuO`81XYhnXD|O=pVa07`4onpp1s^;dCqY0Y^c3&G$B4QRYYdNwT@I#lv1woB!tSDU4?8N1SZ9m$RbPWm zT2%xy@G0Kg{1_GgN68Jxrvr6$sJWB+#jDoN>{OMEC?Ss^5Iqtu2vF{ z#kilKtv~<#7OwKp-B;(j*eGaYX)NVL5FYT;r!x0+y=mZT=vMRK0h{z4(Ntq&H#6+f zl^Mx8=}i>dZF_UC8U)P!Ik_L7QGL;#h$3E4LV4NNflMx$bGSi|XIA6rZr*IyemF7h zxXxSeu0>xBz~7@N}MT6EC6IfWjVHPGqhW7$jkM`V*S}Ezt$8Usfg^ zv)d=O9m#=`p|+h><|x7vfsSkx4$w8}6%dxgD$DXo1YY{jJf+DHNHSS2FdZfO0i}cS zkx96Nz$UehE6PON`fn$Dl?1ppZMWb<<+n-S7w0e|X3TzbXlD7cw|$QHe9{J{_g z-o6+{3XBvnRH*K5A^-=@Z}n6*qkl48sI>avyb`_%B)Ii`)3$BT;3n&arH>1s(S^Jz z;(9@!uKLwvjx%2GY)@AjSgunLr z_RDVh&7+rZ(m14ObzeFnWh?*mTO3*c^Y$;~gx`RfRt8T=DHb@Kkwd%Mk9dB}azhxC zN2Hnu^ffI64URnH?}(L1Z*zKuE}#?!^W18`!#JamxFG-^{BXCxhSao zpB8ab2x(|t{!BV?{=#x!$JK^Mi1ZGCpKCKr-4?Yrk_|$k(UU^~d`Krv#$8kB$mJkX z+-8po^4AXEPE)SoEW(>SAUQKL6Sdy=0x%sovo1I=&>SZ+jm<$u>Cr+CTmDyuPJLiU zssok-{{yVi`n<5dj&neSmEWwG%9iUDO~S0VZBGtg@txI8^^x#@1_M-rZdCQJHPeS* z`YE*Gd?z_Zytn!GyW*WTFm>N5Qj2oLS1vp8v!sHc>NxaL7K2QQTNYP-9tdVqA;A-; zxpQ?-HgIL%t2zOx9M+IJ&E;z~zFtkVdb}Eioj-5IYJNyBlJ~TIt5$(L!62ZAUX5${ zPgAp0k%(ZTsk<}@9!t0wxm+n16H~blRmZl2qoZ3SC7FOIu3PHrT@lSD5@@?U*bW`X_pipGuaITD+i1TSY- z12!B*3#-nB^W5VCU0d!z&&Z)BF zWuTU*AI_dVJJzgtwdmpifEl{zmCYZOytn4xd}D8Z?vRWoHaOVHW{sCEVh74bZqjrg~U`7b&MyHb~{jP#c+_9mU*xR1~Q*SQqETNSL*eW zqfov!^vk862?GG!(3BN_#2z9(zdpQg$H~|7lrF;USic+d#}D*z|8o(&OgaO07%CKn0z3FPt8}AkZ$(UAL2# z=hnM-?mU!E#F2RJoc0wj+PBViNe8=c z-}e|dS#M7kpRZ%Kv#V2dYDm$&v`E7`O06@@ESmPTZ_aBY^nvQjV8o!z=$@A9M&-@a zd}1}_Xy%xKoEe^{2Ay7Vx=YtFw%$iipC0eQpuI_vUkk6!)SVk%x;OleQgs@))g1pa z9dNh@bY)@S-WD+&3~H!cvu0IStzta!O`(hI1<4*LEG%>{7oc*4`(<`=7xAqBsMOPj zclqb~4x$L7DUVmBFA)Wf_}y9$oPcKA^!Fwl2_PcsCg%)tl@v2cy4z3CS>IMo$m9xB zlT*>^Teqn$3hjINXGttNLYdgl$Y*634wXTpjwnLq1`BJ4ez5YBUBwQk4T^vB{Iy1E z)){h$lp;C&7=L19!8%C%ac#^jQPK`Q=5IrFmf~N7;3gtf)#GZPJUIJ=0i7fe& z$^e{1JOR(Ty%F68>1e@}p(TY94lTobTB%+eeKR+=7!manEGeQwz1?q)mdfx+Jduc_ zAbFQGJ+Zjuf9U=wJ3CSb5y;DEijn6YQ}9VD7VoY4NZ?}jduv0H1!x{?J!j7M_y4ka zdv{}lBeUICDr@~WEcvl;@wv!+U-gP&DR{Sduc`iLO|!X%Kj%D4vz>OaQSfmkzG=fX zsRozaE+qFOx2TP9=v4Ii$xHxc>W#k53zqJOUAU*i0K6jeTqX7eAv(PmPR>q&spQTvus0 zgc9lPxDk?1%2%*B`)Qm<9K?nMqU7tzccCT0IkHM@A!aUE*T|?VKb2;oFnBV9Ct?30 zI((xTT|y6JF?9IwSo3?#d2A&rqM)#_!O1j2(nQPXi973LW5zdNvx}mT8d9kFOP4ND zLt*eR0WtsRJ3WKK?U1D9iA^sC@dROGp)wn%s=N%@faD8d zju7VPze@_aCq#QMx>h_*wd>dDu>4VSvW!y}k43sx2M?~WHd0!!dNj`apJ^%Hb)k#N zfF8qz^C1~+thfioCUSM2aRvnS12v6hiS+46499Hq3|Pei=Ky$?a{kHUl0KEM;+DtM zO_&9v?p_p~e%K*SkZQS|37O{YA;o5XR8~p~5E!``TgPC0P`v-(V1#%`vfP2V2Pu4L zVq$ztokkr#%$q+yizv3w!IFqb3!yF%-!;$I-7NV11`$*S^KR{_Gc@);DXMuXEvTdj z)Z?uNUb&~9=T`V3X=N;`dJVVqej({CXf;6SL8BxKm0e-FC*s9ABRyGd4Eg$)(}vMz zY>t~Y5sJ07^(p3tQp`aK7$X4CX*;)g6q0p{aK>lJ)OX0-8ygMwz2gDjxcpN2Fy;QK z(LMBiOUtXDy(|UoYiA8rV*xAU5cp!qU;0UfJ))A$m5e`}wl8-u-|gkux?3@M^@#HC zr!0jNbFawvt7|z{QDz^Zdv<_sEO~#t1IUL<=B~ja;kQ2*FCc|L;q1f!7cNK6zU0MS zZ>T@mpgn-46b00J?&%d=e&YHCv0f~l=UUd+DSja#3oeO?7ia)-IAx|$DtQlSr}z)N z01wP!TN)&4T;z6VyM2@B>RQ?A0)Nnvm_`Z&@3Avy9uWbCSM0UREK>&IggYJetdsF^ zZ9!$(h|1l8pY#1He}XEE?Dh?WxD=fzIE!bFGUc}4Si51f)_3NJfG;6|23TxcKDs+2 zpXQ2;LwX$h{>p}Th(5%!m0omBPB8)BDKEzXOvn$W1G;BD0|6>RhSUx>!=OxfZcZ<* z){cu6s)e8?sgd@qk|V=&8viVnCD#eQh2g8$Q@03L#&IF%CoirOBPS;(0mcaAc$qR7 zT7uGi^a{(c^1Ta2_%5h_Y)Vx-rKG zKYza$ z@4ZHC^7V+~QcrW9KI)<24~JtpHD|+cxnlcFy;a|5EETaRb2~Ay zDwSKk9jH=|ukxK50wiq+%|2@j8{JvN9jBDCYq<*_Txm#SPZ~rynIbRngN+#)`6E&M#@>LqY~7 zhaH|QkQY0fsY~CVRJp-Y*n5~%Mt0EYa6$)34CoR=>&(08Y>AmMD7XJ zX7?1Evt7k2JKqwwUZ_;LvGV4TF#nSc;n*&P_B6JN6w7tb&>BsILEc>Ndb=*j5Dzt+g6=apE`;n}PhxAT_COM8#gs^|K` z%*Q>otJ%?e@ohhV#+{T2DV8&Ul&Ld%P2j@NGUY0jS%%ZBD%_Tz$8AVW2h-oF;r@X- zbUqJBDl6+^*KW{aFZ#2qKW~(u;@O4D<_ssT(em$a@q0yjBIrRIZAEJjb1Z4Z0E7v) zaZg_&79J2RDZEjbZELEf*dF6gzGd=LzhpIwjTv&%O@1KxOoHxWovE7W-YPb%|3mW; zZG@nS**$P}yh3N8#O8qnk$+(@%({nL^xB6Y(8YSh zhBV0ttr}S7HNq?C)U(h3F$ye^!1dboWB6` zySfhdr3i>x<|i7Ulomwyt;>HcDv!youJVm&9O$l?Ff@%SfQF(&SW0Q48*T`}pIHjP zWsp*XVt~iGQHL!1eL5Bsk$!G8F#y;3~3|rs(*da7%<%d1p>mBMeKV%$mbkFEv2uZgSht6N%3FSHF+3;Ptu&E zrlyuIZHnd(##3}KKjqX^j1JPf%7_g-0Y}0Z7jLaU>+;`nCy*zeh(UQ_5S*Y4)X?p$ z<$VKwtPq3ebIa1nm02SoovyHWu1P2h(tFoj30ssr-7Z?nv&oRm5@L?8!(td_b%Nn% zKbX(*RmhVF6rn})<{o~hu8_!vOO;)x+ofic)4)0r5TD5@DL&b9&NC_S{vCF@K+~v# zzNCUcAx?bWv>kI|&FKFhGM}QXL>NdZ33mD| zSlvH>i!FYTvJK~qAYyU-F48rM(*@VbT=V|5yh}=4aKz5!Z0OOg8%04y$8xul;S3M} z&L>jz%dC)fzEQ_oM^(6d4&^lPf3hxQ@|kacJn}E_SZhdV)qHlA=eLVKerLEQ%Idc} zIqhTnDK;l&tlsnRpHXLi{m8KCXc?Q0Jo&11^zs)Q!G}&#>9XS7-l;Ac`$XpRWVh~0tu6dX857?cU7QGR_{Jv9 z3QLJF<*`a^4hPS$(s$ud9p}60EB+UE?A*zdKQ5L7yIiq2<3YoqvDChQhqFMyVtqsM z_$gVmU!a-|Eo$uQ)1}K;M9~4)Pc&@U5T0kYeUFSRH;-?2<@I6IQ9W(q^rTi4IS{vc zNj7CLN6aH&65-eza+oO@rRbl!`nk@z;Y(q$V@Jif$J;|s}Kv+E3>Q5KP8$4%H`+zfh^Pny2!od8#apGs|6SYB#S zlv(J)Q0db&9%2{UCvN#F)i`pMZZPQP9{L+zPBIBe{)WIj!E_;iP@$>w?-2xgU^msKwCj*kPV z<2HHRk*sq`p2`AKlxGyC$8lO&=2sd1yIGx1++ois$o}7$&%i#w1*sQS9Jh00n37PM zwYRBnAS6w+3iI#kQgc6#lAq#&;fJR1!JoI44t+6JNbn*t@9%0xk=3@F)8zLIA zjbf9@$@7Trz%>%;ET*C)CuEM+nhcY645vmK^dWK+;;>-5@(IwW=ptID11C*NpDd(0 zYxVin(eX*8|0z>sJOEb`usC@K5c{^#&LNPAlbkTf$jTNFq8o8zO6)RhFsT<;oeD(c zFj&HEiQ$owRO|{;e?WJ5Wed&5QoFdOzWwXjQA$%~ek_t-y_}YKAs++eOS>my(nv9uL4+AwfbM%x;s#gu>p04P`kQ*9Fzp z3DF_%J9-f=l*K4qC{>_<+$r7l7)uzzB~+!?VYNKfT{rA1c}h+Jmh+vSZ(XLX^xanf ze9o>%7j(bmRU=x&T}SwyQXdijXVbgK-$zZ3d|xo#eKTEY{QVPV-)wKN(xzSJUc7jc zr=J?V1`?72TGaj=qjif0YLDW{CBYAhjWMo*SW9$QfnZ0C+5u%4^y4^$v3wg6^}N5S zH}PtNWV#)Pv~GZ-9(#`ZRT7xJvw4zxXgENEdsXx3AvAPBe!!UO2nd2G;?>WrWThAaP#rrd z1Ce8RpTd;Ol$6^_MixjaJH*=oQNixZaeyTRV-ni%K6-HZV4{!Yw{l!Q?=W4KkpFk0 zKgErQ0+^Wa*Q27C3m_?(`2S?cD}KH~Bk#no1Y4Hj(C(Dt(T>kAual8$9h!R&WyRCC z$e>mZ`+Y>SNRCdrXbap^+PyyoxX*4z!E>k}H~cru+f^!KnM2X`*Vp~pzt_1qE!TYd zk)jvO?kzizybeJ9xbur%`f3my?{972(C@3)nP%$0mz(rM!_$s!I-uKrj?>PY{XvW6 zfGaFakM(Q41^Hp)MTnP*(wI1lLU(@{;KZu!<3Bb}HTZ2gpU@9f* z=ARA1?i*XRsg3&Z)OrR(f2VfWC4`IM5wh(Rdxc z)po5X@p6(0Aqaaa41vgz)hq3XAD%pl1;~TFTjyRgL8K*tGZ|%X{;d*!9C@7>E139v zjGNu7M24*c56hC@sn2siJ6`0#i~r7u@_NPFQ<1eE$_APj8%ffkg$;a80l+C0Hl~-R zp?{%x<}T05-}sorWZen3oSYmviQ%BW6h@MmBbwt@jyx}=#z|;QG$(Q_aVF|5@}rOu zAeG%rV3=jcuG*6T+PUzg7O>+KFIUHJ?maq^PQC}@R;j!8ok&f!g*}w*2-Zs~EOFWH zZ&%QTVe8LX@B2J!br7W}t4Va3oLJ$mvBh(yA5rhI=qXmG{?9c8#(lr@xa_bu?}HxC zHFB-1Xy$#_*vEI(m_gCo_FubpP5O*>!(1L&4`oU*0a1?LZ{wg?MW_a8MON6y@i(0f zFK(^{>GF^yU&(+HT|_c^UCH_cjXd?lt?{|5uptuS!eM3M#xmiFaM;9s#aO4$nR84m zj&3p?ip9ZGG%ZI-o-Ao}Qg`7E(ca%pOH(L03`~WU)LZB`ImXI#O~a6lao9-LEU^Gk zT5nqSP%@5k28cfeiOv@XE(8XFZZ#lEt>OL}8O>+Z|`Z4=v)r8Ckw zIxS+)KPW67@x)nLle;qI>`_Ipb5RcZyXdi@J$P8L)_olL5cwyNvxfS+I9;4L5B+1ad1TJlq@NCZQ2&8VF? zs*JgnD}7TppG)1ktQIuRj`>b50i%kq4Xy@Ul?dCN&E(MJmH6`5#&+J1y8lSaW#l@S zRkYu5Ot&7V*O_@qh@?C!?$?J=nVuOrM1lBTojNR}$szcVbz8efsa@;2ml{Xef zfo7MB(@efXYL0pK=U&gnvn??I%=HS4MTUU~f$N$sQvLDXAo}9%7du-EJJ+MY{$3?` zGl_XT&NL$F)RD3rJApI^WM4#K5~YYjjikN345kt*N+>9Sji}hM*iC;+Sigu8rG6B1 znO!c8GNa_CgnQL$efCE_1%uR%3dNMmZ+RgXoDA&4Y0T{=hZC_9`V*kK6HD5XDUIZCaI&OVVCb?r zP|w&OD^v>AJ&9XOgSz68!`CwvVF?>Z9k1b&%KfI3)|C^2aKZs;TPu9PA{|(Uz05?O z6EAaCshq6v9|*5gQc^-$OmcIwW|QR3P8AFV4zf5X#$`JW7TFP*ktFZMen zW5ZIb=xMXwn6K`;YSY((>HYmhPVyAx6=GrQ`pdoakfiIM4B-Pc*DsB$+f#q|pV=)Z z_o~u4(hz)7`9O&#Sh%oM|5(xK-_4eOS3^mB$iwBp87Djsh6C3^F`t6>37ot+%X>E@ zA(=5^;Qo|M`@&n62PQ2I3Q=CR;WFUfXYgch7s*d4?{Z3_@`Y%o7;~Whmc$2X(tbrx zEDVoCqk;K=e&ohu(4oL5O@w1p*gDPK{Z&sIlShi{+;| z@I~1i=1h8}AjX_Na6*|9>O#>KR-RT_*ZX*$og*0s`|MlRht zMUa5w1r1&QqC?mqdxWUTLU!;X4x8*cNvj(HLa4IGKil-;#z)(qy@lE@Ix!`i1 z(|g#q(5^QdXeMBN80~cd;0I6;+R zQ`7(nl`4=sGHt*mKbG^6A6AW}w%XfCEbiEetIi!LYEH%}$17}ObLLE+x&KH*#r7P^ zv(|}S{`!|VQm*g+dXM#L`aZ&O!JBR&u~WKazVNAOyy*`U?`q9>#|^{mL#=!3qp;qK0C$OZpuQ&bBcwk&AZT z0gNetW_KV5K(!mvO(?cqPyq8`;;t|9BZ3&gP@4A?MN0xe1eF8}_R6Me7NM^in8spi z3%eBHj88Xx5Gz~S51w>+3F}pqxzCDbsi>~Op{f(VC+!)xvdi1g6tBWJar#Im0#?i} z^Y$4BGz+Y@~9twj6k_114O)M$6IiETZGMuk&y`|Q_;YPe4G9ZHGD8tAg8m= zFf-6tD9RI@*CzWYA7wlseG70BJ@(*Kr+tRIk6zwhGWLf-O2nXbMcLEd^lSF+>~xF| zRj($VzgvCPJ0)J~Jg!dG;l?jg*Brl--A%FG`=H&`gKb;QbK2SUH3B7s7=pI|sxJPJ zIrPZ0AGH+!&OS!#&I$UkW51uF(*%*}VI^_8;{6Sk1C7!h?k`~rE^$)P{R-s+f~7)o zfY?&M%0M(R*h36W1clp{CI?JqzDLDw?z>Yst!`n~JJ3wg+KN&GL0mFANYRV{6iH&; zA|KG9+_Y%xbl3b{@+UZ(KIk=2K5RqZPJD`l<=g4Q4N;xXzq%@;hWN07DeQ*w?`WR4 z|MJV(zReTY(o;5X*A5(LrCkVLu8`Gzn}2voxk?TFs?f!a(R4k;=hK3}`%kKi0!n%K z%A|sIl6sOCQ{?-VDeyvE?*HlslxYBhbAxygj)?St#hd&>pAgAZgt&{f;SMG8_IUnsTa>pyZEY-ODct+K55zORii3Oe@^GuvDIw!8YE4oFUSa-@os1!i|RY1cyx_ zGm=0TcKv?#OEj~>pajcrtYhA>25f@5KW5Br&n+?>iF3d7hCcuUCAQ3o6UdtCIz4@J zDIhp)nk9EQp@ZE_CQO>7-ew$Vtvlr(T<`R$Wuukd6iuPgji|if7)HeYhtaHOEF$WG z)r%;8>a1&tTU*>cXl-8kVONce?k7jrS>#@%d9q|s@NuK_JgXx{>!yk+n10~+@7J&` zRaV4R=E(w8>b2!&a4F4=nr3bNCYTE%ixG=+uLR%BC}jI4b>TidoQ>E`97#Kts|#3b zEUk#N`h-}5Dgv}%OR1ASzedJ-zH28CBebF83k!=pLQnEc(J`F^J?TATz-66A+zMQ- zC7JCjHQpaCQc;SPxHwAd!ftRGp*r_7;4@jgH$p{2bz}-)4*s3%xNzP)8T)Xy?>8S- zyE8M{dw6Lt+lcXeui<@>I4lxW1}<9;JylMPoU0TAl5x$8JRi}+B3mtX5Y0WEMw`VW z$14eT>|_w^S5X(7upCQs`Mi%3F`kp-;f!T5^MlIo#+B^=yXa(6t*18iJKl+Duwn|Q zv08TSjC>)s7{3k0zb}9gSMKFmd83LXjgdnKs0~wjdadp=w)}rEvF?2yWK%IAyi(mZ zJPF)3s73w*IG+Z!o^0@bzGKC(BRxXhptU^5Jsk1fXp5QV|HWo>_#fG<|FP%6MfVN{ z-AU*?QQ6h?*`lO#&sS{MJ~yM?t3NEQt#eTMJl@-JoF&=Wr{9QgdPHUZh{4%E@Kwzo zf`|!>V|Eb97ulkBU$$|$(eDU{$F+A8AGB9CaF|%`ABm=QpIQ;mCKp561*r3&W5@o9 zeX7lncT_7=@P4A8_mlxg!*ZX;%wTZ+qfJ{fx!lG`?OQE(>Ci-N(kZ1)BgqR3S9Cup z?LEr=zys-;OG%TxkSOS;rb`jc_<5R_Id?y<2IM@vVK86!mu&MS_DVfSwg z5BNXT81;H_t^JEE&fdJkZT;ZQwJxSk4~qZmzC^%)RWCETY9q5SZyLQXEFD zJ#fX2<8%I7gk_;WT|9=B|AsY6_Vc1=qc6U8bzZ^__-|k@F^7RO*u_u^0(&JAF3jvS zjns#{V&{iHkx^A#n&PtJV)RNMB=8*%Rv2VaC2Tx^orgI~0%3_p4gfk)M9$|g5~22d z-J5pACrp;*!`EBeq=vF#4!$>05K{TNMsnGSRDY#mtLKee?YQ>TWkSL57WKJ08I%^A zkdws@Er7$RO8EY`1$@o^3+po;C6z9eZEBShb z;8<+mS?HgdXsDM2hk*z7%0`kP!xa9>oPMB*@!hW-(rI!qsFOdew>uytlXQ#xM$_h^ ztTpb5#GT7q#85=rjCdGs&X%ffkl?f5eSSTla()M?>473iE}B%xU-5N@W9OP$lZi$F z-C~SJ4--@!)BcAXzn$wmqF@N(Gx5hP>phSd8<6y><3xs2(y&BKUUy{N$;jthD#z3? z?rCM4lN@8X``E_AUE8XU5Y5|y0kIw})Tj*pO|OFF!U^pzU!V6#Y5n|`vwiENKh{m^ z#4>ItbqTlPCl?=Qe_+Z3GAJO>7Cal3}78O3s z{*t-SuIlBUD096}B9dcR>*+IR-W`29^oW^&ecaqK#D+N^=;RIqOaL~hrsi1rqNsVw zDztX<^m8c3I7NzLz&7+shfw|Ur9r$(A`q)9^s!;Y4`Ii?nOf60XHdt^Q{6IK9shb_ z^bK#d#ongf*R!PJhGWZ;w<|C9KA1A2U2Wrr4~H4rthzL|cG=Hrco+!Ui^1kL7Qi=2 z2v!z7`BHB5a`kFXe+J6dl!>#CKK8@qgN4-wiAb5!+NIaJxm-XBW%3ddMc-5j8xm zn8YmHC`DGRE!wN+uKA{$i9v^HQlkeT;9YYuEHdFDPGB)v3o=$q>})A$5hKV9nQi}l z#%&@0xT^{xIdg-kr){zGCT8%R3u;KhJ6d@{sNpU$XMqmHcNAI+6H1(yZ!!Se1Mb&* z7n&;jI?-MO_hbWv1h`;DJJf6XF-c1Uj`Y44ofjJ{2qqr#yE%p4=T{7AE9;ZA+r&p7 zWg|vR0fZ6FjA$fK6l`6(Ec}VU&4R@7pBmJ!zi8tDeQtgt{Cwm>N*N3eFAhf|KgEFT zDaoR&*3R;k=(O$lrSCVR{Pst?Mb*LS`qr<>vB1!7|C(>KcK(k^^TSL;)>bbM1zxx@ z*QuGJZ|pN=t8URw^Itlit^2J@%&hVkIv!J~15f?{#B^LTtPU;UZJXwy#wlT)Pl`nA zn8P^OKM?E=JnGF^Y>3+}$lpKSQi24zIGQtup~{1!=$#=eK~7t?xodg_Bw-e2J!?L< z&yZG1typSU$=|~bZ#{Ennp@O1d{DGF$pyOO9SIiIv84vmu9O zw4^-GV_8sw@30wM#9RigQ^e0hnQ9kJ3n%CI=53x`i^~1pa7^*ro?abI4hl#;zwH$G zE((Nz00;06qgD7WJ~W{TEXlE}3{u-v2~WZKARzv?G__RBG|qP{by)iV@zvH&jcsL) zM|mGg>=9>2;7t*1aOs!DvhJ3dLaV(Z)WnSMRd=<=RrG16zTRt$C`+{k*hWE1p(CC|e}~7q~4e zSmrFh%x2~eqkgQ95x~Y*41cx&Xv6>IRlqiOr3g+BURuVDI5VI#w@C# zn1(=SK_SVs;IPj^>%f$6-@S0-l#FPAv}E?m%vjeaDl0`9u0N@^fp4Bip9{F-@a4Sw z9PjbEd2uXbPAp2d>T1_k4D2{9C0d>oL9sWrx{-dfTv|V_Q<*+STiR<9w_B6HiCoQ& z^2(I#d*)fA65b$Fh@YGrLzmD$S!5ApJ)ncXeQchR7y%0E{$bWA7c69H#F@M_Z(N%0?AKHf9$uZjA*jaKHs$NZu zZS1S@P-Elm4OhG!cjS1%i<2u`>7=(+%Kq+DetP18e}447IjR8NmC(+ic5zD=0cou( zb0^tJrOu`UwDjbmS;UnN3Fg|w7n z#<^vug!$1FhRT%=IikW;>`3{#sfw&YN<->5P-^g}e&lOt^>KVG)tKYNzm zf@oN;C$yLdYeMSR>fld5CzJvs22|?|nTGxpaO(YE`hHku-rZZbBp~tC>i&_#c_5xQ zGbyQA0)m-S#$dVecsrbLURE{@B-nOWpM=M?OlJ7uJ%N6=#Rw5_iE6xIpUeL4f(O z)|t6c!I7D^C(+CqjM~5NdxHy0P}Ay7LE}}@ zyd+e4=T0$5BpQ)!LyGAVaTcgS9TWk_Ai0Y5Up9v8q$w1+;b9qJY9WWgHqUtZ(3RzJ zl~Ka!L<+41R{nUiR_g~cvlzUuBvsZCTUL6;!x94laG(lmrA`o;toKz`M+CxW_WR*% zl7$!`_09bwY**KD%+Hb5Bxz$5Wr-vXN#P1bT(1_yJZy-#dIAaaj1&*(+XvTV;}?(* z7U^E@k~Cyt*3+UtW&fg%CI(RVq_A3v&sZrSi}Ku2Mxh3Ce>=Ro6#(l+PgLCa;Vs;A z2M-_48a#rrpAzwXE3#h)Pefo#{)#!iBe^m3m+_9|{}&+8Bs%%)GtStm?a(I3y}sD4 z6aOlhT2)yeJ-Nu{E+nF8Ap6@Q1Oi`X{&D+^3G@#8<(GO$ zPi^>80!rxf4;((c$hIow$kRB9IVmw8Z?uuf+qR7>Z@=J7Q)5Qi0>9a})a_KuGye6; ziv?glHuMcKc|UUeTFdw{c{Fqp@ykXwSB&rW#|gE0i?`oG3Ys>4CpYAYqZ}JRpfkzy z6`}sukuwg8M2)CFLCfd}iY$xhprAX)Ix*A^folsTwbgh8@l{JePEoHTgqd=S4{YxrXr%Cs znv5Q437QX)yyIIr#x=dE3i&>SM=O6CSMDQ&*+riqPcvgUUO(z~3Dt8i&#@EVAkvE$ zL&tZNGFd7>wSwT83%HhY!&6xp^>kY3Y-%#&9nQzM|M7+&p}q$E_w!`p$uOU0-L2>C zK`ds_H1t*ne)(i^-PML|8xJdbwhGAi5H(llH^74-Pd#hYgdSNM1WIShkBjJLp3h*^ zlbGeZ4I1>=l9hHztQS;gjvLiA^PwJpSU_Ew!UaF;UJhTEdU;*Ae6~1~4$Qk5Ui2_( znO%9=n-S_9(blcAqfD1936UsKkIEts8RdZKszZ0?8**65ID-KC@S_~gj^^KYSS~|E zAN6K35&Nyzgqo_@%Od*h*~DX=gib)}<>=@L{Hs5qrlOf3K6A2m%6#{pIej`&Cf%@U z)c0Cz59neR_e?-)`HlCA=7{L1p@qC1xX9VzR+2+Gp|)ZwYGf8bA_Yp`Q<u#77+?~b@`Y39*+V2PGX=Ax_r58!&5wb5EEiXc^KwV?kJ=&fYF@ZH@d6@%a$9+ z-D9jslj;1(jwFUA!~9~*mhC~q#I39wayqqR)Nd=kK10vA@EKowYDMqHqNqjADcw46 z={Kceyv{`JRT_Tv!#lxiIWkHXF}`p6wrz_Z<{KWl*JT)IZ9OACliZ99AqsRS84&E% z_CV0O9ox34ED0Q4kPLxN@E*tw5~WHh6=gT$R0auLC$kgLrIi#i0EEe&&mZztAOX}L zUr*VP0G>%<;li>d=JA(OGpnEQWr)y0QhOz3ui;cJrS((JP9d6tWV)oDPSgH1)044d z0u*{X&@DxD(N4S-8|OKe?T2!nXJ>SgJsM{Vr&F7|(n>}#@f;sPTUw~-k#*=Jq=IUC z&+HgJS`J-zfGr{)iO!Lt>LnPJ+~Rg_0=G__7>-NTAO^|0WCM|m`WK>A{!CK@?Hs4S zO9U{E8W(7Rf~_OUMoPp31%vx?KU{*0Rh%YiEi!IQ04B&P(*rH{$;B%c%Dh(L9O#%Y zCJ71At(E7^7162sVMG!5Z_b}^$s(cb>$Cgc=Q!r{wa^yP*SmG`y6dgH@6)CDY0{QZ|d2e`|8+szP;SemG(Wz+v6c4h?8H!7u*_;NK!6ZSTid&zrWjWv*y4W#xt1d$P#4Anp=HU; zrdnuJP^F84#SXP@7LzzNCC19^4TMML^zry2Fi-iWr(gpHVza}DIFcho)S(g^ye6*n z2vv3v*n9hDr-VcR3XqFh9uar{w9%tSH|>054KRs@qDS>910ulV%H-d=+jcS zpSNyxgvo&~owxsY#tIqSUI%uUk;7xf>@z2bRxXDe?0qRYk=>@{A0na?3{wV#SjI6? z9Laef<|uw`Zf647F}o#1)D#@LVR2{%`5IE#oi;zWRP;KKV#acQFxmZ6Q&i@J(2)0` z9!M0h=;t2%32C~kGoTSdm4H;PEixMBa+LO^U%0!pvNYclWDWKKZ%6B`X-kVr*SxTA zcv9KG5FsY~{%y$tc)nXBCBBE;H4&PY z%Qbl&38lE;za*a`h5|R+4DtlL522Dplp?&>`>sF#{B!XVzdd^rEg#6&T&o*7M)bIj znA}Qomw5zF9{y!C6Vwmbx72yom0Zx{V|P7lv+VLwQ5j3aM-k}VcVKKD+qdXQbP!OS z99m_s=V>+h8!=GGuiRu;6M8&fR{iJ{kCabVS+(S6vGb2z-)d+OHX~x=g-i7_M$~$8 za$VZ7=Iyi38csO=$J1P)pEy5cB>322zg>H<(xZJ1B|fta0uz?y+#uQ}4ef?*XNL-k zABHf^;h^-+M&2We`t({=6G>_Xabo7kAO##goqZBsr+846z&vkn_8!8ZC-PV%4wb!ED}V}%g?!giG?Zf& zdEOA@zyTG8?SWcr4b$z~22|BLN>6s^qKdLF?57VoPvPvThHtoaw<1!6z$Kz!lr8C6ei9Y)fn$nEnoHoeZ_DG{I=_{gaqfJ?p{&Ba4@!IljO6r!K z9mSid82}1q?9q--x-h*kqlnt(+Kcl4_$6kV1%H)z#b;(W(L3>E=CR0%sTXa|x6`D( z8??yzH;q$0opXj0_K&9@q9>5EF(@J;!u;Ng)#u+m9k<2)bn24RS97P0DUS;45oH~z zH_B<&C?}obu$w(w^`AGszxM4t^Y+gRFTd$AfB&`KwVy6&;QaUd(sZmCFU+F9-&B$L zDx&9Ki{>WwS33CfU}Q+7M81IgxZwvW3g)!VejMWC^uqfO+8sF*1msvW@&)7&@uk^N zl6wd-?E9U3ZOU#Lxhx~e`0Uc&h^4gzj7Mg;<^YWe$cOn&e?kLg!(Eh=kP6ukj*Cw` z6es8DbLSpLm{O;TCQ(M$!C~{DGg^NSPJ?nqY!6T=Zjl)BrJ0aOq24W<%!Xf|XQthzS^pS)#df!g=&#?3b7VVzl^~uFL5c% zK1l!#<0tQ0P98r#V}Rt+CT^$WK?lUYoyc0U0>m9vBt<@Y3z6^ueSds6(-T^}M0_KX zr&g6pa?0%0`u2(1{}xwPFU)q0TN?J=sU!C%sb4$)7dStxFX)WC{I6$!+wM$O_r0mV zTiXL(>UzK5Zu#3=3TuNvT1sKKrK1N|xn5^PY|@j<%i6DNH0qVptb>nSET?9gn}m#G zu42&XqM&ongBwIy&&qspbDisC=PA*q+gGPH|7ISw_e}GV&eyjNY1Te_it8_<9$ayL z(#oKra@Ga}8^VqKGEGFMqVs8elWim_ z0*eGDDybfbr86^+*6@-@8Sa`r_Fgwn>_!gGT9543oJ2%wd-^Fe7)p0H{$(=t9h{;i z(!*?|WSXE~&<(T!_GI?ZN{ z>Je5Hbh)D9T29+R-FyFew77EZ_05I{T4qmq+V|A&&UGG<{b9z{VE}i^tvzY}fLTG8m zrRT-_AkyZ)K`{+5l8X=;t=(hpc9>TP3~2&|wk7%wuG%fBZf)$R)Wq@|VQ>qRAzEde z6$70fNFs@h%M|}b+aVdjVAG2bz9zi>VQ_{yUyt|+1IA2))vG1GE-KG%VJSww zuGH9EJ>O>DR3g?^D^km!*WJXNRo1pp&DLPB{gyOEf6=LxR1O9)A_!#c1Emtm8`l$V zP+~H#70Q|sA!E&cIc1KxO$M`&i10DTM;I9$y~QJYD2-m3?Yfx_4;1t)@Z;%Gqxbx4 zGp`Z++{R-|0EZPS?^F(yXcXzP3jz#8rIu0EivR^+&W!YO+pnu;ZRg!y_Bk zl-{aZIzH2urj_lk)Sb5erCV`mg73{e+Z~L0lss3AqE8I6Zyc$!CdkdpBKO*n3EHdf zELx~#s2FFqf&>*0H|3&TE=yl**kZsy3m-z`A*eOFrk=asn6-AK%&g#ul4$g}B|^$Z zrMu4%BNx&7Lg)a+#_ty>NJ@F+7IsTG90cDe+Ui-^7lujzpOny`Ajve60<8M12?S0~ z>D{ZB>O=#gQ-S&aoHa=_?E)nr#z5d7>jLR5a0FBB$4er77fL}*OB%jUGe(m_7Aw`s zm~Bd}jbN6Nn`f|e(mo*--uSNnwREFmYe0Q^e?+Csfi23BOg(^Ckue0dXwgDd$?x)3 zx%wmscE$j|5wf0w{chi#Ti@>|XB3_YV?p;pgU){(^-my)KVp@aIF7*g%ZJEVP6#6s zHjGDgFMnQk8Zv3x)Vrm<)xC-ydB%pQt8`TA%P646lB+KVp2*#NkSx^1TI8E)jD5~- zUH10`K;m&Cf@HnF9sJsTZ+e5g%#!d?{lDtOBT!F3@?=<_UxL34qoIt(pn>wFE80mfGU4ToMQy2 z>og2e;%gFAX(pr20sRFR6C7@t50^t0A4phxvB)EhZ@5oO(UZ6VVe!EflB}bFge7u} zgVh$&wk>2Tke|5!!vq3UIR>d~@MLmqx#i|n{8ckyjE&cqFYkfIckELXEpx<*a&Gd_ zWsGEO3H(A~wfeNzAi}V`BLVN<;?oGQ1*l%|m)Dg6{ZK(*_@$SYVX@h=mhiQt=1N1Kiq8*FbA0NVjTSJhY<=OS8E=++ z($?Sbb$P4~x?r`hS)0Yy6D;@qv^PB#N6zCdwG@N(upZ)alugxY%Xi}75cF5ATt<(! zLFAZC%DD8f$c}Fnt?y|fASpoM*xhN^S29qcAZ04$(+ypaKcaq!Kza*UJN_W zl9HubWrD~Mvpc8)DnN8wVm5E z3GT`9;in5)B$$wTt^LgBm>htzR5Txfui46L-v0!zVbipM=c}d9E*zDzO~?WGo@6k4 zDid)y?qYzoW66hWI_17A_MbpnC&gBgTbE-I5swUFt@dG*ZLQyxzuWT$=5mG^oT7HY z-D<@O#g5af*Ce5P;d*%67u_FfvtuUm1|it`CS$K#*G19TPjDD!H4Z{{bJ5J|g%O`_ zP~KpF^)C{~*NJSjKdp<~^eKxdcz4a7u=3|R5%t3j$Ho(&mQ3DL0hrMup2B3;wx_~B zE^D8>PJ8A>>r;1MIo%06ZM`TY>cRadwrL^x*-KZ1m^DwCun!t}<(N8wj;qgJ8eO~R z-&Zmm>6%wv{7H})2@(rN^YfO3u*aln9)6bDy#PTh49K!M*(`d|NMgo%TuW{4GP*O7 z!$gAq#jYuRPHgP+M}EvklaN_@oYwG5ZX!7vPfLP*K@!g7x3U*Y_2)qupQI=ha8EES zbleF}Gn9HVSw`MWR>c(P8JSJRl9`SHq62nFDTDH&PgEtY;85y?srYwVWdhYki@U>? zQw0?jq_C0i*}nO!p7TT06><&`Oh6{tuDyGIPNo=>glQ3AAr<>W+PwHoS`-x=D9Yd< z3Qz2jxLt_I3ToNO%)7GWlyiQds@^!lC^>2ss8D;0eUkjg2j%%0k+ix%vTJ(yir92WX1)hro zBs2gwHA--7*(0RT$aR!rys3Lg;Vs9O=LXFBs_e3jxncK}`c=FWJqOvJyBiQvySdyv zo&lG>Pad(sWUk%~*buQM0(A{q`t0rCMNf`v&TP2&%1Li#ui&KX=L}y>dvtfM!JS1P znw*TBb@1`yY|9lPl(wCFj88khDS22In+I34@6=I@E}0)6D+M`|$M%IeQ!t2POa^$t z55|E{aNLR;IG}{*!f4om^KLb%)`ERlEzujlwhbgIPDYf9sx~kV8V3MC(&RX|kE&7{ zIZ9|YIE6xJf*e@iB5Iu1^0m)d`~qU9pu!Vx6)LzL54mu}=OXTICJ|)?izpR>1}Tl? zFSevKegv<9So)zV8sH6dZ__mA*ygiO7NY{}KWwWxOgnE0$LDU6ub({%6OWwiF=P68a8BeJD>hL=u<%5F? zDlSef_e(mNYGo5hGvk5O_MEsi-yVDgh+a>_mZ^ zv$L~%!D}RpB9CBymY^Gcz#4?+uE6h$}rvJ=+#-r(3Jgg<;KUF#{PlS#CV+O-2X`- zK>JwAto5dlUu)m4_?nAS=dOHye|tH3MZYT|q!AS#cZ*>#ePKzSoX)f??uEg2s`ryC z1+$M&w|=ef3Wf`EhBd?|n{HCHcW7{VtNwo+c;s?M0M>8w;?y<3|C!90htf%2p1Og{ zTu)y;;=JVY)KHhOzS_g0M80?k6HC&~UovlO>Pl66E30GQT#zyj@*PAZ~I9o}@{e0FRzS!w!brWtB#Vtf0N14!_ z?9O8-$&6c&Oe3pRRPEwU$tF_ZJ~OZaku=;wO?m+DO2!o=>=DFyStNk#Ku@YJ(&kD} zkdQ^J7qvoqzw%-Pr6UNic$!QHta;Z{Kz5#Z(J)n)v)fYllCHR3Mr?Gz~C7(>_ePWjVQ zH|PZFKZLLcsXwwEoYd_ntKyZ{w(5UIe$d&_rcco?4=zo7734g^^+RjV_;21vESePs za=UQKi==vJB8iulzn(HbfA8mYkG8qR2IEo~a59nr9ae_6u4DZ>*_RF12HVzMHZ7Pw zH}K;WIfVKea$ca7Yjdzpq@^ zjf-9YQ0ZmqcSK-@@_Y`Y=^XG5xs{fks}L^~7YR^P#E@3AWj%^-JT>yJ^-$ata)1c{ z?xn36BFD&i;5SxF)#w=V6qkK0q@^HipDv_yBpr@f^Y{$6KV^kr^dJ=_lr8b4 zP-4&#G@N^SIJ}7_aDv0(Rl3TVDO0bN_EEog$g(+~r&Rx=zU|fz9rWLS6RgwcVBnd1 zj!tcr-5t989V%>_%G2xsOO$wz(-gwhv*+?RJspw#&niTyEAt*_tzH!J>3#BD%`a!` zjUODGc4=KgNLosdgWonk5xe5M>{e~gG^tzD17!kxcqdT>t4YRH4QS#R!H|Yr@ka(7 zjfM7*9NaLO!Bpp zi%k~Xrr6tz*Y)sk4o1()pTmWkZqX2E$KPoiy>s8`iJC zH*M)}DWyd#+Hk`P7kXE83Xi|QynEk9IzZGHFr8ML8#+1Oq#RyD`d=Cs_uM{7gxe(#>fg?&6) z=C(uwtek?hJZWohtAexjeU@2A7tSBKx#ECD(EU$6R~OET*||k~yXXAzmt4YT`MvTe zeN)MQUTO`KSX-&5NW>`e5V3^BqAH|cAz?}%uVWI>>OeqFi?p0V(T7RpS{*;4u_3fm z@b~-fp9hFpG)yv1{5&cy3R;%*V;X9}Rfb1oT&nQYSh&k#i#8IxFNbeXv2N=qs@RqEt;RoV5jwBbPAD)P^j^~M<%U@Mx$MeyXic%S?g zlBFFNi*CYE2ymemiO=$!j<*o!QViw(3dLgFT z`Y``8b&vTZ>0y1kcQ+9)Rr&jM!?t5di?elgEo(ZT>Xp&a=$s&S3eo#!+f^3YMUr74 zrb1dN7+okYfVm_Z!NSQ*J(0C2t*lk*Mo5kvW?*rAohmDlI-EKr|BX(B7Ll#N9Tnr? zKmR>1_PzU$E7mih3Ff7{!{&fqxfI_}dc9yo5Wj5J>i=QtI>52)+xHXMS;^ild&?fB zQlae3%#gkJ4k3h)GLq~~w(MO*WN*pdd-K2U-uL_dpW}GnL(=1RKlkq%=XIXv)w~+g zdKp|2kwHHP$Exor1kAd)HuD8J(>z}L^`#mR_J7WlcGOAXNx53z?~0$uoZaI-zsAb9 zpWhWd@V%+~BC5m;FvB=;_p&Td*p})yVS=TYI@Fs`<|7#tF%!vl7o&rb=eqzuq%%Wt z$UvEclzM=GPS@g}^7lgenOC??|Cu=pb&MM&GX>J6K$%YJNj8Xnwdgi>Oj2 zKWhL-4F)r5@aV^x!5|zlWJIRE?;uksi^B1NW@W(t0@a7i+aui(B71lG>S}@L%n&OyC|LnzgyRiUyIxS}J+IO+grST{7tKGq2Om;+e}Su) z1yjNJC2#k$uzd`0Gk|UxkwY%Q`{m_{xXr?>9dh9Ay|%M6&&i>oq2n%Z1R(yTj~K)u zJ=L$Ivs7e_Inp^nokCYp1DicyYKQ!&Q{D5yb7a_u%$i`B_4qy2Q}9JZwn+g2@-brY z26wGtozexxp=MMwh=AyYaCFv3sIZ zAO7V=mAD|H5P0JtH)aB|cg&%^Iqsy9_#or8nZuU~X;mw-iuzO1r&X%zuJYjT%6nO= zzwU9S!AnYJ0#u$NU`~HMqyS{99f1gf7@(QKQ0mL@LGC_3;tV05#0khS&{~d`5UJ8Y zE=+qvPRLmQq8Ohlfx8LQSOvHvh({5^^X~&TPO0~~7cx`;K)D(it7b?kApwjA?v5=84<~(1cY11bA&V3Yr+m0SIop zPRUr}0Y6h2o(jlnmf0X{LS%7(1(jTn{jsMtd0uyZX0KVr|Klxlc#6qA>>L;|ES*hF}=6ZuA1#BQd3vlDxLF6>#eHjx>9{qc#Z_hUE1 z+fj*MSDe!F6IljoKC`Nh=f&_fCuf#T%dF6(IE~Nzv5}ZiPtE?KISx3^=W09ESZ{Z6 zZhl_%%}0tdYopT1iVb0#9WSm3a=lyXis6kRSIx@*qSHWp=d~qyGW`3#R?^dZZMlHB znVW>DM(gIsy++p26=v$)xc(qVIq71+eaYeObbSg|&Wa4JSQ%BP9GOy5M_gP-R`#3s zXx&<~Jokl$X~XwT4!g{?9B1RT-lqEURhdqBQ=i@&CZb>+(*gBX6`A*pbN~TOMLM-0 zD`e^dr`5)ox0mxwJ}#z4M7%4#w;IFd920lsYi2gKG|cxTL%xo6-mtcAI#B$Z3Hsmd zoA^{EN=#42o+?B<)6gi$bhcir`#UW9sZ^4i^PdKBgXqcE$I4OHVkqYNx*BCDpJJhD z3dg4gKO&~;ySbBfGp=C<9NLjw2*_v#5W2oCpE~)(LePM!CSPq}c_Yv48FS8z=jHI+ z@0JTwd1ka#s(Exg4OA1Ha_y3izgpSR_&M z8MlK~9TQWVS^?-JU|SiY?f);13n)o0@DKlP#(-0R>>33rWOE;oQnG<}ifAYP^}^Ku zdteb+KSbpvsDA%C&oy#F7YXMQF=az07GR3N%p7#=h58u{J@qZv#SX56!)nN02$+mU zM@H&D<7PlXHbU5iKyKdbN16s`M?0)clr;FgfjW&?O8^_P0GT#`f@4|gLOA1J->|iA zUSN)-{7ZW(0n=MRBM+A#Dg1?}!M_1mmVZ^LB?6#H>xFT{}B(d+c`P- z3P1^BEd(Lyug(oI5dp<1_+@5k*J!50;-zD4HOL+eEv>IK-(x?`cB0-AXt-Wl&KYCY zHp=8^EMHq~bxtCZ;IvE8qu6~qqgT32HL5pX6RGvtTeoD`p^MXRWHH10!-(Gei-$}S zUsIcwyR#UM-_Y&IyW6~19)IQZGyKfR|MTdpNoVRN?&u`P2g0J;4x1Z}<5en0^)tgO z0=qQBk5b6@Rm&M^sTS0F3zy8tj?89F6Nl^+TQK(NDxN);%2aBrW;_Ghf; z{?Z;qRfpzwmk~!beSNE~Z ze|FMQ)l{sbpR;SB7YzQ?D)7pGxgwf=bZ*i?)8ZKFwlotvnblj@b*O%ebhlR1A`tu3 zTQYwCr{}rr6qdZ_*}n(cb5f#!QPurYfT$%9VLFc1&rc)X&{Wz^S{fQi?exml?1Ftm z3Mu)%%emEHUxnk=A4{ksVTJd2Z;_#(rjme$XQC! z9Q|1m?{|sQdybBh1l5>V-wm0>l$UvLdd*-Nt16h-R3YAWZsl5jHzf-kW7(qj6Ymh! z&+uNKeb>)Yp8JF)@(xuGryPso9mu3lAE2Z7FZ^p$3dNKF2loz?m#iT2g4ABl{j5W; zd~`R3{`4WFVwh2G_e1vqT8Jz}NdhL!9tv=1;N!<57$EJoQd=Xw`$%^O>Vhmupcs*J zYX1ajQA}iH!!1*k3pT_IK;{vXS7gM56v>)@)ZPMGh}a?{4$4Tpf!e^RoKJ%q#UBH; z8KOCKUOVyp(g!_47Muf!MPIln^=gp66Siv}tcAf90`x&2_ji>S)asxq0=M(se)!ht zckhhBHn|5T*B?eE&+~C$bg%#nAjD__w2*ORB0$0Th~(#*ldTSbm&%aw!V3-v6s54R zSeXOVPk@X5repp29a+2$96V5Rrn>=7iWH*|Lx2Z=`x+t>1E9+RykhEV5&96&1M&AJ zK#B9$uZzFk7K&J(fli2);J*Ug2e`hNwi(w23DBASM`i?w*hQ9gz#Gs`%g>!2zKw>o zp?hW*VWgp1jMTC);P&eFg|5OB7Lat>_Q}B6o;yBfek2>44ukN%#(RZI0i(woVIO-V z@#jkqy=f!ovASk$-5srTlR4=s^XI+ARhs7Zyq31z#C1G(CJEb@2~J$UuTNzDJUXBB zdM4qtojf|A`qj~C{XD7W>zg-An#ZUU>W+Ii*QBMe8SgP%4E$}?GLBbrDpOOlJ29&D zwO?yiDLLRb*%(I!)AeTY=5fBL&Gqo;4+)`n-{NdYm>cB{yYEfRi<`WePpH}EaeHh% zRHm19=gtutg zo*zZj|H#JG{IUUoiNRjrT_TZ@?o7Avcym$Zh$IE8iYauFgBC~r4@J6Gvc6|tBRN!Q zF0((jhYA_*SXYi{@^~HIpaZzUp!;J6k3(j8gK2EcErXMTnujvk{<-LT`{_dNdvpY} z-_cMzi!-k@ieGlp5HtEac+IpO&tzp(+aB66K1r$#y!PM=x1*V$ec*j;Mt>r>iT)7+ z^|zc+|9t-U3q%H-XHI156HOP~ib6L5T`&3C|JC2G1`oy`Hv@ zoz~34?t!jx&K>8anAh5^hqeWGaOKT8F)Otg+b3RP2HfzEVDhP{5r^rFFPx{$tv`0c zaLoK*g9Mp}1VkHgIxyb@F0LP(CPpy7jp4c+TnIK$x#~rYxrx>PTo7F5s&CtjK5QMq z#=~oZ@)*Wl-3L0?FHggg(F*r{PM9&vppy_u1rfu`0j?JgUSZRqLIq_f)pK6T6gf{% zahMMI!((d*DlqIzm-v-S{E4=n>lY7G2wKq~7}ockf`T?k)jZJc^GQnw9lE->)c^WL z0y#Ik8gw#0U2$D44h{|&K#dv034XPP{QmD-N8*HSd>>xtRlW|lkdB;_lhXz0kA;rQ zPoraVut-hK%o>5Q&0{?w<^Yyfp~RQrm-%DM1ivNVGR#*x8_Hoz5Qgt~IaK1nz<@fe8xb#%#&%r+ixxu&cc66tbpU*rOY)MZ{4QcT@p7_RDxbf6c^moQ zSQU9I;Q~UXl5bmCal(8#0`_edsaq7j2@AuA{oKTKbOhGc)=Nx)>jMF-5Q53W>zAc# zqoPQFw2O;{g|&TA8k;8vGpo@iInc(?nu@@~D=cy_EtbtqY!+RV+&qZtdV3hho$oDZ z%%n2y*Kui1x*az#$b#NeY$*Jyt>&N%_!jdDFHXZzQQS+*9_?03Shkp`p!p8P61tkV zsQU0GBk?x{RfX`Ee{laWMc-vqTe4KXYOr^=lYC=pumfKrnUhFJ%eTWYB|VkcG!{?c z{bwC%!Tq$m>P)SlxgI#Z-VQ8b!x%bh6>h)2wA**>C-FFzd&5K%riVFQ+|mu^j;w~7 z3f}11KC1q;fX6sO$L2KGx<#8Q#a|uOH#A|&+)z*+^C@8qV3x7ek&wAzqi=}Do?Agt zQ&m3TM2d^?+`u^L;E6=p0@Wgcnzy^U=msN6hvHj}u+-xHpX$FnbeOV94Ub3XixxT} zv1wOLR(`v(tB~KC>Z=-ZGMAi|<9wU4O;=GH6`uFa(M*<#F*a*(cmi|Jh$~kjxXHbL zJ>Eu$UF_6>;T8V&s~U0o`}gsM?5;y=#b0Nysy!BD`fG1yYZe0=$F^s*Oo92XhezcE z@7&heaja_6q^I*I%4={^A9Rdb6V}gm9A7}ud%0jJr(bkY~- zWP7_m_J!>2bb)ruk*Z9$w0()Texj|ibN|pqNd#K1DSUikmv;G)s=*jDY4VqTJQn|w zQiJ46$M#Fpo%4F1zLH8M9!&l8YlnlPj)#le2(vKT)wrYk>wqMJ+vr6|M4!o(Mo@uG z4S?ns0&XVVRwk891Pt+3IDFH{DlTlC|HPTS>AAUAP-kDgc@q=t_j(qt-x3rghZETb zQeJM*(AiY(EL~#sr)Zpq$h`AnXL*niKrm=3e=&ot|0_VL7#SE)V8-DW82F$KO9q=! z{(XJ@ZLkp|g$$Nm4RkEn@_AW6K)^UOlxRC7DCoAWZ4qnYyLXt$3fNF8@JZi)IJ&XB zYm^FvA%+;4Q4603QX^3@F{IZxE(dpBflG~O3rAe3a|FH-Phd{OovThfz-cM?kaV?|6BTW&I2j$FxeZJFEXl!LvYm?+LivQH+=!OB;eX5EG^X?R7<0?3 zdHLk@3Sv|}`3CpmZ0&+%4%U#x>i)8Sslk|V90r>fp5H1K{-*T#hjC)DF>-CX2Q$aE z=Yv?mCiX@9DQrZ70{b%`UFPVl=sRzpQ{ExuF(EAAu0+$C6vFzD>PqXlS=yPOop8c7OpYAW`&O_P?!kQCPG=h%yN!yKn*d`o{ z(-B9ZP5sGpAOC~rvas*qyyGcC^zqKfF@D0<5sKz;?xxkM^_gyrRQ|obHnpkAHd=u< z@7(66ZNHos=R^cON1@8|_aLO)QfPAd?8Zaj-Cs@kdrP>`u}I1>d3t;4-I?u+vPZMa zZ2kuZTXd=;5RINSI-XB%4xBmAOyrz-rxg@%5YT@No~Rk8n3Xut;})=f(zw*m#2CY> z@NC@Qb(`RtgqN_;%^j@usrm=cQ$>&V9NT6)0xF#H+WjldMccwn;6a~rZa+C$;dU=` z-4}#I$ozr#21?vbh>;@?*VfiZ$cBN0&%{a4S^d9f-u2;*Ki0no4#}N=g4^Z`c|_|8 z?vCYMBERV`7R|W!^gXQ}UM&t`N>o{z3TvcFzyD-ZjxBq^g&2d%X&L{LNSPv-A-&OwL6m}>465L|3{FYUk|J= zGPAOHdYu%+jFxR39U(lQQXmy7M!JgF-E3BYbQ)Tsz|8+omKKC9Q|eU?ELF}XPC@w! z%!L5gX#?Uf7fFoVGg~L1+@z+b^KPA-oN$8=HQ)Vv*gRMPQotu6V7PH3AhLUYyU1Y? z)@!=FXW^B?Hg$)?3N1$yFlHGq2UARHXlfb&P6%=+<{57d%VZ=+19z;jxY+!c=|45i zmJpD+ZaX;edz~GcSPT_j0ENlzyLbJ%wk!mwVboloAZ)(@#1#XWs6n}MG5+zX)a6n@ zZ4Ds=M09-cRDq!pDTKVhu&^*Bx;S7;efT^+B5|xG_D9L_SiX)Ao0tT>K>qX&;ASNP_Jw)9?R)OaqK}~W3=|M(>M2i-47C-rH~1d zl7d*&PZJrzP+jV8L91(7GB3m)f)f=k=E2(WfZ?q!vHJW8*aT6uMZ$l$F7e z9W{0IQ65tbf`_%X+$>~e8$!Ww!&J&pS9+F+* zb*$?=HFg-(czQEtW@UXdn?I?$VRNRq=WBWVz*fh*fZt!&rL%e;J&HNl?!H&EIW)Af z(Mo*^Nlhg+qd-cvKSapgiLUm@?BdIpG+~K(0{cG%%T<#Ga~IQ`lO90!4TgvFN1To7 z(WtfcGRxz?ekhq#HH|I^zBube%W+#l*}x@qV3H z^z?T%jwOo~L8Se%oLZqU7Q`DlW`iL$q8N71_wN3ZZB3|Oq>jLXS%>-d6DS$|-_t|- z1ya!_&>^MR$#S;B=_Lv1>FFyfO}&#N-?uh4q5<``@g0X2n9*i_2haN!C`gN7TC)@s z`EPyQZrCv}KnB#AJAmNZtPJt=4Gi$s7G!4DLla3vNf`_rmPT;UmWzaAn9|bJgf9Dz z{f@EmP2fs|!Q?!Q7uv>$a&pbE;9;XrS>CsMKAuqt(&UM&o7-oQAOwZ0IWtg^kodt7 zYJ`G1G?Lgr2TtL_a52+mlK#Y*)|M81Fm*EyeH~E-qpb^sgoL0h&e;M-*Xj9rlh#aZ zo+r!4)_GW5Hz2goVTQLg0W4<=5|^=>4w}9>7$?23uxJMFr-kv0ND3_phS64nh~w)> z6HCi5m__vH)%c6uf!hi!!cb5#NK3$LK)Rfsoi$2t$7W1jeUzqzc&N_+=eDA%$^dvF zHg>(@x7V7$cI$a+VnV_~LdZWij20k*i0EiQx-Yi!86O=SA72v_qk&08P*V*4+OtV`eBSUSv zHO$}&>8;xgg+Afo9IpuXeN$45;vZjQ%D^nQiuqKc64WZqqWvrnlwsE}3HQ(fe+dlU z7qlk3b|dX_OG_9LwhQ(phvxRk+QO3O1RCiotjNCc1aP2C34_Sx%~FWsO%J~$XM z=wsFeDJ)z};(aM8;k5~l2m1Qxxw#b!BmH#MAG=n0If@is|5*@I9m&fPKkCNr)oNoK zq9l$&C$eWILD}rz6S!H^OTK3%aP#r;p6{jS;}#O>m;^ibl32!Vmb{B@cKLdh8ilog z)p}C}mt%*qGH&s+GzrThG0c0thPm{z;C^~mZ3&nPyAmo*Der9_ZM)O-+Lr|D?5bhrwZ4FkRLBg#p%0h%Qo{u~5khDA=XVdC2M@Diak=M)v7<&Rvf26K z`Dt%0s^dz#Jo^J9FLD1{k0WGBr|{UPR?P9%KXw(`a-Wu)W?D7&oKIXt*&Ncyc{_9e zeq$uAlE7g!&fdD}2Xo?b6xJnOx>udl_bmr%Im-#2XG+*}3L?lP(B59wE?%x-4@x`m(0SfYbi~&H)yZ3WTDM>-J+*g zXV+`N{k5k?m{X((NqajoNjDte3^6f_p9M*Q4Mqha_pUyGcUnm~M?sU-|KWAN#6*m- zv0NUTPbTpR%M+4n@{h8lTq}vB^tbuZ&{ldf!bPjAFA|D}1u824@`pz&GE&hmo0T|K z34?}~V|vCgB>8U5)Sn$2&qfs$Y+~BPaCo?wV|>cXQK!f54E61X(LCAN_bNOpiJ%F~ z%qMGV@D1mDX2D~Z%4K2E$#({vPvPG)9g6DegtD@nC#S1CX3<% zVqok>bi^zS{J3D)cGNJTBaGCEuVTB z`xWatPuNv?SuOv{Re`cAk&$b21ph`82v>v+CFiOC2IIe(qunq+ClTW#j#KmWnA@0n zM4Tv%6dj%y8eRI~s8oGQmu_cTq2lZ9`(f?Y5g$aaNKt&hXSdRo?N{Iv0@-1sZ2E)f zb$DIN&Ym7(AOy4l8ma2a`{5Gs@fyGaKu=%)BY7sBpoauKMDhkm2$YF!2+RNloO+P~ z#znO4+Di}rUI`LQGhn$K668cgsGv;o+43sT^oH%w_kq&_>r8{7uA_SYmPjh}3I#`u05>{*pIb-&~G)gCxOIr zWo4!F3Mpf13ewJrIw33ULOnQRL+*!=H4F|8iun>l;hPC56Yig)N~-XbnT-uMfLRwL z>P|7abN}Xr#8Hz^=J4~8*H^t7i46uRNY@5qT}4MR@xSdFh$M%GhxL~So_D@4!Z}BHS82dN1ZI|`fc_+xQqX!pjPK4S6y{5KK0t?5|v}V zyW?L)e@~v%y$rx}U$O{nIg-8xY8{chLln1xKVesr^{7cUu*zFyxe{A`#NVKTBGit@ zT8F6i*y>6`LT;f!i|xs-+S^A(5XIq>tFR=H3=Un%J6or+#tyFi1}$rRpR}~I&)zpdZwgH zEqvYn%NEX?zf*>4#qX<1aII_D2(?NI7QN^pUKxtaR`mYUo(O8v?Zf{3XWoOu1E;RN z84!J(Cu{t}s7ZQ0rA^Ln*pQ!`Y*SpKw>lA5{~+hPyF9b8J6K+_Ht#d%>b_c9>vA~z zP+fh&`-YkUW=+|+%eD(0)a?~du>%qg*F+(8bWhj5 z-y66+^mUZNVi^kIF>lX_Z*64g##p7Mx#_qx3jv|aOUJ>ghr2d}Ha|5aMAI+gWY|S| zoXXJWQt$cw%TN@f6>=fEM$8}7JB+4(pSBt2MJbz9;B(xwg-(LHy^I`V~ zLl=5@;J8hm5P^^2O%%hm6e=8?S!_aQ&-zT!iS)-y`>jwby(}-EK7E;0k{Bf*AhWSI zO1?R$`$)4}?drAh&0q24!lf*>akiq)uycEz?aJ74{jIa08Ugwb;p!{BW|JsExS6;#ucVOV?*uy*G7**Bvp|rR=sJ z_8$n4$bfE#O#FXap^~dpMLd!}37W+7?lkohZ%wJAUb$-;dVQk3D0UR9o7;0vThuozbq^C+*@V^)YA&AOM-!z$hQSNBpX<4!FPYh?{uY)4Gl3{dD<#%DS(!Ncuhp4wqlh7hQDM@~Q_)nyskx|RTkQRfmhA{+j3~Vtd z;s)|SD6QzznXlmcvocXbt(3KQ;mO~ea!HXjMhFPUt;IKBZO|RSOn&?)V?ld^3))uW zKlVa2?COQ6v$L}~oKPaEkvlp%bYADD&OqTZ^$-LEqcdKR&_kl`nc+XJ3;>OeZji6w z2Z-UODzU&Gg+dTnc!1}>kjDWx5PWtAY#kh$fihnNY;rMRr&I7g$G(WBm8)(`BO6Pn zjQ@6A)3p-&{Af@$l=drDSlHz#!y2y|?D4PG^(INr9++mWomukoW~Ut zv*_SI>0KAn;8#8~_v3Qjq~NV@g|7CzM?1rz(a|z_MmsDVmd#TQqZ^%kOp$S#X~o(` zM%&iCa=z(4x z$iuW-qVw~E$<~t`2kYL30J=R_dS(cmgrK!?!KtM4>X(gDryaLMzmg?+xzTQJ8d=h{ zdBUT;Jx2@KFzFYiyrLrd(>#&jmXL6%f)**Wor-rnh~H3s*Y zg%b9bMFqGD2u_c!TjV9j@7(p4fECHP!*9#7`I2wvz2 zoFBAd>@4r=jrI@ZeWMU_5#`io{&`d*mE4^E_z?w<0r~e1`tjOYuLDL*lA9|niJ~Ke zo{A|~V#}`UkAJ!L;ziSHNmak~@8R)pnQkBNblS0yZ^^~2C36+rRT=bHp{}VCOz>R2 z)W@y!L=MMG$AbdKDW4+Dt?Fw-nH z0c~l`-qdr~yvf0Y`d#s?OxB+@iD*7@pKo_2%bip#5cTH2n4Hl39?WvbNGFkX9AFzi zM>&0?j9Nifp5y}U;D^tiPQ4gT=PA|RQ;Ib0{uX0AIF)ffkjPt%nm2gQq`KHTc5429 zV8j*66L<4Lr;OjDAC7I0^)Q=7_SR9KRG1YbRDAF1Y717a_Q@V*?@rdlqrc)J6s*eU zc~Vf9qRDmq1N)^n{l0*uZmd?`8~ale%ta2+SZOD#ajB)G|Ed~GzN;bKzkCVmNvD;E zs%a*j*eel>@1ps^GTu_Xob$>2p$xR)ukc5M9hWcFwe4+pI~O{wY5>|scs26u^r4v< z>C=2M-w%9*aSoPc-rksKXkKm3lP#Yxf(P<|xbLS^j<0$0en87nO9HLrSDP~ri9ecemVJ)gicGpCZ8r^S zzy0FrSJNGDKG<|8rq;^db%%ET{7eut>gN4B&wp=zuAvo)hGTKVdx)tua}y|t)w`qDVre*=2&p)Ef7{|593Cc^lZMj}G<18f(e z1g&s47RT3*gIFwFEyPx>@pLbA1GW_z_Z(N7o$U9M`y>lI@>|T`GGiEupT3mfd`Gcn ziZ5$*T=NNLJE?|X(m@BizL;4Qook87$+5h>y?4OEAv984CpRw-0bnggN`iYB4K*|< z!C2=dP+*&SzgX(3-25lB5lM+q%gf8>>eNudvYciR?ek-P>g?=nK0n)+M7j|O+Gv4I zJj$sE1P#-bdWd(#fOY}4d=0k7^(X)G1pUNNsW~~YzF$JA+cY{lD%S%T*o+D2K7zu+ z0u(aWGLHa!f)#?L!=j`&o1rii0M@+?pf9RAQc`HG;q-)@u~mu;;Q|24Noz7+`CL^+ z1ydWxGWnoNbqx*EKf(8|0>kz_BNfEsa6Yn-0Fc=?LK6c@1>>+W)=FEKb4f!$1q)%b{$o~#r$PX>U- zGhoG_!2JG{RYP4}5611-1O#glLYlRNc&M(fE?VAaNGd??7l7Qb`0)Dm>wt4(Tn?VT zP-z4rL&-e|mv88PfJsSp#)J6@ z%ID7?nNW%%5*{8|eUmmpGm#{3FIvg-F)=;2d;ZkvPw;&J|7tc6&e!E?&3~2^aW-DJ zwl;2B_=Qz!F#iVCs$3jD4s5YT{oJc77PuUYWZ6T^ef#8LFKGpEdg*s0$wXf(oa@jv7qmXFkmV>kg*qs}WqTJ)Oq!BwFl9JsM4 zAAIV|+>Y6{`#j|qkEE5Ax7WWLT*!GEkW;(8S9ZVLihwBiIWw!7!tC+kuB9ZseXtmh3w`~1ZZ9f|ZS!}CNtn*QXjHMu_Qv77;3oy<=@pBB z!)4#_+t+Ux=}QmZO08A$UtLcK%zT1I5FQp0`RpeU9oTbSGFda5 z#(!02p1FGu_W?aIwreV&F1@~gQpeog)wzR_IBUOQ{K&mTBbRW=>tugegt}b3NSOV= zzp9?8595svnmR5!gXP=gs|5rA+5z2x5Je>TuxORONByfK)fhUBLs_hCjr#6~CZi^< z#veN$dYTH#sV_g7&o*FvnC~MW6fz-l_=uE9F#A{#s0b73JpCm@On9N%YbbbG!(++M zidw7}Lw-o@H|Su~OyS6C;q_e+yvlM!C&LGugL#9|L?pAtu5)V(7)8EnHN`@LEQ~JfyG$~pR)60S~wo{Tn&WjC;d??omF#`vkPOp&P zRGa#~_Z(K&4quMaSsXRCT>hoPb?5i6w(3ZatdAAC*zN#vKi_^71RZhe#f5gUz-sT0 zhs;4CY|4pRXg}ryS05*%$T&n^rl!uFYh9vW^LrFk)!5%pbpz(NcGvL+&nMOah^XG_ z)esWZ(r-YUIZOJzXzB;p9Wn7VD4W?fTr8sB;o7ch62GDER!lsm6?-2{t|R1EcNS++ z_(SsSVSRD;M+5CPhed-B7G<=bDS;#_E9?r&*QAw{Zg6V1KbV^wJ!}HD z1f!0Shf8|Hb`GG{+%ARWZhH=|Qc`X_=f&9CUbfv^5%2$^%IZrfS&e3Gec@^TrNL31 zCr|JpL*`U@U5uOeMxJq8JAZ&HJv@Z>)*e5297fR*OW!t9>KXb`zEazll8<0&p~ph? zW9cn& z2rpuZ-pNw=y|=0a=O?$yYP6D%6fjABIKy%Y6al?Tj20Fy?}dv+kM{R{Aox*I_77DH zon)8ITA#>@J#Y-~*}54VjeN!w35AAZTUcO2hbc2#HIa8L_e&y;Wm zFPS34)6i^_+;_)6x9o_m#+X~D8gwj0@c&62kUtz~mNfn!(u8!fvv74v->4A%&A9n( zepqPW5$15;rBtZ<|C@Ut|wkxUaQ_M(3@%785{Li1YdLoRB6 z(M?kZ4wxccLdMJ=_^|3u_o!30qafuqj*e1uavtfH-DoAjzKAfTfM)6ss!3HZKn%eq z^feH-Uv>-THz5<SB8O&D69sMt=UjP(|Xc4H_C6Jwrpk)>dg%9WO75 zgoK1n3R1>b8d_TBe{ce%{#y>n!<^&@0W?8K6CjsP=RXhqWDW; zMG#&N7PtvQwl5=1OihEq4$!a3b>oga4rBPc5OS{H;JXtl26YAe1>8G2DLj^od@7LMK*dyEo|;C88yy|fh4(w`NY%5L{P;44O@6AN z?rps>_29gghQ>8APBFCP!b9)UvNE!R)BVl&lDhU0Yx0twxCPaAPi{>CP;2^LNO@t*mIGP$3x5Ju)EwscZ5E5*#n2Bp=Mi3RNHc z)@YxVd2T!~RXX??3>rzo;=o-@$2aW;IkiA&+wT^}(Di}*Rlh-<<&Yx7{ZB}pLPV+> zGN_BSRv-;_-aHYSU+BCoH(XZt&wb@3YY0#->oqWkn@OIs=^NcLFW9{+Cx>I)DHH^g3{*W= zU9(|V0t2ib5>_~kyHx{Y>iG*OfS^w=^KD1|U)_kz`mBM4U zA3Mw2o$A?Ip{vRZlE)MITW#i@LIpd@)LG4BmppqL%-h<7S?*h!UaLp5py8M%`Tdek{*b}Iz(=vp z6&X~{#frW#i$FuY@p8w@83@N3lsewL!}@M|K~PZDR8-CUo*V(U5Ut_t1k3%qs!^=t zX|bF8wj|vXeoD#yVd(TBb(CTGk!zvrZ4&gpvhh(iCxN3VRHcejoEjfP#Yyk?)(S=`r8;q8*An9r5cz=@K-*WVAk=-oL!!%rI znMX=ZxQ*q=E2_+DWiOzhXZV$Y^}Ao~8jnXsLDw=EYiR=ZqvEJj4G_}PT{8zVIL>qEWb&4P)xZEoRFF6=?1Yu zuk9aa6?SPvd?s|ap+vqKd8y+a?E{Myr7-o8Mu;;|K{u-D;=|jxqUuH0F98_* z6&cpU0&jYQ%`Q>&=@(NpS=W$=0e8v1d$$!5c%Qnpj+gM|Ytk$CpejKjMYipUi@@vG zRNOG1hxInN*4A^y)9clIs1)f{X>IN8!-FbE=Zyv|zp31rgp@Q$|DGNeJg#;q0{Skr zpB^>4OF5qStqM;Ef`BrvvS{)V$9((j;O^aLQ&5V3mx1k?f4BF|< z$UFbV1|V-Vqs%#}i_tB3(n3%kPevso?zhkQckAoRkY6yy8fw&RLvK$#jm9<_daDy+ z=!#GDJ$jBHFe30~Gz-E<)8Ye5oM_7Y+W-dqvUhT7EivsR-%3+rK_+-vDrV`Kbn;G4 zPDu7&S^YpO3=&)ea2AYsc-{ClprHTc{YCY?y*>3py$jG2#))+wufI99EK^h$Zk$Z-QDP3DM1X#dZc@O3uh(;Y@xq`?YJwESiciaAC-KE*d zXP38JENDaps3Dh|aeRHDD1QQY>V@04Dry+ltL<^p(xgy6FlcBC?`}OgY|!k`)Y{%S zUI%1M`V;GV6JY^`c;4RL={^rL8%cIAj1eGpyfWvYrd)hO!es2%Jx@V<4k)OQyMTPm z*-{QMY{bjZGtKh+1QtKx_2rXqRMG^k8 zHnxvlBI52MhE1<_R#qH zfya;4f#G4?(Q^3<@p)^-ZpV8nDI;rgkY0~=Y@2F5f0c4RaCWAy-m0^;*;U&&ciJqt z$FCx28htAUn7Y@Vsj673)MQKjuv0yB#9XF5pB7m99=~r1kZFTt<9OyGr7R@L#<&pR z^7#e6+f8gTG#rt{ylSB2!^*`}c7=t!R1*Bz^S#c3O}_a8u|>@}ed??5vF%>D3nb?| zDr~Rh*dJJC5q%JRZ3_wJz)kZCr?srvFIRM*=Hh*Y4+T1VckWi}vlRYRMn7nq9;N`6 zaamW~@_;7IfSVa`$sb%E_*=K%l8$&i9Xe4A(fP!5x?!bzVzU%GI^KWU7YC5S6)Z zF4MxBs`I_=h_uays}|}G>(VLU-~6<%=IL*#dw;yIGjkG_pQFOuU{GAR^Oa|TDyy=q zSe+eVz$oVQwfVucacnF$zf>HuCVG;ta+0;CUE$ndFsL0qm7biMI#5pq_t(jBQ>@n< z@Qw}rs^%gU(lY2}^TZ$EX-WGLoV}X?grU~Gp$V(iZUs+cdv#zTfBM+_Lvx+U-p(v6 z?x~YNF)Y;xQ`{BtTORChTimcwYqc$_8I!bzRSj2-yH@f`JE7dnHfKU*W}7v_?D4Ki zymiej6c#}zM?3In2 z0xy(L-rn*n&nmX7Vp4yC?DFD=|pw5B54cJ{aM?2!GZfj!#QB)k+pQnACb#bl}r;)2J^jFoy&eH5__~i>(6Qh(vGBYnZEYJBs z-~cH0Y04WvzZ(U{lz$|hBphV zGSC|nIT{Q~&znZ>J-)6y_1p_xAg%w?!-szOk!|w&DVMy{`lnR9&;jDM54;OxVn$H< zG4k@hja1Xrd<`}NaV($PV*UKkVIY~#bKnMpI&eSI1 zl&sk?R_#gxrlWVwmQn_%>4EbZ4_OE7nGs%N!^DKnjC-Z_8&FcrfWdV$n2In)^_-XD zF#fp%J3e5n_%2dSS=kR%8F4IEr9=W@uYj}&8yj9}X{%$2>_d~enVF`C2&V>UG;)SU zEi^*6nXfD{!xYrNgjP~g60}%SGatS|cYOjI7@0wA)rTbc&+)+j01G78Sn{o%9?0k~ z{YnYH$N=dRxm?6BsXa-+k_x2DlfJGvz_Ee+{2Q!+?|RkF-EMl*jutaE5o$Du8leHb5N=W- z*|)R9s$HGwHMcTs;?8)r@v(KHYEENMU0tB!wvtkBkmYC}dS2b0rtT%$CCfrMEn$lI z171*8y`k+8Fmqc)Vt=86foLNDxx}2^O4Ou`cZD z;Ck@cM4k!t1HIyWOSW3^7nS4$eX$AF;z@08?xn|%)#_(hR9yoCXwCI9{pz z>izr6VRqo@7dlxpE0Wjo3Lhu*2qFC6TnXHojcuAsrUXrSHsht&}|Hw6=+A80iA*sxFvD z(I8|G}r`i^GEeaOeUNz6N&)2zNV_Q|!iifKcY z19flq)Abial>(FH>@s=H>44;>=#Or+q>djL^je*`4>_Iee%H8?p$cHa-Gt3C`lz1L z0^9kgtwExZVbLK=?q5HOC;YU3P98O~Aj(1HRkNE!@E_Tdu%!6x3;66L=&?6_xY8Y~CO7+V%q zndKBG$=4-@sQ1P1wE>!-k|v+j6(E(|r@8mL#bc~GOM*-L zs{mRQ6BdOyhjM>^FpsIymAw_kb&?5pYX;beiXwP$ZquI5?guKBLYri_L;AtuBr8`5 zbNz;f{mJlw(X!imzCh^IcVM6Nrbn;TK6HLYcXncrHe_*X$hEUP@7tf#q`EjTpKEyo zW)B5*`8DSW$F1S}fpv99+Ns3}Nl-eTZ1|l`nr;66_(Ht1C03Up@U~uZ@sr6~I&2)p zbvC&VlhArFJ;wDZD9B?yZKKDbGSWPrA>7>^1zccy)iNhz#}ALtSrdfWNxjg@I!0SeoYRCq3knGUAoG(W85vOSvFr|0HeWLT8@7ph$=EqKc);*S40f_o?cYRws;#91*Q`eH zWl&I4GkFDCxIaBeY^S4p1168eh?FfY&GA8qP3bu- zHm->60elBd2(U_sw!!YpHvps9{@a1e45mIqD&IDuW&=wp2C@*x=$onAFHRG16Eom$ zEHcy4(IHaQsXu?3+pU2c4~m-ru%kv~p1?YJ!X+U=2h4ne$3kOfSG7 z{_QsOfd&cp?EoYR8{^KC(;)3(W?|{k0$~mw^j&T6^*3PGlniUzH79@loNo@+mUipU zwpqo{;wc-Os41UesB zQwB=~)7`DnU)N9hE?fD@{TbgoEgy6m&5{pJX{fZ}TW-ri0ZQ7FQR$Q^3faYPRM-*H z-0!oTFlAWT%BIycnw+I>L5A76VpW(>{|%S%TK{oJsQLYfNBqYD=Gn2C5_rqaLc4Xa zQ4H{;&eH>yljX*%)$Rs#5@O{<*7S*f(;bcVRz227zyhhDOsmY5yAjJ$xLp&HS)d?E zE%yA5{DFUkn8PSX(J;Hia>m3*+_$VH???n?es10B?+ZD6Sf`1N+Q6}I$ljgV>`7~J znIF>ja4_Z3856)456nsGQEQJ5Iy%68{i}4{L~}`ajNFHMMw2{7>A8O!%F_xA*Vpyl`Fi7_Sl~3U*juM=@~bU&_!td9vSM zK9u3`)SxKT_RU0E$g{x5UuT75B*2BaL-pxvn_V--cscJCb;vf*jIh~Vk2JIO&65;{F{K4)w8|?iX>Ym1q#A8HeGnHv#|6#No3_HC)VV?3u1n% zs5nw9U+ChPE_W(p>}Zy=R8^4_MEa|F(tL7W|4ahWp*}x9o>T(WoBWl?fwLJIEogH! zGYkx3SxQ-JIq-x0zD0EGa14wgp^>S<#dG(EWAgHxqF=szFDoxHqx#=Ag(f63*nK+o zfBy(|m&%ff2ydz@Qkinu!^}Loa zgx&EIPi^Y1!ddRj6{%L(YUYKi5rRV%;>T+}@6oEODFcYx^NNabGbt&p@3OM)L92{& z@7_I8H&Ia>Wc5@2IXL6L);*r7rl!ULc+l+n9VBoj3R;twmoG3Ii zHd9vRh<**h&|HF?_9c%8@IP?rHUNClVl953{{uZ7eJ%$y_mB`4XJlLe9@~8oReYij zX%$f>v3~RxRCx>$+a{EbMNlo-yD}mqZUd2{6U>G!hR1>L|8o|`KM4*FRxY=?Eg&F3 zvaeG>0$#?a5)$tbe;!yU4IoY<7$R)gH}_lwft-ICKg|_)`j)p1J``e5i26O)1pbyE z))|~nG_+tVRJ-VdeMkB;?AdYekKZXSEp6pQ3wVJNgR*$!mELXZd^3ZE67r&`f4&{o zVG=3uq-ua&)CL44O+qSATW4pVTI=`k7a&y|7OKVt0QS2?B|uVAQsT7q+j{BcyTw8< zqq`oIE#1xGvP#aAn`6;pvGpYqm-m;T%P^m1LHQnk=w+V$MLFLoE#ho#Nj@`>>rsKXSsFtH#Z|_1Sehd zwF#A*lYiA(Bi#KdlIJqftE>Gae7%*OX&YbYrP$0gz{m6iL<*TUOSZc;Q;n{m5lEIe z-4J`*#f4dZCZ-YRX-alCeUdu0iq3cY@+3CEy7o9EBNMWaFXF+10n)e}xnTC4Fq zZut35`B!0YFd2E-+L$Wk6bDwPikjF+#rd!=C98KIX-dL-JF}{xSAyX>!RhkJ&!;q* zXb{QqPn3K38g))kqhRB`>t;h*d^?D2;#sT>Cc_Ta9}1(O)IST(=Cuo01f<1wRAZFd zOFE_sLtkhXk3XoVKC-OP7jn@_l-g0*u%~Xqr{Z$Z-n#xo`m(oAM1_zLt_&7^^)SJf z5P8>|nI?2WsDCnjr^Mm-01{;*5ZtfCIe`O-I4X15*%5U+PYipfd=2}12H(I&nhuvu%VGQxr?fM_!Fx! z(wj8B+b&s$bUHX|01rk3Cw%VJ(O*LY`1Q;`^bH1dI%DKR+l%u;Kmwp#wtl{-TJ`qT z@Ao2YnY0iC~ zI-x2m`oQH)t?eRqLk-S|QtmB5 z`F3N*|23%ss>Aca0{``g$S(_cnlSS~KkC=S$li71f$*&)&r5CL_3zso-oU||cLS7A z9h6)&0w6B-)o0enuQOFD=YS_seyh*%ovw5=eb1LI?|Txxpy`B*Mf z;Sj2apmSBup(KSRc6S#D(jZSV@OTMGe(XTG<>BLtVFy1CYy$*AE~l6RCS|QGPg>fYi+Fu;IFLh2ck|0JAKC3>W@R-_Tm@))`6Li$&x*)RLM6lKwz1Lv``oLll|ZmYiMcNtDhSYQM_SY2hcME&Zqx; zBikB*>Co2IB?@vr2#BBw6Tj4d<^0+6moGncaNwz`sv0h0Ee7jX-$MMupxTL{>Pk8X z0j}g1KI2xPRLVprCZ@)MhrRp!iB?bbG^Ot&GO*D#Z@vVXPh{s|5=Jks^Yew3MtL-nC`p5J1#7??2Du=x~ z%??k_!fH)1;c#b3eeIpEANeVZqC*+pw{J#}fR~}j*Pf&D3(vgDk_we?Zw{O*%S+2E z`($%mCgQTLR2)rsG&C!xno%XT^5mW#<-y!s5)XbPrzvsvR6AFaQi|ni=8~D7!&!7E zn0c%-KxA)V(7~7SWb33MshK1Eyk7b}Q*W=d$;4D43p0Wys!m}`)h#|vz@{V+Q8lD1 zRudk5Brws)rZEb=$z5`N;j}8C;p6;Dta+nXS@Y)n$a+!9tIwJ~l4@7sQZC7oTFd`C zak{Uyv5v2qVI)qaM4*(YM!RF@W9!13-q`(m!okWVO>XC)wfEauK5(yv+k4-8%Y`>w zI9-3VZA_Tsj(CZ@I_`1sZ3D?d_#; zg}mb`iIF0>f6lvcTvCr;ElaH$qE;z=FB`9hi)t&=^#1mltjyN-r|JnfNtaKT8Lzuz z&X_Lf2Wd-+T)E#w9_*fn>1)0Wt**bM+6P%B`X?IEg!~iL{@<2W0}pEj`E=QzDBo*o zDNd7<&la2@0kdagq;K?Sq%-%m{`Ni?N^q>6Xda z&sC-)$)+eM)YDBNrC2ENjBFj#Os@0go;D}{7Vux}ZD+QU`10&m!jY=`Y}3at8HtVw zrKV~ZL3EE?3E53Pal`i+bbfVVbVGZ~g-Z-4&ed1WJG^-#yLRU}FZF|{7&3M`Q{ybx ziSd3HSJSCUEkgwrYHBqeo^{27MSfthfwVaMjcU3W({PKWO0QmAq<``>Wd?-Ex9DGfDJ#3OvvbwI z*{XWS$f(J}-e6beFn_?(x2TGUqom7KfC?8JUvMMBev`1IIfItz8<95P6?(vB7fZBY z13iM1gi)UE@=-~v-5SU~zlWc(*s~z=m|w~R!N&urhwlmuWMUYrt@KDhkc>nLppiM-pK>56V$&)YD=U8swS*CrxP(f{QXSTZq`)6zORWPM! ze_tu{TXhBr|LU;7u!mZBx2)AHhd%OoEJT`}Qtw0iXY$Bq#+QQo~-AjRFhz)46(9WKR;wlPIOwniv zJxESVO9Chzh|QCSaH|mfKr?vafd%z-@s?(y>?CY0K0&fVG#y=&0Khz^faA4kWxJ37 z#>aIGhWnMQcM%rolwnny*>mpPIsHIi!Wj@Lv-jkzl_Os|O0f3pBA-dAIW@5H750IA z0Xk(-aL~j%E#7r9Llnx64U|G|pDksspddNS58R;$K;w@%sHmXle*5D4GRoe}1(}gwTT>G% zPCs{~$hoL() zlxXV)n`&NcOHYYYi6%}elcc@9zRFBVR!ODbRg$uLEmCS~tgeJtKlOjzk7RG(I6-Uw zGCIAJ8C6%mHI<5lZ8^qp#FB>Gq^%JboL*8ZY`xzu4)7 z{+!lGXpw=;`W50LRjN6yS52;?qvhD1RXj0wU-TgeVJG`3U zoJ)+zH&6g#^zQAIIE%}9N>JhI{N5;5tH47sYZx=!QPIac=VcP|-~4?34+YozHhoyp_kF z#ZTW(zVCb1Fvee z(+xmse&W!(h<)Zv=QRd+%?&OtE?{xK2qH^l+nAo2spFCd5RAiJ)6r^YV#udx26|cj zjG-Mq($aQYwEBD%wC9q?izT4Y@5)ivqSLeKh-fhIvVL;uU|Yz=l=7{W0D7eY5!Xt z!vQn`NfW}15gePm6!;Lvi-sShwj=b5M~s&O^YMFo(q z_ex7Q$qk*E_&Brhb7A2PeKdZSb*{wbk)I8INML%|5S=htDIxHTz)KCrnclC8&$)Wm%av+S z(bCX`vsxPiblRqQhm<#rodt>9IWJCh4dNQgZ}GN#w`Kp$+c1z`wt5XOWpKNP^!lxg zLxW4?OMd0lLg%;+>w}|Hv!hUTg|oserYDzg(at*!ImIrh2^}5Qfnbs6xOV<@s&J60 z<4B85H_k!%f`L^SMI0^)Fl7x*Q3oX|BQ5ES*FvHzcZjO&v82p=HrU7a>F694+cp!B_fW^ znHU;Yt?Tc{j*O&eS?k$!c|5|BKgY|0I)8R$OS%SRvd^D)E^HPT>p%^HLo;{yFc8EL z37zgk6L`~J?e7<8DUJ}oB_by$62NtTskkcmj8tXhQpbR0@=S579RVtw{#@O_01*^S zR7_qYB85qkYBw(YN6uQGtSogqf}@n2DsLsY@c&ma<@@iU4E+DSWDyO4F-xRPshDsC z$wB}b&Bs3u;hv#l-VdUDfRAzBep=`%L5PrlJ8u3}1oH8G0ss5RIXtrH-e2Vd#c19xOzkbq)jXAxtZTDI)a4wz)-OZyg<- zx;ZPud^nu^^MYiUkog(G2RdU~fL$sJ2a~;;)KnZut`hV6RzX65ZG)&+%FH*#h<;s$v^m^Mna~? zI+C0NWd?9oDf8~DJ~B#X8TnNgp9lq2kLsDH%-mV==!!5Wo7nI zY!5(0z@>k2R65G0rpyqN69#n>bnRYqDojM6Fnd|wFlv!l`z+>DdyjDYHyb0{ zWsCKEc6jC!V3D1U6L9t4CuQigu{F3De1eH&G@YqyMS8WqiScB+U>ECaP}nkUqBL_! zc+R-Z5AExX=&b<1FN2GWtST!;#N{@49*Ip2D9{ zIk$X!&e59c*vrDA`!pvGT&OSi3vq5Z+FJb4G5Dg-`A*!oR#VW+c|$L=DxS`zV6yOv zYE(pllnN`>qhXz~qI{j)sGL&Af}cg!45m=T<%`Wp+4#?EIF&!NiN(M6CGc0jl?p}f(#DU<&uPyaelh3-@py)yJITl>{|tZk4YQY0uyZXIZZmnz-*lULQs>YS!CbBx@)4>2tQMt<_7 zto~)YQ@R3Uy}o;veMJtM=6Q86dp~HAnrly8FZ1#igNp|w#=`XSAk+wQS2Az(J-a*{ zktzXhx+(DKu-j!*J8fH>O~`W$x&|+;JIQ{@v~=zPtJbh>UdDhWijYw2v<`ih2NFw; zQ93ati=J%b5!UaG5w!hth_csU9oLEjj5nbql>ojsJfmq5n zKmF`}=BRNaH~~0pzZFgtoT###!6{Q`0ZgGt+@JroZaVvihcloS0;Xk@<^HB#kmnoZ z9UUEwpyX&l50#sEfu`vRv?tJpJwm8+0P9h&b))1UE#yps{4@#y2ST9uv_h;A@+WdW zl{`-q7YnFXINA9A?ajcC6B-%m3ukhK*6av$0su47Fh3tN`~$-zA*rEDTAH^F7bt{* zyC|Ze5y!|XBt#AQoB)|4JkHMkb+;J={>W5Xqn;j?8_S6Bas9yGhKpbN@lGW6XiPo5zBfo&!d zn%S`a-d=BrFGIN2GyE`BBVRT+y{H<+$RLBQsj;yn=FQ)ygm@b5)prO{4psxBmG&XGZ?PNO`*GfQso?JGFEXwJG9hrmaE+Yb zjtMBVI@_-KU3>hoKR|K%*H>w2IU`bv`LYzvrKcx+{%nf|Zd6oOLZh_}xGwDWEM9^0 zjOIb{MO41F2)5gS;U(1M&qSt=5g8+@h4iDn`TRcp?O)~uNP8->6X%BdKmOq6w{c2k z)8`$ZaK8VNn|(9KWaK11*xBO@cp@)F-us^Z(sdW#4m-ASS){z?#aZZ&PR{{j$IjK_ zyH>0%b4mG8(?yqKn~#iRuA#)>S5P)T_RXZPo+wQ&U|P8!m7fh)LnsjfVJncJ{k785yy;2x6mF z$0%U+mnC7n-@+W*-!41y?XGm5ezL0*A9>&I@wbiEzAcRy#mm64#94Y4`+D$bQDwF% zGIR5-uMY=TxQy0m}>I7_`PtLFs~5yPgQWu^9kJ z>ObZO%`X`E;Sta4_NI@u{m2A~)aDOfK$}(VstP3y15g7Laem%BGmCvs^UaG13_Cx; zlo~_)-;`SBWP1hfhc+MGwjZ`7T?I*!MWWavHFFFzqBji@s=+B&e+M`~;MF%Q4?TuH z@7Jm|V5mBxJ?Wb_ZmOcQ(iUbarT^;Uky01JPbT;AznX`1Rm1OYM+I35VrO5pdldT6 z*6lGB<>Or<<-T87GcITIXM1X|^2G>eG(}vcv3uS7wWhh9#G7!YSMtVSyj;M&+rkVV zCH|^2o5TS~277$HS%}n|94)ujfB$;}l#t$_@He(0u|jH6l|#3b-2b+Qac831!SPPF zakPbQM5AS94#C-nu&|M12!xT7(%yodTS7|8Z77Idh{y-JzJ686BfSo{;AT+x$U)jn zF<`C*{@`39K|vEKIu4x`O%(ZrAsbCdIE;2|$mY#vOq9wIZrg!25KA3ROVh;jPm|Wm z9_NXCU3@Q5@DKuk8#}hsMTwD<^BTxIeP!RM;%*_O%JDu#QC+-!`*vUzG~qBqw@X1U z%O?T$M3}g7GCuN(L)IM+&^`6HzjSyL_K-q=tX8A%<+irApL;B@$VAApIXO9`w7>EY zdaa!umz&3{e_GmMK=wV>(V?|l9t3sRJ`Rk!&G1=n_gjKT1Nqua2e87i?*S6lJA|%x zM&!0#9odgvtDDlD>{m%JtD+Y9;fsarUX}Lx%uzQ`O)X@lmttaGi%aL;SI(XL#>i;o zA>3W#Is?e>%dj>4YhFxCa3g~;{=px?!I^b_KcDh+!HF7;uW7GaCB@U(Dok{~R+xlP z1>>{}9cCY#QjL_Pe93r*gBr}A%4HM+kDKeE6A7h=!PVNtRV5gn*@j@mbITq=ow z-I5)Sv8PE<>B|%V%IxxJwr%g5S92I6Ii*y$`LvNotkrXK9Op7GE?z#l`+Cu}Sc=KO zMd_9H9%<4CxGd}0L;$LKfMKSu@m&~EAE~`ME=oH`d1q)i-drJW1ee2NU1&hT5S-fC zs@(YTkENZA45Mtb&SENk^a?zlJT42ZORGqVi51}6uJ z%!o)qDzrxFj@`tE(o*JBsSC0xr+ai*kG77%HwKGn$tYhBw0r){+q-a0Y-MGMx@^AI zg*pA#00U215nEQr+z`CXEdl0j=*o15zp(0UV~i5$_E zaoO+=*8T=^4>0_!0}>ms_m7ODv$L5X5{4By1pQEp`8r54@A7>=rNdg(S4L;1W%4HTb z4g&L9>~L-vEVHD8%+Aj1q)4IV6NdAt*_aM~=q#KHKol4a5^`}dU>w$iEaxT>CvGg9 z6z(8MM6TEh10al7K!k7s_@otHzU0^h776|H7;oggt8TWiERt&5AuXLsU?KHS*P|BN zJ_i*CwEKtaAX&M|$awM3pFgq;HkeoRgy-47eVwgRhy#-EW-wS!E-z;nx`>er3`*{( zfLt3An3EQtJ|!JMqv779kihzi#apVs#I<_EfuhgA=c{YNLo&9yaQ6J;lj9cVEHL8H z5Vc>91g*;A;w}Fm<_8bOuR*UBf{?jCahp>eZ2!@FZxA&fE>_cls0)1dkB9Syn zo6nJ>HQfsKV{IMTaw&%x44bmS{-*5ue&>jst?e}cw>@%mqm+(&kw>g?fJk5l6b9t@5P+s zIexEBcC_nmneO65c&B#aY2d2pF4qP4*(jX6*VtjD~!M$p`OXu6g?pK zEn14PwVyjF%u=J;^xXDV-M7=A^@hJ(Q4dh%`yBk51w6}P7tlZltW1uoHa1lQ?m%UX41txZt zT(GMIarpS`WTIibhCC0zCxIx$G2Z#m?!xBQ$l9^9xw+bB=;rI|7j=7uPAsurxmb#r zS4VCU6s>R&o@XMY%GDGLtPZ{hM6tPc-@FUJ`qQ4!nV_URcBpNi`#CKDyvh1>zJWaA zQY9YBT;|fl{&?4tsKqUCsxmO7w}P7_J=g5T9J#aUZhp_NzQZ2yxIjQfHaxliX+$>D z7eCAR{;x%WUv1NE8}apf>bcvN5jE{1NB852?<;z`WV$jHOSYbN91T-PtpX|6j?YCA zpZPA}Ph|PY>Yos=LY4&ZU8m8#Mp9daPX}JI+9O591r%Zk#Ru`~fiBg_5K^cBezTwr zx`dYBRIlJo8~XhH#5OoG6T$KuOBIzVlLAn!mzb>6@4jr*xtW`PBn7;k}lM@mPRJOgeMWL*#YANjdvkPHJRVbX%f z$>?g2V{DBb2>ncwds@qTy9krg<6TpV*tF; z2{AC7hs`xk?DI1$SA?ma!?hxcMMD@olr8qqMImNKI2Q5?vRzY9WXHvW36bk>FaRv) zpCv0W%E`)lKnfQvFE1&O-?S(N6G8x??d|6W4PT!8gwm(`8TUSaW|C9vbCH?N$}`iM zYS6OasnBJCTYF2;JJDhxXFL0$1#HWlgoKKHg8IKDF{3AX;|M8mG@vfQx z-9uNWFZCsHpR*BUW{D~Is*g6rusJxh5@qjWY>o7qo567=+B{_Ce0JD9_I^Jwmh#Os zC{U9JJ339x5&!sVl9Mk2ap{YMO!U>C@ic-~fHa3XN#l#Z)34%tj;JS(=xb`STm&&k zkCMUuu7VOvQv~JrOdIqS4Xrh^B)P6EKhBeDl?}-ywV%J?K4^xFtKvNV_+!@(&vqbv z%Z$3MF1@dMC(=0GOn7r&K(XRbi`9bp{)DrZ_27BZzH{L22hNQ-(PQO%zQxri$6^i9 z&a!QNt`zo5YZ;|Sp!66#?ABvA9amCTwl3IAv;2MaWD@=2Zp7lB_FFAG%~Z}j*I+

}%s^E3# z0_|$7u?(a74)2@_t}352lB_e0*!g#>9sZgkPd&wad^THVeM>UJj=;tPqJTupHLq3B z&glrEzGOUpPcCOD%IvJ6InFzlylLQ8?p4Sf9A-`#eOZjPlLxErN2K|t+hpm3*05{x z7f}}LMfD?M3o!}+Z)3djf;Gck4SiD#(Ok=_J=$^`R0iO~)QbSaJPq=Vx?d0cPL3a1 z%!sn1CRfKc@nculYVW{>MT_pon4&AhRW|%g)C6P2GIn?GEzZGIz`GpmSc*26YdvOj zJNSc-dc>kMy_9D@)k@t;qKJ9y;B?UfI1z!Qh2#*3fcjGLa&OYBUUi8<&9>0IgG2mz zP7~Bg(dHrtOPQut?mTjMHUiJ9Z_*soOw0|l%<5@Eqc%dRfS)hxQ;yTaR}`9#x33C} z&+NPDSma;J)if2f8=F|&2Dl}-1d@00vl@%$>?VejpJBlhlgQC=2LJ)U4!!~@*ju&r zUe$z~gstb~i$ui9%lc5dM0_rpzBl`Ec zAO-^Xsh+j9?yb1`Uw{0N5FLgsJ~Wxgmm&1dGr;n7#Xk(7c0an}ezJeZY@#LyGeiV* zZ6uHYQe$9tq>qQ)5fb2JVz;m=_7S}>4fef^o(mUlIBa#|@wEpub1-STTn`F*ku8Oe+#;lEA zf$$x6d$UkRSCAS(0AH8MMv;Qb-3I41jUF|6-g0cA69=(~)nBbJT#|8zy0p7OpbJ%h zPJn#MLf7TDU>`}hJ39J=etC;NrUbtz3NZkRI;{+IIt@I?PzVN^)fpe*G3q%y)+fK2 zPLm;Y+nO+-c+DL3sQ*@T!J1iC4+RyuYbwqn6SXzNnKe1G(CDc=<-@);f364OVb`)w z_f!;>LuQ0x`q#$tQxY@L`|oa)IWvlx?>Ed|9OaEq3GP-yor>EBS%GR|61q@{1%Q(b z-)KZf+)h9@f=Swz-^~t^3i?!E(kN+RaEZgN$EE$)3b1H=p8y#p?+q+eQ{VG`d{^ad zRjv2+v@9UzgQ#M~snp7n-X4!TkHcVkPui(HuFu^F)po(}Is^ufS>$VR$IN|KS4K%F z=vYPIDYmg(2)s8qItJ+)7;x)&hb&r~nNj}uLALYJjNhPZ6qdJ!PVFY6pFBX(uJ}i< zT^PLm4}KI2gz#D28ksDba9YB)vJR@;CFN)$_rirA#HIe`sX_L}Uk)Uzt6cuU@gZOj zRU8LUM?D+^6pi;;G(Iha`9^vwGc~@yaff8V_yE$j5d4YaK}u8?4PBC|!9ENg>s#oS zgn+8!U6jR%;O_iv?{qUUnoI*Dr~&r?1n^a-Abp`!BGv&hI8 zPJ;cEjyWf@x^a=sw0x|z1;KPGmN6a!xfJ-IHH_8z&&x`TOmlOkJN7z7gSS^g|F?#x zNp`haz4P*KL-_Ci&9Wl~wCSeS7(y|rKeQg434AQHl&wP_yuc`@d~MpFP)tynDY9I| zN$1_%#V2|LQ$`tWqltu@gO#d`4PfK&pS%AEa2Pgr$l@Mo_;jqStOSr_&Mosu#vjC6 z#+THQNr&RGi%S(LE$x3rkuMFR+5i-U2cbym{ilTx0;3J22@uQqy)aa7g?KCBPY61b zjt)%gtlG0}VUrv8`Cq_7{(s2jl$374D8kLAt*tE)&>fRF!hJvtLdub<=%TKn0r$lJ zKlEbZ`<;PTCJK11n&2L=awMP!NIah1`f8AZOud3|2l(rbo1SiHBYyW}u*#8>gOl^$ zsdsodn$tj5L7}M)4|ecRo_ITA(|X(ZgV+KQRbRe*InV`s(Fk%`J$K6UD?BAQGjnrW zpwfBH&b|&>gOqlF7^Caz>l0_raLPeXN=6>-^^T99AEAOE--*q#tc;9$IF4+<+z(8* zoidkw4kU0n5|ZJ-F-#7@^TFH`F4sUl^3=j&oz7lp3P4KrXqksMk1!LhS_j+5FvBX& zeV@DdnJ*&1^GxZ~(Y}j=!(iR_?>*_ewGdPU+)ES+mid<`D-dk?W3w*_r40>EN9!V& zoVNGEuSiNlAcjZh%i2mzzZ$We+C47DTwHc-VWp1vF^z~Cr{eQxa7>yDbcaH}_ikN- z0GD!Wf~f#jv$vi4G(3af(|4m-3BZ@P)c2t^jYoiJ zkv|iMC)*Yi#gG0FFeeoJbU@tRYs*Ny4keRczNqWeTg#eMk*;Yx1TX5Sb+gSI1ea4Gq1*)@j@}OGyTK>dy}++it+%^|t^)+JtU~XR`wPJYFQL=VsNCS4 zszNgEu7oRF`&%ogRGr7hp>ypaGn0>3(4nh?VQ)dzk|e>;t1>-ik88xtul=@maSQgC zdq9CyZGQmt{~k$$jYfikz@?ia{dv@QClk7AfT8U8>_{kdO3_~CYsgHWde(Y@N?21f zuKSna_r5-|g|1HM?P3|%0O>pp;P7m@{>DZeSWIO00EYkaXTi9v;<2hKKB-UnAP&1u zUVRC|k2dxfWHNf-bcS%9Yfs3~W&Mmy*{!(L>Mb47xHRE+39ssmo+ee;+c)`95r7#WT31Zx4c!D=1(BU@88EjNW9q6 zpTA0u|9;7_!{RUVg0|yb$hPW*+^F}mOaNaJZ#%PIG_Mm{dE@Oea}j=|IN!@+<>2Ze z+jNr;u@F2HPl8xer1P3T{|yKBy$2tU9S?R7$j%Q*`og-#s!?$k*v)6p3XBFsJSheF zj0k=B6kJ;Zk6^t3Lp&{D5&y~F`}?$r!vjglV1X_YbV(-`{3Snh=zB7t_rV5dfmzj% zisk^Nbc^kMfJA$Lhj=qEw>^TOZ_AS-$EKDRBO4Y~IXO=_0XLx01c=2Np*ntr&F>a+ z>xAv3;-Lg6=YIcw)A5PDvC8v`4$hS;R~Q&%?QbT;#|QiR;+#Jlm-fq2SPC8}UR46N zV)M)c=Dp6TgnV2-5f?dKxs0vZbu9NpeNs`CO8#EJL#nVB&NDn-rU5Uz**?_+}BUBi6Mm3|AFm`G;+U_e3zZp<8CAzQp8CMGF< z%Su6s8#x2puYR0b9kVKOdho<6Dhl<##Ri1O3|4Yeo}Adh`Cww?&j9)iIhT~fr{?5M zK%ybZO#o2BLcx#;YEJ4s7kr>1%K^gGwfc$T2|Vv0PJs{31Xw?7vw0}E>{dU&2cYoH z+bI`;0-WG#M=5kd5^&+#Q8)M%)z!1>bP8nEARkMb8SmVZMg(j`Z@6H=MU` zpM&8oR6+&S>$(WZ&RQU1H!Z<0U055HtEx6Fa6P;(aJ=mX{sf?^0R^=cGH9B6GqBF% zS$o2l3Y-sZZRssYLMLzxQTYy*XuE(VQEODjP<^;9X2rSQzC8h<)kDZz!%n&CAHv!S6zlYiD_K9B&+%*bW@?&}i2wzV z;$RbJSv}{GsA z^L|58i z(w)42a%mPrbVXs{6e8YfFH920pp?oriz;@gb#^~AL~*^%m7 z{vBVIOWke6JgQ$$J8q5+hU1{pJMvoXt?|DO@VGnpRd^WGKZb{AGOJWLo~*j`jN5OfXpMX)H?Krfa_a{OM}4CZkrikZamG zS}UV3T_w03i=N(`<$!2Bhr0hjykM@_dL7?VG$c_Fv{hIT0LKma@HFpVL}MNT%j@fe z&B_W4JB@L4DuGqQA4HGJ);ncdV}v4M%zZR&A{ z>6}1s{EYzcq6?eK$G zdK`P5shqIyom`BH7hC|-?;7?FjnB|yr0@Q)?sGNt4c_&NU_MP87AR)FGUgs6ru+#H z_HuA;y+%Q?m!1`Vle?6tHF)mk%9`t0)cQUJ!e|BeFL~d|i218W3JSiWZYrc(t{!!D zmsOWum9bGH==F{;GYD@Aq#QX1hIDL{bbLp1mE%ffEj3>+xjGBtUjM{CgAxr~uSbv_ zkO2qS;b4mI4SiWpGdG!t^3BGS%PLfjP1lMC4yFfQt!3txkGr38>@GiRg@Zrx6Ayvc zkgNcA6j|mOgu1v^ZT~&p0S?EF2Rph2PCIwu84bk-Xwkeb$2-hYz-)Z{SQ=zKkAQXv zrp_{p#7D($@UQ%FHKoBB4^f>gEkPpzgM;(OGw^2iQ_pQONVu%~(G|d|pM*}&qaNM* zpxxkIjjVD-LsqdtHVBn$YPgt|Lhy4?Dm^L4$OMlp2v2tMvP7G|*_l$0Rw%yhj19*^ zDJnikDD=veyO*F?A$h)XiGlXvp|2zM8wQva4WQ?8NyO)E*Suxx& zQKCy#2a78m08HIOn9UGz?k$|?X6Oh{XykxF9U!gFm!;k4`xtrvpgLs=KTj1~y}7<#hZx?G`~o;K0@pEc4W#Y`5WjPYT95)BiCBJ(-Q8E=H@Kd~ z#!8irdk-zbH2kUJGBN`U*5ETmf^{KI6p7jf%R83sRE5DaFnq*RIG7x!F`YT@jX4D8iBHf4$J)9R19;Mydcu+QzZR;9*3=oESLHAPUS%Lc-`P{@8fX^5J zQNnDSJQVm4#&8Dmqi~L4z|@4v5*b8c4kX!+kB?Wba$q+-g#*n8AZ>wF!SZ0+kU=eI z)$AY0GaZ{O=4M5l9FPFMXhKKe)D`*nD<^Ot@{ytIKv2#1?gexk1R)1!L`?>2WAKgf z-TV>USDJ_06(1J>xgZ%hUvWKvP(h};ZNCU`Pty^4-3}5VcoHhs#KoPs4MT)h!_HCw z0#ok0IS0DzJ>Ux`P0v`^V1mlsj(LD94BmiZ6sV&;AsbYb6xXk$a$simwGXEOoY>#b zfLBH=f|aNpvLDg@`@duo{>Ey3eJZB7?;9BbHi29XwLs{0r`y6zP>*1Ggdh~%HO)@U zYgM?`B1jro2S8j+v`7f^s~QRr0OO2sK6O6nqSQNbaR1E9NnEH;`NYw2Ys&;W?xTH! zuR}Tq|Goj8cavdzauGs)Gz^B$;l171LhDM!w`|TOI$*{m16L62O!$c7PG1c~R<&-- zESk~@)y5Z?e)57AnnhFHsPydX+Q43qvxUS4u;)<#ztL9VQ%E4YZbcp3 zau9>l9-cE5?Qoi4pK7il z`SvAE&eZOfn!!D$1D#u3wQl%unizc;5m1?CRUA2o=b*eXqg-|{LT(ae{!86%%h*8f z)2NGk4o4pT0{7xL_2JVp?eE-4b*1xvn36&{(a{Jg(L2?2;;U*w*BV5KQa`;L(I%6P zLW#$|_UHZl?o(qlncj+ePQiAqZm1OvI{A!mce`X9%y!J)<4_2vMg=3tGvjki9Z_#2No!uuLwJ+MsjWo(- z&0~idUi+dQu{;10@HJ0NduCWZo%sgsiWOE53-KG~_Njt`^?)nChX)$9Ds>YAJb)|N zdg6??S*v>&og?CC_F;o_<5Pv}O|P(3p(N+n;y6J4G}V^g)qoXmzU9o5K#HN}-*%Fc zKA=itSC&>H&FC%L=~f2u$235MP<8cQDtkLh=gLNRokM&(aZ1b+-WTZF)7{OuO3ZS7 zvfr>B1q5-X(^gSZRRf&?AHrk8_n z9vr6H=IGXMdad1qgDyc)QA&9jo^c{bRkYY$Ru81EB?c2L3I&cW97q$WIkNK31O#3^ zlI9_FY=aBAM%eJsWy>vJo!R2u+ZnKgLmwv}J`71P)BRui1>XYam-SRbu-<) z2}rEn6?y9LNS4cg{_pDWSk_$R-%|Xy9{0QhJJ#Nas+P9BGZVo#vC!&sRB^4i&kP!{ z316Fg&OP2$CVWHSu7;PdT*MWPzyVh{uGs1IB|gi&ukK0M!xA$qvHT;uafKsH6jvQS zG9Lj9K7pi-%hwTu6qHHM%rhjLlcK>C?_fsc+(cg3>R3RBR3~x8e@}(bJEXClEgo2B zAhb#KEUvK~EiIf)R!4b{c1O(H&}i@@o7n=Iptjvd5@KSp5p2iT`+IwAJ-}7~LzVt} zU&2Pn93nML{JGt=2`?rz__emSrw<-6t$=!A66V?+X+%C~^YAe|1q5_vt#~0%4on$M zZ3%4J4X1^eiEcTK{ff0?1z85YTO3z;y&k;|_rmhJ(DFLGzGZCXXQkI#4dyd{m-A{l z=5qmhaAag`#mDbVwHG8_uOaA;L_z!wdI-H zHur>g$15-3JJ5kw5X{$A^pQ>8-}nmr*FxD0;<45i$jHJn(oK8)qwC^9LP(6t!+EI?fzmKqixPga9rr%ZeQk&WUvrJ4WdEhpM%1$7j&mQGM+B-wxbaWLyuPERB{A zY`;1+Cg{th-2Q2*R+61~iaA}$k&avZs6yq}m^;@lw#4dAnv8QN@WM7t9 zgYN}?Yj|mNK%k9bZ@T_%U3Ygs{Vo$s6MAN0;b%*bZn4$eo9CYgC74xvW#6RNGB#W- zEjBv$r_M9up#2rjU3&U$&yl_?;%xU*j2oYeL&f3%>+#a%KlZgtU)O5L^UBJGQ@U_* zc>C;~_^Zn;l7)=~oZPK9CObdWa=#%@DLF$XXmiMQOrGhs#4xl>a58V@gPT6m&b;lX z$eJ}U%N5e0B@xbIyR-X2&ZzH>s-53jZR=`&#!8*eUvAgQ(}L|^7}un-VY0*1rJbgsd$Dgn)800#R_6!~#8>>4TYIGH z^LR!r^I7=1pV_Z$74MVQ)88rKcMOc{=yoe_j13krD-_NHtf047z;wu}yhF%ftfHRJ zLMwCJg;Sb_U@*#NQ9-TJUZ!8o+}3S&>v>GCD!-kSLKF*un+7Sln*6<&fjzC%1df>< zx~(O>`&)T`g1D!t4r-e8!>6Z>mSo(?_M|iko<2Ei)E;@kX|+7z)f!w1se7qU%NoCp z9j*!Gp@paq>ISOt{hoW_a`%J?N&o7IUP<8bYt$ZaKWar$@^XmO9-A5z8(zx@B^2`U z-pf$T5(l#V%aM}jz2Eebu3w!#ZCaZ#;fg(}J9~Xmzn12{mjedt&Qq2U7y8wB;M?tI+S(^E_cIjBT{NMVpir`QCYq~GxPSj1<#Ca@ z@Mn&9D{@6#&cF;V#LcryZjt(Z<8p$XYG64s+PBYXb@upL?goH#72%|s3Mrx4@?8Zn@ ztFT?7Pb)MbPQ>we{~Bt+Fc2tHxWJ)Y{HD7>mbNYsB+|wJjgK5Q&Z|xZBN>Ur*&~j% z+S3}?F-o?;R#Fy}P?zbksZ55x>G4_5OMQ2!^T1l1sI9GkfACz1z`H zRSJOxM^kzHo!kuz<*`|6I>NOwRypv(c%&o;LgZN0p8k%!Fd5z6-YyT9HM8+5!;FD! z6?u?=zpz;MinYOh^fV$YED=shOF&%^w9Zj0?FaP3b;9-EE$?o0gNxS|TBHw_a5=jV zy@l61}l##So_}EaB$KZPioB` z`;OOF=nWZx31_*}RaAXPY74OMUR^v?Hs<6rY#Crx_PJkY$>;UMSq06N;XBrE^PLE{ zZ+x$R^k}4U`VD$x?#GnLmvZy#p5BeGh_4VZq=;CN zbck^2NY}x%&gJ6=Zk85He5icAT_Z0%F}p-X{=VylT^aOfpUZ--43SHzj^jdQ&mTl5 z6nqzLcDs3rE4*PxK+w7n|$e4R#Ccs$*U>Q~CG{#|PJE`zxym@LASk|;+gdjbG)~l+kC#mjpA)tz zxzjo_(zUUyf7rTo#DI6J%ILg5#clisqSa-(H{LOt#6x)i+NX& z1E$TfkC8N4tMIXT@<-2B?4$3WOT|5;_XEh_!by5p)qY2Vlxne|;u47uHddaJL zc)Vd*DD7Ily-K&>Mw-^1(`$?R+9p;cnHB3s?~W+!o?w=hOkbgpj1kiPSHt^GY2ACa z|LUTM0O9e8A+vr^w#6-d+$2!>rMw~(Jm*Dwabb8nJK5*4XElF$%{7(f8TaD9emh+8 zg!ba~`K^^1ahj5^##??}*mCPjO|jeQrz3|$lB^cpMaWbv(BpMZ&X0bgr7F zqpW{4wMG_061{&Gt}7}k5@Yt@GIJboIwX?P3km-sIb58xBI)0M4=>;0`838Hd^)16 zZS7MV+fR?9UYzHABKkgr6@$RFT2Il}rhCs&NYU$n_51rDaWNMA*$I<{vQzxx@pBg7 zQYl=>)sE;~n`(V?)~s0)b?A|^g9mB^mnz9&n$2Jq#kvmrlQxOh_buvDEUyBX?%p(& zE`D85P8Yv|!o$sVw|n+81ZCY*b}vy@o40)ViD4gYb@ly!;%Z5YqtRPT+_@M)QN2gW z7vNtS2XoPh#@q%OH#8>$y4!sF^u%$SQ6?^$^#6vYXK&h`dQE?oh;V(?^`%OA)?MYX@BdK(MdNpIj>E{i?t1(5swA@nGz3Ou zz#nf5ma1h!=t;_?Qr+>*E*-jtW|?fc$)Sgfyd{J7+sx5mD6j;j}j{QSA1>qnuw8$D>UVx;E!zqSebJ?#Y( z^+}KY*45@dB;hh_hFg!n z+vNR-2vy+kgqttCj~$A_uhxlqQJdwszc!tI__9#y=dr>JKC@COndMXS zc}z>C$^)F16iNLAa-l@kXtY}xJZx<={dUV+u^9XIgXmx&gk@*<+rryS7 zZjaF9u;edPSvYrY6_Gp7Nu*)si0G zpNmT9#`5jPwitaPCibtfB3*mq#;OaSFf)%|giA$9`{LqF*x!EZtW#^)=?)gx zm&upMpQ*!GKyqBrFSl3&fKEt6TwD{ArPE@q((B}Ki}}=P(*!l9MoI`QTxg4%ZWM>) zFI>802bOade`2;)02kfj((L8s7?|MoJ!)WRs5(YHQ1)QHpQD4qq=ZL~d2C#qu`PtU zi{)66$cM0DhYxK?e59$y8(cdjNZE9 z4(E^~oki={#nXI;i@yY%YRu(duQlj#_?J>n z`-jF~ja|C^0D;}=x~HD(>B&9);pA$%6|RsqO?PMuxps-)?zj{m>-xWkZ-N+dFLW=D zG4Q`E8qxJ>VytaVQ5itPc&FiVfq(unkcwEu*?&Qjjg_0Z{%l17bIDLW)gvnXR-=M~ zHKMz;RL1G!Qv6Bb#{^o8L$@pT^d75gcLm)+SB zKIRoahdrkH`Q!Hb-GXQihz$u2j+L{kLh&uAp}*r(w`)#+xuz}PWvuyu0}QpR3p+5Q z=VAco1JHiR9s8gR^HC+}Z-nl)O}PJ39$m(JYV&;a6ohV`IZx4C&|F}HNnW0PrCYpj zSWmpdg%z^C6J97J3>Eq9sYq7Ghp2Oy(|Cm0{ppD+nu{*Z=dybJ=8e%?bU1Ev=?uEN zAzyj{!upc<*w|s&*ePPOLW?bzw6(M>(cq@?D-aW}Vm!u56go_cTrVRFpW)b))Kq>E z9*Uat?+^E@ZunyH5R}z|0t2JrP(Tp( zKn*_e5c4;>umk6K??WpANtQ}uU*?2uJ`o0l$nf*?bJQ?BWtQozlWxCbWLxUk*kP@R zi9{)j_YzF9(#0~y7Xt5@#55Q5C{#VeO)W~cEk&s!&Fl4QMTKAI@yWD^DJq)7Nte#< zcFlTwV|jwm4o`)?vMa3SlDo2!7Cv!D$L=-0F0?w*a(GOJ>qfz(P^No(d|F|cOG3O= zLuatQFvTz!i#Vuc8a&pO=c-3%>kNr{{ZM;o|M{!A;MliI_`p@J+g(j7^Cr5I0!}3u z+!ovrdC2Guw}!-llPZQuV_w-UrCYWK7q@BYlvVFpA+k5%<;$@Fi%Ic(BlSKH&E%Y4G+L+Z9U~-&Wt$j$PEg%lM3atH8Y*7p))J-_q;0ox;$%PF?s_qgC+Y z$n)EJF)fQW1si^oL!3NPB-(%MW5(&Q4-)$Z&U!6a>V<3Vs^WeEFygiUH{W^GIa=^g*goCF`u$jT`z|>8{%vl?PL`-lGOIhN!uddAQdtdHa!)3DMr)s+ ziY_}GiH%U3?U9PP19FaE_o7T}r0G%&L6B8)n7V|r#}1t)8rs@}HLVyKwe9pErf_yV zgoROj>()=ox|FDiYnB;y7A}>#s$q-mF)}gQBZsM`PA~i%7YYd_S(S!$fMpaR(*j)D zKfE|7*3^Msn>{{0J|R}u(IaPD$qIf!SMD&C3bpj|J>pJLVpoBSjsaAWSdJ0Wnw_|b ziw#^NccNCdlCl34`8vI2(2o)UJWOm8jnf|P6sM_So}eA>62S)yh!tA4?9q}mmZLxFgGHR^sRQh9^|h-!4#vo*l0Bl22|l)B=xS# zyMM$RPmzB7IMG<-9fokVP(Ay;{Js7n@kF^pD&jkaOm~2m2q|gIxrf?1CuVXy&zD^o zsFY{WQ<;#vR?GIp>XeOl?_Th~Y7%+qRoJmk_U6t;rQIJtMv3%!c&OE;wuMdf$E(&p z?Qm4zAtOV>e?R2e>LDW|^Yqxq)eSkPt96Hvo#3Dc2ia#mA-~^+)S88XTNuwjtx;zg zRm8&e7yX6Kn05C4%{-x>^<@`>56!0-;o(QI%XjUX(%CqXm~8Iro$BaxbzpK#sj(|i z=i;3k7iJxl%mHsTExc|s^2R0itEq(t6pRm- z@o1^-mP_&Iv&`vx|32Ucue7W|b&^YQi_>IX-g4UR-DiY5e8+BnX6ap$llyZL?#Cer z{eFdCpR1IWS?S6&IT5PcchzNeqVl?O-k|;M9{uO`uevZ=SDCciuU2iEsMsylk>JAE z5|zZ7Nor+GJ#(sox1ZKYZ^75A|!^=6hPwd1EPxYwLnKl zM~S{I_VphcCkD5;^n5nM#rFYuN(BtArcksld<4(y0(Rs?L#95utwTI+7Lc$9`t%4n zli{bo%MT9^FVWzoETeJZ5W-;wj;Dq0D2_vAnw{Z=^XC`un-&BLa%-{tdoU~OckFmz zCCqUC=9(oad+eZ1$^%FxK*V=>rz$7|5n0|34Yd}XNl|&GC_czRmNd`)g zLd8>?3VOWu@^dpmg!EUO$}`JyHL}G%2Eq$Pb-@?QO)~&I2GX&SpR-n7DpYfmRS69p z=SU2G@lM^fawG$x$oYA<3lIsJT{tVJQ+7E9nWSBde)!P66r2!EE2F{Q?D@%^SFT*y zhI=XfuU=g-dB<2peUgKMj(aF5`Q}X|Pgd?( zwW>bV-$Lqja_6#WZoeYNww_BC-f{*uHXcRA;voRWZ{7&WKd2n);z~GD@V$MG?FN29)iWp^x(rkVTy!HnlR5F=06n`BwOf9AaG-(maOA!9e|A27=l(O=0|`C!M$es@ z58A3SuBkjI6J?iX?`d?NncA6o-&W{*^1?3@tZv7X6?P)dt0gU#CLb+6vTcT6yusJo z2BszK%NJMKE?RPI+ia>S;{5sZ$55~N-@GZ!1Qa$h?hgkB6< z0S^d}5yKev zOPV~B4O&34rs1z55HoNI$u{TqTdv);vDw)Q#hU?92+cQp`L-BX30R1wrU_h;8cP)1 zA4{1&U)j^Refk0c0UKN&Er|JiUydpd+cadRq2W3>BEmvfbMM{{D0%tgxWl!95L&#v zG5bGbL%wRgFTakxeNvP#XuqYX0xB>t>*>iaDh((`W5ORiNJNu(SFoWuklOWR+L6TO3Kw z^v85=%zwyeot*h=xW?O-$@I|Bk{s_+^m6~l`^OtzmZ5aaqiFrD;GkWzw1jS^$AsV* zJbLs}H{GFHN(18BS_LLsq+;MJBxZ&tvOJDeBt0NHzCJ21`t+LBGdjY?JbKZQfbBetFx_3;) zfwMtI?Nt{_R2~$IvV$M3&d9mxGdoD~l9uhV&iK|Z6SfOmlQ;NY>e%~h>iqAmc6yE- zi&pS3Ocl&+#)FRwoQ}C6$D>t%Zb>j7VGPVQtIav>rY?!OJ}BENQNkS96?Y9WvIA>K z&_dAt`=+{OMMb(`Y5@{rab*JuW_U4v0SoNDE*OjQ%NEZ&)2Sfy`QF{TFNR2&vB0{5 zYdJKFpI?r%NiV&M0t94W7ZTRO=hv6%5f>FPC(<~}HIa-oU!{7H0m15GId^`lJ_05< z%>6TG&J;<*ICoML%3AMA`{KM!0Bj86c?cUnGY*$aob*`|tc_s-mR^XI2!is1*UXUfo+}Bqqk;5r-XD;Q}S=cqu zi3@!UT%Qem{=D$Z)#IbmBI-plFg z*wjA{A@bh!rP8*O6J6U*Z#Jt=wie%>wHNyo5+I0$edD8h?8bhDKuEOBo#U~Lh-$TI z7Ixz!ytP?d)YR3P8rtvaR#s8<_Q{xgf_c^)wG!&avHtefl)8v+7Y0Ci_0N`K%t$%P zA>GlkcSLK2!ZUE!F5eEABP~--_G&wLW!ku6)#?n`}NX zP@v?hAEEd$ZVhMW_=&x29q_bK;{CJ1D``=637SS1GVd5M5^B^mq)6 z{-sxdRG&Ezns@GA8tZJdTVKDK z@XQoHYhfG-*tMz=vQ|7NxtvgamH3*=0=blH-j$TFr?2ojXvD}x_9Iyj;CMLmRP#k# z7QRXI1UehRH!`Dkz3j-Wbdz25Wp+>FE7=E)f47zy?N zKqrto26?d%%#VsLp%gN3KUSh?hAz6rf8MoYGj5~!q{ZH3`r@Kw4J|FnB{#u0z~U`? zhX5^0ksUkCfF6i=1iQIVMh@6A=rM%-BxFQmMBEpKA~tXuu3hm^sB$^9RQ6aTa2Xmf z-{Bzqx?9~<>AOrb_|lC_6z%)B%^j2Ni{GUEF0~NAO69fwNM;D!UwR%h^>_Iht4-M( zkTRL0THe%5ckoFw7TwmRtnd4{A!X+=!v$MBB!Uq6~jQpCSIu~w9#ufm7|&*7HP0Sqv) zL7^49c|_iKA{%KGqu{slFWX9kkziT?LKl4gap5~o+hS9Rq+z)`Fm=+lJm$&^q*ta% zdJ-WJQ3>m4gQ(;RR&r zCm3|V3n=0O^$1aem?AjUB#BuVT$ND!an z0Qr~?m~x$4SCu5N!#;e(zPq@I^Pai;m#Om0^xez^0haswoi!y%iWS5|3c7&H;UH6} zke#8f?(wML_3PK75Yv!kTSn&7HK)&B_5qFzuTC5+Qu+XMIjj_*ex=kLpLMD>*ywdw zB}MzJa^E%S0(t1<^$sULCr%j?y^}iVN#bm zr^;7uc&)Q>mCvlcn(goT5yA*v*wYJPi{cZx!O-gsRe^YP=yM-@=_ z$Q}Rm;oxeD3YWx6AZD4E-y2tjTq}3<-Q3SkJQVGsqZ3AX#z$aCR%(nDM35tI`Q+^D zKbdZz%I@O?;;0Uc4>g)}qS_~GjO;y2LM4Iq2Gz~&r1!2pc=TNVN)3&>7BW{MbjHi#d` z@c9&-5Sgl>(0)DnIApz_zvYX&r!#Mx97kxTZyz{!KkTE}OJ}}6OYV0_yMmJi$Uh6{ z-N{q!UcDT!SyWwY(A{NS@_{XV{w|w_;%{#Eu=n6=lP#2Dh7?C-?tCAiXsTW5DBtol z)&023seb$LXE9$DJXI4Mn*RqCVYyQz#J=B#T`h1w8eg^Z}V@ z%VzACy~;yH@kq&cg3E`-E1#b_vMGuRx{GqOW3(qPoeh9Vh-ZA4#Trv7owv$A1xiz2h96T$&GXoLnR|%=x0T_>T3sVBRmsbo`a{0~t=KFV31bqD; zG)R7a%O}SuaAFlZ;+L6#I0}O4`Qf60!gVm-nm%d*tS_zQ_r;7%NPlU$;N{A4-`Cf zarFa;5||tk6jfAJ<#4iKBni#L#X9Q9*&GX8IW~ur%jzceW}JQO8-)Wy&*hj#`jUcd^wd>aH!GFpI zew4)_haKbWn{Y0=7;-Sdl+Fm!0ky!=>^QBZ0OdQr-d3)hyI?^K4tf`gp5O2oaCVv= zmyQ%j6`A+%-q8#ULhua;%E0`;7-$Rur@*bekVQb}$9{z&5vM(^a~5$XX+V-FBk${8Ccz5V~vAigiAie|?QBC?Vvhs!lh&&}D3 zjZ=lN>e9(xACW-{R$Tot_K0FURvG^mw+1@Jf|nD&r3H#wS!lD3oK*6P{OMSGg0<*d zyyuH5HQgQN$uhhUBP#jEwbu7;aOK}}NVY1r2?W0t2FdHz*OOai?d!J^nK|Tw9g1vn2B5E3 zAI$7rqo804s^$auP7DFk*VWP1u0pbxgQ-UFIOmh~o=3^ZvcM!!95BPu0zLv0UN(-K@)QVky{J63a1H!Nl<+djp;q<5@Fi z&@ez>T~c1&2nk&7&aB=R#|#M)LsS7aL{H=L5jrR>>@92g2q$PHlBUmT$=N^s%X)T$ z$#urvuW=}ZjBOD{$u{`r8g`0Hzi^JySR~$Wn2pb!h_AVH?-WYsJNfM3^>E3QcPplP z%N~9`1FVkSm0zQI3yCNOottVph0+RYwcc;!neu4;&y|3&w%{RZkt0!V7JfMyq^cX~ zNalQ`SQa~D{H|sLB*)j6=fRwY#RJOitxIcsRs2_lzeT;g}Hc z|Ipq7C;zNBw_N0*q3C#ZhnVwe{iYwR58e778z&|D-SvBilp2Hz#gntkbyk=cS_K{- zDY&FHJL{`M`uhAsfq4yUeJ^SGAlx+d_)e4Wp7mpN_*Ki}kB{cu?AEo4GBAA?)b{Zn zH{<2E%{D3(8{}s8ZCO5*>UAS#0(@qPG_-`|OpJ8>9J2N`GsgpwgRTV5=Ku#a;S?t{ z9KOr>;0oesD9*}( zFuWI@nV9~EAwk#!6p?6ih7BaiA}^^-X|i6o>vK*PL^^xXd9++)gE)ZykZWdAp|Aui zISTy~T!ck}UFie^ zEQnU~$f$yRX3hr|5I1NCHVpAZlM{5)m8wjX0T6`%AFARfUh+z}XS9WHx;29ezXrgG zTIVTl3y8>wj9jU8XT#3xOSec(9u#;TG10%y(rThU-xK#A5nYzEp1-W2?`%8u?|Fb8 z7bKJd&wbdBBq`#a5Vs$QRl?OM3{m{N|>z8VN%rIJw4Ul+fk;0 zv!Xr?&n~X!em=xO^LXB@?O7e12haH}c&Bl$NwKsQ>)nxUGyVJ@zFU*{{1y}1a)&78 zm$wx06UImfCT@2!1fP;b7dk!pM zCdED|c<>;w(HYJ{Zd<%}2BmKZMy-FOE4iyB)B+P}pOn3J z5Qqy(5FW1{l-F9g$e>nrgFi1_e z1alT$08Jm|KuO;3Q-^vQExbqiHwMk3ge*KgGjY!nFJNrBbrNbc@}?v)C>@v@5E9a= zclFve*;~Zmg((I^Ai~ueQTRHBgz`le5b_*9jSY(c9!=oNNU4KQG=?|RN2{ObH(r%hoA03@|NZit<@Z15 zQD}8z_s5^SOCXlRO%!>yNOa6b>2?pUVW0$d!XYP%8!a^xpFck!Ztw`#dEAboSL#99 zlLglb+-uZ{(P7m{jK@kioMFB+`=J^yr)2}}8^>cML|s6scwo%hSY*J-RgVJ zHVQx#x7bK8Ih|vU+NR{->ZU@qJ$p>v!odK@84H3hqu*0Ao-<{IT43@3%PZOdp%Ue@FYCV+|(s7FA+ zWbzMf&*(q5Zz9^4n^7tLXC$Q?EXi7rD3FjH2x% z1Faww--va^Q9@d*r#+ss;ci7lM3jvo zjKEQ~!UnJbbo#XU+{%>#0s^r(4z^^Q-MAs;au0c_Bl@OdIU;K`m<&hl8CVfMJ~|i) z$+V!L1tfmaR!5HEO5Fr3WJvAD=sogoS2Vyayp_R?lC7r~3-}keq+G+0>0d%qDY_=` z0Y=RkvXjiVM`U0K-)`)_uWryL=bXg10CYqi6qNmdwC8|A*18)9l2lY>1N#Ulg4Oq! z*X&3Y@&9Qh8yft7tz=?ovy%Of=;+6i29=EKLdq|#@L`lGj_xCQJtZYY=y?t*eC!dh zaa&=$*iOza&f1q$?HPUrsSb^bD4gE6jUP@hNR$QS;t-V6sIuMHmCaGydie7l%=@c` z{JK)%%z4V>H$VUO-UnXGb;LI;=TR1MQbpoI!eM4B>e8x{OurZLu15SY`2?z|VM4EJp@>@5TnC7f4Eny+h1JMp& zodKX=2&!BIBmyLHL}j$fz=@3eR>VX__Y#|R3F3{Wyu7?1Zr%j<6?LAX=nAMM%F`Qt z?3m=_0T?XHo5s3~eI;p>vJ06n#1oVj@lRotH^5TUM`GSVTcuf^PlI(5s5Ze(R@Xr5 z@(Cw=LYXFc3V5hsXU9LwVkMTqd=D3Z^_ zP`&@45NV6>{j960qcda^GMlajhDMRU=Z4S@TT%YruK+k+<_;jPmTK_)66+o()z2B6 zI65D;6UrUJWO%g_+W``)(+P2k^2PXCMzFZy$e8^y`0)WuO3ril?rV5{N)=_O?xCmw z^Liv$8`P(YLqc52YK{Y)`sg%l%k&w~S-daQeS7DN72>Dxf|z)^SBA6jD`) zyO!w{Yx$HbX>~0zKUPse|72;gS{%b}{0*Xvm0J)0rLU=8opRoKV<&mdr<2^hPqPE6 zOpD4(4Cb-sHEmcvonqzSo3}$?4cYBTwj%+JD1jI6GuT0D9PC2lRIBT;aKXDGD-L|C zso1yS_cAX2KBjOlsyh<50hDuii}QzeWL^2D1n(*9SVWO{<{J>&%33LGXCY;x+<6T(WMpMX%6ZQBOQaaR!oACfpf z5EzPAiO_P0xdj)khja~39BTg%^(dYvvgj13!VbrF*U!-wmneW9<1@46yEj*C#g5*iv0NP-@{pa`;@R-+!bKR`WIs z)ef;sgad{XMtt?^m-|o>_-VXr{?U#fNDLpO3melfOo6lm>biZJjsWMeaWhuw-A#M) z#=9K`0E9z59p?F2td*A5Ax{CIS;7VT6+oH+{S7`Lyf2nAdeYwBS7pEdLIeY+p= z^Uq6NDA}+}%6h%r3>5Iteh&}~2Gmtziyy#jVKYsLtizy9i3)yv!*LDQ} z>qNsgXhUc$XsaYO#ZWXkwOzZ~2`zw*#iy3V3(a!9YOL=OeHgVn+0f^zrxg#1xFM<@7Y&S7BjOtjhr;v-Pd=F`qSjaU? z8yhW}3vPy16qYlUEkhxj5i;?|YIUMsuhqW)u1wxza2m( z;VBdg`xl(WU} z11heyI^hxohymezA(TCVsTivFJK2HcUX8wll}7Lb%YXV?kBH@B{fP9i!mIZTISu2z zujUlJfA7;o7*_a-^GdTLzF8!~d+;t1^k1sn zaNx;Lg+`91an148EqNaim%)jDOVt9Z1l)+t!$hoYUcVlrk^}Bx-NTy^xm8ya(3h+W zxPU!C^xmJBA}HRO$)m6eTNnnQ0$uUMoof#kjcXb8{d{h|l=2&eA>V@VA|O#>!{O{9 z9!StMqydGNi(T}!;x%!D5Vu#y9DAJTMYx=` z^MTvvkH|}eUj>H}tamulGYg0cWKNG$3D`qAE|xF=VT^c2^RdmkFRL`gPsS>gUfd#! zL-+edJ9F}P!A|~>Zdaowl$aMV0K%tf(E(o(%^EdX{wY)ivWz*pGw?F%Boz^%F+~M% zTH;Y6o7lpDi@K*AoFz(V@>3ln@?*G)to z4u#`8IT0&0Edu^h@S5#`I*K3hZY=|z{K8Xd5Bj^Rb)b5L)PhT|nT*tM+PaLqIbk+8 zdMKE7ohw>JLM7TQHwI1lf1(%T6X0w=(lUqP@r3;fSn^$35alN(y?(tqW0$% z(ec;Bs29Q(D;lCHeEW1HRnMNBAu_)N1;}{RO(v`(xW4dV*{}?`eK1 zF1Y9f+aNMQS8;IKHz_NhIQca1$A`zf)HNWw52)W-P9AVLZUJB_bMhR`gTlvD-?fu2Bg2P? ziM0*)o$wfF|M*IG$X$U@FEE81#5pG~{fOcg47tvU!IbHc+HV+gd73EYgh~Nn$TO2G zva|2Ey4er%CjksVN{!bd-fX9~8^k7xB!e>Gm9hp^N%{+k;LJ43aJ;a)o$(W@p+qyb z_XD^tnS@@_|P4%ItMO@0G5HqLf`g0fC>fTR+%1NvXSyx|`{ijTSIH<*zfOyxXRAg0lp|9a-A z4VIIM3TB1q%;Zs)J0+5}SE>GC2JAIc5VBB6zmiBJT08_i`}8_c;H6fngqop2V3Ke_ z!9$)4iV~}{oWx1f0;PO_#6NTa`vM9^45N5#e>F#m4abU&<7~=O7>^a1tqPgV4iDua z`vmD{BB_LQNTP6t_sOsGA^0f8>*BakTMXJZ)Tu!nh-iE@6I&vxyrgtQIi`ftQhiX= zQL=t0!brWBagtVpzC0`U|%eczug715=piYq7--$ z#|VRo`xj0y_o%7aAQ>wbT*vvtH4`BuabXn-t&>3b5&_`sxpVx&31bj4K#m?~lC}~J z1UA|PLOwAzp(5*SaBHkYRt2%ayp-fHAN@$!7hNFPDnVi`@2Xrst?{p>0%`Gb~|WjnIP#c72$S8`{xHZRtd|oaPu@j8C|F|T6bmCAS|!}tepE=wIWX3 zSfP}J1!rLOh}MeQgM?9i6r>=J_F{ou)VuK%wmE*i42{Qj1MVuuVRQ!C5BNd4uI}B_ z45ZWDF=}|$?AZn&<~W`L6ab`1JOw)Tl)Y57JrC6@3VrMn2n#oesW}w>J=P%3C;~OGZaW`$qAA=pINqfDnd<4?l$ZB~d-7a2a(asfMAw zL4(@mnw7IGfZKfmARO2M8`MRxvDKzk5F49w`2?@>0Q;7l99QJNC$4|%!zLfo{Zly4 zAHT64)E~ZCW$`|s+IcQjdBdiUA(kQS@I0lbR^y zBnMu;R3Uc7^0~kLu|d~CN5SoenrA?M_Y&A&Nh0w4zLeVc6Djf8iF+g!pg1iiHZsycfVWtp zVdxS=dir1lA={uFk25F{hB;fDfRUzib2#)B%%wDK?e{$CSY~1gA^tSC&mMb@yuG#` z5&vj@WLNPh(TSSLaGAjE4+tscd&e>t|?zqieW7y7jzmn+6kz&HBmMc;7$ z!C!3~uE`4}ZM!bWNl>1T3p$k}(aN`390Zg&iNxkn0 zrb}S%+@@452*9M|xT!d@7ct?hg0P?UOPF22UW6Ec7u94oi<-5hEK2R3+@^ztD1dkew(ao8Z5HM17&IA^u^wM1|B4PHmfbgM$nX!gH#u zi%3P-6irERM?jtd1aT~4x~H;yG-hGYvX<4)NBexnQ8pw@C7OJ>|@O7=wT=PPSPD4IcZ`n_g@-;Btb=|S_21(uV8{D}^@+O{2A zeeRCtRsFTjjmZGb5GzgYNT>LCo9{$C$*~}3l=?Ty*|KSScFLp>167Jg3%K?%wW1ipGWeZb;HlyxvMkW@LjU8dKH5ml5|4x-yP2aoM9d zW&*&_x(Rrfyqhg%T#KG|nLoY*9+6>(e%hmA&b#0@KAky?Q-Ob~=sS$JZ5d%JEr zuJ(#i#Jw+^`Ddl2l5OWHmOsp3Ow`w$1%vs+5(n=Y>K0N#q=DoCw|8&m6os{5MW9KP z>1vWfaM~{;fKi?6L+b!VA7I;YT;Rk?LR{gXT~66gqiE~z!wWPz)RWF8z4m{!BZSNO zSVQtr(GMgs;O9q?H3Srw!?{B@t$Fw791}bQeO!2LiAJ=&74bTT`05^JKd+%nEn-v| zt-B@X@s!MM8gEFlg=L@%(r8BN(DiqFnY3(LJ1nJ zfH66o*fPNAnQ&}U=>4fpv$IC1pqn0=m|Zy|l7BU&pee`wZPtJrBqs9A@nZOG-8B98 zAAw;m#60CTdnbXaf|3AxwtjtL)5oRURMCAvb1)BH3vZfYY9^Ue5qq}3VzneLefVMH z6%NJ;m7ArN6(Q>IQXU{p&V;@NDIN-(RW^o!OC;ghVe`!~JGq4FEr#wy14;4`dkzi7 z{1K!#8(3mL0We5y@i>4y>$&LUGNxXShIG)Rsy2Qp)yR^?Takd!kg%D=Pmr!5r*MR$ zg!9Wl+Ld)(MldHHCB_&w8iN?4goR)0=8@-vQx27#%+1S>#-}9}Mo8GJFQnu}#yz-p zfuF@|Y-ywmP?b;F<6~pQaKag{AVc-)L$*SYAVzpaCsJpT0i$>-Kj#?m$Hd1McMbe^ zz6)mr9;8OHEjeX9DI{U82J1S<>l&U2_97`X$G^<+{Xj-cfdq_@+lRFxmQPF!2qNVr z3E;qioX#^Zf37%#OMN|Zs5iA6{K-$S>LqYT`DRnr1I75#!=>g z-(BGAQ5q8?9~0~c8^)R+Z-`ETch3H>R3lXQbfu-wUVh2x@G7d4z=)M zazYB$@f_?gsB??jc6@I}Q1T!w-0;L$mO$e!c!oAnT5kH~<6SXb~rWPT!T^_c2G zG=%9yB;ll+bTXMtV_X*i+JZAh2DI7;s0Gr8C5QVDvGayY8Z(^s0HIV6Pcd)=C!%7I z22uS#`;4AIp`Z!Fs`b5V*Z9abAr2%PI$zV3{%|(G8sMQRa3ed&nLzA#O&J&^U#k<~>+ro#{RLEY=?FPRP{Pmv1*fRe7;o ze-2b&4%9(O7U}#KdlEM?@t6Q`WAwlTuqTo^VKTP&WJkk|!a_dIPMRY21mveJ?w(Z~ z%Ejmv69lF&-h7n8#kElrj-U3$Q+})$hemDk{(?64K)sVTBV-XmpN*W%FePq(4paK}ZAfM*o(_Sgm>tj#{MvGVE_!g_r2UD{BHJJ_A?H&D*w4t9)j+OZ-{txE z_}FMxWuqU<{ASR(-#qAPRV1T*?_j9}+cW@KtW(E#B=Fhsnuib=75g`We6PmY9=N15 zwv$Z5fK0ex@!|u3ok7815o|E(3f&^l*hdZ@mW81-!tN)4N}t;&w&?3J1VYAq!3!gW zjW34(Di6vX|8MPrvk%{VW>K~gIZvo~uuHy38bv!7^+88#DJB)!8JU(NB~-HXi%Lu7 z?(hXHm_Pp^M$|B%35JZIJ&RC;oRMLF!X}w;270*T_k=SS?^fLFfC0!F1RW#gi%Xxotq)8S72J|BYan=YFS=9+ze-+Rt7oPw%c95aXgR+nwWT|M>qpA zmj^on)qKG}#4ods_%FD}BuM$tTDugr!vlgs!1Q2$r?GQ>HwHu9>rkb5-TNf_U9td5M-*5f=Dv?qdw>9n zus50x&7wBEgiKCG5TM?VRYbfaz!*-Syw}7@j=%VeO#%#TRf1@<$psPHPYw|B?(Z?V zFoUun5b+eUIHJSfX&JQTNShCoe9Z}a%Nc?Wsu%^PM|+M0;yflbx5?s>5IO?giB$3- zKe4ws1>gxtY%ViJEF+^0bbGGwPzt7fCw)1T=dp!vCl>r!s@{YD)7D89y)3Oij2WK} zD906Uqier%l^D(C+}2=T*nLqAyWkE*P@l{guvGFy_<^5Lh8#8+aYI%?0m$eep&W;*Wt=YWW?9o({(p z%JelD1t+?Sirhxp!&=Pw)G@;^mP{xjHR61wiC@tOiT~=Y04WGN`oWd+s9%{FJ`d-} zcPFiD!o+oh9Sf#-)2&A%@Wnl@A6>6b>O;_Jew{BdmtI6fjZ6eWl?aT0i9m=DX*LzR zi3DH&d}B3@NB<6}DbV%7w{A6Us!p}m!I?Nebc5;*0(r)2c5F8YuXA2NF`i=l&8)t* zZAyR7<+)Rq|0xR@j${nMI%_5L;xZ0q2Rw^83TR9k=sxK1laQJKpRFba;h#Wv5qOwM z2^kq(64!wlVtsH;G)Xr0TdoRl-u$30vJ5c|!@T&pZShrMViWVlWJpX>T8lDPkQ~pe zFk4~?2Z*GEc5T#|igiaZSQ4jpcKXb z6gIpSOz0(urh5p-0ekCVwrL|(@jWlD`II0!Q8}X`WluIK!xU9-Ed?nM__M{7kBY1} z#qd7Z08`kw85I?;^v6#EjPn@K^AHxws(3}z>uC3_L4!pvx*Jy!nHM(2kW7>Sq=^Q0%+k#xkP0{2WpXh z{DnC*eVA))pgy4`qxbGohz8(kicasLeqAs=KmE=PkP^nfPwAf(6If=ryiQahH2o5;9wd z09bptR&%+y=*LiF4b-?TlXu*OZZEliW{9eS6?`mvAI;btUp?lJ6CjS$9byd5ElM_D z=X+lGxh~udD{6vtR!vIQFl%#6!h*Uw=fvxoRDS9$Og_Z?1dMhG;dD7)Um6cMcCL_8 zxDGmyY#GG?Vwi(wgeEn=;Q!85M93^QJ$f|8{B*Y*$Bc&i`#@Sl_P~bYoHCZ^nl-9O z&lT$;ydDzT7H@(YY6x~5HhcW=Rq^+n|)DDN0U$Q4j1!zI>7 zj~>Iw3PLEt)J%+{o$Dkd_DSdT2l$M4StNiJiM4maktL`cL$8xDrhWhWnFyWeje9El zm0)>{tAgjHW(JfYL~cnkz(-JnM8^lnnb1h00)Fu>KfkIc8l}FCSHy-w1(Ptw!t6Ja`bG*3EzZNGS)WjP6cyyqbf9mMh~q&Po@J%I)TX)tDDVNj5W`BZm1CU8O7J-&7>?INMeny~^M z$dk}+Z?59l?$y zLrxL9r!XqJ6mba&;vyqAmoq_|d0Oao-m2NQ>!*lP3BTBNE)<9qS? zjeh>C;nOFDFEpCp{2XjT*yDs_Wvzui<0{9Eg3V8ZX?C50g_Y3Hzh46P8=U|C8B=0f z?9HFt_#8WPp&#qZy9ZtkrP7;GH}tx}7KQfKKBFK<{1h_wCyM^u0$v(i)v%h-!0UvC zkKr%KKL71^Ig2~tKnXqw9c2L~7jkCPcWZ^$lQ ztyiIqpm%`7N5D@n3Tp{8KL3Y!_AF_|5E3d|9%CnKC|k0XElZZ7v1KVL5hwW0%1@N|q%x*2X(T+n7-MvTMFE zE=3fdIFAZnbcV!E^)5NBy8iSQ{*@kTqyyO)#Lce~wOO#1_ZS(qz~4|IQlcjdCkz%@ z%eOuW^mBA4DIcBdwcFwKunR#zWM*~M;vh^AY|j$0`~bK_x0}R1lRJ1yC;q<}Ib#PM z7Ka3huM`Xu(rNkqK-six(~8BPlFbk)?*J;Y{yW0E!eu!_h8mO{hZPXsmWI>-{|1FP z8L*^={}UX%V-3Ag(6(qquOqP+Z4c(%wqAzDFi{*YN5BPHiz7?+OP>ho>=#Xqkc{3o z5l4vqoGie{iwhJ=lc5Flt9W*HkBp9exZHa4>tB_LhfsKi7#;hsWpGT?oC?XbJjkf? z^LiZ{o!PgJz?9h^+(sS@&5WDZRO!A=5?A1nc3L7Tq3toD+Os-GNMMmZ!N)~8?qOJ3Q+HSNFr3C-ujrSXAx%K*;(H>BN{fyt(_AhjK%Inv6Q#B1Txr!d$Xp_8 z?CpB3r_bB?PT~R+2@EZNAUkVU3T%$Uep~7_#8h)wy!D@c<_P!k^9{7RwBUb!-4!V% zm0WyM30%G&pFIUUPD$6X?8|1K1S}9l|1}OQ=%h@UAYMrNLuON*Y<8U!D=l6^l4KmD zNN@Y;_i9q4k2z{H(-)vp@$wzBvd59iUn@pOGqKL)dr5n5VrKx#j-(tNIWs2^UivQn z$Qz)L^coNT(RDlkK@3xYHMSJe006;>7gAoD9ch}-=zqe%=q4>%WcOp7bto1yRXU$Z zgUp(0LD&A>*n@xG6)_%6C^lHBCkTIq8S^!!=}U|0VuBRP>6;B(qL~-9#gBa*Oh%8^ zrb4e*=o6_jv#5^-i$O+AUOKfgx4}dv4zCo(9>3@-mGJGl*TUoVUlWqjTGkgbho0fm z@TLBcORlBpRpo1wV7H+0%uyCUw-K$*WIp* zI_iJ#siAo4>l(YeRR32isN-eJDv+s6@oLtOutysSZSr473SWf3*^yv}pA}ha7_DDj zW@aZ@YmyF1F}58YfCM$J%Fc^rjh%#}pGGZpF=27;t8Osxe2&V_f}%;PWM`GBSSTkp z8Oo5I#ei?N(*of(S>8^*)qyU62EkTn5Dq~qWkH4+yxd1NkAzX-rb!61GvwZEF}hCB zH5#U4D4?FA$Qt8^NwK?MgfSNiA$VGi?nid-ZvJ2(fPk&@ahJn5 z>P~jSSgFx##`C0?qrG+g`tCm8=YFJ7bN0Nf2pX1xpTLxYrdl{R;S0`yDD&K&ozpCeePCm68T{FR= z{8scy9_(P!5YPgP^AlwB=V=Y!T4+#&8&Dbalc*kq`!?y)D?$mnwz}v=V%LNTyUzcm zd0o^G8;6%Qd3*QEg34VOmHy^GeJg*hgd|X)FG{&5jAu;3`oQN zYm3k+XxTx4-raRMnB{g5SPXo{op)2%^x(v_G z8~AV1)7KdeUCfdAa=rzY7Y7Q6lc%@VAehxF*hm^H*S{ybet2?tEq$;U;Ao7HX_RM@ z(-&)vD6>D~C6L=yqwntJYSAIeqq-8(BzeTS$O@QGN7^9wTUEWT>`1= zIupif{<}EO#dNhZ8Pq5J`mT-gKGI_h;j2_xcZ=Jst#g0QrIRd~otDzE?eE*9Nt2mX zGZB;{ftx;G$8{{7%H1SIG6l+iI89t*5j4Faey2*BSwzgIm1QSu@%fH=#@L<$3wWl3Cu*>l49c8ThTr^ zYjtF)%0(75EdSEWgx_DKdQ`aujXJ&kZ z5;9t(-@tF^uER$}xX#5ml~|CBl{ZIm^i4r<3_WS}0fFZhZf^+LD*mM*YQ6+gOXWlq z2P6tfhdqmNpIljRjH9JLq6J!V_hevzH<~Z6?;im#196a!5{{AZ@mbRTlN>MU+t7GX zrNcR>`CpN}&L-$of+JA1K?rm4+pZRCLMssCnR(wEOAABM>5Q!Q$CS|!p%IvA?++X_ z=!JFjDN7}?8sv)(SNJO9kngutAvEUV{(`pk=xWu3d{L_hQ%G;?p))qjc+^4 z)2$HOg-I`+1Ef=dc-0ZxPNXKS7aUzC4+Z^$s`j$OA;)j$I2195ygdDBD8h%@!v9P8 z3+v*rCz9`8NC^_tjG63!ETg{CbuhtzAbm4dj5M+_z!$qMNWW8GXh?}OOuM;ikavSA zLs<|LX-?r>`?)QduM}O-DQ-z3#Ab_oSz7gU&d0i-5H@2I$>4|F)I!@xZyPE(*>S&5 znO)fS*phvnXHudsgK(XOXIqMpS92B{mx;~(jQt9UibA(5dC^TDv3d&-TTgT0Em9e8 zPcez_8Pr#DbeW*zk6P|HbH61ye5>g6XDTPIN{5z{Rj6$lBLedg%wpr}XfbAO7u2l& zoH_r9x?_?l^(#kL1(xWKN%zpB%IqN#gp>yJ^_=%qOS>(t=;ZljK08{)15wYAO zq?fb-p=@Ve{`9qN(O=o4m5B}uPu%};OZ5!k)VoBtSyuULYELdvy?N_zXRA%oGJN%T zdj-gy7fAC(^jed}t;N?MaIkAr%_HPgh}8NhMgsO}_x%RTlPmq(d6$9n2Svk1o2+N} zi+j9{f;Vg1vSzo=_@GRD43a#^b*gGGBBC6Vk^!`l+C}myWU9h~VahBC!l+a4LhlU` z;1t_k2V>5>aD%}*pTG$)P-{aS?7Yc+d-I<7mg*fHx{zc*If(zPKM+%>WQj-+6o>az z)esNSqZ~LehvgS^NaP(=$a0shm$6v%C#!k>MwF8jp{gUA!p|9mV-Q9qrBBx6|4IJD zk{om0_SdGSB!tJh-jODvGZ<*v<^R9AND^Z3uenI5wC>AXBp1bugV|3=Jou_0Zq~udljdC*1jW;TbJFW`mFtN0zL(0&fSYirl3^L0tDo!hoe?^^%F;D4!r zIkLpaLP`i;h+YMx`?3uuh@X+lVB#bml{@6SOv?DNAa=+xzF&IyX~yp|GGu^{@NiyK zrmD3e!?0=w>A}V^YqCd0>KCPrN@2);eqh+#duxeLQuUfO*|7H{KREA{1n_d?NKZpm zg90qja(o}&mLB$L84I@Ar}VI#65J-;+qDbY_gl4)zX*gmEX z!C^h6!S_uR#_P3ih}Gf!BCDid7TJyI@83Y_e(|x6&F0tt)TJc@Rqf+)pYK2W4Ls5B zNnx2!8~?_meQW>rdxtG8J8r36bNfyEmF_EVjvV-C&zIMkK8yCBSpIr{{#WyqiN|uU z9Q*L)gDtdtY7Mq3WyEYp+n+Z=E!3@}(mUa>{SL9CF^N-t(Hh)_gt| zGmF`GI;D=j!AsXV2^@$}{c?!X&(Ee|>@sJLZku$5C;&F- z)0j(34x$@MYs5yiYh9%{fgJvjle6jt?tBSwG}wG_?0?wvBD|V3snJ8!Mq&Z3Yo#_p z)?Tu5E+WDbr`4G=jUrCo9nhx0^K!%1itnZn4yPYkhc{o~z)DO^tdSoa+*#siKQr!3 zERB4iGn1)3{#ySipcbuk@lu`*!TwD%oaxpDUU_LNx-q6_7r|rKx8x5{;xs>e_#n=? zlt>@LnZ1cV7`wCQBq){|kMgX_$_ly}QvyzR2Tbc7zj#rXfKDLKR9a?z>!t4r%Vv-Y zY#z2C&|d5!r(c4uhJ=PbgD@YKJ4E0msu)uG+$Z%@vbyhVQ z`PsSa9_XUx8~FD?_m*3KowHEJjLXZY%exB^6~E+vqqNLkWF^{Miv7@1v0pcMPGI+d zYb~4Hjs8Q4d$M}X8fo7N`hHj7wW=5jG$GQ*M-pqEH`t6jQtr=X27yecY5Q=wL7)%_ zG6IsrMnkuvNQjL_fbjH)vA9M=Re|_7AcNm3(Hme1u~oy4p9$%vcuas|;CV{Hbff&| zQi;1Y2rNFZ`Ock@a#T}ZKcf_vlS;Rfx?Ug3JmhyyFhn;`UA$;c$t~2{nn%9Py5Iok zYrOdd5z-V3i@Sb$S@S5579yldyL{qA%q`ul$2cm-&48(%1T~x2lLmCvqgO5TOBXMS z-coFY;-ZvCe~UJ@6o;lV>xZbx2XE{<3=KN$>h7Kh*jY$vWBOWPIs79rG-pFv_~w2D z9ZBoP($aF=74$3Ihy^h{!sdM*-NoG0bVkd3w_b{}e|_rbB*;s%$&(M!&e~Gq9oUgm z4W$^7QIwP!=Ue+W%jG!!0dgFdQ#_hc1=sxvS3qmCey^HQVWZmBC2-)Ax>DSI^PWmwLR?0mjX`&>{%B?Ieb zsS1!W>aumqmLQ7U6Q@p%1Gihum~q(oC#1`d{J`>4I2bTC-(q1S1-33tEH{KF()S{@%eL6Q><@!@G5yvx*M&mteV!Yj;yqo3oG_O zXPimmFBcCOJp=pP!V-~YI-#}@bbkgZ#E`OY`hTalnxsWw4`}E@NK@ZTPBnI>?w-&W zqYYor&c|P}AoN$GtNrQ?7ow8VlM~jtH^y0;5{h14LA%v38)7H-+ z3vr0t9fXS|GPdGjk`Vjk&9D%pWOw7O)ykXSgzO{Q6m_br2kV0$IG#-IT{l*Io1Hv< z+*tY-6wmlz4!n7`gT<+>!O9v zzf|2wibwZL0oPBseYKPe!R*Dm#Qlz8KW?v{W<%9m9cn3&XEH9}BoC6=z^&s@0?o1M zqrVlYK03knfw9NPn5&u3nI?P(K*1m&O+|9Y8Kc0boEam6dY%(!R@?dGsEPTeFVGj| zKgF!vsl;6u_5a2Juj6201?+-PSGSfiQ@B2So?@M)(a@9MV-?pVYA$MD=}$@fslFpR zb<7^3?Cj}Q^ICIMqyTwto?jd(X!aT`kKp21^GG?*TPO5m*y+=!Z=Q;X*!wx!X)&Ud z_Iqd|x*KpS>g@y~%TWdx=-#78-3&hmC$t>gM$b?RK6h)PM6BRI$l=Kb_oSD?g1pFG zkcrtlTuyw3+xp+VeLI`uz!V+Mh)0Tt1$4pE?;rVVCf&1!*cXINW|(DMJuS=e<2O3* z^Y@>rI*t0X5_o^82hEL**P*)n_AqJSev_lej`iRU!AH40`i;}tU?t)_g8dx6{08^e za$!PCh?Z=nxe`~Za+`_=Qq7YTx4YjL%H|GOl+@vSy35BwVX)$igi zvlfcp#y7vb`Kw`O32_{%S6b4amNx&K3~9RtE{vVAu!fTLD=5J!dyDo-SAuEoP!dhC zu{jN^Ta)37%Ltj(JfeMz->u8@Rqnb04 zdQIcui4!L-&sF=>TTf4~W4@=poZ&&-n4Y$*A>zs6xcfXvkdJ=_B20Ys>J;El3{VNb zDtYI4UyMhH^Dm%rOsDmH@UOm^8TrNEMAMCMDOkUA=aIPR573nAadwJ7tQ+|@dI`x& zdWY!DNhhQH``q%*fw9dVM&k70Ybd;;88WJbcOfbbIZM$%+vhSs6q}}apL(w5%o8ZG zecQG}GSyQk6`{5DCUh`HmiP?zN;WwUFot4YwFo4Kk-*7Hbe-`0xzpal;$^~3BLz5y z- zEFs&;(xSpyTsj@#N9KYV#q#&ul#@)*i(h7C6(Cw*c9t_&udouT0^!`tPuvxg`JJMU zldENBA1ge@au##Zi(p{nrq=fly*28xg)}@WuO9fRcFT_ego?O90Ah!V8OjUiS(}@8 zhBahgP#!oUFbEdYWZyoYx1b?=jhCBa2W-*0Of=_QTes2tN-}=Kx-i|%~TWk$jS~ucA}wEEzSnq z=0wVy>N+Q_f9~~yodP*s{C44fywiWv`t|AixR5lQ-QCTmP20mFAe36C#hvpOb7j^d zB?A9I1b+ECRy}G4U|ifV0wk}%D9OpFKT1)w{*(IIf*J{Fasq7e~dYm;Q4a9 zvIX1>lV@!&d{Il?sF}_ct=LvooP&2O^LnZiTh_$>m4@~qXM%52D#RVM4sm|sf8)vg z=S%E42u^UsiU<;e?#+)hIz_k&oSE&=uXLYC8t4WlpK(~lDZ6qBD3ITAtf4`X5djQU zl|tl5blzgJCB{(1#XDbq(tX5MwF+bowWtzl;-6_>Q<>Thu@`NMc&fw7s!(@kztwy9 zwsgm%kHjm1Dq#ne-Xtp}RyGs|#$;DlSM9;fC8RbRqFdYOSmyjI9(id)i_5%?4SPSe zg}45tap$S8>J%d0^i9T$IVb2aWjjRzI2wu3Au6Z`nfXKVb@=Y+#{0v( z@EiwUsP5av)!A7T@jPPZ7!bxl*9j^#!KmJMpEMC|o`VsiAh-!$-F#VhXqpwqo8f6Ym#-+RD?`=9z9Sl&+Y z=!DFgXixCUQhKGmWEG67WMgR~H>)%2qc$a63KTIDiDAzuOc043s$Hw0f5C1B#;F!iPrXjNu4Tp4vvZ8m(<6`Gzx=13(Zo0g^3%c&02yZ-*0p}EIdo%6J>B8Z}idn>%JhZoOT zg&MwFVyBa(06HL#dI}GtfScFe+|2AK#<{vy2M2pCn^6aqkqW9=LHmpZ&`Fhd&33MK z7!^9<)D|GeZUzRgZ}};{sV6R6m_l(He2N_}_40^~ihjdr?F+O6Z7$Ed>A!Z(nryD_ z_siP(E=o*DP-S_UnVGd;0Md)ciEnq$@lMumD*SQjw6)gnRg}2p&HWC>#EhXECLUQA zrjlm3Qi1tz-P(TFgQHv!BzG-xs-qZ&Q4-)SOaNewzk;O@$FyWvOWl}c-`r225WtO~ zw`qP~@?A@kLFdd3LJ>p5`(E9$Q%>NGL4(HF+0I?&VAxsl(Ad81aM8~yBBAKf|9}L| zOwBX3hba2@udLPF_@RXZghQ06cuIsqV4U~wk*RGT(s(~E!{ue;UyP1j$9D{oA>-&& zg?G}kXQU{o(4u5NI1E>u$mPqgOAG-`A<>4R7FQLKr%~J> z65Ro2Uzir9hG=c@yWLNpPL@Rj=M#rM`*~EItRTYTLiprh>#nlQIDr%g zi!4JtM|bYT)%QC#ZqFeO^*=XWUca=|`ynV%wHm@f`)CVJKpt`7OeifaKuxY)SuTj8 zs-j%Tb6dD@p=>)?sc}nas>7p-$Znb`$AjE* zIF$9izSe8hpwpqKQvCSRxnX1dx!gXiYG`0T=~07ISFTKhAWx)18NUQ|jM!Inuc{7_m^HCijGUczviv1uC7)VTV8B9KA?0Q4O%8-oH^4}1qtDjdko~> zjW;aMH?(?QJLL&WKuXotn_CSqp|rht2B|^OX-$34Aj$axdk5=tDTup>v>SVXT~yc` z;J>*1fTqEvl(Irv>Ks+{w>vZ!fs;%yGQ~MBMWCqn4>>9khsDCi-O;}A4PAFZ@glYg zX^|YyJl1l0T7w7LvNXUq&H_gxb|xoVas-@Y9n36S)kOyG5WLBu$?rIxH&1tXt+=z& ze&hg{Vrsg*+vauaCb8EbQCx*Id?ge%kAoi}_esHXnwj0DlpVQo8!Kj)_XwR{$&1ft zt1!XYdG@EQjklU7srDoPuCn+039(N3BO_C^ zPo6mOlrKsKXV(pQ~*gr2YxN*eDuW=P7n*BQfkesme-HubR zgq%C~w14W;r-$U%sM92tS1wDJI!a0af&fRyV358gWJJfvcx6`cKXJ7T;n|Sq_jj#x;z*ZG-EU~ zFgCVo)haPAE=ZWSmInumt#c!Z?5Y!B{Zrh7%Lx~k9mZk3@(R89*@4T0xwS>2ASkEw zH@Tm^!k=+o|-Z>fIwXP7~Y#Cu2NuHCDbum+!$hyGmLhM-pt&H@F&-zv!UnuY<>YoN?xs%v{!T23_K}O|>Tq<;?Jt$S9@n$vM9`_R5i z>eRe_J9h?yU85p~VUu3-CmIIS5EAwkL;qLgn+A*3jk>VmgXxFtmSSG~^l1i5f~pOs zSZ&bOz)kh2d)vg!)>b~stZ7|-}-j1#;5D|hYyELU;lLajy=ttVAg<*b&9D~zvU4b^nt#Rl1TLzXiZ+Q;JlPsoKA*+8UT%D z78FSA<&_pp!OaVIT=KF!mSEwD3JPh{J{VAF!qJ@Ha1M zrXJS|j57gZ&V2du<@a8#{-v=HmxEew`mR;;fxdUIc6`k8ZxhOA%JHt(j6F$#A=f%^ z@@w(3*k$9P5;@=bhq5NVK`*tmOM}t9OPI;sOAq)q;xRBZX>niAg~tGk$X5dMGa#jH zX-BL89PC1_b0*F_|04PL`*?6eq-|)@^3ACL_q?ay*3Hj~-a16@;nIrZmy11r!M|xh z=P@~!r#a>K4|8@u*|+hMGt_;nP>ehA?KAY!wiK>n&eRLAv~bTUun{w;%bY zcBcC0EWG*Wh!JaGSC5b|!JhJgiJG458&uIB+VnsDp}wovuFa9gCbv0#n(2FAk^soZ zk#j`XQq8x)ozI1?FvTG8KJP_|&2AfUnp_ivgQ`WajMfvpg-iJ`CE_SxWqCoOh3Fe! zp}Um-L}wwS#VKy-87;dAXMpACafe@PbK^M4vEZEe(=B&b23E zQeL#VJVd0n)QUoc6w}D0Om=06il|lVu2%D1GkNmlZxY^t8V~?A@t)r(KZg(`f9zT@ zW|1d+3e>wkSv|!Hml9>|Q65CC^ZX>q@1uRXftU%Tw*io_Zu1DiUkOf$(wB zfh9DM-@-GV2o2SpMPjZ5c1xi*mWDAay4AKww3ZyOXM-B~=63$Ib{6Xq=JJnw*2q}J z#3;q(-h&-HOTG73;rg@ePYj53tEMvG#l_8SS%xcJF)83ACWfBaso6?Ue4l#5r3pjE z&#(;fm%%%Qc2JLs)vh-jHOpj{AoMXrL!ehF=O?j z->Sc%etg=%1OFhrYg0{5R-O0IU;EZCJgwsY_OEWAJ=x~u{E|OZ{h-#Yqt~A4^;NAI z+8`^7A>9yK%GjxC77b#B^v4J{QZkQegsv>6@Cj!co2=9|b1QC>-{oV?e%uwGqpyJKzD5!-f(vygvfa)BC%xoM8+G8BgQW}OVb(xbxd+PnN-u5fl z{wIzc`Gcm*)6I1q+v8eqYk} zUs+*fL{5ddrRCn+kK%0p7_sNG`?r!ZHf^1T4KIaI0C=|GPe~RTQH>H7)8As^M1~xY zr{^^m%$Ta?nHagLyzRzGBXD!NRT)1)VrfNYv z%4+*4(^~a}C~@oFZJb>BYb7YU`)T*H5$ZGwkA|O)`P8`JV>O83VH{7iHG%;oH3um4 zK7|W5ER8mleATypo_j0XI3mZU?W!Hgxdwcb)uS6;QSa3?W_kMSB%J) zy#6(&haQ?{?W;J{VdR2w)J}a~ugINk!*t-{w9DuSkR8d4+%>0t+sZ;=3@Om4z9X=> z^yeQ}*KOL=;Qp&Q16qXu!fN2jqvBZdCa_t~yrv6S@S8|=O7Wc=&RgW7c-7v1>Q!lMU2y{J{qs7oKJ z{<0ajPEod3ys}nacEwd$@$wkm&}vyk<`XQCAaH5CX{nWvkTBEh z&SZ3w+j_cGQ+&fYFg&n03s=tH+CDY&Ku4A;=K%O0yS>xeol4$Uyx|OFF=p+`Pg8Z& zDLf&Tlr|IE{W_5r84UrMad}KLXWl^Q!-FL65x45GZoz{1l2JJ>!vhYK-KRfn5OBn~ z?65F66|byA&v!Ja{I%jSx|yq0;PPLUc7K2ig9|cc5p*Vm?TG*bkzjEcePiOgYN_i+ z=QkJs0RdI=-V6e@b_WvqL6;Q@ChNELNIiM^My`W^kvKWPow#x6aBOw{NxNO_JvwY+ zVSz;Q0Vo_j8FPo6B0>RUPhR$R*mckb88O1TI6mHYTp-?F^^~zh-%Cs{1VZ)mLYcuL za0Z5h4071Z^hR?~jomy2V=E{Qj!bR^s4C&bq&2wHp}BSZ$g3h-{p&BLf;4dSDNh9_ zyGrc{WE8a*q<4*c?(MAj35JP?8WT`q0uqrPE6O{U^66PyR8%ln2V^)c0juoy%=}^x zqDq7d0tS0WmZKiS)M?XvsK=B~nb0rNf`PWOkD>n~4v#YenGoaP66%YB7(7t zEDRzd1Pmen4A7{{+@xI)K)kKut15}PZ*Qk}_qIwiLGF8!3BYFYrR;xctO*MY{KzO( zl&PnX@(J+@z?VF10N>iycpF2w|HMSg(i8`4jqxjTJ0KO~3(|ErnD_2pBWbEi5vp6w z`#M8hvbcb5ey#+4j2@}?_HT8n15VmAa>FA7! zZLgt^fzKSZYg63a-Nhn-S+z;uBytm^a1!pk|L{>}s&b)o=dX*KDnSqGR#z8?xAX@& z7s4+ZEPZr=g+mS+i5R@=HIZ6s9>xvZ8YFNyCd9?r$8_k>L43~LF+oE=#`U48!2QEg zmjRzsDnuiG^-Tc?{E5z2Q(l|&BH+aPN%xa;cf6dfUt+Ih1s&S2dp&tTGCXg^%eVJK z=Y>_LgyMTQCBG;!?)$g3M~H@f7N-%ne6!vCC)N#W`abT6CPKte&{Zp2B?5J!g>x{j zn0xeZ7_i7gnjb*@@P#?%@rN1`Gz|T-o2~5ZFj)0B<(-qx0_UU11*HIHJMYa1*t=IE zN@Pfsw1~tXF@BPQ3IHe9^D_=&{ zsNhGT^G&sva`XtAV5DKx7y%SLXaq5R$xiV-s*T!)s z0BKwtw)i(ZJuRt*g{p@(3$rs6mQXgLh*p2Wpp>>DaZ(x z+$+>ko}H}Fej>vhUkphjC|~zXt-Q&c?apHE_bKO+tY=~ zbPo`q3E_*|4{(0s`0?yLVZSi{f$@1luY!a~in=u1R5^18@jY#l;4LL9L7<{Egb$et zOL!_NIawG+Af(}!S1(h!xF>VIN)vJS?&l`;PdClJ;5xFr`KPPJA7h^@agzX$hevlw zEvZ@$S`7z3t@^cZgG_FhL{_fp)u(a6@0@yrIcE1`ma`x#DA9(Fj3Ig|VZBSm$-Wxb${qSh^uwZ*Q8h_9A0oI9#5SWdU_iiE0XUpG-~(@v`{E0W5JR$NirIk7X_$4;T0BnEu&#F>y5%2d&4A)Ajwc{S{Dnc*CO9Oz`y$dMy2 zmDbB1vti@ z@og}h{f1OD0o<^18t6oJ9UsKXqF$`T#TW!mkc_1Fq-1T4&wiJrGJJC4uvan0suac; zw4wfXXAmuiCp3r;D?J}jH5N} zw!bSg?0kOa2a<(;obIaX-0oi^A{dQ^>|QCP9XZPrp=%{Awr+r00qZtlY*!b~9D6#1CreE!rd^w621Zf4@t6v2e&&JD{A`?yKSK54`S2ahyK>OZLF)l;F> zmDpZaEsp#cdA?Im)zcANnC|vpTsSdQ_rf$AH6>#8U!&Y(a&JcmD{(D=vCeS6+=#=q z0Wrc6O0EFkm*w*Yv7+gN`<57|@x|~5?>zK&?B6f02YJP5!ok3ZO0g#ol;@=LsFmMA z5ynzBdXW|@aoFqksw+QjMav=6!I0~Vid57m^T(mEl^_r#A6bl3nbpWXt;QaVNO0-z z3rjkcc`pA52$YH*=Gab*1?ai!YyxLsr#2%q@767IU-3k+DQl>!85?>VUZ-hOv!SNj zpZz>)feeGcqzAZDvE_C^w#3;a^9IOlcG^3b_aGcH*A2CYt8D)_i>`)ah@{?s$mfDe z(fHb?zHhkLNRY^=E(0o{97a&?jKBZ&reeFv&#$KRCRp#k>|qqDDxBs&|EbOO-|u)> zSVP;TKa4ATghlW1*UOE_nJ6@qXTz8A!lqZ0j34Msh=gd|Qv(C~3I8*UH%! z6^~_4Oag)n(lx7$2Sz1nztG~Sc%dY7cRDX*(k~V*ll@cHukEfDbY5E7;<;i1O!f0W zeu%+qEHYeshZQE%-~PRhB_VC}ZmBRmoJp+be+~pBCCpu4FBueKA8x3rul>l->)U9s z?K9Ah=*(4>Qxhex5OesE^1QY`SHS>4S2)mr+?`S89Vy~MDnM86BLY@unHr@H;1Z0Z z>mYS(t@G#3^`(-7g?S{G0jo_#9YVL(5k^tQy{tK{bmM_@+$I8SS4}fB+aZg>p@&Da zNs}^`wyf+M8;!C@1TgddZ(;_A_tJhOL+ylDX08l0$Ot^x3*@x$E_Q}@Id4hq@spwafQ3^ z=nuas53_Rplc+-5%5t8Cs>AOgW{**+ONVlYb4HWaaFd##Voex>+ z%!n(1j6)+^B}-xBv6={RI!b{lGve#N@IA!G6^0J6OoX&N5QKp*fJ?B5a1p2aqAN3h4cd z1_VF-l4s;JwVQ&5KoQcPK&UQ31;on4jN{8QxF-QaxXM>IeozCA{WS-q>+e z5ESjH=!mo{SZLu9m{ir%tpb52ujc}%#u3H~Yw!mcwBX`r&&=7W6X+*N%K_Pg2uq5V zLQI8S`}&utLRdB^Bn)S+3gtk;1#ctX9ub>j@t$NAv;e_i0u4y*=n!IkuUUjOArzkSjq; zmomn)37vCb;eG9&-{%P~mJDO6qBYBQ(wMo1AWZ@^{AKyjeVE?ZJ*b*9Miqd=6bp;l zF?|0g>5cdJedX-oYS%oyBkiQpAqr|(2B1L{NCmkmFIPHdJ^UZkI1}(O1=IkeF#N`< zA?x_cxC)uw+8^x8DHlF`tdiB*(#k5FX);@8AL{5VolI$QHC3*mL!KZk0;`bC$^pUY zM1_5rwehU>(haCt`S=UD+tY@8_t9d-2}L-O@iV@2{~4I4W0aU~U`?a#bg z8m)3Wvb{;T>J=C}JNaEN`f%dUmC$vSBcwu)e&HCvSz1;s1v$Tl>83;dJ(KiI-n1x+ zdpb2x+Wf(pC*E#>4s`yw4C)WcpbC@O!RU}g=|o@U!OCExOkMa&WIUFlhZQ_Q7mFFT zhreW5%e)Tyc~oRot;%&Z{^M{Q38(ZP0at#py*;y4X9Ws95Qc#NmTGhxJ95lOLRXSMgDk^q!lD^Ig1F3Bl*y18()x?&&B*?zpTkiZd0AwlsWg#`C%k#%)tLL9~z ziZ{P&VR{waQk5^Ng>p?FX`Ymh9GCAGT@Cbd6B8q?NLV%oD4g5ha*-xXMl#{USpLfY zE`}nNGzc;^Zuac`>foSi(=QY{nl4SO?whGQ+i1hvTdn`4JnDDqR@FtK=xqoXZ>w4n zw@Pre)Tb>L$Jt37=Y4x~D=R_GqBL-`P7*JgB;d%c!L!mXEGd(6OjI(e)@9{l=s`hG zV4Vxi3^WeDGF5VKL-hR8btjfhvU(_1!lk_Y6C0`cK18eznWO(_Izs%uqOKKb0>`FM z2SZJR7T}8!rvfox>+Q|$?7~pt!ArhS9TfZbpN4ixE28j=K!J*kX)d5)G2a9m(R9X) z86u?<4`@%fHMwcVtu9kx7-#`7gJ|nIQHiKDUQeI?_V6?2Ns00hZ57iJCpa|s6T?1` z(bRzRn-t~0;S@BJ&|y(Fu6B#sgJS{>>*fWW6x+YpG|m(p!LHQ|_t0Objo1kmYiK`@ z8ag^U!F^Iw)D&|xSBU$z3FnBgAk%9+@-z`As3xxz?Xvn+(L2jgdD<@0mZcx8&pwxt zLR>4@pcJhG4$Xz57ysXb2Z3IvD(2D2vY-v~OnvD}mb}P1QBP0Ux0!wbhBLa~AzdEN z?mRN&)D1Ik5sSt~cl-Q*Zjs^tJ|spz#jo_ivKlMKxA)OdouE6|mu|H&w@_0`M?(EQ zgKj3G=(N35!Qg9G_*V_xXst>iJ8l3PW;GYkfr>ROx8~cF@Ii<=d8r%%USt}H#az|9 zKKL5-iufJ@)?wGbw+xu5W>kx|%^|3!iD-*e;)SJ`9h$QpBtKt3uDA+B3w+Tia~mud zyB`9v#(4;0bJF6uoUz;IJ&>`72$m-z9?}-sk1P$FZXQeNSabQw-K-5uXbw+}T&>L? zqeiWrskwP=KI1N;W>QPb$OZ}H(9zYcna@+HQYVX~5Os2!B+v~J{EjgQgd|cj`OpP~ZoPS15rR);!q2d+MK7i&3agyiIw4=Fw*i}$41c~~=8i6!bDw&idCQH)dP`9(L@weQ1^vceAL>Tl$kwpSu`0|F#dYN)m}78SHTHkL#~^e2m}% zpW+90<6dy0k@%<2BZ_>@;Y4TYOWHsh`sMEUG`<8F-vXB5>U^LK>t^s>RLXx?)VN5&e}0s_}k! z!I)ZPNO9WZ3b{Z6-}anCeOcZkX_y-ARBUWW2gKt-j`^O@q2>E<+97p`c|TwJrmr?P zRARI7w^W{R(_>)f4~mLaBvbVi+kKo_-H@>I|ASrKC-j9(meQ1-5eM$4gfsch_%7iP z5G(!syO^XB;Rb2{#+J>{NV?$kdHl1UHSMpjsW#*NnCTnsR1Xc#UgzH?hb>2{t{0E< z_fKy4L(^8>H#7*^L87?i@~qyErwoJsp)j)i`mN~bUT$mcZqa*vG$7SKHq(^ttUZ^= z0GQuz+#tvcUSJZDS5zI4W~E4(dgSnl6W!xwCL^694=5&itC;c6sqPv4a$;r>3JStd zkcB~Dtj9QfRWfLeaKSO$&WW@#d%4v0Xuv0oxp!|O`6=2Gq)fpi`C-CiVSn2RtEt(V zyaTc5lZJK6JS&${Wqdk)RHU{ewr+jp>bHxWpqLAT_Ek2xvhLAXIVefu;w^!aUbzH% zRL%+E=i%NFJNs@r%%6tss{2Cw3m<+8Em|aTr9=37IB!p)<@P6l@`D^G5dEjR)eL$- z*(-%t3U?H2QHj03y8u2?D+LI!?!ruBo(-Ij--&nOAzK?cjcWhV9CE=}|E0<@ts+ zeRs{Ay6ke_#vT48nYiPILeG3xgp!8!OFp5kDBSP!drboVyk8*}#Dkv%M{%g_8<+C`5rJzFe zVmF&s1kqEL^YocZAPMHTMrP##=YPhQ=a=_)x4wb6L0D!MN&|!9ca2l7Ea)=O7HK8T zVwGT4FM>)+C77jYvd2nhKh2-t)2W8iX01c>&@_N)XG0I#H?lC1nzB z!M2&W)2wYF0lTuGIok4GBCHSNd?2Ua{U7QoJ>2^kj4TQw}A887 z{)i5@4Ee!3Ihn04S}=)mN*)-ER9xIq+~f=qajMi%+BXHFO3{sVi&-P%?>Ww==;0o* zz%(ow{-7ygKo4}pL_iT5EaoR#)CPa%zr$Igy19Av_m+IwOaqD2|DRlsYO< zBp!SC1r7S*S7=r6CwdJe{An*8%tTQ)j5%nvKyUvChUtjEbK1g%;hb-dz}8xu=bA8? zOFEE>-g<~hvHq#7ut4YdS8Ow}($j3Lw9=ni$kDeY=RQZ4+0@E57e@)6%N<#TqQpQm7VQ7UBMQsN4Wd>Tzc)Y*W zx%&f@C`-0}Jm0%e+rp;P2^t`5x~OpxEmA^>(pDvHtLve%;L-_9kWxc>ZCE0FZ{K)w zoS}pNL~AhBgutdVp-m#L(;E>`dl$h>)Pt`@SL#-V@5JpP+X6A@IEq;yvGyaOpC3JX zgs@gEvyS31Sgb*`-P~g;TU>)$>yNEDmD=5j_yj3Gx^3p*=4~e+kB~a{FG^lH_@Itg zKO{L%rd5*uayF<%LWy{?G{=3l2^3oYRXDS#k+pA_PZ}FT)x&wcvtqN?Io=s^T>l2eGgs?EWTveu0`LO zqx;sozh&n3ndi$&B4=%n?!D%vd!5D44-|c_y>|Q)ldIFCf0ng9W;4me>Z)D5JT*#V z)#}ge+jbYJIZ&xmh>%0!e9UjZ9q5_xo5J(u*SM=>MI(r%LV) z9#j=ZuQ5IGi);*g6S>Es7i3}iCd!E4@}joohe<{Yh7G+hdK^B7nJu;gW*u;?xJCV< zwD}JZ9NI&on<%A3ix1P(w@o@8IaKx1hTYp|h95BU?%PjMk%BNKotGo3=6Q&X59mN( zOvIe|fJ7?p_v6U1cJq^5IG~n!ubm~_DyuWC2>J#6__78!#}*WIZcNgykeos1vBpU& z2rbiLD)E2}F535BJcT2Qc{6V+5gy4R2q%c4dt~a|m{hJ`0(7GHPF7aZW0ii5wY6JJ z^sQSW>|>%)DmO>OA&OIx>LKK#^@2H6#A@{JK(~WK{?&EQ+Cblh;T5M4AJKOivBD^A zPnhwzarm~OSl@nbqj;`h!HR@MB9E?E*e)3%bN*~c1+Ds}xu0fujVUibSh%mZ3qS3x z<5kV&Yt4j@^+WDAn;7N!eRKYUvWT_`bGA?4zv%Yr7fo9^T3GM4S=XpRr;&R8&EHI1 zcG>28vpjDCj<2OTef7%_1` zMnFJ-$SpBW4+Ymu5nDGbt>lUn;7O!c-rvu%v6h&cQ?^ei9lqu6R`F%T%v> zrP?+b1PJAVF{0Lau5@Rp9XGvACY|W+0S0gW32>6`CfLG?I%*mOt@A6SFCgNs;Ppcz zvl<8?H>1)79@m08^3m=-!-b8*7vdZWs3eC-u5Bp*kO6){(Y=z@Olnjh ziYt)k8n=v(9(q+$ti98fh$HApTu}A253Gwibo_Oqe&_2=TiIG`Et11Qv4N5l)Jid$ zEzw_uL5Q9M^o~>vixAa09eQY52?owRS>3nULt>o);&h)I?g@Z)V*1B){7Z&kYSc?9 zEczC7J1jyY(#5juL`k7fsbAu!19nkcG?!HKdRJfn&({vyy+#i7FZx{mZG%V1gd@YR zAKo`VIk2oa?DXmJ@T0=%@pK`0nV*j3tv(l>`$l}fqq=XnFaaap_2 zsL?98b|l3y{_W5|F$tT@vFq|<6-I`2i+&b)&&Y&Ftat^jO3SYNrJ~q(vx|xwj#CLo z@zEjQir&|49$VnsO9@)A$I#~nIXMdn(Xd~{U~19p(8W?Mv=%LnleLjW=t#e^y-1No zXX8c6_i)`0AQE0F6QDo-9Zxu1W7a?6gq$vWnG9VKPN+=yO;b~vY0{C_^ZFKoy+$iq zTg$H`zUFjlD_N<+$j}IwF^}mxz!w6N8wsZ_lfEcKEs>hf;z$)e)$Y(52at| cho zG~ztx1}DyN@UFyBSpq_z?HIO8)d#4Ojy-lG*OAW=_Z`#*Qq zp|kBFxg?(xi7j=!znbFVgwuw1RZ(j#56$_T4J-gk2z{{Si%;JrY#?Pb)hgBb!u*W! zrp!UTKvz?S&6?KB3SOo*yl{v?%EA(l!qoDzxO9uO7h0dbmHK@n;(Dsy$B!J%fza?X842HT0u)Kw`m-EuKn1!YDMWaIF|SiHa99< zl$xHS(eN6hwbZw1dM^9WEIRs(5|>z&fOyHLWWbW^ukCLTLN3ADFue#?!y3_;VS$Nf zHMoYJ5U%{k99qN{Y=_EgQq|%WqlYxbc)v*)V8-XuA7PVAz?Yo1fFnWZ-X`o`Nf-if zXH&k*mK_v~2}~8QM6?6?l(Xv^M})((NtKU5rGX>QFdim=p@a?&{nCm>$2@CRHgWbh zKO%vks(I70T~CQw`tjq3#2=2~3cM8hYs9rmgVR*pW{Srm?rMm}r-%;56~?`;Bz$o4 z9meg9KqczI7ZMEs;<5fC5MDCfR3fHOvTujzqv|vb78#bfXldM2=zHkkKfL<+Mm(zH z;TfWmpMJXz3}#5#`hzcvOETMb>IMP0XmG1=q!}4J*~&65`}OL;bb7a_HpJw_(L7)8 z1@ks+gUJRHT2(;l`$|kq+XW)oqwVJ8(uf!D%*?i1EPiFc;V>U`XONlJdIhwcmVD8oI_?LTEOXVPMhG{9VOi(31}!L zt4ZAOy2%wkZ76&2pAg0L#+&a~P@IFXrnPy_qFX3#NG{uT`IY35AgGY8p=F1biY!<> zc&56H4JUE&<)M*LT1nJhP=Y_#I{*0{6yLo|7X)lskGUleqn4|6*Vx$j$At21!)ecJ z+oa%O7e#2=KoSU56k}cTfqWn50=Fda0vS^`q2~pIPMVmWfK-JiQEDwsQS7*-c0ffSG4K4 zapMLL*p5n0y6q4H_XjzwXuZ7FrKPnLo1`;E^Ln-VtqazXg%yJ%StsM3FRubn%G3{` zcU7QCBR676B(SL`UA+NwUWvqVg8mm7-M+9fC!PpA|G-DlsAar}*Hluolw#mAgzt zaFw$m;cl^H47NNszbdsgD;QzvWq!p(kpZx zaz0Y#l~(o$-7_5xK?6_OHLFq$mt{7H`Tk-4i}ZnpHPSBSG|i5CJM`?U)GAj^)|P+M z&M7Y?pqC=9SDtrW#U_<_tWe1AY)Gz~A0cj9Vw_L~)%w}Vt?h;F_FGcqPctz)m2xKw zjVL0?LHl2+SF}pK8CY~&#<+=njuL0M?jdJ4w*;9dFMj%)D{H7!zjQS-C9{*o&N=@K z_#-ic$HT)DJvMON4@g~h7@3R{_?)fWf4$8iE3sYE&O>hA@qT80wq)xGOCA)ASkWhW&?NhrGN9}fRp_HbBNgxYC>&K3j~%)yaLbIL9TXc$ zPPl-<00H*y`EY0wCrlanYOwF3!ot{dW3U`=Q>`HBXK{*Q{XT>;w)?NO_49Q&AmIx5 zhdC>5xUGEe+F(@^?cHrdj8x8Loxu2Y9yej5$%lXHf`lQTfD+Y_QU55_kSnVh zS8%CSR({M<$G}6gL${?q_Oe#Iua90k{(;}$J#IPtmuXTa%YI)fG?q33^8x)pVU?TO#6(&5*d5h#?bqfM`)HxYz+#ANJK<5x~1YV1?_BH4719AME=w={T^7J?~`#ugP5{mp8r<^TqIj#qwv@9erF{o z0ykig?RBqd&+A2Ti(X(v>Xv0}0-0b{V`BxH0py5oRMT#x{{IB|=F*EO6E1yDgy9#k zUlJ$gsUD27%H~!OqYsKpL~E>XgM1yD50#;9nA&EYKbu=$nfji}TMQrk_HLhvC=>|- zoiFau{q!U;S&9)awEu+<2kw@CY<6?<{Yee#Pxw#Vnbq?;jT%(6;m2oAl?!}(EWomL z`EaL|0;ZQ&UZh%6DPxUkP<-`awug+B_-kydb z7s8I@8-f*Q#KgzPhm&XG?XF^sq8iJI4FyTt-}CJ9DwHYF}B>za6I*!)}Y&&`~eS7Q~54J1=9(p0r>giNH=;W7*L0F*ZVHqT| zpQhLX<33sG=!@MdsJz&){8+aVd5aWM(@blVxIz6-n@=zdD}e47b8{FZDP)dv)L6I`}uq zz7Vmg6KhJs)FrKec2fI%N;lau(vn$fOT>f-4hcen2G6A4c&p%;1qtwYg4xy@noe|Z>`59QsxFpeRQ`jQn5QsCZCg`n3qzNkoh_CfiI8l`aIvu~jC@_=JiDPBlPf8+ z>w($T@Syo-LO`n0ohsb+eILnaD+!3k`rPwn|GJ9zaYR5_+^xZa>bPf_7N8gl6u}bS zVp}X)R+NQfBpBv>e}a!t+o1R{xo0gt5BfN|o7Wy?XOBzg`U9v~;`Pky`-*#0c$aS7 zCjCEry>~p;`~N?FwTnt8r4E&*vYMn&99l%ljI4&NqOz6Iq9uy5vdW%?E7@99l1)gI zGP6RmfA_~F=W{;a+xPqHpLg}{aJjDM>+yU%?&~2d`*Mpr7X6ak7#JBS7rqG3(@Kos zRfv9|Itx^mYqJ=-rcAuO8HQ7`r9H)h5#s>!e(??t(-B*=Eil%S68C@2+e7lV8MNL(|1-GpifyyrSgTW6{9H7BQ3xOK|FZXRku3ZQ;!eIBWS>PpR0F#Hrw9X5P%p(*@ z$2u_=1`-kKy8h)xg#nuCFo_{mR$7A`BTyPd!*F=Q3_T6fu(%67o}VL>@vO4)d>B51 z*^yC18gJ@}P;SQ)q(ST=X@dd+SFU^rf|L>H4Q3Ya@Eg06a-^!E=z)0eNrfThyh?Ts z{)qj-j);5RKfHrI(_q`_5DvQi*+d;Z%VzsMM!Ve4?B164`Ft4qK~!UaU8!q7 zd%P6D;laP{Kf$AE3DxS^WEwmYb^43LB1RfTK zfEF}ZkhUfPdXfEXh=2D#5^?9MhJIeY@f@PABT%A~WAxdxkCwdorJo95ogZpFw z5v&)KyyKxr^EZp3+$sj`3>eD4X!xwNyJh$#*F9WL!up4GX0*TYuzXY;!{esAH1;X~ zs!cLgnJdfJM)%a|A%+x!03sk_=pfS^4}{!ryG8i-%GDe2_+kov*VgAp@FnJzk*Gqq zZS7lym(Gt+)Zn(PdpYROaMTT2=X!tcbKKZgqHqKGob zEa7(5ne_#8R1h@+P(lWt(3#t_m&>6?CPgPvSa#DH&Gi(|K++sst=BtS?h-XgVH8+_ zX53_*lTR|Sw`+qa=Kr}6fhv^xHE`@>PLC19L1QBzNyB%SHo?cquM&gbazu1jiw-6)Ako3ie z6%phZO-#zqHT&<=(t4mW#C5pwh#|1w_a|fo#5R1=n^Z5>4fzRFDfrHdHhbfuML!b- zi2`!v15Xj%4D8{IB(MUQGD@K#bJlwZPJnJ-+)Z%aPrjJ5MXwZ&In)|JdDs;8gldgr z9P-pCU%6L9jeywvL%mY3ht+YGOlyBax*g+NFL~2x<@MVcrr(R)j*}`5YHR)mZxa&r@d(Mjv)yS z=Z>{2KE;c!vR}9LR^Q@@yC}f5^bZ?=&o=lx+hUhit)LA4`HcGSk*0lu`im2jiql2GsvxHjLRNGgE=vq^gU_LHZD#9-;|GT8&k26+4#iec>n(yW!X$2 zM;W%8{A(D}Gq^C7V+noJt^hfLgoOu$> zG2M2oLjVjI)_4j&#!$_=pr}KAR^e*x4G@W+(w+_4h$7eyIEfi*mu zy`vD)(AZ@f#!AG}3&a}VsWC*AS*mK4bg;a@?S^%=*S!T% zPcm2Do7eF;tzrw5hy+~$DnI|V-x=}_G7>=_;Dtv$_)A?fhDY2Rq?CU1iGWum=b%a7 z(eJ*9=Vax(kXhAd!UCmg7z&6xh+El!QLmsCf*z*y`WY0yq;te9t1`ckGs`BxnN9z}k~VV}cv6RG{1IV*V4zK*qbYEj}vh5Y#-l zL*oRmnzeM-S%9KKPx>`;!G_(cs$RxF{)l_hJfYZU#e6x9rot0)d4M=y1mM!~A+iLD zKweV9!!onk0N9DvvokCFou_c6(vCW_Pm2?cBVahf7&MSHt6nVi-`L|^k5rUEBRpsK z&Dmy4e(3Cv$;$f7vbU$f7`KQlTNl#sI@8vV^5i$Egv9M>tc)Es@cCt)i z*ggQ#;xv|pN<=@gi;Ih4RxWBdiH9K(m;LgG3RX7=+ zErmn>M2sUIRRqFfWGys51u}xEYT3`FtevGmq~i=IN3_EI}E3vvj*LVJHdB%F-pG90dvNH593kKB2;rZ4OKcsMBQF<{$tazc;Tw z$OFu-gjDYFjWEH-k>-eH3gUK!kJ8P8%B_lCt8TT!1}?R1qX@n=dyRL=Ji|S*MhEa` zveTF&>>$n8d%%+b)3J8l76hdaEmltR9$gDO45;{P_H@_dXD&SB3V&^4V^9i}a*=`X zlEpF4yCq|$&gX!+fFwRsCPXJ<)ted!9weXfhUXhl9j~^rW`fnMpJyysjgW`r)o-~L* zz==Xr7Y@9_^*tGPAmP+dP`EX%z_|%x_J3nu$uNgJSo|=O{i35l5U5jLwd!)=Qp5{t z1B}AhHYas&25Kk;T9Jqz*Vaq(#lUdNTh1>le>`14H&MbYC@bT0Wavy+KEAy)UEQ{} z-^0rLf*uJ!e&Dt`Sk%cv0NrYPNs}fROe&@H?yRsNk2+a1K=3RYv^cqX&$~q)zcfTM zi+C4FH$B$Zj|H0eKhbYF=3oD_J+e8FJM*F;?^K5E7eCvvY0hT(CzDzr^(x~X`@NUt zG(ssu2Dc&e|JmOzS%NGFy? z0TuhB_&RaZ`!1Qtz&lX)j8GLL@}zrztnzUsiXf9*r&#(^F4xEm{w;xX3SC z$?>a76T>&HC?JVDsM?v05L@eA=)Oui4rtIcHRegDoP&ZERcLCL z7`i8|6te(7zxVzs2DAAw%2809=h`qc z6wHDxB=sUwogKarxZD8mr+@P~$0s1GkfLOuU19AGf!%_P7$(z)CvZTbFoTD>hDNBl z`!i?_T*&%A2QA6949iVqXa}dud6xrmGl<(DpnY1L^X=ywXX~8*R^Ba9kk*OF&l$Da z4o9TZ1$eAlvt+jn<{6#Q{>VOZ$`Py~ZE8jO0n}Jj5YK#hG1uzgAn~l2Ylz8A#dO<@ z3|G6BIz0pm^6T%L5#x`wBF+cQ&E#+4Z#4hiIpn

saAur@XyZ-_K|Dr(BZZpK*yH zw|t)FN<)PY&-FQH`nlooxKjYguS%X<~F~Q`N`So=J0j zmV3Opw)(Ft^XK1{4|&n!TF9TFGx?%$^1_{+mH8V?M=Gk-^sbl0m&`S-DwrVO84SPh z5&`$BzZk)C0!PQ#8H&0PP?Axb1W+0j2uE-x4ZCk&b6pIq$ zDN-A2S4;E^p2%GK{X^t;K_*(O&0^wud?p#MmqBn_k9YBOhZ~Zx)p(uYGdw zq}j{1)oOWb#$MT&zUS#&HUA>#pk7-*^(4rkg#vb&y4j;19`h&~3b$cjV<#meLlXx# zgpQ-{<;#~v(Os-ryS7p@FsmRt{l1lrwQ<0Xwa#f`>vxv*4^5mA>Errhpn?AUGP1|3 zlh>GzTnAJ1emFZdud!xqc(OuH|yMK_(+k z+hR+PY+fTL=WwL8)_tw-gPD;_*Q|HPCaKI+%>@ko+qB9afy8APUmfre#kWBCQGs?+ zQNg8pGmaymHDaBz{4j}u|G};INj%`zIItx$7y4=Yc4j4Q2d1W!>AgF5zSt=0Sxufixh&PfkWydqU68zH2N;AUZ5r3BqwWHE4vhMQpAd>EDL)LvNI-le3Hic&uEKSr34bTj;8vVr3&6qVSq_y__Sen(uS>xV( z^3X)@&anE>^7CDTsKhiS3AWCiB~cp^2C8M7Y&=&F6V_M>xuaejB6(>c4(pIlFN;OvOB+5DMw*LP(eMFP z7GJoGiYH$6;T%2y*sny4Vv!nxPYqb)_ZSE*aKI2emfmmzx1~Dj8hK)6^+2j(IzjeF zTuzR(=lsb=r>MeWM&mXiPMi)}{YCLcOK~Q^Kgi#>TY-m^B1o#Wd~zUz`!);k zK=27Z2MeiJ2jMPdm)#bABqv~FXW6Xl`{CQYn=OjP9@@guWz2{2eP&#-U*cpIoJy&9 zrU)Vr)NvH{cbW_xhxQJI-d-$%LR#pv0_Z>3hh%(B+J1HE(tH1Grp*@D*hHhf>T(5_ zCy6Zfod2?H%hr!S7Ehd{GVuOl-`-HAZ@T4AC3;g}W>Do*B9_GS@{F^tIgzhqmm64G_HqSRxiGTx~rFgZHTTyJT zUb$ife8Z*TI0bEDR3h*F?G23IZ^2wqrI9T-_Jklx?L3aMCqx@z@tb4i4n#1R!t%m4 zXO1ZU-_9;S@L!T#7Kwthtx)fvP>I&db`;`vpAJPtVbN_wO4oi6>1T8u_d;?I0DQ1) zSPkMBej153UVeVjX0>VE{cH@86_{eQl1{pPNjuz<-tdlgr()-Yg8X!!ytY1p%wU-r z9rvT(e;1XQ+x5tHO~41i$}rc3+*RVLQJelLUnbUTpGl>REJrX$XFKAx&|8eoBE}}q z7`b?HcmuHmuPnhF*dcMancLK>lQbXyP&ck=pKtduj4{Wre5In|Y>kw~jaSD7g$#zs zfxfm|%O}p0(R}%6Nl`+dHm*@y#?+73VmO05Tus;z)?ocC2j@R8NZW*-aAXiimBPrVE$FyBiMbB4fucsgzZkYV z@fxVQP}4XvXa!P*JH@#8;Z^FTzyKivZV~T%hEjqTJe?Roq1)Y5hSXRZeyC^ln!T}b zWV5%m!M$cP-~1ozq~q3E*fB?}uYYLtHr2l1^HpZq27Vuev{1?c8nSXTe~^Q?yrY4F zTP^JO4Un``22Smfcl?3Vwv8K=C_W37-b6XE)yjRA`nB3NsZa?OGhz^#5vMmg9X zZwu`wz`O6j^jv$;1jcD04;M_tqH5{Z?Ffy(zZ#S})EBg>oNukH%F!o>7XYHb=Qv;j zFckWcbUI8UT}h8hTSWMgZ!F!1I5LWSh6JX%OLGl46@p^PiXN-Eg_Co#Q6$~45H!mQ z15riWBFg_)kqEk80`(o)?Oj}5sS<$*7NJTM4ND31Jw%zndH?k*p(VeBjTWF`pMCm6 z_7v#?48x?7!gfvi39%p|_cZLcbbNqO;S?v%Ri4afrDYNnNDiXW#a>Ct15V-?L=5?I z&fa*v^m}p0c>{4z4KwQn^;e&%w3htL=O|CteC(1uvn)lAw;VF#Z{5EOdH!CqBo@G0 zG!j%24-4^yI&4Pk2@`NL^p=hefH6rqhC*;uCeaTkrM2(&9rX7?5&5RpOBjVKUYQmD znERK>!uk5v3nprh?z)(&@g^95BVDS}_%r+YEQXQ|>);fcFe&WYkzI;rRrD94{0mKL zvWL-@l)Y8j0O{C`g@RbOKE?Ac>uWgYuEf@E)c;Zly8w4IRysk>;qdeATJoM_zrj%% z8Agu#9Xtf!)CpXNOY2u#zpW%SE))(!@;B%`Hm_KtlnILmAhLmK7zCj)72Wig5^WB{ zbGH~y9plC;|9DazFfueW^!J+T=KJ!?#$@Qz6y(7fb)7&66iYn>t4}0S$W0*xpo}P1 zX@D1q>cSi*E!f4PlRmRA(!rZ7Qnd+iLaTE$($&Sqgbu;=>zi1I07@8I!A2=o*;xQb zvB!KDU~InCQ%v-ri3(YY;;1~I5m|@PhWKs@2!^zF^dPe6)UiFsJO8`=*R2rK&;#Ia zo)zWPmoR#dR5XBdDvp{kq#e%TouYEfIv{ApDA@u9>%#+m=UMowB|Z6l#jj-o-U^E0 zs*A~yV*%1`Z?gZ5Vqan!=Y6#*;znvngeV;(_%P< zeZPKaU7c!lG;iJR>50j!ij;Ou*J1`A_rhQL{HL;fFteUGB8;M5)n}O}_ZrO;2vAFN z{F!y*oV=Q@a@ga;u3yt%M;pu-dmY$STkt5Q@O$g?K34&U!|_Ft`nyx+swf9-tD4L` zR-p6YD_iY0*q>4j1O%g1_nq*tKiCp1o#%OQ*8PQQ%_pmEYPQ`JT)1ReWaKj?X6GU- zB5nV4*ka5iq(etosr&NgX;W(!)`}}E9Zaz|iT8UJBbjE^agi+%32Q}B&dbwZ9J_qa zr&_x>?xpD(o8ua99TzjYrl3`2weVn82TgE>2%)CAt-fBLvoz!>xEC0Fj+j=&9y)n4 zJfQ>KEHG(`6IPfxo;lz+9~~*~vltW!67SaSx{FJNYA1?;C>F(_fvgG*g}f1EiKaVh zC`n9+w|jL&sqtbJaLDL?TQMb+(E zP$aA$2V!E{{DF9ajhj|DyC6AR0nz}k%Awy`>+zFeNBBAsk+N4Oq)Wd7UqRdzZu$Zx z+Nz6I0)niQ%9|N;$}vw<_ju2nXHnkM!$LHR01Q#54Q+1oE{VYmz(W#~6=U2Uq_( z)%_|@`2WB!{1&a<-uGSY^zUryVsCX+wasN)vMwZSZ=T( z@C$1EKi3hmGp=&Iq{_$vP5775c&h4zBWe^(vU#6yTN##o&B-A_P1 zI{VNwwFo1;nv^s*g$s>&r*8<*Z4iA2CK0fmFs0=M&0xn+f#rm_GjCp3;+X~01zAGi zCq=YII2KE5MMG@djBUR9n^&^l^Da;HfV9?zpf}_PL<3%tb)tC;#z|bDCXlk-xOQzL zA$xX2jDiUw5CFa%gkPRbtW* zmgB|tk+hs-#U!HzPSjnF=9vNwGI5O|9%B<8r$Iy_wIUI7*az zZJa<``t~9ZbR23y06+$J-jO(UAZ^6v0u@l#*VkVfI+Y=^0MAzm;H=^JVp_uti7luZ zJh;JT0PhSIs2UXpD66S%(O6le`tfV^Tkb6fryjOm zl4S<%nA}^(jK_=^-Oj~Pij}-Azc?FrPScNn^_Wp$#2c^gU;bm=#6wi}_w5@K@QSmz z##=?lgT3n#6r6OE74s@|lns1isyEqe_}=z1#k$#_xu|WM&PYSIEpt?ANk!UN&*K{% z-s%$J$&a{y$`uo$19l)Hhn{>ddO~H5{|-@^$1>TsU+ToX0J=jg#_0sant$6lLPa+f-5(twuPYD8wo zs>cw^O@ht?>oxi+7ffafuUq%Vd{ZMdNuDLaPap|)u?DCIJQ&h2MT~F6so~;T+)7s+ zvc?dKq9{K-c*QoTT__eS62p@*7Q67Rne4X=^AxoxxM;-6rc9Z_ghmG`cJD$@Ft{HQ z)CT5?8t_Wf48wGcD&z*GeLTwI^{?Mzqhm5=AUs+MKm{9VtuDS-A89l>oPJ-Xd>b6t z2s1JqK#$bSJNku}mbU^W6xg`&@$LCj7~GdAW&-g{Fmkxq1BD=O3mPB1v5`mI1!}9$ zWFD?D^NzeU@ROlA`LflPjJT}#>nk4#*c9-Tt-uq8_k}_qaBR_qL6H2Gvs1!k=B3l> znH4K!b{xC>{`Tbh8TPNH8y|3;Ij!cf%Y>t}DEZ@mnTtSq4zWSCytX?P!%iIhYkNXg z^G6-rZ^d1`C#3kfz*~#fuU}3Xgf)%!98P#T+~2@3A#{R*y(IF<7nc zzkBJ7ltV*~XV2}L8ojMDD&IB9>)xFR9Mra(W7e@1ff5gQM;`n7@_Y1|m~H*t4`&{V zlQD?9%%jTIalg4ubP;t@&id(TbKNy$ZA5S>v`Y) zDxZ9*y>+vY?kxU-Z`WojYaY6~IXg2VMz8KprJ$OZs&Q3ad&T)61!aG;f~S6E!#}{pHcbg z-aq@^4oRqK{<6Io@bSu-uv@1ie71gx7K%|;-xI%n$unV%?>6ijfi-rr@$xBLqlLwV zJ;zUPj;6~YkW2{xWe4job0D0oBgP5nDZH;5$WyxBqU1k$%9K9%Q%oT!OGBPv!f4kQ zSrVh0XLP#|-mEb0^W|m-8-$7zai z1tATWVL~&j&9EyqC+Qx{7NXQfVF}*1k6RST4?+54oUg2E0Ih^r<6#A0aj*i~O0-9B zD%WF_fljJC^SlkGN(jH)_)f2cfPjDzui~5@b^sa4NVFS&+H~~wg8{HpHXoU_i~7N| zMxnXF_w#yv%5JOva&?#Oy!OS;f8Y#>8Xl?sZ z?XM&7>wt5r(B0fuuF>X6`lbDsC$0sbi-P+melHym^UWpOq*i=LTGw4;((|na1BEZk z9W0N?jzy-NZ+-tu;q-8}`@8m^Z`$r<{Wi}lKPqi{b@vQ^uWCv3GtF&<>wi~&DOjTQ zyQWxvpRw6uvBrQv`I@sjJfoef#532ZaGAD$lHS}uCZ>JYa^%|%hvAL`dLtl+1k(C$ z>M69nuTYRvt_c(t>Ce8cr23${G5yYd`^D8mJGV#V?_IWGUWt|MCEbz!%L_lhf0pyI zYd3$I@BK$=V~_R1u6~aeuraM2v)bj9GH6gW#Z=d$C8xyFZig<_2aD03Ggo(UuT|pd z@A<6Nb8v>$$TtVMlG^>#s>doVxW{^SmhTLVDT!l_jXp--m{pA**64kD=7}VpwK4W* z8bZ`ZUOImNEOd9Z&PS&sd1BfnIhr~q?`z8*^G-(|A^HzpY&)S$K@Hjz4Yc>OL%+_n z>C^pu<}&%WZ95Fd3+Zj`n@Cc4n}kKMk-ogqHx*Q3EWlj_Dz!4u=Qn>~QN;f|?eA3Fd=X@ButDBwvG0 zr@x#s8K6-!!NbSLHziO=TwDtlr&2r;4otY5Bk}w;UX}60)RF*TXRsPz4Ma)`U){8j zSTVaDl&I%j91%%gJnuA!tX{nf9LdIB%2Fc=Rnl%Vd-4>I)s(yfuobI9T?Q!?nwlE` zfA9|yqF_1LYb;xf%7e!ZxjpTiPp2}Rz^jx{I;)meYo!>dHQEo<&}>u&EA5`_D>{Yi0EkV29h!Ut5rM9 z)!#U}VWcT2KX$C^Zkl_%qfGi|PYI(zsU5lt@-$ULrtB{)m&+PFSs8R7f2eT78voj8 zvC)m{xG!Rs)|e`vuPZF5xu)y%WY!n^o`Vew7A_hrJ3Fn)J$cMCDa*R@%Pg&~nqoF{ zF6H*$)|pzZZc?}Im5d9%)L|hmBKM_+stU=kcGYE2k0m7p zjM++k3KnL4PgyS5wG!8b8qgw_YRopcx<*6=r5%cJF~~~5yaS~q0ZqiA8F}BoubzGl zl*834ixZAoi3ezV;sfHIjaeG}1SKsvI*=*^?fAEIXE(4Hm7xcxT@a2=2T3FXU`A~1 zf}&TpnYyHqrt@AFryhe_^JV8s=caSuYv@YE@prMOD+ZPp3>I&WJ@N6H8_5IDl#4ZF zakzu(U*mb8s1n+HHqt1TT?}-Wl5cU2*P3z8pZ^f(mMIL(gXIa1VVjQUMuPIS@iLyU6KHId8$Us=S#k%LmX4JlxN*?9(KM*)KEK9y&$W+@Zk;n< z*pl0_L@#~p)&Y~+?Z2XGdY!^Nzov_wZ}o9GS(4ov5$xg2owroe;h#XK(M7I~1{5~~&v_X2>)T*G5ybHL4-)47onx@A{iYN7HF^}UP zd>aza@Rj9stZIrXT)J>k^pnoMpUi;W3&m4@i=-42*8{mV_% zU_bA*aq73>-g)8Kde83#O+80*b9=c-DyRK>y|E#U?G= z0Hg?GKXLKU4;L*$Okhf#;}M-S%T4@Cl~VrH*oeaJNsKwy(d`gO14-gYhfQ`V!I3!B z5z89b0SuAx0v%x+vNxEtksjGe*$r124AlrA?u5z#Nz$vsCNc66epV@Jw#(36nyD9F z7|+#m=w3wQ9L8eF=$}S?GduR1hwE;k00|CsnaBe*51_i<#A=WL&UZys_Kv^ckXO_(_AW_r>PK_?2H%YvR}1XZaak)_`q4s9r{AI0P(D%S z(d8iH^qm{l4rOJX=hrjXEqd_QrM*+v0nQw%)kK-pAaqtXJ<@X#Iy(-%som z`8Lb4?7}MDI^(x&mLDD3_q!&UehQDGGHJ^NfjcE$gEy!V8CDzR_!z;zrcRq? z3^TxBdiIb6F1hsTSwAf-ESSJM;P zB!WYbYg+RA)Gcwqpk5yO8GAMf=8uU>)#ri#>yFtDdYG>IVG$D2K+|rJ^!K8iR8%j@ zSw$`pxPy(QfJu{)7baWq<%|)|NO|C*X-Y#x)UZ?sAj;9`cS&eqH7b%u9->umLx1Av z04R`V(=wo+oOdT#6=DxaaBz;&O*c1D{yB`YK*ZmKM>2P$vw0sQUJBxe9UQM;j)8}mF|69ohJgwIGxWlfU z_pZkryU1H8*wU4`IWAK@_O3u7`}sgn!~|@Tx~9vfFO+BrW$>+fuXFXy1NU z^b32;@xLdZa%gzCnF>!FIFHr0ef|eZX>vT*^mfm>oY)aV`Q<-v4lAdFtzm4?MF>DQM?%(Wl3&$i&#Tjt))^`)jIx2a9?Th%g&qQ8Pev)sqxoPKZ9r!!{_ zLdhQ2*Bnu3jL8)Q202vK_7H{3qG3ds5>G4oI@+!ETWtJp-f*SXtP{T95g4p{HAGY$(;(mmTQ%xZ**|^x)_o1{xPf8I) zrKr>B1vJdutrW5WyBN0O8UfKhiEmLvSG0BdoIVPQ0Z+geHrek!c@)8YJNCgePOFQr z>;f1<%c&7LM$23E>L9csIRkJ^e~;N4_afP%&hXR6kB0tDQy8uOBs|6WqWqV>JgHII zuC40mII5BYQBxePu5kM8>O@$Q7pOsJXJ>Kw*$g8EQ2WVXy0cB6K$b%=36vf$Lr+N8 zrXJFc3iD>Oin(HU%DMVnOsM?+4GUZMoHdA(vPO?i5f88#yoP5zL6^)r!;LW}Y_VK7 zT~g+jKbZR$+mj|YDBB|CDx4#*_fkMUL0@-*#-w8yo%jZnO6d3;8)Cb}2XJw5tak3mkZI z#OlKE$JZS1R9h@}^gLCZM95nY+`Z}O*$pF6*G!;O=UemQCV9Lu=+fk=lnqcBx*b99 zn}uqupP~LW@^DfKqg4zLSNrFm(I!Rn+O#kPD@$XqR{i2>3=wJE%|!r2nP?=zWvfXv z;_7CF*N|Qw5EUIXkkx$>{WMHt_VM$=L4#s564`++uqFF#KU(x{_3G7&tKngYj*T6; z%3dUbg9hz_Y(a~qpBJiZeZ+p~0kP2Ii_M4uAlv)vb)wM?pr}Re2PJ~rTJ4{U-GNz< zNyK1!V4XCgy3zJJ_5mJ?#K&jFJ5%~HbS&X;(O?_-UY9gHMKGSzq7=8)&y*~*h50nPRD);^d(8TZC8n*#(y=^<}{Mfm&;=wiS^z;hVVql z;mpu$gEmXH7DX29Ce+|tQPjjFu>iJKxV-xPv3sdLGdLno8>~;LILXykRI6;4$?TD^ zMy@w;xQ=-)=mtn8LMh~CDV6UmL{;Vfx~NsMM(gy;$-vx1xbeTqq5G5XW{L4q^TGP{ z`#|YViShb?KPx6Xv>YLegOz zg1Wn6#TPCaJS7%0YGNT-U}aC4I-SLLucyPMsw%CpTJx;-EE-n(JaH_~+eK`ocQ z+L_-;V@F(HkLnlS*t?L;@{E129&4he)l=)>Sa`hc1yZ-lAj_lY?9~a|zo2fUYDOgr zG+dTF$Uxc|2FwG5)xQD=bzDIxMWpGYEW#F?D;GzUC+d^%dp~ijXdoq~B!F-)&vH0V z;qBr!9S^{JML;$sn+>%NK3T-X8zZz!1~hPkkBCVvv=El)1lqfWhZ@--|5`xZUcIBE2 zWAN%byO0T;vKnYhJdL5P0Dx(80idhA)H%U-Xu$>M-`Px@cq43YC1ChIB1+KP6(sCD z`eG_dejyb>ZUC+%uSQIaqXY0(5*gr}K~L<{>3UNM8F1$umr7?s!A5E5P;0bdTm!Nz zYR0*6VHiXZqz@&*i^qIMkt!a-y<}H}!Z2hR(v7FFb9H51-jnd6z1Ulfmp|#*JLqU0 z^;_Lm*?zz~oXM`?Q84&u*~|aJS!j1joCvOG z&|McMTXFFHxtiNY|6QcXbo}(U^!SOy+Y^+NU%R;+z-QKLugOcT>niUVD7Ky4ukpQM zmeh}nh5Z&uJDPqEzB4)h)ZgG8uUbX5h5G$;lO;Wp2@RduE^DH`&PZsGZ)#{_pSkt6 z!{@wsp_uLGRbH5Zc+P;tl3pGF7Fh>Ow2qwnAD+eYc9nQeP(HFFz->s}WUmS zt-$(Abay3`Rrlu2zU)yvr6cX;PNfL2Z4ukPg%LaoK`PC|UM*L?P2* zEOZ)8!m_R98x}1I;erCMIm}yka~bIn(8u^tQG-)sF&9a|C^}tH;Y#?*<;!!-L(0K8 zV20R4Y{FKj25iR4T@HgDg#&^Vn$rh;9%)vH%YqDIkua?GP%CXLy9DL8X@yYLDYu5e zJ;6Kk-@GZJSE|8_Ya4q)Psprf%uyuCahz5d3p+tCb%3;Jb(<`L`3Jr@gaV%8TQkWy zq6dR4JVsdW#s2Rg3qpK+kmcr%h9Hd=^_3(z_6fSdOa_1{;bj?K=Xwn1FpBl3Zw*Ja zrgWQ538P*%!B4TGNEgn0K;DUMT;Wvc?dDyZ45`M58yrx>k*U#KbDW74(PlV*mfNGxu9&ptD_VkCd1 z!+P)Tz^Bxhk=Rv9n)4hE$?59kx1D~MfRV5cJpV#xmu~<*C>9n2N^_-{eh!0tZ(IH2 zp!)0Ehub-1DJSP~eCIA?u3EuHM#T!{8yr^DS$~L~2@3*>Zc>-EqZ0j*t`AgQgqJv0 z?n^N)-9vbNUEdbz;}%KJl>NG7QBxpoTxQ|paU_UmEo>{+=YW|dL$d`1$GAeiz4mef$#ny{Gi6cE`%tTwmJmh?&p{wiE4&EqTdj8}_`cvp}y#&O|^v zWO9T~Fj5hvj_k%ETXBwP^bOr5U3wNb9EG3iCY(K6vqILz2+~UQbVVKy5G6xj;7b<2 z9B?tR#39)3G?;kDfiF$9z!>i%3kFlVj+AaE{S60MZA(-#6_HP_{E&Hn)Bnzs-5(FRdrJ$+Ohz8lNA` z9PLLN4Z1*WKD$dE}vl2B$#wnkV*%S4w>(-Bx z|8{B&MUKj0jf7^$s0Z!>-)3?|+x@t#d`~L6m+7B!Q>1sswp$Sw1@dbPy+T%JA658K z7p-zD-m$0l#pMA_chy5$VZlXyYe$})-#)f`Z;XxK+Q0R9UT+V>fs{Pjd(dV~>&q7R zg0~kWw-z1M*mZNBtW(m{zg&K03Qm)6<>^1QJ$`54Hok|uuT5V1fO}-8^-J8?$It>n zO{z`?8Eb&0=x3tPU(>m~3$7U*hA~)mL=a>RqQ^S{a~5>TW*y^D$OdgNk+h+k;{Avf z!)-8U1WDe5Jwao!s3ArOc)$n8w`Lk+_%KD_A)|=&-PHMjMR7ci8#WkHQd+M@s2zH( zXuH+*RiWYGD1gV)klU4DjXFy-_qxbN(`U|UW%y{Xy0}C`9&S`1W}Tg+@prT*kOpU|6*IOZK; zn!G;C6FgTNRu2eb@8vaYDw{ETaWDj;KbNuBN5OqYIm=)rfA?W*d z{{W5n<_}Kd93GFQKhOe@cpz1nznMpx>D_X7Gl(f``Vd_J_l zjd!%@_s!KAsD8#!xggdc!}0v+q`}b)I}DbAEoYNTEP{+5RLwsx((lmTXKVE(r`@o6 z{qfE?t-@nWo`b1&I}R>3y1P~9*n%Zo^`?e_8Dla*ovlv{&(ArZsr#}m)5`m>=IPuZ zgJUKZvO_;BN7>!gB1&3H`c&Mr3wd|WCBoa$4>buTem}5QCEB7*i|UmHGK^pWqN5Y5 z-eCBYRv2Qg{N=Ab&g9hZeykOT5Z7SG(VieOkU5UPlMJq`oYn-Q8RMKknPDP}G-f)f z^wj~uIF3NkL8`+}r?K-(ci5|9_1qONquH1f>Q+9-xuD=jK*N&01t)1g+-xR9J@4AB z{k>)?B1GW|==C|Gqn~9?qatpCLN>77-jL=4j9Fw@rhFH@I!Y(rkoEggcDwU*A!w%2 zw{9r|X&;mMd6{%Ofi7UP8v1uu!_=nuVA~-scAG_hUPzwkZoBF3UP<~&Y&!%N z*Qa9D@0OdJ>*q05T7{O}km3W~%3y7$V+1`@0CW!4@}UiblB*P(4(d!{H*PJBa9=Wa z@6t8X86pPwvGFoWS%+>q4mLocHyFu0W=HaTH*>Er;A+!$ybAbsI45}a*@Yw^8uxcWn0BQh*R!f4f2@c@JBPf<23`-c@=th(_34lVuHQ`!UF}Z((p`BudevzP*|3A%AHw`$B$KCi7{|p zr#A#bTTR?lSM^74g=w_VdCBVnEYM zlQF+-^+Pzc^XK2H?8m6>>owi2fMXHEQak{o0E1!s1pg;VqM;@l>ZARh6nv_tc5l{O z#_$4stP~|v*V&0)BNmarp30LHWsweuP?`4rA<7P?Zkda{{~*pKw1tFN5XGyf2T%|C zqOT?idnfxPk^@o#%XKeayvP?esYn&2u!T~^&@?*s_}8(LMPcq!Ndd1ys;oX-589A; z1tNUv;k9em2(@#@;@)IT5@?g(C71&`hf98C9z@UgRPB-nP41X^pXuO?Ofd6SH3mI_ zzk~)cHX8#SwLf*w@Q$@%rcs8ScNRYn?|2!^tX(6&bme4Vc1EG%yQVg`#~+e+o6~5q zY2s>@y31@_G4TfmGe7t58*9_7hXYIcKV4|`@zm2+Q{NMqPv4q6zI*Tv0dzn5R601J%yI20Jwm|Hv zjePI-vGYGtc^^f8t#YzHGL|Pfto|9)bZE(-qw(@e>ahZ~&J?5LKw3^7G^QSm)*>++hpuCp5uCxd?9g-~nuKnbI z3=?D@e+*S+y&b_(t<}sJ$L^bYk+_=6e>vfT2i`v#3NhZhHqn0Vb&UEvthJ&9F~`0b zJxX#)tQt?Z&WR^YKlNt^}_D(^{_5eq#J) z`+1?Z$x6UU8M@l&6oD=O3K`1W0PwXzveTVDy$A4zR_eJhARMq83@-*iQwE%9Hj|cW z0$8R#@(g4?=YFdkD8Xb{CNvmNhq{>$5`hoLYM@@FgFd!BeKsE;?eK8J0bio?VI6_U+~u7i-FI%MG=rZ5{JAzi=8=bIBi0?T~Q?oNG##s5;qc3xV`r&_W-M22s!zhfu>!~KHwA|W_p{|QzjpSwpj2m7cfICk zyE!-hSNC9c$su8hT?5sw8iAF89HrmRo*&L_{47*bqW43+#Fs@53+BDrbw=tH z{0T}tsCM8ze~9^hL&ruc6i?-l^{`#);HK=4=|q+QZKxDIGXBTZ$`^wMNN?v=?w5zB3d1 z+;$g6b*gL#aUzJQ3GiSDCYirY2z-Zi8GVV!9FAp@MWo=ktvF%1ja3i-I8A8>WT@~~ zknxJ1gI+R^A_*7EF{RjFPdVrj6l89MJ&`_MQ>Bk16blP0S!Sn{?2$M^meA%5W*R!_ zK`Q~VY#88>W)_(YfEv%~=ln^GaLn?>K-;VkT*A@`p^2fiE4jDMX=f9X5e0BOp=-h-wtUFv^mJP)+h(K<_d}k z99}c+fR6ZsxFd$VzjKcWmGq7kN<;UiF=QK{cSGZX{1yYlZ*q5`{H!oHyb!PJSuM!7 zLOY%)lZbY?H^K^CO$d2-sx7!+Rjc@JGpzSTfROQ z(vkED%bAzH8 z-PnKoZlz#tx^l2Zy4m0|mAhIt*NY=l>?OOqe_Y&}`D<);sjGRXl7a93yUG18MZ(&j z#g{#4Ur@@YaPNS(SwvGS3P+Zcxm@d3fkRUatsFn|s(H?1Jy`q6G7WT+8h~@@yxAP) zP&0RCk7a?kR8(+X6on8042jigRxu&8qYCjc6n_NmGyMvV!jM!c(V6iGd?2lc!asEx z1=84t>5cN!(23@}IKe2AhBU|+t`oXe$aR28bigh3-1i&tx|z{vZcWh_Kqa?j1wXeR zRxR_xCW#&kK`B#uA9>b&pk~V_)#JFn){RwUOQLb)Lz(CIR`P&Wi`5^zkKC6<&4JtmY0AjPa2 z_M8pdTDwmDUfz$@$TVHEZr3DPJ+<OKZPW@L_z-7b6dDY1Isq+{sb0r$1jlMjcg2OJL+&r(b|X+Ly%mex=w2Y>&`CkeWm zjmid94Lq@sT8lJEacKO^sW|7<37bz*weP7D&26<_c0U8-mt;1?zyYLIJW|4)B0TW9 zf63?fA&(E|sq7iEMTM~#Yn%C+Z%^~M#MJJSu0i0li1(cRCZ9fogLu8{?u=w*_~xFD zb%e_EsyBa3+kbFLO`@#V=iyS@X;XKMBpbA?f5LiQqcJ;R!CN8sE?XxSqQD(z{g-jx z?X7m={-xmSzmf)p>IFcLNwS9VDoYDTiuRF%YTfeIVpRnqBq=Hd#DdEp*ek?C(q(~< zj-C$Z@OMyKHW!6Z;0%Sfk&o8Nz#xoNl87~>791b^W-{?anAD~*X+s+>79+sKh#J|_ zi}^YL8-S6(LvttXR-%ZBl7}m1va?h8A0gFx!oc={%(NDTT(*1e%sX@LTy=;#q5p0$ zji9YtNJe>k?2F&>jKwgm;M*x<3`l_f>GkMeQkuY*^+3)d9;NIGX#uPR|EzMRD+3(; zKr0?xdQ2J%gA%}``TNY1wt|P3NI#r?SqDI!CKa*1Nb1Q=fbWx{os6Tp18Z@BDU3a_ z##d{RbRUH(AcO;(b#f9m07oalT3WjOy%@-_7U02xjLPlfz%D$v*#ftl5XJr;~ zoE+#o63b=9U9zWi)cxX(>l~HHPpNp<-+f$1YQXlVN5`X@Xsf1=u+Nvq`8qOBrX4UZ zP#x^xdH420Wa2|MNJja?6`!TKA5iU#dW(+fer4Oeq-QTuPM@)D7m_q!f86R;=UzVG zkYZ_8saTt-9enpfgg}1uH-~p4IvAecWorH~b=NMDo%GL!js=0Qj}`w8>OYhq*UlXC zv`9PCWz9@@;`A5veZL&whEq5acFTC^ePK_@a0f1$ieF<^%KhsX{qCC~{)2m4Oj7Ph zY|7+ILG43|`;6oDoOa5dmy$1y60TG|q^#Ed?Ah&N?jhX`R=guW?EXA3^i?f5Vqzi%NBJunPE%1$K6903*8sTL7+ zD-8gMv+y1qV`J#;*k;ED^tdpb(+uDug*SnUDdMCm5V1b|YXEl!#W4}zf?FgCYJo(p zIKDM|9K;Yyiuiz%$4akor63jYRonDFc}z~^lhBjE@<)6hHJIHL6t3p3$1{K|wyaB)?^h{s8I1AYT<2-e2%09^C<8s+=U4w9S?3+s^Zx$- zH<`y?A$xVmN-89kIygo|OIB7%DWfu?VIMQcOh#oDAyhWy*eXROWyC=m5|VcP?$04te)_J)hTjT#v`&TBq=GU^XoXXOJi+3O=PK(!;#7wy-!ROe|q6cz{_6 zk^m?s)Q3ELl+Oxfhyqf&jB3^BKYq~zF_#5<9f;7)>2Tdzii-7+8xdUXrgB7RZ;+@f zHyRllIAhC>9V=$5)Kq-w#*G;(Ga@krDS+)Jd%^|`U}mk7J#xy7D>ol$oLte_rzGAn z`SHqD7t&Ocu;56Y@86x~y5sp<*)7|K+%Al9)4Mcn!v1@zA^ZC5EYWWA?A+z1wO?GR zTeXsj;MV%?zXb-)qyJF8IR3{c?NaWuZpT|&BRTnQ_G{dBZrN~ql#gZue|!-tvY zRFuyeJ!$T=kc!3m4QcG%f0TXJ@V%h5WKV$Q>8Bf8EwR4Zs*PIUU-n%b54SwYm9-0f zd++c+*L=UH2fKWaOfPm=lUC^T!6`jzO2|s<-z*MJRbRQo6ruEP{Xo>CKLU1_K1*%? zVcxJwS+|$){eRoy9^&kqvDqc=j#hkE9rZuW)4qk*R3x3Hz!3ESWLE7YpyDv92Ktq= z?e&+NfedR_^F*z-B9$#QI+L+1!#wC&p>RE-gQM#LZ%pA ziYg%zz2*0XmAO$L%3B(ydKdSv93dsw4-wq6EuuvWa#+$0JVcecPR3H=n#8D(krYL##q7qve@p5ynX?^baFG z*tBV4)tOJiIzhw-(Z0*XjO_WjTX`NI5coKXOI(wAF8#G#=6ySuw%^imX+uT-=ymJN z3S2TYhmPE}4~MM;9JYEX9k#~lji}?eJg2EngNm>D4XfvhKJ#urSzdj-rA6s8@=KpJ z#x;}Nmm8lMHe>Doaa#udJ6<>3Hy% zO+t|-)p^mEAYn(E#IX*R6IXrMSRvvFQm(ve@8EL@kFj=bw2?-hR(P(6mkdR@mnU3^ z6#k)Ai#eamY8siaymFz#s8M)kguakt))4%hwc^$zM{bbSK93&x_fu~ZtKbfd{$sdB z>xoSoG%)9x?LnO{greEA`n6OTl{2V4A=1zpVJT*^nGf8kr8wWAC_ebe-ErB{lc@s~ z*EtX zf+0641-nb$KX7dQ=MZ+nR21De%*LV2tHlU}se4(H!(x`{(l6>U2+s9GWG4*Jr%c4z zB5iTM{=@)WlcMscGinh7&^eR326`Y=*NQ-o=q59ax+;;KL70I;HcWvx3R~50aQk~B z`tV7W<|H%E!dKgRsZS&EO96HN!u7zSsIlh&f?7ZZTiHnt*Hp9WDnRWMnnJP z9W+JV5<%}O1Xz^1?ShvaN;@=N8Hb*K{xs>(BX|D}Dm&BG)yXV)|EDN%`?jwryi?PD z{@Zg|k&T=A?`t3QEGXk~&M!)q`JDgb3%O>&xOz3bQzpFL*S%o>&eQgrn#D($-a0hu zw|W;K5wRo}|;a>VEKPF65dZJncI9~&Ah=-U%pgnx#ne`uTU(m(Z+m3e;P{Q7f+ zzPy^kEQE?8n_A{(8n;H6Ei9IPBt@HmZ{z$rQ@7DX7iaU}xq(lhf{0^ZeobZV?4usf z#RT+}`G+_b5OLa;@Rr4(mBxHLf7DVeRmS|L2x){>WD;}&*_O(C?ah+lg|Q4?WW$zN z8Gv)y%~JfilS36qTa^P~dozv7DVc$qge<>?%llxgmgnxz1%!bCcV^pkcsKBaSLO2R zuyskf^M^x2L7kn0ca4BJZj?xKrh~aNd{n6f6PED@9 zp!UCzvJH~NSqi-@(1Hyf%pgQR01ZRRws_LFlIb0h|{6{_) zG68E;>Hq2YhOH_(cke6e1o4w2gOLn-UOV)Gu|=aT=sze;3z#|;6-XE*u4sK+n2>qs zp{wJviiUVLe;M^yUpI61tmBPe^-jJ$#y9nR!XG1Q zPtm#V9&&YL?Io|P-LDk87S&d-jw~UrU6#5WSszt7C+GCX*Xq5yS+}cvH?iPg!VCs% z8s|s30syZnN7Q+B)%Wd^Na>lz3MnOjlyR zM1oFUMM1M>UEuwlWNbNRj8>s*n;D|1fxP=5cWLjwedDZ`U702kG%(@dZj7IISa2>OyBpOYS_z7V9?ioqL%Bof~^AqC+3}`!3HcCsXNx6x-f(Dh< zY2s&VkGhR6^h;#;g$v`xt-bfpTg`qmCRS6hN^UU+M$APwkj#;2K}Z+j#i;?H>m3 zv3;vMRMr2l=Q9t~iCiTAZNaJ^)ki4W;^p+}V5G>NHrXfnQ$2@2PCa_1SA5syPMbPi+GKS zA-6dojgs&yoPPF^2-wJS+%t@ttF%1iEnyZjv3D`n3jF%T{gR(KmSR>HoQ2(AO+fO$SpQF%x zxcx)J-{I;x5yoQ7G*-sxTa9Hz*UpKGr^ zI|05yCAxp-51+qZxmXjdgZA#-d#`x%by+&4{i$`2uQN6U^y%LHPQuckGx-qjlES<& z2(}L$bzZA1!6NaeLY2l_%@AHVDv7{C2|-+^a2HnLPunttVA9x{MeK(cpF zR=~;rC)`xVN{1%!`#-n$zo@H7IEU0s6l=Ww0ZDyTw}^tiQCqD$)oL2&OJicU*NC2r zvq&Z=bn1XFDqPP)N7qDzm>z&8W z3S04umT8pxTliF`j#^b)`$HT{1g^t+%SLYFT z*K3?ycyQ9ai{{3K_V%OdWUoJ`5FGaE0F?FER+y6wqvoRk5WBlBy<%I!$_Ja0edIUO0C>bAHq!Y8U{~}U zab$u|ljFofv@9kfFlBGxz&IIi02!2@E1E{d%O=0Mn2!IBNKl(EjWZbv7FE!%1l zx!0O^X8N8XU&v7zp}W&8sdJ~GqldKrEl8G)gHGXkMIdB2uxvn(XIh-_*0#x z3Sn1T^=o8o+$YX&+LVwWWwDZ_`-(Y^C{G6tXhx-4?grUfa^HqIEq8+Io#myAC|$xx z01XRmQnRg=?fB>6u0QxUn)e&3_dZ>|8(O<0cgSK^2^nyh|>~7yYIkXu4}4}j{b1k zzsR`{n+u7WKF+J0H7f}iBeWJU38u*TaBeeVJmZ434O760?TG^p$-zT zvkkhBYu2*mF$xMbM}dv0%!aJ?>2wb67oA~{coY&`U26tCQ5Fe#FRyfCWjS*5I&#PD zD=DbVZqRIt>3}$=CUqL-aSSz^tUl!7z59kGxgb3o{F5E=VGUP83L1fn@i9odz9L*H zKd%d3!}OoE8_G)Z0V7dX_&#lIyXx(-GvmdP4d#RLH8*y+&fifGxvjFuw=zWJ1HF2Uw(9LVs)j->0M?BouAm^wT1mIr>B`=> zlDPdfv`6B>d?v2X&O}V1fAjHRgCo*7SX_%=E%I#L%pvj1v!Id{g>6h1wm!?^gw(&y zn)Fzgs&ixdWv|!6M&6#fQKM^u{nGB$jmejWmpxik|H7}Uwof9<3N~o|fO6de#q)S_ znPw9G-Z21ym>fIfohLer4=_RlxS4Z82($AA3sTZ6l;6gvfM6q8*|zN8*1~+U6>S;1 z2l1*GlVSDlwG>IZG${0Z<^Q&h5<*Dm^aU5$BgK#WOtuXndvHB`bW==AfDW=vw)kvQ znlSx#1T0$oWx0MyS$;e&ERG?*OiJ?992ADz=m-hei~|0Hi5HLb=kTU<6Ky zblk(q{G38Q94g6O&O(>A2}M}kv`I76U)VVCEve!u$Mn1SBv1ov5sf(NHiO02=^e?2 z+QfqLcZL;5MS_K`;3JoL!(A0d@5J8ur^atX$*#xoift3Gyg2H_K$Z{XksC zAS^QfWaRVo_kdML#I@M-&!k>B>_9M5b7br(f{4+Z6i2Dq3sEs2d zBK70L=e)t9$C=WcYI@gGK2Vakc&W-lKx|M#Q0RF6NS!BKMtC$E^2i|KMH2K3Nj*r% z1Hl+z>xfTu)Eqx1v`W>?%Jx1dayg|Uk1l_Gw8wnBp4mdNnhICSzeX}M{khY~0+IXXv8{!}VRTuD@tz4nhu^GCpK+N zuQ^DrLMh!uax5*OvZ`?j) zp^)$vh0wC8{^|`MMNhM_aP^!RTr3PL140{hl0Kr};Qo zS97>-D3)k@nzz8pu^jZa>PT@0JGVLsz=(xH_`#fV#h29JbXPRsO$Y zF?-E|s$-$D3ECgyCESssWEG2u+up4U zX+Xh2Log@_9)>#%+Fi|SjHF1A`Z#E5Mr>Pgm4w1SC?Y&O+;q&Ed!5B^a`9$y>XI0s z-5#?KVSfTCWTjk!HzleRrUdKe8r?RB;Y}x~<~5JpBp-wT!}w%dJQrTJTAIXxF;Phi zqhs_*MbaQf6QX*cytA;ho7PeY+2RDKR7~jwX31m(f@{!^+od&XBsC!)N#sOC_x(V6 z*+>a+dwN@!d2Nj9&mV+Z`%H8Pk`GzHux|G3+1Z*p3}9q5$LO5Rm1GpIi)`ywzV)_d zg@~wfrVU@uEJIuK!8on!zs9nJIwB*vF%x(Q9dIUS|-Vou|Q)@?$zU?kNFbd@t znTO6!^f>Ceo8Qv2kB_%`b+A`!aqYhr>MdI=7&x@1LVNUp;h(%~C^Do)k_Mf1Po)xs zOAw0`Anz&etGE6hJ5Xfu`82)a#~?(5CE0i=r-%VR5klRmD%=zZ5x(H@&^!Tn(pXcN zKKoC39s!kQ^R~4gG6{2%oym`Cc_|Xwu<2=}JV?0^5cHLc4)TyXVrELtcY3g+i`+Z+4B<8Vz%~ zdxRyn#m2G(x_ zi=fzn5O&9`6NEYz>zIzItv#OPt@UzLSI=eBX5s+Dh$~!qR7iJJLc6FdvQt`PC-+mU zz=*^DaHkn9-Yy#ae$il@NR{roP%|hl&H_vsCKgB1NYJ~Qf)Wva-}}jmGW5m*PeP~2 zN}(MzH1*fsR5(Vv?Vv5aBMZmNY)jj0W%$_SKF?1qefx4~V#3`zFLbU)PQN{6edg-K zGod5)T(z<|Svr1S>JziIgEvlp`1k7UHf=*x8@F>Us=oLM5B~3ZG{4TO*AeAt36@JOB^oO%;?$hIAOMbjqs6*st`%^#EI?~+=ZbRkcDawBFiDjRwL4!-mv*6=_ z@ZS*vyA*zMurFy&LLnj3XRvssJEqiAxMM~>1PP@WhLd-6e1O3j`u-5?Q8Z8Bt|U?} zU`27QWstgS#BXYUGl|Gy_)V|9^XAlJvw@ohe0ca!A&e8*Miz!<_k>V{k(%L0j_=zF zS9r3lM|f%-gD4M+Nr)yJ1U>q)KYL?97`Nlg+^U#kjh#KWe@-Se`dvWFXuy<2MB_j& zIU7`I_929%;r(xX--&AEo^o{;p`$!oxevoV$KJTBqF^GrKS4v7Yen58aVZ_hUhE&! zE~K|%+4QXf3Doz=iy?J!Xu*{k)>*t;PLU)@4_q0Z)%ATf6`k4JG#}7OIziiX3>dU$ z->C=I-vV-C*tvhs#0~27&}nH?MWemePcz#2MqGB?ePsW;3lr&u9NW&ej%j*rhGVNE z|Bkt&wWYUvWXPmT9W5WSZ7XYm>dEQLeRg(JN$%3sjK1&luj8ZMk7?mwVYCIyOU-~xmUKh_nz232!UEZ$@j(0Rwd^y6y$i$dOwp*iK zIn@DNt^px?ya=NXHtzq3hop?uxuQ&=QsF(^m->s8S(*m8oP?^!h2(F=8Bp$6%N6x2 zg9zFLlZ$RylgRePX{Z&ffb&hq)lw)DG{v7pmKHS$Rvq_$M#)|TjHg)gG>jT3=mq2d zR%&XFvuPWKP@yjU^!%;0H^xIU8*94u!c6}3}B z15ch@v@+t~F@JxHdBtYDgnHVz*v$(5vvw+#-K_=|SwFJx9UU{h|Jfu8gjQC|W49;G zy;Cc7cT;=os)3b8!o4BC`aFM)ErlDGHLLFY6-j?j=%=dTd!TuZz{h;!tlYCt?C-PO z^CR8$6fC=fRPnlDF_drpJbyp5ON?l}&4FQpUL&Xy5C1B1uJQ!MpQVvo`Qy`oKC$0C zWgQoq0Gvjq-3K%7KLy=Y71<%;_wDmZJOuCH%a<=-ZLp#BKS1?`QI$8y|DKV+(NZ4w z*;+4A-jeXv!ko`f{$A)^S;z+RK?7Q;9c5`y4r8jDOGS@}6rn|npe|EcrNrBSi$~&- z-c`stiY(Xtof4UbvO<=^LXBVVNl%f!U`X(x08WvLEeuO|<}CloJJ}SHb9A z>e30Qcdu!d4kU)GXL4y_USWt`*f3e$fET(c}3`iuq_h<5deJr>JjrTQL6FY813d=<9qh&{YpAN8Z=a|7)od^LyGQ7Gz2!lsD#<#6pWeL-q-?UZ^liPZ4GtYMJhFNSpl2uB)jQguXS49m9p8zQh@X=mWYm^C^-1Bb_NzpkMoRMvNTkzoVf~ zcX0_c=LCmQx-1}b$K9co1dcq(rIlF#oZxTE9cLt6<53B6#{`4bu{oW4FlUJ%zNP^t?2d}Kkv=XX+JMcrN)~fjpk=1fZ*8Q_Zhvp_bX_FQt`j)?b<6o z%PMZvpqYD;VrQ9$HoeyTcFo;QRRjMrI{UC1wN)gCALvm1_7&RG2MmAt1>uqyk5)i2;kw;C(ZMgdEg56k|;c$#K`)pne_AdRA?4HBueN+9aOmWzh zasTU;dBZ%*3sR(65_GeYr2Hs0Hi-pXNt7$wS163SHLvpi$i%xO?Z(PW9rc&L8I-|g zk&Ffqa7-4Ifd#5}zdJ#8QHu990Lk}cQt(kiC5Nn}q2KTH74Q_Sr%dq=V+wQvmF**( zMq4-BHlFAol)LnrI}2$9)0Z-@N&s}B2)1bSgeL+h57ldidCS(bD?b}T9Jdj5 z!<#lxBpvtb{NtL!r{w-mTXd%Nkk$TYi_XnPi*i+9b|D9oRJF$)^ZWbnF4QaFlml$2 zr1^4aN);f@15LRTU4Rm3iQ)?3O_4yuEKaPh?Hel8_QS`7tUH52d9qYasyd6VOq04z zYCrHY<;tkk!NKV}p64zVh7_ZN@5yvSvge>t8_noDbA5z909Bzl4J?L;I*G6^?8?FI z8)Nn}PS%)0XHsqOu-RK*tP2)Iga+XbKq<{?}Z4i_uZ+5EU@(K&&A!Bl$=81Q!?&HrJ@4tte4;Xw}I z$k3+(8BRl}(;PK=�*`!%nT}tSGC+KRn&p@5yfs-jq49+jmNmhPKfeW0h6naxg7; z?3E8Ltt>8Eeu=N_e|k|&uIHn*I-0t;)Xg;?x5IS6t@QG!73WN=PDjeG(_wh{)hFef zl>Z6La_ks;L{DM1?|B|8!7w}QDGU>WXF>8$1;Vn|D@KVl`Gm=IkoE*IgqmPjE|P!i z2^7y*tuW2;(`Tl_mv$Rw<`Ri1-b7Ph-1ox&@AB>QyMA*;U=Wk$Y{b}N-XJP)NZtj~DR-a9LI-)6g9ksq!~SKOz?PWkOdkA&GG4i= zl2U%|En!hs0?-roBjQ)7zEd&z7b92(Lm~ZvY<{j$iOJD9tEE;&y44smB=wk^ z;%0xD{B7U5_5D@dH@|bR1kC!`5#RyxP8UQHyV{8i{@4^}7e2t21Q-8KDlwS9vmXC8 z+Odja7YG#)+DgWk+H3#1D$x_nGD=*$2mU_9*y^+IttQXUs7C(Q$SlHXWqP^wXRFbBK7RG?=Zl|X-&n`LnenWztzXe!{rL&Mz`Tkr^&-BN z{K_l1X8xZ;^7dSe#zXUWe-dl$BL9r*aIh;30$V=x6RLVQI(6H=V$pDEVG1Oj*)uf# zj_~p&;Yo&uF=JZ};REKZETV1@QPnusxlKPvbdY{f+C}KFCO>_%WnH$Co6VxA-w{i% zmz^h|x>9}%y9n0bx2cPGthSs~OQM1ll_)HCJ^gou@GZ&i-lZ?n1pzoI1ME-F&u7#& z*)@-%n~0M8>RJlM=5N=CaUfzJ8r}1(IZk1UxGI-zCM^;JyFj4%Pdo|0y7$J{d+lUx zCnjcDp=h5m^_>CBX(9RdIvx=W1MITP(nW$Z~EMRPK~~iT%^z zoXXYeXW2QoSZ*9dGmR7Fv%#-< zXhP1S*Z_`Y3}I2XR}F>3s>g?ig(^9b9c5K0Pp^uX zS-KytEKO;v_4~T+O;4;_F>Xoc)wTZE(e{rz6OA%%1%)POf6h+6RC*Sc%=E&2d1pSC zmM?X;oNBk{LH9tP&3`@Uto7H-n)OH5Yx-By=Jw4WEY~{twjyDAuw%Pgbsu~@_`Y~@ zj9qTwyiPe+PHEmwE&bF)Z_~FDt3#JRJWCvQ)HJyBAJ3h2-M1|Yaaw1h(&^mi!qN{v zN-Wc}8A)v?Z}*(ZZvM|$hxH0Ny94++kf z3ZMLrTgu+ge>WbIyXvn-il#3GAY|5zo~E0o*y6}Aiy5%%EX~=h6A;Tnlx6Ax@vZ#= zSdR(sfo|cWNr^J#bv`DWVp)sZ#Ep3W&l+w)D(k;SW<+n>OoI_}S1Lv^I zN<`1%eoa*Er&(B9F4dZvn%ei+f$@tbsOv@G@&}YQ*&cvOqzpk_1Lw$g4`$%MGhN+3 z&)!~S$g6y7=w0#mR}){xZ!69l3QK;*^0&1?i`w*5`QmwVrR`KMWl;0Z+B(|WnG}MI zG%21wH1RN{LIw!|5_^2(C>G4)Y}q(RxKFV+fO&G``_r+)+cT@}=N@^}^1h3<|6@hM z!}d3xgd{!JJbv%njn;+%c*;a1o^S}B{&3<6Y$bnw#hyP7p}Cab_b%ILP+8`R zW8l($cYK8{6;tR2GfA%J%x0b8bBN)?UK=4Fs<~Ov@|#lw*r7j}6;datkH({`=0@h` zyH*(l1j#UCdscj5=JvJ&PsGVSfQV&zGtQ0F=SooE!_hSxSZeB0&MsbhatCNC9HYr@I%mK=G=i4CQFV1Jx*T{mcOZ>+x ze8vaO)3#k$LlM-4;Sk%)uG>9JOA`~AODksT=xrod+$j2Yi*ts+1hOJrWkl7VHw*yH zBiB?r%{o6j;O{R!-V&~v*XWqlQ!IZf_dBZiK-3di!b+*Hn`00#f$awg!ZyzV(}o3* z+i6p$?hGp8GG4=mZt~KhnZpE<|K4{D^e(23Yg#mfLYU&EPdJgoM5mq|bU0^wxF1IK zp_kmZhXle+2hBYhmnL3DT)9U%1!_H24j;_Ou=o6OtFCj!g8sB%vgzTJ&-Ql=q|$gh z{s#{9%@+s3^{r>@c=fp3!qt!akXwa?Z(_36t|L^H<74*;KNfGW4cQ9)*kxh8)S2aK z)L3(X;**#X!JxCeBqi$Es7{{8=u{u(6?*462^$}aMcII-5~r_Os(t)dbi={clJ6=hmrGPpY1cIb3((?(b2JahD;pd$in%o<`ekg;P|t}3Zqf8 z!&89}1vd4Bcce#Eb|9~|si~=qCLo3@**L+$;S+6}3N|HzT^h{j1>k~UT{@NCxBOT~ z(2I>R+qP|64QpjJaoFYDZyii2$)KjT9DJ6_bn;p=fq@E5Jd&@SCic)gI7lgQL$IzPFUASZxrr zBZ^lP4TntwJ4+oA0hatrI&l(xZJeA!!7qd8Mj~tuFzh?*RWko(<&i{gBvt=_Q6jZ{ z8(+ytABrd-x${f(uGls*JctqA7h7(KnTnO(l*Q;Ol^{y<9%rtD+=W`)$t9L%tovFZlhsHi3^SCwIcz!qAH6Uiwl zIpu}v)(qbcL_(RU+V4EK#gxu|1Y?`+fE}`QTfI)b=3Y^@NX;kBniV7^URqk(sGI7H z=*98+nF<_Z?Ud>R>$u#GOMSn zof>7Q_ZYW$R+(4B>Nl$>Yp<_UmH7BqZ9YW2Nb0+}N&OnpyOR1o>Jt3QUJq}FPCaBU z2;q$H_a8gW^8sXHhfN+0s;QW8eE_uTjx{ zd2cs&u2|AKe@?!!G}m9$R}?NLcG2<6n0E>HSZfPq5tvqL%Z^3+sq&aJi@P^Tls@UO-o9DH3Ac<<}P~Y>TKH``&tizBc ze?*nO%KI)1FsX4Iot&ZySMg$GNC7pWWS@6&Ym|B-!}(BX1|+b9A4*KR1q8K|<-d53 z>J~n(acEO|Udrpu9y%+_3Bg7?lRgzV=sza;9_`12gRGxcYHj6M-ouiQDT zYU4bpRJ`b?*arI7#w9h4>ayiJ5V$l)c=q%^yg9PY_iO3p<3)u*L78@?X_Mxvs?Rdi z?veMXnHGlB+u`v&Gib&Q4;y<%K=!i}x=?=fq(jyUCAeCwHB*I?J;Y_f0&NR^tb{8K zktm^nnxCKloXSHvYz9==@W?0NbLriLkjT{CJxJF z>lfvMXQmUD6QL}U%?zk44v1viCM7+4CN7zyCpu*A?jHGi#m5ema4}lGoFnGs$^dU? zw5aD<7b_d*(5!}vb3H?yEf>3o^N-bEosWHe`0H0Qr#bd%a!|$R_mll?=haYjS%6Oq z2x~;F_+`tsiTcK7%QN=RlPZ0(J@G)GHtu&L%bzdIKo(fQrL6d|u+obB>Jt7sAt8_l zFB>UEB){5NVYHG_#}k^V(^L-{0|uOryPBS7jas#W>o*;{#EDM5o9rAUwVSwl+x;N( zo;rT~u359r%+YMN$0Ph87@Fv0{v}1veNy3uF-;#SuNp;W1oRg`zSRvxrYF@^`@7&uF@^;_|( zv}?=43@VD|u(K*=3J|c25V+25PN{q`hl_8sy^~v5&4ee~g9haQ{K!bYA~(Ill-T6? zzO)NtM1OK<#n(=i6J==2UrYQ>9Di4#aB~q(Dvmmd8J?%Wa%D;_qTKzOo&QXVT<-p6 z%39vl@<5doHH(yN&+nV6UzsZ zmSm2QBI1H!g)d#%PVeG6o~{p;{jv12j6o<;_Qii3X!cuHR7Is60y`^P+c0P~?KEm0 zgMl=p{r2ed@gMO={QFi{H*Ffbz3)){6qkEZvsyAY**mAYL3EWl6yoz_-NKx!sGOdk zTV1NvOpAuh(#&k;eo;6hAJb}4%dIJOl57AZzxBxn=zuIeM#QC*FGP+d7c zlR8^T3gtwt)>0VWUlx@P*@;s1a34wox)Yf#Cxm&IJ1M37y?d|Bbbm~jYwD*tTv#K@ z77TVBXp(c30|ghbbmpsN>6TIoAI4&rRXBUnO`c@&^W>fhu@2u?TBxWMY#(O3GVGaa z0C)7~*OwNlroW&7g#ZAwNb z@3QmMUGqYIJ5j!3Z9E$^VqT&EhnsyBLp+kWeEHmufsrrbH(6E0Ucc*cS1D=9=moGS zLYWR5bx{Nk1F#k>Gm7nJR3~?IQ%t*`JbQM$aLpv8NGc0i*IgE{GaSAcD_m%jfys!> zPZV@g*2eTPGz_SF?cMp&rlyhYQ>Y$*)a_?Cy}*C(bDy`dVGN%}FN{Zbgj@{Oey_Dmj+m}Ro#ZT;q>~lvR$F@#-nG5Xwyhcdsi?uEWpLLCu`@;+BR)Z zmNFRXI@&Js3|a3@&?CpGmeXD>pcJY8syF)6ccMwNaA*-n2<2(;Gzbtc zBkES-XvedWk(R^}Yo}26#5fTHNCIosw1l>i#y>+huZMc~?hf)A1#{vx;z^f{s_6+6 z+P5HSI zW6A3j{PcyZBBr??y({L$9U5v)eu{iqmyDg30)aw5SLi^~zfZU--qbDr*JG`}zCmK!>Q|^Rda&64e;Ce=V>i#w(lftXXTIXg<$`UE z1|DM4H)j{_e!}@-P@&27@fVR11!&toYey3)Nq5X&EMaiWd<|T{b9UlMh=uPhW947t zb6?ccqCAv8`10jT-5fj!WlVW^<>w*X--D(D6oG5SaE%r?F!->?7>55T+=3Sw2bu13 zQ+r_rB{FO1MGbc^9Z#AeS7V&*HFIx3Ko}Z{Ueq+S8=*B+>hv@)m_ggRTA&`W456i5 z4}3+(fpvFNKT635AC|N|Z5+a;&kOAVl}&o^G3zyHk^&V))`Ly`oVw>28U0jH&~X0J z4<#yV9?{vkp?`|$)4TVUbko%VLC24e5Ib0|pZ)&(_wPGVWD8B5m)|lVKA8GanNFUu zAVokQ^a=vfT~=mLR0h(^hSVuqaVj*F&vT4`qU(LJ(;dPZAQe_~f|3yzUj#LWU%#1B z6v6}4s32V@5xoJRd~myv+gz_xRC*W>h|Gt%xW&%*Zl*iCb=%aM6SzaIoxNTSh2F87t2}Ef+&c3z8P*FnV=;Yt2S0B{rqWq% zXxzc^Tw`?%@?>}oXA_G0>J_9zHpU#|H=kmjQ0M0wsg7aXQ=zH|!Hr1%V!`Ff*4Dv= z>E5<9Ps%7jxHLIV^K^zkCbL3jjN+q&;Oko45UNx}4$nY?bTHXaVr99jB%QN82YhWz zNm!ghCQ;5_Wz$X`WKVM?o8;bzG|)S_qK75Kc2gX!PT#=(A&DM-nD=K>2P8o zPRpm!6o`~efbg9uIgMfsOES9&Jyt*SYK!m}xiV8EOo4Z+7rK`%TN>W^78Q`++HUI}MJ1bO6f4pF@EsNQq|I4D~FD05rnwqG#}~ z`)&U@$OJ*GBKO+RYl~eS(oax z{l zC7YGB8`|#K7JO6{JD;}nzUFWpi7I{L(~1WMJ84ypSJWHgHM+ESPNd_X`nI>L-!VYJ zU%{KI`%<64CxiCZHK-Z=?oNs&IGsqv3CJ;8vOyZst3ccuwN%JT!VrNqd?B`7yLPk$ z8|oS~tV#@>lTc_%IT%&;q5q5P{kuxBP=b!9AlR^cGCb=bIrCa*JV@$<&MFC{n|HAMyMAy3DhKs+Ma4#_I zG3(%SKlfVY*{YY<@o|+Kr!d#PkJdG^c-Vv?-yQR=URayKLFQrwCkNpCOw{zIFG zYM35b$9MCoD<6NoTSZaOCtriw)xQq(Z1}0Ui$v{~H_z5o=v|;jW|i1KTF?k&F+;;v zT_z7cG`{cjs}>Xhmtxzgsfoyd$Pqsdbqrt8E%t&Wz?^p!ncb#M^TWLmMTyE&_F=H^ zu3^iXinOs*tg??)^yF&DSX2hh4qV%@7ikAG(HgjF>zxjuMmRF!Ijz)R2V)G-#bAd-x zp$;PjxB^cg(VmhYo_IBg^(Ck{vt{+ zD|T6x6drzYW7*~-1A`-M+bi^I*v?pJw(I`9|CZAA$0z%ds_<+(`%ZiW5 zRg}apbDuBJ1wd)kQC~=v)|5qduXs}8&yNyc?D~Mm+Gxo9fWDmiZ30xBM0^J=rc6o8 zIEdw2`Bz_W3-kdpDW&4M=53cQ;DcSOotfRwC!UuPkN`S$~{hr>vN12 zP=rZRwi~#cK6tH8J$cG9xS9I)?aR6}6ixf!K7Ll{AV5cZ(|hA@bW%t#BIJS(cgxsK z9lN;{MzVdw8JbC|mYmwE>D4l?DDmCpo=wRBl97r+sy@)e`VzD-ZNdrhimsDL)*;## zk87MzL%@xX+Q%zP+P2eVP9vVaGAw0|060RFCicsEK4y6ssPDM_)?wo$V9mivNl6Yo zX;G}(b>zwS7`s>2#Q=}}9uE*I9KEZko;w1d-r0tCf73@?-Qp@Mg3ywq|%0+f{5;en+3iVO5$> zm;4={4aZKHkiu-ZRl67!{ThJ-FQX7*ei%$Ys=r7ULc-dRNQ+lP@?-Z7Q|He=Nhq7W zUkFO*kF+h;JzhXVi?rY3&0sWPm0y}vMrXV#O66X&YvXbBV$yC!US(-zM!c$5Q|i;o zV#CVdijo0?y-P27$JEK>GYyx$Qqr~oTK383%!yqCu#C3FhR6TXW0#cYd54`i5w+lQ z)}q1Rx7d~$TbRG(UasaOsDou71fMifEHz2h@(9c3GfWN0Hh7W<`k;ongFI z4z)P+H9S0ALVmmTPwrAD_D2GJ?X^P&u}lmV;DJ?7^3pZ}i2&3bCqpv=y=WsyDL8<( zeQX;kFxD%4e!ORt#YzeqUa!2hpSpGHCYzFomk+Yu*NFFBOWVPdaQpCRPjnvyRsy74 zwpzVZ4xt2uh**(f@VXAvH{Q4ymS}lEWxjWxa2e-XPnxtdsOvia%EHQW@8svt+YAPS zj->c=I440VT>_@)x9GC^T5d7z50jqvMI8@zqfrYBKlF;Q*$ z7|%1O8zEMJs{cu;Q8_#;3k;Ik8M~`nwz?aRV62`c5nkNkSPk4apV#V;e9BT;R%pF> z_@%U*KPvmc7!36eh%ZdYuM5|fBrF53Zw(K1NPm_++caj^dTb(~W2)s6Nf+1ac5O*&a z_;6s5!e{BMUo%{s!zN*iY)MV^;UKlOU=b`H*d`&LX|is+vi*$SWe2<%2CTLbG!*41 zAblG`zA(Im(+<{LKbC7Dgk#rE>8IV!t+#N=qd=-2Ql*|6Csfr{E4t23EC z?}hG`LUh+z^v1}eMN{@g4PiiztBl()n4KPK5F(jH#{YHpXd++l?U08*}~{7hE0Z34}2jm0gy-{FMqK zj=F!$%i8$Ngo$_=auYJC6oHNEFqh z@SmSxGLhI(@tK5gY8LKm}Y=IO} zJj>RD>2V@Zek?p!FX+8k`!H{Pvz*eLs%pSw&Zy6*`6MgeG%dD^^3qa%6E~^&nz4^$ zqTniaQls()r&HlU)JPCCFPpQlm+e-G!Vre6(dJamgxAuQGkl3D2sO;1^k;VXBii&_ z4L5y&s!0GuZwB}7Jyy6Z0Pz-d3cNA^1}pN~_1B{eZ2>-qqm%}6;fAjq$`s7!hZ%^B zuR%#j+>KDrlJDZRA^1T^!{yh-%at}<>h`#}xF*9cp^6h;DLI2&({cHBj@`8L;02jU@7G&@6V~C{laM)s+9~tB|MP)UTCDu) zQ=8F_u+(HaS&}pL2rM??*YpGqp4oQI2vbw#T6yIZXM(9IAu1F@6&LlY0axH5IA9LE z!X__gj@gN{QWloO?+!c^qfnf%u(r;5b89V%;NsV=Qg^XXhuy&^JH+ViWl$@dWX>-* zC%$yF5lJ&!=^ZU8@hJDMOs|}m`qB=?84bbKTN-Y)ew_mVIWeDUq}pZo0({zd$gAHW z)^?z$ud{`p{ENwQ{f@cM&zn%hEezL%oi(idd*XtQ8v@Q;yl5o{JNbFhzW76Fxl6_G zAe(XiEv?SlLOy2yNJK25M|>$6t(aIU(x&pmORlGAiLm*Jl)J20fR zBQpP_e9)LV!#gke6Gy%_#Fv&!)^eksC#l8~Y z%M##OXYz)}>bP$KaYioaXbUhkV9vcZr;SJL@b`Bd0#16--`~Gw+qUyVxc=9oRuXcQ z0Bg|z79ok|C>PcO;P&1~(W2hGIfG#HufdG6xePSOW)^df77PrQ00b()ur(+LmxI0W z)5=oO5QT97Af1iF0yPCwpY)8<(C$K6-lcB(@5e!byH1`ub)mhodmlbZFm?J`CNWvm zyQjj!yx%(ksp`<22tDCV^lG?2xn|tUv|U_VT4!90p6EJJ^fyrLL(#8kR!6_1ySqEJ z?r!ul1rGvNw`kscH``}oV`Dp&?e9Qny*)^+<0uV9tuy_@200i_Ym+zef1{EnZ?yj> z6Z$fJ*pc>|n_LP#aaUkC;6>{&^7!>EJkOv6lq7MI7d^#mdw(s&*fquqv2>!iw zSdmcw*s)`oY^Qp{AX{bwme$r&pE2ryuH}yQUo15y3@g|zb#wBD6Ue@@$p#EaR;Th_ zWf$ew8>+b)VDVhzChe`Qtfc*u(&(1Px~o)p58p0&kW)~{o)$O}l1xLZE*5``7-6%$ zvSg)T>}w8?exTNZe#=@Fn`h$>4h~|zJ*=`Ue}Ts4fcFG=+%Yf3N=y>7654&4mP5JB z*UcBM_P~MRIbb*1GKELVaRiGSRD7Z<6_?e^~8Jr`c*XCUaR&TjF6 zmE$`Kz>eUk3SjS97zpKIwO|2A67(>PIS?rV6ZBCpO*ZYIICiJ_LeG!|(D$vhyvZuO%k9fn{e0E@-{j}u66tQgD7MkdUO9iu^!@nxz+;8aj{GgnrtfK)rA^)MvzV(d z24~+>E{%e7H9~B1fUX+t?c(Y3Ki@FXb!KjqgvI6EkCGt(R<; zVLqxof2X(^3x~9+Vx}>FO!P5?^hfW$>VG*%UFLw;2CGD~p(*Rt(N3fzwLEq9Y~xte zJl|zm*!bTIJp;=>Q7w|%o3v-^R%Sp&RzQ(W6T=wUkw;jnwOtw>kearAj6((!8=>EK z82p)WUq!~kihDCfljL;@dkD|6ulE5e_O4*^25sx{y>~^!fmeh{dNL@eizWjGF}>Wq zbLZ@H4eJe~50^2@&C;S&xTj8VO*foULpW7_T1!69 zeR*ZzqL*x_Ef=$=TV2=H?g?)TrGimtM{wBsA6X7&&B5%ZGL|75H_tpio{G4ODwS zu!U?PgE4$$n$_@Gm&xDZeBc23ltYp9^l5XyeYsv>9-9Hh-#w+aJbH?%?6@WzA$=f_uo4P<;LG)NIcKRLRQoR5o! zHno{N@YSh&*WCvW$QsUeba=Z^;?Y>w-438E`?(?P#m;PHJe`+I6mKSzw%zG@AN7WH zl-=x0UOs>BAOZ-5VG30VN%**o06Y5rt{DbS4QLU8NLRP(eG>`4MQ=AcIyjt*W!=eH z+{$F^f|UIr1NdGP%KIl^YNy1WY6lJjcs7-p%k=5f4@5$&cHq{oT3RRby|7~L3q^NK z(@*pS7>>!vP}+$Zc}D~4V_AjEcMqtjPz55TAgQMg$SkgN5WeQ`AKJdLoGg*DThu8V zOrwC%N5(H?kB%NL1Xy}P*i1=a2lv0F=aC$~?JYfD7Ov=Qaqx1k)yUbk3OoHT_j|8D zlvRBaj1tC7*qU_4sgV?Yok#NvO{VHbX8Sn-bdDsMJ<+Eb-$I3-IcHQY>A%EWfr2n8 zEltf2i1!h=9PDlVkgAr!O>gny#U5G5_kquzk`6yIGIFodC4@3TMrfj}p%;}Y>yMU{ zagtpmuZtxBO3O<_KGG_~A2vrvE3;J$g;b6Q%|f4x!YOir#UdQ15tZ|!KVsgtiySJ( z8Kf2EOr2wCf9|)Sell`nn17n0O&EoY+(r=B8M)b;Q}_6o>7p&6O&uxi93^2*!74I^ z6hW)<+^>Cl4ty)IJ?R?`{Jf~i;4j_eZGqnb^<0XTkdK>Ts#zJyDEP({Y+J^7lhSn?yiQ z1Stis7#5=suEi6vCNadVt6|72cN;+!>SWl44I2$x3}keRjVAlvlIB!yHJxv|<&^&o zd5wBIPN|2{(`%pZ9iQ?m$SW#h=l_qW_NMe$k3TXKBI^`)^3+hcdFYPr*1Sl;$@7*) z56oUXSpOt(u|Nb>XFt z_+!z8+sgnI;tmcm&%2&Ke&(~Jq+>)x_Fvi5iAhMEYj2Oii`JrPoHyd}-Fx@cP8;`R zE1Wq5K}ae!!vXa%@@4l~&%b28Ge}40;m7=kE|=iLG2)3AKL#bhqw740n>%xcx%pAT zuW(vk#&6N(7D#_T#a_b z#-hTPY4i#@AAyvqWPy}B#=*EceOeUH8h!gZ=MV9ZvZsQg&;%lt9kUbOds@>2ge|<% zsnoEw!uZyvsVE-|a<97UPGJrs>8x@OZZEp0g zi>X4f${Namw7AgtYTMHaaSu{N_RBq9>t0V?7P)e1gjeH zWQq9ok99gn1O9ng@1pJLZla@^OSe+h4G77w@Iiiu#v8E)?@Cr61mR_Mx&veZqjSA! zF9DL0UcOwgAGF(2hEU3x|C_f^26$#tD`#hCQx^hmR@a43kH@sg)d+J+jvE=@qC8LG zGt>R(lcPv63w#?4`tP`>_JPVULr0zh9feJN!f6h-mL(}7vat(gU0tMl8S0la;>^>k7O#xB-qi}**RRMRNyd~ z$cuKqav8*cSyM1e!EMOvor;DT?<5zW0RM5|7)%Q&?wy>RT!Lud>548@RuB~@OsRSK zPeyghg6_Ne9nUQX2>yM#W!YRd1c-;U%6&mZ)5~9_QxWzQ!%y#ufXe$?tQ;;K$}NvG zlajDwFpZvEG*I_Tu}CfCumdcc*eS27Bnis1x%(%uI*^}A%cAzW!$cKAGI_a*(X}dyBD^>FC@~X$3$P!TU-*@@ zt1ohg?d{vz$zn$3o0Z|r(=uDPJMqEfUy!_CU&C+xHz~~8u(VN1#h3$AR(VNT81YZ8 zZlg_22FdF-I{;yZSO`J5nV$evL>?t(x?$!(!Q$9;1_^yfdgZ?K5%}(OMWBP6?B2{O zl)=Y_0TvZn3SL$bXqjZQ^93Q~F*b%9@F%=qYk^eI_M}Z^xG%RHw5Kf_!B6!DI6lSp?NGB%As>r#Z6(Ff3`d#nOhmyUEv2A= zT%|tOHINN*GzArU`t+$!>+kJz8#xUhpZB$}cBP%lZIWi@1g{uT_D{8re+7UBFa8FG zOtfnrzgfH4#>Iy(OHN2!b56~s>nqV`sd;cm#TD?);-3pLpAQ#9ZB`2Au;ZSTV20rl zr%?!Q<5H>Xsf@V&U20cMq-Ztgp2VXKz+lfBb^dTcMatoRz7{n387nvk3$++dHgT-3 z%)QVp;aa3!dKg`TbR2<#q2XCO5~NXQ69hB$B_%UP6adWXqg zVty-IR!G;=M&D=MJM3_$`R`pHX}%1UVK+M6Ok)&SeAeh!K0P{C`L2b5>22O~PPS{g z#Ybk^KKoWH%m?fhojw3l>S)z8UNHU>v4aO)y^! zafP@H2YZ&8)vxzYmyalUSJ44QR+Q?7|j zL*xL)jpmZ`icxgwFm|3{I=3EKSpooN`_9<0?;{bXn5+BOLRAKw7R!|}5kbS#|AIqVrcoHgO+_mgaEXwPPc_gM1U!?7J@xh&w50j#n-Uij_Xslv?uNz zIB-B%pxO9$0cuBGeUk-geVbeN43FYXmFC@~!4M#apm*S^6GD>;DS21r$x6Lh{`Y(2 zC{l&|C=ReB0?{8McGCN@6#)W>FtR+wTFtwj3Q)NBCO}~@IE0O>%%<^C#eVV4r?Xx8 z4)Bt|l(uNg{0fiqj7m)j3q&iNwPWQprxV9Ul@{)JEb59;x1#UWOF9-f=PD`Y*K2#k z-;BVn=Kah!)hckDyzHVp{@b%h?LF3S*30^6`aEzr^J99|ShFi2&23Vz7zXQwsjVh% z{+ zW3eDVLn2ftEVv>Hj&<0uNuxjEnCk!i2t4jDLBQO5!44OR4S3RJ{_HCz3f$OkB|i<_ zCkVd)c3$}+Sz4UwIiiXzuzR;00z2NST}@M|Cd`yLUEhNbdPDdCDB zF*e@1{9f#%zvf}NR$cGHM`QeP2BJ$BH5+h*r zL}1`%(-1D+pdX*_Uv&)t#SNK>CTg1HHa(RBAJ5>_z+_~e^df}0dz7bEm=vGp) zI@3Nbe@3WbSxAS!gZ}xw*1u0cul@}lS`OT#;Z2m+FI@Aw-emNAOV)J+dO-~?hF$@q z(IT;8d?&>v=!=O?ml@lT%A$OB-tvZVPGY@y>{X~WdwDUSk|WIm@xPd&gsF?LG6BKp zUIcFS@iB`Cr`KU(TNd}orj@w>VnoEnE z14<^;SFCz^s1JTMyt`pi9t!;r%L7#R@a6v0N}s3x2M&5Whx5;RDjT`=-03FsPhpfz zfh2Lx&1wX|gHui}y4K?uo9(G@%ez5(3^|S9xR~&-@sH3F-KyOl> zegz4#4C}S`e^*2ALa>=sUvf@arxvh<1VwX5@aH7jE%UDUSs5)DiI#?yyG}Qg>(=KB zmD@|r#h;g5Slc9tLLSoPid`$E^gyn#WL$|YGMKYr!v?b#jw410*(h0_d3xHUgi&B5`=mnlBhtQ6%bn|s zrrYQEQQyWKgpU0O4Wi=MDE)@`LVTx=6Xz}Q>_cL@wZ^$gxu8(UkQA_A`x9@MDsonJ zHC+u24Vi;{wy6F9nVMD1ud123sdS4D3#%t1jpBnEkqU}46+1#v!rH0oJE9Of^kvJ5 zl(7+-lj$*dnC*$N+{?u`reQcPWiAg7EuQ7$-tdC!X&F64>?xR;42CnsBN8t=1`wF) zYfU@I=)L`%Ii-EBmjJWLfbIJA+8=hfm@itnV2@J1)zCceY1WlV&p=P6kE}qB#1ei7~uYQY{_;|RLI^|b&_w>vbIU*%s<_SU$Lf_#Ld5M+g zssmZb^dmke58bXZDF;+;b^3tWl6)pJCYxgc=LjwxrDHJ|%^I^TGH@mSEH6qe-*<*F z5H-i8OLoLsh5RQ(IAal~0&E(W0>4RL!UQksq`O$tiWY>v9Hi;JLJJ~a3JoJCDB|kv zp__U>oPN09CJjAC?6fwvF0wpU@G*8lo%`)7pnLAm&N)v8R^)Adqt@`<+5^y6lrwUI zIHN%+W|-x*p^bw6MEWK6uO3X>>iRD`4^Sk{A#C`D=FAd~T9J99AwIROIIvz~ zh6Q*%O<=qPE!o}<{Meu?!-FPxxx9TCZ2LPGqz3{X70VBW>X7Y^=_eS^MgA>Y=W`Cv zoG>R~o-rR!A?ZH*oC^w$MJ^pVgOl8j^3w+|p4iJ>OLflL&?j(S1CI2_MI-hjuPo({>Gr|*9# z3{~1$oBy+Wbld@n>%U*k{sx5o*X;GPjytsHJk>*kE~ol!WUyVSn+m{=7(GbLc5?dh z{`dtvA>yU20{T=f)>=cKhibnK1TO2x(|=9WitrtD;LpvEV-&Iws4kllQY^h8+A{q> z!Q$=G`)W;fT+QQU?b+ynv0wVsM7K=B0k*%Rj-BoAuqf3sWpvXC?9$YJ*HC(i7ucR~CH2eU42wnX^QiTLl-oNp_`YOgvuHo8lz|1kW?r#EdK0LB{C$U6Tj%7V2P0N4Bx3JKq0=d z#)zntOZaJc*7IlHocf#`0F-P*gdYL=_-xu}8=K{j*El|9ygEK%3V1(y-afO>>T@){ z&lM(!&QJy}GgDOvHPX9^-yc0~M(^)+M6|+O*1Z#Xhe3tSlB?)NQFbDZvlrd?BNk~2 zwumJL=t%y6sne&+3SxrU9k)cu9XRMCgo86G?62q)7LM!if@66BI%RMW?&KE}_-N^!|3 zEWUY#?x(($&X)+W;o3{u5RnQ~C88N9yHv^|IT>FiP0&FveNzQx_^QmSS4D_)?~3@O z)6C9pUo|p_FikETiO|Ah`l8P%hQny%5MLl7sad)uH09IQAuW}9_3t|7;l4HJ`u{zP zf4sPXZ+||s;?#p~7P=TiOC%(q^JF~PbdGL}E49D8&yID_O3-Jv=q3a1asfq<&)_&2 zq7YG2XMETI-&MqnNcK@78;nQr0SsyW3n|KAS~mVF{7_09%DG-GS%$NR%UTp_+ryq3 zbMkcR1c74Ev!Ir_qVlAwScekD+on)WOyji5WI#R{9*RshotUl-3Av4cZlj0+*Lp0_WX)>TmGXDpWE(1>hBn!)H{TV|FfR`*SE7D?y^-~QuBG7p1p;I z7W4DXE4;yO%Sx!*w@+Ifnhe=R2hdMPUCd*6 zo8gi~aoXm=K6s6uXq$oV2#nU@}KS9)oYRKfXQq;?f+M z*y6Kf8k5ZVY)^dqTejY zd_$Wg3AYV=&}+cO+?zL}8TrHnLxw!2pSsplU<61++P;h_wc+oM!z0N4cW!*HTU*vT z&}%&}pJ!plu~LYLDH+irtc;AHXhp~<5*t&~gWId$A9?wt8!8bUjK(J1Ji6fOzXP7n zS@A0j&@qkuA2s7Y07e7<@rsSVJ58d(5Qp7;`V{n8T#D#?>1e6YKtDx$r*t8WNg0#BnT9XZ=&Ww{f^nC;pu zq|YXH(B7T0gDsH&xE&#mOr+6-IF`wN8HQ6x32KRy02K1KlwjY!ee3enC()0}-a)!F z-EbNIhrX(=`JzWH3v*7CBkBhTbp?oUHPZ4vAc8n^Kf43|K}9K)W0ZkPN0hQqilT!R z&rb!9xHNzVL^cV5{qT)O-G-hQJ0uEd1<=v-lr4puTn2EH%!TAdF)MehqS6y|h$)Z$ zP$MZ%6~S%kmR-XoElD1NoZSNRyuYu7p`izIYmqtQ>7YAWK_|3p7gBtE?RtDq zaF2o}_U0R#ovv-!pAQ0s#IcaV`1M*IuBew3-%ra)jL>X*e6a?RTb%zD&9lJjX8To^4Bh`93BhUKj!9Hjq9Fur=-+?l zU^^7-h*(Cu;nSU^fN`Tzx>o2I>U9lax9`ug#pM&Au4oVc5rexTzCNHpno-fHWf-WW z$ePw<&o#$zjuY!KW1?*@lUDQ_RQmTkN8P12l!3wZ>leN@41Wb2F5?8FnYE=Wnii|3 z=1*vr@gGSTqJUsxt__fcbmOq&h$*OzX<12pL4gj*}V8@S&w^LXHp7D}VQ3k+cW1{X0Y_Y%n;Q^q-M|?Nvu! z{C#hGT|zc^<6T7XvAicjCEJe8^*VmDRx<$#ZcOF)_v86Z@m%+jhR>ZvIu$IPN(vl@ z*n|;Ua|yS~rBr@X$@fzIF_$;(cPY?GB&SIyKW?StFN1`MgPV6F!BH4y1Ca^!gXb4jNn<;oDTYUNOM?NlT zN#QaGcc>cTzC%!{d`^{+#g|kgxeSFCAg!V!6-2P$oyv1r zefLX8O3TyG_}mWuwiTxTrb0T4$7LUzUT;gW+lEReC>YNn`bo9^>m)p(EEd>^uyb#p zj9>sMgo;K+b!W%V#h)IxP?$UoD3p~q3^x;@_5pMDi+ls#N*NIl?!t5M z{^JVEBzf3Tuy~l#qa)ROW|B&!hnCvPdwW%xH3bqwL-uy)r-th{&>B`#(%NusM3-p} zGarrl^KAEe;ZnNzQ>$seZ@*I0r=H%SJz6D6>tOq!wEW4&um&yy77l&JaF9^??(;7V zq17F~xZGRn+_4Roj{ha90@NS#9DZ6hq>Hfp*Jxm1r1WyK!4*>t<=oKF(9Vm02ycIA zPD3UAI}c7R8Y4s>arE@3#uXZXf}X=|kKXx)3tJM2T_h3&Ht$>*p8 z7;g8@B6gNtZn;AxM3x#Y{8L>GH1uoSaMh5#R9;}wjVhjPao-fj40_Sb6ZF=C zWdKL3Z?f7zd-h>v&+k?=d;wz3qOe;8q3$?JRRvG*BuF9Ii>kQjzhKZlla(R^NmoLI zu$sx#x7&6B)CtQ_#f8S=9SH`g5;6sRjUu9varL&J-sFv>*dvhNU%29}6#$DITPl5t z#@gBzT?j!cE1}B8X@R$N@CqhQc!fHMCyiah-dh;9hE|kY&D*#9@PJZCb}}L!rQw~i z_e}m1EQv>4w#+&7nE}vVmsV*WE)G7VLcITbUgWsf-z0(g%~}R?-=~c^{P0n~9qLL= zTbutIrCm)y78TAFUr&Cbz@!1AQTj`PBO#nc+|Ig7#T-C{pLl8HzBk#=5s+9fCqRZn z>+xxL1%{xwat9(8v9*BA0I8#I&%`cT963ecB;02nra=e9ZV(t$G?@Y@1HGh~@1+s~ z!4di(a{luz>%N`>Y0{*qiD##=iO5lYup~OuOiI zVgxktXQ?hAt&*?s%>>?-ZN$ikFJe~9F0@5p8LknjxKI(V1`9R86QA;>`1*+ffR!|U zH?nw^5ke47;-aIUR6U8$w+#wnpdlvv7o8E1z2`N;5L=Pit+xZKOSJ)=+xB?qvmmG0*69uYJo-1x z|9f)ey8Vb-(VC|cJIk|wWrNkYPRWCgh^=Y27My>vVutw)`yvH6m9|cN@7UPH#2gue zMV+ZBX2GCI5W@6|&F+N>g*JI9c)aG8T8&GtFnevXV+|zOBq}Fjf_x zxajT4$v7Wu#ljn03%SL;qi^@d@dIZ1mo}Pj5t;5u(I@V&%!AGqR3>+E`N+&YPk2sl zK}b#l1H3?;v$#CzCuHuP^j9&P$2+iXDx)-@Z;7CGrJGYoi(Dk<`uaa<8S zq$!>=HeSnaBD>KqXj3B1a%YnYWbLyA`0jX9BlmIL!lNWzFJ&cjBNj90;ABn;sN{iL zB3a1^We6in(&Kv~F8PuB^Xr7LyLms$^V$eKX#dt@>Qotv6LtWtbNRZ9Uwqj#zZaKsfO?~6zvW6^+_(i8M7Mkp6IJboAng`WmzUB4jYWh{YC9S_%>Kiho7qlXSDGsjicLFqGzmvG<&Ln%>gOBvr7Ia^q1K;Vdn;rIUjj;)tZ)hH zVs@|+keAXa5ptImB_t`Fb%CFpAJL1EZAC=?{dHD|G}{po)3%5Mfw(L}5`5O}=|5IT z;f{+z#AMU_PPOzw5@c`T#ewDUyY3~xtBnrtd}ff^2dF>h&ZKAN!|2RZ=azy=U+@p=N9 z(Oh`CAW25M#CaVsmCEaWkAS96Ck`pyWm{QkVbY+^ZMWWOb5HK>uh)%s|2-rA;@IsQ z@xd{j;S%x>jOo~EyjTfH_4ktP5`7kby|n}PBIOj?BiWf##BInh4O|NS4RAVC#O+_w z1C&$IM*|CxAQi7D`t2f;r>F{=9H}-~8uuX)vL^mtr29rHN-Cjz3USp2)1(!b5yXb1u_fe32J>T9x!*I&HiCi|J^hUW!1cx}D!Z2_lJ)!(0 z<%?8Sf?Qvz(p0E~fzYWn8-HT#WpQbj#gh~UAyGTQE(N1Qz<7V3xc$%)sm6EPOh|hv z+bg$5{5-gR4{)|H<1%_+th`hqU=dpEs@PN1 zyzkA7VX}9Q_lf`Dzmx(26~uzf=CPP0vvj1ME3?LuO+}zM_~NJ7>FE3vVYlEu8S#nXxcApfEl*8^8|%uhZ5;y?gj zQAmdIOBgGYqtFeCO1_SgmO8=n!-F~3r_w@-EFB0a_xUyy+Tuqf`?l$6qRySOR*qRK z%iXE%rXc_cxC_XN^Ve?TjE-@ieqnf3qs5oPUNAu6S-=3dW@kHQzN1uz*M2(pePQiU zi+ocOPM^9~lZ&!&hcq$2w4prd#?usb=ZzYb;mn=v^1U{6UaHW3UAl~UKIi!rdq8Qb zfFs*`EOJ6aGdkjnSkH=%%Bb?&BcMD|9&o9qrPO1AB61O<7A?+WuzT%wQqq@-5};NJ zun?GU41l#LM$*zgu1}iPZM$?;RFRz+L+nvobi)5qKBkgf%u2z&J(xMDUZcQ#6qxBX^-m7?NJ()L9f}Y}4 zXkz#&c<@+ zl?-oxp@x?$)OP%d@TwasZvA%^k(pq5Ldjbun;@d1G7kFVK`TCRQql>6CwLw+WX|ir ztGG~wlo7JrgYNDAwmGmm!iDo(N^KCmYuTG0!Cd8H#?Z6KOUQ-LBnzF^ZGNDVUQYrm znjz>0N<*4yx9gFK+*i2l78EE48YBLeI3_&%KU(`veLMLnDd~Wqpqd3}F8~5)!e0_g z!&2d#86UL*TO$#6bi6Z5C=A6=fv9&ocD$-m$!dRcvfYk_?*~MD;rZxHoM&cO?=ank z=*8-{AMcqW2T|^QyLQ)lHB{>VpTQwF(LwRU#Dx?rQE&b!(k5oKuKzN$St{#4uS;WQZJf2G%fJ#yP6 zeE)vgHny!VA6N0x?R&WY-h-$^K0Gv(r#>b4wO-te^Q7b|`VUQ(k@L)BM18JvmfIBc9yTXoVBZ@a&$QK0&CkThFF`{V$6eC?$ zVQRJkHKCsj)%k0&6H`0@0Ik8@)Aq?*KtGa=XK=b(!rCmhA&3j9$h`&t!EBP#QAH(* zH!h6j^yjekITXnce^@E$9f!gbdWcreBP9$gJL(-=uS(A|>NPZTRZkB(JJ@;+0bIx^dv$)H;8D6E(P~ zZer=D2v)X546U?ehw;6sw>q-jZOYD2`0m*YzQ=$1?3nj;!eaJ5d3^ku^Q1K4*@7Rh z%Le#^^gR7E$bW#@0JHmxe~vRVf8}uG;U2G*B>{`qdflm0a;S~jtJ6Pc->duV(7PMe zl&qNE9Vos_1S({Kl}mXBidbG`=jpjmj+c-uG&{00$!W~b_sKg!HmE4%L@v~<7=--1 z3NH8qW}(x7d}S65`?YDKlj7pCP}N~|t1#Z@xudq!Z@_65M<+&4|Ak2-ONi;#3)wCJ zfhd0@YNy#pmp)(goe&$3vJXa2^NM)J5>9OT&B?2#cbfTCq*>O~j*0MfEo!9D&5LMn-bPmdS~XgMihEb+7P_XF_(t5sKgE zw0 zQR>!`eoB-F3LWCr&>UuF(YcBdun-CUsteZ9BxfpIVf*~)G@gQ@3&kJy?c@(nuog+v z0rX5*)JtYV+|MNCL$qf&kt!BCd*LcMaIx@~j z?_9NXaz`C852UAq%F)pgmA*idKHc*!zVjm9i{&5m3C_N1f6eaC4ZgrHt7$jHgIJR`UPQw5HP&4^2!=3>)3=*zFfA zZekK2oZ=;LW7zfAi*|r&{2FCoImw$R4o%R; z17=P%-=CB;k^yrV7Y{=Rg3lLC zB<$VvE!`oO$DbI*`Npr&Ig4-xS|pi{ocr;C{CE+;F>QaeQRf&|9t4wDopOIQ&rW zq34If=cuCh-*2f}-Xsf;3tDdbi?>mgK!ewOt{&ptp ziWAsVdQg<9yAK_5B<+V@pHg142VO4_RlzGGDoSDtRcIrc5=z{k%3V?qD;DQ21=?&# zE7VZB;a%`3)1ZgM2uwK_If%@fK!xmTlm>S`;U`;fT<+d6qg9lxV+Rc9ddkS2!X6_8 zPtcy_w|dG z0pL1g&i0+$M1jVs?`OmscLI(WAV};Lne-E{JTW$nUy_e{R7tHh&HYzqx0NsAvU;P@%o8rjg?p(gWZ9*hi?L%p zzpQ!jsalO1x9H$SJ7%>^6{n{|eOkfGeIriZ1T{dwxhT^D(j8+?J5biEflZBsLy)-% zV`D6_Gs#uZxT2GejBK|z^hK2AjMTQ{%U{2;gaEto`1?4&rBBYxzg|^NhvKEX8HV)R zkbb{^uVhi}Tk9gsbVKEYWc2l2oxLx6)UAeU6Zt;l5@!y#S(GrNli%Fjl1lXS2JQ4IcMdsl$?syO|&tI&4DT3R<+ia@lZArE8+K94PY9MW1|52}w%= z0Z3H5xrEN&>NDHJ5p%Y(}Z_cc%AC6Ko4=)Rb zw%7QX$uz|WA=2GshbK=}@Q0m~j-y}wQQGt8Ez>c;r?O-Lki?BhEh?wRFP=Vi$1{=5 zYHR#|e#_ti73=LF9#GB=M418*k$V)ZFyS%m`k9TIQg|G zp<-%R|MS?>{^Mk=CBpTe*B>pqXyg6jTfyQX%cFdESN~_=aL43x^rd8b=ctt z$4g%Sc>VCr$>WD#wO!EV{=o&Io}Dh#n$+HY(%Jo4Y6drLUg}rA>7u%4dtQ?neb@FG z{#5;~Uk|+=dZqJsTsJSB9q#wa*9 z7VxV6&y@#gbRYMij2;{bKv7pytnaZgh#?u`Ij8r3;;kh8WE--5Gm%$&x}BIpYHcIzmj|iv>W!81_qU>0bZAtU$V}gqhgg# zf?O9Ku4bjRc9QeVnQMXIVFk)4@@#C$%a^U}H|(eC@9pMh2=r_}V#Mtj@1@I@ zY3u3+lV~HA`*L)Yds|sm!o*7B)wCyu%d)kDg1*K)F;tGWcoh?3#W(!(`f$c5re?1{zqPjh z5Phf3v+$R1f(p(Lap>37peS2?albol%wr3C?d?*{$UVE+X{)t-D~~Z(9r_g==Imq- zKm5d}Y?qb)!;+_JJ`OHHt97ETdBoP;wlg^B*bHJ6t`*iytH}kZn8Sq3QD9W&z&vxk2K>g0i1zp&hHk=dha3(OV z1aB0sM4cQaxiU(}u8MRRK72XU*iMuxj!k+j>`LpYdURW^cCE7Es>>&z4M#TfxDGqv(n;qXqWo1w zkUfD8ap$OQTMU%vL|)lHV;x&x+1j}VbKzZmUWL2Ya)5!D+jaI+H8A94IJt=JYgPsY z`EB05T^B?+fGTra&&7Jgi4nECvVy?{Do0I2!=`ud%HO|#_iHhR=LuJb0JI2$+IHx` zC0etjcSgaZ>=dL5voKe05ZqU3$z9LBetzA=+isRAi|fnFn}%LCezRk#em!2H1XB_!0dxn-FlXjns5EEsCYmrPdnJY~7!)0LJ#(vBBp>_FtE^+NUv0A&*-<&?&Kg*|E_hYP^hCqQ zs#EH=cWrE>V%2N;wZH{seh(67+AP1;+`D;EU{c=zm;F9!bB3QCxhp^Yu66c)P(ixd zK8|y|z0b3@aZ=}5_3Zoc?5{u!J0=Gdbq%Ryz`EZ_F?Mlnl}gA=Fyb)AfGshsO`Yn4 z8BP*!*1!O3>*fyj_5lzCS=P608?mp6@qxQ-YMbLNHIb4!`X0jvCjQ?)*Z0rs^tQ3Z zJS~Xau(Y*nHD!Zsp8WbwRJvgQA14> zpTDf1VQ4jWl(*nW?hC_2tRY(Ur*9sdW~k)?XlNin@Sq`vp^vG^`q|m73%tyoVc`{{ zi2qRN*)KxZGYUg>$JU#yF9#LQr@?lC>Ac~QIwBLB%@D%d`vuC{cTW&4zJIyCxL%I` z$CMod{7QClPVsIb)2!dVJ(#J+p}DiXyn4C1HitaXs#E9OrwQCPopt0clGQ9cbrU92 zvyVFHT5aRN7?sPhU6>|m)~Z!CKD&uduFgq7aLa^LLwPl zPs6A8#+4_tJN&->ieJa8MQT?(sTKb%k4$k2xLYZt>-a0HJFL3&@k&jvz?6ULztU9E z9JA(9{8Ssa>1~S2+~NxCx;PdNj=eE;*0X?m`#sE~ACzc4bE#^(AkxRmdLqql2Jw6r zV8ft9QX-2>GdR8GH(Ge#1z90t%mOyHFGuBI*Lo8UU>l4Y{yK^FKbwbM)@ssZEnLDj zi}|(a45OcMdjO{Ufr=xx4a(1VUbUf?_WHzkWo73PWnu2Rk{h-&I{Hk!t&@`ukW2@B zoCk9sKYnaKBe|HFJQY>dXvqYuhB6=>m-T+LS#=LS4!ig69dnPtWS-WQr%#`zE=f|O zE}_GXDS!H{p7vY!*;y@L&h7R)NbIpQN(SBQ+O=yw9b)ukEMctX%<0J6QbXcr-}}D4 zT4ka};2y(wWk3VhtT`^$aY3H8eflV)_k;ITwYM}e(Lz$ZVqlDXY)ewbx^3Im;KFpq zv)+XI@gourK)S7A=018Z(H`f*wjSNPcI~w*>w{lSD=p5Na^`^#-~~f-?c*F-L)VY6 z0Sr@l`O<=pgtDyk7HXAoJ6pwM)Tp+Ny89T92h^>E!m(#ZonS_6tZAwCz8D*q<00iA zx$gHi3) z?{`*ad+pjaDGE_i%=-R$R>EavrbCmN0+N_0oR8~LJp+S-L*OTD+}(|M!oB1%x~>&^ zIgp$11i&|(=`>a^jlUE))pUzGg~^>`-U%^q#a>LCH*enj!{_p>^m;lv)klsTnVp+E z#seAm(%*-rdiCCjfTW16Zl$Fkdb#;vo3y7d^Ts$Oj)-n9&s28aWmi*TP}Pu-5b+#x zzYD>>^VZ8`L@Gv`lzs190Flv00OVMsMvb$(2ZS~sNccrLm`rb(os+Yw2vd5AF0WqA z(`afx!Ig&x)&j74R9ouH^!VWk`6VT7txOG~rzpyumv3?oM0!>~-d zUcD7GG3q#odZZq!?@)^?c-M@JK;OX7{#lQ^Jg^hD0S=9`Uqv^2^ypF0 z+O-#XQjgBOsk&)eutPt6gQDEeJrIQa=T&|;{p8I}f6hiwkZK+3m-$vh?`#{kazv=U zcVh#CJFQ&9>%TfzN%w2ih|q>dz5QHor$xtRDft(Dn4meguZ2;^Cksb+RPCOi^=!t< z$1^<{CutK4^&wg_3Xuef!Dm|Dx& zPgABG&hX$=yvlQdAhc`MS_*((0Mi(tAGG!}*DfFEDZI-}J&@UXSJ-r-WYKMCokopT zQ{>a|>hf3zj8CdxmgSM!b^iP2QgC4KKxYztv1dQ&2eh>VKs^VQ@GY$fxY%0yeK_Ur z9D3O`C^+~$DZgzhVnDg=tv3xkz0j{3xp>24#UDc{p?T@Fds<>#8 zD=D&&%|6c0UqT{Z6;cCpC}))Bqf$NUzA7ncF}PJQ#0izrc*i+DJ{K6Noxd=!aONSW zwg}onqzZjVu?eo%Xx$rK>vF3}lbTVPuY}7=m<+ryD$TRuggoo)7Z&AzUCp>$<6y?HssW`>z9d@B} zlqMEYc&!KwRG~I(-lE0mBJv4K=Q0o;Yw75;cInM0kXQ7NL5U+KQGW|m0BFqCmUd2$ z9z0mXoIpO8b5yc(CGPa@Y6GpU&HBYSxDcNjbaYey)e4{SG+X_Zl~2x~S!&g*pP?tx zYS7@$u7+?2%i#}4z0P$teGC#b>e$?=YaeOadwLr4rS4Pl-Oz8@Qd)&td!kRJ=j7BK zI&`SJhu^}XGr7)5g98UFTE;<7O?7IkVwL|?+6e0pz0p&v)~s3c&gsQH2k*HU+J_y& zk3q7}3XbvgX>*q|k79B>W--;OcBWvOtE*|luy*6BQ!G{s)xKME`_Prr?Zn3qbo})G z{TW*7{QP`38oQ^{srfV?uUAm)9p6X=__n?YN}%c5VNV z)^WS!bpCmjb=`7fQ%Z`|%D-P7UmAr1OD}Il7YZ6`SD6eYTm~_wDIyTI0yD zq<#Z8)r`5?H?Ck^aJ9LP6W1J8Z8kQ??$Q^(mO4R9Ld{}@hJD-Ct*Zm96?{@Es>iqc zP-xFv9ppGN&T-V?8M*|3|8zK=Ov10aEreR`y7-5My6Uo=*_#F~N6S2kB(^PZO;FH2 z$B`qOb?BfCm0R+0)tWWu*skC=xAGAjit`b?M&BDV^JJC9i+}1J*ECR~pXe^y^AhPO zfP8BiMxUBUEqUqMwX{QYp{wB&2ZjfEqJ!1w@i|}FzN85#azz&`$3l64i%U~l`(Ob; z9zJ}iOGqfrTGTadpq9$3M<*N6Y(#_D-0Cp>UZ+(K+IPeFcNFbsJsjM9eSJ@`%5$4~ zcm3&G>;j$@>a}M+E;_3PtR4&Cm!0|x2S=9+&?eNp{=}?DxTts zNBI$Ddu`LoV?Tej}(|l z#8e{>_qG&s9-hPGL#3sq_9I7D#aZEHZ~Gm@pCHLo$DuLKbaI{6NvC*K)4APrvprH( zrpirm-$}ib0UVzU&DyC;&&cKn)6(i8m}*5~v=ZEYK=|A9$AMRK-mSZF!)`FUGok?$ zY?5*(k#|RYTYx+70Th;9yLTV7aZOOKL2A3w~%B9;5r+S7&op>s9YA2n2 z9K*$CakvaluUu5}>XjCW8{A;t{P|@Ly^97=OvE+Q($-MZJhbq1i@y)W zj{*OENA{-AOaIZedV^mEy5XCG}01i1o0 znMuZ&{kg>lxXzvrj;z1l>em~Ku7Bm~RSk5Y0y#td1kh||y=RGyGv8D0JVk`dlqoIk z9VlaMGBPqgU-Lb+jAO`Pc|CCZ=x1&z4OzUU0ZSl#?wTe&^#(pm(X2{9UxKQxK*Zio zLrlK6$suWJw;g8EC^L|?Lh7&P&8?3uX>T`rbVH)4O=mKW z@#uppv(1xM8JJe!aGY=(?ieIDR#4r7$vRpdX&m>lnBm*|2s!!H3Lp>KKOS%mHm_M0{gy+_%jQT0xD#a zS;Z%$Mb}tpGsoBWB9uS?TG7UudKSsqW5$e;r@?r_sS|m+ zRo$mf&7fklIh~{Ae0f6)FCU*U0R0kRjXQ|e$%)q993_rSB>`U{hNhI!33Irs@(_&3 zB$KX0lhrHA?t{OHLA2foX`7E9JJvg<=+vnx|OuwoY{eqEX>}W5fVK- z)?SWvWqCk2h8fOUYrUU7d$t^X>Dlw=ZKzc&XfJ3m*Kgby2nbp8TkAU2k@z-k*REzj zK!8R4q$VbRir!B8j;Y}lMei3yq4xXOl&>DFrD9d2_51hotAk06${!z0>t!DrReH_Z z`uf!5(CTf@j@5s${Kujd`f8UFa|@i7*VH^jv0~Tr*^8a$R3>PwI391VrDosL_x;lW zQg%e&qc=MMCUNRV-2F9un@RF#w95WrR+cuaLAqI3TK-k6yk)FCAB2`kch5QVd3YFv zzHXJjDkyeHx9v3%OK=^#IGd(gzFdOr9#1X`&q61g2MxoVvhz zI-3;F!ZO;VlJ6`^OBvE)KxQW>%gsnu;F41eT?`JkSp1`OhfVE0?Z%|3g8|nvluI2Q z5FZm$g^~gEYkh3H(96mARMN2bjdryhM4*^=FL@jUqNfu1oquEfBzM^Ez0;=>4Y>kubgt%IKHZ?>O}MqUS3}7 zj2c=d5S^ucYvgD1cKNlr;w@cO5+w$kTs-_Fvr0=|Xa0}@emVy%m6R905DcXINW+H-!yrHqV9 zJh@?ML-&T7Xh_ko;rj98M=8NEwp@a@{Gf4KDvd4u>X4p*jhF6QJD~>pu|P^5`?g)X zHUN0uY^b$fqeH~iu5%056(Yoc_3D+N{OsQDFZIfD`b2`{EEYEHh>^f4 z^W{5Z?)GWzy(K+9qZvcf(sZO@QPd`3vPy6 z8XK!4Ygpox7^Yv>vIZGp?ASGo#JNtMoZ)Had#5LV&oB@KSV8Bi<&MK^Tr`DD8Zx}C z;s176T$k#5r43Qsq>BIO+^DYe;ZA6-K4+}x;s_0ItDqWHeUrVH>gy+@)!m$#8hgvv zVyAM}>L+``CdKb)X7C_qk-yh$gM%Z+j9JZ+rv@Sj+X_{~F!{rhRn1o^^`f2wZD6OL zcFw~G50Y77TeFZZSMkcuBxrg8u04J6;_y)meSIZnpqH}7pnc0`2GOUY_w7?>xWS4B zAtH0KX+ucqWt4ztFJF$!yx$hygwEL}6%2eLe{+4)=E4CS#k2g2w$nM9JH9_{nSA*0 zW|O*ik0H?H(Z7B(EhXjb^~FEtM0V+;v;oMBdSzk5{NJ+bHO;F)^n1+t@uf`>4qbAbTDJjwK;A~6rF821(c~weg z`T9y7K_xP8dUM{My{48*B`)Y$p1gfw40hHMAB{TImxI|KhwNA%7Pf-(fO(?Nosqk; zoT=U$0MxQw#IIG;0nd7m?zn$skwlG7&J>!?$4>A1FWla5aaKgy9(L)T1 z=C}XiVX~=y%X^(bm-C#{&%bwbEb7+8BsjLqXZJecuipgOO&PvJcjvaCdtqC)tRk|m z_5kXe$A*!e4sp!~+ALyWYYk!2q3u@}k#Z&CR=uDI-<-mt$s9JG1{8ElM=5yMRo zWencYWBWH1$tL`DmoD|?T#%k47CE-Aixp)cg7MyoBYdYGXWG8jkhD05xVOuid|z_0 zR_sM2yam3ezv*6aONmK?7lc0^5dM$nIDm7}v#j=z-Baf4TU1gaGB0HSWY7ZmX)wIV zxdGD21if|7yDE<|pl!nBMcy0kP}_V*GbNiZV;M{FAT*7QcC=2jB4%{=QD>+~(T2hx zbYMVZ)duaX6lplboyX#bDG~hyd(q)1gz5D=00$!zjKChFJOsV(9n}jy0=7Eq-o1M- zeH!cPvbk(I5UYxc%G<9~Bp~Qb~U5y56@fj;A_Z^MiicN$o2>32Ce?@yI&{CPh=9mUzJVnUawTQTeJ5% z>VkRrHwW`*-^ol4ezFH!W%p z25L8IR1oKJP-1+1btYlc+a#SZa4{`P0D8pBsGH~gG!00tFuz|C5o&}pSy}dH0xJRp z8FMxGC3T3P%!1G&fju-%+sRoPwIbaQ6+2Q#33dBmz(X&K)ad}Xb@{n}Y`=bi)9U5~eL9224j z(uzB}7K8rf7Zv>j#xN*x_ihyqGu_IVs4REu>D!04w67)CD*7TdFg#*4z}IaIk{vQe zIYsXH6|0S0KB+Ag{51+Ox8shkvkdSXYiHluKv*4NRFEdyOO3(2L<2@4gzOs)o>x^? zj&9WlB%8!kG1h+tu5YZfXF?lsT7oabimZS z*7iANtUhvc5vPxn9%ly|)`p~!RL0Gr-rO|Mx?jI$XaRow_;GUMI^ds#0Yz$D_v(4R ze}iAKJvUU#3(^10zRQp3l%Bfv3I)m9qI%=$8m?c@93N12cAU%JeEl(Lg;CdnRzGuk zkeKWHXv-y4^*O^QZ9T1g;?*WOMtW2L2j-W$A#M&T5?k_Pv;I3!A9A#zY10b8|V@CE}qEJmSp$DFDE6+)KI3W}e~P<=Er=)gJ~H&{fI=6M{yy4G?3I z*uX+SCuf$|tuErE6*y5y8Ps}HQI$P%RdkM!Uy^C^>eQ{f3M^V3P1ZS%Io0d-sqXIS zOs1~iuwf};+M3_|=FO8>fPej>Cr2usY~W^<{_Svr%M*1MYwMGJn=1R8{#*Nf&HYJ# z%RLN=rcooj{h<6i9>`HY9XGwR%l+-$4!x~D)WEOi$LbBuLcEq8*RA>_wR79P9ouc1 zHmlgB#^<57i-OhD-PHVR<>;bP6;Kq@4x|V z8Gy)f&E+UkQdAU%43&CFIf{YLkWf-TLC;wF0fy$GKy5cG7t%!oT6TEj*7T`DQej-i&Wu zq@pmM-oAbNg>M$*LFC{42Q}IgTAMJRUcHnaEi4^UQ7LGTrq9r0OK+Qdd6kt23sOE3 zESe9zjf%Ot=c5x1`|iqeNI87?BBBLl0nV0c-(A8j%BMAt-b!F&eJ-eTQvyLzyOZPB z3S-v2`+bisbVzs!SA!Omb8vDV9}QDzNedLi#}5q!$`CwDTf0(l-_oVEg>uvI#60iL z(K#&{c(wLk-aCL(rRdOIv-Kzw?d+BihT(p8?uv^fRuO+fD6-WXng-^oX%#sQ`D~Oh zLZDCOYsc5W_9~dSQy%f(uk5v#0z1(& z>4Qaywqu4OyAn0@^E1wAITQzgSI#Fdkq5AGR9UDCeyr( zI^?d6%0q={Hoh?&3XKL;;I8E4cOW&0z`|hj}SjNtn zGG*m2fNZOC_8wXa)ROS54SC#jKYEkahzTatTxQqvb_cXU*>`fjhJfBWUgH}TEp6Ja z@ih6Ke_mfTx9F>Fn@?8x^D1n0c~iIMVVy;d`gP~sfRLcl=jV*lo(9|wy47OHk)+N8 zb(hEXtJ1agdac4htC|C$Q?llKX2x1CRs=$Ww}Az+DOf2Zs@iBAH|6yap9?uehfjz~Mb6j32Luh;$ixaI?*AHVqsc zZc+(_!;0gqcf^2AJ^IdkofpWk0pfD8Qa;lt~SZghp} zyQdofqLA3x^y;<5`tm&=78P>Q|M9>r;w+P}G1U;{?VoTugjv7QIkLI>+=UAt{q+s} z3^}4uYo~KM%!fN9Elk3GVBKX`g<5np7&JWz4|_V3t1hNn(E2gF#Wn@eVw^Z}!mnuy zE|d16Vn|VW+P6lWFSu{st=mMTq7Cpkvc6n66%0%P)2&}$X<5b%mpq;o(M%c`EX`H2 zi|D$)h4%VRUE3MS_xb%Q>tagr8rO%A^5=C|2ZAbkceSZGiITK?xNh95`@D+zEa%>h zqB?G!6*qSE7|q0%KFy2TyDIfvaF>2i`PHUvMj^57L1X1%up2(SMf@)=r&dh#kDbSk z>0sKEsRmtie1?U_r<~x8WdEW zR$ovKrkm zo(RaDobI_yfI?$r%z*ms)TvX{Z(rki8v8%)>P}9dv1YU7a~1Oe|5HC6d+}1>-lXuq zL%D&{k}PBAnd6oJiBMGeJ@oGkSiY&xxSi`VcZZnfS6jMd&DIgyj_0XwU%&F)ROcH; zlSMd;NavF4!X}gmG#&5oS4wy9WAReKc@$nC|=DoGn`-_ryAc4!Fsp z588}-7-#a>zoXGpYP~7`@Wf#~!>D$o+Pg6t)%X99ZmrhG8-{_$^>Z)mM@R{!I9SS(?HO1fn zq6DjBTE;Y3^2J6yh$#*mcCRmhLgx=LvSh^m91SyS$jGtBtx59zd(1rRS@)7gYz}ag zCTFY4urMrcVcx%*^#+8pWN!2WK#CJd`e0t2D)L%}5pe^6nu5Obyj zz~5|SMo$O8s3%|x0lk`=45TP;yb-w$4O}ZUJf8u?sj8QKcXCFSa5>t7+7Y^=YWzm7 zV&SsyviHjG-_QRXhnoGrHoO)_%IV>dzS{~Cb~FqeuRo;#v)BNLHSjnqZUDEE1DpoW zZzvS8gW!v_XhZ1|Hhp~H3)BG_poaw@Cy3DjYPIaxppXy~$ee&U8FOHC02A5OKZB;5 zdU_ZuG*oJE&TvUcFv4>gcsHw&4ZlLBnS`iUj!sTqa$(5L{L2b71;pjti%*AxA88;E zY3tlz4Y=2!jSWF+yXk3=d<2l9j~OIDXd{I>X4CC4Zt2uhx(6x`DKnbvtZ0N|j0EYw zKmCJWG^78YM9J6K!Sg>o>@v=~{p?FNaEJxfc#vItBXx`1`!LVdXU}1NkYT0xKu1_t z%}(R9q6jTK5uY!YQ|A*iA6qnTCQ=LCeF_dpWDE<>zg9V#AOizCM4}*%(KHl;^$?%* zim8DF7GjWF+>@V|_wF|DDnNCdv}FID3}N6Qg!-Kw${vW1HiC+z*a{{U!~hBZoe417PxC?G&%DQtE3(0eMvzYSgX}N z>+)|UFvkep3pAXQpclYpP0KI&aDoI@d4tbw-GC7319owc;GS4RkVekM#U*!|J?|_G z9uWFu@6R6)l|npO(DEvbXx_g47*4^fS-raa@yK)o=3lrjFjR?%h#-#_Or(L}E*IBF z4qkPr0pKSD5fg_@=ncYm;itq(gD7WnXTgCEPz=Ne*qaCC4IfNJRL-VT&Dntp0k*`pPx@qL)dOnj!a_V7Sn2c5EC z`CHDp$1y5Va1VW1vu7z#N?tn|YFXoW^V?^qjPq}AJ7uO)S%R}%JywJxxAuppMGDco z-ZNt!59yrdbT0@Fkhk*&^*jsBpoC@zY^r9!{S70FJ8FI+oFY$-e-1^5ZDM)P>v%L^TCFIqPr0< zs)ig^cR){tP63Hn4Dux!-3Ff;v?&#*LPB>>z!3||Insw@WMn`evbTqvQ!vzm8LEAL zM@&EOA^-Gro?I?Oi0aq>TLXT z?rG}hmCl_VA|lr9M!^pcpfr%ONJ@-nKVo_DjpVX2%D*AnUdR4g`;lP65e@QgT1~@# zQxSPuk|mnIVi5A6r1;7IKemgintFprC*}1&&)Z8=RW!wfqB4HgJV_Q<`8K z@()>$XESmssi~XbE&F2AF941|RUBAZK*oIq@m1OLe_JJ6e;BZXBnxLNV#xvj-L-;Y zOOXi}C_~NH44Wg6F%V+9n4=`6VU2)gV|1|851Fw8092thDba%-1;pDhAo@UanH8s{ zg+9s5OhrRO0~Wx0h)4J;5*5h&8VhA)GMFT~1o-)pO=w6>YbnBrIEjbAq);bv@C=V| zfr9QZDg{%-3201G%-|kCQasj4VjI#^2Hc<#UdlU#r?)770F-v&$6}DRJ7lrkO z?}y$AlRMu0T~7bsJ8pwUaMSwse-4)vzlILEptNLxfw+GE_K*#Z6#NYV;@IG_nvO?- zD=LaMB9mDu+`~3MR9S+)ekgoH0uIu;1PN&a)&&x>wEJ_)3O!+9VloCRf)N5P5~y#Pzk^!972nO@AB1lU@)xikmFLDsU`~L%uD|<8yVL;M= z=6~aX%3L&@b#SyG5eQX&m|erm9|)5|A%?+^H8oc{8*TvW=J*GWKiYNn-Shuy*A~bV z;yQO5c|zd9rG4BieAC>_Ob&vP@WmlxRgNtOP;6nW{cB@l-r*GAJ(QqeF1=HdLJ@4! zIRY{m_>hkGHp6@lIIOEa)|^cJP+f!Np|F%71S3h9n@z4dbLLq=?*lkWr1ybMMZNAD zMdFt)Bi|M%AS-KY^L<+{|K#3c=*p49u2LDy!Vu1^ha2)!EWp?30pjzNy|)JZ=#fzK zK*#OoQFR8jy}OI_KuRXU?TMA0s^nJeRu`RmR5&^Rxy10D*&A%w|NF6fG_8l|fBzQI zj&9OlIz{DH*O|f%mF%;}r${6@dV*qpDyNmS_@@R%T8gaHd8g!*^}lQGdn*Aht_XIX zmpu9?!4Zw*%aCM%5W@x;;*Ey%03r{*`H2vI;k^PCCiwB}-_QNa z3jQ!5fz}AOisNMPhW`xAD3C`1-ctD9Ysu+A)zpWF22K$L#L=r?_81P)EOL)(S6DL% zZt{efTQHAXystKg1Ib|&+sR!@*>psQD9(TMt1k? z8I`Ml-alwMDIp~ZChaGiEhkQ#Fajb0Bmx#GcW}TjV}OxoxHSmI`*!NYcOgLu%Y_aH z*0O($X1qiJGUm@OqMS_Hckc!Ry7V0^(6t_fkO5+=_&W906h2)v*s9=ULaJS1;pARx z$pb9|zBS}PP*(mbTwQ$!s+b{AfIA9Uke71~ws_S;unU@7-j5*uAc45kKz~qZAtlWp zjDXST86F;RiRac=fi(^+qKM7P$cP=D`zY4RA=T@df@O}zy~4SehKF4lY|dNmP3Sb+ z=MtovA#bS6w=LhR|6jX=@c(0%n6rWCtg#3&kII|(QCAg?zhYZ>e2&ID(}YSHLh)NgK>=EGM3t7IR;{*s=)KHWL!TNOd5h4r0h zZd}93fh7if(of)Lt;-!q;tv1?IY`2S9C7?W%16Halys;Oje+0=-}4!mBY?(5(fJ#U zA&74UuA*kxkWB@J9}-g1GeJITO64o?P% zQ1ZF`dUJph+XjdLGO~jTWorOuW?=zG9aGAixHx)fldvXRay@bIyiCISPWnnbiXD8; z$Yk2IqCNQQ2biWI0=f{S6cK+eP`sXi!2t%N#|nW*L;;KsMBWC+XZpdU1GH%*QNkoj zr82d4lnu_F?4|zh?zPgQ1LKpO4U9Ixy{U=lOIf#hKY?8snx~j^Is^!(Jcf&DxWbbT zG;`2C!3PJQ_=5TO)9?;}i-z$RJO~FSPD>xQzcXj#J5Is>+-Vo$LH_dJ55{*&o`dyN zMCmGT(%@mJw6uT9rQ|uJ%3^W_a!3ygJ|9z53vQRB(<~=Jfi>csy@-)!s4!BQavjr@ zL3=9h}!d9XCdpCp*orK!7 zc+`s$nMi;)`;TeH*%@AIf3W91fYij(R!?Ri0SwJ7Ekz-bbI=jReRoB`1ev`H-QWZ? zhDeA2f`ZgMJUryjwG8RO&kETKogoNCWn^YP3JD2;I^q-rVMrjx_U>Jc7OY%V1n$)1 zOl51vNTr@W=y(ot^PF=GkQPQp^)iHgXYlh@iGKO;f%IQLFPWhscBU!`qY4n6-tjqBqDQW&MN(_$ycoyhxO@7F>e||`}`ct4Edjj55hNr#jhk@HdxP9!w`6F@5)ZZvV%rH>Rd;5WBmlF<-D$u?WNjLoud|lWF*n+!QsUl z7Eyhc_9m)!Hnu&J!=(>SoV@g|us75nWf8$R1;_=9z4*ek%s=btP;BIM~b=8VSAZ*aed z-sp2sK1@ z;L62m5!cwBqN(9OatP^Y zV4RKoCQ#2jRxyhwR*SI%Tq_A!{$QS#21}ctz`>>h)|@+Vr5|Kfz;R99{gmI)kyhi> z>Q8TSx%u&3ncxFYO;{o{BB)tig`N~9s)iL)=q4n0irdZGAkVM8v_KT~mJf2%A%1qcWyfwbO+%sN8K{9adFp0}C}eC9Y#*sJayo9f-biJRwx=6i7^9(x9!^bc(^iHVH+ zBiP8M4nI-q%}o}K>fX`Gt;1lwft(e37g{k&t!9u6Y67o`%#sHZ1>pndGD2$qBg+f=h&(@bPbV?T0pb%tbz zt4~+S4?q1^@5;jOp%0F)VHCX@{cSpPUC*x{O;!<@^)B~z=A(e#r8WrKID(G1`RGpDLLCwMpdGT z8Nz~u=GyMgKW^9^dr5FOv>)v4a^73RX1TQQ&Ly+R?ceCC+7TNwA=wSqv)W&2FI+kj z?kCEcQxzp0TbeGtv+ug*+|Uu7>)&G8RH|V=xm(2yCt!BpDwSJp09zy11KpeH7zaC1 z_R0IlE_R&akewpZji*}NbY=M&+G4|XVZ6}s+?^3geD9EL%}afql80l|jJ(-~g4o|j zT$(aGlV9ZtP%zTN-29rJ%EgbL7nhHB?iV+=jOs)lt&R`-@g(}wL470uVgop^<{rPHx`qjyuAG4nb>NTm&4}bqeA-~h7`waGlKHVSa z28#IUu54BHyH1tQbr;!QZ?4{Ny)#^mH7(>5`oSQ(ry~WnfD8I%1P5uKjK=l|Tbe|n#>ZK=cRHGlCz}9UlV?9d^PeMp5NM;|4fvmrb>=27o2Rsv<@BG9X?7?d zacQ{c5ZQBqlsTV$tP5wp&|>4?H?Y%~0NRRfs=u`7(Gsv?2GsPQewNJ$X$qwXS{hR= zkdPq@1RdUPJWNC%fF^Nw0}MTkshKjbTg!2@%XAVzOaJV)_%#JqPg@7?sbx86Ux>ikTkWgS35aWw)Q9RjaIcKFjR!{&_CWrZG zc5EcXk&weRP`;n1kehSP3Zw$?i*HYa3$EbzmM1~UK#puj78o3Ydz3|XyOw4JRs9a-d+Aq%W61 zarSJa-Dc*;uJO3)ea3u?WqD`k=9zNsJ9oyRtEpS|+md9K^JQoM#Y2PzJ24icSl>+i#n2}oOo+l%=LV6r)AUgvmvXa z-eF%ma+mwc^fe7eACuF`H1BR0atLOX)mIIb-95ur6<%=buyo?&b!~UE@$Th>TAUl} zt;)X8R>_O2aYvhf+{Sd7Cuy!-HJ-`HD~O36Uc4LpzCL=7hINV<%V9sFTP=BmRM-vw zlQ{}M&D5h^fE%{Sv8zosG_2RI)|l%OZQ~ZE-k-nYvl*!H!ga;H*Cuk?oWN31z0FRc z#aot8DOF60*2SIGs{R(DiQOzM3d(%By)DUe^X=Wi#YN1MfOwjfnZj!iXH@?AZdY;b zwc4SJ9rTkl=c%_c3F!tFHL>m*7{5i4c?=ISL?bm=DFxzBq)P~7&r=BZ3{y|hmIO+DA9a>30A8Od2 z#`yiiHDA3pmgrpVn>RzAamm&0(MUZ}m-ibR8jT>1M zYg?cJcGLZMYG;IC^>_Ij-$fTj;@56$b+e%NC)K_ul(T`G0=H{zeQ~U~TCJ(%VN&rP zLzS1S_}hfu*v(k}*%8FnG>I3i{2BdKIahQKM)`y-LXJ^S#^OM=EWLz22F)|pIq5Ne zmD~@%raJPiE9M5b@ZDUQU@@#z?~?nAt}Z3GxD)EXy}V|;^z-uWds)KHsj98YeB~Q_ zb1M%~5S3trZ0~hG9jEu*%b5Q;*1)sjzG5x@pJFy;Mx;-VxRgr#PS>!f1?QaepSbOZ zwWyv{T{Uwboq^Db;3&c1vnexMu0hn`IW{1F7@UxTlhY1WW`uD*9B8~U_-&J(ep%L@ zX*i$29U^CGnE|?d^=UW?p?-xaqv8g#=b??9>#w2dt0+Wi!Q+Ym*_3bGP*8c|@$?qB zGhk;4Mvlj9RTFfUyFii#*^UBs%+__`d7iX`3NJ{KE2&ZKTB2WdWhF4&hS?Fdn zObw7=8KR7UP+MNu29g#+)x3=WHvpXrG$}7mQOzVxu%IAQ^DmlGL{5&5Lf{`b{8FX2 z{f)vI1pmN%*@;9c)#HI8jDkQLaNAphNmn|6zaU6~L`fjpD|A|-ntF6_kO9dc0d)Yz zNnRV?p00%+c`h0n9R}qN@Q0Iv7XY0dmXib^K2{tK_XXC?Wv#K0zJdAkp(4B1f`XG2 z4AX6tjq9^=j_$?HxIHy-KPfh>&#fqki?38JCln4L8B}$3YOuj*-n(a+Nq^$&7n?X~ zs^Dg9wy$rS$Lv`Rkf=_uvMzmX$*hH|^5)$>64XGkAU*t{!q6kHED72p6zb9?{)}up zBGkE;PE@Q5UDE`9_L|ainvj>!AAf(Hk99($XqguxPM%jX+IDpRwZRSLLJ)4xSyQ|J zNhFiTm^YSbWNI<~hB=>s&*|pq89i%iY8Afw;wK<68~`2K$1FX^rTZ7u6Z!nvQAdw@ zeUEoX#R=xtLDyF1B1~>iX%63*oKQ<+uk)_HzRqNtsi z{pfufdirt!Gs3Kf;MmHzH_HE)r68ZK$O#S(fgkf~U)ysYB_yP%VysjjJ|yN=+dY() z7QLmGwQ%wX({l9Kvr$K<{+f4U$#^-sC*4^$U+9wOU-y{an}6ah_U++#x62W8B5HfL z)30eMutH++H&dmB${KtMs;R7~n-6oT^{U+&m+gHScqLa0%-yJYBi_(IVIp%PLjBYH zTWD*~B>ASRjCpThdEvP>A4{udK?d{E?1zPc*Tz!&@3zrJ$3v6fLv6R*sy*Z?H8m7R zaAtWu!^VQ94BLxzm2K)^Ss6#9JN*-di@cL?XiI^LG7nr z6RYzo$``+F8SI}l!0F1 zhC|c8v-u$ppUTNgDBzlxe70uthXNui3kuD>;;offZsH<5`!VcD5o(kp=Qz*u@Hn9? zD&PeL${$@q#+go5-`Gq9x888MW>kg4l3t>BT#;E#Z7+4VsH@$5w#Y0N`0~eO>xiRR z88a46`~WS4Z=NGh@MX=92ZvB6Mqt(>TkXH>BSP=w*}Rvsbf3SmJ?U+rCShAL5`KzG zH>e*wypj}Yw=S?g>Tz$tzpjTuszh&(D*8#`3zIF!F>Fqqt-%(>I#MY}F+A99zN&0)VerBJ4B8Lof-w6f zJ+B>g#ifA$yE##@gEn|(B{Q;Y4x>y>r5c7y_f|LhZHycQ{c;~VX#qm?h|m6NDgnE> z_+8HAetj;$dfAwk`-<5~JX*x$D-?F7`U<(*9|;I}YiyCOV3u|& zGxMdmd(OaboIEd?&86 zKVJVo^O~h+Jn03ibN2Z^RSvyC6=Y$^RC{x(>fjdYiC#Rh!dm6|yo& zDq>3h{urK_*p#+29I_$^l*x9a>}0@-M-4J>nMiOSeH3%6nfN@dYa@(Ji~!pJi4{#8 zP`hD(2qpN7{~?zha|Iu{QD=(c!B_w=m!KLyiS6(2pM(s<6}Yp&3PxrzGsAWw0j60z ze~2_7Jg0S6K?ZUgDC-yg#}1#PrYuK{Y!MLx8xGXd9<-~Fc!K*4i9|h9N_`X;7xzoj z4(0;DHbmwEU6vv#MR&7;*pF07JUWjw6P@9vV2cGO8j>mpL}y_S#KOUa_yc%0u)^Ba zN)K*Fai8ta83@x1H1L5zSG%j~Dfkq)RKdM-&DXawgB8?mh*Mn2zKhTuB#=B4OfBzM zdi7er23`wd()IU8f&0Dl3Rs)~knuDzF$&BGmW4y={a=cUfwM<~#PcC4@pUReqX*9k z!|_mLl=S>3l3$&Blvv!Mjk!=?bIOQFh(WeT|}9g znH78qld{~C;2~xfbxTN9JSq>n=@fZ*kD9~Slpt8J_fcy zDo_Ra7MK0RPF3%XxC{)2r9TA(H;>82+$22Xr;CqnFiFFLmum5h`7Xgu`8=>@paoFq?v)ufSdE*Olt)FE)Z33HtUr5rKTf^J)HC0{w)Jo`Z=+(A-IB6sxB4GUqzdUo|YYMuNbE6@Nup%g1h8!Dg_x^^fD`@#^BX?RI5m{Klm-Y^9W_ z41Sxsw$v@5)xs6^VmsC6)FVMgr^b%`irA=*x7rZrt`H+N#Bq=tqMsc_Sz7f>f+`hR zEc4o;dxp9XzVi8dtfo#3r*O6vy;!rM52uGK{n*O=+*tY5NKrMfX&g8nSkw9hmlT$d z{P%wmY@(SJ8Hu=YYi#K&NwI=9JUrk~P(mH} zNi?q7GAv9nrFv(qRzTdxpRcZ}g|A^lz`;-;ZoF5!ex;r* z3W$|*Da$`Jc=`z);FS>&6{St%ZL)XTQ~R><6`m>$hg(zkP`=!!-%<&9Gf z=FTbf1jGJQFior829Hl2Oo7>bB!>=lI=>OMd((U76!^7ZbyUiA!M~sVt0-*SIK5o4 zMvcr2gRUAEN`dx`-f1-ij^uaT6X@!4z(}H?7{)UOa1OAoS?8m{CWN$QNJ|3ee&w-e zyBT2q3LkU}_oj#ZNGZjEVDQ8S66BDJrnO>|l5>kENnFhQ5VVlTC${NnTLyAZH{g>Q!6xRv`VUQ^}A@`G|gy!r0d4)LKMG zr;R)@C$rX6Q8l%4K^iiz!yCiv`3eyMG-dBu-OgI0xbugvT?Stsh~t094SvqQrMt1Q z`mo^{SNJj3rOv70ogLThEp)ZHl^_&BAvA(ba6#B@CW-umLcR)>snq66=d&0k-Ob%+ ztDruEvdMCPv$z}ceR(#sHbKp;>8eR(hmw zto1(ak}7y`vj1LPZhbHBE?J3A9eu24IHfoAWi9whw$Y{LZwiwV8k?S!*NH(!$^l5zKi4H zUbq!scCtS=iZ_+1ODB;_wnodw#FR_-;g_kXqBg$t!EQWQedDU~Cx;M>eKEX9CAcfO&hO14QzYbhU!Sc_S-z)1N&c|0C8m}3)UgS972_JF+|1#FTOu#RmJgPSKw2m%D13Z7t) z6A%zUsH{FdpUAF}`a^Y&=12}1dyJ7A4P@Z|3^;Sp&j`kX9~ENv0Z^Ok@PoPQUjWVU(k{KgeZbFs$$&MPk03G4 z+M#0F3Jf>ItAVr_bAK|e`tjAmVai^xm=FndA*0_cGsxI9fo)@aVSx{TCXANgTy+G4 zz0dY|$o5J-4MIHu8)`Ae5K@mw)~mwff*&k~eE|Ma;0lLm?M!g%EPn)!0`Nk^tf6XO zTu00}DeVySLiBUk1dykD3P62ebbQht$(Z5fY~whqm6--RA-Rq*VKy&^puhkb@XL}P zOrfr2b}=nPuU}iM?lsFsfwlfoLe5q#S^m4*G7@Zg85u-|zAv$}5`taRZP6XCZ)Kg_ zu4glM8^E?)kHq5LbmNK%A+p1dXB{TpFJI1FnN4@*h;!fHZfZPQ2NmwUAvV8PLuq?X zAb+6=TQ8{h9*1~8nCodqr#n>r<50Z3x6GqwukmUuBne-zX2Oi#ayOwpHco`1B|+*N z_cCgv|2%khGaa;Fw*~wnz|xAx(K0IjU#=}r4f5~LdSomN5h|@`GU{4kh4QO-D)V6p z^U7*!6jZ^+rM_FczCF^GODk2v?I-253ERUH+l}>QjC-#eD=$*ja=|h^BL9c2N})q% z8SdC^wwB!lZ%B5U^nA9aG!7T&$hYEU^TurDd0`w&o^2c~qZ`!cRI>b;ix0;#o+M+h$DFiX&}PNk=A+{Lz?z(r!Z5}Ipzn~%|HE8QC^)oy*~FjH7;rYe!h zY`z?geoKCbELS?{Y7~vkuWjeO>+xdgsd$HFU#C(kK)Qrf`Zg2dY6JCGqY2{eUHG3d zBCl=;PH*6vW!o2oqVw|^m#bw*y87nT2CAg<+o~L@aofTh>$99MUWmAOV997kf#MMk zNbaqtJUA>N?P}H~na;$>r?mal#r&|t#_9bBnJR-dq!bMejoNC#Z=p3Fm6!X*EvM$s z{D`@Ke_=&~uSWcPNvX_Pde@nR%;wW=hd^84`0|ON!ZcFZ>yUp#p84BzRsqlGpC2DYpvS=e)jRsp2L^JG z_AksorMU&l_IL2af(!LzuNpTU0_du%yJ{8i_cLl1dq9v9tZ^Q&u@3Xr@(bWAfOdQV zymMsWLhODOJv+#A9q1qMX-X1l#Huz4MrO2XqV(5f`bO(i?^VQhM6bFk@;{Xs6L|;J57Q zM@ZF)(oA4~zS|0$yCT6cT2QJX3%p+-gPnpsgu^S}&gb6qP2!&GK#&Ho*zf=jfn^nQ zc5$(_P~&{w1(8f7q>c3Xb3i`?o&O@(ElJYy=41GeA9Ibno23X#E|8!nQsp5BCO9!9 z@ACjw2#nD6(dc~>BzKgm6d90^pf7lK>0rl(9(PRb$cTQ*fmz-+9u}26A|^xP6Io_J z=d1xQZmHm=1E|I)3ai%=UWx8W!w?e5Bq5_dgd>090{^1DFa*}Y(HWFJLv@-ex~cYA zs~I%SO-1GHF^7l~E_l0rPV=3PC@^t;E2U#pD(zr!WFMCxp>KiT=g^-cLmB!!Gbyb! zmOoKZ!CY1_`a!KT7W2+yRdAJGr?PS1(OVJKOtmpu=q8ym7+j z!!7rK{Qk*#yN?uen1wjs+11H+w_A^G&3#rc&d$^3TneKqG*ip?o;&X%y-Z6_#X8k( zRXujvq1DbLi@b0v9+J$lUw>YV(CgZn{PiU4z*mvST1t4@S#|Rx8y0OfT`#%^GrNGDu&s2f++y785jtJqp>4W{DNAl_ z3p>sQTYlQ^Z_)SqRCeCBqIznxKEd)>HhFL^s!( zM=1WV?Z}(XaW^ps_2OYx=^-+c<|5p{@`!h{lQ+yI_-5<&f?q_jo|KhwsO;UD|JK&V z3Qn2{Ma4Qu`|}sazoGj2&3aGz4_W9>MTRW=^e3v5Ixt>f&9n$wg{GJ&81L*Q5_pk&RAT1CT4tY6|Lv$8#9kBUrb3o*h z{YaNa&XuC^*+~ruq+fx>2O`E_%{{%n_0%SGz~8OO_#6-z2!6WPrL3Lj3S9pn+qD`% zHi^%mV!-;rcbJWVS3rI}p#+3O7*ZYIh=D{dY$)rkVoAY&Dl9ySEVb*c;s&$^f^Sq; z*N?A=3-touIfxI(;W$JVDd|oFt`q_1%E%>jzd!y%j6=9lD1=MV6wT()jlJ+OD@)N1KEt_}Dq&s7k8DF9C}#)-0&2HUvHTD{1Q*q_bQ}1Sl@!wvK2SaRI+b*@2>~H7}^hf=)^{e<~n(YSjj8Ujcon2?k5H+e_TC(tG zZA`~1D{6MRSE%?Iwp}hrQ)55U-p@04#5u_38{@O2{cTtJVh;~(6ItOPttkH^xdI!m zt8Sl%`z{sXhX1*qn>S3&I!o+rG4Gs?IWBW&pAqdOxk&zGb*ZaJ--lUYG4?}y(pHh_ z4_O2QyB8QlKFx@tNa^*D-!WQxwDH`ZKw;n|+{xhY3SZ=bg% z$$5*b859n8?%7Lv4so67D{k}a$oiriQeEYGd<(z1b3TTM(vLZMM>fLS7@PdIk~PVh z_sErA;`!!xX^kMWH%EJF%f^%s6@wSFr$n(-=yzb)mWA~ivqti`UegFtPc=X}nXOZH1`&WxWq>;m` zpS6u}j$#XKS=Z9qdv+)k&M>e=9?CuH8Wf{4>S-}|L5)-x%N1sO|6vRf#J1O@pMPxv zI-?Qf5=&qW@c*oD1I(l-nnQZyD2!YYuO>X3N=m^8ybM@WX95+%%GUCZT-@JJo-uTH-uIO+_l7 zJ`W^6BoPVeC17vdOe(X_MdEFn>+3JhAzoEYEL1iT2$mM|6Deg^lWV}X;3=TWL_smf zU(CfRg+f6*4X;3CwRju4uHbE$>>j6gteGx7=!P`s-*dsmrL`WXTwF7{T4sU_;bLD4dv*h zrDrcDR*chIXjuSHI{2K62Xpx~@V;u?L{n33M=0k#aG!;W1stlg^72>-)Xr{qO&UV4 zC2ebOcQmTNR zPU4l%8Obr1P!<|$aYc4B?1@h5SS2^TzU-NwLJzSY>%2%5y|-N6vvO#E9b*|P^u>T0 zJ(_HPuKE@+D#L3|cmiT&{sBM9zE<3f_$_%U;~zT()qGacgku^Bb%I@8?;Y|DUK?7J zve1tArhOl|qdycHZ9t&7gxhk>9yN01cJ9K(` zd(JWG;>ii0^0zv?moHz6#dLO#ct(^H5wah+=7lRdFZb+36g}2)T<{osrW| z*qHx6mBcBt8U4R6G4ks#oy(a&?+JfDNw&tMIoz{@QdH>K3rM(qPojZg^GS~WjT_Rb z9~vJ^dv0X(V`_TW6nc_X638W;=8=%K)WS)vW{#BV*YI*f6bTqoMwug(-t-7%6Y}zB z0Pvyl)9LalK?5q_RwB^UO1P5&O4NWz1M@a}fntA4j7~0Oa}&VULZC_>2M%oTAfL=# zcW@9wP~`!f9B|OwPI_}2TssC4fL!2!L=F`7Hf50YZ z7u^E2#ht@_yJ5Jd|6@&LNK%Ie=5}=74NQz+*^(gwPo0V0t=`XGRH|{5T}ywlS_i_xOR)3_D{2 zVMqW<96;A#p~k>xW8#DD9ZCv|umZIfIFA4+w<_cVdI56U!a62EKWtnJV1EZ(0ysZ_ zTo|C9KJUT;{T3X=obd0#NO&oKrsH6U$;zsqbOlDap`re8?k>RAzB%ugy4g*zkOyJe zYF@X{Lk*8CmRHzo@5=0)Cc2{hl}CRR-wnsb;MSP9f`+*upRNCmO^n6pUG2Q` zk^$+pP0THQ+_s;uFAHBa&*38-1(g%)ybC*xY3h559gGn>XtsW1wrkNjvi;0SUOBUj z8;)!FOZ2qkVgAY*FU+JRa?kHL3IwaI&~IFb|6by`K05g+pR9gVXJm@6LnYI%XvSO* zofXS1O|mvwc1w&^^<~0ww2Q~9y`4t7+ECPyZ! z!>zg_LozzG*EvrP+x^fh0&*-Wclrsz8hETr?#sI~ zSK()cdoAU=n<(Hwuo9do508$PkK6O|4&UqRD`FQPX#9wMb)hOWd`;2G=`afT`YvbA zJPzG`b;I0eQztmB(r3r9q^t<%A%i=;yK~v~a{leaeLdIZ3rnLmD?gIUxA&@LVWUoy zeKr}2pTEgI_rDI|q987jzYRU|Yo*a9#bpHB)t#oMoVA2MtfsI&5wTuSKYWGf+)rO; zqt8_zD+c{5caHvGNLsA(xf`&3-@u4VvmE+nrPMEN zEg=~M@7=YO^K@Sil$Q;9_^yzz8^OVgh{`11|A_2S1!}^ED-is_v1|Q$<#|RmfWQWd z9oT;V{;l9jMn;CL4`?n8wS_f3tOyh`_SCBB8z`M1+uH=oHg+y6DM$l8^APeXLvGR6 z*Eb#aQspe16OWMGBCzHRnWHmua!wWQj3BRZ#ymjz4)$Px+Bpa4IfN7l2&pqF=6kxj z5wVz)C(XXL*`dEn!x0z3B6$k1AxJ?4W}Rr%FU+E$cE9lCmQ`U>yJRJMb~d*rrd_>r z%C)?!IGvU2-4Z{7>|s@{R^?Vb6b?~8{0`ZW6@vXy#Xh`J5sW`0<6&b&S1 zk&}680^5+SD_`#03?Z)TVHKdbn zZ;oIr-u0d9{9QbwWFuIe!^j(7zRrA+sjc#<(v87{nV+|!OREBB-uINP=@K{O7H{GW zU(bZEG;PjG<7G46?*}jRbPPs%DwgwA|DG)zV+2yY`^z#jT-#jP@}W zNzm~_bOTBWbaAos1Wijxfplt{*=D=(MCt=MSz%r4|i7ruI@_|+Ldk_WG-7P5{tnJQUj!Yk);KYw~4h*akcCIH# z%Rr*!*LC^e5}CDa>!qc>nUE@F4fEpsTkJwVc3u~oIS960&V=kr>*9wG&1k_|QqlyG zf;VNu+PT6{hTNz>x_KQ+oerAF!tx{+4x&;=BYJXwU$9bpZ|3>?S$Ql5yo*N|7BP`M zMRK^F`ENfS*rr=x5`Cl=nLVo3iz7xn)nztbWl4Idt7J>&Ub+G=#XO<+HYNa9T}BbJ z+W9WhDUE71JcP@u&xl09qrWI^d2*0D@VSK+;OfJBhBPl z(~RA)l|DXbc%j9~Sy#3U4xbW_s zM5^W4#l~^zO+V&GEu3u9eVG$?y-5#Wme2MpvRU)-@xn8s5_0C3&Dz=~W}0J?+%@zn zC75M*S`Led&UFr1!+stexER@NbgE#0IpyY2LI(L)Ta)wH(dcc5o}TpwiRscUotkDl z?VD23551vvnSex^gOw!Aa7LD!03etg8oBW(KYwH3X#o2b0d#e%4ssg*`|)*i^{eoI zE>_Kh=*wDPO{@B&dVoG$7_2#&szCg-@@jBna{Cdh$wRzeUoP6{I#y(mn`Z{nb_xr> zdgQxv^d2oR0-{G*lpluK#fU6SfXDBD7bc|GBisJM!|P$Gydx~*02*mnZ2S#w1TO&zmwP}9JXjP4 zmUS)ID6g!nTm#@#b0pOX2D+2^=55yQ32+g?xrFQiL(1UZ$AHu1N7CVcHxBahK4~r@ z{g>0FI|Pe~<$ZmnA%&ersVLeA%t;v;nG|vbRTULj7kkE-_xTeguEd)v>D|{~JVt(k z$`bH3A6%_)C=FMmvDTgtirkv)_BRGE-}Z_zxa-5`;MGH3s{@qQMNaE**kc6=B`6Ne z3%_G9Ot72?tl1$zJ_7s*3#-qbA{Ib*q=oRkle2td)$Tm3pQCEDW(wU z3G3P?ER49{yKR*yiZAE9{mhw~SMDk~MZx8nyLxucMcpnMzY-D(%7rEKs}tO@;DO*- zzoGH#(@4>mqa#H+-*x0JruZLv4hB-*L@b+noc^inhpFpur2>MURWp~%P8ndN5(ld`;dvz{Q6n4V4%8A-GE>HcNru2@M1nyKI1Xe$sNMI zJ6>$Xz*x zu|RzPI_DPCW|WHZ`{PHZLu6Z~c{~J;N^rUGld9@Xnf6caNUSA9PjAjnN{8`yvf3HM zwBy|OzZP*zN08F}q|hneMeljrEW0o6;tv(zCf77MUmDdXnX^B*egpZU-+bttU%ZejGw3{AiLI^PKg&W|U;pg>9p1|jq75n$UsmtW%!UKD692_ z_-|K>oE}F-liO8nLtTPGe7EeGxl$4*tpw|Xm9-zcIn@hqL`BSrGM}>w)>&XzSpWY0 zkmI}e46l1fs#}smz4~DZjVj0tn8O{-a2MzE6KBErozsNR5%fd?GU%*w|aNywJ0A;h_qkgH=Ven)n$=fCKStV=`lP^N~&Af z3JVrhu0FVx`H9DR40{%-c0}%778i$lC}qS9TwWD!uSgI7nKr;Xuh$E#a!47PI41}n z3(HDQCIWgkyx6gAdB8cGgu^hfyuADf0Pa0u5$@BMFM}ZGGERfuCZ=YSi2$aN`hgIS z_$6Sls)3X}D|M8DHMPlWVgDal-vQ6{+Wt?2rdg3$h|;oWh=`;@A%sH6E*Y6cg9?dg z7-bwgJDY^$ND>mVl9Vlb|KE3=&hPww|L^NOk2+6J<@>!q*L_{@>pkv9nPYU&wt1xX zaai=%W^qCVS3a?&iK(~HqJ=vbhz={zNCn-Wl!MW*gy(_;2&-}=*E=C0A$`yIq^MVN z)T(#y3`s_-=g;7%E(mk3hlS}BwyB*c?dM=v`;S$DSJHN zmkcj%ikRXo{#rj-iGQ#D_~dE|6>0S~S!3OG8h_n0{<@IP=l9u8g@?gqmCW1*tj~s~ z3TSPc56yhEY7wr;SI&~Di~K4&>ta-QZ^`X$xiy2jR^#mCWZ>Lm4XuTQG@XQ@X)Tx3 z4?i70S;#5g9hf_C`ltcx<=UJ3=4Wr_>y_-QOCHW#Zpd`j{E|~zr!eZPJWs)t+xfD2 z*&(jRvDS>zS(8(Sx<^O3yfS{k#5Jck&|@iQ{P5tw0L6%1ap~lQ`6biXCt8K29=zt} z&dcVJ!Q%;OK0YfbDFY&=yvA$kwF4c7IMaGg$kkq}pRqU6NDMi*IQQ^)E+b>l+Ui_S zr^Lke<*kl~Q?*Z**xHCXeHf=N?zmncWe~Q=#mzZX=RSXBK-GCZ^1zkmpObTwf|#L) zFJaT$Ul`%@=cdNjNZJ?NE`Oc^@>L_vAuITt$My9$C3hDLt0~HGz18(|q~B+s73{}; z{{4}(k6$bq)jfJ;2!(2^6SGT21Pw@mXSVm`g zBP}gWtN8hI8e$yjAUr+)=Z}xn!JjwNFAZ|Neh9BA!$MKs;Ms`zy6hzqZP|To;bxX{CoFS;`ft2fdh>!&Xe{ZG%Qw zTb>WAMq5jqgoD2chw&dF(+SieC;zhuj*wFhMQNMNaITxP@2NW1>~ z^H^s2>@_mVQ>Tfk|9oVGFJAosOW0fM;fS8WfB;nsk;u@gFrzI_R>kwv`u5!lSHcz# z`~SRlrzWOnFA|t!uc!pM&-b-`N>J8o^`LJ!5OE}UMQUqj(ewH8D3jGX`~^)X=vHky z_%s~s(phC3Xu&W3RmFIRunoKY^A*2Z>fs3;;o*p>Eh zMD+)nyHnXe8fLoc9lkQv7mv(4sZPIsypDa+;`7sQkHt2bPjt*bFOFeITe8`r&{MoO zr>4*rjlSXV#EhYp56$7l7;*1mS?U|jDN#NF9XG22CUf0ACM@W)^$vx#fy4qztW($` zR62Bwrsm?hkYK*~W~=`?szn;~nOpsNSjg9e=1{L1+?ZG>jm-_DmDknPnO#A)5G>d3Ac`pl zj@Toh)&*H0s1Am~XSHhHzO{BXt`C5N6+yqq)gl$XX9jDc89WLB7$%>^-DnQqV9Ko= zkZsxA)YLs}Ky&q2;DIICdhBs--lPipaPQ$M{sF7h96iIdU(cmaAS`N70vHyrVL z_z0FRqR48{)Ff7dUDKt@GNf1lSVyJ_O|wSgDbnhrqaUkVUZ{hMIWHvj)a`D^kwc`_ zR$d<$CbtNLV@iEJ2u;M)KYe&AgPfbdSF{ zFYJomY0kHgXd;+b+T-r++h*1s;SJ|QcN~4SJxq>GE~Uiq)(N>G*9gAu?=#=c+uy%; zGr5c%==s^XsUcTv(`xrM&yFmYPz5Ir^ehQI+H^DwTrRI=H>wR(4GpV@OkxhtZaeU~ z_QO&`)bl&&Wr+!iudi}vx!cJ3r?@?rMwx#2NKfv3m)Ci|bm3@kLh1DP*7onahKvU0 z^XiXs?KLuI-~TH0&iRCW7qS}}m~J>rCu>fAcg{%F?HrnPJ=qwmT<4=<{%SI9ec_p^ zr@~**nI3K1`;uO^{EQoShr^524uJgC6aI0|PlH>Nb z2JTaoFLOL*L(}3et>xld^~q`Oqoh$xiQ7nDnA2CsWTU}sI+fnQj%V*CNBjDF2H7^b z^x6pv@|0C+78cBJDOo5Dla|e0Jl&o!Fehn|Xm@CC`@rIZfR09&7mPL7Si)XJ+5cHh!3Sy~upO-LP*YcA=?)9|^&R8u(UjK_Vr!zMGEX&>9i)S=Q@ zv+&-52x+@7Z-S&D)z4})^zu;5eK9`NCm>f!W8~Q~Ie%CO_CvYu1{(Efpk5z$m1~Uf z(4sh<{&H@1cBW@=U-Jcv_uY5)+d5j!cP>4!uo{?=m=%YzufuJBjD8y-UjFln?xQyE z{IA0*L(nO%G9EAL6y|Rma*E_WE2xU3C-gGlr+1Gm(fHVQr7v~4KG;%yK78kGi>F8T zS(QyDfh#Ore-=To0a)bdHW6wlA)*K!lsOJ_S=fY&t!$JfP7Yp6lC7-;;t&6Iqnug(D?*9QxitXK>)8ZGRMW09`+RT+ljtV#M{VGs}U`?d{Jr6H_c< zzRMN2T@XIX$x}Uy`oCls8!vlrBcAIP*4dHFR>)0yc;HIrv1QoG5o2)PhH=3kJ2jlc zVUt9%Xp)Y^X%e$#_&svPaYa`lrUsrnbXX$4kPIAzGaG#8*1GJFMbKg|h}SJZn}=%uNMG z+`3tS{>{1(DvpeHPxc@cPrF$rmbr7c7R7Au-Ux1p84&37oZp1qq|~6-_xCo4M{6&A zJ`Ca8>Nlpe%N+dtmM>{N?K9g3JY!O8XDS8`>`px8?KPjvv;tLp6ew&7Yt zdHMWtOs<2ob?ZaV+b#vH;q2)g3davZ(?&XZk@^|8m@q|}ev{2ZlJmMWmNk#1h8$)hHhtIUTr}h>W~)EfS#XsxzOQVj5&~)3mty6f@%3MFrO!kF}QIB0|OBZ@? z(1ghiQ_C^0HSfC>HS6#1nxgN_e7GmrW9Q|m{HY)DlG^i}%d=5TQ5J0wHSYR;ncc25 zPRTBZGyc@y>7E{Hxi5lR$=p-m#w7jiZ@eJ-%_3y^&D~I zQ$O>0%=3ctU5+91rls#ODaYrH@-?s3K5;;LXe^IhMU|U=tJhqENQbma5!eKR=sMC|2>mH~{cAN7H-ST2i3`_5QSlSa7b^i~2%hc+ z%Fe9;af6G82Vs-;a`h< zst0f}`+}B4JBOsAM;_RUY=biBd@Zt$+uH7fGY(C$VW*CupKC3#jRhGv`LQBb8H_>i zzIme(q)A`j^BAxqEPk9ujMlDSUyfj8N@Q8lLrFXS6w?jKjwE|{TG<((Ed~lECRL2_ zsyqzWf9Dj;wA6Xt+;KD%)BOseM=7)?r;YyU{%M2m6Wn9+1U#OhB2X|h`>QsSI8h=x z3r{T<4`Y~NS``wJNCeMWFmB(xdpA4^hZ{FH;x|w1cZggI`)&n(4~fP4{a%t(+qb_9 zEg=VH&zYGWA3hB0Tk-}&YR6u#vSjbBY9`GeF;Vk4Ws)Ky6AL-khK804-^2r-m%EQX z-E8vOx}2KRG&*CfzUKk{pJAm%S1a#Kms^N%OBJ}yPj+|PLa_TDAE<-!R(K7~&06(PH% z$NJ=qM(!lAyAFwNensz-U*!fDNyj>IyA^fjGr zt2=(E?cI3GndX~&<)m&U&e~oYvDH zsXsKet4Qx!s+{yjd6!Qpz7HoCHHM}(at_$d7JReiE?MeK7T9VdHx93#<6hx@tyWXwZrFF@AX8iJX|0@&4 zal`vxb{H95ZP{~r^1_a;?7Ydo>_n9xKho;H%r3*O8n*#6 z_=cl}&=~z_D18ZShSn*M0guAJ4&RYm*uMJYUxNw`Sxg$)*pLk8&_#Q7|Nc z;Ku0}g9A=s%8Gg1+siPIp~Vgg|GYGHKlskrmpk)kMIP6;2WG;9AhUPiqc-=qckOpl zQuv>E;yG9z<7670E;TvjF z#Kg78k|la9Vm=Mq;K4_wwEdhQ^^)z$nq}tmoiLO~Xl%^k9J%+h8)cEL4)>ni?4w7I zDybbZ#$Y9e+kFTZ{@O@0jC4T`I^OKV zxZS5(%ug4)iekArCnBzKA|IUN<599UCykKh25$mIkfw)r?cZMo$B7jDaubp&D`wMr zGUCv-=xeEvtCFOzckd1(A|m(W+I8!^p{#%t+Vn-~vtmyW)?uYJKKi{Qw))plcMWqU zn3&ej<%59z|A-5td4 zu)V;}lCePNhY4_UAh4@*d2xkV?K&m3yVUp{o;5#F*J-}_MBFMUa*0A$SBtKIxv{RZ zliuOmbr>vgQNX8JJj48itiqv)?yk}AZpYLnDyul_hTTmJW|pRvxpwq&ys*xvq=5BL zxxU>Fz-4ljc6Oq09!W~h30*oFPjzmn)uL#sRgi)MrgAQpO%%T>q)JC$;sdaWX+^>#BWST52)2wHD*?G}Q zJH4&SyV$0oH#d4(_gi9N-o-?=Zj+UCDP!I~Ph`}n82d+ZqDA7@DP5R8E4J?8Qbe9% zjcBjd#&_0#zpfoWKT{s<@%}B#icSHYH7nE~8Qn?gskE^Apg%Vc_t?hf!w2e3?iw=q zS-Fj;T53L8<+r{((n5Oe{u|w`9r|>u4yK8I>B$X{AF`z8JMo~&YPolrIq;=L9bfBI z7PF)s^UwVcxO?8PQykQ~$lYNuDH_d7y`ASBs}s#S*!6Nxo37>9BzpSe$xk)u3#GgY ziI&SU4(Y0TYyqt1!_g?>O9gXQiYL|{whtE%nYz>%BD!5VpE>7zV1Rg=^scd=X|Lr! zK=XKcOv6OQd;bT#?K-9Tcdh1s(#wc4?=!T_7VB4ciCb|)XLpKhu$iYbPtmP{7lp;K zO7s$ilB~7MGwwwglh@%L73SW=aqYMOiyB6G#o-?ejXJVy(_=TnxbJ!8fBrBml9szt zd!h8m%z`rCxklI?zf24ix@a@9cS98Os;G0%)g^a7vx5BFmRnV5IOfZ@@}A>k4!2|{ z$79pOOctdZ(kJ8NgS;}>DDF9RiM{2etnrkg{9rdkt=5qs-_fGEX2rILs|S|?2^>H{ z-?Oso>oW!YJCwawMBhj|Fxk{n)i}5=*j)&t-&)FKmnbPIybrc7aMZfgP z$psoMd^u25)Ob(jY7+n6w^*NLC+u`y7sFty@att7=&j8ShRi>U#>AP-l46}7Z{V#qUGY>yskNnH8451TY46<>#2Z3}hF zY`0dgFIh#AowHTYZHi-$-ll+EC%wDACmd8#5(dIpHakums9!qun2PUs$kW)^jB*=K z@RaB|~|l1-m-(2cs(asp5RLrvO zc`8Rhm(Se=D`i}FxUzB^kzscaWH&@M$V@sRCJ^~imq}D$ z9PlEnqQ@Hpl^1>BIE?Yd9$5R)Azh=RBI&DEuhz{O=`5v1;$!zJWf1{D8c+1BtgH$h z$($2?EjBfEqubOV0*gHYyOBSO>||4|JAQuG;KvU%kRB5r`B){0Ra)S;FoIN!#koog7H@dVf7WVe8{$M%f2(dpc`OKso@pd@Pmlwun6qI z?@Cu9!|Fhd*h;YT@lW1~HmoBzig_J!adGke<^Pc2bph5Cn0Lmy%2xhbVqm+8Ib*wK zW{J7bBk~<04-~xFyoM40sR*M${E5QBg9me8MBVGU6U-NOuJ&n$r}?w> zg4VROn_=ig-w=Q6V7%L9>#~;o9zns^0h8zutdQ=c0~Bo~MM$VvckqeK!UqUuy_`2# zJZ%O|uR)$eJS7F1GC0}UHB~h=y)#4dh+ax7x3a7AV|lKivkiYqj!tRAQTBPcsq0K` zP&Lof%nz>myR{YJEn;rc_Ts?N)U3X~CqpBj1Ubn#QELZRuPtYV){@8Vcw|xAkAj1J zbsBc#qw2VzL{Aq^(a~@et)#H9{9sZS=|0#J+}S&4Y_@#(>{G_)XLNnS4Cfe{(>kds zm|NBIF3#<<4IiGH9ocwp|BU>Q@vZWa*yuRJTf2=u52mNp3Fs94%+*!@B-6sYeTm5T zhOYf&6%Ey!`2?Rf=>zpn-p0iOJ%!vCzZ73~clh))$MDui*9a{+kK|_&Id09F%fggG z`J0+)H?5#RCx7X9$hJPkBZZ4Y5h16St1hhZ8(5GzCvfSf`ngH36J?>CQ40FO9J*C^ zf*gzT+Dn`Q2&6$(q+lS`Zgw)~W6}`ku6JYil;k*l=5Cv9ifdP{tYta%Ips(CNbF?B z!y=7_?n~0ttF@O$PFXL>3vIADOmVQb{cw0o2t0Zds^w3Qrp1iiQkpM_5I44_Fm$sM zd$BYjqSxO_K~bAyqtb5le#qv)#v2iA14{xM^}SqKa`%N$QV7v>UOT(x*A2%Xyn@18 z*~Mg7j_=E;bl|YbZq>rT=Zbzxbm^`lsUvmnSOm-U2x+(~w!0dAXlZfk>}rDsn2#F(iQHWM*6)+F)6V-0l3vCg z-5f!EXvE|Bu_z8?HEqUjSvcYO$1iqXHQ&?i&@Bb9>#uX0=bJynEct=j&req6n8!rc zV`1|(D^`R{=_Y$bxeKw%%37ijpUH9;i^cn;C!OXq{E%EhD45XYI#SSjbr^Z-Ufo}s z?a^L3@j+l=aYUGoe(XqY?s9YKab*JoR;M9Xm%!^~XJ2;|ul`in>@;5!ru;>2$%M^w zaiGS)R-C6|b(E(RHD=kH^)sW)_J?1ugjt(K?y|mJWp|$(r#~(?_uQ=i_|Ir8H&_1n zuS|)H;ZSChSJ}f4xC}u_=Y&5={269sqOkkH zje3BDz>H44VCTt)3=mcN6{;%2J9jE5fPuW8`|#bXfq{WqLnxW=2cj0wWPmtJVpvFK z8?%t9N}&f498Y(8aiZe(_0>|KakgtBky(z8ITGg*I>)A`Zz=w?u=ha1%qtrBPAo~_;d7zU=w7ivb1nRvp@+-ueFMVumL~%GK z7Cg)_X^ub>Y=V4`d62>{Y8kS|&-eR9Y+jKVUxpAin_(vV-}K|#q;7sd-84q6L)|3e zsM6)dk+CKnqwcApHoB5WM_#bQ5rR0JcDN4g{Uy-^2f4eyzt#C!l!>V5$6&)YycYZq z;%XFOZfb1o3O*s5A$i>Lw@Vw6UKL_ZJ(4jc5rQ@1jLio-idicwD-n<4;6SdGu^!H4 zW+)X{mivR=clwRKodj@ChC)M!~0naQjfX2&+M6*7m_Rv+AL&CgdG z*p`at=Y8sac`fYX;3rMB2$og(MdOJ%3ybU-=MHTM;4tv!T$acP)O31>u|+5Cc^Sr& zo6G9W$LFOA+Ds-gRVqp)Qgnomoo4jWSr%jT3>qHVr<3&cVctHf&SiI5w)VlOenEyU zr=!YUyL;o^>>{RI*-GYH4iu&Htnj3_O?TE*;19c8Y`j1C@7U9G18YBx4sTec=(x}k zbE;J)YNLRcpKR7~%?s|EllIwa!OGoKM zo^;~$9;4lZy$=^||D`NL6Q!ei1-=lG-JSLs?lQofTu&eV5S=<(hhzX9Wn;lPHjtzSNOzwj0@bXi=~eK^;; zrmZT-X>n<+A&PgqxHC7rgzG`%@J>!q>X#F9!JX4kEvb^|cbjo(wZG74p4&5S~;gyuoP-PXU#*3xsdiY^xb@lbVw~e|= zp$v4T0sZz0mBKNW#n-ntmtUNlxWvOVbNQ2U%vi;6`+7LK2uJM{|5LFD{KCnjV6OA! zKjo7?C;^$j$5Uze{?>mgJeEzP;WlW>@OF*;!IZ*?CPkjcTz>rxpA0`y_~Z|iD2j~k zihbE{&oq-L%^7~(*EI}&1aJ=~aw09HL_e$L@lurOMG1kbr_P$!hgH^{q)M%OG+0e7 z^e59fdI@$5-H;keI1jVkJseYe2|SY@4e^9}p?RUt$D0vYFmPaLVhx#UiS%@g&?j97 z1@RX>cWygY$MxN-T4Aw3KylcppEGZsjAdQY8J@n2VuYdZfq)ytSilh>rsiZIj@ycSvJI6^n0>z)~%dvid3SqDffQ=YMwk zG0&crCE222K?=gXb7z8&sNWpPo+NQXVw3;m9dQANuL4lxB;j|(!eS>@=u|m#KX{XQ zqOT#mJOmO^X=!Z5J|EQA*AE*=IK@v=gk$Qr?cC|xLv$^(z99H$<=^jSDGlU`Un>H!o+aTe4>F3JltV<8b zm2L?Nda`F|J(Kpig_*0iX-0+}*LUmiW2{@>-+x)1KhEAEt1A86bFh%Fhv|%P#JlwP zo6P)TWc=(6f`VK=P!4gD)L_IOqj-6D>vlXIQ;Gwu8+ef<;P>j4>nRR{aMgGkNWG+r7N{wCNVT{ECyA5tEeb#RR$!TTs#%9dgfpjs?hp;)nFzwfj-Q^ zN05xBJcfshJTy10}{K=Ox3V^ZvBvCKl0AaUN+;X&%hx zSfp6q_XKUEeD8AMb$}9-6vrbGozQ*$-eg0w)`^7DX79pBv`8Sj z@vIg=ugquU|4MzJzO6db3b9wMb`E?t+Fx#U zjL*im@-rvzl=hk3d>tpMu#uyrcb1b*QLFU z4@C86SgNvW+@Ga5)}#mas>GF~F7|%!tKZ|>4Bc>kl?*oycP1)>19#U|7)yHlNr$Um z*QL)avbharM258NdHTg{Pu6WsgLsG6Geva^{n76Sog2?`qa^&7hy(VkVzDfc`t zkMht>chd_|vc+sLkt%bK7`XcPZ!kMMJ#;u_qw|eIna&u$3a+r4jLSbXiFeie zfyvN0Joof;bjXd|Pv1~~u(zkjhXn5;7!t5$=o~bhq~$q>r0np!k4eLYm&H|p&m>-%d{#APGK}WUqy&qwsJkj;{G5 z;8v357B=A!YrRAr1ewRWsS4&cFn7WJokTH%AN10JzHTe54(PcLcOkqiR1aiy*dV@( z+CfPHE;`tP7cPV$MEK#&ifHX*h&?PWSmo%^H;|7!A#BY#f)lULqCUJ1s-YaAN4eaCXUyEw>5(X6KPeu``ygzI6^fli%uQ78HsSg~q zu;!O>6+tZO+DCV%gXuROhG!!f1OzpPsh@>#749CDC`>a;^(%jI29UeT z^((oppA#EPB0ftV+d}cOy`7zeW+2a2qHGR@k`*Qv_wL{K2211kd{UDo(E$1R`9&Y# zyG6^STN0|e!#B_?GeZDFO1kNhx-+>mBFCflt)_Tm%CIokEaSy?GEKbuYjt6$lftWD zqp&pfHw62^#%r+J!sAs-3k{_5o26f$JgK_Fp4Q&m?W#*!ebwyE!ZPvCsmi&Dunxa6 zSU+;run=?OZxs&O zZR*R+`rf2%e9fhs%J&=sZPH|b+)~y@uJN53Y^u7#qM5kr8h+;f%6eU5tg-`}?85h@ z{Pb|+rD^0N3di9C*Ezmt;Fbd_ED=bj<76Wd3g)vZ=mJ|8&5`leS? zEt2heb+C`Csas&M>W<5jkvh1QV&xW+KQhwxW1ch z^Gdh$HLx7O$F;wBf&rM6N#fQGRKKzVN|&Ty|K|I{;_1)nX|ie|q`3040;l^y#)d5? zSq|iJo*+mWULscs-3=5?oSE0SY2LTCTEbJLh>g>IloI)$cXplpYc!9CzMet|1lG?b z9<1J@qXqQH_JW}LEKnbrISK5?VfvszGDwX|?jmYNFT#i7DxzH2WZjvG8XOB=JO*Xs z^mRd4b;CKPbd+uvv2h5RA)*;9ep=YmseDwD5T_I(=ACv(x5f{iUQs+1AL zu8N_yNeXyhM+>;S)Q$LfO=xW4IN)+Q^=-K1GnpmkMz@g6vWpS=uY;uDi^{pz1*z;% zaedV6rP;ZwHp%)x()x*n{+>l7lNR%`m2DVO#AQ}H|SKA%gZS}dRTYwF&Y9(JTM;E;$#WktnP1OYdv z$+3j|DD$C~aGkISJVA40!XE;_6vJW`G+0?I?gta z+p^^*FiE^l?`2v=i5mV2j0uul4R>-s#t>g$*O0(K+AlplaPP5AjEj#4OQ3#qYHE0_ z>LfzA5g*OKz+jJE4dSF*)U+XG@&}O|?2T1WM0uB%dIk19jqHQm7{=mAD%H1yeAm~v zYt`lnd#+eJD;pan0|Ryt!<1ANjbUD*4Z-4WO@?3ZQ3p4O*6dQ(;ee7nhZJABty^a+ z+D^i_7UCviyN1lwqcDH$!KzRZ6jVG{^KbM0Twj5Qn&CEW`>^)obfQ045I_hRcWtKk zD95jbk}}U_EV@<*dIb_;ifB159&WA-5pi*DEGrQOtKH`;#fa!{|HD4tzgs*pDQR@z zSaw5T>!IJQ@Gt)~)HeOZT>+VOXU)y|6p}jNPXO1HTVSk(goZk`>O4~?Z7{ljPh_p# zk55S8u*_K@D@zkGWo0F7++M)FbDwQ?V~4x&a~+|{(1!St6%=1D?OG~bwmXyCTzDyN zg~{v79v(eE8l1=WFJJojA>i)is*qh}CpbbBV?t;aE{fiDWqjxP*ye)r-St=9t_J)1 zS)}JjxYANQy`*jYrIk6TLEX!wxpLlK6(VWyw^Qy2{>0*Av_H9OkByFwbjsiCyPUOEqql%#jLDvbcuq!(xVdil_IF z3ee~5Xk#n;G=5MIWu5MLaY(ibt zK#nm0B7I9>Hwb0E0beS?ts6}ddP?|nu*9Sc{!#DQ1AfbPh-r|8KHT0E5!A3>+hX$l{NRlF+TEzAard(V*pty|`f&};L*RL^6qmjE3e`Fe4u;FUtnFhhkQOF2V=$-h*@n9DT2lH4M}q?Md%!fh=MU-oJl;7K?4v z){UgLO01H9NtjG{Zl9b-W`K6U~EDDbC=RaaSaXISg z-rSghph9@cw&jiO92_F4g;Xe;@}{xUvCGV1Rn}YXMS5}~Mb1A^?%mC@voBJnfi?_f z3^s}nny#E@&V8YAD{Z-tA3VW5ITz9b=>*Y|33uGBx+~rTm~$~C^Uid)(PBRr;u141 znr!%1$9eP9=@aN9C}^kmj{mewceLW@@DSg)cP~|qYw6;UV#2B0hP6uvl9H{;;g3%4 zhR{|pqYf7lH1FTn%C|4`fBv_QThM_8@ZRMU_gf{u@dv3YX|{)&%f;zsqIf6{ySAF# zk7Aix&eIL7fIuW5OsvN?>)F6NMMa8XF;@Gcl&uavY?1;ZK`NWpttOGwv~mmYg(P_> z2yYbxTsDgk*IKlaL^d+mnzP<)Jmy*&T(#FBZ0JJ6Ut?K+o=OXfbx1GQw=2FuXyr&R z(yqEmu?_d?jT<)zJN<9~TeS(khdRN3Yz=@pWHln#QyqbI+YjBG=r?h@a^+l8lGaXs zegzOcpxw5a!}GBXHcJ4DR}cvbQjd}!eg9GU`t`2_X%>G93gEPW2H=yR{neC545$x0 z$v+oeY4{aheCABJZ(>54%+gY7lcmz{pz}s)+^|;>psn9DI5?P&u>`q%=B+Q_>J_bY z2Y2~Z9AW*vg)U=U#C-B>*$S1oBD*_thR8ezW;YRp$;Pu=H3h z9?(V{%hD= z@1V9L%vc*)+c@6&DO-E@8qj`MoH(%7{+@~1Q*QUB134{GXVNSWX{R0sVQC_~I;p-) ze}{536Q3J5!=3neKIkED!HDp0&4z(V(o`C{4T;jMJ5~&W>58MFdx$?z3YA2mefJ8u z7eRZm$5@K|MaZjvFJLd&`6;2-MJe|!bRN;$YlnIVQ_y>l9{Hh)l~}cK)GMST7YBFg#Hz0psmeo%);QKm-@mIljd*wCH-WP-Sa+Cmei4r z?&$NozJ2@19k?Bb1NPDIGwQ->K)XgrNa!g7ZB45gc5b@OapgetFvO29aUr4nru4J; zQ?1{!r%P}mdX2F-*%%IvS^mRRA_qehV@?Se}AvPEuD8;tNRaVEn*bBfgLn3BbugdY^n4=xDj;0PR$R%( z#wOl_ot<6KvT-L#sc&h?4`h&yR#&lsD~uivrdM$lZvWA0zARuPpW5o@{Hvm3UWjoc z0lRHz;78&supW4drXK&bKU#yudv43 z!w2|zE5OG8yjb!iEa8&<@2j4oD>>A}CU1SFm7~PTdT;HR_ml7ff2g;wY?BwY#pj{O z`D;i~`*K+3cxbLy0}s81&hgNkmkowD)%j>FRCq%3@+3CFDvz7HToY%EgV=#%|D{oM zR-{V_VRgVkU`P|CwTO=dBOSIsMJm<58hHFyLhcomO{|@W{HTA=% zLnO0=wy;&$3>o#0n|UXN(Q=UPAUioJNnJzZ8ro<2lNw%xXWisE+C_;wA5~RR6+!Q0 zrUUr;r|gassvl(9)di5k)3LXY2e*>cPIODyq+Q*1pn!S|oBvNQY*>-^1M}f|4zDxO zRqch&w8Yr6R@}J{UOq{G@cjxR(1BXt!eIJo&&lMb(zSgr&HFQ}#@w$Hb1KT=#;r;7Hnze9z5b z6C47<_xQ=id(}Jyw6jkaCPD2V;s6%^WZav48f$!iZsptd-_&*q2_=C;4_hxW1lUPP z?&`*YeiPJzF!)u;I?84BK_-(2(lGtcyW|3dt=nc^p+;L4J1MB1w8>bR%TmWix4EjY z;Aq>^fda-!R)b>Nk@iA5@Nv2t6AM+26+^N|1<<(uqnUS1%nDo!6|Y^5TeHum=re7k zrPYdWV;v+&B&~Dh{T=i(KTa@qcfIsr{?9p&j^v>)@i5={??J~8WTsp1nuV7(-_A}F z8&#x}u|tn>rNtyVjb-ZRHII{%;|a&jUzz9j*PDGQTu(0})}#_N6!3`g3%N~o2ZfMyonn(n_c)6=?tdP)=#oZ=?XY^-&c_$u-U}QH zD{$}!56z1)<1ZtO+kd*lt*)Of1UXSi;pIH|?E8E*m7cynNsz)c{?b}3w~e?yW9(C~ zx3}kvj|sr8(z(-vX>xKhR1e7Dx4NkPhCBP2K=<*4j#qC{HVw|u5CCo#+&meFVGbc9 z?H`<=wI{C8zZV7clceh)3khveBO@bF60PuL7Ek37IW4XfYu>EMv9U4&4t)LEXcc(u zl$O?w6LO22;1;Vr>Fc`&hBapz{eu0dq$7^sf_j#F|Nc6&M^RCn+9{_g(I(|W1YHWX?Cpvdw`~WLHosCp5_V`rdAq|VE zlzt#JfM*U{Tc@a0NBWnYJqXPJnrIC}!;T9^CRVR`E!9v zFI|@AVes=!{8f}Ak2yMu$}Wr^g0<+;V~MA}NO#U2k7A35u|gS0yH6W_r2KLiLaYM- z1~-|N8)eZvf>VaF@g%h$2M=RR^0u(?4h61^#*ZJ3(J3b+C1vXV1y@tFqa5)&ShfM} zz4zn^5|ON7jeglG;@&;Fir25XNRhrYVe&-GMgjh7aWZHGNNZg2>k}$YhTj>C+F^3< zuVnzF7y%4GyB^|n7$Rty%vV7{lyGLcU}R@!hwf57rv4?IC~-RKgBey)me!A(+Nd7MFKNipfA{B>mG4evKh190>BJN5v|5^9 zpweig7d4IAC!!;WagoadNL=%NR zhM62DO#%WQoBWm;`vW_f7E@@hmx6IY{9}Jhurz}YpB0q8nkKDuuS@iSo-DFRIA9v#i~WI&{~r>AEnpd^42=l~|;PeUo$ zsP6kFj7a_+_Ze{Z|Gc6pZ#Z}TdqnVqjrNo@>r!YO zY-z+bCb?@L`OC`WlBSSAQcd`NTl&0~=iu#zC0Qjtqtt*f;51!Xb+U+N-c8TK(i;EB z;;f~$bp@QDZesVI?0WX>Hlzkunp{-mq6eJ1--sYf+tzqD2@&%&7|nrG%D9W2US>Dm zFI;&96PJ_Y%9;C?o zIbu!ZbqV)wv;P4Wem%+w%GH%L{|f~vJ^=z|LKefJ@J(-$8+y*1U*A7zmSIxAtv4ye z_0t+yf?|WU%Zo;QuVXN=@s*{FROBKW53Cx<+XF*Mh2)%) zeTUPU2^>s>%fw%giK)seTu}gC0%Ya+VIRy@I{uUq#^Fo>3l#n8}@Ye#em z@;GVz6pckSNKogUJ9qxvN<=Y;KRVSCpW;Jl=jS`G{1dcs7I=J~^GIae^M)Vkjsx0z zbTe0JXlO*6#$0=rpTCtjpb(y0dtDtZsX`GE5wI?xlEm8s+AkNj9cGWh!k9?Yj$%oq zg;@emY6({YtqW=n?mO%yQ?i=vBu_{)a~S9K{Q2|5CIQ1I90{a`0`>OTUp2XmNPfa`yFKPaG{oe`X8OPjcj(@z%LjICj{Ui(X4x`wJo1u@s>U zcGrq`epNJV902tIlNd&PYppOSrcj|#DHT=!O?5D~z_W?HHAsH7PGNWgK~Y)_9o1lu zDWC)%;uY6iy42Laa1t(Rq5F;Y8T0F78iR}bisxtYKXfbZhn)qfTiA2AR{fw7$3qJ# zFDZ(PPiliEHFlgunV$Q=mNs3>6J~sib|F5^+y0Eh?ReBY6w(ufZxig(O1} z*W*UZ9E_z@xv<6{NFn@^#cE*J9H4Y(OWC`3$FG|zPKe;0LHHh7GoyL=_U+rgX1G~G zI7`-v_-586hD$Jp^Fz6n>q-C8xE}w%bgyO4hqY$ED_WZ2MmVi% zWVk7t4^<(8GtwmqNCZilR*AS%EG9`*_SVdlywXr0FxCaVs7_ ztSB;r+}^5`ZOW(+q|p9(O)~p!TN8dH;Lp=e4(UDgK(iZN&*Y~moln*nln^;Mo7l_T z6Qgkb4g-Uq;-Ez1F*Es>&tDx55n(0E1)qbyF)Mbz3g6z_w<^ej%+c@?Io*U}pMJ6D zFpa>`;^TCy*btStYu7GqM_f+eJDI$>x1DRpzpe`zfHE102mQ&DCktyB85s%s|EaBQ z`l1lVf2>VS5XU}&lpzlSy){N#I7IP<@>&V3Hui()4WKN7*`B*6uc?l-`U6CtI=$c zW&|It_9PasTx5%?9w^A-|Bqp=b?7mO8F1xwb$)BK0nmfuk@3jBrJ;clz|nOdAD>?* zCfEGy{%bo6V`F{U83CYDg6xj|GyLQe>P4k2hb7(J$Lf8 z*NBd7xmq)wz4L;YFZ&%TUv<9KuV)hIz9e*=ypQ)XnM8RwNwzH{oQNWgWobM{A0tZ3p_zn=2hYAf@s>06?rvXi~MU5_ja}C7dc~xC~js<{CwDkO=4Q7+qJd6>vwx_apJoMdGODx z#xVd@6vCycq^hjg4vO=2nBt)QdkTO@e*lS{TxeCWJW@}T*gA4h3)0E|+1_i^;Th&T z5m^CuA)8QtX2BJNQDG|dXic@0*>Arp^ljO0ds}lsn zBHD*9QTweeki{BB@oJ&La_foz1rWMw{~roS&V!AjUh&M`2K6B7|&Zp5143rqk$@GHSWKO?kC9+w8174;5A$VTP{c*-J=E z%Wd3$ChsaH76@F`itI&pl2e17A+o_O-|xWH3?Du69%GzEC|jBNlE<4z24il$;hySbHXg`v>p9%bef*BrexW`$v2)PGb16r(K z6YL*=OO`dpE9^kEOQlL$aRjroMb4CZvO}dDF$Mo&C1YbQH0o<}^xBoz`X$7#^}dR! zlZTHl*KKd`DSkqv6*PY9;_jY2J`EeS{wu>y2jc%LYg1f#=gc4lDQ$mVjiob^J4b)2 zuV$I#H9eKPJ?pQHdk@PGo^Gs@e6z!MJYBgcUMBoO5&g)OkCq}m2QMQF+Bg75JQAV?+#c2XH1V$h zT2KkaHPpYmjTQbhQVfUC!;$?4Qag8J^4+k3%BSh)!h$rVo)@$o$v&*A9$qTfV-*`7qdgnok#c!yjrsUB(wl*K5 z(?7uTgeft%MB9*zgsE}}%7_?UwMi1s6Bd}5St-d0*9(n^*a+w&Z9Ldz1Jz2xEF$Zk zrik$Hbk9Xs%*ehWcP&QIwZ0l|FVxV}v&`DFXU_&~kggRM7n9>83@Z$nYZEq}yga2E z@5!)%y!?DXC}eU%Z;r`nZasi$u!;U>FAz`Ws8q4R$D%AU`%MoC8e9viTvHPtCN zGq@;Gj=HeJ((Vz-=7W7TE-*q+N|WSveW#_P)6-Sv16=8X^A&Ka0A$hvxOupmQfTej zFE4Es6+I5~QiebH-1+B|BVVJhDRHe?v0rZ6fBt1O&7-#~ezH!eB!7&~ZuvC1P9$NK z8dBu1)XjwzYz-~GhTapj5?;AmJTG+74S*YG0AA7Exa8JB>IKPc?n z@Rn0jJuKq#AS{e565v0dsVjykSSURGr~T+Axe@3JTh%w!)$*`DxD4OR0j%k^VKBlP zj@&XjqAH?dVqYShJ2H|MzbngmXJuun_>tNZc?<{+Ih8mP$ps)uk&YfSVbFbN$UA{M+|?&f^vQtTVJyOOvHR8PGkNxuyusCGhz(>%PTMskp(K|! z$h}KGB^OjT2__ zgBCmtai_dbJ4+*h;#S!0csA<$_+QqsP8wHji$U>N`y#HRd8NeoBztIHa zNYSN{PcLzLj)@xX`*7;pNf@_a$pn3%Ie-ru8irr7F`>}C0)M1G8T1K^;9uDo^KB#h zMbc%K@)0NgXYJ(?e>?&XvX8X(;$ueSCD#tuOZc?;AFM-M%W724IMjv!P@V}b6HG^X z!R?dOIEdgeDxEv`Ov?ZtCob~QiHS|58G7(rZQH7LKLvChxL}ei8GvWWF}hVifu% zt|o*Cvf7F@rEM?u;>2h|)_qcjt&t#B{6Dtd1Fq-&{r`U_^VqVAvZJifAR5Tdt`I6K zDP>iZmSfM5kw}TsP+3XaJRzx6qDVwjRwbhP-=A;ebI$ko?{;qI9Hid8-q-WG9@pdX zxUMI1fw-S${ix*9-U`i;gxA8l8qJ#*9XE#G=AhcutQo#>v^+F!#f1q51_?J`%usgx zx4XP|fKvbVN-loY%#p2KI!MmU`0LK~yk5^Iv=}Yl zOtqVpPnEKZ%M~SUIrp=>C%ZTfZ(-)Se!JJ-TASo{i*8gnSQWZ;8>~TI$CuSdUEK)% zG#CyZ`9_fdGjU)$$?E6V+haz3-S~Hh72q~DEc-P$8Sl5v#u&y0&A3HJJQ&eoIWNNW z!IqBY#@&E5f>xXpaK47&%TJO!DPMr91}3e9dvMn>XWzcJp&VL9c}h``_`C_6`vVNvmFP|0#P{Z_zWKGnyF* zB?z;&(kJr(dN%3Wl`FT!ST{BWe_>+V{)Rl((>4jK#cOH!MKL zAS}E0s8RB?F%?5LGU6ETRM}2RDgTw`qa)D;4`gk3rtIDoymJ4i*-Q2$ZDnKEzdxLh z7_@8rXvie#cmMUTnc>eYCYt%SiHXvFo~eM>ROjKm!ISo#R1A3jt{}MJjiT()p3R?b zINv_HTWeBG@5^0_F5Gy0<`E02q#5UZe!^a%!M#kp^DrZ${hEa8&fU9@{Mx9Rn;|{L z{o7ZsW`({lE87FnB!$7te}DHZ^XDk)V&W5Q7ngGO?Ac)}A9ZK(d$ov?+H0uPDDd5m z2@_FUw~w-VSzT~a>U9*-!@38B!%;LGn=ey~#p$VK`fS6a2*p_BTY!L$?ZkZGE&VJzu4cn3H zqhm64A&EAawQ{`EyscUCZ_m5*J(S&R@b$~PVZYx`>3`GwgHm@rH&t9;Zf})-l5m=N ze}c*c!fEOAih?CWyNqsAz2%*cX7|_o%MG&HrnYq4wnHSFGQ4f@{O62q#X7|+maRc+ zlS8a7uEH2p?B{-fy$NBladD$H?VjJ;9Vji{1a@=!`7T4v?$Ez18Gd5*F>zq2fE1mA zOVfMzK7`K4bolpf4~r({9L3)z?py2~3ZFDFth#TZJze*H}TsPBPgqp&GSCso42!pp=3P)w=JpN~)&90@8Fv@cD3Xap6= zGF5@)lmZ*Cjm5eGO|czRcT##MJfPmm@aW{m@IYuI0vMx|ML+peS;?lH^DJZaL+^kt2oB->ca+a0j_J2hJ zzV7%RuuLQ zIN!5*BPFZp^z~;ij`iGZdiLp|;FgK1qi!ua(tP9WTMPRYS~S}_*(-mW=b9(uSH2Ee z*1OD|+0S^Tw}16-(k!W8hrQ&kYqlp-Jw~xIJelzYgQ*j?9%9jx5OJi+PeOb=PRyo2 zMp5y9ebjoEi7v2{4O-pcU>yA%8cLbk}k2I4{yHDl~zcWkp(0kv1IHovT%c zEV^znV#*^eL;y87MI^k++zaRFPPQhoQPL!e6{=hbH1(W8-)pppy+)7DdwhUfNU%uW zX=&j48s7oavo+z?SpOn>9-WF$UVLOF^5P5OpiH6!LG|tPTiK%P`U1bLd3iv{Yq2!H zd=2)0z=yj|fdAwrf|aj?BBYDq06{W|5_SaJFusV}R#y}KxR!ef{r@gc&o2GDX7pUa zMn*qBPqc6DKf>U#G3Eg0O>DW!uwt?6vt}x*dVVW6Z`i|=nYKA2U;@{t$Vv0j>>AR_ zk`MU_Y9|a$9H@vF7F=nHA=~xvcxn}nXdD;8;o%im*=jE<6ka%UA&*Yc?Om4m! z7q4thO48#&Q~&-3PBK`UvYsl)2`PkM)=bWm=#x^psZ4air%Ph{on3IF#|WGB>e-FJ z6DD(7d@O9QbZ}a;f4({Ws{!jButr!Z25pc&#b5u*3ZvD3ZkKh|f2`+WWm&^9Wd0hzw_0urbEZJ6Kw^YD$gc&keH02HC3MO z(|G&1UgO5~4!9Wx>=MbwUfVaYGmnD~uhp&%w_s+(ed{pIqfc24hb$D<8}5tjaei~X zEgNN6n-g^v{3Z70w3j>N{ma}jEf0ETO;X*pT`*zg~de(tT~tsIqFYbh-;?`C97b-xi{C0g0SQbh+IkKW0Nm0fKga{J8ee;r6X{ zDZSwG%imlZOa3!(q2PkL&7Ov>f>M0@c5Gx0!9^_m3a$?y^m;xzKKmtC#Iei8ZJC4H zXYQ~V&mC90Ld1ED?ayTZ&Dtv6DV$pb5xEqDh480Wx1wrZcdStOw}lLYFaD>o%7vR2 zGjtF*?KoRTe;AZ+a&qc5a%5M6QUE*o%-7v8WeCP&(V|7CRr8RcHu-DlWXE#WTy<6D zVRn=qTm2`Jl=^oWEs;L1U;k?D@@T63MCVYo$8(z`Owph1rL<9@`g-1Kecy<>-SdNs zWAg5|dov(;*2;c6Myi}SvhLvJn?uf<>=bjJh$0SUj%Q^Us<0a4>|W5bx3wRSgL@PH z9=!0{{H8U~sdOrLPp(P4zdJEFjof5GxAL_rbEeo?7xI!8LU2fM0=3KiNBw@CWiNkH zI>L5Afb~H{w2pyXLsJlp+{2SvZKZiY1vdVyN!T#z0*L$8oPjDTBIOJ2*YL@RENhLT zN4|Tzm~9f2u4u}p`yl`dg#!K3n;G;mW5yUvu65bt=_!nagDX(B)ipG3!VTUr>U%7? zfA)p&@CgXi;z}|si)Rwb&0j;p=%UgoqagFilO32Y11?@nw_IRh z(Gy|sQQ>)m1VhHvj~^G9oX(A}RJvXfpf6$r?nqs|BN}Jh!;ke%%+1?gdFK4E&FQW5 zzXA1VN1~w@VZ1;SH@J{(*E@Zf9w-KWP}ju3&++3MbAf(hKn_s;cXaf|$Q4YU$aU;k zb*CJ*faPDIx8HJWsR%H%x_ZN69hdC&49A>yc7+g7GS@i2;!n5DyBDx!_sUx$(tm#h z&s15_!5}ub*W>IqlJs+0uk5be{2V!B<@=>AvliPq&S^Yabx;HIkC*z))(g<^Sr-~m zzUBR{UA0^1b{Z_|qS>RHn!E3GP*B)2#Y_1WPwtrMvSoe5UTZcuu752l8^Nn%FN}`n zntZWze;SvqG@>6$Mj&QB?_XOb>$Ve^TtB-HTO0MT9Z20Ih3S+}3`BB+aeFbaB7L8y z8mM32r=+7GRJY`BUS3}9J9ln+VztGhmv3IZ+Ctr9uXV>{&mQ6`yy7SW3->j`<79&I~;l&)ghKuSo z^)@m zQ=Nw1&=(v|n9Tzj{oaQPxJzuJk|z3NJu-U`d0R!F4B3*(&vdEetmqk00>o4lWze6)+o&5Sn_KU1Zf|pYC8nqGRo? zqmz(Q-kN#QgNRpXJHi|`?=yQrffhR#4cVDt_3n~wnY}ikUKkSKR`~<>(4-Hp#_N}l z%~=)$ujiU*eY($o^!NW8Qj?8lOqE~ye8sMH(aj|4Wzz@PY8KdCkvSSZ(a2`H+TK&2 zoO(7X2swOn=HN-J9>UNua$vK6w&`Szel+jeIGc`l4vaO(8f<#j0~4Zmr#Ax}I|huk zh5dsj9lLJbrNCoA2UGwHZZCh`lAYQ*s-OIn2K(xnT-&~r60mI zEWHRtSdIn#xcXY+U%en(AOS>Rv0fC{lfhJCYa}i-vC@spR00D{z*9&AxD|GLUnu62 zUkSqq2Jrm&ABbuSOvz|stC&;c&2>LuDEOPDAGulPtdB?V+H-IhP~M@uGt65vEG@%QzvC~c54JTz)MYqW<>ZzCM zeg!ttsn-gBwpxq?Q~X(={dAQ%KOP2eVe1$<(%ao!PUk$~isOt6Fdi?<%POxPFlf-# zcu&^8e*ONPtsu>buI6pp?3fx@wth4*HLw@^pAuCQU;cY_LB@?cBbNLJQbhuqcC9z}$;Jr_>A;-MPezUJ!0=QgfiBd6N@9kaKh)$%zKGgWz)?9V$JjMi@od2?{uD1RA; zv;>nrk3;iBa8(&R4$D)!b^EleZ96WBKl+-p_|NwbNozE2=J4~i>)w5zaK&e4SQHi& zA)aW8{nzC6OIf&a1YyiQ@`vRX{I(k$+CJ{g4fa(5Ud7c2M~)s{FgV9L>S>a@9zF=s zm&|xVu}|S4i}&imgQ?Rh$dh96t#H~KKB=UZ2An1e*$JwPH#t{TNT9ot&I3 zu5Ys$ldC?uJd6J1(}km07KsKRhHP)$8h3KVhN6czDZiVmJ~65Vb9+Ec;GhX*CZga8 zLKgvv(0_V%)UL_h378l43+q?0xdsv#vZo_s#f>CC0oT*dX>f!^V-7`6ZKI+VoII;< z?=suA$<}YZ1xxzpLtSmkwlyykru_L0{->(Dnokp>Rr`ajN1QQPIj%ylJ~H!!g@XRd zq~#rMq|RsxANstaNBxHtL$@BE+9t++ZL1m6bl@{wPQUcnV{(?YleXcHILb$MNm;taB^J3+B-1dW*M{80$AB9`|jPh?+!{JMRW*G)^4m6ZQn~^K%8kPcp&m;gDMeG z`BvD_@QiHXpPf4=^*Gnp`3=@cT)LM6b!>Ch4Nr}n{d1tQ0t3oj(j10(JCWw*v#MEL zfh&Fvu4@@`f)xPl($@B~sL;vq%>xl*J(u1|mnyhsG8kxGy za_hGYy|eN~ZuX9bsxsZ*&KaqyRJS4e_{|sZU+Jw3>^KM4k+=Gd<=IzXeSCrYv0Zze zs4iO-huykuCbjXC0HmP7RQSl)kkq1PNr)$&HL+?%Y7=JEco|Gn_jZ4bPPxKR!RYa8 z_)Ju}s;WCEP2{xaLX~UksW{rPr=H%+$H!PLfip{Z^p=Ju&Nesq#AORh5O-Vm)vH&? zbxq?F627Y~XxIpfVesvNl>WXy)2$Jm*izLFY*-rTRulJfL3TcS2}tRCkL;=ylM_-i z(oAnP8Hj`~!=xQdYfCJcOs0tn5IWOWn@u;%|jThr;{1GkND=-(Q=k=BNhT41~j`e>5;QKAJil?=vf70BgGr?z1X@4-jLgD1CPM z@vE-1%~ayaPz~_|+(=9FteJbrN?aYtEYbbPX4h8BlW%_-kCumXsiE-Hi#A$#>P1p( z*>ZDHGK+3RP!dE?<2kF$!Stj-;gddn`gGclrv_Pi*ofKi_`@^xlI^f5eLEl*_p*AV z`ut71wMZw*k8Zcwu+>*ooPT{}4g5c9T<> zXdF*lCvL9;;)lJ3oD<^^Vw#r8GkpO~kLMFyzl%X#Ljq>TjP3&zC4jWG7_Cyug~OKc7JQOc*XmJGrp!aN z6&@yW1(1|I?^Jy)xhCa*(`$_vCA!FS;!UE&l1Bmd`<3YljvOPOsCt73d%j;08Xhie z72;wGq5XItjAg2)ErEOG#0`hfZ|=MJ%2gktcUkuTW72nhU@&tEjo{Iv93s&OzR-SR zwQ}W7FcCLRH{ue{_<7q6&gYa%4f+3@m=6d1i=Dv8045G>8xkwi)YLwnxMq&|jS>us z)R&sya&o%-G{0@16XFt-+%KR$jUg9mW$=Xy9Y^os8d-?1?Y*iysJWbax{jlM!D7<( zr|ku({PW?w|NnSW3|uGlKBPbRIRD%_S-B&lr+vibPT>>Jlpnx-Ha~3kjc=R7z2!@! z7+f^bq#7YsFpoO8S`2jo<9G$VgU;R@05OZZ_$H}bx_y}GDxlt=`4>H8BY&?}m>mZI zXh*9kI7Y00nqI4e0s~+1R|G(1FVv<}2ujVwJR|}*r7$*7h_nd9ZG9@Ibuq!?Ym zw1oZW-=-y)EdoNTr2oBjj*b%<8WC|8g=L2$yZ0AD5^SO{=#v~f4{KkodGtYTqk=W& zSERP|tmDEW#SH^tK0n*gRv|{Jb3*IyatIhdR^_6ZJ#e_OL6Bi~5)}yE<{g-D4_SazQq$-!q3|ju|>$cX`)(-g&bqcBz zwl{L`!IV*_w5ofEd2|h_AzJ{=XJTH7lxX{?{C^^%b7+N6s-@^|p*c26qG;OzU z-6-3%4_H~+k%69b)gEUmo&FMEwSBUo@E~$^HI9Z=Atwx#h3@q7+oO$QX9@oMeOzfR zZpRkVEta-9T=GnR$t>`OQER*Z{NnA~M@@s-_&LNga??luRQ)$%_zf?Oun!9-TWacb z;bfCdV{Qv%iORPoL|79^N}YxozZ6&Y?AoQzo6UTJaA4AG&#xZVwUuBmBg~W7cphW{ z=-KLgxsp5r*E~NLog_FU1YEHbYe_3{b;P}U7sb^PcUz8kW$bKWR1d#Kwye|CD3+Bj z){OilsI3u}4-2ZV8zejg=wR9tBLoYI7>nvO(1ZnEi?=N|NQ)$EqK=^yjTIV zlK;fBHyLkRLRZ#IW%ubt(qUJc-M^ajw-9~pr%s(a^OO=jZdzOn-R$hV^r!#8>9wsU z0wk9t2`q^@K)a$G9Ew*&Yy-ppF!RTJgDMxf#xm?ifuig*dTQM3-nZAzQ<;#E|J-3E zfTuUSkKm@0|7|hqU8ZsBj~ezfqgb^Yx$xOvW8bwiFwilOtc?ihG_WJq!gm4E*9J*+Eh1PH~r@!8=IDpwS{u*xHo4u~y} zv{ssOeOLwe`h$DJ=pe%reVnSrU9jGd zrLjJ^ujE^1ZtfmjeL_uyfUkI>jo%2ZeiuvMnZp^{zpu07fA4nEIn21f*ql(NyQbh( z{TCAM8I!QMguy>=mlNYnVrKQEXUn#2CxML_aoDtfXhu)&Alc51mHt<+8oe8!NpH&D zyaU{@TX^q9Qe;XPgdCf1vb3{@{RMC>W|$ju&jZOYSLKGJ)LEL zC~g`*^3YS^Ut)cjbaj((oe~tl1u^w?ffsTS=~_qv?Hb>llY}>o>F=N!zN-IP?eGDw zqgZ&xDy8o1EfXfS6o#>co{FwS-I0h5O#S-xRv0NhP4tyuzd6&mt&g{<*;|8Q(7<9h z1V1_VyzHk`gl$l|aeIn7+kv)V%zfyi(Gs6*@4!ww(Tq5YROZ1^4<8zWM;vA)S~eZQ z_QR*oo{c@fS$)AYY$j4jIU%9+)9wfVGcRtft?l#0KAPBXT~R;#vFN>9=n0 zf-I}&g%SP%O|B0eq`J$$xsgnii@VnPSnW>xx;9B1l$PwUE>Q17TJgf_fb9Wg3t!g8 z&nzJmMB3~6U)jEMrww~%*q3#vZ=gZb(y7xobekk2aSJ~(`O8^Z$*&LnG)t|~zkk7h z{R_M9I8)fRTj~KBiQ?<8cp;v35~Pfkbu%V9>idwP{Ed*(QvbCTTZ3=MbY zTT?$wudO}-W1*OSu@cJUa$bdxrmfHJ(w|>CT@mHg1`AuPV}%d2H>rGw$2UM--M2LN zCw#sJBfQ(UZSy|TQ4*Vkh@E?3{U#<_0ux$be}+~7%}Wb3Ex3Op4U2E1jt@!g3$(r9h%x?ywJ1!@}Y;zZ%c-@^T06E`p+Zc*m$@b?3bk zfQYfoCMKE8o_z+~bkh4TMMeV#S-e{xV>q>uMBwZy?WWCM1RMU zV1Ivoj5Su-W4K0sK3_1C^>2n#o7uaXMb0Br9liZ39Y5Dw$GE&P$_UcjZA5H8_K=bN zI(nyFSWu9%MR#_%&*>nMr02=+I0LLKWrt^Y5hx%EFWa-#*EH7@lYbthrR8u27y6#B zjTyVOsfw1?JlNICd8^`Nt1Ov2$Yi!y3=23O=~9*x>Bj*(3k&N$c+~KF@%obVt$`oE z{`ET?Jd_R$vd8UTln7 zmfXfYweXV_-osi3#y2-iBtw@+YPApuB4bfq@IdW1#l;?A@!udPK%`rk^=;M!9drBH5;`W?^_$(s6zh2{f?A z-J%@yw>--_?1_zM2BsFzrQFlCl`CijWX3-|BoGnJR=&)#{BRg|62&IvNr;F!>v ze!9wmWo{Qt!rUrPz8uiI_c-oBY~wwe<+P_}oJTzTJT-Msh8r%k3m6|+Mz7+p%leT7 zRykZY4Oi|b>#XaUd=~Y;oW0fIMf7b~aozUo+t>SGtCHMSw`Vyy#dL48lXLdQUYoGM z>&HCl921jch|b@>@32c#YXyzbI1AVK^r@U^=MjEs=dm#)e2l6PX9`kv!JuR2WAe+Z z%Mg-ysn|tCGdqun7jf|5K@Vp?i<3*oUvSQSH$p{TQZ?pMFOvyPDs@Nv4iSm7X52HGKfjVl z6mP4ws{0t6@vf^R{mcQq)ma6>{>SYmH}>DKPeEfU4IuGalXH5EQ-F95T+^&EOy2nl z@A3m;2nT~qHN@)!Uayu`&bj5kqGbjxX_Qq)Z?+3gM3&7)Ta_*1lt)nt_IDpOhgCoR z@`EMuO~bN7;cGdP+2&!9kvWXmtD{57c{wP895Vn)o0+jN#u%T?2&`;Tf?4FFQ=`t3 z=4>!6J-C0r^iyhl^8Rk;`HXMLIg#eaGfRy&ZMwcYB>mZ8FE81DJe70~Cl?-i@#vVU z4y)njhxzac_V!m>tm2iXFC>#vyvl$|GS$)f&eO0ijv$GwxHMlKUmf<#5`6748+ho4 zGE1}I)+4c6cy;(PWajBH8MN4M*2gp|!&hi}X(?MZlNx)6Bqo{xn~muG!eUq_GMM!m z(zuq5jUbth>fWYPr%oXe5fQ1?j&X}w{xy$a0rrntvrp;n+x7H6D*l~C8c`50&uNTO zMSj9ULfS?{i~O5V_a!gsdY8$p>*B7tVDz7mOP}6;?*r4I(!c*Ctdv9J?IijimUi6% zGz`UXk(eE0Tbf)xyZ?++!3eQ05?%AQZPW7^zG4p-WcCTmuD$i`$i1FAws!B|7)jDM z1R8vNq+{3s2dllrWo_3)QuU+<7-Z(l^z2^gLLX&ODiaZ|bW zy4TyrD9ya_gG}qd)xoJg8q)~Dr0XX}N(%1soHUl-Yxg2ybB_6F*V?O$_e3IXifEx^ zkZ`l%;2q%ke8)J=5h>ng(U9Y9x{vgSpRuArQ_!%^8md`a`ks_rU03rmrM&Kwf{I^S z|DCtbp!%LT)yj8Oy8l?2Wh6oZN@dijXXgw`CiDJXobNh9}Bi%Op2h1HrW7}%f=40DYE_7wp6jXvu7V7$B?6*7dh{} z8$bdcKx#lhF`C@mZ?r+05x`#UnrZl{fz2cm-_V$t16? z9pv8gOvk)nLZBd_!SBlaK?=i`Zo> z*xzSREXv>5>yZ^r<2S84`F(z?Qn?EQHBx<#{3UTtl$Mx3VI#30sr!l7^7}*QV2+yo z=R=ty!>G&{W1Y3@sl~IWvzn{5rOox1jT0HL~WjO%q*WR%@P|65@P-&XP9I5ZB?sfFzQWh9mq(fYEK4V8+yySe9wg&% zGBpVEfeqA^jKlGiK*AJk;JOW2wEJ_8&-VBGK_p{aje6`a3!#>6y<`+x+>J zL}KNBH!9Wn$!nXfUkozKI<3N#1h7HRQ;R3D|$jE)s?Ac3}D%fAWc=4h~@yj8{ zQ|_qq2ukiFA6})Zo9rL*(zRwW8;rf>LYpa2-!)f1^-t4lfIXRBUrRZyKA!Yr0^Q&| zocGfWpH{M#-G8Wz?vxUGWvWFC$2TD8>g9tR4logJ8tSj=bn4hUhrD*k^zvb5Nv*!yE27wPBK2)lyZcwN7i%a~JDTMNx%Bs@2w$Cn4 zfYy_#rVR);OE3ihe8~Z?og=yMbe+bWKAvyK@<>E0GVTD7Z=3sV{5QwCHzX!ZDZerwT0sQ@V>o?gaIg}W{ZfKZnz`g4#@C%(->_k zN0MdN9m4rSPT>xy=XdOXexnjQP%fW!mddTDZ9kcVEK|8-NePunRYe5@=iVc1d7d## zQew$0P6*px+~+FXRrP}M8}a`(^kopFA*6t8%> z0^nuRbo#BLVOD^InX=ni%-+|1iB8H!-Y_yZ{}7pzon6W*hzz_F>T3)1<>bC}=R_j2 z&JAn&7;v4=A9U={Vd9-?A5u$)~nG|VApgdf9Nx%gWD{LT4 zdSChNDoqpb@I>TSjbXihe!tkWyptmBGM;te%lj93p)InQJi5x}5{y{ZxTW^&+o!Ig zBC&7lT3eL?_iGO=?_L2O5E1(L@#BXrpR|U)D%qIy-IpH1{>{DioDFr0{Nl<|(ORv4 zy9bOJM7Fta_aQ%T#{6)T{VdS}%YDqD#Nj#2t4Pv{XZzGSk9Dj@Ij!d&L1*pN;Y=SQ zx5>W6Lyn6>D@pgFSd_tG%0y9xRAI}`8R(e93{>Gv7w?%&@iSsf_xrmM;igejLEgiL zEvYG9A9K~I(o%ZZ+{Jb4yJ2lgxMV7o8PWt>N!&kW&wfidYF7Pal6 zdt^!1q*_OoE*?32_;mvvlm9!%I2T<%o@o}3S*Fx|OtRL-_lp*UKQVV*P587iQFqi= zm6R;r@uh|4Xy}ArAL%Mjr?>e13m{%Q&i1uEmfa-I=QkJ+Y30bABI zd|9(T?nQlV^h>ueT(qF@6W(nSPTXCZ!Gw{(yq{iAaDP!=Z4DqZWllYH`cE|d(UezJ z)wbbdTW#hyX83CN^GoZZW^QD34pD9(PZKJoP^YzI({APl%ZN>CRedgGO4Ua9nvG-J zuH9UE346kLrsrpk9lHgtFe9{d=7tSd5H2r}g9U0P55A~v$bmydX}G0lXKnqMH}L3* z6RXHGLC6RqLVjFe1f49DOKBRgWXTelLcjWO@$=`7M@CL0wmqQxc8Sd|CsQrwgy#2B zR-Py};~^A;vJIrR9Ke+u>kvln34%_MlXEnuK;?2RDYRRoGIn(4UR<&jL~J;kd|0 z$y)P8l>>Q@31Sb=RqC8cDtZp zZ`-mSmUEA$L zDX_I&;Djb86`R~}#}bsz39qJ4_-Z4bVU*2r-P)2q&ngz(YPwTqDRu6@tpGco8>`b0h=>{HuGL3*1_C;_ zRvPp4+|@P8nwlm^Eh2X%narOb%=fJvcV_N_L7fq!nyAN-SBh8q@1*-{*)?v_5U<0$ zPXGQK5b?u3x7(=iis~xRk&5w$4oXT9yn7;z#BQ#MkfvA+X>c5Gi;8@_FMeNTXZHk3 zxtfN~!`aYqpMXEKdR~WRbdIMLXM(jH*raH(fp^d}%KB*2)(XqV4fo$b`ZOZSNI;GjD8BIO%D^>?iTQ%FoqI=0y^i~JfWx%-JHQ!uSa)=us&S4<&rq%Z4$35I{ z8t01%+&7WYSFc)et&TW>O(8ayj{EtA?N>7WTl?{LTL;=0fpQ2MnM%T^mJ1hy$z=<{ zkzOwzO>&bdC>AX?%nS79Kjx#kJQ8i=ZedX**J1?0)mVf2wVkViySJGXrUBLRm~y2K z4KLazVsG2IHI_)pB4XTK;O;osDkPiuStUmI@89R39{Y`h&tVevxRi3&!Hd^urHAh- zM^HxAoIGh$Gy6lw?%gjyL6@MWnJ~R|<2h8qc~;7$a%I5LZk}H(?V=Yl$UGq|=f0@6 zi0IkpXZ8^BGH>o&FM;*oKH{<$F^o@FGG)ADl@+K+b;uBb(jTOxn3sA3q$u2~BvzDN z#Z&D{>fLJVL#hfZSGsK!T^#uXfh&V|*?eRJit2GCy-go22v}=wI=cL9wc^2c`tx}B zRh%(G?pqSWE-yCjs-s!A+;L+!7n*vjay zQ6J<}SM=xYPv3m8yz`%5Nvhhl?e;>|cI`<;S&5->Zo!kY6Qeuy`TOC4Mg{#vQ^L4M z32Ci9m*E9j^0vi40UGvRFFftrkeuAY`>JxIX{{v_cnQpP-@Yb7hmLB=(MUs4dyf17f?lbTM zsu&o5{{TqCvWpIc2L7FY{_!3Is>BSrz~^lBq%{myK8QDul4X(C|^V)~eOe z{{_>8!Z02biolYb;!m5dLJ*h~Z^1Pe#oU^-t7=tplU^x`f%EJqPp`*d_y3*($iV@t9v>xdkTEb$P zl7o*<>wFJ8(6U8~4BYES!MX4njDBXL>jiYAmEG6rjPn;IA3fx8nvAxmP4w0 z`Bfv$f9zJ;^0$@W@=F|dw2}`%UcVaaRyw_^?G+X6r=hXOv{IT%5@y?9G_Jd~8eIG5 zL%X$~P)2@zcqqMIbTs8%b4hx?Q{n4ZJ!|JPO=j_wwQDa59Ie;zEe&*AUyG~rEWO~x z7#+hESDsyE6>`PXOXFd!KIZ@)88|=plC|Tj|NfbvJhVE<%ZCVV0MB|Z11Z?#I;)w_ zoqIM-_q!o2Ge6y2*w;~GW&B$TYKf{U>odTQ(jpMer_UO#yN{fO_?rdW?m*Kdu%$)5 zTC2se_EXyI9M2&1F)^^-q*=3*N4{j^Qdi+HVvhYRI7JOSlM2JXblBRo_2IiqKWQaa ze{*sE`tc0kWcrL9U%pf(yC2_c(;sB;v55EPDw>Ul>8x1?`GI@I`3~-ux-`l(%Hn1t z!Riv|G8_2%>thy~#@eeiOdLv=9EFtq0=0@bP_T6qd zS^f?SNtLS0VI->GAI=`Jog?kCqpl{jlQ?f#=rGth|FyAS!Iu~5vNukazD!C&u< zv#6O4G*u=iFLj|Ap$YQQT63Dl;A!73lB#8e+W{IvYW$Kz)`U%Jk?N=S5{Pn=v;+f1cF#xhiTCx~YO6x1&U6Wu zW&`Yfpem|Ayfo+e)|*ndMbH>-KR_MptmdfgtK5&U(VzK>Q~?Z$y4u)n$V#@|m*quA zP6_!q4;~fw%PiO*oEcrz<}F$@2x1JGQMGU1JLk47_5_k(3kktc)ii7T`0>>@oAnqV zz!%Z8yz34C`Webuv=0u293a4`N!yiBU3h5^;OJlnh!9&PIch4nM(92ui1@Z`u5M*9 z)c>fLnVHX@Ul0e4LTkMtJk@e8271U_t{AQFa3X>wG04 zA`0dgSt5QdR}lU!s8Rls*6#Xo{4oLaH9MJ*;engW^`Ei2#?W0udx*ix&_#-l-zDjD zSKRre^gkxXxyye$>^1Kncqz-<4wdyVyLr&)>$xeJ%_e#Santto&@&@4=FUTO zxKsO^E1T1qCf3*5;H4StuhgMK0KnenhKG!h8-t39EM3_*9`07$7s_xp^L=xKT6N#! z3;aDP;mz8SV7qgu=Fv`uBsZY~^7FM;)Hb38mpj}jVUkEUBSq@!%gZ}&Wo4C4!V*W% zW5b%E9TXMA87T#OAE;cX-?p&wo6&*=AsAQ?$XStK2qvO0LOhrUf7dV&_)C`52KaF* z7lubaTkWIk8jFP`cS&)3MrQ0$k<1+o=@Esqo>y^o{O~&Cm$Tw(cRn+E>-=q*nxtS& z({0Z>^_cp7!tb0TsTvYLzUiM|NnV`FSs8pW=)TYg0vCm>dYmQm>hgj(%em_Q(G*wH z%gc8w>AD0Xp467}CbIL&hiMkerP}&`wQ2K&ry2zm@c1X4K|5G3FR?zfx->KS&bFW5 zqC}->tFHc@y0!O`GMsV9-T1)>W_2Xn2gJ`J>W#uRnUZSDwA0}tHtRAJ?@M8m=(R`FXXs&lZ{B1AJA0&c+KK(}&BMyV< z9I?wPE013(7lVEmL-X9L}jRv9bzA98y|qRtNN~ed9G#u-F2`bV*;nV~G`g3qVDt8W8~R%nXi# z&<92UYXt8g`XCK%2o3Bv?bv_;1MY2GuK#@#sS*B&e@>5adMs6y=&=Q9g-e8I`hMFH z{SSozSwOCbDQI=n60&17O*)PTqgk{3`RU|$e0p4s^7$Y&)!5lN_JWB`|G>uSho0L- zgB)Rq1jHexQYsu9%JMEfTP;`_0tOl=6r8<%$Bx=!R1vuD8OBCN@gKiJhuUzW73*`A zLCN#NJ@@X-WD+9_s6n|tQU7+Tkc znuhw99=Yz{hBizPRkYu@Ngp?jsr759bZ?qfU+PyMlIbHOvQDvh3w7(iE@MZ_cN>&s z@nc2u(oT4{vULI!LC~n4&y=6#xo6K~tQVW#CiyMtTKhik18-luStIhmv=2c2o;VYwoDSOcEzIH zr26utYu}#sp8_vo8nZqS$DMYsY)^&fx^?aoq5^hzm0V5jUFtWyu%rBPjnRKxGm`WW zQ}Z)C$KM|%CPR$l9>OCpejM0;$o@Sm=1%Y*$f-7BlpFoHAd={C<&KTq>+8;5Fwu1P zh~Ya`QRGV)=xO=XA{5n>+W&-}p_7u5-O^v3c2!@|isG;q8jGOrg^uU#j{Z~s`FPR* z6npq$alEHYk-4#mt~(QKEvt~IL{rW+Y-?6kMjTQZzqlHa&?$}|?5&{K?{3)d#IReVi zjUrK&08PT(u9=^Mk0!f_D$a}6*6k}vk_u1!VB^Mp?LHVz7j0bAREYG~r|vl}ET@==DugyL9YW z!XXzmLwHX)x8tTt)Mg^UVtI`~ovXE>p&)a`^2aNdUOsKZ>VKhk2$KjECNwco*CN00 zlQ1T-LGiM9s5Be)&T|iRk=L?2#YMBPddxF20v(xBn?$2dt((e0-qa&DAYdHM3yoo& zw8!xcpTAGiC|n1i4&r#}ZxqUyj>}gy|6`;KxabXyUd`K!hWo*-W;7~dKmQX}G{ub6 zU#NNMZceah4O>z?%JH2y;ofD|71MBMsEDIzZgW)qi`PLVLT(DODTY+H3hSf*fW72D zmSdEG-dRWw$=22*$(dh@kFXmHwmSrH1ny6Q_uF*H&+VTnwV#{-vp0r*a!wErsGhrN8y!>D&I(EhkAlhE#m^Qcj(IKz(#mk1jm-S{8orrYYef zdStGLfIu3@z|l@FQ$~-Hl?67c!bTuWP4&(zRs;&JUU0{Kw@sh+^P8o+p_N>v6_(nw zzPy`_P#}@v><{d z#$J44eWtSk!_zhFGJ@UdgBPgk4 zf>SwY?4%Y9KUoD}sWLQ#RQGsaL^{<*w$UG~t80djWq5V`A35$7NfgP`ij zkx#cRzYzwr2UQh|9g#8^6clt#&#yb(7>Wj&1Ov#4;;H&wSV<(O#_FLFDo8{n4}{o; zf>DMu5Xp1X+d??jU`*1gR7KfL8LPI?_dGHqmQ(*EeZEYMtag8das#fX@2Zc?acZU3 zYCPGR#Gc+Vvhb2=xDT5_IkQ9JUc{R$;FE@4Xx6NmjYyoDtLOLc|Cjn0<@q~R2U&lT z4=R#)L=1XropI4B-|J5))BmtM-?+=4kdk<4J$_dma{byfDYovEt!eYmjkVjKUa*`b z^RW?v55BeSy5HW&(b2fO(yi6*bpP5+Fm&E+zRo5wUWG1m{d~W_OzZbTD`kU)4YKZP z<+jpq;o+!7+f-LylO&9L_AL5~dGX%k8g36r&VFCjmL=qS_V3STT&*q&FR@Uv zS&lPe*fY;@t4&ti*_^0IR@nuDnY4wwZz@Cx}!7jqADXM*Nj1%q^Y$byiTsC0)iE_Ut z9*;D*%DXi#7oNz+9JjBpnh9O}%$azt=NX8NSHUn)0JFKnG=t2yFdT(CC#m6kk_}(k zM`!ajQnj|uO0-Ab-@+bRnwn+28VE~+^DD;|3)4cj_0hCquc~FT5wPl3EE!bpKYsIH z@Mo|_*hvWz zuMXFeFe5fz#QIHEHfIRJU~=QCF@NmmAN_1((aUL4+@n7t;UFg?U zSDv+(_%9p~VQXU9y@zVE*vkO95~_y~hLRvpbQn;XO3Z2vlmDniJ0yfG-MXqwk0M97 z-E8LmuUcn^I_B+b`SaU+4H2=uIL-tiUbq0U{mT~h5IrB80Wzt-LVobMD^v0bmk;KN zu$j8w?g&q(wnNzwh>T@RFXsoYd6_5#MIMj?{Tn2`f3^>+ju$7nEyjonW8i#V^Aqwd zyC@EG%6~+mM`Ws9?}8~{4#4$B+VTk#w!_;mU8;IDS}IMscj^+p@NE7Hwnu=x2>X>`jVUE?jBal_`5ngoaH*m z=y)Ot3;m1l9qq$1k4*ZkqX!RKj2m8XoYAY`>(}SE*;%-~_?OeSCjz=UCc-1<)I zg1hiI+3D%+W`V-AR9Z_UY72n~pdg~`jbj{L+Ne zYDNnehSAnH%(Jzv01!b;L8Bf85Rr&*Z96Ds~SuU`!RUl^6_ zpt(ExB*|q%JIM`C;COby7nR|=S|9R=cz|n%Emu+%6DHlg4qPqNeQWE`-K8E$Q;^&R zGll> z$(}$G5@A%x=za)pnQ59jW4{X(%<9HMwn1PK+$MtP)6ADiO?V1sZ%9;xtC2oioF|e& zO4sl2|Ecy@JhRMs$c)Z%T`o&zU!gTA4;3ApFRT8QTm1MCRs^ z%q})QweUySf1K*3Hi+*a8MsD!PNDpcjG|$J5ybd~aoTBSlM+j5&H>n>g-Nw9OK1w} z&}=ui`j+O=^^gn)g^f-aYvGVkS?!`M`3*5RUp#rsjG~{kqcoP5Eh{9JFe6FLJ7py~ zwY>D5Z4@8V3!9JZ0bhxSc@;w-iav$YfhLZn<;|1(Zwm>ylvuG{lAf~fL$@bidjE+K z|7KUZMt;8E{?Bhv)z-hdG;LCk`23(olJst=N*&KUcXPSky`+poUM<35^{(!nIt7Z0 zi`WLO#zkskwn$Kk92VI!lC3j0X-U1ZKUqtU`~a#kghU2IPG%$ZIL;!-@gn%1Y9-O0 z$%JeDTl4w}x_)Wq-I#dF-~2&GZf>=mlIJ9#v`|!OZOyJLYipYd0>IMJa;KJ-H4nYv zn_j~tCC_AY9DtczVV-@are!5oUg2}CUk6c8|J}XegL+okSOK~rDD(AZOq-TTX`OCJ z>);V_f$;M+H2QoAXUM7P8NjVFnR4c0B_;GtaFJ=<-7_EK)fmU+w6*c0}^#! z*oRdlLBWYm3Gp}`NMDCBd6uhdc~R0YOZs(_?m?BC#)$k3({7I}=9G}HiD_%MI5TKD zSy*UhOm5-dXJq#_C}w$j2S6R}b&(C(cMM>yu8i6I#ZVv}bjBM_kGhm1uLzH*<-@Mj zMuDlw>$XvLv>(ul6~Xr;Znr8N%b^B;b@Kq}?8lEkhpkD*-UC4-atX%}?ANLDlTn*X z8Rx)1B;;!@olE#2ByH?1)sIGh7%+0=^|`~Vmz(*WX^6aTGNbL_yOM&)-tRisEt`01 zwMj&iKYXJp1fzjs0t3w+VR0F)qu+(Hg3IB*9-MDgZ ztSO9s>g)6stx!2JUSuM2;YhBl<~31=rL$(mVswBHMO6~B``6n&?3wqbUAb(W8AF+Y zFO+gZwARI8lB&tb%Yw=wKi5SuLu}3R2H=VMumvAhMCWO6W6_G0E6e#2V?jJ&NwWn5 zp0GJu5HA379$NVA&uqoDBJGOiMrzxk$bO8zv~!wJ+O;$^O0f(86V+7O=jNrI90<|% z)3_5>`OSr|wJ|Ix0ft*a-RdCdMjrDCR>C02qWguPzLwJ+)v^dE%I<|W^xz5ak2x{x z3Z=$c`H2=I9!tF>)Nhqe=~tM07IyXUN2{7PX(BCow2o7tJp3|Qg7ixV!<&E(hWh%e z^HNdSUHMwS?a$6j6fDtswF0G+1qH7NF=Zb6guE}TUJ%iwdQu^EznV^^oI@-lI3mnx z|CBHCm*r;zNPkO0Oc>yoRui_SKa3<=jff)trs&`Bxwo18V?CTuHmbdF?sR$7mUmNN zH75PEM=eNOamDnAVvo=f#*5^dNZd6vb&WI!7RUVQz|!OX>t_Gza9Z8T=?{)-%Q`Hf^%C>dIbhvzC}Jyik|`_8HWEz_WAIq(Z1fAc-f|pfb}V7L zTF3wVM(JmS`3sunm-$OvD(St33?fj02o%7-(EWrGFN;NHJj-`mTis#KWGvgFw^UXs zgAp^SYQmo&7@*+b;A|gAg;EI3Oswr1;8pZ6`^m>d4hLS;uS4@t8nekCvuT?cJ^R-5 z6oNaa4SpX9g>U~fOIk?Sdv$UHN7S#NEw7NoZ6(SJ7^muo+YKb7(Eb! zs_OH0sKCVJu1Osr~pikS{Z(T_m=FeyWAcNjNE;>&4Ows@|1I;D}s!#u6jK9mYc zB67xdGA(D~gbAyDe3c$FhF5=7Qz(TgHvF_6+PfIgRT#c}^3LVW%(gG-y56ak#wHi* zwGCh2R(;KSr;+t~MAqk)4b`=Mam`NI(M$gDpu>ZXHGGX6ba_FbMc9O!nkg+Cx-Gpj z;JKag@^p8XXW@TcS(yLT^fr?nqO0Tm=MW*%xP|BK-nY+4sBU5I4fW6~YmQV*-;`j6 ztYAW0JpP8;U+$eSK{08>$Y@U1DV^-6iD$GIzCFCev7Mx((V)YZzwht?dH?=UQ&c?u z`~OM4Csjgh?KXqBkG{^eoZjQ!+u{22R$iK4MR6+SKIyZZn!0KT8<^b*917(i*+w9_ zRXk`xwY;fzuB(RbV9?KoZbw~Dpb;3por28+R@t+1T`<8zjt{g*q;&Lf|8Yr3Yty+* z3^*y8wl7^^qD4xeE&G%sCW65{tFTTfD{t+4_n&{P*;wR5CKQG?nphJXHXwzFPvO0B zF819hhF2owL7bhsb?cUXF@(2}X^IMGy%J|yPop_=UiZMwYJ=fSS$W)MsSUB=$0#*k zWzQ*MtQKoRyIJLgY(BQ5h>G$Fuz6K^FY8UX8*Vl|QBwW8XCvy~4s{pK`6I%Rf9rT0 zJS|(}JFoQ(?F(YJf{6GrNOsM=QhRZA2Q02zSPW|w6B_Ej(;a^JVNsF|H2M{f6nRaW zPZPm@iHWU=`wHqT{F{CMwrm_+qI8@>c*+Dxgg8h=HT7*-E54ktl(8odOPS0D&mMYD zemeVm%%cCy;kDuFzu}nH_Q#PlwW|2L=W{J1txv{>+I|Vmub8_v@BGB*nXQ%lTqaI> zURnMkIJ@_R-0(fkFRKo}7aH6%?De?Sds;@R47n4ed(&ir`+B7(YOT140qOT)1OKft ztx@dK<@_&Vfk*)0EiO8>hMq=n3#boW9(i^hHV${yETLxgFV_j!YOBy$@xn=m!2ozk zX6wCeO#Cbs=C%Bj5t&gw@QZj9A(OtJD*gI=s<7nSh0p~C_1m%WMCyFWkINW5>3a>K zV!q^XvW25UfU zWs6*O{V-Su3O&{=e!j4_!)jhl61H_cJ@Yg_y_y{1ng#$P>C1V&io{BK4tcyxkdJub z!isiNZ-fVXWOwKC@Mv|4F8-wG6xW98HE*QVpYl#2@>CAjS%i}q@cyHa-TQKx{t0#Q z!>%)ZnpaU&!#tmqMoG`erlY>Jv>p=`iPlL@@ENdm!Uz@Rg}+3^;L(VRZTUVnrTnPP(kZKM~4&f z_B?H6JgK4w)UY-Rt6U**#(QF#ko;%L)C?_%;5*{y7fAs~u(w6c7iN2!KPo`{+Uclt ziC!r`A_86SVz$>1_O7^}J&9 ztN7I36HHw%YiT`LTw_puw3Bs2$dOO+6F)nde{-W!3t^sEYnaD&sLih3DuNhZBs)mk zh~l#W0aA`yK_k@5t9pjl_noB-b1)Qi!+vgUrIMcXJs1D#ImPkH|I!{!(&D2jVX?=$ ztikoPDJHhFpHf_WS&v8kM(A%;gRBrKNjgIDixIRU!>B-?7r=S~$BH>v+@sQb$2Lnb7g`FX_$3D+ zO89N3d#f=>*4%;tW1->@so2{dL#lw-WuGuBC#-3)EN#wrDpmyCrC3KLesX2ooe6sm zD?Lx*jjG7ifVVH&zoy&bd8Du#-0@+G=L1`q1|V+k{Q2dUax>1)r9La4;}Gi4!#N_Z zje;0xoVAkW3+V4&b8fEYX#anFeFs?1ZTtUSl8RJ>hN7ZWXlPIxNLDFHrLB@QjUq(@ zElHA;w3H}mNqeCsq&=tze3}@rPff z!Ogt#kC0!)=wmF$WBH3e)p7I&H4iL}I|2O-+6nywjCph2HRLJK4$kp?gaSv_PndD{ z5gc~*_|AaS;kF)y_B6UeGiX|fU&!2?zw4!&;X^F!O`F_RdvM|RoaykRCgvpK3xbv$ zn0s*20OV#E2T&Gc+uPfX^v@%1WS}KGt=_7Y&^`0ENVQzEJwsoH*`4~o$JO*4>U5nN z_j!}>F=g?c#+$u)TIJm9OZMy=zOrIt`L_$l>l$A>YHc!VHD#}Se0+Oyz(vD-c%t~! zZvAF*aBmXHw3w`T_6Z{paF#;c(Pp$Y5?v8)ggpOiC`N#tjpaeztF=K=S%mqM)3h|Z zIj6e)d%jWZ>FxFY;uSd5zML~S(Cka06n1MF_)d78EniY)fJ^!a#ekL^y0bvpb4cIiG^Bd{!Ll%Wsnx~P72_#rn6Q3rL;Q0mMT>=l_4p(qB zB;7p-2+rw5P(BS%%abGy*8zBM5Z3Y5s*OLV0Nxh#pU^J4CZe=M^Z^FY8FF@jCG-{_ z5XoWjnwYuw)j$PIZMa~+hLe*_tPfUgpBzGkdKBuw={&dU67HrsPcD9Nud{JwvB zeBx(^KbiAKVU-d@sRmmwA;wju3z%j*w?9VxX1589Kx9w^+5{#1+`kefjV)ILP=;(X zoM0q=!T4k6nI0IS>j_K*3bTZmvsft~!vKas0ReipKN%{JUeLpY)bT0*b^sKM1Of@r z6Lo`-0jg!4{uQSf@dA<6iZ2QmZF~BV2vXjnvWXjg6vk0>c?DxO(N<}5dLZNHXQM!F zpglso%RmX3G2~~!3Trgk<3QTW?tIWD_K+I8AvW|jH0&SDmm7^Ng{BIFb|N{_K!W@P zX+vTtFz^##O=a=|c+mzd40xPb)z`>h%p&e3XhZjJjuOB{cr~P}fXym16cca_I3)Nw zj_(v7v6qij!xR+^!y}+CxY5SCuHtS8;w~rpIt~f%E-}iS)&7RRe^)9M_IdRE`#W%b zUjt4-(mK_o(SWG)-8t6_h0wfZ9kt|WUX_FxZs9#WyntcG)>J`aUS zb|^0f)B79qKACLidv$TRz$P=Hrbi4y7yF#zQ@I*^7vz6KIwkb zF^lr6gcxi5#%V{iKrYqd5+OYh;D=xnv#tL&2wZ=HTslUAUH~TrwZ1F!r?OWgDIZ7^ zwC`)!4A zkp-d7&=p$(!Lqx(#y}5^7}}=kKaCi46>mDv{3@F_Q5=%Ua8K->fCUWr>a&d_+2EgaUtB_=&}`k?LzEEsn1gyJQTLPVT!lCn6@9U+c{ zg~oqQ{7Rla{5apZ**^pwVL60^kg!*s%Pzjp-p=R1~MHvXy!gNp>OV)}|OxFbJ=W#H@j2V$dRiFeoqY_fi`i1Za-K z3bA@@>TMtmu#yOk|IbPS#tc0U@e+_656pRP7a&$5L<%Hn#6Sq(2I{Hy3J|e-pcNdG z<~^U=Dgy-xN&E>3TVlfwVy+?CC>cu#s+V0DVJy?XV&4OlZXnqLsElkr&mET5YP^7;xbp}Us;&KMZV-PGK@PFX3kCDlQfgwZ)C**FRPKS(jP$Lf_n4J= z6A2W(3JzFcFy$S4SBY77uU%u|)r|K7^`2Bh1u*x6*M+ZY`Egs0Z zL#o~fLN9krc&6twD+Y^$yd#$1=;)V5!mw>Y1q|`KiVAIB6Uchvwgt-RMjS1CNSFbD z_)o?2g$RidHf;wA|)7*hT9kScy8mlWQLj}W*AyE-T2S#Z;tYw+4HBK8BwM~VGl zsBMiXlvwM2+Yh#8nR2$txz6ZEUK+a#D2`iQ4OI=Y2!w>z-5+Nf(ZukIpeY#;hY8tG z8bes1fsmo_HB02);k!EmCj%B0nfq$Quv*A+0($1Cw(_|!{@A|`rM-av07H{EpZ+ZG zqH7d!lFvt%B|Cg915A(P<2rnFx)ntEgCl;rKGP?n9PmQ+++xymsCaw^*+Ew$hdF9m zyKGx84K}>Eraa1&c2dXQn=B=TxBPI3Cyk#+F~FFgTtC|3DhWFWpZ~OD z6X6R9(ng`^V| ze|u=1*gaW9U3+TQBUxt`K2>dHW!ifa)ZJQSh{Hsb-Kt<{g(-|6TbQLj?uc_?3icw~g^1z__aDOse2BFdj>g1Mru zL7bBSBj(cVArc4k;hA82dcr#B0Z1YZ11?CoIl0g&=eI)vbf|7ZHYNkonUh&_-7Ymm zLO%eBLDEE^|4Fu2L5v$J9g*RoR@^h97eEG}Y#f#6v> zXN~<6{Gsr8`(F0KMtGn?j1!d0hUEqS98QpW*qvK3FHxEMy{N-233dUE7S3FhC>z6$ zva)&>v5OS0ZdZ_q`C3vK{Y-OoTNJcyo{Agn|jJ@Wzkh2&*1 zvWpiNoBd2Dk}7=Se(ju zh>W^_(k+{4szUQ{+(4>amZxYTtuUMKkgY!_G;zC&iJzFIeqD60U~5CSNEo2o46TF; z#20|&@*Fmi>oh>DKb{4W8Ps=4Cj1~#+LKB(O6Gbri@+{y8YdSx`2qnWjTDT+iC;0% zJws=Mq3!O?C9R0MfFC@QaMCy)=6OdyKZscF5Z)1$pNxG1Kv}FtLk0dUE|ldQ`<`1Q zD-Gm8a1!vhBZ!h9EEYHSjPkf1C~$mH^aMgtws0kxlrTUp_}R~PO{*S|)LV}#gpzDD zf)WNA6S^m`#AFV4*4z~ehM|Qxq+(2WK^AP;$}LffT(EwZZ4evyR6piuKPQ;MR$ zn3TNIVvVL#+}%l}4Z;z#sE+XKC_ArTeqGjm8E7fa2x5Sp)AnOLqGjT?{?kUa@4;{b zbAJ^U)B=v=nV^3U4{3LAGE-c_QI)0(;Xj}brI7ri!rAGe3kva^&QO(+1{{sa)aOv5 z=7^Bf19Bu1c3viYqRVxr%dIpO2bLx-GtTYW-FOODLjlKWQ-DyimiQP-ij^!xpk1oN zzdhZ72KFR)Eq;FBC5k6-xDr4B^A>e4QNPcTJsyLDSTbagB+{yUU~9UCF8R0v@k6Hz z)+TX^p-QU?8Hn((hDIUZDEFK64LoJjLzHR;IoI2=wWKGDPKl_vsj zbF6r9y@{F8pI*zmV#3!d0S;uqF!0F~o{|SjOB!0dFWC*7WG=(5@uhSN;`Rva)T&u1 zde23c*Ym8=pEW~yzPRbXxmkKgclFVpg7mzsL#QerH6eaJ9%ZDF=)A!zpgO-RHhb<% zMjn`xa2R2H9cz>WS^#`E(WuZ6&F)|8rrn8jz{%gu{N|n7;iN4Q9YUW4P4{m(-kjG+ zkdjB-mH9v3$r9a9p%x5fgo4E2&d*Bi)#Y|evr%M18heHMu1rS)UNenAz zGL0ysVZt0b2mSL8le{)`3aH$c8Z!3M_-|O7MBj-96gE`oJHZg5?<92`t>N+XnKM^T zQGTY+>0?0NOIXlSq5^T_Ob?kjz*rN(NRAW|Ycaw!!qLmRe0i-cJ258V9LjLAAqH#o zo1{A&IA_$&&7HeFXX?Q|B%(p{iwehGbkn}@W8&nfE!X1|xT2KM9zQl}YSUGN5*o7x z2{^q|*=6OxiElkr_Iwh~x(f!?jCkY~d#_i$%dMPSPIa;i_oXl!tgzl9%iz{%45K&>N-~Kr=w7fu$P#!4291nCZ?zHCRp~mD#*dsz!f+B_ zEs{$z2>Xm72n6X^;|*@P$2diV;bKEOei;4fL~h}vEc7vbXwJwUe4Ym*E|d=hQU<$z z5>AV!&9fQ#%1~L&@iJ(15gnv~6s+zic2wbU_?Q8Q-D(~fYXn>%iR@QcvE;6Lc`f=I zo@DHaztlrwK+{uFL`j|}5$0)j3r>s;B(zNrp8t?{;jCiENp$4EHcocF4%)A#=9<`S zi`fe-HxZE15ZWXiH*}yx<{vs2B~3z6(oU#7m;B(0)ENcRLJ_;yx?Sggd?{a!&4vGA zQhru3ikKW=M~6}7utrOrnJ6+U|Mqu|a0A9eKr3J?x~z}wyHT+TPJr9;7V1aVaO2#2 zgnCM(`BB{9^@t5XN-D!(h{+(h@kF<>fiHYdn$_K6vP*N^Rz}``qZVraYhU0k-hPfq z=7;T%n9^O_3}GZJ_!a^X3?!EtFx~L5l>;aDMy_4I=fjbY&rPY^A_Ufp@4XVUweg=6 zzo}be`8%O5{PU_Be9dWCaEPALK8jT0AeIwbU}k7!<>X9&EzyFgn?eC7p$#1!hm*~^ z|N7=~G{{i%k$6K@5j~{h;Z`U3U;;{ibK0UL>;wogfl-KClVH0I`dO!T;EUB_+?E8r z0@H_LGe&XrIz+lYX$87(E@Bo9{_?>4a>&{FZF+=$<}$8mjRz0mCW#N2SgkPa!05CQ ze}X6rfgOl0d>2Onin>o}hb9O!2)y%MZJ?@xMZWEQ8?B5x26m)3*dv`~4k!oVD=NbJ zVK-uD{}DxmCxYWbpLMITk1>v@%3%=`6I|g0-PJ^QX5n-qIf|jD_}MofXyO6f6A;Z4 zBZqtV4H!C+UxZo%V=nHmA{=b4*&?Svs0nP^3Ji!dTv8g5d(ir70%{^FYIy3`S#;!p z%K^8454ypQ!IHpF5Us(5c@9na%|E9VY3!m-5QP`|K)n4UhK4gW>7egXa5Yfiy)JG3 ziLQ16mq_S^FXAG=N}}E)+&r-~+sJ;C!zM$63pEd3hVpNd^L&50YWEQdC(UXdk(l@V zc7?}oCFq=y{Mm&=ljH)UeAX4;wh+N&JGA6_)z^CtME!Yg{N3|YI6u(y{CyeR`7dl4 zENq1Y6hKT50HBJ0ZlYP){SL8h_|T`_zGk6uBqYup_7k$?=W(rsBT#Y>uo8=RdK4-U zT>N_!&5Y(;F8XiKVTu6HfW1gC9bzL4wVh`Aq}o@*!TwXc9(P8ni?}?iN0|0IUa6=0rGvP7)V`s%kwkF!a_5 z&W7R)gd?6rr==Axw7(243KLM9f^9+oTaFhA#5oO}EV>s$iTL0HsbWW)hrtTr;GF7_ zNSZ}20%O#OCTiydZnGg~TDYK^jS$;HWMCL4A-fsbpV^6613;N>KQwOQx_KVezPGd`>OUzP4%LmrJ0Zcukq%)h!&bYW;t=?e6@bPb`WP9hwUnv< z9zrUxAlCu~cff{SRZuz=5ABo?+>Q7*k62ig4IDX*AT&%UaOcs~)z;LI5N*(I_Dz>! ztQ?!V8?uFh5kc|F!g<@;eNV4meIvhpPsXv1&V)RHj)V94{W%^JWDTc&zoW&TE4-H_ zHp=K^_Ua<|hB9F8XrDr>{B8UnBc3Dse*+3cv~`C_#@oo= zSWP*DAM+FD#W~ufLZEtu&hsZT2)O-4Qobh8uM%$pSue2toP>}jbnEHp8>_cP!;zq6 z1z}p8n!>au$#uh+0^LGqB;@Hi(9|J%z4dNt^*%C^m6FK2#=f8m^DQ8Jcac!~_|F^{loFyA3K#&R_6 z2QP}*{DR^4ucxSVoXZ8^wMxBX%N|%rwgJ5c!`Ru?1&O4d_>zuQjk+6Gl2CBy>+vw0 zdoDzcy}~tB`v9^%sc!)sMQmK}2cCuW#3OS3WWL^s1bw073aoilz9m9>pVb9V$35n| zGV=WRhI(83?v7P5ZP(pD5&Y6Tm-yJN@MLimZNgdvDdnmZH&f?+g<~yZhVpG}cnS=#HqM31& z)DT_`!mh7ka04LK3l|gLf#z~=hb`ryGgPn%y88NI^!othv188LY*g?BD8{Op@M-cc zfvoO8%d8Dgn#}fl2wKAG-K?KgWoe5$&;W~kT3o4ce-Z>Ax@P&!gi;G`EFEnyAN(AhQw7O4TvJ?OioA48pL38y(rvFLS9LN8#e64Bv?&! zTZV>%$qE2yNdO2=i#T%g#q!^30e4~<{XY_kN#+c;E%=SKRhW$-apU#xDx=+uM?tJj zFqW$v^t^EkPB0R?15m6A5X2E;Q?z6AZECST9(josS~ zi_}lsq`lM5)~xN9lwInSwdbIJL5_CvPv(34qfCdD^}VgPC)w+v5hQi=^-z$I$wb)iaquP$!SN^<`FDD8A`8CS4N02nMA~Y07o8yeO$BR>ciB>I#h)5a4 z&SA8Rg<6P&hIW6>(kjAM3QsD7Xf!cu+Bx7Lo#xVjerCQs9bi66P+bt@&Uce$jXC+*3`6s z$_XI}c9P?u<*d+Ww#4awfXuINbK9M)>ZUaj%d8da6}{``+vp&@S1ZR;Nmw)Y23qsK==5=KcYH- z_yG5l`SmLP#3uQ)j;P>xNZWFs(gGp2bTYZBTtfTCBsGLJl*i*SWF&42X~`WexxB)SjG;@@K=k)#H|L`FKxbg0R-R8v9dJ_Mj3 zgt$7uQ-=|vKuFR(qzvI;-T0`sCz&Ka7EZPfS)^1KLES?%r$j!4t`=jMfPYs><6?w; zF_9DuL?j73QrZK-1Tm4!Wi}eJJ-dg84C<*xVW!7)JKn9+4FogLY>y0n09F9#)7|BC z`5(aw0&2xUa<05@bfORF+eCj_TJWQyJ(Hal*zf#O3(W20$&`uTX z(}2K}h;?y=Ovvpe=0MBHWe6C(5FGM;F17|}j4<9K3Fb%8k-!5QH|fLZjzyu-m@^Bs z?m?D6CVW(;;{co?ZbYXx)p#4Q&3TQNg$OE`nEFvUis{}h2h$rUl1IorKUjJ72DvUl z{(PU;9zQ6}GA>c7+zFFzb29&(Tt-j0I}69^6~zv>Tc4vpBx&?>IV=U=`@Y8>f5jpD zyx5HL&53q*Q_zZ4|G{~iQ`kI@g>d~o|H%@m3N1U)9hqGB>I6}7zj$(Mrn$-SeWLX( z>Je6={VkksE?g+Vx&#$>&CIt&P|)B~4GTR}(->GG56^`&e}uXSqwyi&XvmP7Uw4M@ z5a$5vCh>|Te7$*#m4o9k$!rAMX>G{#T$7-|K#Uhy0&9D8S51Vt%zmpT9Wz&LXgr#2z;5h`EMbDB?3WqkP069Zjx!BFV_Q zs9rPQycFiN{bp`tG6o#6DT*a-q+{D~q;w&!tsuPuu#JYhEbg`yIJ5CA=G-jV#LNj` zWC3lJeP-tAY<>945Jyh##}Ve*f?#uE&FAC0^242y6J9vgDS}yQ+QW$kWY7uE1 zI9TL2zB|yUi{NZ-IbG6V9y};D5L!y+<@Zlx>rY&}0>3UUcq9YqZT)UGCXQB_nUq7F zj1d{D7Vz)A@vM(LYgQJsbF6u8Xgd(M0N-hY!0pmsEW}qxOfzIY1t^+pkpJV zOtZB2us!635@XRs&m&yu${0s{D8#J}v5V^d-*%;E0}ue=Kphg1TA%_56gp-2Kj zDeZe8x10SgGP~>)w>-RpvmvF>@DxB}(=5`|0M>(1-k$nIDJ<)W*2vO@(Ud4iO zbEa{2!iY+>1FE=qa;%3i*jWNp0pvO6PrgYo0L{eCMX^u=BH=poBXShG+-(Aq4iYb| zA+&Z*nG8L1X=$mEz9~Egh{*WC!TNIrCQ=eHf#x$aln~q`ixPwGJ0{x0Sr{U~o{}NL zu8km=KSJP<2lxUp;{miKh}@*elmeHZ)dTvgV3=m^N5)?hUKnfdpid>?fCvNBG-!P@ z-i>naPp&lquSFb#8ZTENzlO-@M}Q@|IU%QnP@NdK18?2WhpKO+i0=_5Q1Hr*>yaog zX=` z{yyj&PG6)|zvub;8x%^S!Q*p1#w8+l2NzQ;*8I4|_gfzGj1TS4XrmE{N}?LY(*UCraQ>* zV_Wm{zr06A9-N?49VC7ds;GJH#+g)c@M>)8Bsmo~X_;v7wwI8|2HP3XZ*u_PX+d{&u$1?tew(_g+7j)!)~T|JkNsFgA8@ZaH!j|> z+>AvC57;3Tqz7x(oR=>c)7;GDFGI;GIF-A*aLtN0ZNHxj9!|TqNu&HfPrf@^$bdVI zkD{2Y-t%R6W%o(w-hapSLiSv0qyWVSNpK<9b1jvPFGRS1q7n)UJ{sY2#zvCOYq;O=EZjrMuwQ zSI!ID-6Kbd0wd2c734$(z2`yf6^tarH3K;cps256(0F2=eMUsRxHljn$owJNi%6Hv z2PcNJk3;QC0_nP`aeE2=5|KrM!3|DgWMn)7pdTBbbhjkH$Iqixi)QhgieQ-${?JlrzVi!z_sDYd0OQS!kd*=`j+(Tq%AQ?9>UOte(cK(x5O^q>!?MGbgX9NrzPQ$cx{7 z=6qQ=F|81{j9Mwl_i7O+wUywHp(_WQh z?7cZ;K>^MHZhx=w)|}Kx1+R-i59{%UyC=xb$2Tw+xJ#lS32T3w;pcIZ&x(a=v=#bQ za)|~BHh_Gzuf-S9D?bq%bcP>ds>!GsE~8x0ItDVLrPqzV1{6&-NeoA3knTUSru zW>TaoQ};rIt(#`f*r+~62|DP%^p12lk8?=dmJwI zgDu-|#>Ul4UMJl+#G$}|jU26U;Y^Z(RYQ2!;5%;yO7d}w$hF5yJwpCvzw5q}<@CE5 zBTFJ@t$$_J@_D(fu%`5d&SCbadM7z(MC=fKV3KtOT+T1oo6OF^%5@$GB9vfGF zOrxqA#-K%LZ+M|xH-XV6Dj;j?CqCuIcW^2I!mCw~TCxi%H|VrZj%FhBeNHR_g(V0S zi|Z|Z2$NWN9ar#?%Ny&fFI^>fjJQq|2$8oZfa3!dXO3zr0K!jh@p_>7T!cuLfLR;m zAn4{23KY?4qc26DHucshdAJZVk%VkiVj@h#QpB*g2cu9R8aA2707LZ20&bzZ%@K;fmRc5IlOY4FAR3#0|sQIn1E3x zXKsBWV)`aT=bghn#e9!8E1VRUWvJremT($*?{|X4nA9#WT49+P$`xNm$T~8z8s>kn z+|0wX1F*f8zJ7}cH~GdCwrRK)%&1ft0>-3rJG#?)Wrdng|EE=f5vS zD}Mv-UvHOEQSz@=-h6DpycxNoxq=bGH210*Z+`Lzp_2i*Anp(PF}<(qiMu2 zkZk`J>>DwGK&C?4K2TfTx-K*00ak*zEo2AiIwAxjmnIez%*0xmnwZ>1p8$qgaQpVW zz-2Qsp!YFCqsKQ($Ohj@1nBT!Vo5qf;j=rvLS`c6&}RQ*=T=AUV!yRn?qp3m>{O@$ zcTe+#b=jUE?)!pE&I#k?>qzDkakIcC@gXlf&iLNfCD*D7o_gN!rl;7My^EsTy>NJH z)!#MYXmIiMKMV#XC#sh}nLv)(@wVTE#CTspqc7tIXa$!$kjn)SDDu`w0sJuf267>l z;nwyQM5c!xk;wQ^7N-PaNs3%F9 zmKXwv#nB>=rbt|XSC<_DS`Z&C|Mr7ekjNq+SSK7F$Z2}9 zN@{k)Jk&dZQ&rk!s) zW@FYh^Sp3bw9m&}3g?niH?fj@&T+$ru@A`1x05Qjla}s$Xl*Oi`A|Sw?)|8_FGlxq zTB(p5J<+C4fmc5aF+F`LuBz$&%O#@B-{D=@*{{26-tLXLJHF8{uW2s}i^FC9W1T64W1b$p_edb#WQ$>KL}-xdS#{4i~}j+G%5 z=Ns|@&)(`-rFh`LVz8+Jw{Bg=8EPRBYCE}g+cqBDJdF2!%HDqU)eS!0o)1Ab5+0@(Le-F;1+#j1NkrJ{bIwppNnOXEfp|qt0N0|I24#MVUS|cP) z2l)2Q8<9B;Rneu{FO|R43Hxy|R7c&4f3|g#jgoRAwePxy6tPRGm1h&`dv|OLHd0r2 z12`HDg+%L9Y${V6334*#S7k!^VzgddVr5|zjImK=>9w%!7ll#fpO>d(KhWQu&sd(7C-uv$YY z1d98z#``;Ts3r9L{QNov7q=~ESqAw4c~z&ZWn~b9{0BHp2h=v8kgN2PG~!P|U5tDI zV{|p&hKI#89w`BkWi2zEC>koVd{Anf22g6NT3W6L8#?(6*>MCs)7^v<+Z_V5bUpr+zNrBH ze@;v+buq59tFx3u$9fYB2**}_Uz;o_@+%Obp}M|kZ4K}0^R_6iwM<7;LQTTiGiSs# zB$Skt7VO#_%!f}FTua=OC;d+obvBklO~lN_#pvuj`GP)$NmFPAbbBp>6MHEkvgbD= zRPf!9T+wZ*{5oFybM1(3vC~nWy6P#9=0}1-`&VywWCrw( zNATi+NWO&eg?;ElPD5XlufKs}Y;Nue-6|gfRmw~mXs9=SPc41Kp5CI(Qhn3dmiIZY z&*{GGFf2S%cQjEofHlN((&B`q!80ZT(u zj)U;ko%dN{_{6U(@xFk%{5sA(#aNA=-sQ`dtzG7mg1Lj=6#HrJ6@9Fzp`HFWp zJ7y(yp&%!Rg9i^PU-R_#j&p?wk#FWbNhvJbMhyz>e3ThHbbV`Ul+EgdQudlKavol8 z>Q!fF+nVEndP-3|c8mwCfj^*t(Z1BZiCP_S?Kolt$UT1J#*HRiORKN1?<7~?cbDEI z|H&!c4^6?MKHd|%p9I%0b_n~=k`ll3>GTS&`P1b0kJWUC6V*9oX?UYqJSGjI1CHV3 zd{tSw2aRA-QdATLOh_5}sq1X)0iUL(r&IB9^@F3Zi@uhXwbfgR_idy?WT5^(N7S;_ ztCzUCy6R`2S%@cZkvRoK;T6gNFg7tt+Jy}MtF|$Bz=&7@@q25P{S0!8q7e%q)DjZsHe;-u$8|vx#g7oP?i;N2i_%!>^U=q?0&)Yi+K{lAk+D-L zU*Zc~AI7Ml>ItJm3f|YR7sF~RBj?J+&Al{L=Q`N?Zq*zEP0g#ItXCuS!aXr@z^(M4 z1goaDwkiG^GR05k9>*#nhcy%)e$4Ena0G3`{7>pzPEL+(v#$~_A}!JGu~Jb{P0Pe@ zVX5|g06sa{+R~DNlz`p4l$7eu3k4C^7Ydx0rqIAsOf`a#gp<>WiOGM*wG>wgDJc`2 z-nhuPHbGiU>;+OSm{+g1e()+z1j|?B@ro?X78(!T=1YsBS@<6r`l!`fvGg7HP~Y?8 zm}}yB)LeXeBfa2vPUCw9Oq8v}P1($E+~?TP-g`-js`u8M%8&6oQp z?#TP^R;tsZ$}p#MkdQvd_OAm;NlAppeeC@b;s5-el9@e6;!0OX{e@p+*79;}=cjIO z?g(GI`s==f-f_<2cb6dMRoW2}Czrv4W>)MS zC*m^M+_sZHu;+Dh-ggF)0bM*-^Wy{n)?X$u2KC!yWIhjMws~f66A+BK9}^cx2W7U> z0pFQ*5un}6FjX^#-h77NCaGtWOBYaZyfr;NzPOUJYVjrkfkg;xyR5&tF<})Qjklj) z2^?9T)5Rrx`)q7B{a!wx4Q!hi(@^8$CkCuBx}{6MibZ(8qTtNjidHtMe{k@|`HvUM zyvc_(deFdt3q>aYH`+s!x4BscO_huc2mTYDDcb|ONrbF1k^>RdI^PKfa_g2X78e(n zQu^x?501vZxO-XItTEHREySDI*vfN#K?Aez)zYQ0BzDFhlz>B)H4fhpiIW!>!Opy4wJ!U22e$_(1eM)8|7M`;Pc?<~@mb$S7B03Cz?q9*o#|AqXH&Q1?pSHBorh_ud8 znnI&@kOWNw`(DVF6H7t<)+LuW`dFCZU?{=!MObBY0y;7d%yrui_0l2i(;WyMFR~uB zgYgj~4aGrb$J3Y?K0~LT^{gvapch}f0jBOoezeIr!o06tdx<{n*zx0%#;yw-Dw~@* z(IDiU*^|u9xW*)vP5iEkw#%3p`BWHfV2s-e1EyiPj}JY1>9U~?i3iB|A{*Y~WOd|K zM^sClRFmD0x|T$A4Gj1-8agxLo7;&2EvXs+aPgtm?ktvAraJnbvqabCa=MqbaAxT`2} zUHC$P1XxzB@*-iXlkz_|qLktDMlWB~*4EbkW8*G_ov=f%=ZC{2-wCNb*6*J#92^`} zzINT|Kterldr1kUt^Rf?hO`k?Wlw)4t)0Q!CfhP>H z953723bi{FT&CFY?_JeRY)Dw;qok94W<4kzXGvS_VqPBnWn(ohu8@Ws-n4;~=L`y* zI@cWAn8*kD=XJ>SQ;^EvXFvCgT`K82TQ9+$-g~~DL9(ALEm{^v>NRWDyuByQM7${V z@aunuJQjQOO^T}F9zHb-iw$*_4!QKY0DP$f_Fe^w$tEEByL;k|mS<-02$69RLd8FY zGt~^aa+|b;f=H8t4V;Rx9NAv;-4h*}KAjW_Zrk%nO3Kb=hwAGkM0M9!v9a$~Qlh~h zcfkDw40z-(YirZqxEQI5M+5}8rLm5@>AK|{ezY)?d#do)CK#m{Q&s+|z6r?49heyO zihcEq7A`yn9+{cJS?Yd-ukT`Dst)}v*~>wl=S*OfyNC-M;Fw)Nq;w*w%h~T~#1~_# z9etO6sieMs1zJZp$h28_`F26kvliGV@rjEs1DKVPp3e1vZjrK!i#(dAMIJlv zYY(-zyFm@{ZL7QrIze;@NSve8yxh50b@hG55Q@9ow!Pj?k9Wksh*syw>)4bDVdqlj{1JeS5Ow(aexin0hf0S;OeRK=ZAI|dk&{c zmDn?h+1~k{BZS(uWqa|r)UOWNa|`It57Yb1`YHd+f8dwsfhf9mV=jfz@pW^Ud~Xdy3% zrJCNqXBQLau%4NjNl8oF1yU>tARJXhL`3V@3OX7#*>fAw$zQ?3_zY^t!nPr@E+#g1 zA;`-e`ueMpCg%%+kz39D;3XWI3*8kdiCT-J9H>Kpk6;4h*e2e~n5y$c`%bx;@--Y1 z-ri21A#uLBKi0_se0Zm%ox}s&W@7>{ZlK@<4cZDa2a5Hyekn!;bVTT5D%k(Ifgx!B zdW5T7A+wQ$#4|OlFg+A8((eMMIrj#>c3-fE!|5#OKTpP0A_ zhXUN?v>1QrJgXnnDk<&4^;ofn@A1d>2B6?LBtLjSgMn}FzJ2cdV|$+_CMSzPm4&{k zrSGipu6>w$U@)ag*HzHIdG8)4QahH+HDt=Ihc>X{Fu$rP)6nzi~XxJ-w%@Q2t;-{b$HPS~CL6&6+ap){wn)<=ts1jArnk zzIyY|Rv2>8Xw zOBQxn%AYf5{^uI)MD_D+_~0iD(!(Z=q63uHG8-e3=_le%P?Mt)3Y|uU)-9tKh=@4` zMKS|gODv<>1ckF-mniYFkl=|2ugHU&drc^>WnPQ!6cU zqVh6aUvuZ!Axv4HaR5|Vrc)?E`#pf`n&96pWxP%sq_`IKoQ6o365uR)frZVpjQ!5T zqT*s2pyo=Bj$6r0LwM|aR}=2vr-Hk_3@5)RbVS#=<#LQ>m-dF;z02Ox(gKy^!vx%; z^?9hc=yq)E%(|5bx%|DDo#=oIOHN2EJu2$F{uD8 zxKhbtue@<(N4>UD37pqJJYo_WEhM($ZVgmGKqT;NgAI-e9apOkjWy>x@z_WMVvVw<7O4As|aOy0o ziPz-CmPLDp5o7RaBHj+V2bSf_7ieoY?zncU9pPZ@=tgt;t`~g9zQtsW6Y?cS!abm@ zz?4$rH5uYHiP7|2a`^L-MGpf~nL~X2Sl3tkK8iZzQz>ql($YvflVM)E^w9zEMH>VK z#o}x*Z-AzBM^|(1g5i_Ayw#w+cEK^6dmJBpF*2}?O-yQ2^@3ZpAwLl_HoB*f9%< ziSh1`yLXQP38rIU5ZACD*`BE7i(sF10s?$sjL;6YjM-gKd{JiT=BEP2=nuGj=V~y(Tg@IM+aaWD47X!QgKKtUcxkd zQ?Bb0&JXT%3=kfT_276H zwU4)xl}9q;6IGAWrC0tOH{|DTk7D84fB%NJw`YiQ?88ZAG%Z6}v+FtTPEAdT!b+qM zMTT}G9t0}OCs?|{7Cl=ovPLSnJY zY!wr=`ZlQ@IIs&T!3a7cRWhPueY7Ds9Vvo4LbH;P>p0AtVg3H)^lLg3J3A)e7+*R& zd*(nb4vr5A!~OmJDUdy>gi6p8ik5R+7|jE^ofjbO#gP5l^x?ypj7x8XzdsY6vqL8` z4uOu_b`{+Ok|BIvnx30%C3b&wys+dE4_q$wPAj6@G4S^ zfU&t4M3Rx<$o#xR ztn7U~0{1vyL0BMGRR^RQtr-|K+2ougNL$9rN^|z?57L%dm?0>kMi^Hh>EN&v_~z|_$YmUIZd04?GDp&3n;$Ur_i_%@OitxDGjLnJAk@y-xW1C=c)~lc@!qHKg>D$`QWzXRNm+r>H3OgE*aC4&O-C& z)9)YC|3|TGd{DAI`sFS22fL<*ePzXr)mCrnH(gLyN-M7GtTmld9`(L2Jd4Cu*d0R$2pNFPwc@(nDE zdN?CP;0_{I;)FL96}z$cu$6(PF=hbk%5P)F5PnEcZ!z#^L;s#npWL+#FD%3d50KR) zm)03MT`pXLEZE~!0-|xJaq$uMC~D<29RJfHJ!J|EC&x#5it9qFjflAkr>P-DC(Yj zi%0HDNWdq7%D4dRlx5~dQBh`)R_?%xSa@TS;m;-ADM}w#!9gxmyd5~FAi`OMM@n_$ zrnh$lW{6nra|C@~2N1Ug1sXFZ75-#gLPFouM09UZp)<3y(_2~fp*Eh1ueStoKuy5Z zJFqx&0dnwATr0ZMjL%;qUI`cw3bne9jyKRfWIm2KuD+ixxAEZV!@d=;gb z1Nb;{%*59h{N!CBaL|lzTlTUeN6x(J6l(n*vqu}9(sOQq=YdexvtL(N1^xK`@p5FO zL1A%=o}}cNixVf8y^E;WrK`Kb(rShJhPs^2wwB!05j#{kltm)cKFav!d393UUUkL{ z*QlMaZyNmMvvKuAcSh1w!k%VYzC4}-)lZZz9Nc7ifU{a%aF?&jLU-nrXJS|Ky(IH? zh8-Jtel-@tg*4GafwG1H?Dlq+)Ue1x{YUR6aCIl&$rt(XM~po8^Hl# zUh%y&-HMZOP*3~O^CGX#SFa!Mut{94yUEzj?uHv>k<#w$7x#mk515)R);OG?uCDG? zQ^PMIDagahfIqQRK|z-MiIU^p{0AUR-f`wTG}xLkkN49C?Z*x|yLX*bh(!5dQ~Q2%XgObUBdj&Gh?!j#og=xU^$Ty23CkIMSPGkVCL z#Z9iFy3fC+i^|LCaoUuinvo;zm6>>cCj?rglNKXH6__-DY0B4bdwUWQ(^`3XZj}C< z99UUQAgurbpnUy0ExBRoG+YH6SQ+LU6zmjvKVfD%&>`M9>`|Us;2_%#39=z4j%zfq zpYhuvs6NP~`Sr2y#Msza%G0MclYUtM41w$`U%%$GYhy#$Pi1+L}SAU z?3E1c+1bC`N_W&9#=G1KN6^RYx?~ly-M+s4bYT-8<{mJMir@pmutn8d<_>gx{(Kp1 z&Q(v(moM(m4(>yFh=?ZzLw1{3BL(0UH(D3eIus`!{418}Id8`BDenXof|K^gPS;th zlP6z*`wIYQYx^GUs3`J)`IIEj;KSYu)lo&lIG3ylV8mW|JlkEP7ZDEe{c8d#X}z?1(OVf zX(&z2dT@$HUti6_ctqo*oQWwDuW&+X)BPn`SFfJgc=Qa-@#Ctk()=8k_lqaoJa^== za9M%u3-9W@tk-9)PXBnr`b8m0(czd}&YOjOYkE9S-rFf^n`Bp5a%x`3UHLFJrlYPWT$b!QO^I}kxP0x}*MrRq7jFBUKhFE@qcaP$k5fd|fy9RP zHM4t*A4T%>3HF?PG|)d#k!lg3AU|o%vvHH3LUz}cMX^6SMTR;?di3KoxdiTri?JA2 z^SVy8FA5a33DwRteHeS>^Z7#GqJif1uiho~oYM>2R?E5OBC|(*!UE?X0`}eb`p+8+ zU5+LX3MkyY>@OeMVAEFSFMO)IYPo*aO@G{Kd4G`iP{Vchk3ZY5?0s@{%fm{sajzFm zE2*A;_`rc3^8(|ug9CxAmU6uWzPqJ8kb!wr07VKJ_B%06fKAGAO4s$n*k}Te^0(1} z!b*nUI~n~tFZLS?i#vwZc1)jW`%^!j0oH@SS*I5Nw7^ShN3B?O8Few4C zc#0+_yx5570vP#`E{C<=JN@w%@F^7fu>;hX?yGylP>0|C42UJ`(xlYw{jrAkpr<05 z14Ll%v9>0VO3|0;X^i5}%gPo(5(Y5yvQkk%FrGrnNmMC3R~5jx@SUHYK?^)_qt^z=ZJ6SYNB zQquO>y-E@^+ZFES1I8ml3$*?~f>70o&;oFEu+1jSz4%bkFhiulsByKFFR(lX0W7RY ztg2s(!3fu`HL@>pgLD8?nxat9PNAUBJ+<%78rYZMu`csb8n3LVV8pv*ihO-KJBwb| zK2YMaH|=%(#FCkQr?)?z?!4{maj0@1jNEHDRLinYE2TEBOJanE}ut4VF z)p9}l>~bDd_mbed={L^@T=DddlU>PA{e>$_j?yieig)=EjSn7mN?;lo?D`=beWR!TNJ*>>aU zQ+|W+_;_a9K`jfHuZA`O@jfikeBvr6%_TRx)=0fD>d#$%ovp$*VO>50lbShai~P`` zZQr?rO#4;VSWnffgd7bWJ2{TA< z>1){HIF&VNh8#_X{^nr!!WoCn8z+B8^d&^jj4X8-OUtdY$T{c|tz~t3!hZ9+u<{bu z*>R<*UhOTp*0bG*7NS$BZ{KR$vf-s4=Wg__5@Uk~48jQoms|ulJ}Db7bWPK~gqsvr z&WyVbvA4{8t>A6RJ9;%pctMJ3nMu=!&~eNBqH63aQ6%}bu$Z>+EXgz!8ac!q8s z__?cY1$Hk$CG6fF)z?on;>pq>1OY~8Cy&%lOzHa;W)qtgH?}bcYu%}~W^tk)}TcL~3k}XI_-hQ#^T&oh;GqmKN zhKAbg`XD|j0x?9S3}|Y3(SGVC&e(lK{s*ZVAdXX!QsC(1G}b(^N8HZN4w35JuPfE~ zm5Z*Bk?oC{xP=42-AJCVUG3pyq2%vfAPH%lP zKq(-6DH$2uBO#cXP>g+GV7?z4yIb2xSNB?_{NxfsDE1EwERx$CoeV%50UT6wWck~V zAGt|^&dtqb;pMywxBwmRO$_>oA$ryM9hwAPUEY5*3EG#vExhLov8*ho;4+A$=%F$t zR1KiDUH8@P-#4_ja*2tF!H9SzKK}Gsh9!5P@aX@LzE~}?w@V+mGai#1^@0U{tnYny z0V%+YI5)fjg~{i5Q(eu1AsU$uXsk}1I#o6KE}_Qa^!*f;eMbxm!IYh{pwbtz-RYt4 zvG8L9GyS63)j>P&D_*>KvG96qwZ#Y$K-h56mlv#wh1V6BSHD$Fd0k06dA7IO;SM)% z^bLB-lkb`CSGN5){zpGBSCJWM4~?t-Sr_v^)9x`bSvc4tSRG{KyouFyq86ieD5uzC z_tRfhHh=%o=g`Td5O#gdw1xP7U5TwX54Y`BI6 zVQ^tNHfGG))YV26YC$eB%ti&S>~>RDJXbI6eSUgM`}}`+DK#>{ii-Ft131vEzk9}z5P1I$Ets=mpgh?QYY&hb8qj5W!&3j^-NlG#ON3u zJcEJ+A3rwBa?Yd?Q2QDkyPJ{XI1;xyHFXj5n%Vuq9j>wI(>vWMlrE+0bIA;w z25STL4~WehCg{7UZRV%d$vFUXWk93sO06PQZl5j?7Xjh?AQXb!xQy( z=X7;^DPO%&?;LBeNb*o^#?Oz=!DyyxE)^}6km?;zd&YX~yBU+p-OspJR@l4b;ReBz z#jyt#X1V0sJWcN?OG`B?XVK5gT6*WsF1~#&iq^iB_t3&(nXV1m@8E!%2gslZ+uvD| zKOhyH7i0$1kduKG%9((9wEts1XL7-W(Dv=yue1G~r4G#^-j|h_7F+TzEXVY18{91i zp%%&y(Rbi9-};`H#bMtDR+E{jUjXrflaf+XS;*v$%}u6heh84Vva-5WCE%T=b45Us zQE+;?BvPzz0EZ+XLjcb^XVb<`{ z=#i!m9OqiUD}EavFT)KeDNvoyUFZ%I=J1f~EGN;A;5LHl&oKcBn6r^`qy+js_c(YO z9niCdflSoDQNZ=Zuq_ZlGuOcHpk^OKXqKqhC$syYISxV3fpci6+~K#pZ6vzJ zLITu?5*Qr7$2rzkfGHq}BRnmr2PK2E+Ni2fo8L#sdh*AYFR9_KzYO*;fS2?y@&pt^^l(iAjwmP*gld7ilW0x9Fw0nSNtYYHM zE0QtR#>aS9*E!%uH#!E7=t)nxF90%-m-77GyQH?VK|7ZW^N_CN9tTQPOP6|hd)~kB zQR`dUC&|peeH;13mDtJmkW~HqMO!{S8Jkmvx(uU-zRuCnHdU{*69=N|T^_ZS_W7Qe z)jEG(c&h(IEp&ucP3xphG?lJks`vHddYN);=$K#0&>qwNjcc-gI)vLti>5DgE_7v- zEb*hbwfq!sQMm0R*muP_eXy+SY{2O>$?A-RLN1~2y*WMAa(dcgdAq{ew>7YzAAfj1 zK4Uh}eq?uyM(r`ca*|Ke+WH$E9L{a*e!Xao{tg3!=Cfnp-@IrlxU|Eo>`hph?h{>y zC4z!M6t}uMAzZ{V{IlAh|9m#Ga-H5rx-v^jCYG(k-kSPVd3Lzwz>e@S_y&qkM6NG- z`_{4CXYt~`;lRk2>Y$3ZD&IPTeN#`HPL9utxQKD(o{Rl7+E}^cS7hT;3>RkKo0E>Ga3;um{NufCi_}zPhyE~0cA6#Xb?)B;qxN`r8WEfPHZqZD?p9noCr3zvWF^aIAfUYrquF z$GF7A`g&ZFi))|?oKU1#)m=LD4wxY|Kfo2`L>sX(IF1J-{# zk{C@8fDQM5fI$&`A;^qCE9E8{e*4NZAjp9@gdf#4HIG0^pgbk}5f|^tg7W42yRJ3T zd_t*V@=KpTH68Rg=L^t(aCiaZ9WFqtyY>JQL+%!VNLB8pG%>RZ7e&U)te0y1#TybN zJ>LW-Khvpn;W%*e|M#^U7_dQdRBF6vC)gLcHgpFGky74i;Qo1{B0nt8&BRWKHZ%N&B%?xK~>jS z+bxgczPM;T^_W3P$ zxEo4yxNG|Ta4Mny(INGXYH$&j-708qcAY-jcfuz;#3qojeD!I0ENXqyos8(%!4VdV zJ@1n|CqrC9LK4izo2Rkp6N8A>;9$F!0K&!V-nIH=)}~!s4f~sm=jm$*LG4y>vL3O% zH*<~^)RDz)>*q8O_y|mKK)k19-Om~ja$UQGgf9CH85g`;ji1T%86j+)7 zH(ndH%xgdxfq&)bB->~W4UOuZQA^jowaXA`awaKq(hNGi>%h?jOII(y z3lGy4Z1WdJa~&%r9lH(n(Cb&{ll0N2`H7y?cO)Jnj;^e<#BK1&dr!$HByiU_4t zr&s|l{`XnV$oMt@RS1YM26lTX@C%VJrA9IWPX~*(t)Oo?Ff;(X!TE(hdrANsgqjEo z4DS?JV1bMXX+{Jahs%KLI5Q)S$sd6g&??vY>2L0ClaQ#KfK(lM5<%btMmeV;aYq)w z+F@@;s2cyq@J*`f-vGwt!&*}_mFPRT$@k|5v9$Dpd}Qt0@GyR++NVQknjKsbEr5GH z$EC>I@MC&$kr0%ldb*VNIW>&D73mkWV*5hOGO}J~7Hr<(6nIhUm(o(NkM|}fW^mE@lV%gI}xfhc2^Rb+q2+FIpFAI;-&$ezamndiQm3!BbkBsnN<+csH zeZ{}3>Oz{rx=&r5XI+cetSrg%=QQhE<~Np>z7%+Qp4ZW7u7t%-Yj?Qc@i)u0Yg{(< z6IIMtIjnrNki?J+0CD@4m4!7JU5&QB!ybREePJPZxDi$FkJ z7F9~}#3OUd$&>=&`zOWJy{YMD8CHSSDrYQN37fNNj3g2W_|!TQ%~U0=OwY>M((c`} z^J~xcw!v_Be!4XuWzmne4s8E&>TLWV)rIA=gqB6(VcNR=KFR&^e1q*tW>&tL0r<`~ z>UMyy`4qAaC|?^mzrE(rFpq9d%9K5eianGS`rzTV>$UxU8n^C@P0hlX_E#r@AFMGc zGv+spw>k6kfrF^JAS+YV^rdi7`UR)h__ z#_WNO0YvX`4f^rq0tbgM=IF_hs{r5@ebMM+M=aTh(p7=d$&Svf7Jwn)cH*ze0Y?_G zN!M0jicvRNwy1|@uDg{XWBq&EK&P>}Wxz@C(B)^gz4dmRo1fdbQE^@WhIKdQMPpA& zPWSO_+XXXom8R^Vwgy|zMYeeHXe^ZYksU8K_L4hYXQ_3Pm<0pHuV>a{tus+duTZG3 zU)gkOw1S!M%5Ix|Sg@eBnP8K>bBD>of;_)WFi6_aZ=}f6;PF7&?6v_u5m7VniuMrO zrmG9OTZje=0f7eZEnH#|AQj*lTu;2$1Vj!@(9NMKsa6A9QP4EJN3TyNw9v+oH>QvC^G(m2=t8Y(Y zL1AGdEI%*7#hL|SX^?cZ<&&okk{`fQfRQWrJ`o2aGc0yVL$WMUh!w|mZ5j(9$sth+ z6F-6j8y@IPdZM8c;sq7)b`01VCx6)SzWox6h+aq{)w5c_daMGu;2L1><=8tsq`k(L ztge5L=m3&gYupG439C0c*rhf+AUBv;&wZI6JOq@gnQ9t&fCs@;LvD%i2elHu)H4J0 z7j$S1u<(s>78MjUK+MVX&!3O|+x4tqXMr0gqRD{^l8MOQwMsNn+VvPKasVFbs+CO;H%ffv2$DmXi zO|>YabibezC(TmFrbwwL;oQqwcZ#B@)b`D);;x6Lf?~o`qUxF8XJdx ztuH~brIR~i*^$n7dUtt&fM7u+8_(xqQMLW@Xe*b|jb=*y)ajC;uK5nFWfv4&oS`nK zuADHl_}GDh{710&F*^lZ$F>%_6S|vR>fE8IBYOD&+JUEC>fQ$kPU0^*e9<2%UFN7g zDs+;}&Wn#oOQSg2^`5E+vM{+oWh7V0IF%N2FjNsMTw)qR$IkW+KBUQ(c(Hll86AgB z8c%5)9Sq$lF!0qcoBbMJb8aZFOtEWyA!~N6g)&s-0rQ{tig(sHIr9#B$A0$1T92HI zqi6iU$l;I8ZpuevrOG&%%AH)@D8sG~IF(QICbic#ix%^jixx2!aZjI?z#g?q0Gz9z z)`a*v+@Dg`=g{5Dj$cg6noM=;^!jI}4hC5XdDuo%nhN9-)Mme>PCI< zNRLo?r56zXMDJaQOt2nx@URP$fw|G-V}*+KkHeUqr@H&@>4S|8eYhjLckOn%&#Amp z8OV(Y$vl5w;6=+(-aVhUA7{`ZS+6Zbg^o_S?4%`q^7CBKu=>DBA~1FChCVTzJP+O& z38;74?a#K|7JWnbRO)V!wWuF`!P@XJ z5n4=bj+=?QgVTkk!P3Q9BGmB6+ol#7=*b)0b#ZcP8?&-KeY!+!SBq2m;kCuBiq-Z2 z!u{cj8^oydq~HBlJ-KmPTEdQLj7$xUJl8+7-l=?|sF>5>X*+(kS;^}Oo}ev-)85<9 zzzJBw z8Nrgm`k(QTO!*21D%b^rVEzJe*9XdqK5ad~dAMu$ut5z6wvqst*08E3Cs{x}%3m2Q z_BXe%MnvcdnZ&d=d1*L+G6e@aF!J|btEGLvSWAVYv{NE=+PjEB)@n3x9@4o{thslNx>7zlb zsIXl8wJ)9b=m)Ed-L}PL#W<~rl=amK_J&Un+LZMsot|J|*|r*Wq>+m-d9=Io5iU;i zJ#M&BaIBYFn-&-&E8G{fCEfZr{sUDE^uv_Y<}c2DqwL>>UiTQt_>Pj=DKqz0XHOQ>n=E z>ZP)80O7eA?WyOD3@1YAas}Yr^ku#q4xQ;PAy)!l-F-u_)_a##Sv%`@@z)Tto52y> zZ6Go^Pxbq{L4nSZDRwSKn)%xmYCc0J6)2mOC3o_nZSGh07~2TpIVVD6Q$M}uV=a@9bIv6GFr$Oq%Zg8Q5v2E3Bo?_ zd{gg^EWh~4fsDc!-LuD2@x#jpnGS3pY20gFh~dK{w$Q)d8$9OTn-6D4Z190C?`3n< z)6QOp1TD1~Zve#|M-{wwUn(Rq=I9$F?uBFva&!us-f4GR1 z9%dG7gig2-hf7nBe>-e?XlrN`G#nGD=uhq3ynB_resZ!J&%R_j&V>tQHbH^LxMC8$ zCQC~NJh|dm67EOK`?*Zj1bd%|RCGE&sUQ$>PMsMpu=*eh5i6z#Th|7CjRB^Hz4-fV z1UgV>veXu$o#+61dRzZKkQtsYM6!dp>g??c{`j0gAz2U5x&b)9n(>jb^5+Ft(ZJvm z(gzD_QDWc+{dOt{B`=D(`9tCchgLB@@&rqK^R8ni?O8LiviE%;%4Y#9f+sRd6rOhjbmIyr0JmeH6p{qq0_OS83hM<* zD0pcB6cN;X6E+Wxjjup4)0fYShhV#b%?}G<`yubtWlGW%6a$EG6D$&-tFhSx$7bv& zw&M?^M{z;H!9+>wS@4G8VCB92v6pM?xill93}9iwy?F7WANkBTMAZ%b#5If0YIz|R zE?cjLhJ{5+FmF&ol`R^s1jsu}RlNqzX~=wY%Ka3MA0P>AswA=?i=Bvgd&}p<%sSp; zBID-XB)3<8m`~wa+0FWg#HHQp&N zNPfugxOttVXPPYZ)bRDuvx8Ei4FBU^?g@S2;_%;7kXUL8v%~MP|M*ibas^D5!NiJ< z;hjB8fzVj&hRW~=!=CynMqrP!~Py-*A^QEdw9Ody-TuT zg?{oM8@|485wg2P25{KdA_gy4!cdBRptZ}q!n9Mc1gwPv`uhi^Rd1NO837eoULK5$ z+572F>{j*Wy+utLsDD7QOpSZ3_c!DyqHsk+3%V7 z9#cSObTB&Rbcrg?UF{qGUt;Q++DS|(82caaF)ds8>K($^9Gemk`d(b>ZyEz;=J z?n}S~zD+973j8wG0t+S{3Q(8U9l`GJv{i4KCYweR%SpR`%`{FiV4*%-DWwwB#s;JY zrS;laooQWxOUpRtFt=LnXZe;rBSjs<#Lw84`3HMj6PqsuigMNhWXXHWk&K3PHcqprbuS;H`QYnL%7WE2pm#CNUZ`^I-CaGB4JvFgO zR%1bQbg(}>IK^i{?2H+@U_8mzJMdO3_`GNnt$0#belfqCLmc)^`;YCF_la^Sl=;}! z_XWAX$0<=Zu}gdNUi)ZbKxRbsgu@k}gv^!o=8%i$L;RI9wfZjU*6R3xoUBSG0k)55 zcI!LMEQ(t`0Wxi?L3iJzx*e9^bhr|DtRiMND|=bMMl>LXq9m6~bo_Ld1|Ep6C_qOG z2bvEdB?yIweVi5*4aS9e>dka8?`f8+W6yS!5#*0+O)S#=o$ zG$2sRW0#Yyl;jXtu5NM=KUxz2VF4D(ww5}4zVDD*KP+F@PN(22giPpa(f6? z<}M@#fp|O8nJgScXOKul)zi~7d$Bx8K5HNu{I7tv!|kR=gnNJ*Byf%mFGZNkrIH3=*B2176Kz~u(BN%Ba((UdgfxkWYg$y9of;)30NBfgM!);bBgll zI2G1<&9s4H^Hc5fbZ2y1e@o!|Ms}>R=UJT+0=ND9LLD_zgmer2`_l8$Zc&{Cb2JxJ zSqNBlIK6g22-&u+=<#|)xMx;+0{bE#pGfcXn!Npf$Y? z`a}>sK3?oD_|ttR3s=2d7`-r?ReK316~w5F%*a_ebY%eWU?>z2$$W^Q6!6}d*PAuF zEYsUVQU#9lBv+$p%=om9zEkc32hrgb1_m}m_ytX^=b)?t6&!s=*%827FD9z`fQp(+ zRI^ANzy{$9^OyImaNR0z;_2(lS{u}!OH#MQLanT*gG%JgudkcdLp$ClF2bPA;82X~ zIE^~O?AmKqI+B}wT@asN2qDW6G{6I;Y~G`xC8UYXy|udcv9W_@_0GyvWn6 z34Fgvqg7Uxh8ovCYtJcbkG|uZ>zd^I;>Af6Q~HaZ81JbO;m*-6e>?3c>BrXWi+&)Z za2ic7Ehcf<`3MmHY%1%83gqJy`F3P_SZH6`e2~yBQGcb|*_zTrj1=!=j)m8zRPn5% zgDJr?6IUa(*Y1Md{_8b0o;;}Oe>@AAD%L?7=6H&?1b|`9cNm!-7&LL4HtLK&ee&-6 z7Tfsb`#?+wPQNB+Q!RsDkm{rx5q}wP%e!;wj2=neRi%WJq0O07Q*z@yKoxAXc!<7> z^lN{da>yK0jqwgBJyU(It#;AvMi{Az2b{`>} zR5JsMikMp?Tw0Q4yj-^j?BO9>w9=%n&;f8&C<0i!Q)<{)Yi;PRhQn341?HazT`q>m zxj2|B=389M2Hpu55dl>7hCII4MwCcK)_oKT0_$>TzPC7C7hK&Vy#{~MIq~vO?Cx$_ zMw`D1aC17(11arfY&x{iF}1Ha*L}Q+XwmdB8~CW{Uo5S*UN?%SzVkwf#Y4BBE0a?& zDa4RNkeyp!oF>7e5T3gOvT@Hj5uS3-Dyl3Eh9-a=+Kt()Cfq2HvgCQo?QF~_g*%&& z5EXKzg19wDC(vRd#{uM+4y-OR(GbDS$@OQn28dtu!5$>3q5t4P0HhUs2NQ3A=J@)q z{6ElmGNmt6&H?anPtk-}tz;4HiKILU~yTS-oNF(@9(3KYF z<`x%|!%+d6Qb9vwK1~oxBnqy(K>Y{A=ANe;$Ix^(fOpStNZ*?t!g#-`5UJDqn#_sW2!~GW#OLcd53;AyQN?fW30tPZe0zksD zz=`lRkamS6f;gc1jo=W2#*%utFOPB}cgZ?0F%QqNsUZ2l%-mdAV9Vj^r}E&zfVDd) zao{ow8^R+fD7b!O5i>~-r{E;yApLk!p%;h5>Lr8P8#z501qCU=m4(>q0nPC`mAFkO zsjPe+4!!Bs)jSs^G!{_va7qqT=}of({?!WkJoRcM=VizaOG))TkIOtRy1{V zd^$WUNvu#$N>k~3fv^@dhJ+CH%}iK{Wjz{Hcq|>A@3BxOCM-hk&YS{P zzg@o*6%3P)m2o$JdSHFV(EDh13^%8Zs#ptBp&mm@(yV_egO?CB2}y3{25Z8Vq}tlk zQ-^v)Kob%uD2$#I3wYNuOg=TO&h-F+H4Uxy0sl%~e0Mim-yarP5v!u74#6ua{n+={M_QmtP;v zzDPp)-O4T@>2^ww;r>R?aD~GeIEThKDMRlSILwg_kCrmKju)}kXe?UH8`HiLES>IO z9o5swD^PS@j0&*ZC|J&~S-k3Ay`LA5pKnw1wx~cWe9V1}ReJt~zCXLbv57!4{ZUPK zho@#>5Nnas&(Di~1;5(*3?B8_7k1h^cofWdoBr;&c}_mkv#vQYwwQ(!E;|9#dH#~a z4FdDi=6(YViUZLPIB8OKxPCiUXPqcYI;r^Oidt%49XxVnCH?0Yht{8iGT3SqPIXw)2^!Rhr>=OKJ^lo;~P8|49E11NNT^;HI zpWa!zBj@bQ`R?6o@a-X3184^JCRV>+cUrjkCANTaetv=#eL1mD-Ak zNB_KV)4z*mKMczSpssaW98?Oy=xV7dElu3nt-)n-VC^!M=z zU*)IcN>hw`9?h;%TE%h#A2^auP)Dm00UkmmOcJS2DErdXgaZXM0v?_swdbORhlYoP zL32FQr-W{S?N{=ZSIO=3?~!TvwWBIbIy zY$q*U?mRLVCfK;P-WEYZsWJGt3cNi_lX1E1-iz8Qu>xPS#$thyek$a|MhUi9u=WZ$ zuuyi!gsqfL8E6+a*L9YIDq6tyVrX-7sq~qyu9&@frF5s8D}y#JZ>fu}nuL?U{|xh&mo%JbsK`St4neMa0(Ej&wu(CMfC zu>~%mft7S){r#sv@GPnjUrmdG3muQDYCI!f|HrXmIGR4MZP8%_DMO_tkB(P2D;4Oz zFLAWK<@1Hw!%xcR&njOFOg4wOz8H>d^&{Q7k1S$+F6_t z+I?biZdKTBj1tL- zDw_bsw9g#~5OP>AYTDwQt+cvf@|+U*UX}a3_^7K2PoKvQ7>$x&Vi+Lf{;9y$|`<5W&^-O-nN zpuK@#kaIJejsw@_IQr{t(f3*GN>g;$b@*jAhdFzbSB%Fw{AEANSeuja+Fp3Hq>x~9 zI58agtjPQMYGGJO%4yJd`hMni@;q`KEa85o60b{$!vF9+Bv>YeXvLDFY$x^BX2NmW zTTTBG^jqAd{KYFulb5@KAM)ewo-0|wSz-7h{pC~Xv43*Z@pTk_0JhN0hY}nDGS)fR zAS^p}wYk8=VnXS>st3<49*asvQ4y7(ZS4zK$b$g%0Z8-Ns>hsg?PlDfoqd77rqy%c z#ugT}9X;vjI6$6oA3$2g-%pm(yRR4)z@>Yr*7*$RD&`k}rs8lr&Mp9^GleptqZ-xd+N)+Is-s4XBF9f&_e zcL@cw-l>nODT`$E%YhMSJ)idWm9jC@W4Qe8t+?d5jI<6scme**=Xsu-jnnq<-G+Dj zPUa~ThtBUHNZ1%lDL;_s(wGSo?3x|%aAff0E9n#m*Ns%hNzt;V_WSQe-4xDMoU(FN zv+#Yrmgg-7gy3dEi0f?Fbw5MC!P2V`G;1><>;Y}pX6EK&t-w6vK*~jVx#P)xtl8B< zP*cN3yWT>np9t1!4Sg`xKrTDN{ayzdlcE3Lfe!!1>F(+}`(OFAfq}S4NBd7W!IY3F zB#90Dq@(xDRQ2^Ekd%@CG!z zuz0YC!n{UA;?>onW+ib`Sz8z2a|Vk(!fpc-?suLUgikVXt#W$|kd{|E?91RxV}zJm zxc9hC9g85rL*V8_(s@Db3p~#n;zorpKuWP1TJYuHv=Kkhelf`m)*;T2yLZ7B;&cKj zNW1h2;H;ni^$Ryd8e3mqAJ8$Pip_gq?6fP$_zZaoJ*Yzqk%%V+OKFeifY-qblkM|t z!UC(m4>T$oK}4tQLvPzKhhiUjoDZB5Qd*kW-QnKkd7+UXvqcq^ZANKs4c4z~ zZq3SYovW{?6Fr6EQ~#_I^8lyYAf4;b^@aWKQW%c3S}*OhZfnFBKH`wZ#nh;D;;aF^ zjkl%=Ur?79m@nn1xu>x`waB4JvR=Bo)OyD2Ol#NHe456t2a`_0#$nV9PixP#I3Ppi zJF9Idt%U9e=-A#lAzZ268)LdZdQqA;shpYIe{7(D*g1o``Z+@q7y`qp!8=zHrp`rB z&-#F$#c)Q_+doL8ske|b>508Pj^c;*23zk#)c!ssY^1ROvn3i}IV1qUT7>wqXWaT$ zaqCFYqTPzlMcCI!H!wwc?gwhP66nkQTl0D_A_DLU7BI)49QY=tcst{pXD;|-nqgYu zgi;sf!9F7D)8~~5s+hM!D7#LjGNKL#U(a2771ut03*uz z^XCnkGazMm>SV`6l^?n7FVF`_`a*l>yk|VF5A4g0U0q#}4~&Hv03kVWY#wh1;yM7%qj#lzn8e-P9ZdhAGPP<>NMK@2h8ofbZjZeJnHXs*AT1smLJ_+y zeB!vq-o1T$85FfWAT4!pSS;kk0?Y^QD6EK;dEqNCHukf!dxvG+(bsjrnUn}AAK)MY zw=IP0JcDS$bs*?wJheKFj_l2Ael=L{7l;oQehf(Hj{sOk^HOtYjN|tR-}9rW7F$aA2>kB9x&QFNLxjCKfdus`J%D$tMx$U zB97aieAH8{LoEYm?Dzd|E?(@U)mr1LJDZEy(QUi%%kbN0dBX@-amPO-^V@W-_@ge@clvmV+x*G#>qNcYN~=eC?3#x?j(aa0R#N3o z+;)zC030D8HoOZY?(FBdK_k0Ig^%)TKITx<)mS_Jj4@$;w@CgiL%XaEXxO9epJY^m zLzL{b=!!8{7~tVYM`zVIb3_s=Y{Q0Vy)!16OayZclXR^s;XZUdf1WX!Ljm7DHsi3)kuc1ikW-mEBv_@y)M z?wc1qTPt1iP9X{mFmUuf76HA(6h`1|l}q~UZe2oH7_GZNNQR%^X;4uNK?8*rh$H+C z&|}}1$6CP8hc}pgrg`e*;@I+>B`B{2r$qDI@14wfry2BLJ>)vvD$_q+5EnlYmC@MQ zE@??m;94qFa&wDbDY}m0&=u~O-jZSKR z351OwL|o3oF11>=eV$kV9&N;#2(!PtXLvXx>y7+JPI6l0Pd5=$zra~oF=rUMhpqBL2wI*EP?2K+aVTk^WVIA6W)LQ2grFf zy0Mi-FVJm+FI6RFpKe7WA10{=Y?(0zP$6VVr`b%X>lj`JZq> zHR*a-Htl8_1Dv&~6J#F$KpWRU+2DMx|J_^~!6B_+_Vh!^44uV1>QAJzZnG`^V>E0$PvDS@1_ zroO(u+mYLf0Me(bX0GT%%XZc3ULewwB+b}!yb0!0fn73m?(12PM^Ob$my&Ne*XEy* z^hM3aak%xbLS~Ga`MeOrh=DK!OMt|6dbZEgraj5{SrIdz*0dqXZk04sgNH4U?YXG9{ePodg7v--p?iVXHld-?jF1+-Z*v)^&_*ZORFWgA$II8-9v zzmHzAS5c9onKTS_^R}LPim$Gb|S*Y%amUF?H2ccyNA3TWOi{@VuM&c)~y@WGvLddveVEUjwNnC=qw!7 zb1#dp-BA?tUN%w>yY=N0kDC)G?}gMm7s{*9^yWhZNdwr|-ViE(&k=Ju?HKKz zHBOwiUrc?zf^{~wfVo)C+pplY$vnfH+s63&SOV}<1-4q;^cnR0rV&4JPDCVhY8 z-_b!1$!u_tJ+C|F#CpSZK_at%I^o3QMU~54`=1?dLPCy5BcN?a-F+x~mE2$`mpQ*^ zQS|bNOPvS_5Wce=7q0mXb~8|N{V;;9wZ@s6`XK78e;|#Q4-v6bYmRmcZpradkuhXW zpUyaO7~k3`Y;~z6^OFfZnqC}p`@nWJ1*NHK|-D4M3_-N`PWSW z{_DYQ>t!glRceH>e_zAdvA90-e%4I?*Y(|z?l%(MwRfSar@o;9`@+cxuE-%54|y5b zMk$kF8-eGpvJz+>;3Ch#1FXAsg=UnFo*qK<@acO%<@YQXQ128iT=!~OZc0vvj# z*LQYWp~D>UXd;o)%9&f<-dzwQW@E#J#7RSk5Fn6DGV;=`)ARFCadsW1#^M6Z0pu31 zNG^npxN^X$GZ+nyOcfm+B2bt!$qw8+ckUdLfCNbp2$CBwK5!DJ0gM#{xqw$RmiI9M z&~dmoihL`&Os;N1mMc_k=N>H?(Ht}z9$OMtzR!+8#`po`ha^wB7$t6tI6!$HqC2li z+XYy2c>!dZFJPTzlIC-T)W&thy$u^`My^fOTr$W{Ag}#2^a>)85?r#0k&>Be-$8M+ zyfY)QcxVO!O*==vql>>Hr9p%8mskW1in(B>ryNLhAY>ZO+UsD4+H|vmEmjR0p*>@d zCL$ql^BZI}baf%VB;Xw@tsp)UsYii9D8U#Boh3x7ubUFo*jfOIxcsJbdtrHbxd5q; z@g5(5R->F8nZLrzR8{k;?z4e~B_#pUjK1&#(yd{@1^#ruR-zSN6foh<&6-Tv_t2b4 z8r$GQ%A@^#xXweH1G-*?S~jHziFSk)q899*=}BB2ANldVwj{GK$w^A%nAes?+qnNL zIYPGZzzF4j?P0g*NbRNRcc-DO|!HTW)UCk8Gka)qylw(mW5ZkE`u9Nx?4XRyZe>0{PQz!>OO>nHvad5V*Hne(UM~a4s29_WWWXXa7OMt#j>ux^dW;4oe(_{ zNXE$4p@z#6&RVDbWD`P-0->bo3u{$GQ;>$UP;FQkHSkrH8_!PHQi+rvN^2oWZtNPV zl^c-vqFJttHh+5W-q_+}8yjqG?M<7uV@OGOln?}t$3|^iT_4%L12)C@C0}@t^*t)y zXa>3^|7in=E2DV^A0QH4g{trNa;EI*S%Flk;)Vh-C4f<_vbhm!%>LYG*UhhkuAs03 z%`^3Qc|k#wS3tn_Uc)&XY(z!AZ;YjI5086hD_7Ut6&&`J5En|_6{j8gP5F(@q4=Q~ zHtOi8bXPbWv&Y3lHO7d7sRrRYPJ2J1loHiStp(E+v>=rTc-J+B9(SZceZ{Rkyx)fz-xL z(|>~FCMRpX{rI0%A5eVY`aQ9EOPV_znzDjrB==}vE^>hZA2f&)rL-LIyO4JdO*Aj) zQ)It74$QT$8z1sXZ*@q#empe2v%-gx@RGb)xccUj!7J0M(l)QB8oN68 zbSO{VDYvX^zE1VyxO_sa>fC~XpjG~`rvC77Y|;$6V4_C5x2=846Z4edRTMk$QJ5#f zzq+CwN9KYcTg!b{-hRXQENZ$vk^?N(ff*S)QAQwKO3KdW+=qrR-H798aE@L+UrgTw zt*B;0Y4dNb3%L_)D>vd+b%JD526CVT;ewsxlIVzUN(%E?20h>UdT%KfOV4vhYe9n0 zmGb*y%h1~)JwkZgm@%s-7qL-*6*T_rJRK|8U8Xyh=ehi9#dC9|YL@5elfz}FL^{#f z5bLq7g?JHm_WK`h7E$o3yB~_K(ba^TZt>* ze?H~&sHmuT1dIbx8bJQMbXk;%2?q)*n&1JwN%C*@eb`l`WDPo-Kv-iB1vN;V6DLxB z#G2soCzKhUqYvd#X8zznZcDurW%tMK4W$WSi?~ACWC}1o=REtP>A@s5w@}*wOA1_)#Uc4P-;y|$WIRMP%;CVZR-IRNcwhM;Ib_ckb zffEP-pB(Vk@#jNZmKPuh{E?}tslOSr=jeIp>vHcO`PV{8wvV5mUu1G}Bo`;=z17Oe zJQx&sAO@yC@_7Lm7XgyOg=BmHi=bPhmD39Gtf0eGg|sMI9!cI9=&XZ6uEdE|I*G+n zUP!d z-!24L!fOuQiJ8ffY(T7kc?ZWm_X9ii(bQ&e)d81*PeL6hOGx)Qt1V4J#MLoUN&#x$ zbmwlW5zT$LLiU04vOmg{kQQ;xZKt%5TE1{8y#9Z>%=!an+)J?jOH$N>Z)|Cq>pl+g zXeqVZ+qz|xp)!>MKmgc1p`rOqCM*0_VZbxRl#jpU9=V})j(1b>UIlGNTpTBSLBc23 zc1l+spcmpx{e*829w#_2kLMULF6e%zVsFD#t@mxcEk7lRMJMWhQhR;%JMM|E!OoWh zt#{G7#e89K9V3u@s-3YJ-0j0(f~-(!L8rS9qXF`p%Pa*gUem{mX7)rglWP05 zmX+`a0qFB73&N{ABlcM@H0uZl_MAf2vKl$XfV3f+89E(+ zf`kXt02nHqQoZB`%Pregu1+0fMb2;%S7wzigxiz)rU_zS61 zaMWgHK~u^zmYK#B?-DAsgD03kHGZo=y$6M)i?J=p|LwJX;%s0qsX z(|QOg&wp}v0DnHtfgKBp{ek%ai`g4%h_Q!{_)prbhzJx|aHoM0U>YL6mKWM!i*i~V zZ0L8ibC?jl%k?G&;<^5VpZx_QUSm;ZE6TKxP({f2}uw)ua&17Y1L|%wfuUcV6o`;@b|Fm=a z?}FR+{ILz*Hsf9nZ|#ng`)6hR^IDh8Y3fe{(%=!NI4JH$N;7jRLfRYn_f8=y?RN$I&98w$9v!=;a7Rz2~Ww5M6VcJWL){zZZbf z+-|z&d5Cl1>z7Upc`}zzlyn{hY%hedm_=r-4Si*;qPw3m*!|7HR9~31>&HTe z!xWKR|MU7<{|Tn2onT@Tokz4c+v4b&GQ}UT9 zddl(F$Q&*zIfXcZjzf5q*nKB;u>)f6@rfL-mwRg*9W5Ealm_h{>XJxRn(q7Yap8$s z0M(L{otZW*=4SarM!X~y*@8t6WXm^?aBjOmh!`gnOUKsp?-CBXe8k!agl|_416NP#pYyC zm`5D_$y!@pLBSu1kK)qp2>DmdWY%0zT-*!=k~}b@peO9p#Fa+~odkCa zrA3-?hyarXFFi|F0y+;F1d=dJSRZ>gjEs&lUb=(=r%q*MbF(xE9VUZSwm-ns_`ji; zEa*Rg^?yP!3EiOz`JCfRVD=NCYDR;LgF{+P?e#h~qaTokP>L(TVp(-DzoNo)Xd1bN zL_Pw$JQTTQXv{&Y#A!kbD%MLh>yTDu2aRh`b2Ic%3FKZ-<-o71CgpegN+T47U8V>n zVPMEBgCJTIR0qjsAJ@O zgb`mWVA?Rn(kBXG5+f}{l1X;TMxXNXpnP@Un4Oz@bwCK@hJV9vk^Ea17ly667EPLH znx*bcKZTF{pk3f==F2J`PPiYW(K_5TT6`>CkTTm~07kAL=x>3IZDx6y7<4ayil?;p z>$l$2lHefyep~O)C?f0mX2e2nP*O*HWW7)SKQqKk==}ekD$mzvb?q=7;>dR^1u-#a ze!)D(B-h`6g$!6)5k|qXvnXY?{%`3*nJ45F0*R&SXWiXHZN?}l1>vj6NqtB8rQhTe zIIj{f$)$cQVM&KVkEJPtOnt}KtY2EX)$?@pdrsBw-C@@~UUH0hK*0ab;%7eZbd4~1 zd044jmmgf_`;LT*+g~HN@U|;td3V)1XSC;>M)gMvZp+1bnwTVcKa_ydi5s0O+!xHU zUpbAL2o)6b$Z2LXG#VaXy65#=Y3e4(e)(0b!1>6ruda~L>at!Wwe1}qC-t)g4;A_L z42l^gkuopb5T2}ZMkb|DZnkql`%;#zIlEx7V^8Ht#opL_^kV0GgIVK|Is3zRxdqkL z)`m=`bG2=o*=cDYZx|h2^Nfj(_R@yU>`@g2_|-gdDj82)haMFRi_uYGr#0bIr*AGq z-&bYhar*S;)@*vJn)BG7L6Mz9tmJC&d$Ujb*DsdJHVdbrF+)#J!i7cmQqFqD;6T)(o2a5UZM*gQQ=|}T z{^Of`&qL=7HS7P!)_cJ9yuSbAQE6yLk#kF9yZ8HkUH5f8ujd$v2bHxFhRjS%Or{ee zTyqFGB0FJtAsI=wEMaaI0Bs77wutiI_CJZNZvn=Rl*~(ztvihAa$+*J;UU0N4JEi1 zxU6*#S&PfS)F1s)PTA1V0KT(nK>%87pcxnIqk;-?xJfGIyV=>cW3;lWHY2G_*nK)d31by1U#>El+-rBy9za7v`C!HJHooRfxFWq zQ@Z&$2PY@ZUA-U@?x)&ID^&59y4I z3@O|K7@NwgtHb8k0ai8oy^@Dj{%^^J7Ys$N-@O|;GvoI4`*)X?CZx}i9U%IiQkw~O ztgNCo9e|DCvySI50T7gvb193r!O|=(P3(V8Phe0OS?y3zAYiQEjWUcmm;!}aE>Bj$yEHWnOmzpnEe|K~i0cIu%z+YyxF|+DaoFk;2_TF#K?MHt^XI~@k-|m$ znx~sTFu_Mt;IMP7QE*5Q$Fn`YMAoO#KEDw5pPoV{-))oRzvmFCZKtjz4?oj>2%D{z z&(DP)wRs0K+u#YS<)*8>f4VqyswvYl#egeBv*Mil`r3vUTh)`aD>v-%Kws*aVb0fX z5C7s{lkYGf54FC(+w1@0MblT9Z|Tens}7j+bZDQ0;agt)lxSTlH3c-n0vF@DP6Ray zEZe%$Qu$7)#auk+xgJmPRj#N93ql=EHV%q-P@4oN)7G%xzB>Hr4!rL4ZyaBpF!^#F zJ&~imn>-NHE&goid4zGQx|8;iX!{aD&tZiJ^|k_Qn1)W$MoMwM%HVzIvA99()b++K zC*YGeD7{;&P-xunOk(H4EBcfpTz4jVrC-RyK}e=CSNKhi(+e3(8Oq8npIJGY9$%Li zTS2j7?9?c|)9`q{*CiTGPI-O+phi>Ed4m^}>&`nq-KnLuogyJ4c8fsuC(KVTSXAqC3 z;O4l4Lz1^KeGGiF-#tdba$Toau-2`<*&ZLGN22EsSqkur?h+6a{Z5se+9;Rz(&KUE zl*9Mh_q9LU3&%QbraOzXFWA_)TDw{ocof*THM}`@p-C=x1E>+0!ijN@4vJn!yf8?Wif7QZ>^X1% zZ0;)oZ&Q-P*GaF$;~{l7j)$|vd5{DD6Jd)qBlh*{LtVSPIAQ($2BA%8n)!{-<+Y8~ z$af)@do|Hboj?D}SCN;CgG0m))r+K+8fQ*a&8J>?@}%d2s+>-zOOet27>FYy5Ap=l4MWN0J{Ykf8TX&017kTSO|N=%u8?z2Zn zhnVw$QrVL4nK$TLJlEw344KQPkK!>`J^SjN=mF8?GYlCHC9{ur^TF#? z5F}A}{?+=JnwWWWr!R@^Nw1i*b@(Zfvb*?o{VY$&p7UmQI}Eihl^bn$tX=WagF#WdR4a~&>AU3GP9qc+jx z^=G?J?tz1L9tFkAmyHS+9Y zb9hF*-nA;|9ox|S&AI8eTu<}rX{EO8E}rm)>8c{*4@Ue(pxY=itfv{Kqo_StkP+e@GpqT09bLifrwtn?3or3aB?sGXa%77coAMTF0 z9yR`R-e~V$x?1S)cU<^Bp|dJ6ZVeiFY(g!%O1xMCY}#_cXF&+1B{nx~kN=?yN8ZN3 zg|%%+Mus>E=&{Z|_M+WZrl*ow?bXfAhat-{5WQ9OD#kVe>_07d^J2FZn**i}@fU^@ zWW~vbsGB;%6YeVK+Vi_JPTAPtC9Pj{PqakKg4ixAh+~jBswiPza<$2D>{7f)6j{8y zg3^Q(KBShekRF>y=*)*&aCF%$!2ptmM(pd3ZzUvXtW!H5VpMA&1LI8 z?0^Z!6L##mHLhC*;z^GP0Z!RD&I9e4FxEtd;H%fpc-5B^($eTx`L6&do06K^|Dx1H zf|!Kk$3y3cb&O&$veWC+)1Exx$XQ%yHRD8bx>p20EeZ~5cNG{wT3vZ6?o7^#{w0yepigx&jMu1v>?sy(vCRsSazle+L z3Z^&Aq@#CU`HVG+K2?3Q!GnGP<<8ZcMlkX{ixIV<_NDU0wN-Zi90F3P5<9CNdj7k) zTEtw)Hc}GvfFNVv1zQ4QcXPawQrE0vFb=l?5v$&oeyPY;Y;iBW5PNqDTD|hT!--u-S zIge*u{yB|PX=AN-W~Z|44|QkK-|zCA4{WYyf2sJ;<%mVabJkjy(`JnO7rVc2$-IEA z%{n>jl!2x`SGSX+BIQ=GG!bi!+j_I(#oaBL5BBdBHl}6b#3}= z+Ti^wVL0%YbM)M=@0@clZ+%nSu*YyHEg~dtJNc{!70~oHBhPvYH<&{+$p)~p7 zGG8)Yn>~HCDq_Rvpm~>VURNIC4qo@gyV@s2|FGrL)K;A@y3~J2`Wm0l=l(?|UQb_# zlFiP!R+gTc@yAO~$+eyEC~vkreLCug>mviAoEhab-tAi8lDTFzhiB;R+sQuT^nnZr z2`l@Ul%*Ql3b)YS2QyjCJr8$JBGjR_WK{9}_bD~ziPz0NTzvT5YaAqBZtrOSdBdI%h^JnaAMi zPshfle<1Sh$vfL^uBBLSyX)A0sISU7)d!X0YD&k44^k&`$05u<{8+C%!e_w>2v14J z+j<1v6R;S;VC%KCwAdc~G=oWEd3kwa>rOW_|3D;>q}Xq}`&Y9xiV;arNW;@??7D!4 zltZU-C|muBDHtXa(-{PGCc2}aLHu{c+hQdkHW9LCns%ZK9Dt?d=b}s72L=a)b+kLm z(Vn0-5gYlRE@RF*7#bnA%VXu_+@xR7*@&nNxCqckRPGD8$7Pp1udL?jD- zEX;giCH@Ufncq zXX>#=mWGUHVWvtAD!4N&?Ch0`oE;emhapib^|oso5ik#Hfj}^8jEsGtq(?N0BS@Tx z-|%^plmPm*%%pvOc~{e=5?tdRWYRlfb{4E5U&Xw@1I(OMgMj zDe<#~rMl{F`N`RfBg=82ufs|!YGS_C-s>0!$16hGqJrr@MCWBYcEc9C5m-jVo)fLct?k_3dx0?_cS_xp8xAF8_th#Xynv4uM1mN z8Xu}V+xv!mqUwcS5GP3>dhBZU&|rEwI=O(pC*ucvbOPsnOpA`gW$EkJAqDpr37^E7 zc^i6`7vj%^gv@Lg+p^=d#sU56w$d-{jg{0{%Z=GxgT>XaXT}RR9W*=G);&k1?mOl} z&9u;!KJ#O!x8vSgio`cG7VU$Tu|rYIm)OmhKFQmr_0KrGYPGBE2-EsJ_bFRps_iP% z{M60$YI}U+hg+s5DsFRoPL`HlPVJs;dt9)oMq$dKU+TVh`|vMepQx2NhO;jmj%{Ch zx)->qjW%zYxm%}Yy|YjVDGdI-j@Q<1Njg=b2&(^Or{&{VpFU0HmWQs~^K znjZuCG#Y|M-91xhrF^&ZSwDDIFrCNTZ~m>WFTzmqWaJz*Yb0*rCt; z79HC@R2Icw^$%YDM0f6eIEUv!(is#SyjjM1|O#OTr%&e-w7?rxXBAtHqptS3-x3YKpmyCUJ z+TS$Qph)}ijDIEKvzV+iruueZMF7)jk9Xq{zq0eI^{LN^0 zXYG{QLgiYhV&vO~JZ8^Jb8MZXcPZqbV`#Zix=_7Qf4)XOqUXbdDK)hfAWzR8%^L=d zAk2JYUYopR)+(Xh=icn*JN58!)U*PHg7<=e$iCLl$VeNXFtypYM*{-fF1mNcIgVTJ z_+HxqE9qpPEvU}0ZGgU5sNmFMeZA_;d};iTvpx%B_6X&q2n^(5-tUmTV|21v8<^di zEk?ZO@(!4LZgoLWDK(MQ?Rvn{$jq%!Rz7#S?8D$BS{VJ9IRrjBF3vgA{f`GcxfL-e zWU1Hm#B{ZT!-!lpswkmF5|#GWtvq~P8xxm=A=7*I>{((fGKJgpMD|eXqIFqGNvV~C z9yIjh1PQo)kIZF0&r$W5(lG~+lHj>gLx#(=^tL$4Pw~LCLR&F!vI#%n{5CKkW*4KU z+<%Mr2p$y%*xrxk7&WJVKP%b7f%d3kIc z=ue&EF%QFHF96|l2w76A1X~Fl#m{o{>+qV~jlmYDQfDD0dZP|XG*4d2R= zA)SkI=XN9|?-=e#?+;H_{*k)& zd7s3F={ZM(H4WW8P23ba(Zw#zU-75qQ=zIv`d9No`4rPzw)VM+sc!;)(d0;RiPak{ zPUP@Oz0cmgO>!w_4O|j?oD6qR@Gqu&p5*cJ?Pm95ea!4z7_sVn68CqJ)dSVX58gKS z7t|0If1KQwI#zAzzR}3Wq-&HZHOerU4gjI%T{i<(0NlkHo_AtjUiM0af9y1M> zJdu($KJorZ87?gRd9+u?F2~oh@35g)GR@m1Ty$Aud6sS z>7OspU6`W%(382mXYg4cEd?^l6n@2QP?&4G7;yEM#?kv~o*K$u22z7ML=tje&`oFE zO88ovO?SQFiO&1k6KC_3OM*)|&MbH7c~13yvbBF+(QaGb>60ID*rzD;$ITNA;S=8X zUJv-D*89D~4^eCszG?c5k{5I4=5wzu`;=U%EG@bA!Oe&2^-P6)10|R1+LPDI8zwG# zQTLxp*?evdn7f-hbH)jsyY_Hs12_J#iTZNod0$ID)YmWPCm2bY#^F)>A(d{?pkPur+rd3V}#LSGKp?X{;;%rdgaoQ8}y zBVR!_?~xzjXu zeP8W*r=C|<3fwET!+&X*+~=Dbt{IV;E*ud#G1BRg@7Ue{>cr$^=Fy@VipaAn6gO^kj{Z7- zS#K2pHSm5{+0=3LbXU69d)d@7RH`jm&5!UDcV{gP_>C7*QluErFLZR7zUf-*_$HBM z)$-hSp}JIgd1Q`n$wCm%@0;O{X$lG&X4Nv3f;-Ye3{fJ~LJFUGxWd^O898p=mg`2j zLFmbqf`a6iW*+uO3kQe7vR>CxQBzw$*N=-jKO55tiSfWrDU_Y>gRhkJ`*oks%5f{X z=ozSi+yy8Fm&YdkZGV^0#1gBS(Qbr56HyYi=Xj00y@>Ej;K2xPmA0wn22pmWh{w|G zO3enJM~@#PNyx)dft!M&u;Usw$rk+|lh`&rjDtsW2yDa@+AdwXBz0Gh@|>U*M5?5o zYd>U7J`k_%D27*JDx{wRfHcsx^ zkRPw{%8B@%j6ZWwsj{-L@av0lWdOQ;3igRy{G^bO&@;=|^uRPiBPS04O(pXIS@JbB zt0^rPWB5Rb$nS9kXtl*IcUJ7I%Msb4Sz7I|4>AV?t*?JNFPCAs1wSrKApL>HSDwpr zbqDj8@5iJ&zM&bI-UX+##A`>4jKW$$qj7ZV)*J>b6WDQ%05b6Qzg$sqt*ZN=*e#OO z3`(A#zrWH?-K;0@!o(%-kX`x*c8!Russs+DWEzay6*1TFuDq%(gl{0)O2y_LAmaf< zzAq2CMI7f$q}}l$h0%Q0Sca-yniD1~DZb@5pLm(iX?t2JHyU1Ng!>B(8=x4Fo_;B4 z$RB!l>JI1kYXVPn?r%N!ed_)m%WV|d8XZ!#=9P8H{|vh17mN1~4N(01cP2o8Uj3>i zdkef2rP~XQdNbr@kJUGvbk1E|Fo6lG=leLGYfViy{sWiX)8Y$<(U8K6+L&~9v$HuA zzKMz2o^z=WbX(S3=`+7Lm8_Noj#3Bs)cyRyaxs(`MndP@JT4?W8i}mF9IM1BDC^98 z{zAG__bzD(6I-R!s!t``7KUB7KXae_V}Enu$-Oe*mxq3ib4uu(?EP>uK zFACq9o}HaZmS0WnmPaNFA91(KxCQWab}944FHeZuU5RI6u{!r$aQ>Ch$#}PkLxWdi z)?efEp5Ss>`lZa3u1DdURHeRpj085 z{`HO{_H(Nk#$U~K)xNnK@w}jDso7hGLuZb+DjAHIQ8MR@ zy|G!*d+TQm8=Yym$|Z_APwo5|xlVlHwOfj5lA6jRTu08QYIvre<+R3TRgRqh%)`@p zKqSgyaygbsEp}6_gWOI>TF1}Sfa5jF!=LZZep@&y;K8V@*SU{(L9rRav3@ z)S*id1+e{x%yv>5xfLf?8D^>a^;X5(S8m_-?#2!M&TcPGiL0-W%Ar)lOktQ#Y4qgg zmv^@!_s_pO2{G**U!UdayC3Y1#HxkzS}u1E{&6<|>6a@zlXhC~nl2L1n)yOd3KaQD89A70048prSymO~jrxZKRZBj6{OCtX z!-VFr?da1JD-b>*tfQmZ@fyo(Xz#(mYjN7|U+#;zqy5UVzWtN#0LEK)%s%Tl7hHa# z4ET-k{-*606?r1x4()Y)zo6yi^UE3Q zDg@QzD%g87q~Kj+g?e3`Xq7#tWFfNxFw!_>W= z#X4?o3J|_BCLNj^6qyt9@$rFFt_>-wN@82$V?Q~_#gGY$6mtmb*+d>a`Qz=|a*&iQ zvRgErVFv)w8o+w}dnZ)>0BnZ0M`|4dEVd-XZ`Wp-Ubt{U1JWu0+$f4WR&0Vh;ASwB z6LGfV3Gn`-5V@IzC4fSm+p=lQt1C`25+-Bc^Mxm`LIP(2@}+$V``p}|<>K5Vq^#$r zz43tn_98pD<7dtcu2w676R@MBR7FJv3okl`r&9SZUlzYV{9xpCj?&W+3(EH{>I0&h z4RvoDxKcKW`%eDyo^f~Qmy_c_Too>$Z*qm4fd4j9=T~%TU8aoK@X~ZsWQevvWh5;# z!?@ehO!mKvCGNpdoqwD!39#*L*Pg%gq>V#aQ>H829aON)c~H>Gpr*zsC|LCL;%OPV zvmqKgeL4L26LOn=7G>q-=>`v83!l{3Un}A{b&Sb-c|PWkn#|#+%PIQacd*7AYzp6H z*>k;b>Qle+jo4xUXkWkls1KG}mv->5RnXg$9>ZR=AkS4!4xSn7TAtVPT)Kp{XNLV8 zk{+p+$M~{A%ij~-taRx-EwA#-ilx!u<3KXcD!Usjx0jrM$$)(LazE-DP|BB(NbH*_2>3#5K{`hyH`5BF` z-uADe=5uVH$X43rs{crmUhzge#E+k1w(DlXgqbXNZlBuI^klBkaY5<94$b^|+qky6 zbIu8)hJo3!&n-79FXY5puV?EIj~_nf_g-Uqtma9fsAS6Ei{+5F0(N$Sk;ucH*e7v6jx37R74qwaLfD5Zv>spHkK@Xl^ZsZ za+%3K?{rw;ya|7gb+C}POa5=9NLV)rGCk@p&%2)@1%dCyJ8OlVtS>V?S7=I zJ=nkSW4D#Ghl_&u;^2nQ#q(yt@kuKwJ4XHaevFOBZz@mrFP$9;3!khUYTGatvFx@s zM!qj0F)^vb8}sT+Tj{>?fkvT1WrgvNNz(b3K5@br&~+83#G1Tg!##;*&reAc^MQ)e z)meojza}o0KeMWSdd7&CoXLFmoEMc#=s|l+7xSJS&9OTfyqK6+G#1mPR~5=0fu8(y zVSH6db%$x-scOs7a+=U}YTl!I2SuXZjviQ+p+3|6rUyX%8G>B~t8HOsmbHFQTlxG{ z{n1?U#~rF(vpJtxvBc8~v)W!n>k@QaOB2az7OgK%-;p`ZwuU}Xb8l#h zl3u(j57=6XZ^QP-Dh2|0IsdV31w2-kHP8-b#@J2f`;j4A)Vpg`I1<0sXd z{KV+e`7cV@2BENmtp(N>XcTX~tc!m@ATxugDvP`}tx0g(Nd7cHjyGD{VrqzaO1*7B zy{#rLxtMw9K;%AQpNo48-)L8naAH3D^7LTNRUOG*&`M!c`&e3q_S_I%HQ zKW5FghFZkgo#pNh_hR0V@3m>*)DjoFJZYP{ zz0xvQ9m~DK_$@0dilt=UZP0NZF<)mV+jj7d>E(jw2rIoSpg@(9Bj(1~Xt^gqexXuj z{^u7(?E1)`@!fcPhR6nkCpvC41p-p7oZv!48`}UwPjXGKH#nnxCW=~zliEbZ^h?(G zs}J3sj+2i4-)+V`PtD4T)@JUykh$J2lfF~^T!;?6Uszk&y6Y`=xevrkS2ey4w@s5K zr83oU=ioQL2bEqdYkDb@(#D%#^&bC5wgD6rIfkL7ibdSRs*)u<8Cka4Hy*HsH(KA> zVEo|VW&(vcq|gL0O<#K8tjN7mjIWtER!x4}`t-$70b1X%71AcD@94?7xupO%dwoY*L+bpa zku44mLoPHR+^nIdc6R&q2<|-e3_Jq}?Bv(fb}#){z^9r4mla{rt@eI~>AQ~Jp?SP- zbzoN3;FiqkD2p ze;JTCH8rCjlS?7tlm32LT1PWoOiImyXqX2M9yzt^OgaxQuN@WjO98niP9Xqcv-g`V zJ?4I$8M_2~&8I+N6}Xvf_93fxak?uwICwYB^u_^mJZ=cvkO;NI<`6=%vOjJJ3F~P6 z-HtTK%gV@nL1qEEt#gId!3^vDNG~QXo)I0zs_#mCRRIeKrDY;4NC;+AqO6lIXqb6V z025AZ^%yA*E-Tv^$~GyH-EtNGlWv#=)W!a>MO2g)l?zYRk8c+xhoMV{I^HFB(p?3- zcrgpXfh(jd!!TQ$3%QH;Y_+-09r6gKECWrh+|B1jmEH0)1CA?2tirz zE`twSZuzXtRt8bVEPyd!${;zZ01cgDZ0)7h3=9Fg>hqph@r6j-mg|i^X*{Rg8D^mS~U*i{( z8JKJIfb^SFQhYtXo{P&HD~p_0sXs0udFWc5S{*foUOexW@jfHn`yuY1`g=r98s1YH z*`uj4(7vhVHxaAsY99M&Ys4EJ{0r6j6pSBt(Y)b*k62VFo!0)IW)S8YgawI)L?1t| zj1;Y(FO)~UP+TS!?0Qg!ze_`p6dL)@%bGMrmRBXf05AeS4F7WKeIp<%dyD-i+vH)i zfCG)|ANS=Mf> z;3G0wSoKTCW4abrK_68+m3Aau>ipL|bnI|?rJ!EL3FW4*G#Zwm2R$c)cN&T-)A$NH?>qt!)EDm5^v@U>w6LC!g<8w2Ix+ z6Rzs>eTUgtor90)$Ayf3{Tz1aJnc$}=$tR)%;|@$da+l-R9bdVb=`lb6v~6kC3gQz zspJ7wY-~SUj$L#A*#og$E9K?Q_;nkMQoj}5iAs?a&DUQ=@gYB5%3kCCtJ5j+FCCRI z{ECC$P9l4~y-TvEX2z^Y8X`_fwA6b2=^JlPl0{ODwT8QUK{gnogtVG*EPEAB44Vk_ zr+LW+Ou*37O0w2OjJbP$a!vSF@WD9Sn4wn*MZFEMZ_9n~{P`h+>FMbzJTNmmN`rwliXVGUg*t%AlznRfoC^A|1vTdcQM`#ha$AO;|oA7BK$_lA-JCcMzA z0VqfuG%8I;^NfQh`Jxu0>Rrj7b~wPCs)O~G*`cV$%awuQhpPA!R}4E^2zB!IQ@XC5 zcGA_7h_Wz#x}l=|h=R%I5RRA+v#}v5)h~3`n=w!C4A@k4?>`GRu9Q`EqS}AoIppt@ ze>~jn4A^CzVnb%7w{PsGx#B-Pt2r15UqJFont=m+`PslbA1X1@+?hD_l~@ zQdQj_uNn?XLj8Gl7P6T|P0+h1Lr3*8zXwFhAZ*z!8mi#riZ33i5M+`{rK;rCr?d7(;MsB zS&j%0NB9vO&Fs0{G}2WXfrNkvq60*r3B}9%RWCy3C^1iBcMs>2#$$-(dIR#A6w=x0 zqVx}pO-^c|;3Ky&Hiz^k5)%6>3_hZ|hJ=Qek9Jo&;mEK2|7t^axpc>V7JhH@*8A^Y<#dg*z`N9t5*}HAKFjNx2wP^CFv>}Xi4!pYRBVOka$i)S|Gj* zH(Vu8+uOgsOi6P+I=b2V7*a|wfl@(l0MDjB$JZoUd)T6 z%Ii+QPQd+sCptzgd+a+zZ*j}Tg`^as3qgO%4+}fg@{y-mDE{bpZvdZ#xZ8XB`c7Tl z)mOD2jvE>qi?vGDAAv6m&Y=JPWPEFY;?QM2|1|xZ$Qrq0q#J5rq>stSH#}GIXlG`F z{wfvMTioeE9zDh{>7w5mkM@krz(H;s#bN&!d)5!o?nCsGxFkG%`k_TaRCK3?MrFd1SM|5^c*fOH7lAyKgDPo}b7j+Qv_lm{ zNdo2#h;tU$4o326z;5C-WO1=WXuRk?9W;W!tc;9|LYqj`-U=CVwibiY?M0h=gRIG+ z?}f3Woms2+4IUocl0a$`3jk4&`i27%8p&t|68B|{4?xIi3Oj}~I5{{VN7zbqYwaB! z!g>m$jSUTOdNWPQ$T*uV4%I#!0RnWABKcdwSgBTkngfg(ppE^w_a|O>3cY&sW>;3T z6`s=T)t3l`no;lHAoD6D8F8FEbqbw7C2%I5smamNGT`u1@Yr#4bDP+|(+@j22u=ZB zU@Zr^aO>hy+gc)-XJ!umeY`5sq|T^3kCy^D`enxWB9*3QNB^N?t8t0aUKp95H>6Q~ zsTZ{OvT;B`y5o$K5LYHssu1hs7GAL(9E{^%rvB54*|O;C@ciRmK#rK%eVs-Zihuz- zc2JP`G<~p$_MGcKLrdRtB0|$E)ihJL4{8r>#@C?dEy{UNt``10kk z7e(kJ;yr)S!T%uQShh91L&%9L5R;W%i>DDwO9~2H$62LITwv$lR#-ntZaXZN892hI z%RbTNJkQRS2Ae-LYHuA#UE6JHH4)K7b}8)z&egL8Glm%4NTq=e66Z=lRCG_`3nb0n zNij7xrZ4PZ3fv{|XLDxs`(&Bk9lcN986-$b;M^WG*s$aLM!YsjeL=3X=#GHE`C}3d z9RMSYROjCymNrQDuQ12LB+on{C#Ufkj@$1fM_Ax5dAsicPoEsQ>=yAfTz=$bpvA>B zka*Gb@Udfb{^b<7Hwy|1_Rqrxj#$8DHIvkF9pT{j!h(sHvkD4~8@}6Y5loEUM7t7m z`*rMoF(-T;nqryb(k+KrOZ@$%ulC~XF^2Q!iE%_QFx)i+rPyk}>u}Usu=F4i`f)Mt zj;hL<>gX!|9mfjjxUgu?AK!IAP;l?d^n5gDacDk>InNhpG&Bz%j>`6it90K^H%-bw zjE1nfCP*X{nhJwt>dLj>Kyo9o?x$n_hp0D6c5m4Ehd|H<79aLIn~f!CS3WT>y#^$d z__9h%OA~?G6JkJ(AzUl#8@c`PX`sQqg>preV;qh!WIVJx&Y<}r{(_q-=Ooe86L|x! zR$rX3FW5ra&ndf4(DL~#NVBoAkz19_^6s^`67}28(GZntf4maRRhrS^3_2TsP|gWH(U7?16tlOw4h45#i0aNv#6+*}hPGgwD#} z;9=us{NquM{_f%RnPY@e{Fctqqldv@zAb5S3#}-lQr8O1?3BLHV487{9j** z>U%^ym#=L9*KgsMkyrU?XM9>39Y^;_u$5K5Bcw}Io5~uA$Wij(1dn;LtbbF zTmdi>*H^y{NlC~2HaZVeV#D;tV?-y3Szx1{mP!&pcdQOM8gU2%&?NxRvL~M|2qqW? zqFkc!AY)2IlG5g4RrN_Qb9dLzkNk-=-88G&DQP{@Sq#soYuPFU9KhPQf?Q{sUdLg# zgXmcejAcq^a7_~mI#6t3J#GKMzplFWb`$*$)H?J`ng1kL9Yn?&`E?+ypPRCIMqS#xOje~9sdp??I7)H?pRdDo1bx7yFh zmBEy1g&s&TPHcy2$cP7}EPH*~MfAxg`Ht!aVd!qrMxxh-B(z~WnotrAg*Hy|P1)rs&lw4GLvOZ!D zaVJN#mv1=)!<)@kX=zq+ZAl&JLc4#it0WxkC9HoRbf`iEdq>qv5#AXI2PbO3(g!o~ z>@qaGovi}BNCraZjCfQ=&b zj7VHVA*g}`12>|UGwK*%HX#`qgT#{!H^^xE`57f>NUV)(>8Z&Wf?E{iC7;R3N#ZF& z900__V1yqjteaq%F1)8n6-^<$oe36&a7Cuwea#wkv$N>hNI#8h<DcZ3>k&{yDQTiz(A(f0Tyi#|!#@j8-0Xa7%IT z^OPA-RA;$<_|ui`!Zn_C3F>uP0vdK98+yDSSKm)wn74P0{so>$j(h5 zG=LIjNURiaMf@9pnqNatP5vNUCfqyDYrgPVPeJy0KR-*imw0VJJEKUNqkRVj-JgWA zZxTd@2=~O=R@4uC!YeSprPHus5`u9HO8lE+)ipJ*Nm=UY5wv(-q6>lzM&xy}c5A>Y z3&KoQY56QwR4%Y(OLYr|mf`xUhepYbc3Sp0|N3mG zr->ya@UFBMae}+S_Z?RowqU@=F75^_%+k^g#0ap4!;0&u7K0R-4{%oUY}IlwIDnBy z&?oFf3=4c04bZ=m)(9Q1Ts0)pp_mb-Lg_XQxhNr_fEG6qF5sh?P5VcPI0kPDkO7cJ zCPVvBY8s3T2*(YBBOoM#Lk+Z{>=M-Dq_uS-UO;8Z|3@b+r_OTl@L|e_^*@TxvllfK z0B`xtV*`%_XO>Fu}k;-&>__`GKx*z(V!aR5PGVB<#3ii?^Z2XJ8%yHa@1>zkPk zXEbB*_^WHCng1V-xe^TyX$r7GR&X9zqbCw&d&$X3T6OM+5P^fxARt0BQgkz`S3E5I zfZ32@Z37#rbUYF@1LF)y1KGKlhWLo$v_( zY)PhFa&sfzY4<&ExZg(|y7X%!H11l2sDlYlNLkqgb?6A#{>Vmp4d475(b0VwtNna2 z-C+z^4`NaG)T7l@>N~Lu*$?AVYrYWtKtRZ(Enm@wepC^rT?tL86MP1UXOC`Ijw}}= zfw-FoS$TPxUfypWM2nS^OgiPM$;odvqJoJd_^`1+S-*ZgF53MuBm27Kv0NZcjiTbc zE)BCAxKAOz{oO3XKQ-F88N(KUK751F!-QRU{V%*)60mMRO#V}p<&}0hb4k3>bC2)e zGh8Ecln4wq9{O-XYyZA2I6A2h4=O1)(2GY*XyP|&Z^>1jWPfx`vtf(Lt!?k=T{Mca zR!?)168_KEji~=aip1gh{-9^4yr$+M$G)lP5O=a<$82>41G>@~reI3sVpn2`MiU-H zxvstpb@K}9ycMR{l(e+-O>k8MZ(r^#40US)F8^~l8*XEzZfPlq8J(C*!Vo6xwh}j! zNr~rrSil^GO9CnoVT*M|gHj7Y#>2$KievCy<3_Lv^uZYNI%+^B1-6NR0@o%YG{#a! z=T6!(qE0G7l7;w;3YAJ*re!t{mp|DolB$oe*dVAr^7=-{$NjY*UT!(-(uB{k``fq1 zp$M%GpgdPvG0Z1!Brny=2tW%kF=ll)xl8e&vOGUk-Rc=9IB*WIGb$Pc*99pO(bdoF z8pVdIf9?jWm&kWqh=xR`9SK?NS-5-pofP|8}uUN_Sgcc?m zcQwW!k5s4B4C-S?tZ8Gr!c)v6SFG#JDu2PhRYvDp@PANpP0J!nD0}|(l@c-L+^sc$ z6E)iCIIpW&ED3W#JaDXx2_z{ElMnGV?E$uV*Ko$mlR^3h|D?!Dn=OI$G+;a!+4hFlkaD)`zc<=;qC*KjCv#`lGsydhgt}et5Z!SIzNa0^-rv5lZ>NOO+nCxLSA8Ffy2Mft!ewf%UY#Vctk(0Bmi&Mes z>|C}urgQABZ+-P_Ehtd@`t>XSrcE^1)W%Lh2#*jRO>hrUop4=$T)i34!>VftbklFE zt2;nWx|aqbiAaW9Y%F8gYoOln9VC{C$nX)nm4-IFPec0VpE zlB*mk8PM{Y&Us;PkH&%u^P2tbBupB3&x=ajeF0Pb?}BT1Vp!8r;V6J<|4A2vHjhn2Xn@3f5_p8fi6DYXX^;qp!7761HVwSEEO!0< zzk+TK{?H?(%c{h;3LnagyW-ygmXH+)HUjXK`{VL>=thpUYf0wRZ@9S-1Os^Kt%e&n zplsK$YuAEeVm4qK_9v7rD|@8EU@>ZcawW6(C?#$S8r*$a_V$^!FH!LEnK}(NZPkla zT+1$^=XK~ z;yn~8+)-$?w&;twAHi|TwyuE_=LTG10>;)->rOxyfh9>C=N!-fzmZJmnM*jY#55RP z%ewG4n8P-1r22CXYflD^c7pylT#Y_}pa^@b&gBUK+(t4vhVh?IvrxNm)=-I9*w{Y8 zKPCwGG#CgKwlermeE$4bQd95nFjMUQz9L2*#fiYu%2ucwz;S?N!u~cA?Ka>F{61?> zc@Q@iJy!Gl|yKLH7*rOWJFP?YpkXR==fQ?9?6B zpR2%{QCp2w;{U$%g9~hAkTm}9uK~Fd3HnqajAc6QOKdj`2waE}1QwWkWOlRy?PtZK z3|0u;C)P;@U?aG`kUTVX*VZo(vl+>rIGSzEXdregfDRASpI>vFuml8PD1dQ`j&Wg51TzukzAKBf4LqzDWCxdkCRzuJ4REz7}Gd3vIJB{@2WMc05%m+vxkFZ*_E zY&ib3$$^cZ2#Zp~#Dtrykgr~4z~Hz7_8|?)a7`kVXxJ;lw1~RcZ24W3>~F~?kdddx znaZ~tcfTwsxY8;Y6JAQNjKt+bCMH*L175L1Nr7R3{0vW^dpn74UszmGmZ%lR!pA46 zj_Zw_L~90UAti!58>$L|bUX@<#7MI?7tcElpC<)|3@lt- z)1i2cUz*sYKLnpD^bl7yLsv0je@|#lU>(?tOfDH>Lxnnq=5*|1gw?a7qv__0^j$o0 z-!k($p*Z#EcRo}neOGU0GRtRc+9LR)fdqm0eah!n;g7;Il2W7j3N6|0lP4c(rrJIw zCi=hyS{pdUkNU!ng$^bW?uhKe_(hrCD3&%qKW~M}!hdT1F2*#>VUC#F zJ5z2!!0^eXB`KLzGm@f$S`CI~FqkGMPJG$2Z`T35BbI=aW)??$B-0if%@@~VrA%Iw zdmKNl+r$r6a(P8Z2VZ;q$MxSULv>*8P#w>BJ)(3eGfZ`_iSCDNhcHKJ}bEf0cX3^ETeFx!Z@e_T?Wt^}7>_4xem3vH$3F zG0mZolx;1J&oZcICpoa>b>4e5n(LEa&S>+fJJY`_jo^Ud-4FFG(Z4^{fBv#D^uTaa z;=A^xN1O3kFD}T^FGw}*9$2TTN#+!c&?0HCtE#M^D&yXHV^s!n!VngiRFe=5RGlEt z2?s?t{_C)m%F4-+5E!xva(xM-Ar1+XkkQdmv;Eqd6yPcOF3&_PgNt*aiIfnGKe~6W zxK|Lrc&s~ZJV5*ftNB(y=&IX*bx9%_Y)WTYVK3?sRMkeMDczDq?#g#>X!Wu|*mkBb26pu=Ii({6nXt2`LQGNH>7 zir9*&3rL>SI{0n0CD9FF=t3qgY9BM&+LtDsLM?a=s|x>O_toZVKP;E@nbkE@)7GSn zY&ok9DL0ZN^-WDXEX;xJU}Ugt&v$~HVmQ`!QIFt+VPX15kA|Siui@kjt5xY2gVY5x zJGvdx@B{@_yndZT!$ye(CQ*j`AAoc4MwM{C%IRko)y0Yg^($ss?;9>u#F}zMhbPc% z*WT>4eY!FF4HG@_5h3pWwWfjV%Z;fDQ_1ZA@2^FcI&1#_o+5uw(={pE+Q@t-V>?qI z`smtDO{y>bnl9)y6Y1zLoIj5z`tt7)=xRzI!}@JhM8%Ho#zHP8c9I0NBZK`F<`;#R z{}{^G>*9=*0dJ88wvMiO4Y0SwZ|Hmpg^V=A*wyD36zJT%Y3ulFVnS0#haUD)KQlTI z1pq$O_gA`~7n(O~@3z`ss7BbeFcya2yBEiZYYjC)sIO?Ih0Mz#A?G(@7*fqaXYjJ*3gNZt<077(TISZ_)ry9eT@C7f6; zh{hJsq}}0o6%V;XhYrELrdSvjas-zq*eP=3Y>MICNd%ABE@R>mw5aGD2O$R5Q0X;G zmne|zw-VmrpM?7UxE&o`iRJvJP01ZqORD&!S%tL8t_$-Bc3o-U{|ub|Yk|sl64dw8 zs5oy>gRjTMF=0uT-q>o<^=ERFkeuA@uDvD6s3W9NCdtlJ>S&bl{-MgiV2XjT+UV*1 zcGZttiGxi?Mas>E3)v~d-@gYz#VH^q^|WgHbr}F&sfi-L50BJ}GH@C*!lrtrUwDhY z+7~EWSv;Cje`P$WCPm7LTWw%NP`;Ry&}T+gUa@n>|7t`2{M(PDw*k>R{ICD#H$&VT zW+Sal(!V@Hp(z$Vm9oW}>OzmC-~hqt$Le?UJs_*X!(zAG`eI$BPX56{7X=O`N(70j z;Rw?*H~@ALfM<`BD_8w}Iupw)ET4VByp5*CxRAX8lB4dNW;3F7qA3y?(8A?Ks&)hx z&o)~D=o1r#S!3><@2$5lO`0#SCCrSN?KVsz`|pe5?CfQsg8x0?`T#w!%Zt!Ssz6;M ziCig-0}`3?kbJ?E)iJlDy?t~%e))dZC)EM9<#9DF_Cg#@ZtM!!M952v+<*G?W;usW zhQh^`S03OcjTBGwDyyhqZ9xf-LtflXpRCQlBAi9)!gsyUL;4QsSy@$R^V>Xk8>Jec z9InOnu~DZ<=`Y{tdQ}06gQ9B)GC@4WCdZ&0BV`C$Srnu{S6YacLGOpn4LqWC^S)z4 zka&#;j}-u)S{J16bh`C25`v^NGmrpa8HC`03dl2c1k$-Zdv@Z^f<1@FW($JN<0C;K zqJ3D8?+~Dm(Z;xkIWH>4j&rcEd>E|C=fmpTw9qB=N%SfTsB|;15rL0)@x5U8<{`W7 z*Ve~8Lamti`2W~C6L754_w7HWwC|--w5zliDhWl4HcO%u+C!xcNfB+@RkTrvvW6+7 zNGg?S5kf-JZYf2P5WSz9rfGiv*WqZUnd0$%?)zHK^SsXM{fFmAUj`mkm5y%=Ll^UO z$AJ~eL6PImAJ*FG5o2=L@|N}93Zt)Q=Kk}B*8kIC#f)Gn0?HpB`U}-%cV~1Ux^|)2 z+Pp95Gp_cxs2Rmt+f~UcBbNZviGvfXrX#Pvy%R&Z6qZXthbToKSJSnl{w!sg*`}Mn zi*dh`=Z!+ebT!~>PxWoxs9XKsq3{u-6+phir+59Au6||&h#mNRzJz;xY{DDMd*Yk* zf%h^`$3IOBuu1Q{wcsIZatS$Zjy7W`?6}Ci>4}j~-l_UpkoNrf(BfqxGe^G`ykTow6G$=HCd=69zh-#jNY~(Jn6N2K$HY9UNwO?8iu-ZSMTM z?VC(8^kKFJ8B(9wgO2Uu9eUyLVc5s#*E$UtFyL`)1J{R-*Re*|+C5?_knWGQ#xuND z2m+TwhL$S!-6dk@hK7aL+S*zn4+8zLQ1UdJ@oen)o^3q{ZWc%j3xM290}6A8@KRVy$ZqvBAn^c$yW zk(d6gt6lp|S6W(nX-Q7>k2$J0Pp@rDTlMLBzYhAn>~2JLA7y4`;UVXx-as+YVngPKH?;qA?cWMoF{B83**FYW#4!2{M!-N9&OoIV4T4IgPk zh^nj>{H{eJmjapDnQ7H!*oZ-cFdOUo;2ZFHm^6HG!b z6a4l_dnpDbmT{ZUlstGagU$4Ixyv*9E1Z0I)Wf3*&5PjMsL-(h^Hb+xV%soMF!aC=tz-d0s;(H3JeX~lVzP!vd&Hl|BAMQDR2j!JM$~>LD zhYz1Xt}QJXS`@&q){oP!D0`FC4=H)U&o+~ZS3OkL-1MHfl{NJ*F9M8it9p0R0)CWP z*Z#Af#S=Y+RF5(zY5{Khdl1sy_|oJf3KU^H(5>~LL4%xoH$VF#2^l)AyFYc~s18l#lat99!1YoG zdSymf>!}R{#PZXBY!rPPQ7@|Mu`F|^l$TomY12IpIy}64Stjb4YW}K8+8HOKpEX(J z@MK3+s&Q`M%qMgIUQRRL{_}?&KO9<#lk{}y2Tsz6KGQ?BALtq1u@7!mQYrtRm{dHz z=b`=o$!9_BWM7Bu>gnvC?hPyv@^*CZo5usw+Uw_2V)W$oGB!W(aZcF6n32rV!8w@R zUamcH2+m2M94G+@ghU&@e6ldUw2Lz}>?H*Jti=9WCp`*saBTjzPGn5YIdcwMWJBzUnQTI7 zc}>Dm&bs(ZRI+PNh?{};(@uk`jalI+Km^4~69CA$Gx=%NonUydhK?)=>xP=zl#}q6 zy%}0TEY|fO!=>`q0yqN8;y_{XeRg0F;g=j@7Ghh~)sD=-vk#JT^GDGI^WD~jZta%<}B2aBz z^{OU7m8U3bF&*qbV}??xaHdv0+m>>2K^&V#DLd-Hm#KME6x{J#5%Uw9pK>NeEjw^v zn;sU_HegrJGRC(1~^8Tlu{sy?{)3N202E{^nZIJ8INFNRKCi z0q3M>E8{zeuPubYbW&a{ibs*xj6&@@CXr_VrjNLnc7RnDLOXgX)E|3Km@?Nv0vCGo zw;Qi%w=kdmYYrf7e7>YN+D#lCA-v*CRhv$7W=fJde`0%VdF|88%q~b>b|Bio_G&dQ zgF$W~!NHbuzgAQn@7)4P5Mf+EUF817F`Lz79uT8M)*#Wh4skS~b*Tr4Se!F6EGap; z4Ns0wif(4FEPQ@G9@@O|9B;LKbVba+pm!WP|1aMZjb_F2T_2eAws~=AZ;*L=$)cfa zZ#|i=3vD(Qn@i5ky^-ydA9#jcHL<>3>(DAcp>X$oiT2juQ5%AW?09ZJeB?h_sQ#Z7 zY8NsZ{U<{04RUlb8K5{zuH6F+$s&g9J)CQS(1~s+5wy1K-AQxR)rgq|g(k4!IY!f^ za!~q;kq!~@u@g=lFCHmOG>g)XkCgl8`=#CN4d2kC2gFKq(Xa!*uB|4R2`H9(tcCiy z$>4p`>QqA56u)%WRp0QGq%qv=`d+3@W}G@0n=ZylD2G1l% z2>Bd2e<&UNIfeO-3 zE?T7d#9j_t1|%CjkBG5(addA`ulo@(zERox7mqz$GOB&kYr%gh8SIumAQuzz=t8wt2{B`kp(4RzdZ*u_ zTnp}`lDNER2!%7haKt+?wt;8rHfm`Td_&toL+?A*vCSf~lNh5Wu|%Hm_CCzFOhm^8 zk7f$82g$t;<-4lTEON3s2FRYFpW+8{q3spBx-8bL=`8Zil}_aETRM#;dWNGGL0(FMS?ytXJ{)bN(S0q zLJD87&3yV)2>5{=0HGWBwEg>uQ$mt2X1T{{@Mh=~)a2*Z8xP`Av<=7r>EcZ?f6lhg zKX1SQ^aal5L#n!FU!L`}1q-?h`Tn=jNqt$0#+5^ABGTnEnE_`p9fmWY_Z~swR?{(<4%DAEC`RCVS|Hjd3=VxkX^k{5)Eq;LC zjdkdvp%-OlH}}Gsgis?~2Tp-69pa9lz=Q$lkcoz?1)>U;vi>n9*+z{dgZtTL!LJI! zLdDB|aXk!$ny0=Fa(5qpYoz{IFKVJsFXCGve3oFSB}VxH`-X2AT|AsUjCd!VizLtX z%gD^6pjsHKx06XYR%iF6C0t*Uqbw=}kco^yd?ZER0GoCcyK|@;%6Qe2T`8jI!ejjB z4g@d`%6sypZ-dO}1u<@zP#su7UNXSv&SE?pfqccS*m23tbjJZc+v~>T@hBx2zfN{+`jL*aE169D>a~J*S%3tFit}7QSXf(6WfcQC z8dx!|{HmyOe)VJUZ`)N|s;|e`46P$uAci-(Z zUG7kSzY`Bdy*{c=w#oW4zkSo{&(5)KZYKZ3%<5|`I(|fI{dCqySxDTau9y1kt!@fI z##4Qhvb}E1MWfJ2r^3`eXs(3(69v-5b=-e?EOu}13UWf1hRX$b;Z(on4W$#mO1UAt z7H;`S62(=feIkg=A1c;$3N2%n`F1%M7Z(K8vi9_3!o`9)-%KG^6WOalwwkwsViNVh zNU!9E<^UU*qRDMGDq>t6IOi@HLp@5ca^>Xqcii{kbGjIkeXYXO&&6+FWyCSk_6HgJ z0=4`;G#&e_DDTBonlRZ2w&=|i*bTMP)Zj z^AfCPWC`npv1h)r6tRFN{BF`~f0fwUjYg`QGL(yfVm|9++D>ttm7 z&^vH;O{-sgb3_nTKiA!|rRYTO zEbK1V;%hbAEnl`+KW{0Nen7L;8xmgZcQWbo zVgrx{|C5qN<2C!FZ5Ez(#3$I%4w>!bRGVBdXc2EV;t#h!m;0Dd2|XtTC|x3|ioPah zCMNqCTI)uk2^=7uVHJ&Eiu4JwlWEA1nM$|H__xxh3)gK}dY}=W;{xjw$X2V%kkc6)&yJ zSDzndq>uTyxobp3MExNGku31pe)!(KsrnNEkTE3RBeZqhE2a@R>KFEXeCG20{VMnQ zLaayGCyo)>L~+UORlvwJ;v47R+1G`F+|d9KdoYk4NX4`t=Qn68?7iQSelLu7+MBoWyx8`6*xD)9vxj(}Qq9lTtwj>NZ(OggM5>#nnLiUM|5GWY z-Av4${`*|j|4a2>y~B=4t#99S zv_zv< z&00SC-1+mn+(RoynS@=LoxDaRW}%>;Wt$)FrE!>gM}HWZL15kk{9NiC=3}g)Av8mG0Irj(_ota$+s6y^n0Atbi<9|rRnR3{x^3O=PqaH&A|1)Fo55; zcNwQPoG*PR9o?s|-{W}sPlv~u9};!88LJa-CH2~VP=s1XI{9wnH$sE&br9P_wtgFm z5mDa|CJM#Q`ugFXe^1SKF|8(mvg~QKVMvZ2;Q+N|Yo5+ME zcOy~>TG@q6u(Z?o(H>;+u_r|*;+_Gsrj$?$qLbwNT`a8lZpr}#^18zy{5ivo8~5M1 zX%+q22QP{#(Y*k5AJj+s(YQkOvf z>N&rI()$TKLZpw3%rBPCN)Ux;Uv)NSW@Rm7Cefay1zsz-8S^hMZ;@f8OvvHp{31o} z-Mcp@iSp3Pt@GfE+T0ZMja;hV?=G19gW({N%2i}XQ>4D(;wK_^mw;V z7p6v?l+{|)rM}3nPe{LiDeic>ck>+?ef}F}f*jk(OsG4xaENW|zJ859>>St2K4koQ z#ve6!hTq@1>n@I**y9v{&dd197dzPnY;$X`gBX$!q`!H>@DNlULL4q8TmFbP-x>20 zdwqng-Z~~`K{)#aDRLouH~wYqXDm1}Y63d7*Er;QorF zmPAT*P}cqbsPg@;T#*541ht0pE4DeyjFG;+zNnMw;xepkb+rjGe(H+Qew6z6 zs+}IIsjcLjU1BHF81RQZ-WRTp%#t9R{iQye&?xR`_L@Cu7B(-4r@0d&g2g!azffbF zHh6OZHP*UO1vF@#v@4DrX)oRa;0y}o0U+e}^HdFEZhZd$LX_|&eNbuQ(HCmd4tj^) zS~_jg&Zd4f2Esb`sP=iKx`f7DEaSxdD!4bx(I4{@kS*VVX7AkeND`@&v|cBtb8KbHs0;8Jc48Zx9)CU4+hmgx!%aFCs( z6(Jcoy>A%bKsOo?u=LB@`xCtDy=4fd!w7nb*4Lci!FKU5Ybdq9mt52EHuS2H^ zTamuSw(GR*7w?KWkTEK#*I&Wyspevzc-!ubY11XY8iS% zXxV=UlKOvfSTUL(I9TgHWlV>(uX1Bm`o_i16G*qzV+)P0kJ7hHN5)oo0dW}5tl!wN z#`W|gU4j@Niz5O97&HvaWLk-w$uamdlJuljGcuAvmVsOT^Oe^K!Gd0=MxohayHCYB zQnC;_2B~cMvK*glkd@eUc&PPMHmfGU-?KgD@UDG9LMKyJ6zF4@GeSW@8gnTDm+dwz zcdVZ+>3Cc8ns}jdNBPJji!8$ssB0$AULe%_I;Lr6vOfn>*#GL)`G%F>zTIQ@W7B$Q zDNVCr{Wt+Wp%@BHzAT23&})xgkTR>kWn?yXlEscmCm5%=qt!r2tbq3rL z90fzg?argz@T*(NMmE3Iw-AYx=B$p;!JbRHB^R2>%Lb%0BK`AFQnW}k09|g53B|+l zCS^gz?_n@j{%j4lt@{4`*PbxM&a6{%tgD)dP(j#H0B$muPbVPU3nrgC(z|;_Q?Saf zKOEF7FX^!NFet03v)KT(?KTq`!y zO&VQ1q-BpV`zK4?iSS4JKQLK?0nE}Tgv&sfmVW}A`KWSHF{xe?zmgny|`tl zUd}l6TSsK^Qn)>Au-%jBK5Q$*I@fpf!M=bBl9{09M>4TeL zJ|^Z&NC;Z5mzRf#u@;~+C1vIMp>}%|fy^+UgZ?ZSTk!pwZg#c}`KKs_? zpwtV~-A|?8WZIVcsU@=|bY?T6hEiED-#~Ts`O0Up?&hEMs;0x6-`@0D902epfEM7{ z4Lj_(sa7X@H(z^r!1t;u9ZZ9&}&;K5Aw5-$H; zKUlQM3Lr}pqZSewJ5xKz%C*LUFb@F@pz;ca|78 zZ5;g|aIEow#*2>YJ*e8^@t<;4+ULe*nL+;vGadHXhjc03aMb^b?MU;!v$Qkfw=H^G zsy9SPi86)(Ngg?IVvksGu$l*k1}2G`ug}Sc5S$qk1QUpc9&z9slF;K=r0?{_WdJ6( z^7GAa@A`dKcP4@%mECIA>m`AOO)u+mW<7HNjz##jjG+;fP%pKWwM}0#cO&feCQi&- z5a(DUw0`XV>Az&O!#qkv&YrLjIui|_fNX-gFU7L(A%KFuq2b##Udbze+%^BvR4@x) z3sTt4J0&9@otm&W<6?_>P;lt4sP3QpKnH4{wOX@g7iyKv?^T|A&2}ZPOyuE#{SH?(%a%Vr168XV z@s&#Co5AgHOR(ey3Q7#I8yP))-_t*w>P@sg_0BB9`rBj1ocOX3=6Zz-9exng>-S*d zz0Ui@vWw!GGrYnGI(F2X7Lp-rF8f`~-rsxd@eF+niT;BX%{R{7R~NpfZ6lyx#bmpT zs2u&T=0Bfa$yu_!%q`9F#ANC3SFw8abB=gll+Z}FY5l3%lKfjw+HjO4r9*Z8lg<2` zGznTImC{EQFboOYRPa(>Tj?*B1Ih~k2Ay!^N+nr=xnfB;6NyIrG;AC|OuxTAFhYB$ z`4xCTN=Jcr32&2sP0y6Z3H~Y|0LPm*oG76t7`6_L43#{COoxDPHYUjFcm3se{MY=U z_0!BkW&HQ@o;C>@V!tWf`s+0@QBeqBdI^jt!0#~{-~A#A9C`RoR0inw_(MAY-)^sr z+*d7>*0j&49c77iBe#7oeQ|JN(EsExq1s1t?nlE8SK+7RVxdV=c$TnT1Lk*%ExbBB zMZey?HT{+=Jri6Cs0wUpL}85`Z?W)oL^^jnJ^kzMtcCjeE)2v)XS>y^aqO8+6|#8B z+zhLVRvHRC@-}Am>1)P?9m2255=1{ZG*s)?I0fU+j8jxq^}#ZaE3;>+f6;|qDVVkh z`JymoA;+OCbaidku)CR2Qz88H9+Q@g=I}4ZR6UL#-&afqXN2G`5e?b7V{-IH`JDs% zHg?8iYUrW2-Oy=))9!d~*nDHBpJC&hG-@N$%*v!hciqYfo)-_V8tWz1c9LIe$&rVW z%@=xp3RyMQ{7~Z?!*$nrTaADFW1A57|ND{A=l^o(n~r*0EjNytLFsF8b8q?E-l?1) z6#O`r2;3TQd-Kx~3~PzXXY^YEt{oQ+#hO*kxe2_p92S{+`|ZHyqS)_Y@s4%~yWzMs|5IG`^H0m$joNpw+p>npffFJq^|>3!pgj$&-#5+ab$pv$ofs zWU4Sg$6ew6%E;3moeO_ZPbuNbQIZ`wP%qA*B817b^Q1e^Q>}FsVRR_J6)BN3TDENI zarEetrHWAlfc}Hf$X+Bm4xtVwBe9F}1loxx)S6wO4;<=v9_3|8y;i6CPV$_vIlLNk zqe=I-3a>9;yM?Wk8%rcMc3cBh035wxe3uc|OB){;>7Vz#)@g0_!%M3&js*))M`tpjh|FhlgCIB1X_A`Pv0I};O6 z98Q$@`ctM(RsWU`4PU+nj&)KA@{(^wlIJIvbc@|qbs4ZT9IuJLM;@#>96f)i?5L%? z5CI(b_O>qGdVoJ-`{4u@M959}gy^rFpIe+B=+?8XE1%ra&OLfOta(4zh7JbT5Hm== zNfk8J!_03i!UIleaCcQNPZ^)#PottHwwE|KA5b0^vbHSPcujpsoIJXzUBU2($QEGo%~7c=;@vh3ZdnpG}5Y|8OVYS)mV@ zF?)8&%jKF`ugdJpajXA1Z|&N(zE5M;naAT_eAy&SK_SuoK}t%N;Mgn{4wb1b$*&7z ze0$cc-B*6pB>arSFRgJ2?1u$3Uf8|7>s#gm8fTEBKgET8{J19h%y{=r^Ggbuq0PCx zVy16igt>=?x&vZw&xsYRS|4i0NZ{X zs6%h;3Jn?gzCaj_F?p%ZrAN~D#JQ1_$wUkd;1xL>_lLLofwGaC?{O3U`Ir?`WN;`vW6Pt% zjU=Woe9ihe*p779k}AC6^rLKA$@6Ptf>hSO)Wq4^^X}j@|Eqb{j*Jz?GVHE5kvqHY zdf6U7fB)IP-}yz{=$splaj45bXKUd2oP2uBG(=KzXIdA^vJc<_W{^!Xfq;A|y-s&( zUgC!(T0&h##JjQ|MLB<=C9@2u!pTn)>Szmc@Iv5?u}yJ` z%L{MONx=fYA()}nmDeXGlaR~r5AKtLSUd*r`Gn)K=0xzi&#;Al^0%_wNtvyv8)f7 z;12WV&kqveqTp-vwLC_2X?N$KLYXmZ);5!}MiR|-(S#^ORije9Rc%9Sz^J}=6>4iVTEhScF`HSx?=}+OBJp*@m24=a{gMf)0$b`AWFJI);pg_e5Hqk zE%a{YXQr67X$7*tp~uPdWW7BJ2)>ME?9kDX*vQDcO@l4Q`NS;sMshqgpcQ^L5ANUB zC&V=y;6b@0vWtO1-;)n7z=kH}%whVmXWMBn(-O9Cf9z`4SX`{@TOzNUt<#Gq;FvH_ zC!*twJ&!}K2243*<}WV&P9^&`ra_kaQ$LT4^#9}Cb+&$NZTH9TZzgu{*ds)4MVpmL z66fLT){l@`A(a}xV5rP6lc?#RZ`4?)`q?2GoJ$HN!_Au2%;q8|UFd|W*~EqhXHL@(F*y7S6!84)yZd7uOe1XJ&p)(TLz-ZraiK z?2)@m(@<(fbDuTCNZuNC-~G`JEHN9Yc8P#a711PzgN4`lTq~=PE$2!#HGXbK5D^UO z!YO-EIo>FA9<-nS%9R1ML4JJs;0qT9Sj<|s%%4fEaOzPj`aN0oZR1+`YY1Vk@0F|% z$CKXZbE|ar@q*>^F3db>gGSxgWYMBCK-qt9){gQ*7gbEXKS`PR%m256uvECvSbmP7 zp$D2Ae}Qcnv&Jc-)+qsq>RW0qYQ8ik-#W|5)pj=EX#H?WDM+W)+G#~$qHv+n;wxBV z5oZ<={K;YtFQ4IZ^zQxp2SqlD%(Mr;AKhAuAC!CVl9ZAkCvU`y;jwe;#l=5zza73%MhdW`2#)`y4as<5hF6p*$Yn#liwIP*1UZoXQgl4Y_y1Opu zo%;6bAIG>~{S%M0){)VXdD!cx#}t*&hI->#fzJ|^1AR-3rnvqnTN0yq+j@z?%{hZe zmJ!##?8nnB@#b6SJxdCC%d>LVC9t+-kb9@basvSmS3kX&##nKCr9Q>UUSlgnnMj$Z z>$uVA*9#77(`}??LUm?@2nxWsy7F!VgMK2>tWNp)Lzi4X7dy?9nO4e!2d7?ZEvq!0 zPCAV@t8p|Ah5F3@&Sf}x-~H15)M!BY6lGn4-W3!KQC7~pV`zVb@Os}m=HZ75X)6kR zS(P>Oy;87V&17Cn2{2B4Lok;$(S-|Fthk_FTRxWzoP2hIG185ay3K`;F@9K>`P@2& z@tRrFx$LEdT;Ou#w$hmhkMIe(efy@xSak8Jf?G92KVov-{u05^+CcH;E4CPi;Iv6w zV6K@s$5yPm6~O_V8U zIXw7`NlotDnqc4~InE0w#h*%?DJ6hF-M{L#K0on!#0tp!t0E+(d?XX#S?DDKpw zbq2dU4&vu9iClSziZnDQCr2~>;>B6OZK;STJIg+({1tqB{TGB8&GuB%q43Hey7T1j z`Z~EK`J)LA;(@o48JbEBH)}>qo8d4znIL&m2f8_^$&2(PbZqzUmtGwza{xcNXe=}| zOVHMPLJSp*R2@vsNc!b5H?bm1b7G!pt;4d^M0`z#W}>j_B&o&7zS(%K9Q#Bw%0ADD zTd$zl$Qxc27YFaNVCV_oyJkIl^jOfm1NpawQ-Znqwz4UqcYVn@p?q<7gL84?aa~HS zxS+8h%Hk*h@9gPK)XreILLn3!6f_;9Gy9Jv=y)bpKHo(0uuQVy-KsQ3@0oG<;6X!e z?Oi&S5}WgIPiNJ(7z~_qMsN1)WT~WY#hEj?6X~?8UF78QJ}pp_cOyIfYH2Wec1!yD zcWzbDqTs|=M@VUeTiW&9i6e4+mUI&(7wf5?hm4poVWIHknNu1{l z`#Hz`{e>$(H;?<{D)Al3U)7Lfqb6a=7~CK?Q3SYVhZWdN z7W8{^i(S?QypuVXmQ4$-2I5o(J?_-8QM;!Z8Rt3H!JKrii{tAO`Zh}x?r_tqUBy9Z z7l<4%=4}YyNJ_%B z;78ro{8m4DQ;*Te9d9@$*b`O0vYn*vqf2JWM7?HGG22^h0R|u^?%sCHzZ$dB$9z?Y zXELqDSrh$vkpb=Y8lDQPy^-=q{~`G?KK#Y6`^!so9)8|@*X&E@j54Fts?TdkQq%A5 zF*)9C$(RY!4uc}{Q-{SEkL?j3Ta~2o`Kn`a3!g%Dt=ROp>1P98w&)yotXlh7hQ^cd zx(;>~Zc#%<-5wry{Qid=cj^p=(=Tq$cN}XmDvQMY@YVW3A`{qXCLM5d%K}y?2QCh} zbZL}c!p3*|u{b@?V9NcKZ)$!;zs1rA?)ot^&_-TuK1$ptpt3uv+a#s}^4g+vpGa3h znQ>J|$K^3ASx?ic9ZPz1wA?lSDCM&ySfj$gM6vYcq;xy^iy&Q1t(rD%`rz?nl@Xog z3kwPsMj*~e$p885fOJ4$;0HFmiyANrj1_yjCKWZ?)a9^LQgK2~VofzwxK! z4;8}4liDMLe&*fRs+?_e2)mqy0X`{#B+xSNWi z|6Fr5DB4~%JqM{aBQBHsKk!1rs-dt#sqIf6X=!?Ycd<Z zjW<76uS#y4dH?%>yp1(B4@=T5m1iEiD2|LrXG>!%o4q~L`V@Du8l${wGqSrd9?WX` zOc&23-Kw!HtoW9@Zb;w0x5bh9kQ);r3PDo)NQ+6gu{#yY7}4afQF*cT*XxR85_UC< zb7YmRJg%(G4WOrKR+07esTFbf!QX$UZc3RLc)( zX{YpQu(zd5m&P)x<7;YQ;719GWrx?PmX+>|Eo6MNR|^RW(lao4vTR-Y(o26+O7`og zMUR!v%fkI8z;_g)1QEMYUn*_=wr0i3l^bV2nuw$RcOM^yG$n-xnQ1APbB(*^PYL+6#UZ~uM+>h3dh0@+$;3--14 z`PH%;Z^Dd1KiUN>q0^L$_#@cpCNhO3+F#U+(bq<}dw85D=5eA1_wWDql6m^b);w#R z(}t7=g@rj@TsxqDe|DVWI69kcvIOZf+dodF+!8JC@e?PcGl$6>Fmosx0JNP$%RA%o z^R+eO+DJ52=Vo5K_-`hn{QFzq1Kqp#3g|Ut{vaud{JfRhPGxsUJ(L;! z{cT8P(D@hrjL%NX9Oe|+&sN7tC+4}jI4U7u+I2d9celxLnNEGDmG_b#bxviImYqI; zz!S$trn=9Fpr7%;K&9yv!2R3hi zKU?UBQYL_jiEDY=W7RME`f8G@Mvv<9X`rIw41@~6?kzJ2{3OWoJGobKOT?Ft9x8&3l z{RMd9PLQ|4gd?mNVPFQQS5>`z=`@Ocuq@(=A?O&Vi@0K85yT+WTN_p7^)GMUtnfZL z!u%<;J8{h>zW{rT)l7w(#5+VYZ;|sMf^V1_@JHVq)s=vRpieYUBB#BaPaSsg;!x^e ztDDY^+-%kJKU9Pu5JPiBrUK$|d|8)L%LoJi;r^@jP2`b!fuz~A5s2jXH*VmTVZO0> z)>&Xl%1rILA3HZBxbpivgMv{Lq$$d>iq%3Yzju+lt@bBa;#-^j-)ct5@pegZ*Clgz z=VpCdEOG81Oe@vm)gr~6-_l=?v55|!Vr;dL6IEy(nNTwD)g{B2@AFKvFU~!le2i+w zh*ni8$T&#)-Y&(Ad$;$uTTNx4IC!u=eWWkF-$l{5BJrk>c{FPAO-YzZ^bB6Md(zjy z`yH_4ULKQM0d;lRen`WyUW|u8fMt5X;4x$7<|9rDZF&|NLNwUcS#?KIpz0+!)l&4H z66VNbRz6B49yN|exm3(bC)tS(fV!ou?~23d<^tbZTI42Blc|d@t+g2r_Aky-6z&J* zkH(}#<$hxqe8tDe6h*{A`0%7*J8yF6+J-lK)W5%!YST-75xd^?eORv8E|Q@B-XF zn2AfZbpr9OM1b!nfFTos$5=|#f+?!cj@vLN&MbK=cpa>$B=np$R`x~H!8i_I#G`j zhp&p?dX-U4wgzURBPoI`fQ;HXZWD}DLtr8{kv)$rQL8@0+Qbf!wJy^P434c)`4nGb z37~im0qnR!v!zAtlH`bUVYC(^GLg@g+cl9C8ay^s9y0b)McJBvPm08O;EJ&m{`mdN zv_l`|0^+lJlozJAXr!64?5IcDghwB{74G0=?#2{Bd06cH^ouQHQa(=q-rc8dj1 zjIe1F9>`{lxE*lr(h^x9LEs^o_g7ub;vI@WGoGSD?-^7DAqRSg0n!P&!r8N$o$5Dn zqA3q53x}`1KA&|zS3okJOkSBDnWFx%hN~__TKs z)$Ixm^KB4gzGE!btjQn&%11qmqDSDyP|U7aI@+cg>hp9m+0O6-?&rzF)TwSCUZ?h< z1VLsLh~S5^zT})Ybxi-j^k5nT(5nwO-T>}JZFn;k5Ib_7ITIMbspp{rb)JlHDr+Se z1xU-52U*hjg#Df8QAf?=VK_!5bdyMK_G1N4YpbMt_r5(_R;g*6PBX{M2m{Evgu*Q^ zp9!XcX*B_DQLFbF_jF`yP%RK20s5Rjug#fsym*i}R)4q%z|RUkzY_OO;FU<$CRMf7>!i;SN8oi!%MM{ljAk{lhPL3DjOSzekcjKI-?|qb6`helTcOau)ch=evAw)d zfA_9k7StR+tbe_*v)X{-;-S)at>!O_%v3B6!qW)z5xfzrd>l{Qy`I*uwVbLU+-n9o zKq?xFViG9vtIfU#=(vo4>wOuQy4V0BdlHeJKtjT+NDuqOpiThu{%X-#Q4KsW!%iW{ zkt{fbXlYu@rjtelJWLxd_--=hfY9hwrW#+r=68+BT}xXnao$R162%}C*iMLIU1Q?S z_;aaiVQ{SItiL$KPpY6-5NqsXG9lzI>H~5{6DxaB@<0?rf zaRcG8gne@zU`cmL6KIv1vIT%s%pf~{xj(R;osTz*5cP3<`IDI>^>VGcHdIscv;Sz@ zU#xw4Iz(aK)m(3h+*$M42h3^dDwgeDSljQT zEM@!A_{7d~PaC?v37erx7>X@~(@BS$mc=DaORc zgMyDpKA?*qO9$>)En@j0+fix z;GpUZpgYg#evyr+m)liDT6WvR2`Fdzag3@$Pg&Uz(de>!ta6(*=%GmXj*M=_BBD-J zi2;J;ny?-IQgUmaoLk5rEnU0UU#o2JI1;7#@RJ{0mX+AWkyeFr!owpYB%fnV8zjnF zOG{BM^|I0^DgYd(U1XbwAT~TbbBcR*D|Kg4!rJ^B@$(#MrA)4@Ieq)~sZEXC?u~4H z_FC>5yVjDzWAt}VGtN(ceE;7wByqO;pY&JO`arA9$DaoO6k?~! z^2WKob^Oaxe);wqQM*byuN}1LpR@=mrb^Qiaun&5i7F{v@RLXdtP6^ zz9D(tN$Hvc;{+QWD-PBBN8MS3&7mTv9qKvWU{2s%l+q^L`t($& zid;46o`@-c(Jbj*!oK-z{`l-1L6(>7HDwtzd0dFdcbh)7EnGy}v|+kN{Pk<*?njRJ za+9eZhp4Hk>G9+!o?Mck&)-(o^cp{thLTbipHMDWKqD#XK_Cc6V|IGq9of1hKfxT) z=(^0x^x9)~6J{QJnFiMtV!5Tfa7FtJq`K(klYhI){VH;9iTIndJ?^UdycMGc8rv6XyM6)1@r=%jA2LNzLMcama&WWCB7k75I2BDe{ z=Q)EYL0K3VAUb|(jA#?*YeYF1+3EXnB1d1t)a!BvX#}?-*V^$&|V8H^eMpeaj=WRh5!-3!FuX3@TGh{0_T^&%pv( zb0fx85w8YQ|JhpHJKULCHN!40?@LYXWs_?sP*a==MW#UOXdOMhLSU)_!ZJ8F;i_ZJ z6+s6F1iY*{D3A$bvn{(n-?kV_Q()B4#-WeOQ%TBQqtd2_$}IE}vaBqJK9jFfW~U55 ziUROsba4;9Xo9PpsrNHIR{!bSC}a ztQk@-2ZBHF)8P4;PQYB=tui6&+0siEL}1a|F@d$>(tN|YbNz*SqrPq`f+TV~E@+&# zuKh|2vm$!c^tJB~F_Z696NW78MaT%?@RW&_JvHXquQ_RD^60QcVw!W;Q>Nm;TZa|@ z92)+a>U6bZ!+Ti&A7lR-<#_w}_Q2qJCC;Iuuj@O_@~`1ffE3%&5=pZZKL(wa9sa;fCWr!fcff@6=Z8B&tD==KPVAkUDe21qA z%}U+bGWqw13M)wZNfjKAIaqU18jcyi>|8xPW^ntGI`kRUGGhH;3mu(r63WIXoquwy8U~gJL52OOwLjoBrVpH~_mhAj zs1gg1X;Bzar~oXf$_J6MOenv)C|JR|f5 zJ*pXRm%{nUn++) z5~Y;Qy>CnU-17c7b?3BciqF9V`udl=0>4aKU6|A|rmgW+WhE8EPSsN^G!6G=U-DM{ zRSYpsEMR|Mw48QAe6=qBhC+bUBW3hqTk7fau|aj2#^z`$G|Fzv{QBPB-e`!6BkGDH zqHv#p!Rl|ZVS$d&cnYPSfDp00kU4Po{CUlqmhIY!3cZ42tEm-bOICK|s`24@O$CTV z;C8XY<$~tEYI}AIzJDQc+r>o zkP1*p0;Q62O6)31a*eShLJPnIfy3LZ$lQ_{Iy!J`g1M*jzz)RdJ zb9axYoRr)n$m5JHHdk+l9dppl)s=ioJYsV$U`kgj6Au|VzkRiRJ!CpP878wh&If_; zYUICY*EadSuW%3uU6FA@b(dC#zSG!0=y}~N!U=VzX7b3^j_kd-oV`W}u=f@HdR=PM zDfq;k9_{_w+$+&0rvlHNn}l#;w@{OIHM5^YKdyob=mz3*8-_Q@9!;$TM@!~g5i=%P z+xQ7SaVR?l@9tuPlX$EBwv!R^a1h%?VQxEnR=Nq_BS6zHU{5EI9zSztjX}=0PTji)3k`~>s83NzmM=%GE2Meb)XnuANzsDmtv^3mS%LvF zz@<}f;^4tLNQjqquu)T@Vq=z22vTrZWCK)gE4^vcrm@IsCIvM`eu=|bTO@2OaESDM z(?>((I1z_Qo_1R&>Ij}0%i1su9kuz!E)LZEFSh@56ym)7Zf@(cvZ(SWY7t(3v9+2h z`t#;p0L2)D32QRcRUBZpF`pcJw{q16I1*ZiJmE<~qdxTRubLXrpn<)SSwKO*7t5@0 zK}veG&P%@FdBv?=BT0@O%_X)UZDUU#b8wqcJaNYUsMV7bnzuR7W^GoB@(J!> zYm~RM$;0h3uwV)kG7TZs*>_*^Rmi~jIiLaBX8dqUz>fPO81QOx2*Ii0C8eG~L0$X0 zYeFhID^kh}qgEfysC6-~_!%tHGD!B&Zo7x4Krnzn=0sV}^))gzWuKfHwM*Ka<@P;E zV3S&HH0Lj&A>V)Kkn)JmCOWfcx3PlF4Y_i~2v}tF;I#ND)27w)y&6u}S;=vM8Z%0E zF1&4i?WfHmIFWM@2xXN`%4NsRF2EtF5`s{tbdfb4jBL%>p3TvnRDra>ifE;9VX|=+ zC5c#vk-0Pj{kUMs1pYc>PdIfi)v^$JpefxFRC@u@B@Lq>fk9@O5zN~p^zmeUL$Y@0 z4ZQON!3W$vU@)0bG#H{y6qI5-TfbOFbc3bx1lIF`Uvh{+O!VAXT(4Yp{*?T=@&ja< zD5`O7pwO_E~5N5NpOls2e+e`2{b`4(CYJQKLy-f z1+}_Nq#v0X#cEb%-CIVN#sATCNz}&2W&cq}ub&`o&Hh@uD3Uvn6; zNqSVl!gFFap>P#4J`N2qpi3+=O`!^+skQ=p_$^{hl|lK5xO@fxTjJe*Jzsd^KTCJSO~sp5}C zAhRfZ!l{Ba!=j=L(TOPp#3qO#33igjvYCew>T{F~M$35pHbPwT_0bps`JLSM86)4; zRunz%$Hb`_%FjjyniP$8jJ7)WymnzR2({?LgxVmp2E7qLL{znW>mS#s%Efu%&kKR! zrt7p;FlmC9Wj-mJS*n)l{Ai99nZ^`_VWV*`)#TvomPBuVf|jtBVaIx80mN04?FW-E zka1zq_gr~-d7w9U&88OzvVO3J*OoUI=LROxY-#GD#G*c2Vb#ZR9uxZ>pvCUqyLaWo zHT0V}q9cPPZV%uBA>M8icv^U{y` zL5>(we0&^N0;**|CFZRruj`BHaY)PyhXXq6)82Id?_3CAG8Nh>4}pa7anpZfYb^qJ z7V;XQIp2EtesXfAxaBhU{3M03+<74cR1Xp&dI*G+^V;@{gieY&S*k>g%|!gHj)=>< z_8|Nd%C)CluEr(eqCzDpC^(GUvjqHSIzEy&KHols#9Aot0oQzrmzI~5K(wS&AxV{h z2w1?mDJbwLCe9`mI90~w-xs=8{)$i+5OY?UjkH-nst^i&Twvr4Kj+28Fhe@~?wvdQ zIA<5I`A!gI6zKdzC5hpiHa*Abk&M^|qi1jbeQt6=Weti*d0NDvdFG-o7`6~O|{`xDbqyB0p?O{<3YN&NcKMWOsa>TXXNSXJjDg>TDl$9y8@ z!hl(9-?G#Cn$Nb|nlE)==__n?&K`b#+47$Apf(3v6sa8eNtI}Uj7LZyq@qzh33Rmj z-H7ek7i7Goa=f68b~J4TmJ5N{EoiHnyyK6uB$TSNT}SNr713U%5p{^p7KE6b$W?CK zZIeUy38(B0s6dr<>DkcOi!fXQQA3H>CPUP$sADQV<>iup4B;+BZzWa2SrCvn;q3kH z$sjv}8?-@2Mk|+kx~{r3pUZsw=+Sdf_{FZhob)zrDqXrvXneJb z58K4$)$2PZ>LXbsb3V6v^%6)^U$8^FCZBu?(ZSsm5bmT3E@ymQ^@siYmTZ{$Lx<8t zNE`vn;R&EqH*N%vyiAXjy+lKo?5m)L#Y2RSfLD=;g|Erkc1?43hLU-pP>t-(|ENnOCA;TM{`Y+N zdUb2x#V@L`{qgl$s?PF5hpB4o=+Ba5$(gPzwa7mtef;TGgk?6q3$1-rOvC3{MVXF~ z)*gJ#>v}T*=tJ1f69Xy$Z|6v*W(+d!*mBEx4lY?%l!KsjlUl_xWh{oa959Z^WTNA@ z2*`fExTtki1wzHSHZOLbpa_d2LPKf(yp#Zegw$759pJ^^g)E2}lZT%xXI3ZneIVo) z&}U0Gj#k@G4yY(fp9WXhqJB=3=|!_aQM?DT8N)}6ILX{Y-V5b`&6iLD(Z_-T_(Lb) z#66<)MVULbo&YI%oB}2R{L!)O#r(;?sByVAd2ns&W_uln_or8hAyCF;Rxr~-i2-Nd zG9>BF9gAd7<(@G+&TA}6vVV2EDe}N#>V~jyMV|(Mj}T4xM42RaI324iCnFQUCd{G(BW|Q3kZo+sgcgo|V9pkh$Cu|14Ol^KdlpSTwe8TMvj94o zu80lbha`+l4V-iK2^n`qw|mRh3K^88rRA@(mQy^CT{@2~UB203`~q#`KhF_8iAk(Y zFbgu0Y99eYFWeTJ`~2Ie0g{`Eak(FjO%Ldz^!o1<&G7Kmj~4$??%EIPR5RsQZ8I|Y zPF;tU(=;cOfeTzQ>pG;iY3uXON)1b7}XL_@HwGUQ*W%~2WbcvC~66W!_2PFr$j*m(LBS;(=!TXiMW4Em?@3w({O5?r?Er8k4!aFh+qH@-a#i- zv;NJN5ntYjnZN}@4R#M7psp^E16AzVeT`5+h^l7%_-x`zXL?8xS^!O?qD7C%5W-w= zI%<~!3OAtA#;y8Q*E;20-N02#e7oFDfyoXr;WQ>i%^wl1Kv4jW(Z+zOWl{(!u@yly z)kmV%cIa>`t0NFwL!t5>){3#*<{^UbfR}jRWb#Q`!Dv8V{8W^K{exqe>L$%rLzPB zBSa;zN~UPwn-}LokdUX8@bjE2Xi?3Addd78oFKMJl}dn(c?wWfXjY%lL1aFA_8q}0 z4ZRSoqEM--)nZn7kEs73>_Mq1C(Y3AXRlG|bnM)~K`S2jmDnpTs*qoC4~+NUxFYeM zm1W-OkGRwRh)gHHx}*P(ulInby8Zvh52+}lQk0C!YDkEbt&k)kR7gVhsLbqW3x%R& zMcJeh$xKKRLPAC8nzZ{@`2t10&Og(=! zw=LQJdK%j)j)4QD0R}ULY>!8TKu5z0DE3@5Jn0F z9hu$W@cD2Mq7_F0y!-$Fqnmvu22O1|P0jWiNYW2M3WiyZjCBOqh1V-W@l(!CjaVfM z5J)07WbZ3ZxaJW$7zR5laGyN5DbcAw>OxDzhv4~CRmUcY#0M%?LWV-W2tLe4tP_&R zoHJj1a2Uj+3){AJ>)VTQ6Re0kt@He>D1eibvxGp60KIAI!JCjzv$0=w_R9W_k*U$C zt|-!y$d7?10~ez7yJ~!gN)Jy;?3oZJ%GBknxwvHi)*j-i`I77JTiu6b8-y?5sC*p{ z>>lL)N9ish^Jx(lP~On1yd8Eb>oS}i_I^_W(!UBPSSek|0(3wuQ!vD&2<%t@$Q+1e7#slO-Yz!=g=skJANyt@cP}bM(ixK00AbsyEE9PEG#4ab z5GWFym5^FMfOEmoko5w@guV`LWGyHxpri;3qYvB!dIh?NBd9DEJe+96p}LR_#r}n> zE%!1d<=t!1eq^j@lfIPP)oAz=A$Lw_8=N5e7qYJ-lvxxU1q2~sA-j$t&O!axS=LOP z1}?{N3Xw!HsmOLkeQcQt7zGMSDjA#=DcT%SPDi_9UnA5u0r4m;oz`Dr2 zM$yJVQIY!9CEq|ZV*(t4oY8cN)wLmY=bJcbC@v;xuB|8k)gc!K@0?(i|7Wepy9lAQsO03Cjd z01C21Bqb%yg;7O{wB&o}ffx`IUNZ+;I0-c9N^=B$p;*U-XFejm2UieHF!{Jhumw5z zoL>5tLh4|S6m8eTd;ACS)AjtMvnM9r%=K=XmADSP39;kfz426#W<~+6I~GcAU&lA=7^W{8?x20P5_Th6d;_ zd$2u1i@`_|85b@xTopKN0yDP*g}}x_w9Z&CPnutL=G`ltMX1#ejN8CbwJZsp+IIbH zbsXcuTh_S-HL*k`>CMmqF59V-0JC&niI_%N$3L9{WsJyv>O!C zDAdfv4&8)s{N%;oh>8vtdKeZgar0ge2q5t8F8KY?@%ExNq$PZ)Bj_Mp8LZJjBUEy@ z4KWo6FRVj*%!k<yY0!zTcx6tFewJr@++ zC^lCpM88H`%&N+N$%{Pw(u!y@PH0C4Mu0?;`$TqvId8{B(x?)ym}OY&oqi9EsuG*@*Vwtydm z90QLK%>um(smMt=BD^k#4NpUuceJkZj;PHD%7~Z(KHlP5(p|&VKw2b7Wq$r@&gV}# z93}H7L9Xs3Y)R zZWHhj5bnZ>N$@=&cJ@9o`~MBNSSrk zjT`CSP%7QI#rWuvhOf2G565!!dxa&J8D1~S+bi0(GBxaGTBfogI9wR_IkgU=frmck zz0Z|GE_8q0zDzRJCF}r`%DeKOu#PzP>6f7h_fGf>W0E6;llqN>ABs0X3N9F&<3l-G&`!K9`#kSP%6@EL*$#8Ch~yqTa& z2vA+p1pH`-!;?fCky9ivc+rC5mm>Arvc7BLX( z>giX0h(Mq6tvKm ze8hh5+XM9HBl6BvRTewQkZlb46Brk+4P_HRgV$5_y z8_u@-jNz5sT{vmi<x-?J)av(wK0)i*NZyNO79({0BMl zwuN2C+*LC~=A|Z)wzycD(qA=t@rPuMrM`Nee(MQb3*G-pdr;pTE-W$BC zqk6TnPuT`I5b08hYXUo!+$=<4NNF*n_Gk}8Aj&h-N;cy9##Oj54UCCwm0%Wf^tPw{ z*)K*#{~#aWKgb514DyV$8ofA$s{cr%fPYt`V$CDzMd&GRo?08kgL4R-cF)651Cc%J zIEhjY6&!Uxh`t2)JaCusJ$iK*xz;7lVu4Q0+t2eQWq&s z;`jqLQH=>SByr++>c{wdFqrPO&62FPwYGLoN{3e2^$xViKLC))XQFNBM&C7)Jz0;_6yfk496)N^PEE)3bPHqih63=i1fO602%5Q)i zOblvNTW2O^R275_AUbmSGAkql5EVJefl-UEeSHx`2jLp!`$=0B1PeJBPjoaua>(|9 zpfzA^P*vcPLLSY;j)6!I(coD^Dh1HeBc*{lCyBfV^rE3vUCWcOU!DXH{xw%t^=w7| z?KYGb)s<7cElzH?SFbU`O8Dndt$6%KlxCQxr2T26<6hCAG981cN{`V_69<15t z>iDue@g;Y$kd{_`%87>cb$uP@c-syBwV}A2d`Y@i+~F02^R*84peJ21NO(0fGlWpl z_AiKm08Ss`4|7Egiwy3>n=ANM{8_F!zEBPk1yl`cEQDI&_C+Og7(WD|R@nt0Zhs&~ zTs{*#kRA~UZ&}Hy2(^mmZ12g6-q~8mQ+WUNa7ZG^ED9g|EKX

Zv-CoU;Jsi^v0k zCuv)=sL+18NKLS#5#u5iAu)$ocD?n&3tzf@#r&(~J2vz4cO~Yc^5{N}{$#=5m6*6i z9TE`2NhG3JL_2k&5*p0cye0U({u5r zf7Va5-iy}unw;fggeG#zGCh{EMBx&CfDO|1GV|gLLM`>)1yDQC?Q^}Q~ zSw5tq36BcUmUzHqP_%{JTtTaI@?`q_WZ=B|!HTH&-x;C*O3xd=M@}8sC=SA4Cs92B zPx;)>9rHmGVQOEq3T9JD%9ME(fLwUx#Ds61$0Z~xDx#6rH6P^1wITXDIQpsw(hIQ1 z{SF-4WB&-W!N-AgtX)?YsE`mGK@NQRu|L6iL{{K{Ym8N!sy%y&)j%B~WOUbt;sB32 zK;KP*MxH$zg|#UtXwz?N6S^Mh?-%>4eydHPnz5@7g+?1VX)COfGhdbhVhbm^oC`KedL|+ylOVnUpd~g0qX&7jZ~$p)IsgHd`_&b- zT>1d0TBKSf5Ly|CEpc@u3*(zwJFj1QI2j00fMTeBd>Xr%e7~%$84jJHY~b82mc#w1 zhA_7GBV+~Fi5?S6MrRBTD;+ov_;_A*02kJQc^99$Bf8mjmuouh!P0KSqQO5z!~JrR zrVIxvHfU;Ur6=5Adm&yEi26TYH0i_~Hdl;_SpzK7RF zB}zEVSXi;p=5AqdeE|9uN6{b48xnvx1G{cz89!|dxqn{(nONvmAYplfRt<7F5>=UhSWNzApX`sx+w1i*og~0=XmVFE%em{s{jMz?)V3Pu!6VW3s3%>g2 z5NuI^lY112);&0Jn)2MTAVv0_Mw^m^c6Oe`5s0Gg2P*+n7W zyW6&9aood)TLcBwa2gOXTt29t%H@~5i9uj-CqT*5YcC5#jLz8Yhy1q=2oB+sz?bbq zO=B(gh$xP#BI|K|N%i1#{Iv|=n}n8+Mt8BSZC(J;M^X^G770}WE$%uxAuV-}V8_7N zQhR{Pe?(N0s`ADC@F9>Y8oSL8oFhM7n+>#{q20ob5Cn__Z!%}bO$qu=61@cxa8^$^ ztl>cR8o2|70R&1z8b^?N436-bR*RI{lGRKCJ1T6=B#u!iIhX{N6#X2H8@Y)rs+obV z5iAxp3+Z|Yx+XQdxl= z7d7PSE%U|6{|9qgu=70}LzKhT)@^6T=(?VSzE5iHXut-vHE{w2V)qC71F|9FZ?(H> zUX89*yyEb_d zO!ZHF)y)TqE9v+!kHG^X5eazD)C&-Ai3>_(AkgQ6WUq(N_6P7xQ8@ae7uVP!4;6=z z1vVaJwHBjCKxOd}m<@F%K4S8bfQuL2v6klm05Eyv4o0t5G{8LJ*~Aob|Rd@Of`AHoO>4FLJ@ zQhvv141?d{SzqH3WegQeSjaLaG@o$m<2=>Hdxm+ddBOs&adU_su+hR6+tMjfG&AE) z>WiOKKCJ1C=#I!e35{WJkRu~%SiC=gz9g`PW%L)BV7nma$amip1zoWN5&~QIB01s9j6vbHaL| zNZ!iF*O~E?(sGC>Fc3L}G7}43)+?Vl3HNS|&V=J=#`FqS)GIsXtQG@a#j_q<|5{Nl z4k>j&w*PuazXnejk#4K=TdnJ$dCSS~(RiS!dMCW*l|u#7nY>NQ<_6li+B zIsZ}IbSt6l8U+E81VuC;iK9d>Jr8P|;2~CM1n!|ITo5i^kF*BLi@@ir1cy&R-1L6P z-s9l~!q57YN&koP(2pur-#sT1UPqDYSb*rd>4J|qUTSSt%B4{a{AU*XH*WgFM z`zw#r>bAt>>VztpU#gc^y626dS|`Z^_ItlvfOj6MDgx2~$L(3$mWSyagaQCrgf(CW07(df5ahx%3bF+QMSYSsLAD8> z@4^N=+qdcd`odu1-~>6PFL<2w*Ot_COx3NjvL2aQKy zUMS8GlE#4;7bGHjr0B~z6En+S@*PLe2#821|KR}c%e>#I*0-&fiqhYUUFv|w#{a>5 zZ~iwjGGL``mL~_C4wrK^vr;9KUz}tK*CH!_e6q#%+@_hR~b(u4G{z1K~Bl2C#R=3 zzyKhInh5UPQ?E-)EfMUwz=RW00M+>2uw_AL$U{IpVtp+pIeyr|cJ0vON1zDt))7#d zIDFy!!Xn`k?L-kngx!d7Bko)TD(-jnU&t>vHtt~V!upr4#r}sD$Hx-2=t8dW%A9-P zc)Bs8%4RHL;;va_4Zy42e0AlY;djT}AL8@Iz0u5W2WS9E_@D+YU{G7ez|ijWsgDGl2vK4*?ob3?BBHFogQ5I3+{Lb@A zK#d=GpP_qzuri=}Fg}NWAtswhyFix$Z9HS=9PUndglFMNrmfEZuAe&oYbdBbdw=-U z*6iB2_5KD>))aCd40uvO?BDb=s!yvhmbCuIxzw`0&2m{2FnzDFO*c>t)CjAzbh^I* zo3kIw_784@dGYAeMkTj%)git7VYC&ziag=dE!VE8=72WgO75TBNBm>DEv9CLTDyAE- z4SIO@ZjWrh+=PsTW)M{L;XN_uNmE5MRS*_yw~+!3H^%YQh+br?@jw(#(lLnU`|{<> za^2)^IV8ES13^PW1+!KL&&ikf=jCz`Zv zumM+g;a{MdI{aHB;@SQQP4G7CLojzTW!gmZ=MYmUo)Tt>XjiDB39f=`2H<|;X#Grg zlqAKw>}dV#Ul9KN{c!pJL&y^EE0xk^HmB`EK$FxeKl(hG+^p&SO_c#Yvi4bx@s&p{ zwadTp%Uw~S&9W$uHt69n-6^TEjdVGLaFi_&MY*Dcv0`e;ift5SqS!@?JUUWwo>UH~ z-tL8fjwO%6+D_Deh@BvDpqzp6+Fp3egigl6{WM{R1aVilLIx3yPp5+zD8V0`I-h4S zJKa#ALu&TY7nk3o3(N3CBJCg9OV5R7 z+;NW()RPz8G(XoiA6|tDcBmTd{wPWBiHX^${!};B6(!4Y^|ee7(e+>f6S_&=1Lq&H z8H2(-P`=>+6blkCZ<2NlJb+HaNXtQ3S$@YzKm$S-*uYo|iR|9_KNg}$=`pHH-$ z%BQ7t3wgYbyKOsZ8rTqI2LqdNt5hHP{pC1#Lt|*u} z$Wq=zP4cL7b93b+QO(TH7R)=pue$pVWDL2k-$!OGHj=z5vi0yi>XE<+57dcRRdtqU z%@a`-@mJ!WUbAY|xhi22WtIgVGPhhq8IqPl!9}8oiN&qlbr&#gq*0P+S;UJ&qGBV%j~9p=7zHy)4_F!} z8r3$~fVKc388JE@D8xuY12J|ef6PQ@w!l9@?v(^%_x9sY6aXa(Ei4))#I8<{rF(H` z{oK#3kf) z()#~l9FSh|=06eWa%JInRwvrmSDtgbyOBn&%S=@Kt=n9fn%g}RxJ(dU028Jww+BOR z#hexX?3b(WE8T~03iwY$;W($?wJr= znRK&on2`3R2gEW-9Y<`ep_bgBNu=`e_TK%Si$%5~coTHyPYz&Q;$Eyg ztSYqJ{aOvZGB$sd;^?;@>UiLXxZ=J+n3K)kCuuvN7$fHu;b!4TBT{kV>L#quf_WK* z&JfQ+p#v}5g8hm?;$GVzJUwXtj<(k<$TzWkqey=KaJDRR38AhbNw?4+2Dc4*EzArc z$TJF$D@e;#tI#{bFR zitf17oO(}egGyU=4~Ze=8;P9K_$dmd_E$Nw>tZNoD)B3l;w--Rc&_I@$+ta@IZqn< zHdA=k1v#Ctt4`%h?|i6GVcXB{#|c3q(E$UgN&_}aJqGfD#MbiimLP+)OB5^wjviog z+Y{zMVnNx04thZZuG9jse)7l`Y*zk zTja+8d7768@jhuh=J13Yw<9WKlAjI;h8X=v_COX(Jfm3d2o!7QoSN^!-b_j;;6S=# zz%X?X;cy?hq#_=xfcg+49f=%-x>!bC9AQmfvzct`*AvBzj2);IpM|9bsk-|kXJe=f zQKJ7%m}7dt`EUJr%LJ+Z-Q%Nu#t%Ill8Q+tBs~??0&a`cr2%Las&RXYVhRe9TV-o{ zsiiB?tP+oNW)c%V7|ec8BKk??#ZtLRrYDUs~9m279 zVQ$Sf9pdFaERF2T?i}103sq8DnxHzOnp`{l#nCORs$NcOM~SQZ(#fW}Dv*ctQNO+0 zS8?VApws~jMZ#>M8?ms@g(?HuA{0uXONTh~TqcQf0tr7!aPB48X6h1i!WMr1vN)3# z>>Ukw{~=LPZAo@S04r|CSq6brw-M1tBRE;O`aKW{Fz%K{o1Vo(KfDm7g?v)ag85rn z%;>5}-HvHK4p%|x+6W0^!XRqfk=V0b7!!JcKb?z)r55hH*H?u zJsZ~Vnq2vYxgRDUNjv%~%$ z2j3sXbcZmGr4(Jdr^lZOa35!zp07qWc07)uB_wN_l)t#Qpk|~Bo|RQZE*DHMHQSLJ zxEVfgup^iK#9STEq#zN_wColS$uj_M#&FLtgT&zCW-{;?)8?yKI&cb935p6xn2xH? zLbh|K#TFKMOvL*AqPzpXp4mML6>TrdE@VV_qvNr6z!qH#sT7GU#$`B@%h4~00*J(x z61jQ;a1YQ!>KHE~BD}6i1g05CF?K$teSi=ygw6ugJf!yVDW)YY$qWwgmy=b*Ip)b2naJg zjo#~p!8#jYmo-GM*3-q0JD-F~6ivhv1ppcEf?yIj?B>*~fNFQJMfmDfv3O@BU3_5*l%=IlDUtMInoCCf8nduPijJg)}L$Qx;EZNecCA6S*!iyjGGn(N-Xv zFozrTt!>tV^ugrM8ke{ZQ!8piSkV$8C07EWz|dm{?K`Q|5qU>^yQq(M!Z?=<_?39d zhJXqc?S_?2BPMWYP~c)%!-8S z1MToTanc8glK_{|2yFqzlaP)ODDGHl1lo5ZM6(VT#m5qL*hdK2iSU@rwdL{{)kyAb z2@Vb>jX1dWdm$;m0PTE4@lVo9fg@oi-L9xw7;te0Z2?n{dik)d2`i?lrvPK@26vJ= zjS7k6OTblo?`Db_V0l0d&w=U?1;ElVf`()47^-}2)Sm|+A1#bsNScIdBP@g_5DMu3 zcl0*oHvRXHL1R`AXF3UCsc(>V$-fFY2-&41CBuI`zQ7Mb>KN8*H&PN(r%Ag6G)?Mu zn{&HG;)u`{8cJA9%T2?;gIETFY8DEyP16kxjcGO5;}Oi;Onr z;FzaYphht-CmK8+O+6x<(qWf+Jn1Hh-4!O#AW~_=a)-fUyP}xk%#W|#0vh)9>(`9R zjgC;uSrAVUqFa*l2?B;*mEE~dj?^ju@publUCuV)SVBkPGl2k0i9P7ry{6;6mQ4%H z`xQ(s$7V`mGCLqThB6Rj2dZqqL)Zv^fPMv_iWF3m1t3*vObe(0T?mdd*gb}&2Q%kJ ztcmx-);4Vk1KVqlhC*htz=3}d4`_7GZ~H94$nAZo4!0M7wY)9<+{2xLQk}mis~*Y|a-7NLw>Y7_ucaNlUJ^#q4Kuul}ohmdnI z(Z?lXa+P=y(C11&44J4Rye2|Hw(Z&TV&BP^c_&hBYFLk2eIL$}W|8Y+HoDEi)c5d0 z9?*eJ2Z&J#nyBDtU4V`YMUsHPJ5nTJndHyUxgo7qlvtmiA*%<6;hVpeA@8aI!;>5s zsKY1vi0gX64Z$gu?He5P5<{QJFnjFd)R~Z=%~rO}_~WoRqP<3iQQhOWATb)4z#1~h52-7puuqympbUIM(??E(<;R18AeoIg$cV`o)g)FF zfKA+PM0^+*H#9rXDF`zI8e~MlZAq|2Q;9MS`OO!+#F_PTT{OaX*#tGL&)w|;_$iZ! zlY8nb$R&y14uSD>AZoqV8>XKvh#fFZ1{`np6W!u6>{gYe!)>5Kb*=x@6 zUCi3^9x6ec8sH=brCC>l?yMp5J@}G{d~c=L8GUpc84E zZD}L(Uc-{iUN#JVkew-(-Hd@2aSAObk#tQqB&4FX*jYwkTpvL{5JbaqA8d?+67Ffx z8QAvDszciTVK$QLd#aIk;oNPvzTtoHE7%ffa*k}weHe!-Z7@d~gd_?b7p zY-yFJyD%_F`Ud1AU_R%1EhgxX;3J1j!&XhGoeuA6ej-t2>Uip-ipk3K{Q7$4H9>J1 z%a>aHn72!~n z?{<=Dbl&yLPVG=}4Og~A;kNCnK%wf_7yyAUafHTHufcYzuES!Xa|u^FTdbh#Mi z6U{NnU}1G2FW1uoAuUfkNc;qmf}*~Rc7lS8bXz!X;J>Io52*$*RpOL_Jzpz&0G{99 za$H>iK7V_Fs%!oyjW?!n&)cZgzy0Gx<%YQxFW-G#+Fu~iHGQ!*Vnm5CL%dRlX+Ld{}G^;GJi&-e0NH`cP^4-&GMc`T`jsr)( z&q`NGy(5nPlMFl2eV${);G5-UW@amcR;^rFV|W%*>ZM$U3Lgl-Ee*tP^_G|*ZwS`=0H72i5Q zxp#wF>)f0vas>bVcrXy`$Kefz_Fs=BORp*T@Lph?6ir?-=6a-I`HGce!uv0+KDda( z^|MCQ)2BN@LwjR)=m1=z!{Ls~<}318;E9kTAI|#q;rTZhDA{MeP$RQx`&-bUXTHB( z^DaU%i_9tx;u8Pz?72qYT7|kYHOuveKHkTeBsJ_YJRDf+5}fZFAw4px+pyIn^&KhsSunE@;3 zjt?x`d5{yp+7c!vrdvTD&IsZ3H8C}PWiCohQRQRCoqYv>&b9kDB)$#|FlOp9_$wI; zVqUB~w#%!DO1ti=F2SVZ)vH&BRXo$(4tbA)s+fq&`SYjnxhpFx7eZUmoEm)Z-gS5p zR3oJ@lJA{&YHB|vo?Pn;v^PN8iXN!`o_~H*<>97>4~o>%8#=&VK4#n8T3${Ev(Y6q zHOl3eA3;t6_Id*%kWqKF=<*2(t$@;ymWk<#%EO+ni^cnFY_{X*@xz9nd>&BHu}sS~ zAu11P%M+wADfLsnha|GJF1>xLV*f%dz8K=e%XoRlP#|sM!^?SKCOsR$AhpMA-)ymo z2$;9kZ2qM)=GJ{Jc^UZU>dE!A^z;{XQ(49)^c@|=;E3=AxYF&Aqt@%{;zCv?Ja_Fc z7#?_Ps;hfLZAXoJjpchokSeH|+m3_N0Rz%N6!GOqF9pZ@&*MONqPuiZcK*;0!zkHTGIrPj{|3 z&j;DrYmpUV>D-D-vXX;Cnnxli_ejmfi?3Q83zZdo+bnFjRjGx_Yt`!51qB64M9kmLIqi^>?)v)G2QW7k40%GKa3JnYuHUp_ zgWPMAy1U4cF&AYU8O6D;aq=V+Lhn`{@wc9vn?vTN&|y{+Q=Mfhl-+`TQ#l^Hilb^%G@+KxcD7U}#^+{2-TnR++ z;|}3d^$!P~5oGN9V%^Jax^?yD4T+Avz8i4LECPiikJj&nJ{CJVyE61SxHub)_Ts*9 z)rEJiWK=yn-65g6cWm?4t$XF=sYKZr(3#bM|L$-y5n1Q%;Sr;gjIWg|R!}4*&kklW z(@-&S$tV~ZZ9)|yk-F`+iZCSgugl7~g@kVY8d8`==*4o(|MRXff3|sIe7qFQwM*a4 z%a@kqCH7*osIy}m<#W0A%l_iOHazm}<+bA*l>WQsSg6@ohsmg1+Q$2l8skQ{Kj90WrC`pN%}>F2!oO0&OYY3MbFOJ)cUAe=VVy31 zU+u5N4QKWRl!;6|0doUdq8no#FYJL;`BV1~|6H9F`T+m31M>1hXk98#2Xe`l0zG&P zm}A&yCBr$9yY#?LilCht8v3%&{``?6%m7fY!4kp2$;r8JZ~xiWg_U_$sooQ%i;0QZ z?bx##aSj5V>;0AVa~$ZPq&W86dL8wmMT#ma`x;S+-wX=!!6s?n2rO?1X;bt)_FuS< zySVJei>1mlU}`TfybPwR7&2`|wO!{H)6g7K#j1N{A|k)__Vo~Rlk-kzG!nIz;pBqU z*zYHP)_n=bW$<~G=AIu}!NOA7&yHhSh>w;!fK&Q9VrTq*f|xo5+0fUE59F)NMUB8^ zys50r+(i6d%RjfYD5~rdT}!_NX<1|&gvd?D*FCEY2Pd}ntC(&SAKyiZV>h6Y zm%}mu>c9|Oey-Z$q2@IZ!en4moI@SQ*4xXlsg~e+-3$#i6BNIzN^W5X2@X_0imJ-^ zpxc1yJPYd_^le3-bTj<;?mM3IO(w$;%jJZ?9$%hnF*1J#Sv;(uov?1 z@*d|1cexWA%Z)PeDn{@X)zq9dq+aE32t7GYo&@cAstXq`0CHbsZDXUTqEwf~HYz=D z%X>C&7C9{YH?W21dvFYU4canGDg`n;>>mo?@XIh>nkE0=)u&+ef1J!wiLFl6b-fzZ z(X5ioR{B#eFfLOae%_m7#6&|w)83!eR*H?_EX3uwetuIZFyB;E6q8Q^If}hSksu&1 z0sKJ7vXV>O3V?$0~y% z4=1RPkIxEbW^Mza2l8ryp&hr68WbV^>H>6{cPktvILLBD4O~=NdFb`UzuyWE3w9QG zU09$A+U+^Bn^XjyvYs<*A?}c~wiY1IfE&ANO?9=Anb{RA!>?FtoloH8`&{6~1)69R zUu?kU+vJ?z047iW&i(s0u!7{V)3938j7&^qg?+^Kv17-Tz`#J|Yf3D)RZa`rh}<1| zFFDu_Ax<{TDD}~g&`5tO@RFUGoy|Z+kBdcbqK_MfZ3CLNH}Bp(+q|5Xwh&mHqKblV zM7hZ;RK~X=B5X8lt*rbowVfd{GIA5|`p+mAK)yYFwC@6^raP zyunQ$ddGCS%v;DUuyeHThJOC+8JjJhm0v-# z8YKq!bZGyC*!&|tGBGkL>FcwDC)*fYzGO3R%^O}?YCr`6z-VBMrB&UQK8_BdPlLr}q0BXun%8J5eA$N&=!iKik?-5BAbKP5 z+u`TpA3VEhj`Ibd0^5wuv=DpQE!=m)1(}&^FfdHP(sC;TQ5o&*201fN>go9by?00+ z06ZuRAPXYRQFD&j{?-oD`yY2*=kUF{ET_+oBjBNiHk ztLEwH$sXsMz`X1dHR|(1l!MVV=I_2kLGcM{I}lDwSXo(zsnH_*e%vC#`9m-`e1+m5 z`}_zWP0`PsK-|tIW@a5AQTO6#TblCF7Bc)Y?7tc(PS^-ax{V7gEnfc`JRg#yj3U#L zl6-WMFQGWvQAcZPXU8NiE{+2KE-4COKu1w{?o(sq@!JO;Y6={GLIW3WDPZX{I7gEu zz?M&9R1-GD%izk2D=Td_d?o!0g%tR>4BOxK_4Pdl3@k5pW)A}=UoaigAu^imo@Hbd zL3H=RZk}F#1;AJzppGaHqp`VHoDNapMPIjK-MVWyq(`1k-iK3i7%2aP$B(Nm4It-3 z=q-bu-f$Z4qsZWLlVeX!8L(J^#&7S0A&m|z?-;y5vXW%wou8hZ7k5#8;o-9ZDfHC8 z7ae433CYR5+Ij;61GKcXa)yQ*>MX_dPn=l#eb1RZgFxUOXk8v41hqD2*REY~-jH=S zn8D2^=d+<5KnAY#qxxLvy9G|>-!I6zyc=cNh$)?j2xrG|2cL_}r;VH6tb1xYwJ*kH zt`{46VavyIQMZncjY0efK_LiT6uYBo(7lIx-$HDo$|4K383u zs6}ly$lu@mQl>5Aanb|V)uvKP`?PoJhL1jW&b(w6y|a#4YUAy(%JCSX5#g;rtu*Gd z=~j3buH8YS~wesh0&+3?0+EQkrD?Nn-1lHGMY<*V?`LmgmJ( ze@E;=yTP8GZQ;>&y9!%NSZqB5Xm5=9 zb^QHIEScZ^tK)Q{)AW}H*U*^laHA=kDy(Ul&WX*hwqy_UY|+*8&vrQAH!~xlrL(K3 z;@~AZW~r5|;*YzYtMXADO=tP~rI;-ww70#})bFv7vub#!czZL~pEBalFAriYT?~%z z`|r+BRVlx*110p>MO-(ix)StNZ%+A_E zKFfwb}I~o>zY)8(Y_EA4PAL~}DBsC7*?QY@gAq<+4Ot}c6+xVi zUU&Q3qiOndKss(|))%$-%{oo z8Xmr&o4QMuot=Gp{~+2Z3Pvsb<-*we{# z{AuFkR8_oVlaq(XHW-GRCwosG(a?Ehm8GV9b(`m~AM0>W@+pqz+_z>vCSk~HTFW^~ zm{kiv)QN5$Oho~vhX?w327v%ZhSiV?j-9@u=R7Jt=kNda`3KP!SW{)^BGS1B0 zJ3lv{_cB$l*VQ3@wVAA{yyv-fs!Bzc`Ja_8CoGmfHtHPQ{;hFpYO?36>x^lm{K%*t zSKh^%v$5*;wXYe^+piBC2syr;`<9B(*&k~K<~-T^PQ3K8%PZ$;AE>#KJ#$RPg2$)OfY~5N~-o_uZyJC9n z6HVT-Z(=dWhWd|fwr=*4w&QwR`ei=XHgc}$g7y$w%j~->CYi!7^Esy?=jQM`oc%se zxp+E0@1IEN;hLNh^PA6#_ZSg({65|K3oo6`vbj<1)5|8KFE2A(P8~AOnqNY3Ig`4A zZje8A^V26SQ!2OFGY*y-&Se>YsrH|kO%EN|f9L==#jv%xCTw0!06X~EJ}+DDS!)^k zRKDcXK1R8Zzw#duvb?Vs_+%n8pw+5HTU#qI-OFS79`Wlb&owmf(wy44=)S{%W%Q+|O zrqyu0WAVobrxi7C&gs`uZtGufr)px!;@`B1I*?QP6?F9PP6L=HJswIcL0!QIF@@+{ zz$Lu`qJ^YrOF=|PmE!vhl;!emH6ni%Lo3I2aB*>2x8-vnd2qMaoRqGf-uBL}&d#q@ zUUNH9gj|AjcQ0S?wUCg@*8!|d{`wV*l0x4JMn8r9`-?w5|K&STU3gQO=jyd89$u7UlNS+Ik68Zi#n%6{Ba8PI3kqRUmI)QD`TmJ!cQA zFcJBJrkUV#rtiWoK*VZa=nRx$#n13)1*^DRfLz0f^5wG)h%A^(-3f8A)vs~&8rEr zgYykom51V{TZX_&9%p5hp=RCQ3Hap!tah0Abp`d2G94Cx(jzl3vz(VN{X!Jnfp%>c z5K#DyRKQPZD74n~bAZ0!6+o3JjqrsII{j3uSPF#fiJBm+@+_5f-<# zajm*$;;p%=Mtsy&dVH`g5LU_L^C*a4gY#5WR`e(_S%p?SeyGuG9mmJS1oKx_o=zOB z$bjnmR3U9S-&tvtqU%U=`ZO@P@87hHJ4!otyYF6#V)fEq;oWI9mSW2`zjeP6*e31! zDCeV7FD>;=R7?|D9A!Bt-**p7+*raKFcY<2S;A;`o!h;blB?g0U%bgpDEin|xo?Ok ziZhje%0-ZIwoGiT?xFtotg6%l>>ji>rN+5qp^9hD^iNB~Prv2sn-?Y-lQQ$EFdVN6i(Z3MVcxzVvkAytFQ2o}n>;={Z->AH;qRhTdx?NkLy7%RXp5qez#@iSA)+@S<6pzoRKVV(l@p^Do&4+}g z6dsS;zPz$m2c0gzj&BXTP-B3N>cdmn2TO0bGUrd0E1Ed`^!j|PuAuMc*6}DC(VW&hj!#&X}!|KZ#MOB|*Gb z&P<&zRl>fZa8Whl_m^0r_>%r84qj7HIN1E zAaoOwlAeT}M2rdquQ+Iz?Z_Kl)z{yj+61n+6!(-{?ttAA(ND0=Cq&TP5$!qKt-;YW zsC(_I(yCwzyQ|g}|Bdh0KGeK=hjc<{aTK8ne$fY7k9OIzy%3L>zqopt8hL}curlE3 zmO`xa?mLW~pE9jjpCfzuwq^Uoi@#sI`4Av()Jh`oWCo%M`ifSXWrShLB7(D{V-V1L zP7eTc6Ys+qz=ej8R!ol89N5M>_hkSCHW-QYC_6ZaBG{5i-bEzrnnC)0C8Z@s z#=q<~Z0z~;wD}YA>EP`tEGm+ZQ{eWCh~R+l{avj@#27_o)ioOIa|qK@L=^(s-8zGQ z$i%Q7t?wpb;gxW)DQao0UP_PqpKULS6jC^8boTc8p)KSh{0dCH^+q}5cMdl# z&=l^?o0aby`pVhc?-bd&6AVNE>PAPab=X1`6csm$h^$gmQ=7ei<{Jv6-S(aI$TqYQ zs{&d@_K?8CRZE;66>S$3WCS7w`HWAExo9W?QD_msxejbj-GH<(&XGFV$h@QVfKK9g z7Ny6;P!Uuft)W;{cn=+}d_ukDh%wA7H_J_CZ)SVpy%V)2pbbX2UL;a`fwuyPI)%v` zpUt>eX#gC|pZvPqJX6>2T%wlcK%n>K=UlVho4|Y*OKlH6@Yq~K$h3+PwAS|Pe(TT7 zxNzFGY%Kz6@7^WZmD4hQX*rvKo)fJ^M3Fwrx*r#K5BjWR@1sxD@<~bA3dncS>NR2% z%J^W_Wt?yzT2q_W?<`$RQP-}&(bHo}@zxhqG&KBGRx>lh{xTv{+BJ`Ji}T3Pu(M!n zVp_faz*ply)KEj}l10(tF{1IyEX$hE>o%dXG{;Zhw@Ccd-z8iZ@Zq?3HpLFfwrmqgKFypJ?}4z zEut~ATl1=m=GD$O>qK6x{(fFedCx7s(t9ivx$_1L`E&J$tY^mh)S6S0tG^WY^J{1y zvlFx*eZ22e3dJSx?4=Jn0rx&EF){e?;G6vI7yW^)Qr89tjx*{lJ)QL=YVyP;ksam~ zN~`gS6}z}8Y59yo-G>rfw3gr1Y)W-%I&$QLH&g5G3r$a+TXC;`E^;HHZ}}L_;oaFy z@@IZ-8dli@zJ{x!Q3_sMTr|YlOI^eh&GVAY!fd2Q zfBxI-vGK=CqCC6kTc;f8R`0q~ytOf5^;Lt_H=36|rJ^_oe*gM#P|ZiRW~^8H-Vy&- zH_AGHOor}zSJi*troD6i+tabp`8?n7?I9eaB^$DSxW9=z6F!_2E~a5Vs>xV5s}6nSBT#~DYX zbF!&^bp6!0HE$C2>_z^XD9;J{=LPetsAKaaY<{h0(5{^9`_-j>G|g9&cmJe-*bKLn zVXS8Kqm-2vW}GoI8rvz+4>bbwWb++M=#~>`L85cddDw^{t#6#rjG+;Gh0~{*-$!mQY<#-7bE4JGGniIfTyJKwEclmH*$&UM9KR-Hr&0pecU^bv z-Me!2w~N!$I{NwgSB=Yl8VX7Mpn?W%u(?_`8lG-_&>q ztMD9K`R!^%V`hT&E~h-oN)1*i>(j?LzisF#lqxDJnjFt6@Q1vp9rTCLT~(G5j6}W} z9PA4z!3KDIHua){s>UOE%udPW%gud?1~`rh4rmC)#reCyu0&n9qw?G(v1+V0Kh>8@p9mKA5TJ=Y@ z?6(a}PTmHAC#pia!-uP=l=p~0FZ>l)!ken9tb45N>^I;a##Li*7wT~a==}KU(_-X0 zZx0ZMRIeRco)SO-=ifmUWbQ5HA-mN3l$&hG23nZflz!KOqHO*p855CboSZI`W>`vE zS}|`hTbxGTTPao^ClQX0)B$bJkN8Mdva()51fKH44^%1#6#5gBlZ&yLgWEWzAsAXQ zjF^fI@O{LY)E!MT*#3IZ%d1RE+A>5sfc-ewp|z(q2HWOrw!)1eW8gKr8e94zO^b#$ z#qZz!UB>E-tA9ZFw0}SE+|STU#|m1fKhJy3bvew;wndeloWHk(GWoM&DL5ce1Gnw3 z9<&x;v7@GV&W%`2mh6aR4;~${7Ydzth`A&vzCm0iRBJUG2}V7!mAW&xne*Jk@54VY zwcWj%H}Agg#WmjSCz7$vy?f>xw8%~!$r!%QJ9OT0E4#En;E~GDtF8f$>zk`tEUF{E zug&^(D7JU6iK%nR0{v-SJ0>=S(Y$lnPQAQfXN~3EJ$7n^{DYD5XFmpe+-g)t}WHFk2={?!zQ@%M`Q!lV% zKKQDbZ1~61wv_C-5)b9?t5RmlwO$vne=z&CSIL*|8RwX5xcHJvy^1Tys5T;l1qU{&W_KTP+We#n9kZZ`O9Nl z^t0aS`N{DZYg#@kWB$Nb+vbIfd+a66e$lW`rLR|>?kN|us8?jjwjbU2C|~4R+65J< z`-_Tr3M<=Irss(_^f>Ad^-eJ@bYh1-+dSK! zoLcri;_giq-pLNGuCFc5+tb&~#Cy3?(a}vE%Np2hIX^BvG&t|cu}$ZO;<00Gb9ZK| z=5y&Nlc8RBZ=9Sv$7b^6d7HfKq&)k;=Pe~k+9zX%?3zC_$z)&2v?@E*np-GS`9EBk zsQ%D0lm7yfJwg7e*E3@y47aV;>ikBg&$$oVxUWY~&% z6ZHWDC&!7v=P>1F=j1p&UhQuJ&wMEiA_8xDR?&eI@W(LfTbc|~!Q~e*G{+q-!bzBF zx%%1)xWE{KLgElKwzCT)k_3d4lo1)^DO}B=(NXroI&LWXw_tZE*dQz0X4wh_P{#QY z(*&)py^AeA(K0fYVhc{4i$X}=a5;ZeG~Cy5COAd8E1q^4d~W&Wr0$#%rpiZlRs2Ol z$-%QNJu@8W^%`8ZKg{l+R2*MItG0uX*oe$5ohX?1a7Uouf_#9Lbt(QxnP*=-moFG+ zu(4kdbz2B<^k!rvV{qcE+Xz@!VQA-OTH0ngh+Ycs*3UjwL3D^y*7F8t0RDe$od-PD zd;kBh7D_~^Y@*T*8D&;xB`Z;~N=n1tWJDPWiPA7al5E*Cp^_xoD>TU7BlG|IIQM<- zbN}xD^*E1noYU#*>+%`z@p?VqrM<1;iHUo08M@;b4n*m3rbi|D==8a)HEV7&@hK-1 z)9$}}HEZyl-}&jWi0%~>r58!F7pBWqlAa~>m;bpGqwX2R*I+O|Gcr(rI*w_&LSOIg ztJ8*+MvIeX4Y?WNh4qb%-VkgULa3Q-6-%{lNb_m8++LcuA6>(j=cr%G2=#c%p*P#6 zV^4KR=J%z+S8pA4ru!tvtQ$X3uDh7+W0!1}R^G_;!_UMubY0wB`2vYr@2mW@y{7Aa z>`m>&kJO*kBLiJocBy4Ei%(Yjq?_pf_*R}z-*N&`Zfnc7t{yp8F0b1|%4q3+Go3HL z7LhQ^W>@&R*zlxoR@#a;cU+&`-K-hVSK}O6TGI68d8Ww^B-(~GX5DX_Dhf-C*jxMg zrz(w=oP_hMEG16xuO8+Y@~kWhTk+tvU)>D_J8_ah~nqS@7a4PKG)^^;4rOO zOSiH_Xw9z~xd(3xtml4kB4uFR7^7fWR-0I4&76Rt)h&ly z{vQ@zLtN3tw`yGT4Z|~SG%n0WY{vOE;jj zc>Ni0F*UN>skt!5lszN%sA4dzciKw$%F|Bv+IWwCG0qM2=Cw3}9*UtIWQa#q`K8mnOP#6!DpKJP%eUHhTA z=i1e;Te(6da#%$~Oj?Fq=iZ-i4)^#w(Ovq&IJ@KYhyRWeYxgFJ2qvttEdA2c?CVX% zw%YAR_#c657bf!`X-!4S`tvA9UEJf#)?}m~&v4bR5rVzyQO89V=c7vz?q4 z%-WQZIACY4D67#jb5XqdI+K|=R7-wnr}?Hq{sl^keCT0WnY%$j8%P??$H(V8ioW>` z(5;}T;n@a9@?Wp0;&-SF0*q3OVq53HfQfKXVIil!%b#sTN!Hw~&A9%nLHFRGKiUz# zX=pMqZQhJzSnL3l;_=H~VJ2hgYMEiuYKO6mX=nL|vHtF7(4)3g|9&{!@~<~#zCG*fC_iO@}&6-Xw#$k z{KQYJ-H+BIdPQg#mxAr!MT!~Qwh=8S$G&KdUF185dgh0h^xPH@$k)1}gTkQ^k^o1- z55%gOVxD|KQB0(8~OqxA_8DeGejE*#;>h5#B?&GmLYs695AI>rhHka`Jvsu;E%5_BxYoz6G5T zD*l~}HqY_>5^Q~)*%e5;@-eXpZB~YzEE6&P{kwDuTW$9(3}o?fZ+{nGW0YwYaP&sL zN{C6wLy-$r;j>*mTyqBNUz{8Rn_7Cog7fDn-}@_T*7TZxm(>fY7~hCO210S^`5qO; zlP6bES_|U&t3q}@KQp;*zjd>x-|?HMruC(Nu1RtUUmXOO_Kwq*34fY|6l21;c8{-{ zIz0EvqO0fJESL2)T|_B+X1+aF?z3N&$8CS1#m2IE{0@DeUekl~E}RU%d>)_U$@?_? zJ^uXg8kaOzUYa)p9&Hs;yK3q7C#)XOY#S7OywbO*xct=Z(~E)`6%DS>_RM;ft2Tas zV$Cr5&3&$SPh$aY5TEw8Xq2ey573~bCv|H~m zyyUYkoOhU%{8Bx8U@VHtW}``wcvrzjIxYS8$}QoG%4fp~&d!q3Uxt-6a#Op{3jvB^a>>-s-K#mSM(l(u)4CYlmRXSzRojS!yt2lE zMc;2UbT%acE==uL!dB6`rMHb{^?&U(`RQ}HvV4m6onZ}u5(CH2{l!`aV4nR?W zK4aPrI6F9{8+aUS59dKf&g_@}!jPNjAe2Ofg(tG<^!4?Lun;PZ6tOHi36{nIBs+*3 z7ro(1h=O;t!2vb)mqR4+MT;cr)BE?-D26lfG0aDV z`^Jn)Bf#H((@qYbvbHu3a1*HV^Y>6on4(+?q}#o)Hc^i*{Kx_`AfxrY0(l$6Bo=SS zRZ&-ius#s=L3n1Ei)QMQQKXLm6NVNFwt-0tQ)tix4Br_bSRx;~%VY&`Z%%URRt}3mku^ZN} z$<1gT13aV@mDdpbwd^-2NT? zHT6Yx^Cy~Z`{eeo@${PyT68Mj+VQ#JP*~ui(4A$!s3+voJBsvEW^GPf9UXz5cQ zbW9)EuVp>Fknfr^U~_MGjQ2T~uSP9v&K>$5{!vx8)@?NZ$-a=WwU$Q>tQ*>Wdt;@v zJU8*a@@31_xx^o+xXr%vSCYO0wW#>5D$^Fv?$#ZikrRKCr>+jXcai*N&%55zp`rWQ zH0w3WyC8TK_{}{i0~v+m za#O%M#Z*8}gWEF?i4Y>a*x)eIgt@sn+2cZ%65`W6uUD+I4Uy$JVr;wvN-XoKL0uT% z7|~+eWP>&s(SIO^AsYuR#h!+1M4kK65Iu&2w*8Of9;?Td!sNLnDKMrX9q5yT7yeQ z(b3VteN>QjEjtGX`lBNKZF!rKdFhz)Gm|qiK__QD6pLZ$hY_0;z$5YA9a9q%WzZ;3H*s8Azu`n>6-0*&Y<+Eszfhq6o%Z>`Zn>`RpzRJN zEiHvDIonk@Z=Z@CT}H_=wHZ32W5xnqFA8V4ByOU$`AF;Zwj*G8O8$)Ivl(*aM+E)M z=bNJB{L30g6v2%XuJi@%u=aH2%h>zJ&(2F}-|cDxcWz4a_?lbFeK#Czf-h$^+m2qJ z4Ha$3$~?y>o%3l@LuanWf7Y!FUA%M5lENFVY!oZMM=0ru%(4Ep!_51}#4X!SlLCu0xLtTMOcWEuHu6;W6 z_lEiFiw~?rcenr7?HvBt_Zr;M$Ly?3>CNyLWzm#)#4>>8C?8 zeRbb8NP^Hq8=2`#Ue`?T=tH$ZMMZ_&sicKO8ehLq=LBY*8$43+eleH7A2F)?M{n;P z%$VCEDairvvOMO{md)Xkz5z_kBSE7z7>-I8ljh`c?#r+UNC8C4mz4Bb=OHdQ6j4VH zj&uJQ7$7$|p89D8x@8RO*YjL>6$tAXJyhrRbJM9gO7V4XtCD9uwAF?r#9wyQbXPmd z?0P^JJSv*mF7sj#p3-|18>q#it%c568G`s!kl^mZ<*TSG1Ga%sEflk)_~2OZ%Xu9N zDjs=xd5UqyDF7d3WSfq#k#Yy2Tpx{EsHv^Jg#t7$bm5qq)~0<4O)*3v$RJjRhKA0F za@)LwdJ*1tIk*ez@)3t$00t-j^FsrOi%I%GO1>aihS0jZ0b%egHI)%2mMCx1=l6^t z*L_d%=+jdSD@mAv5E^67WnO}8=$zJDRPlazRBW=pXqENEv;4Gq~(?^Nvb=Z zf9X5PQJ-U%{B}+P2A%dl-@Z+Db>F@{A*r3o2SWr|1Qxn)YeGZaDZx);3*_^x^Gt>W zbQ|`Tr%KrVYV(*hTC=l1G}!p%RVqsDgT7pycf|mpl(f8T_UyW1BA9+Ic5B|=JYR&@ z{Tth^vQgMun=K#RUEaIvNk`=14TmYWLg{+#g_rNUH zdEXiRtp@Pz6PNP;DHeXnOCmYtKVBD)?R#J_VXSZ?{FLwL_UJ|nZ)1kBw4n3&PamGk zBpXU1azj6c2xt&o{tmGT6}O{@y9&CHpW>!=wvZ1nMNwn$20IK|^oRW^W(AKs(i%DUm#SI0EIB|QsusD$)b3>qyvZEXz6pFZsRd+Bw0z3z_@ay}W=B5S6 z{epa*F-Xk`v<+1qo$KMBLB&^&^+1duQ1=00y_o>Rne*o(y|2jAi;Gtj2U+JPQ499b zqn`t9E;Vqoh+H=#OT2J87P}_Gb>Wb^hsV<9WEFm;QhyQ;87A&LBLw&Oo{x}0Gj80t zO`V4|0nH2xAXnl?F;=Vt#!&X@!}HK`@`{VIp#u7$y!>W)?Fe^8ySbH>420DrIqs~U zU$!;Q6?>&~xU&R3p^P0N$79GD7f^O@6(o)5w38C8X3;5F^+5sfMnhhPhsAMf2K_cV zd4vL+Y7pK;{X~6o$IhJ)Aqq(2@PqnBmgi2IsW%a#t?#_L($+xF{kNB5?$%s0QhGrv<+kOcR1JWn7S{@dl zo4P1rp^)5i_0^DAo?k&!!+_n-o`&|~%E_F7MdPo#Qiftf@)9;5+Rxy-SU6@FJZajt zDA6oE)@1U^?Dh%q5bf#PIVJmBIpiLzOm^P)zsP&>PO=wQ};1`bu_4B-5eg(@bU7b`!>awg;WrtqNqe;*&?pB9=@&=!7tenFu} z7;YOFcqY!xg3@^q;wF?&nqmhuvzRTm78SOO@otIgRH2yvRFll~%zcxwPW)2eKG*<_ zDjr6{ z|3;#7^yuMHKJ`OnA?E1MAuxU`9xT66D#gG-R^GLJSE!%qo|`t|lH9 ztWydFX(tr>N{x4ixnB`FXvM8y)I6X+~u_a|nRjEufw!GVDRG%we%d|Pj>9^sD8%x;5hkoP};}D>b48SG#OdblY&9XiZL=%;Sh>P6-T9|eZA1! z*5-V;zNNW&O`;~PpZp3xc@LB!nb&%9X7Pqpbar=BV^?Nq`?X@xUqm|=z-P!Vc`XO4vTz06xE(90QO9Q>T5(A|u5=E^D)NBaP4t%++_ zwH+VyoE6ipDtax>L$e%mO7q^@okAia+La-STZ#4ve#8|x(o0K~eE(P)P3cDu)|w;@ z%^bXFmmxCM)>q5n{8+c$uV2ui-ONkd+GJMf_=E2`EeFz>p*UzaD)4-jlw7Pj-ZAzv z{dH@*pRltU~odCEv>GB^HY_cyQr1_ZCTv&+IgIzVtLQWuR%{ z7uT%u&dFHu-j8f~6P$u9RE3Lz4l3V*s`9%M!}9s^;)`ZhroHZKnl1TOXmYOl z%NM(r;_yqUpGM}A_y1j+S^A>dyfpsNf9p_RHn20PCvg3^+jdC!3(HL3LulpiXjrbU z53um`xL|De>A}5wlrZH6o3d_F14Q9Qs*fqZ%=qoid@&)m9Nq0v-8I*l%=we)ll`nb z{jPUpbz|+G$+O)nwJ>cs#_fZR^?~t5-?{ehhfQSLn2_F}!H%M%3(ACoN%FPvn6x|1oL$u~!KQe`7 zul!=~*iCA<;!^=P{n^#Uvt`RNK%X>iFbxEj>6_S3#Aln5kzo=`cPf~k@oVJF2QR9N}wSYzl4I+Ck(BB6oE+~lg3TC zxu5&uH7u<-Hg9Ij&TFyJE-E^$rS$_BB2Jt0d1DHGthaG=7Gtcn&g6LC3d&x;8PnwC z95$PcnGa_-nlAXanJ%8SRsN{AzlBZZa>(u&^U<^+n{$>i3Hx~_9;w&ahQ^+5m|twU z`OHkfR=3AY`{~+EH6pM41N2eedFRu0{b#+6>Lvv?h z@JMg*k+`%uot~$39JSZmYm#IAuKw(wwJZA)H1+d9a!y*%)J40A@7+3CLc$aM^VQR} znQg_9W?6YBPkNeWU3u*BKH z`|59Gf$RWg2om!j7NQ)~zcK9WwFA z0k~!bk_>a}{&{Yh+9_gB0~R41Om!L5T!7=)h7|2Rde(3$LjfuhawS6>k(~FVtJ7GN znnviTq`dBnSoQ9*}kITB;TF z><8VJe7@UGJRBC?eHQAO_e3U>6uX>QS@!5;n}5(%!OYy7Xd=Nq9*x>Z#SAITzVjoI zAg*HSoX|FPdJ*Es%(L4MDQQ9%NJz^_+IO^gnOK|?iXl`^PELf|z!1(lA%dV1CFXY^ zM6UU_c8N|GMnAN;HW|TLQ`MHVB*cL;ML-%HQm49yxKRI5m2q^W&uN!%MyN&w7muQ{ zvM1W;=}BH*)RDzcp&hi

IDAkq#7?seqS2vdlpD@q!sGD6qO}6xB;uBGy zj%jY*RNIhctBuc=-o{3y_j)5g=TK46GrKJjD2Bhq5orz%GRo`me<07o2$*7Y#}fz3 zTxKSgxjjjHXPTUq$!2qU-@|rj0&-XS56)ZZ6(%=HKWM9KV3d{^PxHRwsPOB$q02Yb zI@tZfueF<5x9*s0&uf|CX8CfauGJ>StXK0~eTJ2W?U@Gon8V{y2CY($=qQR}_dTw# zl`Nx7JUICwH%hp{&-KuAzCdcabJW|ms;^#I6J+Q_CuzTTVlP#?Q0DxlRIBu!=eF%m zv-^Fit(_&SgBq%PC3|W|oYL}U`YvO{@&P!C@T5y9fnOOe&oOR;qM$j^_?kVjjLy^`V7{KuVwT(zH6B#wfOnqtHyTL zCj87;^{LCw>7PBz0@bV^A>e6=Yn`zEkXU1<8|eL#+OlsvTRW;mN#9AW9(vmV&;rrf zXxDjBchv&)woiaA1tYYp1~WDV!+Qvhfd6r`OAh#awB$zsByJ#w60tciz0{^)#5F+u zRPorcVl)=Mv|c~d2*Gn1KyhxwmX0;gUhBA$a6CDsZWwcRoB_E2Nz$3r3%5T200T82 z6khU}=)PqA0#cJ?TD*aZ9q6Ip41Q7pdIA2ZEf-#OR(<~L_~rm^VN(bFz~eEfYa+^A ziJ1dF5H&<))L9-DM~sV#D{(+P0$3M%>8deX?9{4Ggn`I$Qg=M%KfgRGI+~c01M!$k z5)X4TC8f1UGXyVAzK8l30%lL7pTQ3w+FpB%-ApvexJNTH*;_4AfUhKil?4A{;`cg} zLID+>MR4d8s{~TUXXf^9UliML_}IgN5`@5vpLKexJQB0^hN}&Oo}0>zHUJo2=`lS z_jXoy@Pbn>S8>Mw@W6S1k3b%D6~%JnXL1To2$yeA-e3~R%Ve|L(a~uYzwQTB445LMzLr(GNmLe{eJ9~E4;k_T57Y-pLXE|+u0s}cAM&f@QuJhy6jiU7e zzVde0+RDy8(5)X?o2g@_dzP2IZFVUChW_ZCVwE5#-p3L{_f4V7Q+b6lBO zq>_{fl$IB!(v`MBm~#mc&NpM@HAb1chV8Gcx> z#b&|68qbyKRL0rYEj|HzM@JqJ5t1?shkyxfIQp}E37ozj`$R(Etv`>>8r+Dh z$`6lDO}$6$q_XBOTam}nNQlg{m~R!tT`@??P(x)zz>tdI7*H>P6TbZW4Gs!)M4h15 z${s#U!9YJ6=mnOQ!@DIEN@@_$9xYuSgyKQHk?%E%qS$lPjs@SMB#_CW0m3yt^Z>~{ z+)6wsHlh^Rrv0v=fzXAjY!b!ja6X#@mWbTHzE{)(aw4KUt&YwUJtL&%glo==Km+t| zjI3IM89{$b?B6L+N@c@H^ynURq`w^|r8Mm9rcCFP_?g#&e*y*tsv5*viCGl^4f`J) z?}RVG&5g2>iV8d!%B}~m-Vdx56$_)s1SZAf$B&}X5NpH1&Q6-fN}&fC;X@$pg&FOW zC+RRw#Uw*lbJqh>b#zq;;6D{R^CaS_(sHznwuqR~4_RVVvlaH(Wq(#aWr^h!odWHZ z9>ptjbed;Vs#JGe?CJ(8W+W1~eWrI=<#`;+$`15`MS>BJYztUDo$s4P-ppNReqamfi9aNNO-@>hC(^1z3K^@ z_W8aqHtp-Cl9{wKWK@E?G@=($>HRr%7d<}~>Roi%7j&YNt#`7uZ>Z{kdaOz}uUK<@ zg(JwMU~}8c-D8)s_F{%>Qz)3-2Bn~XX7$4fq+M)ces zdUFPGDn_4+sm;q6P22kJ8u8;0djn1h^^JnW#6&@;haw{JtGe_FF)2mK}pw2|$RD9~lLrTsgq zEhx4Bwq}7?lX#-r=LhXZ>d%>-{}{Z}+}fJ`S!pdBehbNTP$ykkY&m=dC6N62$rg5{ zuo+xM?+|>eB?=w_uhngmd29w4dIy<giheSwZ$Y9SRmGeUIRqrVtQ#wp$T z@aITZ&(qo0224E*_^l-F~m=4oipJg-Uw0CvRt9q&#;_e8JFUJN%a1}m@hz)nRMcuGwB>UNColRN0p0}KRU~3Xl*5z4m zy1yaf?WN2Ddi31~Z_KAu7>)0xYW=h8y~IFoH)Tj*-#nw4;(Dd!=o(m`O^(cEdAwP; ztLI)^;l15PRRu1qbHC|L)yGD>u=~?SP2qA5zUqQGIjI+qwGWCq|1lEj?af^%n5|Zr z+VesS`!V)^w_&lvW?0w%`Jy{p&6#Px^Nk*Ob3$+Q)}$z;=b1EYq&|f68!6W#rSI0} zpjG-~(hTQb!}TlKE0euGf3dl&j4({*vVB^C!0PYU&Ctl^cU`zqp(MsZ&mp$8+4iaN zOSnJZf=ZY05oHlylq!92sQ>%r83h?gM@##H2(p>_LCQCR7Fb*uTKtY|Nz-Fj&G65T z4ia&m4VJty&5)Cm!?P`F5^`Ohg9jUfW4~}mG>1hPSmkWsZ2|f=fw2o5Q z+7NWqppUo-&YrI&=nDCe2mN}`DlTIty%zvc($94KyQ zTcWrh9HUSm)SsSd5X~#-7aia0$+>?l|GE0=OJ=@;D(&JAyn=%y`xaL)@Xd-M^15{0 zpI@cnL7(~bVyj!Z(7~j3@yYhA^S?Xh*HO^251rP%@1PB9m2Fd_X$D8Z4rUQoCw)Y| z79HJMm16)>F$l)Ywy%btt*hH8N4QNdN&0q`W=Z)eqj0II5T}CWuoIDNamr6D#9);Mh&lZd|Qx*uW*=X7-fh~;5!`gdXwNBx6| zA8{ZqMdSq1)5YtCF)RaBz>pPp>9-JV z^6%&XRgeUP%}}L1lV!@WM4B|twz9A&W;4sb2^8ts;H=C{QW^$Di)kkd1chHl`wx8~ z{R;^9CCOl>P7Dhn@hZ#(SuIvJHgY6^g`$s~Dbv*fF1JY20SC=ZbZDNd+wNTSLnEH# z5JLr1oP`TB*C0QzJJSFbE5iG-0GDxI!dF6NY{jEs@0khvfU*$J zw&>yx{Y9HK%U6Vbs#zRbY%Biwz{_IFv}G!-+W76`1AovbShr8F_U-%koTjFkx|#D& z+dsX1sirbh8y9z_IQ@z~ev)gM{lfIwPHAWav@b5uAC-|#{VZN zdRjs+hPn~*s9Djx2-=XjpXTs8Z54h&~ zwn^twf{wT~7>Af<3`N?$4Hb=sW_6vBVZ*wr$lUD6nqRQ#!|FXf$8TPu(HD6yU^!6x_jRxD({7vuX$n4WwXzq-VW8$%wgO_WRe^D_w%3t_n)J(k|Rl^epS+R_^t4yfNCgGB{&QjCEo=yX$_& z^7odNMn_V=(L8M$8!&YW{;|(gGBy-caRzwAN7A;(J zEV}roCPIDqb%gG_>S&Cv1e zu$#w)Hw}By&H%xlkjcQwIuN1{nGeWpu+6iQwD~N6xG0)p>lmuJAlLw)7Hvhll>gvP z1GnL3CN~7O$6I0p{d(b$+XvJjhanCn65oG4p?c2YP z{;JUQ*pZo@PWp~$l$R1xAPY-V^v%~eUp!w!%&(JYTN(AKu8wE|yQj7-qk!g1M6Z#N z#g-i{5WNv8yk*XGo4~`65F*P#NKYhlfrE8l6Gy_cf7k`L@V^**?#Zd4uG1o$PgO-r3widKV5)$k%U_n{mVX-{wd>2Bn zY0O^crg4{ZKyiSK-iaS`7aDDB*=;f0vF6j$EUt_Ul_bCEnLeEx^EUg~*_q63GSCXz z@Z>2Sg#uMwTV4osy)`diIa(HKZV$?A$qWdPr03M}k@Ka0hA2Q-cyD9cudOaKHSuS% z`Dqv#2gw^;i)Vyr;KRpPM=dQ6sh`?C{OMf%9pGkwgt~3jeP8VIX8)!don4LpG67UzPzN1 z)}iYzgxoDI)|Wc8eSe_sLYvsofU$9PX0ao0i1HV}E}goZ1~fuTbZxdv?i7 z_U^XcQ6A?Bs#)bkBE=2Sc-6$z#6ykkWuA-Tv|J|AKBEd53`)!Q-3YzY=QPB7@zwsh zx2@-gF3>8vbx}#a^6gc-uS;j*xa#f<-}^uB3V!n)pS-;2Ii)Xsv-vCEyp)mN7w&F( z#-lH0j&S^1?WA>%c|yK6xcAw~-fdhW;$LE5pziavrsEG$@-YpyO zScm`rTF^3A=WZd5eIn-LDIVi-QwzPW5mbxj0h}505*I@Jk4#SRJP-Y}f}-)Wp>tib zi(kswWi)pu-Jyi2E2H~xP|FS9Z`UuF_wz@&4glUZy|9JqHC*-g9Mn9HpX6Au`_VuX zk?1MbOndy?pqPVRbjj3zD8H-8#?%X0o;_ka@cDAY!13E0z#_;E}md5vT6>5jok~?v)h% z?uH)|Bl}d#TZVL!rQSP!t!Bh< zXMzE7B-B#0#R;c2CMk&?`ea#*eUQr`9w3|Tq9LA@t4W=KPR5 z@#BDPY;LAMe7tu*y0d)Bp@%Ik)9NJb?Ci*S$;r7w^yOZ^>v(s9t%IcNE-3K64g^sI zXg9dL_WAAr52e@FYXa(j^e==wEcA{Ah3^MIN(X85NTMc?#|${>TeF5-Bobp!YWStPx-9&kwcC*N9o^5hp=Jce0Oyg8?NT5v&1LX^1qM4@U z&m;E!wseTo$nS4ApUixp7qPeO^H78gA|HxdVUzbKTGoqtU%c;M>8Fi;W|>)xK6Pa3 zi&dgUpL$c0pjFIYh`p`aIDTDm8+?buv~~(=m%#kH_=+pEz_EVu@ZsltU9RGrF1M?^ zYtKF41`-kxdO0TMhA-~3*XwgR!~yzQI7nC)90E#AKUuB`u{ zZR2!&Q{$hKG2Zb^4E~poCauVS3!zi0l>>ybsc@H@UjH`u1SQAPsw4RwfsEnAeTCtF3{gpcXDOgi;u?NFK26G5OkT>d6a?Nx3oE zbeS0&2U?bOP+u<%pYS2{GCcX0o;}p(F$HuB{DyMJS==Y-!(={T< zYHpds4*K^^@{}VyOOq8yH_f=3M=u&8O$pKqo%ID{0IXik?2Pt5y4psbUK;{ zv^k-=Z|y?3X&`4Ni5Or?N-sVS#c~|Dv=&to1Zd0SG{g??^fBfJm=o2@?>Ha0Cg)+`4cA})DE!EzFA591arMn?z`KaiA|H}w- z^YsB>n@=hp3*_Zf@JtijsjYp{*ks^_>(95hGY$RMuO9Z*SL#bIRNsCR;mLijQeN90 zW&#HD67^L(RZl}PpHwg2-@3wL_fh_?+iDcI7rC3pMs58+Dz~`amvM(BTK3cT>g?eL z4bXj_oxc1#r4YA%G~XZY3wqo<-K%VyTwK%OLGBys=AD&i+~@qn6A+6>YdIFV;FsdJ z{kTK6NQf0!1=csuIjlxCI@`idSakoE(P?^o^Uho|eVv>Yi!aJxrMeEq3AQH}>2SY>^7spTOA=Z`)PK;s;H&;><4U z=JYihAwp3Z$J%>l4Y(DJ)Dz~*tY!Dnylt8aQnQAiAOK_?d?$qIcrd#u{ZHPnBNE)`pXj^XFM8>NTVGBXz|HaWR^ z=P!R|#5y(?*)y6I9*S(Y{BI%-Bi-ipk)m3ef4ANL%El&C7BBXSQ0N^n<4H?9-M1e69WFg~8QVnO5tpxs#hFA-URts%Vga8m zUj}cW&q_QA7ytyS4RePz;-c@QibFtUY9fdl@-cvG&iNSL7yS^gvLOx%c#v<=Ttsz< zYR`#l*LHIP8;gd&1nS1ygW@WTP?3c$`!*z#fQ%i85NN|$2&kHM?8`j1s~It495| zj-zQ)(`g#I`BDkziQf1}B9+N46`MEK{phM(AN)WVBSTjmk70uBni(`PwDgC^)6XBs zI6}c#-(#{zG^F-5efi>s?2%VevY<@_kn~hAG*_-#Ws|dz^-avLXhE2cGW>COB))ft z*olD6wY-V1V)~@}cRDTpT0a=(h-+BffeEB4^?-+>y(hoC&;m5Z3s^8_h+Ud@cl?ZW zznpSF!XtD`(7Mk*3@hmpcVuh%Hw$uDXH-%sY);#B6q-t|rPH>~vCAnpW)OPx3u$@h z(;Ku|CkXAiKhC{@c{2!NJ!1pLHWrdDult`l%p?Zrz5Si7*46Vh^lI=$t;nRK)H!}@ zQDAOsJKC*11oo=`63@$@({_5q$HrsQxo-2-<7|;RTK-@y{(efQn$!5UvT7O1 zB=>eZ>`CUkT0wsCX7i{#d>od|ebq;r25RV7%&9xDZfW{bV$e zC?C4_DwDuQMzyD}<@{(Y)!(QI42PY9n1wV!2{~a8)m0Eo^K*Fhd3vR=yNHPcgINRw zs&!0-SX0f~rLMPTGZN(i8GrNFD=P8>l$^es_7Zh9si;6aqy5PYAquoIusFFBVF?Kg zp4E029vYnM^n$Plj8qG#yY}ejYLs3ar^6qG;xH;Q?{pqMe(ZyprX>j0OE2p+JkWSAN!1Ts7tBCL%0SFA?8L+aE4iDIp z%Ha}M1ok7d0h5tt95$UK_p!oor=6SSzKa&np^IB@(RlAAnT%Se1A z5O))xD?TvsL~%?TM#9G{DERgJ!Z{GVksv}uXkei?!3Mwidmx}?73E5*eKrJqs)|ux zec^(HQ+j?x1aEzUMZcylDn=Oh0OPjryiexD2?6IBYfLg)Vd*gC8JuH&gqCuzd@ImZ zdi`mI>h-S;?@<*nA}{z<10IQ_HYercnw{~dP<6gDmK^$<-RFv@z!(4W7g7crlR|hE5`faIR7eSxU$5$W zbp`R{oc-r(=5R{TdN!HR5hGKMCu`kdOL|F7bliRxZ+FxU_r0tFJRCqvcmCw>K)$2@y)~iE|Zj$6u|%Q(02P|gaK_t^9X?2mJ5P2@5uH-qaC~- zipuBJm|I}*_4zR#KDI*$PWMoK-3^$CK;`gM|Dm5)(t-EK@BU>Feusf5kbHgziye&B z^p{|BkgbRNhG2+r)pSnngeT|(9Ua|4>)-d}9&79>Ls#Vd<;#5PyIA+&qDS(|Nj7zG zUT*FMT*J7`=bzozg`yiKnf&w^p#2$g% zVLY1y=yYTN$km^6!GocIIkq|ed3jD;+qqc4eIi#m3YESMX_C;y;XZ~F!V8nq->lrq zQ!Nbt5+;O!SdzK}t_DaDSD}hgx>~+O+0bye%i^qfR}ag9IRmrg{tz1sa3myH&7nRs zutJI03hj|L(^JuLGNz_<6!b9p?{c+IxcI0yzn*=&VVzNQpR&5&o@k9V`J2AG z1E%%TMA!y2M4!)Rcb3fvd^Ugmz=tD7A3+LHfB;WV_3G6tgc7@Kx6;DEjA3gB(K!y( zt8;G^b8di&lI~fnnB>=ZZH8YcG98=WcJLrG(#BH6FQ1GceIhknB%xG_Tl*jRyyQb= zn^1K;irpYRw#vZ_KQbc@lmBdg{%5X90T1RyAs&G=u@ea;oNB3$l0klf;T6A&=&fO5K-QTj~@k>)N5o#zhFHJ?DO6i&~0IK z)tZ8WtsFB_yoHl16f_0Gn#a1el7X zyN`Gf7TZmhhw?nwKO^v(O$3Ol$@X11P)&16Uq1l99F5soQJ7?F2(8mnD}aS= zV*qE};TZKdaJ<(zR#sBsQI7rWZg^Z#IFa~Us>%?uVU+TS?|plajqB1ZF6eb3J#@O^ zKlOGM9R>E3@0X>WQqNU1e%Siu`9m4W;`oYO5DO3f!KS%V#m#rupRF$XuIzl}m#d{Q z941q80Z!HHOkL)!Ox<2eLas)ki1TejOZ_+Qm3Ms(|2(qs^~=ll3WRI)j~CrR(Q_<% zY~Akrt*)tks~uBr^EU2)n|t^p^W}p(h@njjgY(|tK%_|YgciaIja0wOwa8(?8l${w z+%96S8C=U{Z0!sCK5r5x6wyE$e8KbK*fNFNO2HFVp@%l0A-ieCK{BxhzEV^*U+sA8 z?J4-LX>`UyJO>Wc=zK$Lj*rz4jbCzd{bL#(oe@|tKtyTPMGF(j z_MNvk^JB%sDC8X+JmU@@Bf)Z_`tYQJoKZN`ph3Lz=x!l;P9Bhojag%vK)c`3)3Xz1 zgAxcM$u-4Iv9}CUw6G8WfW4GXL`hQIJg(TlO)uH&Xe7fG>ea=*`)$oqPO}ygTKZrc{AGq}gW!!~1UV7^ zsx)W;e2sjFC{gI{wQ(Xa!!#U%!3GSbJJp{t3Vt(U2#099E+1A6LtJO~j(bfxvkHy- zu^gbnMk3=R_Y6itpis&eal;+*xiMzgb$pUgP9i37V=yxw0nFi z-dzA1VVK6!Q)A|*Q^Upes`pR(lqOr=vpz^I?@^xL|COuYOMIy8d`w)@D*eToO|RCp zOjaWbK2j#r@qaE_=Z4=o=_E#_=e*u z^(xH(+gpj(FWMMKYV7Ll>Y@@A%~I|I1OVs6v%j_mK2uR?*WCQn)RCM6(ARYaXD-|O z?z6bgR-W}IeAV9UJh`IY-*$aW38u!~LH_*%5I{HO*aIOEFymrA7RZUDC-HK&%gTtK z`5vGe`-rgybRQzY!|YBUqQU*w%To}G20<{YhE$iU!b!<+5Ak4=&<9xDGieuB1K)$T zg!6g$$+DfMlFA%HaTSsIGd~U7Jq}~k66y&+_!{gxf$$pP5=sP?&=JC-1xTd*dHxfGT(1%^VE`lZZV6S901s&3QUpyzl8Bb2c3n9}ws~>6c!P>dR0UA=@@$2F z&Cvx*kqm2*dyLt0Fe{g}Z{!U{5j8~L#Ro0Zv%xL6nh@;h>TUR@i&7&vL!{KG89z`j zh&F-u-Mb_mOQzoRB625Udwu7Ljcchq@O!Km=Dv9S^&?I34!W@f#P z9y%f_((_Eog)_Slw#h0%aLVh0Qb9rj%xePDm{kqO4e+`MApd|r{4Kg2_}~WvnaApJ zYgoX>L>P_8IKlu+)bRH6lS7>V-6pE-+>CT$keXsEWR(e1Q2=CdBf`~|mUGW93NZ-_ zvkaH>^*}alO>xW1gUp!;?f^L(n*+Bl=bEX;b6^VZdc_!LtJbrzHAIC8QS`uPI|x${ zHe#w+qtsb_m4iFN$9dK7>-zl|+D4`f+-v!OI{z}t@2lqwj0Lp!icTdO-%=jq`jQ|! zaA>;X8U`neSi%x5qC&vdj}UCY!J|6Td2{$R8_==%c! z<%6dh-GK8TJdjh{vs*g0xl(_adHLgcr{cbllCrXx3T1jo;~_S{nsFmaG~5JwxDYXj z=#kP3+ZJ=*CnO4zp4%uZ>&%y>95DWBlWb1QoirC<;u7=qJgplcy80kr(dxLAc zmBD>h!UQ&WU!bW?E;I1N8pyL9cuQolfZQ!j-TW&7X6=O=$an$;f^CTDt=76A+vxI3eh)XP+%kCX3}{?)mS^YrKzbiqe0KZ%j>;W z1_2o1Zi4NM7Y9nowStJQ&d!^NgJ6j%uB)5KsaKH+KxhS>#U2+!S6E#b@$87VBT)Ye z4}c#GM=u@ffGyvdO@aC%q9Vwcw*baWEHOel_`D%0#x&hgtfCAf-nklai89A*GfwAJozVgj znTc^dqs>5(7Q2x2qUz4uN?&JP!V0Gfe5SumZJ(6e^Y1Fjzy8U}V;>{@k4Oh!`Wy#L z!@_Mve*e#dT8%W3eRAF}xsFjjjb**xy6&7}c=QI^tH^{qfBvjB@_OLT4efa(Zc>E1 z=u`=yvKT6vZ{9Asp5K!87^d$2`7VF!;O_Igvj)@6xXBqzEbKVbD~*uyLWGf=3Tu~3 z7UP7w&c63(Opq{`7w}n~Z`eFIt1q-ug|_A>Ci>1dD|^2h0aQ_eqp_ElJr4 zspIVHOXra)pv%P*gTuU+6Z&I<&=|@T?YtW+ckUFPWk5+4Kv&Vf%rMhK_-O zws>2JVaT3=)eEgKGBquSKm&-jX5+?HkPr=41gN4`gKU_A_|}B?!!DAXKe~pUJ>1p& z8c!;$)14T*^`0pIV9j4jk066dg?I@^z>%fp&5pGg1&0+0c&=vPueN4B*M~~mJt88w z{u*tA1d+u+k~?Vu%)wq|1sM&3#Y*vbjTIJM#>vQ?~II4!Ua2CKI2$%WM3I+?s{C4`*32H z93&&azF`vJ^5}0XV~#I#+RQfXZ}+|ta}69vj6tG`y0pF58`Jv&D15S?XiGosWmIh* zDLC$N>h$@RLshT3P2#Uk>x|(T`ma|TFXPIay#L33RsHOqe#0H!KNH3xqZ%TC%-4<9 zC#Xn~qdiKNe|-Ws{*3Ca6H@+O{_*46Lg(2% zA2B_}4VGiI=d$DJU=%ifmpBX&7WyIF7oq~A{~uS^0gh$c{-08)B&DP>l2VGOjO-#$#-|_K^`+4r`KCkoq ztqTs=V%qee;M=0f*1>baA3c7T%grp;)U=kLB4P(T+Tbg=VY0wU)MQY_blzmc3DE+K zmdo&T0NY&i*$3}Q>v?j_^u~=kp+sbgBd0C_HRNA)RheZ0iIWEW7jxhV6l<25h4e69 zapfgdn-QZTR1PAd4l;4^1bHbfCAbQuA*2m)I~cP+nd{?45}Xq5Gq4%NMMYnLlnun-x1>k|cpw`E3Ib#D z6_;1gtDp+yyG`sw`&8!!zUacx4>%odR5{w&7R;l$XRhFL{3X9%#AxQi*Fagz-ZOGh z{;@PS-Pqhi|FfR<@Ar<~&$7VaH-O%`AKnKY0+uuU?on;DS}C=-`pVan+e#y=RqS$4 zCN+2~CV%kaTxL-gxOaao?+S{Zy#D9-p7uaDDRn))fMs%1RmQ_1P+9XR&RE%LA`{bW>&2Gx3Di2zqEcmg8v{2vT;(iLr)Ec-xxZ3gbr=kn9#0L4xrNq!_#K^j`fpQYS0EX>Tz8xpCszUd1ZwPP`Zlf@SE z{V%N8d)MyZLXN%4U(?t{Bdhm^M4SyOJlgI$aNOOtz4$}ySjPEE-Bx#xPW!R8?!xR{ zW-CnIcdbm?z3COB|Ki<8S6=a&zmsjH=E(H)je(|uOJ{wBR#uklwtt^_CKpTn-9(>S zG{PwxTT&v-G~FRL6OUzkDK{y`R-Bx^GS3>#a80Dk_0cl8c(YXK%D>ab4DpOcv_Y zQ04a#7x^BoR892hOL_;_u|J#gC?=+eZXW->quFac8G;b-B-dZJzuXJClHEoxT|Y={o$SNfGZxrqU{~ z5}Imi_h5sb?vpU^$i|1?x6hl z@tgu1s8fU3f8U|-b^T@-t_#!Bx9S#?S}eTf+SQ<{x68Bqb4L0XT+5d!70wwrk!>OA zW_oCzm#dictBhO|R;6eG5T)5o4^PS~2*MLXpo&#%)|jhpHTpc{EbRh$xQ4d&YBlbi zr>+>d_KUtOD~kxl@fYM{&+eYCt}19x9B?R--st8{o4T)Vj;R97Fu8lpfHt|k9+UmS zO;EHAk9Pz`g)-9arluZ2ZY(WsV4%1q8;EKGPFJZd)9@|_p5}nb&&U=3qYEqe)vGTW zFrhsFgTc^@bh;|AH_A%2sG9kAF*o(%qUEIc#KpD2F(EZijhl~-^0{7`D~QCq0@wAc zHN5bB)%CTaZxb|xqfW9nf$H^uh&cFZh3QAwUF@}s*I_%~DkP+R{P=AI=y{a?VhcEc zo>2=3G7i4ar?KqyJ!uB}T@wxy2r7btzwqaaCFh=;;M+(?^~G;|ay|*NvX*$;p4wz` z$!S_K>hrxyu!8Fv8s7CgGUVO5b!)Oo`|x3Zpl@KSzMp#%8WCaEv+IcO$*yAB7%c;n zs!>1E%Ibk#D7~u<$MvJFB?~GsF|j!i<**L)jR-gyvM!G8qtIPG$xcPT=J4(H4K8gJbITP4g|tSGeQ2u{{gF-ak`)(d%FsT) zq9PE4@4p}P>;;W;zaCFkh|;XQ?%$BtAZh>LgNY-@`ZO8I+GL3})rl&vuIO#E>{DAB zA~BCWF5~f}M*XMziu}hCPot5m#+}~BS7>WxRSTSW7$-seZe(cLob)jn?t=q^MUxr0 zd&`lP>Y)$p>?3wAzNC-l#{2$W$(N0ilDe3KbNG(Nqyc++kd?I~p%|iS>HjNNTc+e9E~99-;53QRHVBk z;;>v!D{UMm9l`hPz@bBaIWjRNXM>EfYoQ3oqX6``JWJ@pw6&^*+Pvce z#@8U4Ek<>|``uD_JzKYL*Mi2=t*BXUGBP?k9nC2ghJjdAdeDmb>}E1|mF5b$8G(2{z2d3wLWpO%E8{xa1rV8M`r@8FXpicF z&xis5u87#-{qoy9iu(YrP7>y11ws5H%%;2ems7o`C&zjV28)&?V}HU`97bee|BfP5 zO@L;aC{)`Go^2`3AKJw)@5^)y&jiB9FfR{RJ>q?aY%YsZ-=$AIrd@)HFm;)~^j~#u zHwUR!F)xj)Q7~6=a;D?C@_E(}gL4h=R5jp``uwDolozP+;h>7XITn0(EN~d<(VTEA za%~GL{i2H3jN*k)1QM=VUR@mPYHeo5Xx?lkhM4k*&_pK(C#TeV_x1s|NJ&n9(*Fq{ zpK^@!af~=vzrFE36#5OB4XKa~_25`|87c}xBA(WW;Kli_*vOl3Vj^}Br*|_554QOJ zHf*h|n=|)ja2m`*G4mL_N(63=8UDATD0bWt({%rRU6pzLT2NI~M@Y`Z;nc0oPb&iH zd@lFt_-O@RF?rdUX<>bj`tMLxJe(yhkD<*Scfl&@FJZE{7rd3f^{cMtaU)si9stB4uid}OPN^79XD(=k~`^&(Gy|&2R%EGw75{T?t8JC=^$uC04`f{?jj|eT3#uf$j z!CI&|N!f#kGzCP$DWx-Kk^rULN5q2QiF{T3#3LO6zD@KLCO#AZFj3kHf~c*PvXg0P zS_AG}X+_hcdj0LiatWqcI;2ipK&DX=**@3+&HxioBJ_!GJBM1&w!_qc;`I?)P;kAk zb}X~$1@cP8VUn&Ka(V)67%9wAJ*SvYsz!=gM?%e*3;F59^o&vAqep6_;zI_V`blSK zfS-XP&4KyrTtRC|dPatd@7cl1J(AHXPfl#d=jk(Nw#mroqb99hF&2aDlVC?|!JpjE z0K3;vRJ@9%dfAk2Q#bxE;!r zpS@>&98N{w3uZ#o$*FR}mV?<3)h!b^!O{p5?{BX!I&jcazHE^2|KBAxDDY~!<>j*f zR@olzCG5V;4HxWx>iXqZEp6crNWfkgDO$_FEyQ3gll zzO}Ik1pm_=a#VIv^8Lib9_?ak7|R%RSxvcIxCDoU9>4V+R+aaV;A$+MSXN@#PiSuh`0n90O(bxLT~*X}Uh-|kCCVcH5^EH%$_5(f^5UamDn$kugE zsd}_l%~;mCs8U=be}{n^C)p7Npn+PY5-qvvx6DxI?r0U|JG+k|Oc>3R+p?FOYkcnx z7Sjv>u+|?<8nl;;*3GpDL#WniHOV*{K<`@86n5OPR#Ur4me03ur$W%G4}_g$N1R(| zbN{X8el4WX1r1s_ZQHQ}C%SFR$9v2?Zt|0KhNOHx28fXfd@CxV>kH$8`I3+?9X@>6 z9<)?*V6JX@BqGfbwKHz8@N))wMN8nh7N}~;QXI_b28Ft40>${(*qwpP^dl;`18#)g zXwaHn2aihs@eX4CQm1&b3(CFO2_v zYmeLyo8I~#+tMxinIkwCva5dO(2|kZpmID*fUU0G-6i=zXH+5ieSW4V_W8tcuCjGx z0z#pT%F2@qF-TW(Qp`^NqrnI(VzQd~n$Zf-YLgw}xr zF9j!|x6VbtB%ZIavpZX^leZlv3#Me!%0>#Gyrg(_{dE487TP#BjW=Nw&$T6@Vty=G zyC3NOUBtmKGy`+1M#hVk@x7lK*wqkqy|5Y01+}?&T91`=gd0tC6Us%UN5)|_3}iSV zBMlWzR*9a%&V;53p+`*}=ew#8abr)sp_pv+81JqHrXHCd*xj_Xv~Gb`L#LE+>z0&I zT(8~k(2$ULWKaS0&_y{A-@wa+9fp)xFpWl*h`r0RM4YUNdKl2i3qM50l(CwBOOug5mG(k2?>=^1FNzvuSyN z7Mh`%Zo0q{JsU{AL|k3UlP438O*%R{MA3KS^(tp&Wx0p4?AeW|?Wc$zZ10c`{McQK zHzf^=KMIvb20I+YIXx&uRq3}Jpw6lWLlPDhwJqSK@xB)f=d<5s%H6Tpi6)0hvN$>- zp-dq|b56_c<%(g>Ve`8G77Jxs$4@BEI?VcM)mKprbL+`4_q$K$Ky$k3p%Ts$)`Rs= z&*#NHUrwRFr(M3uOy^o@Et72QQTSsYW9*ji5lCc|<^TasdL(p_`+AT+;DUhF<5diQ z{>q+A@>JkZBWm`NAIi~^nt0sLNk+ThVan?mniv+M{T6zPES-n9?%YWuIU#qn#TwCN zIY8QEP zd7Qu*N~eRU7?A9TekhaiJaF=%j=;g3@W^YW@{YZJgRBq#IRUmSw3DgAi%LEeMcgg3KO);}mH&?B84g1B02^1!ow zA8DK1)hrgJj7|!^CWHs7X=&BR<#UZqw$lvoewK{B4cjJa-6X=w+A38ltf8-;j$Jnz zZ-PfL?QsWE9`BO|d!4w7*0##6bot`wVX3fqyGT2+adT&qK?oW|zD;yCgBT;w{xny% z1&QGIq2G1h!%bVjBu}%ux7R}}ZJw8`Nw$KgxWoaifkktA9?IBjV|MyH*E#EloX z_UV4Bc5~fJrOnR;4>SqrN=qNS@aBeqh6sh-_wF(wk<^rmGma}M&Rr+6m>UgR$6 zHvZ5?^!h#}rE6p|Bb%;A#e7d3Aew<8P9y<`zwJb`Tbi&xI00>sbyE(1!e}vE;st1W zln#sRhJ33lcBVUapG7gSQjt@ySwdD^PC^iH;Yht+$z!Y!&*@BBSfy{QFq1s`-#Uc) z*Hu;7z1smu&807^eE2nQ(CaUD!)Nyai+~me5Y6!B2{+Fteqvj<9xk4!uqETRfpA13 zP$MMyHwqTFw_gW;MVdBDh?ul4yzUm(LC0f(^hC50UDbI|NL1Y3l8ls$PeX_Syc$jR) zK~o-{4@WV&Z>a^AsYLqb)YO9hPax5hkzq{wW<|wr^vy3S5YIh=1OBi` zr4nKyBU8xYK|_Yc<6&TFnM!Q?Sa?0Rrm>=QpPajYV!Jl?=3Y)u$<~B*(KAxZ7Eq*D zCLa@HLN()mqH#p zy7&&zb3H~bSs`@9x_LIy&kSrIdLTr&vEPwAn*y)r|G9{`1|YL!@{JTRzbR(hmWIsn z;F-};N8GV9l4rmA#5jLlv}se;b9qm=5>l}%^+LQ-(gbKx6p0L|Tel;u?B2b3bK_%o zXnDt?`ao{E45+;#BDPKeHrd9*vlo*_i7sxJFVh-h)_-Q=abu$Vl$Z>&_*G zexul#)r^hrN!&F`fo7pAyHYmfB>UL#QwZL$)V2u;1+{ZZv)~y$aUu;#NLyGKeTGrB zx?u9w#Ga|x?RAk?OiU}Ya5R#n10zyylg`-WbW{G>c&#*@y9lwly_$vc;>7v$+1Ld! zqLC>L!pX@doDh@tXFdbIgdsw#t8$*0xcIxlF~3@%VoHb4kCAqPl&IZj_o182MFr(;h_4_nmEJCXm@yetrT(*i2k7&u%1YJ(L2jXyzYT>n7E_)Se>K8 z#Gv&Gi}fx0|E*CNR!qxF^mj7JPiE#uaney->WU&AuU|VKF*6a0ULWd>F5Hz8U3f!D zUxFCDhzKUeWn`nkmFHsO#lmCg#j$pwK=)9fl|8xtUI0rQ4y_ppFq|S2D)mQ{?Khyq zR3U?B4{!jXXC2)5vsJ{@B#U7eW^N2L-H6~?T94a5jQ?YfZ0sKk*L~}AECAF@P&cD~ zAhnTjz{MFl=j5F>HATB1@hKxE#W-ug?|rxZ=mQ6P`<6$6+^ZWQzOBXNG~A!p-UD;r z?+G^^6KD*<-WYb{Yd}Ob1ou0+FcM~dUuMpx0F{{vK^`WH1wZ1pKkXFuC>OROLF@cP}{r z{KaODE9=TC=jtd@qd4yYz=Qo9^B6u!$A09J-_ zH7zli4>5xfbP}ywyJK2~sk(td3>s+>w{g3f0y<|Sf^(pYa;Yu9wk94-r~xW9U&4G3 z1HdKZ$wrtaI%&*2s2Hl~$WgA`-`r@CT1au{#~7yn->RY5dG?w9ZB=H!`*|;J&-bML z=KJr+8*1lv8z0GQyg^aF7U*P_yiMyYGv7USGZj&XUj{l|-{DC&+5X1tLwJG+>gv3C&4PK&i2P2O zlq@#~3Kpc(5>VeP>Ox%*IsICZqy)^%7pQbZBCZV6UmKlbgRRkyaLBOS8~cF@HO=F3o{Rgw_V zdJn3Zv4YAezfJu9o*|T~@SD6B+854QM`PtSj&r`GaNWvhjMBR9uYJKJyeQc`y!=cA z=b|oFm4X9H{9bMmlGtgrS#HKLs7HPeyJx&{gHb{s#isx&hbiyv98^_G`ke2~wjN532cj@1@2NoE<)T zR2v=1LoHq16ik0n*yT`#d~XKgudV6lqsVtzyF2fZuGI}6wg6k|qdDgIS zrD>&-eGFxUQ`%WLVGzZS;5I-<-b_kSrmkgVY)4jlDj*}9JkpPcRdyD1PJilm#0z`- z{T#A?_<_z@nvC7OdzU968mFnK-Bk5G`2ha_Rh@vSug`Pj?R4zZS;fX?fd`DEcfPaP z{EJ@bLGNMm<8sHgT#F-2KMbAf(#A<>34~rigvzaHoX}E(RlSmx^<()PYJKpQq9D3G zhF{(1&hjln|4WcKN1$E!t$t60k7pl4ro)5bgGo%;^79JIHkeyjg#IwyzwPaZ1zvQT z2Ta0kTx{hUc_}^tQEgr-qK^D>o8IPocbUt$QXn3p5 zvg0Qn<30k<5#6CVepWKt!rXk(V;QcXtLF5HPyiD;7O9xEm=H7!4ckrL1Ju?;!+(hD z!u}wcQT70A-j|*h6c7{;Te4p*TnEVnzfrrJCn_fPcL6;;+wn@j&=fU?6eBc1K*n4o zc<56RJVVG|4*-(`cDsjb)sNy7(1rw*=`0-q!afjy5f>L+gwt+Q;^XBjFLTgc~bdUq`oado_TjvlF$s>y?gySxZ>4@u4O4?-B^!~eZop3jSyCi(^STHdr30M zDZ}B4)N+jY&yBud^zBZH?ZGZALuC`xO(^Gbz2>^3$~Gr2&$9DFG^}&q1Sdmo-pm22 z0(LPaDC9P8g?YHlG#F zAYhbTJJzs}SnEvm-&&Y)j=xUmLemh8Jq*q{C8fVxsFCnhJk8(=Nylv6_y{6T2cBu* zv82ZY)S81~Z5GhC{~0OgSp;_)e`dqFH1bCvaynd`|5${bOih5qN)9wldlX~L5ixni z6cTb-2w*x&-Ww2}mKb)|N;#$hv<~_gv_55&{byVU!Z!~jrZ211Q<%D8BN z~M-hB6l3$L+<|gx>cfK)pe#1InOF_xyQoI zE&OR>+JlgHhXOgetHUEA9^i)nC$|D2a*h(UyoFUW0+Mwr%lNyRPMkeeg%HtUq_B;Q zw=vIdIRG`2{}dn8ACRtNxtXZc|3TBDqG7$;3Za|+Q+y5wLdOY-KrbZ4Jp%(p{huJZ zS+#QI7SPd-Qk5I&oW~`lq_7XrYa*GnR*)^=pDsR>iSabMFWIk=5&5R0TcX^%&Y}*7 z(4;^yiQOe4xQfFu^+2*pR1dtZ@svHE)~z@l()!j3S-by!*m>r!`%6QR@3GX^#P{n; ziS0doUB5~DV+M2VNym+!FYa#^jFb~NIPvC&67w4R52CJhRWJ7|Q|QBeRf_RdSO_~B zBgGml2_oCrx%VfUA`$c{;0D{!7>&am5Q2seN~h&&G}1TQ?H0iawdXJWQ-VU#_>80) zP9G*sACytwh9>q=QI1kclY7P(<uMTy(` zlJSh%*Eu=uLBJ#Q+dK`+ZC={sZIhETL^tpj_Bnt6^bm-Pr5hGQEaSlAfG15eRjajb zSETRB*B!fRe=K>BK3Se#x>_~7Jy_i6_2e#3D&0*UdEbqt!Ay#5ifr$SY5W0JTKq38 zj`>uadndo*m~lCQ%U)Gt`khY3_}R=t)UOiyf0HrXzQQ%UZk8 z?}88|F7N)KY$f)87a)a_@7Z!;&XK4~qwjum z6_=cQeF7eA%)o=k90Y6@7}-gl=&a87x#F=y(PaOApBT@X3phmEbMbMnVZf>y0C54c zS1e)hxqnITF2U5W$xZKt(R-4$_o>LSKN7%c7>@bC^1Brl zwyLJcWh7<(MGB3cYeUkT1R$ANG zX#HW^>e|{onzP=#d9xKWEXv#j=e(+3SllpcQ@I9%$aI3K2^uFJ#l`Hty3gi`I!@Gm z>ee<1H%xDxm+Qb#+s3IHB^0+OQ;BlUQz}*-4&xe)_36vcE;l`NW8+dzNeu|RmL8u{xyG=qE`u;Ek(3+k1(dsGGa@pb+xr!L7r3b$nn1qFb{F4o+p|a&j4#B ziYAy!)}-m}5*9>gC zWTwaj4(*F#^3V0CSlC*zS0&;FC8zd&2xvHX=nxrHpJGNJKp6F4AV>0Q1LwVtQe*Gq z$3nxyd-0rxR<3s8F8+hKh~jWluyo$)5DXBFKdD(b1*gR2$dQ1VsZP0GqH-cng%TrW z>_f*dD2)%J0u$^ZIr-Rnmi*3Xpnx!d9l;ZXPoR9^rC+)(m*ua9sGX-DNdSnp8GTy` zlqh_=zS`NdS$L_NONBig(_R`MLlcQnSroPUxWFxgHw!49>5m_4LE^Biu=0h4`f!y} zh3+#YZH_ES@NCNZ9-6a=wg?c1rlT1;=;++clpW=xR(ZDP)rDBs=v zkMu4q8#szLCl@YP!adxYCWQtA6t@2c3=FZuW*^?o-(#rY^xNYv12?-I2DqKL<`hW! zXpJQO%0T~yx=|abU9<_1Dl}i|v%iDc%tN!K0Vlt{wi|ZmT(WK+drMK~h}r%VwBu#x zF&8{_n|K7A=u$3{j>uq)|GyI*pMmLzJIQ#)0Yhv~AVF|%W;c%Z|LP)`r+b7(18a$i zPmQ=&iy<8YBjW%FRt6$X(x_o`4uVvKpiw}fm(+tkMq!dZuzxtV_lWmp?6sH{(4>_Z zrK6YvX?@gu`*DIej!y>zrwHnA7F8unQfZX|+AR4du4QAthJCRGbpc7NMPxw^WU+V- zhS`A?Wq@t7gpeo_M_BDe37;lL`8bX$%zSfZs{>r>Mt&nL((9rHWo* z^oB@*Kw!|*wRzIq0cQDJgDSfQB9?D}663c>Q`BRzjK7K`S69);EesH(S~yZf|z|7Ng>ke!0B#^+k7 z;L@~u2{aJZ zzSm=4uUoUN_sa4|dY$j-c{*!fOKo&Fed=pio7+c!0XHD)pZCCHfy7UL3)>fbt?j zvxa5!So7cRjI2Gc$@1tq2%6VkOXq9yLe~|mzSqw#!oqUT&eikW*H@T-oI1H{ItLvJ#)0kR*X#@6nAh8;_>#z5DqVcM^vYc1PYNhiP zVv)hF2JMnxtsHIYJOF;2H&ycAV%!A@AY7knnA++I#m80}=5fO4cj+=Y4I7(`Tif+h z@OX#Z^p(Q~gvw1Qqp1&<*aTi?BXyZDLC zaj|O+{qIiw=CiPI8RPWt@L~<9-ym|0C)z!a;`U6^Hv%+`gxC!=<8> z@(n;#5-w89jp-a$&>;+Cw-MA>6)%LMCiLv*{LTxM5X@Lz*TviZk*v`7B4da8vve7e z1#b}&sze9*lNa(2*w;~y-}XB~M>@8G**ne0k{SSiK%osVah7r{;a0*}*zz`5CE_F>n?4-5Y0_qiufoJkND( z9j1RDczN&E9WPRzlp2fh?6~x1V`Xi1Kl9H^bci_n$LdFX9Jg<{vlpm z4edk{lvBPZcFhu#WEjEeqV*u7pA@wpAyX=PEFS?bkFF{nf(@XO@bRs?10O>e^wWOP-IO$9N8i81Xf?K`;*Vh@v8uR8AGk z{vc57gre<2N+upm%vPpD?o(Y^rGCvxtfb{R!di~uxkkGg)K@Pg@kAxtD44mNl)%CoZ0XTJtDC`MC0BAXcKJaG>@D|&GPr6A|8|8iNE_3UO0 z$st3Np_5Y{QMrN+fcOe%E*g10<;Vm)h=h`;s$A$*P~9`~uC+e>?PHrKdIdBK=D;n- zSKL#o5DQ+gvCefWWo+Ex-j{MqldH?MPM1r3^PfH&5b(LBtdvmpnC33GocY~=lSgyl zp*nr))bQCy7kRNhU4m#}At_^Q1 z@~c^s;%pQ4H;QblNWS{!#-psb!=EfisSZVrmgp_JR(}w8t@}E%t%zGWE&?k{(F_za0VPBr4aR%<=fc?EgE@kR zD5<681c6=9^6*T9hHHB2kP46`8J*ihMk67dc2WZ;0kDAt?$zckpdS$$WPa{Zi8m*t*i68^Za~*B=R`$D*0!}}5Nc+{|CBwGyyg)I4lD@o07aO8K)@=-1^ZsL zJi9R4&qA`EhSvo}P*f-zXQByzz8e@&nwB` zgs_zdID&$Zk38-;@kJY|8%XkmmQ)(d>)K?H33;~WUq&h;EpwyNI5Aelg6#_lHARW^j$O(9rG=o68Jc200TQl3MzH4P}&vpK8 z)2pSf{rQ8?3->48OS??v!&OA4oZCeBYqn~q$;W@?71_Cc-*T~{p_V1sCeVT6Og{HK zX~*-sISBX&Xrg4O#3^`HFXN2Cds;w(%N(*FT~#S_2qx~ppvear$XS75dh6Z7Vdww+i;9|gEV9>Xh|&L z;E8^KSczhbvaH^ zLl3kUNDj7bX-_8T@|>riR`u0gHqC@k`FrO)XLS*Wg|)SH_gR!8x*osDMKeCoNkZp+pq3()b6OqtZNR{ID(- z^n$F0*!C_FY(io4UjgjHRUmFH1(?4%z!DUdCEd#)$#ni1huvcqM(2>INl5r&4$P3) zah%42#`W+-77N(AmM1F>ZTjD7RUP-ab`~dJ0 zPrY9)w)&AW)}wY)pKzt~Ajdd>k}K&>&4E~aU_25v7r^vZZ0`8!;ttPpunL5j#mu6s zAOl4z5pp0bhr5eLXSWU2p87RF$Q}bampg!KOeLaA)adBE@CK4Wl9e?c+Ckr8f`g-N ztdt(Q*5{a5yU=b7&6m1*NQ;e=9g1CkraKX{0J!-xAY-EW;`(8I)u&JFHIH=!FylY% z=*7C#1TBb~(1T4Uk8>R)XTU8iF-f|enP0UHzcIQzb4i6kwC#Xebh)kL#wnln;O)8g zy=q7D9JTT+AAC&AT6AM=2+Q5OD(lKQXz*4X0c?96YV&jNX3(V*jtMQz=fOU%vO}Lj z*-zFf+dl8xLSb>dxgp&@=;3WJK>vL(*-0zU!D^mc>jaWd(l`sl$wzBmjYS zM?1A0TmLis&9Llb->&J=yyDaM3_tTgNaBJM$G#6uK&rYt&9%^g+!mr#ghA%yzec0+ zvpl1OE+OLxFhfkID{I<6eR@DNZ{&M~59H*uR;&5B0%ichZGll?tp5-zmr1Z~h(Jg` zNES7aQlo;btYZM#usea)H-shyGyA!R`Z$ByIP&L;Rjcaa>j6%~g3Ma8hx%)5nza9* z<8mNN=L(o!>w6b!Y{Jt+S^z11Hbz-)F3PYrBui9CIOHuLMkLVm2HwdAcVq9XkXHjR zK~IRM2kFX=<+JLkKcn%-2gl;!R~+m)1M;mGw1OJyqJoU9tN~W!+H(D_RUkLe9D2j= z))sFIvpnG5h2_D|DaKhkB8b7lqhJPW*3N6i(gEjz6F`grL3E_a5{le57RCofMFxK` z8|=|e4oq&_un2GbQAuLtAY<=2U8V)IEg~}48m;mSPP7R#5o$LfVLu2Ql<>UPFxW0$e zX}TtB9hb5~k7r0JWKtIRv&2^MupTWKY!yl%#!lCe6{wr;^5gcIZ)2077;YOC;_>-w zbxQ<|OR(8LA08qu!)DVJ?Nirr0>|t`dT~$JSAC?|SAhx2IrU}*G10IEyhLH_fMn+* zn0~U`q>&;3KYXWMe|JN`7aqC(UCPNcrVbNYqh8p#~3#1(wE%0tPRSSA%0jMAZMQ zkqQeZA=MHzyqhst5tj&(DnaVDoqYzJ2YMk}p!2<>4pM2$*FT97$Tew^@I}^R62go8Er{p?2 z;NToZJi^nLD?14bi2a#t(dcK;y27i3Yk=S~G%V~N{fVUH)72|X;wE}6d2eD&t}(d= z02R*-DkYTRuFccH6G@W{voe6t9{GyJl$*8noK?w*+ciZTQ$BPQ4=t%06m_MO)ab0f zD;ZQ1?Wn92SX-8$@KTb#Jogd$aqIRXnQx7y#ZX!#qLUxa{}AZf;qTD3zqFA15FN#) za+RxXL-T(Nk3z9|3GME-`*T*O9rx5c`Oe5fNypOo6tURGvsIO+UN(se)+L`^wEWTW z+ppKE6_zK-#Xd176H1=P4%uV!cO0`#hK<;LWvN}f@UqD!6cN=bklzH2J>%4%h5o=D zSk!_bLnqddvVX3C8V25yOd-%h<@Iq(oJ)YR@~Yl;)hn*&p~=C2+4QdmfN#^A*b zk*(eu5_#LSqX;f9T{VJpqWrOutNK8z| zABiRZh7I?$chRxCJ)N2tUC^z0Ks|h`Uct-xE5B53;FtPvwe9+ZlD6n_oAK-}yUe&MvZL zuZ*XL6%Wr@aCVg$u5McktCN4f+J1rVggU7E?-nw^zl2#Ml}ITr2XpNKN;~LDa|N8vs}m3u)X+ZD-~LF| zjGM^AnAy(#Nv>UpgSD)WXj8a2ie8gSfco zSZNZ(ObDSA#4f+GO~CJ^W5@-Yld~*j9>t@FxYiJ5yw3vO3~Gg~2QnVv-(AWD1uzuU zp}3u{Elxw{(b|LCNCE_oUQ4d3673%m=MZzKT9Chg4J^x$!3;Ar$^BF($Q`O0yen<- zLg09DdH@awxVh*JW%w4hgIVRctE;PmjF^}fzWy%7!}0^UsUU_?g^~dm;VEow@W5mO zW6C8x;4FVYX1ifq+Ndq*B_9@Ie8xlpTyi|Hy~Ii^3JEouIyw(vJ9%fEHSBnT;;l4Y zT#FEvQ^4d&)N2LPK5qIaxwt z=-8+g`(;z-te5BB+3cP&9dija=k@wCUI0ckt3Tb}mfrgHp4)h~!rJAXsy62mUh(bU zQuFFmXjk$(?yBNY@g2$QC@lwZA+*r&T3fjz+qP{R0nmO~Acq~xq#b9mLUX?Z18mGQ zgG|}+&vZ{(2ZCMp?Yl%+woQC|k*t5S1t=Z|p>Tg}+umlvGfEQ%CY!F4g{+a(ppeT! z2wUhzbF(;^0B9&6V*J&gN8m6bQ(}oDEJZ|21u!@hY@8Y%%4Ls-e*Y#u>f5WgTq7rm z!TI9$un_J|DE-UJ_UDJ-{+L7BlBeJ2gdKrsy@4!F3a_D{W0R`pp83gu$60z?6pQfo!Qu`}SR0698O#f)SlLhxKx{an`Z9LqeLWyeT}aY1 z=W}@{SXtbsewBmx=&u%sq(^3?RviPiFwYvVwtBRp1YIHK;SsFU!tt*GcRUxq@Bu4w zw!@dq1C!mwlLa8O6aH!~F%{tXL;Jf-T=+!{cph6?fAO2%5qe9q_Xl-8&OD-|R7>U^ z@ZB*o^M6)N(D{B4b9Lx@dZ?~nHoGG(_RX~{%O;kjb1OES%4U9M!VM0cn2`Z9Hl*KD zyBZi+5B*#QURYqiONpaXDw-zJ>%VyQEmT49{LBCC`+epGoJ}q0nY*y)EPI)|>{0&E z>GhseOR@`rya25o-^K%5>raYGF;kyB@>Q{R$RodrTDYn)>+O-iol>tV35Z2(jcV&) z`7wa=kZ;EVm%T77qjgW3wS7X4H4I}#_Jj3*%Z~?fqIpJk5U#=r zq5nGT=|-8E@eVLowL^d=%W&dATDm}FQy zyO0f1SiT1YGFEKZyYDV$a!Y~}h&~=#%45X&Uj*X=0cl!3F7kPTxO6fA5o&KXOa+E= z*uYSx?TP{WLGJJe;ed!&FsS^z;UuJ`;qSgLwmExNfD*{i0Ku{+HOcd7?=OgTH23bk z2tbNAHLNpep7eQQR1Nrp=}rmqKyaf1eD0ZETQ|% zISX1ibhXD{ErQsHW zag0S!tl&bte!y#9TsXl1k#fB(`TMtTW+?7%ofj0apwSPL)dQOqXqxt4ButKYT!}&M z+_%>T6Vf6~@`xl-hs;r-)h2J2j0bziMXfCR4A2=EF%%2T3+Lah{Eo2V9{5t~Kz1t~ zW{ArqU@(d;JbTQ^aD2;5d<%&L7bj6dX<3CD0bcq~a1NRB3MSiP3$q)o2uzDIOe`t4 zZzrR_H&D1%f>n3~4vktuzkrHs4#YXEkx1AjQu)x(+Cz)x!L8okN|ZqprkOfA^T*z9 zFt^(0W4hGA0`|tf-FX;y6N#?p*|Vuz)9|2@;kA=v(yPI?i>6(_Ud~pG^7FYBTen>eW1QxC0kbw>)? zr&aJe`4ZQ<{p-ht8I>wT`(FIoqQAI-O22{F%cebPu_VVJm#}1iiK*THxWR+qMFi%4 z@9I@MIkKQ>Gp_D1g~}CNae)4V_;peA&eFIk!f zzG1JIYZzy56|a2s+raa~0hVXrf5D3&ni(LuuER*QAR0xq6F;&aFSCADm3Ack*&yY) z_u5s1RZHSo-~Uf$&I_NH`LRj=6UZcA{dJ_*I6xaN?i8#3Td}F*iP^^!mY-!N3#x0J z@#UdU?*o124JZ^IYCmC3Oe#$50lwuAgws8Rl+NS07~t636K(|u$B{lsAcb!DCbWNr zlLG~_-t;r4P7&P~GLu68O$zutih6-mBL-qpYi*TTJ#cn62hs0BaI^_dwS6xxc4G6Q zc6!d7A)yS`fQeiXM6IMGI*#RTi zaB1|-fE$Rh@1*et=_|N0@;AX4$Q9(HL_8)|3~0+n&Pq}k4FRQ75mXij<%SVxP{Met z;s#!QL;o04GiYkkGhy{RjYJYFj*!q%EZV;8Nwe8%ODqDWSpC53w`|^=babbY9Zmoz zP8BFsTjOp2dR3lyGO?H8^%GwaRPSS5uHSAP!X~pZ;APENiPv6ONm{{q0h$A51&>M% zR`GzPA=q3%CAa>}bh^50X^GXA*bLF)6C*GBQ6vO_=UaPXe^Zz0+l=(61!xaEwh?t$_&!j?Y~!J(&{dWV55^g< zUkgz6xvseN*Ux9?6BQk}XweMwDH|2LRSFEaW6d0|Ns;jf2Y;}r!3tRoBg*ZcYr8f< zgXPj0?F8?1MX(D-Sc1>P1pTj!_ql%lg`q?OsiBo^e%S+7+(mnEy~s0)lzEH{L`D452!M}rE<{k6DFX-6 z%aeRnV%6?Z#{>hhEYx9s2P@;9i}z?Y!2mt2k`ld7CLrrV98o})Pi;-jiZoyw1XHeo z@=w68bPk+;0r$`aXPgja8R5%i*y=Mde-Z^E>5ZUuR7IXKv}V45Zyz9)iz{gJtPGdT zX>3V7^3dLBLRR$99r@g7V7Bq`y#k-OdDpI83GT2uK(^S}KRp>~;klX^cV~sTi?|a1 z5t*TN28gpAiu&~O2w;-YvhKPha0mVeU`5Ibiz#&N@K~jyu=&=ag^dRg0eKJIQNS4B zMU(ynLe1P-@@qx|M~D)75f}h@iWM3^f)mI@H#Yh}4whwfl+GR|N6Q;g6~X#pyd z87xQK4=*pW5x3<}58vUeg5stM_AJ6@<9@sqz81@AnWY2qJ$$R)xj@fG4=7Q*ubgC; ztgF56jexd~@9wPV(p#$UVSPTYTh-d{hiRGFKDGUsZBAzi(uEEjczj+Tjzl8vJ>rNwUBp->pt_)%yl4Q4@*wj)}P}qht-@+|0;^MgIz}i7@zH)SP9jS8S)M>#GCHp zZ2v(0yx$1zz!E)O-QjvY+sgtPXHfTvSr_pO#M#mYJaOcM)ZnnrGbiwlWRV@e#`@=4 z_7pcXHYO965Q&3%eL+BypkILUA4V?vTO|0$5TJsBH84d1)iytD@*~TTJqQxIAyLRcpa)ynI@L^C3 z+Dh;tWq>?DNPDyr8`!qMiy+cO5auiJnZROy@Q!ZUnEP#h7RD|NAlT{}FlNfL<j6ku`9bEgy8a=>GaeB(YE=i#Gz%MPoi#y>psn;=@#_JjLl3e z9`+SUQXcYqX?E&P$K9BKKA8H@1={26VdO#b@%4~ANLZY?nkqRM5e8|EA)(z-3B>@* z&C@Z!-@hl!s@2>_5T*Vg95LP#zxNV-RUj~8Ff7^PtpxK zn#(C?(;(xqaA2zD;ps4uGIMp8oayok^9GhAeaCG9ui^$V{~>%uE!#<$mb$=eGKA%1 zo$Y46rB^Pu1167KCOf`9;TE72LmIWAP%N37LXPnzu_<-c%Oy*`qsr$(OU8y3JNqHL zUIVe6xb06Ng%_OM3?CkzSGYeAwYZcCLJS%fObV1)MuL#Yl4J|uxXMam`Q&_4!hS#g zSW`pNP9680=v}8k=0ak3`m`MqAXHZi<9in~pFBBE>+qZk3T|qrO}A&OUSP+nB^WyK zO9A+7J*R|6mDtk&n*W|zNO7tH#Yl7=L~tz*OT>nBHuC?L+WVA7Uc~DrWkL=+ZEJ>H)>l_klsaol3_96in z=I0j@7!I(*jmpj?$w~L`n?nSIL2?)W8ew7NT&MhNsKN!0jlnScQQD-()>aVHacpp=UB=C##!&p(4=~B5VZ`e|~ z%((Ho4=X95xZVU=iEt41N3Da|A7BM2bRGRccbd+tr0@Ej{Y$U^kUco3;Qjom!&EB# zo_P*yL*ZgJd|u@cbhJTg+M@4w>aTX1RnPBt+y|W)KMAO*B`CkSA-3tz^mu3RUmf{z zjs??ntlQ^3_gFUc394zJeeuW0f*v981FI6+S>L8uC_Sq`8hNxTC*<3{bOJ zCt}gxD;X#Q7YFk*;z)tscm~Zte?uB5B!=7>dv+On=%2W;e}E;$tMsveE6gFL<;gXi z?$KT=W)-1$^~Fa_1PjRoIl%E-H*I>Eu)oV1+wFfEGK4^)Xy`sV zb(I8-B++?YL>MrkQf5h*%2+u=JQch6HED>Za6lo&SuaWi{poOqM+Y+|KLis1lEPZs zJO^T>F8ofArA-ZSjR2>YK|&6xcECL9Pz({si&Rzcr}$foIuBA>a#74k247EKA5-#X zNNB^f4Zcju{T#%ICok_k+pbwyJk4~6jSB9Tg?XW&D`S$plHn=&t5Ad^3xETLBSef4 zZ5TA-!A0nJd!a#QdJKwaW@eC9q`i}lox{f2t;sW8o`i87nm6zkP(Z~xw)ENuu5yNr zKxQ!t^`MetaEIiVwEKh)xGrJjjusMBf=1@dHS~1AvS<3()~vbkuHm2ZGgjZAHIE`E z#PcGhT}A0D4mkVo!}$TVG6aI_e?LB7Y)of6UA7xUHDfXZle}P?*V=+9Ih$w?cxz@J zP0U}+&Ud+JVd??@T)kDj9oK`>gElAJJM^lT>XpfxJ#j9T^pD*6A9knnUex6@( z7UCMXZ<%gC%m=N8_#;)oc~{?f{UpXX;_8Jcx@a~E5eQ`OP^DbMu1e|BOCo@Q$t7g_?QYJhVHCR;NS`!9#$7}NV&$P*>q>4$AJK>o)W{Vo^oYnD!LM7|aotyC_Ntq_GQ?GwL8Sc0cLiS?}3BXN- zRl!Jh69otMh`OexwQ1tw;=MSrFPn(JKNy}=1;R_L3uH<5c6C9tLmWkp6PI z0nh>VU3oERXaK={=iAh!%Gppo@3X;oz60kB=Q`e33NL3mf6dL88;%ciM;#DLU3oHhOO5F< z5JHSm2Le55yMDLRu05Vzu|IN-N1U2R*Cur)Qls-1NqwPDgUc&3=%|R)D}75FoBCPW z#nF-85(*eXG|zhTcCMeo>LNuA&Rgc>qmV5kB4T5^;NMGSVh65+w-a4cr{L-Z2>WvZ zEFc0NRf2<=b;amW{A=n^ER92rzy(BT5wc>bERJR)N#)-C7>%b}+ow5;la?JrWto|X zrAZ`=#?pp7wMf9v#TR9AGaYii0%|jmbXnlwDV)h*NWcZOJ?IM@-g&^nKonOv1Orr8 zH#qJp8qQJE^qd)rg%0;pE;b#aLO*um#GR{e-go|TCaHRuT7ciS)mf4R4{$7ymyPxo z7h?h-!Bjl7)=lQ%QiGlN%EYQ@rrrv1*RG%>S%7LTp;Ntu;}7N!6ut( zm`mOs4V-GMtU?(l@&m-Ep?u=;C1J2QUZU-qq!Gew$UZIqI{VGE;;V>77I*TiKvD{S zp0e0?PwieGv7`L=aniS7`F~$gyr3~esxYjrhtEew)3c9b8mYT5MTmfT>Gd7z0S%es zj8Top1`cD=AyFFGwRm)f&|8Dt^_cE8+_quE3#ht@odmw{|D)?W;IUrc|0P8st0ooDj}poHZ7GTGBPWBR7U3ieN&yz`F;OhUf-O~ z_Y}`_f9~tL-s{S{g{DwdIY!b%1&(A0xPi}}Dc@ZDtNzETK>U3TVe(j_c^6Y>C)77H zDZi-bH-2jx`IYLTVkh;0dWq^sC80RaI|6%pC9biVOtLh!{U91ivUpl!b#(&bDc}`o z!qB?{A?!1p-Aa?YckZ-0Pt1Y$`EFL$hT5s_d<)-vyhC||WjsWTNX`Ip08An0^jE+W z4$Q2EOpc@qeL=l^4=)6=$<(`uuCTSWRZsW8eHh1pxsnJ?XjRoi@~@lWeMD|sG#P!N zl%ZcG@Crl;$+!A3IxP$7n^%fNS|CDK_E)i7dX9wD$5V$gBM z33mzaqT86nc-415kewqOO9}*c*0)r>z9U=hh>+itK!1HPMOQ-qve0MfHW@8BxKeXv(qRtahc^_yVgZ6E`q&51V4nng6;@zb0YL7 zCywkJsG8Gq3_6yXydlUZW@2(78^QGk^hj=c{;xlrX=!C;rE6DmqReboLqd6x|F6fI zNUVTqnPWl|UD2yTCIB2AIxf>8oaKx|A<=Gqo5I7H7Xgsr-e7wPNhgc%kqkv-ucF>c z0-zuz7ec=@<2`#Sng-`O$ZMo(&twbJAqsd6A&rm*LC^<~zc1SceJ}hLhT*q<3&Y&# zq>=!N5-&-)djH8UuPQNtS1dTd4mmn%HL0FRxG{Ei9^td#fISsRABjO*zIIPbj6bBf(W`>R{S48htkA-r~Wg*r= z__{HuAdj@N&F$qjGSxt)&5hLp9(?;`Wn_Fa!3Or77ND6g5F!~nw?^K`HNL7xOf~gU zU0;>kp%X+q^W~Ppsh<@;?;JiF*5f!(h2%h?`^16D{a$e97SvLxSrC09$pvE$kzf;L z40Je6c-e=r@Ztz+MBzM}6(s#26u~?YyX@xuH8vD@t;6*{qcntlE(sPY+4hD+5wz!> z2n5D$^$uwY^h2B8LOhS~90C9`e1IGVA**X3u5LhLOgZs@Lg!>Tj=mR}ptond`AZHW z|KSu?O#;rpstHmYgM)(=o{FwpX9$wO2yh8##RFNUFMI4f{3jP zx^mL>-9(I5=u<*Z!MciZ_VdX_lh+`yO-+pe-FY0tG|muOCES$6F%RK29;}I}_@?f@ zK4VA;$tfaI!gMlEqaa|ifazjPRV4$NaJQ~pyy(z0K9_Y0Kh0lZQ}i7>{ci2sD1R#V zq2`{I+r-~~_^?k8m=0!%LJuD2P$X9vAExs#_ka3W{!)( z|6bovbg8lDuUm@jG*!Pp#`n)vzU+`{#^ zl7xC;vN)Z$7xu}>OE%3X!M5lCNd_$5 zK_rtOR5zsQ;30X<0CY8MQ9#KLLefW)$Fu*JcRz++b?3)DpAe&=)=bk*D zPcq&il<4YPl*c8HF%=X@d(IMmz^r822Eox3FA4S^E>0{+iy-J5Cte1Urw)b`bxM5< zCYeOUhJXpW@J8VW58M!-N`G0I?%Xjvbm%LvUN{1OVZ9Jq|4_G%b;deQBKZT9_oyxZ zED=Y7z{G9M^10zOWV0cr4i4Yu_H(Ja*ba>io!WU_;^DxBP4gqXl+=R{2Ywci)q2X| zmT-R&dfN)j-(c+)j-Pzaf>0T#6z=0eJL+_a6&;=Hk4o(on?Kqw+H?5xBr{FleuU_y zFH#==>)fdJFOA$L`L}qVX7W9JQxkq4SJ4P6js^)dVROjxieQIrD5k`cM-p94z5|$o zSnmf3f?4EWOssL&QA9||BM+d^5D-n6(`Q8&zoHUo8uAHtPb8b*k7tDNj8ewN#$7~j z_$TRHz`bfS1u+B=vw);^PQwG2jAwM!%9Z7~lzgz802@f3wZ0}WC;~Z5-v1O@2808x zAX8g8&kj6vTep&g3ajI3bCApEmx&mw&70pgkw)d}p=h$Kgiu@%F+MJjB$8yDGD7>o z+!~7qkmDk)YJX=}*E=*&kacKu5s3!zo9b5Q0R1I$3v}&7!)Kohzzw2yq8tQ*CAQAN zvhnNi4!~X{^%O(3dKWQ)Lh)RoUIaDW|Mi1p0gckrnQxhvj;jja4^$tS*rpfy6m1=m z6X4qot2nrRmC*bQ5|xI<9{|_z3#nx8%1t;~Ep}p9i8AX90Z%)N?5g%JfYHXeFN)l- zxWK_|LYo218M8m0dx5SE0W;}*?(T}y(KJL*{DLPZbKExPiAw>mPvlw;-+Rv`#Xpao zR+_F|3~1#l-(l6O08nPm)rDBJ16@$vF+V&RE_^B>tG&fH3 zUiYuEM?;g>VHj8g=NzjsOnH~X8Vpgu@l)-X*iA8 z8Z&c>EZ|2aC{!h0gqc&6e@5w66NaBFSMH$w!P*%rzgv$qr`UW}PB1zcbl$_LW9Ug{ z(c=wr;f6z~YKutn4{8f>Qw&{$dVysBK)rAtUfa*U9&H4No}L^b#yvxUjmM)%cegvm zIQ*gz@2_PaO2G-Y1p2?@);$M1I=PpO(c=Nv+J#hR65{u6mw^FN0plSM`aZcR?|xPm zGGUODR$cvE>Vn&+ZQ!bKp#g=xgtCKxx7Ysr3V;?=;klfNY0S)Qa5{>i3B{AdG?Hse zcXmGmZoSkAUYfbP?$)HH-mP;?PpC(aly4b8HEbJj?>(Mrd-$G({i5J4{e-YU*pVx* zXQUPS1%yB7>l`z;_I286*3R58?#zkJHc%x8ZEcPc{_Eg$C^=>P>pG%oDQPSdaiAHH zVs0O)CT%e3i%4t*L~#ViMqVUV0O82sI|ZP51XI1xP-^~^Deh{(S{r{ywa_R=!pWpG z(r}W%frR@eflNO(%E)xUY*2-Ehu{L6Zs?!%zE*(a2D8T)mSlKmAuk*HBRzDBFaYrq zu`q=HV`F18hHxNHB$41b4B?n>yk`yJOztWko}fjfRxKD?4*&|UE6FDmU%R#%ky)r? zoa=5QgPg>?!i_=PXe4kR9^+djGy~1}xsa~MZ^#3v5xxF?!#IDa1G^w@Bt#O99!Z@Z zy#}SdXRyjmk12w18zLfg)wlkX%i2bqV_fOBvNN1@3EafquA-~mZ*K_^T=S5Z-L7ZD z9LH{RvV<7ybr31P>s&bNp~oA!enUAy^^pcKlB8XC@5R<&qt2k$(UP<@Iv2lPT)yOR z+upwsfwyk|A}dJXyt?`?2U&jhf5D6u>STPityB=J*^bBdTG^{%oz?481(i?JQ(N+y{y4JdjD}g` zF?qMDyPrR%pMPjp5&XWNL>R);eS2~Oo@5(HO;p?@+%eASo^Twt7pGY&cl)-3=U>@+ zG#z^shZz65lxP%1b{=!ydyj_3uD{_}^rQk_U4pMwSg^q!)U6s>#A6`}fa2s*X8sOluLBSf~=t;5K9n&6gq(hS%fGTxUcBhCU&OeyqpQ1nC)h^Jqn`$w zo+Isuj1zblc07$o2R<|Kk15Xenkt2Up<+R$DM*}(;S2wgxq5xs{Vu1bH^Fm|1-AObeI$Vf{e5)u0&{*z z7iNFg%fF=Y!!aj18ip;njHBnZU;1DJKKX|A{+yUDM$**s3`>HIiTRC+s}q>ZQyR7ZAfuTz~+| zy|YLr4xn8kO_tt08W<2j619l!Wd=UNzX5zCfZ-Rd2_{~%G-JH)Knyr`Fq_ZZN02oE zRoub(q_lto$_K)jph+X*O1Nf`6N}&Yfkn7v2@r`=0D>^CEJoAd13fc=YJ{@L^9asU zbj1@DG|u43N$L*z?0&gVh_=FHO4eRs-s>42h>1>x_?8e>z!}xMNEYa9{>fo-rpR4I zWM72v^#Edk5Lu-JbaOz*3wjC0NE{rn4sbb4VE67F=TE1#$pR*{Kze%AsoMYoNLMk7 zk)yu-kG&2!KJ=x7V)F$6cL9AP^BZbmHs6u#dLkS`t;1h)FNA&gJ#Oldat@LZAiav2 zCSx9xMpMn{AH7+NmL2q;Ge7o2pO>;?)4h+&Dv9seIIy%wXA-OKqT&3nOH?^IYtKEm z#(A*m0Gt_j>dnRlm(Vb+sWQBAQ2XzLhFK1VcCqSA0aw|YaZ9npBnE5PDYlx z?Lm`jvKp!W*gUrZrW20vIX>l3#UW>b2zTj^z^y{ae!>}GH2#w|3{wzxCZT&K^d5P~ z0B{OTV#Fm2Cjo;Xt0Wl-sNolfgNVK!7!IPVBd8b+XQHbSJdK#xd=g-b;#N3}UUa90 z1uT9tXte3k6uo-&D%7(HFFUEL*sv#&nDMn$6<{+4N?)0!NpM+{xK@(ZNyIQH8g3U> zA)EvYbY)PoVB*ony@MDDuX+XOSRkP1H$?O=xl}iu?W+Vd@q1dUscHH%tpQ5KL$-R| zx(6(V!z^IBXKpFNn{3;%r3&}V&!{4g9OQ2(|7_>L6!xRT489XW+ydFOiVMoaqx zj#*pmVGtT|cZN(JuVD?NWNs{)Jiy%tk(vOEoh+6C8~CjMSspbg2%ZhHQwAjtKNQr6 zC~>jVHir*aqWc3WFAY&F@|&PSBMtX~7njVUb9#@!v@LWDyb`%}i)f^FnmMapPtClu zjw#xn?$v%Bd6yK8z7NM!c-az7=*tVUFNTFxCaRpkf)fHF_l@Q5(O35cJ#uRF!e0IP zG;_5tG{(VO`foUT0uK`|S`32+v=aVYArW)en2jj`_T z?jdBqqkO4ivFpP6(y#tZ%i^#AqZ3$XVIAJE4S2^fnO%=uGeN@bprR5A41xi-Yp`5k zBMi{bK;Zy2^Cm45M-|i2Y}aY%n`Y9@ndd(N0o+vs6D2`XaS>>tI0+%m*aCi0WaK3i znz9)YGI3z(N$a9kme>bNf zF90ewo)ft@LzKtF+DD6!doe~bF9SOYm`mh$*xvP=r!Hegt6rjWg>xx+9Hn_{a1 zu^GqmY83m5>siz8F@FU=(6rp0Rvu#h`;uZ3MA6GwPNQma*S6dE__u=SXB*+7gax4n z=V{43Q~|v0TU{Z_yPCALV=mesG~eNGzV3jhBCQWs7O)4RIYOxy6@@sfU^*smdQo8a zt192=9Cpnmo65yiX@%5h~o!F9S4gkCHH$|cbG2KoB>8f0~}ZNCbq_77Jj zG=qFu0J5yWZ3*Az9=~O{3a!64ybhae@w%F1+OVl`@X+F9=o3^F`b0G>3%~U2)aC@k zBqQ<2ifvrm)?d{V78TyLGE7Gbus0B`R#db{u0!HLo1(@dD-K2}dvIf4MHw2zu*gU` z(HNhAXQyEO=aqM4;0*kJIt>V?Z+?EhPV(M+IXOJYc2R|`38RKpcKvwcN+~H;`5TMy z35acZsDmTqGew{B+zj^Y=`|Yp{=FD~TT5r>D#LXS2aNA70RpG$L91cXefIjb7PeL(hHtFruJK)ISo-wPojiT|bUDYs z3MEe-T3T9)t~$j!jgj`g@m0ILvIfsX!mfu!?|RKGO^M`)E6SA`dNgtGUPcB7dXQ%o z6{SF+wI{{^vZli=p?0HKdc|A5jkd2?{e-yK!tRFA=rB8SDa@aFd;j^LAP$H0xvc;E z^Xyka^U9pIOy8hVZQe5Va*F)SnQGUAf6!%ORQiW&UObS?0KDNXDcPUcrI%HX3We#U z0hU61P2#j>V5v87ZEK`NP^nZ_X`w%VcI8_G@yjL;k1j)@vBNM4L9*Pj6Gx5|W8fm7 z*c!8kOv=j2!YlarqoboE*B@Ghl(3E@1e3zt%z%5bbsi7B;f@{lP*7{v?_bw~5)+96 zOHD5G8yK+DNif9VOXDTT0Do2ieDxIlFV~F3m%RFt7`R1we0`KU)f!jyQ9^(d) z=nLk#e7c7PJ(u0+fE8?F)>zr})<#A+VMW|vA1-ltj!)eIe8CJ9LU={j^Me^TA3a*? zn#Yd^!MF6?^roCopFZj6>-*^^YUt=Zdt}}`yc<1yZT1NP01;7L_+e_FoLZfmn@iEv zx(|aDGd&AG1$|%0F_<cIox zZStu_m2H*|ESBo(wC&$2l{UCS1hPXxP*C-j2u0@IlcP@VyXyQ}jnABjvU#*8RXV=x zXsJTw-t8}LS(h(!;tZS@!ms4LH?=Gd_m9>SiVgycW>BF{cf*ewPl2L z<#J&MC41%L15V)qd_H*F|NbZosPy1e{&Na`eGwLzv@UOgl1ZV+*zNmlZ!8>U2_h7a zj^nj^qY(5_gM+oOA|kf&Ktpi`+(&Vl94eg@x|Ep>q7(0 z4?A{tNFV={8vy0{u*uP^v^3Tb0hKbubp@a$R~6?|U{rxGipROHK@EwV@lKHF2h$2hM?b5%*)n%> zxL$aLt1c_uQl|MGL$2*#zZM^r9mxqn5Ig3Hu!|RIQ;Zn{DC=l9YpdX?0vJzzD{W+Ztaw$=4}K6X+bqHa+traB z3yzMCmXNl&=boGcOa*5tc+06%8|c(rI_>eKWSXY5v2$|Hzj}djd(|kKxv-d+5cI+L zK}J6Ay189&#fo{TRT8Iyr!ty;&M9bfqO?(VL@@hrF|B-im_uP<C+&b8>Q;JQNibo4me0~@AA!F*o zb89D?IECZVk2_Iydl-Jdt-pSi34T_(NcBIzNonu8=QL;NpuVg&MJB4NLe`|0w%xGC zLPjgu$QpY@M}c8jWhi+}$+`XzJ2O@lQ&e8AUE)Y-^EVPN1$StfWolqz(vgJHX+6@{ zSo`o8kGGGHrk>89zegr>|GH)9qXN;bM#hI&XNA6a*)xQda}c?-4xl#$k*XAy~&3D^?kclRcT0?01gJSy&l#~fAsdIRdgR^#mnykYi6wRT}>TbO55gr z`<7!p)fb~3Vv9`>i@aJ=Pu|2^F!uKC`M9QV0l;V0>n!ltJQ%>s@=cqT;T8>jUiOLh zVQP_WR%>&i53cm|9gW4sbD@4ZN9yPEXO}@ru%HCm$xpMnd|Ubrc?UvJ>fhQ=p=fNw zbBbGXU|^sCV|LU-2=8%bk%m(pmv<&Enodo1#|{oOl9p0S@rf)ixukS=clX%YJ8H?2 za|H#eIwJR{J+S4o62K64w>F~=OI_9T7-nW-qPV)sqdGMA^(m__@b?pBP~umZDTEr7 z!YCC5mxiP&hcbyt52BbN8dqW`4z(N0!6LS4J z4|$NHqAmdMB`+WDU*31-EiEmznTI(V8um^)^Y-0Ty_86Mp!bT9juxf;TAbn%hVRYx zYS`fN_Xqc;|5YX*KQYZ%;*_qz$`)krM04t3cjVPut7K#rV+}Va8uHF07ztZ3o03z- z^+JeSXx&vNv_+-3;%{)aZn&yfo9DI>+|cWQ5`;<4e%QPkL_Y$K4yyqJfrXBkFu?k- z5CKBVP}a}k>1KIeM6pJE@IrFSp`p_5MN_1stIN%ieIgUEViCqj%XG!vp}1Mv+uJPx zvIws{=`quPf#GBDE|_sD57JhO_u3Ihqj8kn%W#(O=O#MVlNsS(rw|& z8LT^!YK0-uygi!ghXCzT&^p*$)oTQ4!Y0xfDQ?4CjvSAhHQfeq0G;qe#{+$odSaE3 zvNFGJf&-o56aMQJexMU#laih#-#1K7L-@Es-w_Aos4!z`Pa#Hdx1-;OhDt#Yv9YrY ztFIG|YuN}@Pg1oluYwbkx1d}XX+CjM(y<%wQBF=^q}>E?Yk;t4NvnmJIQk97c6L(a z3Iou(zT|L#dAJ~hn7Ft)x-7^T#(P$W>4b)cdVv!{C-%5Ux*L|y@8?Pp>L+5Rqr(6Pr{b;hszd%s+dmxd%wNt{xpaeq(F!NJzmF8Y=04PJ zO)vPL-xPK@RV{kAmW@g+J7=4*#EO64p_UY23!r9%A*nN1&mHv0u7wu#iHP;OxIt{U z+Xv|YX*$5J@C|{p@2SgTL>8%kOiYZ+oIeMP>UWkRd={b*GwoS*l=;oj} zsGd2a3`t-xSmLmYi z$Xc7YN}H%UbhUVuw6|{$)d?_<(EXfi68iBd=c)xkT6+4R(z*@6xAjd!^D1@1os$)hK}Z|xcv6|!=)E>wmnOTpZ=bXkc;*&U({1; z;-5TmRUL$$9G44zIJ+C~bz3 zrom!w(%1LT`%lmHc>d$sIlr%&Kfjvq$+{XY%ElJtn=ZTg21U3KJDGri1weeJJqgf= zG-5}O-g9Z8mMuG{eilfeCF-Pp;9dxNtRYaTed?ir8=mra#Z*Ky@#*weKE12!=0P)~ z4Z%v)FpSbY_1P>?X1X9AYY(h%;4I0A2_ilie^HGTW}t|td*b)N#b4#U8(KpIi2G~B zvq`>R+T7dAeB#6j*l^@+@jGg2YQnc2w!V=pIyh4>+k*5&i$VN_Ub}W4gJ3gyX(XcR zd*JxmBM7V^_cRCE!7|di>29Iy+l;I3c+=BfBSt2s5^P5Fn|zY<fTE2eW*t4vhR*iK4IS4{XHBatC2Ll)meWBj1{q1)^+NL9T4w(Qy zYKgpb&EDLlhRH_0D4EM8aSMWuEdvN}4q^xGJQQaIr=9}zYAyZ!lGa7(dZy`GeBz3T-EPNrIPU8LZ!fGt+<)*O$f)e{!8>1}rd_V2 z#D_Jyq0jd(zvM6qLQ*bkE-kHKm<~l(gHoJOb+Ei<#WAT&GjiVEJU`r_elmjZuBc1l z9@(*Ag{u_@(ReQ3xRDFq_qlZ|6%~0g?vSY_BBI7?7;l%oe3P}*#N=c;nobuNm)v)9 zd0{#TB8e;77*g-&epEQF^0+bed_TuOSIx}nvMJf@d)8gy`k$lr`-{CDf%IFGHjF5J z#JS6R!g`rWK%d_&kVI^>62(jvr4vo73Kr;r74_+GmQ=XolN1LAB zm=6D^LAwnNfq9M-Gqw|So+<>Y5CGSJ6%=?9Ighl^-zp=MA+Zhv9kg5v!`Op7yak&P zw!W71?^!68tq9*_ldi5MV_^H|&ztHd@)qgirXnTWMq5j(6+|t&G+Pjfu$1$hQvBgM z$v9q8T51Izong@;{=)qfnJX}7v_TQ034Lz7XvE7wvQkklWz-jFe}L-W z93ye(~uuzs$ARm38*=VC(oPmVNieh>WkLG07=`1LQ)U2$8fClj^xwilC zXyzIm8SzK4ad23NCc$D%zSl_stFfn&%iZ3b{f1HxhOPyLI%$$#ot4gg@7oK063Wr+ zm7{FU62&1a$G@%D`%hiuwr9u7M4GMtd^{GvyQg6KX_MDfVkk+cTXC5tUvGhzx%{x` z5vV&w(Wv(t0l_W8t+0trzLH`6^@>p>ks9pS(PqT5aA6Sy6U{9xLWb-9{Ecy81{M|} zAX{SbNZ0gYCd76bRXe+ZXY+Z4p(0#MD&R3RIx&+yhxR$KRa7?Pt-;yIjqk`JdBCvPw-sL>ORP$>U2aD0-qqS`@vOqq}?QJp2w`x`}(4xymQ6wYU>({T(j2I*v z>FDYC>L>ab-~H@;#z&BWmP{yU>o-*&Shoxr`BZ4RNLdVPJdQ&>pG2nayB-n2MSutJ zNo+V&LHuy7*Jd1I!J~1bOE-ZX(5yCqUb#YZDsUB4_uadfG0mTgPg7IV`5Ug}=c~-# zP_S7yfBjlsbBQwd?c29rz*hDjIy5r=X|0F|9RmH|b~^5x8l8efWT2yRx%N1ua)hX% z=wYTvPOtKtTe{Y!^UKEK0Q0jU*B5t-jukZRk!`KP_=pHPZxEaRjE_o6r1;;^_s0;j z>x_lN*pI`w-@hk}d>qUPGzS0nKx*)WYftHNh@Mp_2Q?Si;`no_wkvel!7sXw->g=;>N8= z#*acea$JT1t4{u%zt?y+F13arnfE^52ka#Pj_oNR5U{kfJLq z56&0FLm{vYi99~fz7UGAnvIPZY2}N)ipHJA0tvA-YbfaB0q3HRzpRS`;ERsfM&_E( z>FJJzXTD!=icL?itULo+9UdQXoVx<_@;f0R4ziL?B#kNChN72p2#JLDf^Uc5h5R}!pB zk-3jASieY5gfj%o;3^?l4RKdTaejU5{&O9c53zA@EVq`A8!Vq4&r30#-4NwWO-?S1 zR&paV)XX3h`n#-BZCo~suwJT$L5RfR(BdsERXdgM>8U|#?m&P~i$0s<^(bap*C zce_;U&-c^Uc1w2pBM+W7Ih zl8}W&hyq;V1<*K@Fp50@xsRrPwz~(urNH-+TMRG>?m>*h7!ZYkBw}`5%FYpV3{cr4 zw-E{S_iaD&E`Nf-%M#56fss*LwZvLF0F7e9N4pUt0|VecAb_>qy0$9VY1?ySrTRSis1r0R#x- z!2I_XDaQMNr7s^F(-aM}+#AVV{-RF1M?QdB^364pe!pVkpSSuq=2O$!GK3XZSs;Y`Z<>&Kl&0s>vXqlXl zFb|!oDmI?&*b`rA34X_11md1wA7w!x{?N9>m3v=MA-w}4lW4O{m5QFvPsDUbVoqtp zpSLGLUD$%?B|l$ADpMf5fOsdKB@eIk{I+snWQ1!~Wo;Uw+VD>NRTbJ@QCz=+ zE?!n`0T17_v^2G#kDopT0~G=c3A~NTMgW}HpdAsq2_{b*SD`%umQRrxL(A!oWEe3qv5xfKij7SrNK%!R_;eS% zdV!@BJj0}yhbndHtIYDgzCKKh!CPEh?HIu=1&}xo6g`nAy|~UhK9D_<@IW; zX>C^PZ7UwKmcU-nt3iACEe{M~^m$x9@!9iy>HKicntawBij)WbSWHF&;^}swXNTyCwd-VR8u? zkk3Y#B6 znuF6SL8qyiYB8TU|X57${(Sh;fHM)<6IXv!*73!7##U z*mu=5<;$S8FjvH$rhpw~PPh;%S<@H`(jPdP^cGxF9Z?IilD!@l5#f*56=p_en;7tG zceGd`~xT-+WiBW*bF@{focb zK;)9&7tpU?19T;vX#RBCzrWb^q4ofW0xetRqjmiJ&R<_$^?`hE@scH*Yyl4tA{Z~_ zN^&VZfAfX~O)V*g5Nfp~!F^8xvj|Pw?ht)t>z{(VF9tmip^E|kf*Os_&22cGg9Ziu ztw7-RYb>7fg5Cvg9|>B%1x>I%O(R@Eu+b;rmv;)im2rRbzA=u`7X4%21E2-4^I^vhD zA>O%fPuk|x0`c(NEOAdsJIxg)5mTnbessb z!@02b4c~OVImN4N$D)01(=jVp*&cJJB+^Rnb?^UgtJT_i;UB{0_X+UX&qXJ5>SIZ; z8hR~!VaY|Ovw}T6o{0zD3LzAcd%g;)I^H2X`EsIGE91hJEL}>4(8U@7Ao`4koLpRO z(8hsSNqH%;RPFblOZo8tJEI2T1tuBb^uqJ4itOI1VE|oiBx+;;v8(&gE5r|vCZh(y zghQYq?GzN#?Bk6^Xoo-uIc-Io287)yv%anlfrQ?`!q|2^eh9mMq7w*CNW$2-xy22I zYi=1PM=4(iS0H>_sLKKnH{5K#5Tk<^;zWH>8Uw^H2#W};Qbnji^PQcY*)Fn&MWR=> z|NLS};IG!Dk%7jt@g0iEhr)PxFn4n0|X~kCBjPb_!5y;;jaYy_^ zNZ(-u-6Sj`ybD$jDtE`C`UFq3HtCq6M-+h)0vKFb0?3DIq!(Z>UW zMnB>ud3Mq0ZYyG*jt=EI~Cf;DjEn1Dc@YwCQ~LsNO?5xo=F zLn_}ns?DLx*D5IRfC2<|dJGbaf$!fnu#YDm63yVZZ@1(8@dFYEEuxKnIWW@CrWyK3 zFNnYCAmbsx7vR(z5Ap8^*hcz%2)oYW?RmC;-?%%Yuxngi9+ATY?bwQI2b3Vet1B&7 z>2P~!q}Bjptm!;Gs0pb9h;=3m@W3Mm1`p5?P8_nLUIU9qc0wfRlT=B3$T_rAt4?n( z2p<_I;4R?YM{h+T!in5;QWVEFm|tmOPGesZwpdva92$wANeWZ7A3rW26%trt(ZP(o zA96T`)C(7UL6{mDEyml33(HSPnhoeRA>v725AtZdr~B(yfBYzzu!EnJ$5h_NIGN1H z$|`8cF6_F*x$Bjj*(V__Tgi#sv=)OE?C;m^~KH_F3o707Xii;aI_$3q5CSLfab*F`!Hi{yuPH*&O-GJ{h=|C* z$*pnHuQ0*l!O+t7|4#dgwEKu#+mVu<&W0BspWq25gV$cxW#i#t#EXxIps=Fim`=uy zX?{f)FSw<+K_6C==sY6+5l#@^CCorericp{F7SJ5i(g&{aIcS-*}GT5n!Dzfv4W(?tY$*S1+5q+)M5MzLI+B3e}J7fsA^+ffj2UV$i73#Nz6LT!NGhP0doTviG)GfGh#1 z_6+83Uj7wIGdbi+g^ZXX9HJ0?dFcX8iBjwIW_VUkx=xS#p!L6Jkk;q|7|MZ?2G>FH=IBZ2k7Top%au+jl8Bw`*3PzpZWFVWKi@5eF1Cx&8yo#k!; zWp_Y(kd6yY29VeOj8xQE{BTmRhC=%jtXCi%2c;vPc>OFdFPgA-td*7xsr@B~_xaaK zQ*C_roVZeCZR3>}GO`Iz8w|Uuvom3=k+`-*GwW{9_linNm6+536=8xn&zHRGY@dT3 zvtxPVn0|UZD_h=!!)kpDj$hyH{m&_qk=OD4nt$NGT^NNj*P*QTbwC)EN|9RomT!wp zv~~GXX}~9=js0i*gM$;p<8>1Z$-)X;$%~0{&={=`4Aq-Bw@<2}p@f9$MJeZ><7v|X z-$Dt_pux_a)!oCOr=bx%YP9zJ$A*RrO;f|!`sMB%931IbEy9M4sHmV?a2v$)adV%K znP^`rEb4F!+{Wl+zh}SEc!tXOdeB$YOV5UQRnG7dt|b$x9>LgiM2)0&RD(BbMswx@ z8jx`u(110DS@K>Gb|(rANjkjMiV9g#+1W(05e&i@aDw8QiKTO|x=H}$Mt=d~Y+!J3 z+3Q*v8uTD!goM24MiCUW;6_8p`kR7qXH9Iv`&=QlNiTq`P+oK!fI**_8sW8E*BI=4?@%3n31+i(cz9Z zg+fLRkUi1SZC~As@8KS2+$cxu_EBLeBhH z4sjK@zCDm@;WZ7}(F>cG8sWzR%&#{dFDE&a3X`cRcxXYLq9IxfH3H_$JMlLnBdMr` z=Od{$9YCdkn$L&yFYh=IAaaGH5-lPiS0~j*3}8<);rxh>P&7;jGZQoli>_CE=wAvU z?=NL2SR=c#gb3t?`;pvTmlKHnkwMSq{WWoWKF*arwlYWR-R7@{Pu~k6kAwZVkm`DumV}iaF6%g-7W@{F*G_F-QHU^T{$Pxc|o2O{17w3nZZB{JO|YIUG_&d zE?SaZ^vQFVFn2Ls4)YbWmCYwMf~~}7VN;Vuk4c-czQ#&@?z)+`)#hbpw9bO&bm>2r z|LhmsO8>i8?&Z|qsZ9~_8yp=~Kw8nO>l;~SplE{lfnIMx#!_>uAVnCa&0@#`?2!=4 z;N6^Lk$;8{lQw1zR%yQ92ZAlk1!{97EI^Y^=MMBAM1f=7aRARyb>i_P;ymazgjaaj ze#$*fL>drf^O0caR^%DAAClHsBPu!{us6t#Qv@(VqbzDIUb6!21RL8~6mjhe2!au_ zl)GI*i-6u02QzdwQ~jci@0JpD5pX8f_r@kCKR=vfgGLt+-*vVtjh0;r!r8kcO~pG)3IU^|kK3^WQZe`K$NHwMD=gtz!@Dbd8|}oe5B3P0Xk`C~KsHH^h0Tc*&UJhD>ymFQhl!z0jV*bU}m~q zr3_{0}-fT*9ZeNVJnx@*^I__6#ktwXcE9<`;)hT$P67z9{Msi?0B)_~( zVFQiV;ngdxdz)VK+d~xTwobvNluTnI&+*r^kx|(i0oaefnh0oe)x0}xv9A0CS zT}Rlg>+d(u4zHwTq0ji(lN5sj0FZ)cd~Yx3ZUQ-Am1R1JjcxMPh=^qYT(vvnAlT18 zU5k!8UN^|KAS$Otq(bQB%bzp%u+lXEd&BV}1_&q=HFXyfxF20F824aP@bUOfp>X?_ zJ2x-QikRexppHmJfU*(6Ii&tK>QIRwY?{d76U;L0eSc>7yGdL+1?L|IcEZ45qkuP* zf5itgHVpy0DcsoP?gI?o8pd#t0&J_+vJEBPyg3I`6DC59BS((pPbL_yf|QZ8;g~ZZ z#Aegk>;p$^AiDMVjEt&eegT0HWX2Qp6H2Frz{~7#BNIt@E$}gatVS$sY}`WtY&sgu z+OR6meL6X8hRZkc7&+i>ZbZy(CBtnsD+&JaK{ly-ArliO@^iC8In-lijhjX(@t2?P zS9fxfMY)Pn4(|v`Ok4@ELom?U!uonQU-sZ_81?jfNw0g6wNf?HtVeEp18v4mRvfUP zbu-48wYBS>b44%Y-?C&!l4^sRZT|Wn`fB$5(h`WSRQ=DP zq{uKFyz3SBO8nCPopWg#@)`!}L?te2zP`uFGd%fIbFsPXdA+r1tA(ES z6E;^t?u)17v|t>Zb$l9!CW7XvVwMb%&ZTWZp_h zutJ`#yMPoe6zIg^vX(epWGL@r^sf`W(%W~(xX39nT-&&ff2C~M&w@Q=ZC2K=%_CB3ueI|M^X}u+_HNbp{uA23=@sqNf@4JKdbM z73^r#0e4q{c@aVS8(37F#grBYKTM1n*D2@byxOE#)Vutg{)WpT?BbFewd+@hatywG zdpNuH=9Q2)o9C>=ybKW#4D39?rchJ^b@0Q>pdlPajSNQFQ}&R*XS`bfhWuwEaD{Yu z#UNo7|JR{Gd2dFSTZoc?RSBQ@8SOY>x`3P!_%_AVw2mqn6^-T=95-~#O2U=f08hIKtQzK})^iR9s3>4b^aB&QgS348{R1OkoKnm;$u6~qR^^=e z>*UimGs0~+DWsr#B+WAFw$hlQ>kkG@(vk9~#L?E$Y}>XCCnx9xZZt5aUf?$lA6`Xw zb9$a4ESo5Mp_eX|)EGPu7|b{kN&j#%Mw20Eh(nq!=u|JKg7;4AMAkQW?s2DL+~`X} zD}qgK^WhcXl3vD!m_iQtwlK};4}s@dgq~UchW;(w&%Zz7{Xt7WF@+c1mq^@h-6}ck z2F`&j(jtBV7=#!WELfI)6lDwn9fdfFyc;q;UI0pG4aMW~EBh4DBbC5WQVC`}sSywb znz1L;Wb+8Ecr`qI1f3zE!uiN&r;?16RNQ5+>J1DJa|$t6K4WBIDTm%}yNL-ERMyR> zPd^>b5o91REJj!2vcWTOveGTi3Ja59%%K7s0`*&Tkf+xACGpe2E<}BL=JrkO9Pq|A zOm+f)oC3B3#gyvJ?U1)=DZp!h*8-(Aqg>X)!UDKVbOP# z{V7A>bf6valUob@210@MA2`qgxe{SXGERVYA>;JLi$!e`!l#EvYMSp(1pVVm0r9d0x!0cxnX%-_9_i!}ic9CSmNuAWT@HbZ$b-MT*=zI`2 zW)L1fS?0v2zHbtey0$_WPu?rD5~~^QYrvgtH{{P@Fq9Bg7kPlix6G&lc>Epic!G|E z1`XmT)>X^?d_$ETt^NHW&^H6NZ-Z_Y4WxcOIL`q>Vq$nAc=Mcd>e9lGA^roHe*wN4 zIa>r(;FjTb_wdjbDRu$QPMQk%=;UpoCW_p6f}d+Xk1xInw5|$-iO-diC~IuY)2@L0 zxxU`aSBBCW3h@G50;o4&zypD3qwg$E4k@)R1a=p8`SLj|gB0AbVF_tQN+|n6z+U1a z!O20o0Eq7)!*MJLbBB?7;M+Iu+rBbUC{<~|Z7@yCRTmc*rys5oef0P-x;3pF*ei)9 zh+J0>5d%gfMcXkP5%?}YPHkCcL1%cFGe)% zZGt}zl?t#dw2l)30`nl3oc)8Y~Y%O@m z>wZ4+<^Z(_2JFjP)Q_E&^soSQ{(U#t( ztS@fo?JE)tZ(hxly>eo85WhR`e7qfl7##>l9(--qzB7nqfs+Q@!eJzMIn6YMsH);c zp=%USzJNWgRsgI29;UdJKtI8i#md7Ih&NwYL7{$T*ab*}Hoza#swJWsmR}^XVwk^u z8>iY&uEIlzs?&@F34yR~0ZG)V^W-Shq1zQ(y&5vROqZMHm$c&TZL>2oV_}Vj-c@*o z10M_)K)w!Ud^DJ~gtLbji4*Yxl=w&__an+5+bsDZA|^yCgi8R^U9qG4d8;eZMk5VO(PXP4tJ*#m2lDm$(xjKbJH$`}W1}P=Z;DM5W*el6rhE+$MCn;B1Y*Y1}!w?q#gov)fN*XeR$y?hAHgbhvWN=9H}tR znj2}Irl)N@Kpk!!IbPcOS+qEgfh7t%5qC4NRqB{6ZkZ!_POoR_f6fQR<6RJEI{xXW3k9l6;^I99%6P)IGZp&sues)}RnJO?|~DysR!D(?604c8gT$yhIQX_cu+Xe* z_Ok|`s5%V3?=%MA?R)s)s^zm}O|!gZd-yn$JT47{{80b#p7mLypXa+3cT4^5_y>)C zQK!FOH_y?@^rqo+wxvnxdKXv>clfmQDvi$DeeIgO{b$`x;;ZW`Yik3%XqG%Yy4lxs zvbAZao7=|wdsBel@pN@pgzBzfVAyW;ZNH~p3_tCNGJl9gHQ)egL@>k0U2kwVHMJbo zmdq5)^+W6i(|FC{C}s#OZT+EQn_W*JvZD=;*nsi>%WvNiXMC4D`@-<;5*rRqeAO@z zA83Nb4+3S4j1&WAhWWu#_BvnhAx}~65V+w8-hsv(C^{p;4*Dt)w#Eq^t%jZ+ z6WMa{RXZ+XyM+Y{rU-0D;ZJpQBPb>&O3HNFo5DS-DIyT zFX*aSohPK8K#3pasQgsM8igfT${dKQpgj!>q&qfq)#jdFkFLP4XPC8;pm@NCh~VSp zOP|K${Tb6B^DwvhfpAn4=c{#oXw$AZJ=Fk!6xRCp*KbSk1Ydb*^}{{mI@LUgrn#d{ z_GsY9xfqAhe8#PLTZVVFs-5IIZ)P!-C&N~0ceNu$^}%%W*TnyVW!$SEPVl z`XY$iZMeNnWioNK6nDMrQO1RaEnih)&gV~c?+$%_iyAzo^x}o-92%tfWehe=)i)@Y zA3xGvrZQ1{%zf(gB3oO%yy1Yr@5K4~Nr{roKho;hD^rt_V%=hEEomn=xQ z;hOnjI9DLo=e@~9MzW5Zy)dRHBztPVyl6#m$9j&p-PM% zM3o!fAu9TvS=&y$0b06t^s}U7>*yf6qqD;FYkrj}r0y##l|6P!p`D)0Zz?#)INxMV z-ZqimK7tyUPISiT7hhaaIX5>;5?{`0;(bIilzJj$MLFG%o2JQorkTy=nbLavZ& z@4~AD&iMDoSoMhbe`!0I6HAVBLCAKaD8k}PUfQ-jH z@=4+RHSQfTll)4yMP)%N2P!s(q&fFHh-r0-2xl)+DRG{5;czsfn`X zl)|T}s+EB&mNeZjx^HnXwtj!_DS@1w1zCANlO&Ine(-eVcxM%9^71%W+H|Ufn&-nG z=R8ci_2m8^Ti*dr_5Z$qG?XG~k&H;Ggp^WdMOGP^rBERvD>Lh8Q3<6TISL^m+4GPx zQXwllNyyGPIGppp-+kuq^Z&ZKKHt(e*KxdG&+B>a=N`X0Ql4&3TOv3cdf}EP-MlWT zz52!zvez<>fCfc+M#JNgHl@}}jB{IviCEWbGiyXaTERLwKfg5r5xESx(DUdj%;&lf zemC8o|4K(qU$k&KZ(Y1h`spU}VZqF8(G!JY*_LlFGJSBJ-&dp?D%!jaQBn-b315-= z&3GrPydp{3IK!f>s_NOKbLCRH0r##>P9?d^9Ly2cur=dK*RydN=qWTO=Rko=1;f{y z28>uX9;>%`X`}r9xRm5Tqt)xLiPZcnfyR~QUlx&awTwP?TcmkfW(MuDmS%frv6}lm3dA{HTRqkD>E5Z2kB8w*RZq4iwYBX=2(L9R zqMW(O!GUx%*(=hsJS-~0dRc;*ufgL1>U$Y3-mB2whRmQtz};{$Dk|z!j+rno4~7eH zK1*=4u~7}lMeC1E5uqA^)6I>C|5A;N->&Q?d@F!#gZbyo!~Th45BH;QPz97@@d4Gy zavq*N+xDVY7=amp(1r~qN5sU%-+>ka+n$(HIiE&1qr<>meR;k7^v&ASso*1qE+sDm zQAY$cM1K3|n#~|EiM;(Aixc*0HAJ%)!0BSJPPR<{Dk7mb^|p4{o0=_r=pUwZ#l7X{gbv2f)B{| zpk-86F8y;m;+N2P4&&U^NC{`E$FNx(%^Z$A#IBzB5nvOy-)h6NV!RqCI4Z~7NCGIPvHvYns~CCu;Zy;nJ;v2 zTq%%+LqSqz?V5w0U#~nh9oVd*aunTv$`RR=|6&IeGhmH zpA2QO=D+ssgYAh~;YIG@{-h+CFmBair8i4=2XVyGY@?KnS&JE#HOYtMT)5~4gQG?m3w*UY+} zHfE|z9**R`UbI=Te6^nR-7F!QXm+40q@u38U0+7W(x#@;vV<^Kg!Pn=T~-G~$dddw z?OlH7hw2;Aqrnbos|`8Vbw1bI_hZvlatt@AsM$){?ny-|6+m6gj-LKs;~r+MwlO|5 zPMO;tofY%^Y7&aCJ>h2=UcVaO-rd#&)#8Km8NO{YCz_rFUf=mEJ_eEmeqMv+rlii$sLZoH|NW?Y_3u4Rm%c3tvh_zvYWU@*hc+7+NxP0=Bjfhh-CZA_f6vBo`-b65 zEZrA$HtozhHn?ZDBYFQFz6HbU-6&lGGvSHZ9{FDq)+jc0+%_HBbWqJgbgSU*XCi6u z`#d_r9nuz3e9tdgJ+PkKoZ9iotojb{{yomZ@T`D@58qN%bW+mCvw0}t zz+9XFhUWvgfxU#R-yfQP`=BE;u#9q^sR<_~NcV$lp4fcdnuzp6HtlcBU$LZJ~W^vv2c zX~)y-dSQZrYjNYac(#evoBmE^xkU>&PQ0eB;0(_@QYSmxL52R}_EAR}5xN*bP5Hkc z(kCMnF!%l64;7CI+r5k`W=@w);MEIXduwRhEmR-sQ<=Za&scO6&I4C%i_f%`K+I=` zXy41E!J+_;vQs;bv5?Hn9Hs`+HXhA#I3HTbtYVJwfl{&d$Wbt@fi`S1b> zmcFd5sbe}-#b;9IlZbzVqKSod*czME8OI+7x(DSo-sgYRm}*n&h>p#jPjVU!;`?AG z({y$~sDcs5M=F{Xj6FG8rT3(iZ11vS>#zP)x^Yj9-k9a>xZ{C>!WT}hI&B?5lZ}k6 zz2QrCX5;^8N+LZval?vwxa?)dsKRSWMw3AVb3pMEdGfoWSe|)MklgsusnNTBE;7b$ zQ8_Q>JJ{}Rm}~1-_fe+5DRhw$nMcxm{bHm^G>(5lN6~woU#HoeS)-F(f>h^Z<5gYE zi6tFXYOULPItE*VuX}nX@#T>(w2#WT4tX?Pqpa(zlf+BH{lG$cytQQ{R4imqW)eN` z!oyoXW|{Q-8qW@vefAnV3mJlqr?LX7~u>bb7l4fyEN1Ka$scWWNS9811wmHtT zl6+$`jCrkcW&<|fixN|3sGr`&?Byz$M0q5pHJbJsS}s4SkWN1yU+(CXUfU5`@|t;T znTKSSz=bi1yPZYm<`?lkhWwD=W3<>!jSjbajY^*K7wAY(ZJIMt^RB6Se!5s~%e4vT zl0~Ts+>NOwgL`~uq>@b&`KqV#+`H$KetcjnW6X&~)$HluCQlgee9}ywe(>P-?Nd(a z;DSIJ*`g4c=67=Uai?BgA^bZY-AH3X7J=1nYtGpCUJ%*W5Tt2nd;{lZuE#q9%Y>D* zBKFSUKYxw)aC0N*-@j#l2M+unn-*C@E{xL+&I(?XH0sA6?EYL=HgI9_2rmTu9ymU; zddWfp%;8h;s{4&afyM?B>bOur^5`~z@o;8xO-HXpSSA1~f{Y;Y+Txz7K!UgU;stai zDw^MA3GO$cD!6;aX8aDAbXr;mVI=e?`TJR3%Z1T>cPg&ng$oe;KJfmwL+l{&Ud!S! zXjPDJ5$;mDy479eXEGpM|3R(mcj@$kZxh%s(8KMiw}w3yS}~ytoeji53&5KwKQFwY z-^TDZGBL4Qlqc903X>wxB5J!*4}$1{5E=DD-lI9ODk|f5!IlGh{0HiPQ+31t@$bwq zDSKC9;khOQ18y1IxtzY2bH!O$<9-8!^Kymd8>~L3{DN$F(latT{}L87H_r^f05}N3 z0z#Gzoy(OKpLM+$3R!vye3@nL!o1l~3TK;@RM|nieIi9lxhrAc&8*|hCuhs?v)LB( z>ut!Qk2ZALH2bw{s$mZf+~__MDWCk$^!-PLlQ{T4v;xln1to$Rqa?OUFG4(Pt`tGQIGv}=Y}ii*WAFM^`x;(kyAA~dfsgUXyA4>zU`VZUP$u)>)V;Doc8W{{*UawjW$%pP2?&FOD%69 zr|~d7#-xtsQ4VjKm}6GFp$vG(H8LJ}bU(|CX{s@!d88<3?a6H{+>x^?h`f6>P>~ZY z515wBCte>+DI|}Si&_Giqk=y+_&I#{p9wL#Wa@7qQ9MaLw!mIlK_Ir@koLT5e?yoCCW=+dLo}1g`EpC8wWRfym0P0dOKR7|8j*VP13r+ zDL-c6k)t$^qDXU;du|*9@8eejEoh`ax_6~~D zg&@L%RiZXH)RqTS7Z}O9+w3air{PeCL8|oP{6)G3Gsfms zO9;X!AxgnU5Ze|0#h1pW`Ov8B@f;yq5opjam8&-@(r)w~1KsV}vw0vpfhMb+Vqx|c z>wfUEh)x3_bj;_-9v4MPuw%Rm+7s3Y2!y~jh(lKHI@nX42m#qFA>sZ4>%?CNp`EO=+4!1Mz^Ot2PtgE3$F|F&oa5#iq!t$Ux9 zEkHFTd@%@LJ0R+?dWGj6P-mcK4}-R>#W}F!niX_) z{)+?E9;YH~m-WT@QHs4;&w__A5(H9x7@t zSQs)%o(-*)@^Db0(h&f>RESBtt^&~6`In_esAAjh(dT>h=vz~#to0G=;5PkMAY(OI zalT$tkj`37Pt&CyI#;9ZKP0X5Dd*Om-aI-sMr?uTkW7uPN8?7G2hS7v0#I zmmwauoZTHIHU-05m+ig4T}wep8xjNl{6RzGseA?SiJtQ+sAZnNzIt3UrDHgFKhAZFV~~U1grn zbhq9VowM!DedYb&ufy+Kck?Rk30!fshua@rdkZ$$Xz?P!^_w=8CbZYoTtpu4)6~@8 z?wf!0v&n?dh7!H0im-Dr;|~1r=~JmAnSLDPjqyCs-hIT?aqRWmga-U`*iCX&Of4w4 z+<{P+J@vpQIKw|~g06J+TI6MB*VwGuave0V2qiipG2JM1UIEuDO3PkGZzy9i*q=ig zp)kEAFj#EMuTidtRRZQHfeqzoc*iU&d8$Ovi9Tr$yG=hc?vd|qpRl+h^cb^El)1Hp zXAR*Sqj#36UsT`F@C|>dy$k+a2;ICLSjP? zNrMET!}@*P#K8h1oPFU6LqM>qDUc}=Ya;qEwJeU}cY{0?NXAp?4ceF?giDyICY2VG zAZG+DW&!%+)MzOACUq)^iaK~)3b9m#^PGdj9K}Pi4is`iA^YKpTvk~oHq2ik*64PY zer5~`?{$oRUFOUQb3*jS(^QyuH7VP zjrqjvFIQ}Br)6Ka@!5WiBH#Wcu(x%E*9Bgkg=S0jgRtr?eRScJt&RNZIS-M)E9)P> z;kEi7fX=hg!MX&!xJfm^KGyUO{P)(QV{O)yl0d;{dGv|}d3p&8n4?PCg4JJc-!?m% zuHIknK6!4f(aVRub#W~c?JD}Y2Wxb*X}&*o0I$UCU>nF3-Fw->XZa(ZFFlGouC98T z>W7`~LQLDA0rBPC?PxD_r9qT_iT;~u9MNf(Ool-rLkrCl+OT4FzOhNMiVZ8sUtqsM9T zDdl9{-UjE?d*o5WL8^l7FzNf5m!Z_H+Bg`@}4ccoxhf>AL1ExwJn2vt{KH zT3qAtBh{qvcgr3fkhKggVPV+{Od++y51q#bwMdC&X-=nBeg1qPP^3hc@BxIHk?;ys zO9B>v*|Ze&{slP^aiJ@&YvG(C6yr;tha?E~(d`Wl0XUVXu*@WyO3C$q7u1@~J#hab zWM*--oq$gXJL_hL9ghH+cMz^h@cX|^r833Lt#SW=b^F^3YY>b-YymFAZci*!XP>R@ zR)^N?P8%d9!7g?2AL0}REG)p$cO%+t^J$ls(S1qoX5$}z_tWKW3t3rDL)njRw*>4d z2+3-qbs+md`%k!?5;jw439hLo{@Yk`*u|Jkkj;S*Kso`z2zA`e1a|t0K0GipNML-D z8!oC!86K;(aZPO*Bs8U1`bH3zVBkxr+Vu9ya|XYy{olIqPa@AowU#fy7KHFL{Nqx@ zE`+!4C{!`OwUQ@KevOw85r%ENO^q-_Szn><({3S(6NUg_9KZHn>fMMn$$9_;&>kb6 z+;=kg=1syL7-ZtC#OjQqX*_{GCFm>TxFJ@`kA5QOm+Qql zeYMKVSA3KDdIIoNCr%`(DCxMNLT1ard;#0?vtje;gBy&?qkf%NR&!~j4zHAsN>!Lo z(T+Lo&@XVJNb&GveyJ$Gizbd!YF(xII(-GaWmTX=TszD+^_KGZ`@UA$ z+2y2@9|recAUi&LzJSwnw&w7r2g1TnnN%Ut$TGd?Bu2%h&!_Z#7>i3fe; zMDD)2dztKej6tmMFQqN>b6`m~8(O4)h}-tiWB$RIh0Vs1%L}%}t+{0Cl-|{IaG9rh z@K9fevGsA8pPI*)^L5GfZsTIpVJh-gwv5!9o4x_7o~7#A(_`PAOK<9ych7Zj(}o+L zWlZ!9lP7*=3*|66;mNX!aHj<^=m?5-DX4$1&Nb~Ay~K6dVqLezeDyWLM10}@Ie>>dauPfkwL|4{Or_!6$V5fn(seF%rpKUNWefrO(rv5NTRl`$3$ zx5(?cvOO*}1XJ+g!zs@?Q-hj8 zaQIVvLNClOfIObrp%_p1)2PWgp9lXYSxnKYn z3>G9>#{DWfSjwV(B#w`2bZS*0SWqB*fPl1k>C$yiuZSQel{LT-vPqJH$%43`<`1vy2`nWu+Zyj> zj(;&T{L(wM*YncwA+1d2!0@Yrrz>|S^6$tQzdCnMV2S5UXfWGjLq&&MTg-fpQ@Wq~ z{TMEmcO`RT5t!&F(OBs@DQjX&r;uvx{F!B%8Dp02Z3V5I8KL1q@?*DF-R9X7l}i6L z;45&5N-%}Um>CWZ!J~E z=c6$q#QGr5h^ zUh%oza<&-1eqV2Qmx30lJqVih;!7iXJ<`i-^uegZV~^~O9mGa``j-^}iccV0_zn6Z zLac>350)E`m@Xo+uu(<;5ndHR33sOlRUMhT2;VpG`CMh&Zi%0a1i}r2oj4Qd=P|tP z5b3Hdhx|uWC-N5FQX-cT4XgCdaA%%;1Leb760{nC~<_9I82?<%I%MF#WuU@ zbZQh#8L+>@FTDQVp&`FMXHeyVDnl(x{Hn2u(#v6ZD1*-k%_$*eIhv$HwE#T`bi!qX zLc6_vK64qV#uHoEOPIo1F`9zJ0MDDni}!(ui0QU-Nb%M1_uKuiO_Jc?=Kg}Cp6)H| zwRck@f0<^B%IxHBSjEmG7JDO|mge#FsZX*Tf;v}MiACo-MP(aektZEcu(Eq$70aiL z|E}bvc`N9{|NW$jw9n&kiiGU99+?p~xmZ0mAD_GNDKgYoI?5%r_HrISPW61s-*0u! z_UFKnl`fa$uIf#GrkA&w`B8dAvjmq2Fy>Q36?`a@yRs%D)SK%*cTeSrB=VAX+4n@b zJZj19Y%1%Zd{&m~O;rmop2ed$6og0_*??oep zM&Mgom7>dIPX;>jgLpt@XJ=IotEnjcnZx5g)OsDO3N-?G0K2pl>l?3a*EIf`lT=g` zE`YO()(8iLg{aRM8sY}Az`)eP@CPbK#Yet@a7OBltfRubMm)98XIV7+HRn=fxSDL8WI6(UmseV z2iyz>Q^oh};em^5&$OS4^beSxK`TMbNAR0|{d$FaUICMz5^b=RI2V6pG;ncAoiSI2 zYWfsFQ^HkVR*n+cZw3P>$-=35MDus@WE33R-ogkK%YA~B{pQUZMXDH)-2}`Qls~8u z&Zu|v)buu>g^T_Hlj{R3es_cs+w9cBySvTjui)ai3bKZ1Wpk?6C2sSjH$+NXI9v`G zWSfnBn^uMh*-rb~lA3mh|6e@nwb5eixZXcM;Xi)&eMfF?q{QxL2Jg*mI~X%EG9DaT zYsU_6T9_+w1Icg|s&{FooC8w~{)3QSry% zgZF!Gf6?U)raif^d8gZ?&j%iDQ~pm(LCuVB_sRy&3FdaF(U#9DZu+#iq>7gO_EAUs zL5L9tcK0kgb*JLSPHRo)ABVRvzDY`SDKUGFO{FOv608>SKR;c>P;HK+uNxhA4Q7ic zY=4ll2>|HBk$%)nM7WJ~1_~b72uC4!vq0-WviX<@ja z^8L2*NB41J!N5lFAyKHyYa}Ekxut^bQ2G^Y;TX2{o+kMD7QhkT06e@3G45BmU=R;j zO&p#h0`?;eKY*YKZQiVYB}Uo|stANl;DN@JUTnEzA}NNP1~YMN5{uiVE6N<0EmxIep^bdsu)4|0Ktne%_bE4?I`d_ zzdi{tk&wel3EkfF;ml54lf*M!T3QMtf2>8YNeO;QF`5;_AIfGL1qs}%bIy5C^1q+LE*2!DjEjI*9-j`T+ z_vb~e$yWPJSwDCEEner0vnwf05`jHptyI0e+*0c_P{mzbJQSaNDluk@?fJUKb%bqg zQ^obWwmr;HiP}V++)1=Iw>7_V5AE$ujjWP6y8npcmvJTTsHAq#n8bA5vv`B2;cs}> z`DbHub1VpF!$gJ>48{m>6e>mS7PzB1!5;y6 zz-+|u92#16C|Jq*6CWX3hp$LTGiHsE3Q@|<#D%v44`FsDp65Z_3D9@pO+5gw+KoGQ zd`3SysM{e6si0{KxM(6df5=ylL&qh*GjHnEjYIjj>JU73Sm5Ra{G9%KIBIb z?*EQ*K+>JO!Xk3}lA19*e#BND!lbyWALc5X?htwjn8jQ}tn~@S{MyVXu;(kYYlIyd zGEg}BCq7r_qW(jg`|U4-(ck)CJkZdtKpVQ?PDPzX3mA;sVY;I(B476C+6fBWw!tQX zTnURPLV?oUEE}H+bO}|OpIb?Z>dZX4mXGfyF)`uhQzcYa1klj^z?^^rAU(c>8m+30 zeHJdB+u`B;t`R9RfCdN)3!reAq#$<-R;^NWpQjUv?UQAOTZ>cjGi5?qK=OB4&3|pu zsOV5}Jtemp3Q*Xl2X)G4A~RbF?|f)9#r2k7383 z$jHQo&FbA#O*ZG&jrfU+n}VP*l%{hsYDrR(&df`5rM1jS$Dl>ruel?C_Ou14VHD6#ab`hM6E`;~Mp!G!P z)u@I0i;(6H2YD$RC!$Yra8Rfc+Q54aIUSgBiTDwOnRGDl<)x&gh@Afqk6qDUjo9n5^dLhQgrWD_{)>gu~g6Kf@?D52^)rL0fobZQUJXi;#!`)PVQzoN8 zTUa>Nt!Y8Tj&zHU3mnx&B90tj*L<|fZj)zMW#Gr&?_=}13g&q{T97E}sk~*~m%SJN z+ipZZ`5#(6=|ZhXE3VYW9bzFJJopjMc)$Z==F;n}d5uhAt~;Xi=$II^zGm$WI>Qf2 zrej#+a^u+;x6S7NJg#5&eQNLK_i<&DQ8t3Btc_SmCbn_`o7M+k1Ni}9{sPtMsLYtP zq-wpLvgbKe%2mwOMT)2?Vu3eiy?WOiq<(%_T%*J6d1<=aRodfCiHGHy=C$#H0s*^t z9cRbm5~lie^hYI`=YdzhyP!r<@LuxDXZo#BPT}|O`{LYVIM4b$K|faj?&yHC%`lKY8O7onB{1P+s-xQqXsdXBpg zn4&@h1-XU|!ZV~koy~)W4_bGkhXxS{n(lM^x*%p=jcxw{Q`0BW?wxNI5~e6mb=mvu zc^nDF8Im4hT&yMzb7MlPLe3)6Spp!uf}fxqVb+0ihP=@^cmXyyM@Cu@K46EHr6&ni zL!|s#L@W}rQo>^uF4T2#fOWYg&EDdn-wya6Mt=vkc7Honrv$|DPgv*W%X46u1_lDX zC2Ww)%;w8gImZLWB&e|3yWsSIrh8-}bSVOI-{Ru&s)KP25naT!q6P)@fBAB}>IqR% zi9QNOD(d(j%mPG1!W9b1ps*W<8j{eUz;Rqv^+1BbGUacnFekQvRWe}`g)A>VHyrB{ zgcMos5K>SueS1YkIq?srKcT6i?J*;ish>aJf-<#i_0OmOfTGVvw`8S4bh?I6n(p1= zkg*^QGkVtiFWTFdd*mcPD)mLLr z?ZmQx?d;Qd@w2QX7%|Y=F049oL5yoh4QKs9Es}r!Qr7$l(OQUXSryJ4HrXr^78hy{ zI(D`A4S`GP%5ho8X>nv(4s)NUWpi|$H5WfrIrhdjevOG)P~2#YbR!9pPGleg0qca; zkwq|Jor1gc6oWp@Pe`4SRdy2a{)wU>n^kbBC3Z%{V;{G03O^G9G=RhIvK?zh-~D4b z7molx-Cec+hd>d;>8=J2n66^ISq{hGC}%XL~OLF@bWB;K0BR zfuqu%N9V6+J`c-e!iq*ig9~!28XM`~>vInz$X`2q;<9G9&(NDRO6~HRk=r27>F5wP z5&uUWiBQ$Q)R7QlU~?=S8(Tx<;=+raFsz2hFeFSK6x2q-)>%|ED3g?fZk)(=tFP}i zFo#JA6!9t*Dw0S6^M=p1W#|lCtBIIe;$eribO3yLCTE)EVOWk*d?`vJA- z+DijC&@gG3XX{v5p@}UY^cbj5>>4p9yDh{+{_&&fmzfrIVU=(eVI z1(B{0H>|BZ={nC#+R}!vpKu_X_SR|pDAz_;#WZmdB1T~D3s|xlhAj7IrMR7)76O|j*RQ7@b6eLuDI9LCtqo(BV|76=(bFuJjDcqOBr@uqfOR(Nr>*)VQFQ! zL0sImJlR47CNyByufoIjJKPk&-78pca^2yo8N&lW$OW*legGA`Yn>p}&cwkWEIl;M zXCFWMyT7^S6{aqN7J=|?CpfaL1>TLnW&KFs!PIQGG#JB!1IO%F;dl=EjiG5Rj;dL4gx zqWmstZw^OG}0xG@~$O)20ddPlkE zXqy$4lRLlg>3s8(11Zl3U1tZ@{kTZolj>eX&)ur;!mZ7mlxUx;JSI7y5A|_BeJS7I zNCo3sZ?~ep*XHfkzVkkj-33Wb7mOvqVm)=hV`z>C-^Z(X`ViDr8SI+ z(fX04v=ve{)$5XIrqJxF=y2@sD-fkJH!$aT{W4wlLTxxVFn#yq=|Ghw>Q)=%j?q5s z8qJZtdiv8nn3yrAuA^gb$Wk#$zDGL zQ)?paFrUewpICqC?;oSha>&~V=_@fj!`%}4744SeaMZ&zGRJJ9kM%zPq)^42vw1Ck z;qCm=()-09PVQ{pw_>pnJDZ03o+~0(_JzI2#H!t)Z%A{Wcv09^-|s;#@N)B*7(d=t zZ)j>AuX&y)?AAWkd5;#e=`VhJX54=*`_bLIGe_miFUR;Y)%m+OpDCTZKS;9^7xyfr zHysyjYPg?HYZ$XZR*Ki`XxZv@-d$r3NWDi2OSL01drl_B#UY@PuX?49yi`=^DUpXe ztwRvZz3LpIMUMf`(NtI8E?Kuc?9rpOpN9qp4B%A_*hfoq6(izPwF}iOr7=dNs2na- zan_L+RN)L*{lMHW0r#(>FTP9+(2B;#yX@`@_>La|p84q3sxwmT$}j|#b@_1<#*6h$ zO&40+pl6iEtp+PZOc!G3K4nnkJhaO~_?ITYqp}asg5{f9A#D5%1pTH?QHV=$0X{WbPMzPRK2=&@8vzjV6hqDhd4Y_qtd81D)ldQP ztQP+KcV)EUjJE8(iS?)iMq6ZMO%P(kdCchO(akB1iTFC@4xE(X&m^TudX*na^Hy>3 z!7;O+Wx%HHQ79>(v;=;%R=h$*mARGq?sQH3k?6yk^KWi@7u>ehv0#q&mZj)$cT&Mt zhXEr8y}M0ERJR4adv?yEYP^K$|q{2sO{xlRHZ1%!t7@Ymu8-2GBktNu1tFSwuSS@xH+?b zMQrTBLt&4@^Cqg59qlUbub7^@csv<2`*21hQ#N^vz8Po5s5h5U(2g@Am8a)U>4xbr zji2T=dpU%k6-hfPE<5A!Xp&K)vCv^eE?&Z}I-J@nW=`Y0T$(%+lppTkSdq9_^U1^Y z8oy|)Uo4s_7e3dvf`WePSY~nBNz*404#x)Z0;h&Sn`q z=7w}|>-MpPX3o_&%A8O<<@fkUQY6ngGQ&f3oI&36Y_jyYPEkY4U~SXod(E@V_Bb5| z5GK8ef76@!|_p3Kc4ryx=p9M*(ET>b!d}z_vu`#SH}X~pIGZkho^=y@Jo#Z zQtM$auUs>)zKwhNLT-6EJEtl#rYzP5oYLFz=Z}$R22NSyV-)6zQ2Afi&h5q)G&VMN zhE8RAuU@mJecbaL^t8adR_lpv+g1lj?jn&f^&RkyKX~w}*}C&+V_n{`?`LCU1N^Wo z!O*CkF-ueb;VP@#Y&M)!6*`~is72|`8bBfwpASi%9q+A?8E}CfZ*CZh;|#|RMK=!* z&hrP@GXcl}eb6D(v&e}E2s1?+`g33)|13GLEg4f`|5a|YYI8=n`Om&MN(0c3k~7a{ zU{v4>r(mI7;UAy(M0#tZXZvyGjp<(G%V6)*)RRWA#K$4X3%|Rh^X)LLAJ$hj=z_n~ z0?h=&Z{3QbFvjB^1B=t>BYpLTz$N#5_l;!&^6@a~{7IdCzPq4OLYG1bjyJ7fgK%nn|S2h|PiLRgEv50Ko5ESsCZZhT_m;+t!E!+=Mf}CB+J5 z!V50w(|lK7)z#KUgUHt}L!AybgW)*b`X(+&PIzNbHOY2*>Nj0!ZVPDWv8Ym}Od<=Sa1b#j^~To}DP#{I>RkfZemq{HFEi zCd>sJ-rwtzwonaI4ZfkL6Kbx7BPV88S8>Np*5$dG%%kIn!&aS2+br=ewt_RHKBvSm zEM_Og+-S5=jQ8G}=*2yp4}RQy68X}anG!hDrMBz4_*~y%--(vmnV(s4hUv7|=kJHs z#*g#3d5r&jUm<&s|K(jKCu^U{D~B7KB-E*Wlfn_p>}Q8Vf69<&6PlB~yZa~Ys+gOr z8D!aL4bGc}&39FDz6o3L1mxQ**p8&XnDL2zQMG&XSL!=Sp7|1}1qmxv`=*(%jyf{Q zoaG(sbnQ&z{Ak?tyAiA7u8-B;lb6swQnPH@Yxd{4A)~hLHKq#QrW3{{i?-E1Azs7Q zpnf->#)BGq_-E}?lRYhE|m|Fjv>`#7z7k8B)0St_wH=3s|wc-3sn zRoi&m+Ix30!eTCQe(L`o8T2@Nz3HQzEjQM#-k8|-B6(6Eqt5-s51;6*K|kxe=KRPC z%%%!3nb=FJyH{r9`aCn@(#~t$sW0vB^}5xVieB0_ngyKxO$vQ-;8GrRGAS)hA3}^m1dY6nYTfq`Xh$7H?CH>lP5Ts{ zoI{Ws2r6FRu8UfxZa*pT=X2M(6lWP}Nt;DQ)qqGh0OdJwA7TkR+>peD8|n3POR#`K zZq;sJ{4d~X`Bmyf$@X*^n8h4N=CMV|@T7j(+5=Qp+gZ6ZI=3>?m!%N1o)VH+!zCI@3+mC9FSbBg=#_Dr$ zN-{ ziS=@0|Glyo|;RRwyzf)9tvDr zCXyOPcuzhrC|);9KKEK*N<({PE&EHSxwel-AIB;&CM>-AQzjD!XK2*W{?$pjW;I@F z{l{Pa(7Hk!U=9nt6WQxF@=gP8FxSsk2{p1!%BFIBli>(#?DTVpz29@{cAUTfX$ zG|sftch57-^GKlY@|U$-4I5 z0z2k>M+*`ustsMK1q zzYZwjtWz-it5Vi?CY^{QxKQx4Ck;*SM87%Hgi~Ss7UpHX9erngeDdMh7=`?{O-IoZ z>!TMy7r{Cx#jb{3@(~!}zH0mY`5D|e#K8D_YoLrYSz!6^FActU(L(9ruRwQhC+(Ln zNIEHi4i}E3L`?u!Es2<1CdS9FjQ0%;)Pc4A0tSWJ_&SZSq1+QFtAg3~MNEtePUtUB zd5`o@fL)P)oQQ4n=1rSsMcL*Mtwhh4sCp=+j03%M3)fI7-lTDGRj-_&3TZ&*VUlxx zBV8c=`Eyn9wv7)RIz+J|-byTR-VDuHkxKBXAbEjMV6XN5$Dm~bOd36QLq&s)1ZWIxD$h~}n(fZ}ppgI}o>PF$^ zcf3ydct&4aI|QSLwEMu(c8_RtpvD2|cfwRlJ>*$-n)Z+Py`7Lqvmh0j?!8cOXy=`0 z(xva}B+lG$K05K{il==+^-FEaDR(~7skc^Hy;s>rZ#X;U|GO7wcRcZq^ia89#S0|5)^urZJoDD}^ZdTYbDo){ zA2kXd1nyXG(l2L7j@|Xe?|a6h^D4`REA!t>c*uZf#i&=an;4wj!QXO0Tg$k`vAOYd z#%5p5XV162uZ|EE-Mp#hLT%yvkEc_6N0q|k9*LFpRg(9_j&+#bqslmy6z&c*?CJ@w zdE=(U(uAME3zb0Be!^6I7UaPw@EB*1rUTuBBPx)u5hZZUg zEb3CtlV&!pQ|d8h1m(!iOg@K&Wjiy-tB3+E{f1cma; zZNdZ!9_!tnwSS%WMHu?PSF*!;dIt7Bu7o3f_r-nt0+;b4F!*|;&o2#Jev2N6o209Q zZf9WK!`V{|sc0h6IP>KdaDp$YsikEI?(8df*z4Y+C^$&$+&La$kA)4~Y*w$m(BFSf zb?bx5)t?z9Jv;qJzB=t88ge_NpEXm-RPBg8LS1Uy>);4%12p}7UYkkk>8r5BKCBs( zSA$*|4xk_*s)MN~)brMH)84(upgVnEQE}w->2K>}1_Ze!PEa`R7vP(VOrK+z#J8{D33xg)<8?tk z@+%V9x+631!L2N-1!H&;K(=7sT{jwYge2tTj;G}Rx`9RRn)4r`pFZ7B7@C5&9A6Vn z_aiUiS&a8U*7@5NcrhnuW@a9r)KXW6pK&DK(%POWekUp~FVvKe@7}f4TXE8IJG-*E zLHn%tW76_ub`9^u<_u>&c}lR>eIBb?mzCef8?LTt``$lBQsoIW^l~^5u_Bt);T)9B ze}DKpos#mC&;37_@b(1}N*(dU_?*cp=y!xuL3%Qzk?NJ9r54%{pP8wwrPX%h+AG_- z_{~}mZ$(8!SjNV*dKG*pxBn zN>q56({B@tnTzHHBuBh7RJKZ#<<0wkGsximlxJ}Govn!z4A!2XHb3>=^~q+K(%!c; ziwD&|i+{K{n0H-t9p*zpeqZwz!OfydTN?A2L$^e>_#IAX)CbyGh4LJm-+85fcqli{ zu-Z(4;`q2K)1+2e&ry$JJeDJ6{^m^WDDf2`04HM&gaKbv3Qxi64cHXK6WQ z+B$kyYMmQf1_pm6d~<#B*pB$yjT=sGe$%b%^Wso+nds)dVnZRUU+BYC0|v_7wD}rJ zV~%TVJ6-f0^R(6UA0(Te)9b3={f;(sZ}7F4Z1Yo0H@mDq*=Rkn{j{$rIM|H1cI)d0 zZEbq>RIV6q*&?C!C@en5MNMqSX9vw>x$aTVaioX_9R9L;<;rNKxmqPd*YP%m9yp7R zV_npb%h~t-+%$!mqZ-WXMs;RU2U=4P&)k{A>9q0nLi-gWR;Z*c=afA7wnXw167xOL057U0`_ zK>%qO*V!9h8YN<5M%k*M(Dv3cdnNA0=KKq3xp~Sdf}cKqtOwYJEvS}efYb8D3y991 zfA9I2s@I6^AzFO-Dwc6MYDWs3y9^Bs;&F@xLeL6*w z20jS7Z9w?3J$sfdLG9qdzIWW_cUuN`Z&@24SF+szuo{9L?t!MomzIAU`p7LxO6G7s z{93S^^~x!pnQ-bf(Pu2M<5OD1+VChq!uQ)*897!`Yno`Q)Z<61tlZhvb6cCEY44A_ z_3NxA1;`~H^pcR%?&=c$5c2O+w|(;)@6x4zpG*Aa>n6qN@8?IDn55FI<67%yEXqoM zHd$K?Pct^0qsb&3yimMLBdq>R%`c@VI1#)rUK$0x?g>6JnJ+b0IM{pS$YVx%^}1El zdYu_xnSyg&$6l*_Iege61a0ZYysoDTsr!~5jxONSi8Zr*QR zw#y(>BTO~6Y%}@ecR3SDFN3?AzQ>(C07NQQQ=2?yO(sG3*_Fg@mdPQ0P!4#Afvr%udj~P7fuQ4&Ctlvaf4^li zt7Y&pwums**mdg7l~bd1S*Nf5m^%*GV$0G!%UBNc(-hFbpr#dzQ9S!uM1ek#_S)pp zO?MIay*A1X=SLG9)JkaQEI`PfP}vP1MGB#8RT%rQlh}k@XOrT?e)I*P1iWKM936+n>4-c= zo|Fzkad_5HnA=f!rLUw9y33Vay2#@C>bd@t!aTC%Qc`D3t#xm8hIUf#le=tG@Le2&{R4J&2Cqua$E@jj4Lxbk578zDPqVu2}26$;V_py@U{5zV_4WlX!o2f6WC20SeRt9w%~Gnk(6Wz@QApvq4WXO zc-xoc7cat2@w`5|r)S3E$Pr&riCZlSzzL-$c z;WxK>)vDeaH9C(7*$NMrV^@7+V}zd+Ck zr~}n>0^jAB49^ejrZbRZi6J|n4wSnIAm5dEs_wFP!uRelaY2CZeBh^1A|AISJ#@Dk z6(sh90jJ^<5;SmCqqHK8&j7hW3ic4@X+v{RZ0E19r{`I9Q4x&^;KCSukG;4LigR%Z zV-UcGEn$&>XVKd$JU3?jsEET(#J!DP{P<)>%OFvh{rn0+Ut--NG;&#x`f1a~jiX*{ z?6>;ODB5PjxxQ*7o_X4J9%r@B-wlg?_+umA2h9+@eF2Io1KZ`6`0pP-dM5qM#)*Q` zMSs^Z66s`i*ZW=n`ZMXuRg1FZzKll2+8{~odY3A#nm1R^YN}72+IOqJQEs9y?!kl% zdF+YiAu`V_l=rn0Zjr(kdSSbqnj^EBV8|JsC zVu0w_+10>aa;wogzKvVgdzhA`ZN9I~Pg#LQSwUvHQScV~+8rWm11!~FtG_lo|Krj=V|Z7m(GlF2jF&b!Kzx2Onjc{hzdfhOrn9d2216iP6T9{*E1 z1u+Vu=AqIWQhX;LVfm?ipFZa|*g-YN{y}I0N_%3`3{{x?vi>>$@0%Wh)aVTs5=iVf zYp+FsV3nKkNamuMgm&t%=Bh$;gD5_A^%Er`k05&4v}vA`i z_N`z3IS#lhfp5Yk{RI#k^r@9G=_NX9YJqqu=mulZfXbPOZrGqq90GtZus&nm)A`lT z(XkNV@wQO6@+1 z6=ZE*jOiYrI$G+iCpWV#n1}X(K(l^d**cWQ2)TPa)X0tKfJ`D*8!7OtumXd+xq-q^@jYWxa8N1~^4Q!sWb_sj3gy(bni{s=qEZgN@mS1a#unZCX z*F)=n`?}-q`TzRF`g5KIiUka>R5NnH(XhLBzZ)FBscermhwaa>(P*yT)}f>JZ8KM1 zHWUl??k+vWKTZdA$$xI7h)2mUU%+pw^VW5J_|i?Fh@6F#{q@mpX#Qt@RZ6;HX_wdR z4r_448+1GEnUR~kwt72z#^i#!FN4Mw50DV=modRw*S=0y&2isgq5r)3J1z|u*w@+K zK0RP`Q0^rltkwOmwn}||{&sWtiVgNRf?mCUzkrqY{o*14T}Q{1Ldef=%>)Yi{Xe?i z10L%&{2zZr;v^*vC5fc8Jjy6Sp^Qk8NM)5h6UtUZLs1eXBrBO2*%6hbvPo7_8IPG& zMF01VzQ_4~e}6Bp*E!WW;(6}Rbzj$eU7IkNW2OlH_f*E>j7g3)ofCJT%EKfdSyxx* z{|&sqYaKH8DsZSpFwX8%+MTDBcnrPiLAdf+>pnc#9f@n6RIcEyzX25rY~Q~9Yk&Ha zCrW4xK@I2vOUFTd@vNREBDBb3qRmcx1VAVm5p-Qg0wLfVAC5~(szd(~hoEp1Y|2qO zbV!t^tD+u|L~7~|(nDfOd+jGUns*TnEFuknqe!bj8qYQDW>`g$ zs-5k|9RaEc@*#pnA}O%-N7_T&gaB0UKq&x$d0CN2I}<$ua(H@QzLX2NO(|}yy{U7` zq)P|`(Qy3GHpwS_of=NUC0C1IZlGqu(zv2>cp@%Gpmxdq|XuCLs+W^%<27+1U zI@s-!(Lxam>ZhlteehKFJLTe@dH!4nuhV3(!@#EBGl z`Dv2^mMIP|tU2av{+QLP2^WRrG7C*`Q8hw3C%@KF4uy%Ik4qajBE-p$5rK1Y?zZ0>wP^kQ`-t!-Rm1cM;!)m-(-~u^qaueqavN> zi9AO$=_nv^AO>arhN%9x>uI%cVV1S7FY z!)5>Phll>V|3zr;y&dN2kV>HIFv+0d-ppD;ED_koxQ^gGr(mAZ0!{`u*_@Vd^&?3Y%)1h9-wqYMakfF^wP`&^(5c8! zhlWI!mS3mg>J9|snAz&anJ|AIUGK-Ivf%9Mp@&pa{@a}ttINrGt#$4ZLP0UZg!3dW zF59#fu20Ec1h#A0VV18_^LQI{G_}8nex}h5CQmf=OZ^(Sru%vaW}5xRi!~wrnfmGXM+;iDYG`T_TUuH;M;lR3iOgh& z08NriC3xafT=7n!mBV*h_ws0)AX4q4PU-3jCf9F?tu|GAG%(U0u84P8QIW#JfHx8< zKr>JVq{V^TvZBMW1=k|*aBYZo2&9BQX9Hr<)C`93cw>Ag*Ch}g{LGoBN~|)?QPlzA zSK{c)f1LJ2*Uph$r|JmE}}{+o`pF8!wn!Sv`-n1bnZA>lMTTK9!V} z@4PCqucDSCDdd}-IFj61iC&?(eXtYxJ2eLd^ItyAVtd&!L=YmV>Tm^jZ<|1vq6vn! zx8>#aot2Rb1bA$$(OxcR@^U-3e)e#k;qYdyu?yy+r!dw4ak=4VKd@Ws9cGc+&6|4% z(|4|5qP|2uQ}5GKQS}uD=JM~yUagc6u77`tf8)3Y(g&>*xx-rF=}E@FGxl(j1??56 zqjtZ44!SiO;i)RHG=FSH8!W?V2L0{$$*H!f*!ik6n?EX4{0CZnm$#Lonv9pm@u#fW zr=J(RI^dhf{2{8pH8x2*LrE%J6_gH3ohtwUw+TzjrmQ1YnT;>KX1^7x3ji)scE|%o zR)mF0yFXVAhvaIqxpguWhdCDhf&2cpCLuzny~%$tGCHa>?q*vI#E?Ax>V71qOrS+Xn&Ij?IW5f~G#8l%#O;?Xh9u%KSd*LNvaI`%S51w<~5Ix@bl4A zKjH?4C@Kc)mm$%_Sn52UXYy1F`&Tzlt2AG~X5DM>c}f|`5l&|njL*@)2=(%xJv;W7 z9|EP$&TjA!UR44V;3d!No!Y)(!+zkK_px%9cl|x;8mu^!l~ri$vU~f@_TC*vjxeQ= z9F2~V*5~UfEc&PgA4Eq?oOt^^i{Ar7$?iFp@1Cu}vS1#I3PwJ+9a6{I+Yj_P8qYaI z9^7gF`%pc3#C0rV2QM3iC5OYOwufmw77O1Z?g{%4Jo=M4`4JCGtUGN__-)S^eS;Gf zW7Q(Ta_4)P$9>Km`<%9kt4j{-*Zy|%>?VorGVQ0=L&X*Rz-9nB0+J&&>S$Ma%G*IY zST;UW!P31Q0WGs2c_9TF-3lafK+kyG^+)N(?#@o8<1N4^+<%w2*Pw+T>5TN-VM;p% zrE-Pus-1SI4tj_&Q+Mi&@Hz9;_rW>3^JwJ-;B7(^^gh#^fh9r`kijxxU zhKl$&JTv?oJnx|tWVlW^0BuVwPbEsld&Xf$?9)tnc++ziwMCa z=p2k{d4{?WPhj~i8C`J||*h=VU^ehA&TDNH3?l6caE^7 z4u>+*op~$LMcwGIP4TpG+Sf3YGWi zA1lH75GQHaW@!d_a&%g=LO{@pD3#dmgAEIw^(Qir4dj#WaaPX4ZlkF{r%Lgwp zL?tJ%!#oTSR!wX9K#uIlTMJDwY>D|B;hpT zjz6LZHH}=56QdaQNJB0Ro9`=ButR(GDASlOc8Du0&0|sBaHH9*k9qnw+kZ}VZ}g4k zw<_0mNc>~9?)|rl7)9n!IxUY@=!AIi@JJOsS;_x$i;&q*_r+`G)Su5*gDU@l*gmg) zB_6zW9_8JV;;Hk6SQbXZkF1U|?$Im{SRCx+0*eVF(Bc_71>2tucgh^hG|P8OTi+)C z+uIJETadQ%$bJnUwNO!&YkoC2Inm;l#STU5@BpAce2ZC*fQn9-UF3w8YPsFAdcoo?1|ow-DU1F}(bIqdluQA12ZhZ}mKo3*x9^9$eDrhkZk@KWI>vsWCDA5oP<6}jKGJ!#J^*sI0ao8I+{dC+&m z+`ig|-63r*q(z~~Y8XVl?_ol@*>K?LpI&Q#cX(~bzXquGnN-D19650!9@|&x^8H@e z=9k4*y|xMzwvUhAXfv^Vi73ubt&@cPPwm(*>l5w^ZC2a4bba75k>A^spEX!G^<{Su zRoG)(8Cw0&n&J8w?v791jE3&t9_Tk@+~j&A50v$%SYY5t8m&q-EW<{G629b2J6DwyIrq{9XKB+w1-@b4-0XAR`o-yX=UvoB&yv&Atr%N;fWFiz@k~ zJQXbp@X@e>g?d;1nT9|Eh$a;+6b=hgyl_|mzTO7pPMG2sx2@ z&5+jPajqu(8VKHd!As5~lTDt;Xo1q<`NVX#V}}|@k7GJIYx_N^Jtqs?B%|Js4DbE3 zyR|A3@XhokC1Ur>{&}>#`cmHaH!p=;IjfbSua?(RP{Xh4oZN7h(&QoWIs)4EYpG}7 z&n-5E?^z1ihUXq6kA1f}mbb89qZHZ%a+FSGN`vY*nNbA==IrDRIyjw(oViv<3win;gYildnnq4>eSKWL5FM}srBa?1OBR{AlV=va>kJPRpP1PDjJ9hHD?#`Xn0X)h9~PoX z)!FQ1h{pRXV}?0N8w=GGui9)ur2~w)FTC|QIxC{`o`__7oS3KrH759}C*xZ~giUYP zVb=i6i|n`QvQ}c6AavH_#{@sXl?$S}-$?>Q5k`RNV&f;hbKZJI^3^OpKEzU`z4);S z3G^x7ITKw2__8l4xyeV89X77^Ga_xe@|;^rE`45hI@kM2gdjDD)b?};x3{unXyOiU{9ETg@SMyG57-8p~sqz(Xo(&>PD0Z!HWtf1Dc7Vg_SP}>3% zr_xQHLM5sLo+N7qI2`~-V6FIYvKWUIz-Amf0!FF%p{0lQClKAM+K%98T%kmB>+I|t zZiqIvZoS{0qI0X5&f&WQZ+NGq9qmyi2E~ar=GpAvSWww;cJK-4Yis{?4{qelI*m3M zV|2I25bSSApjjNA;h*h(Wd`~?x`?XC4(?*;Bf5c!X7-o^)kSp5w~h{?rvJ?%-I{tO&$n?E_(CsrH7P($6X605qrH-GiYvZr1)Mw!~_n|B@ncGnx<{W-VDqjc-- zyM}V+I=0J7c;~b^ls=FDg?ASoUcOR4@ICLZRgO#)#vAW{OiryFE1|M*NxPi{uht-` zQe-2Ie4u4a4BjuJw%Je9m}-irGo9w;efHz?j@n|GpHo#ApRch31bWSbubXZ-u=bWE zBSiqxdO=f&ngI$j*M(r}08^WfI&$sC4UvBv9cvWq`(oB%!1rLnYb8!VqQBvBT~tg& z;p76@81AA5_vo6QEPZQ0*g0@>r}Pe9sHv#HqIW(_=$YUsfZ5C^Cjy-^tOxO`#PV?{ zw%SXWQlyA7MZ6l@v{LpW*2^%w-bxWqeueZ@yEJrZ@{n6RCf%^fFN`rj!qK?$VsaX} zS$7sU34k9=c~)6}ihIF-n4{ToO)9|re*^P3TAdK|Gf=-(qRp4>hE#@l955{@mg?P| zHB-8V72|9`y;+I9<0?vwpO4|DLRZD(SF+x(Boe=H{&chK8}2K@l0ajF`xof``0j1g z-=K*ke|-&rSFH})V`cBUqN##ztp?JE3sUzULj?unc|AB0M3Fm`Jx4w)JRQ|o+_8*{ zOHzTE?i_@wUg_;$zL0BPZMEKoU(f2y9T+Io+RXYqoSlhlHpYMlg1#KJjZ)#(F;!pb zd6^P(CG~i^k;~Qzl{rVpBbS;k|K>l#mGYSkE#b{oF|<=tjpxCXagYbz^XWM7nq=J!ly%#~w^`ap**D=wCx;AI7=Q&1|C8%4LrH)FP+L>8OwE1Z z#R3s1WdhXNj8KQ5j7&nC>QiGW8lvMeh}|uQ5HlDJfY>CaK5UCIK8`7@E?GCZrsgOv zvY$c#Xe%n#%^W45J|bdb#Fcwv^#+3X068cv5-BZ|e5$pT0G&Xmp$-W5-U_wDU63F{ z<;{|WUI>ab)-B00{6jQ|P|@JQ!xkB?s5^Tvn3$L_nflPqpYVu? z$%)iN^#N{7QOC&qBqmbRw30O~@-wUN_3IT|)R(5WtHrP85M=O^?YaBYU_scjTWOF@*1F)UZ8io0{`_CxL&)%yq61|T(WapbT$RH7WA@w7 zi^6i`I@!gNQ2@WhwQ}*6L0&~S?T8TTFmh2Hb6j#pzKkh z^?LD4m*+_rFg}kZGYWm zEImP{a^fe=NKg0LI)*oxz&P0CO7wp~ZiAZL{>Zxha*;ZfcLBrDenPgnNVQtlU-k3T zw6aPfA}mBCPkiv`58ZWE25&S^2CAUscxNR4hNAb7h{Nb{InVkIA12O~Iv4gKNdoB4 zDsub4@)d^~SDuQxVUB1uJQ2VS*D0cj!t4%$KGEdCF(Wu(J9qAMz z4ht%aMnkmj>jfjEw-DM68O;MZ;)O))2Z{#D9`K(3iv7@G3kV6F^WkdvVissno`Ky6VBEJ^E@U*g|^GYm%Q3k^*RIF zWB&a(Pn(ke>j9ei-2*8v6+#-=D0*^=z6J%((o$H^fuH3b7g2)}^+kgapJFXnzPL86 z_VV4etBibJ_*ZuAw2P{Xw>Hnezl=NLa@?V4`NfGBWr4Vj0Bofppqr^d5ZW>`;Pph2 zjF(LLU$UZ}49kcC#@C;34qCLf+`vrMa#y28og?_k~MAW0aX?nBhpl)2oyM*oH9q!7Dej5N52y7;FYPJv#v? z5%>?oDZ!d?NNbv#xvFC0;}7b(3|=NeD;#rWNZUb7^YIN7+a%Oz>$q?pJSq&^yjS~elJp*5Nk2Yf;qZX9-mY6A zvA$N(MWHdlv`2Qzv}jf%;8R#gMO(7Yn;~A#wBQ#_^(KsO$GFrEK^_sxGqa;+bs zdQCx95S3=n=q(8s1(X~hkcg3BPrzub6u_^tLrCL7*_$^NATNn~ZSzo(m%t2xanFmT zFtMT6fl#RTkxN00~pb1%tWx~En8@J8--b9@dy#*8P*zfDOQ*a zY)l9j1^^LQGp4b$RZQTUoq%nE+>_}p-!nw!!FSA;L&&IS_qp(*|9y~uu(60BkRm|s z1q({qw5g>d>Ag`k&I?e?tpT^`)*r+HpXT!6BIt0f4!z>*3w1ne3zFyMhX>gTEeM%J z09N2Um)FP>0|2q36BdYQWcb&tomJ!Ek;Dkll$zXZr3sYt$Iwt8K^}b>2wH|&cpr*gLQXrW@{b% z%7pa(w*pn;0U=r^{~>-dS4_{GgOk`EKJ=YT^wS*!7E-0U!P$rAkuw9^?{$(g-6APj7d(*{?~r28##RC-b@g^dbuV{vC$G!|k@&WO6Zw zgYy%l@Sxl$hO;61X~8BvGNnT*NZZNmfZ)!VIjqLGsD~3Oh5NhCJ|g!O^eOx0sBw*Z zCxC~I|F$BQOi;7;?=Xp->-i|;ic}_Sa*>`)$B&|{z_616ssltU$lqMkm9`b~Ou@W? zOE?dZzOlGNTLTR&ntohY_{1eYjtdLV9WJbyI}643R;^iY$4g3I2^9Q9`D#l17nYWb z>P|p6_L|e$q}Y4W+c$5hLTwebm~+v2(ZKPVSz8wvwnAA1jkhW04XE)w?=0aEhm88$ zh?A@tVJo8Pycf6^R>XM|2jpI@-OUxzKCiRRqw7X-sgR+;d#jRq#vfk5L8xs~?=+N~?426YAonn0bJUsHZ-weY&|qA1JQ-wYKX`X*SM?ZLo=<7U zexQr2vl0of@ao!f91?mqkp#zlok_nxgVJBe{@ASTgrU41QLoBy>)<6CX`O0xUl=#p zV=S!>K40-_Q90tD$oQ)#fp!uB6H!;MQdl%FsuBAGq+-@tO$JYiU>j-((lry<5aYjm z3aiSPpTr)}N2Q=OoN*2Y*mx$ zDV7=9BlYf+^$6DIkD2HFtSZK;H^g3^Sy7MrCF;JRf9E3)|mm z5c0l(Y*(UyMYoat&Ep&Ww^-@PIM8>D!%~1}P1Q45Ywez7Z=^mPo8P1i(tUC4e)CU{ zucUAf6yW87RXQe5$6CP^m4sl=!ODwfzY#_f9SA;Yt%bxI)@hd*UfA*B-mb1ntk{E& zskLR`H|E)@(|(dMO-)TH()`V>h&PGDysRgImXc_Q<(tae4owsDfcP)n^(o2+&Xlllyy0qG~N+9qI zWZX4#eUAyEQGuFlW}Jfg2G@S`hy&yexMd+?B8U@Mx&89;i}xB(JYUg9x)U>7z|3kW zG^2Ziq7!$ch`{lmLr(d`lMYZr_GW9d0vHeigQ7AIu42oQm7dBAzH=u^cj`yb_xba@ zFD7<`Xw)4=l&f%2bQLdK#`@MSH(Qjc^=NvbVJKWL)GuPhLLY^&!)L& zqWIeXfpo-P8)}BuTLUKI7=4|U)$>#u022A`w9`#)k!v5V?7sXz=(8aCXL<7sP)l^3 z4%g$Z&mya1%=?&4H1~(xtZNuna7uq&B3|UMqlO5j0LD*`<8neom<0`UGU*kuT4@gd zkf8Nc29yH~0k@;Tt`l-w8ss$1EEOPCS?IM)`D+B zUSuHw{8YO>CM71;0*jxflAjC&f}q&(;w+$Aj3Cwqj|RRFTNN%YBFk6Xw;8!y=2I-OP&=crME@A2_!(B!clc~u7fe1~d z62c$yUKEW>Tias4>j$Awzpbd~(H8N6rmj(vsGCXAi+W-)7S}>kzZTHHf3aa7htHIW z*(IqslloV664C_=e9y4MHYIn=_oTz~>whgr z4h9Y=9`|#2B)?x~mL;5d?}(F%(mcIgObqi#UJ^d+aH~rtH;NHf3*ZZJi{t-#5V?q+ zSP6i3%9sJStOo3-tmMThuw@H5?SjQrnQl@>rZpCuS@_Y0ntvVJzm@IMI0V4rUVayb~N8bso5^_`|WWcEymsdi!M61!m&*5#L7H9qEj4C{4?L ztpM#^aoaBFc4U!tD-m=8BQOD9+uhr{sIL6M?)C7(p&PwMH*c0uc*sqLXW$qwPN#wz z1U3CBMZZ`D>!lNs_;V4DLkwEOqd>%X${+zFvSz@vi8}e(cn}+uQKm#L<0TU6h^zuqF5g~t17;@8yw^SBjao8NB0N9Q+f3%Tu z$hje1fhd3Hue?%nVhHn~jcIGG`|q;>OWO`07yKcK$njLXow1V zSgN~fF!##bCB7*lkw9-_NSw$fH)CS>2|7Qm%sLy3bVSmpf}+E!q%~1qg#)#7^vc8b z6ZD%+dqm0AlJVuW)i%ZR0#%}^QbR2A#Ga*iWc5l~g>hi-X6xSkNu@%j9XuZ*N}7eP z*+b?sw<)D22wh^Gd+XSQACTxxRr)8Gi-F=T6d$#0V~`}PH?9d9?4QIq2%+Ue5U8wg zCIO+mrTR{U**${2^Tqu`IHMcdN1HFEkYRo02`Ax=yOkElW-l{boP6;>T54)K0K0q) z%tVGaj!nG4Vn;A`>@@vb!-fNJJrJoMsZ1#(W}CEQ=-vx@bI6PUa&Qt-F-zLp*)>%^15*Gmkf&Hn8s(=f-cY!mg3*n^sQ%pF5a_{S+%jQcarFO8 zi7#P-cHkrtiQ2RkZJW?eJVs_V61nvD?U%`|@beURp3CE(5r|ftf1WKbN%y>TJ|(k? zooa`7YWLORG4UI>_(h9i>>JJ^h_Nn6Q=+D~BV)<>6H)IUeLEc)7{F0FIM-V4v5%Tp z;QAkh1k1eK+DB(%Rd8`M9Kmg)DD9%4DB;b;(ZUGO`hQqt;sp;0(5#l|)bCc3x7Q{%Q8~S^_xJ>P`(Y0a^;gUzSQyX_ z{=wOE?BO;Hhj=$6$2O+w1F-w{HgH<%2U18#@MthRM;IRuV{VxS?C>9hO>o&2`oZ4;8co2p zDdC$q3-e9Q&DD`eK!##6*U)@n(m?AY0Q-YJfR$XQw!YqHScFBE_!02m6W5vK z&(EIZxxV6ai5r?4;w_nWG^$FD+yRV+ur6*;+7&QJtPd|=sRqe;5-QZ7{;%gNBZw0N z0SQ2#wnSMWTM4)#7%-0l`n|30NpWy(PV<*Xa8riYXGy@5b0;ip`;2w$-Me>E;SrS2 zCFp!{-SGkOZO%PkMs>>?C2#=|yFnp|5@u-d_F^U6Bxt;zKrIRwDYFMe3h8uB5t%aP zazoN1#2i1rwY9avw%N0#Syf{R4vJ$bhStDP5LKiIeeTwN)_HJfo=Cs}fTS`V=va4>Al6tPoExU_^ z!(xZZQj4IVp!n!$LoKPg3HYxfkh!D>`PSQ)!fmg<*^b1)8U=OM2SR0HY6&7pE@3F2 zyZpLYM}AC#t)Q!rNwWM8!^4)qbu4?b^jNgQ)<{&}*M zQP1Qa4k|?R4m}@v3jX-?>)}%DJYO&oN{m0RUcL+tPI1R=iR55%s&Fmy_>CNR?Z=%A zl>l+Qi*g1y(bGp0K(E(km-j2F1yosQ>%Ge@^E3OIC>4RvbTY;mm$iR>@WFhLtE21dUd?&lbg|8kkbKS@MjLiM79aX3soRw&hdhnsvEc!kS(W_QOGY$!9m#W z>ig}|K+vqT@JVHor2GIf&_KzFvUDU#z2|bBER;^NwVmNh|frpoyZ!3 zE+c^uKus8M0GUs?v!H`k78VrPy!nahd)mm~Q+64?AAAuar9fBS(?RJG*J!l0#4MT|wO>koh|Y8!4N%ZPVDdZ=b?8 z$jD*D-XXx|M%*wYI0B256kO2}JAJ=rd_jlh92C01hl5ZnC$@4*I;9h;^GM_Y1O-r4 zLFrCI#Jalfs*Tuse}v>w?A%F)ax@-`wN+p(N6;b-?~WzYsf(3b3eN?4p*UPIS~RrNLTr>YpthtPn`Otr2_m!z_y-B;AsIYe z5~E={?l3@b5- zSR^8O6X8(4$3-uF8HyDwL1_Q6Ox6dF??#Yu)9T`IhaC~MH z&dmTU16!TEy#d%i?u$Ag(Zdj{cLU~g9Y2a|C9_~Mk-$!HZEWV@iO}tHj>2R?Iw_n@ z*>2T6gI+MvSgLLQ9f3D>T5n)J1R8)?UK=dF>*`*-c<$TV< zS4rUHtiaUWeCB6Z$o|ta)bm^36Co17b8#Xo*}0uW72*Uoo08JboG;jXKS3=BbY{&#{R=gWgys9`eapi%Zp zmtM1foEFSsv z+w8{JR{Tbz2?ngCK)3{ls&sc+B+Xu2j1bRZ&IO2Ro@AREP<_gNhWZ(G$ zb$NMt1y=;^I3cRRFmq=@iwy#tu{2~DSxZ}%FnMjgj(Z2UoX_tLj?arBDvO^<4g(*7 z4I(hz=F)mU_BsHY1uy_sVhchfAjv1Es87Yf;E|4VuLx*R6(C0o>vD5*Ghn44gC%j3 z2TC9gYXo$926rn!G4jQot^uXm_8c;bpIdM&9kZl&t(@;#vVmIJ+V0U9E&p_ z-V-E>MI(Jl+%8dlV|VULu_&P$Exz|ESF64VZQtLJXjC#+l3FO;zFBzgNqQKbb$X3> zbKz$vV2w>e^kNs;$Nd_6>&gTR67b}crU|b3>|Sv=3?S_SQ^y>aIr4MlxM41&9}6~Z zqo&(UEP@X)J|X*rU+(E;w3K`IjZsQ~9@Ucmvpd(J_cjP#K)Z9!5^h!UJE#+vGC%{r z510P)od2Djcqky~3}PL}3`4^W!&7!bTYDIK8}t9!|N0$&?z@A=ck{1~MDok4cbylCYbtKpGblt@A9nxGv+_vnQ~O;?^S`=tz5ztg*8~jYF~L zJlMTg1Fdu$O$#UKkR-H0TU%S^F08Ob7Y*_7oweA-0C=y=Xy@4PABSxr3gwb?5{|kX z9Q+vi$J_OTLG>9eU|pV*liBtp+x13*I{-4e2P~-cCP0Cc)Nw%y-+5y5sz;EL|-m> z;lLB0HjqK)Dh!1r#|wBadi%SRfES1vfs9JWwgg-$oMW`a zD(lgIZ@!K9gp{o)6#5tMr|cC<-@dIPf&l2p%`xDC1kUWy#RvivN3nJB(GyJ{69tnU z9yhv=1(;KcPdS<=PYqh6-SxY1?2tiJ9R2O>Kz&i0NEB%tk>Mc#oLEQaoTm#Q8xf?x zxu8{05k3RLd55`A{hDaOhPk_LQ$yi2$YUzO$6gv6bcQ?}Y-2sjhKO}b9?$N5btE7T zK+NYeFYwo(#~>=#($XS0c8nBqp-l(D2=+=$;K)xK@lYPJjU4`*PHUKmi3Sgn9Xuko z@-$>U34rT@q~|bhze2`5Z2f!$V2Bv$p&|-q<(5k5>#I<_2!WZ@pWo+*L{u9{7oko0 zbjug0BOjOuO#JVo+MO$xjK$oF?r99|W;ztPM1Ka0cQsjeAi&F86m^Ef1ks?xy#r*R z*0=!$G942l+B`JjS?9r?K7D#*+OP}+EY=bcGe}fE2;Bg1BsE*x9m&5a-cfjLNEc2* z#~~j}fq$PkR>dlQ;h< z*ZAC5>ojTBHEr0jqg*PViczC`E@)2%tHnDdCh+kRuz|E0wmMPVDQ=vz@o> zf~q1{)z7U`>rS4Xzk1H`LW0c~UJIzsG}OiDd+7GMufo4tc)jo8$XWm_v7X>P*aJ_B z6`mZip7XeP-iJzPhCcV0;aI}<9r}l58#0cZ_wHaX29p$DZ^V%a?bdMf!)q%Kb(w{KaIjL%kwZL*{L6yUi|)3%umV#q)6(`Pktl zctc1S)+Ss=m@PVk#+DS{Lc<7a*hCMF8yOIY*L)py^k+WaWpq);Nj}R6?2slz_T=k} z@q|$uv>EBkv6%w$F+rUKqBb}d?^yqya(*-LNw|>lea%Sgv)funLOC!2Hd&n?cx^03 zDZYwVhMZsWvXFhRWrPF3UzUj6!xh&J3)5<9Yl&r>Ps7WGe?|-V)_$|18V)|wfgSMOc{(TX!Pi+D?Nlq0o zq+txCG>lq2t^lLIn)cV0Xnv}xbDm%>q`34OnESPCGOw#{HwZMXdR}Gl)TANRkX(D9 zHCHAqm`6v-GMEuhl8;*YKh60IjF3b<>X_f6XNmiZG@9d?muxGxl~Hi z77apLAj9Z_f=a4{^QZ+5%Y5QAMI)}Mq;w57!t1XLjzGx{bYlr|_dvv40+r&6Z?>1y z-G4VxmP`ST-md$cy>U4|w5OZ?3Q;l)h*x$#nH_BW6B`dEm*5 z$DIrh!mWCmhO)(j+usH28&_(d3FcbLap{v2y%SC5`iki6;k}TPurt!7Yi%XFtB|fX zEy^KYI3h12wioQ+=**ry+M$v}<@R4Bra`DB;-Ng!l7K-K`@Bps-UVP#1!Tc6Yi;J9u{KYy zzBpeDbxppL2JRbp8<`y!W!=MtMaX^RCae~-dXR98wvMmY9rzE?rQmvr6GZgC_rwqs zBFKEuAcm`@qeTMH$Sc9wCr(Ov|D%h0eQ}8-8>bsN1<_pM*86B@{kJj{&2Z z9R~Q3eb%fDjKbdS$MB&LD4Cacyu+ChcmCg?H59>_pf&X7M&)769Us{|2I{7{J7r}M zlhom#E>a|D#87kd^6Ve`(%LD7{toYH>WE+g0+1G2NI>&y_Vpr$N4=CdbM2uYBF4n0 zR2jbg4e2)N>i9Fmc9*$gkwSk1F^Rd=i$ZvzK+{+~r@0hGcD<2087-B$zpc!Oa> zdXKFHmXdD0NhbA`G>HA@c?-){MchAx?3C!*DX1WC!hQ! z#1M_My5^YQAFK3?j4sqora0hUWa5HJn?L#zAMpi&+a^Y*xQ%>##AR> zZx6(X&d9{n6z(zNb4HcXKobN=^X5%r|ABr4pTq=|!8(?shw;l?rKz0 zL=F6&Y7S8v`t@DU=hiE#-2W|DY@*kpiO#8k-Dnww<439i*R>R(!de@<>1ytkLY-sQPql`T?uoQBs2}*W`XMU=QEWknPWJ5_Jz7KADC~!40?42UKuO~41D%OXR3x9~=7M9?B|QpJ zedBPis!Q&g+%}^K>Q8gm1rqZ5CYQO|W(74s)gEdPt&CXQfF00%)_DhVimc~> zITmh}pJB()mqDw(2p8;NQVX|(y?qxR&ds1f5skWSBIo3G(g~uls>{d}hjM|cJ~aBv2)F5B zh9V3uUgZ=AFl6t;Bv0U(rne0T{takZ9dH6dGu!KGz4Tr4y0=Q1+Cmwu{d-jC+M$2pU`gUo-`!&62FLoC1jdJ zCbRn@)(C3dq3kSq-pOc1?FoZIrr9j=BKr&{KH8LAFw%T$6k|G@wAefp)XIj5A zfkBCNBSs~l`9sQkggqmw0jel}?X0kBO3{8ix1^H6BiFxB)Msnys_)+OJoF#IrYz|u z%O?6kB2p03?%Ahot5-3C&XF;+5>B_No;XVG{N3|h4o6F)+IybH;re=lx z4Hlce@H&YvWHS6VKVkCx#{`Xh)lQOOuXAJbl!wD9Q*}pL;sW~x71I1KcMlI|Y3=`% z;OW*{5KiquYt1tPffu0!GDDgE%3>-sp+$rTVZ{FvX{BMCVEO6F=^a5-95YCL0r1ukgW2vVvR;&IeK3@^Cht5Z3(b!5@WpYtC%2QI5pbLkhBYPF z27tAJ-n%7L2EcTuhDT<@j7%0JM*t|&0RQleKw~!jIw~u#8CIU{wx~5^ zaD!jeBE=Hvr)9cRYLJ?H+WrC7GLHQi(9<~;f5R|AAh#x~v4>x?)rpjhtex~x-cexi z>K~#70qiJ9n7RerE=w$cGt&hB{17w*6QOLF)PfZnv44icWz5`1jon}GX4Ug2TwIRb zbO;$n8CBJC_y@t8LR1e%1XQ2sqcQzV;X3{;1>kI(=#Ns6C4ilJAubNt+s<@8A>G)p5*1_m8asr zCCW$`aD{kQdFLW@c$z8!{GAijwhN+(zftcE7RsalQ0TGJWSpMJ zM0Z~wYh3OR?f_}aM_4ne_w!!hvN(;tHv6gh^yRQ6!9f9VE*I@nR^>_H9p2V`PZY-(pU(UpE zDFY98BKXh4=N@3b33vH%73IMy%LM`(vf&Uh?GL}_1VWlhpbyq($S)thuKQbN#lnCe z3TPt(rbey*h(_T|j2a_RP*~u|1%xaPO26%x&^Jg9$7ZBy4-6YM4|EP5WFELXXxHY3Kz!p^7VsYE*jy)|)n z@Dc^-jr!4K z>xwIcq>jO20xasZJF71Rs8$8|EYDtMc76g-L55#MG+)z#lvZvvRTY)Mr=7bqhJWN! zDV>6#SjT@2-*Vl)b}Y7qR$WRvMHUgOdenE`+FWFI1c35)K6#||ib?!uZfB&^y!xMM zi(b#b#fw!pcalMX;=viROnP{u$m%XAzy`;+_vU`$@y?`wSN%;aebONQ?u{E$f6~r( z=pPC7nJO$uqubdd!NkIQ#Uk?I=cKcKXy#Oe8H1v_?#Iw9`;)~oGo76q8%LgHLbSL# zh~u>H)q}oYo^RRoc^&Viq7=^_>dto#ZtlUp_gGRly`!6S3o~B3tKYGF!p&IX+}(Ra z1~Mr=2N$T>oI6*HC-W(awa>+MNY{RfIT?%i*rcSSwwKAu{rwPbn$N(nPEwK!TL07_ z(+q9pusy5RAMmv~e)H$g3%+Z1FGVat2?Q-XOgD`^fn#5Sy;BcqWeSH5tzGK*8}~i0 zv~;Ufb47&*PMq9<1J{DhItPY^JY!=w;%)3T1Qx->bRloSe)Y|PCf3%Os+=s0Y}HF7 zb8weL)S4YReAojfR2^E`nH?`nN@OD|^@tO$%yCuX0-jqWAI%RM(XZdXe}UeQHJqNJ zCmM1VlBkoo?OUm=bL{t~XY%m0~= z(3tDc8{E2GsRtUc2zRyKj(cm~Mym-x80Q6rFa9L5&?qh9`SZ0Hw<@~q@Clyn!oz2m zJUt_h#p9EWi))dtZY$lPgK~0-3kym~J06W*SeoIkKbc}tu4}yYLBKgk)v&2wt zyj-#GIwtDs8v_FuQu>R;L_~HgOZS!JU(E>!3Av6Lb$93xoa?W@1eUkyZup%j%vQPA ztS>d9`OM|a5apgt!pifveA8O;?efM0k8RRwz06fkZ88@d%53tZWA?b!@(vF`7lOI!C28p7-?55} zNl;KQ=o=5n{F#r{ePI{AFn{gnh>WMTI0hH{EIti`-eO^DhbOM9VxSDnHx78h+aa-N zsSqZw>S53Fo}nN#%c{sk7cSGchbxiXRi*psgruh4z;o+Bn zxl1}8jAQQ~I-`GZtROnx3#g#@xvj2Zw3;u6&^)t@)%Zj$Jsfw`(?9si6-FglS^A@! zm*b~pi0Ejm2TF1kO&ClJj5}7jNA4AF0@Z>=K3q7Q9x@So_wJ?4qo)tGQi}fZ+!EJ~ zxbgxJEuYkPF{(g|ZzXuSQF9$;Qd!3P!^OHK|v#OM?Nj}1-eR}}! zhC+G)y0x9uMw|TXh4uGy4x4}O7>ndr-XNb^AhC3^xh>lN8kY}(GXL`-$GcL3@}Dz@ zUsOga@ZMV2(8|@(&0-+W!@Jty_GI(!fpV%*`lo@ZM2AlCclnz9yg@miK3=}Noil9J zw?&KZ(YDJxKDzlSlo}Q-d8-h5H2MPydyD3r<%nVV{7~xe@s*Qv#p6x|q_t+Z?Moe` zdfh}uIbNFA-y9G4a$H0_<|;(py~NX}FDenX9?0Vh-9O`+@9v8%#B6u*=;jaWb}zlk z0ME)Ugb3#+JbXw2Cgdd(v1CK_VW=Fh0tBD1!$5Gw)Z4p!v6(q_znR;_to{6trO9k~Q71fGz@KS_ww<%e&?C{u#==ie@m~8JKkZkV)25L)3xB9!3|9FkQwVA8rpk!dq8OlV!Kd( z)gI^xZvxH?2LLjW4p*_)jGbDr) z&&iVI%P(Wv!S`xY;D5!0gBtRF)22;(v8M(9p>rbl(xsP-!yFl*YTvFPFX#CU1r{fY zVsl>du%@ODj9sx#n*;y3c<~|=3ya*|y&jY9_#o%2W)TaEoub2|h(k*l)=0T(6}yE#y%{>8cVa&n&g>h%xUdiZ=4>SX&8?a#Yj?yB#S4XbagIBDKpl<|Fg z`h5!n`N#!dY5e9Y_r^-AMwZ2ZOJWj9J31(Mq9$Jk6aEaviB@}a&&WoZo%3RF>8mcH_ooE zlIUx^p&}_R_FSp4;cw;Q?Y@q<6dnl)P5`{4sfc@?zK)bmo$nb_N7=v`+J}58(6H-a zHIsM;!{{#CuT1IM+pswrnE*^d#ta?^25G~KIcgeE>V;O9sKoE=kySZ*^iHwesZ(5d zH$V4H{R}J6*c`}O?Ub(*#V1!BRC-3{>{%gvAw=!yrHx2YboBJr39nv+&w2Cao%Fvg zRR*UUoHN4ws$stP#y%sm|ByYn+5tcJ-}I|*Fk%7j#qjWOn&;@Pbxc2DU8$_A`+j%| zI%PeKeqX$J;jdkT`SARv0Hx>)(08$|T)9zg^J*4^;9P)SprWGUv#>RSE3{J2FfaS( zGE=F^E)%Gp9UXVioG;wi+>g0H#yzezo7lE>tMVOJV_se-DlpzRrKS7eB=tRTIy?v~ zqDtTfPr;<~=3CzH9hH%XlIzI>gfm&615t~Knc1cAaPG{7T~FbJc?!Y&$$Y0;(-t6Y zLI^cf;9E-B>zLb6J9y>F+J!l+NxG&33{G!^&UvlbS)}n;OzSLe!rRB#%@(bA8hJ#p z=#K5nUpe}?24(K-m=Im{zv~ejMS)@8l9#Oe{;eT1|2F*i)6p7VaZimN4mr8v1Ho!H zZ!o(VlssHeaVFis*mR4wO8#R019D%6>;nw!69nh&IkUYtGcj=y+Fcn?@v@F*&hTMI z=BDjC1iWbWW4s+r>qpqFWg7rD+=38a?NKfo5uCKMz@-qaYInjI@?s(x-&Eq2i9=Xb zr8Y54DujM9V^9ajwRc84rzQ&iaEdnO54i$HkO%s6CGI83n1PPql~ib0*iCd~*MqN^ zTu9MQ%(ADgz_9^RqbE8hRrn|~Qw$cvhm|HvR;+k^vCi8c>Rg^JTNWbUr1=AmS+KC_ zQm{XCix)3m8B1%%Rd@F1fWA*EZb`_kY?s*HdLVB9586JQ69qmkhy>wJ*srwTGa@RA z0rnT~N)x+HLHLstJvx4s`!7c%8<;P7V1C>iTiswC{>SZz?yrf7_2S~>TEizD9i2-e z8{49tq9OkuUEcwZ_1^w}%PiRy*`p-MD52~Tk}{IEU8!V6WQVMZ6j3T6p|Xt#dlR=RD8ftJia!^XjSFeSf~!=epkOq66BBlVT@xw}~ssAPDHe8%Op0TZ$iAmnD@a`}Jk95=W>kw|6i-Oy+Wv)$=gyU2=K4rHh1^_RHxb@L zw&vP}Nya}XlupB&w<~&OaSJ-f;nP^4=L}fczuY8GJ&p;N_TGaB4`xk*3Ylm)q{GL@ zR}!Ct&L+)u*4qE2jax4VIwYra!7<;#!72gxb$5Y$EfUGz0fFU+lh8wW*c%1JWZZx$ zI}nFufSG~}BjLyMnL;8WzDB8IjvH|XHVF!<#MSZQh(+Bfp?vulCFwoEdV^unZd%;>o`O zkASb5BpZ%C(PQD|xGnGS*NSI`Y`hdQ1vM=V)oY*6@4a|&J7A)i`oYKbgC^o^^XKLkRw5nnkNBRs)E&W z-yQ_!o<~&LA0EsjrxXpp<%tCU`bwzd_q2*Ei#DaA* zQN-M*`R}VrQky7C%^Lsy?`l2+4{sZWWzMdw`i`DHip_@7u<|Er)O`2xJ51D7vc=W! zndofvSEn5vx%QRB5HZt=K^8up`0)I+-_UMn{`*`qO`pMN0fy?ePIaGs zK49eWi3ui&aDnHagRoDh8+;OEr3WY5+uFo&S)(vG$R>^}EkKox*7aia%)~?q0rW;p z$u~d)Z32*d=3Kmf;F$ZEYsiSnbLcLJ-eja6TBNxj5k(?q{IM}H@$v98Qlc~^rlyW% z7OL@=phn;hyLIap33DgwlP4t&7!)Cl<=e7_TI$4XCp%`Ma%jZS<@WVBc=6ISXI@}U zF)kX_Rrr1GEBmg{ysfFJKv3wzDprj6aG(a=3ZPQmzn>N<76?K*o^S;gk)yAzHX_{u z-L%=j@UZW}gp5v<<8j`If~oOwB?E(X4d-_uZjXtZn?C2H0!?6Gz}wH-f-DH74uXFJ zQl;smXd=mBT8C^^1JhXQ#ioV^6TsRBJ9Vai{7^Y_Mi4PqJi@X}IgJ1*>|nD=H*Y4$ z7t{zwSt`9EVobpASG{oIxoVETKQ*8V*G_Y1n+?0-I!qk*s`1BG>IdliMKCW`1S>Zr zOWj$&p6d)6+{C&Qwv{G*Ht!ac*6?a>Rn69-&x}~5XS8N6d&24EzYmL~_@+{yMg2Sf zE>QvV+O3x5WT()n2L$-#Z>}@aTs@B0HLcgdC7_Vf)z48EXQ3Cy`i3&+vi!*tC$498 zxX`M{)y2q3NHC$C_n4&p^H&uhTd=ruc}Cj*Y6tQV(yhMiU^?WO3PK%7T`{Qk+=o0& zOjj`}@QR37>0WARXz1cune$gYhPMeDo~biCyB}v}uJ>H*<>9I#(vv7CDFazFE#T!S zhC9;y*8zhS;35+XQ=mp_Q!ijHj5S^d6Nc({;1GYblms^A7ZK6KyJeFE#+4pCh1p0I zavws~F@sNqhK9!WSPE>r-5}lQPoqK*Gv;nfCUl|ep}w&bh6hW5*bl72&%PG4F>$h{ zTng&*cVGHu0;TaS%}xB91dcFLkfCk?Btqnk|G@-RbMs#E%)Vay&+&^zs8HjXs%ldK z=PWIiE?<@c#BI9cp34gkZth?V^G}{UQHs%>dLB(oxvj71eicA&Zf>HY17}*`co?Ae zsjcnC%2MabdRf1AYlO0Q=%M{&VrKp{Vh`-5vtPROThJ|( zfvsD;#%|F1o~-IMmYh0fwTgZ1h6Z+##Lt}tyVzcYOuzPCrCY?e6pT_9p7i?c1eS@AHsj`gMjl3C+nZZ1hH_kg%sI)wmM> zpIhj{w>)w#J=ebHBS94tJ@gLLIbbY5+A?Sm7=wkKJTl?fqfU#m<>=1KZtKmiGfDspmi0)v^GpYL$`;^ISDz{4cS zAM0;Q{?R0jx9yT`A&0W$t*`p4OXb(oryjX}K*^`Xn<{Sofm4Q(?mx0}s@Dt&IGkI5 zk7C>@f)BkBDJ0L+N8Wy1p(dDk1Fm03YqJ4Ihw~e)w-@fY7@$39 zfYhPXe*9JQz<~;ML{`y-9!q=HQjv#gHF5FqJR`8kqem^HQ-J9mKYkx=_vFtepD$K^ z6=qjYe%L?rF%C(P@DQy7Y=rV<%bxvO7O_Uipe;dnjpJQ)$u)#8^Y~3zy;dV0M8;)G z2q|4$-$3f0%yn1|OMfJmd}7F=xMtQx-8%gJd#USK0X5O;!ZU0?H#fJ|*|Q;ofaGz_ zGE!7B34u5YhNe6yEQsoMg;@y#J|4UtyqZV;5SxyuaHvuk)X6^pjm*tu{|#~zfFxS8 zqSjEYcp3%<|KrK8I}L*Pu#R@#=`zLy? zihf{%grboop1W^);{1Klr<>{QXZqlF^9o-~aOWKi^V* z@|L^hS}BB@70b76`&-pvng;3!jVxoJg)7j7BeM*|<0-6AVJwRJnW|rf` zHw3o?R$j`%QO)`G*57|#z<>gHQgP5W4`l30jSP<;55i<${^m^($?JE}?`d=9;R+Z(Uw|6~oDc}Z~!~cYjUCx^qv$yi|&#^ePR6a3Hr{(12ZiR(WpfUged{iv}NM+NO zEduEL(e}!~q*8wYR!>@D#>CwO{~zs5^_?cET*{2UULI(QFlZPkkwQ)WiF*3fxpP~g z`tOq@&7dq35KYQXZZZHl1oe^EZg?Xkq!P=ry1GloDCm!1b|t3FU-SvKP`@0Lo%RPx z{kxGuFNHz7>)SUeqLH!`Vxl0!1^f2H2X&ZLfOa!gfNbW_fNPOi&G| zM>8*R(6w=zm0EZ{pxALf30XBEzhIFkANt)u3#17$6tbgZ8QPV!-LIV1qL@o-yCy=iCfA$V$vv03k@ z9K%i=%isN&4_Y$p+aS-3(@%Q*H+y$8EiH|&Kv7Fgjl9}Y64WIR%+cAG`VWRU9u{7Y zV1f|v$i=m_HXk##u3PiMsDK>;S5*jm%kp2m5J7$>(LlEO@v)bJazX3m4HvTBd^D;w zMufeCQHhEGJTPe3f}%{g0u{*7bTdke9mk?uo_(Io`lzOsQars~J{Z8ifjxe0RXI63 zdH7!#6o0H~$Pwp%BUAkLp%W!q2A5x7ClIK&8+po~`_no8bh`0Z+hw1wx*i8*6C3M5 zE|~@<+6jnJdJs(G$mTl(mLIEwfH~4rd;RkVfA<4X0LAi65RqT1@d4h7jt+t46-VVD zSS_GQ#8pL5sP%)K-#Bin54UD9;8G&+VNjyfPdf3N*KOF)l5pYtd0h+!cyr0ld1kM{ zHKHx(^Z9&c1eE*y2>dKJ=((j8KR!P0MF0(h(=KS<<@4{}-C&t{eBHWrQ|FYaxO3ayqf!SWLP#!;iUgBC14G?|w`JM79 z>aWbayNkB$|q!6$$M z(-m6BFE|v{)n!5m3?S}*TUF%^kT^26esFxYXoU;qrV_<~+l8?g2@oZqG4PQ7DlQ@? zv&UmvxJE5@Y$Vt_SV~=w;lqO{L`{1$ceqyH_;E>e>QDJx#KF#HUmjs6%0fr0JcfT? z+?8;t5m?`DKKZJ}#l>-~qBemd9DlzGBp`<#g6xR}XpPZ4a@U0oXjc~63eoW+@&J|# z&Z=mS`EzqWf6809d=G&@*s8OT>rBR%n3K>TVD*|iHz~?b5U21tMU`a3-?(?K@z4b@BXDSJz*-qDT)G9LXNA|T0Syhn z1e5Rky{q%KgldW-%m(ZqKX%TWq5XPUmJEv$V0G4+Sfc8~e_V|8xQPI+lpm7Mi^1~8ISXbf-wQ{b>y3b}C3Itc}9 z(V?9<_BgYZDBYY@IuB&XuBluF>60r$`{&QDv*iodlE6=hH@4lVv7;jxtE$cMYH?a* z@wkI0CBQYb_u=BIIVN!2&;8lHd(SEqCWL$ygv@@z-&pN0{iw| z#ppw;|Ebr4AW^Xh3gILK8iU8R(zKA5CfmMm4VV;6{z3RrN^6p5FEtRA<@^P7VF3F$ zKQ1<8fEOibX#~R4_d$Ypw9sugAwYR&$xGZ0kj>)uak|gS!s3IWOBn;5l#>&cS4&3M{d{gxjeEiJDwzlG!UmpaMuYF2f=rS_R zrqmMpXEGZhO#xAS3?!&$Q#SVq{p+1EtzK4AOQEue|`C7S1z2F z`Ez>y{zgK^f;^mHbeR5m~yM;x?7gJ;y=w`|`|N1PWzXoRK_^?A^= zhrtyB3mL3laL)x&3?@oH$pDwq+~S356Y+pB-$WDxX#xdq(?&`$c( z=$9fDJ(o?|DZi^j+5vEWRb1VGu7b;yC4QhE>1`g)VZgym0Iv&%pT@tRn=-TwR zCvIXI=i}FkGvJ2fFYO{P##9d%lkahuDyNx z)^?w<8pE@U3>K_S(m(`=YL2#G8Q_Y^;{Lq*Hyf`80E%{RXsDR}Z(@UKI=E;auj@a* z?D>qPH>xk>#$CLx>|3RLK$FV87ydeM~~9P)in?80IdZe88rrwjjBtr zvx_9^79KLF*}70%aPH*ZpUF^t2!qOSXAut}P{C^f{oX%*yx>wl{ja9v2AVMxe@c9; zhL^$-L4h`>VsO%q;CTQ+@!pxpGcCIZ7!1!3Y+amHr*~{5LbibC?)}!euP_6yl{Z>|1 z?p@7egq3^n>-ZD1#;{IPQ&TT!KQ#q-skflb<)6PWDCX=H7^fqjf|-MUR?FR9~ng@`*D_cZZAf$XH8 zoOY2%C+xGh=q6&6&jvJd>Ad2uUF(6K%nTwZ2Tn4&=Otm(_W{SE2SHgf6MO#Rg_@;h zqH~N|tihf~O~1j$Tn$DFum!}J4$%?R;sv4t5!Im}I54K8^~Lqhx@q(GdwF@XL{AP# zkI-wNojT|Uf9g0y=>j0$aD{3(JIg>LfQg(kcul7bg*4a?t64(+t(tgR>%1UaGK5qh;1k^NV<}iT6`|*2P$IS%~&;ofTcpC}= z#73{L%pbbe)(pMOI*85$fL;Q;0skZ4h@`Mh@Vwyli(Ad^{Cs{FvE~R+c1`!C*I;uH zJ1$6|)7;#mW-9#iFVHm2EiA<4`TqY5hoi@noDk z2$N`Y2+w9}3>!F{l~wt3ix}hAUeH)5RTXqRSA-z#cxsSdh zTw%Olapj9CV>eOvah~~~x;Wzk!IMM0r`m#d>fJDf19LM&YKJ@bx-S+xx`_llRaXdd zN{As?WY%_Sd0|WyQGE%3j{R{Ho=3~nM6%C>ViRuN)Ta~uO7u22Pte*8f2 z$*U>RDAvr zoe-WN)f`kcqG<*Q5qg`ct{=yS0;w2WuV#L;LobytL=Tt_I_v=oTXxQ~uC|r|JTRzw zx5Sx?-d>%3(xp|Qv;hGp)dk54WknhF1J#Cw{(U0U)Fgf1RnRN_`@c`g+-H8KcE$e~ zq40^l$NY4i>?-%*3c;?W6C=FNn#H)a z^b8EXig%i&cLDN6cTo&Rn`z_5FBhWlwt=WphcT@{L`;kUrPTkc1UE?pr1kOSMuk(y zj%_t#pAZTU?te{3<)gW8!0s~`P&j5#L1uosvtdmh3J1J<7$E5FRbsEQu(Jmd8WVW) zi3tdKxrZL&N#7Gru;q|!rvzw{;9@1X7)fbq*)V^9|6Z6fybwyE`EI}zp{3?T2Xg+r z{e5ZJ(M68GqSNdQt`OtKjjJ<{(*ePTvpYK{ zhfam46b-2U1WSw31?97RBi-uN%4lFE*=q!HYWn;8{Rz1HufC$7&0>csAQoq&1_H6% zMAQlcGc$fzcx8nyF8usC!A4GZj^O@?x%7HPk1jVrm*2gMq65!GL5>%ts?yN2FPW?# zP4Qf6yRsPmUrlVlsE+7m(!Vd4*h;-uS*uKicb}iW+p!Jvy;op^KWLZ>>nFD+o_}G11~TwEHog| z2}u}5%xs;B4}|M=mf*9x`unBl_yVc)tgJ+ULf9f&fw*083(Y2PyYr5ioRwm_oI+13 zr#6}6GPks{au8Z^U>xpvontuw`N;p&%gZ0fo<)E|!&xW<3ho~~A8EyV69htOX({g= zyK$U`?;IHGFo*Nw>cGL2Zw%Lp*VNKd!GSf?;iZdV24L3qxM7w}omFxWb?b6KK%(dU+!y1Q`XvFUs@80v9r)c!5Lb>I{SznIQVk7_tTWprji(m~SYbY9lC%!n9w*gq@ ze|Rmza`N)bQ4Hsxl^8ya&_jZ^23#)jrFfsFrjoYyT433JAf|WM)Iv{*IfQT~l$Vz; zJW7BCUk!p#b`Fk_(|EAZ(fQnUPo$-zyQUK3^~LJ-FQiRD!KmCIW%ehT{O^jPhn0{7 zxQnF(NHHVda6P{1w41v-D2G)j9k`NNpKP)xlz1@MfFI}uOelk87iDKZC4hqHaGyQ( z{-JDEWvz>gGnEe+QPQyzNcD=YE+aZa>Ts>i`a4|MAYlV5AcIVZdO2uALC-|$b`p~{h6bOJ@+wi3fPgeM7w&|Vly zr%Gg(Jk}MWdXXv^=UChVh@r5aB$>@;FVLQIg_ z`=9eEAt^J;(Cnh4QD)S|*Z{x;3=`YUH%1Y9!r_F50stFW8{_L!Zx0ErgOQU8-MuZ6 zv#p0v;Y|T`AXuc77-R?uPFP4GU@!{>?7o+9&WQY;BMQF7)z1|O%HdW0_5$k!vGF2G?h0vW?<;7r~WnRXnIG0y_ zvN06`Cq$b7g#>3Z(g*Ya_hBXCanQ|UKnqSt)yh%i;0T90ZmD8bRcf(&K18Q#OP=}7=823*5cZNE+O>p z-ES`!#2bJr#lCp){#U1A#1<+PuH}CYRG1|D!>Yh}EetD-jbI>&VFhq)fha~JbU|Kd zNWp2U2wrNxvO6}W3$GbKh*s(}PS-eAQ}F{A1bx02egWD~^79Z%5lB3ef_8^3%+H&_ zF9k)$hOHarZ(zdBho2LAA=~@XSfVI$>?OsTYbw4LW&U5R)K8z@{2PwPrw`!_bfI`7 zFthx{w2NxGBv2Ny^f1 zjGSrp%G#%#iJ)K9%UV)b-MRzmXa#KK)3=Gk!VJj<{e#gfLz0~fL$6? zzSpkhh%o#EwCe6cm@QC5JVdTv5W5t5|2{y%&1NsJZUF*|AZ28UbtAk&XzNzHNE@{8 z10VSau@g@HUT5cQ_eYprkwu0@$Y!p`K#~A}JbBkMF)%2|J`!8ky=%_+7<|v2Br_Wt zl{XV-du$-QS5vj@7?u?RE(IcQi@wol2x4tCL41JBkYRomYdUE4nd#)AY=UevRrTIU z(Mm!Qf9H-5jDiWLaS+iO`$7wa5R1vs)(*SP8DDNVbzG@5m1WM(Yk%dj^WQiJ}33lScc#x3zs?F169JX^ zZsL!xSY0sZm=AzL8*rlYUwFZ8N>&*0GsMydHc>^pX0H<#lgo{nYyP)Ol zRgev(`RkU!q~&jY0hwO7TuOJL`N#G zV&=Z?=^_n($0HU=#G8pu}tREY?{G%Vc? zWu{Y5X0*js;f4Sys2&~eW84(S2A%hhOem!5G%-n+FjszSLB#C5WQx_A?Q(q;03jACA3i(oQW3%3X1iKhXaU| zeAw=@uS`Bw&4INPro*=-t^R!YDPZ5j&9^SPXz{UrQaSEo_<;LNBAcuyR3} zrRDwWj&00LbzVR8G$%5{NGQyI9^=2isbhR&4D{n)Ux6a;I=NVf=rFgsLVPEweNE`J z=9CkQER}nY_D$~ii)Zj>)d@U(RdZwKE|>S5-GiSz@S|X9$T5u89A1@XM^JggO(DbaFp`s8YOs zOhG}r(HU_zHElK3Q{ZbL+YKgq)LcheZ{Dd~ZRr1rj{t3|qGC}A!#+uE;u9v&xYOVp z37w#qYxFc?7a$yYgWC{y7T0$tN5^isXo&v==L2RQnE_`WK!L*@1su{g9S%ZAgu{(MWeN3U7Ex&#}ny2i%DR9?m?Gg4qV%;Wrg&fkJ^%t#CkhUmSa zUHL5q?(XleoCb$R7X;ra;Y%Ri7a8k1E=5H}f)NG*Ask)>3>AAaw(1Ip6O+yRZ?2tX zsVOOMkTe>Ibqi?Rj+FbVg+zlz#^n^7kuiDo?vW$|LXDi&v{x{6v2l6j=>ghwJM*A` zbLCB7&j@vSVPVHFb@`XG(Rx)7T>-<^R&tZGzX@IE9>a!b-1#OEzP}fpFeU$>hg)yoLsJ=V=r43mMT2u0`&$}hvbZoBy2o=#nCcp&?}~f zg26_5&Noqf$F6n4#!l(IyXU5RPFW`9EadE%-QO!dw(XqgP+n#s%i(J`EsNs(3*%|+ zoi^tC9H!Ygc>SY{=i9HJJTen(MpQ2D+c_GRRr0Zsk~BVUud4J!I`+0tSGT;I$Iyur zpKkcICR^sk`6tz{mDZtLRFP+5?+@Y4tR>GpXC)o;-9=z@pv7cZ9G`?xK|3Q`pHD!! zl8`(1aOPH;RqWeZ^p3RLXxt;8yGL8s`EAUUqlR<7VrkDJU%2DS$;(Hr7%585wOTa3 zaI*bjv6*C2=4Sy<>A=;ObsrBOs=tbzbh?M?1B2;zs;C|g2%agK8f3C{@_gmTlIxyF zxo=;TspN4+p0SV%O-8=?(L>S_Yb$4CN=(#GwiW4~YA5sbWLU>qa0x6!DN? z{qu{r@2DbB3JLE7V%`bS1~fEwk)16qK|u3)(V@ZeT8^d=^AhC(Kx;SR57Hm}$g}R8 zIGD(oq@;I5R~Ser<|E%u6Xb_dCya{al9KC?f!LORg#{H69swvF2OT#8y#Qd2U#P3u zyqoLB^-rF>0iaqo4B-MM?{05C*r|LQ@{=8>*PuDG1F{Bn^5|q@=+Ui>b#1zkpU4t?8bc{aRY3$fW4{Cgn!8 ziJJxl9!6@S;z!k44m-)>UB~r~5k&<6CLUZuD8MHHfAT39K=5>#gSBrS(yZicfJ-!#ftqxrY;WUNT#or!s7CA(yV0dR}yL{}}HYpp? zG#$^O6D)vdx(!l4f252WFK%btow)7do6x37U1t^QsEVK0Hs9GV;Zj8-vUp~XipEd6 zqeZ(a4$STUrr@t&F6i5{f90g4iTm0oZRGLUY0qiaw@}f{URlZ6+86MUEx2z*f>NR6 zne`BjH>v$I)jH1Y8t<;1ZwQy_%#sf@@SaOzicA;Xn7*-_+(C!x?ib6Fq~xYsc|{aA z*N@C5_pcwg`sCR87tR;YN4F#kSr!LMr^XIwpUz6%Pr6mLiA{dmR-?@SHW6&o(WskCQMz^|JobITn)7VK1%afw87R-P+PZpt6K~rm&E~GgSZ2Ume?B)!E5|>8>1n z{>W*F@)E%U_JS}Y<9-^Tg(5phc21XO>yT5jg ztIZO$0)xU_aSR9#QwXilq7{wVgZZP~Aff~?k3g<)g(vJmrX7(Afu4Y91SqJf85|Z( z4=_>;x92qtreC^rNfr1Yj3`s*c6l1ZzAs`)T^5=T4 zH^08f0fuy>j}WdR(m5Sj*eG^^k+&=dkgv5$@#nktNhTbk7^@Y^%^5C&% zky6#HUCqbO?@1fdIBUBt;dN84pbuq}j-km1*2|zAHN9KIC>crp9+?YfATzLgp8X z-3FU#125+*p5ZX~5i@ym?Onx{pKC-H&WCj;XtOLs6gY6mI1@FO7Zs z=0{K{$gNU7P*Xq~{`%FczwTZEFqWt8|APuy3f%)DhJM#nl6~Yfl84`5lhoYN2laQV z`<#0Mw-9YJ5>DcPu$X$_Kw~{BB|;)1wFOZUAUgMJ!orwa~gXn*;^MAC~ zG%Oniy)J>T5Y9EM9!sG9Wo18)9aK{jMGC4vvO5Wg;Y>zOWiI|yV$22-2R_=`SscwncYK5q0cRj| zz(h=%{U%W}9=VS0zQHYjoal=~km#ubSr?baKwwdk;v=^o@07LoHYh^F^aj&audUG< z62^-(!maZ2C!KTj|IxsI^ypA!e&N^ZHLD&Hb$zV8n7T-AO!p}_#|?>kfo_&3-Z5pq zFSljlzGC)1;8g*@sVEowoTQ(Ka!%{IM}IL2cvkwx|9si{`_Ee^Lc~07+`WI$^`1_^ z8uO=n`JJCIgei=QZxpe7W~Fhuo6EqG!z5$I&8phMw$0dfR-f0!iiZD=PuKcH-;;cM z_g~y%{UAy0^R#P7UHt6z7@MKe7UNoW=1$gZyVuOEg|2Hi+^I3zu%SPs?vQ%1^m~?3 zVagjLf(@P4%2$l!zuU}?-5hLQQj6Kfh=R*$}bp$mT$?miqDVpt|!{)+wuvz?_C*PT2$j-k(v z-#^>+{@yN4tu^1%FBWuoFE09V4+ZpP)GlDD#lG@QAq+VugT_3hxl#*V)=EWr@hp$8 zRamsW)|~zaSNrRm0lslo5PkppQdRR!SQY2qj&!63*VB+r^LI4S3COKCtK=m?pe8gn z@UwIZYVY{`vKzFHRN8cha`J7^hau{~3{5pq!;A# z65TB)2scE?=b^nx4xE%Jiazb1;$om$!*h+Nz|v~i-302zAY@#jf5uf^R&!G|mwEa# zT5nu_oK;{o@buJ;y+%twK!1dy4QkQBrEN5;5XZF&01LvdOpzKIGO}c#r%~HT@f3Y3V>(k%ee!>qKkwKBS1}1pJwV)UAg4xmj~_+=Ph+uoDV1O!=s)DSqTC)eK4q!x?DDEvVm>+L z)y9S8{#f_pqkw zh|G^!Ig(dG5jDhIh4DjI$Er;Rzf1PsOyyMHe_#u#xPAZCy`7O>H_pCM5*}HW2`kK` z&7szcnHpW1b6jkov3S9D)AydJ?AFs>vwBKM@-OgFLd@rP7ZyY0|-AdEbyognUIw5a-6M7lB zc|KlzSkZYeV5z9UcUZoHM2Z5A9Q~-`?JHg9c~=r?&->ThW7Tn8)y5xG`nRMOl$VoE z7A>1p+UM1OGAnXn1(Scg9LLHJMfs#XMp>iEwu2F?G$==10H2QmqVy}HjEz~viYNzx>}=PaSU zLr)3*{zJn6U{tk`d-c0q)UIr0?2EU((upn?&wUw!bnN;+aJgO~#F@WSfn@0f;Su@k zYbqPJS22YAzr;NKwI!H(lt6C^+=eP=^ec`l|W> zEJsj`b$jmVUj+nfp9mZ*-d=9(;;;5Qr2*K^Z&^odZmtb^Ca_Zo2cgKU8+24vE7%5q zCNO=jUkZ1^V8B1sb3L$bx^&<@R~z^x7N6jFte>5T2-;*22Y@Ro4RM30o+0}KI%DE&f{ zC_saHBkPPWY1n@x<=!RS(kH3dRt56MdN-rR=&#IC*K%&kJ0a#F`Iwl*T+UXpSlE%G>wT2JVmJ1xBHgmcytqWMuQ}tCyc5#c*#jn~=VAB|oWZc{sS zQ97QpcF|T?Tb=d?Lx=pVwdKz4(iHypOZvhjC7bl=kEP>TjzhaR_ZU!-BBjOH>!02% z*730SC~jF9b<%nKdS-Na>O$&)(oliEZ1+4+-(w$6*sL%9aj$=b*7##vR?0hz3LQ_E zvE|&nYTNNHtxcW%?s*>WJ6c)~%*{82tRkhHx=ePx&;v8aY{5Maaab(SD8@P-7Uu5-lD zFevg35VA_L-P0}wn)@u%Hr`YxJRi8-!6!&ThO=`7dQ<#=l|eH>eR1m6?SZ-m%;xaH z%Y;EF3=O!w(ubiciyfG12{*Eg>x{^UXhjCdox)5z3*BLr<42A1N0YGd_-4pU$eVir zo6(=fghRyXp!gu4i6IuscH%+S(D1L%$2wsX$X3gK?7Aowh1JlMzg59-*hFeS^~e$k z*TP4DM@|U8T|ij4>YOa*)hx(id8H z6!~*AqwUuTaoKxWhXFx`zC{$O-g1oUZ ze4C??h?Saq0k=2F*YBCQsfrNBB&sJNSJ$zykfSYQyZNE0GXIi>_cP&*Ev=hr=x1P+ zX0o)FD(&>_Dpd7YxJ>&pNnek?7NHl=pl`ElC@qzJTWh1F z>H|xwpfm0Ut`Q%*UT-FrUTKr1scSWYXkfL%!d^WtGz#$|Aux%`tmPGrX8f!Ix`$%W z54KPoBuGJ%`8)EAVb)a+#xIog7;xtx5wdLdlTGC_aDs@el?W6X(@+%j*%<#N(=us9 z?MTAvN6Xv+TJ-A8mMi{M#)sN`@AsT>kmBHNCZ#Gd_`aDpS5qnWa(~)Uml^YYp6gZP zAY3(F9Nz)nn~rl9PnYTHx@K(LB^~wAFx9_Qm41cED4>SMKgzz#bMi~PbliCM2L&$X z-s?B|CgeqT^7An!>D(;sE7(l(dd|~X8f$aV%O>D#(fo&>U4mL-pGl-1!D|8+?L_ET zrnMBulxvR<$;RZhc)fO!w=(2#Q(S=gvvWdbDxx~= z*w|>?=QhjluOH=UUEN~*@MEf+j{7#QBZ`IBbevX#4jwTZjnF3}30_$7ploZfz$X-* z_`=HH$fhKK$tY4mz9&ZU!uAauqDC*2_kpHplPI=$2|9+(&Tg5gX;+t=+dZGfD2Ld1 zE>Mt)%T$9AHp2b-;x@O$2homL(`Hk6j6}jfQ$Aay$fOZG)qiND1PC$Z%u0Lv>9_B- zb8XstD;*OPEwglr=FHaIzT?_4I{*F1p$UJ*lj`Tr(UQ76m$z`BMm8@rGSC}#XB9pC zoHZ_LKhna*DSLT#QkkDj%6hnS{=2E;<@>fhYLvplEp;s|*V4pwo+S8B&o?tj-gy^v zXUjHS!$!Y1>f|aa25hoYY$(Y-9kbWPo_2|OzM~|8E~P`XQwAe@OHnO|&j0*kXP5a- z{qG`N$GCIn$!oE9MF+MoiWW*t1;*LSh~E_pGFjv6%0y9&PL@xXxtDLgM?pN$NUZLKKru}*w8KP+J-fXr=L z28P{_Tt&G(=gO=oD1_4OdX@918#6slr+=ZypEe}(n2fSox8AXcn_;a%4@akF8SxBp zbITi3c}#v=1(>#S=A81JGzkp)x^4}{J=c4B&Lo`9&tErJ!XVh)x#G@m@v<=0nCknL zWdo;>Xd|0944%$ZQBtQ=FJ;ul6sD%iN?A{3>UhpLrk|@K;=R?=KvkJ-PI=_Q?20mi4kG+*#Ux#uCk(y@ZocRG3`6-7Sq=&USkD%T!Zu-o;#!jHhHlD1P? zCXWoi4%SIqQ*WeM+R)F!cVx;f_}RZ0+wMdZ-sR3yr6bjj2dg!oFyAodyuPamJ>A6XCY&#i=W|i$oB>kbLsikHM*ElksrHN5jtx?gkA1jfcLK+G#JUd_6 zVQgGpGkG}e7?bA=&Sau<>czjkKCfF6F-F|5AU%Fanr)+o_#G&yuZ(mCSYoGmp{aLE7qaz zdq$Fu`d0j$9XU94wJ@`Kxd`*RT)xW+*FFC%4!I?CoBg9#Ox>f-75|u{c+SY#>py>D zJIU)!t=!#HuM1C|==s1UU-+}8x49sX?bN9c{ZeOZf=ZF7baK;X%IR$m&r+ zX0O`&752w7YezB$-Q>74k(nAloik$NG0DcxF{bcn79q~3N0Pt0a{=&3JAYWtV`*qV$ty}a zD(174Qrt#`{zPdoe=CZot7wpUQfM@gI{JE_tn7igSx?1lSsl54!SosRJ9;bqdulx& z`6L|@aJlIDFFrf5)dkP+vBj@j`pHPY`7&ma4IIP|G*`4zol!)saAC^QlehuW9L`xq zkJ)(M1MN!?oLF`Ez_sbv{DpcE&tTqP?$VC;D-CXy`hYTbW>O zjHOMHQ(YtaPeP4?-l&fR(aGNfb~D^2M4l*a5DT%LY-?pMeOrr;cZ=B$JH#+Rgv^TS zO(ZcOn7M2bI#ai~ZvVHnwUrQ2GP1EjErN~fHyk2gA5Tt9O0pC7G}sy*>=Q6J-OkXW zRpt+V>)5f4xDp7VNlHoz6s};24jz0%#PFai3smdzexerQ4;HN#!aw@c2tJ9#TI$Es zgFv;A3u*e7s~n5-4n2epx*MAxE5XN$W5Wi*k}lVO8h6BgI%s6dliaZ95yTn)rdVoR zKZMjS;c5*TG9?%;G`>KzWg^YHUo$frL&%3P^eVZ#FF~L*I2ibkKnY%Q;XWreM5I_I zrKG4Mqy;iN$VCw#h9psl=0G;1>0P|~d9d5fCI2&2(&uDlVd%qk1DEF7qEs!0@-r;s zpZ4?gCTFFzXB|X)t2ke`n^V!~>e9lIsk{(c)@SK%KdBR+5B+TJuZgNTnm0q58O{kD zX=PQ7VPhXkC@)t7H9$@hHNI=U;vr3=GFv%zL!I`u)mF%Z*3ZrL9$p&7wZ@{OSU?>i z`g!JPYWl?JrI(lb=`2pmUuP1?pZ{D7c&Wm8`L=5(bjxS*XIeIX?U(L)=yZ2#&^dnF zqyC^EDUqY&+(AM2=kv8aAIP7$7QRh!7~#j5LXzix){2>op4@e&MxQ~8JL*93{ap)b z7miH0V@P)DqBWNB70{@{qAjLt*FKE(MHp$l$trR$Gg|2?X+8RK)9a??WW#NMNQ2w+ zY?`a}^>2`@mWp<|Hvl+4U|g%RG}5@xn6&-eo1#J&w%V2X)thRAM>A`J&j?6)%F`Zh z;&6S@)cClvMDZK1ljl~W#iF0%s|}~OxsG1d&tJ*NNQ=z%bfbv!s$S62wHu)?oU6Sy z{)4q&TAX2Mxb>RM_<|LUR7~qn!2-&JEF+@SU#9E_TBQB&*=T#1mcs{BT?rO2)(#p6n-edet zRbJ|A;YG*6nzt)+9d_dY1=SRrzCN|*d|GZHIeT;^+ITTrXXLoH)X9z$-rnYY&dW!1 zOy0kGRJ*ma@Z@#%Q&H}FF4~O^6)o$|XSc3MZ*=WoSP;3>ZPb3%h{tUvU7BN5epK+n z>{x59ut>homgqJlq|Pif?gmHNTWM1vJ0?={qc3hDY%tZcrUGyID#1IMl`g>Uhlh__ zoNem4V7!0mP%AZk7=ag(lA}_c^WgmB>HmTjeKa~;*Z)UBB>u&I*8wltAERt~$zuy}QU&-0(dg`! zBpWR_@&hz4KkZhQ5EpdAz%p15n3=XoS$VavTa)#IO=q` zh2Hipy2$_OzvYdfPr;n6Yv5ob{uu2suYdqKa!mk57z1xJav9uAR82y~cHh|0(64A| z#D>u0V_E9*TL$Kl?(VB#8=zWi{w4C?IC%eRLHScK{I=(QL5vJo1OWRc42V6D9U$a= zz;PoQ(f>mxLP-xYv2s7kB=#2!>VErO_i`+38(|YcBTvMbVFerbl1tC-ASMPCoG6Ns zQWx3fSohYl3_+)+s}*jE8p*tY~TMe))= z_YER3qx1NQF~!GMbin2PCD+A@4Wx|KQA0b+|2-JiZ|Ev-{Zb7-;~vU|rxrlKNy?+^_k4YDJpjFY{I z5;9AYO_F5qy+b7{WIMKujqg{b!Faam!T`zt-&h$gmWv#5KRoMX_Ti=EL4G zpVmtJ@>PRt^rcpR_?;{~kBm%y=U#8G6F??8z{54>qv=-8m*gW$`~AD$%EoQw4C^Mi zqVovlaiVne2Z8`UcrZpN?|Fna#>aZI&D5kGO6?KUl^Zl*E=WVx7`cM{aG`JLNW3f@ zZ;N{7imCCtSM*_}FYgn6U{+bFb5tbOM!)2V5ENkOX&h%ii#!THdH;SAEL?!`a2hd7lXs`t zU9cAppuV;erv@LjnoVEfDlUtEx^Vj}9#^+NWM#CU46TxgzX>i~lbii) zI=6fGp`aREd?yf$E!Dns43U2TPzo+WQ?2d7v5MZdhpt?&BCr#;s8+s$Df`jC%*oHo_UTA#8H*Mn5~L_bZvi^O)vd; zBF}e%+hnkE-vvxbM)uKSBui<<$tk=Rvh0fGk}xMd0E3s6be4i zC)5d#0j!peItxKa-of%-pn6l>BINSk>ZAb9M-6^?xX3Vf#3 z|B`^I3-(HT42K62uN> zW0Q(OpHf43h3z@5u>JQHFhbrf<0HqF>c;;Pdu} zRKIgap#q+yK#4t42cr?|zJpmha538>Ubml!+6;%4VV_^(6o_BIt>_^S(cW-8#RPJX5Pd?xMg=mLRNYbEuE_?VzJ`F!98sMBky6#OYHB)_*~h{s`Yw@K4c+B|Uahy4vVFkBI3X>n<(YMEUb1gP zLj!U(gb8C}d3nsj_~Z__YQQ;D0Qi1zGk5|SM&{of7P~f`L z`Rt5*FXeiP6LcBl*s{E-W)`IuL$vYp+e2kIYYw z5B3-tQ}G!2mk`QMi30!Tfp-EY0-P7$a4bZ>tv-s{T8-$OC`>$UMUj8tQXj!Y-fHau+}GU1qN&p1tUJnAuO6x0!%l7W*D>QW^FY@jATK}d zHWjuXC=X`>-Sev7yhQtYQqHyy2h2V^jT(QXBo`s%{aD~Z;WB~h;FSAv$M!m`kYVe& zG`1)HNh|D6&mQ(jVlGW=ELUn)+u`xCDKZyXCh2Mvh5%j*6ZFpEbGNK}*(v)@IfQo7m26P0i1=u+hs&=cnU<-N$ z6bfkzDl4^Ky!03v&dSPq{uh?PuW9w!h!Gu;EFvPgZ{LW*Z(e}Rnv$6g!uRKoYf?Ys z%Z5#GUxWKtWq$R~la>S?H)CDay>aMpn>VlPWT2$BV!EicX zXU_EtL-h+$`m%?ZWdy28AM;<4xs47qovL8p;;!^{&{Tnzp;s zOVY8Qym1j`maWJ1`g`lKS`kybdFd00TnS%u5)4qv&Nmsu$%_}5y=-??2{}()y)@g^ zt&`u0xyt-rw3%1YD55tZaT4xL3R+Yh;vy|9d+P8|ysMMkFWOwsLDx3s$moAV8HE17 z_k3tUjn6ULNnUXC+Gw>&7`7ixdeqt3IT1oO8o=T{T{v=*{Zo@Vq%p>2$vt()-7r$R z<2M_p^*)T*ZKvY2+eaaF?eO{+mi8+fi`xZmI{o1>!{nIrA6Z`?8?f-kai4vdt$k_X z!_5`i+b)KWR$cifCq1Q}j+ezC2VY5p)d``_sq1KgN_16p|FVK)!)fnR z7V00HcDcxI7vLaj<3%QBESEkioJ3~UEIlb{+dQu)WRmXg(xW+SkD{X877;TywLQmJ zl>>H;aIU<1Az$~@Ik}x+uHEtD7cML(Il3bQlceo2As%u6)u!}`|DCPV2EtKB8TI@;{C6Y}{4ui^) z)+6~*=iP>fj48K&6nGQOv-L%W?vvnD+=9mjxUDuv+?O2YrmpADV{aI#c(U&x-N^R$ zSFc=b26d_7VXZ`Vr>(`8hzD+C$RxD~meKI6Af%R*96_y^lAUR_l{f;weog9cr892{ zy>jJDeMW{Q9mj>Hmay|wWB)NO{{0{mG@1DyJg-&I;alHD6u(jFEwlb*cvAbg-2bZ3%VU0`#w0uMPh^XSLwZ+aas-dsTP`QGKr{P z;V=`)n(`lE*S9->F%p5*I_QRn+Ypxj_b9&8^77SzXSwKne0%^}grq>=48hqbHvsY@ z1dasHzDt=)KEkCFf&fqf7e0O-_F42j)# zJHmu}}UpN#^|$de}* zAa}=}wCj#UxB)vgHQXex6H zs0_FW0nL-r=Q81H@NX-~>o*j@tpf!)q#Piu7ucrV>ZpzUOIWQ2R&ONc0FWLaxEN!B z{U1XtEY84t2~tpSQU9_q2B8`(h2To9oO72h3wv$BSVe^rRM4RBqWuf+zPPaPmkC%* zj1fRdJiH$#CQJG|1`bB*o%;yXgJ>8{=1Z4h{VvQZ06PHH{J|HBC1f%Q^SHNfo&sfO zC(15g&d~GevUDE^R&>6tgpd zU}FF;y!*01bvxta+Y&(sc<;QgTdRgJ&`|X85kh^R1xC@9Z%MaDZFB;I-37VQDu%2;WlC~jMuC8ru{A=^ z4D3gCi-b-01E}b0e!isjt0NT_^#{2|_5|Po-#Hn;@`gJaPr8?`a|PCg7eZkLVm(UQ z8Y5_Z$ebS|kjXp!?!fKU^)%uY5jh4<&go~fTk^0F)2kJwt=no2wVP{gK40KdbE0lL z43)LHJmtET&t~G?_X;3orzItSo4A4rYPZ!lL0vC~i8+KOUWx_eV4PGS9t6!kfa2v! zOz)h%Y@GNL!lmpk85yx7_PfYPGfYfwX1C16VUAlui?j{Ah(OiUYwzkg?@EkzG9g<0 ztx@(RchYFJ`(LNzbXUVNkmJd4jTLKar0+YmURHFH8 z{6kRvpZ&yiylfoZmP#7)yOvFEGTOyd?pd9)-j7P~$U@yp*7lDfH|_myU3YAgx>JV9 z_>}hP!kotqCfk}>nXL@=Vu=ysGRy3s)^ELCio94h_=0YnZ4L2_1@GCq%q39YrDNjv zd0%G%Y5ks2wRbvgzxEyO`9pISBYn*`@MI=V(a&C^_REQjTU)-4W|Fc#%q73)yA3il zzBiv__dLzDUi;?NywDlCKYRSr2H`EQJR`h8Ea>J|v046;fX`XNJ8#~DRl=>L>(A6E zy|%KBpq3Q{C7N$gN8o&jWwR|ctu1`F_eVAOQ`J?yxQ=6y>ywz5>oV%2u)TXCyf!KD zk&iSJoKrU5U!7SDe?+oA>9SiFbO$%J#6hOJKF$UZP`ycg35lX3J= zig8>vVj}JI%cGR7J|C_lZT>WgnU8?$iOsb4S}fSMuVfp?OQ&E7(_vD@crt8bOuR$@ zN#g&Uqt5jy(ZxEsG}dnPiSv8Gi?;KI-%gD;26)abNW_gR+G;CiYZC$_Z4H8$_|;tv z=in`4Cak~GEW~*LB`$Gd6>I`ualT5P^^RZ%>Ol@T&W4XjyRQM2ah_SgjSgi7mV_be zM=S7fJ|NJV;pH^~`VMPf1>oxsk^I@UirVLtDu+pu@e`J(DVKc zzsf@V8R&zro{cekSiW9wGi?BQKH<0jqJVrh{Cf)+u)aUd`0T%9sUV4ALHNB9h-jxB zWZnsxcK9j&^9%ebSc3rtIjT;)e+&qCc#5@7rNhAZ>nYqlOcsrSB!pm)0MYU~Az}BM zg3DhZAb5lTF(n``FYijQQEO9^J4y|>9$Z{=4b)fV0WAZDA(+Ab^msC(fXVakV2in^ zZ%E=pAee&!3xUXcAAl)5XDc;Qb<7yt4MK&+0n zb)t!dHwcC)5Wok*0TGRL!(0U3+Cmr!C3yC^3{M`zajhyGU6)EoeDk$5oM5c z@DwZ2#mBm=Pb`@@?d2e7Z3;oPTd$ z@14>qtistV2`6THePTApYW;2iE8e9nZ^f)G_U$#4P{r}2!{@7y>$?3oGvJ!vt*CV7 zE1>9AMId00So-?y@fUigH=5AC@jV%ezJw~zO78Q$H!J@f9xI%#fz3g-M^$&>)-XwPtzKqXos5r<( zY$Q_V05WuOoXtEL#WE9KvfXuPU?nb!LA@q9YZm_xF#;clhJx793oi^~z`O=7zQo zqcPsKq?N!T;gd?s1!R7`Wg3z3MgG>sJl{hk5}i->a4(kF+uexPeDQTcJU z-$qUSPb}(ke!+lIIB@<~h{Sum7uW}*jj~-Xh`Gdh8qMNo3?ASrs!Uc6Z`3!U@w+36 zpJUm?sdWjvNd~*4M~^3UH9RPRk_}6)$~rDhQdguUOZ1!YMqeLi zN_o-zAfPtA&19_aaf=_uGVNx6xTVtj-F(-1J6Px9Njv`D20YP-QERi(NINg9Grrv6 zsFv+{Wyiiz?D(YP9qI_Wr(Yfw9qK9@tExNfDpTrCHCH~wr?hR_sevAOF8zy= zL{a*ye)_9Jw=??jP2Y%H%(=;5^0fma4k=G&d*;RDDrN^t*=e@)W-9o;BZF|e1%%r= zH@j(3TXRZzAjV+D-A2R3=JuV@VZCkGBv|Zjb(x551b{-cO^B;7FOR$uIJiP^Rm71} z?8a7v8s3fmi}I>k@27t2NF-rFt|jYv=Vu}Ieb+g zT2U@Nr;T})7iwzefBv&J<9IItI#9TTG(%0gFNvv=#zMmdfqVrIzQ5718`tOAAuo-( zdL5rjR9fkD$??D$oH`ZxqWHtBvG_0=$^ytyU428eH zxLy{3BqPHCEWnETN5PB&OkmJWC9sLHaezP(5Ljm7F0P@Hz=VN=P}-+YEvHI>%~a)r zZA5lt-%dwxb90-0?BS4vtQdsX-_SMcZUQhKaEXt+@B4Nvq*+0+N-M+BLft6@Fg;+0 zfq+^hd$hzLF`V((F%-Z6L3RG(#Zg<03bpAt7?8l0-kaKigx3O0{G@NQONfl=8A+eL2b@c!fNNeXOm6?4+o73YT76ZqY# zfIBndJ^kexDHmit zkP!%xQfd(E&_n(Rw|+$O1%bV%nAhWo2Z4wqjpUBm3>6;&cny9B!CHYc7x{aJquWRf zci0m+Zpyqh^}d7s<(60bMO|GTm)LG-L%FcYEY6zkMpmHF{1m?^_*#EoUSNy4=hdnn z{9a>qq_6@*f9D+v)GML8r`>%*uUF78#iNSjZLOq+Q7DQ2_?xDqmRw1-cCx~ccoZ+% zYNV+6qv#H)D4e0bdF>RFE1BSiEfL?vm&Iyj#KaFnN+;XW49?d#?z;OcH6ELRNPk~q zSr6)2Q(&9%NjQk#N^EiSfLhM%e}u&@DdSrwZjU_Jkr;r-fro#>6BECnOAE>o{mR>x zrnGcig5Vgm2s}gZ8!E}7{yn@LwNvIBFEZUzZrG_9PpzbMIbsL?G7%`TkIDUo+ebPQ zqlN=b%e%sv_$#Mw@G^ox;TV_cqdag_rDkh}idfjxXbtyEvF_i`i;}9t=$XU@JwstG zd2>ePkcr*XNxGk7LhXmbuVLMmY9`d#c2#OTzA`0z$*+sJ8OVJ>=G)`pi786qhu2u` zjA~`_4r}YlavVhUwzVkdzLg0Jxfz%uzCJ5kOj3`(ZX3ayYVQ*i&oC=4Xs>mAx3Q+{ zuBYQ@tZyOZx(WLTPD*@?i!@$;SBU7s|VB z?VBb}PlP#lGR_AjGt(AK&-#A;vdhcv&3ACyG)KvXnWBnB6liSOIc@G`Rwr||GvZQX zC^b8gcqd`(?UAg(16hMFOJ!TvYt1vM9Zjz^6iV?Cwm1)_<$m0sytkXsJw|Q4`%*&DI?Ve4tgTsRTADw#qdnKw;jsiLG|9?LgOrNAs?$1$wTTIwfhQuOQwncu`C*xg8 z-(DZ!EsGsK!gF%(+EDv9TS*m;>51R+aOXwr3x2K~3&E|6J?%okv+7{3W z082a^vM*AB?Wv&&(p11@BVaYq!8Y~u^w7lOb3qdT+`7Q_*@4(ii1aCNn%B)`#34Xw z#B2{yW}ZjmWt}g8m4YJp!5`TlHO&u;Qgd@@!N>zrjiu_WQWX^ypfwPQ=(nE$dY`ISe0_h~{~Mf-dD`@YFB{Xw~cjMdfCd*Nb^57p|L(7~@3 z6yNKW6<(2D4>?^$W5*0^lvR?+c zn|CC9K9$@u>hUKvsZ9kQwWVkbZAh~@B-}9^V;4}dm}<8$KXIZ2LcoE=0(U3De~HAn zqbVC2u8B*I$>+UIya*31UQantlYl8Jz#ylEv`6!2LDq^3KV-dd$(J)Lhdgof`=!Wl z3CZhgG|uvppK!vPL66a0shg!3lSx!<|3`O`a{CH6JwQxFS}8}v1&mMY%}irMNpICk zN2!fB1S=`Kch>7S&oJB{`FZ7mkI#Cw>77fAHj)(PtK;EJ9qvAqc#{oKz=i{0kzjwO2V6v*` zn8YdSAD$plve|mEk@w0cH0`hg{!O~ZS8Z#85Z{~ zdW4CLN}4{4U#BaV(Kg*}GQVOO-l~xH=6ODe{0=NoCiMXv`=PpYzuF_m1nZ5NjG+~W zvNi_@ftqhkGjUQK59yySG#O1V2{oU_#7mxO)AxK%%U0+8bz)s{HzzYFX?#nmd(cCg zyPm2iqpCD{Wu?htCB)PyKSoUS2|ddkT4>mYvLh+SA&9V1O|)Ee0jBa+U!EWZ>g~1j zuvtl0{k41%SL?}?Q9BfK;|5KO5_!G&lx>~Tf#>_LbU5h8k4wu8kA0%T7dcV5#Kkq2aKV z&6%SAfvGN^)pg}l{<}N<=i?q@u$r563mcB9xX}tbb4W!^&F)iV&`SP~_{6^Q*fH~_ zf&~)SE1;YiuxBkdN-N38e_X~!NN(hE;(goCNs z4DcpFuTCm(JRHjbMdooProlGo828jLrr`7Ha0j1Pyr{b{gx*7F5OQsXQ|_My2Om3# zeLgKCa{>Zo!IgO|WYp8-RgfL&UES8?k>av4xn3dte3ZR?uImz35K=avnd^^?i3x-x zs}Ch5efP^D2@33)_A@`2e#T4v#RI>+FxFZf9TQ^#E>&6%bE^8c=WoHc4BfidmI5Sh zqmsSxCp39mX$zXoMG(K*SV#fBjoUI>~w$IoBXZwN>= z5ThYkQSjSA+Tm-^84O}zb0Zsumb#aU6;Crd6`;oZvd?RM=+7G$#$3rwBXl3L#4skX znMij|3YW4xm2SIPNf9Q;Ztz8x<43Bu7bn{@j~b>;5B}WAS^3qw+mt0z5mGMqY^lFv}_$|vu}EyK4tf-27P#b;MDeheE%w<*O5cwC7Q%{@klzWn+MEC<^#H2E?0>{2Uiq z+K%fHe(4Hm$4Am~{202(du@uBe)t4pKCHlyaKqzz#C!FeDUrsY)sBA0GAL(%y6@)y zD8txS8+X(Y-AvH#5iS#lC_0{d1WU{dSmiDH`@SHgd{Ua4S9^fHcuafTEp26&T^H)C*9xLU%<@VjfwrqRHzMG z^#{0%&7l&(f^#la&wal)rW=pW6;tul6hIABzJQB0^o`?}d%D?0V$2BGzjW)kI2!9ZrJmfBG632e#@cIk z?()_W%bXc;L58L_HR(OU{BR4D_rbsQ3CvNF4cPf5Dd6eo zkX#|1W-T`P+*daAcii%Nd`~iaguBoue2{Y~r`8Q;Z{xhXa)QtI?{BA_iHid{hYRjB zZJ*?C&_WMc2bz(dm~PaOteGcwwoI1Qot^o7wwi9CP_L;$+cy)rS(JUW1OnYvu7@<3 znA9m+D)*4FXckbi94;=y?B1{$ONjXJfq8ehuD!n6{pQO0Eyz)2^u)iNWT`8E7MJ&q z4UIk_D_7MYMK#Ff95Wmvgj!jh_a0U3H8=MH9MOcXjZ-n*?j94~2)CPf^rae)ibJ6_ z5u=;Q3cT-c-!r+^bO8t&7{XfN z;5&SZdUUZHxV@Jky8yt+c0d1<&1^LdaYkUjI`TOur=*_)sPRuqcKmQxWMKCJ!AOu= zT-HA`J&?v%I(#Od~)?^sT_aKaUBIOLhzozPpR3#DtX)gU1G6T&d z<4;9}X=>;$PlIV(=lA>wW`6m33g9vR<=2AoTWbm|sE~aH)bKF){VE)5!}u`^VOAuW88OyEAP{swC9doW zZvaV(@V4H!hjY9IWBGlAY5hIeh>w~Q#IM0%uf?;@!jb}zHwP|6Khfy{);{R;pWV$0 z042y@&a=g5!>{3N8XSBHOzP160|L9*R+yfCR0`O^2xA%Wp|GspU;|)1LIOepnlE3r z_YLuYrAqw~q3UN~Sw9Z>DtHf=m1bg0Kfo3S{`_YORb6s&M#)lCl=2=&al(I9)9Rk1 zwt**dc;mg_2gV#V&S~)(?jMzeLq~J@uH~xnOeR`tw9l`0e z#ft}!!H>$kfCHbq3!#_ z;Ureujt;3$jGyBpMRyQdFc`(^ge|#rk*#j8|&qbgT{|_n~lem2HMs#(XK`s?wf^4 ziy_*Pg_u&0(^bPGo~!Y#E;3=ON7lcx`I?G6=y8hHOIciB?QqY<2{$O2KkWh0h(2NX z@>XknV3*M#1q#Z^1k%N^c!!F)c1A%Yt4Bi{4t@L&H&c;a3IwMy8 zf;ge@-hHsG^yc!sTeZN0P%bD?4nz167Tu@rt^~=-EnSI+*;k!p$_%xteVO>i>YfM@ zUwTZI)G6%49P9LHap>)PQa^@y#Qg|;6|J=Bb5FB|%k{R~rhK^c{DhmE+&#W4mJC0= zcu$y;c0SbPfv9Hp|bmg_8y;?-(xQ`OtmM0Xtech^r3;R zoTks;Z1&`rdD6`f6r+vXYogT;yIdA-PWGq-f@+Ga&;@>(dJ2j&<`*nu+K z0YQXC8r00QwG&RVcH8g>QQfw~+v8fW#7DW*hgQ`V^~hmHgLBT&b#(`)skTfZ#eR=i zyH1e3Esq;uo_Cs6t9P+tKyTfSx^67*#7g;s!NxO>713shmVPL9;ZHZTV&@Q z&GaMK9rPfC7$ok$7p6)W7vuTxI&#!X*&y;@qr#}~Jx@SU0aEI!*Uo$fE!`s}=Y`_{ z3J1s0T+7lwFC3))^2!;6Fq!|jhu0I~)FMYS_|C2rD!8c((+f+VImrRl>#wwVStM-%rgqrL z+WlBq29f}Pz1X07AxK={7gJ>Q6+G_)LoV`FmOq`KL{L27+zi$CfnOH|2U7MjGc&uMv=um8pF)lA5!P42 z(3yF?u}fPeW^pawcYfc>H*X_2f`U zcyPGC-_NzfG0o0u*_bYLtwBoB>$vIe0;NO5+sz82V72;CiL3gnqmKuORo7{oh%gnai`)*2-1 zv4~eC^k00W*wNHb=Y;e0IOskOwdzH^6<5sg9~+TzTppLQ_hajb`{u#aNj`_KOfy{h zINEw>6RZBG&LHNnw*a+n#Xg2u*1L`bG_Oyb_4;f_dF^a_tWK1Iip ze7rG?*!hh5BX!0vFgc{-51ZP(L!YCKp-QpFtW`FD7PFWt3r`Wv>o$q06lc~Z6O4kI zYtUkeIWMEH#YRzUBqiy-*OXFHo#P<-l%Y-iQU_b!!F`iYsiH#Rc*y0ydbMkT%LA*h z&4zDQbCj~$3l^WvtZmibrTlgyR?VwZ2A=NABj9>r#A5{NU8d%l&sx_L^8hzPkK(EB4RFX@=KVUIbcHubN!q5B*K=^@|I#HjgvVbKZ5w zFw%)uCBp=%%A`ujw}9%zgOWBH40fK{Rc|gm1MhYaE}MF-2B_|Vwu&2qvmIwOr7YF| zd2MkAKoB7;Ed<#=3KSDzxafdOeC*QT-Z|h;YlO4=wYwBGal$Y-GGYX8!FJ9mun8e| z7swSFD0)Pp@wprqpEieZ~P!>ZzrrRZ-cn^M|+)sfQAx zfB{y$m)Bo)K8hG+k_RhaML7NZ`0>N&Bn0%Lt!#7I0u=g80_C7v2QGM^^g@PQXi+2f`sE-@l@?k~Bw2+}V5R&O=4W_0IG!aadJo+n zfH@pujaHyUs`A{|n4pt2H4WJFwy&Y}?Ge-Zxxv6N4E3x-D=NOozPHb^F_Se(Hbyz`VL zF>88+v`xj)gcrD$ZgPvULlY-dZSAxhwLx@$m>l@yGfTd zcsTi(HZ&M5(;8DSw$Y)iauy3C;8@Lb=J#r~wniGQ2I(CY>R9k)w@JPt?$@+9#d1sT z^}<%sK}VAV>T;Y%2y_I83h%n7XeaLy&ZQ?3(!nlHcGilp5o|KSuj&0&fFebFsGdCP zL<^F5$Zj{$lN^n19pYV#`gRtOR&^;D_pdGeJmSqLUUc( zj$U5e|AytO8CHmSLpINjJ|R13lWL^1d%iN}1{&RJ8&Fm@u+g$4n zDQsUfRG-trbcdYT3e^JUlC{;Z1o1)I_?RO#k(5x`cD;(fmSNYmqm<`?q zSqp{AJvM3vk%3jWC{PfgaR{mm5});1$!&!jQjAsk{?BU@vcUM93Qn&^Jq3PHf+Bfg z5eoSC@nVkfJ`qBKd4X|+uS`ftFp1!3{rYu0Olqtit^)`f1ESaG)?pj1V_@(K-b|+z z6!wx9zCv?w64HV24j?>bD>@;Ql$1nk2{{R$G?%%+2ZEr?q3OAK^L0-+hJqVVV(@SU ziv_au_Y(-V$_EDvq^Jc$OB1*w<2VZ;w+#wcX`VlOh61zS0SNABOjQj8)0pR1NTr_y zKMR5!XBIS~Mx${i1}75$@`OaL!8k{Yqz52@VP_aj7A4}CAV?W7tHP}8a=(~@)$}3M z85BGeH@P@4ui$SaYaqdi4J-u1i}VNSfo0k zvw?aBxF`|ZA@P0t_J!)abW&n~7p-;^fjA_DxEgpL=36*q!we`sy*4rbHbGqoLYOkj zK9slbi+V&zYqHLgz0YK1-PCp8f3>s&OPrj+QCQ5im%~#0{$|9Z(K{WgkLUaOEZ`qH zH~BnNz+O!|PfE?DgMZB(e}3P^%v5`mm#s&sc)V9ewUhH>n%**muHWKr&<+<)|JX^k zU1ngBI9%SWY$YgBPo5SMNjRd%7N4Sgh*2Sxu_+ zIF?-BeMPY5nrVP)MOJf{#slrdcBBkRjQGTE{tD7Ub63V0Cr3aFl-}8ebA8cFJS{MW zd2D2RGi1;Ewl4QqhE~T6jvF5DX{0*KUN!UB5Whdpuq~BxB5Pf6U_<7>!M3S~>awHD zqZJ988LibDSQ#Nz+D3r6u$t{qKdvp3o|0p#xF5P97e z^Qxw>j_BAQUCgaNv*;vW)Ziac>v>$NaAC=LYEErp+f?X<+HbNmMTg7jg+*hf+2P+0 zUwrRX0Iw^1fApnG*8ozfG*+R5Z#5;EqX>p7UV?{B1n$(TUquLvi26@?c*(B|Lmc;Ds9(UWB zSBq)IwSBmJLBQbSg4CyvADN8Hu%OxHRQxj^{$k~+P@scqQ+?;Xk+5Mxlu`H7VLT7F z*`H;5TU!aBA*n1+?vU`?j&nHY&m)THkGdi%DsT7H{^xjVb;g+!cK84FBkd|2PXlsz zkC#>9%>*H9IC7j{@b6fi@BNeR_9?4~<5HkRKkyr$t5AYE6F!ElWXS zDG<-%RE=R71m}mPPoXdM;KYjfwvj>v@_!^%lT{FsQ%q}*f?o)Vml*^BMt*;wDvAZw z3WBqM6GHVntDo=xlC*F~uf29miv|#i>oR*dr^4Y&$Jk$%`Ifjp`CIPU)Tw*eayYS)-f=yU&kr(Iyijz z!X7G#KE+yW2;;d^;j$#SzRu&d+P01_42)pUt1ZcHmF!_Udjg_42XxO+-t~GE=PftD zrB{~$ip6Qpi)AsJwz=(W##P;^OY<1VvG*IyfrTSIn^UU}=~Ja6HpQ|Wt>%U1J%otV zGifa*F~VXDi#}DoB@*x9A^NbbD4QEhP|^wpZW^0R{m21ijZeb@}|^vR6d z|7aWL_;EKYC*sNgfyT(PV1T`470)?P5leN@PTK3!T+(&_In=w12H43h>X=lDyQqDV z(?4}8X=UwAYS!XxMab(RmYmOjl&!NUNF+`zi>xoQT|+}Pf>$+Am>~*4!z9(yu1HBRK0Ffn*7Gc6 zEQ3M~8g344SZ14Tug!ReGXmimvPUjKiu-uO^R9Vzx$3MhUmB}$wITP49^QwYc;t8W zPmEbrOdK3oo#llE6$t2~q3;NLzkVf0Dj9&_rP1g<103Aau$r#M75ZiVivZ-KA0^4kGC zg~NLpN#})F2=-q2$JPiHz$Yl2wzY%9!rXji#IZAMXD{SnEU*)nN2^}P#}8ZRL!{F0 zT;mXEm0;Hb3Bp6Bumd{w!p~G{7RExb&BBlexqhI6q0~Lkr2%aX9W6w3L1F}A_W_$L zo9CiI1E8i)iHbVB5dC=YH3Tt0p98l8Y^#M|d@GI8);LwRVHt7+u=^SL@_NSLOY4j3 z(-LgZ*Wg)muN^60FYTIX=(%vaaV6TsT^U)X@RDG=2oqT0!rNwJh&dDvfnGRaxB zqgI#bS)le}qi<^DO>5$l5QDDn&(F29tz31g#v-9aP^;kh*4Cp<0o46{o9(>KgmQ zx;UTlM{3NxbaKj9W_d*}G?_Vkv(EHrY-5QIAkg)E<>hJdu9N%6eF!H~0qz=Uhr zTpY#t8F;%6@B9B`UGOgsLSyur|JATS2fz0A)#?B%m(BSnO~OjaQ9gLbugdSZv=f zk#*_Xg^foOICXE_3ljh@5L~U0+aIJuA?srS<`7j)$n}FL#xPLhngQp^wS=wmb$)Q1P^k9Mk<=IXc&C#Q39v-s1S_uPFZm9035S3hypbFrC z7dWK_+681SyE+*n#3TJaF%Ux+dY3J?EH1L&9D?5~FhokmIW^V!jAym&)F|oA`+~fR z><0@z;)w1;Y)<9XRRifpx-nlT$4v#B9@3lr5k9rpX8uu0#D|+J!_+}LcCRM2NBjX6 z-_nMmntG&>x}td&#r??lJae5SJ>)BRhg%dKl3uNCNynd#@;~-*?Q&Ep2kNQIx`aG2 zWP30CR#sU`=@0*@JH|nquTPy-_FE-6oZHzZy==Or9o7`#<8%_-csV)h04faWH@VK% z&3<4pGoxi9P)UA&Nb>>I7KU8&Vzerm_4VT;{?8yLmsqeY_Gklj96#ait}9^t#6M7j zAT1hu6b=grh8$=Eryz2$iA#1Y3bt1r5fu9mTmasX8(gmGP+P4^;^zeIU@_>}o_hP* zYU2?`qaF6KF?cr(>n%ph1#?rXuY}Kh+!iU%uR$S7IA!Pij696bcQ`UYD2fa=Ghf}i zo8qQwKGha8S;{!Gik;8KltVe{G8nEgmA`-g=hKYeS@;7S9X<~2CR!#b{-*EsBFevg z<=z!tOs6Q>tBY7xwr+3lWR;RQir)2Ze%1od1W_fS#t;3xc((pQ^ZF7-Sly0fU%4`) z(8(5Z)?8Enqe{!8%cy6Q*ZaCkPQbzX551>c`)h}R?9sZmf_eJA7E=W$Dnv-usJD2y zBICFa+5N=rg-D;)#RzUEZH###o^kLZ{I#;m8V!>)Lh00WLh0fz$b$P^RZ19b{HkYF zJn#qmkDB)Q6jswfht+^50`4_r+y;JNIN<9Jp8gPcD{S`4qVV4IC{{6yy=c#R?r}8n9r7~8?sGNyUdu&(B+`{|jU3|;4 zmb&!X+9Mz{7-$131XzDos-9SNBY=hovYe(t)L_Xu&*AQx=p1o zx#|E?FI7hF5Q!<=XJ>x~v;UG;+~Q^-?{@ZZ+plqrkGfW^M_y}on|_^ICwTqXzUf)X zp|<(?(?v)ByD#hRGRJZMciB`Ud}c+Tm{oVHl%wJCsQ($VqV|)6?(A8D^u08;28~(< zdKw{$`phSZJ>{=UA+#0hNNXxQ=75{T2)iEFTX+BSil;4rR=t9ES*l)#wv3h|QXvQd z=z}J}33RRwkByLH7int>;mrE`d${BeK$;z>KasT!1emhF@d}KC!l@Z*!509<(C){> z>5;)x6()JUr)-C(W5NrOVYa(p}4p>_60Zt1%vfj%5p~WZ~*1?cLEJ%wVM}Vg_ha>erLx60tQdS zOWLZyy{^p`38GAkP*ibrEK>CbITezX9`J$OVf)=1w&j4xAlLNc3E+b82Xh1T87nnJtN@m6Q^kS9D@)BnL@j=1z)2 zv^CKIU%dKBXi&HdcwMy>GL_MJm9UpC7{-q}wswlw&eGm2(}W!}ljX(Om-@v^kXZCW_OAVQb#e92$ycH8R4kxvk_$LlLd`V`=5Y>3iPe1YZ* zy-#D-fKMV$Bu=%KxGm#dm6gzVQ0vR*QftU+02O z=ff!dc1f7@jvTWYweE`5=G#0$P8nK{zn;Ib9xpN9PCLvPtQq>9pK&L&uDXg^DXy@a zB)e9&q-b%l@nf0Ua9y&0LC;ZCB!9Zw260~X7hXqZ`@Uctzb6TgF2m8C5T38>Uz{(q zt-^`VZ67*z-HtO^j4fMw!{u(BEt~8O+I2$#dPLsp-7?1^)LNRH;NR}CF2$pFZS`03 zyHhUaR&_o0yxbo=J3lSCUOEZ11v_lBbWa zgMwRvX4Z z&RSZnj~khB#9{i1jLWjXlGN;TI&s>%^vy*+`wWAbbWU`mu>$(QB-C|wxYBsO2{mfl8A)L2qk;3Y>^LTG!$ipiXtOL zcCso;LiS4bmX-NCuDkoW@8`Mi@9*>Shg5uA*Ll6q_j$aJ@w_THLF{gR-4;F$jmB1$ z?qMYmR#>ixZ&f{q4m+rkhOqil>+jJtV8weDZWDaKQYmeA|J2m?LW@xDMPHip8 z@v>3AH#tHo;`f5pg=w7AnPZ852m<2qUb6ETwJ=cd@EDJIfB*jd?=dbN zC8#mJv$L~at#!jQO)$sc;N+GHr$dS^GBO`&PD!TM9%rD8mho85s+c7ZXMtlS0Mi@*W zhAa_u*P5O^P%uP_+wc_qj2Z9x9f2#I6Zb<4b9mR@d)kQS966odI800x6COV=HTdOKGg^q4M8O#W!|vavZ;Ojpj} zdQRi`5i=7Zk8Af%FYMAa_bzvN`fkU3->Q-VX}!1mRo<7g;crb|&q(QC@Z9{_IyazG zW|IB6O;xXR?XD7rp!~*Xi>s0A6)Gp%sSbEKGo!m~sp{1pnw(vb*(q^meae%?pY^&n z6@A_^p?7&rw`{t}u)m__$!t<8$4!5BMl!M^WZA2gW?XK2HU@lNk-M8&6`Vq4)79QM z*HhMK@!ndrYx$a%heshxvc8{+Un z`O2!}DjvM&&N-FFUhyUnF@!$Jewz%j;VSW7>1Q99@$B@5m0)nAfn1T;OxPvBF=7Uev+qBZwzqi`}em>E=Go3iPPKo`aEr}t~GCLj2?+)k8{bz3<(cb z?N4#MegHWy<$Rq+*{Sso3+G~j;&{k2 zU%4-53}3nOMWeme!t$IdcZa8y91XVsX_@TG(-%DB8K*ja)n&bLmT$GNJgUzxyELZ0 zcxA266N4zqpjUI&+I>aBCGiC_xD2D0#;qt0kJawStYa4`kKnj+AoR-_)>3IayR#b0 zlNBbW;v$Z8ADZZwlemDi5I&t3YqVZ0Fkh*eZ7Tw~0C6i!t3T!XN1b{q)?zy6)@gG5 ziBZAN&-QH#oT+u^;N<9R*?#`Z6*_=nW$T6R3$=E?+mR<%uZeCaj`8AToC#aJ_+t3h0Ms{8P8lt+WzslbKS!>?>QiSeMoP@f_NFB5Hj8sd8 zp(Zu4+_@79N>Rm3EkVBT-N+mO#kzV<6IdZqgKdd$>Lg}FXxUK zkTew?S0}-d~0bs}Td;&IX_fypzp3uEEo;=j0unkiH5-(DWk7#M#FYJJ6 z-CZ?VqKZL|)T*}eKi^zYaV`!1Ks}(H!nQP{5h=}CIiX4or>dfan)3r&m(+2NHB8-B#4`m~+d z@FabWp3pj-U@bBqh%yhX9-t=_kJDBaQBIre;zk!K-M?S&6nsb3DM@3T#2!G$$E=*xDZA#GE=~I3?Rd7?@~da( zkETD3f1S-ld)bPNEc4Cy)$R8KGiM_22s(UBWBByoITem#-#Vkuh>XGe$H)|Zi7f$CX~e6 z5d`W8xFR~mUUDnToi%`}MUJ0XFAvOLMPLtL|D8Kc;TJBGrf+4tAQKjzypz|md1@z* z`^W*d5H0F86LPB#h#9Vg37#Om2eSTY?r!{L3I7KbW7c(~w@Cc18XU35rc5>y+WF`_^(y)@x-ZU9IfS zc8dHtl$@^<%j0pj$tRJPQgF4F!o2^Uw#H=os@n##TT!innZ}(8D#7$?)=1k9&xWuj zve;N(wLCmZ_wA#1Z0J<9+v2$J>ZP?_`3q-~BSMb2%9AN3L>2d@kUoBCFv5qv7IusrnY;~mglY4ZTYa~@ zpRdVZo#P|oY~+NvAT#b7O!xU@q~Pwyvz`6m7kw+Ai^D60LRXFXOk`t3A77=>T43t2 z9rb5D)#AL=`>Q=NG|l`CFrwdp#1+)fB0sPdvn>uJ3&l%KlyldQCKuE}XF4+qP(wl5cN9&P0WkH%L| zPD<)gjr3yfrRuBJ@?mb{Wc0wJcs9E*uOS0!F+nE_w#9Zd0uGy*syY6x>ph!I{cq*Uaz07ZV+W`sL_ni zd-TC@35`nRtn2t_#N|eQceNk|y$e>GSEk#nBV^}YYnV%+SeSr=a>sMn*+{zx%zQs-Y^uYlnaz?BBqGH~m1%XQJ;^$~zmjn(ju3)Ihwn#~FA}uwb z=ec zY`*sJAqUA7WM<2z406OB@&6J@-ptL-O=KhfmWAl)wd)Zf4(snJuf3;yy{SpZpF~m+ z>^tazE+QvONAP`BmBO)O>+mZrgxss@B|^$ynbk3{_MK1aP#pV?_y@u(9%fHFckRk- zY_$|4Xxre0!E+*j^T6W{O_v9D@v317Voa3%-Ft67WC(6tnPtdc{=v-gF>jn#bWjjb z!K$D8!k&qdntp!6W7|?qq6#}oPkysm%w$rS3<(J{s*~eYF#h#vw$(iU)~~C#PyPzp z=DDhNZaMM6{9+?rLhe+bw|>;o!)LDdtQyeA#pheM)%_;QIIlFaGsixA)p$Wg>{SEE zHed)>6^XEK)YmW`4pVC$X6ab2*|X1^z94r}ooHB@ql=o0aCT|q?|m<$F(z8vu~IiE zxaLOdDb-4Duo(F4y5DWoDO_;SE$+aUnmq7UZYAE%%p<65xzageAFm$_pDXUFdHDvc zlOaR$8fmVxhWoS>hArrY5U=sZYuPQ#_#DOL&$xZCnx%3cc@z5Xoe5$6x)7|FFGeH| z-XT{rE)8H_8c<3y6m^b^bDcRmCkFXdd- z@ugoG6n?((AsJbo7XRL{og)s>8I;}yQ{U`e7j_-n{mK22CnlVoQV8Hy~_jj|` zWR1ZR>gmn7ukdtWwar-RZbv-V1LhwZdq-&{Ap~AKJIBpG7ydooMzwt8^@qK$58d&& z7LH7Yjgz%=VI^M--(i1x((ZOTFq=5eeYD|qRFwy&3Oi&6 z+&#IZb}Cg04Qxw`8(rztae``+dwRQ>zUa4=;MT+LT{79L$@gIAghld<@?lN=Y zXw(jFZga@x3Epg=QsPT-napQj(asX7@un^J=sI)hGTD$OzWx5*_RxX?foSP-P;~8C z3}opJ^Vao$ZE-hXdqhsB{*9L}onaOhl$t7e>GDjW>|zQ1*U3^|x|-D`G&FSW*|hGH zydpMD3=f8bG9BSNBX5F`Gm0So8uw+ zZG3s8nB+0}=rGHAr_;MO1x=tJHA{)+mS5W6-4iBP5kSkyz{uELLGyRD4Ipx6Py|Vw z;(1bn(g?Q?)Lg!4_Sr7&f2?_7ERx~zl<`lE^!6D`NQ?)m_FD%W? zul+8oaLGP@;llXy7ho=cqOiCJ?QOlD6kr#j_$#MP_uvoF)Tdt)|UeH6v*4wIrerH2Fh@kNV_7{uqoA_8K-ub%L^A3 z&e&ZV-R0XiL0f+6IR~acQ&Xjrw;nr* z$47T*Q0?6NDvKd;uXp}ih$h8pSO8h4U$#~m3-^5vIi_A~F+Y>O4?g&!$?4;Zku&p? z^GKOY)l zWJ0n<+_I&ZzPr$;UsNMUFFx9oq7?$a!A5d2`&>$}CurVqKTx?==pI*=6vt|~I4h6P zn8fVtjWIC-)Smm^@xPOLOVK$$psowgoVuu~FIugOg$}YM%GI7**Jj131!xMecMFbl z!^Va<63i|e2%yh)r-|6}Br;Qgv8p4Q<^k!a^={<@=A+|@0+fdDN>4xINyXz2dHv4H zMUt=DHp-D~0(TTmb|*^woIDua85{OZc5L%Z*u=YQ5%tFnWOz0`aIeYva8ILPWsYPw zKWd3LPn}ir)gsufnJsmGOw{$4`W-f(@HZfRvNx5rR$zHX^sLtqX5Wc{n+=(^LQ;RYtt>1-)Y*P?5A`8C5 z97@l6QNN+q5FT5NqrI8N%bB)@)sI>|fAwk-oC;3ymW5z;sc*c)jcRbZy+!v#Lu|k- zuMQGV1>1~GAt7#}2M6uuZP~R_19`uh1(4jkzn5>u;5qgkiMKC4E5hl+j}P(lKjR|M z{v{h$l~$b$O;n02LBPQw3~E?J56(yj+`H8k*12rGmB(GsuxScM%3)y+o3uUOdzLdU zofyeD^|xK~pCqn;h~o@xo`d`TU3}L@ZMmpzCZ*g$8aI5ZINfz(;$F`)S6M_f5>=X8^e=M#?4O(WAHV`p-0tN&{%Y>vlL}@;ByFn)LMP zy78jLv_Hxk&($A{;+KDTK}XW`#x7Bzov@#Yj4W|EepXjExX^tLCJwI~yMYA}lEJRQ z!9xRP?Jt%W=O8&cIXi_SK7m4%FBErsqOo{|1s*xGQ$SWk?7l0Cn}~WqjT2vM3U*p+ zzrCvzxEGiprCL5;sEB;8&I>b)-o6<26^`+g{mq^I8V6SBki zBP9aVYtnKa*rJY}k11~-lrY_CQ#u|uuoAZNdQSUd^l;o{NIs+ye#N=< z_K8Q+_xOcds0`PWqolG8y=w1U73vHKJtjkT@t3%OajgY~RG-CwX#A40Ov{iXhW=z^ z8}=Pq!&kt-F&_3dIe#^MbN8F@+f}Jb_bc>Ou$)-mw!&MzqbfynOO25t?*4!)>-{-b5@1f1g(}Fe3)N0g+FWEE9 z4X;biPF@QxYHi0OKh3R24nG#(^i)Yc&2cpH1z_qLR!iSzTZapCn5_L#9(smaL%x`E z09{`1nZJJ1Gq#Q+%E($&xW~FCXu6}t+bm`L+N?4~%TPYtb|z?N%9_l&C)V?{@Z>}!2LJj5b zPo@&X5#Y0EYVaintX11z_T2cNUz*JVa&l-8gK`{u+WGT=`E{Hc$alCh@iWeVj4V|@ z#-~R1g3RO5;_0tm9`lAgWEXK5kCw>%CYzd7=g+ibH+(LA;ZuV_;H_Am(D3lCt)Ixr zycX)R5cc_*g6ucbBET~eUYE$W+quU=;mlp~ty`5=xp8fGO|P45^XuRfyq;UM$M2gUfH`hyK#8NjqoJgS2(@t znjdL}{NCR`eT*4l-mhMX5)5yB7X)#EMos|-N9*cpPb6ajG_FMwdV1C8&&O0$Q0C#2 z|5m(ck~UiU>LxJD{I*cjl{(nl+b%CSNl$-Kfab>fP!cB-Q_q)O(CTcbJn|;JW}my8 zq_bmE<8XWUhY3>)3kCR1p(_{~8j7v}R~0H{q}uZD+O?jCXCyFm;7CJ#y+8U zcp4G089LosWx zXJ*vZWkJ_)cX3BwFYIJ4@l=)8CE3Q@T;AGR%z6CVqi4@Pm`S0q0(ijnC6tHApb3L3 zGmjkt;?QGJ`?ScmFdCLzV-wU7eziJR6Z=B`$dS3+ z40rFB-G*P`P9N=R=Ba18*ZkRUmwwH5@u9lY7h2To@MyT3u=`Et5&v;@C$om2ek#Y5 z^9OayR9++F<+hlLKv>@k^w4|12~OL4V0L8Jb*)yqdmmjCpN@_S5!R7Pm(DNWuG$Xm zRCS=*+)%?h#5t}dqh7E6y+Qkok~G!7&QX#D4*yDh7?4f46__c|v6+`O``tyB=&-|z z^;Gmh)N196@>?16rC1h}apPcx@yFO5pKzufY{6-likXSyX4wo~0{wlH`_M9612IY%Lg&1A)8|p+R;d zj;iU6iHHI!DVbC4ze@2$b2s-#$(YZl9?)>dsFL6!3O+dq)p??-kci+EMKFwEU|P+- zsn+xgP&Z^LM1C?ME#Oecw=*z?y7R@YTMWcO2JTCx(6^mFbEeKB`okr85G3J!(rGMp z)I}+&?mwLFjOKM-pY9*uc5TaaL8v%PS%{fXQd0lv$6(+)c5b31$9qRIk*x$zL&Tdm z5`+{43S1+Rbnm3aF#PU-;?TVWp`(!;vFPjXC*lhUR~P0D5Zi(StV6f7CJ4( zo54f^#XCB!He_f~QpN4SXU{t}`@*2x%+=_t2tJn6v--x|d*58RQS}dDag(|nfkc+Y zS1}UWDWyx)`?gWN-?}NNE#_NbzLQE(&T4&3Ow6CQ2RM+P94)V?faf@2s0EoI$^y8J zREYF;FOx-uBHX*6cH(X$z1e6}9AoUw%R5fGTxCSEI&VMem!qq`s=Plh?Brf|hnJiO zwz>)Le8cl}QPVVeGopZY&G+ux_s%G@8@|PF<5pgkwVTN>Ks|nQb`A-tG_F@^SKN?2 z_`ffE-e>ttTmJWJV5`QnJi2vp2E^e#W=b__w1I)qOyoM;{eEQE%i5oP(MH!)E?M}; zT28l(>!aKt@(#A_YN{AjCrRi;MaDj{NmQ61L*L{rmf`dU|?xBjYhNl+Ilb3ZiR%eoB`X#*k0N{n2f? z@G4#7rTy`kJ%VGKl7x)gv&vFT0Vo7NKLvX2<%-?R8we3A!iDqG!;~1> zt+5GAAT=EaRoD@a&Ro8H8Lc|MzH9ed0K@cyqBXHHH?bWYVW1@B6Ij_CAM|oN5M3wo9Xe0_-D&k7Z=r36QATQuD+j6+C-8nR4;#=1L+etV^0ik zsdTQo9}v#jsmG+A_=#}ZD{g+@%wHP1T&3Due=qa}-m+I~>mQUw{KTXFgOLK$) z#?>B}p_9S0681k>@a52=kMx@_F!At&qPn2ze+LG(etO*?0l;gXJ$tAOo{t3g5s)$G zRLzeYz&R~D^-cE`|LS{excr&r4wdm$zQ%hd9c?>!qj-G5iM9S5FtDF8u^s=x%B0`_`70iCQIIq^#>-Lv{h@0c`7kzBtZf9&i-3ZQ8A{aEy@X%rt28 zMyd+i^dn3fZU_l)-oWBMCME=u7`j16NcRzg8v6M0wvoC7Z?~xrcecZ8 zwc9O!GgKu-MLY=P!3PeHIApv@$;R_WcogizL;(|4xHPtB=m^qfLcoWFRE{Z6MbwtX z6KBpaVX$rsM?f~3059Xx(o({J74PYNZ$V!Sk2KC`1l^u+@P}inyC>*mvVW>@5Wl0a z{{@m*3H9OZ6s{XQW+g9;hB=?rGl>QUc{Hi~>V)v~_f~Jy;$q6nNZ)V6isSh!24uUW zr;nM7Jesg%QMIloM0@(YQX9_ox*c<4>nK)k}mfOR)syebQVM*!3%Z%d6` zYnG2GDfJK!0s8vh4Aq+UKU_VM>T-86HR4^v?E~`IAHBUt(p|hdnoCP5u*T4?28M)O z*Qr*W2uybQ^y$<5RMkH2A<#i$y%x;gUz&4CYT`#xB_2S4v2-&U&zOlQNRVELuqyY; zwj9JN8iA=T;k_b_;FHj%rl#8}?)8~Eg2`!V0l$%6>_Rl_N_|=xgA@Vx_$WV-x3m-l z-=wRzp}nLKTM4}4ZO7Vjd3tcpy%`+LReuE)E04F?QjD3El@EsjXls@$l(%hyJQpN+ zv%pYfI5jeF3h_sqLv(>n1y>J1QZh1wUx5&fWLmdN!smq6oCD0qZixvYK8F`;+^t+n~#!_sd=ycp}Z?tawE;?G3sl^1>SEq$Nu^y z9av7^v7o`=Gkhh*cG;4J>~QyXzpqMkH$QC9i(&+vSbb;rHrKx5{&(VKb=-q=%14vd zMy2H#M#ph&vGbO-mff)d6VqO$S^aMT%CSQ`3^6pW4e#jIND zdpvGJboaAHE(~>~WV_tqy8@onWTs{!k6ylH61y-rwQ$p*8U_{~I!WU~J%`9)UHu)q zfJ-`D>K(Fp+zQw3e?SS%iN>yo#qLLqQMVs*1g~eh#=w?bHUynn%s>4m7-@YMf8KyD z=O3?YXcVVRZC-I$eQ|!+9z)02PNCVcUodR8kD^uS!4%yAb0=182#P;gaa8&!^@*+L zKmq+%9zn<(34%IkMUJjbzg|$?x4}F$Hg>KOM^ z;JON<%!@c83v-0f&4fu$QKRy=g*PlCQJYKFgAG32bTg>hge@%;TLb|gqA9{Simq70#jF1Qc`%LapZ{x5p`pK zb-YmszvxKI>y6vDZu)O^Vrg%Ow45=G6%Ej?y24?1kR0%f2*mXM=w+g z>xu5(-h=Y&o7y>X+a^BlNxz7%BPZx}qYL0dKYH1>4mC0)JcRk?`4Az*d9PeeFNL6< zC^&)Ta|S!6*4?CqQ6$)=hizT<=iEN-{@Fq=g)vXdcwg6@&@TA#KKooQ+$iXmj(T_$)XkvfVL>u#TaQjL7|2jh^xyB%)lL(gKEL|? zCey^iuvw|Br%iKV@dyKBdlk{x$LI*Y13Rg--AtMxT$z34r`(l%NK*ds;xO{y_&>_H z%5K)~53YptPodDOD@Kr^){kFvSION;*;|36BxA`u_x|BySCUZ5`FlNH@ECizjSpCe z;sGrqRA_cR$3zbvY#zpI5+EbkX#W6i0&Q09 zIm*Wf(MV{_)#dR>N19aX~B)M)U^%~Jn z$I$Xz`@DZ*mAO>N=>waGOJ&ba3BC0%o|1?3j+)whDS0feAjo=TFvSP;{QJ`Yey2gU zvd5J9QkQck=_cNW(Mg9aow>KO>X`x@g<6L4E{^OcqBE!OmrCgbHN83=9LIG1$J*V2 z<98R6Dap=-tYdz~k%|6)=iZe`4+Qf_J_+T2in&)?q+P|3Ibhs|22J z!Z5$u1cD)3TVX7DalwAgbsrnGLm_UR97=hn12bC1A~l;`bNPXRZ67XF%n1LpAz@^k zIkm0rQ%wC|YwtgwII0bbwq^{v?P*b2mMeP@$fM$AFQU&iUPEdOk#Ep5@o+5s)NaPW znA;vO=@mF(KWf-L_N}8Ml*p(Az(zL%QwdB1xuxd+YSR-IuWg~mWRV^uQN#3~@H7RJ z#|mo?Q)wO?&RqLrT)2jalh?Lw+k)Ru17g{$`^E=~_2~f>CNjdN8`1CVuy4JZ)PWwQ zsPvEG1IDcF@g$M+nwpw0)9cKM0P5lekyE}9FR+t>0%<7aOL#54QHa5gbw*9*dU$xaY$fQxI4MX#MG_a| zQq^k@Wf~RigY0v&#%dKAC`HA^6AKEse)sHX(vTxlTr3A56(5AzE2y_!szn<&Y?y!u zp#*2_dg6qJnMkCh(~w>-Y}y=)Mc@hKHnSV21$6_1&c8f?FgziS&!+fZ8KI|l6JC2e0%afy`qN2F6vOfChMVI~PRA+vuDKTPJLB?culG3x9r=B3y_g+G|hPz-wW4 zto`Qq@89)FN{WhqxV8rdvb{9pk#wZ)uZhcvKnT)J_&3m=TTDEG{;9)lScn8!Tcesq*bU zNJA*7=%$l-fnUd7?PGu6J9VVrKHYs!&U*Oe`+@lK)#I{}2miq@L`C`Mt9GZrCQ4uJ zg@3h~GF!|YO|S*~HDB#iRGcQe73AAr`D^RDyY%+FAuTFeOG4>u3%=Y99Ox^p?NnLs zldk)HOjGw8Mhtj|30YZn@v=lXfXrN5ptF|pe+r+Xs4lT>;^@>|g(huvT5a{mD~k7v zwP?mN;85+}tQRxA;hGN$3(i?&$i|fkz$k+lp4?;=;fpHwR+XJ&{LUo0np{9{$) zeu2mCV}+m}R6loa@boIA{6;vYz`J#Y*z`OXXJ?WPUYoOk-?P7KmS!ErXTVhu+P^i* zoaSI+WetEIn;~JnZ|yCC+f7h%etWULe7Q;BuG!mBcqT#O1x5Wz@NQG=U6YP+9 z+%cGZRGIPqJtEUavPz2#4ud}P@Z1Y6V*O)L0hYkQ>4^rgCq8g!@NNhK&2kM$L*VWs z7(g}5ka!FM#<|I;K87=52^B9)7i)XXUZBf{w>Pg!p7mDhHTF><5?bwR3$MFaeDOQ7 zi^b6H@n*R5TFYozf?J8{66!ZI3rmUlYZwXxq9CA3fZy%Lk#IV)`%bAEBiUfAgNvq> zu1_2Eqa#<_@D2K8;|sauVsTqslH?Vq-v&*G8+j%a6QJrpUkU0e+o;!!`~T~2+vk22G4GSKdJ9&$_` z{H*FW<0eB)7gAGG^+}7res!aPD?x)rbub3t>(a_xu}iZ1&rjlv92}C#&Px1Ip8^n> zv}%{UxX@S(;cv#)xcguonZG7TkIthU({BC!p)%6agi;yM6g*qUZEe9XF61P@uAwpm=$jFI(@Xd30jhF8 z;_fa5Z>wNLAAylD%gG7r9GJxr7Rj*wBvzB(QJk1$++J0BOLX-W4L?7d!j)!l@PP_M z^I@mYo;~{gV$qhq_~L1&+s%yunoKLgGXj-VC*0ynCU( zrl!AMzOx~^Sb|egee16uZfd)Rf`WkjfaA3c+dDWw)U5&uWv)op?Ig|J=;QrSy4>}g zoyCBGj%9mjy*PgzH|9I>)EXNnmQG*;@R<6rgUEE{@tQZBpPz4QaKyfZFoH1HDE@){ z!&Qyy?n?4D)|(g}blX;A#zTJ&{T%^YV!I)Fik8;am{5M0n?y!Z!-yZ}0n+CuL4g6v z*oDGEM=92Bjd9o^_>VY@!;9M?|Ac5Ye5EkJ+W_Vym!#u+ z!zMfjh{}RF1_zGbpuJmm7I9-VMiHwD7!|qg*u6wFpV4z3VYt<5S$6yRUbYvW>FI%E zF55><3L-jhJW>Q{S+SP#Sf0pVQ5N+U3)0)f7O%B>UKCjjB^T_t|)op)pBxS1Gp{c?im zBL2CP!fSg~QQq6d?q7dV>p85PNWQpdKI3o#Rqcj*;sWcgzLBCGs(j7L?;PQTPx%tm z`zJ+YkJFugBNalP0*nH-z?6x#1E=GVLk7oQqb8}7{C^c!4pp!=y%!L)5$gL~b2ovc z(d#CKf-eL>ElZ)D)25(%R@DutJF#J5GblBG4a29?JCWqL3C_c#X+>GMaa+a38}(yx z%b_WrJkl$De)(68_c53x6fRBIsS%#Ge&$(Eo~*^l38fA?J6n!@n4~+8=%CDP6A+M( ze>3+vhM~4Fv9OSPyx=Dt3Ui^VAZt&9w!zU$mmYH-$e+WHp_OtOD~k#9g^S`=Rt(aA z)S=nOdquNP4U(06$Zh2(kq0eUk#u|@c@*9S_(_L<(+$8lz(NPtl$dmB?*8qF^{%ti zyKGiu?UA=|S+<=XsO9Or3kdGvK???6UOB#8ERzt{b$;Z)4hj0ynKNgKGYOe8$OunW zV>aOSGJd=jXzcB9{QmYAJp z;uv)&s9*k1-MdS3E(%UJxGuz{F*CE{*mwIlo+SxfnqXKN?|)D&JA#wE1SS!B{dZ6o zDtX*#kj>i|DuL11*WT!AN(Fxm%@ zh9PUy1wNOD5_WtrFhc{}J<<9iIr*)@%2Y(YY_`))^!4~HlzMISP;A6F?H)hhcr2yM zgEE0iV_(O<1OD_)C}1Kgf^ObaKzoMZEO#ykDr=s*dG@^@ z^?Dsn9yvk|xZgdsACeh@Wut;E=k#c2RjSsW&v9~LM5aof!447U+5N1LjOHB1e0MxC^D|J5n|*hy5{~-cSC#I8mgSH0~+Z$k6xy%Qq3_j$|UcW z6f*sCfgzz_ZtwN5pr8^0C?_HUrQb`z-vDC?Ah_@K`lRpj z<2@tsJZ<{mS9kk9&@Uijgu#k?46Yfv-(yF26ars>_ELth>;k)fW*{;P{WnpMVDc3L z`(p)xySeq&Dk{MWxnO~ybYjG)aDUO%uK&u{LC3ve7NQ*LIy%&F0fmbT^|ha&w-peA z2viQW*T=_)ydNu$4US@ph&@-(0~L|j+A&#&vCeja17Rr^rLc_RmFYV9-B8X3u3RFp zjX~@Fed*Uv6aYa2Mp6O!1=f5d{9=aKd|>-kprDV30&AaWCCy)hWHt0!!Qu*@95DSc zi^Q6{31`rq%r}mBSCqg>y&S&qIIXrCHz9}p4=eP&5&2sWpxe8B8O{7?%Rx8;V(jp; zQ|Ez70LW(qlSEL^(trOpK5YU;A8t&qM2~qC45L}_tlU=&ns2cx#xl}HzrTuFH z&X@C$BLOKZSa%KrilWh2cg^18qIs9aIig`KFZ1>#C#?=o&*+%w*he6GL|G@=Ed(;o zF2JM_DWHM|MIKF3QUA4@!VjF8a0p3?^D<9I2ok~LLE>6_Wb^wd?2bq-vf2l#A2HMV zBiPdkTsERp!nu)z1=kJIFkLB{;Lp*A{LuXoBW^gts~#j31kGte@EzJR?1OvxR8_>nMh8F6rOW?~AB2?jwX0vJUs zv6Pfy#^DW5G$4t)?0r;G@h~5|UQF8kd-n{1%s`0&qyi?=829(uU}qU~$G-FiJiV7$ zS?=uu9eX7t*x+jti82Q25%uNciWEd`h5vymab+rtpH!8SXSb+mGV*a>sJcmd>qd9uDf@rOga))*n)+-S|Q8?`m zeLIV-o7kd==I@79=f>b;{BYmB5gZ(h!y9OZkgyV+-6@7G&n&Mwb&2s~nB zO!Tj!^4BM(;s0m6tbOsbt;&H%-E^ruGT?n-rsSf_@=Z&3!-vHs&2M}=7X@_d` z;q&KPC>oA90lWuriA2}myQZ@84bH5g4>&g=lbe#D5)lu~lPBedt|w1!#Guh1&oh=T z7cPfq_V(MyF@@}%cmR`RVXvim!>Ud_Bq2JCyA>8G1#h7yM3AVDuqVW`u50dsRt_U$ z09(@9hZU`fSwG%*eLQ+KW1MkhzWM#g`<*1cmi1F{7lS3Tv^7-a-u~8$nD*Ot_5g9_9^19?o2E5iQt4 zLO8)-xCL+K!ZG?b{BVMsiUoyHHuvjR7e0Gt4vrhB*T%4iR%o#X_X9$s?(uQC$D?%N zrx*xE`+6r}wf~*_HZTUtjNpZA8l0Kq z3ssZq2p>qDmzRQ&C4UwB`7!KIg}OMDVblU|G3aqS$}1~-K8bghp}oPR{!o*p7^#~$ zL8_}W64^t@@SSOXVCv}js`N@)YAV;ag7%|?TP1juK2b3q-x?Y)BOwk^z=c&??|is~ zp^iTiYk#{M25i5=m*5L3KaqNgF+SJOW+Z*0sT05KG)^315Z?qBZ0?~3^Mg1CAME;^ zN4f*?lAaznIuqD^b#-^cBGgvf>Uu!HAGVXIa|+R;6KJ1^b{O+dHyU92?}#poi%WVc z2v<6U408t*8A7Rs6KbapFE20PxwR;N*TceU-fugusDSu18Y((2w)@$jP-j5yW@y&R z5@W(yBshHdYEtu6VaAygWVU(7NC_2mlv;Y_PM_}aR0g^wp5A#YIvTXO5UlsZf~=iR zbhuWdcCkh~iNz03BZB8Lm93P5Lp(vPK)YE;N563+q2P0u>D!$A$C4RihQs$B-wm-l z{*Bu6_5Ytu`?0W~5}sQ(d<$Dtnm*_cUnEz=({y#)*%-tDPj%b@9wB%-*!FSCc!wh} zColdt;h2maanpLKQ#l#*#z!hK$J6Hb;fay3IIeRkGU@Xj*<4> z`7FwVUO~dE-m@#x($A$je@e?`nsqYDZ_=Rlr8K1=zf&UCR3Vm~Y1eFK+%tVG<{Uhi zRJ2dE4<@)Y4nA4){t-7a(E4M%jRvb z0+?GL$VG=b>Z}nuE_uhbIJRGU>Q`*%uNXCksh}^-Ug~?F6?>P7iA9}48+q2yFdq2H z?)>FGUMCy2fZe-yTj#~yrd&f_2;;q$BF}tK)t+ABJaBXi!+uM}kPdXyQ9UWf4o%MFj`E<2KrvOh?C)4|Ia}j6B>=fWK8T|4gzO?`tF^447v`T zl;q?unDL)tf9#TqIm>ib+_DM~)JE{D;H=C1atROO0@B(O0X7c6T9Ss{cDjqX##84+;y-Uz^1FI zr}qNL7aA-T<+!?M@$p9hfgnh!GvhKuwO`OT{%WTptKOxb4j$QAXeu~oY??6`e4(4G zfiuNJIqwzG?UceSl2TGL3&ZJEJ(DWUVtM)bPSBB{L390>J!wo{`bGN_h2+Zon zH-8@V;idpr=b3Y&K{rO4_K)RdQh zSAU(t=Y+7Z@FQ5TsRc~-r)cAdM=SvRlLwKOvqK{z^=Jk~5s#x`P~e(LaPXQf&m8D7 zb#ci-1T_fRKkResJ33_Jy>PSxC1QJ0-Lvp;X7Djm3JP?64{v$gj%71FmOpL|)`nY` zFVzo6$sIkHRdO+(Hw8(0YH@k16R9*Fv4FZDDHE6I;*?&Y+gS{XB(F|AUPnb`)8veP z0gL=BZan5&Pj_wgdG@unHKkGC{nYaE@|mI+=<)toJ~{j+6jpp>^S-9E@QlB*q|Twdyz)AsOK3b(PzbCu>H zuBuWP5AC6%xk7CEG9Vw|f`ykGvA-_<#dyY{Z3CI`{@MKhSt&l58ByH-^Od*N;jF!1 zHs-ExIoJBaTJ7RA3t8k|+3M$3C%@k-D;+R-av(|R>hd;mbMWeBzMe7(@&+|NCJ9GfL#l?EB{1Tz{kl2h9 z^3Fqt&fz>%a{lPi>o_c{s2p_0#M=$Vu8t7Kk2Ys1wYRr#t&)bM^h^8zbo2%WYh5XOudPEO7la6l~( zlR&)NgTSuW7pFb@F}FAaM#Qt!RLiE97!t^Utm_d#LDuJgc*`Rc++-|2bW!4#7Fo?i z4gC65@n;Flx2%3M)?w)jY1d_XJc$MVl<~H48kv@1uZt+1v!qc&Xry?*mF~|6; zQAXeuP(UjTTXh%v!;j~u-g%S4M)owef$vk;rT1+AyFvy80}R zO7nOkiUpM-@|c_EDfG@fZUb9s>F9`Gti(-T{@Ti?;$Px*>@!c<1LBbtN326!y}GWh zuH<1HA(DSDj*r#Vm$tZ&Wqk&V_LWU%(XAGk+07s-;)8kb`Nxw(gM+nTRwV#4x#f23 zSOBqtqZW9&<`x^;+EVdt8yXw2@q8F{4$J4gS)Krd^ia&_E@mdC1AQ}S@?P6?%Hp4d zy=gOlZH}s%w>U4)&sAp5OJ|)*!l8M(0&kW5p8t_i$k88GI*kGMMnvHv{9HzvpKiKdnuF{Oc zBrxOLw5xL}Yk9yaw@%<@^}MU|l_$pMjv@Ffd9*cWkD%q0L5<_U@NgX{jwP0 z);sj`A@|z1)X^ezGMN%ujAH93AHFCu9csM&ZIYgqk}M}ridAtaZ94a$@W&EAsu@H* z2h^u(r4Yw2A%mi}$C`isEMYI~&jdLo9TLflLf8`EwU&PAbq7bk*y6&%w2Swg%R+aL z9*^g=`V0NSmNHGIM%og5KgJAZaXJUKcsJOKbr! zUP=O8BsnX~p-c>9m-EG5g}D7YNTqgXTJz80jkn}FB%#+J31KpZ45${zDi5W#WYsbO zhRH=m`T(*}a4g@u}=`@X)P@m|;3;1nHur$bwd zLPn-gvT%qcl^wKGuy@Q1hl@Na(eWEKl=jwS;D<;IhRisS+nPm&ynYbnKNskKEo+`> z_qL&~wX3=_DsO5fMI!dm(A$(c#_Q(viw;TdA?iTXZMuK|StIu@0|k@`DVrUWmM{OH z=0*E%i(Sd1btOrvSyYuPi$3qulf$Di6zaxn8P3V99QralX|z^BAjHw*Q&` zXDBIg^d@pqChzBrueF=%P2;!x`TVt-QQ;syXU%Rj1a?kNql{f{=Yd(^Bg9z=b#rsO z*nq3=IFu6qM8#&ko#lRna)462f|jO&NCaA;u%st$_FVn`-8)}z{n70V44*KQn4)`m zSOZ)_`LMz~>RNvWPx$4-iZ}H2^d4$5w3b5n!x~=6_zb(@e!J&}s3g#OA{$bIOcu7| z@r^W#vpd`if^VPkuxFj<}z4&)4>CDjc>HK30Tu`*ya#qk66 z$dM2zs?ioBVq$uo4Iz|5C_^;xbPp?srV#9Ok4OiY33a=hZBIC|spYFCi$rDI*C;e0i$?Qqr0-Ugp_W~n$ zH^k-X{pe9BUOjsJL|EFR?RvF(3H{3*BOY&me}n7aM}gv)@bwffg&LhV5 z(41MfqjucoHHG@*kY``f-I}N0?!(>ze zXWp$LjHF#19lqXgSW94;OMxK*AA}x8K-M_JJ$TZAocM*A2BX4PLih_&UPOL=K37bu zDZjj&TnIip%pINmvwi(l4GlXk_nBvgA3RKvV9cZ93Nbh)GX+BIYgH}u08{P@eVRok zJLt$#($ZY25Jo!kb}i?Dg9r6L^W$lciaLMT=R9O_kx1YJZ?`{rW_;X|G?^m~v(tK+ zVW2hi@V+aj8;533cCj-HScISfmRm5gL4D@{HT^C3MWkZk#~wHDl_b z!-u7?v&&jrPbOcXqiAsBG~;c%j3>skpfz2y0SbXfj~|owX1$B{)8Cg+ivTB@Lc%A@ z11vxnCPA-?+UFwA$2dnmR=4|m&r&1;DivK!4G%ouzqR1tTb-fA7(d(jc276pel7A6#)*yXasr z?1(+~>p@@mbE?kP`=|E~?S8~Wt8`JcKH9-o?6$1YbC%hl4B26 zqHi^Reou4XTAbjbu5xIr(DctdGRtqcJT_r#4Uy3ZB1nolO`rbfkmRFZ!yI7Un^Jo# zB|BT#4u6X@Z%LFA5P#_5F|)I?8Zj?koC0#f%~f4;-`%|yvtuxD`%YjPMBRF~7pz50 zW?#m62uLcQ{Vcz~os167PDNrWNmgmf;N>m#f+ShgT*3}t1WUQCMN->*u#G8Zx zVf|QER*Q+#4EHZS^6}Yly|T78++p0s;S-b(FCdzgR^i2Rj)5Kdh~iN1`m~5lj&Q(* zL$+d^8{jXo4TldN9DeQ8(3{V0PZILE8QWB*S_L zRL-!S7>wtI9uCJt9{WEpt??x+Gv`V^)zpNgHCvyD)Fn)qvc^E(iGfzC6QX^(Q{1C!W0xFwm@;O+JA| zhsm*yjjJ)N_k8O}`6IRJQOgpyrH#KEe|tOJ@q*z_1H&ra*p7km>Mj<$KY-Vaq`b20 z8W`WQjdyIth9u<}YxLM3LldU3>1i~z4s@i{s#V_S%zlY=Vbi813*RUAGg3^>p$xyV zimQzRqp62ew=e9eFzCt2Hb0|VLl|QtBO=O?n}o^53Yw7N!rkt=px!X-4%#?r(~Vmm2)1-D;s zpr@BO#zT=v3kqV*LaiH|5{*-|!b=Y~m=jdA%2LZ3sZqgz%Q}L4S4e~7%b6&NcW6(5 z^jj7P9u5pjO(+`Z$FDae6ngZ$9zPC)vGXJj3s8bUY}2BTcQ^+D0KqKPIoe=U%WTmY z^z-LWzh)Y;4TwkAN-)49qguZj`4Z7buh%pV^HM0j`p%ji7Oik=e?q-U_yde zITDjEK2*lamJ~``0MOVc%&gD>-XRAknP!rsXr$itu)$)+Zl~ju(1uN)u5XL$jM#fq z{QmY9E>{4KY~n|`@Q#x+bM!W5U9&E4Xy@6}8iWke@gi^~PAmpZ8yR;-ERlVEG}JzD z>iFGR|MpZ};=WXol@!;*9QT5>7M&G}c7go0MSrsIP@_(@*ZtOE`5{NQPc1ejYS-#=*dLYbkQY2M3m)0GFi62@IQZEfv%z>q-zAWbPG)mJpD)o*f%yfV2S@wBUQwUU5ZACla&^#y?|z|4sHB6pDn)n>Q!@`OU;JsbO|| zrP+W}j*U24K=D2O{{vH|RQm=xS~JwvjlbHn+E)VD zElbHg=Ak?#uFTh}l`-eD)aId-v}2_~;XbZLzum5Do1*JG3|E5;L_Ov3wS*u~;)(Y^ z8(4lAsxDcbIRneKMZ^6dCo2LJ}ZVpY}@?7>;kM<&)8VBTe>&%1EujLQNh3zVKC&O)np8ffo_PSX|PNGwn~ z^!S{VAP+De=HteHVqQ(}%TYHrPQ)G@(%9I@HJ3@hZe1rzgd8q-ioXi-x<1v=4~`ew zK@@G-h|ti}*pSh#nMMS}P?&%&qq*+SY!W7oW(DSH@~=@!zkGRKcheMbQwO>-i^&v^ z|LuC!+VI@XI!mY{(OWw=UUOaJxl7D&3hb>e`0z@PuKc%od1OnDQBDyD(JFWSdK78z zn9QYBR^+tA0jpD1w;i}MZ9jLHu^+wuk+-+R_%YA2c1R$-@H3MiySw|$-0Qn{FODze zl+v}KIF`@IpM3G{?J11^PON8i3he?nzTY<)XC^yBZzaq+mW$$h*(S2$i!0>qMW zKw$xNTP|pSduA_1!ZSlHPS9RdGc{d6*RP2-Kbh95JE&rkd}EE}ldvd0YS!n_hxhGU zws1Nfcig{!zmgO^&YWrPOUUN91?_q?Xx>d;l1OAag{Y}(n1eh+&{Cjh?M4OXJ6`v_ z2oMX}!Hs6EghHyT2QQe%$zCIeC`_)Z7Av6S zL|Hkxs&TbKGk7$di%Aa>&!JEsVLKav4%oSh^f_b~ebmwZS+sL#Y6fL#{rb;6#_8Y9 zGYgvB-)oiMTD}%Q{cTwTgPzTM0+{jb$VaLRxQ*Ym!vp8p+3pNzrB8lF*%7DjZw2)z zDK0IYqZtVH!5Wtb5l+CM4h<#+!6J3Ng0u}3>d?5Yu)mOv0*VO+q#R8Z!2Vm<6$sP< zIYXuQZ@C6TGc8Pioj8G?7Dwr#q-2S&d$43G6aZbAT|M7(Po`w;7g#Z(XV-EhWC^u; ziSzJH)j(T1y{pFW%o-S-HyX2OX26=She(W%B_;Cs>4AUEuWWECdxoHA11dNmBI58d z>DA@K^Q*P?4o}!V!Xe#rpYpB1=|wk(#(KX`QD}a?*u>22z?~gEUga$u=S`=_YnD=6 z4{|j|zM__vEiOO#ZUSM8AQ_3L3@-8w#@|ent7AC*R$g(=0N?;+rcggW%lG; zLR$bMbn&s7^x15sXpEvMKgDSWFBF8`Zbz!5l$64`bARK;lL+XP^u$+bq+(J89xaE8 zMTY9xS^Mgmnn>^(7*T@zZ1xp_YcR#PMH6d@S3T{b5?DRf62H!|uH9==c=;7wdE@VQ zi$FnC<2)OZ`7jxE3b!^N*uVb_thKB~-MH2lLxM*=66#+d=}Wq9EFMi0V6Zf#3k)}{ zE?<5|>=1x%3?yd2lYAtcYE4ZIm%r>p0>;HJH)G&r@gIEOWg331afU*n6*IO{1%6Gi zzYdwk*r0J}rF^LHqV{t0T^)&=I3g$)@e`N}dn|BpcLmqX)5Za7da9xSQ{Y0uLz! z4;tpAL`kE){0~!TSVfrQaFU@`xX!}!XO_AhttSzEY zG~|F5rKVzJyNVb5c3p$=A(E>Tfrq zD98JW%LdLEw2w{A|I{OZN$8za_7#MFjzNm_6JbQJ2 z5s8`WeIx(aaMF*}^|H0$Pgz(_ta3dqb=jAt^xj2pyYDLlv}iX+tV%G6nS6{%R)28! z?kDIQ;!vz!rlcIb)DQX?*|Cv`;kZbqq5o@&-6~nuDFAW9fcNf}tdmsf`y&GS>0wdDioR*!5YPe_; zB*1=LP8C?j*JTbuTK>WXa;i9B^{!uG=#C}n$xa3a-^aa6a{ojgr3{d{HJEHdJ7=f* z`Pj;bdFVwkTFK)5TNo;EdaWyG>J4}aeqRkRVjt^|7!Cmem7M7xoMymcm<25_km0mw z#y$tmyBGKlq$s1G)&RqF9!WyVg<(UgBu0YDEzA_h9R$7p;I$vxPkc17J5nED$b=36 zqqsiVEP)@VZI7ys#J)%Pa)OQ>1wD@~jBv5o`LJ9BR`nuHSPJ|6J$R5)d*+YTJ_{R) zPC);hh&qo5_oX3AK|EYfhYc|e%=bbZE4=!ATcH`GGgZ~p@P11qb0kn(6vt=YuK1}d z1B7f$0iJI3f#RbJkSb6M>$5_?t$G<6y4TWxkQtVN{+^z@!0@AB0WH0lW22LQY+o`n zCHKMcx}3!oD_@T$@@Nru^{} z!BLN@=w1*P8{}BWMbVJz+`i^zZL6YzZ%NOA%3TK!igUjHc}vae$0LuEwhavpBcNdQ zV7-83_VvjP|5Qp`B_BD}pQCP+HKGt-)6^7$6t+$%W{z;xRpDxkgQ#9eHoTPr85RM) zjE}X|P_AI>!oab*l<&xqvz|TXnSJT)EO7aEAt?Vb77(SfesqNvtmx$FM@K}vg1^>( z`e*R?z<@r!5bPNiPPegA$1 zK|;{&I9FlnCz=XSA6K|{iMH5D6UcdRXvnPRGnXC0$S{kn(Q7r&OmUc--B^pPHliI0 zgHyBf?pjJNfu1mtOJMU4p7qbfqXT#u_oH1+tfM`+-a;v6f@h9^TF_HrKL8Jb>-Jv4 zTtS*MmDVRr1rA2Y4CH@QvGX_9*M4%4wOi^dGI)&xwO)2`XINVZjL3CB(SwN)_8A-5G3p8+qHPAs9dbSzK5#;EDgXiWbn7*N5QDV zdngZ-gBzzw1)tS=X8z63c5dG;0}us&T)g(^azHksZMY~F;MA40hU)V2AiTwB{B9AP zzYrE+M0lbf<$d+NSE$Wf(f<2o(5cEP8A-_pXmbR{2(mBw_)!k`$UhUQ#$5zmQ~TL@9hJT%P1-AQY>j!g zEyyTm#{BIJktbnlLYGs1Z8;zrC_9NE_xpe)?Q>XGmMS*Va$HHVFZluVW`sWjjmLP? z3W78;w*>^SkQN9E3Ey-3h-Kkftc;JIJYf_}O~K_3b-jEd^ndZ>2gR;tGQx;n%&6Tr zL0fc8a(hgN6jv-dOt}p2oa>U!Za$0TcS+7kP#3k&>pANf@z!u_Qw~-6&bhDjuEuMP zN36~mEu%2-%^#hOIt9K8L{kH{g{V@ZKs#MR@NNZ+i9zjw4r=qH*FU@8SG;j7mS5<| zOuM!Q(*OZl44Yrqy}KgI$;x^XRip+8(k+~vHJE&!07*itx&R8sd7x>86=ef}0#<v2{`9&A zp$pzKDD-6kN5eSrV{8ihUEWs)UX7s*Wij>`%0Pwwj32Qd^a_NwzCTQDso1tHTSBo8 zMI|T8kw|)Mjd2aB>FG*X?Vj=(H6KA6|000@1Ij`X!_B-xix;393Y5*V@b)ZnUT((PV zn>{Kltj39pDan1k+MA z(zr}R*GPfi-_*6A?0LCu zc+0!jf3`&}BOnsqO19Iy*PFafJ!0Bs6Zxf@S$?TI=cA+ zW}8=3oRlzCEv*-%|0t?iorZ$(4M|{5Hffh4Dz?Oi^gF*!V{Wl=f7Te`7(lIhX7Bhi z1z8-?^GQhYrXbuf7N0EiSu5t%7YHQLR(pBw+laN?FEjXO2X}!Ev|; zHWdxq48}u7v5t{FM6!WFlMwub@dyjBaL1-@HMe5L3W=g66cI=)Z$rmbyk zQt+6|m-{|qEvk?DhV#o0{tj1x#$mN16o=wvN-oC>a4sl_L?;7G+2}Jbm~}#k zrszKc0|@;mMZz6FI(ix#C#N)aG&B$b90G%e^lS4fr)JKLKIwl#&Arj$B87TL;kYqqSBriako#ZdKd4-tAGack~dqguc#VG#^6%xBkW?!&!P2DB~?`X zF#m2zZ|d`yghzB3nf)JhlKulvi{VfS1K`~gGCCAEcNvDHeS4p>WLoqqfM&RX+#0pH zww7r>wwK~TZ4D|C+iASPypMCOu|m6cC*iA;QuPKG|-OQy-UrC$*P z77^!Q|M9(`j;Ef0GN{OS0wKsPXet1yIE?#o*GGw9C|6FSH8~k;PGkHfPsPZ{x}1D{ zT$_^79h(WsbS&zs*5~wrepE=Z`Bo#tPUpuMJ0-VRlfjv+zW6&3-yQgERb%`N;gCVk zBg8`~<}%;6fFBJ$-X``2lo3={c>M3JzO%dU3L z!py95CvJeiJeQwE0|7ic88@el$@)$Gr%xU@8pD{sy1*bl?btonYfiigfKG`<02?ca zxYZ*^w@4*trlbY65CcKCkS}A5{D+4+1&dow@u{uaFBI7qb)DQj`iwvI)I|fgPZi8* zoYB~)!eCe%=E@K%R#)P7i}Bkg#W*QdW^RZGi zwuViVar_r*nCr@oLOVC(FxT?KJo7eT5s3QP=e-seMp{f)75xa5@u*+R`T993tDm>7 zEmj8vd=TUWHx`bJxf@>i2r&?0qmjTG%o4YL*%G2&fR+W*oHIvSLofN9ObeJTpnI3> zFq5e2j?0NfSN}3ihEF&cB(ATZj35`0s1LGh{Yj)<^feCI}LmaAQYep!p6|r zR{%8*mf<&aemp!pM%I6HV|B}d=pt{$!)y#F$@VjoeqB4XExx^5O*GC#snZ7tb-dM` z9e2=N+QGzBg*iV0d*Hh3b6Mbri8{B&wqOYsB+zH@%H!!s$C!gc1K|t~6`|45AxNPq zwzX?^RLi;{*%aVdcac(jVn9Ec zwt8d}_DaE2augmv=Iw3g&krtWtrZFAod3kE9ppV;zKVa6nX_n3rW9XxHD6RLG{nw6 zNbH_yw3Ihr-+4hcXN7Hk2Rj|lCGVyeHWk70-kXKPSC-_BAKfrcI5Cs$Q%VE_BRC%v zP}LrED$rQ5LpTYE{j>i4zfPSFoSdtg%xYE16n-nk*?9#46^H_VpPw=meB$TM;XWUg zcyq{;Ag%xWPA<5+qvHao1{{Ry>Z7xN9YCp!1;6MT5d+H0Fs&&J+KO%>(?NSEG{$O4 zgkX?Z5cR8$PTQ*?38v5 zH-@>~VN7$_o5RDyi)ewb&%6Yt)eT3DkEWAank7+tAp;W^7G4f)>956|3cNjImpm~h z4Hxs2k;=|Gf*+FTWiXl&B_dwBOH2cb8ZaYy$UBL2=aHu;_tiK$3Y9N8EsdBKh`f{<5W z!M2pOdv@GJ48#oMm^FkRZ?sj2x?M^tK)!ak;7(mKq`wMjEj-@cg$5 z`0>-%nca)ngfIJGWQg(I78E+5iOPRsfBnjL1ndl27lFnhU6yx|%VvCu zGr#3;>=cQJ^=azaj;xtxqk_>k2VoEBfL^{DkK*CD+8xn>&~C#oMLRKx!He zY8O#QDJm+)r#e8q1id70LnG63?2EcseHTs(cmsW?)d0KzRk;BGjBJoN8Yukv$x@)Q zA-y780;c|D-FS?#7QBCz3XfViz_18V`5$3rQEa>y}?n5eZT!=Y2D%`(Z-Tlx9D6@z7*{0Q84^?w>;8nqgmC)?km0}z?JtOPVI5k!l)i9vE)xWEWG^M7d>8ak3gY4) zy=e{5pF?hm&hcyEMNF~NB%ZqU>uZ3KJo50MaK~N)=9)eUjGsTG}qwiJg z`a5^H{y0Qw{D>jf3>W+|i`ZX=u87aT4;m>5gSTwk1~O4HDe#D+m8yY zaa4L+-Co`7ENocn`lLde3l#VJy9GP<$%2(2UluGky?{G)+xK@aol#p$T`0MK{>V2G z+`i~YP;a_AF76ij;%>>^f4{$}-!s;QZqYSmivuC{-SuWi!0>_4GXl`$F~(kXoMC{x zV{4kUp6%iDo=CTj#-rI8lh(Y3QLY~q;87Fp+6AFs--<^JjSX{6)u&ItMH4iMAVheT zu&YS{NrPk~3=+-dWDSwi(~DrUdaV|R4A$pFl=<3{>-($05l9gta8^)E@tLrs3Zj)? z$xca;M+4T>+_b}%1Gp1D%o>bx$=7QS!%N_Fs2@^bB1*^TM&za9XV3b6G8Y?aebdoI z8}l|;NfApziNf=FrwZ5`d^eHjS|0zGC;)5au|E3+1Pfys5mUkJbo$I0Y3t_ls_JU; zBYT!@vlas18a47sbi+*+v+bBnyMo3;-mbvJcPC~j(DEGhr$nm(gX0m8m+q-V#I7OGIlhtjb ze$1;q_JsafZz^q_oT$w#c!Nc|;@jwh=2es%mC7qVx6EDu0!ON@MYGlZD<}MskX}ej zyWX1r;6E0Dejp=4KXaE(`Z!GZXq5DtcOw(73O4~js+mGWRFLR>v4v6tBUTEeJ$iNa za2o*lf`~a3=jKD}fNO>n7J5j`Fd&2s4-fAM;6zlF52TYQEExoAH(9@hlR1C&VM%)Bc*vm{8yV5V z#zBPgL>T;|t!=P*X)Y5lUIe5cQ2P*WU5JHQw}($) zg`-l}3DF$S-3H2=1*{4lU_B;0AMSo!Stcuyila@AE3w^p4~uva$ZC&s_t~fhGHrBk z-s5aC{f{+y;b7d5?a^60e!|FzW6rcKtml$TshQRAX_K5-tFX2pk6hi~-$rM!3n^Qz z))f40eQO2x7aq{mjlX|xS&!=>!V%_nio`3^=dG&A8UMDnuGie~bNrikroWEhPk+2R ztn8h565e1xDZp07e{JQmvJW3F;=_kQq%dLZxfS)$=~=OD;0h{FtspS;6d$L$p^ zE-rA7NCU}GPS$-Gt>gA}t@oY<-qZ8g`mfj}d*? z+mxw@$Dc^BUrE4 zOF7(7_r0GVzP7)b580(&tK3QA9vpi!)J&r0wE;Q6XrVirbBKp>-~7PisGtxvf^E7)4$6Eh4NFV&vMZsN{zx% z`mz7cbtNs|Qfwi*Q7Vcq(>eujIkeQAB)esHJqp}>c*oLmjc2xWqRg~WMGBRO6g z6HXE1AL2y+{iYtcj2Z6IDAL(6M|s9q(cB!121D>nu*JctX>|sqI-?Un_$cu8O&t#! zR#nyJXK<4FUJ$AEg2Vtjf&vEfa^lW|*TCZkR^`c=euudEVW8P2jy&qtBs`9F2t0?g zX5rhEWt6bLuwLNf+@B_3`o|ZdU;u!j-IYhBhiMke>fvJ=CW?x+PJ;*_FYGk^*lCJI z<2(jrzl5ngSa`A~yP~!;G0{hGVwxtn2>OGhxcPP5Y*>j<*Q4M1>j?)8q2al?*`X{* zV`icpM&gpe9ugf18XP9kbmP-GGrdMBfK;0BS^(w+dLE)UMzqgsppWBZM`UNuIvX;t zcfngg%;KQwAc3Kn6nTg|x8NMn{bb||``{tI4!+^$P!=q)YF|@#>f(sq=K0NgFd;tZo6pxlXEHYkqA7AAL1;o4VSixst2k-iB8`>b8ASFvWK_ z|B9K(4)M5v%S`*bHmF|XwE^g<7=T;rnmV()$0l#8YU_&=w{--QakzK~^rk%6yfYqJWk}}U zAR}Zd8C%?Ot3Gr5%y_S%v42yKYKu{fFK8C-Fl3qqV_QM%f1u}&F&}ty1h7w#dzS15YenGp(|DhL1?xelKj!$-VGw<>CfvAkjJE8 zlG(Jo2C6G8za7x0tcf6c7A&yqCR}mP7$LwJ(U3w=1C#Y?rM|H# z0`}vxCt6&hjDy5HxqTVMwUkOvbZ|gBpo90&WT-W~o^UDIcET9Tn|}F=(S_Wt#z>6Rp2N+^77I)_ttwD$Pin$W9leR z%R0y)4tWm#o>$_wcvAFcGG)nU>WgX4rOGIr%b%6KokC0isxuiP0d|4QB#W#vR&!gem z%d%}30>?7-a<_f*Xz|#uQE1{HIO}+&(1Y~@bq|CV*U~P{4$E$>ij8TvrrejmOxEcjeLKHDyL(OC_p4JHxKT3T@ zoS3|_F*;4m1huat@K}Alauejfz#|#Q22Q!;vHF#4t?19qbwps$zP%z8*APg6ciR3u zdA0;QLU&CJ3~R*Y0}zq~b2K&zhLzpgi95U%al9~x;5QUi2Quxb-*e4 zW@ZzCG>8zP3%y(xsV41=bQ;b#00JPx10xKQ*{t$Xp_o*L@(XvnXm?%PKq)@zV&F-`J%{%^nzkbAWn!k?grL5x0 z>VBSLw>yl7>uKd5tu`K%a=UODdZS~$xA|SBgZY+FB%YR@9L)??%CXNv{lTKgBr}IA zGa&6iDY+)YT`WeIcfo!C^HSCv{*^ObOoLIiJ1zwBJ-0VHVVCW=M0W66-8=pi`J=_xq_MHR4GX#9_w8+TB6s{B^DoXZg3$XY)@BwKE3Xq&1rEaKnsh!# zMXcY#6W=#kWwoqp(`9f*c6}%`9O@R#fLw?;j2PKU{ny6YAQF-5myHdQjVn)_TQ$J4 z0)__1b2(lCz%(TNY)!Eka#bywLc!02;X9Apkya}SgBAfK-vQmQH2^n)i?(H>1HO3i zf;euY)0^n?={34Q8ximwM9!)R;IiHB4%5NVJngSC!VnKQ_ZRnp|2kaw8Zuo|aLFtx zCPxUev|`b-`W`b->klD*tQw;aF@I|ok)jHqv5*$X=>7)F7qRXmRxK1Pe>c1;IMb|# zBryoM-GSY^MTxT&japKCEk##fUk0(mC`8|fCTO3{QMw3r55oLv(oMT#6p1ilqZ2!8 zN8Fl!qwFh$mOjDSOVGt!Ty$xzAs@WHUlNG;7T{YHOfr}aDt#3Q(@xfTKtPh+<|Q&7 z46*E4!GS(Y992|L%Dk%E`0pbIxZUYJm0qlAQJtfPPb_slY-(frA}?k#dxwYbV#E8p20U=47mx$>9I4SR1%+jwwU*z0Skm^9;o&L6U3AiaSXT?e0w+ zvRIhkbCZvQUxf?ownsg&KLU>O`dunnTkkHVmEB0 zjm7^^h?1Yj$1=JPA7^so8kz8GnHGaS<;?_%`T>CSEV&qqMjkc!S{Vx-r500lOw4N; zQ-!#S_$6>+7PL-~LlYzmjSPdeIt2!IyQ! z{$<)?H}(PX4gmAk)hzwXkpP@z|A8)pmy5Od`&tHllc_51FS12}_U5l<*7!0TY#(12 zq5Ub&?ainD8w%$>Jmdw@h^PTYeQ#A{2h_#%Twgf+VW$9hBdj!%h{$_!GYvT%&^rPT z%m@5M1o2;{Vc#LCA9%|MF#F5vgiq-N#mNE($T8p4z&7NhG$yU_83F?SauMR$AG9tI z>BxkDNLl$QS=WiV`PMrgcP!;Zj!X{JB)}lzP=|7)BIJsRAB>7n`mbj?_D9~TEMaVp zG9)BqOM75@i60>?@Wu7;I?$1rYTBU3z}83RDa@RGnk_KZdj-S!3vdGu)%TYUCZQoA zFO!o^OIdg*j!(Y(DWr8x=FeFiq)0fK*@qdkSjt$0oxToE6T}jqfdh=n>)_Fm&A5PP zh=?h3W^3n^6{4!W=_~g(>d2$);0TnsEuruzXv(~LYiKexw73dU?VvH}|FX!L?WmHA zGkCv=#lMrO;*nb1D-Zp?q_fJKyD#AD8Mo$HzT6R3ox++kTb86s2nUtYxS>stexvaS zd8GP<0qKxvbV4V-(d?3ED(tv*p@QIo=$4Q;z_Ol_eGR2|ZI({RtkN%Na)?Vo_TQ;WzPWO@xvWdtlCOe4x8qYz~uzn(KmKcaS{#wa$ z!PIovY|0XeBlBqEG^B*4s1Q=t3KWwTLeqD5V^$+G(dW+xAP?`e2h*{@)DuV;EI64g zPJVl02+16#BB=OH&I$o26q%|k!xpS9lbBwDhY6@K0&c3~uOegM6XqgsUtir*iGG0f z$+$-HXQHlu@6I78k7nKqvWg+p<>{s+xs98kj0H@IN*5It*5+&o=`U~`c^Mf9s@d+p z=ayun`%;65pRSVoO^q(gHI$Y1kl?vuhJtks;KaKix0o4r%J|EL#riA^fJX)Ly|0^0 z4Rco47GuihsmW0bcs}uviD%Inph&EbJ-evd+&B-xwY=}#vNcVwG-2j_^Pxg%-RZo2MLBYZJ&cNWJ~lE+^7Tj zJFJmp42iy`)+vcQYVPW>!tm0)Hij0dWD`}OB6_n&RoiG3eC80{d4@Pqqac+r&bQ@z z@0`H7KwtR21RDoPvn&K1K@=T{tVN1 zVVr|`W0Rf^#bhBx=AUG{`uc-;40E6Q00?0oeM5M4-i8=EskbnkKj3R#I3*wsNfKK^ zpjQ$JyCaGfZ2R;okI{aJHHv6L_heduY9eeahGAmDrJpK>Y`6mXjsUR20O&wUd3v`! zIJd@24h}j0L{(*GP9du5+c)xNz&B*38hnhYCqvJM;h6I#p81B_XLJb zRtFbNGL+(riYx8-E02fpsET%tE)!Fy9|@SomvSuVVSKV(3v?dP+{h!XT3_ zvRfJXA!m&4K1a0%y{3+8y*dKWZ8v^sAiht}eV{VJMd@QBlu_0C5TzuRb++xvVW_@Q zIp%Ta&(u403?2*FG8%%IEl0@cAYv;aj%WlHn;x!BvcOvlf0$cO?@4vGv5vV}+cdT9Sevs9g3`NL$@|c8tWf+dT z#MF;@ zlTARt=4mC}JWELCFdPgp?P?Y+lUc;a4XlgE#^F1F>xj~sX*X))V=Z6V$^|M>#-kMg{bC%(>eu(Pv0nLvX1A$|4OLrwvdCv=lJK@UYP7hqJ?xi$EM^}RH6Xifcz+e z%`ig>`guTgfI32e`6^kcO`}O$<7UWPLy(Y6j?K-sl-;d`YJjK{!IoUdRV<&4!E67@ zQ}u45`RpOiXslgpQfgN&J9rXVN?_J{TpkhV)PM+SW-WSGwf z+EbF`fc%bi5e0}^A&CM|5J`yhT412ZCt^3*89q4>+=I?A@@|u~E^Gz+LjA0X@djCK zzzeLJc%eukRf*{CAAF!_nE!;0Cpub=h?A6+AGf|D8@!Co)j0;LVc9E%VWtNWuXiZrvH`1O0=6)~Uw3 zaQd4~$yfH4zwcvB$(YeO^VWTQU8jt@qL20K`i4(&;5&chyRh#UmH-OPqea!3e~pe$ zbiqU79r6YTSVLT-ST9T8$+hCd5)t6S8Ur1PyH!)*^!NSPi@oviymgXgC+xF5bmI`B zL+HfCRvw5J15QR4<{2V!AwiEGKR`u*VMM=wlxk4-0*fL_^qpizNEk~CDLShVDFTWr z{qkY_CYgo6^fGrYn72Ry=mX;=2^|mx+%CczO(M()?W{9>{7wH@H*R{oh;8Vs5Nu88 z9<{#?%L3x90wSF$^$?N+ktgLh)b5SjX@|+FY05Ao3l`uV=ah8IffwMm=Hn(CUtPrF zzi-cSWO9Hv#;itiH=vNfsX@;|XCyy73@2eu6vv0d*3&Rx86ls=gG7h8*>s0$jI%M3>U@XXRUZPWwd3Dd725&~~+HbF@R|}lkiu0G&Qw4^t z>ZB=-jGyPEp21!Z&@+aw#a0$Fv-LGoQ&R*;Ab|&v*>H2Q+DB`D3QzJE<*z&%Lfeot zm2?gXW(s4D+4_tZ{@tKxQwt4kJP6_1^M+~DjtlV$aAyKr0}21`wTAYm`ua%1B!L^q z%(ST* zW;H5C5ou|BAu~YyVxS6=tcdazSn`)+v%{R7DsGqxVS&AtM8BZ#@v+r4RwDryn!QMf zb3Z3yPwLt)GVwPcWQnsnZWyXbr6tq{c`nGt^8>pDbpLodgBPz)&o0n;+8G+i6P`!D^9@HtNvmut_@I>l zl0piaIc|ZPyI`ZMVOB?&T2in?#G?-zC&Q{agxM%363-r#$~JR*xM&omgl&*MxM*B; zytyVxDftlV<)Yu;?md^ql5^Rv`AgN&BeQhZsEO}xL_Onj`?7fl^m4ybS!dKn*HSc` zg-3Jv-haK(@#I=SIWUvMU~8jcvm+^vTVnvx>!~$LUK_JB9m);2as{c ztY-#hTqWY6IfzYm5sbWUbwWRn%pdFAE9r2P5E*fHcD4o-Kz3k?9#``xxdoAMTdWL3 z@rD~~UiRYACh8etGgZ&NOpGYthqhj}@2Xt8{>9oh>XZZe$}b8_6kLT;+vO z7mtvDOp>O6MfzagDrSWwU{D0biy+J3wLdA9AwXysg6c>Frjb$L1#%lP$O6xKX+jl3 z8pImIcR~n3(rhB&VZRQMH0X=Vu;CFkHE|BGjv#+qXl{B4r1ep9Y_OB#*f`s6n`*|E zlngQ&wQrN>6Qb}#`2*NgjYuiej!V`DDK1J+q2Es^w1IQb%9y|S!vh+FDdK0G=~ zd?ZApj}iyzEXLqp8hs#!1WeI51Y?z4Lo1m%oL=@0L`IsNomjn|n7ece(qm!ICgVM` z_Q&@wb>2I6hOKR40h;UBiQRN_LDtWI*B6Q-n+BKt^zhFXWL~JxA=-G8e|Xc%&Gd@{rcW>b=S9c z_~bMg7{7SZ+r#Q3WoLcvL3?cESfq+tnskf$8=C^@q2UgPO!HGSv!mMrI|c_7&Q)}s z@#%<|yE3bB#pk1+*|%2ZYQ>bOOuG{Sbb+oSGlT6dUJ=`7`-HtVeN)g*IXl<;(Mpz= z^7Q>+&a%0wA3i%P&U0>Y``j1mBV4|7_J)|J=~xpZ4#5+)4?Oay*TaY-!V}FELh`i` zCer!XQg>2<`}>BDw@2MeP1i-l)W&RWB%>k z-VOcHi>Grlz+AR+*KgXI$ucblTqkX9Es+ZlMOo^WgU z^!=-fD|?Y}ND%$oL%`%JTc%4|u;-c*vK*>v#Pka4H_ZLqIeWXue2N5dY=J}^G4e#< z4;@9o1$7{9*q88|Fk6USf7#AKa-0cLxMmr=OG;3U!7!@M#>;Dd>Ui5hlo#aD1BBf* zFWvqX9q4V)N8&9;d+K(>;4NcujERJP!AN8zllbHs+BY!`Vp9b6D`vn0PZW{*!e1zB zF1eKA`vx3Wbv3-o5@h&nxi01jBN>!eqb?Mx=!o_7q}|BTbrb0_8*_^{(UNp8il$5X zFds$E8R7as1K#`l?~4Wo(U|j7Ks2It`!O3T9~rSpX?^?7Ep$ol<-u`+|X`A@dt?x ze5dP=ic}oVe6+Gid-RxTVW_A}_6|8JE92XcpA8J!c3Q#YDUb206^ zpRi8&xF4p>4Razb9SviL!Sb)#NZpgxa>09MvV{wIqA#QZ47~PmC~uDQQ9Jl7)-p*9 zU>%_NGu2J7uME&b$lyT}d(l1y$jZhZ4R*B;f*kuLE##i{Lw z$l@3IeO2h6#wrUt+N+~bICkU2Ag*a`&3YEWm!K7nGgXRw@+oMA7#vpXNTm1)bbaae zB#PHxTN)SF&3C-S|8@SnEYOyDr8)M0DP?mWT!G?TtPZ0Q=zCCSu&8x_%^2S1FtCm6 zOlCj7oP-{xJGHmFTb9I7-C24=7I*|~l0*yz`koABO?@}Vw2>R|^yxto1DkI%YXEEq z%F({~vWH|IyocL)EW*-}TxJ%{q;&HQiz%swg@uE<0XfQ9H2Cy7(c=v0zCUE5!@tgqQ*bKJN*Vor$ZdS8|mXi|KE$ME4fIUAb zBfC3Ll+H2zF#b8YqccZm@kzXK{Z&fbrNtdqPa{p)dn@zg1un1&5LU?Ceqt=TSA#Xqavl z7a#L@xGewJX}hx={9LELii`{1J?HA%cS-g|&+Yq@qgiWNQwqm#oX&S%-mdL+t?#*~ zZ^Ptky3=(rwW_mUU5-xX?JPa(*_SOK%9$m1OtoC+5!n@C# z;-=UJ76BCyETtA8Sejy-?E^s%FST6aOknbIu9vR5nF?j@Rf*%&y78$661VXbVt|O` zjv!TUltI=K@R}G=&Xas8OoGHN2VJ^wOGiOJ7&1cEl8$~U{>0XzjsjBvMW}jA&*C7+ z$j!}-7G}vrvO`_(sE0!m^n=I&tAarSSu=1h`9n=IjD@EI5cBR-8@TxA7d{|XmCt_5 zy@Z%8a&HFu`hvwpyWVBaw~3|*ogqlAf51o=7wKa@nhYd`&=d@v$0V_GU-{stNi_*AE^z;L%hXe1T3Q zEUJr1#<1+cSTE+n?pvo5bq>is*Zi(MTB};7FAowBb1~^GgDoP{jLgg&Q6rOMyZ)%^ zUAs%Zw&2=IN0r*%h=)377F9?}v1a4qvNm=4U~sk;q%&JTk>j^w zrfX=$tl-3wH4OXiu>4iM>;31LdpD=8#P7wku}x>P9WFlD&9Fj#Xd8Me zIzDGsznQpWH8k8f_xj@Bom_}z<3H0PyluMQDOopbJf|tH>e-*0HqklMw68mGr#(~Z z9%Al@>^tuD;60LH6HROiBw&){!fxECsr5EJxS_Q+!tJF0YyOvXE%cL1DLmq$InwK9 zvhF3S$O&?uV)xp;c_${-tqb#Hav_90KpoAn_iOaxC|5vkjKo`^y$wg`MR5}F!4tha zsI&PfePmRr!-hvRs8^r0mqCM&Y|*a52X`Oh+VYzvu45v7* z&C%D2Od!Fz0Dj1IEx3f_Po&@C+6OSGQ!R69SOzmt1vj&>&}3vNp4exQbp?r|vho#% zR*|uV*lf0iln?yf4?2FYwFk*Y2?qIW2^L!{-Cn5=(!UY?A{MkwG9--?rc^{jXv;*V zN5CphZiTV17>mCONdlt5{1~3wxZ*O>=folpHi+!SkW{*)5OR_L=YUO#!ugraN@_6i zRxG51Vd{l-IIV%8I%wQ@F(e=*x5RjA3IFFRvcEEHvU%e_n5_2jwP<2PFA6%&pTy7h z4Rcddfnexbg|FQ%C!o^ytBOUAWkdwqTk)djM+^4;jA+~V}*qmyc3?M zTcm$UOK$j`24(lwVC(VOVV-Gek>GiWiQwt_LxQfG7)}S`vY}!#DJ_yNFlcuS&91Ub0os}xj=OT! zM>eD`^am5dU*eM*mYy4ye)F`TwHzraBn(xe-W<~%BKc4hupx(n#=lsb(9o+4Y!QeX z#6P%4p7(3-88I=q0HyI)7E;G>KEDcIy9H(i_ zoZD+~sTmp3$fXM?of2nPH(KL6Vj}sKW(d&&1ab&Y`Bz9cTc%hfeptdmN1PTD>q5OH z*p8=RsRBs~%$^=B@_34$2r|+jAQ=eJppnfNR}=3Ei1~J;`a|CXOoFhL3!!5a z-$UxW8wz_{4~kKeZtoS8VGkI8!pnJ2(bDqR82?kq5IRPeKb-tx+~4tlPQy(p?GMkg zKmL6fDY@SNa~i4UcB+%U=3647$xIH!?r)6vUwvKJ3m(BK6?bIHH_+Ap!wg_!=saeJ4pBL-`^tFg6ILV z5Th;uu0LZ4@|NVJku)!Cl(?fIC!%-m0?>Q>rCUR2FeC>R*F}aHYG)u5&bgS6m-par zs|xi~o_gzQ0mC3;BGTVC`e2psf~u&7n4N#-ZD2p_Hl&+-WdV|%kL8BZW>eR7WQP4` z-@rvad%~IP!1^`Yjj1AD%wrOyYN>GRne2(S9ps9Wye`~j@d|bWiL*!e{{Io~8LL0E zktiXkPzm*fI}(6d#=+ovs-7%VusN1P0c8p~GP#-N+0asr?a=0RR_sL~??i+bx4mV; z7k8+p3Db?rg(I(n>!y&!nx*rZIzszypZA2#QH-|a}KDT7q9$& z_tosx7q!ZhWND?X`S&nR-|w_ zQUbB_FDO0f$e3JGZ1t?I3caNV-}V=LHBETam0XW5T_INdS=?tmc`>RQ!znfE(}k6M z8QLq9eY252-}AoRW-%2?GO^H&?~@05a*J5sy!&8iX|{XmXIKj3>8VFM>c&hyor%Yr z$2VVnc&!XyxMG*uq<)Vg-t$XJ>h|@moDtNzVlp^Ez>6SL!%T>lgeQ|8hgR2 z4(5d$Aa0GQCAx0;<(t z9Pz7gAVYKsbXUJgF@i9^hH3$L9srNS2%QyN)WDcg*V@`z4L~%<9-vF+%C>8O0ZUG$ z&lV{B5#L8hwIUCjHMc>Q{Jx1Tr}7PA*!I)~ZjT>NB#t*qqOPr$Dh z_~UV%w)X5y#hMNdnFpPij~CJVcZS_?#YP7S79}4)rqazles5`-b>ro^}uT`BWo*sP~u2wyF1y8TfB z&C;%7P@!Dv*wPt@d1oB8++1G2zOBl*v4~%Gb2`BLjGegjBfTRh5Cw<(B_##@*wSw^ z%lz@PcUTYP>f54!ywKG<{ea84QKGc3pH`BGf{n2~(IpJ&e z;vDOac>J{t!fM$Qm(s?9L_IsjoK+#% z>-*D3WH-{pB7XgfoGz{2R!WF2jq2}FD^%WH^xGbziJ#>d#fMDSlgTa{=DQBWwW!Rm zH7+gC>@OyyQRUw%yq0c$;IJ0DVEx%AAt_Gcby}5HDfM3G>6E?o$0R&hq@aP0q&*$8H%mTY-GM=gXrA zDo(@Uj!%Dqiq)*B1(oW{=uLADFV&u;YFx_h%2a(k|IpWO2SY_})SM1wiwApO7%3q0 zC}ZHjVgN2?HKgUuu@nC>FxP?KM*C-B4yMkSR?b#ANauHFj*T$@g@Jz~-2B;vX&~#Z z0j=G=a`h`e3)A*m)dcJ#tKobb8D!sRDH~k^%=NmWsUAMW}?@jpVLvh(&HMul` zTefh!++|dH#dX?5jKi1ecfBzg1GL_P>!n)efc+G?gX9+KFzD^?c5Q?N2P54B!jS*@ z^DEN~*LHMk8rmGTI6^PA99zCe83&wWz=MsT;8OlAz#!%Dly}F?`?_iXYyk{qMUwJD zu!*sq-ToHVN`x@x))2ZK(B^*|Y;SqWAQ&T}Zjyx~;oWMq;1WX^92AHtJ|g}Bvu$XiHj2bRKD;}HIj}}p{Ro#?SNAdbC4#{-z&k+( z^6)@*eCxn>Fa!E;m>7a-R&+Z#h$qfShjfX27HvZbv5IWs^!p@V99`pSjdE$>2fQry zDS_!4Q7bUG^aQzEU+ukUu1ui*N68lu+xjiH^&js7z4i%Oq4l*i%q#TXXsg@6`_hUH zPW(8X-+@=9`@(K>*)H1q{Us=6GK{a2I`l4%eGvr1QwP80TII@vU0xRn2mhLfHPcp1 zQ3J&u?Q>c9nU{yEVx>L6oxKAi_iFK}oa{RGiP~e^YWts27!Uc>IT<0Dkqhe^bJpEg zda@Sq8)nz+e_m>5Cu8!S!}}Y>%W%u#@ptcjXt!972QMrnp z@40H&PBpOco@zP^@1g;Je)-XpQA?}o& zLAN2Xebd0*5&V9nl!>hOHr`n3*0tNSJ|2rsmF@@Vs_#10TfqxSucS9t&6he;+JD_` z!48W}9-NplzL$o4O52J-zag=+qNcl3)r0wI2MHBdi=5XL^r`B;S7Ba%&q(0E74deB ze4EnrjN=t3JPc%>`3<>WBs8ns!Lt9{DP(czO|!1a<#kNe>pqa$-xH^?sU3TGWzDiI zIM@tOK0Q&<9u<35GA1eK%&QP=$h<$s$e$xO=!WbF#FO%1IRtG37R5%@Fp~vC8d=0u zz+i3ylhdjKV>i2=m@2zTi@9h1P~P>A&vwFc7>Yku@Tp$tnm`t&{SN_(3b8>#Vli)w zv`-xZ#(RGCc6c?@3R5Wf=YZWFG+>~L-?>6+lJEldEh1O}MK!QpI{odk|Lj92f513l z6T>D={>c`18o(PMu+;8AY0d(J9zYJdVh|T~k zx)IGAIDpsnCj$(C{98b=gpkz%l%aoo=vg?>@DcG23{UB{&}aro(M4Ru!Knjz6kyXK z#7^Qzz-~JWYFX6alZ%@B#w?sUWVlu-4$u7JYXbc-$Fig6c7Jci#W(M5`pU%rdF2aH z{j12{!}sBqio|}%m~L0c$2L7DcrE4WIFz!ndkK74P4y?m;B;OBe%_9;OmEE3*leW3z%EmB6L;CU;Z@Z5Prp30yq>3m)}!ANW2*9>$w8H!As@_~ci5k)J)8>}uPX4j-` z1(`?P<(kZ#KN0vJ9G2NKY*972?c;ZD_+ZqKNw`m@q-a}Sz?eOHx&B7eQC(MYn)F!> z`j1rmr~n`FftBZYr^9GpABvk+GBe8=co*j2uUaU1MFR<4!vCo}!ENy4ylJR75x`PpYoKVkYf9q3^u>OI1yPsRdpiqgd46~okLlT$xt$;fUv2|F| zs`WuXy`2>jw!QX*5qI)bW8`aj+Q=>Ol~XX6b-}IjrA#;3Wo+QQp3y@L^s&$}v)z19gWQZ8$BsnCpZeGa6&3{ONj+aKj zZMoV+r7zaI(0D#mNZY+!_fZB1zwy#~dPZ5>Trq~B;>-O~%=%Gd2R9$vS*{bu3NFdp z88hwe#WT$pk9>zE*A?FV~`n8!yR1)zvqZ=YPmaw^`sb@E%9 z_-B!?_ascA9VzzNa+n_|r4bvb(W{uz}}~@zc^h~Rv&fo zUUKtd!)OkH$y|(u>1*^P>6siqA$!0CkXL4Nj&5ENzz%F^|3SUG_V&tdZlZ7ypk#Wr zdcbU^(kvQy+qGcb@zDg*?g4z7c9nvt^(&=W4}*hwPM;=+gC*4%nu)1{Hbh+y22|_s zBUQ3O@9=3AYz|LZi#>-e5|fe=3FB$7h&}?GZZkh3g&JV9JL5L(?S?)cMUI(g&zi)^ z9t40=kBO3`{mV^ZO`B^#r}%DLl6u+5=$W=7?!&->gy%9BusjU+ft;YArWPuZ;rks< z$Yq57nO}Illo}A;X9x1{Vz|g4>gy0VW(e6OK^i<_#|~dK-QzZUFpl0n*s6vj*cMEM zZufrdSa2VQSpbwBjesB*gFj@`FPRiBI!z5n$8!Kf=@=N00Grqn3(iMyH1bGGv!00u z?~x;P9j+gHZwL3Cju?&&Z3;ar^>p|<5OWa2I2ZGF&wWL zq)EW0{l8vqbbHF@|Lc9on_;u7zi#h!qT{{$&!0bl8RAsY%}Wu6)z$k0zqXg_2DZTf z^)on>Up&L^CzKZz#r+->hA!r9f~VzjJN90E8|24P*d=M3hx|HNIVHEqW6 z5lq#?q|0rnfXlf2m=97M6suXPZCBlL$RAvn*Dz)>+sDq)zM)!8s#^5vlh`+4{{nP?}mj(syi>gYGMLi3<$ zZ0wNi+=4h(CJHR%NKgW%ERkzd=EqKS22rT^?z`pZ?ql&d^}P-Ab3XG^DV;M7Vf0&0 zztrb=bG1u>yY*?a|88$J2F(@~lI8b(n`ZfWEPT%S2>B%ZAq=A5hu^{h5D=Wh|gd2EX zLLxLEAW_ujrApR{Ed(jHyTQQ)I-O%cWRL(&O3xfUN`sFcNd<+mxFrObU0AH5j|^$~ zr3lG*lq7*ut8H5?MwgBsfs`(V{BCc-`}fC|!a1YCk0vWWza<|5w_oJuhCgZAg8FR? zXmvqcWM&)SFk%75jXc>&rgmbia`?S_;LBm%`yL+Fe18drf`UTyv$&igJa;- z`w);&u*0GQSxSC%+wD4-4_+}biGZiU0}Z3rV%*8h6_7qdxaLJ?=kri9s?77ptH-9M zMnTFAoMvyVLql&D3x?C*GX2>BkF*UXE?v+>v30SGQ@i>cUzpN^lJf@=z2DEdGw@`d zekb@dZQwHC>Hqbz>0c(h^VePQkDrfpo8vcb?fJ`a2Pr`*PxaDQsW$Y|6&yrJAqt&e8Mv9jBHz2e1l;3B=IL-uNB_=N7-` zLJ6f>Ha+@Mx*w%v<3-wl?urY=Gc+!ol76Eo8tkNXRl?> z{PP_U>qX)9$PcFLtU3DdV^wEdMV`qlnBwnJxhh|Zp68Fdd}TdOrZQ~FmGKGHBX?#B3hy8~^L-cS*2)P4+ZaI*Tnl*PA zqZuBOnq44sAgJv)*d8scB6N$Hjr9}Tm!NU$8ThW>L4Z1ie?_3Z?_A4^quwxH=|(4b z_Y*B_SaYSRaxIb z=KCjrGy8*gcFj#>5+3j1hXKBk=eUR9N#m&AiQHmlw z(iukWCRM)7Ku)~t4FwC3lZ(NjSpd8Ozu8^R?Q3%=&4LdHxI>7E(#FAIvmYa3Ae=}E z=7CU9zz;PwwE*>vX}{MO6qJ>TK|xY!3v^$|vpsS$h6bE?8^NUZ5cqChHUD783~ACe zZWG6+7Z)2qX&C`A6BeCdSp`?Ad(*E$>j{bp7fIlSM}eyr2&u@$9R`^;Wx;WmCe3e? z8K&Kj!26pQluyY;N)u~9NrQ?a7^bh!G;-*0B*etrXf!8C2vy|eYu1U5^YC0TwYgyB zg4vK)I?h2*^TptTnURoJ_5F;nYqT%q;Sg=8U2$aP<5_j7gO)QwL~)-NaNtsk6Asr zV^l8e(4~Rj!aLN*@=`fT4cXYvHWph^^;XGxb?tC8E+!4?x!qONKec;lIMi?T!-?^^ zXa}F}5BR<#YPWtgev7M8p{Pa?zYe5^4CPn!r&}!3WR}IDujyS?-QnE~Gh=#m^qlnw zY)_Q@T}>J&sJR7s0dJ_AtS`~Fm06f?yY&6&iO3`iG9nbJ8*_KnRCdyT#?164RA3YUZQMXrL;(ynyt z^c<6-(+hl&e-HJ~8Vf(q&y&`8OG#+6RNI6ex8U++doE_~7cWc&Wqpe)m!r$$AJf6P zQJ_Y4Go+2mTwQv2-}UX|Xh;P#HQDz?C-MQtTLz>1acJdNS(pl5ydZKZX9J_Q)b2nM z;}i!Iy9-&bTfWvx*{GBcSgV?*v3rgB8_{RoPup9;vWERRy?r4SjtefaC+a7FG5Bn< z!?vX9>sLh;6_UOE5J$|G=;5G40PvN8^`ec|G`n4qw`a=rAIWTrXY4&?1aLo_$qgL$LWliug9G# zI0oG(_HNW$Ka7@$*5FoN|EweO6qIaIOPUAG=I_K$)}GUmw5|TksO#*Et-J5)U@FA3 zcztK?O{qmuoKOQ6mqTig6i6p zHLl_PyyNVro{*8IQNkCq+M?cXLQax!-wSkKAK@*c?!9uaysy)&il5yj$WCKz5?x7K zVqS_u%{s_1xyGEl&dwS+Ir_2k#e^IiH6w$S@1UQii;J)sVBWsK8U%-n)Lme&a3!=9 zThs+#+xkdLM6k=j`mL0@c0PHq-9K}N2UXU0OF6PR*sb{Bdr;o(A)oj^+u}Ck^R{5H z37OvT77-DVr&YmMa{yrf7`^x!Z$?S|*1CjxfN(^!FqcEoj`+u`eDiXqB@42(DjY|##A|7n= zt&=cziH?YXFOK}LI|r^ExGqw^zkTrFL3+2XY(i60)3tK@{!yn9@RD(oU^CkQNeBxx zxIEdbXH~N3IQ_S|k0SK~rP@ zEB!MQlarb0>8R0BUQTOuC8c-KTx1ehspIaHP$oK8bw?N;BXO>H+%&d#RE^wV%puKm z{*;nZ9G|FBm=PwZ_xyhqknpBW)F`vszuz6ICi3MA4XGR%wE_3JbG9051=kaVY|}0$ z7|Zmmt!2?3{e_-1xt!22l3&OI?O$POq*s-yb9tK0rp1xB5cjM2CPsn%WqnNZW`!3I zths>Ykie;HBz*Y#!n~I8hOToatPcUgS{DIFk|akrI2(+9E|zo9PKInpcv*dmjX4@M z7qW;#MO;m>N;-M|q)Thd)-B<32Im8JnR8XiYx2>Ta87>`F_HDrUkTA8X$N-*$HobjHAJ1k2^#s%fuGfc z=&5BeYUPH74)_aQr{){n?UPz8q#d^IQNxKgBtp`}Y!@gqGY1}JZljd$Zm;qFId)dr zwQ}i!pZ0xj<7GiXP^aXF1s7uC-v^$4SMR3kF3gZ5;HXZ2w>EkZ3S}Hvg@!TClGiu- zECt%w+XuD$dH{FF>5lC?BEz_QL<#}jwn4@P{;g~QN<6i|sf&g9B7n{13wwMtJt~IE zJXd%i@uV0=Vt~(M)Zfxa#t$s0sH&18&62Q?kWj`gNP**)UUvW9xIRvud=MSGFNt+DU>UK7$Lk=zpB?zu(AgZn!{1|EG2Rk6&VK7Ki56OtiIWSl4_{ zpU%)Z^}bJDQPAJvWhNBjY{VOM2mv7P)f!5(n;ELzPA6fUS>A4=>}xh`U0s9OZ%o zhYZS+9hxS7G;SL1@YB+YeP|r&$qS5+In9AFufCMG(=b;n>{Uo=?zgyz;qUSET@%}t zYR%fDVH?L8Wnb|uS%~)JkXlUS^0gABPoR^_xfMO}yggP!P`OOZrghC9x3ah-#OvbU zL~or$VdlwA3^|xyuI|PtRGObVWA(m0`_DI;HIr>&M(_9tc6P4loU#6v{u%}$p1_1! z-kD1C=?2XuO_1V!m?VTgv2!s>{!`hUac9Up-61?t@8h8X&1zi|6V;x|cI$~hk2^`- zp_CQQP*~e#emJq$KUW;%AY?4`(W9gXuAP^8VxbRZ1aZ`A7p%w?NNZ5Y)(v}RX9*5a zmHn_SwwAD}z|l|(YtSy>sKSE1lFZu!d!Utp_>>VrA+uDYKi?Gg(gg3i2JkeCdHx*y zczA3?*rua$LON%ZAOnh$gb;p2sPH2B(Fwfq>U8w@(!QIM2%!l)xW$A!342&Ta$`X36rna_UK96jUf$ekIy(wx~)nK76 zWNDtL5e&o-WL1OJok`;&9bLyfFWDi%_f%o*fuT0)DUR%)_vDGIagPKBwWB^cp$8|N zx0FOE1frG!7zpv?{@1$BoN3JJ`JbmZUuPYCcyRD;q?cTt{)3SlM?ZcPh+gOftfqgs zvb(pU)LW6+$0u!mBKI?Wn{2Y;3DzwcmtWgcbPt=gsJBBR_a!%Jyn1_nIC6>&4Q@Hz zcoqB@x83r^=VWq=@D0@(vO;uxwdLoQO;LKUv#0=}u)>?>zKiq6NR!=5bNwa_e@U$6 zuZG?7uAkc&n4tIMBt&tnRP9ZUlSAQ>MdL2X@@{5S*i=jW;C6Er`MJ9S(WJ7pwCVMY zoM?c1_SpiFwxi|5$ow_JZ>gdvln9HYLBXTOwf1xME$7ZDsx|fY`xe|eQxB|x znX72GSq7w)6S&5s`!emIECa!Ix9-Ky7LS-6X8kT;a|4KqhJc#77pl zmaKIWSbQ;QY3E$tuIS}=lhM$q0#_5M(CHW%Pg=fOxpImDpf9I#lhmX>I~7Q!e3noQ zAzgdml*0`ecK|R{y?d+r;iLyV$`S-d@X05|q>@|>@{dAofs`aHkv4bktaxii$17$a zwa$Cvv+x6m`RV#+a=BJoJ32cr*r5{@Gird{NET$d$rdhj1qPkHrH-7h4VJ9#fuBzi z=tb4Cbz>j5Q3L3YVDYWD*NW~y>}L%KG`AClDVlByBP$aiauDj*LenwoDpk1)^u7^r zny3eZttMgF!=J)H)NuuGO-)V?02u+O_(qqOAS5j;w5exkwj`rY z_yCWGlKnAtI7O+?LKB%@82hbn*XgqqBAZDrL|328WY*wKxZ(C*Wu6IXaR2!dQEmQ0 zE&tE={eJ6SLfhu@!=Gp3?M&99eqwLF)jZW%n%MTOLpoA^#7j?>Vcat+el{eey#H*J zMP~uw=A~vL6m>W~nctpkUIgK)1|O_cfw;ux-J5d}D{ra1s?qvMCpqiCNlAD`hxjFN zMkhMdk(^b(>@i+=Fna`2tEdGNwn9St=Ifi|A|<@u(&1;&ruCo$6W$wR43a}T^}@Xi zPs^nA^!MDtc5s`v{7blM~77G%7IA#sar%tOC3LrrlAG zT~l5j1-vz9xL>skfbgYae|J9zh{3LIZeIN$P~*vAMu#aKic|@_f1g0R&9XScM-sZr z=VJB`v~ufUP`iPN65$c48K|!+4?nxCO?Y9tOl1TvDT=KK3Ssh&G z&(rY6Qx@WMkA_GB{QEMmW)6FPg6ad;i za$)8zWsN6XWRRHK0ErXCqt{6V^YqVr0$q!mrsmPp?3zd8)q_BWk`DDPC_gWHdF>R* zK@Ea{T16l*VK{n}TT04%H}FUYJ=E(YfdruBKrR}P354zgH3fRo0C;YzJ}h)}Ue4r) z1WOWCm(<|=G_hk-z{SiG$PA!_gE?ZqbT_E+pvpr z-i-e`a$)Y4N5}AD)r6C&DEj-ovV((I++7hD?X(!$qmylV(dd!mv?@!&))IDtFGZ}R z39^R9)=dmw9j(jZ-!ycyw<`FMqSA7yGgUhok5?=^Edke>PSK5P4wvKyjCmoqbDV4X|rC7r_uzK8v}^ z-mJjD3Se^yur)-EyeoQofnfN3sYUQ9$fp58Z|rn%a8R+fJ_(;rNkjBJEFYNFZ6Py8 zPYXUZ_~;uqYC1m{3DBDv=<737ZB;I101^N$DffpKE6qP;WDwvt9r3}yH@^Wc<8YAD zSDj5_B~p<0@DK+M(hU)o4j6RCHH&223W5X#1n`F=bliQ6E?LU$Fjxwhl|Zx=Sek@B zY4Uic!S@N^2PU?4jtB`@%t#3Rgi9)meI~48&_K;@^zHh9K}K)MT}E)G$VD@0K2J~Q zk^v>C(MQ+QT4{2TU}J0H2Q>X0G-O29p6-clW4X z!vX`|bLVKFxF@5cO38Vsq$JX*ztwx=ZN(Qr{Y1`cxQjb%^rk3`B0zl{=f+o%3!JabHBJ7Pc93qX3UJsN~0ZoXPu-{Bp`Roa35~401dv2V9=@7tRi;9W0ZfKYZk{2TXmIOEFmB( zSfT5gU2i6peEI`Iu0QpB$M&@X128&)o@acx(ubDFqRR_Z*Xf-zG4b(1BO@bMaE!-} zjX`FKP@+aEVw@eDlyo@J?zUs65m*WA?eCu#5t+yXGpekYYJ2;F90y zk?*d$h(AkYL%zM(^E4YTz~L@G6{rPw@?~(y83hrPS!asYNf7RY!$z?Kb$qXt+P&v3 zzdnPr%I^a-tcouyD=Ri8CIFmpU;`16kZ8H|_szkFmN}0H8$#Lx8{h^vAahduuiHzkH9#OpRRJn)9YRhWdS>g} zWm{r6V40XqkQ(^l!7`}qN5OAK!hN0z?)1tW4_DU_Q1K}H`^$;h_Yp7ZX~ zAQ^@Qdqao`jQlBqRl4{Kio*IYUto|5M=K9E_x-{=a^XY`39vvg)D%pxf@@mmj4GP% zwix7!)gudcU|{ZQbXZvJ-NH(ec=Zz?O)rFD4)959U>_FU0mQ8V3hMH6)=`j4$_*;G z0H|w{r9G$#35Odyfb@=xhLFw*TN1QRHLDWd{wmy)C|)bE#zq^lG= zC{}1$S*4MB(;a%2%*oSdvs5iqM@+nIE++PGa;o^ETMO_4_x9Zvi=YF58N@iu8 znWAhzgZ^HZ|MzSC7v8@Ad+Y!IPEDjG0nJ8RyKroI0Q+-vqvwK3%2KEHQdGb7zW3QE z_6kx|nQu9D5d)o7SjK}_5MSQP<^MSU@x^)_3V!WpF#A!GhS zfdqrVd0I|Ef5l|p|1!y_bdiVPA-13&{@lgV(|3;i(26i0sZ-d6p@&sc?k@xJ!EcR) z&#NzGcy9NNxJRFRZgen?F^TZNc4uW~(xaT_3`(2A8mvKQ2#97G*u3UWx>q=Zmi!Z_ z#o$=X2%X^&k{*)yDH8h8zXCSBZ&KTdk(U;<9vbd;nv?;zk5oJUeg4jasY^$1FBuSy zz`dvL_Pa=_Xq7C5+qe1P+XOvRn$2?$cmp!3f}!5jUI0Z9bUDFkY1oGea}+=cZ~~)J zZ67cL2V_j`@@0CM8^Q>`zOgau&6{s}ccF%W3n4VGK^0g@J-5W7?J*5TAmQMKks>U? zIvBfikqzVIVZR`yuwCC81Vb7paQHAP7!I6sfDZ<8HbA>nP!@xp@S;cQRuyDTsC_!vox5;w@0td(JVEX> z;!^%;7OH9Bhq4I^tDujC-xi0;9V#E#)1$ygDs%aDc;0=Bnil_-Kq=Zoy@ z6=~uQ5fA}h1SAaUqW}`7!vzNgWfT?BLRC?BzzMBBOBgVr`W@(idlY->P095bme&*b z3b`Aa5hSIWVnz+y&uMsg&fl3h@;a;|6!3$8y~t#EiT+*H(uGatjl*5mFHOf$dB{v(z?qD7%y^I*L-nQK3fK(r0Ay9_wxh>vmrVMhBXwANffjO+>K0n>En8 z*ZCzbDIq7gP|iaVn;3mJtqX@h7bpxe_7o@f`WysxheoU^heV1ouLA?%qg zf`45x+;SSI>abxr-}cwU{EP5TpiI908DuTP!M_btN0hGTG>(N61OB0EZ4CnqC`7=#DOjcMzU;}*zg$Ma(gxZ!4zaT$A`GBnQiQ}8vgRhk z(+r1SUwcP_Oe>(@(lItZtgtNirl6vN6%P6UgM8&HSLom$rL2Bg1mOjOE;Xa9jLBnh zka;t84SO zi!NtiwcK~^655qh;^8SSh~5ZEOU8BmYrTi6`MMMn(*AqNcMN9hHm_uH-Wp#zSR_Ho z7cPB1=kH8_U&2=$#O~XRorJXeQ^CP(S%N+Xd$d5u@YdI<2WyP$0BZdjHmz#AiO0q}emRO(D zSc3Sc!N{j2Bdn`00namD@8nKW$?T&~F_deDh(~+4DO4$p7(H?|`Vjd-{4y zh1IvjC+ki_ep?Rb?8Hv$^SowcVa0utyRO1Rc4&P!BycOzP0pML2Enj6%S;4e@a8oO z&@QLnI-MD2-_53^|4!S)F(MQD(h-;E%+Z{SVWAw>HKPq79=M`2DuKVQ*s4h~LH~}t zI+8nwhN$3Dfx6n&!y~%i{7gI#0Iv`r3715mV#c$_CNQ)xdT&DqdOVquxt_u-I6!M_ zFF4|6#lbx#vmAz*RX$sSuv6(^rUr1_Wi_>BTQD%FopDtZ+)CN^Et^fikq__(Kqg=W`Jlr_34Ht@ zC7SR3Yp_9q^1dA+E{Kjel&0u{kPoTY49^M(P$H(Noza!R6@v^D1ab7~hHMVt0xvRe zCH~@v0@w*IUzpodJ^uh;$NXS9y={px3{mKbh=^{0^WEPqD7Xnw&RGdDA^aQ`;0g4W zIrA!Byr`_EF5&|MKFAD9fAQjG7d{Q**owjPbz*9&eH(8P2D`&>AlISdC|!q!(?%8X zrTgl|R4h;?m6AFwLG<~mM#I_7B3k^jYar9%Z+)BYQR2GYxZ8&=?=rDaU+ zKIj0mJvaN>o!4$xK9%J~Iqy`V1zVX^>@9gzsbL9@j5-WxnH}YdG>uD3afcqW3+v6WWNNRgS41KVf zq9yLX624aESJ9~FvsfM!y|B~*of??KQ>SQ6T?LOQ1U&oxV|BY{`6jDtnREIZ&V>F} z+MET0u-lvjHQ8bpfHZ9N{rswj!ym*ai5=E5QjdKCVd*x6sDy#>U4&Gbx1}WyeBLN? zz&t&{Vj(s@o;XPHe6}P|rpTJb#n_}%w34N_rtK##Trw~?2Kty>@X7{v`O}tQ)DRxN z=IuY|AZ~n}v+`UG;~!F3>W_gq^=zYJOX7oN8aypf$?I~Y*{BhwImDHIi|}Tf0zYW- zz}f(~L&rg!R*%K+W5*$+@H3#SSFTOuPE{lEfnvS~sVwkW8n8&YL`Xc-vR zpGn?14%iW5pbhYnh+RKufLHqO7EwX9VKD_ZD&!_aVD=g@aEYE&*~4GU@3sUVzU?^l4!vD(5dlzN|GN3IesYs0k>ujH zjd*y>e~3)W5771mRmtqTlcWv~ z)S;KW&|9{`1^Wjop^!P}Z7117hZxAHHZEG{5*y^xqqySLmk)5)SN50ENZZ)+UeNy#4K>%}dUCcSdiYf|q$S3fHcY zp5d3~CgYuoilnM}diE0ZnjEN+@#wow(>iA#KYlQ>gm*%HiFu-9x08QVdsnFK@LY9T zh5#WP=Px~11TJsXny94Z-?l-)p?lKYi{L9s6_Z;v_gD!K*Zd0L7bszq8K*PGc zgqDbc(SNcSW7I*KWisCF1R^8hJ4>0wbrf#sTQyM*7tzLdeZG5FusZad(iYM@4Rh4Y z#}`+A;Ef`q55^9eP>Vy4PdB)9s-0-w%w+!&F$VH1EsC9S_Jwg=xQ_!-gj@NAE>aB_`D_yTfxi= zr|sw8ns(rjx4I^cO5a{He!^Z*_c^;*?YlA;R~LD~S|oBW0QSFq$2ziM~A zo1?leRbW{0T#U&1fU3E3Z&VyD8y4Ya+$s8jlz7iTqC9e~6L^prYJyB7GwvhmM? z%)E5F#2Ar^0IO%P!oyKx4Zy04fOlNDp!YLj);Z!0;^g-C1Q8)k#v%R=j!hIJj!amP}6GhPxIha4CN%aCgdw08WA8jlaxo z?ie^UMrCj>?eie}7@UubfcK5;I`tE!-C-cr{cI!h`G!!<4WabxY+}KLyMrw-s0)pb z4l&3Nf&vzb^fYt2@*xKVi>VvNAsoMswDa0ufdmBh8NszTmS2&QkXS;PI2k!P-N|0c z+#c9l%NBqU6@Jv1oFn{h zk6GQlcE9`m{bs0u4v7GByfHe*PPlKJbP-pI;s3n5K?;L!QWUUHU*=!%+PGv$g*LgD z-ZI}EWh&wbY{ZX|=&uUaR-Eg(EFFEl(f5}0;1aW+*toHqMD*JCEFq#+L(vkY z?(?m~T101M;1&dy zDUAJ|KeqQBN6c!VQmk<(^|1v;#%I7Kt=B(<19qWrAUC-5v#$@C3<3ZX{>*j{TDgwl z;l(?wkt%eZ5z)pAmD=rgV(6Ty3BX^QT3eNX0*DMpfEQgj^MGpvDrHEht%c-1z}cnj z`^cD>m;!2V$U~hB#0=QI+<^&NK38aK=6y1UOGcvUSvo&Qkm3Ki( zJomM7FM4}RISiKFhf@VKaNZlmK!*jE0I=zRoJax~P22QRhmu}FX(=OEx}DKt<%$P zYn&^zMid{#`|ta?^z}d8uk^`xTDKuw-xXY6H?k;}ko9{J%@mkjHBmM7QR$G~Zr~u@ zTI(_0M25nHCAYH2J`{}Dd__#-^8K9$JR-g8<0cGp=*{~p@Wytt&pk552zg}i_XT-M#e{G+kt+yFEZHsk&ejDYNzeYmKjXU&) z<7MrJ_ELK3t>t3B6T0M3$zsS=C4OZtwmmo^%gEe&lFYcpL{-$9S@Mqqmy%SOC4`4w`8k;u(@S$U2c{Ds6GVrverG<`@^W0wzFYxX1a)$uYSD^Ks z6 zt376qD((I^j1=1*OZHhUcNrOG|MzQEwxmQ>?Xpi#XO-{H(bp!-;*&eqjY*sqj9L2! zLcR~&4b(lY3?=j9xQj9IN25Z?+NLR(v$8Y^zi)Zc{M_!tmpvG3=cJU&;8*_)BBr4i zL#gLZ%la~g9Lycn#^K9%a;82GoJ|MG`A~yD<%UrPwJx`%ztH2=Os;sjzGl`C?*f}w z(>3x5#83TMkL!}}9eP3*Ng=QALiT{yZ|(T!TaB|T%~%^~rijxNR-Sy6&I3;u6_#+a39P{kB42f~IgLPM&^` z%fqBW=2Z71=X0GFZkc(RggGC$DHC6V6#fYd@q;W8;L$!W3;7jo)d{(22m92N;3j`$~)X z!H@jScWL{S5ZpFK#1aB_w|hvuli>p5B}m6~>e~T1><6%Z83ERiQ+co>g<+mP0^-0t zOQL9)9!T_M7`)tXdM*&*fu8$E$q08`T=LVL&!8Sb2<(MZrcj{+A|QxBtr~nBr2~%sM_jqO?jypJ)6YpJi zqiNMjeUKD7y|wr2$ezV*rkmQ^xD!b-yAJ$OEGpP@my|@hKinbw&2ymq?`;EI8jA@_ zQm%i$pNFN;s&Ucl&E@PP6O$?tcMrCa{r`jH_^Ro=!9bvW%Zd(WP5xvR-sv}!?!pQ&dV$z0FfV*kqC zOF3zZ88tANe}J?@K(b2p2ATlO@3V`2TB-9A{={AN(~*0RETC!nwHS7--)RC)=DH#S9&di(gOX0u|AJ-+K?NX*asy zWWEA=AYf#NKbvm`Qp*TXY*Ie}qzDaKY)Z;mIPM`%Ay6gjexKcD7t~scii-6VqAF1R z7`54n@d`P4Z(hPYhxP;dO_2n+ zJ}qIUhp;Vy%s?DiEA+Y@9QvV-4TshD+O*;r0RaKe@e|0*1L@s>0|uE3!7aa7it#YX zxng7#3eyF(>>qd3FDWSz4bHr)o1Zuzr=)}xnPAWXRhIWl_4!ll(!gvOJl}Vv2Ys-x z2nvZ?7BNSzS?t;UkEMt`&M1xg4M_jgFv2%wz$TgUwx^ZSV7m8VH9IVwKyRX=cxm9a5JrkmwX9qJF1voFo5o&Ha!5b`fi; z{tr2iA9<7YhbvUyg~cKc3D9vD+B@@JQdftfAE|}AY&RHACbJ+k0{?xfMQrs_9z0lT zK+|>3_<2;1t33z|j7?6C0LO4`2m(d;20S?|oG`8dMj?<>y={SyDwU#DuWtps>c2?` z06EkBKf101F6VZCC())QQCgZts3bI0Pg6=EB&9+_8k(p`dutF)o)U?-kkY0hEA^;E zX_L?tn)qLLanAdHU#CwWr#j~}p8NiN$MwCg>&vYSH$x?o)?vV>=+a%4tB>ab!bIOd zP{%R- zykTb}dT--*FM-P^;bD^=1@#N*Q2>!M;CJ0qcuXAX&*wwD@<%obEWk^K&O2hVJP$yF zqoWLA7$6uDhCxV4TR+dK&5Jl=))c)yq(&|g-^x*UvJA3682f>R$9}?s=>AzjMIm*zCRFptM6VS zNsT?8b~d-{?Sn1B&@?Cc#;;Tt{j6Df)4%WN5(YYlX(O3~$Fd%UhGZIOjQD>&;AO?Z z>tvYvxpl`AFRNDDtfxKYb@Duj${&}(d(m0 z*Y8o}7to(>SPcV`O}93x&>fR*Zus>(H(CjuQ9EVfwv*r0^;LX?Yud%=>*ROpmH zt9OM=K0%vTzKiwf-mr{?C^5zBLF#tAoKX5I4`4|UmSu=SAcEaQao|Kj`eaxP_u4A? zo^+FJ%e$t^NAJ8|WtA8VAR9_u5)wgq1)rK23+h50n+cT9K9I0;m7ya|!UCVJvcH_i9uNC?Wfk1@eqJ7?L4$0hPeq z9pwDo=EvtDs*xM@2N!@NCf-8n7j=QlV&1ExtE-0LJ#i`N-yGfMSX1cwe!H>p8Stcm zIR~ft%OAnWDjj1_a$P&V3yH5-!Gf)dm=kYbajqdA6e-(#ASKso7+?y+IUxpBb_%bg zXrt1$@|^ldSIFd9sC~d+*n>|rafcXPsxC`FV)+M1jQ{y2gkh9l&FkN`41bGJG1pXn z+?(Gq)xO*7-8oIKk!tx>`*{7&a;VmQ`AbGFN~k=ze5#oLo_lud4vSl&ya_U;^jVFb zj9%CK^SdutZtv%#O8vhQ)XOr&_Ib;7_ATF(%4wTp4Z(vXLvC7A)4N^KQBedvzhsiF zo!s&9SkIlHIehb%?N|~QBB8^7={_qfeV1fUXvwb_l^I*vwPhBda2^}nf6weMqxa10 z?12VX{Sm9YyvF4-l6{HR2z80lQ7BPVC(~8c4;;7@k|psRG^(G|X+P%@%F|7Zp)vMb z48n-Kf%m6ze~_*ZiOdl!8z-iap856V(mvWj>p+=0j8ei!J)5N^HugZ)g~O z^Z{Xx5Caf4u5Nt&g^OVhnDb# zlutJ0Kt}k?W&8E9FrnDSivTWDAq7Rz#C_t zivy5mbh8L2=R^SaL$!GKy)%?&?bYBs^#;)0vaT;MD|C8zMd1#_f3^$ua;HBCvyJ?d z5HLim$|CU>qvvmzh%?48cTcj{Slfz{DaS>02ild^V}8KLmzxf+EVbdmoTmBR-^U8n zS9~2%6QJiYzO|f#v)0aRVV%P-xW;!hspC>XLGzAsRJCM%6Uh_4zP{1Pp7mW1CBORg zX{@N!LEyRLyY?$I zKxiN}#@nKFf-C({zV|?y#=G%#6)X@yR_0l}_@q5STFKt+7Z-00h|9<<;0mo8O-6O{ z@}&t;*zr5C&2Z)%zSa84y{}7)CMz*W--#0>UIOb&Z{OxZ|4Nt*&^g%Dp1A++*kuUf z=KDWzy=7XXs#pMT5QwMqOmDvLh6-X{=;OtJ-Pe?j(}TEa{uOaS0~sk^&kWm%?!z~_ z#2(25V<-xiIqPH{CRT6%)=|5?yb)*jZTo-v1Lwgv8v)y~eZq2$88jZoDhRkUybRsY zhfdDU93UrA5ST#p)9wc;F9J8@?Ynm)#pH(Y0EGA`*gAt_dYHS=!*gGK+tK3(e_l;Z zjJEhWxj7`1`NBtA!&RM}W04^tnG2gM#ySY#?uFX=zbf56W0ZhS-RDWX$Z0A-cVJfEY%=RN^$sbb*b>7_`!9M~bSg9&}X( zW7Zjp<;Wsf5uTC3ZFA=}*y+FMlrLV;Uv1b3I_0}#cwBK2L_t4C?Na$oxBkrtxVwbs zmQCjVkR|J7P`Zmi9V5Br#t&@&S+1>ps+j@(t8k9 z6Z{RUK+1dH`11vz64%DW{@mhuGkHnAc;EdtmCd&+rg1SkgwN>=yBQ0&-~14Wv5%~1 z;-;k&{$C?^igIpd`FqTm{g`&`fAn@Ox0Wy3wacj+q;Rv+mN1MD^70p0IoHp*sDCDR zt*Y%^wLQfOf}1@vskv*l`X5>I3ctV4O`%w^Y%RR(8wbn#CP@wyzsrSyF}AKVI7u^LD8s=4No)f z470b!5V9pf`@ph4te3Hx?Z6z^Pdj!8GP1TMO;d8KA#yp73eb%LV)#PWE{JmB z(M9=x*U2c2^ik3O%%So-3f=;MFp$C3S$C|r_OaR=n`{_U^tTl+26@K(D_(puieH=9 zHr{WpD&QDH4{eZV+grt#EKYv{4y$G%zxT!F*>3Uz| z$9rlBoixHdb@x%!279p8NGCM)8DjPb2!h93Kp2>Tc`z8%U+)2Y7BBO~t2b%3glkao zzTm6HJ?A;d8e@4_YdG>`WlqRq+hjtc=&oW$Xi`0P0zL$UJW*9uWoMv}EX>sR;e#J$ zh-PT~!5ixAJn-LoRY~d1dy6CT-vN!1rC^tHa_X|nZ0ZDwy=unJZfsqyM(q;RRB*I4 z2*I5kHLMFjyz_cHb`KtZM6R-ZMPYfsVHqGogk7zCk{7JO=>Bw?ZLVY6CLlO`xJ-%M zT!h0UB`r;CbdXs?e6U)XXsDo3mEsAr)6+pLaPeQT;m4KhjML0>Ica@jbiRp;g6INw zqJw5yyf1jzy;u|4J$crfA;#9*Zj}GAu6U>~<3ab&suiD8U+aUou|-C+ymB?Mx!NeX zMdqDw>>Yi)SlDvrkC{(})p0NC2pZCUWfx}UTbqWt6Ks2r-ShKj|NbhIeam4!M%`a% z?4&|?^>6br6#DQC07!w>mh>kF4s)Nlvdr+^75m{4<$rI9uUn35wjg1?G25$b&y{gxqB+9HP;4+_!dIZ0$Eq_ z#1|>Zf=V(y-Q6w07JKycsRpV9blZ7{U;$?9Y-l!TyK`#0Ioqtm&&q%W)~ zG&v$f!hatf7zP0i>~906LVwRmblecxz^o(BGN^rT25Dm((O5x5fp<%of;XH!c6i5% zb9)fMqXaf~XSHF20(yPy9O7K}x(ZXrRi=4N1&(c>8?ldX`<^{gSP=yB4?LXc5QCMG z_4`J14&7}z*=dYM)Hqf$4lSrEKwP5sCgnNPnX&CJu>%P8IL5o9%P>k*6O9267u6@~ z5#(3$9;r2pmw~g0dJ}lrtjSJgsw9csn(7K*-mfn zae~XlEkb(+ogSHBf>^}1uN>|l{}oHp8qr3QiWV&mQJ%jkIqv%EQ~4|fcMIk**c8VB z1+uYeVl8SxBU95bVx00lXG99dC1k*fx}aPYMwk)1+N^O7_`tZ$TEkMT{O6Qt7Uhgh zEd8yV{Do~M39np2YpJ9VrH$|+zz7T6l&(iH%P}1Iny4Z#KQn7W2mdPnC_kAl z^nF8ECeZwgmKK(RR^`iY@J)#)8LX2W9Qq@?f5r|$to4HvO)lxCXWdW@f9W%dPe@pp#49KmMZ7bVl)NYXIk&|D#UNuj z6s*<6UU1*O%{L4YN!pLUCpQlMI_!qi%lvgX7(lF9h>L`o`;u=Pziq=`3WFt5(qSd) zxaxXN&RX03CXy|yWk^L=D>b&-z-?d3h4(c(zD;9l~kj9VKA*iL(-sP4?AT4F&y6=|>D-l+f&H>WN2zMBfdsAP$Bhzf-$m zbAr`#q1^UCSeAX?Z!Tl~ix;vEruOzLNc}cGF%fz;!gdlSES*@FtcECWkg61#pPz5( zg96kU+^JQ2Z!;jW3w~1$x0Q!20D4)HwXsqx#Ap3db}ZbD05ndd0ESzTR;4|%Ya~1v zM5#-%ifV$^-%>hMp2M>1@vxb(Fq$e2zS7N{w6$Dxb+rpA*X$jpJ@%?W$#wbn8>7X%7Z&8jd58cSr>d)^bb zubQB%LWBS4U|IQq63NJIG}@e`+WF_6cor}pkzbmVFU*wt+F^Kf6s9{tO{hM&|jEA{1c#{Q;x^L*|x<$4UjZ*#M36g<2QK&h14R*JSVua zi|GA_QZZYzZ`6)b{ zS}wg^8fH-cdZRM;<7&|n<=?mx` zSUf+evxeS2+aaRr?a^QGZ_XaIFkE}n9}qjvZMW8*)9+`ZP+Ym1%kS*aOjv~Q3uO(} z&Z9z+0m(i2xY9bs*%PhR$54j+vZz0xsDCJL2vsm#1W7pw+K04(4jojM;ovf&6FgxL zsGJB9@ES>4-5x>|ksX9k4s8@nKX*Bo&5rqfs0z~&!Cvk|6lGz13TJ5D#)I$PoV z{qdwOiT+=wD13!SmbsIw%0QhW@A*h<;mdOQ;+3;Th+8>{2QGmP=SJ!vEnf zz~ve8843kfXl8&X-fJ5)dVdi|giaSHiWyyrosMotJ_gF@N-N(Tn?r}OEgApApF{WU z!JdSK1V&gx+BG*dT}H!1#UI-ae1TX``0h8v$9z7rCf`4D^r zq)0R&xD>&IgM-YgYi2DaXJ^DAp`m{I5=n3KC_UD9UK5>V-Q>@)Q~(tKa>C>7CsE6H zeIM%xZW~m-=5zS41o^KPbZ!g-b3#DLivtcgm6_N*OGh~S_?99|Vul)QwFVyl(?>5lcT2zb%RvlQ+;vzV8myRsPI^gXRKzmVP=I(OODtYQk#e{%E}{P_0@ zNPhHxc-6?a>WXs(TtC=%8x9#@2#W6%_mB5ytz9uTIiT1gKXZ()jVguZk4g73O2eLa z_Pc)3f#R8+ogF<6syaVp1ou>QoS=dce;&X$0*d<&f?z^&9hos?wUzns!w|*vrb!i! zF($?je%FQ$1W=%Gvygd#3W;?8O^2CNRthBh%DLY$H5-{=9_o^$*Q7 znjap~O7IPu8eY5@26m~j;#^p~!(j}2D@ot;0 zabWpwVbK#@C~{|5noj^*rVq#Ij$o?rtvRvY#xop^r1$pR&ks&i7(N{M{P$y={aAkU zqdyxD$uADbA}1LDE^dRkes$cLI*Fqi3%osYgp}MBISc1?-Z}Z*YrQz%x}||=nr7eV z=rCo+bDH@&EC8Ai=YH%4zKRcN`d~R|UVg zE2w}2jyy?xQ%J*_d+eQFTDcJFT=Sza%)|ybik4Pbysex4-Ii?S^X`6YTxLKAWVc~6 z^s|zOUV|_}G&W%EOCCB-iGxv)z=_iKY+i)*ZAl8iH6}5XB6LObG#|QLm{SpC19m;- zQ4~0840lz+onvO=PHZm;D?ogCVj)OJBW**zi%C-UnUn{RRg%ei=5%Z<7xIJ^YIJfp zc~}NjlUL(f9XYZRm;>O}c?n0}TCX7!7GPII2`v{Mf~TuUrT7&mRG@*AzMDT+G-2>S znYPBt*ihmD4;F&l3Q=RW=C0hxH-|+q_z$+pcY^=NT*6iDLYTif2j{k{YgPa9 zvqmt8kz0+nSpuLe8_ zhKN}SvAe-4ZIN@wI=e1H3vmgr5x$J4oBvi0I1?aE6V3!D=eljop{Wt5%oXRcZEE9l zQ@VE{#qpIHeZS+JZ#C2Ig>&Z6Gtix(W1^doeJNh=QoOF<#BTApEz1)` zw~Q2ro3Y$AqYtHs4*IK8;xhkonr?L`V{)&HQYe-Y9PGU<}{Efsw`p-<=)z z{U%lV4cOIApP9?{tGD?AVh(UZLf+EWuDSM|c%;M00{8@~LyarO+cfkN+3>=`-DN%= z45hnBQe+dtv599J4{gP$U^Y`a&J3hT%-(6u4Tmq3)`qRN@>V0=)uIw8fCrQhWW9Fa zm42KKk`3^rSzswIKOfTU$delp7jQQ~!6I-E>ZeQFrIZ?)Z|nt4*#P_4o*hw9;~+x5 zH^>dJm(kSId&1ozNtJvDzfNRfqEwkAGL`!J?J>sNa`4m1R+^bGR=R zdxkMirIwZXzLLLb5>BG^3vN_Xbp}qT;Za{$@X9hoXQwgqaK$ucLie2h zV4QF`?V@Ww|KE$WUmsJKtg0}a>8&>germf|2ckpVq{^4q#}l1JPvogg`G2j-mVSVA zQ~i5LdFKLHl;D`6l}ARMt*%jyp-+Gyd2Kbb-3oL*4nX{%a`@7S6&OY!k-)VF)-K+J z0ORUP9#zAefJh9Lzw%%@8uC~x0kjz~ynfMU(&0O3el)yV)Hs$AP#-bGLNA0N5{!UQ zivhJTz4_7=>-~auPL2<_*xy}Rr!>k2?g8m-sl~;uM~WUFtOyqJ4FT{Ezk$F{I$TZg zx2S*p`opf3t5zYu1nGokHpBA|^z<|_yue9!_rJaaF_*y0tEHvOhRT1cfYw0#1aS{IV_)pK*X9U< z8Rkq37+Nm>cJA>(crI_Tv=l*>K^DxE7N@0eY@^?_%D~NZf8X7r`lGIw$2=D&2zwOW za%J68UK#0IlOA0ct9yF;IX27k;(*D=93Lb#0!$Ya#t&(TS{!q_uaf>!6*a;$aiu*u znSs|(7WAxD)d@WsI%oX0-FBc-V$`E8bk7!cm$Co*;1g5+7r@CX+H_23j?yI057d$u zGq{A+Yb)Q`$X?xd--dne%nm1YippP=-4*!MOL}97zs0XoJp(J+Ntpv9T=1g0Jfh1I zBw@b*d<0VqCRWy@pm_Nvw6^eYKwsOJwI3rUU?SVm5J~W-!PJa|<1%2# zADZlNkkkbO7?EMua+**7ddw?Yt#0YJ9z|-#R}n$}wVsJjEOx}{BOWqUNum?`FHK-k(KQ>AeY2cnT%y~d~7M7G` z1k9(sefwD^-48}6VT@*XE`0g=HS*O4MMYBd?Logz7(3D;F&nP#+Y($I3jgK7pMy>b z)$1jr7(JM`Bs2xeWRLxHHKyybUjh>#O7^>umybBJ&X3+>7v)_4rB$1ZEv;YHJ87FT z6-3v{U$pdfF7oSns0OejhAH*Md|^2<&-{0NNZAo&WA^7vax9AhKW)>hzi6AeIceOX z!*MRTaZ50BPnKU$v9^z-lv<7;HAVx4h22>JbPKG08H*TJflX3YCWsN`OufHuQsFwO z_h-GKHf>7$ne<#14Ko5Nm^hxw>w!7|7q-L+klR3tVY7Q2L|;MFk}(Mha2}=Mh%{sp zDnGcLut#|f!Wx!z^!!#{-)=otXhtZQq)c4J`xIWeF*+QJ7R@7Gxx}E%)Bv*~;Fcnw z+Gs}!sDWqS?1mo;AHhrD!obczn+Vd{6lgl&y}MxNKu)4@RuxDIA_;Xr8jc}l^>>sD z!H&fIE4FW8!gKP~WP~6}8A5;oBowr5bZo3<>zY!An2;_~IP3CqtM`H%PA>>&#Y&?UmUHQJ<^*IP3ZXrU;ka~KVT+&e75&C! ziYs$;0Y(yLW;xHzHg@?sYusnezI`wahxQ`nx&c8x>WM@JMTVg^N|e0YR4^?F!fNb}!R16r+Sk_y6g4eP2$NAV)6UJ!#Xy^Z z?bZIZY@6!k`%YWlpA*-%+9PAhWA;nH-h!(G!v+~^d;_Jf=&H7LR{F6!e63cjan2;G@VRl!i-E?Lx=9rrGDrp5fpBt@Tyn zX8ijXetk@n4>fse#GFJX=7&pd*%zKm%UU`$px96WmpockvYnv%y)-uB(eVp%ZbYqBNkgPbT)O=`t8<=`nz>JIg5M{!;Q$M_oj=kvj1YDO_P;jyd=Kw=S zSSk9zXpZdf!v2xtO&aRzx!^Xy;v^*gS-~%Eq}J};&AE0+b_B{R=jWQBgG6A~hn{tJ zj(IiMbKvEaOk$GX)kjMwYNn7NT7FAE3Y$sXepKo`TouV21^NaXg zej9bYl3QGe?IucgVOdHjluO=_QI9Ol(W87hP1hNIa<13%)eM(cEMWN+bt*6FOAx*r z>{!9tnStMN`XD(}z>o5haaN{?_yjdKcP%Noh;g+kP+hWU0!sxr<04bwtm)yQ0F#W~ zENr{A!Uz!q6J(IR-rgm6% zczw*eO74Te0!Ed|zU7GMIVZMn=OEM%0RAzt6ZwDSL^i&Ed57Yff?P9e5f}|67TO)Q zGo3zS>ewMbHq?MkL#(t(Q!72t-+iobJDh0&O<>dnV^%ykNNVWMIK)r5M32};xi`;O_M2u}53W*xiw+7(`FUk=ozW%EcmK$W$z-o9`tzNbsVMJN zF97Hpt~-Zv?fA86eQVc3mur`WOrm~hPz)b^#jm8PM5sARkk03V>+m+MeqLr(w?1uq$Njod}G$n#=i zV#FsJXd#qPa=l+KKu|KnrNv55tkc5G{G{QD@`jv^2zj_dSeOoSQ~C2U`^nzr9&kG=Cz0N|1`0ZYV0#Vj3^ocgRl?8 z32qEE_yhz5ZaLZ7N|7oeDTy^wOKuypoeUoP0p7(QC;RWU+GrG4i+hMo@+Q#9f7x7L zKXm_buLvdU5caHCTIO;Y*)RFYD0U(M-xAS-WF*Z*};Eun^mCvTR`1Ziy3K zW5zBmrsyaz@pG4G;$Bi^+N7ra`GPF^Ul-)9F>^(etef+txfu3x|8-tX=3S?lTqv9Y z?zzylnfR)3(cbpmE+U_iLH?6D*|X(HuNYGO?1mk}m*xln>3-qD;91hWt!iH}2MmV(|77!u^2E*Q;YheLR?8(i}UbD9t@kOIIq zMA0ILJ(g9NwFi}sLfPG!}m?m|J81x{7DoPl?1Sq+A^XMKSq*WACmYC+kj^pd{^5&QgY8=>883Jb# z7z;;Cp7KPdV}Vz0axxq9Sl0`lkGD6uWY+|JH0Ph}cf8e8ub8*b%(mcy%f!#R_B8DR zm-ge-v4Y&W3}ZfXgpEbTW9f8tGsO4aR@i!b)ze~!yK%2RycxU~_be&3)#=1!Y1{jh ziz=Rn4LXv28$1UyBz3RKwMDhLl(gP**!$AzXug7RhSceB9V0Qi!6|zBD$^mM+ai_m z_d*Z)y&ZF15Vg}#dDr+Eh9k$fU$ZFSA6c;^jJrw7zu2y#0TZ#+zYxF^fHepT`(ftoCI^&vItd6S5#Wvq%w09e}3L;F4axm!^>thrN8XG7#XoUBQrDQH* zVi=D8IUdc2#ZU#}l9D~as?#EGKVLCIOLCh_J6e$45KLZB_f_Hl5Vy9ek?0LTwo+Ew z4_F28wP^)q>{~zNt>PmrcyuwqJ5etG$+CuIofWNQNk&M3%o73bJ#9-Gj(CiI!==uEGi`a12Tk=xVy z72D%8w2Y`rPh8Res7MK~7#d5@%3@YXVn-*qBWmU2V=R~6-HoacirvJ_SsSP)(fQ%cp0^KOB2s_%GynZh6A3jQO{0$-f4>vIKVBJl z)R|uB^?q`;rukmn!&k1jHwJImr*#EQ&(l|3O)e$O8)ph(nyf^aC<=fZCMyXFGUAoF zUn{$bP<~)biIX@Zyh`2J)ByiAaO~2Md@w97QK{-w${jhpENS2SZI~G7yYjUKk27Ev zeI6$cb89fFXv1y*MSsA#!K{K3gLOw;@H=o;geA7}A%H2E;oDf_UTZi^pB?)U*p9Tac!K*48Fiw~}m0$s?Bzuvw7aUGJJ`jmsRk33K<{d>P__ zJ+%ZH#ns^w4>$ki9@gbiELq!HdU69+BRsU$rXIqz?u40`;;A1>CFVo>r%+Z>`8!#xJY$CsQm?js@~h+N2qdolv@UU>P7$g@vv-|#lj=h?#co$eFdbQi47 zNg3=+7gn2h*ZW<{nugE3EP)s$u;6Kj55>55nOy>AOloh=*Q~v^b$>trg|c#o-R-5( zORaGhqt~7e)a+ZGKFr|7tCknv$3vgGZ&|Q@it)$a#OdEBO8zsJlVbFCSMJ}x^!wxR zgN=);)_e@g!1?Ame7~m1>9V4*ah>gavmYEkuDCEcw!cXV;XP|0(s^oM0#j*hVAkwK zI?rmv1A6V;>?Rh>vM|@QOMnd%>FMy;&vuj6A@X=Sy3>A6q(Nx5$T1w|(gdgiGoCrF z_6=?Yysww08bG_g(06z)(-~pQ>nA^9L6GYA235_x@9!*rh=cUSs~j#BMto@69I+`V z0D`^Y;o;<$=Pq1eN~iJpqQTT27}~-@XkoN;?V-Yy4)jteMdwCp-FfubR(gq`;LLZl z#I<@Ok?<6DX{Ut1U*MA1w+Fu-D1!iQLTz`iI5+;>vutiUP0HwYpI8{t51k?27ab+y zNf@>gyd8tD&=;IEb!-QM=e+asWf;Edw>e7 zTe$Z0DCNTA&zexp4e8c@=f1}OrDAdSdTlHgmbj#-8~IgwX@=b*;~J4fIlQW1N#PEl zf{uueHY|?7a0Yh_uPQ3N5xoYc6Uq-Xb+%ISWUtu08}E22u7}6}bti?xDL>*smFL(k zI?*s0B$5R=QQEOV&Brm)`NtdWi9|@pw1UzT4EF>{b^xtV6Tp8EU`e#|3@3Z;f1oD)$o z=K#1a9g{B$P)DxBE#N`Yg8M=b+LgH8D;XK2EJ1<2clrP&?;7Au7XYBB$2#uRVf7!W zU~n1NSjpP4EnL_G7il!qso!30fbS{c{jFXdsFl8#1Oo;QQOAh!?QQSXro6U%ROC5y z%!=pL+3VFUUW?w!rY?T*0!suI;?=}0fQ^RT6gV~T+`SJQ0PL(qwxi=<3LOt_nKQhP znL=UJp5OU%KZaJ(a_5&Ns0I|6YSn*Y)>{@Bl6f}sT+c+)Nd3w~#@c@{Hi}JF|M*8a z*vd$0lhCd?IY{;)=a;3MOS>~9UJh3le6`;<%bJ76f@|QCRpZYn=l1*iQui(u5+Yj+ z5bp8v+T=w*gIR5*hKoy{U`3IeJ3*UO2jvU=go>k!*O*h#Nnr9NrmrAZSXckJjBi9|0X)2C|rxz@MI;o>kr`Ha**qGXj4mT?x9Q3}!u}56}k1MF|mGDR}pX za;;V};vpm@jKQ_?@S+Q-c=O?xLsl>V5J4P)@NBNLGsmBi>N{!YSR86XRAz){M%o?U z>GAz_s{H)?nBxQ>?GVq%r-R-*cr-z9A;>E-IEUs)s?R*N`?xRLu?zsq%hfmnH1+jX zJ`PzE((;4ZjQ_&v&CWw(VmBQIdgkOsN2^`s*na+onqAoZ5ya(NArecLEeotY48DDe ze3{D{{Cc^2Rw2=pd@9t{dtSVqOrlJIz z@JQ*9&rn(OCy!#(2;fg)krNV4;C9<@S5lhn*CQ-QNJzjdsd8ZJ2W8FyG^WIw02AH& zLuqEKcEbJ}<8v}2IVj7!!t4WRZ^Y(8J5L;`9hE}a5u>qp-AS!_bl6_w^<&~ak$Hp~ zYf1vcBpNW8=C_|b1yA&e5QgI!VI6d^8K{XM@+E^2wgn5???xkWpsy$Wvedb)Q={6^ zHU4PhkRLGFw|!Cunmqpo$12j-E?!)q!)z!1QDCgcIGV9`xr(_3<&Fg1qv+k1u0_0i z*KIw?zin$;O^+H}nt-}OvUWI6ALLCW2D~HwUG3l3_Wr@hw~^ymX+zm@yo$sBtF$R< zcY5Xkihz@6*W~HlIPov*#UI-%#ij?FJ#;zo?;e1fLwUc<^1sF6e?L0E&OdcSiKM70 zla`}Jdg^A5ikSR$ybDFCnv(S6xkuSL_9f7zoT(0-#Wbkf*78w6R>H4JbP!vF*AB>H z!i7`E+3Oe@THxSgh(QDl-N30`|HcC#hC@ppGcqNYMAAiwJ~1X!P*O&*i0gBq_Av8~ zwxkCKLM{yU_fyvL1(?aCjclSARSgfX6PV4AUy{bg8@{9UFK=ChhGyc$WpZvBe z*}&!=Ub%}6lXh5v5d;@!4vb@n`Ugk8ym#umuYjtr0BSc(aw#MQ>FU~J?F*qMind@< zPS#{aYAxS(O_O*_liNqd;t-|LXL68n+@xetcK{nBq$~Sc6CNRn7FhS3Q1h?-DjGSZyhKI+If z-IvS*?C-)sd&%YPwk?Oi)jn`*58lrvCo3K5*e(^2nkoolh*?VgoDqSasPk>x20y44 zWL()Q!aaOVEn5C2{neJ531vpnP-Z&8=b;ZC^03$S{P!aK=c6(;5qG+OUm;QIISB)C zZK`RI$k3M!0+Lurqr3AdN46DbhAp0tsFwYFqpm--DKGd&gPj`(EAAY>Rbn~YR2cDa zXmSF!!dU>zx_~#2C1|NYNrvtfaMkd1lvn{4@?xDD3S{S2Ofp7NhP<^acFGY$3c!^! z2?I#_0fi)SeRn57(VPu_Tj;RA6niVm-pf*@0B|nGj{wmZ|01rtp7?<|1z+UD;pHMC zU4OL-rx+|=2Lt5yU%cI%K+_;AEVn~%Nos_io*vH8;-aGAq>kzN3zdfHJcioYu`I>N zI96SXrvfGIh7V3o;A1^{Vp^uzx(04j@T+hzp4wrk90){$nM?EPfwdvlHBpzZU$=iz z0;VcSGNc+6TTZnV!xkA#-3fmOJA!JiY)#-`o4Ip)Q>@g9e# zeO2MHK=xKWfBsy03K}E){{Z#SBfgSaQxv77e0OEF;KHJ|IaxjRveupx$lNhmI}RYA z%o|w|CHdkEQ=mHGwjpf;sayV#Wk@rpxza%am7B1f+-U01w(<`6Ly&Y!-cz0rPoCSH z8th}oEf3dn@?43cM@3IsPEk&!8OT&D&?qHv-JA#hZhbN^sEN;g=)!Cr!lgoa>Qun0}${X@#tI4Iw ziL=FeXgoH2{QOBCOP~hA@DmH{tbvA|efka@MBBrd)f4Cmpki<<_AogEtpKE|RdG;m z2S(H{U%njvf=YC)l2QaFD<9O`{K4Ya(_@*OnyNfHFff3-v%rv%evX@mM-Zz5jIO=j z4_~|?%mfR~0Ge&UtdEJl{<8>I-xLIzucHlL@Z5t$ZjxgjNl50X202GHjjCW1L94Gn zP2>h(p>c^{srJB%b;!*^^6^V|`7sKfm7-N--?&i)n{3Iusz1hiz+sLNHgZFORnzKJ zuKeNq96HPju}Pvn<0zQmbdbeiTEvJFld$>zvM5){G6ZJ_ixJin)EG`Ynx>|S47x){ z;W02frhb4)P`nR#o}>l=xPEQts`=YkC8fg9m~BUv?Aebs?t5MoQth#}Z#hyp%Yt_hFiY3l_Iq zB2FV@TViwqII<0WNlc;~T$j&|j9V7&TXeO;vst1uWnVhqh0fzFG%G#YqY(#l0j{PJ z!GQn7ng4vyXU=2HI14)c|FB~k`nh}OwPhE4)+vdK78RE$@@j@1wk{Jk@QBz7MiCR8 z^KEN^?C$uGU7zRo&fdrHLwGp=t02_CUcBL3!sCO)&=-gmV)avMA5RmpPeh20h>Q$u z#i-1$q@<*ncS)24B#R_>l(cV7+lQM&$~+XT1n zjI$^p*P$4}JE%LoC2h11Zk!Er(gxdHuhMME?f@d{C!UQK78X_0G@8-OpcX1Hd@T%{ z4MJmISdpg`6Z*w8UeL`XQ@!ZV%k4un?k4|lOk)Ldls(iXVqO6Y;s|a6NmOP8M8hQ>ywAs zSh#n+>80I3_(l=}G-l=H<<&x%n3&oi^~2IASe^6u%D$KwPJ|TXAw_{GU2Fa^(JFC` zvm-6Y7S$YFekz7^7uV@!rPqtoI*31#-=8cPcgOAdWumO_&In%0D7->R;Nx}-nrryUN`UYuHdfx_I7iS zp{yWMK?##;oE7le;UVdRj}p&I%+u13y%7WJ)~X0zp^!S?F^>jjbps<_!eLfBe<&qz!yrL*>W*q9@F_nG%S8Ctw4Pzqai4U}@V>`nJ-Iyk5WFe6 z;GYw28~6C&Ok60(3OoC|TKy->ZX;hP4d0*3B^G92@jaQ|ZwsXu9(b^^iNbRq8p}yD zJ_or7+470*gb=wQs}k)Er-q!?*7bLIv8PPdh?TA52 zPV*XLb<-@=V>gXdpL;~6H*+WZ&Y{ke?`Q4OnBtT9`s9|}%K3%2^im~X#N%cjfXBvN zQ?mZ{=qWW0GPZ#T91&10okn-Bbans0KwxDTlWJzGMMt5|Jmca#jQLb0}UNEXLby>DQEy&y%!XDO%+Ktgaa+A-dBL2erUg4z{zJOLXl zueZ)if-Siyi^4*7_FB9B_%0X)l6O?H1_tK2qMoKV6|78oG`&jOoki?T74b+BDi4wj znVX{S(Rwy^qLf4?9rCTM*b^pK+SHw_*&r=}&i5nZ z8%W394;Y2wg^j*22dWdJW#V6AXATITEJ6S`mKpct7;KCQM-bceFjwg$t@Aex-*O?s z?7$r-55WzKazWXP4r-r6s7&aQ5Igbci14s%dwmLj?zL;zw)pz~ICDsJvM9<$^|@PA zp-Xh4i2l%JPeTKBjyx!t?7mzf>)2dA(>HGkv~I_5Dk`yF+}~ z2Je4KJIPIZ_JN09a6`4|2ALw&t$$#Jg-M)Ij4PfkO#gAGXeHT$qn%z;<6mGbZL1f} zXO*j>x!FhC)D20;w?UE3>8!8R#1p zCq==$zS_zUc!{ZLi)e-yJT1p(IEKWW}`9xm8=r8;PwTyiTHQpVrmYu&ybgQ{xcRyXy?pX1F)W zNtB00Gta3tpB-c`^Yb~!^OXb_w)>3jC!Kww4I1({u658CZbc`2ekHIP>>ud_5q)^- zyXOFr;=>mLW|61b^P5!C79#Nia)X`5#oIYr=|rYQ2yp08*3=owr}!Gu*%ZIA6&%rB z7HF&Wxo6j_m@67#S|aW|u_1NuHK3+O&5GA?Jo7<&^EI zphp%8LWPTD*6!{L+*}Z7=?(MJ6ESf#?_1cr0^5u5oDQv(5ILxhKUfHQkeGh{S`^m9 zQyo85toY~BVW#sCTP3}`=fOd2Fc`T#X07rX~}-epw_Ck_xiIqNp; zKpuI}nNqj=&Ye3WTcsby{}`E7t~*Ieag+7H^=+Zqp?=nt9b%5 zzG$V-Oa$EBxTI|%>}o8&f85PkBZegc#Ml_dHUW=$wNMQsigk!^Fj4X#QS5<+o@385 zZqzdcr^DgmMS)ET;O@SVB?i5+pcnuWO80?eL?!uvr=L3 za;ZVsgKrXkSLFd&Q0bw|*Fr%I9VIpkLYdtAW!lBXg}_HMc^8|_Q<|n$iG_{t9iTh& zrV*iHX(8QohWrv)!NoVc79tRr=^K1mm7Ni_#j}e!V0iEQdjx(z{Pxasm$x<9my8}|{F>$S zoWn0WTR++CblxQqM{Gz42rMOaTCaMYlIcc_+zIjeNb_HTt=F@&38VmDV!g!es%nlObnnA!ICv{Z++iad9yx6T_wLl)9sCP(YZ; z?DlHu8PT~AM2BSsi&dw)7P`RQ&U>VKF;k!@E0l>QgqlMdPXzVwx;1kU^$31}R)#W# zCI<0`Bo(~u=2$GpK<^jw4RD`;qVs|3avLr$-lFUyr z*svxi&^8l@h=6BN{f7nm9T#nxof#gzc$?|%QK1Tah2t<^BKA0Hpzc=P?r^fHb!-99(Z;LxO{+S`9fTq8+8 zbjepeZ>RnAK)vp5{1~@s0WCIH!XR+2gtA+SN!q`!L|@r3|DVWZ6fRTloQn5tj+nr3 zuHHK_vdU2)5*a>9e3WY_?-}R|!f&J4ClE#8V)L_#uZd6*^clX?L2R6cq!fFSGxwYDXo9%3--q zK-#fXNE)GO3t=*fO4Un=EboHiIh<6g;^La9RJqAq$ko-Avd`RH5Oa44BWhp+yeK-p z4|<}8;2a=PNA-tSmBKOc&nHSr2yua;Uw0G~5@*Z_vF}(H_oVMz*$2S)4@sHx+9qw= zCandC9$>Kn7R>dT@_-HF7MTP9s^GB#hGPNwj-xs&r<^<)m`Y)IeW!bjHE&$WkK?&} zp;3b>iVo^L+B(7ePEKsrK4J4)<=pF~4jOA?>Ln-$TR8vP!y1N$brI)j_IL4ng4aNJ zB2A-@56U7u1=bW`^>Imb!om~&qeUvBqKOyPoH&^6F=dxx>En@AN?Od?dTFub=7(ME zpHFgAJW({hid#0FTBJy{Qc9ee{apUKZR*hjaF4&h_An!+B6awh3ess%a$ z0)`NK{Z^EM^)FurVREI5fP#SL_cR3x@OJSta9mr*}4T=hKa|P=}Fs*^6+QV22{~ z)BEJe2cey3{z7ONbgbv6q&A93*)yG?+j@I`w$(X66_*b+u5lNmUc0@?AoRPIvemii z=ZpW*o#0cNch`pZf2>T!nCQVVyH{UHb%U^BSMnouAsVtL!eQo+4u$6sg-)_%|G0s# z(%9|J%lhL%t7eMASuI0SX-Ns-tYd6G>yyKiFgRKAsFHjtyI=d zfFm&RadL9Fg3pXpEjzpNfj{F+ ziqY$MJK{AA7hz%2x2R%XgS*Og#gq>%6!C;aHsqwGrlmz=7{iOn5cMF92ouQCvCgk2 zmbYL|ixL((xXf#9@40ACGJy3QZB-y`F1lU~_%mjFe~lg2b*cB{E}n`!G1CuHb4Dh-Nl}9cjmlYKz~()qTx=0q*5ffT zF}M2O+xWMCs2M?j`GeiSd1`B1_4T)k@rxOM$BHmxZPD0xQ<(#X9&FEar_26$3wjFl z{&)*a;xiu3()mKpCwGh8ZRG6Zzu|B0R%Bxle?CNNsapi&fn{H)Vtbdn^UOf4)Om|Q zPl8zsN@|1)fDm>Gz%hyO4feVu6lX1-qCY6tZ?+2uu z95_rA$$^8^wY+8Y^Jh|2)IH62%fj@)!h2*M))lO`OEt-^cCAw;oOj?IoVEHm3_${` zLgIO9yiy3F%8diEpkE*h0c;2KPlnf?l3}6_{^rU3e8C$Or(7x2chp0T(H-(^)-%WQ z@!FE;J)!7EaKfnBkXf0^J5-12u>FQ9?lF&G2PpNi)1E553w0-_!SFg4-fB_A2-?2x z%LBV^aMdKsD;Vh}*%QdpM z+zv{p(vAz=!eq0lhA{iwEC9!(3!Bf2RG0XD;K#+oLB0(}hhv+)D7pVeM02w2kgA z-0J|mO2~Iy#6h7o<2Z(Hir(XEv3p5eM+?@uxKPAF5*~{WYp#|*ROes-(yKH*Tp1T7 z7s}gkVNHm&!D3Ub?wPB!sHt5<6dcS;Jxh5m9PuSX`sA_9Z99=669!8)y<# z@#o7pT*_J_QQl_#5+&7!`}xhmFN=4!1q%#XiOCn(MY!8|gFzt}TeinzpS~l+ePql? z=hjD$m|y@Efk>}+9&FmnSFbW+fQmj6?>#-PQ^~Th^TD(1?LDn@wX_(-7=RZT4>>16 zJfyB(K~?(cMry(@&925@jA4uo7OveXnxVh8bxEZl>^TE=lAx%~WC z^0IN37&%~NfQp}9t|qRy$YWH+(IYD-lhnnMe1M?$Ggzhms@B4MhN z{gSUe8`vvnYqFVSZuj~lTwt5*LBy#95?*p+2ZSk-l9EdK=i=hPJCs}V$y}%B$Js_0 zW0;67E-06ui#6}DHy$KGoA(jtiE#<2Be0uAk zOWxf!p>IE2DmX*X>UVx{*#Cj>>u-2mu-4{MR+jXemq6g4Y1d74n(Q5ViAFI#pjT8* zj{DSS$x~;@_9TKG|1c+L5n1@V*`ptOXZHywJQdd~I_6bsj{U)#ndbX*4iMQq;fU$2 z|A}74+~?~q=*BejYz`mpZteJ?(D?D()PyY{hIZfoqw752xqjRB|4oX}P|`#hS(%lD zh>S>9qHL+GjLOc6$||Hpix4WIVb4S;AyJBCg>16-|G0GD_y74lcTX>``*$blTi@$* zUDtUY$MHUAY_QOjN1(=E2ge&BQ?i2}scau6&>TF~n7q_-%VW%KKlv-+h2vGIN?b+j zjC8wUEu=9*ed4AVl{1$C-ROEAqn>q+;KF6WlBRlj~W ztL;U{Q~HX{;?=-n3KX{kP&3FqL|@-LO%=vUJeaV>kpkfLiU1hK*AU|Yp}w!kbj>%q zk1jFhpc{vFjk-tyS=*{)Zq6+iGsCa973w70+%F|!!oT%{M4b2t0`dZ$Tc$d!@R$Z z^rEmH(6w1y4-5?QX^HXit&+`CtAO~GfAAN|9+%h;rkSBf_rM;c-Z zhTh*#i4tkE_e-*`fglZqyd)5A${^D-66XcH4~v}({QL!RUUYT}e(il2UfKG){ZQk^ z(%A`TY2~!A?_0-Ojkov>mpnbbaa83=`_2<+oe-4_6{rZ>WyZbuw_McuQ1NV|eIvyc zewBM0dr8n9-TV5-46}}XUP<{_^7MIs`oEV&Y&_cLfAi_E=Ol}3 zUe6FIBpXJo(YY|a9@^18EbX4VLnaq{k*V+({n|Rg!GCpG+~OX>ErZ#9<#-wg`oX*(H06YZS6U4`YU#fr@% zenV~}v>B&+o^u(}-$9Yc+jB*3JDMv5&I7QYPesSwksQp3{8!I@_hvkht4;6BtunZT zv*i#~=F^;WJ@x4n@5>)Ho}F=eQ4+vhrJK5rvi8+zx!%23WuUBA%7v*@mRd`wv#Ui~y*@iWQ z24=yvk78!Hplc!?hCvf#Iwd5jo73cufR%;W1U&d9u0z>UV^~`VWHZ`8Fu<4X{CPfG z{z+#9jC_#kB+-aeO5=uWl;j_cQX_coq+vx8#f-QHKII`dK-4Cn$2p!>mQz z4M2LH%u0r!4O8E8-`1##WoSV;I{vF+i7r(8@Q3NCWwHn0JdiIslW!z*dVv^Br9C9R zBUnFF&Jl7TJ}~&jS|GR(_`+|!vS>QD`pnlY=b@bRz~Dh$zY~X;n2yix-Nw5&uY_Vj zcxmKh%Fjn*stoV;7kxWI7IPV`Q6d9eads6!e?@W?9gnGfbvM3n2(-tmEVtaoRr&mvMe@KlT#iHfqJm=lKQ1)? z`iV0{yoU*1ay?*fXj*#&D&joi!nHladio?LWFP)^2VzJm--*6ki>DDMOuwX`h5!;G z%ZoXQpIu$BFUCE5NKxSml4n%7>Bs6u3(HMpWY)-u7CcQu@Qe#djibNwKZRl^YN}U4 zDvtqs#r+*DTKI0j>A}-@A~_BLbtvMi)Mwm572o|ZEzRMJE3*nt6BNg9YSj{tz5(I$ zlqe#x{>|xS?KgxOD--dX$#e^Xngt?NvfFMHp+Atx-=Yl;&<>`h?9lua-w*2+f=&Jw zjqspD(c{M1)G;vwN{1b~?DA@MQe!R$2T^V#76N<(mnv^076_`-Ow zHf+#$Lx@Qh$N;SHguX2sHu#YM0s_yKSxHF?SsYqgVYu~T zN(D(kB3N+bZbW3HKki0k_K?phd3o`wPU)cT29T@yPzz zE*rw_4e?V>AbMfWR~wcBG}%E4fxL(Rz3S)Jl~(}ek9bjWK|;CQGT$qhSM)7}cfM!OL`JZQy9L#J0Hw}teb~#f za=nOaS#>k|3vye|vK_o=C}VJN-B6CZh82^)dWh`wUFt_2J3hGPbL6+%8w?g(q;82Z zp&hmqDHo_da8^{NUf;_7`>I;0b<|5-&+%dVsS*BJoSkxxVFu-0?0P;~uHx}$8<}=I zcG>wC;g0Ewi@DQ(-^tlyd0uTl7BoIALi9X^2P=8=R$NS@4P>u!ei6yXn9R_XQ~E+o zbJ;$-v_IdJzNCHlA?`)cB#}_vrMCkGHP$gqW&S}y`d_`}cMIWP9yVhz_*DfB?CefD8;&>_M;r`_$8Tzb2bo(wn{qfY_r=!fu=azYrL?TTv8zEk|v8vjssLe!gXP^ zfxhmyip59n#i(7xdWPl01G$3;B(Jt|qWXw5KfT2nYJ;e)8!CJ>nM-U9V|s zTIpBJ=;=OU;F3Y;cIYs#+#Yon2o{ zCngSW*DXJjZ##&y&-g{`Z{HnR%Xc6U`fa1%ei@#F-{$i!rgWw5%@ciMlVv@d(oYG+ z0vB2siJLHJHYwYvHXFO#F_C3Roq9**25OVfR~@9L3w2*>Nac#q)qkX}nyqj&n8{k! zUi{g*8Fh|WETd6jeJU>pKH<(?@?Jo3Y6ZuCW_$oD;~C@zy7T76SRZVXP> zT8pEN6i42!y6khqt57uC`%V9Hsd?RdYoevc#s_=+Dj8*h3X|sBH|1w!E{wQpvso?? zo}2MKQ(Dn4EU*SC04NF#^6PtO@`+c*JEF@vO+0MR`Lk-Zu{_F^y&Zp%A!5INDr~3v z@3t3Xjp!>eqh$}|yd__6sb94>uGSgpzud`R#L|%hR$#JKt^o zS5LD>v~6OMp$LfrSggx(wIS(9X(NIA8gT}$(@s~m2UrZF!otxd4_j2W0Yaomi(_y9 z6Cwq&5Fyg0WK1ds1%)+)doh$m_-C3?UJ|&e;V@;CsfXZ*oaf=l9x>w9og0hZE+B;V z9>T6LVqk_YHT0lG3vJK_A>b`oL-vnvTx?rxp2LYthJZ}Y&HImoytoxb47!;zxZ07C znIwYIfW>X(KCA#7>ETDb2N~S)h)o*}Q^+yA zGiOenB2i=P3M%pEZ>&`$g2_5{DT3%AW0JCf6ni91Ts&i`Ue|GZC4aG+DsCRP^y||@ zvW**e_~a>uTfA7QNf{P;l{UQ1yH^bFyykaG6Z=t|Ujvl!>aG{KcsUTf~1)L@dC8`k zH>xJ5SNc_Z@`3}aw6jS=!T!B|=lZ-16y6maJMpBg2u=`Kth-Dn?tq|y0V zvoC$2-GMI+oM+unPKy+LNosQG=Z-Gg)sx#htDzy^WJ#mPmphd2^_|{Kx#8JJZCkrj zB=cPZqP)Dm{|H6T>IgTfUw4bf!z~0w9qG=h)yF~~kzE6?@n5d4wHkjn`1rHGJ5BXI zY!CESc1L-6W8NcQ4hXidw2QmM6PWeEgNZR&IKpkeF!vMJ5y!<-B1Z^|0dEo#El3D@ zS36JmTsGdYZr##X-xy$sf*Cg|VvT|bVmV0tp!tg7gmHx>yzXuSWlEo)rSj z$Xk(o0mwSy=YP_|YsB6ea_w3RMjaBb@%Ui#>U<+qH@_Tl-+*6BG0g@7+1t0f_^+~5 z;s^qbl=hy$%gJn&_&19qmd>0}!9vElMU)q+OXY1jC@7%di9`D0R8f@Y2RGmiZmT58 zgg6Bf2Oa0pKtf@|At@$^uz9Dl`2k8rl)#iqxS{Rir=%cki} zw5(pJkKdG<8~>8NaBd>8B|7Uu%DsXeORO_HxAE$)285gi=`Cz z-b6|6WFcN|W=hMZu{XQiEb>S;a_if_r5++wRu61vkdF`XBdJY!W#>qp?cs$goBk zkBCU;!RlnwG^74f?+a%?Nu&+;3|X_BE!<@s{pgZPVqWhEU)_Y4{#fwUC9|D0(I-## zRf!iUrpg!jek&JLy(B2RGMd}FC|E??>SB#i{pxd9y?o4;vgXZu-M?gBN0Y5}HTHqx z#=}}MpZG++N(c6GHwf3YQc+a1W+&^}qDw+NXEti-zHOf9jlI&*g=Z>Y8}0FX4;5we z=$sd)a_py-z(t8o45ct}QM$XkHxL5~nmf4l zVjLS#zAqIJn|;+W>_P;NMWq9RZ-9yYfLS7{UqZJ;hw9e*+zp1|Z-{}v<9+zh=&xTq zXlT7~8?!<>6&-8et5>fMIbgu#Wy=B77mR<*ENNeHzIYD#vbB&@AGoU0mbn^Oyugo> zSk;b-0D#! zKX0hi0k?ytTy{{hg9HvTB+UeQ6U6g|oFzlk1zok8Z+K-cSr}(ii)(Vas6X`1E4AeAkvoU>ylr-D zVp+L#a%drkTEt<-gID*xeTxv^^X`v&9J>YhV7mPeZvo3OUeefe0sKmLagy&DGFy)N8OqKA{EK+O8mPofbj@( zY8Z!%(UkqgTg=EsODwt;8eI=X5d4t8rghoRFvocR(ft&_v;h|c>*4mwB-CmGMtA;I?+@xvI6XL*gXQ1FPtJ~YLidR^RQ zXdCn;DC{zL=waWn#}r|IC$yGvN*Vp`uRGb>-~V=tUEU>tXp~3LC3=AgkmVgagH`0| z%X-IRGcuSFDgM>ld+#L%pS{HyqX{Jg@nQuP@dd12g2zBEb%}X()LW8-EUPr$v~_C< zg3K#@i@eF4SQFtxw{9~BR zeGjUXyW&R_;Ce@-3{E5f5(KpQjPze=xXWtx&>sF`5xwPl*MrvXFs1V^a@a4tKh5e^ zbM2|j_Y!Y2hxeXlbG8OEKXqK(8Z(3NICwOus?UDju{+9{ooNjU;@~aOnQ`_rvn(N> zqn%8id+d~&t7|?htJ+o`6t8__p*LeE3 z@WDMX7Jlb`y!tphd|pyjbDJ7h8M;YlD?i3qmiYB}uD&nmS?0^WX_W5lp&C+vWUGcs ztolpITps>Bb{Lr;E?| zZY%uhs{dLANXhiH8}3TjW;Y4kgp7vD1M~vem;~+R*zX1aVUUuNg6ac7s}S??_87Q( zk$?PS-y{NzR~Oi}SFtBd2El7yH0qjv?A=J>=rpadsR$&{NURB){WP$WI z0Rv(D6ptqEP%D7MCt@!=tLhD_JdUEnp-Oxo^2o+g5p-@!F&u#FkjgQ8yo#WXp|yx@8>79MQEB z0MrmTbZ!$eYnLD`4;`;M3%9Q(lX)EogajV;__4*OZ!9kl8)m9wYk zdUbU#TS47&jrpIO-KR%-cxO(iH=uU@(M%`0M8D8bDo%B{Tjco20p`1c-&q^;T(j3* zX{ep4mim&r+ zAB-;Sw#fNcWBaJ@c+*5xMQ21yD+i$*z%w;I_;l(dH#)OSPqX{! zEn0DFsgW04rxE#B(KL25a70$q+$LK_e|zNkJUK-Lha0q>ZO3xmOEdnmC;t*NV!;gS z|N6o7Hsg`C=|V$gNPxf2U4cz{$aY3Zl+AtThO5rpd8%g5fYO<3|AydkZbzAnjLT~@ z{|OD{jV#RgMUdrh&Ih)paw_~@5@gH;ARKKoR)|Hha)@1lEr?Htjuf_ z&|DkH3@yN=^N#tyU97~{GD1T`GmLpeWX?Rn$T41N>h0}CXJ=E*?Ynma(S+9~i#SNA z;Pstd<~!15a7gg%+-Wd-U@fBpBiUIIm_Zo;ydLTcw0n6-8886TDNsbHs#m~oe%kno z+!8#+Br$d7jG&gBC0baE1ER#Y0$k-%A~LEAd7uX1RD20YnH~yTZ|>B^Wi!vwnkkAx zU%O6VJx9Ts`DE9P)ycsJ6Ktn@wQxs)qvK5ocryNNeG8q(_|{{~hgiGBq`?slECLci zhX?_#FV#vD+~RX(j1H@-atekP)?4jE!@k-g+lS`WaCdc5VqdLJEle*?rlKcrntS%Y zD!T8JK%W^>asNw!rCYUHW4dJa7n_PZ!{^UmVK~tn_R(!@{I{AuTcGnj_WHT>`uj(3 zUAwG0=aFD2Wn3+Ilb+|to&~dYo2CN%kKFn8!U2`-pUZ^s8adtCZ48}tuGR7*zKy1V zI#2YLjc*zy;nETR)sDVKY1gNUcD?CPr#$D=325WogX^_M1^8)y+gX>z{vTKCrh5RW zF+8AZ8u+$Bo|9d#2Mwgi(b^3S58ncu#zEAv$0%TZ&3|Wk!LBKwi?AOnw9!t1NfX_3yqmPq)qDFzftf-u9gRU~b>G$3DOx15nBu1rF_fElZ-S$nR~x(9v(4lhKE8AZANiGrqBQ? z1ri&sL?ogPM*3nzT})Ke2W9WSk_bn~kEp1~;l7S!2a3`SyrvGYJxtz|nOX;EUpvUK z&^Q7k+32p|)^nOnJzBkgbPag)4#r*k(ID~eN_^^#`uy_fre5db zJ;IDpYWfd^lRCzfjVnIy+NW#RYd^X0O@a4S6?gm({o>Yz#FRik70HEqsx!}&@4Gh7 zmKS)~a0o{p?UGzuxz5&)x8X^LXynb^!rYcEzx_T6|0HenCRR_ObJ_1c@@m{)+@ak% zJ_K9;Wh=I@Y~Q>8Z*7>^$ZfQ!Jm1bFo8f3<*K41Z63AT@Nm zBCoFz??I$0In%c)Z{8PHcfRDwsnFq#_mG(^#oZd)HLM}9WVJY{8*kJ~)`|OL4FP4) zN3~Q=Lp}&jBw!eeZ8QDo#a1&ge6Tbm=}y>Qln}Dd3(>a&YYbO}jnK0*ABdl$f;b-V z)aDSRQCLpd&x4@dg0Nb+@MoLlzS~c_ojgeeOw6Gp;VJ$}B$L3bz=)N&`MP4PJkt^E zdIUI%{S}rWT091fBl}qiYIrFU)NL=ySPv3eO?iVNnk>141@t5=H;*D!a7u21_yjSs z3EMhM&piD6`qUQ>GtkDc6s&oBF(ehRO$8VzZcSuXo` zeqf{eQLrbF#%!s^XgXqR*vY%n3NUpOxLu2MJ@~|9F=&9~Aq6pc7k>unNW2y^HY(g`3ZAAPB+b$S|#Cvp{Y4SWSMEi}7{({ED;Hfj8h_s*F2uNFJoc zf`DyTcle;?C^Fi?k#%|1WjS`?p4FyxUK8x>=a+z5a|55(ND<(zEvMKNgkWWmhOPTAkXgmXwdT+IVeiMI<1u0qk;Dh4nI z#%;V4?RLmfwz19JvbpjDeeI{;5f~i^n&L&YNca$kS(du@rM&}YKN2wGN!&`lVgwX1 zy(xFq1G)wdCSwie%RR?HT2Nd=a5-{^?+F|tvZVv7ijA5uEF=f>c6HAIJb>!c4?2{< zWI?#U&T!D1#TKy0HzT5sZIV>C=EZVTXXD4rVWLa%i_weO93 z-eBkuxUmy3+9C+8vs2~P?Su=-UQ`ImKuE8Vq78UG#kh}Z<*LhDSMD-W`!g>BbqQLy zeBpE$Nc%V`>0XV?9YC2(O}C?)gyAN)D_wt`XBUj=-4J`ddL8Y8@sm4EE(t10-sW}A z)l;nX`CXM)z1W345;y(6R;j+jNBb!L_C*H39Y-V5QS+)5q?^86Y-DjRmm(J8o zT#pt~*kF6r@`=h9p6B}yU-A=mL|I!S)mB`|$dvMHQZy4p%h`#!}G+UE>ua;KpL9w~Xm~ z6(eH}9ZZ%~7}k#JEGjB$k?4zH6wWiicvo${euhzNEh7bXTrx&tF%!17&+WeSj1CDbW$!GLF+tcZ8N zdBRmdIHLA-t}2vb4LRk?;|B9MLcD}6Ow8lBJDhNlM*I)C0yJevPve?m99C)>R3-cF zPx1a4gwl%b_3-p7S(wF0VjWA>bJ7%k?-M@qK`@z<^g6--b&p&j{Z{Xv*b3S*$puv$ zLAa0@#?$o}VHlThgkc=EU-*XzTJDy;7M9)B-}PYwWQeNg5KL z!2`^7NM%B+cCaY1>fGWo#c+!EFPnzwMKAX`I`_xvB;g5vW2O&xUF(YhIFQgAGHK|- zBquUw9`!Q1tQZl~ITQa;YHj?g=a-+PIolTZM@LSJ|GwuQ^<;*B51oslC&hb*&N9Qi zQL`MrBR_4*%#NmMSamw_|NdH6e(plL&L)xm3o~75Mo%6)IR6&X9{1iW?va=(>fSy# zIp)sfA#dw%832<@x5Tnf&^s&YOzSqtS3F(~6e-A)w7>TY?#rg234Ia^H>q9Q|L8f@ zpugM~Z0|K@QvK!g{uY|+xu2YiX}oUQtiltaCeLr^qjEKtv)ispJ+o=!a1l$(gM70vAwPKnmj0f4=rj$5oyxhx`Wfj(53+0%Xjj6PYHleWw@7vo{$AA0hz@F zpS|)365HpHe}~zDa1w3@fkF;^IB_wYPPZkQ0dM{iPuOAIx?{9ZF~Re!4N%_oQ)VJ7)7ck3{C3j>21bQ z^9L~y%q9~IHF{HW{V7p!7vnGkK;nVzCF5n48)SPZha<%Xs*Vp$ek{2bK|kp`6g-pI zx9;Y{OZASq>fIdKEb$%cvj~< zbAIuwZSdN`>jD~1^(}v{1)g*Hn|FNZPi>YFnW@&Yu(~jNY>h&)c=*=dqiG0MT@m7m zb^PF`A15tt;K|_X8s2eIS*rq+sn89Ha<zHA83p-&pu%?V&)(J&qskKe z)v(ri2fJQdhI#3Ms$6F!nCat>iF^B7ywIMws1cdv5Y)L=z#Jf`t%5Qz*KRfQWSBY5 zV|Dlh#{u2F!7#kAQK*ew235pZe= z$f?AJ1Hi#V2FwL-v4sLGQ4A-h+fpAdiV<5)Rl> z^zy)^E15-w)rRBxD<;ArJGO1-jp{n|8o+^SQud-DR2aMzP8G)m!c!h^v{Ro3o2JDV z>P_PA7-o`}{^34ygs?mA>f)M8yas%q4e6|b#CeUaoG^cm9^KXdLL1f`;!@PoqDsFT z-P)9jV2L|mGNf`;i;@b!<=B4;I9J5ZVJ;voZun!e@*~PWORrW-3nAP`=E#?)XfiNh z$foqskZVgx2{ncnk@x`^GTnJUZT~?!`1rzd$<7<5-j8ot1uFG^ZZxYz5kf<8i^F%Hyc&&CF`DNB=}$OuM1mPGE3zOXR3~ z`yUTmtF5`q^?1vaQYob_?1lA>6sORFo^(!ahyhs3S5c&-T*E>*~K#|D^FjsfvHBK zvp89kjox(Ub5mp5;-yA)t6Y|R=e>2~-%n1(&>BWf2RdLi{sWI~otOay4?w6tod#f> zTfvgvT?k#_k011p7DD`>X(=d37KM;~`&{46^$$P<$8C{e!>#d|IRY__a%o!;aI=Zv z7zlM2u&~$t&89Xs0z^(wzHl0IKL89^leH_M>Bo;?T%O;FTBbMnTQ2t4^sJ!l!=@7Q zFM~cB6^tW>f*XkO>StFMVGv>Ra_Ejm>?7Y>ZQ_uoY%wt(#Rp3SKTLmZDM4ED-#CtfVHVod*)d0Td+kJa4-tc8#4rc zwEX^Yr3C5ZAn|NsKW+W~pX3E)%k9G!<^v3!+6JzvD=00|w6VzvG_DWMH`a?9iA@ov8 z!gk65rDDp}+_D>?j2D`F!+EJMQ)L=I$~q9S@YA;Z^_Awm6pEZIBW#Puy-ddPq?MdU z)|5^+`5&n{(m2sOD3UFA^;x>|&1s919I44{{TDdiZrt19<~zSeVl&N|$jCE+ji!3a z*z=gU-)Y`|w1jeP+Q&X;76A~bY##?#_~=@xsW0EAz^{~`wj|DIg|ATnL8l%6%TAp} zJv@;4m(R-|zKqVqjZMCpcGpP&!J??|L!fEVV;(Jnav{Apom!bo9?SD#Bh-0>8^&HG z_pE(wzD*rU_E(ohT6nH>J-G3=o2F#T6$6Uu3zcU9M@^691uE?J_dj$VnlLW9U}3=6 zXPbGk;WW}9e{3XxcpTTbXR(AD*%N1Hka8}w?V6g&yN5u@pK&$lvXLP8g@(5gw{Kr1 zS_Tg3|8UP2Cz9l11Uar18Wk zwvJ(B!sPr!VF+QD;1Wz&IV1P5`!2?7od-$)wTK)4+JXN}f_%Wy0CLIDWP6Cvx9Vjq zt7tqid4q!xy6hm-kyz);RO5)V z-Kc#XqD2R2b!E;*IW=|q9hn?uOC?Gk98hC7?mc$NF)=#7)LSKyVZrN7$Gze6@gYCk zO1t_g-km`#zpS^&_B@?*eWyKrNKmlg+(msRxKg~f?0FQw34mLGMc`ZZ4d zvAWwPll(@Lk~IrZJ?FU0JcxugGw?eL7CkQ$zyC5b_M z>2}0+ls(pE->b6^fJ&wxCbVdrEbjYz5ObG&>7Y?BdVX$h+7Z6+YuB!+HX6rh zkemq`%>=Y3SJWuHrji@>(SOeDn5cFmxeKyeduvFN#}vfpRHCd+{fx8 zXm_~&^xIB36W|JYBD!H&2QwIb>J>-HL~VQV0j3K8@~ll*@&`7xXka{z<)@!xNmdAd@ev4R#HyR>@d6Mk1I`e{2V4=XDdvCY-6ZO6$q?2ExSXaqyBB;G|l%+$U|L*}|VI!J{` z=xeH_)m_YFco2#GKo@l6~%#_zP3r5S#EH{4|S$pO_N?q!u+j(yNRvtl0v~}U4p(lfL z2*-AgGYyWnrNV;N6+SJ}z#zGY{(I*;w;Ih?dOmjCRjAWAxbnc0sM&9Kp6p&(8m29t zGO>grnE%-y`ppYl{i!oA15FJsyf70;&T=L5JnUT<2>n>JHD=iGFOn!GR*rGkG zT4XrGb*f~W;4#0Nf=06s$Jln878iY&0X#rTjs8sD9d z;!hlku;2w#@}cL)pq%YOitgJ%;@^p<>(?+2=FERg{7VG}v}Sh8AEX!GNWD8!I-*9`9o_KdEOn5yoRO|@^T26O`lfOLqvlU0ee%27q zeHN~`ExiT`mJGH+`h!6*g8!r^qEVhB_nKP(tbdL4Q*7Jh4kkJRq4sU;54LE|bryR0 z>J^uYqZdMQLv{<}c8GE__Wz5>5iw`};BNzUEOu`d;*RZ}a+k)2LaNu?+>7rE5G6T% z`ZUZ0N-8Sv*|{D)LEX~&jysy1YGhgufzS5THZ%l+UFjufX~|2jRFGd*gbAuA$4R@N zbw1L3!~eE`?3#zi7ldaER?seGArx{X)-lNEmP&lxu*Ib(rz&RhF2wZDIB&6UcbCLn zNuJTAUr4=O(yMSCE>t(CjXV<>%b>ecwv|)+*BYqL{3>krzZW4X@@23HhP}e+ADUA0 zY+^BeH3sEBCs?DmcC+~PKVsR?Y*jL{&?UC<@TVPP?|9V$%7@-99KDjbaE5;Oi>JK1 z-mMpvnqkXqGoEicciD#>&&5zsgmmeyg<;u96>gd8ddEF0W5>S=)VaIUd(Cx^IQNgh zky0dNG)=q2nD*qMlES&4yWpgvJ1ddKeEw~d@dYIhnX0^#Z_VJFha~!FO7SpXjx7Tf zC56pS;ai;R@wwMOo2%yppJ>}U#%0I1sGZb)6{GR)@0v0rHWNzadLZ(c+jD^w~u2gmCst1GoSEggt=@Di*l z)S;nRBo7ol@k)7H+${8<&hGWTz~;Hno31&899%=W5#dwKk4H8kA#_FgHBI(iJAP;F z<_z38c18Ht${(Jp)UCh$2BfIA%h&4ao}zlFTmDe@hbyP$&qG?xo6uaj7Lf<8KL(Q&oI;2d9!fwysOK` z{5x+V2R#?M!O?f!uVL7c4vy>uF-bFQvqFH?ukZ=MH`o_;lO_{3e5}(yO(s|d(8h;H z=GWNGAl9`teefHDL4;o%io@(jwO)iVt}Kp;@P2nn4$YIx>kaA%pSeqBVV^;Yq(T>Q zx&N*ZI=CY!xcwIyp$g%XK^;NQ#hI9T9usZJK}7()o6F3ckZ+8h`xd>OgX3VB)CNI= z*YG5~sfAz01x-w@cXAaKsB}q*j7lB+Uc&K4UK!_?;Q5z_Ppi->djnuVBnty0&42{%1t#b<5e;GY=1TTNeWRSxIhVOk z+S%=3@oVl54cac1sp5A`d26qcvXS*>#WKuA+jzL)o9;95ahZW@PiUz^3%Rx=v6WvI zRJ(pddT)=q{RcKTM#pXVptQT#R2-~!t(nnV_N>k;*$)1508TSs#H7-kM!wG&b&$WA{?N+N- zFj1}DHKt*?zBQvSmPu+RIzvrw{LPk?lfMRq+IyY1SHsqo_nU$9beSAW+~w+Z-WOe{ z_6ro->&*y%_m~;u(-xhLI4smO(sg3Y?@Zo=vETuQrW3AmiVRk!ing zT*ii2m%*`lWWI_AZ_-2w9GKfgW3Y6c#^X~w4VIT*>|V)tooZ>n+4CriVw0J#54x@& zYgH=rV0e-=;K9-cKi$Gm-wfkm>edT%fzF#^)IYu1)k(hT%C7sg(+A6U9+H14vMS@I z6Xn1_sYBjub;yqkbGvu#6QOfY+$IYHWkoPg;1IP!)@;mBF$-_=)``v6pyJou_wXIA^Fl>d!yJb9uv$}H~g%Vd7+&Uik z=?L2AK?MGd(FdM46qI#^!* za)yWuI2ohwfAOqROVf7s{r|H%t4EqnKWkifz*&isC|PO>v<_58AL{jW=WUfh=5s@9 z0)r=Uu~fhe4Y`=^BoQD*5CtQ{`oLRjN7`;6i8%bo+Q5QC40-1i9e~l^3|iIyyQ6x#oN74kp^5jfCWdJQGZj9tv9K(9IRV7%Yc6Di9v&3+B0YUQt_asHNQwZA zf^5#r30SBXX?epg><#tEI1@vLAVwvgJ%@loJMx{2bS9AS2%rfqK$7A)41szv#RJzI zdT~d-VODz_Dp?Zwt*SB#o4FQl0mo)}WhJ5F{Nb{yGf-2Jf>aY}1r|vs6F`6}Je#xE zEa&!d>HWepTf5p1LKGv@smbTXL4&{24t=uE&Cz;%N=iD%7*1lJ;g}Woy^>c^wgn!j zy|QYz;jhTLz0L`gK;X6_sV&uw@ibHa=QE3TnaG z<%!jt_onrNe{+w-)Fcg!dr7R0=B7|K4zQNRXU;G5H95O$n@4rc*eP_}*WkNUeu$lp z;rhz2-n(qg`v#nx)yujcJh=1VhrRQe!3Ph%<>orS8R)ATs8S6v&uM({U}@EkuO?=% zIWKX_uYJgO`qt47iHX(D6^6_Yy&uI5xfU9Y`HbA}lIqeA>#19DtjpQy>xCtmysiEgehpD*fWy`O6mS$(py&4f?4!V^)Sz5nrgxUO;vzJ4v=kL_< zRr3ZO9y^lhmMX{8{Xg-IwpD_K)q=Pw!$=|4}#cTH~zW zr2rK0Om>+(KWvXVeILTyqJR82!;$;#e|?G~>~`mcQ}X+3p993AziCoh5|XO|kDb}c zW*4neO z>uZOx#vQx!;(c)YR|;i~z}8hs85ue-d`LJC-1jMpSwoY-f`mKmLd>Iikw7o|*aSt5+B70nUD$m-qeFPQ*zy zYmHJTB=%o=aW30n8pEcOK%?zJdlGzXz|P;=k`lY}G6Q~+oX8x{S0l2o z%w0I5w)A3`OV&il7psA%Tl2?cf}b|kN}lp`x4QO4CPUuY{o<9g<2CsiFN2PbrAB#a zlsOxIEUW(VI6lMr>wM7OPV0{Rx9|1upKJ1bk{vMZYtHs@*l>g6K%Cnic3-AxSK1Xw5w!WGX?8PSzcC~ZSI*~nQ`Ic9~r&TNon7b`FJYpJ{`HD=V~|n z+$_RYK&vKjXa7uM?NgVoF!%gR*5j{^KiU`cDSdi=O67fE;TbmNoOXTwG<%Pz;Cv_9 zsP{=dXM1#CZ_z##=@E0d>@LlM@p+}1m0nSWHw^DT^%{G#!SPr2D~Zvw*K_X9q&Dt# zRP(vryic8e@6XkeDV&x1yOatSDp(rEB9t6jHGJ7yf6eOLQ#dTP-m0uHU*X9GMW?OW zIgTj>Q>$ON&7}rEo|5YdovISg)oCs~Ffp67=X~iMFBa1sCnXvF@f=YouXi#R3s2tO z{?AXp-u;taVbtMB;&L8|B_^ALGWACv#9IckbgY~edEx1vaa7>Pjr9T)%ApHwebdun z+ONzgwTj{=e;GI*;7)Nn*32dI~2Ibl7kT?bOGgMKgq46F2dXMmO>1cIuobiSD0AHKVPsuClW2^n+hYx|?K8z1& z2G79>Pj&bfcW{NFo%0lf zH?8d@_U|_=T$m}eQ(?|*OlDh{Ec2~h<#&((22ZLme{hEb%MH_79T8@*u0m+19$AW8 zH*fBW*~d`)QOI+c{_~eFis^PuImhtrdM~G3WJkxg_l|iKBBi?87fnrL3Vw&De)u(! zNuIc&H}s(BZxKy}-uvgNYg0Y6+=OqYa(EUKr;i;y8h42EeDOS{ooKtdW;5?RzAY)d z`uYb>sA|)gA?W}|y5XQx1Uv&d2woi1&=7vtov^`JJl5Fgmkxh)`x5)5-_D77Segc& zw8@LEa~&&;fl`Zyn^4JjQVhP>^yYzq+VavGxM~&LQYSo-XvBx%B;Dy>r2XH-SN}RT-H@S z+WXqKnr+^@HL(1s8ONH~p%t;YIwPsDHu2$JPNn;XPcE$8ZEl()YWQxi!MM=iMOnimb77B? zn@_~p6z1~rMbFe7bIcs)Un_R&K1=FU$Aw$BHp%o{E39!SI5_s%re|>U-KIeeD_ft= zh%UYzre_N!uc@AJyEeeDA}YE3pUauzeWKjf>3@BWa_z{orl$g~E8HrY%$=Ah8(-o7 z2c{ooGUKxxxl$*m$wAo|e#Y)le?cg`XgSRG98H&=tEEqA*9_Di_|@>1=hF0iXwS^W z^>&#Srxl;+oGRseyHEN`A%$}Ep6Ny_8=GUNPsc-%tV8CwPt+;b5RrbJ6DRI;R!0lv zKJdUW6?8y5oN|xp>T-va#bIDtFLQ9=4Rjx}4N{8AATXVJ^Sy?(d(mi=zpFf8VqHp6n8te38W6GPL@ZJLv-U{|IUGXz*c!tF;<>k)o_b+#xoxRmZ9e)-C(1EVX6QK$F=u5p8D$ZnD4ZPo4X?SPX zfLU{Sd|ajPg0W}4RL2Ll#tRj^YRA;h^@zt$tExEpyw3<3Y?d8}b?6Xmi?n+Ag5`fN zD--4c9jbpWE8*+m@nw%=D-%mEh2j=sQuljZAK&2+5y|D!U8;V~eY6x|fn&~->AWGS z)=TO?ePyd_Wlj}zpg9%Evb?tRK>5cq0V5WxVJp+XqrDf}Zbb--ysceA>4aNV3(E;+ zB0qtXg336<5nj=?w0v@V=ZVA(SKHuY$|S)Y8V{_U)ofN_C=-)c72vyzTIw+6D>S0Pw@ zRP?mDc}1%|&qjt-8K3Lxts%)iLZ)i`?Uiev4w@kGIQPPpWzJr3?gq_ zs(T0jE;qKcmoK$2!xoc`=Jqk=Hl^S$Vu5XW2iH>J_30TIJ*XxmJf_o-y;wQZ+0|tY zu~aYP?47Nvs>Z6AvF*FH1#qM4NSyd_Q5|Dr4o>4INeKyxXheKQ{Dz5*%}(yqlQLn3 z{I2esxkBRZ=9JK%1N1%l)6z{xBgOlJi-gas@fYg1=vjq`DTh6C3d-FfrYD-Q<>j8BLXS|enQ+<6RHZ7}~ScQb2FZ7au z)UJoHx4aQnI%XjwsqPc7J6}x$)D;$?BW}ze1aUES11c^ZDk@`}QPy zVYUlvx7N_PX^-HH4vIuKfa^|eEH%0J_GAqrPg4EkO?gJCp5w2gpCl&>Jl)UFe;5^F z-5%q{U9y~OYXGsr(9)RgBY64g7~+eI^_@_L*V^k=`H?de2&^Y5DFatMFnbq6NVJWN zs>Dv+=I7)4L~`%gVXc9o!ZQ?jmn}fv;b%GH0|`f!%qFLMgq)V^;nd7GGI=ao&3R}O zD<$qi4*VkGx$j?{={!gOXycapcbBZQ9iE*COPelO_TJ&#%BAbd-Z^dj=X*`@Rz5S4 zDfqvNk~7}>hMkhD{#}!pl<1r;wNjH}vy8UV%QJPPVLwww@wR*-xkfW-@3XBxnuRMi zU5&w8jve5s>$X!)TR#XpddPkrTNb{>u73KyxPZ#J_a|e9R#Q|rcorJNo_!RX0(>c_ zHPR&n%PxSFPfE?BX-?PoG+tEc=Cq~cpnOj*}23V=9tJmum zP9NjqZElr1w`b&!ZCBSc0vPqK=P zj*zVE>?E?4nf<@+I^W;#cYM#^%jxHH#e7g0xRv5QC5*|QQd$vdg`zZTWH6t$!)2TLu~ac|x>SetOJI4cedgCcI;9)>7xjFM|w5 zA08kIBzIh~&L^26Nxg@5FGncVP}^z^vunKY?9u8kirHd@I0vTE%Xa3Ny;k0GK%mu^ zF2;9^sZm$^_C-n{EgH4`8gIa;b3GNYQ^MNrPuFls@EYO)0^!z|eb`l-e~yNR2BBcR zD71R??x)X+$Vf?fLVM6<_8Q9U$4~`iKh3@FA^;TV**Z=>sE>~h*9DxDk4jWheb zgoi+JJ|XLxY>tJ-PNBRMtQ`2hIY$WLC@Djtu zb0YzH3i9&u3U>PX@wwA=RIURDs-mvaC8Vb-l4=7^Vs2&Jyu1x4WsE*d(B;=^F}cEa zD-mXgeQ0M7diFaBlwV09XI2&a)*%cy5?XB8IA~>rg|~3>t|g~)tVhnJpsK+?D?57} zFna&@j)JCKhwSc(rv@@AbQaSEw&Rl2pKN=bQx3A_*uGZQR11kuW18r?9(3BRjy$)7 ze=QCESpLrgz_;vOQ&QG$-V-KSGi6>&DFs;~yeCquj4MPotM^Ot0b3K{+ZAL6lsmqr`AZLrlgb_(?PHoc}o0Ddf<6l zmz3GYmF;jfGg|+~jgN@&5ky`B@6z?s36lX@e1MAjl#%7+1~C#m-KU%v4{zm1zS4;p zTF4pszzq?Z+@Ml|A`(0mvJjn%2PwLvQ}*#dPB0cz+t%h-Kpad|b8OS`zxj1C0|J?V zTxRtrXQvsr9vy7XltmgnHH{(MJ`S-dcLpBSD0skga&wpd8TF`myH`))d65y9b*Yh* zz;1i(Q@rYZRq`Ge>2-odi*=baBJQ?xPy*ct`TqHwDU?*tf&Cj6eKBxEvPQ=uqmJchQ+d?06wtcgH3GR*i(!4~DLW5uZ)JeTC_7h?ngH_<>>}qcU%p(M z*3kcSj3(a%5LgmcrK5W0k}1| ztYu7t$A?kl%QLtfXf;$e7JN+R&e5N|*}5+O=Z#U9^nOX|HclCTztP+LXCqqmt(H>) z>o&0(x-H83l)!?2={DDU6h>sPr5mrjp8a(_vu^Pc=bcv!8CnM=qkaz3Tc!wWk#DzA z-)^a@-xfi8ajmTeJ?s1K7CG8X8($8g4>U|_dl5<&f{FE?QiN=!oV^U=Xo8-p9u_bm2c0{%q zVNQc>5?1Uuyl+ooL!85+T5KNYj-MkjVRmYi_itjl7@nC4+lr^LmKBxC&M4Yqc#%pKR!2i-qit1M*Mj-lfg`K zad2Gs-E<`b{Cz`1-n=E&)ykmPbZ-g^5N%z+AoD9!C+QHQk<>t2T=tswyIhU4zTF+v zXnX~=6U)c(p8{Psy-cI;rK(*`KesWLiz0Pop&{7&3F{KNGJT&fhW6_hwSyGrBV{?M zi$S*{McT&NBSI1398#}`<`bY9$hTb=7V zW+^H5@~6Ve@X5g)-b%3P3yzMtoHiIPQIL~6AcVc+QYF70|NH*&Am3%L{BfQ7G-o}W z!ZB%f96hDK_Gx87F`JPn+ihXEN0f1xlIJ&sUFhDb10Ew>7rVysaC@;z!FZU8F} zC1>3Efc+(WuI=3L^Sg)>fVb>%P|(pIJ8yF)QmL}w|ID&m@rkl}OI3f*DB3rK|CQUJ zCVSavNcWIQz7a`-Bn6-d&4{hfP~vm#-(Qh-8HfY9$n$eEtC~p_k3DA}I!vW(Lw})7 z80H2xgdSAyqB+L!#HC67r!L0F5>pe0z%LJRX+u9XFS)3wD0T|a=C6m_SIddA%>e-& z72cyZ?C*(nM0U3Q>LY$NUB+8y7=0{8p%kGyaM$;Ccdt^;9oby|gufUC;I}V@9=1>{ z!16k>iHE^s=*_51t63=zy~;}6CCHjsX-x1Y2pnE%YbMmiV}^`SAvo)XyZPd9$f00J zz-Eqqf=^iVJUkhGK_RTIO#=|Zc^uYy%p9g`yom1tJcxB!KFy@9IyE3_m(`g0JxN@?r2nU{=y|&nqqbeAAb3B zSbs^90PJ)Hqy896kBW*SIRL=Ghmw*fBnRj)xk7A69xSL_R=`6j_3>f$`Z7L0l$Bl8 zT=`OEAE|pCv9OFb14=4ZA=Bg5t(F1qB~dsB`Y@KeTZ32v$`@y?G3t5=P8r&WS;qyf z;Ns4zm)AdHQu8wHf4a>F~1y+vm0yWRuInX(0d-hgCMRxAI&My(zTdNbt?nqT%+SH z5)EXO00CFGGBJI`y(f1V$Gr803B0XQ^F_iq2s4CwW=oegY^0-$6drnY3fKX;?uv+8 zQL+XG369V@#{!?nfK&reS9Y-;rZY6BsG7Z9M_HU#3vQUUuuC;R@iH+)Gh|{DE9w05 z6W>iR{_|;5oSzKnu3Gf9{GaE|n$md0{!{Tc`G-AYb1NTUdUgA}$n6ecjDieYls*3I zQ7>tE+i&6E|+1_x+P;pHLY*TT-@>*$oL zI{W_pGJs!jC*{FC?4uVf+(+kry0)cdA1U;DhKIM;s6Z}@hkVx?h6y)iJf1`#@Z+z< zCnp2wvmJ{?%NqlRug2_obaXu42&_xnYAB3Qjc33v;!$`-?!<}PKpB1&dB`dPaYtZx z8yTIWF-)HFI@~V;+>)Le=1fWE-zv_OuXj1wl;^bYuuH)3)3Y-k4AM$v!6;LS1^Lrs zHV5Nd(HTOB_qeo+1a*i4U|Al39Ohb9T2fMlV=V2^PT@BIW!VpAyo749D^S1i{^qPsFX6`_A%-Wb ztB%xuQB>FbIKV{FbemVmP{y#sY-nG&#y4||5 z)niOg``7ebkAg3YRZSb(ocLz16w^8Vr^A zBS`j(JLPupO&c6Q*MlPL_w6Oz^5x4d%h`&g(k=UykWBY(azEz_U;rD@6HtTI)?OyP zE`s377+!S8(0$@j%h>v3xd`YiTA7B123cFz%Dm)ND_63MEaOv!!sHiv3TzD~+cT~+ zZjgvIWm)#KMKcQ*@TH+QM0`CI->RFc2N)^{4@IJc{j9QbK&ai&k;8(5>z^oJycj{u znZOVHd6mMh%EyFo^qf3Od@z&2?J6YZGz^Jd)76!^cv5SiyX;A1OsK6SVp%Oz&vlrz zimC)oprEdv1mQt#zBTl)v)22p#W{)a1m0n3H zd$_Iizwf8bzueDv$mCg>d*#GLo1eF7q06S|hO@aTPHx`w;YtdUyWmHeu zV`Q98=iGkb8$utIKL)Hl{^)E|!zW&o_k6;%eBm?_98LD|y(9fzZO$rvQ2q$pDrS$@#qRsf@uE4`6bzu?6gM4)3{{(B9>q zDX{vl!JBXK^LxkJ4A^Xv<^{cXbQd`)*mV7MlB8uWuC!~VD8nzRs4yLFt*wo^u$L{{ zVZ`_~HXdz1JUDK9jcH}^jz9)4zuj!5S7Oh4RhToYXNh3#9pt<(7mGFl9 zJ>%RtU+%Lwm*}2rvV5+bm&&I2!z!WiCg0iK6!~u=>nOU9#+d&5l9ut?Sf5{XOY@_^ zT#x6su+5E}L|mjuad||o>SpKk+QlFr8pC^obX-0y4k!GgAp!a5qE+9owAfySPC2%2ks}|uY0dR%cfIJo#0(viNXl(`5 znuJCRU6y)|t))~a25a#&*NH+Vay8v7m%h`WCR!Q@#ly;P%!bW5cA2^JzhwQ9r5IWdCq_B30w_XYwXA~wFTh#%lb_}k_F%bV z2vdE_*|C_@I8S5PYI6hRBKz}Yd>eQ9+#@wRfz+y>pBEf3NNr3rY&;07b0u;H2YHuv z7V4+#*TpNM0IY(WtR>)HEbWXHB{)A6proe+tRhW*-keE0_!h;67+kkn7K$#lP3yEt z-FUCh7q*&Acc$PmreuR<&RagsFK(px9<>hqGb~dmfhxH7z{{xXgIA(h(qeecBd9?_ zfsWR4JXVsaJwfQivpqdo6WD}UtLUa_w{^N}J#zssrR)N#@HunofQ&AV*h3<&C+#V^ zEP3rzU`q(b82vwg(6gg`(%j5U0q|FK^+ggg?a$@2xoY=7mxD6;t~Ywx0}uQ{L-z-i zJ_ni(00Lc2istRrjEsyi&Boc%uuIuyXrA3lltZBEuBaws`E@V0LER-g%ChXIUILQj zI4o{xm==WJnjotm zkv6YkXBbDzA6S5J4rUw7)$`B|#79NF`|~v2Zd^0$v{X2ArtqQ7-;CH}AD;tLZz)pu zaGGIWa}phsGB%K}n~6rvh#DC-I2XYrvF%yT?p@Pc3!;p`A`YN+0OP%TII2k$5|CG* zO(0C-W_tR-$4c>i-}{jY)5e!Mv#AfPj*Ty%zND#;l@c$JNibw^_qL(re@(e0+WKFQ z)%nuDh2`dlbR5hyyF}@_wY?2 zQNlhS%qjW&aqH{2t+G)MsN@O zpQk4$7ml_HLHNRj4SofkKW<==&Z7)Rw}PF)8+?Zvl!!oUjXyadfRlLqTV1lNFK`O#PfLVi1MEI-O1!<4qhIlkGdY|ED&doN-fU z7ZoMcBzfEz*Ju_g6GHnkK0f}t3y!B!mG{zb9T$Q&1A*-CT59j!3sO>L%h^_*tu)Rw zz#BoB%9Uhx(h^!6Na`vcOQ9LYEvbeblkg`5=G}lVKYjWnQI5nL3kMDwQcsogc}mz- zrmF121((;w&SR zBR9gfw8T@S2U{3OEOkw5hr0tov{$>cyQ1rYZQu6vr&AB$kfVo8H1KU%O-TE&GAXst zbiouKr51Xn4}RQ=8XC!1Um+p+hZN7>vA_cEID>6DYX6rV$4f?r3dWWp!H(Bx#)Lyf zlPi}59-P%aHT-I3?8eQ5#WXb+Y{}z)0d7pI{+xG*`Ljh&o>F#AP>s{6i=dr)^T+l?swtNcDBt9fBhFMV`=?EzE*Nm-%@CaA*#tMSu8n^+`s zKeO}LwAthv)2jU1YfP@7vk2^sk_h+%ZMJ7Y$43NiLVlsZX*3*2z0wVX*+ zZ6t;`?2)->PS3z_R(f(`B0f7?4WQT~|D8B$aOSuRS*G>(1sp{Usxm&20UKr~GrtuV z7Gf2=G_aJ=wAsL?ciukJxYe;@#oej~U#o#n;qb|l2+OTtgw%)dka*T|b9n3FH)_#c zwuPW1s<0A?x)y+zOcEG{0g3d@>Bg;D66FVd2;B)W+W8Js3prb~%-CCY4oJ{p&MnSA zU{??icDxtG*P7+V_8MYhVwz}xk*R$OIs&8q%&!%&9=hz%uhxDML6Y6{o z{E}x%i^S?hU#_XYUF7J{@`W{bH^sA>(dpW%J6vfO-#0D=b%u_Q5^S3=C1H8#_2|UY0Dr93T)6rNz;YmR z7O>ynIf`v!WAEr>_X-O)qwz9U5@M+bJV=lcb#U&#G!h8z>1e!g5Ts>}YX!kLY#L@r zV;yeWHLSNEgK!8kd_@U~l2>7LJxH>@C=2s`S&F=@XL*DPkF+Amuk@aO0i0;FntZ|v z1nhyGy1Q!VMgy=|i#}2V|D|%foY`HJP{@4INcJNM1rO0bLk0ul)AM4@@%MyOjZAr3 zsIcJx4YUw_kd|QZ+UJI>NgeI*v%~F_#(MDZ5IoeO$|uFxCH9`5)8I*di)gB@|7Dyru4IHA z{32 zI!^2P5R@Tf?)Mc*mtSA})hbMVr8~3BCsq^&wd4muf3uSaH_&Eo4X|W(4M(pu;`aLD zDlPpHw{1-{o$5Ax^cuGekKTB0Rz zC3}%vijr?=mAY)v;n6a?#KS;tx|m3Nl2!Zr`(KyjKe$@dl%`|6tB^DL$%S_pf&(y6 z9C*&C?FF=fR@}oreBzpX5e?L__++$r^jfO?Y+%uMpq>;)?^mU*5q^Y{>RtIwURwGA zdGlf5xuxj4i|04M4=D5f{QbH1?76n-;!?WZOf-b;LREo^0X#IGD3}xdbdU-rFH94` zod1GdR+eRAEgw2)$M18yP;tX%uHo)_(HoMX==ZTwMUEtfU$iOFKgE%~2Nqy9-u7bc z81uw%f4@8!G#ng!!bjKb&8p0d44HHFKJUxQ6aZuD^%s}bS5<{$?pq|jyu-8yikC+P z9j`iGqHw_uH6;lN_qJP|>u^TSluu6C+met1aDDJo$|*1)ok2O6Uci2VH@ke!Da@d% zuqhQI6q5N1-*~h0ltcKv={sSfD^G5mDsWx8 z@zm}9S!S$Cd zM5SedLT_3zsOSw1?8p=}g%U`NSB?=jLtzUH7@-N%RZ~5ofv>5KE zTpGV|^QEI%^7-8F-&Yu+6ux4_AaXGzk%`%5C^}fl+;a*p7AV=b$<$VFDp8)u6aT1XmcEDpP+zaHicc3GSR;~6Vz#>x&GCeppf`aXY%qT zJ35njWDOh&1Ef(o&5arWc&he^2%l?FP{S} zGeD+z6pFJ-5YVHyzJJzg?yZSQ#}jUZ>sP7VFNmn6B~goSFDeEwEm$Z6ac3z;OmcHI zz$!oTH`)hH$F&rdGOycP_TpG$_nyLz4fpi(P^mtC^5n>un^}-I+9Miqo-O%kuwhZumrX;gl(Cn%dW6 z0_|J2t{oo}K~^n#tM^Fw0xLH*9{Oz4i#fSH?Cdfs`?KpJ%7lVdb0WpuXemrj6IFr- z>U*6WH~#aKQS!%h576Ar_E-dalwDM|U5a}7y5}_;N|_adgfJtUfw6DRz+R@ibjmrl z)X-h$D122zTv8w7%Y=Y?Ft{_hit2AAGGesO{7opVEYyr=J@=Z@YUzVk%`)q{k2fBo z$=!MFVgm;u*deG5#N~crVU+@_Yh+|)y}|KVPt#keLsF{#FYn@uOq4^0H4xG^*A~9LdX6yrO?4TNkrIip&bCx2gh_vd z-<*0x3=iu8oL6TaUbV`Ob_5q&Y&`{A^Lrq;~oFXC)m%ZiM%_jRS zX1+H(qEG~{pE>I}baM}>uKxLvzwi11$C--7jB}u+SGsL_On5sQ@#t`)6|1=_@AC_< zEcz9lj_j4WUGjCkcp5Y;`gYN>+Ll^JgTn?sE;D~buag?Yx+4D&+9|{sLL-xQYJ7ZM zwkPnP|81u(B^lo36Jc@<$5;U1O?(n{AZlDG9cc?WWi9rf-wW3N)^-!p4joH4r;X!S zUMMx5`YZC(qPrqBrq!CyQh7f$hD|Md%ffIr2p_~!A z)ai-LJf4^uo0`ss`uaYCWnI7i?eI6MO3F#=Hk%mmP~wM;F?Z)UT@&_U*gLnI^nPf7 zZZr+^+Wr5OqI|(FyIfUsb{<~;XYFjiQ}Y|`Gi9{8yAq;sQ@h_aALgh?hHm#-aU-S* z=-0TA9t_&^efmy4sw}_tB-kGcCi=SBU(1-_sYj53!S*~&((MgAdi*$w(2PLnwp)os z+U$se{9F1sq?$qu5=1(PZ=D(`wFOHcK0>=7wqFNQN=UQGpdhMJ^X@XAa$V-pTNtDo z*AQu6LUDGtQr5nsbZzA6;vS09{gTO5n@l$T^P?MZiHY5`MOLdAH0?jrH*YzRmCfh# zS6SG&ZXVqMmrkTA z0DeScgj9vL$#u{qT{Ydb<0Q0TdXQC`39&qSxarjq$huHZ5L9$MwL!S=<~{}wK;C}` z4H69@5kG!KOLz6kyVLCpGfoManbt3OgoV{erAr`8kxKBlNC?Sq^?*-{O-G7torKck zn>+}>-<1*3GUqwzsxOzO+vV=L+oZMYaLm>5pgvJX|fDD|FO(?jhxL*_1{b-vVvAP0Vae$rc! zQHqq5lt;|;`5Pd4CB!wLaZKSPH)2 z0mb?CK38ofUAO=GGAiZDhKcvBjE98RIP2y6^?a6gYnzhNtv-fEK~L`ju3F`*Ix5@H9)y^TEl zz!XqN>o>l;xoC`XDzUjp+=sc?nvz$4^M4e`4L;6s(SVEy#fC!7vQ-pgJPm+y z{2o1`t8>h1jmd0rpkv)gVK2)`KFw2k@33&u;|(+`$4TcwXH~m<)BU0lzOH1o?k0Z zZ$J`(V0emHY*^^*c_^{r-OzOP{5ByM@ zL$Ux90HQ<8ZWUePe!GzaGeZDBmCm0p)l!43-*I8mNfRY+_2`?4CIj$hOd%B*VH=)n z8-Ad_0O8UlA|}PDdZRJCJSwCq7hAFPv0qWQIG-k!D40=$$Qc1*wM8S%G4`{bj|QP! zie#Sl;q3?8!CwQH!;y-_S*(47vAMaQ#bV%KG!^V_-b@L#A1Tqx;RWAI9Qh%L(R%*| zvv(r%KRbaJv$GXQG5o=t)x!Mod8jiOC(G&Gy5d2HtFI)Q?K!FtGEn=5Eha>uf_tgF z9B6(NnMpt;)H5rA##+t{W^x}natRVI*#-;!9q;OdDK*M(S|f({Ny{fbt#p^H_I4cG zZF6j!eEm~WCD68_O4z-7ci*i!@$14<17Z}ZX9GgP&?7-mvgr9zc6c1{7e2O#4%Ir_ zxAyE<>Zp1j?FGX>@-J}8B!sFyjz7KX3`caf{Ge#g;1Dy=IIYVq{gSmKo+>1~u0BJP zqud@-Gk9vP&@#Tt5l^PpOAq&E`(61f-w@*^WSRrOOW$8EYuC8B@veE|?@x5+q=)9_ zHb4&@3EdIK1u=QW@oNkvX<}jmlvzl~J|aAYy@MB5^cpRDs1W*KXJez#{5Prv0tGPG z^-vr3_(A6OpRV9OO!Y0j-5G=e6d=KoZCv^imLg}ZKX;3jm& zk-!UsSWo$4D2)o&N7$6!Ao{4I8zSZHjh7=H+{ZiL`7fibl=oMdR%)(YBt6Q1MyshJ zmiO`b?G4-cSA2hREBlP*rNHo-kkz^UOgNUU_P@qe=%k9y?~m)*8g z7pW}{^n%NycPU4lTF=PAS+9E-8A4(cRR6$92>y-!+Hv8#%MRFy0-eB4h=X0YwJ`O4qfj%3}ZjOp*7tRB! z55FkLggh8snNPpb{WNqB;rMHsfO$@Jt8J?J`etr*3AUb6YQoM`3kF1U@%!ueAkgfGb2NO%t`lea9Jxg(-Y;R`A(n6RH1~OPt%asPuQ<+nXXFA|Q5&prFvJ9HK-o zbyKu8bbMK@W27toR;c&8yeTb(UEa-maDS03V6bln*>#djE@^!*n_MxDkZ)`62sV}e z?lN62i~~Rv4!V(y6GD_(u%EOqpdv|S*Y?KdrE23@EqyGnM631?hUEB!00~iZTnOqT zgkD6zyWT&4{#<>IM(Yq!4P%wV7T28;7|S3+k96=366NsmNHgn_#H@wM3kV}rk!mwR ze6evNH)BGF%xVCmU&Swb82UdvvnREd{s959ebaoYz-`H-7eG|GK~n6)R$sa#?beSg zX#8YBfYhK%Jot6o3Dq}<>?`N&z{BRwcJXqvv+Ex0^hy9f+w(j?W9!QWpcodw$NO*+ zdx=DT_de%NBjGN!XVz4&0}{7734ie{(hGLi6qqbW2i$>voA8epEC|&YbPV_;*Sgn2=J*wg;BHi` zkpIHSq)@Awl0pPz9^rDS&t}0(YCD(|9QleOcdlmM8!kzAA_0)8*%*W4}vnIzVGN&3Sqio6N~-EJN};epzAP=mShj@>9NtZ>g39sbzZlw64lD;@9mWXcZcU)*V1w<;u>&~ z7~t4sUI)K4UC4%4DcG|1x^?KSII60^B6m#f06s)U#WZKsZ(QEXT$MuCfrmuqa==cq zm0$uCsD|crbpS-YSV08a0xASNv>C+11wiD zL~h^vOCf5Rv&3crs9mCqK9OhGXC`}`S93^??~2tXO61`U2TAd5drRYG_a5PLuVq^F zz?L6MGm!z)?@Yh@#XsIeT3sQ9H`A=$1*1p4WEiSr~({o7%=~w zzl4Yfg{=mF?Y6OTuw+AC3fY-?gFO*n*uv-YtA!culHE0Z>w2#S%hs_vuGn>Xd8jmv zgtF~jV!AYB>mgaKlg7ob)OfQ+bpzii6RG>NKSKDI@bROwLUpb1V_#l9A*X6cJV81++y2MV*=-o zM?J}ZEl+wMsOs{^0nwwgbZYxHQ%sFm=U-gZF+?A62p&;2!Ilp916h~n2d>y%y*9ws zB8*#ZPcO^$CGwYWa4}*>LM`&m-c4KPS%m;ll97BK9z|W`r z89?Xw;q^~gSIHmWais^3lcU+5>)ZSnYJYc~dr6t$_Cpf} z8ooM8+o`kNArBid*f$2EnT%q+NaCx~zOsuF`1-P;=Hk#Pwew$?z&$F)9PLmRF5cO~1^Ro-_eB+wqt82``-@iDQC3kd{E|3hfIV*JwQfw zG4{UtQW&IWcyb(94a-D9ymmZn_!4o(BMm9`(I$0G953Ke+$2kq5lHY809v@>ThYfJ zF*^-w>nu#arr-MUrNmQs7s_Zd!;TEfUBDO7jT*NH7q<%CnY5%^Z-nrM`Z$3wt|o`A ze{3n>#T%9@7BxMuQP|9*`eb9m<|iqVH!*j*qU^@?qjUP>BbQhx{Aw^J2&!=Qe-3SX z;UUBq@a1yW)|q-+kU!wWP!T&S@B4Tl4d8jajMKZQtGI?yg(`mUejHkK5uG{T{r5nO z)Ov%LU)UtF`)8zOmTK)X;yuS?pQEGO_g$WTHKpuhht7;g)se>Ty6kOUdCpI|jby`i z9ZchT)VV)j3T_z~JV%%WU#R4?@O}6-4zOyLMI2? zv@``w<3jk8zlv7~IvNLC%$5ck5mOzQ%*Q4r8NDEriR2rx&=yO3oBTq>EwQQ>N2X+W zvVAO2z66D9Wb{*+7}AXpk0-j9xv(?a3uEt-MMVQ6YIVB}B`7V8F0djmYlCL@WCYQS^ykd*hep*V|SIMtTTNgdb{Mx@-BNly)?+TtI zrXq)JU3ovP)tY{tz;`0yU?*U>)%8TPLuM;PGxC^RI+b&p)GKc?^2z7)_bR{pId*KV zbBX16pCS=(06+rwSTK>H^_^NyMeU2b(%E=(r5{8 z8g&JQLs3R}$Dz-v2bWjm;DB1DQ~Gt=gETrX;3Gh8lbpT%8}|}dGM|b8Gyotj zu@j^MB_oPuWslj=0rX|J&n1h_|I{I*K!>SNyc47b7@#538GZo)rTXbIP3Aw}pt~h` z673b!K@&jf1ftqZN2hc0pp@nRdclmZDEGRnSN7j-Fkb6#-@FL>LS^Uve2T7H^6}GX zXt+v&8GeZRWBF1-umKnQtt+^5!hE6}F5X15-)6c!S*S5iesEqwY6Wii&6hRM+bsIn z;3E8NQB>AafO|;^mq**g7~E{~g|dqMVmGHonZ7e@V2tA1L|V9> zx3M9H*&yaSaCNHW^N(9&Fego*n)F=f7BFgxLV|da=eiR=YHG-tHvLxJu??`u(+vcK z)Hi+S!-Gr%2bmCt=R%FNO9bEI<>>u@Dw4up*P%p(ZGi4qQ7po7{xv8P7pHf*a@-Tu z+DG{2BqHq4G+S%<$_seV3$X8GqLi5ml*TkGE_$4lK*@j6S&yZn46;fXi_VtaXnd7c z%0S7(B9B|5v>Ky_cOg^C{Q662jcF0R)&{Nb6X_Rx` zQrth@SN4RTFhuI=t@r#(N=u2j8m5-GT!-GSKA;BSBqTv+Hy;_FnS*}jN(w1Oi9Umi zivYl&m)Un5aDIY9=LT*koY<#h-U>j8KQ}Ykk8z?cRZPSUugcpo3h;@K(7E^ezM?!rkkKLVNGBb zupwzl3nN$U8<36QMBdl8KKgX}gK};J@T30ypk$ND6cW}})ng_#hEM@hnD?-q?u|*u zQbFR6KxmttBO>v1#~Vq}IZgSWbdVblSb69WT&VWr0)u|=l7 zJ|e2{d(*;33^0YVh!YfgTSC$5=9V;juh%KGNVr1Duf6i$8r*1D$EXxGC)rM!dEeQX zT0>!48&em7xn}gGP!-76uiE|!&~)W`8O_{Tqw=T0z~F;{;ZC2V%d8ThIkoQap#J1o z>}0(12`7~Ut7)Hi?cuVP%eweY>T>u%-kaeSmWRjB`(`*^cgdI=ieIhPaBkf4R-J9! zdG+#TeZwQ1J!?yb(um<;;%iPAbMC+1@@b7WUT=k~sCrVrl5D^88<8b0ceIam4&0Y@ zR~(8sbd53QgKnnRXzJEJmxZ6&sR+yc*xW2> zWnSxqyi$(rn&_{Ix4L*Fl+S99NO$!aF<%%h+;3Ss`M#O%VV$nx<(KkLlCO60mOkb; z?sI&wcav9-zPh)nL3XXVm=d&vio_9!h!A!e2EDG*>z@d=N*hR zRygb|WIO5?xGmw%wI`Ci=e8DKH+P7k6?}hjBJypoyS?lYi&!~kK^*9vt&Vo#;DnCs z?0>RupZ|iu&jr|28avGOKW`YwyjGgpl$WRBJC;uM>v?khY%b6219>A+-VvYzv}EQTV*^qFf+p@!9C1VU$O9}r1n_8)c52VO(hI;AG`wkN6HQZ zhJRa05hI~J(Dc@rO9Gq~2p9QV7;@pl zj+jU0wv?j%xZ3D_5Di!i(Gmutxv(yne0O1Q6v+S62+K$1=03%18jmo?gpXs za_lDmOal4W<2ovca0!(&^xSP@SwLc8-URkv%WCx2v8HTm69vKwVV75U@X_|8Z3w8u zTLNv-ne=9mzrE)*q4#?GLWeky0L>+X{um`h>$LIT?C6e#oqTL6ln8K99MW6ngS<#wmE>d`*aapUBd{5BOu@ z3{;5W^Xc8x-l#r4m@~l)1kk{s|MP2G%!8rKKk%T-REVBtW$_X1G?;0L{CGzvDJ8{H zXSViYNLH2ymqrY$EiwElMg#QAdRk3pX^00WqQCEKadQ+Wgv^4pLS&Dqe z$LlT&MI)oJM?P2mXxJY+WRWo4gvKq*pko@XC%W zu2^|ea3J{M)=0m}iUxm^$DD9vc~_}@Fs;@~zfw%82gmhHa>{z`RJC-|Q=voi1!i6J zF*1(%{Xz>j?>bHmi#^OUv2eLOE$q;=AoX0-rfb$Rpj^{GTsSx9WT`;**+aT)EOLWY zx{7(4F&^SUwL`g%RV=}+@xDoCn#SrbcJZ<{wl|#{yUhA=;nvs9Q;z*do_DkNEZB~w zXa@`qkHp!AseKb2>8VcXjy)jCdxTrVV7o??T3(t|io|a2*^jMF)OZogqmyzu$qH#) zJ=yd9dm{NH`nRh^jC=F?wCXWe{nXKxsva_9yV5i1XyQ$C>ZM9l-gH8Fr@%x4RRj2G z0-N4Y3q85Z_>!SAkcH;Z+!Wpg$EQIRmgjt{M>eH*-*)NT%=6(T&;Hl`*Gn9xMC%p5-8{&=)Zvnsso1dYoSUT zFJd-{jf~u_?+CKraeg!`0Wpb~5};}@o<1_0@h>jBd#Oe=n1Twhz0=fQOI0%Y1oy12 ztfdp#OnKGWelxvWpvhPEA`GcdHPDEV`8-k@5oe=1x)*t)sJTOt!e$e*Befiqrn$ubI z2b@oC7A(}OtC-@H^nM=u!?1>N+Ctc&r*W$d7uBmY(9bUUMuR4cl4kMu700rIrWMmO zWtttW>s}?=CC(2en8Z%myPc%_o@?2?EHl2{+B?={?}jS)w8!e3JT+eE9@#i|nO4k6 ziFG_@_tw0;QtI8jW09Ry&03v>UL~I9fvr_$bc63bzKu<5vdRfr%!N4#(N{ID@3;JM zE^=r=u-~FI@;v*2j`VTssj*V0h1;_ZZ{G^oc}lzYro3!QylbSCa9IC!mVDB5VV;9# zj!_%4SG#Gny4Xyl1J}ZrlOkhICik|Es)o)_2*)SfV3nIN-5XIetf5pF%H1^bpsQ&< z=ZDR@+zCdvu9>8;;&bU5ebw!vKZ17c)cINAYnx1aV;G_1lL6z{JEz8|*9m6EpB84r zSBi}CT9-04en-91SDtqyUWcmw@sZQ(Ge-*JcB`4*pZ?*UU@`JF#))~o=&K7|UFJnb z1In5?v0<}Os=jG9=C?7JTYZz+P9VF5?jzto!NWyQ*0&t}VVlIJ*A~Q*WTVw_>vVSX zV7uD$?T*7D64lHLgLHGHVZ(thESIE~1P_U#%nB0s*{!95W)7m--%v$8#=u@+$;!JM z5XS(smzR^PdN^mLwGRmcZc19W<&iKz+JGbnXR6YY238t5nEvi6z7F8&mZdzIJy3HMocR-DAOb)ez#yPaRdmV(QwDaw#rjgSy52JkQRKY7MaaJst;N;;U z_YG6b*7E%jGv>v09ms~Tqd~(?8q*1KgV2-LL(3^8dH69>Qr7+>RZHZKnpvH}dTZ{_B~&_d?=*y!xwsBY&Vp_+^`TLrp1Yp=#g zL*$lztDrN4hNA0zReTd8v3Rx64`b0)1himtuUp=x|AT^K&Gr6{I`!9(GKZ`@S>$m5XB9O9np%UT`N zFZb1Ub@KEX4loCdw+UaV$YKg`=l;jIUB-~VzpxB9ewl3p4sAc`Ux{> z=dQ6C-WaZlxyn>MF`Tru>XTCTnb#iGe%X7~x41~$m-X_gGxp0AeCF@*^r&5a{@n|{3H$&ZnV6&78mQ}t#dkV$q4F6_*AXM8>w~nid?peOs&6;-tRk zeWk9wCK9f`)jE452FG*a8;7gvxWgDC9{jlu{`vB2Zk@^+o-q)gT=p~MgNU+4x5H$h z|GE_g_y6>a%^0xZB)(_>Yd>R_pj2Q>dq=pFLLD?jYVi z*OBwVVvl$4>nG*I>w7B~W>X`+IL#LKOG(}JxzV@2ud7t#K25vj2Nl!vp(6p;VmqX= zw^rCsALV%+`F=npKaKG~Q~>W}52RC%nPodPKdVCfK!^Y&OAs{cnu)py{1~!v9U$0i znWJ^KEg`?$dAr{(je)&A`dVlt2pXOr2Q`%0$k?%!OZeu91LjfquRznJ)r|6W{DF!6 zDX1|2p4Gj6jJQ-+OO4kyBWjD^}d%Di&n8`aY3-rD0P76jqN+q~%U@xBpF1zx4 zY4Ms{Vmw|xBD`mk0X?-R5XStH3m{xQt(LYuE=|Na^{)HBiwdg(dP z6>hn%JoLq)i?{qjc8u~j(fGW$ANQ56Cfqoa!O*VyEXwm0^MuHhZj#0n?{@Bp# z^Hg3`ZTo)4nd5#;ywUfC70;h!bye*V9N50bb=<>g^6lqgWnW{v_|w*NrH5Z0iO4Ll zo^R8i`_k_CjGIG^O(N`#g+^hQdD@rcp0M(7my8(vTI8Zec=y-d8HaL&=hSq|+$w{y zlJlfyaELJ7yGvra=UbiXn**9_jklzGhAsS9;6dFN3kNIbdOXUOF6_S-v8l_ZB7 zKJ^1C&piM7YB15eFt3_3_HC77YWr>pL<-vuS+UwY+m5%|3;s1Dbe5&NJ<|Jfrz9(| za3d@^T6C_@d$JT2Gg}jL!wm;=s)C!!=OV47xqrvA5hH(td1jqdvh^mSvtfO<;!P`$ zeg5Ia=OQ=$vT_zv`CQxiV=oiS+-sBI^61&wS;P^bVY5a{0}o<;BHdsVR@jRHFSuZyGBC$Ts+~GSM)5vBD*EBI0wSp$;eU(g9UYZNj4VGA` zNQ@Cg!+7-_fjuwsdHtIC_5SHyB#6OY=Fn9QGqaagLoFDnPI>B9T=r${8H;Cn$H6BC zH4P^oPXni+K0tdzQ8!VbtBAW^)Uxe7g`d?Jz$lDz|Ab4r$YuMvO6mhfP~(`6`)zJX zCK9kJM@k1~9lvZ>Frp;xu&qcpH4Zfb9Bj78yH+B0R)Z#kTn!ol`E?ydQR5R5MU$cGv5Y#RKCS4q-BxGobN5XYQ-KSEatf9Xe|LBf&Xi z30u{LYWGU0nx0R_oi{)>KP%!z6?}$#ymKyEU~o45^>kCavES~AB2D@C>W`CK&h$oV zwBKs}{tlgL`GvPHX|9!@_Zx4ZdWeV##hGufYp?y(rFmKCOK!qYFmJ&_jh{8^_el7> z2=C0^-#71+I+RsvxX>2A@xkD|l>Rm`Rc;_+#TwZ=B>Iep8m7FN(+oWf-fbzaL0`ei z;z=VT_Iv1Gklgvaj9!zlThdS0Irv6dFI29^-C@fvuqwYM znNd;q$mxLcC3AV>zyfe6{^9|EdVBm9MlSjMKf=BODC)NDcafBoltvmP6s221xK?S6wyFt2?u0>ixM7m??r8{@ez3=nQIWyn;%$yy^afo&Kv;XV9u3ueaDA)m2 z5KPDF5$nlbvywbbZr4Id@RgXJpL5L15iXm1QhIp=VoCi}?yz0C=hkIyHs9N3zArM!hBq#mzAyyQi3NJ|9NQ?G`${T9s$)WkJ+<3;Z9uOj&@28; zh~sJUX?cA-F-Udjl8VbjEQ5sk(=wzZ_S}=wlCe6sfI`;LuD&H_>vdG!Nfc9SCUssQ zFEsVhhi&kZ_#Ct$vNSkUz>#I3&ZG2l$L(iTM+RnHuE;#3$m22Mrr>aKbv30=H~yy{}p2t zRz+(zEC?0Y5{)}u<0Kxzvz0}BucgeK2@z+waAxF!$!&FA5b(eOuY!WPIi5GByGXVZ-j zDg!El|8tE$7|Vf7W%^^;C@6VDsRH^f9@VJiAW#iK=vzo92z`SBoB@?C0^mH$DZWMoGDRLlD4+1o9??|K(= zci&a+N$a7Sh+aql=kjxRX&}Fi&6q^kj_LtcMqO> z_5Z2_?G4`zc$x|yYfd}4?UwH|m|yC>u?7a$z0p@Fg!|S=k~P2QAtFclGb8J_UQYklpPj#GJ9zno=a*yN)sLeR<9v}TD%v9IQKhM>>TB(A z{%g4koFi%jiEk~(OK7KeVaF~sMCj;1>js}l;F>?WeBM8-4$?ip_4j|b_u2usfm&8u z3Eustv61DMk0~|v^~-Q^V{4quGyjgldX}joevg-;UuF4wX~bGYS8s*;w>%?vAmB+A zChXXfudN zh??*B4nLh(^vOky9K~%-c80ykjyPr4om#x$(aaJ-p4k=Z!y`I^RxruG93Dp#G22!8 z6*k#2?n0JD^0b5&5t#kEWuHF%cV<# zw^hca`zDSuxtniQE%Fvv8ft1mO3F`b ztzVw0?$@iVP6-X%kJ`MP*|Rdj@87Gj>YndL?FwVrf|-n4{^apwE4=eAYOlR;HtZ3P zie{PJHmwXADTv$LL`iH*O8x?gEqNF|QZgkbCI_X@Pe?!+4oS3r6Xt~G=sGC8@-=n6 z>Ht*L_$6H4gF0g;l2S|kac<^PEiw@}Bd2*qf{j0Ui;hjs6lL~sJ@J|;Q1H-!qnZkC z`)#m5AR~jPG+|dLU<(v@OW*>{h{`ilS(VeGyDbOj)fvJ8YOES`9QRDH1$1aL;7 zqDBG7I3|CiC8DowX&qbxUg0unZqhr9ZkD|;wKtFB;h{P}9&1a`A6BD`;*E>H@sq{}O zLD~UIe{3s=OEv5^1U4#cY-z(l+yqB^10|^H0NMvO2p~ckOZTuh4V~W7NsAtqxzk=r z75i|;XC_{#Y1rA<7z7ytzy3d_Y+z&}$H4%{`p`RVpnh9wyaLf(aQex4?j)PlENF8k zSYCvJGUGd!DMMtl0D|~XF?^(#XTgc?KIC0+Yz^h8l)-OJ8}drKu&@1e%C}(P81XnG~Gw3{M_0^61jI@s8{+8uFG7mHb1-`OY}N-E}Qz z_baH~d;nE;omD(G@(M81^X@~ER23F>*YgSbp6-XPy_YER$O>+~4SmaPhnf|i?ND@x4pYE)`ITQv<=zBVfDFvPz>T_Ph3Q9ge- zw1yhrt4lbPP72h) zVo|%xc%a(f0DjM1x;x`9GdY`>>CvjoZd^(C7x6y?%n=a=AXe%eiHJy7fP@0 zwJBx$b)HY;g|5-SkJX4DTuZk#UpH#&@Py1kmCzjBJ!Phm+QhRumr5Utxj#(s?%K{% z`LF9RLn0BCKNASPIbA~TL;ZmlyLY41Jj=iGHuClw{qxD%wdL_r<=y6D&PD0*HizlQNB76 zaNR$j+@5%pBzub7-Tjm2U#Dq9uO}=#KRs0nZ9*0>!W(>A)|zH3$Hms+)l$&i^{mkR z@8vK4sdjU{r`zy@hkU+fXLkdpi^gdt29mr^FK#TR%gLlV22oaAl(r8TlE4l?T&xiK zJbz=LXazDN@Ud?|>a0P#!}ed^-_Ir=X7FgV-fP6Jwcs@lXrKaT@l>7*Y zxk}3Eym%e*L0c7{E`D{qg%Y&`@D)VN$+f5eZ9>r`y24*nG6qOVDXIgydsO;38!PoT zDk&3$cz~=g*GZ!|L_HMPZU3`N2Sii3f*2MM(Gl6=zLu!h*~Ct3yNkUJ zwW4QymzVEDz6B_B{ZaNu#Hh9S-I^tU?qv7!*(Vf9GKp}57fmip&X;7}!hN@CzFDd^ zH_H#-{zA*}TrAH(ddAsC6~CG)F6^xYei0cU1oNC~Yi&gX(kqefo|#&Eqdk^;Vr{GalPZexGeb@O* zuxb|texgPZ6XfD7FcMlEp8KC2N2pURrp;bo<>6WM6cS4LowkXhFbyu+7PsXjFqCOK zn4m_Os~+UEWG-z+q_5^^Bi^Wdjp1JIrk}v|{8b03vO|MP8n^&AKj~|mz3rze3tgpA zB@Miq5pb>2%t(4FrB&TspiIldxPJzzmX9OD+&OrptR7!qM#ISo<$CO{_amYa4>j9q zTGWPZW^aku$2AwZ6j4y{dPP4~k|8rN_57!}5Gck5wT--GP*a)P^cVEaki0#1U(i&x zn{>VA&)oeJ?dMRxca6|r<5g;&PM#c}rTGJg*6trI1>>>V8VSKr*W6`T8LVvTc`-~w zbA19AWWZ9hUgrfQ7bP~itm zgd{4K8JmaFYjupqoE;4fj`_Wo#Jou#1vQv*je(f>Cz!%CbjB_lzb^c-u6>-Mg5Wv> zse^h+$H3OK*LH2+0tP%wTiczHC!p+jjj6Gan1a>W9$=XQx{_{2V&dn9hW25_d%k&1 zP1Ilw(^1Mdtzt*eRLL+=QOF4j^n0i1G}G zn}>aVOh>3o-I9T=DZPMb`P|p1-~;{xVUL41&29YD1R(MolA$(fvOEf5!MSrpvpZ#C z^?7{Fe~z7c5ChV9Drjh3XRi1iYkCBi9-42utQ3w;5p^glD>=xE_KAnkLeAAsRo}(s z>4{hCIoPT?nfKPJR)6z@DEO0KPymwlBrP@~R-Hv+V`FD_m9Ll+l)&;=_XlEP&CyIY zQsV!9Xl8xUQaCth!12*Ela2XF zwe1*sGlSUST(!4S?&C}h{VK_hnODgEoSZi#=l2(`kbaFslW2Hh>ER_qlzS;MSNo;M zame*jO=HU_Nsk*uH<$a&k17|$R3DO?AFk+d>pi{`7DnA*NUx)t z>bR{VKuF6GJWt>B5nC_~uYTc*4;PoZ1)NdwQ&o)UA|loI!e8_aly=H9kmu(7`~tK@ z$~Qaje|XiD4Gvlkro=@I4pPd#;9Pe0)X;s`)#h6u?AL+|d4SXY$j=FblT%;(%2G|k z*jzqB4m;D!`VmB$<<8#CG_8V#g!-VUZu)~`702A?JH{B;a zzm+~ed`>7+j){qVQq=S}bY(8p-k%6Nwg;Da`%7BuT1*@)cm|o!xhDD6|Mq@XanCgp zK|c4pj_fpSOS4bEOhz^rOE(z(1$(LEsRyfm$&kmn2~@TtSpn+g^HkDThMbp(!|!FX zLu(0_5j$}Or*#LH`p(D3>AegJ2_}4)16ANu4UWQFGyZs*7-%IKffqU?Teu#};bxRx z{IFk1A-`=gl16IrvnZaSNwg_$Bz`K#+vc@!sJ(;0+0N;L;iQ!dfa8Kb@@b;g?<;8+ zIB0EF>Ah^XfSo)Ynb82cW!@d{I$+wV}2)E;$Gyuk3+iKePe$w zA?q)xt)n{2&a2!EVO`al_rhKjwM~2kzx^uy+3MYCZM~b_uxF8%a^B~OUX#^K#8nSQZaaDk z1vjsK)G6ugH&xZ*Q`7&@WLUa++z92Kb(!gPTgNvA{b+oxqL%3Fw(u#$TBLIqZP} z@*Uu|cAi<3149R&@70HNQP+NldGL)V9wPicAREL7gaJ4<`wl}<+oD55Lvb{9+JGAW zKgBHQ!G(%R)f+U_CZl^V#@YDzh!+uO_24RU?kqx~^TL`sIv+u*Hoc~%W>ICO!=$8$ z2oqN(Zyn5_hPxg$3 zkb5hzc;pKa0>&k5#|sH~&fM9QHQe1ro;Nezdb(9SbyY|zd^-ISA`J@N>VnqCVw;|e zvFoHqdsvX2+w0dm-k0K#H?m5UUn(SsmrC~<9sIrTA05q*91f)=1x*?o1V%(`62j|i zra;^^W>H1miKi{y<*q|CY_3t?!E^dMf6MuTU1+%WqK<|Ek{>byNfa_PK zZ!eaK3Rixyvjd;aw>ZjmSUZ{o>T60$^y8#%4nnZ2?B|7LWlx48hrs?3qTI4U9Ee_v zskoe)+U86DwM*B94aF@xB;1Dmq0@S}4F{0H0@k@i#X{FQ0zZtv3^-2Fuhe9Y_920s6Cdo!P}*N;v);m%Ea$oA6v8m_rp?3a z;09s`>uH$4;`M{%a4m@jeQzV>{Tb|Nt?qoMZdMLfUa!k2s=ubaUk}ZQmS>m`8%i@E zO*Vlp1Qlj2gcC{X(Zsd|7j!MPD`x?-lavUgxlur|Tu?vEL82a?lhn+6?7eJDkC>X> z)i~p-J?9+=c=NA_RhHqZE%`;HriR|MmM>vI0=d+b>x z!?>LYtaJeMXMXgE2zX!p;Nh}N5%dz&U=0GLl%F^mE*k{S8)6_`JrW4V!aiYRPubj$mMDG4JGmEJ8~NqL7&d1u4v`r~J{SFrKvo@BW`Zf0my5f*=a8Abism`O^^? za%^X6?si0CyQ2+eh}{pT;K5bZpt~2|oh)EO0cbTaxDpz+`AolWo86TI(Fe_Lacrs` zAYbo6!jZ?6BkMn%pMmM;d7cXgX?|I9s&tQ2t*40vmS5@04@Bx!_;tqgjW~qPZ`;U- zN8aeXb2dF6(YEAh#cDMsN*kKkm6T;729-Q#bn7eVTgS$9ENcUeCdnRxMc~3tOjU5@ zxY*b#fDwKS2|0Jtv160vbjI0TOAe_L(A+&=*YH z3KI`ra&;zZF>~`S9HnDr&JS;_q}_Os6jLrr5b8-0_R zYM7`bqT!E$aSaE#nA;pd!S~~VF=i&Ri$ed@8HVLrZ3CMg2Bb&7!SDVrKi7xcrSq(^ zTKGlvKdDQ#qPuLgMZ0X=uxlWDz#DvI4UX(t=@#9YauRws=z*B$XAu8DB+zzI%vMl|}~&S&*{HmeWq>fTNxP zsYNj}`P9YrC%PzxMd3@uoMEK9c3v@&n}dq?Mo7$QwKcqa7a0@c zN%J%ZACJ4(>6N`-t12Bx`SB3HlUkxGIZEA6&>DpeQS))ebc-7``2AY)$}>i<=)zg% zP-?C|Y zsKw0(vo+Ig(CmB-4XAlc)$@6PPw;t^qe!nk6E0nJZ!oZ`qlUl7X+yVlh9RA3VX{GB zyC0r)t>}`)(EP~!?q?@hkLj;i)+wmJf4dzMGcz$2)$~xGPqEHJZlfkLFjQ{U{04{@ zV~nFOUp&|{y#$=JA4e=@Ei712{>se{O~$95Ak+oC>Pr=6HMJhFXv}Nep$a6_!+`uS zs26saBI&ZO!fjB;y#yP*FIX&dn48#2E59EVQ6qiWd2UUUY~1 zX-?nLiAjnjWmPk#?P!`~<8)MlC{KsQTwPAinr$o37~?%+0?Wb@RY!7)tI7+3Vx2F< z%yQC&qt6OQO}j#gH8{w8DNPSBn@qA&Qo=x_DXFvPex)Luuv{eqTqmW7x%Z4rNAE3? z@%2-L$4u}2_ml}f;`$2-lAivp7zJ7CG|)yzMt3BVFe|CHY>5!ix)$(o{4PICd|yTr zvKo=hJ3H9mLaXMd6?zop1-+$#^9a|&n(D=ofp;lIM1DFn&mvtZco`lB_p)X;_D67q z48$}(7%x;y)&0y_t+aD8YjALL7+u&hJZ{ao;AKM|-yi55c9rHp(Z1~HJuocUOi8V} zahE~oAr)dDNnyU+L4KOTE|O;*-2LsQQQX?0&ExjtEsZl{yV_*Ar7#S724D&tX9v53VArytn*j_CV! zH@WE@{%;?rA%}F%ls1^vNZ7-{W{03fw@Edfy0mvs!LBP%*o6@zZyh{W%9&$+_~b7l z9BvnMR_(chYP-Q2)lYMFFTfNu>LBLNM8f~4hr`s`^7eOpdU@c$*5@&GrnaLS(J6-+ z$)?6R(o-+=vo?AQJt>2qiWTS= zPeq%=>ZH7Jd%Q~P#ch1CI?Tp1CWhy=FzSEs6z^>_3Ak@bkSy}!R(Kke zk>L5=pV~-IGKSmbkM5qoV0nk-P+obxh!XY0=`4~_X5ap``!v~UlnkkFLO%9K1 zdm{SeZ9m!S`Z{=7mCGZB{F$4J@V|jE=(Tw8fk2bRJiB0! zS!BD|Hvc~Lc}_dX#%7H9;7i*iwR!5*0&+wg8-0dNR5C=#fxw#fU{(f%Gsvr|60`x3 zO%0wd?nwfWaM0r21o%sk83sLg03C^HC4V=F9y$DyHE{7%+bj)#;>dl#bY2}+n5Y8$>RtgHl~x04HhEcwAor)M%UGKH>S z9Hrm}d07^-vadC9<0(DMC5dCCUXOgy2$BBRk#BXfz4cn`->MD4YkdG}MiKNLQgac$ zeBB_P55Bv*>=#Lh=(vtl?FM)tou}VyjUf641HbaV>R= zyXf!F!?Yr>d-X_M3oW|TuKY2q!pYA(R-rrDuUBv9%EsQoSYIJjwtY;%y!t3)G&@ok zv(-OE>kF&8O4kkc_Xi2?8Gj0@=R#9+-#K`0r&fRMzhMtdX|$*PPrYcp7>Nm08Ov^* z!G|!fgwVAMo$<%fWM>iqgZ+kEx&Ix3sNzq+l z(Um^)>a6SbgZM?|Qd>W?9o-CwA?UgbdeX<0JQUV17f=PsR<><_D{FFlM-TTgRs>-r+eK=zYH7DXzYVq+k z?Wd=D_z+PeBqI7w^;_kb^UIxS#bi-|pAKp1>B15ezL);VQJy}eGu^tS6jLwvBT5~Y#8mwXN&?vOlf`N;fpY0_h`b9 z zfvEc?e(g^?P&IF9Q(&Nha7lS9D^ApUSP(T6S*q`3CkP?gFF+SE;2H zU_yZ?VZpp)2Ux98{cb1~4a%_OD0~1i)3d>2zzb9#2Hz86uyef%?7sGk21vp{@NNN{ z;Zxz8S8UWhJN9U^Cv1jRZf{&uDXfcy#~6uEl}vx26B1QAvM_CbQFs_4Y(*x%RzA4| zZu@`Q5Jnyz0#JC-{rs!gKZw$Sl+zj2Bn6fRu-EY?S{p}$bXb_pWzh#VhHd+)tN3qfd{V#MYwkUT zA2ko`o-GZgSUX%!7|3JIxpjYOhr5A|kp|y?ea7P2aeH;YwwYZDO6Y&lsR#*BV(kup z#ETAT@J@lwnI1-t>{{Y7#u0pOI z)18^l%DwWCb4O#(g^5B4t|qjEDa{wTx1C_ zm`^S`FFQ9i&m)C=<)d+AEON1MBL~HXtz1keO{G zrK{1{PUa<%$^qxT$KHpeTL!;+2*ApBzlrV6BMp5z0lspZ<@agZMs>j|1(eY9w!wfI z;7_{O-B`YFhw6Zi-nd1ZZ-u48QP?<9JIr1=*5&-s2IE7K1B+IpM!#>=S%~$bdhVDn z)raSJIb&{!l(PnjqK2_VJq};DC7AkE`f4WTc8IaV2`4-D*6rQIl815X5v973xDC6h z0_H8laCmuxTj%@6%IiIzq=DdBLHq9Yq?rj%QU|&)hg;W_VLb^l*{;&oVKM6ZFpdUr z=FwJs|0s%2{SD)pG`MxZqR1~&$jf05pa_h~$|5QpowlMW&d-4&3g@e>Gkf>Hbu zAD@!~Th0z1TOl^PO}Pimnbf#-EFw=pw?^2j(M6qceC1%tGLpQ~(7H#8R|sZNPR_=T zER-b^4|=c!6658-8Z`yjrsee;um(jB>L+^I*)?tz%uXYGhif32ghv4Vp1%_2DC* zWSGc1ceh2C4Nu4A_K!b~1fCNC>I3jT%H^2moojj--#+P6r>2>jv`j5&XG%&ivN&Os z0B|+DdyVJ)`(CU$L3Z|v*#R5t@5^!OxYhnp`pe7HqLL?_mLJ;f-QA}v!0h$$P2#^8hE^Kfcf;zK8A6c@9qPd&;s04c_NhR~=0zgLXngbGDu1 z<_W?V@90Z#(*Q?zxCXzk;n|2rgnY}%^1U583!vgCw^3@Yh`z;FUt}ECI>#)7s~5fy z`McdCSd-)SHPZ^Y9V4&5h$rol#iEs2NmhtYhYL4uN4}en3k2f6AN+Ua!|kAhfIxQG*o`lLs|_|w;l=4jJy)@&oKqtzso-Ckl23}M zFT{|>u6#9*IK_GnT7?^UXM=OYoEQ)5Q>=5uc~HOtgG>gdSe6(82H~X|N3`#d$pYA`8xVb=zfuHpNFP~)F>Vr)!7vDbz0dN z6tYA_;(Me3icaFoc-H#6Gt_-x`rh4JV`@kGSsYz~-1|r8t5y$6 z>zXu-mOBD*pZ@Kz0FA;M!~FES2cbo;xGC_!3a^`2JLZJ)z1Jb6Y$^dt1sx!N@{d)9 zG;mOyHrUC^VgTkkKxom2lwQ4e!yPcj_W)D@d;7rxQ0NUGwn#8?TnQe<{rU>x6vy8F zX&)UM!vMJKmQUQTk6`~A9ta!g0&B+}P`J18ID+)ObTRke@Ab{hn813bi8r5S%>g*< z0WxMxe_Ya64)*o6gW#toA0zZ zl0CM0Vgh)lH@b3WV;3WT&S0l6auQ==&^|p!Yi>Pl>H}fcbD(np5Co2sh26KVhfZ}1 z>rf`*k;ek8x>Vcu{tcT54ZZXCS}f^bYs_h^{==U6yw0zSmc0wOPj~FUx#l9?Kdoc! z&~at{)vU)Iq{S87IKA*zVu0Xy*tYTWQ-cauDFp~&96IO_gE?>%(=xqz?+;5P!Rr^0 zH^XEga+2Jr!yi=^QKCkpSkByMs#5~nk&L)Is&_lyOE1TYx z12JFnp1vHKD^!f6TyeZg<@k`ySCS;x>8+_cx%uk%$SIswzVJEP}7iC*B$ z7eCY9`%@bE>d7ZEB7_y6vA4Al^*|&90;qD-GfVF04<)B4DgcToZ{OrdP5V=A^w8hG zf<4AkMv?wmSJ445Xv+9zKpjmMw4&_tmkR&OR1J@sJCOgWW(T06iIy?Y7xu)mvzZ<5 zM|l^zzR&oOHv|>?>@r*FJ#g2Y`Ec`iAm>jFlHPswSc*1eXe`yzZhf9gA&qP?PuMHB zU?Sobw3+6!ez^NghrCPGW_8%JE$EdssWGxpqAG2Mdh5WmyrY?kGz0dwb*wN-_M$px`e3nw1p|lG(<_$1lqtk%T>dH%JL)1CKa9t#iN_2pc%a zJjNr3;9$t9&Un>d!8hyLy**gJhg`FrF3osCV5SHTM;o;aYgGk@Q&@;L-Xog8q z?thPccta*%gOQmF+na09u>n(9{CP9fp6G#e0QfrnNh&4?)5xShV~{L9XtMiQa|w|T zS(u6n%q-uUNadRq<>zL%3LaH{`^J{-W#W5E4&;c{_Vc12KYglATLj|>@LpZ1#0{*W z+Lrvkf9!?$0SF2t%|YYVrzw(vLN7Cc(vyU-$i`f<O0ACF;Su) zu*eM^0x%B{d`0v?N{RspdxEBI&x%c6zYYV6K+GTShkHPz;De;|oeN0r?Rial8%!dxYdH^Erf$y$G3d(+aE$LSZ7W|3!5Jo{h!h z^y$CN8U#3CcoX4MiolOEeV=mo1s0n}E@I8Op%tu9*kk<_hoP}(?fgOPThtVHMK)`s zt!+(iQPH%6MDR}`x4i1|vXQFX+4+8Br=4 z2hf;Z7J2FTr+%eT0$&VT-9ja0gM;HN1}sQuH&e^aQTbP;*;wyWH>xj$3(+H^r0CXS zUS3RuQ0*n2SBCao7Y16s+h8_Vd~DIbcf0zmBDTXC=l6LTR3YDN?(&C7QOjJw7i9uS z0Q0_aS5V0J9DKNME{SSB36VrI#&E?oK4Cas5M;-Dix%i(>|dStoEw*H0<;?5T z6<;0dNh68ubXwi#**ba3NrLucBD%Na{WtU3EcL5FZF$bw@p`J@bEvz%k+hEERct@2 z$0{<>Y;j=2X><3Euw4eb7nOtr@%hb$PkegiX-Y+uEz;|D`ly!_#jn)U3O%7+^mD@G zM>|WXugCZgIRyy+vKvH=eeWOb9xps&U+4Rv7%f;G?`7Sqp|lQ9JDkx5Aa0#^VL2!@ zgZd15T8%VNF31CT&-sgmAtxRUp{QfXlC$TPg^0x4CQheP{rTh5gCXD3w~MWe+6qfG z^>zjn8x!rOuFoW8<5C(NE(R6zRURshnQ-%>fj44 z?v8a_YU&pN3s}5;`z&wd#Ob3<^epod(26y>>au-i0l*LQl5H!&d$=8CPK=p#IxZEl zvqBdwJdEQpco)$K*>?fqh!gN4eC=NT;MSUzk)9sWOYe;j76Ax?DZr~HqN=J2NF+x9 z_n>KH6bW3-wP`}mt9=32hd6ie35lq8?fGZsENhxgKX|=y_Il$6{`Ootq{kz`?it2} ztQtV{Y+F4rCB!x#QK0D;Q&Oi;0)Wl*312Qbi7XdusIJ02-uo_6%%Q-SaQo^?Us3Tc z0&xz7RW$E)fDPcSqei^1#RvBD#HU$lVFi5s+_5@QuAcD=C&tI{SF#gjLbpvIfkY;Bk?-?*~C+d~`nBPPbI$R3HJIxNu$8L0$Im#V?Kcks`GVyB1kX_5aKJ)UZt zhfIow|7*j5RUhbmj`eTFfI6{bm86V;F6u^3>`XV=)cxr+=_%2tZ#Z zL-b90PoIt_S1H_&M=-Al^ckmqp5u7|3HWR1yu7Vk8!Dko_Uz(?O$+tT>!BHmE1&r2~bN!s63KmvZyEZ&zF z>aCTYZ!pGcs?UTI>6h<&O_^<7dR_c;`1cu0l4zw+7D_hbLK^&4dhw&VCmdsd zx{!8)dC7cRQKtyI(k}iL1{2U7RMiO)C8O@zaT5b!#6h4H072u=?%nK*^mKQl-tN=W z(*Ste4&eyNqzIwefZGXAzA^vqy{*$Y{mJ%;^dk=n#?Pph1fFTtJ4-c+rxm%`JeFtO_6_u5p zq*RR7O3A^GaEfZ1G~T=9{aTj`o{#)i$q>91|4kl(uSSHgrXF(4%Je{q@!82_G)Do( zq0-$I9W4dyKbIvxbHPF+gHs*~Jd{gvFinU^IgvY1kGls5*=A0kqWFr{Rg9-&_BK{l zKS4?jMnfHD9+5g5z4N1#ZN6hIV? zpPZK6!J`Ov0?yTCBi=uz%$|g|M*psxl%3%SWrkggh%)8=ET7qa9z67klC_eKQsD{H zs)q6larPIoAJ8LpJ~|QKy4Z35+c8e8`S{?z*uSw8{7Ya3;c6EL-xHP38E@IhsBYN# zShjgobqs z(fi%PJ?HK3ljfaBx{FE_Rn|Wc)SZWWc*VN+n;jw-?5_Se=1vYVHe;&C6&0awwP zWY%LwEcxh#=+MyLaTIr?U4BR`3Ey$O_4;vsdEbQ0^n6b%>yn31NL*+&xr*Ay+~YDV z5N&r{@)B92e+e?hI%a+E>fj#?I^5hF{dG0Q8JlTF|?Q0*f zPb6rIVxQ#t_7lAQymThTOD#A<}0K)F&)>Rt$O9aa}-1ivJ{G~5`_!swMFQ%lNm>n-Ae72{}$bX=_0=nil-J|jT z&=b)+D|MJiDks0Pj5OMIZ(M&zV|=}5KSojSQK$-iO6Pv_!L7a89Yuw_%9`6co=KDt zz{G=&w_>yjMQKCcSmWOuFOKeikqQfILoPNoA=E_|5*Err94fqBOnj%^9RSAyR3CWN zvJs>VAlaL2V`7N`S%VGaGtODAa1BS;`2%sB*^jbg5}@$}Gym5TI(>7zQ?P$C5^R5C z1R^Tn(+?b2JDOz!-6arFp_oMa(54k^t;(dA}ZaR6^eD3#Gl|Mykjrn;nguPDynPHaB%G;jV~aywMQ$e zZ`5=+mWXOcRY6diD~~L8 z0P}BOjK{6qPKX=b=7s$xSmcSZ#y*gW}Z>niMzQt(2_s*8WNti1&8MmRr9BgR^u*EF{&a z!YQmzI(1qlE8F=qpOTeDyiWDs?qu0u9RF&WfBOu&Zn9y}X z0p}!{0JEKzkM!-U`vT{}g8th|25*Y&t}~-k66Q=>8)Aq#L|Cw6y z#}}MuRz3R20&j2L?=KFo3ytcA)F(ED3qGR1i%6v$ z=1k;&;D7Cjh@arDqKQ9!qg`;f(&s(9zP;XiZa*KUL`lBlS>G2dmJhU*mTPJ&w5=-y zu;=Q;T$|0uI2QLEJcW?&eeLENII@l|k81%rRE<{K>X=h8fSSwqnmjGZ0Wk(!d@sX7 zv3wz)ga%>Vus*yxcY}>Uc|x?r97*}zLwa|C>UW(-p3UkxCBeu63*;iVD^$K${yEQE zCX?;-2RIx&JTHI(9OQ@Mqz$dtM}nj2<6Tm~LZN4<2|b9`#K!_}!LiM*YHBVeRTXD- z33dpFty6YYcOZW`z6*(t0$$u?p zOC)h}546l5L7l)Ht2g8gdA^Kn_>c`eC-gUGaZ`Rb-07x#8myZKNqCW#{vUljjv?UG z*0gbj-O-P}3`grKVNuXhjwwlaR+zwOxQ9ea8VUnBq@dtc+@wSu|{9 zdOwJ}l!~5y1vKXiUhs0aN*+c=G$P51HyY5apNVN{;lSkAx$AbVk_5gSc<{caq)hKh z7HDx&@EV2!ah2_S6FIoyB#(F9!~hZb8GwUt0D2c1O|)}^cUsBTOv7Lzl>Kx4%p@f= zD7SB^%it4_o|Gu9Ca%4dhGWCLJy@A0Q#t9)KoO3#2^VEjA6S({Bi1A$B_bLDenveVWD<|uv3(ji^aySA0I%<8)_r*d7wU@jy+>(rxETabk0H|RuOzMV zX3xP{a64x#p&MAEh#h;XLe%zV)ETX>;=*L%TCIgNt(8xvP^g`cwAA&=|m(#{Lk11OUxB zy1nS}8iHfxRBQ=1mA+D7$6bjXv}vu9!K?Y2IOiogeD`Uh?>nIz^$21|mb z(`d|=8_f0uOzDyLL`81bw>8V$`$!l!0fKhs4o9cIDs&4UEr4kR@th82N1^vd=yG(f zN*J#X+0RvwUGZs~$)pf>5X@$VZAthugcWT>s1emAr8=H(K4!aOW&n5Xw{KV=*iRZL ziSe<57X_cB;Db0qa3uds<}-f~^H}7AIsofCD6pf(^)&6AK8ey&n<@Rr=>hIEKCx$? zoQRz(&_j!~!BDRKP332iW?mIYDT+_Tax>1WrzOOW?{6pZ0a-n^a}C76HY(@m*S2`2 zUtU+g{BzS$26^gHA|yJp1Nkb533=mm)^u&!iJm>NEmuAsc}(C2kB{S>)I6*JqR58!zUo7UmS!MX%u6vS9^kX1C*E*kY2jG z{^G?8Q?PC?H2U1%IG+nl*B&5V+!1IFcNSV{1k;Az`@Mcmg^DpjHSl;8+yOfMv;G8R ztXSBBNtC88$JS)Ih>15YupzFY1{WPlIjqB$y95k`A?P5F<)0%LM4pF?PTvxE0exhA zJlQ{PHjb6pL*wdNA|^H_>MMO(`Z4`^1r3+1_YcCpDu1PCls8pgG=j^^^6LTOo>(#> z+TdRy_mQP86ZpUB3rDiEW4}1rlu(eG$4_b7KmU}X=*!vgWIZzA;d3+_Wt*d}V25?{ zzx|q^%gw0c8++{tTn}icS*`p8%?LLv`XmT=L(lO{qLefXmz>m`iP3+UszVg1YV=di zJqIQ$b1J*KimGg1Q=jcvshV9Ou9FT2OnyJJw)UvJqLY%OmXFrG15ryaIx;Z1htYER z?wPc2rI8qQ2w?&`L`eyM>PI2QKvMM1_n{n_H6Q`v3lR1h5KKcJZXK;*?aaJfsW*1d z5Ia{$Iz}c=;=~QWc=rFXkhIj_R2uBd={V~p`Q{HRem3O!1cR?`dwPF~Y2_bsu)Xh= zbP?Y>s9^o-{ag!KXaSRlVY$Qhui*-_lasSk`&Bnj@ROGMB}si<+vhptj848sTGJ+B zPlVex<~=ZD$;w0`u)F;NaR0z-Hby3Z4souhFdeC9fBq1Dg>_RX$%C(7CiE0suhrY| z1?TnWbW9{Oy@U|hdcHIjTgcF6zypZWL~1gYeOQopD7^K(~i;c&YW{^jjMrTk&F zz3-|{_5{}9241yu6)IVo0B){ChwvKW`2B@BSk~HwF*g5bt#;d`6i42OK-v#sFNNJJ zDB{@NGj{5MNEw?b(R%=-X*Mlh6YBWo(dpI~j$YdMJs~v0H`1&pj4^?%VhAkBlOlWRo3|P4)`O&dSOr z+1axwqpVUPWF=&8*&~}IGqSUHh|K@>?tZ?{_xRm^Ux(wl?^_)Y-mmxT8s~MM=d~AX zK&bxJrY($;gK~8p_{*|}uUnq>K8py|P~_&2ewu58$&@y+p+?mAm6U=aOo7%5qb-WA ztCSY7;|QQn{K zJx?)V*F{1wQ7;7iFPu_7MCefG#EQasdJd6}DXOXlYmK`13{A7&=5B_#;+|hF9R0QV zfob+ex2k$-Va93x)hi5?S@TR73wrctRRlyD0n10Kf6l7y<5{>dQb*^-bkXJ0XM>Z4fD>Yv{dH*A3tk55gp`!u zitDt*9K7oucPAY@rVS=Xv7=ZwF>Wz4Un2}V`RT{0za!mX@5qGRRB*up=jyy2`Np+{ z4UU^je@3~+43Cr$J?~$MDv@}a-?{Ib!UC<6FPE>~8}XuPvHmzVSl#!+<9b^Ddq>HQ z$Hg97@yp769qd0HuKjv5D=$xPs;u=D)|@Y6@-L1}Bi=}m-t@{xXKk=)v;+ONWt9C`~ zO6i^h^-2WN>;F3PcJx}S{}QOhvXqADv=ek&Vujs4oH**#4|C=lLzRsHw{R|3&7Yv! zulKUHRlD?G2K$`)pnaAm&sA0ttoc1|B$0gjY0uLA&kdY`fig(sWi6G*-|rYX5P-QT<~J^HxLi=(Z1S)|t(VoHq$SkmgsE1GcAO9JXn`8VT~SrUbv&lS}Eri5J}H>ic3 zP~dH7R1V6>7Vy}Bx$aSKB~W#FoM+_P=YIJ=^IU{0gc%`N<5Gk@w#&jl^2hJ}NxzAQ zdG*8-T`s|Cv-Dbpuuow&o`hj?aeUVJ$c{Dot5-V$f{dL+ssqICfMgyPNb8l_XvFR| zHI35UTMuoVmUl?HL}&D%@pMjLs>F=(i_l1)#YTcT>p&4p^?6W7?B|Yzo_0NGP`G)m zJ+Gwct6JN28CaFel68DHPY0$-U4X}%sUPA!X`XHXZ`KgN*OQk3kNg`u2JyRRW#A?w zGJWvx0u})5heL*WQW-e@GG@T`0nBf@@jRWJme%361Xz;;`@RAxqx3l6q+Cxvdgz6* z%Eo8v_>|&@Hw%t9T3(xw$E*0M<<>G%{!%E<;0J-#$h4rfXx#3VjFLQFQUipXz!^5D zYU$`;`QkEha^eE2s#j)vsqS!mV`lPXZPE!CTjnr7`MTs9t>jnmy=s&=@u4T+0)u5_ z3R$u#_)HHh6#F}6fKjNSm~r1y@tYGT(^(jQS|jU)3bHdL_{;YoFhY5oTlrEk!yI9l$SISq7Wr1dsRtGXMFFEw#jIQ~pB0k~r^CwZz0+NK z{R8g1U-Bv^7bx>f!sG-`P2-ic*!MPjUORQ37+_3rK($%%l#_d@^U%k4dR zTV86SF>zUq?{7Iz{MrZzI0uc6tDbIwkV(t4D0peLrToq(h$(g%?kvjLFc?Vg(N;XF z2&kdUSa=|s>occ#1BncdcbU$;krdD9uS3)bitqCzJqR(T){?0kcKfwyhQznFELV?K z;)ncYH~lA82dtk0cu7;a|C*LdGECb@RVcD1fXGpzgrErz8~m{NYWoH0PM5g_kCzE| zF54cneCHJ!EBxcpR=APzaN<5F;ela1a6cj5BT81;*1&AC_WHY)7ki?rF8VIzSdPaw zl41_21{=p9c1`_hs1m!Sdr5F+kkw=6V99o)RJ&JMLCa;hL*=qR2Nh5a&`=JU=N)|m zuH)AmOzj1jIuuV2%*!0@o?NTk&ybiJCJw!QzAL&S^L-aN#3tXJE?FRF)%`7Z+V2V$ zch#x+>4fG5h)rL6koL;+XoG`LQlz-TbnixNa|Apnf)=6NeiqK5>AK15JN~CW&9;TJ zyP58(<(2j{ygtpIxyP}0YuBkzrG4z2p2A}W`Jlh|IPsF~Rw>6f_u9KxCpR}2EJ2g0 zQ$RcC=fE%GG_UMLLKk5Ed(~p&Ackk9-;Q51$($f{lF^|&f=S4i zWXUMlQ*ypn;*b+uQ^TNi;u-^QWOG4maui8#<|I%rkv;rtE z5f}s7C=Y#wJRJ{VBSH)<14AGiH3DaGQ%0J^JvPC4d{ORE^~ONkk8@7rQ=ive_>9?X zn&8i!Dmw#hpFdE{udsa}D&g^s%DtKfyr!FAr_Hp>?(r5~%I@J27)Yn*=5||qdwL4f z`tO@Uxv_JbdK}iHg$&;dpuq}iw64-(Z!z*1e+pxfOJ>fK6M+x9<9IxmvopCt$%P;5 zc5>u(wB{Jl3~rXeKRrxRia&|(XSF}Ni09`akre7ZFB3#VcZtH};r4!~6Bv;|3FicB z{16j>eb^io2pA7O(r`LBaDj~*`Z@lV=Q8H1f$WG{?CC(FTx8IhWfQ;fEm<1WS;dxy+SLp* zPP){VSLDq2I(OAaW-h=Um44nnjLKlE3q#tL--Jm=%ix;&bw*NftZZZaaoVY)}$>n zurt+$O-}vAb9FA$dNJiRpQNR0@1N{tPW1*z;Gx*K9HiZUOG1Vt;kHx~lVeE9%A!aR zx+iODN`jgas*ex3cQm6YwRLi|P*P++=9VZr{1(?`Z=Kk}q9ziLikI8jj9QL?Ub3E+ z&esVO4^Ib}8;GUJX2K-(-q;>kxO4vS+0a2f8eDFAc8Sh>@S}!av0MoFrNS>Q?4C<( z08+)C`9Ll9*6aQ}RIn(p?~xe+*(WC9Fss+5E_O%9GDoxLeWqdJ55X#0T(JTbQD}Jgtt=-CjB8HFO zh+O2N{hI#E1s!FwJj#RQt{(0AM>CSYPHs~BooZGu=>$0vH^nB8U^}0mTtcce*Bj0> zzm0r9V1X<@Aa#Tcs>0+Z z-n~>ktnJGPvfQ6JO_K6Vq=bjTz$1>ctcWSbMS$z3Lz?t45Kt!!wXu{6;C*q!(8~Yt zO^$`I}8;L^}MIQ!aZbd>lPU;(Y-rvj6XlgY<>JjrfB zZEKANlB1J(`(V=|QXKoiEk}jhaKtF^0jqCYaOZ72&)@64RfE0tzRn&Y{Zam&6x#O+ zSnH7AG@#}<=FwrKqWUcO04%$nK=_{dHSMBhbRPp{C~h1kS#XF8(jGZ8Z&G-D5A^xC zR=I*f)%Zg1Nz3HB(Bpzc0^#8GIPNQCvuz6CR50Y}zQ*3z9iJ3cCNA(mhq$=d49Mal zClZIN6iDM8PECQ9p*fT1akyr_c~8hM^7L11i?GZ4CaPJOOhV9<#YrOe&tJNHTBHT$Ss$%VqhcYu|IEl=WIpzmfURNAD@?Ksp1X_W(@s|il#Ty4C~p{L0U_xg3u{*0 zacDr^LmZ}PquQ(|dd8%71ut_6j%s|hPnbc`6dBQgbKRt>BY-V zWA+uhOQ$iYnEs%sXZq?7+ztzr<*uX(-PoEKB(ozYLCeeCe&A%Yi4f{?bDtO&>`B0? zg419jP$EH_E)PR~zzL$?zQu`<@1L#A-=dUbNK0Yk73OBvxwOo^$a3G5m+50hR{7=+ zG9o<5Teerfm5Mnkota>Q`lsqi&YzWF@O+BcuMS1vTg#Z>JbQ&*gcMMl)4c>~`Jbnf z)(b1ADyM^w;L@%iubm4|dW{QW)9)F@&n@aj@^A?zVLSlAg-XvNfj(RFbi9C5pSEMk!&L_5x1=GHNAs1FbwYOJQILZ+hwI;AqAr{(gydE3 zc-=u)H#K@HBlZpBwQFK2m1rqdqpNq`@n%?K&&|zzk(3-40DmgM6Nf7jC%w&u35kRl zDVvIzI8j=zdiS8vhJp8O+T@=V8&4-YAAjlyhRHh0#!vEMlupW~&0!r#KoN~`M1Afg zwbSYlnJDn({8t2Y<3$tKix2Phj;N>_itB{sw6Px#z0P@Gx$AR>XX6Chf5ksNqc!l& zTTP(s>>3F1-U>pZuDSIewelMvUi$m<>_-*Um>X!Dl6q?yklP`ESfxBvlZA0Bw&d=VF^=Z8h1((YF7qxnXovcaJ-`_JGl$e`t$e z%+wcC=jm^ayoMrrL-%Q3T!?cX7S8<>8y|i3==7(;<^>7MoJO>qRG+YKtMgx(bV$V3 zV&sH$J&ycVG)~ulAD{n~RR1C~NWy{Xj^ftHcGp&?hkNR8Z@x!EMRT9qbVmm-h25+K z_m^mXE}X-A_v^o4zX_pz`zZN%l^Zn$lOJ%u?OuwRt2zE{o2$3Qv%H*(LRF7PaOhI{ zEi1pHp3FDWi=#eI;PAQgbuztBg#gb(*`Q0;>qh?gyiB5b4t7tBB2_C-v;kAF{);K> ztUjz>?dXVBlp8sgS#g&Ih!Jp5Rk4%3OJ9F;)1nZFbn{^Ecf$`0P>fO2Ca>K^DRzdg zJD&AL{<{M-zIYc=>ud#$?)q+*P zR0nJFXIK)3;EpM~E3P6w;|e-DWbl6nPR2ABWX84%M@IDG5@`+Xiqlj2f+=&Vhr80g zK5+JQDs6oRyauL^@8byb8lJFWmKN9Da2Oj_W+BPdd63M7c=uheKjH;`LvTk7t1Qb! z2Nxq4XQ(6yreek7cpw~e`tfi9!ecv0Uab-MfKs>FPIX6w*-8@Ao!M{M2z$h9>^91Xtx;| zaGvE{!&eSl`y)U&AOwxZJX6-@UFW_03?K&MSg~e#u3?-N$B6LsYkWx0`V%~gtCSS$ zbX1Sc(n;>=yw&KkcrqXhu0C5q1nW}kLkvIHuF6BQ7=~JQecPOiJ$Q8Jzy2vBPM8P3 z{$v-7%Hl?QVio1|YV<-RPrZm(RKr(vCJ~8ta?YUU(M~2Gj2am~F;xm__X1my=x))- zpC9)Pk2DD~^Nvq%gY0{?n~C#q-C~i&{p3&W_hq`ChW4x2BPX$7&+{M|SC%HbVb!PA zyn`f${sn&c&-YM&LRUY#yoIB6Nw8^l`DfQ=Sty18O95rXcPd_@_(e&I`J?kCM9`U( zk-VmpH8QNM}X3hhXrRbbB52d_{qSMjH`_2gH-oY0H!ELi zm?qYb>nIDZfyN&e3fiWO7;0@BQS360SD8gPkH1*RF}X_KvVjUzt=5a`*Y8(TU%@%7 zQg3pp5(Cx8*aT*c6fwGyt?(O+8yo9tmdtWn$rCGGA84}N@T`_&zihsmYr*Ml$MB5> z^Moe#YrlNXW^Ki=N$+!O=U*`#POgP@D)7@T0|y)aqM@YGVCEJ$QhndwEr@Db+L=1~ zBVt%J@Tj|ciuJ-fUS>2*%&3{N;a`P59@x+&$$tXRm0-$xkj2js@irO~1d$i9Knqju;yN-)Wnb+q7 z^%ob!IRki*>l(=&v*w%KZ7{){mqF4E=-%`qB30XxCq(LVMh<-sXqE_{$JacyNdbqG z)q#XUe^+4=4m>j$=|cQ~ZBXFf5Yo38Kqm!d&e8rBQsyl7rN?*I5sVnpQs)*U>}v5z zV*2F(T?1;w$pO?AWK7wq)bwqdp*B;|9To^&2#V&-KMTptCfqw!YMCX+TYK-w#l86v zvpR@Jjm)8hJPyQ4=6ufF^)FUtujB%w(Cz8P^X%O~`MAQN^w{}@3ma3AHI0se!PxaK zXJP7*IzLhOjfn`K^-t$vCVNq-9tyN%|AVVI7ccsK;CQWsjBrDrKfl=N5|~|HPDbau zi3XyP(VheeKM}Bor}%Zu1VlZs{WF2z3YgI6!3gFj-{hM6Y{O%8_@&#xqbj9;NrrUk z>m5t?ne-Jbz2n4s#yTOKEreHYzpHYi|z*>Rig(v!X z!K#_t65vdAXqU9^bdgN})B0X;s7)n@fXKAdMd^C07v7GBPpEltuqh7~y@?sp9<#|` zZLAFaUB69#7Cg#8$tf53b@1AoGmT)dSfvS_vO69JXsTGeRy2No(V_N&Ci{O#T0tv@@qqFouC--F}mV{}WltM?xdF3Y$SlK~UM z^xOOWl+U(;qJ@shw9L#{BWrobYKi0_iHQ@;TCO7FNI?gyc2l!EBHI-)U%upz6BWV? z3wuku`^onY%j&oR4r+fmAn7sKC6oQmO196C|0yu)GcZI4MoKe3D9LTRxrN43zqS*; zNI@1lN zsBD?QfU$DgQ5)^hbcVEq@elnBtRMEqo($AeQc=ACgbg+j4-5|<=3OUyj#zYBSX(nP zGNQZOsJM0*NPnUILFsIl5+A3sm0D1uK6?{nMVGWowG)CYX{jAnC8n@evJxE03l8p9 zyI=dDnnd|3&Nh!LF|UN7z5}0C?GVqvrRV~*WXfUM493iOZo>XF;@+e_zgL&`=JJhe zDFLNi)TV=5ji7JF#_xJvpUONAq4mCS_3G8y<@)P}Rj$~{e!nRaRWo~Ci}b?B9ICI` z%xVC1b|%puJY?d|nVwr*F%3?o;DlckiyKPvdG zT@{*EPP*1kL`@We6VuoVRnZA(dlU+-g!>CFc1@lO#8P z`naSfFF+y;R7e=8?kZMvkCVehLIpWF8jV{wJ$de>TThs;G5amIj9iPyOLd$9bSY6nBn;h=n!JqeJ{qe?tQl#&lS_L3Gky#Wq(4hzk$RIKDEt1Hsg;y|Xhgbm3if9u(B}87!VMG#7|=pXm*} zRhHm*+wt=}4mg%AFWZ7D5aK$}>tJ2UPF4YkV1bac!6+ss*4~HZ5Hb*dp`A=gkk4&7{+*khEuLmFqyfXyXE)@O8>BbS`V-l=a!fL+ z;lv=+CoLCts%d@3&O-8m-dQh6_~3-u(d$Pk#t+del(P&!mGKp%A0Bx!=MD`yuf$t_ zxWXboI9vA0wsqxttuo zms8W9znQgV_u`EeKZEu5LmcP!8Qkpe&ALjq6qMVGYJ{xfSCdkBS0YX&B0hn2?vpGZ*Fc9 z$_C@;p zFEqT-Br&8l(ahx?n8v-Wno07%iQkgUjyQ}*{r<_<~=1rB;M7YT-{$8eQF z%i;`zwbjwGi})doQ{}4GdtaKh-zEtgo3{*-ov+0DAa&R9T;!0Rb8hQ1Xs@9+jLv=W zRD{zLygCL4pYnrIXjH7RH#Q)J{0Kw%Adudq_no+~+w!YkSK!&foc(gTe8h5&y3c!{ zzaxbr40^_T68P^E59QuP&vtcMyU5DQIvd9Nt|5$s)oH5XvHR>IOue?ixx=I{?Iy%C z!Ov#Sp$s(CUxo~da=NL(yNj*e9cJr~A6+dny4NpTYh9%GS2_cxLY`vp9QM-H*$_-G zN%cHj1@LP|tBw>D{$W$>aVh`-eBtdXE%*;U={Vkq)jUlg%xDPH=djH?>Aef?9Xea= z*u3dK|4$O{uv{Sh<~z=_!U{j9c6^d>t6Yd{%yxDDY|N=xAc$ueeeO~MlQ#wDj^{!` zf*lxFox69;NKiJMP}{%oMWr=Wc2Anq>fAlbN-qA9LL)~UH*%Qo57{OmltI6`L;!IJ z{r9R#P>&{0PspHiEIIIvmrbI-;8|UW`I<~OOBVa;o%BlpmXio`ay@Cw*_u_`2$Z2Y zct0sOP(~U-O7KBJifQJ=mJQAff-VDu7|0n<#vS&?Lj7^fb`?Tb$4 ze^tNM5Mpq`c&2%C2ksNGm>25-0u3%tP%Q)-7Z7BfKr|>aYk9`C0w&s;<|}`yG)ac* zNvZGaQvj9_O~!KZd%3HT5sfjpoQ$i-5nP64*)9be?rW+5*#_fE6Mp^n4bgpk`=*JIv_Mm);0N0a-e+WhZzO!e_5xb7 zBH?zTy@)`0s`6LA-cmB8nHVH|xT)hhd5=R@{La8k!-th%^$=;f0c*i%Azq(3x z*$zweNrD86h!OdOm$ks>nZ~5W$Y6;)J0~VTKxIB2JfJ@L-fJbB8r?8Ul%n;E)uJ(a z*hV3v1%79ld=Ls5&~~3gg}-}&@l>A-@(UXw!t7XE8`1S3ULNsB ztYxn|EZV!ffFX#az}U96m=w{25Qcfy z#DtE@-Msl%!VgpXhPJBH+LpP1SW^$7gfE7}}4UK$O^8t81H9aWElhS-yULec3ERdIbypRwY zdY~2%(QI3WTYoh7!~gD(lgUG`O-p8y(Q}U+%)}tJAckVQ4Y9ASAYI9RZsU|Cn zju1P+#Q`?R*6+e*5*YHlfLbCv9Jk#Q=CxQG7mv{PxqJQ)Tl|EP0X>octC>g%%1GXFnH zD1$~lXEkWB?!6(&0;7b+CD)lZ1`hXyHUJAiOlYf@fJea!nC<~%%_ES1_-?ga2Hm-h z&~;FFAjYxaq##3hcM`@lP);@h&ArfL!x$J@t=4t)7lfT>Fc4>1gklaKaxd_ytyl6! z?tC%-B-h;X&V}zY1IQuR2U2l8xneH@sSBhrPr*z92;_Ex1{H_M@JC4rXr$W`{FEwm za@3R=^0ahd@pjRhq;2@IGO*7yim@Comp>69g>jIcRWCfiW z9w=>=bD=A%xKiJT9}*J#17Aqsrb?x67WJH$8}}=!K27V#-+w-lV~}wzB^EIfCgY-h zyt~q7+8X{|J>)hi_w)XsgwPHrZ2sv+1P4rU zZ-BuaCMITkSC=g8E0D5BCR+fRcXuoQZ6u;a82{G~yk|B5+7Q4e-=s;qmj-qmhGx+K zd|gm?07D@fAn&{>P~>eVG71wYs+~?)d-9B-khVC6Rjs(lW*Z|TqmXSr@HKatQ$j4oA6lR{|M8}RT#qo^o;I*2k;Rz3O2s#q zDjHj3rI8 zx=ojL_52{9PrD+GF*O$dB2-&RmZmesy5u`D{y=P^-@nIrd}JsJ+c1D0h!lG8gGkEV zGAgPR8!@(R5WqS1grbq??qLohxMnj4<9~E-WfM|xbUR~?41$m1#&Id<_q-8 zaDabrQ{jnZ{D%Y#?Jqbd&k)Nu@9jF0bK#Wy>tpA?WbFkHKTS%JXpFNGSeryRpl zQ0V!0N06djjAS4a2Om~>bA#}6bClVbPF7A9y&&q!Svff^Rgr1u=WzO&=hYnDvDBEs zA_|Jjtnw zA9bF=DZRG|tagA+1pCc&CxinMLH0nx0$VCuA-_Mr=Hf&>94;`}su5x!eK`2eH1$-p zYqF76LY901Hd$$lG6h9NLc|Y8JY$=`%4g?L4$NE*P0=_`rbf`8&qO zljQj$ANde-txB2BA9Xe377+$co6A|Ex?#L%P79s^$ z+LZ^I;pcK(O`&c+8ZokxaRB#jxN^Irz?cWgN-+GLvp&*Nh_Qtx{T`p0N{?&{KY{i@ z0ldYo;eV*2%w_m#W^wZ>m<&mUYf{2OsDqXS-M1Q+rZRdAX(w-T-^@&hwP0d|4Hm70dPvvBN~HEYm@AM%R~R6&}Wn!)xmlaIl; z4Sc@ERqT&Osyez6c0!R;W-N=y<{Zsp~@@ZHJyEfm_+G>U< zo`4*2+JHq4B@eX5h}wzldB(UefNroX(PX;mX>>>Myz?XW5|Rz02ajW7?Fflb@#K|~ z;^%HHrwYt_iP8R#Owp?3lY(DMTbt}ZWiDuz;p$6Ax^H?PyO7xCfqX++j^Cmshso2< zR|^VSNjokAR7}UuiU>n41uH8SWS^1uj!kHb;YP?a`U_ck zc{lIh&lWN*>9fHgwQc{v6(a0_vERYEd;_f^mHpz5JnK*-?yNsP(VxOkVp$Os&s)2R zOmNI8mFsTwiZbhjVcuFcX3fuo&PLOMh9pdm@X3^a7r0VKVO}J|DC>R4OOCt0m>#9B zq0#3sfe1&yGYY9MhQ;=W>x zDp1-x__~gxGFb;+5cWw;@1PQ%;d*k56Za|I%4%NX$?OjU1Ka;>A%e8%BuqWNuE0sg z$jpo>l7tVRu-ZvITg#7dYia=;FeEl;f5!msDArJ${REPmy|%Xyoq%lq$}#Su1!6- z;a6CR&>*qFS+>PF-DsK%62bJS z-*#XoHHEb&09L0kqn`yEQ0X-x5fMtbjH_EKcpv_&^&e%GLH16OUDM~MpP{w^-{SSk zS$yS0VI*qWSE!9a>4Fe{p;6W=GQ)(TF_Zz17Dz_~&-G2_yhB1A%$%|J3$ri@BA!$8 zFU;rarK3TaSH2``OZEpUbjVB3%G~xb5pR*1*O=1d$VuB)&@*hq7l#5BqRzA30 zDif%cN1EO|M}CPV#IQ-uO>R=y=t0W_&O_u|P`bZ~_-f*v$Ub1Ijsg#ODu}EP+`t2d zAx(m$;N?_bc)Mxx$hZkNSSiDLGp{VLVa@7+w;f@eAtk1kVZ|Ome8KQg9h>Iim9895 z$}71U9iEK6RJn9a$=K(_CnWvU;N?KGf07Kpp|%0A0IkfL%qU$n`89!ldoxXcbgkU6 zW(#Y_+wZHH2K#G78UjAfPx~{~%r_=#rB@EAK==BOMcaD)gh@wFPYdX}r?&Rr-T7Y$ zI7M1J)@68^z^7q)bGmtL;&#gJb}7s$l;e3VzdF=^|AE*vK5cUh%eoB_X41-K4gwh*+x%U4xWNb=iP=_zPi>Gipy}0JUNm?_8utudd~AHB_s)O{ z7u+e1W|jUv!(}p}M+&?QKCa5KXWg$*8}6yWZ)a8|esWwyR~R)Phbv;D8bEUE=;BSM z^7}V-W0rLqso7#F@5bMaY>IN|I;Afs2^iAULv@^vXgIk9zsLW06B=9tjF8y3)WB7_ z;Kp|R#j0d_rsX*@QGkpuWe-al5f~X{eSi1#JaJ`A??V#1%BUd-0QF?Eald6($h;IT zR-0vHKpcJrbS^v4QDC*8l#f*9{F;}4O_{vL1edQ={HIxJT5(36+e&=kQ&~iR3onDe z&<)H7AQIy#;er4T@wo$VC{>Fc4HbL+9=_6~P&MF`5=Wb|b|Zyoyz#keZ~7?C6#hnS z498*d!IBTyl+GwBN(FZhy0*P|$CB@b03st!qz{I2rS0tOLK)U5Vig)Pha<%NZ_D`) z>W}a5R1s0NK9OZ7d&-FqJ-)8)d8BfIBZb*QTpaoP4<8sHA3`(*ehgul%-(45OWPcd zzkUA2L9)n-Ab9BKL zi;Sa7TIY;R?I507RuCn|XOx*XF;jNNR}kq8b`ruqQ4!8G>GbdGt&`UUpCToc=|VM0 z8e#U7V3*Va83rPq>cMx2C+Yio`Re7_Fl>!z+#sfMfW&nRj1iv)nBw77j0G@kXJxS= zw(YE})?b-OBTq&urE!lk!kkoug@3T`W_+P{eN~P z9KW{GGpez~!y{nNzHl32XQYnQPwUg{j0{YqDHzL{>zH%tekV=^2OF%E@k>>RGQ#us zcWampO9CR(1PJn#*sep@8g0`5<7*p#M! z=Si$+2(u9erK?j>Ut}V=WtQv`3Cv+(>I02?bH?ph@T@;7@B3SrB2M-%5)*$ohCL7A z`0v;ZSyu?M)4$aoGZPbHL@!{*`31YL@S&UPzy_tALycDkg-0Y#GqM}4`AV9EiECw1 zN%s?mAU>EN?sDYZZV;@+j!jt8%kF1TW?3qAu(!95iiJGI^I(V*V6^k>uRo3-!2oBc zazpA85Yi%qCK?bK$o~GmgprQ6_9cn^DRcnP+_zzGc93P)8)?pYp1}!3&L?P3i8$;d zX1NOshut347`=xOM$$$@~aI2bW;57hBk$a#>_1{mA_YpOkiRK zGW|W;Np=wHcg-!kOKN75{y#Lf6m_-(db6tdE=HgL;{WZ@WRiODVil%#>w}(XvbTbJ z4PaRiszTcfS~@7%GQ{2}o_)3nii8*wnu%w{YMVzcUKj&+jj~EU_1So&Z>`j;xC{M| zT~sm&yAZ0WNSGH*umAl(NcF08qEe>?hMeEj;BA$x3Z6!oF9d{XD{XpO{jIPeljg*} z?K4>rc^MiUMV^5X6&3YFk)Qu}EvB>-;1>TWalvX3DRJ8n!xo6tV3P_nfo~t{W?V&p z^NbJM*Z|Ll%qpDp#bJ-}g<|x}mw`Zq1W5aefWRdPyZ`8wTM0!h{x&L{Dt#vUi7xHA z_d+i=GjffZ|y)(nPj}cEi49hlLmBxLdvRY zW6g7qtx=W>#LrQATDW<+Yeadk==J*!F2p{+GOv;vcAx@9m3M;ZUv!D84~K=68W+ca z+NSW~v7%h_?Vz3rFy-K7xRi>Yp$lU#g!TbWdi(+cV5$p;`0iaKcvFEpq%4a$!{<7a zB0XPXL0xU_#@Q{`YC0ffz!5usn+LB7#SDAN7>~aS(`lI?@wjOc4FYTwhoj}P`3sV>T(O>Pl-rgebyqB-~ z?hfh-yRVXCH0}ikLMxnFSV+jBlPjgF`sLBB(_XHqHq`5>jd5#0$gVIyPQ5d>*|IEV`iVSpLyf?zruP!Id-N{Ek*9CAaLny zSSABo(Hes1UFabOkH#4rrpcy`4hotX8%}obRh%cJmy&uANGpw!V~~2{NXwx3`H>-K zpbSGG{I4NH;Oa{Kvx3u)h{>=$!E3qCRFWik^nK%sRlqu^y`1;6QuHefMa%_;{vr{? zZ4YwEFJGZ^JzT>ILOj(Gpw`XJP0Ai7Fi)X4gKY?CN*TpbjMT=E#BblexZ^Dj*;GSE zC%CGr3K&g;wlMXcR~ezEqa!HN`(bL7Fczd%Km~FK9})?o-N&#In14)MPe=q;=~mmM zDFbEYe+N3s63PRQ|DSQr#Bnwh>Gk$7(noe1cx}&1=L}+%*5v3GG|g`9u+P8SUoGhc zPZKoQBfe7i4}+NmTp-I5GW(YbO`EZ7_}>~|52<2_?Lt^UP=lCsy}9<+uF3t3WN}~#>OlGS>L%loZ$u>LO&}k_zqupdZ%z^|6urS!POoVh; zEcg9_Hv#B@HEosfZg`VO?5487CA$I;E=PoWWOa}V*qd2irL^ncv#SOw#r10t&Xijp z@SQbx2?dooc3{K3iT}d)#{s|BMaA-ebsz3K9%Sj>)}iW9ad=tHGh!mbQc9@wh*XAH z^Fp;(zo4a1f<;M(6^VNc=l4|ynG3fRQ)L$gv2d|2bLw`C7iVZ}81*mI1PF%{=~TRb zk+-o><$C|!!}SG;z3#*t_MPac|9;$QpCBtRHfy+misK%xoGD}Lp;+VP4t=ItPkVJE z+?@hNC{WR>swEvLK5B{EF}t-0G)Z<2jypGRwiF%!H7qA9OA*LHN{Wi)ljkxCo4E(z z=2$n(fBDk4b{n?n00hYp1S=j`yDWI|z88oR4w=u<*zR!0F>}XTe%9NWzI*Fg#jYeu z>`_Oi5c$ya*W)5XOHuAu8}6FArx^|ZZ0j=R-3g*QGHDQg-jW8Jz7}EnuI1}{6#8cC zGl`lFEd8?9U(V9wB=cK;nA~4_zh6Eny{JedO6*rC#1++q3bPg*Bfs6jVm)$2&$Oq_ zrfVZbvS1kS_UNR);*U;_He`(=uF9#vEHO#b=C3hrqS_M_ahsOt!X7Pj(z<2aoNL1n6%`3?&|7l`z~xf$p9w3a~AFFboOkL$hXE%1SsWKO-$7e zZ{x|~Ny-G9E@FS_MJe-^5ZhN-TFn$n82tC)|IfcHoQp9fmoI((KmYx||0KyQXHAi$ zUJ8Yi;I{Nl>x~b}lX*4UHEtXJbw2e0YzWT+7HP2@VnFPs|l#*9wM3XT(@yZhp%2doNsmLi{&je4uCbg2iiz=uz2Zu@Zc#W-2AwBZ zUdl|?q5rx!n(*CjsBG9c2ggaa~le-{#}xb%pgQ96~}KGArIC zeC@;n;Jdvm8>4muc@hNFUy0}_1&(9z_a|Ir%_PAN45{P z1APd0&AjQ+OZVp!b6Prt#nH`Ct2KlCG>m*Nabv{d= zMqdhUHibQNh5)09cy6W~`JYDn0VSpkn37P%a)^o=u#g;1@J(<3foV8$C>3LQz+}S= zY9v}H($51k$~(Y?gdOUy6B0P8$NlBY26r)bMz&PF0)16zg%dvOt5mt^q!pX7w$))t z-Gdi-2rFvhW)bfH z{vrxxVs^KOC8JGmU=jOfH7QDXLFJ*og#cgSH;eMTMv^sRl!@lZ=v1EypS^}bohkXp z{Cph%Y4NFrQ{KIMcjXYqy*6k`@Z=;9_O39^NSxCx}ZN|idSa&{>JF)=?4YodmHz+@7%d_XgH~)qC!()Pn(w0 zPp4z^Ggb->Wle99_~V2{v~|6fQ8_V9D+Isd&@H#0ZcP)VY+6FH#;)WhSB?AcOA+TZ zWOMo7Uqqo;CU*34D8*B4z&&!IW~s>c=>y)C%@EhppvN&Nsh5%N`-4jaSNkN2(9a19 z2xOO*YJoGNt)n9y?}S-VR$-w!oX1-}>&RT4-(`%J&P^IH}>Pf{{vRdpkMzhm4GyNH%$!kf4@O zVbMWs{0mgWEcz*L9Zf;9kZFIV(rm6X#<|&z(nIo2d zH3z3Iu#a3vBM)Fv$hGUrS9coQ?(FW4?wZpCKRdWtucD&~*M4zJ>-+gyVWHAe2QOt0?6fh_}P@ZBb*^#Q%QOCcJzU!Hq&&|G#??XXY;8 zRhnAG3U{jEhNF>j;+qd`aDcc|J)^Yt_cxKVjN{j_Znd#@|51586N(~;Q|#BSiQ5-$ zpMb|sPaKadB>A_J0d;$7acqj8W2Is3Lcsejq%|+gCjUGHCNjJhud;h*KJwm|O0Sm=4 zdg+t1K2Jn%Ut-V4D)fWxKK!kqAr#BYTi+DsnXX=+AcHskUmx`BxqV#!`WpPFGV(@e zz+2w5Ltb7Hqs?Y#qxM_~dC>2RD_U{{`pqqQqK+SJ(MBo2xaO?ST`V6d&AzLUBE)#k<7np8V&rl+?;_0eEj7{>{rke+_@Lu#OLhJHgk9hGHx$>nNb;K*Op` zPSf{;YmY+R(aecm+xX6Ped5E#60@@Shr%e5#k5?#FXR7ypvw1;sz5LNua7(G^5NQ6 zx2A0_pF=Hhm%P8v13{!%yKqKJt5YFyKF zYOfR!PP7aSbJNlo;Hwe0Xck1G+88UwX#V8n<Yoy1=r;k$} zW8aye_(~V)y7(qOOvza$T<97DE|?l23iZo&i(6)C~nOENvWpv`Y7`2OFU$M>zv1EtG2)7sxeS; zuiOQE_$6C-mQPaE6~ST;dB=f~pCx6oD7^$Vj<%N8`cbTzYu>e~iE#Y!s$HoIaWL5|v;J26o3w|*+_8>@250R>hWYzl^?ohTRp z?OxS%FqK6G!N!T13tzl4RK!Ct3)t=CD6rXEANQN53QmQPvNY8gR5kzhftH@0@ahk! z+*>*w+%TKWjRaA+w{_`rFLPA8ZFghj-&W-rU-d!JvyYs-qv~df$-l1n)sdhqTc~|k z9yQB4jnYJFDGLN8KmmS?;3B+Up5l~_`2_UB;8VsH`rV-+j(4^Xju zT*m*{K{8MpH@(q@+pO;9RtBwi84w(IfB(k+-Zt%I(-??}l=zTKx~;b3U-^CE#gzp~ zi%a$Ri~7jONR2tj@UQ<#HilfWz$ai?OX;dk5Qk@sq=Y#70`jfD7Zsm*fs+nkSuXu~VRe=kzk~ zS>{Px%Djl+t-D5A2Bn4zdHI^l9HX%oSy({WkuqrfKcBR) z8Ro8N&uOMV%@|c+JJuJz>WQCI5EBz%LCTEm=J&G=8P#l+Cu^T*;Zz9;uyh5MMUX75 z@9)3=e#uXcz4l-+McTP1RlN62`*l@Sn+($A(k?h8+;$eb&%U??U*wW|^cQd-)Xu&Z zq7jHmTIA9-D@v@)$Ri4!t@oJeVD{77DSx;poH4V$`K8pzY+L|(xZy9_x8I0yx|I+a z2VNw9F?KMYo_YS%PMXupWa_O9u{xM(^R#13g}_zeyW=|p?>02pD;%Zk6pcbFwXM1v zw*3wqeaz$Ev7(~P+LMiAd3OF>qPtj7*Nsj-(d~-HCtx~7ci!D z%tYyZu^Z~C+FJ!Wj!b!i%G9orn7vxh;Y19&I8SqWBdo};;=v#69q;3d)&mR5qx&06 zGwZcs>Jvl5Lqgyi$cd6$xN*Mr-=@@y#pVoS}8k*yZ5-yk4Pun_B26aR4%TCKtmX;*{z?-MB zW(%L^4pqGf2`>jd%&AvWro&ec5dyk6VH$-ZD_EfwO#;VQB!M*t-|-)HzB+ z&l#lAdIP=yH9KWkz2?ToUtpr&R3Vv%xbDAz4R)QBr;{TTqS0&>3*&e{ux0>I0e8yH ziw~y=2wA6^nfQc+JjIPqYQQLGpUxH+H|#nLA_0m(B*H4KKLEcu;{c2-qV=0kfjAxo zqR({zG5+`YnuxAlILCU?R2o9x#VoN~UT1;n(gN82Yd`rp&p}>{j7K4o=Q^JvF;`SL z#{16PQt(W$qBp69ZRueP&Lq#8^AG7z+K_`UR=8Yh- z?yZ!3vm2%$a@mA-aVLW5EL(S&otF!P6I~Zy-+=0m#VTD@D;xQ;k9qRfJx9lGK>?Th zP-ugO#uf1P=KS?1`z!g!vyy`R_pTUCJv+#tju!_L;8neq1jPPvI3d_Ae7(-!D?2@4 z_N9z_eqT@8`LpM}dwlxu)Eo{ZCamP8oK-i>n3yUMDhaSuryDP+cLP5mk%rx}3k!r^ zHAbuAx(&fq|LGTIWv-hiji%dS_aOGp0I&mH{oVh^WGI!j(=GXHNkD#wz4D7h;+?8U zO=@%}Aa{A1?ToXg{=iH0#c<;h2ds~Ohn!wuWK+QXFwrYCKn_hzpij^-=YNqqeE$c1&4JvjhW zoQt0V=k#Ci8oqBB7Y!G&(hB)_>TvA|J7weP%)7ncPgtIK$q9h2P!(JzW3PsOefOmE z^pB>9UO8;@gMOg*rhRE3gKKkAlz&Oij!_9}sUgJygaZ)!kK5n03o9#I0bvemZ+$!) zY}vcR8H4IQ0$n|M)dV{LG+4gvCaa;fx=d7g&9m87am3QJE-y^pPAYu8kNiQ?KwYa%KVsq zfbqa>?Y_Ng|BUi+uUIvcTxz_UTv{}=I!dk0jM%tOnC>AU2`9X8(;6!S_) zaDs+HAYO39mGfh7p;0IxgS58~cquqQ^`aUNy1TPiNq8@8E*Is|#K$>Vuac3)?%~7l zbk1vxp+Cd^xX978>fg%IySChKFmbtJJg2BgPJX6>{4$1X!VToaoEUYIN%B>@`IPb~ zM@`teME$s&&x%fDiReq9+IbCHj$b~+T(s6@dw+Qd0ka-5!`2}9-&Ujb-&TVnvlk2z zW6$m8WR+GoGB+>W-u8ac^y4ZCkfQ~lzX0nEY(6rQ_y0jnRn^tef?(k} z5{Sp=y#N@uG-PWDQ$D5)%Q8?ab%gZ5?9_r*Z=+Ns_@HKGsV2v=`&ZAe^lvl3evVfh zkF(0=%ZZ4oT&p&NYQW5|4Ki{lNUf%Vkq)Q@-(J39`X|_U>1?g3*~z-V*#%Zjy^D*> zra`Zo2rW-AYHKEn7y%J>I752rW zXhrX4mi8H$&b}{lYaq!sNLuQoWAY+W8P{xUimaG*sD2;8eCJECU(R@3EI*b- zM=%rBMX5>%3ya=-;pQEme6jTX$+~*wW6~zRg@r}Xst9ujmZsVi`;GAo_t9un%x;p5 zPqS~gf~e-Jm9n#^nt;f3;EC*{YsvaMo)=H;l=tKC-YpYV>$PV6{K5A`sSi*JtG!3> z2OVSte4NV5mr#PrqIzZRI%asMKoGUA(4VQ6f*$Jc6M~)%PzySYuyCw?Up3nz-_+q% zl@ed^r`4b{ieFmxruZ4Zq%K}!HOVme)13$zlg?2eZ#dq4efXKp*oR)~-W5kCh9i|! ztZW}D?fnJx)7+$vZN;|Y07UOK( zeDP*=b(@i6EM{Y)k^JC6kplJThSj5+)#c^0m)P74JUlQmo<&1!UWV$kl*3VbFax1> zJA^R&yiS{Iv%z~(uAK?Ed*FDxG0+=|C;(+eZfX5CCgoB(S1BJgf5NiAOq*%(ZF*owC@Qlun=grFy~&9xQuR|EGxx^GOhz z&T9~E{2Ml8H#z!#;`CJoa6TLPoeF0-3CYh={J_m-oe6hS<`sgW9?}^>{Z>M*qzBKq3HUpaB*W#lXrSwV?F?6h)>-2*Uq+0;ebl zEM4Rg8g38U{Cscde-L*1(yMN`;W4y_ohprDg}N&Bi_fzA zh;|KkFm~(xSY>bKkvkoAF2Z;PugcY@Nv$%t`*WW7eu=un(cQpra>hmc#9rO^>Vsr| zy^^yDjvcOSKdc6W=T>m=%kFlAiEIu zB?ki<4V3TJ4TozDZ(T$wz7bBP9r`U%^EY?6aF)5nG3-&5l1{v9aa4FN64gq4g$6g` z44HfvCX*I|-D=KAe7INW4y&7`>!w;iek>zwhRkFVl7Ow7)&Y~~&yS?&A?_IV0T98| zr5cJTWZ-n?FiIarR}ja0MFxa1mjp$65V#|+P}1>+1E+XTEIDv^n0OC4ji|dz-Se6t zmQsU$e5k>Al}1j=q-Y;Di;ArIvl7g!1T|ZuruzOD25BqQapAZzTVCH~XD@^Oxd*1Yk3N(V|!9@-xeIz?W2%Jo5^2CMgs7~K}L5yEGg~|03kg*ioyH=Aa*ca zT*uC+&MG-om;rweD4w_2_wS$36)PqB0#2c_UM>C4;D-=6@$`9R^XbeOW?mu0Q#z;9 zJ;r`^k`ZIt6K{JAOtoj3MebX+kOHk2!5*0 zzwMLc1%?RXn0?HOWX3@h0B~AiX749~%r+O8EY-al5Av$r$Fm;&9O`LLl{HZY6;s3j z2{)c*y$>d+W|tm7zFqh&qDkTL>~pZH)OYjy&4Z9eZ1*cg;gR|GS4M66O-v%yLBo9; zXXax-{1Mu4&qbABMMFk{{ zGY*Ku8l7xMKjj{q7m2W4BLs{QEo-9oEz_|Fif&SC^+gDKZQeMMMzkp&ri|PBc@>I8 zHKD-Z;IdVlbWv~PdfPf-s8M7 zc0AM3!^>;t+SwgVrnAqhRx-w7M0ZseAMp@#AAA5j)1@GLdQw zpIu563Orot>=e`*R!4ZhLJ(IH9iE?OwNqQ82BSQ&a?X7?3Wbp9da$ikY$|TGvxEW| zI~3Wt-+&>0J=aQyS2PzE=7euWmTDkVbK$9q#86+>P(`U{wR%~xV`qxU3V>ZpU`t>0M@-qE)-5mjj3S3B(`8kOF zbueN){b$4=7f;i$X=f&%4a*nNgg73M#owK#XhI`8l!L2_Mhf<&9b;OvLDLTKN_0@C86Wmr2u#sPg+!YY8XOM-1PwJcBOQF&YHhVEHBW@u1r}GU z`yki7&D9Yqwf5?1dw4fv1B23mfd_{7WAh_Ld*a~q)2dSRYIT0nK*n)dT4t|9ET^l> zRM;lH;MBQ+UODQ+gOHmIWWQoTK?R`WK)+`JD8p=}pGSJ0Zj55~Bw>2#8J)%O&=7ih z`okX=@yYU%P|XB~d#i?kMg=uDG&m^fpaOXK-BhKjR6QeZdzNrl#_Ig7+Yv#y`m+Jg zyuVG*TDSE;7md`*imQ`E^$A1&N+U2C08~h$J$ke#-mJX!(TzO>t)YAV=3ekZ#92qq z*<=H6GC=x+Wd+ZZGGnbLJAGeiTa$csRuy8aCIx=9ULseqAFmj z>niDmnw|7J40`2ughC1c#7;WKR0+t8XQm#wbi9A73;i1w{_%?YA{G6;q z8#G2OB{$s!f0u(zeRg6att}cErQtqiN(@*xDj7sFJ4f-iZoBA62R*}$&a$m-LsFYA zqvR`qCgvK^uEjDgu}ya%vj|!Z zNW3yGdxGT_7;8|`{wqxte>h5~WLYtY3E*A!Ru>94K#yok|NMM`ve&DwVvnH-_OB>RkK7Fc}rg`%CF(CZ0FY4r=v25UCKSe{TC6dNMb&aN`rog()L=c=`0F)XS5RDyH z*d_;>B-((09mT_Luf2lIOw0@rt?KWJQp?T#C&fcQEAbA7rpgqTmsiZJAbq!*YjBk- zd&hRt^3G5*Bzk2&#>X{ICvmiebqq$|qVJ z9iCS`)Vsfw!4*sN!j>EyuE{iPg_A4Cu$19UEX4?Et?V)FjA+o2G>Ix|$@IkpR;^%2 zl}j(YiIt2;HK|?W?dCzsZj<)8B<9Qt6hAP|3ja68FIDZ#sDv? zUG@wMKzw~h=a;|62pd}WR}L;>_r;;z+AJem!G|Qym}v%TSXpCwScf%wh+-lH3o&U3 znR;@}J}$3yneIsl_Zt>HROEzWgH11tnOI zQg9I2Cx@CN^fUHM7cufh9X7~5>R!}zG4OrR#_lMZaFIeH<{`WKRtDZo~i+J3G#O7c3AYpHlk^&BG{a-eoC|+NK z*h)EB1;k%=ZoN|%_1Kvu9LN`p%*+b@z0I<-?G0-{Skd3S8<2?6=MXu=?H;Nb$)5P* zIExGbzSs)uh)Xb|X-SM+Cs4pfV2OLHCUq66B#JuZL z8qD7<4(*O*?l;>|581$b&vBy1%VlesG&57Eol>|v8!7uHKiXr$6*l*#H$OAezwcAF z`o!+rnEDT>43$E#tj2uYJ>i0|a}I)tZaAD}wuzhJx+2yhmLofzKLQY=0Z?7n54RId z=!NzNb-9Mq{2TK(>@3yX-KE^-?$%uvp%=W~%;HC$^PsxL2dyinED+8!uueQGvJ#!a z8bp=|-q-N^8oT91JqWY@xx&EugF-09%I0@75=P$^OV^T2OoUXLCNf|hgfG%Fl$Y|t*u)&or6?R36Ju#F^qxn4?695Q z->-Ztk9x#dMz8ds~qe*OjO}VN&hK~D|F21H=FHIsMgY6xaTvTb_XQ*gB5Cp za~)c5ZNa&g($kk4LY-K`IfL?V3`D=FTIJ?SQZ2MJu7JvRYxQNs_~rLYZk8Q~ID9kp z1$=^8S3TW--rHyL6*lSrd6^;34XkJU%(V0$ZbuECgg6UFuFU%}#nI$wGmDWx0Qtxm z{LSDUvoeXKp5iYbZe_-Auk+9=hpih{SG1$y z#p{X3qPiL{XGlCKG8g}w#E)y~;f=8_*mwp0%Pr90y<76$a}HJMfaHYgaEz7L8 zs!o@XHYP5(q-YJqn_OOgEo7;o{YFn)J24$`_#Ui797^#rUYicK;qW_OirpB2jVk9( z1G33nPp`J5#wpyX>mG94eIQe*l z#RTb4l^mauF-F+-@jk?k9L#ZfjmlzcYI@;~yZVVD4N|&hW?QTH^Z7YB*19TzhRVvv zSi^VYOWl!8cKytaAAS2sN8g|29B>ZbM?}01YK7mmS1hyu^XAwGKXJv-QvIMvAO28Ko}yrFyH88H^z%Y8JKRg z-`|@wY|d?ITl|=hX+i|#FYPy)Un||slnC5r!*y6q^LhIg`~=J3DF*uYTQPf2jDJ0U2?`gP8Z3*t?63D zcr)p;86(i$S)lhUob=fa$(+5v#%ico-AKk`jP&{iptYopJGGl%C(l?>UR~sX;|5W! zAc!O`PAm%^@L+aX1ylLh-f_MxHpjwK!0Y8(o0#c&=PXR#*B$c>*(Y@S)#Q|3CviY24{Wr+n z0-#X@&05!31<9=)fwdVP9xf~@dN+YL2I_^UU%vNo%(S{z3G(r&izI4-Pnz^b?yjoX zjG+PGX3)~gYIMi{QnfDL(5oHHdiuQoe975VO`u8G)sQ9`ez)A;SPMC*e6P0zHYCQ< zG#`aI7sTn+VpJ)gRCbbj`GM+AP}(*G==NPe8Y!;MzZB?g!yYfz2TGuHGCp@eY}bc& zPa5G?2eYJ{u{(_H4@dkPE3OadKp(HR^Lh28>VtxHty{J3eO!A&S2Ey%=~fipqf`BKpGu%M?2@PFiGN8 zjOhAB|MjscWDWsN@E~_$-Q#%{=D(@uF}ALi))8Z&|>jDO%JEugF_5qpKx#uXmTr7}8i!y=_D6?)??iM(vim z{Vz@<(l*uwMwqVnX4egj?l8rv;h;`GNlUGiIR#-M$nBp0NITk-^u@fHm)M{JUtmsH zee2bbId`3pYX=7oDe@P1?drMCp3U?-t%uae!ESKYct&vzcT_C2Cr5Lx_5p7|8@_Mq^F~br)MtJ z;9p_{t}EUIXF5QX!B!pePF-93d4NNZ$-Hs zJSMJG$%h;MRl|O6O*5f5-QbAxmtR??h=U;lY5#_Pm44hu;= zZF4-Vz(EHvJl8*PwJMzEWM_tmRc3P_99z<@Y&17kR^HN7p`hg3fm(VwZE)pX=fdkF zn*7UExsEfkJ*%5}K$6Mo!SYQ=w&T5^#oLK0;<9D}(Z}vyr^^ci(#8h;sf%Fn)QSb# zI|mu4=AixE$x$(S53p^8z3_?(fMnFU;D#B8LnsG~4}RD3_p>j2K~Niz0_SICJ?Jw< z4;LOZM;fh|U+3a`2I+Z_;u>uD-XU?&RmEdkl{{#%O;SVh#6g3$+X2UzTY}gDCNg~( z?0{J8F@oEjKiIlZWsfVdV^NN54!Z>= z)T8l*!-dui6N5VuJS*KZi7rHrbC)t^*?%;=YHSeASRUMP*(pSIqYv#pf|d%nHM+I? zs%*>(P9^SAYlE?F*mC{&U^ZZ5&;1~pL#IM27on~{5Uz1b_+vna!$+3gYcv61`hah{ zotGB6rX*Z{BlPoRRfGjMCN8g8ko~ONTns z)x74!U{3mhK^s+t&=I_O$EpDNb)qN3I5r1nV_Cu+S_>c-FSQpz zIC$+*h334uly(BvV?U(f7_A`L5H*?}(%_T@r-COH6G;ch3!BTTb$Y#*Tn1#v&r2wk zpqbpD?>?evZ{2Jjm$huVs@Z31`|&xb&`);|y7!U3K<^QskSJG#$Oba=qlTM6`UIQD zyEc%QPOQ4@Ly_(}k)G84X7835$`K#?+0MZQlcq8md~oiq>$+S<`~lu|$mEm{DcCwE zt&P^Y>5u?y)V@12vz3{ct-G7HNrvC{?oT^WzQ7>MjJ>Tnd@`lc<)XmvzpZT4jjHdt zsFKEWHIlyL9e|9(I=Gv|MHKuG`Yh$?JUmZub&-eBUxk8St!71@o(Vl~QaG4p;e36y zS6%VaZ!u5}h_if(8O;HrpH@HfAMsIBbLq@~NrG;ja<>!DaK}Hhro^}XSr^ye&tOB- z##P|T7_cH;tD#}z?VrWyt1uXv*61Jgo3OAj5Z%9j&;Jkj36iBij=#t46BAjXjrUD; z*$MAtzrEar^hC)zEVs|49R}DT*%Joo!s&Wo!C=<2X@*(a7MlGlj&tE=a-aEc#3!XPR3|l)`ic!a5W>j<&tyix|jgB@}>to3tNzt4hY*HnSjp|^( z!k2rbP8;hIP}Si@L}u(K%U;z`%P+R3@$qRvTg`K|8?|O?du%13$^m7+caKD+5Hxa8 zj0n7(Kzk+^HK3SA@<0dXVz@?Vbf~^T8Zd|H0SI)DFjGn1#j&$sxnBij8K)@%dH@R<4ROpx zP@CT%bMF6HG}X&Bc!x{$%8ohjEv+mopfoz3{vk+_Eb$eUjvZMQQ9gYSi^Jnw>`C_iqe)sbH*;hQawpR>GXufn*ZrHM+B)P+Mh;d?&9MI61PDF>@Go zWx9gArb|K6!#7iwgwSHJ+u&QSKm4)~Lx07-`hDfk)uh@zAt9zjiE^K&FNW)$ebucD zcW|!;A5?AUeLXe@SAL>`f1mRs^e!)Ne?Mx9m_B+_>WGBJW|w0%-=kXSI zq-XM5Knq&{G2Jg0XIMo?QfSJ5yfUC`{sVv6vBh-g`+h5s@ z7{ea>%v!jA5;$YDlgXBH?8f%?*&tCFd;)GRf}I`yCt#Q3Xcy~sKE4=_vioCVyFmJH zUcVl5Eb5Ju*S%h2-+#S}{UUVl#Yh$nK?Sb9txAv{#Q!J(BUFDUgDpWVWm#{>v7uNX zk*e}u=%BG^V)SZ&ox_#cM}S0Jo+h!=QPMy&edERus6QzX!~iBy>ENfhsY%HvOea47 z93P$^9`HpCq@fUvTe*ok{4Dh9xQv6YtgAbw#ZK^5gFA;K7#YSZ``<1`EbM30r#s1{ z>@J9kav7(A>#=tBY@#)y<`xnt4&rIG*R-n09!lBy(uN15gfDSeGJKe@m~t*Y*46|- ztSUD5fSg!UL#G5R1+n09q9TIltJ@tO5?8}`+W#>O(HILx zzzu5xS~I4U`IlWgeg{U4**YoIPd4$vSGWgd$$lBLgQ?~|txvD+@aaSj^HE6}9Edve zQ1ZTqD_TCVWNL6xS-9F38t!@6b46R|-W7>bOea^3m4Cj`-*cwRsaNLv`<>GM+9gGy z!5O&)S!!NKC5DO_%9mFvD9nn0*(>}yERR|Mcl2ri6rR%nrpR|n%ARe>;cm@ud<=a9 z8?9@xw-wVBt~JFzZH&dOkvKcY?TcvKU*%%IT0bQ=mwbqYqUA!y&aB&n+(9A zZeZU4P#89SCk9o4zyexxT2m7Uatp^yK?1%ONYn?05ATdXuOVUwORK^$+iR@I=_35F zw&X?bL7*E{@t#kEM=q`38_e_kA}FsSQoE8W=~GHo%iDpknq~8mx3>v ztj%jJ2h*z-{|Uu}5N5({MIX8F<0HO>N> zPIrN%32-BLpB$DipE=Yx%RgNqixIIC(rq2DoCh;4a%$#P93BtY-f<&PxY~SF$4jR7 z;t0&e^#%72A|+cY)Au!!IZ^4nb>XZzpBNMTGgIk{ z0{dWh#$PMP;0|@B_Wu_E@b5o2=vn8fn=a%Ev1NvQ{=CR@URg|xce;!rl-3dYeC=nz z0(*#7TY0^8DWJj)&CCQq@!)XgFK%UKDmKP+JJI~f|7dN&d+mU!z^2~=G7#LrcN)Ih zqX$geG&{!KYZGAm;A+runm*(-mjuUW!{23cP&&S=+&)Nfv9d||0jJ+Z!{N=`oE%^; zqR(t>v{v+6zYn^r<4-<+C<9QDd5w22&E1M7?q*1!SI#W+k2)PowU74wX*6;1zTad! zc{5W@_-!E2Ds(&$XB6svddj8TRO{@EsAKkf8y>GGtv`u*+=e+hwcG)#ApPIu6Kq%4 zf`|GZtx8^38(!E7@csVz7!bM_g02^VRU=T29G#a|VvGz13tUDmEAIJEA3P}&sOJhl zjacu!J@;y_4WD{yWv)DGr6M9!i#Tl>>UBTuys}2mYp$5`mMCt|&ACZ`aAjKg?LUCY z?75OIMqG#93^fH!l%Z_M+|e--01sB4^cI^~Xk&QV-NjuF&ihpECMP9ZzrR*Dm^aN( zTG)|XMeKj3P~D)DX=^&crFvt(k(X8>m*SR}Wii8^Z)Ywl8D)>>OduY;O+9nMrp}!7 z>Kezme)cv8j7FFw_!CeBMy%K}jcUg-5aagrw5=@)JZsP{e{sH^Z`dW%qf8)B- zH$m_&SJi6LP3I3xgyMke(xy8;?GVo_vrOaInPhudy;*^T-W=X_vzlU}y+(r(X{L59=eRLT^|-u^+>PFx#xM>V+8 zQL~t*;FFw{4SDr7fJ;v>t`DkG<`(ZMN)Hz-Wl&=W26z|YV-%&Iwk{caQ~t~`YulGm zicATaz|u2U%MOX_tgV+5*75m&$lK-I?kr8n{aR=c+ky6|GX8c?TT@pfIF~>Or*uz1OoAoVyH2g}ny? zbNY5y2I-D5wPSs!wKu5mH&DBXCU#t&|UY8e}cpu2b1Kl+T}FJI?X7#dmR06yvF zi#6oB8T=>#feAGvGE_%_U1!=z-%%@3-~+MdbRv9G5het80MYYsSJJsl41{Q{&R$(s zkEEst^y1V;wsdw0b$zEI71C$AYaW0n`|-V?>(z>Qwy@Xz#|1qH<49890+$L(xCZa73MzukMPx$R5BTHQ$58PXj#2VFfdEuqDX zI!jY>>A+|MUR-cvt%Ocw0k={|dZNsC37z$#QAt%?9so+YOGLk)BPvdfUhD%;F1y(5 z5AR%x`_A&1A{j3yY>f6yLg$Y53}j%-pb;~L2|V#iK87WC1}nE)zUnQ>BxwOnN|6x3 ztPsc{N1pAw%N=}*IIq0ZB-~;3zNE{p$cDgt_BnB>)N8e(r`uB8ii_NSz1%dSvBfn8 zPm*l88A|Qd3!SEhw6soyPx%)a8o^dAW^gy45=l9Lj};tuzqoIf0f5FZ73zV>HT<{R zb=HUjCnhHCl#f@YO94r!&MAM%WoeOjc@^y(MefnqR7|8m+s(B11M$kk<o%So{nDD_lo5Z58E1M zQ>tmG@$k#)T0ik9dWGRR@nK+u(z@=kdR?!+zU&1KfLWB!dY~Bp#KK8_FK#t)@gRyE z9US;&Wo5S$)6;uua6CiP(A^s{#PEopCXOSCi_43x!jag@GAjj6X16x>ANN1$PK>fG4_3EE_rgEQ-)M@i z!g@7?guYpN`pE(1bR(sN+rpIx3WSKO?yE+#m1Bbs{8fStiLJP6c-X$kAdvH(?hXY8 zivnO?qMIo~^58>@^`+^Ofs&nEX$s&6t8SEvkF-zouH94wYp0~6M4GuMQ0Av!o(t{u zac29m=kzDIiA1R;(44eb&g==qCu5nV&j)c8v&6TdSX1oL&m9SR0d}d~4(i4TA7$+| zVs0O6U4RXlJOAS8kHN+9RAc!$4^X1;9%kHH9TQ^TybU5ck{DWMz(G_-=6scwH?Q<; ziQV^C0t3xqTa4(4u2+yb$9cY(VjB|CS{Q2=-L)vG%B;~1?Nbl+enlbK_jdlqcJ{Cd zw>tr^#nZW|zvH_^za}~pqy}{O>Un^>>jU4&_RWm_eIWFhsspDJJgZhd-= z-yqBOc7@!3L+k2dESDj+4_g(<$tgsgSRM}cIhlG-lc#z>xD;;ihaw0l^A>#8WaG6g(MypwL1)e zfXpfos>F2ZQk%n-X$TScq~9oN*Hl>uNj4VP94T53QBpZW4&R&V!vPeKf{Wjs;4CDbK_y&(Xl81(&H>*xSIZ ztv!{pskso3$IGtjJs0fP2KV;C`~KU{TTRhK@V@(g!<~MqRF@07;k? za}hjH8I+MMV(sXvTl#N%EGfnzkD{%~L~E!%Q0D{9z5qbCw6>jlH4VND(5H=!{|NR2j+1oeWvnBZ zf;d!rU^kdU(tJ~5&YgXsv+)F2L4qg;j4s{Dsb|o3e_{``T>whL z13w&WzF2#-F!%uQFKcMIn3Fws*0yIWrIZhs{-X$dB=V7g<3G6pki6HvD5E6t%hCbi zsvj;uv|xkwMH)WaM#b%I%017RkAF6lOvM+@5KI6i{137PY^O}WMTG>rmze@b63G0d z$!md*Lq4si|J|E2i~mGSWS#(8KaY|%BkOwYz^$dlzb4+BKebcS_$7T5VaT&-YyhHhF`1d=#?Rz*p@x#D! zyKn6#5Hf8M8gt=ng4YJimvUF)u=V;qKL;`Ap0&SE8bVF!56iDNf%Y zEXRz3xmLw%nBbt}=8Z=`ZqCh2@inYbNd9Xvp-L<+p+FQ1dd}D3mK!_0iEGsp>5=n3 zayJgwY`li9Cw{uCPx&Ml^2x{x3wN!99V{Hse6l&MKTVT4xhu2)P#UsNh47BhS%T=A z7H6ejJA2!rO*H1$hma33FA8*|hjwXcI&yAc8mH%Yer9xqv7noo698CZ?jDUFYmcs8 zuXn!R4J;3!+b;NrRsl(i5;||_<=sJ`kVvA?c4AtZ@^{;+_w_-~?37LVbKjO8gFJGg zhteS3xE&;bo7C&rZHMTa`oar6YwBoWFQ3i6PI1se2H<~+T7&br7&CpPV)E6iR~_Bm zMmjpt-abAw`)f+KjMITQ#nsi-P+va_q_%>5v77ei&YrCVYi(%S_sfxvePd6agPwma zNPU|`Ck$2a%{R)tZ+f}2Zz+e4v6}zVm7}w``oIDOM7IP&mM-T5+apZ@Jh;L>;(b{}YY^BukvT|Y zc$okKl1I^0wNH|aGxE>TMjHBj2((Pl3oY^(D?7h{dVm)<2D54UWqY6WS_ZDo@bGw9 zmfzY37uftb=8Ef@2T>e}15^d(%~S7{>5Y>GRr7ouAr%z%5^`Ba3r_=E=Q8__Dk-WOj zKJB(^<>6sn)!+Be$Aek;1 z5}9rIs#0!B0BsiZWvbY&2nXR6uYlI}#D6#ylJ}pOsFq9Ej{h)(kVDhh*}gStWqN)_ zNRus6O7+DTDc)?s(>`=zj~{=`w2B^a{Wk_pHpVk{7%W8v!zoziqYL6fK}=SxOa0v! z&CdaGAZqM~Uh{f(Yhd(v;E3@|Px`EK%DFgA+Zykdru3}-gq-5K+m--#iy+}g z+Umvzrx63-?1BGbVSXO8*(NnLH5yu4>~k987eJsPjZ_0`KJ%-q{rXHJnleEKyBpXh zn=IpjV-*b!MeE`bJ!g(-oySu?dPfd$d&KbNcc-9X?bl1Lk{XY;UtP5qX&B07wozi; zAF+@IM=m&cvqgWx-DNcA{p3OpQEI@<2m1%|0jV_Vd3BZyAU$!L?}l#bM?fpsI#`1W z0%Gz?5Y`8*hEbpLs@#04{Jr9M3V^5%Na;H|6?f(?pofcWH|HrUQzQjzlk;a2v}u_d zQ2p*Nm#aqi!Owp3y~x~h^_P7D0rXVrnDp1(>504*3%jtkB-@zD4fk~p%?{F)OqugJLK>6ec@Kv|+edu4BuPJ%;!w~unwXC`Ea z9lmCf!bBfFJiReUK$zs!5v-+nVVe3O^}^nD*-d;<5Ui=-J^vDDQeo?3;=o)!c*w+b z3bwo4p7Bx1uPe(|?2d_L*}qP6^orAnwZN#_d-%kmunc z`;EyCNQczN8~YaZp5fIh-uJ;o${Mu8=vez>4c+3)`lfZls0?)z$QS9}D3<5m#S>9I z;>?!FQLo9$r#Wr-o;>S+-}VnRVB&N?$rMxC+YimX`WJZgrScBiJH|62Q8 z4q&dU3}J%8qL14g*{6&rP-z>hSY69<1Ia|)(K0+=Oy@q zZpiJ%GUIu4o^YtVoI}@F%^hO4mH&C6nYzZRb^o^$oh2E$7Sx5T3q-Dx@;6LxCoZGS z%w7oCCrkifroUMFt>4I`VN&-qZ13JAoUojRioT6wl|=gPmyQUCEOeEmiF_K$#p|+1 zSIz_;zAT}T7M|^mgHQz({GG&w488N&TOieBG_g@CAdLW&o0+VVoG&Y}q^Wbe4TNG= z;~CkRqJi_WO2VbG{7`T{1Dh@2O9J_SNHWdm8yGX2i|g7RLNlwMRVAe z$^)V*1au#*?+-&-bc=>_fF49pXhg-Yxnj&fruTWg&F{^s>!EEuZ6`9hfy9EhAy4a6i93h7Q(romXJ(j<22 zx95(S3?~$d+S7lE*}-jX#2h*x`hWO(3#h8IcYpYRbfeOp3P=cubW2IMba$6@ zr!O|)EG?z;!RP9xW!4zja2-ah^oHC#XWIlIU z)+T!!^+~kQC!fKT>Y=5!jTv*Zog1{mqr+#S5H@H6;0Q8@sePa5>C4$++_CGO!KU0t zd?eI+Ywzmx2^Q4&4ZJ}CI)68ECpCj>KhoB# zbK9wRau~u}v$wj_et^1tcwjTG3#}97>e}>GuQ9L;wUIA|pZ`p2xp4KUVRHo?%&U1V zprx_o%3imH=MT)D_3xg0ydHR`u@ZqL{cEY#+GS6pjUD@F*=tAP`kG&8cy$M}8`qjd zDHCWLj2tFhNqLQXym=?Rd4Eis({n%TGqbiQSRKvMl5H?_LEu*-rcJ>^XsN&?H97~? zD!3R;!DX%$s?;!L0=V%%Tz6Q&+L%-Q0=ohy5@3ZoP|u;+0~ZyewS+RXVj~bnB-yK0 zD6BnSweB@~zBTCAqW*GDm2mCn72s`M+0E>?crDtFiTA#%-db@^fb|A>Zc3k!{)g=H`qBT{(KN4YGilsiuOLgofegF=x0!Q7sL&)iy?lXCt{V~qyuYJi zjKYUBALMU9!}8N}<4~5C3nQPK6$$|1zm%KvIiJ(2wMz$9hFz`a8{F+|Cr)a$-c&T; zr~$0QKN?D*-`!Bnk@cUJ013jMy)BPhiO{rd-sv8T@T@!nrEW2(g5=L|&*08T(a0Hp7gfV1l>aMVo$j4deGxSl^(O4fDz2>6ApJNc=0U?QH- z!&!3xM;w|}ikb1e9~ELA`TF1uW9r_j0xxkW5H+|}OXaG+j=lw+EIIbR&Xu%1{859( zH>*zwfHdT)Z)5XZ!3*oV>@YAsRkZkh-fCslaH0=X0Gv1O)BzPWjcb7o!_+*?#CMQY z>28^zYitaUkPRrgj{+~e(;kKSLP?6|SNOd>S8w;MMq!Me56myb<}-b+Kb`d3w-mmq z^yM(SX^$|OM{F@yec>fAmN!UcMa1!4Pkxu?5V2GUtjrM zg-xt&ouqR?0Ss8k`|<^lO^cP)26MKK0`BYnm&`)aJjWx{dtkDapEF~19WBsMst&zg zmIw(7?YwXh1=z)Q)Z455WS2$E+Q5LJ&lO!Y6We}*hk?^OUZCzpQ>i+ce)Hgr=OABn z%D6P%ZHo!db6rv!XtG^LgGzf`?7StrW?E(NH9$eFu?`SA5COiN)mf3s&TOdu{;#tq zSmz`&yZfW|(64&)`zqzlk%BdffD`1qg3j7KobAd%y|+SldbKA&0!C}k4FmxB2*9ls z0dR7s%7ZIRZDWD=TV|9 z(kxPX=gxP^--o=(JSd~lzHvLI3hT{l5OwY@swb+Cr;Ot*dq!4VzS|xn?^Z7Pxmg~! zU(@VyYGm)S4zR_)KP#n&u68l`ACvC5oSf&D$(gA0?iG~P!I$pGa8 zAS-SIzATOtWk^8#J#qVPW@j3l_E^6r_1cz=YA&MDQ5VbGt9MdfW1!5#lZ(_7xs`I& zIyr1J0V6t?OeU4dk2LS*d|r>{f48<;-M{^{-$fkOQ4&&!wP$xG0ko%6s@^G+0o_$G zWTUqFMr3kfT!_PZ+7ySX0ro!pVUY|3OVqp^t0d!WW zPQ9}8dSCn$f6GJ#QzLxQ0cOk&xZ=ODn?KoPS`Y**)qQ+vW$R*%WZkoraR7Z=OWt4t zW6)2Z>;#Vcp6^YMQA%I^c!F;=k`JiwZdjkZu1>fGE?1FVPsbF#RF0INxn=G*9n;;8 z2sP&wmoW~tg;>f+JN`Vt21AWvu`sP_ueUjXeV$;LmH`Ytm4N0d5EE9DlngWLciP+> zP85=Sqi!}9@Bp9g3CXu9c{eb}()voN1aMt+|4{D}D>%%ggzhi%{t)^MrI4|zEc!n| zVM1`ao|YMA1G37Zfx3Xp1e60*)yGTrW_DKHz(kKUCZL>-RWB{;{&s=M<$R9hF%}7c zdtY5c{p8N)YQ-i1Paq_uPzDTRKvV);_LEMMF<#IKN^k+DGUutsFo~OT-)fCes?!q$ zm&>(S2EnPUx)Z{W{RJ_}Fc{YwixkBP4^d1>o2t)mZGz20zd34vyMBzaQxAlPfC!f5 z{4_u4NdF`=dBb#lT z{juTF>Y_;>Co(<96;jxdJ8Q@o7LUJ#nx`kfSam+l@@o*5dooRP4mDE9@x5Cz=d^zH!R%SDs<&rU zR9qG)BGf1>h7S{X5u2!(8-3NGmn@B50FkxdWYgw+m17NBi(bUO1b6NB~Il z|H3RRxJZtVy=8{_)&S1Noc(^%OZ;tsJ|CO&JgPJYu!$0o@~C>h3i##p=rKm{sLoU& zAp1&;=N;QIjsiC4$~U{{U_d7OS}23erN+S0!Ju)-DVLd;eNjaS2yAFqqT6<-U-lh} zW9*8P7M!;=CruK{z?jr@P>zJnZ-`*@1C$i@M^#HN5D?XvfRdCEX!P=cbq};E-%&rk zGfM{qFdA>Goc-|Htr!Y@YA@M)p(nF?3ppT+mJf0Q(i`;Z%t8JD;pjoI@2zm;mr7UvYEP^K?rAhKZ11nl~E?H~Wg5X_&!-m*ajFnx6}FV3YI~=|o+(eB2DMoF7NQfl(5W za)nhCK#V=8lQ&(SS+_wIhPBhunnqB2YHRyjcJdpdxBUJh_Mvowp%f6Y z%>ud_+{|u!(>k|D$;sABZ3~!8=dg5trY072FfxbZX#kk9sE9@2!eHy`)%n>Otjnql za7n}@B-ee7XIfC-d#}pr={{Hvk~^-hl9^KPocO|izG>v>4&bs?swg=NKOwg*MWY^A7y{qFavB zYfTBC6u1Jxr@8%|M;ixZjn0(-m`P*aq#NsJUJ2$=w1v$rk~8FdE}iw}naO#Rv|)NT zZz2F;B51uYEvZD|?uc=UH0XYJ^9Ja#91kOUUMnEert5^A0~YGt#9Tp_UK+r;?|9#C zL@Qu6uJ3M2{Du9U zm%iXEHQbRDOH|p(1%tPV)$%4U$X*?jh$MM;=yX9d55`B~^LwqPVIn~E^JI#l;q;n- z(QCf7>q>&#d%6nzFP_s9XGE=^SwR=!R9{G;W!@jCA_9#nuRy?`xzm45pXhR^+X17E z(e~Jpd+QHPZ!39tT*tjoRd+k%n_Gz(IIU6qaT)dEv`m5;)uC2+6vk@bpPID5% z`V2?GoO>3 zV*t+>kqZHh0m=KTa&2ILLZRyAb$>0ID6Q4REtYxlL)6r~3p?dtW}!t`Ls#wd!d;E* zOwHK#WCWIn=L0n4u7hc_;kC=3`uuX`0$&xp^aNS#lc5XTiP8YCS7bWTuiREOX}H$% zI|u!RkJ~%F8;=M8G|J#wAU7T>(q7*Nl9oT$N$0($Cwxx%<$4u7dE?yCbkjzWg?ouI&+&_C=a2dazeiw7KP6^$ZfK_v>mdb;|npNbXR zr5A%;+igR1w_|m>cl-jN_?j=T+4sEGg5l?f{42ou)jIsn11r$+5;qCY*x7*?+P%P) zNb^;z-rmj5urGy*^iL2SUJEu$MNae+o~>+0)$XW3n`BJO)vZSC&kN5KBctHdP0gF+ zj>p$6PxsJ~`jX${7I=|2NWe68ZDYyQZz}+PSI#UEXs`W(^%mOD0lCJt^PBA4_3vuW zv#uG}rref=l+js)(q`u@^*MI?Ztq8?fcM#L0(Qcg7N9=H;gWl6$FIwbe%mp#9AkFI zOWq}AzXv$4{9n;{yn4z)O|CwUq3W}2lCR5C>3UJlsa613at9`fwXXZVc*3_*1M_zG zo|d89{nPkdqip3s?>pH_?K(XY=5nZB9&BE?H)EGt930%^v)5|HwFRsxeuh_Uk8P75 z4$$K0(n}|AbXs~eB%Uw)`y3g-f5b5UT%Pcsms#PyaH;~B%)NqnXt)S!#(lnIXs7VU zc=Uvj>tqD@+ekLzK)Ay`()pM=ekw6TO8xXxM!N~jX~rt+WwkV}lh)Q&@ofM_ znh~?z#yqNI{imU)(EnPg$@U1tAohERC>6}p0EUDG&Rd@=@FQOVLXlR5Q3ycW(&xAO z^1C*vNb=q&X$wMR@k6BDz@4m=Q03#5F7rBdbaYio>yrwH8SU0`YH_!tZOR{o8rX9J)h!sPo$M@KozfZ3_^_1Oy` z*{`9i>kBX*?=#4A+S*j{@bVZ~1_Xv5;X3`V9KJjj2e2n6J&ejP=>Pj49EDzmh-X7YE_Ikj;#TIFs4{6 z`HrgTTT0OIN`XIGu>~Ts*?sh7rt)UGx_^j)AYID?me?Z3D$2@L0F%W0=9@U6=s!|5 zyyv~4x?Q;Aax_DM09zLUbd~oOwX&1!Vp(y(Obe#Q+a89^ghO~m3T(7R0Aon2$|4@f zhnRlK?m{UCcdmINaB;zSt!ok`bfA)`}lF2K;mhw2dw*Q9_f$if}J@3fJ^x+pD|9y~^++p)8fwQ27 z(;apz%MybN@LNj3x`B^*(AB&F43^K3HUCIeJ&k$Vlv0Lk;`HYu6BJK~IkLjgP^xQl z17j?l{c|20+uvWl@xfaBvbf*^wzd9kae6WdNM7eaF#(Xg0Pbb@dl0fIVNj+q9zMPm zpsfIP>JPH{J&rz}_LUlGMDVTBp}7M6dZqm_BnxYVp#ZD{n>T@hsRzH9--q@s7pwm< zpt#GOiUp?b=blewFJpM6m96XoKP<5E{Tzd&JR`LUb3{@m#h-hVk_*}zsz5Yc-GI7J zzG~+x(4FVmIuPLg_m)!m@M8Kud>tVEJ+_VwD%vFc7;NC#$|iislK zhudCL^YUt^%_4hNpcR7wA&j5v*JGA(w(@)?R2i`8^cES^4E8`2pRL2@zi~u0;HOLL zm1|s|?Ty5c@r};T@>>8q?~)HJx<$Zr@F@?EhY=cSzT)Ii2L1ph++y=tjeE73XUQ|O zk(3QP3kw)O0VaSqJpA0G35ep));q&6oNp@4daTyGFPcd|RQSa>G}KJnRgby6P5>q|0SMrX z1IgRx{?ys|z<<#hPZuPCDE-_*v7*HmKMZNqe z3)@~`n$BrI&(A|@tkP8)CB1aE4oC*MW0lzNC7__Jg z&0^_^jVBfo*#aOO{QHAeZa|;>-(Q445`SKPe9rr8DHd>0a9jyYI$@L&YC^8jU0@Ig zF6@V&ny3)lLrct5s$PFbV6;6gtaSa2GTO$PuiXl0*#XbMn z&rOf*4&OZP`f`1KasuP9 zwKfoP?U?BsW`Wy?BZ_eIvC?}7Lo~xe;~IdE0x=d~L1W-+^&&PVOU^2~6%vU{);2$-Qh*RM)bpTyh}EVPs@fmXZ1S z@x8Y^dWdLIX{nm2De%WC0>V#gr=F%VvWd3|Gda5C5ZO0}TxEksHejJ(7e&nr_p16b z4KWHixXoDUUqCjBSBOKxERg{$*}p&LjV~5-{xj03Oe21LUbKq^gLH*|LWh~A0Y*{} z2t`j zXe;yeO}6e&o-Q^F@QW4$sDNY)X}lqZ_Z<-Dg&hi5`xX%qkv-k?f#3nGnG9musM|{~ zSGD=YjdYmDQVf|2I8T6@n`^+vZ8HDkhd2o2FhzM_pkzLGXH0zo975F8dj7Fk%FoYV z=o6=K!wV5bWhd4g>%-TH+gc8SsQe5=|6*lxl>^53gd02_(ysYfOdPsm1; zuACmT+*`)E9tAst8=}yxe@sDONDW%)mbk3sxjEpGn*9g$PoxMM#=z5!j3-?_9at$i zagz?SSPBCpHLtT>C28phU<8O99SsCjznS*JOmKjurMzN@aW4UywO{cc*PmSV@$He0 zsR|HXeey{{sQ_gSkYKieh3%1NSj#*G8u~8eK*MM^pcv)Fk z#2I3&-1HtK;bQfe)I|Bkn?D6gG(yv^8sPTue{Rer{T%@b2Zd=Zh#qAGpO511VsKssUuzfH_hM{7)?(#sdu4R^80`?>R!+FiW03 zFiHd@Tja5s8766fu^kB$x>R??mJ+i3m|>4fCr?SUB?gjI^P)* zTG`qr!DL|m$H^dY$v@DHTpN;Naj4Fb3H@FdS^Wv-r3C^BI3SWOI2#qhex&$`W!(`f zN<~4o|KS!Q>-1N5$pLau?Bwx0vpNjB|)nF3!V;vBcDwLV<_Q;e~*v(tZRJFJEF695yd#geZDS=yzh z2SC4F78r7gb*2I@#S8&$7(pE@^RC-`Dn?j4*k_6$r2PYe_Vko^8c5D~k(wG98xkc(jwSf$$%@KhzXeGrDP)Y9`-bx)gr~(Kj$J z>9W$DeToZ7FUm6{n$YKyg!aFq-Zmw0JEk?i^(B%^cP62C1SJ5>T^$fSrb(LKhIsFL*HVnsq;Z;Bvm4-6!O>e_gPvpN_8#eB#E%8bAsn4RfBfQ+1xp-UT~)0T z`8otR<(Nc*XCgRXOFi8q?;hf&@6@QhUCtk+Y?-IY#6f`5*GMsNMZ1#^3IC$6Uf^Gj zNw1UYv>z3M753nu`J5c6U0(BlNK6b;D^|axDrV{L@9*yE`5sRrX~30=TuOq6_s7@# z6DKNTlGS^5QZo8R^tC;mq4e->GCVT^nX_HwlJubQ*HTigxzdSo@$muv{{Bio_h(IN zXOz^`0B#l%^xYrWlmk!W*cGWfg9Xl&;j$i?X7igQ;Jy6zCEce_cRKLJJ2zo+JL%A3WT`!pZN6@Lvt%#l<(0BMcdL z3-^MD*Z(Z1AVKzfYMR|PE%7taF*D}?YZP4EXqJMZvHQ?3CSakwbF2De&=T8Wi+maU z-K1;FjZZo7J@^ibFnC{2Ei142FsZXelB%X0o2K4LFIo8&z=H?ixFVsds(NyL0nCCc zfcVPOq@0&_jXEsAaGmPO6J&5h2aPAmYNJS<;EiTR}%6kS~MFyKFv29EWO?7bx?a@MGUgY>BC8GMaD~jrG z4)VXR)dopD^bsX+Z2!wg_Hm>v_4gZ1W;pl&YAr_q(K9hMz2r{@7->>LZ$cg(9t&GrvMM`8MI7L_832++ zv4W9H{+E8e_B9uLh}7A~?_6Yg)AG6?kni8WclY)E0GknqjBIbC?au+E+@W9U}y!_PAEWzs7wyE?a+*=M)hs$ zF3wH*!{IMO-E^7{3H@1tudeEpD7GoGKYzUVHpB*MzQf;aB`!FIK01wf1^YJvGfNhSd|nNAqCFA zCq#|AVqU4M;{%6J$|RZQjKTUZ!^^^C`ZN!5J;aG0hV)G(Q+l;=)6&vjm|*Z}pa%K_ znIS?9q{H#)Y2Fe|Lqo%%om23Vl9d&=u=t`8W16DHwM-~3zFNfC6Bw$t|5;{Ujebl`qfleP*jGIGt>n91y&M4NFi?-4Rz$*Z@DE@lo=hJt#TqM9} zI1U6vpqD7JQ#58Jo6d^|%u7(PvCp-zHX;MfilHi$)V!>(v^mQoZ2AJQ8l4TRV6h9?l{|e+DJUy)k2o;<~eo*jzTKs#hGc>Bx1c27Bu`jS}Ue zrfv0rGd!H19x|zF*;cv|NEI(7VG9L%{uhfu*pw)NGq#|O`gvkUvgx3+Q-YM=9Y->j zJY>uYoJ|{>XMhX`w3SgqmY2N;HE|GDYcYJ&!o$D#T{5fp#g4^n6Y>$4s56054#YA8 zHbTWWZ-71W@nB5%niJTzNS@8);APwy6(Ey$B+}`F$dDH&4Jo)*!3C>E^)a{jo{l&9 zhs9c*s}9@Ir_xFFxd3_OC&(_oBR}ah9~1aX8y2DOyMk!`myh8N+5H2A|K-BnyrGSE zxqWo)8d3_oD7$y?J9ORA5%o4@-I8GsyWwfYee-XWK>u4;GjLkPIg8H5Qsa9l;7@|EXn8v^672h zXgUI9i^$GT6Lu3bGip9Q5+LnPa7&ssuq$31U49=C9-1?@F4{a5VX=34Yfm9Z{|Eyq z$Q>B*hESrAcPSN*{l5bV)*ci^*zDw=RH;S2$b?Enj<8-op>7}yDKd%ETP-i&oTZJI zV&lu=8+W6w{SH59&MSUL2gfF$9DVs*>&u$5UxCY;}_C>Wx9D&tn8Q zpv4A4|5H}hu!rFE&@l(BBt?zg;2f(O)(o_|kCrv#62m zx-q2h3lTWDpwISu@EVHSdw+fT0wo&##8{%`eA-8U=_HF6(t0vx>^NsXr=L;DyQrRig7gRUbNX{AC^?E zKfPi^oF>^a2lZ!kwQIh9TrZ<5?d7v-=*KW3jhJJ7_Kmoulk-5z&v`$CLyy#}#@@Eg zngzA*2P@NracM?RRw;_-1q-jQyy_lw?N8eK!aEgwCc#lcPqJ#oQ}vzkM-2Ka7pqMk zFK=G$(l5+*BE}vheYWZlf2{)X$rNE)*(iTr#jQ%8R8kg{KnHndMm}Y~n-yRF!yodH zR(n4_0_fG4j*vMg`%DedSNFqfqsx=6>$vxiXYF-CE>%0QdnkzIqZN$pSE>C>DAB8+ z>-pxgb4&EGEoQp=Qc0N`J+#7-|bb|xpeS3@}P{Qm>eTl8S)mlD`C?f3P!2?2@!}1T* zI-MZsTW-5qZWD8JninsUoCACxS8Ls-!{ZKnl-7-fzBU?+h6ZZFMIKU1fD^*-#2*x8<&`8JRc#es6EDZjO+^= z56CK%peMYD_#KwG1fPGyuD8E^tQ!Ac*=;RKHR%ZTXt!)w8uo7N z1?Bfj#9#S7ut^VM{e*{;-8S0WX*4{S_!)hq4)4DDx}|MPEQ6IX!?%ZU_HsE-Y#g=| z$-9vJ)nZKzzwdpt8*}B+<^HWibyg-6aizTCe+nG46agsyb2zfYhzqsa8Edmq%6y3?$yA!z~` zIv+7RC^K7x4iO?_8>DC?ThUQ^GZnLN?@Ixpq^E}fZ}$(G9K^a%-!sTrn%x6Pu1N`pozZz%-+t1VBfy&GGk`6Q}B_9`n3=~wdQVI$O$0;DJf%+gGBp9Kgp&~05 z)mor^hY57kg)J;L!<`!9s;APvaPZxN#2IY2@m=B6{s_vvzJ)>vWY>)kv?Y}E^sxR_ zQi1#*M6;LCoy}0@G$zrpjn0GM@J%D!d?V=VjpPdmqMbZuC#YgTW%xOxJa&r|H!t2$ zNe7bf05498_D(+R@1aZr?p}rE@1YDgdJ-gc^2DW^E-2v29C(z+Q$2YztppL4vVpMF zn`UQnP{UU_p{{pXfZqf&sAq|I?A8)KDW?H?pq6RTmNhu5-+Jj{M<%~9ZrB!xFniGl z|MUeKh1uJ~Z*G$nhuB`X>7D*(&3+F-=&>8yW7Dd*PKPJGDj}Q@a7AUu-}XsMzyGN9 z>|!uyA#2%vCC-&EN_|%+IksK@{o;oW0vb!LlkhWLwV`lb`wi^ky=&-X1dV=DgPMi| zDtC49!vZ%oqxES{oy(D_IELKz>GCImL27~vVXUzDe>Sjiqa*K3qMW1l_prSz=*L4|jljI@k;Zy&w>AyZ@#H?Qo>VIYEn==f<6s zAP)dWz;V5otaYKvvsTA{%BtW5E|UCL7aIZyl~Ylmm!9N!?$SWr{Yq+&VDy|?pL$o%6mKp^6GO$-8$kc>G*Id3<(Dph^iJ3DHnm( z4zB}mE!LXkHYJL#tHSP=Rw%_>l$V|^T;co35%-x1Kfd7cX)M2j=d^`7fs<8iKk`GU z_^EKFDe-ch82a|OM)7JFS7Xpz2umT5*NvF=_a;pH7rl8>sU!zyf~s(62*E{jMI?_4 zPcL{j+eQ_w=CpTJALWvMi)r8gL4xfQ`AMaJ*~cmG@gw~0cs7QI)0A{2=~&<|rL=eE z=_}Rk>Ke8Ao*HY@)_z|Puf#w_E`$wn;MJM@-6-I|6Jl6h`_YTFrtB9(95)Y|F};K^ zL0NzfRY*n61F+$;x5X1R4_YNdHf<`G*Ubqq8#I$xWP`pw53t+FQIRsVs?M&TqMm;{ zBDW&oF0m1Mb0g7oJb(%j77s=`pXcitB#JKE#^NI_TS`w9CqS2rL*tkLHdqbb;JUZ* zS|L=wH~9b^p=M=#Ii)mWC~4Dr+1$z-ol%6v_fE_9PCh8SyBOqPU-rM!3`DLN`uYmC zT<`OhcQ7$9h=_?j01gojzzL2JBUEGuXer5pjt}s%3y+RI)gmVdGKz5)7uQo7wQ8?I z>Ho3|l2FWl3Tlv8eJL-e_)}AFo_ZdFR`G+vleQ~utM?LE43P8tUsB@Y?Q}_yc;Y(} z6J9@Q+J`2=g>mFX;v$uJ*V}y--+069q*f7^tcc7cE#q+W18w34J^x+|4u?c8^!6mq0J#Bm0xX8y~_s$)c%%+(<7WA&?41= z{>metNoL(hG)pnnZq4lTr=hjMKeoSa4?mODO!4o|T@=|-UwyeemuV~ZItdk`_(>)5 zmLj7qZKo?%qKF{+DCczYeu)7htm?$Jtx$W#lhcx4CL`i4!L4lGDOyP6Zs^!J79ht; z(K+CH34tVwwMs%4f+*~D(Cd}g$6M%GhF2OE2F_*XZ{J+D6Qww=ahG?@fG0khW`*Bd zVVPBTs_+^IpJK9L-!#_TxLu_&zP1npVzA?~Sh;b&lA)oFVZFdNK}K*ckl_5p){g&M zIeFM2`a?>MlUo61F9+fndzSVv=oF@P7#YOjrqOBDTRC|5h2?`z-BSyEb%9T7Zsf^Z zt{0bN`pc25G9A~zr6~bKza>WM`MjJubGzkfS`j*sE?XrW^9nBL%$sJS!R)0b0!Zb8JX zra7!s6@O)}ql)S$A~--B66K$^BJc^Ws=+$AUH|%(?a$`J$6rsXZ>^VQNum^`5o?~> zI#305uY0@|(dsXBFP*H&Akf0fb$G$vt_lheL;ZB37dL&+E~QG;W2T0QL3Yo2QP1=D z8xK?d|EM3^DC4ER@{5Gk$`gTv05@Ke7V;r_*bz7OShJvgt#T4k8*`IWtK+Duw^Ot% z?$X9DNIHW&8l+ctV{$2k0gd`vF;G1yE|&JhAlO@bB4P`ILr6KjK_DlIcYX)fW51UPheT-cCF>DVILB$P8jAx9OVD z&&2OlDUdMij3O-WpujXWEuH+Dg#!U?CWnLO#M`vtO{;1i5Ic_F5{qwyEN((_`=h&h z(vut}btW6{Xg>(?&n;NS6W~=(51BbrOKoL(2&D_j%y~C$OG78c zlg$nK)9#>ez39T*{Qk|$u$+5AP3m|`0Rb`)tfb`S*RNY>92(}33KkEZfhUh4pH-|c zhpiO-Eap#(2G-94nz{lpH1SfPJ&4(?$2t%H;DO}(nM+0qCduZhYiH2d+-8>O^$Ufk|L1R)EXhDG|KUVMJ;M#Q?BfXty7G+eiLAb%H={-2UD3|L=X}As3|{ zvxRH5%B6La{_~aepEUM!%V;=O4k8?t6}T;d9SjnC$f))87s27+hF|I~RVh>`{@|{- zh;k4w2zEE6$SJHGdv`>0YGJ=CZeEP1T>lt6I#91svItVVd$6;BV zgP53icxp>nG43G)!}k;}W@1N2-dN*LvRu2fZz6umJR&CPN5N2~6cogX#HXX=R23-U zfBibTyT4RSEeUmRj|)gQAs`lhoN!!Jn*xyMX#EkcQ05$AfxzP?S^l`iK{1T z-*NzX;)73&(}q^;VvNbuY&s8aSEYhGu>G=Rl3xp7F7a}{ugL$kgJ&hOy^w|2ult=*F&3R5wX z-Aki`z*8r0>qa6icf}_6ySsPush*Exsp{3yW`i+7T6@>!;?R>{hKD>lo|e=nSCp9t$2v|`!|?iLJo=7J zaHj>@!}F7@ zeDRR}6(y&g3;wSRaI2-wzCnddLYE$Ym=)^C;lWRzL4ZgSGi&0!yqNW8u?wU$@RAGr z)1WN`b+Hr{=^b@U;tAv zMVT2JDeS}DgEFpEOfZuEuQrdVDIFNe8S>88~+FkSz8NwTWz#(U``RiGfey=L(K1* z?(ZqG^8;x5|I$Dc*Q=&fk9;J-)Q81V!S)HuPbtt|@8>IP+l7$$2clys&h75cTFlvV z*%>lmh56vK&rK(H40R^~mSUZ?pd7j_uvWu=(UR>17B(p2b znN+)I=5Q8|PF+;Klpu z<;|A`j!u$@MG7yl-AQR5Qqu9^zdlF}TqSx=zGqzbGiG$Da^o!L%EURd>%A6FzJB~W zwOBKeLZbPJ98#_)ptuhZILPp?$h1a+EU&n_FGsv_UkqS~>&d+x_c$45c0cUUf%wZ8 z27Q9uEQv0Vi9P4%7^|HL~ zJRW2f*V3AdN4@A^ zHKXZ3zdY;Oa#@1j3s8M1Z|3SRJ}htCSMk0uq=GCK6)RUbu6%;q$>1Iij*5EO@l70l zbW9+)TjXlr<9PmihJY?~2AMo15Lh0M?;P6BU}!d7_*8gbo%l#?JC;HZCCK;dH&O*1 zUqam;r|R=f9|)G;_Vf4EInDa>08xpTc^CU*G#1b7caBq(nYywY>5Z4tKgMCPEf7Zp ztj~8`7axx4J0qo84>Zt|^^&I{g*4n>4hO`d9UdN>hp%-B=>$E+9_W8SBEPg!aQ<$s zOHhYnvsx1@aDO@3_d?yIui94w?GRzcWJ*gb;0~D>)5DZ-&42g3{4Mo)5OV|VPlm*|2X<~12=sEO{?U~OYB(+TwGn0?S6SsqEpe7+@G%^tvGOh z^}Y4ve0iS!)=L{S7Qu++x$lj8w+VaZ!GVzx8%U8sUlOq+-M2a?=K-u430#MTarp1~ zQnX4#K{HZR6dpK)v)<~Ubf_8WgM)Ntc93fGjMMWDA5ASFUo>Yvu>z;oTW zCD%=Q`|*ON4S0G_`wKhB{B*Yu1u?p|EA}O|4XnOlXqER2g`Ns61O%2M&Hlbd1g#nJ zgWic}^d$4wmsn}jYCI;opqS}uVGh!v6B9LCeQGk1T-Xsfad1!UO4kXuqnc^cNrG+P zwsxNgYhc$J``w(7rwOXQG!YPi$Y70rJ~6;#@o^yjFw57m=Y!FeaW21jOn@cqa0=y) zEFB5(ImrC6HLEpyeB>0n@{)M^_c%6GEHTS=$Wu*?{$>$5_3-Yx;P>T*H7?m?t+vL444hsK-K2M)DF+Fz1ZKJ@2$s%fpus)Sh*+@{ z_CFN-8XkbOIVLxE5pL&pTch#!$;kB*@#0%}2v!(+rmt3DIh3Wn!L``o_H+fggb4*6 z7$NaB9sL}paz^lMNn&Ef+Djtl-d>_o&PV9F- zUXn|B`x>e2Vz`fLr6G-_Jg;! z5`1^;=Bzf=@Po^)k%+bB=t;21OYnMMf7lclDDT`osf*?l^l_sA7ZlIpN{u{niw^?< zNY?7MV$rGMPyA$?EYJB6FO~<9k<+dBL65PmS*`Azf+-7q&x=Lj&BrL#`-9KiZaATUo&U;NjgP)u%8ie}{%DOG z`t}yp#<#z@-Wb$of;atkk8L;>LkkFm3zh1cDHK3X7M1!a_J(;4;tyJw)XtXK3TLnG zhL>;@C#~9phtQo_#A#L@BkY!Joxt*$93#-(`z4kX=fS2=fB={2eT0gE)IOBdYyf=0 zfQ>pul8>(Jw2gd~rY@{0v>Euz$Te$jUHr2$(r4>FPU7f)GAMsAD^pP!`D%oWhzRR0 z*Mbqbk_U886ellU49!1>95?Z?xuA&(uk zvp+&oMjeAwM5gLRildb|B4f-;aJ%7W-Z*AT9=^sJUtEZ_3k3~FVF?#2lwTXsBAQKD zU@CuVTG|bQcFk2;$j4)@BshqjubWb)yP+{W9Auq~sn52iEN$&!@Q=-_IX^{Jf5}U% zPNQ#26mR`PrkC*#1wr5OeTgQmLac;A9TH0cx-F)e>N@NPBVnc9f1s+=CUalOvRBeD zdVDZ~!0_Di%|jz)F{*eOMR31>YysQ@haH*NMu!54Vb$edRt+|vNK&!$5PqVh)4_r( zbtQW$s2KFF2?2 zNom64o3SLFdf5%mOzi?nz0Nd~p1L{)JA0dw;!w5XVuLGU-P#9?TBpmi8TQdJS5`CM z>%IMIl0N8NQrPvuJwA)ZIo#;Z1INoX1}Oz>F;MeK+BF&Pa}{Ct{Y8 zG#ZSN5M0L61qINSmWWZ{hkhA-k9Z_E*;q^bM4l7-V?k+`^BI2f>(;EncOBo9X}VU| zddNEzt=6Q{9)^JG1T>?B^O9hl5jiCev>LV6U-M(wTp)uLinZ-*zt_dgBOq9F;?aFN ztwHx;r85kZ=)PMc?_yDdZ^_AxVkEc^1$N>131Gd9-sp0vcVShpu& z-_t1x12mHBngag)uCT`q&npJQza8^Y+X(p6Sx7DyPCfw=vzbm}+OggAHPmtU* zWk)p|i0bJ3_*_Z`O~T>p!}N5l$M`ji$AhmGwWiS^7S@$ZpR5I6$q$rvcRSqIHLif~ zbnHbuqQP0~;CTvprdL8x8!f>EYWlb&yb=33W^D|g{h5Ypc%TZ=vT)VtruNkJ zFUXrgSrT_{OOzH<)&$NU6&({&W%BR8g=|1Ow zMFeXaVeG71fom^#!%Xge%>Gohxl2AtJ%E8J(a}JR3PO2){IKNWJ4W~8UxL^C+O|sQ zjK?G$PG@5e28)F*PFYCrnsO(6Kuz<*QLgNLHaIlfD*YgHAF+}Y|PO7ykH4800K%$NeP3z4^*##K|uhH)0DQ? zA@cw4Z))Na6Rj%8sJI-)OVg#`6tx8_;t}^@k=j))x>t@p$#l*>4pL?mF-t)MbK>xm zW%D0@55${oT?$$M3w9xX0(9<6WQUec6D2g$b!5M5{KT3t9Bf21;$agFnz?I2K>Zls zyv^Kfj~3&t_705g(d#plgqPO?SwYnU(l@tBW-N$5qi{eqG$j(2Pmk)}4Y_#VwfU0X z)S*D)Kx@1H_%++@yNc%R+tm?k2xOtzlMrP8Xb{jfd3S#OE408Zr>5@JvnTS=Z7-mo z!a#rCcSn2Lw)-|4A)mf-(0ONYd?Ge$UA0nEJ{>QT_{j5@TW){mP9JsNnP=$x z-+P$wny_1zHvsuMx{JAkv*p5qGkD0^Zeb_K;4Y|0c?uL}XY6`gA1Ul2K)la&d91AY za2`EkGZEYwchzqMWvFHge^2Sg^SIerUjCckZynO#v`Q4dUkV|>U;d6!xncbAH!fJI4* zbSvE*f`oJq-6_HV!@&Q}?)jhdoagNC;mF3gyRI;w&-dQ@e!pM0_#q|qR)xDyB{vV} z)QMVrf1En=l+{NC!GxOGNQGA=Q)QmpJJmRt|E^6(KGdTuPMd+isWE2-8OhoQ7h&w?^r96?sLMeotHqz9pL z+*a`(?@sXq6>t8Mvi6w5$oz__L=9@#G?fc1;v&vlG8TB6lud|Jw2WYCxtMNI^C%k-x6UNpRUii45c$B@4ZoqCLq!n%s^u z+)7Z9@xoZL)X_p=F!(-Y^XSF})u(Z|r9GJ4ms{ktMYGvc2>&XR%LjFSG)@KVvPPKD z6D6nwuSJ^|s5yE0VV?y?xj1qo4*K@!nUOSs1v@^5^5-Pnb8SJC8WYk>t~`Iro!iya z$Q$7erW>3!EX^z7$9;PmAHi$lR!Lcsalm#G&0;bP(etxcPm$ z@Z8em4IG$JQlEM5Fvm_<>-WwlD+@Z=Fn#-ey399?oi-8kf$tDTeF?zt0v;t?uA+T3RAvxLd(h~QZPK&%xRewxPR<3IpQtdbN?HS8SlM@ zns}l2x1?V*99i9Wk;L|jU<^SZDEtgwGv+=P6p?hj&}(VYIZ$0_3hiTH+P+HXpI>zy zve#89ZpE_dUHs@tjgNzdRR#vs)F0|Maz!r-sS<>kt)NVtoScfSMhYHvz_@I_z1Vu} zrbTH|E!)>d&?NjaE&eNL0xl8|xL(TxUO31Iy%peemm@ z+>-eJHH81`&qq+b2i~8@J4tLK*qn%QGHeBuo{-*)}-IU3KNiV{23uhe?e|AR>)S4Ze4dv1EF5LuYay_56KjbzP|0X3>1#a;xojyZ*#<$D7Ju zoY5mv;%*Ild`>Sf6~M>`v6b8_dFsTnpRQDVvLPEi&lUMxez^BFyJO|~dd#Mj)WPVx zt>a!cIjh`K^(i;iQ z6?JcCw_SVogK~xfKf9Q%t1GX}$I-j6*WqC8zm^H}SfTq>LU2R^7?O6wDX?sk1_K~s1;lRT)l(mBHQ zZ(dS__7M9J)!sg(sXmPtkP+GYQi4;Mz3ua--KR*G1yOa}fHpLIJNVQAGrYOx6>Zer zLi}&Pu!fdAn~w48q%NnkWZXKQxQV?mv$;OqFP{6r80(?-McLoGR z^vb;G{eJH^Yg#{&vTc#Z$hQ3JiG_5XuON|M{gF_K=$GmPhObTGy}akqVs3*`7aT0L zua2)t&K6aI4&9(L@#?G3(BVP4UXA^b^5M*gq_OSxL~>P{yB|5!2dQbTxG))TiYHeK z6j`vj-rOd;(flLE$KS?|#mVbba6ab=oL;J~Wa(;3=H#)kpGI!s{?h zLo9D9tZdo`b)}p>kI4V`Z}g7r*{8QFE3evdvY)&TV*3(Uf9ewAj*MW95_8QnKD?XU zYEd#XnEj$y%Ej{YhzY91-2qiu_B<>VDbXGwoU+?39Gl4crljN{LeAKjG$&^qZ8IP= z{PF6s?>gC%eOl|h$QG3zuI(YlTeaHSx<0a9ow)s{a@F!P@M3}=@Ayhgl6WgTGT~4J z9&{(9XExZA+n&{@*BV^jI7B(bK09VhJ3KcV+`GcoPT{LJnAtN119J+$r`Mx13shfe z47e450Mj@kz(&(mWiQO;Zc6uisg3f2E(lDfAK1#J%H2Dpyc=tmII}>_tl)Yb~I)0E*CawF@MR^D@h9G z=U1cW0+WcsFt|b*fdVN*!nnU=FJ`~c@!=72`)xtZ)Yd;%a+Qz4uaNUcNw-tw?%gAy z=X(|~VOP%zYt_{qK%F%?`m=aJ)>o?J+2%DfPlvmG+$aA`Db;#?qgB3ry!(MMqm4Yf z(Opc43@$CIfnFrp>Ip+yl<7-v1X~YTGyGf!*<%Iare0#2|Ldmv_iy`K!;)3F{!@p6 zTuLY0rPfpo7k1x*uQj13pSEkvYT#HOQI`qshYLc2P5Fl)Zi@ZU%+{$}h$g$D7S8s4 zD(Xyf>&GVv`t~xq8 zN;aXt7RgDP^FX9>$`xFf^C7EtA?1cW20_c(b`8f|ZYd`!(g-<*j5ba6?zSQI9@bl0 zLm`}>mz7?t!(!mtb3f%^~Q1iI1SIFuM5-O zKBk#KN>EJ1)p27!3LZB470RjRERSTVa@;WrAZ!%Gd6@OQmd#0=0Omxm>!(f_naFLL ze;Q_gxGctrcty?Bu~1vjX(c!YRiV@F5~WC^tpgcVYP{m5Mzvo4Ca5GXM%}+i+;B?y zVD_wFEEvh_&fKfN%i&wUpnqa~gg!iORe*8+`_A-O7k!B>AD7mY`rM?tlX@1oEO8KH z+L@KYnSU}s%TJc6R_BERg}2f!RNG`3#A4enGnz?{Otk6v!})vaXWM6L@{W#0I;T19 zQZC=1-S&#&suDLm7@TYMbnqG}xIVhsBjZwLyUFGi&D(FC+2W)z0kJ34_3c zmB;2cv@%JLerZLHS6EGMh?()LF_rXy`}TJKyi`_))y`dhY5m>#r*9&Lau5Xk=w$2S zjgo5n!@W7*^&Y)=ep`mk+`!#DkDjpzJMN&$Qnb5yA>)W$tq*&`#!!&{{tt2RrL){B zlRffGAd-qV%yzlQTjlj#hkx9VJf^GKsdc+^K|_(#1*N7*u+=-iasqiclJ%#BRx5K? ziE%+e#rk(X7=krhfr>CmduV8A7K1{vmqZ8vqx@8JvZ67l+;=*8B}n zFToHSdjkhpbnSrk4NAsrm{)9;O%%aSfh$~G3fazQKIfU(Y9o@ZI4=E3$3OU?i22R^ zts~<1OwTSyh23r9B_%7WyoGvF*<=Eq^jPNnMjzVS_U4|%-_vgOabsgP|Ml_p!#sp| z{;TW?4(>3&LUT@wb0y{DUGQrVWZJ817XFb7n@Hwczm8g4^wFlsthL&`8}j6}N4Y9U zZ(wEw|5wN&3P@q5=S3#*ao!_g6?*Vik3M+D>gJn>gLj~O{8vp|t!FFR%E+=Tcri0M0b7^)-p$pVIbI)V z{ga`#h#9WN9FRGnJ3e@_$WGs|n1manW%KT*OE+9B2j5qc0q$KrWI`hwU-s1DAb1W3 zTN+W7YZs{AR#FPk-Q21XdOTT>yWUc4S{wORYr?BPTiUB;KzUSeTHpI{XSvm0qKaYW zm*bB})`+4*xq$F<=ewegrmO2{GL>exVq#1WT;_xFy#B6hIOwuwDGqCH=^KKpv-UV? z&d0|ulgsPxOT_zH*Ok#Pa&(-0_Z@kN43|!&ABPbbo`5F~p5+C;j{If@I&v#|>)Uje z;#yKSS~O$*dIOhELN}J{cI?EZ;R=6p*#rbyS}-d5z&^k67QzwAFHS$Y%wri@-U%z z@(e}a&ogZpo}eCzK)|zDOGctO-*tM-*qql&|9JUg z^rK&o>P!XCA1GgsGTNQPFZSLs+PqjQ4W#(rxp^biZd+cKoiw!-Q~k#6rx;w8uyy;FIhy0FxDx+-={N*C z6EFXq*v#7eC;W7&K5NxEWzuXLRa{mk0=8AjqOlXv7iA?SDc`=e*rDtDE)X8-(^7wM z!(SV?@%-ocJcC*$$YR{^1&H_j@p@rLQQ@6Z#C?+z?2D9U0VW_sG!30h{b%6- z{{y?a@Bb@g0k17QhS(>6IOT<^OCH~D)<2rmT0bAm%)X^n4GyUCwJ6pC+VXj>fXg@Z z=3Q;i#Zk|>-Z6MHvFV+@?1bYVa-RnCImAov0YlD%TOo@MKjx=wC2b!h2x#CU@b9=L z<0UHbu!v-im}oDcWYm8zRiP55@<;Yqh{FbrX01_?19?x2=MLbTj~T99`f|&nbSf+~ zPgnbfQO(bvuGSM;9MN{we9fDBgx{Uaqbd>;S@CLREEIQDIhi9^vKhK!43p+uCstM< z*y_4{wqYt_eEY&JY5LyE`G~@uDYwPT9c5ty!gEx$3)Z|e*a1X_q0Df#q}Y8UMb!S# za37=t)ZWv82GSbOuS%Y+dP&B>_ZypE_3Gmq%%^+{yPK3u>&}tuX7H5hIKSnGxnZmk z>3imm^sQg^0ZU3iD^V34IEf}>D*d*xE|38W0)$zin!?=YYvm|GD!+Y{bZ^}ISVR`? z`8iY9Fg)Z59|QKW3VqI+R!w}8egOKclEld_!9VLWmCEy}c0;iBd0Jo7Ix&*)-8ryS z{JX*q<}DceCH+Ke1+>VoG0)!CY^!GuF_!T`Vq^aH!7txlmd`sDRqmxU^c! zZpS1kf`9JZ6b2d17ibVEnaIB}`Uot;5NBWb=nfP>D-hX@{X=qK6EJIj}Tx%aB13OGTS<%{;)WP(iXth{z zMJr6QJ!mF1Bvt1FmH+391ZD{$Nwj7(M%M3fd1!EV`dS`Nip#(WwxrIFa2Jc)1F>{y^GpLo0e=HjROv(6QSuqJwt*H$^R zf}I`VJUbQESbl8J;qEpR!{im!Zl-w1MJ-x7?9yQC5WKiD2>)i=JJNl3Kj<>SH?7>+ z$TUdDj5W(yWwKIC%q$02P6alw)vtd5{~=wuu>3vzJC;+HndhExh&>A%u4?T4@ug{N z&*P6`Fa1$i#8waB)@VV?cdRVYW=(t$k_+DKa-<9n0$gph;8r4de-FL2yje~<14e5^ zoa$>`+fIAQm{;e|-wK&{`n3OYgxiVx<%9?Fuv}?nOqiB6|BeTACoNG?83vKrPaG#b zmKDJd0D&8DPIg86_!pZh6nXCJ@al;X?k%-m{{72o#tSzfm8UWr`uR0=x8zTZVSkB< zH7R+bISO%(;=3NXd_NZ+D55iNQ{*zvLG}^kEllxEFEys`Bj5I`dSAFU*ypi%jU>x8 zmXQ$S+Iu^R!^JjxI-<>Sg);@IQz~8X>6$tkQ`CsYB#pRt>!3{Azg3N79^G_9h*kc; zZLbq2M}QD#v`A_0UK080ovYtjt@4QB&~n{Ujw=gNFCy_woB}+9&?AFtN%ZCVMDgD` z?l)}%l^#BdD{t&akkPY+qTaaimFphtClyrpx8mP51ivcuaGTHUWqxDjCr!C6Q>pzHT;*p6koJV=qns ziLo}M9fk~5^Bs4Wwu$In{c2P*Tn9w|c$48;v6W`9J6hVVggNCy-k`Qsd$eKn?SjwY zV%Y1VzwlnGO+P)Pm!xs905 zc<=(7(^q50v=t*wg6vd}0g2W~_Q~X49j^xI* zWtRtv43&AU8)SlIcINDe}2&%;?5gzX|Kf zp3j0bBy(w;9lzmoe_+W9&{#TMru-judX2n{Gq|_^^W!TBX;&(E9|ei3Gxggpeey_31LFIOC z{SrhJ|8Pu9@BGDG;kpyLnAAJ=X_?endU?32M}FP;K#Zu&Cf8r_&ueA$N_Kkd>kbv0 zWHfACyfp1wy?&CHbJKWT3orUWjIzGmR{vE+t@F!Eb92tkJp_pH@jQF6D)i9k=!aH3 z{|rv2!arZ{tOv2?rnmlHu3Jvh4OR$Rv!9VeHML&7ssxKlf_7r+SzasR!u!lDhw`%FRvMqO#rhh~gh=_TeK&Pu7gDJcv+S%M;t) zFju|!!cdk?^3;a~G2>kEup*O0sNnC3#Cq=nUFYbLtIYLCQC!4YXt)Ji1>+ zP7x$-+98W{vau0&+|;);mD3(_fW8C0meJ2pvzq-CH+!AL8|7m=NOrqMQ}(=9_a@vP z)k3|#;hUM;3+JU`Eo98VBVud(piL0MRk1o0p}*&*wY4v1B@0@M{i)*#YBgi%5-MaN zic?*C^e%%_=ychy%w(zxmzd3UA=Jk8*#02Wk=vr!)8#}e{`TTg@%dwMYBCRU zowJ1qj)@oiYArd+KM3=MJ_>eB+A646BR$~;qfVygKZqA+CI$QWw#>ky^S-Bal*daD zF){Ul>*96<{td+86XNJFf%c=MKjok2e=ty_l$JH8xgw0p!>xUVFWiarRj@x-q|4eY zake_ii4f~Ix#K!m>8JK_8ya{@{gB|5OOVTcSISKS!jeG!2@Pag6-1ft1W~Xy!3G)# zYRG_xHPjki`9b_kl9+quVrxKaMBUvEstOxj9$d(tM+)j>B^ z9993oR0t*WB8k_w@JY2cIAXC6rU)P1;qCb^Ms%Mo8O=hsoLY@tzUHU?F#jyG3DR)6 zuQG+z_VVbSc$c($oxfM_*zc{xJw}9tSAx1Utl+>N)vN*$90aTnY51KUQoF_?qZf2? z!ddK%-ekPolZnY&&e&^edH_$07Qwm^aLs8l&j|VRV3@&5# z(FbfJ6`&O`L-Xf!c5D~a_iO?Op%Tv!3DSLSS$+S?wlpNY+iXB$lP$zt0M{Ho&geb@_;=@o_XV}tlMBefr3Hg#FkR8r(oS<74w*i9w|y>IJS@U z_TEG373Yy^&tuP}^*fX!Tutavu%+bXN~2jtZaJIuf8D=!&0=jYR+UrUy@K$16xmNU zAalPUefEqOQ8TiQ=dZ_4SI$a-!`}- zVK}Hsp}To!7>D;=={iLz)!~wV;`y3ADbju!yHp6*-x7`w%1e7#c~-T$A=n|&z(R$1 zQJ~h|PT++=5KNG*+;~Ps^_8AowzQhWW`AmLB~Ck&{6g1wznARw&B}UoBDpZ zFf*)2wu8o=RYKCydyF}du&Wt+nb7K?RnM{BpV{~JYW-N~#PKJNJStF`GZDMw5)+K*IBdnlfD#&HSeY>T*kMy?k{W+wf% zmrZECedc&_^-fsSWe(~O7~U`Mgc%(jrz9Dtq@@I_ z#VyGrCM3BA0xTycrH(5uBv)3>6PqCK{P$s9zHrJw@W0Ng@Hq(r{*WOPbG7BZsFx3O z5a|^@2bT^M95U-}^92nJYO1ql1&HV@K7GARm7d{cELuG_+pap9Hu(4SXXMnHZ;wcr zmkXt@UVtewI%M(vD5ATT8!ubPlsfJ_nTIT?HF&5R*Uz?@%bj)=dzUwMW{z;Tq?h~T zu72sUCc?i4PK%j-8PAVh>6mE&h^p=Vn%`|A6-W4!Gn4H8>593XyD4)9r3hhl8mgxK3N^e9DcwV;g+u&o%!nC^SVT1m^!hCwzgFj7mC9`NTv% z1=es|eD+zzA5NN}izZ1MkThTcuwC`Y7KS9#Xl1 zrZErXf=P0I^zgY6D5EfmJh!GyL;pR(0#1c8Ljf4}1pGc>{sIq^gQ7VKvs=gLS#0oY z>9-`kBZ-d%oUB7RZv zt(T$xtB!9*g_~*peSrf(Rp>m(js&X^Gkv>b7%5#wM?j!v;?a{JCUrP-UeEEa6a<($ z%zWA>49`48lRPcAcWBxDAB;$E#ra5S53av?|B>e1@84R82CxA0>VF{)YJRIE;gJG4>OaIoEGnl}S7_KNw!|I4#mmv>lQ5 zBGY(DWWekB^QW2jksp!qW>K@T(`-jfIsh;N58WYQIp=DoapPmg9m)&q8yFo)Wts9l zVZ(?$#6xY%76!kytr5t)yf74Hn_uR_rtorn>HpB<bwp`VAgs2}@E<^rD#-3_TNZ`LuHXbr`I;u&f6wJ$|nd7-D=Ayaad%7JeM^7eto7 zCi*lQ~<)~=6@ZM|Z=*Iszo_;faGXLYJzCIG>dWNS^ z&AOQ3&^qF*{LUphR8%=^4l*<$%V8R$ygeN2kVLHgneifQVpsCnWN!6({n4E3lY7K$ z-g`H*@S@(7tVp;8I!@UwV^h;^JV8mGGUkuN%t`CW_Y-JcNLEa7AL>Ev7%MBwX6WOR z-!5J?e=xx{_m;sCf#Cea$}CDhzZB1fKqNe10CD)64(f~5@06hkZ;p%^Bf14gZkC9G zbW=-9)yol#)TyhQr!#MO1g*3-uv6RA_r{Aq&GPi(fEl_{WQ;zzS95mCooDDrLLY{Q zc&PcjX+F}fm55z$%#Ov5kJ`!J^pTrtHvUuX@VzWdhz*@Z!G=MD+pkYf;FuYIzu9f8 zXV^@~#6Vp70H-TXrZ)R)zB;Q*3XMh_K7S?x5$os0Nk1*#X(euf8#NdEdv+1)a4LTi ze~}m&gI{ynP|jA`i!_lNnb+mp4=-m`S z*5T>BH-&~$0epq*P_oO46J`IH3+%Q5FV+rZ09wCu>bwOi!>g6zRE4sAMK~U{Kc8B) zHGX>e$i{{UzPtfCb^F6)Cun5uMOKqWfwX?nu%@;a{NM?gi72HJaBy5s=Akg_q>;XN ze-D8Ga}okUL^0UloROR$D!_IxT{VxL=4)HY%SRoZ*$QfEL=Tdzn+|tpO>_}DHDZlv zETQj2!`K6!N^Kl<*an3gc-bTnl^V?l5j)&-*Hf!rl zq%)b?$R$eFwRq!&Dut;-xf9zvv)CV3t0fB!y~TR)8m3VX8a!SUak+5TRoD(yjZur4Po3LC7%=F54?hpV$mxL%WHmtm0SKEi_M>3Uj5J? zr+rRolnsMN^wpSqRLm)I)^aFhAnQNJn2GnnE!{*5nY|yaYXOfhTp9IH;>5YFc@t1o z#E;!D@1Ar~Wq(kRX)&k$ZwV{7=gvv5@_#?0foAp%^erxex$sYXE6#dyJWTJt16kuA zF&4ZP&Ev*FIqkP;@Umrw7PL;_9dk>&4AQCV5`oDp;xMCCDRg`;e;NGsj1GEltI)aP zu3-cDA{m@}?_FM};iWw6GA?)KyQi&f?JUXVwpJ$W{53UClUJb9?&jbt1p(qgy`e2v zwxm#1GWzmg`C28y&X=xfNSaAHRnz6WB;v^wM7iyYR<{Ki!_Wkg^8NR%KLJEHUTR#) z02}Cs%lim_m-Roehl>`0h!=@UD=R}ktHK_ikd2=<#|l{%=a#hoiM}hw#mxvl+@Jqscb35Jry3So$#+^p_p^VypHrS3~HpR8F;Rq`1G!j<|p25Q$T+d05<+ zFMM)Ns3Vij5=xEg`|y4TtExe99b8ctj2kAu@Q=FpTLL-&sSGkB#*R{3EE}Qx9wK3BR4A1Y6>>-@PCm@$?J@t%3 zrp_#+PCXua*|`h#P!PX*bq`$SWaBSx+{qW|xI87hEN~v$2?cG5w1RKd1t3X19Gr0) zcslY8dl(&L!T$apnCz3ip|cWuRoWTvO~&c6-?W9WG2@2d&k1V)tLJVEXJsNvE^u$W z)fi}GGR|j#`$CX+eQfeWkLTfJ#fB{B zyC?K`pynW)|9d`dO$J4Fe+i*S>lntKKYx*i_^T()7wwEZ(qWE zW~J_LO)nb|biT^u(h05775`5Hcp8fR*?!{z=XKAJI8#kC+D> zt!piEc>*s1c)84nS=b`DL=fZu+J;cI+I!>uCXA=uczhxG0#-cw3kPP`r3 z8b-VGr%M@uI64Z?D%_TP@}!|?kF|TV+hOa;mD>#+PdS$`yQQ#_$U_ zof=;Y!l_)J{Mnfe^8;oYn%9A>0@tr2Xt@F}%hsPm6uIQ$jOcw@iqez*@P#Qf`OI5{ z&Ci<$(J*VDnw3Q`kZ}irxd;5HtW+o4A%38Kx9Tr(n140IBDtcr0`Hg=xw&Z8p>#jONu4mgEwn@ln@k2C+8 zD}x@zIB?qSFpS3gFIsv|PdWYZOhLVXg11t@2+GCM?Rnv{EEMwRuU9H#>@_v3D_9w` z?<6oU`wy)(6zJE9Zwm=|v1fk0bor)`Px%G(T?d*PKuvS=@`t^D8tVQ#*UWyiR7gPFMI-yXRxaI(AB_AlUA=KGV`VnXwl%v-ewnjw@H%_| zKM0Eu%E@`%JbR(y)f2+IDPDWFFv*D{4Etw(G|23?c**Y3&nqf}+z-dMs_Q}6)#s(b zqkH&MtcN;L0G?g9pWF<{GE-Syc95qsawWMjrazYtN8%TE!y1i|6G>6>(MzXuMM+1i z&N#UE#Bw8Qh$|lpdOcmHuDrjCll}xZN);FWn68T38kd|SBwOmp?Hsxluf)l?i+uP$ zX3afZCx!p(dAh*RDSxj!eAq~xmEnjr;;)Bk$~mpZNlO9**6auNak6Yx38VP3zy~bH zC8PO=arbG#i$vMDNM-c09Byh8GsDECs7V~TY|Y>a`cJ>C9MH5dY8id%aBW(f93ghR z{}L=k*KeEPDxDX-$$eo_c^wBq=RvM83~|58ht=(qkn{e3Une*i} z-qRq8Ozzygef!A`rhF>=nN|kMc~pIUi)3f=B2?2eT(V;nnUDP;Q!wSGK2wM0G@Wm| zRNTW?uIfJ+7DhhpKJR|AX%~cpc#!G8oGcp*?pTSrSXf|di)es|nfqF{2QlfuJ27qb z^vs}x>l2$80ejw8`1)O=WRZa6`XF$x#ZfMciIH@lt)1Fa&u~jm7f}Q#xJcA|$VvVp zmEM#`t9bxQ*AB+I8(wEu5Ge&M!Qi}6NfdUkpDzF$Qqf|va_gVQ$grY0q5&cs)0gG5 zkmo9&LtjUQgwgJ*ZY8pndH|4c?%-eM<}&3oWF`i4qcVYi!A>anxqd8D<7E~hukpR< z5QD`*j-)~^Rfv6~4@z{jvYx9XFb&yF?F6J4EYA51b-;ZzI$&X;aKH2}uksbb{p=sd zTtAB7b5e~tU0nEZ8G{U7wDJ?`&3B+eMEH0yv7f$(aZltab)4KUf8M7a>g;uv)Uhg< zgd=w_r0?On-@LUdPFMVFYMti{v+aE$T&o;48}j$>4SWLb{^4O^E@{_19BsqR>0S1S zQ61K{NXq8UPrnfUg>@3)(q2_dPvzzBnLWRQdMV`RD|McPd&89hD>gtA4VF|P!f`gI z^$$$HJdQ1L1%$tDsIY#_IFN3WrrXMXU0kNXYtDP`-&TM3Q6n2de`KlUs^_if@{q{t zfp$a#Oy7a~iGzSwIQ+tJ;B7o*Dfu4iLx<}8RNA)j2o6KaO7wU?_UaH%Y7UnH7K;fq z5R>nuZ$fasMAWrDQ}BgJOV&orNiJC8E%?qf+gra*Z^%6l`iBOa_(|f4e8uu<5Oww{ z8v}v7|F0Ks#;gwxmz=9m$9?*+Etv{`_N#8zxY^A{058|nu)u|qm7l-4pS1Y!iu~+* zmFF_u?}LNumTCM6f8Z74Onccy)?60W3Hj|>jqRHyw!=xajzQi2b16g-H(w8agl6@h z5i|a`>fC?FZstr(?SGWf3#OoF3yK7j+L8z9aW{9DzBH*Us~|8uHSF=rics}g78VH)Cd!A*0?J*0#{=q>AC>3=WrPIQGNov*~~k!nS|Ve zvRXXRcz99n3m;yZ933rFnoNkiY4>=>tB@?z-;d2B-Q_LR)yB@q0L?OW01fuRfaBp( zPdFHO988YVVYcbdc#o$d%w$aQpn{4G8vsD4`iV8dehsRL;k@lF1U1Rp0= zz-&PXvA72A4gmd7Fs;YeFE>pBxVi`B9amj=^u3-4VU@W|YGLbvx$uQ-YJRBYY6yt* zhTRieO7)`y5H0(9sSsU7p-!wg_c_aVU1F0zvd1H{`7NbyDeZ8?*0Pkx)0^k7d(`g_ zEVZF}Hp5M8$0-)p>kpFb(tm}pr(|p|+VQm?8*cb_jR`C92s;7`^*c`Fs!FM_^W5Op zmWRSgfIZmt@epQg&T-P{Kf_u)t`vd{;-Ca4pE?JbcY4YJ3jOp%(B!Cr%1Ocw8e#cK z-}&(Qx!6jh#tR+C;;}e5_qr1pNf8#-w@fj|kLOJ9NB>0@Qfl{lcWkBo<89Mf9 z*f{bWsDSVH;Cg0vFk=+vqaR&4g-m-UY1b?u*|8Y)%)}}6dsk?(o?H(I2)GFO8E$Uw zSTTo7-@kvq;dkoIYgoskt?xpIEgB|WM)ciRQiFEV*fGTFGDlZ@Pxq4NciY+9@hGMI zCATU7so*EARcH{Nem}-!YqBy{#DYjbQvoRBIh4f?I5&H6(dfD;U5z%#|fEUYiM{`X;;46DXkX6MffF4MO zCLZYc_J<{fRmiF~CfaHDL)|AVcFyqL06Q@a*wna=SKP34!#rfwk#5&6fL3tjAec0G zGw;D~dB@?#(^YlzOF}}p2%{h3#g>pX0b#7s(uM4cwy7!C>3*S9cWf~Q4EU<|^=#{X zzE=1g+DrSMp?yrmd3cy0x>HTpPFq;x$o(Xzpl)!H#(sY&d1l+M_ieM_&c5f$L%srF z*IXd8@nIu9_h`PAAtrcHpk6BcPS`NCBo2L2`0@>PK11-{Qv)-X8<1f3M>0Qr{;Mv9 z`7868+!2G*P-a7TGE0eW{R6jPa0JClc|rG7m=yvls)Ot1`HZ4@V)IKaArsNA%rb{V z4@SFvdIjaF?04pG*PwrFB94>VF)y!KwI+}Fz5~GIZS!20j^j*M?YSt)dO2O6Om}2W z{7ovBp))~ze_`2V@^mc<#EoI+ne+)6;ONmQY~{i{J-?8>)hSnYqdXbczl*JlrW_kd z^){omB#B24I*IF#{{3mZfu8xMjEN9%LR~dHKp}Q!3*Bbl7;Xp5>R{fJYPenC8$ArF z#mabdK42J+o)mrFAW}GaL{R4=&hSff^G%yb`P1D_A@t5>SCH3KJ^1%8enOpTj+6yb z>?Vkoll%5yp(%w#b@H=Y*LP*!K8v`|9t*WDkPabtahq3EyKFvRStPZzAPspe@&k`K zX*~QSpKwcOjx(wA?RmUytgZ2)ctb@lIiX@U+v-+CXgYvbnVZcL5NBs;>)aB52X zx^#s>3~#@}^gPNJHDyRC#)(A=5ReWxTa?^hYp(2g9^=wW>$a8m+9~*GON9@cX#XYM z76QWyjHc|K&h2^|dN9`(9v7{GBV!^x$kJgWe!y11!mr?y%GE51{(y47@1;oJ)^td zi+mJ3L#am9ICf{-!@lg1s$@#uk>V?mm1Ex#mg}#xtl4kgo^d3$t@qts6hk`sYC0AX zLP9bR=_&Gq6=Xn4uW9Moy%BySkF?HZTV+7|UC|P~j=p-9BP`kXVjlMo177aH+moeK zLoaBEx+6q&jrW_@xuA}Dd;3*N%0=vr$IX3?mQ%Ii@9Ka<=hYZ2{{y&IJPw%sk@*P8 ztp@D12_#OI@@Y?=TTH+F9lE-@loKW@TYzj1S0Ai+Y{uS%vS&xVhiK$kgt{!|JnUeT zcXC3CAIg3G`W1`bFSQz(D{g0NHjR`HgulKd)>d682ui>GFRi*dDF$%z1`qED@hU1_ z2Dk2ImCm})|6@cM+@bsb*onk%dp+*_NCy5aGV@)Ln83TY+sa`YX$|?k0&VM*-gsf1 z6bLau*>II<%VkF?(0;qE97Yf!L>hnh8;#ys_0j0gpe-OA=yjqyzTve|zU!L1WwiD&J4yzPV2N1#q%>)vX~=y&W*CLu7K2nac=hzRUoM? zp-IX9&I%$j0I%>6fahcqsr4Ss_hvv!G6InuA~hplze1O{n_0?k`se!<5Z~ScZ;PJU zVPj$dXd{3gDLPE*9R1UqrqE{Kj57P@%0a-dEXsaL|8lR*$l8?*-o6m(y|ExX(-J`< zmiItq5n?NDBz0cs^l^E{rfHMJbFY5e&%+NP3=8LMfF{VbpGjdUf?0lGcdQd+g97m-H7a)$xsbx z;wjV%_1ebJ^T!|nU!}}8_8Qw@0!HK#A0HX?JLMt9kEcdu%r`mUwNgdT~&K;L7_3>$SBoAl;-?ey`P@ z0GB$T0ZZiPl_SYp_v_;$-TcZ}A5@dt>?aFD`Foi7)Yr+r*US=&*HUcfdF!mi-c#X zzWc}(Y_{pvS<-Tiuj3Vyfo3YE%r_Ep=fHATrX=(;axitlh!0=e&5$qZzWv@0Qfiu9 z=3A$bOly?g!rbf5-%yb@sIbK2MWobs0+0mCk#iM;XB<0i0&E)vVB z84H1piST~4S{}Qo3|ZKA+(y3)7Ho&dOyV`g6?ON$i)N0wLxGxd>nBV6#_0Zgqb6w#5?-I^CD>ra^$MN=mS~^$u2mpp2dfrNtH1oW{O;E2|IMY zDLZPBJz*jkXOetWQvNLAlUY7v5X@tsTp^1q1e?o!+qhsncDm3)t9rm}0UyD5&O{n? zB6zdFp-?=Hc8vYPhwxvdRBIOX>t(%pphCBqPhSrFVSw>FZ^8C8<7WYF$d~UU>6mPa zDh~~=QuiBK^W%UBVHOyEYBhgZIRa2OvVdV@DT-(`YJkl`N`gGwQ1xS{jEzQEuoWnB z)*=&q95zkoq1Db57eqpc>Yd zbd6KdL6jI5z?*LdwrIAsiLMcbC&T?gS?eM`jIju-c<*gxL*ytzUm2CG$jQ1>EDRw` z*)yc*KmO$k7F@_lcnv)w^0aGYDt!=wO^UnuR;lT!OfOPF4bBA5iXe-rdZ^3-findU zD55@>NlNQ$8;{j5MHw9_KtAXB&3~RTqa;x*ZuKQ(XEkTc5Cj4|mKM)}pL-7SFy>5M z1P*O6%+`Y5Ndp!H=%1$BUZAeOMnHNt1()glI!ge zO>uV`x*`YDPP%tDxjn6kZQVU(rhQDR-IEP6!Al@wbTT;jaA9E*3JTdqh{PcL+mwW+ zebM;v+4BCV(~PD(DM%@;p>!l;He3{zf*7X0pUa4EtEMv*+p4mnfn5$2n1jN;eq$oU z1+qqiDJ0^e)i1H~i6ph_Q8P=+^s}X2^-54`Ag~u5 z)?K}J!!_xL-*4I01|QtH6lHn9}tkzuy2ht2Q2(C&s$msuDNNyyavg$YVu9< z2rE;Mcrik<>2k)kz55~Hsemr z7!o!3Y$Rm3>NFBHcH0Ec{#W4=hs$uo$kdo6iSEP4qc_c@0aybT*jFCFC1jmgwV(ZW z;Rg9r(Q6uER(@W9P-BxoRqW1I>=nk$O>vclEx&7Cx;>q76FU$(6wflNm{T4D@n;7e z_5pZVwLih<_eFAoC~HdADR*eI3JT|bvz5B&n*wSZ!s?Gb-7z&gO_vsyhQsC^%ro)q zYNH~{r`IM7+gew5I=QTNRx?}KHk)aI3eBsZ)1INGKZ2$C#mf4<>pftgPQCNu1wK4@ zThUF;!m&56IyQ(nEw{<%M>m@|{H;H02aGW*yjQL}dpdTKh^p1yQLmjci4Ld;Uu{o|~VC4UnhhDci!jiEP`E!NGx% zn5{}4n04xxCjP_us}bfO^-4zlkDCi#Zm2$Ui1;7Hxkc>9*PnHm*!{P2V5j3x4=$O@ zxU$~c*B5*Z0BV;Bv^y+@9Gl+X;ci+GW zL;R5XpvDasfSnxcnSdo8?9!l$VUsbcg>6;7htpB;4+w&^sR*JJER-7MtJ4iu%YQt# z6%#!VvH;0ZjJH_x2yu@$^fNHa=j9yfCDj9e|A6(i0#ZNBtk9<5kT5wZz*JW+xS5 ztOr}QjO^a_Bmy<>-y|fA;X+R8>w*d+&xPK!;i*G2JsqGrr? zk!-Wi(~l3Vbei9-MsC6av;_bD(X-cUH0N$SzjuDC!W8Q#e$(c?lKK)_xhGP4-M%&G zt-B>9I)r(-{T+gK22usEI0^onb-nXE+MM$M6N<#nGVq8jjf?+EXVzy9{dCwYeR;TP zx6HxK?cT>idf=9vOdE5BVjT7N3NbD$-MSJDz-U#2qy-rCDP9el`QWBCkInQM#it_o zcmVe`HWJz1B*`Vu$$nfz;J+g%4_{|qSU&I?l0{`buOeCdT=Jn_YuQ+DgL3P4+VcBb z!c4NKW_;KMPZ;J0sp&s&8R4vQ-6zCec(L~7_pKmkEIqy%Y1x~022 zq(MNsM5IAVN?N)b6r`lPr8@?JVeZTK{hoX8AD{D#Jjmf0X3jb9e)nE`?X^&`$#e<6 zc7b=c?&iP>Tsy>_uz$Zr!Jr@rPqiKAI0KWH$QlxT2!qpTQsK)^^8IT^5g-J$3v0Rk zBF+kg248DC$uQ)8l2};k4|)6_b(hPu8Uyou`85^cGR;8yd4~?5HT{Ua-4`_zer}C~ zgsB(cC-^16pB4nQtMhS_5dv5T7|HGf9dgKfToI<8th*Wvu2dhtb0;t+C}Uudr)^MU zh4ti9U|gQu)Bc4<=b7)lqx8~2N-pVuCi=uo8q6z(Fo~+h6%4$a5!`nYe9%5Tf_|^f z#95qkb;>RPAK~S7`PIbtU+rUWPs847$eZ-s99~}dZ9bbA z0Is;W@dmgv=Ck_&iObrZ;J%hM8E$AQY4nl7ODr`|@9Qg@P4f!_)wJL*ad3cGhlYOa zNZDumTtEL`f2*ATOcrfU`Bc-JSI-`g$Hx+<3mKjs(!KiM@)N^a#kJrOR#mKWAjz85o)wZF4eyo+__m1BdA@a1r46+fxssm z1?CMr-3t!E;!lG?eGQaAdkVtDm|X*)N>M*Ue`n}`nMR{}L102AWv2rwq^_1vB|~;H zKqCDF$ksH!4p_Rs?SDU_ih;vZ)CsOTjh zGbp1Y2)dy(G}~o`+`E)GB$r@19#rVS=E1nW8E7Gi%gJ#R03}p6xA*;=`fNyzjqd!; zxesGvEP#hr&iFGZ7+duACS_2>3X^;4!IA{jK2O$Tn3!$E0HqI%E`OFI6PK7w;h@DD zFO-oe(ZC`r{PvB?wC@%L@;F}Zc>9_K`tqIGbVz;8l*j#u0wA5yHlS{I@nX;rV8A~! z$>w((+Wp?d2l;bkAMh}oa_^e54top%;AIUh^S$K-v%pNAh#dzQbX3o>3M%eJF8TSO zQSJdSPl8@Q22vxh2#Uz3T$zOQ!a~QT0R%D{VhzMK!+Wg9=mq)m;z48RTl;Nsp{9M?l}^S% zy6f-j`7_zR7PQ3zSt>|JUyQBOq(*UGU4WsGdCl7T_`#i`sd}w?{dyw{U-;&Yfo|aE zQtc`|-HD*IE(^fC6EFWn%cB?uPy%I}h4ewJ4WJXX@*gTCXfueyPX&K$(93PG+NJ6(ie}rxwumLI8Gc8{XQ+ssCN*zKsC_3a?WnMIsqFfP2()&l_ zi{vLq{K}4fN#B6;mzbp6(tRcZrXZAHV{>zn9?O0tz)6j|w)sz*?Z?#AoxIWoMhs+f za&q7?+oNl1yH{xv5)uMLjPG5?IXR&@IXT__Bl(PZlMz~vbfhT(OH26dl{AyMy6^H| zy_&ox6rbH+|IL}WU+4s*InyKNS^sxS82r>d0VC+~mC(a~Bv!dc0(!_x?AF1ZW-t&g zz5+kG^(av)REX?X{9B8yA!=4LCAV_m*4qFyi_Ctt!?WZqh#aft7WbAZ^(B z4rS_P8yW=fB_C>BTL94${5I;w8Tp4=#)%@M_a%3W*LO7c|Ek&vI`&v?G`RYPkF05_ zs~f?uFSfV0j~ncPwK-e!J{OG{FaQ5*i<}2Mk%0yG|FlKUt{Do2y_xgHPY(}&@D4un z7)*MC&EX<%Y^`IDG5Wuk!FBlq-iMOS`~Upou^rUqxc?7I-I#zBTcVM@+~cKZHc~H1 zkT03jXJdgC1>PWN6@x%{n*!#^C9TbXO&><;{k+z|6-2ei@~VRV7EMR{Tz3-d z(MyiHSsyeE3gi!1KbG>yhW6#=d3J^?6w>$<(s`^O3>lcA1XjZO9#sG0;#>0hkyQH9 zKY-Nik*%IFnZ%XPnzVt9G;WrQ%H`vmR9zuEh=rlkP{snY1za#1416xIZeY{K=i8{AgA`Z~GDQ5y zf%hMf_CD&B57u(1yl}@%4nZlAGZLt-kTQYxEqrg%TV2_#gHxPwvCs>Zio}MemnOEcwysNZe&i{0IuFWi5Zwr z#tsa~%^X~2@5RgI%f5as=It%W#l`h8Ee$h7eC!dK$F$uacwBf9JC82qkKMf|KT5Ro z2_G^)><^ce6q8kd?PG2S9O3@=!>w`M@!(${heDmbm3&N|%FF@OnN^CazBdbg4TTKEVoyc3(w>tbbdws28l< zgm)jh9DXH~EVR1jpx?#r_2Oj`(#D0zQV@sHa&uqv)93dh3PnhYfDQT2VqL!O3jg`*p(ss*jtuhxAlarlxd=ka#p-k1ldG_Aa{xAy+T{q~J zXYZ*nCjwVV55&Dp65USvom6%27~|ekx~2u*^w~RuU!jsLDq53E)7Ieixk4pjaG1&e zz6{`}qLMi`Q8@oPOiRc{us#JYMT0DQ-pi}kRO}=~GUUe>Ssq3`3(v7KGXu|8ot3Qx z8GWoBIoEq)neDs~9J_Da4q@EVJ*HP0@axB>xnI=YNHx?3Jn1rDYf}tiLoQSjjh0Zb z@HKSU-n(~g(`Ac#w&9*8&j5z^czAgqMv_EGDl;bnf4D=MM7d2B5g72j6n}#6HQ~CI z8GQUF9lgp_hN$gt0S3rp)+1Ryp_C)G$MV1xI+D799mzO5bAmRnAh4N3t3RRDG_JhZ zxn@T%;CqzR-he)o=neDXrlDL40}JKtS(YCJ60iMXFYC)_|E#hh%ZnL~Cpy2Ut(AdJ zIvZh~_O&4J`+lQp9EGO#X=1TYd#l@~)@0{*KL5LYZ453Hkf;I&dDo@EW<9nVAb|sz zN#^F}KtLZ)So5Wpq$KM7g8A|#_Wazfr?)r6EGr~v>CZk8n+a!o^PsMsq_$H9)9#xM zI-(yFs$xV33C<-%TK`Aj3lC@V{N1Mb{vU^bBwOX9zJSGb6iSt?wCJ_X{=(T{XA!^q zP6wQtOWx}}4e7NCYaBwJ7N|D2Cg^q=G%R@ayp5ls^$BLEJl4X3DDs$`BC=BEf38ws z`1bcH_Gi>|Ayi<#pCREP_(~kL8T4nxr4g(m@I+N2WXF%lRxQrfEXmO~G_5$^v$H{{}Ety++3r2eh7k8t6 z@C6)oD*F#ZYkuN($PE0-W!Lwk~8^Nc$rkf8a zz&(B=h>e*en1E8IDe?hKRne4bGVUtiJSp(Jh%*p{8MUtsV;F|pS`4dWc z1W4hAhH>A&bA3)u&I1?gLGWtJ6cq_c7`Q-R3k&J#=}~|Z_m;*;h|vm(v!tY7(7{nJ zs~&@NsC206%d9GVDB7pu_!ES~KVlaYdhzKCoEQCHDS|=L*QC`@f9Tcs2JDQq5}}Bi zn;tglOf>1tUR&S84UhqZ{R|IRgT1qnq-7*VdT}20>XXjYbn&QEXo4Pi@v@QvC9#%) zv*{!%9Q0^udU*zdXlc`W8pk8uvZ+UD^@E!(6*qc2J5&fhKIEge=3aQ4?2{n5{2-mO zcL=cGd`?Qb)>8tGpo*D-A$t|=)Tybdv`dj41N>eN6;|J@B!+$_IUF%s$gEy3>y4p< zB{JUgjq@g6#V_-LzT^P5one5LVy(eU8wPX?S3#wO=;7`eB0*C_Lqq$!8E~JFz^FzB zn+M$Ap?+RPLn8$EHr}6UkQER|!H%?Vp;5iSyH8q2hx~qFBqcqlz5e}|tzfawaWy>r zIwki`(40L;6IkbHu4S$ZGTX-ldl%>-0Non z7@@-hJtJ)`)cy4uxwlRz*G)4IJ5#K z!yr~&W+VtOs6{<$7hSepxH+Qfgu^P{CF_?PddIM^+DoyIx}Up#UUiWy zwE4*Q1X$VSpZUExC?>uy(RA~mQ|~A?jJd1`x;}oS3Yj?%Kl?UbTA#bCO+Y?4Z^Wm) z>e86~`v)|EIPkL=r?OPWa^sFmDgxrf7(g) zCg*@dKMm_0$NSs_9QT9 zJ`-xtd-bOX_4bPD$TaGa7#lGyRc(#L@K3UW*@HNYkzFH zQ(Gj6TnwewlEBwEZn`GlTJ^mu$G^`!yiPNNbAB_4H^u5CP}ysu(`|Ge#jqtm(yg=W z1oyXi+8ZsLFqHW^``|GvNq{2l?(L0~d;yNJ;8t!90Nz-%#8DC%_RC;wp!L+Q78Lq> z%YyIo(g#;-JPZRRnWcw&%zoYMXg@KtmZ)1w0~SdKAzvdac|F@~XRf|PfrAzYJtEN?^$>x=An;j^=vE(`fhxdVbtxd;KtVU+SQ2# zKV{}t>)^#;_AXXP7Tk{DmxT~yC4{8l6?5y=Q9fn_sYEUjlhwJOJpuG0WFwH1RHU8; zr(7iFcaR{VuVJ)`7VIcNFI1W(`=j2i{lC=Cp=0@SbJGw1wb?%8rPF-KsHd$qwlrOg zH-^UZf)$J2Bq-f2j*Xoz9%EINbIxuUq?{m`Az&ktc!Ga1d!4WPjE5YXhkTWQWd7hC zX^eCnO~TKOdb{dbWGzZ@#X^FEg99r^5|K-=Re=mtxmf-FYssY5Yii^VXkTPO@j9&3 z*cnPYMp5eHX`}A+^%<-ideVrvVxuoCyf)4~)!#igou!vCl zbQFMEh-zReZp?YWc#YLO8qFE-Yi3M<}AGT<;HmefBP z23Op*ps;F$zOM>p;KuPa43cWkB;%J`rokxEsMO%riw~ET}{j!?3$s82hiH zYis6@lV{(6&OUNE4Mg5v(+<&qche>mLJ=VPP{P6jL+3EeV)~P55&y!d?FIYIyh92$ z$)%Tp)__yv^2h-ndJ@9^#c3PsDaG`hO((Ga>iKS7c)XcBZG5`%`Tbsbg=4>lmLCYD z3^P7Cen%156a$sMUYro4i6YMhCkY1{%O?v(+D|dbhd>yEs=TQ77sd~pO6uBlGrlOf z;c(M@U>5lZQ5XuwL&%=}3*S+3Y&ve|{YLy37Q)ihnSp2qDo6=Y*-fiw@6%@< z@@6;>56=pyT3S=8IKW*8O1uXWTN{htuT~p83rHC0u2+jW=WYbiFyqzArog5T0&-5# zB=fdcxj{w$r>Ad}_4WObzT-1+azb}@b`EJ`q{pY9>2rkQiE1sLO+I|?=bzl~v(vrK zu`-}8+4IeqOIxL>ow&bF6P%Pz2X~X%N6k4H$)+1ErPO6UH;ghW?lr|S?s^!klin@x zQJZZW!SN#`wb~Oa z=X*OLJyVvsWp470(1n44o{JL;SXo=N0=kkrlaO-U^>|s(?&9g4v7P7M`+gsg|8p3* zFxNyFJ2gVUozD19J?Gl9|IWGbJ?jxwsNU1wU+6NPWz#!uT@KEkEhL%W#M;Lmh7U%6 z02R>U+AEM65IQhPG!G*~IBIo{cgEA5!TaGfvj`nsJPQUa_Kvw}7)%EqHl4h-U2uA4 zAR@?nb=h89M}_0FB_KriO80V-rha{8cl_|XBwd&Lw7KcPXF^G7W@UZ*j!2@XGu4F; zAj)dvv9=X(Vqmrv+8JyLpR(tfb3Qhb7@c>by}hiTU#!UTvJj`Sz;E$PBrG;oRj)e} zfI#*2*0|_6&7=im?U*EVTcJK}!htMD*1@72S<>lmATV?N(HJFKNbXWn`Gkemexiv|S zeaO%ec}_T)h!hMKxCry$)bR5}Iq?Ol4hTnBOmoS95E z+1^|bZ@oKqopOE|?d|b^;+}oKy`n-b8;^N>-pIY-aq?B$3r#I8)1RN7g0z8zgIft0 zwG|f^%f5Ld0Ro_^np%!J^UZS_$HQhvCh%_??QL@BAFQ|f?!By6hcrG5=EZu^6bQGc z(!+zN;pr~Zc%XLd2@_oo2OFIp+sdA(y-eRRzvNTfVcSJe?3V@r)q%&)C7*a+U3pvvK(g7ZL8 zAy}PZ=kjlcjLhnfP*vZHLaNFdkMqFTQg#VYT}SJt%m=gXHGa#BxwdcGyT`8T z$X&T?<$Ogi{BEW*E7JtO1opzBB;h+#A1Cd9>mDyI3(Y+)7DrYn#RNe;d-rPclp;8C z`UUw(eM!4pGC_6TN0H~lmcC2%jLuFM)*b?22}ULBMWq+!d-uQ(JqS)#!$LfdJ9#T- zGpwvR*OPSHx;ukm;&jlv%oNR%j_t9Bq}&Wh^YvQB)8)I5>)6~h%b0=jE1>d3M`J8_ znl9J9*fMp?zyxt~IshnnZw@qvuKWn)0%9&OT+^;0l*ej%4M*$J{ileFe{J;8i@G;w z>{tJY++*gB@)?*W(8_|`)(SP+Tmue}2juZbK+(kL2}H=~q0o?zKdVI@{)1a6tEsK? z>gA0+W@g6JDI?Y?n}Q31Z^>gGn4IKKf7x?uqMkOc)d5_wFx&FR2LKKDPr575pjHn9 zKGV~JQczC3h-M{8MM=q=b<|O#@AS{)L4~f2n=z%k>U7~w3@|k;94(n2Qb6N#>Tyi{m-92Kg#{xsvqW>%cL!_?Vd+pt#qIASadA!_~G@| zz*Ox)KTx5wSZ#-ue&{~@u(;ckHvPzH-mvh?bW{7^K`>cBwKWxF7~MUGI1E#<6sTzO z9{;9wHl|mM4q@st_grb$qP4fmST>)C$CPg~tz1s=H^t|K2SP!4OTiJAQ_V%IHF&(V zzPE5>@4bcB$NGz?60H^R4%fh4)!}f!AAnz-cf6wO3MI}zZ`QN(Mzs*-8gTrT!&?Yc z9z;Y_Z#GDyQ}*C3sP9!CT}v!mb$!!1U2LS%@gW~SqFG|qd=53ehIeype#udm@pil! z{#5d{VHK8O#5?cor122aF=5er$R1}p)lCm)yOF30km8Jd2fWa~Gd`g;l@#V{@#8SN z2tc@qoX0RgI-=1-*C)Q%)Ton*Yqr0*(IV`&KB82P)=ydIo^o(C>RuZ@NF3;GK!{ey z^Ks1C-oAuk1sXW^5-8b0AcVwxtBW3SD>-3JSy7fDAwQ2N)J%b27F}QT>sRmUh(V?Z zaa#5(soeW@*FI|C{cIV2of8%+@tpYLHT6r*$ErBB>Zo~{D6(`-4QA2dGU9c_?uBj~ zJn7CWH~7)IZpPQ{#;Q;k`b`ou{eMJZ032}Nh1{nA|Lf6Y789uVt%)>c*>H%IMp&(sQxb|;EI$|5h%gJRr{j{;yu|ACDXZZ52w@$+eaMguld zG4E_VAp;24Md)&KrGA;>%HWp=aCx)aB*Wfik{Rnf&HdCp=IBQEgvc*OaVdeTpRhxLm+e>lWh%>Vm6*yk9B>KbI(s;|c?44+sG%ZKSK&IVNR>(RbV z?zQS3Q^;J0U-`$A)^kiBas5OwLoYr$cX1N(K8YD+SI#Zy2&wGipy<9;U5f-#F1jj=U0F1f z-#nmcL2~}m2A15f;`)7^EJ~ZXt87F`N?YBkjpdu*)V4osQaFc$v0{gL-}1*O-im|Z zvtJh~uDP?e8RD$n3yl4!i6fJu6j2f-Z&n0dcc_*%BLK96^)0qKx{Nqu$uMe4klxNh zh;D7-2PM(+x=&g6p=59T10IcS$KTYdv-SNbsrKzl+L+9( zty|QmcGbthzn1CTEu`{4AB(@5ZokZF&_;(`^^2Y!k4kj=_4W4_pZtC^9^%-6rmQ^^ z9z!8?h)V|rVWG2D>*eH6E!rpw>Fd5K3!nN4N2<&1+cA`7083e%a`Yv}}w7S_W3+x2EDRrf_+q z#zH~iT{B5BR?ZFTV6ifd$86mA&w@?7c?__5n}-5F-Rj(*f8M$52n|{`ZmuVAj*_3D zX$h=g7A084GIVk6y()begJulq_mJ#+Q-KrO25iJ2m1otar1d`nM1|F*dzfhd2F7{- zK;(Zv*ceDg9PR$Gi9<8PpjC3Ry&VHCC>c4aqVeI^;o7Kr3FVzD+oKA23dwO+`Clsl zA-m~$Z}BFF4G+j3UzQKOqp`BG8i*!?Of@nxNV6nFv&6kU78KMVHg33h#kum+ z2xhXGxr2_5-C`3Fqhdw}&`XR@N!fm30L}$lvIKaROnUoHsh6XtJ*ORv2@h{6;k~cE zc5_^U%PQ3HN{!4pr&`)*%4!c5MIJ?|nyi(faIOcg;=SqdbnZ+BsbtTPx99y9$!r7% zP)%J+0IAm-rLIx4OLBq)>cH8!=pL}Gi|lz>oZl=ijZfD2_|ZTB;M~z2wT1qRo%A04 zxjphx_WD{iZ>buB+Wp6DEv>Lmf~vS9Ibb0azD$Ik7zGfXyCX&8J#?8-<+)v6=`szY z-}b>A7fU%vj26rFEf|Jy>#*kAuDctcu{HhK2%e5Usy#JjPu8dtxdX%Puek{oe-0Hf z_A81#d%JX*HEGu0^a8We(F+Pn_Bc=T1}xL^@rFVDk*M@L9x~JNL1+#3yy(lwR>LeH zEk9u%?9%er>_+!xmn{zKCudxFghciIYkrFSF#`??0Gdjha7 zM@Wmnb0Cz*_eia3Co6zu>K6f>jJ20PxM;X-h(zJ`;a zFf37B!{Hp#=dg<)@Ql!7C@iPZTyF9aZOF4AX)oh{{hlB zHM~V{tRlzMjUpv)?UE+n~Ung z&Ii(niy|ZB>EV0+t|80oI|^@VsVPWkx>muVt8K=p9=fksk)7MN)to0ufkAr)Cvh)1 zb_HWWxgAQ+_mZw?;TDKOU;SYh5;`S70(d4=_3crBqno!OYubR&jdd_+t;v_1lNo+8 zTvws?F37C8vJUGtfjTrjBVliAAb}{WPXu*`QE=F4>nW+WXDTcB%O-#ys;Ih=~;7_6Ij)A?rm8I3}&{RySsNO zzWqkBn=eJD;~Yym3Y8oKhZe>vqU4p=@1`qGtt80hbe)}stSOgu0(S|9oc{7VO3^*G z)x)=V$%72=>}Y@< zKEt92Mk7J9p{aiVv%HFfxO9ZDlrmt!(`4c>*7WV)9P+WV%O9Y}6V2t#HHwN~% z>F?OgJ6nN|x~LjEKp!IlLxFt7iRcAcxaslvdG-{v)VP}K7(Phlw>GE7S|m920c_T- zO^VR>@l|a{KL5;u2?devdXWFI1B~@L%GTi96q7DzI0O&DGT0^Epb1e@+7n8_{?I_i zkSC`MI%78k3iwbJk9vI}8l7I=N-*GWZ!h)c4UqZO$sXSU&xHyF2DP|pKr#gk(zxsE z!-Nu)_c~>vnk7>Imnu)=zdl-EaTC=4u~%AB(!JIj#p%AQuIIIxxRvNrHb#Bg5*}V8 zR9Bx|$5(>FniPcy%msmWI$$XlMB$rgKy)w#fmQjFVU&+yD$%*%0Pf^^ULoIotf!b( zUkV+`T8zv~A8_6?ED)zV6A|+^oVyu%4FEY3z2U=zA(VR{sx#6gW4NLhUNQ8r$sfcH znmb<<<~W|~81CKY+K4|$lsdYR=t~XE+dmJI+Z!mFMMMwJTua7cYb%aqhJIQs+|^ry z_94<<3ts&zeyQ5mOsS-88bP&UR93 zY!<$rvkQOP>S4cYgXMV?;a((qa6?KXvJgI7-QI&v^GnR{-Z_cSQwUreyvcsmq=x~gq>BbkeweDBjdE&z7qdMBpaVj`LrVe-m;jKQlRbFRh z;g3$6TD5`Mg~;t1N&4#f7!ElWU(30wh-L*_Z(mG($1YwQ4cxkz&+~JFOD-B~^5;D= z54$%442yiS$KLlR3iKBtMt_3o@e+t>J!fBmj%crplD*q>02`3+=u)`vopgeeE_QwF zndSaAnvYV?Z%&_DK=h6B{9^is&4{RkCGY%gAG^O*ZMOG;CgUZmXb3&UpwrExaR5Am zaj5Ydn=$pJKWuS2qlTm2A5^877AndtR^c8c#oT6l(7xWcLm>*H7yqyV;sK+8yhn?6H z9iEBiv=gfxQ88Ev}yy#FKj5@NL9rDql^hhYtG+hAoqWs1}Ny!WE5aiBU=UU>Wt z%_b}iH|~sy-OwP~xO>f4(ohdnaUwUR)k@7y#+PVs9qsLfEJp5-LqZfpOaZ^Q`}4Cm z$GaumCs*rvw^Oh=TePdQERTme9un3pZP(*Cl#6 ziN$ABn(a-v-4z3p>ffc6!MkB+>8cM{1j88X!qFqY*Y2kOyhF4JZ38d#uK~<-l!l1n{y0ulH`RM+AFnVaNwrN>P6ztzIn)NS# z3@v>sucZvgmy#0RFEFjl4A32h~<3ilvZU0yYT@a9eY$(b@ zKfft@tMJbEc>Ll%fRc@amHc$XPHJ+DLB2&}<|Q^{sn zaRP7qQqWhEIqe`-r#Z5M;gm9q4P>Vnyq$h45bq-S<1d2ee`mwsr)(b%+x4aY5iQ*w z?g9b>l|cMeZUZ)hj}?RJliWuVtVFAS?x3=&kJ2+@exzeGlHx=0avR+u!et>5MR{;E zU<(5AQ3Id|g{PqMv{T0wKT!Jujh$qWpsldGS+EiugTJb z&Q#UDV{`DgcK(V7G`DUjc7KLN-U3hsg`tp6@F^vLV=nT`_*=>eL3Inn0NwK;?}ZF9 zmde_;)v@nkBWEQx_N2&c1uR(8UHBD8#ZgvVSB$y#^(~P9$;8D_eb8?-I>95^Q(`=& z>+BssrMO@8p}dYN>(-ka|JoL9dU?vIM;EtL+%~Z^{_!06 zRuMwgPY24d@`hQ2*=~-=@De{zVmjSZuwt?gkwrC&c_WT+em&Y#Wb|8C+4(CJfA+7E zfxn5&QxSg~`hCs9o7Q8irw{Z}BR}jj74f)eNAvL{)#ZR_4u+S9ee}icK?B4`wAVt? zXl7p?%<9~08SDfJQZ8#EZKWPP2XQ-)OFlGYL~&nJX(~k$^lWl-YKR~o+s~goP*5QF z+uTa1Y)~3Vp_OZ7>7rE_~p2%_tk!0xW zcxL^B6pF6#0XdX`0P{s+-uKEX`l&xXD7dA4rg5p+V^`(lCO~aVDWtD&*jJsI#+5PJ3l29Bx+6 zfxa^iwOt2rM0>Aji3Fh(_#NKp+TJyuWkB%BJ#fZuR5uf!pcwSLZL%jcuyM=?Fk-CFnQW)F?NkCApzQN2ckB2PNqc_<(-CjeNE5R+d+?$6O)z)a=Kw6r!AkD7 zsLQKr!3K1rrlSPnVC<}08v<&?mCk)`Kc!XE+y~d8BceC`dTjYv9Kebm^^@S2KzA)( zkbmlkizp-pG#M_5ZekQMtQDJfhy7^SoZ3;S4vq`k=RN=cMbZR%-!Sv>#Rr5MK%Ze+ zP!DZV6NJ16NU>%6Y!LcV3Izwk-I35}eJz;!sMlQjv4;zDt{TrrmVD5My9dueGMZaE zyq#BY2s$Fq{c`dGnDLPw1qea$-yXvr8-9!BGwCmsL4vGu%Kbv@q4F->ODDU`9@@_q8s{pl>+i~Z$?+wX_b2J|(7D~ZaIyQRz zNDi`J-Yfv);%}`WRv_{gt$kR(lPRHnbM5aT0MO;bzp&>;^hEwi$ZI!Jbbg+~Yu8dB zN`;LdireZ7DIS!G0hR~k{UBQDjUy7ksYreNVwbPeO3ppEPTZ;e9s;sh=+I9|0E+MP zG1^3>IeYru|G~D~?sxJnQZpyWkizbWIR7drP$_9zPI#72%KwzeTEa9ZQn_QsM$Fj& zJMx`o=jLvMVGC?S@b)U;EFOR5i*ws)a%R_M+#@7N{6ib8_HQ&M-e_{FMO8x?PI( z6$=RJvx!N(1?%8tOr1AR9u%o&hKUZi!J+1B|_pkVLD*e^L<2QTCpR`b> zQwP0Av={-p+XGrHtk8Ab2yf~&xR|jj^|ZzXED2-?rGZu? zTrHCGI_-|yxn0Mf9s;l)wEr+r?wKBNe@D*Rx0UCA^_>KSED*$Rq_ofHUz7T@ZlUf0 zI#T2}Q-!PT;+P`#v7DS54P`jU!VPczDbiRjAc730S$7>yzdVij%`itq*SP`rbLMQ) z`FVgdF0f}wV?NjZJ0iSr)&DbSs`fAj8Ibc9j&EL*y4P;qWqR&C*7MrJYdPB{2hb$s zeM}Ut$t`vx-kTN}kPGGgD4t< zb$`Ta?-^d=0vY2(JhnMdRFfsgivh_-?94ev+(;q9s-eHukt?FoHknXMf3vw&zf~s| zY;^FX1k?UpDdM8ZFV5VSI#qV5$o+91lt)R#oSNn8D}T~igIt!sOFc*)j41p<9@Tz# zA;&b_=nw#FX5p|LPG({%LWlUC^gg^ji6h|jbSllhW+xsIZAS;!;cN@m-A`i}vu!ov zx{bJV5V>S%3t0D zQ7EaklH!$2dgXCF0d0K9q4&s59R^$BBY-q5z)$6$Sjc$!ifoOIUEM8vfm@Q(LuC2R zoRL~bcw~?J!vH-t^=40o)xDrB(oGV{>dX{o%|s2@%K(1!O3&hnOSx=k%3d|J;G&&< zSZlRl*zyT}Dk7v|M7Rj2u_hc1#COHU8(bUBteGt-a9yRbcWMl^>kIhcZMgEAmrO!Q0W=NpUqk z(3xG_xH>6B$Oygob4^(>?XR^}mjDb<@ry~Pxc}amaLv9Pkcs0%OSNBqTU{^#7ZlbD zRml_gxb>T@%EJWLo;i^)%lA7Zir6a_-T!8u1qdm~I3wV|(@%$6PcI|FZ823vc;43+ihFOmF7muQSXixS(M5vR&b9wm#`u zeu~@8WBCSW$Xa!cfoZun<$VRc)xL=7mj@KQ0U3c*kQ+rVX2BBNMeZc>Wto0WEDqG+ zXi71AQ;LQcS60uxELbAXY$^}d#dFf5Vh}<=_lRY26Tl0V+a3ft5f{QsYDZ*|zfs%) zdqM#;6}v6otwf31^(SN*fZA}3z_K47Aklpn?TxwfZI0ipy8D&v#Q~8A(h+fx2s%I? zoFBP1Px9V{B7toX8&V`N77eHvve41E1Hq*5*W-_EOJT0=LjjjLo{llilv(+`3IS;g z2eUnIL5u}NpWOXdejSwF5+F(dPTM9MBX)S7KKsxsx5zD0D*$L$f?}l-L}$C(tjZg5 zFwFO#GfUt=7G6W!26F{yjPGUo5XkaTAa?#RJZaYLWqV2@X1Fw?$-E5Jw6y743Y~8k zTfBz^bo6?CTRHzI_rS*S5w?5>3bOzCXp)p-dDG z^u%Dq${*CN5f73MOjrW&*oYL1I5%yf>=SWMPyVj%?)bE{xdT@)qz9%Uz?90X!tqAL z%bPhPj&pwWuYD1J8I;7qTW!5?@(E#U0D#0VEI z34}=io7iui`zs$>l8d!CF$njAb^-5LH3qr;^MU|#hw$RHJazt z!xQpA$9Qqm)?Wb0v|{9hJe`Z_0j!-5y<)M7@?D9VG90~){W9gF-;z?l$?i)Dzdoe~ z>&N?=jQ1EUvvcAu?o}LIITeQkNghDo#jF z`QLmsSwPHDKG~E;kH+qUA%??>C)d~v5xnunWfdej z+1Y2*X`j|Q%Ib9q(8q}L4h(e3-;xOgs94ASV`E}tnAHf;v07&vl@6? zH%*V2ny3RU0yI*t4L|;onw%_uZ_0qbgNaCcZ7Cq>jwUCHb8Ks*mNSP@rGCT<-5ew_ zzP*ZbzV+7I+tcOAXNveG0 z-GMJSDL*Dv?@wWceLbihYunQxdBZ_~a0-DG!lv)H609#tORR0zW?S6osjA5CUy+59 zDZ6*I&WwdD0jh2(X`2ZxyqqvvN8p%%cJKMBzBzvG(3^rq_U43 zGLZquRVmytP>3;AHE8}z7G@)}8ysFnLwUl4o0ag)^KJ4t8W~`l+FOm0P;%Bi{jGX6+gj|W`T9d*wtj`ta$;h0A4F>DJ|@?*wRh9<@8v9FFq)! zFjE|7JH7Y+o&uObSUTdbC>6aKH`@H=Dil^V&`3u{zhxOx;~xNWK$mvnH)u zINCGcc?90Pecedel+Nj@);$riMVkh6Id-@7IcE2QdBs7)BdWVN0DDL^(q%SgpBaansn9%NfqCQT9s0x@0=Vl~-%GY9|2wKbwcs^dPt2m#9>F0;lBwi}jcr%`R(?9Kcrz zI*Q_SFy6kI1#2FqT)r0@$WZWRrPN?7Ag(D}yysnN>fNHZrrO$Q@<_}iT!s-4o6WZmPB1l5k8Fa*Mt zPncKnvv1oLxjrVxqF$i#@&bP7;u(#9AEDAnHR`p&i~t3~^C{p2fSu*3C)MLuY4HnX`pAs-&xHdP1`m}q-N^|U?N8HiB<-aTQ1S3|8R=Gq;`_|oS92Z}5kdjkDPSXR6LWMNwbu3jVVd!WL6a6F=WEpq-^`qnl z0LdA6t!qxf(+!XOG8h+xA+H<90#SlT_3~v_`?x&3Rz9nP*Ef=#c5YLT$Nc5v61|*q z|2LcLE1UKFTF5kHD4KgitP@FpZ#HbPv+NSPmft!mH5r!Ff&iGx_IU40Dl7pE95LPY zMK>3_`doN(&do_0te>6=xA(uT?7%FZuw{b0Vn!=(>_hID%*PRZF> zTwLO=pTA{u1r6EL;S9t*qH|M>*gN)BX!QV!BlNd@&pmA@H@*$SGhsuoE1UtZ3T7Lk z6RLm^3O~AQL$BjD^{i6=5bnu4p);~K2-fXDz#n74As_MJZraM_HvJowc@fb=_siEx zvSZT>(|DQ!65Z|$$yM!I+)NNEY3UQlWFysOrQW;R&`PhYAN2qH>}j|>RxsVK>LIIQ zzLi$SX|IJXUot)+0*`iK0;sbMsZ_WkgM+7hQDmeHsDq zKKR_pd37SBmA9NC**Q=3BXII5e_d^tz9J~%#KwbdoBQ?BW1^ua{-zef{FLz##)mFT z^-Cjz!M1XvnvQPCTf%%z^R>Qh`+VqaUJmGbmTCJSZ^iXQU`PYbF5hzzjO=yFw^#c; zA1A3~L7t~hW$Tjc%24o_i}u$DKGpDIec|Q1ngrsoLEVP=>kD&g^Qc~=l8fJ@VTVUt zU&y$g@N6aFaa$IwPwB#8BtRmHzc22nrA_qqnCUrj^Yo1CQ3)A28UEzbyrg*p<#+(k zf z%$KcSU!^xiPDR#qDLfHtU;aL) zv3i+zD_#Xa-@biOy>xLW6?mO`xbY{()Fafl`nWIi6|jQ84;FgxKuM`xPZT31+#1QLfpjpOJ{ybUCz(oR?&tqE>^1Pu z>8Hwv&dg{7Laxz5gLA{~w-Tp8*7}ux(L2sB{@2z(^LyO$YBkIm{LpAPLxA*!1Eq_L z3myfx)qnW^e(t*0+qKbNhWiBzY#sd{-exD?IKPs*l0X(yBf5TK zz^vDCK|^Ey+@*;)UD$_`aJ3#d#M%4L-~puqc_E9Us`(2CpG=^zMe> zvw}~Cqj_Dg?N?fUi~hWDVY;v*l2v5q*)!*BLSt8oHvCW1+`TJ29}?pK9`*Koaf`m$ zVrjort)}cY%r3)L2GUPW?yq_WeUUuxIQ-655J_88*}7R7lQqIwAOrKaWVGTN0A3teycVP#MF$NiFbJc&asaUL_h8?==6C;tr%%9DV3++F-n)M7HA5w(ZH^rh-|;tEi!6u9{DDp}~*TEfn}ky3An&b>whx zAr>Cg58s9B(;P+<%<}Jn-hL*qF3p-k?oq455O_5o>Lr`h23Y8{@x~C_gU7CIIk4bQilzlFN#|) z64nuq5wc5Rw5*}eEd8r99u*qhW+-E9yzCy6#Ax3%d_2!YPIC#`>nFN-pl~tes6fHUm$B2NR}63iVV3bt#l**T%qWcPG{M{_pj`muWW|hU@AreZ&YH zlWC{eQ7GY0S)!|OJh_D-9a+<_u_!PUZA|{i_WO;P?OUb9p!!Kmz-@l1Onby$8xDWC zFZ@1{K8F5G(s~450%p{;6@TYr>Atd9PXt*47W2CV~$NB&12u6n>=yuXrnn zBy!P@sV{M1n36ATi%VrOOpI6gyn1@)u`a{@2^GYBA&z;fJ+ZLB`k|2qJI8t*MxL!i z&ZicZbE*y7Kk_@rqGWeW^Z#ta@&9r4$_bI)KfZd{^fBa7Y%+D&Shm4z*==dB=cn~u zENGW91EeSyUWH0OY`ueYJ!JyRAe^$xY5@B3VXj_gIT zYo+^b+}^%S$A4th%giWDx?+7Hfdi`%s5JBUz9i`vmy}T2)*sw*K~G2#!=H<7~x z_XP)KVTz{e?yJ&z*xvo4n9Z@4(5iEVOA4mblAXrypIy47rm48>op|?lJL=U!{nrcY z2_I^q)R|rM=&YieCY>U&kBjJQC7#|Wz&^b}&FrciGCNtlkqd%sUPj}x?qQLjq_a#r z;2xfNKuJ#Op0H?vKX!;lxL=G|p9xoDaeVdgI}}C^tmi~%ps=`@m}4K)D`wieDsYPe zzBgRKB`nF#jJj&)Z<|^TaIr)apB4FVr=4#AG$_?-Q0gcnkPQ|ez70D@r5wiZd z1wWl)a>E3gR-6uWAEf+nJ@3-sBCM?ri@bTtV(Z$4*AyVdvu>7ejoypvUc-9gmGk?b z$_Wq-tmyzY$zwEG*D+MMbia7D(bsfau+zZx%hD^y3fj=(om%H{k-c&lHHCXsBB1_< z(3zu8!LxH6N6GKo@_iCll{!{O_DHM4_e+o3rUc7t#&r~~Tzca!ab+M)*lWcj+&W*t z<9)Hq825vY6GGv=mGGIk@U^o);AENz0T)%}@=U2+TQ}hpdW@4S@6gq|(GzblN|YHk z1jiaEmtO6#=*H@J_RNX>)QDhZ#tr?thh==UTPM$lkydCBH?BF`D9>9tmlAImW zl&`WhlSN@V;1q7m>Dhs7UohSg#Ca7SA735Ac=p}>C7et1(!6J0zIDfXMd9mCxgD#! zV>J!#5!qRN$c3Z?`VA4m*6Qi`qd0-yo%*;KHFh_KstAYH+qDwcC0yV2FQr{r8}EI_ zKu>6`j9yD?-{!Bi$6=s(xA*x7390pi>{zFX6kQD_gmkO(24PyyshVaKp@j?=q0h09!5dN0I&WXtkK$X=l(3@)2LnRg(#Eb~xe^eiMlm<#{K5QKCEX}2u_)jzs4 zov+d?6z%QV7jr-8n`bkc$n>(R-S{$l=OzosTBc;iTdK=S=wP)c%*3-uO(hnVbC-+H z1d;apw|PAQu8BN;f;Dmd$ftQa} z1QQLc%#2Vre9OCDzaDcrUWrclDc*))05WY1ec@B2J`VxF%${o%Kw zdg=d;CH{q7jI^Ok)aGcy?fYu01PE}mi_ZhtDimBLuMA|c zcR}Qb`$;fJb5u#)6YX$xAfBH_1~yxie}MxMf-=C+qJ0o5a*J1KU!~6fbT8WD^XJx} z;4KNVbL43CjJ&+_(8_L8jZ_s*Oj!FlR;!=Cdh3X8h0c<1y#M70nYFT8_^`y>62WLg z2`wM1@OTnbVf(%7op|>*t(!61{bGyQ^Dm<|3R}RGnF>-n;sM32H|d=xdqX|L30Cdj z!ov^a8QO}D;T-D%rlD$f|yp<%+yz*`G%gnQfuO7zsdIahX;mu42|Cx(b za*2r8<#^^Y2>cvT7TuQEVhk$szrj0xcrCj)<+zhsr5rA0`8*A1CV*Vx`KGDQJ3d>6 zJz*TqtMKrenW(-zcX#s0N99`dL62nN-l}VSS1RfwJgpHR+XNEs$Lr6sNluHEBUK(n!L;XO5%=ZTSqOLkl}@I z3Nm(te!-2*ySR}mITGC`5wvr+2J1c+UElx5UR#+saETEzb(`IeB9mVI5Cf5R3b}Q| zneYK9hq-#)B#0pCMo@|TAbzfK?0dA9w$MeLozfBJ;2vN^b zlZ8^w7m{*Iiq2{!Py6jr-+LX#zqjGjBMkDZa=_0-~}u2Tl~5DzKGVk2pYUa zc@N8q`d1^o+!rSWxl(;*M?bUl&R>!D(O{vH0&hWTl~F_aZgZtTR{dj_OCgwH1cL78 zIPjFNZEw?C99gPTQR;aD!;X(?f1;}J zw?`v5>)FhuBFAvjJkpmmi=Upg=?Zxe74t(N_~8>a zYUYhd7{=i)YGi#UM$h|-N8NPiN1B&_=g>v2 z^Sxsbp{6p`B??3zG(zn{BdGl#e&{S`M_tT$aGABs>937CQzBLUwKp9Mi_iSDyE5)f z-C*C&m5j!2T-x(;W!t|&5Ig=*U~V_8H_UL^wBuBJH*Q7!d}4HeoK%e0P54QoNgEgg z*0hW)m+LJ+Xo%}alNR8#AKp3>bH@)?`hY72qc9sF+7n`@(HOha?~Ida&GyiqIY=^{ z_J71qR45=naRp`T6IP!gF=sk>s=O{Xa&T#YpBedx|RzLosGTf0`rcLOfD*4-yWi&j&2M>EVXf+GXQ zyfvNudxM}HgRM=hKrT81IkDy>xCL;wMsu{_r&2~n-#?e0^-}I#7mY=U{Jb|j!N;YB z{1yRa*!RRJ%AyTOhWUAQicD;%Aai;DJ848Lp~%2vNMNp7OGjQWL_=ZU+#3fWKz6;J z7T@T$P5X(hA2z$w7-fIw2G@59F#(&maacXJ$k`Ar;<=gzw0(nGZ3XexaoLoi9LVY2 zdM?e2YqG-D>hv2(XMPByhF9);Yqnwr4FB?tx#4~|X9@l~gBY#czotymVc1u8zd^{| zh+lncnhFJwpL9p;lIh%`DJ!uK&BaOg?7{EM5y21|5OQoxz=)^TZvSuT7|tq@*=4QMB%-6voQCgqJA08=U+#=D|EnnfJ~T zaTF^n+uwNMiM6KiJ{)+Tm38LCy{KF6su{of6y`o6=uAJKTY+lfLX95m{uBMXr+tGw z=xjA3Sc1Tr+MeM`Uei0JKgv0UX{s_x({HbvK0LQ9T#<;y^68z0w`M9mCh78^`j<|x zen2%C7D=ucGoAnZi5L>xhmWaXd+*AVF6AI?M(YirN!8|6NT7SY<10kB zq?nugZ3}lAmdzcrj(|h7|9HA@>A^*sf3 zg-7Uec&!N2jCccSv<#hBJ3i4J?jzrl2I-aMzI>j%(b5b-b}i z=2>?MQFv*GQH3+3~xfO(7+3 zF32)oqoOOo(syVe z4d`9#jldfbYo%yWjWNBc%-gOV^QIfkcnYzmr}R!J{QaI;DCx^jmAz+Sj8;6dysuqY zew=`Obf(>??uW&;sKVWV&Ar)4ueIJBpJgLdpiHfI2Aq3pPsDnZGF#5-JVTm(C?s<@ zBUoKlo?1&n*f{u=N0!SMOa6}@14w-oxNW9?d{yV=B}!hn$s70RuDcA!;h$rsvL|QR zIRozwCuX%dgLst@E5=yAymDu{| z&-aMyqjg9DP)B%-e_pzA^CnR206%H;hm{(AA*871M?2@T{5i&;4{mYF=R#AjD`ZDN z#PJMQQ>g5SmoeXNPra}l8_jhVagHo!ry=K$+oW@Wb_t7Ycz5ON*>GFiQVn{=tbk{S zHhj+C#2li#Y}fp+I{XI|NP*fLq_^uZUBerEl!{758AJ?i8eZ?}ylZX|bL6L6hsaD& zaImC2tmu1BM})wyBhSjh^XBnSwTcQjl6T@twKpKn7tD|MOm#{n}hmw_{u(c)CNgI|-|^VXXH!(}QfdJh&Ns6D~Nj5ob}1 z%}5*(HwgkiBL5k)Gr7ooku)Lm)^^4yM%Bj~g9EQ&)-L zI@NEiPE)yWc6;5khs9||`epz!s%}00lZ6;t*!}9xH>R(yDK~nF-W5L&|5`cm8%o*x zu-zH5e2pI&@@|QU@Bi=sxxlqSz)X|Q28+61h<%!+-y?@8iu0O5_S7wLwkORPshv0$ zV!Z`U+O@$rjkZrT9kV9$#9+iWw)q<9$S>-&>w#Xp{a}z8pBxdGP06{AthwTk=Oeu}hqvQ%rR~6<^e?v)( zV$Gv^n!LCB3L=cry0zD*{>SyD;*Q6^KU*FeEqr(L4+NmaS%%eTi6fbvi_N7*;+r+P zZ9*eo`A(0d2#_Wx23@daf}MK=Tr@>Twl#B3g1iTNO@1i+`46dx`6Jo43EJS82h z5P|-MqRoVa0uH-H*RadhB3d{HSM}1h4*Lffq&?3Iid0)$tERtvJwk=OVm=_%N|E71 zzc6)%OB$d@t+PO7>fMRC3_a>|*@rJ2Dl04yPUr!3`ER7td&E)&+oDVSq5t$FYsq=dOauR*%3{$_+Jh;H~HK21-Cyw!P&1Nur@NJ&*7yg#_?*ADfjdTLWY1UF< zH!uay1T!ZLZYmG``US9JxAbQUGS+&mfMvg8!B3j};?MyPWiM>li1n<>$vM{%&eSnl zY{YBSem?6(j(B%I|M#;f>{Ioq=Hu$}0-a3`Q27X zO`ikX%jLfgI+$p%U_)yN^FPVXZ^IRG{P*GzP^|UbbundW@fs^2L zg(3~4g?o`>(@T~T9s_-|S8{*(pTv1T{6u|LWBj#!`J+UmwJ6_y$b>T{qvGYkue~t? zT}v{~MypYHqS{KYE#mRfe0AZ}VgfLWzXj?PmOdEQ5;dyRncrGoBgxigsq~Kf0zear zY4jv8yzoy_qWdGZ&Bl1gh(3v@c3=qxWgVn)SNDY>4Y*+MTnv5W9dqw$p5N;8KZVW} zu83=@XC?%eH>7N_8B1T6cj5;ndHq(!uOjVKmt96KS6qV!eAWRv7^V{|LIrf8V?&IH z5D{1qg>T#~PQFPS1~1uTMujqR%CbYbN=rD1`+BgP5sbk_#)Z|SEV)#Mzv$B%O` zE>B<6SP{^dq(3m@IWC&=n^I>(Jo$r!mNW|Mfe`r+2%?Qw=b6 zK!jW5Vs9~~dJc2PnBF~|V(awtqB z0J1sc#;2DKmTNscvH(TO<{LEHSRgIPzTXbON<|Z$!+oL;o?i`2~hqv#g4AZ(P zqe3Mi+KhEp31Ix>5}&e|^ZfqVkkHO|P7Ty|tFUc(sQOc{SLd7ee8ETM70jitJ>oSq zZ00{O=DKWKc_mjHcZBviBl%cC?zS*#1g{H;c%-2Ii6D!^RxSQ4R)c zwdsmS8<FdUJd9(Cfx2md?7#H7=fFq4HrnA5A+Ljx-xWlG>vdWRytnZqCKmbu* zT2+;X+n1QwmcRf#*cIs1{u7`yt+~kS>96U2YWCwT0NCfUdsop#B7jXI-=8mY{Wjzh z+A@F7E3LnmPWZH=SMt@Xs71QxjI-{Acgr=1Mu5Fhsh4~Gvfrm4dYb;m+-nXo(R4zL zZe9u%$c&3^?&X3L&Q!s#!MH_QtbWO^YUycR%3);Blt6y--uvS2-%2knYtUD7f0WL! zs*&P!)xCRmxZ~R>?5@-d`Qq~;z>VL)loZP19jLEg+}bMr%H!N7nGmzwnuJ6L2dPO7 zEn;hP(o*X>z71U^)XiMhC8_@G(u*2hK_=Y%%24DgWTH^hIe$+lpE+w?sH*g5T0R{w zckYCcL;We8XIBM2A)`!td(QmuKx%0x&>0vcj%cSh@~399vHk5^>GCto&4JIt6CVi( zzf`wbjrLXH&{d{q9={7hQ%7EAmeO=vKDbtPEcgAF%M#^p%OZBDIT%l7L|aXyD`&J0 z92g-{eq*-8`oiP;QQxXN?=jJlrQh1b5t$X7t0ku_NxQb{AdxM8%Zi@l%+;N?-5o2; zI9JJ*qM}hSm$e6^X{1?joE`pGNh4i@XA#xJPalqlBm0awi2XN2 zqwk0m(!>jG9$!YB@6LG53#TlvIh|`&fjX-oD|4|zCh5(CM9e-6ejdN_d%YF-8pZsg zdOEl4iLWCSiYrQFUQDdAMVP zi;#|Bb(MCX1K1+ZhIG=!|MAxGNMR&Hn)Mn5m+f4M=)C+Zr{)O*C5pG$Ywnf-b`>J7 zqxrrSB9_|A^(C}cKnS|z`O)iZMfgs@2cc@mtLLAYL@{QeTC7A9Csl9WE`R~cv&)H* zzcR^!FI08`^MN_qRa*v}YUcz#NkP~g)Z>j<`%nj>d_9g`PZL!283o{5+kMRhc!6rJ z0;#S`TL=tuN7XOidd~Qoy`b_`#iRo%BXW)R{8j!g^!B*`mVJM23d@mC=RzQQ#8?dJ zfoq@sc9$L#&bTr@FP6qTx({!JI}TBoKbZMbC*fESDIzj)D)_~_1%m!5L*Pa+%TKs( z3?-ORzo-~`C8hExzJ@+AXNzho6@U))07t_dt3el!4|@NcZL|&-fW5d_W~keP1f3g= zA+`CGe2_H92S)Va4!ckA?Jm~d=_$r5E#w0-YWLYKNxVLtW|}Ab%V)x%LTGb~Tfn_+ zK*Na2zPIxpvx+QH;RusT{AFOaoNjXw*kv>t$r!@{^MXi`@8p_5}m2 z%YUArI+&Oo8yI=6(G6sz9|YG|o@W?$pT2xqr*b*_Av!Qu4cY$wYRn*KMovyCxqzXx zc15pm7|PtkA-2ZTqOhpgX2jcOsQA_&iwPo5S8>2XZT3y<;IoL{wnGg&#Rqj#-Z7T> zm|B`!i%tz4?$=*2-5$Jq!^CpRICWkS*aF6gJsKnM6JiBpt!r5y z21Ud)E<^;)kp4?cqS#IEJNkC_A02A{O?eJV1LZGt*>iV2)p7!psGd?0J-SXcUoC&( zSGD}cqrsSugUW+?ERo$*6gnXv2ahfrBVZKfRWiM>@ajB+@Fl6Y=kZIxDtae`l9*Xm zsbKTM6z%&H*4To)yjDE_0hT#*#qtueMX&lTsM?myD+_5t8u{Vn?AO^trx2ruO%(59 zN5~C>(PvvH=-!Q197UxZ2KM}H{5-PPWbWFOS7C^A%>*B`jjRI+qN;jFBVtVdf=1o6 zzj-t$t$vVrAV843jQ)>I5KX_~cm=GApa@XE@PU4)^{Y%ucI^d3(R^fD__I$SXFHQxZ25!D{TJX|3jSwuaplg3g3X*ZdE|abv zac0XRSUO@B1Wx+K2k@D>DeC}j&bDJ*+?&4!_TDi&9gC~l4O47uV_2XhHRaI^4}&Wj zPZG-r33J@u93O~v82RwJlR;N&RcY^=t>G`xyCA1UOdDyKEnYUJTG|iRv!T&L4?fxjX z5NV-*x8dDsghZ_j4h>1EeTOSl((2~#_L!Ld05 zpczTt5x$Pt!KN*}5=W%`-^n9>!OIa>>0(_ABj&MQD<$vSo*wCwUtvfzp6s~2{)Rc? zfKf$Hwhd{~r<{)&+W;I-2`#L8^``k{8!6icv05_zx&iwzaLGP|d4tKDZiagoYCwOo0!7-WxJjvq?r0CJ@R$oQjz58ms)v_7y<(S1>NYYcVrxBaoqo;jVkHY z2zuZ9*M`(TEFZv2eN0&XJ#14V;x>iz&#({kC%&3r0~{X$*M8IU2AG>ytRRpM`BEAy zo#Nu+SY9LIB+u<9z&(+|-}x*o)z3rLVUNt0C<^Hbpor3$@yk=ii1;?cF{yFdLV1T4r3YG zM66?tRCFe>1gzL(;rgdWtl>HetK^*bL~#mgtO4}ZB`zjy9ek$4uczOH!o})gHolIy z7)>8$P(g^J*lL3B1qCt+qyrP>!m2ud5T-@2*nQbAuzmoPe;1$r zswpW`Sd&b#=k=EVPFI>X@+qrX)%4yUNO}Imr`=xWq8+Ma zB@Wea3{eh7tQH4_0d(Es^P~dNb2~HE0R#Q^@8Nq*Vpi!PzmKU&dhq3=FnY!04Ol5h z4&K`q{4yHK#rkP>MAZaV>Z#G2x2Be79=EG8SW7qM{O+L#5}=T9s+Km?y`a=@IG9V~ zQ5EN7d;7Z=VR_Vy#`DLgVQ+%25s;ed_orUMaNM;ahW9Qb=Gc^fA*WA5W}@Qg5&$Un z-{avF;q@bxkHFmw0gBXTYqc^EUH6?m>0f+>b8WM8a$n*HFdOQ~Q`Yn7(RY-Z0*Z>Z$kx;h=Fp4!i|u_wsTd=nONri$AnEcPAW1RybTIMlOz7SedvxUMTJim6TW(S8ZDm~{Y3zJUnvC?z`&Kk}D3 zioW6-v$u=JVNd3E)4{c+sW|l}Ba>NFbA_wWrxfwlB(m2HR7t!SUI97l_!1N-?V&LR zdU8847{_~JeMH0#L+Y~e!w=1jsB6wyMfRE8EC`jzA+fH1#o2>`9yWCprv0lb(n%Ur z24()2zNClXCDU7XSiWprqNJey8aUt1abZyT#pq<2iX{SLRa;tn2wTS?2vV0GGGxk}E~7RE$>Jfh2k-wBSH#=Ym~u)+-0t#N;; zF$ZNQ-KWZ4pkTw2a9Uyb`vIa@dMp2$yK$b+PdvHO`hylbW9H%6d-UwtvsogKl(w}$ zLx)nx59>eJbANk(*N6BLl7! znCj1Q#W3V`_@Fg>VPnNyk*m#feEt$BE2G=f1|Y7t8lQ`g^7>nT93jSh_Cd$jHGHle zANDbMgR?Fvo3I0@8UFTNJ6aocx6-wVspYIrBHGX08@+gx zvUd2>v`xX03U4>s?r5)>a0mlXlZ*0n=}g1Y7=f8udQQ$Io8tU@>PeT0b5>KDE8gg( zc*nj`_1A5oG-)LzB@=h+VK+skwnP{9n>t5xy_a+4$}KduIJGGVRj3z|CL>a~;!b{E z#x3gZDz(ny z8f-VU;5q|VCnE|7)uw4PfBauh(8&((-v~-i#QZJ9)lQlE)Bw3=+S~3j{Pkocu6-yb zbrc)VE<7nC1PTlCJwAC%A~7*L{Spy-JRk94f4_e9rw9o$Dn_)FNT$NA@G2tA>UJBD z0amxB;rR4;5Ae?Y)m{-hGwW<@U73MBpq?F56a!DLdo$JcOR9P;sl{)}f>`k^6Uf;?nyszLVz{H4U|y7VN-hbIWw z7W~~1(Xl~MMJ-fOa4o2-i47y}TE?oI{ zQ^gXC`Ge)1C47Vt-iy_37KTHzKi@mVRMfhjc~p~IH=LWxJ3G}e^5wT|+N{?_#`IRz z@Nk!y3*9D1nCWN$Ih+kIerf3?f`25KfQq6}L%OA*DY*-#2z@BXCl~8x5?W;%Mqhuj zz+rp9_6ida{OnU4=@FXZOeA-M8Xx2I-0=}HD0N~Q0f!#ms3F>pP!r>&ql_zZxyGh3 z&)9P6b!v8LQnBQ_o4>Ki3WNF?DPgv*mx=E39072GF@(T`U&}*E6CORKV0ZCv4hAS==E$ zg_*jurZ-oYXg(6RHD=47W~bt=X#Mbc)Pk$H3aBHe;~tKu9X{^3Ly^0t!SFAGlk_FJ zqi0P25QM%s21&D=4^?7Ox!$sRjxc1x$}5qw|53V+|M{I?EvXSt4uvnX6tXN(nk@o4 z<%D(Y7R#5}M|WC-K9rK{7pud*kB7Rpu`8yl1|?m z)#;7cSgrU=Z^4lBtK&+4dRJaWbuTLd)BV+LkY*#i8JI=$B13sRs!bvKOumr|Pi|v}4 z8~|ui>e|j@{&W++iN;(f{vs2f@S+rI-0TE!h^>NdBEfR@$By!RqYms{-7w2BWJSk43<7 z2R1IUp^kV01Y~S?HFfVX(t-ht+z{YNg0&m>@r?~IkARaDMLjsbKYUKG`tJbi&Y-D?}?fVaRl8VWXp$09#uTc^JqYuzm^JZp$=mXO;a(G^Sf3mPh zHmEMZ{7OmA@p&oC^L%Oa+VvMZmLgnVJ&bL(NfC z{$BD;m18;G&iS(m>QOT`+ZIb#jt&@V$}&-qS1E^NJQX7=WPbz@HHN~$@AfTdRwDXN z?)^5m7mU^qyMx0Vz40$y<OAN6@czN;c$qq4gX01?&g*2Jg0U>w3eg^Sc z_PpouGJ~Y`&esVN185v;>X_i{s(e&6y0dr|BGdtlxN7q&Z+m}8|NNl-eSs2emmm)i z0}ToB`uk}<*wNw0O%)`~*3RIQ99H4|R}6Y_K#YqYwmk-7V&m%{cOaY4^tA$S_6B6y ziv0;5j{)}mJq^tMRpQyrLuGtJo-!|f+}|s}Vr+MwSY9CjT+E0*Wg`GhsDf&`Q0%$q zf8$-x#Zhy(ZmCzE>W!5k!YI1t5CQXhYwt}i{=qoz zV3mXr2Bg~qz&MJ`X=C69630B6RWf7qn3aS?UZJfkP{y!Div;1cmJHOvwzm~z`i;@% zK?-LB%quX8*Dk_3K3 zfr8Eb8&#TR&|2_!)%a?HlUVPRx(rQ@Wsi4p%t$4^dxi&?6(r>3gN6 zxMk-9PA!{9PoJ8#4@5w!H7PM{C|8aEVL2T2`;(SOMP0I}5Rr2BnRtFx{ki7!SbomS zz^ajgDT1@}<2-excZnjoQ(G()n}9SoayxuC$_C~K4aq8(Y;;KFcrq#AR7EA zv;=R-?KkU)xc+{BDV_x*uDjU^WFzO~6wgS9H2dRwR9oG+b^p#4zm}y|SD%?T)=Z2| zAW;T&2fe;4zqFB=3lFTH8x-#mZY%u&>;%5N$S@rzgUQi-r zulCmjwEk=4@F>ozNR@scZ5B|J08eyVV)8Y#s{p{e_9+C9q{MSqC9C9Xov*QlLoieY ztY&h_GXe0tJXVjQRH&g^*E!^d#|BRd-ao7g!zReC;ig64VKEcTZ%Hi&PGMR+yPc=j zP2hSP$ioDs z*~U4gq=*7^{U(!*wXYWdfdkNsvla|gHuSH*=nOHU@9;UCk58m{A?C=)5C9M-=u^@P z4k;O?D@}>+K5k9J?cad^Co^TJzS-T>L+N`8>)CI|^raejrzbc;J-jxybbGI{bDHteFp}()_BX z-r|4UZ&j1W^rm>G3Pk1o93=Drq60ItjqSd;CWd*(>~t3SCx558{?!ms^am_sD+2yy zSmbh~t2!-U{AgCTln9kOZeXu{<6bf4E=7Q}|JWS^TDtr*f=#!S^g8>#r&yMpCcDA% zCFN^i=b>;w=tEeVF?-GT(m5jH3v~XeQh{Ie8oV!twk_+#q;^U#z(V%9D^?1vxrf4` zp@IEoqJ@i4JHoX+@1IvddehjB1Mj{Q$4$M&NdHCX^Nkzu zxhT9W7-oZeD1!ws-}E|%L_c|OUX>6|c3V3p!o#@CI8a&mMN`d8{=jq@dyeqJ3$3WOgBp4TA(;KI6z^y*{&r(du3Pu6PU|fM*F2_xa*B+pm&% zaYcS)Y3HoCPaudxTf<(Gh3&p_37Sxi(q9x*Y0WCntsfhuoh@weZZuu{z|5;&x^&CR zsytQFcIQf_?Ope%NRsw5KhyJ@f?_lA{m>}@;9)h0(@fw83dVuoJxMo(jJ#Njm$8?t3 zS;@I8qMmn!HP;n(wF5}}ZGG{4EjwqHl(}SLukkHF3XGim^qF9TTcEeWWUkcon_HQ$ zs!aT3M`Mhyin9I0&cL-jdnPBL8i`PI%QN*%Yh|R!jO!{-?K<$~8PxaXJwY&45IYpr zdGg8-kAwaraEn*J4i?Q*f<+U*nj!376%1hk@TR!`_~=36gc_YIBSv3uRbfTZ^=V)o zLVSxNGVY2mW;#A_Wnds+M9Wm#zL-ltp^#Q9D4BQK3v)3~SLYl;{aYTbZQY{(i*clv zTrn-MWGs1*do;H>tnhIC(ULZ-GYfU-=1=x}KyD=T8jgve2*%7L2i9yHGVa|Q>F_1o z6|_s4wmiBX{;o02CpPGE;RaR0&M3+;<$TOM)R%9D26@{DhQ1krazr#@FAk@(=>D*( z8>ZMfi(}1fMxX(C2D~FXKElHc0!KOCut_#tM=p#2q0O~#uuC6rqR3+U^+OV@6NxnKdmZir25tSlbMbN>6V6{+p5zdxUqj4 zNrWXN6UE)Flz7A*5rRiDZm#ROz$mB8xk7vb*hU3<>c4sD!-fT7Jta;z7LfGd%f zh&{BXO`*~qVInSg3U#7&Ez1Ue9ub^4AsY9JK>>||#OE`YOL?~+H-&8M>Va1Uzd`cq z^qv^85Ie9c@p45yPOTnW2jE4>i(pPdWI37vxoEI((Ko{;s8S)l-0M-=DR2 zj6(uqLg#<#TIK%h=nT3o*mV4o_Fn@y^P(jtay~w!u~@W8?Lux3?T^#^s?!x z2j#X>L9FDrSgL8eykA`c`&Ulkz(xP{BG$D2(fP#~POZ0I*uA;c+``Ao%F5x64%OOb z#eyE%&-Gx{P-rqo6<#-*->e?UVjpfAweW?(*Plxl-ge1%WLcI_FsA!=F8V==NIq}y zJK-=B0KaFLM0BJ9rt~+2pYaet?-`c7O1$SI}Bn(7R+ocnx;%Km!9F6jFI> z>d~$`S9$;+RicN+U?VFf{LS!N4-aoqfswiFtOcy&X&(&`3PNWRAuy+>5h>@$YYPFy zy5VKaudtER{0T#bC$T5E#Pse%9331T%|TTf-;?@wb}CstP=b20acPvBo#_Jlog^}N zMlUd8l8cM4P|&(N<8-@qI4I`BXflmDzNg`bk7Yxywpcum;rP4kJFO_BE=o~`jmOS{ z4pz_+BEY!)uAX1ykH?L>y!e9+#`JS=&_RFpHucBp+C*?6fkY6?NoNjsGXMA!0&<>!{aOOADfkrC#gC_9FFz2>;LIfKU!7OvYUP2 zRngnh{L1ZbKZV;y)ept&S0Ap99}TO0R6{|QSdbX@J7C{4!SHXLlN$Q4x1B4;qq!`X zVSvd1DU{&v{yYR)Ljw5y+JMFi#dje61#E&0NfXy$lMwtK1;<0D&8PxwWuP$$_bNh6 zNQj5%=+Q&CY|1hLR;mtcTzw0s0LXqI%Q(eusy*52`|Yj{n)2Go_Gv4Vnvqygy?JrC zv6`KI!Hwn<9C{2p?4?z~aqz)bB!+6agld`WiR^2|u($q*fM1ahwVrD2D{G$P?E83& zKK1CF@5P5?r=BE;*punmd$zh-b;fXXR%3oyU@(E-$HQKk8>21CexVNB)>`hBCLeTZ z)!_e`=PI78$6;F_XWLNZ(>0N;q4=d&!YwW7uyA*LA1ihaLx!8OT1;Tk@dmUHXW}R2 zI<90XCfvq3Yfp#pm$)PAb)Q`8*6pF45a%Wm6N`=La-);0ck{n;sU;te?Ru=m+nVj< z)X~u{iGH=Q&o00%_i7d7YbEkkk2xUrw&h<9K&sH-Nh~EBWYEs7Ii=95u)7Vb=y^)hDie z+Vc7doA}g_bIIbkgXJ!o2v9D!T0`X^Xcj4uev|-*sHJjyM)Hv zo}^;lvw)mt>Rou=}k(X}hkmA&DWu0SFxfjE+g<_WE;dcVtEc%ChzDbi7WRHx?xLAx> z1`du1yLooq*~>C~nM$oEp|iVuK1n6fvj?RxmoUO}UT141;IofvViLAVYy7a;`_Y@0 z3dfHNf5%lTvR;&O7rxwC95Oo8YXALNLhnn629c(7Z@k5fMA`GD9(#*1857Gfna{?> zVRhX4KIl^^+2RMHiFHbs|EZ^j>QAOiALd*VC6N@>bAFogK34gWy%T?2v(Z16O ze`(XCZ{neCeIyOu#kyvbUunEcG$QI#lL!2sHhl82@LejTN5;?DO*nO63J2wIH5%L} zXp?~IK<|umIj1zYvJXDH8Yc>qUGO2hn-F@Er9l);aJ(TgvN~Enhy4}p7*e)fN~&5> zX(jKRoL%9KQ*iLu2+^!kuB^8gzN{O+v>-A-L5lca9}hm$2H!i_W%*xUXuJd&zOn;1 z>0ZWF6*GSVQ{Nx0*7uw=s?@p<6t28AaG}3uJk?KdjQK=XYJ_O~x)V3%zAchx(!nzY zID?$bBo7ODlsX6X(8x$Gxba|e;2=C8Q8{m!FDd3@_Gh+8Whw{B|0fP9@RW{z@|d*2 ztKmm*$qWt6baF!dBD-AqpTm`dHX zutx6q9PfAkWXFWdJvKM26^MyKI_drxBPSEr@H|H8sofVodcA67O#7CaTQ{$ht45wH z2K5fL^6vbfr2I&ncCocyO#N+yP9<@p=Z3|^k?0rd}HN>o{W02nYKcqPL;84w7?R3MOVz8bpg^T z(|b2s?cP>bG1ld7_KLDiSfCJKbsHm0MY1j2GA=MA&k>S@e9h+!w`FyE#7&nFwJe?C6~t)(1}B zTdxX_XYN0E=xZ3hN)z)_&&}nyL7F9%P-|jnPa(!5jL2C1{K2+PPOL`C$HUtD>CX%8 zY+oP+%2r2DY<2@l7P=kpWVWV_2c$bxTANQOa0sW=UDX`2uT=BuXep_&+NP;ig%`y4 zODS>+Dbg%lv=}Q)j$xY=88vl2p#RCaV?MsteAef@r&#F)E9Qf zR5e$Jy5fjFi&T7Hb2Hz?_~1(F4~Ir>JQ0{4O$lVG=gr^sSu<)FcD=%dqC+rZzW z9VS`vk%_BAJG~WI>FF{$Iuy~LJj!1^$u{Zb5>2zV9&?a~CP!T+oOsbmQ}yN2z+_dx zXFk;6Vc2P-Y9HkC6>Fnw)M}ke{N-r$OFbEB!s1G8@tW<-16A>~2Pda6SBB!r2#?>4 zk)hWVD@9mhW|)0aW#D4zIgAygCih$+vOYQ%`S|s%V<-Mtp&vLzpTJE$wTgOYc1`$N z;7GxPRGCNRLToudo*?y%%PdW?0_EA(Xnmi?73Zv!UxkNl#wFvuTpi)!J5et69eb~antix{PGI}=P))C>hJt-(H{?C@sb8odYw}& zgc!TXZkJohFQe)hI`ENiSRzHtGAyeuT`NMr0sb79N0zcm4%WuU-9%HRCdQrZENlF9 zD1=1$iOz}T#H#WXjGSt&Q#x*7_l{dD_1W-6sSd~bJa)eFQG7fqSNZwjq0|35y%2~V z#)f@ZME(E!kc}{){j4&pEEOij%|K?+Qd~r75_sxv>xZ(GzI0<_#;Ovl4~h>yo+8WL zPM~&{{%s;d@cPO1Uh@c_^Mp?oOcE0l@5JO66ufkFEH&x=d^4Q2l(6hEaXaSx%?MdD zoR+n4b*73}qJg`k4jf6@E(R+aKJ#v%Yd+)^M-w ze=!A{|W{e{8_m=;jwVCR~G#63L_3gzPRi5s` zRic|E>hDT(MBs%SPGk_Su8^ziXbA~0ayX>lUaWc{l(`wz`n_e{rPHROlc#5+G>X%(IQC1w+4!p5 z4NA>YllR}O#=cx;e`z*d>u&m2lQI1KSIz!LuZ+Egty!lY*RSo_*vxy3~Lc{+d}K8UZFU5jQkA-mX$d=h!Hj zYoeyGOZvDq9l7a|*x|#DmaXrv9TKyXpZ6r|u5ChAsg}J%X$!L5D1<{+P zzb^$PyW>>83OR<{=iy0Fy!}<5Co4^coQ?MUEhd?~fRCAOqq|!C71{rXvG;&#GHcsL zaU69-gm)AWrKuoDZ_=BJNE1+cN2T`~Is``r6{YtoQbG$z4TKUMr1ugcgb=9#LXi@B zlCv}Kcg}y-=bW`pR%X`lz+y=<_;^WAQeywzddm$^q9p9uF@o`_kx+ zCHO5*xe>=q@N1K=Gjr}|S+7>RJj>T1#=Y)pxmZ5%Df93aICy;d{*>x`aIc*qUk- zJ+7|HPOX_p+dRHedJ|5v5RNJy5FEdgG^B-5ZV;Mu^0KnXuj!NmVC(U+L20xl%jy4B z9pKB$7g0yQmJJ&ZV&IdIV0fnG4=|!91oBzTF@X0vT(Ct3p1x!^e8av~FwdGZ;mxE3 z-P6&M?a6^y@n^dv(uadEbMuVMtSt2Obj97LVP~gO3Z0cCJh4+hvAB?J)I>S4dl>U~ z7Gd-dF)&;d*q;Iboor?BqRDu^2IkQ03zWi6^zAEWJ$2{-+a?janzwmK}e-&VRV#{C!?^z^$VYgRsZ zs}=T__y$^Q0=jpYDD65H)bAS-1a_jEsFqtqty*~-wHS&0OD(rgV9R+@gRuM;Q~x8E z%g>7Kf;{>E6#^N6jZqhqSZp--R>gk$bF4RT1!%9elp z_$AqDe8Y>VeH6q_3rp>6Oew*KvBtW3bdsS6#n^7j=KZsXgR{Q0sAj_fYW{IL2kI?4b@~W0k z@S#nNhpi=FG&XlTpk{!b$pgcK57OQGvy$hV-TC#P#g7540h*@N$x9;SVc{{N9>Qq@ zTVjmI$7ff8W;gDn%fa_Jr+7i@T)WJ6wIpjyX>+qB4u=!KxYSNAYz65BH{v#K^y%4n zBC@E5=g+>p3P%+csqfE(R{b9sE>uK{)i^7eDP$PSy_SG;NJ4#4e%?gUe1tk_P8;_r zk!@Pb&dQSH^t%5x>NyPto7+)J9Bdn*NitPaY1w(G8a=(Z!#wUE_TXjJt*l%o*ZFz5 z^!B z)pt@DSutIX-`M1A{mG1=!VAhWEtOoGaBhsmKx&hyysgn?;GI0W-M&I(pjFf@!WApS zA^A<-x>-asI=E0j@s5-F2SLZ&!+pcF)X>Em-Rw<^UzR2vUG`Hc#a}Pqm62jl^SfZ> z+V8Vm#B&esb-`_6mhj?jSA5;)?xGlNWd@m7T169^@#y%o&grLqRG5GG8o=^WwDHGp z>No%7R0%V4aBy(-^fYmCDP?G%{`#t5tDk!}_h8~09O@qEJM&!n4*7&0Eh3tt_~UV) zr6$(a^-157-Bj(~8?M7Nj6r4SerEeoPo7Ev#ap*Gs(e9;$$KY@6AuuE`2^y;BlGU$OzeGuOhEE(tq6q8<$3Sqt5J3eteaKu_OnA`dR?fIra#y7FBnXGL371kKW1MGdh(el4{x%qy*m|M}(m7v{wp^SbAm`WNV}LbkjgDQe3-h$&?c&m+S1w>Bw5+bwuAY>Hb)WXW!jr;ARqAXNNv8@=97zL8EU$-huZ7 zL-Q@6yYN1vNh^QtroPQ=m@E{bfja8#13<;|YU;>-&dmG`<)1}Y#<^p)5bm$eD zsRa&dPH*tsJ7wfypAI_cUdQLfbI*M|poft%O4Xs?2~r*yV)X7&RRcf4udn5(z{|RL zyZA4|UK;u`35UMSrEc*lAHPHxRN_CgWkR%yQX`yGU0GB4&ap&P#i}WX`~c?}Xp0gH z!T)ID6Y-i6s&I{gQr6$Ns(*!8rfOXQe&`6}#;q-iT6eor>gnSukKKc;YE7zWy9NU^H)X}^5;+@l=5t!?;n4oFj5FSxQp+rC+kkayUp+(4A zK(i9pkN^_`t2`%}wZ4nF^w-y?3Y6qz0Ag1XG`L2<6e|b`ZCYMew_E2BdOQyc_xwa; zVsc*RQLNuf_&T4*j}eH*7Hf(Zdj~$)#0~iBv?596S8^KGbNVlxcj*#3S3jTfGV$y& ziziC0dN~sGw;lK~ry6-Ul9VTE^s(FSq16v&{NWd##KOmcM%EfHwZG}FRhfo?K9H-~ zd}Od!bKUFzps!e>gcLJ9fY;zmljGp>wSw;dN&h-Z7|bP}Po}IjT2&gxu56uL;uKe8 zrhD{E$Ei6FPE=> zpvTGJjIIoS1sxT%L|>#;qL$}Wd)f9RM5{?-ab}HUm{rplbn>AgM*~0=H1`ULZD0FC z^WL4@$y1(rfQ{Ead3M<(OP%|8!#72ig~2j_VhmZ0`?gDjTEHLBH$_l$AgdlsOcNg5 z70r$jJ)a#oJ+nNIj*DpY2tZ-CwDh6S)+JkO{+N!IQ+JLP{yriHPTV4dpAp583(AvC zH&oXZ!0)dWEgh}UDeL5!`;Y!f*VIBC&H;e<=KCFxCbeF34_52LNp<=FjonT)MT1J{ zq(+uQU~Git%N8&unbOK~G_xL8+FGQMxVw7Vt2{!>ycgShR-d|mmK3eeP#lKEh>ou5 zlt+sw&VSMl|7j~aqOXF`PP{YBjoCys>j7Eeolljb^?l_($-6_jXhouB5}-N#a!tZq z?>$5)=K5ZE=TAIDel<1_go-L=$=G5{(%h{S>;{YO0eeJW`ZqBbe4FUHHfr8aC}$^b+lOwMO_mn5``( z>3-N*#6sIcSZ;6ID@?tzGZT>%?i3)$Y1*Q0hPT_Y-!2_OA5%7dGaeG+s(SO~4ddoA zTfpI8ZmabR#hpHSNh+=Fb^kYHD>E)1$6#bPGGe)u%S>&89lp)`$apK{z&K*`Rk)ZC z{eY}+t+8hQLkb!x(8keiM!en_PbrD_gHg|?rs{RUzsHclAJQ%!?NSaPeQM#g91 zP5I3a>SxEp@5s5EdsP~mMtCNqPN%n$=O4)d$|ff|das-~Brvv^m#s>95%1Gc-((T7 zKWS@eZ8MvK_vCE7{>$>DqZ@l4!7ujf_4nqlF_lcx3(GmV|EyaLwbKPvzrTO^9AOOD zX=5VbSr}^P>8@hp=Z6i?bwVTU=+fO+PlI;i`k9dz4?0&pv6!O32?QS@ZF;-}?6-en z%zf<&W`dlf8QtY&o;T&%-D^E;R_-~x;FVVdTG|3fBn4SJwHK4$a@ zYFLXI+Jt&%9ko|4oLq$ZwQcv3ix;oz_NCe>w2*^*BUa8il{Sh7euJ2Rw}Xb*@;98d zv7Z$UQAIEM2*bCaN~TQYd7;F@oGNYurQG-qN%=ub&>v52Pq!_c^v4KEACIp7iOKMp zy#qkU#FFt#A`KA5axXuj)EVxVKbpu91t&21KuVw3UCF4G_)NmT9iLt9=arF=u%2;C z@{yuDTNL>4A@JPZ*)c5R2WymrQHYHJ@Hm3Z^tWL82w`S=frRd9pQmyo&PL#AZEZb| z#3k@vI^Vk0$qst2m{c*_%HW-)TUAhSk<>ZI$FC$ckcL^_VVe2qA;vjO(e+ShsImlq zh`JiN#*CrqwBBNBh;eg?(LkS@k$m&;T6gi*V$*I19%bu=3c4GLcQ+Ot!F1V*nI zgO`mZ#8(Z&SDQLq0fCZn#Z))bZXg3FL;uJ;S_YmOrnU#`$GUoVw!WR41D(o~P#r`- zm9lTIAA8<(K}}~EiwU7H+}8JK4aiwl9kP0uzukznM#$;&kec{ zQe(vq3IS{BCU6UfslDiVe}%=ZpoM^D@Nd1~Y|jj|t1@x@$UXH=;W@f&fxMi6AVKHB zKv!VkXl_0ph?Tq%e#DP)J_e_a)B2yBGd@;;!e#+%{YO>N=q^#P?!ksU?yZ-xWNpk3 z>+%d(I6pAkP+B6RaiOvj6O0&)PoZ$>6{tKl^=lhj`*laAqkEWeglqYA$&2$vJu4CY z@feP;L@0g&U?@+aU{d}M$r{_r_D>FrHUs?-d(z#8FJ@;rOi~)*t`{$+bDp8AF)76s zN6xREQ_@&cpMU!v-D*erYPgg(EvH;WV#SMFpKIV?ukJ54KE5-o8dPa-#vsxf9-s{& zBnP3r^@1g_l^7tf|yz90F9vrHYaf zcBb`i&0@fWCyk9feWbDPUsiWRj0bzoOVNbeNGV`WDrDx^sjzP55HUEsWPHr4^)~1} z)<)$cP+k3|SM{O`HvLz-pxbpoW6(-;XN>VYp6YOIYf)IoN?IbPp0$h9?o&Xs@#Dww zqY413&w87Eed_&P<2LiPfQXt^B5{l-PU#bvhDikW23ps3zxtJAP)UDd;*nCyxPGk6 z)}uA^P_utp(WVndYq|T-ueQr0^0Lya+03yXNj*R=Z^U)^^#TT%$vj}&w;Y}A=Y<%E?|5v&&422;r8z^7$6_zNp0pd8YX$pzWpPRgXu@&a0L<^(|elGQDSf z_mG8>JmJ>z@xGT*9S={&$!l2KK4@tLJ-Yk48#=K)$1t?#6F?XwPn2653BoXynzq!p zA^)rv70YA1LRnrIf7ukT-_=!}*F)`YsFN-bDjb%i&s$kfjN5C)6n0ibi9Bqm!f zp+T=HBR}$OXM@P21tho>*`f*7AG zrTt#Fsh^{@*Vww-HZ?dHqm{Vgz;oI&aND*D6WUvo)c4(e(l1mDEb3bGah%t4r?Yg$ zxAO)hXRB`92>*W{tsv@J^|RLtk;(ZlWb$458Uo|5Y#W{VpzA*>j9;4}XqsquxczJT zWVN_fO0c^^)i-!2$GMkN&wAaueU=w_?g4rE?nGwpytv|T2FZ_a+Cc(26eOHIpLlro z5?PGS*aId0KZ@kL`x*gU`g&>OMK%cDL;C5SmCAx8&ds>CvC?l^V$hC?`(Rrr^YAkD zMb@dvE|Rp_w>w+lX4J@lN|iznvHInogN#7Yd{QV@O!?VO_%@rnoQ{xw!imM_WQ$hz zrJy~vT$J0zjSTY2$zJsrOP!R!a3!pb!bd!rfLu7{xV^nRkGISpU9rPQ*QoPhXCHkw z6W$I|0#AXe%W+*SjL|F%XQP$w=Rfuh!MBCKdvNpWBUPE>z4fsI{mR5^gR+YBk;3+V zWtOO2E86qTM*Y&tA0SSN8(`57r>NuazyWu_K0Yn8|%b% zSw&X9x7SeCgp@|(_1gaeoy+&%V#_$9)vceL{eC90y4T0dXsrRP7XfYi&l}!DZ%_H* zPHFTK0d@n@CZKs>BA1m`njmrOqsXp+oF|>}Z(kmrZPp_npS<==N-Pxh9}Qg{2rO#J zl@%k^B-SlXhSMxFB}7ep`b1J8W%)}wCZps&j(_h@Je#eW?bo;Dm4%Sbf{og(Wmy}2 zJwj>-1jAHmVACW9$EG`ja@1#LjZx={(*R1?H(we_$;hpuq*rP%#`!jb^VHo`d1^yO zrq9S7w6Ji(K*Z6`r^osa;v(iDt%u*Zv<&XBja5;D>s5Bf+`5EZO~kEc&IeYsY#@or zX=zRY@6UgALZow^qATW|!k2ad2a5IjSR4XU2Fu$NJ@sA{h5O`I7JicVs{w=Vdd&BJ z#{d1U)6t0=UyaJYhEkJ`h-Ta;5z!Oz0Rig#({wrM2Mk(cejEy@Z*a*f+?w2%S9WYs zjs*F7q_lkiGhMer?M^ns<8<{)SCul}sGmz5lZl2rB3ft&aIb#IeNw!?%G5P#FqH9T zmy{>Dx%=oOA{e^-jpz_q96>x4tw^e=qRe?HAAP_?hYzmQ_pMa-Eh#~$auFX1-yZP= zuKKW^&)p4)o0U%HjV3-g1o@SS+#cjm)PRezu;llE-MVQ1pe~=ql;ct4hqsQ~Yt>F- zYz&))o1L(eIt_47+#k&R96qy*pZDBSaa-|5m5!|G6miCvB(BVoE^866)vF5c zra=i?i`}H;TKmf|!nbwKBg3$Zc$SNKw>Q0+T6G`r5mnf^51@UZskMPWb$URvY^+t~ zVa~NhRA-7+=@A3?5`RzgO}wA~){l~smIix$;fm`=&xTkZ8#w?zhbf3De}8@0kwW|S z5)!nuhUx8l^JOP@=I(E3s4!8^yxl;Q!ANzR3mJ!mrLs(yC1lI4l;5!ReU6^RX ztSqMJgdSCPg}KjBe+V-8)bzn-h2p2d2^By#0IAzZ8TH_`@r$}~s!h~0pu`Q1Z^$z8 zr`5Zw6xVc~ymuF4AF%$c=`kzAso<}<6*Rpjm&iA<)$4w!*JejYC$-S*426|x=UHD# z4c;A{{W$rA?$M1aDZxf7>>(civY&nVCL_P;{@edv5OncHzzvX^5?Wt3UchPn1g^?z zptJe??3Jzs)0J8vng>RMQ2Mw1(V_%ONRQW`0xe)OXIt09oTH1_+DQmepzH#Xu zgo{f!@X&gv?quT(&jjZR5p!YRYcvzT_SozAr-vZ`Q+=DcRfj@EJ9B0vb4&tDHYaI4}Ys8Y+>jSbSu|3^UOFrb*)2CD-a zRj7S}w1K~qNuxf2*HFtE{`JOC@QxL7ZtQ~p;DD|G^}9@7gKtbNVc7DEL6 zK+`1bzY=@32^k(z34X%CA&;)pK7g`(+VV7=UV08V^ndH@?tZ%exk5r!9yc>uplR#+ za*1ndM#VMbQ)c#pzo{Gkd%G^(Em9M#g>nwT1>B(lX(IACZZ1ui8~~v1DMP z!o{s^Jd!tbd*=|?>LVxL`Xlqma4&yMKH?+)BFAL$XTkVv&rq1*Y4TR`1TdE<)RfzO z|L0%^dwZ4QW~)J)>Tbt;^((G}(F;=t3%PY(Za04jhd&l)yW@dnVPQeeMzQY_pcH~d zlW!4No=HRM=Png5W;z3t4A)6?+HM~Ts%5ParMd@hQ9W?UlG^o^L3E{lp3=Mrq*i>( z)(vGKbdO`XR6LsY$hf19G@J3Ewx40W9%)x?4kt~>%YdmEZkTUZ;l9|pJF1UZMyYH! zhA2i+LkaMTlrcB`!K!e3Lb_zd;lNv(ffuYY#bQ#|^N*^P)J?Hmv>o*biL{8OD(!??A^}q z55<1n{OF`unmmx7Mqx31%2Sgp=+yZ?*Vk9Bs0r#vL=^&@SG$t~ka_%=tAygEc(mi9 zJZbu?%9K8x=8GFHj=6wCU@LgUC}b{;xGkUahQE$MsEr?!(*0QD3#WHkp^Cmn07NsfK?z}N8WE82zLsNcY%!1*)*N@% z(1P$|BA~;rfSOP9zV6hkB^H7^3UojX(6>FoV5^UwYRl^lS_0TGs?apH0XZf-0AZojUl@Myv9yPGliGij~VID8oE`Dc6cJ(Hl{)H(mA2Gw^z?n^cuna zr?q~V+lL9&21R+V>T8)15wF4P?d9|?X%;D57ppIE*!jmG7kARp?9g~UG57B3_L=w(Z6-*%SIR@220K1#0oeo0*6RXpKGz z6jKQ$TaqRpJkMc@ICS`j!VE6QeWHOKu!RCKOM#K(x+4{eZa`QjGH<-4(Jjdv2b7eu zTvMgzxXo2%t_gh=>U$(`OaAR20gGb0JCucZJCy^N(KBu9_pH!%Q z?2F#8P1Z-70CO1EUawy>RHug0h#J5l!a+lA9lV$lvNIrC?}<}@`1`-NEZ$Oa)U`hq zt!|=r?T7byy7^95h6YsnR9HYvW%(ZQN^|o->~BvmHMdQP+0tXK|I04?=+R2^zx7a( zy)N}7^7x{qu(Xil`H*IwPa<7B3YXq}Xp^kbF6}Cw4AvXy_mjN)pZJF=&Fbig=Doe$ zEWnWQwOm8xL};=~+H3aqJ8uJpy(eehs6^&5Amb8}xTzFz^lOIr<@KAqD2}BmS z?)gy&y)Co*v$-W;>FLlsbq#Bbamw|zYSwtDOs zzXrT5@$%<#cHO zB<^;+-sexNI+?yHf{Y01g%X;P7B6Yug9S#p-LKH52V+A*s*8t3+?hq|RUE_m)D!cI zyIr@m$~S^RGDIKBU`{>o{I53fS6QUW$$*aWRD7Uz(_)^*M?%CwDowNHEmc+eXiFjM z9F5spiFJie9n7|D5jnMi!BP(K2tnCV-@X!}7fcydTw0a24K;S-zB)}%VBB-X*){u9 z;8G9aiY&)6Txbwcke9~?-FvOQy7SP?ZuM#@=Uzyw1=7Ei<}q2HtkrPCRHFCRu_-`r zUFWK@Gm-hs7PXyi!g(QXCh#0xp+@eQV9eKG-omL!wJXVu|I?->4WGH?iT<6#5sa(* z<>AfRD0Kaot9MzqCq*m_)+V|f7gh%|nvfsA!8y*Uvh+C0GscMC_9236M`nhVsm#rb zwL7`Lt;i^P?cO7vq8nWco9c77A9%Vyvh;a1JL_7~4mfa@-+EOkdmNhi-$fl@udrO~ z0IwMJcU7<{kpA)x;_6*-WAepepe?JWym*$^C$xVH#?wk(RT`BoMxq*jQWoafF-k#G zzJ!@k-e_!UHgv((#<(j#SHgqXDo9|?l|(Jga`L3&sS{7&q3%wukReecbHfN1DO^Ltt|#$or7f_fVf< zDa1a|Nt<})ol<7F3&b}9NSCC&7!jVzS^#CIc6VnMPIh~U7Ox_} z6DJmeYO@VrWPykC)`th&p_QB=V6NOR(K~qFg_NVqfUL3-M?Hvx?BfF6dJ=E>Di{(G z4**ra*vqF+#)RCW{4_B0U=3eu?gOmq9rwvYQ}jxdST4WIPbtMrJk%*C1l#@+f;~sf z4Ru)mfArN%h>NlBfra#Ja9DiIxP^}@hwKV+4;wrs4wuiuw!2q^a5{M>p{}M z>iQo{0+zs*&r(aDxZ}pf)k=<0KUX*V7ARn}mSV-L(DEh_ryQ5<@aB~(gcK&rVwaUW z*1NNlJ%^_)V^_XYHlFn{0+--+Gcj>T6r~n;vQM8Pvv5(st(pyaG8k;Jm$0-@Gk-2* zcU7#RD7!W6;R}_coJCLvQA%F*=Kk$hh7^mu8n^aCqmp%ufj_8i@H24zRN%wN#<)Pj zW>Pr*#3cfE!lX0QxX~k+67XG|0pwnY4Oq`H(2N9Ihi!FiA4_30dKM&v*cTPyjv924 z0Z&x-sfuT_Ms+1eolQF)$V?$#4 zdYd+WW<%tk@MnEM`=fcsTu4KBy(Z6E;{gF}J=9kDxwaUvc<5Yik zT&W|yCby^J_O?%vPrPh22QAlG;U{Z14OjJGf0spWAU6dUQyCj>0f2>~z)b5vIlGin zAK|n7XRYi&xAG`7=kV|lIrmQJpKb*SbV_4$a{(T7=LXX!@bQ#{UKx*L%dusVQW*BZ19Iu=Ivi(Q4Z5~)X4=I$>y9SE~3z@77@ zg!mreM%}iW%z>dC3A>!R*-8Ro`~|UbPpi9(}0Vn8t^hf=m-|VnesL5IUgS zZe2Tt3&bRakiUZOcQ{mE1F*kUg`DKZeET`w!T!7V)trpK|At&h)Y&~8g4!T(0a@3s zU3+Nj@?O%Xzt{0_a10tjN4Ju;`3cu?p0#}#@OUhrZRs?V?xz`J&6> zwd3CXO6l>P2as>By7Zgf*lPzah{n0PX2r4`>nH2Pl`D4QSwpUx(h<$qd^yUPUvL^LrWJDh znzfaDKZb)V3*@wp6#jmveipcqJBH!Zumr9nm?`t>I|fn3fwo1l-Q_)Kvym!gZS39d zo)ExNFKr{V^vU}nx~f%Hw;)X*rEzUJPH6nHF;>x9p{~_hBOu_mL~oPc$L<1Pc>^}` z(}{(M+%S!-yvgzw$vlpY%c@OPAXSltYT2!@DFB{41CSO1Xb@z>05u0d{+M9ilaC;Q zm}0q`aQe;k>s za^K5!%lAM$(Nk;GAuqk$X#*$cin}tSVhLI=4PR)`Dr&-QtgIEBSt-8SEr5LZQ; z=Hg&8^Do0|X^=`k!i!u2$@18`I96{*x~F9jX!#gV{$U4_UyB2f9yymQ)bD|&w}G8) z@QN{Ec?t#m3Q$NrufVl;D=_F@g)3`viHpnYutQH0B#QPM06tqt_NWB-Dn)uJPsR|K zG!e9&oAfe;7A8ddl$gGKwfoc5R7fR$xCwD5#~4RD6l0{?*S#D504~hZuIJUxo$s&G z2r^e!*TE3+Md$1oyUhrdnfis!6#+QnAZi69(Z0WNkbzIZUQikfzp8v0#X7-^0`=aS zUL+***RF?<;~hoIAGzSR9$RIBeA}&pR5uHY{A`0NWGn+06d#eVeueyGC*GIVkpVFB zq_D~s>@`F#Txs8CCUhzUqtYTJZ>_>(3r@r?ipG1I+3KN4lIcbgY8aktcSFknte*DN z(BqEOg}Q-fBXjHz@q{(eELK~5{F;|P#~K>M5`d1$?$i1yGqQ}p5@!KLXMbAo7nb}8 zAckNC{d?q$vr-cc^T*U3M4dI4%eyv_(a^rh#RMjf<;TV{!YvVc>@x0}eOr9iT@^hy zhdKgT>BMulbLyJqQpQ|IL?EZg_Mx!9PtGyXIem4I_V=@n+&uoZW+S!!mI?XwXrr?) zh-@!A^V(D7w!e@<{IH_(ll7HYSEv_C;igaRyzbRQy>h{D8T1*BGLquL)6c$N{4da3 zPk9Tw2Qbj3e`aMy_Qys_Equ0bt1<_AT3#bTo92->!p>|gY&(k=JBz1g?=L9OeD=B!Rs)1bBIl~N;p;eXvJ2k-0t`+aYJDgR*#y@Fp#l(^+Lw$WG)!FnWVpZ z@B&CbN~w=Sb%q20!QT~60(nOL46(c?;;yk-)Muq(XFOYjsC>65;;}#TJ-{&o4rOv46RGr{Wvg+CxB6H$~41i^965lj8`6kfpD zPM!3t&?uAm8b6cq<(lRtBFN3gT6lp3(k8X{HE=$nN`l;fzLsktH!^yus<#)X!i?)RCl{8732O@!t1Hps~CFS)!xF51WtP%tk+!ITu zY@b%AoteV+pt|ue0MHqO7x%gb9ddgrr>THVeYN-H)RrAur7hfk>MbJAaFCr{%5I?K+s4YQlipj;KGoe~Ba({zK$y9xZ0ZBZeg z+Ee8Z$rZq@?+$g7lS(zt9mP9Oju($cwfu=G%{ELnz`b}rB(O6(Gh2&F>u|jd%~X|F z{ZEV#FuOs6J~MFuMRvBTa)QQjf3Q`s1x=ZL=}ajv=ejo#rD%2}9nq!-*anZWsd~41 zFZ^(XTiM`Vy7VM-Q^o&{{Mvi`EFg9tY*F8F@K=-1=c;2_c*X)DOq#n@5!#7^A*HN}`pj%LM#o3^z|< zwZdMv&D8-NPyU9cuXP$;;yDg5?nftLUgJoOKvxp*EkXhbC5u_9Ce9QzM9@o6&ZB8s zGEZgy3&~=CS(Iid2Asp=62(uHpI_dPHf=(A|h-eodcHwUi~34F=4glwZ_eoA?S}r^T4Tp79`TH@{%0sYwfc zL3?*05JPkIO8~7{cnY&9)cj65CqSkX9PkCw=RtqP`5MZ zPH#zUy74P!?Jv21FjZ6Z!HO(wH;`e8P4XvSYx@Fj(S#hs`k1Ezh?vOszM^ z&oGJR7cBg`Zx0S^5aZguu{c3-gk6Gu_W-IStCg{A=dK-L(5Kr~e|?AWGh5DUndxF{ zXQC$W_{?ZhHg}%{tWd=?szb8@*R=EbO#0oXo(Y`gG^Lfmw#ZF z6}o;hRC(Qs| z1^q3Jm#Ry9olJ{{DkIxn1BB@W%*+-hUbN+oT=?w^bM5D0kAUB^&qQ~yAv`8IOu)Na zFbJ$=-+a6F0)<$WeFC05Ug56y1YD+{r{^dD)ulw|Jm4-Wg(+76Ke$P=@kXTD;n)dC z_a%u9rT97m&XP9F<4V|5$6n%^xu2XouBGK6u(AtOS0$J`%N(5R|IHAK;E0z4W z7e3E-Fi<4ewty8UZVkAyvh^Ij*T@1|ium!0De13E=6X}8wl<2L0LBso>7m+lptpP$MV0r@o16<#x&Bw(+F}8Xu++2;sI(s6pg`;gLOT~5W zRC{n{WM4d%YiPf04w|Q?3Jekb_v;Jw-m~`?mzEqwlM5d_bQ@xv&2{#j;?xZP=o-@z z-{)j%>$Fhk@E5-_Vx!vHV*h)kR4?0VEI4vFf_Drz-GcCAV&_Q@QY>ZatUCICGwv50 z29=yx2##`zLST;irXPGUE6ub|n(<`}cJSwv36Ox{;NTRZU?w52L=(Yd)#N=e(1RD zdZELgmw7DV$BzP$J0BZ+q@+&_3G4|He7%r%Meh5VJc+z9=;7^VLf#Wj zpdkQ;y69FM+VKhQlL1;pP**2>)1jK}YB>sMIk!l`F6CbPQ#?H@fOAc629F7KDFM52Ow;Zj&SX%fy4EpXLyGqx0n?U02MYYxusoq^1 z@~oM~s@9ceDH5hs64S%N?>r;3y#S7s?%B1t2$=(lJC`7QyYI z9UL{X#3gYI*!2Jrv(I-gunG@@X@PY>I`EjP(;|~$RAKFxT_Uitj9LH5f^oaCk>M}P zXHL;M6^*xhT$hjpskn|ic?vu}>zkZSs;B7^jTUQ(?_5-_p2gh$b<>LMiLx#JRgHW4 zM^?w<3{0hDm2#6XtA2?Hv!P|s-7cQf3Y2K5;?2|7(jS~wvb-oTW{NDalaLOUJ=P>h3hi~IL zPlZIaKi38ttnS3ZU{iEyqYP&Ht|qt*AV61;SO|I+0LPko-bUUVJU_vhb&D&Gsd*ZB z2Ph|pGz|=hvt9BjcixKH=19OW3E8o`A3P$DM|!c=GU_4c`WjCd0SjcuBzKCWan?YT z{bGk+Mqr2@elr76W1=4x`T-c--l@-r?7UUBILbHgAz)N$K*IwP*O_N-*=Z_E$SJ{Q zrb<*f6m@%Mb524OlqY?%3b(JGDKmdu5jat=o!Ed1yZiRRUtedb*Ck>v`UD3zfe(7E zf8Ie)yIV9~dZa)jS|k)XKVf9N+OQ&`-VXvKNj5I2yIxO4vpR-#Av70JT5Obos_ZFo z5_2;gSvokhANK39^M$uV%<$Jc7=P)1Zq;V#X=3N)^AL@B1+J@XVXw}OW4Tqg-gL0! z-Q~G!$cIwr6m^RuB+Rdd#k7JCf;y~$nRTYlhe_2 zTRN`pV%qE8sOho&0TRPd-eOw{PKO2+tO9_2AtN@;q!|OEtmf`fsd#Dpw76UcaKEoI zF){YG0E>gJ-s$WgY|#O7?BFwx)ZlP<&tY$`wunWYCYWZCi1XxrHqbwZOK1rmq^bV4 zy~Nj{pWf|~;;aHIbrNR2O72uvpaI~e_TSeN2bPr5v~{`q3hbtwhdDUVf`!~jAmPHR zdo_Al z$K0-M#$5LM*(hWIeSrGHz@n8-Vt?b>hHq1yea|Hkn-()*j<-N~=FrAbym;}o&EChS z8Vus>RQ>+D$?9x_J)Jn0Hm(k#`JvBs1WuAaT8Y(zsLwlgof1E_D0dcnJ`D{%B>vai zM|bt_f4hC&Cc;tqRxslxz4FzlU=tBj6ojwl3CRl%WoWWQy`RC+qi+7dE{^H9@6&d z5|;!c&+~UZL+^4#2l%T4VUeylD|h}e&0KbNFtIE3@GC!X?91fy8j8a^(m zE-kMR@lhjj96c2naF0Kpr&+;36pK85U=Fs;yVXwLwh1%Sg|b{z3DbIjb~2W^;sT=X zAcexK1_jV!d8RYuU2_Wo6nR_EXg0fh~5vQ`Y21=zTtW5MJ{fq06e}Q~Q$c3$G(Mz?NjIE3o|LcpwI=EKq#`ote^~>O?fYINM&#wQt2TY{^@H%#}{Z$Ze^#9)hp>mlYLz-!O$hRK0{#bkQ(%V;H z@Xjaq!}q_LSMX5DAi+VK!E+7)ST~wvnKGc>qms%yfGpH*Q!>@QDlDT0!p$*BZ(w!W zoyLO#qt)TK#6oh`6!XT1-bFt^>|n(>*$ng>mT3(}gW1JkU&Byp@n3I88AFx0M64Qz z*Vrzn4m>Xz_CSveM*`Q}(@_AB9p)9TcHYK2ppS{Ck0u92N&4{=6*|93kx_7=H_)86 zfPZ;hZ+CY*_sdmq;fwI1O>9o&%YRhJ5TF0S|%o znZdHn$K|yrW!nJATN$xuv*%u*!k>s_O^q6uf)RV+Kiv984X*GZww$AVmKd%kUIGKi zjK@Y@d>-75!Z8GzR*FM-U)o6PJg(3doRY9lhvJ&n1p-~TP+mN`m zM}q|817Ii5>PdIG3``4vW#uFEWo`q!#K)Iub{6~0{-tD`E=Z37DKt{6yuV%Kx{qV8 z5^$%o55H_~W|387pffaQSSg;0>{SXw{mUEu=*)=%%fGz;;zc>Oa}#Yd5x`m>=`tt= z$>KcaGFNLZ+4fsN5*&J-~X^kdWbB4@e-5|i^E9L%g z;;Ku#SD;Q}E)ZDDGEFNIk<-aLw=SIr@DIWKAMGZGwz0~c{#JG{ zxj{@Gx-ppWA4*VDkv0Qyeg#vSWGvd;5B2 z8&uAE*8r^>V|2Xxz0Ib!fKGfw+E@jB_TRN`alQYq;(3IQXx3Ji;XEI!@Y>8M`p!4x~n*~96S8DY8(Vh6sCqPNk z8l|ayV3LSGux=Uy{wY%+bQ*{wUlCXm6iUkWBoktQy+J>?4Z%}!uI=Eje+Cs;?u|GeZd(bUH$l1x+&jQ# zbUh7UpIz{a@k|$O*9=F&GYt#dv2i+yGr;;on_xg9Bp0C zMx1g-qfx;bU>cvk0DBz&?%#2;E3dh|v_<>!mM~!Q5Zh3p4VVS$=_#At2d4&Y@ zTMa08+CYclRHyx0X&^5Mp?FA1aY~gBgZnC2d719+L*voGIblF62(YF_U`E{ly5MUp z>xE4omRbk?ThYn?d$&dxnE?V-hqpYHPxRtw+wF9Y>0IhFam^F^XKYbnG9bAz55|x^ z3IubFKpesF{gC5qy24laiWXJc_yW=_O;Bn))_NTZ0?S^g;ZF!#=Y)29%o*&C ztpor!X)HyYV$rEx8AB+Jy!e0t1~jvyf5P=Uzdl_Asy9h znCU|6ruR=4&dzo_Nv%EKEJnGY$AbVuPUmi$>1=l@dobOrt1-CVcDV6vQK)uzSRET5 zn)$%-3&p^S^c3{^cz?19QH7p%7%z_Vb#pXm=Bv!FQukl;kHb&qPkRrSybJd@@E{B$ zV(}BR#+cZzQ$cptJCVtU;{|6%a$v3{3|H}cBtdSX@u3vhID@RFhO1yU?k1p#A77(E z;@5t>3eB4pFjliGR-^E%A%a1ocJ0#95PUBJB}jn}ecV%XroXr#47a7q>t=pt@ToD1 zVljOA9EyO-FqZ8}bmD-~$C1}X&0 ztmq+$n^ov&-Sv~HZ)49v9*4U1t{%pAi|PSPUmmmxudjLO9xu1IEMDS((j&|ah?Nvd zmRhcVJlMps5|SF<^q!6T>zi$v>tA-9+V9=`XB-_&$)$t`?)cQ8c3Z6CYr@&^E|Yc` z{<-o16tJ(tJONJp5w>)R9DWk&K79_IA#aOH=^E(5`Klb(^E(tj5E@_ zS~qU?J`i_EeNvOtdX)gL#_nB%0qjXB`<5W_#feBbuAk!Z+Dlo5Z!#rPBc2|dN%ku9_|^N`cPeQ6)p0i-Ra$f zafuIaLsjsjW>0EG>PW#D5WDLG)IAo1iREY+7`mq~K>z1@4fTocYPZ?@?+OerH%p6p z)h^CEzMaj7zONMaFUFZ@HRzoSpmnq$6rjv~Z_v@C_``JvU2j@zD>z}#UUihQ%f6TW zz1h0UnAWFPN`;o@sts?Ypj{%MVZW)Gei<<9gNvbu(_CbN(e^RgszE7BQQwAV(CV|< zjURb??(sBKYyW`rce>&WW#2d^-LB2MdtAMRtD7*LV)3GIIj>LT^d3k%%hlkNM?5Pd zOw$`Eq`71APZ&J7t|oxL?YWM7<)MM}TGa{MUi?5|=*eJ;4}fr~(>@BQ9x>R)0B5TX zchfpvLOmrmsM+zF!+3_o^}Rd%5-j7*C%e{-4bekt!-2_7hGXo2*!tPCJR3dj>vhQe zh(0)RM0?&%MD%OpQC)PPo+EqMu|OF>CS@5!4+^gTEqqqgs`tOnGHD%lnU6@*)_E8!otJ+Be4QrLt%Xw50c8lrs-E@j z$7nDf{nBR+z7A&Mp=8S=LG-+C^)Nrpo^4v9BjqzwXj+P7(ia;AIJan#%N~ru?vdiEWU2g9>&=%eayG{bY$BTI6=7S(c;*m=<5*Cw=IoMQ;*Dn=DhcGZUXzo zP2P9&RKAa|XCywd8+g(5-mb~L7jHQUi6M4RNnXc4+d_NqVDC>q+6aKrm*xy*&tE4t ztSmnQD(Rf{^=M6v8eZ`o=Z+0GA`OVaUH8MY^B2c+I#xwh;V^=Gu|mJzZHifJDDZ^S z7eyo8I$5}G!ohGrRAc~AA^A09iZd=grDM{??6TwU0`#5r=mAeWWm&&z#GFEh^Z<4? zac2sOUuuHmLO<@7G--I{3MuWWksQu8v#YWvt?k5d8R3Kn~?o7`KJl zz^e>*p9D5nppY}DENOtN#^A4xM$Ta~!GC+GPPwn+xK3VPAF2FTxMv-wj@E65mCW0uObfhYW+GOEU{cA`k%`{<%yZg2Tic?{)q2PGk!}2hyfRVf4rdSCDHNahhAjt0IC7CBC%i0a^c?1k)oOf6m8(zll`$dJ~myB;3P|_0PgfuDP^05nZi(trh^cM&eI*ReceJ|Q&;Z#C-s4&hb$^)6J>qG6VOu}}2( zZCFN_LcX4Pya9ELugp8mWchvrhg>IPJ*aW?UNdCjG*Njv8W)NMe<~<>#a%O!PvNvV z1BZb^3g(7j>0UQFr41NE+(u&W@5)w$gGm9%h~!`2bFeHE)gX>1uWm-PygdK1_oeB$ z1{Y$m10cNyp@_5Lh1<`FDO-c#Rlc4{-WP9vVqU+cnzc)3CI;us<#2tu{h3bJ8Vf}$ zzbCU+Crv@=kQ1;2nriE4-Ne}BG8}>o2S0+>J`nC|x!L1#XGN%dZFazXJMwxY@pFs$ zUBg=#bRZ157XdTrrBZ=Ap-j~!d&FMtXbz0JM4dabjrQ|Z;}94S03Z0{>(#k`5X)7# zqt>;Fb0C(W#y@|L1ZE`P7)W8xg%9m0*<|=@9Q4_!re?^U;kF0G{?U zF-QXCsVK#Pdt)Uj%oJ9LaG19`rAgs%?#St)b*yg79#8SH-p!gfip7&OCW@L&_yAS$ z{Veip-r_?lTuI&$?^&L_;S7lsk)Uno&G%g4|4?!%aE6et?Vx=DgdxHF;ZDB6Wi2=& z*lI7;$C=pO6bwbgZ9|2Eg(FycI5#08LQ0geofY@f*yg!3VTOMz{l@tL5B>rPJ2Gpz zlDj#fvY9J9K!dR8RMJSkw%TklI*gn5d+_ktgg9+Uwoljgz{ra0(u+ya#r%EL#_x-@ zQG`(T_)U$=j|W8}*n+o*DV#xoAVGJA4Z9Y1WuplFS~FM0Vm}x%?x(Ah#v{(it0d08 z-VD%~?4OWPe~gn|blf-$>moONdOsFKzOq)ADwmQvp}CjJxA!VGqx~0~iBW#LkJc1o z^7+1h@U!QO%)afB5pw$B<{r5RhNXky@4eSF#XY49tc5ojX6c>S!|HW>nGb>g zM|hvW6v4A`aI;jgf>1VzncgUS*2JC?F)D2%rj-7~BF!B&q|o604GOvRWoge+N`{1E z*xaB!^)v?ZHcmF@)2Pf*a; z8+)pCC;$dzWX~Fl1{`vOo^W&W%H}39F0out|HLS82s!tpN%qsD%mAIGXv9FGeNIUV z;w|d`b_Qeihuw$Wg+^Fi)v>Tn)cNzqb(&8+;!Lnk-M#Rcg3Vxq@+l5?BUsC>*m<=J z0Jwqj*SUau0_URk^6~zr{&5Dq*%kj4LAQhDRs%NMk$V5Sdrgx<=P`L!G2wMENwTJ)lF0aYtt-Iqnt!wXu&}8KD?%ifb`96MR z(HK-Y{5SC6rB+MJypAPSce7F#Cm)t!H!xY;6jsC_JwT*X$HnH4e{pMFM?RO36@s=a^n%fFu5V1lMD`)}Q5&wj@2DVy6| zmyJ7>4r%5N-mC$kz^vy>lZlF@X$u{zHSbj47NT!pHk&;%SbL3vo$=CCTT{4<9NXK< zW1HS*3W*h#<0+g@>v9Unuw(LXwu6HQW%9uYeu{8q5MV*-FQmq-V9v#8PCPKg{ES2$ z+jHf^Ze1`m1=XqC$(l}k2ge9yRwl`xZb$=2>r$8UaCW8bxlchhQLRQ4j{1TRo~8~o zoG3dY?L9~JN+i@Or!tqJK=`s^=J0W@^4&<1PZxuw=KVp?Wb3kR2qRkeMd21>_-Kx~ zEA3}C2&(%#omw6yei{4N77c{%MGAa5hMX+-9j1iuQNJADr94GOM!GvyY|CUtdiQtvY@>i!(v@sM+gwMMtWGWq3^rGE$KE zC$gbe=i2+L9@rOWsFF0nCj~Z1fLcFf~tbUgTVA00z?}Qe6vu zln8Iv$K(QU+-1%RjK}livc=sro998%T;{}RJ6d?0KWh2&^MU4%6uztq>)-rOsSXpU z(fX0fSIVVt zgL8I(I+M>Hi;VcK1&L*e?E1UKkxDdkLx925)HP%iop!-Al8Nm3BcRt~ViWpQ-$gzV zzC9PlBTG;nd?u1z3ps?_v1950t~IJuom!hZ!I2&mndCRxpfAaQn4BVRI%cL3AGX5u zqmrHP9yu%e<7I%5mw}65ve+=X#gCXa-E%O`)OXp_;V*FO+s@|oIrOz(N$CS-2K>03 zx>R3J>NCEEC)=`gktidyQehYWb1YrD;Vq>*@(EGV>8oAq&|e$PvNP;jbi*@vx9rcj z8Facv`rRljpttUyoJIa4?1e~8XL$N!^14zj>EJg8*d*l!G{!{M{6&#ie>i3g*g9ZT zOaL9BK2_zZN1Q9fy;9W2A-EE;Y&~!t zABPIOhl9?J?+p91D*5GT)$OnVarO&pbXm z6ToSR;l06fp#umO)Q!JxcA)@vv5hbO$xy_<>qLa(+Gw%A{_(F)MEFiOdWCcOi*Z_m zN6>8QMqKY6326>g2}QZ5{A$q_kFZN<3;!?kNn z{WVj8r(TEn>}<8*7s$pXfAl%KUF0YQKf~Mj5A{);nn6ATwBKbX%+7S6Eh)B$f&uKZ zb=rq!b3?3llKzwCXZwrQGnoUCV>dT5s)47*d2mj27o9&)ILPT>&V<34pLyFMvCxw~ zLRe?VUdhB;b-k7VjtYB)XC%iQzSd4&Ub3(KAe0J@Z}N{mJD+u!cEYzKg0ptdrU0r9 zI`aMjtfjuZSvb)R%$dlV%TesxZ|!Fhm~GMg;=?ylW`GlgtFq^J!1&U?BCex3e4E4J zNEg(u8R2+q&AW!MayVz^&;|YvvFOn~b5Zxx0fLDB>b@i8oNwLVFMp3jFZHL4o14U| zYOGMu0wgR7$S&O1wxP-|c*jBC!LOP5Z62+$$9-#>|9H2cr2Xv1b(MgoOUcju-^nx6 z__N4n(h1)>^6q0yJdMn4j%|3wKcC(m6;aZzQ8KQ6J}Z)IBzw;B_^JEnI8^v!r%N-( zw;ya*x^X4xa#pFk6z=7YBw_tu_)X&~X{L)w_80R&FY%CaT$KOU^_l5hv8H5utYVV3 zGVzUn^n{>!vMT<)R8IN9ub;*ua&}Wy}ZaHrcG>k+R9Eh;_M--iZ)JVL4M=NMW&RXejGk9S`eFeSV;~^Lo>z3LC45>#MZ) z*a#C2Ct6<@5#A{9YeiHilPpbKwvz12!)65XXE~+Zo*wmDn=#tMo@{|_d3-f+ZN}g8 zby0c;#|%<=i919=nBZek)4_qn47E>QS~{ZVToEUjPA}^k?A%)Q^FlmJwDrY&xn%w` zzb4a%9*qrFv~nrP zfKZC*BU_s$ejU7}Yp~WvzDL}e=_fH_-};&OsysSn{XxIFJJmRAb^aEu*xtba{`^dQ zr;yO$?^lJp!u9=I4$(M8dHF$8G(;1^6}4V&qMcc?0Eg z)=-v=hQtqpYmI#(@w&{lp@(U{NzuvXv1(`-e~XAVD)J6AVCih?-Yp4gpgRTvYEu&lB?UA2(OTDmzabWvrlY^hc z)ho(`&cYmZu*mk z_vOUI9$_QLaCzADJZs6iM4GZ>e(dVw;S9oyQY8tI9{(324Ig@T5C=MrwQ}zpovKfA z7z*L*PeTo`;ZutU7^g{u96O0QS?*l!Ai02?ul2T%i+bmmU0I+aw!LI`yoA8PeXWil zq?+03H=}>6N$P1aMkrC?9cWjla*K_BZBVkad&cPHVq5621NYF;{MmJ}JBrVBv+G7_ zQV7az+>uIDB{xxZ+8~#k>`#zVvErERQWa-*i+H+dlGakY7FYlc_ey zRQy)ibY=ctPw%v?*?>;Xf!6a^QGCiPoWqEoCbI?tE`MDd0B*5GB zh!9Epgk%1uwx508!&c>PEqc0tLI#s_032=hS%n6gikvvbsXL_@_vx;}oI&JblHc0j zI^_;Ke836hR;$*fTa1F&(5}CuB)L)$oJWY06dDdbT2)17al$D9_r51H#o4k-TZ|f? zG*w9dK{Yrk!F=PjSHe4HdG9May$t&L)1we0tTWngsD2(Se#qv!thIDRxZAQkptpOu z>vs3khbof_-wPef(xygQ%fKgr?j>qx*Cj|))7M;?JTjsoV8P)zig#e17M4%JwSgL= z;W7R?bGQ=KnK}4zy(P_I$99Mr4e31D;h_EJ;7`c0=g1UH7p^6Hs}jfTn;`}dwO31s z9GrE{bl8zo_eLc^Hsx`ne{a->2>Zr`rz_iIb&JIJrtu+1<%%B-7EtUMd=(K^v# zR|OYM$`RY7@$rNB-jJmue9~k2HYwAB=C(Kb+Nf~lwhm=8nM4O-8y3(p@iV~x4vGlc=ujWINg~Lp9+!U_9OrUqq4W9XGw1rBb!ME zm=@-ijufI|zcwKVjzzlR_iVZZalr$Xca1A7C1sy6fq&qMqxz%Py_i+jz3u(p6eq#N z>+-M(LWL&WbtnqiQ}i4pZu@k;{BTZb?1V6(su#9_^u}w9rUy~%xiD^*^Zq^%%ip|L+O(y2bBl- z;SER`82@m;{{*bvF6nUR-qpwF89Rfu+!rtNz^15@N_K=>^!`4Q zOWFf-8VA3=Us3?AkW0aZ*yarfwI#&+4`zZ|4`9^gh!gI`02FkI_zg8Ry^C#a#Hwf$ zhiOkK#IsLLvTkxyyY5?89;(JTh|QGX1|PB!dP0;&(Ah{C~6WN(VawTVPL7JBP zduM5>ylPdk?M0QLGxN2kAf}$`a zrU|0j)h-oK?&zAzOB;b7tta;w__{b$G5NX$!v{!S3q=Z&B!C2K1%6!uW5XFQ)*fm- z82I!pX1ZHd!tv?RP?p$$@Qsk%O4#nh<0*2c$iUUm{!W*}tWg%e3H!0rDi_^vW$B7} zTTOul*e3OyiKCkN1x-zEfZ>zGIRvRl)FOkyBP(Lvu+>#w#fR5lS~(>3_zin}ofGN4 z{ZQ#n`FrPTI$yVCSiMs%n0do95B>LXEduExf}`Tc&R?trV_u3;heuA;k|U-jcD84x zba3E=dNTFDR-~Rv5GHwM#M1KzJ)7;Zctc1H`wStGNiTP2@mZN#;}3(*Q#Yht_sZJ$ zURRMLuDwA0lT+;|P5J$bpm?Xr`=8@b%X;bS!D{Eb48G{i{mBkSoY1O;>@&$}(vprr zv(33fB&*n5>V4|LwVo6cqrPULx6c%X&55O|{6x6;8~_qcK#6G@U5d#ga3WBIL#) z?@Ci~w|jCpK~#!muq?6U7dpP4tOnFyF(Sov7u@>cZDgy#Zn1NNe$=|s;<#3U;mn5n zdWjA@#4mFvuN5YC=BDTZhA;9j#T-l!>kACu@E}oU1e;l>c9G<{@yAuzo5{-QIqkhY4oL;`OgALE0}tGuP1%06On2C!ts}?rK*blq5r@HkDh|8wL6CgWB zJP~h7UH8Yq*S>$Jj&c~XUK51laqu%Pl%e>&vTFI>Gri0j(#n5 zaze$!@zLKm1|USJZv~zaZ^RmF(QwDQ(qTvc#RwG%v0?5wDjQ7+A?bq=pG<`AgsZqO z{lNn{A9$TZXfh?cZ&~6ixeG{-Aj?UFe3dE>p*hg7%WxK0yPg=0bvej;GsL0SF@t>C zCF-FqPKm0kcBTdv04qT|K_P`c+!@To_oQxm;ZI`f1Lft{#i)nWvXj@Ah-PlQ3n362 z;K#}5=JEQZ8 zODty`)k4-rv$tw{7E|_%S+EQJ*y6+6`uibGW;@m9gV*U(=k}bgRd_AnQB6J6r|@BY z{9^f{DCrt-DRgELYEjMf^WFe(KNYL%ZA3s=mDgCT6F0#c zXe`x2kA?J%m9{cnh#Cm@FhFFEJ8Nz8q!YJ2|%b>|28+cVu-sW&o!W>94}dD_Ek zL`QN{>}<{Cb&@M1WpOCFNadc#FuvV?9@{(~8 z>a<91i|ub}cO|(p!qVb6MVtgbu`wAfGLQ~ikYMqo$Ir{&^|X{n#Zh}J>2w?RA8~Wt z(!ljq1wqKF02bh^99^F=0^)}(m!-vti8yKO;r#~tJlRkA=yZd7&lge+#jT9$NRq5# zA#0Go%dagi)_)k_0E+2pE6ns$0?>C}9(nUJQ#OYOh5G{zcvRct-rH@aD?mNqk9!gg zgjFVg#`d3tKuS{%_tzDN@Xj8(1RCq(0bU<+CUP)hBbyX!e->(Z-@hy+MaS3PcKzl) z`cke!Dk{9y(6_HEqs|yunIYYl%N!A{cUUHYfBE`FHoKu7-|qOhLL*|+o1yM>m-wrM z1pSbCJv~ST7UCtrSAa8#XvM~?d~Tj5Mto=nqGX#??%#nlD+S>0NM1OWFuT@g6{PDa zwv4sk7>o#BTS>Pd_SKg-(2Cc_V-AMSOxMzwXkSGGH>RT6!%DKsy|sDd&LSG_K2^rX zms@%W_?@195K$g8X-Xlk|zLw!On9eLoUi>*Gg!P5P8vY9WhdvftP}N5f*u?9BsMvbXXs@8pc3G@RMC}A^=p`T#fcM7Da;emx<*P zp;*ZIF^?xNaeJR3e`tJ0k?vI#7W6!u}MKf!q1q?&BR=?z73L zgjkl|JxO(TzWt+w4swjg%FB{06zT8ui7-S-5@(FlEz5#U(HYq3b)3mGZ`j6p+ki<1*udiqx6Z3$L0H;QC0SA<5Mb`_D%DQVa@VnMY24P6$jV zw(L#^&189(`_bC{-FZkgE?%ttPLs1FA!{(kqilW+aP2*-j!~#1E_^@I!NAhp6Q_0NKvBvJLe6ceLK<%EKz*u8@9Gjy6i~$iG+DvTqlI9!PWT?y+jSM?XK=~kmXn!Ub*<2nrHoRRS~Pc zRIv`gDHhc40LB3w7oYl={M$z32YJ5O@Uv9bGWib)!Xb(cxsacd5^xCpeY@7vWu&XF zvkYP<&Jd2Fr3oG#rn}yu{rUlZiJieB{QUu{{4FR;e+9>~u+ed>$_A4P=qUJe`hm|P zQ>T_Ke+PG#IR5Z^if342C^TGUlD=g0YzFOUplPI$)w{+MmZyMBh0xL9UVK%I)KMb= zS42%y#z+>0RWx)F@jt3J=$DdhSqpsWB(rrs#C5b<;^tKRwC8z5E7jbew=44Ydo53C zK~BiD{_2Yoqu}XsFGAx*f{R7SjAkDWg$XwJ>9azrHEkX^{KmuX>J>SU%>%P73iwWZ=Te;6oQ#FvC&5lpqZ{1^<3H5+mlt!7(UND zv;#Afj@JFAqr()TL||c_cr*G= zfnTXbz!+j2?8wQ-kAsOPzclj)68p*yPl~Z;pBr^AxgpobZtv9?YA|O!?23)r|Lz%d>F1~ot7%Nv-(&!c`a^rHeir@SP(Ck1x{w( zYsu9TPkl%6v3l}arJGZoLycZjtI(r&Yab#Xj^5UTI^DL7aPatb4YbQo(~`;MmSdI_ zn?ET?B1_yu0fuqsFMY@2wU5-6jv49l;#P$e$th^s8dl2I0(T1}$jhe_!g%iW;AP}^{~*WFx_5T@81V0cC8GjORV4R)`? zck~8UJG&%24-VNM7^jS;V=ckM{6m)BdZ zjL=|X@BMx10ce6Hn-W9L9vP_0iJGz#NH!O1Wkz0b3Ar-)ocr-1I{P@z_vOn#T0%jK z?z4%y*lzp|czS>$MB!{Jh>1n_0E?jb^1cdZ+_qv%ps~NMCH9Q={mrt|W))=#%IAj< z#sT-e%Ti8LxSl8r#$z!JY=2JT>%#Q4*F?`5YnK%=9#dMGf*m;3+J+++|KcuQ)b3m9jbyxMIN8a zigoM$waR^XL)7?cOb?H;qxBEm>7s1ax^bnJi~YN&doO)ytLs{O?*k=C9PJ|BU0BNQ zF;qZ`{sIpad01mKTThoOZd-pJ3Rv1dS|`0L6(C(?MOe7~(4|B^lD$T(L$9>v3PjaL z1QVAFr9#HuzN6F?+L?@W@J&ufMEny+!Ly#lk`SMKpYQo9czuy-0G{ zm9w)&z=HU(vmABON_YCrAK1IUZIA~f1yz6?YUW8npr$<9s7FpqRtxy?R628O$i_^( zJ!%Q=JwTZyIt2KiNm6Lw8ICrQr(z{Z5zhZNeP3v4MAdsGW$5#tBOysdohqhe9- z@Xn)R@fY7gbtEgB`M%1<0IzEtX<8TaA3jx2zSXTFkWL>goRuk@1vHLpFjKl}@?|{{ zT1!67?gpZMg2$lz@SOyp-PUGR%Z<#0c&?G z+U(bNeXCVbBJnH8`{jMT6H<;2eO0974I<0|;UtYz*2rVO-RU*x1{;fT=x(#z0XY?H zFOd8YB^Qu-MW5?5sLro{YPORyG!pSBeb>!!DyW zEV4FZyd4NGnpAKI65!%#_RF_k2j9(J2h-5i(QDP$;hq7ML#ibc*aiKXC&$-D z{|i9m1jhoG&fk!70uqnyMG;-?C%-J!jja(o9)ow-$^NNjE#gGwM7PBcPD`Ewen*fh zFBi!wJ5LC=f8hQNB}=A->ciG}+J3Fy^fEXh-kvPNhZNJN>Rn*sMW9qib$b24y`B=P zuP=U~ZKlqFRu_Qqmw_No#(7w}aE#CVB~Bs%1X$G7v)Of;ZD`4F?rA7Xd;rb3STg_P zsC1^$wKl-u;Dql`S?iX|ZE_>!+Unq;SI{_TIvxq_3}7ghob2ntYiqyrJY!lRW8q44 zVNY)yA-ktg@Vt&z;AVpW^}iYf=4YXhO+X$0;nuT0zwzQSFVhjtxk=_#+j2sTf{*1V{3BZTAw@@TA`n4 zM5EjgP-=LAFZ0UX1RlMjRVv|mf4IDRMePjQSm1^66m31bS3c!D-M=3Yo}Tn!@DEj5 zMI%E|{8&TMjQKK~5qMZ{7|S^s({Pl9%ce!9_L9WIYXt~AHhI1}~ zlJ(4-0eQb6<5Qs<%%{uHoj9YC;ZiDY$?<*OP!x`W_w2SR*7Oz0krW4D9X`Fuin4-g z8rI1WD2dy8I>GaLTKmMsv=syVd{>G_+o=a{^;jg^lS7EsQ37paaz0XHinbaXd( zV_RMh0OGsh;UpZ5PMh?j2KoTaXHcO(vD)Y7cZr)|548@oLu}hXQZuA@^*&wBCboZw zZ6AsNs?B-EK=^$h+Ui8|%1KQAkrhIa`EE3bNR%^3UuS~$k)e9n&}z6dbdSNEJL2^L zyos07hjko3f3=XF*gN`pvzA0P-c8H?DnHSyWjT=@vVl-@QMv!DQ z>{F`CoxGG3E;mkkZ{+snANdsovF--I1}$nK)h~ft8+Ma>9%KJ)b~vkt;Gy+cYK6~c)J}U}nPkR1hjJ-enA>CA*3D91p z4rIX?83>5AcOPGL-Dr!{c=^`l43|tT&(TK#zTI~m1sj?bIn?JU7@qB%QZ1yE3_39y*1D$6t;R^g#t83Y{XHPL0Yfm zkZEK^Ez8{9FsSlP-Qv+w2jD*=GF)D9XtN26`nx79MCi_LF1RsJlR!`G2^uv?_Cq@n zp>|cfp_WJ5@KsY3T!7#p=cJ1XbB*-+-1M5F3Dp+PE8PF*Mp*+%g+e zFCE+sieMw~we?|}n_Ujk4mE4qnAdOhBv~I#*Fqs)4YC)qC|t4^9{d(rVu=(bqpZTx z%Wu23y~qc)%YACXVUtzBDWee3YjHybnseo`+U*(1Yr)2ywi&|!VJ(kCmO`wA1lF62 zi7$$Qy;xPyf&=Y%fN~}@r~%DE`uNuQ;r5uPWY809WdX<^m|QePvB9xggF}`bFDC*` z^9E8u>Ixqe`ffHE_0Ae_O@7y7ggaCSiPm==$W=}>m+8=U9%x=e+icN{i?j}ilw1HI zh}w}Z3@NI#2i5J^-9zZJfeyc-aFn2(op9L9=aBi4rgahEwD=Cutcc-vnLH|Vo`PBr z+M$dS%9V(%-Ir}$+o>r;AoRg56!+e-mV{29y2^6xc8fPZ_&6u9xgWS0#6~zH_!1uzKLLBExc_v zdOwbdlna2K#WeYQo{*~gJ8+YjbKVE(vDcRlyNwnLCWz+=i!UTI*YHpDP>tV?fASDG zk#Ye)l;k258U`so<;s$ylTBf=g+^EQ^_B_Y!9X&#>;YJ1{FZcK$P7fHe+#2VySMi( zRJ_>dq645Dhf);OXmDF=1-dqYC-W~|+9o+VVly-*y)7;BID{A2G0NPg&s1wK?~OK} zLWgX}74dp#gRH4^Wu6y?UO@gw5tU@?SW5w6zziJe3sXQM3>ma+J-WLRtr)9f#YcRA z(OrdnYaRcb@yrt-#FoK49m*g_8?n-53K|_)TxfB0G*%Ha>&FkK)%~7LeE-P0Y?ZWx zfec-TIB?~Fsn$V*8gPVQU{8k&NO>&)8$6H$%aAPxE1VI08Z8l^F6Zre zseY*b(~S2Kfzjw&)&ko8Y%hZ{!K=cCdoyvsh1dYGrOYxblb+YJt^a?)`H;#VzXNtf zf?P%aGk88@@W6g!%%|&H`!B9&1-fROp%)2c#(_A!`4|0^89LfjFOpSZTUo{zLKJ#r zr=9XdfW)*xtz@{O9jLxQh_Kx6Ad@U+1LH+mkJ>5(_(27=BAKaCyk`&(bC6RPvk&P2jt6CwC zhZVacdoB@!LCszhfKI63g0c`1|H2v?jKSa(QJU&s!HU;!2YIj}y8})L>L~ADl3!JH zcDc!*?{)JOOzi`@(AEN4l;}OU+3K{-lJ4IW%3GkxEIIzoq2!N~SuJAYMlnm!R%{iG zuNDf*uFl(!CJL>0{QRJ80`8$|YgzWKBzY(wxUht(5@u<=BjHMSxZgrA6xuX6h%&5X z3r70?pcJxl)S(RPt4RaG;v%jasEGNpm}}-ALR;qCoCHyBh6euis!M(dzSvc&6KL?l}{@(bF%KrT0W zv_A0Pu$AF1m8BQQX0Z!ed!Z1Q2oxhb)xd-0_LEgEwt`h|W`keq9)__Io>a~{FP#_e zb9Cb)zfp`Vn9d}*gNS(re1UXD8O)OP`UP4*wR^yXrfgXxzx*JK{mG!jG__5?jviAT zq{;RCflv@y$bt!$dT3uk8xhG!TLj28HcFx!$*e4r3P5YHotnZ%C@WIK{Rmp{U(ad8 zLibucrqY2H$FjWGw8;g;>azk>UR(ezW6)P>YXc1lWOWe)@yrUhZ%sZy_L$gWfFwX> zjrP|4w&861%i*3&$zjekG-Au>q#ck<;O0<1Z|6u>qbLkied9dPqKN1(baVikQIZL! z7-yQv{xD+cO39KCVNqW%WxN_g=!lB5#(p zf*kuSLw+VF^m^G=3JnJBP*ps3=O4YZ)2v>nX`@}7}p8((Hm_M9LxY6wB!H=$HexRVNXZy zv(TP0o}<{}G>ztFpd^=mme{i#{VA7z&PhUUIb!PTWqMm$mC!~PilL$3$|1*gI4=MzveVW{~ zXKWI$gg_%lNr6-d7t?6TV*N6dtn2%4o%=0<4O5&JWd-%SD$l|GEGRazE^Bed(c~nO zw8z|HvXUC0Q(dM9db+{)eIHu+{!x93Al0~Mv{`(w#iD4$+nUSQ$j30wP-V-aU}srj z%c9I?_wE&4k7mea4z@b!jQ=-y4O|n9!2!wEurJH9*1$-uRf)VQhRlMHxd33Q9e;2q zSVaqC34b&xM4K|+_Q8=eyaKVjF|k-u9NKch~+Md&I-Y0LMxeacRk93c~*#Qdi*OD-qdlf z=$tFe3(QQbp~)*b%ZbFuQ37}fVp9$b*Y|xHF?p&#_GOSS&p;KlGf|cNGJ=QP{C`Y~ z)hR4b`_exwPZ))L6jNMi7a_mDL9-ksb{RqB(CJ_(=A)ChFQ?9&&4zhPG4tKeWomU& zV1i5%TAZPRd9nI>MC%V4Vs(O&TZJjHJ^!lW+gi~d*38b9w-}8Ir{#Knh|W!1oq3`= zI~yhxF;iEwlRM1F7n)<0Oo{x5f??M`q>dx1lv2lsa^vKxiwgly*?+5oj1_|w46ua7 z8k_O@gV&n2@IYq?g9RVe-Z-GRmYVFRx0y!?g4Zf?^jMB5L`O9akT2WM5BdZyj>?l? zM-1G+q5psF8_4f6{|_om{y7BaNInSj!$;5yxV>8+BN5D&KuE-~r6FYE`2U?izDk$d zD*#!6MQ-JL-BLcWcAh*s{2(Fta3gu&0EG`B3Pa?Mo5H6I7RkCyD0~J^+9dDlq4cS! z3Ubng!l!WW$iX0m4n;Q0fevx7Eo2#0t=`Vw!i`iVGAsvs;~tXKnPo40ab-9uz*To3oL*T zw!i|a3R_?SmBJQS03mFF1ymLO4}yihIp$7S`7xw#2EGg7XQ_MWbD*!Sn2$DQtlS5W*H%KviK2ETB@@0t+C7EwF&9!WLKnA#8yKR28WOR;x|>&IlrCod IZt&;-1FlC_y#N3J diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-version-info.json b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-version-info.json deleted file mode 100644 index 63d062713..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/mlc-version-info.json +++ /dev/null @@ -1,634 +0,0 @@ -{ - "app-mlperf-inference,d775cac873ee4231:reference,gptj-99,pytorch,cuda,test,r5.0-dev_default,float16,offline": [ - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline )" - } - }, - { - "get,sys-utils-cm": { - "script_uid": "bc90993277e84b8e", - "script_alias": "get-sys-utils-cm", - "script_tags": "get,sys-utils-cm,sys-utils-mlc", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline )" - } - }, - { - "get,python": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline )" - } - }, - { - "get,mlcommons,inference,src": { - "script_uid": "4b57186581024797", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_variations": "", - "version": "r5.0-git-aea55edb04b10928b75d1cc318d6bed2f2e4a629", - "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "pull-git-repo,c23132ed65c4421d" - } - }, - { - "pull,git,repo": { - "script_uid": "c23132ed65c4421d", - "script_alias": "pull-git-repo", - "script_tags": "pull,git,repo,repository", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline )" - } - }, - { - "get,mlperf,inference,src": { - "script_uid": "4b57186581024797", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_variations": "", - "version": "r5.0-git-aea55edb04b10928b75d1cc318d6bed2f2e4a629", - "parent": "get-mlperf-inference-utils,e341e5f86d8342e5" - } - }, - { - "get,mlperf,inference,utils": { - "script_uid": "e341e5f86d8342e5", - "script_alias": "get-mlperf-inference-utils", - "script_tags": "get,mlperf,inference,util,utils,functions", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "get-cuda,46d133d9ef92422d ( toolkit )" - } - }, - { - "get,cuda,_toolkit": { - "script_uid": "46d133d9ef92422d", - "script_alias": "get-cuda", - "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", - "script_variations": "toolkit", - "version": "12.4", - "parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "detect-cpu,586c8a43320142f7" - } - }, - { - "detect,cpu": { - "script_uid": "586c8a43320142f7", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_variations": "", - "version": "", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( pip )" - } - }, - { - "get,generic-python-lib,_pip": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "pip", - "version": "22.0.2", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.pycuda )" - } - }, - { - "get,generic-python-lib,_package.pycuda": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "package.pycuda", - "version": "2025.1", - "parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "detect-cpu,586c8a43320142f7" - } - }, - { - "detect,cpu": { - "script_uid": "586c8a43320142f7", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_variations": "", - "version": "", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( pip )" - } - }, - { - "get,generic-python-lib,_pip": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "pip", - "version": "22.0.2", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.numpy )" - } - }, - { - "get,generic-python-lib,_package.numpy": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "package.numpy", - "version": "2.2.2", - "parent": "get-cuda-devices,7a3ede4d3558427a ( with-pycuda )" - } - }, - { - "get,cuda-devices,_with-pycuda": { - "script_uid": "7a3ede4d3558427a", - "script_alias": "get-cuda-devices", - "script_tags": "get,cuda-devices", - "script_variations": "with-pycuda", - "version": "", - "parent": "app-mlperf-inference,d775cac873ee4231 ( reference,_gptj-99,_pytorch,_cuda,_test,_r5.0-dev_default,_float16,_offline )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "detect-cpu,586c8a43320142f7" - } - }, - { - "detect,cpu": { - "script_uid": "586c8a43320142f7", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,sys-utils-cm": { - "script_uid": "bc90993277e84b8e", - "script_alias": "get-sys-utils-cm", - "script_tags": "get,sys-utils-cm,sys-utils-mlc", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,python": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,cuda,_cudnn": { - "script_uid": "46d133d9ef92422d", - "script_alias": "get-cuda", - "script_tags": "get,cuda,cuda-compiler,cuda-lib,toolkit,lib,nvcc,get-nvcc,get-cuda,46d133d9ef92422d", - "script_variations": "cudnn", - "version": "12.4", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( torch_cuda )" - } - }, - { - "get,generic-python-lib,_torch_cuda": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "torch_cuda", - "version": "2.6.0", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( torchvision_cuda )" - } - }, - { - "get,generic-python-lib,_torchvision_cuda": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "torchvision_cuda", - "version": "0.21.0", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( transformers )" - } - }, - { - "get,generic-python-lib,_transformers": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "transformers", - "version": "4.48.2", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,ml-model,large-language-model,gptj,raw,_pytorch": { - "script_uid": "a41166210f294fbf", - "script_alias": "get-ml-model-gptj", - "script_tags": "get,raw,ml-model,gptj,gpt-j,large-language-model", - "script_variations": "pytorch", - "version": "", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,dataset,cnndm,_validation": { - "script_uid": "aed298c156e24257", - "script_alias": "get-dataset-cnndm", - "script_tags": "get,dataset,gpt-j,cnndm,cnn-dailymail,original", - "script_variations": "validation", - "version": "", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "detect-cpu,586c8a43320142f7" - } - }, - { - "detect,cpu": { - "script_uid": "586c8a43320142f7", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_variations": "", - "version": "", - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" - } - }, - { - "get,python": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" - } - }, - { - "get,mlcommons,inference,src": { - "script_uid": "4b57186581024797", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_variations": "", - "version": "r5.0-git-aea55edb04b10928b75d1cc318d6bed2f2e4a629", - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" - } - }, - { - "get,cache,dir,_name.mlperf-inference-sut-configs": { - "script_uid": "48f4622e059b45ce", - "script_alias": "get-cache-dir", - "script_tags": "get,cache,dir,directory", - "script_variations": "name.mlperf-inference-sut-configs", - "version": "", - "parent": "get-mlperf-inference-sut-configs,c2fbf72009e2445b" - } - }, - { - "get,sut,configs": { - "script_uid": "c2fbf72009e2445b", - "script_alias": "get-mlperf-inference-sut-configs", - "script_tags": "get,mlperf,inference,sut,configs,sut-configs", - "script_variations": "", - "version": "", - "parent": "generate-mlperf-inference-user-conf,3af4475745964b93" - } - }, - { - "generate,user-conf,mlperf,inference": { - "script_uid": "3af4475745964b93", - "script_alias": "generate-mlperf-inference-user-conf", - "script_tags": "generate,mlperf,inference,user-conf,inference-user-conf", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,loadgen": { - "script_uid": "64c3d98d0ba04950", - "script_alias": "get-mlperf-inference-loadgen", - "script_tags": "get,loadgen,inference,inference-loadgen,mlperf,mlcommons", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,mlcommons,inference,src": { - "script_uid": "4b57186581024797", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_variations": "", - "version": "r5.0-git-aea55edb04b10928b75d1cc318d6bed2f2e4a629", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,mlcommons,inference,src": { - "script_uid": "4b57186581024797", - "script_alias": "get-mlperf-inference-src", - "script_tags": "get,src,source,inference,inference-src,inference-source,mlperf,mlcommons", - "script_variations": "", - "version": "master-git-aea55edb04b10928b75d1cc318d6bed2f2e4a629", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.psutil )" - } - }, - { - "get,generic-python-lib,_package.psutil": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "package.psutil", - "version": "6.1.1", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.datasets )" - } - }, - { - "get,generic-python-lib,_package.datasets": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "package.datasets", - "version": "3.2.0", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.attrs )" - } - }, - { - "get,generic-python-lib,_package.attrs": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "package.attrs", - "version": "25.1.0", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "get,python3": { - "script_uid": "d0b5dd74373f4a62", - "script_alias": "get-python3", - "script_tags": "get,python,python3,get-python,get-python3", - "script_variations": "", - "version": "3.10.12", - "parent": "get-generic-python-lib,94b62a682bc44791 ( package.accelerate )" - } - }, - { - "get,generic-python-lib,_package.accelerate": { - "script_uid": "94b62a682bc44791", - "script_alias": "get-generic-python-lib", - "script_tags": "get,install,generic,pip-package,generic-python-lib", - "script_variations": "package.accelerate", - "version": "1.3.0", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - }, - { - "detect,os": { - "script_uid": "863735b7db8c44fc", - "script_alias": "detect-os", - "script_tags": "detect-os,detect,os,info", - "script_variations": "", - "version": "", - "parent": "detect-cpu,586c8a43320142f7" - } - }, - { - "detect,cpu": { - "script_uid": "586c8a43320142f7", - "script_alias": "detect-cpu", - "script_tags": "detect,cpu,detect-cpu,info", - "script_variations": "", - "version": "", - "parent": "benchmark-program,19f369ef47084895" - } - }, - { - "benchmark-program,program": { - "script_uid": "19f369ef47084895", - "script_alias": "benchmark-program", - "script_tags": "program,benchmark,benchmark-program", - "script_variations": "", - "version": "", - "parent": "benchmark-program-mlperf,cfff0132a8aa4018" - } - }, - { - "benchmark-mlperf": { - "script_uid": "cfff0132a8aa4018", - "script_alias": "benchmark-program-mlperf", - "script_tags": "mlperf,benchmark-mlperf", - "script_variations": "", - "version": "", - "parent": "app-mlperf-inference-mlcommons-python,ff149e9781fc4b65 ( gptj-99,_pytorch,_cuda,_offline,_float16 )" - } - } - ] -} \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/model-info.json b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/model-info.json deleted file mode 100644 index 698385a23..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/model-info.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "starting_weights_filename": "checkpoint-final", - "retraining": "no", - "input_data_types": "fp32", - "weight_data_types": "fp32", - "weight_transformations": "no" -} \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/os_info.json b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/os_info.json deleted file mode 100644 index ebe11bd71..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/os_info.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "MLC_HOST_OS_TYPE": "linux", - "MLC_HOST_OS_BITS": "64", - "MLC_HOST_OS_FLAVOR": "ubuntu", - "MLC_HOST_OS_FLAVOR_LIKE": "debian", - "MLC_HOST_OS_VERSION": "22.04", - "MLC_HOST_OS_KERNEL_VERSION": "6.8.0-52-generic", - "MLC_HOST_OS_GLIBC_VERSION": "2.35", - "MLC_HOST_OS_MACHINE": "x86_64", - "MLC_HOST_OS_PACKAGE_MANAGER": "apt", - "MLC_HOST_OS_PACKAGE_MANAGER_INSTALL_CMD": "DEBIAN_FRONTEND=noninteractive apt-get install -y", - "MLC_HOST_OS_PACKAGE_MANAGER_UPDATE_CMD": "apt-get update -y", - "+MLC_HOST_OS_DEFAULT_LIBRARY_PATH": [ - "/usr/local/lib/x86_64-linux-gnu", - "/lib/x86_64-linux-gnu", - "/usr/lib/x86_64-linux-gnu", - "/usr/lib/x86_64-linux-gnu64", - "/usr/local/lib64", - "/lib64", - "/usr/lib64", - "/usr/local/lib", - "/lib", - "/usr/lib", - "/usr/x86_64-linux-gnu/lib64", - "/usr/x86_64-linux-gnu/lib" - ], - "MLC_HOST_PLATFORM_FLAVOR": "x86_64", - "MLC_HOST_PYTHON_BITS": "64", - "MLC_HOST_SYSTEM_NAME": "0162cdce0038" -} \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance_console.out b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance_console.out deleted file mode 100644 index 57ff14cfd..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/performance_console.out +++ /dev/null @@ -1,72 +0,0 @@ -Constructing QSL -Encoding Samples -Finished constructing QSL. -Loading PyTorch model... - Loading checkpoint shards: 0%| | 0/3 [00:001` or unset `early_stopping`. - warnings.warn( - 100%|██████████| 1/1 [00:01<00:00, 1.42s/it] 100%|██████████| 1/1 [00:01<00:00, 1.42s/it] -================================================ -MLPerf Results Summary -================================================ -SUT name : PySUT -Scenario : Offline -Mode : PerformanceOnly -Samples per second: 0.704645 -Tokens per second (inferred): 48.6205 -Result is : VALID - Min duration satisfied : Yes - Min queries satisfied : Yes - Early stopping satisfied: Yes - -================================================ -Additional Stats -================================================ -Min latency (ns) : 1419154830 -Max latency (ns) : 1419154830 -Mean latency (ns) : 1419154830 -50.00 percentile latency (ns) : 1419154830 -90.00 percentile latency (ns) : 1419154830 -95.00 percentile latency (ns) : 1419154830 -97.00 percentile latency (ns) : 1419154830 -99.00 percentile latency (ns) : 1419154830 -99.90 percentile latency (ns) : 1419154830 - -================================================ -Test Parameters Used -================================================ -samples_per_query : 1 -target_qps : 1 -target_latency (ns): 0 -max_async_queries : 1 -min_duration (ms): 0 -max_duration (ms): 0 -min_query_count : 1 -max_query_count : 1 -qsl_rng_seed : 6023615788873153749 -sample_index_rng_seed : 15036839855038426416 -schedule_rng_seed : 9933818062894767841 -accuracy_log_rng_seed : 0 -accuracy_log_probability : 0 -accuracy_log_sampling_target : 0 -print_timestamps : 0 -performance_issue_unique : 0 -performance_issue_same : 0 -performance_issue_same_index : 0 -performance_sample_count : 13368 - -No warnings encountered during test. - -No errors encountered during test. -Test Done! -Destroying SUT... -Destroying QSL... -Finished destroying SUT. -Finished destroying QSL. diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/pip_freeze.json b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/pip_freeze.json deleted file mode 100644 index 51966ac45..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/pip_freeze.json +++ /dev/null @@ -1,77 +0,0 @@ -{ - "pip_freeze": { - "accelerate": "1.3.0", - "aiohappyeyeballs": "2.4.4", - "aiohttp": "3.11.11", - "aiosignal": "1.3.2", - "async-timeout": "5.0.1", - "attrs": "25.1.0", - "certifi": "2025.1.31", - "charset-normalizer": "3.4.1", - "colorama": "0.4.6", - "datasets": "3.2.0", - "dill": "0.3.8", - "filelock": "3.17.0", - "frozenlist": "1.5.0", - "fsspec": "2024.9.0", - "giturlparse": "0.12.0", - "huggingface-hub": "0.28.1", - "idna": "3.10", - "Jinja2": "3.1.5", - "Mako": "1.3.9", - "MarkupSafe": "3.0.2", - "mlcflow": "0.1.20", - "mpmath": "1.3.0", - "multidict": "6.1.0", - "multiprocess": "0.70.16", - "networkx": "3.4.2", - "numpy": "1.26.4", - "nvidia-cublas-cu12": "12.4.5.8", - "nvidia-cuda-cupti-cu12": "12.4.127", - "nvidia-cuda-nvrtc-cu12": "12.4.127", - "nvidia-cuda-runtime-cu12": "12.4.127", - "nvidia-cudnn-cu12": "9.1.0.70", - "nvidia-cufft-cu12": "11.2.1.3", - "nvidia-curand-cu12": "10.3.5.147", - "nvidia-cusolver-cu12": "11.6.1.9", - "nvidia-cusparse-cu12": "12.3.1.170", - "nvidia-cusparselt-cu12": "0.6.2", - "nvidia-nccl-cu12": "2.21.5", - "nvidia-nvjitlink-cu12": "12.4.127", - "nvidia-nvtx-cu12": "12.4.127", - "packaging": "24.2", - "pandas": "2.2.3", - "pillow": "11.1.0", - "pip": "22.0.2", - "platformdirs": "4.3.7", - "propcache": "0.2.1", - "psutil": "6.1.1", - "pyarrow": "19.0.0", - "pybind11": "2.13.6", - "pycuda": "2025.1", - "python-dateutil": "2.9.0.post0", - "pytools": "2025.1.2", - "pytz": "2025.1", - "PyYAML": "6.0.2", - "regex": "2024.11.6", - "requests": "2.32.3", - "safetensors": "0.5.2", - "setuptools": "59.6.0", - "simplejson": "3.19.3", - "six": "1.17.0", - "sympy": "1.13.1", - "tabulate": "0.9.0", - "tokenizers": "0.21.0", - "torch": "2.6.0", - "torchvision": "0.21.0", - "tqdm": "4.67.1", - "transformers": "4.48.2", - "triton": "3.2.0", - "typing_extensions": "4.12.2", - "tzdata": "2025.1", - "urllib3": "2.3.0", - "wheel": "0.45.1", - "xxhash": "3.5.0", - "yarl": "1.18.3" - } -} \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/system_info.txt b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/system_info.txt deleted file mode 100644 index 7c9f21e24..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/system_info.txt +++ /dev/null @@ -1,2616 +0,0 @@ -Platform Details - ------------------------------------------------------------- -1. uname -a -Linux 0162cdce0038 6.8.0-52-generic #53-Ubuntu SMP PREEMPT_DYNAMIC Sat Jan 11 00:06:25 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux ------------------------------------------------------------- -2. w - 08:53:36 up 59 days, 21:50, 0 users, load average: 1.83, 2.21, 2.76 -USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT ------------------------------------------------------------- -3. Username -mlcuser ------------------------------------------------------------- -4. ulimit -a -real-time non-blocking time (microseconds, -R) unlimited -core file size (blocks, -c) unlimited -data seg size (kbytes, -d) unlimited -scheduling priority (-e) 0 -file size (blocks, -f) unlimited -pending signals (-i) 770559 -max locked memory (kbytes, -l) 8192 -max memory size (kbytes, -m) unlimited -open files (-n) 1048576 -pipe size (512 bytes, -p) 8 -POSIX message queues (bytes, -q) 819200 -real-time priority (-r) 0 -stack size (kbytes, -s) 8192 -cpu time (seconds, -t) unlimited -max user processes (-u) unlimited -virtual memory (kbytes, -v) unlimited -file locks (-x) unlimited ------------------------------------------------------------- -5. sysinfo process ancestry -bash ------------------------------------------------------------- -6. /proc/cpuinfo -processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 1100.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 0 -cpu cores : 24 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 1 -cpu cores : 24 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 2 -cpu cores : 24 -apicid : 4 -initial apicid : 4 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 3 -cpu cores : 24 -apicid : 6 -initial apicid : 6 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 4 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 4 -cpu cores : 24 -apicid : 8 -initial apicid : 8 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 5 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 5 -cpu cores : 24 -apicid : 10 -initial apicid : 10 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 6 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 6 -cpu cores : 24 -apicid : 12 -initial apicid : 12 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 7 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 7 -cpu cores : 24 -apicid : 14 -initial apicid : 14 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 8 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 8 -cpu cores : 24 -apicid : 16 -initial apicid : 16 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 9 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 9 -cpu cores : 24 -apicid : 18 -initial apicid : 18 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 10 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 10 -cpu cores : 24 -apicid : 20 -initial apicid : 20 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 11 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 11 -cpu cores : 24 -apicid : 22 -initial apicid : 22 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 12 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 12 -cpu cores : 24 -apicid : 24 -initial apicid : 24 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 13 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 2723.689 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 13 -cpu cores : 24 -apicid : 26 -initial apicid : 26 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 14 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 4223.033 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 14 -cpu cores : 24 -apicid : 28 -initial apicid : 28 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 15 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 3348.056 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 15 -cpu cores : 24 -apicid : 30 -initial apicid : 30 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 16 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 900.380 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 16 -cpu cores : 24 -apicid : 32 -initial apicid : 32 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 17 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 17 -cpu cores : 24 -apicid : 34 -initial apicid : 34 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 18 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 18 -cpu cores : 24 -apicid : 36 -initial apicid : 36 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 19 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 2172.987 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 19 -cpu cores : 24 -apicid : 38 -initial apicid : 38 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 20 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 20 -cpu cores : 24 -apicid : 40 -initial apicid : 40 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 21 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 798.951 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 21 -cpu cores : 24 -apicid : 42 -initial apicid : 42 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 22 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 22 -cpu cores : 24 -apicid : 44 -initial apicid : 44 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 23 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 23 -cpu cores : 24 -apicid : 46 -initial apicid : 46 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 24 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 0 -cpu cores : 24 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 25 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 1 -cpu cores : 24 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 26 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 2 -cpu cores : 24 -apicid : 5 -initial apicid : 5 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 27 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 3 -cpu cores : 24 -apicid : 7 -initial apicid : 7 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 28 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 801.025 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 4 -cpu cores : 24 -apicid : 9 -initial apicid : 9 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 29 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 5 -cpu cores : 24 -apicid : 11 -initial apicid : 11 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 30 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 6 -cpu cores : 24 -apicid : 13 -initial apicid : 13 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 31 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 7 -cpu cores : 24 -apicid : 15 -initial apicid : 15 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 32 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 8 -cpu cores : 24 -apicid : 17 -initial apicid : 17 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 33 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 9 -cpu cores : 24 -apicid : 19 -initial apicid : 19 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 34 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 10 -cpu cores : 24 -apicid : 21 -initial apicid : 21 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 35 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 11 -cpu cores : 24 -apicid : 23 -initial apicid : 23 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 36 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.091 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 12 -cpu cores : 24 -apicid : 25 -initial apicid : 25 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 37 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 13 -cpu cores : 24 -apicid : 27 -initial apicid : 27 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 38 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 14 -cpu cores : 24 -apicid : 29 -initial apicid : 29 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 39 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 15 -cpu cores : 24 -apicid : 31 -initial apicid : 31 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 40 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 16 -cpu cores : 24 -apicid : 33 -initial apicid : 33 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 41 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 1500.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 17 -cpu cores : 24 -apicid : 35 -initial apicid : 35 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 42 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 18 -cpu cores : 24 -apicid : 37 -initial apicid : 37 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 43 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 970.771 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 19 -cpu cores : 24 -apicid : 39 -initial apicid : 39 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 44 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 20 -cpu cores : 24 -apicid : 41 -initial apicid : 41 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 45 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 21 -cpu cores : 24 -apicid : 43 -initial apicid : 43 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 46 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 22 -cpu cores : 24 -apicid : 45 -initial apicid : 45 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 47 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 23 -cpu cores : 24 -apicid : 47 -initial apicid : 47 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - ------------------------------------------------------------- -7. lscpu -Architecture: x86_64 -CPU op-mode(s): 32-bit, 64-bit -Address sizes: 52 bits physical, 57 bits virtual -Byte Order: Little Endian -CPU(s): 48 -On-line CPU(s) list: 0-47 -Vendor ID: GenuineIntel -Model name: Intel(R) Xeon(R) w7-2495X -CPU family: 6 -Model: 143 -Thread(s) per core: 2 -Core(s) per socket: 24 -Socket(s): 1 -Stepping: 8 -CPU max MHz: 4800.0000 -CPU min MHz: 800.0000 -BogoMIPS: 4992.00 -Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -Virtualization: VT-x -L1d cache: 1.1 MiB (24 instances) -L1i cache: 768 KiB (24 instances) -L2 cache: 48 MiB (24 instances) -L3 cache: 45 MiB (1 instance) -NUMA node(s): 1 -NUMA node0 CPU(s): 0-47 -Vulnerability Gather data sampling: Not affected -Vulnerability Itlb multihit: Not affected -Vulnerability L1tf: Not affected -Vulnerability Mds: Not affected -Vulnerability Meltdown: Not affected -Vulnerability Mmio stale data: Not affected -Vulnerability Reg file data sampling: Not affected -Vulnerability Retbleed: Not affected -Vulnerability Spec rstack overflow: Not affected -Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl -Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization -Vulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI BHI_DIS_S -Vulnerability Srbds: Not affected -Vulnerability Tsx async abort: Not affected ------------------------------------------------------------- -8. numactl --hardware -available: 1 nodes (0) -node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 -node 0 size: 192709 MB -node 0 free: 79355 MB -node distances: -node 0 - 0: 10 ------------------------------------------------------------- -9. /proc/meminfo -MemTotal: 197334528 kB -MemFree: 81260148 kB -MemAvailable: 192270948 kB -Buffers: 580472 kB -Cached: 110942764 kB -SwapCached: 63040 kB -Active: 32407452 kB -Inactive: 79605680 kB -Active(anon): 457776 kB -Inactive(anon): 74116 kB -Active(file): 31949676 kB -Inactive(file): 79531564 kB -Unevictable: 6160 kB -Mlocked: 0 kB -SwapTotal: 8388604 kB -SwapFree: 7703112 kB -Zswap: 0 kB -Zswapped: 0 kB -Dirty: 21108 kB -Writeback: 0 kB -AnonPages: 486956 kB -Mapped: 311000 kB -Shmem: 41996 kB -KReclaimable: 1566044 kB -Slab: 2381696 kB -SReclaimable: 1566044 kB -SUnreclaim: 815652 kB -KernelStack: 20272 kB -PageTables: 33284 kB -SecPageTables: 0 kB -NFS_Unstable: 0 kB -Bounce: 0 kB -WritebackTmp: 0 kB -CommitLimit: 107055868 kB -Committed_AS: 6897048 kB -VmallocTotal: 13743895347199 kB -VmallocUsed: 157652 kB -VmallocChunk: 0 kB -Percpu: 122880 kB -HardwareCorrupted: 56 kB -AnonHugePages: 0 kB -ShmemHugePages: 0 kB -ShmemPmdMapped: 0 kB -FileHugePages: 0 kB -FilePmdMapped: 0 kB -Unaccepted: 0 kB -HugePages_Total: 0 -HugePages_Free: 0 -HugePages_Rsvd: 0 -HugePages_Surp: 0 -Hugepagesize: 2048 kB -Hugetlb: 0 kB -DirectMap4k: 70549092 kB -DirectMap2M: 127111168 kB -DirectMap1G: 5242880 kB ------------------------------------------------------------- -10. who -r ------------------------------------------------------------- -11. Systemd service manager version -systemd 219 ------------------------------------------------------------- -12. Services, from systemctl list-unit-files -UNIT FILE STATE -basic.target static -default.target enabled -graphical.target disabled -local-fs.target static -multi-user.target enabled -network.target static -nfs-client.target disabled -poweroff.target static -reboot.target static -remote-fs.target enabled -rescue.target static -sysinit.target static -timers.target static -apt-daily-upgrade.service static -apt-daily-upgrade.timer enabled -apt-daily.service static -apt-daily.timer enabled -dpkg-db-backup.service static -dpkg-db-backup.timer enabled -e2scrub@.service static -e2scrub_all.service static -e2scrub_all.timer enabled -e2scrub_fail@.service static -e2scrub_reap.service enabled -fstrim.service static -fstrim.timer enabled -hwclock.sh.service disabled -motd-news.service static -motd-news.timer enabled -procps.service disabled -rsync.service disabled -sudo.service masked - -32 unit files listed. ------------------------------------------------------------- -13. Linux kernel boot-time arguments, from /proc/cmdline -BOOT_IMAGE=/boot/vmlinuz-6.8.0-52-generic root=UUID=cd1dd39e-4e7b-4be5-8864-c6790fa67706 ro quiet splash vt.handoff=7 ------------------------------------------------------------- -14. cpupower frequency-info -analyzing CPU 14: - driver: intel_pstate - CPUs which run at the same hardware frequency: 14 - CPUs which need to have their frequency coordinated by software: 14 - maximum transition latency: Cannot determine or is not supported. - hardware limits: 800 MHz - 4.80 GHz - available cpufreq governors: performance powersave - current policy: frequency should be within 800 MHz and 4.80 GHz. - The governor "powersave" may decide which speed to use - within this range. - current CPU frequency: Unable to call hardware - current CPU frequency: 900 MHz (asserted by call to kernel) - boost state support: - Supported: yes - Active: yes ------------------------------------------------------------- -15. sysctl -abi.vsyscall32 = 1 -debug.exception-trace = 1 -debug.kprobes-optimization = 1 -dev.cdrom.autoclose = 1 -dev.cdrom.autoeject = 0 -dev.cdrom.check_media = 0 -dev.cdrom.debug = 0 -dev.cdrom.info = CD-ROM information, Id: cdrom.c 3.20 2003/12/17 -dev.cdrom.info = -dev.cdrom.info = drive name: sr0 -dev.cdrom.info = drive speed: 1 -dev.cdrom.info = drive # of slots: 1 -dev.cdrom.info = Can close tray: 1 -dev.cdrom.info = Can open tray: 1 -dev.cdrom.info = Can lock tray: 1 -dev.cdrom.info = Can change speed: 0 -dev.cdrom.info = Can select disk: 0 -dev.cdrom.info = Can read multisession: 1 -dev.cdrom.info = Can read MCN: 1 -dev.cdrom.info = Reports media changed: 1 -dev.cdrom.info = Can play audio: 1 -dev.cdrom.info = Can write CD-R: 0 -dev.cdrom.info = Can write CD-RW: 0 -dev.cdrom.info = Can read DVD: 0 -dev.cdrom.info = Can write DVD-R: 0 -dev.cdrom.info = Can write DVD-RAM: 0 -dev.cdrom.info = Can read MRW: 0 -dev.cdrom.info = Can write MRW: 0 -dev.cdrom.info = Can write RAM: 0 -dev.cdrom.info = -dev.cdrom.info = -dev.cdrom.lock = 0 -dev.hpet.max-user-freq = 64 -dev.mac_hid.mouse_button2_keycode = 97 -dev.mac_hid.mouse_button3_keycode = 100 -dev.mac_hid.mouse_button_emulation = 0 -dev.parport.default.spintime = 500 -dev.parport.default.timeslice = 200 -dev.raid.speed_limit_max = 200000 -dev.raid.speed_limit_min = 1000 -dev.scsi.logging_level = 0 -dev.tty.ldisc_autoload = 1 -dev.tty.legacy_tiocsti = 0 -fs.aio-max-nr = 65536 -fs.aio-nr = 0 -fs.dentry-state = 1059595 899336 45 0 315521 0 -fs.dir-notify-enable = 1 -fs.epoll.max_user_watches = 43912503 -fs.fanotify.max_queued_events = 16384 -fs.fanotify.max_user_groups = 128 -fs.fanotify.max_user_marks = 1048576 -fs.file-max = 9223372036854775807 -fs.file-nr = 10784 0 9223372036854775807 -fs.inode-nr = 773459 39689 -fs.inode-state = 773459 39689 0 0 0 0 0 -fs.inotify.max_queued_events = 16384 -fs.inotify.max_user_instances = 128 -fs.inotify.max_user_watches = 65536 -fs.lease-break-time = 45 -fs.leases-enable = 1 -fs.mount-max = 100000 -fs.mqueue.msg_default = 10 -fs.mqueue.msg_max = 10 -fs.mqueue.msgsize_default = 8192 -fs.mqueue.msgsize_max = 8192 -fs.mqueue.queues_max = 256 -fs.nr_open = 1048576 -fs.overflowgid = 65534 -fs.overflowuid = 65534 -fs.pipe-max-size = 1048576 -fs.pipe-user-pages-hard = 0 -fs.pipe-user-pages-soft = 16384 -fs.protected_fifos = 1 -fs.protected_hardlinks = 1 -fs.protected_regular = 2 -fs.protected_symlinks = 1 -fs.quota.allocated_dquots = 0 -fs.quota.cache_hits = 0 -fs.quota.drops = 0 -fs.quota.free_dquots = 0 -fs.quota.lookups = 0 -fs.quota.reads = 0 -fs.quota.syncs = 2870 -fs.quota.writes = 0 -fs.suid_dumpable = 2 -fs.verity.require_signatures = 0 -kernel.acct = 4 2 30 -kernel.acpi_video_flags = 0 -kernel.apparmor_restrict_unprivileged_unconfined = 0 -kernel.apparmor_restrict_unprivileged_userns = 1 -kernel.arch = x86_64 -kernel.auto_msgmni = 0 -kernel.bootloader_type = 33 -kernel.bootloader_version = 1 -kernel.bpf_stats_enabled = 0 -kernel.cad_pid = 0 -kernel.cap_last_cap = 40 -kernel.core_pattern = |/usr/share/apport/apport -p%p -s%s -c%c -d%d -P%P -u%u -g%g -- %E -kernel.core_pipe_limit = 10 -kernel.core_uses_pid = 0 -kernel.ctrl-alt-del = 0 -kernel.dmesg_restrict = 1 -kernel.domainname = (none) -kernel.firmware_config.force_sysfs_fallback = 0 -kernel.firmware_config.ignore_sysfs_fallback = 0 -kernel.ftrace_dump_on_oops = 0 -kernel.ftrace_enabled = 1 -kernel.hardlockup_all_cpu_backtrace = 0 -kernel.hardlockup_panic = 0 -kernel.hostname = 0162cdce0038 -kernel.hotplug = -kernel.hung_task_all_cpu_backtrace = 0 -kernel.hung_task_check_count = 4194304 -kernel.hung_task_check_interval_secs = 0 -kernel.hung_task_panic = 0 -kernel.hung_task_timeout_secs = 120 -kernel.hung_task_warnings = 10 -kernel.io_delay_type = 1 -kernel.io_uring_disabled = 0 -kernel.io_uring_group = -1 -kernel.kexec_load_disabled = 0 -kernel.kexec_load_limit_panic = -1 -kernel.kexec_load_limit_reboot = -1 -kernel.keys.gc_delay = 300 -kernel.keys.maxbytes = 20000 -kernel.keys.maxkeys = 200 -kernel.keys.persistent_keyring_expiry = 259200 -kernel.keys.root_maxbytes = 25000000 -kernel.keys.root_maxkeys = 1000000 -kernel.kptr_restrict = 1 -kernel.latencytop = 0 -kernel.max_lock_depth = 1024 -kernel.max_rcu_stall_to_panic = 0 -kernel.modprobe = /sbin/modprobe -kernel.modules_disabled = 0 -kernel.msg_next_id = -1 -kernel.msgmax = 8192 -kernel.msgmnb = 16384 -kernel.msgmni = 32000 -kernel.ngroups_max = 65536 -kernel.nmi_watchdog = 1 -kernel.ns_last_pid = 7592 -kernel.numa_balancing = 0 -kernel.numa_balancing_promote_rate_limit_MBps = 65536 -kernel.oops_all_cpu_backtrace = 0 -kernel.oops_limit = 10000 -kernel.osrelease = 6.8.0-52-generic -kernel.ostype = Linux -kernel.overflowgid = 65534 -kernel.overflowuid = 65534 -kernel.panic = 0 -kernel.panic_on_io_nmi = 0 -kernel.panic_on_oops = 0 -kernel.panic_on_rcu_stall = 0 -kernel.panic_on_unrecovered_nmi = 0 -kernel.panic_on_warn = 0 -kernel.panic_print = 0 -kernel.perf_cpu_time_max_percent = 25 -kernel.perf_event_max_contexts_per_stack = 8 -kernel.perf_event_max_sample_rate = 4000 -kernel.perf_event_max_stack = 127 -kernel.perf_event_mlock_kb = 516 -kernel.perf_event_paranoid = 4 -kernel.pid_max = 4194304 -kernel.poweroff_cmd = /sbin/poweroff -kernel.print-fatal-signals = 0 -kernel.printk = 4 4 1 7 -kernel.printk_delay = 0 -kernel.printk_devkmsg = on -kernel.printk_ratelimit = 5 -kernel.printk_ratelimit_burst = 10 -kernel.pty.max = 4096 -kernel.pty.nr = 6 -kernel.pty.reserve = 1024 -kernel.random.boot_id = 266aa90c-a7a3-400b-90af-2740886ad693 -kernel.random.entropy_avail = 256 -kernel.random.poolsize = 256 -kernel.random.urandom_min_reseed_secs = 60 -kernel.random.uuid = 70480901-6603-4a55-9122-c38014add3cf -kernel.random.write_wakeup_threshold = 256 -kernel.randomize_va_space = 2 -kernel.real-root-dev = 0 -kernel.sched_autogroup_enabled = 1 -kernel.sched_cfs_bandwidth_slice_us = 5000 -kernel.sched_deadline_period_max_us = 4194304 -kernel.sched_deadline_period_min_us = 100 -kernel.sched_itmt_enabled = 1 -kernel.sched_rr_timeslice_ms = 100 -kernel.sched_rt_period_us = 1000000 -kernel.sched_rt_runtime_us = 950000 -kernel.sched_schedstats = 0 -kernel.sched_util_clamp_max = 1024 -kernel.sched_util_clamp_min = 1024 -kernel.sched_util_clamp_min_rt_default = 1024 -kernel.seccomp.actions_avail = kill_process kill_thread trap errno user_notif trace log allow -kernel.seccomp.actions_logged = kill_process kill_thread trap errno user_notif trace log -kernel.sem = 32000 1024000000 500 32000 -kernel.sem_next_id = -1 -kernel.shm_next_id = -1 -kernel.shm_rmid_forced = 0 -kernel.shmall = 18446744073692774399 -kernel.shmmax = 18446744073692774399 -kernel.shmmni = 4096 -kernel.soft_watchdog = 1 -kernel.softlockup_all_cpu_backtrace = 0 -kernel.softlockup_panic = 0 -kernel.split_lock_mitigate = 1 -kernel.stack_tracer_enabled = 0 -kernel.sysctl_writes_strict = 1 -kernel.sysrq = 176 -kernel.tainted = 20993 -kernel.task_delayacct = 0 -kernel.threads-max = 1541118 -kernel.timer_migration = 1 -kernel.traceoff_on_warning = 0 -kernel.tracepoint_printk = 0 -kernel.unknown_nmi_panic = 0 -kernel.unprivileged_bpf_disabled = 2 -kernel.unprivileged_userns_clone = 1 -kernel.user_events_max = 32768 -kernel.usermodehelper.bset = 4294967295 511 -kernel.usermodehelper.inheritable = 4294967295 511 -kernel.version = #53-Ubuntu SMP PREEMPT_DYNAMIC Sat Jan 11 00:06:25 UTC 2025 -kernel.warn_limit = 0 -kernel.watchdog = 1 -kernel.watchdog_cpumask = 0-47 -kernel.watchdog_thresh = 10 -kernel.yama.ptrace_scope = 1 -net.core.optmem_max = 131072 -net.core.rps_default_mask = 0000,00000000 -net.core.somaxconn = 4096 -net.core.txrehash = 1 -net.core.xfrm_acq_expires = 30 -net.core.xfrm_aevent_etime = 10 -net.core.xfrm_aevent_rseqth = 2 -net.core.xfrm_larval_drop = 1 -net.ipv4.conf.all.accept_local = 0 -net.ipv4.conf.all.accept_redirects = 0 -net.ipv4.conf.all.accept_source_route = 0 -net.ipv4.conf.all.arp_accept = 0 -net.ipv4.conf.all.arp_announce = 0 -net.ipv4.conf.all.arp_evict_nocarrier = 1 -net.ipv4.conf.all.arp_filter = 0 -net.ipv4.conf.all.arp_ignore = 0 -net.ipv4.conf.all.arp_notify = 0 -net.ipv4.conf.all.bc_forwarding = 0 -net.ipv4.conf.all.bootp_relay = 0 -net.ipv4.conf.all.disable_policy = 0 -net.ipv4.conf.all.disable_xfrm = 0 -net.ipv4.conf.all.drop_gratuitous_arp = 0 -net.ipv4.conf.all.drop_unicast_in_l2_multicast = 0 -net.ipv4.conf.all.force_igmp_version = 0 -net.ipv4.conf.all.forwarding = 1 -net.ipv4.conf.all.igmpv2_unsolicited_report_interval = 10000 -net.ipv4.conf.all.igmpv3_unsolicited_report_interval = 1000 -net.ipv4.conf.all.ignore_routes_with_linkdown = 0 -net.ipv4.conf.all.log_martians = 0 -net.ipv4.conf.all.mc_forwarding = 0 -net.ipv4.conf.all.medium_id = 0 -net.ipv4.conf.all.promote_secondaries = 0 -net.ipv4.conf.all.proxy_arp = 0 -net.ipv4.conf.all.proxy_arp_pvlan = 0 -net.ipv4.conf.all.route_localnet = 0 -net.ipv4.conf.all.rp_filter = 2 -net.ipv4.conf.all.secure_redirects = 1 -net.ipv4.conf.all.send_redirects = 1 -net.ipv4.conf.all.shared_media = 1 -net.ipv4.conf.all.src_valid_mark = 0 -net.ipv4.conf.all.tag = 0 -net.ipv4.conf.default.accept_local = 0 -net.ipv4.conf.default.accept_redirects = 1 -net.ipv4.conf.default.accept_source_route = 1 -net.ipv4.conf.default.arp_accept = 0 -net.ipv4.conf.default.arp_announce = 0 -net.ipv4.conf.default.arp_evict_nocarrier = 1 -net.ipv4.conf.default.arp_filter = 0 -net.ipv4.conf.default.arp_ignore = 0 -net.ipv4.conf.default.arp_notify = 0 -net.ipv4.conf.default.bc_forwarding = 0 -net.ipv4.conf.default.bootp_relay = 0 -net.ipv4.conf.default.disable_policy = 0 -net.ipv4.conf.default.disable_xfrm = 0 -net.ipv4.conf.default.drop_gratuitous_arp = 0 -net.ipv4.conf.default.drop_unicast_in_l2_multicast = 0 -net.ipv4.conf.default.force_igmp_version = 0 -net.ipv4.conf.default.forwarding = 1 -net.ipv4.conf.default.igmpv2_unsolicited_report_interval = 10000 -net.ipv4.conf.default.igmpv3_unsolicited_report_interval = 1000 -net.ipv4.conf.default.ignore_routes_with_linkdown = 0 -net.ipv4.conf.default.log_martians = 0 -net.ipv4.conf.default.mc_forwarding = 0 -net.ipv4.conf.default.medium_id = 0 -net.ipv4.conf.default.promote_secondaries = 0 -net.ipv4.conf.default.proxy_arp = 0 -net.ipv4.conf.default.proxy_arp_pvlan = 0 -net.ipv4.conf.default.route_localnet = 0 -net.ipv4.conf.default.rp_filter = 2 -net.ipv4.conf.default.secure_redirects = 1 -net.ipv4.conf.default.send_redirects = 1 -net.ipv4.conf.default.shared_media = 1 -net.ipv4.conf.default.src_valid_mark = 0 -net.ipv4.conf.default.tag = 0 -net.ipv4.conf.eth0.accept_local = 0 -net.ipv4.conf.eth0.accept_redirects = 1 -net.ipv4.conf.eth0.accept_source_route = 1 -net.ipv4.conf.eth0.arp_accept = 0 -net.ipv4.conf.eth0.arp_announce = 0 -net.ipv4.conf.eth0.arp_evict_nocarrier = 1 -net.ipv4.conf.eth0.arp_filter = 0 -net.ipv4.conf.eth0.arp_ignore = 0 -net.ipv4.conf.eth0.arp_notify = 0 -net.ipv4.conf.eth0.bc_forwarding = 0 -net.ipv4.conf.eth0.bootp_relay = 0 -net.ipv4.conf.eth0.disable_policy = 0 -net.ipv4.conf.eth0.disable_xfrm = 0 -net.ipv4.conf.eth0.drop_gratuitous_arp = 0 -net.ipv4.conf.eth0.drop_unicast_in_l2_multicast = 0 -net.ipv4.conf.eth0.force_igmp_version = 0 -net.ipv4.conf.eth0.forwarding = 1 -net.ipv4.conf.eth0.igmpv2_unsolicited_report_interval = 10000 -net.ipv4.conf.eth0.igmpv3_unsolicited_report_interval = 1000 -net.ipv4.conf.eth0.ignore_routes_with_linkdown = 0 -net.ipv4.conf.eth0.log_martians = 0 -net.ipv4.conf.eth0.mc_forwarding = 0 -net.ipv4.conf.eth0.medium_id = 0 -net.ipv4.conf.eth0.promote_secondaries = 0 -net.ipv4.conf.eth0.proxy_arp = 0 -net.ipv4.conf.eth0.proxy_arp_pvlan = 0 -net.ipv4.conf.eth0.route_localnet = 0 -net.ipv4.conf.eth0.rp_filter = 2 -net.ipv4.conf.eth0.secure_redirects = 1 -net.ipv4.conf.eth0.send_redirects = 1 -net.ipv4.conf.eth0.shared_media = 1 -net.ipv4.conf.eth0.src_valid_mark = 0 -net.ipv4.conf.eth0.tag = 0 -net.ipv4.conf.lo.accept_local = 0 -net.ipv4.conf.lo.accept_redirects = 1 -net.ipv4.conf.lo.accept_source_route = 1 -net.ipv4.conf.lo.arp_accept = 0 -net.ipv4.conf.lo.arp_announce = 0 -net.ipv4.conf.lo.arp_evict_nocarrier = 1 -net.ipv4.conf.lo.arp_filter = 0 -net.ipv4.conf.lo.arp_ignore = 0 -net.ipv4.conf.lo.arp_notify = 0 -net.ipv4.conf.lo.bc_forwarding = 0 -net.ipv4.conf.lo.bootp_relay = 0 -net.ipv4.conf.lo.disable_policy = 1 -net.ipv4.conf.lo.disable_xfrm = 1 -net.ipv4.conf.lo.drop_gratuitous_arp = 0 -net.ipv4.conf.lo.drop_unicast_in_l2_multicast = 0 -net.ipv4.conf.lo.force_igmp_version = 0 -net.ipv4.conf.lo.forwarding = 1 -net.ipv4.conf.lo.igmpv2_unsolicited_report_interval = 10000 -net.ipv4.conf.lo.igmpv3_unsolicited_report_interval = 1000 -net.ipv4.conf.lo.ignore_routes_with_linkdown = 0 -net.ipv4.conf.lo.log_martians = 0 -net.ipv4.conf.lo.mc_forwarding = 0 -net.ipv4.conf.lo.medium_id = 0 -net.ipv4.conf.lo.promote_secondaries = 0 -net.ipv4.conf.lo.proxy_arp = 0 -net.ipv4.conf.lo.proxy_arp_pvlan = 0 -net.ipv4.conf.lo.route_localnet = 0 -net.ipv4.conf.lo.rp_filter = 2 -net.ipv4.conf.lo.secure_redirects = 1 -net.ipv4.conf.lo.send_redirects = 1 -net.ipv4.conf.lo.shared_media = 1 -net.ipv4.conf.lo.src_valid_mark = 0 -net.ipv4.conf.lo.tag = 0 -net.ipv4.fib_multipath_hash_fields = 7 -net.ipv4.fib_multipath_hash_policy = 0 -net.ipv4.fib_multipath_use_neigh = 0 -net.ipv4.fib_notify_on_flag_change = 0 -net.ipv4.fwmark_reflect = 0 -net.ipv4.icmp_echo_enable_probe = 0 -net.ipv4.icmp_echo_ignore_all = 0 -net.ipv4.icmp_echo_ignore_broadcasts = 1 -net.ipv4.icmp_errors_use_inbound_ifaddr = 0 -net.ipv4.icmp_ignore_bogus_error_responses = 1 -net.ipv4.icmp_ratelimit = 1000 -net.ipv4.icmp_ratemask = 6168 -net.ipv4.igmp_link_local_mcast_reports = 1 -net.ipv4.igmp_max_memberships = 20 -net.ipv4.igmp_max_msf = 10 -net.ipv4.igmp_qrv = 2 -net.ipv4.ip_autobind_reuse = 0 -net.ipv4.ip_default_ttl = 64 -net.ipv4.ip_dynaddr = 0 -net.ipv4.ip_early_demux = 1 -net.ipv4.ip_forward = 1 -net.ipv4.ip_forward_update_priority = 1 -net.ipv4.ip_forward_use_pmtu = 0 -net.ipv4.ip_local_port_range = 32768 60999 -net.ipv4.ip_local_reserved_ports = -net.ipv4.ip_no_pmtu_disc = 0 -net.ipv4.ip_nonlocal_bind = 0 -net.ipv4.ip_unprivileged_port_start = 0 -net.ipv4.ipfrag_high_thresh = 4194304 -net.ipv4.ipfrag_low_thresh = 3145728 -net.ipv4.ipfrag_max_dist = 64 -net.ipv4.ipfrag_time = 30 -net.ipv4.neigh.eth0.anycast_delay = 100 -net.ipv4.neigh.eth0.app_solicit = 0 -net.ipv4.neigh.eth0.base_reachable_time_ms = 30000 -net.ipv4.neigh.eth0.delay_first_probe_time = 5 -net.ipv4.neigh.eth0.gc_stale_time = 60 -net.ipv4.neigh.eth0.interval_probe_time_ms = 5000 -net.ipv4.neigh.eth0.locktime = 100 -net.ipv4.neigh.eth0.mcast_resolicit = 0 -net.ipv4.neigh.eth0.mcast_solicit = 3 -net.ipv4.neigh.eth0.proxy_delay = 80 -net.ipv4.neigh.eth0.proxy_qlen = 64 -net.ipv4.neigh.eth0.retrans_time_ms = 1000 -net.ipv4.neigh.eth0.ucast_solicit = 3 -net.ipv4.neigh.eth0.unres_qlen = 101 -net.ipv4.neigh.eth0.unres_qlen_bytes = 212992 -net.ipv4.neigh.lo.anycast_delay = 100 -net.ipv4.neigh.lo.app_solicit = 0 -net.ipv4.neigh.lo.base_reachable_time_ms = 30000 -net.ipv4.neigh.lo.delay_first_probe_time = 5 -net.ipv4.neigh.lo.gc_stale_time = 60 -net.ipv4.neigh.lo.interval_probe_time_ms = 5000 -net.ipv4.neigh.lo.locktime = 100 -net.ipv4.neigh.lo.mcast_resolicit = 0 -net.ipv4.neigh.lo.mcast_solicit = 3 -net.ipv4.neigh.lo.proxy_delay = 80 -net.ipv4.neigh.lo.proxy_qlen = 64 -net.ipv4.neigh.lo.retrans_time_ms = 1000 -net.ipv4.neigh.lo.ucast_solicit = 3 -net.ipv4.neigh.lo.unres_qlen = 101 -net.ipv4.neigh.lo.unres_qlen_bytes = 212992 -net.ipv4.nexthop_compat_mode = 1 -net.ipv4.ping_group_range = 0 2147483647 -net.ipv4.raw_l3mdev_accept = 1 -net.ipv4.route.min_adv_mss = 256 -net.ipv4.route.min_pmtu = 552 -net.ipv4.route.mtu_expires = 600 -net.ipv4.tcp_abort_on_overflow = 0 -net.ipv4.tcp_adv_win_scale = 1 -net.ipv4.tcp_allowed_congestion_control = reno cubic -net.ipv4.tcp_app_win = 31 -net.ipv4.tcp_autocorking = 1 -net.ipv4.tcp_available_congestion_control = reno cubic -net.ipv4.tcp_backlog_ack_defer = 1 -net.ipv4.tcp_base_mss = 1024 -net.ipv4.tcp_challenge_ack_limit = 2147483647 -net.ipv4.tcp_child_ehash_entries = 0 -net.ipv4.tcp_comp_sack_delay_ns = 1000000 -net.ipv4.tcp_comp_sack_nr = 44 -net.ipv4.tcp_comp_sack_slack_ns = 100000 -net.ipv4.tcp_congestion_control = cubic -net.ipv4.tcp_dsack = 1 -net.ipv4.tcp_early_demux = 1 -net.ipv4.tcp_early_retrans = 3 -net.ipv4.tcp_ecn = 2 -net.ipv4.tcp_ecn_fallback = 1 -net.ipv4.tcp_ehash_entries = -524288 -net.ipv4.tcp_fack = 0 -net.ipv4.tcp_fastopen = 1 -net.ipv4.tcp_fastopen_blackhole_timeout_sec = 0 -net.ipv4.tcp_fastopen_key = 00000000-00000000-00000000-00000000 -net.ipv4.tcp_fin_timeout = 60 -net.ipv4.tcp_frto = 2 -net.ipv4.tcp_fwmark_accept = 0 -net.ipv4.tcp_invalid_ratelimit = 500 -net.ipv4.tcp_keepalive_intvl = 75 -net.ipv4.tcp_keepalive_probes = 9 -net.ipv4.tcp_keepalive_time = 7200 -net.ipv4.tcp_l3mdev_accept = 0 -net.ipv4.tcp_limit_output_bytes = 1048576 -net.ipv4.tcp_max_reordering = 300 -net.ipv4.tcp_max_syn_backlog = 4096 -net.ipv4.tcp_max_tw_buckets = 262144 -net.ipv4.tcp_migrate_req = 0 -net.ipv4.tcp_min_rtt_wlen = 300 -net.ipv4.tcp_min_snd_mss = 48 -net.ipv4.tcp_min_tso_segs = 2 -net.ipv4.tcp_moderate_rcvbuf = 1 -net.ipv4.tcp_mtu_probe_floor = 48 -net.ipv4.tcp_mtu_probing = 0 -net.ipv4.tcp_no_metrics_save = 0 -net.ipv4.tcp_no_ssthresh_metrics_save = 1 -net.ipv4.tcp_notsent_lowat = 4294967295 -net.ipv4.tcp_orphan_retries = 0 -net.ipv4.tcp_pacing_ca_ratio = 120 -net.ipv4.tcp_pacing_ss_ratio = 200 -net.ipv4.tcp_pingpong_thresh = 1 -net.ipv4.tcp_plb_cong_thresh = 128 -net.ipv4.tcp_plb_enabled = 0 -net.ipv4.tcp_plb_idle_rehash_rounds = 3 -net.ipv4.tcp_plb_rehash_rounds = 12 -net.ipv4.tcp_plb_suspend_rto_sec = 60 -net.ipv4.tcp_probe_interval = 600 -net.ipv4.tcp_probe_threshold = 8 -net.ipv4.tcp_recovery = 1 -net.ipv4.tcp_reflect_tos = 0 -net.ipv4.tcp_reordering = 3 -net.ipv4.tcp_retrans_collapse = 1 -net.ipv4.tcp_retries1 = 3 -net.ipv4.tcp_retries2 = 15 -net.ipv4.tcp_rfc1337 = 0 -net.ipv4.tcp_rmem = 4096 131072 6291456 -net.ipv4.tcp_sack = 1 -net.ipv4.tcp_shrink_window = 0 -net.ipv4.tcp_slow_start_after_idle = 1 -net.ipv4.tcp_stdurg = 0 -net.ipv4.tcp_syn_linear_timeouts = 4 -net.ipv4.tcp_syn_retries = 6 -net.ipv4.tcp_synack_retries = 5 -net.ipv4.tcp_syncookies = 1 -net.ipv4.tcp_thin_linear_timeouts = 0 -net.ipv4.tcp_timestamps = 1 -net.ipv4.tcp_tso_rtt_log = 9 -net.ipv4.tcp_tso_win_divisor = 3 -net.ipv4.tcp_tw_reuse = 2 -net.ipv4.tcp_window_scaling = 1 -net.ipv4.tcp_wmem = 4096 16384 4194304 -net.ipv4.tcp_workaround_signed_windows = 0 -net.ipv4.udp_child_hash_entries = 0 -net.ipv4.udp_early_demux = 1 -net.ipv4.udp_hash_entries = -65536 -net.ipv4.udp_l3mdev_accept = 0 -net.ipv4.udp_rmem_min = 4096 -net.ipv4.udp_wmem_min = 4096 -net.ipv4.xfrm4_gc_thresh = 32768 -net.ipv6.anycast_src_echo_reply = 0 -net.ipv6.auto_flowlabels = 1 -net.ipv6.bindv6only = 0 -net.ipv6.conf.all.accept_dad = 0 -net.ipv6.conf.all.accept_ra = 1 -net.ipv6.conf.all.accept_ra_defrtr = 1 -net.ipv6.conf.all.accept_ra_from_local = 0 -net.ipv6.conf.all.accept_ra_min_hop_limit = 1 -net.ipv6.conf.all.accept_ra_min_lft = 0 -net.ipv6.conf.all.accept_ra_mtu = 1 -net.ipv6.conf.all.accept_ra_pinfo = 1 -net.ipv6.conf.all.accept_ra_rt_info_max_plen = 0 -net.ipv6.conf.all.accept_ra_rt_info_min_plen = 0 -net.ipv6.conf.all.accept_ra_rtr_pref = 1 -net.ipv6.conf.all.accept_redirects = 1 -net.ipv6.conf.all.accept_source_route = 0 -net.ipv6.conf.all.accept_untracked_na = 0 -net.ipv6.conf.all.addr_gen_mode = 0 -net.ipv6.conf.all.autoconf = 1 -net.ipv6.conf.all.dad_transmits = 1 -net.ipv6.conf.all.disable_ipv6 = 0 -net.ipv6.conf.all.disable_policy = 0 -net.ipv6.conf.all.drop_unicast_in_l2_multicast = 0 -net.ipv6.conf.all.drop_unsolicited_na = 0 -net.ipv6.conf.all.enhanced_dad = 1 -net.ipv6.conf.all.force_mld_version = 0 -net.ipv6.conf.all.force_tllao = 0 -net.ipv6.conf.all.forwarding = 0 -net.ipv6.conf.all.hop_limit = 64 -net.ipv6.conf.all.ignore_routes_with_linkdown = 0 -net.ipv6.conf.all.ioam6_enabled = 0 -net.ipv6.conf.all.ioam6_id = 65535 -net.ipv6.conf.all.ioam6_id_wide = 4294967295 -net.ipv6.conf.all.keep_addr_on_down = 0 -net.ipv6.conf.all.max_addresses = 16 -net.ipv6.conf.all.max_desync_factor = 600 -net.ipv6.conf.all.mc_forwarding = 0 -net.ipv6.conf.all.mldv1_unsolicited_report_interval = 10000 -net.ipv6.conf.all.mldv2_unsolicited_report_interval = 1000 -net.ipv6.conf.all.mtu = 1280 -net.ipv6.conf.all.ndisc_evict_nocarrier = 1 -net.ipv6.conf.all.ndisc_notify = 0 -net.ipv6.conf.all.ndisc_tclass = 0 -net.ipv6.conf.all.proxy_ndp = 0 -net.ipv6.conf.all.ra_defrtr_metric = 1024 -net.ipv6.conf.all.ra_honor_pio_life = 0 -net.ipv6.conf.all.regen_max_retry = 3 -net.ipv6.conf.all.router_probe_interval = 60 -net.ipv6.conf.all.router_solicitation_delay = 1 -net.ipv6.conf.all.router_solicitation_interval = 4 -net.ipv6.conf.all.router_solicitation_max_interval = 3600 -net.ipv6.conf.all.router_solicitations = -1 -net.ipv6.conf.all.rpl_seg_enabled = 0 -net.ipv6.conf.all.seg6_enabled = 0 -net.ipv6.conf.all.seg6_require_hmac = 0 -net.ipv6.conf.all.suppress_frag_ndisc = 1 -net.ipv6.conf.all.temp_prefered_lft = 86400 -net.ipv6.conf.all.temp_valid_lft = 604800 -net.ipv6.conf.all.use_oif_addrs_only = 0 -net.ipv6.conf.all.use_tempaddr = 0 -net.ipv6.conf.default.accept_dad = 1 -net.ipv6.conf.default.accept_ra = 1 -net.ipv6.conf.default.accept_ra_defrtr = 1 -net.ipv6.conf.default.accept_ra_from_local = 0 -net.ipv6.conf.default.accept_ra_min_hop_limit = 1 -net.ipv6.conf.default.accept_ra_min_lft = 0 -net.ipv6.conf.default.accept_ra_mtu = 1 -net.ipv6.conf.default.accept_ra_pinfo = 1 -net.ipv6.conf.default.accept_ra_rt_info_max_plen = 0 -net.ipv6.conf.default.accept_ra_rt_info_min_plen = 0 -net.ipv6.conf.default.accept_ra_rtr_pref = 1 -net.ipv6.conf.default.accept_redirects = 1 -net.ipv6.conf.default.accept_source_route = 0 -net.ipv6.conf.default.accept_untracked_na = 0 -net.ipv6.conf.default.addr_gen_mode = 0 -net.ipv6.conf.default.autoconf = 1 -net.ipv6.conf.default.dad_transmits = 1 -net.ipv6.conf.default.disable_ipv6 = 0 -net.ipv6.conf.default.disable_policy = 0 -net.ipv6.conf.default.drop_unicast_in_l2_multicast = 0 -net.ipv6.conf.default.drop_unsolicited_na = 0 -net.ipv6.conf.default.enhanced_dad = 1 -net.ipv6.conf.default.force_mld_version = 0 -net.ipv6.conf.default.force_tllao = 0 -net.ipv6.conf.default.forwarding = 0 -net.ipv6.conf.default.hop_limit = 64 -net.ipv6.conf.default.ignore_routes_with_linkdown = 0 -net.ipv6.conf.default.ioam6_enabled = 0 -net.ipv6.conf.default.ioam6_id = 65535 -net.ipv6.conf.default.ioam6_id_wide = 4294967295 -net.ipv6.conf.default.keep_addr_on_down = 0 -net.ipv6.conf.default.max_addresses = 16 -net.ipv6.conf.default.max_desync_factor = 600 -net.ipv6.conf.default.mc_forwarding = 0 -net.ipv6.conf.default.mldv1_unsolicited_report_interval = 10000 -net.ipv6.conf.default.mldv2_unsolicited_report_interval = 1000 -net.ipv6.conf.default.mtu = 1280 -net.ipv6.conf.default.ndisc_evict_nocarrier = 1 -net.ipv6.conf.default.ndisc_notify = 0 -net.ipv6.conf.default.ndisc_tclass = 0 -net.ipv6.conf.default.proxy_ndp = 0 -net.ipv6.conf.default.ra_defrtr_metric = 1024 -net.ipv6.conf.default.ra_honor_pio_life = 0 -net.ipv6.conf.default.regen_max_retry = 3 -net.ipv6.conf.default.router_probe_interval = 60 -net.ipv6.conf.default.router_solicitation_delay = 1 -net.ipv6.conf.default.router_solicitation_interval = 4 -net.ipv6.conf.default.router_solicitation_max_interval = 3600 -net.ipv6.conf.default.router_solicitations = -1 -net.ipv6.conf.default.rpl_seg_enabled = 0 -net.ipv6.conf.default.seg6_enabled = 0 -net.ipv6.conf.default.seg6_require_hmac = 0 -net.ipv6.conf.default.suppress_frag_ndisc = 1 -net.ipv6.conf.default.temp_prefered_lft = 86400 -net.ipv6.conf.default.temp_valid_lft = 604800 -net.ipv6.conf.default.use_oif_addrs_only = 0 -net.ipv6.conf.default.use_tempaddr = 0 -net.ipv6.conf.eth0.accept_dad = 1 -net.ipv6.conf.eth0.accept_ra = 1 -net.ipv6.conf.eth0.accept_ra_defrtr = 1 -net.ipv6.conf.eth0.accept_ra_from_local = 0 -net.ipv6.conf.eth0.accept_ra_min_hop_limit = 1 -net.ipv6.conf.eth0.accept_ra_min_lft = 0 -net.ipv6.conf.eth0.accept_ra_mtu = 1 -net.ipv6.conf.eth0.accept_ra_pinfo = 1 -net.ipv6.conf.eth0.accept_ra_rt_info_max_plen = 0 -net.ipv6.conf.eth0.accept_ra_rt_info_min_plen = 0 -net.ipv6.conf.eth0.accept_ra_rtr_pref = 1 -net.ipv6.conf.eth0.accept_redirects = 1 -net.ipv6.conf.eth0.accept_source_route = 0 -net.ipv6.conf.eth0.accept_untracked_na = 0 -net.ipv6.conf.eth0.addr_gen_mode = 0 -net.ipv6.conf.eth0.autoconf = 1 -net.ipv6.conf.eth0.dad_transmits = 1 -net.ipv6.conf.eth0.disable_ipv6 = 1 -net.ipv6.conf.eth0.disable_policy = 0 -net.ipv6.conf.eth0.drop_unicast_in_l2_multicast = 0 -net.ipv6.conf.eth0.drop_unsolicited_na = 0 -net.ipv6.conf.eth0.enhanced_dad = 1 -net.ipv6.conf.eth0.force_mld_version = 0 -net.ipv6.conf.eth0.force_tllao = 0 -net.ipv6.conf.eth0.forwarding = 0 -net.ipv6.conf.eth0.hop_limit = 64 -net.ipv6.conf.eth0.ignore_routes_with_linkdown = 0 -net.ipv6.conf.eth0.ioam6_enabled = 0 -net.ipv6.conf.eth0.ioam6_id = 65535 -net.ipv6.conf.eth0.ioam6_id_wide = 4294967295 -net.ipv6.conf.eth0.keep_addr_on_down = 0 -net.ipv6.conf.eth0.max_addresses = 16 -net.ipv6.conf.eth0.max_desync_factor = 600 -net.ipv6.conf.eth0.mc_forwarding = 0 -net.ipv6.conf.eth0.mldv1_unsolicited_report_interval = 10000 -net.ipv6.conf.eth0.mldv2_unsolicited_report_interval = 1000 -net.ipv6.conf.eth0.mtu = 1500 -net.ipv6.conf.eth0.ndisc_evict_nocarrier = 1 -net.ipv6.conf.eth0.ndisc_notify = 0 -net.ipv6.conf.eth0.ndisc_tclass = 0 -net.ipv6.conf.eth0.proxy_ndp = 0 -net.ipv6.conf.eth0.ra_defrtr_metric = 1024 -net.ipv6.conf.eth0.ra_honor_pio_life = 0 -net.ipv6.conf.eth0.regen_max_retry = 3 -net.ipv6.conf.eth0.router_probe_interval = 60 -net.ipv6.conf.eth0.router_solicitation_delay = 1 -net.ipv6.conf.eth0.router_solicitation_interval = 4 -net.ipv6.conf.eth0.router_solicitation_max_interval = 3600 -net.ipv6.conf.eth0.router_solicitations = -1 -net.ipv6.conf.eth0.rpl_seg_enabled = 0 -net.ipv6.conf.eth0.seg6_enabled = 0 -net.ipv6.conf.eth0.seg6_require_hmac = 0 -net.ipv6.conf.eth0.suppress_frag_ndisc = 1 -net.ipv6.conf.eth0.temp_prefered_lft = 86400 -net.ipv6.conf.eth0.temp_valid_lft = 604800 -net.ipv6.conf.eth0.use_oif_addrs_only = 0 -net.ipv6.conf.eth0.use_tempaddr = 0 -net.ipv6.conf.lo.accept_dad = -1 -net.ipv6.conf.lo.accept_ra = 1 -net.ipv6.conf.lo.accept_ra_defrtr = 1 -net.ipv6.conf.lo.accept_ra_from_local = 0 -net.ipv6.conf.lo.accept_ra_min_hop_limit = 1 -net.ipv6.conf.lo.accept_ra_min_lft = 0 -net.ipv6.conf.lo.accept_ra_mtu = 1 -net.ipv6.conf.lo.accept_ra_pinfo = 1 -net.ipv6.conf.lo.accept_ra_rt_info_max_plen = 0 -net.ipv6.conf.lo.accept_ra_rt_info_min_plen = 0 -net.ipv6.conf.lo.accept_ra_rtr_pref = 1 -net.ipv6.conf.lo.accept_redirects = 1 -net.ipv6.conf.lo.accept_source_route = 0 -net.ipv6.conf.lo.accept_untracked_na = 0 -net.ipv6.conf.lo.addr_gen_mode = 0 -net.ipv6.conf.lo.autoconf = 1 -net.ipv6.conf.lo.dad_transmits = 1 -net.ipv6.conf.lo.disable_ipv6 = 0 -net.ipv6.conf.lo.disable_policy = 0 -net.ipv6.conf.lo.drop_unicast_in_l2_multicast = 0 -net.ipv6.conf.lo.drop_unsolicited_na = 0 -net.ipv6.conf.lo.enhanced_dad = 1 -net.ipv6.conf.lo.force_mld_version = 0 -net.ipv6.conf.lo.force_tllao = 0 -net.ipv6.conf.lo.forwarding = 0 -net.ipv6.conf.lo.hop_limit = 64 -net.ipv6.conf.lo.ignore_routes_with_linkdown = 0 -net.ipv6.conf.lo.ioam6_enabled = 0 -net.ipv6.conf.lo.ioam6_id = 65535 -net.ipv6.conf.lo.ioam6_id_wide = 4294967295 -net.ipv6.conf.lo.keep_addr_on_down = 0 -net.ipv6.conf.lo.max_addresses = 16 -net.ipv6.conf.lo.max_desync_factor = 600 -net.ipv6.conf.lo.mc_forwarding = 0 -net.ipv6.conf.lo.mldv1_unsolicited_report_interval = 10000 -net.ipv6.conf.lo.mldv2_unsolicited_report_interval = 1000 -net.ipv6.conf.lo.mtu = 65536 -net.ipv6.conf.lo.ndisc_evict_nocarrier = 1 -net.ipv6.conf.lo.ndisc_notify = 0 -net.ipv6.conf.lo.ndisc_tclass = 0 -net.ipv6.conf.lo.proxy_ndp = 0 -net.ipv6.conf.lo.ra_defrtr_metric = 1024 -net.ipv6.conf.lo.ra_honor_pio_life = 0 -net.ipv6.conf.lo.regen_max_retry = 3 -net.ipv6.conf.lo.router_probe_interval = 60 -net.ipv6.conf.lo.router_solicitation_delay = 1 -net.ipv6.conf.lo.router_solicitation_interval = 4 -net.ipv6.conf.lo.router_solicitation_max_interval = 3600 -net.ipv6.conf.lo.router_solicitations = -1 -net.ipv6.conf.lo.rpl_seg_enabled = 0 -net.ipv6.conf.lo.seg6_enabled = 0 -net.ipv6.conf.lo.seg6_require_hmac = 0 -net.ipv6.conf.lo.suppress_frag_ndisc = 1 -net.ipv6.conf.lo.temp_prefered_lft = 86400 -net.ipv6.conf.lo.temp_valid_lft = 604800 -net.ipv6.conf.lo.use_oif_addrs_only = 0 -net.ipv6.conf.lo.use_tempaddr = -1 -net.ipv6.fib_multipath_hash_fields = 7 -net.ipv6.fib_multipath_hash_policy = 0 -net.ipv6.fib_notify_on_flag_change = 0 -net.ipv6.flowlabel_consistency = 1 -net.ipv6.flowlabel_reflect = 0 -net.ipv6.flowlabel_state_ranges = 0 -net.ipv6.fwmark_reflect = 0 -net.ipv6.icmp.echo_ignore_all = 0 -net.ipv6.icmp.echo_ignore_anycast = 0 -net.ipv6.icmp.echo_ignore_multicast = 0 -net.ipv6.icmp.error_anycast_as_unicast = 0 -net.ipv6.icmp.ratelimit = 1000 -net.ipv6.icmp.ratemask = 0-1,3-127 -net.ipv6.idgen_delay = 1 -net.ipv6.idgen_retries = 3 -net.ipv6.ioam6_id = 16777215 -net.ipv6.ioam6_id_wide = 72057594037927935 -net.ipv6.ip6frag_high_thresh = 4194304 -net.ipv6.ip6frag_low_thresh = 3145728 -net.ipv6.ip6frag_time = 60 -net.ipv6.ip_nonlocal_bind = 0 -net.ipv6.max_dst_opts_length = 2147483647 -net.ipv6.max_dst_opts_number = 8 -net.ipv6.max_hbh_length = 2147483647 -net.ipv6.max_hbh_opts_number = 8 -net.ipv6.neigh.eth0.anycast_delay = 100 -net.ipv6.neigh.eth0.app_solicit = 0 -net.ipv6.neigh.eth0.base_reachable_time_ms = 30000 -net.ipv6.neigh.eth0.delay_first_probe_time = 5 -net.ipv6.neigh.eth0.gc_stale_time = 60 -net.ipv6.neigh.eth0.interval_probe_time_ms = 5000 -net.ipv6.neigh.eth0.locktime = 0 -net.ipv6.neigh.eth0.mcast_resolicit = 0 -net.ipv6.neigh.eth0.mcast_solicit = 3 -net.ipv6.neigh.eth0.proxy_delay = 80 -net.ipv6.neigh.eth0.proxy_qlen = 64 -net.ipv6.neigh.eth0.retrans_time_ms = 1000 -net.ipv6.neigh.eth0.ucast_solicit = 3 -net.ipv6.neigh.eth0.unres_qlen = 101 -net.ipv6.neigh.eth0.unres_qlen_bytes = 212992 -net.ipv6.neigh.lo.anycast_delay = 100 -net.ipv6.neigh.lo.app_solicit = 0 -net.ipv6.neigh.lo.base_reachable_time_ms = 30000 -net.ipv6.neigh.lo.delay_first_probe_time = 5 -net.ipv6.neigh.lo.gc_stale_time = 60 -net.ipv6.neigh.lo.interval_probe_time_ms = 5000 -net.ipv6.neigh.lo.locktime = 0 -net.ipv6.neigh.lo.mcast_resolicit = 0 -net.ipv6.neigh.lo.mcast_solicit = 3 -net.ipv6.neigh.lo.proxy_delay = 80 -net.ipv6.neigh.lo.proxy_qlen = 64 -net.ipv6.neigh.lo.retrans_time_ms = 1000 -net.ipv6.neigh.lo.ucast_solicit = 3 -net.ipv6.neigh.lo.unres_qlen = 101 -net.ipv6.neigh.lo.unres_qlen_bytes = 212992 -net.ipv6.route.gc_elasticity = 9 -net.ipv6.route.gc_interval = 30 -net.ipv6.route.gc_min_interval = 0 -net.ipv6.route.gc_min_interval_ms = 500 -net.ipv6.route.gc_thresh = 1024 -net.ipv6.route.gc_timeout = 60 -net.ipv6.route.max_size = 2147483647 -net.ipv6.route.min_adv_mss = 1220 -net.ipv6.route.mtu_expires = 600 -net.ipv6.route.skip_notify_on_dev_down = 0 -net.ipv6.seg6_flowlabel = 0 -net.ipv6.xfrm6_gc_thresh = 32768 -net.mptcp.add_addr_timeout = 120 -net.mptcp.allow_join_initial_addr_port = 1 -net.mptcp.checksum_enabled = 0 -net.mptcp.close_timeout = 60 -net.mptcp.enabled = 1 -net.mptcp.pm_type = 0 -net.mptcp.scheduler = default -net.mptcp.stale_loss_cnt = 4 -net.netfilter.nf_conntrack_acct = 0 -net.netfilter.nf_conntrack_buckets = 262144 -net.netfilter.nf_conntrack_checksum = 1 -net.netfilter.nf_conntrack_count = 0 -net.netfilter.nf_conntrack_dccp_loose = 1 -net.netfilter.nf_conntrack_dccp_timeout_closereq = 64 -net.netfilter.nf_conntrack_dccp_timeout_closing = 64 -net.netfilter.nf_conntrack_dccp_timeout_open = 43200 -net.netfilter.nf_conntrack_dccp_timeout_partopen = 480 -net.netfilter.nf_conntrack_dccp_timeout_request = 240 -net.netfilter.nf_conntrack_dccp_timeout_respond = 480 -net.netfilter.nf_conntrack_dccp_timeout_timewait = 240 -net.netfilter.nf_conntrack_events = 2 -net.netfilter.nf_conntrack_expect_max = 4096 -net.netfilter.nf_conntrack_frag6_high_thresh = 4194304 -net.netfilter.nf_conntrack_frag6_low_thresh = 3145728 -net.netfilter.nf_conntrack_frag6_timeout = 60 -net.netfilter.nf_conntrack_generic_timeout = 600 -net.netfilter.nf_conntrack_gre_timeout = 30 -net.netfilter.nf_conntrack_gre_timeout_stream = 180 -net.netfilter.nf_conntrack_icmp_timeout = 30 -net.netfilter.nf_conntrack_icmpv6_timeout = 30 -net.netfilter.nf_conntrack_log_invalid = 0 -net.netfilter.nf_conntrack_max = 262144 -net.netfilter.nf_conntrack_sctp_timeout_closed = 10 -net.netfilter.nf_conntrack_sctp_timeout_cookie_echoed = 3 -net.netfilter.nf_conntrack_sctp_timeout_cookie_wait = 3 -net.netfilter.nf_conntrack_sctp_timeout_established = 210 -net.netfilter.nf_conntrack_sctp_timeout_heartbeat_sent = 30 -net.netfilter.nf_conntrack_sctp_timeout_shutdown_ack_sent = 3 -net.netfilter.nf_conntrack_sctp_timeout_shutdown_recd = 3 -net.netfilter.nf_conntrack_sctp_timeout_shutdown_sent = 3 -net.netfilter.nf_conntrack_tcp_be_liberal = 0 -net.netfilter.nf_conntrack_tcp_ignore_invalid_rst = 0 -net.netfilter.nf_conntrack_tcp_loose = 1 -net.netfilter.nf_conntrack_tcp_max_retrans = 3 -net.netfilter.nf_conntrack_tcp_timeout_close = 10 -net.netfilter.nf_conntrack_tcp_timeout_close_wait = 60 -net.netfilter.nf_conntrack_tcp_timeout_established = 432000 -net.netfilter.nf_conntrack_tcp_timeout_fin_wait = 120 -net.netfilter.nf_conntrack_tcp_timeout_last_ack = 30 -net.netfilter.nf_conntrack_tcp_timeout_max_retrans = 300 -net.netfilter.nf_conntrack_tcp_timeout_syn_recv = 60 -net.netfilter.nf_conntrack_tcp_timeout_syn_sent = 120 -net.netfilter.nf_conntrack_tcp_timeout_time_wait = 120 -net.netfilter.nf_conntrack_tcp_timeout_unacknowledged = 300 -net.netfilter.nf_conntrack_timestamp = 0 -net.netfilter.nf_conntrack_udp_timeout = 30 -net.netfilter.nf_conntrack_udp_timeout_stream = 120 -net.netfilter.nf_flowtable_tcp_timeout = 30 -net.netfilter.nf_flowtable_udp_timeout = 30 -net.netfilter.nf_hooks_lwtunnel = 0 -net.netfilter.nf_log.0 = NONE -net.netfilter.nf_log.1 = NONE -net.netfilter.nf_log.10 = NONE -net.netfilter.nf_log.2 = NONE -net.netfilter.nf_log.3 = NONE -net.netfilter.nf_log.4 = NONE -net.netfilter.nf_log.5 = NONE -net.netfilter.nf_log.6 = NONE -net.netfilter.nf_log.7 = NONE -net.netfilter.nf_log.8 = NONE -net.netfilter.nf_log.9 = NONE -net.unix.max_dgram_qlen = 10 -user.max_cgroup_namespaces = 770559 -user.max_fanotify_groups = 128 -user.max_fanotify_marks = 1048576 -user.max_inotify_instances = 128 -user.max_inotify_watches = 65536 -user.max_ipc_namespaces = 770559 -user.max_mnt_namespaces = 770559 -user.max_net_namespaces = 770559 -user.max_pid_namespaces = 770559 -user.max_time_namespaces = 770559 -user.max_user_namespaces = 770559 -user.max_uts_namespaces = 770559 -vm.admin_reserve_kbytes = 8192 -vm.compact_unevictable_allowed = 1 -vm.compaction_proactiveness = 20 -vm.dirty_background_bytes = 0 -vm.dirty_background_ratio = 10 -vm.dirty_bytes = 0 -vm.dirty_expire_centisecs = 3000 -vm.dirty_ratio = 20 -vm.dirty_writeback_centisecs = 500 -vm.dirtytime_expire_seconds = 43200 -vm.extfrag_threshold = 500 -vm.hugetlb_optimize_vmemmap = 0 -vm.hugetlb_shm_group = 0 -vm.laptop_mode = 0 -vm.legacy_va_layout = 0 -vm.lowmem_reserve_ratio = 256 256 32 0 0 -vm.max_map_count = 1048576 -vm.memfd_noexec = 0 -vm.memory_failure_early_kill = 0 -vm.memory_failure_recovery = 1 -vm.min_free_kbytes = 67584 -vm.min_slab_ratio = 5 -vm.min_unmapped_ratio = 1 -vm.mmap_min_addr = 65536 -vm.mmap_rnd_bits = 32 -vm.mmap_rnd_compat_bits = 16 -vm.nr_hugepages = 0 -vm.nr_hugepages_mempolicy = 0 -vm.nr_overcommit_hugepages = 0 -vm.numa_stat = 1 -vm.numa_zonelist_order = Node -vm.oom_dump_tasks = 1 -vm.oom_kill_allocating_task = 0 -vm.overcommit_kbytes = 0 -vm.overcommit_memory = 0 -vm.overcommit_ratio = 50 -vm.page-cluster = 3 -vm.page_lock_unfairness = 5 -vm.panic_on_oom = 0 -vm.percpu_pagelist_high_fraction = 0 -vm.stat_interval = 1 -vm.swappiness = 60 -vm.unprivileged_userfaultfd = 0 -vm.user_reserve_kbytes = 131072 -vm.vfs_cache_pressure = 100 -vm.watermark_boost_factor = 15000 -vm.watermark_scale_factor = 10 -vm.zone_reclaim_mode = 0 ------------------------------------------------------------- -16. /sys/kernel/mm/transparent_hugepage -always [madvise] never ------------------------------------------------------------- -17. /sys/kernel/mm/transparent_hugepage/khugepaged -1 ------------------------------------------------------------- -18. OS release -PRETTY_NAME="Ubuntu 22.04.4 LTS" -NAME="Ubuntu" -VERSION_ID="22.04" -VERSION="22.04.4 LTS (Jammy Jellyfish)" -VERSION_CODENAME=jammy -ID=ubuntu -ID_LIKE=debian -HOME_URL="https://www.ubuntu.com/" -SUPPORT_URL="https://help.ubuntu.com/" -BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" -PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" -UBUNTU_CODENAME=jammy ------------------------------------------------------------- -19. Disk information -NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS -loop0 7:0 0 4K 1 loop -loop1 7:1 0 258.3M 1 loop -loop2 7:2 0 73.9M 1 loop -loop3 7:3 0 11.1M 1 loop -loop4 7:4 0 258.3M 1 loop -loop5 7:5 0 44.4M 1 loop -loop6 7:6 0 11.1M 1 loop -loop7 7:7 0 505.1M 1 loop -loop8 7:8 0 516M 1 loop -loop9 7:9 0 91.7M 1 loop -loop10 7:10 0 10.8M 1 loop -loop11 7:11 0 10.8M 1 loop -loop13 7:13 0 44.4M 1 loop -loop14 7:14 0 500K 1 loop -loop15 7:15 0 568K 1 loop -loop16 7:16 0 73.9M 1 loop -sda 8:0 1 0B 0 disk -sr0 11:0 1 1024M 0 rom -nvme1n1 259:0 0 3.6T 0 disk -└─nvme1n1p1 259:1 0 3.6T 0 part -nvme0n1 259:2 0 3.6T 0 disk -├─nvme0n1p1 259:3 0 3.2T 0 part /usr/lib/firmware/nvidia/550.120/gsp_tu10x.bin -│ /usr/lib/firmware/nvidia/550.120/gsp_ga10x.bin -│ /usr/lib/x86_64-linux-gnu/libnvcuvid.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-encode.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-nvvm.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-pkcs11-openssl3.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-pkcs11.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-gpucomp.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libcudadebugger.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libcuda.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.550.120 -│ /usr/bin/nvidia-cuda-mps-server -│ /usr/bin/nvidia-cuda-mps-control -│ /usr/bin/nvidia-persistenced -│ /usr/bin/nvidia-debugdump -│ /usr/bin/nvidia-smi -│ /home/mlcuser/MLC/repos/local/cache/download-file_9907d37a -│ /mlc-mount/home/arjun/gh_action_submissions -│ /mlc-mount/home/arjun/gh_action_results -│ /etc/hosts -│ /etc/hostname -│ /etc/resolv.conf -├─nvme0n1p2 259:4 0 1G 0 part -└─nvme0n1p3 259:5 0 487.2G 0 part ------------------------------------------------------------- -20. /sys/devices/virtual/dmi/id -bios_date -bios_release -bios_vendor -bios_version -board_asset_tag -board_name -board_serial -board_vendor -board_version -chassis_asset_tag -chassis_serial -chassis_type -chassis_vendor -chassis_version -modalias -power -product_family -product_name -product_serial -product_sku -product_uuid -product_version -subsystem -sys_vendor -uevent ------------------------------------------------------------- -21. dmidecode -# dmidecode 3.3 -Scanning /dev/mem for entry point. -FAILED: dmidecode ------------------------------------------------------------- -22. BIOS -# dmidecode 3.3 -Scanning /dev/mem for entry point. -FAILED: dmidecode -t bios ------------------------------------------------------------- diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/user.conf b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/user.conf deleted file mode 100644 index ed30c86c6..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/offline/user.conf +++ /dev/null @@ -1,5 +0,0 @@ -gptj.Offline.target_qps = 1.0 -gptj.Offline.max_query_count = 1 -gptj.Offline.min_query_count = 1 -gptj.Offline.min_duration = 0 -gptj.Offline.sample_concatenate_permutation = 0 diff --git a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/system_info.txt b/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/system_info.txt deleted file mode 100644 index 7c9f21e24..000000000 --- a/open/MLCommons/measurements/gh_action-reference-gpu-pytorch_v2.6.0-cu124/gptj-99/system_info.txt +++ /dev/null @@ -1,2616 +0,0 @@ -Platform Details - ------------------------------------------------------------- -1. uname -a -Linux 0162cdce0038 6.8.0-52-generic #53-Ubuntu SMP PREEMPT_DYNAMIC Sat Jan 11 00:06:25 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux ------------------------------------------------------------- -2. w - 08:53:36 up 59 days, 21:50, 0 users, load average: 1.83, 2.21, 2.76 -USER TTY FROM LOGIN@ IDLE JCPU PCPU WHAT ------------------------------------------------------------- -3. Username -mlcuser ------------------------------------------------------------- -4. ulimit -a -real-time non-blocking time (microseconds, -R) unlimited -core file size (blocks, -c) unlimited -data seg size (kbytes, -d) unlimited -scheduling priority (-e) 0 -file size (blocks, -f) unlimited -pending signals (-i) 770559 -max locked memory (kbytes, -l) 8192 -max memory size (kbytes, -m) unlimited -open files (-n) 1048576 -pipe size (512 bytes, -p) 8 -POSIX message queues (bytes, -q) 819200 -real-time priority (-r) 0 -stack size (kbytes, -s) 8192 -cpu time (seconds, -t) unlimited -max user processes (-u) unlimited -virtual memory (kbytes, -v) unlimited -file locks (-x) unlimited ------------------------------------------------------------- -5. sysinfo process ancestry -bash ------------------------------------------------------------- -6. /proc/cpuinfo -processor : 0 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 1100.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 0 -cpu cores : 24 -apicid : 0 -initial apicid : 0 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 1 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 1 -cpu cores : 24 -apicid : 2 -initial apicid : 2 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 2 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 2 -cpu cores : 24 -apicid : 4 -initial apicid : 4 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 3 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 3 -cpu cores : 24 -apicid : 6 -initial apicid : 6 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 4 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 4 -cpu cores : 24 -apicid : 8 -initial apicid : 8 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 5 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 5 -cpu cores : 24 -apicid : 10 -initial apicid : 10 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 6 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 6 -cpu cores : 24 -apicid : 12 -initial apicid : 12 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 7 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 7 -cpu cores : 24 -apicid : 14 -initial apicid : 14 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 8 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 8 -cpu cores : 24 -apicid : 16 -initial apicid : 16 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 9 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 9 -cpu cores : 24 -apicid : 18 -initial apicid : 18 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 10 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 10 -cpu cores : 24 -apicid : 20 -initial apicid : 20 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 11 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 11 -cpu cores : 24 -apicid : 22 -initial apicid : 22 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 12 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 12 -cpu cores : 24 -apicid : 24 -initial apicid : 24 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 13 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 2723.689 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 13 -cpu cores : 24 -apicid : 26 -initial apicid : 26 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 14 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 4223.033 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 14 -cpu cores : 24 -apicid : 28 -initial apicid : 28 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 15 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 3348.056 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 15 -cpu cores : 24 -apicid : 30 -initial apicid : 30 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 16 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 900.380 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 16 -cpu cores : 24 -apicid : 32 -initial apicid : 32 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 17 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 17 -cpu cores : 24 -apicid : 34 -initial apicid : 34 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 18 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 18 -cpu cores : 24 -apicid : 36 -initial apicid : 36 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 19 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 2172.987 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 19 -cpu cores : 24 -apicid : 38 -initial apicid : 38 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 20 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 20 -cpu cores : 24 -apicid : 40 -initial apicid : 40 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 21 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 798.951 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 21 -cpu cores : 24 -apicid : 42 -initial apicid : 42 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 22 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 22 -cpu cores : 24 -apicid : 44 -initial apicid : 44 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 23 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 23 -cpu cores : 24 -apicid : 46 -initial apicid : 46 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 24 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 0 -cpu cores : 24 -apicid : 1 -initial apicid : 1 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 25 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 1 -cpu cores : 24 -apicid : 3 -initial apicid : 3 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 26 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 2 -cpu cores : 24 -apicid : 5 -initial apicid : 5 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 27 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 3 -cpu cores : 24 -apicid : 7 -initial apicid : 7 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 28 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 801.025 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 4 -cpu cores : 24 -apicid : 9 -initial apicid : 9 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 29 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 5 -cpu cores : 24 -apicid : 11 -initial apicid : 11 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 30 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 6 -cpu cores : 24 -apicid : 13 -initial apicid : 13 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 31 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 7 -cpu cores : 24 -apicid : 15 -initial apicid : 15 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 32 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 8 -cpu cores : 24 -apicid : 17 -initial apicid : 17 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 33 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 9 -cpu cores : 24 -apicid : 19 -initial apicid : 19 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 34 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 10 -cpu cores : 24 -apicid : 21 -initial apicid : 21 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 35 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 11 -cpu cores : 24 -apicid : 23 -initial apicid : 23 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 36 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.091 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 12 -cpu cores : 24 -apicid : 25 -initial apicid : 25 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 37 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 13 -cpu cores : 24 -apicid : 27 -initial apicid : 27 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 38 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 14 -cpu cores : 24 -apicid : 29 -initial apicid : 29 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 39 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 15 -cpu cores : 24 -apicid : 31 -initial apicid : 31 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 40 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 16 -cpu cores : 24 -apicid : 33 -initial apicid : 33 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 41 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 1500.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 17 -cpu cores : 24 -apicid : 35 -initial apicid : 35 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 42 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 18 -cpu cores : 24 -apicid : 37 -initial apicid : 37 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 43 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 970.771 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 19 -cpu cores : 24 -apicid : 39 -initial apicid : 39 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 44 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 20 -cpu cores : 24 -apicid : 41 -initial apicid : 41 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 45 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 21 -cpu cores : 24 -apicid : 43 -initial apicid : 43 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 46 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 22 -cpu cores : 24 -apicid : 45 -initial apicid : 45 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - -processor : 47 -vendor_id : GenuineIntel -cpu family : 6 -model : 143 -model name : Intel(R) Xeon(R) w7-2495X -stepping : 8 -microcode : 0x2b000603 -cpu MHz : 800.000 -cache size : 46080 KB -physical id : 0 -siblings : 48 -core id : 23 -cpu cores : 24 -apicid : 47 -initial apicid : 47 -fpu : yes -fpu_exception : yes -cpuid level : 32 -wp : yes -flags : fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -vmx flags : vnmi preemption_timer posted_intr invvpid ept_x_only ept_ad ept_1gb flexpriority apicv tsc_offset vtpr mtf vapic ept vpid unrestricted_guest vapic_reg vid ple pml ept_mode_based_exec tsc_scaling usr_wait_pause notify_vm_exiting ipi_virt -bugs : spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi -bogomips : 4992.00 -clflush size : 64 -cache_alignment : 64 -address sizes : 52 bits physical, 57 bits virtual -power management: - ------------------------------------------------------------- -7. lscpu -Architecture: x86_64 -CPU op-mode(s): 32-bit, 64-bit -Address sizes: 52 bits physical, 57 bits virtual -Byte Order: Little Endian -CPU(s): 48 -On-line CPU(s) list: 0-47 -Vendor ID: GenuineIntel -Model name: Intel(R) Xeon(R) w7-2495X -CPU family: 6 -Model: 143 -Thread(s) per core: 2 -Core(s) per socket: 24 -Socket(s): 1 -Stepping: 8 -CPU max MHz: 4800.0000 -CPU min MHz: 800.0000 -BogoMIPS: 4992.00 -Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush dts acpi mmx fxsr sse sse2 ss ht tm pbe syscall nx pdpe1gb rdtscp lm constant_tsc art arch_perfmon pebs bts rep_good nopl xtopology nonstop_tsc cpuid aperfmperf tsc_known_freq pni pclmulqdq dtes64 monitor ds_cpl vmx smx est tm2 ssse3 sdbg fma cx16 xtpr pdcm pcid dca sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand lahf_lm abm 3dnowprefetch cpuid_fault epb cat_l3 cat_l2 cdp_l3 intel_ppin cdp_l2 ssbd mba ibrs ibpb stibp ibrs_enhanced tpr_shadow flexpriority ept vpid ept_ad fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid cqm rdt_a avx512f avx512dq rdseed adx smap avx512ifma clflushopt clwb intel_pt avx512cd sha_ni avx512bw avx512vl xsaveopt xsavec xgetbv1 xsaves cqm_llc cqm_occup_llc cqm_mbm_total cqm_mbm_local split_lock_detect user_shstk avx_vnni avx512_bf16 wbnoinvd dtherm ida arat pln pts hwp hwp_act_window hwp_epp hwp_pkg_req vnmi avx512vbmi umip pku ospke waitpkg avx512_vbmi2 gfni vaes vpclmulqdq avx512_vnni avx512_bitalg tme avx512_vpopcntdq la57 rdpid bus_lock_detect cldemote movdiri movdir64b enqcmd fsrm md_clear serialize tsxldtrk pconfig arch_lbr ibt amx_bf16 avx512_fp16 amx_tile amx_int8 flush_l1d arch_capabilities -Virtualization: VT-x -L1d cache: 1.1 MiB (24 instances) -L1i cache: 768 KiB (24 instances) -L2 cache: 48 MiB (24 instances) -L3 cache: 45 MiB (1 instance) -NUMA node(s): 1 -NUMA node0 CPU(s): 0-47 -Vulnerability Gather data sampling: Not affected -Vulnerability Itlb multihit: Not affected -Vulnerability L1tf: Not affected -Vulnerability Mds: Not affected -Vulnerability Meltdown: Not affected -Vulnerability Mmio stale data: Not affected -Vulnerability Reg file data sampling: Not affected -Vulnerability Retbleed: Not affected -Vulnerability Spec rstack overflow: Not affected -Vulnerability Spec store bypass: Mitigation; Speculative Store Bypass disabled via prctl -Vulnerability Spectre v1: Mitigation; usercopy/swapgs barriers and __user pointer sanitization -Vulnerability Spectre v2: Mitigation; Enhanced / Automatic IBRS; IBPB conditional; RSB filling; PBRSB-eIBRS SW sequence; BHI BHI_DIS_S -Vulnerability Srbds: Not affected -Vulnerability Tsx async abort: Not affected ------------------------------------------------------------- -8. numactl --hardware -available: 1 nodes (0) -node 0 cpus: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 -node 0 size: 192709 MB -node 0 free: 79355 MB -node distances: -node 0 - 0: 10 ------------------------------------------------------------- -9. /proc/meminfo -MemTotal: 197334528 kB -MemFree: 81260148 kB -MemAvailable: 192270948 kB -Buffers: 580472 kB -Cached: 110942764 kB -SwapCached: 63040 kB -Active: 32407452 kB -Inactive: 79605680 kB -Active(anon): 457776 kB -Inactive(anon): 74116 kB -Active(file): 31949676 kB -Inactive(file): 79531564 kB -Unevictable: 6160 kB -Mlocked: 0 kB -SwapTotal: 8388604 kB -SwapFree: 7703112 kB -Zswap: 0 kB -Zswapped: 0 kB -Dirty: 21108 kB -Writeback: 0 kB -AnonPages: 486956 kB -Mapped: 311000 kB -Shmem: 41996 kB -KReclaimable: 1566044 kB -Slab: 2381696 kB -SReclaimable: 1566044 kB -SUnreclaim: 815652 kB -KernelStack: 20272 kB -PageTables: 33284 kB -SecPageTables: 0 kB -NFS_Unstable: 0 kB -Bounce: 0 kB -WritebackTmp: 0 kB -CommitLimit: 107055868 kB -Committed_AS: 6897048 kB -VmallocTotal: 13743895347199 kB -VmallocUsed: 157652 kB -VmallocChunk: 0 kB -Percpu: 122880 kB -HardwareCorrupted: 56 kB -AnonHugePages: 0 kB -ShmemHugePages: 0 kB -ShmemPmdMapped: 0 kB -FileHugePages: 0 kB -FilePmdMapped: 0 kB -Unaccepted: 0 kB -HugePages_Total: 0 -HugePages_Free: 0 -HugePages_Rsvd: 0 -HugePages_Surp: 0 -Hugepagesize: 2048 kB -Hugetlb: 0 kB -DirectMap4k: 70549092 kB -DirectMap2M: 127111168 kB -DirectMap1G: 5242880 kB ------------------------------------------------------------- -10. who -r ------------------------------------------------------------- -11. Systemd service manager version -systemd 219 ------------------------------------------------------------- -12. Services, from systemctl list-unit-files -UNIT FILE STATE -basic.target static -default.target enabled -graphical.target disabled -local-fs.target static -multi-user.target enabled -network.target static -nfs-client.target disabled -poweroff.target static -reboot.target static -remote-fs.target enabled -rescue.target static -sysinit.target static -timers.target static -apt-daily-upgrade.service static -apt-daily-upgrade.timer enabled -apt-daily.service static -apt-daily.timer enabled -dpkg-db-backup.service static -dpkg-db-backup.timer enabled -e2scrub@.service static -e2scrub_all.service static -e2scrub_all.timer enabled -e2scrub_fail@.service static -e2scrub_reap.service enabled -fstrim.service static -fstrim.timer enabled -hwclock.sh.service disabled -motd-news.service static -motd-news.timer enabled -procps.service disabled -rsync.service disabled -sudo.service masked - -32 unit files listed. ------------------------------------------------------------- -13. Linux kernel boot-time arguments, from /proc/cmdline -BOOT_IMAGE=/boot/vmlinuz-6.8.0-52-generic root=UUID=cd1dd39e-4e7b-4be5-8864-c6790fa67706 ro quiet splash vt.handoff=7 ------------------------------------------------------------- -14. cpupower frequency-info -analyzing CPU 14: - driver: intel_pstate - CPUs which run at the same hardware frequency: 14 - CPUs which need to have their frequency coordinated by software: 14 - maximum transition latency: Cannot determine or is not supported. - hardware limits: 800 MHz - 4.80 GHz - available cpufreq governors: performance powersave - current policy: frequency should be within 800 MHz and 4.80 GHz. - The governor "powersave" may decide which speed to use - within this range. - current CPU frequency: Unable to call hardware - current CPU frequency: 900 MHz (asserted by call to kernel) - boost state support: - Supported: yes - Active: yes ------------------------------------------------------------- -15. sysctl -abi.vsyscall32 = 1 -debug.exception-trace = 1 -debug.kprobes-optimization = 1 -dev.cdrom.autoclose = 1 -dev.cdrom.autoeject = 0 -dev.cdrom.check_media = 0 -dev.cdrom.debug = 0 -dev.cdrom.info = CD-ROM information, Id: cdrom.c 3.20 2003/12/17 -dev.cdrom.info = -dev.cdrom.info = drive name: sr0 -dev.cdrom.info = drive speed: 1 -dev.cdrom.info = drive # of slots: 1 -dev.cdrom.info = Can close tray: 1 -dev.cdrom.info = Can open tray: 1 -dev.cdrom.info = Can lock tray: 1 -dev.cdrom.info = Can change speed: 0 -dev.cdrom.info = Can select disk: 0 -dev.cdrom.info = Can read multisession: 1 -dev.cdrom.info = Can read MCN: 1 -dev.cdrom.info = Reports media changed: 1 -dev.cdrom.info = Can play audio: 1 -dev.cdrom.info = Can write CD-R: 0 -dev.cdrom.info = Can write CD-RW: 0 -dev.cdrom.info = Can read DVD: 0 -dev.cdrom.info = Can write DVD-R: 0 -dev.cdrom.info = Can write DVD-RAM: 0 -dev.cdrom.info = Can read MRW: 0 -dev.cdrom.info = Can write MRW: 0 -dev.cdrom.info = Can write RAM: 0 -dev.cdrom.info = -dev.cdrom.info = -dev.cdrom.lock = 0 -dev.hpet.max-user-freq = 64 -dev.mac_hid.mouse_button2_keycode = 97 -dev.mac_hid.mouse_button3_keycode = 100 -dev.mac_hid.mouse_button_emulation = 0 -dev.parport.default.spintime = 500 -dev.parport.default.timeslice = 200 -dev.raid.speed_limit_max = 200000 -dev.raid.speed_limit_min = 1000 -dev.scsi.logging_level = 0 -dev.tty.ldisc_autoload = 1 -dev.tty.legacy_tiocsti = 0 -fs.aio-max-nr = 65536 -fs.aio-nr = 0 -fs.dentry-state = 1059595 899336 45 0 315521 0 -fs.dir-notify-enable = 1 -fs.epoll.max_user_watches = 43912503 -fs.fanotify.max_queued_events = 16384 -fs.fanotify.max_user_groups = 128 -fs.fanotify.max_user_marks = 1048576 -fs.file-max = 9223372036854775807 -fs.file-nr = 10784 0 9223372036854775807 -fs.inode-nr = 773459 39689 -fs.inode-state = 773459 39689 0 0 0 0 0 -fs.inotify.max_queued_events = 16384 -fs.inotify.max_user_instances = 128 -fs.inotify.max_user_watches = 65536 -fs.lease-break-time = 45 -fs.leases-enable = 1 -fs.mount-max = 100000 -fs.mqueue.msg_default = 10 -fs.mqueue.msg_max = 10 -fs.mqueue.msgsize_default = 8192 -fs.mqueue.msgsize_max = 8192 -fs.mqueue.queues_max = 256 -fs.nr_open = 1048576 -fs.overflowgid = 65534 -fs.overflowuid = 65534 -fs.pipe-max-size = 1048576 -fs.pipe-user-pages-hard = 0 -fs.pipe-user-pages-soft = 16384 -fs.protected_fifos = 1 -fs.protected_hardlinks = 1 -fs.protected_regular = 2 -fs.protected_symlinks = 1 -fs.quota.allocated_dquots = 0 -fs.quota.cache_hits = 0 -fs.quota.drops = 0 -fs.quota.free_dquots = 0 -fs.quota.lookups = 0 -fs.quota.reads = 0 -fs.quota.syncs = 2870 -fs.quota.writes = 0 -fs.suid_dumpable = 2 -fs.verity.require_signatures = 0 -kernel.acct = 4 2 30 -kernel.acpi_video_flags = 0 -kernel.apparmor_restrict_unprivileged_unconfined = 0 -kernel.apparmor_restrict_unprivileged_userns = 1 -kernel.arch = x86_64 -kernel.auto_msgmni = 0 -kernel.bootloader_type = 33 -kernel.bootloader_version = 1 -kernel.bpf_stats_enabled = 0 -kernel.cad_pid = 0 -kernel.cap_last_cap = 40 -kernel.core_pattern = |/usr/share/apport/apport -p%p -s%s -c%c -d%d -P%P -u%u -g%g -- %E -kernel.core_pipe_limit = 10 -kernel.core_uses_pid = 0 -kernel.ctrl-alt-del = 0 -kernel.dmesg_restrict = 1 -kernel.domainname = (none) -kernel.firmware_config.force_sysfs_fallback = 0 -kernel.firmware_config.ignore_sysfs_fallback = 0 -kernel.ftrace_dump_on_oops = 0 -kernel.ftrace_enabled = 1 -kernel.hardlockup_all_cpu_backtrace = 0 -kernel.hardlockup_panic = 0 -kernel.hostname = 0162cdce0038 -kernel.hotplug = -kernel.hung_task_all_cpu_backtrace = 0 -kernel.hung_task_check_count = 4194304 -kernel.hung_task_check_interval_secs = 0 -kernel.hung_task_panic = 0 -kernel.hung_task_timeout_secs = 120 -kernel.hung_task_warnings = 10 -kernel.io_delay_type = 1 -kernel.io_uring_disabled = 0 -kernel.io_uring_group = -1 -kernel.kexec_load_disabled = 0 -kernel.kexec_load_limit_panic = -1 -kernel.kexec_load_limit_reboot = -1 -kernel.keys.gc_delay = 300 -kernel.keys.maxbytes = 20000 -kernel.keys.maxkeys = 200 -kernel.keys.persistent_keyring_expiry = 259200 -kernel.keys.root_maxbytes = 25000000 -kernel.keys.root_maxkeys = 1000000 -kernel.kptr_restrict = 1 -kernel.latencytop = 0 -kernel.max_lock_depth = 1024 -kernel.max_rcu_stall_to_panic = 0 -kernel.modprobe = /sbin/modprobe -kernel.modules_disabled = 0 -kernel.msg_next_id = -1 -kernel.msgmax = 8192 -kernel.msgmnb = 16384 -kernel.msgmni = 32000 -kernel.ngroups_max = 65536 -kernel.nmi_watchdog = 1 -kernel.ns_last_pid = 7592 -kernel.numa_balancing = 0 -kernel.numa_balancing_promote_rate_limit_MBps = 65536 -kernel.oops_all_cpu_backtrace = 0 -kernel.oops_limit = 10000 -kernel.osrelease = 6.8.0-52-generic -kernel.ostype = Linux -kernel.overflowgid = 65534 -kernel.overflowuid = 65534 -kernel.panic = 0 -kernel.panic_on_io_nmi = 0 -kernel.panic_on_oops = 0 -kernel.panic_on_rcu_stall = 0 -kernel.panic_on_unrecovered_nmi = 0 -kernel.panic_on_warn = 0 -kernel.panic_print = 0 -kernel.perf_cpu_time_max_percent = 25 -kernel.perf_event_max_contexts_per_stack = 8 -kernel.perf_event_max_sample_rate = 4000 -kernel.perf_event_max_stack = 127 -kernel.perf_event_mlock_kb = 516 -kernel.perf_event_paranoid = 4 -kernel.pid_max = 4194304 -kernel.poweroff_cmd = /sbin/poweroff -kernel.print-fatal-signals = 0 -kernel.printk = 4 4 1 7 -kernel.printk_delay = 0 -kernel.printk_devkmsg = on -kernel.printk_ratelimit = 5 -kernel.printk_ratelimit_burst = 10 -kernel.pty.max = 4096 -kernel.pty.nr = 6 -kernel.pty.reserve = 1024 -kernel.random.boot_id = 266aa90c-a7a3-400b-90af-2740886ad693 -kernel.random.entropy_avail = 256 -kernel.random.poolsize = 256 -kernel.random.urandom_min_reseed_secs = 60 -kernel.random.uuid = 70480901-6603-4a55-9122-c38014add3cf -kernel.random.write_wakeup_threshold = 256 -kernel.randomize_va_space = 2 -kernel.real-root-dev = 0 -kernel.sched_autogroup_enabled = 1 -kernel.sched_cfs_bandwidth_slice_us = 5000 -kernel.sched_deadline_period_max_us = 4194304 -kernel.sched_deadline_period_min_us = 100 -kernel.sched_itmt_enabled = 1 -kernel.sched_rr_timeslice_ms = 100 -kernel.sched_rt_period_us = 1000000 -kernel.sched_rt_runtime_us = 950000 -kernel.sched_schedstats = 0 -kernel.sched_util_clamp_max = 1024 -kernel.sched_util_clamp_min = 1024 -kernel.sched_util_clamp_min_rt_default = 1024 -kernel.seccomp.actions_avail = kill_process kill_thread trap errno user_notif trace log allow -kernel.seccomp.actions_logged = kill_process kill_thread trap errno user_notif trace log -kernel.sem = 32000 1024000000 500 32000 -kernel.sem_next_id = -1 -kernel.shm_next_id = -1 -kernel.shm_rmid_forced = 0 -kernel.shmall = 18446744073692774399 -kernel.shmmax = 18446744073692774399 -kernel.shmmni = 4096 -kernel.soft_watchdog = 1 -kernel.softlockup_all_cpu_backtrace = 0 -kernel.softlockup_panic = 0 -kernel.split_lock_mitigate = 1 -kernel.stack_tracer_enabled = 0 -kernel.sysctl_writes_strict = 1 -kernel.sysrq = 176 -kernel.tainted = 20993 -kernel.task_delayacct = 0 -kernel.threads-max = 1541118 -kernel.timer_migration = 1 -kernel.traceoff_on_warning = 0 -kernel.tracepoint_printk = 0 -kernel.unknown_nmi_panic = 0 -kernel.unprivileged_bpf_disabled = 2 -kernel.unprivileged_userns_clone = 1 -kernel.user_events_max = 32768 -kernel.usermodehelper.bset = 4294967295 511 -kernel.usermodehelper.inheritable = 4294967295 511 -kernel.version = #53-Ubuntu SMP PREEMPT_DYNAMIC Sat Jan 11 00:06:25 UTC 2025 -kernel.warn_limit = 0 -kernel.watchdog = 1 -kernel.watchdog_cpumask = 0-47 -kernel.watchdog_thresh = 10 -kernel.yama.ptrace_scope = 1 -net.core.optmem_max = 131072 -net.core.rps_default_mask = 0000,00000000 -net.core.somaxconn = 4096 -net.core.txrehash = 1 -net.core.xfrm_acq_expires = 30 -net.core.xfrm_aevent_etime = 10 -net.core.xfrm_aevent_rseqth = 2 -net.core.xfrm_larval_drop = 1 -net.ipv4.conf.all.accept_local = 0 -net.ipv4.conf.all.accept_redirects = 0 -net.ipv4.conf.all.accept_source_route = 0 -net.ipv4.conf.all.arp_accept = 0 -net.ipv4.conf.all.arp_announce = 0 -net.ipv4.conf.all.arp_evict_nocarrier = 1 -net.ipv4.conf.all.arp_filter = 0 -net.ipv4.conf.all.arp_ignore = 0 -net.ipv4.conf.all.arp_notify = 0 -net.ipv4.conf.all.bc_forwarding = 0 -net.ipv4.conf.all.bootp_relay = 0 -net.ipv4.conf.all.disable_policy = 0 -net.ipv4.conf.all.disable_xfrm = 0 -net.ipv4.conf.all.drop_gratuitous_arp = 0 -net.ipv4.conf.all.drop_unicast_in_l2_multicast = 0 -net.ipv4.conf.all.force_igmp_version = 0 -net.ipv4.conf.all.forwarding = 1 -net.ipv4.conf.all.igmpv2_unsolicited_report_interval = 10000 -net.ipv4.conf.all.igmpv3_unsolicited_report_interval = 1000 -net.ipv4.conf.all.ignore_routes_with_linkdown = 0 -net.ipv4.conf.all.log_martians = 0 -net.ipv4.conf.all.mc_forwarding = 0 -net.ipv4.conf.all.medium_id = 0 -net.ipv4.conf.all.promote_secondaries = 0 -net.ipv4.conf.all.proxy_arp = 0 -net.ipv4.conf.all.proxy_arp_pvlan = 0 -net.ipv4.conf.all.route_localnet = 0 -net.ipv4.conf.all.rp_filter = 2 -net.ipv4.conf.all.secure_redirects = 1 -net.ipv4.conf.all.send_redirects = 1 -net.ipv4.conf.all.shared_media = 1 -net.ipv4.conf.all.src_valid_mark = 0 -net.ipv4.conf.all.tag = 0 -net.ipv4.conf.default.accept_local = 0 -net.ipv4.conf.default.accept_redirects = 1 -net.ipv4.conf.default.accept_source_route = 1 -net.ipv4.conf.default.arp_accept = 0 -net.ipv4.conf.default.arp_announce = 0 -net.ipv4.conf.default.arp_evict_nocarrier = 1 -net.ipv4.conf.default.arp_filter = 0 -net.ipv4.conf.default.arp_ignore = 0 -net.ipv4.conf.default.arp_notify = 0 -net.ipv4.conf.default.bc_forwarding = 0 -net.ipv4.conf.default.bootp_relay = 0 -net.ipv4.conf.default.disable_policy = 0 -net.ipv4.conf.default.disable_xfrm = 0 -net.ipv4.conf.default.drop_gratuitous_arp = 0 -net.ipv4.conf.default.drop_unicast_in_l2_multicast = 0 -net.ipv4.conf.default.force_igmp_version = 0 -net.ipv4.conf.default.forwarding = 1 -net.ipv4.conf.default.igmpv2_unsolicited_report_interval = 10000 -net.ipv4.conf.default.igmpv3_unsolicited_report_interval = 1000 -net.ipv4.conf.default.ignore_routes_with_linkdown = 0 -net.ipv4.conf.default.log_martians = 0 -net.ipv4.conf.default.mc_forwarding = 0 -net.ipv4.conf.default.medium_id = 0 -net.ipv4.conf.default.promote_secondaries = 0 -net.ipv4.conf.default.proxy_arp = 0 -net.ipv4.conf.default.proxy_arp_pvlan = 0 -net.ipv4.conf.default.route_localnet = 0 -net.ipv4.conf.default.rp_filter = 2 -net.ipv4.conf.default.secure_redirects = 1 -net.ipv4.conf.default.send_redirects = 1 -net.ipv4.conf.default.shared_media = 1 -net.ipv4.conf.default.src_valid_mark = 0 -net.ipv4.conf.default.tag = 0 -net.ipv4.conf.eth0.accept_local = 0 -net.ipv4.conf.eth0.accept_redirects = 1 -net.ipv4.conf.eth0.accept_source_route = 1 -net.ipv4.conf.eth0.arp_accept = 0 -net.ipv4.conf.eth0.arp_announce = 0 -net.ipv4.conf.eth0.arp_evict_nocarrier = 1 -net.ipv4.conf.eth0.arp_filter = 0 -net.ipv4.conf.eth0.arp_ignore = 0 -net.ipv4.conf.eth0.arp_notify = 0 -net.ipv4.conf.eth0.bc_forwarding = 0 -net.ipv4.conf.eth0.bootp_relay = 0 -net.ipv4.conf.eth0.disable_policy = 0 -net.ipv4.conf.eth0.disable_xfrm = 0 -net.ipv4.conf.eth0.drop_gratuitous_arp = 0 -net.ipv4.conf.eth0.drop_unicast_in_l2_multicast = 0 -net.ipv4.conf.eth0.force_igmp_version = 0 -net.ipv4.conf.eth0.forwarding = 1 -net.ipv4.conf.eth0.igmpv2_unsolicited_report_interval = 10000 -net.ipv4.conf.eth0.igmpv3_unsolicited_report_interval = 1000 -net.ipv4.conf.eth0.ignore_routes_with_linkdown = 0 -net.ipv4.conf.eth0.log_martians = 0 -net.ipv4.conf.eth0.mc_forwarding = 0 -net.ipv4.conf.eth0.medium_id = 0 -net.ipv4.conf.eth0.promote_secondaries = 0 -net.ipv4.conf.eth0.proxy_arp = 0 -net.ipv4.conf.eth0.proxy_arp_pvlan = 0 -net.ipv4.conf.eth0.route_localnet = 0 -net.ipv4.conf.eth0.rp_filter = 2 -net.ipv4.conf.eth0.secure_redirects = 1 -net.ipv4.conf.eth0.send_redirects = 1 -net.ipv4.conf.eth0.shared_media = 1 -net.ipv4.conf.eth0.src_valid_mark = 0 -net.ipv4.conf.eth0.tag = 0 -net.ipv4.conf.lo.accept_local = 0 -net.ipv4.conf.lo.accept_redirects = 1 -net.ipv4.conf.lo.accept_source_route = 1 -net.ipv4.conf.lo.arp_accept = 0 -net.ipv4.conf.lo.arp_announce = 0 -net.ipv4.conf.lo.arp_evict_nocarrier = 1 -net.ipv4.conf.lo.arp_filter = 0 -net.ipv4.conf.lo.arp_ignore = 0 -net.ipv4.conf.lo.arp_notify = 0 -net.ipv4.conf.lo.bc_forwarding = 0 -net.ipv4.conf.lo.bootp_relay = 0 -net.ipv4.conf.lo.disable_policy = 1 -net.ipv4.conf.lo.disable_xfrm = 1 -net.ipv4.conf.lo.drop_gratuitous_arp = 0 -net.ipv4.conf.lo.drop_unicast_in_l2_multicast = 0 -net.ipv4.conf.lo.force_igmp_version = 0 -net.ipv4.conf.lo.forwarding = 1 -net.ipv4.conf.lo.igmpv2_unsolicited_report_interval = 10000 -net.ipv4.conf.lo.igmpv3_unsolicited_report_interval = 1000 -net.ipv4.conf.lo.ignore_routes_with_linkdown = 0 -net.ipv4.conf.lo.log_martians = 0 -net.ipv4.conf.lo.mc_forwarding = 0 -net.ipv4.conf.lo.medium_id = 0 -net.ipv4.conf.lo.promote_secondaries = 0 -net.ipv4.conf.lo.proxy_arp = 0 -net.ipv4.conf.lo.proxy_arp_pvlan = 0 -net.ipv4.conf.lo.route_localnet = 0 -net.ipv4.conf.lo.rp_filter = 2 -net.ipv4.conf.lo.secure_redirects = 1 -net.ipv4.conf.lo.send_redirects = 1 -net.ipv4.conf.lo.shared_media = 1 -net.ipv4.conf.lo.src_valid_mark = 0 -net.ipv4.conf.lo.tag = 0 -net.ipv4.fib_multipath_hash_fields = 7 -net.ipv4.fib_multipath_hash_policy = 0 -net.ipv4.fib_multipath_use_neigh = 0 -net.ipv4.fib_notify_on_flag_change = 0 -net.ipv4.fwmark_reflect = 0 -net.ipv4.icmp_echo_enable_probe = 0 -net.ipv4.icmp_echo_ignore_all = 0 -net.ipv4.icmp_echo_ignore_broadcasts = 1 -net.ipv4.icmp_errors_use_inbound_ifaddr = 0 -net.ipv4.icmp_ignore_bogus_error_responses = 1 -net.ipv4.icmp_ratelimit = 1000 -net.ipv4.icmp_ratemask = 6168 -net.ipv4.igmp_link_local_mcast_reports = 1 -net.ipv4.igmp_max_memberships = 20 -net.ipv4.igmp_max_msf = 10 -net.ipv4.igmp_qrv = 2 -net.ipv4.ip_autobind_reuse = 0 -net.ipv4.ip_default_ttl = 64 -net.ipv4.ip_dynaddr = 0 -net.ipv4.ip_early_demux = 1 -net.ipv4.ip_forward = 1 -net.ipv4.ip_forward_update_priority = 1 -net.ipv4.ip_forward_use_pmtu = 0 -net.ipv4.ip_local_port_range = 32768 60999 -net.ipv4.ip_local_reserved_ports = -net.ipv4.ip_no_pmtu_disc = 0 -net.ipv4.ip_nonlocal_bind = 0 -net.ipv4.ip_unprivileged_port_start = 0 -net.ipv4.ipfrag_high_thresh = 4194304 -net.ipv4.ipfrag_low_thresh = 3145728 -net.ipv4.ipfrag_max_dist = 64 -net.ipv4.ipfrag_time = 30 -net.ipv4.neigh.eth0.anycast_delay = 100 -net.ipv4.neigh.eth0.app_solicit = 0 -net.ipv4.neigh.eth0.base_reachable_time_ms = 30000 -net.ipv4.neigh.eth0.delay_first_probe_time = 5 -net.ipv4.neigh.eth0.gc_stale_time = 60 -net.ipv4.neigh.eth0.interval_probe_time_ms = 5000 -net.ipv4.neigh.eth0.locktime = 100 -net.ipv4.neigh.eth0.mcast_resolicit = 0 -net.ipv4.neigh.eth0.mcast_solicit = 3 -net.ipv4.neigh.eth0.proxy_delay = 80 -net.ipv4.neigh.eth0.proxy_qlen = 64 -net.ipv4.neigh.eth0.retrans_time_ms = 1000 -net.ipv4.neigh.eth0.ucast_solicit = 3 -net.ipv4.neigh.eth0.unres_qlen = 101 -net.ipv4.neigh.eth0.unres_qlen_bytes = 212992 -net.ipv4.neigh.lo.anycast_delay = 100 -net.ipv4.neigh.lo.app_solicit = 0 -net.ipv4.neigh.lo.base_reachable_time_ms = 30000 -net.ipv4.neigh.lo.delay_first_probe_time = 5 -net.ipv4.neigh.lo.gc_stale_time = 60 -net.ipv4.neigh.lo.interval_probe_time_ms = 5000 -net.ipv4.neigh.lo.locktime = 100 -net.ipv4.neigh.lo.mcast_resolicit = 0 -net.ipv4.neigh.lo.mcast_solicit = 3 -net.ipv4.neigh.lo.proxy_delay = 80 -net.ipv4.neigh.lo.proxy_qlen = 64 -net.ipv4.neigh.lo.retrans_time_ms = 1000 -net.ipv4.neigh.lo.ucast_solicit = 3 -net.ipv4.neigh.lo.unres_qlen = 101 -net.ipv4.neigh.lo.unres_qlen_bytes = 212992 -net.ipv4.nexthop_compat_mode = 1 -net.ipv4.ping_group_range = 0 2147483647 -net.ipv4.raw_l3mdev_accept = 1 -net.ipv4.route.min_adv_mss = 256 -net.ipv4.route.min_pmtu = 552 -net.ipv4.route.mtu_expires = 600 -net.ipv4.tcp_abort_on_overflow = 0 -net.ipv4.tcp_adv_win_scale = 1 -net.ipv4.tcp_allowed_congestion_control = reno cubic -net.ipv4.tcp_app_win = 31 -net.ipv4.tcp_autocorking = 1 -net.ipv4.tcp_available_congestion_control = reno cubic -net.ipv4.tcp_backlog_ack_defer = 1 -net.ipv4.tcp_base_mss = 1024 -net.ipv4.tcp_challenge_ack_limit = 2147483647 -net.ipv4.tcp_child_ehash_entries = 0 -net.ipv4.tcp_comp_sack_delay_ns = 1000000 -net.ipv4.tcp_comp_sack_nr = 44 -net.ipv4.tcp_comp_sack_slack_ns = 100000 -net.ipv4.tcp_congestion_control = cubic -net.ipv4.tcp_dsack = 1 -net.ipv4.tcp_early_demux = 1 -net.ipv4.tcp_early_retrans = 3 -net.ipv4.tcp_ecn = 2 -net.ipv4.tcp_ecn_fallback = 1 -net.ipv4.tcp_ehash_entries = -524288 -net.ipv4.tcp_fack = 0 -net.ipv4.tcp_fastopen = 1 -net.ipv4.tcp_fastopen_blackhole_timeout_sec = 0 -net.ipv4.tcp_fastopen_key = 00000000-00000000-00000000-00000000 -net.ipv4.tcp_fin_timeout = 60 -net.ipv4.tcp_frto = 2 -net.ipv4.tcp_fwmark_accept = 0 -net.ipv4.tcp_invalid_ratelimit = 500 -net.ipv4.tcp_keepalive_intvl = 75 -net.ipv4.tcp_keepalive_probes = 9 -net.ipv4.tcp_keepalive_time = 7200 -net.ipv4.tcp_l3mdev_accept = 0 -net.ipv4.tcp_limit_output_bytes = 1048576 -net.ipv4.tcp_max_reordering = 300 -net.ipv4.tcp_max_syn_backlog = 4096 -net.ipv4.tcp_max_tw_buckets = 262144 -net.ipv4.tcp_migrate_req = 0 -net.ipv4.tcp_min_rtt_wlen = 300 -net.ipv4.tcp_min_snd_mss = 48 -net.ipv4.tcp_min_tso_segs = 2 -net.ipv4.tcp_moderate_rcvbuf = 1 -net.ipv4.tcp_mtu_probe_floor = 48 -net.ipv4.tcp_mtu_probing = 0 -net.ipv4.tcp_no_metrics_save = 0 -net.ipv4.tcp_no_ssthresh_metrics_save = 1 -net.ipv4.tcp_notsent_lowat = 4294967295 -net.ipv4.tcp_orphan_retries = 0 -net.ipv4.tcp_pacing_ca_ratio = 120 -net.ipv4.tcp_pacing_ss_ratio = 200 -net.ipv4.tcp_pingpong_thresh = 1 -net.ipv4.tcp_plb_cong_thresh = 128 -net.ipv4.tcp_plb_enabled = 0 -net.ipv4.tcp_plb_idle_rehash_rounds = 3 -net.ipv4.tcp_plb_rehash_rounds = 12 -net.ipv4.tcp_plb_suspend_rto_sec = 60 -net.ipv4.tcp_probe_interval = 600 -net.ipv4.tcp_probe_threshold = 8 -net.ipv4.tcp_recovery = 1 -net.ipv4.tcp_reflect_tos = 0 -net.ipv4.tcp_reordering = 3 -net.ipv4.tcp_retrans_collapse = 1 -net.ipv4.tcp_retries1 = 3 -net.ipv4.tcp_retries2 = 15 -net.ipv4.tcp_rfc1337 = 0 -net.ipv4.tcp_rmem = 4096 131072 6291456 -net.ipv4.tcp_sack = 1 -net.ipv4.tcp_shrink_window = 0 -net.ipv4.tcp_slow_start_after_idle = 1 -net.ipv4.tcp_stdurg = 0 -net.ipv4.tcp_syn_linear_timeouts = 4 -net.ipv4.tcp_syn_retries = 6 -net.ipv4.tcp_synack_retries = 5 -net.ipv4.tcp_syncookies = 1 -net.ipv4.tcp_thin_linear_timeouts = 0 -net.ipv4.tcp_timestamps = 1 -net.ipv4.tcp_tso_rtt_log = 9 -net.ipv4.tcp_tso_win_divisor = 3 -net.ipv4.tcp_tw_reuse = 2 -net.ipv4.tcp_window_scaling = 1 -net.ipv4.tcp_wmem = 4096 16384 4194304 -net.ipv4.tcp_workaround_signed_windows = 0 -net.ipv4.udp_child_hash_entries = 0 -net.ipv4.udp_early_demux = 1 -net.ipv4.udp_hash_entries = -65536 -net.ipv4.udp_l3mdev_accept = 0 -net.ipv4.udp_rmem_min = 4096 -net.ipv4.udp_wmem_min = 4096 -net.ipv4.xfrm4_gc_thresh = 32768 -net.ipv6.anycast_src_echo_reply = 0 -net.ipv6.auto_flowlabels = 1 -net.ipv6.bindv6only = 0 -net.ipv6.conf.all.accept_dad = 0 -net.ipv6.conf.all.accept_ra = 1 -net.ipv6.conf.all.accept_ra_defrtr = 1 -net.ipv6.conf.all.accept_ra_from_local = 0 -net.ipv6.conf.all.accept_ra_min_hop_limit = 1 -net.ipv6.conf.all.accept_ra_min_lft = 0 -net.ipv6.conf.all.accept_ra_mtu = 1 -net.ipv6.conf.all.accept_ra_pinfo = 1 -net.ipv6.conf.all.accept_ra_rt_info_max_plen = 0 -net.ipv6.conf.all.accept_ra_rt_info_min_plen = 0 -net.ipv6.conf.all.accept_ra_rtr_pref = 1 -net.ipv6.conf.all.accept_redirects = 1 -net.ipv6.conf.all.accept_source_route = 0 -net.ipv6.conf.all.accept_untracked_na = 0 -net.ipv6.conf.all.addr_gen_mode = 0 -net.ipv6.conf.all.autoconf = 1 -net.ipv6.conf.all.dad_transmits = 1 -net.ipv6.conf.all.disable_ipv6 = 0 -net.ipv6.conf.all.disable_policy = 0 -net.ipv6.conf.all.drop_unicast_in_l2_multicast = 0 -net.ipv6.conf.all.drop_unsolicited_na = 0 -net.ipv6.conf.all.enhanced_dad = 1 -net.ipv6.conf.all.force_mld_version = 0 -net.ipv6.conf.all.force_tllao = 0 -net.ipv6.conf.all.forwarding = 0 -net.ipv6.conf.all.hop_limit = 64 -net.ipv6.conf.all.ignore_routes_with_linkdown = 0 -net.ipv6.conf.all.ioam6_enabled = 0 -net.ipv6.conf.all.ioam6_id = 65535 -net.ipv6.conf.all.ioam6_id_wide = 4294967295 -net.ipv6.conf.all.keep_addr_on_down = 0 -net.ipv6.conf.all.max_addresses = 16 -net.ipv6.conf.all.max_desync_factor = 600 -net.ipv6.conf.all.mc_forwarding = 0 -net.ipv6.conf.all.mldv1_unsolicited_report_interval = 10000 -net.ipv6.conf.all.mldv2_unsolicited_report_interval = 1000 -net.ipv6.conf.all.mtu = 1280 -net.ipv6.conf.all.ndisc_evict_nocarrier = 1 -net.ipv6.conf.all.ndisc_notify = 0 -net.ipv6.conf.all.ndisc_tclass = 0 -net.ipv6.conf.all.proxy_ndp = 0 -net.ipv6.conf.all.ra_defrtr_metric = 1024 -net.ipv6.conf.all.ra_honor_pio_life = 0 -net.ipv6.conf.all.regen_max_retry = 3 -net.ipv6.conf.all.router_probe_interval = 60 -net.ipv6.conf.all.router_solicitation_delay = 1 -net.ipv6.conf.all.router_solicitation_interval = 4 -net.ipv6.conf.all.router_solicitation_max_interval = 3600 -net.ipv6.conf.all.router_solicitations = -1 -net.ipv6.conf.all.rpl_seg_enabled = 0 -net.ipv6.conf.all.seg6_enabled = 0 -net.ipv6.conf.all.seg6_require_hmac = 0 -net.ipv6.conf.all.suppress_frag_ndisc = 1 -net.ipv6.conf.all.temp_prefered_lft = 86400 -net.ipv6.conf.all.temp_valid_lft = 604800 -net.ipv6.conf.all.use_oif_addrs_only = 0 -net.ipv6.conf.all.use_tempaddr = 0 -net.ipv6.conf.default.accept_dad = 1 -net.ipv6.conf.default.accept_ra = 1 -net.ipv6.conf.default.accept_ra_defrtr = 1 -net.ipv6.conf.default.accept_ra_from_local = 0 -net.ipv6.conf.default.accept_ra_min_hop_limit = 1 -net.ipv6.conf.default.accept_ra_min_lft = 0 -net.ipv6.conf.default.accept_ra_mtu = 1 -net.ipv6.conf.default.accept_ra_pinfo = 1 -net.ipv6.conf.default.accept_ra_rt_info_max_plen = 0 -net.ipv6.conf.default.accept_ra_rt_info_min_plen = 0 -net.ipv6.conf.default.accept_ra_rtr_pref = 1 -net.ipv6.conf.default.accept_redirects = 1 -net.ipv6.conf.default.accept_source_route = 0 -net.ipv6.conf.default.accept_untracked_na = 0 -net.ipv6.conf.default.addr_gen_mode = 0 -net.ipv6.conf.default.autoconf = 1 -net.ipv6.conf.default.dad_transmits = 1 -net.ipv6.conf.default.disable_ipv6 = 0 -net.ipv6.conf.default.disable_policy = 0 -net.ipv6.conf.default.drop_unicast_in_l2_multicast = 0 -net.ipv6.conf.default.drop_unsolicited_na = 0 -net.ipv6.conf.default.enhanced_dad = 1 -net.ipv6.conf.default.force_mld_version = 0 -net.ipv6.conf.default.force_tllao = 0 -net.ipv6.conf.default.forwarding = 0 -net.ipv6.conf.default.hop_limit = 64 -net.ipv6.conf.default.ignore_routes_with_linkdown = 0 -net.ipv6.conf.default.ioam6_enabled = 0 -net.ipv6.conf.default.ioam6_id = 65535 -net.ipv6.conf.default.ioam6_id_wide = 4294967295 -net.ipv6.conf.default.keep_addr_on_down = 0 -net.ipv6.conf.default.max_addresses = 16 -net.ipv6.conf.default.max_desync_factor = 600 -net.ipv6.conf.default.mc_forwarding = 0 -net.ipv6.conf.default.mldv1_unsolicited_report_interval = 10000 -net.ipv6.conf.default.mldv2_unsolicited_report_interval = 1000 -net.ipv6.conf.default.mtu = 1280 -net.ipv6.conf.default.ndisc_evict_nocarrier = 1 -net.ipv6.conf.default.ndisc_notify = 0 -net.ipv6.conf.default.ndisc_tclass = 0 -net.ipv6.conf.default.proxy_ndp = 0 -net.ipv6.conf.default.ra_defrtr_metric = 1024 -net.ipv6.conf.default.ra_honor_pio_life = 0 -net.ipv6.conf.default.regen_max_retry = 3 -net.ipv6.conf.default.router_probe_interval = 60 -net.ipv6.conf.default.router_solicitation_delay = 1 -net.ipv6.conf.default.router_solicitation_interval = 4 -net.ipv6.conf.default.router_solicitation_max_interval = 3600 -net.ipv6.conf.default.router_solicitations = -1 -net.ipv6.conf.default.rpl_seg_enabled = 0 -net.ipv6.conf.default.seg6_enabled = 0 -net.ipv6.conf.default.seg6_require_hmac = 0 -net.ipv6.conf.default.suppress_frag_ndisc = 1 -net.ipv6.conf.default.temp_prefered_lft = 86400 -net.ipv6.conf.default.temp_valid_lft = 604800 -net.ipv6.conf.default.use_oif_addrs_only = 0 -net.ipv6.conf.default.use_tempaddr = 0 -net.ipv6.conf.eth0.accept_dad = 1 -net.ipv6.conf.eth0.accept_ra = 1 -net.ipv6.conf.eth0.accept_ra_defrtr = 1 -net.ipv6.conf.eth0.accept_ra_from_local = 0 -net.ipv6.conf.eth0.accept_ra_min_hop_limit = 1 -net.ipv6.conf.eth0.accept_ra_min_lft = 0 -net.ipv6.conf.eth0.accept_ra_mtu = 1 -net.ipv6.conf.eth0.accept_ra_pinfo = 1 -net.ipv6.conf.eth0.accept_ra_rt_info_max_plen = 0 -net.ipv6.conf.eth0.accept_ra_rt_info_min_plen = 0 -net.ipv6.conf.eth0.accept_ra_rtr_pref = 1 -net.ipv6.conf.eth0.accept_redirects = 1 -net.ipv6.conf.eth0.accept_source_route = 0 -net.ipv6.conf.eth0.accept_untracked_na = 0 -net.ipv6.conf.eth0.addr_gen_mode = 0 -net.ipv6.conf.eth0.autoconf = 1 -net.ipv6.conf.eth0.dad_transmits = 1 -net.ipv6.conf.eth0.disable_ipv6 = 1 -net.ipv6.conf.eth0.disable_policy = 0 -net.ipv6.conf.eth0.drop_unicast_in_l2_multicast = 0 -net.ipv6.conf.eth0.drop_unsolicited_na = 0 -net.ipv6.conf.eth0.enhanced_dad = 1 -net.ipv6.conf.eth0.force_mld_version = 0 -net.ipv6.conf.eth0.force_tllao = 0 -net.ipv6.conf.eth0.forwarding = 0 -net.ipv6.conf.eth0.hop_limit = 64 -net.ipv6.conf.eth0.ignore_routes_with_linkdown = 0 -net.ipv6.conf.eth0.ioam6_enabled = 0 -net.ipv6.conf.eth0.ioam6_id = 65535 -net.ipv6.conf.eth0.ioam6_id_wide = 4294967295 -net.ipv6.conf.eth0.keep_addr_on_down = 0 -net.ipv6.conf.eth0.max_addresses = 16 -net.ipv6.conf.eth0.max_desync_factor = 600 -net.ipv6.conf.eth0.mc_forwarding = 0 -net.ipv6.conf.eth0.mldv1_unsolicited_report_interval = 10000 -net.ipv6.conf.eth0.mldv2_unsolicited_report_interval = 1000 -net.ipv6.conf.eth0.mtu = 1500 -net.ipv6.conf.eth0.ndisc_evict_nocarrier = 1 -net.ipv6.conf.eth0.ndisc_notify = 0 -net.ipv6.conf.eth0.ndisc_tclass = 0 -net.ipv6.conf.eth0.proxy_ndp = 0 -net.ipv6.conf.eth0.ra_defrtr_metric = 1024 -net.ipv6.conf.eth0.ra_honor_pio_life = 0 -net.ipv6.conf.eth0.regen_max_retry = 3 -net.ipv6.conf.eth0.router_probe_interval = 60 -net.ipv6.conf.eth0.router_solicitation_delay = 1 -net.ipv6.conf.eth0.router_solicitation_interval = 4 -net.ipv6.conf.eth0.router_solicitation_max_interval = 3600 -net.ipv6.conf.eth0.router_solicitations = -1 -net.ipv6.conf.eth0.rpl_seg_enabled = 0 -net.ipv6.conf.eth0.seg6_enabled = 0 -net.ipv6.conf.eth0.seg6_require_hmac = 0 -net.ipv6.conf.eth0.suppress_frag_ndisc = 1 -net.ipv6.conf.eth0.temp_prefered_lft = 86400 -net.ipv6.conf.eth0.temp_valid_lft = 604800 -net.ipv6.conf.eth0.use_oif_addrs_only = 0 -net.ipv6.conf.eth0.use_tempaddr = 0 -net.ipv6.conf.lo.accept_dad = -1 -net.ipv6.conf.lo.accept_ra = 1 -net.ipv6.conf.lo.accept_ra_defrtr = 1 -net.ipv6.conf.lo.accept_ra_from_local = 0 -net.ipv6.conf.lo.accept_ra_min_hop_limit = 1 -net.ipv6.conf.lo.accept_ra_min_lft = 0 -net.ipv6.conf.lo.accept_ra_mtu = 1 -net.ipv6.conf.lo.accept_ra_pinfo = 1 -net.ipv6.conf.lo.accept_ra_rt_info_max_plen = 0 -net.ipv6.conf.lo.accept_ra_rt_info_min_plen = 0 -net.ipv6.conf.lo.accept_ra_rtr_pref = 1 -net.ipv6.conf.lo.accept_redirects = 1 -net.ipv6.conf.lo.accept_source_route = 0 -net.ipv6.conf.lo.accept_untracked_na = 0 -net.ipv6.conf.lo.addr_gen_mode = 0 -net.ipv6.conf.lo.autoconf = 1 -net.ipv6.conf.lo.dad_transmits = 1 -net.ipv6.conf.lo.disable_ipv6 = 0 -net.ipv6.conf.lo.disable_policy = 0 -net.ipv6.conf.lo.drop_unicast_in_l2_multicast = 0 -net.ipv6.conf.lo.drop_unsolicited_na = 0 -net.ipv6.conf.lo.enhanced_dad = 1 -net.ipv6.conf.lo.force_mld_version = 0 -net.ipv6.conf.lo.force_tllao = 0 -net.ipv6.conf.lo.forwarding = 0 -net.ipv6.conf.lo.hop_limit = 64 -net.ipv6.conf.lo.ignore_routes_with_linkdown = 0 -net.ipv6.conf.lo.ioam6_enabled = 0 -net.ipv6.conf.lo.ioam6_id = 65535 -net.ipv6.conf.lo.ioam6_id_wide = 4294967295 -net.ipv6.conf.lo.keep_addr_on_down = 0 -net.ipv6.conf.lo.max_addresses = 16 -net.ipv6.conf.lo.max_desync_factor = 600 -net.ipv6.conf.lo.mc_forwarding = 0 -net.ipv6.conf.lo.mldv1_unsolicited_report_interval = 10000 -net.ipv6.conf.lo.mldv2_unsolicited_report_interval = 1000 -net.ipv6.conf.lo.mtu = 65536 -net.ipv6.conf.lo.ndisc_evict_nocarrier = 1 -net.ipv6.conf.lo.ndisc_notify = 0 -net.ipv6.conf.lo.ndisc_tclass = 0 -net.ipv6.conf.lo.proxy_ndp = 0 -net.ipv6.conf.lo.ra_defrtr_metric = 1024 -net.ipv6.conf.lo.ra_honor_pio_life = 0 -net.ipv6.conf.lo.regen_max_retry = 3 -net.ipv6.conf.lo.router_probe_interval = 60 -net.ipv6.conf.lo.router_solicitation_delay = 1 -net.ipv6.conf.lo.router_solicitation_interval = 4 -net.ipv6.conf.lo.router_solicitation_max_interval = 3600 -net.ipv6.conf.lo.router_solicitations = -1 -net.ipv6.conf.lo.rpl_seg_enabled = 0 -net.ipv6.conf.lo.seg6_enabled = 0 -net.ipv6.conf.lo.seg6_require_hmac = 0 -net.ipv6.conf.lo.suppress_frag_ndisc = 1 -net.ipv6.conf.lo.temp_prefered_lft = 86400 -net.ipv6.conf.lo.temp_valid_lft = 604800 -net.ipv6.conf.lo.use_oif_addrs_only = 0 -net.ipv6.conf.lo.use_tempaddr = -1 -net.ipv6.fib_multipath_hash_fields = 7 -net.ipv6.fib_multipath_hash_policy = 0 -net.ipv6.fib_notify_on_flag_change = 0 -net.ipv6.flowlabel_consistency = 1 -net.ipv6.flowlabel_reflect = 0 -net.ipv6.flowlabel_state_ranges = 0 -net.ipv6.fwmark_reflect = 0 -net.ipv6.icmp.echo_ignore_all = 0 -net.ipv6.icmp.echo_ignore_anycast = 0 -net.ipv6.icmp.echo_ignore_multicast = 0 -net.ipv6.icmp.error_anycast_as_unicast = 0 -net.ipv6.icmp.ratelimit = 1000 -net.ipv6.icmp.ratemask = 0-1,3-127 -net.ipv6.idgen_delay = 1 -net.ipv6.idgen_retries = 3 -net.ipv6.ioam6_id = 16777215 -net.ipv6.ioam6_id_wide = 72057594037927935 -net.ipv6.ip6frag_high_thresh = 4194304 -net.ipv6.ip6frag_low_thresh = 3145728 -net.ipv6.ip6frag_time = 60 -net.ipv6.ip_nonlocal_bind = 0 -net.ipv6.max_dst_opts_length = 2147483647 -net.ipv6.max_dst_opts_number = 8 -net.ipv6.max_hbh_length = 2147483647 -net.ipv6.max_hbh_opts_number = 8 -net.ipv6.neigh.eth0.anycast_delay = 100 -net.ipv6.neigh.eth0.app_solicit = 0 -net.ipv6.neigh.eth0.base_reachable_time_ms = 30000 -net.ipv6.neigh.eth0.delay_first_probe_time = 5 -net.ipv6.neigh.eth0.gc_stale_time = 60 -net.ipv6.neigh.eth0.interval_probe_time_ms = 5000 -net.ipv6.neigh.eth0.locktime = 0 -net.ipv6.neigh.eth0.mcast_resolicit = 0 -net.ipv6.neigh.eth0.mcast_solicit = 3 -net.ipv6.neigh.eth0.proxy_delay = 80 -net.ipv6.neigh.eth0.proxy_qlen = 64 -net.ipv6.neigh.eth0.retrans_time_ms = 1000 -net.ipv6.neigh.eth0.ucast_solicit = 3 -net.ipv6.neigh.eth0.unres_qlen = 101 -net.ipv6.neigh.eth0.unres_qlen_bytes = 212992 -net.ipv6.neigh.lo.anycast_delay = 100 -net.ipv6.neigh.lo.app_solicit = 0 -net.ipv6.neigh.lo.base_reachable_time_ms = 30000 -net.ipv6.neigh.lo.delay_first_probe_time = 5 -net.ipv6.neigh.lo.gc_stale_time = 60 -net.ipv6.neigh.lo.interval_probe_time_ms = 5000 -net.ipv6.neigh.lo.locktime = 0 -net.ipv6.neigh.lo.mcast_resolicit = 0 -net.ipv6.neigh.lo.mcast_solicit = 3 -net.ipv6.neigh.lo.proxy_delay = 80 -net.ipv6.neigh.lo.proxy_qlen = 64 -net.ipv6.neigh.lo.retrans_time_ms = 1000 -net.ipv6.neigh.lo.ucast_solicit = 3 -net.ipv6.neigh.lo.unres_qlen = 101 -net.ipv6.neigh.lo.unres_qlen_bytes = 212992 -net.ipv6.route.gc_elasticity = 9 -net.ipv6.route.gc_interval = 30 -net.ipv6.route.gc_min_interval = 0 -net.ipv6.route.gc_min_interval_ms = 500 -net.ipv6.route.gc_thresh = 1024 -net.ipv6.route.gc_timeout = 60 -net.ipv6.route.max_size = 2147483647 -net.ipv6.route.min_adv_mss = 1220 -net.ipv6.route.mtu_expires = 600 -net.ipv6.route.skip_notify_on_dev_down = 0 -net.ipv6.seg6_flowlabel = 0 -net.ipv6.xfrm6_gc_thresh = 32768 -net.mptcp.add_addr_timeout = 120 -net.mptcp.allow_join_initial_addr_port = 1 -net.mptcp.checksum_enabled = 0 -net.mptcp.close_timeout = 60 -net.mptcp.enabled = 1 -net.mptcp.pm_type = 0 -net.mptcp.scheduler = default -net.mptcp.stale_loss_cnt = 4 -net.netfilter.nf_conntrack_acct = 0 -net.netfilter.nf_conntrack_buckets = 262144 -net.netfilter.nf_conntrack_checksum = 1 -net.netfilter.nf_conntrack_count = 0 -net.netfilter.nf_conntrack_dccp_loose = 1 -net.netfilter.nf_conntrack_dccp_timeout_closereq = 64 -net.netfilter.nf_conntrack_dccp_timeout_closing = 64 -net.netfilter.nf_conntrack_dccp_timeout_open = 43200 -net.netfilter.nf_conntrack_dccp_timeout_partopen = 480 -net.netfilter.nf_conntrack_dccp_timeout_request = 240 -net.netfilter.nf_conntrack_dccp_timeout_respond = 480 -net.netfilter.nf_conntrack_dccp_timeout_timewait = 240 -net.netfilter.nf_conntrack_events = 2 -net.netfilter.nf_conntrack_expect_max = 4096 -net.netfilter.nf_conntrack_frag6_high_thresh = 4194304 -net.netfilter.nf_conntrack_frag6_low_thresh = 3145728 -net.netfilter.nf_conntrack_frag6_timeout = 60 -net.netfilter.nf_conntrack_generic_timeout = 600 -net.netfilter.nf_conntrack_gre_timeout = 30 -net.netfilter.nf_conntrack_gre_timeout_stream = 180 -net.netfilter.nf_conntrack_icmp_timeout = 30 -net.netfilter.nf_conntrack_icmpv6_timeout = 30 -net.netfilter.nf_conntrack_log_invalid = 0 -net.netfilter.nf_conntrack_max = 262144 -net.netfilter.nf_conntrack_sctp_timeout_closed = 10 -net.netfilter.nf_conntrack_sctp_timeout_cookie_echoed = 3 -net.netfilter.nf_conntrack_sctp_timeout_cookie_wait = 3 -net.netfilter.nf_conntrack_sctp_timeout_established = 210 -net.netfilter.nf_conntrack_sctp_timeout_heartbeat_sent = 30 -net.netfilter.nf_conntrack_sctp_timeout_shutdown_ack_sent = 3 -net.netfilter.nf_conntrack_sctp_timeout_shutdown_recd = 3 -net.netfilter.nf_conntrack_sctp_timeout_shutdown_sent = 3 -net.netfilter.nf_conntrack_tcp_be_liberal = 0 -net.netfilter.nf_conntrack_tcp_ignore_invalid_rst = 0 -net.netfilter.nf_conntrack_tcp_loose = 1 -net.netfilter.nf_conntrack_tcp_max_retrans = 3 -net.netfilter.nf_conntrack_tcp_timeout_close = 10 -net.netfilter.nf_conntrack_tcp_timeout_close_wait = 60 -net.netfilter.nf_conntrack_tcp_timeout_established = 432000 -net.netfilter.nf_conntrack_tcp_timeout_fin_wait = 120 -net.netfilter.nf_conntrack_tcp_timeout_last_ack = 30 -net.netfilter.nf_conntrack_tcp_timeout_max_retrans = 300 -net.netfilter.nf_conntrack_tcp_timeout_syn_recv = 60 -net.netfilter.nf_conntrack_tcp_timeout_syn_sent = 120 -net.netfilter.nf_conntrack_tcp_timeout_time_wait = 120 -net.netfilter.nf_conntrack_tcp_timeout_unacknowledged = 300 -net.netfilter.nf_conntrack_timestamp = 0 -net.netfilter.nf_conntrack_udp_timeout = 30 -net.netfilter.nf_conntrack_udp_timeout_stream = 120 -net.netfilter.nf_flowtable_tcp_timeout = 30 -net.netfilter.nf_flowtable_udp_timeout = 30 -net.netfilter.nf_hooks_lwtunnel = 0 -net.netfilter.nf_log.0 = NONE -net.netfilter.nf_log.1 = NONE -net.netfilter.nf_log.10 = NONE -net.netfilter.nf_log.2 = NONE -net.netfilter.nf_log.3 = NONE -net.netfilter.nf_log.4 = NONE -net.netfilter.nf_log.5 = NONE -net.netfilter.nf_log.6 = NONE -net.netfilter.nf_log.7 = NONE -net.netfilter.nf_log.8 = NONE -net.netfilter.nf_log.9 = NONE -net.unix.max_dgram_qlen = 10 -user.max_cgroup_namespaces = 770559 -user.max_fanotify_groups = 128 -user.max_fanotify_marks = 1048576 -user.max_inotify_instances = 128 -user.max_inotify_watches = 65536 -user.max_ipc_namespaces = 770559 -user.max_mnt_namespaces = 770559 -user.max_net_namespaces = 770559 -user.max_pid_namespaces = 770559 -user.max_time_namespaces = 770559 -user.max_user_namespaces = 770559 -user.max_uts_namespaces = 770559 -vm.admin_reserve_kbytes = 8192 -vm.compact_unevictable_allowed = 1 -vm.compaction_proactiveness = 20 -vm.dirty_background_bytes = 0 -vm.dirty_background_ratio = 10 -vm.dirty_bytes = 0 -vm.dirty_expire_centisecs = 3000 -vm.dirty_ratio = 20 -vm.dirty_writeback_centisecs = 500 -vm.dirtytime_expire_seconds = 43200 -vm.extfrag_threshold = 500 -vm.hugetlb_optimize_vmemmap = 0 -vm.hugetlb_shm_group = 0 -vm.laptop_mode = 0 -vm.legacy_va_layout = 0 -vm.lowmem_reserve_ratio = 256 256 32 0 0 -vm.max_map_count = 1048576 -vm.memfd_noexec = 0 -vm.memory_failure_early_kill = 0 -vm.memory_failure_recovery = 1 -vm.min_free_kbytes = 67584 -vm.min_slab_ratio = 5 -vm.min_unmapped_ratio = 1 -vm.mmap_min_addr = 65536 -vm.mmap_rnd_bits = 32 -vm.mmap_rnd_compat_bits = 16 -vm.nr_hugepages = 0 -vm.nr_hugepages_mempolicy = 0 -vm.nr_overcommit_hugepages = 0 -vm.numa_stat = 1 -vm.numa_zonelist_order = Node -vm.oom_dump_tasks = 1 -vm.oom_kill_allocating_task = 0 -vm.overcommit_kbytes = 0 -vm.overcommit_memory = 0 -vm.overcommit_ratio = 50 -vm.page-cluster = 3 -vm.page_lock_unfairness = 5 -vm.panic_on_oom = 0 -vm.percpu_pagelist_high_fraction = 0 -vm.stat_interval = 1 -vm.swappiness = 60 -vm.unprivileged_userfaultfd = 0 -vm.user_reserve_kbytes = 131072 -vm.vfs_cache_pressure = 100 -vm.watermark_boost_factor = 15000 -vm.watermark_scale_factor = 10 -vm.zone_reclaim_mode = 0 ------------------------------------------------------------- -16. /sys/kernel/mm/transparent_hugepage -always [madvise] never ------------------------------------------------------------- -17. /sys/kernel/mm/transparent_hugepage/khugepaged -1 ------------------------------------------------------------- -18. OS release -PRETTY_NAME="Ubuntu 22.04.4 LTS" -NAME="Ubuntu" -VERSION_ID="22.04" -VERSION="22.04.4 LTS (Jammy Jellyfish)" -VERSION_CODENAME=jammy -ID=ubuntu -ID_LIKE=debian -HOME_URL="https://www.ubuntu.com/" -SUPPORT_URL="https://help.ubuntu.com/" -BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" -PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" -UBUNTU_CODENAME=jammy ------------------------------------------------------------- -19. Disk information -NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINTS -loop0 7:0 0 4K 1 loop -loop1 7:1 0 258.3M 1 loop -loop2 7:2 0 73.9M 1 loop -loop3 7:3 0 11.1M 1 loop -loop4 7:4 0 258.3M 1 loop -loop5 7:5 0 44.4M 1 loop -loop6 7:6 0 11.1M 1 loop -loop7 7:7 0 505.1M 1 loop -loop8 7:8 0 516M 1 loop -loop9 7:9 0 91.7M 1 loop -loop10 7:10 0 10.8M 1 loop -loop11 7:11 0 10.8M 1 loop -loop13 7:13 0 44.4M 1 loop -loop14 7:14 0 500K 1 loop -loop15 7:15 0 568K 1 loop -loop16 7:16 0 73.9M 1 loop -sda 8:0 1 0B 0 disk -sr0 11:0 1 1024M 0 rom -nvme1n1 259:0 0 3.6T 0 disk -└─nvme1n1p1 259:1 0 3.6T 0 part -nvme0n1 259:2 0 3.6T 0 disk -├─nvme0n1p1 259:3 0 3.2T 0 part /usr/lib/firmware/nvidia/550.120/gsp_tu10x.bin -│ /usr/lib/firmware/nvidia/550.120/gsp_ga10x.bin -│ /usr/lib/x86_64-linux-gnu/libnvcuvid.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-opticalflow.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-encode.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-nvvm.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-pkcs11-openssl3.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-pkcs11.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-allocator.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-ptxjitcompiler.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-gpucomp.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-opencl.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libcudadebugger.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libcuda.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-cfg.so.550.120 -│ /usr/lib/x86_64-linux-gnu/libnvidia-ml.so.550.120 -│ /usr/bin/nvidia-cuda-mps-server -│ /usr/bin/nvidia-cuda-mps-control -│ /usr/bin/nvidia-persistenced -│ /usr/bin/nvidia-debugdump -│ /usr/bin/nvidia-smi -│ /home/mlcuser/MLC/repos/local/cache/download-file_9907d37a -│ /mlc-mount/home/arjun/gh_action_submissions -│ /mlc-mount/home/arjun/gh_action_results -│ /etc/hosts -│ /etc/hostname -│ /etc/resolv.conf -├─nvme0n1p2 259:4 0 1G 0 part -└─nvme0n1p3 259:5 0 487.2G 0 part ------------------------------------------------------------- -20. /sys/devices/virtual/dmi/id -bios_date -bios_release -bios_vendor -bios_version -board_asset_tag -board_name -board_serial -board_vendor -board_version -chassis_asset_tag -chassis_serial -chassis_type -chassis_vendor -chassis_version -modalias -power -product_family -product_name -product_serial -product_sku -product_uuid -product_version -subsystem -sys_vendor -uevent ------------------------------------------------------------- -21. dmidecode -# dmidecode 3.3 -Scanning /dev/mem for entry point. -FAILED: dmidecode ------------------------------------------------------------- -22. BIOS -# dmidecode 3.3 -Scanning /dev/mem for entry point. -FAILED: dmidecode -t bios ------------------------------------------------------------- From a0922756649e0ff0758a02c4791a9f1efb05fd66 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 4 Jul 2025 15:54:22 +0100 Subject: [PATCH 718/719] Delete open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline directory --- .../offline/accuracy/accuracy.txt | 6 -- .../offline/accuracy/mlperf_log_accuracy.json | 7 -- .../offline/accuracy/mlperf_log_detail.txt | 71 --------------- .../offline/accuracy/mlperf_log_summary.txt | 4 - .../run_1/mlperf_log_accuracy.json | 2 - .../performance/run_1/mlperf_log_detail.txt | 90 ------------------- .../performance/run_1/mlperf_log_summary.txt | 54 ----------- 7 files changed, 234 deletions(-) delete mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/accuracy.txt delete mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_accuracy.json delete mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_detail.txt delete mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_summary.txt delete mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_accuracy.json delete mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_detail.txt delete mode 100644 open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_summary.txt diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/accuracy.txt b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/accuracy.txt deleted file mode 100644 index dda39d94a..000000000 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/accuracy.txt +++ /dev/null @@ -1,6 +0,0 @@ - -Results - -{'rouge1': 61.7021, 'rouge2': 37.9679, 'rougeL': 39.3617, 'rougeLsum': 56.9149, 'gen_len': 1302, 'gen_num': 1, 'gen_tok_len': 610, 'tokens_per_sample': 610.0} - -hash=f3de59b8a585ecf82203af0128622d3175107de8070cbd39a8cc839e297faeb0 diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_accuracy.json b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_accuracy.json deleted file mode 100644 index d8dc90165..000000000 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_accuracy.json +++ /dev/null @@ -1,7 +0,0 @@ -[ -{ "seq_id" : 0, "qsl_idx" : 0, "data" : "C20100000000000087080000000000002D290000000000005201000000000000CD740000000000007801000000000000D574000000000000E2170000000000009B0100000000000007010000000000008619000000000000C37400000000000004010000000000006A3F000000000000B57400000000000075010000000000008603000000000000E2040000000000005201000000000000EC4B00000000000007010000000000007C0E00000000000058010000000000009B0100000000000007010000000000005E0A0000000000007406000000000000F32B0000000000003C06000000000000BD040000000000000D000000000000000D00000000000000E2060000000000000445000000000000890100000000000078010000000000002806000000000000E217000000000000DB74000000000000B7740000000000007C0E0000000000005801000000000000F4020000000000005E0A000000000000694700000000000075010000000000007401000000000000A3060000000000004F0200000000000031030000000000003001000000000000CF6B000000000000160100000000000087080000000000002D29000000000000420100000000000073390000000000001601000000000000A8090000000000007401000000000000C01F000000000000C1740000000000000D000000000000000D00000000000000C874000000000000C174000000000000C2010000000000002D2900000000000019680000000000000701000000000000E2170000000000009B0100000000000007010000000000008619000000000000C37400000000000004010000000000006A3F000000000000B57400000000000075010000000000008603000000000000E204000000000000C1740000000000008E03000000000000A809000000000000B303000000000000C30100000000000039100000000000005F72000000000000300100000000000016010000000000007127000000000000360100000000000086030000000000007C0E0000000000005801000000000000C1740000000000000D00000000000000D274000000000000C174000000000000C2010000000000002D29000000000000196800000000000089010000000000001601000000000000E2170000000000005201000000000000EC4B00000000000007010000000000007C0E00000000000058010000000000009B0100000000000007010000000000005E0A0000000000007406000000000000F32B0000000000003C06000000000000C1740000000000008E03000000000000E528000000000000890100000000000016010000000000007C0E0000000000005801000000000000F40200000000000007010000000000005E0A0000000000007406000000000000F32B0000000000003A0B000000000000D6010000000000009E1200000000000075010000000000007401000000000000C4740000000000001D020000000000007401000000000000B303000000000000C30100000000000007250000000000003A08000000000000890100000000000016010000000000007C0E0000000000005801000000000000F4020000000000005E0A000000000000694700000000000075010000000000007401000000000000C1740000000000000D000000000000000D00000000000000ED74000000000000B70500000000000075010000000000001601000000000000A8090000000000005013000000000000C4740000000000004F02000000000000310A000000000000D4520000000000000E0E0000000000002307000000000000890100000000000078010000000000002806000000000000E217000000000000DB74000000000000B7740000000000007C0E0000000000005801000000000000F4020000000000005E0A000000000000694700000000000075010000000000007401000000000000BD04000000000000AE16000000000000740100000000000052010000000000008D15000000000000890100000000000016010000000000007C0E0000000000005801000000000000F4020000000000005E0A000000000000694700000000000087080000000000001601000000000000010E000000000000360100000000000007010000000000005E0A0000000000007406000000000000F32B0000000000003C06000000000000C4740000000000004F02000000000000310A0000000000007716000000000000CA0200000000000016010000000000001334000000000000890100000000000016010000000000003C0600000000000009040000000000006F01000000000000360100000000000007010000000000008E050000000000006E04000000000000C47400000000000024050000000000009801000000000000070100000000000070660000000000009E12000000000000D60100000000000007010000000000008E050000000000006E0400000000000036010000000000003A0B0000000000008901000000000000B303000000000000C3010000000000002F620000000000006947000000000000C1740000000000000D000000000000000D00000000000000F7200000000000003704000000000000C47400000000000016010000000000001406000000000000683F000000000000D2040000000000005201000000000000CD740000000000000D000000000000000D00000000000000280C0000000000005201000000000000C3010000000000009E070000000000003001000000000000590A000000000000C1740000000000000D000000000000000D000 - -... - -00000CF6B000000000000160100000000000087080000000000002D29000000000000420100000000000073390000000000001601000000000000A8090000000000007401000000000000C01F000000000000C1740000000000000D000000000000000D00000000000000C874000000000000C174000000000000C2010000000000002D2900000000000019680000000000000701000000000000E2170000000000009B0100000000000007010000000000008619000000000000C37400000000000004010000000000006A3F000000000000B57400000000000075010000000000008603000000000000E204000000000000C1740000000000008E03000000000000A809000000000000B303000000000000C30100000000000039100000000000005F72000000000000300100000000000016010000000000007127000000000000360100000000000086030000000000007C0E0000000000005801000000000000C1740000000000000D00000000000000D274000000000000C174000000000000C2010000000000002D29000000000000196800000000000089010000000000001601000000000000E2170000000000005201000000000000EC4B00000000000007010000000000007C0E00000000000058010000000000009B0100000000000007010000000000005E0A0000000000007406000000000000F32B0000000000003C06000000000000C1740000000000008E03000000000000E528000000000000890100000000000016010000000000007C0E0000000000005801000000000000F40200000000000007010000000000005E0A0000000000007406000000000000F32B0000000000003A0B000000000000D6010000000000009E1200000000000075010000000000007401000000000000C4740000000000001D020000000000007401000000000000B303000000000000C30100000000000007250000000000003A08000000000000890100000000000016010000000000007C0E0000000000005801000000000000F4020000000000005E0A000000000000694700000000000075010000000000007401000000000000C1740000000000000D000000000000000D00000000000000ED74000000000000B70500000000000075010000000000001601000000000000A8090000000000005013000000000000C4740000000000004F02000000000000310A000000000000D4520000000000000E0E0000000000002307000000000000890100000000000078010000000000002806000000000000E217000000000000DB74000000000000B7740000000000007C0E0000000000005801000000000000F4020000000000005E0A000000000000694700000000000075010000000000007401000000000000BD04000000000000AE16000000000000740100000000000052010000000000008D15000000000000890100000000000016010000000000007C0E0000000000005801000000000000F4020000000000005E0A000000000000694700000000000087080000000000001601000000000000010E000000000000360100000000000007010000000000005E0A0000000000007406000000000000F32B0000000000003C06000000000000C4740000000000004F02000000000000310A0000000000007716000000000000CA0200000000000016010000000000001334000000000000890100000000000016010000000000003C0600000000000009040000000000006F01000000000000360100000000000007010000000000008E050000000000006E04000000000000C47400000000000024050000000000009801000000000000070100000000000070660000000000009E12000000000000D60100000000000007010000000000008E050000000000006E0400000000000036010000000000003A0B0000000000008901000000000000B303000000000000C3010000000000002F620000000000006947000000000000C1740000000000000D000000000000000D00000000000000F7200000000000003704000000000000C47400000000000016010000000000001406000000000000683F000000000000D2040000000000005201000000000000CD740000000000000D000000000000000D00000000000000280C0000000000005201000000000000C3010000000000009E070000000000003001000000000000590A000000000000C1740000000000000D000000000000000D00000000000000280600000000000087080000000000002D29000000000000B303000000000000C3010000000000001B0F000000000000EB0C000000000000A8090000000000003001000000000000D4520000000000000E0E000000000000E11F0000000000006C0E0000000000001601000000000000E217000000000000DB74000000000000B7740000000000007C0E0000000000005801000000000000F4020000000000005E0A000000000000694700000000000075010000000000007401000000000000C1740000000000003605000000000000FC010000000000008327000000000000890100000000000016010000000000007C0E0000000000005801000000000000F40200000000000007010000000000005E0A0000000000007406000000000000F32B0000000000003C06000000000000C4740000000000001D020000000000004F02000000000000310A0000000000008314000000000000890100000000000016010000000000003C06000000000000973100000000000030160000000000006947000000000000C1740000000000000200000000000000", "token_count" : 305 } -] diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_detail.txt b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_detail.txt deleted file mode 100644 index 7a2fa5460..000000000 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_detail.txt +++ /dev/null @@ -1,71 +0,0 @@ -:::MLLOG {"key": "loadgen_version", "value": "5.0.15 @ aea55edb04", "time_ms": 0.006116, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2025-02-02T19:03:46.726516", "time_ms": 0.006116, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2025-02-02T19:03:46.726520", "time_ms": 0.006116, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2025-02-01T00:10:56+00:00", "time_ms": 0.006116, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "aea55edb04b10928b75d1cc318d6bed2f2e4a629 Create benchmark checklist for pointpainting (#2068)\nd67b62657a63c7d64bacf92d5ec3328c03ef0d90 Increment version to 5.0.15\nd7bbf881ca0b69c71cbbee18fa26cf13c7a67be8 Update results.cc | Add another significant digit to percentile latency display (#2066)\n10d123c982be145de8e818309440486ee1706439 Fix mlperf.conf link for equal issue mode (#2069)\n9b32a74c53fcfa6a7f6b529dec6bba534e10cc29 Changes for final report generation - PointPainting (#2063)", "time_ms": 0.006116, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.006116, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"a8ebd64f62d0349aeedbe3295d833ebdce625c2e","/MANIFEST.in":"ddeb472d62edf2920db1f8fa3beebe3e831557f1","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/VERSION.txt":"106b7e43a7c4da84939d61238eba5c8639ddf3c5","/__init__.py":"d013101621ef06a0ddc5e7d9ce511918a8b2ebe6","/bindings/c_api.cc":"14d178b64c7fc45d090e038c08d9b78ca943c383","/bindings/c_api.h":"23d9f99e00b2d196e095fae0bb453a391c18d601","/bindings/python_api.cc":"4dae966c92acdaa373b04a95adc4ca353937f154","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"02fcfe6d9cf958eeb4b6f1f4dbe87ba7eb4d7dec","/issue_query_controller.h":"ed20934fd3507a15949d501ac154be38e766f6ab","/loadgen.cc":"6daa9cd51454a699fcb55d9aa6bf9e54dd7b7a97","/loadgen.h":"ce9fcb5d44951e7e9048a83b7c1a41c8b8e0f7d8","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"49e63158ebca654fa4b7c5f3321054cf4d6c3a30","/logging.h":"5f6cc9ed4c4859641e05b1bb6a0ef860c852fcd8","/mlperf.conf":"80d879f8a0e3ff89863101be32a710fa476e64f8","/mlperf_conf.h":"fca712ecaa27e093c1f76dc4beb85741d88eaeb2","/pyproject.toml":"712fab87b72ba67ef2a068d0f9f47da65130342f","/query_dispatch_library.h":"1f18e9cd3ee4dc89a387cf462de1d0ceb1ece975","/query_sample.h":"c4f399103bc3d172079bbd4cd2b0ca0f22eebc4f","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"7c4ef762b985ee0504bbcd482f5985123f7c5a1e","/results.h":"fce22d5a588d91fd968a6b25c27896dba87bc276","/setup.py":"a5eaa6f713bd3dfb6603be2c7928f0c295d7ee30","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"8e05582d1fbe9dd2b809686684c3a0ac41248723","/test_settings_internal.cc":"a5cc85fb7735727eee032aa3e88b5d61c1f11a2a","/test_settings_internal.h":"2bb9e9ae53904cb0ca221f4a5d49ca7d9ec3b0ca","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"9f23d13276194588473120a8a6ecf5a6ed034a23"}, "time_ms": 0.006116, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "test_datetime", "value": "2025-04-01T05:14:05Z", "time_ms": 0.336565, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1194, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.336565, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1195, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 485, "time_ms": 0.336565, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1196, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.336565, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1197, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 1, "time_ms": 0.336565, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 1, "time_ms": 0.336565, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 272, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_test_mode", "value": "AccuracyOnly", "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 273, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_offline_expected_qps", "value": 0.01, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 310, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 0, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_min_query_count", "value": 1, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_max_query_count", "value": 1, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 6023615788873153749, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 320, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 15036839855038426416, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 321, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 9933818062894767841, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 323, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 324, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 326, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 328, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 330, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 331, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 333, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 335, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 24576, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 337, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 339, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "requested_use_token_latencies", "value": true, "time_ms": 0.675343, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 343, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_test_mode", "value": "AccuracyOnly", "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 419, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_target_qps", "value": 0.01, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 423, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 424, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 427, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 0, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 431, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_max_query_count", "value": 1, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_min_sample_count", "value": 1, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 6023615788873153749, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 434, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 15036839855038426416, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 435, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 9933818062894767841, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 438, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 440, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 442, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 445, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 447, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 449, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 24576, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 451, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.675462, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 453, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "generic_message", "value": "Starting accuracy mode", "time_ms": 0.678933, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1085, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "loaded_qsl_set", "value": [0], "time_ms": 0.683901, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 608, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 0.701097, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 427, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 0.701097, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "generated_query_duration", "value": 100000000000, "time_ms": 0.701097, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 796473.383806, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 796473.383806, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 796473.383806, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 796473.383806, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 796473.383806, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 910, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 796473.383806, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 912, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "power_begin", "value": "04-01-2025 05:14:05.071", "time_ms": 796473.387411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 562, "pid": 2383, "tid": 2383}} -:::MLLOG {"key": "power_end", "value": "04-01-2025 05:27:21.542", "time_ms": 796473.387411, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 2383, "tid": 2383}} diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_summary.txt b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_summary.txt deleted file mode 100644 index a05d86571..000000000 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy/mlperf_log_summary.txt +++ /dev/null @@ -1,4 +0,0 @@ - -No warnings encountered during test. - -No errors encountered during test. diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_accuracy.json b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_accuracy.json deleted file mode 100644 index 0d4f101c7..000000000 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_accuracy.json +++ /dev/null @@ -1,2 +0,0 @@ -[ -] diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_detail.txt b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_detail.txt deleted file mode 100644 index 212536b05..000000000 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_detail.txt +++ /dev/null @@ -1,90 +0,0 @@ -:::MLLOG {"key": "loadgen_version", "value": "5.0.15 @ aea55edb04", "time_ms": 0.011953, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 53, "pid": 784, "tid": 784}} -:::MLLOG {"key": "loadgen_build_date_local", "value": "2025-02-02T19:03:46.726516", "time_ms": 0.011953, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 55, "pid": 784, "tid": 784}} -:::MLLOG {"key": "loadgen_build_date_utc", "value": "2025-02-02T19:03:46.726520", "time_ms": 0.011953, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 56, "pid": 784, "tid": 784}} -:::MLLOG {"key": "loadgen_git_commit_date", "value": "2025-02-01T00:10:56+00:00", "time_ms": 0.011953, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 57, "pid": 784, "tid": 784}} -:::MLLOG {"key": "loadgen_git_log_message", "value": "aea55edb04b10928b75d1cc318d6bed2f2e4a629 Create benchmark checklist for pointpainting (#2068)\nd67b62657a63c7d64bacf92d5ec3328c03ef0d90 Increment version to 5.0.15\nd7bbf881ca0b69c71cbbee18fa26cf13c7a67be8 Update results.cc | Add another significant digit to percentile latency display (#2066)\n10d123c982be145de8e818309440486ee1706439 Fix mlperf.conf link for equal issue mode (#2069)\n9b32a74c53fcfa6a7f6b529dec6bba534e10cc29 Changes for final report generation - PointPainting (#2063)", "time_ms": 0.011953, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 58, "pid": 784, "tid": 784}} -:::MLLOG {"key": "loadgen_git_status_message", "value": "", "time_ms": 0.011953, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 60, "pid": 784, "tid": 784}} -:::MLLOG {"key": "loadgen_file_sha1", "value": {"/.clang-format":"012aad77e5206c89d50718c46c119d1f3cb056b2","/CMakeLists.txt":"a8ebd64f62d0349aeedbe3295d833ebdce625c2e","/MANIFEST.in":"ddeb472d62edf2920db1f8fa3beebe3e831557f1","/README.md":"20a55bb946c2c0bbb564ced2af1e48efd096b3a8","/README_BUILD.md":"5f6c6a784e9cd6995db47f9b9f70b1769909c9d8","/README_FAQ.md":"01f9ae9887f50bc030dc6107e740f40c43ca388f","/VERSION.txt":"106b7e43a7c4da84939d61238eba5c8639ddf3c5","/__init__.py":"d013101621ef06a0ddc5e7d9ce511918a8b2ebe6","/bindings/c_api.cc":"14d178b64c7fc45d090e038c08d9b78ca943c383","/bindings/c_api.h":"23d9f99e00b2d196e095fae0bb453a391c18d601","/bindings/python_api.cc":"4dae966c92acdaa373b04a95adc4ca353937f154","/diagram_network_submission.png":"53dba8ad4272190ceb6335c12fd25e53dc02a8cb","/diagram_submission.png":"84c2f79309b237cef652aef6a187ba8e875a3952","/early_stopping.cc":"0cd7b546a389deac73f7955cd39255ed76557d62","/early_stopping.h":"158fcae6a5f47e82150d6416fa1f7bcef37e77fe","/issue_query_controller.cc":"02fcfe6d9cf958eeb4b6f1f4dbe87ba7eb4d7dec","/issue_query_controller.h":"ed20934fd3507a15949d501ac154be38e766f6ab","/loadgen.cc":"6daa9cd51454a699fcb55d9aa6bf9e54dd7b7a97","/loadgen.h":"ce9fcb5d44951e7e9048a83b7c1a41c8b8e0f7d8","/loadgen_integration_diagram.svg":"47f748307536f80cfc606947b440dd732afc2637","/logging.cc":"49e63158ebca654fa4b7c5f3321054cf4d6c3a30","/logging.h":"5f6cc9ed4c4859641e05b1bb6a0ef860c852fcd8","/mlperf.conf":"80d879f8a0e3ff89863101be32a710fa476e64f8","/mlperf_conf.h":"fca712ecaa27e093c1f76dc4beb85741d88eaeb2","/pyproject.toml":"712fab87b72ba67ef2a068d0f9f47da65130342f","/query_dispatch_library.h":"1f18e9cd3ee4dc89a387cf462de1d0ceb1ece975","/query_sample.h":"c4f399103bc3d172079bbd4cd2b0ca0f22eebc4f","/query_sample_library.h":"8323a2225be1dff31f08ecc86b76eb3de06568bc","/requirements.txt":"a5ff7e77caa6e9e22ada90f0de0c865c987bf167","/results.cc":"7c4ef762b985ee0504bbcd482f5985123f7c5a1e","/results.h":"fce22d5a588d91fd968a6b25c27896dba87bc276","/setup.py":"a5eaa6f713bd3dfb6603be2c7928f0c295d7ee30","/system_under_test.h":"18d4809589dae33317d88d9beeb5491a6e1ccdec","/test_settings.h":"8e05582d1fbe9dd2b809686684c3a0ac41248723","/test_settings_internal.cc":"a5cc85fb7735727eee032aa3e88b5d61c1f11a2a","/test_settings_internal.h":"2bb9e9ae53904cb0ca221f4a5d49ca7d9ec3b0ca","/utils.cc":"3df8fdabf6eaea4697cf25d1dcb89cae88e36efd","/utils.h":"40775e32d619ea6356826ae5ea4174c7911f6894","/version.cc":"cbec2a5f98f9786c8c3d8b06b3d12df0b6550fa0","/version.h":"9d574baa64424e9c708fcfedd3dbb0b518a65fcc","/version_generator.py":"9f23d13276194588473120a8a6ecf5a6ed034a23"}, "time_ms": 0.011953, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "version.cc", "line_no": 67, "pid": 784, "tid": 784}} -:::MLLOG {"key": "test_datetime", "value": "2025-04-01T05:00:37Z", "time_ms": 0.360092, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1194, "pid": 784, "tid": 784}} -:::MLLOG {"key": "sut_name", "value": "PySUT", "time_ms": 0.360092, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1195, "pid": 784, "tid": 784}} -:::MLLOG {"key": "get_sut_name_duration_ns", "value": 825, "time_ms": 0.360092, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1196, "pid": 784, "tid": 784}} -:::MLLOG {"key": "qsl_name", "value": "PyQSL", "time_ms": 0.360092, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1197, "pid": 784, "tid": 784}} -:::MLLOG {"key": "qsl_reported_total_count", "value": 1, "time_ms": 0.360092, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1198, "pid": 784, "tid": 784}} -:::MLLOG {"key": "qsl_reported_performance_count", "value": 1, "time_ms": 0.360092, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 1199, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_scenario", "value": "Offline", "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 272, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_test_mode", "value": "PerformanceOnly", "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 273, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_offline_expected_qps", "value": 0.01, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 310, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_min_duration_ms", "value": 0, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 316, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_max_duration_ms", "value": 0, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 317, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_min_query_count", "value": 1, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 318, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_max_query_count", "value": 1, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 319, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_qsl_rng_seed", "value": 6023615788873153749, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 320, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_sample_index_rng_seed", "value": 15036839855038426416, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 321, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_schedule_rng_seed", "value": 9933818062894767841, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 323, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_accuracy_log_rng_seed", "value": 0, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 324, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_accuracy_log_probability", "value": 0, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 326, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_accuracy_log_sampling_target", "value": 0, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 328, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_print_timestamps", "value": false, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 330, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_performance_issue_unique", "value": false, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 331, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_performance_issue_same", "value": false, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 333, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_performance_issue_same_index", "value": 0, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 335, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_performance_sample_count_override", "value": 24576, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 337, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_sample_concatenate_permutation", "value": false, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 339, "pid": 784, "tid": 784}} -:::MLLOG {"key": "requested_use_token_latencies", "value": true, "time_ms": 0.678566, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 343, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_scenario", "value": "Offline", "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 418, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_test_mode", "value": "PerformanceOnly", "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 419, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_samples_per_query", "value": 1, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 421, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_target_qps", "value": 0.01, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 422, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_target_latency_ns", "value": 0, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 423, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_target_latency_percentile", "value": 0.99, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 424, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_max_async_queries", "value": 1, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 426, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_target_duration_ms", "value": 0, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 427, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_min_duration_ms", "value": 0, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 429, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_max_duration_ms", "value": 0, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 430, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_min_query_count", "value": 1, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 431, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_max_query_count", "value": 1, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 432, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_min_sample_count", "value": 1, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 433, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_qsl_rng_seed", "value": 6023615788873153749, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 434, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_sample_index_rng_seed", "value": 15036839855038426416, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 435, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_schedule_rng_seed", "value": 9933818062894767841, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 437, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_accuracy_log_rng_seed", "value": 0, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 438, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_accuracy_log_probability", "value": 0, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 440, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_accuracy_log_sampling_target", "value": 0, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 442, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_print_timestamps", "value": false, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 444, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_performance_issue_unique", "value": false, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 445, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_performance_issue_same", "value": false, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 447, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_performance_issue_same_index", "value": 0, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 449, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_performance_sample_count", "value": 24576, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 451, "pid": 784, "tid": 784}} -:::MLLOG {"key": "effective_sample_concatenate_permutation", "value": false, "time_ms": 0.679010, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "test_settings_internal.cc", "line_no": 453, "pid": 784, "tid": 784}} -:::MLLOG {"key": "generic_message", "value": "Starting performance mode", "time_ms": 0.687073, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 836, "pid": 784, "tid": 784}} -:::MLLOG {"key": "loaded_qsl_set", "value": [0], "time_ms": 1.114049, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 608, "pid": 784, "tid": 784}} -:::MLLOG {"key": "generated_query_count", "value": 1, "time_ms": 1.499232, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 427, "pid": 784, "tid": 784}} -:::MLLOG {"key": "generated_samples_per_query", "value": 1, "time_ms": 1.499232, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 428, "pid": 784, "tid": 784}} -:::MLLOG {"key": "generated_query_duration", "value": 100000000000, "time_ms": 1.499232, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 429, "pid": 784, "tid": 784}} -:::MLLOG {"key": "generic_message", "value": "Ending naturally: Minimum query count and test duration met.", "time_ms": 1.625438, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "issue_query_controller.cc", "line_no": 482, "pid": 784, "tid": 784}} -:::MLLOG {"key": "logger_swap_request_slots_retry_count", "value": 0, "time_ms": 739620.939581, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 902, "pid": 784, "tid": 784}} -:::MLLOG {"key": "logger_swap_request_slots_retry_retry_count", "value": 0, "time_ms": 739620.939581, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 904, "pid": 784, "tid": 784}} -:::MLLOG {"key": "logger_swap_request_slots_retry_reencounter_count", "value": 0, "time_ms": 739620.939581, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 906, "pid": 784, "tid": 784}} -:::MLLOG {"key": "logger_start_reading_entries_retry_count", "value": 0, "time_ms": 739620.939581, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 908, "pid": 784, "tid": 784}} -:::MLLOG {"key": "logger_tls_total_log_cas_fail_count", "value": 0, "time_ms": 739620.939581, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 910, "pid": 784, "tid": 784}} -:::MLLOG {"key": "logger_tls_total_swap_buffers_slot_retry_count", "value": 0, "time_ms": 739620.939581, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "logging.cc", "line_no": 912, "pid": 784, "tid": 784}} -:::MLLOG {"key": "power_begin", "value": "04-01-2025 05:00:37.882", "time_ms": 739620.942762, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 562, "pid": 784, "tid": 784}} -:::MLLOG {"key": "power_end", "value": "04-01-2025 05:12:57.495", "time_ms": 739620.942762, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "loadgen.cc", "line_no": 564, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_validity", "value": "VALID", "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 674, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_min_duration_met", "value": true, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 679, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_min_queries_met", "value": true, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 680, "pid": 784, "tid": 784}} -:::MLLOG {"key": "early_stopping_met", "value": true, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 681, "pid": 784, "tid": 784}} -:::MLLOG {"key": "early_stopping_ttft_result", "value": "", "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 706, "pid": 784, "tid": 784}} -:::MLLOG {"key": "early_stopping_tpot_result", "value": "", "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 708, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_query_count", "value": 1, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 712, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_samples_per_second", "value": 0.00135206, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 768, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_min_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 774, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_max_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 775, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_mean_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 776, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_50.00_percentile_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_90.00_percentile_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_95.00_percentile_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_97.00_percentile_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_99.00_percentile_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_99.90_percentile_latency_ns", "value": 739612850832, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 778, "pid": 784, "tid": 784}} -:::MLLOG {"key": "result_tokens_per_second", "value": 0.412378, "time_ms": 739620.962677, "namespace": "mlperf::logging", "event_type": "POINT_IN_TIME", "metadata": {"is_error": false, "is_warning": false, "file": "results.cc", "line_no": 822, "pid": 784, "tid": 784}} diff --git a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_summary.txt b/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_summary.txt deleted file mode 100644 index 974855c85..000000000 --- a/open/MLCommons/results/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance/run_1/mlperf_log_summary.txt +++ /dev/null @@ -1,54 +0,0 @@ -================================================ -MLPerf Results Summary -================================================ -SUT name : PySUT -Scenario : Offline -Mode : PerformanceOnly -Samples per second: 0.00135206 -Tokens per second: 0.412378 -Result is : VALID - Min duration satisfied : Yes - Min queries satisfied : Yes - Early stopping satisfied: Yes - -================================================ -Additional Stats -================================================ -Min latency (ns) : 739612850832 -Max latency (ns) : 739612850832 -Mean latency (ns) : 739612850832 -50.00 percentile latency (ns) : 739612850832 -90.00 percentile latency (ns) : 739612850832 -95.00 percentile latency (ns) : 739612850832 -97.00 percentile latency (ns) : 739612850832 -99.00 percentile latency (ns) : 739612850832 -99.90 percentile latency (ns) : 739612850832 - - -================================================ -Test Parameters Used -================================================ -samples_per_query : 1 -target_qps : 0.01 -ttft_latency (ns): 2000000000 -tpot_latency (ns): 200000000 -max_async_queries : 1 -min_duration (ms): 0 -max_duration (ms): 0 -min_query_count : 1 -max_query_count : 1 -qsl_rng_seed : 6023615788873153749 -sample_index_rng_seed : 15036839855038426416 -schedule_rng_seed : 9933818062894767841 -accuracy_log_rng_seed : 0 -accuracy_log_probability : 0 -accuracy_log_sampling_target : 0 -print_timestamps : 0 -performance_issue_unique : 0 -performance_issue_same : 0 -performance_issue_same_index : 0 -performance_sample_count : 24576 - -No warnings encountered during test. - -No errors encountered during test. From 0569e20bf223c86dc4ca64fb51ec4cbf263b5534 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Fri, 4 Jul 2025 15:54:39 +0100 Subject: [PATCH 719/719] Delete open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99 directory --- .../llama2-70b-99/offline/README.md | 46 - .../offline/accuracy_console.out | 28 - .../llama2-70b-99/offline/cpu_info.json | 27 - ...nce-cpu-pytorch_v2.6.0-default_config.json | 7 - .../llama2-70b-99/offline/mlc-deps.mmd | 70 - .../llama2-70b-99/offline/mlc-deps.png | Bin 4575585 -> 0 bytes .../offline/mlc-version-info.json | 694 ----- .../llama2-70b-99/offline/model-info.json | 7 - .../llama2-70b-99/offline/os_info.json | 30 - .../offline/performance_console.out | 78 - .../llama2-70b-99/offline/pip_freeze.json | 73 - .../llama2-70b-99/offline/system_info.txt | 2594 ----------------- .../llama2-70b-99/offline/user.conf | 5 - .../llama2-70b-99/system_info.txt | 2594 ----------------- 14 files changed, 6253 deletions(-) delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/README.md delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy_console.out delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/cpu_info.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/gh_action-reference-cpu-pytorch_v2.6.0-default_config.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/mlc-deps.mmd delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/mlc-deps.png delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/mlc-version-info.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/model-info.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/os_info.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/performance_console.out delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/pip_freeze.json delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/system_info.txt delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/user.conf delete mode 100644 open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/system_info.txt diff --git a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/README.md b/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/README.md deleted file mode 100644 index e41bc029d..000000000 --- a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/README.md +++ /dev/null @@ -1,46 +0,0 @@ -*Check [CM MLPerf docs](https://docs.mlcommons.org/inference) for more details.* - -## Host platform - -* OS version: Linux-6.8.0-52-generic-x86_64-with-glibc2.35 -* CPU version: x86_64 -* Python version: 3.10.12 (main, Jan 17 2025, 14:35:34) [GCC 11.4.0] -* MLC version: unknown - -## CM Run Command - -See [CM installation guide](https://docs.mlcommons.org/inference/install/). - -```bash -pip install -U mlcflow - -mlc rm cache -f - -mlc pull repo gateoverflow@mlperf-automations --checkout=1d794d8fb58eca31070dffc1c4bb60c89f0fd4aa - - -``` -*Note that if you want to use the [latest automation recipes](https://docs.mlcommons.org/inference) for MLPerf, - you should simply reload gateoverflow@mlperf-automations without checkout and clean MLC cache as follows:* - -```bash -mlc rm repo gateoverflow@mlperf-automations -mlc pull repo gateoverflow@mlperf-automations -mlc rm cache -f - -``` - -## Results - -Platform: gh_action-reference-cpu-pytorch_v2.6.0-default_config - -Model Precision: fp32 - -### Accuracy Results -`ROUGE1`: `61.7021`, Required accuracy for closed division `>= 43.98689` -`ROUGE2`: `37.9679`, Required accuracy for closed division `>= 21.81485` -`ROUGEL`: `39.3617`, Required accuracy for closed division `>= 28.33004` -`TOKENS_PER_SAMPLE`: `610.0`, Required accuracy for closed division `>= 265.005` and `<= 323.895` - -### Performance Results -`Samples per second`: `0.412378` diff --git a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy_console.out b/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy_console.out deleted file mode 100644 index 29ad5b4cd..000000000 --- a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/accuracy_console.out +++ /dev/null @@ -1,28 +0,0 @@ -INFO:datasets:PyTorch version 2.6.0+cpu available. -Loading dataset... -Finished loading dataset. - Loading checkpoint shards: 0%| | 0/15 [00:001` or unset `early_stopping`. - warnings.warn( - -No warnings encountered during test. - -No errors encountered during test. -INFO:Llama-70B-MAIN:Run Completed! -INFO:Llama-70B-MAIN:Destroying SUT... -INFO:Llama-70B-MAIN:Destroying QSL... -Loaded model -Loaded tokenizer -IssueQuery started with 1 samples -IssueQuery done -Saving outputs to run_outputs/q0.pkl -Samples run: 1 - BatchMaker time: 0.007839202880859375 - Inference time: 796.4393861293793 - Postprocess time: 0.02407360076904297 - ==== Total time: 796.4712989330292 diff --git a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/cpu_info.json b/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/cpu_info.json deleted file mode 100644 index 191bfa66d..000000000 --- a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/cpu_info.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "MLC_HOST_CPU_WRITE_PROTECT_SUPPORT": "yes", - "MLC_HOST_CPU_MICROCODE": "0x2b000603", - "MLC_HOST_CPU_FPU_SUPPORT": "yes", - "MLC_HOST_CPU_FPU_EXCEPTION_SUPPORT": "yes", - "MLC_HOST_CPU_BUGS": "spectre_v1 spectre_v2 spec_store_bypass swapgs eibrs_pbrsb bhi", - "MLC_HOST_CPU_TLB_SIZE": "Not Found", - "MLC_HOST_CPU_CFLUSH_SIZE": "64", - "MLC_HOST_CPU_ARCHITECTURE": "x86_64", - "MLC_HOST_CPU_TOTAL_CORES": "48", - "MLC_HOST_CPU_ON_LINE_CPUS_LIST": "0-47", - "MLC_HOST_CPU_VENDOR_ID": "GenuineIntel", - "MLC_HOST_CPU_MODEL_NAME": "Intel(R) Xeon(R) w7-2495X", - "MLC_HOST_CPU_FAMILY": "6", - "MLC_HOST_CPU_THREADS_PER_CORE": "2", - "MLC_HOST_CPU_PHYSICAL_CORES_PER_SOCKET": "24", - "MLC_HOST_CPU_SOCKETS": "1", - "MLC_HOST_CPU_MAX_MHZ": "4800.0000", - "MLC_HOST_CPU_L1D_CACHE_SIZE": "1.1 MiB (24 instances)", - "MLC_HOST_CPU_L1I_CACHE_SIZE": "768 KiB (24 instances)", - "MLC_HOST_CPU_L2_CACHE_SIZE": "48 MiB (24 instances)", - "MLC_HOST_CPU_L3_CACHE_SIZE": "45 MiB (1 instance)", - "MLC_HOST_CPU_NUMA_NODES": "1", - "MLC_HOST_CPU_TOTAL_LOGICAL_CORES": "48", - "MLC_HOST_MEMORY_CAPACITY": "192G", - "MLC_HOST_DISK_CAPACITY": "6.8T" -} \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/gh_action-reference-cpu-pytorch_v2.6.0-default_config.json b/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/gh_action-reference-cpu-pytorch_v2.6.0-default_config.json deleted file mode 100644 index dbbba4cf0..000000000 --- a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/gh_action-reference-cpu-pytorch_v2.6.0-default_config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "starting_weights_filename": "https://github.com/mlcommons/cm4mlops/blob/b18ff890ff559e21d2e27a3b54cd26467ac1fd9e/script/get-ml-model-llama2/_cm.json#L51", - "retraining": "no", - "input_data_types": "fp32", - "weight_data_types": "fp32", - "weight_transformations": "no" -} \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/mlc-deps.mmd b/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/mlc-deps.mmd deleted file mode 100644 index 44eb311ae..000000000 --- a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/mlc-deps.mmd +++ /dev/null @@ -1,70 +0,0 @@ -graph TD - app-mlperf-inference,d775cac873ee4231_(_reference,_llama2-70b-99,_pytorch,_cpu,_test,_r5.0-dev_default,_bfloat16,_offline_) --> detect,os - app-mlperf-inference,d775cac873ee4231_(_reference,_llama2-70b-99,_pytorch,_cpu,_test,_r5.0-dev_default,_bfloat16,_offline_) --> get,sys-utils-cm - app-mlperf-inference,d775cac873ee4231_(_reference,_llama2-70b-99,_pytorch,_cpu,_test,_r5.0-dev_default,_bfloat16,_offline_) --> get,python - app-mlperf-inference,d775cac873ee4231_(_reference,_llama2-70b-99,_pytorch,_cpu,_test,_r5.0-dev_default,_bfloat16,_offline_) --> get,mlcommons,inference,src - pull-git-repo,c23132ed65c4421d --> detect,os - app-mlperf-inference,d775cac873ee4231_(_reference,_llama2-70b-99,_pytorch,_cpu,_test,_r5.0-dev_default,_bfloat16,_offline_) --> pull,git,repo - get-mlperf-inference-utils,e341e5f86d8342e5 --> get,mlperf,inference,src - app-mlperf-inference,d775cac873ee4231_(_reference,_llama2-70b-99,_pytorch,_cpu,_test,_r5.0-dev_default,_bfloat16,_offline_) --> get,mlperf,inference,utils - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> detect,os - detect-cpu,586c8a43320142f7 --> detect,os - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> detect,cpu - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,sys-utils-cm - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,python - get-generic-python-lib,94b62a682bc44791_(_torch_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_torch - get-generic-python-lib,94b62a682bc44791_(_torchvision_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_torchvision - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,ml-model,llama2,raw,_pytorch - get-preprocessed-dataset-openorca,5614c39cb1564d72_(_validation,_mlcommons_) --> get,sys-utils-cm - get-preprocessed-dataset-openorca,5614c39cb1564d72_(_validation,_mlcommons_) --> get,python3 - get-generic-python-lib,94b62a682bc44791_(_package.pyarrow_) --> get,python3 - get-preprocessed-dataset-openorca,5614c39cb1564d72_(_validation,_mlcommons_) --> get,generic-python-lib,_package.pyarrow - get-generic-python-lib,94b62a682bc44791_(_package.fastparquet_) --> get,python3 - get-preprocessed-dataset-openorca,5614c39cb1564d72_(_validation,_mlcommons_) --> get,generic-python-lib,_package.fastparquet - get-generic-python-lib,94b62a682bc44791_(_package.transformers_) --> get,python3 - get-preprocessed-dataset-openorca,5614c39cb1564d72_(_validation,_mlcommons_) --> get,generic-python-lib,_package.transformers - get-preprocessed-dataset-openorca,5614c39cb1564d72_(_validation,_mlcommons_) --> download-and-extract,_rclone,_url.mlc-inference:mlcommons-inference-wg-public/open_orca - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,preprocessed,dataset,openorca,_validation,_mlcommons - generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,os - detect-cpu,586c8a43320142f7 --> detect,os - generate-mlperf-inference-user-conf,3af4475745964b93 --> detect,cpu - generate-mlperf-inference-user-conf,3af4475745964b93 --> get,python - generate-mlperf-inference-user-conf,3af4475745964b93 --> get,mlcommons,inference,src - get-mlperf-inference-sut-configs,c2fbf72009e2445b --> get,cache,dir,_name.mlperf-inference-sut-configs - generate-mlperf-inference-user-conf,3af4475745964b93 --> get,sut,configs - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> generate,user-conf,mlperf,inference - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,loadgen - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,mlcommons,inference,src - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,mlcommons,inference,src - get-generic-python-lib,94b62a682bc44791_(_package.psutil_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.psutil - get-generic-python-lib,94b62a682bc44791_(_package.transformers_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.transformers - get-generic-python-lib,94b62a682bc44791_(_package.datasets_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.datasets - get-generic-python-lib,94b62a682bc44791_(_package.sentencepiece_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.sentencepiece - get-generic-python-lib,94b62a682bc44791_(_package.protobuf_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.protobuf - get-generic-python-lib,94b62a682bc44791_(_package.accelerate_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.accelerate - get-generic-python-lib,94b62a682bc44791_(_package.absl-py_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.absl-py - get-generic-python-lib,94b62a682bc44791_(_package.evaluate_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.evaluate - get-generic-python-lib,94b62a682bc44791_(_package.nltk_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.nltk - get-generic-python-lib,94b62a682bc44791_(_package.numpy_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.numpy - get-generic-python-lib,94b62a682bc44791_(_package.rouge-score_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.rouge-score - get-generic-python-lib,94b62a682bc44791_(_package.more-itertools_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.more-itertools - get-generic-python-lib,94b62a682bc44791_(_package.compressed_tensors_) --> get,python3 - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> get,generic-python-lib,_package.compressed_tensors - detect-cpu,586c8a43320142f7 --> detect,os - benchmark-program,19f369ef47084895 --> detect,cpu - benchmark-program-mlperf,cfff0132a8aa4018 --> benchmark-program,program - app-mlperf-inference-mlcommons-python,ff149e9781fc4b65_(_pytorch,_offline,_cpu,_llama2-70b-99,_bfloat16_) --> benchmark-mlperf \ No newline at end of file diff --git a/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/mlc-deps.png b/open/MLCommons/measurements/gh_action-reference-cpu-pytorch_v2.6.0-default_config/llama2-70b-99/offline/mlc-deps.png deleted file mode 100644 index 6150b12ed8ee2b21e940e41b65ae6c6c248cdd29..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4575585 zcmeFZc{tR4+Xp`Rs+3^LY1DmxWIlCosWmeANn_FXDl zwi^33gKQbY`~7v@_w!sv_j6yz@&5I`hdGX}vCQ>d&h>MC&hsZoT}|oWf#U~YFxbH> z$_kn=7#9x=23Orj34TI3^WztIlXAJN=c46c<$^MEvV>hTb8)nDaIw2>&f#k5_F|?@WS1|DhC(>qr+xVC1E-S*76u1o!2k4LUX?<(;Q!?#(69NX|G|gUZ&LmD zPx&@U{r3;Sw7bB~{@d%-JKF#D5%Bxyd;j1=>2E3i`=>5g?uZHo+YuFHDLYmLgs|g) zkSXlMDj zmeH->OLXTBNKw{EI%)89X+-Qg09fBECyj|w43rdwZw8qNP?Y{E_H%s&~__zo}Z{|BQXzNoKG z|6mvewsW|x$^v2GpGr<7Vf!FYznTi~^Kyo;L5|}C1}1&w7vB?krTiHZ5MBg2PR$f$ z`X`zS!GWBd!MIt1Mve9v^CF#^8Es@A!CC7vYiv#}uBFf9OQBw@U=mMonGX-l;sTa@ zE~nBnzHOU5`8J9v^G+pPd|OD&@0zODFJ+G(xF}ZQ-!ny<)WK52ad7xF0WbGnM?pZX z8~0uH%u(w*NSilD>&M>Ude*gBMH=7&$@&VcCb7W*w5s*;)L-U5 z`Axbxnn}H}y^ObkA>IrPUE99C>SYz)QQB?GpX)@BK75*}Q%PotDfZ0&HdD`mNf|BvqxFPwrbamhkT3e`g%pWl zTmpg284Lnzzb&T1a9s;cb}n?YLvp{AmS(xm`aG;RgA;YqG<0IcYjK;6A~aH>H1UVX z*Nsvt{}NHak|0^y5j3hOu{NmoWI&sKe)V;n+uEF+wD&D>+IF?k@IBg((L(IoMUcS2 zdn$Jy`5j_Sc~d_Cnf8XMz38>6@3fLbogAzV0-_yjFof1r6HvXZ!GPh~bt+{jOKF?d z<}CiGwaSU?B9HkseS}(eu|=|I13bV;>a0PCbwpcjLH!tRf_PmQ8gA=?jJ0x za_&9RU(&naNLp84ep&B9p?D^r%L>wH8}64fQpecK9K^@o+yNJte>HSEV*8@3msJ`O zm&uh>_o>iING+!b_f;PuMUfjDt6fmY`BFyw7`u!tJDs5FcsRuj1xf9}w)L1$V7nZp=#)XBcd98Se4t|>&4VP~$+x+gbj-#NS7ge&d$kZ7Lr|733wGB*==O*8Itv}k) zOsirVdoDPlr-Xj~z)z|DczmS`J=Ss4YkhtGm*iPQG=kRg*jj?;>unZar}RW~p^l#H z(!RkZ?>oaBjPBOOg{ip(?ODkT473h!smpz#7pO$r1`fEshty`g{eV;anks`byUCVQ zKf#--H#k$ruN$A=KQR_w9qZXB7tS&ArbZ$5kgW$J zoJnO!laQKk>a+quY7{K!0piAbQ_#cJ?@xMpf;myFvHR8%G(K&cAnnxrqXheJ%B4yc zc7PqVbZdfyD7QIEO^NKI@$uQU^=z_Ca-J^YzwcakADu?C{mz*GaA#%ST8xK81=^iI=Nu6;d_h4$ z`hzedyU&QJH1c0k}$+hBZAZUMYIQ6PQ{^yC&@ZH5r6F3*I z^(X%G75<`_X4+OWhRmWd7?+XmnJ?h_%De_Nmy_V{D8 z65jv3{9>7xI|2k3AhXxJoVeur@nH)(_P}40`glpA;a(2YC_V2-50j6|QGf1HiVlw;s*) zGPZhl`_V`MP5mJQ4|Ak49HFX{_{U9oHZCO<0bRB*5HHN zGfoH)&+&`JfU^7Q*I*KyC3tz9F#cn%`wyW@- z^ToOpH=)l*n^_h@FJfu7z7>kM>b^H#HlM4YXMPZGG^i@7X_LRnxOy{}%?s%Lnt>+3V5d=+`tUAs2=F zHQL&PcfU(9lSc97@_O5F04}Jy0Fx~IdJ#brfDUjHj_HLvGjH9LBZf0|BGSA@y_vQo*DioGPmm6N@4c@%rfK?CMyj6`n`93q4zN zX}vsvP5}hVWZ@Hwh|ovqfR3{<%xvUanlk(;&oI#?po4$LCrIiNY91;|boJ4iL^O3^ z^red=k6dSa8FreswQffH3F2^Qf~(J=&0nppe}$Z&xvxLsRtM-}?9YSJ=-TVPC|M29LA8Lo z`}8c@YuQl7{{0#zxl#e+gXTkTHdZ5D+N_zYUk{z2BheD;4c~Ze(<=#p328;L3T;MD zg*Qjs0gja|=m^hhMdOvr0*+8?&9^+kxUDtVz0ivdPU3M=y07A1Fc`E+Xl=DwDo@(w z6r0G0!=iEOUf)02f!i{ZXx)KH!JXr#w%yhhWB_*r^v75K_L_UCV^HqqoehVuZwK>8 zc(ourF1xlsX&l#U;f&l|cfm`l-vP<6ZAoXnwl;gtiLxOpFwkce#Ja0=ElN--R7U~} zmd94N#Wjagk!$z6F(@|(%3Hk{W}npoubG~n1>h+dx~$|(LBP>B!}6dckXITvN4|N6 zpp8;-c#PwBIEN!je8VKe;Z7%|D}8er2aAZUY4CAMu^&hQ5SsLnRK z!J#%ZQ*lAq@E+qo3IAFfo56}U3J`q&UIx}-_+Yp-p z6uO%XwsC9!gE~RJ7%&gcS6O~*eD_S{lV1Jxk6KC)!GPz9dI9$AAgCU8X2#bUlEpoq zKcs5Y_u^6{w|j+*L!}+TvoDpxF{MzzsUrQH{|S&u1Mn6}VTPFElKDyJC7UUQ^eE`p z1!=VP)l^naO0QF($cwh_k85@mBEO`m6C>}b>#19rmC6Sn{#IvviuJsqPy=e9VO=$W z8ZIhpr+Pnc=Ze9g_QYHz)MLc{JQmTyz#&hZqcqY@9L1obh?Os#MAbdP(0G~&Ze1A` znFeGkOD5Ag;G54RN};*|5Ci~}bF-Pe2!fIa6cvD5*83SDfF%nFNSHrH?8`e{$cZav zHwXnaz! zpCC9fEm8A@zHPk;DB8kYZ@Os5_kHA!Xhqq(XauG3p3T(` z|M`w#YOoi;Uh}C#83&x1K_c1e-oEqNe}GMugWy(6ORiqS3$Fb$G*0zqmj*xW8X%1lqLvt$%p}iEa^BYi%5f};o7&wVy@>xqbD;o2fW}AhS{mIw!Jb6<= z>Zt1$a;Pl?Tn1}kU?A18H@M*l)=-c-Bm=Kyc#V6AR;+dD;F&7-t=AFh>V#zS9wR8Z zl7`HjKoL`C_~`7}Z5LDj)5Y+H{Zn;UV~WgEJDTBp?SVcfS62aq_6D}7a_B!G-EUb&coA<7#J( z3&U1G$QWr!H*|Z^YB8rSni|UDfUP(KZAy^!m#RW|oADc%BRUp(M|ao~zJ%kue%xCG(k333#LB5U^vzwjH4~vp^OS zRm=flDwHi|o-x=83Rt(PIo0(six8xZ6(5R$FSC=qCauk`2J$?HS2b*gvB8iyV-qvM zX0($i)6qbNUnJjdkHob-SAl7c%Y-b{*oXR4u|}aH2`x|68P{KOW?$ zO^caAzU~0WN%yTkYK=>jyC?a^R@N4xBUUb93s}a^&e4)11)d0icX{2mM&$;Zzh6#e zG^1K{K0e}1-RroFevNeBD9tj**Yrasw&UTuwHw+8II?HT)j%nix^%{zirHQREHaekVBU}BdQKXw@~Nb*76M14`?vJW6C zs$axzfT-B~J1W8_P^QI;2tutP&3W=23Ts2#jqfY1{>jf`F>1R z7tCL0GYJJnWWNT_Yl-OuH~S1W<5#d5MS6Olxy2u=m9KmIHCB1o;62oUT8wS3EGNg6 zzS6-_PgiocZdAF?$i`*(mxP2x0ipryH2O*P40?qk0M}6)2X=b0o`$27b2|yqy2+~2V87F(wurdq z%0cqTRcQnEYj4;Zxhfw6$mX^IDh_)WKyu{It>RSI+o))3ajIuf3}=~B7!nb>5ve7{ zdzag_0)QdPx=TXmXsQ8jCOm;^u{8DBzR{nuSZKk|B{gYm8FQaZ#Bvo{O4z7|AdTjE zDJdO~IKW)yb(_D!y)X_W%STYTKE}9S*E%U4?j)YJ{B&5Tto=l zt$Du#TC&-urMYb#O<&|!JRQz?d@9_udY{+NJEF3lIS?F2F(|$QrNGbTc#H+i0uM@7 z8yD}K(5GlB#f#YMz+IWPm8eWF_3>LT028+F>LWoK&etZ4h3`IMkuex1hbX596&jw0 zrh_9D(pC>X72tW1;(^jm3I7xaus%g{($HxTWkX+*dk@X_4lmPbhc3i?kc53**fWbV z;X4xgq}3W0iQ(2(kc?>>|4mZwQVj%^e#JeYmO6z`gi4;tm3b>cieKAgw2?JRRszBx zT4|`FD|GzXRL4{rpx(?2*LYtd3?dv%Y!y{D1j8di8wo11fiCv{3~TNK6f96h*COJv zKbl=`?5sL1xx(|XcQ9M67j~g@iIUe??{-bdvEr&;qXwbN&{w#J(Ta|sfINEX zhbhPEHV> z8RLx-PZ5DtStbo-H5yW8R+DE^*WB%5Y8}nC_gZLa?U|r)QrhKelPnr2C+93G=m7Xr zvjETyNR3tmooN(pB`(d+S3yh0?_CF>(?XB}`nOdZrCQsj7&1q^%8w*U zmD0DqtR7WA*H_MvC{;i|4uqr8;|BzjuZ{Y^$`h07ApywX@oyn3{9$PiV1xlt3jDjj3mBStL0>yO`RO7; z-D<6Ty~Xm?8abl>KDa%6@U0Hs;eTmj|A3`O-P@mGe%*=7*ytWjY3Sc2)OY9K0@4Br zpTaV2sJYdUvMzw!yJrly&2R9OMrO4_>@bSPG_gGIb0GM(Uu2aRu)H9cw8AOA=K$Uq zfRF@)B4}~EIeG87v3O5^fu`E3{ zWONs->UbUMX*RhEp+Ysk5H2uIt^gI9f-JE0td+B!^|?h)^0I%S>+gokt~~x@#UAtETRc-L0M~q92<{+n-P(peu^HPOG^+NO$LWf^ znSJ7_B|?TaHQ+l`Xi&%0K^$m-0h^v_1chHxC(B5<2HzT-iE7RJeT${DzJ%CcAJA;#($4?jJ16Mg*?5Xv%cyllnF4zTNpvNkL~1As9Q<-pPtV zDDna<`C?+_2M`JRIk~^|Yi_eVr#miG^ZlO{#XLc1dUdPFllP^;w?IWuJvH^Lt&Prp zs;K;+Oh3Pn3UZYSh>61rSUR9h$J6A|@%FoQf0ZU9peH*2 z%Q?P-B?U-E5mZqn?*YuNR4;S6jnI7F(oRF74K28WUoit5hiemAK=-|}-PsjEkJ;qr zZHTi5g5iXEAJN6fpFFkbA^tEdXWImxGa^;))_qe~=SgD0!t&tAp$cDaubcIa31}=r1cU)bZUZ_;~xAK*seA>MS3h@$Oa-M|0*vu zZv)nMZ4~$@?9mjk_4%6DaY1Vaa$Aeh(R1X(R-vD_caUrx z{?$Pe3r#F5-Tb6apD$??;_tvdiV)KZv2fNx+Mz)s>00fNUj47E6aJnHR^>;0kVt+M z&8wshTy{1fJs8;~#o`evmgQg=#1@dqK(6<)dn4F$P&hy&126&DaMk-)W>!eRVP+m; zXlBAmTRR)tx=%EZH(B^@30{x20PhTF+inArp2O0xVk!HygU?d^L}N(?CvKuY+rkT) z#hO^xmX-wLQUa>=F0E#=tGA(U04^-P<9He;Owk^l87Q|FM+Bb-jB50k>%1ES)6>WT zFaWdHbOLJWv-?cgs9vk=Z~BSY$X~bIbug&KKtIkAIp^W4+97NAs%V$fH>E??K1q%!V|GP zoa_ZrmrBR1arw1zlG0$NUZGnc|3`RP$LgSh*}xm&mvVk5!2LL?8li#YE}M%pFFW!1qLKY4ob!`vV<=D)_bh zu^hm1K|9xBt9nd1$!W;$44a(GG}Dkph$yn*qHNvQ=9nT}yzcKZWTG1o$@OgKV-bp{ z%8&){O~S8M;0B6#tUFG9y~W!ZV?h%J{Gjs#a9kjqN418JkXCY1(#y$zmk><5(S@v1 znfpRE5(e*Qc^02=vA?V%T-V_}7celxGi(Mgf=+{0(~$rx3)jqs7l)SdUgg94#D&F% z%D#m{qWsN)Md^cZ)lpmdV6Cgu%4@E{j-hsyL;PQ22EWhM|H(7sjbUFr6TOzFNZ`~*53|VR{z|=&CU3r3UxpJ&SOn! zR1sTqm_44Sat|~lwDEniTI1(0H-pW|mPfLb+~0;nMCXTOoKK=cz0-#Lu1MsND%e4u z);JsUnr}uas~*kpcg^|szj5B$$T5BQmTZpA)3LBOBCB$<@aevi<%-H2me*GFMy(Z zXJ*r=7E2xkv197pR#)RF-aag3S_Y92-sPZir;jY`ZHEVVFMq0X@tWu{^qQD3gxbsX zeu06|z@!g=`fMqC6*EBJ4k9&Okld&!&0p&nj~9azBjLLq4^Tz4)DdLdC4qFnyp26` zF?xt@?Cb?lO-2D9b1T?-98+z;_>8Y?2}eDsExk2(asrHegpPHXD+1J(8^aOybyff< z91!-SZT*V#Sv6Rx^}C(uPfyR1fL@Q+nnoYn6KJq$wPx~C=6Ha0nv+tmD2aA190`J? zsUQW!WZN$%N0O_|3=HzTyt!rO%XA1DcpbUJ4+!gI(YbwnDbKSR=V$UUs}u9TpfOa? z?f}fGEXrQAtqTES(UF)FY_UE z0yPk&pYh!P6-~FT?OmljzZr0@`EU9NS*J}eJkQM~D&7iIG5{zSGRi!X8GXiIQ$%kR z1A5x67&-A;@|xahHp#-7YJz@~!#tG>19^gK`jmRK}quOiX)!45BV)66M>M<$)yt z2%xSQDH~(93667&{5W0Y!CG7XZ*4!;=aws)3cz7%z%8XKB#%JsBAcApuriP5!Z-as zdb?LHe^rR^-zhZeNhz38`PX4?r*?!)w;D_5sh8_wCs4kKEGH}hpgnLJX>gjJh<7G4 zN?ZLYyU{ab%THh{C;v9R==$IVsjz>d4#euew+0Wfuh;?Z`k0VCRI3?4ggG7S%OH7C zK&*y-1(R|3eXvwlV04=k9ku-;c+j)?P@1_+GzMtSQar<%dASW8Wkwc$pcekNamrDE zX@FY|=jMn5h?Iz+J%Zc7WUTf9>JSVLh>RX$M?BUHZhx6ouUQVr%1XG*JvVM>KH`Y; zt)c<;noZI-4;M;fYnbC{Fc`bhO~)Sgdtd|1gzp+OU?h5jR(LksQ#gnBiw8DvyQa$ zEpe#HmP|7Mh7^FZq8-8H0O$lHBI?3{87E}uV+qfI;hc&Ti}^9w6QRjXE}1rR$&8?o zSu8?E--4qRg-{HuEKtb)pi2-R#znVxhdwmDC9Ia&K58N#QhCoch+%5{LuL4}6h z>}UmwZiG|0&a^0=-XPIA)amLR2c^AJ{RvQJfo37-uUE=kr3Vbl@h4yu*3t%8yps|v zJxa@Vwx8Zr0ZY*zpSei12i|kz+hHs)R7@Ff80Z1JQX64k zGz|5p|K0yIe}K;G3cm^l07PJ}qc|NQ=-b3nfX}16i@#0PLNod_zABi{GqqtlLij`> zf~_eKN_p0$zih;ESUToM+l&%36`74hW|7kz$w*L1Bubalo0NS5I?2x)xH?};9k%*Y zlm%I5uODsF}HluDeMu`v3?S7B2q*RthF*pl+uc zE6G(Vh8u6g3=R0g>Z9l>(HI(|>k)%)3jG`1SaJ{Zd0Z*y5CTs_=B z#9$y_YUoVN{VQ3vd7!->lsgpYDI{j0L^%O3CACn3j|R_J?5iE$ORH zi=1;^t_Pq3%H@DoQz#{TX96^L1!$~7KWeoHBOA-09RlJOAlOZL054$v&+PCP=?j@O14;XCy;p7uaRGSTYth63>Sm@fm(CI3aUeg zFHC~B`O|V`%s?7WHt&<#I!k{~xbG5X00pfi_U7!+k2ZTSrVeAGzPjYJ1ntajIlTh< z*c71!$k=98KBC441ewLplha$i1AX{|Al8Ww1KoDu5dxH`&-k>}s)veR_4BfiMQ02_=tX#crgTi4!g8`BflS$%+stgm3XF%w@b}Fpg0TPy{nKi7p%76TrMcb-MrD~`Tua%ut;!kt*3X1i2N+CE zqW`^N_U{d~YcVY+p$)CcHatVV;fKVMm6l5$j7czrt+SWNME*`|!CngLArh1&C?XHpp)4#**Y7R9X z0E;#$7=Xuyz`PHT?oGo%e63URaqOm1y0V&YX5&CEV7MwjWU64F1OIue;v9#xlsk}2 zE?s+a>ADqz@rd7_yFDji3sRuVuQ{*XEYgbMQAC|S-^mn9c!pv`Zp-@|KD5rJa( zo(8?kmTO1o_&|+>zKO+QfXo2M(S#7WM;buE@a(@#r;s|%LkF0SZVq|~ATM8li3CLj zXOv!-kqbAOEag?EKpa+vC`+oW8g(T8QZfq^g~D>Uys8(c?(jp8dz^a0iSvXKK7L{W zw7T~A=0D5&eZy#sb*m+K5_=ga68Pi-8OGoVAoq$$yX;N?bSs#(JY=TbHi}1)l7bZ6eT4CTjJQLwFON!nSk-t(sTb$UzGH>k*6j;GZsyG4^NYLr} zYuBl~t$a)axpP#el$3RbU4M!BYuc-l_O1)%{J6D!-RLY}$SlUeV}y1qte$&UuNag= z55!?C%9_E`itCir!iN5V9|mW_7OkgHzCW5f2nc}Z=IkvKnzcb*)sEPTt}hsq>yx_! z^v7iHxpF-Zueda7x1#Aa)uQ+!7ol2`Sm9Y7huD3A*Be#nMRKz&jX~_LRQT|j_3jYZ zs)Oksmhb>WQ#l)r6x0Izpt=!R3jamk6OI*wdVX=K9xhavNLXnnT)}J1Q06y4XQu9c zUhZa_U&$0WK0G~-RZJqjv-B+ zK~#}DOHof69Q?Lrc`&KpXr%TpOP20JvbdaVxWF>ay)t_X&m$b)X>& z`i+l9n)wap6Q;xGX6vAWIBJAP;=;{)l)TXMdG%m~7Chc9*?3U+xIY83yqBf1{WAfU zEZT(j0n9Fpkge(*mzfVPYPB^}2_GlFXx#uRM`Ae3;d}+vFdn>B6|iR|kA(OjF-&U*~6f26h+9b3D z5E(-E8Ag?{C!*>WS$f?*<1{?+Ri7W&1c>jq764j&q3`^FP`)a<`-4PU*8+Q$u`Otr zz+!olr;I?#(YKs10zuJJ>k&q^w->BW7O0rErSIV}Fd<^1=juhrJpgsky~-mIinqbT z+sMMWO(uy8(T2B|af(?nv*UD{!AnJWN~TmW#7D_{aLF1FS{*Exu|r|Nf6JR?Vt4@v z=BOgGJ0^0w3ylO6`0DyE4;m+`UGG>@0>QX`{qF#kx(FdKP}WFA=gUneTwK&DVy6>+ zvNox+NgD|QV!6g55sV_jX4Kj+s&RmDkRvm=yv=Q!g%s+@$$ICo8FjDt7T=?yPrR84 z9%oBLsAQx>;l7s0=fFylcgo?ssRDAI*|R@sI0DR8a?w-`T)LY8Cd?RvogivMu9fUs zKHF@M3QQzHy>P2`jZ5DhzTs2d>~C`&&vD);1P{8{xvaaxX6h8J4|P9xr!AG@p?+Mi z@A-iQjADVw6*~lGnJe+5FPfvCka!$VlZC)-Q&tx`3)i4$M;;~TRKoAN*e@w$<(}r` z{Hsbye=i#6;gUTIlej=@oBd-Mx{XB-2@Q4NJtJ)ewZxZ`D?Q1YdOhsqCZ73LLK*6i zBeN%CB^*&8jjgU@TEX9v!1MGb^x8o6|7~kgc-7#H)Jf{c@)iwS?lL&?yg4d}#FM8* zb{9|bVU|~!py+{~khK9bmAm+GAT67?XVLRs!_!xQo}w=N%-)_z{#N6!h-AWNMU?PD z;{M7xBJkUhC&@k_Coc^kQS$QyXzuS^Vcs@&PP-G8Q;Z`53P;2@_}505D|jFp zj$N<31Q_-g1E3=UQ+MB7jxY)IAthwd)6XmUEZVJ4CT?^oqwzW@;irkBw8rZar-y|$ zX2GwKL)BEnKEx`y<`?PTT-U}i=-Z+jxtmkYJ|yhsJeE{HKO!XO)wkKV`LkH=h=h>C zWrZ5KbzM1j?1r3=OQMz*<>uP4&Bff!@u5w*4Yl>p1ldTT7B;a*{n#}bZ2LN4D3BnF zWuT|OzWzCARd>@~EOGXNaW#IW4@~0ruwPwL1Eb})+E!iE_nL-oth z6yvdD#UR@AO$iNt3m*QdHSZ@6o(l|pA>WfHI$q&@zRjDcqM*>i{yYpz6iRG;5qKp= zGg#4KZnFS2^aII2#ZozxCABnYY0&zv|1PGlhv}G|1&iuwmrMKi>^pga{lvjbm-d}| z|G;?ngFC~6o^rzS4So>#Rh-Nz(6`hFRPR0lmQ94y;9M; z?xpp8($9XNcGtJ_&Sled)(-!w$I{Zr{mjO@#j|zzzunQc;hVmot0l2`hvL=&-rdy{ zk7F5w9<(S2p7Jh@nF;V&k2*7u&S&>1^9HqiG5rY*zd&Ug#i>;{<mtMeju|mSL3=8YcE5ZW&n})>*nS=DqK3 zzquqRheC?7xSz28>;))$T3s*ZE1m)z>^avfqi?zrDp zfuhxX?|Uo$`zc9iAp_fYRk(Vp@Hg*h?L1eQ)Mhgu)s8&B$jN9i@&2TMmmMeW3)fmU z`k-~+eAldPLU^3`2{yViZ2C}mN8*95m>-{tt<{~&_Uh1e-(cgpTsNX1K&%#fP)muq zW>KiGVf6L>k-jW-k94+cmTSy)-540hvp``N{j*k^USgW{wL}4}ToK&j%i}Kw zjl$mbE80l8uzjI?)F|>}>UFGpy>i6LLRzJHJ}$vi{MDFSh}QaMb}8GFE3c0kR=i7P z*(^bya-khM!g<(#eq@csd(4|}Jd#=#-S(NLf3-VK_De#qZU~$Jmv0TyDJC+WI2&pd z8@@MUghRMR+@9Enop#}7MRbLhl_`#lo;x9wOK;uwzS^#+*fVQsf8xCv0^#hmvFz^1 zK^mCeT*Dsa6damAK5&9KG4cGc%h#-3u*O~uz2z*{hEpUtC3fX*!^oi9U!FS7naap+ zt~0QCISN>{C@=F1`LLf#qF&M>c}D@H`RCWfr7J@B_Wt>I&`;6Ii&oUnV)$Q+I{%X4 zED@F}VB~L#{=mi0y6b6T-R?F=#yJczj^H z`Z6|@SK^k5vyy5P1 zhmH5g>cyWC@7ZP52IIU7c?NYu8oMojOr1`<#7W#Jvv@Xs&h(ebN^-K=tY~)0wEzp2 zrr%bgS~@#m(KC-e@%AWTw%WYND6xQOD>9Am&|96@6LEX#IM;?Ofj@T9DzTmlmi^h+ zJ2A1QqcHWuM*9~Lr?x}b1yT>|`jpt6@M0RhUs;>(`=knyl}#&`={Z+QhHi;jw93() zKv@QLyGRw%2gDp3{+e^Ae+_4s5*|YJz1+vazUzAvf6VYmqj>E4M6UN_JK=0-d7=5W)B|^=Hjnahb1tObu&#Tq z!kBZN&R_3j$=6P1$5EB))STV)9T8O&1fkuq&@kG{q>rzUjmA<^AA9wNfNe-omrnFq zNzI|`@Kt9y6G+V(>r{Tetc)pZ1Nn*9elAvzrc_|BnaIUYnFrHb9BL+I3iI^e3Ns7n zk=9S_dC(wYZ??9#^X|~5+H1Ua-`;4X2r&D(K~nsu-dFUvD%^cxJH-R$TGfY`j>7!A z9xUrK-nm|wH0DMvU_}_tylCrkfa*bm@GVtVO~pEnlTwE?b^kL~??;&rxL6MM^Zrslyyl><8{Muv1TV^MOyb_vA!$9+=r!l^pTYY{59Z?Q=SF zlv>qeoPgHgeXZOaq-1wT@;>KokR>|wzWGJ=WxbmGN&$OsPL7@Os4DH!KPjuddhgdE9v}MGo66 zIk?ih5;|Gz6a+lHDVmZdWZ#Fcof8EZ=Kxb&joV*QX+OX2v+v(*13XmQfjOmPNng?I zr#S!9t3>_tRij=1>9?Fo?7`j#2Y9rOd!IhZIpu#gm7aLKM1-GrRa4ip@^25mNW-($zl53NTh@zLT7qcxTuL12Tag@w3?pf#?t< zNi@c+zse=eMbv>ks6;D!b+H@q%5L&e=Yh9wR_u>LgD&6FoN({VD%W{i+Te*qu zTD+xN;>z>R`rv(SZe*|A=IeRM{PM#9AJLW9i3>^QXCH9Sx+;A8X4l>qy$`bxox$xs zr4-!$*6?E3!J_Nd2qOFE= zQm0N@jcMo$M2bn%DbBUmzfSlNHd-CHaa{)86tI`>W3Zg^uAjsMP^$8q(^~s~ldAUd zMNzWad^VoCbEqTYs~`6n%7+b)kB)izt+Df^ehj58-wd%oa*!dbGk7ZLs8|f@(w7W* z#|OJ1q|f-YZ*@3M@*QgAN}^9>L;g7JNDmMK08a|Kk(4^H zANd!RnVJ35qduV#ioi~y>z}{6Gxw>|j2Z*S65d68RQtSRp*3hA$xlrE-AehX~lKB(#; zdMSy4>8v1|h=283>N+7L$Le5ai2f@({G(dCi*AO(pSig?#H8jlb!YF#hFaM~dz|Rk z&sLbt&`AjmJfFJ&Qv35JsSeGKy_U8Tq54NgE}q?Uc@KxXn;7mh=UVFEwb29{$v)a^ z0ko81^a~u!09F9Qf0#b9!4lsVP-{nEiEon1|LK2;=hEoDu>DovMe_HB94vXYa8T+9 zfS`&>{>KXs?0;jeWH!^I7s~36UTI0_5|jyE=}yYZU>Qh|YVg0cFM|5gTFo9ipWH)M z$=)bANBK3onONS z5?KQ@tarm&d~8@%Kugs3~gW%jA|MO5H{iL{e+f&tvbBb@>e+a|+K0GNd zVQ^++9-n9W`0?!ea`6!nr1@?TK9sDi_9$=~Y&xwBupsC9msXT~&@sKl$tz}_3xzk0 ziP9j66#OI!{`7VnxF=5!hX)pkp9v1u&%0~DIM&L7dm4_7INWhzjdxuZRNMdI^_3m{ z!Y3@`KVYB0z;yPEu%mCYHI05pk^XOdjOdr}phIy`p1SZgJuIf=fC7)6$)ncIHYMHU zI!hDBl-*zK+v*rs!u|9(>!?Gpm#zKJ4D?ggnGgC8mp{wyX38f<4mTLxJ31wv&?na6 zUoA}dh-4}j9J$CuYGFxxA$vr-TQX> z@Gf9Mc_TrM$}Pd55<8{0(SkPH{P_z-ycZgr9M-dp7tuZ{_R+kkWG04ES=a;3cT`qt zms$QrGiQa=3waj{ps|p}1u4jtE z^aON9a?jMh+Zc@Q|YZk}!m7rW3o zRW|1g${reKmeUW0N_~}08G)SWGe6Ou8zUrd=<#4^N$On^6MrIt=K;G(1ocK(WWoMtvDkCi|rTNBRrqTl%8!n)!_dCPp&k^4{$2I0ciQKL^T zHfDve6CDR?tQ9yxRl)hB;jVFcU7)wgi>fD#wd&$1PiaMM6`ndl)oM^ZcjzQ508XUF zoF0JTC$b{1b7SZLrkooxx$8gHe^2v46LLJu`LXy%V;$>c%FFTmj(1S!G7rQ*Tm|wI zY&UCaU@}A!m_lP^KD6K4}kyV-m#x8 zVz%u@#C3&^H%@x(GJ@92UFLX-;Wcf=2)v*S-vUcV=huTEiRbljq=ZMvLohhHyV~yE zt1$&?Kz&W%zkalP&%vP(K5_B2G8fEt}jzZ`;A2ll!GNKIuhFT3GncEi@O9-j<_COGa-p-cnd%4#MnyCcnrM zC8iqsFuzxf8b6nmb%i3~Tj1k~3ji)GyU)sqFawEBUU0TN_H|Y&)`c5XiS&mvDqnSo z99v)CRfKxC_T?@>y7%A6Qw?J*6ix+z`uW>l?=bDaOV5QJg)B zs%dRjQto}ps*}jK%~V&BwtW4&H4W(%?JKZX#r} zEJj=Y8x!I;y-#y5M{{FS?PXGGK&BtC5NBiG`dWOyi zvsXX$<19Lh5#kX0P`UMC?QJv8zpNtnf~x+IdzMPs8_~&D9zEShTBVuDZtNK6-Ak7e z!?p4%Xog64J9_eesR`vnV`;MZ)rJRs8=$*jq?d%HiCKXnn1rnDb6 z+GC3^Yt@osf2S{sO#vBn8m&|K8CUha4^RL+nf3vN}F}riZDDD_FiKMus zl&If_EPoa}{#=6LmVj0G!QWo{Z>=7`htqR)L)8Lmm`t+?X+%-A7ZTo=AGKY-2g@qe z_>IEjgcJW{Sa}FmjoS9gWqCvhZ#po%Se9kDn61LrsLgC!f9hY>zgSL$STEsTJN`^4 zDa-YIl#>!|uWC?@q%k_4=A$d-Nb^Tm3$t8MxSHH&?QOC#IDeoKvQE;O@y(4xz&d+nBqM^WZ=@s5DQ}-#+=Nsh6 zc4lc=c&t92DUcTB^mt-+#M>a9Iep}g=kBlb*I5dI2x}t>Yg^kZ2Lhi?LilnM#s&{@ zj-h=r)As+)=Xpk^d7d_>P|4K&08_n74gqcvMlKP=G2-kO*h5x=oqWQHww5WF`{P*b z8@G64#C!@au9tk5Vd?G{*Ui;pJYG;q?bvALu{KCz+3qb|~N-8Yw0O9C`*Z)-|VVBjKup*`|OgzU#l0w>$q7$PE5w zfJ+~Rbx33hyYs6gAZg*2D#=G!6RmQzh-N5?&R-R)-zm~d8U91fJg+2sohD4cdy0Gw z`}ZtjoX!_crUm=CRLk6xVWVkt?;}qHofOB#8RrutG?1^WLRq3#;Ufn}k6`6@ljMKc z9??9grdk(O=q%GQ?7F00Ttb=Uu9fz^;Id-!q9BKeFk{@bG?BRSP1cn;w1iv8#VQ%@ z`ocZ)o+KZ`iO}wMzYcPQaF{5noxeIabyV*af64^REcawsJ)F9c`K~JpaQM{AE6!N% z4}7jIpPUopp~PE`rW8bD3;q0MwM$*iCOedzo?FIrFlt;o)5memxQ!|rhZNzoCN=8> zEIzoT#83!S%QqgY=aPBK_}A#*X(47w?#c4l-RWg<%a-kbE$V1+BgoNrU+6DS+iulG zoK(@wYa6s#67VSkXk_t>3Z3<7h8FZEpkK?k^09uF&yO$8tos!qy9=fA@ldh_qwF#P z_LsMZV`r|5dZqY#)2Db@yl$D>7}a={l`0ch<6~>$-fu`vblsuyI&rtGEb;yOtjrx1 z>Y@`-!T(GJy9u2a3l^laU3;s?Fc0`*Sup=537y>jcS*?4z2JV>cYodLm^ZC)+`v75 zDvnPPC?H-iB+}&G&v0F~^Ay(BnP5aTQc}EDRaOKT41+PvuJ?+OmKAJ&6dM)4PF(D6171

5g05=8gDZ2sWR zIIVRqxjoSF^e0)rL;HS}FWjNs{zdq3P=m!1?s{rrk? z$r|5414@pDra4WiaaYon|@Dokz zPlXDfO8wnu7JFRkIydPf`lUvTy)n4?tSMkCXs;k6Af~O1gey?ugBm*U%_4YPl>cbJ1n- zqN0Pt)&2Dxmnt#$0B-5NKm!)0kU`VC>UR*$ooWic7H5CZ+@SAr|8r3KVChNe{>YQD z&`6fi1?>u;5u>DUHsu$Gvon2@9_0=iw|Qdtc>MlX)1YR8+dR&OJlgbsPP#>_w~BKqybamyPabXs2!!9fcUI&8OJI(p(KfM43d%8n2jdP7}5eyE2=cX z7yn#D-Vdi+j@r#09!#U$kcAF33`=qykzu1mziiNW+o+|TS>=`92lJcu=U2MaiE}pT5s91*OZ?CoxS8BU!dS@Hs;q=OaU#8f{<8kM@PbPx^6D zgmCz?RDN&1_tF4CoS4^VYJ%Lhhbt~#u6{lo%-&Wvro6J$36;N^K6|Lv9Odjx7@;ng zCHNUGI+!?;1`0)qTg}OhKe)0smnR2tn7Pba`(*CQMSgyodYGqm6hvu8m))NdJtMlH ziu(Ry^qH(bx?-!kEQ$Scy)SO?Mp*^-W8s!QSSyoET9Dd!)5vBOvk}>QRY2up@)(zX z`Wo>3u~d;jaf{Vt5j4A?GvJ#W$y5-dYm0u_2lS@?z1?N@{`O*xk^d?7p&z#q{gt}% zyUPNu3^gfcO}&c6nm2-dO$uK`5!_2#?zvzVd`s?Sfeso~15?pq&Zms_mqYQsZ-oI5y};a%Dma=Zb{pk7&C z9dEZ$#CvU$t(=Kh(@Awl+7So7?zuEKY}87>>_5wyb~y%dPrbZNL^izp4GuzJV!Dm= zruoPhrm5o0S>1enk+G=z8{a9jHow{yRcAWpYe)uDdG7_JTqAK)Ept?6XNEQfzZ&wB zErVv8k}o|olg}F0>S*;dWuN3u9&D!Zf|S1-pNUmrn5yx(8jE!h%A8#DpICO@TlA09%Y$Ne;cJtSIreT!PV|>c+Qv(ST8q_ZJ1Ll{Bn5MQWU>&GJ-0HTY&U z@AILo@+=SwS30d{`>r~5>+Mjy|8u-<4D7l(060r8tsCm@U-9NENaN8xo;caZXz$y9 zmgT0oc-xY(qtY^zVI)Ip^0Ib91l&tn@JwubOPAFo>{?L&xY3YpFj&57{Oov4PDS-*NQ+m61K!I@x=5K%_<{c*jy*H% z{}19I*8Ug7VKQiQbzj)ikTL#^)s;2-rRLA%mkEN81!#QtLi&sqv+6KkmhRwZ{QP|3 ze619Fkb>D6=aw870jnoJG%735kxhl+^FUz90GfVvk7xU7X{A9}KDK~CG1GivC1!En=Ee^WZ2x{tE zW=lQy*2dh3KP3u#qH5TJf*~p>BxE<7m90svznYLu*lKN$RdTbQYqOG>B=w+C$JoT5Nmrll(928d=C&2TNa_=M}5V{$7_QKqt zhzCw+T23~H{J-YsrS1)nF-MW7SKhzfY(0&Y2PJ$}2h`VJHscZS@8thXfsya z+oEBdete;*%`0hQze|uL@%?OJ=R-&4yKQyeS>-@V(1}|Tg8XoWGPFm3H24cIEYDWK zw-jqGPak{nGh>cykTiy}rD^&j%tCw@8!Wzp+(51y!wg*2dubPP?g$Y!N4HSfxcd8B zSY2Q2IFo>QCk0NlfNOa6=2Z$h?vUE?H$xBDBOLRCG7W|o&2}}7;~C+_A;`hVlfIb2 zCUpI*ZU%FRcG9IId<55&Uf5taOCzk*8;@0wg3dN(zE02hNsNdpTj6<)src-RJrV+^ z0HPE=l;LMueJ%Ws?_>nke-%|2H4V@Xzen;~hm8Q?>am=raXHBE+^U_~Sl?;lOKVZ{ zxf+Zp_#5yM_n25BEBozL7umtBf~K9jLK6nNtuWl8X9`MA09UpEu?)Cc%3@NSNFj?u zfsIOX&?sM!%~A8h53OMrhB7WDh32NT>>pGuWm@w6CRJJC8B926S$;BqV{UNHR1AM(F{D`;ZPkvZ`(JUoB?%j~yYvbUEhU*8|LxUdX_=v>D% zN?tjmFW(zcitT-5XG5p^-R@A~D!x-o_iM9(Z*;C63|sr-E$UiB^N(K}qH0gi2cBZl zXaAYAYRRA8NTwng4Q9F)y@7u#+|Otb+nl#%-Dvf&6kG*;KEgG?!29fL33NDV)4D@H z6Zj(LfCKlc{K7u>=JMuHFl3*8Gz!vy@$PIomUq?a&{#rQ*;?8ox`X)f6vn;qwG zQ?R{KwLgoXE8e$Fv+17=n?}6^3GWU^Gwc$Ao+>t~}w*tT(&1oLMkz+wRxuh9JU?3-(*Ns~sLyA&KXmxB}Aw>B4*3wPVHnY#=B zs=rb%y~Ev%8k$68tGomN7Is`>j40oX2QEHj+v(D(&cEX_d;w0fw}765@*m)WNZ9};O?7Y8n6pJ~HCMOYZtzar4Y$+O_Qh!Vbf?onG>)Qzu8 zS}VQ4zyikV9sGNWRs+gHvwJ*6<2|9yev37qSzR!09-bJDOijywxdg0jkpGQw23Yoi zQV4c>_LxwJgVXzWx*MG%-Y~X1`j^0>YsO;p*oy%`pY(6X)4&uJ_wN1$nDhKv#G8XG z`=8;_4qQL{{ptO?KljcJJ=RV_(?OjRly&*i(zVx!wGcY%?REEz+1jkQo)DPt7!u9bTeM8F*6vqssZlL z;~;$yd21v;_tMBw>KQPy9Wr*~Jc64ux%Jib1<9xXhJ7$nycWbVnBNz(P$WXZ)T1{w zU~rP0X*kmZl;l$f{0A2Y>TJ&K+FX_KcfH}iRmXNMmEq=5gWQ1m@hpj*_Z%1EGfx=> z5Ya)HtrUAfJtw>LlN0-cAGg&Hmf{G$oCHkowunE#{|b}l9D+-)G8kPFXq+IxLK+vw zN;8UNSG-%%b-oiUU%Yz3d#mf{>sfEF{ zwoo) z94#-4JS`}Fd`{H%I~4^f*Ns5o819K^rWaSNPkFcfu50Fm*TSDi@t$^t>`esny?^VB zPTYyQ#WQ9<<{Z{%GLX1AS#&;$h7$d(pFt_jp?|bJcC=lUi54Z2E2aK+6)T1RNmW{P z{RaWU&QFpBG=>lzp67{+rd%ENa?5GntWqGS<$2kBA1{VQxqQyZWfL_%oR;ykBY%6` zR&(U=b{aV|j7S_|)ZzmF-^i-zq!8Hp$ly_cYn0lKtKqBpQ<+Hf5*@+y1v~w{)%yt|}_t z@0s{f*X>_=adHw6nOxx6BNt*z{9oPvvEkjaMkIwgK^qQNnSJce%N+a>L;Oi}d|UYJ zzQjhk*){BLmbH9JUmpg65IwdzuT^cWw~_@tVHMiN2cI$quMh+5yP@R zs~+45_?Y= z?}z$FEy|L}^>jyS(C1vT%6VjRa-*UgyL{O{rC>e4B|wm7$6s*Ddbl_-WbWs<;%q~R zh(gXS!mSccy=HkdBq7JWEW;8OIDIgdytweda4ssS4sf-yG1V3$)2k2 zJs!XZH6tM>zf3Ljs`EuKJ9k*<&lfhz&?dSKTlpThY9&dlb2c(Pb*91X3@#5oC_jzP z>!^p_4i5q5Q~ANnyH>@kYW)?A|X8UM%j z_`l07ezyK>@#+6#zh%wK$r!&LK4{Z)r2V|`l-LI;wf2)5XH{#(mp;88n{g`IWIsKq zXR1ZL;b)0wh_o*0-&wy8v*)hfjpSL~?;is%wZb*cX_WXXRjs)DCUb!A8H?XXMCUIK zNX~n$=ANlOT#$;}j(H#rPhS>!iU2FKq{q~|!4i7cZ2iU6_o1w+Ch+=Bj8f`jkgl>0 z`vMa0BoW}7Q7{H~WnUd}Wl{4CKU@{jYvJ5x`+RvG9=*X4+q@0#K!9w2vc_ z?_H*NWfGTR41SfF6v>rIL&~BrJ%(99hGtbrVCb7_9F4B;;yW)>l%tZ*dCtWS_GE{v zm#uD2-oykQgEfH8L$fk6XBpZ4k^@}y@}4llkURUk4RsWF`AhBZ4KicW`h9GY#w_LDkaS=LNatuOWi7#0wLPJ@cKoe1U&7IQ zhpUoa7!erS@l&eyGzm;JFE0g1UXcTG8Od4wHdVO#X#ol1nLmYR~Yx z%p&hQzHW>}iAA*F1%CPvAv3UhygTfXP;`EM=ZaSpb*Riy0#;pfge0el2-elaQyJo` zizLot0Zo2pj|$RAZlsLj>^eD@bWXY(TH}-c-r5(V482Py=tB-S(V1-V3(V7Mk}H}A zD`Y-F9`9bdnZvF~bvc?+dZ=u}m0{?3Cu)=V>ibaV!F*1WcG3o^(#TKIxg$Gm&Z+iY zRAEhT;c08G%kL(1Zm=Y+@5Xvnj8_OFTB#qaoQ^uFRQS^YNidOFv28Lg`YQjl{QP&V zak}b_`KxromV{ulirjm8IB`2meDZ%ls{Vp4T>G!`Q^sEXoFG^`IsS^w3S(lSPV+R% z{b#1+o%G)Vvz~ZHqT3?Z6Hr(dk&_p|5v7<65XhW``NT9^*`R9}hCg7(kz*|xfU58h zk*JpOkU>N}mH&}~5D54UHbd;=0zyaP z+S#k=)!kcIELuSaIfdmlUG5{PMo-1uyGQ8$0NqFxT@Vn!lvm?^NmsYH$L3-APFHwq zHxeJuzdB@h@W<2ZI*4Dh5uOlU&w|Uom?%u zZ;utsi}hE&llQ89Y`3a=%`N<6h^0m7v+zkVxyU1hBYL$G%_ZzDDKg(D@ zrGYK`fJbj|EZmG?J&ef6eVhritT z$#!0MnE2wVzccVe?a3>YrFgv5O;LjW#DMRjpDFlG0=E9WRo%7(h0^cFC-nx-DrE$ipO`LWZl~))#%h2n72>v z$|9N5LKF77K!gsTcoF=x%n)m0F?_VPWr%h^>Ax8z!&*1zpRNBAIA3OTRnbg|RF;p^ zZw(FmVWSzJLbq&R#OagQm*Sr{2;fGUY3Qo61IR0_XRfI_GZ=Nd=soHBK$4b66MgB> zR@sla7Gx)|D+X=;1@7o|AN2d!ezqyuW8YzB zj08_wkRMDMY33a85P0RR&$Z*=>HJ|wuoJK|-^Pj&fxmhX$7`u^X02jAqXvsd{D{_+ zgqIL4@Bwdel=8h|0Ba2Y~v#@9(-ZV%(ntDCD_Z?7D=P^hz_Qr6d4}c<* z@x4By-IsDxmw2W4==vSpPkr}etG+ju(wAMm5%oV?&~^bL%YZVJ4pl2Tosr}*%`300 zT&8?{o`q3qxMES2hi0ZY#wWYStrcj>wT9dTnZzXw)uL|@x zK;P(I2AU>l9oTF@pjU&KP<|()?mpG>FF$W0aONkhTf+mKdQ95HRL@9=*q!76gCH3{ zufYqy)1q#{qYhX#ta1vCFh7>tAZ}H&a5y=(D!8-%o-n+$o7TI`5jI8OD$LqYYc7rO zU2Y}WhX7VS4p}qP>%KXqZif^x*}5%Y^-$*!jjy_6m~&3RH4zLl%|PAhV0UGzpRapx zN$aB?cTcU)F9dYf@2MFP^j3)DR@tdqAABJ8>z@hWBfzRE&~LIaH0bN#|267QY(VEr zDfNZDv0a=m&50<|c}dN$Q1=IU=v~}4p1kl)8q_F@Nns-G2}#8_wmDj}KoU>8iiv ztgxI9F6Gy_=X6>{44Hz;7Kt}^*q*7VHu|A>HlI#^ueV^t?vo&{T+@AVH_|KB2R%5Y zrbf7KxioO?o!;MysS{q048LPa%&3pd=)v)zOqp6$MdlD z-R_~oFpKxp1qaU~A)Jxl{?nGWIU!0F0VmLGSm`cvN0Qqx*TmL}K3Icyy)QzNsC+T^ z@C`FHcO%pBO7ZYfo;UW23c>inHOeq2%ncWTlh~Q4d(zz|=fmIC+4CE`c+*IpZ(e04 z6u&j7Q3a&-P5FsNa|LJ24QtN5>)b(%vdS_y1$Ttn^9ZsP%?3FSc zJ1_EiH>!r81(Z+N{RY>4dBNN4acm2p5Y^yUyR#2wRg*<<5_V`I zsSSUZluf40ifSs*%a0|7GQKN=wO5E^bQ(!CgwZx?abVh?}mpE_SeJ@c-)jI#Sa zi<=g0$0PriF<9ZpKhTc*Z%7i-O7W`2tB=0l(#M%M)%3Ib0;$%IAP4E++hbojp!qrV zThaNIr26Ps(!Cqgjkdm7lC`-+VEyggx?C2%o3plgVEjx~m9ic(I5l6HJtKy>Y`ul* zO(*%ku5;NczBO7#e74f4C`c|616)9a+@9z8OK$J>|OVq z2%MOaZ*`|wOQRo0ma-j+2FbdFr`vIJh786Yn`t#^EZNhtymxsm z)-;%Etu&df6;}*8*t;YMuIy-4l`c(=*Ds|v50dX2^`%cZ7!17|kEGM62BE;umx-Bp=D)cT8nN0tXRGMi< z8ZPYVCg*ovzh6s*C+72c$o}{PXQmv@mEfthN-^}eM}_ec!oAnKtcw5itEp9fJ((rO z&=oqFa4Wtm2mT3mww|Dn84>B>-E6B`EcSaG{?jqaL(=){u-nJhMm{qSa46z5^cE3`?z=!I z9_$jnONo06^X_WreR!~EqLs>K5)w`uWNH0g1@Ijaj8981JW;jN74zSk(FAMIAG8x?jQ^)Bst=W#=U|yvi_r7J0oC|(>Gib$(bd1 zFVAe%89pD`tu4(^qCb`}gd5pPput!G6NC2M1Ef%SbG_c(SMd(=<8tJ?`(Sv#^4s!-kSXI~Dtslxz)3EWDee zh8(IYzx>dn(_vHo7@gGB`N-b%WTYfhyZvs|hc`>J2ZAU6W`j36I?bOYr}%GQCxO`{ z7Irx<$Po#O-lo!ARs0xdW7=$HD~_;BxnnH*;yq;AaOmp)yd}T<;b(fy*HY!w_&+yc zmPZ#PL&iF-_l5tkZB)aTW#JaR2;byQV!FxN~C9hYGrF zdlr!-{l|*}4ZEP!Mxy6$_0$%-!B1OzWlaKfb|<+yJo1W)K^mV{^F54IA3Wd!*CkI- znM3aL#4fva-09!3v&;- z{q~Mr*JF(Q7I?u<+Bsw2K#siGmL9faoRw|1*kNoDObAZZmtM6r%2PmMECfbLoMmGw z*0R;J+rIyij(!yC&SKJt;kmF;f6BP&n`6o=VYZzy*)sWARoFiR8}wz?S^9D8c4b=E zA2|=~$fF2yDT4L5gEkMEE=RRrtNp$fU4~VLoo}fk8xdH1;X+jVdcfBG1$uo29e99CB zUgWz7?^IO))dRBeBRpX2j}!0X!T@jXe36M+%W+rvsm=YqAXfr^xbb{kD ze6fr_LfZZu$;^p`fYwkrFMAHW!tZc2qZ+FAQox9Bk;duLNaIC69|9$JZjp+p`W zzW;19NaWn_csb%D#Z*gjEoIo<*7$g?AfJWNn$)d*-I#tcHw(3Jf%8Of3#Nx5d$rX$ zHICCOO%Hf{7)s-R;JA*o@LdP~2EL+hPMl)QIiw!~_W;OALjeilG_DZ8%E=zL{(%>m zEoyPqt2_XOj|CILuXZF0InMI|xJjaMwswy_I=_26L6PRl-7Zs~DhKZMJ3h(n>;}ED z4@~qZ*cv{rhU{hYP2n7qzZZ{A)A~OMz@{3>h%M61g}mK~z_NY!7%u3{&0aj+j%J6z zUsi$ajw1rBjBSS%*aebz64Ap_drkISZZ|`oczNx|QLE3g)g&*-%t z(zjN=`y@%XeI=XRJt@E3#neZbMAzpuy69TJ@=EPi*~w|tUW<6C&2Is(cy0V&aYiW= zrKoC`Yo?~hwmTlv(ho+ux*g?xs)m$TcBm-Qo z$ep`n7hm03IY~7YjV&e~-&)4TkwUKGMZLSWi%07AK4;4+KEl~gEb@S4+M0nekD{k+ z_1gPT93xx*KsGOW&TWt5m&tp_mUGoi!bv5=(rm1Bcy&T$%M(La+d(iGB{C1Q-&z`r z-oHDMfk&Ao1%(w-v2ikjb&G<$h*H#dih3%J#`H`@a8)SG(~0o3n_;hOPzJ9zgfL8G zE_bJKvEJ-k^;aXFg11#&#Cl>Q(RWe+8IfG}{j8G|Ek%^D^k{xH0u-fp1(W(y7hVl5 z-Bg2K49WzQEQ*|Yv0AR8_aWv-tD`GX*1I>(_vh}TN(Xf3eR{XVWHG8Oi|uj17Me%` zT7qg1|3xONr~STV!9i(x>%$s}55XrqGT_9TR8`q%R)JHW^GnOJVD=vV2;(;mB`ma_ zVMcS3MbeCA-r;tqzoExn8MIztXs?kY1Vsy9*6!gCReRE*rm85qf1lDPhwm>IL7*EP z+-GqqsezDH@MU>F>t_5*y`MEn%FMKS^PwZ$k*J9xygkb?#7Ng+=Ay;S&vB*wiXSQ4 zAg{9O7gtaQr0gd+O-Y~D{9KbvHFpzsYp{EsEKKHR6>UKlz4HF50p`W=F$Y9Vx;7d^ zwc?p--NY8Qp70?$%iV&M^}|AHAuwTlcoWkzxHP85;I@Ca3Fd{d^Ec(+ZCYuv?NCKg zNZco_{Vgt!1HJQVE9|EJEahuOwp+~=yon?hU~Ie|)S{n2kgYO%ix0vFSzdU!kX4qC zqu^+_5_r#Wo=3IrgH&Y<3+yV{Ne><8F~Sj^MGL_ST+&bYgQ7!}%x3aA^wkp(`YBtB zw7#!ilA70JA>1?}Ge;k?dPdG__-TK%pxtX*mPDAl9!=~PFz`0(Yx1@TJp za5!}@C10$I7jIlThuhBm0YBMpfbV`Lc=xKne3&ShaN9q}Bc2xM5DcM}QAEm z4*WUc#tX{qH^desT=$P5gEo!cdJpEDOJKX1^_WtAMCpef4cAV6ML9Vy3MJIkFpBprU2QqMuUd-M-dNbV zbBA_3at3-C{@d3V?0{^*?U(-*)-Eaixl_e#-cGDX_vy_yWX$z&vEICvC99^4yml|^ zHRkfWrJ=sP*(CMu5DGx~nidkiB~R!#?XPi@T$8LUrG~^siZ)ZpgIB0K_;+%i%eWim zJu?1=DxR##n+{Ws#TfWIV)Z$n2L7$?K;@2`Jlc!!7KU>N|GHAq#k6O`$_JVs=tr!K zzh4T;6+dl7%K(bIGUaF~x?r8S`sZ@REun z0CEe8)WneGHh7u{m8T>kU2V{FNoI+z!<-1YXU_talXUIx*yj?xIv;|EfJ}=j#U8p5kbR#S#lGx`zhX5=@MJ;M1bC%ylJ={w@L!rkaAaDW5%aV~kOl$tKjF9~ z*}$H4=iV(@a9O{G_2$vKfJmiJtr{xZ`4fMHQId)Bc+SJpC+;Ra-*gaiU=RJ_jU|7Q z+ciAEBB)RgEjTKS24)OlaCoSH@V9(qc)eG_gRsiiNQdAkd4vj4U2^{K;3rdMwN+58 z0M@K;GE*;Mmj~8WAZgJjscCjh6<_{Y`Ih6&!mSeh$U_=g!3*XUdsf*`&LAFK989nH z(w|XN9dMet^PLt!SNZT@C0iPfWjzEwQ$56?`^oosQ_EFkWKl6XzoQ-$+Uw)ZC6c?^jSQ3Xl~T{xFZEF*O4jo8Tc*D#fZ1Y|6O2j zBLlY?XQ_4?Sll0GB&(OWD@@x5wf#9cfgxGm$<8$Gg?A;7Y-jYJ{o+B=bnuaoQEY1? zAxX5tq(hZM`+@^<$MAdQU!E?xx=JBL;CIciK1l9;ie3Al3d06ceDdX%GIPoSWv8h0 zFu#a!T`I~+Un4FaJ7=Ou#>?v-2|@{H-@hlim1@hGQah^8724ntHZ@Hg`i3ut<>+4* ztu`dm^zMIkjqYB}5{fU(Rh^S$rRZfZ$W?tcAaGgU+)vL6i~0O$T$b2`Dw!<0j>k@U z-p|MXnoOVV;7O5Ae*WF}=ReHt;T^e$AKxEO4bx)a+jd*&u0&6mwFsFLEv|CDe=EH} z8+I4?YS4v!q*XB$O~3K}M60Pvtl!{W4}u_=(cbz}Per(B;u`UQUZ+;Ej}EooWZtDx zbbuET7*y9k(jsu8?~pi;f6Y>1D4np!8*+Dj57kS*J5xK^EvzTPVjE$cp_ar#LHg&cAPic(K}q!jIG7Y%4oM+ zgezAGu5|J_#~Px4$ECOvluzy86*uoJ9!UP9?$tMfoY)QE@U#sjipGhnx?UH6DmmmB6@Z9?Q6 zfnYG|Xh>00D}R_p7^A?<HxI6d2_jIH z#q6FDT?xkze=NC{&q%7EI0ekedY21`7irH;#`FFArz<@ZSRhX$=k_R!JholGW$fji zdA};+?|Jlh(^2EFa6O2>PXX40KTnzt*>1pn}$od~e zF2s6KGmc!gIhpybdPRTrH}pvV`yOuJKj*JwtvR2Nj>} zc2;S)cyvAshg1ly@>DxzmXlG-AK>>6Mt9=o0+8eEyZ1{tb+-yd-+P+Qgn38VC*ek{pV0r0Cs`(P zbb9?1v-CL0EIq_ytyCmeLri$@WQUt&^Z56I{5KMdrEB}*6B)-^_wNjhlO6u+qVpHa z-}x`3QxHz>{r*V7abwP=6ZWRiDZ8dj8)1igWG=dE*pOfS^*`>=D##W1y|Arm9?7b% z?dza^yfj#P3m2}d$3HTxQ&yo^t44Sl<-PBUEEWphk+qC`aAB|NcPjab6XYZ@+$0A{^7S+N~tl z6FH}*>S<#c=%Tke*){S0+=2fPiNi+4?bQl3k_dIu z`Q>K)gq0#@tia0eS}Jh?tMx?=^!AUZyrq^ZJeP14Apoe)+6zDUan!=f`g_u`QvnD` z#Gl}t+Pknpn|al=5H}#YSfc;gs2C8&-2rO9Yzn>YWFw$<1@ZZ>y7a!UxBMGg<+}y( zcEiB6SeGB7;;R1I1DT(ZaicnR+DBz$W`sl9uTb54>2%P>r;237P4+w~QSkA^NT0bt z!|a^Pn?cjyOohb(MQ^ImKoC{UdLc%Ojk&6ddC7LEzEpo^41MpJX;6y~i>)NXfcz;R zbUeZ$S@(!2av3nu!T^V!?MGFNaIyr>{urbr*&8~G`#zxP~2}U%nqPpL<5`qL! z#C7j4<FMII|CR~`G(Y=cQiq7l6D+wJt z%eRaQDpPz=mObP8EAsd7uL&e4&Mm`2d8zb@wT^3QdjZf+N01^e&gfgmLN=@Q66L*T z`?gSAmS(r?&eO!6Iw#3*96-AtB5UpT9qU8;Tr>+67WRl1v&8I)C20uan z7$l?K;{jbj58bg7Plm{XKHd8i#5ZwcC_iw~i2UmMnSiWruHa)mI?B zH|KJ|;<$ts&XBjU;1@$IH~Fw@R{8D`2bkc?&O~tFYc( zixOa+@bO1bz;l$=I>FbNp7MmZlD!dunf)bd zTbUVOI<7M^QUqeAp5h3w{6p_Ng-ADkmfJ2~S>()895kR=iPe@+2Cg%r{2Q?E@P9!{ zya2l0;(q$1nmu{WrOUm(x9e)}9i~F!Ok^wJ#8(1K$s1g^t^0q!_SP;&{J?O4aQ0 z6}u=gNg1F?^fR_N%glBCM`iPh;FY+kLUZG*>7pb5?b}`J?9wO4$nHCe`zS|zX=LOl z#v|9KS5BqaDg2PfG<7zGXAmoD zWw;_l3x>E5bxCX8@C| zmL+shQsDES>u-{`?3vj}0oU=-uq4hRP~d;!*4h=ZVV29^39Zd$&}h{QWx!#|T7+9S z9hhFg1GFFCy`}q)hbtP`$S$$T#_D*EbGaycFWR*vf$Mqr_QJX|NK;0 zSneQzhCsB_te=Y;?(agB7nk5R-rdkp|F{h6Vb&|H7%?c;oS~$p<3NAnw3j~v)|TGK zqvsH<23|DUvf`HzPTp$5I9hleOlHS<`TUTO;k{%4x83&D+`Hdc{!EZnMvJpk0Gk@@ zWULd|@Bc7F^&miP`fupmA-T+Lbw8Iw=VngnB`(%01C%`ce>{3wBb&tb+id1b%zgiGFMt z++)sUPYTyp#ctHV?HX%xnsWJs(ePS6&dE1LcIzAGBz2Ra-#?@R>lAp zx1{KxCePCi`8%%Hk9+>rPvv+#tobh&RcToo!8~l_Z!6Lc%w=^#heqd$2DNmR> zu{^!^sKgo~ucMZgf7iIUS>}5|6-VvJUjgFCqHSH*H#Ex}Oj3FZry_~DN5$b-`=Oju3!m?AR^j4{vZ%GU( zw7-}iK@B83)Le7h{90JtVEGB$V*0a?1gpBgYzMsu_4@#f1y&v+*6gVfV0K~ky7Nv} zb1$*#s8SFgcf)~O+>1llv6WIy7UhS8wjGR5>T!YQR}7uc7LBa>Hk?@DYt z^k-kaGQX#Kk)qz&xsFa;NS6@hO1E?OWV8<7DtqPS&rxjG2SN5W>wtStu6%hcA6(F6 z3bB#)#Q>TRM^QML)~VkIPb5FwlAERG({ZfQ0<+CPQUO|aj@D8vV|9|Yk>rAhL>AN- zTqAX`KJLx)?sd(b5Oq)mxQ(E-jUwewOWfQMUPOyhYI`>s36ai z97LM^_*pT&=rV>8hjs}u*%q4a=<5MTcrDKDrq4GdB^U;oJKPu7UPxq#$hw zny*PoG|`Bb`daODa%jMD?eUCjIAi{_+=|VBTuvWGmGE=UQ~c!isf#|{$#{#-UJ$55 z9zBBoKS?P5sdo>MeH8)Oz^z~Bi79S2%AD##krbfldP_rdWl^XW2-*aZUJkq;9~v@i zy3oS74>GuCm#Q<6x_>G?mg=!Fa%&NjZP#~T;2%&G|9>cZ%cw5Xuh-w&iPN|Q=7$b`v&cW4tqL*M7x_(2qVk8&Z$;QX#iBoiRz-~SG zsLW1iG+6tL!DD&i0T)%BJ~$ITubau>m)|?Azw*xFtn7lF#RkfByo#-%d#wJ-Zx&|E ziL=8YgZ*XlSH`l~b~X*3ICqCmDsG1iUM^vM^UQ7#hDA)a$nb)lH(=bd0yf+MK-g|k z1q>O3`vMiku%lr8qaQka_X`34N;SKIb|GKJrD9^+eHt%4D z8vBQCO9|W;M1z^;g|2AB=f9)9lE6n9C*>>$U|8*@hV!zmxyJX$9`LzRjhEQFL51iZ zQT7NGcoK3zMMNT_ctE(l8#5K6xq7}N^-ZsO7>Gok5uOO!g_=^l#Za#Gl2?T*O6W_j zdO$T{BwD{QMz&R<+X>^vL)UenW$4Zwdc0{gt-nD?z}_T!l@bXYGpWg0o6zFjEewN$ z!CwdO_*Xo89bte1t=e9ma)i+v{Id(JE#SQk#JE}0f5vyqgqAV4M@{tU5$O$Jo$}2szMMu|( za>ySxWh8Aj{Ce4n#Oy{C^*(^jn{LEb!39;BT-nfQ+h@T3q%NubgL=F0={(L`>B}rt zn}6WFWhYh0jXeQsSunzmU90Qqe-Gxr#bOg==`g?jqG#Um;>E=G9p0~pw2_$~tVT?_ z-tsUMb>>=FHC}|E4Mv53|9Dm->T5ipcPL4oTdxU1l+Q+Qd~ha$X>QaO6zU zc4B1l{kMIimpUN3oEa-!mE|gX`az-C=wzrKqrVtO`+A!#d0qs(L5cdF`zP3uH(CLs zDgy>^f{LB*BI~D>?H2W%L3@z4IVmJ=Z1igyabV@3GEw4P&F=K1;qxpy0 zE~xHdoKh#(hgZ6Gn>+7CL`4%v+6K9{^(6Cp(b=R4T#AjWXTBXWOvPZv3Ez(i4_^Lo z%J84Ezv<@s=>HhN9wP^^IcoZ#pC&x`Nb+M?i}zcdM%ESWn+kfi{uZX#$Ny+@CjnuP zd)2X*@aGlR{F?ZtujHEh`_AkI)iB*CPJjN|xS+4QJz_(>=HDA%Ib0>lFb7~xxP33~ zy^$y$yBD%O4V2u21kR;ZMKrVtvLU^*IfKF*-AMcwuwxEuXn`3Rc`Pj#(SH|*q!xkduY-kUnSb$o7bY_n zlat-fj~ZBZCZ?5d9*zv)Fk;j|44}(p~jY=eX6GwsG zi;XiT3=4$u_-{hivNZ$`T2jS}Q8WL3#Xj3pkDUQ6w*i9-_}q3vf9rPGvHX~IMXME$ z=7`lDUHTG^DlzsRJ!us5B|*K^{;(7-bOi$ic$NfOjC)J;gTN9$x?QLPvjKOGEHKEB zoG5YbWE~C8Z0|nM+tXtOje~OoHyuUv`i0VFM=6r*Zhh7P*F4mmQpa6?v;JlsA6uqz4>0g^%lD#9s+FP zmGcp}Tp!sv7v1Kq6vZG%2oyYTX|~9)kyDB=q%yKhL+pc}4}rTdExH-1CF0tYr<%-EWM1p%yD8NR`w7jiThL>#n;KEJ_& z|7l)w=TZpq19FaT*nN>-1$-x2uwL5crdp)_Dq+i#r2V5-H_LI>nX;!o6F! zzV2XHrp4o=IcojeG;MQ#mX?phmI^rnlouyQT!h-743fdeCTVP|c!sd0eL7(t!?_2f zv%ATSq9Z2b-D6Vj+^b1@F|@hPg7w03b-!*~!Pp5u2L-&2g^0uH0tol; zoh!(k`SaTQWws|LH92Kp7BpcERDg*NIEU~a@-uIoB? zu9D$3>bQ1R$$#*5)^j!JEVrSd^K)34epc&;2rrHaD(@6xMEgZg3MVTg0#G4U4~@6b zl)QB4;COB7yl{_e6%z4-*4SN)X4KUhZYHD~cJ2YAe`UALU!yjRfRqOZrzy)T$z5vT zF@A)1Ix-!h{VLV34DNUbYn*Kp*gYk;%BeIHam#f-CTRqiF=}c5oM00I@J3i6*v)-8 zzoaT(dVUOYV4jNqo8x-(d2>~SyEb3^JAX(O1y0+4h}2$k{|06-no0o$p?EDQ9VA;mvFJT%u*5F-LS%`v3*h$N*3b4gs8EixPJ^?2H-DE>YBnUd#HmTVB zBR!l+Q!h+}!M(2c1_dyzz|loc?vd-%Gp8pPOaK?YpEXK)T(0G7b6HEKeVM%Fgkmla zt{~Y&Hb_&UvYY7~kb-gY&xN&;+64y_Fc%%r9sNipcW z#Vjm7vGRXAiad2*PO}~nYzdILL$9GA8dtSAeWKj^{i&yJ*V!!jl?C*xx)h(;nB3nE{^6UahF0%UB=y zbRRBseoPhP1F+>sy3P8MsJY%*H%PPUvf zOsz~q6xVq;U4ESvtE4nD>38QV!%5dAh4?p&j5l58!v&>WngX%gUlq{ zE#0#uFah)(R&#gCNLOX^TjPlLIBMwFFzKT26L7G_g5-|l^B9n$W2keT6B}S{DRwf% zr>UqOO#R6qO4hEYZsr?6SQ$NTRm&=Iyz4h%zs|jf=T7wvI9gEmlP_?6 zrF1m0^kUO@sfV65<7H-DWVj-pq`C5W)u~VgnSAVZC1?)ToKlS@JU3d0lrLnuqPU=H zsqQxA*52+;J)D8>IfHe3rNHFHUy*iHW9vPgU?}I+JZoDlJ^>X*u)B^~cg;IkT| z${RHgU)^*`qPvJ$Eab4KY+&Zx^N#Rh0Lh{zK>I~*f&cgGB$wjyG=`M7cl^V|D0)j! zl%!ibi{l+u{BFV(sL%gx@u_y`HdMBEk(*JUXPkBq$=<*8E)*;s@w}Qfo40Tko7Q&+ zDMb)WVLYeOZF*+9#HQOS3BY+#xm6bcmFUg6^2;p5#2?YWe04^2D2Oai9n|QbVT&qT z_MZw}qwO7;uf5&yT!d9X=69_X?u(CgyaLDFuOrXgYk-u|2s@ap)=y-3C&?8-6-Si- ztDnh9Qt`h63`x~1N4o_9{nzX(60nGYo`+}-7zaV4k%CLe&Xt^}PPYP}RwecmqP*_nUUC$jkR#uSN7cI}z^ZRBic=I=k z&acyJn%~qN_V=VlWtKqp(zF3SOMKc%oaAv#gg{*ldY1Eu1TB*+FxUW@ozU}L0|FBg zFZqbsQl{{&;;qsjzYijrmo|E9JMma!W?sHRs*q1A>;B9{!+J~H)dEsy18_KcpaPZl zU(4_CiF?O6NLG0Sngh;p1{M&+l z@F=Uf=aO7s>p&pJIYp8DK&Xf@p`X%JBn9V8s28#u0*Y=Cpe_-XBVEXZMcK!)EKuxR zLqHJ9&yRKAkqBq7$N&=>6X=)8t(ear^HV+&Qq>*2mJA=4T!>@kZO@iR$Zf;P-Sb74 ztv}&Nk~;J!J@}!vGkF8>A7BAvgOEBMA>*TLB_p4VjcY2bYM*i4B=b4~sVdLu5=`wFOhbtyrCr8HnF_5Wo?IG)MBN7pU++!~dVl4p5+ z&}jHu;ZE0s0oCVUWR$|O5k=LdTtfxa@4pFFozzW==K^9kBKv;H7E!&IyXK;M^yg&#z$z(i+~R)d|;V2=0W zKe219lR6RbYQhIXUZyZ?NB~DK3^12?kYdRfYB+gA<+i7gCKV{6fSq?86lnxRUl0hL zqj(BX8qFV8URZ(lRTc8>a@F$*MxNv}Q;BOM1xSscG{wY>OV88`kDkcu(=lkEHOwJr z&ZNkXeS|}sS;>0fXbRE_HOL#7S~`t``PC}kPy%jqof)y&# zr`6lX^q-X!korMTmq=2dY_eC!9va0_=gPxxWpTg0l%6Myr=8Uh#q-t-E$<=T44J}M=!3E5IwdMuRS1dat($5Bs zLm!XVMX`1>iwP1m3zQm0{N7k zlQ#{59%#$#CTtL%U`!y~h059B&I5sfzC+lW9O`B2cLO{@9kYJ5dlOlM>=&3D{NaYk zuF>5skA~F`f`a|Gk#b1zGm?d8NDX9~f=F&SUR8F{&ZWXowzzsL_ms6oJ_8p`U`S+VSZr}=Tugp)N zAOJW=CtiE~mI=PxIPjm*I?^mR6G=9T1kg*-Rp@|}_;=7lRxZ+&BH?>q?LC|8dsXK8__GL! zXfPp(irvKBlxK@<-oKdE*6Lma5_G0mDMSpeIl=E!u)k8RNVedw`iX*?kzV&pV4+|>oQs* z0pShD=WPTC;pri>bX&J^)STvm6-p5h+Y}oEutCC#kd(8R3 z;n9buS86FE!Q2{O57xNd)?6`84)2`-1(VzG@MSzt8(%Dhj^JU>f~2PF@`Su%5jQ!G zX$+Y^Am~z)S_aJ#($odD8@7c8@V!huVEY)=h;_p@Frm?sx)?L{Xzb=uCJZ1yD@6l*b% z3Lb4p2g%L#dZz@MrU)VsNc<^8q|G0U6#gK<-;+eZL}Ik2f|Sn7h5FGK6pLY8~u@eDS_4wiKotzvWyv_`4g{N z{FT#4gYEbi^mzUH~^0sR~pW}|a^}czJ@l$8Yr*}u*#T4Mr zfE>VAvfAMtcfuK;ZGKQ08sE67+~K46PhWA@ zUk=cMM(GBk&ms)zo`Adar0X83@QCjJlv2){Q?L_bZ*Fj$&}#Slj0vKfuhJ~*;8U+$ z0bc+VEI=c(vePS&S9Kr|J9&d;UF&u`<`Jrp@PW(uj;_KZY(U`9YxPd4@Ax!knhzhzFb7J|nq3WC2Sb4Eg}#Xk2b=vHM$ilt&FZ`M zIjq_(CqAGZc(iDkfK$Fy3;;=_Y6f0nByT+Hz61=YUl&$7XpO=5`o4 zb;N@{ssAYpGv&*Fq!k|-m}It?PLw6EtMWOzPkI-F8)Y7NGqa`oHbK8$!o7tIev((e zdCUDTJ;XF2i$>21EZEOcPixIaxz8|@!_uDFj{@oru>KeH?mx9A4+Lg`_0hqmcy!>{ z=^W$~fE2pNxb17;Ee;lV-IBo%jU>Dv`T#B;U!do^?f(+ii}i5fw3K9h%_!AFlF#9i zwJ$IVwWl!!qAUzv$Crx1!osCo{+z8C>|&9xg{gQ5Le^&?lu&c8k>RFQ4p5R=n&rRv z&wYXrEL9i8rl&iKn73h8k^WRc6R=*C7gHwHHcC@G8E!<)MQHq;*<*3SzLli&e?jugykA>WD5;5`pX&;!f{1APx=K3*}PX70ZPz8yxGs633&L- z)1FZ#{l3_Vd|DT0KcLi9WM>rcR0o<52%hpBmK@4$*;I+R#me|&IJL}74Rk?}_N2+t z(`(WZOrE;P0iTQUgH#cyQ>7AjD>xZ<0($Wc2O8ful;5=aeiy#Sce<(|@@TDETqS5( z2Ne%9Zg!Zg@JCyuWq_LLn)^gnDXfd)K2tj<8%Z z7R-GxU;!;j%NihQQ4`4QcOAH`S+!7mfplQy$=900Yoo0V;LDq9e^<)sIyoe2|A>)U zZ(d6b0ZDP0FLggY)qyA%n{&(XrIIO^8NYzWq^3mJ_1LPTQex!&2S8>>DWC??4wB2PY=n%? zGCdj*stbbGcJi`=dRCM}f^qj54`SvQi?dM+_Q+A^Uy~(4U6a2U$_AlB?>PNhwhou^ zL7wq3I@-U=4$2rvUCGz)ID7>pis0LfpsbxMonDt1_jdu*^jkQ@fCvlc*BXy?u*?tM zhSaxF3IP+loHbTV5gyL!E=%KZ|07kpK(&{1@!$#*B8I?9yEpPAgB#^l?a27JZkrSS zI2gUoB9pRe8fbh#5e;J>xI>M$vRI--EeH$){Ur(6{nU*m9?OzJBv-(p9fS?_o(X}p z-*#H~#`dhVw!h#saCISC zlH-JQ@rhqNV`yo@4{!0qkFf7Ji*HbDS{bl9KDC&jwmevU@=aSKaI6%=;B=pf>2`ob zdX!WSPa4c4Gcri2MKmLO?Ax@8=!Xi0+y>5$)F5~UFws&jgOdzo9yp8{zK5Rw6}n|c z?{fK=_+ND~GU8JCTEy5|j`hTle2!SCSmqLw^CfRN+DZ*8Pt`_?TRvlSmBB&I=iBu) z0s$b>_iQ)>GTGopg7@OxdwX=F^AC#tGRP$2^ylMbo$CG_;M|F)^=)f-5eRc7WYdkz zGx+E=8ukjtV>GaT75y5jfb+AkzM2xEjxt>IIVc|sr#)Xa766QGy##FZJcekyDe1$hLIp z3WKqd=K}8Ks0c_++gZ~=p}+gcH_?VRaKf5u_U1jZ(4zI69G5Gy#C|;sZJa5?5+RN- zykjptHytnrfs50y$--4R)Zj+jjTl2EZ`a|H+gwSyXTe&SwtO0UAqt#z zSauP;*AZY6hxuaVQSxnLQD9Y4ueU)2Nu@Ba(d)EaA5y+4`DD2^>=(3Z>Y*FVtAlbt zr_L6Cel$^*Si(KGD~YTx!QBQyLE9f$;Hv|mP=*){3(O0WaeJ2Z?8>nINh(A4Z?`?7 z+9OD$+`O3z`6of*+F{U#9gbeQmknhD;Vo!#lM49%NX5{#w<3`%;2yqXxy+YdS7T%K zdbx}Vplj4XeT(h)-5@@DjFbH73cGBfxDtnf) zf5qhWc~@Pq_W^Tq8~OS}$RPRjNVx862$*ak3Qpow=FI)A&iZMXQ0&BD_1@jd=gLwa)+tBJEig80(r~V)he6lothi$Uq!I##m#@@Oaq;P8;il9c~ro@Ed0` zHmJEA@g=geLNqV<3=0k_?Z}bWBiu1FrIWs5@(?3Mb;U=lFH-gRIB@T3eahK^dT0!N z+q6c#&iTdA2M)+cU@O1V`VzA)fapV1l(NF^ZGV3P%qTf2n{*8-Bcu60d6PA6TaLG_ z?oHE`m^gpoDE*yFo-!>+ec1D-%HG-OZ~2Qg-j(j`d#{Oi2h)xcsFths0I5wGMor<{ zaK1?JO`VFJl&ai{t*zW@Q7@7s+I90@t=)mg{^a5PSz6pse5uoCTk~)`&At>6{|{s; zCFc3avwyvu_VY{f7HMB<=Yf0_Bs$0GW z&HrUtY66bioFeL{yFAO5ygv-fBq%;hm^siJ{%#_OCjHCX?g`G#6d|X`Wh3Xrk~d6W zzfrI3d5x8Rh!SPPI>o!yp*)v{$^4{!nv!Y6nc9 zm{}DuX$9FN<9w*snw|y28i)-Z*JDNP&50g^KEN;v*bbr>JnLM`1xGBhcU0r5P z612zvK{&$+p4+>YiVn^mSb>V-XlGQWkFzdvD?bdlonI-h++t*a3n z>dvO7JqnX_#8bM7^KTR($2%DCJAH3Iotv0ZmPO^gf|aq{@V*S|kghioBv*0#45*70 zs83)1UH8S0Q6gfyQsPy&Li6DBTZD?V3{ zDi5IsN9Pt;Jpfk9>Hd;kKN=RnOoNmB^^fb~6ZVTsCiYry5xvU!$Actg*AQ}Hkw1AG zl6+&h`^9L4+4%_YYfGq=NGM~Hzsb92;t}PbG4z{5@&dyheoB~VV2D@wh;AjxYFlM> zzLClpN^n0T@2a=rS-}1{;EuxZh~3@ge!=-H)3a0?shxf;kAowT*$XyrF0AN+9vK&@ zqPFTk$H&xJTccD}Qh~y$`s=*& zFy1gk8RCwsi-x&9h%py@n>iT#5Lf!wju5rJzCS!dqu&Z9IrHJ{z zMFn(xl6WL6inCAdR364ZAfVkkwQn|-tE|2ByM?OiSb(+U6r1_I^?;Wyg}|R}lOpJK zUJ1iffg?Kp3%iu3HCOvXsmJ>#IuzWTa+S0?2eSCJ)zsRT?$q#|hSARCnrFbHz|WJc zuN0Mh?kf21Mjr^&WI~=c_7lt#kpp|=!xgQfZUD>A*@dr$%ffCIYTygtL=UA)@<|e` zJ6lZwWNp#uk-hE3mWk0wVwSCgF2kq0&}BZoP;=r?I|NgcNiq;t{g>l!nCx!0*IGFqu02g|<5ZSi^(VLVKDeoDc{i}W z?x>Oo!9+waCQKAv_)A7X@nvenq5@4uqvur}k&Q_0bvT6)h?;3J)V!9!Ja0aw_KIDN z2T@%2uW9Rxp(xdrYY0C1h(_^r-fl=Gi-AFysKu4HZr6n|_95~&41N!NY(L9D1YqR# zYmE$v2ezOy!kh9`BSPjcY$9ww?d3|x=yk>`FjekiM04B8qC3?(VZ$R)Q_ESM^Y&(Y z0#PJ)=u>VdyIhr2_76eRGRbCwS zAMd5&&xPK@?w>XBKtnLVo2nW0b|_EY@W8y+Ue!iNluq=-F=X3q)$tZXte&1H_z~Q3 zmzFaB^%j&tl{73JJFajD`8TT0vUYmzfWPzZL|`?gjEtAFh7&mv&9!EHj6^me+iz;( zgG2PI(REwNErClB2n5p;EZS*%s+iST$4J-RirCcjsuCLb&PRH9&=?LhU1-&hz4|%8 zCMGFwKO*YYB5G88-uCpu4X4C{)uL%{q>%9Uw7b#u-^{I5-Qt2m2#v33!d7{?=50Uy zc(e1?WFCj2Jo<7IVx!x;aN0(T`-4NU_O&8$jW-0QEVn=P#>TBWiP(3WN}c*@#4#9| zOQm+~{=KK+<$owggODhQX|!C&Ca9nJW2bqTc}MSP#F<>psf5ulSdj(LPPt;Mf;r;E^Wzv$Wz6hD4qclO}M#o@>DdI#zW zJwYr4rV3_bCYANB_L%2$&tfdz6OQHDK^rvLY=Tr0-{yX`CK#F$r2M;P&Gsk+tTaq!YCTMkPT(#ykD59cGnT|@S{hw`Dcv8Gi0S_3Ga zL4*hbf^TpeaeZi$!9No#N-Xd8`C#!} z?=7nYx54o7o<|6KRW(*T_bPwn$&_#Io|0MKn!lRG+xx1%QLMaBYehxTtT;8;y}zrI zyn4m`SE8WP@^XuN-DTq@g5G%+tW%SW4RKe$AOL(cUKZiAf5*;cdx)gb9jvA;uJD&w z^RI6Th7SdQxudG^d^U5*mK8aNvSoS=Sa)a2owqAA8~dNlXCe?#yx;BNNz2jk>mfyj zf4TOS1o@QBWh0A&3n%u}HIvm`Z6(PY`;@5Hq#$(+PiECW!Rsib{&<_e#`euqECh_c zu5G&8eN;Jf_KrY4UfQ@tl{cnJF)CIGD}a zeqmcz=_WR=S@6?7PYeUm{`@}FK5}UCc)IQ~GJ|AIJ14CFzW9-VJtN}ZV~i+UE%~UN z)SdAhd4q$|n|sVTth=!TWEd${b1}!2hSLkKsine-a{X$iHcC11=8Z3>yI8hkS_8g+ zJDd{(*WO*lBE zPoLv|u{_@A-h8HSDdnE2tdJP4yClUJ`*Iw2ETaPxQ=QRNdDmyqCM^I|D3*48sv4QS za2|hnlleY|N_6l!3IY!GSBfF>tuZdCNBIIni0D6H3b(i5ghtT+=#k1`dio$Ox!3r? zTtRo|3q$3bMD`hQlE#aEQG9DRZ^&h>6INu^6v&t)HpTim{+`N_W^1}MZ=X4Yn+V!} zC^D2q2l!O5P`%JI$FcIr0x$;{Og_1)_(&9jcE>2={e2UcGGjwDG&XK?y$&Bb`GOb* z3%NO;7#i?dtJ&YXT6@5y7{Gnw>4jc_&K+sE%w946+%k^Vl`-UEB=Yn8W0Tf8hFZn> z;4H3x^yrTugq`V(KJ^{`gy+w4d^vU|MUwqv`Q*}~bngBws2^r4c6HwW`U3ZNP+5o_ zs#_9xZ$2v|-V7>xD6PQ{dftpCUMZp1Wn1&Dv)h7!M!K~WVQcz00guF9&Ud9BhZPJP zPpYxLN)YC!u*a~ap5HuK{;9%=S(x>$OUm}M+mBTV86MTZ`Mv)@!1>bCtz!S1FyB)f zPUPh-J_i`|a5-LN@oF0sjlL`!f9bI29ZFL46P*f?h+Bjbg@{t%yr3g$|LpVRuJ03O zyA$tN#kUf0!o7QZ_3Mjj=WU#E>1neTO5(dhUDCR^j2S3ue+;A^)?iVjccd4*`w=hk zO$Id|s>gQ-L8%N8jB#FyuH$28_r51n43>%YLZ)eXOB%AS-w_D7|8L_f->lO=$#DxK zr(3H#zr{Ox+?y#S#LRJ%;wiC50ZWL{;K!mXzdb1-s2XC-1KK>n#Q6MC(3OW>+gjE- z7FIQL7oxY7}We#@8~Q}{do zXUvY{1*`pdsbW(;TV6*N3^cR_jotk2v=x@qdYrYF1F3%zl_KNLj2Sj1DH}BnI1AN^ zwx21Hm1MF_O5l)X)21^-y%n0^jP5KaZnQ6*ESnAASb?`mT9Yc{lFE8*t{^4Sd;&+H zgqaROukovB`4UT7jh8XQiz=d5KIA;S;U3e!ar_Wb_>R~At=_25`Nl`ZBggg$my~CX z!;DX}&+Cq=Le{T$c%>xj6B5)_HUW=rOBCt6OM5ISD|PdUOzx{-oG6ATx23b6VBbLv z>oe{!DK*egT2j^U3pSJ3__itI7Hl|*hPFm+;;Xqbkq^DK*x9_8qGp^g+^gJ%9yo&x zcV{npNJTj}^BM~(lr!rqpO2kH=d3eb&KiI26rl$g4!A; z{Nh&e3?}x61ob-UvL#0Ej?En1gP0oV)tFw%yLH#BRl z!MZhfQ4r44qUbht-y5^uV7Td;T#6dmxhRQ2ZNHGu@uP7oLR!IL#)sCva_scCou&IC z6M|lZ0W;FN@^!EUk4H>uZW~-CdV3@^IX_#yX&uK8SN-(eMN&D2HeCLG>h<48Eo7H& z9Yl+FO|3sAF4J@4D?9#)rMcV3R7~f~c?;QZyLise+jAK&wx{Repj+#Hy*DXJH+6I= zNdDaf@gqrF{ua;QUGoOd*BF6*8nmOqA2J;wgK@-U?yGnY)f^?EXDLnv)V%d`TL5JH z-~Um-IZ@T{`QQI6c=?TK`M7G~gG@;q$Ycc|LuJ|-1<@~8yR{AL!JDA?0eQEvBoQUqemjhu$Lnxw+D|RMrKRJH~qfqt(Amf zbq>Ir|E<^Y=(+uK;Y+W1tpr?BnXJG$meg;%Ri3wcIfIIQS#XW5j%SK>i$2u|C8gPt zm`o--X#QR;0?o5PdEFZ#1hUwd=bv0SGV<6B5*0W!CPdJCKBA`ea15A^oK-pqV6L@2 zi*HPL)#)8+1iz%eMlr&D?4A3nuWUJe=#%?~`%-e1%W`^%USzRRTX$Xg!52>!#w)1K zB<=J#5cDwJs;Y>jn4|1hB@cEQH7QMRVz=&^(pV)Ri4zsMZgE*l2ii1faeddkh2140 z1y!Lz&~Ztwu|-4H8AW{4I3WHad-DYcx@M`vbtajt0IW4O^*mf&vQM@Pq1P7j@Aw$| z6h6=R$)~z@-*a~L#7w>v&mA_d2+VNY=EgG?9ND4Z;B%Ag)JUR`mT9uxqkw;FCNCKN z<7(DbuJ#KUH1E!?+rN@azCIn;=SoV|mtB#jwng2nh)u7>A?=Yyy<;vohq@SStV&2J z$IeG}IB0gE#q5Vcj7tZnhd?z9i+Za>$yTFmY!KB`yvgk$s#1K-KUKQbslLUl(evsL zk$iu-*2!XPZ7TTf?;pBn%<+Z`-(=XmU)_bl$c57+%#3Pn+v1hm6jOKu*&;sEI0;58_a1YhjYYX_w3YQPuey@mXVV&WmP|u zjac9$XAt_~-trNB-DCbH=X$<_tL`-)m^-z9@Kif?j=Y(*iq6Ip3UzAy) z8*DDHTD_ru6ghU)0) z!6$D?$O*qmWIy5ieDK+(u(TV!jPk;L>AnkB|91bAso9${E*%~0+0xIXH8azP%**C< zN>S!Bt_7CSnoQ8k{G??J5~~Zomnw)BI_5cT$gOQ>pmG}<|M~nUeZ{sgD*1!6N=P|R z6)nbB-`f_i$mgXwUch%(OVU{5EmSjYJ<)|=;vZ;0YpL$q+u1stX>K)CYb$?YRasd< zM0TWZ2@#XEhtG zf8hBX7IL`qAoO{XhM&q+B|gV<8ofOyo)x>fJVTwUi7~xBYtIw3@&-2|zI%kr`Edn% z(G7(}ds`Gii{hUcYmfXY3R_z|pCj8@C9emc*e3e_Wba+3bN0Uu8$H{6L48p|kauOn z*UJ8gq;8AD!FF07C*4HdZSmfJ_E9~^e#84}vrMqsmqk{5cn>3~Je-`) zFRAJmW;mXIK~wN=qwfL@HAH-QxF(q-`XY`5xi4=sbM`Ke6d%<)<2_&cbo#vYrCE-3 z8ymi*l!a1O5eE++0rc2}(WKap+1OBocgJpIi;M)n8vyfyC2c3qs-p7i=S2_)V)R~w znG(id?IcgihMnf-{ych?T#=gDl)5CH$30V?MJB=co}Tl;H7LI?FMfzsRu&S0Nlds4 z&pr&M+F?nt!9~aP+1E(=DyY=-f4?gsK0q!%pKn65VKmXnl2m?sQI5e>`F`WP-^)@I zcK$%85C60nfd&b6P{g4tyuMsVNVz^u|k6u8PRukj8usAX!KujN-e4 zhw}508RhrNqMB+)zb#Y+6M=bMjoCiO$?Ct5r& z<;CiipNVfv+i8e);)`M0~#g<9oRs_OJgp z`plB29OuYlU0WR&{P&9t~{t@t7HuqNIw}lflzY@Sgm4mJz>3xzXBV76byMNzhLA zl~bOczd(Nn{mBUJc-ZhK$i2U-=}RJ>^7b|Vg2Y`%(cyt|3R@3p!a;^dg^Q+8AeF7b zyzu;HJ4u~iKL@NVlg8BF!FCiwItvS7YhpnGoyd=}=VKSguP!tk3kd&tW#X^hl*^h z^iEn;#;2`!TdCyluL~p~%x83;YcRg7s+%gF!nd*uv)_1a9QSzlV*#H+!3PE=t?;;9 zyowWH#k!^PfPkY=Fd@AAXAdcdZ=wf!TOKt-L-C@n#T%cvHC|BSrGQO#p5(>WrM@(>icwp0>+UZ~DqcQNk^%ef5a9Q~DU#I>kkK7IHi7-~n2N~a*&~Er` z;f0X;k#GbS_%PnbilWSrP|qfMVerWRuCcVP=qWmKaG{z-h2(1BhWI%Oy$ZU+@E%9W zhqLq`k^jf|VaG#F>T8_^=Ld?4Iw`aFZJt>3Y+f!Io*xz9LS+X#nB!CNHoMN~JqUK~ zz51s;Lf9Vkeudp?q~wJ8^3xZnujdp2k1>OvGFBdJ;8j!(jrx`-CN3Qm56d% z-U9Z~Sa7Gq{_L}InDltH1ZJ{s7wVus>Mtu|{Gu0_^wo2DA^y->9;rf>|% z?=#1Ng`R5mE8s{-0926Z4I?4Jw|(uYi;cwbVZy2hfM{At3~0DVO?!MM^)CbH49@8; z?~f|~cAS9#DqVSmgy(Zww~tV=adw`JsHUYvajU;!I94K?Fkx1l!K(ypIJ25_jB#l9 zIbZ~4yZsWkC&fy}C=;*0-yD{#v+vkGq%TXq>@IO4Hvj%tudQ;@55pOj+|sZHEaNv% zf7VF|I+gujc3!wl0_46fY_I~X|Ge>gNtOQYZKO~kUG?B1wE?&F$P&JIw{Lj)yvZhuD45m2%!P?M}wTox4T`29b8y?0O)*cSEMfPiFB z$x%^M5D`&w5T0ACO5f(CcNGE-uLR& zS2f>%Q!}MABi-lhz1DB76OUnc>b6i){mk*_&w0J_8>HRcH6&kY-_@V)5>0%SM`Lid zbM*>yPtl?|oP#4XvTridZ)|+02I>iN;3YXM0$d+o7Y_2I!9!Z52&FI^1VUfr%#Fn* zRf2cQE%(am__U^h-xrE7xtc4Px`&?pnk_ARU;SMixM#GESM=|Li@`uj4qe}{24)M` zGmeoHKIx^ONnvJ%+4=n22lOo86to|l)@*6s8&&3ngAo>Y1Onh(v6E-A@MZSpZ%p~V z72RM1GpKd$J?}9q*h%*eLNW?XDm84bi&p*3qSQ7Uiv6~yotEo=lQBz_vC%z;ro1rg z46rr}TC8J)1BD}3$uNZplr>O*z}_0c)(%*d{ujy{{$ke;gq{N94@LOz_~)MT`75J1 z?6(Z1SQ->J8NmxR!VO-7ktN%7$5q?4oXpGg90Fk>(ja`YRs?7BsuOnxH`_GYRF1b= zCkLK1i81=&a7*5DiSudIo{He`OR13Uckg)_5*8~^pPYKHtoUI$V67!WRGoN`^*T6C zl#;>F{5?%=YsZp08|0{SoQp3rE}b#kZ2NIXrdGpB;lIb^7wnDm;t!%{N3RrM#yP>7 zt9;GWmr(j8L!YjAocc-OnR*5y6;|fegd%&KUQKN#O^Tp*;&o2mn^#%jr)osz&aFzq zd84LSNyyhq6k!g34qRuWe?L8m#+Ll+i8~Q96FCh3npv^Cy`MwCZv>YS;3K+ap0&Gn zZEbJ&miBoOfk)h*THVSpv20oOUh!^u+wBCqu)O$r}jzKQwKp{yJ~{wy_{;R{#?iiBaH^iwMzvfsuKcDwdf zM4ap;TDscG2FtOgo?9=9j)xinxuVu7-b&QCKm>O@LGw^naKHHfO7Pmy3OO54Ut@Vxp7r`I@A9M-vccEM6M=oLTOY+gOz}UqXbT&n#sqR>> zJu4Y@TxnNW@yAoMJs2^Xr=Zp~-xbUtdE0XA0QB$*W$_lD`X5%TA^vPUnB}T%de!2vb zZX?YrySB?`ErQ87>v-kDfV7vq%E@f+ zlkoP_tpf=$mM}p0%$;m+_^C}KXRM)SJC5m4mbj6uZk?>v9h1k3i2TF)i>jt6yaytP zIv79)H>aJnEA~y^zj9WsbgV*mA#C;(9; zO!7f#MuDaMT{da(LoPqa>b-FgajsyFUGkB5aSgRq+50bhdVx^6^nJfia<<|+Xn zO*IYPzPh21IxwvE3Fb5c>9?uMd+j;Sat_7}J3Ve55|R$Vw) zB_9nQ58nrAcXBiJ%+mGuJb*#F%q&%YgKRI+$vw+vQ{r+q(0;h?hhxyAgW3`KO# zmmf4JQ{*=k{_M_^=z*x8v}b!1%I6hpXRuUZa>uoC|6+aBb{w8CGyR)u7W(F14J#&z zMtsCVJ~>j}`<8G2;!LE9$QxhJSzgd)Ih&*KF5JxA)yo?<=Caq}u=_bZy$0{7P1-9iM)9hx(71^C9a-=ynX3`S1`Pu7nC3}ZGX686~Z zTM^o4tlqC+f8P~`^Y#C+HnRQC3W+yT4=Y&=O(kEoJe>H-`|L9Z)f#6kV3|}os!JuE zV~0C39xiUjpwyLCdvXATB@a(c&(WRMF>7n1Kfmf}JPP=#s=kC~pA}0);HN-5_T@&s_B1^fz3UQX_4Kq*~775il|CaQhzqosRkwP?D4Vk(e0wuevNlZ#eTE3C47 zD!nZH>KC*%iyq-w;4BH$7?Ha20wi!9S;@vMo{#WRCTOl>CsH#v~b8AaQ_g+5$WOCCaaU>f}$dxcvp8`J-rIsUivP3 z*)1CQe<4QLicYTVIs;h#Yhk8TY;nC`E(Wpiaz^*)`3Jg^-Oebo!)ONt}KD3*}E?i1}t^8z*m1fES2s z;l(Mi>PQi{yu=gMTuT===8D6d+>@6B7hs$z+C+=U_^0Ln(k`>a=Vm#28|IAE4a1=c zw&9Ka;cI^nV)g>Z-;RZEwN@=sQ;X%Hc4?1^O~ zOME;?DJcztx4%Ep50B8~E$`T=5a`TxdJxOlt@hySnJB}5B8(kUh!D5l+dKx`eDCdX zl`@%u2Rj{tx25&MwSg0&730|Zj|eCR;P72@o|l}o&%uw31U4A(7>CkZa|FMNmt{~t zyS!}PQ$<(F7S9B2Vyuy1Gw*XO6vKU=;_#{A4ULily{GQ<&j#T|0KqE+!D@4;Up1~M z%p%6hRZ@L>SU=u4P3(jX$aj5tmyFk}^4pzAFhyF1IG+_Jr2tV1;On16vThfsWf*LKESAo%UnD@!V=b&eTs=vsE8Vq7@GQYxm6dl&@Za3utFp`CFXD zKu1Q=Ha`BdHqK#m-ZM?y`G=PJnA?g#MrSb2zvS#b{(G>U3zj7)*I%+<=#^uupjxvo z$|IV@hRX0UWa58QU_C0Sa{EJ3$K~i}K3#cW!sZF3$iqlMShCaFgyf#Y zfO}qO(>q{!aq!j;_dmMr{bw4tG zq?Z~~aO{67v@^d#1^n^yrL0~>^i+7`PNL-b0Zc`v5a@!kdpl~McGfvPN*9Xe|B$@6 zh7LdA0L#SY*cagirC9}_KN8FV!l7;)U<#Y9lWcztAClGK$TPR~_F~W39nbyR=)JXo z4;D?HSeNLj?AaHB$cYj<(?3rv50X(euAS2a{{-%67c>x6bOje6?9ghhC|pIq$ApJtb36SuPA$FL-T)B%Rl-73EE4sJVR*nA0&h?5Dr{DOlW(ev?( z{x!E8qH88JPyJSsNw~C`KvE)^8R8kBqtFNJQMK>JzZ%usb`1f-cm8Gjo4CTsC(v2K z;#jI5E181-JnXvvLd2j5sTOwRCfSrb2LQ);F@@&3KNIxeqE8&H@l_w~>X&)3vAOOw za_?HU#!&t+4XDX>&Xsy}WqE;8@8HpCuF=(X%)TBzfy6#&*^L}0_$RlyM(Em3#bN#F zCMpC(5W-2gjUDl|8a6HSCmeUd$+ACMW=wxwMc0BGH5hp#=3mGP=ubLsqxmuEtZVDb zlBvd4_o?j`T~r#4C#Qeg`<2GTuX}Gdv+rX_-9N z<*`|!AJ;~Ppa>oVqSl< zth+2`-1*I&DPnM-Zu-q#I)0h9FY-HE$l$xw zryh{&l~xntRzG16^Z$2I<@+3brdM?BJ~~VP{tUmjsly>_+|cLSkasw9UdBl&G`rM? z=PF6c32TXl!)D@1Ke^}|ezfhc>4i##PeUf5hG~_=VAjaY3rQG6eshkFOLcYomOT^x zmG%imL9}=A!ck?=4U+W{zn+eb(PYSEwr=qPQB?`raR%?k}8B^P(rj#`T z81d8L@c8cBbr#_(t#%H5(>qv)d&U~MBhvlMN$Z<_?2rJ`L=XSO0e&zv1tF`~N?XD>0D- zKXVN!bngH_*4Vj7xIJ6YLL4qq_Qi=m2v({#obB+T4Vk`4$ugOz%ggx{wMQoXL^=Om!! z`y?2mFbIe)F)kl_cg=ZBs`P$tZpYc3YZ*h(pUG=k5YPmemgrd^M_vi1cv&v-ZRX^V zpQaT7yj=cR1Auvz`j2vv-1QX?myMkCr=Tp16w-omjMrNCw>lo@JWX$N`dV!iB;g_} z|FzNT@u}Z&w|N=hYse$6Koj5nx^$TKU7Ef8&>L?(yoA;Iw{=Gv_gW`5xL1J+i84)! zDgWU(_(K##pvmn4&4rPw{MO2b{EErD4kn}_ZCndO`R8QiFLFExI&N~N@ z2OGKrciwSdPZ6NnSv4=0o3=akb^mA`ksBfdiM35$!nuIu6)b0uN%^}B&~Mfa$&mm zK{bABRn2Ay-c|ekF*&OI$LZ|5?{2(!^x|6o=WRvKmsAD^jym2_rM2{6FQ_N-wq%CD z4V|y3n7({mW~iunL$kH@!gT?TSLQ@!%j-jWuZ{icY9>)arP{WUr0GdtTy7m!GZAm zbP#-{fSbRxQtcd9K13bJ5XM_Ggw3}rQC_viJ~QFW>R~^|wZA@;&cBB05#uiQGRT|| z;bf4xa1O*~V5NrH-LeI3fqTn)yz4lLDz!gYvV{?n7y%d27d~VKT7*s)Mz9#WE9ZJ< zK76~tqd4mM<~Mhnq-yr!t@GzqMu#u?Nx`i@A^q63!+Dih4qz zMQ$t*KJ(_n9;9ETYrXF0NKl~~%w6?gNN=xHELZ?Inj zFF9m&isDY(?iW)?H0@2GN;7IRAD!M#mOM|k^q5gH$rYqHqth>Jg$*|rXYg@1Mae*D zwQs-WnzY{L#~y;BbtQG>EW60K0ZgmPC2%d87X`mA*mjd3UQ!tn?#+!xkNi7ki4U~L zWJochPplo++b`S_buWp;bl#qt3M-q+h|S;G=h$ypl8HnevOew8jUgH(lIP^l@6f`X zt{fWkk8lD;5Qb(FL=)krH?~y5E=MCtZYHF?4Aii42olL^#^>ex^&6kqpVxS)fd{z? zBCYo}6JJ0EikC-^B7Qsy+z{ZK&K=yp8%BXWawhlRTPp!w&BM^7zMgHW1g?W6MW_1u z<_9VC0>Ao0Y0vVbZeNfP8=jG6v|=}3xJ9u@(EszaY)X#6zLv(y>q;FS3weBo_%=kQ z-WH#Eds~JGJS|W_r&nTb_A!!4NZ8CzB1DnP0(l7f5OC4R-4Z~Tq#0yJVI!EqIPyMM~^}uh(kINCB7pAO>Q2rJ^ zj()m||688d5vVTL!Q5kQ{Y9_Z^Av(dbI!XuW0@hm!U~U3L+$>OH1(B|=zDoxl*;m< znd7i*G9w{{q@mkK`FZ)cBQ-u4Y*5!*OOF2PUuCk(`a=OW3kJ}b?^-+k6Mu+u<(##A zo5=YF%Kacj1}5QS|L;W1I9*Chq>Ge5;ciaQUgz1Z>JMH|?PLcIhrGDXa&ZTL^7;e0 z-ynV!BEKpInCigbT0V!UmUFN6r%XO^QB@(p!Nw9ABnc#Ms zuRM1CK}(R28Dmvfw>Dn$_Jd3Ls^3D1z#GYcM&c~C`T<^p->v*w$?0(DNX6EE3sHH1 zWA~RM2M=~d%PaM%TBI_7!wC8DU1wwh_O~%!&9#CM8wB0_Y!Ae^NB0jeWJX0hwUzC? zq7W}CVj&Q|FkHP>2(@pG}gwT0pi2uUOJ-2meqAAXm(gojE=L*d4;K{TazeTBg;>$Ms0E0hi<5p`4 zi(S131JQ~X&K6}cI($5QHS74r_Tf3BlPdkwZ`@79KEf0TUTkBP+{7zLQvhl%dwgVa zxNv0S)89PjWFkY6+#l-Y5mWzO?$fAES_m$-m0fNyYKh8trVU0r4P1>**_J4%53wf> zu{Y51DuJZ9=+*{&3(i2D5D$RsiFiQ94{B0q9x<*Cu8%VOIBw)9*dZfW3SrRB?uJYa zu|~%RD_*GoXO>4bnnBLNx!JaK`h(#>Mvu9%$3c(wwS#@v;0pb;wHE_Zu#-Vx^T5qk zy%O#C6LWEDUJix5O>1}%bboKX$QYPHQUqk#TL@=x8@KA(lYwn^XAqV1B_4p*jqymZ zT$#RA;3ocaK(;^wU=#bCquLZme$9_2Q6W5SHw_zh=YWp6Dj*ZPVPu`#J5Bf(hVeTs zn=+ft{#DGvVMaeh64r58xEn2&3?xPldMH32b~l_a+EId^te%Q;^yy|!)OAH)onWn6 z!ph89kk&B3980p0WEyZ7;IsZ)S>TJd&q)kjKq?*l>Hc7)0EB;jr_zCWT#U;mO&Vz6 z%HdK#R)B=;%5qwEbbR`V$X16PWIjjCR-|`ID`=fj1;^h>)aACH zf9r`A3Vi@RSS#9N#~OY7@?HUUF2bkf%cR-u@sobq1*yt`AcDS1xWxth<(coo3$Us}C1Z61-!9*l8K>4$f%`Flv8Bg#iREk_y_ zVaAI6xEe|7fQXw}4&V-1?diMq%oflN^qEs3b;vWh_1n2Kw=atMTCpm}je;iuK zk>hVdd-C_|JH^xp3GN#qRTG#5r0XYe9H?Di~g#K!ev3b|Hi`2fJEIMwtEBt5T5qxGuZGXHa4S0j(TnbbnK>76aYMSj`muFHxnAl}cdSwrv z9`c`3Dwh?i1-U^OKhT|6B-y&U<32}MmAd5zGtmp=HizX=ahR~X20yW4zR-R#>MAU% zKjU{4y#L@Xn@{=8oOctk|CN|`Mf`CH0D#vgxNVPZmjO&wdR$JKk{9~$8HSM3uNjzX zNTqbWwHUHZK%YuK;GZn8wLT`h|ZZ%;y?sJX4RW70==_H5 zMGCe3W9-TEsUsX}X52??3Pztw8U#A8UA#4+dB_a3*k%TJU zxK{YaEA{X5Rc~#rf;_Br*E%_O+MD&Se6^v8G;PI$Py+43XFTWYZ#5>(23Bwo8hLWt zm5nzr|8pZSQK5a0K7W3-muh@ft{?GPpl)Y1z%aazq9|RqfaNY@TyHXbC5RE%_SQVX z`BKcnnyqnKLSX{~_Rv=fu0KQ|OvZrR=KM~2JDB^!TI}geE+!U%0QO$dC!Y4a#(0nr zCQp#w#7gy{{8Q)fxZ1ft`-s5*sjUsW5E6(9SM-rB0<@kgD%SAJOOm znlycB?W%a$sXAoZr%3kvV~*BX57OUZ z#TEtAcSTq~ZMl`Qb#MH+$Z*EfOCN~yxZp?OJD$ipQJbQZgf$0oQTwwEFpq_R&)b zYdK{x{e;ydmRDzSL(=uuNDwN9MW`0JVjmXN@TB}`NJ%A)1G9Y$V3h^xq!R?GEd=6! z1Xir?ub*houOs_ojDs7)_s{21>X`LAJU5?qMyCw{$N`?Zd$kkrx?)H8E8wmZA+&U# z@dC7+Ak$Qwn76m0jyErHEy>b7v6bxm!Y|}A#470L5GJ=bDeWp*9Cy6t(Mo~ye;T-K zlWQeqEvmDvNpp8T3 zFit|S{+T}_TXG`UEkNeVUrTr}J^x_w*57hrID0glN*SsqBv*eT85&=fm>*5h5*CGE zJ2;Ie2aqOH?zS>Y28Ed*+idu*j6ygd~ZB}Pa3CL?em5&tvK8+Mb^K7~c)tQBaO z_RosVbcB_iMh9vNw%Vbt$#;HwisWq6IGFWZ@yc5Ez9RHe$=aH|r>B8zR&QQUkmgT3Drm9YybBWe(nK`-Pj+$ry08Ee;c-68c^QtYGcFnf*6vm65Y+{sv8h@|Pl6X8L{}ZxvYb@(`sb|zeDHiCX~bqF3EB?sS7<Lkn3CpRPIj17I6A=K65%3gww4c{x237fVs#E$>D!|hoCpK>i?Wa`W zrv=V*`hp4#sKW@vWidv}0groLfKtV2RsH%_Wo255y|wUHbvRoT3BVbn2a* zr^Q2sQx&{}5HmG5>>Pax3L|l>T|)^nY0ACkj|Pw;Yd+Sj$NMQlpL7aI#1EhDxiF;A;qJfZWfzj4 zP9AUmTdPAiqEOFH9BolowV&rXqaY9?w!|tuYuNdsWXdyx%faYHGyDVi=Yb*Ure{-S z$T5Lav^rNZ_m*<|6J^KTCfA|KGWz7$gCa0La4YqbZC@i9x<05?=%Gi2y2twDXvl7z z4GatOJ9e*EMkqrz|_(q0ChKmEiZ%ojy?x?&Qw&Tjteyu;j8e-!wVLYefd{-&OjGe4iB$7f!#un&^SiOK^D; z-b4=y!jQr+czu6Me-hbGtdo&4zt zlka!AlH=C9k}tD*DRHUpcSRxbEWIXT_e2yrcYFk^YBVCxvahDUw z*ySxYHP^g1!Lh0+4-BI(6DF$tW>>ljq9>n;b&p)XFKps!188*~>YjWBoNkAE-jkF4 z66T)eFhAY^^+o-9gyaY4-k6iymEMhcMCXyD#$FZu>*jeWb!hUWsy`{&cNbrnNGFGOs4^^pK9tsJDRG5A}K;1?rPPs?3BDC*N z8tsVSC5xM%Ham&Ui2i8?MTM?e_>0+dawOaYMx z8UrLn!!SMKn!g$Gg)?`J=2uIhXz+9--I<;+*{Unvt3I$<^5~W$x#k|Q@|h-I%FOK) ztGD0^XRLd5`((|@9_?Rc4AbG`@2lul2C-$@#`~%USLVlsTP#!IwiWI)PnkbVr37x- zcz(xqV)vq!3IR6>cHIei{!47pXx=XWW$V!DuG0zVXNgWTSGrp_-$T(CjI+^x*tD~y zk1F}jddMPwFG{&LC%C}-O2PVKu}nR{rHtmcS05i(o99s?VBdtA{BqpWn^%vIuuADm zPCqqb8*QjQV{Z^GB!x|EXL*^(Jp`td$roKa?hYB7A515s6SbdOA>R{aY9COWfGlXs zvfR%_Ev9Y}tq+?}>%xl7ffr0^2`?ULm-#TFgMRaxWEX2cyhAc^a;Cz9#4)|-1Vz5M z3Bg89^?&ybjMqj_`uYA1CpXXRcu~K$q;`n06&ZM6n(@=6JXkk#D)VP=l-t+UJ=Df=wf`j zyCCU9KLzp(ELGwimZ@)apD8ey(M9Mw4ZFn+)E}o> zjlK(lKMgFa-;F^c3BND2TITz)(&WA|5J+7ybPgf>#K6g2c`pQGswGD)4!mRUxy7pv z(K0L+1j01(UU;_K$+eS>bJ=&FaBZnp)pJQ3Y85CM)-0R-CigitP`g&7A15GlYX8aa z!X@}i@g}S+q~{q*cDX*`pT=|X7V`=R`2Q?i?fSx|@eJ25-@LhLWoBb~G1{Hh-t^+; zAtTrKwUS2@)|=((Ks#|%xE_(i*}2b~ifydjw2JHLWb%nqhOeWPiPRfHR5`xZ<{IDo zhxE`DSm%hrw}Y4sRdbV(S+u)5mfQIcbGepNOA{`?eNedYioyi9bxCgx>Q%y3C<&i0 zWAgSA1G_HCQ+IWFBA~8V{TG^sJ-2ES$$w;pOW*R=i&8*=P4_|O{=vH%(ttN$zSjrJ5Uk_=W z6Mx#wo2UQ-%g@B$xb}Qxh39=` z`5n%S|2;W9ce^f6&pXvd+^1@UWTxlec&Msnc(Z%_YdUJ#Yr+v4C$ECpOGV9TJW~## zd|zxwE+Kx^o_a)({{zkNEa!y^-0NcI{&VJUbHu6}-q!5%*DBab#cmJW1YZ(i>=G$a z=3_rn8PZG&v{QPV`)x4})fmhbANPN5c&uOaQTv(g3l;wHe&aXym##q~B7E2Zc$kJ| z=m=HYy6n*+_$UZwgd`#8N?xz5&FpxV3PIXFreCKQy-W`B?a#bv%2^{NZ}vliP#@f6 zKBdgJ_%Z-LGgu^fjg2LS=H+G$5C)FV#p^~=ZmnUka_D?SdKroaE`+V2b znddu8xoqb1XY-1XX|T23uOJz5AgP1$9+NGCt#r{qFq3`MC@p5orb3hddj~;m?EgYb zNJ9wO?^K$q8f2p@#1T=Bw-s| z=AZw9b1C|3k6AZhn%SydOG)v|u3bs2(xKJ8Np^##S&a{^4A;Gi$of;heaSzwQ)sP~ zqjgY=Zz4Uv{Q+x{%wSNw`X5yjfEx%dG4^QbJoloJq(k)PqjrpMoQxzfgy3V8W3bdaFDKJWOVy`5MPEFE zdXr|8k!S2%}7$~~yg8x)06Q)SnM-Wlfn zzd5`3KgM0p?q)H>lr;d~{(EAAaZkbvBz>Qa2hz0|_rrtSz?*bQm1^fXnYg82v@*G*A?jSi_94g+ z2yTGCM6G3)TgE_?x7g7XAjSZNUCyK3x-IyBQ-Jt#_7Mlh35c@Kb|T@!C}!CEgB7Q1 z=v16o46BlHG$jPK49pVJ;-XxkXl~Y5VdK@5!a3$a&c3ZqcIM{Gft;-bmr=rzBbNM> zK%Dr4QE@TLm(^%#i0lQUVZ{b~fXTSvUD2+W3ZNa`_32_ zZ#~#JP)Z>rXJztpq4H96#B0^@+W|#G03D2@v{2A0$${5umB{67#p+&o&jZJD_D>=V zwhSn#dnq_!+ofQ=EZ5)%WI5r-J@GO!12H-UYe~yEH#kjst=>qbLE#~<)o`os6Xql{ zH;n)o5YB$cqFRI%-`u76xX-9R1~mKgGH&n7g9<5UPgh%@k}Q3| zn#k;3W2FMhdGtPmk7=`Htkw=^{<5r5Gkm#Jew7!MTKWLdrIgn}eW_?w7f86@?|0>7 z7OZ7ws6b$;bHw{9Y8AMBL=}zE#8Ao)D`JmSO=13BZzD;5?3ta*8K)8m-NQ5j{x_fW zNq-emNME%)l}z&@d)1G2+gqgbQ&HL$DV3)8oKvgec%{f&&%6jnxw+x~(O8Y0B1&0{}b|qqLzCkesp@-^)iyjHv8d-Xti%}#$xJ&bG<2FU| z=*oF$F;|cTzNy=WQIX^ee*cP})H*JtkNN*H(5lVeH75GEfi@L7INOFFK@j_{J~@fp z#a+i!Tx@0Fy*F~NEy%fhDu>U5HV3i#u<1_PPayT zQSKe!E<4opSNS||A_(Oa5S`z>m(uEcOxu*JEg-$dD;n;oXUoHXy~dq6)9nV*3St}^ zz>Ovd4VaP6u0-0tCF0Cb4!&?xLh-c9!u4!x_<6wZ^98sT1fjYrvx*|Z&->w3YFf)c zYc!oN{S*jmkRvwP+M#Y*UOMQz^`hIrhEDIJi1MM|?UyE$4`SSnnN#whjK+pu=0e|$ zti@WcWwhkQOQckbI$@JjB<4KRW_Y%zw;b$rdH$~PWN2ES9Gpm$%~?R583IT_8t!9n z<~1KL*4M_T2EoePgF3($xvQMzeVHEI+HMe~>el;4)Wyi*+VC^Vzki6+H0WLGpdNJ{ zSLUr}KeldLbtQ$M2Obig0F=1im8^Rn`pU05K!dH$;ZuI76kV?`@SqqGt8t`;8v6C& zSWf~_2dz_rneq3OVUEw*u8>ye2%;^kps2KU6&I36?~_z^E~W&So`M@>+aE59(Vf`l zz3H4FLZKBRaL5c4n%SwUlbRk&zc%&$nOr31%gWQ5R)SPT5Q)yRM-uw459ZiWg5gI$4=g}0thTj%!AyIzOCdnacXC6`wq zXj`8zrcQ!5M+7gSc3eZg8(9Sf=ZeHYKV8$a5X*5+OT2bW^AcXxyM`58<6)xG+Dtk< zY$M_`7peWHYcNAF@&QFN6t9aTb8SwcjfP@SC%?GGWQ3VD^|8k#u*TDIkI*DX9yi$1 zGy8g9DH(8=>>m>j+?8jS5a{;d+opYN>Q%}Bl+%HqZ|g4I0;M!}8e_LW+KjlEgc#)% zMSacY-d)QIc*M(5#NfLm0s&sv|JZat`55wY+QjR+l4Xlf!MHHIGLuIwHZz`@sY%hp z*E-dQzQncF&s>(?Yc8E2qV7AnpQ)ko-x8_9p-N&D<}U*h$x{0lU%5E;7a!UK_Jqrl z17sd&Ix$S}Hi5(oH+aog(b!x7@wwVatnh_ zRWNh2DS-x2uQ86H4yxZcM9BZ3OXI!X>od+4=5lJ!Jr%?XfM$W2A##ltsHR}5IL@N{ z6?+mbS==stdw;hwtd;x@<3)d2HV*58uzri6Z+F5L7*cVu51dRU-e~_Bojr#gt>Bw- zQwcvKAz{3n;P;VkFF98@J+T@%O}McVW?C1;W(n#WqB4KA;qGk~iDKNIxP#rR8z#Ci ziCQ<;{U{LE47;k7CrE5;(=n!cJzqV!BlKH@`}veOL4RY(>mnTBYLz~WbQxP#O$|5J z^{6gD;BW(ubG^4EdL$gc`(3ylS6TJufB(qMKfRw{3lh^S>lr06&iMjW4d0=K^2IM+fh)nmMMfGI+WC&key*wxI}x0PC1r|+sI}1 z_vtI=G_w%?V@qz&wcXuIIR6vE;~=RZswHKJ={ODVM|4`I*3&i#h=vzx#p%>6%bMRn zVtzN#t9(6n5CIm>%5TVAH^S566-z~Z*ct!owkU{8sc#|TK{#k*q@&ME6 z0`$nGzE0Qaq%@r4mBng#z-#a}>DRfD% znmLM3tPKxA)_uhbLmikuIfRxGEjHcRX_~n6fbsh4@1~$Ro{pq)He7>N$+wVJ`A@RW z)sBo~;&Bv6{o(n^83m@|e5bCG6MEtZ=W3jQAJw@LIG?nVA3pZ4B-&}t>nde6>q;`a zT@_pT!@+~ob*R7SNj>E#C{sVQzD^EZG<8HMJQAHE#cv>AQ5VTbu&t zcSU$ez^j+Ku3C4yTHP)USHY%B$C%t4ib@>}lMeIx1LQ~f{-1r>Siyt#B0p=p+tdq1 zqa|3y#uQ7_Oxo3&mdO}x{yXx8#3@{Bf_aTT5It+^7NWvyE2p%$1xT>C${f5j?R-1j zob*V2m5$6oO4Idl0K$KM@{@i|MMzFpqg7e8aQ)#v`6Bd;02 zP)V6a`u3-V<)dqZYoRUS$s&6%TWzv}(NL=Z4(|N|cZDB~X~s?8m}Bms=kemBFIN0$ zy<;%E7n(Y-HFK-}vfo!VP`ccM7b_1_58XNp1$79eWq@4_FCV-XcrJ$4R6kyEfeJ0c z>lXwZm}2<}OjCrT)io1b;{!j43`FVQBA!Y38flWdG3y{8x0Q7H+I9N>1o%^5y;z>5 zB4P{vlG1LG$BQ<=orboy%MLA_t|icW2cZIiKHJ_#3XVrH=KFhY9#r z+`pHAu;uR+Wc~{X{^GYjM0e~&Cm*NKKDrR3%J82I_3cZm;hKML@Tx@vD)pO1WVfs^CxuE|+ zz#_;c!7Z_R{!g2ChfV72tl7$_S7f&3L9hI&(%le+OReBMHZ4y~L=igl1HYL>3Z+Vh z3yrSCLl3Ic$0;@3*IWdaPz-tG#QkGFf7yB?=q&xmb3I5(pGlAo+T<~!H~9pq!St*C zAcNZ#7O(6acW36xZ;4`eQAFpB5coU{`|e~TbQzbQe0>HZ(K+LLR=He|#C(00J;G>g zfa|02DGoAk@-*_mH|-+yD_R|NBiNIxO+y(!H(Kaj+O4?8Eo6zPvnfb%e=q_1eZu|)Vk4Xs++a@2TDp@prfcV3tLdg% zH$iXe)NW!VS7IRHEiAWMc&y>=12WsuId#68Ex!qjQ*dl`3AYTX(kb}>>!>yBAO7w> zthK(oH-J_0%a8w~#GZ}4LFvba2tHWkTCoQ7DHsh%xb%A4P3SJ&H|mr>rHb{cZT{O4 zTz$QTQf9{%lUgya)jfxsMR_@h70qZVug&}2{j#kn^MG^yB4}7sp%bWA+LC+A$`J@W zhe7U+Y$}${19~_dG2&dfXJgcerzV@rl83$bcE1K}B&2RJ!ugk2?ai?K%7T@AUwXfI z&7#)wqtVj4nC(WEpJP)Zr^rWI>HSupbdVJn&QIM}cERrKr~0zeUFq+%X(6NY8wYN- zaDLPuQ3pQNf^EN4-XoJyUD8K21F0tmw%t1{*P{$D%YU4tc|tV>ne!cj+#Rt3kNmQ- zZGOKjrB$%|w13kqT0W zuAz<8%9gyCrBADmVGdVf1=8r!+4&c+@ql+M?EE|(HD`OM%;kQ9jcUiigBp*&_{ z5*Y1hKeg4D{+0{E9@_is)ql0d*Dx6tsoihpVv8a>o;NBZ$D-ltH?A$9)w;vV6QpV` z7YkN+5?S8e>rOIBNc^Y2ixi%FH8w^2A(rB0#Sjd30Sa1#l2#`Ju7AAC>B0s_=Y3uCk8Xc5mvcwfk zo3Kh6i!;MMxfFeY9L+rL+It#R{3JH;lV44z(V~sKaz>{gq7k0gjMD!U8qXd{giszh zXVwy(@o5vP%(MiA#^+x1rGkB9fhNk3}NpHnr-5|+#ks_})$LCp6X^@uSf0<(Zj`QtpF6r%@@%!Jj@s z527XUXn-z6X*asq1q$bd$f0PX+fEj0=NRQaMtl^DZpi7l)UaE3qyL$((6WT+xz~ks zg|ALa2LM4uXm3ft=Da=!xjXN%G_{#Wx_8h%91j=6e7YE0>$XA(&ku_72w!f?x=F@w zRynl5!GjidtP!)=JaFN%;+XXsog+NMv{I;}@0HI054%A^BFKa}vT@LGY9=YLK$}uxBVpTb4h9MOws$7AvY2>wmH7Y zeK>Nw2``%6>#xW!85na&d)4u>@rhgtUuD$mp#H%lq=T6BjLes+Q zP%DNmg#3^&RMFr&(|K=uuKJkbeSf@KaIw&ABzz=+uZWEF$$jkmiwRmTd}M+|ToVfk zZ9)R}^@Ij+m$4(e&%>1q3}2wCEf%`95ynTW3;Vmb=l&nQz62WT_itNC{jyfbPN@_^ z_AOK@*|V>MlI;7w3`$aD$&#Jy3`UkQc2V|y9s7{1W8a1uGv0gt&-0%1p6CDk-_GHb z%JQB2{w&w$`dk;iIV07+4UpL4<*y7Vk{Ak>8FBatEZtvrogyif%(l4k_3@SUD?bMl z$pXl}4;}PpqU}Bm+0kLKq*ePq;m(S(b8KxF*g4l9H4t{yk25Zjdemg<>xJjrDD2!R z?O;djC7$j7+L31_dkJ!~_(|W3&ZrAyV2}vfzuSM)f83w7_x+_caHCa@+{|xSblzp4 z0{2UVJzEF+>k&ouksesJB+sr{5f`UWlhEY|y64vR23KbL^&3|S zWELbz8lXx3uLFTMH~t7#@G`rN&Ng0wU_`kFr1waH4?qnOJI6O(KrWQd{P#8(s^V7M zlfjo$Jh^Qdsh>Vr6jT+vk7&fcApi!`~&_(YKC9m&3_^TFow-K;t1-_4J|&tvgR9^O#KIsBkp+5 zp8%agAb9afkyH9DSW(JZ&I9r08tnTV9{iuvx82j6(Z%bonydsDy1I&W$x6Umqqiwq z5AM5Ig2y}tdWj?kr3VW+;{&-HohO{vMI|x$SXa(V2U`(m|ej~C9h zPK|u9S|e>vLW}=iaeJr`yw5(yCcGm0;NeeR0)TyK{7fbd9@Bb81Do;z`R2gED1ihr z;dNr!$y4bxOr4AXNYKDz3RibNUReAba0V$rAD$2Hf4l2`+JAke!H#bK=6_8;7D-TD za07xO$=w8XTn0o^sQmdpG>pKx)>IS1B7-gNfb8IqnBnVj>>{Xsu~KI`pO$4td>`yI z2p4wjhYGEfqr|1Zbo7EPf?gh~zm%>6*6a$Y#r~VCOcbr6q$R}qv+@i7s_Tm}JH!&9 z2WofXOKk@~!^+A46ghMDN2G!HtFN%3%BdP?sQ}e%l9bJ}gmUa`Wh(~w9XveULi`i4 z->{s&eNr;=Op%?Qel;FmXv3t(MOIj{;Zi%hdtx&ne*z@5Eq>Q94|VKFcI-2!UX9vr z_Dw(?!p&B+a^J1HlidLNxH<-rzy}rrRRNlYU|s*`kWj=+*Sg1@-WxsKK=0=x;J*eG z)}}Y((C-{D`DOjjf#d_(JXody6PgdM<^XiLX$~APv@oMjL)vT*S>O9Um47`qCpz+@ z70op40^a=x+5VxI`)KpL|A5DFUnAKgrjMl;t}>_1&pv%UhPi)S^CBePLi+mci#Kh9 zL*#>|+kQ`w7vQc1mx1+4FQ%3?uwa}v{7cwp2W&LF>Qamgm>UrO^kd8cK(-NsoR6tz zZHuvBOZOi6?h(r6Gy@8@8vm=L0*-i$_uv8>4xazmt=|cL_5Y4U03Qkn7}1k~U5{Fw zo!@IHYrOI1tH&Jc?8iv!7$r@=i_sfK!TuiZ?U<_%8uvvPWRF36vJ8)&O6LBnR}E?~wha zh7k8Em<(Bcq}t>Fx`}`K)d$>H*Jpq%fmB|clG10sdfDx+$fE&>!RULCgUqm^Knn`3 zSxw^m{SffixaQ>l_S_rFqA^LOhB_zc?>ELaFOdz3EH~_pwFUeMugJNDs_DUi+fQ>(X zFA}5fU*`u>CtqS{sdN_dQ{85k6IJhg((h@wc%d)tTQFc~bMA}jRsAZgETg*}9eLdd zlf}qh>ZO7oMq!o0qC0l4`w)MIt$;=CSyiwX!s)ljN3wakucP*32e9aYY6a-0Z>+H@ z%!I(7-oCKVVqh>&(!r^%z;Hs&kiUKyIglDqEh}o>pB^2)>>#1|W~O@D(|<);&!2!f zuF5}@8pLw22pZFM`-#6Pkkth^%m*Mx%;TrAMfXwPKPWcl!H;uuZPlQ#QU_Xm4t!}O3}nM+h1MN%Wd@#Vs#g}?_F69D1ELtUTCPW}LdJV0X|1QRqe zu)Q2Hhiod;t3jW;Av1;tP7tl2_OhN^9ep4v9hNcnT2WCs6;*3$U{*XLlVB-3QM%(2 zxg!H?5W!f=R{#lQD<2l`-j?S#T{QqyN54L_0mw=-STz6c@-Gw8&N11KZG!s`DH+IpKT;gQv$zQd*ebBwlu8ia6oFPNp&hJKX_(nq=H+hDuz@)) zjlq8&=F%h(2?>(_VVKMPmDTfYfPP7f?1>{a=aXN?&%#)?{~{MEU(>K2Ht1jK{&4vy z*e4W}0ZdotImn{8wmE{!fW*I1z5Bdk$M^6=`<;0wxe%4`%hc3lJRvLq^)}nDrTpz* zd(0Csuxqr?ig`!fImskX;o;HAc8l8(1D01>E^9j6|JXG+HpgkCbHhQi^Yc?}Cyz3y z(50bUY2@#4*Phl?|CDwfs1Rf#zxt2=gOD#cRYfB6BJUT$Rt#{m=xBTX6GBWR9tFpPYaYz^J^hJ=IQ|&ZQ3?z%vU= zur4Zm)GG&ee848@TKtP(u#e5i7}F%1R3Uj;Za_V?#yw{Y#Nc-ejRTQdD-ZScE{4+u zgI%HYVV;mSj69%lkW^v9qogjtGLk>ZW*PFHSU2D%_Jqnrmd z>uEuFLx5#qpi}$uz_-hzDY{f2xfEf0sh1X=Jm%o;SB9wmRp8|_l7CHC&-0%(6K9Ul zM?xr_fbTPl_}7fGrkZ3iOL^(+g70sgPusfHQ5|>#G3NCI<+PJ=^^pnm?g8!oedl*8 zfGETZ<6xX7wpo3kDkDJYWGJKQT0_bNlssV2-YSxN4QLYdEr!fo!rF;%s)Ca|=`{n3xN2QH~y_Z_GS7Q3YmcXu?1OkfY&uklh)gD1ok% zIdIBAfE4%JoW`41{At-&P{H`aU8BkkK!cY^cvK)-+huDlX1ueM_^h!S8vur{MbCkH zM~tA1hzHcoB-%6f!&j(m^lO?onTuU%pe2jpxS8AbFptQ3PO?IxoDj^}hC- z1n2*Dwck_~8&#QBUwuFm_m0ySG*ZJ$@Bi~auw@(@bXEC3v{Sd=&pEU$htoC$W-(GY z{&~N0mF5zKlm<&dDiW4y1p7$y=3^ zev@Is%;CxcKrhqZ->Vaif9Hd@|K7aP@_@u~O9*;o`f>y~&H^v;P>;Q&xyS1MDFKex z|8~RiFA9YJ0GUcaUYDWxn&t(qE?iVy>Oa$Ule%z~77WFG0%BsCYBXnRF9cK3KK=kU zJxQ96s5(ghN4`#*ARwV^=p&Gz{J?Rn^JbBfCXrTK=5+w|`&PXtb<3|S3oC%;W(uF! z6B!Zd-xzsd3bL@Tu+4-s2WBDIrXLK-g6H`n7MMMx?{Sj>aS=1DFpy1=(-MN6f+fea z;jt4^y+M=`DZq~^S&;|l4@o;)Bz3Fz3%-q`*e1RX8-XSXi*tI*HUC4I zV0z#x2aJV)2g40M=YgNoa(K?qPu~M4RE?9!I%flyN8ay#Kzs>@q8e-XEw&ZUOdmv0 z6Lyhy^9^9ri{$_I@04qhx=3)kkE$71wT^%_xobHdzCxAVQQm1+NniG(-Ru?~7{KAGp?nVin0wMxUewuXG zOW7~h-p$a9`VlJ^9|4`JA0> zx4{YQdjOos3_E$aY-=bm$0VO8!rLyWOg{`9@5v%;neX?1BH5>C(P9LFnU|zZM-<(G zroXJM_Wg>=TOSU4(#YEB4gPny%v!0;cHhLSUaqQIA`pGI_V7FpG-UnJ@JI2J|RbS3$rmBZ`O0!W9 z{WfKPLZj{u6&)e|!94KMkREiL1+?`T)4HE*Jz`*)Y1r058YmtVtgJr$6-x8-hF+DzM;%x~zcLOXAOjd3_<^~xa80Lg^ zKtcHgR+Ef%PZU``A)7A(N(6NXN6ZKl;E#vbHF1QboPV^%I7cpX5jbpkxu(SQ14&Lm z+q0|PZ~IsA6SMuK$m{Ip&U83-;EvJxH^%IQom>PXt0r|Y*%mmq z*}-MDG6JmwHmDl-GdqG^0d@>n(#m;$>R$2sDO}^_^CV`(UxBzI^J3;>tCzkx`tu>} ziaYREP)lds_G~`>Nese*Y5eztnnp+_7@Bdq(J6FUW{08>6apb$ro!0EWbCfYITRm@ zV`PNKp9zLZynH(73APUuGsEi>d8B4Sz&N)4O!CL#4AAQ$$*z*xvgsxw;AIAF=Hk;Y zN?G83GbXoUz=W_McyY%!e#7xE=s=%X*0wtm0UiQ$jQ6uKu5EIQHxfJ}6-;0+2C8hb1-+_Z$jVU1-4IgK;=#B@ciiKMb07F5n3+KEMs;Px_uvDkr6O! zuqD+l-D?SUQhsJ>3VoVFmc`pqY@MLA3lNA~_Io57qV)Eu+4H}2U&>SsfwObez1Rt4 z1o|?MWFewtU@Obi8*=^mmmg)8gxzSo>(5{8qC#cYoAI+|iQBb)@-moZt(O$$c5Je0 z$H!+!KVKh}>gKb@fGmwnG9%$UP?`G5Na7|bBO4KHK0potO8*($GMe9Q|u zJ?qt2@0nPtm6ul~MG;=F`r*SB>}R$1Ife7&?>aB=BzI!>4Yqod-80Kq=KTA+9+HuM zdC2*WNi;%#n3`K9>pAAkpOTO@?xHr;sFpIxSrGJ+;TzlZAu+w3X-R{moU#;=kLf49% zgMRq=C+bdz^j`;m3|h@Ch#)yp;OU*?6>+S^C9){K89!}2fuT64oBhOR{0)jpj-)KaP!`lYL zGZpOmJZa3FbD|&T2yTwN!OQVSt%jfj_`@2Bvacx*#u}^zwf8z}-eN~$J9WYZQ7NU} z6Dep1d~BMEfZtoy;v?b1@ho_Ar8kr9%AYT9r&l%vhO3|1-nHCd$&MG^wXTX#rqt0b zi=&7XMJ{1}PAx|eREFmgjA$s8I&O#YiU?N?RnyihauJtJHjTb6)d~3OrY#e!zp1?o z6xb?8RNt2>DS?FcRU7TD_g`EwQ)g`C2-H#udEKd+F}-|rqGd3@xGZ$J;bfW9sPrzC z9@-PN^VT?%foauWLqY##yk3@Ors2wz1~YTsXB4f&@>Pz3qP8T}=wvIwBHKg)=UhxN zq<|s8xp#Shj@RG&tG)k$6?DmdPdW$kG&)ZDM_x82wrbTpRnMp+I}_+pVDyVF|tDA&Ec==-fO%w>WAd=f8vgERW+X&pMPlE^IoxZtGSWlLkg>@ z(%-xU-;(VvN!oENr<~@;#g*DSO8kezv*z`LbSbuuqqI}bOBZo=tsg!}OrtKZq@d3^ z*Y3ct(zIV1Z>l$Nma8ghIm!IBHQ3@Q?YC7el+Yxt2?}aTYOHhlwzL9EF^Fu++KyJ~ zW;FzUM}mktZGFyUM*@?s?Pk6IA+}h4temG`(Vr&r>q0!$>a(-ClBw>x>y~ze^iW0fI%AM1aR5#Bv;-U#M{Sk2P{VT}i+5l{+6?^MeVY<96uU`}*f~)slg3 zFP#+{O4`_Mk<#(iya1cWnKWV}@;g3G4=3^CU}!)wT=!4`LZFE23iXH<%rLWRf}U)1 z*PyDm(6IM*ZP4kr^`HZOi9xJ`qWrIPZ(7vJ3f9B18NVgpc(fy;$Dl8Hk^~zzF7>fe z3UQUw6lJFAo-`;qBRGWG@!hwcaTY8cGoceOXyIK1DjN^fZhk%Ibm&ZZFw&{2t$Ac% zioyABU~xSbES3vwg8feB>Q8 zP&me_9m;P1b)P;xxd}ZzW|h{cZxY2ZLpmswr}o-jITH3t27`l+k=2W97lSSxz>e(Ccn4p?0LDmMiPlw^%UVn^5i~!8kTFb-!rd8ln$K? z{d@^FEh|66H_7dz)|dR7^aG7;7te95c*rCmDj#JMeACSuEVU{fXRor-=YBYMx2(^g zu}LRvoXft-(Fkq?9;o4DeZ&lrJ-t7!BQ2WKq&RWQI5!>rMogB;9w`leXvyLmIWwNq zgDlF{jLmluO7K*dzOLN~ooJv6a7T=}?4nFc18Zt{s`T7AP|*|`*$z@LvTXmY8oS{Y_D!H?cp#`rMb?V}rW9>4Ha4lJt(|F2 zX}=L}mwiUMZok-~iYf}*^&TFFXR41DN*9UX#0Ybw`-UC9WsF@2gq(%#Cx~%xeUZ2u zekt>wTJtJ8)pM+jNBrK0EoE{Z?C6i=uBpb2Pq)`Pqt45DPbb#>y}LMi*6{R%{p|iS z=l!8Q11}r!xkQ+0!a?AD_14!Nq6!oGUB8&>!7&$u;rne8W0r5Lz?>f(lb;!v*9`-& zmmNU~FYcluq@F1`BdS@$IJF0L5+WThMND*2T}hM8pOql9vhY^cz8YxF>3P;O$)Byg zZl!_#$oBrtvcF#)>FZl;261#){(Cd#qT4LK?HFH&pz^supIV1+anByCoW4n$CKN<% zW=+HcBld4SYb8Qa(AH!AOJe(-AI~ciAcPmgNas!M?n0?DrpI5l*@|2Zj!?@3w92rv zxhDL6LfzKxJ|pw^4>CUxtK|IxN=2%d>z&wyW%U}v+F+F;)!F}4)oTcd_z}1YGInKB zRP7ImImP{TKM=Cx&>xzY$#dc#iom;`plUCv9<0%$NueH7Eln4t^XB!Zjz3Zc1=o2S z54o85h*jD9Ud4*$XV$94;qmR*$kB;fLe<&`Zs7Fe$t4E%tAjSg!uzoCv%n^8t6Dwy z^mFrZgYvG0$Rnu5rdA`iN6(F2NMn_0zL@KwsA^Hi08{_7%*&TWaIB`#AaMO=Pfpvy z)gFq5y>S{q;iKUo_CiATgTH;LcMAp)dmy1e-p`>lKwX}7_(jXqhTJnhe3lj5=eF(m zS)>~WM$?%ObbI!vT^ZydLpQ=^i;$eA`6k4`ciPXz1A@RmA3t?2({HHw} zx4L}ab@jdPFKeuxKJM!$1c>cX)m%qh1Rs)bDz^X%uJfN5tIwA+NTJeX^+giZUd+2(y!5F{)YJ&&EHOwDHfj@Y`y^J=rWaLl@;Pl&=S=4B_az-TH7l-o+4#ky&4wC*0&7 zyX2M!Z6k-iQ>NO*D}g^=R&p`td`--)gCn}BU{$Y(=b-pyCus+bw!_ahF7hMsNx2Uo zE~pm*{;u`6Z5Que0(HsIl0y2#hHu#H^2_Dw07v3R%zZum*`u;#P!NeKvvL-PpqF06 zN(6d53Mn|${`1o&`T#oT8S>^pc)d~qdbdD_PwwqFDPhhEL9z=oGMb5L@%mQwXqZFt`v6AfLSVlEZ?_)$44z=i>a=f2QQbk|MheIJD1rK6Enp?8&0tUwna| z%*MO>Me@u(c%g*7UJKv1>lVOmba|3WBnl;9Qtw{4emoI>`;Y~tm?N}VO!}_03HIY@ z$=ENtubwAkaIdzz9X8FOxo$>=;ydsny3MtY#hV({jqk3!Eb5KnSo)U z4AgQgn!aUl;w-}<>$MSB(?Hv54{m+lTY9SL9wb&b$tcU($@qKAh(AAgIHw^5UBBKE zxt^%P8;gG*2Ac;*!_O9Vlh4ZqZO$b}Hp7~a;Jwj^ zo5rz4u{jCwqjOn25W;kqdek{D1|Zj~!ZEEl-J~{RtqC7#*QB-Uq)?`erqT8ceHMn| z8d;#}U}cXJtaGmVqrpb^@YQJodDZ$M1ip)-*{56G`DjX$nUK9|mQ`W2$tR z_9Grsxkr~!OJd9zdy%;si`kt5TT(WrOjp&xMwNsUIlRf)6G&0GA9GStVhz-vrB`?{+M$t?6-FB%)p#sGZXpgr zH?7Ns1~}Z)Gzvhr|#sk&HqtGk&)R@ zom&5Uc_n47QJ_<=dfzy(L`7Tz8?H;9o}e>4o^8CzU%A@)+kz%yZ+oZhvMg-gB2J#@ z1Ap6}R<^eo8q9S6#mIXKTguD3I&a(=n#wN|WDqHfCEW#ylQrDVM% zBlDzD8;832NJ+|FKNXXvfz37(3TYF!mh@Xyr%N2h#64PY&a+-sYBMA=bAvV{d7857 zY}#Sck=6hYdP&{p&!zl4atpwv`8Zmg#wkMmvcY=NvHl^jJs==J*>*?yB&_EERm;WS zp!DOsqPmRuv$JEuHUfrr_dK~p@A1a}9mc=LS+APtwRD#?P%GT;AR@lyq83KkFOHlCfH&3 zO0=f(O|3TX$B(#*wDdMu=K{zeMrl#Xz)>&vpd#$<>gwFUaNB)f1-!IdO z+~i_J*tw|(XTi6Nr0TWL8jjo}YwkUh3v2Hj_?y$6ZO1W}RD96q8@o=WK$kK;eVFv^ z_?Y#OP++~~2Wst%e#Q&LCdqM{l4^b!nY3>K5Us-VN6iG|~HJiAzxL@$*`s44CSdl7IcU4dh-vYi z)4A|<7Vc&;^D$k+_@Ej!+f+BbZzwn6&HkV0f~BVoZ~L@16#_MGDjS&3nqR$o&T0GB z%}xUf@uQ1KQu(akP^Ys?;0Cp+Mmg^SCAD}_bvJC-@-2s{ziwzUBmAg|C&%doAF9I6 zusJTzMDVZl5|VXm-z;EyO2o`Wvq2l`GnKqAd`KB}ev(ZaXkb2ZI#_4P>gozp!a1qz zbxN^7sa4Qv1xPM4_-OeqEBG zQem`??r0lu8iVI0v-&-b1q;tsr3hYyQZ@f?T&|1-&Bw$c+;X!C+vc{%W%BHUMy0!J zaSDNV3x4s|k7WsE%zYno-W>mK+sGKlo*!(Xfb#WgAAg8iSscF90hHz#*jc{Un&F_ivyab-AB57_$54RPVc<)OTjuA9# z{DQy8Uqr3c-AAej%@Pn>fv(lr{$alBww+GytMeaCQAgq^>K#M0fbIQe^7O|5#;e1G zPU7Yz(GpVV!s6;8_|)_gkbk#=7DXBfoo;~UuwDPK zOeIx8_OHIf#mxMUC3wKW!^YDfbPc?PU7{eolAo>W@;~C~yEWa}>ldd5(luGAFGBEm zUP@4}PI+UyUzIg)xlE+P?!#;cy?(dx*5SN$stCm%C`V#ngZy1w^X|Y1V>&~ALfK~Q z$U^+DE#0Yh(>%vs$D?v>!f`GV*tD%SNeTkM@I)VPfQrT%!%C(a4?cMb$?ebVd-O%t zOYq$6+*iPl6&n11qEt4jx`^pt29DRWj;L;6b^B2{%_8MVFoa-t@Vo zwq7KgnOW)`Sl8+)=PXz>wmdK^3(pWx2r{3=VW|%LH4SQ(vDHBdf(5^dFlSO@p7%k{ zEdZ_Nh{4&+hMxoY>Q)YY3OSk~L*FB|*h;nBwGqdYaj5S6>$EgMr;8_)bXb&gQ2n?# z9e~0pC}^U$nM?;wfLvs4wIjcci~qOT$&IoX{k|?~KrOCnXPXVOBCF)ZAjT|e6P}gO zfyn<-BL$KbRn-;&8oQo}ekZO3DI*jGs6U6CV7kK9r0RuJt7Vo)$0j^{6ydbaHbhgN zSC4X+2`k(u$G+PE%O9#cJqz{^_lJ^t@OFpO{-#KZXUF!d6kRLVWCjI~;*G_`Z+;S& zH*3W`@wi(ebm8BsfQ;o6bH0E3DA}xvRXW7nIyFImi1_S;%o$cLjd;*gLa@3HiDk%s zsleqNHp&|bE^p5_NI|MIQRNbCe2tT2YN<%7=5oU7>LbE-P@>&(?;E501q@xZHbnDW znb*7Xm5dBqWwPc;rO|=S`Hy$yP0|%X*oQj;NSkR^!I)Iw;hS;ep7lX5_Md&2uD?=t z7qNt&F>BC^sgwq2MG?RpFQV~(B$WydGZ)b&qrB|fNIrM6|PC--!ShZ zE*Ppex&APLRu@$BX|HpcqK`_$zSV|@Fj?!_EMe+X0@U^jrDpe#RDZcf4_v!nd5cZJ zLA1%OtN1(~S@q zMzV+@5}g9v>WYu|*3xJ@Sdb*#b-2Qpb~>3L)-Lo@34Mm4@FToy;~OY*s<0^6J!^2r z^?2k(Z4SL|t|U6^Lbr}PcqXGev_VVK+AAmUu)r)GRF{txAQja zKEyWlHW#{D7AeM@$CjCv4>C>3X_W{-D-7DdweNodT`g#_zkjX7i%psX00{2C5dhtp zc3zS)yCQ*o^w~|bRzh~{xBVr%c8*&IeL+YGzaYzYXj$|5Cu%8Y^<%K&BB467Zw6x< zt~2!STM3@12qp{ypSZf3fje#5L!`S+aHQrcRJ z{E&rogv6ESuCAlihmOjIA;^{qAyjpBu#xegjaCXQv@RCd&=m!oKvCRdg=J~d$Sn8#Mgrd5c zY!Jzx9a4`CQpJhk_5wkwi5XebRd4GW@Kas%afNap%!e7+fmsC(gqdzle*1F5PhL6k z^&w)*#~%Fl0$@jBV`r4S7}EMcX&S9{Q8)B*v{sEP=8l))zHr<&QUS@yqL$^isz6i zr21>=h11k97`QIb!?M1}49k@BQ1RWVcXg{xLwokGN}72s%T`hIIPcamYwzBM9r)Mv zAhT?VpuiWC^^|%kCV6$N_LZ3-M-~90nd%ro{fN-6IyiCcoP zU+7xMDoy(40;H6y(LeP1L&W2-@uiyn0nGxn2xI#2Lep{hiX8W)A2W>`=E!^DU!4wxzzXW7k>IO7c=PqhY4AG`>trW8X~d|;rt zNb#C}34pLcqqZ~=88ZdKhFz(jJ5nOZ>*aFn6wh?~?J(QXJZP$1MqLl;Zw?77oWr-N zY(kO+l&5w*KY|w8>3Bp8B~BPM3FEEvPPz`tQ~S9!kH)Z$^;oV*txVU;d~)M1?5(Is z*W^s1M+3a6uG!4jDzOR22T_XBErj>OfB(J`$$YgKNJztagtl)1I7LOn7fOAx>-CBA zJp3-_AFgX^TD-bSmY4q#vtEqHR8N28laP1;K3!hk-H088u9V@rPg}^qGZ0fqbq*fI zK2{xS+dZdO*!_p~gyNWwTmA)shsO>iEXG9Eo60a=g}hHLFM7)K{ICBxtg80QoQ^|M zx=FNw`-ja;$wK5DtxrpOOBz2Exi9gea*HEx*B~9TfCe#uCS}~DK9Te5?UQ2`agYs5 zw>mlo3VjE9hclLxs@z@kqd4w;t*v?PIi@e&7?5)98q0S)x=szVvW;3tVhcfj9P$Da zN13zbtdvti%bsIFwr*HAW~ziNGn$RUaNuJBl}UlKjFAO$CB}v(c_`rBT+2wDuAZIN zR?9VMj>w>z^S=n<8?CqdK>yXcH~^T|{x-$jd0+>s?yRq;YB{|T%uVDImL!PD(J%fP zKk)qpwENDP3C-Do>GiCTdG?(F#}VAgr8lJr@u|1R&v;Yz=cOo=F4&-9CHmWUtvnhy z(}MEAxKU5GNyDJDc9;7LIKiZ@f;19A;%NR6UOPH7%>@S#Q;$93wYPp%xf!R_ZVRUU z;;kWVALZ^eX?_Si*fNFgA|t07#dN%i7r#hA+Ybk4%++)Q&@Yhq^ngVRnzUgZFyV#E z@mH`yqk(WRbbFzkRTg>Jksa0kcO-xlUQ#l0dU6Re4aGmVY5y`;I;Euj3C)6xOrJIf z5LlsD$IRbF(wXgDf{Jfezj9hyk8)TIRi=#EG0;nXHfD95W6q^_b9-x&GV<~;!5NYd z=C#?S=A8PF!{p4B-;cHG{;bd^i10PF5uf*J=oy;EG@VXA2Q&qzD)s2VuXQ^aUs>h( zmr7o9N{AYD-LeGHBz=nLS!c)3CKT@U;Dy}9r?%j0GqSRI=;LMdEDD>q0=8Z^9{sul zPF^NkGt?akf`XvGPeU;TbaIk7X3sUnMN(#-0>eiRvmiCAlf#aVothhm?ALN!-SNfS zA8_`tlyDvR;!Mr5I5TzKF;SRRydWA3B6R!>ti*A(QY{J-n$Y&6!NKG%RteEsy@9;S z8XQtb>@-`~5znVC?H{3M_A;-|&LnQd+1j#qi2z`5JL0x}&ns_O_IG0;G-%uWT&rK} zv<}psrJGyV5Stc0zN4htS6Gps9Ww;^j~+CMB$$yN?i5&(eXsaQbzR&_>QJMi$;LD| z>X!lf`oaDptl>KVpG`~c`0wxS?rT{jDQh)8VMGgQlSarL8(e=2NJIL+Y?tZ@NeAk8 zqm_QW&Aa#NQT%kYI`fr0{y>THHZR{WRY*|%J}&J_B5M{KQA{07nkMV1^wJ<%Fp-E2 zPp{zPGWio1#(Jv~EMG2$wqM{AykjaC zCbqHdAg=?UY*2SI15$$x$FxDo>4At|!g#m-N=(p?Ozvy%6QpjaT)lj=?`ULdAg?Sl z+KJH7dwrTF`D^au5kJfX@%7S%fWBw#Gf=*Vwp{?l%j(6B_ah34A>77I-{NxRLA~@h zPUjH3r8*)3mk6_+!MotJvI9_G==RXQN1G7OI`}M%xrxPfCHkHS34jGNUgId7`0E0` zdl&w&w7y->Q$2FA<-I`CFkwb94_5l)-i^`R!K52^qQ5xqq+>({TS~E&MGxmcTWWN# z`s96qb7>awm6B?&gulA5{00q)Vn{xjdFby8e6AMp_VU$ttmCi)33uHgmKt(ImEq%J ziglD(RVw?prZ z4iLCwY|=w6sfxMG+ru}{{2imTpSK}lbeh-cwpKG5j>+||kG+T$YznFptg(Mn%(?yf zNyFUmB-*-Q#8-ntdZ7IqMcBcRr9EW2;JUYmR9g4UH*@osb-DWQUJ=99X$-(m%M!Vw zFSW*H5CjA5obvDwqprG>HLCui=BTZScL4LeucxO>@aB^mHkZRa<8yWKsAtMrzq(rN z8^S%6yB(^;hbaC%%mdAIfrplE>W6V=65%BX3FYC2v) zwC(~j$dhbx#$@K~+^gjR*A2IAreH6MM%G4$8r%4K?-^KWA*i2+Jjp&k>h4nY2mjpf zp5FjW9L4Ky(iC372XYZVm<|1FWHPdt7LlLXt#1u_uYiQ*6g@p{egG(vjwi@Xt*-|* zJtY&z*H<&R6P8l+?BQ{@DLy1Ie7%xwrf4a1D!lVy-7Ny(`$Z3%jmM2s!Ei2KL}n{( zK*7P&i(}I4d#q&wjIVT{?OhhnGc2vnK+6y%^*;9Lv6<5{J}BXlMw@10Nws@fCgT$;gBvM_opZEx{@B)^;Oo!PYMYmV4$-kp_(61*xAV7@lH}~jv zM+UB&=FEROVsSUZwMq#1+lJld<-TdyZ0G~2-(cF>Kd9*~)N;1#hc?T5>9582jPjxraj!gIbkH&s1Fj1qT}rBM40_>1k{GC`G(V0x z$>8x0512>6#24WByI%KVZ|4ksm$&pU9GKwb+>~_pZ=w|KflM!!vCa<>WwU?38v-pQ zZrjqNv$(t{yC41_ZiUJ`Ibe#&JdqQW>vs_S5Yu73Ef|j&=TTPSWJJLNn3$K*w)>Wr zt&8u)Pq_zH)IwUm&na4t`rCALT5bLFkFf3F=$vB{7s&iLbv1{$Cek=#^|b%&gqjkI z8l;Mqn=32pQUYUv!m?`*=e(7MIeiUdV-*Ry`VP{IM?gOB8PCTmRq8p3llB&4d~+kL zxExG!9l#A`{5mKig>ME6&6NxK3I|+zcV`HP41^vl!u&cLTPs@GG-;*8G8yzz|ob|_> zPJ@}r8_I+I*H24zzS)ybtMYKCG?XEcbzv9kfcyKs_YMNS2}bRH0bH|M+Ar_{{Zkmf zpVQSsDv50l3AzgA4~Xku24yz6h1c{$IE{d(m$wW@itXn`kuRTslETtzZqsNRgIOg=8V+NSQ`R(@_C@^)6Vd$R*f6I{y9 z0P)15Tgt&fB?H{Qa{VYe|aWy=Q3tPff&O=)XPX?}&g#j(__o+3ImcuRipvQIQIN+Z8fz4tU`R2Hztl zc}KUAcYP-J%NG#}5bNdTYbL!A2#Hxg47b%={Te5XLukl=9*AIVlZ>Pk@OoO$d1H7h zPswyTM{w=d*$w%yRYP!75%$U-?fpR`ZS}x|@TZHTFKbUGGV1byPcu=TYZZt`F*X(o1X;l9`fA1{$z<|={{aZP& zDq{hEPXApN5soibvG=rajM8r`SEX39f=`pWw%=_OBF1{yz zVgp(SF$*`%w%iXrHYEzqq$*?9yhJ^dYj|0ct}9T*b)$zfdA@w2zCPEhyDa^c>9Xml zt|5tw1=O6^>NZ>JpZf@Ndnf^XGyW|Bea|`U9W5-+H5pCxDy8KT9BJ@X^gIEbn}gw( z0BB_F?ShdaC_enKUlxDRD{7*rXXPok^)alYODH%q`h)j`BP|q5ly^MioAsPNLDD8< zDr72q$UBNFOwEEQZX@@|z5L&mX$y2Pd-hd4NMjA>OXMC;67<97>QW}ZD!8rgBj*n@XLJB~rIlGO$H2+8QHM6R*J-Z~C?%lHI8FsSJpD;TE$!R% z9BK<+-2jNpC+%=(&D?U8nBuefNJwc+j-#jw*1zRYZeeFn!6B4C^>vmod}ogK9NCnb z|I)&PxM*%dBpX)(F^OBGzse=)9jr6x3k@YB>~ zZBGJ~tihDi9>9=QC;N)Ax!?Z@`WGLgx~RI9d&QQX@ z-fo)*aZ!ZgRbxTZ0oWJTRn0qFU%{Z&z&~vA4k!>9?e=UFp`IUIlgDw#boL3A$QSw% zki3rG3KLDvoabh-u-vPzW?yK@Sl#Z@733wXA)6t^q%)(&!}Uz*QYQ*k_~#F+p22j# zP0+r8qL_tK(4rFoLNSY z^WA^_Z-5o8x$~ueJ1Jx`Je6R6E5h&j+rhC`IG9KJPHBIszJ8vbLGA~>-(?uai*g3y z;u4KVQ%KYjp3=dCGJMkM(wV@4fd z?jFzim$y96?~JFY$#P;A^%$+f+sgyeUaqp!W`f2p!&%uVeaEwo3|YpE7fz~e@4;;s zXwa*d?jelF@3LuBxh~bi)vv4U(YsJco1K|!Y{6Pr<{yxiUZ1*Q>@$4MC+6Ymoe+OX zzF=!M>8)MQeoWKUT3(u>WG#bQv$2y36{lh6CxH&}m(SOJXe*WJm)}h|nfgxE^mw** zQOh)0-CtnrMPCkKHFhRwTs{5yn@SGVKnKrlGKzcxc^VE5Dyd|c9*J)WSJNoAr_u=L zo`!4eO1-LD-!Nzk!X!<+``gSP)d$OG#3G|XPg`~Ek2w6DZPyjqeDqC2adGzvLs2Fp zYumk zaS_X7^XCLau0i<`i_yF;dE>F|lB7iZJt3`?ZyO7UP!1$fS?^9C0hx=gaj1@b>QyBD zn>v`={Xtq73^Sfgp^L6x$@5!cf?bb`t{}pP!KY_H45_OUa1%-`87E|SD*WJgP%m6p zxz!E<(>T{SiTtu6N`@>h*p9C^_ulkj?vlKH-SVtWKl-})p7Da9u8Mdj)TIVoMk`V0W;X zd(1EYk+2`q#m}Qp5S?}l=(tWDzFx_3=#gswec!^0dp;+>dHk-6sxs}1`@Hv;4;nZA zNTLYN^5F@ByCidpa!&CT%2k7av-J$t*4JL{+x4{M`ck<~-mw&71?l0)XJ@RT+ieD> zn3>*71{y z29~2i(lW88^HCe$F!G4+;+M3SujS5!zv%LZPLI21M2NhM+0^Te>l;z+vPxrlbNQEI zsb{!6^<(;G7G}YBrF_YDg*ST-3MyC#&VdSw6Wc;2vpJ%rl|NbGA6nCr1o#`&+1A z5+@|E12xpeGIqH+e7oGh_mvGRz54ZQKaCf^{Erj<7X0;fwzrt;b|W4OzP$7o+uDbM zY{;I=a&Ip4&sK6MWnO)J4rRrTer5V}^M{x&>ihEpg5$%!BN?v4JSgXzJ~a}$GG}l* z{4L#2|MMgr|9z4ZxbkpHfIk&p%`eY!>k#h5FTW`#HsLH$RF)#om8O%`Gr!_0E?1&s z<2LiO=19Mh`svKbh5O23+L|Z!p!%;FGJmrPzLtyfkMTH7fBwdsnQN+Rp0?9JFHIpO z|MP7Q{#bZD-v&MS|NBsDv1=O0pW~8jKOUZF>gewl){mXP*i=yEC*y0fZ}wSSXvsb_ zu?Mx_q+}4Ml@7nrcifvlUQ@$TPd~9{;2ugYR@>Sk>s{-7hs<@}XZukm9V`Lwi5;?o z!~Do|y)!Ls7&s{$w84lX)fuiOIvhiNXH*Msx+uE`h0eSHx5**6O*r=c#wjy3^xNlp zV*p;g3>_=W548htQQ|GF_k(3_W7M&Ur<_6?E*n;HY)3gi?Bi@L(AbjUBr~DDC1s%& zkWY6ue~uMZ9<;?F9RBkqx&P-SjZ8Znn^f4Ag7)zAEyUb~-RJO|Duw@i!xZ_KZ-hryFXl_jjU7c^Zc$*TiJn6-Lhb zenz2M9o+R}*ju)eJ36zRU$XD7C2D{5wiF@u{!hz1YaFQ7FtKIHvCF9TEh%Y% znvHX)LTRC<(6LWFzf9FtM-p-w%e8tN5Cg)m0JLq+^!)dQW?w_CNxBz_QDH zGnE`c1bBNlc5|bw3>g&zL`1_WmcmDwrx)PJM4>JOuTWog-2L9aWwzb>2-mH72LGNZ zDBw<^Ozi_Z&M!%#3X@8H_k_C06AJ$B38jzzE?u^g^ir}Gk;Puq&C%VK7=u6AqL$yD z6k%W=II81;XMVQbfAuA4WV^Qp^XFauuK0sX^X*wE`#=;bv9bM}pMBsr7UWq}6}~QT zjI5(j<%iWs#>fiyafcON0M~u{+V56)UfXY1_&~3THbyY)^-c1Dlw~LJboj9-HH*e_ zaC&~{TU1Jwd-*rsfcNlUzq!+0c7gFPSV;p{QF1)@UJ8qM4iDTb4S>x*KYQ-y=C?9U z;*7oL%Z%y|%p@sD9)_1<%<#L--ybr>_{*n5xepb;WAjNwueu&ZN!e-?>kkB3S8_bp zb0$4eN3|zOCEA~sHhOsk)n0kdZ-GQSAe#y$vr3L#DCa%AZ+-VH&K{SgU-M-^QC*^b zZn#3h|C~kS)QU+%uO)viXIGjc4@8yRd=y^!)L!rmx=lIaPb0#-$ zzL=th545<<)-5|YPQGY>e6|a- zxBmHTvj6>TKOam29*lV1&G2^*R{xv4uKlJh|2y|T+Y`@UF-S+>B&NQ&bq`A%IEj)g z8EvSXwGlpdw!PkLa~mp-h5g%UQ{+)@MQr$lu(3@nkT0TAuCqh6*Dtx1(j^GNh?TxV zBNB|r(H2+!2}Y{Rm50n>Sywq{{-!A9?7}go_iNPj_mw^kZH)a#Ck zZ(`cJ2|1AjkrU|#?Dz7_`H#Ow0RDNczuRL_aMjYpz~yQmxb1Ic7V|ennQ1nx2d5bW zQcx+Uc^4mzba6gLHv8(g>1(|tK7n?^UVsj@rWk-|XKjxE+7&cju3|*Rdlm2JX*BPs+KKfIc!LNCFNxi*tW7 zpvX;K9)zv{{cWhw*@L}{OXSzRa>&WU(fRsyYa&;}0`jU}4{c9)I`~t|WBw96Qp8E%uaT4tEky@>a_~y}#WDvToG3+m{~8EQzpHa?~C34%rnN#0YmTs{L(92;)mN zhUPx(Rfqxr*dqAV=^HRllQCKHAV8`Z?9kp3(R${v}jD7{7pX8 zew~4Y^%Gd5Ndax+tS{HDGNZolQX>sNJ7F5W%7C)UI!)?(b!pp@(O-6`slj~_^0lzo z_l2K;a6pkm8s|1XyAuVYu@WucaP34bOew9!!kdcgJVN-*s^a+Vt@g+iOMJ`J)u>~l z;DADf^5-u&Oc+j#z7a8}DG=4)S7 zA)I@@J%!BN-AhsQOo!hsxqijcI2{EifoPI||IGAuEXx16r`vZaFB9*SlItjc#(SEq zO@VM8Wf405x%G?x6PG|3M^dPI3@>O9AN{*+_xl%Q^#;1PqqYT5E4L5ao)Y=>vSHDW zEPYi=lE5j=hb}+Qo|tCNR@^-(%)S0mq6=oHS6lKw-->*}p{JWsb0hbO5Y2H1{@^`X zYBO;S!T{&6f7ViB@zJ06F2r=pspL?2HUkU>Ue~`Dc8cUT)*b);P1MvMaIsih{#6Ti z#MA><7Q@2yte$~;36ZTPs68kNOz~G-!&{DuJ7OqjUB#1$r*W!5Oe?N`qQ2=wdOEjA z-STB^SL0V`n^h%N&iX3Akb$CvY3!2 zUPy?!nYMMBMSHFt;KYi0bzZp5N{`Lr}{#`gEjP1Tp zuzVBk=qY(1<*D*;!cV_bn!kEvE6yV;_VkQ+LWE3HRyWfw`E#Aa^#db?L0!{PeLWs~ zUICS1Fpc6*cZpl zm8B-c2e}2Ruwr?1hqR|1@fGYvXUjxhI8dMVxH~tq=i%~wY=gj*zfg6zJu7zJBrW8^ zT146}Mn8(&rsLp{p2SF(jl36+5CGO+YY9vvugc z4vP+^y!jMVc?0Tp@q)qz3{DecHnuu`_z9K?4gsBxS9?GT%&oZXCr@6`J8Qg-zWJV2tsCC&M`<_t&hpOXuIFNNwOXt1gTphc$A|~ICx5F) zf*B{9^v!DL^F*R$*R*pE7|}p#0IKttS|c71QS5w9;|FU4Vyx@PbXhp#QEGE}K(#e| zax$*S6&>d2gn1xpc!Y~9ZG8M~ldYx31++gv6|Lm39Os%+& z__h5l{L;GFnS%7*A|I-90p8N+3G3-=Ubo=G$1beZ6>TmH=4(8-pT&CGI!>$e_0|0B zmDl#{jkL*q85K2eTzmhY1$W1qtm>0s^45^ySraZgpr|stZZhgA zb8mLS4^AabP`V#tkoSUFUZl^{QPy*+JJ&q(zPi3{P=IqR+MvI>l&^mmtT}h+x$F^H zgA1r+|DR&K{lf~iEEOf!xB0ss=J=MnZ6q=C>qZ}3O;ceKhQ6|w1PaX2VW9d@i~!Yj zb0*3}8-x&sEXQ5ZYGocTfbLLr^q>LeLY;ZPr?0RqR$JM0Ft3WgVElbz*=eYYf`h;N zjO?*YFphH$*n{YElNuw)HYem_TRHFH5mvuCqBy{k@o;lv6C@<(EDtXP0@egN^HYtc zDw=m})l<<>PW$M}Vom*cTB366^-`Bo{9M~zVJm3!#(B=FQ#g{py|oaMbX5z{K-6p}ot>}+#M?_0ENv0;LU%#|F z&Er5mN3WdJ!pC9r20%wfK?H9%0-}obb^UB zMS0JnR(9=Zhs?GVN%(o$^-t{mi${eo$*2Qg>*pC3zdX9#e2Uxbl4bA)V~XTV$eJ|o z7qw+E(!)MwNp8i1yFVe4VUw^uQ`JZlepo>fw6e+nyP6z*oof^r0JgyXz}|w(HU2tH zj|&ebbLTZR9;?x!r>!111T00rhdnMzxe~vOgh{7-obnK=hLV#(^}G-(!J4Kw zjVvES+lrN1GGZ9M!qMUF678YgOBDzMhkdv*Zqbe{KnXdm?l4V^$dmMO9@ z*AXMgI2$*t+5`RyqNkBw0pg+<(&@F>^xKN10FPlHmmUL}nYgN9iz1s(rad-YnSpU= zJrlc*y@GS?q-3t1jWs6`bEP&Gs zv`*J?&p)@p7i*!W>eio$m}(cDh8#10VJSyPm*BdyWLrw=i;KrZxErBvQMh+W z(tkd!V3OMEz6lwQzTvozT1$A69rDflB4^nKW1!8yqirGm{p%C>*RxS&B5Jg1X+AK` zfB`I-?oqXI{WhD6VzN^|HI7c!X&oAMnn{(8;>2_-yZK>3H$_0hm4;YiK^x2r#0yvv zN@2P)Q$wfB7x=&=XoOUU!aLLcQ8o9{R1*z_4iSb6XPW|Iu(& zNAJ$cW~Rm0yBFP8|4ItjVG|DctC%L&Mv4i@s%K<*HAbYLOn^cd9EyONnEtxD9sX8< zU&s-2eF!#U_Q4*fTZOw+r_*0>esYvZ|7cXenA_6vVUv;{-o=afSHQSm4yKaO)!9el z4+?tO2U(!in{76FB>qqy^r`#KyTZkaPcm1~*3p%<*U_@;sKDLHUXT^JeqHdkD|KS} z5bPmv8Dh~rbTIYySDc#7<>JEg7vjTt+b|JC*Cipzj{!!j9Sa1jdcv_hG#A&?$yRb^ zww4#4)Szj7*c-}3>~#P8lX?+R@rLRAc|WvmdIqyjUcVnlt|;%%;E>#7xOv=Q6jokl z^@5NcmqXskZBwT{Hs>1481C40V>$=3K0OxB#LH8kOG+Nj3*=-Yk>CGoX`xV%fc!4# z+;6VkJzcTEh6J6#>jF6WD)=J(yZ5{**4ZWl)ztn|OqtK)#?4+DWcBSvJ$f4cRwmz7 zRA$%HjI)1C2cL$teme?nXK?Mkw}jlKwN*I?hZCK9Hj>z^HF@^2n|vo>mn%88x1U6W z_Hd%dcDs&UMqd(0PS(v4+$#T6gy3o7>RSm}%Im^bVw9M-jeUk`C7cF%L7%PRCa(|2 zYAn6D1B$RDe6@0zuJ!cZfisH0PH>b+%n>L^pg29`kKx0Zl(~Iqbi{BgZk_avTORws z=0}k`78=SWg=xCbUwl#Jv7I*~qFz|kqY%J5Juypis@r4*Z)`=Z`3z#}(ln3eCYyNP zh?kX3gTAwPprz})<0{KEvetC<=zSrIj9t;;s+z;QcdN)`fR23tr({z+400L78DoQJ z*=M&NRf9;zNq!(0ur%nsT`P-YHs&SxdPHf22=Cu{if;C;fF!Z>r`B8+{4>^tE#^#W z?Xi9XSt`k#QQROn@Yx6~QaZ+w|Lv>4Me=pMrRn5qEzk)?fB-G;Nr+=O#aQcM2df%^ zbmC_e_(zU#ri&Satl=XSOurtB&a7Z;s~eg+#fo5bREmdJrmupmv!;kx;g(hXW0WuS z_%T8X(cV``(Y~rk!Hq}>&D9N5Qnqo~q?iJVnCokO{gkWs!Gl|DLT63P>g#M;;*LX? zGxtbaUJ~4coOism0VOtLru1hIxCThwT>g2d2W)lZZyIObG3`Alz4X=HXJhfCVyEfW zM-3`bOljM;jeY+xA^&Gdej9v$`4=&jKjRl$m#N^>(U28Ca*#-D?AOnFf=I>je1JE9 z_18%yj;`x6zVobvhk0}}T2OSdNnTDT+m%GkTiPk0cGUGwe&OV|ot%Iap9#>RIHz%`=OCNt zX}zYytPM_%b5a6#vd28n2}`sOyiGhDkI#d=25IcOcwhm#gg1j2fB#=sFk8o60|EP? z{ZA61Z)1yyn9qGSm1a=j7ZCY;+JTfA%S*lcSB{d6j-Gim?^iX5GZ1Z??W?-F()d84 zj*d=dqGWJXnS{a{56W;bamWfi7W z1B4#W^IhpyJDa>J%}R+n`D&|QT|M;)+s{~A^f2pG>Zeykj6eSxhn!pco8^jz`jkH- zE$bkrs<6R09E0+da_7fb*ROWceG7x8Ntqh4D-R3N03krEk27ThERA+fUv)|CVr_O^ zp?-0$y0oqFpW-RS2*jkrB3q(^H;*Zp!CU~6XMs{!PdZQ00 zZ@*gwXwCZ7_7F*h9BO+EDdu_0?N{*T@@3zDaT!49hXC>!bl-)p0V3&`Y;H6*Z)>?L z)ie!kH?G0UOz=IS|2}0gFt>5`x@c2_O?{s%8|GC3JKadB6O5lZAG73xx2pP%47AoB zi*um85b9_134tFTsZ^m~KlvP=TybgV`pnF*s;)6sMUo|5IRZ9Dq3@P2IKs3G!pVHa zzHc(F&xR>RfRnkBo`6OY4bv|!90I3 z(57F3h;Xo!lTA|8WOM!I*SlG=&zs{*Zq0__w&s_**FOXM!eLW-Bxf7BdUdBS*rgb1!1b#<)UP#>mZ-g;g9m&UKZWcj3E>#vpK-B%J+5e4Ur)}A!fP$BwW zT(yTQrg1r^wNwy{1c)1E@BHzTso@%G#-gU?WfU^xq@lv15SfM~A$4e=*o6)WC}x|DxqxpcF0{oMVPY)Vvlm&`zSiuQ z52bC=l-baSIXJSo=!4B#EqYN_hBUY$*B#I2l5daLlW8!*#|GI&W3djV=7GJ zurgzZz!7@s!aCK0!)9DWWapaosK^eN4xiKs!k5`H0HPq8L!$vEALsPZ^-`{70B=oo z$zq+*vOhYagBA|IO&7GaDU|C|2*;woS0~PVak6f3TUh8Sv7guU@TDB10=#T$l%`kvKXWPn3yaF>7k)ZM{#`TInG{ zUC91^woL;YMC^)N&n?4nXKLQA{o>)85L1G3d!YO`2=Um1;&#ZR2UeO;=X8}i(L z%(Cvyk0egEFU7i9LH-=d(h#D$r^_4+*WaeA4CjV5&GGYK;?6=@y3M z-xum!5riCa0>M7+NqIf7TRQNDO{-cR(5?OeLl?e@>OEGz@SlvDh58CKL%{yccslFI z#_TBlY09P@P(Z?@PYyxQYtz}d@NlNW6X=uG8#H;srFm`8ubBzVVgSHl^FiH8x&LyT zSS`?73S_Ze!2B(-x$1=jHk#f{6qX@C!`u{uGyNGJ@0(Tbgg}b3GD2 zFYt8%RsOVYsRAm+>y)zog3MbtZ;Ax{wOdm?FZ(Zsm?DljIht6#{} z3JPcH>;?ADW$rq%@4JmEKVxI}G^4(XAZ>l^3tFHX8G9Nnadrs1$ma&?bHwNKM%fzh z%{q@B_}`DLLB!1RTy4yHZQ%NP60$VbR-*>qL?}Knt*s?*s#RP`RQ<=W<_Ly00K&mF ziidP!)bLW$@Nu{)o3U$rP1p#y)WPY;b~&}g(Coo{qne-=lv?25$jNKu(|Y;S;&$r8mkp%*>Pdz1@GF2@i_w=oM+h&sgUINgxx#O8uM}gyjRq z3Vf>H_!UY#d>Py9>rEfjNegBvseP z(rRvGLCCvC9@NiDI9{-Qhfj1ITOP>gG}T`w+xT{C!KDg(nTh@|<^6;oKntb4{&Ka0 z&E+8%f)$7;T%L7qS!=ixP+w$MZMke22&7)UpXMM0${CzI(BEj;Ub*yy4}u!)n^%h< zH~>RPv_xLuwbUKN`7!X@p*N=dZM<5;>$9ov?}UFr=$eLjU$9pq{(VNQ>{&&=mP`Dy zRn25GIghK+5Ci2vO|K~01^B86dChr2ox;gEFk{xezD6^4ngDlzox5XPeRcR1T9A<+ zyU#v0KXW<@mk?mW5RQn^azq&en`!Vj$OhpW9LRAO@OhNr>YFG)TFn0jP$rS(Bn|A$ z*#B$l`h5TCPg7TT5mY(;F^p!2N?*g;8PiCOQ~|UnQDZ%$a@XuTfimBe z=ZxsBjrD4p{B$yX@Lgdc+=8a`az(dcX+T(i_;7W7VA!IR)qAg0QT0VA=RJ&{aVq?D zBJ|Bb+u@NiY+hXs6eghr3y80rms4^;BElLSEbX0dCN9TzH|p9%`n^yD z@9d1URNZM(IZrEo@8w1v9RvQ^MV}j#eHx=FQ!E5@QBj;y@DgK8&bf+fA`>nj2aW?= zk{6=EhSX7JW?wskp-y^H9<`a5S4#92lA-6sl3EuaqiP&X%5VH^jo0qx1&`zlP73L1 zhMX)!Vpk?R4JJ6wYk+A6uieuy?aX(s+4LX;?wkjC=y%&{Ux61b;LQZElwN`J6Dcr* zD`{!bW>drlQk?o#RzRoU7KwHN+}hIK?>djP{oi>$2Ro&9r57KP_xs0sJEZNw4ZuF5oXEOQCKlHMr#~z6z z1dT!mvn1WT%Bvsjn7bO<@;e91lC8(;DlG<%uo9-IIJzpA_aiyw7^BQDUtS9JcVA%> z%mND_f}jySmqAj5;KP$PG9gi(nN?d1o^}L(3cD79mJalxaBi%RTRzK{S!9ArS>4c%rYB@~Us}WlNkiH& z^kY#I5)!e#vrb-*O)8yU&AYFj0;ipg4xQs>U5pgqyq3-`r_{_NAZ-MC5&#d!2agTGi(SFHLTnhGh*wEQJlr1BLJ(6RLCBacx7`)h^41*4D+zo7ra@XnoXUlpi&>=*(=yWc>wEy)RM;XY}d=G=(F}kW*X&~ zO~LB&Vx5Iq8qOe{!_+ci@y9*!`F@f(iq31rob8DQbOGh~ zaV+$7yV7PE%81lhKVr(v$93xHXGw)~XL)mVK8Vxie<4k2)fZKC%wx_D)KX!9hBD2%ksA1w;5?s9*~Ao3 z;3{Y^Fr-UBq&U55DQNfdI^r@Mi{D44iyGE5MhRa@Mqj+;Q^#X9Njz=nB5)3$|Mou# z>%l9w`)OiF)4&mew1FUF)s7@hTxjTx$V?Zs1Bd;FP}po|!)&dxudwbtyIf-%Y=Td1 z|C-A4)%+_n5{$x=U#`?O)7jG@*Rrg_0rSNM_8|Ls&S8W?v&m2+4AV~RjJ;kfTNjR( zp@EL#nL5m?H&?yu%>H2fS_#6-B1&>arJc6G^pFTvj^R#zL^Y0B_f3kju>H_Q^gv90 z^2UPz-Qz=M z0>KXSIkHX^7*)hPC=c2>ZxF$&xW1(&)H=0rVKt6_i)}@|!h~!8v3eALVKjFPF?Y(LkgM zs-s8gO1Y}Bw7lFgR8{LqO043OFzK~f7`hKPNYlzE20h^cOS?RMQZhZMvrCO->o&+z z2Fo^uovjL=fQ%5 z5@EK2R)ykWJP0B8A0(%?`ZJuoQB=+k--C`431A@0F;*7RG1RAfG|L)gAhc&MQ`}0( zXwKGEUp5cL98q6y^@u`BlQ4n`T>DY9;W@# z+b-`EF$p1-1boiln`DqLT|Yb)rQt9mmYoTBbR9jk)_RN-+3?K-A9LwCS$anDcVh_H3k}@X@HJI&c%XilsYmvh7PKywGG>nAX7z1 zGIi}`N^c+8iAS!wuPfY8*9N1oZ1jZQL&E63BM-X$WPxRmTtF1S$2BtdmTlqlKbFcb zY&oVwRd0OEvKwjw@W%MT-UHn@D^kSm=*ib-6pJh{CiD(X#5)wEvYYBSet|aEHXHq+ zs2XKGnsN>SvAi!mp%bg`>~5CVDSF(=XBD}u0nR4^uCtPL#(h-iU1Z=>QWfU@8u$p6u&+3-k%*OUtwFggxxCSF+lYK<<;exOg_cHV88q%sUEruh7tkA z|1Ue>P?oeZfL>&Z41F^rqXFBB(cC>g-!b+@ZZ~7>-0#O3n$dWopv7B6W5rx7K!>$~ z5srdO##$|2xkVXM{+|#${tJ@xz@QN$r;yO>bGbeBes?f6@s2Jqg+x?>MG}(!eg|)p zx1a%G=$W+oE=;Z40KaOU2csg~hqA|fT79>2&X|{3z&xolc~@T4jYAg~L%qW^C-S54T)ojeR{grF=YuNN6NM z9bOP@5A|~uWK!-dXQJAS42w0|p`{d(zB!A#IIF!1o9A`(@@d?{_vN+uc@O`oLVN7x zlr(R!v>{5U>@4MHI0sN21)6D zg`?jUSwj5t7HWNA_NtTBCxs$f%hcSop?`^|vGB`ca~njV957oe$vO?geA?64<7DS^ zX?K%jstRkcKCBiov(UA*UT8dz3@y04Q>jqg=Wve5z&`t847i?O>=!R(8UoecU|F?m>`w&5z1hryq=3swZ_eQS zGf2c`#^@R3z|wixM7YNouLZnegh}+4wcKr=#61_gBG14d*={eqhS}X(lKtdLh{d;x zR1z5OoOeZ><37tO8Ay!)8YoE}7CpHSDAR}M4KRRztq`_UrNF5}B2Hv5&~|Q!gsf+~g#kZsuLF~Op;SJia3uhp_f7~Hg?R%5 zpELaoGUFVBZD=s4Z~CJ`T@#`E{WnrAQ%DCb+!uq)t|8vO_p0lHpE)8tn(611!j|N% zw{Qp437?-71JWN-%yQ$@e+x5NfEWbxW^7hc-SWjx$H-Hc5z&#V@sFqX)~EXR9yw-f z{fLfCrw}N&ss>#2OKzVZBilB&sFA`CZrx7ZymDL>_#S$BPq=nHU4dLW)0Dn0wKGBY z{KSX)jHv+D;@Lo1qq7yn>>7xa+OPAk%wS@Lt@H>t9#=064e+GqUmd1Ra3jM(>Vv+( z;`+%2zFG6Uc?t{JirR?7r%;(Xm|qc^d_2nUy=CzG1nUW#ig9Ogk-JK2#P45(0hJ(D zN`FUFnB7#+pA%yk<^snY)J}RZ2DLs{%)N*D&= z1e)@4Z$(s)oMg2^Y2cWfx%u*`}{^G(MFC)zj zev6Ojh34xjZeZsoKInJVnN(eR|k#t~C=cHu#Fi}am$ zRNpeqfH#y{0X@)4rV7qtUZ-UQ#|W$wETWS%g&$1?G?cGWwU`g|e}64Xdwgjf+RcT_ z@%!NOlBS8m51Uv6i$+p+?YqgzALuL->z(g`JE~5+>DdX6ih?JEm-(K(^ z8QfLtsCK+26B>4`vKJ>q#p3QQYLQN%(}>xgJ20}ixCQA(vTr^TFhZGkA9LqKr{KNa+~goS3TdDl-s=W{ek0W zJ}^2&0dSj}HIadKMiFMA30qkUB7>KJIoAW-7lQ=e3aoFg9U^ul3_33>ghKNe=bzoOk{~n}*v;A~hMvVdq58 z42qZ(tra|FgR|4{lCAM;h!zF1edk;^qIv2|fxU196mz1%(A(yzzjgZ16Y~`Z`?*!^ z9)pS@Pxms2vL_(*NXf1OtOWuUyv$0lY708gLDb^T_7#|ei#!J!L@;ngq%RYmC3fCu z+;cvH`RIK9NI?W1#0~)a1K~*_yqKIYQdDJ4y60l+P0w%Srnc8j)=KL!-dOLS2`Y5^ zgvXye_=)!`7M4}Y$Fa8Mua6H^ZT)?pNLx=|T9&0I2jym4xvO#>oiu6PD}_$jys_n64|7(L zYBD!(wf!>~b$|qNY5$*Wi%@wL8aD*LLTxI};|)k7*~x-7mTzaQRVTkyk9oPD?g--j7?)niYR|$qdz5E{zGUd)Z2!tr68hK-nnGrq%Z$GT-M#e13&? za^Uc0-=1iaUejISEPC1v3JjV6We%w63N337!Yr3a?!KAho(r)UXVP+kVTd16Hw8u8 z&!Id3tEghe3uy!R5`>xvaRTBfwp=Hxzu-f*ZB?3RZaP5Rqh{^MxokTQ(`)Zal(4M&qlig6EfJtQpSJ2O|tKPC&-v$B{_DMy$I(6y6g9kNcL?{)^@AQa~DyzN- z^!V99P3z=AlJl(v`ivpK)M19<79&iR0!83uR=m|Pa7>1D+$*tKrczZ`VQ>%Im5Ok~ z%|PHi!sy#W3^#bsosxsR47$KkMFr{%)siz3)Pai2hA>03Dvw^%%|aW5>(FQYM$PPN z5R9^k7sW!Kc!FyV(U(quzzITim>Cj3-QvS-6`!!N`29mci1XD6+;YSguC=w``N3d}HO$9?*0!M>3y{$G@$-ex|=@ z=(%7RJTB zqkEZ$c@{rv7H8jeeY3~}xUoU1d$jImDHcmqz$FR*|F9)Sf@u>pX&|jxK<4`Z#=5wD zRuM#xxgzz11JCOj5QG=Zwv5EzCv-^P>@OI7ky}I|96*u-TylOl)0T8Whl?lx-0Y<* zn_UD1;KVzHr&lHg%^U$&iYZ>l5Wq87GX~|T4`*uNQ>osuv%JZZRer5VOL1b zy5jSosCT7-tJ+3AB%Rk(w&3RE==!^(GQJd_%n<9!ZxP_1(4)Hs8XdaKCuc#z8IJ&Y`g%Jy*`vm66qJ^=2I`(TD~Uj)fBrwh$@l^H;-$4SQ;Q*KO&1+qA6k!{B4xPAg=>xg z_;O;9G$4w6zc-&>6V(JP5g}_q+B2zBbOK_B#}yo=nI$>5v=kuU1(hM5Os~r7APj=G z7ZB4LNWauno8H&Z8Ngn;)HJ%sU+Y_4tW!|ifT^%jh$Osx zVu4k(-}lR$Ht3(d?+Gd*Q-xams^kRe!vqP(N_S#8)2fIe^BlBQbXCe~i#Dqi+Pc2j z#M95i49j*ATzuz!DV~J5Ko}y*>Fb`pMahpG&Qumu!Gl5J2=h3hx8{r@eXZ^G&OZ{~ z2(1Ui7eX{SQUL{aq}sMlr6dE_UzM>!3qi(Dgx|I*9EL!!A4c&JTtUo-lnX*x)K(z- zf+0%LK$?e*9_QCtkB^&OnJ^VAE%ssz>mdrZ*nK=l`SV88r*NHRjb$mMuoibm5pi` z_O(eHhWO$T_o0^iMBt~3G&%w+F2L?Wm zTUlyioqf*UdYtAl`qwfgGWk4#KfgOX+81&+V_y$0NZHzVkr}zFl_OSq&1@(?#%8${ zA-P=ddtx&fG;IrZI>ZuTWX25ole~4xxKLdc=-h5Y)rhisx(VXz$<@jrFnk2!jIr=? zVJ2H+TdC7Cfl{Y0$Hv4F;MRUFGT}+4=vSbogNbf0y;_m5F;unq5jw$<&=bwuB0mVN zF0_A2gXx@iNW)Uv2x%57w%m%yB)Q#sOfNe!7q`z(MzB@UF*i1}MxsWk(KZNlt&Y;s{t7ISzU)F{VyRu3z za2p;$Gkb9Tjhp#6=oRXvxFDVODoO6p_TyE7o^HnE#y3!4t+o9l}$#M%>Tz@$nmeOcEkXz?kLnt(gys&m5^(NWGb-*9Tt-oQqk@ct9Cic zQm`}9p8Z*$EZ6R`PG4K|_0G5M)VD?Fr1kfvY!2p^;y6D{-+Q;Tz`wQDM!Xn*;kB6H zel9;Lu7d|xN$taQncZSTKcs!vvyY1c42q(Z0{wYQ1K7Oy)GYSl;c;2Sw zR>AdQ50h?6nJ#~>BzvZ7S}G^wKCO`$*1Q}2w`d+0g}9D}VYxVKnCL?cDL&o~Ys1B3 zdZy={CFY&)D}Nvk9@KfcJ=jlnTMbEKtC=Nur{!vlSyzR^jcAV+46Q?bXglg?mH5E) zVB-A6V=Z^Ks2mqfk6Idi@4fTKv#dee364NNDKovcP=lz6@BJnCvLsu(OE3EM4C6cO zHh%XVycXagsr<}5T|UfRj1nT2AZfHaxNm!a!E^SlOFN(D-TvU^x88YAytEwE zME}SEFN|gH`_VVVuSes*6_Q`wI&)MU%XOA0bs+onSb~f_K3a5SdD#Bod@z%*slSGv z=q2NY#I3)@=53-dATFyW>qHt#{N6-@CM>Q9?=(vMAU&EQ@28 z#EJ7(ir!U{9b9SF&ZPG2DLQ`Gp3UEZFnOY`VY6c{xungFHp*n!u1(yPvT0)fJ(^dY zL6Xbn|Iqc-QBgczyo*R9DV>6V(%s!kD+fX8CesG%~RUq~E^=G}lOlH5lILwOe*-^8T>RjfwMO-J( z#zscl({vVtcS0_a-rugJN~lUYeD_`1|Gt2hcdCveet*eWGi7D`JN0zy{le+6TDjZ2 zxd$_?Y)HJ8naM>iYcSJOA!VME_zA%#2;T5R9aR%CGH?aA-S^{&wyo<-jNby;nml52 zlZlf)_ZtJ72#T|_a)F#X7b?)|R2_7-o^EALw1ZwEQ%?U4yoEmesrLEQ%Ot20wlICn zgJv*lnh<%WC3kY=tsW0V1^4($$nKyZ>8jhd&S)fAkX5%1Z!ja%`2HTo;&9R`u^!2x ziUf*?NIji3l~~J*-F6>5fHb3S6c^TY8*u_38{N3EYcuU`;ue+N3N%!q1}n?viUkCV zzV&cV{`-8pq7lH-{0M}KT=|3+*x z?#0<@H$9h;q<5S^B#H-gwA=0EU^GTXyMIU4=&)AE4hjpNv+bOYE&bvooy(_v2gk?_ z8n~VZejR7>bZkppJ>dtg11UJ{Y*~b1cR<7{t zYJ@IYqjCbDIFgKc0pN;DN7dDd}^42!E%971c*aeWV{66g%DedMSdh|5kYB z#Q(aRKgUoyC(h(@--gSrbYt}j(B(ko?c#(WSs2&-vAeE@#>9PkspimWH}1^(R|-lP z!jc!W<#L*m)xh@1`ED|+E^;}k91PzSOtq=)y8Z8D^NKl2zC88<6K}qK%}O`yZ|RDJ zd8OUYzI|`aYc9e2$4X+X;o#QvK5x410>J{+(4Tr`Z3uKyfjd)f`H^u4o%_c2nyWe9 z5A|y!ZBB(ZWu9gdsW$Vm(ZI!>Q9jo5TgVD$k{NkZ`@HZCsi4>=5AYHNnymjgcjI&S zR(`D4-FmQQZ|lop_+cL|v6xUB;<~wkCgKflcdeQV0iWuuKJc?PAj6&MhEc(q)3xKA zjz-UqeQFVU2dCZbFxNL|Ao}tlygw-N`ivr#it_nn?|aOUc{KwuO7>E%o$9zOo> zEd&tAE7kLSb_R zZ_c27n#~;RJGs!~iCE|EZf@atujqJ5xWVaT&~!RKhfD;t|5;NDrYg8y;Nvi&c!23Wf@mycaj z_XQPvfRrK@7>RJyU3Ly-sf3p#fC91=`L+>5m&pl*b$-b#=@I~5I)o?~F2JK}5#NSh zlD2YA%I&*l91Ie~!@gRK@y?c!JfDi3hlsR@wO3ph)n>#DW1l2?a#6uxGl&~Z>?ZDB z<5YYoPsQ%wS(le?5ROEvcaK%(`(x6m1%uQt7ZYdf5;7jroR6oF4*qE;{P6BBc=+58 zexXWwd9=~;_=mN5Ffc9)g18%@XS;quxemk7Nql0OO>TRIC@(^Beg~PRH%nltVC5&x0NMrbJIAhY`$K)IV$~LkmQ$4Ag-U6zk$wa~i z)p9wC+|$)MfBo>?DfNkQ$J(8=iaqcFW#wc4pH70vI(+XlzMWdDY=8o_m$GY!=Mq3V zl2dIx9;b+#1lNC4R|( zVtTpHpy+mrO-NbV(@(?EF;zV68A;>sL#Yts;yV@={j4WfZu<5f)$k1ML#1=J4Xj{ij7#L0qs}`F4@VDoqa^M9c$NQBV-K_% zro!0~kHy{mRNBdW^Y1h*1#s*Ao?jIw9f?1t1lom7)OXte*$?#{zT{VyStHRX0< znywiMXtT0kK9ad!madb^U3)yl^Z z{Ik0DKm&0l9Vo3%e-r*+2UW9mqkL*HyU1JgWTD*~s7gRNruunMG0qsKCH;W@5+{n_ z{UJ3O1D(3IhcsSbUg?*sUt|1&$`%w3>X7ddzp9^GH|keZHGMPJ%q(_q)_1WED*}}d z%9J8mMhO${zfD`Rn@m#7Cx~k8LpWkbpm%gwO$RjIh$tOB!Mu_KU|u6O^R)3HfsN#+ z>WRSj3rc$V#2l`&rTXfN(EeXkKZlnbaLm($o%Z!*TB;t#To|!ZQ!lhDR&9Ll8~I4* z3Du%H;V0ukdOej+@^tY=>K-!zbtxOAM}C)$sJlAYIQ8QTjt1H>!zL7+vn^7+p)<3TjTyB zB_;`jucU--<}5$^q2f|Z_8fA4i>PQ<37?UqZ?6$2#Mk%Zb;4lm4?jR|(yQ=e%2kru zxfSCmC)TaghBF*D+rRpX>yDuG9v740G*?b9LL?^DuLA2v&<|q8g)tKt8Jd=k|7uyF z`||CK@d3S<$FA;F%(?J>r(wh)s+qliiyvT-$qwD13Ha>o1ebB+N}Q!~*;mu`XylN~KMA$3gqCY1_&JrMrTACiZA6fMO_0%wfM-_~qjJa>EyiZWBPwSa z>X3jq2%@a6Z}!4Q;A35G&Yv^A5PicrL&C_+bWr2FZC=Ah>94(NssUiFUP!sBu!=nk zt!#KBZu33FN&i2IF)Q1_fV^Im-$IWvN*R4KtB4OSBTh+cqXonoY)1 zFPrmtF$YpZc7kxq&==VmiAyPNI4WQXxp@xU?x*)0GaDQU5}iEIww}kskjXy+*RK}I ztk4<>6pl>q_#gs}=YPA{nn)TtonA@tK4v_;Bfu*2?@}IQ%tb_1)OSz4`RuNBMNpgr z_rU`-I5h zA)lkd`vEM*J{i@>2vC%=jW#qbBEbyP)i;PjoOdSkOEMvscS{AFX;kBHgqzzbcbv;$ zAshbFG>CFVf>sVStF%jx6Osc57yX>m`I_d?_4z_ZDQ0Ie>XSjPu<@9V-;L87Sv%`` zEAX<*Os$`B>%aQ@n?`j1V>avC5l9_%YOz>QDgfg^5YU&31rRu10;;_6h{*xs@VLB3 ztUr7WkabeNe|ERu-xqXaOjMfB+SBuy24n^EMF~A^d&tCk&yQU8y;SS@U^91^-16pQ zU%7*3Y0)wddHOf$vG>}Pk5+h(c^s6{<4i8QXIAMy(1I7$yPo+3Mu%$A5?;%d&8;ikr~ zU;4UeO+SToS|CZ%#namO1XdV0RrMK5G9&^iV*Zth`^3U#pA#x;rBny=EI#!$%QPd2 zHo63*ctP4BUujA7k7yvUidU$W?LqWr4wpI_i`>?d|67T|8^Ou)N!Kl9#VFqZDsc5i z73_oKr!{mp{FehCuVfEyY`9Z3erO%g)jrJqm`SYBaExOqd&u=%mRg-ZjvFla9b6pZ zy098lXS?t>Td}riwRcHFe=fpmvJLOa;hUEofLPw`JgyQVyDlu5Oo~k3ger<1Q?*R5 z%_?d7VaC+xH<^ZGlYusgGspTH9sQEdW4+Nc+EqP=&!DdLvQ8XdeAaqrisL9SRk7AXAlLaSze;$>> zlMr$VbocQ2u_R{InVg)i`Fb8}Yl!7NTD;C zMAHha0|9z3oOOR>s#|Z8a?}3>H;o}8KB$h3C+L>;=Kvk+C60rHIb{-&odC6$^?_&o z0;mH(IO?dc7JpV#`8&GlUz!2yYmBPGDpx$AfAFX+TK&b>Hq$R;mAy#2Y|lk&5u(gv zQ_YL=Gu9nIqj2@qU$J&*ik7nInR%G~67!E>V( zsQZsDj~w4xvjz9%`J(Rm8Qdq9Cx-3!_FAjsY4&{yHhy0vz%3Z<`mR(z7sW~6rPQ)5 zT>gD?_-Xl4+jHnM?*^d9NxgSXJDu;LK%=$FZs2-fng0%hTVA(+2fPzD`=@r-Sj5N2 zNr~&+t!5lUk(@HbdpS9}-L@5_UDsFazUq?rZatxdtHa#(uWgMHpioAVlTdON&`=vT zJ_W>R%#>V2=J03EyqsL3XGCS3WD5&QKofeTD##~eBp*1!0UI^c{=QO1J86gk;Jx9w z$DeGYTc;6-NNI-t0Ti-n-z`s&!!l@9#cP z{14h7F;Mn5Cwsz`lQ4Jrqt-ktt3qK&_%b`U&{J-K&ZJ>z*_T3-$=;o3AgOqyE+-kY z5?}(v%-bj6%E->7cm8RI!%x-~als^}{x?74=xBS$Eh{y89IM12RG8$?2Fhx2mkFMK z9<49vb!>ILuHgIq#>t5@Q0}SkAs&UzvoH(pPwF4dp9O#B7FlMjG7Q_j>-{CF%fVXm zgeXk)8PL0$SlD?Med*AwF3aSbJKojxu~(vUl>lzQ-jW&4B#_THL>DhqALcU2ggyB# zM2Pq6dEj_s-qb*dnM_*cFdPui`Ff|b?at=#bBFQM@KM86RvUpzU{{af=AbW>TlHE1 zv`h89l>@q)K2oKo`(f4{IrOa=fox%T^Oc!XvA*seYn%-lAa4YUnQ7lF+TFo2X-+TF zInH)i#QRPZT4R|M<{##!*XGUpr4DU_3K(!s?Hq=1_;%7xyTB_O9bYg(0n#senX*_r z@?v9-Kl%aNRDAAl-Yk0V;F9v~)A#hX#rUn2yWZS!u;0(yHS?o=xwA!n_e3%Z4aR0N zKNB(O>eiXqTeDGjc&yLd;(xMU^vu1d4zg!+IdUt2yAJ38!M%%q>F*VK*6-c7U+^;i z9{nNvoP<^(U*}kSP0ms=7!xB^30jRmQNY2a9`gd9*`Fj&%>T)y+VSDtxMD#tbZyMZjVe`dA?m15M1p08yZ5;?+#e@^~ ze5oFLHmUWx2rhZ?Hk#7$(>lDbw+ewR6>pu7L*mYM@%0d6eJrGC6A!S*f_mM>EkyF8 zBdRF`fH1;}Ri?rWk@SqMSXQ520^2WEF`zTQn`(F)Hrj|GC@w)!qN8DGN2NQI^76iE zUt&&Yymsn9U4PS0Pwvo_hycviJU!_8mqI$R$_^}Gp_*FcLJyu z7Jo*OL(HlN1IPXA$wpmuLakKfd43?wD9DGJW4Hi^s}MeW+TPuzt#j2Qi=DQS#?{+U z>hM&-YLTf$Jq8ExNq`W8{-0f)GT$9@Q+Op#Bvy|M4SUBx33m>*I{HayFPiZ(eV$|Y zgG%MF>8K5wY%M*c)W?lP73h4^glMuR$K82e-wS!_z@^nUB)1>oTp=-BDF6d+c*yV^ zpiBZ4kLaAd#0l#;;u!C^K7eDDlese9ot}%=Z@m_N7w%529htQ(_9UTm@F`M_zl5&% zKra+4Y2!53VsV>A-t+^@vcE^`(oP1zj1zX0&ij@bSi6F*~L&q)0_ zncUu$P#|a*{8VFaQJ-n)?+sb^+#>S=d@=t^JbEPV`Ov*c3A=R)OgaF4^P-^9SNFRKY zdyW>+b06PDDHn@dqaPUhd19)lW>Tc4 zU0$CLzwf%Ga>ovoo@j;pa1juexhlqoS-kuo71-kXfbHM-TDG9`ruyJ{GqQ-{sATZfHHqP!Sp%ap*Yy4a09mZ zVOVx&B_(+yUtI{Nw6ZN9rT7KrQ*S(vyq`Z+wAop0gJsVHcfleh)fitbzH9_||B?*p zp7HLwx<->Lda-A0G@YCG;TkaB7JpKR(xX8A_~y-0JJ4%)5vX4m3c)si=vmurngBYh zNJ$$0ZG#UK;JdQ@!luQS^Huert$P_xX3CS=Fz3d?%_Z#P-Cdsv{sZGmj78GT&PvUq zO?4FJ;HJv5L0^ZA;~z^J7gggA6>^#mbMWQN$ls)AQW3442^+Ip=@+JA$1%d%7>#JI zSw|Mm#FnI>v15+@dbJa2fHcGofBr3mb2Vp8h&Q4owZY%&E35ruxInws8pC|_?a@aA zU}DHv^k(zp)U+2h-mr!`FW4D({xiN6IFAd`j!mqN3!>04=0~PUf zHRH?Gx+F-jbg#cRBY~-U@X(6fDH(9K=MaIX_r+fRBq3n~^aP;lZ-HKmNe;#TRa6#> zOZZ&%v##?1B>Y_H@DOIG^ox*ABe#^}Q)8Q~)oiJF_IGYX6gEdpqW*1zV6TFada#!M zo9ZPmWv}F2u<4&}z~9{FR{3O5RzXQjvO6&=r5;?XHfmdyUHR*o#vr5~H{$sWZsC+R zTo79)ubX%4X1c21%Rx^#DKDq$FZebr*hsNp`2hlkivZQcQ|uu#Y#5s)i=TzEmIRXY zs|&So@Bc>4wpTn+gUW&6QYfsmcPiRjBR$OP> z9o2U?95_1|eLxygt(_4AAphyU3JVt^GiTRdjvx@o8QV4g7(KKYL}>vU!H`vfdEd4@iQp;#d4G zZsg}~-;ExIt{kW8P^oYr!aJWi*B&0b;|@;7h=pTD(8Uj_s$i0jko#eDy0~ zoe)6xwFLi`nz0U;+w~Zss>q@@8J=kSAcgs8XtDSIAr=)!Me$bB_%Lgbw0XiTw>9V0 zlOH0a+~h z&pua11~Lv^R-YEFz`xg8v&vm)e^AO{J(LQ#_5ejK+9fNNs5Dtu&cc1tP;3P0H(eY}A{P|<<^MD|`TX{H=o(q98 zzEb?MZEhn*cG!FKye-vm)-$~piuh6v_(gQI+KbonmbSE0op0B499&~jLGnQuHzIEx zoou*<&JWbJ=AY_EPVBXMpjo&Y0q>6SobHT~#ZK&BqhWi2ysa+E*Fcb=zSV<)yH7BA zm0nsm52ul0ECPEIdp`RXlDn24g z+3;;FC=R)tD6VI2=ZQTw%ArVF83CA$@|%MIvO%l!Z^S`Ev7fY&!WO99uaVLbl)bKKi|Cf7{G(W z>l2u4@HdPyr}W*Q^#SW5LOYG*l_HU{{QU@km|=}l)3}6$m6kh)zHm>_H*fY=13Kf= z&B=yJ|E}JfIR3@4{|4kmB%fG`OJCc49orNI;l;O;UNRcbO&V!&3+yeV_M&Y#^y;6I z8diyk2UzqgD(hH$v2T)D3EIg2zZ%F)L1-9G>CxyaSy|b1l}O2JGp7?YwTW6Cd0|*KzOVeAO5O77iWvYo0UiU=Gp2agBT8lsE!uE90=QLyfEvgkT))mx5NFMkkbf z=G>oO5YL;4x;z*+o-Wmlh2D(X^Kabjz=VA`Sd~#!vY+Urjob6590f`V3!}E#C5U0r zZO2HQ=-)D@=wVjRJ&9~YG?%?9Qjn1=SupCAeX0n;MkgWZfSM2$#+!crPLB4g_MROR zKi-l)cPec1lu1v|_z~>VTU>hkw$`#^;cB^iu8 zZF2#cR4vBG(B(cOd=K|sk*NI%o9Zo02i*7J%HN$F-Y#XJ!dH65oM-HuLkSA*^IW2yj74eZdk@UOd=3^f-Vjx&b+`Px^UoD>g>;>-n?- z#~6WrOy+6+_;O4Tw7P72U~hZS9DQ)2s5 zNgpE^q%fH@dbcJc)>Aiu+W2f~zDzRb+r~7>&y5VU8B9|(Kz!9cyI?PMs8Z>h_2@AV zp3_f4>eN3luO5ur@?T<)vKeWxCj| zUtqr{Qf5Zdz7fM`%-70auPVF8cFb%F8%a7f#OzIxjQTD5Lrh7M$YNORs5TRQL`pr( zNoq!qHj+%sN2P`;NCCJz)@FZxG%W@W0v=!GFgf}DY>-d_!_?dR>bvP6fko}h(?W^p z{{6oemHz%H0atK9SA|uWTs!&YNG({U|BVD!vK;Tnjy>uA+oktR3%1|SpV{rsaVRC@ z89B*2MoKyLW!}8iwUk;iED6#%=J)81f1)bwX?;iw3Qkvyg>t2hJVJ6?FPLS0UxCOr z^6b7(ORs1eh85K!i_3IlruXqLvs~zj@(47J^p>A~y3G#+gsY5Uh{&~FQsIpW=ZH~0 zQ!d?X8-7W!vrqSWP{n5P8||wkuyjha7Z2dA#J*n`Lk3MrVFJXf8&Lg8UCiDl37KVD z7uj|`{D9z3XC=PH%ik3awO?%T@HroRlH}>>E8?9PxC&V%jjEAm{&P49m0}&)n(Up)y>p}h zwhl+8!^c~1VDoXi*D7)+lvE+h#(Pq=x8U>MV)W)lk@Dt@H6peajm77xSe)rZw1gg@hR?j-V9p)-2u1=6BAOPr|~T3)@xanhNSc<+sE6xi%GZWUO9SOKr;FRK|N5D#Yv|FLV)DsC#uIef9shM@9pI4F2vMKt>l{Ri%1k_RMlL5bO5m5=WIzg z>w8swbZHc}gge^_LTdUIO7=+V?|;eS-7k zu9h~o9arwW^lyIh&I$%`R2x&u zZf)Zwql|^Rwr%T2!y-Ru%}2SG(>w(M7Pua?A7hCcSf;UiD^0ng`4og6gW0iY7;~<( zY%&Ent|M*S#$v7>%JqF|PK*p|U)*<$dnMd+U)9fBHFZd$WcvFtQ~Njw0z13EpL%`H zCnpGZKW_FBYOjE{<}@bVT8TDFAPX<$30Yye`#C;g$yz!(75k9=ggH8Q_|nR~eKi~B z(C68$wO^1P1L3y*ZtA6=djlqhP(1aPai#WYFHPO*z`S(m30m;4Y|Y`r%wE91Cojj0kwI6&Zd9zj~oHe6+9PTY@xN zs}l?RyR-v5>!+418>cRdiOa+f6}YL-HcH?Cq%0kNT?Xb~?b$X-pVf=XPJ%tkG+g`p z%CW;sowmI0jXTF)Qvvz;wDq1S9;eu3vwMDJbG^_DU)ZzW|H|M}^Ke|6Zb)ZPUI@(D zxun9ozIL<$ww#j-?50CagTD?5^`-xJv6HR4jx72gUuw-Cwe0KrL+noyG*H~(U{g`} z74vN8SKS+2{m6q3!7s+vyQvG4*l7u0d)u>qw@Tl}ADtluc;UvxCvOyERO9Ymb$CPZ zDamNPXqM|LYfwJryoF&P$;y2iPWeF`LCAD`^_I+wN3RxKQN69h#!9Rd8>mna8`D3D zc6=c{7(k30IA1e0SNIIw?VjovFD7RGF17&vds8_3!VhD1KB+Df5AwdW60|_UB33C2Y(TsUH_4NY<#Wq>mcB-$m@K` ze9dL_Fh{9x1J~Nb>a2bF4&&Ivld+14v1wFG*}+7m= zti5RKon8uF*ax}M|3IhiB;k{sD--n-C9(r|x~AlYUn?@wrLZj(YSfSRw$CR{^5U`f zy1F#OF3tqMfZL<$i=UMFn&0w|U#;0KVt}4oAE*@#CcX>qdFoYztLtQTf{LM4wAIy+ zBCKsCAn#ZScu&D?qH67ihCI8<>Q+5Vd3y^{T~*wYYjg~pc5nD`9vo2ri(LZ*8Ze|m zpk!=wO1%%X`BeN&qhLSZ_;;oclfiZU{Zv8I;^3VYcDEop&DkCC6d5ozrGkE84RbVX zEU&|X&(6<{9Pq9-u7A=2dSLj^eg0%@#-|E4alD+E^u+n}jBMBe2#*G7HU*crVfub5 zf6KHVwqJmbHfU$=9G`p^_Y>#dSI(W>Z%$9pk2RAm)d!3nD zfGReigxZ&X42;D%l|vp8BG7CUZhRiPr~%=QWI7>K-gMCw0d!VWW5+LXLrs1(2g`zC z(e_ksY3{NnoWC$Bf?X^78AZhh?3N}fi&?w8GsKh+N38uY}Nok_}Ah!~q`fzDI?$f@!9n>U@wT=Zm*LDh)~a&1=P@81FaMm{R+Xs{pe+RM*)$9W{1 za7F$yxU>79=a_m_3Zd`#le)xU(I!DY=4=M0@Gyvoiz7t(U^D|}CC2iT*hfx+e{gML zyfkP)rFslR0IIZ15sTH=-Y2C#<|HGGJd8`c>MqpY?SuXCwIN+z+CH?5%e;6grW^Pf zm_gX~RA7v2#--Cw;UJgXX_6rq3G$R>OxKkJTA~ zYJBvEZ^p9naC|{s>rZeBZq#YR;qvr$C(mc4LMZdAfg70k|Mj)h)zxu99y=L&%{Oc( zEeR~Fp&`W%ORh`yJUvyu#ON;S0{_S+Jbo(N)e5o0{_ihp(A3qI5`pC-;lZ+g1X)yH z?lKBM0jcYZk3lQsI+u@Lzxu$b0EL;#w0{EP)x2hpi{a=s|0+Y5*kNMV9;2i-Wpush zqOtHJ^SNWxpTnSNzK=Duv;AKb^sRVwDhf*f`bvX?U@2_?WR;msbhzA^8Zr9N7opt@ZacH%Rqm(Jv5SK6aAGSO8f;^%g(q9?XT zI%@pK+L7Hay<*%~-8&{a9bAG``tVDUeRoD+NK#s**ziku&Yf=KRAf#=r zb!F3d&AuNQq-y(`;Fbl+=#Z}s*X0BY>BZHhp;E^5n<0p;M0;L~&(R*T`z`B9j9-fH z(qdcHyTxRd-jekA-)|`yRfY{2yZ3WbG`)!ix-YR+Qo(tvX+mziqB4!(%ys?*F%}c|o zYt1`uy2C`jy2R|gmN}MzDGYj>G4zA(ox+XVb`>`f)pD?#zr>T8*sO(^kiie@F)BNz zySuaBx8`DL7@vhbPjpw9@0QS(aF>&!%Kf$!yFp$f?{htXnl7gPqUDF!YkUC-foI$y zQ@uZqJ)I1IX*5oM(t#hO=aD@F*}4m3{u&APjhdZ z!6NJaP|3h<-VgPRa3jVoxJsMUdY?p!xw_Q$Y>ZUYob z2!s;K8GxB4wB?Kww{%06bx&og(qK(?{Ow_0uQhU)m!jf$uvR`?`=xq$QrN@&jaWr8 zYs31qRPOog)ftzu5Th72Zkqs&wE*myNz9icXG+kNH|Mz8NV;V7DDWTA(`BFb(O4d` zk=e^F3}Z1uc0FTV{wWvwVPbf zGkvNMZ5M!zow8ne;B7v#h>pd*xVitg_PG1!Z}mek z{lAGfoyL%Ew4nNu2$EP>6N{&1O@%F&uESj+oyNkJo!}0?%B`ghZDyBDqI4Xoef(hA z1EcAgMmjZ+plTT5G)BxaI`$V9|9rNC^Px#4X;$Kib(#?!3VcY6J+GdsbUIf`mXmLx zLpeIyi?RY^|1TlIOyy(?GdSP3qR5nP>|ps@`><<31rDIzvL|axKMb&cc_N~;OB3Gn zWjeqf=w@i3L*00e7JdgE?!d0U`gN|d%=Lau&ZOVjpYwRSARIt;?Q%i(1^8Qac{ zEaT-e?`s>3q&!952AiemMFdmMNOj(g*0n&==d|od)VZvTG?RFv>!iLeK9xC8uc%kn zY|=~mlD1BEv&7_+r-BQ{yj&TXWX4` zD8rocJc#}2y?Cp_B3DJL{osNI-BT&oULeQV^8;!(3B=K9A6o*)?7;(nWBI9@n1qj= zu-)B`=8F{oSuZ!8C^UD}%TC!j1Y3f2+aO=xh{i*ACIQVjj_#$CiN#R zgCq$A1hJ-O!i!hHa6wfdLX$Aiv5AHoanQcEvN9j*axb0jWmt8TbXGU!YXT%SWx*Ce zK+N;!GPnuobjFR9O+@C^v8Z7_9d(D#h}isC-@xW#QWDdPt_-CpUrDgF*%x;)f?eJg zMSed5)MDZvxCjSSRDJLE&;8e!pq?FB+6k$wvtC+ep&q+d*7A=Rbz3)ARw6sDTnAR| zBNKV+%d(Pa;e;h`lfKMH zJ+Uy_$syXEy58F6`3wYE(k4h%dpbD6MhWJd7E7dd@_hPl=i5bS)~DS5cw3@FHj-_5 z#g`p`BKuWy|Nc|S$4!Ks&yMLN?eT04|Nh}LpRs;ybla$(vO=+j%bSfTZb`d(J3Owe z&LZGe>F1x1?ly~2>nP(;4%K@>*lKUVj43L}AyRO(cop1ZiHDc_d|D~SyFMCRYU}Eq z;ns$;Kdf_DZdwgRqT{*@I3g{Xv#j(kZ)=&$8&9au>w!SX`2|{sh1OqJiV*`rg>kY} zA)bQ&SDDtVn@vA^-}w~hw4i`hgaLxZt63*$15G#cK3VK^Q}*l$#MZbhn!V-V7gN^0 zy5h5BVPyx(k{F4`>uK+wk6P&uT<0ab1?|5PfPZI<81fF4rYd!*lxvNxl1wJZt&n*=>e;5X!T07oAityN3gFkhYkJdv-` z6MiQK?-<>%`OkN=Oj=oFki%aHbe6rOX! z$lk!(-b@tr7w4hrZ{DC+V5HY>1?eTLj3Hwo9U-Y*m5UznpLFuPqfM+w8~V+^=eQ~Y zTkz=OH%hGgyy}izU12o7Ozev_-Vu??d^dqC6k8SAck2GBzX@ z2rz;uOPpzFEjI_WKbwVM9RT2Gg#~^xs{R*n{tpB14|Uuyn}~xV*}T*MclkNB6HN=h zy#mJmaM>4cRMi6}<_BOb6@6G?I^gu6ynG8;e}+7?`ZcnRghAbB#IjdWma~Dq>-j2p|)> zU?P3P^9ACiZ%yrCNGP-M`8jNn;+=wygp70a$0VJ>s(Jjm1|rA z#!*t@SQg-w^W>L;b(QFsTWkX>ADHvWoK@ZEh@Dg{%qf=?9yNqk4Q+6cv@bP8Gm9FJUVBa3XaF#B=3y=Su3`a< z17ZHp#=xd24t(BlQ2R-da<}c+F~9PU7bRc^podR4EcF!$Jb?LPi64E-fN|2ueT|xvB%n|ZgI!fBu4?}_yCSR?x?~C7G%neesf5Js6~c+Yh~w=+@t$uuKtuH2B*#8e-~*58L}GLfrnj5w-~eF9qc+$lPp zPv&wY3931H804${4g2ZQwcWEFbMIA*aDBlORXqVLKrE164H^!kR=-Xah?2hwAbDH|N z2$D|*h@%W%3pl75?gweY8E_+7AmG#vU(z;4Osc7^POA0#AL9U-vpI%b8Hvku;GNsl zMZ-7%Y`yF4ERhgU+5Hs)>TTqLpDedxUeRn|hWT1vFL95zwG1#df}uLVBwbazE#Wej z8oKoK^;3&9H)C(MkH8?{@>mk8s!|=+K>b=-HxP|+t3;_-^AA1Rj|VECB;8&s^vtB< zCP)t;&`v4lm+e>5_RhGx=HbxnQr7|%H`lX=V??Ko*`@3UW8^wtr$w_=AVxW$Z}_Sd zH+d1aQ$s&U{A;gnc_=VPWyIeY!`^(9VCp9IQx8K8y$f)jx^BbSdV#GC`9)wcV_jS{ zROU*r%i4zXw>EKy^~=IQ@XFssTT5OZmG<|~(Yt&2lO%Y=yTOX>y$rXVC>G?Om(^I` zn}Od~pSm6BhLu;B$hqgJ0T==oq5INCy!zEhhWAz_{aMrJo-Y65wCv zM&Y5LDm35(s6r}%d8#%=TZFj10XZGrRM)NZ_|ca^m3B2NE1qR<4lPX~2r%^PrR0i5U=@j7I1%bGboL-`MJ*3bBQ%XSYuj@94W-J>r<+@S|R8_STYFFTC zcG3@uvW!6HY+>o*;F1AYHX_Mvf8ss)*bx=N_{PKa5Vt` z#0+=l%5c-?$B+0^)rnBY6^W)F;T14IXWaN(;g|X_r-?yRLfKInJ`hieFg3mjC|5N$)k+TiH*|ydma1)=+ST1BlSe(xx74H0p z1R&m(2p*dZVIym9X;b9pe5HZ#o=4SqmxFaVRc*dVyBTKZZS{4b3K&cV)Z!Iz)3z`h z5J`AhYN4VSp!fRVn<{gH$*RjiL;s#wfo1D&@g}Fo832{(%J?X*GxkZEPu|4C_f;xI zEdR|5yApp3m2F;1;vtT7eSq}EgQ>2Md23YG_fxMA(Pgi&h^J#AVl39pc{W%_(^OPx z1AZ^u4hr29BZd|<=6Ap2F}+4gn234|eHPYbkQgXY;YW(xiy>pyx}W$0QS%-CXbt-= zskF1I*^$Qty89ae7uD2?QZT;}?EJkL^FntP$N8>shT{rn^N~LkQR{Ys_Eh&I1v&5o z^$fCAo=M12_d4>?viCJqE>}X!8BC#SSL;`QHn#={XMcm}|Ei9NCVnWvtoUobBJpr@ zN_|otD*f^b!EvN42tH%{t88DF$=lDrgB&r*Gm$AOTumn6yGl_NU&ovGY>T?rV2EH+M;P4@a6M;1N>4pl!}nm6n`=AKGy%CtZR z0TWXl*|k3k3Sq(TQl=oSXD-V@qzr8Glx42|xPNon3_{D>ocvX&B@DpsG_98L4UG=4 z6?^b7zwN88?a|)ZicT;42A4g&3A3Z{#YOAp8(!1Cp35P>e{Y)D{qyqD9qHdhHjLd& zCyz`bv93W)aY{k2(WOXVz2Yc~x<545PI3^#W>pBtFSKMZcp$D$9L)`d0`s2W;W5)$ z?v%^F5M%~PdRO!coajU&uaI;cGCEkum|@lVdr<;|+|S;s3E~7K@xH<&DCSfAjr*|7 zBwX=>31FPb2xxqlmvb3$`AQ{bb5ceX7Wu6DTwv{kP(ZAID~5R<3NWOcdmj51r}Ls< z8bs#ZHrA~W3wEHWnEcF}2Z*k1*WJWER|}(=zOOhMH)5ocg?LaTy(!Zd>~y0g?j0$} zDfH23tgQcR0E<3M=4-~!PW(^mz160N9}v7=j>PZ4PbjGF*RaJsO&-mz47|qF~M-csrM7|eN+K|`QJwrQgBG?z7fi)<5N8Q?pKL*WC z4Uaoleg}VASOExNSCarvS2myU($8NNNQMu-B4bpL|HIi?M|GKXeg8*Fmmn!EB1niz zNlOXRAV_zrgmfbE0=p&1&46Gzz@`UOi}N!bgD8ZL+S7JC6+2VXmHGfb!p)6b~U57nySq zhj_hmI%KWb{<~0Lno(h_$@E$q{xbGE(lwEN7GKXboEOV)cXXg#QcwIdQvD97A7HOm zBEq>hy>z(^cNs2*cBlbK$^HEE0oha-94OP!h7Nc|IQQVQMAzW01C zt%w6nqq5|dpb&kuO3U$LLWv6HKy_p2$1*7LG5-f`z=AH+AqFZ0h- zhW50Q^D&U0wwQS(bT9p70a7-W# zXL9oKx#^0b>sR~Pk^$YKmYAZ84#}^7+^zdQto)zW>x47VG&XN0`&bO`p zQmn2iR&G{`QuVDi{N`HEe|kt)vy;%X;_6YD44cK>lkzIR&&Qdf2In@d+5w)e0;7jP z-7-G?Uo$K0R6wQk+fz9K_>;TS&!gr^XnOF>c?rT6!D}8N!>B-Uo%VAr=L2Jf`N|cmfA=n6;V(mDgoEvoN5!nrsYcv)Ds4>#oaXaS zF225*(zU-Cqv!BIBW_*y^E%b|!O~rJ4Dv^Nv-u~SDp?EG37t29tM%m2X_>C{+7L9z zhIT2)=)C+8rL%V^*|#T`_zYu`VmPWU+HmDqcrU2NgCkdEXuAGr5O=vHw0)}ST&S1D zWZk8QnlFp>p`JfCT5dS92K_RE)%Cp=FMw~Q_R`Mhl)8iw!zt&jVs){g-9>k&f}_Kk z(8Rr}RIdRh3mt;LLUr4v-F!UAE>~1d%BcXi%y{u^j+y(qKCnxc&y}1ugwnmPc*la8 z3Q93E{+3b9>L+4%f9z#kB6vps z;*DNO)0`LaQ&qFZ_@CU1O9}>)_#=GQQF&rkC#Ude^|M#&(-id5U3f3ICBx`1PFsU- z;%v{T8;)mXhv^GJhqMGPFZK1m;^IiL(|x#y2?e|_ciG?)XimWtu*%_4uiaHH%I6s9 zKOwbTuf6*(vd%=dss?NypoM#xeR39EO9UP>?HMt_}p4$1mDqr$+#5!AzH z^hTl|DC~^H{oiijhA9!>hOP~^Kx$}4)l7XTv6xR2KR?tYqfdmAfWUU-{CUSEDTyS6 zIJFug71VxClG^x$^2NshwaVT6JEFp}aC66iKkT@EwA-lrknmTN+&!@x{D5iOq$ta> zZ_7d3(BQ)h0*L2l4{!PJn$3NVJPOHJdLSD8mJ1U_ca~3ex#E@FXr5p;L4>5dGwP=T zbr~1-4XqEER~v|An9k$ByZ$SvT!phhC(=F+_NcCGnCDTE6d|Vzuz0D zhJU@p>fnmb)n8}?lfF}O*3FtAfbst}ccc#hFV=8^?L|Vs!67sal9%Ogib;#c%g8WZ z@7{*XBM+SdA=Z@;&Z~NqOAae%JfF7mOIvxy=tB7(v=Gz#Dl&Os_3Q+SJ~rBXlBd6b zk)s>^0vxjH!d`CPY+;@7OMWldIU3)K2e;YA@^cWKYv5_DI2#WKAKqkY+dN1XfE#%& ziv~G@bv54#m0kd|T{m?ND=or+b4i?y!u?2iCV<5tT8$=KhmhVj-n9(BKPEp^Sq&YI+;!C&vXutq zKWI=T7C-5t8rMC#F)T)%G>Ok)vKKqe-qa~I$MW6J5G9#C$3PeO$s8eE1b5AeO}RDD zMkH{mw3yluA3C>R6rTd7iRnb`jgPHZnI&p5Cr6znrZQ+W+~aaxX9qZMJqKLYdx$Vz zZ?|mc+y+O01vQEhdb+=a_SWYY3SP87rgeTlZor`FSE*~@8|+U_??};;Ts z8!n=geoo@!yI%>sx0j=Mm+o5IeFI&r@5k>&J+ci=eXG|Qt1+=4A4fz%Iee^6@iyKB z(q8u{!tv=9q<<=2KaWBBc7cXGS)?5?;bbbyd*H{3h#jr4b{pWVpKNv>InLb`Jw2We zrxN13AmPRMi;(u|ddlR3w&@nFun7DhLwhMOR?OP3ydH537Kw+JmR|Taw9ISs)e7cZ za5`>%!B{yo6z05Q(+pao{T@G}=@#O7ljHru87%OeB{>ps0;>GBQn&zccE1BxONu4Y z-ADhrmnk!Zc#^9jk5T@B1KK$JH3?h^Sr$8Qbo0&K9*yVP2p3Ow_3$x(UjQ>9sbvya z@D`khqVC3NhMuYIRX&}ojfsAvPeGlQXeg@!LLSIO(3oD!wQsVpKhCZmVDct z6Xl~hs8Q(iG|_+1+F#SFGZ13qMi?OVYya{p2s?^*vh*W$lY-+->aWzm6v zGaPLfp;riiqR)&rctaVLv{%rVxurr&5en4}O$z=-(}3P{1Z8{p5fmhE$u`maxftkW zme#2tYK!P@p)|eTd=1>_U7r-<^pon7Z2|G=!v@0uf06_PUASgDPpdB?L2eOY>ND3* z+p``8MzS0U#%HH^`flG9y!WTQgF*{_O%%PcZB-KRiJ57)3mO7o8 zmyvzUW*DJU1_qeQFrNV zE%};UWk9LReKc*1_f_7C{DRY-7o0qNq;w?I0J&OF65e_%4hC=oI`>Oy+Y#H{cO`>p zOAK1f#g4vPCh?RN0W#68mhZd`TXI=+gr^K?>_BR`SS+qf$sK&0NQ2d8JR@&oy zB#?vRdh2)k(-P^9p%VOHH-U<+$sjR0@kse@2T0Ks-gnTcUE1)7a!wy5(ZJMKxjb-R z)uvW_l`~GknE(FzGz+c6azuZe@c8M`2c4_KyPq6^VN$%VYwm>8#_c$6+3{2T>S+t6 z_xif0&Q*U&yN_dgMNL1%?=}_-MYr#R@CE7ldhHfH^-5sQeXV83;`;E2BQ#)e~_6(r{dh39FG(F8S<6W($%iYoav|)R^eX4fV-I<{I zhp7Fnz~%v+!$~3cd4R3n8W1aNm~IF)zNW{!+(hbX6PR@)tcOVMzTrib@s^idtOzU3 zeWF)r%bhPaW;OKGUg~ijQeg%cTsxX)0e|S%j`M#)(8s=FWG!>AaeH(L6S#&?2Ce`{ zjPma&6vDnW{xfh!?-~gc(4KnXpo-_0MQ#k0PXm;ss&);~q3j|$0nn~@8}fh9PwG&^ z0}~sirOv-5du4?vOn;O%gX(DH(nF8|hDxCNLSOqVK)gz0}>M`*BZV}-n zoLtxVb-?C*yf0fbA2HHEg`1qYEVMUxBF@asLTC2eoW)Bjv*m`wQPSyV!^4Uc?YV`+ zt&n?JSbCj$*Em0&9PlJ4sA_%-I@oGxE49Fn`?g4(S9L$;XjLyFiZ&u>=n>SlLx?E$ zUNXhW7enFz>EuI6$jB6|PY??HWE8A)Ycmvh$Z_F$uFBfE56|Y!|D3>Oz9z_-`sYMK zhuoX7NI{)nVjYZXM$ZI!@f0Ut2J)O%Vrd0OH7_6DW7&QuYhnP*Lc+78CtZ%Xh}>a$ zNn#!?RXwD2Dr1jwD$B@TdMCIsm2j3z%Kw%dD>phHx381PMD>~6Z*fsl)P-;40wXzU zw{W=oG{4(cyy|VRizTh$lV`15?RiE*Uf8y)64Ua)mRQh~|2{MUS|++KtY`v9j*RG? z7)$MICr5AL&4!J|{II)3NE$G>^+CXhxn$ETKJt|-kE^&)iISW?Tzq_}TW%9i^z03S z*Bb24x|EbWuj)MywdmX&f5h`_HMrFRcMB(fh)|TZ1&( zp2O+=2Yd7As8E3G4V;aJKKF!-kjGb>(cwtm@tw30gD1n4&w9nlfbgScdVZhRPKsH| z3|;hdE_CH}c{^YZHaLi^f_@mZDHqpHOfLc#Y4`o>JW_;E zXL5)r{gx)9TOuMFE*MO#;D%SuM+xokZzx&k+cm5`C>$8Y`nimo@XS-^-Se3g`wP>f zcG{)`Z*~t)XJTSn%=u=3wTd55+8(3QxYeW5Ium1mq5OO4tVsC-p~*`s&`kueCo(?E zY~z7~bJB^9ySG^~04M~M$6){F)If9$iNgg!1gQuM{&05n(daCw%4Kz=NB|-%S2-!< zR!KHbq+QLGBJJsj#X_DXuylYleV-UzM+}{D?~8QMAE$;6{#YHdJh(Z_MmstVfveCZf*dr>Qxhs_x+$Ce=tFO=aoX04r z>TzNQIBbWW7auj8oI9Q-9^>_L<-jIdH4TZGNbv~bEx_J{*^>eT3gS9hg77n&cd-1c+(Qyudyoh)Cd21Pk0zC}k z)x~n=;6&p1{}wWT;4wKAy{%?65ee0Al>avwhKTls zL;v{s+%ze2u;dH0ssbzgiQEwtc;p3!qjr0`&bI7dW!pUCdlqEw?@v!cvnM#N8_?MW z2s~+tVluLnRsT?Nqn~Q^qAmkP?eKRhObd&YR8jFHNWQ6<+B^X zzu8;3u2O4*0>x`_=d$FqEyDJyX%NOjoD`7*nW@9I=Rw zXkUjy(!-V%|MD43k76vNePFZ!w2QQ@exXEbdu_>g1gEpvS+8H zIQSaK5pQnB2J!NRT;==i)LIg z&!U1>I=*%`h=OrZjDA*ELb{c=6ehYMH4p#^wq&2#{e_Y!d0L7HMAAW^kp)2*WDhE6 z{SxH;Wg2Kh(;Ysv92ZtvknDP=UXCO#I_W{9GcXF1?0-f|LyHqQ820puC*7F4GL^uq zlU|(T{TG#v3sBR!>gj}De|kiu({%w(D^fgtaKcM@lDmByvTWZ1@TSF1Rsx(%qQ4)@ z$+{g0VucixY<~gYtLcMOT?HIT5cMZ*MZ~$<#pydgU#x+Uy@ zTIaN4*H!t)9KCIuPm8KcxnYu>({Zlp%W+S7o2(zPd{?oV`p{xIEqcqTdNl{0mL3a} zU(A0}o?pjs&F+cS-dZ&H!*{EEcp-Prfa--j|8m8<=ElWOPy@mMy&Z@%ATud(+D(?E zbRq8vIhCIQ5r0G4^2Vv~D-8&6Vt`alCCo{ob|r&%NlDP}Ee-Tu<33hIG0s0_jWmp( z<>di<=6SuRj;4y&pu-!<<{+1+OkTz}_H>Ase>h)+lAO7K8#DX|<|^youdfGiaDk$P zG{;rc{ui)3r+0>kP4-s#2)OP&HUqIxUTm`>z(+7`(qz|{xpW(iyJYY zorVkoH@<0%nG{!1V)%Ql?X3EgUkAxCt$^-4jiJ{=*P@lgKYM4~2Cq&&U;VShn>4(< z>Mey2D?eRaJJ~te>FFNI>Bc=Q!Rpnk93T@NO-Ej2*hgak*HW_-ewJ3Pq=B^d!>B=j zmL=ny&RQTYtal_vl=xYQxqADegZ+8bKYv%@BV>l5Yy`Z#S^W#=lBJVVRwpnejT0IrY{S}bqcd~q~ql=*3-uFGVhldN%7LB+V zy^6Rfb^q&2GxCN*!riG?Du~;rI5mp*Qbb;k`qCpKD5$$|!igmG?_tUsCIp;5d&Zb> zr_Z*ZO_9}~Lw=h73|D+PCIEcST(_7XK;P*d%yWBjFXK6qTE|vaX(|?~C;U0jQ>WHr z6{XU6@$6MN0?&kE6!;_)CkMhvO(2_S?`p^OzSg_w#_sVmQz?nGpb9;>BI&+nF8YF? zJB=qS%=?lKQ|ps>#tO+_pRR+m*D>j--Vj4W5lTo~*v?h)w(Uxa*w~yU#)}lfW>0f- zl-Pl}m}QrIfU?!aA{<;t(R;imQs}O~s<9#{{W!Pt$teY-wuiI$5dN=J9 ztNW$R+7Um^7eSxu3GS$x5_%PDDXq%+FG4!I^A8~#RJ&fSW-oiY>gH4>T=~H8G+s)* zy?(G(fm2ko+<;4t;IHKE`75|NlHxnGMRXpAsS++%* zOUq(U4Kk90aXJv4+-oSKe2s6B&+$N7 zYx>#-!w<5714Em;0;9%O-SY#}4a#jdZe_m?nX22HGabFLH`98NgF{zs%F>q{_3f3L zzFe3nLn{sBzC3P?7S+w9L=+DTqVZL&t#VybpF*%uw}=a6WF$qqSBn})PNJ7dp&pJP z7EK<1O`W5BMG~}p=7d$h-+#d*vUElBUCY4@6q2lgsRisvXYK-DjA1;MODq{uZ8Rai5p~T9O5Q*x;o=Jw609Q;$05-x98W`#FZ4u^@j5%rXHA@;3;39=vJS zI|W4QQ}C=a@<+G$u~4!+pM&bZggsgHJ0r*vg?en z;!|F|T5_xsH9;%RDWg#QdRzZBEl3{L8pMe!CQmV-(Z+r6yMFD_qr#T5@Nh|;hEuDs z=s|bQ=K%M=sv92;S%sh>Y~twV{S93D*?t3BfeRVC&Bv+!D4{~QnYY&eY|g`ko!1gA z|E%^1>&W@*0>&ooHgL+-VmcD|A`SkEDpAxwCzr=w4}5Xh-k?l;{Njf!)WM9urlb>SGO{WJE|jTC=Y~`h zl9NOGP%iQK70=0wvRqxwdBKCm$~j!vtd+RT7^wGFxu&}-nm72DihaCryVLAtzXVHo zFfl1)X9pQiHOgn_1Pgt+?ZM=4JJ097R9`XYQ5yxJEYkfC15=QB9cg+|uSfYoO`)0x zq23@P8w)`RwMpHDmm-h#(dkbgFhP*43W@u+H}^WU_@FG$^!Ncfw8kH~T^rlF0g@IW zZC00CMZSSqefADz3)?VfC;(}?a~J=9@aV`WF6dO7HKV>3OD2@F@9!=nr6d~Ca7p=# zr1(|pY{Ctbh1C-?e9n8G3vGdb)&eH0gruWS%2cd8%78%A+Eta%a`FM2T>^T zWHe}0qInubg3a3BsA*)$K)bw+O@>Y>D?X4uasKWa{Wf3|{II22^U)RqFC$r1mD`WH zJ*SP&oaY=?bfP-9G0w>%wY722WeEZX`k5OBPEI&56b@spAuU3ypnGbe&-UA(ZQr0J z`PsB6IV2oG=1gI;z_T*;;d^wP$Dg(9LF1?3+xwBF}JALDO-O{J3?e9Nu({lz}}jz}UK z1gr*MgQV>MO)*XNSZVLcM>{Dk2-+E-o`lgWwkxy=E%;k;SF$U&5|RbWH|U^3Exnkp zdt<0fa?;knwMRe>@&O{-)95RQORVL-&IFL3CF;kL!-7#d*d{-|U@&Uy3!hL;tR{N= zj8j9n87d+r)S*OLt{&pNssT{W3{i6)0icw=?V$rEauOxhuXC8}P6(a$m#FJ|g@0m3A_k`3ykphCg*6JZ*(;@9# z<6hT?oe5}81jz|ckqn}75)ES3Ytuk&<^HpvRp%%TL$G#^0P+~2i-&~aZBQ^{vihsi z(Gtv>Tb(dIj0Ot?o{>af97p1!Mo6nMpz(P@%-EjRGz1NJF#n43M-AmJqni&COtcts zgtX<=<6nZ%xf#*z27fm8`V~@8$p)FFJf-t(mpJm6w3ojj%e7t0RXq$B7f4K0bzF00Rnp}WA19`X+n-`IgLH0e85ZU-hg^5|2CeWWq2tiqZ2?t!Yz2I{#e$UA) z`o4+>+J_8c%BfJlwf2BDmD`h}JKo9*3;{5+nCS=ll19q(#{OxrD7(<~{(WRf zg7$;}!EMXet_5AlSFYnPf*-Ad%})G?g=*&$KC{Y$#>EfhK%< z>nl>y7Ob!8nD*$Cgsbc@Pmth6+!N?%5jnitEpMR?0FQ*>%_g-62kS}S-&>zcmU?>n z?8v0a%!~wl7tra})HT2&C4V%^zxCT9v3*KYQ_Dc|rWq!gnSA~g$dn+|(4({c6B9`p z<+>$iPJHt53sE9h?S{);xo zrbOrJ{yk0FPE_Qlfu>NBBD?WCXOWC? zQnmJO9SD*T!Ye2b6i!_ScIrweBU)o|C@y4U?j8Cn>JjYo7ByY=XmFarg9k0-k#ILu(!$ zA38pHXzNUt|6;rQB7Q-Q{?a9-;SU7_6ck)}1^trw@ysInQE*}i=;?wN7GC+$8|`+w z|44h^Afe$xi!z4t0yWvdPGU z^7M6MEZe8Fed+}=5@C3KHW`TXkY8qHX28RBfUOn$Yp8Q7`AX$%)uQOsU4#T?CX0kxtO`d7_6wb0BgEl}3T>SfVG))t7=IMA}N*=ybL^DDhy{lqu$2Qyo= ztn+>s$K)fisk3HC=my^N_nDSV9dFPeaw!cLPQZvjW<;KjBz!p1fhS?2rtAx}dA*f@ zG6ZM@kzSmr0mg^sbYS$0ezhYlImTE^pI^Xd4hG5Cm*V%L4ofYq zQ+7}&xMNK=B%!NmV1BHT=pCUjG)zZB4i{3&zzB{NW`5ni2hk`lbjN0vm*H}`{Bm)G zbbj24R4hzLKx5*tCqFBt9D5^YEpz>8L`v`Y0}0hwbVtVkDlquBLuI`liu;)OP6G_y z7(!hX3M2ROOgUxIw(TiY;W>%CwNWS})p|7X;&N>+^wPjAG8joJG4@stV$Q#?Wn5~a zft+`HqtEun&g7%wIt+;jG+wP(?c=gN!CqyT51G^KU5+kR{-1u z-=YlI%cVxVe6sDgU$3uR=fC$M6WVPMk8&_O^pkanLY4%);P4X`(ztI&Jq0GM=4e*4Jj*hrbHPO+oR(f_eE_BPB9D-OTl00ay_?O>Y@^|;mPw!Bu1ci{VU(b<{ zP9NEQC{YXD-xuf|9W`uc5c3Yz9wYg48M#`-V@(|EU&0q0l*8-3si7$*>AE|UmW|Op zFfa!ko^*nBrF4EHXrPgWLOH6j*kdv54*u4m*sgDCD3lcsQH)-hR%miZFETALy7{SN z=`*(v>f=ild6G1#a?w7y}F`#S0;yHKv zCiFZyg*>kM)OfJxCq3vSlD%)3TI^d=@k!d*MiEzi|0>RI>m#WLsHZOo4DBz6h0)!? zRX6s2d-eq1NE!T5HTzAulX}mc`_SW-V7%|pgQ0Pm4SQzP&WBcKT>FIY6+0Yekoedo zV@rF!gz^L;0h+jauLw5^S9&2*0=VL1Fv$VZolK6#@eL8^=_j6w3>2Z;YL13qDnNUJ z@LG1R4e1}wZ~(?hzGLIqvzF?h+Q_ma)|!C&N99G5ejcNjdRT$wKFSIpjEz~U@h2IO z0hlTo&G|5<_q9e=3an(?wB*6QfrqjNY$31&Fpvk{;!ZgPK-KsD%MJ3oCqPImOV(MTlrdA!|t z+~x4^?$KL&^ANWIqt2+ownEzjt3xN@7UHZLd?&^M6ix-#IZr{dsFnipho6XB?t zZoUgV)~6}W*)$Usku;B8hv|d7DP0KHV#wscr~-c(ibP;3mt7AaW%*D_e+{#vhU(5G zGW}EdGn5oll&{booQE`w*b4iC6MB~$tCxPfX!8LE(K`;f=u{*oEx*{U^2kYL%r%UVsZEj>#j#wH zpI$TFwvoiUQmBmEMu`=LFtQAqFTBQNk!RWgfdVJ;vIX1=e8s)W>1m?SOG*N``)xkR zj=PQ#vtKP6d{{?8UQ_dNw=*oq{yLH}fH@A2`XBk8u|ouSelWC0DO90lo@Gf7t`-O9 z@Apl7#$AMBSd*TN}MF5}fUB8s1W( z2c-&vae6%|3RwsmKgx*1uuPT&d3u@|eghaGe7wGP((LJMi}O`(i+2_26jI z4Yw1Wh!<$JCr^ zX|PjqU4UH#dL}QBk%i-NOG_u7^BZh|ROES6jP)-gY!x;@%r-$J|EoKJu~)JYMqz?t zA7J+-$8N^%LKK8dTr(n13G3i&Yscm-uEMclQn(vQ*cknkAAzKEOP)8wh^5KghWD+A z)F={b?>DSX7(~~s83v|2T8ddN>S@~F*nk2w{+hOS#g9;i?jOlkyHoocGIZOH&h;*&1ZMW%bBQFIEpq`mmj}thkF_OL8a12z{G5kwR%+(?Xp5A3eUw8;` z?jaLira`)i2oMj?=$mi)B`vI(sxg#+M3NCClE9k<MeIifoPoi`$o)G(gc`qmQNi%UC}d${3;g2{o#x8Cgtmkln{z3zDx9|qEzpjhjuqKH8& z`CAuRvbuxX6xz{(&|5T2C&6sfMy23QMd$DZOK8S!Ia_0%GcgzHe9={$j>`n_?e#|T2+#HLy1=l zdumIxN+BfAmkB8=Ydm7Wv}nxnDE!)51Vq=NOqA02$EQ zt!}!H?dLq+5!AE3Hj;HVJd!VRk65<^+@*nnUuFx%;98TcL$A&fSYm?$+eg~b|h9i`JGz>6~ z^XDly>d(K|m%~^j8AEdsBj;-l%ilifDYtA|4uL8H$0A5hURulC*NPeps>K#{YW6-a z%_{HffjtGy{B20j_5D*|T=iIaoHux2DaMNqH37sZ-}hl6SxU#DaE-6XG_-f!uh(a~ zea87$@7dFALvCu)xV_*jIaCWXD4#^;gu73dzusPN%J;NZ0@pG$jN8;U8FT1$sKlkM zGap{)c4uDO&^qz`^bBACaHiagT-iJO#suiOdG$M z%jeXc7kSyC@rDuW$9ra85b{Iuz0WRS_0!&=LPOWAu$Y4MH|>2V@AxhJMboX`bNCLn zP>EUL1YtE7%y#V?%nSTv9HO2jVVr~qtd}?;e)^WVoZC5S`!yZk&cWMT;o?XjxN?Q5 z@^FdZc-75}lp<&GgU`Qs;s<|yG%Z=D)cNxn4fAY-q(fjWc6}q>o3s}$F-%#|EGt^i zZjH&n5SDj~74@itK+?AO>!eJ^@1RYk*mq56x09rAHnV8E)4@AFti#V+NbK`LFvN*l zv&T|@HZGZ+!QUpa82|j6)x)WFe7x>Bqz2`mb)y7_9<1M)0FOY@=@g%wG)^4zQ;rXt z;I~Ex3R84*sML|hskCBt@G|Nzu9@n8Q@XoBRTpnKwhz#Wc?#SiBoW41Nbh9k#ks={@LahRU)s1=nt<&;DMXrA{_p7J{|G zu4;tTTy@}1ja#u%NEr7ypLa$fGg;Eor_-GqYTZ1n$UQJm`Dd?=)PQJzGWXBxlNMDS zHEpc23~Y#yl>Vtb4_F(!4JCtxPL!&&^iO@G0>A#-RLW*iLk!#etx18Roeyomg5w%+o(~I3FIyO?oi;GYQS67oZk;zYa~N~6_exGAW|_}m#wvjPm1)-PZbW5IJ1n;yPi^9W}=SQ z5YFUkvG3kRzxeFVLQ_TT|B?)M>7Tacx_ZiuNvg(R zg65E?yS(G=LyCGD7B;?IfTo(uX3e? zi3Zj~H^ajoTrBKsHs5_@eaQ&ZA9l#zq3sE6&T}*A_)&!QwD~ly&sI8r_LRHr5*JJZ z+o5JsT~DM4TR;ycuM}g!BwecaXi)rtjFN`h<{r`ShndZ@*SS4#3T9L>wPM)_&Jpv{ zXUhNU#V1qz%jSF}IFK$^5C?Oxq|FOjttAZuCki}5*mGj<)V{bxN?OJAtZb;kX8>~T zH=Ov1q@5l6J1!;BPXuw;2NTB|1~%L_+et*Dx6={w8|WVHt01eRIBiv9T-s}x+F`mWeAdDn`J zVv$4U%f};x6UdC*q6esDx6Dr3Ww$e9&rI73}k`U+eU$72W&{$Q6=v zRz%V7$anY$t%&|2xV$l;x3w2~eni<^FWRDWX3Mv3^J$Tf^IDdRvrG5m9t;%ns@&mfS0_H3nE zgp+UF1hrt2XeDFolqouHl1|&#fPr!T#A~0I8za4FNpXg+D+XPn2XE(?{U8|CD|UG2 z_h0wF|0Uc`wfet*{rAro6lp{ae{t9mxKW{c3w~X-)68n8Ej?)stt25wrTh5gtS_eH zynlU3EsRcLlJDlHLP^p&htO;3N=i-(U%NUSLebJg^@SKDIBrz2H$+D=6tA#Bqz6cd3zi7=p#E(3yO!Mm3NJ3a?kmnGRB{9BhRZub)FLapDXB{ zq;9$R&u^koKA#FV+Z;mSC(VVWy*r6lh^|vgp-B*5qscZ86&g3e{PKu@DRUcnG$_d` zUQu<|>sIA;t=s0I+&w?8V=feN_dNZ!Z7x^YB0<&LfN?^5;1_5fjC=jWdDdwJA6>Q$!#ycYr#(w|C2Y^+z92rdudFGkCt$n zC`*Hlb@+xz8`&x^D&3&P?COhHV|+I%!7txfLoZm$1m7Y+F_k+w#J^?}f+yD;c6xCk zZDG>ua+?Ke8_UO0Kjc6SwIG!7lHpPV3F-;u=VPh=TFl`sv&z4>`~Uu}^rB&d33j8( zKv-e(Wg8U_vyNQ>_GY?LA2ihHAmx!>U{gBQeYMhLDGfmql++hXKI^6_c4~rcjk>;{ ztTh)1!_~uFNd3z1Y23C-s@6A;tIY@C}(B%Eqt_pE@7cZM-1zq-}GmUzfq~BvxbYD z&|LCwHCm%nM4#%V?vvo5ddute*PW%FD8YL~4vH={y;)xnrXK7ql8`nXm(C6yp{@{z zQ1tX?a7Mn1@X6jbdyA|N4n(gTtd0v=UDV;>|E%uj->WnH^Xej}iT=4dxY3N=?wcVu znwX7t*kKQOC|ts*^c3n#mqPPUMuJoX*jyb?|N3H%zx# z-08dbjQdj&;>@7UwNHwGw z@RBMmD#Lj!zl(vwdE2o4{yf9u=G^2KmaU*@n!s8vKp4otBtR^CSI z4QW!l@STtEp4x@y%985~lK~Skr*&&)`q&coy=iS0hnbr4^)BE$|Xq$10)ewvJd^UOLzO`9nEuO?%7 z(+lV|pYmu!+pX12&)BVrwl6;;dsuDb9=XESz1Qmj?{-b*Y(gB~E#vs%*x#G4PU=6K zPbm)$?mri?P}KF`n@?hGcI!ZMZ8yK|msV84qEYHtfozP5a*;|m|LLClr8Mu&JDUfr zX{>>)fzvI^#ENGGRwa7@y?#+60Z{6}->dSqV@$Xue3`kJie#xU^y|)CH#C2?;MPp>7gI z4tePP&X|GCWb~k9$*--+aW7t(TNR=Vq|S)8xwqF&U=yR7bHt{&w&c3=q_vVL!Z;_)gI>yVwVkvIOtQqq{Qf?%J zHRVpx#lm}mxq8@l(VQC}@mX*5T3g8J!QFg_3wN^{+|3PeHsk(`zN9PY`g|99>X5_~hr4Kan|fS*xn{jT07k zwHcrMM)~X3#Pbi>TE~buicsn1MGg-b&4NT}pO!kSCa)O4&GdhEbo>=P$4h7oV zB&4YHoTihv*2`DkYg`nD5fX7U+Hjb7Q3TKupd-ifjEBl13iFy$I$t@Bwc)bkpdOg+s=o3vqKwd4EHz$XjvtAsy2(Z}-6Q@0? z|M5Z78?J}`w;9TCv2~n%B3yx|d}^e=rZ#oy=@Dg&3Wv>l-p?}hzhv*+Ap-bp(#;N& zL2%5}b%LlRGyQQ;L2Ut_d&xSyly{It~CAztR%Jq?{1>9v{dedp{D(`*z8p4=M)Cr|Iv7uEOH89Ak~GEt~^zds_m z#WmZ$SZmQV4T@D&)V7&6arnLXR;pFrUW^B9S%tJ0vTh!@9Oe~^jZdRaMm_S0z9>b+ zHc4zqJB!2iobPQQzKYmD+!LR_T^st0q~F_9>N0H41;0we*Jk=W|MX_I{pK9b{o`$- z(jUK3;V$;hZp+CU;MNknZ+P&aOh3p&&jvmg9IOy&0j_!x!Oxl27mVrGiL`FbZUo=L zLoG0f)W7z1aenzu@hH6H1-bq8H??t(8BsV1?kOcODUzDTYyej6qp8Dye7@WBIl}Q5 z5lgTjDH|xc^_3-o!lExl?v2*GB?=cO2j{(HM2NH4F6|e;=;Z>6e~=`CH(KJ=aBO8e|vDfV!7?Pw+EDG?+Lk-waJtuKhm7uev?_& z*MNFl`y~&I4`D=!t#z5!y=P2aq?x(!=YC(OoK;#GR~Y%=lW(@Qkw{~rVqY8%1WX;5 z;ZO|LBxuayfn#tO2UZQ<4EXRk&oZvv^Lcc)c0xq|9m`GcdQ=84>*q|Fk-n#;RB)|e z^B%e}ZZAwWf!$r@jcv0?V!-n6#T)qT7?9{@eCCm!Y~JA$4PCbQ#~bpm@1*cC z^^7I$&{mxjEHV_zK;UW{%_@m!P@i${z(}8`tzV7;t;jUdISKJta{(JQ*LVW#r_O}t zY8SE|(U4}j1n=mk8eI-?3V3f8dqaRrYW}sbdA5-Z4@6Hv*eAD?O~7R55tcbp&&i=N zmi@;r23Zp*5ln=mqbs;UttS`gT|8biE34~7oUm{J5_*PDp~D9 zokyka0C!uwe|K0pm5YmOS!pJ?saV#*aIw%Fy1k5cq()`%k+lE)3UUAQ3O$im2*4G- zI)?xLq;)BO_@o&3?*!`Abkck%s8vf)#-rH26ubjAZpcPSGS&2*Y}Ef_>@A?8-1@#@ zEDY)xGy?{WfS`ah1|T5K(4ivT4MPk{DJ?18IfT+Rpduj600Yt@ISe5%G|aqvJoodw z?|nS$TOVubIk*;j?b&<(|G(P%imJ8pd9D1`eX!~%tch6ON&NjVpc2hpZP1qaYAtB_ zBu=&-jYE?RK*4NRB$H~NO7cum2m}4zM+;nELVI#yn0hEh*(3(&y$56-Z4rmF=W zk;BW+;_rgG;``+d9fAmiE>sFkW|7!s!_SLF;URX!dDNj8{(Bx>{|Dbhsm;@7@BM%! zHHZDVM5lwE8vkQg@B!Km+S?KGX=`z4I!c~VRzJ@BS}Sj7!sci)p@35 z-@Z19?T*xtIZM(Yt9k0OcjDgNn=4LJa?CPIn29`Ii;X#^>U5swI z<7|pFyCwjVP&0Yhv~B8BysfeHX;RMlSyGXt>r-DY`cR#NWSJ(5d*I&9k>b55^(N3~ ztv!1_-Ja)G7>Zrvwsy4fwgOz_)Ii8nl`1RPm2;Z|Qh-*zed`V&Nfm3koG3K>?kQf1 zsE1MeT)$j={16A=8U6h~p~?lSf2os) zmD#Y`nz(zGI(j*&#tC}NPkdr1bsAmk#`E$z$Qd%84Gpt4YB1$G?VW?FL;<|@d&6mf z4F|q`QDVfS!DuSPg?_H_9Le$c-KkHX?tXi|?hKzyq$ejeN}$gRmE%8GMsn5+w8=y> zxz1;IhG&yNz5&G^;+7Zm{Zv;O4-|0Klz*zy*ZG+pm&q!={xBo7iY8U?A za|6DBPrXS8)#VKBpED;J_SEPfYx4S63FC}is91G+w2sgZqJHw13gl+h{)bHM(%~wErus$0b6OHv-ef zEC;Gvvx)GUT78&WMqGo5-AMd)@+OmO^5Q&wp7eE0*YK{b%j67Ay)p>Ws+rWc^gA8sT*5N6^FXkQv+me`YDGO8-Ewd5LpA|20z_!_XMGpfUDKT z7UcA20$XhdWHIl6CKLNv=JG?FG;V{gaCzo3H*wM~%n6nX06op<>i_JwAh6%E+I9pl zkp4MqQ>}kCJOAeO$E8ZxOIWweL9PTnxeqIoQUF<$N0DKL1V@TpG$D#nRe|ESQ{zmQ zC_MlKaMk!xD7iwexKzDsL<3+=jr&TrywZR3y=Vdxy~#d@HNR z`8uNPE**LDd&^PKFac?>Qc^f$+O=Fu9+S^LCImD)_q)EFhoFp*vw|&035I1!HYaHD z0zl?IW6q`Dka=BJdZN)0N#6G_qslV-vx>-=%x@pK=8orqVSpA+z3w;=a=^M)?h5*_ z=A-P}$!RMC0Pd5M0^ybM!=!PB;)*0uEjg@?fklCDd)muC0y``U_6tReTL~%U_FyUbd{Y&t)_ zmcCLQDku-id{DHAGD7u3=hrXYlEH1mr!p6Ob4TJeP(p7Zinmf=E^&!3Up0ShzEazNANRK=+fSr(uR)7jR``<^xH2Q; z|0yG{{W|a=HgEKSTd^ao6+%h+eo!uIT2wzL>e9+ReqIgJxl+Loznp7%3JU;_!v(6* z*n)DRGrKt+y+%cH*!cI9VZjH`y%s;+n?omj=PzC+ro7FIyFh|G)eh9=!5&U!G9*nW17v4wcmjRp02YPu&x zO?>`w=(AfsF-+#5@XK}B80mXuYkLWWL$f>>0INO5UC5Ky{{p!*z%kI0Q=SQzcK{6F z&K&{1s}e+*?3tDRo43J8Pwe)7!sv1A|1v*4|56gCgTMGMmxiuL!fq5O>ZzN9Rjx%E zbm~#x+D=iR?8vE1`D-Gc^Sg&Hh24>FWa8QiZ=h%(W$r=O1xc1?Kv(+BO5`#GggS4O ztLnqTUwr>0<4Cj~ehcW=6}sM3Ehk+5D5f=a&sh1n43372^3s{pMh^&))1i0>@-lH! zzSKQkB043X|D$gXxDY301Z*B8^`Dj9?q6hs%v#PO4{YnP7^KiOSo~V1k(ZD?=sOy> zOiLTCOFmR^*Jyo|2MU7&QGY-61sfZRol+!IKlN&J^S55`S3Ae|FOY{BcO~nKIQiN> zyuz;y_B zrGS~K1$rYL9bKZRIMI=E{Eyy^O9c+gYfun8Kwbau^+e*xUSa)@N_XdnP#sb~-lTw+ zjbbZEoG$Y!rIwi6he2R7xdW#sddimE04U$j(8k0XR>YaM0?xf8tX&yb) zO%EZdYO}s)`4V7@=QscjA%YCKdt02r6gYXJhh}|6_5;eBSn-uUSh+SqiPWKtSa`xs zvn*K0mBmEbvYajoCy@i`{0xo<;9U+-@7O@CJ7EOPDvD@<=obFntkS7L*85dB7@7{) zL;o1sz`rdx$4yWX{-?N_7n$Ldr4&+>7^-Ts)whRk2?=~=KO=j#UtjdxW0Fky_@ECz zDgfWwKbry|WuR>a$-vChCm%Y^v(>uCl9T77)k%>lI#+=-5IFGVvwAYPU(bV1M(rtS z06Y7*(2dXepk0*ml<4@SiBCSy%1{iugPqh0k!#bZcOh+{KM{vrAr^;rGk^SBr=+sq zrH*FhZ!~WaB-rA3ow$|G|GAa!3H+(>TxYtpG6|C|m*e1E5AnY`Dqalv*ZO$o0pBNW8xjeJ0_V zA-gNF%Af_i^N1agov7?@4-U^`k$JCusL{1=}2X5EQAGEkNjg)fi zK`pIgjJ5}Fz7zp6u|~#2BKXtGTKCXoGB=Y;b}LE9W*~8FDB-M(ff#BtT$;PO8hVFu zrz?xQ9-P%9{WSK0fuh0OvI;aPK)wdpNCO^>p|k-ni8sLxb_DH+sbS*yCtqdt3p`<< zw?VIJg$&;01;CmPDEXJUL9dMhGU}mJyV=7qu>LF!JbHeE2+1(N+HK;nApbI~7i|CZ zfypmOWvGgGGwghRtB-EvnhC-8a#*=21ng?v1E%E@U@ZJ56eY&}*zKZJ;&jQ!i@@f0 z4qlSaE)7?`42eW*huFeN227qRIR}ad`k7gVLx~;)8If9%_&yCB%VZ4h;N<;sWnIlb zS~09@dHgvI(3r>d2Q%&RD2f;EUV^;2$9PW^RmP>@-lnTelu1a4yaWt!gE&N>S2dgm zX4zl=yTwK!2SF#G5I(#9Cr%=N{4a3Awbr^oUTMoZ?zg%m6IJYSgxf_itd&|0zy7Qq zTBGLwK@zG`E>Y4!Ng~-bXR`vIv&}jR2aE*@o?k4~mc-vAcPvw#$eK|bxw5*rv3E`| zsRE6^Vf)s6!^h>3w3 z@SrCjr84=0CeGu))-1|~sS-JKAd5dBX5|E$tk z6mbgAfhF1t2KZlN0IGc2=cVm;qW|y~{rw8YF{qd(h6Sy9V%tZM|39ofZ*)votNgc) zK}te?Kk=oUwOqxm+@lg#817e@v(xM~k|(4JCQK~lbUZVs8Z-ks5IW8Xx2j;L3z;HB zI049}nTYyX#xn*v9f&jWDj)(eJZnumlSE1qkh^Lb2HN30!4Yo8lj{Tgs1_;-B6$Y& z9|`KeWB_shjTrq62t8fp{hq@=?Fh;|`=d(wV~i5bWP^NK%ICGM;CXlFkIsQ0 ziB<`8lrjw1!e?GjV26%6tgDb!MES|`xkk2fqJ>=N%a`Qf4s#F;NZ*nJKJv9w*T%um=Em8GGZ5c->og0Q}#XAGfo9Gp94{aJ5>ZO6`KLH z^V*LyBnrTQnWcRG>A%dhP(d{rPkL6PRQe)}g8LPy5g(oNd9O$_gqR2Qo4jm~p zT8Joi6_gv%Y15BjU~w1%t@6s+ZI09AT13wcm@`bEy-fp>^6MEu^2x5Fe=EhzQ}HI> zX+h8ai|G#mTn7F%6kM+JUti0#kt21u#aZi1BqV8$AG(S<_OzFQ1!yL#_XVQz{iDAm zNua#PP7CH8h$>VpKZ!sX?1CV|=V;Eb!JDF^8w#M{-;IGNRwUuZXRU=iIYTG{1EyXM z{LDgiMb;28o9dIQ6e0epk{e|jk~=FYg1P41_bm&hz0Bz8R*ap|reuLvL0r=XB zU?$T#b_D-xa3mzls$diUw${&@QU_40wY7DYj1gLATZy zz~ca3Vx}q=7Z(anI0|p#OVrZ4*^iG1WsJ-68l5Tz`w>#)-r0KH=X(;lhI^Iltrw*X z%%;%r_-0C(4uD5BZuKZe+`$410dy8%UJbVF-2^+-5-i|A;Ihtk9CP}=Dii-3q5WF! zJO0w_k2Uiz`?G!EX#ZsarNqf$ttZMtn?=?x68cA_@5>LBGt)Oe`)lCVUK8R` z$|W|1AsUKN3h6Y)#W#KFs)H*4dzZQcvA-(&b~|&_b8rVgZZ@QYRRP?oVhk?&xbv)$ zBmH}k_=e%nncahC`VUjHt?>+EICa8qD7A<+{`j*U^UTtAQTtRS*>{nV23Jp;Ebc8E zEt7>f$?@X%aTm-~6`+qxO65&4-yZ2S7lt0|D-5<}WRxhar;VF-l-({yZ%*YE^A%|t z$>#J3$7aXgii?3DhA(dlCW-ICN)fhNpFcatGZkt0LUUL=(GylBo_5$ar}NFk(h;3FlPU8>j< zs^0$nvFMAw^=b_8;gfYrzoa@tX-0}ZwsVC|-Igy=1~*R%Sw3zG&+n5=JSk#fUbdbz z20u)%Vt|vDZjrMYW!CZYE48*ZR8)=_h`u2pMmQ!1FP<}^5n>(BesI~Gdbqyv7A~4g z+_+TQh>$!P!OTYS7IlW7Z{3v;yc?fUIO0$a8*B0Hzm;h7J-q6U+*#KapoIiK23u0@ z8~B%$Gw@Y4wp=l7FNSy5YHfPne{~F@R{MXA&lR{St&GFHo9)H2W9k)dTSw>a48$iU zVv7pWW4h6Y+g*ClvLXdakazor*~F+-Yt}(c4F1J24&ynj>8GT*er?&E;E*c^NhbQ58lSB+4F3UJAUpy-4hPWK3otL;g)_4-p z1Zmr<-kj8hm{VlL=eNydl)j&T<8KG7sz`hOk-i0aSI4P~)QfSo_0j>TO!Xx*%XNzm zoBf_oZIWMl(LY9t3p}j44bDT8b*9W43m-SO_p+4O_&FM@9raG=mBbUq%8)jZ=X{J5 zV^A*#zKzi*6YAe~rN@lVN-^3PjaPE3dP_+Zr&63N=5?^FEWOoUjM0f2p6riw*sf=~ z&}5v!+`J80SmU(cIdW)mOZz#L)7V5P{wa2XY0yCrWb4X)2ny7vkq|YoqfXd5%!Ng% zUS7;UeRHEJ7LjqXYFcWN!Qv?W@uTpR&oL+&ZnDaNQ^w}i*rn9EClJu z4|^RJFUSjC=>HCs%GWbpd&=|Ci%4~ zQR(s6gP^R6)mdJAMu*XTT};l^Ryqoxcn)oooxemB;Z;_7(CgIlCliU2;zHplobs82 zKS71>pEdc;K#5^eK?WXYsxSx&1?kOTOlyw!lR4;os}G zMXXk}e@Tm??x(uYdOFvKdVHQQY`Ye_V727QT&v@)!!q=Fhr<;uVu4A;cDNqY8As{x z6KqSHe4o8L3_WH*?#Bw@AIh#djWkv-wMrPd_P=anl|O5{{e?=$2HRNw05Y+(V>Od( zI$pc@9!OS-I`AdKq}%N(Z8K@kB8%9(j56rKW-{f(NSv@GR=mDp@%)n8cXzOHRs!T} zJ4YW-L&^&8)XCNzj+pX4j^vpbZlz8}q*F%mE9^j3O1;bzg^^xrqbJJ} z5iOQa)M~K~QhxSPTkh99Ch#VNUNNEJx&}J}BR08;S zj=%k1&xO?_F5OoW5)y)~6cuQxt6%=LVO*}Wu&|)0s33xyeJ}oLrE_v$||f5m~{>)0It!;F^4Xitf|$%%cF{7)t_XL9D3A zA)w%1v15-X6*Zv>)L(Bu4OcNX)X(i}ar1e9>x<05)OL{Nr8xW2F^!DIm0jy^f}c%-FDwftmdPP>Isf=e%_zhCRAM&OD~s#sIKjJGxbnQm}^yZ>=Sr`>#OfM-Sn&u~)KCE`sr?kX!QM z%6+ifu1F~M`}!47q!LDl{4PJR@Wu9%8^p;zV$b7r@rlX40ueL_FXk*nHw$&R`lxQ3 z;yQVbpgQP61Louz9vbGE+THi*2kqyb#ZycW0dik$=2uP=R(Ke9Rz0*=5<(Vj zj%6Rc_Vqjr+PW$PBW$EJ+7ZMPAS0kc^dIhCOhTnKE1NwimEgT}PuaeJAK~P+^T}uf z3nxrq32B`#2hciE1C#aip%K-XuWuQUJKAL<<^74Q3rnNcsBE2T^WFj>dqa(sBVjsn z23j+1Jbp`MCM11e%PRxElzT_+bf&bZNE?c$?|H+JNAq1Jn~^4Ng+o--$VwMa$j;%B z#J%?7{)o4uDIo~A+~AKC*RKI&K@3!>C*Y;sPX+258KtHB9a_aF)JU6a85)-4=GL^h zq+Kx;0(k=kAOOX%){YkZgwg*tlW5qLK7%xlLisy4H8BQ%)Uc z!LPvRjoL=CH=yHVduK}b98yZI zt_xjDDCb>LHsu#~_a4H_}n7#;2*HxM^lv}0a|RanmIcBCSmnZ6L=0Lj@~XX3VsfD4hbk(KN8mal zbOqdQ$06mm+|;kiZJ~p$=N47S)n=>lE%!)cJVxJZQrXGp*Rrs(-uFrvC?bK;{ z>tMP0ps!fvTsd7+zlE*DjciDxf_3(kVw#vUjPYaoW~$22mrcm2qk)X3!|tg05)Y*q zjo|j;Wi_*JyGABcg(C2Y;qys$?6Mr9T7FF~ZOhF&?QUt-CPo?sGlrYlaPNUPo)qct zC(Vxv@Rg4UM>EC*cWv#eX`gKd<@D6S;wdWleFIg=Y9)`ElXOGA@~&=1*SQD>>xr5P zGwTAGPla_OCxL(^gbTdwqDv`kH6U|y^>!*)Hz`~-@O}zjy!IPqGQc;ZKYTrv34dG3 zAjJ2XX8lgXRJ;!5&@$@ccy0MpgIXF)^(f^^vW;A@cnKvC2O6v%hmTN?*tq$n0juo7 zat?XSV!rG$5Rk*CbRDlPmjgh7>$%%s2)~)9=$?m+~Hu3a#Hr3%!ddyys+uf?GE8%0=1NyM<*QGDbFQr<88sv5V+UhU&7ojYE+lLz{FW*YII!I2_2 z|ES}cynn(864!ahcuit*^%1=I*?HXMJPKyX`tpZiA#yz8;|%ML4a#Ya-n3;V_;12q z`nMzDJ0AV$xO1(iYkKJtb0-MSAMs8T6I*piXK^HKW5WQ4G3gUWn6_+%Ma^3~D#er? zA3jAAqI!v|9^va&;6ZT60fk#B#uh}hccQxEgl^PNO+5ZOes3wkX|0A{%r|;*Mo3R@ zE>N<+bT@20T_0qZQ=J~4qb z8vH?Lr@DQK26m950UkVlCT`0q6>@F&|FsjlEk zv(CZ;50&A`33kq#H7BQKv_tep@yimTKvUVZFJmog?kaNQUq=IiIk;|j6xQpou>7fm zgH_>HpTbadCzn3{TTS#DAx+v|%S{2viuuyWjsY-&TiW<)#mV`KgHk51soo{~yEIX@ zm8BC4sMeO5A2T&$Phopg*n7W z7g4E}>cqINj9Fwr{rvpY)|3<#AG+ICqB5(V0P)741&)-;C{8tw7lI|@g*>(9wuA{T z-V1B8Lucdyu9*pQcMTr%f40|nU5YUQDeF0s`z&YtHN{qDJRnl}R3i>C0)i^ekGtmI zXg}S$e&|8+95|N)LO^JPum?PF8M=vEe7BxPOX@$J$+0NkC=L$R3^e1Xsa$|IC3R-u zj|GkU7kvEJC7|nLM;2^G%dW)74fAkyX05L38aY15zfGh0E%Kw6OP2cvA5>jE%}0t$ zNa*2EL4oyTU2Zf}PQmJoN&XIY+wJG$i0tf=uAcCbCasM?TKI&e3sWoz(hYQX7h zM)Kz8FGMgy%Eu~)%mnEJ#zTOJng5fNSK1gMqr|(kc3T^jr3g(N?p*o63FfTToIIw@ zeb=f2JQ&ArT=AV+jp~J+t%KacvmpRZaJ%VF>TT4r)pqB!rjSA3nIMuYV0x?O5h)pY z`?%%kRg;)YELx&R_qSuTf)QmMJOV_$@sE> zFdJ3GP&5~pAj6TxWoZkfk|^UPEp;_SyM6&^KwS(OB@b@RHCD|WpO|&cm`vlwaf<41 z-Sa4;bPnI2rGKdPEN9*_K5~Og-RyZ)_G71zpt`pF^3Wf zH6`cjUx*>JjPZUveOGccQ_MaQ?G}tjC8uS*Tgu0NYPcGcyEP}o z-BWtu@NQ;Bq4v4+T~NPlISRB2eK@J?*YUXb)TJeFJJayt`g=c(7dH44S?PO8r*H_b zP;%7L&ZcDVx%LS50Pyf*vvx#2vvFb<+r$xeeSU+7?0X+LsevO|Z^qV8pHOuHm%qJ!(&T&gj8`5p`iE~&U@9*}Px9qoZ9S&s( zb8!?rjKMX|*bPYTJ+-d%b#@M9CScoOJ|;oH>3=XCwhn!-S(Zwn>|6Eux^g4zaTN26 z-i8L?5kU!0T${0SXz0V5i>P2rw(dX5gjM=ZR$q?7(}6GV+A{V(7fT^ za6JsHdvt=xH@7Xii}dP>EU|?^)8J#t;aNd2LO-+^vUx#^%U!$XcDpW{LTwdugP$s> z10hf8sGQf~m$ZeH!pF|bT}&Cdfjt9hGow)9m0u$dp|qo*1ao;;L7f7U`_7j}{<_A< zdta>#EZ<`8ve+?e4Ifxsa4#LJx7G$fsfK%#eusNPA((gLDAl)mC z%fA*+$3x2B^PBdrndN{tYz>Fg2Z*h@ zcvnu?GnQ}oSj0;(XN;6dmi4BP3EAme*R@{y|4P5QIL?KU=)2S#!q)4T;t~^Sadx3$ zVT}UW>~aAQoP=Y4+OnTJ++8M~+1UiX#0~)gswi%MczLH2v)Spk1qa7G+hYmdO);uc z2f6w1=KKv4(zHX1=uS>>xDKQl!S8y?q~O$!gZKd)oXFu++LjLK9N=$%aw@YiZDnRn zh;55`JWG=;FqV{Jke2WG^oh9thZlyV%Rs(3OeZE$_=`7$wsV%nK$0qFA`BX5kTo7k zD%(Croxz-!HALF^5_9NX$>Ge=XaG;rwBrdWM~FT$>p-~K>b$k?VgS&NOK`eu{>DcJ zK1!^)@SjQPw!Oqqx zbVK1P2fK%XiggZ$8qId+!Ri?>pT)6l_S(ex?0kteH%f?A6{#(7SZnEwWfppdkVb^+Eb1Az&hB`E-H`S3QG79X zQ7mGb*CtT*vXd}zlXeZ%MSq=2**Z{N0wagR>lGuli^{Se0Fi1Hr7g=LYxyg8o^R>7 z&`3jE;uxs4FNSvvYX6zP%YaKG3^? zFZ$P-q5AQ1b^i;wT-cTmnKapMA#0$E0z5%eVgTko9BRnuSic*fJ0rhKo`(1#_@m$ z2lKGt_-6{ zY$3>~6)OtnjC5&TbornD5@ATXT=ySl?wPK;{i`E;EiL+9UTBMBUeU2Eez;e$->(1{ z&~M|@PToW^6~(Zg%yU%k{0MLnn}h@WL?wxPi$B>|lVlElqH?Y++*6%d;?a#;hOu-4xzFP{P>EV?Vy$ze~>Y!@!X+3=X z$YXM|EZxUhr{KjV%on!RnRdA5AY!>OUEyzcTNq2o{t=hb6{SJv%3~QduUp!L=+SljVH>J2ZKL- zq>4nb?Wv&R&b?C1t=avls%^8KA{vvMHKe(7AWz$+w=psZw?_e4x)5J=T4$FtEZ=0R z>CMJozT_<7;PCvC>`03@m z^P`8gs**Y^lC|>S0d75$5a_(z0O+KUsnf*XT0Kk2V2y{%$y&>H-%c9LIO7N(GF~%3 zKwk@=Pxo-M)SRVXgL|X9?bflwzSdEd(|K(0sS;%ers&E(-}agF5pPD(Gotd33aBm} zN_kEim%^TwXWG>uV=xba$i>}dN`9Bi=yS_(>-4ltT(y5g-NR3vLVIP{ zAGs{Lg3Sz4jaKl=p8?UN+l&pGIPkUM-DSdnlr-x|pBFW?o;`~Zs&AyQU1&L3N zHqh>Mc{tMoIVRc7Xo9SIFYxkGE)jCz*D9J;+{t(s?70Ub_o!?L9UV~j)t>ElnmyV% z?EG~3L`_I*_7>Q(Q6}H9GwHB4M+ycf_cr>|MF?9Qfr;KBQL<}%){a<9+9y&%FDMj~ zJ^$SPe*OYLbDHY*L99hpX&n{300P3c_cAO;w^{+=vN-nr9{!-=uyM(~!BHECWxl_b z-Pd&ebR&ey{O!o)0>GxU9?!m<>^t{9bMItiV+2u_jj69&E>pO{3!Afc#KxQW3}4H5 z05sXkN7_{dk|Dy>g*eM?ukpHr0BZVC_s>z25eKC;72kvClD)!<)D_oAeyEI9O308K&mBNt>ZF^zgZbvVz1P@#}ABS!bj zht2nCb{W1;^a1q+1h2!uZeB>WciUyim^Q1`-r1XdMc<90s}w zwxc7pE?Net@8*remz|QqN7q|#mfAB?)-tkJ);Cl+bD&lhoZrDEEcDO=e^R0C`($%g z(aw7#dbg}==&i5Wme-lYPyxE!=XZx(Hzi}m-S{|#sD-@1yJRd!LvL+07;MZuiw;&{ z$wf4aDgjtMF2TR?Elc^F{%(1xSX17w@P4-(rGwZfc2?gaD+I(dtiq2+l zMF1zT!Ll|MdN~P@a=Xg98+Qi&@A-*K4^CNh&TTHY(V7frJqi(XYmAAx;)OJ(kC^3& zw75fe?YS`L7h5gGnX;@)Bxf|t%*+sLb#?a@a2q6FqJWxrACNsGq@CY2qsoOfCgk2; z@2Tw$(5Ao#MrY(X%1W=?uF@YB#O+bB&@{$HeNcT0&M(yeJmjHCisqUk<;<7!=A44f zYfRxoUPviBzwJzI$$HJU5S}wNZ{Jj6npC6EO%oj7Uf96#)ukN{CraMcD@8^Y^D4>) z_nM&i2i=psR$;-|;EX0>k+q{fM%+g zDu*=z-jgL7$c(SKvHvhmR|WZ^j_=^7mh8FB2C88KfY!^U(mvT~-KUpgfeDU6&q(Tk z1uI7FUFgQHdLvqG(Ir@^d(;7V0L|OG6hM_Z{DHC~(j0iCkY5TkfcO=lxr%zfU#*eilCnGEtW6Z*p zhQE~e4g5;7vRfoSH~9S~hs{LI&68z$o1JOzkOkLtFY?N0OY=)HM*xOjo-BGP4QT?8D87LoR|uY8#CNzDf{uP9(>HuqY);X zyZrr@!kpfU#eMbmRqRGmRjuENC8CS4I)H+-*)7(O-Q;DENtJG=VcdvVO_t)0^w8dz#j3)rX&r28;wlj)B1h0JZ`hhXM;(Bm0BxLRW!>0oO zjl*heQ=eL>t`+uYt~Ozy^!-25Dg&*+zLnamX9J+c$gYNquWVG+Z|j|Q$+sc!hUy`V zX^DZy!!X*G{4XS7MYXO!Wa{nf7wnLSE2mEOT6DSwJA(?nzvM*Ok0ST@>|9RG2b0#O z!>nty%X)@l*hW2IgrK16hQk$|olcH(Yc)K)b}*H6xL-yBAMybPqi)gub8{tr(i%ec z;nyA`Rq0Si|Aw6lp--%Kns8)J{H%Ig>aQ1mxi4Q!i@`Yf446Anz*prFxZ^z?tG&>i z1`O@?%ZjrtGS0FOO7YuydlR>R4M0O`X;xZh^ii|E(YXMC>ThnW0cKaK88+$>Jc=E1 znK@DuPhhsMRE9dG5v4)MWOO+rTuP^%glxaB!9A`9nSym_aaePTp!gu;aeW6GF8I92KLE9Ti z1cU~+o*`o0Z6MmPy^-jy`(xhKV7d>*BW=R?VgEJjL6ejnY&&zyJ^tI?@@eQjnA_S1 z%dy7I%~eWp8zsB%36^JK^?aI}eW@QTk+`xv2sRJY&hfrh_sh;lyN`eSLDMLW{Lz*U zwiu{r6o+*V5>+ZYQ01^-6W6_$Z02oeeJ3KB#M3K#O4A}U&Cv~7cOZw`{4tvP^DZ|^W>MY0rwl16kxb?wg+pYYdGs6#`D$N0k;|s zMrkc^sXgu!Ea0yl%;^pKl5Z?3Q8;%NVWv)I5g@)iA&O0z2ww4ZR_yMWhR)#U{2IPY zUXt{iIXV^gBQ*wC;PA!;3g5%g&pWL#xJ@$eon02Gi@H^rY#E~krj3fOfH#`Y+-EQJ z-7rd9f}a#nMh$^dP3qS8j9ByFvjy~F@?755)=&0k5-7?cmerEm$?A4hwt&*RNsC%( z9Le&ITko|qiYDLxcrrm5_7OOvB~8gcOtz;$YSL=LyVuvmf~-EZbR|Yia<^>{FgcWY z%2jWro$|(Mbhbn9;A+Y^Ve(-AK=Krs!8r||It@^lwHtMq(aJ5U9EjE{!)xxoj*t0| zJ*!jQAkr2==Xt8;3Bp!`W@C0Vh>Iibqg2ai6Tf+W<3u3Fc;MGG+9<4z3Nhf%u6?*a zi2-`zQmoH{{kKszlhoqhh>wZr#eC#P&MaD#^^XqMo@__fy0!L;@lCpmMNJ?^{dPVI zr|&jPrZu8+cpVljL6W55M3uId+jtlvgOtG{K(@Wq48sLl@vScZpNzB z+0w2l-ArIcI2G*9T&<6tDsAa0a%T_ONS6=jwnOlhj%u11l$nYvjoazypbE{X9MHRa zRM;}O@kr6}BQOybEtMLL2J_az8lZlp;}6e^)gVret_v-2Nbi291Uww``TUZrl0;0^ z?${LUD2j1BwI84Wr>)mnuq9Q-VciK=eEsImnTDS>cLO4`%SLBv96H{|A>LH|gwTLB@?GCpTN2$h&+1uPRZiXElXM~oLdGZEja4uxZpoeM zg=CMTv%JSE;qH62f7^uPnUO|%YKU6GhX>+`5R?VpQ+>cIX0~hTw*!8%s|dHZcMx~t$ zeq=dMvy`S)LE9*EK|xXf$!65(a$<|&j5%K&YyFfla#Joc;kIZ=L56W5V?fUm`3PL2C8BNE zT`mvPS6z`hHiHX~CTvI5=7wEnO!9nf#3CC3m|5FkNfAX^!7wH1GpkXu=I&yMV+}sRR)@T9bcHa$%q<$&39R6so>xrt?`1tdEY%B z^W^NSkm7QCt?9liNeyb>EM~?{VT+6`RrDxH zL*_t_g+~_zKhT8x#JuHKr3#`t+OD5SX~#7@jrQ7VNHLC@nC^|29Lxq0KHqk~U}cuv z`BhzN7JHbTosV5hdtdh@?%-aBrEgQ;abnlr1Bx$XHR{z{~w zA3nWO;3DLS8y=Nb12THjZMle~l)M|Uu?ufL7UfqNH)=EO(0=<>hlF81R|Dr`(*8yi zco61#tP#OCkriZ2ukzDpKe+YN><)t#5HTT2REXV|85)is42e}^Y7YkueN$asjn#G* zp|)Nyomg=4rcPnVa-yYFcXlrU;l)sAo~oTRrr*gF@c+1WhY_xotOMeaUwbBIM4_>k zZN6l8A`80Q4_$TL_eJ(2k*Rz7SJ_!Ru{%~1H7_KOj*ZdjxBJvQKRk?@*=t_+J-(!8 zYorRW%yPi3c81|KCKPZ$fs1nZp4fH!%;Z@WidyIHZPZ+RFjI4jsL1R#@<2h(xAVm? z5Fms+H7ZId!+>3Ebu`$@Eqze`U2Se~ZjXN`F}+EnbLk=mnz>`@=>3YNV zPVdiPw@#!}dgLgQu8e=V{)oL`|7fM1Au)1|598=`)MUg~N|U!?i%N{tO11P`&3WxJ znV0XkKju)X1|(CUvk-HiZ}hAHOCtJrMm&*)zn%rLcCO)8;4>#`9w*ydlyfGB{k%jv zR_d3S3dvCiNv7z_yojlihGID)Dh7f>@on42_q`Y(tHRbsv8+PAD!lm5tI;&S#F`r) z+hk1fDugOl+05z`gxJokSrxilFKxBr*Y1(0 zMu8gxj&V=G@yCkRnkg>0i5}voyd-MxplOg;I=`e|Qc&KQ0grNx-a$uM0l8Zul@D^M z921AQMTb$In|=A6iBx3P1e5|LMN&*QPW@u&o67E&!o~R|#sI$C%z(GKEcrF+EMlrn z%7-EzT=SdwHSUKz42v?Lrh3bGtqA3EH(r8OyXHw4Lvb3gThxP6c ze{LilT9Y5|8DZ{`of$bghaNH^ zWI|`@!=_3Ez!L%QA(Acy7*9p5kr&#Zm;rlrc&Q#5dxxZ8>Tq5V`bkli@OcZ=y8 zfV)c7U?&mwYKTnvjmzLYbNd8l!j=X)PWao(9h=FZfOxPvLDC>k9#I7%0>CU*vgRJF z!etnMAm$lb-=qxYtUUu;Z{#3vZBw36qavFEQK^~zlBK8u-U{%bbQ;DZk5x%`@I*=x z6>K(kF&suU<3Nmv*$xl-r4u^lVuvuOA=QnMGgN2y#SfX{z{ z;|o&V*qxQ?QRAU>N$)-wOIur;;!|?JK5(gP0ylVZv|UF#X*xfNzay}lEF+wf@HxL> z8*;mY0${dC_=KgP<@_o+MT)?aNIC={Q`HV`>EL~poS?lM6M7pm_aG~io%y1YMa{)1 z=?<%VjMqdsxqob$YTOKntmZv2Q`xa~-WTgKDdjF3YFUSkW|XrE^ej?9b5XM1sLIMKXQnhrQ?N&)mKpr(dSBxP>!Bj8balu^=sDyM@mHbR7Em6 zAjhmGJcPn@si+`$nC%EF4HI*-_Zgpkx2}3%8l>~ zmzJvPZMR7@X*3Em$UTv+0tB?^7H;s{@)bY`9Lv`lkP^NFl1$hG{s zO5VmQK40R(k9M+Lf2XZS265HbG^b!^q~bkNB-v!a*6UIo zwgi=eTgs5Fh>HcY$=_1mi1(@weoDy~QkGxxKvn94Ayn3_LhilL`s@69PlG^N>Y!@S zhi_0rtPg!`OWZI-Yv2t3?F2NcHo=Q;OuK9EpdyV98k8V=F(Agy))?P*L?10+x%yt_!LXUmQT9^L->l{LM14xLAu_=9(L zvWGJu30W4xpNRU7_j0P+IuL1bQzHhErA2r|#pM9s5^O6tk3W`Vle(oy%y_yzPQCeJ zO^_~GN*ruSt@U6Zf@vcNH;0DT61@OB5BLr`f*7$Y%FY}iG=B4J1pBizZq^BO;p|88 zILZ9~B@;1(AhV6a6<1`NHC^qH3Ec?9M1v z*TPm82DhAGFrCo)9F;BG8X%=@5e`Yo1ZfZ zp>lF@z`%IXINJSWFicvH$o8rZjfkIJYFgI!;U`I@a_9u1<*&_xfI#kL+^WjUYA=HO zt{YJ^+4}W@fC5FzyZo#n4eGSz=+QYS)Cn26#;v~Ieym+SCv(ZAiwMf~A>udg>YFZ_ zb6Y(r>7mI{jDp%MZQbIQ93M_(88LfGdV(c=o1E;wfAgrUP%ms zY_jj1+=Yh4++G^I+b!vd?O2QNSU_(}>#~LQmVBf=>!GYTU3SF|L^eSBib+75IOkS5 zS6_ndrl(af#S>ZOWl)JO1lmKKD4&8r(d7zx{bAiD?dK}>04gu_UqtW~jRG+-zg&V) z+gN-sos%N-!y0e_v+fTR)Zy9FF!~|1%i`_r(_f;{b?%EZDI5EyrfDaB*H+R&;?ECPEjntgJ%0U|ipf3CVu<*i&es4_PS~5uYifG1H{GaZ z>@ak*!n{&FO2WBly{U5N^yJKb`%}AV!@{;OlcDvgyz$1`+l43INGadl$_-ADTB~>N z0S^v8Q}v;#{2a?xcSkVAR8sflsid2r#H^T?gcL+VH>=#&E{NIN_7!VU$>*FU7jEcS z`|EZkLcJ`=kq~2S%E;`Uo2W_$1_Y3H%R8F)(_6lOJs)L1RAM9FX?dyXx~;MvG95U5 z_|uG_R+OF0dL_Fa4BQG0CCglZTCD z0me+ybkqkzFnkE&Bus{peDD~=-@f!K$FwouH>%^TeTc{yV>5k2<4*Y!5H8Tfq(u}G z{hSmLOFxeL!$&u#*kFf@Ec;~gCBQ5MiTlU5`a~<=9QWABV?d%Vnsv>{p2~=bHD`Dh%yxJ!qkF*)^n9&11n93$*32Ns|Wg~ zhgxSZhWR&1HF~7=ni!QJZ33yHOu|Fb>aQ-yi%!s`?%sVTE1_>dDjxt@#7WN;4vhPd zHcM>aDFCKyeWyw3l`*|`F^A&fNGDufK~SYlfz(E4=u@+uIT(qoz)iP;)V7b&HUJJwLftwa zOaK;4Ca8Y^lg53CWg$96P$kHn2Vmh}cs^5Us}gD9#>1a_LFb10lRQA)4rCbz-9VKm zmHdSNR!ontvX$GMI;zJWIi%H5!+w?lx02+IR4|Lu6&ViuYkf)d!y-;z!-)5|OBJkP zQ=qDvV7tyDD>}gp9syspy0l83h$1ivjZGrizmP%+ zf;rbskFNev{0S(z@1w1>aIpjIprB%rn5e4dF}w|6%MHQz2K1UE#i(&>-9}zF{(oem zPztRC1(LA3!8ZlK@#0I~+G~-#Fj#GY#G%k^C?wlk;f!NDtd{F6IUeb5xX|@d5U4c` zw-ZN>yPs4A=uirb6^H!zBgrQFigpw{+~QqFE!EqC;xnu+SJ{^Pu8^8Qp}1>l2Uso@ zx8ShOA=LGuFE4vPR)g3LW3C&SLJI3w8S0RA_5}Y66LT)+qiceI?#xDuMPl>Q!)_VO zN%Ie_koa(FW)`@vK_DGZ0<{+<*33$$@I%C#`%uEb00z!y=O|>mv*e5VNEQoX#gr+k zDGe2gXiCakj=b-<(_8&F@yhZ}mSOWoy5DBzuCOggrfNsquVk!hBh0>tdYFDPQLC6M z<+ck3_!*^sgzdV9s|fHDGeT5YD{Fp{3$3N6ZMu-Bth=u^jknXsLr|Ic4i$o(4Bk z*~-WLd_V`ao(6=um#Gp@w8D%;=!0y37gpx!kkV`yRGNc-7a_skN@Sh;_%T921-ld^m} zf(C$xX6Uj7U!q0hb{!y21dms-7?|3YdX2lv)ZWEv^6wh$9e}+}ToEuDEn{-1v`-zq}Qo*=_iMP#8V+#gwMJ|bu=@MP=TZ7%zFuM@PD?d+LxPa zuD1Exzuk9kY6Ea||DaZfi&1aJ(R5qQZVM{n?Wjn(8)x_TH_biiAKsah$z zYMFt3aTEw>NbCt-JAlii;}JgY!i7o-u?eD356o#6UV5k7R}a}h3H7=Ow3r<04**hD z7V*94-#!+owtgTX*oLbW6e3x#Rm%!g7dfCXfR)_VeaIF{7`l5IOa+X85Hf=s^;@%P z9(XQhD~RUXD9QM=O(~n5L1#``vGc3W65yE!WN;7g!2MEkYSDO|G+4}t6Y!%!hqt<=Aw%2L0~988d&kFK40JS z#JY^GgLmHrm1QVtzE$k)ePe!O$CrvFIibr`5`E3pJ!>yrk^n7J;WD?)=h6z>eFP?R zmXF}VK{6?0h$PtU&gBvts~qm~b7cV!`7uq_i+O}?$A^)s>A>x6MV(XP_p`qLV_O(u z3P|4J>mx38Y1n4=+>S7Z_f%+!qcyV+h-S z?AI&oR%E?yuQO9wjXkz;qZ>Yuz$0NgQQ;h<25H_T*91Qwda33APlB}0s*S61M;$lU zYJ&($X|9SLFa1bqP97*w0UUo%uR*r4Z}*25e%rAfiC~DNr&S*0_z)U49DGoY44QVj zcb~%l7@OIQ{tO4c1(T&D-k^4Xym0!Gzsm8T8ar|N;+g=z;h;Y&g#pf6Q;&|NL#{2& zQMj}ySrEVfMGn)E>m}FzWr7}v1~NK1*1JSW-aK3oLAfvOY+W5w5)n13&hq+{U!X+| z-@uZ}OlG>Yh;Wkx9F(ez^);G4(0TciMDW&NPYuvVut{NCc{Lg$C5=oPD*frxk)}Ir z8~gzO!ypwGuj@c~fA%EGw|Sr8u3Wg$&%Azxk=ee>3}`X7aL0eY60g5woYJN2liL(Rp2*PvCVPP=?3l=6;|-K-myg_ z=SlrVl_M67qN2I+X!@tF;>e^Sx(9r9gFRoVH`iN=BJ;B!}mU-CmCr96B#D4Z;U z6&k)lRN=Xb&W+ydR6ezT(s-lTy(DQ>TMZW-?|YM+eQDsI>wOFyi}hlGC~;hQdqN?4 z@T`qW)y+szpv3hXmL}BPu&g4!^7^P-Xo?andK_j&lI~rk&}IFcBdmiKjC3D(W^7PD z?*HJ<7;cC2TG&wL^+yvGd}-r_($?1gxynV8NkZyXR%X_KLJy%z3m6+ujgU}H_D_G~ zSHUM9J5CpSvImNR z)5B8Xo@zdM~5Bz8B>~jfQ)6Pt}81i(wea3(` zmX3B>cjypT-snY-UB92FQRN|+SJEQa!*0|}=T5@Uva+RMHe~!u!^=QiygF(Vnh%@W zu7v~zAvw-)Gg303$bhmRQ?d2kn%>3FoJlkF^<;iJP6BZtdT3T)44VlJe3-)u>Quo1 zYv+`HEHblX5Z+uu-a@J11_8tnpO#f(z;?EN+;07*2H+F$S~06Mp)EJ4bv6KTA&kOE zIbTd_c@Rj-4MG?UybI9VWS4C@CP9Mi*yr7@bOBnS#s)eS+Tx_JoCT8{_>T`t-7jjU zYRH$`mJ)D|1pbIQd)8XV@M*3U=wq70Eqwb|3jOIe>X%V^9+a>(_rfFbIvh)WREc)= za6{KJb3R8iNkd3n0pZt}krDC5T}+2BU;A3d75`0tS(rW#|4C`v69;A+fX|?bLeI`9 z-{3oTc-(&FY#ZepDRUKHR_;2|&91gOWRD}B-U1Uq=?j@5Of%2M@?ZnmBjv^|aSXdL zHhXf|y{Vi@pIgjT1K*VVA<`FD*x5az|0Jt)L#Kffz;{DRvKR~Hn)MIf0-Np#P;F4y z?j5sn^j;AuSLzg928nze+2W_)*d8tFw9_k#r6C?NH$Xf`^)kE`<>CXh2h$CUEular zM<1c~9}8^ZO`0BXVEe{_nSzP((FU`WOFJ?>aytsNoT%N6T$v-@nel2@j01Klr_T53 zp?sdWYc;-eYiI>qDFC@$)*Zz5?@FD_7KXco1!ln}fm#8e8Q4<~9e~5~?Zfv=X(AbX zfBX{UZ{!%i&g8rq^}8QW-A>0{uDBlOFx7F=ZP-uUxdjQfjYfzfrZ`$-sA)?kyU_@5 zDgq0AT~>e9GpE$TLx>iU+lt_Q9P4i`bKWC++sC(RpBzctUZ5l|fO$yKBT#b0X z&x(^v^{VTX8?LFohjHWj^}`zX0KffvZlkCHG$-1wuG6iEJp&durNUoF0IXDUikzpY zLSjR!)IEN_a_{!&1typ8#Kd=PyFSf7d(UsbT7MO=Y~WLD)iuNYYE?z4o0cEc@A}i! zL3SYtvlj);_JIyp7))$+{FNj~R5TzamLQ=q92?6)&86V9W994UL1JJ1*#y3$qi`hh z0k~Ce{jml4_f`P;3NH_}GygrSm>Oz)2b9^@&AHR#I^DQc=Tf z>XUuc?S9@g6LKTS`X&Jr_n_;55If-0{j7B1?|q<+N@JvMu!unRfeB>-DR46tt;cc$ z9*zL!Wl!PULC1l1>Z7ry$uz>}o)}|9_v+HM?X6p%8klUU!#gcTx6;crZ--{!{b(NF4k=%IRwLgsHud2cpe)j1(AR5&4vRR< zB}ve-`T&>ImoMrG0?rx1vwR{9M4^GfD9E#M+WAxWH?cEDZ)9^p;o9~qb*raO3z55D zSIKsTO`F|?TLg8R{0iIClWqYs@4_kTjp#1#O^S&XaRkk-{?@l@$5A-Uv(p&ALCD8C z6LmLQSvs%J_H*xj4g!a$4V13t=mv?FV;QC#qZXEe>M@7vXZ8!b0jaZaNzXd>ZcxL@ zw@lEy{PDnV^#m!ChHdvP2Bm_`UZ7A-^XaKuoK&ySJT}@dA1e~|p1%-BxAF-y0*N+Q z^6v%R-Xozx+TGt3vCKaUmD+L(s^~;HMh5?u$zLg`rqrsbY zV>)E|OjUdSL!_cH+1-PcF}cCY`Ny*H#P3z4nV4y~h2gNzE)6C<^3i86tXBIMW-ArU z@4X4ocPKGQw?Y^ZMIZX#E^aoah5uhZfiEc-Ado0A5h8oQSmXrRukCvWUQhX<@AYLU zT`OYp0f7sW?Ph(C?qZW(rsRXM9-$^8S(KNl2E6I&-ub$urqe8~?q=(37CAeUMf`L+ z!I#u*glX=p^U`Txv%YDF%<;g+m)Nu^pW&@OkABxezXY&C(mG=F{u!x$M%08&l!6n` zi*Z=irD0It(QhNGCf#HfB4*&clnto&b&z z>JsAz62S>mH_d^9^Nq7WY31A~hN3}!GY9}&FtqQJZp94)LO`+(7ToA|5Qyu6aNXLK z{tI`TqYurQ{H@s7fe6GYwBWo>zK}i1ub`R*c(@hBy~Bxn zfKZC^$g^|dLT4+^1stm(O1ncHjb6)Vgfg|SU}~mIi0TLj7qC|WXztIY^+HPkAiyKC zT?6f4Km2PXc?}U-&#GtflN<)JXLLG_W6o>Ozjy6Y8HG^-bRHZPOF26jJAsRqvHj*a zAiyI>LOqvV$_=?15ZG87LEV3QwP7#<T4f0iCSbc=d>bp+;6mczx4Q)RR>EI&S2Ou?Ps{@ zeXV?3!}X|@u5u|;5L}~U*im;6I#VGo>l^GzF!?#&Ac`;^OsF0_%&&stnkOQ`^>M#StsJlPO_NnDeBCss;a`t!~s9ojV-d2{WOvFyH%K zgEu5$)!*(Ux4VPqgE~IZq7n~Y^iFCb9iiRd|Jc>}l?uYqn{L}!s-+RNoq;Q>7`$9K ze0sz!Va;Ao&$;p3!vK{QL@(RNS>T;vLmNkV%bTQPwOqSu08*(Fxl%13mXMk?9@O|tlQO-p6 z7k6RxNNVQ=P$sWRI)?31%a84zzn~*veufC-?+q*IRTwM+Bx#7Z$X{N1Hm2HWVQnxx z=K-u54cWbMgX3Wpb-i9XY4vrqP=BTqp)@BxL#TvoaYmZ^)yP_uiB_w5R`&aZuBoTv zC2GLqSTMZ02{cW<6PF}jdtA6~YYAcn+Gz*_Byu>OsOvnZO3jTtxLN=Nep%oo`vh4R z6EcBrWI!~r&0|FQbMqZH0x1G827qa*8}SA5eYU5-H9*Svs=!gWU!VGzV;#}&u-;Lg zoM>#2*-q%}a50b1oUK?wgTVGH6O?xq8{j&R(@CS}Uw~eTXRAvr`|T^6_V(tvQ0_eg z)TgGVJp>oEgJ<~o__Chw`EvSP%FjEkb-h1Mtka(OT5rVZ0*#g`;TBkqx+2&xTr1E5 z1BJpjth^zLpzoi;k}brH3bqV$B%y2q@TaB)OujvCt4La`0LxMQ+x2p4LP~g~nw#$% zfO;tx)&ATJhQil1fo_WfWwJLsJi@BQEGd0Hn7*@1kA!DSUD8DwIpX8o3!eJR*HzRN zbLqTFb~1diN9EV52GfXPn}MG%<@gxtNHrnG~l;^ zg`iec0jYh!buv{@B%!_NA#yxatsS7;#Zqvd@!CQ%4J#E3Xonkcs6T=oU{uauhGK2x znxMQcWCk65X>>`BQBc7m#%yMoAOFjm0yn#DhUu>dgpv6cahvR-kB3?L8D%wqM#9KI z(p5%eH}Rt8OG-0IE?YpAzT_XIniB*I@Lfy3YM?*Zr=Wlo_J_MOcAc=1Aq&oq{b<@q zfJ4Bzw*RZqguWmZ0DLO!y1Exya{&cTL(TfTMVB*It&*`TYU9a0>*-guU5&Czt8j3j z;>qwkYB=nYn^SPT5C@9R`|N;FBp{B>7o^<-{g6!>EmwJ=48hdipOOn6mFHA?m0XuW zVPcfSfLo`VDO$IBRCx~ExdnHLKOWL`F!K-D0#_fd0!Rg}bwlx)@a8!EO^fgU9^?_z zGD-hRh7SI8<>7GzrUmf*8d#}uf9d9G1N}m6!{NOhHVkt{Br7{U^*sEc)cG~Mb;H&J z6?jNTf94fs|56MRO2|B|Eb-3;xwB$C?(B9t!uExv-YVl;$g1i_vS0&-H_SlFP- z=j}^M=iw1%Yff`=6XyXAa0dM{n=vMA^a2bZ2&YmXC#%2r%JnOorX#%M8Ylq%+LYde zfq<;8lWr;{pC@gBOKmkRhCBV<6-8P1Ffw8Gb*#Z}D1d{$zF%2vZ&)qnr?Av(A2{ET z@M%N!qYWTUT6hcZys)jGeIVct?E(P3SF&=jWO#^|$G`zE%$!#U2hRW2kYE4t7pH0K z^%CN9UpWdSeKRscyOThNx!)E2Y}M8H`wLRBDv@3E?T6;#TTxE0w>73IZfdVRx%CZ160wGjJx| zW_JChqOj}F-YG2vQ}{$^h-mw80I39NWL8znt~NXos;wCBP4i#)vuG<)`gmevMF!&r zV}EBmgd}Pa-hTgQC<5j|rRH6BD0Jz5^Wx$p0wf}ct6PW%*bavf!2%G!SmkoyhO%v2 z37i|mM*GlqLHeXS1l1#pNE2$b@{jMdVgkBw;ymWo4RsPw&IX$8YNi+!76$ z!p3Li`4WLWWz@Fw_#*U@PF0zcJ4eWSc{1Xz_wGr8Y__%vNY8SfBYOg&7%ThO{qRMo ze4P)a>In#z(Ox&3mHzSKCVw@j!v_G*YVusou9TN+z%*))4hv!ve?K!N2RPM_Q`MaH z?H?0Gofz?pZ^DPm1ap4xOdm0Y-N1_Z83=b0un%z=?^ZXHhds|j#Nlm(3pak| zd>&c!(FD;`w$kUO`X$ryV|D(pEko;QtoxsjrKuDVZNp}`e?pN+Le`=`&{5Nu@??;! z{NR{WXaJeoPso}q!C-0`*wK<$Cw$AIPh4{6p_oQh)c!CAo<_qKlrhm@ z1h0}{%%A=H_Beo0rLBOdo}&^oX(H_tPD(b{bY%Fr6xhPOs!)Uf*^Bri1Y3+aL72cD z0t;DpcoPzK?4V}JI8Q0}IxW-R*pu9t=>_gdxA{CIJEYBVA}?+8iAtIX(VZjVs@BS~ zw|LX1nbGvT$Tdz6)eL}rU_b?1Z*;jlTMdM(-M*VK$@}0Y4FzNX5&F)o7QBi3S{848 zg?-7bB1q3m1(*Ag=elFj)u|pshTi^B?;q(ef4AysvIYrSz_+d3q~~#JCA1&+3Rqw5 zj?4=BnM4Zulus5yU1CA|8wfHN>}usjkZ?;>RxiN=C|J76C5E@DY2% zF6nnpT{ZU3H;k@%?L*&&jbv0o9AAY#4nlx1Fa+Jw1yvi6uAaar@q4W90;5G`{qr=t z{`>c`Q9o%lr{4OX3z0;}IGGfXurw#VqL^Kz2FM9khN4bdrKe*_=ysJCz{Q-+Eu}qK z4q^c6NgaJLmjAHIVy)1H5GQGKI1i(pz`~MXz4eft_ ztS@rDqXPqQ!%ptzheN=2u2Eyr+z|9+WqH^rDUE;kQWGH~VUZogr0;(5KwFtZObq|T z{sf6g{AYg6wn>U`^u<7$d{D@urxQe&-0i3T;5I?XD*o0gvNcMJTq}qjIj!u9Hj^Z20&r}?=uL?9*4rmlGH;wyWEJlp zPVIX)KWY9rXlY9W&Z~g?6FjMPjw9OcNLxe?_qx2XxKl3MCm8j0Sa6Bpde4T8xJqHB}3`n6J| zkxfii*YHyxZLrr}wvgBl3-$Qcge&8-$@1w@WTQZA1PV&k98Kg3D^+zbw+{A29JAYY8#LxQuRU1fYl-zu>&Kx zZ!KZfJ=Sh!fhdQq@n(|C0<4z7{rJGFAEO!oopkJ|w98~jy<6>S6u4(<8%cKXb{gnmhR?0&sVQBo4~nrF zxECqRedIr?nl8RCA!X$O*|Z+{@xJ?g1|;*J(N#~LJf6I<{dqo1{I{yNe{g5?$OdEH zwPuYi`mHz`@_ix5eX;Id`&X@Z^E-v&O6Ff-I9kjmRSpUPQfVmNl*|D%{d+d@$4~)Q zTDCu48hiz5BOXuk-N&~yG>DUqve#5T-gVTfKU)lU#Icv^Q#FV-@wYof>D>IZP(xh* z0N=V&Itlu^-=HmLA)tNx-W6FsX;<3k+QNl^v1DpMSn#VxP~aOSREj?9vmr2gVC(e& ztlX2UMnBRZs?ppKM1dl6^6U#+2MgTwiE@9 z2sj&_ckNPS*u_@Y0r1Z)aK;e-8I8Dr3r``e9b5Rbic9$n-$rb4QPH7g$X|Q*Y1-MX zzM1#(^13K($+Jg|Q@<5J{eJX6^;_E|w(=VE(aXmO0r|YS`u42e%rz;do4(vrrBK6F z0{XdZ#w|8JD;eeS`HoKDV86SsVA76^$`6c$r2qUvznzHmRqk5$Gp3E6z$=l{*;-CPq?9x?mnim8tJj!suq zvf_<=;*Kas`%l*TL1g_pqW*|{7WY&AwUd~qJ!f;YakV8rfBB^9p1_W%M?&hjDVOvz z28h3U`t?2SOQ4SO3%LU=+d)j{B6s64#fRmq`ESh*wPC(`#x(YwuI$@jX7XI=&zwF~ zf`9Dwmb_;=>dCj;{CVtYQW;Oqam1b<&%4qSg~KMhmW$PBVc(?SqzY{7YT?pX8}~#} zVkq-_8EY~3pZ3G@5TyDa%8ruKNp=|H^3)~C+WcwTsnKBihM7_K!au-N!%iZYFt!j5L92P1EI-Di|@9|g})dp#o;f9`SDVgf_EFtRh^4m zlP{~=-I=uUHHxb(m3pG()mcy2d9BP`pDXblbZ~H439?O0OUfU+nxcz%>P*}>>F0_% ztP?Lek-NRve^(mhT@bn=pf$mSxXgIbhi}Z#p$;2=&l2_WNu;vkhpTe__MS65k$w5= zWenD-37-(D2sy^kU1A9PvwXSo1s!L&2U@#?$9XVX>qY0-XbBRZ%ac99_e@_TDc67% zoZ34!Y`dyQ9K7#lbRhj7?R_Wnx!v&ThllZFxv%dq~Gi6I|L z(@@(SUHRmahUz)XN*J`h__$J%(bA!buwgQTG8KzUK~0{tHP?^tM?&8hNSudnC3Hfq zv=X+PVUI|vl@E^@OF9=h51z)G6&8n68s;O(!BJnO!JiEN(^KAMHW-BT+NK!!vWPO5 z*Tn-~?(ba4ACB!Q<99KC6DFNks7>HsCDSNtFr-*{;heSagq2UyEoHeI(1$|p>ee*k zRUM&r?J@=~{$s?Uqa$UAn4+2-A7rLqd;Z5-SRa~i=}+>Uv*JXjI;%pPK}9dCvuxw~ z)nV0Z{^{L4>09M6p^@0IYU{INvFV9tBGb~8Q3-KVvARR0N7LgUgPopynWsyBV9O6W>W3Wl?)-sF}Bo*UF8og7PrwYHGO1?4e<$?<@bv z{PC;Cg=2-Ves@<}n6+uE$`M-Q;LbAn_*`aNcf%-X%s^SDhoDJU#}Q*;&*F*#Euy4* zkYzKQIe}SHCR$gPHC=a95_`j#nEC`oz{f=_pWldQ*a1A_muyLzv&y!GL3_%D;GQS$h8QT7Xoq>aVJD=_B)Hqe%+OKi(hFvsPWH_D|2kOS8KK9)s0I zHqrlhlm7b(2L6J;-oN1fK@q|X14k_QEskl&3#6+IUJM_+fGoG=8WD6~vWc9&E3(zdTwJyBp$gVI9G*!}y;Oc2|p;=?V}*yWwD zLjK~k(XqAULhR&fF#Mu-^NP5|n{}1oZsTKDGFDv{UN_tcD-rGcd~yQP-={Y;lR${d z(@v*53Z*tQ$g7ciA9L+&CFDXT!MlWT|Qu zdcS}FhokTR5PthM zy;d-0;ty{$H0iTravyIP-BFQIyV?TOZvXuJL;lVnNsa6)X{@?;f_K3-~w>&cN!feZxZ#6ljI=AaHe1RYDJ%kca|0JvZ`7qf+-EnVb?_@zH;k%fey?vE~NC3*;dY0o~{}L zN#YXrRrfckUEd*kUzrH!rp=#GPBdB?Ff-L3(qwO^b^DJ!h{rW~q<1_*4OM@&4T-*< zSbH@^c(>a{IqXXrnP8!$fexi+d}PF5Qjt1ssB(l5h!xVh)gY2$MLH>ZT~3$h9~gTB z`x=^}aNUXkeR=;WU2WlAcB!dFft%ue>F2URM7Ni8p)Da3gix$JgvTP)*^eom>6c@=gk}ZD$L5b%4vSuwyssr_&FU-o zfj<1r1iD@>E?ay}W;P7SrEy`NA8^6XKTNsB@qIgDJYL9P=|@f19rr6Au%eQuP-UUY zYph0Y0x9PKg}|yVL^2MhPw)q=F_pM>m|@@?p4(TyqVK9hvHJp2GBZU9U2}N|lq`fz zn+V=$KczFoAf>pFaHrFh<#w3laodo+1RtR0UvqHKI`c3;z5#1g8})Z<@nQQssq+cB zL8*t$O$T$KltGyK-in*;?xP3t;uDj8$Oc@%)VgDfM@vjQOW7+c&U&f0zmneX+2cFo zs~nz8-2QXq;l3Y_I|#SWmF-v_H0<12`f6*m^|(R?m-B_k&8p+*<@3_O$glgLqG)2m z2RHtFQNN=*Rlb0~Pq3vd=&Rp?9=+d;htgQ?xf!gvpm^*lnwcFRcLCVE#&#SHBKFBn7P_d#7kOjKBM%kB-JLD_-)>|*(M@yZ}`c^>+);nFnXOSVYvwk7n;v{L)5)0uJ>F>K8Hnl6!NLQk% zjV7((*0{B#OEOBjFLp3Ug?(7W@k|9$ECxD#pfj?c4 zCP|5k3|I63@pXBdosCX>qnl^s2ZX#9VWw{8mz%7X^@)puM9K9Z=&@g~`GOqb9;U>> zjuynEAok7qv0wew_NFC`o&H@h`}>}md~uT&NfdPpbw~1I73GTh3J6rLmfEk+WEn$m zzR#VzVTtV*vZ5N;NMIu#_>siN8idtr)T%cEL)AjXYELo7W360tL20zYyX=d*rEF{y zymO@aPXYx)3wA0}=8YE(*D&{;dkSv~d2Pa;ZTmvH@-FZMqmexhw+&H_NSN2t1x)tm zlSzRgFJtDJP|U3$ddnJXdiyXpxphUw!@D6~sP>M=*Ue@%AwRVc>Y!CHg=(?4=X&bl z1~N)K@zzzjHAvbD27ChR!)D((9YB|RLYH%Vm+R3c9A-c`@IfQ;!@i%ca$T=(b+4|6 zq8>dvfooaDl@&tCL*?=ZBU+xFdelI~aVNb(P{uvtu?3T1VR2zz#42xu-;`|={X<_` zt*`4dugoFb>=>w!4KeG*hnwVmXo1)*JfX{R4KF7XKWDh2XcLSTLWZxSv(`yL7DGSE zE5LdRCdP{B!~-=M7!Q;*I)B|E1~xx!#4BI=mUpw=jxDe54E>O|{Veaed+{L&zTUnk z{u-~|y=TUolx2JYndWsZf8L=vrzM5Brf0Ygf&`vXKBS_l0mMu&P)$ZA$L3wpsl1)W zL)z33&FsPt+;tDz$1wdz?t9FO0qugdgi94VN>x z@SM+X1$Qb^wy%P$$C9z4+}@A$KMZE@X2@zaRL|)*NetswdG+(F6~mid!|TSHeh<;s z`WEFOad8hK?B5O7N=9Asp7B1miT7HPl@`|s(rL!eZTs;|S$U^0Dhogt`q~tH2hivm zn~ZIQwXj)2z=tY*)Y*s4RM~4|6S|?O_j&r`wd4o&*q`^==IFOqU@8^;ecFDcuf>k7;fO{F`kf38ocX#MPr$~_;V zh@5j=L}r{y6bQNPa3N!3Fsr(@p{@I8S*h!Cl^)E;_A8Ej0@c#;07VP~Hq8*?Nja6t z#I>c~vl3yX&-}&8b1IC6>n}QQJ@rL75VJ@aK9^i2&rk%9x4WyRvClKwXp7=}=Y5rI zpQ4XiS;J+B6_=GDzOW!8Os1}yd9Qx<=7Yg6swb8O=|f^|xa1RE zHfNulq>?ww6`Zj6fcLxBcZ-e3?)eP(4G}0{P@9^#_6zcnB3CaLfv!ouCP)rL+90sC z%GW$9N}{6no1NXm=-*!diCWkz9KMF zuQR5^v0(%u&^4bTnxslzJ>NK`DTl5(5c4`~vzy`aor-?2i5ZL`&+V@LQvCNx&VR>U zQcj)u@4~ew{F-DR?@q(5)(>ChWw)@9>s^A51kdrrRHaZ6-n3-`84 z=E^JYOaog-F|;qUEyBGmOZ4lQ?M)4xp$avzrxoQUG^5+1IzXH&0+%{momrK8)1D_d z_KFMQ5*IaRxg21_N;Q=Eq4~nHc8SNgVhy-g%E6de&JHRO?E(aF11bA-Laww` z<_+%?G3y8XOD}HQ-AA~5xGb%-#v%q?clNJ+7@w9IceX#6{YlN7*%SSG;nfGd09Meu zO769j6moestmr=&Qr*M^r)-|>?TNcAZFYZn85xUaZN-7c%px1=5T$SI=j5<4=1J8H z4$~d*E`l1F;)*IWGvBor+Q?dOXS6U71W^~d*4#eAZ1+C>Q9-B2V|CS#v9)Pu%F;}E zOIh1w@!oPh3uP0;uA|d$=-ILdF%PfrgibKc2bpSk3$HJpa8~<=*FHTH3XcZsCY1KA zlg|L}6CbSh;`h)6g?^K}#Wg8%xgG#@2zQQ^SkgY3Z?uX^PboPj0Lek%lyg(8m5U@T zil16xF9GlDEnCAH*9k>z@T#nxgIE`dx;kD_a7To^xNv++`}`Sf?5zOC^cgo#?H$PB z25oLHR+;M3$cl~Y+E7PnyMo^62)Wbw>67=a7b7g`%rIyGtF74`Jw{YJU-I$+I*cmV zyn%)ntBi>vZOkbNl+7ThGIegaZQXXxhfxEn+4G6G?9wYD*=0Ho1o6fn!kZo$@zG}+ z*yQqUjK^K?KOuD?f2HB7-woD=!~sd<>;ut)W9zhoLLeVs>bJp4m6Z+k2UadXDa_C? zxP?m|WFe9J&43wG`N%wQthh6GIte{5mlYQgZY4vDeMzsk18_&lo;=pPx;M?YE>6iLm^8EAsZ{Jb1?3$1k7n_E41A+RAzXtmWf;VzA2_sL@ z1%Ea7#Ei~#%`oZj%^JT6HX937)s&=g zFI>Olpp9!l&^lX?bX67xSO%V}8HI0LC)1Ei6 zsh*qdFXIpxIfXHy7ssC%Gzg20-K#P@coW_2xQ@qv!^R#CvGzv#5E5ig(QB+dvS>4n z0O&j{VW1RH&1)%RHfQf3E~#>T9#3HJq5`%Ck-GXJYWxkPa8)_<@Q6Zi9R~c;Ml5zg;OFzCRWMXO4{gss+-_a%I9;S0$AJnN=#l@c-J0?VQ zueU2^Z52Me#B;klY^TkK(k7B6un{~lk;(HnB`~zO$k8w?O!wKUuo{Ni(zmjCVzl*c z=z1_?Vno!H}{}6Fy!_d4)J2?1+1eRi0)g!(L>8 zAKc%Jxco&-t##z}{WX5I2U^xk&S}r=s@n058(U$X^m>Ao<4ZyCJ&fNjRmm;D^tbf4 zxa;P>;5m#H@y1)C$25C}{g^aO%2^k6oVWIGv}w%4!HnJ|^&cvyjEyCm!;q_aEsZ4d zuaPA_kgh-dczwTiHZg`&&K;^2?EF5eV4AB)A&Y*g&6mcdNQF*;+qktP@;VhL0i;D{6LOm(ra`F7Q&+Up z&LZ!jHA4FP6tiy4Io0p-Wf8AOkA+|6uH2Dtbs|JED1iaDA-lbGpqeKJ;=b+Y{FZcfBXOn)v`0apu05Jaxnw?Plj-)u(!#!^-@s@{U+wB(i>6_I3sOA zSLjZ~=bSQiEmJEu&Bnhy-*kO>d-KTZ7N;5XOIg(n4kl4a03DKGQem(Q<>>O_VqUnp zajt;-(vH9yixIFNV`qPLrD64|)){S>FWZbvX4p7-Darut-9PrmFN(E7RxH6lEK%5l zrbgU3s#tp8>gElzVL_84&-dMWYG{~Sm=&xGu~|b=M@L6D8O)}_zxKQ!GnmEY-9wZE?5vzw)22Dc2*Q<()Ad8^OaXE?L4RFh}MF3aQIWm!2M(SWj)(7hJ% zT0T0zTtsy^7(b-k#x!hh?K^!bjgT0h`lVboIBBiE znP9RdhPoclNTwJ6F7Gl>^Q+;N?B)p(7~Jl$aX%8O1`RI2!Pq7f2G)pLP)tTEhhYVt z+qM#$CJBO>&|ozPXbG>^$A*3Zk~Gw{&ndt=6ZVH-?x%j$ow@vuUqL)%VD8Yu?-=JK zclZ$ro1N4N^Npk`x}|i6Z9Wn-a<|itZU(88Aqg<=f@It+ar;SuPn>CZ@cBnZ==uLH zlY25~2e-HE|NG+Kil1c)$ukvP#}!AljN+yyot6ZiXp+0BTe`KBfO-3)N^a#yyw>K0wH#58_{)DN6>4%D z+@f#ixK>)-EhfIYZSS;1p2~s(EE8>E1GyH^s2yLuQO|shCP&_u4GuOLzjgGA+1${Z zWby6M_s#1Y%%5(J)U(pkB2GhPhaxWgYK;+UMrt>V%@<27t6qhFvj4gu+J2+d+E}E_ zRMYt3^Pt(>C(X6g;Lu_<%s4?kyO3;2bfO-Ff?o1#ilpzyc_y~m&Ez9>rzkh-%fi_E zKO&VlL%i@#1EY-=Qn^fH2G4Yg>`y*z2&b=!NIhXCMp=&?fxnvbq8Y5~gL#`)a{wIo zBK5~aV3Da6Mui%Q;98uV9C^&w&0kHQF)a~uoJf0hxWwbz9CR?L5j#mx2BCzpyNvW3 z6VFZtx(&@fq5ncjA8Gvfz@wqXgW=s|F$u=n>Wus^v*Mlp+Eo6@$D>08Mm2-+!)#D+ z7qb5KPpBf{FM||(Dr%00o<$CPMRjH1)Omf&(tIY~4#&iKZWYSTrh5n=lD zFdqMf-w(&Es5S+q8=2HcC9wJ@tE`qPl%GJc#ZxPc*>lD$s?X%Ebo(#;Z)#&y$nVTV zDb18s9a>fkTb{cbI~g#!);f)MXuHV_xn`OCwx62b2q(EDi^^qw)caZFSn<{ic@gB| zMSCl^HS(zYmp;*VASmS)U28^fbEugJ@vHlj+1m)Acx8yePN(|zerqIid%1Gr5>o2k z%6LWpYRZJ`$kO1qTlmGt_BYSTsm(#@)gkf z{`#D9=+5`7z}dq=%`@1!qlrBWhMJmfUtGEOc-lKUJ1Y~F7d#bbPaoZL^c0wDRX6I7 zUXqh(EBNFzLpbu>(@zMd_GGMR+dkmgY7*avTh@UG=k;$F02d+>|2{2Qm>m3}a_SSf zdBi?}uo_a#_Y1o!^@8L5$wwc)!=w z18>_B-HgourR2uNpeWz}w11)2$YP#RD)w*6_&n=8NrQE zDFNR%Y5(`tlpd{{$RumxLR=9hOkZgBDjhH}QzP{}oZ`4v-vuq>)6X(;Hi8Y{1w%rH%#A-aC<2;WTJVrdvqRQ|h)H9l6hGe%9UBk}BG0vWpNmK&_s?op zg#BSpI1qZqzaDDyLA^tio0LO}wzpd%)BLff$Bs#X#OfgPbj@Q@aEKuNMbC!p3k~b( z)gid?=6xDf_1uyEmEH4c`qkfjSC<*;*4t1|iBf;DZhqg~Qu6C=Tl-g(`hGK0z4!@C z+z~8hdZxs0hNNvNpdzcGYzRp#JE(crc7TSb>sM#ZniwA20YJv|6&!2Z5CjX?4bJK&$ zRt$oEX^UV_?=%!$(|icR{J#3_xE74_xvZJ*<}E(ZL>)eI4O|rwq^t#yB8c=Z zq9Pzonsk-kl@1}W$RZ+5dhbnY=p|G|i1eOFjYr&aa*6-8u???FAzx7|7WCvDBA>`vaeKmxtiTA2K8g`@Pk&?M~0Q-wxc(|2x*q z_V(;#aa{1h9KEs%E7r2tK0bQdje5M<%hu^ybA^$Vy{r_}H0cRzgrkGt#+VD6wOK|b ztz=^n@>}7mP^O>ti^`pxt%>T9-!pIheeTOly&Zz^%?>=dT#G$sRaN6T)SE2#wiR;6 z3LKILOHxu&4uKdnA{G$98Gt*WJKVYAJNGR`os0kB>cHq2TL_LsZiaXS+Pzgm%P%@* zKf~x{!DFWn?NY7}129ccY@0@Jk!#lnEAVwSVwqadyW%oO zR{~+AcW(fr=^Z+Vv}6UlEER%L#6)1Mj)GSu2`rMGt?>+Ywqol155eVq zhkx&DSYEBKoQnDP*PeefZ*MT*;>u+=W6X`Krl1z6$exrghSL(|qvG7yHaGdZugaS}OCF&RK(=<m6{XU)$5 zr88QWMuun0LmQu@{j+iT<>KLDx9S2&@rDv$Nxj}XH^JV&l02m6!TZm>C=XpvkHE6P zd-YIawA=OIeDMx9y$K4Mf;y*;)_>A`qB}jqHRe#+KweF*uKR|X!Un_^`ZKF8thQ|4 z6o-4taw|+b`0Fu=AM$>DY^W`88J7>Y^w8GY>GPhqhBfm;yW?6Ur4&iiyL3NE^s?}e zPW_uMcIl9&n^C$e9haInXV!SWEt3=`Rt{YIrhjy$ulokEQ#wOys1SF&&vZjFp`9vz zM#O4w3*WLaN^K=}K5adA=ZDovphJo{{ekGXGZY;Tb4!qQguZ6Q(V^Lwx#B+?80oenR}3|tGqphlj|v}5A8 zd~#Dog`5}L;;|n=EskK{nW2*c4t&NGilw{qQIU^5(97fw>$W?v^t2nvpm9m;E7RAj z$EInyaOc0=gb4igq5Kj?{|Ls(yi79-q@R8Z7+XB0Hi|EwOLp0lwHFIwHu3)+D!rLtksaf z@(}3ipsaDL^L}MFqWxvAk_#>=nR7TqdB9lXs*H?L=8B?E&@vUVV`IUt3+}&LRXVCu z`A1DAaWB>_`z>bG%e7lg59Vrng5Je=-Z~x91s%P{(Gy@S6Ls|%>~@>K_qTeBizhK% zSE604z3o(R%XMSsJN^l>JrkL*3{^#eTRnCqG5)F;gNC5la8(%e=__&gq`SdnP_e$( zOoG)oi64E@wByqlrd0qeVpX%ckaE23N5xAOQV;*{YfY@1d6-L&YRwuObpcAg*_eAN zMAyGLkZ^Pnitjk5&ru~a(k^YiufQK<-|FgS*=Hd!563UJ$m-|sJ6s>+#8P~aJf0Jv z9U)A?y#SS{Yc-R}XvC#zsH4h`GQ7u}1xhRg{I(Nbp@k-qezs7;Q>|w~n21mOFZ4=h z>e(T|Sbr3BM+cIDBGr;JQU&yQznu>qlmE<#C*S?|yO2eGpKFEpKVG|6;x60BES1lL z5b@m5m8)K)Z{0uzw%}=5v%da|Z_oVM#sor5Ri=(M>eXGd)Zuph{Un|=Jloe-bq0P( z)64l)MRoNx`wnOFmf85}kR{~$WKCQ*Xm#C~M&K}#ekb)Gzl zxKH_bX>{x9?{?$xyB>kTUW-3BEBoha&BA(0QL=z)P-p*(Uq(iwxKGR`3@Go8+43(M z7ZMrUw00+Pi9eJN;S%w*@5hu7?DF7W0C^d0e5t8(OXzB_3KE2s+Z^r&lcBl#QeOU)fM6>dh_s@ZL#hoW82&yF7UUTRT;Z@`eD$>z5mZYaUq$?zRvQdmS@ zl+@%8CT%9oTW3WvEPanwns-v}K=Dof@G!}x$LbqVKF)k#CUzW=>3;J>Q>Vm#c71$& z5%{!|9KF$Lqg8AhU*nf!6uXJ7IQ0{kFWYcZ+H(A~!g+#nQSVSkhl%kxg+FuejsIn7 zBbJDo4r*-uKA(riKqVjE8Q$y-S6uQ#xrMK_`}h$0CB|^1mX7Sq7t>#wHu#IJImX7% zgjky`mz1fCWO+Z=i1hEYnRm<9mDo0$8yo8hTL1_I*;-p&7`I7R>|`fnbGilSh}#+)8OyT%x}zwF^v3x>&Uxt(?y@3+6%%) zHu*cR`t~5R8*B4!T6$w%AO0Md1^nH3n9>fk{0ToziuOiA2cN;wc7Wt{o2tyOWrr*o z1yhxFh-zV%eDj-vOwj#xFCDf=#(E@~D!A3Y_cjYX`(MRP(gzOkgrmo;MA)5gSQtNY zTA!0Nzy9fD0N8RnM&BFM*GnID+-dXzrjPMZf79n=QV-*?8!GKYni{FyywEfT{`Rdu z2PcX(P2y~08~U;Roh!y4TxStN3&aXxf8Bb^hl4K(QR-8dI!yAjUZQR%Z;L!`e#&z7 z%1waS1+OM*&LEMOTXDx(&Wo2%b2YiQE@;U8{`a2+?d|3gD&G0rXiB*iHZQ5QbCoc( ze(wrsQ0wnpSt)*(Gv|7GC{Zc2K>QZ0LMdUmN~ot*uNK+uBWahbl&M(Ts5$P$T= zA2Q_6JYEUSVN29PWtG);I*FKT{xX<`&%Ljgmba19Y9VsJx(J5CSpcg3_SzpRVVWl2 zXCskTW-G7fGN%>3CbzR7==2cZe$$KQXE_mMKY8xqBE$D0P%6Ur1`2$3ZPiHLGcV&F z__=f~(?OM3NC|mXm}~7_SeC11T4AnL-t6N$NMusYAtJ z+UY6oaHEV|+}X;>E*H60y0Z9f;`Szr6J95&;tYLx}%lZ6}H3Zwp`ltHVMicuNh6*!_Qq0^p*Jc3eh{U zOUFwf4|j9f59vgh95%}>Z#s<$=Yv>3WVP^Y(L=_>e6z#Qu#V}YNB5u68QV@esrl&L z5BBAa)|fnM8()xpdM8xbfcUNc5q(`Q8(8n|8Hicmw#pDHuHXN(fu4Ya{~wyEw0VM!J!%6kVfF5RVnES-Ert*1Rg%*FLGH!t&7Tt8Yazv<~(1G zp439CDj7eCL7K^6yAb|ZPLxXdo{A9bjTt_s%qEGarFZX)nhlIe7%z4yKj;9Kit(dy z$O|?3E;XPd&Ya}P~^2Oi{QP|-0GWq*bs;%XDe@5i~ev0*k;g<<~I?yzq z*%R+=9?^H`B@bygkQB+e;j@vR1sOS5Dt@wTn9AG*3o!R5zt?qw6P2#y_x2_#itLVApxkR z@+%c}|FX~?9Li+VDI2x|;i%K=Yp(Mr>>%4x0Waa}woB&!IP@9ytK_Kx6+2LL^BxK`rSwEh4c7?Iq5+VfCO8dQ0mAJ4bqAV(BPksDEUdi^64@QQb$p~&m zL=C2U*38)AL(Dzm#JOUBa6UGFg-a>ep8+?4J}612=33m%;`q9$alPXrTEl8;W`aH& zn>T>NP{o-Xjqj{k@`~EAO!DtGtQ6|$5#MOu_sZvLs6pdV(+px>3rzT2rz{X=s?s}j z>~-FUzK{Z1=X?~b2Tgyg-I^E5zD}YN$BH`$NczGtm_xkI8Ikqb)Is-|X0(!Kx$9ip zf-;lPc3!Ab;dBQqh+&>=u2KC=r8mPRV1{#JS#>0J?cg`!^gZ+NH9*qPca5r= ze{8;NmT5l*Lz&6Ajq~c`9TehI^yAH8S~}f z;DF=1sbE2jBjAQw35AKcGbYViS%C*nPj{pyJ~&70^pBz3Mv?9+GI_ z>T1D^L`UI6TLm2J0X@dM;B%jGxNdoEUm<&oH7&|nx5;_TAu^YqyD{RI6EOHMyI%oUYzq z|0@XC=lvM6U4-E{_16ReJ~9nR2~>&M4p~iY`%Vjl&!J3}_YmgHn2~H&Y&iDKzeMdV zG*MqZEihgsFc22FVC>^>YCRxgXO>YfU$^bK4>-FJ6F2_F+A?v4h0!tDuKlliOQd$Q zRoC5rpt^cbnm1R_DnfO%DV-=^pj355)h;7yXrVnwm;@@ln~*i8gx{@ena%F(v>(l! z4)O%yi>fmR0SS@r3;7}S0G&xkH{PjR^exg2I~p0DC#i3EJmw|wcA!;@!a}e$AJF+= zMjN%646;^e+ic@I{K;*Sj*N1b@LOaL`L)VdL)O^9ZcZsViUcGT+fhC9`Dm>u;*Od> z0B+y9hiUs*X9MGweoqInpKqVDFEzt;+628l;w2JF`nc19QY>ma4Zt%`@mAr=7FSIEUtR)uVlq;68JcvsUsDMxz|6%}>ybTdCc z4?tX)@21`5M zhi{kt4f3d-;?Ve+1KeyCj6bKvi=@@Ww5w}5Ko8m1bpOWFoT)pB}tTP+6WnvR@ zA3;J9$q+uVgB<%M5VTJCs^ax^Wp5Cl&U6i^hLk-s8t1QCjOJC-fW&CZXma}ox|I&; zAK74PbIv!miBq=SyMoXl0T(Tc3&eeJ6qhgnBXYA`hY7_n^`KVtOmH0)cps!)CO_H) zD=*EV^e4BH+5c7(=;*Ee+~|E{%juI0ka&5?Y9)!V%lqMH$G_&KSAQ|Cf+M{Tw=SX{ z%;4`?lZlOffo>+>fBVin!jQdm{C73`i=)^6^7AQgUG zUYl0a+495McU%Zo=y@<5(9`jlK?=M2FGpI6aM<*y9H-0rh!~kZ)#3prmR%KBmknk1 z_^hnL7*!>uO37_=voaI4rGlpIo4U_Fly?qpYI}cM-9MZ$SgHh4GsT83izD&K-_n2y zB*u%|;)Kj)8##0492fD~&f$f7*> zEK>n&s;4=DTh}tozwl@;GkpfRn$|H|ta(J&1v*8O z$@b#Zq|A@VM_?V6YHb&u=bU2bx~+^B@aI$@x|uRsB^!y^!qLD3rUF$ER3|+lZ$-o&$Rx5HUKEtZcxZ?8>bRP zBHkO1v;}rvFrtckBQJvWH~EYZkls>E%uL089UnNsz{dmR%)A7 zZWhLGbFGdh%mrUp!febjf@}eD4n(sONA3~JWU8KS*YoOzfpXz%a~In5)tc7YDs{ku z^mKGr0h%hTprX!{tsWip%Y^SQXOldV$FZpQu-`c);lHFBiHBjnmr{U;iugSpKO1Wgs*K;MJ(Z(xiCrMl7|;R} zHEY-^;dipKs*IA-nLOsX3QE_Xkzy>p&c#-}8Uze4u)HDYB{y+4mSy10FaZKKfHW01 zZ9mAY5p0&}n0>ZX^q%eqPM@NIa-ExcYN6Pm=QNm3H*+=pwUV~tWj0a#5E~yey@`)S zPq@xbMHk1-cOATQWeecZNDpzzv!SkN?w(cpx;Hr(T*m)UeOx7=X~Z#NIk@Tx%7)sR zCeB-&Dtab0H@9SH=rSE^D-g_}@7;p{Z%-e8ny8YJ@U}-RUv8ynZAO7f)f*TUC}VX8 z2a)gWb`k)=cP<3F4E)F0S95)cj}skbHs2TyD#}S{QMat;pH5dcl;y>gys+oaeu*?U zj|LXcp-YZ0#8@WRh`~faC8JspImPVSj zBYCaEO&Hu=P^D6yQ?cTht@NC3d>4Ne>x0~V~B=`_E57=Bq|%nX#k`GqbKQMZ}5 z=Qx6?xZ=IQ5avFVXmfYYP{%Z+YMllItx#!eMXh$Uw)oho#YUz-?eDhmur=a7r0dX! zHLQF!5>-PSZ719wB^M%;Yw|{Vf|Hm9E3{`70H8+_ot1zEsW6Ye>#uMCV~qRAI0l;<)Cu?RRo}ypN+kb=jm;P`r$Uu?~-819FPI`5gLh*j9g&slmVEKuy932&O*T0?pSujEj7u4|t{J11C zaVD(0N_QNOO7YPYm60;^81-Z}-Vh|;sl@;*@U4L6{g-!Fne`o0&Lss)ucaRAr<&rx z!M;!=16=H@gm^bSA8W{82$}MU+SCAvi9TDX%0$$sZ$5hDez_-xrq)$EU9EOIy%0R_ z^hhSV%V3fr{QDb8ZY5ch!NJSG|7vVLSH!F2+7t?H7ZFwRJ|6S_?&qbNF3L^-=jM7g zgX55^I9H-&W8iu!t{S;+a`L7YFt0Oh>P)?Yu7Xnv4|(uzrY_LbSG=^X0VYQSBadtz zvI=I%iqz#}4E9h(12T9wN`vxqP|9T zHR;Cu6SBOa`#Cdg_NGayeDe-oNk9vjPB&3ge3q>u5q zov*qh5<@`4FAcoDP2)$pmy6e}RSmKE^P*bJ3LV<{Gv9kvXgJOO)Cg2-RBx(0EI?}l z-hOeK$UL@S0$BhM3Df!CXA99VUzwKP+%_B%+4zbev~{|^ZmHTb02`z#POaKszq^|u zas1yK=f9u%6K5owgMRP+<#uvE_UtXqj~e9lcM$dF9uDUFN)j zT`*{BX68qB)izz=g|VnI3sAn(-8EN}3oz|56g!D4zdk)m3o#GycgmE8*+wF)hwIiO z+ptb^Kp68QDJmSlt~@as-_Tg`7^^iS;bpZ!A!&T zZ0Wi1sJN)RkcWk}M+ZgkT3;hgl7X;SZg^$ygpK7Z-0C~30tvY(GH3i};Q6MR*Jo$= z@zn6WdV8IgjoO}?nF!lZ-8G03-@T=u?x?rr?;%z~0ZAJ9R~2F{%SJmual(2#AuEf^ zRP5aBZQgzvo1OMJnZarf+f0~1>ytB8d0ZyuPeQtP;`6hMnU?3dq+oHypZk@?o0_RAwT&BP*R6@8%i}dbik45Cy$eRf)`+oign#}YG`8ToM4BXg zjgiL6QMWOK-_Jd;0Lp0?^zG06oMru6X}>3Xc9E{D9InY!!rTpaJ2a0+*21OCV=0IF zX_c=+ZSv{j44Rd|oJa6x`qecRE*(FYh)SJ+NQOpEAsj7wdmACsChC^%_Gv6vW#9$f`a!LHOb@ChVmsv zZKfR7N3vLEJjlPgrY(G*sKpJfGqsu3|6Hodql^cvy5epPZ7DBT6B&zF{rTyM_N|*^ zJyw+`0Rj-evn78SrCMhVlKVw_TK<~wJSdF>plK<4-nScCd(_z#GQwR3m;i1Y221om zhXX-1eD@?qL_kFyuhODqP|%Mdo~%8YTYqVT!caD|v{k$kIGVNyF%$=(5|*Kz#+fTq zO$Xt;pjPh%(E-qMUx43~z-6nC`n%AX%(-6SuGtJ2Y^|*}{tcT6-7Wb83Hp@sDDalR z=)1+A*rZCdTuett>NYb&tKFov6IA?5({y4?LR{QIH&pE`+z%Vl($}-G~POnS_*-yNEyBx8M^8-e97C zW7C}JIVG%;P!;f|Yro)C{Wr{Fgz9ou>o(UJUH6hN$0ebb6k2j8hBID>`3S-2V=tFT zWHT_y-Oxl}cCio4Uv=x<-`WhSh?Du$E&yG#lLS&tFrarIPH+Gd1Qwth@sgi^d_X~5 zLyVIKVzp)|#MaHNs{HBoWb(P18a@>U?KkkT*=`_U@so6KNmwvFMk#T`_JRFP7HWsTI{?Tp5Co zz9;T?i!+2N=AQJ)djleKz+eb~gdsQJoV8N>pR40Ord1!f%*>#We|c(OAg;nfLP31O zLfGqi<^>y`Q(7azn}Wum5JwnVZ?p5_HV>1;u%XPB^}@jy+L}n4-bVn|LY(|ldDPnc zqfFQzYACSLF}D;P5hggimy^@FLJHW9sudLbI*kZxFkXX1?2iO@c!4JZED}$vEci>8 zwAg4WZl=k=_t@5w&=Pp5mgV(>2R|uSV|+JpQBCWK@jO$`m>8|8b87Nt!kN()q>1=m zatDlb&+Ed|YB=^NHF$f$4agvrb#2KAlVAeYi*>I&I~O-k z+4~gFLsor&Rkd4NI zoWn@8evZQRv^|C~9NfOqVTVKJ0IFe{q~lNajKkn|y6PSlS2Y;Aaw|o<@O=qU!nan% z{Yf`Ax!;d2Z^XPfzaEmK#R~a9aIg`@|ES!3^E7hMdkKr*M1wCYc^OJL{*q`0Ym>agNdCk~!bL&j#*AJ^DebWs{p3o}HMh zs$wK+c6sY!enxG(4p0oKzOGQ=L``puTSqcWqdS6YXeL-l-jnekeGv#a8y2sl^2 z4Df+k0A5Ky3XMdnHJdSA86b^9qeW*_dnTxLbRx;Je9&F;oOcR6pk%N{Ml-v~+kL3O z9lz6oaRV?DbU)k64U3%SSN7DEc4Jlt5Y%L^wV||h^nG9@9Ht4v76@k$#QD(qsn@Qx zZf0Q4AwzBnraQN~toN@pZP&IpU>+UZqkJ-!SJP)4d^52iPM@1+e5$|qI^QT(Frv)n z4mguOi9A6+7(C#|q)hdhZ7gX9U?akI5*x1*B=KMMkpTR40p>pYhA*%W2{W%3Lw34{ z3G@7FQxuRz)GrEe%>+UU)8(#vK>dRYqCWnzz$9%u^5Ld#M|BSuY!p7Wlubmw7#JHu zGDmfwoO(gb{_t`Z^rq3wm=-_^X==8Zk>*2bIXt)99kKvr7<=gSu6gkk8-2mf4BHRf zu(_~wR*pqQhtF@6t?9UjL%A7z{yjRLnPo*jnpmQm*DMH``iI7=!;7GQGoQ-}^IJs~ z{U`%HA20aCG^v3S*UxV{e}fTSBLJPr-E!S2|E3B7cF}fh}4d&pTO#I0lAJD)GZOnj1T6} z+Z>VF!&*~4p+_$YzCR2IN}lrDwe|r6n|ja7ymt4_f$>2kKu1tWGq%c_sGnS{5h-#4 z8qCa^ia9O%ys24q1#qw>q;G}8I?Eb(c`*Zi?D!Obr7$%(6FXa8x1S!Z`gR?q*IdH~ z#R*0*@3=LRnDJ4{SsJs^>q=v%!`h79j@lk#8SQebGS|bj;o?0`g@U)aOM!^teAn4-;}}D;SpCFL6O3a5 zK|zD_bw6aG)CDZzS6|1>rJsB+mpggrI|0MKJMG2P<~ShdyCo;5m7251{fpsF3)L!)4rY^!wL#LeEUJ8IB#}QX>n3h&vL1CGfu?AHzw(ulWIiC8{-&9 z`)|^7o~gIvBaMFhefaCw53{bR!yJrbKY*U(Use7J%iq_=fZS`-m$hph7$`2BVL2cjJW391t`VIFGO6*8^9uOkAaS77*W3QsY_4h?G+t$8WX&LslJW_RQm%Z z8>Nz=3*{DD3m_vx*v_}o(MnklxFt$b060ZIDv<2^M|I$=U8>UNSN%z|{5FehGhPGG zgZk0#<$^q>HP8Kn`LMyVu@Euf%{d~sceSKnF~aErLmsmg#!- z#eAfok9(y!gA|$7-9(~UYXQRC%r|5fJlhr=syfxU`>=H}QO*FnqvoKKSrL#)RIqJ}P(zsP!CAbM%Apy6S_N3Fy*6l~(a5HW#Vx9;B4A8da>u}lj1?3l` zKu(ig<<P{~ z75dkZ|3C#k)qC02ynoLJbdS^>#0~Lo==QT>Bn)QY;Ly0WHSf^Ootl?d5PgxCiIao6 zKgp)#`6!N(EFz-#UYdqyt>08pQ)n$1qZRF9>JY1CvQy$)S|apC>5C0$S5ol`g+mc~ zz^e#|%E2`E#2a9}cA5Rx#QY0H4^NwBTYbi_uHLM^BcrTf=^=Kk10JyaE z^gpP`X+=7%J*2Uamx1&=HPpBQ{$5Z9)iRj$+!QeTKr|!b@i*zZuAhe@yz0@|dw`$& zDVG3qg@|V8`+{HEa30v~CETTj0~q9v<}pb>c6i$ZVQiCE5N( z3?Sp~BFbC{O-}^M=8FD2dM6D+1w{t}XrXNUwI~pf08m_Gqhw2A;md1l0&eq7Pe$`{ zRW|BVW!j0~jwl(I7*>FP48|VeKs+gD+%#y5>^)u*nGMd16~A}O>0n}LIpD+nqO0gH!bk{>z%9^EihCxsoD)99_ZKvma(%i5BXnwzPD4 zKTeW%G3*Ar;W+-R3ERTR8zKuFojN+jVbz0|OC1D+J0@zkk5TJ7%r)F{ z-%U-GsqF5y1~t{9?T$X6m(Sfo0+)(RD9v_F#e8*})MHyF5f-%kH{~o!NVVRAafsm4 zno=Q+punq+4BadGx}Nz5GtF*h9v$(m2Uflp`;2Uo%%!G0veG<_-a7>5W5T88CW|$R zNK3DZrY2)mTiW}7Z2jjPESj7R7hBj_uJBn{Sb!x+B-XBjW*AhLnX>cSZ@quLR?H3l z87*XNZa(LG(85aCvrd2WtVI1Xc0saY zjw4@ZV!6x>$BV^o*OQw|&jzAc&ZPiKwkTd0AEBhFFzpab;k6Zc?ZAQ zE3plTJ0yv<_$4 zn_8GAfsvhO#_bcp)b$-sA+YUFGdY6t=uzQPnn{XD`+CTf(PA{)oKMn? z9jT|~@i>qe3yS~X?0TZiWOd5O1BZlG(SRa51DGiIOvR?s8)$IJ0eHzUo@URHmnt-U zo_w|d1Pl$4atw^Dv*)v|?k+sheu9+;8Hoxh8QM0!(qHWUdtz0d9RBbh(v2%WK@6b5 zcQ+tFZue=tv13QrFkX74LF4OHd)DGv^Yr0EOkXl)9N%^Cp-Y60mv7LwhMNK<&j44N zk@1Do4M-K{()*8o`&$1JxN?Dm+n7H|^*m4_UR=kxe5Q4NJg|f}KLwlR!`C8Fy3wu- zjY+@m}@PsW==4mVN+c$isV^6SL)XUq~N5HCv&jZpdg@GQKB!hBX438@uNK6;gw0Y6lCx7FtbpG_p z!4^2+AS1fJ=!~k2lttmWcR<#a;`2zere~zyWe7OInvd>txsqwtTyd;%nR%s>uj6YD zyT+(#-L-?v6Ak3&7IlB=pSk=XApZWX!u8>^NR{y0gBOU{01N*lM{&`2uK9zTr(TZ_ za*H4igu#DiwTn^OMAwl)*!Nqqcf8$0{PVM17mf`Zr9FV%xyXt-ycBa8BRHTGTNkLT zx)gvu=-mTao53HCucUY|C@9MrGO$%>C#_fK6y)63oN4`( zu{9x`17er@fXASczDfq+`_4azMn8G}e3aCHoAquD641hghQO0fy3qJlHL1{yE^>wjs0X| zZ2{wj@tziwfY-nKsoQ6jjeCq%*)ExxcBgK%m{rxm%yHnuaT3f3d zMq4q}dTNnnf~M~jsn3bO;X2E@CYYI;ZI28hYq`D(GMV~5brhV=CI|0FJlu7<`xqDj zfm&Baat1Ml!NRR-L%q_+9p0*fc$rt6!@wa}2I3sWPC1r3 z3LealE$vLI^)TYMhO&>ra-kZA)qOFQNy+4AAc%BEG&yu7mmBf!&Qy*frw|BuITCXGqQHGt&Y{CqnwgyhT@7cpJx)j1u3m3|A~Z_e z-uBB4T;sfJ%X+Wr%YS3Q|MvrP?rJ<%DnIBUKjY9CEcH$VB8&9C#Pf%ty1Fqoq2v#t zd&%t05K?q6RwpfXYE(`?PV?*QY2X%FY-h(d44?K}@R)A$1u6)xX>u`@|6wOrZ^lIMFW_Q+aB)kHr{`av`Li*dGT zh(mWC6-@%2+y=47ebx_s2Ff0vhyf;E(9e}L0(?eT=aFbtuXu6&is~R3^Jw{vymY$>Y6shK{K3Pwqb2pi zj=GN4i`=`(v{62Y=80>I2la=R8{_xpzAQMjNUl&qlL+)(w zde8E74#!1l5KbDGom=z8*z!9Sv}OP>$p8;mR3)3@aaaNnNC9AD;M&ilY#>`0?Qwnj z1uI#m*mTf4j&0c(lZwOP;N&9D+KbbFqb|2KMcP)Wq>Ho2} zuJ;!Tvaa&)Wvll*YJO|-IQ#Yt_(-`4!l6JLQZ(==9e@g}q?4`ua^fZls=+}8hrN$~ zl5`|`LLM_el2Wo4JKP?H#1_t*+?N+MstGD*Ou@Y;|z_hK;kqnxqai^mX`6n^{rkXsz{K)x$ZxWAsMy+$|V_^VZlgHEn80zTIiVqCXuAB;_uIy`2)Ce z&ffMzA0K1Tihg3dXjlhi{3l$Hfl-six)2>bbf50eMP zC*b-C5&Stfs%A^Tnptq~aVFMhofYx^X1ux~?<527RGeC6qZQxrCzLG2a;LvMMbl|! z>-?zjx0HsXV_BM*c~QF$W@{OOk8?#cVNpH4#c@0+bvplgx|Tc>jBc`=2R>z)VA?Li z#L`l!Ak<1c{ldkI#g|M@f?}r(o~^{cOo7cVwqvvkI7m<+9#AJKiAUD_;;5_F?#G%! zdXuubrcHzDK@TS3qq6ycaa4Jss;c6ZJTji->mCrdh_eBP$3g5)r&+d-c_9drh_>@( zIp01Hf-OKY?q;)E=uNOb0f^NfH5%R?i(hQp0j^0ykl#M<)U*3l!oK_L-_PEm{>$%m zHthcpciX0&XXjFVU1TDp_mBA7dT)oqqk~qCp>h3aKWB~jLHiGr&L756=FvsbvOJE9&A%wZ5j^K!;^;%Bak}*%U!9Lh z__Fee$nv*d-==biO=yycrHbKg)tA!TO@w%JLoB-!Td~4n*AEvGV2=ds7eG#+Xt$#6 z8Q9uUpnO(}<1Cr1waZp)rmHLNv0w_F$FfEcsAvfwjshmBC~!Zt_H;QLv_x)bHD^iR z=qmviXi^Dlx|xGe%uBPTK;!8xj?#LnV+u&geHxl%zVcKIC<6_-BVoT0UZA@{E`wvE zIm*f>j*UmY07^ZmXqMR#bb5_y^0EUeIR~2-EsiU0u*# zXuFlTfS}-`Yn6Gs?N5Ax;r&s=v)F{|A%gpxcAK}y{F~)9Y@?VlcycsMyY?{O{EFV> zmyE4n(mc%WCc5G62XZ}=`WOupi`RH?553bx$^1xcUR{~?-9eOV)~J0kBT+Adjcfy1 z6DtPDn~mZ$g1hx$<}dGcR;8W~!14W3nD4)GG49YZ|J(N+28<(@aEn1v%$yx|)mx}; zAZnNJdJ6C)!DMytrg6)O>wJhQ?&y?QfifpD_Z#q@`j&V*WEU z#62_dJqd+P?|WfnW7Ci(##IWWoPNaFWP54u`vvJ? zLTWLculqi5nN5GvMVNqaTA&wX-X17ivdmx+cuJVjC{uHQ%=CWau9X%T6&w0h`_5+N zo0J#C*0@P$15?u?=644J8^*_!yqCB&5(=8TRa~Lhjy$R4hl+KD_fN{1 zZfxvu&+@uGZEUm#QDxxy0b1fU-6yDjB7>O37tlk0-htfNKa(kO{P~@;2mi-~!5Y@& z*!~|~^Y>V=0-m@U&dTfQjZnJ8a-$baBc6FFf-8Sry1q-L!vhvhk-yfR{9D~g`Y}2u zSz-IM{%kXZ_^RH>!$zEsg(C6Ee&T~BQ9b<^_gn1YEpp_&a(oWaV1{DhkKzHtaOX#2l+^U|b!Ow( z`<#2^zk(O89F6=}HWo`C;$&)#vO8CAfx_Y{k}_m^dr;>+xdoU7pN>3|X@g3lFSGph z?ERelt$vN7+e?m9%_k~?8%oq?@{djMvudF(m$7yHg!Dd_yn1d(zvY(N+yw%B9B&pe zj56e*+J|oow!-%F5ppH({~u>>0hMLig$qB3A}WZEbSp?H-5_C5iXbhGf^;g~N=iy7 z5=wVTBdrJsNH+*dNr^~FC~)@kI^T>VuJxbuziZYkV8pp&$F;8=_hoW?stsZ0C+!~T zM|F$_Wv>>IGxHpzH9SszfvK!=mD>%HcFam?-sWZ;q>-_)+3(+Lv1(T+!$5ErHa1m& zR}k$@uiCj=s@}Xwx;lu3)T_C{bEq)WYascV->g`Q|m*V4D&YcT%{=gP>Zr)82 z8%_+@XITNk=8bETYGywuwBy)HwR{|l?%n_MIn{?Tl#S&YKXf2PB%eC4Xz&1gQ^5SB zhJxTmq0fOc$O!^wV*RSivt_f)qH!fekEW7RSQj2qrU-kCZ%OG9%t<(??_h!A$<_-f0o=LBg(j4MAQSas& z`QsAeANr~V)><(h_qfNox^q#&}tTfb3KGhQ(c!M?jN0_r< zJNJ|@_*%gmFkkg=$_WG6s3Ok5gB|coNFCW{umq+iCP$nt=6f@30&ozHZ~=7USLG;m zyf~5&kG2!$X=kJJQ!UGc(?;&X%`MU(x9@`jgb67%q6*-->2K$L8hrDc5xutk@V)Uf z!++5`vSw;zQF)x_#)%{1u2ja>CPo%6vSnenxEqQ+5OHI&BlUYf%=pM{cf6IS>5uQl z9}+hQx(59=I;g&KlNv!tw;?W~D2|JWNM2i86Oh10NM+_17T|>wdzZK~Se|bIt%&Tb z^*|Fq*V+sODS{>nC_|81c4GBecHCLAGA-(0(XacId!~n9*UI=h7TD&*ASWnBMrV z@;_0DOTt2ecrcG@ja$DtdmQmsc7u{S*RK~B$EsaAOXuz?AXIf<8#b9dYvN2Mlcz`y zh`YYw!oVNj@S7d`57*xCUH5O2A>IpKxmvM5#XetbED`af&m3JCI42b|wbAscim>(Y zAU5J`&+yDc>Ad9Vg|0mMqSNrFNNZBbbruH6r8mNO+&XDY z)9`0>)Fdu)B_StAc4O2bK7>Y0G-JdL1|?O9ApU%fQdnzKvXnF;5i6RAp1{MtttfCA zzA=AX`loljzb{>RF~>qp#!GnY79##A9H-2_zqBc3i6>iuVg4*uT>=-GPZ4ng#+KaW=O8w;|sUI%ORQ=T0!P(uiFsM@e>WRegc z@iqcC2>CR>zW47h%#PT@(rd2&T)M48T1aTPE-vE5{^TdQM%Zw7Ke@)AX7k&+Kk{JY zgKyI(3ZbRUr!iAa9-&xw561E8wlGpvAVf&#yDb>V@QrlV(|*tg=CRA_uMi_J6MWsr zq(#0ERZWEB0U-&=VOA0%qGoHH-Ni!)sWXVQyi41-HJ&>RcgKnENLSN!Q(>?A`k-oT zHdM^7elhKYkS90xWHKTr7=iIcp07xn&jnTRUB1@^@~CA4G8aqFu*Wp^zY8nHi#c@i=WxHVGdzJcYkW@=;@> zB$crJ`PmWxBN#n2!5IEammZk&X}UhZ^3;LXk3=@!-_xjdchj1*eyL?MDvDe=1#3>> z_HbR3ZNG=*naD&uqYau!I)7Vm|DvNf$9`KhqGvp$K9U!X<4{$Pc{(b?!ef2r*N20Y&o!kQ!}t#PqXun@Nbjojtq7sDYFGC_3!xk5FWT=$ko z@uxw4MYQ-+3gm8c{@)$wtYdnTL@u=ldY~_goqxbI?Z5g&iOce_=FC&HKj@p@zYsPI(V@^SsncO7ZCG#r0vaWSZ`7D-7i z1B=A5Mk!z|u9MII!kmYk*Q7f|K=L3$iV<4$6@_b*$cd1KK$3;$E?>HQNOUr$reH?E zl6~c5p*B$dL7@le>ZK`Y&eonlaO8`BMiT-7e~2b@-og5cdjp~m5SLt*M`ic! zty_tpS{P4qtxOE)SF6Zk2|&y9Z~ zdTXM^|2sKeE=w-eo@RZHYW>dLoS|f@l0}6DcJ^kCbhG#>?a5<6q2&7v|BSq9?gx+hX{wp2J&4?6arh zH@z9I$M7&=yh?8k`7d>yY z)y?A4DnVFToo5cKRX@y_I$}8K`Mu>Dk6ERoCpYqDSek8+hwVNUR3K@nD_>`D`FQ*F9Wcjz`ehX$`fM@FJBi; z8}o<#gvlKJ?|wL5=Vlu#1o~ysAiib9p}I?yRu8Z0b~z`AAPEjm59z+U3e(YDvOxowYk@kwo00tCUMg8V7I2DONDv7*v{)b-~1tFp2|uqzJhp`oFQ`K)YP zaPJGRkFhHQitjBO-}nwqrO9LL2%ZFpQa#ONGSypvey>jEVl0iosVn#PdrHPMqu;b3 zNRKy{+D*BK7?2b#OQ~{MW?CEfNB6sbwb74YiwBXNy(5+B`PN$XsbODk#Nt&RiV28! zaL-2LZ*NZV5T&UDSVs_4m4KG;(mD6Pn`l8TfTZl2s zQ@C{LlJa*@1%*Wd#Fq=iX*_VbOAbX>pC!&=&{aEqSIoYqw(*C8zrTD*H#kDXy3!P!cmI&VaG&^jT-v{_4IgqVFnwgpTKpX%We1~s^W*m4W^q{T9(F~53KEDIj0xVNt_v#^ls7vcDXdA-rz z#q|J|C+Sd)TP0LoJSHOr7_S>K0Yj`?$_tXo=I4(>qKF{AMaQV-E(U1!6l z@e-=5gXG4TFfFb*=?|zNRB%7cvPL5b?JhLzp-t<5!k^922cctU{(4UigJ6!Npmq(D z6b>R?M)uec99z)oR&yTX=|Mt>*4CjUW^0n_1jlPKdRB^voGtYuJ2ho+IsT~=6B7y3 zDJk^9WZVpLT>v6kfVJ4(-cA^xJT8SoMpAfX4uy-IW~d>RlauQ-1L7KBSznysJ!K7z zECSI@@$VZ;OC?r-($*cZ^aO6X5|V0=(%=)I;hivc8z#I9(@WTIP=7q#M;IeG179la zRZ5&ul=enhC0h-i69K!w;6O8}#CrF+ap-Mj1d)LR%0ma-%g=#_fAIOrogrE0nqmbD zotCdy%a8Q9W`|!ZV<;K0PvW7db$XI68+|6Zi!|LBaHoorQbg6*M^0l1tl+EJ-pqLY`Vz!l#yi{V!=)Cn7pUdmmLeS8sP5wjg-ZIOgh=rH zpRQZ_MDTvst>c&lk;Jc`I^2o%@9R(vgaW&Z1$jy8Iv#t+i1F93HW1HFfm+{u+O{lt zBEiULy#NQMI{%2rdfmWR5@G;{#YFoU`i?*_cfVWK)D~eslpDB^S0gSJpq|IUcmEKP`~>Q6Y+7A9R36KP4e zNR1>hj~XgxLH!|zFI8-F1_eWkiHzm#s>@jIj@Rylb&M7<=a4Z-uFF>7}29! zsD~|t!4|ss|K7qw9Jrw0)?0s!JNmaDN&363vpqB3Xasmms?0cCHLSKYzp*{3$TW+G zbiHvZ*@K!7HqysAsG(~%_jeGeBV%kf?`%8OWBV$@fDEsy35|JX>?>6i75~EUY8cCH z`|Atj@MHTgXX)E?$`X= z9T5I{wP2!In}V7`75Ca&-32-6B1u7Vgmg$C|G3tz7?t?wn7n1emj{vzT>~W0Z!Wbf z03sw6bo`p|-qJyDNRPv~C6d`D2p8xx4EHwz1hNDW$R#E{3{KV08rrTiL=dxSkv91w zdQ)Jm{6!n3lKn1_BK~>1&(+YJ1U&`nVE+5Tqxpkp929;(+YbQ`ssiO1#>=zT5tU26 zrwP+E?r>cZguKRV792LHvTKXzw!P>cUM`EZWAq693Q`wo!0@nQ!4Nn_Onhni zUf}H6vke%AXXcmLAjT_>7lzAi^gGRp4BJpOjyrR}Gb|8JTL?~D6!nVEIN`eJqUnaj z_tR*bdXl97-Ax_t*Nb<|sxHH_w?8`WwR@>XT$}`l9)g2NrY&a@t7o1Bg0}7!_=-Fi?v1fosW_G8Fv@kD0Yf#E3%#FG zmAJRp?i2C3pGgWEq3bbdPv_)hYerK49#hxZmy{$~p8x_C@sm&zz&wzAtE_2GG}gXPu3=yy|LE z%oz>#Hd95*B$>y#W26!|h*3?|m3wbybgwjm!l@cQII76%oSGv8B^!=@2l1Fs^_;IT z*9uT%jf1BW>b}A>OsLfwOBe_U2$%q70tx@8g73|T4-h<|XU=ML=Z?w5n}?{FoH#m} zEWSiaLv;ZngSxtPeoSPrE9SC6^}*2Ti^q@bTdgC7R3io4pJk1f#yKvL-nu-*%1S68 zPQa=i$<{KL-}yQ*HL~ZTrld(?Dq)MXC&$&ddfoyq1Wh4oagJ{a&RVsS*NVlxRj4wM z`Zh@)56qjItC+j(Lil+R@(w#~{^9i!o!^aZI!;m7Z>$KCuxFkP?H;qVh~A^GGQ%2u z@6%gD0?T~(NFn(U6imP`fh6S zva(pvYc!ALxb@&7Krvc)?ZeZ?)wSv(kYUEks;Z`YyE|ATg+)boxR##@BY2W=G=~n@ zxc8ExC)7M==6-se3+en~F#r356&#z*o?OwgF)@URl?;QtFt>Qt8mSL@c&tbJ)FS-!)x;|V`B+>4H|lAlr4`rR z-Hiu#hBS{`9}EJweHzlyz}8G13`$N);;|U!@q*yypx0`$I7NFXxgS6$0YSHB_7pf@ zp?S0F--iL>i2dP|J*I@TO>KLW36YS2kKateKF-?6mUAO7~j=I6u<_Fwgder;(?>(0{mnYTDNfA9JsI=e+2@U0E z`U??o0y1Q{#?7U!6e1F)^XK0s_#;U5t1X4NOaU>|3t(##mbPPbrD4^!y#vns+=RO_ z`)?8wm&Qk9$t48TRwSR=`tGJNS2T~{tA2tmq;AgY?t|+Lrvhuq8VtBhOclLr_&6^X z`Cjp_Jt$|yAOl~`)-o~}n<^e`f=r9`aAtOP58xo|((6w-UL^pQHJ-ENd zp(W%qAEXU!nxxS@Pu%uQc%Tklc>}CmMnFu+5j6&G0|P%UfqW#~bIaUE?|n4l6Z)^c zMG$YDI}w_x_?F2iVx1GOhKgZlkD$27w~!F+B#97oKYla}hV@aOOdcuxOipaUD?~{< zcEj)h45BHVS&HkG5Od#fvj{00jH zLnC_Rj36!oqc-lgUr$4ry`9Fs<d6bJ?4k zPoGo;QQ1CCjtIEn2N$z2KWG@kiAYpX9pz%vi3$x;i0m`46{wR5ER3<;dP|D_^m4Mm z7?2x1nh?U(IleEqDZIH^a}?2|gs#-Iv@}l(IP<4^$VYm>K$k&eFp7zf8ZMSNnua;w z)nqRc@Nk_jZfzJjuV}LQ-I=ZF|KYq<1Dmq7`&cD`+w&T3pC>MHtg5bgKlR4n-5&>$ z(~US~A?fES_f(eQ`n7=00R{$R`@7yZeF@UfM5GKgTZB%U$eiSgcBIEfmktrb(PPK3 zOQH0D47Iry5YzctEH5u()8Ha;^pL|SF&n@!r})W0;RoO%C$6H4_gqa*$3X?%VLKcn z=xaIoU|RnBA+35R`CHou1Tn)i*3HxFWDvXInX=`}JISf#v-2~*YCI!>oS=_Ld3dB+ zSNtG0<7?CYiE++~dk?fHC+mZcAI2Tt+_WAQ?8FCj;wwqtaJuQGJ!-3S}EIVuQD??|(kx#*0}doS>=#XFq3Ic16DN*U@HRc$X72$U)Rs@=Zw@vEF_ggs&t^ zdQAV>QQs>XcB8WbROZHg^Oy03ZY=Q$(I+U;>Ax;!61h6pb>0??Zu^CoFJEF!DkMub z8!qf51%pb3*N@|pHZ?U-K<+LsB`z+`bNSKf%RKg?|yolc1I zmGGNu)t^+kIZ%2+_fAC9>?1@*@oI`hV0SVv3!f}QL^IfX1i^`Gn(P9$_`tvBX69dW z^U;7()b%E0_OM1ib;Nf#gdx^AAAd06oNEt#K8ymTk85XZAAy0y)zMwp=aK}}_q(LT zqRKQ0(PS^gQ^{Mt`*{p;+x3T^`=%R5>PxD6>7%0P*bv*Z@6Ef*H?5QF#c4nZf`CTpR#?0>xmbBO(lzJk>hNJjeyQd7_BX{xF|K|O0H3Za4~ zCnxbCE3(bT$jGQiNk*1A*ON|xCQTA-+zGvo%t*--prmBS-^Tu6{vUK8F6R$AFr+eS zZi|apUxhEe`?m50s)N=JD>iVFAQfC^yBJ~{%xy^pjxa>XMEqsZaus*r%RKS4GS8fJ zVAJdiYhUSr7fixu72&@XzChRSiKf>%;v<>a&{6@fYlX{ns6=$?5Kg27kN5b zR`LBq{>aBGbhmS^hNPI!+3IseN8Twjadj;ZSkMHPQ>I}(SJKa?=Zc~E(?TV>$0tuU zzIK5eZAuq2N_w*@XG=ra8>6!cdf>l&>%u^aku(JO=nCR>4`;Sp2 z%@Gg_xH0}?HX4O)vs!=OMa`e?f@Hr*`Z%S+<1}UK6bu1{oQzzQ+T-igsgmn#4Wj2n zwG>n%$Vbl$9z>e1XkI6$p|P|jC8dRxR#@6THPxoJ$TjURrE4?&Xm?V?AKLIlm@jer zUSVQ;f7@07F2zd+bN*V}A#2Cnyj}gBz@Zc}nUmX9Nty9U z2RxNJ)>3fQ-!UZD*CBz&%8YM07q?TBr@F~S@u~3wd>Wa3!SOc8zreKr5vm?DXa&HD zl$aQD6=mgv{~-SpZ-2qjx`f2US-?Q+X#%f@s=pj>9E}8rfY?#gLnr_39vIWv_j_oq zw>ZGC{rVBWpA!737bdWecA3EW_31Q%hn*lVTid4@5P`m@^lhYn-LfU+c+_Ct{pMgS z1H+(gd2=qSO53pd=$WV}wH9ub@}U&whkR|6e z0D?*8y#8qg3GiAR$j3-kn1Y?#np|#uEuW2q# zp!9-yoR|BLr$}+x4qNoB{ORJvPF1V zhgC+CRv5~<5>LPJl~m^ye3ka_2w#JD!7R9ox@J>zb7e@n)Rh9AimCxZ^$XY*U&6#? zFrbmwZQB-A{y0kkG3QURapVc|KV60%@4g5jj*^@WTikd`@O!_tpoowx z8RspQJgY^&4CG_)qm!r7if1`M8)!*R8TP-Alm?I98)jH$uTo~Lc0DkTEx#)wfRbAL}72O*&Eiql2?ZSb&0C8Hs`jZ?`%HRtH7u8*ZgBid) zL9{I5Qu_L}Kob^%D+0j79fYce76Jx2wx8G|xTe4Lw*%L+PDp}N`StUW@YBks@lCh{ zMXdprsHSx5z|4Ut?MLLz%cON=7O!rQ_a{4qpUKg9YT5VFL?O$3g{fmsv8E>#a|?-w zFVp3cQ_$quERSD(Sp3#te#eCo{`$4^s9IbAVCUnrZZR&2AeY(}*0Z3JOD}fRlAnPT zne_>#;BQ&9+@l^g=QA6tazb}1pakO73*zK`03wKX+rKgFrJ4Paz21fW<`zKlg4W7r zP{;v9j(}Qs#D=%ka$)@r_w9Nfo|dc?Nn-p1)}J8MVWDGqH_Kri0785y zlJdfeu}AJTC%LGA1&ysLZrq!)ir@$v8~w~=waO=6Ltp~;V?AtUVWHyg?*6YcOooc| zE!X?H@81Bh1))3Gv}kVzxf86BG`VM4gM&9wToMO5FSH5pYUe&Eicw+2PY&S!VP~e= zl70}sKYlP=!>h#?EvxZ}JoBuN-OdRj1kZ9PGYmE(>36^9idMM%j(Ok-A;%{Vy`MV|Gs4ja(>`)Jaqzh8B45JGX#{$GSn z1Lmd#aKU}ycDFwh?(6J4@q$%YcmxxG*Kc*by1$w{YX$yYAmW)UhJ`tyv0r;Um;V43 zYtCg_EjlqG;+=Az%7sKlwPg4LLy8?Q0RqCalsQQZPJ&+uFD~shE5n^3i`|A3^IFu8 zd&{=>3TSm_=?CADlb!msM?zuAi7{Q=!%(@1-AE@D0u{cI7_1nE1{=VDi?Va!A=s~G zR+&+_s|hVfzQ?IdZ(KnTTic1PuIYI9ytwygqW$n9m`$F(tLX-2M(YHx>&JiLgiU&Pr8H=#1)^?CFiTdx842m5S;YS zki%R-_*(YN6v;%c<8&6ikY${8@)}g z$UdF8X-xopf9F@{>ftgg-G3$U9%LuKDD*IosS7T$O9#vV%;&X zLo3wpBIw)RifZc5Jcp~(nShAp7VZTy)(I2LyB8Wzl^*V13NP70bH-Cu`wV^Af+*Y?H=UM_2A(( zeCTm~0M$jOIOJqydx6kGDG1N!P8X=D1gvTGMmEFhc{n+fV5eZIS|~<#sVrA!W!p8- zZ40B@mXYJb7|K~1My2106=d?}{vgdw=@JS@Ni6WA=(cUu1DjLs^(3vy*|_;9AMku- zbW5Z7JOlCa>yNj1V`RUVo;qHqP_*i5;i!NYm_{Va#)an(GnOenx$}>NL_~*zA|fK-1);vl=^Vp|?xgxiVk+Vn z&sLM`K3;S`?a6(S>1PzRYg4!nEpqtoM<$&vC2XOT;H05i%Oo?cAzLF#%4YnDsP>ov z*IZSxakc6Viix+-N_8`mn)Otm+pR}KWmfT`P+xz1go1(s^6bfQb$fh8&GeORXDZv0 zx}Lp*7X99-p+=XQhQ}oJ7V+R4d2eNGCaQr`eEexJ9pkG7~jDNtS%;|L5U)l z;NW0L<7@t8buavwB9=LfB-8>ka5#vYi%ZT_v_15I*J0G(9%PLDNqWUc?1Q78oPRI` zV_?I1QO+orDg$E(__e61NYYMns#0Y*_;IiRf<$eXviVauN31xx^e1fE!_!NvD=R!! zV^s+=+|jZDg{7?CQGby=Kmbj9(oVsX=t27W`j4MHF+`(f+q}fYi`1cv|GT7+j)@OqjRj*ffka3=B@AEeI8ht`oYBU0!SEZp<{sR(7ZscTWcCC&8{&Y;&o5><0L~L! zgz`DZpilxeXIsp$ULC&8c% zvXRkSr%`X9 z$I<*7nLcX$3z=fM=~{FA1C^1XAqy*O8_J~P>Y&kp03BU@f;7;JsO^)x^!TTY*WT#c z2ai8K-EZ})m4>FC3_yiL4b?k+*fZtN&eR@ULm}?j&W<_ zx$e}9OYu=r10Jtn!3ZNW1_wiSTm3vS#(n*2$|{tUl~Vv;{X;j1f34EVha~rb(x2G^ zPj@LWUOh4ye;Y57to>OR%0Fa`99k$$kfR88;)N^tQ6y;+%q^xzo1w-zk||qe=6Em{ zy4cC(+4TWkwR`vObw)L}wnlW!EHq`nl09ooOiUDR-MaO!`E~i#`IMlvtKHo z>F8rfP$83u#!1zyVH}R0FN9LCTmB>}S#DAB@$6{cC$`NH;)!-N z@AFfRp$Zcv*w~a+R7{}h7{k=EM>HE?l<%h?KYElJC%0c`8z2=K*dhq}I<>RS@ z_jAcBj>5cMX=_YoZ$Dvs-sd?#oR#^I53)u-zI`4_wp-|5pDT27aykw51;EHU{mY3@ z-h%|(*X1o&ln@-5n4AoTs(}jYiH)X{Cr?5Ln=dDbQ0Fzsf@4jhXL|;V;}N<}mSfQ7 z1a^&sLtyN8h0}PqpOyUx5I}|3DF`^Ic3i($5zx5P(XHUdCQfpqr;HOW(+w#JO#E;_ zIvif-TZ1`k>su@6s)K&6nmRPS{(JBL=s%aXwl?U%!W#JuOe_=}Gm=wKl)z3sEsBgf zR2?15IkB*i&SSuQqKhBl;^u~})p?!)K%L3bi^*oy)g8IBkJaCfAKFI|oTzGHD65h@ z3~k}K`SMWsaD2!WfMN_U{TYK*tUB?hg7lZ*d&n9|heNljj>nE&NpbO-QwY@eMO0jX zGe2Ppt1z{+JU;m3@#8xfrvjR4J4j4I!v7U`WD-7$VfwPl9m;$tMyhgJ#e@18&yQ5V z@WUkbe^(DBuXUMB|E%6_|LUXuwEFrsYBU+geKjMr@>ow2p$V^#tsEq4fc+y*lR%wH znd{bFDk0n1lurQcS(un`0al@8#s~_51Vlv8rbjfvBffz$O9Fg+s>Fc};Mr*(L5^a3 zeF0LU&o4x83;=&434MSl_jC@WZ1fP)2M}wfhwr|{39<4l8LO=(=_^FkoHIPP*ku0L%Tp?AXl@p<$5|6;wNM_du8Qr zUw=Q&KdfUXH<|_qf~64}B^iNuZfR~lGWaAcEOBkk7LDE$99%`?p_md9iUtNKu|x;# z-ACn4-M?jdK6Ia5UL)9VmuVeQd~v^zrb#=ZIpvzvo41CXKCYGs?xENQ__pz_!Gyq} z$(N&+)!&BA(9)ryI6FJLtc(oYA#|jvxb9E(8tm5p*lQyc7^9J&EG=UFU}EC-v_DnQGZE|u$KblrPy6vg;LGp!L+DTT!_iH&uXFPhf}?yrP)aAB zeQLE43Jtnm%s%Qkh9p{dy)bU@RmU9o&WzaJ+=~ozrm5*^A6Ge8b3)nOd-t##Pcg{O zMm10ZRYmkm#Q|=y{o=5v4RmS!&zJly7i?^DKaK@aWOdxiC?Y=b531`Fo zxlC(QZ5^lCts@9_%cPm`?l(UmZG#^kTZnpzm5|@~#>eC;8k{0BD5p^Z8CLgbVRrU> z-A^cI`~b!77^nO{QMUd?lqr|cc?6i~`}1e>=(ggU*hriZDjJOtd0|*lY&{N?I?(Hl z|64j1I`&D2_mh1g2S*H0_7NlL&Sh%rxfXlmqdFyyu6Lc>?%FNaHl2gOKt8;Kaf?2h zK3Z04FG_~Eh|rRc`TgzrH4hLTrfhXi2oKcT0*Yg9DkIObESE{49V;#jV$H&l^QYqn zz}|PCiAkaJULS44P)&MVFii3$#@a~nRs#Uq#HeIM{hT=vENy%>`(8h=$ zqZq&?B_Uz*#{c9`y#~}86BBcwbO;j_^l~EYShQaNFcj4~JzutfCZ1|NYSUz+yR8OL zkYUS+0dej>9HZL|F(A%JF1@I}6modjXRti&!A<-Ou+)nTnJ-*;k0Q_6>iK!ozJbG^ zj{lA1E}}-xOXi3@Gm_Heai1)9b&tB;G5jd3I^3PnukKfq zuD?7UwPL2d<1B)7J}a;pj|n&C7Q%yO;#Y@hGQgaF$Y*3{msd)Yo!eMI_b%59bThb8 zPYY0C4h)@ysk;G30|ge`Knx0)_N2j>5R3KDHp3!Fhg<1^`&t`&_~BH;n>S3HoL%6m z@c1CQA;rV_8QpM5dJMGIIreHRIsBK577*FTx_H(I^b_V+b>*%k@2T-R^*Wio?17;I zD!eYrc9&MAic(I85#t9Jni{9501CL8=rm@Mniu%#A>z5K@}Z+i7wl{+_wBW^>f<5n z2;NBw!+s(%fI6F>kx8_CC6>>E{Lw;7M@Ll0%wz$02T!c1s3^vGc%F$dyDtlhCMnS3 z3lXU4#L|akr!*yE9rTOaRmb>dpy)rKU9)IBE#P;g4yH&Uk`x&~E```2dO^E+QLpNf zzQk0R{VVS9Roz^x>F*)v!DOHhfvi2=TbA`1&op3wLPrKthAMnc#oDal!?81|66(BX z7-XT)f{?_LlSgii3SSJXjuhMGM{r?`ota_2`&a1HKRtBZc{!XLU9!p z6?l|s$WvQaUq2b9Qe;4~ieS(Vwf^BSkb;7Pn%gkw_3PJYDVyg<=)C`@`Yu+x(mpdU zkL{lvTqi%kItd9007hI~K>+I#(A-Hfo&mz%cC7S zNnQ84g+@2j)T$Vy)V=8EUguvQkl1*KXVXiHNBwmD!QKo7wUA~}J~e;BfY=TED5*Ez zb8*E?ZW{dpv;&8ca9c}_NXa+efYlO!B(L7xUI6fm?OYA`m|WC#(^Ch{c(j|t_%jeB zf`6DwT1PZ&0;Vz;3g77=r!x&@f#wu~cT>81fVcvXSp!S~yp4@N_#`^o1oG%Oybz(1 z-lD}&&WJvqay8y1uZo4vgfBqj{_?SA*-t%(Ppxb!-<(mrn0xK)dTy49pa}USk@)|`V;K%}VSdHd)zI3B$h!lzb_(foQ_Mp_)Ry+sbS`LC~A2#OZvktb8o zhz@-8m$P^^79Ts--m^fEyBtb0;_8zAy^k`hXHuuT<|w4Ysz zcT>%pn9%6!Eq-fR#w;SD@%S5)#G9c@?rw~eze^0|iv@dCzS)mo=(4g+m6MYtRAVIy zniZ}#2dT{u9#qfyq|-JqYp&qr9-I|m)~7t8Do=d!RNkBM0&-3#|1-DizfT&j1+M%c zr((0sD$P>J^Huec@@sxhp4(e+gq({jPyQa?z)M@}j*}{nd$V4Sodrz&cF_84uz$~s zhcW@5C-r;Kc}Vw#R6gg26(uRKuCw__+$18iQX1Yoh))$uTa@bHaV?}iy>_@hm}N{g zvm%S(CAlV11SzwNMlqLBS_`@5k*#%ngX(t=j~=6n;h8(c%U{Xs?)t!fvEp*S4)q5) z2S!Z?s+J9$bm93td3Jc;L6ShMN;Nz(>|}aUQa%~w$xFF8IXe1HyAr{{mceBDKoOO- z8tQMI{Y5&FxUsj7P~Y+IhZIeVK+e!L{TkGmpcy)*wWEAb%o>VD=w-eY8$Vlp+>9gD z3z-u&URI0X)$zVr2>_RRoh0{_nlL#Y|o0vq-HP;y< zai-t}nfOd z&ePByaSqBYA1y#3UtP2j^sHb_hS5nuPi+hMD+KpBrk(=BAs{kPgN`)`bngGE3EkA7 zXMqFS5jhML9Q9~(bhMNDvI(TaKYzn@+o&wU51H+~?0))E>Zcpe_^O{R(ZugOmI(+d zfjT~%SO~`6mtZT?n2 z{4(k0mY|<1L-Y%-daAy|_`HgMN0i_0kGn}E@^%h}&t?~1#K zvUTm^Pn%|#e7XDKZMk(@jdNUkwcXt)k5=yc0}DMp4pr&c!p9Gr?~NBOJbpy&(!DxB zZP~l0U3;Z%J$KgvE`4Wql)|OzGj@@Vz}!N-T_5S_=GT`ZU&=O$6mwnevyXqynP1yD z$$2Zrea={WrA<8`Xkm20-Yih_&dWZQQMW6HHA~%dLE`@@L{jk&*m-uyb zDNpfUk!JYXp#DsYDKCK|o>y1&4F@+Cu%qckJ>$YL#l@sACm zkeZ50>YdgnWaO9K-D?crJ-ua&M!W*tz;2_BCjXT1ow!gol3P(xC!vceULIuvP~!8cxF^;N=XETRA73?O6eT6!Gr!zi*AoZvO#F1CTJ=I@ zo7i)=^_von!@zrZUC&rO7Vl&jTeF@^WNNWk!PYqW&2%X}wMExKTFu?#`KJ#Y<0<(A z3(NDt@%do`i^6j+w_}rYbntLvG>W1=ci?%4|)9IW`c&QnQhQ+Q#*8qQ?D;g zEo{d>Pi-cNi;h3sUsEV`*q-uB+17{cJ0UR(n%^yUpI>?7?eN{jAzR97k4uW;$&_8C z?mf5c{_5x1quJNirkhBZ?@5}54ruF+rqsGmUREI#z!@#DZV0$tz4Of-9@&UyuJ}^1 zj3Z@0IiGh}Y(CGX0vb@0w_Xu^n$Jux&B$v{ZKBppuly2|1wE9>_5TU6w}bb^ed)DX zP!*!~vnnJnFShyi*G%J;jHFD3_{aP#qVIE>kMLcWD`sYXK=|gy&FbxI7nd|YT&FYq zXdCf}I=hMQ1JsR)dv2@dE1o(0Wm~erXCbFo+D)g6$6D>^#~>S3(-f}B3TU;)^G3l7 zN-S9LT8KL>=b%gM8+S;f3wgY~MSLw_L+P++UgooXQh5TGhjRraGO^^ zgI?x(J8|`ov1C($P6HLGs&Zd{KQ@OtV-9c@( zcF*~xqSQ#`v&xd8+(I|r>Kc6P@mGh%R`-6?VXxu`y%Ffuj<0|y9LLYw8`o{otOp&x zu6+4&_U?wqz??&EW!>zHU#+31`{gJ9MWG40y=BfNP`6x}6|2%UHMezubPdXyMDvZi zXNazolbiIVT#02kJ4krSkWZ+X`;yza>kL?R(vp;Z9WsH65)qo;%`{!c$dCs-d~8}X z7%r4nMx)@i4~)4BK7JW(j%}I@Yq-rRLD!g`8-raL`_}c(}RVOGGPexw{N9WQ&NDs^Hge^i_zU2{4@f!qok3B zx*~Fb{0Ei>PR;*#tzDS~Z}jG`g<9|h!c5j{cB^ftu)kH#mNGkk@jA1!Q}Z}W8_L4VnALLZ5ZZ)hssi}!79xrDu3$~Ec%alIl z(EheK6yu`(;2? z@h}dChqrZqFquvsBilF9Q?=`C(0?#7!+Tr6!Gc zk{=7?EHtIDGRWS`qfQ@E3so83Z0|U%yIk$0@$HSX#mz$b@Tv99wZ@JCrhA52NvGQ5 zmOeyX*nX*X-BL!eCBspLSm-jJgOYaOn3L->`exH2CVS}=)SjD$&{Vm(<$uf1Htl6+ z_h=KN7UH`|m(Jkua?khd_u@sbOM3$D#>JN37k#Lx`AOdMYR5)RFSNybu!W9fr)fm( zu7jR@JEzuaex#<9`wX8}Jp;YT;2U}(i`*N4bgIkk5?6P>AJQnxlf_`vg?#>Rj@*wj zDn)m0IN*~mRDO7t`(f8U;sSO2bMR`^j(Rn|$AkHV&i^Xe3A+1O2kUL)Wyg|^@$S4B zVxobPmVwX#NZuPowy))OrK`s0A zhX0ubt)o^pX41F4B}0Pxk|gK|siX_3m=Hcy&C-^e0 z37&yr%6p8W<$VsfJ~2%ADcXF7f)!_syEC%3ezG4UzBsDsy?8Cw^d3rNQ0rIEj#o zW?x1>L)q)(*Mv~c4&sp-ROm27&^mW3@(1>Pg5_iZrUUNgA< z{Qb^{cjjDk^Y3o%xWy|;#Z+2dc;7vkmBdPPSCuELt1IVg21i5s6_d5NL~vQ_g5M$( zR1~wlT(YKej3bB46B|FHRuky%(PAGTvU+q)n%PMJLbrjni$v9bl&MV|lKW?R>ekGnc*8GWo4# zab;c%rmeG!RJ@QM*!e^kpItJ*x-&Sqt zdZH}B!%?H!d2+^V3XI=Z0;<@A%I7arnl7xJrkk|w5`mGRM&CV-PFtFIXz8rv4#l`P z&$4aAJ2jB@sqoJ2e6Uz+D&48S#9E=R;pf&yTtqy|sWRhk7mzej*)b0B)OTEkLY>(Dk4{hb+k82$VPR(*%mkrxliYc(94hui(Mz{Q7WDovN#>PSIx+um6s(G zZoB)eC$jd-Bd~jZ=DY3YI;FZg4lXK-ab}HnFY_z=l8HJ>KIoX0t2_J{Ka(0FGi$r2 zt`?#K@zmIEpOB0q$TDBe<9w$9yB#h3K)jxpWpe#5pm3be#y?#0&35Q2U~Hww=3?)& zM@U#8V(Hpj9d(eW_)7lvKHnp|0gnEHiUpO!6p<&PtosOGaG75#agy^LgbJUAI=hvP z0%D7!z1p26{B~h(lRwn|#e7<9<3d3b zF^h;u+Q`d*fH-pAdBHu$zO2|{<1T57A;rY+Uo=xg#n=|D$%=+j^4;yi-FFsNc?_hL zZ*X3N3EMq^>|c5(L#;Z;s{5XOiHIc3wDNFCbKk6Y3hxQC#Ftjq}@_C_VC=BGsQLkmrJ*VQ$CtVbVRnAp|Y z`y|ThG<9!pvt}S)p=I`?zRsRx-jfRz{bIXc76yPq(54ZoPVH$d&KE0Y_-Nx%rC*lG z7++f`6<<{jHoEb6ZJ?ZgXTyi5nN9;6mbZog5NqttunLubmrk!GfdAR2X?@$S0V9?) zzqFQcA(36)ULLa{tA4W?DT!Ulwx-s&u%v8{IMkR{2Z;1mIOiP7-qw4DiZSTN;5g}> z(x*H&pmXSZQ2y-gYSOWOzaYxITsP>)^kw3{(N->6-0g8pQlGPV9%!K zghh2jWGJb#=b}QoUzW+L8fa|S32Ft?orSKCvu9T8##LCjCVEwnGjxYlF1ALzN#k zIye`yI4hRN?|B6*Y;I`sU$8AT)E#wFT51o80RLLOvnjN=vnKAhp+KCieh=QhX1;R0I?Rq(*vA=mA0!6$PoGhaRZ`LWdAYNOIrIIrp6Jth+92 zxt6jl5c7ZEy`Sgz>}_FY_5gM!ArB-G{=HPVPMwJh@DNH+Jo}>XuP=`N>jbV{7mnll z1Rg}yU;eoQb{2s_b7B>~o3m}gi2zhDwy+$`J9qOu zc=ga*-9o4er+H>jh)BAMwFe!@s%1VOh*J3lw>@{3Ex_H4wKS}b3-DO${2AKt<+2RSqxoe3*h&FnL})r*TzcNRloWhC&H ztLq5`JsV4pYOmc+i>MLCq}wo1QG2p3lw5pNl3f7ud8ZT46gwv-d>D;sT3xNT*StLH za1Iy+OQq?j`JD&BgZkl}s8GV5L~+jN>?-&msPU3Zvn{9YIad|xO8Z9SoQ#fI4SP04 zV^}oh>*(6|IGd?F;NNT z@KwKytM-9oVN^8T>5}9nonn`hAGFThlQj2O?I=mG6B##`aM#W4bW5Bz&s7TR?hy@i zG*y}lRl3|!%B`5bo-Z@F6W{x>!ShYb(c__!&TyQeVe;%=>nSP8rEc}Y;KsGeu+;6N z`f2`q9=>7fkc1l6^8=1MnImsZ7ab=#E%~9N1iVp#VsK=do72_XVrji{tO*+gxjYMc6Qk_tb}1v@3W#T@IGAg4`~K9=LG z3j1htFlvXuF0{|x&1wOk zOk(5QLhW!>-(N_3-YR=(Y-Co0qa)H8X>K^?!?6%Jkn|ys??0`>V>o%!8k20A>*%i4 z*wu@!HIB!j={fsB(K^ZqYh@WwI5t^aR;qru}dSOD+4ie2<#!A zD3L2q6o+rpGt)vQcID{t`mcKC=v^n2_kXSQwobA_`E-aVr{;}8*CsZW9x5r;*8rJL zN z1(8}OORf3U2JiNl4Y^@_u1*;U^XG#9@;8P&czm6<;$Ur21SGbEXP~5QF`;$E@*&8x znCF19UH9IGl(p0l_;b%<=Rg7bL+X;2szI|9@4e~u9vlG*UQ7?~s3jhi5+JX=8h$hq zU569mt_iTB{9S)!!WsAAxq7iQ?~`d8Al#$ULL~I1Jt- zQLrj8x1=JF z&=)zmrfldr{lkZOd`B&^tEzq*Bw-bS!Qoe*y_NRxUq;DmWfFcXKc7}o8hA;euOu(; zcVlE$j%R-u0upToN`3DkVY7Sh&6AR1-@L5>AgSGPfw-QYY!;+Byj8@!#>e)l_~K$A zW|6}p96rX&+eLryA2ri2pYQk4+t2;yXTWsxt{K`T(qi9yv+-r$N4!6RKof=ASa%=Y zHz9RN3VEf`di(7yD=Hm8N&VHS3sQwMFF0tT|FXDnRV}z(hDn-wG?QIGrWr?1G0ocx3WpcL;xEQ<)g)e|q0oq6rA_AIk5MoKI zM*w?FC}sYVmV|-8#Ii zARNlx_nU<&aqiy(jVXA=T~k_bpj!(aOATreH&a(1FAX$gf<_UyULJ2z-;mP*kNYsN zv_>!Q`8rmxP_@>%7>v)9ezVQE>HVzH&pq_{B&+MUmHi?cj7I2tg)9HjUwCusFNrMZSJ6(4_YC!SZ)ldT)$WT~(3n8NfKx5@q<1L;pR%|d(jH~J-{-LGCq z>$Jf&60BOqd{a1~F;%pmjQ~QL_k6lcg=h?6659&adeW#?m0cAjGOzgK+z1>u;E0J>91_ zCB}F$#2{l*SI5=*RB^{^m#vN}3obu+)Bu#}+JHF&kl_Hl0hQGG^j%6yAsC1GqlY`d z>rq$-7yWK+X}K)BGK)Wc5DY8$4^H8FBA8r{Z4PPs?_n69+rYdSa{$kMuKO2D{!+Y@ zpAt-!(%!Q;!UNMFXJrT%Ln8wVL?f=)^T*hxrV>kbJ>vR_k#+y@YQlo!w@8Bx+oh0l z5`3r27e?IZVP<(#JE!V^9P{9$#N-UHKS1{kzj9n(W=*RwnEx7Pa*C-!pbF z?n}|y8KjwA=>qW5@lr@=is!dNJ~IVXLxbVm zgX{PI%c>U%JSj%ljFqFP1*bR8czF#TK@67}Ln&>8!t`=!0xMOGIu8~(1tEp31B>K-22cK%rEkowORC6aYuk}mQ;pMt zzMpa2nNWSBe4Sid+GL$%vBeKbVZ*BVBfWp)p5VHKexv_ zB5@|MdRsg4^?_+G-@c_<`b>EG22;EAw4gCW@qhaZh?Gt?=)~hb)R6d$>=ey|z1eg&Bch1Fur4i^S5!yYkmQ;vOjm>vM(5=9jbxhlTR)$$k~Z z`t4jXc#Z&MJfC4AO00IvU*bj7|KpyiRMxszS|UhWLr}ch5+cfN1HpP}N0POadkYmj z2OdL~E?~`ISdjXXcUmn2JyuJbU^xyo+%a3S!MZtV-F8QWCX`J{acs!U#b1H*5rD)MoB+KV%f$17t zb~a7v2yF|A9xSnx_;rmKC-9f^V3a`I0EVDxC;-VAS&48$o5~ZK^nez0W2{bOHGq(p zcgHcX%LdM21}Zx1m|+56E~&ptq_s&M*{(*yQKs6mNrj26-zwMs6b<)rj<2QY>YBs~ z(lQWDUW3M>p#pE3QAcxqSe9e_TB{C;f4+uMzYQ8HO!E46UDhN@eZdgN%B z0Wex=Bs6Ry?z5ywC8E2B~aT@n8?&q+v7}?4~yT| z%GOo75=RBMxZU{c<_8ZIQY~y}2jb>**^<5L$pZQb`2A8{k$Lr@E49US-^sVee3aU> zYPMdw_+Qdg$y(8;eNodytZhCinApkgO#?^6&D$+)6HAt%PPM@i7G?8lDNalTl2)=& zNSYbc#$(KY&CS37>0D`Qd`j@D>#OrKK3>4_;k;Cs1DCKC(!!qF_dTU4n^V@PPjM|g z5~L}Ha469e>$>M%YTX>&x-{zjbaS_q8;R4ub;F|7>+6eKQBk&Fh0kBO?LI%mh1v;h z{6ad}Asf_aC1a``?Mp%3svNT$H<&pfz0(T@?^mX2sOtCjI$yq2JZKMHAq|x$u^Wi$ zYka<=FmKF>kvKYQ4GpG~`^LU2G&-A83h=4U&g%^hqAMfjvgze(CqxYmy=aAKxx}sd z`7mdY#|G!t`~C#J#6OFzQuMkv!;(NxGF_gOsjT}){yxpsjztpL428d`8dLP$xR@VT z+d-if`#@sBE0j0iSl8;Ji@%A$(r->|VLJHSjvG1Gg8d!feyL%{!V~hkI(DcD(C%c3 zGK%C9x4WINoD(DB>TvhMA2xMy95nDF{zea80F7TQljud&!tJa;=D-Hfllo@!Z%@ZM zUNQ1-dmjLz(-p-I;Z`ZTLbni%ct^o&^2WCAylcVTBsST7G_2-nleq2I&a_9Xzq@wd zc`vJG1kOON)y5-Gt5ocZ3S5Me_wRm zRk~4a_j*5xh=JK86)StaeLW@IOOMQ=D;XW>zmLHtSInQMsVhttbe_FrKs2y`J;?j&!>^=f4|^1n#6@$; zyF~TPHtGymRK3z5N`J}^pH=)NztuD|9E{WcWPa%C>%ZF7-0RI+B`+!{JIKbCms~Ef z>>WCmLWOpLSTC8p5{JdL>Q_)yR#`h_Yp(p`-K|;cgi)lbWo?p{d!$c`55TNuVW?kT( zAz%k$51d}!_nfyI{Ks_GKXSY-V<#r!WB<^3k%Jm+B#jBwD%%^kt9jQexaD00R@?k< zDWI10A}$@=tjBGFK49rN@?xjk;J2;Yuy98>_&?VvMUuIm;qJgiV>RuYZK)eoHeDq$ zk36X1E5`4oj|+z}Q&cnY@d`TJD}0+}3Ch}wK!yEq{F)+Q0A!r`xek3tRfLk5ua0AV zgN`(EtUx!!sQ);bamVs|E%@8g9#8%rn~|0C^ z7I(>l$n=c4R=fO~#wN|4Fx*)z5d(s-x?6JsoQ-S-p{2;hqE9oeKLe)LJzQIMnw}iE z(gPw?DC{cG{{fi`kPN_2X1{rJ48%C7>r()7P=`Jf^-4-g3Q)_GK>=F&Nn|3I*9BSA zCEBz(SVHpse+TAV4+M~Dxg%Ej{@AO(E-3og5M#a{_~pu(f1|#D-2l55Kwi2~5KSuN zFwneU$t1RMoU%S8$y#0b8xd#eeG)RIaC`)8bVS~5UA5X|`igj|?=`QK67Ruy}|6qiX*_Lqx)Y3C-0noA>dQw*OrwRgCGwh^agX~+k!Xid%OKx#* zz_>zw@7Wp@Rntz;zLsmcKL)1%3GI+GqI@>m(_Vfn!5qpe`|qOA2{|+X7SWl~F+p1{ z6q=2TIjC*)47^w~sn&0gE`>eg3w4P7NUc}x zev=r$fsxE1k@k=EJYdN49EH(l**NRIeawm1Vk)pCRIcBQQ*Bv2v)I+rrUS0_qPD(#CaFn1jyU z6eW^)7xRE}YI?+uZFt^`c zZ}!GcP(C*6n&8+yf-3;tl-Fv!JM})3q?qMFYJb3Da^RP(U5Ks4Zz7_#c3G0f*z4wi z^tjPQPU2D+ZMjG$waV=FSP;DyMUQkwE15C)*whn@|PFbkqVDEw= zbA=Wpg0A^e-uF^X7wqwVe>&S96>fOsjdyqxm`~Ub>MXD%_5j#0r`tFa_cJbCE=3Q* zVFa*fX!Y=MOv6Zd2wCQv4Ov!y@&9T@&hL=@aiRMgw6LE~O|koN(~p5b*B7gTKQ+*# z2d=y9Xf2X7dU>$QmE_rZCLBGz6yu&XQUKhud3(!r4WO=L;CGPFLFc{tV zG+6gpyBP*=q`d}u;9_#wXd)K-2M17QGM3Qnv41f^WgXiHwe(;+zQ^BR%8DL#j~#4We`$WQ1XuMj+@5xP~y0##mUlek3e+kRQo40N{3 zL7`@+ZCa`wKnUR5l)9+HcIlc7x~cYHh!ak&s3ng|9+v{&O668RtY5x+WxYugINElM z1v}q}(v@K=;A5muw(&NquHUb8*3XJ?LY@TmIe4QjJkpk9Gi11W0An8#F0S7dB*(R7 zCs^b8FIBcy;{c&=pInP8;M$-fY&pt=mSw1F^!6sNb#Vi}{48x#8`}FV>cR(w(aE`{ zT(?56DeFQHhn)&JftVt+=TJH?x;M*|=Z=^6WaD0DMAx1Ui3n^doD&Oa%IabmOM3QZ z_QTi+V{@BZluOG=hk4+PKDpF#IOLZp&~7hiBZkGvy(*V46#UWaOCa3o-RVw8*q6C= z;l0Y{$%;2FNQSTl&+EG~Oa7Hrr*i6EqT{vFa_ym+d6h0$zn}Zh#Ug}zEI5tSmPIrI zXR#qAEF}itb^x%T*iw@9&Qh-)iS}5?T}rR0YNVWit1zTGK_NY1-v%DSH{J)TEB0su zFkv zMhyW|v=|KX0HP&G>D~P^!_d(ehUnC48t+~o(`2u_b&||fvw@Bfu0Qn1F-x=WpH?52b!+n z`bv%v@!@vto0JisJ^(f_=!C$a_4_#S#ahx1D9aZItww1o%@sc z@qLaoTDbh}+v~M8kGaONHDsc6D7T>lbn9HSsvTd+T>C_aUhIz{+fJdg-{lQd2LFG$ zd!YAk6J_-w?7^H*(>tJTAc~>-rp^}{NqTONE5@d#7Yyg!Q5vjW_}qQj*5tvBz5gbI z*%6ldTtaO~wK)m!itj6@ygYp@E$@}Tsk185Ko<~6d}V+Orjbo=DoiWVa*RM4r3lcT zQTCT>==}#b8^=WBfd+c_7VINxPrsF@oa@eO8Q|6gnoP|)Wx0vM*Ei%M2zwK(08EYS zpFmFvM0*nVVhvwgO|RPkIu7_KSqK-?nBh222T6_M`syK8iwTBK zBnlGIaI@SPBCI|yme^ywT1Bm&fj#tj{CTZCp!7Mu8!-1ZTe%PP&6dt!`6Gi2xK%8- zN86tn^ePEz1K%>Zg_Hv9s=i)mws5p7UFnFi2;2^;Y`|M0m_vGumKbZ;4|%PEhfL>& zw9)u4F7rGmY|G$5GHjOzV8L)>2Jn*C+iv~~U4qcic}a)zEK=U z>PW@jq7FT-nuZWz;QlGeV+KF9i0g8VD4vn=2~Sj@C*{p~&h>Mew_WD*gThk_)T}@6 zew%?ME2{n(V9ytb&T$GT~ z11H*zj^q{V?%tU=07YT&wK`mjqA%rH$Ko9k!Sdc~=BL%$Q5VBsfY%9>1zavRKwF+h zppW4gPm;BgbJHUu{o(`t0Q_YrYAa3lN5=!{^#Nro z%jxI;E}<7>pD3w6c!avWkLv)LJGO8oZ|u;(_f!z5Gg-Alw{Mly&g4D-?7{XZhgoK= z^v)^X8e9tPPwW;eelL~GP=ACd5)cX_If{AB z0QM(tAMaN8%~jKvhi z@Zd*lGr+ketsw}uT1Z8hM-2+=mmz6vVD`8kpr9t@_SQplt}YM0!gfZQ!EY}CkSX7( zQ>e5wH{Y>f8z?SGzO|Pq7f!dLmXuBow8UaAmm~^S+=c6H0o$xU9t6(sXg@*mv*SP64m3c1J~;qkv8*L^|H1rTj7U)@A7#X21Sk2ZK8`5I&L>gIDW96(yS z2TksWgMN2#bb2UVo1^jRD8!oE)eVYydgr+w^-fn0yiJK!ZoW|7xLW95`lUB#qI6WS z8hB-HUG)()0M{4uv-Ma8?~Hxm+H8P#7US!|k<3*gKn>Y*)()^Xl3jl_PRE)%+7((N zJfSjb0DYU-oIUI~Ip2A6#Om70Vsq38w3(8i~lS z(HPrZ(C&d#rZP*gRUqTz_BZ*`H1-Z7s&`W$lcjp1s)hY5NSJ5HKc{f5V%nx;4trb< zO4X!y-ZhNTBePa7>-Uk_b>_SpS0FB}lCaR!8P*i9z``oF@Ng}(A}ibv8=zvn+uAQL zlAkVWJG|l|CwbKj0QZ`C{uw_gv`cWE9KX=%h^P^@g8UXq-KA1aNH+ViH`8S$Q6OXq ztjoGI|6;zcSyIvyfbAmV6GO{^%?TRD4Ognis3BO5(M_lDega7ISUrBo$!}#eX{DA+ zPOzCViUD?%6LHnuqNqSPW(WV^-FuhEKUD=E6 zIAbP&o#a3R=K#D-;QdL|YRZ%Fjn<%mRw^3qmjW4ZGDO^4Dt2Gisv5ukPu&jmqIp+u zOUszn#Rbd2?k}Ig_oLBXqMl_pz6kd_dW~8l)@coJ1bZ?xeEGB=st1(S9(|!K@<1Ir z4wuZ$7CN$xTU-mN9q~vsH}j?M8>1Gn+uGGQry>Ar6$1(uZe5rk#tNEB)V zhyBlYJ%Lsm=2Dhl`WHNXEY>TaZnqWk_;33y=mNVEl=UbohdiR$(tj3kCn1RHFXUTx zAVzc+CC~A?hiQFL8hM@2??QSF;MS$?Lit`X{BE6KhV16!2u44$ZmCU*xCY}qI}n`( z+H^$3knLjFT#_{3P+Oq9va*9b4%X1#Aq0w=r=l)bQ;pi=j`zo!meHX?%2)M6w8QmN zFcJV+3ekkWj^yNH1{}yGj3u}7J(8c}eypvg69l@~q$RsMeaJBqLP1yeg5>H(ZxmxC z7P==!0>w+uclt}Mv}B7$XiKneu~ zDfE&m_k8wdN?k+)f}?J2 z?WAglnN(A~PuSnbitGPRq8g~PZxP{$;$qJsO}Y%7l5St_rpa7>FyXIve{zLICl1bc z@o~@fQ5^Ht&T2W2SLFY;_^N%9iQAyP4_p7@qkHSn{Fdf+&qx?!v79llV*O)g0|8fx`iu_bZz&bH1psM6*te3Q`8z~{ju|8ZXLeWz*M5PU)5}J1oyxAB@p?+ zZ&J6CuHv@#srW}92C0Lby#ZvpyMlhqYDU2p$2;xoc{O5Tnarbw{*>Uhas9aaMP*@; zSaHu@gmM2gn)v&!P)1J>%uysY)Xakr;z$U#n`pc9jqyY+r7Z9zwE&V)$`v;v z1<2~=0H7 z)PmXMhI#ahX_~7QC)!0fsYSo*3Y|vVT+3Gkb9W+1ee|^-i-#V)vu-=Di=iP2qZ9b${9pd=prRp|vcln1i&Y5txR%m`(x`2NiXD^^hW-K$`i zwsAnR%AJIHhimu_!EXF|J;i$4=fA8ngSKvU5wr|1IW}oG?oQSk1WX zX-<_R*9LFp{sXm~>VGP49^gzKC#dP_+Dn7i@#6|C&!ey;PCilb!2ODN-46qPf>@s-9{Isc@-B#qG0ZY@g4|xRE zN*@ndOt^nwpoHW!R;c976U+$5jFrm6JySt+Zhs7Hb+2U1h{+FBIAHq z4g#&y>|LH5dC^dDy*2!pQ(?rQrOWnmwSf0R&u6>JH3R}RRHjsev zF!0Jyyk*_rUZ)XsY*#`ahrPBKw%Mi&j63Rv!Zr7T1%PrD%11cAJ%2%+Ou$d(gW$;= z0lKqXJcN#{b>h_?ue7>$n%rAvXPJZCpS{L;yuW)0x7nVQDcUavuQ4G(ILSk3`4n-0 za)?oLViW+w`k;p-P!@GOnu$P_1}<(9dA)jbzSv*;KV$K>bvc&b{h{>%iF_ofXX zeqB#2w%#ED%&pltS6D8% zUOu5TUP_0LulMBqMepA|?zq1=(W>;m?nP?Q9Gwz`CCFYjN+ApE=TMQjrZIBHHBfg^ z5y8Rnu}jgWf;kPOxAb@tE?HV1EJhfVW_r`;aFaPTyV%pL=eQpw>&m%H zpKON#CFoA}BT;S!DRGrfGt-ZWLSCA(KHX$MxpsgcD{w znnzH}4z|KQJ^>(9+c=`GIPai|YIP%Qp?DpBp?*FZn;SoZYrIt(EySAPf`2fpD4cwrQ443mUZH?EcKU0Q~72U78jRk*8rF9M^z*4WQ|ds2a3CsZYrNW~lhG_D_q$rM0K3sVHb_oCq0Of+ zmT(=@I+g%pp1K^RF73&RhsdU^9&N$}ptmn#+(WRY?H9xUnb|_cy}w-#xHhcsf~ND- z@4DX(DF<@Ja*z&Koj&P{9&?$ml^5;)#<5DH)BsYI@@qA9m z@yZWLL97w(8oBFdxzWZm@j@yl{DvO<(qK3vVQ_FTXjim_yL07_C)dmX_8Cnd*rE_q zMvRUt{qniE7>{=XO6|WRtH6k>!n7k|`<+N6Kfl{h_?jbmZh9KL4{vcj5Ph?rh5~K& zMqx>jM~C(?Ud#fEj$VT~Khg)!c0VZea=cNVL{0EpNr&K$tQ8DlBCoxD$Dstg!V#cA z95A70DoI<4MLfd!C>!={5VrePpM0MbL+op?rUfYAd)bP>hl4i~dEf0lVjn|Tv51T` zzDol4!FHcv3RN1CI;$F zI3dAzY3Jc{pbX)>l;Ji%3ifJ|ZfBJ}U-@yGAJ{gJA?WNi<=|mcKmw|^5Rjym{`n9! z;ur17r;e7FiQWi@E^KgwN;pyj*d`#J)yPaWZ0scC7qU`+MR3Z+PQx0J;$TEkZ_CW4 zkRD)*?6a;z?rqfZFmJbv!nWjz<}Gh`8ZZc()3B@pBDMdSya_iKk)+f#qnhp!{sO1gDaan0ufl zDymzK9wZWCRIFt~nxYRL@d|Ik(-D|n2=Xd@bmJTX#ca}w(zE5-wn{J`ooKQ>xFDB~ zY$?rD^|E~d&Qb&Fg3Rs^4@fPCQt^9C&a^LMKS`585K8s{lZs%#(gIm&kQN46UJG`k z0*-@?Wza043)P6#V~f6td4L8Fn*E0f*JhVFzj<|ZZAU@-F(oWByyGEAGY2=s0|EY7 zy5vH;SPiLx&kuEX#2>@C<=U`9h%t;!C~Yg3h>t*N!)cD?2p1%efa;uAK*!*M6<|!b zmLv(WlNuSgk`6|(6t#{Cg4ApfXU&JRh4nCuVn;B;+a1j>1ND}SuKiw)0~{O@iZJ~? zc3Gm9RgT{#90vf>(Be3Jqb)}LU`u~o$BDJqU=jy-`pY>X&i}-*uY#p)+}S-JB9Bip^Y{X!3lXwOKzGw!;L&lky5M)A-G9y z_q&d+=_*@UJg;>L;0=I(2*>>#wcddVt@X7IwXb-o%eyP2!#bdHJ+_dvVc+tOS8X@9 zxqWQ7oe?H5q-AAuA?b+L(5Uqo;W;v}>-xlI`W6-rg?F5y{HqtVdxT@Q;(G8;fL<@>oLcACUc#GDv^{f@UAvPxrq1`h6n^8*ugVh#Qew#s8xIHKb!)(-&j` zo<4}a71W5Be`_(pj*i){ods5%ZzvRTm&ErLO?cG6kAN4P>~m=>$Kp3yWN?h7B(}eE zws}W#kVq>Ega9_%c8F&aB_$U7O$h^Y^dkwU&z>y+;nk|F(D`W-1ZO;OtTxz}NuHli zms;pjA$)RfZbU^J>%Si2>=O^bnPb&zEY^Kc<9zy@4cUKCj^I`}HLyh`LA9DKyUJ>! zXaRi^>ok-bzsRKu^HUlTPGW0@khD*QNT;%*qAarEL-#`Ot$Fkk_me;TCd0pddqHiW z$n_el)RL66w>p(;>%tnFj3A*ntqC|in$6U_aKXfTXuK@-nmS@l6U;2o+F!rz`mhG> zH(Wr*6mP`vzL|KZ};ni_ezA{__*RJVCZn`gbmLF*-z}lTQy7 zH}(3$?j+>Fkz#)UKBQpl_7kbJt*9*e*v7p|jq;Tf%x-r3?5FuS)7YWm&_%ZYl@mtH z{tbHXZuQGcat&KAAGZo2cGf@bB<8KODm~2#`aaBa;tE!QTC(E?1sM6g8Gqx?9fA{f zW;TV^n+MbHPr}TTb8@lJ2h#c@BP(6&JIktli(2X-qC-0?D<=vFbe!sq@4gDDurc42 z_^7##nCFe-z8K@=w$r06AGz5^4v=s%;4W72#*s%1+KYBggod*&QkR|IXrs=Nv*OjE z6&kEQU^rrw+W!TGf}IMPBGnu9^X1^|Q}XG1cmJ!_rRwJ%f2M<5didPRiDcf8-aE|Q z+xFs@6^{3V!Zapj3`49l?&MKL+8sgZwuyPFM~u zMygw%?x1=?mqVQdVnSc&fZV#%{*K-#8>z#oivn#R7``V}#4N$I5TS?(BA7ejm|5D- znnVyZvpo9~l-xmbk&>k#q53nZTwR_pK3N0n8)^P9dh;6DB@SVX((V(k_(iwK$T8!r zMK$fORuS*ZSX-a_e@nQxcz>H+h3EmV;bolRAH&L9Z?0-~XGV_aoQRHjM!<2EB`h9J zU{O07HCsr+CDv@l&h4_mXV>cUA?l?z|^Yo>ch##a|ao@7mp7d=g*xA=yOD({u6DLA#xc*Z%ok{7}N5RWr-J zttL(M4|DnDYrIkkak7Sbbv~*aEG)_)g*-!V3WG%?f1rn&JpUvjEc~{!(`c4LaRKyp zJtZrAeb^;_>b;)m7hlZ8O9{i|zvbn5-@ku<@a^xnbahXJ4BrADioczWz5O96>1Z+- zw)96&`m0wS+r5jhXLSy8(YG)7p?>}Az4qzTzl$9zH7kCHM2da#c=hVlOH_`|bt%hY zi}UAhjE|oa9D5UEUm56Lo6Guq(EN&T8l@qdKB<1T+AS2>{M#=yn&U2+ld;tgwD+?% z6@Rx;!$1#Z*803b`ttU0P4Pg7U^CIJdL}=`M0GS#N^j=Nk-|_1FcDUeS5us=wY6B7 z$s3MTSNSzwORa;d#;dI%a!PW$+rDD#DJ4r?ZmBj|-n7*&We>6%K`RaI@tBPZo;ge3 z(oj#s`Y)SP$O~`Ox87-WF(cNM;=7(&OsbnYS(Bowv8A}OUhi&M)jK~#AER3C*9^ga zZVPk=daizb*x4|`y1A@pcennSv{>3yrD=rKp?PvL#e>9Oo72)ombNKv`If&Jx04;O zK`cM=UL8Ohl9p|55%oBxlJYF?uqWR2$#@RA){@ zdpi?yMzG{tzo~ri{^Y}|=~|Kr?_Q95=bu^jb$%k(x%fO|jQR7(cRyxO?O9>%QpICFh^gyZktM&l z-Gn+lw~)JGQ}&x+`TDLxaBAho;`7~wHr8{v5X+_aY{gW3_lCIHQB)(MghX?oOGDkz zT{mu-y~>?Xv0gu)-X-@#)4XVQm&}WqdD0PVQRMo2Ja6!wYsv8=d#NmKbYdIM_aB<^OjD2NAu%fTXUd7OWkz`eo5HOS?>}+-)Q;6 zpKkR)c+BDQ4_SjBufM(cj?cupJEY7hK|I>v=7&NZk7&aP5K&IqX-12m9&-t5s@Hdj z(^*>cv3&O@Df~X+Q9YXd#RpB|g(zAtRj%w!hD%4;y%dU6AVOWj&8^4Z2&Rm;Y-}a` zDJrn+9P0$flvDTNri?5pSxF(8`QFS^LA~!UbT<%ANyUw4Oq8MwzL#Ud@Q|i8{H2eY zW1$g*ozb&rVsU$e@GX>m#PImipA!5Cmp8eNW_ORLcdgbzN2bCokO7kd-7EvS0Q4Px z&I2t4Slaq#2zoS(Ys?T(N?5zH}X8~No^=gu8#+U`4sUh|&sd^{e4yym_+;vWAxOg7)_ zC4uGpdSQ6LYoqn-FZ)xx$3?VN6k#(z-rq(J3AnHrCvV`Hpp%JR@E`^r`Pyu`J0m8b zI=Y4n;g^-wjEpyyOF2)Hh+YF|34!n%SVi%FKJI<0Y`^kIFHrc*@rrwUk|lmC-pFtBXkn6@B^dvQ+}-uZ!(GV!*C7HX1pRQnj`3ZSOu<8I_Qk z)bjUCBSTy&T8LXt>%6~+pFG+^4Dw4)%i+gFDOw61E@<43<$+#50{)dRT|;R+A4CKA zoian8Q*xN>zY`7LE1nw-*ZX`g@60b@YU69_C`#x7^(PBODy!S~uTz))DcO_i1!@=fMlt&b|Kw zUtcVh=0VYXHo-G66ud)qxnlFEJ^#%5^!H0F#viwg$lWHV;zR=ZkFZ8ZeLQBTC!rn| zf7TrBK2K`ic=y7hk|LNAnCJW?aQ2R2s?6%X=*c+^p7SSf{&_yT8vcg6RW9F#`;`mA?j zgQaa~Ab#>ibo2!&sUtD`vR`E}{L7PE zz4&9v#z-xZ%SjwjCnIos_JP^0hgJp#Ebip*Ye#PSZOsSVU`~chVb_MEha;&&nNf>h zoBWC`o~^;5FNd@IM6s1W3AZT#)r&mS&zt45R*_+hq=oBEYM8T6sX>RtZw zOkHY+I%D$K;YyX8I^{W1x`?lK@~x6>4r6b6pzhdkS-0xEb)@6Cme$Dv8B+lzsfZ!t zuMKPN!4?fGDy{jDz$j7I=p&Jz8;e`t&LC4%NB2svowNSY)S__OAXYdrr67ET6cXua zZi^j@)K$+MJLekkM}p*IqR1ODv2#CSRL7(^5B^R>n?x(znW{xnHTGM6ofhqSaP8~+ zsVewyX^?s^%iqk*bVOE82zB%B)k+z?((x(@>@-d{@OOKniYhoBXcar*wu}J%B{?Mh zNQ6jr?=#*=tbl;IDGCl#ex}3X;GW*P{AF1P#}R|vdHClhzdAZbg6ZXal}79j%rqY_ zSDD^I&?7m2@#lV#CEyPb#{G|olK1)c_hZI3a${9odM6V9PEsXOnPlm&Pw7mkG{ja` zxrvpSx!|$tFO!ZKJ0wndoo+9iY;pOUCQX396O>*SZb2rQ{Lxm}`uk+P*bQC**g8w%G- z(vfK09_-%xd4dA-C|_J4t$m}+#UTgJ9qY7BpD~xo0@wD784a~ar}G8n!W2y9O0c@t zE=|-%ZuXl~5Xa({WOSHq2ZATkM#h8K{IP5Q#k82_2CA}i02 zP&KwiGgQbju>Fh_0SPsC#LB483#Z*OA4R*$z`}Z>FFt@}6u6q?X zPvI&4PZm&}+eKezZg%PDWJF=Hzg^4;-LJ1+5BKsa%6*c01{F0~W~IKg)c7uGE|gU! zQ@*hAT`jQS`$qL;o2MVQAMxV!tRk$wePX0F%J!(hZlwR&hbAwl2R!|QIibm zqUPO(*DQvJ9c22VTG`!>_k?ytl;@F?r%|xeB0IN>{Z0Cn=-cu<-k7movpJcPw09zx z)Z*%mWI?I;|FHEIP)+x5AMlWp7Nn#@1!)xN4pBr>LTZF`Gdh(<1ms2qqy$7jkdW?X zB2og5t_h>ZBnAxF`{jP#_nhbX|KFXnb7V&pw%_miey&ek*EZ8DFDaAtwu-g?a zQ@E|CpcC;dp}FR()7RY>`GZ8T6b-$p7E!h0fgK+NPt&98XXbCu3f^y0%VQ6pIJN0z znBGobS%X5w*RM=|=D0^LN-t6=I;OHXMXZ)St)CstC2NVd{WdwtE|!&pWe*Bkp?IU4 zy&e{lBRff1HmG*rK{rvw-+_5S$T==(X_h_iO$pcfW-|n`Lv_&+Kz5_I$)Pseu{W3R zoyzc(8c!bskH4knHjX_RX_f!pZsvaG6xQEUe5!{tujiMi&X!5<0=Q=PVqwWgpluk9 z-X};&Cy$|TymgNq*LdmUA0C9rmlU(x(yFcgH#-az6N_NX2ECGCk-Fm|{&nNu`8xz@A- z%?;X<1LqagYs!Z0phpF#8pW~PcV#BTnd(oqYi7d%RH04dNdEkHA?PKkl7LG8>%fLE zU5`6))#E28&cBnU+oKiv0j^UYs-#JrygH-6evaba%4*Js$sGAG#?QWoylD|5wvP+J zCW_?H4!sLjFOCI$rQf&4Af%1>@D!ED3dc@=%XV7s8WX?{LeSh~Ca63aMB3*Qfn-+O zBAX#p;qQ0KP&^KLWqoL{X{TPd%6-=4+I8MJzH-Sn&ruYiw?kl=77!vk@tB zJ_6rRd^Ve}mIs6rI5}UAIwfI~nO3BaobS9?z#5s%&4`+GP3>eGy44&E(;)gukzMJP zDuH>fRg&C9WgM9apIZ4_50UK2t6BIUqVb9wr1o!Y_0VJx$un=I;hjpq+O*s1Hmg1N zNECAU^2ZkT_q6A`n|?FFuGFCTmMc1}h?`U!@wcDEch5(dze>rSOF2g~5I}?;2r8b> za)%-p2Nn7ag9>P~E`qNzz*!ZJYMyORoowD=!`0FwLOnl2kHo_arL2Mi%a+b7LY+wd z*$*A;x=L>NYB*{jH!pfeG}o)POjnBc?wyITdeA7irz$8t5~B&{-yEs4fe*H!nEv06 zN)&13yL4{*mVKj*y4T*hdao?)OtziRp|H=pm9Pza`#NJn3-+}Znc8jvJ^gOU6-cdd zz*5xWcHiZxZSV}g<*EsjS_lskKl$sce}1}vP--b< zyT{WTA;J>BbwaZ~RPC2ehzicj%4S-X#Qx&^Ab1$KA(vP9^|O@dGtfE-c07x}GuRhV zfiW_D4P*7)dR>)K{zfncb?{z5A$TUI$qY%-h~9ZD@Qy1im`x|tc&!?G*$s#^S%Yq1 zgje@RlFy$nvpdR@rMLeGF=qy?%cfDW3dmM<=#w=4mRT(M{CV6RkA$=4BjnKLhzfS4 zKg|9+Ul?o8?>Q`WMDYu`slZ?Aug$F64w8`XOJpTx34unB~un{DAzViDnZdFP17 z53i~?JEKwXmq)4jQ*E;+0sHnL1FHtwc^rxCa>3Z5$Vl0_K;$U4u>Bv4U3URNIr)Q| zmnGtVKbj>@9Q{TAyI->OacTZ}{H>nCD2EZ9L-?~U-cv6FwK7}r$$)Hvn_OkAvAx+c zJm44!lRm>Vsq{@1(pcm#xA$d+O+Qe{U%>fer^9tETXYVN!xE--EFlXDm6}@dY>C&S zQt9Q7Rhxu`hgZB-SA!qkjNmvhb$2JJ4D0Sb!z+)yXGCqqs(%3u#Me&P>cH*R{qYwhjr@+s^n`&I~!suTc3@?LFu@;s*%5@O#xXNJ+*zECn(hFSTjQe z-~43ntd}^I9{0s_>~Zz>I`m|6)H!l~&4Od4r~YZlA3b70HdKFZ^2-r#CMrq)03+AM ziwzxp8iv57n0HrG*n??}p~v6YaXV=Q@+l#O)J8{wpLP8e!~eRk*0IEvwdDT5{=Xo$nJj{Rs;(gmf&A-p^fzqR1|u2DRP76SF* z1}*nRGuP+JEk7y!!iszzzA>)=mMy z>nVeV$g~M>;CFaA>lt+KU2NRbD|2sYV~K{A(0K_Wx4+7iM#_QwcKTXHKAO zm3xf6=WR>BX}PA`glzt5V5mmB*(2|OaJR3U(FZhIxzNZK9G;k!!b-QoR$Mmtnkeat zlzsGa+kUXc8JOc#D{FV7n1Nji60I(Sim8|K582_&WF{~5b%HI2j}_pqL?>UKxR5kTJ7uO$I6J6JjY_(ecUi&)R1#=L+Zh!VG?%bq}QF zzzrRXZ-A5k;o&#-wv*otpO@v6V{eGthj?C+($*oDv!x&FM#9zuQDS)^EeO5XYYRq9sgwv*<~*rnYL z;lm&+KUY6eolvXTA8Ahj(#~(0ziQWTL-H{`^lc&~;m-hn?97*I#2zO@swuEC6D3&+ z6wU8~(pId6!M6^^BzxIj2dDWXrP_8P&I=Dwx8DoQJ+}8v0}<-aM|4%yRj)!meyzCP z7vfsAbvQmM<*x?#$p+5Fg#KUt7Uy5j2<$MQTjt}_J8FoVWl`t-+u(E9t9YcC!jQhv z6P`I4(cwEeHYa@Ca&z?r#o}5sjp?eycDih|u|7NPlu*ANb87Df8I|(0+rJkFfzXk8 zA$3t|pWmFihkELGR?`1*(1zx$j&DKt+n(_&SkTxxqGixi#YIjPzw9syX{ouKq<-;q zr=LCOcfNEVG3kCRY?3NgFHP;4*l86xZ2O)3AZ*`VRZa8u+S|8!PIW7Phd@tFTVa%c zu3Bz?%ROT`zoXAhV_?Aye7~yYBi0!gIqU6jo!hq;zfUToI{WYDARQyvqr!~xJi3NC zca#w^7SD|0uK>#`!(GRwM+fxznx4I-?hMiqRn(yC0)1g@^(HSIvhp>&Ej%+NWEtKVopyLp zI=C$z&^(UKXkPyr(>j;vF#MX+C$z#YHdY`tjrYPTjH_^2?ka*gG)XQ04L6T!Qa=SQ zGDG^?w_3$M)R&JRsnH$K*Swl6dKPmeIm7Y8I%zEj&^T~PbbYBrtExr87CDm^*<+%m zD}{HWzBAqW-{-;QpILVPEVNwvuQOq6^-~i(lHG5f-#Gx<>ml(iqdYk;OxQ-jpXM;gVtr~xJ0XKS8)X{JZ0VQrNJQeSxS~aZ;Pw0Q#yG{Du{`35=9V_N zNII_Lg>&5Ck;!SkX-S`xH~n4gra@!y5by>+!c_c z-W^n&cFf9>-%fkEHdsw^HKpO@0%Bu_A#h7mZjMpFsAN?yj-Sh9fGbs+lQO>y?aN5U z`7yX?tNJS?ORjB~&pS#+J9OQMLp9PuMYw{>yWxrnz znq`lx&0DhEttH4S2L#?7+nH-aBdictg3zi9RB&&a1IB#aESAJq$Tpf_X>iNQQ2aw2 zM%^U;>88LfdgPhj>x>8en-kZbM<+x~0oYz%Tud6=wwJGt!N^1Zt>^28`uYS>U0kN` z)9y8r64BAp5^IID>~R+Xa4+`q+E@kv^qJv{#3MO!p)@Rjuwo$t!N16Q)qG{!0yHeo z$?exh_Tv2q>Lm&X^#5qxx=Le`-!#2Hw*(GdG&E>c!YXLB7Oir{$@N|!`sS~DBWzvZ ziJ3u{7Ben?&dT0Tuxv2Wdox$u6lz=lb*o-L6rsj*QnQZ1(bcp~L}h43cqiLE$e=45 zSKag0K%c=q(u3mTloaDYZ?hld23a2q84=fup&XI_<&hruIMmrbD~7kWBaYkt!nAmu_yxutN5sEmr2b4XJOyT zlzPp?4%a)r8(`i!#5Tv#=9TPLMT2S@AC#%J$!y)hlOzgf zexDkFf1W#HN6@~L4ld;9%sJ{R*J{ zZR>?f+j03d0T*Z6butgLShf(DHkkvPgpWI0_oZd;Q#L3C-MYZ`h_cIYOcsB@^cT*O zTA;q*idVLGT_))3A&{E}16A_@Cx*HyVsZ9j*v2SYR>9p?e{r-K$LA{|Y%~fSe_eDV z^k1gnLm@kdmWuG_{xE6@-w)X`=8*0)U&^DyjYS%naN{@uZ+wIampI=Kl((BPtAA@K z-7HjArsV8qVl`fJy`?QG^m~$5K;@{(;{Kc=qT;$jIMh~=Cc$R2dTBp`v#ULToH!yv zoRWm%1J(QRAcU}R_ecgAS(jS!xZlr4=_Ia2uRS}Gi_2OI;Zit1QAxRVJGn~i9YQfI zM}9h*lyfe!EyOj<3bBNm>uR6f$_JKvHpg#XddBTV%6%n59foZoYRB!nxY~+_3-UU)ehLl#$^0MTed?mb zo7Og6MtG=(*RmwiN$3q06(IbCZ+|uzcq1P@ zMQ9W))LW!nQaamk9?_UNPanw^U#d&1|IT5lyV%gg!Fn7@(>-M$+xNVu`bm+gmu4;! z8>i^Q=$Z70Ml(1(n?^r{S}Vyb;&0ux;;o~n&eFGzc>?+#k=kz74B;4$Ii7^d{Dv%F zEboT-)!(SvO;D&Z3#NGdc$MyFGdFA6`_J6Ys`Qa0{Qf6!*M+Fz$IBM>Z?42r5ZgRa z=z$HJVRAO4>znBBI|e7-L5CjFZ%fKmoGb{9*}dJa>>EY^(YMQtVQoG)Zu{g>{K&*n z%MpQcKp0;*ED5zzV-r>ZW9N89i!EZph!H)>>u}W4;(x1!BBawcmTML{vuwkxax+W( zJqw#_`OqfG$^Z}HR5xfWB+qdt(Hy%NmLwz(cq(%`+-lPbB_|!>W?ldN<{dnB28S!(|U%b#hv?a5>j1E-8zena@IHKl`6-si^KAYzAa~lIQV87 zR#LK@o-G2bXjUfI+ToFobuZqti7+IQL467S0AnpTvRnD(%Z9I|hj{Jv(EV(ZF&RaT z2BXe`0iH2@U!0h1S1WcX!)VcvA{B-@KVo-nOqh;vGxpvOIe(J>PeLw~nN3JOS61%) z9+NzvKqDjNFga+qo)C0SOxTN0k_kkhEQ=T`+`Z14r^t1jbGsnotOXxH0$}j|0J4V1 ztIve*^wz@5?_o0Spr1N#C~6)#;*b)wnwY{Md9Rfl1%}&_@r@+Spy_~`vMPWwq zjdnbz)^Ox=`Qu*#p~twe62(b*YbEIbjMTjD6_0?N~qHWPRE zsfyL^ozpc-irq1i@2hiI9d609>dqbX(=PH7iC5w9;-&DeA(<+)xU@x(htIp8sS%Fb z68S5E*XdxPSyAo{vzTGs0PrubdPH#2S40{JM`Mg=8cgfpC(V#<~h!_+P1Gbd@%^pcCu^smE5I{@@^)Kv$S+0 zdb#^T28`IUwj^JS$r$Q(O;la(z)}Av4lNjaq07tlpRe*X{{)+zv)%5m49j(mW$R9k**9)1^9!Bzi)l!Q7Fg8# zIhGOlw}d^^P=gO?ccy9XV$_~1r#}`~s|=fs4vSJH6;Utw#V_V(PXbyr%8w1!r;&1h zArN+91^yYM)pfb8eoT9`yi1AN+cytyB^UJPD7rna8RL;|AdYz@$8om$5@nLKzID=_ zzBA>SN4c--{d3uRU98KtUbW(yiRt8FTa(;x>WHT+{9go8wf5>dgw>ADZdM{+1@Xt+ zwpe90U!33-u!0iD-0@zl5!_n|Pz<1lE*zdZPnG*y;K0^TyX|mfEfV#H|9(5J-|uP) z#ZjY9LzZoBSYNLb(askEG!0v0NpthKD<@)$1=5OLVELJu#%1bX+eDgYl+eM%JVXL( zyLb3{TkTeCVn<4dNm_alGEIq7Ic0&UtS`1OFT1C^Fhv!pZNsF7z&px;fqccbSi!pB=URliNya@Sa=Chg z>?hX&{E=AZm7|&Fe5*nZw=gnH@r+npc91m7{}mqsY2FIew13lWwe;cqNAKRyBIsF5T zi9YsJdX@ZpDJP(!M>Il)W($$mldEvQq!JxF=A>e4dv)7+IiuBu4S+aEbdWwc@x`5O zA~CWbTB;A5>>$CY9zYWWy&33Z4~eoDH`Bo0Kb+;mc@BP8L{Pi8qMEiVfV4AObngnJ z*>?rT53xqi#!MZjMM**f&m${7`kKn@e7$oSpfqFHh#=q1|B2l*MI+h%4j?_`Y;dR-^ffpFV!zvZupxgtPX~gT^&get7Q11 zh%2oQSw~DI>2i8yk2qEiJ=lCZ#hntA+UM4HiF9WWin6ROqWkOTJENy`Qn`_{1=6|XC7Cm8u8a9iN=K`g*fh-Mdv5zzWL=ke z+aPXxYGo4qfFEo35pjbbZRVA-wstnJ_ob)5n5Ow4)oxd~>APh{WN-Gvl1N;o9VnP- zH!6c)8&0A!BuF;Xbw|eqZwDvA4kECaI|5o5QTyt`6N7GSTH%+8u>QmLP%=w#(?6$(LI1Ihg+=gI9O{`8N z{ddccT!okp=dA$39c`4+eU+NQywWz#)tI|)Byi)oTI0xnm0Fgh3}Kc3gIX`Y@V!xn zu3y^!qBG&2qFvp-{LdTqgnz=?XtJG4H(yi_ZrAHUp>>Yttw#v}G6H%E@^QZC#ZJ*p z5y2~Tr9}Sbp4`1N-Hh7iGCC8@bs9YN`mt5t%iHrQ@(Bc_7u~Dv4DI=n8<SBnec>5@=fL)2T<)@Q?X8^q=b?qt4y#Obf9dKksGC<2%hM9^O zo?&AP?3lvH8Wn$dbN3F3rlKP0Y)nDk-5B~4gaZ}N-_Ep$H{-R)G=PZ#qGvKE5Io<% zFAK*|L&ls}$XQZ|+qcuga1V-4jP*ipPM7;jysMMrxVh9+`9$yatY!WljC7Mt$#LkB z^1tQRAGqRZKJDF9e7x0I@ta;UWJ4hkl#<R zw3|P$?RZ2H2mIld9}dyS03T0f*a{LfL1ik)7~2s|3-Z2T-p`&oI?*33E$3g#z~}%& zTNq_sLumd*bO^^{QqdPNMB?2yE@vJRE0|5YibgKHQx(>{*wo$*E4!xbX>$-0@WkgV zRDZXdKKemOyOrURWe(fxv-QVV8$I=?$x*IWkA66ul%MYM6EAoPb`7glA}HRkuC0;q zvw&B+Ha0gS`*^Zp1(Pm>*C}a}?to>7Otp1p!sz0J`1OAf)iG2-vX?bUXa0z zVoL=_$o{m5i>DNrp^GEzEi8;;qiJv7ske~FZBMq7Za9KLz!2~R@poI>?EPFJM%K+u z+0-{&yUU2l$%O{@+QHDA*RQ?k8N+)}%!J%ZvRB9SRZ5z~IDI!?QO@n91ra3w2mk<% zlIh4fYDF?O4vPzUf$aZbK)j@OL2Y4 zC*NY^1jnW~*<6o+8W^h}08E!eI~9^W5Ct4848PmY;-SmYWIobg zJ$hzw8CYRif`Z1*-{a5KDT2#~*%D6aP!{-4a}OWd8>rxu-*aS$wn_otPC$75;Fjx_ zk`oCW72LWR|eu{c^t{NM!n+Jix(ON8sb z?lC*TK(YVuA3pU27M$%OGEa6fpyxzVy*PLTCC(}oxRdJ64;x_mZuKS_!~`faRX1&( zphKrcsn~Cu4oPaZpu;TX#iLVzB-g19`g83XX{rSL<`>2gW7i{tFVuhjb1z_U5FMROBo zeC4j=t-$SZgGmCzl>M_@`BHq4pTw&arnMbVSrv6&C!Q!&$gf(qlpf*lsC%NV|%R$xXh{Wb*k(;+-sP8QBr|*Di&{Qq+)Ctd>f@T2k z3ifO@Nrzw;UP{x-#PTi|rVKCt>n6#=3&GE^x|rr%9mj(i$Es70%8m;^GLJSt-}A+s zEg2!1c6 zHz&A_GO`_im+4vLCj+ zs1X^9d;9v92_LE518r+^+v~(;i*ZL_b1U)xS408RN;U&%+W3TJY!f¬-^BnQDs+ z?x?EvDvajDJ5`yPo9mmJ@|#LJ4zoQh63PVD!SwncTFVxkO%}^F zPf-6SUM9Lw-`6GMqvgM+;VGTXV!K_EJ)iC?r3X4WK1Wp4#$0!t|Fo07l^$0ZeL5zR zA(bt|H#WqgW1L9!b&}l*KP@y2tLt-OIMdOljueFlEVhq%BxBl7C=`Od5q=*76wfq1 zc)cs}qy8$|rK!6MtDVf(n!{C=X^a#F_+41H@oMVu5E!2?YCkNZ8xbUMVhD;z9SxYR zO`MgS4=()pb?j(S@-v4`ZjnLZI>M3b2UpWacmK=|g=>ieuMi#h7%om;R}Y@{Q`D_D zg3k+}AzW0buqjd8!A{5yS>hxY|4*5SjlnV=10&*7->Px90{BTw?_OWx`S+0;MYG91#_JgP;Ym~T@L7vQ)FR~xU zZ$JV^T0A_rBfC~%#@fO7eQXoq@J?Qbtl>n^)9n*ckL=%=nvN6|lbK^{`Mqu}^mC)qKKFqMk08*A9=`bf>1YRU#=6sztFnm_FOE`|d`ck*MFB)Y-?aKeuG% ztQ5(zCPT!1OSNA;8vMn+157DhE=R5Bgk2tJywzvjl%7i&`UvdI4ZNxLH}UMu^tq$l zV$X%2qs{i+5L3H9K#|uA{~6MCvC?*bbW)B&a!pu=_Wnh7olJ6`F3?G9LcWYyOYX=|* zheJ1neo2R06_oJgzVr?3k*P!|LZtUps*#SR7LnP4bMl@2tD(-~L87myAR&ta2Mc7K zpUSckMv$qKSJ;(WwS02pr7e6j3otCP4U#X&B?2?&$c`Vui=}hIYxJ2M57UQVsO81C zXg2N-lO^ZL$qgODm6C%kQNTQ(UuQJeHtXK7DjObtet6HxeaCy&^MUp?<4;71EU;Q1 z;WZ1Bxy`_gSU%g~qIB{O4s2Mj{R1v{v+}|)r!X?=@O3smd7Y^3lTgTSr+S`S%%Bcl zVJFd7HvMWWasNKqdr;1(E@M6M@%&!@@|&nXf0TJJ7uJdZ6OO50+Q@8dY~E-+`v!77 zCMqf~KYsmRNuRUgf&$f19UYxm3&p!MRmFontw9ONdG3;Oil-0fEK0G9Sj9>@$a|qj zerK&A0aQYa_+cg!{_K4TcS$sCD$e$3o9Lk>hZ}x5ZlxYf;wln*m0`Hv{PiICdNLbE zF)z5*E2-_wl(#(mLM|oxU;_i*9&^fqwVIT-D`EEwPV1g-06$@9M#$e@r^+qu4iO3; zdo=d0LhyB+2z@fkFea@dgzV|hW^wb0t;eg~=z#4)YpJV=W6G@PY;3T9N~1d&1gr{Y zS0Z&aiq~0nS?XKV`5ZF2eXNN+l6M09NP(7c+iwMDriCRKOPD+4W4-BIeW|buZ7pxw zsb|(lv7(RjV=Lhz?Q*9MHqfNU6@_w)H4L8e&6Zxd98&i<0`RK>{7)}V*hUMb@`q1)piX1h(mI?nolf8w++*D4aP}#p1fYOXt3nA83m9by>j-e{1cWym zKBfKn{ga-yrTFmJ!a;+Y6aVB1ecrrBT;F#4wIFrdc%-9U@ieO7MT%}j9|oOLw#Ztw zcyRxj99*NEv#e|_5UnL`dgxppW*b+XOH6Bl(ikK4(o=^PQzbvfP zT=6s|SuuhX^cY~vaed+WT2@Tp{u2xu#}mcvMQn77jx2mHDl@shy_%KtB7CumX^m+s zO^N+QpM)28xM9jOuht**27jPgw_LVM1@E!p>Z%7{`{{1`pNa5@6CqKzscT)7D&V#~ zK{@M(v4#sPPgS_r+($U$Dk{d2va=T<7G4~gHScL^cU<9R#iM5P%he;FJ2;|CXM-$8 znw^(y&)n+R}b>lKlAzdswDJEC$~M?Hyj zRGL*-LgP+%1NLeC^TnF2Ji9Fe+*v>xHQe`_aZ&Vv`R>Mt&_J~W>%7I&IsLKf6Z2P( zG0ui5#Be3y{VDvP-#*#1N_+L~6UY0F0r10%Q$W#6H}%O3zrK$ordiJ)vo~_N3YzoF zI@$@k;Z7HZaFT?S4quK6<;!c!PR-n$m4zjmMtumZ2h)=PFGUrPZxFeEHD-My9q+U) z#!M;gznx+IEIeHCslW(L=xrXJX8Na3pI)5-i&6+)MT;1f8QXK+HP~@dI-DRdZTpEY z`?}LRD4Iui|6-;N$(@dUUpl$*I&hW2dn0RBGdk#KYC$6L-1u z)=KKW0pV3K-{1(YvzIilh<92A4O=S;$xLj1{qohs0cwtILGKDKrs*4b9py<`KUG!z z8?C}?wSB$vbUsWn4Qi7EwC_T+d6twB@p+$cr8%SK^T_pLCKjr)GrJvpJRMJ9*zHQ&UE|M!9z_(g49# zLXQePOUnz%RomS*5pimnmDImu6Im|`8^IkVJ)&U!$Y)(QwRe;fu(a*rSfLsm&enc} zEz&DlC9IHQCRGwTngJz!Wt|VjzSAfq3=RU^Vb(XE9!yRC`0GgQ2=$%%xW)E`Vx%wd z6%cPJ9SxXup`%oD7HLUhzJw~$JpUOFE3bTVmGNwMxuKHjyj|8jgeZugY?}7raN0tS@;gfbg9f_o=t&3ewO^sDcPTwZYaVMjPS&Uns(+6cn z*U#CU#Le}qbmpohCRdqjpY2SY%7{-`xNT0&y*>&!snvkUT&!&c zA@Gd74g6l_bKm{lKF6oJWcQZY^;eSBeS!QtK6lpbkB6jj) z_0N5j*+;43nfnc%v-MirQ|R;X(X{_TZ%TwpxNULY_|^)V=Zzgv*VJyOt*m9`G#TM9 zu;Lk{B@AM}ECOh&Pj=cvrNpPuP97~oGI)2;;bkk#0`6}4_uO1kaQd7p!j<|zaq8Hr zD$6?LKMOzbK{{>w%jwiJ@2RFzqYl@U|>golTaO97vTqVa|czpmyR6_6tBskGc2_y)l{|*FAwbHwJdTw`Le)|TilDd07 zt{XmZy^uRqHW3}``pq^z9UFojg)m#&4JGPT#d3*-d-0P>Q z?{TQ+_0Kk*I0o@sQ=hz=?Oi!V(TiO^nSB%qwxlD=a$`>Xt1hF>L7STV_a5NI}nw_BV=_Zd2xa5WK8P7 zE)0Cn-W7@rJ{vbUL&pKSe3Qc~r$UmjzQc#?M>FD88#0r0>9eM&M`!?J#4J?)CA(;zuTY{kbziIbiL$u~;D`G~?pdAg zGD&s0W1iTXNYkmx4oL#9gJtUMWcYyBA*aa|xXZkD}|&z`(=sd>re ztW_O8BK$7cE$|D(_%Yr!charrh$-1_97;{TvgZI|iC|TCWe2B16Q_Bu0(niGZN`(; zo#Q2`5Iyf!{R!8DBFcY|t`X2z43+=4Qcr2jisGff%{< zZ3$jcQLTw%v#%^8^00S>ZO(=_fc2NOSXZiLN9n#<($+6ku9xiZ_*kkh2EvX z8!gf*`Um-v^t1E~pROcwOp`P>&t1aI3zfqEAOc|6C zfssdUnz$h;3d~wo`^YpS?u~cECN5y%^rwwPQzmRH0idp1FKFCT9#VFJP-wFM@~{AX zxgfczd_}Y0gV3W9UJjhZ+f5?a*>xCi>RTpZm$PXpd%18Cg{N`nXxT;VC;N?mgR;mE zS%i>7lyAXrRu7ox0Bx<*S$-$7-QoW1-dGyWCLmK#9{avprsHDeth+V-LgpzR8v*Tf zVml5UpR}j5(0!|QW4AqI<;6Mh)6w7jM@jBN0n?kCPfVkHdnae`6ln3?Q_~?8#v}vP zr?{F}`Ff?Dz(aaq+mJt+K0ri?`N=h4h0>Arl|Ua*5!lMP=O$^2Iq6WyQ=-hd?H0CM zQYifHk=V`$Z^9#`wzeziGdDh9+qF}_aTBgEVv$r^TFn84U+Q(~89({uR!&Z*&BkBq z=3#_IU6(C~!>Tr)-UEu&T6M%wrUTpgL?#~{=yZM4<;xR+c6Tk4ZPO{8)4g+AVbTgI zvu|I-h2WMIrld{C_rT4ROSa-_a=<13t6Iq2OlbE6eiz*SR?tC{IH_zk;MGqFtM3TB z(tIkS8S*=67dtp$Qmn>`Zhhc(g4I{qy?rSA7ZapHJ^YiEL~;1M6!@+$o8K`V5Yf-h zg2|VQpIQXje?y}F`Y3=c1@Z?+d1)aoUOui(aK$)90hP%+C{*6rWF|0roS!-)b5NZH zmJ1`Cc^5lx5Nvaa!0XN9_v#+0557f%e`e?Wek_lKz*X#dz^#=U9=5^yGaP^Y1uA3q zq5^dl!hi6MO6a!WZ!qD0DSRkx!}Z%oa{pXGB{0@t=pA8%ztF#!{1slve^nuk9kEdV zLC5VF89ugU_58>ST6fq~2!`MaCP&0cy=~kn_G@~7MR{3gWVRX@f=?IG`%~~q2Ivs% zkID2-=huMnSMw;}$W~YA!<%$xOQiSMcRqa3kw0E~A1)Kp&b*7}>OCDxaZvNO$>SGv zg$H!B457qhku_mvW*JCSj$B#WT2Cbu8riTwhi?k3S{%Dn_wL>eM%0<@G$NCe*0&G7 zI&^!5G5IXL5F2@pu^pbg6xRRuqhK&aew5(<$I`pxmr*`tF`2L*?5)uwYX5_axc#*F zR^W2ih;^f$TfLVi1F%G?l&13XYsY}YL8yPh))v%gDDacF;wE?1w$vskHI*H>LVtK*xIDE`45Sk_lBfTA9<=w<{m8+ zz*sLDIlE|Gri3JYH?lRp_Yb|+yttnkh)1;by|`tN0LP>&iZMK~mj$d68^mjEPyoQA z2Ot#o=}D4?srI!$S~n;y9egGH3M)QQz2o~}y}m9iA{lPA@@65A`9T>wtDv0zr+c>c zE>~3%&)R)z%p5~f4O z>zgh5eqYG!H8*W)rm7EKmi90rRy?))(?xUC$0}Sd%}N=q@`RXYW>Y)LmfnJwn)5M^ zu&r-AeNL{gC;oisSymJwnu+55BYV1^t%`%i-VWshAhtZx?SU4yIPZ?iQz8xT_WcU_;!TI_rMce=C^Qgd!zPZAxpso<71}B@6Q(7R3?O>iyW!5@Bm`4D z)6IXs3_?-TLVXFM_B9@CDfghp1w{ra5ewA>fd5aGz0ass$0j_EEl6O+pVX^Y8oHc^z*`B8O z=NbzGky`qV`!T}&XWxE#FM9W=jboM<`xqX8u^!rI+m0~I?BWsQmz|C3B@8@ZnNMjt zCWD#kCf&!T; zll|+}PWpF23Z9m&Tv;=YjEZVrI_TStkfX_hrfOEmwKNXNrK!RMZIQlzLW)P0mc3dV}`DI(GV-*5$Mc>1Q{p!_4O$DluM|)Urbn^cv7e6+AE#soOoy2 zF3;{0gvQB)x6*=NY7CX|47+1|cNZ<&#@HUK^rz?QWSVJZD(31K&W&&#)faT;cUFfIox^B>V+;gJ@1*$sO3=;Lwu_Q zgMPAS%w7WO2i#iiS4s_SHN{-=jZ%K94_uh}Lcgb6j?x*GI<&?h_|)?Kp&m__yo!}l z%njFmUiZztUXfI-D*QZ;Q@4@nY3D@vY)yR$ab*OaLUVxvQhwTzSpXjaw;eG2NW5$4 z@rbh%5gU~Q8~r}&y|ZHjeDuttPr8X3&wWbP#bebh9Ecni#5m%9V>udt?uA*igRZ=X zTE6_K2!EH~6YbYA6ZH*Dv{5%SehxyqKjZu5SsI5$g%A`YhX9?$(MT<=Wl1g7Au3eV zqs{hkpFexdh2!|7L2$f!`|{nn{Q%i;d~sM=m?0_5xR2tamiFw(|3oEv9?Aqv>T>VD z20vdb`7i>UZPwd&kws{aF3X4}a__LUZL=Xsw+pz9rVbP26AEtAUGSCw{(A6rCzw_N z>^Epd=-$FJBG*Ze5l;4@`Qt@9DEZnxs&ZHz$X&D5ASKdouCzdqRfKIW(;LLpIlkw% zI~_&MCS~P7N-b8Hap%yT{fw>F6&s|b@vD6TQ+i)NTG_nXJ;rAmzyX9sqXBG0C zwf*#f?vw8Ii_5?2k4IC2mTazCNZ|e@2^lf;W}v8Xxp;qXv&Y<~fzTm}STq!N&*OMQ zUfXh65zzz8blYAEmpBZo*wP*0=|L}dpr3$rY|@H7n~WNXB6u-^_ahpw98~q)>4 z?kMSuRbo_LoXEP)W1{5^)|XK>=A{^eeMTXi9@SG22sHdzOl9aT1iI4BUj|_zkG(8~ z;D_)S;e|`4+n%GbGBYa60zwzDIJWPYeL8m3XQ19zo%db$*uMpcZ@2I@v7Zg*%U#e%Y4| z_jO@@KPSa-`Cmc2mZEHZff0K~6BXEst}=7uSDJaDw)Ob(`n-(Zotov|d=ek|pv)&U z_Y%3sb5?5D>OAder$Wu{{YJ`-8js%zmg{$Ls$S0>n8N%WSP<9pGi|YD=Q!E74aLq- zMhiSY{|#JQrVSO$Edhw7CRD{P76&ZVuMtqCsQH!<*wI30&9(V?2T;=n-$-}XyL#;Q zM_jvAB-M^cW@2^@W=v2_$c8|sq0$@s7LEyc|G7(TbKfm|=L z9c8?fleeR8*tAeRH{t%WbautbwN_y6bXMd0hqS9@(`%*`)Etd(906zv#l4#hJST;I zJW=gBIs$Eojh*LBj#oMJd&d^?_@NQNToXX5o^3moZeBakj}yRsMSyss6W`jAs0d%Y z3QyNihsBFGSMMLS29eZL;9`edG&Nx!+OOF#yw1N8m|8l!J>GcXHCS2s0w9Owlo5h;VeK#%S7IIauU@lI0Nmyynp~#H zk`kx9ubGBwZW zyk&O=0E@yU(TrrfWqtQig@=h@Jw`*{5{z^?g_H^2y6b_)R*K9=hyE~1$ zny|eCD$e+1sSqnVFfNC9A<$Q@c_x@*9`IyDF&S}DV6=^;E;L4pV?G&-;v;~-CEs;> z)F1&MU2kM8(n;w1qUm4@aw{^wh2?&61wQQ306hvQ6r2=@=gU1EoQ&*)p5ZX!%TZz{ zBiK$a8#xfow=3K`ZGZlVd?6fX#-=dCZ@Rlda7)>f_V>YoSSUIK+WmYpOs@|>gQ=E= z->z=|JYKh%f_40p_>K248TjICd&ZJb-oD6_@G#WeKT&17S+9igin70Ks3{^iu!Nam zhLT6^M!@W`^#DV2CqFem^|9Au2WGn1$5Nqa=?aw9qgkFjQAn0?1t+yM2A(;_ENirgPhY*SovPUF)@0GI3NXp6xS;sDWM^<+B z&d46wo8R@)Ip=fM_mAKEadf;Xb-(WWzQ*&qp3h5}>4LriSI1l+bSm)e9YZh(^Q5Kj zLrmU2Bm==f&S~)LF)r2 zA{?Cgi?&B9*)2bt_ctDlReh=)@h|cy#2xgczSTP{@g=B>j!_O9TD?GQB>L;%eb++F z%`4{G$8GdiplDfiWI>2HaWy&a{-tWNcA5rAnCKk!a3(lh$DZX4c|$`pQ!v;2t(r0W z!QZSQCxbxyB(p%Wi>Yu9IrZ>hC|!}4wSaH%PmFz#s~*G0YHQJUt2ag>UfW>n?f}wz z?QO_DH{KYzJajQSsxPD=)pI3QY6$wJMY5b}##cc!zb-z8e3APGJA`u!|`^!`&CYBls6ZMxQ&5Uh{6dl1e8cLMn2Y?pZY1KogGu4R|2l zr}ptOCMapr^Q&Wg8h9}{c+YM1Hp$jL*9(2Tz9zjvlh)JCGWp8$ ztL*~Zmn^NB6jSTP(!-OIp$Jg)MqsVpYqUJ%orf^Z`G6U>4?Sb?LGw3R(gcQFgZCzO zCwY$^USMFmEB?rak=ck6EQFPfEMK5jn_C3bL$Y1X+Po$$b_||gO1(#iIeOef9Y4B$ zFmzQH`m7Jb{Up-Z7wXN3%N#yDsI#2yvJj2o`q>N-*+f>WKD$9OkdSDCIAtyK3%#nxHIaOTv&Q9W<;yY65p{m z=2T1h=;019F|9X<#s#;*;RxF9W8GibPdu%yc8$r>p{sDm;w3=VU=qPgsNt3IEhco0h+0wTj zDV`xVfjPU9oH1SF2cP)?FK~15jQF^ggcoc|jFKy7!n>6aXDd=Zu(@~5%yaeB#K(Lf zX1;{j|CA4t-YiE_Rv~2izhRc-+awd~xbHFLl%$Zs@DXv|f-KMA=mmRX)W?t7R%J)Q zR7G^xu66r4a_0WDA_-@at($KBf%0k>zTTi(IZ?HKSXaLA+F?e>1Kh2&;rTbzzcnn3 z9izkLL>|#+buOq&DdcBRkTk^Z3Ie15Fl6S^WwE6#u55%3>!xt_W!`q&(YhzqXCO;O z76S}m{rLtZvoUW@?Hn6Zhx7Q&{x!>)$xoitgqzgoKhj$tPObydbqnl$tx4fHWVyKu z1xk%tZ@g#oZ=^jV3Etm84D#V*w3HFPNg0}`)pLq%p(1oPj|JNX=M}i9(5!Dz=a`BM zuzWo+qL_nnS=Z1xYMX(k*ua*DtKe6WHi@-IL^zHrpY`*_m?FOawx6O8e_o=FW2IM9 zIsS=5j69Ss)QkVTJ7^^7ZT1Kd<9knQI6h&-VVb3z% z?n(53Y;CeHN5l`=-WMcukszF#mq$S!q@w)5{yhi@2skL$>gSC|RK?6Vw2U9%*N|21 z)CmF#2K*J56(!-l*^9!vQ-Y*y_dXx&21v`DeX7plr)?<)Es6JHjB6_03xfA&0VO*m zYr!<_eWW+>g)v5WWWxW+!48@AkU3xE4O$#9^M-~?JF`i)jLeG_A(|x|R9af3i-QlA z><<#$q-=X0r0)!kg?}IStMVGbkKya`#>5@1$$yfE7_P~BA9_yZfNa*A(eGTX0NVzS zpzv^@Y%EQgqslC)Qg}aecy%4h=6zYN%;@5ITP)Vuw7X|M*IMR$9><%pa&Y7VO(f8gcs`)J z$$PYmdt2@4XpL`kC10z7C+gML55?22pAe`HK=98chXoy68>G8uLwYVt;LmFOuW5(9o)RCPOR^cWqaL6Lwd?G0h#nu zdh8T#`vpCq`op*AvXuY+{SzJ9P!r@wKk0jfZweh~e1xkH3HzZPCfK_%8>A6{ZF(9B z#&9;jr)0Mkv*L0+j=^*XLo1YK4OLer5LEaAbQcBtkJ#q+S~oT$^*)^{RIG$ra%)v7 zdRX?ohTg{^i7B~t<|0t1f09u@K5zMh{xXO0^Q3Y+sEap%4zVAeC$kmtce^WZ@4PBw z#Oyn?3Qn@hznaWCqYr{B*Qp_C|= z6j`)OdEARDs$YNhq(*$tqHzxZXcS0rmaNZOO&YSQG)_g8Z$G8QU5S34_U48gm|WxR zjxwt2K-yOk=A@c5Vi4^VwmQIY-spT-JePt5PhlW*O-w@K5-F#0nZZ~0>l^Z6_iYJH z*Ck$nq8n;;yhay1)FQVR){3fPL%2*o{)DCur-dxK`~tj-1`4k(;*wt0+bxKC)gn_} zt-8_)q_^)|S987uFNXE6&R@Hwcla_)7Ggf z1Cg7+2kphBrJCcgUXy~S*uMU(uJDEZttBF6*?adSZ9eXw)`N6V%Y=#PIl(h|S=@ti zCuLsRdB5MEBnLC_CO-qAPz*TUcA-7tOe!F@9pS)(T)b zWx(DbRJ7rIyO4M2Lxiyv3*fA&2WP`fU!;c?DG;6YpnF}@KZ?I)-a;UZkH@{O~6)sM+T(#biEDD-M z$+Aj>BqYyi4!+BB@@#+CIp*iq9fMH*=fGng@ue#(NtNxwuj6U1GL1r{cd+8=L%q)k ziqJdDXIrYwz0=sG6hVb|d8byteCv=6=%?dA9OhkhE*YwXUPVM@LjPNBMQ{er^3wj` zb|K&NIt+wc{n3n}fO%bWSz@nvqETp=xD z=6b(GDS%&$@9BmQ;vjuFmTG}mj)#WVpwzUtu?NOuXK*ZHycMoe@H90_bhi`h4UIXu zRV-i8)b~;v7vUmkW*a{2OLkgVg|0K*)+o|j%dZ7WaKo&JR&t)(c`bmfhlu>$vF;^n zIY%?EBO`p4- zd}kJX)S|y`8Z(Ko9NpDS*&DsyO7Vru37X37jAv~vhgBu?&sccqqt&;{ofEQnY)YV> zQ9*M&bGk8^>s{Sai2jIupM=?{CO!(h+WJnrr`fA!KM%{fXh(DVs86(^=tYMfgxUwa zGZ6KN4~%2Ip>pt~4wb%Aywy1nS*rhi_zff#!)sqioMvVOsQ3cxA8U+EKR~aBB5Cdp zrP9vOL^vkjUf zXdqmI2&i>heqB5JhQ{vO zc+C741cb@y`@&@1s-^3ij6|~Ucyq)(Bsw!Hn1pT*fI()sDxYSfRjPxE6ht|8Q&Xhu}-(` zvaZlK#4lWzcZv6KH(7(JOuytA1ZnFy@)oMzxkGs`&decyrPMTWOy^%(uEw0?!%Ecu z3NHK^FEK&$c5yidl?Us*V|wUy!$`mcjK}ZaYb)6CF16e;Lgz(~J@WbT$x-bpTjp?<9OUJ_&i*Fz8E^yAQDq-Xne~A>spnC=wkR6K2QW0%uGMaDsI&s z(o!N8mI}fm#%I3VhCTKoctQALZ6wb;ca*<=Apaouf^fuh;MuO)mTqNx&U*EFhMV@) z?H7-xv_sEAn>SMk5D=4312RsLMG<<37Rb%dH2JJne^z?ruad?akT0$>hW9Vc$zEI@ zef&s~5wU6r9_(C>4Xs$Y50}mWW5m`-P%Mcn(Li{`hBeO7lohZriJS#XjuK)dl#yM2 z=lxB5v!ll$5%kZ#pW&MLL?6n*KA5@+^gS!kq)7z5_!pC)V80A1)VGZ7&gpm6hQ7Ia z(elRr8-C>glCjv}A7f2YU=1l`?18~tt^e^BNv0P!EqTB|Xa~b{r;81Jhl<&+K&X>> zbC3;kZ5y)W_l0Rikxq_fgdnjEI|=A0Ek~2Hx!gV6XCp^x^46&8Vr%t-qix*P{V}$U z{`P`1Mkxd{Kc}V?5CHjSh$nL$fVaQIB?(GPTh3tLr(?bMEG9-;E`}WQ?Qh#|!%qsw zN7k2&7xT6#*8n<7Jp3~dQWWGOEXrmb& zUd!+){-GLo9PYlm=b^Q`U%b+6mjAx)$1MRw8eHkr(P#GV>)g`5IgfnlpL-I~`L(s( zv$Myhu_7Y7x{UTq9*6$N;=^$aj+-}cCMY<0>>!}9@uM5$pr=J6N=dzZ__P$dXYW6F z;4?U={`q5x?aT>VmF}1$O70M5jn2(M%)v?tywAy<)w0z=)5kxT%N|>U(7w9*S^pP` z^z`%>c}6&?9mj+S+1V*Tp(P*4y9@XmR!>xk7$nBK(_MeAHK+b;xhVze{4X1hoSp9@ zWjvij6Y7+^FR5~MOiajwX^tsCCX1LGMERP+T8+)Q-lsOu9kuQ=0zW8-vQNJ&MX~&7 z@517k&da?$3u8{34bK-{Wq7+I#jyg?A0%OxYy`&s%?GeY$@%u6>bp)t*9whna}B2 zBq$eK-4vaOAM3neiM9nlokc1_0qxCzm661<2;H!Mr&mP4#94(MpfEXH*n!}F^Ha8UTpH&`SO7iM6L=P5Gz4>P?Ymp$qL1zMDj)3i z;hrO&Iy}xVw9K(N5VSSAm|ZoxepF<1mmxaZ#cENF-i&)&K-FKx2U}JV+8~}FA{7+k z@Bb3yi#AnK^k>#=)BhRh<&@Z(Dsx>6?O52_035?!DZ++zh9n%WzH3gu@wNgAIPHR0lcn|#TP0M{hefA@y8lm1putZYVCF<&^|PtfvP z&t^!T4h^gw`{AJ6^LqdG2uf*%H0X+T+v6+g1il=9I(pWV7MS5E0ydJoMB=5h;4#6p!74!ys>7Z&%fu z$+()gUeGoBglB7kUP&5h2+61t!6gi!Dw;L=Ces(WhXW-$%kqRCT@}oII2i?=+ z!cCiz~0i0_JN8uWQNO*#SpB@jlhJde;3_Sr)dE<*DXqpe#Uz*hUj zKm@36Wt+E}W#vO)ysl0V2hy+|#rgd+5}8$`%NxRoUHi9=gT!e6qQj!;?C0fD3#)Gt zPQclq0R5&jC@6kA>Gx47ykHC$P)CUDh>$w)osKJ9g+Q3ZgKA8_a3UmQ)RuU>axLx- zj4+a`s;oDrL$kW=jkP}Eed(<;N# zzY83TDxFpQK(ZIRFjDkUrjv1fZY4wKZD2B0Rj%EQ#8XKVBig16FaM;6BR|H(a(y;DlPi8aZzVgD1{l9 zM3epW(>Xt)TMyHW*io+>nMEWW9uinc$nn##3Q&CTWwh!lR!B2^>h5)_)7bXtz`1<2 zo-!C_S~X0Dh~THcX}}>Q+?9f46s$gplmH{+nGm5ZweRSlmK!`^Rv&+~aTWB`S2VM$ zj$Gv4akE(60>6c)=tzaCL9`GM4riU7gBG@NzhM@IF8-cOIRO&IzB$`v@`Cv zdkEsoTBLhNBB{zubs-)b^o(yuX(I1V5!BUXH8wdnb&gi-J20!gn(89w;-c!Ws=~*^ z(*l7;vGg}|ApO}Q#{#xiQZBH5Oz*Uu{=}!Wj0N|!EjUY>jZ+H1LA`WY|Iu^@emLmx z{GTDp&LI~8z+0tPL8Jtf6B<@OwHir5u9gopydD!#g-9Is3*$)arOco2=36zqK^y&b z2n`h$o;==`8h-Xkd1v;P8Seaxp$>c6?JK-uK0BFbF5mO_o;u%xcj32$7B)fWkJt?% zJRtl-WKwW(Zcb0FkQt96s95v*rvaR-Et=V7kP_rJFU1CHM^s<|RVqwOd+4B)g3mxO ze|7b|E5vqbf}_u&>jtW|&+ANH@!f=$e(^Gf!6gTPcyB2^6aX3@ zZK%w-^XZZxDh-x;=bFngj9pUj$VY?bF*uQ@q|ykfQ*YI8I59q(@^GaB8W=$E#u*C| zsu^gY)dQ>mSh0Bjxs}`eLuWxq>@0L7K;JIPi(dlUT#+!W1zMa#)>1{B%| zdF~0Mtq$j@o2BvhJP(%g3{x;qBGB;b?-6mTlZY8W6Ct(VdxhgzgR-F@Mv-2x;1ml`W=ArmUL{x?piqF4V`V-J<6_Ra z{ZFtX{d*QJD0VNP8s*FZ#s15Aza4Em+VO3DveUz1kGFlb<-9opE-WtY7E# zC~xZ2F5xif*IuA2kb42~`}q1ip!@9j`wN}3%FspA(2gho-0y@)=* zcf!W50;)T^P=uVPzQ3P4Tf+Xtl3-O-@{qKMU} z#3dpM3JbXb|HKY&jk@VKc6^?&(=h3TS}Z5`T#-+ZVCss^V<pO+d@*XzSVzwq4-AwMThd}vT=Lq>e{iV|CboX%hM8={DJrN($;aW$_Ia*YO2&Z z&C0}>Chk~-iOSWz^;&+W%>T@$m+gllWk>Yl{I%&CgJ%#^>Q;qZhZSw@333Dl=!E8d$%QzN8mC+B|7MNPv-;L(}>;M>hGD?bCPeBor?d4vYGY zcYe6T9?ALSXFM0yLYRD8(#MZQ)U|NWikC}U2t%I!GY~QUaAQC2pI`JhCcX}kgW~;n z?YGxhRkOILW{u}o2iwi^zmLQ(iD{Up8VrESZ0&{Uu~?Dr{SJGe*&7nb5mjja2#~a% z1dXj!Pmx9Ah7W#v!!3FDFXtzA1xCCWR7l~$OWn3;l8G|JDh&0>=d zGcCn(nfb^KR+z-0U-`cEu=5J<_VuJPg(bC1XvlXm6tRwfo=a@j5*LR7n`AtDI7>uG z2HUTqtbExx^}`3zq|Q?(>PiVG${+>m=DO&#iOt%#ZlM9iJSj_=iQRo~j2mYPHPvlBj7X{inII+kJhS zAN5zd!LQHf+l`709JP41xHnGmzqy}0uS|s0^?@yo4?zH}z06r}u?N9pjs+L29OUsF z%5Bag+_E)buta_oq(>1Fj0P@zya91}e4obl7Yk^;gc3dOz4cgX1PgVw9m37d#k7}*s*PaU4-=vE)i#h#Ao)mW*ZH&@Oq;pAJbBAF}nF%Mh47m=;{Ko zE-mh_U+uu@`M<>7>Td^~nCvp)+s@EExA<->hJScGhzU9hk@dZ~#;(0D44mRjjhTbs z?O*|hOP3#+wf`7DsOe0FR%7Vt;2VTymXgahw8qAgT?aKwLm4QPs@B^VS7a3rxtG&hv~kv&``jE^8*B&W9Bv+^`@oEYD7NZd}R zjgW;H{MoXFY}JwD$XmbDG%?eD#_kc+DPQ_*^3_y-lX{tOe2A_!2pA{`(XybtvbpKu zbMLX7qoYiA`zpafi3~$E0oJpGOb_3S(^B*>lj26ddxzP4QkTO+JnVf>E-tRw`T6tA z&3`tM^&u_#etv$4-dkx56`##1hOnm2aOumDHy__aAzY!gocVc^ezUfcS1=Xo+O=zF zf&Q$gF)=rH>Up>)w}x!3n_2jf<^c7q!PWigX$!Z#Cl`R^h;h>2?RIn^0qsT>m&EYN zD@LC`>C&zP>Imx+%L{*84o6#u98hW zJuNH+P8j6m$l8S*H@yy{9EA!qUG;gu!KUcE^Fain?-4jom4iw^H%Y64L&ndQ&8h(#zk!e7`Gt`;fc$_eLlW%8-NR zzV+Nhc~4+PfJj-9ySNy!6O`iNqgvRQP3%hT$iO)BpxcFy(ODKbd8pi}%4>~r@tid< z4CbMpjAjINO=7DA9isL^J$xDkk=coys=t3)RbnFDEdg$BQkeSuYgWvx9e?cmED!)Z zf0|IM(u|uD|03@tHEz9{Q*oC5{<9wywY6D|uU@_^nyiS)Kwg@;v7z?!n`|obfkHvY zE5OU|q0hCD8)MNy@3^~L0TOGj;M1icR~!U>&ZQdeY;IIE!a4^BpT$w*660I|eZeUU8-F+$+&v^kdXqZ%`tTu^{rVP=)7)hM zNGCNob{_}*CA!DA)};NM2TGOKvY#x%l#H6MZ|BsY6Xn8%@kn-h;SO5lZvE|rC*b@Z zsrG8m zoAen;PF&BpZ=k{lKPC*SU=4$;?;kar?>9W`u zQk(LujPo1wJa#b~QGd!-nf4_?d=p8TAqzT-bzMx1*dGeuAMSJ(&;O=8Y_Ce4nCR_$ z&1Nj^kzytlneT}GNdHg=(X5p*x%6c(emS2h?E8_93#ULo`$`9G(8Kjf@7x==FajPe z3he|K-=LtNKy)sePj4ent~1SMYw_VN0Yv2&ad*3j#P7-ZB058;RfvO&yB}*h-4^@1 z;{(LS=Y4E=IO%)oS(`+wn-N*b$pCLI&zwwSB-7D+`uzDTrkxgR%NO$QFp({X{p;dZ?;CiQ5hDlWMF>GXkv#JmJMw=yDSDaz^ilQvWh240 zcd^Ve=FnxQoXJi?b$?Mmc9UyBpf6{p0xjj!{BLA~KbtfN%xtJd69-;kpTDo#$R|O) zZj^UaRHm3q(FBQ~Z5Hl<7Vv ze$oB#oDt@5oGxW4m!-n4bsi?z=8ADkf<|O5bw-XFwf9ktWSg6t-VSbVZov3PqPF>* zl9CI+?DjS+Dfy+eY&q+_& zv!|!W+wk`q%yrW~BN$lgnrm%vZEGpxwzV~-iK#&6z!P6NhFgz5P^72pfp2&;>}Jbt z)GBkvt`UuhDCySS)@M6Alk5EzWfZCR3f})FLr?+A(mnWDDn8Jq+!7z}{~&`S*XSrc6z1g_tA7 zfwtxJTXP96z#2h3HpGINMwD+TFueBC+nmVATWK*1eD@i!}icKR)XeIhzp1_zn z^uHqA#Jm>$Pm{lrVk~}QI&|IBxdkb>L0GuqSN%kVU*NQyZo2YH21R+@X3G*#y}dmgAX)a$C{Yzj zDExeV>1g#9g*PJn{aM|YmX;2~eZa+E0vE6O0OYb$QY3TDM$4Ul*>b1u?(S-n0-W&V zloTkMl@3G2F9_N~FsSOU&DX_y>Hf)tSUxQjUX!isILlc7vN6&6X3(0!^{tCKcz%~T# z>>n@VH#gMGowmz8I&QD|8}9(5Ffp?3LY`SD>Nf#|+H-U>-STvaqGv*7p-|j9kxmQU z8m&0BiKp=Dr%5lZ5ny}W#-q>o(}XBGU7QEy|7 z${&6UzPCPmEu2hH^5tD3Ld-!mmMzavxVQ&ScosA?!5!?CEdE&Z%CvPZu& zf~-oQF_+q^OCElZP_L+;n1nXx)&_M>mT#wZd2ot%Jg6aIw^vmtiWBd+WRpJB`wG9u zuBj16X=0&79c&dk6;9`stjbF54i%=i9ltur;O|HanV8VOrID;iD} z&`sNS9uVqxOohV+`L<(Cd{lOs&&+^TzLZ3Ol9pf3QE0`hy{`JwF zidN2?9gWElqZ4N+iNAk+2b~JPGcEb`om_sdQ>G0*V)Qe7&yCX%&$v`v%qxAl{6w>+ zbJhy+jjjhftFM|K{ZhCW8T`wrz6i>sPJeo-;4|SrW@f~*R8BL(wkM9?*U-RySx2W9 z9vw}A*&$wLydf!Gc@~Z-t5DLl|1xKpXAUyu|9(`FAIqL7XIb-Z zX0X#B?=+aRwfm4w^T-a~2TX-CSek^!SVn^QOsGkmrxx_!$OS7-R%)_v)Dl0%_cQKp zI;-@(5?dXq359}=>epw%}z&DkpNyyF>-wqC;6oXVf$X34#&Uc2Bu1i@6XS)61CN4+(`um z4|16itO6O)KuCy87Q-IAaN&YNhSbq90sQ+9noJ53laj=92lmx}`rqb( z&S`2ll^&AW8~%;$!ddgr@f@b4PQx?}-)qUQ?w~Tr<&$3C^VvSNYcsm4Ev6KM(GI&IzBju`f^9PsnuPgBzn4BiyC1r0zx>Ree=0o?vdu>c^)# ze#oF-5>XEV7n=5iUgu4$G4sioBS6~UD>MDEVkM}1p2d3h_Z2%J#`qRugeV}`P*zn< zDlJt%*2~?$kJ}%=a}aj=zff*Mr>ePfRhqSK?aev407U~?`>UXB{S-f-G3T}Sk79nl zQ>zpyKh6a`|3?&s9V_Ij(s55tPC`vh?bl|~#NNAlf|D?uqbtosrZ;uq<05@-+9%(q z&^`fWU0hrMx22=-@t#tY;2{xrm_21SV5^H8f^8=);Q>1^H54VWSN4a~B#vbYE*F-cZlo`Bn8 z*uN`Pew#Hd!;8a!&U-z=%7a#G4J?&NB$9`Jl;qIy1si41Ed8IcAm*Cr=@V(hzZoz) zKE6@2>c9cs-O$@z=(`ljQg>8^-*9*s!`e6$~(4XCbL1z7((GAjXQu zAc=38S58i<$v{VOUvOp5Po1@{r-`;PMA2a8_|x{xeVfsl*#3Rx%}N!$B`oP&%2NTp z#Y(5HDV#Snxq0VFX8$)QhOIyYzBb0VfF<60%dwLS%m(pM@0{8$JK9)I+}x5gyUJsG zGr+NlyFiiS_RG80gsRwh-j^s)0j!x1Dpu@|-=CT{e54~2N55Ecn-Pc_)p8smX>H|BI;CsC?{VOiGFoV^ zkzZ7lijARwAt@y_nR7IsGvPY6KPqtKA{-ym>+8}^_L0@O(&~q^1*NWeR-Jv{FqfHy zDVHN2>i1Xfg-Go`K1Zpc=-{hgCg3is5@e^y+T86Q8Ynhd$@`Qnyhg}gWwL^* zCYZ7%6i^p*sCr#k)ZLJ3xfsY(Z!|$^8C=)YInW*Yoob|G^c=ymMbL;jUV2h>3Ph?L z_O`aRb`A~`wi9jTMlxI7)YMcNH2}9Bb2D*lk1Bq;jnA{Q zVdOgW>TA~zbDhiMw(a|KW$&K8{f73(yYRVxOI> zGgH$AL!ovu#SB56;=Pt;RAy{XTm-u#-a)l`ixxcCUoSJP&GbLtpfVqmG0aQ=#o^*j zl_rczOC#Y40v*cWDo0=Y3*_6@A%ez1J|A z+K?!?L}dDnk$ZYlyliMfs7{j7PZaOkTbZcRjEw8|&!bRIL*uo6H|JTdy2|IIqFf?w0CnkOGI-W zi^jV|OIZWA?Hc8c2!@G+H$j+clta-A6CG{g#fGLz-2=R*Ozx#Cf9=-N(h7Ku0y;Mq z78b|m(cwOQyw2_lCQ20Nt;`jrzP>&w4GoP%&T5ywv9b7c$CXhk*R91n$aNk=Hg)u) zq3o|akXs1S;gmW%HDnW0O<}SwTDG?9B%UMdJ|CubYdDde4eE(TcDDRoPh1%oh+nV% z+01bKKi*j3G0aR)9=l|ne>=C^ecy}XL6pY=KO9i9#1acbvSy7eOi{GPc?;_bEm`{# z&Ed){K8vA2VkxGsPMv7~uE|g726?xZOtM16wOI11I>`B8U++ePsd-7pX5=z5G6JU- zRjt9mz`#0WT$+=^hU}~}97rWH9L#Nnr=~XZ-RZo}YmZVKXFF*7h*8}6KHEXy`OjZL zQxkyezRKgd&a=8R8PYI3Jp5W`?Ay1GX_JCFjp}P|dg{}LsNQ;$mgdMZtZXB$ne0FB zCO5zQ(4VK5xpUEt>u9Q9IwneOZ=$Js+Qvwq6?uOU>mGYQUZ=z--LbG+N|=th2Rqkg zjWExEgj92S-ljxzgXEpQX97Ex)Z>Y!gaGo3v3ICm;w2U((c;?|NXWZB3le`{{}!%i zZ1WHOhG42+YYo20cfQADQ&CGthXxo~vXN*0W{5`?R#w`(V`F2X)KhdBF-c0Pb?~+uZAR0&Pr>K%v(@t&LCzO!`>OS`dWX6G*vD$g#|3tO zyz#n{rjoJxBP_|f2ED$TE|xqtfGwUPNNOuTr=PB8b63;MGX0b8)oP}}QN_ohU0;xu zI&ex_N*TmZzYDl3OSEcY$p4$F=e0*ubQ9BJ2E$`L=gS5u6dTY;pNvmG7%H+1*>jn+ zLZLozAs%OYX^0FDud?@)?j&PGb5yAU;|FZ?Oj9_`mHQ(rMlRbUjyl~vJy#IZv;w!A zU0677ZgXo33|Q`0BO{}Uoa4QmNsX~R73GX(*TkIUSeBURH1 z%m_4<&TN&Cp}yBzV1VZ+DfOLPSmQ)^F0jSK2j)H02(U6V-z+J%)vz5dsNmzLG_}T%TwHwE zwp|FD2(5aa^5VsoZXYq)7j^p4H&7YI;DdyZciS6$aLH7r+*nweJu64oeKc{~$$Gih zIZzp<$uovvgZuQkrvGx5Ij*Pt?<{{?9R~wPX^?J!TIlY(7gV=3T{^pDtQoq#^u*Gi ztWBesNVj0PWAqh2`pWxHd91?uWh}xSBNT9BSBFaic=`C>@sn~H$-BCjJG#m)vT;gk z2}khbTLsA^E2b8DAh;cH9<6$i99b?vM_}GEgVWQ~VM{+4epK$J$CNVTUSIi;YueU&s5hn9)zx;7zbPjHxB_PPzDbY|@S2>7*h=R7AEL#K& zl}evXD`Kd^VUz#5xkQA2u2_|wi-*S3&Wx8~hB@psbNiGg*U?VD)j`XV7|2FP*k`xZ zmx2pT5)`Tun?vn9Wx2aMYnGoj=aQF{EbL(??|RU<-0M`pL6xH!`C^j(CTYp&DI3Dh zq0Vzf7UnPB;RZDy9pI!2PfH_9_M?z2mkk=z5h&YKG8+LY_TH4x- zT;S~9r_dEbqK?!!7p5)&tM8&_Bc)vT=}>BvGwtyqd3kwqIYmW0cgej-IE96&b@skR zOw>Wqs9@Xp$&-f#wo*=&RyWt(#`lc0v2-^xyUPPhOm&kpr~a>97iD>uqcXYPAT@C? zr8&=7jb&|N!z%3Qc3Ifg;PI4giD7I;Y7*_&nN$_V3n&pQaaZyAGn&s*O5KJF3lP_I z5%Jy?FO^om`lORbM^@~;e5hKgu`t=oa(6Y3X1I)$BGRo?nwCmL_pW%YQl(lGk-hY65gseV#bDLn2Y7qm+*Fr9UsdM zmwmd@Cf`?c> zLn7zVJOindtptZmg*aqK2M0-EOQOfRt8s2?O>z~cec8q1zLbcmE4aza8=3Cv;zIYB z-cUD|f|@$GYA>Ve#nY!xxtyz}r>D7`<7uXMYl0fqeJH~ai=B7=#A5g4D*h#WrhFTK z{l{VVa$Mc}Y3G)~8Oo29d&@n8lhAdLQs46NBv-saI}Y<1oa!Gc`kxI=8VaZPb~={# zoOD%9W1=Eos$TE1&gY;KC;F70WKA-{vT3c9z;40HS>G1U|IYCyID=H(>5i>CMN1(+b%qDY?@UNJ+DSZps7(c$DV9!%3HYH zf2^+m@7sDCIg~!1L%Z3%@Ro6((XY}__n(IyG<9)*Y@lWKmtXyaFCtszc z47BFSGrcug$+WUIJA2e=XohZN&!j(h*&s#d+L2}it-Z?5wQyj#?09xha9=legrga-Ee>NT(_4co}hF$m6_hk zX+bJ+1+Dj;O!9f;!lpQmxo_Xrifx@h>)*jTeflP16U>GRS?*JKB}e^k^r&4I^CDwo zwbr*S!J@5iYchko#%d?i2ZLtF^7tg3P*Kre221+urAz;tJk=j>Q15#MA4dCs?u>%X zzpHO$$FK+S)zNaVsmozdY|v_rGcGGe<{MazlGLTA^`%J6nV8cLun>KhEi8HZ8v*5u z%os5B#R(PT-$H^VDV5M4BG(xvIo5404G|@yp}hRzCd!TU^^x(t8$;!yJwTSqOxkq{ ztvA*ZMo-+&s;szX{+GyP@B-qhD=I3Msn8Z`20lHWY(G9|pM+~69iEV!d}Y9u&1VL; zFL(y`VIzc`U)9@A>^2LFbE^VMoY8B;EWHK7O+~33|=H8gtKkEw*M|5hB`jTmdI95HZ-GI>dRJmX+Io( z9=BuL*n5<|C{=b&&Nu0fi~WT5_Sx`!Kd^IfoKL+M`*j1_WnLi&K=%I-fKm=2A*y!a zeX|tDJ}t-FfU6}!2vZnLBNB($v2Ul))V!hpX6JD1_%JduQaU=Rq=av7U*|e_>$3oN zkRu+u&1M=2$F4Cy{y$f(64>6>Ko%VGoJ;m;fl82nI}J@hynsSr@7}>{?IE>@mvLLZ zyxbh0_{UM)p>e^!#+l8@!ME_{HFFahAuDOS*HYi{#q4ZQ)#l_P!KA_n2>i*($s4 zySM4+rZalT_+A!;7P8P_)ibJ1;z;w^`z(d2f%;hJKLU% zq5Zv@G_gUujT+r#YF7Z`v4*h#f5vOvWw6;W%0?~4@q6`C#J1TZ7k;F(lW=o0oB=)C zOjq|3t9E(h9M$h9z!iy9k;t4|2CLU6OQWcMY~R(B4FQETxA+d#O`bftvo&gJPA{a> zuN@z<#pS?1nn6}!2Y#390}M(S&DoAnuFCdUHZe)Rj2T92$AWy*NP>J zn(xe08gvc~oTI|Cf9Y`-oSa=j_w6V879?S4W@g~EI|qt)Oxc)Kgd<++D`^qGo1dt6 z+}VVKMP%F4l0rB6X`Nk2H7I@8M6UgT!Sw!dY+O!#_x7;>56%4`6J`3*z579pKj>^$ zpV5@0HA`_$+jc5PysUR)rC6}({CRlH?S5R9h0BfmB)z!~$|DB^o?8u3F#_u}LBT00 zJ;!($+y%BXZFR$%TMNbBu1f_iGpW07!WN}-DbEAIX8^#8_zW-?y>E|> zj!uH2*||9cjp-dl4%gPorQEp1kusj6PLVw+82Te$&r`MA3c1t6^|MfkGChhbFcK2h ziERGSyRm%+JSOv4!3kht(p2!*EB3cj_CYyIW%A`!DFc)2Xnq{P@a0+`z8R6p*Oq!uuJRP`2$Woa21&1N;)>B>+kq}OB8y~&86 zZSXk^$3%RF?k7s*LVyTfBT8FGM<pj4lO1JOfVDDH#MZjK$CLl$sqX>?I zs3@S6AV`raAT>bn+CT(!RC*B*6zL+psz6jq1}UK?2m(@LXd$%x_j?eHzwh&T?)Bo# z+&N{xd#}CL+WQnFP~HRu1xXRaj65ZQDO5Ov;kx=*t56PHcWNZ@S1*oxA%O+FiLlce zZi!(|;UB2EaMoU06yGVrc>I-d@@RN?xX$=YX~k9S9!#-&82IxZzHxL-r@nTt)&g*>-oF*6 zGX}pyPPVKi(cU!tuM$V;)TU_Ws%Pj{)k6aSLP*_KK0d$N+S=dgG!xR!;>C-9zXaf< zj6FOQ>h#cySf23mWmjHrROQ%k;b;vswf~^XLjCyhRD?27scrzv21ZVAv$HE`Q+h-p zcpc>OD3Cn66+QU5gzrK(WR>|8BRkyF>p64hZceh}N_a?QAc?x;4F6)I^8y4%;r-^l zyiS7?JyYK;6ZN`PaC~Q6&Uji|rE2mowmgeo15|bFcD8T-Tn`98!)c%q%VSm0+{Ru= z{N2sd*sVkn=U5nHzK&n+DIlg7;;63+4-1RMr~vPVENJ>B^P;EoCVitPeP5;F zo6SWKlQHH|T7eq)z90!Pli}^Unn4mA5p|yPZRlDd@V65hXryWVkxKj(>_B zZ6)qi@a8<97SRjao%7iZC;dQQrO>>x6KZ_*i*pCB^#h-RVC2(fO=~!aH1F!+A74L& zBHr8Ks0)>=Dfc&rmPLoWg(Xz_+%#Pi4$T(~l>g+<{NWB=$7t%3{Y6rfskUFkVUds2iu!Ex#WJSDzi$GTox*6E>Uz}&`Z{<|E1So*|zQO8ok!GwuZhI zrTa}FoESXvSU9Eq5l$O)|M}HEH61a`KfJ{*fDCtXLxZ|C+&@@UU8KKBFXu>#!**}~ zuZ=0G!9Q&K`}BA$T#+6MP?ZYH(i2rABbB8 zXTPh(#l&3wr`HDh!%rcl`%!0T|MS+!f7}|in}=55yoesv}>Ho zLZ5IUu^#E2D@)uX5#bV!LJig2ssSVCoV2--^`STP8EVx*Py{tO2BgrN>)tFE~Pm}AfDj_hmMg) z_7qiOebQ6~nX^E^B_6*xcRTskV*O_0)F&X9_+QK5Am0WqD>*H^A)I%^1mz_u&VBp? zuJNv=io#saQ$J#~92co4LERJL$m3I*}~hhmKf3 zh$k?$I)L4lSkc&p2n9EZIzq>-<*}KWqp#Dddq?embiA7cBE4MJ^@q_nhrG&w>sD5J z8OfzRkTN>jMVv9l<*~3cM^mm?`SO>fiAxWIJmCvOA!sDYul79ab9J=tc}*&ExnKPp zNBu;|DNuTEyxFf@-5~KIAbaVzQ2&7&2{m+aae=drv)0bc%(Nf-4$u3iU@4FIQ5k7z zRS+b@4jNTEW?9BkeM{1YU@h*}jL9~c;Dzr8%D8X7Ely(%oN@3}I_k=ETWae@TPM9}qqCX6jKS4Rb} z{)C#3-41kjb$pEYl>tX_B|U==*Kk=WDN4N4tNjyANM@ZS?H{KWSkYJa(u6%X zn*r(Tna?U~V<1>Wb3!m&q?#SVfs4lf`s0?5N-JzSpk}TGIdaPZRdeu7gbE301(VAS zTEi#@HjLGeg{$o>8 zfZxOE%CDxgit9YD8jZ?1d^@F|dt+Cf{LSy@;DJ19{Pyh{DueFx3`=q>Tr}2AxPUZE zg3KT6LldL&1W?{d@|t^?U?)l44@MzNv1i>~;t{TzW6-#-CabYHNL}p@E@EfPLw)m5R&0RZ zC#mIq@M0{I{Xl54db;9ycDugCLr|>h#f=OOZC(}X{?#lCR|##+FMi%y8|b?S85x7i z^nWT(joR~l!^*|s6bKmDF4h%Fon_Jl^CE0YxSApmM+>2a7X-kIj#&W6R%NwufRd)^ zCCb@0tw9hG&w!S9ZG`}Xiw}Qi%ps*V#&I^DLN=*a{OzIy08txhD0sOG_q=Wp5^#K# zWa|F~SD!s?z zzo~1-*>5Jl)e%)c>E^Bk{if9UsHi}?VMuB-72fY#~AVktwJN^U6ARk)a>SP zTb?X&&I8eWm=2o)tz1Yn*;Xa*N7Ui;ql&X*b)f?%JUu;kD7YSmPV>|Ncw<88Rz0Jv z+_|ZNP5T$=UPgCWZRaIf*P3ar313ed0`*!LOch3%1ua+(mb~)E;r*BM@b2SJR41_# zQQSAyvp(g?UI;Dj6bDdU+kT}))fZ=3nc6t@{W=I4iyPyQ$2>|DdUiT?Z>cKC4-(^z zDSB}k8RAHFm5u{H6DGkl6%kO~NzX>OEft_bc6K(X&SBz+!d4hnPjKmtbWsIo6okqn zDx-@eIqpO;0Lw+x$wkruX&&7|;*ye2`V>b#?s;6Z_n-R+|irDoal$<^%bAIWtu zXL3D_vih~_yCI(Tf-G@9MVmHGaRi$Wt>Mi*xD`@NDi5D&snlQaM%-`SJOaTBMl0zE zPMImZ0U-hK;W-Mf^wx+&N$7Y?2kq~jhOl6nv)Qb__wL<0_4av;_8F`Z4}7R0H%5BHI9S~`I%hPe?k0V?QykLZ!(_eK^gOTRnXoyk|{igEPCVe19#y}r)M+x4V zR;&ne_bEGfcR8xZX!S3-TN)VWK&2^*1WH@m4nfvOHF)y5&ON5y)fVFO=fL`pb8zyH zb1;G)`x5ULQ9JTSoDPG5Y&!o+BbfDUWZ!1)14B;#)^uAbklX-qBtY{EcuD$SNZG); z{BCM2P*Wo#B?aG3ApoW4ouwlhEQuT`o0h!VhWrSeMeQ8l*+{u9z$FLwL+L`6vXB)< z-3rr#nZb$$jIZ28Swsx}-&+{AdE*EzL-++(|RBshlDH7It`hVP@B*pn7{}; z9zWE)!1#G|74pkvfC~|Az4~Mw3aPDb|(&a8&Wd1(6Ha zq=yQdA(&X&$(j8ZYgYau@A!@ru@?~bri|D-BDU0fwU*!rQ$L(C9%^R{(m_5oyaee$V*+lOgb2j2c{*1r7h+o%S#9s-iEm1 zVMG-zF*1cOGfW3t}st-2-9iiCffFIk1%e?0SBiGmBT-KafDom`&u z48#YV<$L%YRE2_0@f_>x4}fKan!D2d!&p($RoUG_qE8#$gvNVFdG}2g%Xx^6>138wafok@ZX#o>w}N)nM)S6 z!dJqj)-|n1cL2o}72QG`&cM$S6m=bt2r>cH;!xbfpf|qFOb<>M`)2cXkP~UEyyPs)Xp-_A`1~ zc_e?+T;~QjYq2n?%|j|m+3xRy-hvbFS)W3}OGY-v$~a4xMLF~R!^KwzevJ%}=bhFx z9bR?!c!DS}6aT~Uede%~c~_f3F9>$QfBX26Nntzgl*w051T5!mC!aj0TWJAMp3gY_ zu4k{-VpAQ55VXc&Xzj8Bt2W8tsTBfCW`I?K_LMFff{T0;Ppi) z`txD{vhSj57QX4g*|KlfocE*_7Z<}>oj(uq^74D4b2e|VRs}!q&yKLoUJc07{m?&T zE%4c3%1)O6yEf*wNXP-mp39z_hGeA65Bb#J8q23#&@OPa30KT6t^ai}3<$%6D*Y>P zQhj-jIweYml*aWSK2mnBEp2JWTRcP-aySte024WZvkMH=D%ZXJJE}q@L**d#)VUKvGGJgAc&Qc`2?m3|KzSrT|zG zk{)IR`1iGTQY{#Y``T~TX@C(t05ausXIRc1E-oMLxoq?vpDnPLMl3*zf|^P!@Z&;K zE{9eJy#-J77OrKWs>VeH#^kqu&GI~FRkIq8bbOMy2Uq{LT>n6h429I*8jfBx|v9j6JqWQASMqQntrf-TkoAGt!|j6K3RyFQg_attmBG$ zbu|fGRKUf(x-Nzfd61aLN)lkk7^Ku*v9otG^0Z1JmoIKlL2+~mxNE@HR$M``7ymNt zM?4%7qpUF=j${7f%b{O(*u4#ynR~cx8;B$t05>cvIZ6Txd4^HJMlR@>reaS5WFjjP zWTFki7~N6YAi_a+Ja6PA+;e%vT1I9=jB1l}aYvE@*AsxuLY* z?S_lq5eh*?K)dpN^>tob!`11!Kq(=jnu5$iFvbyH{S(Fc>DS!5gR@pzO=s}w#6Iw3v7>EYnMon-&KCrO>vuw}L7nF{GRj?u$hAGt3ii#AK#$vd^>YJVo1p%>Y zs30JgjU#6K8z>9zAaAEWu&qJJB(F}>*KHdgnE=aLL6+CEI%w_&TKc=-XY|Om(a}W_ zzeJZVs-L?a(#dc7Vn5I`=ol0&UcL@-QGM!X&sNl5oS6QIW?#1M48@r0#zvkTnil~; z#yP#S(Mq1p%p*VfwJ7-Ox!`t6U}gk*3WtR$IIgr7fRG=zK7@rlk2xoimh}gOzTjU| zMe6ui)xf=EXFy<8+4riMOMw6y7;%Ia3OxMJpHG&N((D2u80A)3TYz>3(*npsi$_H{ zB_*YnI&1|NP;DA|0k+q#`zjJ9u$zE}q0$}FIj|Xt4<9}(J0n)~w6C2&FFKF6i?fB- zfME=3o})P=1mvY&gZdh9%A31W@I=oxdlElXf)^m7=>orwcy`3MbJVjlf7G)E=;u#L zIj!(^irv0}Rv?@8%A^2%lSQrJIx_1^R~3ML_GV7YL2hlmpIX}MOZZT(GcpJm6%-Wk zX4!+aP?&;Eaex^6Z8S(9XEchP0EmkI0nfCh(obGSd&%|m(x^8xQd32LlAZNI9DTtE ze7O zr`XB8U{?z#*8)!NedjEo(Zi1;;S3wb}NQFm{ex5jPp>-GiXKJQ5;=M#NsH( z><|Z<&f3X|i!MZjNBDm!1Kf`r_zaeyi0BA|`i$yv*5pUq@UvV4AMOJ(ax5ZsUKP1{ z^QMYY^z3x@jZJ#q<(RDhbBIRzi4NXV>UBduMkYg12u5VG)-Rv>DX3wFKEA%b>h%9& zP9V`{=qPaI>YAG7?(-^+e)uol38eMOk7`Q)qJ9d9|KtN_LJ>ice)9|mLGfy-w(&3j zYU48?Bu?{l85y{nuB4D#v!%<6YwlfAU!3kSEyvfHjUD9#=ut&mU%=X57&uGT10FKi z#vAi2qBQggBUXw%r(ht&2u;XwsBQ;x zD5AolPFwo9ZF}B0FXoqK>2b(@$1FKHIi92=;=E{%2dIho&to3om(?3%@cTP4G6q_- z#W$2@O*c8&)YhI0hB_<|3AV;30Ko+7xC#J zTLnrxur=Esiy`EcL}z%x88SMaMpT56ABiG00k^#)ZfDjnp7DlT2>Nvk{l8oLW2X~t zFC1cTr!EUqIcSTWh8p%6s@cJ$5d@C*A$YINl;-uI8l8sDARe(Qytc-xZ%8599$^49 zK_>kK^^)^0{>-mCap84URq7A@{Y~Am%Z5|%0AP$OI!3Q71M%ggMo- zt|JSG9a!+?cgxstqyrvert87_HpJN z<29OO@Dc&S_japX>l3w5V*S<4q#?gCcxo)Ht<^Uz)0G5ufbr#w7z`_N3p|8tMG#b)LqiA} z?heKWKqGO@9hlKlQ?duw515NNCX$DF&v2D4A{=kv&>wH$VH~u@8*tKhDEETB6pL)E zwA$*gYsu|Q376yR`k8sO5pEkBy9w|B4NuCW0fxN1!S|4TusOB#%WIJD0L+4AK-8h$ zaty&yLsuGaQwW+YteR~G_h2rNqrfA2w`iqkqYh@l19BZXMUY=SK}VRUYnDzV0*V{B z4}s4!@2d2EX3u#8Rw94A2;GC+My7@CYbkZm0Y-nn9vHAJFly2Dn@7HNJy9Hjta(B0 zmPQJwWN%9DBk#vnqxNmc`vJ`&Y+?(WPdf|c7gEVNJ|1eF2qpttfum)Alv@u})d2Zn zAK(|UM;{48FxWAjCw7Crhtp=v0BXSFR~$PB8-+pSN%sl~=DMJV`G5cK^;c`r7f$Z6 z>hP$f3Sfbte(}BU_^1xPz2KSr#wibv*ExHFit7)Rj=(UPRuO;g1vFsEdzQwv=1aIF$gilh7tjZCJJLl?b6SAoc6`6Yj27mu$VpA2fO+|Z#|@o^RRP>OxIXE6xhzmA&Td+w>+*-yqZ6}wfD6Hy zPDPly{@z{YO}}zuppROhEyJ2GbXCs7ZK2>Ws88j9JzqTf&fp&au0G=g6W7Qw0Tkf- zi?{(8f7^Mt8&hL+k5+ONUR@#1%%9rRl)f|)r~SntHHqnxr9zex^;ngDQE_Ok{i~#X zPHQ^u3%4aZe>Ka-(an&~4#M047zHUcL(O6$NbJ@hSCt~3abcisbz0HHl|QOf0Wd_E zNn#!-2vd{Ninpr;?D#YWXorb8(cxLs;8|N;&%+X?cF@3I3s3h?@x|~<&*<8zy=`nQ+oB5-4|WE2Bhvo&!rt7-Y`GiWyr}RA3>j_Z=dT`vvj@?DGm?a# z{Iy=5HFXDG1XUm#F!cI?oqNFn?Ry~FLqdxV7y)4*^e$47Yoe)1H-R!g-LShFdKUDJ zP4zT@b6(%ny!Jw~K8~nE@bY8S;b#YfUzP?-1I{*AXj@d2pP$xfq6yCWAn(f_lWXq7 zVR#z#QJ~;JpYA&qT=4WoO|`c&uHX1At3iXAXR-1z77MO;e!u_<)@{tS-_&HGbsZ;N z1gr#v31YE|ODoR*^{lfg!z7dnWW(;9himP)jN-N<&ptE+N;=Cdsal$fTX;qFKaMF) zWCYF&yEwkvz@LiPjd`g=yM6?(L+!$!tQx?*v`6?Ek%QGbHQK@=4u_peE}{Uf!! zgY*v{4HaI%WtM7aMFNbu!Fhw3YfIQ{=pSt8eyFYhl6)LS;ox2&{{@;H=!4?%0vfGT z_}P9PF@o6uIA$15N~Noy)+XP3hRs^1%jx5PUF($~4&|uh^fEQ!Z%fj*Ya7QDdO(Wr zAM&xOb`pBDVy7{|(E!dsfriBMdr)I5-Mby!hHm2qPIJtQr*A`e5B-mk=&7i&cs&{-U;hRga|;`rYE((lQ2SvS8D1E)gu`4=fRS#Jn6Dp%_S&|# z2oy{KcbN{b#sM@l%dlU4*X%@V9JG-L^4nn6QTPG;WKi`=pH-8|q_{w(us&GGTdt-& zRSU&U8gN7D>C4!VDFkdhhYX;7`x$&4WB!+Sn3M6%{Qx-E4t-l?V4icNp*)c3d}m+q zEvsjKK7zE?4SHey7u2|TVvzp|#!JxXRD>cyx~DgjZ`aNFS))=_8MrAd75((L@G0TocJPF~5q4$6xSmzORIC5ZER z(i~?LCvV@W0zk;@R}TG6RwW})IXgQWN`---t#I(w!Cq;p2h1r4$`=X`3GpEb5QOVm zp>;rND>nnpMWfi&2g0Ovr-LspC4k|iW3k9b?}7SJt`?&yxOJw^>k z;J}QC3&u)Ia3cp`TQlaZ>q660u;Cv$V5oZn6*eNv4&)zE$UjyYAe}<2Xa^4nFC2n% zNq-!IC(SI&i0}~P(>)|}ABsyeb*F=AO8e%h9b=r{+8oI)wQ6>b|FCqAW%5WlNx1cTVX{Fb{}v^0M8zArq{?d{(8A9 zjYQU@yg%+%4LM1EdewQ0u~&cns9EK2XK)~FK>vV|;%_P2bM10{TVU(4!-)n8&=AP? z-yALqdw`c`kSvT9h-dAaH=Rnt43%Ic<#%a&4CyMFo5RWO!US8QzH`x3$8u%YW=VZq z@>)nk4aq&oRp6bKv7MuRsq8-V8++~tmz|MKa^w3KFcmskFue*$XD#`iNm)v~L>K(7 z7CwcVbpw}T&gGu;+FgNz?Ddbz-xrf*VBm4*%5wvV`*Z^Ltu4iJ zzu`RS!dV(sS53Bo_njB-AzmjB-a5>=2P#b3g+%=cHwPdfpa-TKgU($%@(kRS@Up3} zX&y?%u{&$++P|&^=*XYJeUApD;YL_76$+Vma z{2@$qOB?}N8@ldngZNN)I)Uyei(pIxCiR#!IQRzs!sG7`Z=Rd; zBqU8WA~M9Ffe8zkwnh(TFt%7bM;FE@PB=>gADaB=Za77>t&L~rZez~2Kvm{vTd+Gi zCt?w9<`_By5Dx#WGwLpy?DWGncq|N;DRab4VP4g#!6iZ4UT=p9nhDm|0$BH}#_p19J^R;v}rc?vq z7P9!-lZ8!kJU6SZsw!iR_DU8lIH5m6Hw#}U>%x){M8dh-z=fJJZ}3vPmNvfRC~Dsg zg>m}jFvEd4B<5#0IvilThd(+w^y2AZKWt=hNhu8N->SArnSbOW6oZ9J{ZoU{{T{nC z9Q;6c6SMM$sbJ(2XsOHIPhVOeaDU=$d1R|LF~iiz2BS|zgJCS!vJqx(oLXq8$uXRo8&MVk{b{1IexvcBfzs0jdS#m^UszG4T^q z;#QbA01*64C$D4?^!AgF1zC>)1L>|Al}JCD(mj7_v$Q!*tskfMWwdLxG*@!UYOCC1 zeeJD!om~{Yo7<%Q0`W8+n8Jp!EZwCXd0z?<{yDbo+reCkCH}|!SUqmdA@@e%e!#-@ z(+M^66I;;O|BIZ6NS^FU2)9M;dLzaZf(vL9t1$lPt(O02G(q5D$n$*+Q1if<2m{^I zq~GX{rTmsFl^iL*{P0q6AJ6})-?6$It~;xEn40+^wlaO2$IMiAt~seiBzsMA3M#qq zDFzc*3Uz+_a~@8npRf79ipBy-^BVn$ydS9dLLCbnPttF-Zl~3? zYkLKVGYoZl{y?gqJiSm7(o4|i!Zp_G4*mbciS8lTs51ZS)uGV_OZ~S-n@Yp-Wuwmc=D^mzSp&v08~gxsq1WzZBNpEwjqX#46TU~2%z7BASQFaF8f zK6e%=bkV~$3}}CrBr9xkbhK0MJr5WNZ9sYtE*mdIp!Dk{^|h!K_-W{GB#=e5 z?`3q&m~FvOwuPOYCK`AP=3uK@i$^2rWtb&&xuD14mX;Qv*^J^LhQ`x)uoLj&vBPhj z&S+O){4d{?RQYiY-1QfRuWkbGFf&$t*CLU;ubNBPz{*q9tG}lP>Qo3^cm?vz80LNw zNq2))3nOvJ3iPD{KOdx@>c1T1bBnM zL;C6Gj=0HU5WidpGe{Ce2v;Kn0l_5%{`+Kn7N(Kz^snd0lgh%J6fHMjXs9O0TUy&w zZVk{=8}ChqhWi1;^btzty<6ciGaK%gk(9N|Y4groi0bFHby#g}$T=se>WtaG2Be;^_XW`; zI|`V!5HPce@tD`=di{HjUXj~%JO4ubygI~Vx#(1!c4~=U_&}%m1h0v?-^uZq#vyCB z0s26r84-Uq^GdF-4T$VuA3XZUWjignZWkIAXAnm#VLQ-OU@3%HO~J zJfLl&bz2fe3kd%ehVbR(dc^(Io>7h(1otv1#Q&?oVV%oWoA- z{FuXTY+lw6|IjkaJy8cHpp~cMl*0~h!6iWq)fbo7R4Pn%l=zj*4QOrHz{pf4Sk1f> zRCr?*XPXi&V%XkFoG|TF>j&0upU-tqGA7-ZM7*Dpjss~o9nCP9@T+k9bkRXyV zd#}byuJ_x=n}v*`QTySt8|ndP)qunZ3aDtxekqwsAZJ_jS?4E=YQydyqQlS-W#|IKwV$uFSKJZ=z9z?TQt(y*9DM6?qDl*a^5+KUjn^wG9iB2(E;RXy1 zL^96$BJxhU!x&CwamiX7-}#^9=lP9Bd1#j;w%!i2`557JUqGs?EDmnQ2=ikq9?Sxo zKRVa$f|I4oMG5;?d;60LtETJ+L+ju_G4N4>VT*^{b47Za4j5-ObQV^@icLyBig^ek z5y9eJ&Pq?~hJQ`$)*1FS&NzB?xAbit z!=t?HZ_yq?bA#l7ia~kd8P6v_oq#zaF>hZSeNv(_M0VTgp8Y=70g@DuA{M! zSCNZ7e^3eeosR6O$NReRuuY|s63d1!S9}VtQ5`o>Dy|5cYI32aPBPe}+1k9gasEKC z(c5>HXEH`8%epMs@UhpS&Vnd0m{FQz;(57es!}rOR0E;0$H~c|pPF4~*wd#ydc^`N zSWouQ-Swxldvd$q@pT^Xnw|BNqkjq}SIBtL%VlTE606vIl!fuZe&O^IQGpq5yFh9| zeyV#-Vn*mBwd+6^f#|v0kl=o>LiB9$K!!-9zdiY-ZNPG`o<^71%(gW>y-AETui>=L zZ)eOUXW7$ukNw$$!@3i!`{V!;dq9%7{Yji;uzR_I4I_{}av*K01y7(03Qy0{rcI_h z4-mU$Oy@iSuV$}<3o`Aa6O0!ijd zj+trEY3B(9!naXYNHlSJ;GIL>4q%&BYH=lswAI_eSU3hAfz9UoW%fr4?2rSj70p^? ze>$3sM?7aftUN-~{XhX3%^n3}6dS|9=<&tY&S;piTa0+LPH6Kg#w~7-olu$^!`wp0 zzcxvpU}VueckQNz=}mXg3Z(3JYo8GzSHf(f`=04{v=2T$A_c7`F0|}B*S)K+0SleT zPnmttL@$gL(=+g$eGNX$`>|_1ER^pWe*xf{yN`lV8BjwQTJ|>rpGkfpx$!V!9 zz^A5jzt_bXc}`czoUO2%sdxe_#Ovy|m5c~6_nuu6PGNLCJ3|!Qvs-Dpxz#$o=Jix0 z&25)gUohH=J4$zluF?ZG74Gm$^DsR;*IhDmW9GeBn^U=CKuz&u^8qnhSE_?`91buKNSTyth> z8h+Fv_Oe_D@g?gUl9F>}m6?rn^5c@DTLh;DhZsM^il%!HYnezgG?gdl#?F?*ujzst z#Kjf(HD~B)Wtn){?a{0>s$h=S3-j}7trk%;E;PRX*wI$Iu@#oskOKnCzUdF)*;n^i zrdE~*(uaJg0YZ4Tkz!MEkN0xuIveRq5L zPrgf+h%LKHE18>mm6o&Pz$5sOmcjKma4UXu^zhcG0LXIhN_6mLH0_jRp65tO z-!?O0r%deYy?b11rW@}ir}X32z?Y>9kf7>j? z@qR6tEEyFtzkJt}UXI#ycF-%kk7{lf!YBgmJm^ETI6L;I$6E!X9z@TjEC=l|WpKIs z+GTS+7iZ{fmWjCh>_oH+Q$w$_XWB(Ara&a@3ZJpVHMyEX!+>|au)KgGq$AtfIyc2_ z_#kbzW;B%Qlw<8z%&jk2KAinZd(Y>WY9<|aR391clJRM+6xEq=ddX&gk#WXj_V*eW z1v>EfRcEVB-SyJwK1KAQ2mD2MEkvZt2kZ6M@E2`O^RPKw@fkp(`1#NWY?|_%>dcs( z;#L&H86}E*@5^P>x+>YDju5%Q#qJ8l)8m^cGgB!P7h#svvzUyK>6mf_y2oUNkN{ha zUA>9%o)=$ov3J0M5yd^^OD!mf%_|bWNA@RuC23YbNdeIEG0%6B=GAS*5M)yi#12$+ zZXqj59_4!(sINalO5X}Go{`zzf@cqu%$3DQvs==N?3QjUO3BF-_sUM&iq?|CqDh0< zrKJy=srtOpc^Q(~M;hV-776tP6In$J8ki+?qni_Uxf-XKu}?Dd4i8kh&C7cut( z$uU5l=U|%k$rFpRx;o4K7%6t1LUpYLE_OT6{KRhD1sGoK1E)t1A8nGUv#Yin&2MB~ zs~irPgh~X~`gG$UaKMJY)6$GA9vfv(Di`lT5xy~DK@e=pO@6b0Jd*P*$4gG}b6I{d zS0?DeAahQ9K*qJSLI+VVRu@2`;U;l9Mt)%bvs zz6`)pmUcdBUaY8GW}W9_axax3)%7fKvIbwwBf{^mqX@J-o|idSR&C#Lf)HE$bp*aY zKgUfE2rh9}nEM_*qoM9!t)fy^iRXW4J~z{l%Oo+yy7uT5%M=qt=a}r7jF3pL0kS(m z(|l$!tzB?>xOF@=?Wl{fY!o|*lIltw`A^ql+$E~yB1P(3*8$?_ZTsF6GKS8BfpZ7w z{`46LB2o!MMqFlEJAgDdzS^F$N2a22K#n@D7zXPcI{<|cl$^8%TS@>NvA(5Id5H|M zVt!3_-(w%xHk(u-$nK%307S0Mj%Pg({GdAWl0B#=uE=OiI3>^I>8u)fmQ-lW*sq$< zxtcwb2w$eMd+ir6Bx29-uiY2f#hzf1h5IHSJ9aT>o}Juc4ec^EzI?VcMk+~p_N7|q zU9n=XMA4f$<>a}_K*I@kwc1>qd{~`WvIPF0mEMMCi6~OKiB<}HI(LDbeablh9 zwF2}yu+G|VfSM^1pp1X8Ni53Xy;^Wq#L_c@o{*E4mVq$RB^iC0HBcRtkUz;i!P|+} z)WXK9Ey~I_@{rGrLLAhPn;T!ghX=f1mvO1rt9C*q2;6UHCeqYyZgiP7$4c>w}hv7G^LbrYkQ zIok^U=s=X_0jBMq8!DOoCNH3@MCe``Huo*sOUP`$`JUaKPfFbFJXkr=evFZNDX+ar zy|rwnq&=Jl6_q1dGG$~~idPZU>a5R@O1YabS*l5MC^wm)*70-C%|vGOHV)}j-)L1M z#ARgd@?tBqKF3oj>|ow5svxbzC(y{Ls=A7Vf~stAOpjQD;(`6$Mc_fUNVSef`vjj{ zmr2PD=9WM=#+fwIV0|kauQYasSr*CNKAB3Q&3^Jk-J8IyV07Nop>nYj0q(gr{$T#_ zv`W%#Wfy|N55YDbxGJF@3+u!{O|}j_un%zj*IHoSA|R6HlofUe&bArocnqmB+VF)u zX-X_J6_vKJVuM=n9Zj~-i_?vJ*-7lLp<7vVyDtgOvE)0yYuQx*6w(=-p?}LOaTVz? zPYUk8t?V^gARsR<$%~(+*bxiI{aKTFY%(ddZTa2N_{ftZ;r%0wP(JxCv}`!#OIZURr=JNfvNAB#*toyDH=ajs-%v7LAf~DU zpon^g!B@hVJIIqZlXoasFE@1PeYNea>Tbx#;*w9=cf=#Wf+oUZVe5mLv@P1`FU?{rMM}h>+|S-ob|f!>xK!CJbih|6`>~GTZJ@L^)ilT584JosSU`#H}fEZ zT}_V+{E~6{p+SU?r4;JR^<65nzvpM<&Tt#8g+G00LSAz+{s>Ql&p|k?19t5KP^xt^ zm|N4-%Nf0A#2uU!_TPIMmlr4gpxws4>)o)0^>y9HOj$^i)JbaItEUxvqRA&?22w`x zHVI>vqJ`ocUqZ(j$GhJIz)4H?5mG?Z@wOYJfR7Y|S#}KURDiN}Zp|8f_^J;n7A_u$ z5zjUvd)Tyl?&?d;z=|iD#oOY)xa#K~-d0`hlwqYqXmcOP&Yhj$4>lVOlBDNsK9Oel zurAHe=c{B_?SJ}$dd@kdVuH&_lgf&=&&FQG+i%kjmFGSAAntrsB20d}NiX&ISX;vu zr*VoG|Ldm}#u!cEwHpp(TuRdpy|e+Jr~(&N&lBwrt_i|T50KZL%sax9spVK(oORTF zxc|YB%+1t~EUS~Erenra*;!O0;^jlO)K(wYQ>pcJjSK(?Z z(US94g*hi;KP#exm+y$IB3G;RWjRE67H#~{TVY~mEWKTiFE*=bGo{E}Z?H{I|8ZNq zNw=)$x8vxI^ksxvSljq?re-%~5v{C?Y`f=d@_FhuN!m|+EJqK%qVkEjycBOWzkh8; zk(6z&iQ3I!3xh@$Ip{zaIa+qh?b{#3rKI>M29ynm-Lf?(bLaL0uQWf*<|WEHSstm0 z4EG5LxCc9E0)QY>%rFgQn;9kr8_MyC6}w7PT}_T$xzcY!rylqCwns+AYT&a+Kzcn9 z983Ixc3z_wBa{?*K76Rlu+eiIY_8iNIQ8FnEg0Q)ho+YMiABLzWn?#**E}T6cu3{v zw{&K=W|P`o==vFXZ4Gmox+^o4$5^L7tnyIe{>jRg%D0ASI{k#y$&R2WOigQ&gwi7X z9DDa0Pfr8^H&PZj^Kra3{kUHZ&}Xod`T{U9yk3uNQO#Jyao2#k)~gxEY*OG!})53!~W)=yX46=#KLii~B|KJ?chQA)fd&PHqnpH5#>Mx4}eL7Kd6iKm+%Ew%i(kXoSOL-VLZ z<%wE*&g%sG4FII(Cc0|EY<}U%JC@Z~L=Q^BJ0?!OOf!5<8;wd~j4BhSx8NO_?FI}c zdrpKgUBY%yQJE2BlbD^+>{0G=Cwh8QeZBtp8!ETLsCin26-|kM!v>imJDoNt2aIl- z*YFjkhb~zg{hk%MO?r4t2T~0WPd&yfZ6(<1g_g%EPcm%{zA#|zh^Vd7TY1AFqU7$~ zqQ&CdA(~H$#a;$>6CGA`D1d?dHj-x{a-=+4kw2q%g1`2^w{MAYQ|uo%B9t7&;iK^5 zd6NZfGPej}*C#>OE<&&9tOBq;1ma9T-+3ytZM zJ@bV^^yIVKj85Lba_$KU*_moBMlHEL>xJV?sjKNZZ;EfZQHBD~_ zZmYAat80MEA|v`q+%W24UMLID=0hkKsBHLUcYa}8tBReiz|V2h0(^X?0P8TlhUTDK zH`LRv5u9j&Sdk>oHA-3Z`j=1sN2c%T9Gd^?m#WZ3GuT7jx%9ee+W6#zNHxQkJxDFD!2jDP zqz{GanNCf)Nnj*=@lap1Gc($tW!y4xBv0SN!bNWoK6W9tCP2mQ+SHfW0_fsvi&OTc zmYB9)BGPHqgH`%Z4U4v#2k01&y}wp&=!`Al?_w1>I{RIc4%VWdQ5t|t`@9YNN@b2m-upxj?1a?4s!dE)l z^?VA3Ted|~ewdHnPU|gU7Oe(@o9n|svZ}ecNt>(5uHoVWA!oWRe&i^n(<J*|dt5VTPrauS_Uzsn+esy)Sz0mcEoTQ)Bct2#Olk`Mm8`7B6eqgkx%DTT z3G@vsmVBI;lXOP!F(V3(%8Qddn`m6{)#~y8POaD_bNkI33GLRJ znsw{e^+e~a*mY<9@fG4PX8rX}96xSkQYR^M75$N|^Hgz~4LeeF*SDSomWG?~K;ENA z>*3pWUKl4aW%g|6DX^2kej``7(=O(o48|`nPPCzkE?j0N&5{JpWsBP=vZw$XM z?Y=EB*3n5xCR1qojs17h67AVbd;}f7hxuX~p9HPJ(H;b9_FCV~y;x`W`OU46e2+)9 zZaGIj+gr9fl?{D9cETx>sxZ5LH{qE$G~ zr0aF@g$G4={o^CP9oLJnqQMYYm;)R8n*7`#Jp67ne-v!NHFRd4(U2 zC@68??j5U0=Dino3IF;+;^XuKt6x$M#7Jne?HY&llf|dzinp?L-%_V^+TBB)2L4TV z&6t={Px0D@`!Pz^VX%7bKYqGeQehG}{^8YmFLgp^nTD*>Yado?zEeUD*R#|9{(M@D za$fsW>d8l%$nEVst9kBx?GC;lTmDI+#P{&q)Hk9*9i=&pgZ~WZ;)UL9GAet&LY?5S z{?0N9G0~xqr8V3&l0#csc}o3E)=J{^TcQFY1r3c_#H{sj^mDvBW?8EOX2Y6d*x4@R zjBJ8|4gJ&8(eMlQ?z2wy&(6rtR8t$4)kSZ=^L4d9<^9`WuhbQ4+6I5ypU$F|_zf=E zdZa)mLFwpNo}`kB7PG>PxCGTUmnA{KI5A4cM&=|KunEfkc39Wk8RY3^_v-y)PU0I&YV~OX!PTKkUw1+KA z%#HtW1ikJ|L?R~$LY4pKe(Z$R%JmadY+#?SwK*(uXRX(kDn2tIiKk+5giVac>t z@#h0F`Lu3aC&={+$qkFyv@O~pe#1=cAt5*Vq@}>>Cx2zk<;7W88=RF6*sU?~8<{_>tw{?8{rCobB&@6z9i-*YwJuKeD6>z^~rc3K)gy4+0ZpB5=@M& zVpW-{(f`-dz3dt^mZ zop;9Bq zQn**Lwy;>^IByr~w!82Fp${`Dv(jl+jLzH1PTaV6@6LX#)T`)oT8ZU`KmW_;YGldn zZGZk5$_JtuS5K@mB+fph;udLY>H*T;37>R1rF)s-&ojce`|-ygeJ_L4-N&V#4y>%= z@>cTHXOyu0D;9H)Ny!R&a;=8TS$3PZYURO>{x6_bPY;Ug=U;66Y9}qdsEuGJTRi8! zv8~NoeoN5v3yB$Co*R{>h1XqoU-S&8kz40TUi&^NZQXtUTj%1X@4c6lwmJPlo*r>R zRn_MH^&LCz=;k>rF(laJUY6<3u6MVMvt#`e?@&12csbc1{9(%#=TqUCnNmC=UzWLi zx8=2>)Re`Ztc{?4kfG^Uh?VS(&CPvhRPx_BbA?GGJg~gy{MzqO|I446n_l8E)wi#x zi!th0I!zCJ*A>W=k)_=iXt@uEICO65(#onUY1~S=yhL_n<5l-|oTS0jq6;l02TZo} znd;_^tZ7WuIDR|oT?< z%I4D@HhbH|dL6c<3f7#cxNBU$xsRsFb!=y+Nk`Q=bwj!Jwp;=GN$}Zt`>SWR*9kyY z@OZDESG0273#XRD(vaDkWPQ}nnsgofUs2N9y_Yn?-0CZZr3DUX%DYZD{CzTDEAE`l zwaeST=(avoLqfYzK1{r%&d01yEFXx;_v7#FA;0?rM3g@FL=) znwRm?Z|;j0x4PbwHF+Ov0av?BAke&V+TxW>#V1*CvBQ{CCv6y@7@@9RQZyjuw#1P2Gh%fWY3~|u6sP567RDv ze)F;ae!udRvR<;xjwfOryR7dBnkoH1y1p{3sy&Pw%CQiq!FOH{tPC>dU;}9D>W+rZ|EplK3g!UTlRhf8&~+;$gj9 zIfAv8d!euyHMlDq?+m*0@*?}s*|Fuih2Vxm>~0wJ6T}AwjXqur5pUS5vQ`D#w+pM) zm;yD5xUb1Rpw$WejIFSFvAVx6+l?gl09g#!ffx+z3Y;FQ={UZ;e6BFlG-KP~ZnXc#gz<=CWrVl0uL zKVK^A85+jB@q(|aiPe#NE=aNSUC8z625H`X{bb%pbIbc>^%r}++%vgl)x__;o4+Sr za0tTp@hdiUZHo}Sn;pooCI=<1KM_tCo@ym48#xwqzXE^g0s-nj-N1>hbL?H6+rx3bBQW&!^ivDKHHo2l~=RqAs@oXiK^q zivsnN6xR=Vd@6xr{1ES2$EDoD58;l(6z{CX>e_@Skzo(lBGj9Sq#}h!Xa1)HQ^Y4{ zGZ}4l8n`EMP*&!p%SAzNQh43Nw;=Ow`qIAPN&qqc{(`scilja8`nf|MYwjiK4*+>c zb%oH?qjR`Nd&jXuyyK?Qm#%t#KgyiOumAp|;AktPn|I$0K%3xuA@i=}bh)prjalC%sJ|23tFqCAn>~+l~d_ncA_h2M9``tI$ z#IaHf)n|7%_P>@iB&2^A{xp~>+^_YRJ-Wk$(9&XojRndef#fr=1M%G4kPWyc7i$qd z0Q~>|z}}debbPt^@6QZ?^%rz12IDz( zN!A;4>?VaD=Xmd~i2z`CH z>{q18i3xxC(ya9dZIJDr8Unat+!r_DQ1jVxkOI#hv*W!Y%Cb4MKWK(oRW&$Mr%YN| zZmKAvW{1|iQ|RmKHjq?@m7gCteGE!d9KR!DwPkz|GR9S0l8G~Nk>yOJZ?-f}diFbz z`u!?~1Uf~#K9^cF4UGff7qYeQoc2<`^(KU&{3zZ) zipo6wLfAO;;!OWsTIkaVHCu@Q5kEF*CP)oAUpA9*Z}Vl7#_mlkFPD%>u1D_eokY0? zt<`!gEh^h#%qo$iu5RXOLn$9Yx0me3*<6gP;|%@0Pq5N0ffb8ag14RbsYe6avcyOQI%hNe|xlp_cFurvQsMkBj8DCj59Nw)t?Kth> zvMWL{MGUK8jd@OAjIW$Z=HE-^XKfRk_St^_?)5W5?%eo;ncXqC!_BD~Q$_Qc;>(F1 zUe(aTnWqCf?3m!Vu3xn~frOq`bx7K19=QqU1*bH2wNFdK9Al2741a3Qq?S65dBW^t7K=tET0^!VMbSr)z~Z z^3I+pAOunx0K(8wSVloZv;FGskihjyvhxrUq*%M469$dhUQV6gBFk7^TWdG(|NP+K z>6n_THbHrdwxhUj^AoziD3rSR&qLnRBL3zLLPFv!Pt&*H?K#r#4-bD02k_*Y3kX2( zd630=1{py}#H5q6^ZQic9Pg435C4xjUMBFzkEc`i)YQ~MoJ3+hEx+DAZ_8KCRT~C@ zG}Lr-IS=2AdgIp$)y8@=8-3?5o>iHA{7uyxu*1ijAjn^6RwfZNc&UnaXym&{n)FY0 z851MDa&QAaeCqke+98tKNfJ-0>7?~?O2ixUOLDy!z(Q|B(R~tIkX!^QUY+dP-bzyJ zGQNhnO-F1XKqzh;41zA}kWF`85_UD?NQjz!Pzfn-Tg;qK%BQaz`OLr5s-2!aQ5g&i zi4X$K``7o8c=%rk`Zz3XZ%u_0XgMrBubH;$MIZ_S^FGm$wL=qkJCOd?i93QXEWzH4Gh0tZEsmOYkKz~iAMr{Dy9%wb#; zHS~+To+sakLGC+ARwf{9Z5vMwL9rWNu{MDRS1g=%PtBm^0MGj?Yh9-C-j(G?&Svv3+LuS&*6 zGJ&^e?#v z%I!5DwwT)W{PHaVgbFnL`k6PY8)g{sAHOE`8%YYOLmZ)N!lx!2ZaC*$>H^vhr}!r(|ER<(HyMa?mvEJN44f$<4d9_i0&*8}=Y>Rd=|NLbSaQ1d*qt}DXu z^k%WgQOOWRJYU?AVWcT1vEY7EyIWh9R~pcJO_IMeXI*BErA7#Et81pLBp=$Fi{)t3 z-?42{J?|$vsUBe+`S+*T~Y`4lf-Wc3R!`r7*DpRhsgyg zaPspfXX?3-jEut6%GMr%nTo&qA*`1^Vr46}3O0aLGnbqFwJi8@rGetgCC6pyjY(SpCT zXzM5|>(xM98P?XY2pTD=bvKFqaD`2y9SWtVLzLO+rJzjWv(T5yzAKx?A!`Qc;3PCQ zr%y~Is6A|Mnkf$m3c#3Ehlo*AHn}%IAg^9!MOLOPTUtIkI-drk2Y2TIlPJm0L?T{z zj269vgDB?Cx}Vc}VL3R&oygjH!F<2oOks3=lB z2NDVL9x2>iXWK>jnJl8!Q{bM%g18m9cIlUs#0S%-qVl=t&`3-MT zGZ&#~R%d0_a{V-dPGKIo2wE-)mUP_Ai6NSsCn#zb)Lll4w!XJ5jb55QKX-m@(b%=O z5dRGzs8VYWA%{Gk%|R%ZxA*(tKZCrU^=Q}UTN!Dhz72;;mu1=FX7vh&MJmza2d<{&s|cIO!z zGjCl-h^9x?KiQzVxR}YTt_E@BzrFTPa zJUQ*XAN)TR38fRsMq_{DlM|5g~sjN z>5XzUBEZ4}r+~=9MUx3~uA-0WWB0ye$;z1$eNq1KZi$5EaICGl85pAtwuWRn|Cm|7 zV2v#(bvb?$N(5G}>UMmkS|q6a0{A!K8@6X(>K-E?goC%m_Ew_)4MU}vloS`YrR1AW zF~)Z;k1ai2< zLUeEXI(K@beN5j7dHNN##TbR3JI-m@jG91!)r}R|V%@&H2O|aIl`Bu5i)cTtCcRV3k5CdMp@v^t!*(-23sf27ETl#4J!d9AzFKKR zJ%7$uM*?B<@_-l*?jkp5qM3|YM7fmX2268Je*-PDyKn=V0HFnD(vv>3yKstXwD4nJ z-CEK;ElWMxmCW=Uba2h1yqbg2d1d(2gTX1VwglSnB$I_=%)Ofx#0mcXnLphWPu3Av zi1#op+^NBj`g1GzVPG&B@RY`W{h%|!g7?M{aU7dsJK%jwJfeWfda7ol8y}6S522tt zU&qvkM=@ey^(PYGESY4{=?Q1vhi_hcto2B-P3(pTgr#!Lzl*HZh*Uh zMBcWJ0A77UwLGFK*6Fj}RuG}ua6qREwTBFZY{6;-@MHrRv;am~crKgo-L5^IFz4zL zo+ue#cfdcfdml`LNPdQTmhk);k73`>A_j&L>zW#i?FV^==lT8{6=)q1q`h5>%|wpj z>dY>{&kt8A9}yf5wN6D(K>_2K-t2my0s4QD8+%Sa!QTkxvln~vWEF*;{z>QBvlsNL z>l686Q&8x*c4`~o5!#)XX#vIP;jw2ot?h|R@E1F(=dr7+BdCWzCtCn~Wq%J&32-9C z|1EX4!M-{_ue^YQUN(u}i&9Nh6;)I7V7k>ma{j%#I&s1N`uOQ`*gp*__=tuENx?o8 ziea*qcuyw|YPF<%enEj@mA00an=zGtq+I^*-(nz23tIPtM#9Hr1Ev066g-*y{79n0yGH2mx32)Upw2G|g#ci?ax9z5NNS!?x&u>=teMi& zoCkut#7uTQi4ttjToEoJ_mZQnrv#3xfkx2;T<%Mnv9 z#cAMFTOa&pM?U5j2@$!R+5h}|lPXlNT6Ydu^sbO!rRdo#~7tY8j zg9Vn!kl*77sI(kMNYEUb^KmyiAcj>9M)EM0{oSu{aawQ0HXOa?Z=n?{QZihixn;ZE zX(P{xuXKQTdGE4GX;_dy{79GHohTN{pI&iQH#!F9n}BEYn>32v*bf9;jS7pLToy|I zkomFF>+ukUmy>xcKUa>X?PhMeE#+Vw%U+3)uRal}UJ2i`rLQ&jG-T5iR`0IQGx8)g z?K!=@h`es>zPDk9fB!CVa|Pw%5Do=Y7QyeCN^ZJ+Iz)O6jrHH?0)mitt1@6G0Jod%E; ziB3iFA$osT90yK0Q%QZPory{e$8(FdzDo;m=s{n!Bc;l`O^xy@Y>9AXd%Uyt{ehxM zw-lP9xV?P!yo=}#n_Az7yO^R!e-4b5TrNUB9HHAA!4j`FsFKv|vEHkimA~5vh)VI|+HG|-l(^T@GaJLhlH;WW^cnO&GLiVnX5ZCTU$>z;NNNru^f2miw^7z#z z#{dyq8Nqbd+u6&O0Jk=j{$yj#+-wpvd``Ia;X4o(hKM1l6M&8+X*Lqmi@scTE=j|gJgB)P zw7TRN^gBm0&dU0S86y4dQmx&QuKGlFHZKsbiHotA2lLuMjsMoy=mu#8Ott<<($W#y zSZdfK2x{{;lfRsyNl$$J61Upw7VSuxkN!V|6IX76!P~*(Rh~W zcaYI_=MQ4AV!kmq=h_^c|F&nz{!@zU7mJYp#>TrNQ)jYL|LU6`>J@JK9f7mtyBN#f z-k3lF!z!0DVJWX#{BflN$ME>>+K_K>Iq7t7XFTBI!f>Ahc2)_p%}&{^$*$vWKnPKp z@t`AJH;|)tjBa&U zg8iodT7-zPqnZkE%svYtr0gu#R{w`T5`;}CTF!Mc&*v0O2hf7^zn@7NvU1$4N8x?7 zSs`iec;B;sp?w>W~Ogi85zYW<^%fH+GGzdnExF`Txc8UiUocbOZ^5z_jLc_&VQ z2R}lvy?NACQ?j+;Fs0-EF1OCLzNH0;Q80~hVv5@k;JN{<24qBGuUkiwXRHUqyXRxl z9g&d=;m8_+)7bPS;%uZj8<2jovqCljwNB!NCXdiKfS3GaGxhc^OTAC@vX% z$zwR*EQAMc8!*3^sduDkp9|nA(&F>2It=;4d~pTszu2olN7nAgb*=`6?^Y2LzInbF26sLy{Zsos z^9d&-e)vJP+x38}883^qbayi=%bR?KHvrp-P-?f9PgY}OB+SK zWy09F>gZf@1LL3s5Ax!Z;-3IX#hamG9jyskjg@)ieKMbEP+L!DrOTYwvzc1XWr=8u z7x`hgGVcQMQ7V4uaaS(~ogL67Eu%plqgS!$=qn`mS3A05=p2~1PXW5SFKd0aYv&e8 zT^>i*{dtTQk06-ZxZ#mRHh9XmxgYaNZ*NZ(swgNo8+pm9<)-Lz%0oNF`M^0C50AX` zltCeQTr@kMl<^Lo5j)+jz61%dGma>3PDSu;eHLu;K0^utUNm;o$iGu9)S%rJ&sOS~ zQt@fIOYjpooIrUloT;Gi?S)irE%gWPJp6D4i%{+^H=Y9)!_TkaZk;(|SbU;WG!&dK z)PA0rUQ0kEqBjoBRkW40*J>!|Ar;>YNIz%OZMsEB)AiHw+K4Ak#j ze1cLF(Z2C2NyoWqd^HZ-SMDM$!5E2EJbU{B!nimd9Bq2IStAlYqc~@Of&-qQbTHtUJU%|}Q;w|ix)ZoqG&-Z$!~klTif*FK7wvm`s56VAkZquL z6tgad*Mgl!o=4Y*oHqRIq%Tc9;6QlSAE8ZfQT8%2&Zw4gp`HGKPHpfW!@d$auGiI|%l2(c)z zvakpN5vK4FQ^0XXYO4MtS=!n{1!@EvK$muoJ;uO@ii_)UW1*u%K|nx2$)N(T&;R%f zu7O<^27HT$sz7IIyn}syl@c$_npwU4=~3DCb^cMvi0&2~7U zjw%%8uU9-Z-?YlBmi^4ws%;bYCbJg#CH-Rv5(?^cF&sTtE;An$&cio< znCcpNa|be`@my+{$B<+mgglJn&h8eM{-B(D@=`1>oZ?h?;o)rPU5)#};4Sqcq_eS7 z@$iAIAcLFCQ50S8&c!H&o{uy08UiE@jCjmOSfUMc`r^&^UL0Vg3(0NP0&#y_bg6)b zb)kAn%00kdKj`W@yFSSC9s$B_;wYP(u8|$6n?*`-XYgaG{^NURO~ES|{>??OljZ|) zmPr}BtB|_2JWYP5c)%c+kCK+Z7@qRf;0#84vPV0Hweb6b%lt%6=h2B~7PPj7?-eUtdlKE3+Ld)CBF)77QuPb(tpSCDEL)C|VZ z$|*!>>X~G@_mYTdv#hLOeVYHfTYSl#~g0W*chLPOUC!l0lKcD#e(>B z`tcDt9u~nGNUEf;0K*6dd^yhXfS-&z??aRP!(8hoTCu-D_mc+H;taY+j_0;~76Ma) zF2<#8j$v zN8{flBG6BD$n{HebYg94Nq^l4A}udn<0R}F0fY`jQh{}>Skv<@Ig-8?+VL_1E@^}* zw{B8F{2xT4cPjKdEPuvuanw=dOib6TTyDv;?Nf_40zIB5xjA(1Y#$nK*nQB}`qewF zxEPs$K*{6U%JMz0y18?I zI!?jfPB>q)JW4%|0@2d4EOk8NVqbEGy&M!rw{#f0I9x~6OjfFihF%T3KegxIf}E8kr&b26~TB91I_l4IYdCE5Q^2rTFTlVUH-50$We zZtnZ(1&^gu-OBBGHwoscTC*Su*q;_5<;~I7msi`BZ zp=-Ef_wFPon{lX7cZPJf5`cAIFOE9wO3QaI>_ezQ9P&!5c4_q#3V+i{-^->`x4kki z8Ib_70H8EHgO3w{;%hPeD!w|~Z}8#QiH7OW31-R92SiuqK-_2>K*}=i>s=rHPC4-_ z*EWsk(`__Iz8_pIH1d(hI<&9Lgx$uQpsa?l((Ph>v2(8Z(-pQiQXNJ*&O3N*oI%Qv zX~S2I+y^)?peO~m#4E;Z@~eabPbyh!mKwc=q@?iM@`^v?DbZB|*yI=76tJRo3MUgF zp6VGI!(u!gt!SzVi1{<{*eJyUgK<3fMP{HYQ)QWNas}6rjntoM=flo2T|C7fDtbRZTZ1`wFIHsQ4{`+K`SHRQ7K|c7K*ytQf@-sCA5GO zen4JR#0{!K=0RBJf@`R?S1AkE-~M79Q4c$e(c#=i90m@8&tLnLBSIk{u#-FUEP$M|<`vv)Hek0I@xu6ehjT2VO2nzEHhB)0ubBWx{1A+`<|VlBNjJ9%ZLZ-!MiJq`g$z_#Va=$jBJY{*^hp+ws`S$dxrv*fA^K=j>H>NY z5%TaBZ~lG`ZEj&Ld=%*2M>;c%gySiU&eDu6u~ifymeZLNlWVJA0p~Z__7!%y=>LFA za$$)C!SQAX6^tugE6QtNymneGlS$gRMriiza|)$Vo^4K9tL%m~($L^S0D%Tq`pjtk6RFa(83K z;85XgL)dgXFUB*k)57d9t`tr=?PY?Cw;oWlL}GLnM~!>LZX##naW=~9=b7Shf~KMI z8k#b^rN1?F36`?8;S#B{b_8A;%vWEcjp5bKC#SYSR_?B8j!%cj^-N4|G=2-T*t4t3 zgmP=~vlN4c^}nTcwwyK}hPyQ_JIp|(l-}5wGznH_9-W=-rH}rH9+P!-<=fqzx!)uL zc?05^50VaRto`9+uw7f|^;5Cnx-6xaV6U+HFcvY#qvuh_%U(|&a8V{OLaPYys@>a^ zgMTsRO_~F@t`95;Y(vhGi60ZE0X^&&MvH>jc)SgWML}`NV%g8W8~zPCQ*T|~y<*#o zceSL+#xuUQkIqin_-sDJ;H%a)@*V&F|~*SKkvMFpD7!zKh~j$hBZBbQ83v$EJB|UCq_fp zg>ip5EGp+&r<6Z&ankRSvk&yIsop8ZD!1AuySKFo*C%Lbu!;4WvT8yDct}TFHRwhN zkhnf~9FE-u(r{KkS}T{xEJ=0{q?C1JmT{l)6_#+*>>3fcqB8Y~9Tn~%EcXsEFuev` zaqC12QtMB8l=jJCMVn=A<=F1l+3plG-G!8)y5?OV3KXT?_iQT@lS=2fr+nV)lJlC(ZNXD`q*f?(udw{F; z$ikA#MnP|(V_^hef9q^NMdkstzmju^fOLq(G zVT@8gWk{Ji+(GF*z@er&O~*S!%&2N!f(3T6YQ#>0*xUWjnd^2A_+`7dpE+pnF8nkl zP^M2UJv@ll*81Y$%JP5oX4rtK!Yajs0|ngjZ2MTGO4Lz3ykQ{)p9+f0=R|H_f3nGu zB?VnVfr>10M6ST#C%@ZyXrJSo|8b80bBlxfSvy(&2mQY|6@w?l#9X|d8l)Okma~v4 z*(jKLd6ozxQCl34lia{mUybXO8K6OP#sfX>Z}B%JW`LS_rbVQ?6JJ|CTdBgjbg`uc z^%rXdJHPD5pQtIGPLZR+^$tiW65(fm`dM*6`|EN7+;g-vEq`Fu3c* zD#OgI84?YP&V;w$(Sgn|=yLoJkeJ*JXBV{Ct8JzO?+21e&A;3_5%TbmdYtsSWO`;~ z3LP)l`^Cd!7zt`U>I%h0oSikN+@=?Eu+*Eb_${sk4)?BnisVJeaa2`FGd+JXleP-7 zuvm-QmA$oTIl$DgbF69uGdIpTdX6#j^=?!)EAi{e!Qxy*qd-jkAWjOQ1e-Ma#AYuT zj9lFFvy_#AGSItr?oW^rf>{CEsq1>Wf3i_;X*rPRZf z#y!0)gTaJiP-`DxXKI9@z%@62_`8MB#Z>dH?)6i)OZ4${+092^uylzKRJ~DilI?W7 zsbZzF13%I;D3~(0>Xsz6(kB090Ec%)gizsrK%tq1G)XCZmZmUGLHtS{s|}E$fq*Z7 zq`fF2fu;z3ckaFokT3{V?)i) z#fY!A@h82K!{o4_0DU#JOWLtT`v+;%(ss0Ycq|-9W7C?&NX2;e^V^ONEd)Db7z4Dv zksl@-wod^{{ed0mqUR6?y+inIPF?lWIW_$@XeO9CDNAvhD+?yNwxr~RFJM&N1vQ9lBQ}7!Bs!aC%sB?+uUo3r= zntzQfP<70igs)sh32xSXi@<4Zuj%%&F}lg^*knQiJvnCFY=~LnV{HZ94CG3`Tr?+&g(HIi$YQ2Pwa#k_IaCzNxXWSLU==n$$(1Jr>brfD?%dSyw!PVOp4Y<% zH<)x!)mIa3=%}{e&L=RDyVeb2~U9NP!*NH z5{&A)N<*9&61pH|Inf0V>FlbA@7y2rhC z^ND!l)xBRr-vT*~PE*pTMQKg1*Av4Vb;rW|+{Z9DDQ^X#GG(DkzMZRc&XSkanzQ-+ z0pSkHJp&F+6SH9txjx->?G<0^XvZ%zRgxKDM~h(O=fb&$zW^_k&Tp8WV zjv6Xq;jaQnb|lQX2cK~CkRnFGVIe;X&fwDgzlJBw5L^BK+%lhBT~)sBdoat?Ix)n$ zS_8JAcFWt4kD1XvAZ#S%yx;^nGR81H&$`~oecziK!HAe4C^Mz~{=K`whNQ0C`b@22 zBF1ZbW3&ujifdq~NSW^I&jw@0AL9<2+w;>{>s|51o+(@fgosGw@$iGi*`&>3m|%km zj=L9Fr(yVogd(DQmmWv}`dTl4S2%QYeaT`)vF8nfA0GDhRnOK$Xyz!Zmi+GxBDDOb zq~PCYO5ER-HbE8l-lx_G|$5hn>d8jI+l&ZJJj-np#4FY&L`Wweta}zI48eGP-`-BG^2JNplSS4r zl&<@oD%4Os+iVsjs}A1`IIw%5o{f{wUliH2t)>?Ani3OM=@*@}fa2UY+h)iT7-CL(4?s-n$9Qcw` zV+@8b>Dno+~O?WS7A|h z>US!W$hxvxTq9v$Z#>H7k-oDI;>CU1?Ah6p zki{P;t`+S^k04TG1yKa^>^)Not=g*Q0@(JGV;OqdnW)Ho>TZ%KJ37k5YYgS>1@hf& zR{6Eam){7Uxq2>&ip77_XTv-LPLWM^@R(k_uMHq~4$PAJuA9mm3`_)@u+1iZM9viA z1=^Ba(k{Sgx~(`pmg1iF*=eGxd=|GCYu8NlD!}QU6by}XFC*$z=jDr(X!b3mqLxTJ zlit+|OrVF0^?MZa+KQ4Q^6UA~2V%q6`38%~>dj1YwNsCejUxMPm|C6spY2yYfIWbC z8qzNj7O#mdrwVnB;MF{{B@LhWI6;$qj|JR#(4*P~!7{{G|(W zWbafS-kH5GnVyxS{e45e^Owqe2voM5da+JsNgZuhk9BqCV}&gz4`|QT-n~ zfAJ@l$3Ig-3KPb-S%7Md929JdM1Qspl^YyIHlH0xv|V#+HP;)CP0- z3LkmQ)R|haKpC^#u6HP$$rr3}pN@0EAU}5LZmv4teAb>ILj|I#p^y$XXh6C)+@0+` z&OVo~2G)eClNJh~wfh{+`{s(@)sh9#%e}r$HWc^6R4p-wCTO>#zTSctsS>3|_ zgk(#&IS+*rH_15rZZ@)cx}lBBNis~k>zGDBUAehcu9w_kUnJi8ljiPh4jHk^FJ3O> zjg6t;cO><;|a6QByF4^SjFEQ(^i3;8{q;?3nk_Fp{BF?@ulEEQm+th z1}F6=Et{@`!^2j=XV6DJJG(=$Jy3#u5EjHj2hvGP|L$7_u^ZL~>^YzouafKQ7jG}l z2bk3?8_$PQQU>=(TA}qNa7|+k*#l|c#I!|CE|5vslk|rXadeW3rN{Jo)z8yGfOL=i z%E5W$JMDQvqPje49|&u}-7Hhe@47ude3zYqO9xS4_y`i@s2*d`lHW~DWNjZMm2=(S z&Xj!1>nlM`LtiA}^>c6i!N)|dGWC42nLb9WqEOpo4@BqLv$N@#y5BJeqI}Bz^L+A@Jz1KDNR@Vyri>)dU zh5QL1m(_z2&ORO>iJ&%8ui^#4_}F;=LFpXJ+_BXrR6K9+jJL=)eeH1GYdYL@^=RS) zTE0OWD9F+dU96{5fODcSvIn>koEp6;US;3yD=N|c0YpZC z)C|v0?b#M)+Z~4Ox&{*d2nSUz%kxX0U!dAz7-0r~2@0{9X37N8xOpCSv6*hx5S|qY^KiMTLvuN-ta*($5`ssuG!i6Nw5K0h!d!wB3`kwnp8D4HPA&S z{nIJmTnGO#_!(1ne=GF-c}9?gICwY~NM1bs5R@ofea{k$`^8>LPEJBx90^(%Mf3i@ z2Vk22Jpi+?fTj>*Qg7&Pb$-K(l$vc}P2sbrD=d_mwtOh@T0o_G=3LV@CP{ea^fVzQ zEokDGX$$mZ?{B*+|2Y+X&Spnsxr|jh<+*;C% zt^s>0>F;OS4WNkYti+m>+Zod??oyjKvmi5aGY>;3HFp7busV#h-c10#tg-~-s$m{f z+In>isce$$>@d1+{AZi(M%%_8KI9uy2=IRgkFRl74G34gw)=~>IhWrIs9V_ItJp1o zZCn24F`J40`l^xgFc6W=0GRnrMSV`7R%}rPnu?fp&0pLd%hWGI^B@ObqDpSD(T4XN zwhPQHg3Z;%1x3G>h`QzWRDD)Gp~gm2+_OQ8H5?^FV;yBgfLcvys=XB`it!odS5?_0 z@zw?7TQE*5yRHbiFzQ9{5H!SCB5pt^!BG#2@teKhq3B2?J)1WEv=JQNedjq?5VrNZ zUeY&}@bLYIgv|?6+LiSsoNhzF{P1&$s7I513;I$DOXj?|YZJP#-KA^f+8xF|yzfUA zP#fp^z|Z1W|K(-$R|rI>2?v69NlLPjw98nRV#4V#bdQAkWtjWqsMN2d1b&m6PLGjQ!ND=(_zQ_|3JYmFa(Kc^ z=gI+A|1kCZS9z^r>0dDV_a`KC|L#8WJ=^aWa>C3U+61WOyGch<`8(T&3Pas1~e_OQ0c=8e1Id572cG(Vk1ORngrS>h^(Ke4V6>dDCfwuCCwyKUg zhFYC(-^tjs9GOj{`qGv1u;j)I=|eGmmZqznaMx1KyWdc~76eQpj<}N+`m58|2=4J^ zw0S)QoL_bJ{~Y#p4&e-oPm&D%-7pHWi#QX%F!ipz0^4NW{T6sQe7s-)T(yR>@O-Rin@Qa-qFSbS_yrdwO!PSf}?s%`iA6OR7(U_3jye z1hB#Gxg3(=RdO4tV=ml~GjrHL@=AWLtvDY;ApaP!N0T}znW=$0Bw@Fz-wO2F6*}JV z@A^G;T7kb%2KhF3zmk0HjHlb`u0p(yki{dv*laEC4v|dJ=i&GZu z*Nc#;VRc;B3LDsA6zp%$2pM8yI{1f#VC6OgEw-&)nQ@o7GHpPSPY@kaHz#0#22N=o zUTADU1Ng^B#`1g`XI-dr(+&0B700<+=6jYlWo(!ydF|tqSR^8thL=Tn@DVo!T{@Y2 z$-(w$GcQo{gQof*mL<3a`PnHuXNfQhj$k7>f@J5CM@zt z1{)4`{T^&TeNuyE?%m-sH>dK?=w#43&vx2~M1*n016{;BkLN;e)n~2)yQddZ+iFq# zoX(wDHow^FrM7gg8HWtrFSq{r6Y$Tuz=16$IjorfF)uGqPiSl1-rjRq`k!amT6pnZXn(e(8L@-9lhJ+Bl?$E~hju-1`8+u;MbhhTf9o)F>Fr--`CL$}u=0N0<` z#HOYWo1>zloWrC|>g42ZPtDKp z+uf7%rVjf7kRG;g9_#SI@y^Lr{u#?*@9qSVBebVpQ@O$Ih5fH6=Ah-j((}wM!<*LY zg7@%RCVEe<>SWrY@$e>~gJMK2fu1#*TgXnM@$xwqp0+4XPR>ok;MGVGCSDSL)=j!? zFeyu~^~zqejNC_tKv5nBFLPd3NkoOfi}#v=l3eS|!)pIO;sfiVm~YE5UxjII)jU5+ z5m@MkrI28fa+=HV_b-q@xvzGa6G=ZS6jTli#Fa4xOd$e$QoNyIbmtL!v8+iqwqtjh zjobR`$0e$}I|J30Fd1HVyMC)ueEVO9L009c5$icKyx&8o#%gk}m*!Gw9RBLCbQDJz zDn`+8yUUzB%U;@YjA)Qc(LS~5r&rN6q&P(u1iFL?>~8d87AyJgHp-_XA5Gju%;l+} zwU`N{XQ|P36WR^+3Bo}iRe8$h@aV0%`NF8RhAcKuPRiF8rpKcfYpSbfKL?@%7T&7J z$uohIPEE)Wp$zpRD-rx7LSm*yR{?cymApEP`_p~G*U`1$a*P-Ts zya`?b&CK7wdrd_aeZGUya+$Su`K6z~QOC>XYZioM&se?kWs}sfZf)lPPOC*DBMU~K zDz0;H_-47KW9%m#9U+%SR^H(jA(sCBuC5#JYE>vY0%EsrIDzzS7DmslWgbG5XRyYs@L_ntLAfNd$+xj*shnc336=I>LZD@d=+dx`rse$LW96o_sG5pzb8BUrcmuD-tqL zGYEW`8Fg>4Aw=tejRP=&J$B%AceII%f2_QZhS+6Wz zF8$lg=tS6Y!GKR8={NfF#<6Ru({}Bl1D8EbaAE>}p_54o&47`0UQ%u@d8gI2-^Xwx zPi?!LkKY+T4Hlj!c?eRvH`n^tmrGQ1zu>`@%o{^@aw;DYo>}^8vmW355Xo~4#ROBv zL?Ox;r2u7%lJ(BR>3FMYvd2PgKWj9B*zGJC3D!~#_l)Bgsoq*CUMzOl>Of-a=|EZc z+&|)4zJVCBbngEWiKUXYzj6YQ(L{ukpEnpxbp;1fTkx=owq9i`f!zrhwl^*akk%nN zpE5%v+pT1MfXF3lYqxcCVK8RUfyvBIPq*!zPwF^(qpV+%zxfA|H#kinE9GByw|Wbh zn3$+ESPwH~v${<*j{y^p`w} zU@?a63ZwvAfeKfzE0kD>_Z4`t8Uc!kAi(yc*+1JUirx88&lQ9i<;vBnLU+Z(rIv%8 z#mV*h*H|;%c#=_PwoOInmnx2OYaO{?WCF1y2wz)ITEru_8&Oe%hHk5MoOM+y85ERX zu=B}^haF=+h2Uz~FHgO#-b-kEmcO94yz2Z;&!o>4BHGuB`1*UIeu=Vbv{x|^KoC~< z0%2Q%3!$I)tLf9@xniDcB2;+RIN0AxdA_5`wUW$MKAg~gv}qZ=|Eq4GhkJH>V0Ib; zzcb$Uq)qMV27XCjrD3gZ(L6 zfd*=TeieGg=`IX3>D@abKhEw3yA&Tq75O-(e0=!{H+FXd3#yJ1IMl0EoL{>^Syl3< z@yQs=%%+u99cBOC{N!Nuq&i6lOmIHU-gCNnb2!P4OKqE9k&_jET~Ps@Gn$-)0xIfX z2-tt2zJ7}Z_um*8(|okH}B>s~>34x@tsfK!j2&^uPU{KsF=a>&i^o5f)_s6>l;4D+7mO?49 zS(CODs)-e(KR{5-qn!N}YrkxfVINH&nl0DY6Wr@H^w=5C-!H7%07u$hQjtDwxo?ad zP&Wn>EJDRabR#y!T+P{Euf+f zySCp^P?S~#X+#VprKCXtL1{rcr3Go}76cS20cj8^>FzE;DUt3HhHi!)&K{rltnXXz zbB=4_TGTKM|NnjOd++P|T`txvho?P>T-~E=7v$6vl=3xjitnKuM@5$U#=35?d6>&k zV)PzRawZ3WL#A!)p4Gs9s>?I}7YKw6D5C-7rv7G18_*nJnD|vH4&Gggxi={H+dr`N zk2XsRcT@2grL&=f0U}L6xf%Qw4zG0Ni*}jn8LgJGA~dx+v^jZPxgo$CXU~Q$pqo01 zdl*BzVa-k9gAE(9owFkm8Lj zN`K;i+A57%)_90O;n!`Q@T087#r523^$kIIFlnyenDJM$7vJiYvipPF7#VI>h$|M8 zw)ZMjzbY2vs<584xSg!lm5mjfDKcbO27-7of)gJQoVbdeP%R4Z@ik6O)&Hlx0K@tFa&mGVot^*c z!BH;Xe*zSG_$EjX2$Jc=2P)4O2trUD@9rj;ovoPa)&SF0So{-PTQGvGTZ@6uA5)Rq zNWic!XkcJ~aM~girr9F#Fc4fOJ6pkUt36g#K70S@VreOQV-hsarAb>R z#)iH5S-hqcJoZb>Z`5vD$k)AgE0^chJ+~&DEW(p;=W`hqZ$vi9H~L1!IygOgBy2Bu zc~T(U-@j+@F-7)t{fiw(o+m#_2bs$_R5JyjCyTjCS%d8ujhM72lYKdeBssi7&+kXL ztWnH;3VSkT^67Z-mbqFE18>PaIVTrQC)@SmJc(C5k4CLxom3_>?#7f9Tv%HdpFS}M z$P@7No{VaCcHZ22_~r>86s9fEnSPVFmy9QxSV#I#WWriZ%DEr6zBN6k6~4z3y;Wy4|$+X6y~@${A|qTd4{ULw{teMgi6 z_wD)dhlk_y6H6>AM6j@E=s8oTRHLJ#buu%!GO|9L6y+98@&w*pxm~t!ei4hV>di_e_HkS$T#lU z+ZOrP)a1Ef)qdbx_EWX3dS1?!-Xv>^jd-?I0H&F0WTiee>YbPWXCQ5c zTBMnfg@bltJNX>H@kekL!n-rNGdxD!#|NtpVP+MA28fjGG!E?&p4-U=fmMm$2BGFr zMWx0?V=116 zfme8kPJjKPwB5?g`F~f6OUxIh4dw^i!Fs#iLZRbcIUP7Y+I}H8uf0jqmJ7Lzm^saO zxpu-7@_M{R-4IjX!MZWzj`Fm<3f`&=8tz6ChK&%3j$7=NhaW>_iq*ud;J7Q52(E%+#irjj=wre;wQWC{C*G}b!7M`s@niu1ja zr;N{Yo)}tkVKg9hYkh8&?3a6R-}~kefSYuMIc@CqI1NKhP?b;mdN1>0LaLQtRe|9W zWp}&0cfbjs;qZH<+M{$79}_SqVZ8*J!!2FQ3ZFgma6cu?4J5J|))RjN$%$Z88Fw3q6*uX7yxF!~W3yf+Qx!ghT} zSTZkNbSxV1sVDl~=c+z}h^Y7~k5%1N_3^6@1v@UBP3AUnalNVHN*H71vacVm92Dnm zX_+jv9Crt~v%xuzBGE9bGal<$u5cRg*?0 zCSrNJ3uTXQJvK8lBMJ$`lzKeShsBrO%WLxWf z(mCAYKhOGET)%+J(aB4SI>aq)e8rgMDerDdtu}6%rX}jo;b7`_O8p_(2Z?RFKV@c_(9oF5fsx-scV^z^UW(2$&pgPoSUXenTxWv+bgo!)=k- z^vM|PL4_ffO(&MNkkANnI!|k} zG4NV_hUt_X(_8ZixvX5H0|$q)i37+g9{QeR!Qm#meo|Adxp6p#K8xkbM|o7SFv}DK z_2>#bulas2>Hn)7e5-pfMerZ9^8fwmXE;eoV~7vUVpoyND=V6OlZWFiEoA%qah?a$ zbRU3S5G(o&&+TB6!x`R86p+hVb+MIVDPpf*W4U|K@X2YtG;$qX@psu{hHu)p&>j0{Ds&7m2$C$)Zms?D~T8tVnLh*KA9JiKV zAA678G&C5Dy1}Ocbv0bp!rBYdm0TR0Y2l2xhue?8^y#^_X55E!(Yj`PC4hCVr9dEKL6`y1L!xSo*;#-we&l zJMi-xymi%KO2-A#9)~3r4n2t3W`*kLp;7ql;Nd*IE!gxs;%>+a-WuDpEn|+M;#{0> zAzaT{`}VsD)kP4+L6-z> z{(AQ0=kch(>u6zRml6g3g*qC1HiF_WeIEwE;^r?k+A~;Jlhb{7J`iq`{&lJj^buEiS zVFOW?YDvGmVDgt{5qK6jWzr>%(n>kkxVR$wOYgp>{_=ty)j%{5%{#^Ez?(!rR>Z5q zOFuS9S=)1?WACe3BIT1068?L8loFD51zwlD^Ih9vGMOP*#?(86A|1t@`D<*N#{1CV z6*ik?ufA^P!>sE*eM!$oBH9KRQ`G z%v^g@opdzaoM2h(p%*$6e=sekBF)2OzbZSFV}6{iUUpdB-teSh?;Re#^_de2`#mZ=)MHmw1z5|*Ra0z)G9*q8E-NRW>*9;d&Y1?e$mqAC!_ z?(5dz{}3X-WNO{-pkH0K?a${r($v)LpS86}v zKM%b`D(qvdetfKVSvFjDcx9Sdhx#H-N`uZ1mCcZ_vULKqo;0PXOM+1$M?gw4XoWgA=s0AIxAL+*K}eGfbdZnl=h!f`DgyLxpY zhO7PUv#0hY7au&xz$(AZP&RuVkS{P@p4qhy4$B^YB@CjpTY_5nh`>`!iQk zF>xKf{Xv3FGdkkb-eO~{t+^mD5qq6Qee4xYMo>{*07dkVy$=Ay38NP=39b_SKDfr% z+}_wSbg?#Ubz6A& z;*9W@&*kI|mkT)?t~e^bP_Fy5dz$Vr=1mJHXZikC819JD>FJOCy9jJ4GsB8e#q+;@ zmuu}hH|8P(>;#DHbQJ2ZSv?cTAUL+po-cos2G#d%!RU}q$)Q$4vFoF@M4M!z#a@#4 ztzX0?NYX3M_1+9+5R|{)Y6-SBw83?3G3K5E%Rnpj)Cb1v$FAk$Y>A*1UcF;b-Zpmw z`lN$uF;l%`B+@u42ZGYtF-Df4;>%`Sr3i@A3KrP<1v%qnt8G1 z{&b~?eahgl0B8|zSB&r-JImLlo79vL7YjJPb@t{O;?}15rAS=TT zJqQC({xZPrs(k$QQ;^_KHG{IGhHQ&gHxXl5|?05{Ejh;hC`6jd_ zo99L4F^#EP1^e!fuk0z|a$7uiy5oGA(DM&C)QrZkE({xoIeZxByh1tilg0PbZW|cV z^^y8Q6_4*J8}uf$xkM{#o=@g1WJUS2T& zKD%NOIyUslZ@R*?d7{Z+J z{Y0ASNxkk^=v+`_wE7&u_OD(2zy5orz7Z|(U#qm3fbjsM=wcBdBt*}RY~!p>P_q4r z2-&zBtwJJhTsjm}UvxAfPQ^QhI}I zXnyOJrOgzH!X9$(xcY>_X8t5iI?uoI>4yYYiaF*uo65oqB08UvKbvp481Uqr zm#vs2u!PTvxsb4_J(_ow!()1S5zg&g9@iCaraVazOBEx?5(~bUPT4KpeV&fXLrH4R z&G_MOg&xNGx5fh@wpj>6jRW2BxPuz_9p~T4sB8R2D2%LP??q+|+r(7> zy?CYBHDjG@#>_4bk}--vs4H8TUgk>-UbvxUn`e`cIRW;^=MqO~qYS>orRWD9?Ks6_ zcPS>m$UUb#QJ-I&wjCW2C=x#b0Aj)-0b*ZetsbC-mtD;n|4PH)2AkQPCQX4Ddl=7FoB zeu50Yo3=c=dBPqV6*}rb?INx{1o+k-8&q*SglhOfwYEC}k}_Mnb)MAzL6ySZ_w^nu zH-C+gMNZt)KvMRS+xvbVH)@ZzjXUp_8UHk2VTpoSj1q)}RXw_h-M)@A!FX^uY`!{9W~9AzT1q|9;1j9Ge=jiX zmpHq+>izZB`u-mutO=3ci;YutWA@wnA-cgGL5ffWEOc@c_JaxsEzI zIz+*K#D6BiV0Y&_>fo)2NW~Drt$*`^j_wD%yf^LmPfjwnw&IZ)EO6^|bW7){rq(B& zSre`|wA}TZ&}P$vA1-zhPb-ofiqW{v{3k<23tFSA6S$DL(1aNRbu_fWCsSUUTad%w z#iS?U^_EFLJl->>-9l2E?C7FWaL5-9W{qM@_K++ja0nY8+vcN024dm6_ru zp(eeFI7mOy_8wDw^sEL+N?q(n+16cI-20Z5p;6?1Cjic;Bh7dCYRv~#1_rq8a-7kahZQnslVqq?OBe!i3yk#FT&w-r6tyj%P^ zNBxKoK750hDU3OsL5HPfKaOlIl=bP{Q_DoF4l}RQqT_FQZ_KbE%gUu_tE7);fv}8eAg+0v|ZV?LbzAMPFC|ar=w+AnHS1i5kNkM~B zc(V1WWIKr9=)By+`%iI*Q1!;*+10MXPB@bX&_M8X&BBptDL0n6ys|21J>7u*MffOI zRJ7|ax0O|S<%!ig!N5>%RY;jd#8;))?`(l?>%5f0C_g_iNq1r6Buy;cU-rHqo8ImA zl>trnSoPp!W%HuDHMCMeRN;z)pSlDnV0u5$siGL$lZS>Bc1+kHKD8AP5YXY1hw88OuHFiJaAb(fIC zH5dcz?yZGFc3si9UR>Wb;jUq`bGZDA)+b98D>ede3tk}e5+!(YCM;L~g?>dI`}5tk zpJH6?4<*3wRf%IheBwYz9~zExWEhS6f82S38&28N_Dc=?+PU}EKZ>v{Ag*%xi?eW`C&WKRDcNgC&# z&OLhzaZ0(%dXqI_`%YR?mW{c2sbw0wX!y05oqO7bllCi+kGhek^_1puVUChDRvs-n z*N7Kvv1C5h0R6k8=9}Kc8DTRw}0R?bn=wUDjZf z$zgaZu-Dgp=Z)3SL}cro+TG$D=Q=tobsfQ)ZqojafL{ zR}d819+y?+TTa;|v2ia8C$2}Oe4_5bBCRqo(w%XvP5)}t-z;@f(kvKElT-I5&jub& zhm!L0vp zd-VX`+A=v=jw5BiYG-Tvs;texHuRG(@t?DQjbUbH=E|CygMVBZjIrSkD{Jr|AujGa z2nY9>!rXJO`+xs#tfm+Ey)B3_M2JH+)}Om5Ym<}xRh;|xTN(^8_N{!;pvS;}#aJ2kws7R4(mbKFLgc(E6S=E1{j^)eEn!JZ>IlKqHKty=Fna z9`DeSEeNGb*J7VHWygjtq1U-!!qEwga7HS_Pfo1*RhfrKV8EyA`kJvH+^mf-myDsl zAiui=emFJ29T}B|$PV2xT`vHU%H?jJ#AE~l0MrKMt3V7buXFoy6c5ky_|uyDylLP@ zy?PSsqL7Z-$2f9qxKuJc)+!_bO!`UQer;G{$kqr#ueO|ZDm<;9W)9s?$#`Z^+KVoj zzZGGAu=+{>8G|yHxsz+S4Sm1D5%umDv1$P&Ug_lP5gu7Lo=feg1%ozV7POugW)7aF-DEZVy(8^O6NKVree;gJPr~xb^*jTZ^mKQWBn!)g1;0v zEWtV*Dl*oaJN(cX#IUN9$DyLSO{9sG?(e08KTGmo&xyA!uDk}t@AlHUfrdj#Ux`@w zQ0|}j&{Q}1EVf)^LJr4H(iK&=gZnoLbj2@XcwC}%?VK*R;#|xuo~xLD)y@Ce{xxO_qhBdziOKw{2CY{dMFOlQ>qLr*?k3oG|F13O-@DHa z=HfU0_htjz5VM9BULn}FGd!RL!;JV^_l4{dt*aRw=zS9wr*oq5Xq_KZDQ^8@&J6pk zWIRL;t}LU=7fAsI9NWIX&v}E4lCY-O51B;OC1eGG5`chx!o*X{GUYsv4}dt{LnoaI zyJsqD1;`Cz>a|a98-4!gYoigt$zJtihVy+YI+K3GdZLD<3ewV~^(Py+kXO`!C+>ep z9xEnuKyEW9al!L+o2kDZnPda9H~TmRn};VRH=aMIsqiT;jPm<+Z;d@{=Acy5JxV>8 zGx2M^Yt?m?ZfV9U(|8Kw<*!jaO4&ZyCU-ekj5_j7aY|2~euP-_<<&xq)egvg@?7s~9R#FwC00Ie9k^Gxt1wVD;|&pf;x;qm&Xry9yE zO+K4!@%xnXw0%~-Fm?n*18hfgkxbhBVR(?=I`{w_7FNTXlRT{avDwt?vJ zKYFIk_q(E8HaZm4Z1$paTu)YM84$MyB^XlyN>c9BQ=C8iscO*ob12&VJ*k8%m~uyV zdD|fJEEyOp{guT$4UP#+nu@6mXUah5!s-2w`z5|tNAK^zNlTG&zqnRrcw8s9KSimwbTdqVvvAa@7+cDN&x)vPUy-4=77%q~u}Tv4viy z6Km5|t!+^pq4(wS*FFy)@|tz9oZx}NOjWlBO$ZSIc7dYLIAn$SC_pxki*_*JJxbL2TcZ4aacmgQZF&GAcHMZEVV#_A<*Ta0z(250{HPDl1uOx(UXHb8^gDfxuNb zn@*zmU|@?=QT$I5q(IzbrRgYThbY8kc3LhCs`QuNF~Bv9{ARjv{<4aefO%$L3Pks|azb`p#B zA(8Yamh)_;2N*Q?v{&+PV~1fdpo8KHgXK2+Qz8V&*6sF>^AHxn)f)`fRl@v735)E- zqy`Y8GMwXHGCQ zT;OiAF<*CBhFZXIx-PP)`gz z(`zgG@#+_7qMCc2x}ndI^XeFZgB=5p4knOg8_loUUC%F3ei0LVILwVk{rqCI&bR9A{rbL ze=ab|##hnkt#ugf*rGeWxA?rf+^WhwyD9p)XuD;%tF#>A9&+s``*pwe@IM8DW(wW) zPgc{GwQe;A&<~Ln6fL?4$xvAb)MG5+vzt+w>Mk@`CVOJ~rKSP($EAs>4Q?pL>v(qq z^URvzp>un`x2Sy>JVd!){n0hjd5995~;ZnyE@zk%C5f~g$Ng&B={SubZt ziZ0e3uRMSZrRMf_TF%((JUlewG-D7nbCu7Pn-2U{#6emQO}eHruQP-=)faMpe)dwE zrK#MK;&8h?2STJY$g@Zm`%Fe2(m}y}Z(x2ET=uAB!$^Fj#-~QB^PCst`pxlWuI3JD;q-p!tG~VqQ=$ z)W5kZxt#1J;C#$#nUpyoue0qC(3GJsugO5mbYHBBADT?#Fv~nc^ZV zFUuX`(Rl~tk-D#AUwKP>yW`1~`{63lAx+fPvm#!n98~%1%@aUQbDr_3)1lnhg_R!z zmV)Pw@876Xqx!+PlW-o|0KdnP$qJ8c3QL-fKJxhsYJ9X(AH`5roN69Rrfa|Voz{EA zoZD4X!R$YA%^%?7mgu>mw<9$p`f%o(sSxypHu&M{U<7gWBhFp|z`7<~{_^6u>h|f` zw6fv@Zlq31Fe220m$dG(+v>V?$nV+u89e8%+KTf{7Ko1{R1<9{^w`b+V%Yyn8DdZy z=Xbwa-P|zl_ctfPH7=ZU`Gs*X{(R&D4q&X(Lr#9P5+8(owXdF0iZIu3cp702j z!nlUsyWKxkx^rX0#f$pey|)Rx2;ly=48iW3Lhe%0eX18lc(xM5wcni+|^`3 zP`8lymmiDb6wL&`Am?LvxycO=577iR2HiZ6FwX8Wa-w$C^)25&vEO5-$|pRJ z@ow5}p1Jnds8C^N+a2?aSG2^Jv2TlPc$$7{TCWv+yT~VG8Mi2)-A8HQ3{hqqbrlwX z1QqcWLXBLw@?JGYR5VvSx#3*C4bD8V<I+;DA$VT1O2&Ga9o_i>7j1z^_8J3*RGQHJma2|fMyhWp1$ zM{jPiSvG#D=>7UI>*Fq`8J_giZtNo8Bw3`;JeZDvNCILW1oxAlH!CtQUfz3*8hVTt zZkA7ewtH6LJ|>BrQ0(`sVh@Egqz?g|p_(7F$0DO!2QPWWId;MK* z%YVfAZ0Dv3hmE1F0|0FAVO>E2S|Zu+Ssb^+Nr5iU1Ss*Vh%jl=7*1onDNnZ>Us#rV zwS869cwbFc-vK$G3-=>)F=0f z7DTs=(O`#F6Nke!MYZ8~?R27n{h2{A$c+Zk>o6W^{$n+?slBqQ74~(P;w`LrlCC6% z35lcaH$F&Qd@pvv$6Hzrj0929(IsP3Q@m*5OaB?uw6(RFndwh|V*bBohbt>9>XzV5 zD6!fBYnKSnV15)Q;Sm!0C+qc|efg(B8+;HYedETBLd=IZFhDsp0h>;>Up5mexHVHY z_97J(6_zq)2jUGaca6QGc1_7#lUQrYZ-p$qO;5^*xwR!(p?PH?04PQ{h_oaoiIps| zz%}ebhhj-k6yC@y=$_xT**uvq1gM8ucK^WH4*tL|LD!L4K%u@`FOhbzrpoPAOk3Lc;&ogg|7PS}IG)kWsfEaHF8}!*G_ncz_xvT)4fgPOI%f zbE|j|EjcFo3kz?W)?U2RY``O&Q98@OTmx#Q!qrqfOq&95)?byr|7Ns$bsRvr-}H5~ z4g(oMaCdgafY0X}r&`y*vy225iLuz*dz5p&Gd{Wy%<&%9utiD!J5frJCPUw5H1C4c z4)Hkoh0aN$abylU?jkd@$Ro?RAlrGgq`5f(3<3YueIC)?6kK8tjRJXvgx)mGZ)5(* z6B7ck;m*yqDP{PZEOsU3T`Me}o8N3e8tg3vLp(DAfr-yZ5qF84VMO^?XqPTQ_6g|h!`xZuNUNZg z4-Wm=DX^a!-4dOjHXlj!2!mNNCUkSQX)-X5-0!Q8F7fCUMV&iM`wTAMAAEjzW=Z|D zay74y_pHHJz->Hbn3xWeZ3V;KQ}0SNt6eB<3fmHM--@Qq#)t_UV}auRXZvbyMA#!){X7=O>t#fOx(5LQ(tHgTesN(OZ4j`96^ipx^$`cZjK@j! z>~8OU{hkDxE}`69eCy~?VY1-Af6d*|F4Tybi7EpLse%?jw1O8D;o}ohSaDPAb~uPO zKduhvVDjE}?BrNo0i`kAX8vVZkIpAlq&l$9&7PRH{_?@(O&bbZy4 znnu5Gv11&~4EEf5b+RAcUFN40*M7MA;ljv4&^?)yYO#-Mo1?-16Z`}E&F?vpxA$LO z_~>B%(#4&2!`1#q!l%F4*!u^50K<$l4Z46Z<-Nn;D8mG8f?L0TMlK7|e-~#3-LXHz zx%s7tsBqV8Vm+1-A9TOi3<-#^h!}f^w5@rD%y%0xaqQ;}6VWQBFtu@#ntANgsxQ(5 zL|PMbZwLd6;V`5kV_EY3K!PT6E*&feY&T&Di|a9Tk$X2`uBDQCr^rH77ULPhll4MNhZIQ+J6BW*vsN(m z@97y$k?lP7gl)S#W-C&~+H9Z;((|nHud)}vMzekaFoMtSa2lBm&8>|ZXjds+K)^($ zUguFh?=Ub$-PFLApimh0(A$$cujs*lSJ?njShV-0k@_y0!` zrP2FTvKqr@FIn9EoqD!4v}%e#Jb$hdxLiZlBZ4Y)xbcKnYq?5pw>xe#cqRWUsmzND z-_4Z1eS7|))x6tw^l7V9Vd}k4ccJoacl2SITsFCKFbY4cSI8b8#QHJZ4~PLp+t|`V zXjg?1ZSi^565nKteApz7*n%Sr#?T~*WahkHK0sQ{l=}u4H7_ESg8#hju0nzp@|jR% z5L)@&EZNMJ_uB72mzryfehz>hGFPHg3?6`#nrGadyj-tIC(aXu5EEN>zyob6FtVPBdQn<(fHm99pa_W->CPeqnXz31j$eA7KHAAe~#Z3g%{UC~E?7fc3 z8=B+ne_~$|RLi{3#gB_PmLv4fOGPUx=l7h(`hg?ANwDh@m_)$@fv+WXEsp9RWaCHG zZSXmWS3!kLi?-tgivfT4d1J*kSm>u`T_$%K5f~y&T|dGS98LPM#0OibMLp6n25qSxT% z)9>ujGWpPJfpN|GTxfua2Yj*MzrALPekS%8Qg*Nfn||naVACM)MRJU{uif-_bmbqd zf$F*!s_UOEUACKWsR3&paxwK)r!lxj2+6thm?WkWwcG;fXWmmS?)u0#!5OX;xcO?B z+zyED29Uih#QH*j#;?~b=cunnY(l6Mvj_X5OIwYa1}^R@p--M#(kPlK?8?C zzPF4-_vLRCXPoWzZ<9Z4?4PNfoOrXnR^qz_y+njJ6cm_EL7eF6D_G4T2HvrpyvEU( zos!l5&>N*EqUtF#p7g(kXuGF$U<|5MmU3_!XA>kxsc zC=MV0hu zDLB9TMnqfz6oZm-jxD6({n0Am5$9o#SZkq&B=jf!F zZSLvJH&Y4zzkb~_ss62KdU`m2Lxz>@OZ>iIbgI-Y$Q;V$1O=aj282%k($>}%5K`0A zBjT;dzgiE938DY;UnHibrx%SSB_-K7AyG5}P!8`{-^ZA0L1vsp0bvelr>Ccc(?!t4 zJZp%>Xp^n(r>CcfbuQ#0-}!duY(gI;pOTW&X6NtoA$w0xPjYUvF!WW}M#ADFN&gvY z1072wB@^+B7UUSCm{^V}(JyG#k9YCkc7{)? zS$bo5bRs}y$EJOd$Nq;G>G}yW#sPB4Z!_^Hxf1~$RQpuxY_pwE< z^LCOM+Q0vkuH`SCC5cK`GSXw>5nx4Z1+y^NAB|moI0D*JK9my-?BC-N%|htTt4-Z* zw=dAsgiUF@+&Cs-O1)0n>FO05i8pKFmGdEql@pp+;{$1pkcTu<#@xA7s-xmW17U!m zB!a|uA<8z^z{$@hg#}OsfV85zk5bhGZu(ieg^;-$mcrJM_?xiu#^I8ro54ziug%** z(t9_>nXG(XW|9#gjFhU9vTBB4j{F{S7&_g!S6@6(qhB@?`5fYydxp*39wgNe_X!H| zN&YUacd0e3w5LY5(azg^P=sS#LIb$vrt7!<+2grd4;y^TMLmz7?sZv*lI2ogwSA6d zNAe4R@slY(bcTTujTu||dR3I)oNMa6_iCL>>Xvn0Kpu2CA=qkpecDOv)?(eGi+XG$ zDF>U!exW<0K+Jcgsv+O_h&Mf9ExZRcL*<(3~${2Nd8s9^>PuD0p!WtvJ|?n?I7?kFBoSMarl<^!dLXbg}#ud8@K! zd@em&9>Yb0R@R{YG}IopOTQ-zi>ivoX`UNN-{9^DpS~sKBSBas>$c8|67{{Ml5J~H zW-_jt^DnyklYN^j@-Fc<=6PQezT5hDb?OAocZ%M-W^$t1@iKQ+8L>5}9AR$B-dfC4 zQixh+=<;MCtsw6ngRI&n5IzPZave;(>q$tpP!SqXtsw7ej}@%oRm~-TSI|(8|F}eW z6GBQ81qC;TohJII0P{>bJ8Qug`rqXkri~d&b<#%ozjTK1ZV&}$D^7dJq;CN2Snq@p zC4!RjubJ8J%d?o@zo5Yc(hT)WOa0X$$w|3=hDj$?@`(Nsik?NiFsN$O`Be(G#gI0V zTg31JDX^tmFyY$~%3!Tydt#uv@bO*Q-fo}Yf?2X$k3#F_)V!jaU(|&-OV2(hWf(db z%G&5UJQ0@9=W!Pzgz;UAp6t7_#MxWVqLkXqLGm>4?CxdP)!~=J23yu8dgT6}2h4BW zQ@Mcn+l{L|714%0zibCtmC$H>vXXVNM0fgr7ip-6ASsoe;7{zD1)T>{Q&Ax)dX|!q z#OqS<>7jrgn7>0kg)ey(2(<}ZCd+(T(u}wNf%>NRiD;bu!#m1(N)FEX;Bn5(v}JnQ zeze3;TGZ?dy@at$qEtxmgF8$%Ow$cC0knrk7hMTBJa!MV&Oh||z3?E#FLAU z|Au@1t2wrMMMn=8ZjxeO?7Y-l-j2kg;_Yc0$kKuY;b7&=k2Wij=rg}wK{-%FS z!hzdna1(GB`P@{w3j8ngdmi0=u_+{asLz7Q>anhK2-zKhWIdLI#JuexE#&VBN-Niz z0INrje%bDsiG)yIA+X}Y_${c_ucZ9`rJvjubkZ_9N*OBRiUcF1b297KB((o|D{mMhU!j?L-^h-LJgn%;@n43H2iR2CH# z`9Q9Kre#ntY~3tDvATG9K?VZ!dObc4ef< zFzlyMF4Cks?hS?3 z8}+1gprUX2%3pU4`p?A%GPl@<2haj;qfOA!65m|%ooeCXF{|!u|I)HK1H*2J$0X+{ z5witgZCV18w`}&qk?Aqrz~A&DOT2f<`;_0w^-M<+OVLZZj^7@661ab}t48=8a4=D8 zRd@kJNNBXO&B5i9u^~wZMIR=F8;3Tqbs&pZ#8GBz*S@hWhzQ!JM(*2BsbLI)kJ^W% z=H~YvTffC$JX>nE=qw`I_f?2tMtPBh{@Dwezcz~daBkxYPL1t0*jz2bbr zPt3yn>DoW4a^_iYRJ7;O9Up|6@Ak90?h_HNHh+jOFh!}j3|6@XC!wqIvNc*A1lu<1 z*SnA3i=Kng*lVgF!i`ap?Hqm)Apr{fB%!3A{vCtl+@Ma)07M9Lp1W%g`pxs4C^SM3}N z9T%M=ub#CP)aDcEZARar8MRlf%1!M>FS5DRS6Cg*Cr{`8P;qeGZm26rS?Z7rEn7j~ zb=|mG{<<5h^X>lJ=_1FoKlyaix!nqqZ+FHs9UM0=Jxyhh&MeWaua(p)XXy()cdijR z`pL87JW*}eG9B9KfUJ`IytgabpwcVaAg_ns@jPqOd%Lwm-XgM{J=CxliRk?f&9ckRNiC%{#(bE8_ml`f7JPo{D_98ul1M8;jLc6=; z1IHcGb*Co*Z&8Y}@>VGR=~a7+ev)g#C%xraub!?a*s)k!3`C=km0i!R-5l&TpXzML znAvgX$*!O_H}7O+hyy9A3uBTs0+|A>e_LoA-S7*ISUrdeXGS@}9L3GWi zdqQTzp7*<)mA$CxF;R=jg175+zu3=r+V_GQDuXSw5dYV|ODh{2i`f6?11ulc$KxIT zc7(1fp#O?{%u@*qc)CApX?f1`=IWwQVBg>Y!!zEOQ9IrX{J z)-W}M=Hwk3c`lSOeKsjB;CZ^(Vwpc)n)RZ_X*!ecq)2XMP207zJx^CBlR?T;Oz&ij zqi(C2f<{;%jalt;Tte;s<7~Gwqvuh&&a+&{sEXbth4Ve}^>L>czP=>c**oTg*_y)p zFC|3J`VVsQ*2fSLQ~@qs3!B+70WVpH&XMk}dwo#?E<2g~;|(5<_0CrM;t9(uty=tD z&qhKm{|w~?RV?SV&kym|Z`XPrI1w#Zc%68&X_LqA$1HSO>}{T(wHz-ePuZ#U_Wf4O zqC4~{>!(URai=|NRVba_D`HsAmB0H(AnF;jz@v03B$Er%#BuYcNcw!i)BUDSkLnTo zr-m&N(h9Oaiv;zUt$kH@7R;G4bFwV3G=$h5C~2iQIEduoEt+zjyrwLF zJUqyn^{3bC>|`I6?D4&BnlI(BvHQF`;ET#?5&J|zsE4_)5b1{7vGzt(!JiyVJ@cze1}Tr|u5_tP7WBX!Yd;sV9UPoFEu zm8#)>I&STK`+Yw=%3i?g>yOOj)4EKDzv*1A2ki1o6FAYAtH&w4`n3o8{mD~b1L$;9&JXDhR(j7*{pNTn9x>~joUAOVXD1(iaAeifweOr~ z)lf>uy!3iU($VmvrqVk>~C;?)7cww7FsVRiA>>_Yb_hOsdhb z$yGI*%iH1l55LPqiyYJPgvXp$VL!H|prlk{Gfir5nQg04#X}&xcOG(cr=_IOrw$L5 zJ5vGfMA6(tH=f|VnBzM{W!~)Ut9=xTvW`a3v17g5jfeuccx;rxln!^4$f^3U7=@9& z$!cn9(S55+OTT|b*=wC65Rd8YtF+Ct_OsH5GJUomtOv_YIo5T#JA2RyA1)KPZuL`H zom8gSGyF;utrjh^9ZVele+c{PxT?0U>y3feE))b23p~;xEn$Hm-O_?ccSwU*(MO~t zq)VjZfOH(dz#}brkX9N5q#M4u4|)aPKR$ks*Nffs@t8Q!zr;=?a!_On^km~wD>!Mdw)<@)ju^9eaS`owkJV9rjV*HiZ5 zUu~31NB6aGDjHV0Tw^laTh)hKo2yzL$X%X}W$B!*_nu>Kp-H!0e^1UewqL95>QD_q zU`FCaltaR;3m4%1d3C!Zg@h6SeuxoT<8Pj~ZHEjEy_BN9tO>3Uw%D@uLI*c5NR!fQ z&1=ia(M4MN^`C5e%VTu1_Ii80QBYKjJ;7kq+})itKTlVvPH0`C=I6H}oEp;U@)=mJ zD6`jmdm6g7+6++mT+{Aj7@3Q&P`ei@|w*%aHS=+kh?Bire*dX>wH;_?g{f)l9^#ptKgTk z_Jl%g{#BO`g^RIJa`UpR>S&6;T-D4+@sQ*9#Qfo!mB*f1r<-n`j~`uj597XKpyfhGkj*W=%OkBRl(<*87lT)qq`cUb5UBwSQH!DzA2Tt%mPwUY`9S)RCaSKBA;? zl`X9;wBQq#E=X*sADba#a5wjrxq*zq*K|U@SlB|@9di%S=nag2N{g#j+d>*2-<3OnbJI*9Tr_EO4p!* z3p0BvtAcWu*LvK<%NCrXzr@~Pi{xQTi;WfY_LWXf_>^c-AfBFn(8%08u*Yp^W(rrD zC)NlFz-IBMFW673`K|<~Han## zKCdbJ*6n7gjZ$h@YOa~a?P9z1K>SRDQ(0+gbm+_!^^jKGQq~`+f*=J>%=j$a9NC{g z>ouE&uVOq#BTX-43CPQPz)*C+}j;_|!su=0uDi);+9A3R6`>~Oe` zM(c9Eli7!BW_o>}gp?jGoDhRopnV}-Yta(ezzsbl3oqfa8MuEwzU}^eR8*o2nAd14P3rpZLdYY4~n|A;XGkmP~WVe6#He;Q0M#wF#t(I>0QNynh{;BrlirvI!UBJ z_?MPcF-3)f7ASnLUd4dvys+dUg&di+WrXU~lAJ1V`#yK~e& zBctUZpRgAdkN!-7v%sYGA*C2kNXkkTixd)kvsfPDuWKkK2bkf@Tgr43^Dx2@*>JjJ zmm>wPB;7Q*4yPHU;O4is$!rwkX7?NK-YRAusj86m*238_XQa&~TPj8>nV2Z(MY^5) z9*j#R$J>1Sn0YMAGX7FRe`<=1sc!tMPVxwX_E+b^vUk`?-_EYGbgh-#{5h(!ENTDQ zWa;##jrE>r)$;p|u$f(j$}tCw*|x$mNG_tx@!l z%%?FQ8k)xSwVmj;xt57mziKh4enn^6ZZp*9{U&>qQhfBYgx zYe`-Xue>ZHANw>YSGBv*ehcZ|-H=_0^>BQ-S|pD?_n~ zu5Y^Bnm~yUq~KY=j;UCd*vd+;S1edrQT#Dm8%A8Pw7%y0T$$?~l(mR87TKPQyt>ph zv%IKYch@gAd_l*+;JT>qp^=)%$pClw$?IT@Nh2n48@qXX4)FGD|z^0Hf{Fybv{4S!!ePz3KAQOsLC|d}n^SL8MZ{Nz5tyY(rupE&|=yn*Q#x0MgHrYx_daqZMEt22b zu)dEV(*q1zOP(-ddR4=1R6bfxRPx{?3BY6$%JK zmaop72;Z|uiGHqOZ8&gKiL0nV{Mh&c3?HB^Gxvi61S_& z9rDG}#To^r4Tl9S!yit@IW#Np9|$bGILtU~-T8t)J@{QGQ*+T+D!J%fNA&8T%^KuZ zCF3(HcJ+8%O6FIi|jpikZS*%oa~9~xAaaD8KR zQZQ~*#^Zlk}Wl_)wupF;M-Kdbjy#N zZx&y2aoOATu!lyH(6crn)ZFKbEgkB6HR>E8_iA#9=T9OJI#6uTSC&^N6`kMw2i#$3!NeBvy?SdpPRZ-cqfqTLb?jX605U8Vz!$$!|7 zk4qYrRa@Id*2ig7r_SBB4stwhNKF~0u%YsxwGVKJD&GgAk%*0jOD}5WzbOICQO#?w zeid3ZI{WC+h-1a4r80BIC-r?wN-g%e>B(|b{|Y*E=C2p+CtY(~U$Wi5zG`dhy4HJ` zdDU{LucJqhV{N5)Nw90&ZsgIPu(|S`^QForMk#O%(^`*9Gp=UgH1p3m-1mTA75`Fa zXIm%p@dq=vO`Y_tHD-4Exkog?ZG)j~nvLO8$?_J*O?hLQF6QMpt-tEK! zp+OcsF<};GW@@U}1dx32v*qcEKFc`nnRX^*h3?WNxF=X|HqX3=Il5l|(h&rq)qxg4 za$upM)15mnB3Th}RY2g(nk5FaXPkbpwKS8}yYwMQ4MCky&G=-tKd^i4#qlQF>el#a zMJ2{6Yo@1ARx%=Jf-c(r(2R>;@Hn~3+r1L5>8&9qMW8yPS&+MDK__EOkM&_5L8IqI z26;V(lZ#f&s!(IOEsWNk3#ydSp+H43BfF~(8NUER9q)phIasB4e1etBx2IrT} z?J1UAkH4(96y>|~Pw*q|#dzv1he77aCw$Yg`Nnv!&DxYNR>qcd=ZN_Lx0`j(?4+>~ z{p@g@2b=c(f(GT%M$WaB6EFE9wG9QRt^Ms{BQK6Bytz0_j8oY$H~Ae_a?Z6y zp}p!QfUHEJ;bYei*Uel~YtUVXMfgxib4Gr(zfe`>nI_bl=%=nA2e#{A7U0pu6 z^^yi-W8>^VQpEi`2QP1Oa`NS0+uZH8;iAV*W7_@0u6PdGL#h(H5MjgriNuL?wY4RM zg;CxyL;_RpvnMNSxwH&(a&j8Ni?3cCBSHMMI0)6NbB_+6z#92iJFVQ^qs;7;a=us5 zRCmCXhC8Tgm}+B?Ndw2j)#db)qrw)uFtV_RR?G~0M|w?SC_-og^oJbB>PCK=4IBy1 z{zKYxS|(f7new*|L+Y5c;JfM-MH74hb)()`>c|Z4t0mVDG8i$@T`w$j1%NR#(kIkv z#(U6Ha%1XTx`DA@M_=i9(O8@)%NRWU;Dj-m&)9m`yn^oh&KFt95rQRQjZ+L_6?EBo zecB{sADmBb4GpVLkwt$#mpNM(<({F}v`DNV4%gW09}vZT_|mm}RmOP8o$P-9~x zBqT+|L`U$i!CCvDbz*BeAynmb*~Px+aP3Y~7`E^Xk@Fkjk3Wpg<+ zJLI1idKT<)@AjW^S$JP?KKFzXcER|@=SyYy3D#GWuK8O2v;6_7b03eVrazc+^0byY zg9PsD5r}%`wiGOsAhi17@Rh4qlQ*V>d`7i4#v+Z(O4k;62DHWEmFv?|8g#2uZP;J7 zPBC1b^Oma2qGqi3w3V60S%?`Wr^`4!)*5>}-?PA!u+UBQ1e&p8n_uoL=-0H1zRI9Bp!rt3s1o;e4bMClzOhbid$m|3JlfM> z?iFoH+MM*p(qhXS$L_)y?ovC2;JS~AD=&TG+ox+>mKJSJ*38qWB#sMZnsw7G5H==E z9bC?suJrSn+S_}Css3wKj9t?hwDKV>dN$X+!JE9Wu^^_$L6d%CV|}Ck@%~pX8C8Zj zFVl*G2}i4o@ntLZD}W4!Z6+(DyLH%(RyD|Pypfk)3y|tylX;`LRCVqk2=ps6V>zp! zss3yfcfb)e$4=uLo*)0PoGrv3i^`~!G|^$}1uSk$eJN>anJ4&J}o>(fkDA$Ey7N=KG{E zct!bHY78shmxeGroYc@!Xq^~;1tR_s-jd8)#9Q;?v8+H+;p8Hl5&uJ{d9-KS+uL*O z##Qe;N07|1#i_ok`x)uaDr64x8?@!PO|Rr!H*Ni{Uohq{RJGuDdSL z}zt zN6AB8)$$rmE*8;o@Opog7grDSv;Oci2=o}PrBr%Gx|^l();@<0)7si&l7m__P1WR* zTVtf7*stc_(K&k#QOP?p^euWh84hgK8@fvzDM>#u0hN_%7az;aBqzL2_hm|N#=lZB z;t6el6p5a`^O40fd&sk{qYCZ6g8A)ZYY^0_Y%)>#hS7r51ngQn|8lODlc?ZRv)Asx z+UCZ}*8MJyTynF@9=H-u3vp)49NYxK^6$HtNMYopBI8MpQBTi(oL8=>h{FkXm-CZK zv>mKD6-Q`qH@bWf^vlaBzYYF#l9kX_#YU{wQ=9RK13GR|@n^Vgjm+`Wrt=(x_2SkA2nZ%z*yFhu1?oI4_Ng zw-b&9ZZ~)0l=bJr2`yuLHp=*!F7BB4vEjNNzTYnGncMw1R0P^@ceRak#^nxQWJ`JZ zB77y-aILi#>g5)#u9va3us8O%qubJSoT zW_YH}CrG)19dj5FZ@C`vtjjFebwF9%+^<{5=zx8$aiKDemQwtkfV+_rZ*1vbw)WmP zJotm5`C8?|>%zuVkZwS|S5+N(S*=)lWA=e20Yiul#~g}YQ4b&ghZb;xxF%LPv_=$r z@A=Jt4_Z7r#>vBD(EQD!Ag$*x34-)?NAlTJfyvMcUVg|q`K7hpGl@#a{W&r(Ma z?i1c3oUE+cvu7NK&jHp#FiZ^w%}HZpV~Wbk`j-#7%h`PYxHnS3sdZRj`tyYpEH*#z z9|S3W`s`UtPmlR5A3wkM@~1iW=u)hFJ&2^{YiowoLS#)cHa9N>UkZ$h@b4Ix(~H!K z9W?67$K4H&Hfl(jPq^yJFi5|Dh7_ zy!l`1{qt+XlUHNqSN7Np;z;yXzVkt+`OB9#{*`G)M)b{XD#^9s;Y75vRBtY)s~bQ5 zdH?9AC7$D_&;<1M<92wqI@5mWPxK%KgB;OcV|4MQG31&&anMsr$zMnT6|||XE%54@ zU`omzs<>kKjv1F-g_eA*-Wb(vd19+^$0Y)L z%t6S|UU3g~1KNQNEXPRme+W|5)Hw5jU{q4UjYgXo+aX3KTIS$8-{`UL6*UEl(!|jw zr0*MB_LoXFHsW8;NB1e;e(XQ_3QwEctYo0^B{A1lu!W7$yq&vj>9ONnOOmG3ctp3G zn~Z)vU6s)=ox8C*b1)Sgi6_18K!M=tR**^2x3Zg*)BR;dAUB#<8s$ z1Qu*d@L`vG*L-X_gl0sG$;aD-vmBsBFq$0QWyDW9@hJa&95e`0#v6i`D^{o!6#wO5 zRRx3=;bu5HafN12L8#Ss0_SJ0*#WEj?}NtB++5DdDPWx9R%~IRDk?y@OubMu5xur| z`B&@2nLsniA*Knq}}ObKpr)IrH=B-8WTVhV2s|%TRcecbz%7+Aimo z{D^JkxWAPI;|}){>*6=!(%(uyIgc9`&*fr+2r-R((V}YM^trjoawBPR*>Q%Er^LoO z+X6EZtn&J5!)wum@ousqg%aD^ykw)5E?v4~oiD3$Xl8h&)ZDvta;3G+XGi&48O$4a z$q=Hgi0Q`qn#lOV3h1Fq zn`tzdMXUAW$0rRhZk~VDh+sJ3yiAtN_%s+B8a5d50Tohrr(*|5z4>0=R)KI}(`ehLBIK(w%@{g8%ij8#iW*C!BcV;{{UC> zUK=N2c(02SX_NIF%Lw&->`eb`yXv^qpSwk7L68^~P-hQg6J}e>pL{ob>Ugf~sr+4+ zmClfE>WD8HdM)iLlMJEXKE?AS?L|CEhI+m$aqA-1bVI`+hCB{9bx=xu#kR2K8EY{u zQ}S$5VU^Rka=h{#dfar;ojw6Fgv>LP${C`7ffr%SP+@$~JU2Hty|M=T4d1#AzBS5N z{{vtQFK^e3?tXl2zZgGsN!`MEPr8!uq!Xm${vakoS3v65pUgF724jcjz z?9WGk0Kx9hX8;U!`gnZmyd*}vA6aUI)wKIOW+#^(ntj+HJ6_!1&{v&mt7Uqt;jm6J zQ|vWKL`Fi~Q0B}r?5PaNX&KIwWy@M(TKiBO3LDBb~KcV%XNH7)oaPJ&m57&iNeVRK#1 z^RGnnd@qo(KOZv|_ngJCTBnsaR0it~lTHse-dF135qPc0sVPueqRL=Kk)dKjxq7DK zC=$cSxr=AxRZ;lCd#~O^^ch@?EarN$HZ|dUoEBjZ z1NWjp%1q(FgR|6#3WqgEtpr`%)RbjQbF+&0E^RijfW{qq_TiO3{4_CSHV+?C*<#2V z8{_Z%opM-#bU&k{rsj0ye>jnYChSIzt5?(DOU03c;1N$EBMxea05OMMiguO*Gb!;Y zQuv(a)oD&oPftlnc^f!L*;*aboHap-{J_lH5^C_*V(H`mKWB>KfaHVOPvuX*(2^I2 zvgVX zl13ooMOL+c{&i-ow6R)8|NL@b9j~|^HPEcLyp{0&<)ZoOm+H)w@ndD`(*?BA9kgY5 z^qE#6)~jM|*n#=|4!h#_fy8M5y<9{MbdpS_^asI-$eqnmAPkrL;`IxbBU_ z1O;Ch1--{fg)1qjsG7#GWoENzPD)qHZ{G_F(<2H>%EqOcA#Y86{lrLCYYXl%Jvw&Y z_-r3JUQwvj$&+irbL^SVZ3&3BXjVq?1HnU{w=UP@k<|b`;uY;7NmX4Ane}UnyShwe9e^qF=#><}B!HcY0P2$OSF5_x8mH5x6uKJGiy{ z`Z|9ea+v;i{o9bBKFdWUM!TTP3^XyIfYJ-!r+BDLchKIyfB$CX{l;NVZf-SV3fCd` z#iQ)uzxq}{du(sqAvk>gNWnyBp*1B11?!(+LxqW=`1o4@UeMF{oxf+vM=m+Ap#$fX z-#PL9Zum~5TxS@PJ|uplbdBOxsgT{Q*0=*QNyKzT)`6xgj8er_nyuyRZ1i>k3L!g> zx3rlRM^7W+=F$?Yu zKh9~qbF84#d>Y_N1zFj9mro!ueowBStLU2o0F+RGcy{#m#>lR$xpG0lg@71Ck!w|z zI7C4^D%W^+coXz!Eo(_J$jbaU$LR91V{Tp^TJVHNEKiF9Q5wE={wS3DK+pZ;%t@)K z)o5ZiXh~7D+X4CNEXY8Di@iJJ^7X1uf_lWyHBZ$nS`4id+fXCUgJCu`|S!9CiOA2NWVO; z=K}YQBGx_RESeFyBmZ8EZ!)A7aJ;hd2tjrm4kqMta98Y>>4dASWS@Pw_jG*+u)_=# z)6G3SNY!u0K}rH$3bq+l6Aal~|DENW${FgiFca0#k)^w5@*xSLBCL$nY(eFzUx;Qm zSOAFmS1?*Sdhz2W6w$<3MDSWaXkD4CtCQlt=g$6NbGdhk%MH^B-zGNtNlVagSuC;` zRVg9GGUq#DEMTc||MqV2FCJ4~rQOE~)oZ%q4*=$ND(!qcjhw~uOzJR$&>1D*WVSQ} zRl={2wfqQj7D{WJ)Wu^Et||w%H6iR!_%Sxd;RspJ?ma-ce)dqWPk6j{ca3o_GHr~K z62NX>GfyGzySZlC<>0M1lvGsXPe^2r6%`c$bK4!a(5CB~oR`;%ipKjyQBK05RjPg` zm^t@2DdtAn#!mND`q>%g8B3W_Ps6_uoBejgmI>L0y4b(bm>ka;YH5NeIT>_e~<%V~LBQecacJWMpuk)eiPgq}HS6g>rJlRxeJp`9tOo!}4 zWd4U1Sr9Ey2;%eZ-L*I~m@^361<$o&qQ5f0D#F)QWOB|i7v#bh&w<`+KEJ60cN^!b zB1_PT;Whp+XxfoiIl0h*CbQ?BXyc`^QO@&pisi^m5Zm6%H|f8Ea$ zfw@f=+;_cSepw2<;s>lR0Yp@#ySI7Scy>(ty-nSWTz6?&r)(1qTkLBfUin*Gx9Ax?uV5IU# zJ4MW!5+r}asIc!-i)jn*Hc?c?#PmRQb~hegk!%D6n}`rp{zC|9X#gxR(wou!290ia z<9)V&MR&pW=>DAFV58*b^57;*1eH8}b@|@2LAJB+3)$A@LXUb&>6zkRUSvw2wX!~r z^gr)FVRgIDqa7!G@pBW!lXW9mkW|(ddbIk{jeeMYkPF5=*B>J)7HBI7Z3%+LiSjE% zrlu}RAk~J?be7AcO`EuE^FhwGaE-*>el#?$i?JdiB3|7$4h!5z|1Lfii>*%rxh&^? zjD>ZU%mVW zg+?S29z5HNl~zG8*_OQyJP5VQh9?%}v30Wda zEzl4@2H%XV7;Tzs;v92RQd5bbaPd!_9zcOi2YBDpRddDv12UlTdGj56{(0cL=51t# z(AAaBpEBXeOdd z7y2c;gxSKw!Ww~Cn#=+Ofu6drDynqfs~oUehEB6rH^4Zkd+ld3AKAJiel&UY`LM@> z34gD3=R?aaEirfhhh`??8`=CB@)rytg}t^kx|qK4m0EeDEim^$ZP0T6A{xxSgt{+LBb3b3oQVd3HE9wNeQ#SnF$K7I1iG&W8p zHhf0gm_wRM76Rjaaa<218$JYF&0)wy&4Y1P;%9Mz_RVvpYLw!iee!xGF}o&Z8?XhNS@c9rvWJ}KSD zwtspjc>TL6L%wK5Q8uVV?Ky>JtjEU@M)v$Sv?oaiVt>*Gh~dC*v0&o`Z|Gu_p@a?G z=W$(x+83YRk7Qb)C`(yMDFp=#Xz!XV8?8&=bjgL%?B2-A&VFA1)1WxIyDH8r3zOVN zWj$D+6hv%k_yVdhyr0IXP@Nfa9a*Zfro$~Fr!7=($2oBi0Yog-*U`~IcbAV3sNPv( z5q|fn$QaJ){ttuO6>R5TLAfn!m9eVxc{7LaQa(W){+4<-=2S!0bZE zS#TJ=fP<4Df)SMrI^)oFpaE}UUNZHRj*f0(go2n^MMcRmu*GfY7XJcV7-D|z?5s&p zhi?^<56*fb^UTZRLen+oASwbg(9sfJ}ma!o=;NeTDG`0B*mM1r zrXK&5rab(uPvaac)!iH%ZyrB|hXma(asTw%z{pYbPkjQSu~~YG-6dBKBY_Nc*#)7n z3Tk+pc$9BI^#CX~?{?qbgKUu1A*8~;&`0}~F9TL1@AlB-grGE&laDV2B9>DJo#1!15SUG5lK)aGEF_>bJ`_+l{!7!T0}<5o@wM+JVwx zm8&ms6Jbj{?gC38+xTH-TfyV4@QREUZ#lajFNU61+HAni!e7N7P_GRnAH}&D%K2pT zXuWvZ?s@XD$wioQ1Eks-uA{8x|j%sp*uU{eYxrd`VB%@CSQ2>Zx?bqD~rptA# zI<|`a2&-0TL>&1Fes&6+3_}5C+C0b{FR(H8$F(ie37l89QuEd2G~pPwx>u1C$MewYudOn}}xt zM3q}V)bv)b8dVh~fO)@QDK(e{h85*FdYO1=)j_xw!C4K6gG2A@-krS(D&+qlFLc@J zu#lPdRqJf0NGcM4<)^_5p|vAaiK4UbEa1>09L}r}joebcn9y2q`6{ykK{~nn@ojjB znog3*&EIi&LD$s+c>d($i?B{I;De6D+_bHs<=Fz~8d{XBv`PFAf?@9O_ zg;rO%axHGCz(H+UDQww`oRq`$q^m~b?GBLr@6jU?YCanS+2{+v=X&2gezANS*3Geb za%uOtJE8Syq>(m`{Lr&sksMa|Tmu?&J07_c5u*a>0lxar#S zOv8!YhpmVAVyMJui?SA{BLK8^)QyDVtBg2NUwSY`m?ja%S_W&wndo*{aG>eRBmWCA zU6GYm{B?pu3Y@l`U=N1MwDlbB4DdU2$ReHZ;wz1{u*&}X7v_qJ5_H}{sp&28^aX4z z;56SqP)Kx#W|u_oako0Q&%?{n#wL)kBjz8CLMy;SpSDPsfz<=e9UaF1NtZbvp*5G0 znhMY5EwGsjpkx3_<#O!EGY#mOL@f;NjofT(u>dDe;hWkTlefJf3Ny^h91c9_biIP%F&Cz$`Su5is;7ZC@+SW zb+FgkKF3BP)_>er-+5eqzf}pRZRu{_Lr5=WU24I^hr>e6pzBOetH;H~K_05BwkARL z-C(7NXt{Io*h$b}$$sdn z7}GrXu;}~Sn@OI1vI2jR9NYPZ+vt(JQPOeHhVJK6}KkUcYn_)OqxIt8L?gN&*4` zA~=(;?4b1#5f&2%ws4wqKVzvnGrXp}hl<-w0qPT4SlJth)mJIqAKwRdQDIr=9Xrkc zkY!$Ic-jjmp)V9cmfR)dSM=0>S;x?8OUJu*xc7Uw(6)f^_)RuH)=P0#5KsvsQr0Pi@PkZr0Y=atr3#D{bJ@n@vI2S5Ww_K>V z={WFw-`pGsaE3NcSxqg~*KM_rSaRE>BKhK22mBQ@+l!!k2XaM@&4}D)$ollv6b({>fN`Isssi}3 zEvhf)TMtPQ|C$EEg2TucA9sR52`n-(6hFF(AGR*k)L)(Pau%G3BSm^2pkj?9v>A*{ zOhBY6k09>hC*0;9hD~f&=*o30wu1_bxW~bl=HxaC#wn1Km3P+q`0SS^o@O>LUEQbb zcoN8aVDO+g7KVE6RChr}viZ@*W?Ua#`@nYi%Q<7va6V(TSq++j+5pUZc;V!WQ@Qv) zS$X+@U+YDWXjgkXeQ>oNQIXLD!B}2iUIgU1yU2c`*bZBqYuP8Vxn5$q1?b>_(?rv7 z3H0ysyB(|1)=F-EK9Ts{qoScvM!hM&3a&F+a3{KcEx3JRyG7#vF1YJmb%5i*)ep+< zatl>U#We5V3Jz=aUO{9uTX(zUT_|VYR0BKMW}#(zIk@_(57=8C%V6#|18o&Qggn%@ zu=tV$#DqRSoEB9dL&9&KXB!jH`z^>sq2(14t*xysB(3<#(Uw#`%kNK!%@cwI`o*>& z<6e-oW$6M!Gk$p;faas-N{qUw?-|b4hxRy4blnoWv;|cj9z+%f&4}z4b9ey~@RqZv z9n1E0%clt0lMgozXpvM^G7RBu?G4s%s7%BJSl5~&nL_C;aBW=-6DY18u1TKRlcx4xis1j=pMgS%7FvKx1We;e#svMZz z08gFG_{>rcSYC*&g~Q1TMy&$osDa@HBBO+ezvda*EPZqUqJD_0`QFN{7fC>!pWo5BJl+~zhW3jR*i#2b{ zyk^ENG`17bj=cz-&nO3hljZNl=mPkh7c(0=ifkFEK)3bVv$nPgo$Rloya%AuK1HzN zLNm%sQ%^5G%6VFh2#KAcT4zrj|3TD|%QeuRK_~9lLa%l~D%!Gl?jE10vK-i4(#O~Mueq~Z zEdIO~-!w7;a~1zJNCY2f>Z*dSMo&+VGMz7|uK*lCmBRu{{Q)pokb~l>_XL#4!T~_X zGteTox#A^amqey}kpHjXE!^HV7#RC;Tb1SJsnQ*R#pHF~BR_60*BxGsl01kCs>Ka; zOW8k8d?(!xQfekO2!fy0nF(oY*x+@T!TN+;is1~EI z`C@k_D~OHrlVDy28@+;-7Eyc}p#7s8pMz{eAU66zoy|4v;NpiCDrN^bQqTkHNKH*` zReVbAFc}pHjOAUS^+eUpZ%}Ca0?%0{a&OtvOdV}!aW`wbo4g&@5S3u-H)Xv1uOB^6 z<`pQt+^bm7nEe?rLe3Rs#}jO1LNW8&^fe$8d&e?JwATl-QL2t)girnz zz#pZyHm=W3W?al({uFvtlFemz`GUm2n=(Fu*EDKuHm-OCq-U|4*A!^vZuHjO_}Nai z!&8Hz{sudUgyRQYJC4dA;4+P}|GrmX>f>?HmBfj?uci&$6u}!uRD95f_s|5?=*f|E<9LuW67ep3yzwCp58!-;Dpdx;>H^oqZ?Vaj$Vm?D_R33=z_%K;h%|a z8gNq=b^+SLa5t4SGhD2RV1&1Y4Ie6}f{Ns>V?F=lF4zSAv#7zLA!Uz2m2h_2P1BG)l&`^KI?DrQelsM?j3CuOi6EWCn=wuKH86Q!f z&OCxvGHZAZE80f0V3>g}#MZMRv)%OJKK-(0aMrtNFKr_)&_uGNo6GGE-)Yhh)Att6 z9eEYDFCXUDU>2xKmk+^6G3A~(koW%I&kwT0!q~N(sHmvGOLZSMDn`Ye=<${bWAF?%Yxl31Fk>dhM6c8!-}dYfYw>K0j)#1HAyL=R6LG2~zA_D1 z)4opleSNSdiJf;S&(|2cFa*mX%bzE$3s#|kDyzhDcG{-GKJFIH_feB1WEU|KO#)FxK z0(bkO4%q*%2a3nfj;3NMaZjI;j&~L|p=z|}<31juofGI*U{C$>g_t(t2^f^xJOLSh zul5}4$d>I!82KP(tTsY61(2BA5me!g)kAo*;(KdP{iFr+8GIoZE-rR5rypK6!3#gX zO&5^ucp!!pskwz#aWKpQyhp2eCF)|GI@E;^6WUnBik| zcY+CT*S>Ym==wykJizgWSmuS%)VE)Kxe&WTbyaAoqSd@M+`N=`%yAZa!VFMGi6{OZ zOdg|+cgM)cNMaWrl|A^2n~14D1Y7u*{{GK)eM@+R-<~)5aK;-zT{8CtDWpbvWu{-b zO)M0EIk#T-Jc1Df)5o+^fd<+B3!zuEf(6bT4u@&_!8dt%O3cB$!_W=x*t=6(3T-Q~ zXcQ@!ardCtu=Umz6&HIuR{GPTM1W`km$-R!xlwog zmc7(I$9m%Amd_beN?+!bTHoumJAm98{w$q%J_^cI6vh3Z^6f zeBSIBVCc;9omvMAP&z@0SRaM5_9s6zH8ts?Rz;L7%CrLtiF7!4vK+??fUMDQrZg9> zXCh7!IH10jUt^vIo6aCmgc6*#()?!-HXvY8;(UA#wLV2?pUgc$ES&QBr2$o!;|iL# z*E6Sq?Z`7QBQ-f~JkSg&mnw>VE&k$l zFI9NJS(mitS{8p1vg{RW>t6~x@2S(LuZ*IOaxr*jG7?XpKD|PekbgPKb81UIp~SEs z5tju_N$j3{@}^S!4N@)zR1U8$&4{oZdyZ1WFSGOWqq8GTBIdvui6HsCG7!~x*qV~sF{PxZN2gbxua7}1#HmLAC z0ptia5}GTrdqBVu1CM1_m%!K%1!`eMM|#;&E)3lIw}tgK7`6Um_!`A_#GzhGUaS89 z$%>vox>X2CH8)WB;cJh`GNu9cYc^TjfrR*Oq4w;k zRI}C5uHxnGom^Jd^Pgwiu=wj-iwq`kGW%xb~P0-Jl5hF#fUtbUBGJPBJPq_|C8*1!QiRmVj!i;lU zy6M&YOKyxG0NOD5iOv)<>+5cBxFqtnqf%Mi<+FM2BrvYz&lgT{*3%;xcCdYN0YyE; z9fXnzLmLIz+b}`E-_5RIwEc7BpP$+M+K+yqa1rKobAHPf(m)4*3I0BcekIdY3}Io* z1|KZIn+1wZ^?(lNUg!zKuqCS2piR(zqBA3N7)_`&g)PL|p(2qVwhaqeni(o8!JbjEz==Ep+74&W0bL#0{#2;+e@{#qr{G{SC^g-;S!rJq~A_!te zoDiiCl4B1=1tQamt*-KrS=9BpJU2G5+4qXcAr7*TJ_ejbOLxY145a8Qj_6dv)iawL z30xrwy|dw$u^a<#44c(O%eZ_XM@0WH)Kbe+;Qjz%`FG!T`d_d-|9ly~nRlEEzHU#j z#eXN*rxAU_YA$hMpLxJkj=_5lo82!;e1uGvYPr__~ z-NIYK3!gJ1E0FU63N+|)px26Yia~>Gq$$z+KPydXC_(8L72<`SU z`m58o_h~y1Zg0TGNzccHo<#Pe)ZvY#ZEqUXG&N#xOL~ClE<?gn&tIAE3yBE6d>@0=`D+4?sbPkPJ`Imr}@I9P}K*y?3 zmH%2uNXVCmWh)wJD=gP##g3@O2!bY=-TK(Z#s>aRXM5lP#&p)Iqk1sH)oHRli8}u! zOjMYj4`|9?78z(ev?JyRkN*=%=m){urXP~jk`zV%nS`>U0iSg{inU*=z-Uam_xOz z|Jt$|a)uOm!oNStHLOpI_W1Z1+IW83gZ!WXpI1i%8*g1Mn+!Ue2V|UxdpwEEu-6X-+m6*tF(;0C|ZhMYzDKE?l@E!m`g4hABwK9UxSK+vZwaK6rdkgZ*=mUj(5mhH72EjG68y0D;TY znQd-jUTws-B+9Dw7XY6^ueCu9du&KxA^Z;Z>#z*&T`oqmPKb{s=u|trkH>MU0?aM9Zh1txE*lcpCPm}?^O2?G*P$81zE9rZ zNd`wEY+%FY2Fm>NV7s6Ya`><0!sDHg?Kt-y)$`Aaq5mS1_!e#&tB$AsCSLsUB#sZZFmaTn1kif0j*61ICEmSAuhW+75XwX${HGx#O5Lm z-m)3J)``os{o>}ePR8hIJl$15)Rp#CdT(#2!X?n#;i(|_PNX-$WQqr)fZxUVw?6X` zQ~`bJ{4BZLizlWnoygp$k4zWB#YpB*6V<`L)3Sfpj*}=xy^9J(z#L)h4tTwG0^0n| zh0DBt4Oc-$`Co+&N|CYY>9t@MN2)&75W2Nld2q>;?n^Mb7%dAHYa_5y6JcyOurYMt zWpf-N6Qq+75cb&|0Wmp3&y6?^!Gj7&z_LWJ+1X9-C0l_f3-cCQ5|b%eSm z;@Izx@)C4PVB{#5FR6@*l&{1{TafXM5dr34!P2xUtMkZbUybUNfqNf#8Q+FdmEHD- z#}`%#%NHuf598~Q^-#|kv`-O~(0Lj%#(1NZ?Q`rrW_lI(}csR zcNVH&ZqH=!nA1*55+sBHs65!l;$UzIEUv88s2aeHjx!O(tp;==709DH_bx>HWo=0U zC9$z64u1GO{U8o&$jZk4Cj@LcqD?=4W)knmXiZm3M13QuYuK{tK06<{577=(?@*y| zC^RnItt{X=0;q?W``mw>(EWd%PTzCfI=p4SRpD&gTceJb zh_)aDg&RIY7##6;qX0j)kd7$U@qdX&K31TyoJe zDN%D;g8H9bP&mNO9vVJS*C|2nPVLi&fgaQ!75^IQ5(XEo5o)GFiK8HNG-RJ$KgYo9 zt^?hMcm;aB+26+R0=OFo?u`3&SB^W=FMppG+8l3bYcn9qIK=BqTzXsQCXc}E(*dnP>*@9iOIi+IXxzc3wcDLWRLFc;u>Qmp zt9O1gNb^wy_v7DpG3W>^P@qoC8DEA-DXKpMk%k<-qpz~#4X7{ndCF(VVoNl-V?QFJLS6+qqQIn!sM zgoe8K+hmu{JixlT@ZQ=z|BR@)`<{oo=(UL-b z=UY`-)Wq2Et#gB;4fl77H@lDT0uJsB-4>UJc_&6&zK-t#7ojLbJy{r#iAan}(_y^= zf=HTxx{oq|qBb!}C-9Hf(07HH8CV70xlmgTE;XQA&}*=zxX4+qMI6z;BF?LW ze?^=Z(0v}bQVoowi=7osiKX*3AcGk`wzVg?OkKfV9Dm8Y6TZxYpn5u(WBa+~RcHL` zWT7CXL=&vwEy0hPvDt`%pX|YD)cztG958nOAg>VxY3-2q#1$<8_(Lg*^|D-*WOV4`pPbn$sMMU#) zapYQ1S5m5WP>=Ei+YQts^`j>JrRp#OAqQVbwIewc&aNt`3?ReqItu!k-+xqiEg!l) zEH+mi4S(?(51E2%#uhvVp^Pf$w_E-6$~QP{3ryV9)G*=Oi+y-r9jL_gap;@$XH6)k z#@9D0j9dX#W-T8c7l(D{ll9|YEfd=Y_@pPTz`NZZOf>d0qHo|5mMJVZf!7_Fg;#H+raxIiy}7P-TSR_)Uy zenpF->b4HTG8tY8Y^HDYI-^l_xBmIi97;@vDt@%e;CJWkjm$6kTHkZktV=BpAR{X8 zRgs-)TLn>&4c{^FJLohHS%QJ{FtmE#1h^qC2hiLQR6E+M0SMMY=NR-jnY5-Vrpv+D zUv%XB0)so`*775A1s@rSbWBza(1}3Pjrje#y6*fAp2FaaG&cxf_710$%1R4Ssq8xNM3wxKKw) zx$(NVv(}Nk6l}-LiMRe8h_&WTcip~^vIM$t6%-W_eSy)8)%r6*Q!D%0d0gTE`y=>i z8%CKyP}q0jiwho~mz@&xrdSB!G}7zMf!W3gu4WYy;HsbRUl~O2FSGKl7Aj&gOXEl2 zoc?she(uu)n&g7oR1-;}qWhRu#7nMit!hIk;GDrz9OXs~*vILH50l67Gev}a*E*9? z7{X*mA)c@rRgA6!oU_O@6;NB1UH$>UX2Wv9A~eJz78UD2P6jmI8B|bkCp=ZkEcq7V zD{eLTg-{@)KA$@}(CnTl9r-)d@S;%TxmG>Ke}>^t>su22%B?*Fy@0^ zlas~Qo1Zkm!U=Ll1h7=9L?1x>Zb2CX$}XlET^J#aq3ZaCs_JJ;3m9dQKOBLQQ~l=kuCdel@o|HBj|W0`qYGU>k@Gh+*tNE+5q3wM!k9$2llXERn4Z z>CDJzl%j}wha?MdN8u$46lGmtWaWmb9!qFA9RfDsHJP7pk;a4XSzsuT!9261L>&ql zXs)N1KrisxTCM<$vpJgDOe$`uqcGCjq6w{n0SMZVR~BS3gj0w6(wLZjF*2bR<{~=q zGSB8<_lJ-9ur*oyB^t(;R(}+21+xzFf5B9>o53!$s2TpWj>f*dFqs2iMm^oLZ=aFA z{=45gZbxd<_=6kLvfBh6dISH&`90^g+_SY5V9_URxyk=EZlFBnvhUM z&{eaMV4jhj+~j_c2`^s!Kn`oYL0c_TL=J@si)<0l@4sfD9YrVE-EzLK-Q5TxOSh~=@q05+->J6pn29y>d_u*N80#KEIrN)AWAb@ZI@2GZ@$8A>^RPDN#vG`_t#FFN#nddjo@ z!R)YISjMb_n!Txwml{jsReDmSsS-m!0}V{KMC|~)M}2O9(Y>py+b0X{<{76L9we(PLm{9xw|5o|pJ+x%|NTQ0vE^nqe?g+ffw&e@kdyPfSgBC!Qu%dXE2|LP}|$jM`1`hshYk_o%6q|}iXX)}Wi zDS(DpuWM>XBB>E7a)>2oQb6m6KE|3+a02OfWT@j29523;!?{e8fbZ!=P;A|yN2vcT zIjlCN5PIp&46=(l@Q!B2UmN#rb@>;jGBQ+K(1-&+-hU`r)?{100?~`umv7Q%d|F@B zknC!#m+blB{EPYMPOSU`I*{IY45xbr=#2$m+N+;u@8Y40P&8U1@}F2+{29;b7a6K9 z7>+iq7|=9$4**PP0n!c6VFWS!HY%2Dqp~DJtw4JoHDXPRps)jFAL?g7)VYGZeCN)s zQ8WPJTuj&rh@9uI!h>G=L(|URNCy)N_VAW6Fh2=;BBsp7`b-w)78YiFEnINCnyyC9 zPDC%wBK|(VHKn;OJ)HhRvV85|2a^3b**m*yB2c3gBWbiZ^pWFRG z6dZ>5pMIUg2{Hhfh9?9F46?F3FdFs{TD|%Jyyn3a9X!uL4Z3SipVA#$+ivK6k3-92 z=DvYfbB2sJ_dd%wcUEpFI9$%z5 z9mhRoGBb6h=SO#fNnuN0hBb{rXs&puo*Ca0YQCdveN|M_<-}VlXcGRREE9aJdyI?X zyyxP?+0uPtV#gj>)9f8t5)d*&9%V`@uie8e0P~^26DBQSq(`A4h_$t*riPN!&`;a| zBektBBGGbdA69PxGmK0}vEaroL<-e3y^Wq9Y?w!(C+ST{l{fcRmy2mH5@aj$s&oe4 z{uJh)`y8Ga$@8~w-;#UmffVz+855R1{*R1` zO7P7y%*+jOP@`d^neOiH@|tF9^~d@6deDC&Rd0A7&KoXe^&dtoWMPrDYxnLwI=jiw z=+6qmrqzo0@4qd={I-+ID(ShETS-D=Vopx=mGrQ3IeatrI>OUTSSyStLXJMqhA^Jz zR$7?-!MO~g`FO%wG1Zw(!jaovJ?#zqh4sNu;W%I44$RHPtHJw`qK63gt<&`FqSvLc z5~lWc>*B2DtrV+49TS|bFmEmn64UFqhIx!^Y)!1gwnYs)iAgU))uDL&A6C!!#KcfQ z%^0@Y2K+PW!sGp?KAWGv2IXowuIxbAY09=pTpYt(y;{aNS{n{^ZNoW8NJx;{Xluv7L;|!P%;hKjfn5xOg8i%k z;o&A|A#PlNij%1KJ`xvq`3ZyPB%l$C&9fm=P;>wO{ii`fz#1S@kAjuSEvvr%n$;U@ zloMi`nT7hB7!i*HhYsCFwnLJ@;^GU!{LVk_($cW*`&Vd2r;pR66xUsBv^>V}1~ULNvyICze4D>hGdLY%Lzgt@AkR&*?egM5Sp$HyY!A*-kD*Tv?t*RG^$ z@k&vzDLnFWtro}qy~7@pTSh^3sgp^Jj}OMTN38nS4Z&bzE>Fpb3teH*8BS`^(EmJK zDv^Q5<)LUWihc(SPyIG z`6GLUVWz@0i7ljmEmXnTxis3AH(Wwek`tA7!h!uWN>FUytRJ%uAtM>BhA!ZjroDy~ zp*&Uz+ByYX#PIs#qN2)We5m;p6`7dX?!UeT{5{OjPzT7JepxIMwM*#Hd1h@c(vA?kml$my9n4TJ$HSHqzuo1)%?&zM_` zukXLY;Smw&Wn3f#J@YoHbkhPCcDcDbSh*}dU8bd@3k5P0%T0hz16^I+->1_eTILE^ z*_7z5ZC{&*HY6Bj6jZ&HhjYLpA|k@vczERV=OEv0SP*dYiPju5A+T|Ag)iZl{`t7P z_Gn}N^Cd>1EitjmD?7mr&}05;wb2j8RI^oz-~-gry1_eR6pp$~XJ1j}V!0^Zv>(0I z^yuj5N{Zb_w_x_}@Qzwx47x^UM;u2vZosmDG_OIg<;Bg4^>75SNT&MjTm8w-EVKo* z^o>e8(;$G+ZpTGZ!c z8r`3kk3(d30>>gpD+2=qC68qs991DWspda<2`5$mhQlhjZGL~sKi=Zpq+*ZW;Hy>n zd>HF)>9aV;&L*McnaMC~MnP~hDX=Ke#mF?Wv=WA>s(yZ?c#N#=_Q zUlkXB5-oN0>hrc*4tDqsVQ{%7kubkOGmQZI7EH@`c;p(yA)bpa&S&5)qL9VF&fbDz zRFvelZQI=J>x0&}cuMJJR4x-g^_ zB9kP%$d8WeFH*q!`R}E{D`&PQ+OTQ0b{Oo~nwKB*dXfUeWlkmQ%LpIu99|rW>*UnA zYklnW7FLS-BS+JDYHV|KF!kXDN~-5wQ?``Kuu9Jcz3Y(uZq`rPhLVv`TmmKxt&|-! z7z|sEh3ShZidJRk!*9|98H}2wX;UMn`SAgj&(gs#z=4J44$%?8of(i z_QnOKOzg?)?X#_bcC^9@IHBXi9#dOJ?KWqYrfA_u2S_8-)5;-z{P;eL(ubD5RZ*{7 z_kej2+1Am-|3L`>pwW~6O00kPj+J4LO9YI%jP|19rmU*+IhHv&Y5W|dGuHDEzz&b^ zBs)!i;>2r!R$6Z%Bn%DA|w2LtN0LUD6b_F-j@bmW) z@Se}txw$|q_9z1y15W&QJ>7wz2Kw zgi{%1#rs!!*_R3z_5Jw>^(PsSro1R@f(^`i{D3@nN&;KwG}pHssissjPfxkmy})uC z_qSPxy$~}GD=!BRCA?KrZrG$+FD%a=aqAwO4f5McL0MZ{dw`VkywTtPT&lWL@SUHvRH`0ib~vK!YMweKHPExIhRN#)n{OY4pxf zVEXaHQJ7z^o}#k48V~Vd)~>KgPS*h{DkEULBWvpFu7fw`rt$0lw&7p<)U168^H1qu zgzG6WeqDeEDM?AJp6X5uZSLOQT!fY0ym`~Zs&Z`$H);da9WLZ0ERr+)uSIgPG;DmQ zJLngehf?+h0pCsH=WAak<**vwZF|``&h%*DyKO4it-LGTK37{GcV5Yf#rWS$aB^AR z8rXpeNkblRD}Js08MIXZU$V4P(4Bh8&UmnakxZFhA&)7Fn$PNnjk$Q`12PZ@gf%jr^FMd=19 z7|Wpmj#7aBX63T`dJ7>wOdJMGuLt4V=qR=%N<&ULqpXFm)t;eRjo*G=A4RKXY;-4+ zQik9A5q5-ne;3|TAPA$5o14b4?BbCv6s(Y0g~pfhC5+Ibb^PX(6ZBWgDPcOW%fO6w z&sHpo@0%hQlvy_RE6Ej;s=RJ`cI7g8*2cvNdvijmiAQAt49SSC$6x?u45v?5v9>!l zsbl=Hg@vjhnz?1o&s*bH7FC8^(a};;Qte*`D)7lQE?`6g>NSKTPUCSzLW1FkR~GpF zOU-cR3`gtX!-wHy;K-n6M-G&V&NP)A4$IG1@ag6*v{y>QE*j|8&eE^)JPcXPB?HPe z)?Ayl_qkHoz4?7!_$%e~!47^mxH|-GVj7>bNzKbbfu`*2T^W@+pX{M9{b@KGbiUz8 z-_mKja-k-imhrHD{@jJ_*nIJ@$2iQ-gp=xe1D#ZFcXwaFdLR_TYYj(-6~q-2g!s)x z=d9wx>WD%bt$-1!XMJfta0%^Ylxx00W6mf{2y@Fq7{3`iuL;o_ATPJ4h?v+!N@~VA z{4wo80Dl1{e;qt_%nS^E=l1~+bW517FG_${X{~u@Gv>WtX}D={Ca!3&gQK1p=0Y?j z!|ffLBAE;2Hr^T(j%R)QyXbJCqDLq_e06&-Lg27rx0i#z8z1dtwDbnLOY zjQi2?MFfyElE}9i>6YL83+bvqy08bo!1N-owBb11nSB(yG7eKG@-V2I`EDmYfcEJ+ zKg14$sgpv_7VeI^4N8V^ctC(2p)lhH=AN~6c6N4+ufkkt;AbZvkqC{vy3H*FVFf(z zvjHo3mLi#zo%C+QvE!)|d;3B}sbIMN&m$vL;{e$R0Tn_{N_x6~{|vV$r(obNQckcN z%d%3Cc5J{1qgQU;jE-(Z&~K;99Gce9*3m~LmN^7!!%!dV3S5<-7f3DS{`D=aP- z@7r~hf3y`?dxJBMo44rC_kS;OZHrzUxT3J=lL&h>*ZP{{z62uNu`|?EnJP8CM(9OC zzP|60wNu5~Iu{s4%rzKQWw2Ajj+Y;ACk*bbP*zn9LBEONB3ps)0kf%F7DA|~pk@z= zq{~xHclQ}tCL>jc*wI~RlX}w_f`_|@&i?Cgc2ldWM8hxGHM0f~GUrIekY(QY3j(^e z2LSIj;@Lh^@UA|GZU1|_62bd?J?1!qm`%*SY6%zy`2Opk)zHmsSkxlUWgGl&(Q_9f z8n3;aaPz#Y!pwhQhl@cP`4|vk4M$A5`F7zZiZ#qU*owkO1hA+Po^UB)P^Z&{j%!I# zXgy_QW>x^{-!ncT0bfpHt2M7(x9_o$mp|Z9y9LATMG{~AQBi@u)pEiH8s*f1HaMg|79R_M-vZd}AD zqgRaI%}{bcXEq)IhYi>k?Xj7gODdNE`OYsi)QBRK_2z$aa&oA~gwTfSZl5DOa!pNY z_jZzUI4n+^%-WZJ_PzSqnzsSwlb0Xwz~Q2xaKqz91~8|=86M?_h)Yr$K`6aPdH9<) z<&}A9HOrm9R}FuKO#hQHY2f7Q9WMuw7xi*tu2%0|NvRF`;{(=9a9X6Qe@e*7hZO z2IU1Ik*9PA?vU!j@G;bpgeR||NQ1bVwt?F1g_*tcFz|u0DoiSorEv5l{@i|t8!42W z%LLVKq${pfsJJ443#4lNJOmtQWuLMMFO1&%8!rs%AM^r)Hr+z2g$>|80nR1{u>*l- zM8pqI)u!;#@p7KzYM!JQiu{UCG5G{G5TXyyk-Zf`Knl^3vR(mAZ9^!g1FMPb@qc!7 zD5ffq(MbW}jA7?vmw1|oM`Z3t--7-HH(pLo1r-$)JXQ!FSX@Cp<`v4KqHgd^tUI9Z z42m9IiXs@Fd+=}#XB!Z20bm-W9EaO(7+F{vfvdoYp>N^00Nw>SYm!@o9v|(YbLo3V*)nfWtv9mZ9$h`LKxw!}?8aoIRW3*fV7wu}mZT@E_>H$-GjnC1&y7>>2-O*XueeZcTR z^kyBl=9R435@Z5JiW@Hrixhxf?hP9_BBF8Ob~!fJ$Tvg(&w0LkZw&Jcp&uQn*V73A z>douw>eySeva&+XTg1S#jGBRJZS5?u%D93R#1%fzsf>H%{K#DB{D ztMvu00|WGw&jg|jp)fs*SD7@;DC!y;@$~2Uz>W!jgw!FDgYjGpLDdv4&EA@Eg>Fd^oUCpt2P)5x+|h=r=YD5)+5$+ z$tgN&>C-wSl$Vp!PJ*s^-K;N*AmNqE+`+Sh+HZHjiN>c{E4vF3Mx2}>S9K>IsT;C9Jf54& zfqoZ@DuZ-c`~E$*CqZ4nFpd$FcP?G}@8rpoP*?`VWo5|%lmY=Nh)vkS!U8Wqh1)=% z_0pZ#RS&K{3c6)J`r-=mqa{Vq1T~sheN0 z&tLH3w!+dD%a#8ST)dc^eIkibGpgfTWH3d1QFb_`>1|nts7%_PQ`Sl^bCg0q6A?=a z3%f?^S?t7w%n-E%ZtIl7Lf}_GDI#G15CU-2^a}{6owB7u+$gAy7^bTKQ(y|6T%sTY z@IMgL7v&e_=mbPDAsH14hY*5b7sUjPwCH}2iq0h2SPrYe; zVW~lhX9>vIUfOK4uIThPY|L>tq-^j8AlN;tA<%eQL~OL>`4P%Uimuc};*V({MJ>0| zJo1e(klPuCj_&Ws{ListnU8l2jU7G)j&nLLJWAWTR zE(=;N@nS(`RTap0*fPVgIGXeg)i-b<36K19K&Z9?Ar^jVa}6IqaHmeC-xBxv2lua^hj13;@ zdbe%`0U3N7!Y{LC=?7MFYU=dV%+hXrJwOc+8BOB5d4`Uz8X=4K1x!pf^WhzzQ+G33 zerwCVTpdEalS00JEdi?^xxFy~tBsyUlik-o<;OqmbCcIN#tIS@so~H9(UR+luCxGw zf-+d`a{XtT5)GMj-M`Bjtav;ZrHgcs2mLAz@WKhZQD)$PJ1jAt}epLml){j zS&dmiM>a%yT};!iM)g7@p!ucS8c)#+*U{)_?7J?dx{z~KpRLP3YE`Xm`iY6noBmE${pL+CWl*AdF-I6+^&vlb#hH%B zuign1y$APzfWQgaf(-j~ri|-J@23w_sGad;q28am&44XC+x#Hz#D;oU_#vtk8x9R@ z!@-@_8&!4@1jHgZ41CktOK&K{9?5+_GM5ZSV%BzWOt1zGp*;cYF+#`q$d-zaYzS$J zW7dkB3jT&ryobQe8wNW;46o<}v%$gr`){HOLUZHeFyzW9jF$4E((Dkq<1E{gdIkVm zVNE@us_MHfx*4!I7;m=5@F&JKn%Qo}DE^hWreCg|r)({2pC(!@Hoowmp#pcai}MbZ;-m3sfpwlP6O$@dY0&*bAIh(sL5$eFW5R?DVwnl}v&&^Zp?U=#09<2s z5a|lXwOsJA1K^9hj}IP^W6Q2^7YW69TG|=ld+|Xsw`y>fVtjZ+^8V`-1bXsKIT7e_ zwF&{<_BdV+{{Cg4IFNW%8N3Q$?LM$8JlsPor^|XOuOlRA7v>UOF|l9>Bck21!orRK zICiP|V49C?z)-OZP-H;KK>`s(H|f2}n|(kjMOV)(P5`6*hh%z* zw3#XM^YuHo%x6J=NGb{@FV{UC)ASpN?ZaRiyMFQF{hO08-7Wx`1>mm~Zh$$T2X=&o zlbHfhn@`I77r~pIJD$z< zWrcmVJ24#f>!*Z33fc_)3C2tLdbEbpBCd)NX$r!}Z`sn6%Qx*OSLbJxYH<^((~cJZ z7pG33`sbh_ab6&`Ed?G5VEivozPH1iLdk2}NMV!L$#9zyp2-m4+pkNGZdq%LCzd<( zVJ_AeZ${fP9}WXfev8Mgt?~#8x@3TzX=5091g4)pyAkRNaL3@_3P5B0sfoZB@j&OT zV~ZfTSPQCsG=*cEYwyg{ruMuh2&SIPgX@n)dedVmU&{LXv79q2N19F??2^?4B8UY| z^8>cJr!N~;&Qek*8*=ddtK~lXRe0Lu`eE26$(kJ&Aw#hbl8 z+mY7?Z-!VqKKdTW1OwsM?}(5lFHD%Kt*hI>*LcINc!1r0^BA{Qmlwbwbk37mqgeUb zvsE2ups-xk!O$obalv4{Dw>Z@NnwQPD^g&%lsRtxvxE&sA@}?^3Z&%e)iou=Y$0)T zmR zvgevCon*lq4zF)lN5i4PZ8#o~ewfM0ek)z|-!+t03JlxaiL?i4Go-l zn~R`xG2%jm?i>K0#dusC4ejaED5VrZ?p%N|P83xzs4Nj|<;XDP5>1B}r7J59-aOce zCnwC2WedFV5L)B}Wou)@7a0{?MiHEVMiwu^PwQHWX>zIJltyUx1e^*9y8yiVcr(s9c6KN@qtu}wF|{bY z`}4;VzIuU|B>Wz(&XB);UFY{%@CBU)zxbCX*N@})tg=8_E_9?CR@Zq!b!9GN#UT=M zhw=I8jk)9e{IArCb^Hy#3O)KS`YEFUoHeJE>=RL1^falk-~DXktGlg7EZ8;j|sE*;S zy{Q?6?i0e`&JulWiHOeSlf{vw?ZhiK2W*AlQZ}^f%>Bp{v9*?ra+l z3hvUwK*$Ei60uP2pkokt-oI$(ohjW<$kNw=Q zo%Kbc_c((Hv|K$KZd3NaYBm!%=08)YToHkDB3bXBBllqDfA+tz94sf~R+yYrZ>TZl zn|=JHMiCSytEkR(PU_J4SQ1Cq3tCZg8|0w_z>f}PO}kr#zf!I%A7odKbhlMn6%r)nrRnx}KMz)K3{ zM_i*fAqe5{&D8V9A5HMe>e+(4%RsET?k2*-#~`u7Na;6`(lO@d(v`(}nKWNa|DvI1 z0;z}M+v5gLx070ipjF^b3PE=DWLDXLLo8z1(+Y^AM3Hkh3`{J&sV>=&SLgHZ7 zD3I^XK!Be_9t}9;(8+NhZc;rwj)-u?yO&>sMjK!fIDhl*)^T7odL1>(djJS~$eoA$ z0|LgOB*8OS=Dnab5X%5v3#sDf8NgA%&Z&ac10q3dJ?yP-(-Wz=Sp1Y*>#>60zfICrX!bATDnqOWFVDRpW##-C z1ye_&jK4sE>o?`uP9x^G%ZM^BD^BAfrx$wU&w~F+hDToe*yo5}<}o;KE33@pg2<)@ zZ;Mo<;ZQH3Ux6eBhTf3x4*qQ^)kAg4kDoanCsfiECBxqRB7)?aiPXxM5WBs+pk)K+ zfcFc7lnE#l`3*610VoDkf0zMuL%wiTWw>|{rg@X2K8rNhb2Ir2;=xa;9W{t%>+2{%xzW?rBLquMrP zlVZjFj+{FBfuC}`9en zNqyhh>_yP;QqLu&6ibzMi;7f=)CQ9$=I#`iI2ziS?qr|p#9Qzyx7H)&bNVchqztD| zNK{nQ7`AS!IQYD?WJC`Lld-`zOX_65YK25M%E*&bT$SfzIJM<`o9Sq3D4X*7X{!fb zXQre{J+zBYVaW{4%#fX}&_=teUd13=gfpD%=$TauZ)z)}T0gp>g=J0yIp1&7C1NjG zj`rh9OMe(+r1D1X2gcl)B6fFFdpk)52k#{#v*{1v6tN#qF6quqR;H6~PfAFTeXVuH zcfQF-3dfXrX5y;=G^`KTFD2Se(ktAGQz)|SjjwtBXwlqqJqox)Z_WzmkHNPwXb2hf zrAchb$aJ>xu6g28sx-52tfC++L1k%#dLT1VjZN`XPVkOhTKYTO;+BQ3WWBuK>k`io zF*JCmZ0)tjauz1BT!rH{?*^aOnDJA%obf%)8k@o}ua-OtT!uovbL%KsdoSgDA?aQF4X=w-MRGZ0;Lbp0GUT$W`clqqu=OwtNQJw}fz7AhmdVuBv%lacLl6t?KI zQCb;%3e1B>%3_3PwbZIb&m zT9a3g_Fqrnyw?*?JRzOCTV|~; zjV{ZG#lvh*WKpzIUrtUo`N{I=KP}cc!Tz}b?->?$&wF!TH~cux4HPeGDkvuYDjL$Y zGoq}}Y20sN|FOZx=5CMKd*`pkGxxsj8C=LZrq(@RyS+UU=ioEdI?F$l?%aR$SaR^y z?A-5Ha}GYI=#Bq*UL#YJ?@oZ>MEk!cz2nX8>9oA&QJ3zQca1pF&o4Bt3h|GoYN8u# z&WbOONuP!Jwo`MY*m64r4%v(M#eDsqKTpxEI~@|HZw^ZD@~1;e<7e;HOw2j~Ex0;) zJJ4YE)1lT*iq?C-79H$I!oB;U;#m9O*fycUkB(_7R%(LZUyF*%iHe@w^N^LM?K{9D zTuODL6ssiV;ml`U6_VG8F{ji7wxrGz%`a|UR*N%jbeH!(8x=mJF%<5L1-SHt_K|DD-53z#rH7*p8P9zkd3)BU0w)GFiyIIhx+@kUOuJF)@GLvz>s= z%I>*`N-wbGF!j%}Uyt`&^!EMA>(Cqby(((3y8PUJvG-ERY}J1Cd#m5)YLRQ1@Yx)+ zF>_}_%9W1G&KQNEA*HK?7aN~Qzl5hdPWRNHwS|4!UVF!G4UNn}PUjq*=*YulW-XJS zDI_f~GrXz&kE~n>(PB2YZpEEE>82Qa6>}+wiH+6G@7Xg0o_;Y9Wf7H5f(o6L8)VM# z{pY=)0S@m}JS{Eot5OM=n3>tT7w~>~zpbOx;Qqv0DkLKX)Yq^pHi9q}w1R?Pla1rh z#cj)>a2?_(=+<=fxJK9{zP^6{vfQ9A-Rfgav?&6*S znSGh=WNo}znQGN{0vRZFknNd;7>&dCGf5grt2RV9OR!9b!pVfL#j^VNxe| zDc&_D)euK5s0nh_j}hCSZe;e$Ml&@td@GJuBcH}S*)3N4>}cX>=M}qDCr18Z#eqgS z|D$ucIr%w#_bl@jS9KF_8n^ll)Rx}O4I8uA^^oR{h1XZ^u-0T^NgrvrG4j`9ue*EJ zSnc~pW1BMYq5pJqxoO-vy45Q*Ji+hUBkk~}uiRVZEnmqwC>=~`?>#P_vdA&2xi75= zex$f$rM>BMbN-2j^Nq1R4ko{fl*sk_$74+UojXS#u=H15mrHf|HECt5x4k<1P-&Tl z9aCe(_Lri{)~|JS{CJoozkR1y@b4Ha?;hkWO;q!h2fg8*qm9352=6tI5g+|K{Wdnv zLqq;HOSeaN?-O_>@M9sr%qjYbIHvUxW*Y%^U_bZoLKPLaL3gg-(Gs8SM}smKCNu+R zoED3ytb1)#RzJ<1T-)MUxscx9V7(zkI*2-2+WIBYZgbJ*cUxDjS9-xeL*fqZmNW$REgO+8;Rp6VWz z8J%#l><wE}IdyXDMA?S}878udy}vTbDg%H0LT+x^q&5&R)p92vEW|j-P3F-a?yjNWO8i-H%%%Gw{RkujVa&UgYHO z@MZruZ?b1Mr}NLpTxQobI+js_{#tg$-XSe5O!{ zNE>SwD4p$QR_`t?ES3^nc{ew}>vYMtN=n+mYK8giG5J1soq=vRw?)oY3LYa3{*mFy zQ)WlnuL}MUBDyEDpdiqDufuX^rr^f@{q0gCR=lyC4z?k~_2Mnd3v_V3WirbTYyR6y zUtX_l2%a&p0# z!Ss`-PT?J}xPZXG+eu{q{)<=Jp|GR|2^*min<0OtTJ5o3l}eZiK`G#PH?vPnTDxxn zS6Mvq(k9_-MN0M81Ns+=dcFEpqrzy@@3~QO(yH89{l+NL84}PtyYip?N)vT&<6?8g zv81MnKpOSMvbh6o()Y~j?&T{Sq)VX{hn-%y=+iO&<_2Azp~(t8*~H`a$~)JZzOX$p zF=aQED*QPS*)b6zs1|FWq+3gVxM}!hOGa%gb(!)liW{pP>gp>O?Uke0k`CQ_b%@TzvX05Wg5*yn!I~C z5w4SNYUK68u!+BAp`pX9Z9y|KbN_Ru1&eZL87d{CB$Ea^){@ zljM%i=G-&tj!7#VSyakjDh_s9-KxHrXE(6??*6XTDKm%e2C_qkk~5u5GJdYkOxS$+ zSdxn^56 zoh;?~g7=d9I?9&Y+HGGVB@O=(x;_7%30wy2pP-E7{f=4RrkSAyxWxClj$iN?ZYa=4 z@fKeLabfnasUG$2{Fz*ds}o>Hf}0#d*y$v%Cy;u&MNEi7dDEssGfg5iW>Ao_!sABe*8f%Q`zhboVei zxa*;H4Q*_m`M|rQUHLQz4|V7dR$aF&7?+nF@b5huz5moFPMdLa$;gQ2k>{;{J> z)OC67vPIe0xV)%FaQ;f(AlkRidToz_g6a)5i~q4qIg^J<-yYCIV4^=o$Nb?)0^fxU z)q}(SDa+O+lqJrUJX4fhj292*$n}_bHy^Z#AffJ2Jei-W2r2*5qb~nv&l5gpnQ$8( zetaQ5kFi@JA*Gpj8}UURxj_z7)53qW!#*!h`!Glx` zBahE}1#w)f5T%`e=VTq2St?t(ok;-#Id8ra{6v#GWhJh7w{;$7khsQEwixEtElS-b zmsb0Ua+~8t+Jq{@w{bM_UNZ$%r}x-+F?J640#yQkQa0hGO1wWrc?fFMl*QN^Idpc_h zJ>A&Fa`I6xTkLaLYBE!H1K{6Y#TglZ#@A5yAB5x9}^vUJ|<;ZEX_*nEd4VFf<- z`=md6x^8F6HMpK76`msN=;$X3w32S1m!cf^i2ciiv-VcTeQIf|E7Ga2UMbGbJ_}(z zA?`u$^{lsV)h@8AKSzDfp54Hsn9|a=F>>bJw%7W;Dku9)!8Bh%(dfUQUmk@okBZUJ zv$r(Odhr?bms$dFOioMFXloOwtQKQDbxM}~c7ndzzIme2+TsazelDqFnTnFSCE8@M z{^uH#J;PcBX3UzE92dNJv*RxVj^J!vJ`i}TBPf|cBJ!@Y(-!xAib>zK;tiQ54(^he zt(cOaii~^q{LwAhPrS3RZPEhc6kBUYD8oS3)t@P~QHeA$R$J)qy3J2iBPwgxNW zyTuGX{r8O5&q2ah9^yqttZBSW_F_O{t|ILtMzlZs3?t7ddT3KOQ3{;CsFRRLpY2gX zWEm+K^~u%WmnO9##^=}PY|}+O{prs?zkcax-{CRL9NIZxGTeVb-0auQ(EpAJ%y`T( zsrIt)a>;p=T}ZrYC-Akc0R8Njj+MskP2-DB6^Lw!cM6jQxR*6W`}Ng$>*xX#8?9n+ zj+{To*B{HRTYp@n1~LgaCWVo2j5rE;YezpfA8{NC7mZyD{gRULN@hRpb7*RSef4p6 zh_)@Y`CM!nG4XC#+r(+%DGdW>U2@%YNJ2%Z)I#Uap6nri;ekv*uka{S8%%sm8J;D^ zajCs8u+V-~V#D&I^H%yjy+}qW`K!LpOLff4A4c?Y3!Ee*(vI5QlBH`|^eSRG)eR@pi_~mTk$U zeVK3H(#I!MjlbC~HvjH+FMDL$;!GyY9}6EJ51Bs2R2Qw1)68Lc>xW%`HC(k?(w&!j z?a|%>1zFk5s;Veo-$_b--iwDcFUVE6g=?NoeBFF3JC)0frTsqfK}MDgS#9my)7Q)m zn_}4bzver$QZ#Qr7S`Gn3x4Jf`g7+lb!db}8%#Ow<*|rSJ?3aGM@wsyl4>MxY3Z4r z9ai|+SJvHqJa}%O`ax2+T-~crzRj5NYTnFI@ed$R?aFDfe!Z>2Vba-t?oJaKIG*R8 zo^qI-36#EPpK|oBR{ECvPq~cGCqo7e;SaTYzn*Zh-bzEN-(O&=cbCVUmK*Q1kA0sV zC-1*1{-i0el;bMYYeM_?E9&YAB)oaUDBiqSRwlELAEptmWMq6B8xibJql@UI4NT2w zGdV4Ft$Clb`7_h9lBwZ9^14&ZRl9bls!|8W)tx<-&S@mjWcsoZo<);VS)uCL66-6@ zWfuR1SDr)}xooMyKZNa+2xu*0iyv}M{#b=lC%n)n&&X$DrT>@EM0>bQl>acLZINVt zFPFIB>Xfsw-f8RK7t|OjdVR#s4w&CGb$UZ}*;_c9lx@&>}*%vTswV zWDg;GC_9tv%a}?iYmsD~>^s@_nJ3B`V;{R@8H_QunPJAv{O@^w@B90|J|8X87~}iB z?sJ`Uu5;xv_7S5&3>v$Hr$ep9?y7;opcFi?5daF6By-o)fc+8U;N56j_rceHmnq!m zR#$z%pv_h9_Cfmz-^>jyVqK|m2|hs;^~>y0KF zNa9D(n-y?8=4!mVp^c4s+SF-5fgHd%uHbADCEba^jz%t)sFEacR$?C%S;4QOZ8cR+9$P)TM~qx!q~#>QiiJ1EdZ@< zL40BpjkgVO4zmex)`<4fqD(tPC&~HWizM%Vucp?Lie|QJUZ%g(slrV%QuFrfeLViE zJvc40yMgdq0bE8|C`4>5-&~NVq6DR=vz6g2y}g6WOm1!{vTHX%Ac{%|X?c^{6|;Mz zj(BtPODL)FKk_L`MNM0ltjsP-@;Slfzp7RStu@`ytIDE+7Ls_zLzTz#?aK(cwro~O zz8pSeVAe$!Yq#;?p zl~=xVD>-I3&tKm}*WM?2BS)^2{Asm_VN*28CMXho_meseCoboy9~8jnuT{o`+0Sr!c))4+;>slAV@32I?E4xkY56(;jg$!V4tHQl^%P$1Zl9G@ss5o|x0}r{s}e=? zGJi&cMz9%I(cl!tAyG+1PY0!J;LtR|oqy{f_OCK{P=!1ld%L;fc~Q|vi+;@xTvYIs z=brZi1Hr3+3O<9h{L1|C44(7HQF$FPTu5o-x&8j|sCYtyT~8xA`hOI=RM3YGV~}+Z z2(4zhPY3qYy$P+umNV#*3W2e9Ht0y5NBN7ZW!~U$UYiXrz-2{BCRFqF6~D^r_;+R} z0|OG32}&0`groj5>dgYCdF`3AIoUaAh80LbV>$Yn$)~s$3>VeN@+$rf=7LAZ#(D?} zHuVDs7e%E%{A=O|3a{ILoPr=7E(>smkDS0%2m9gM<0^!foZG+GTTVv*%m(3^r15MV zaGAKB5mYY)D&+qp<-PLWb(A{i@?w`=2@LJo!}fbiY688rq6rzR z-*lnue%?W)1@A9iUs}4aopAH!p(dp`q1!@O_<3{mfDq@8;4QksA#QhiC*wzlo7Z&?6rq^`YWcD8I}>IlTdU zYkThv<@W?Fh|xV$)_+W2_x-4xZF*i~?__2gZ9U8_ss6w@fO>4U$z)9Ozy?9AgD@2x;3?nZ2o zRF!jbY!t{d3dWFw`JYkm->>hg!@ipezeeOdjqDvrwqG;JP$4MNQguCo@!y0*FC>tV zX1USogCwf1_hf^AGmnR@{`G}LV!Jf4M<<)m1a^)r zg;sr3;-6c(inj*QKK4yE7ZU}v%)N(w7wS1{s8n~or6Si>@}4B##$>>;fWY^cg<=^Dag99f@12QGazF5Q%og8UV6+my1E zl^t8;Mjs&G>ZWYWW+D+dChVVr&Si@l_mkGT0V!3D78t-)2oA>a=Sfhy)9orBn%O9s z&QWLYTKk!I!jS!P0~8IvQ6Nq?{fWZ15i1=K#*XE01gwkaq3slw}3% zDmX|n^K&pZwVFGSy*tJg-vK$saUsMe6#fXp*mqQ;teoNC;)6KA8B|s|4~t%h?kJAV z-5EkTW~@vCBBi5;xJGF4jtbptk4{P$qBt!tLvTC>YPAXn79QoT-dw)Ev6LmL`jtUV z|H9HzBBo<+OCN?+;ab~hF6ahXAq-5=n*U%1J;jI$7gT`vBT8yJn+p1Cb8~;JRrNk_ zBd#sXU#xbzg*x)T28!yS8YowH-vGbV5k0Z&>O106qEd zhh~NbAvh%u%mTLUynzt_KrFLlTe@IUQZ21nIl0%QB%Bp7=%l{mmv;Jt5b$BZfV~wph#L-9P50raoxhOjcE+tSUS~3i^zi(GI?Z$KR?S!yEk@qx@>=)QiXC zJj4eD)%Tl#qUb!h_aNcT>kp)I$swpZspl0%2}X>i;9yW3pq1PD9;T{TKhA?6pv5E@ z_^~tOdMA_DHhc6SylrqxUxcVrfVL}fxuS06JsLH$jBAiPqGBENY$DoXzkt6?o>#?; zNvnt2Cr7c>!9q4lnUL%U*Q_Z%GTzu;yA-+kV~De_b(S{l2Oiy&W8;RL>}%J)_pM75 zECX3!7FuQa{Wf(flL@y_w7rtagFkon`79VMFJ8{M(D10)!3a13H=>>vOg2#2rK;Fq z=(U|3&hBbv1A;>x{;aYvigD0J<1EbJL$2+CeRX-uw$jovr+-UjuEMNh_WC+%O^`8; zar91j^w0uD&y{RmH;2)1ZWm70&W)FdSS1LQ=nOd|2@D4%CMX+((E~7P7CZIU9>|EE zld2TV-)_pZ(1B@iMi6mnwqyNEnxiWo)Ji45C_!M9(`E1`;_?M-uO)uifh>ucLe0(2 zyO}3w#|M6*hIFE9GgTktc8<#`$QX&EXvoZiF{+sIAJ1T04jbBTL7MFIp1E2$e2UQQ z5ePbxR0zYR%*uS+@m7lW=*XTAjVR?ZQDs)+TcYEk26J=eB~HzGVn&J#T<4J6lGsBL zycL+elpmfD#&QuDRZn6XmjUnqX-K#KyyQW<+b5U{sL$~HV5-Z)u_KG<*~Eftqwe}os4vRE(^wE8;dJkg9y#r)wkqAbnTNp=Q18KM52Zas4YLGV&;MXDW4%a%iW2Zj6)A+B5Dy#=Wq6p|o$*-CSULgTLxg$7QkB>IlnUWTsGea$XLe3tS>&ZLQJFe7YS;%Z-9K}D-yu5mq%4PbM?wpU|O z$r5P;B4kS=qY_rBkO%SVO~!H_s)%~1|LTwqVl*dyW90=LzSn*Rtb`N$_ZDFwZk@SU zH~_qJ1E6G2=Ars*LiayZRS94~k1L;)jHz$NXR0lM#TK<fL%SECha#3Yq6u+Z{MIS=Tq}2}uEUqPF;9G$&1y4boCMtaMZd-c?SS z#F*kpQ*#4Z8fAdl!>XlM7k&7Vm=NAg+&_X^ux0v96G0) zNRwP`WDzZgPT>^?!QL^zT5sgg39W|g}5X=>(c^M*w^m4Fysr5!tTN5Ya6H5;~~TfE-n>+_d4G(6L#l#xjeGqS3yg7;XZjX(=>1`xuH-G(F{*Nv*EN0N-+9WFf zp;M4$85L_2i-6pn;EMP3N1)-*RzFDywJGXdjq7$EkfRcdR~jCgWEKx*nj8@mcXpNi zQ$|`P*am5m=`NKw9sSDBLG{AT3}C)V#oD+F9B#v(dOKZrN1om9e9(h09^LSdE82ZB zjLXTYGeoO5nA*ZTQ?l)kj^*(&o1Y43nISC6;M_-3J#$%82dCWYV!}p1lB~ha95M+F zI7J?lT_f1q6fuI*~)|nuVs8I83Dp*T$!->H zLqKo==@WGVSV;c$sPOQ%{$Q&XrzQ+)r+vu%^7gvih+J{YS8V`z#9SPPg3Q2=;v4fo zIYQH8d!X%@y|Le3tIf+Zw}DT1@Dun9iid<1IdI>4?Oj}FechXNH0v)!3Qp-sEs9Ha zmxGvXI@sv$8#K~z zc@y*1(Y4Y0AVv7iX7FT1XsrooK$u)*>-Mem`FC*x*RI<8cE5W9LR*xLmWD6K8-$O> zZtuULP?P6Oyyz48Q9w~n^?M%iKwm)8GS{Ra-pq|$obS-z1mD1KZ|%}V{M~*M>cZWk z@Asst+%*Tv<%f$o51_Y2NJ(yl;=B86H1LcS;H~%18J@EUm|7GFz9_W11YUqGHK#c@ zc4G%$rU&O{|K;G2PY0*}8f|KQ$zMJKF6v?>yoVE1ub<3)&hFz)_A7GV&*g2sH;k&a z@EJ8hyWl6Q_0<0Sx#HU^!7k^d-?UCT&ae8U=-)CYq88d>k!(XLed$dYczZ%CBj^T` z$hKuPX@YxROUqBfQvQw-c-3QTR~Nf^TMw!#8;v~549u*y1T~Bka)p{b8!qcqJLZYX z$Y@KNWqg)`aC$y`2~=1~)B$9&4sXn>Qin}#|0^sOp#9$gJJV5eO=v8qm#j` z^uJ)D3YdnkPncfz{&W|nmwkZ!Bx`K+#|}X5%hh^&OM&!Hpl)%9n{s#9J2?3MgQD65?p!j5a`uGXr-yIZ_f6PIGB^u!cUrv;i+q- zg7g6_OWE>rCQ}IL-YhyOn&JncF zsOz2!qXkiN%6FS4stc^5xlmt$PNCr!+_^)8eJ4?ON6hw5`Rdv~F1Xn##9gmaZ*WJG zR3l;6O&|b+5nH6To6r6s(9$NuSgp#h8LihUkau;dI9v2#pmii8h?*k@XWKfeZ?{9 znQBfMGPg{c=Om6hF^z&b1w!?~X$!%OcpXVhpA)H1_`@DU)XnQnmilJZ!!+7UpXn}tsL;9mocCHRG10madGtQ)p8xmf{A z$laUwlc4pa3rzV3w~VUe%}vghCqC$KJ2*IppUp=$O?chRO6=@*>>R#@xOMDJhKf!* zLgLzLk9a4E0y`Qu1j=hxYp{V%f@HIM&1Xsef={2GBz+oLi&cobPHtZSOJj$UQ7zi- zk)12-h!rbHDzHK@d2gmSdvTHb{sW7pm!mGgEMb8>c+y}nz@(GmG>o~S3>2OP<%4R3@c^?6tcu_?L)B04 z_|LW8zmhq^H*A^G6&3J*7#C&XUlsrceZYYy7B>TT@YsHzHLVqxHKA&t}j zYlV+UcwO7`|F@jJvZ)TGonTTsZD2K#7x14wPn~mb+xAB_KVncYp)=m-li-JEV_wE@ z;4UsvHj1xDo%6hz_~C3`ry$A{Jk{Rr8a7DCd*pT!dG`l<^EElSszy5H25imP`1BON z`hmIDkOrH_09)uc!vtB^#~}eN3L*2XgyQT!>x;t-O2tJQ1rvRN-{Sy-i-(82_e9b3 zTFEvaLy0A28MBObHY~3&V)jLQfEsDMR@)%{mOGhPfgT3EvG1^61z`DXh{RABWw;l_ zO%zx>bLnG)H!ufdY^jr0fPzb|Y3O56;jBJ=a%i{CxO}Zez`6m#4xW`?GkD=LAoBo` zCqB*6ZU0dqjOnZ@I!q)x*K=_IZF%6H(ua&<`0;znh>=89WiRJ6JNAe#zXzqNGz`YIOCe{)|_? zUGGrd0@!aV=pwUE)PyuWLG1nDv#4Hf5Lz91eX@*>leDJVR$H#S(fcNt^pKVju%u8i zoGz0lDg2J(+dbbmF-+_c+Sl*`wuLwL0q;UubD6Yf0U_OR^k6z4TK?kN4N!H`Y?TYY z{J1_tufCt!@Q-5P?jM!>>+tpyv>o`I@$Ys=GjX7ZGC{F7f@w6%=tU#TBjWW&GUBbW zc28(U;LGWMZZMxv*#THSTn4}k8s?vY+(rO@kVE?6nN%QkKEBR9^p;g~rB2Yg`0cZW z!^;55WtoMx7+TtTxSlcr1_!1AKfK-H?iwP?}@3 z`kw!=kQb2cq#y^c)yaDa>KVspOa#m_{93&3f$FIcveU+iO710OmTNQ?>{dX_KiB6M z?e~H%Jd3GQj`;D8$D}3Z_o}Puv2(?^R#$2P4nvCeG>lZ?5yUSzW08%o5FDG1ucPt4 z1g#Q;0nIo-=3|%zw8=lBeDBw(`*njgMZ1NTxXfAck}~RPQ>H(AmQIroS6|nVakQw~ zpv;T{JZd?k-sU)(&L_OmHYM!#Yov}3cR{dp0EMc301j97G0cHSvZJ+vO}B$3^!#EBK%w(m5>FG6%GEnt>ecgy>V(USgbb9|$GZ@1{e4NAqjeZIw_#Q%zE^Q6nc4A> zkfMlq8ONr@L}Qbnkuo@q5d4zEgn1QKBVd-n1mdlMg!MPiG}R30}2K!szCnaiQG2+;1~Ax3AQ;qCSN_tVup87rE6 zwSM^b)k@d`+&&gMGVWM0EXde940O(68&dtt#D&AvqdOA`ffq~6cfe5 zX}CX}2pEz00t}dz{J9%$h()p`ht4yV*^59x&a6V(w{$%Rrf`M z!aBf^vt8^5QKQUI5auj=HFFYkS@rtSXdqRBEWVa`rBdyjJREZZZtl1iqey{i80@K;5@!k z4Kr2WTIdzhV$Kh3p;7z-jDrQ8PqI}w)-i&)N#P^u+;bD{KAmW;+7;1nE4OadY@=0b z)++OrI)ULzUbWf)i0t)!FyvO3V-iO?4dF(*mg74WFKOalz2AYEh3GJC?^IJRtps3G z!i^$4NAM8K5qm^?_SH`tva|Fp%{r1ZDwAGZ(&`E3FGu6y=_P`+m4q{7JnP#@TIjQy zt=h4b(>`L}4q;x+7XD$bVy#gIu8f0s^^*l=@X1}?nkTu>*(BYo6(0KU`ow=`5AHeT zOZj1!kPFuKj-ovxVQ z%e=|-ZpC?ccP|tt-C)iQ0D#LR2*A9-Xwlq$DXzn93E}@XxW8q;M52jofa&2JrBF41 z>k>S+l9fpQ4>9ajbN*QaCyw)5wO+keR?pZ*wFL3LyF(5l`Uo=x53 z$JV|d<5~egZ#6U6w6Id9@X^C220f{1t+-NK%cXVh!>H?^CPev;Ep5%++Hc~EJYyCR z4=Zshvm+g`1wvYky-s!D3y`(;`Z+GY<2ko6L?c(6V(_zbaAau;G+F@>C(hkN6;kIe zI);WL8;9X!Ij&O?yGa!-1)$Z`3|I%G91g;0ME0S_Dl0!yddwVgthQeO{B@?${>(>8 z4FVd2vM4flihKwBqbY7VO{8|^ns@oX*th>c!V9f7yt|TPRvWM$f6)t_4xQSN*8#N8ns1=q=% z3+;YP9{^=5-kaD;@c59s>F_C;?DF$x2A_nnNg;azSF{lOzhr52vRUoQ6mhdXnW;Tv0G_+1m8Ouq50?K5+Mg_k$kS3h@iS_EVMR%$cnS5Gu8Xrl5kuZQ&A({QqjV zw>yJ2)`b5*b^P~a7+bm7O1r?Bj~`82iDMEHyeBifI{CR)Kwn^y(8g{wwNnNyOYAif zDRTL0{Nw*)2$Oc4$D(M{F@+wS+Jt(~D^l8ndi_E5g{A^0zR6w4z2m z(pZmb0HEo(`cczDKQk$JzTIIL08OVaiU$>NerInu_%wqOQ;u+)?ZKayv=z{xn9gi=AxyFM!B~i!k z`~5|1>PtT(5_3FqJYNYV^_3r*k#QlTT1dxGMh()vAM;3>=}v8i`_~FfI>Y`rlYZv- zu4Xv##bk_ImTzYlo7SHOw<(qEl$JNap6x>rtZ*6pcHcGiANVNNajnY0fHmGEy>P69 zQA3-CX2KR1=bLc;X5yC{pC`ULo@OFQBOSUHt2(tQ2K_n6IYso(div0?QhsJ9a!j!E z9Z1&P>j1SPsP0ZzC0>%L=1MvEnXj}v5dTQ^jkQf#z~QM4Pjff+esat!4{(pD#Ct-$ z5w|$whp~dT#oftqF7-TmAqm!qgC&lhE~TmU=L_&jNG6JyRt84u-s0jzgI|}^Dkh@b z7aGL1ip=$Awp*i{N1t80aB`J7jl^X8AmYz#Z(1N|y$OD4XJF%@iFnYK(-bNhz zE+73294WKGsuiOi%?n-2)vTSsHSfFdH)~s8xdrfQev%Wh?nSnK>+?_!jyXQoz;S)N z#B1~j<@@P{8hSV%k!UF_HpNp`_^ujJt&b?j7=*qg$;o)8Ylxdp7gss}dtY{r2W1#h zKIl>ZJhD@L@AoI0fXk?#bsmktKO02f=d+}(dzX{Jps@&Iny;LIf@sD=kPUqiH+w(p zjod<8ah02ui46gcpv;r?_BPmmYW(ft5te^WfqVf8&g&py10<>*eFZYg{vC528Xo?4 z5fV%FB9;%`&5Sq#+e|@CSs|n%s6L7p%b>?kNrObqq&c`>r6I^&{NL~^V8XwH)Zvs~Q60`al z_OIe7tS@eMSPl~aiq8b&T&cC2+4Ye|fy<_D5B+CXN5?C_Z$1GzzyPF8HJWrHw_?dA zhF*TN)1ZH^4c*PrvAA&&Ud|*{L^7Y0MoS6!+K^{5!R!ikBdrKvZlnA6xGQ$vM}fto z_Z?~|8I~OUFhBZMD#RTsL$FA2w1wEz73Ob!h2(-pYh#lhPmnO{&Z?`Q4ZNv|W zg1a?tR}r=$6`eHQWA$9CiGnom`4U*iXzTY2@@9rA;7Xi=*AkqBdEGsgQk68g(Tn2Kfa6+Ay}w(0%z<^(!uI#s2a`0DR@RVQ}u=_ z3Alg_&CRW&J<@i3U$n&N+zfRrRy3I@`o$q)>1pQGLbQ>^TF|G;iYe0MmNa4C5<$)2 zpeA725964xxQag)<$Dho=0wIZJYnWcn`Y799H0aMR`dfHVrfr{sy(Q;B=+*zjDW%7 zx+e5arISgPz957;&K#Ls-;$t)rsMc3=CXAN>BVp7J;>+3Sx5km|Cu!-c7MLoVai%e z9gBsQD>Bx}=ZEc=O31h+&<+YhX!-J}fJe`d2mp0pdvu@KJp-z7fOXEc!ggji0nFTw zE{CGufoHc&u)fh7SwVQiFG*Y7sQ-kTe|vzu9njoy6b1bKVfS87$?ff~j!?~Q~IvlCy{3mj%c30e68L022 z)XvP!9Vam35P@-ks?;-*ftR>ZLEqU4CoB<{Say+NF&xr-RdYI{$r}D4&GI5Wh_ZX3 zo$3x%+S#WQ;I`Rmo~h$&@`!`QPn&c431Hv~O!74qje1;LO`A&xyI#$iX)$%wThAPo znfC{nM&l8n(g^-FY2;|z5D;aj*LkI6G#gl<3Xum(6Z7=9Whn~Kw>AOuIc21{)rT5k zwt#cNQQ8Ijmz&75t3mrq@paS*3iE`VJ6P7XesKq<7TVwNC#n)~n4I@R$E!=4CJPEO zU8yj4X=3DhF}N+_L!PB)2H1O*#M6Motn7(biIGu1k^qX6vYh-;3-_tdSOm&Y!S6-L zHZ_bgRzDW*R&dJ#Sl3GO^>kb+%K&a<5^QsO9TRR{aR8VB!-zS8226q3()=b6)>+cS7QM(%aKKLeC=C$KT&dUv1^@wX7N=u!$t}k10-6 zq5aeem}st`OjUo%nvZhEGc|-pU|@A2buAie|FyTLioh1v=?69_{P%79%PrP@Vg6a2?+4gL zI{x&l;O!n%pfbhRkZoHvB4YN=jBXqME?{EIJLEPCtx8qn)2*Q6F0COswTfS zVmW)Uv~Al~x3aeH0qdC%oT`xX_ULNel!)JF4YP?qB;*eK#$C&>?_-%YKlYym+NK8l zJT4UL3qMF{k+mL>?5^-q-ssA=w>*1_wty=UD0tJODR?KqP^*0EbUFVbBBn5$9GbE+&HDfTK<#*O3`5RqRK*_{PEZ`KSta%Cnr1WlN&EK zV|&@_p7OFZtaBx1ZFMm>w zprZ(rPJ=Nh8;ZwLa9-&)(6(m6;PBNWHYEnHz&D>idh_ zsb~*4O8{M}TKd|x!_kRvybK%_Hv~B%dK1GwaAZBGK^=O@lB|03T&U-)I6wcylbGr} z((m+Z2737O{|u)HKMDWIrdLzDgW<})U*KQ;HXyEo`0eebdTJGSdEB{|EQ9ou&+Zq> z)HEKC@FleGEksK5PrqGe4?J$#7|IADw1%t@NoTuIG>W?!(hT;IPn82+$=O8`5vR&JMm@7}#u zJ>kSVC8H-P%C@=5@76&^@L`W6?a?JU@Nf6-g{9e4Xo8h(U!|_*km;Wvf)62w2L>)V zM@B?wKR)r1@I_a6eWF@(Kmjb}JAAB}!7l>Fp@~qCKd2aN8gOnLtUz#yx_rfb{M%ng z!|n{!2W;7QJ)=3;Ts(JfFubs;>U>gC(ie(3jx|%+$0}&eiGB0eUtvp$##uhMpcbnI|Cku)hc9S4 z&9FJW02d3V$PtIf4IN7@ZnjwBu9JP0C#V2Qb$-FkJdAy`T|CZ1p&1R;;^w*Rj5yy& z)87yy3>83SlzCkI9c{0X(x(_?b$6%1@0~HTT8F? zv9L56MSNIcIqV!>A*B(0ZjR=`hNSHDZ>YzV*RJgDQKU%3?!}wGntQ#4KPe~)B_`)z zy>Yo~OdZ)Cc;@|2`}-vt>In#o_iXUWbIts9Ifw_Z2gYv~{!KgFJC>}l(0UAhd&3ot zz2d-``+J!Sds#?~h2>Fb=@|l~K>+>h;4E5#E|_U~08v%hxq3jw@;x#f*is9+>v(u0 z5PLpX|E#~~wFeui!-`FLKgQy8*gCtEX8D^%tkf@dn{S{`U}u#~KYXvjvSO*vXD5f9 zOY58nRz8e9$%>_on>>Ac2mVtYJhzh7FmU8EwUirLcgNIJwj4E!q?t7<>9ggB zJcNoSy=BVZazJ`4^&Hov+<)er)&frF8%T+l18VHQYoI+)VM-*=j>3F+)BzC`1n7csc zX{kpHVU*ucu0WM!{^G+1Mwk;k3DhTWudd@T@x4Ky@4J8#-m| zw(cB{+pVv4>z7sJ$8zUqJ?@s->NzKX!50|XXKS1$KJP13FWYIbdaV7f^Y>|%hrI01 zn>}uRF(gPA1zq~k{rpQv5*E*WVIy8Qax;M)lGHwe)({o8k~wrT4~rPm^*dfw^`-Rs z_=f5~9hRaC_Fo@Q5Ky}>%7(r6`J$eB9k)(}r8SV(H>7L0w+NAA3%@O>!M{r_A2{q+ z-@4}^4n2)H4{f%#9AOEI{-%t)2^=BEOC+ z0fnmm-Ep{8VBRL|^1Zx&3SXQC1!;j4Bdwas@%q?Hdl_gYR+^Qi%hCa^JE&3n5V5fj z<79czXHeG~nK~(PeVUN=;crN++Sl@B-M=SOPkBBF9^2>M`XMFaaNpv(Kws29cpkea z=@(*7Y+2lfvdxQ2NofWLH~0F6?!>BfT~Svr?DhVmtE>CM~x>~eHQmDX1--{o0dS^wcZ_#?&m5QaF#u9a`)anuBLvj zd3?x6n=4hgkUNPW^3mo+41jnmhrGcKzb7|1EiJPiJstmMcixnFnAxxTDBrOx8PzsA z_E8G#L(9p#ACT1r()HdKUj{dUt*F@feg44v<07un3SSD_5ZL?2dZhSH;%k&hVBceSeZR$-hp_L14r|9Rb z@9vlPU5$<-r7!*NFE$Z*>h}dsEwR;G9i)V;PPz@B<6To)Slnn1r%nPx&r4+jr5+*( z+%kF`ujZleu{_xy^0Nr8|~hD4n+x7%A5$g8cLd9;#LwLl%KV)vL7YX z<^0(*kC%&0PkPshLh^ch3bKaVYHmbLIN#AQ5$7$ou#@*ah%heyWH%Ny*g&4qX}Xvk zl)d&?4GEmimX_t8=GW$Q*?s%}IBy$~2|rnW7ns@~jYU{CsA{W-6cFslxktSD(|2Q-mA^ttx4u2^8~%c!TY3nu@Xc#G8p~ z$j^@0YjxCM;%MOBf}j(rho8To>X&VpMmM)Vl|G>ku?<{$AtV%_-f#P|`%!NG zB(^TPZ*C2bwU>Tzu>Qylb2LL=%I-fV$N&BH@BU0RlJv)cM{*sp z>|Cc*obSo7j#BeIckOinS1|6Ih= zkL-)UojmTDrpIY`=Z4YCQcT9kiONUC>ySoK;$W^{xKSKSS;v@Xsi&`U`Q|IN8<(%& z4g0h{d%_!9?AD&^I|n8@!Q_~`&K_Y-0z3*DUT8k=hD@t(=kWci}lys z@mcH+_bY6FJ<%vJ{8o{VKFjsXqmD&J=JiAUx??H39gWfBoZe1NUfa){l+jb5UZf-_Ep!D8OIiVi?oj3bG`78)#S2B zhhVbMk?$dGNJhtx!@uivBCkjeuxUr$E&^+0zT?L2E5GLE9`66r$GJY^2G{45)IGq6F1T;te=&hm)8>T>$E7ab1gWoukN?p1^}!`N{=C7_D5)e#ii%U%v5s`SRxC%Lf+ciE<^QQKKbhtdz%M9L&9} zhWEka9A=7~JRe||TXU*$9Qmr&8{uk(K$)y(V)jGH=n7Ik`PxcwX-Zr_XSE3XM% zifRbT)DvbmEw_44^0V{USw3AqGh177h)$r}xd@Fw3pX=V8Y2swH}IrRkIFy$PjCylRa50MK=PpaFw+n(BfsP??Tr!acSn~ zF5gX$dI?8y_5D#tSqv^CPA=`nll@=vvWn}+uslS^WOC(vpXvyw+@X~&-foNQq2)~q zAsK`A&T4qN$)VsOY5%5C8d^ZITe#aAE3J=uw;IYVz{iy-N} z`svf=o^B#BG5>(pZPz)YTygn>Y(tlwi~qRf?M@PiB3e}=-00X8-TEND-kAZu>xQMJ z{VwxNJKQavUr%V;Ht|GYp6O5_m$&Epv$eVVv}#!Dn!4}%tJ^f#b8L@gk{WDec-<_v z=o+{q@&1Eh?9xKUR;I&&^vhyGah%DCBTCT)yWgZ8I_)d7uUA+|+bmV=77Lh(8H_pX z1oI;*4rEW>2y`tr7z>`=FzNJigm+q%ewnxU)#6fPfFZ;UKE1o)N)0`CA?tgW`JaBZ ztLHTiPdstbN!#2f`2X|g56jf!sVIZY=|fKE96UVo%QI-YT3U%;Nqc*jFk!G?TWb9o z{a9ju|6tQqN&USiy*;TJao(zC2w721Ev?J?oy3v%?8Gnf_^XS{el5!;Vn@{JdqxNA zP7A?Hbe4Jf!VGI_#1Co-S35DnXPal%~x=#W1 z>(c-E$d&BB@V|Jd;im!W_rGQ2{NtVU-hQ{8&nBd~<%vopd7e*M zApP~~m+n@)#xDoI=?C`7N@%pf>u=Jhi_u4}znjpvZ;NHQbiFI|WASHBY~odZHv{dH z4_$q=aBxU)t54Im!IVEJf~8kA*WByF!b~0-a4~pSJRCaSvA+7&7BxO^ z2RinW6R{=|10B8@k0ASNKI=BV7g-+J&c-A*i%DN8zzO|swaxqnjPJwOWjwtIZ;KSS z*-YM_Jb#GAYUH|m*UY@I=;lR0N|4=4qizL@dj%NFh@I^JFzl?@je3`O2Tl56|#}CD~HH{AUJ8yz&b6wg0*L4YBf%QRl`?+KBDsS~% zYF#VZG&5UBr}T)YZr#gJd;ILB=>>3BILlio z({GvomfX26j!cc+8(~(@o7+%4! zcV{`?94sEkTt8dBu_tx!TBMaQdnv)^HBBi^ad7`g1}MEiRx&hoWJ?hRJ3hbJwKv5` zNO;^W(YS45`gw0*Z@jXtlF1OiInwQ!%p)5(Df;Nbgh0bCf-d^_1%j@gv+T8Fy zIT(Jsf-6GKRhF0QDrgac%&^g$e#nPX=O#eX=84Ic0V!|<42J)Fc60v6d-gX-kXQJh zh6B1n4B|f37CjUm5k-|?H-!5x|KfRU2F?Yraz&^(0(3GVMTLbA2lVy*7*mg~tnoKC zHiE4i|BtJ;fQqVX`-cZbq`Ol>zgH}BnHFBGd0fIM(j=HL4JP1xp~PL8CaI# zS#|<6-!(GdfHSn%w4PKp-Ibs=WsI*MFftVFm8w;|M*U12R3SEA?92ZU%W`3b%{K#$KO{o z%7R`mW>(q->17r76d}SB>-`gyx)YeOq4=^U-V(TH23BK@ceVYhS58U@|3VZ$I>c`gIjn_kxAFEITi?ZJW)G0?{KWnM2)cj&`05=-wlF~r1c`V$Hc3JsVyBJ zw_AI`lQuJL-$Fcs7Ps%opwZ<=*QN;Ht(9aCCrvRYTnPbrja@F46PnC^`ed*fv{Hr7 zS@qw?c}>_lkgme)YJ97G$WC&?KxpM0D&TVcn-46rQNwxk1|G3JMo_oe?8)PN^tGn# z%$*(xETBzL5I%>xVJ;&k~T@@t{rp|}tgeLqloC|M#aP;pbI$96W;}Fk0 z6nWm(Hk7p2{4tFhzeL}(x&1A3+lbQy%y>RC4?`B9n$8HyEk6Q6NsG(rwPxia~u9Y z$IjZ?qY>cM)RMN^TRvEwo!G~-SSHe6Yr7LjvTqC7tC0)4Ji>yIlaF^e)yPk>*Y;|A zl;e($W(|jz=yPghjZe=Q8I9&TgLo`b&^dneW|9O-d@6V`Tc&qaL;5FMA!84KX$VL) z^_;AG^BZE`!AV*Y&s^ww^518Xm&#TkHdxgEcMOs|Lg7Biz2cgd$M$ruX?E(s8T4R{dvxbOZNTrBV-uc_M^D9Rd%JS4 zZTQ}4L+3anY3}~%rCDK7)uOwanrsr=i~X7Bujt&bhF0pnJyZG38~$m}PtC{YWdXAK z7WQs+SiFr1(8Y}E8Wr9rG2e-9-Dx7A z6B{;yBsSMG=*L#)<#-&GnaI?TZp+G;K$K0AkNYm54JHzRkiS_)m}q$wboW_)X)sTygOf-Vl@GvjFHEI$I ziJ~Tbz!TDMLboXG*vM`0vwS(_jv1898YGpxe0(#N1j8CG%NtVLGpl{oY*YuoR_J6( z?@Vyr)7vodFCl@WV#RBMSLT#B*L9Gz{o}WE!?ZT|h2y-e;%9ZhL;*!qaVo z2#vh--Wb>C1RoOXL9A0bD$x{F7VnbDk~qugW9Zxm6kXzW5X~!r>ZTbxXK|~mw!n-( zJ(I~x>6x|Cj2TcX)8&IJu+zljAg|c`oncrmQ5o;*%ADNHz(FT$=-zf27bV`zZ1h1r zZQ{9&013%E-WN5-a*bcUv|Y&N;$@6FDw>&P=9J|#U!nV5p>lFcO1}$pUT$vnTqke> z^wNoEmOirpW5pM#!B829==uob|JPe|}&meU5(P4Um0USs=N}TFou9gtg*bqqKM9KN~u*6YHyj_esRAGRR+b<+!d~)O9K3k79 z%yOb?){5U(PvB;s3+%n|1wVp8o05xrNLHPvBmYjAyKSYxVNMhm*$msj&YUxYviJW| zLijC!PQrSu@l^6Xpls$Yy5*x3UoScs`virYEiqn|)&y%iaITQ;pm$)DGj8d^F9y(L!Uz zd?$WX+8Fx9G$Re&QYjhtz_4w-UX>dQq#yT9J%0S(t|Kfl27CjJ2#SaOekxzgdj3Vj!+BbP3W9&*iQg^X_c3d z+o?#bXZNiMj9xJ6I5`XA%*>1R>=C+G1?A@6a-oshA7Tej8D<#jm`Pjb*Q&325WMi2 z1)y9WCU;Fj3I71JFie_CCUo>7({g5+}G}Z|VKH2$vwCN?Tn=niDV5Ya6H8zO8Sp1!u493{I?wH&tLW z-FOKOXk@X;B$9rY5h?!GAjS>~t95}l_!W`1)Slw#L7f6viMRt2DgHsJ_j$dG4GA!IeY=MLX zBB;jR@wh|6Owr{DF(rfPtHmpO3F)gt$5dJJywQjflV=sa&g>vsNP;oQkv{AONuwn; zo!^y(`1Y+g_8k$G=!xT|I0JWnsyc%R$%k`h|LN@$tz8 zw7GbQ@9OCzGaMPIFu;t5eFPxWd&6hX@SsdGJe&V1=UMK;KRvkhcUbaWmBXy!nv{?* z?uI1c2iH6JbEYH?zDOjJ_3>j06Bs=fq&HJdFfkz^N@u#NeyDkfVrxHaJz>g}L{`+l zwto8LY6`GETxOqRlBC6PApyueg}So^m$tOpMQPhc2GZR2$B3_ZD6@yj%(CnXFLj`f z;W?sub@S^4IEt#OaRm)6_nUh%uuLS7PE3xa*dS8C-n;W%K@? zF6Jgm8Yf@@%403mfUSw$eSOnRZXs1mcCmwJ#>1A3q5IL!@~DICXnxTT55;{?ZOrZp zMwFDZ>&O7P2M8|vIU!Y3x%t(d1WNV%Jt-fZ;sW0uD7He5e&R>(p5s3@4oV2$1=tN^ z_e}V1Z4KbT>y8tnv6)D@f*1x*f|`60w0hZ-z84~(HGZk}n4IGdIpL?Pd3%8kYM7d7)*6L|;ZWD9oc5g1>b=FSoI z=dGo&nZRC2{u~h)qtCh!?jfa0gqXQh=6%xrXL0bQ=>oi^Ex5|!76Xl# zy-knz$il-pUWpBpLim7)=lA9&8ks8)NG|&@bqO>exTmkP#H*W$RfH*TLn2tOD>mD3 zq)lq69r{0e1i43-WWHxeEcl=$@=P>UIuv_9)Stikd`$DvV41I&##&kDax5E*Lm(Dm zsf5i=QrrlB+@hz?(7mmqGiCP$JPgkbN08qF0cuOE{2NO8VtG&l(N-D-)AbtWeM&35iGF=;Ok z#8n;Q4Ll-mE8hgIcxSXwUDynHy_BDCN_{;D+3pj&xM&!1cd*DxKJ&HLahIlwj2?Xy z0>8YtkkFpAeXx_%aH>B!$c?k2Z?md}43qc+=eaQ!pPDh~XDzb3>Vl^%x4 zPHg$wjR%W}&3Xp^fM*;^rhO7u@Ac1zj|m%G+*ABoOAQY&ed%%Iaa#9z=L7(gerdze z#=R*5YR2zlMvs%<@d?~QF%Pjt-mJYJ&M9*73|<5%fB(?nWO=)+&%AKddxqYTr4UV!J|uzjs)uGE0+ z5k8M#b=)FW5D5lmrb4>B&26PS6TNn_1*+8IBn%Aq8iXyp@w=UUNhQ}b;yZ*-j*}Hi z_8V<1Ewa74y6yx~FfKikUZ|z|n-GuJKI_igiN`z84QTsvPa7PVD`(eeUOR*nz7VyM z4Q%YlqKcrCUp!Wfa&piwbfvXVb#zE=1wL^{^}7h?RacXPQNWp#Y$1bzRWda`L7|r{ z2p>Pn zcpe^K#F=8oWCEst*G{fI*}XDhudex|INau&(YTYAvOFMlme8fb#Bc|)J8KZ#)Z}Dq z=j-w;Z#02OTGW~@`1*vEgj)14EPNr5t{V1cm|{zusNrVa>hI$UBaxS9v!mNe46m$c z=MGcF3dBn~-X(TD`dOr`>qJUZ1uAeAFPHLVPRq~CB()`EQ08FPwgMcQVB_B z>c`QiixzS7!d9N`bJz~EU^l~ga8U#ETc5LyQVuo`cl5a@uf~1PW~|9lIOW>9O{{Dl?}rJt=+%ZHlev({Qqv8 z|9mB;up_!)XMaHoH&uviTd^JJ1=QXB0g$=Tx>IH`4rh6=S{DjAfN8%d+^Bf>uKna- z%Ja6hb?05#Cw&#C#!P2QFADt9#t#6l z_6*MVyM0mp{FQhs5!gX1f+sv73~hJD+&8i7odJ&spdd6WM7v}+aUJ>a$(>K8SViAh z-+j3#KKCHcYr~A}z%;-4y&rAiiZEc_e33=#~ZhYt6pcp(dLSTc#ux~vE1+F zZIk*xx?IzTW=E!Bg`(!Xjzl!<(k8aOz1XWIFq!=yx+igJxJPvv+V1V)j0qZn%5`X?L6Jq#5Uhb9?77 z<;a}tbu(wovYS}x=3CpX&_yollZSGN&NmfI;9lIjWJtQ5__!ou1xU%b-2wxfXV(~v z1}Jxt>t9PsaQ<|p-C@s7^|zb)DvF;@zTu~4_LOX9Rt2}dwOUX0iKW7Hkv#_0aDHMBtnH$4q^Ed?06$(Fe*lrv8D=AgQK4@SQ z6T?gwv52qtzKPXEvp67MMFr6Il7D&?wRZxJx_}=3(=sUqO{8`8V& zbX11ZgGe&vQHV?W^IKkF;%&cOc}D-`@%l*~BHbnXF~(w6vP8ccDUzl|;?vXSFrFaJ z#b>ISwa%udH-X&$-MJIomqM=n|6p8PAVUL+dChIbJ9jWzU)_Gr*aIHAS-1F43DZo8 zj1279f>SMv$1A7y#Vf+MuqOSyis4O7)53B-_h_WVfB7{Q7Bbwq zB^2`JSP!KdrFgf_=vG>#(7a#cA6qHaKh6*cvPBFhzHlW_@4SzHzjA5e<2-M#t$^bF z8D2dzPt&Vo<51PRMIFubpLWHjR^+t-E^~a3%7!&rA=C^yY&f7lp7G*Pl9<7Y3Hn7y zo862F(Kk;4o9)Q&1gt_t&k1aN$I9_kvQ5ITZql!Dx1i2O(I23-bt~CatS^eM>8+3| zDu*W)enF;_))mV@q9U9YrtnZ+36oParn8Qep5|qaNAO$O=?d|r{tN>iYZtvYhbIe$ z+&q@0!6bojHts%XIEh&@|w5GfI23WncdCGjZXQd!AC~x)or)z2jX9XlJ>5 z9kQYuZ=i&o+_eeqGd2B(T~r3RLr-szi>zkd5?P4dd6Vh2MF!vIM~WNq06+rtLBxgx zZ`t>LY6JSS2dyWlaR%r{P7oZ&*Ef`mMbYOAA9a+RI&I?j*;nOYgOhrY+(%5yakV=? za&KQyay0tu57qN}(T5L}b5+p?=M!8ox;?_r))w^FofeQt@K0YSE+mGB7GIMR+z&q6 zkM6T=$w@Kg=~?Ydeo%~lNyxXF8j6XDiHPQ*7NrNd3+Q$~L(E%wM8tJQC@MP8pWhK( zgTldxc%GJ(|6$%V{;ey;+&E>g=nX(!&8i|fY{l3+O+S>i0d_j;v;Fx9&v)YF;_jYM|KZOFq&qeFM~ni9ADFOg+}#>->T z$0xJ2UGTEovd)pA`yR3d#^VCPEt0RwP*=%dbI@OcB6#FYMZ9in>=j0x%&mXud)XzE z{is*NW39ONIKFOZ_Pg91?1d}&P$A)n-@GuW1m#R9ezQ5v%c@{~V9jgW^`y~lq+yc0 zWYyU7>@Dqk-J>3!iT5pA{%>gGa+>zqQ@i{M?ab4DfQX(;*am!HIC3S_u<q{D+Gd;CA(K%X1M$M{XF1ogj&y~cr1a#2%< zue^SPD{Gl^{TN}8EvBmaIcIE3DUqAgIYPrh=A)>D%53?UDL>>#6J=&j&Y6`U&Fsn7 z4L90(yDc<}jRqkaNwuV83gv;vH4)z5RhZ_ZT2Q`8J$C*5>7Ki3Qa5`dEuY?KksK&l z!`h;=hypj?p3~p!IfhkN^JUw`fpv}V->Z1Mo^f@!WZdRp0~%%jA+c>Y`}f(|^o504 z;bU7n8@VFK@UJ@h_Th_q9Bbf@<|p_W{V4wLW%}-!+5dw*l00?a)(b9(=(6qtry+UG z91x32LO)4}^!WwP!UWRPO4(feW3xNHUM%4L#=+IG8lVoI8lbX>o0We4oO=pZGO$v2 zv|@vqY$xKzb=3UiwWxGUV&a~;`p_P){U9xiWBha(@@Io@^fH!liRh7TeF;rA2}XVU zwdO5Qe7}ABBHOfiBOclVKjTEM3}+-qlU&)^*(3a3yeRyAOIx-_7>EwS=8S4f zwf>5o)h=dL&VTH;OOu@TUyAZxzmI!sQ`n8mPbw|0={o|iqf&_0GaCo|)q|dG#2*27 z+Y4Og?R-$Ir{bk|b`uV~3_!OC+gTA2x8x+Gq|E-=;|e5bX=yP5_RXoztSAL~_Hu`$ z!(0Wh<0yOZ@!kh&X(4KI3JRQ=u&}VWB>(Vt7A$`l@vxLQG8{_W?_9NLc+<JQu6TxI zVO0&A`k$-xBftvDmFx#F0Iv7wVuMXf=U1HX&hFyLoG*e}`|RL62s7yBF}t#A>O>j_4goWUfGgi6=RJ7gLVl~}G~vgah^kzWzhkdx_FAfzIV5hQWr^{8pIrK;gD@kBQv2@3 z5-i*_3X_e7(^_=B95dzx&?*1^XUB@JpU9n*@zT%DF$*)DgR#pz9B{$b+-z{YQiwWm zRrhtxog{!H*0fh(5YpHTr6t&3ogF2LR*Yq_KK8NZatriiyc$;>J=@VRv5^3EfMVia zMwg)Nhr^51KHDdKP?7|9_u~u#V zov$yY8!ajhVx1?kd(=OVKQ78IF7ES_;CyEKaXN-z*-MXp*!yh1MDgkCx{R!Su$y#$ zS4Xh0cAywffY-R&dw9>QF=J)rZb;)ll~s*1o>HQ-n;TUK7U{83I~|nfxy-fRt8V4U zf;IZ#K8dm(ZU1IEYM`9uNC_2lm*xyP%Y>I!*yQ~_IN0!{v5=vHe>!-OveL8HbC{6e z@hGLq=iz@bO3fK(M`97?3JH)A{B`dXJU&+k;XRT($8+A`#lcy<1zNooE=HeA zu7|_d+a3SX%Z@W8Ffm1vuTt)A5NNe>rWB*I{ai)yw^$QG<~LvVc0(;L~p!q^!Poq!Hi8R?EE~mM$k8h$t`$~ z<*P>d-S*wfVu;1n5}RQO&%%oMNxEXxHn4Gt9?r9CKepp#v~QeWaB7EYN&@RhHbsv5 zYGC_R^y%;6+l$%qMR9AkaLVWjsj=@}tSs$|lPE;XA5$3%Li2DbgTucd0!Ax;g+5aL z&mX58L@*&XsrLdl_Hcc%egHl*#>=v1@z+;Z#@ahA`wjTBdyk!ma%62c^~2v~xyd;2 z(>5|z(GSBhfErb_c!~0{)|wY zY+0xO6@4dm51}&a8as4(zjwHm=P> zj_){Edo!G~ZNk0$>kA|k6n5e*J$qYeh1rMp&Lek*EY}2QEv5X6weCFfoGqeaexdfE z2kF%{lSu#ZdNCu)+!jmE)L4{-G}3iXVP`1Lcz>ipUT$_8_WB7vNwwxa7gdd$gH7Ih z1=9pfv~dFnd&ym@n6h=cyDYmFH##))$V0QOcQ2NKCF{GNAYq8}@W(isQl;~^zj;>Z z1&sldyleb?ByWcCa#gj=JpcG$h?MK5v-a7Yn|&si;rc@&{oqh~kAd0I?mM`sy)T-j z&u^CpH{$T?-EVc(QKKx z!+m>???J$q5kCo*cBZlX9)FsZv^Ca;dHL}91cb#fh3^eiOdNX?e~F7l2#Ew8Hze)YNB(U8r~i=Bg%N?t-c_KOCX1BcU49v z?HBj0rJn_K9A%i6iM+nlkSh~en)lx;s{*)O0`1yW-BjiQ*0|FHo`y%$lc?>YX&_AZqiVaw&VGu= zFC+}h`}%dX$hY4%yyiOf{h#msp0x|&v~6u1CdjtC;XrkG!qXFeHqU;w!gN;=anx_2 z5hFiJlIsZ+RAHx81i7hFnggqH7Rmi~=M!ZQXJ?o~eaOK*zH{Wr|VY1Df-SWd5dv4We@p7Bxk}q9xDZ?&R z*CvJ`#LBBanY;U{h9PhF{3h5lH`ju6*E8-%Dc+fyp_5_vWN=g^ozP?T%Tx_!|^GJ^;xXmKgPTR6K$6$7RUvRsk07N*YOb6AUKb5e9`J^>D3lM^33_R;R9Bqd z%LhMg{e0Lr6{m{*Y@|{N9pt-tI>t%8)-8o9k{jfpewzAzwt`Z=g;jm9MexmaNZUn^ z;IIpoS-PQ94JkmyW+tUQa%juo>!_wxptS>Oc&B?GKGD1q^M0QJ%@pWr=B=)#mTEpL zGc|h26!yqVV!i2U^77kc8;cvz&JbsZ?~BhBl$T?i&7VaU_}*q{;{brhMiO}ZMdl6} zKBsp@>tt^9A5&^BeaKu}n~tQ^(P@a$e83iKec^@VKTCJlSV|P}$}H_)Tx@wZWwQ~l z0%>)vO^oSgF5j-`5e~%P2{>O$bo9bTfq|j6 zfW(~tM@@Sf11QOA2@99koAL%mM(pVSh>EHsz+_WrnjuC*N#41!Hxj@!Rl)T?IJA&(sBHAtOYN-I2y-W^`FW zPDQi)D$w!e6oXwbtS&LWI|^r+XLCOQf2yv6x(xo&*_qcOD`rHirr_j08(w%`R`$WX zGrV@r-vX9%-Fkaxe&1O+kLGN##r$pZ2;FhcU2%RR5$FLMN6M|*If0*R=f^a7M9@$w z@Juu9ED~q;t!f0P<7A_P@X-2BCm_;Dl@{+~+c~8G(56}oGua<+n&-;!X`7ER&GlyG z@HJeTN?w|6udAg*rSrdA+~sK!=Bymy58Oi6SnfM0HNu%R+v&Xw!5*D4eSo??b}=H; zZ>bzELZ$^)g)(vVm36iGI?O?NC3NC%7imsAt6|IkJWu0UVM&%pJKQFDB(Ucz^0;vS4@Jowz&ZUnAD`RwGtrHRXqAaTWcfS z?5*{-O=yc_1G$zKuY*a$hi2A++xz1Ch@9<>NIvHDW3tq`uuPS@AG-zFUL z9AsY4?@lYY)Z*dMX`9V2+gBeH1DqaA#+-u&9>5i*YKqz?o7X<76-@jDN6>H{y2a>0 zhAqDye8fe5HtjG|jj5LDH2gYvuf{aw2EKvZ1OjPUm$?A$?u6IXC)evDvtUr4LiEx9o6i$oi+f2(BMzi^EiKbD&y@WxON-&8zEkV8V0As*n*G z9mN3oEmpY)3JD;v;obssiRHg_D99&OTc8=I&-FY&4v1ZEI|G_@g7aTFxPQ(5z5K=U0fmsOmk5a#Q#;16S#N{ zEBr6C@Xyyx`NO|sVi8Y^LA^|5V19AuR6=cCB%Rx^wv<@+dq|vmx+*cS*&KQdQ*nf& zE~ew+chVF~$7uLMFMG~^M|SX~bGxN-ayqv!-ZT?!Z5`lldkDSI;ac@VMzw>vtPwF- zkv*TAfV<))kYmM^n3>JL$;co&DhprxQZX79ZfB>fj&Mg<~a$H7#9=v%oXW@ddt+ZU}wgzc4{&nl0-J3%6nk%fk9&s6tkZ`%w+!9&5 zk^)+wbjwE3vGH-Jri@gKn74G>;?2I-#qfcqGA!HiYduj{`3IDp*~e6U?o8rkC++`qI9HIx&ZDxa=83>jKH_ zto@^8-qfKb3C95HA!da5^pzh#$afK$Z9tLdrhG;K92H7L>g6@FwDC31z7VTb6Z-Ft8!`ul>i{pfb;DDwR=}jY#Is_cZ=}4yLX#w?v?{;7TA5R zg9XZW=a9Xh4Nk36<1OQo8-E@xTDWvxTr8raRKi*VwPvtmM`9Y5l|!ztc;ue+27))C zTO8^*?!-J`}>W>+SH6P90$lG zDbp7u;_PE>?0`hAglTo9q;WC_J`uGE@bH~8BK?(rB5LqpDa$2|h@9;YJHK&KO#Sv* zB-ZL~P)vL_-95ScMIs=F|+B@I!(w&$gz_8ac`XjyS0CPs?jS_zAS%cQ`%u zzTp{fSOrae{m@wnkTtzYcQ+^CUC=oMgQ4UKch6s>18l4Y5f*QO@~XNgT3KJu0OE|W zli6;lfXOF!xc-Bx*a8>@&3$fjgW^#8^oJS5pdQ!lwnaV$B?}CA$7{P}sENXmvkbjM zD3OKGGh&~H(u0!?qZ0k5zrW0j%}gA3Oe$@o#wvK5_~<}`06PU?nLQw`LAbicm!M5D z+%s;*n5G$J9DTB*=|e+CvXo?TGlTjp3QVe9@djVUpAvs=vuwXjRNT|U#8h>no}Km5 zuK$;rG;IH~2rt+^oQQCOw+l<^ko&%j9Z`RS$iS{dURt+|CwV&cFdLyHM7?>^zVgl( z;POCdQ&9U)=R`oT-y-;+VMOma&P^C89o^1^e!Gyt-K-|lr8?2PLpv4B?9ZP}fjd%cMav1|X${lb_ zo{$9w21fV+kw|+-$Acd+z4iZk7V)UQTW{$g=LQ)heuCkS;YZhJ`Bq+OoLQ*`iEI5W zX#61tusF&wb%<-(q4fate)48J&EY$u|8Tz%)!VPj!o@|OY93VBKm5o#YomF2@M`uL z7kHUU8z|LDj~ImKI>+k`mp!@10wgNK5frVxQW^`Cv*Ly4IiRP%R`x?J@7WXx;x&$7B(t z{9FE6t>dxbwY9Et1Lbkm3HqT_RJ!9sGixRj8blsl|GeC|gr`T3UM9Zwz$kJocyG~+ zCoF&koMa*TDw-$C%Fh5Lh)$g-f$vn^x;nhXFwCHr`RrtZ_6B|_Dj#lP^55c9d?0aS zCf|0d6LM8FSe_;hYs^-DO;N@ryM0cblr)k2C3)qZ!OSA3zEmT@$(q?QE+mJq7T0~D zp7!#)->BGq2%qOu+FTv5VnN5|X{%c?R`DR#8DvX{Ofhmoxa!#GaK9Zt6CJGM*BwEv z_6>?wPG2-&+zVG=mJSVy&v9QL%mW9naf#u@t{=Urf#c3I5DYMD4#(bx$fTZ)ngL#( zvWns+FyJ<>cVn}NNr}G8^?{X3$Slf;jfEW($WpxV5jska3BX9 z82nhRyCxEJ4dl7qr9|}&+i9V?@GIir_{!le7?iS7RF>iJF!bt3doK7O$V5WYG?Fbl zW7%i7J(?XhD}ypj0B1}7m?t;)9*7JgI7cp?;Yr^O{Bya3m!r=K@mc+A>N{a@4rWr~ zNq6|b+$|n_vjIVg!|)>EkVsQfYR+ z6f2*FU9}q40A}Amn(E9iWxsZ5z?Y8rhRpqq!*&!;tXHF}3px z_7LM(9rhjzltBNabg+I>Jg+(6ymuNBY7C3d$FlvF>o@#nFM$ys`{0tXAj93ses)7->X+bwGi+BHC3XR`Lv>+7nQbNU1U$+3T#GQlB4g4 zh$~{s?{ezu>8`)h!BH4~6O6N8XTv>|wTGRo^(aGA5}HWR`Qy2PHTay4E-tYe4z1j^ z#oZv$oq)e6-vJ zC9YM8*%BfoBJ&@>D&9=PK6p{N1GZf~dMV>rO&M+oB;;h`X$W$Uec)4a9NLx2#Dtls z{2FMzKY;uqbaV7qSInmYy7BBspC4{CeRhZ!js4ISwMtS}G&nSvZtBR&nkt#VK4#e- zMXjiOsqiovm^NMtI?%pNH2`unj*$g7EC`qJ2aczE(4?i6B;Jnd7ffHier0W1?&sCC z-kXuBM&VNRDgdqz7ITGT2W#T-W$(*)n0c##q&qPI* z-YiYQe?as(M;7ecXU|HNUg{?TPk&o;_{oiW&)b0~Ws~z{KtvX^BJuNWV*_h7&>)bM znch4GVpc^jeWCGJVFdtsT_R8_khaM_1Ti?jKQ9hP{aiegW4Jy2wMZFVqmNUFEHjCO zBo2r@ee&~7jB1{u*$XnlYppntZ3dxMFUK{wu{UlZk|IpRk=>3mq(46;Z@wDSoVvd7 zv>j5HchrLz(-DZ)SJAWir{FCl^E9~A=f*l^mb(9XFXR$-K>T?aF0&6XPB#DNmY|VV z#K(#0b5wTKW(792$~zZo&Ef<-;T&X`$hu+?abg6@lTtds7LY!ah9#|NAH&&?TVTeI zBCumzDd_Q4f7UXYi_4wk_`tN2cAf>VLt;D#qkZ5O_<|9IJXfY)Tp-DX-K6W5rF9edah*1Jp6E*}SZW$uC!$6Q^7~l#ZG$T6hFDHH-eO1bju|}(%4`O%Q7zOHg zT0h{V5~-Z9Gr+Fcm>^moOcGO5!=qP=9ls_d$vLXs15yEM;{-?P*_aqXf^*+6f*(J` zZ$W;IeIp`HOu%mqKHzSrFbKg(?z>er{igB2JQhGyUGE=syn9QGpQ(`qO7!xe6Pj7f zFv$7Ea8F~XyFSMnsH*8D&wnRD#D`u#)I{-?HuBsCyq8u{3IH)%0UOo*3UvBdEjQOb zSD=Man#clSgJ9WV?AJRm6i-516VcjG^bPdm1rxY>&ugZfps#USB@Vbc#|xRJCzrOH z$TzVq11{F;L3SY%cz8i`YjycI=^c2ddT%m*AIC=#kmvW_u`R zW&!vr>B~(WQ_n6cKI(yT<5!C}VQi?AvPg8C8#OTZSy|BvJ+84qlv27(<;dCD_1Msg zl+uEP1>EREevsiuegy)oxIGT?vw!k?tu`9MxCmBydSpxVe|NqP0@T7C*d8X}T2TQv z*GlGh76VpJ5WMY$x&%`VHF>1~1&ag2v-ki~N2B}QC!^)!q7Q({(D(0^SJx&iH->Pl zt9|JG_v_h;47OQlH}SEoa!Qn0dDPO-1)3wUf0^WuwK>(CA`j@|E>$<;QQe(?CC@*h zcWSJdj~^*Ne85&X6%-Koj|sRT?~7c(yZ0|&LJIUafaz zY#leZImks~dARyI<7_>hYNZp0fsnBn141ut8vBg*njR?Nz-BwLh41L(RGSzMo+5IJ z$IDao^>h2M2~e|*I3^gviYX47Ib+-IMVj+=P+1gIIH%saOXRjadOT;)^eH@eSy2jz zjH?Edn~%ER!SxYinj62-O>GNuUK8wYx3~bz@!^9O`fs%}se?X6sS_z@&w(kd{pzC6 ze7*DZlplX7G<+Oz-(T!aXPEA@|q0>fdD@!zsuAt2qjWBOv8feN_u%41k@~K z1#Nw+pK68=oQhuoeHbf&f`QcrCDNY{N<)#;iq|U!Pf{lOgM!(?O`XK~ZA6A2+(yutY z+(=v-URAmFf+y#NdFyMWk76kbazt3s*PU931OkOd+D<_@puuXaukzPJmw7l7|L#{!OdH;AD?`S^1U*WvA2R+N;ww&A!rS>Nl)*ay;5i|O zy-81c6J#&{GLLj)-GXd$e%3B8!C&jy#_T7`CfyL|FFcfeOc)ZZI)c!ejxZ>_F&tHrSUA>O%ADz z9em2@yFava9d8mx3Ih883vlL`X(H3x)9p{lVqZtuXEYk8iW41Pj(bVTx(&n`A;IiU zxOA>7DI8*Mm$Q-y)MO8wu3N6w?2C_Kjgk>%+9MC;~1a$S&CIS{phM65WVDFISibW;Xe$J)fy zzLyAdla~brko=wC)pt1Oh!(;bX>TwsYW8s2p~rLLdB?jn>V!#sGVt`G$!?d~bE5FH zSNT~p96uI{iyfO#m$&8uH^dFO^s8&rkA6qWEJstF{{GV+%goJ953JFfK3M1)2h48P z_SlzNji$2q50+3;#aX?=;*4 zL>cmh2HgU?|0AHU`0>2w4vi`br5bYB_5d5-=en_vvy?gpEo1gR-^@z~0MzW;4EkFs zKeOKX9Kel0z3XCP@-P_?B~;dbR*AF&_d_=ZMTnLg4>VRl6xzEiMnV9k0pCKo*P5r0 z@QlokW)^+CLv|+5;Y2x-Mal)Ay__vPc9WU~JA)MLE6O;A8+F^BG@wm;gRjRf){~ z&Xd*VLfw@L?~O#m%GGD|G7J0|a8U38wBd=KD{k~oUwT_KzgoNSCoa)I z9X9H`3BSy@X45a;U)Fh!x;byAIo4r=O*dfcTPR<;ggn-OJzy@QcEJ2zpQh_m`B}xM zt=FewEhvx`kBVt|0|O@)<0#jG91*=#|rlH&PFezseux=3iOdy5nS z1nl8GpwO-Wzb2w*&rFbk!1HReF+}ItMPXsNSkD&+6DDiE`g0%LU++I2%wFUVyab5- z7jRyKE_=tz1yojmjJB?|A{^>;P`t-1c*{;@w~Z3yYsTSbczh5Q$Hm*Hi#;i0D!-#x zvyJcQi=Be;@>D-B`CBjoZI6;??`d`y&r+!xjkr2d>b0;~^=Sf8xF+Tqv8!BXJyCet zpwMy)a3e$=Zzhe1)U5n&kO$r9rcbeWFHvoCB}AM3iyLBkmCqB@!(*%=&r1zHN`AU+ z7AQEhTPA-|(ZFNH3$>pYpsjLeSoyO!@hYrt>7qX1z(J-pfQLG_w2mZwL2A?Is(1Qz z-BqgYVeNzL_)-y9r(630q0H!_uzK7-3i&vD!sfuhtV1fKPqUM!K2g&a5g1 zq<%3wX2QDbaZ5JNMg2$2PYWG=TcSv-1JCF+gNPFIZr{WqVX+ z{at5Cu@9d_Gnvak#_EqrifU{mL}|gHf{+oRTk9FZy?jjA7CFQPP=o3tx$}JxAIPk# z6gUA46Q_nI;Q;?rXEHk-b%)-1#@jB6*H-z_j zQ81;4GN9*mjZ5@LglQ8Uot+hAUmlk$=i;f`*?r3yjDL$pw#`Iwaigf_f%@o)ddfPB z9sso$#!p|Qx7nqNq9g-_& zX4aGO=g%MgY{}u!uEvYn(3KmMI|r1pq{AVbK}D|3=xT{b!VC%r0t`U49>bfH&^G!$ zwpv_sg#X-Nohx}Qc?MhX%_e{bU^Si zye5#Rq5_&zJ8Yt}gCc>u`0kZ%%874Ufh;#?wC<+4+`C&hB&N>vUH51G)lzAKVm~Qm zik!~&Z1vXa9pyL14i?*q36wdkW2=(}l)q(?T*}w(Oi+kej|G<|_4G}T)@_w}e@3dNAVTzhDerM~ovgPd{mv#-~}hqbM`D=J-JiCbdp;pQFe>CtrSKH9T` zp(%RN{ zlBkrVjO>yqM7CrkA|oOpI~3U?**n=ILUxgmEqlw(-ka=fviJXdFWq(D&+mV|UOiop z?&A7F zUT%x;`!*Q)+{J@_B?@Q8txG4EaI!b5so(;G6O zMRDwFOWMz_k(c|sBhW5Gm-h@mCof{ip}R}3Yil<7>U$oJ;RT!h{e5V42-VpBm2Nol zG&?ukWG(Tgbjs>2Ius4<8;!YcM3#;L+5>7gah^SEf-Zl1rX}O%NMWFA&L_&@glh53 z`A)~?t(NR~!4S$ctav-GyrTv`9zabpnVShSxLGx{>7X@gS@>W0chHSN-Hk9NE$NCo2O9ac4=8 z0d;P>E%`yK@%zWI3eH-fc<#zFmDyb!c67w&-(NHHpIdtH@uME3LtTnRuSlkvAFOdZ z-{$8($sq6ZEu2GQGj(*f4(oMtF+7yDz(&AuF_u|zZ=YRnF!C}<*jy| zGtPtcRhBSu-B=)IwdE)g5IJdAX7SP{egD{Zpk1DiOxXtnGbirto79q>W?4#OJf6?3 zCQ5@nJ{3Ckbfq%d|4Gug)^hoYP|cshJ-f170UC_f^e6I)*{Q?@R)X4EBuI!|Ys-oe zfmZJiCY$Mpbra_p^Nq>JI;yca3gYChn{}8{-{Z*{ME> zvni?{$5(fscJ8F^FYQ?llCA00?zS4V zKyept9jDLBRfku@0EbP+s-moBldgOCoi(}5N&U^Gz|YpJw|v`016`Q5v7h=h#0en2pxTP(>a(fJ0LFvE&%z{`&Q$1T8Ss zbLGqj&&)&vu zz~_=BboKohZ3)r%p7m09aKZ~Qx72i|@<6(TyQ8Be7g6kj^n)9-_wpB8z6P*!ejncg z%2VZVx=(Rs6YJp#n`W*V37F8cT8?7~$g#Rc;LuF9+v-WNyO>xoc1U}3q42GhcGr1r zX3E)Lx3GCOb|>a_RrVb_iqq^ruw-8})wTbaN&$T(hD2`_FQi>`zDYcd>$jN$%S(8( z%ucMa*H)*xMzBsw2VsAkTW11HrjxKDNdrD$!>xhoW zhfeKG*xgU=`r%F$as?^l9PX{!F86AYVU4R?bDD@oYjZIoL$g{p z9UXTMH}VP~0hVCV6n+oR8n5R^i(|Cs`dsLhOCKyp{0VSw4ci*CAQvQW*9M{saEqt0 zu=2EEDl&AO{D#syuS$30N?_MCz!3=F+ZKvHAKJ$W3a`IbjjN3vZxhPMq_wysHKv6; zV$I4o`p|{DZ)pcLqaEszWM+dtS+A_D=cqgPJ#Zk;XEH?;Y5%-1BO3K`l;W9O{u^~a zkJq+P^X;oD!?4L5i@lA@&q)vy%`n%&`tn8Tp}Mh?_*GHh>#n8ACfJb>+5;qya-5WE+Q znKm~p&Eb86D;5vkE(-$<50(0T;4z{oE?~m&HS~)(cgm$^V|kfO`J64XREYiZh0xa* z`}py#%MiKaMDA&HCe|?#i!3I6`-c7LbLlgE<3TIjDel)_$a&;ki;GF2EF=V)Kd{mT zc{*CYlh4y9Yd!{F#;&K)RO9N_|%hZ|ZxwSsDKhFKYolXkP{x@eIoNU!Q7k%#a zJ5!d2UXdn}6r^92A4V#;zL_P0Aw3xU)99^IJTB@b;812hq7*s!7z6b_N0XiCu-N!+ zTX&f$ifp(MvlTyawP9(#Rwt;<2S#B&G8l{KyJCPaK-FirE2eVVd5gdT;zMEx)Ss&) zlaR?Lf!IfP-v;!RtlxrlbutdwXCEh6=qB}4b{aYJwBq&W->FJ)6C$Hi z?DAdR^TSC;NeVJgTVfg*(ikDOGtg&4O5_AYoUx!YE+|MbO~9seR?r}2uJyV!G$?Qy z(u3^Hx~>r?H76(Pim1pXI=W%^t|B0rt@Ed&sZc0V-}4V1;91-Iy*t|bAoHlh3sn7% zWA9vMZKrziViYL+W6sXauB3Lz!-j_w$)76p-~m~GzvRg?YeI7G1P}$$pKWi4aBz?F z_~GwvDvU0V@^p4)_cvdM+hDRW|CE4mvcKjm6+b!GuvJ{G1rb4H^>7Krv>2qdP{hB) z$9LvYLXBmSl)5f_o?1UMAh(g~1oS#qR5&1uE|8!6#*N15X%k`BCP&bng89FiZnb=q z%ZNTFG9`Etc$(1G-dh(ruE}iTlTthx zqKr)MJbf(est$5OYWC}op>Jke)6~>7f3^AcAV}2`42<)O zkdv3}qwlMG`8!fTgAB?E_PJs^?CUR0X|7Kj_>I^4LkJTfT61%EA73n5dZUy|x!m#< zQr5n{w)^5ZE8DQW+JWDw$k0~^7p5GP8RZcMA&AZ*G0o^6KnN0&(f7B^Pfq-bZt|tL z^-C404Wlv)o{;kI&U;a&rk6D|rHZJjaePTu#z4V@z(qFN^H7&DiOvjFc{T*L?5Q-Y z%Py2`PneoY;rOU{y$kDp48L$^7LddQR9^(`t=alX2R=Uhj!!V^pRsj`#@CnBq>sC_ zPS>;sgdo=LQ|L9be@MzDgq{Y$QBoQopF7Z@LG^e^8Pv}RC=>5O!=R51+p$XalSmzN zOEGcJo{O%g(?|sJ&`t&RxXBbs0JI?3^Oltf`!b)Mzg}8T+?vUamxN8NTzQyCoYNh5 zxv4EjDM9v+rN3$mlM@}K{ZmRqxCZ>~GwIb9V?oZ@sgRr1ljg-T{s=n!F zUjkLjk3vE$FkYppSs26C`Dogl5RWoN(7psiDV5aJ)cHN&1t^8Gyao1qJOu5Jxv45icFymuvBfb>({AM6^ey&h4tUbxI$1U z0vwWP`&Z#DJtMmeka`p21sD2IiaqEO zC++U+d(R?sk?r%`oMik>Zju{Fx2?7QLi!|Bd#Ow?7>CjDE6d({7YZbe zXtT#%OD9ErIQy$Ig)q+kv6BC_eS-TDv#R%_Z{vGj_PD=hzxDnK4AH%Ayic=b5?GWD zJi(cIpQ>}7`V75ldZ6CS`fYAc96T{1!o0Z?(N(Bx;*9qDZyzY5_x}D1t z9s2y)Jqv~pTCms3hjL+PJB8Wpbs~%T)VapUncY=>_y&+Y!w9iKsN-$BUZ>bt73HG* z{EI~1)mkR@Ha6Vt9YxsZ#hOM&@K&ekBvi56oIdx=VHx^9*Es$IDvv)t!&*SZ1Xc zSRuXIBezU`IDMcXCG|SmHa3gJ*K{7^Q_F=JhJu1Fn4-wcDGFj}SO0noMkJRfL)CG1 z2yn1MZYX2tDB)`S5NKJP8W9&{3VpgUFWeX%^|NE$XhiLeR&8^CKL|^7C(JGR!#X%9 zW#6zMadk;fA=mW2NJ1hV? zqSAIeDS)LTua-yW9X-|?Y0sr#3#jb{u%Or~cut>utktP9GR=1FG0vF?a`&emtJ{V3 zOpN>|VMkGHZ8AZYTFPfjzO5hJ&}+yKo%<^bc?mKqgC3tg4L6zdFkidx*1$wI^>rO- zWs@nfYlLF|dwxEQj)Yw%p$3H=p|mlLCih6zt7jj5UW*ww($qYeR+4M1mY@kTixU%7 zSLhNfoAH`gp;wsdS4m-9VIfwsd=v@=HGU+2^h7@eHWVz1YS7AI%X+ToK zuWUQjc>Fl_7n+p1+tt;+(Aszcl@Imax;l^LZTsT&Zb5J_{ry0qZ0+%Tpzd5zpl1RZ1zOjjrrKSp)f`y3?a1=v8|WG6UYZ_CjdPV>))kGsau zyAPYV(nX#=jZ0mMw#~Bn${sV2pLwPDZqmE)ys{`-n}Vp@4foOyJ?S_H`e$ZF-?P!x z7g2i(%Q6<&DUU)l6;T_4U8#I1gxe9(kTTIeF(Hw8egFj1Uo1!Np~VwnSisdHs@k zW&9tv80F<~RM7jBaCar-Bo@dIf;h-K~)6O6FV<}M` z{&1dv;59fFLG@{Gv2&oPu5t5ZKdJAL#=|_1`nx<341;0lozO2|=ml^8QGZBee!@sZ zF5sZj8X6tV1QX3r7Epy&5Li;^!jBA0vj`8FY%M5$U9oP)v+$&l6t)y z&Gp$X7ye*zH>}e{_vMg4a?5;g%QQPD=hRNt*RS!{-`xLn`ntrY1SMr(ZmSHD&R%Gk z@eO8JC>zZmee)LRzjgT+_OrEAjv#KH^nsa;?LrXkgTO@U*)-_l+Zs8OQf$~Dt zSxzgehuoWJbFt0J{1I)C)bIgg*4=SY6hvkaW`Tsw{nB&Aiv-ht?!ekYKS99m&~-${ z3{jw&B;w-+AU1Mf@a1;xQ?n&su3yXRJ#^?PfG4cS`ub!ZCNH!M%yjVBZ9KtHvwF%5 zTHH_5KyUbzr{`tacf232($U4`=5}{8F)~(Vzvw{V{3lTU0_8d^ds5$;YJ`3&y+BGz zuM2Sv8Ne4!k!L`*W7@+4?(>g@kkhqK%FoYls9uES+tB;sA9svn-@f=OPq-9?Hwp{*2(zCvtIhz4`@t zP?7bB)>07sO(G&%xS!Oj_n%Pnb@lf8&Cfe(!I>k}F~lm5%R=+h6CfK4F#AoYTE91k zK91r3=qM?)7rJVxO?h6R%h0MlE2yrhc#)}W&kUNkJ-@^*ay#rXrWGwJzn7%g=6d_~ z0Ltu)p2@MNn`(S}`i-bkI*LfdU^S@Whv{+M9MZk*7BDzS^W|j5Dl`^P zI;C$?1tvOfuf|wHTOoL%ZQ<9-&>`NxX(L7xhm&xit*^n&@BxoKHRFuKlkO0%5~A?W>j3z<^(KCt9#>~ir=Wbp;8>lZG#i6Rd$n|PEbP1mnDM!# zTeMyFCC&*J5E}nt;m{n5^H8(dP>W@HW7t}n@(c#!{)1_ACRnEV<%@iR7HG3AmI;ux z2nd{ZaPTUhqvy|^r@_Ad|YR!r-kCR)w zLBf^^vdXGjyuDh>k035&+V8m96b3|Iro9U8+jHh5;5Rug`c=X-29OT~;Kdyv{S)4I zm#op^K*HqjmyEH@|9!9$P;U}Al(*X)dVzFf!zu<*YiqXf2ww>LC-HhNUB7-DX|u|D z_ToMJb*b6F-NEK4E{*kBw%+Wd;{^pANJ~-QR~zV)!UvUG&;qG}QA$|YaujhjmzmZ} zFU4Q_q1F8bomD9m~`beG;c(p~tp?sv$0ROse-M%W?JTv57RC$l*uM z?N@RVne|1yn^7Q81cceKyImd>L~lMq%DmYA^kk==O=?KRx$^cASpKcTsLFFpVDI7U zp&|y;o$RxqTL*lSAP|YvcU{-kOkjxxlyXTutE$c=Nl>~z_9P@EFq{fs;%i5IhJKC`gsNY%s;U%d1?dRZXE-1QBTNVnLB<&N+?+>Z%GEEg zb(axsYBa)~nVGrBZ!8Av+W;Ses<$EzSo$T5D`Lc7^A5_|5%uh z@O>^nFd=Uc zC(1>Mi6MQ}vznCv?ePoQLPA2;Ot3ne{SF5^`$^dO`4P64PDCN#0N^Y3(SR|0#*!Lc z_SlpZ$=1-oKxzp3u8~j#hgKAgOhp5G9Jgb{>UNn&ydWDab3hlnpc*A6hx>N({Z(o; zC}b=a>NCQxl+O6IF!21K!DomdTyKW7{9~*n(=xm5sd;`i@M|$=-IDI>>%?@Ardo*3 zBU=1^wo7GY7njOf>~fvw@7B3Tve9`?@BTS?pB^fxH$Fo_1uu1r>d%`oJsEbz8~@PSRBG0z*R@zWmQ;)k)560 zi)eimYI^KAqoF$|r#xEJ$Rm-N5JZ9d;hOI;LE@(CG`mNkbCjc5e@soqMc>s^7ijOq z%Zfchezf)Q_I6pwtB|I_%PcH1-@H7q(;>c&uFZ6~l)mWre3_S51^h59)(CDM@rCbB z0|mrKXYk+Rad2`%pH=C(wfadjM&;Zypf!B{@jYQz{8dx%ag}!D{{}XI4xOc$EHM6a zKwz2X&JKI0Q21{NT&(uu1)-R5uhj^el`JfB*sIUI)josm`XRMVm*&fBz~ITv#M2-# z9{`tSCMPGy!Wy@#s_JNp9e+I-$Pqa?xsyA8z@?3NLdU88qiSezrsa|v>|R$^2KhL| zywD47f5!xGZ&#(%5Kxz|XwYTX94fSMDGdlX*Z7`A=;(tJ{wqCYUX&8Xj0SH%9pmWN z-7PP8=z?`RARyq9%wu8p_bgQI9`{Kbkt_1=7wOhkkK@0;0H-17gn@!(h(A_BvAtVc zTbYP&GcYo~`$R+?=!cjN2dS@cFgzd&LIhHCv9sTY?eS~qZS!t-ogt|^T4oYk3d-c* z$Jj}c$AZR3>j0(%wY1_sq_2+5d+xwS2cMxL%hQNps2hANv}RT8aOXDHr#(uqM9cZ! zOiUzrTDILI7X0T{6v#b%*!ryb-m6!+@EGv7TS0aIp`0AePYG#hQ7I`uaDHtXwRYIl z)GwB|B>wwM*_T(+5C8imVb?o1F;N~KG!-Z!Qb`_l-dOWQ8hl)bsLHHApR;RwtT>9f zS@`AUq%U92XUE0Het3B8sBR4{|7i9hBm}!4(lHqhlGGzSBI0SL;&FSla}c6w9<6k5 z8ysZ(IbJOW@Ccg@9vNJ0L3MF)FZcwwnmkcw!MHE1K^mjgnJ5G96<&)IQ&TwoAUvk( zFd#0>-U>|uq{?|lglleZ-i&=`NPjI!8-O*;SHUr3k+cWV59j2YK%Z%*Lf)(zm~ z(z&#$Y8_9HPv_D`;o16wYJwo*80C{6!?KzCI5s6}t=+%(F?T+m_U2}XXA6X$z(Yi> zp%%H&r;vYg9Tsf*BZA%PHa3OGL%^5uNAF`!lB2$P^C3(l2*WVt)=WaoaeHtY9{&EJ ziO3b3g$BQ8HzS%{NFn+n2cIE!H@)X{KLYH4mGLOV!Nn!aj);iBB7Q+#7$Ao% ze?#YrXjK;bNzhBlP-M)_1x;OOa5z$#w7HAeG(7ZT6yLi3y?gf(Q&I@lU|qzK8(7ap zOPq6Z5&_KtM}(ckrqleR?(G$qQpoInEg|=Ezq7*i#M2xlLue&%dwWGxToI181(shU zpIrCF3;eZ~7y&AwH)r0$qc41jeCYqLNl*N5umw<2i-P72!?=92}hE3m=8t?}9ZR zS+v~r5+?*jz#;b|(NXZ})2H_(Bw!&1^1uz@7?5SDS>P~mrGTeNR!ApW+uq%Uf9;x$8Ti=f1tEL`~Y7YtYyJZ1JZD{*{{+vvMzRsAEs#tRmR7VjNk&_peu5IbCQW ze60VxW(&>*ljDSSK0b%9sz{LKH_qIqaXF~IP(?WQ=mU2k>QsnV1p+2Ety7PKoz(Y( zbV7&Nqcd+`&(6+ja7JMLn;8JM`)J#rkI$JS*QWk+Z|Jjpa`30K)GY899)h3y*-&5a ztkOL40De4N5%{E6uU>U2JtDD5KW!`yd54FS3XT+8uY$-LGnzn{kb+GndiwfUzjaB3 zCopxIBP*tw;M+r;8;U@uv{|v`&&e!l0kV>Tzz+E8->U%@OQ`WHK_K;0M zcEi2}hyVm0d|gU=dlh}C1k{e=R?y&}ir{TVMxW_AVRj7;F0S#_76IZl5Xgpb?NcU7 z8_SPG>DbcR{}Zb#1h4;ZtezkPtZPWi5dMIOo$U63KF1S{o62oQnXyCFhfPG~I1DMM z6!~(TJJ@$yLToa4ix$s!BYA5lCb)^-V(YITn7O$xN`x>vOV$|RQP0{UkaJeKv37=k3WF6d z;6E2FQQ-eD32kokdlTc|otb`Qb%OXSVPaQ=Ofqo#lPKzaLJ!=B)ZOZy zy{N0Px5S>jyM~2k4 zTvC!)D;LBkbVP_PeJCU2ys`00@HZEX%LHl~*KXV(Tx$yF@QyiwaEF;SH6j4xlv+ch zqHaJ=D9qQ|-tO`^Ssunm`**ZiNkW)Zy~@k)oMN~Fm(H9NF#Jb0YJ_GjEuHLl<#DP?1W9UXIHv zEv+lyWhTrDV47k*6Up(EljOz@`*wSlxEsvXTT3Kr18)?47p>P&9@7qs!==$+HJ3V> z@m!}ltbCTEuy%3Cv1c(;WOGrCuba}`O`Dc2>Xh^ANLR6A9$JiKuG(bEzcR%GQ`?jD z9P7fRqR;;b`XuT;xAW@o+Kj zTkWmXvT+x(S2G_wRkb`_%dy*NpIg+Us^J zKV@RaRC*P>^s@8l%i&~!fK553vMe2Kzib_48Qi+3yC%h4a(q39| zpvp;Kjec|Ewp>&RTo@@6ctnQM$dscG)@BM8xArR6;Wix;hE6EjdJxKdmxPPTP!SOk z3G5S`%;0udPgcLW`Ak%pcYm#;t@}9zBPvDCSL6!zi4#Hd2fIuD8yg$EZYZO6W5?YA zC_-2T4q5c5u2eHSuspirRF?7ka1=H$n9lw=oL?)#q(`SyP8}X*TqSyeZDn*zGV*<- zuS5ss9X0ykFdI20Np<@>))G1r^9x4)x4A5UQ_?&d$!*K4_+WdN<4wcOu~%L&j|@Z1@ZCKqr|8bZdFy)w&7vQuV23^d=-yB z4_2&xT~S0!ORIjcqa!Hk6bjYwR9E*}SW!{Y_{M;7OD8tmBv>yLraGd1)n>g*A;=nz zoaN@t0OWkgXV+~axlO6-cXoGI-#K$k>}{{E^y{Zz?Sm*Z7Hzv&4;L+;G66O)Lys3t zAK>}bcNqC%SFT)H@mAlXt}lddb9-c4iU4QMocKgWR#U^R!zjXtA}Oi(l>P2T!SfR+{X451O|nVv zl1|xWc@r};Z87d_jr)sANJxAJv469Run}cuX6?zl($Z2%Lk&Y+c2bx>VHPPl=)sg$ zR#pa$!07M2IB-O+uCDdPV0yh@b)=(NQCy?ngBe!#p>3Yjw~R^PIsp#IueH?F_>!@4 zVs0_)nxaB#Z3g_csD6Ccpt^K)H0AAz>XyIF4#nsq5{Iw#$kF=bwxR`3&F3j|h1hz# zD`S-_eOo4WoN7E4w`>ZP#-n9*232Q0qSLxL>@|99lux1pB#c8hmAiANAc^VH1LL?B z+F*@?A(1AZeAPTtu@PJgzcOG8=+0A*Vh9#5WzFz`$+7Xl0&}r$Y89Y3?)m*~?&5q9xtiP9& zvaqug)cR9NXVFDlQRf;Bg(u;oO!%Jb=!}wpx&y8x(N?oeW#50*xZuw(vL<9Y$r+?of4g9F|rs`edgz*xI4w9k7R zixJ`enMn*Q18od4v$~$Uc`s}Bm2Yb)n=aTJmVQmXp+x;H=Al!3$9eqQ-CM_qdJD{T zURqdGcXoDm8E9))Rp@3QEj`+c^YcfClq4rUHC1W%G?fSZP3PQ9@JMuYw4@;lrKl(B zPx)X9uq>4SZ?ooLpoktp201mQ9~}UdEx9vi&(ikA`SZ{2YM~TQATFpNJ)QZ{1w}IC z6cRuNDA2G!`+DhGJo7WX<0zH%E24f?uzNj_Z>?1*tRFR)KeE4Q2?GvPG<_5>MYnXN z+)GQ~Y;K(rb#esg3O~VR+Eo!;tv$ z@S3o?x;o5LOz3e!VFyIl>w0k%#hXrNdsl0I1`pkbcxZ*>F5Y_@VZOe;SIrAlHo|aG z8o0XybF29HmLv9-**Q<(A|vgD>13t0q#?kks7I_6ci7pkgM_lxhiaI5EI;~ zX!`&Pp)G0W(_B~tm$BLb_3|mi@N#<8B(ipw%0@;TP#I?<`D{5qfBqbpQdTA)O`?bi zHVn!qWnx(+wwpDWFbaTJQ^}XES(ha}JWAg831`Z23(Y3Q?O65 zjE|3>6?NqRPjMA-5dXd?;5TKi5&R->dB)1g8Pdqd#}^ikLS=M9DTvbC++1P$?#FYm ztXUf1Nx5bcxSGMCAqqpoHA-O<38#v$k5wQ+b!wYcwVR;69V`?(_Sj=U6 zzg+Z-TXNK6uL`8qJRD{)l&*J^5OgZ3EIx5WodYbnB% zFjOMI&q(NVp#99glKx*m6J#uPVSKF&%On;OwesYVxVl$D2bJuTfY z!G;bWb?iTvMvngKDl}wCN=bnTegrFrOMDM7;|2zd^)|M)Y~Wfnk^tOaHHT&}a8_U< zD8|p?@qvvH_$Bakf!-X0+)gYPYWE;@Fo1x*kPZO*tec2E9ZMb&rLrejD%n&EG22fyRZp1aLDm+zioK&$z6>}Tfhb5KilHOx<^K+)DM1z!oQQxzcG$tOzq00 zs^z~Ep6d4Z@+95(CrUYye_>l)>E2d4qn215Tp+T@lkzZusiiup;X>xe)N?2D2*=Ae zG~e+knfDZaZlXw)0g(7zlFyRRi9-hfk%1ak*=Y?Hpi~flll9&Vo+~Aci2I~E#moN7 zvJ4@zw6(XNU*Azrw||`6)tmj|QH^{?HVjyeLzw*=wvyD+nYnN3X=q$RBSZL4wV{{R z+Urix1v;s!nT~{@c~83NIs_9*0ro>Ya|+^_$rOhG2o;0gx>$Et7oqxYe-A61q)}CU z5Z(A#wYS)--CNLNc^hOtT>&yZK z3O?OYppQzqWm}}Mu`9rsi#VQ3ol`>BoO0s+9m~O+$|eznGN~4ZKc)}vEwQyk&PGI3 zG^ijh?h3Gvf#{Gv10*jC2S>0rCMG6gMc%1OskG+cP&<#yy5mtvO_>$Un_okvT$pZ- z@@7y-qj>^XE}Avt`~Hf*Bf=EX_VttaW82d) z{x&7i6C>NrmIxLA6C1pU0G%4WUI03mxU@JHF^?Y|g(9P2!9Ss#F$a*&l?z{2W_#s) zB%7zwD(SmDJmR{-xRs(jOBXk+ViOp4ICDflKCDwUbS@cMSP8*mM0p!=FlsngclPvz zHWD+bcmOf~b8)e`^Q0m@6!5}8Mu?3A!%3eZw83gL)} zsKG)c-&$N)@HH!lOgMwe_!*x5EJQq>_uda1b;CkTF8wN+?>H+P=d^jYX7IGOsag) zo`m|2EJ*RGB1glGJ{s{MH~aV-3Qk0uz7Ml_iXeKcxe4XW-O&$Z1sMQC-jdfw!6 zcXkl&ic>JmR!s0Qyc7h-lP61@Q7+q;qptufL8}@Y8xdebvj{*LF{zt;QA{R`MBdnj7v89_3H=w z`bh8u_DqmSW=2LrA~uB};|CfRwv)jMv2ZQ>*jjim+_-Tgj1&XKH3p*T=D+6U;EK9a z+FaAR;-n7(#XXvkpwuI&PO|#GUi4iQS3P1pneAxfkyV=cYdjKvjmO)tw9R9aNvc*~ z>yhbG8-7cQ?tC_dsWEl_a1X<%)Do5=l=`*_m*VLAwJ4UI8rzjqB`GJ~E6%M|UPzyj~?$o{htKk_E{< z|A|)bnxpy15Nv`(7W8#N;2h$B+^xE|rB###sfr zR6|fOsqe)3F@>a9Tk`EliuFW_!QxN$p!uRf^|zzZ$OKqrb=VG~X^p{&jfy-KzY1^i zQ60Y0WOWd*xmsGRuQgZ{e+4IuN^*E$h%)qZ!q-q%ay@h- zjz_>Ub@cIkk)vN}6Q#QPPXc-tY&;tDQWJYRqUg~MXLYIxu_{uqDxw~7w8OcTTJ<+* zef%4=l)K=TBY9L;ScB;+J>(`8KvX)g(55~cW9XYcoXfJ<+Znx3F@a?3%EXKs7>bOR zMvHZ-7+F8(<|L-fr<<#T85Cev5f`qxx8bW-Ap9Zn=X`#HW57H_$FpdmieyxwlLNm! zhu~c<3%xVqQ_jv9jx88Y9G_&9Xv`H9z+Cf*-RuxFi08N^6gUJ3`gB?aCQ82tiRH$S zEFAgq6e;7f{a2AZ7WDO{Fi%wr#T6vv8+8KD6V(_M6_x3DcmNNHADDc!+c{loxidyw zb`gQ((XuG-ac?r_eCG_ zHIXJvXbGiZ_=5jhmI{;3494Li;!b@0{ptFkW{0F*W*3h<5dVo^6}bOqz~p0+F*FFF z&5?zh`@J?E@A4hUM+VNONIGs+QlULuuI1A3R)0@vND_Ye7{czXObjBfn4wF&_fOVv zQOS(>w@KxbOeai@?Na&GI^I%RzsbtbUteFa#VnexK>W{~8QC>8 zHGu_rd0dY!pim*uGxd;_*t;6;TLS`DaGH8a@1hw1qG5ix>;#mm(NF1cOZMCcc^hA%z&e{wzAsCpF zoXikyztN}r!gTO>MqwW~7b~-QNWBBPsHgvCnLp||jt)ZOaZ*#)c#lpRf`QxWRI!c9 zGmp3&_dMe9h9XYkOhfAs-XS0*@aUe0P99=qgbY2JMQmqwZq6HG$MP&MD{EjTZbmOS z6;{iaFBK(EmwR=?K}B#HdabOiBtMCl^@K9qGXpFMEkMFywl+4b9DszcBS?4^m?@Pw zOec=d2!1J0X?}E^dJJ|WLZM@Z1{P{R1V?n|EWjZz(E6d7TUu&)gf0IO*p6CgTnL&6 zPRkRxA0+P{^nt>l_He1a?V!W%g7tTl9SjW;wFn&A!5R>n!)D9U3y`NZcj9Q6BFDlV za%4901sfQ@5#1pDh|%x2$#=ox@|Ziv9Cugattu<)Qe=HStj71$56H{xKN9)Yv>%F< z+L91sqf(w(8g4jz-|4pdSp{%J^1*}2>zIs%(CmgVSV(YQ*XJ+J+cr^GJPmHe$Ip-U z3FH!J{~8QGAQDWF_>4p#ZFGD_5X&WMm?w=y(t@MB2Nh6>b{zmyftv|Q!cAcUH?C&4 zQk~c{r|`AAyBqWs5?FH)=K1fGR6j2OhP2O6!(6L%604(O{UX-Wm)&CQIekh5gH&%n4_&VNZAyJs(Ew7)oe3pif z#GivDt*(nDta+af(;*>f>*y-{bcpAalLqVrr==D1Gy9^2+=1BC)Eo6uk=$>Se%}(X z1jwd;>~K&G57^~1r(gj$j?khhh`o@3y?lo%2NAip?dFiBjsYMwH@%#2okom(Bg6N)zC7iDE$vBgX4A=jIWTK&xRe6(P(WDQ}O~-dvrep4!2ua*x1Z1 zn1H`)6~cRFaD2HQByk0*0XD>FgJFB7JMPcHfidDj;4vCHvYedfw>JM?5#`HAC*(w^ zua`zCz;}z}aO9(SDNP&iMY-us~l2_Oz9;XXwyS;%#a=gNUZXGu(!7d5}lrni;Gxbe?~gU06_f8B(vi! za4|og;{vI*s%kkz2v&-aFpq#}e_k;n2&6sl?)=xb{{HYJ!t(Y^suMrD87I-`WG+Pi z@Ac6Ab3Hyh0aLML`fVz!b_@N7$5J#AYq7g=AmCU^u%wcBFRo2-?XXss=`2bSZ*hem z2S0Op*%RQmz{?|qVP;XTiCkZj@@qobo8-^Q!e(tLH$ zfA|ge=Zv1Oth@4Yl#xhM+|^gFa;DNS81Al)UWki}lSzJZ>A1e&YKz$subZa6D!V>a z*`aX*cx#D26cjyp47kNX|73m;M*ww*in=dN)Qtftcc0&p2*(XwGlHSFqnL_-PCp9Z zHl4fTlu~Puq6p0ZfK^yAkBN>b`X(kf!@R3wmtHs%B22EoOJeN4k}mPzmHOXcQ|FQd_w~xpJdba@O3hRlA+8aorRc&p;NUS z;vK$rED50GOJ$R?FTtorgZlB1I&kqwMq6(Kp%fvNu0w)=)IqZLcQ+A-!5BZK9~$%0 z{z6DUxHc-F^29c(E=X&`A5C=HE$IM1^O-io2DtuO+oh6FbcVBr6jUILroi{c>M=mr zNH7XI5()}m=m%!0M?3?NdcQrxFT^uUF?J&#{&Eh%pB&FfUg95uqQ~VDYf<3^n9d*}-V@ z0zfQS4j%=yYe2)+mX%83%j@;wOSo9F>o$C9R-*9bnRy8X36^)X}prdTF1Rwq#h%V!N`ULj6qEVYMtXB(ELM<pd z2rjU(JFN6XZovEK%@0@Y!e_Qc_9m*dk+0^nDwM?^cJC{Ddjm+Z&v(?5KTQ&K&c3cvmz)*S~UBbDRcN-t7il1EuXR%@bVWPv0mCiX<; ze@vt4Z`@FHl#*<&IW+xrp30|XBJEgm&53cg6-V-hI?Q~2H;;VF zhSVDx&N;f!4${{Ct4A4+%vU6m*P2!9iU!NkdE78Qu3@3eUzN8Vpsv`2&}Dreogn#W zL6A4znKRnJ=`-daFrul4^J!Aa&TMkM3Vkp-qk-sBhbEzScXnbQE`K^@7Y>YB6|=*N zH_hD4j3_^uDU=_>l-Ja>}({6V`=-icr3}degI0BNT;#H}@p;rTjD-%J|U`7G{ zfUA0zvhJFTb9a1FMR2lGDQYc!`KA&4r_kTQY z8K)3CtUI_|HZnAP-aeu~^aDwGe@9soU*W)z8dehz56>8M4oLjtDbNJ#*VU=e0E}`o zc+~ZSc6H1t&$a$lNtS@z{In^wHqwwGJqeJ8i2I6YRpc~G=|^O|1HzF&)Q?;&3kFAd zm6Qg_uuyX{t9&uwEW^UfXP2R?modNgSC~%piZv-pJ_!Napb}K1g{~8#_I!bP0{bBj zOaZbUV0nSAp z+SdblyV64CQ#42KG={*De^%ROPjBykwMo%)d~!@Ac4zqYo_)$QW@B zf6(MN**{1Uq5e5_9dnuM`Sa&vwj;Z0jZIBgw*?Ny z1!`dIpEc)LT?+)xIdgCi^O&!zO_TpRO1Zy|vVXH}OLA{Xl&KoJF&61Fo&#}ObLcQ3 zYsb!SE_67I6tA)gLJ|qf`&-*&J%C(BVrJLKNK7LbooQq9P6}Rf7fgz{#EfcYz6Wz3 ze2--1z-~$7D{4dbq_j+unvg(;2G{800Fw~@_@l@UrRW`=uAG~jixjh0-#r~l>e5@8 ziZG__yW-C~cKazxF#(OO;h~}{7|y{kRZw-Bxi~<3VPWC-ZAT4$L9WB%?5r0gW03zW zkATD7s+V>+4`lBBUy$gpnu70mu7{Puins))1bYt(yd_sh*n7YY>lJ^E)xDm#ZSh%==yJL)PYirXU1_J!0 zr6n3=YZd`>h3=pDopfb%LNd=9Djq*^+fq^7xODq@W8>cXj|c!U2#<{16DjDOs4oTp z0vUG;yIr8=u(eLbew+GEd#ENtVgYZO7Yio_3qXT^GZ#6KjVu#G5ft{17}NEosTPeX zkL<{UKV@)7yvD05=*tps!S?sEyraAOyM}d@1qz-^j@*=GR1x1*vATX2w_!D!|&{p50W_zQ45$aEl*cciV`lUTzk1FAP zki8wzn>Unm0ch|7K0=u!Lq`ztpt{XbwdJ4)9!*Wz{y^E)Cs0pBV~OtW5XdsVYc64y z>l6)ditVDFZV&$agRjZ{;_HkJmGEf)!#Q0JKoC=D$L4BQfbuKzlOJzi+v89#f`))s z#@X+1egw_W7g5DNp%S$8RSHu zlVoiTi4Ntm=uQ&wT(umxP>Pk|=}(||Ao)Pi7#8vfFmJ6|{%rk-ZRmRR*}e> zvK4eL+XA#e0k=wQW35BF$nP{N`Tw!?-GN-L|N9T^GMW-ai^_;(lrm29NNDm-vX$(N zj8rNGsh`@UcI^}4R>b=@Dx zhfO?A3G3&VXWxz_5EKa z_cFik``ug0g$znIKxhn^$z1m15<#{szBSt3!D_0S_n~V_)>+@W?p2E2!1qta$eXVr zHcSGplWC*?La?JcUKBE0@_L=5=#O!|AUKy1y%$S3Gr|vD6w!D|Tg3}*7=A070xk+t z3Nj8Fu5E3EKdbotO#Nn9)IQ!ih5`u)8N^>T&ePeX{pRT^E`zx^FTZ8cn5*knvmVb{ z2mQL|g>sESO1xf50Rh){HI<#`nXzYC%)`PTTir6^V}fM<*faSmT?=Cm zi8SMV*kb-^+Rsy4(GqbTo9^a7)EHm98h+dY>RX-pmdGvHi}gHQ($2}4LzxH(I#+$- zB*n^7C0rIYW1}p4MQUQbizu*dH(oe38IVpH}&pA+`&&zPF5^E?YQPSy@@0j@Zx{b}CF2ZvylS>?})ZumEAUCXqIp zxn-aJ{N^9yu5aJGA|oT|jvE5?BXpZbz|+0E)YlBHdNPP&%<}uz-9aeMshW`h%|v#% zIhleHyG)~o*1$85Qd748%+_I(==}P;MoA?W_8TXVQiLl)%YBFRo0k;L34n}_)hV7R z3GL+*Eq8QEc}shD!ws+8n#E6}Mhx|jq@6X8AFb)`4lS%y>JAe8pzEQbyOD|>UAVIG z<7xM2A9qO$s+YPtO^cRifoMh4!gUk+P&Ta(Z8_a>6?__=mtyQiS8zi>!Ee0R&L=)F za(oyQGHOYMCp>#N+k$R&#N-ON`|yN4rS#MM{QRy13;{XJSXhyY!9m0iFTNFyCnhBS z38ErhhB++A|X(3!??-fxJ0+lzy<8VnO18XU1RH+=B=8GwbW? zY2%B<8uj~EpFMjvfgQPspA|6J0&2!V{t9<;G$78RoT$_S&Y%Awa!@S|9q@^gwi$*y z6XIB;Immv%hhogp+XJJKT z480&E%&WIQk4&76k-L8bsjv1ivVvpHra~~mKr7VAs(P03>!ohE_%COL_hok;&dYq; z^Y_o~Nd`c*d0W9J5%}~}HnR}td|rnnC#C66FarAJ{ZU7Ya=&6y^S8a_!{y7D38nA& zZ^KE-EuPRWIN|+4L8b`$6roXl*`33$(*He+dp99)VCkJjp5GaHv8B~1-#_kQM8UMu zB!@f+@6M{YHAFEaomDpBO_-E}plitqgXQjt4d=EXE3c65K2SyE$@3FSLdL)o`uV?h zPkR3BocSyWbvYn;EB{zCL+JTNF6T?-k6;%Y8)gU<>G-PssF6+Bt><_K&6((S{I{dJ zEp4vVA&6DIy}hfezJIsDn^*HEjLVENw&?2HdxDLvAh;+hqI<0M1Dv8e!j=rwp=f#1 zPNTu}<|K_3=mtO}mWQRni>RSM!;!Quh?bBnggg4ARHD6-lDTl_3v|#MRaL#tVJ8Be zg_)gXrXey(y?AOhm|(%pp_5ow$9^e|X?T;i->z6;ac5_s<@6+7YzWsM!iSIlR0!yk)zHA9oDs z6^vXJWE*aPT$8jJGS*c<;?>v}sOu5QHlMZ;TtA_UJ_VB51!P3%7O0$*n174;9l{kZ zsJMtXMd(=C&`hg*^Qh?gC^x*YkpVC$87M$!zXkzME>GZWZ|Mi9`}8JoZjVUxP7Vpl z{pO@!RajJ%gOMY_xVSG5d&aWHzAL@%@2nw+Aq|M3BDICN3i?h?>;gsANxJ~vFKyRv z15O1Mc5-Z16&4DOX8;i4Bh-pZSL-ZIeb~8iRNO7z9Ik`5cB|rJQ#2#+9wrq8+L6LE zec9OI69WIX8r;*7l*|YkFsk{>W?$yyr@5yMw4v#@NZfpYAc#S2VX>!>kHrQyj^m~~ zaG)+IHT~7M?=9*p?QGc&O&g){wRkAyFzJB2epyG+gP+$47F!1X)r8&F@2i9d-Y zK2z?^cx*468DVyCBC#lj_R`dE>HO9&yLWka)+TQZbZ+s*Thde-KIPWeVNFvE=DT|) zj>zIkTj58V^EnaSt($#Qr$@ih-L!d$bzl%?($5Cp_)t;hmkbiKTGPT4KdqI~KTPbG zm}*{?2iy){xYeErpYV7~ zPd@QhVMtcxG+=YsjDu`cJFR{*oou4{P~34Txvokqr1YFt#h=Oa2QLkQw_v!x)7_I_ zj(qT{#0(@jHjt}Q0h@fW!cQG!n4UlzDT_8TeD`@Ka`qoRJ;5-AoSaZ57*)DTa*LD{ zoHsji>RbT6!#KH8$-R(~L^P~keOI`0xi;g7_k)_%d!c5J2 zU>X< zOwAnH2fyBM!f8kj=G(|h4}$L&MmhYOo^UsyKMa7DeNKrzdWi7tk+dl!A_;EPy?ghr zo)9gc5|>pyj%vii5)XvfU9+*>& zUc5#M$VS<}?tCin-Pk?U9<`C9HHcQ{5b;jT__iF9=6V72-9|DA}=NA=3}jA|v<}7tb#Kb>rfl$iZ*>p}EIE zL8RdzdIfRp4VqylX89v{T;r~;t{XTkwwf@?M!#QCT3z|N1}r|1OWp@SfOURRs&&w# zG35~mztCKkYP;Ip+S+SPkgF8Y`!|%M1`?lZ2Rz~)1bceA^|ioW?@oj}l>|U(B2SQ_ z=dV3y8eV||k>+#oaKJ+HKYl#Rhkf-~ElbmPTNL`cTWEG}cNIV9@-FL5bUoPt_afvB zPu$&kCbK#rE=;DY?%hAB)5DiQJRuLc+FbEHJHr=^cD> zVYsB0A@vP4NLzdB&@?K#8kanaeeD}6PmYh5=>i?-fy1740LTlAT?Rkr)Z{7W-2uyV zf0_h^0XI?wYhj*xHYQ|R`u#6oMU+@Y0 z`1H>6B@Hx3?UTtKyKJNo+%5p?6S6&o(ho3A-}NQ>jL$2=38TqU=?BqHxTWybaLCY} z29|~4s;{pv5uk2aV1B2YDDq?_#1L85&e}TaOt+f}`i$Ro;QRph*oxC#wZ7m>sx{Xx zY!pqE#e+mYfwxIOA~5BC>hzm=b09XRxzBO@i&+e)cCGw)J=V4eUo7ZZ{90p|FwEXk z!%fN0KJ2$wWs8>WBx6jn@vSHd>i;bYeP)WGk>LGkO`Wpo(bm@HS=v?OIv^2l54ke0 zW8Xr`t=t-%2I`Z~M2kclQF~ZUS zur32hHrH0;^6?P5{iC;pSv-v}iP%6IMQ6BM8c{e@`XSsq3$il|9j&b3VT$ae%*%=l z&3}Ee5O?HU9!JzyubdOMm8@5wIWy!M3mD9scdF%w#jlT#yF>McgERSn2Xy@Ut*CG{ zy)?`1I2KSD3cq~&rqKdh2GDNd!Rbj>@!Bc3$zA|Klo&Z^)6kN2-ayDdD98)0vEP8u z2Y#S?zAASw&`JYpV&ZQ=97&@fJEFgR238E;MCSZliL+|=?%iU#<()o{pQTGlo9Qm1En zC&3x$Su5n$@zwLHpsTdNiO2%bWz4q`PW=|629Jdo=AIw6*?PP^*`E^4)>-waJY7`Z zvr6mhRrCsEJ-kDALniuEDqNGMMgL&-%(SH?!d$U7ad4j|BM+>OV4us75QH(Nr^SbO z276HIAEV)=4@e@zS*2f|mH3tL1?0)I0~O8??5OES9JpOf#^7=gtQe5q&(lvF66hi_ zdm6U}Tc$QKeHFmwOq3{?t{g}MiKLXFC>4}S9)byAk2MA+NUu$gQRrIdTs*D9$7kDX z*Hw;l^n>q3>?TlHC%jN^IGgzzGW{whnuVK{wD#a6^R#8FC&v@9byS7-wjHj6vU7GC ztOHq6I`QbNK_aVT%t0b71Aa0Y0<$)08{k^u*$yKFVq7O?Lu~zFsKjU_acS<13X~ za2yH~yBh!}u^@-9#^hf&&dN0`dp0}mTp`$cM%&T(g5@9~O@~%ls;62Ts zXg!v1GT07l-WlWF8xj`R*jBp9M35NC1!xSkV~ zpQ%6vB=2mCuu@LPLZfE()md4B9ChHwZjY=8rJyM<|w00*9)*gIzk zs=wB$>nUSCf3(1FWkoCle5ypieUAmV;*!HK1e{F#vDYoF?)()D+w)^Q=8>Kii!RANYym$9aB5Uc3_uB?Gn`ZyNu@{%T13l4#QbJN22Sjt5iilNONPV9eUwLm%HSWB!nFMOR77JyAj(#f*ynY}N3A zhubQ?gxg0Ivk+2pg0^hoVr6At;x7=*Y!>DRutp``h}?PmWAO!k>CKy$XhrI%ka++a z{BbOn(DfiypEdfW%vD$^oZoEgBM`2DqaNIkH9b-L7#2)Y!GnGM{oN0SsN;pE1w`v2 zfl|pG^Kz@t0Lrnpzt|<(vIq(Q`xl2pxHAa3?q5B{yCXDJe-Qe3B!d#oym)j=Qv#h>KN%)QHeHJ7eB**YdpYmD}QJKx(GZyyFk zp#6Ahi7atuTNhz&Y3Ye>UWkm=zMmf1KmZ<0oUdv163&thw82Kss+Je7wIZUFKOC$! zCQ$9~_~U@=#I`qXBoHknHGpUwNR;t^#`)O}9X~F0b@Zl*yc(-iaq?o;!6PX?aIT+4 zeZC^rboj1F4WVUf42!O7RqKNb3@aIx%=Y@r7 zOSq{Dz+qwouERAAr}buL&VHBrP+Xv@>&OvGzteojYi=dKtEy!^m9FB< z2alGW^l5v28Fbk?_Q~me3qNx&b6Q>+_z_8JSxOvIsK{Uy2>;V;;Y%kC(PWppzOkk) zc2>l&Iw5}<65l>CRpd>dKAoJ-wkOkNkBHp)vCu#b+g$tmiFQovfvl!PD}Ilp4HD9m z@aD#PBVVb>SI=F)+Q0A62_?iX6)l#2x+Z6BLbAeapQBhQ z(7n*Sbg(lv)8q9_z~=sV@MZ%(y=Sv@GU$rk*;z>DVzO;|+e*>#YPAxn6StH=kD+{O zaHi6YE-@irmZ8Osm*$m`^iT)pWA;cZDG5ST-U}8_-XOvsEWehnvrryD`4ZwsrHA$2 zCfl_(;^GqZ*eNr(K>LK7{AWi4)ST|DzVmDm6%>rTdA$rvBl7h5jjoo8qYh3hznXk|G8&$?!bklJl^C-LfU=j)flTg%k}(m?V6ZAGiFVr>4wv;Qv?(vj0zd;G4zH z?y(2Sb1y(GNYFezWI;s=d2lk;+VLeM=mumDEjIm#C1mCWv`r4wz<>~r=%;Bv-in@r z*=RM`tKmkwpkW;xd^+S?&pGB_!+aJzR8!)KNRV&LM-|(kk;nXDCLe8zIA`&-S_1JI zQ{<1%>F%w!y7EnVSK*Z-WWW+xLoj|R79#3}W@icMKAqy(-LoQtmiCf#0pgF@FdSIW zwDviAE;6*jRP%!OaUvSrKCuQd5AQ*_XfYXV3-WaxP~#eLhrs8FBpkjVvzuWIp1(!!Rne=6IHUc`X=q9C}2FzeM<`$NaXVb1RFIM zf=-8ZiQ3Ex+u#cQ>;tQz6D%6uw31b!`SVXU>RHPf8761nj44)J4z7E3*W*(%Krrss zXWNe==z~&i9_Vh`uz9nu@eFQ(0^qYaa*F5^ll!t)&}tvlg5(i9AIEG`300U3X%q_p-lAz2KV=I+hGt!Gs&o5DowoL-PZGY2>pe z!1Y&ZYz%!;BUbGtLl9sagfuivYJzu$}5i)CuR|6Saq$;c?eGYEs@ zv-5=ciop7sWsuGt7cJ)zFx!B2xYTFbXWA+bF^!l~#Ubx#9MBSH4*WPflAQc|a>NtxWjDJczJM^cgdIdufO^-r<9~>WnaAX2Qi!dtz;q zm!ALWc>XTj9MQM2;dxx;&d_cSH<@;f*YBjs8~8!$ov>`cPklVT2<)nhhNFE#^>#KY zoO{)+xKmCRB91J2Qh(N@<0~B)w7X`b7U|U)vJEfK8d;cG7ap53@Xe1q9%(UNG~M$3 z8a-D+#_!lE_b{yTxZ;O&*2`p%msL4tdyULQjx{PUXB}@IjMr8}goBUGDx#!enn;~? zwh??Tgh@el+AfK~dUW6g=YYA9h`PgDq3`{{QesLW&s&dnKIuPo2(O;?Zw@dm4B! zDD3jfX&kWJO2t^(l%rcv5Jz*sq!1}UMHVwP&p5xUK%0Ugl}wQ|GEIHFlHp`BAA?8U z;VeU!RXM^K=~*LuOPB^y)?}&>M_%<|6)^K(kXC@Mfrei~=IOgxwjEspX|?pu%H(yW zv`6w1S0D&DGdld_{McyQ*RL22VAzo@tvA}N=N%p#ECg%c=3;mVY`HBuJ@Bp5HA;+c zUUw9|X#-PPU03wgNl&bS&iFjPugA3`+*8=8_N=G`v-+DC)m z=PU5%pR>tQ_z}W7-5hhGc)j~+B1+CBm&DRJ#f18+#IU?Am$2e)M5wygmj`v z(WM>!tXu)nBWb#NHeSbj6z)fBj$q2eb8v8wDXSneC{TiOllj_F6~`PjVp8wg2cs&R zW+)B(p_6PYloy1|3VI4Zo?Sl%pYj0r*}9l#b@LRc%pu7QN4>}cp&y)h;6H0eSjJBC zH>rp+x!CQ?+|Vmljy^np&^|tybGGKAmMlUc_bg;n#C(Ek3+up+oUrR9U1l|xrMUmVmhV#S}*;~eX`O?^ooi862i{X z_66crqq|eU?`Jf}h?P%9mq`UHk~lw@Pe$*^Ut1&Yg3j zvCh5hch@1Z$#)!`T)pNyI8eoEI%M^#<5%?CBT9nJ^?J+p7_3^HEby`)pA10{ufKaM znKl_zPEL@3V~j}bheQTlXHe~jdK??!tSPOl3wXvER)yZ82?8RmC#_w$^v(=Q?7geg z5S&!Rf(o7X2R5r0ZB>^=yP`MvRtpu0^3fjeqq&xDnkz1@89pVk3x`&>f5R`~ygk7O zel;nLJ~r>6%nJg$`Ue^bd`27LA9U7rv+(g&0#oQOo(rk1%=1d06WFo45q6tEYxtId zMTb&qn)YdEb@64MXoFuEpQ&--6@6-OuZG4NyPjfE5gH27E1DAXibtk`hTGhb709xr{Nk}apAG}~S(n*U@zb1=@%2vcmqV&Z@NQYSR>3RGAP$h63 z8KQZGPrnXLCpe~aBkg)?s=zTe5k3qkEZ>0i8yKeT{}5vYQZ>_W|Ky#V(d}tXt54X; z&*B%3#zq-#_dmM```89^D)nib0>E5zk(#`chClf<#{Xm-LvqL{_U(i6RdzCS`Ev`LmunN#@rClXREpwck0`0#(?p{k`j(Y^*cF?P)AoPE8Z%}S_{J600`H?^b)>9+Dwz%nTQDn9LP(_flS67TfzIHiJl<^8{J=cYisE=xGJ#KO@d?Hf;djV zg=X5O7}@Q1eVFSQL+Z6c!_~zadv`rHuwKV}=5Wp5M5x?1)HZr#iIZ|OIrX#U#=twe z4#+*g87seK3r0>PIhK3<_J}ZC3$H{#{X@3IL3&t^jLZuogW~%C8R=Xpt>?~!oGz%l z>j&2VWF1dofAlE$GR^hx!f;H(^R$}yVqKf~8|Y(rJspz-1nM%KJxK@#Q#yYv4;>k? z&c+yAY`w=;aZ94BdM*$$JRG1&mI?-WVy{jB>)I#3fi)(}wY;>{^_YZ;g+xM5;&3Wn z6)IS*n~#qVX>&mqRLuTd>KYo*TuX%uk(vdn7gNf4G}TM+2Bw*hA> z!spig_DlYL|1iDbOUzF=^Lp6?#I~_JXVha{ixsHVsrBxT$pvsG|O$FkzWk`|G!g5+1F6iuhvKaifQs7Uw-;lwVN zw;^}u6vDz+pJ}|dhSoSPg1nd5v^R~J0A&b%xw|K5Y-`@>jt}!Dc!PXI_|U-sT`zlz zz&*=+A~rGH+`e1oG}_JFhLO4k*v6Hu0wTn_YXkfrf6#t&40AqtYM3#^`1V8>Svz%2 zKkM;Dq+Lwb*Kg8G?Vp3jC3)M9UuDbu4OLH=cZp=GnJIQO-r{;q`Do!&flN#0p3ov? zHWHr);xc*`UeK_?i1|HV-w6sd=?LKx;1HQG0nQ?a6-=#2R<#S13g1H?Sg7;kbMBn9 z0S+#?Iq4Et;*nf1uoL7TGgTxaB5)&YbQb5%DdHlp0J<2Ue-7iSy^r%`0E^;|F&QEM z=GwZ2Np$NAeRx;JH}*bb-7myzR$P=Tt~Bg@t3#?A`O4(9U8BJKVYn30+2x7th02T_ z$m#Md&3LxTk@F)DaW2ctzYA(&o>^C~qBGFFi7X5bHA?>^alHwk^FZ*;GtMVu(K`c; zGZ6?y$(dtXlyphs6J|o7r}HrNfBed4+dB0i#bu~r1fQ(9@(>AP&jx$8>*)m*bBC+I zLDG$&xOI6@G16&sJwYbXSDu3yCVgARSn5E{k@L-HD5VW32YWQUt`i zCeM#M8y`Hf-0!0M#=GZ_^1nfu zPKI7k@_elpJ{p`n3pf_hUAIMUC%oT|i}e4I7IYyHgm`xXrId+q3-!`t+rQtAQ582Z zy@JY~N9R=BWX*rjCf}qZB+Z6M@#8s>drDn7V+Z*#a-{^qWPj9H-_5)5x;UXO5zbWe zBCX%z^N%>12rrPT5joi*(5*%lnJ_hMPQ#ekoJfC?PzKg1tQ_3cFVuoRW3h4ukjv2I zEJ7*nM|@%WnbxNts>=P)Qe}FlK)h`5s2*eSp77peM!i5s+P9Mc(iI81B@n-&bR{pMi4pX~i$P>H*s>ny+dm{0;D7hmGh!EN~ zWh5J_fRJmnlhRDK=STVx(bg^ylMJQ=LL`f@8!!=2TOTM#SazT&4AOgN*$9mnDycuI7-)?mjZTaRw%YP(J3%*F&<;tSI z$sFD}1hmkC-lRlO#^IEjqm4>+hp%&5@_eS&or!ace==`ghT>VF-z$An9;?07yx=8K zoP$sjlzDfvbb~7P?^>PW^TxU(3uEFxV5vm!@Z-)v`-;db!t#uF)B(iEU3g{6b}nMy zBQw0FmJZ@mQKI0-H@v^aaedy>t8*x{Xhdv_Z85)H5{@cFQbo`ypCy%vFk|JIcX*90 z?=92tcve`QszlmCF#1}pKuk_F8vnk0J{3NLrc9iBT1ajGahpvu;&K6&a*!{z7^X zv16M9TkEcUBT8weZjp6--SBfzfxW%1)zLmXSsN7|&t_;VtLS$R1Ss># z2RME=C_yCTJE}+r;9Xzktn>zcjc@gE``paih?g)Lk&Qmhq+~oD zmX^jJCY3dLApq25xMF-f)+&c6q=?*^h>7!N$pX&Tqs>VEHnBXCq{XXn$%V|NqMu7Q zoMEvL>16B!Q(!lZ8xV`|h|yaB)|bpS=DmdY@&Tp@G~>@nrUKCfh^ucSBw!ojJbK=WwkI8y+;bExcC>NjYJvaze?;D zv~V=84|x#tPJRNhZ(=25c9gbnD){6#Z{5j|p#yn{HzjilH*M$^#$N;ci^||XE36#! z@vpC18GB+O1}Tt45%Q)wNZ}`$B~n4t-1xh{hDEJ+5`@Lfi%@Q8ilQr)e$ag(IHTer z9>AmD=dZJFnSPE4F&lL~DLm=nb%uqQi$a;J2RbOoW%BUZQN=npt|TmK%B%B<&UESi z+~wK*y+4vH1|)w572v*>Yk>A(0+*qU!@w6Eh!t_+_I;-(u2R=5h8nS*p+k5{BdmPay755%2_vD#SD)NI&z zOTcNMU}G_z-h$s{MUu`Sj)&QyX{==-fxQ2bzviQi^LZHDfwgHH1nNrf zV1ms`!cRuA)-)o=35z17^RcT*S&x(Je z@BgR6&*^T;1y$MgRcGx&L!5?0EEH8?hGO6P{*CBEeU*9|MC5)==0MQRX!^IDS40U2<9j9XDjEK9R#6X1Q|I`Jc4sBT{ z-F0)GUzFObj5GKw4b4OpUVI(ra0;>O>Ff)?Z+U;0w-w4Yc#x#KDy@v6^0uyioq1IF z8d3~N(!`A|U2mYK97Y}ab?v8vD{e3-JUG+MA^Y4?1_l_&q+_N_Z{qkIi`$*Y6E5!M zBxj>M=9ziMTzyV?B+g<9uUyGxAps62<#F)M)UrjRUV0mo5lZ6fA+*otg5u&_Bv;)c zjP%uQ|4zxtJTXktFi$8AAMv*Q5KT9^n4X_Hh04c_A7hIj?d}ghaeg8jfU%@@*&ODA z7cF@6+QNf`r%qY!j=)euW840(u45I3zjC+8?M~gYRKNVI>AKhFU)tH#CdO|>Gjxe$ z_{Y}9;n;&lhyjE?6Oix-XS@}TWSx_EBLpel*^@1`HI@!q0+1jF7fEQV4TWJd=lvm< zq)Y;m>32dvGDB==)KkMAR7etL!@QTt+pi_Uq6{A^!0<{$+}9DL$$F6F-KsJTrDbG9 zn7j@6JMI55iZgba%S}0V}e~|=P%fvb# z2ouLC)sWx!b42$-1Cc5&C~$53wvAy%cEd~D0X$3@(nAK(4UT78hM(zg%KVDSHUwr&#iZN#_E*-Gtzt>%vu%fdSkT$RY&Xlv{Uyow z(-(JDmk5mID>SclNGfO3bU(0L^;bsMy4*<4PDqTR`+6P%4gCL4{_d}E?TF*1uUzCN zO?uiqCjAaUxZ@xCd)LAkdM5t5th_uQiR>d&O;Dls&OiVrxq9^cs4IYg`>Ze(@M%PZ zBD1jJQ0vqCBCHX82$u~Eg>-xwYo?9we}z|4zb=hziXL*UCIgOaiq7L>O*g$#y_p+8 zf7<4)vUgJ1Gp~RDy5^;zm3TtVSZvf%j6iIZX197>6p4XXEAb)cX^Mz_;AO^xX#dGh zBGQD&+4~}4D1bAwnj~3*UE9NCKiaEl^8o8g4Nr1I0-cOBo`W&}ca)Mh(3DV&InAYx%Jvjl|>-5{P&Q7NhA_6{MC25ict}APh?=p~2vRyVHfsQoG-& zN(ddnAS#_Z_pb57H0lRVK25LUYBK1QsptA`{N!8C?f71$X1+ zJ|!rCejS+eb7+$fs~};Fz3)_2z$G_+8}0f=j!Ii3nRZtkwHB&^ps^TJCfFaMxLUy- zSEDLoAj|uF>z0OVboSpgY7W9GMH`p{;2)zDEyq_x^slXlSyH>`>!WmdC%gV7(;crr z4py%jjWaY?{pICo33zx!a;9h@XtXA!y7o7Q%zN+6D;M< zSz2&7Bp;dOBV$12 zroZjJM{%jc$a?q)Zgr5TlNX6w!k17L{v=h0p7Unf+G&M=4Exd9rem)br_;k=7+L=Q zr6c#kopx=yC=7%ZSC3~!h)Jwr8!w2Gde*N$PUEg;`xTM6{=H%PXkHj!wf5|8=(+Y z@c3I1E`T2Bo7ftKIN!)23dIv$w8r_NgY@kmgyW%E1kwuH^ z#oyiHyzS+_Nm8v3x430EI7gDi@t~n%*uGq?c58t!lHDCs&d2j6_=EPNSB`f(RM{fqE0 z#BdV~5xxV#Nu{PwevXmw6iddndYASHTFl&jg^Rci!!_v%FzB7IEr4CJDxBa{Kul^V zjl4K9kJ4X;Yiix;A!(q+k9KU|uNB?jyg(wY_kgzpE_sP7R-{8WwAnKI5JabWxt$JP z&)in|du4e2=7+D!6{#H?-buBW@V7>djDE<#Sy_3()C! zqP@}(B@{;onvhxW-x5Kf0@^<0kVZ^q6ZnZYpA1C?ZI7Vbb`YQpYbouYxx6TZMJsin z>VbGWO%Hz1Zvj2BK;<*W9rPFJA5+?CKE<(%L5js@CS1kVv4 zv8(r3nbOEXusbjweau0D7_5E6JTR(2H?mA9KEIRrqU!?!{M--TZH4;9_(xs6C_5Qg za<|nNkWoL=57fpdpIxHOhMQh+P1Bf(-ns;P?Q>IW?;FPOrlqB+$fA^kSXwoyS!ynQ zX+KJ$Fpxv}jI*7N(UBWdm5j-Cc=PEF1{rtC3v_C{ zMtB!G(Vi6y;4g=|EAjICHX0SC50yu?teZ+1%aQRPG*vNhG(#&`6o%LNpLog`h|J_g zwBSYLf7?Z(Y-fJ|66rWmDeWgTeYd`@!k&2ZjG6k&5I2;E1SzNfqQdS=fssO1Q(Yg1 zhz6$F9kfqK8JU?R1M)yx@DZC>y0-X;>oI2U+`M1p4i6yI^sEEE_wReqvpwVN!Bt|t zAhVEzW5xAnuV@`<7!y2?NsC*~v46hWd(U8j>{_YXq=?XRq@A1Jt?MjKICu2u7K`{Z zjChAb5mSCri>5A|%636=!SQX&SFc{ZX1Jqgc$Mv|fPkByi<}fA^HuAEYFibfKHs-% zRb2D4btFbRf=|Ka#kFe`rELA#W)p{lUC?gBkJT?u-g>C6qEZU|U2PAzq@m|2De{XJ zEn1R;_OxKul>3+924Y|sNy_6GChv8!W>T5d;lqc6flK&uVq;_Rdmd;FBcS62mS@%n z^g0gzPFsBMHa^|B-RGA+XtjT##5F>(J#bjc0;2?(*u`vPjU(eH`YZ1iM=6-Bk29i_ zqY_dJgzgqM^k4SgCdMPabbbW??%Z`E@81>Xxm{Qz^8OKidh*jv(<5RpbEx}i7?|FE z3|H0$6`wf3r>}35>=Cp*I%x*=AY%?Ux7MLUw++I(AN{NBc=F{weFo#+`~?f>r*ymV zCNFQp`^URH5A4|Sx5`QKQpyM7a!{@h+CKQXPDJD#Tyh*n&?1}sL@BDo07R$Edl>!s zq>QhsoOCOC78zNQ;?T}vgd2?aYijyzz5tKJ7mymyl$Xy9&wBMr8sy@F1#E4e*RNlH z0c9l%x4)Ogn&caU5ZE(~Qm%ZU#8r|eUnY3voEue=aHDs8C*p;a^ldBPBCm=0#bG?W z%qn^0LW0~>H@1RK?l$F1soN8mELj{W;i<9DWiLZ6n0))0Ctd&Am2=5|*)(M( z@4!H-&@|ns*>mRLkvTZjL(2BmSjgT2Ff6g_IGy@1GBi|j-K^<;jg5xe1N{8nvcJZ^ zcGj)3y&Rk@hY^Gf;b0t=X-Arh`S{QCT)SWnBRZ?a>KmK!U&FhZeKg(232&pwW zJ(X%((0|+Em(!93JdgNuU!VN+acyZVZ`)S)Czmchwr{$(Df{nJ$@APIv7Uo*eQ_3} zDO;Md7Y`{r8EPr(ucCrS*zD(;NAeUH!@VK{dw<=vl~n`MrO$!*+N z_C%VZ9>|5^eZ3dEI!gt&!@ole8;22cJv2>CLsiJ0((?%lvJS;$kJ=t~`y!CLw*vw! zd2yHRJG`E~dr)NyG#QLg+(nv&L5s4zd!{^)n?04?rKeC1*AyS;4cg)und2pBeL#ew zQn*J>xka8zV7`gYGGDBaiLaFDqSKiehWwK7v;KiQdd(ZQOs9HhIUg1{eCF4(^kY}E zAKAToWM9TeHQZ9ViJiC2Fl(EA4~>sS}lf~N&sJ~ zu|USNJMfL&SBMm2jjQ5kdt(!8gWalDP!GIj{#mWn^|iNi_v^pEBy@N4Em&Od@X+m? z)n1(>D}KSKPa6!5?`A70ma$r?vk#>qbdGYRZL&vexvYQTlhqW}dmp7>d#BB&y?GZd zWViP{fAQkIBvhQ^2Z(*(h1T-9`>d=Y4a9!0j>t%#;?|8GGNWSir+DQymBMMrX@r5U z!}j(sm@7nc-@XUR4-EQ!%gxGBCe`*%wJ$(U%V7q?l=tWh(|-&N4Veq2rOze)%ug$N zp%{O8-PlbHgvgr%;j!VO zMQ@vTQYm^aE-4RBr`E}zYDAHA8!i0yGrw6XP>a-&cmJo3c$1}Vc)|O&x3?kdB^ekz z#9rYrYRdJJqw7WCQm!ixaa}pm`8TZLaPdT#$MeX@B1znPvX%UWZt3eKB#QTOFOxrH zU=X0Ev!G2yJ>($n>y`BoTC>I+piWo_pLW_Gc=z|!jl3?MEPLvEd%Ci9)9KVcT+R5l zy^k-4kPSksSKp&w4>^I(ro=K|kJ0~mJqFWOjK3Tzb84`9)b)1P^rY}tZF8KFF3_e;BWQZBpp?)C7#b&J!;b8$IU zB2#PScx!`y;qh%`JK-{omk%SKKld)qJ<;2yiwop0%dc4R2kvyH_mJ|L=Fgvbzdm{L zL^#L*XYZc}cJ93NFrlVX$9iMtGS5T;XvRXMe z;#jI?e0==n7(n7fxQvc$mIrdMG+udF^yZDJt>+rI3+v|2Y(rSoJ5c8wO?E8>rGL*_ zuouUVn{%7jloyer%uOX9DLOQsS&ui;mNYkxB`cBlGo`d%>TcCM>VZae+0NYJ4>n;w zp4ZFhWn;3Rclj%`QvCRZ#i9F#S*8P=4_ak(r5|@c;-gY7mwmC2!h$tmz6?8eEot4Y zylrOL+1Z!n#l`83Mzvi9Yq~wa*Y_l;KqWPsmvU{lOspxK^_OlFX8uIP#brpaeKB2x zqwG@aM8w3TJs_}=s`3DYn;8!2%Q2v!%`vCNS|27R!MFw1nABgXH5_ibCqSiCC) zSYRF@rR|YB*=Wk&l+Q2C4BawB7Z&-Du*vVhG~>VTKyCTll0aMG_iV~1I@USjwc6*U z<2t3MtE=pD2b{1f+P%-RpDk@mlC0aXLieDzMJ&Vg S9Jr@^la_J5IbuMGGf!n!) zFZO(92u??U7IOnx$oiuoav0%+5)S@XdB2X1MJO1493t2a{UV~G2US$A8Cc*hb!OE) z)>O>fc{)`^rQ|5czr20ieA0gc#YoZHt9|zD*&KFjwr$%cZSg@8;G=WF%&9c?=#i6_ zK7>Oz=Grp?b)rGpWo_Q(zH-K3n> z)#kMw$8p=s$d2(|8VlpW+5f^6>1+qZ!$~9b7JBxovocR!?{3#d+1lqOj~nd=uk!A^ zd|It%)&q&F9Pw>G=55fH6;Y2d(2Q%Mct7G+Z_fDq<^!ir#gIaiPu`P%L_1$J8uG_^24)&`wV!BglqwLNi>hpkji2 z`XDI_vTalVgs{9gU0jw(?En+q@;95jKC}vX+s{`GI_}#Q?ccWTFS<@!&z(7yKD_-A zc+;9heV$xfUjEo4BrME)GuK8p@rI_vF>6at%HzqUx8l;hWn}L#zj#f#G?NN%0A5vJ zu-bG(TZ~6dCH70vtXY2lSS+w)lms1FUHE6h7tsatrG@fxR&JtPE`9j$L6~zpzeFV4 z$BzFT(Yj697x*OvIP8PpzI`j)s(bR}$?;;+(!%!@6he0*-b~4(>(^&*uN&C-Cx;lA zJE`}uTlh(tT&J<_`J2DN%8MWzRv_5SdE;t2wWIYfFkix{UjwJ^`1tVUJkiw%Lcd^@Tf)8s-`9KM4gBfCJbXTO*zcZ<*u?*P_>?RSwecl~&xkdl@^lxUpO-uP)l>?PYTbKUBLyBn8yF0Q0(`gPVfotQs< zFmFw*vkGOt0D@crT}nT1Ms36&c=~kLv}x1mTHt>hqcpvzFDNT>UCENnX;qq{<=_yH zX-X4be;IS1ZQGNl1EbjAz_X)ra2hRHvV>j{BSU>^k-15@3~&(-&+!8*@gk3cx6j(Q zX7y@QsQma%CFb~r$9df=#KqO_>eyZ*$iHoSo7ctB$Pi{JDN=XrpN!@ghH?BxYzm?8 zIuD)-DDL2;NqPvV*86wpKKJ$f^Bxy_#fyqx zZ`nqS`xGOYf%;OYvGZR>H=Q5k`-iqEIpcF@lSE3joM=X6FB|=Owb3W zzyp)_?dJ@0lq=X={O`>)J?iY7ENk&WpEv)_n-cmU!Ebb#tx73u!F87o6abdj+0T?Bg#`EW2cSp7RsgM-`R=ez|q@u z=u1WK=Qmbq-1sh@smOZ9U1`sHq-`mBN~EN}&`mdc<;k4o{zaJ*1C0}C+02x^dBM-VPI-TvT^T7)wW?N0zK z=f?(JIo-Lj!^MxK9O9YdzH$mBS<>9nf?=yAM5-kKGVKDEox4Q(TCZFq-VDsW-no6- zm;nEw^+Qb$lq4TBONx2#j>#LUkKw>dDZ;ng_Q&2or=v%BFvBY(p=9ncX|Y|2US5@( z&NzA1N!TRhRAo5~|LufdewynSMP*h+*K}S-UG!*fZS@~tJjY8hGNk-&KqoFX_Q1J} zhWnCjKd-yHb2?~i`;p#=DiILy+6`Bil!wXj<9G|$QPZ%U_T}c}yaoPTF#RbxQ_alH zIZ30ukv@YY`1H15%+%_@{0C=o1V2G}&ia5wb%(%0vYq_t?c~KLvj@QPz9dQ=+Jew0 zPhLyP+O_Oe3*1^oa7-_EZ|)y@CYtA$;!swuY|ER}0%a=azTG~tSd_U>4YR+E>c6*a z3N>`;mxxzwy0TaO30vb(#f>8Bdb}qpzuzknNV9&hY8P4o_oj}ag3wJuHBOQ2ZAU?I za5yyWl%?)3yPIz^(Gxm*E{^*?JOb>Bb9HEAVod~a{F0mli#v-09v;aQU!o--DdqrG&>l?S(3BMOCSP`Czc`QHk#Sq_Z8^ua!gL8CVSeo z;?^>7Vw^_%_wUz^HNM6yy9bUOc?Rn9_fq*67gofd9X_4KFf)6;KKb)}@^_Vtl?cu_ zd&!sCARf#0_|Mx}WO#bw?NIs;HXUf@v1_Fsm=Cb`%7~}N zl;5rSC(Ah>H`$ln+r)W|YvUF!%6<<4{zb*bTFW!}*F6Z^gdWi{u4*q(iE^>~jT@S# zp)vbU0<5Z>EC~wO>imxE6rR;sAZ^WL8#&;QuUy3W*I5(k&>zhLT9dAzqhs;ZNh z3&_m%LB)&1kub5#gV(@t zT&}^jOp=F(=P8<@bz&aQrKsld4@(VFP8oZwJRNl>;}PnC@;+-8aFIVgxgt;ZD@l<1 zU6M-{acN1N;-XUCecF>K)wF7FvNg)h{!^d$mheA4+iZ2#=Fu*$#D?2q%C1gVdAP1s z{5@cEM2rW(CG&`ihKAgGpm33O>-O*2b6w?{+h{hx%h#(j?TSL!D9MZ2hX2%ARRVwi zWyE#m;-%V>W_phuI56K`7#~xw;kgVwgxOqs4G^rWcH3jP~mJnlOeELe5hcWz{w$$CqX$g*{Ji#r&+%e^f$3stYZ zfBkyq5jCrq?9?$KMMd3Hr%pW&59hdniXPdU^4KTaWDc1*7`d44hweK7LEH%N z*}>a3n{J#4m3QE3I4Ujnyo0!iH=Jn!5&k{UFsD&jh(;wM zDb+};FxJ}V(ucggspezdQ}$>EwCJ9RQ(S-go!|_Gr*i$dUpUW(Q4f?pC9yO%^1*uS z`%e?~M{rGu?cD})>_^>B{q@{-cp<-)%$DonbQ2-HdBe70(c0=mIw1?oa42JtOj z^xXN`KYS;+Lr%%Loy#?hS-Wc0b_5L2ugn}OkML9WA?`_BmgMfHoqzF5tmLr2#NzbK zNJ~Y%*sZR9gK#;AQJpp$pjHKy4#e#&5}Y20Z~(rqp`mNle5JDK1}o$HpsUE*wUQpe zAtAs}rgeBbFN|`zXfA?u6?W_>Wx65V$=fjD)jB(qB1{Fs01G93yuBYmr}87n@PYvb z%btTU^9aZ=j@h+lr`euI0E=y>)8d(DTYKNrNi|FPJdfGrnzejwDC#-DV({Z~0ku6H z3JbhZAFwl}iv%KnnR&_bl&=eFU~5~sB*wt+F5k1>d(pb(PSdCqZ4@4ouo2$#MF&P2Fv{C|uIRlXt^5 zbamc4&M&o(s;_5K#0oA)C&7USO0#7qlRI;lnrxGz?A?cE<;lhwNgKp4ZqMV7^!Q*C z5ZY~ObSi-BP|_mFExbcZ-Jitt+Z1YezW->-*U84q3l?zy!Gn)+!mZo98E3EZ82TSS zvD>_a6U62!D0TWr|4Np@9%*g+?h6+zFrq<1 z&F9ab-_VFjJ%IdE$Gct9T#$?IuXFSaVi3(s9{eF3%uea+e*h?Z9u~%Fr0OH~4+L3{ zQo$L=f+bkB%d5V663A@R*j6~1Ri8R|n1HCRlEr+A95=YiPVBn$>|JNsjRPUE8k-L{ z1uvSTy?B{f>5J9=vEG)~=Op?qn?bo8GBkVyqK-qY_(xCA9aK?6kq6&;T+)6Fk6hm1 z@7bb=#gFn8|DsBKNY@IM105F4C^9P;+SrJa zecA&Y3w5Kk0LnLbU|;Sej4NTiMpP!1F{zNUXNSWym*jwUlZO9-Mi}*`pgAtPfCh=J z=|rcZ?+3fVZo;<2gold(kQ5SV#lc~fc(lDTO3uR*APkL9MEwnHO^@^staUcDZSp#c zdYn27-!k*7q)?Mm&0qQ}6H5r`;=OqtA_H40yoj{~(d<)AtZ4?q{GF$sRs*i1#ZF z47Kro$~78);{35KtQmE z7?9>42k6IPGw?!YlNm3+wCPQs^~cP}s+@iUQdm;n2`8+zlRrDZcwOvo3u8jCx)6!J zNgdzN^Z(TTk-AbdFS?%*4ed8QVA$Y$(}FX`blbMSg;(zgeNq(G8LZy9-Y&0```$86 z;aS`9dI5pQJGVcwfAA^-%3Ojy@Zmde^YicAyJvb}<%4gyIQFu z4fIoZb+yQ6JapiIA+NvE`E(|`a*MDlGdTI4;FAEPQv;P7!HC&*K{?A~aaj3t)%D4* zUmqgtm>fApMY0}he$pz;Bc-J5rEZjugx>nZ8cW`cj@W#fPDg7v{RZ}L9?r5HAF(60 z6~nRCp8gttc2* z793v$~u0X=a7z;sw*?5pTC7@Tm7~6g72T{L3%a5!J|R*%nR0 zP7@JL;^eO-H zm{WG2@AN7Y96H3NY@Kt8hrAp5t4;U6?`va}(t~&MK~)<-K!`^4;*$ya=y%GUPZKmIW|!l~8Eh|u&DP^f;5^X@^^cb7EC zOtR4C3g(SK#KyXVGvFx`T43ERT(l^UfDNYOLD~pIylRnAOcJnStfd=AL=l9CY8(m% zVxpq|kF76(hkF0s*KNBk>Xs0~tt=r+*+Q16>}20bvTxaU+E5AEBHP%vu_XIgrc#9L z%VbY>(~vzghTr*2_ulWl|JQ$Bue!IA_{{r#&U2pUIp>7c#J#1`Ff}cPKEF``I!49< zqz1iubtDTe_0cm>nhe$Xl$krU#&RhsD$Ylda;4SAgdG6`~ID{IJu?b;*U%%e*Yll=gXFw66xVRL5>A}xI*!zbWgt03~ z%|zvk29#4Xx_H~MR9^_@I3sQdEWeXLU5~45DLKtZyuH&bdy&qP5yweRh z(D&*=vWqYF!N@N}B1rW67b1D9^55%PmLZPh2ldl7)PH_^4-2KvX5-YR)FKL*?1b*O${L|1lxj|}#*P1m0{Zr+3w!iNO8&TmdK zoCxFo7vXWg1~E%GIbAbP6GWbx@DYf{RBLKdD0LW#1xnQ4)R-_rje#j*n^!b>xh3s(jx8Oh{RX( zKXv=g^FyvwU4jQzmPJPFG_3p|U!KI%WAa@|?&U)?+mRs`O*qe${WdzDXCSL@ZlRef;p@m)t1|{5^8eN|B_{ zLv<4d1`NQA2#WJ!W@!TM_94JK(4Kg@DPb2pE@EP0L}(VEwt(k>*#J$`_ zs}=QUslFD*T?v3+9;Qy$FmR~Wi8)!67^{!X^MvC5p?XqjavnGZIyepAl=EKKbj{Lv zlqoBGjzgu<;{7T4Mp~*l7YJ;vEiH#2UrtbU1%}g`{KeqEH3mLJwF$0KPfw4E!lRvx z!a^0(l2Y4qom>csKmWNincWte`w{7!LbTdbv~sU}R@lee+1XM2O225>NH{#x z?&5z78y#O)wu zqw=Y!PU{1Bs&E_@Y5ek>LGTn(52qbVlU4%@lNt#zW%wcnuYWSY|NHpF4Q4GLN>Hu+ z5hTN%ziM~SJz-G8H>LU}wL#s^eWisu?t;~VyviMH^q#R;_Uk=q3h15;S^RF;(f+hg zw>YZz%J6r2M}@DfxPnmzYW9v~85!0A?-NWwfKiHiT8O}4NC#8e{$|Js2>iIiz2-qt znK?Lag7EcAAXEPB!F`0C{Kwi9pn+sn(AzoKuc`mO@3*09mrQeqzB~;|TtXQBNXXpO z9ysCcV1lZ`k1P6SJih zZ|cj1v)oJ7^+@XH*S(OLm@Nu*(a={!FZ<|vT9@Jg1gqsley}ae5h6_a!^ehr`g*NuT4JNMV&lX zsPz0bcen2D?llpfc3rA8dSyk$D5$T5xZB&>dJ$F&sxH-Vrk$a`a)VUuC>RwAZiD6J z^N~k^>4=I{CUXgr>XJ4XRJ{5BRQ$)r5ylww5?a`uff9HQWfPi0-v4dk*x{xW_y~Qp z&*l~+4_F+;*mU4HOCb`vXb?KY4$O2qpd}&6v<;B}J{Rbq-9d0{{F0>@BLp%4QqBte zfK_}-FatTH)S%1?s+o?g>n344SJs;6mB1`_KlR4Hl0ur_<3#3vo*&g*N$aZ@N2w$W zK7VwnDoeC)J*T)MBNOKo6rlA|ovBRQZ<~78hrF_xae((M9T$ht^-Jc|yD3iP(WlLI zjJ==8UKBFibH4$6mDkVTeBQM~COxu%IFu;vQGmoe1com_(Sry_w;Q1HL?)*JRjGY? zF7Ln08wUb_Ioa7M)CvUvBcxa#L`vk^f)NbNmuhMQpgMse44`@@942t=HtIu-41NK@ zLcX74C&Q%YcKq~;!g~!lKvByySurSegC(^3hTT3*}`tTnOh5@ zGmy*r@kiLXLb@6|w2u4tgsD{p)5sjv)V!&mS8fQ~Ri3o1=&C+OMNEhAjts1WNBDfN z3`xlJ;s;0|u}En+XZgSD!dc7=T>^M=cBt4E!SwtRb!p zWHS@>zv7`-I>f{Ku>2|?wskTz!JymBgoM*ECtsnf)|#PozX194OYUyu&$ZpP=1Z)I z@zwg@O6MHQf6)sMf?YblRS&O%n*_ohQ+n2Q&vEkha%5ziU(+|Kts5lwk2vgDfYJ0v9qu zv35b1a?dFi7N^tpbr%laAAs{vGwOs`ZYk+-2CNMsLlg@MSr#GVz=NiwrY3?UQYu3z zd@S;ylTFxt?$%$=1yNu^gM+y)N9&-8nTd&vNT&P?0So+6^9q(LULdZ4eS4(hQc{ZG zL@_nd@fqjA+wzNybE1%OpA#-6OHh%&ke09|pr|u6ai}HdK|G@vC?3oMRvv;r0CNYJ zxeS!UWv&xlwa9VtW%*A=J@IczFHIVXfoTi1GgNCA%+hv#hV58_{MsIYh1t|ATQ)m; zsPE^Ki0h1P4^^7i6Cf@S$;!2Mbms9jlpe)%q^Wu8X#Ht8@gtHV_OYj82%k6!dk>e<)|DK~Ee15CWR7bE5QFXBbzX2pLLlc;I+7Inj za1~E6tgwqSzX;_4MHfagDFGlvQgW{vkdP4ZF{2Pue^EkE+q+N4>8)%6Kkxe`QP}=y zMZ49c+toB$N|67zrzFjK+x`z_(Is=pqQ@Mt_9tqJb=GVS1$tnE5 zCz8d0LE`#l7sY6SKMjVt>og{S;{rV42`G@5jzd^afc}-+rzL>%_?dFF{9&Lr)$L;u z*uTGHFpdAmRMW*8C-&-(gH%?xWnFjQfJp3fHI)wAfDoM2V4NarK^95pb=pC!Up`koL!3zL64Ltvv$A2}p7I=SXf8Y1e15EQb!x=3pZ#2E|&e3kC zDoE_3wzy&w)nfSsQRmcOq2gtHymg+1g+<(p7q`_;AN6KuXQx!b7X<~iESjEQq8KI! zfWY3Am4PodM@8e8bK0FAJ6-}KCD4cuG_d;yy)EdN6mI{kVVPBUdc?D5JtH-4pvF`M7JgMN|&!IYfc?`X`Y7c(OU>bosC|G*0`NU=b~ z9+>m{%Rlo*{~`hp|GE~x%Hv^wDHH|i{bR0D{>L41`Z&ZWeFQi$eh9&_du53M5J^4=pneo2L0=>(h1YRO$O@y(F7dCak!GWOJH#!3i zB;r0RS^r|*4ZtDpN_`iKMnzljz)VdE8T9=`oOU3gP*iHO*PtxQNkaV%9NiZrEVRRb z$_n$w1C9E?jum4TxFPDqXaW`#KsZ!`vL*pK8k)>s;d4#`c?>gY|K_6p+c&W$ukvIs zmD{~-f;2dcM~ac-2gBo-_y6|5No^mMfYIbfQAYlbJ!zKzZjSH+lWO(yxA!uBzzZ=f zR99aCc7Wo3p=uZV{}?~aKxov4OLRI6A%u@&+F?=-wTGr=G?3l)l*fdMi;qVa6tqH~ zWmHV#gvW@?xC$wSPYrMFTSSye|ACf{B%3A`ko~Mh*!RKGU>~`Vekm#YR&d z8+hFB{rq9hEa}@PPzlZCl~u z9~fY4;>XOLL%4bh=VlB%!4h`2!ThsRl|76x;9!^aGOGJ2R%GO*67Al`8?*L-Mp;#@ zugXJS`mN4!XWWAP8+OIzN15xHRmDpyq|*vRq}7Z3g*{8f-dy@YTT#kFb^~9nssg^Q z&`)2{IIoEL+F^_N!58t0+$|}X`umRGU}tClXLM5{{Q*@7NMGyz(G}hQ{oK2lFTOnZ zXg6?sS=j?$6r-y0XhuLW!@gp3cIyL8t;et<&m)gNe~_)J+S<@vpQ0iU;Dzm-y1sop zj}cfNL>IW)lZh6soh8~nHOkA&Cm6+rZXTgKs&gl!ChT{(4?2EK93NhZDOP;D8y>MF znO)Fh*e`B;IQ%uY=dD=H%ioolWB#;(|<6|Bd{lXA+ zrI);wOG2!?;`+tW=V=Qr7RA*zW&W5G_?UZ>$3x{|wEPDTJ5&dERm74+c28!Ut8Oi% zgi{Pm#aR#NHlC8n6PKkS=X z+TxbHwt0zhgG^Y5NTb+e5owY!?2NkOdM@**!@Uhs}78sQ@W!rU)F^G z{^DSkDC7a;U_kZ|0dK(}FEYc93A_YZ;4cd;)fz3Om;+=uy{egpGN|=*jM*#(X)_V} zfsi0M7c@{TSx{krgiUdIStNioENC?1310}s1)Sa4tpqRwBhIehhZJ zsyFIT92KnWFbz)izT@>8mWx|+OO09GId^oQ-$v{L%<@qvY4E)@X2n4c2@al%R=&LU zcJC2km`4>q>g>6p*q*pvJ|RWV%Lss9did_JxUJJAk_)!vVb$t_}pA?J6CCyU6PYSV?%9N4NY)>}{d zGfG!O?Uv(nE}Tab_A8kPvk*vWGNb|K{&e0O1zcX0)sAIhZjJ~b+)O? ze(L#bC9;8LjjKDN!%l=QsoM8R$DLl{reoMlR>EJc=wCdHUp_U*%@?l>2$H!WrE#JQ zvP;FiiT6$l`xEf9(&K+@Zwlj{L-G6UsV(L}JB@EHYWr^S4tdF>puiQM>L=CZfeR<- zEUm&AT4C_-08rnL=uyIeZdYaZ1U!a@o<0$yB_Br^3H%bc3^@}W9d=YFC(M9S`*RJH z&Kn`(!WD$hT_B{o+@Qli+sMfN^rWCA{E`8w4VpiGq@KGWNK69J4HgdfQhc1j!7l7j z8_cnU)B8ZnD>$zXyn*>1>Tvh;0fmA&Hg*$4qck1Rj8!|VbAK6QM<<{!iU?cUs4YBe zO^{UsOZ4zbWbJWua}6XBrRbjCNyG2C4A9kJ)y~+ck64fs+PUCp=Ynrcd6bVwyi3=<4g>!1F%vk zGp~{5h`O_@pl}REe3@7N^;dlI-0QsTs;fQbeF*4 z=rb*Y9u`AO61{zjHy{Qj1y$)Y7&!7L7@6qi?)noZ+n?Zj1I6JT#zo75s@+XIB}Dw(x%ZSV zJr>W}`WAhFNr*zM@TegV(1)9fWgx3}-mN5W(&<$r(X| z;2lC-yvGh5>W0B}DiqthU}$6a)b?e`fZH8=ANs(c&kRs-e~DjxeU-DP0lh!qg@)E! zFeuh;q}l~)hpv7gPS;l2_?eoZ^#*C4p#a9i@R5#yVkkB<5FHdKEm|_;cdUScyZ;{u zU-}=cMp!NhvcwnHjtM?*M@e6gHxbd43;FPA;@O`02>j=SZ91CdC4#?x86R&leqD;@s5Ol(z6W4tqz-e0uYajoUq&0 z__+ENh1bVQ4Ta1n?pdm+$MJF)$7NTvZl%fWuT^GO)XDbnT^L5M$L$*D^l2;?Ee=nv zEqvHnITmzWa^iDyz_Ozjdp~Qa)||@#Cx9`>vAUYxs!i{!hV{M`#q}FQd@B1v*Btfg zgblQgIwmQ4FsbBNz-Dck&|QV+gQX7{`QK}3YqRC{av?g#xin{R8-^s}07I{1kHgHM zY0uh6;24&J=2(s#S+kz%cW(Onr#o4Jp)qBHrfOi8|5u3N{?|3f2Ts)_<1&hl*5tX3 zQHQ^<@dW!xhn zYZBWlK|9=&ws-E}lLry2|Cj{2`m256`=GqK`1k)dE-LX`7Y_v7yQ9AL<4ML1*6R7N z9-4fc`Gk4@WTaU^l2hXCFN?L~DcR~$Ts=B^^#$+W+wK3Y$FZuYZ+lBFHmbWOnCcj- z=bW|l@Slo|LKmQ&!F@s71YLE>LiP>1DYJReS1*D_I6CV|f&g#5UyfA}ht}D%FRh%l zj3#I;Aa9+|s6_=>gkc*YV=z0XnutfqsKw@5CKd9^O>3t(jPRskg!7 zHyw4`Q9W#cHwcr~vO2RZkz+fYbZ~uzUn;s2uV<2N1XIQhN68~Phf71sB~ZK~#&XS^ zwsVi!rU@4Yx>MMhe=cIw7DFt!znE|Qg9;_wZXE?C(@E$^fpvO-_2i@WikBlQ%*h3{a|eXu1e~T)yk33x~|T z1d327!I}PN7{}>opDs~EIt4j@$Be+BM;(N}gT@02oAp-a@BTQ;QX%QhBXh@qP}ER- z>cY~sHH#|wq@a5MD&L~Ewl@0}16ySHgE^IZ4WhR|X)6G`39dlg9#575e|}o zn>(~bvwt?(BYs~BC_TQ2#$SgYewM-g$H&|C(ro)q=7^Ko>mAZKf1j$Ts*UIHE5Y8i znIxRu;)wOs^3Q44t+g0bAx^N6xZHaMBzQ=G~jId>Zbu_#>s9kxzPqy;Pk& zl2Vq=MB(~>)YVPF;s`>t2CtJed8NC6yQue>v@^BU@Ydo9-Udgz7)QCnvxfe|r&2aQ zwmf?yU!Hg#$^cZ&=9!ulQ_GSo7ZnT8?)JT5Vo+}v>N*dbIry!+|k9Tq&c@gQp6vd zK5-U9D52-}aF;C`?ySD^9Pm;7&fDhbtZ#D190LexGyXZ#I6M8j2}q$qosy81P7@s` zbXDXF73m`?ATLuw*|5&hi})j7@?B78jc2l~?1JDcmoqnU$RdOI@7B|;aTu87Y}0p) zD#d*^MQi=jc?ot!81lsj%MHbzMLLtuU~(UagxEp!4B0P^pl_#SpCyU=4ZqCkMP79| zz;Pu(p9m-$k31%W1@901w>ymjV}_inc#yVfL`W=gyK*K3egn8S6PU4KgH)fTz~6KL zAr4;2RieD2(l6Yd0rCZfI)m{AIShhUzk)qGrS9b7;_`oaV2uVbR=|3J+Z3}{zn)pU zu_JT|XzhA^r5P&A)y~fLP*RJ~z8PW?Mn|ospHQK^fps)0-B^^&j^VVY{aqnHUn5aW zP=|)Iax@cN5A5kcv5Ea@WmVPw$A{WCP4G{8dJKyUw9+h^3r_v6AQ#$Xv+-RhoRKzY zfFxB^Sg3}_vFe$dSG2a7K$lwAlM|(eb>7t(nT@qmQpZ>>;s%O7ihC9+>7s8mhJ3VZ ziJpM3MjTCM5&PEllJ^ETU$KsAxcs97H-KNiwmN$}s-LlkeXz=jSJTPq704|bKv6v0 z-zXHc^MY@%iY?`eMo~D^5is4M-(1dh5J4X}9?_kps0i(`w_eL@sSj45uSCJ<&*n0i!{c#0cZmo1o#j#PI+1A;uj6++qw{EzE zuav15ENG`(RV&GubW}RSh+doLI!t&tnZ(wwTkSp-b587;{glVnsTdnAU)=(PWNF#) z3wstLERHF=9>-}vy}4zn{&e7zf$cNZXF?J~2Ft$w;wy@!tw+#0Mibg9vROS%>^AzG zLDE|RfwZiyc~c6(+{*UT`K#Plxm7d}p6IM1*ZIj>Hm_y#P_^&;TBwRyit_THdqb#$#a%KCbJfJaLX~mt(K%o$qI1;M z34N<|wPx{u$6=ZOMdABn>9w3%sSS~-3AYAmHkSB%n^{q};B?-(^Iq6NzmSgaOwQxS z#YTMYv?7p72Auh=^{q-GDk?ZMG5!Hkj zS#F)oLz6UD{ghP0jqC?b!PXch9bJsD&8^s(&x*aVT#q{JuyA58NRYDNfyIrp{;sB- zASIdI#=-55yPG(c!^ug~)vH}^J(U=u+O*>#`m1h0tWG#d%(zq0i z9X}#m{wU_??kdW}u%4L5Hvj%e@A_7!dY^g_rlEbf#{YrmxV)Z$Pq(pcOu}h#Sze z?DNCVv^9)*B74z5G_@wAW;{M?EvkW=T}R*cda6dNoj))U+c~LsU8OoYq}dKI&5dTS z?b3WcuZ1FMhnH-@H}yT%C0C*Rma~pqRS6YAbq(MaBCl~}eyjNLvPJx-i+MZ&%Wh=q zcx5p-B=iFQ4WvImZ{;ZNXLb+Q0mM~`)p#$iPgd-_s^^x_WmhDA%^yu%edWUI#2RzE z*K+*T=-5xmjGwn^jyZ~*ReYnL#h@2{D6oC>!Q?$_kM%5#s^MIdTBX<+t(byqJ%AL)~ z)+f@_&%z1sdCriZC~!6(w^%C~!CKM-!T;T6GsOPV(sk60a8^{F%j8Ln*9px$Ye)n2%Ny238iZl){(S3K>Ezx_uB&w6tl>V!-;@iFrg>K%xpaK4h0_b~g&odFPM-05h#Z8)+FX37baJm% zNyp2;z~j<&a|;~%72VC;OqJkRUMXffFKsqo3uz{AQ~` zAW{mW;LHGuD@Iv ziMVn;-lnhco#WW7d?%szp!%Zb4Z-H8bNiC`c81*BYxv4W#~-mxJ6*}y+^I2X-b}UI zz)xtTCk77MnEdRYe`YHmRDX~!NF#z~o!rV-?Uu+_XOFRKP4Qh#Zy&1XRyVD}wm4pX zqv_tio0R)*@}xv;JvxqFciu~{?o{D0sXAE~gpd;HAEn0}uCjdJ>um_p@X(P^d@E~5 zoTMckNa+v~}EyB!Xy`a)3O*4kYiA_q2X6n5Yy z9B;Vl_!*82q|z1bM8>RZt57wr5DGUv|G=u2<(Y4p(jdwX*Gtz zVUujsfMl2S+2ro`C%V1)7=jNAx?*j$phb|aKL}EE1TL}-kFN=TZ12o^{Pg* znE_j8(CAKe%tGCExskKU34dd{U%Cf6k+CXpE%P|?q1vOLUG6nK*Cwx*%;J;XLMm%^ zJ5q5T(>1mh&h;J&*q!y}q4N31+dDIUo=r_5_&04g7Z9pL%*}AuWVpXX`EQKUh3O~} z_)FnU8=vcFiH-K5;Z`^68;uijmP&UHbgK|?=CfO==%t_BU5-?aO*lRHegBN(e16N* zPRaO86xrbsz!~RXH^Z&d7{+P4iMzk^R5wmDGs1V*Zx-VwBbPLMYFw_F0I^s#HYAWg z{dx8yLcth0r;HRYO~9<$>$^m~N(jtOF0~q&-!WS=hlq<_)?De#F+YYfvzr%6f<|+o zj304GJu7!-t<5YI%^zT^6h)VM`Sjo`|VB1tsJANIH+AT!A7#?v^E7%=tSWS6_ zAuTc-KJFagYNW1~Gk+{4Wi4Q6P;Gy>SkLlhr`!c;mlu<_v&I6%Vr zr}p9Jnk;=#V zLhiEBR>opCSDjCfF9u+@NRf|dGCHvVlq%zbkv-HTrVYE9M1Q*ZA=KxkA8d0SS5_3G z)R>Eo4$#m?-g5VEn>`a+l`7B|!*5e7<^geBi%$iI;arG}$&Ju0n3JjLY4RMk&#DCz z{JXDMc@@s`o_(3dvWE zT+VN7zoVpNx1KcaU%0#DRiY#GXk z+8iV>G&NI%XpQRKyLpTCUJY=C#SR~`-F3S+-FhCgIkrB8si+<-X@)@}t0+_+p$hr+ zpk2M)GmfgV4^(cX$Uvvw#3$uuR_6x$(>w=)izK>Hh%!7U&Pbwee#MfuLyoxu4!RZK82_ zd>C36%eKgSV8;iK(RYJC6`@8V4t7fQBns~b<;`BAd{;H;ut&RaN#z%_~gt`jA2=h%<7C}W^StC&;;49 zJ`mu6A8`}wRe=Fcv0qD_&=2)lACt53aC+cF&4BSfiA8UDzW9>C5)yfjz(PdsLp=BC6n=VQ#ROkiZk2AfW`0jFq7AN0A3j4 zf?kECGJgD#_HMv7X}ng*L0_fTx5aY+_TM}O+*+|x)pZcTu?Ja43gUQH*FgUgBI%F1 z_j(5``PoBrL(pgkcDG*nxKwrZC^AzY3yG+Qc}Ots2_B z-$VeD^#nC+6!Vc59jF(to4rMVn*C0_)&pJN<*oWX1S7pPGAlEV8hce;i=Sn1f@J#? zH`!YzZHxv{bEqP~)HHi%EmwUF7+Fmln>NYhV`nblrsJ`Ndk-dV^hotCPxb?9*To0Z zyROe9p)?#Eh9zaySH1l)%T1zCbn~+MDJw0k#g)|M!f{=PPvf-NAd$(BvhZl$QdCcl zr>E_odw;~ND^)VraKvi3@(yf)a|77Yfx}^?>PJaVk-%mY6}r#oB#reAJG0ul4Hs}% zr3Ojqi5TV0e74>CIZX3EoN(KH1FZ;E4f2{R!}wj!?vvoq_KcHf9nG&@^ey?Q=#oX#4KWPL)$ED^FJmIBciRzy?0OO=mVvV0)u3ra$tga4P;&?Ehrj? zorC1jdaV6HyM6{`Vd>*Ov)YoR?+sr!cy!Nn)9O8U(hFH}%{BGOHa`bDbJU_Mm}yeQ z@j!Bz_|CgRtX{FGv)9Y#tNU@I{>S?`%uHe>Q&4<{8|equX;kvLXe2`&v)XI z>wPRRCAF}z1ihG`(#z!W@$PuX71ylyw_r|vo=KH`-6+0u&{w#lKjlLcfQ1Fg2YaR-d*FyJBv=aRdKxGYU(O+eTyyS z1do}{F)bXiv%VB~5y_o(FP-kzrT2|SVQW=MW;pAhGKH9zSylE}CbLi7M#mjrc1}8b z)ZUj)v+z^$a8S>m85d|UA$4rM{`2RyIx}+#+%j7nVF+Iz{RT0<*@ETzuxf$h_YJl?j#P~hLzSm0n1^| zXbgs6@qTeMo7f#G)kQ6Om;3zWce*a0?PU|pYRBf(`T%x=Q%$9(H4X>j&y_|`v#r)z z!>UDni47a*@z!M)yVD!Z&cbWhx3LLk(@fZC%S>L&l_&W__9o7&!wzrkP~SJ)kt1ji z^1AaJ>uz4J9yXC((Rb!mJ%&&jw^Dwut{A<#&@8=k3)~w-9G1^X@KNV1F2$<>4ZMIC zdRRx#Z|z6ppzmVMnfe`Kh4@n4KoC~6(oU!0C9j_eEp4%<B7A%?2jG0Y6K3w7GQ=`>+wM@Ng zOuVM$(uRYHA3>k|-I`S~+H>(@cd-e3;^pjsrDn|VkBLUwlPAw%u{#Oiq|=6Zz?UqV z5|WS3rGPUPrW`IQB^B9^NB#yAjgWzUc95Z=$4U~s)bBRga!c?1iL_%yB_-J-qsn@E zhrLpuu|vipTw1hT|KhVS{1CP~a~5WbEWF(_xt{1Y+aXLAcK|v93qzJDo%J2kUFYGS z(k;|CKMe!Yaj8PB3J7`&r===7uG491}Eg(l>al>oaf$(^sn70duZNQ9@fEvgvuKosgO_8xRE&u}#mtb811Yq-?Jn7gpg>n6wv#BI$W>UHOOE!=q z7JzmW4%+!6JL`JO{My1?H@D-Eb{fZdv_$%~qTj%oI5x|>**2fI z)4}4zEG*3zzB>Q30A1@qFh8d;7)*Ni77>5dD_g@J!Pc<_0?0le!%)X}isQ2o`e3EC8@ap_6S~aeQ(emZ%v}%EQKGE=_f+CdF~<0T z0NwO-0zP{ilJx*jNuQZs_3d}JO#EF!LXJMw@Tf3CtbrA{V+U9_5#YSq$(o+6EYGlU zf`NhV>a0XyonePMDHdoI9`@`lminhpqg0!SD`L7{$td@AA0M~v!1QuD_+${^N1gt06`Ihrl?Rl3jC*ji>QAEDb;2j=-ev#gw4 z`Q4U#yvA2lY;C(&e8HJpH~^|76dP$DS%!1h`|!}&^y_tcy$JgSLN@Z6(;O9hrFq#} zcKTJwv2enzv4J|sr$|MzLSo|6MG5oh=$VQcf>sLcM+=Dai%=B_$`3cM(FTS?RmpC1 z$Is67TS-v8B5aiQRMZ3uXmRzOSP0a523 zKmJ`u$0rs_aELkfvmMcD<9dD|amyo6FFFfod3T7#t)HG~>BJc!Z=rL8kG`oh*<$^B z842|4Vp#H*!9g-8x2bN5RM&&Aq*o;vNrzpiB6o>_=0rzqG_+AYh z^&D0Lc|`xw!K$n2=z`Xw;3KFbflJY`2{yg!IBUDs*t@}x^~=F#gk&Lh_VpMBPXc$p z&n#KpsM5b-@Q`@>d_;>-y@sAD&`5%u70Q~jvLw1LH3b0{CUsWn;Ym1hc3Ue%+*i&Uu+MUh-c1zM;Agua>)tqD$&)z-a+>3v{TkgXSA> zs(zxQ;R8(yjP=JZPDy5EHaR4%*b%pbLcfV|P zfJ5aT3KFLy&q({O97mDN9 zKOhcho6pJbfU9kD$++n^Foph^CKo;BL0ZS|R;g9(u11TjVmCVpzPlP0jWyR8To}C; z1=x>;>s0erIHqC|lX2pK`^-V10B} z0$b9{KzplrLai|L4S<*X$iN}FJ3_u4Uv-_e^rl7nqDm(Sn37LdCOv~mZH8aoY>Gu0 z<+R37cg8n0OR4iTPjoCSDrPo(s*D65jK_ZQpY!-_2Qm?t zgji>e4Sya@z^y;(2VG(CaVNPrBEQzkjfTp;*`hJf3 zW+eu@;u^-{{U?l(rdB;dyaUkGccn!V)($O1Mj9A!L4n^5WXw+e2fw9hxT;Ky`vdVt zOZH$^;N~vq=B+Ax(MjLcm3|2ZVCeKc!H=K)l7GBpWQ1rD**MX|9)A&6po63E$#_3W*DP6B z5$vg>T@CK10t8FcxE*YBio6BS4MAod-%I)c-o9kxxOqVOh?x7Q6KB*FpK3ZBX(oR9 zloRgct0k9JY?pcqO1#w$KeZdyLc^yov)vROOUVEH`8t3S$MHbTw)iEz{#-04u8^zi z27P@Vh&K8Coqu55#tE>?Ti(4>lJx){1VX-Ahb*g3H7z^4@{2EtxV9bgGyn;OVJXLx zz04fyY6GBhz1#eZ1P860qYITJ{FB$O1)!Nnd;fmwb_E!!EM%pEZ|1n21`R;J=IRv% zph^Ixjt>%=I>$h;oZckSo|Kj2DKD=6@-y%vWv2qc?>IDv-d~JI@G@}SqhOEm(U zv==5+3G_j`X0p-RUP_a*NRsN`T^gq^P`WgbCN)KYuJ0o?TJTo|pCg?WrKKjK^gw^D znSn`N9ESLtKkw|h-@p%Td9ajH@HvczUKp}3iTl_?e7FK1#sMytUZ8UnU$b1jVhBwV z;WI)DIJc$xRV4noL$}X5$N{HL_G~QGf^EjVSJnA|-$jJ-@0RBu2p1J0u)FP&kE|6r z*OxT2472AqL{36=+(X|W(DHip-6JoB=)j#-YRpnYcyv~}oW;XQ7%K^HyP39$34LAr zlN0A*z;~X=YULDhf?y&{EQPi2ry^v?FI<6&y*Zs{GD^u=lmrT7$Q zpZdUnjWXwjFS+%DZ1W4~=DDRHY}!ZDLQeI8Iq(iv*2XJXCok$%yU!~~lGiL%i6Pr7 z@w1S$1GMaM1Dj!Az>E^Oj#ed3^V16rLS?AdkKNH3Z!FBpQY*m*D&R~+i`$RpnuSH5 z0{bNwefZIV*)bsv%#<#=Kf;oFzoyt@Kp#1fv)l@5kBz`5^K2 zy=qOv`SB~RRt0o{E^nGTn1=3{VHbd_X>fkD5pZi)T&k`@VR-CHo`GJ~Hlu5D-oUCf z`Ie~{8UOZ96y3O#j&g&LjydnzMMBmuHnJ?7VswEvqX=>Lx%%nBG_KG+(a&qEd zVvY0We$Mzg>C4Fpt+rlD^lS8{M)kRxwICWGBq?2XpSb9(irvBd`S?iPl_xWPca?jd zNjnO!%_qADVqTKJ8xJzBM`Nb&f@>>eR$D}cghx9}Dq4sUJ+?48P3^{oI8B(Aq}3H0 z!(Y+)J4LIpc+Ryrm*@g-vK6qWe!GpDxY1{SYP7?&gx3Y>;^))NFU4%E}JW7l0z3_QY*kkPx=JC%b92 z9D?}4P68c&pc;Ole>Y`A5XFGzOw}Kx88aY~C{&G#e0)8+*&u2(TuF;hv^VV^tIq`s%Rn)Cb7~55Z` zabb882tu$gIk!z0phjGXv%8b5osOR7UW!9iOR+4zHt6^7F(vsw$(uOQ+MVUd*#&y+ z72J>0{<9Kgz#85F8WH!9hL0uiic#Ck@ZSCOLXHYs)BNiFn+;XE(QBYUfk7~{uWycH zSa@S8NK~Z|Rxh7m^k`yxW1U)K99`~7(BL&#k-g_n7-EWxGB^HIDrxFFe}8-DJ~%{x z)zGt3$pm^}L=g95$m@zRagtTKdnvFg?_)q9X?TgrG&J5!N=aR^qP&HzA5og&oyrG+ z>Vl%Uf3++H-fWJ<5n9LkV{CoV#mV1?HKL-LP_`hcAw;H$t!)Y@X~W6YS{f5QxQ0q4 zP4cA)5FM%;OT4Q1yT0KtzR?S%qwFE#jt=B~sFnA2z*z1#^Nq@*5-$!>_fz%JaC8@)cLkZm9I~vlc-k?r}9t06v`;Lq^f_2-R)PhBZ(38y$B#HsJCoBi6%U zmPuOZWmUa?zqQcA)0c#^N2^Bb#gVAky?lZ=@@Uo`8bca!{B=CatWp*7JUhnYR}d{E z+svV>Diy0_&Z}~DWb@jCfvYc>3t^a<;pJ^a$JITN=sJ)V;J$;lz_Ue0%~V{Mab`|+>zA|2fx`szPk z7Mu0hy6f0{cR_GTK3Xl%S89m<`?&YjpG!aTZc#hAC)*OPj2MhwvK3?=+(Qj89PEtV z&|ZC(zR@{t`CVQZpe{5Uf~V=~+64LvPq2CS2H^3>(ixA{SS1wzcnc-N-9pAI^T`pBqxK#f>!LRSoJYs6jIYO<@y z+bU0Z`D6&c8c@>q6-GL8IpB8eO~3mP zHy1>^w+(8mOcDuCoiQuT$7c*Ww>aCu(B|0?smowp2c*fXWq>PC2awWqmNqotxt25_ zVcL6WW#LMAWowYm+FBug?Eq0I1tq3|$O_wuTLWjJt3Ol`@V^F?f|~fB2J##pDiNIH zx7yl{l>Bv+TMYr$L_5(&nB@V)j#`tJQ|Q*8&wb0I4Q;I&pZ5XT)n`!AD`Q{Fx!3$O zFf>U?+zu4reisS!nj=qT;^3n?=5y;_qTss)3*;xF{ljO64RE-@jZm7gFP-=eaFx$6 zgmlTmg{A{2(q$MddD)K?IslGOe|mCBxk(J4W`ZI_4{ioO4tfsA&4A;_QC2I;^?m#I z7KW8^n}UhKakxCJsj3U+mWHMq4amv)R-XXJi6>J1QLGu7dF_B%?N4JXULED~Un zFCRSc2Ej+Y9H2koE?x3 zEyJOo|0(h}!-t_EX@zot%jKq6r_Pb>B74mSWQq)KqnUMy1weCiscu+IyO{7~Z1b{t zzA|z3(Oc-*g!5%PtLx9AC@yV;_8eN$94`RdSN`&;x@oQkpT)z+&;x8b=~+AycEm9)My45Hta0~9}m zEd|5mWLK~ojO!UW+h&4dvu{9Kaj!lDSQHRk0|}dh_H!*aKssIfeiPSjpzy^Vio~j(1Zr38jEb3x@REn9E^CBkM54 zNEZ4BKvE}mmkAL#FzKte>hkH&F3hrRc(}@ev_JA6()(XQ>RT5@&+ zK*MV$$Ny%+~i4T!YHe&x!2Q<#TEPaG)jparxacSeKt6uqGh2sqCJmXM5~2O_ z#9e5miEA3u%mAQ{MGN>g1e}^cGKU7T_}ZIG1?r?%@Dx(P{$)UdCz3^A;^YL4UeEMm z=U=-A5NZ~D1DFwlpEMHdwKdZoxc0sDj8tcXIP}lR4kJD8KpmazHBGl1sX=RVvb7k8 z|1On|COHgbM(u9Y0yzRdH_sItCkglJyPz?C3>> zm(g0fT>Fywb;8gYE0sI7HGe$8rc}I8%ZFJsz)VS>2@V#c1nL$CO>hKZ^O0k_1MkBi zr@pQ6BYFL2%w;`Uk}n|A{N$whIE?okTHuy+@tPN=?$ebR05QKZJjt~jSx$860_!2m z8W#mTIbbZQ?0kM%hLtm?iKLybhPtEgM4{f&PtXBj0QD^Ql-6Tf0nKA7P|3 zLr@g?b1iT11ld#*e0)5BcvTiQcMJjBzDe*+YFD&JK=PjDTT%c9@n^6iM~%a_cD=Uk zRnfLVDAIe3LWey8t0(ry9`ACV^aiLvge)Ym+Aqcah$%XA)@pfOlQJ{6O_(1We(Vw& z5SHZf{h0E1B`MIlwXetPC3^Hh0@j}4V^Z>&+PnhkR?E9E-*B2)retDu=g2p1p89L zhOY`GtQRA!!-$vzgFSGUVtbd5y+8X#se`;%f5t(iqIEfXo;X!Y;dHcHdN)6fpUTCf zspWXX4mpG*O?<+D?6;b1SmBDBIbKe!oM?b0=M4=_aE!)suEEjD*yoQP5mEELCUw>i z`xs3UI4Or0I%v^Ig?v>0gvU4LYYMIy!S( zbZ_76_b7|H?r@S~%rn0M(2}dnJlLaccTd;{6_*BLf~e!cPP(*+I(|+;%*YM2z<~7< z#tp2?ai=8Yhjw`-rTbVIR4Yl~vSW=4U<`tfGgm%T?ciMn$oVk==~s6Yk}Ykk)iM`g zmlk>1jTnH)07&~-QjXjsTG~W;#7v2zL78m6eknV<#Tj(2creNL@uTDM3>O$dy0zli z;N$ z3Tv3rvHMn7$gvI<9e^4jAh)$<{Kt`0H3I1M@ZJJe)_Hl|_(wp+0(hV4{i`tP=r}q! z0pg=YJdhUrgLON+hX4Z3z*3`p2FOO_`ot+q*8MBGoBIFC2lR{40AOcG|KDk%n%adh zJGBHO;LG%{EH99Obx6YSfz~-4g(m>HK$oi-z>BeL;zvx%`Be^p$e+nP=^JvO?~A^y zInx{!Ka<7IstpTE1H!RJRbb1rIPz$xzX>evq$?_v_lKNOhSKcV1GPXuD~TA7*#Kql z^~pjWur2_Ky2gG)=87-0K9m^PF_0}bHcy+Va6y3Z ze--xOj267z!CHT`hrI<5p(Bw4P*hD$x<3eG1gkEE*^V%Qxhy~Fla09$8FO;-a(pi+ zz0OCH@@p#{V1_RY(|yK`;=WEF@7N9aq(wGc_-TSYE3=>%$#(m0E%SY&U&MJK<00n$ zhAA6%`$I>E-)DUBy8lwc*e7-gCqwR~wx&}<6q`WSU^gI02b*|c?BB=4*D!2|*~Ru7 z_}g^s`97gmJP1qzP>AvNk!HE8B-m(rDpw$|0J}{6LVr3qVW`UX*sH#ioh3g8Ca-A3 zy7HwM?3>pjQn)Hdvhf_GSMsD5OHkt+&2^czV|Lb>xWjU32e)YBpSD@m!L~o_(I$9! z$?r;56{^(dm04s zYNlL?sW@E@a0wLy$V%Pe6WWkJkd_*AQtgI__lnnbA<_Gg!&;2XqIV!I@waP>gdW^H zXKI&L2?wUX&#bIM!n)Jh6F~KHfHNb4cF76UBIT}o$(2`wp!L46;g=ye&QgT|oxtOk zlasxNb(1E72Nm4>>_Lj?^><5MW7v zPeUOu|MWhfi97pb^d?cPZ9(lWU^Wpt>q$Bv)16blFgf_o3T>rnhp8 zjzBXB*egyx846hEuTfqn{`G*??&u5cX0!wYBWmX1KTAab&mbKQQopXw49MUTaSi)X zkzF;D;j?UmjQ=V(I%)4SUUI?$k#cl}{`oV(T6#JkaZd4gRa^79qvMxr{<%1~<&zoM zgut>d;FG8CMrsbx>slJ`8A0X_XM4Z|eUmm2E|*jn~V}@)70X3c@ph`rw{veI`;|ZOUm8Zwoqx@Wy33yq4Ac| zr5g^zAEW3bXCQ!4kr1}0xM?(efGxVkYP(7e_B6m;g;(>vT;YDFjNuv@tyq8*4N+|~ zI}YDcz`j&GDj%0mG~w4|2WuJTNBGz5a7nc)BYVWkx z54q#=1<(Cc`Lna;-0oa;mZd^;l5E5VC8g%%rz*p*-@j8oUU>MVZ$0-J)%)jverzzD zv$XlZ`)N%}!6TAOpc>^b7Vu=^n7E)e(p%$Q!`cM{IZHLy*=FPfneEW#>?q_ z(G#`jQ1~m(u9csXY&;bmh(S%#r;E11p>lNiFAYf=ikbs7wE7EvZMA$cZqqMjTQ!Db%t*66}PqmqPW+WSu^RnKoRluiu;? zGLempb_BJEy0OWK{yAoTSoUQ?rX_e%c21D8hsStZc7l9;AyVyE5)A<*{d}Qz7Q;TUP&-#^H<%&_J}y7o zqzz^?1Vd+g3{c2lk=%+AqZ%6afJLU7Mt-=Ke%oJ6;q`5%eR4H~RB(MeN;fJH&-E(q3bdReiZg=R-RIsl8Rfp~kR8K%A z;rIcCp5`*MVMdC}1`A`i%Ik@>Ij?Vy#rQM^TJNu3`kR)t1-i@m_bpM&~?Po&=#2(Zy zoqX}sYMLg>g@n~}|CsC=dNvWhK5E?1r(PJ@T&_St*3mt+{*xWYFEu&N!hXrdlkLaH zqC$2~`Zp53^#tuL(GS%HVQ`y_;Bi60lbO>&k98kNV4ZCNIk`d>+-?8GdZ*FhEtkQt zE&K9Ok0WI2PiI3tyE(H(rP`qTNXMUHVNGlwWG0o)qvsOeV{)=Z98Cfom+CHHP}83^ zR!wNNdziOs#yg`qnB#frdww(~EP(kOFZ8rONKP@_ntAf6M=9{ofPkf3zmynKh zlyQXu$=&JUKuP!hJ?-JJ=+2O?lM{s!;ja?gYnm^Q)f-T)H)JQd`%(*`Trwen z;kmPBJOd5LX-LQZ+Rj(0TN9hjJ#jJJGffbiiMvkP(5o(}ew%dCUqmk~ThWgvc zel8iDy@$#t zStixwy92)2V)16f;_1XIV0DcAm=hMvr!n{x5--CcVubsv{er1k=a55DhAko3i)#g` zc#Z0GAgn!|##YLpKCiljmcO7$(pQ;nWiIVK#at^*biqwU5(28{Q}T~F(IK^71fR6f zto)bhu3fc^PZ|ML)%NFdk|?)j&=!x#dS5y;R-vVO^4N%LBf+8XPIK&9ak(3}2JXma zR9BMWQ>#rQd;+-JY%fbHSdRI@;G!_Mb3C7qw;~w{Fd?S8E~blQZ*)f)EzHBL2GIf* zLu_WEdas^+$X7IDWP?6&-qMNX;+N8|H%3@`m{F%U%G2Q^nKTOySVXm|m0TSpya@q^ zM`#GD*;KUSW`gNbGTQG)O%3zj3UM$!SSYn`d%ctuH5itC)A1-WGE!9LlK1OF_gb9C z%a?esv3wo`RwG`j-J+-ih&h{8K-*Y;?~DNwnTN@jZjaCwcT1{8l-cQ~goNs&8U_Ar z*@aNiis!f7@v~cld>1YpqPCze2%vAT(XgjT78OhLGubfSU4d>DS(#Qz6B+&}em}6_ zdN!E(M|Y~>bH)L?m(#%kk`Vzsu;JJskWq}@z6}kBG$<(tc8~~MWK7Hs-zhsOlOMl- zcVQtQ9LQ@rgx^?4d&_6#mK)$Vf+>KLyT!79M%3&325d7qqu0tGl8xPVYWdiq4M4TPaX2q(*rU!Bp5o(e02I=%|Fgo8U6RBw_s zJqHJ`e1*fPGp^dxQ%FsuV05j@2`;K?9vCS9`ST1s1T#>qXFLW+c*b^}%&ORJUL^zu z&&q|i(AE3mx<%VegSm<{ldR^$U(olu(U+%_N&{jL4R!SmuRQKmdSy7m-_x$+0DU?uM_-@S`RuTl-rlgXwz#=}t#R>z{zOlE zqT@GP0zY?cQw5jXT*EbWv~t+%JM+6A#D0yK>+DS7=ROsYVXR}w?Mv30X=ws)mTWj4 z(B$xtPSU?tA4^AAnD>7b2_;=&U?Y3l;*zbGjwMYlJiZ@U)xA=xjC>bD=ylM#a$)m3dGHhvS z^b-dPSk5)yEMY^I{Ni4SGQe46b*uNgmwKzW@rI3Dc5QCw1{ghrgpAz}oNMb_y^9a= z0M%Q?H zX4!9Zv)eTXwN~htS5>ul*Sv<8S!Q-@FM~1d@9gOSLbK#p8iBNt_H}i7YI-H~pndtP zch@7q^C#tP@J0LPo9{VYLA@_ExZ&2}(12ksDE>ijTISN8rcce2!1ba(aVht}I1~BRP3B|B)cLZhSSdtvcD~@Jq zD$DTUt))~RD;9PPUK@{SS7uU{$8k`rA&x}o0L1*l!xD+WipC~88_%L#SruZFZSc{C z=vS!R)w^)WLTyjjyIOW?jn>}UpM1IH&dizTWV!t~w@~dnUu(UoV3sbl3W|b5&Fh028Lck)3Z&Cen zoNjZfYFXa$3uwq=zOB8qs3`lQOVreibzE-JT1!PW&;3JHzCYjjF?M!vh-w}l68nJ* z^)J`fU~Sc8XEE0`E-o${a#sJ#-e#c&hOi>5Krc!?xTO(OcvfNsmekiX-@W^bqv`vP zACDPnXxYfg$z^@kzvQ}g$FhMBShb}*JApT{v_Amz4eDcno__x~c7x9=H7F1764f>5 zPqGAJ8<(gU%w;}CMHMN_d37J0o(vas<>1-x3e6Zl%vT5qfOs`q_hz$OJ@@&hfk>Z7QUHCek2bMvI$@Q)u28v0^6i|JcKtASK?tr8ujF3-(^(@X-F zjCL?ll29!o66uP6Pr!1!g_xSoyz9@%f~WYSvx1#mN`9&+YrT^HwI=IEbxH63!pc|A zw|9mNV*2O5@#lA0sb-lGI}g2gDxEei(b{;L^naiJ%-_)WPR+M`b~#q)nbCua4t2#I zlAxJSY(je!T>GOB28!bsbfIo3}6 zy71;I6#gq~a$&2-fcfrE=b8{?LVc<3ZH=ng>fGuQ0=tXVJAn%~XDc9H>WM6Rg`u%K zDDW*MoJQd=gSpkZ!HTa4gBudoJU5bhO=DSVDr*h&$1fHP`*{3{x>R4({9DB?YlFHu z!)<>|CENPQDs&`uZ+B_wioKL$1B&eWty>q^`|=g9PaL_sgV z6%ILq@%}f;gE#pVEncj$|Mz|NXMq`74&se4_l8PFp^)N+XdrV~!FaJX=rAXJ zjyO+ILgvlScNf98=Hr*y-&fj=JcV(q*xla&d&X6UJH{4k2Y#(jpFbU7*eWEIRA@TG zl_VG~t7owpD6yvx7Ao#=9{vE{J99-nd3OsXTmm9jCpTOU;i?;E`t~j#1TLWBQ^nrq zJU0!WCC8!DJ^WE@nTpRNK+{=wcRkNd@~87gXwZ5lujM)Sj)b9?IGMjij;E6*@#9)<@kHLhg#B@P|OhVlvL@36UVV(mU^ zALAnnOa}8?dm*#@W=bOk=(df#824ipzI~Cb9Em23b^)Z0a?ddK0qBT}B9hNOKYH`~ zQrmR+tRiO`m1-N1vRkNlVDj8&EwnZEr}e|gLM-ZD3#HK#B;@vj~$S_Sij z3NiR*G{v>AZ+fl~=*fo0#@=$c9e0zIAarQVcg^I`(V zgn)AC8QKwMLEP^Gk4@cJTH39>J(2&WLJq}1@5;fO3KJK%Z8)OYoZ8~ZuHVIp3}gIw zZ54{gv&SX9x|9wU7x{nNgkg7@J3-|!T|Z~-Hapa|KVOVCkjQ}99#D3+=c^ZoOYkSLVcT-t zT*S3pJ;jm^fY_Acr;ZJN!OX{1zqKhIKIt}J{1gfL;gOrig&t)65ORJ%9*;vkS+ z;kKFe=>}dofB#WjQu}>;Tk8c&iN$rlz!(99#P5YeJ9E~GHr)?`W)m;GZB85N{q)-C z$p$!!I@jB_RB;aFJrZpD{Dnl9I0#WfO8zGb_putc*)h@Msm!(&{Q8E*Q=Qe}=7S2M zpT}WbYC#1AeBw>(hj5p9BN?&lFM_!QC9Dygi@_KeE+3Bl%|SS+Q6Bh}fvKH@&HKmyr%liwB$Wa) z*#BPV)5Vf*y@j>9o0NgQt|;(W^_|%p9PILee>XuKAcU^}yeyP_I9ig;Z*ZXQX{Ga- z>-MaT{Qfd#Bj(#ZET_0tvYYpik1mABO_zPc;NXpUaze;T1_ z7~4r`n|wENVodfkrD6Az6hBn~nhzS=%Vze3Vvcac|AOW2BnkGt!vhC^ffmu9wZ!LK z`R{(2(c>wqpj@L|Sp$>bD~mPV zh+60093EOEz=gXD5=izTWy3{PKA5=Q0|y%+UceEku70h``hzmW4gaBiK7glsSTF)5 zTro-sFEBuawuPOBYGvPJ13NIo*7&)Mw5 zezzYMJ@I#f37fZ^%mV_b=lpM8c56wbAiFB}JfbT`&jaCNE#CU!{j*Lg??c$|#00MP z;bUO49ihcvo85LNO-oQ%*jo1m4~2Zzwrqt#%qiU?H8m+F**UGo>T2C|75ks=EpI*i z(CHvRM-{mNS12YXrc1|n?%Zo*i3I=jJKLaibA*+JWdi^9li>!?ni-0Zic$$|#>~*2 zMQb-fQBjt2m%(Tq&TuS;Rwo#Ok0&pS%LQbejE6p>Aav+_f1_nvy7Rr<`R$ZHY~_W_dtRLH zU?I)+(<}w$BL2qx8p@Fl6)$G)MSypD z%#G_k*a^}6pBM{fGdcSkt~vkreW~2Dsr)K}x}mG2(v3BFdt!`NtK_O6g6e8xbn@b7 zq1UcG`|lbUSp%DMb26P6yw z`s(Wkz?Bd>G<)$vC3Q!v)tiih&U@~W=S-vE$%+gsPJUv`>U>Miw-VpUxel?9jP+ZM zx(k;^iiQG7{;xvwUwNCdZXB)(N}u$lgmQ5fcekFtjE(wsWf)`cFtkukm?$`|gY;pS zsMyYRym%(a{A`kMvO#8TIFw}MxMpijxW)YJu+Q?&_S+1MpJki{_yQ!dl z0#^3^RwF5G>Qi6PLE2hxD5W<)Wd==FrRqbdd)t`8Mq$a%If4AUUm7e%&kzCtCRK!3 zMOx&nmfy^X@o>hxBvX0xf-L34ss=tGq7m{E$huZmLG)^^uVk$s(Q4hX-xYGFd+H;rPL*80 z2E$0LqsaH~1nC*_-x-T&bvNXQ$tqlsR^a_eK|$bWGcP}g*b3$eur9a;7PxOHC}`~O zvc!-BbkV5(@%w^@S#F3oWYi}_k!O=!`LPWQM$fP7zD4#h zF6i-xK8Or_lk)xjcpx{`9!5%vX(PIb4P2jE#-D zrd%j|F=9J+9gA#pjm$GBrDtXR@R15(3KT%E78e(fkgJbfUh|e$1#VPNm?-J#bx)W6 zougC7bltf=kenz5HfhN>fir0|&yBNi7zCt24F~^sj?T`(LB5%rd;O3=SV_OV zTMpcOfP5I$@sFn&-CphBM2!gmOT-JCKw2W2CoeCzx|$xye6j8WnSL7JW61DtrE^}_ z%!~{8%aMmL@@stp6Z-Wy0K!@UFY$2L{Je!}&}RLoK7t|aItZkN_aC^_XLDl88&MT3 z78bcVKCv1F_ib$8)34|9;ndb_GOnDUV1>J{#S;(yxrflOa^#brU}96cSJrbHarrV9 zF=I!V)E6TEe6J{NGM2!G(Fk+CyI$Z1_v=WFVa%fF^1` zkP#?fTn)tuo0M1i%~g*N|NR^Lb7M+V*1b_Xc%JxXWFh4BP=G;3QHLv{^j;Z3@+%vT zA0IJ*ai3gyEH*Z@#~eWs#(I55Ig@B6LMtDd1wj zTt`mRjTDKaRwO_H71c=3eyAX~0#Ch*dM|9)WLhcz$OvnM6bj{7RU2FIEPKh$!a9bM zk5i?eJJK@vAm}mmfOX*`g@DzI07;0mTuhci-=|c=^R-h3vI90X^I3lJ7T7tejY$u$ z)T5XX+M?X+VVzy%IN2YtqJ(AIvp5@bq-j3TrJo%%Ry1 z?~MfkWwrzzB{@1WS@&G3fPnF@;(3ykQx@V2KrHxW&s^}H{RQ&qm^@_=WnElAl;v^^ zP6os5q9T4bYgO@?#d=37giR(%2};1V!p+y#>XG zF0--GhvW1OAmCH{&XFy7vVrG*UddZzWO{No-lnI^;bOo1+3VM0_;4UecCuu=iA|&q zQ;Rw)M<;wgX}muxQ9(T`nzFtQZNsXA7NuU%Iaun4$e`MK=T-Z~@au-&A&Szip_feN z{&+#RqB^gPF(F)OZulbJ^-xex4?vzi2#|cd=TyEaLH+y+3o4XoDWQ6`_Bwe0M*BQj z$;Y}PMZ~)z%e;ImjJ?caTiZ67<-Udl(tVMG3jqmI4E2n$i_nT9PU7C-y2zllbY#sw^^72LI&q+5G1 zkAKH@MsSgwxn2o|Bcs_eJ#T%{?htYqB>(*sQ=T2L6$PbqjTzRTreV~Z{-5{%9Vfi}PW ze(5;Xw54UXnZOg`Lv8JNfOM87W@U*y6$b%T>Kk~zJRl4`l$xF{^i*7m*Lk{*Qlns# z-6BmZ^81t2gG2T}aK6~nfjTw$=FguhvFvvrr&~l7^-NDsS2>Og8F7BbRzkPrFHlhU z)q?7D_$NJ=OR2JzxzpdQZjDw1d!{a6fUxt*%zlZ6o12f-URSR=ElcUOIrHr`oN#Ij zAsd7YH+gOKXsA=d+nnGcQE^;XkBkng3%|c#INX8q#s09{a0)ZfuWz@i-e125X5t+d z)%O>u4*==F7Wxe`x% zX|oMyz8Wc^?oO9z(|emU37e|-4kqj67sLcB3I#lw=Fy@U$2uNi~V zSy9b%szB<*+FHM~$e%2?dYU+?61P1W(1%>Ho|MJHZ>&?kexqE`>dREo3fY^E}D#sAV*2ZN9{>hu+!xooIYkz_x7v3U1#$$eHq5;kBA=UJhY_zPPx{*$3QMa^29& z*?%p`7ViC!sb2%|Vq9!p*K6^wx+aLoqN1O#%Dn_(VrMY-f3M>tk_YKxhON`_vzO%!2^ z1MM(L3`a?C^R{<*IJ+_23;bmg-2|I2wLxaHOY&1$cQ>}Z9a`wPJJAD1NcZzTQHPqp zocaiX3|M7Qy{BoIg-mS9_x&+0OYrg(lJHzYOdRB^-|{Eo)ZLTMEpPIUe_dc#Md;iT z)np<&(T7xPv}*2q%9%6GBhW7D$LD{&XhbW=&@1qm?#q+qHbH1Vz!411wPGCcn4Glp zO*1|WY1(Q$LS&usL%Ba1ho=wpqCruM7@C3<3p#^opT@(F#`yQu0T|~nzwzRWYP8fo z&`edkPKq|tH9I%gJKqtH|UHkBJ>4N-|hc+Q#O$VXzN;q2mbu9FX^XHyYU(`8yj;(eIl|g#m-_ z@Tk{P%C`cVpWoHbpqTWT;UjlwBRw@saJ*h&9od_+8G;tSBv^$0h*-vN&Bn|^iRGKC z_L5sogj1#i=ijE(uE#c{}3E&Wd1-eY_vFF8rQbZBh!l?sf>R`%4j z?*$-Ny)`g?T0fy)kIZ4V*tRw%Y#UOU`WwxX7u_tSpCClnY?lA{-cvyQ5K47%tAG}Q zTnj4L?v{XFvN!Q`j6evlqeTDx;dY1h1YS!|j{)tc1&bC(a|qS^sLU{2Zw9W7LFGK?-(UlajlvWm^OLt}#qo7d`;j&C^c8NJVNyC&i|6+zME zax(}>_+9>bEPPO%n%eNw8PZrDG=a348vp3%x}~D3s(44TwHCWS0VbRIJu7HL_eJuf zn-ar5SqZ=d^d1<(3?APL!W&qukNLNphQjLSK0h!12?&eiWT`kZ!cCB6elJ}aOYt5+ z2~%Oz#Y{pTQ^!oD;8~IvVHLB_bYG6mhdz_BE_nW{g)1KZ^(o63Kq7(z-$hm1sy^*r z9jgd49G;`Xpb|$H6O|-~Hv>&LyoVaD&cw%X6sr=C21jd9iwX4E&p~NA*e2E$1n3yG zXw0XrlclSF`6Vj?otyh!_uEoyV;fu|UN*$4evKNuOJI+lrHrOhbALB&jIG{K%R;;s z6=ao=me)TFbt`ctzanxh7u;KYEmxzUdv}@52zKv07<3y|&4<}5n^W%;B*dfMv#wBg z+u3SZ^rZ}(HsvUwGXevN{r`*pjf~jZ*UQ^WZDDr5B;Tc8zi(^fZx92|7DNAdNrzL% zV=I4s=4(}uk!AXSXdgsP?AG&tZ?UucLGAp=J4y)eTk75~R#BQiF1%yCi+_FpyV#_g zu!?C7?@)j>&bnttz(W>F7My#Prm~i(zDvLxKEA4HsQ9#uDe05@DG^nkpiDZH^==-p z$!tI>Id2XQ#J4Z?p0Ay^2yAxeSX$PPd3CUT#9Om9arev7ohi5>6*WfJ!1js3%SQ38 zW&eowSa8%`!$f(npU>(EA|Ul&)SX*RzhpNEYFkRSUl(c)_&stkWJ447Pwc5^^rDSz zPuG(Uk1L_&E?{bdZ-G2|;aO4uib9WtYuR_$s`d@_)cEd?$DGVW`^`wlEkd4}ByD$Z zzD=({{6GcspveEIgy~`~lD&SDlj|~+F(Ycd0c!RS4fA74(52Zm%EnDB^*kC$uyWNe zyf-Pv)vckhk(wv*_R^(#=5;}Zm07ju4Ne#@+5FuGJbB1mC1DnW^ z3lG#3AwhW8%zFXPhf-5gBAdL>kN+K!;`_f^mL^*RaxxCCo}Xe;zy1v&0ielRrMcFd)6 zaTUScdlWs{27CIHE8i*hK)?&lPdROMfPvL`SXUQ$u*9f~`;L4w5@f1yD*z5pw|{6w z*F%t@giB=evwnN!As~^S8y9AuTv9 z?8-sI;TELBvLzIuYUeG{oHT6lYlnNViHj9bt|#f)od{zvlAih0xTXJeoxf_x(w{#D zQc{;N1-FepIfn%A+l!QxnzVA|>r4$EbxxeJ|Mt3V{#PehQOu9m_gZaetNV;PYZv}d zE<{cyOHx2noZGm*qzaDVT*W^5zcnRK{@VduGV|Fj@b4uh_@8m&dDsPC>Ks9tZ}q zvxWiZ@!|!UMm!DP(fNFuSW-yHG6?H0xrLeT%y=a<5~n##`F;f`X!=YZ+Fx*RrnF%u z$K_AV4!f$DIV9C{B{D5qc^5u#(q8CO-mGDfdPYt_RfLJ$&rjmjlh|O?jrcWwNik<= z^J8_#yTzF^7Kp|EjEyR^<=G76ma?vZ{X<^gS?PcB9;-r|D$K^3|H&;jkd+qyi;G)` zyT}Nz9AWLPTMY-aISd~ZNFEX=`l0zb@Z^IZbYE8BQS6wnv6)f^hsS+3cfQ&7P<8v& zL3P5pN)9Sn2C559B})dg2}8yR^eb;i!AKon<%CtS4aPjDL1~|(){N#WiYxu!hHLJa zcyv=Tb~1Y%k_ff=-BW{JASbCgIKW+&M53c_OhPs_Y;32M3V@Cb_x6mJ+^lFV;EhSi zy<=h(*dlbUjc!p_m%dQZiggL;Nl;KWARVF-!gaCk2|Md9o#=f96YG6j(MiL&ZVYgEs}o7YwR_x#N(5Uf^K z;oysUT?pxkANP@LMpIF9T-fv=R*~tdsE}D%Ta$;x%j{ieU-y=`{O4M|v4}1&xVMO@ z8U^%q^~UAA9?wn<_4!Uhxmab~2TB2YPfu$Lm|fw%&z@ajU}OYA4}_YFN!{HVs)!v^ z9$XlI5@is6i+MkeOMxXXFYiBt#s^O|HKTL0K7Z!X<&~P;M7D8{kN;|cl2fx>7as~4 zxORrrw9d?!sw#+zRy^6FqsO@clLPSk#FB@AnL^`MGv<2g9^?Xueg3!2(j%iTnG9^~ z4PxK-E;Wa0o*`I4V#v+rR1uEX<$Jl~>pLXER_e|>Q`ah`35bRJ}!rKraeDAi4fjb~}#1-gxo<5cb~jSoiP$|Cz`Zk!%&A z?6QRj4ZCD-h0Lt%y|a>pY_hUa_8wW0k#X8GitN3|_ju`gU!V8q^ZVm>-fm~PE}U_` zp0DTg7>~#Os3yU#De1q7uRuWFH%B$kW+W4>Vbxj0Gxx$gF0UD#MTxumxT3(}S~_%Q z>lc52Igi?QS2uBE+ljnlSQ2Tr#9m@pTX=VL%@v*5A{E+RA%JBgHX^)!kJBSx=n>Z5 zbUvE=ipA%pc$Lm?7_M8HW1rM2_Cjpu_+MpRAN$xp+5P@;s=+rMQ{iU&BFe&rTRS5h z#?Q_SQ#bXmP}K^2-{0*Xer!To9$!!{Ok^z3r)nQB!=@8|?)#ob4mtknH`ahE%ET@# z>=998DybT#zeUNQg4thTfH4egwGPM&XJ9zXWv1dC+>^g&V@X9x>FI}WfzEc_{}ZHx zPXE8#TbL>aDN>w49)#jZ2JPxG`BH%+e&aLxMdTb`G-C?TtlN9fw$ zff^@w#( zUYMFX;l~jnk*xGB(Surg#ZQedT^}Y;1Ph-y;X=bOHR*APo9@L5S|kGOb|{e?9E_ln z6mYT{V$Pn|AZS&80yt=ukt5$T+1zWtyT3JsR1xwC@L?5fA{HA zPjpfvHI3E4q~opgsVVXWm*|Z9P3`#+--AC#%T#q3KP^*5^2d)4yMN26BGK4>04cLC zf5&|Vg;~O4`0rL$B+;cMCufqo+-JDAw`cO4MCa^Hw|%?6wa66d%-$3rll`aIm{nX- zVgTG8?;7X}fjB_@pBN9EecHz-@BbDl`xus%meh((FLoH|ixk34dZPOJ`U_If>zSdx z5k

rqwgHglhy&S@sVvSndQ8a70#q7tgnd|`GupE#?MeyhDDE04ND!(zH7<3fm-Cb8QNM+FQh?JjQK_EF%5CpZCC3pxd2(#^vG}Nsxn&KWB0Aw@g}Rp` z+g9TyWD8564h22jxZEwq^z9s73oo6CqwVJwOT<;WQ4g_zFPWQV54O4t`&vuO5>wHdSa_VM&=pzEC%K*&(G^AI4Qv-Z zqkP@w`q;-Td4IBYFG~!)Xwxwqy`7Twy<|eiPTRTEkosHP+U|#)JTY)Mbea`9r)HFq z2#bpTw&?&pOY4M|xkY$lWXS8RI*w_cwUV8?^_@3k#`?0y*VdHY&XgBy;Mvuno7rWH zOTtgoUd40D+tqqr0v&$gRFssr&>Qd?eFfd3jdLym!DDesy__WzUHWdyFGJo1iDTCW zjkPF6#3I{8G)l(vBGj;L?4 zKB(ho#yU9seIk&=2}UZsZCZzx$*S9$7HQBmlDvn%^ZhmPHTSjPpn+|C&c#N)N~%1A z2V1cT?@cRz^N8slYS|=r7P@kat%0ec2dR}#;iHQmPq?h$fHx+03DUH)Q6FcE_1#4H z-DVFIKGc_WVj zIK#MLn9v!hX{qj0QnFmy1aweZY$9d*0~Z&l4-^(~cJKJTeaofgQO*8mVZ=&d61=ie zY;-Q*_(L>Ql{B`CYV0~rP4C{`vNj-KQV1h0c0Vwz8^|+W8#XH+*xo{CG9ZF_z)q!j zmoVdU*1)W7Bm7~Q#xwM*#)vCs74YEXjCs&_7#a(s{|M%bgX4oeZ^j*A3_L1(=ebe)n}Bvuf*Wx1m+b z7Wy#_?-P+VhK`KRoxq!Q=UH{sec$48b0oDSyr`5^)($>nOjL1vn9FI_T%k(jxJhGc zS-RwtzE>L!+GOmyW=K%|{e-@)->D!_B8ea~u((A3QKDWR9rOY9!hA$s z{HBTWLzV_oiTE(yxQ^Y!AbM-N9QL6b*M!fp>S*IU^Q;}2UGMXI zeTmvRjoE`=Wgr=39F#i|^#x_qIIbPB;O^3O@%81mW)&VuE{~Unn2<1G;H9fHCbZ4w zB4@YX@l*;b+;zg&m>?vCY-Yl+e~jKRZt^N{}FrUcs9In1Ou#?LS958zMTj%K7C+WEU1+ z72MwFu#6X3aGr46|D!G7GQh_V%spHkPMe#Gq|2Oh4^q62-8)k^u#)aX7({kG(@Xf~s0NF9_Ji#q%U9eU?w`@< z74|Ux1r~O8zc$7(LFUwB*qw2QO^Y>B?<0FVXG`D1t@b2y<^v`&|(Ezp;p4;GpY2Qto@}fvxGn*q^J|!iWhYJ;4J1p?oPuLK>?Kq6SXm?U zQMHigy`s$}7U_Y(_!5%72U;VuNl5{_yPZ@Ip(XLf$nDj6vDydFI?@Y3F%Iz!=v0;FXDkiT61 z&73&sWma%RC7niT*ev-<<0MOIa<`G*NNoh@(iCW|FI3mAg>t;Z2$8;3Vkr^~h)o}m z91k?seiL4MKT!!Dpwvmu*-zS^YklWevB<~A`-LW?6({a?<);}2>8cOnlqWeoKS}aLv ztI^Xxu$cv+*phFgJ_dK{hHF@|^f(ri52fZrXXCKPBI(!3PVnK0CpT}a?`-Mr==ClH z=)F)H$$}G1SG4l6Vr360$iGJSx@5R)fG9U|vh^U>**PG2n+?@^(IzCG+{%J}w*&{f zmer3D*VX3SY-b7Ez-pzg^D|JX3!T!`arCiSPK>PC}7VU-e7`@v&<(eVl^ z5uxFbKuU0aAA1sC%s*V>F{{*T5v;O5aaK++eNi3{<@ce+ezha?nB(-aeQIKR6vU)c zO=h(XoeE(owU$gl+67$QPC3qM9gD3b+RL_wyg)OPQCgOWTL z^vGV{<79F#c>-25;2wQ7PD;wyiQM<^v}%>xJUSjCmt>+dCJS4fJ-90}UH61`q7ac> zD76ge9f?r8LtT7un_-2Ei4vYUb1WZyoh7RU>?!9R#-jXUZ@E1tcmz>jG1>+OeRu*2 zLpplNX-@GX?Bc~grPnyCxD+;FtNN$MnHA9VBbLjx9@@~Cs_8oZU{kHTGU_8F!%{VF@&w^}_ukRhStkj1ky?*k(=db)+aOZ~vI;yUT$%w`|+ z#Yi`?-JhR1MhY`B2>B-twZkLbnjDvk@3uk(zath`p0ow^6YO-O9w*W>GlLggbshET z9zAM^58Jtz9xN$-z}OE+M{Wb!aZb3^7IZ~^D!nmX416J5W6HSg*1xA7_2n~4oQfCH zCX5-Pmhj)he6ELcd?u_Kd2Xbf^j>q8B}(<3w!51r?w&(~d*urS+l>hzf*$@bKJmhK z7YcVrPY3J^oSm_kosJ&)GW*(5YQ}5&b>mrd2;B##y_fcMEK3WxscBtxo7#`ond@JlIwkn0`Oe zRp;B(iLVOM-*`dApu82DW|0EU?Qwc^_|nP#hpob-hh9O^JNa4YW;Bt(k0=HbUbdnU z?t7(gf5z4RO3u#M_%fkzy=9%**uFApM>Q?u8crB}=8metVfyodiAr$Z_xY_{b{H=w zIzGmOjk}e@@b&{L?J9wOp@8Afzu4gX5rIfC!+{b#JnRJ|*`-}{EfH=FcKOEs0zS9J za7*`s99JBB+4gY3+Enlcq8c4o;R+)HbAxTyQd2NzXA9S*10{#lw6lfJ+6mQ)=J02J zzhv4J+v?+pku`$2!(5Gxd~vMu5{meOfw*}t?vp2YsJiW?VFN)Z>!PBBhYnA zmr#$zxEYH@9(;i2XTd3(c|(+|iQY^2Sp3Albb_evTD09|)^?n&y=>XXkp#e(#y;6? z5jXRqIoOkR;9#^u{ykJe+sKnA7Ep4yR2J_*5o%;U;ZB6;HlVx?8UV8~QS_Z@h+`&> z`7QyEl^mt0ThN#u?e0}jQ$dw>4<8?zq2#xBw1^U2EDMUQzTyzlxpIvxSQBod&G$DQ z7Y^n*lpruvUdro(EIa)hfOQS8@2}&O4n5>5IC8> z(YiX_zg%3Ur0^}P)`Z2c9x{Z>ic3hmNGi|_X$&xXu~e)(4N!AW5G(i;8m3jyy3E`s zq;q;2SDX+-5dzLS=flxX9^OA#eiz?y$k}phXk5R3-Mge-?~ard)|)qP7+(cud01Ed z3Dj7sF}aGc0Z7T24lU=@-X`IoiZs|ao%OGjmIu z(M~)cQbMBX>?9RRl=*Os5CXvP=Uwxsw^An)X$#Jd8<)cRz4iD9iBdT($s%Ws! z<5~6bS&o)*pb?`INl*kLGBQJ^3}rnMO!C6y8}qYbCnh zFzxNm{Xj~Xb^l`h!^B>Vw4_wEe_x*>Cf1NCwQTH#fYOO&ZT1Xo`uIS%gBxr0*eyWk zJmd7!OU`k%{(E2$u5Z1bw1q{b{h>EjTiH?xciC(r;ef^OZ}cYuXIsM2lO87v@d2(! z_9PJGlL0uP8FtD0_i_9+mI5-MbRJ!bY?q7U+C+IJ8(SJx0>!Exx(e6HY+Sd8F2>xN zeM}>!a~={}u@k5Bz{kX-Uu8%YWW_-lM>LH6u==@RD|9 z82z}g_q&axa6wjl#kY^WK)ON_ONNnF?+ZR-nlTN7JVU{)3Oll84${8jMZWQv%cHxq z<;c0AzsS{SaHgaxDgeCouLYSY3mJHDaGpp^7)e4p_;yAm<>WHDW&plcRRE?Q3Q zkZ`)8R}BfzP@X;$iQYp`>$eoT$ouBwOATnkP89Kiu|CDHtL?SW4%Y57f_UdD<4shB z%GqyW=*iuZFJQ;X9kH7>!%h$_P3y6ZEc`jXHn_hH4nlF`27lG_2Gg}fS6Q?OTRjKP zRTp;}$!bu%J9fr{BVJQmIJW9wST?eNCTRY##^J8GC@Pt;l!{aLg!d8C*UR>g*nkt3 z4E~Cme$RLNFO@Iy{#=ZpD(bVZ`Q4M9h)%H0fWCFj!GTWhUW@35u+WA=rw0_Gw+SXE zO(_VB$S(04Og9@WPFUrbk&(@Wiyb{L>D|J@y1v%0s%p4A+NG$}g9l{`um-$BRyM3^ zsh2Mau3qhflcd{pOTg*#88Gr9Dp{t;viEK^HvLZ+h%a8$OE42TT;+oZk?6w*Ux{pG zwHA@pI?UUvcS`hzyl%IwZy$bD-LSa_e#(X}1^+xv8SC%7Iowt{7hr3=3)vL3?pDEF z`pHbS+`dZoT-EllL_weS_SWcTC_gy}|Ac0K^`lTP7g_GA3>=j0PU!=i0@vDFoGZ42 zh1073_~^+XoQcCDp=HqIlSlKv{$EOov$VIA{}rELV*IfH7Owd`{Z^Yh!~mR`Yr4t zZdDnMwOqBIM?2vyjpwaMH=lvBq?4x0a@=iaOx+QR6=zU;d&LyoQx=TO0l) zY8E2il=@sCJmKzwCw8*80~=&B?57m`9C%7fUR)Hi$eaI;J$xm7J0atg&y3?y2&h4qKnE#n=>YY(W*PIb}s2cb|$? zbM4#riNO6z&a#uGYdFEUHo_bWZ^+<|P2QJf7_V{=`C-gt@U_n|%jc*^t=~UwB0BkV zR=tqUBxlQKu@2wbj|U!t#vKnEns$t*5BP0B+m{59F3Ch~L~zJZTi|^H|Jvd6OLET2 z7oS_3>S7f|Et7^|2Lr<<>w3wx?eJ;=tTR_Wf{ODXU=J~Se+}!q{7ix%^>HIHajI{A zzo-8?Ioa~-pnv(x{F=fL(7SbO-WN~m%c0$QT%3ci;SRvAsGdB3YJ9+g+_|mh~ zGURzvtF*$ZsybJKmX>b1?j+!u4X}`qJ@p!5yQ>*TQ0I=in+eToB#=<6JthzZl@uyD zIhk%~i2mot9BI9QTuEpXC~1ubJ^^^kp<}guxhG1@b$1eBee6)Vm|m_mLNZ1E@m8NE z3}{HaXuFog0D{=HHF2=@Y>d0RaA(LCj!{&LCE3nBU&90`oWq26NcLkf>0f*0jUd5m zhQ@>SbSPzOYoKSnUUR+A6C`uy;T`Hk%O+3-WF{r{TgW1mD=27zyISXP$;u@HQs6(m z)7&r3$zTtE-To{q*SrZbO86Z@F83VMpTw+em;K5{(&8=>iaBN{Yc^Gkvu8)gXa+0!U zhVtRL^5D2=C_xgQ13&fT!Ae0(UTJ>ufczQHcjRX^G|UA{z)rUd_EES1*C9h1IPUIV zHq+{eiSQJ0v25Z+=Su7A8y}@Zcm{rE5~afHG^Rh5RaK{3BRZdP)>fT8IVzwbBuq%^ zhZ3KZlt#T{Xoi{mi}bsp89}6mTFu?+_w;cM3^3My$YAI4V#+e%-?-ts0CU#~s*bh| z%&Lj&wzkVZ9V1a-nzY|Er$?LDkhvG|kdWF5JnhTby6vgA(xDj)xM5byN(({=PNtE_rksQdaY8I)8lo1Un;`JJ6LOM%*Np|d@SN&&|k#taz_N`H6-aPo~P zYL0}Vb=sF+0*NvLumOU7rd^gv|45RWj*m)Q3$Drhcpbyiv1Rk*JTdCcm(;@5Owz)= zt-AyJSr!^1!;7XjFz(!ot8e>JV-Hxpl-0=Ty3&M#v^G8I{#dGOt@yEuIG+>4X*!&eb0O+pZ-W? zKtdS32q&egC?6p>1JmamdyN^V6495Y)9#gMn4ydN7CxOUM5dy9NRMxbjUG=m?aoEy zu?+S&9kV(tJy+`>A6;FfY3)LX(D8|$lkpJ20`sAj_I5}8jd($|{$~j{%L#(rg-C{E zdOxid`=lC{ZyHA?6R8*=#m3uW^o43dr9Rb@kEdR2*lxf9pwnGIWRt*J2$sO)2RO~| zFYRcU1-_eklEzptY+-(rGf51>MFHCn=T3wkP1|Tm`cmn7xDooFN81S-0dQ+^!b3Q} za@lNVMj3^IGVa=1OAFS)I%X-t{ z*O3hy;~ATukqY^B4Sq>5f~s>kMC|ZIeq7;AsAA*4=#`6|I>F%wbw44%Vq55~8VLDa zg8}Ao#?Jj@u&j`Rx^hp}&N%shDgDn0b>)8oBujs|2? z&AG9*_(rSvRkaos+E&2s4s~PpI);#(!bz z7J9VXZVij|HJ@PTlJliwMrtqp8@WdcTgRRSi)^cVM}j*mC(F%&WAsh$9rtj^mgikpXb&dY^0_-b z8wxf=et`<&@whyd+ymUkgB8wti_$#5wZoFO?|IUe`5TPOr^FTdB>#yc%gVW!l?pSO z1~6Dav(?06=`hRzAadeVAw00(R!(}*+Fb=k#V4$GlMZ?2idi6(sn1sqZT;ndP zPYx@7T~{dO6E_>%BDH6U@%*q^QysOuoyV6LmF$KnMm`w8*C=#})S#yZV&w2VU5iWi}l8tOQp4v88>t>uj=qR<+ec)coYr<+C=Q++1Rd z$zR_Bc92>T3l{%Ge2^dh<9)6l%hwn&8Nh;Hb#kn9c(iixVoK7j)X>lXqtJYa1;i;7 zNzI!Jx(d=NDrAUyGJfDO9^QJ3f``xFBxPrCm~fk}3?X?OFhAwO ztk<`Rxp=bA_%OT=GLeh4OXp%j-_bSwVz=?5X-`TF&6Ua?JlwzXme#?2QR&?N6_-CR z*!KTOVt(;ADg_4SJ@`nsgzv~;RN$<-%PLfy}?{>Nx9K+v)wv{zS4+0^5#ct3!}wJ4rLZ)cSUN^& zbr~v2w1lvwBnktyK9J-EWtUKoM6*0y{@HCdNu~e;$V}2oKD7O*0Nkfx8z&R8C-J;> z(Q$A9wr?v)13`Hr34I8YEE7KnPfYAkQ_@m;KfoF)$czi`i-lYXTA67fS7E+0g!(48 zki-fNpiH+8BZuHyF4>9`U)XZ!M>!FIpQ+) zm7;2Y{#pz#t5Tnk$Ygu@bCWE*1vF<%v@ZD+1d#QR3XIhRAbTiatH9T)jkcr9`roSR zjxFsg|5iXux;LKPy!NvazZBqrQt)p&o*cR6s@rPVNj$TXyoYV@{rxkfCnkEhO0m6o z{ob=@)EH-vUdPjT!bb!$M3xu~vP6sRMMjX_b3&t!6A;U1h#()B}7L$gT}!Jj|B<1oB%!Fz49_m=fawRYr{CS=10r0=Kk z7h2iMKLJ7RTt(;Mfi;j@PNeJZbRo$@2Pk46mB7~b?j2F|gEMZ>!6p_IES;J->1={l zT-SSoB^DscofqMuMEsrw6N>I!YcD^q4CJK`v^>uEIyIl8eJQhI{v%0N_Vas|mb@KC zv%;znjKQp3Jg|S0K6vosVw=(pEoMd+;RjpZ-E(&i7i*&pTkf@dI@p*7dx#KiX3+VD zs*BTzw_b5qM)RpI<kbcz6F>Y(jH4gO$w7Flk3pgak{d)(u{#eJ0XieL=e)Fa1 zsCv%R>~iHy?}$Nw*=n#tenY9pMDXGFCN9c~&b$5!L5)wsRlgd6`+OpJ}R+{YWtsK-Z*zK z``)ezr=ywy|9IZZt*ve_IN-|tG>IIQaK1b`V5|)6hcN)!3Em=sjGP)lmS`|zVS09c-M}R^crlw`81b5TG;Q{C zW=~e(^WH7B&olUoL%r(CNBycBh34uvn!`E0ejQ4#y*KmhQ>Nsz#4H-lPg=pl;&QSP zpmQ-NXbul=o=u`h>BZe8pilJIJ)Gi;E@V?f-QG7L#Gl#Zh64M}xaO;cg04FHld>-( zZnptP4tLx+*iA{fxu<4DM0j88eqm0G@si<`YpPuNZP=A)1C_j#^roNdqxD!vTk&_T zUNMKzP=eUOScb`IBsmOOjubYiMWs(eTn5K zPrIU!xu6m90B9M+r?iaUi_KbqBRw2t^!j*XaxZjHZQ`gFaefDW6||ox3y~hN#{t*R zpFchaCokaS?sGtB85ZCQ2dyQ8%l0BkQNJngKLAn?Y<~FhB;pgU8*lh3anH^^(249;6= zd${@E+d79_bf`ubj#%Fd8=N-@8WO$q)JS?NEu15uUQa?^9_tlEs(BUxI#w{P;Q~}t zC`5!XVq?KOYO9TXnU|MA?FU>*r4fQs7e7c7A>X$wTDioDpV&?W_j$_|?s> z0!`=bTg4Dc(x4)q&@O{Ay^)f*+d9%IuYoVt3~jCE$qq{P?d3elCr_x~zu&|m*t}<8 zzy%uKuY-k3h~hG?t*627x`5pr5e;yrKn4MQq&s=rcTG)MK-wd&mwce%mk*yZbEBcB z_ig8-AAc4yD5Tt`-gCe6!ikQ?Zk;5Xt6u8d+NIHxfk)xdZSe`d9YYVg0T^Mf!~1?= zSihT+Buk0mDmMjaX(ngIk2+cgv{tMMCULJIx@I~~=68cje;&|u9(MxyA=mNQ6{;Lf zYrcYWq5XMWacnpMstmktjT2FS&CDI?*1O@ivuwZYhg?1 z6}864g{+g>vw$+AL|Wvnmpe`H`T_3z;n??tH14B{Y``L zN&Q0cvi-F`W$T5{kvj5xZ>UM{e0Rz%3D}f>wxT36eXqRqmG0wyy8a}M3pP8;7AXiK zklzuW4av78)O$icygUodhwcF*Pw4zXVEXxUDwn=QNXr&LzbTM0oMx*`7v$>JrO=Q?={vjpbjs0cJ7^2#S8~80 zBX*enup!ddoqQJz*9L~dJLwSpvXD*9zjQkIZ_1Qpej6KkwiV&O(;ptMwU2UCOzFFCc$_SV924l$RG0 z(qOn5o*K944Sve!XIaeKndoXk)X2g0?{f9|G?BIhAoRg#4Q3VIo3|LajO8B{Sb2&% zZ&nIer_!O|JP=aJullIEu&`odn-dYO8ahAEhq?ppp61Zth7S)4-QKYYt8MZPq4l`L z)0{3^SS0Y{XjEmw3O3khXiwZ}g~ z7`B|1467qP%!k0wUT$cgCDah$ncfpp5f%P zC`@2K#nQY2tT;3sofrIzwD@~6N6>?s&d)w^c~zlR+ZdFj}sR%6ql9Go6Buq($mw>Frfi#98$0o zLg1DB=sGbyJy40$xQt}G24+KgOr+s6@`>hmup)pP0HX?gVbp-L8_m|Q3T^J#4uO}s$cPaZRx@!K8Eo+G^H5!c zegK34Yz+;r{0=a+w6ScZ27q+%ZPq{QmPB1Rte!uAULOnv(z!|)NgNR~H?${{TAu+w zg;mX7MlpMI``TAhg;gyIkdTsc*6d!@(9poH+x{wR`FKfm-|f3k4Bzab^~o85vWZ!Ygk%kYX8?a?Soc zYdxV8%=>wjE+bO=fOxBVd;29!Hj?3@AQ-5Ly=83BlPYyV8bMRt{r>xV976On6BZPt zgv_qg7sgF|fq_4Z!^JY)4jbS38AL?1|C%5!v);JT&;sL~=3IVyVG~nRvjPKv5hj}O zq6h$YNvnWmw*cp z<-_f`XE;AL<=pDOp9uP{Omv(@nN0bt4pMc^8}`&~e3R2nOZ{}K!2U2jyKJv~@5i~B zD^~qy8P_kW7Bl^uu^{yfhCj>8cj1lu`#9Sz5)^OrOc9XN+^-mz031|jWf zfR7ZHFe_|V{d@?S5)4P$F;a6gO0MmMkO{g1`s*_S*>}%$@43hEb5d^nzUE?I4Y3=d zKruV0IcRpVKRQx^uJ`9Jh98Iw6`|QlHQnrVEcikkZlmGh5R|9-mkT07<9-1-g-;)7 zgeO&UvRs_9TBju^^|cQ6uA&2oEz zzVz=!#-!XzPAvayu9xDk(CCFsWj$yHhTN$_qviM}aqDpuaNY@ph;naa^bGE;aoZJ9 zTN}7sFS^Ly{T18Aj%GRgr{1;vG6P@q06(;Mn)i2O5T$pndJrUvJ(*q8VYRhlhet=W z`(!a9!nj|?XztPm&(Cd*v!Z^0D+yW+#U(E5BJzt9++^371E!j(@1_%_1*h!4Cj^E@ zXX~$l_WFW2M=&B zkftAvGQ4>&pGk^@pHFX2bVb7@Ja;uz+UMOnax=}G{)d+cwuGT-1qOC+(Q!vVxUly% zAn5e-^Yg9=2_g6gQ8GI@)mky=!L;6X7>r@;_oeK1Pv%EHvXYV*U34-v6Wey()rw+!CLnsVG zJptb7VP>&Z2jOMiaN;dy5^?c|Nn$JD9DlJs^FVx}BlcEhpOW?L;t*53P&yV0@mE6Z z>C7sbgXHnBdzrbky-_X@!&b6#1K%DGuRUrm7vBa7HSu8!!9vPBn{Vmzy~K?%P#qOc z1l~M}wCzD(g$k?5F8Y;J4Q$4qE$}A_Z~V`yKK*;ViPiVtm4BZs`^|Oik76f7Dt+vb z%X~3DU&Tm7b$EQVqO~wR+1U8WMCp}XefR4x&)U@&rlEFqO`owQaR9BuVm;?DPWpLYA&f7wXhyC?G(HFf6B^XJ!r0fp%n z8ygEse)v6nB2;oN7V%PrGFrbjhA}zY%VVr_)WR>>B<|k52Jbs$C8=RyAhe3c-=`_C zW}LqcH9%nZ)tU2Y9?aMXl5^^?=_|tv@w_fBEuH=vMCx6$xM&U|dam}^3DK(LYhB;9 zM4`|+F6;oyC+T@@`#Xdj(hSR%KX%fi?gv7kbx}Wn(ek)81m8^&U)f9O5CS}Kz-P;D zz;a_FavHF0uXHRS{{x&}!AZ|Ok547HZeHc`IOgi|W?bj49uKTN`u@_Q6(*elNK=5e z-Pv;nKXY3Z0|hvz7~2uGptKaP3^2ZTFnt-Z2ABS)`9BE*CD0~ci_X})$&;ifatB$7 z|2|gU$j~CQCjLGi7++?5ynV^S;v9{r7K16`_H6>_zQzb5j;^nNsOu-)$AE>k1y)`R zvjgaw`^z?erJWbnEPaQ8LQ0n8Y!t3-vp29Npug(&Z21)%rl~T@^C(9l6$nKS=>o9X z!R>6aU?&tkt$sNmu+ED z{bk%{>3;n9@j?+)gn!P?GAr>QixV@sop2pI)s-C!$P#t_X3m$Eo&BJ@rM$WB1u$x0 zMiTyhhC&j@>6h~;CkQ*pzc@L#SFi?W9`V2rV4^k%PydxMmf{5hY=#U<^e3B^pT%3pt-`j^>3y)#!FFD)0dwN{A zLjV|+8m(DqAoBUkNpQ5zaQJk~f*zk{Ztbl&b`UvT5IY!RV$(T9cKv~g>ce%6a6`uR zbcjoV7KG4pH4YLTQx6s-)Xu+l)t^-OyoUB+U2U-_y+hle(Kr(eMWMh;1xI+mm%ZyG zPAe-tauNkV^^fvqg_Hef9o?EhxHa;dN5-8S7pgANz2BTYkvI9@$2jlmu^EB_{`=60 zjC>qaxuO#evrLm_ut7u>*?n(F6a%I84RELcUfH+ zf|5MSc~C|XJ9Z>Oz1Zq!fW!6uR}v=}lqc0au&|Z&@uP&RYn3Dk$peK45 zVEXAhfx`oLI1KCHxk*lLE?JSGq2aYxzQ_*a+1W6HSo#0^WGOGF1UN_ctRqpxKK*}e zN&bY?{=UALf6t8+pqQ$4%iq60*Ar?KL_--WZ9e!(y!#SVEB`|j0eOLwlau#LY3Xwy zmp~8!Uk6LA=)>s6IOb&J<(uI?W;3k8JRh^!8tKtm4xhU z9Y!O^`?F$Q(l5AzH7__l{v<{^2{GM|w`S5Q)!DZSH71z@oafxMDQ_y@h)RYDcHKQa zJ?Rs4_%p;LL1-Avz?>9+2359@&lZ%nrUf$)8X6mWy1Vt3Sg&6fhgIRLMDjqU0FD1Y zjx{j^k!V(|@n9-{ zx_WFfr?9Xl@DT66ef0{@e6&od`w}s+=d<>A;wg&v$zZZuMA2bF-@(TmvwPawmAz|- z;UvLjEfhjw4Adn!H1K4=9bbX~~ zRFss?uCJR`Y=%=`@xFntg$l=Ow= z>=D;V{#uO3PTi60Z|l>FB66c3kJie=4@91SKZZX)^d?1KKcM!LI9Coni6Mg}Ga!7~ z+6MHl!EB{e;iK}qnODv z`YwTrX>o;2l6Jq|UCpv%rp0o3RX7D}ZqEJ*(VVaYH$;-f3lZD!8q1(Yraa|jmo6Vty zM3`_4ZfRLs<|otA(1egMNZ%(VASq8rFBIDRdKe?iB!0q+*6RJGVFH&v<>}C8yBRQ! zfjX^P_Tw&?#GsvsY%OwpGAyB_>M9lw_yqL;ddotnFN z2GOGexR+>e&z^T~=3>jTRL#QvL?u$~)t5Q)>L4ttk2Ga!o#-5TWMMcHS2s?k{ zHWyq>rzbE6IC*LD@X8jay}iBm@1WicIUlXg1zhCbw>k|{(i}l@r{Cxg`2`_)9KY4_ zdhG%rPuE9b&LYWH%#e9AfW@%fV6xY&AaAPsH-o~TBUZIicklWnd%*lJz#c@i+<*mf zYzW@15ZaMi8&5oZd?g(nj=4Bt2adzT{bN1=y*y;oP4xQJerx}*{Q0Vk9HCFsdSKT; zt;L9(h|aq$)bmg3|I~o{^c6XG_gG&G3ggCpiTps6uJDQO@|MG2)M2giJlDTh20oBn zIA9I`_kU#R{t1V@`iYhn@p7frZFpAorF35|GLRyNnZWBq`);vC4p%B68Z_^^t;TZhd0|%Po>w zLHchr0I7-Ii}}9mMMXt8|CzZqD)izf3g+S5n3H+uQ+i% zM&cg%Oh{Nkn-R`KB^OQIC^)|8W5Qw`&nnoN4!_L(`-o%%+B!clC5|3Gc@mXOi@+Qi zk>eE;g~TZ(8hO$WAHMnrlJ?8dvU}~MyRR0pk|U`sztj3tmZ*o%G2o|2z4CqSy;Ndl z>iEsL?`pN~)yWEZD$`TJoTme(PsfmEUt?mrDkP)<2U2VLw{H(KC-4!0q@TnUB_yw! z`BrGyQZbX(5yu~%G$!d5vV{r?{LuJ+L(rC-%l|Gc|J6XEcY&f*Y+WD_vc{@uf&9M=giLox!V(SBQ zlimp2k3Wk|RHw(KHUbL~BrH)}@ad~mAAv%5s{Qw)ziGumDFd_KYvC&|-v7Tvu6{?DOZ(A8-Rr z0u7OD?K#|KusHsGMC`29A|1B>d5G5e!j*r%x+}b!t$agVGqd*rg1FyO1RkaPeEow|m^me#rUc=u976;%LS2RljN-a5DV9kg@o&hh(`_7fB*`91pJ{Z~K29>!Qb(V1&v4JX*0OVu$Q7ylx;5Vf5z#m!aQA*mZqk)_w4gg z?3yKafLgHsjzjw!H6@6El?r|VAaxk_8XO#)Q~G~ry$^OoQWEE!%>DaysZzmU!h^>s zcdYDSiA7v8Cm$bq+P(01OqZ`3Agdwh|9>?Qr#!k(ivjfqvXtLz4xeny$D_6WlTjJb_3Beti-mVqIUkMa$o9}Q({Id;6BilfpBFkwie41e^pPpRl>Eh{+ z{&KttRgi$@UHdQXV{Ex2A4n6H$11VhVCKVJ1A|o9!I0X_tYe-5vU5aq>nVaW1MVN# z3qOA@(}m9}6+BT@1@E#}cl19Hoje#ymaAK<)1CP!`;+UQ4GURECNu0?1?%hU-QC@n z=QKa*oq2jQ7J2c7GZ#o6_#k`3e?6)cIP6jXb4$QSJ|v`55H!|%6Z<4h6Q28bOSl%< z_8NADIoql~P)&kkTCY$I(+%NDn853u^Z)dT3$S+OluVHiQcbeNKr=^Hl$Ms}^cWji zctH?tLERb{L9wyd|CSVDc!Bx=2y^`^B{ebo?L~FyAmjX3$)OK~>Tln@!$VLL5Fwru zf*j9&VE}g@YN3TNUgvL)SI>B=VFS6=>SQCLVT+%io2&a4#j4Lkg>|H=>&6QS#%X;J z76(=6?pg~ymK#tiPLKJkS1};r54?W^PLBV66K350KW9bA=`BJ?=YcJDT8r6oMkZdQ z9(Sx|(1APilNuY`_KbuH^5}F0xZp_lzo#yx#{|;?e(a14rsv)%?kT6)3v#2t*>d&j z4Gs>7O~l8GeBcdq75@u#*M~F3@-)JLYvY%-sIelvY zSmg7soUgv&>CH)a7rblMrd#fE7{}VA9)rulEmRRgepQ>FZM#uS`$J+S=NJz@3rAc!M$O;Wcqac-6B38hQSYvdK0D$3tTL z-~BUFPvw+N5;%7n{~H)wm5`A5N9cH~#n$`uC}}O;-{0r3{R)NSHR*5gN{q2@-dqqD zX8M0@eRm+&>-+voduvjp9Tkx%*_slisK_iTGDAj2w972YNJgQOm7}+rQ5iWx$tXL@ z%nI53uIKf}Ip_QNz5nvFDQ?Wg)_#pHk$oaXTT`*&YK z-CN89b=sUD3Iv}CMDh6V{QN8$0RoJuzSh4_f8YJ(xBzhS){*zOS$%L9U$PT9ErYAb z$@$l3V8+;7(v~~<`7zbpZ{GJUH;c~zUv2)rW%`fpC$BI6_1{Io@?w}iC{okW-NIi* z3%nj2+Gv`MpY!7uR0dU5-{2T<5UR?e(x@^<4(9nZRXV1TcCz6$ycL_G_s>|vGQ_8$xP=TKM z9~!!0-~Rn_sE90aw~8CXux!rY@h!`|1+)t~rs3H}Dsj1c%Hy&Wbf>J<{oO*iY) zw*_W_p(i6~en{xO_1rdJF(--)82VLIl|9et6|0irj z`YxB%0m%6hVTqoR9lfj!dNb*V)-f4m^7=fBXeIV`t;Ny(;Iz{SOd9BpxH!&a}{8>x1Ea|ZDnUq@P^6UDkI^c_REKm(@`V#oaBARBjG z{v8@@3bx{otLp>i*)T%poYwX^L{_NK=|#|o{QS1xtK3{%-q&ThFvX5c6iZ(jhCaap zl9F6L9=m^c>rXU)BTs|55CFU!AicHL8{tors(wW^%}Qt$MI4-x65 zSMS)dW3%SXTiUaXon3y(y!+EGi3J4bzx$5ItJ_tb8Bc=4!Gi`TPMr7`?3{0m82chR znhm|9=jorbXucIp^77jH`ua69_f&~l{aXI$(W7Y&Lgzm+Q_0gKwBW|X7&7OCIbyGj z*zVT8cySx(DpCzXsQG&Q#EB=gf1HUkHXKz}zD@@S1T6%J1=7;ecb}3cjm7oaRU3n> zM~4Rx>i1RYXl_~eG%c&1A5S~^xo`R*>|cZwUaS5q=h>b0(m#+b?r6-|H}lnp@}t2g zBT{;rdu?aadG!dlmt|7!?fSzqWIJ5DwdLUeDAt zWJW(M?RyBM`V46P`83_HcKg3okO8Jp_}{dXV*CyFIL6qmruInq?ZW#H-=3x+1`_g!iq_?g>(;MlLX1oUL&MhJ z(!a2O){{G45sSnLwkWnHIOAWR57ss#QzsV)U)=ZT&pmf%PVM2@Sd%@k)mWsH6k4{u zVBN+0iX_%MH-NFO!LWhRwcKm=T}QHGhcwHetB?77;@8$WeBVBxHPY^QNa5^qbS^FIKYBBkZ2|T{Y66@Mx9_3xnaM>3v zykp0ip+9Zc(ti9n>&hu|1dl~tA+Yu?`~FwKrtfad9!}>yCJpLK45YxUL6k7Ak&%4N zgQo}^#9VSX;S6lV&R&$1NlH_P+y4iO+TT9)UnDBEl)Z_^auysGQ#=y(-VO>{Jv@9P zCV|xEz(g$_O3KR9jX_W#6+ldE(RH^VFoUj1r^;jI@6tJEVviN9y&Z^>d3Qx&AuE=7 zJd9pKq4XU$LXb&k3gbE^o;?VWE!L(>XM8b0Zon~iRcpgrP>i@zF_x}l>afuZ72gHi$sMNONM0_f3zQnB5G!+!)O_?%f zngg&X4q|amk8LWs5@iCazZ1l}6@EVbVd-5}|a0v+`Gv><7+xp)GtHAvpR9eXs@zf^l08-M%i?K<;r?)H#*Yfl$#koh7P z{6+Vc#Y>Ztk(hSV#$wUF%8saxs8P=!{$A}1Zp>Tg>9lqpTZnPS!JY9N>vwyFMVZyC zZrV9wB>2sHSifV$_C(kRwfo;Sm#vi^{Z$xOYK|#|U*uwrZ^^31=-({UQ6+xt)J9lnZv9e;;yLWqv#!}LUQcxhPKRZrkJoLr8Ar(^d`~?fL=ffVk zc~PgO@61^j9q>OC{V{yL(VF=crGHj@=j(x^3r@Q+_GMbJd-@*=KYpm?P?+cD*;H?E zOpHLdq0d#Bke8C{O9G!zP68)WE+txR7IZVcbV)ee&enGOMixq6i~Kg*PJ5u`&#HxN=f*JeLz;Qdz>*;xEwTZiQ?$^RS zf6F;J-%BJ4L&w@_WH^s3%Qtt1QBIST5jmY0%Sw?_?dikQ>Jl3mA4>XTvJ11qpp?J&gK#+ z-VGjQ+K!U=1A#J-e5>p`-@4wVD5)J15|7DJ9pm@uzVRd)LmDvvO46qX9l?wGY8)M( z?W1Huz@Z;%0ij)0HRVH7R6?~g`BBYnUt`w?+W*c~xJxL{Cq>eHJx8zwldwvN#o|6n^&56dB2;=H|OPvA#V9 zKG$T{lUF`!M^W~%+3IHyO#xlRR>0$=H8y3vxlZqpWcAPeN^n=;PCm|H^0sE zsZaB`xw%jF7z8adAK5A(kdr5n{(7^@Zi-X&K}=knzr9?YQh*|0itRo}i{SlcAl*`Csojf`3(Zqs=hKZptnKN-PW zz5iEhYub^Tny<`*X;eC){ktLnHSqPv9(UbJ=V)+}|?)Q3GYX3h)= z%i3lB;d@#7Xs>ghGV97+3rT=m%tEQuvtnMWvz{F*+NHDiWjx%@y706y9+&xihIww| z52~uG$rbgv>U)T&6XR9b*TwjMNxByUmbqkwMjzke=SdwE7#te%8pK%@l#~qMnWr(+ z!oq^rHcV;r-o?}lMNLhZ=aza_XJ+mrkI$I9#&#mA$$G3h+!7cV*wo%$umtZJbHk|s z>EGhu`5&Vzvzbf6W*a&B#hslOm8>l-Mfd)Gi1wj2bzGgf>ds;Ro#zfs z)0mGFT;0e^87^qaWF-y>bQ6nVcz( zoHB>19UxS=y1IHuRMb}d#xJDt4nCKZJnYHN$?3lFJ^AIy2>RfWN5_lSKZ2L%zEae) zXU2>fM%6=mC4GZ}JZdo6 zK+Y7mxOPREI$)nt!T0Gcj;+F}o?w!{WX7Yfh^}=$>BHxW3=IukR-+M@w>_DKYOUxl zlnyG!uO8U8JAi)Y|75M0f3nsL{*Nq?^QE=@mH(Blo`!{OgpbW1@$0(G`f$nBC4oVd zQ%q@9RX{N#@9qWMBc@Ep99ug(vY+*j1lj+V(|BVe*o+$?q_Hr;ey$!KJeEls1xoBl?Dhc@Jh$ z`p3U~`Cqrgl z`?QyJzn-4nvwaziZ839B8}|>)VZLU=!p!;1uM!vhN!&K`6}QGglOx!@;k*wIku&X8 z{&O`WZ+Gzh;y@3BLPaV=B|=k5BjvGgHU{u|lcoHO1>l)S$#{vG|2RlC>V8vGQ@lAD zp>T&8O-VBrU5^9~d zH6b(2VQBltDHO*ya*id`$tx7^cq62{PX9=u)Q}N*EE;~0?LH0hx}0RRV*JtzR!Lw< zE=D0_sd6Ge(NC69ZkWq8zNsJWR*^L{+B|5p;PejT&GDHH?xmH2AA(i+xGlq!avw}{ zvQ$Mrjo)oKxVY$pyJ_hhfF(-!IS5kj z-i!<w8XC=Hp)s_*GRLC_Y7IZ;@g zKYxE01I8ny)?QeEOSsb~#^WF-#|4}|4j(C^;Gl9G~$ z0(frR(6kE^{fjEnfg{KZ_j~+U<8y?=fe*bHLm87ZJUmRpmFuihzJMz-FaK1%9~~W~ z*X>m`;JZ=Ql=_*g?YH5Xya^Efa6nd;tiD`F#?`5p-^q9-FvH+C9{Ay*uw|)u zta)ZfTR;tL|)+u=CdYBshN)>Pp9){UmD!?xBfW7@@cu++@Mj7Fj0r<-$6jz zYk{_(SP^ zH#;HVi)f(m#Ye;WRn#%lACay_;>%N+2e<}qjjy*_MP1}AxxnOze zQsi=HQ4tXzvadt&*tk;SV?hC)!tnVYBVZGNaHqi&{5~oO<#k1>1?X%hXqi$5IjKQKAJUthFZU$FOtc6rStkS>!&PEy&dR= zMAdvZv=8@Gh?iI7vtB=NdexeY7C{W>p+C_<@e`XnaVLK3^Xw6VsfjDU;C<_zHxq5} zekqu);Wi-9-&$n8Z^z4HZg&iJJmpw)^UkKFqxCVz?v#4parfSH={1{FUrbi%o=fqJ z%x8bb`W4mXjr4h!!w77$)jG2){3e#PrP}zXd*z}JFOF{77pv+D?^aTu_Lw+je_u1j zYCrNlA3wiVJILjCp6VP&+P=LwzUADZDO7I+V8Bk}@})>rK6W@JpTB(Z5?p3@0XlWK zY2x==O--FE7gM#i1GU##f%gZ@tIEqcP>va9qZSeJ^YuN5{Qr2foCOVq#4NWoOZI%) zOJ0P1+Wq_YL)%+hEqvSydn+&ymW{2*?gkJ}Xa_rWI)5cWxy7SzJVTs!kmMV{##WoeaGglDc&oQ^$I-0A}Ct|#LpOJy#T{nz* zxnN-@Z?rW&yHv>#|2Uu`EB{)!>t?8W9#_||93S%>>V;jxrKEe6zfcrlSLXFPj&x9+ z`@xdy0Lmay*e8iVN5&=`*z=a-g(T;S^YbW`=NkMuJv}`?lda?g-Ol~HPCh~-O0+cP zT5P3ilkxD1EjX}zqK~Sc?7i)ojDN8$p(YTgBBPOW#R|EWw6rt@In}}=k9RFHP3=8d z1dqTF$paq=&bWlvhoLAPJmk<~&nEE!SSvaXepzOHnN(FxZEg7v=1_@jAP!u6Y z(xY*&x!Ijdv(0ztn4b~lX7`rvb2Qz$&uH@%+W~#U*d6f?)O2OTcf_0OyTp5Er{5_q z`OG27sitdWwb^>9Q*uelldZ{OY8E+b;>yq6U|b(UG2PSKo7LIbiG?~fOZZxO+_CWE ze?OQ;wWuSXJr=WUG1p89_08V??~E3Niwu_K{holOp9w6*cQ?*1$^?|H$2O6-WyM@e z)OO@lzbeYi%ycev13Zm8duBvUOUwOv&=s*}B%1!#^wCxl>CbmI_?sSJ41PQ63~vhN z+=y@IzShu1_U`4HUF+&F`NI~_&1m9|G0Rs~;v zn=d%NoTpO_C-!CVijl^jPe1!*Eg$hn5A@e`E=+&=Mb-P|qQ))qX3?HnAB^wbG3HbY zNr>BOEAo23sh;{T{ZH}-6CSzoIos7d5l8W+!O%FB&99hHT-lhR@B$u zU*49jk0qmIvrnF!q@dva&9e=SNf49|Q@Uhnd5+LV8i?&HC2fT#%dQn}r0hb%0{Pc4 z`%9>n<47+G$#f0&IX5vddZrLJE*rPU4kjzrA>yb@nyMLXYsH4D_ZZ(9B&&Y|g4~dsYr#)Yd97fsHJW z*x!h8SZZyf_g{u>srNXUBD3CC`(?JQa<`eixfI!uS)%N&rGNJio4W=DrP#AvkdNK? z$Hc(HKSiwWSw7e2&uSkYcSbCmb2HIGUQlbL|456#j5TJB-a1R?w2tXX8pe=nrfs1# z)xRCbn4pQgUa;xe!mpCS4u+^jai^BYEulJlFIijj^upz;1LOzlt9zPGzN?$tMoUxE zn8tf(uF55iH=rn+Cr)+9I0XD-)FeiNosaevQF@W%jlOvTzh^Z1gB^BZ7vN3>%$GQ$ z>O=56qfATQ*1FP6w(=6jxq*DiKMfxzF3^i2C2Z3Qp2*mmvC%$^y<1J!JL}ui$3{sA zFl4-by|hqz%$}T&yPzi}71z`zZ)06p+fwpHLZ@t)-RM(y z{q8{jX&tEp&)9Xs2TRg#l~|jZi1HUW*s_E(7Jg1hk~C>+xr0>vx#aUB@oM&D-H5rl zpoI#evMLv;@*pEC73Svy&d5kY$Z(M!?eg$xMV3QNV}I$=rLzjtBfyA6HjSTY4gQ!W za+rYuw}+nD3V+u|OGQORY-cLJ5reYiQsLCoGeK4cR#t2&wB!F!ei@)w2ga3u&`m5s zH~bF|!RqF&a6#1+`EW$gx9Xrl%h6kLNL-i&X*s&UHfW~4R_EZ+n8~voF?p6-_M`BO z{iZ0eB2`+DT{liXY5g(Xa1QLuweE`inQ^PNodf4bnW^Ir4{!0OdV5=wlYga06|*4~ zJFI5WxLQ5m*lcfZ!~%QJ@*A-U4`UNzAjw_SwyM}LcNE7yI{yBVi%VzJxAR;#?7Q6f zRL9JL^$(!$OI1GHQO>Dj^(T%cWXX~xaA@MxtIh>$D^~>sNi&9`qzdb(Nq5vttw)+k zBO4M`l%J_lV5Sa^Cut5d8^aejZyexD3Vv;$@#9?$CctkS;<*{O_2 zoP_trR{i?dIr(S8pZY;B*RiMf?=Lb$6lbzVl2fT(UF=sP0)!?QmPMo0M;km!?^(d| zhw2_Azi5rMT$nu9j5Mh*W__Z1yWj*{ebroxu7a#2UtjG4uG_rZcBU-VjT7p6{@HJn z_3O+4b!W52(z@iNk5-MPO0EIP4@gE%v@jI3o>qEaM{wi{KsBMvd98?yp-G3bswfqYvMM`35;lp$Wi}IFX*r>q=YUN$6gisAd~2T z>2B92xTv)8sObGhW#uB8lEv)_*$Uy5Q%NTw1c0ZC?0&`O*jvp%oQeF*8yDDsxTuj{ zI`rl=9T%N?Kk#g3X6PVY+cToosk_)aG+tzW{iW2tEAraifuA;JMmrJp+{jkTDb=Zr z4O{MCuKw4#WLFTX9Oiaqx1t4upLqH&Nr{a?Ig4D4W@`}G)kdxN!3YxcBhNY95uCu`mOW1v!Ua`GFB!=s;UXrL81&Uar{t3i`E0P-&o<;Y z!b2oiRZ_ByL^r}!Y8!$1p^Ipc!UWn3fU;PQ6fV~>R5NzvHDg6Da(0wRjlIyzl0xof z@OIm!%`j|AY7BINr=2^M1i4i?pBy1&fchV*5K{N&og$4r7RD5j!|{WK2J?8>?)JO zmJcP1KmBU!Ah6ol!1Qifew&~K{(IQ3u=>`%2Rkl1NU{D2Fq^l7Fz|9?6 zio;dW!ifa!8eUso66j)3co>>sxaI+klt|w}5vVK074gP#n-);DV$8EzkszR^X3sy5 z(vZdtl3Da_lQ{F}BX;N;fEc-sw=-8?S~ZO-4WmoDrlvd9)zt`oiu@Xh3k6bpl!{Ui zEOgMlONic61XaOL#S}O~Le?v;(Z9IbzH_%Z>(;t2g-b@iuO3}H=jI-giVbXfOH^no zZlqUxj8RknBo|n^v-jsC#J`K(ssfYoZ~Yf@8r3*Gv$nDltlp(^*P8!h*6?432l7KX z7QF|mkzg!LA4ptiZL&2xwF2?^eeclnwt}tu**#g(Ziy$bS;(Fa*!+*er`1-KB7$Gz zctnuYToU+jYIm6=H$VHBTU%cIKg&i+5VqM%ZLPB);pWuiTcmdUeCHw-a8X5#h zPdmj4ApQGiVe5Jr`^D;@is$)4?R1&X%F5G=EdsTpj(6Rakyh=TqN1Xo$>6I4ZR%#{ zxv)6xu!do2mvy^Fk@&@0nZRwIYPorM>K#W5$NYNW`l!p19*q?jr;av~bB-cUxhQ3z z!9N57t@64xJV+a9fvu7R0yZ#KHB4uQw@4AG%}g-(sHn(%hx?I_Ijun-vvYjd(g)c; zbvF1!bY&+UdSe#7I7K8r%b`LfQzmJ>ntG|#@}{(Q_x}8x={{9d+CEwNwI;mQKiuDn zDT}|x6pzgyP(z~fTeng6l3=Nb)bLy}D6Ntc)1pz^Zr0tBXhrk}m#kLt%9#|$;UtYC zEie#B>{b<`8N2aZVIc|;4KE;7;7fIN`v4zxu?iV_gC)3DBQSBwj~WDlrcK{;9UyDs zv_Bf;><)0B#ok{QFCv96ah!WrgS%XgElpGJ+?bAaqy#DKQIMk;fHqz z!j_Av+Zim+lzQTc)mF2b5w{ukkY? zeR;546@#%ntJJ$vP*u-4tIk;~OeuEFEGOQgvNCqEZW?9$i>~6@0ui5L?|zmH2IgoS zX@O@Bg0^#MDPy40sWn0^DjO8pWEGM`hdfi{7kzZ6AqeW9q$NlMx~*HQPpBJW(XyQj zcd*iIJC)(%@8@?H9L$F1-oCy@WZGDQ+a$UbiYgM#bR~X&SWhR5vjp`)zaq(E`ORzl zBVUyVI*ty|au&4CJA??v89&Vc7h=IDX?K>C+L9{Ro~z7aGTVNc zK+ze!7YB>>tmN=_>*Hy7zhiw`v|pCNn_W9O1>Ewp!DH?E8n+fV-we9kYp*(KzIH{e zd}T`11Ju1o!(aa%dJ1z1tdHA6S%S6j5QOEslteBnC#k=@{Ee2}$jjqJdh?jHss21d zn_`-|0=j+CRkMAg&JImvRyH;^Pc5eSLFMZLRnxg~@+|M1bS(&bp+2@5`03+S4MwKj zkC%F7;^ykQ!P3so?pI}!kTOT9%YSK9ZB{CgADY=;b=4WQwUB;4C1OD1`+npX-yUS6 z;d<#&-KBGG(z*6`k-R|EE){8%)0f{-t+!BtXq!ZlO#87UQEM{nE#XAU!pG^>3DnwW z4c2O5_3k&{j2WHYIsN4#Gb0@-_ro7$!LQo9(A(Q%Y@*G0vKxMmtadkJY|X3_14Asr zo+yuXhGxWwyix+i1~*eyU7cRKW9fo17N?lMtUIbhwXoCB$FrY^TKg`xH8eDMCLdSx zXOX5p@$EGFx)ADZ?5oaidWQk}ynpVz1CLQ?H8u`?N{RKsY`DMcheaBaGU1~MNf`Ht zkc7Z361yJ(UCZi#pQZIu#pS55&!A=u@B8&_37CM(?_TjM96p>y6#X9f4!On!;QaI8 z7Q#)Fo4E^jo_Tl}-%*J92%%*~MT-g6*F}Q`A>bomk>p;$CHA1lBz@!$UF7Y{uss9l z$FAOX*{NZAoLZ^vP*=mUV+{_L6| z2x_b$&9=Hd0Xqwh3XB|&j~ye=B3yf{w{Xg$MH4EG&w5$+4VGAZ+W7*|9QO}s_9)+I zDpGBu3wnTuFyApdNdT-C_(9nDkA+|Ycmi$SnG2zAA-*4o<=cqx1}5F@0*o40a6MlR zg#991TArgza5q`9X7s;nbh&nXmR$`gR@E8=Lce_k9r{X()bi)fpw~!MpnO8OP~9pI zwcp|lzt$>$H_(Ae^`B}BKJsOc4eM_)*%Y6BU6PYTJpI&LF0xa9x|4|bQO9zb@|5BP zH{+XMRoXvrUAK&cpNXbLVKz~iDw9g8Hgu8aS+;Fo^g@K8hZFt)C4{g$c1`~eFGc<` zhP+ES8cB8ksV+#SMlH2*(q|+>PAty4;yrFsp5_pUT#Y88t&S8`8U>`iENJr|X3V>Y zI@<;lE4-?~5^_Qy_?wWZJ28CfJd#lU`X(S7g6<$; zi5>6;$GrJ{s3-+|Abt%lM9o$=oL%0iY*Ote1)j8<+-jdEtQZ*#sOwz?HxCV)c4)b3 z2l!fjyVKe-T&XnNl6i^#Gi!m58>gm5?=zHNyP)#)nOzxPDSaFOGUhFE?o$;w3ERf@(;!cCqne*nG@Mg*D#(n>LvCO`;>lPRZy|^l%!pG*7JZ@&HKw(K6Tf&u}pX_;} zRUCNt(8fjsD)A)A4Te5!%0Y0TsSG=QBY0#!+%CIF?b_GVlSwiLaXp+0Qz8`_WHY{d zB|?INxW#?Qq=?M;dJ1L^sSxLpsKum!j^A0&>0iQW`eqTShsoJ;bK?}DooxOHu350j zdX3vZW(Z>=0j?K%X+1f5hg7k#XWrkU6N9j*=!%%ML4zVrw6=3%zM=BE)2XmQ)C#GH z6KGQERWeDatV;aO^avD(Df_V14Seyb`_ZG*zUkgB@sy=w_3v}P#)WSs3-+u0>nOv~ zI4NWYa9n2W-GLnnvyQID*ZWj%XRWtSQrevQS}SGSEGo^tprGJutgbBVd4J4_cZAp} z1ea;RDs-&PZ#I=^j_OiQ5{GW%%m_mw7lE_W1^S|8_S@JTvhFM8CRI5TuqXRtV1hYV zw33KvFHTCUR}@!Qud3v5u)_h2)yas0|7)cS{-P4e6z)n$o8>>lw!WU zEozgvc;=))sd3jL&ZR4^-m9y|{cxgDX0nyJt>9=y?A`uvUAend1_o;)X4>tHJhR1V zuob!e#xr z-1gkw#Tb#X{eLVc-w5~Fi)nL`tJT)Szte>}oF^5gkTN|#Al{SV$8`wxUXo^bu$yQP zI>b&WIq<`4bRG+(AwsN|P_=0hi0JlSlVgYNZKk3+MGiy|t~k&Y*F;aQHDveP)y$JEh^yUhpxKvvJFrs`b4;Mwql6 zFq(76uV1c8>Ulm_M%*|nLQIwIkU%w_-=JC*fk?h4#Y$SwYmtdnRhY1oM#JxhTLRKG zq7@-%0p_#G9<>JAIN8J3JJrchh2O`^7UjjW=g)Ov7xr5m%VFR%`wy`iPN6ns(a#7ET7WEoD+z-WPk}zTcY|u7S;8eWr=ps$R-%uV2=on zCEyEPyt)@IG@F|OUF|fjek{1G%zzxq3199iavEE%US-L9Lpd2iAoGI62(7FZk5d2B zNBfBFP+x}!d@dwRb@t99Wh%N?QQS3UpI1i1dh_!jcZ1%zlqh2Em@x3Iz;w8cz4;0=FX>UKk}Wd$CveEL!lFY7fn@u)sB z)$nS=>$teBgF6|G?z;ofc0Xb3tTrB3`g;x>IIunJYJ`g2D)eu}EuBf7#GDt^*bgCA zmj!#zA&SsE+k`&0z{|_a3^jP@cr`$+qxaYL8U_)ymK7?KVhS#*jAbtzZoeXR_hNHf zxROuW3t#}yd)i~Qkoxq1lU4o=Kp?$KmJa5>wfPmv)RW?9InaZm zZ9=8FJXuEaZHAMgp;L`+;$;@Q0B+6?{8WwtkVM?v9Mhpw7w!EY;% z8n`*YM zOI}lxzLIJ68&NTFaoSh7fY$ku#rC7(BDCWoR8Df5L-Xz#T7#h@s$-E|)05j$))!_| z8Mfvi+_4W*YQCbl0hY-1y|Z$(=ecwi@evPI_ocC;0;3o31&9E2xq4LxSXVbIOexbx ziY6s`Xu{?0hWr1_tAfxl!&w5mLtbdF?_|B^x6i?K> zp68=+{~y-&(=0kpnV9f{q(vN!P_|q3Qc?iKEmA*^CA~smFO>TJ{5ek|`AUTR|G0Pd zp^%}gln93z9~k1$kLlHfZyX{EMWxZ6l4=N!U}NV3!aJ> zvS5MoUsw?mLpVZUf(5!KKXLeEEFy56O9u`OcgSV~Vco1WTjS5l!(Qj;Xev=Hrky15 zp$UahQ+wzARi9QSi>uYSL#`5hAw*z9V?zTm&!dcMsX~I$g#|_~#wvGKq%~p73=8Gq z0LF2lxiUUcRjz+mJ{GtGqR*?a3lYB_KCOJviuzXARJ_C;v)40w2LY#Vx{v%9oW}gA zdP`eKKKz6<0M)zD+Xa#x#_USm=ledDN?b*QE0M~Jdw*4@iYqSHr%)wa>2lG|MCWFa zBj)Ljd}F0JKK<9ed9;w6#azb#0vqgGx+ieekJ`5B^p0BNA3hM)-`tf;c9CGhk89IE zZaq<7gDW%@JruKx-zmLjo72@65RjzHqQPGCElE&qzCptr+Wj{$Fo5)?zm#$Ptf7x< zhOG(;>A~&-vNqwGM_*wZ zZSz|ms%XgtK8a6A$Pr2Fa&x_l9TiC0L@is67sZ99C7}^xCY%vcxPlvVrvA{Ii2-Kc zG=0R1#^Kje5-*mF4mV2YfC0PLW0E$|fI0ZLiFjZ7H!9Bhk-i#5TXNbeQ>I5dlhZa( zNKV_{msS&pZNX%Y%YTkTtlr4mmA-m~2lkG6&Q;4sY^DC18cvUd0+qTy?e1S=(Vnuw zX$M1huVH0cmN|QLW#jx&cvV+;QlB;vKf5hX{wd!Xt(tA8K6vV(cY*-(RQ%R)adDU5 zT|7)0hQL!ywx?vHHRiQWZcavq3<*ds&UF);i(k>jRE(m`GXwew?L*yc$IeQ9 z(4omJ5VlP%Eq_0l(NclZeE651v^?7kSRDGlvpOz^<@j&kd6ep0*c=F<7TVYOo)ueI z_Srs}^+5Jr9}3)(bg9U2!nQQ7KJ3pvPy4%c z@E_WBW!Q!sLE!opbqL?Y6(cAr>erLr9ng_^SfuI!tb(sj5*Z%-L~^Dm8_WBvG#XmO@FyE(NyF=mh=P1QWQe1zRil%L-{7B$+WL# zA5RIj0WMcu7@bRJzwtVzf5Q7R?%C+{BWxdP#)2iGEL@N>`L*8o_Aq_&Q0`7?Xt)w> z3iY4z!TLd9aQ5!*=D$*Tg6xX|*r_!xPo6vpV44Dn6qAN!B0jjt zFbqb^p`~>Yhi$ZKFiU>}X+r})(7v29to!v#NkQPGMUGRl5aZR#v5}FxWF7caM#Px{ zNMzLHzRKR2itqUS_Uv%iM}C|z_81}MMsB>fFMw> zI1z%kSRCqDm#75(f=>0cf`(Q*0;jp_Y)R2K4~0o8_D?2U2BbgX(WwcSeoA{XZ+S&k z&P`rRfUSNkqfZaUO)rvMyC1@*DLHrLj95up{9>UZ*O%Ps zulMuo_hIEG!ngHGddmb~y*!nJ{z_G7_2Gh?F|!{!sDppFPoW}(ZrMtD`8Ih9|5?Vs2fWU1RZyFTtVd!n7AUyJKCf$V(ZbTqUzpk^d8E2;I#Yy9aX;tzL z+J_H&=yUirXy*^yPK~p_oW=-UF5uLIwli5*kqt_`J`XXqO0K24TC3yqj;ln@7;Jv- zr3IfbCe=$2t@c}h#O3YG?M64;6~}@wWg;e+t;afVOS{rNNGO2Ah=T+$zf`Ho#A2=Ga()9#}MCgUM)a5@e~ z4<0=0*%u^;yA3V58)%)SNQYYEkI*21@|JGm#U==Jy&F7g4|#=l3^!X^S!H`%=?Otv zOy{CFKBMzQVr;lu&J-k|S(&Ew&)L(2Zz`$FOW4+DBQ-f${vyzCY^DDl7w9S>^tw>< z-&K5KZkm_N10Q>gwKmy2j!Enxy||j36?N^TrqEyW4;3lwiDET0?NYPSV`)bz#pV5v zq&ij}w>yb0oczYETcS13e;nNTZyR;mL_^emZBUFyhZJI9v5|K_upQIpX!|OmpJ4zd zx#QwbEVyU#^XEC2pS=x0-!`G}Hqb^Gf`qX>sCDqMc@BE3o7|0#j)oI`6wMsj#EA~Q z=Exw4u{5ElgxbCha;m#UW5Gsp^oqq@G9IPa848;GGRotcN~z`)sCd_niHWUxuiQPk z7#W<*G@Y>nt+ZV&2vI()JRyG_&*z$ne`5}8OjZ6*k8b+mzGv#c+I#OCef#TN)~9de z{G6#Zd<0)AaSfpz4qUa2E+`=vl!Dx;OgN*2;7_>P331rX7_uU4_xC4!SO-g_zVOmM zHg}jzV{A-YYu}Pid`cr=yyGG7!Hw1@h!|fKrMXaXTI*Dlm)F_V3~6N_haSoaPp^m6 z;HGA-T^H9VE`PC?WSfCJ?dblMwP+l7S2s!8NMcJA>5e5pnHab0LF^VTnWeu4i8?Jo zPZ^0iLrmoqfUo_w6)}MUhpUyD8zxa{vb^XuG)v-840`Nd6PUa-S_GD-pVh`;O>J1< z>ob(MJoK~${DqptAxFqCjZmwBS&<@)xnr2UNsjgfD%cD;g@+i_ke-vkTKW3<$=Q+< zg{KvcC-U1=i;9X0+V4c{EkXi*9a*u1Yua$c$Wu?2h}q*UOn=q(&DQGjzdH8mxQw?lnV; zduU%%h)1&aTn&-b0AsV7_#zd@wbZBLmy5b8lzr-&y}raI8ad;!@wsR@nSxrB&s`85 z&AIK>)i96**ykkn2_Y8*_D8JB@hxFfN&7zv0^vufmqS+}ak90%vsJcR@KTZoifM?| zic@XdbxUI)ScbMkD1^m?Zh-&F?7{+jMROBVse8126e*17;B_>Ke|0`*ij++A(k-Md zl6D}piQf=*M+s{4=Nm>Kz!I1~7?Doo$TbLTOo9jIZd|xze0j?501}Yk6;GG5<=km> zXZ1*vEP7GrNu#Zp^ooDKiAFBss;o(KKA}%hjyH_1m5EZ=O&E+fn7XvN=beh9ASJU;!m2)d6XWvXt0l0Od5$Y-A7jKGi~#?X^*JZx$+ zuJWAN(5h@a zf#LsWMnr}l5bsI8O;HXFIc3BJwj8^U;JDYn=2I`k867kJ^!EOo{q7NwZ-xg79go_3LOX!Rtq=ETS|UC^{wwSy&xCtBNH|FakATtgDMXS@6hjLHZNMwTOXQhn z*gipQ*`b0Z-MN*QS*X7rp$Q!dH4pJZh=pX7bz(g>(v8*$u#ein;7ZDaXdf~s1#**D zH7pt0;<8h}Aq3j!6h7M&KPWm4_BK3i*O-H&1k4Tu(34N`-N`y=u=Dc|`{6!TZXK)G zM}WOYWk=7KWq+-#ek49)KI{%rP4KW1HftZ^VQ|qAH?_ByW*FKzm&tprh80!cxHoMLbyOQGt4pRv zl%$Ct)a?g(KnGEJlmI$;Nh##|QQv$NE;o8<^lc}zP-~4i`+ouG$pypAHeYK#}WpZgC1*-Hwf7!m_ z`yt|+!IAR{+-N~j5gibG-7zsX6kNhRkUJTN7=^!+Q&PI8U7(0JgHOQG>TNKr_@qa{ zl_a17(*9XP&MNj1S{{Uece_3c6>dm{P(`i`>qduqzW@zkK&`sv`-|uun%a8?X)@5Z zh1l>84UY9Fj`bvU3{IikenL0X>WZ$@^^o-5JYS|IIw~tMECNy|73|K(Co5rTzr8x? z^p$n=igf;*%s{0H?;E#t#jO7lJ6xhYhq`zwma$OrSJhHNSS{tyAl9se=X*uLjbon& ze>Orig!k$mDxv!sw#V#dV8IYVWOU)i;Jp&WMH5yrbYo3|NfjF239O-nqiMj4#s;9D zjRf#MGzj{6O3Bue2r>-P=P`4^O4Y6bF-k^~#F_C9LwT}vk1+?m8JC6CSI14I{z9~= zbGF-pZ~x#qo$fkt94T{jBfTRdw^!M2LTi{0>C;U8|Cj+oLAFc8>M1W(&##&S{{E*y zX1cpV&Zne9fcE5*AyztJk9l<^)y!mPpr~ko_zddk?SD?t;ip!cY{Kfyw<7%>-4v<1 zQz^xD8rdWFvNlt%tnkjglC8o8TMPfKEhG|lg^X;NU{?l+iU-wG?~qar>bo~}-MQCi zvXXx3i@gM|5YtNxnZvTY^dqjOqOp1h>9mJ5AfmX2avGzlJJ5x{Ux4c2C z0^L45VT{pGgcG{RxiA8<`)7$WR9XfG7xepUa$4k2RLiF?kn7wm;>gJ%pVs3Og-^y{ zPdQIqSF0E4HpMqaX>yHc8(LstyuX2_L-a{*Bn?8(Hgx_pD3r;1DkMrpK7`|kA@6p| zk-*l;fZChO6O@_d!tPZ)KC`9Yo_2=Pf}JE0FMj8tdPcU$VvdFauPzSB$+<70q@b3J z3X`k#_5Jn%V7x8|%Q6qaO%urwrUmj!HF#FL42&_$_`*T;&|Jn-uh6 zSIFv-O{BC8Bk>gnm2L-28TnJ83!xAI#Xc5cpuRL3dK*w!51zb^p?IDw6Zw=rnT)Qz zjrj9=k=5FNljOA5ze2rkri1@HgNEcSY#8DrzF`#=z2DK%QAryEJfONF^$GlQPU->H z$J@`eWx)aw;8r$4r_iGU{+WY8@uHw##tb7j!illVWiC7ahSZ6TVhkSG%NFBI$jB7r zoH`PNNUIsmH9;_EMi7)qOq(MRzC7CtGG>Mz(M7b>u^h)?*}@@~V7WIS=me#K?^liQ zls#bB)7vVR`d38FmaUR=@7x)&_9=tr}Q+G7N)W(Gjd@HoY;iIi91mu=)8=g}IgRuV8=GMp&ZVUAo6~ zf$8z7X!gR}cNQtgC&-O_BG>O6?{4&&9}tllqo_si4dt5ZZ!7F*C%_Vf$q6UGBEo6Q z%T6wY;#zy3oCauMsMvRqc-f2QFL*C1udhTM-m}}LFhiZQ=Il! z$=EL1Voe&A`u)=f{0Y6%+D07xe0+Qw(L}k29~%=`>``hna^8!J(;=i759|D-yI~6x zg#=BKNvf->laAhfu6U%8z}xV&!)<1-^!5U-hC>c3e(3D-nu?4jMNPEkp(#mQVO1jg z5hoRoEW3bvAsbYsJPM2$laHhwbvjLDmv9>LWvE_#8N zxtmc^^ndoavXrz2gC;2ZLS{4!wnWI8qRD!^mW=+mSCaE7Y(^-9WddvmGcUcdcx$wu z-WhA&Jy^>8Lh8|D569Jh1te%F2N>V%9~#a7tP?Ka_ux=iXrRA;-h*jQ2Tr71`T?jV zxMRmdOj0=>9=URz5;~YSusAh*BB-pypk&}8=F(z+5Ta2IL}M9T%$~L=vb>!zrl^24 zo|j2A*5@6D!yLBJfS$C0d{ePL7Rr*01PRak`InH+)XSp-jk;hGnk8|!X+aFQ7C~X* z2gSrC%`{J|rjx`R{ydO?z|b*|v3`$OTu4M0GHo~6weFx_E01(UjBg4JKks+XyOYE! zJ~rYA6%YZ;gS&AedTyVbqw}HlW7y690a>ru$Ktc7qY0x=Of8HI`sPd-Aj`lUjP>aS zD25Qo7^z?|6fT?&D-<6q4!jSv9(U&bEocOaw#Xfq-*rBuQw*7NM=(1&71(KW*|@V5 zL-C~EBidfRRvey=dqxSR7IhsMK5=Yxh&1jd?qqaOvB%C|fg(5+bnl z9xiKo=iJ@I)a@-JQ3I!s4W;K0%jD=(R*8l5q$xBE^;-RV#vMB(&&yL-G{O$IZSCrF z@MoSA^BqJS%l7k|RCipBX42A_6SviqzhI2)Kg-AG*jqM*vJW)vHM%oOQjxy-zu2^w zAlJXbJz9j077YH9l5HD!4n#?3H5R9A6;wF$w$oz(Zwc%GV+KFM!AXMT8JYFv?IUf+ z5(ST_NosE?L7`IGZ9;Fr3T&1e@dK_VbqA?U4$vc?5Lch!nUUd(TIn%yCC5?1`RN7T zHDK5#m0|Wq4O}4GpG~0-($*`|gh{IXqKrXHOsQd-nsN0;eM-%U3C^0#0+=D+Ia@D< z)MKirFzsTmNlVKeJnIMgP|VF+r0B3~1^bpszR~aX#Av`Dg0!^recwlZ(DW`WlvEBN z3AB1P*L&LkSoz!{tp*OZnJ@lu0Re78OO-J2VaF7r#@X+DaIR^#jgIPQYZRoT3C#S@ zmIc*7N|{OgFnE0dJO{bJ{-|{wDTQg9cxMu=$kac1AO4NtU#EpDq6JsW9wAk(BWAQE zMh8ilPBVacwAh2G0^&50eXX~dxP~IMN4C(xBvWoK`>yt2~HRi9^E8(WS>+5Unhq*gq6^Ej_`bLVZZ)JVzI$T!3SL%6iSHTL0j-vsWl$ijal0(PR!)tH|%S;+WA zINVYha`h<0Tm_m&>xi!~aiORGfrM=%7$g}}|41h{&%B89umuJ>{P`AIhNd6l9V0Q8 z7ma3g3n#OO?A4Kc7;BUjC6{5mkKEX5^xh4Bn@>IZ$TXfs__X5sJZE}BTXuIT<^bd4}!(;q_l)TRmMcwTGoUeJzsJEP@ zb?-mZ*xZbO_Xv42?!!BdL-fZUQe+zO$3|-|e>b@8-&FlGGT@&K4;UZmV;_4c?60~? zC9)#c7?_rb3*9D%Dn^9-sY*vuV-BiPBi@Xd{$HbDTdSR|rCBI*F|;7kzy-}df!kgk z8EH{Tccy7)(<|Nn7+7IV0?&_&m?dh{8kOK#O~MRO;2+F7-MbVGTsb1v`@>5bm4S~% zVW*asmRg1?;@njgw@1pe9G9 zK`5hRxGyCedZkx2ne(*mC>qo^Shfv7IdNcm(jUi4(GC9No#}4PO`k*LQb|uUrY$l1 zSMBZX_s9HMu{wz6M$mV?gs?&6@5-Y88$`BCNv%T5Opuor&|4**nVg(R62560JE8dZ zyxne&fqqcKh>ToTQNc+^8gL`#Bz1Wuqk%K6H_8-h*sB@wOV#Ux21RZ^pMpgZ6HHxL zctwH>k3W89YQ-*02waI*w1z?#n`-$IQvZdnM@RMY5r1*>9H)KZ$JIXj`&+b6Kiylx zzcyS`X-X1Ng&|;*;9-{ji5Laq>tH$f0S9^bG#Eh?_acXf4lb0s|p#VMQa|kU(tg z%V;%6><{Q=n>Nz;w~&S-`m-h}k+Pglutn{~O!&UjX0KpcEc)AKL>}d1Q|Z~V%;y}7 zDtICXIR|V?aPF8lA|*t+0>eBl#HmOIIX0wI3$Ph%kp{bfmMbW|e~KDWiv|U>yK$$E z48Z^~mFoOCY4}yrqY)b?u|SR^wvzvjxqJT-R*Qb;b6I3638{c+_?kv`MOs^Yg--ne zd0a+J=1>q$WLby9=t#|2s&pKS%o1Cogre6eiy6UH)oZ?LIop}2;$}2>u?MSid~|FK zq`;4mPpz2##3oU-V{i1XV}GxFET3x{^+KJPQTnV*or%3-PUI(8Vz+K>Hrz&}P!3EQ z1jYRnNw^u)tkx-W^eeP|dyba+c~ogR+5$;CMso?jC^wU!hcBHLuHoUaA!97*_5nJQ zl4UUns)E9ymwQYR#A_Xf=^i2@gEE{8KZ5VgCscG_Z*LOMu^hmFIyj(@*y2rhL3L<= zEv0k%|6}XR1F_!T?LShQ=aGa{p=2HknL3pcQ4}(zNJ7R4W$4stG9)EbQW-LZk~u>u zL#9m05K87Gc~PW(YkywK`Tnjy?mhS1j@a+s&tA`Z)><3x@thAoi^jo9r%ukuenL^H z^sQ|ErrhXQYe<$Z9na#3)8&1bL||h2kNRvG3%LUM6|-*DlNo6DKB?92{T2HRZtK4q zXXkZIgbt4J(o3Zze@kD_Hq0!75;UY+{|8`vEy6b@|z-t-{rCmXf&y9_9U1n3d zsaP5C)@mx<1sf8cE&|gU*fSZ&IT3&OW)JL9bUXq^E^Ps{VfhsCmQ4u9=q>XZA3%e3 zCZa}l4>m!gNgKjRLN%f8zQf^GC4M^McVk<{y9)!j>3-=t0wr*?us^Y-?J zhaeRCITgcG8==}$h6!psF|p%6D8C%s37v09@F{5uPZJqZc)VN64hw945>y?25|MG+ z#(`U`2_!xsALtr;N7Nt*rxOl>pRh8*7EQ=A5_dftk_BdTFN|Yh$u*UBWZ`}^CEjGZXcNzc12LbY_?jum_< za^b<@biLAvU9+Z1fZj8nY(ozuB5;l&#QWu{Dzh>9RF>x;0>vkSkd$koysCzT z%f0!p%=*4(^+^k<#ElTQY1aNeL?rQuXmV&_7a1x|#C9YT7MflWh}uYqH!<81?4~PE zyLsyB*l9oZdCNfnQek2zZ6cA0LPDQ&!F`cmkgij-(2KRF5E15bgVkD(68*|ve$Pad zIYx&oNra?5nfEq2l=EhRlP!SAGL)=EYIGLw&szOg6?NK$`2C}_xYdPUgqD?joFEAn zreb7=ue)2^BxSfod`sO2mjmqk4&AsI+E}?%UlkxHw^nn~fU6#ma2_P= zBn#=N2t-4m8*~NTv>J!TEP~E)J07TqT&5``WVgLdI;wF;36t9ULyeUU4LxofFJ2^F zG3`$v(RXRsHbkpoFH4H_Xnpg~q13RfEP1Wfo^mNS^ji5-Pi)WBNn-JgWk69vi*Uxt zBetjNWGN_Wt~t={Pe)S$LR70KFInyGtflkmlVte6BgpQWFwKR5hW3JEhChorHdm~^ z<3?&1l#&KycW96j^7lgmWUX;yj+fNey2v)x7Gz-`7fQy0ElB9BQD|tDAjzjhlkhu2 zCoaDJ3^PJzpT2 z)TXO&_5300%eOLQfloG&T`@i&zqzGX)1P$4Dm+BCjXoVo;dQ`jD~Z$=9yqR=nDWjy zX9;TpBmAlH7|z$aL>6=V)Z__yF`a9Gl5ygMv*KqjPH1PMfoT^`Ah+0~NEsrxH9=}O z1PyLsXB5zCt2KyAqm#zS%4C6cMn<`FI$3+t`IxLO>;sL_Z}V%&MkN6o!u{*D&~31& zBV9vD9k;lCe@Wlw=I`n>fSKyoyc@mT+3el)1joXxzv}zPVl!@feU!HTMsK=?#E}m_ zG?Qns?~6p-&H9sb(hf&xj=1SFBl6j4QkvvX;hxdrxcRQbt*_>e4}$iQV}Fg9mQWca zBHriFtzc7A6HTI&(^_o^MIP~hVXLJ9%1}C2D6*=3nCb{bYGZgnnweNU1hh7K6a43N z7kNH*KSDoG=8PIvW}-fPpxs_rm14RjCP4(P`98swO70}el)Q|99H`1)&*r-?po`Hg zsxTVxm`>JD!N7#+GLPd%V93jBr#80H(voIFOtMkZ|91~q<3NP8C(QWh!{g>r zp1=LL9uJqNe%Vj-c>tbYywseUZF#=f|sM4_)yOF*WUHFxlJO-=G) zu|Dudk&-`{$+d)zM#{R$0VKA}SNO*BduTCOt&jvK02HC~=)FS1Z7{hzNl<<)O)sGq zXMPwkdXq!MHN@x;J`7d=Hexry(j7!Zfy0K+H#upoi7x*gm-gVLea>9?KZ(_iRFYG} z?@sF6o{zZOh2=k}pLs1Qny2x5(bV!_0YXmfU5SK%d>IEkIXxIi|06YMj0n@8j?Mxx z>1WX>c4cO6Lf*M{4n_C@u-TMs`w$TV3z4Zl|UB7 z5i-!l()|XqCNdEH3L?0aaG_-AVnX|D3D`%Iq~$+&9GOORIN5Nijk^TGILRzm*YJ;% zRHp}3;u3knhpgxeAiSHT0teDzQe|c8N0;BMhbnL}9mL;&qx++``R8#goRT$u7-qm8 zQNJ+*_Htz%`%>mYWW=21>;Yzzky|R^RXib3ioH9LOWsnCLBhA1!V_vj&UM^y%QnBu z0?Ky-9c#OLk^*#DayQaMsAq zYz(=op<+uZWF){)Pe4A6Y3SYKSNkWK-Nxn$4NZsy1JI-aJm)HO^?0X4VZ{8&6WuV@ zJv}QCk#O>)O$q7rwrb5T?vK4nGXEzQr?&7GGYI)L)s!u|oLjAo=Be-X_0_Ftew&Rw zmS#4Idh@DpTZ>Q>-|sHvN7H^H$;IBxaBoE-cO-V%u%skf*O~w5f5zl3 zJrG-s2dlnGA~GAq3|(u&bz;?M9t}(J3R=*nE2vD`Y)Jzj%uHWEjdq$%4B$$|E93bHjv;KYF z=89_wapJ*Z3rj7mg;)Iq`9y+}+xL-*iyK61(a$PkMbj6YK_$v6U?f9Jw^qkWjwp8!sXkLGpPB0}hD~P^cl=ifW0ai!G?e ztYYOVSE!eCo;JuJEM^?&>Ms8sXb1oM?$6e&iIu1hFD44?J)%j|noXqi^jB?esV@Bw zQ6_{&k&%J7gK#XKvXz<_zg@^K6+;ff$tAi{xOm3oKwQ+sIEYi055P*0*s}0VL{c%g zG>)cG(s4EGW1G;)oh@|I(DCTm0AA8WJl%)LeL4gYl7U(Y8JYC^WW=@hKn|G1r?6|N z^pV*=PCbb^NF1?f?xm^NbFm~?ONI=Q;lmrwLP!KX1}*+CO7W}3b-dC8dOrUq3_U)O z3z@bDoh6&*e z2^{fF4a?g<5W^XnnpRe5Ui8R5=FnJuWPyZF1PbpUbf%aet&p9Udy@ofAaTGj-ZC&K zcMvag`)3RpfOvbzB;GFP58|P<8`Tp@5x&H=kC!ec)LP@m2ah9$KBz%--24x?OjP>D z<&JUL;AS66_u+*c6z`=()<}{J=ova~*aO*}T~7*ys%ogv48%oEAeB}G9rhz2U%U&? zbiLGY|GN032g}`}te+!28$LJ74ZSrXb&z+8ZVs8tBi)ox zOCMN0R@P|u6d-zmn5Kx|v(fGBa5NL~bGd9~?yTq zBgVvA8Jwty%y}{M@gwLRNP~pMPI(=dPja1IfyC^PJVIER#1XpqU8!yWQKLP?&7S;z~oGD012(PFUb zD0xMP&S4?e92xe~aGcD+ju0=vLalL0kDx}ODT#w6>e%xut;qvAckSrQ4G-W5+ObjH z&&h0!u2PTHQS4vh-CMfFq|Nx`aMI&=XhSP1D?x?pd_*LBKg3(x)B=C)CESZ}>VB*pk(&QG zM`pyj$JgV#?P|XEz*=c2PG=yph0ojU|a#3jrbo!f(t`Zr;?^Z~)RBu)(zauUA@nRGslq2_)h z%ls1DkUxXWWWNDjw-~Zg=Jt5mr2m29*m~NLd#7Q_?&G+s@G-SwAJhL0G>^4Elc8-U z4|zeTX8A*L*sCcn}b1KnKq&in6LbFt4w4;=LQ5D8u&Jc zW;1FI&Y%`7BwQh3iz2jDosXRcexL>UpbAYmsGB6m3R2s4#D|wmeWTYK{6JTb7>`S1 zAwLy19QibW`#6+X^ z=Zl)R%9;J{U8A5G%zoFg{mM}@TA>Ji)%4Hl_f|AFXCXRb7FZC+y_;_*GOaffQ2AZ~ zYhf6V{L)rk1A`||e0qp}VPd@Z6ku~am7NjPgI4y>`)`~y7jpQK`6UTz&n8-NMU=jN zj-4VN-&L8ZEHEzLYW`uCPdYo2@~ZL+}ryub5rzkyb4zxt+k9{|E_wTb-X-q zxFypCUuhD5g@ms{OZCTPKo2yspnOIsImv%*=>Z@&>W_;bI$)HfhqjO&6u(2)I-a^t zy&>9p&Az&%f~1GCU&z4srJ*OxfZ_;=Bz@2~S6awO6l}EUleMrP|RdM7J$uV%N? z>tkYvG+5znK_rY_`CNDlo|G}irrl@wN0`WkAFxX9l)WrkNaK(V$;&>3TGw-u8PJY1 zJy0JYdDwO}tgb|G3V{XQ{>yB{g9n#Dk9kL%^Ccg>Q!fEVa)^lhT*0ua`kI}@(_+L< z@_W#J9?48hf9N>v^a@ox!P#ivs9?f@EQDi{7lM8pDeDRSX3;nJ(1S2<*mrBX=w*c* znAjAu@4)&-fa18@G(HE)DEjgyN&l^VKR@@uVq!{yBbVdgZz!8ZGzn<4CBVTYpM3xP zU&3*dwZqHZF~7!$CKFEAvBqBy_k8=6G~Dt2c<69Q-x!P1qLx{H*LPa*Ci`6%ED9H@ zyYbZA?&)T`tc@y*kIEc(aFwl6udg~5dZ)}_w`*yzdSGyoM7hM#j1mbI{xb1LA=0-3 z-##)`NikPV+1XpY%g=wO-;$ZO)0C*3j`>p8ZT76>_1m;1`(SNb@lW?@Q@DlLY{tH1 z=A^2wZ11qVQyZpJ&$e{8EW3vrMTu^f_HUBg99~{Kl73@0^V*`6mK8o)HU7`=wuGK_ z6AR&1c}u8OvqTq#D|ZeU-PcSga9WZ(E42J*b%lAg<{~QBK+DkZ;huGpMi00xrl>6~ zu;Yt!sjS6GH}3vE<{sv|ZS@WRlH-tBDfbSK9$1NQ))N&9mdo z8mhNc77-EAzzD}P0)Hh{O9rF6_^!1k7qnJnIQ9tgO^MWy=6r)|>o7x(>k6jk2VAm@ zHs6B`)%z(#uV-r?Ki(_Ft)=DvK;+GIMr9X^ z;YGfL@6K4hO(84QJB<&jrQS#qVRC3-8H0rjoPIB#?eU6jyEd22Z(uH80I$o)zTH=! za;5gJ^OWj3J1eXrDq=0=nT`^tjELN4&hIKJ?wN|Ozfxc#wzU19ZL1G)E)d#nfEwZ% zy?n-0Z&!2kotL)#EhA&q5UdV8W>joUj36&oGe{nsqgi!vph(BaVu~u{xVrO2)R=pY zfpB1iqy#B}H4P1x#PDhk$uyN)t)j>fJo@jZ4CMpv@;bku6pvpHHAyFSLR_9?0LM`HI6 zm#=N2#mq87*3nyKToFo9re{Jz)^Mmur=V-}Wbgj`>h!zRyPkPkq8;B)D}?#JJbc5S zqS|kEcUy#Utz20Ju}89L(IVALqB`O8ZEx_!<9O#*&z?GEvFp{H+i)XSahuHpwWMr# z;J`8PZ&dClKQ%r_yo4L3bMT-Iy+9wyMjRR)bNZ)56w#B)@*Y1=Kh%BnToX=T&uF|8 z?Bfb0(yO9Fj1Ou{eU!ma+puS$1w-~wh30DfnI4GyAA-YKnT21UyB$YE!A;irQM(dzV3bxNgxUwI0ikwzl0tc6L%m zSMO{NsyH_^tS)B` zjs9Jpf(VWM%PkM6-05AH>)kT&-CLI9$f~(i5ZAP6({3tyWCK0tpL%AzNA-r_qo7$1 z&)VK>HcF`ddl43RV?yl)ocYr>^Dg8Gh=`;HSy1u3J(GPTQvm7_`kkoq3*RpBcZ7;&<4&CbAifT-f{fhr)&n z*IX%V?bk&gPT7v(TJz3`irTqBJ1+4$TaW@snOaD*k23JO^Q^|Jk7Rm)xFMP*AO z@Ke@xhuw*?zSU7B$Gzhg6ExvR*Ross`&hGo4k=ss$PKTwg~tlNs(D~>dH?mElvUG$ z#GV=@?DhDu$CjcZ^&B0O$@=vei$;<7w!^DSrgcr(%#aGS)`hyj8p$C>#QZFjuprzF=ljkcR#Uj+og&^=sEmzyqTg9J8?rY?Y=j2#^v&e9r@EB{L*+2K7~Utv(6^wq`_rc<*V!DlkGS%?$>TglUTN;z)Xm7dB9;tV&pYG( z{lA6d?_tX~CW03SS-v7`f2{Ta=F#1m;HiO+Y8TeF~b+7`QBJzLo$zOnBXZhjPRGM90p2Sf;^ZolFV)n8bacL9|{2VXv z`l#iYKv~;2apT9A?cd!D?Rx~~Ir4??z#4nb0w?Cg0_$eWe){?Nql#4S97k7UwN+bZ zHi%KiyG~r;bNuz;?}vg#Nvoz6a%@oj%4plqBDDVSx{bEH3Zdmu6W4MS$_@fWBwV{L z+@^wWWHiizSAl7uU{LlpDQX4H}X9UfoeOaO<^H&V*7N18&EfT29t+6fmg)H6D}8a z?b?ME^}MB}uOEB)@~j52wT31BDym-ggZL$pZgnm3@mh^x5y3sf=9NxMsGvo?TTNso zq|L^;Y5W>#b5eWhjYs~6@#+FQhI&V0?+QvZ;^yCBuC52JavjuLn{eTfR&Ic^ys&)s zlbyJgw71OT9$?jJ9dz|^#V=B{_SQERs)~PX!8OKKbx$=f2c2dJuaH*hf5eA?6B+9TbDTX~Qo2<2$C9-CoL@KJu`J<>7{5ySFQCR|#**6k`xG#1`@ut)SYJ4KzEa>jG-{5f+Hm)+(A90ZU1=LY@+TV%rL z&(>k77BvE@$M9oVtF2umW|7k@l6uP8w+=Tp!811BzqZ9hMI|yfA>LzL5n7Y+&RXce z;}?fl8>j;<1DE(1;X->8cYs+mKjLYtm4h++W3Myu;7e$1fCrCYldl-V-Lr=Vlc<^{ z+b|!OJ<;h`LEgCDud8d0qHC(WWy_iDFaL@}MMmaZoX3bsy~rWcKo~nxr@!CDb4Q*X zQpAD=$qLnTr{YMDe}xC%x;AcdN11)*?;B&t37H?(}yz2!d2LTg#5lKowI;h5S&WfmeDw79ZX z7Wf7|4An9`>QisWi5jT8OClArk77H{FdYIBftZzsz^?`EUA|of-8HmdHb z?Q8c3ZxoGqgigF4TEs+%%Lzh6Q~0X5IO#f@&Q~}n1+MP&t`!bcCD|AT!b1TB`7W6C zZ#A9`XJAI&dyXaY8BjdZajhBJ5A`A-)GH=XQdA4Cyyrz_86`kx$F=zo#ghljp)j0Y z9-W=t^r|PpAYIdXxSC4}5!AWY73TZ7JfqtE4^E_bV;Ln1s+*)H)|2AvwLPfz-uV?u zWze$s&bxMu9QGHU=WnRf@>FeJbUqsYRvWR>3?77=$Fi`IVlylw8dL}{H+S>UnXYto>x|f_> z6NN4y^0c2kmNYWr;4PqXKy2#&huCBRaYNN>E;1mGTp_1*VOZ*+JhuJ&_cQMm7jmx{ z8W_a26$b)psy)PqaI7}oP`SKsxCc5jV-Z}YqOcWrLo=#u18u;o^72&dcX1IkK7{Ix zM_Y2tqZjkzV=k+(50@KSbMYWnvV@mAbouXl#EdQS*#xi_JqwsHyfqAsNbVDy{s3SI zN4q;a%7sNL@EFAP&^#r9l2zYjm_<=oWUpO(ffFashI9jAaygS72zY`Wcp*5P+i8la z=7=7@n-G017q|K2W_ATyRu7VijEyah+OT1R5b5m--7(m)F`jf*iFCRN4f62u&C^KG?x%cifumi`xa)q6QCjMFUEa#OMj(^>FLhkFD zf6X-u*4TxS?-AelAR$h;rgz}NAz%uswI#M8B4dzH=S?tgeYo|zj9gU(^Z1LoMXbmwj!8G)D`(; z^GA(XO7!r(33SAmaHCK{Dmv8-p$`!H>QI#2&LS?dBG=$V`lJN}SunAHR0}TSC;XBY zxT|$5mT*YTOB)IJ!I{dXrx^}%;z^Wf!QbM>7D56g70Y{AUCmXlo|ZX%ZocW?BewU= zB`whE8pp>(M~ih=RP2QSwqfSk10rq@ab?r{tn$TH-!)e+Myz$X&?*fHj&edw%qzN@ zeS!nUq8bg@9&77(y8YNB3}EQUvmntw53>D3h&0LVJpMNE%pjLz-giZS!U z{}&keT_?_M7dl{a0I8OIgj=l`1oD=$X_8Yt<{6E9q7QsQX=hDM%`qHRrzxn8>_TJp z2XSkBhCrL_Jl8Y>;p0tD58%O;x7TwBFLOO7zXmIp^loD1oFe|LobiZV1};nHm&7(( zTQ$Qq1OVnlTdJDSu{_9luwao^j^Q#N)uHt#&^A(m|1VF z=HrM`-zV1Y>*(0xd8w-0`Lj)T!7OHhW|+_ie!}t76onUE(%mI~a?YGNYH2^UCTMl(RB z8-ep6MYF2Mck}1hh^LW{2z9A`^G3bSL4$385^FH%lT#wMZ*lxY9zVW=8n=_ll=0;g z-TG;!PrWZ5lJ4!w7#Y1=+*bLpO~LJmWAvNy;Y-Jh3YCRbdO8mNa5!z5euc#TD(Kc|@{UX+|R zyNx`_&ljHS<(O^|6C0aE^Md)?R$rZ&Z|~}8glvaWhRZ_3LFF` zSISjNO3LZE@jqZ*EFs{f!M0w;iD;V9+SCpdiIdvusIKn&Cf?9eY@weEx;t`m%^Bdz z2s7B$(V-8PmogK;S#uk83e&+ILs}jzpi`WFs(R@6lB4J(ul08buV24+`}Xbmp~K2} z{5k2FX$SNCo_siI7rLe$pFX~d6jk~Cz1Xx1`SD{)RM1Q8!EVm%p;Pxn3LZ{hzPxLb z&a{0U<+sk8Dw$P(A7I>yWp%Z+BS+X^zS`1+TFc*Lt2TB=H8>jIR^44^>p-Fs0#lys zMj#cxaAtRM&aq|5ILYNwy2TGaLfTUTum>w6RQK=MgN_25&1d$7JY~au`y_d%|0*$s zh~qWkB=9Cs7`<7m$TOL{)-F=EfxrwMUMpuog=IpP8N=SeVGzOqJ({9vMYNKSYfa4l z#)2ycc=2Av1;rotNOAnhuh2h}KuRW07^2{&hY6-K$q3UTm1G-bta1GKLn2P|?z4$~ z_%P(^srI1h7pNWsp?3Fhe zjj|V>wNupFHXuORU^RqP7i!*2|LZ?Di4cA-Tqep1z+7}uytR1V1s~Z3Gjxj%d9_O8 zTZ+39`n4V=v}M8hNj~ zt0Y)LIUyxQ8%LYq=CFglh27Y5nunHZiXV_(YnUaRPMNWR6u5sm@s zs*v>w8ojjKFT&DI>^QBn8f_D+mMFMLqkG$0Yup|sGx%5A&3$oAplHBgxK>c0q{b$W z(NoeA%Yu@k3p*-q}&+<-6uB_&7BC@Jys@*){^ zMIY20$B=cpOzTiedZTPX&!8S!7s@>%Eln3>QuIEHr^_#GliqiHRs{FZ^S_Gmqit%( zg3#^UZ%EU=4UlFYL4&&X;1C8Gj1^g!JL}O*gd4e0U_RF0_0JWghr42;>SJ`^0K4Tr z?DqhgAw>0qH4s?4@F{ZrIx+kcbA|NA6~>IZ4d4NUd+_PWdi;D^fp_bZ?Cqu5rsp;; zzEUsUkeN;wi9A8h*68naE6Kk7mP+Pe& zzy$G_B@2AR+nX*;kApd`t@5kCoLjoWUH%xarG>3wd)_WCbm+Ns^Dy~$ZM-gX3|J$B z%R3~x%};!7*1@SfARK7<{5@aSg-~kK2Dgw_r_6v`w=C(SEF8}Y>a+rf)aiK%OTV)L zoEMX>Wu!}qg~dlj?KyTVnt8eKFS@kCaYQP!E4QG*JPY>V<@nw5!;g;? zUk?mCsZ}l(g2NrWcJTm3UT+U~L#Pv}-1I1w%q8TmR&_=Ufg-=d(X#*T{o%ctLIwg+^uZTyS1a1K40 zTGw-&@>~T#0sgPWdDR@YHtDjtbK_Wq`LW1@uHOfyffrgZV`_DCvm)kJ=el*#QjFRTi(OJXy?EC<=ZBkr_J)%Lu=8 zUq_!d5YlD1mdL!3ycfF~;~;dMIVw!RUupYW#{08q-2O=ExI72($AQ}$80=lQJ!rY#;)tz7^55iuV43~}| zxDjZ9Vj!NfVnEO`2u4(q z_x(n1t8Hw4?1Y9ic@zX%(K9_hE=~}le1z}Ce)Q`BV(Fw0UHtzoyBcV z@68|uGK`6hjTPd3mH;;cUMkR1wn3DIz=jP;%xeq&Vp2%Q`EZ^*6lPSRLktkKgAg`6 z_Ta+MxB{BDYBFzx&R!t3;W*j6M3@%5QBV%szXuB?^UDK%@opTKg(K8JimU{^^QJ_P-088L~>945=kF3-nX2_e05rA-N)DSjQPF-uCDS? zZj+vOb^&Nh!aHVU=)o4t)i>?zmWKy(nhpz;@L|{;RaLtMN6Gh3!^!g>m;lQ^D2|?< z?^b|UrVn?n6~7ngUTU3EpK258U-Z$rx8L~$1WGk`dcD8Ha8FR| zgTM|8o;&*-Z1xE)zt5qbD8;U$6V$4?HC@AJ;4~lyD7hAa(+Z(`H~WTRur%JC(75ab z;tcv_SIl{3$2o6!yYjZM_QCCdzNF3iCXF5ewEH=!mL;~FJ23w4+^?-YUP$0-aQxgT z^JjotN|0?xXs9Cp=aoRP%#I<`>EY$e;vU?>_z2ai>guAM<*XTKGK3<|oak37Vq123HAZXZcU^m zbCJ6vC!XcNBBy0=HcX&4D#+__(e-)5FRcoIcYF67RMTI!tgy#Uqi7)TqstSQL-#&M z3%u^%X}~?`8)LFgj5p!mPs!oycBGaU>x3w3aBVuf$J451(3PKYzhYe;9Io2o^FOXy zv0{%jr#8V2+k$rA-|KL9A!`{66~s=M<$_kb?O}V-2Q|!;RE0cNL)tX)Hx*Drr~sn` z9lFoLk7}v}cDb9;{;OTdeAA*g~#;lw$=AAq2$OS1Jd~TG8FM3cZkkvNXYEW zF%7qFExjz)L8+W3oCY|9_JKDb>8eKkcKBgIi^V} z?FD&uFSUHh$RiB@TYcLHJ^hWz)8kGx3O^Jcv~Y~pL51n<^XcAm(0jN`MsSC(bx2@B zL;H70#n3A?XRLrO$ZAKCQA{|%vSW^H=f~FR&rrOhpFF8XDOvU?Nal56$F2GtML@3} zElthUfyKN9=_h<-IOh?T>2ORP|5$F|KmSZwx^yW$j`PdHc+DpR?E%-70%ixY934nK zp|Zq%093$cRbX-(978lA*q%W~zpMF~wL}iJYpw9A6#BnuU}3ls9a zf!D8_BQ4kIImHfgEgsUQhEDv&UC5<5+>6^Rz8e&TCO&Cbc&SY0vKW^k>gWk!|4sVt zu9b&KoU|wLqb_Ko3E=>?a}1FTIiF0l<8;M3KKGb$@l>{@8|vw|hYT@$JIh<~e=g13 zk14-JN1o}OwjQ7A=TD(JZJQa{2g}OKE4Hd$W{AAG(S5CEyPo##4fzL4=K5aKjL9@K z;m(R)dbGI~e<0FPLfo^&F7w@2*DPfb4j)n(f_e;uYtQAWSI*zcV!dyKgWqi~eP=q$ zp%dGPx!&s5b~4?;2#=TbA(n#=0m5C3t}104^s~(2vhv6lbBwGs{kt66OW47it^CuN zAhrBD;A$KVAET{XS%+i(ip9L7Ga}H)ztuq2 zv;n&njvVd1d#}>9JCg84FuHp;p-WFb{oNkl!q}Ctk_3QY0PFJ;gRWSdkf&YMr}-(1Q)Tg`HwS{8`2 zw&BpKN2?NcJzA(z`RKFUOK$g_E-o%Ns#wzY;>10A8?ieU_vK*@h{G`8dzB9SM19iHD|-V21Ixd&UPeW4TsZ4rzQ!H<33pc7c;o+KzHgoU@y(75tj1M2 zb8wL$`S3EX51*f~6eU~LiK|9qHj(lc7dC{H-c3+8i3AoO0ftcv1>~o}_S+ztttZAt z1~UNIZMxG@)F6y`!2P3TdTV(o5~ju!e|fujI_%iL#rMs1*W`@q&Akr%ioV$IwqsU* z1~bOO|3T#KJ3Z8VdlyiCyqqdKMolupRyXUNzi*KGIjXRB)za@73wC#6a&uwSB-Z|*FWYo^i?NrT--bJ!<>G;)V(#60W}!}PDg$|b!SliVkm;1M zYy`K{qInmV&b!d@VvBQ-9sx9N3ZVos@))9D+p(j-ImCUPxV6#b!R%ulGS(K+Pf;V2 zG9rK~xwvh$vV4PLU@v4R5SP)C9vKRSqJ$mrn2dZP;a%H&s67puoaoHVhE5D0c0*!x z1JMXn<5Q@c!i+%6VTdFc!D0<`lJx-A>lImuzB&e;g#0m2tA$qzl1~j|lzIC5*O{lH zSd2H$x{&7zGmWIItOcVs?m$=p*D1j@!(DM0fsvO%2FaLhnb$E0(F6nqpLTVG91of< zJsMuldV3~$HM7=+?+-U(93RXc8iztje~Dh;{On~kx`pzzyD(~2c0LSCx>2O0!fp1f zYWbVhmv$QPud#FV;j`M2C$+Dw!#qM!Bl)M?{*R};caNbL(}PCQZs!hFRWG1)t8LT# zSZE=^-Zw26*J9dFW~Os)&tf~9^pi(UEsG(gKEeSe{B0|toW}N;Hr_UujKMsS0jP~h zAf`T=MW&k2I{lgY@JOJ8CmYphBsrrFQFTR8Xj0OLK^m8=C#7CyPOa~6x?Nk_0vmLE zxnX3O85)fMcu-BIN;`I3AXg(t7JT>eV~i^M%y;yt$BH}~Vrgpnf_zw|_@ILn2st{G znj`=IPB@4Jc;D{4>TG_nOh_+W#H~w}{Zw|HuZw40kHr9WbP8CK+>a&Ue~Nm^G?|nM zs7Q$hGL#q3^5HuU6cbk|b@TPV=6|`*%lsm$# zMy6v`?bG+6=78?>83q+6y=2}dAvd3iK}T0*!UpBl3pZ~ui$;fC4UP(IaVi-A@oWa# zhvD!yj8rlMp6n%EuCu1P`t3}Kf-Ly|7JoiULI5RP*YTq) zoq)mau3Z+Mve&^QcU~{I<@!MBT!Ht?CEs(5fQ^56|Nd%b(YdG`=8AL~Z$cO>6EU(P zMdu1O1|FZ>LZ}~)B7a_zw(_yxz%7zJayphhyQqxy328q?T_o(#@{fNdvFm)qE~3AJ zlriG1b%!_PpFB6Q<3`dBicNw=1Dz=@-!qOIWq(lFag?o9R(^rb=nll2Y#?cU-E zJ0OvJ@^r06@#TT*9d?GpwQp)`i&08gui|;tCpf0j7iSq?hKx2$o!e<1e#_YS_&1MC z3W)g4Tx_lxE71Vk;WHt;2~P-79btR(7YY*bmMiGOIVV7}2Zdibm~cIOvg)m5XU;s- z^i8pfTN_Y=7hQTm>c1lbo0y`bEJ(to$oZ#ndvGSwSQ6kyE$oDV*S8rmJHHx$A%L0+ zeE`C9ZeB_oQRK;s;gEkAGd9^YF(=0O+aDL*kw3-v=_mv_u_2kq9Gv^#C}Z=I1OD1-~6b+%KfO&@GM z)P{xD`4vkIYzk_3F^HM(yq`Nw+SFR~&c4ytZ@(#8?J2vhnV=74PQkDBWtsh~=8&Bb zsR~#%o2Rvnjd3K1_+{v3Dw1*b-@{^W-cwX&<^1~2zP{SrTu)l(^LGjW!?jjvWt)xM z@iTC3&73+#SzcB)##yecrD6^lu5`b8wTsr*JjS21fX2{uaBz@`Y}5&KSK<-1?E9rU z(ISzdikBCY4sfSyxt^W?8ZZ75&NgXZ#Ggoo1>U+9i}yov=}Bk_0@dKvO08hhS!pU5 z2{-x|DHZ`&C@5-Hl~KVeCHBcS6ESb-#}>ntNkkH zH}Sv`X#sUst0UT?Q>QYqxg_ zt=+C=v*8{YGWh&D(^fhTTW9t$vrtl7pn@ZUG0OD8|ElZ!WQZ_Uxy6t|o>n41iYR=^`7F<%DR4wx zh$P_dH!!dwD$qhcT$nWF+XUai+m)=wZy?dD25LI^0Ce>b#4AW^`>BiujRiuU~ii26P)vf3i=ne)~3x#Qr}$ z`R&Ug`H{{rh?8y&$x+s~CvEb;eL6aX;cImqdA}~#G(5ou6x~OsOnPGm0zyjWm{4d^893d>ibz5NOsFg}kP;QXX(J^nugCqj@s(tpK6?tR90(grQv zpshQLejlNnN=mp+E87 zHRc&riD%)Qpuaqii7wC(ki1ERDv$MQ&HrZY58w4wE+tGYz4*B1jqvNO<~(;Y(p0{O zZW5U96CpfyZRdKavl|674K?@r>TgdE{Q12mzZJ{R>tdaL^zC0qv5%`@A4G%yZMMwD zjge@Eq97=?7NxFzpe6WdR*|X4i@mSx_Wt=anCzpYq6#%vnZjjCT+w zeRK>4A(jmX+V3|tHO=g)tgqL@4?wG{{AkrKR0t0--GM-3`eJVd52}g}u|z`vzXUV?stx-)ah!Of6x(y^?_NO*X=RTA z$xPR5Z1R*fn55WR@WeN?w>9`4@@Z((%pAf)fnDc8*>SK=g^wE} zv;>4odqq=f=WBbUx|xC*DiBgJvS09&C?`E7F1;y*8&jVk?4j?cX({!FetLc(EpDww zR_-@gsREYt&WMlqWi7wwW7wGpBh>jjfQj;Z8s_Xs=ODRcT|tD6#{HPXMM0PuNxZs?N61@@wNq`if~E%<_UL$+ft3;-(Bb zj2+tYnh-0Z8>UV#SfU{d)vE1c7&NYj+E`mz9y|8vI$N#}t(4ih_PQD$Ima#jjIDcc zA4RM!bYhMh6!Gh4GoBUG^0vDFDgzb%R~g8Y1b@fgQ*vBOmo3xVB|!_w8;jH2$6Uur zcFLH%kW7z(AX)fyF4$zzrE_Gf(J~hRPJJTj@RPxM!u{3x&c0a4z+>whf}p<_Y49~B zxFY+`yzNzh)8FXO8=!-f!cZ?99WENFAfUJma!7HIqU(Y&`lSvdi(jy~tXXeY3|i6phCmp?M{DcSNv0So|=;`wv6tw@yT) zoC$=xK7HD!p>bsb2(z3T{#=8!E1xERUezbi`jK6zv=K?y{}91K69ixPAZWtBH71b_ zM!yL@ysPk#o>K}?8%2S_#4r5825Rz?)PN;U52mMyaZ%!zhWD#oVB9$C_P{<4ETJwdej;(;$VWjr{&JJyzV(3e~Uo2wp0AnxxfDk8v-c`R~Ov}WB!}M zF^C$$XOMvXP^eA7D#o$_oLanBcbY@Drfh@HWou=Ke5bCrVMEYPs3Rgg5> zJcTL|c$#onmXvW|MrGn_R9`_|yK(KB#3A#;L|Dp3d8)!0iXZqe1BAh+b6d2bGx_aU z(7`QDaH^pZ?Cu2iqf!%f!S)*0K9~MQlO>bNvP?6S81@RaJ>37Ggm`5l?yFR<*Q_cxamed)nd5s(c%i{-6hWIf&k*T_HRUIFtx%HDv(z==ShzfO3vJ z>x!bT{0lkzw;gL=L2ppq{rU5qMB!{~(+QD1$k8{mMV{V;z%$GP;p&~&LVv8So#H8T zG5rQinNJh&c341DT=i0-N&$mAg9+yvHq%__eqL)IvjHRvZ>v3A;0`lv2O9wfF(uz|dQj3cE zdfE3}V=((+FZFat`_d!KoSf!PoBAi0PT~9KhX0Rj-gp+mfqNR7f2Odpv3dXe+E$Fr zNsXf3_8B6^@rQ8vnR5MUGlTRR-;FO1mAj;S1xM!4S&F*$rfhUWB? zx&Y8BrS~3q+AgRybps(G~|8< z$Zh!a%Ri8Dc>lzFm_Z(4gYf@`KT9IbFiDa7ZhX&F+f5QeS>?~|r$nWkJzkzzCm^5+ zO<#p7xA+X>b~1{=T%Ab9a-SgMeH*3!`ltO$i!mY6WslM6tcN)anxL`AaoB~0uZwN;Z#wb`>R))%LHM(3ViqZY{c z>mR5$LFl0;4_ZRRO0djo!abpe)z*9J{HOv!Y)OmR4s3w=(Q*7_{dP?yDFHizJE@3` zi-WfheSYR`42VcC^#}VmLC2Y&UoyI@Dxvg9Y*UjFfkZ^Agm6aMRRSuhFBzO&%3@Pr zyaN?%D{2QD8+}!iuD2bMZYAGV?^@tq3|5oDS|$Uwkt($Kb1aEF#y$J#9Jcj=*#8LJ zwOLALfw=~%@+X5asK8$1G0Ozl4^)dNGy&+lq9-FJ--bVSd9&Cv&YI)SlmIZ1AcF~H zx`vX$H)kR8@(Exv4WGaHGSlTKx_9K>cnOMSB+os|g$Z_W<$RYJ7xJ|2OW4RU8*^7= zHAcG`&%~3kJN`C}qL_;jwHLXBkBNh>Dk8HGW~Yfee*@<{4c$>DtGnYV&FbzatS!Sn$FnX{j`$q6lB21ajFH-@?ITslIqQI*=^BjNt_ zt$=3AL<9w^$r@o|@@I|m0G7tTBIjjp{>|mChFyKb%?`ua6nz+o10!KQNNPRR*#DaL*wIvULH6joM6haF2c-3QOIbgZm zAj6`;0T3Fkb>$f#6fL%3Snb=l=KOPPLlj0j*U*%T5WJ%ku?;-8FdnBRuzeT@wK*K4 z4wNI^0p*SM?+$-I?TSEpP;h4J**$~*A&htOuB@`ct1$(jQ(ze4-ohHM_4Q|{!;G{$E6e5~*yH(n? zH@Vi^2nuMf(_gmG%}FGA6v-#!$&-D=b#k38EoINTaNR|>Ss5NQfzs?|g5fx#|@{$xBzCi2W^szkEn(Nxdqm!pCc+eRzz z_~wa!rjquqZi_l8d5ox~EwaK?cr_si31Y>E2(z9~6Nm;RM1Pt(#nzBHc-l)y_Oz#r zyuFq4HCW=;x)~BBFL9hh36{ozLW7|-qjJ7#elqBtW6sWR|Hzl1i-5EvJNpjnAZd1g z)0O@*7v`<1gtGFFB!3A0#3+Jun3buR6lG{TloPObXm2d0ipCb4o1LtlE4*jIqgbfO zNkpF`n58fesLw2jwt<+FtWbyO{e*g8o!LuT6ds9roYHr(_1ymcg>hfytAGtcYo~}V zCR-_TGC<<6o+MH7$=I=YSKU0|M}u9O<>t?w=9DZ1>2qDR6K zc7**|DpUr=LupdBGF=|(rxmy_K(ZlvHiG;7ZFgPq_=m&=huA&(%wrLihE{NLRzv9e z$4#TfKjA}KqctvpA#*7Bq3H=mq6moe2y&kQqo+Z zemUTAd|^^84k9gsPBnfug$P@iqiCWVpxwsD>mL8x;|W?2paek*0W-%6xIGM@APxDc zM41Va@ldtNA0|aB&#m{u#Gdjj>gIUQ=1c|6GqjR^F1cP%a36t6me$rOi`jA=`-uqA z>m~6>w|vF;PcU_~^7w6BGZ>H@UTx|{nzybV2rNaDkR%bNt|-V}yA0E`jK_~P$*oQf z+9v?x|LC*S?h%+>=YG*vbOos0z&|@lVk<4D)zKLUe!m1UvHzg0HKA3q85y$URS9z@ z^$^)34U;?lhb~n~6w-gwr3PaAd1$N#) zA)-j2Jv7sdBM)PB)aL)80sLG(TPmW-WPYhF+my?i%P>C6AE`BB`wS5Z7#ci7yH@*g z5>pB`qwN^%waWN__0azNXUD4?9LDb-%sVFfk&)+bcbrtD-xf+?0w1S>pc zz8~-KCh6`B%xS>giC&{m+4MhcNTJ;4ufSOJm+>rHRynL(lZ_ZrlM@dQ16vn2kgd?T zTv$wU36_9o%pst76-DEUiRNPm&jQhIYz6+E%izGBM6~S?Tvi3f=D%l3qx1XN z&#w9Yf8k}T*C642nh=HjatlIEjMWjMTnP@z+h`=~WSR?!W5L7B-eh=%VW~An)>vKe zMTq`UwgKTcj*TuO$ZS7xs`n8O8sXo_pDc93^ajpqQCE!MYY{dAv8v}SXoq9T$h+UU zu@iH-Zs_^0lh$}nu3G+&UC63=+B~r+elBX<6WE9+Z_)Z4sMI9nLA~=5U%*b86cc0L zFV1p$m}*vla6?`$8tmK}XN;|#IFCLT#gAYGcUlsB}yGf6 zqJv)ZyjVa(=_4uXvBug(6{$I6$7w4*@;kb;LPxISriw77 z%*07aVvzoJ{42O1#AxE5O?8LO|6bql7iH|bQ)awy|JAjX4e^mDAa;Y<+#p~Q>t=fd zu|Q&M`Vi0Kmey4}XSKlVrFZ>*_L!-7G(;!>O?yCwaTajOLELtToH?Z-I^hP=0Xsz9 zQYE{X87&1zu7Rx}*#cREJ+&LqO{w$XAhWMV=_Xz^8Np3?X!ELxO@QE~`1MmL&jahn zI1DhQBZLhQAwvLP>!g~ zqqod(^4x&pNO+72RYkRFI|H*LQP*jzyJs(qc8BgBgYj`O-)vfTh$p}pR?bXQm?DF9@laR4|^i!SiFk@&wt5$XH6Oi5MqRC=^CzPmZ;5u1m?s7Y6E!) z+3GZ^(mIM54k*|+x}qSHsE|c0i-|QH$O(hMpo(AB4-_KEbSE0S&g%7qlo>?jn>^~V zI;z6S?15E-Lqi;6IPKjCpM8Q<3_)I2zqvvBGpQ=CVZ(`~5ikDFCD+-Jf07E)b`xtl zDV9aR(g+;*XxY&1rlEXFy-)0hJ%nrtog41DyG9NPC}l9De8vTV2J=%Jts|Zk@w|GBGQ;pr{qtj z$=GBK-hjNry9q+pSNC4v7-`FbXgpW0iF$?_D%@A2b1>w~@Lg;|OHiBiwO_|`btltG zhHMK$YhYz<31go9YpgGp#4i!ELg;8?2VV)t6wed%{(1SQa}zI~3@GPr%En3THVNa8 ztRobX)4`E6dHe-Po5M$%wup8u(Rq4Qd>p2tQRR2!W03&|8tLzmn|yAiAkNM1Ft%Q$ z!&466>9ar4Gj*-R?~#Qk(v~h?&P?7)CZmAqm^N*(u6&<&0j!7lZxS-MWFd$W5_vLv znPB4oBkV1pqHMdiVJuJ(5R^~}8wF|Ul2Akt1SzQjQBdh_21P(bLXl7eR2rm2ItHYq zrMpDB8HS-|=HC}S_w(NG`oC{|vzD%9!FbJi?!Av=AN$zv%r?Me0-@fHleBDj5aC z#b)a;x)z5fx&g1%cZB>*2Et`0GyN;KUB&0K{}ac_F8NX`lJS)zcZ=)t<+~t&0t#t^ z+}I51kNk&18f%Q?F?`wupEiv80!r#X<8>#zVhV19pxXB5pL-n=OK6=T&PGOiB_Ev7 z{wg~b*t&M=e|(K_aBLn;nEZgs3PAr}M-%aRK?Yomr{IX~v2h3q^u#PmlQXH;$fE^; zVP3zF$N$GK1?(4Kn04SknstMl00_Nd&)Vs7n8pI5SBJ)ihTL~RNjH-NGmKbhwIfw2 zFpf}_`ZOTr_+B%G!%$Z@6hZI=$LzI!`_r6KF& zoHg(y9ZK#51cT7y>m3x#WI1i~IAGMC^}uIy0{CW_oJyr!u><<;A06zN`+p59A4aWL zuap@rzd$L&#}U2fu!zV?fcV?{|KFRp@9hWm;W|YzqI$Y%sHc~Xl+H+XU={*fVUEO~ z)JBF@1xsOQ|8IQ^1Qb}F{^uCM1m)C}Swf*%%P9b>14j-fx)E@_-NVCH>(&^8GYHy{ zIR`R6{hA3o1QGAK_83?k3e$r2N)oIY9i4$hHIf~9!CC~ zUkk6$-AiHNuB2K=7@0fnUxJtnMm)ft<_q8aA1_5)r&-@!m`r-7vW90-`6C(v!wl`p zC@fN7a}e(Su@${qZ}Mt9e@}z}7Y!*juEGDh|F6T#{QP}+K$rKqkZDvPmq9^tHK~{% zYRTvvf5e0KkFy{>+_p22;*n8&gkiYSvhWS2nn77}svKFx;B*-tc|N>{y~O|5AE$#w z()BJ9b=oshQ-47*zE?v=-+?u2UVEZ$(5)h~65)j|%SL;y6Nu3h+{fc)*T^dK`LgNB z7RCtRSGV`_X*OYkZ}RtbvWscb--Rsa8MR{0S+C7881zAg204!UH2sUi9-He3K!M@~ zCQ({7PBlM-iQJG0=6@#di;R%&$Hb~=@0Be1|HT-|qtT=nLS6sO?TF3GUsFVpal|Lx=XhXa;av90PglO0#Q=uAnl0$k)pp%XDQy}}s z3Ky!9=3l5x?BDqQPSH3k*@AI3hh^&5uU*d%QSC8~@R{?}BO&K1BckQ^$!Kb8(}Cjv zm5;%pFdj^t(%&PKl9>2Vj+e0VPz5aRk-7LrgzWZN_=R9qP%p4tfHYv1ziwd(5Ab%# zX^{Mgj~oB_U*+H>0$T~vIaD&Eg@IYHv}pj(E+aTRNKFXB1DIs#fwE?A`~o=Kzxjv< zCcoTQYnd2A>%oiw_BcX=yKT>mAO;4|6`{dCV>MjT4d^l&+59yVp=0#^7aB<3eCy$o zS=xif+Qv{f_x+2n7O4MsMsFC?ZA41KaDP{l-$e+QH@((p+K*J<2MIEu)ej^YXj|Xk z9=|dhkU`cl*(=C$_b_m0;Jv5`im`P_&`H<&&jOtVgvGxsylnU8x-&Au2V1o5;r@Fh zp-d5^`gSN9;TIvNOyt9yIdED}_AEo1>4I;!3_4=}5@{;^qTD#?d?S@{q= z;WbxXJ8XqWK>Is6dXTIaF!mgU z%;qzVD-KyhV1`^=NM3md0Tlo1!33niTmAcbcub`1fDdaK=|^O~Meiw!66oq6d@mp* z_UV72`1~aHIAq-Dp#XSnnWeAeUIQ=+gYfxH0YR%kJzrETfQ_JgL)p-8{To>;U?tZP z3aV_q{J-urvvi>D6DOP+Qi)W#gafR|jNFu;h};iY1-`uRg@sLURzRYFnDQ_le}n;? zzBi=@c@5bFfRuWmvqnO(QH5tQs6ZB8>?8I= zMqEU@>Nbg7Em|-o&8;#qEU`ph@cPX~?LOvW-DLZ;`5@YTr?8H+)1N(@fK%%lpgJJT z(z0K|`Mbqn0sHG-^VZ#hr%%m@vZvwprDtdNf=;2{0}%TFOas?y_E$}A53Z7kQzGo` zQ5D~NU}i^qZ=7Mzu9(a~AINO=(BNQTF~a4wjBlaZ08k_JXXQh9q<{=x_s#}zRPSB( zvND4`cStxH&7O!45{)JJJGUdX5Yh#X{u>I3@6m1J*{}p+PppBMb?vqDnkA9`0xUZH z`jMsX(#UHt$+)BID-C-QcS{fy$jw4Czh|1U=gv7656b3QkesEpH-W|nJk#sF-a?i+ z50%CG-njrVLfV%vKOm_VNh8b#(kcfivJMbneWjWf1R(cslN%RABa=Elf&bH8xIUG0 z{>0z29qsG9@VGaGm5ElG>Y(TLoKCO4>OrI#^!TzJ;qPZxPuhvJQqIq*@Z@~3uks_` zyIT!*EW%-JbfcSf0h9p=Zxb{1qaM%qBxz%>0lNX;rt0vt6I-U0Tb9}yC9 zQGWaM>F#z%RF9b?2Okq6wGa8*_p-GD4eWo-dO@U(B!ODZ=qDW%=}@Tt{UnmuLs$Y8 zu3l`Q0U6(at&Xodiy}UD;2HPU7o0dMzrvl_Lk4@os&Gj4dn_Q-+36t4|Le4dJ?JZ& zkcD(}9y-s=&&TAD8^W{;;(k#>IQ4b`ay*7jSX$t>B5wOA@m^$DHHMh!Rn zp>X$#H@8#_Id-~Fv;3N=tUY<5;)`SiH&4%7A?zQn)|glEb?;`x`?KQ&>8RI3E=x-$ zm;Ep}#7U)9{7n-Z89DUJ;U>j|xXb!;_d$vitgpaYaFS^{%~L6mF_TBfI}kshW`7d8 zhjR6wtd8T1IITfag(qTB5*%~1rV5nfMBz_`7M|CcqIGc0f zNmFF44r?;l@b2}b{`!2|dV$cWyR%c!h{9L)uRh2mIS}kcV=G1wBpCJ2sL8F!!-)If zKOW!Bz$7LkgT+!b-2kNXh5_ra)R(!*Aeo~ZvJmtx6AJ&D+#xty_iFNn0QRhQ6rtM;G<#*Q z%V8;5p|GWEm1_&PKBGl_NSguHf`gw^)O3Ti`|g90WO9hFlzrB*GrTg2TuTO@I7r!d zINzge`|F`Iy>UQFe7)(Ds&K5%j3$X8zHQPuZE5L7$?}$vo?ZR(qXZQJi>n`gwc4=v z5aVOJ{6ea@q_a|oQuGUWy6>T#@lH}U^`F@mH`ys1E%Eo0=%Yx7>^%(9hri#_ClxxS zKPlU&nCx4L4q00G)$TIj^wcKGe|t}w3piDwSf!gqH(hB!4ao|joF1aRN^k*b*4iM?yBi4 zr2R>f!K1g@qb}h-zF*115UvOI)q31xBSy_nHWYvNht>|Ci|52?c21UxO@jS=&&8hU z8wc`&p&UbR(#GIcs(GJSUQHT>1rWF(lz|6g(E%^BsDI0xx&3+a_$kTwy!`7lIDeX) z7fX(Z<^m|e(xp5$UgH7rI)ZJC;1F3(^R!^VBN#w{xH|dAwh4A=m?6SPa#H@DL3VCd zP0`~n7VYrAM_=c^RTDDu1Pu_xb?>j2GZr!5GBu4NpD!`sK~&y*1{Ams+6#>>En0{G z=vsJZSJ%TXkVim4u%}`(o|>JR3B%@gcELaiGLFWt?kr*V`$Y)5jP?3b$vjU>OjNNi z8C*KEFB==q;1uv5vwJcna9(c*$LQdZGB@mcpK0h`A~8$&|9On(b%(IBZg2PUw(6y4 z@wDq;N=EF@JL6y?=SOKjm0!Z>f27!4|kJstMa&Ma-4 zYfYJ7uZg#O$Kf*7{8g+x`yQ_s!4MaJx@5{UfJdQwckYQ4&fEos&9xzT<&6;Q8O72v zNYW{9nb2M#7!U0j%y-q$-I0i1zT9zsLMcT>w!&4n+Ir8Snd=6nDABL9Xm^da-NDlQ zDdODxc1gck8$L9B!np076e*M>AO1z&wAXv=#45w2A+G+osS@Fj%(W-Wp{fQryY|og z*R+kW`2pKEaWdX`CdybEu88Og{P%ggH3C+ATQ%Uqlks$cOhQeK z3<>DUDSsQ!3Hd|52PSYmnabSB`^-`T*wWj}@jM+%!_(_iF|r^iNLg=C%gSN>s~1!` zzm&Llrr{@M=Y`GJi-+%9%y6Dg3?tlTVw~rd*?tR>=d1@XR{m;1RDXX|uG74RSSx#f zh+Q?n_#EU0?z zln@}~Jrj`Vs~9i=fgK=FVl+0DZlC#93?AiN$O4bXMhjYPi0siBNH^UbOeQ!l7wsB9 zg#x7i+}}2F7<7@2o6&=K!x4d4F1>~>4U1-D(ohIxBSntU_s4G#Gn}LbYw#1y#t=)| zxi~q0Uw;5GoCEX`r*Uwezk`>#gbCj?QfvaI3~X(FuGxuSuH*1m zdv-t2Pe!>iu-{2tUij3?Hmg-%#g_E8RV!XlTHLH_b6!Nt9TzwO^@! zcgfDzR@QPpViB#ktHJpL{DeA`RV*D|r=Y?^n|C+fhKmlfUaFq|O*21+u@R+p*+w?& zi?5wi{1L9cq0A-0H}mczE)`C9F}piJY0X61caP@jK1-Jq3pugGqA_1O?MUpaMX}ev z8QGeG95_8g9DQ1}fKAr8a`Ayuf=)q^Zj*I(*X}6{<$fdh9jDlZ-fVZxgFV)Cq0bL=Dmo z$yFty>41*cQto59*!mp~f&)+rB(Kr!NqycOW`rII6GN|YxID!#u`9hLE@HfNLS-sE zZ+_i^FbKzJee633b^g8t9zM1#DPY-0#uI=y>C;qK{|t?kQ`IyO-(i!21wVsN6ZI*m zTwqlFdl)&Hj|H4H4LBUo{kyi31mkbW`H!X)R6@=_GzX9#Nc7hxYR3?Tb5Q*g*TME2 zp}NIoWKgxW;q2Q$+w(DjCUlSPhAFH)yqv6d;Q^cE8*Ykp;5(7lqt>2fExbiRO+-yP zH&^UrtaAzeFfBW~ZANOP@&L;Wd?&MVfXOhZ#Efjzp!xHh0d54M>H{TkJ3^6YA#FYA z{YMAG9>X-vxsKh7|6|?n5IeEk&VarOriqhLs2-%h>WC*AvFst$ziuJI@Ak7`1O<CVvIsx_CK{=M)`GBETf0s!ydA7+-A}uxE;cscR5n*BNTGv{Gf`IVvlh#o!jhMi z#Mia&?s$93xk%o3IWChI=jF|ImreA9Crmzw>y+DU}C z-w8*QG-?GfC2a~>s|>w1j(o}SMBL8X#61AB){VA)tkJfbjb3X`WRvNW_(EFg3z~51 zZq(p5beSwB^FTWqaA%|R1~)S(akVy{4}QeQ?4UDb*$E4Y-kVIzSKi-x;&>)&nb1H3 zD}&?11}7?(JS%=na_ebj-Nz7wc!2!76#F+M|FD;WhQ@ztxdXe~lfBkdV{d%D543 zBjo8wnP+#`X8xn52a`rqb%fJy#87F<*`IHy=9bvrOx%iyIHZTE&?p=d;5{JYo7Ouh z@nvyP0!C&(u_)?&PrcJ@B+jnExH$7jk z^p-ebvx}~nrL#!(B^;n|cU&3@Tf;hhw|tMnk76_r9V$MF&V2KNaC-RO6~!vTS)B9H z$3?uHD7!e$$Stj5-79LX!K9O~UidRu&i0J?sR!RU&UtiGvmjNsyHLx(*|Z~=Ik3iY zs+z}7zWR2}TTV`FT1l=3Akc?@w#;AM!siUou0Q_dp^*0QHu04l_u$>6-)o+O?bf#+ zTulRD*@gDLb^cB2tJ_#DEuHjQkxjyt2$d_2oL~Rq+Q*jHK9@Z=U^rN&_r1h`p=tlg z23B!a?der%YP=!?JKH@Y*(|8Yk>i^lQ8Juhw0ltc(;Hh&z@Kn(7}~ggb?8rL zJS4VwwFOd#&UTHc;#_F1oK#DL)I~LSTdVlXHF=o>SA+$U3kYX#$ellRL@`1qj0=6^ zS)@&>hPTobMZZzo&dQtf>{g{ogK&_ONv4`V{2nh!}@{s5s&HaZl_ti>DHRh zQmM_GvXOa`Zh3C5`gB`7dv+*Hje)oUthDCgJhMvnWwf@Qt$m<0?Cp`n=ALRs=`GB} zD*7uSGSb?~CG>KHz2^Pl+~e9m7;(DWRRx0LFAVdb{Ae?Y7cm!uc{y;o;?jnA&wF=Qn9TXZno_Y9X zD2bYd0k3jiz6?&3KV8Ap(t^0lMSCtL1A2JsI$BzGd?56%WB^@xkFM$Mj{i0;r1u&Z z#M2T*55=KX1DFhkr( zvJKQm8q`o18X!E6?xNW`-sbC!${UQ~=i)|-CeuUj7gXHVN9V~Z_ZWB-%nzt_UCutE zGrG%U6M`=wk!WT@IOek^%=v`gnqDd2Wm@K>!-#Ty2SUetewkf+T*yqnRbTH5h4=Ja z?=PA%yPJW1NitEiq!D)^$+F#7p_s*01|W92ElA5=6a6&1zh4)4yZxS?u@2SO$CRn! zy-YGGcXcprf`H>;D^n>$=%uc;fhS5G04whBvUPG)bhvhB$8t+=_g{(abFibCZY*U? zE%_{m*$s%a@a)EU;A9p%jhAal%Y}rgOz|zE8(^BoJ}jzLQAQ$3^LIx~y+4OV_GVv= zt_EheURh@Jqhw*lF3D}|71qm!^yhZ1fBkwm#vS$cgg8S_1$%r~tQu;I)IGFoFpS?l z#a)tQ5TxC9Z|m*vRRW%uxW$vtKJ&h-jrX22p{;s?89ON~YZsa=tl9)}hh!Phk=h*Gr-P+oY zBCQF-XKVGYenR^nijS}rZ`09R;oPPb?*XJBW;=s+eU!Ab&rj}rl$VN?RaylRSDIZU zCjZtT3lsZ`ikyOHFe>pM(nM$wH@Cpj0l?HyW_SF^#SUw7=VFAvojE;Pgen{nNBx(H zBo#r;<5XVP8q2zkl|ieOI>%}Wh}lgCv_&PLNH}rAB+43;sW)`IvI+`bMsn}eue$GS zpn3s!0o`^By+SdFYn3qP@C+VY>|m;}4I)n26g(a&dp_TIRVnRV0(v_|etRA>l3`lE z&9f7|w&qD7E{`V55GN!iv4(_>r3h~VY^nT!6G7+cq^K#ru1Bv1S$u!e`C}C}^DT#x zb}Ke(g@8(8E6<9rcENaPqIU{Cx;yMAW7?aVuwdWGg;PX-m(9?QKw;go>m$6gN8Gtm z;p@A6N2BpCvafJ9b}#R$`VxQhE~$HaFFlIKvy)bRD%K)UuX`0kJnv`dprjlYOwmiy zuEkn1+Zg=oUX<6Yk`@o`OeSU!-A_%=>$PZDzMk*GyHkcC(9&9L`5)5pvQlzi-ciiQ z#?x@xzsCpu(1>T-9v*a@7R2>>#)VpV;lGaT+Uz(j{q$=hx*ExbSbJFdv2Ds&-)k2V z#^35dNfE#Nh^>B`sOFg@OS{`@b!sVe&YZM*fYm+2%Cq`cJs{$lKvM5b?o_TA%?*`# z8JScJ@lVqP4o9rmy**kiOZbyL(i{qD9$C=gY8)!TP^hGj25pJnY=Hk4iA!`|EA|=WCA=grt+$PgOc~>E9+fO(E#9H* zvA!e;8r)30H;J2?gv}1m7F9!Ss^KmAx6@baUzGYgjRgyeuc?-{vdw9&ah`t6nEgJ* z3gebx-CyFUAqJ4F975J?LVT=%a3`Vhi+&?+1~c%9!R6|&S+Cgb`s^}(qA7Ye&s?)4 zXY)gLyIYx>4N+#t$+J!DV*Mo|%a9lkg4{yAoqVTs%&pJ6O3oUr?)+F9Hpc~^i{w4< z&UZFTIyB;m!c)_poQ@3x^?N_y{n`#rVpIbPD5ba$7*PVbccXd$lwb2otGol~M>Vw|NV8kc*M@HRR|BRC9Nbd!M)JV1qm|vbb-fu z@-0-8t?pS6TUL(9p=Yp11?V>X3_t(n`!pk(>nv=?C5Bs%?1@f^qq5b4k!g- z78QVQ>`yEgI=o`RF2iNmDXLZh!l_^zcy>qJHXI0o&iVI;4f*&alP^e7oF z5e$svojb7zb6sAZ@IizZB_!kxVtU=;@Q-=U`MsMn31J~D#h;dz=)^*x3<4b*GUcrhmRT)jbtXx7e_~y#O1xM=uM zetQ;;x$k#!+uLhtCTT0gI#d}TxJOHNF?LqGe3eA7_2jWNpPw{;=0RnTTUpMG*Nm&L zD3PTw!dxp|@5{>DuF*Ngfn&i4&wMpey`H!87DYN+W#(()U~`Fz`=XQ=Ok2%Fm6SQQ(qL{ADp)P#g15%2y$ zm;1WQaQ;|{*+EoOcF&w*=tf$Kxx)GLOeoVWdzkgfT>CJ6NhRM_G#vqMvcMxtv}|J2pCeA?D7q)P>J4Ua+~( zlq*G@yRq*O-M+z~@?C{0|J2kooM&FHp9&kH7vD#JI5GUX`?A;0PW_LC)}H>burqDb zEUo<##SZwTg|12Iwj0{o7f(=D*4HyidANUJsJA%5bn#+-k>i!qVq#jCxf!b3+v&Z% zotY(z4p~|rq}~9dw@J#O;)F6auB)0$igKUDKF0brV^zGCl;gV`j1&|ub#yeU8yHZS zo1c}F8*0x>F%Or*|C!PrZ;W91GpUvEMeZmQ(^8p(mB^w;}qgcU8v}e-+Q^R zaQm>|)ZxRnm0@SOjY-j4Gs~ko#N@lWy1r}n{9iL$#^)^pRAUY1g=7LWITas1r3)YN*f#f_2uQxGt)9zzkk2O96E8L z>-vSa0&TB{OFbkv1Wm228M2I8WWBwqZ{MCfbK*l)QxhYrc+sbO93tU(Y%(#{s`Voc z3?Wg4HJm8&6g%DCzBSVmyoNm{clhiJt{3X z>FNfx$zYzZtf;Vv3G^p!WE(bBB`TTxY|G}?1s4a6<_}IMC}>BjHx^A#UwxT#R^i5^ zNzF^I%iP^kN6O^(y$CQrcg<0vz?#+6en%sXC5SP$F8PiNbq|P>9?bW+zVJ&L8%1-N zj$QA3OK&~M7&>1%`Blf@ z>rjC6vg(i7&Q6t6m#;=N@>>mqWDQluZ!i#dBUKPy@}TJDstJa4;_k#V^(b?k)7nWzRk+exiU zWb*8G!6)AAyglZywr^_p+EMO1ES}LZ2G6-|;$&|;BRUuiEbiasCy5%}7uoL}JACIU zrr7mZL;yN6R^G}_cY1VE;Lz_Mf?@9GnRv6k+@({BwBetAtn!`lEO&Y($eDwS&0`iV zJYQGVU2W` z7k`M1$i^5MJ?^d!y0^1^!Lap52>$$;tcd+&3&USm*9PcTjy|L~8f(NJJcBMir17Yh zVLYBB5ioMkmv7v6WX$2PHg7{#jlYiCClzj6rF+hv{6sxZ$~FQ@&Gpa+X_M|*T;Z{b zS(+2>tyN@aqBz5BrH?o0eB<}3C39r!@A!jnzmYgCMcO=1t{fHTr^(s7h41JPzk1)E z@#M*f&5gllWIhhl0(^GAWi_>w9J9_xG_H24so%QISv6iAE_RV>bCkqmGTCTxm-%H7 zec;2sd@ec@lPjCk@%+)^LO&mu&OLZQxiaKH5f*;0)-4uP_Vg9ooEpnc$M1TNR0rI% z`>k+_xMhAcS&ddW4b^f-D}q_jjHRkQ0Uz8IJ~kbnkn3V2pc)_dbZv}iT7M9ZAJML) z@xF-(b=~X!(A3L>>4#)L8`6GwEXUUOWZHr06KBd!W1VG8(Bj|(zsr){Ptk)6+B(X| z7$cwOn(9kApF6NNYVzbsU^-{FTW(hCrWYC9yN-AJ_wU!81g-Ch6DKM=JM*51D|}#p zaTZQ0KkBFpp~!U%ZmX)=CnlEu!u&({<>N_Zk{cu`57)cVn99|Jiwln>$!*V`Ibr z`mfF7)w{i5tT?u0=-liC9>n8|i^B?1=a9JIW)IG~-Wpj^8V@9*3)|hGnjdi z?_*+2fxBG)C=cH=j2wo@eERxyd%hW!yI0lKB?r1I%$W`rxO(oc-gsyR7P6oTqg0M^ zuH}e-@uT^5sE?1#n>eF=WShHK>5ul}+S;#Hhh2Je?Iu{u-R>MX5W7;#<)Sn6~)BKRjZqKi4wg_=euA@aO;A6 zQTg@$;tHD7dq9=N;n=B!<Dq&6XvTV%#+zTI3Pu|)56sC3E8mKal~sj>O41_p~> z8o?{j&fu)y#uWl#BH|b~O*U!nF&(=EPr>@w;iJ>{RF%2Ry;@gRW|<}XXbY_;Ct_WG zI9W4E#|MZx3{31SQ&e?#yJn5mCEB1S=9(dtkce(WC0d_wBKrDNSM|CMp}hNPwY3>T zhkF%y3>*CwzBEpVKDw>1I)5jg?L|vW1pW%;A=|M^sj}6ZBCX3Y@u*;zQ7PK%ZXqB_ zpu4oL7w$|``gY+q3z3H&pJgVW^=2Z(rcS=}88;u1Fk-vH=Z$i*vN;!S+R@)jQ!Z+q zP}h-MD6(XDuU0VXq4uffAWs=iS{JpXvs+JvZ+~-SPa!kr8B1uI(Pl3BMD@{Y@%4uM z=euKbY+6Igz0-QtdBL+{s<^Z(I+Mn^cJy+OEp}Z*KF5*?tIBCIc5@E?3g;-2hR`?O z(?NF#50bW8@`5Dk?j)y87o1Z~>>Xdtudy=uTC{2^n|xh=Bzu#@B18+vX_n&d_})?r z(PE3Ihtd8t7Aq|xZygr z$;J%b(hcEPLHYPLA2N^K&52_waV)8McUqPg-tblwy@DUAg^IjR#I(SfYY@r`H=7$vKe`fL?i#D0< zIQCVyoxP`LXgQc<+!pkOFl36W3cI&!_)KV=skuLniE01Sr6(% z4!(;BbnKZO=w==DmN<4^k(_xkVLR{7P%&%HqbBAP4Bcayg+Fr9=x?oJy5zLh&z$D{ z?HZd5<3}?U&RyxrD)&Qq2j0BtzqG@_&aUsdu5JUDtv1p8X|cgGRtfqXcN}$1&560p z>E93hl)?wbhFv_)i1yVQ$54A2^nj~G!_zkK<#=Iupuu|ih1xMy|~ zG!zOvT`sbS+-6~V4#iNC}4_gh2%U+Eo$)h${7+PZw* z5US>F{Al0~UMsxS&8^?~#|z<^UsZ>Mg&BYTOs)JZpH}2m>hr|o*C)f5uiTGpY(#$C zCN7oI9j3ZR)3emq&M;A5c=NiCRLz5v^z^rlpWG+O;!d7qnCZ3Q_|~Q~fD--bu_1rQ4NLM0qBy;M@qQ#Zjr*N(T$rJR4>>woZecj4 z=IEE#!Q4Mm)pu+s8?R=Lsp{+RGi!MoAVYk)kIZuTQ~-#R$fyp;9{wE=&%JDP<=gL9 z(UVf$a3iEDs zGRw?;ohx-yNhy42!sSgDJCVMP^I8|3Hg^{DQZJ`Is~liiq}Zw|^yRvm$$d8x6M^lls=A`spMsX+t_Oug0>N0OuT(!FS>?sM`Qlr_ z%_bY=ROtF216B0CQ?s<6w9F@s1L4iPQixU8^;0rNnx$T0IxA6e_z080*|d8!nyR5! zDqCsK@saTAV_0embEFC`cr>jms|}CtAP(=Rc$!1E|Hzs2H=izN?tfBnOc6`twnN z)t`zBeWFMBe>+w8whi2-6Xp>Q{8skV?0shK%JO_5NI#9qialsc{QNDIQ4wAA z>?~FyX8nG3Ma8S1UP>zOZCd5;qEMRwMb&lcFH4^CcbHO!9eZ=+y(Z0jg^=NR10TO6 zu>ooU?ji-I+z^ZAVu!~0=PJq9jXxbvRym@Op&fE8=uwd?KNVb5!7AQSFBjJHH#e&X z2N{oZuqkLoK=mZ25<9>cD^V(YhELwok`C%8nn_w;x=>aRpL`3O@yx=K(+h(f@dh=$ zsEkXIkfNCRf3R2kZ&mg8Q}*{qhQ86_u5*)sXHK@XZmo~z=WfrA7Xh15b-3D-P5q*d zmk-egvOLH=nrFu92aWo=P@9%C6m{*0QRPrua*7V0>`_#fRqHt^kUe<(#_;=8l(_ZS&Hrvw>Jbbb1!aq<)Ai4WEFig>Xf)$$TK z<=-@3KRe)(ORpy?LB(&ScTC*L{!o0Iynz87q+L0a&uekGrQvf&nRAYdSeG3(H|Hvk zPv9S%I>(cB7&{efymWTUlg*zRD^DUyuWS_f6^pS?PM%9ljy%iBiNxPvZ;Vo$wA|Sp zmb3J;H0el9{(;KF!$VX|?DKa`oWY04NOCnbYs{m!q=FK0=P$x79;`^fzh?6xoEqHk z=hL!j(JA`H;Z>(i%3vUA|(X=6Q^zS!&vFJGSLH@COvJ`t~0 z_wI$!&?+>ou3}u6H`41;T5Ku$Sb?)4At%C306)@g) z9K?$)zJJLE<4M78&Hqg`{J|GFb5n$0@|%t2+j`@Hzs+yaqYm?Xw1b7ALeUPnX_kGq z6hJ4K+5109|4Syosrp?Ln0Bc!9!kz5$7uj8Nc!Sl1x1WDEDbA(dbxjqw15Hh@YT>m zd3iJ={ley-aOIbQS8B2hCn_gSwq{dEzF!s-%-ernamjS@i$E=md)dKOupR-n&;YOo zI(Oy_Sro6iY2!Qjo3M9j2!jfV;L0}XqM0f8msiF$|Ca8hjkWm zhREG>AH{R;43Ff2G!gqf+F<^xOj}m!&F9g_X`}6bMr(g;9S&A9ZfPCl$_i)xiW%=d z{&67U)#OX@kI(xYW}~-S&Lk8Evnxz~7L^wo$(!%Z4roGmCLTjGL~+JfE4T;9*vO+tc^#oYc;R5N%ln0*8#H z{0yFbEjstGmf>@oqxGs%Ad}#G!e~PNdGglDNQdZ*&UqXDtx^w_r$IhhYE$^`(r$t4fOie?UW0oZ<^n%?TQ7(5g{+sI_V?&j9b zEbU4A`}ZqcPhd!Ez0bbVl_p;K;8v#`C|^BSYqeCDW1&KdLPvLhO#GwoQhsvpcSukHd2T(D@!Y;rn7g8`4x#wDl%k(m>G=5j!_>nenfmp%Pw&M7 z0Q_KI>iqmo-P6*N zSqBI5{Dm4;IQw1t%YY{M0r_4+QdW3Ke0J^g5ET(Etx-R0!@nS zs#lgJebx%&?!+wp=h7+?iF+Gs-(+?BF09W@)~4h8&AoC=755u?-#@uq|HxQXR#wQY zRr2|VQLMrbH^UKxMc#=w(z?^4OVd5l<|v!hv#Dl2cag`>Q>*};Imw%Fk$_~gZW97Gkb zowE2e2FY%9I{y6l?33zr{8Fiu-Il%ZB{dp(6BF*I7D>^9)1qGzP#tqYJScBB2BEC|s}Jbf=m;|AQJMK=@?Ww^%)iZtUg0^pp07jNE6LmqvBS`&V2mfxY=1t|oW( z#J54CiZXU@5>XCZK!sVG|xZm@ie&D2Gs za=);fHH$ufd3e&NSig`GDeZF}+(2g*F!C(CmLcA~#Ljv3YwdV-TC~6o^mU)japR8D zCh2G7l=Z0`Pqn|jf{*EEBbXIXka>1lQRR8A#|*c7J)b?m)r7UMC0rNyx^O6ZYti_i zzk42W#75f#sQ_h2E+Iz_ieY!Gd}Sru6*cU3isp*!SM%zFG6E{|s3oqOY|b0f0k_%u zRfoLj_T0lhZ2yh2e2@JHS;(7>WWtU@zwSY-&1EF-M#t5 znc)lZ&NxeAhry7jD=lU5{?-a-*V+2ddV_W~^TW25Ez}16X-8QyGfqs5HFCK+RnJ_a zMm`Ahve;ESpm}+8Qa$B#$cbRPDhF1v4|RiZMq6f)kd`ToZJVE(x{q z9qIIK|Kdx0wpiS7v2V-dAZ6#fDrY+MuxxK`=J=d>woB}Uh4mLr?#F4>chM?@GWHjz zeA5)UXZ;UwWskKsP#hk0447BE(S^D4aww|ioxY=eH5C^{<&EzR&W2Z43Ddw2ZDAxKeuUxK}qdxqe!QV@Kxm>5Msx zD4*=S^dAD=BexMvVhtKa>s;)oqQ?Dz9f$ePZAQIICFtqj$q%Lt20KPBV`jR zU&2PprReDReX-{5JBtoq6RAz>3x4!untosM)y`UIJNSKH;{~c9ruZtTe_|4?cq&Ll z^5egz=AQaZ)Tew>q$PyYF)?%-W2bZxx4}&LI4P(|C9~fDdIKTxNysjNpstrr8F@!d zP4RK+xUv13<*%}+_Of0icHY7!iz(UPzkfe`JwdFg&79-ZBzDWzNr?O8vAximcaER` zjaETko-dTEaRYlHm&6M5Mqv$o#~VaNC5`rP)Fs}%Y`Zuh)bkA-9-zmU0`xKxm^&nu zF3>+TH_wYabnxJ{JLYi@z%1iulc`!N|HGfh`ugsEJcN7?;tnKk>(Po9Jyq-}tGt7J$saeYt$FC_>2;grVawZXcOcqm|7R7r?fwVNL|JI?)CR5=l=tRq5t0Dnsfy-fE_siJ0GzY z-UOV`SHq0vVP^W@A$AjbRimS9=iVIc&6<3GUsAZmHuZW4-l{J}VdZ>w0JCM@XN%JyRjJoCHS~}Metq_47Vn~sixa-gor#nW2lo@gt;`=iIx8imVX;*cc%1IUi5C;~ z%cg68}w;O{6 zPCvdskv)7kNAXb-GZW9lao~$otiJ;BWo*M+xs5yjec>9aA8WVR_n@2r87k-uGpNE|Hvq625X&qk};3j3y(bIK{&+Y*;&^2d( z-Ww{?o*C>t4Z!!$WcS!ul}B^p5jfd|yv2WPYMuj6~Nm^mr&hX5etXvoBC!TCI=i-v4wVW50`P3cs7geENmH-MW)ki5^#n z_B)2&Yc9HX*e-L9WvC>DDCws5%6&wgEXQHoS#q_-b!Bmka+D^^cCEngmo}MF?1>vE z8E%zD`@IbiOTD2*Z;9q*Q7#A$2za*p4WBhBkloU}wL)trLa1gQ8?9w^QA^?XYJA1} zJUh-{Ec9U=gRCaENh5cu+qK8>ja{t`1uTUUpPFaOCdzH!P51iLPoK>R6&YAxd;Uw+ z=iAWJ8o!kDGD%0t&M-9PVym&y81HxO#CRh#)wFO;PGs?+z62{7l;}>M?3rtdcl-`R z7D5`RYrCgfy%@EWk8K3odtMEtP}xol9g+Njd&7!)(IoMEA6Ws9+-tWC58r&NTZT!@ zz-vW+3oqo>!waxvYw2Gxi^#mTwmWC~s4Xih+~7-!88$pd^ZBd&-NXkALmY3ojuK<3 zidmwC<_|;ZzmMzy#rA{d=tx_KyqiWwLLn?lwfO_$F}a9uM{$0h)!M z->~LuuNgZenGHgp33oUNvAmKp+tSZCcykH3u%A zxPHTrIYDu`zBWkkSGh;EK^_ygc<|3#-9l}k%0Q@6mlpze#qggsMwM5g|9mm?IT7zlJ^a{>%z{K-WN-V{rG}erh=iw8Rt`8PPVlrBnCF7tej3 zd;h`XAaQ2idH33DueG*#A>%xUUR`wXT7T(cBNCU|mQPd^@Ljvcd)+tM+S~7D%i(2s zyOY#4=wJW_9!a0$+H2wx!(p?-aoX*TRFUrcu?6fWd%M9g%$Fls!4fQYsrTkJB$XwS z@z(QTSw?{FD%1!$KY#Tm0!KZM;U&YKEHEkh8*(w0SfX*1wcO$vdq81M}T-42G zL$$O-E$l0r_EJ^%`iyqFObB3HLdZEvc-IbAbeW&new`nl zErpRp(x-fo^mTj29w$Y3@3qm*cUypmDRZ&Xf*VKxErq@wyv(#)+hwyX_|(sR=he@? zxJ%cGg&NLufikD@!si}n{mXW)Fp6NzQ@vZ;aRZVf)V-V?<(2)0mMz;CFTl{B9;A=% z4is={(F)GCj~5lbf5c~j3$FEYT(k(-=j?N41;gD9Wd-m268$;H#WvZrKotfCL17;c zUN4l|t__$*w*h{qB7kqpg4BOJzjnyexU9xFML|TCdVWPO6=144uJr2S{;#fuuLN3I)M}~ zeAu9Ti~nrCMoD_$%I}t7lN;6+Nf2S_p3<|Z&u5boHS4U_{TkR+Y~YufS0_R$>=n~J zUGu*TKaryhmJS4}7NZYIko)&&($<)Pg5w+nv?A2ze^ z&*Y^M7+gAkn;%}D@4G|b#~5sAPA9D&hXV#2bxnK>%#>0F(wyarnJB3Y{F`OA??^zW zY?y{1WZa8-5!SGnV3>6y;5H8iR(PW$Lz9wD1P_lYyOpJg+Q$k)Z$NI|6cBdj!e)Mn zv*{BB#^J-We90+Cj>nrvn^WrU0aF*JY4`8SC&`jNeCVoOzRd1-{zhz5kCKgzX(%m) z*xvr~#ta0^yng7~2lK7%%}J0lTz#0ozwEZF2evozyKGgE1O6DO!~SVlOoI2k)7G~y zP8DsZvR?O;zn8x?Q8Mo9j}M7uy=iiOQl2P&N(wP^IGL*4q7oc{jh>?rY+Z}(cYk)y zb@s$`v~LLMwXHHcI==XeDAmrLZLFm{LBb#k^xd%RY|=*esP12x-VUb~yjEyHtp07i zMg$s&eqf-jgq94Ysn7BGpIJTfI_9BkTidpdK+li*aUONKrl^^l=T690fwjeN#%TXB)ax8# zVzj(T*1zQ$qVS|z*J%wW>A*I-<=J{?xS5@u-R9KM0V{$T(}sBJ1gD_?yL- zX}BaOheZ$2P1OTaQ`8_?DEoSp4g9-TK?Ecu7IDGSOqc%0>Z|{Fp-T1*7%2BYez01t zZJ+K`N&>p~1_oAw?yy^=)$%`H=!asmtpyvn1(^?i5KPxkf?g~NI6*=qXyfYf(O_y1 zpzZ!|1$u3ini>TJ63^Wl&fTK2egkz-ZWb>TkI_r%xFq);S%EqN^N(!n=dL`W=hB@Z z1L{7^>=aBoQ;(AS*6^E-e?OgVEe%I49|R~%-o}VS;hWEsp>b$D5GVRWS)D7SB?V*Sw zWfvFz1{Y&)S_v=k=hlbtaQvpf%TGVa_Fc1iRb5J5)d|7jPh4seqE*5f{pxNL(2G2w z?dyx{vuKQx+Kb?B-RV9fp7+ed%gQ?BvHHzqUU9VG9|hh+P}g-fZ`AmZcFhwkhy&or zS2wi4KIJY;MnPype-i`fONwJEn1euE$B8*e5*9<@# z1T>xH?nsYfy;{&xr?~DU#C_fe0yA8;7Jp2OyaJ-rH2*ZX5@J{=6y}8i6mI*fdW(N6 z-G7bpEUUxyVSiC{Ke=`d9Ixf9uvIg%3cSrbKi%E{0MUL@vO44 zmmKKQ+(V~36x@>7&peg>u0+K|`8E|wGcd< zkh6^QpMqtM_pGZYNtb_UoVE=<`MXc-_EdXeXdEAV-9;9#`DM=Kzu#HxR2TrZ=3{nI zsSInJj2c@LbYJR<7a#a6L1c!)V3w^Hfo2mYERA#6l=HetRbn;$0oRf%ILpoVPZG*SabH zNo++}QUIhpgJB#Ud|0mmF0F3lx$^2gWaVfh!liAl5zq2mj9 z5d1wUiF&-RrRIF4Hz(Hjd7aYl^oI}k8V!IMCO9Q!6}ixm13TF?f>){EUm!jaMFC707B}6J_Hv z9O_!_HGcF2{sLNj&v~6r&gED^^azpsfU}qmaO}Km!31I}lZK(sGd6Ts5;%Ymhyv(% z$4=cBdnGm`L~}1Lcgy{+ae%602;FG=PWb{gsgIeXLGH)(l{>9z>`d_mY9PTcyL`G4 zcO-~|bMg837$1VcSDlAzv(Am&`d;x#1XvJ?Cz~I^lVr1N`v1qC4NEo>0_mGZ5$^JC zZKI8Zz5ichZbF8plv6sXYq|&$sU0FVwmCGEocw_z?25au?`=KO(6^ee<=ie~-WA|y zF!&iMD<8QYINvT}WAgUcJ3g*zuYnG4`OIHz7hNTap@k#qR2c}0v=-VxN5^BSLcpvm zjiTEGG#fS^t4SNgaAF~W8yN;Ba*M#?2!Q}y$@;L=XEFEn?Lp%q4QMgWXx?=g7@I@- zJpI7t2_6b=;*m2YP55RM0K?1u+;PL%U|h49V*lV^%A#RI?0-o2qb6bq8CwllH`cMZ zk}WZbn<2h;b#X+zgPOG$FIf5g7AVDkbavu*cJ}Rh9Ij#?Z>`KNA~C|we;_}4{ri%I z)W4GW-y^HkF{yeV2q-HO4?zjIbJxV<7aHJ}UzK@z?V^zA1g(hO8M4GfF{Fze81G`2 z0W}&GUm3qdGgf!xoKVHf6)*n%+bYC<`jzJS?PoPLSa5@4sk&FA$pWeai^4!$2wv(( z+1M7|$NN7&+$?B|CZOuGRB(>z<%$fXl zUsFV#IbbX1?vwck9Piu)v%haXMAuoyP4UX8aW zc&|VHZH@k5prT|&XXQI%Xhn;uzTy-v@t)x3k+fKDs*i}S{Tvv3W~hig>299v zH)^9_2#-2IqSt2=z{80kzyKC%#eqw8`9x{HhU((Ag$6BNEK`ES`~l~L;~rRJ`QrOfUfcY~N3f*Ufuqs*(;;Z~}pKJBxBN4L8XI0k+_XjX#Y{Aca0H0X4M_mh5 zcxLwf&!T9Q&&j(-P0bO?6+ro`|4pjN;NVo;c22Zwv(3-n9}h_1xO1C~JLr2Dj>Now zk0A{&E_Q%6P$s;D&r0q7%P3-_K0PEO)OM9hT-#Ict%e5E!-t(YEl%I3U<-bZ-LICLEqSu-4-w)6z zDK36toVmB)+*2vT8vC5xjLqmSJ}ukSKf;uK3lYPR(ML=})Edl~l zPCh<9z}k#%o`zuFrbJwPd}IKIn{py1OBp-UBwt{f@>t4&|C;g)E$+Gc0*rnPJTt_Q zRObt@U+cx{2FFqJZ5(r9Kc6o{^07D8J|5EwhX2TsFHH4^`MM`n|~1F=Ei_wUpjSN6cHfuUy(3pIi;leuP*dJ0Z_b&!TA|40Z+W#X0LW2rZht6kp~>K5)JN~7 z1CGa2Tc^;;ss%monFbw9n+Ysn1NFj80X~9m1?ACwFW*VA>CD;!yP6ROxYjTVH(_@> z(dnv@G=$CRq2K0=7D~52lK!?R;>I(e;<-9gK#6#xPkvlok}dAoe%F{BQT;U6*j?E& zJl7ZtLR+s=TatYZ8>%=oOBvvNa>-#~?YucL3nGa6OI0cjRw}%Nh2tbMJ?{+R0w&5z6_C^Aa}Zn zGqLXBxqHiF`_BTvWmMM&_8-HoI-huxrbT@uXc+Dm>3uk0Ly$=gr+?yTriJB~>_uvE z9sXR&Fs;TdKzs;PW0)J^5Ii#Vl z(mveab_+-gJ841P0#YW}Ry!3sot<$Fm`+v;ef{3l%NMpzd27q)adfuFY3J(?2=5xIPO+@BdY5jEh1YMKWzU zQRJ9Vt1CbF`ryEXbz)d7S&vG#B|9r4Vn>>e8xC)#Y>!B+tq*2D^{m)SMPmA-^fX9D zP7d*c4_QI&AbJ5ATvjFz)b+!Q78pAiP(b|3jbN_anKZPg-a#TtVIi+_#3{j@M<*_P z$G&MKB1uL%Azh*wEb5T`OWkW)684IvGo1R`WLHO{b$uB5F$j;1rm0ra!RWyRocarB z`Uz{K-hW~Qq#9OO=qa3oqx0{%{pDo&{yt%yCV5T;uQK^Rxy!>MzwlUhiCJAulAv0oNBJ z#0tFf0UKyYf^e(b>{}y#>2AnNZ1k?-|Cu2mWsszwYeWno4uErk$2oVqnS@d4g$8*o z-Bhlyza*J%P5w#VLj~zFhPMM;%+rd=k)y6`?6Oo`~8Z#~$ zIXMNrIv#8wN<_dBJGK~P#all>j=wmn%)RAn?+PRj;Qs+oi9Z5vwCMPQ1Z^-iF#!56 z74xZP@9%$)hQ(5+g;vjz_V*=Q5+8$&DHPy^d9&ee0-9_}!{d$_`=UJ|u%cQtR#$&C zgvi|iRk&^7v_4=2hQN0uCC@gd9o%c%^I#|K0EmvBqSr6PbQ8Gj1`sxrbrN!9|89NE zwZ8RoB+!1>I)C|?0 z@Mh`fXS@N#NH+H3F4V>)?Ww7bV<8?y%)JoHYkgn~qNQbnU9GV`i0EmGj2x1amoE;S zem$Cp2V6N0JU1a9Lkd5LlHL8{I`KJz^ZiY)%Y|zc&ooyC-6tV;JG|Th)Ong%txOQx z4ESOTA`cAqKRFp>zVo`#ZAcW4*P4be_n}Ue3K6cjkFbVCa}r;Wu|?j+9?*#h+D|YJ zu*E-iQ0M#vs6}eYUHy8u=J^KaTW>LwqqHLSUwJCZ&l0@aV(SB<7|2{nYP2kv_$_WK zd^6X|T=WBX-YnM6(n9B)r9`VzVS9vH4B>YY?9OxwaY>TqV;s)>O4l0c^Y%jLzs*G= z@POR){PT~`5~*)ld*nq8&yhkTt6tqbBoVr&|dZGq0BCexIcB2W(X>aAk@-<+?xL-TdADh&?fo~&0}WRUrY zq^5|^2xkUQA6UE^I5;0b)cE=?j`YQhdd+ne4bf|Ky7n}7bG8{U#wdD6qpGZ+M7Td7 z&o+kac=*)yIcBPzf$A}YO}x5{MFG8Kc!Y%g z4g5a@$N};nyo5$o3~j{Q3=*P{T@i!QF=Y1Pk&QN*e^4JNvgD!)PsYD1G|~`GtL9zR z52p_XNE`b+^S8_=rLb;HH&^Qj$!duT4)MxNi40H!Hu-wv#b@J}U(eh;)u7I1(BkP& zS|wB@_gG~q2Uq1E#O&RF01ji0h@*?k$Dkl=o22zsj@2o!mo~3D{O$`3{!pd_^jonR zIk~cORv8#J?3S;UExHHJ&JpPGy%1{KKK>O$-F5NGRHPt1)(-tWcjqTDTi7$zpx6-9W}(5t_d&%D9ZofO2aadM|I zfppfN0Omj-Bn;omS-lvAO5#S4Xn99Bisc!5JR10Dc{Z)h;UOS}NXg^!G=@7nzo43n zkfUd|vmOc6SMP~WpD<3kFA#leV*GI)%_i`}`cuMC4A> zV3zf5px49>ZK*f!lY5%q04EM@Rj=+|IpR4U=XyC=$>ZQ-y)E$YmXc)Q>{73m%SJ+6 zKY2c|aCujj@pqq~e>mzx#dK~BE-qSOE6}>N{Q8~BZtJ(gZ^tO)7qQJR4&lBOzIS5R z*R(3~x%1BdzW+oJu$(B`;; ztB?I04N1(a!pf3NL?NrUw}Q`wqkXQ=nG0B0F$V){AQbglOB%TKDp+uaB__Wdc_$TS zNq>B5dSZ$+7#CSuiSjmG=~Ct4;}f@=`v(2>Jxt7e=q>bm1$St48(j(JbO2vimqCH$ z1i;##_1rcSvd>Q(S}Urn+YJ39uweFouW$ZOd)~B~jE19P60`=J?10vVf@1cNj@B9} zFS+6S*Vw+wcFVGC$fmSl@1g7iY7-M0Ny_Ro=Xw~7-tWTk(}*jWD*xOZxuKYZKvUE0 z*t!@*$1r1)l4dk$jiE1q`MQhIf7ZB(Wly)Q@epi_K6tQ+9F_~qu^Mj1ZWFr#=HO!9fjU&2Jcov}^TJ8w~_Zn^K(5_4Jz(JK2?!e*bN^ z){CI&gnEOe#Nl+*ZQru9k6ej2YpThOCg5(0oNV5*>bNXZ7+E9j2H7Vy zs27kaWW&$c{?zNxmfp%Jo_?m6^fq#K^)7aGih(XOb||CO)EwD5*0{fdStq4H&v{DH zk@0RmzKI5jpIU;Og21DRi5H&&qPvoMDu8Ikzr_pdF-d7FkRzU4dzlQ(NYHRPCZ!dT zAL(R5*zv1t17%rW_`Rl9w;K=gCZ$Vg$fCJzV@R}Ze<6DHDJ_hguK6umB#90rgfLGy zGHA8BZ8MW5mSbhr!?7Xh)`6a9CFheR1tkQWx}x_V^(3j!taOzlp<^>6Y>#?w0~gjy zgoegv$8)zE>Vj*MeV|*ry6VDaNzUfbs(3f)r7l@}CaJ2)Dy{bQ;OMB#7^=MnW*Zb}0W~{eWt?|MeB=t-7^v<~K;-D$w)< zVyp%Z>qQ;=g%%c8ITF}#2PnK7Rq3Ur^dTf@)4n`qAzvSFAaJOtcND36|L?kG55Fi!{Kk#u5LwjcL4unBJU&=b=XfIF;QMq18`QMc*O17ouUYdEo73y z0^{SyBWc-1Ae$eU`3Yka?60r9qkG?8w;n8oHjA{%nR)|{ov^SrO>wOQleeMRKLk5z zpgyjk#MKGP8m8rUt)}`%P6G?H%zew*^5vQuibT!UyA=bwbWs#g`h4hIU+;_4eY`D^fpK6d-?)dbUdeF(Y2IGCb+t*sxg2 zCL2ANs!#g6xcQbab@Yx8-+1-T^t`Oz0xV~|q~N`ph?}p^jX3MmhVvmyyoHllCkp4z zjiX-;q1zsPUq+j`KNWVWC@-XXFoD#DZzA=+Vhtu&XEEA-j)Icnf<*n?LtVdjS4NaL zYs>Wt+pf5|o%{M{?2HJSy#IO<`hDjcU|6S1kYB}c4xWBQ@73xnkn7@vsWW%NuDpmP zcl8vncFtDw;+KKM8DnhgID(*`L*hh_fV zFqjWkR}4tXNB>pLf2A|sn1q4&1MauyGSYrNHlKXIBk6Ug)&2lO3$+ug5q<1#tWyKGZQ{M-;2G?FDjgy8_3T7y2eD zqqV9EKUE~#n{;9AvR83GEch3lemB2fz(+|A>a;9+-wcdh=*1{Gl%TqWC|bx*(X;tf&l<;N1EFaBTApG(<-y!;p_TUH z@7Q?_=IcIn)h_S*f=LMdxj=(4hNu^AV!!J@AQfr5`yeVE{1-!8z%AtdEWz$D(O;aU6XoM+tZZ56^Mf2G7_76-oPMpKCw7QQ}IzcMV#^dSg-Yy(NE3I&3S}`Ce>3gC2KHl{eAD+ z+93KmD81_I*SoL2?_3icA0OLh&0{|Ok>Bt^%qts{n2d}Ng?oyaL-~8w>MD1Au??zt z6bn=-pAkzq>R;Kc z#rp586V}=qtOtz*<$>8dt20Oo-Q4?8`sDni6dlKD!`SwJ-hjk2N$kW)IHu57S- zp6~7r*Z><5HD6>#nF@$TFL)b?G0hDq?$WhZ;~w_<9dwejTFik~A@M2m->2TiDS)5Z z(JPM7!ueHGy4}PQ+ZR*B%$Ae?5|y;7wpQZ?hOf)TrXx^2+FS6qA{_M1woli089bJX z0^x#6@Q9E)O?OF!5@LUMqQ{d?i51FXl;M7rje!E&!$TUJp_YMrm(TP%u1Xl!WT7*Y zc%@L9AmHg~7gAqSKe5{QAYv233dj2zkZcu zNnm$aLTdX+8B3Wc%?nL)QIZD>{5f2wp4gzjeChtTMu@XJFRmHFDeBZXQHV@{M3Za% zWba?Y&*{EbwoL%1VU>Dg@cQwPy{19OlaJ|`!*rZ%p-3Fr>e#JWwke%nPH`a!&DK_-JV%kP$&yxjQA99dUb4@x8wh#3~2 z8D^vI-u(iMD*fNOBiAHJp4Hmz4bU-xV3^CZm0jAoh5{!PNdI{XMEPI8hHpeqY=Fk` z^nAuQrYasuHN~lydfSEe%ph#xPg}k38@-xXl^$P6`qOP+WrZS0 zRBgjN)N!k^*|&qO6!aX;rJ zNCoOiUCq#JlrF$km>h=^HR>L1T`Qkx2jBA&4lk2QHp*zv+FzT61VWJ^L1O6k6r%QT zyO!CGS4G7<43@KRKe}Nhrtk>AyR017+`iT$$vPxG85FoT%gtA>d{^Q*zrb4IWsJhF z*S;U^n>w1-Q;vAh2}`#KQ6*ZslW4Z3pVO9O@hOsj4A>{;aa}cbcA;j&m$N^=+eN&T z5^D1L;C!CU;0~K~wJRDHXp%%ca(Ur);}NF@lu^Az6|%Tw95g#?UF>#TzVUa?dR~r* zp=l<^uVA!^@^mf1^5RgTY6PWOO!H|2X0MAx7n7idIb(W^jSozlXO^8v*7xi}!H!!d zLBaTUZs#Y^V{jwSBW>V}xjUu|4Abc6%7svMMHoZ)r%#G-L&lCL9FGe+CW;S$7Jvb= z2m(azFkgTiDjv|Q3)O6K|4y&|wlC5@?M(br!=&!sul(Sfb0 z89|13V#rJp=D0ofc!C%3wa2?Nwg2OhI$Vv5kAEN`kyibLg(b&IR3=>nYgZm%!a|mwzQVC2 zo5_2ntqdOk4T_-yAd4Wy!DY9B9OiTP3kY-r`HHx}AfX3bFu*`4`}P~J!Pt;e!0YOZ z?6aZDr%!KcH=nUW+NP)J02O;terEG2FUI$a#(%(&j#IP(txlk@I~x}(wM&3Ua(QmP z5yKhd^N?0G6ey;v*B;D@dNDTV;X3Z6G{o9uJNo+bs=FQ5bfyY)C=Q+L0v)$l9>e_= zJ0|?Sgv?w$9MuRMr-D7b6iA}@&t#Y8+V90TbBn2rcoKI1xO+c9y(GpTWrsk3IR_++ zP|xa$Zo~cgb{L5LTyHuhRYaZaO;GKt;Z^M-u-UkEA$BZvhAX%XcN)e%WC1@^tZgBx~*3$n?j{iii`xf|82-5LEMD z{$H-LTw055ASWf39r+@ZlQ6I{N7YYdqq7DTn;w7 z+H$C(*Y(Ey)@P7SL#pw(^l5F>f}E=PP26F`$AA(B(rJ7ZRqddkVM)L^iWf(+!3Nan zGJ4Er)8$omSHo_}o{{y(@3dNs)T=FxfpXaHG~UN|cX8A5a|N#%m!S;W5^|rS-{J5* z4aTOECKl|mpFtNFX99fYLJi{9hGgfGlR@gRH?Au$8FH1cDT9Oie0%2&0x*MI-k+rg zyK(UoUdY{CZV4%? zt~NRp6*_LNIdn)ZgWpP0l}OA$%9&7@`G@KjJi4sF0MkHgzhM*XP$wD{>|WE%|@+D!^@@K-FkCwj}sUt zLKg7A>qoc7i{}Ebg$YZ(DG=ek6~c8U8}i4BfO3^swFHJWd{`vBlTV*=CaI<8A?oJm zkbYaxq#5>$PFj~BQ8&39moQ;Xqu|5%y@sGSvernpGV9c5w9X2AuNV!^SJiH+J#Z$a z+J-ubbx+Rz3@$Dgy=JWVDC4WnWv{G+)3s#{x#-BKQxT}_U$Kot?krTRsoFa zuZIsv^Tku!UcERHs$GA^8oP9cLcq;Hi3!clHXU{OggGZUNear!7IBxCmnr#d%CIm> z?ussIthRjBY)f8~e?L#%#Kc6~jDv%t3VMFLc5z>sF-bAx@*JIAQxjk9v)U80e|Tt} zWq_Iet+s+D>AtvlYV`mJzXZ#Ap;-(uXKJuVe|kTAH}SxR!S(*|%NJ{+32%NYVTkS2 z-B1v7qKIVqMm8q&4Wy@m5+F-kTx&i440x-zlazdQ-@Qxf`2E$JfrV>E(^vQVca4B< z-NAp@EaBJBi^zb(Ka*?ne#0jxUESkxd(`pkaYvq@T z3Xp!eVU3N|?X5-5&g$a=RT3LRy8&Q%*R7aWA;~n4-S!mQvR*L3MM2C3$lObL%&q;PaLboL&1_nzIkp>X zZpTNJ{&28)Zx9%Xa0nS&DA7oc6_d)!B8*S489_+99ttj#cirD(V^2J7c2{3Q>oMV%hu^E)Xv+!W4$8Zu@>(XhE2pxPul^`YpF z(`kOz+egV>xZ(llzHuKY2-^GSkNz||E?5-yZ#x`!)7JU&YZ&TiHS>c7kS12yzVjs1 zbNDqcONQ4e%x|U56JM8nc_07XU^uuj()YoxrX(@E2`gA{YKtOmmIs-zROV)cwAkaM zUz-1YXRfQdJzhwOnXl#+#KBI6&}XJGYc7KV_dG`dpw*Ow^p^&&3$~9Utb#u?qMz1t zwpCrKS+WIygfQOEhlwMz+&27&cGLAbnM`_9)Wf!7!-7hi*CwHu?nq|AKxVF^DXuve zr@b}IJE+EfU@!>ex-i6Iz0p}1A7%jdpO3r97d-W@l_p_jyK@fUdGxI&_{wF*B$i19 zXm#zSWbh zr^;kfVg*@TGqZ7O)Wf6)u2*MgKm5^Q`0UlEt?md#*6!K`gYHQ~!=gz&qK2BlGiN%Z zY3I^{VMholX;~l!{-Wy>K)19pQ}QQ< ze%}qCj#CtSd$Y(pbaYRcNE4r5?9Z20Rb6?ujMwVQ%C4JUQ)FKODp4#bG*rNbE9lHR zvbC~cbii}r+Y#BnX*}jVzS^6=aJ~1xB`n}-(7vG-7dPF0BjoMoEmjo^D0FDd(WUBR zm_i5Pvfo#J>>khxb=8n$%)Z_Q}bTdFZ&%CxqivA<^5? zLSE>#y=`^%EMAL-mV&mKE;jbQ#Smdvh1nf&jS>Hv1;Ix4&EsjO=O8#bY0~Zn@vY6X z<2Uhazl2LzL~-3|A85{;#7`U>l19CH(Gc|A<&KT5^Wd|}by<@i8(4qNLraD=G;~6J zK|aq3PrwI|rZy}O9dS&Q@|mu0d#!AA=svvr;+;WtJDLI*rT3OxU+Hl?)vj%M39>3? z*!Gyk45-JPK}Y{+6<-{!q&x(jZHx~h&^1k}h{%3;Heeft(S}S--=Pp4It7BHk|!PTm@NiI=6n8E!`&Sn9vM3bXp5IO{N(= zkwPK96&JTXdoyZDOk+RtgBa`igpw_z#68<}SPXUeZkEV_5kEyoMVcXoEP zKc}0TnD98Q_IS0{iVk{Hs$)UI=kNfrh628#b`UIRG+P%Frk?lPaY>q@M8AK;%MAn@ z0@w21j~M1*)774UZa>4rh5Pp^O1WKKBgnpCx)*+akWmm20GVwrI-7eeApy1@99#Y{ zh{?_IWR!_j!khwSw!zz<1lZL5;=y97%031L<%R2djArB`zaY5VS6AOpgp!;!U>3W| z%RYZwk#T>NOy%G45ibTCJDFn2y7R5B)+gv#SXlnWk3He%ulnruPk8>j2_zT4H#e{N zU(pV{V9ZtLg<`jPSEO0IDI|mWG2-g~uV9BEg5Q6CbaYgwTTFwfqI8M+jii6B)u%6< zoOqp_oV5FKiD+cet;$TO-ds-3H~p7Czx%y=cgglyb26HY24t>35U3uolo<4pR_?FC zO25GQrF`fuC?u4M&|r@P=?HT@eSN=A-3bWhMDoXu#sp>4tIzf<8noXzO4UyP$=|RT zhuI7PDZP4bRVNUULINUaktQQqq?&0Fv>K-mK?Yr-&yF?m>qJGa)c_!8gkGsWgdCTU94L1_oQ&T0nBVRp}$v|3t&e-T!;G7{S0UDj&@ z8R}DS6zK)e7KFvM&pSL8k@d??g+S@jOZF`@xc`UlbCi5~H{n<(mFqjj=;L>Jn4MB# zh!p@hc;3%@;*Ad?F2^cvIMF7ozg;@EuZ2K>3Ji2h>d2dWZUqrfxO4Mn~CYA{P{3$ic0}pyw;X4@IMclQ8=3GR7Y+kAC(BY($q;3 z*CvSLQA6d|EC+*se7`9~SYX?I@@dc2Be-FQwnR+K{QwCKu2+LYN)P~#KT7jWptJm1 zGZ6RkMJQ!tR!zrWJmQy|>(E;Vc1ykbuMyFt?y=J059g$9@ea` zRs)Xng6hoX^jFOG^^b?0lvkYf@|5iAe7%V=M!On&Wuuk^Iba3?_Vgc}VekGPkM?82 zs2Uc7>$V9;rh3rmB$$2VeA#*(h z;lEx{dteN6=6pi4#}M%k!ec%Rl11G6zbqU%>9ezMZxAIV#idSia8p_lk&dr!TX*6- z8BDwrjzY+dmBzk#<$Dgvg=WK_thfqbzb?^Bx=BJpJA@qhHg-N=@g4i_T?4Bh^J7tA zVcUGG8dQQTPPEYw9K6#2!Wo-gq0pO&0!8$g1YJlOh$Cx4 z#q(-2Nww0*Mb27mpQO@RzD`IX$}9#ibJXpU0`>yFijXg|EW!CY0;HpTd%a4LO%5H2 zY-fJCJZ55=2^9#PDWL(#M3;)z`c|7^Dk@d)HFt335ThObuol zZk4;tcnDL~*Njuw=RqK}Y$pfW>4t*^QV{qbCI z-+czx_f3!o9G&v6nF zVuv@foi~Q+d`@{QcZSl$s6a~Cz15CI)=mV0_aj#YrrLI}j)YbvU4S7M@_WP?ZG1#*jbxDZ1OvN7qo~q7UEAAOP zNPRCc9p;JoM4$8Ip5Sb=Ag0Spr^CZ@P6tF5(99@+%c30vWA`X>8DdCe zWMm?bG1goxre5Tzg8A;p(9p_w5izlBg#<2;H+d+$v#U|B628Vde*8$efC;0ulK|>p z&Apzyh-MwGmO}PXR+^25Vutc6{-Dy*G~n3e&@9RtjN9uKS~0Br!Rr>6kv=Nypn z`2tYt9$Vi}ZV|iPXd9A)fG!I7t`(!R<%Zf{KKW$5UOmb*!iDKo;Rp-&PnLXs#!lf zVpx6o|8kSNe)7T8V^qBXQ99RK1rEGa+KSObsk^&9T9>g z+C;NmpbHN##oe0K)Jwu#UJ}sUUFdoI$Jpoh+|(& zIa*w_3z}$d`emlb@kjJFIDjgd@r|RpD|AzOR(qy*V-9TI%B&IyuIb#OYLyF%;{G9q z{8+jD1jOfo8g*w)Nu43OYRQRJ_T=iX`#AI!$X12=?t&a_M)Wns`(wPBzRTfxL} zgnC^sD9Wt$qYf)Nflqw@5*>tzNC8VwmVrH-h2>k>_V~Dbx%D`atQ;kX?0c+(mKaUTp;)uiTWm$hprgj*XLblTf9{X}|weW4K(1;qlz!`s7d zdawjn)$3FcS-sD{boDNF{b`P0i+%nAb@I7mfv0a(dFF;C1}go-@2!#M%m3Co{P*{q z^adBlj^0=+NC1fMd#m|Hgh;&6m1Ds&4cG*LUz|=H7N=pA<90L#0^xx+dDUsHXJK>G zXCQSda5fbc&wId!o1xW4w=63uCTIt_C@Jz18~^9v8doYh9m_gxJ}F)Q4b_ zJ`e@rvHwMn)#7IG7QVJZeR_YB-RKMf^!$tpTH8=X{D2s{RANRrYjp|5FBW(%^5d1=9Atg}2Zkzsl*3zz)}9mYKi4AJU2hY}11C z*;R4J&Dbbt^*3VZh+XHf{&af-S_%TT9gIXLo2Hl4K+|I4bHz)$e%^C3D9z6YiDMdY zH=5Tf(PTxbaT=fJo11Cim+P-D8Q>r@d3kx7jW+MUP(g(&Ev<9iFlvu&D885 zdBN>N#^%&WQtuqvEM;9Yo(L7X$HylrEz^95W;+7oJO5LT=K%0MWX+HMPRwW3Y#qTq z@7TOBm3;bRw!5EQxG-4-$+Uy=nuUdhDr#z2lnLtxx^$iu+uMI`J8$}7q}!gNU?N-5 zM@~)-7Thl$(Fz%xIBV(ZhJE>hKfy~v62-brggH5{j^Yv%d)Pq?amRdmm-Ro!W-3!; zq^!BwxAGhcjA>+Gi!9vC_w3l}e+}OKt*orz6)7pS@Z#c>Ts~!x?4k&Xl6Ug!-Bt}pAXepj)xvy)O#bm??~x&fW$|AN9G`d8VXqoU;e zB*o-bfRH+Q?mCIg4j>xOC|Ftpkuow4 z>)4rfa@HmQdlv{{f&|RYZg=k~`=a^uLDI3Xa-D&;c1M$&6Yqtdu!9i+4xSkEQ`IDd z^Ga9aU!X6C1c040swx@Qh;c2kr0HGr=gzeG{;a0_2%4F6qBofy=+7;%$BI*t+vhC5 zglB*~T&v0+XGSz~|Nr;xc5aP#IOFxYp4WI>k9%5xZk^k>s;xr{ z+V@9$fu>I6VHWtqmvAXTxlsq7Dp4)F`}YM$D@MM@YDKIl1gHna#>a#&-0`s%3tS@p z8z`n_GGb-hQX9;!ZET%)qs6}woWwVMjX&liINv8QsOS7DE=u4K_68iwDK*&Gt7$IJ zoe-STNh0;dZJV^cEPfR}_h&NXtEa16FI1-&yT&AUbWP&Nq5>raCw(MeDiu?HZslb? zBFPCRpUMM})0pGxB?;evu@LoyN@wW~aX|w5XDqF(_|AdEYhAJOWNjBVZIjxv+>K8Z z)>6;OvnklGu*<>fybCG}lP#>N-jfoaZA`7_PH|UpmUGnw0YZAkku5(edsxRx^i{CN zV5m;rM@=l#64UB1m+SaNS(uhp0zb{WZP3~EIxg7c9~^8!UcZ7*d%?hDD3%|B1Iln? z>A`lxDF1L>8G&>G8&`MX8cYX@pFdd*tUw?e_x$)`_oJsXK%!TBZh z7Xe&w8%==={YovL(PaYmO5u z|MkIxv*)WlU#>izAzP_58+%5?Yk3tn?3mAp5CEivV`chBW zXkrsu4_Wjfoj><0BwF7_@?W#UIuXyuvO&et*Jvt2(oJlw(RxL`mP!4dk+{DJ`V(>j zPx@?#m9vddV?$Ew)|>w;4jU8&&5G1Z&(Fp~s?a$61~4leA4 z#u%MzG-^@q!Bfi0lb4oeoj@00lGq+X`hX>+Aem4covh zAT<9b+dj&*5h~I8OAKA>te=r_mx~zS(sMH|NET*axDTjj;pPEj{)02)V%P2esb+aG zEtXG2B`OI`T<(e-C}K^YKwMp(?6BM8wY9MKbfG@q*M))iV>sQ$x9xd0mlX6s&F3M| z>!vW_Z~8#2Ner(keB#;>{m0(E!k^K#`=4K7$XCN~Ezitb_*Wgronp^1){s==*86L5 z4{t1dWOWw1c}DiM)puD7?BK|L5QQZ^UWioH4R$`janQa0*OjZrJ3#Ojd+L1Nv%UZ( z$0BZE%-?Dm)~~xSAGyRUnAT(Ks&W3P^Ns{7f+HK3Tl4-~fp<`20|Q6U=*2*h&ii3c zW*z?wR>j9lzz{WI4SeFDa{vC%fI8Ee^P=nUj&G48K6a&*N5JeStnB@_RvrZk^<7SA z@%<=o@1MIqF?Ge+(|`W#i!N*|V{QWzWCd0nDhs zz^Qrk--7ciOQ(znlpH$!F*#vi0RJOK#Jv6C!-vv-E5EYm|87y=;Qj|(>A63IJ$t6Q zvD@z1G2!@jaC->p?=L03j5sEm`5`(f&FFoNw1ikH}2&m-$X?2@;Shww6_p{ORYpvVE!hARVH>*LGe^+bVC)1wP zh57FQ1OG4LT>Cth0ytJIOocdzjHi{+46uYVhkTMU;I^gL{gdLEkgwhNHcZRu4p?1wSgJMQ_!z;J z&!)UvwLQmTJ;yGB7}#WF)F5!|rOSwt@t(n>DWj|`Fg7a?H)xM9h&B@c!ZbLJW2@H5 zRj{#A+db=CJAoI|7Q~;}8qnVb~Rg3!(qY$oRl|#ypOd;UvgrUpr8U zY3Dfo^TMDH5ah&KLHlbvajjuZ8!8aSWnmo;Kh~ zpJ*8I^DnimC~e&$N|i1r(g_SlEqH6z>aGhO@bm~VradT_y zTX)jb^?9J{`QZd9xT(S=DX$;Jce{vh{g|gnDVg|!7O^mS(>^P~Q_}A$GrOjD*n??n zbad*||G3sNbL=fCY1KZ_|D@NcHCI>>A-`;VR0npfmfpMg>9DJAnG2|adqPTus&#UJ%ksaf~ja6~XDQXIf5_6r4h4*a6Ba|Brr z+;_5j8b1_yseM_nqwIU*O~slu(Zl=Ih7==b?r`pK9Zq);odT$jvk9VZ+kP9Q8So_de;+{PKx|56&Du z@+xW8;yR?X39nnHPJ*pO?2~^~DzB<~QLChKK5_PA8#ZG* z$S32RudnaQ*5Qr62FDqaQc_%yju+ZrzI^$A7G~N1KWWhVISGmD_k>k^vLTILxBR;^ z@=_22CND2bq2lzK1hu?{mET|PI{k3{to%2sGhT0Q`rSBvD7#~7wHrh> zXBYqRKMFk-7q!;iY;2jngo*z`JyhA3CmO>ugf~t%g|cyFGr~h&ljf&78{W4cQS|!m zht^;j#!ULDtFRoCy%y&<4@duvERN)`*3Vbsg!Cj8(p1(x*1;Sk-}wKmgUXb+0s1Q|QlhQ>xx;kS#AK1dOjH83v> z1$#thrhl-dEMx&4tKmy)vF+{!z9O(P^_tZMfY;CjfFSX&ptI)2R4QS5t|jVKNno;zGvH(ux8 z)KTcP%v=Q*iJDG?ws|#qX5zaYJ^Z%_j&o!Z%1=;zDqxof>}MOdE82mM%=|w z&+*W8&s_gaZ?``b{&+yJXvarraC_lrcd%7@`U03NXCry^}JiDK|r&P>J5$6yD|X04|e* z;ofXt_v}`bE)f&`dX`IfX4OQFv2!QA-_jj8kfr5@9D~TINY5JI5+ipSu=AKKyDl0q zK5SqwJnDE5NREJdI@7Rx%G+Bn#pS$?TL= z>*v?Qbrr&GFR=LGi#!j$T6ae+ILqMuj_mD_cGP9lb;YLeb}TZREe+di`)1v{CaU%x zm^pf29}D>Qbl|u9WxMe5*&CrWvmKWm6~2BDuWtLL%=88^PjhB*B%HteYxTJEFa!Bz z3u9;S&e*B6i#gD-${ckMRG&0)AMZ`M#iSL4rw?_Sh81;LXOg@L@Tp z+qL6Qj+S~3a4LJQ-x1(~CH#}|yn^=wodWrL?P~U9Dfy3O^sTUDr-0?$Tk0WFNjD2X zz7ojodn-)XFTs)yFB`fcgJ|WnWV`QaoM@>yFaeu4n8inN&F4|a15OGxd-&B{6V^sF z><+kL3O8ZfNg~N`jHMg*+}3Q=0{XMYhZeUMh3}u`jD3Gh(OpXb-8zcHZWmKSo(|cb z)Z)m|;?f&@vgM6;`#mV}W#3+=`<3u1<4__Z=#v0GE&rsr_2`ieu250~MJ6PQp#204Vob5A@e>Mij8*9Lq$Ua1M&v$%w}ER7Pw-zLCZw{Tv99H(e#hrzo&*y|9_^ZoZwvv{|(A^-i$+C z-`ZN7>{3%zUGM+9jOXuj=4@GOJrJUrJ0mUs=>0d^mzBwym=mk3vH`IBgTk=2Knm8}GEb@g(*q;*oR*571C2e+{_R`}S~_QXS1 zS8vm^A9sAr&W4tzcA-5dCnvLCKV$y4L6rIS$9+X)TIv)qqqvt~77O~F7fi5+qDsM^ zXQoAUoQZE6T%Z{?+@SQ{kZPE{TQ=q&uwX2;XRjvJlm#caA7E)~zYZz(S+LKJ5EN*N zt#;SD6j1luOuW{rS*hCSZ_49!uMHNH2MSO+>7Ou;S2Hp)9?f>}{*Tg>Vc-5fWz2wa zfMo^PvV4Wn(Jt7u`$k5`U{!xx?uBuh;-|v$UR)86s==OHGC~5|nCm<%yF_wbn^Zsx3XK6&T=}bJ|h41mCYETwxQnTNPJ(62Y z>j?l>$IIyNwrx*?8}3R<_vOO^Q6}x6Ms~_$H8y$vsBiU-y(i=ITKMvDk^?hcd2GYg z+lLNwVUO%|@od<}@7?syM_Rq2_-W-mZ&5O*?rQQV_lR|MhA_DIuXdOE3mt3EtIC!duCwkJh*7g(p3)}z2e!3A83sX z{h^8fYfk*GEmmz*@w0jBrvd-ig#atlGnc37A6pm3lIzd@NuKjhVQ^S}7yPn9e|x?V zV>yzyw$oz7JgI*}p@-OARxipq-qZeqRIT8vGre7#f6Li?3)wiT++5v=w|jVg`H+-@ zu#SX+>)d({o)=pk>Z~%uo8bBD?Ta=Za00WQiemxZSD612@Z&;g6`m~xIV*qOIhu&S zze}TxrA#1O4h(D`l1iA2I84QXb^KK$iNp6>Ke_VUHdXvuqk0u31IxqY|Ap6~S8 zqf^sAF6ag)saaO)!REqUtl(5_6Na{-d|!5|cA%qUvyeIutC<|Ho{f1I=Iz;&m!@&% z(9-0NB#_^n93IVIbBvGotVe)Rc}sVCeqf%wOAYRtyK|=I9-wHFs9jtdKQB32XqSIqtijdc*l6p{m@jt?A)ifRko@qFjz49~J>-?6^R z0+rD!V*y9}(4^*>)C_4OcQKCZC0fIryyZY}rw(YwbY5ITX8~p7Ykx4@N@C^&4_ z;3a!SzZ+2)k_&kNcM#h7(lV8Rco2I+SV6%hh~|yVw$j%R?e1P;E;E~aF!u2|qfT2t zbXHG&H?&kYmaM(xbIwS2??VL-hMi|@;RJ3c9d@{2s}ElEz{ik6`yB=!Lzh1}&c|S5 zv1#RZ+v!*P_khuD>&kpid<5$UW0TjP0)2&9xh9^YkZz;DzHJdz(ve7^>;_;fW#3kM z26Ki+Jd>kZP*VJR2*Kz6LAtPBLX-%2k=Sg8RLu~-an)S zPNlzw;gF>P-)&6I78d0%DX(|RrvUHZ(5_}KFxgTp{Uh6J7r|DsAY3Zj`V$qFu9Le^ zKCbG;Ma9&7-OZx$86t*`yK%QA6RD&AE~9*Wp-g#kTY9I*Z{8FQOzm^tvq8t&V$XQr zc^E)DEqrnEa@=Tb8CY-dFkXEFp8F-MQ_odepH7(gyl0w>Wv6(ZXXiVbb}1zP_lc8W z!5yKpXD@YDl13f)~ScJqCVdH{45k)fIm;up;cG(Q8MW!*VTcp zBOte6QpfuTZ$(s%r-zDxZ`Rk?600B06;5@8P&ieX94avuB+LcY7qV1N)H2Ux5NOls z$a|}Q`fA*!)z5?PehE6@^?6J4M7#cT%dbSgBT@R_Lx4uwH^BtxzL~w{Sr!Llk)oSq zDENqinnb}c=IyA*#WbrDewSIzKQl09lF8y=gd)n|pijV2}U3vrjV#?LOY#&x?v&`@Ck)d%)55^4sD*IdBKO ziaGMv8!*xE&K&aeHl@a zgID%h2zOAMxkZv4pa^eo!ECO9y6oRbx-NX)0jvIc!Fg)yCCdkUZLN-!yf(JYi9p;x zLhg#yv1egw`pJyEo3LXCX9xcEei>OgvD?mLM(*gmHfvK;k#B|OS_X!hNXwwLAJ?y4 zpYQsX{|dIGzixmg(!^ zKUz`s=+Uo>vp3X4WLdtpBvo-;ct3>Tx*G@1V0MoXF*okRRdzs3YYMiO1MnIJ69IVs zYr-^KGBcA75C#}bR|;nU-hJc{yP&%~(S2*Joud~E$ajZMW%!(D7%rp3X1`j6*c5Jz zam;K_ega1A1vA{Rp2CS@16M*CwtB#_da`iiSF&1%g1N|-yvYMN>*9h#QyqkSBHLs{ z*c+V1e31}O8jxCTo>*n8YJwh#nF2HUxtr=OivD?YB)S+V5&@4BHSWB#el>+jc+VBs zq@5ou4N6>{USl4Pe(+M-{9QmA8h4KP3Bfd@4Yxf&FvBMjy~0J@`%0BuU~Biw@1Xj^TL9P z+FP>QIf;Q~|688<`o?!$N<*&Bu&cx7ICXuxIpjW&3|~eV%}Axn<*12EbXG3pO=<}! zJ)WEk!~jhGW0ptY#yo6S2KPU9EwS|FYmMSXMz{Wkm9Lc*p6!TVME-+paXVWzIcWQns5=GP zWt+3fH)1C1&!@G!KHP0qk;`s~_AX#S1wEesG`cuB$+L@fQ+5XM7Qzaqj0T~)8>eM6 zi6g3IkU`>8^4h`dTlmyfZvr^~rZPABVK+!&Vy-O~UN$S3l7sJ0axozVaM^$&yRD>B zzaI#uvq8(fSxQ3jM~(WiqyIVOA+|Uj8^!qj&Hw*fzbcl0`p1?*-v@#r zDF|zSXN(@3k3=83O8lBWeGQ0za+l!lyUX<6MAC1@#-0;7B;O||JJ{F9N+s)o?G6=4 z0-Ass0(a(C!1{iMom?jW8U$M}zsbhFA3Q1ZX*WO^e|4o^ITsP(v9?k54Hj?}eQqnA zKn@jOTO5mzfcI~BIG44WJoo!mB%hYr;6;nFk7D3@z}%u^dGhWZ)1&@4R6|dJfei|` z*RPF0V|>8rDG%ad^cGQXr@ouqRVFe1#*K3=%j0dAn*rGV>-$?HW+Q+JS_3}um2@TO z6@CAa_ZjweY69fh?CskKF8#SrS}8#!dxw@*z?MndHtB+};u35eF_QwoFvvI?vDEpJ zpA-}v0TBVbLCj4bVA}xC#Gds{q~LhjO-eDH*JqhO3?1xyLe5Z0T3TFukBDCO-Tqwg z$qQ&6A3qLQEYQfomp~3;4l6TEL-)wR5~f8x6CC&U6uGBr>Fod5O;^(SgX$r3A<}lW zoj`-rzI=B^)sPJyK!q>i(~z6G+Z|gaZNJxHFg*0MKKugjGk2u|4O>HvSc#7}f&{N(dNXS&@6-hCAqcQD6_3(XI~? zupf-RcPVrC57o{9UCwJk*H~Z6iI(~d-u;Vyl3YHxx6>fS_*H#QpV?C<5IdwX3f;@O z=)NnZn(fPJHQuL>+(jV1k#_xfpz)Q9(`_&}@8HQ93r<*sF&p~$Pl@kym?Wnzky84P z;uI#Y4}8JfO;4Qucx{$h*4J!~UKV}JRA_kooO{czyYyY|Bg8`qI51IxQuJMqt+5fk;#u|WU1kx~B)deGffD6U1TLVZ(- z%RX0R4hhTZo1VE_HJ|7GB9u$WNuhMl3-P&#l}KNfbGvx-=*J@sT{7=Zo!;$t$vbcF z>#)$QaZ6=eNVrzW$&LvZ{eo!!sExN)U%u?jEBVlv_4da}$)+TyFec4v$GGaRUw3!G zJmuQJA)~l{VU4WAyuR!P#)aEm<6gV3C)~1~MyCV7ad;=wZ5$Eo&GFX}d&e#x=4Ay- z8*H$dTvlH2X!Fs2w+W*?_G??Je~r`q<#q4PKsOW6(8r*9+4V%0CDX1|f=Lo#E{B18 z%Y2;X<2NY8E|$IfAG^hdc*q3$1P0N!j=a1k3mP0ISfHfQ=QtW& z1zTQAIf#%EF7C4xcZ=#nXnSE0HTCDZJ$ZIwPED&X{N0+TqWFHY*g)#x@>AHKj z;-zyJ>;{vi)8*0IWf}!bsF36Tw#S#h-CSw#hRC#Ym%?4jdevP24-QqqgFL1Y5_StuwUR|pug z`O}l2rkqK!eX7d$?8?akD^t^f++k-`_9Y4jqW!)6@OW8ijQrPkV7z zmf0jpvw?m^VHvaOejJqQpRf3QZqUM{{?)D@%z_+j8duJ(#6 zO?r9ZBU9+pmHO(2o~iC5jDybMb8I28Fhv_pWjb2|#0m{;gj+&|ogdjZD*QT%Lt z-pK+Jj?am5^WQs*n8BV|>%oRBq-sm+vzPWfSzNru4CK%ov%?R{?ByU0=J%E7h=ATp zu9gUOUlRcn{h%U9BU7{TYwY=_!@$pA`PCN3zrIk(!TiolBZsa$n1dEt`EyV=&V<5_ zf~%#-S@(A*ndKckTSdqR{!`!`6hcIRE$+U+KpH9!E@ivcWjm{B z?(8!uK|hU0EWbnt^Whjb;mHpc)pJ1aXq$ixuBgHlSu%+z5pm=NDuKX}{U1JDB6ZZ-?jZk|5D<8NTZN?*gn8?DT&PLFV|DfOV2*zeT&#vGZYKIJ!Rx-A z>~qYrD&lDjw+fFDSaOy$Jl+YMLYa1lj=h>CHa2<{*iM1}uN*D1MD>kEJ%28rphVpJ zXDU3=o`9L;2FGWP!7zBwPeMlsjyZg1d;p*jf%*3Q`*$mc5qP!#Vvv1NBtwEkG&Yp! zTKYb%MO%aOd05M;aLjiPfL8oee%B_>KDY4HXP4!iE1<2p>+~Lk^0CDY<-_<4I3D+g zKT8&-WU}a+8-*?rv`W`OgIe+#lg9C5Ot-*obSY?&GN-yA`VCB&Z%^E@W0Er3RSxMVWJhF8RBy|6>~VOY zF|1eM<^YD-{y6bIA(mrwi@hBk-iSU5)-C$W+zGI4bBK~|SX{p?AHSFZxy?2oq zV0XkTSQ7#301kuzmEmJZ$s`cyxMzi`)v^5Kh8G3vFec4(QK})#P zz2^VCNH}Jn76t&@YhX>L8L_w9SpmveNbQ(_PMqtnSqDg2mZJel$acve!*{$+0D)9v zVGcL279%5ZW*BwW0u$>nBT+3_6E39k2E(@h3KM7zJ7g?Ppk%%r62cKe_kY<;<%Bj* zI8QlK$8Tix(qnVwekQ^GI}U|tUK-m2iwkak^sP&Z%KN#YB~A@#SfFQUyT zVG29{*9|XEI7KS1?d`ieE%{S(xdI2` z!Rk}gcWJ?`lnc&4Bu)PK^70__eQ|j6j_JQ8SziQRrJq6gTpAp>v%I5w!@r;2l{$40 z-Z4m;pIH?@f%g@_r;0FD{8jhlw`>2fl~o~Z5j+G$lmWleW1%Oq-ycn)@53ki(2JJ6 zNIiu{`_%ZjUOanS@n!D0nQ0Md@;rN#2Ueq2Reif5&dSdl>li064udE9X zg}=FUMb5&pqGOxh^gcLxn0G$ytvX?-UCPrsqATdk$4oiLuG-~KBg&&af$yU)=p@MB zYS+*(m7>{(PheWh9f~pNj^~uH%nHL&0wNru+w-1 z;&j+m3S~&)*1L?Jgu!C9+)5X&w2f84Xl*WypiDPR@FTY0!FWq;qdV^kq(VFz>+1D_ z^}y}9al$W?o+9KzG`qjO$xp)}ZA!2q8+%9%uOHr@h`Fk#aN`T!8=n=w+%L4< ztW9ejczXkt7DBnikQo`VS_B3ei{l^@#wmNPVmZGHi_~*Oe3~T^*dx zDd60fezl$yj!2<-An0pHC~NUJk@ecQ6jebY$yG6E`LYTN%W3woFX@*~e6X%pS9G^9 zHTCV;RrjV&_#dKt^iUur8M}k!(i)LU(-heMd~x^9L!F(S3wXR7I26RIANz-e#a1yd zFHdFxc9C=v9?kw9WMN_XVgLx!N8@_iwHGZjdU7wiPdr?A=ty{bVGI7}JA(_y2hy+3 z1~)yu)O06lnkQ1T(r3vc6Fm04#=VEkZ|#QfsV9Ea(7Vzf#q<%Kd0Z=g=JTWKyI)ub z1`>%T*?OqrJ?yNKM*CS<-rOLdsl_q{n(y9ASo(BbPknp5tgM%WM@$ytivtUhZoM&j z%R}%m_3=qsTWe7E$k6^xb|_Oy|GS43ZvLiBMc#O5|MT=Ms~@%AO>Wt}M;uh(;Ke?K)^N4mza`YOk(auGg{_j4nT zdYxFxHz^;N&LlO_^FNFS!)9ZxL;OF5tY;pHU$b8zPcni-Hxv$aoadVy z&LxD9t&7YIGW?0Jjm#2G!Y8PIE+drfYJRbLx=wNw3Arl+mG!3mCdffm@wZnM0?j zkxiPZtLyk^ef4!H6pbvaQj%PJCxhiADgM}*61wwK`Z8gnf08zyOqr2#@xrC{8Q=M1 zXlYq(6HI!U7T}D88R|{SPn;-xwmcg)Lyj6MS_n#SK&Gk5`;u?rt%z=)5CYj@z8w*A z{arZ@iD@2boFed$AA6?F(3?Z$vhY%X03{QYCP~}}mqe-jLdP9pN znbVI_DCYDTDPJ7S=C|4iuU{fBo&_ z!U8uuTVJuwbn_}tT+|?fZhsGJtxD<-q4fwg7JjlTuY5SNyEDx?_w(o5r1kYYG};oz`$$KrNlW$(gf`#{|>OZrQjZ^zNYth%H+SudXFW<2=%-L{N&)(&(aN}Lj^+eHnhbxTM zl<@YY{>Dj*d3pwvAnrQj4_xV(;^_%qKDmQ_koXhnHwT$5~G;%iB zD_^qSX@R#|jZt;SH89VFZ8G_lfE%Ib3t~I9rBhsg{`2Y@kK@gE7EzT2;Bc_ZVOF>C zafBpec)l7@KT}hs+BiA){J8S^cwm}p36>g+qV?R`(sWEEdXyKl=XxN*ql6C3V>DO!`jr<~D%|a$M#<%f(A=_Ia8fjeax^>xILIrB>{Bft3g3E0@-4uy39>TMG zq^uNm-KwwR2L%Qts#S~6Xt%fwd#-27K*XAlW^DRWa*zSqLbSQVI;gagpFh3!g#NQMI#9^L7qyN>(Q=R$k9!a~ zaJ4sXucg+}De^SA`csx7gRGM;PkO z;`Sm9*vb1mtZJ~Swy>->MobnLS?bMoD&VQni6({TluEEuR`~^bbIok1A)E%|U2((X zh~k=z8{8tZZ3OZ;{^dS&utdOY;;zw3eCXC^yH*E$I|BWuVvMhC$p4iO1U3>p@$?%X z^?DxM_MN0vgfPg6{>{~q$MqV1)^uNFwZ6jSLpmqDnX7n+PCMC1S;*U3(%AlOLR-&K z@uHD-So_Z$ixf=FtGJMJCEZOx{bxQzo6(xp1my{pVhU}tt z^k+s-d@LxT*K!J}ds{I!r>N`O#7c`#8Q0j4rcgp&U!!K5aUcZ zMf-p9n_PZs8gq8bYs|H8{gqxU+(5U#-TOpm|9bidL4$g|RnFn?1L_9bx(1`%dJ+uQ zM?Yn56-J|!$}bg|K)p%E8rjdx$*H^j`D$fsioC<6R$+}*^}IAzB|ks?qqMFr09!}% z#;YIOz&*Nwq*k+IDSq?)&&?R?dfJr%-nAjyQ?3E4-$v=|Wi65IJx5Fr;|)t9GQli)#7s@6yd zQjbmxSh(1@a-|5p)u|Ie8pvE&icDKwWnQop{H{7&nydsOjX(5gYgFf8whnndJMKgW7BugzOg=lAPaN}ITAd@>EIPStP!#OduzIydS< zzGaiso~J_hdFx<}$cPUqy{_e(8ZsG^Hg7VM|AU#Q$EBiVzMeO1{pkOVGKE4K>Cvao zE(H^FGdy@om907BtmQ5X(aOcB?zYOwWcQADA1{bA zS3iv}9MEJ{&9=P~<+j~>B&qiPAEj!A`7&YJ?=M0 zvhf`fy(smN{T3a*v#PFNTIxUnau5Bo)u4 z{VpG+lKTL0+c$XXcKf#t>j-c{YO)+W zYb+$y*Kvlsf-W*94IX9m zKTY2`gQZC!BS_OczR(j>>eBB%SEr6BQCbu( zF2Eglgt|7ymcF+`|9w;vcc<_hLWNsee?Y(N}lVZE+r&#mFrkGz666t{`&p1 z_WJ0jDMQL?boKmm9_LDzp__X`-_rR8yBy#?bRG3Vp&2yMVCtAVNyxpGPwuQXV*8IO z?FXvEFo3efVZq|F{k|Ia_{cb6Y7T7s)Q$J_sw@#4(awI7^I!no*dyTcTbcI zx%9vN-ZTZ6>v!fS4t5t?*HiFd1sQnPR%iQO_-d;n3(E=b%^>wmPS#MLe^0-9!AcBH z2a5sw`zsTtSXi3E2Kp01lXLV?k~zc<_A?Mu4rrz8Ke!}Cw@Q|%_gOmrCi=M(7}0bU zfBNLnByRw2GCrR_J^|j)9IO;+x`S@Lt?l0BeJm`}Yy7HykD9h>+S+cZ+?8GwP<(W$ zq+#uK4{n-^Yc9gB}w6FY&&=8q^XYF5kBw&l6Jro_%5KSY4S(k zy15f%W}S{6zCY)vQExD0%_BVveU`>Nh05$P{ZDu0*q}%onw9Q|ZoiJLjzemQtu5Yw`79Mu(=Hd%VmTX?N{~8B z``rPdM>u#!jOa~h#IBREq)O3q-@B9DCvvC+q^6!m@TGVCgM&r#E*lS^hQW3CR+S9c z@^v6_WJrVI`B4)C0y>mR;t?UW%!2E8`qa-yS6|r#%fYd%=*0h(y@{^MP2NK~ zMwc(YfaBD79*?hZ`I5c^>3i{;A#wc&L3{{{ zDYJ3a$=M)6_G(<8wHhJhIg^@2nL}Co4=&a9ra;HXCP#}A6uQw6Qh`BwIeIcM&55xR zf+h8gkXI?ieUr)D;hZT>BLNx2!ZFpBXRv^O(nmJqu{oGH6@+ATWoi19s zR5Q~gLvoPktFxVA>NuZxMo5-yu%H@M$~}+JC@$h`>7C)zx`jOSrGG4%Aip5ymiWHZ zRbiPuUuJn&(Pm=!;rJ>;q95X93fX5&&u4!6ttVp}H7|ps()`e~A@^TzGtV|`5 z{e-7+j&n2dgnkZ%!X8Wa)TEgeG7@vMvC)7wr6D8^izkg>(cKzC%7%UnZ>WQwe~tTh)i*|yM%s+do7iN#cV;at< z%b{{~JNw4mO9ox5)u7LyYt1*(-=Bq|JseU(xZiMOCc9;;p+okHbr5<1wD_cgzEUocDx~ieuNgV4TWO8qs5{Ds_$lVLN!j z)Wqb2mf+UQfB*_0b7hvsc#I4U|={c`&X znuI5ye?@!Lra%*Fg!xeRlg{l-=7X1$H#qoGR<7@Yn^jd?JG+sMjpF?;#_y%L9<;yL zdoWhT&1~{efYsON78g713`xH%_A#+@Yq+W?kAbyoxj%nOD6ueyvl`J~b++Sp*1qyC zhNIIKZq^wdHY>!Sjj#gAO}n9iM1B_B#+YtRl^M%c-&pn< z@^>eQVSoBJ^RBhb&IwHf|AN;Slu8&PUU{6dOzaSf&tyZ_TcuA4C>sS|fnQ#4e2Z04 zHZ+u!GSU6L1;cj?srf`t!Lq2K?$buTbJlJ-#f)t$3F3KOu4L-x||-UN?K5`wxtoax%`-iKAg1_5Mk-<3Y>GmA#PV z#!lPx6IMn?u{JBMs_WgNYU|z4mn=)_oC{&>tSzGJwGd|-1&M`EWtESZ#$>@OZqLg5 zKm6?NQ)#RTT=fZ)V>xmmK~ZRO;|v92l`a82mTO&~Gyu0;R1}O4a|&|L8D?c|iAqVm zX_ZrSmy@zqUj3J`*MK~5oq;Ih?G1;qA zo)_#ipJwY}Rd8g+)^tf>VZRHwtss1Te4GgrjiO@IU)hxgnX)7XeKL-MWDph=zNSTu zE%h_DG~(wWsJ9y#ds__*w0dv*IWl}4v6B7v@kgxtSBGsMQmqhD(XB;#$bcxYvQv8f z?cQj4&UJZt5$`$u0;t_*9x28qCl{YL+^*YR6Fy?oM7R^7YE$A*wB9}u2#T~M`1FkZ!S_u!-o6pX}1+wNYfr2Ms|s5 zXdu9l#L;j4`{2Mp7UVc1bCJj8Z<$(oJkPoYp~=VCG&$Ui=g1L{abDKzfr0fQ#N^3i zYB_y*W9}GhUS9E7P8Jr{AxrnhtPLN};_)*3cFn~`kuM_+{@?P8>CZwaeW{dz5@$Hz zVv>@-h@M+H8qWhr z!PSTzOK_U)3fTOeI$HC|hJz!ut;5UZ^D}-4gkX1$hMJ|l{hoInd$3rX%M8(fz2ER} zB2Vgcck6}o$=Ka}JtA5Ny4*tup>8#DEc6+gc!d-Wv*IVWf;l)Mcg>PIlH|1&Cj9I^ zZ)k@fjU1Y2L=9G>E6p`1JyrBAlN`C=fV5={GJCd5;?{RhR21|GNaRZ)kCG?7cV#~t9`|e*5^LhC+BRG4a@^DnGA_T)0V2C)iCBHq~LJ-J$mz-KP5&Z zF=w4U!CkpG2r|tJE+*;pMeN1g{DsXY7;M|EKAZ+39Yd{u-u7LQ+%fKr=Y%_Btv!7G zQ?!m6e##x~|5`U??vC+*$Mwt{#%vpwyihfkJzs!mw3uoY!BR1+f%!Kfy>HcEWorn4X#r=u&wita~G5LF;lklNRL5(4LwNpXw8g}Yb9!BhVFFd_MA+o|abUes00ddR4VAML>srFu)^e`hK#&nvXNp zLEF(3qBg>B@o3{pqpvaT=Z@!Nfx6V1edSv7by$;Juiu-*(P9M^YZQfyH-^%)adl2o zU^L)QVKkRf#NdE0wmaoWjO3>0v4_`VfVzR2EX+YSn!{Al7=dk?dc&No&6Zf@Wt_)p zl+zaK9pO7FHm;+LM?WohJ24`&3@6Fl4$-|0$yl`1F-~Ctwb6|`uEtx>9 zaaEnb3Dx}fef$bRo3+qJeqFrvRXodo3BeY8zO5uRR~(}y(EL&vT9ac&$QIPNm+8GI z-th2pIE#ZRZKgIo3_5|c66Ju9^<92Iqhb&_&)Hj3Jp+Bmjk7(b%1Fn*Z%@A_^kW*KR0fj-wDCh=u;2$ehJRG@3 zBWg$u03NC$g!Y`q`KUGpTCEoO0|o7*DIxd1aH+u!A?_%?_)M|lbAvhR^4gGM8uUKL zF!MurK1b)#REIg3SLKSN4f~D~!pY23W4?D^9W?CVG0(GJo6^ODQPhJO(Lybb)44Tj zR!;EL&73HIp`b@q=(V`5wk~y8a&8VuRWK9uvwhF(d4_C$DGFGs%pApNI2M2`m{8Z&%8yo>`9i%yrkk2bc?Yn zWVudY(2F?*lxZ$PH7QHfsFQpi`t$2Z_?0%thv*lp#flC_<@( z44H=vO`MLYG>{AxLW-2RL`hO1Q!>w)lX>R9_Dvn{_kI8S_nzPDbj0)QXYcD;*IMhk zb_jCJtDotqb+|AA57(gBDWR%>dDOqqXd6Ex-bju$vX(1L*d`n`?Oeyh(^<_rZ;MyV z5x0oR@hfPNuY5CkY3hCtbL6X6YAd%)XieuE=#PH7=+b|`eA9rq;Oo+F9>T`O%f_Mb z%EU#gY-Q@OwzC^L_a$&$bW)Oz|3IsoYa|K)*3NtByUZjV#~5B;+S}gUUQ)+5n&$M* zrSEi9dtvD76UU|=9`hcHoUVQpFg;%SoF9R}rRAQaXS3tULi--qJ-MX*|51JDk@K6h zQu7DafgwFbiu3ZUH*U9-kR8oavl;odNc42TojdJ#zb^v+Vq5(!?nG?3`wU%`Yi&wi&-Pm2B#LjOz7TCC~qR~rYF#4SW+qh`>Fp~eYrrl8FvVZsQ^ONfPXQ-#HPG3OpwKg<>XF14@2Ag7IV@u?hNtfyy zoj%?68Dw^boJ3M;ZIhpDt^TVTbC=Q2`bUpfrY-WA57uQX?sEQyj?eMV3r!!|OTwlS zbM{7Y$XwJW@5Q^8)4uv)Umt@$?_cC6kG2T7#nimAA1>EzazW?G_*3*{@P(&14VuhQ zD)=~^H{ARO;oc%PW%$ZM4Xvlz4;M19Xdk%Nu#MfPgSc zNl1|A<699KC3z>Hwd^*xYkrU>hS%>{$ypVYVCeS1Fjr!eGT~SzTlDq5o8|(@>$y!v z`*Ij-Sz8B09AS`IG%qa(v@^)+m7L-ynFo8SzkPc=JsLKBSVhIFC``ik$rgiXk%U@Woi%49*3a!Wy2bqkfxWk;_7(S}8m6yxKJSpJe{=@AjTm+#azh(==?clN1>1XXiD#&&nZD`%u1Y>ni*o zxYo4rogyuLr&?C&Rrb8cTiqym-?q^}$?RYV0sRHTCe_~a{yv!ZD#C2pT}$T_lN8}6 z?T2Doyver?{e>TL9B^P5%zz1k5O zo1bo!3|b}iEzWX0CanmLk)I39!MM-nR}K9&cFD4pKQ&cPKHmSs z_vlor%Kenes+y?uCruK^>K^;vJ!0*>!>U+QaJ^R3*;U&%mf!kkWTT#n%PZfXCP$+a zG0=JFA!9L?J0~f1$nLDoaWX9UcR&G}gN<07P=yZR((E{DB*Y%6mYR67b z)^Ykr5D9Q{?A7>mTtw?dQv#OaNcnAo1ib|GzEAWI%lf&DUp6u6 z=y2l`0&F__<)hZIV{adD#wI2!Xq<4_vxsSdZR6ems3LbDBw+Uoqn5@&*cQ;)80New zBh*b_OUnm1>?#@FG-f_l=yvc5k;@uD)ILc+s~c0+^CO?OZo_4n`li?eOnxv`PE~o;wELn zAg*;fr#W30B2^Iyiu+NvXH{uMML3c1&lRPvm!2*vEsZJHkJply_f!pKL@)lw*8X#V z-YBkxe=_@CV_M5-(*J{AWS8BOB#}p7z6krQ33%4mH}6gWt6u4_e}33!!j8(0SMVzr zlT;ra#qUKuk2>?NkL3_xK7<@f#l?1V@}iUJ9o^5imr*793M*RoE|V>KL(^|U=-xJdM>TISe0kLBO9T-jnCym+DRxoq9f_+fY= ze0`bDoT+%Vf{|_UYrXzJZf5bx2Y8A8cd{D}aV4-fiq{_Wde+=exWOWeib=GpdAx#`5j z{x$5Vcm#vts}COlp`~RsFz=Fkq;TG|goMKu znUl#y_k|2S&PTpE4Li@nkzeX0TEZFa;gISUjBWp zr(oz8b&K;~g#_UbvF3E%)vM;i;B!odf8LrL1T8hus;F_)_vo3Hb5%kf*D78r8V_|b zOiTZI1M^MH%l=ZwWPqa+U#Iv53S;l^Y&|hxU1gape&P6i^No|vEYZy&I) zkt`TZg_Qv5SkuhR)U5~H$0@<^&DH6)QT$(QUWJTxXcf9?fB4`t*qSj3kAS?}iQ_@> zz+QUVN(MLfZO9nQb*rw{k3TLFGky7;jZve3&7#I`xAD?>KC9E(k6(04YY|ow@w6A_ zu4I*NZDJovZYy^icXI2_S1u1;<3GLS(6aH95$=6`R%taqrzF^YOsbO2SXR<)yi(pN z(qSofo*M!XtAK0oospblg0DkuC>l&WWI%v0Z_etnF_{)t~Wz{Ag7$Ez|XCqZ7sqO4id zks8o^D=autyGyOCq<{0l&@R~?Kj->Ots2KhGvO_B(RpPj!oxmkuv|}=os^Jm%cko0 zJ=0E52norUbQ`vqPUgMYdr9S6G4r=8%VgqF8BAtmj>RsVbQ?-d{mOa8VzfuA0}QF0 z+si3EN9$A}4%QzzNdInc22;jr{7^!dJbhDpZn_RM78uKpQ8zb5S%>=7NnsQ9foEIR z=|2{ZuTp4c@Go#>Pe__D5zZRDP(=7?dzKF`o zO5Vt~hrDE6dmW@0ljvz&IY8}1_B?9!{oD8N=Snda=F{iTFUcO`8iiJQ2s>{wDZAn8 zKzj+_7VOpMTcmY(2p z!_2{F`Nv2B>&5s{<8A&C|K{b=IdQ)ZEfwG8t+*rn7fEI>hX^V*TLAJ zsj>dB_|vX5k5U8**AMV3KDx_y|D1S!y`i2UjWN`8ZD8bqOhx~Jp(f|DZ>8e(LSGox zrtLC+DjT2sOvhdc21$>p^h9BOl>nw`!&S7@LwtNEU8cu1{UhL*n#|GHH5Oe?Z9q1w zyv%>OX7=KZc>qF#x5CbstGT{7;je*ex~oe#Dake#PSIug>~}YTpbJ-An|4^W^<&z$ z>pE>^A;JjN%yIti!f6-W*n{V#Q7@6dA>K)f<-fz9Vr1fS=w}z46 zqtKEG)zI;q-06M3I~bG&I1teOMTfdsO?pzM`T6r2^72=39VOr6c^;yExrU{^*@1{o z3v+UE`W&SGMaz3edb&h@kjUp#+ZpdHn|`Ho%5n{iy>b(cP7e=#RA+w0&d%=bG|*OL zB_Uq7j7IzQ2KCvR!S~l!9>a|rG;i&<24ABmT{G`+^uS(R`QnUM(S1pK{+~N+9^X~d@ivRi3Jmc3p28F`be&nQUcbV}G-{FEjQ+n^-y`WuYK_P-{cVQyNfv$U! zkx|p8@7$dfBzAM3ARR5MzO$>V^n=eDpV}|?c8Z++!lrUv@cS+^ldnCKwbsNWQv4HOI)mAWCn_M^mPyVw_fZb?=xn`-Rq82OIv z*xi)dIxy`5B#WvlnuvYhtP9JAhWqZN z!*5cm+s=fYb)$X%r6xKpak46XEOhm8hr%yc&6e4vHJX2H@4WuxS^GdPZ~HxSzR6M! zzR8Tz(U32Isi%q~l>0_6ez=^ilYGADiDoP>c3{fLJ~x!_k(+2VT`8c?>npbs{b4c# z){ftM5LvgB*W@pnTnCgog~^8gYa5jf4Z{zEjz6{^(d@=WY?7g_3d4c@Dk^-Dc+1aY zQ&O}zIXM*ydxonP_lLs75&i1bt4upFj`Y4yJX8!+TAHAJz}D7QL16h`+WT+4)^qL_ zNlH#;&-k0?C}t++t8c)%y~0mUHm@-mI9nLX7x4KE_X>}2&>jaDF>wewVR}Z{yK4-5 zoADYp=ED(V{(2dovdUfFgxmD;k+HE_AT;bC8CPvqhN0$DfhsXaA3y%?-<&VGYBPZ$ z!EN{O%}p$~X1fmd`@`)3df~8Gbre3T!NGg_yr{y%rflwf@Z}VLsV}gPm!;%op-e1L zGutHQdepd0PkxIpQ?|WxXPQSD6?i9P$9>`JE@rwmt?{}Y5@OE7F1SIE?7Q-olmkcL ziIr~9ORH8L98~aG{jtkaF_~DR5s^Ao^p6YRmtDSmoir8SL*>4zNk;MS@VAYV-S1>5IAm53gkv$^q($NUyR8i|ejh_jP9>2mB@9FjI@A ztULutLXkf}esolis1~SyJAEY|cmB)s;k&O?^Dgt(TBhY|OUDL1H8pMbBqgmUItDC` zSPxC@K*B1TLhSsRVNWlru-+K*=fV3w%DYq4%EPAGTRwf-k*SsTl{2rg`DXG}vsj*0 z%HNX>>Rjsd9=qK4ZO_ulwK;8c>$q=n>_)c37WQF80UU7DklL(vD}LLzYQF=A@5&~z zNE*p{YcyDjY2fa1Ej`D`%oMY#T{V{3OBweppub*jI;DzmcGsUqvouCUGxW2jN!G|0C2$^$#=CyiYTE%7b9NlWaJ%l|EPhDG#!DJ32l@ z9b=ZObo|({_VgxS^_mNd7!0|3XS8^mugRINlQpfA-_MJ(rW-owOr@nA$3Mx2^~|Xy zy4!Vx_0D*GsPf&hEw$mVn21M@_Q0{V-wihaL`pjkox{W&mLqi80R-f~c_TA8H1w(a zNAYT!oXK_lt1Xk)j=y;EqHDps8;d=&6}o#JcsC=+TyGp6_mNn&gqy|7Cwd>Rle06_ z+0z3^wY($-DanY4h$xUd(Qx;f&VJN623mq_!X_}X4NAYnm6hD9maNW*jZ6PDQ!T5b zR4z?ROPgOS%h?XklH(wra@8oJNpKwQ*@_?VJ~l3HS4U&xslQoSb@pFrcpM%sPjJ^N zubeSwtR3oK$>A>Sxb^)o>cvo1wHU|o7pJeIs$K$L=!vQn*#}0U;txdYo|cD7^F4{T z&~kS6)8$9!PzDHoEtR|8<(&2h%Kz5#PT;Hv0>NWD(AEl7aD1@RR1r-YFX2PeG&M~b z8#~oEE=}mFGA+L{Y#{WJ9BHw!(wn@P(feTy1c-9)(9bQ!$f(ai^B&tTauLS1*hi-l z*Iu-|OGMAw`SUzz{g>&o_Y-qaNl>+1)ov*6%ZHb>$TdoNB09bY79>5nQtVOls;| z_Lz+jc~@>#Eq#cQQO+2t6a*e3M?;4)Jc=?hq0@pY<`mElmfL8sEpFe;0 z;MQU3M6uF+`Nj_s2wCzc;UsfzTMlhKf%g3(C8(= zE{!%KB4y(arsUfuZGa)X_~+^FDp%g2s4K~4wAK%=4QS6>)qgCMov|}KS+MopA;cc+ zm)jk_lBuH1O7s{+XC{ekNFM&1CBL~b&^)Bk?7V1HEnSw!`IK8H-z$mtW-lXr)zFZg z|M>GiMXKX%OT{%z`|QEaTNcpAKVnZ zZa>Rk3%afaBP5=-s=#$4*f(*b1&sPaNuTQ@|3;gx5K6wV`>iTp!;5 zai7}eyjw{Qy{A-AEoCVUplFYdkB=uDRxs-V!A8QhCK)F%?W$~%(?MYrkS&n@bzs=h^D`llUJP0yVZ?0EEO zg9;Xv=C5|}pwHpf=4R+qLU)VtC*N6@T=T9Z2x)^cHlsp!|ERF_j{{VOQvZyZ{AIPZ z<^%ip?n&%U~~^~)kCYMrTi1m`DT42B*a=>)l-^JN0>3yjD*FH|as9O1iy-M!$^21F}rB zt8zmu*mlwE*IbuU>p5v%2Gd>|&Akrvy!+KPHAM^z=DWZDXWgf@*Y}c`Os$I5#K;9( zJ3G9j^()=F-dzUQ#JH?C`%x3?v9&*`#2g)sG6uV#`6b44g!A+9o$19l2|faI6FxM0 z@tajwWzy%i{4iO4)ve_T*w4X!q--fDC^+is`UYHPYaI_D)K1A{f&pcPO;e^Xz8P0Q7{VBc8wC``HW~_2}&F>qYqH%NPS$sQtVGqq< zp(w|CO-+m~9qet&E&KY_fP1OnYsU<|aHijQ1>lhl8cW+*f zPFUN{qv4{?L+8_UtF?Sp`ai6(IEP9|!2*kO{Ix>@#eqj0&veZ=m7whwa`!ICIhJDw z78s;5&|iVQ$8%A2xu%|890VwCY#Mh~BC&t;u00s6a~ zu&2_yldGC-cEoYwn1?jw`k9Uobt_m*O^gCizqtKZmSuTuZJg&iSBET9T&M834bpdu z$O`w9|M%vJ`#HN#v^lZn=;6aBOHZ6QQTfuyFH$(`S4*#hHyC^!tUy*I4>z|^2i`l9 zcLl-?ZY=Sx>>NBkvpTK)t8KTfdU5-7o5-$R?6!vw;|4I#NMZbryp7P@>)&2czzxf# zGY4)tBfXCV?wiF_e)*vFpq=}FPuz}uS#PI)&KNl*0S9!4W;)a*z(t;UTC6c9F?$XyAdhOc0MB`qocc9 z#Oe201B3aqt4qV)J|#p@{%Jx&QS>okB+B~RF5^Qnv$>UULd0-D8@e_lJ6VZ&3_Y3z z{lvq@$esb!pV1h9#~rE@u=FK4gO7fl{s(b#^;}2KgllPQN00UciX%_nkOHrsXj;zy zdrMuz0o@wvu;Ejd`UQ)uhdA|F@9DpP?r#UG;L9Fxla>>mT=VWSe)NOU;qH&$1WnPXXwSV@M;@uWpqFJEIwY%PkJa2#-Y;#{e68l)04wIWbF`=$lAC+ z%*8UzKFptu|Ku}T_CXm@4!W!xr!u`74CZ1WP%ywi6AnmfzI!(iAb5kV*Xw0zS=q7u zm*3i@H?gop($fqal+@I;&YpdY1v~F3)`9*Tpco&$3l|b*&RsZO8P;0F2~{fvfj0Tg zNH7*kVbdRPZj`t<^1SF38swV|v}F*w93h0xuVwJbq&JTsVhrUy5jFh4_}hEp-V79S zrW8MJeq74@f^u91H&Co1usQ<_IND;&tRz+!;q&)^yzjY4mj@L3JGz}hw0d<=DbiCT zP4kDufeVgjI3?lkY2MBAY0iNA9V&RM zj*e+Xu}_|;^Rm3W{TG{Sqo)ucG(mex-uQ&VW$d~FL9QX=YV_vvnWyNOr&N(}bK_jE z7k92evJ?lWtZTK{aJ!|LFzQr_`u7L*U*m4Q`@a=35Ik4vn(3w#=0-pwTvf5Fcb@%y z&~$W6OA)g+j!D162}1uDd(Zv!8goLMpr8gI^&{CE=tcqV`C?VJh}TeHVFObrI#=3O zhZXvimzMzOhvL8U+q0b=PoMGvch%ulW^2w;UENhFYh2>XW5;_k^&K4@@A}iZpP+UT zIc&4f|FMhKl6AV#sG!COj__{HvD)WX7ShaOgz@-nT zgg7N^KAzgZcz2`;|07FO*~ogtm+T@F0-`4-E@DBBkvi^%lwD7)S*c;JuOMyvsfSRZf(K%Y z`xdVskGj8#B4TQtdo+<}A~k3R>i~-k&It*y&90F9d#lj#^}pAvevfyf$3_~PdY8SB z?t*!j=&~c4Jh#nDbs)==-SC|7v6~c*9}fblyyxgTj9n3P)jKVVDEU+VB%+w^LniT{v}&V4E|H^#fHQ3gw$MAtgcBDH~@+K?od>B5m0 zk2)F%s)x@~hRjRGy=6shsLMKrsw@IZ@-(mu1P92s$$K)b&dBj%wKiWaw?|V;t8=t9 zY~5D^4TR5pTE%c3OD>2#BlxM)zpDK?=X|9= z$V;T!coGu>?wd`ok(D0w$x&f;qI0{y-$r24*q9T5U)AC(yH_%K`UEXpMn_}QyC@TO zcoFpS-5yF9`-08%AI5g<0(%mT^dW3gx^1QgQ6x1;z~@Mq_SQT81gXD``8pegB$YxT zQDGd-vu7eEez`kO)ona)Y+OVr`GGA5gCY>Hw5haz5zPTUPF>#<*rlBkBZZYTaVAZ_ zJzZ_Nto>O^?Gv<%&ghmbUODJQln8I_ABCYR0F!A9#40nKQ>bG(dEY!;xuvCrg_%yS z?tD4ywOJ7vx{xe|Od-Z^T+$$00Rdt zUb*FC!-&CO@|ci=T*0^Hoxxw=8eP(A6v-x94pQmXVX!$A^1xuk3w-h#Ci-79j*>L+CDxh>9|;PQ8FqO^7w(4H1Uc z`sEUMDab}4BH-;Cr_Tu!yse<%L2@cZ#WI0|nD2Y*w>+f8-^=iEY;tP7V=Y)Cy21>D zx0lyDngcmDY?%s5mg6$uzJ3C)p!{deE1%kl(f%3Q9IxBIkd#Q}<>mUXbonWLYD2`M zlv*bhG!XA_B8H-*ay`^fSmkyE!*9=Yj0XDp`w^5+a<6c2c>Ly#o}bKEbZ^jc|F}0} zCVa;eKdNu1Dd|!9?CqRycZb*B4GP_ZPNasTU3uF%_^GLvEdzLmJHLLG8G1z5dLHje0jt0s zUhw5h>D9$6XI6~AxAhZN^oCT=eTz57B-@oi;Pq@yBSHHXf(^*+)fpqEwF;L(Sx668 zTPq0c+<62yuOmJpa4UlxT5XFwuw%!LuB*HEKZ%boXApR8Fo1uz8ShT{+Zo!&NBe(W4%&mblq@PpS^W@>*f z3Rf&EDKX?;N-sYlq%t9dKUXqi@-)nP^a<=CIcl;R7|%CL7acxw#IUp^O3kPNAF%uI ztoQB8RIsAoT5@WAYA-KjXtoripQM)QJ|T@%)zr#ySs8C+f)PqOWXBy&HwMG*pjW$)j% zsyKU@>{GbB!exO-YtZqj+c}SBl!aY1Q~A#vGy}_DdRu3{#5Tz4wuu(ZJ|js_V>G?C znm;Y1n&#KQ?RxIikt4T|I83%O3VOd^X1Q#JIl3VR^d{_a1xiFa;4#a}^+k<-$B_I8 zJ}`LxUf>A_BmJ{xwk3o;I-#pexufK)+Pb=H*bSn<=Zg_Q}Ng=!-Kfer$ z{h~=nvXjC`tO+N7yF~oOhjqy*Da*9o9T1&^B<3-x4it-jTs%t=S{5E**ORVC_88*b8hxve5HH35zs&dA|`$ zJWB=s#DUB@gC^tzczq~r=c*+z{BlyPG|++0YNmSNGHkU z4hIvGX9tNx zdG+Mzn}A$scl9-ov zSYsU+wLo)7pgt?|%KT=QDrwdH53$7Qftmf_x6WcKa+;ifWr{_ZgoK2*-h^~=);$hM zy90$76468>V#=)Zv%}xqizs4A>L9E4-e0*(~HC-I2=$> ztxjx3@OK)nvG*a1fS60)El$qcODZG*fkWKji(Ksl;up6+ia6ojq=VQ;21fwYqlkU; zYB3@{p=7%aPznYB3P)jE9qfcKaS*narUVYB^7+XtMQcROgTV@)r4|b%HbGRs)_T|V zly{WH-nn z-;gz~uHL(J=~4wuqCHLAn{rlS81#=MI%W#c(_Rv#a{!0rhAyE>Adh@(VSjC^2#iMm zS~LZ~V7D`v0QUxd*9%_&(il5>bzfA#(?sDO#F{(5sW&U|CmJ9=e}ksY08Q;tVc!S? zF@?OKas0Rm$`4u=H{phl>)_O)OBC4xi+-bn#I=ik%H0(&bK8v0oG~8kD5EN4Mws7M-mI= zI2@?TbSoqCc2~N+?WLKbSSkvHD~2Bi8-Hy7PKWE#-?+S@cG6t|eRG1KvbeFuG5<)*M-TH}2p{)*p`3q;Q$?Y()Y@ly z89|*owE7JM9?;Lx0@$SH;_!l0E`WnY<67RU<{&6~#(!y+Q{*5jZdgIlCa*_wbQ)Rh zcbgy+LuH&fgaFTZ-F9m-Msfm%2FD$wnH#_-40r9AFj6 zR7B-B`5H>Vd1@!Z87R9VP{#=?KU^N{ERTKpvZU>qZkjrOz~>n?0O#*aDS_-CKRyTi z4uAM?H+4$&;LqYz2x)nP|hZ4HtFnYo;5f&CeQzQTo=d z^WO~yf>oms^Z4V;=-gG_6V$}wlz@&j;cCQ=ADvKO5Qab~`Bb*{Fa6ZdA?>7r8zpy8BDRY-{-eqvLUTj% z2&n2#zJ0b0jspV3QF={Eqe}km4|i{-_7nQsgQOf^ga|4vM$|8q>h})1f8PhEvL7zq zvs7eSM9jbhP^O%(AhKUX`ZJvrbe@K;`In2a@wPt(^m)bYEv9uu@MJ z*ZshdvoHZL@^onl6(7Kzk6C+QYG9=cUN<*>lWGA9759 zsO2FdfESFaq<(oW0E%b$XU!bRA@7QDrQ<5w2Cb(m6e5Q>9^8SzOLCy*eUO*;DH*>MnLe!R>x? z0|!$l8EF+y*~`|KmXw6E>N&h^!ysalqDT8?qI3$;UBbh&+j8nu7%pmqnn6>k=>Ss7 z&(tO+I1va@LX-JL8N9d77`$mp#!$lk=uJbFrnaLk0XiTrvseCKi!&D{RUiDM9Ps(1 zArpYh+F_;F)7|~2-tU?^`0RoVZd72HhS^RRoOxkG&?=f z(jaV6`}(RZB6XIM)yS`tO>AfxGX~?pvT~^pc7;pUg|4Nd)ejOAg|QRB2k2;O-Z|_C zBKg|2Yj*Fr+Cgz)1h5E^w#G1P;$>f7AB8JNi0x> zaIY0L*L~;L;(t((t(4;5;2)t}B{R#VsOU*KbtxCq?)?r!6-Jb6E-tlki7%>VxnhUO z!Sb@28nXe)iL=yQqYPIwDz0@roi*74(lounV@+x;ay=d557&kN#Op9A#l3-bY zIf+6HCEvqNE$~_w=TM(T2iXv2qg5@0QG%c9+9qrm->$Cnz$ffT?|L_mH~ML++Die< zdn_UwiKvuaU5P8Vyd}r3vc)Rq!GkOknQ5=khG8?%cKDCqNUrbS%w7Cb6UR1aXlgFi z{@M;x(1DSDe5<$cthtMK^J)o!*S{`Pt|DMh&Df&}xTJ`dwfcPBn-fLn5;KwVVAM(p z{EKJ;`-v060*R5lXDaClCZ(j9QpzwEk)km7($o*nFac8at}rQ*d420zq!-cH_7YC% zy}=y3J(pA3NC%-!?$m+2Gj9Lh1kHB zpv2dkyFE%W6q_Mi9uA{m)&5;!kcPG%xGW+p%r=NtXsU)Hu^NtFX`(M{sH>mais&WB z>?JGuqkoh+ee#^}6Y%ov#0mTNs#U)?4t6L~<(iBJ*Qrk2O^D|3fTCB8|MF;G1n8~j zrTUvtZ1*{3;EctDh+Wu2I>Cz|fx&%Ny`I!WK-TWq@ru_0z2#!J;U0srpMKD~^_zwd&FPFj1H zijWm3_x}SOtwo|mFbHP_Ef7gU_X2n&WT0`w3>Yb#3f*lx;qk^rG^5wWsn+9xpRqS@*KAO1k@^~l12i!%CqQZkJS*HZWq;xh&g;FT$$!*$u3 zVKK1~RSdDIJ@x1<)-8(2ZbWh9JCR-rZiwc->g(694~Qp&6rJ2l{f_|!t#$4ag2uu& z2}1AK=;$a_!UduZ8H7f#CIbvUJ`k|vTxEMVteEUFl05=SsAwGk7y(Hn;OEf{n#q3B zm6%iR3eSz>>IeEA;kk6by^A`11}9K_9uAXoB<-omz(|MZ$V-gAM52V63l0Dza{%v@ z_U0*V@Y9R}@Fe95ybFHP=%?rfH!YR_dHyfU1Pg&iJ;PgV#!-I?cc8dgIPcO`85kV0 zlrZ1Yl#=M2#(rh2JBK4PpM^04Z{#+yYZ8lfA1b^Z9U%M%yR0uq2D zIfF!hGI#(Z_&(SJCzCJ$6D&9HmU%b! zF8lxQ@>o8UqUi=1cPY}55kw0YE+ma!8{J?2Px5bWZr%p`*OntNcDZzg+f!yYUJ{Q2 zM)uupVm+z451c+c8lsrOzi-CImt;}*hK}Lpfe?wZQycKVAh&_7?=Kh^ZRRqe^#AIZB}V5#mX@1rymKY#HDZI9KLkfPwczCKMW0BD|yL% zRr8>e&u`yN{dN7)&8|hP5xygaf0K%zq_ZWk(?i+C0-?v5x~2dfO*0FmNm4&}1omIG z2otmjAalHP6m-@`0jDTYJ0t{6-+fojP<$9f5Z?emxX(=6rF=bM-jFycZ$wF+bOe5c z(F^+?J(vuUHJs00di;0`<}B@05RjHW0aJWbwRs98SQ>492AWMUcYiVoPUyxQFZ7+< z*gjL~=h2n3{I2u!^OJ5f(i8}Zl$AsJ)n*@gPkRxBQ|e*c02Q*uk(Y~+#u_eFxDnJ8 zq{&tANEl8?*#LB)c@%A`F*!{_etEM3E@SA(z7sHEMBp;At&IH26Z3ZQYA*%C{xM@* zuUPCaM`$G?>7$i|(%8D%J3C8iGn{??({Q_OT5}o;Lry#eLg@aUkWCpQ`B~+A@!I&y z;CS%PaX^cyP~;7kJfwr^j~^T~_&9#n~JCh*s7-?k0ata1$4G~x~c0pmS& zP_*X#Fz8#Q?YLXw3A{^AWaO5WwKb{rsikt88l@Xw?w1=5ioma2hGwuBdB47Kj9drDBxjq^^x}cfAe&Jzg7Z71Kr{M5wh;4mVqVLtU%X(R0&)MNe1er; z1$3y7#BBH>8|lj12O(UG3FG%4f`nWuTCc12ht}qU>Yr5Jc_;T)V5gF(MJ>>1rG8Xo zWP9Id0kx0GcWlHMKD0km4${=VhQNH3TW*oX0elY3Wn^3#s-|_=UsvMT`PB$Z=!!nseMZ zqq=`D5Jse3(@Xhx2T^e4_H!3l#%6UUL&4on`6PbTzdf0tEhMNV8LQk;`RNb z&Dbt_6O95&t^^K|3qTOYse*r1qhG^@n@XR;UY!Gx`^ReBMZ`nOY=nM>8J|tch-m}K zX~|WXyh1gOz_1EK)yDNz2-s}xvy!Ct{vD~WuON6BN7f5<` zk8h{l*iS8#@0^9=o4ZiLEu*{}u#%6Y>I^04-LQe5u(>x-c8yMoIb>}mK1{5Wn)MwU zSi)Y)W=pM6gQWvz3fp4A+4QiHiF9eGDl%6fahFB`PMSmhv;VQWseVB!JN*i!`uy4} zI?oMu?@0y_7c53A3MNTzR1lbAPj28s z_mLt5te`LJkuUAOM;iUkAVvRbw0m{u)-A(R%&lz2&xQ)ex z^hM1`lCHh2?;%NUw%7?OpycsE1>Pb_A8wW;5hRISB+FBAnGCiG$jf0vC1&)kcv{ znpLDOe*jg^$2 zX+M{P!HLjo3t=5A!%QQY2bm^i+u=wtYMW0-EBFT+$Z~23e89St0$FG@I@0s$S_-$c^EQQAfo!&YB$f{m=Rl|FF zJ}v97<(c2DV7XnniH>yPR(4}}`9y2qaTlq|()qh4SdZYPPZBS&1{~NCc(y#6%6^iKZwua@Jb=PxS4oyV!q_ ztiVxa<-E-n#9<%xUjP2CYTqOx-GlAmUE5)mmUt>>iB}B%qq;og2GWf&0I#$|;VSh*=*a?5q z`VMuznCu+du}!G#x7DA<@OZrB&d<{Z%(UQx=_L=1-`Ie4jKK6v4cMRu?0#0qpl%}A zsG&Dd^rgZKh0Uf*#!|xWa&d4z z^ic1UE762NO39qPxW#JD`TCV3(5J4W11T2onqOn$!v2X1gkuXO92*!B6=nVtpy>a& zY7MB)8Kbjj35uCXXPMu=eWU9!hKEF@@(1Cs(e9!1@@#&3gc&(o^HCa*58J7?JTCGD z2{Jf23Id{{T3#%CFr{OelE4zICQb8q!on=JGWsi6Sj3YOjhN#niH9G1>IG4@>%W+o zn%e%V-yNO;yLM#MOEyXOOQdYWRgkgaAm{?_-t7Pjx9i6~I+{P}psBC^FBrLV28STQ zQcsqD+BM#7j5MQcS7xykq}GF|7}|1{nJzHsbIPw1z;fZ(k~b)r+Q{nI-qv;q9Y3gI z4}t!VZt^oEVSCj~0P=5AM`nWGE3b`eMnnXjSthkGnG>p8Q|$2v!OHM!bpWw|BGMfg z!fXDMm@#t4t9A=s)d8qZM47Wu&b7?@f%d9(mcdThOGTgm0u0V_Jcr&@3XP(W1arNe zuqPz&0bMSv16b$chPeYIbwOK8b8{PT7s@ud9b`1XMn9W2xtZk8)GMagNc#jA zt)vzBPX4L!?F8R4%`Od1MZqXlMCLR6ezZ-Ii!021J{hcA9PCGYZE0$jK z)9Z&K0FW5>l2!k?un;$kB^P1#9rRb$?Ho*N&=E32LT)D95j1I35(z7RFvHV=8Q|<0 zM2<1WZ|rL8=wP>$iTmyTFJ11|eZR2vR(3;bTQ_`?6=G^FcCSQGiF(zNh6?(BWfxH{ z0Oi-{l>iM{(+0djn#3r~6LHt9Dj90cU^$L6hstiu>nOz}+A{;lX0{-&j}z;gr5!Oe zq4&|d7L^FZDx%h&{`>mgf&Q>*-L0y@f@7EkPy`&gG3EX}6WWdKWD^;YuD{u*Yp(NO z$*v-rk%VmS6GuNztq{`($N)W~5TBl22?}Y{Q7V*Oar85J%4m1g`( z6sM{9@EFx(!Yt$jTo3=y0AFDMlks$7E3g4%YGN|?>ISShcyC9}Oi?@AtfoJZ3~SH~ z`-o2m9!I@_T?kGF0H$l63tEuEc1y1wOl{boL*=NN&_8<{6;UA05|;=x5tegMdy`Ox*aMg#C)+k<^MfLOX*;z^C ztDFTdNwPO1M%L=#_OI} zY4jr{LY7+Ldx$-}4M1Ic|CQ5UtEyu0iizcNCO#?pB~$UK3sp)u;8J)BC^vK|0y}I9 z2hWE=@1hb=+KUQNg%KmbojYe>1gKZ1{5Nxq0CwVqlfa2HkD#`$s^@lw*Q7pCkJrASN38 z;t;0DR~%83nYk^89#_S5B@RznOWtX)``Dn=?N!XPB*W=tf{IYpZW`BOu}zvRRvWii zJ_O|&jy%+kEH#*|kdNLw@jZc2qz|rfqW9uutYKoO4P7ES5a6*^E=t4C@*b1YQ6of6oS+GQP zc%6ta28~Px9#Qla|1Rhmw1Nku2?{+ck(;igrn6W*Y@}3dTYe&n@E6oyRzV(3wx}~F zAWhwM*I)d#SlmR%{~>|AwtfXUzYz`sQtB_lil9d~)liObFv}`{@WZ zvz`;%YKm5BI$LNI?M~bIqj6VQ)R_MaS>#_!KcrTOqxPMQhL- zQeKT;rt6L%&Y0@)lq$8pf5Q+9G+4cBLr|}AYz?WNj}71ti#Vrcs3q=f2of>98zM*Z zFNSCoM@?x*D5^;hmN^XpRRVs zgKbyV_(Fox+_xI0ZEjQWpSVaTV3k*hL|`XKLDIT`<1ET{t=8 zAm!hk)!PefOcQgwp5&g5PvJ6J9v8kAI%bSu&&>k_jjDbpOFEc&b>O{> zLvsEEQh-~e%HnfGTEXCd*xOMC+RbZtO@fTnT4LPE!ZV=y? z@DZ_1kV!y)xE4_AYrkMr-`1_Ims4I+mehg*EYwT>%|zEz4JQ!>$(?d#96Uw-KE-px zCs0J}(G=w(jj;=q`>si*{3fpV)l+wI$tb#yJ9qNC-hE)v}|loq)~MEfozLuE;`*;q(4HCD0oU`}& z8JIyT(bHH29F8%UH8yWsT@iIW)NO5(jvqhnP0V#-9&}vFc~@?MLN}dLBT1-4rWy&wSyFlJmJVbz&XCl#qvU zadA_yR^EZlzPsK?xyvSZYi}+}AcGO?_4!WlFq1~$ zF+tXc|Ko}YfBg7Ev_fYB!R0(XH*jT~8&U*+=0u;04}>*$F%X4B zik_Kx#fA@>kY+`%egv7~QV_$LC~`gL6iI+iu;%SoQ%hO-*S7^%cH_#zl?sI|R_(yZ zw@H;l3e#!`VM(}}Ib0N8QH*LOBeU|Fe>G%}mBqWB(Y?PR#7XufA{bE`XBPalybxLN z>nEMxz*I>+2EXQnaNX3OZvvT;Z9C;mQ<##5qc^`>SaN0acTUac;>B*eZ9{Wou2&cb z1>cWT7H{8gFU~v!vO`|$?=x*Fi>jED57LzWX83nUv*jl9uqECa=yya#UW`fz`#CP| z8_b%ctLMvXa&CsANA4)M$E2LSGI ze%u9u6IX7b1JcJmPMZlZ$vX82tHBzL{t!}%IboaLEzY`&)(TwGnYs2?&`A=*iDFC^ zeuDTVDV_%73=GdQ&;VpQaOdghbYX4gwBoV4={M7zLGnL7UD$Hz^d7DQf8F3(&V2ZM z>16|H_v5D0iDeyQ)!%AO<0nO3hNrqR-k6^3j^Z=XiTx*Cp8W-iEX@~B@4j4gHG;RsXMc%?6yK5F zo^{c7>>Dnbq&$t~zGYjxBZk|$c5pbw@$>0>x!fyz|39+6JdmpGeg8CSkR}aEB^;Te z5Gmr8O2-%tWNJ8vlq8gt>E3Qik(nezw~#U(4V0*aa}247L{z3kgd+3rS^H@Ce1H3o z&&MsDv-jF-z3Y9S_j#UGG3U-Cucv7x&WyBXf22^tj)?-GNZ*lhnU4EDep-{F|^e;+3(pR`KV}LUh`l zYWXO-TE*r$POY+>S8R0DT183e;2{L$fYayz7nxgFtijkm&SEEK~a~jkqD- zEGa9KL5sc5q7UGYW;|;262KahhGpNPJH|Y5Viy`MKXce#pqeTEFoqmb&Bo^7DD z8c%-f_U+|X$JSSLdJTw)wNU?ES%k%08?7a}9AR`PTT?^?%O8KpSr8l?+&$2issq_T zH;kU)O5?xK`&yz9O(fhzexat+o+4`CHJG?#-twYpp2%|kg(%iuW|_{pGCFdt)hRIZ z-)7fw^!gjJfz!*UX03_}m?>9!A-7hl?a@-xmX&iovz)feUtHw9JojsO@#(ymvPzS^ zhAQ3}KA+zEcn*@85Nxxoo{wmyqUGyXuuIcpwVrXrm+6o3fWF98o+;@y)t`Q{*AQuq z(!HmpZ^{gG%Nt#=Co_Dj+IFUNQA2&rP&*i9-7NPb$0GK(Oj9utvNHS%ua@?j~!5}@BTgWf%gkh{`?0}<|Ue1q?)9J{ZQ88NVztsG8R%x4*k>fE^Tre;&Hq2h_j;$cq z@h(HZ!XQU$1-XQTk8@15+fvS~pLNAx+d&Wg4ND^D9*x+f{Q4BrIBIKHgv$w45B#uU zS!D1k&ruq~E45_#;x@GhhiRMhSMj4M76g;q_oq(a+%od6PpS>|Og?}9a(rS5BDpc( zZO1~0nowJ__U?UuN?A!s9I4JCvk%toEh~C^@Y)TBu*)c*R@2k#D0%bdvcll7hn^np z(($OZ^%gmc2RE36GhBh)AvVnDbkuIlRQN|IS^v;%9v%MUY$e{!ho ztSJW%Il0Id2TkA;&|Wv(@DMCoV?tf`OUB1eGxqfKEO7b??CZ)~&2!q~qMoCJ^`XeO zige^3JsT{=dh5(@IM&~uQ5KSfMY1>5eZ0w{o}M0RUauA2)eYi6F@9GZA8W9h=B1pp zfIl`w_Qa#Tl~wZQI(L6Zl+&5}e?}Bnkt^E85s8^sqkXp>+z|IrAWW0Fd}YkJ=BAz( z*-Bv#nFpe_Dp#JmQS>FVDdzYhZ#%2)%R47tS3Nk5cIu1Cb}9Y57ZwvF*CEdfoT)MN zbwIS*M{6Upe=*|i&}^;gv5K18G2>~5Nhq-R`T5ObzC=C~@x-h)@m);wdUc@QtvV;* zNDgR5MeTiqB=FEUk76uO&+Q+%0?G5mXax+K_!!ee@+p6h9o6y?4uNLq(f?W3#d}%g(*&M%r-bp`-)eBm_ zDG7kqG+GoDI91GFN!Xi0gmc8^+ zf0+zh5MD7*7VV2OKvPoR+~I|JIqhr+97DDnLj&wp+t$`*+zeX%MKE@F^BT@=EM@*p zXaw1}m0j}l<3j%&n)L8tq|M*>D7H9aNJdp#X=bkJlsP{C3ga|aE>u7)WT{jn7EFnV z5E2p=-oZw}tj>zf2Z+jKGPSs%b4r3IQ(gf)|9vlj=>DOevd|>J#VE_-@Vee1_zd9y zlR(HKrCdS8$sYK8JG*ZNjehC-KjT9FsWFwm{QbL5;r`01pBO#!oQ?~o4s}Vc+1(P{ z{qCf$zCi(NdFg@jxMy<$uSNXJUn<`Co{lKu+E~{VuM}cqnWDnKX{qUUDbI(iq6Yt? z1>igl>FS|zoc}<6YwhgJ-7s_J%;Ltz8_4Z#?Itzy4Q`YVV{H=+3}CZkJZI0Rd2OtE z^F|Wga&}spgF5OcHUJa9WCt;XnF!OH0+`?p-%o|A&hgP+7d8K!nCAjp(2VE!7lxv1 zeVE!K&ALvS%UNdMWTK19+asgN9*Ghl7&}9KmE~X#mRUul=jX>VwunOFxQ9!F%iaWOa1%;bRZNBMz30zi!RL&k|OJs z7$?J7mzJy#>(Xxl$fI<3T&&r;s4%!8n8}%E#x~-+yWxYl4YefZxtH6MQ%`7&CuPYC zN(Khv+VuYzbiUopu|XH%GqTkg5S!lp<*sX%Qe=e%pX=V`N4PExtNZQMByLMvu%855`27vz1M%-f520mS1 zWHzq zKS)eWd};hi+1Jlc3e(^99l|;sz61mq(xDGFtjC9-k5;e+Gv+K^SU?6d`BAclilU+z zI&VAJD4cwX^MlsmudDZ(h9O0gBBv)CH_Ku9^nWWiAQ7Dd6_3&yY=n~Lm=;KAf{xT? z2D*G_JVVZu4ZtI-7N`u-V}eb`0Daj z)_LL%@{@6Z20r43QDU1Zlr*6C`z3o=N(ve^7Zhws**u=+74#CoZwcb`zKk`v-SzN< z2<<u3A=`^%WZpeR~_ljyu|<RL#* zuSmZ`UP8Zi`?e0!tcz_um>cHVC%2=mx8~mXZ#dJ;XxFZISwYa0l$6IK?M4+I(vSYD z!us_x!2VB7`{0dtp^y=Q5sWKb#Safnh1IK%sbAFUK7H+}q2!Gck6bH~)Ogi0;*1x?Bnb52U5B8v0vgWSo)r$+{ zi_qN(X@@IqFy5b@AAV_?8CIb}Rb^!nlBMqTMcez_xllXMAG;qLUj;FEBO47Lp!?1jTf3ESEZthT?Cv!Tsk z48M8&>W_3V|0!E`>T~f%lY^G1b~@Hc?hQN7uQi`3!WQ{k`o3LK!&xy}e%`FWC3^G7 z2w%rm2EPaPY6q;uJNND#A`!b>iM8)TVNW||ZTbuvEh_`81du|4w1-a4P+Z7ggs#P! zd;dfY*FM7u0*A1aodKPZ;dKY^VW6niJ3W0h3oQz2_iWs?YGFxcbUBRC67gZ0`w+~F z+I#x#`)v?;SsosiT=dw&*#vR7`{zll!u>mU&hJ}z}`IQCiwW{@JEx4PhEF7t?Y! zbX))-mv6Xf&dn1}GpFJn$ra|c>G#Izj4$so@3P@*UPASu|S6w1jdg@DUi+^-7a*Qn+4w z3$puICEx=r4Ctu~wcaPyZy}~PQga3vDVC3JF^N3{e4r0g)ytcK!9hCG z9#e=%;*~j&(h9l4JR!6MS^sqT?8jMHN|Fm<#n_F`ITc7`&F<+Mi_spS^Rxx!jVG%S z!=GgLChf0HVyLc6?H?d#Na7nGyEfz@f3=1}@xTBzHX{cd_KZ|%Y7OKaOPAGgq4Nfn zCV5cc-K4RfV-4eEb@X+gHP)ATH+{LNsGuBPVKH^HogDW}Ic=KN+2VjNe$)g`0GCVn z(z0bUwIzZbQM&l~c@4eqIvG-(GfNuLwj@VN7vhkLi7E#)R95TF7avcH6$Glt0v8+u zJ9$vT!+T$v5Wq&ssd$gT8G{Hk2M&-MAk1Af{F2Byj z8-Y;uI%dN6~~vxocuTyCE6?FX*5HJ z6G=A%riEX7IGI+vrcz_8vPFbWQOyDyYOJ1V=^D)+Jv|m0yz(7t2gJt4)GLOKF*PV? z9NnNu@0_ufB^v$U`!XzFc1EI+?t=T0f15*`5rTAro*_RlB}BAbDO45oH>_~Y4oCwe zog@W|-h7>T>h~0VqjJbcOZ3W>+P9?Mlefw-2m6fx9keV(Wn~FW7REh*c6vwfC)14A z3P0!A-}UK2yD}sT8`=XU;~m#o5Hn)@R6x_+Ja6)Bhmt6Haedd%38{er+Cc|m$L}UMOqxB zpAhPXB)j|lnVBS>N@~RyFGu2fyb3E#ME99-cb&tIkxoTpQc8!hT2m0AscX_%mHl{? z zka9Y5B$?W8sR;>xH3e7>U#R(#)7ny4P|yf=1(}g6z4tNWNS`Xl# zwFxH6JK4Z)Na`0)Wo@ZAtH3on`FE;&WPzM{fY{bck`W*mquZRa*Fyy#G;6jhnUUoh zDa$`OuOsEjqQ&DlcX9ee5zY%)u}ypq!vll_E|m(8rwOtzt@BsfnQ1@#7m99C(GO?u z3@NS^8@;X}yC8l+_lYHl{M5x!`D2aGGLP2dEN&6&zCXR(@i-}^QMKmJ3M|~cGd*o# z>`U;FzGbpi4#E~79nR> zSP{GptE}9(c|Itgzk!I`xw%#mH8nM^8!)Hd3&6#O1u~(=V849MkBy{ah`WCMK;mJ% z@W%xsz9f0AXnc-35p))udRw!SaLIWCS;m%a{qkie8&I?lDA5&Ds+ExjAAR&aEC{)_ z>xZ)oZe(YhQ}ywBmDQ^QkI6lR!#_!{E2e0sAl>4EYNS{hfkT@!)V^03tMT#EOa${* zHTvK*SrTd2?rt&eu6knY+Tf-oS0>?HH8u;qvRhvKO0K%Af4CXdWv^O!nom?=j^5{# z2{g^W9=K#l`OTAix5g`|8Sf6TE#m}C>9!@5BgGhq0HA~NiR3=t(vAuoAmjID!JWBW zHL|6}T?2Raz;?4i>(c(m$Ry5gWPrfG<}Ut<6Xme>>anGFey!)-;&ZlJ$CSC0mufXG4NV*<60E3LUeA$p!<^~)m zkr8KYMnpyJ?B2S}eFx;iYwnVDe-#>}i!9SB8;m|#$ZM3x2k1bR0(%n~{2#X}*=$9b zdf(Nx!ge1f(oC`oriI&CJbx>KVx4m9EGFyiFl%C5E^9HiCeu&^rye+=tOkeC zmN$&U!^1^5&QwBxuS^@aQ=3SN|K7v3AR)$tMTC*;4uB`JPR+StO}n5##nB6SVp6TBouu7ktvR=Wrqj zt#0Vz5F?}ney^*0OiTjM1}PKj)Wnfku1%r4pM5!T_|RmUq%!svf8{P+Y`iXca{`YR z=GL$Hh>cnHVwC9E_`T;#a{M`#qqcT()I`M=)EuPdFpc*3-5w;MrN@^d5)5gfWK)Py zKfQ~Hc@(^?<87s#)RtCJlMuUY68IV1_DGKAObp&|HrY_-4_cS>mR{+3@;@M~jfcmE zhEscB{Yq`nXIG9D@E=#%Bsg;6{U{Gg ze-#0sV%I{p#Zv3ZVHMBUI})IWg-BK}U6FqiA;STUcu@i5W$!Vv34x+Z?NR7!k-kFc z4r#h4TBH=J1^B!PFRV%3p7uDX#=Ir)?izb4`ki#zrc<~0Qc+urD3$vC9INhse$TrX zM>a%+O%U7OKiYSW6>?5@9L;3T(uCy?cklMuxA4Jj+7BOWeaF^%xa|;$&@F=Z3S!5% zbop60%|Ho|sphgliPgP&%-D0o+5evXf{=TpxAZbvJMj`K;QxhG6SN5-C}91B2^034 zR(`K;a^#}K#HZkQb-7F<6mUV1bo-VeFCg$L8?rtI@@(ZcgiB4W_=E)3p+k5%WzkKG zi`k9Cef8na>)h3$3t@xJOkvo(^?U!`J!`cV@2BDV2~BWSdYKj94BWZ4{ z=7gFkxVLGHO+MP@l4EWDWB;LfjAVuS&l*S9t*6)fWz1Hm(R5nVLqmQX*Th##{!Ql%-L4*>C??;4afA3P%HQ6kd0X-Wwh#&%Hl#2CUF@wqg|U0T6{A>WcO@Q_I(3$P z`Qyi4`~m{fL|IBuMXFp8Mlxic>F^h6wtNfuYnMCfJt5C3Hq~+S9Kq_6BLfJ61HB_=nTM^kH+}xc!cp%Gt)%m)wIpjx;@Z6 zx_&)<#fts=O~X1EV*?g@_KzvJ&w4q1P)@h_Vczx#gKiNijGgTfKQrYjk_yB9Ao_HA$;cv@PElJc zD>JF=`ewaAmKG*!U?ovkejZy^Rdsf-XXo_d_s$z#mH;=p zRA|fTc)e3rZgOKfT5JSa4I;j+QWFm1Oa=w^Nwt(~$OH#6-`Sf&b`u2KrB&K1$y-$1RIjpd9<~bka6)54VvQtN3jXA1;4G0-FPs z685_Z#US-@#Bb2sIy#z7cVaRLPn~?GNEjg7nk;n!BcshLRSkuX9H49-cF$MLzJdIj z3uhB?dzHoQ^Mk9aOGqWRs+8ItEV@$I5UceDfk7h|a!i_E1 zKx23Z_b2iL!M-i2RMr5%=xCPiTI*Ulo8}d#yf!MpQ)6*h@Z8l+IRrEzm*Dzy9Y4o1 zKt^%8RJF^#J#|YiG1T%WT&K~Pc?Xc%zWq{RG2wBU1HupDHl4_DZj{?1jZoJuFX43C zIvqu;E}DD)Y&j265o3eg|3AA1et|3GBk|8K!ia zcJE<}4KeZ!iX<8~Lm1<0^B#8>ump`2;6v9?DnIW=?ZAEG;0)6#0{ru`m_iQcyso~! z^cXxj4bc*DFbog`y*|sy#&VP}(;XC%XP_Svk^}&*BX`sT|F-cdde|$LxqbaR2(gct ztPv244BO7vEPYoydS@iQe@0gby;EQ-tNyHQ#I_s1Q|YS?pUBLnu4-%3u&Y({ZG$L7+p;$a;is6HSZcGO(edQ1IqT9K0&&^US|<_G0hlpj*w(TLm%# zb|wKhSrui&RL2zNeZYDzkXbdEk1rH0(4AnbrjWGf@jEn`LXdUQYhZ+eVh!I!V(jx`fxAIlPmj9{f8oHZ`0s7wl-CF)yoq8MxlCVYy zTpr4>h>6lqAaHrsO}r=CY_f{8PkldhY;Ctw|I=&NmQ#BHj*fDFMJjL7_v|t;G2yJO z0{#2rqi#_JH8oih`Pd*{yN>j?^q?qF^cp4IJqt_>fIFqWrJt5T zS~F73M?1(`=&d&IX3!xQkGjKrwAh8ujKj#7sK}Ra%oUQ!6r(QY3>TO{QkP)zhl3kO zwxSv_kpGkd;AOj_R%3P8At{$*gR>>YA$2OMt*Tmq9Rb+&lgzuUhYzqZr`6sRgN6@FDm-i%=ay9>oAg??GgG+RPUr5z2Nw}Q8!4EDTDnvn>uPD`;*kLk1ZPUIZ%Rmo+*6ZM<^37Eo;Frbv~eyXHUN2Jj!rKTyM3 zMYP?~6^@s{G-TB)733uHI~M<*e`HmP&v{v)g?>w5xPmJ_(km9s z0ccHNFKpi_qb*p&MHqnY9-QxGPEi}qmpp%7gfu7wFwJIz2(up;=O!e2jC(jBf=Zmp zHk1kSuB6l9!?!6C(HT0+G>o*y2;A6+!P2ga9-V-~=K^>tFfXGjLUdoL@xx~~8h8!_ z%gV}1_QS8UX?N~GX#6g00*6u@i{*Da%tUr!7rE_5lb_Slk6BuyLVqH;7-wVzOrIyW$>W{TN&(BKeA6hX5bPpTfN1fFGXhSJ z@e-&Rco!jx_4i#AOBl<_H#hHq$ih+{+f?KU#08>_`ewWK=_gbOzJO^Tu=xL@j={A$ zsRY)CU}S4)F&!KX3knNEV_aNZT5jP9Xj%kBb;4z7|9ZMof7)WcgL}>36)L06<1Wca%*e=0A78BrH~}3YYc=mClH`|sXwXKye#8w83ztr z=VgP&9})9pWp7>-dQ)bKY10bd+#)r!2*J29j$8(Zrr&Lw)wrGI1T-UE*=mse^FQqM6OX-&5}?04 zW<(t9J4Oza-~Bk`Yz62^#)>qWeR9#404u&*R9el%xRBuy7K$p(R6!)7uCv}NK8*b+ zc;}?_%A=N-!^$@GpcjR=)QS>A`9$a^sZlYjB{k(f!3;}g+7z?nKNgK%FyJ? zU*&7Q6H7zV6;qC42OWlSzHzez0HEQ)KADJ`FJXntJk6=v?V}?0U8?B?5BL9?7@x&n5O1|%_L_gFAxVn{#o7qlPO+?BFOH<1;CoAV;< z4_Qq9akm1>(7WZotmc8)1J~r{a);g&_W_Vx3`L0zcs_RKz)x7=Q3I_ap*CM6wwjg5ml z$>KecWloXRX5h8;l9p%>+Mdr^ohi80P`JG+6O2T#S}1V39ywsGBBmcF9a$gC-roNC z?Y4cW50OaUWC8@>^@dPk8@i$M*9ay(173Cc>zgNa;AM6rK1$(4+&E%P@X;Y{2Q0EA z&2gU_`WxrciSa{{Rjwtbu1+aiw!ctgWH|^F#@!@o1U=;jN*1z&=NF<3L=q#b*wbRE zhP-w!4j3A(UKVyOV$wTQnbh_EQdh+oR^#;#pa(bi0Lm zuhv>myg-heQs@JHZ)hkxT0v8Q;QM2Q&2KXD@(Xh7Q<=Sb)&cGqNdLmT@f{c|EiJNv zz}9D_y2mPI<@gf33Ojwot_!==mWIPgK(d+&`WSJ^r5NrA_YE;_u*wT!#HH9E7@;Z& zYa{3aUH%s*wg8^eq6CpA^kXXQk^0c7x7`#4UzJVXDkd5cixz*dFD)x8&T$_^)RE~Y zGb|Bwc?vDGbwj&ZL0vtWNrpdUtfgtbfj%a_aFt32P@ zm_3i?Wqu(}w!gUFwtwJrw#od{Kl>w+!bwBZd1D&8SG+u|eVI?uxp14ag@FG!D>rFH zn;EaXyELIgn#SG>+pdq!BR|Sqq(g&n*tV_R#76Jmvri^!aj;!)+n*!do|kTn2m&Y6p<(<*QT;MFc&;Ob$Bf@fV2g(js|HiWe7xt0II0>Xqj z2Yfgf6xOKrjv(6;k9fmHGh8VAi$nkT%lWuhq!*>SdUln?Brue|-QF4c%E%)A;bbpa zDS7$u8H<%ABiy^RV;55Jeqn#qT$+BkqG+a!V+%InsV9EC!si}&DfW(A^dW#RZ!HVne_j_d#` z097$zMwLO6s0M{K=x+#}F$ifwixwG?9@9#TG$lgVCMYDEuzs=Qeiy{6G1NZOq}hl(0^%g9hWuz1wv^Tk$t3G^^V{XpK>xWz{Nj((Y; z2SryNy?T-3z^hT{iB1t)JItOJ?9ohzZNQG^moK+xo%((YePa7JDwllo3C>8B!DI(f z4b(l87d}<`{*^y?{V7nsy@R1{>C#~jd?t=wIC}`TuMgqblAwdHNgvM^kIc0oL zwufpaZCi9nq7g-hkHjmIk5Jd+vX2Pfyvy~7LRiqF3H!3v#k`_s*_{VPD-Duv&B#3$|mS5C+pTj{X=gxEMKKHCxuy#DpT04 z*~F0O!B0p=gPr$SnE3ephutfkkx(vFW%uSJYLNFGB;88^jS{s8!Tg@H=(ww)1fs9# zX9pfth$wmU?;ZGL{zOnH+?=Gyb|i8*W+|2=QgaM1ILrvi=1{}3Q4SAjffj=xEzEFi4$p2$n_4tgQUl<8d}MHVmBfMebC}QM z67f|1C9v>{%uM!OdFa?We(WWUUVr3x3iFptyx?PdMaUGb zqZe59L;LD=3v#OY*~K~4u#$L5+DVOz;iXrC<%FJRm>iKyKwdN>H3MHPcxDH~LYJMG z=%Yu{k}yrou4bk}Vo7nhd^phArZfO2G_4%Zuz{jb83qyHCis>j*I<$k2swW>nw)KG zvj556RSg3c*t5dqGS}m)NC(X~w+xW|<+fB*zLF#X{$FadSDXUG2Bc^7j4ewah2D!! zTo9?6Hl}zVrziRKE5nWoYQzLhwTs(Yo;tGRexm}l0e>vXz>DS6y9@fym#j!O^MnXQ zRuvivkNcG}@xf*nG^JX}vP9rf^&H8@vg{>30!%EwO~CTH&$%l`AoQS-eyS)W7Wi|? zuaJh3tx^C)8YXw(zyU(azxxg2!8yA_IlJed>2cXk?9#W?~|s zBZMdE2=~?7VlS^1761Unu(t97Qeyq zL@Q_r?sc?=z{pBZ{ml8F{R1HknB#Tx$aw z1VJi%CuJjeCEiR<4n@sP${5-Ocr9nOmLV;Zu)n|l+!6fI*E1w?Aq_ANUItO2>haGZ zK?LMvgUiXQ45~m}6%W)lkC~B2O2LD6|K{`J*9L%Mj1h9)JF;h90Ba+>)U#(N(QC4t z5}+;#(-NN(_~>+>b&``aIz$}BGBq*peFxF}_F8&8-q>=FnV9{Fg%fF-7w^!!^;bzh zYDh8|1L;IO9R|k?I5g{i*t=QQE%QO2k*3LYTqs(Wnix25p513!2)i^=1*(ed4GHC~ zzWLdI@85qVUZD`ZMvMt(S+5?eJ@UE8>Gl@UUMEaz!?*2=&kWT{Jg>&_MT+wWz7Gzo zc4B?SZaRH>eIx#f1gxzy8c$Yc}FfXUz`8Qwc3uvXc!z4|UdvF*yBHDT;Ix zEm)8x@Zq&YL+6T9OG%$Kfq#BJTNb7QE>vh+cIwk|A~NBP6ONiFg4!jK32Bs(l~pP# zsfTNTH$e9p3IZGb=#e9#Eis;!!UFepfvz4uZ=SuYhu(}xek`w-BY`*FR5qlN_9Jzj zKkv}_bEG_eG{4OjUKsJ(Vgk~y-fy3!O^uC_e$h&G%R^3gWNihS&KRGMN%NCj5uni? zSp+pLFIp`e%sM}8PRIeuB*q77i?=Kxy@)I@!N6s9szY6*c@FFEQwqM@P!KyP0L0p_ zHh2)blR$rCxp6w-AuZ4N^8yf^{QR_X`kNOMdeHyimM1Z1x8Vd(HbEngq?r)*G+~Ew ziFW{Dn^K^gPABNPp-cr=cPM3+x;D=C^P|=Jf#d2qP#(kFzkeflcZ#`G?&xr5J`B+@ zsWj2gHI^{oJDD;E5+VWLk|LF=W|D*Pu4BIo_ImTk@H64oRDO2ji>U=Y^7ET-!jJpf zFK)5Sa-<*Fel*quG@j1fF&yULsnb_(qvcGCy|E-)Ne06Jl$JL-nm$gy#wYfUyUGT6 zKKW8=#ceyy&N5mTp8i4AYZ+7|46q4+=Y~(baMt$Lr~uMYvTX%uK+AOF-o0I1w4dUY za6%<{JrN;~vz16kz?p!YuUvp+48tCt`u<3$Vs*gHTq~R%hTlpPeEQ+*j2;;J zI6MN3q$xJ2xSxtlo$eIT)A&o>qsTUA+^#8PBj1pHR>LEodiVdXhvzPS8QB`erPUzlet}gm25({87)jWz=C|nSv@u4c|lX9xhJDj7VR# z+>jrK6DZUyNJm>|N3}|Lx_pOsT)4J4$xzTpAl-jS^H05n{xG7;iM`BSr&gF`2=a! zi$MY4^=jA7opitnNZOw=jiC_p@tI18XJAP+vHNvCmSxU9@nsKhhb!<)vBfT>i;hOZ zHM~ZemX@4Vj%?zN&jWS~y^|~-?a^5PBE{&)Fhxe_DqL3e3YkT$Ak|6Pv}r<__q+fi zW1+Zk#HNc7u&KPEeJw?GZEYi18jpOvcQqi%zIsn9P@}Fj0++XR|GqO3E;pnJ@YKP| zJxdL9gEyTZ0_%O6y~1zas?yG z1azai9tH>9?b{dIj!)W;77aPXl^JhabWuJ>l3>t)9lhdIHf)dsVT5r4 zCvEE>WFVx^Z-Z+rNlc|9FoXnb1r44c60vEuyRGC5ly{HC8_+-&BrG6tini84nm^i*Njk>gvQ2(kw)y7mbo7zbSBDLmzwkpP?u=B`p7V zCBmSQlaZNw?#xzq_-_@Ft}*xq)#+eMhh)Jr3L+-Q6K;48bL}`IAyc%lh+vDBU+fWu zbC+z?Ze#SkYS|i#p_RDv?R;T_BQj0;Z6sTK%iIig(0uYTa)8bN$OHkkZa{u;K&&*&%7!EBha?(IH{tg-_5qbX|G(VTw1rk~U1uq!0I-Uo4wu|upb>3SlW3ZMKX7r) zYPU^!MJ>Y6ds=l5)~?*$Jm*)fEV;aW&)C3;*hn0Z>Tf%1yY0qt6>wISRQUsKCcJ)? zRf05A^Y1q(C^OAd+Q2jm<6&8{>g78|A1&nd=tM>lPb4$k=Pr_6Ip}(RSPgrKrWP4R zbl9I)r*Gn+GR^I8@O4?}&|V1yJZgIoLDKN5wT%t1Q++Ouq@ehYh)P~e1t}2ijo=;- z8%K`0q%ZQ5t_ImMF`K+;uQ(igdA$UCp3%i2m6Rd9#vfZY0$;2#WX>*0e)D?hZE+ZUxy*!gjlGxhlNSE8g& z;hHcAN>5$%r++NNS$a2`5GFxW$NLV}8fy|XL!h!OewZJMMAra1AmX7=Vw{JqhJ zM6Ug-sYFu)$SH|j^Q%r~A; zVbcC3$~9lO9v>mdKkYpq!BS8%g9KCPxDO8x{#86Vh&=e-y?a*)`C4)wx;{dE+UwV^ zUvu|K)Q^sbos9tX8I=_k;{F;VnN;#r?S!LZz!FpIMqSq`HDT{|Vl&X`IeL1_2+U8n z!5giE@Nc!0Uq^Hs9rnwt_CXT4vNkousIweufF`!r=UQ>G zHD_b&Ya8%VHrt>QhE;P2G}vY+1d)h2B;eDBz8v)ggC)cbm=J`k%7nHiDR&s~Ug&pA zuq`~7&}d6Ur`LDp&DKeuiJgR8FkbkbN@dc^!?KU>+miA;0q_(pR@}8%xcl>|n53Hy zZ7JhpHKCZ%MsP(&6*X~M1Zl2enIsD4=Anxs3QJfSNQNO_@SUu)f{4v8>3Lukj`X0~ z!_-ok|EbhlTRiAk4p=Fdzc*k)HKGX-E{>xIg(B}`jAWy*vbg~LgCgRk>{ZG#@hhL0 zo)YLwHfHBxm%qB+MW0CPSa3E{?Zn?2N@G*2tFxrHefso%scp0w2+TNmX`4dCj&7S6 z_g$2x5x>WYCUjFLl;6T8cbD$k@4})e@|DYCinT=_gy)4 z9PP!2&u)ym2dl)m&b?Kmyb+@rKFC}QMgzG-4&(c0Zuw6^SdXH%Y&#_BQgryphD1fv zDFTl_0&6Ac3ma%Z)W)wWrkL!Bx3y;W`mGoTHkbE4Z5SH_RjXWo@+v*%OzkHUv?w%G zG?4SALUl2Lk|X1#kkXiH$YCVqz=C8kQIt|)%MD)*8Ud>jmPCu*;%46SCGKDl@P&-m zolOc6ApkEWO<|b?BVw@xlI}A~q1|VlI<<$}@u<&I?})=gLLC)a2BI&74qT#wBw0Nt zIxujyDm*NL)W5yhlzn=t;CbXRu4tDKHJvM=nymERV5c(v+kq1^9$x!2hDTq>|nv+_Dws zca59&+HZag4q_u=CLjg?D7N~p?$cp!%KpMiARS4O7;^5B?Bm4(=Q2EM$JCj!6Opl8 z&<7IPK}z$@ZJ;BfDf`;Dw(a|&gIK6u%X}|gVu%KUm!Wu|`UEiXh^tq3u#q?Uwj|(R zuu1zJ34bm;xjsvssliDDVk9xR2Z+HH`g)37%1OZ~@DzVv?d#(S=xJuo1wDqGXhEsq zP;qjQzaFmJhE;L#R;_?5>(TCAyR(i{D~bT@Dh)|CKcR#5Z~T51#Jl+QX9UKjAGUpgG!%uY0Gt?$R~SkF-E%O zkOyLUDFQ6V>|xBO25Bs?w-w^AIB`6J3Vb_|Q65~P;W6Z$Rn}5`gkYYMRH#x?(#8=f z&o5~Z2=9{cb~f0XT&NHnE3s5#1>x7Adb&3$2(u3uUJdDelPxeb3FWt?Ti0f!x|IBy zS%xXsMDaq4twxNHtY8ow9)jM&1S5rzfU`jxA{_f;Ye_fT%T6&09g4S$s@Wyi34}*) zy{?m?pp+5&6GiY6eHN^O+ay<5?G81%=dvrabzE@g4BzFV{>isO)Ju1MbYA$NjV7r; z(UqZvh79L*#bmPne#d#^-Md%7%@z?WWU@}4hroV~t$4H_;D+wB9{lG~^(-FSd-d3e zfnD~FY{eHU2_XsQkSaqF!+RFsPaKO!WKBe7aaz0JezJ*xc)i%SeD-~{M8Tn{90j-_ zh}%f|XWCo0o-5e$Lv{M}>44QYGEW)N$OarG-wR^XWf`@mWrQr#-Y5lb$bHYsM?My|x`=B(P zXVChSk`l{V`wMP2H2;`*r~Tn7stZd-nsi|YkFRm`bXUFjbv)++lweX9-B9q6Zgatw z$FGP-(HNnuaarCf1?8ypql`0g`!x6fm6Foh%dLe+^0xVY63?ihs+!33$kn(_&AGcH zBOYTTNUR3RV?b-f?TWa7(n1>l5yPhGcCaTD>J+zo_h9F)B$L1l0SrpqEwU;1-GfQlzUb zDOIeOFl6@7V9%{$?3ISk5SMU{;&!6!L&dD;=M{Ir>KXiMD6Odn`{+7FC8cMsd*}bL zJlFGD|Khg`r$Z%)nh#_!=J(5Z;w&ZfcaVRQ#2F0)^y*+M(W&QYpIeDN!mE}5CWH+~ zjTqoCw&$n^DbfL+zq~4pj6OC*Pgj=%3UUxm**SFRj2V=Dp~x*rEgAwPkdeW1e2y$C zXma8J6TFst_V1uZ_o@j5QMFX1J_LjUdf%x>lmiGlWzqXdLajxbh4pXU*(J`OF{8ki z@Q-VZ*`yd`{RmW7>e6}kJ4yZ6)f%b_LItl^X1d#0BK59oTxDLd2aTOds%xa>@Z~_t zC%gPN;Yriukge_ZW`K)Rai04FLB&DU^TWb;?)Q8o|H^Q;CzNehyifF~*r#;yAm)n} zE_`8uxfFZQoiz1zC5{d_4c^73No&*xc-IUJLI+D-|I#p?jahq+82f;<2_FvpN*opS zWu&E<07Op}y|*DYLUrza7IerbgE3VDZSZTFFF$v7hNHhpKMj}tXRXX5bP+4aWID5Z zEp@GZ0xtra$MxFXJ!<;)gX1`U2=B4W&tJvg04b1i6a=1fms(;bj&&Z8s2 zNrUR%Ak!t6m*`{tv%DW^9eb1l!(9GRBqM03AZ_kFW2032X($hF z7$n)8M(HJsp@<@rU005ghNy2{p4d2zF6l8z|KASK}BY zUV%v1^OStY%1Rzb*uOHK@^~c<&Dc#*g^5HDKEu9!uh4QCL4hYRTI0QE>W;CDs|2SB zl!+ZlkNH{gw?^EO7u*1P%`3 zKaJKcRs11-VlS-t@gsAn&1%$$Xt|+k{4evJogm4At8j2S3mrf%?5w2(_}XO0^?$<3 z=tja1Qwzw8?-2Y0so?zUpkUUg-VTIF_Sp*y^tj$qPkzMPS$XuJ}y-fEw+s zgVORC%ZPGn#CCr{@=ycN1c*h*v-i$~15Qprq5K8IuINFVn&IQL0HkSeglqnQiyx15*n`y9PE-FgL*E+5QFN9#cIM;`w^=poh$rc;IA|*U;3x4fKB54n1Wy+m^m5R{PrMn0gXj40gN7OuVb%Xn!|4G5ppphjR z7}yqu-8Rv#TX1Z83PIsDd*h6xP?8%^o@m593%$jmtB~Ar>qkfWeBI!x4zW>K?E<_F zmweD0@bKB$buzE*?Kllr{rCeF04N__Tzk(TtpaF<+BIhk!e1*Fu{&E zdi;z5A@JX(Jw6UKHI^Cfbu`F8!PCh5J7;jZksW`%wuHVm3YE! z#5YM!1o;xfx$DwgUeX%agVfVd2_8L)DaA@EDm?!WHA2Ng=1Qqo!GePiNG9$rU@^#C zYJ;y|y$X7LKu%Vc^W+R)D4Yzrn`Wwq@TT2X(&wqItCNGZD%PpDfbiOYu97;^Rfg>! z8~~8VsSx)YLXLoZBLCrZ&83T$E)63tk(5;hxI8?kAnZS3Phs{oR5fD*a)Nl0Zf!1| z6X+>W3Jr;SaTx}dTwYPXE620Qi(TaxhDJ=BDUW1rB}B0XsVPAV_S}wpntw!{)}Id< zGs@S{B&gM7i&1K4uKyEK1@sC3l(A9M&_I+-!;v=Ny4AWSzEN3^;A5(aRMSg<5VmRo zPJt!CW2^d;wmK22!qSMSAnbDs$Oc-Q36>Ly`4Pj_X>liTk}aAd+u0p8Ky1Z87Pqs}Yb%051i^C@0ZnxmEIz7D zkPN=1CpAytC`U3qO&Ig!&Sk&Yvoh*=n#t74eG_ORX4Z3TF*d9jR=jbex%fu$kL?4s z>BhmwR5d_`z#7|p7&^|88ubmWCe|0ACvKxXo`LdBst0&2{BdQ_fwP`vzN;7=lC$2){YTFG92ML=Vgv$k_bnBNxwd>M#u0MtPU2Y^Sb25 z-ow=9=|}jgprU{Lek$WcHkZ3i6-Utr!MQrD?HHayl^m*Ff_K2Sf42@cGXz^CcEEI} zhwr*SUX-U4x0p5}D#ZUNB072(8#!Bg^PB)1w2deq0B~6vi6P~?hb}#3*v>aI0rBeR zjE2by4`kdZ@Fqg9*n?3qmI21WU1pwRML5ml4~+GxLPkK)O!+bj>EGVHc82Sz17xcg;|;!Tj8eK)kq)IvDtiahcr%?Pvu8Z53*< z)hQ6xzk zaX=fwX&FJFSWrotBWb?^+9nou+7RYoHrg|R4G|xTjEv8VZUElyhS8|wqh-j#da!X2 zY);#5SCCbXj8EqV!SEZ$tI!aUbU*33%X&`Y??D_1{$u4r2kYu7F5M*o9hRSi0*<=u{r?!ph6<+n4fyQNjYM@56uw}%2BqG-Vft;5tZ=dg zWf&K*{!h;3+wTmTi#<42btfBUWT6)rME<#@M?VWrZKSE|Wp_C$Diofw1{D8^%;iP? zI+k|Hff$NySr=W7lK2QYr|KFW#7GUIrq&-FIerL<`@eQ4k(^m@#wM+Maw<)u*$KPW zg+*_J;uUu;%a=uSJHAO(Nkc;(n-k&hgRmn=tgOpCM~t}; z|bGytnZrXQ&baXb{O8bGSQu@fk8f$9Jb zn4_aQ78QjCr*mqMAb z-Ndvt$n$U?cW&v<{SPu(e6eZ3AEbNWXGf${#*6nJBQ@F(g^Nr83C1ja|K@`IRZw9X zM}7naBeZ&qBep#KsNh0@bK&0xN$YNKc!WNTW&+KE>@ZnSXrj?fS@=i_Kpje&IK*n< z{r6Ec8bW|89l@B2sI-@zLd)X(KH1@W`=Hl$Y~;+Qn?XVh4jzl^Pqy*M^YTOGQ^{cc$A>kt`UmLB>8?_ z5FA#dUC|q%5CButM1lZt!hWvx_UO|cfv9;{Zue@N0#;nPO@S!vSE08rU>@E*SrpWF z?)rYy_6dkL-AJKUqIB$}wb8KgHn8WqldN2!ptjB(DRF2&{+U=(D zQ$IW8F&gBqf$`9%@*kvm0(`0Fn8mezB>i!M_b$o>ix`{(*6oy96?t(FZtlm_Vzxr` z_h}Qyo2phKW26Tf*6Y+!p6Y?24P;Um5HfKVq~H+iL2oyc=)1v#+Rg@5K>JtlL~4`n z>muz!G`BM-M>pcU$0dZ&Y_O9oPk&hx-G}Hjr4%|-7}h5+;k(iefOWWCeFWx6*$?Vu z$e2u9Kr%;%`;!yqIvxsIV~9H-*O1z-Vmt}CzGgHB+xDMy-bIr0X;+qu_PvhmuJ23? zU0JL*42Hr*`52c(Z_RL=5BDn<=e4Nw2{_)A2w8L#qX7Rn-V_IHMBefy5ae4p5@- zHZ{!WZkW+#OvW=yP9;(_;`LsWc}%QRhUN;_D^(mM<_374VF|I8CwSVVg~p4iZ&7BF zU{1gbNH{)+c~-9CuV%Lo9^ch@Tji|GReW=l|5h7%m@ZLBQJ0ASZ_;)^J*jy5Z%_eZ z((O_+iP1sz4X$oGLodtOmyMIuCk#=yFK^d?llaJ;5?u;U0oy_p2Dv!`uOQBU$3cCB+OJST>pV-kLb`;dEObhh*)WGWGTr>|!*P{q@h#m6We`@#g`9R_snx1}_qciyZpK zk;jq=XmkH4aaJ--$2l1W)w(KRXKA>=pcbvfi_~nLI;d#5a^=eX3k!mlHo%gvG_xv` zh`p1N%wIL&-@k8(L(>3m9K>gKv#>zef0c9;@kAqq24a85yN9kDkjaMl9#oRH^5~BX zI_~VO4MI3lP>-)bXgM%UyMaz8hd9~ei%v1H`zImh2ZIB)o7(+b6JjjCWlStCw&->{ z?3Z2H9NH{hxpxw6@9Lbz{@gd_6qSh@M%N4XNs>mc+5nKto@no&*1!IyEMt%R_qJGC z?iSMvwc9%>R%xO4ax{z>1VPv);hH%0EC4XcU%UWp5piIsgWFa0zSlG?G@X-)eCjR2 z=0;Fa!`NXCPo8KQr(N|(MTIPe7Ks;wc%80vqN@C(ZF(y7o zI73#ZK?r*Dv>Sz&V`9QVkuc%HD~*LBK<5;yEL!bbc-rHrg)6IjT{HvDZ~$Bgfusqe zj`U0@_E+s7F{U0Y{%sRGJCN)|q!jlZQpxR~QC2;kX84Tseo4!cjeJad=g!v_o_#Pb z&SpQC$u%<*o6+?A;Rt9{3?I5EpZsPpgny~|8kg@nJQfKYHmz9Eps+4Ho3wPj&Y%BL zL^&D^Rh@{rQyD^Bt7OJ$>Q^sF0Qd4$J^EQQ99oFu63R0|sW|&2N7c{0XHjhq&cXA| zr;W{0YW1^aQBs=*b>hq_Vj-C+UXQ66TaIk}C&wEk2|qs!$}}@5+_3VDD4!at<4Igc zT6$Oy7vZn%pqnR1Y{alo+QZ{;7l??7*-ak$eBKksbNGdXkj(7wV+!wLN4oL=fE z3}QEmYneW)w6s~ul^zpdNONME15R2u(*%S*TR_}ESruslq+_`^$6f;~%4Db`K+e{6ku zJk{Oz{cWx%Y38C(X%HDw#z;>>MUo07DUoE%d=2So4k;C)$V`%uWGY1@qKhJ98Lv6H zGDW|2-kQGO-}nCUdG&eHb?dewCAN3VigWrG5WbL2$C z3$b^^6;IQ?lh|272~Fvg5ESlxxeL^cH1lGp?s2ygkGF3+2TW6d`qHe7%*@n@xg@zp z5S9T1@C47cu6-)mw3K{YA&aMN8?eA%M}G=VO^ z@;F#(*eD`%$pK;Y9e4jNf)wCCJ^kE|%k-WsuBSrk;)kjUI_tZ)QujjP@8iHfyjl9g zoAUU0jJHW|F2Ik{m-64GjKSvOL6FxP;XfQpbLa})RtgcHL~}}bnaUQInY39?`?DVm zWc+_Td9iwFk%q15XA^nanhk_E-Xtpt8TH@aumP`@??{w3`*;!8~_$B6?u1`)* zUOz_@TO_*Wt*a%1%rODNhivSonOgC_*4oO^*8sfb#Nz*gJj}> z&kkK21PEYEvZNltkBzd`Au!3{-$t^hdg<@hEc&MqQ<9gVxw0?~Jr-o0YMP}#1 zc&hUFgxP@cicjB2=kK>|fa^Tp00bx2iWb;-TDK=3-`tB-q0#2cC2=f-BKXVO-NB0D zs>F02;97|mp2mwzjOfds{O0$BmM`cUA?R{4tV{Sjq={MJ|6cB1C5+L`AN=t(;CXWv zZYbr2qY8KN2G`)puW7lX2KSN>L~r{Efl=r_SVv72@#Kd-;yQfr+{yFCfJq-8&oXH_ zIpM_TMpMrRxTV~?bQiXVlwcRRAuvTS+(1AwOIQ4edMk2TPdAtGLT}Hy z_f{Xv4TNDB2ue(EL4%=Jp|dDu;)cgg^W5~ZBZvF}uSKB-w-@qCY3>`w5_@NOh}uS_%`kj7!1755&$EiQfyZw{i+ z90q*aLN2)HiO31_5Y3v=A+(P)BOqV+aEj9iad>Y)lA%OTkbCy0{_*IexsCL|?=DA6EWf-Qo{MbMGmW;=fUUIk0%u zy1IdDZEFu*uoshxI+-rK(PS44KJE$HJ2(*4x|2IUJ~!&`B5Yw;T3NmQE3J5Wq#@E2 zKmmXTm>A%{y@^u9-kLz~G9sei z`Z-5kNR?cQ2@9zGgobv35u)%Ym|E1ocf!rWl7?a&Mc-dm;0@3TgrR$KhpukroXxHg zo%;+EWo9SfdD%$)G?}93*F6{Ch`2zCh2#$(_Mfn}e%?v)@0`1E{Gec4G*K_Qm>U+S zeF8%qzG5!@8={BYjG#}oxsmr{;~1VI;{rxsiF+&~LJ)TP3{dC3^ ztbuIMH7}`IClZ7+L*$lTsMYDb?rN@h%2G+V!Ka`Zs3cdJe}444xZ)$PTjhOgGiz^- zoU)3Tlyem3Sg8HL(S~(?B-97Au)a#@ao5ETIF{;7J+{-adsY} zrf?Bc)~fqo2;%!BGyq@u`;t-960k4uEDh)8;>BS?h5%vFX(qV{zixra{d!$p-JK9e zQBH{GEkSLQ7B>rskpYc4=9a=5!UQ8Exnjz1M>*mjJ-t5BO=z_J*opb1I1IE=zRRQ)sLB6C1uHHZk5br+;@mt4iDyxiQ} z%HyM^(oa*#fEUqb#O9uJL?VGsBv@gfCx&~#o#K547;=P1o0do!kXKiKb)0X4c;q&S z`b~ZL^`~UE&M$MC?!7tbF~A5A)tyOLAcpA73&gBk z_Q)o9LqIu4@Z6+*M*um#af|53X7CIKA-(_zLtKULAei$n;oQ?XCMqLmU2+QSy;LJ6 z<{$!Wm^cN5IH!6yiRTv(f#C7m6D~$FeDCouY!;Hhq*;zxgcmYLO7Mh9=Uy`GP(P;I zwHk+-d?FAd521U=&6F*YG0u~`Xy;M*la-a3^pWO*ZTbQ<728B%| zRC0E$-jppZY0iRt3p43x*Q9_loY6`xe!y)Y&y_-E!Aw~*EiL7DNNW5u zc*oOFxX3R;w-wDco_AzVne4AVS(JBzp7z}W?aVd2Z%htuP1nftJH_}lI%F3`}kofp~ z8y$0ckK?Q?GrGzRS#24y58%N6KcfJnUQ9GyLmVoGQckZTS$}2cp|7Mwc#0Vk%S;Tx zdC4wGOLHlIy9624s#U9Qb|n??FZ8|WqeUJx)R@f!$49oQz*iNhlD=_785nj-wTC9F zm8T(8j*N@jPS7fvI|SiGtGeSGdMo;RgsvG2xU;VlTbdpFe?o!8WP<$cH4%6)QmuD| z=)fhK2=5xb8#19uhBkfa!DPIhfU+>=+zm{k4gILf>>rBl8Lj`(%d<}EUxnbQlZEyt z(S;tpZJgu1G+0*hiE*;rH%&%)bbGdDkaYUnM42?rvYHy{cK@E8)%kG&vXX{5jusZi zTxazylwXQ2ILe%^E*Qy^kP_M;keB>iYJnL~d$D?AThr%@!*2Gi#Rvc8y;^dof0!!k z@>lIQ1n5Uz`RPpw9xhLv`Ez7H&r-1uf1en2W4cVNk)OgoCk?Il-Q}xTd$ja6)_d-9 zonqcG%V+nVyq!l>@3*!@pY*WGh&GKB8uPKCUDM!NcOdfc_}2i83x;#>n;mr?qutWc zlOMkAc8td;sx)xb(h!ac%0*(Z@(8;fBRsph!4{eK)TT>E?+{}aED^0h={sFSMn)I& zMzi5S!Bge|feP*iGl zOw?{`X?dkUfQyG1dR!9g<>hrT^*Ka$=qasX0S85aZlO1dygKI#G>p6h;oH2L9p_p_ zPrhJ<%;J7xq6`CuM-{kHEq7py^RFM!I$F{hDaa-L8TUq`D`)cb;;;bLLtLA7vws z9apxlE!D6X6#!9VBt@3`0;VD5xZLA}1owaj5%*=*s&L8I@}bvttR>-4=r zZM@%}ohs9L>~`vczgEd^L5p?+HRa({v-OczR6GPeWUCLhtWl=a@sc%5#=r1Fwj2j& zv?h>9hg_i5Dvd?_*>~Uo<81*()dO?l%dV}heR>?HO1S}|IB)VkfC;zO9|q@FJlPZl z6~;p|uMHww_xv0m8!OS#)n&nTM_?t=Tn6Tqc!yw!CG_DpuU|i+vlgFg6z!$tFl6KB z>g>xkrB_bQ4_&x=7YmJ^SKEo5+JVPr*XLZuZZYd@Z@(od;a6nGmknH z&Sw9xowDb9bLO3(Tem)p9(JU*KW!`!bM9AZ21TNom;H^9W_GO_d^(fHAbU~m?jN?; z0gm<5?Tsh>(P@brNeEXCG1>!GBfWYj=bz=j5fE^^!Z6WEeG~>&>pPHqUyVO<7OUR{ zYclB$KfeVf*;o#aEhyHDlVF|{-)t|RIQ;Zki4G1>{zgbxXd}7@51y>R2=I&^NrTg1 zt4PCe1cubuR~~YjvymS^(kQruvlHCE3>rpfk2MjP$`3W2qfTklxCz*7wBA!_De3_JH^LPnh!my%M>vc%*q zIXOA$mB|AA|EZOlmCQDMrsUaYA=LV-ZdB(q@IjJFUfwzmOSf!vxJa%2)2gL=4jtdU z@?-moZ%aP53m;f7xozhv!$TFZiu3m8CR!DB&c@4jpM4XHgFEdRZ4Y7%wA zbbrt()i2Xm(3IVf^aDH0~)Y+mm3JT^+50b^3f!UDR%X72NR%=m9 z&&9)|FUH^(?LB#HQ8pV^)6Gn(GjERg?K}&gO0v{jWV#rj3-bCH6T#&8) zWRa@|5)d23j~_ptl3kMgU~a>1^aun=9C_(E2>EzLS=mwcKmYvG=szEk7o`4Fb|G7+Hu(kt} zX*xp>8#&#t3%_C@lFIY&oMl}iRV>yb|26`^76DbJ8f4wAT!z1*66UEuhPcV zwqo~sI#&s8zoO?~J>`15%(Af#)yNW~=G-6u&fm6xbwMa*>zf@8Yel+qe<(doG7A^k zCfB}8?zBUBhKYW)Q^%zT%J&BQaG;20_xR>feJiUOo|569wqY{Wo&agHNMdEtWKVTj ztF)Tfh#6n-8OCAUS`-WA2N}@O*myK}!ODxzuF8b_ZpSBlxO(NvZFp?vXiw^fMZIRI zl6%hRQ~P#5(IU6pxO?{mt$6UYRBxE*-^a~E^3ul>$L*hsx$+P%G7lynEA&hph4ZB9 z+kvBa%ZQjKPd>!r^Tge}d2>r|3>a6)ltP#XaWl7qS-azmAG&mwSXo?Gk4)R8sreom zqlLzXZ{+fUlPOW{k2uvi1eliGIWckVlW|>WqM}0ODfzqP+P?@tA4R}w9W0i_2!uc7 z&^qUp3gUPYneJYF{S$0Bw!{D{4_@G-*ko&Hv)mXLzQHZC-w(<=Yi}lk1NEXGs#hju zMx7*0a9M@(6!W>~ZrxBC`g}j4u-#@&>8rVc>&*52887{97w&Od6Ruq4xA~~> zyeEskzyC31{?75kW1E*;7u8RWjQ^qUm%B`k{o*udlx&oJ%+`mCE|%TpjEa%H`7QoO zrpLgKS=ut(CXuqu^iOlPaf=)76`VKPy9ZGPmq#L4L5?Po88r1pSnQk7NHp~JcJJH2 zzvHSx&m+mj^NLzd@uZfvrBFw*0Oif2N4r6NW@ke$xtN=~CU9+^JDaZWhxz~~v;}#B z(HD#)uBflCKj^wCWQ(E_gFcjZc_C>h^EJblk9dejWSCgVy5;X*eG`|IpbOS}x#|3P z=!)kK^)U;MxwyFWT|ye*IVm9HffSD;*k&5XVfN7! zFRpcdHQi5eeGwW9Wfpb&NG-%x@;(XOIxwLP44r5VCZ2rPtf}4$39_Ix90>;p?{Vyr zf!n2Ss=~sT6?IXiws(XUFItqhk{j!{6o#wb@|!mwM!cH1w@%?Q z$1Fb?27douWW%0=;BCC+Wn^UdnfMwuOMy6^nyN9zEfJk)5oyq5%iY}- zykL^?VF`K#2-&6mhRR#EoKb@Dp~$**Z=;W4`8W~%bO4<)Ga=Z#MWWwwx4r9suer}H zl#~;%=~qSiC;sGVE3_nOuk9>d3xKSm_89zkSdckDUxS(9$YhvuZNU z`zC^Qgq5Y`zO7r&DjxClz`zgW1w%`DX`U}G8RkVpwfEVkDnw%Y@)741BhIMUjP8YV zJ6$#WKFk>5-GQ6o+5@1OJ{gZ6-<^HP4qOL_C<xd19r)r-B3iQJ`8PboKP;0S5T> z5n#4r;GzvsM3a=FR2>3g#in$?KzG4}=uHbyQn{2Ja&pq6W718KIYtlsOMoxb{ zhP?;_>(^OsWZ{K@xEP7-fN2;Jp|@^D5ql(keSPVb2-=QchTRxY%KC($0crK-;q6xo zB~2xfJslCfGl}NC z0oe!Hl1@;$4sB<-oc{er}KGeo@bEG{1?6Jt~nd`lf{l3OCS``Ruk~W zyL0SiH=;l|p5tq2ZS5;3CcI|NrL8DeJ(;?P4p~;9r&;tL#03d;;7U0E>lyL;6L>2o z7a-=w9unj3B}t68AM50kD_&kR28NK4WcQaSs9GZ#JMh%z^P?29l0%3z6?Ju%Y~$@J6FoMis<7qMD>kI$b4cJcWO{v6 z5qF%T!%mv+wH}QEVX#4wG7l}4`P9#gyG*s)F_T9bxPsq)T#DM_dXJQMtxnUyW9P_+ zud;VFJoq47Ia_@ZNxpfB3|T+U&{FRH@x$A<7vB4p1oxElxu5H+;k%A8DEXvTrajWa z7SqP4Yn>abG?P~s(@`JC-&JtpC18T$(5JIY1ON!z_5wRWHYCFUiPa`Mi%v;fAYM7n z1_aDLFYdO53f+tGhA^tqEKY9(*3LO{Kuhqp1a{s?!S56O-6)(C*?39pT5_WA2*DUA*BVjc&2CPX9sFR zTbHCaM}t|wD;7``ErdJz<3MT-foR?t!#mU`;A zgMm(?M?G2rcPov4EamJ_Bx@o)U#u~;}8)yc6iKGC;D`)7_ z>6iSF>e!tfGxF8PSLwwE^O2`>bfET@j@s_#@!X}SzT2_pCSrw9Ch~#6ZCcYA z^$F?eA7f{uXju&G(P%IE@|CwDqod7nDy4CO(bSd52qp=7R{~6cFv>uVHHN?(QS$cf zQb(|HG=sxv57jOWl-!E@ZwkW#Q)a-Sn6fc3Clw8L&abdqITC;i_g;T}QEEPL=G0?_ z3-}iLXkys&uL$`Zfxx)uHr|fz3p%Dw$@fYQh~>_k07ci@KOA< zusV>S`_$v17?&u_QDLXYg>_+tXlJ?I6ZYHM_IV1pe#)5y+n-mgI)}X95tgfkZghddH87W0Ty&4GhKi%m^1L_P7%1{_;Vp?5?`;TB?ZdUIP&`T z8;YwH0YSd_y$V2RJb2Bk_*xOL{@Eu}Yjc)KhM1e!maks0l}&R;qDXW?^xz|w<1NIxt@SQbvvbm+@Mh4(g+qcNxBzXS~+g&`c<~Q`c3)->0u!^Ig z?`z*$NX|G?i>O3v6vr4CJTg`5dtQ^B?Cs4U3sy~xja5gw*mngV-t!2q65`zCZ3t&j z%@g?#;a<{Rm6XduP23Ji4<@lZ62@C8H_D>IgI|*mxs^}o@EUc=gUpq-u3sYM+XgGJ z4>0ee;K8ou{$U(tah_~8+rADk?48WKKoSY*^xxNracKE(fJt@#af$GZtM^E?C0yY+ z^*xVw)l%c)lh)R{Iy&Xo1g!P_@mn5%@0X26t{BZ))`Dg>Fe7QJIp)10n8dFFC_XYW zFSfI@bMa-Hk|^;%@zF^1HMvp}D_!dfRK)e*mAS#>%}r;LxzHEm-rOfK^f}cJhi5jI z=p&^3z-7VI;|7x=i8v_)re`N&h%drQ1h+e7N7Sc#_^`F;-VO{Uwree(q)6^NG*77Z z!ZlsiD-aJ*=oWI;Zc)8^ z;o-|kkSxb_Jy0xFx@HL{-xw0oeUPQbYsTSu)xkwM8vId&6Z*c_$jDSitK_hZP|R{H z`Blq$CY7G}O((4AR0%%(VL)aIO+8uYu0?+hmKLlf`_Uki$qa+eg-P$3MpNfU9$bl* zavwvsjYf_bE|yq#rBJG}F+^pf+C4Z*JeqWZ^0?d|AQ`d*dQHp!kG1SSe&4nX-^)t7 zf;XJ-!>!oJLxsq_gc`%)(--8#QJROF#UAbp5owHpj%D2-;dHKFlB)=sM(!k4Lv*BQ zb1z;zhn4*tdOwu6yyR-(T8!#VT@bFVgz+a3D5A|HVP=&v%1!6b=L4sCU^mAD|91VC ztal)^lCNqP66<4BuJmB|Z$5|#VeB}goWml`X9Pufl1_enI>fZWf!;YP+(@c8arY_lkb zli)359#`j1qR}59p8&$dWNj1oI2#}CyD{8%8@}`O72@Khv5Neq4gwe{Xi-6qwg#e; zggk;6xp7f?{VWMdO9x>QPkerPervTZ+a<1|VlTE1=SIRRPG2*^o6y+U*r}ACESf#i z5wBX>aw_SyF!u0&xN5XfE&l|MtJoz9y(HIAwZMaK(C{B)!$~XEwn_EsTnRCf=1jl%tNhEU zYW?euE^nM2m%f@1S?X&w6_*|AJC6&+6cbD#o0PU1tw}PTG5gLGWxoSYENXkbv=ySb`@!I{4=i^C)5) zaQKE3WkSJa_#i^0RhI66D;m}cO3h>2LT_|*W>GgxJc@II-8srG2?F`KQJ|@R__6z_ zV}uNl^8|zRQz}U6X+~xMPxtBQ82>6+fWerZO#3ies`VvQ&7JKsaH_*ls1ocq0u=wbXtm zrU=imxka(1p$!Q*&sPAN63PN7%oAZvv=F*UDh3ppYz2`ju^*nMfp&cSzA+8mFt}PY zbE>!Dkt3hgRgk*SwUOtj9y)NqoLu*w%~+(PT!>0XUp=Z%yP~xER&cOFGlB`t$cV6` z45$(+6{B%PE>c%60i6-(2Hmhk^bEq_3^HS)qE7TRzLH+WtAL^o%LP==icLb|FAiF; zk4$C!=ls9fsQ|hFP~dQRF9pTB8JRZ{yJPH_U3c%R@w=!%G&)fNB4dO73bFw_YBIwX ztQp|)2y((D?EmDAVt?h0-mM{zvd&zhu5Xtq^AB~*!X2Y=A6=8z$3nmgLSgjnQWqah z;uMH6V=fJl?0}7J_OS;ee_h<`ptB(Pp*UmyoxX>OP2QhH*@Q_JZlxD@4t&eBBy5jX zpu`qb{43^DTa;K%LlmKEjj%W&08k=x{siZ-s1!Wmz{Uses-ZXDAOrs}j#aa9XI7d5 z52TX=$9qq@FvK-9Xsp`mO5b@;`8(& zsfYCLH4CWT$;Q0P1be;;6&A6l#KBbEEH17^Y~9MMqmAAF?69E+feUQO`lF+mF&u+rRp#jE3|ih{bMyQD zVu#Gx7%2nbRr4|p#=w7aAaGiFu@T%acmVQ90puYB2GY+Qz4_TyDFsdKPBlN%?@t6q z6S@nfa0Ij%A8;{Dkd_8uhhYd}#D?&hlVFiU60zwMTCvE3=#?p+GL+`B9)+kkjKX=wiOg0!rB^!k|aP zRSM~c;t|^RzvHF{)2kM)vOHeZ-cHN{9$e_8&NdP)1X$L=dUbE}{ht+~s?b9GDs{q# zbWEGg^%^iGg_@@mV_*V-S%Y!Ot&Tyf8txqkQmpleiA_!_sD9qrQz5Iw-^Vv}YlUoz z17;uLRJ1P5rWUJx&68X^^7t36^6nglxW`EntS)qvO-4GH96w$|R3?IbXyyG~(BIiP z^#Yu^NE*NZH16{Zv+N^$mF@#4uD*#xQ)n^=ijAdAw5?)Pgpw(pP7}062Zbh}kc|xHR;e@0 zW(q88n{I;p`##x^nVFdm!(UBYUkcr0HF>j|V9?|2v`zj!?i$2hAoS2F&gRpQm5J@ z#sD6xaQKM$F#l*)iF%`P!RAD>j zpOfm&3d!GzbO#WRYD+p!b_|S7eBanjqFpHcY!eF+5{r;rXJ|h-bNQLML%%X{55!1G zuQ{sX7oOaX5KvXmv6hs;JO81kbCNN;twZ@-M)jj~z; z^Cv<3S+n;i8Y*^;sfOBtJ$qA9!vDtr&}kkL9vFo%-8%jz+z5&@gbud{dH=zK4Gft&`-QzfCg(mr{bM|E z*bL65p<`XgQ6yjkV8#4|1FN*JZq*`K2dEUfu6Wp zh#F{ zzx13`f?~qL+8!Ry6p!Rz!c4PH#4icP!c`jYVw47iON_CGDj&fhh!%Aul$Vv+*I1@~ z44kR%0lKOBiRD3!A1KAkkmFgXBYG^jjOospG~idT=o zoO$qmCr34K+Wd2_Oh&@Lxuu0C)kGgKsqaUShaY2OG;W83?Y-_zmQweO zvhx4b>5<_gzku&6MU%4#zmF#Oi`iOODt0fDaa=RV$Ac2)W{r<@f?B4s7i2(suSu~y z3GKyH8Iwqf^^DJ=Dgn8>__B)kI5OOq?)aHO-NbM1wKWKopI-Xr~vaL`4O@5QrVI!#>zC=1EHV?KP?@A}fnz%JcD zQW1y`$rkf3AfJoq>F6kf%mKN%5nw6p%sQ)JKT=Bwtyr-YDhDJ)+GgAFoYr^z_uqfl zy%D13IRTWF=F&l%V9@5%&wMW9WM}bk5dk2O ztEa36JEJ_A%vjRSG!UyhRkV)OIi9$;6i2&K@_SwU9z*FuY4L~%^>Qqw~XbN=Hq{waxZTj&S z|4oxT`wlRG<&Lv`q9|-k=UWr_3}vw2VVo?eTzw4jFBFd~&|phmC~8R-o`uM@6B*Dc zJ3AkgIzAH{6bqrc0R*ga7O8U_5FimRCZNZQ*i6?~sCm}|%xZKLUF*v`S&`rx$0beNG@Gv?EPa z+fN}Zn`hVc7&u6viy&=!E5NxCD0ic6WG&d05A-8&W4$GuqTuzRF%QeSjQ1?Sd?%Xn zeITa6wbPZm#Ef=H+z=k5QDOEx37WcF?cO+;jD+8hv8q6Q2HG74;+j0KXA}I zJ9lp1&IHs)VYi}1$-?vVrErQXaKK4&BeeNq?)(&%COcfV=~hFpj5*gk9N<3a{f}3W zGm0rXVuv!0u_3`#lO=%QcXYeM>FLcMdN7?7&}VWkPKhQcc*23jX=9j?9&uGk7aP}e z2n$E)nVFU5+sgp(!lLddyC|JhgzdoT`_HaEp>x4!w+jX8c=$B!%f%L(B0?@{^Bb8VjXjSaIeG7R+LF z)zmJc%%N4zR}?Dm9Jmi^#b1=(5Pq?Hbj*VsTnT9vaRzasE=mR~C`(wu6f#2oQULCb zh^zb>RhSCBdkq#5TW}M#1uH5m3+^2Jqg+E{`h=7Q3K?>Mzbzq)A5ppw4_weg%rGm~ z=$*kPp^*a#$l3#^okuhvC*Ie76TSfl5xr@*wY7ou*8g^H`zfRZglCtvDDiS;g(z{ABFPVNZk$DQx?|hCxZ1}@dx2_Eu#dVBk-kCu;BAPL zV)bm;lv+eoF?j-is+hmGVfj0rtZ`ME=a%@k0f*O!Ye>$zu7)9L%Z3)zu>f?4+L0qi zivNAh%-wK?IZshggmGpkR!^3o6?K3mafc#TyoPJ$f;|{P=gWq>Aq{7d%BAGZn?0tc ztute=WiEx1>LmnnO&suv>SYu098Ih|Ns^qUX=9^qO7I>9X!{V!GOcf+d+7(j8b|8~ z!9rz%^kKhf8H2x^wRX`z?Nv#}s<>CYo6?p6B_O0garW%A+EZSwrJ>e@Qt`}t z0?k=ONpbOnB!7w!nv=wc_K5I=Bcw7R7rNYC(S^9EAC{OSLyBkagc2TxhH|UDZM{e( zz5Y)OTx#HgA0#jDWd@&S&{mwStPE3t89U0bI!U4JfFT^!uwqDOb_1bjD%duv6o+rj zQU6OuoKq!nqAW7-y!DVGh63icocc)kEANevgsLvhX4(JLN?vtdg)iU?Xd4KbYU1O^ z7)XQPTWEr*M5>3t5d&Juv;4vDyE;1H#*Kw(V}z6fkbAs>Ai#BykCQwGQ) zo-TaTiXB>+g9b;9!1ODU^o#WtBp2w?G+~0HA0-W6NN|s{A^w++iirUSt-cGIH6oA1 zc(J2wuPaxm&IOWd5ib97D_YLH(|-(tQrM5Ej`VO;@&^x1b$VhVekWczRiFP#592yN zP2NnS&oWR}vN@ zr&@8N4AZO0Ta8&jXBG(Jorze7NUe4KO$I(7E6gt-SH%i0d=UfKZWeGD89ucMo46^| zAXNd@=u;oG@g$-R&MC9t1jKWq4-o4l{>9TH3KJZe?Y*JvD9+v7Z4<~%2&o6n^2o~0 z+qZWweE?Ua@xvLp1BbGlwV|?+Vpydege&fz9XqZ}0Ac~Sh5-XH3CO1@Q-?uRf^z76 zM^e4H9B%wR7a9y=-Wf`B!)!)!>3)ae<+t--i?gr*bf>NVjBsF#GJY~pj= zj^yQe>>dBiqJ@{LNO~A4eOjga%%eIq6FShO(?2{Y`UHBK#t}rp)va>)v=Zw<%N>1aJI>m80P>s+ zo6rCC^}#s^XvoRERHgTY3^2n8B(hUYb%L5(FzwW(zcsLPK)~(GmJ)pg4mNT1fILBh zk=EdmNg;_-g+jsbK)5c}I^K?GtvJGpASw?c*ijA#1y}voL2e<}%d{#|=t-RV`z*f) zrAA)f^9Sy3!6wWJ{P2#a!tb)MV}bfw>?jBn>9^WYUhdnnWlQgA!2WC3tO0|{d#er5 z4luBE$e3I^(TESM)0`ko(J!5mh64`oCx_ZIDT36~Xe`>q?~CifqR=?)*;Vmy-!1s1 z7qA99g@uLlCyQVYV=qEaaMk5U^rJ^Mk5}MZb9_{5P`eq4*2VRn!1F)J<2?{APmn}V zn9Le(gE6_PqG7{qL=XBLL>^T|0Kh1jw`VXc%i#5V!p-e#b0MJg_huAhJr7-um~TDs z0r(+lwm{Yhq@DnX71h<|Y_yOkK&w8Hj!PFMKPI7*NURo%><6Hxu7oliJ-w2ZPfblt zJJ1!U1CbiF;L90JJ~E&pyz37+)xGjo^maua?s@%>LdxPVMf8x_C)9{^Z7lh^+pm{1 z&uZ)FNG%_ITUe-lrHS=*zv+}k1hJwE<@m^@d&s;;isKnE}!(@ISX;!fL$%&AovE`@Yr(p;kc+CGBBteZ6gwt z%Ep~#6yE!?aXJ3_@|$E)iW0+N7nw*c+aX0Hu7XV1+?OG*h{`%b<3!&oZs2? z2~`e9XCJEipOF7ZcEhM5cE}v)@C9E)MMj#nmqj6gQMQHvaSxduRo?zsk|%$5T1}R1 zN$((0R(K}C+&C753Lx?9Z1BwlDdc{mibOP4M@I+UbS4n#2zn5q>W2>>AXMewjx%~A zsSX+~xi&-+KUE?HA}Jv9oQV>h6PF4F4x*9BH(hBJzmn{XfitA%XfrvJkvQBqRS~*- z_AK9?0ZB9A7~CuX9z*Hg^K&=fK?4+U3{`9BiJ%678d;Od^h1E7kI9up*M8xC%F>Pxb>UH;F#O@FpiaoM&m&Pg<5 z1C_FtkZaK<(I1ztJb~LmjUt_4nnCm|c$ym>I`%H+;<}Gj2MOWcvqDBD&FO^(DbA0E zY0&7a@iY7M^kQMoQ*VeY9O`yse62uM!ve)kStMXD$A9D43D_4Tl#k(w6LAwD9tImTiR!H$b>pqjP=UcV zOnM+4S`5+e;6)_XhnL=HX=}s#s}Tc9AdKF=b&Sr06{o)A7@!85epVjt%uErCCCh#B z(q_sL!1yjLizzKlB7hPKb|vLqBNUg_DD|1a2!{Tm`bV_{1q(^MPkc6FuJPl=41o$@ zHI@_}Ma=Ihj8x|22T^c-Wzj0iM~@)c@ZR)?Q{Q^O4&RA-aU9lhdX1hU39${flLqdQ zSu5yV#pU=85(Kh}Ac93UI()L$#ztq01q#0s3z|iv*Ff$F+N$DU=q8svC~dH|`FrVe z5#1njRiX;8CmMI<_u;<}rQrhIPueaeZz+cRip%CBE$j0qni_O(L7EHWAL$W5xe=6J z;WZv3C4B?Ni#4mJ`WRvDw~wdt^Nupuw5v6zhli?NQB#C_RVuMV&RoybWsf#YFSfH5=NjDHuJlzQHY`qfa`HIR)BiPh5qo;_{-qlTgENG;F)v0bC z$p-xFsj`o#T_KIp{w?duh49Zi1KJ}1B1^8f5kBPZ_3JPCfZycId<66)h-f*0o$8YQ zg9ZwIHYREuPZe~A9--b0%~y&?{4`L|{AyrC@6ak-J5{qOybRbKA#|ppD{*mCHdsf}v z;N1*b#M#OkCKS?U6}TKEWm*xWA3_JPPVZ3;3%4|cCry`Yy05k)jPVgb;2V%ox&!;D zT+{i|HmQzhS2yGBF05dj34&t@tQrC>k!N5eIM(3Wxqp`tWnnfFw29UWIZQ6QpOnO8 zq1*2O=S5?c5OGGIH+cK`E)bJ?_Vr-R(Y+?gfP%Lgt>6S_V?s2iCPkzn7w*AU2(34g zq@fennHZp00tT!(R^sVQ>L>i}2>^j|(`J!{(#7bTJ{+BFn6FLQPi2*e+GI-iGTVzoZ zG$0QbEm}ke){q8G0J%vXZAF5mEwf<*VRXNpd+`(kqd&m2BB?&zTiz@N@LP2=Ux``3 z^;7q1^x#cl;bk7q`giY)Dww^w`29lnk=_W?Ie4?d#!XOhau~YU5Nmhp)MGPd_{?S;7z|a_KBzXkuD};jk1ll)Gs+2bQEQS#Cmad^T zuqNhnTr$J(Tn3J@W>Fc{6{xL!oq`BZoI&!ijr6Yo_+SE0FUMV%E9U~_7PGv~kP)eh zLkS2IkLIdLLo-QP#v+lDdi~F(gSz@k)zZnd{L7bC9_pEO==9dSx1?9c-`|vtR%~kJ za={5~IP!981UVvFbZrx~Wta6aZ$Ye|@#85M6Lm|0i2$}B9|0e+c zC49@Z`^f!-prImJJy*qmZA#5cLo)+`JQO04rve;MB{}N#nQ4AUn`M`9rlaWu;}9mI zUE}X)XIA1o&)s6adl#PfytzcBuvGCim`0mL(8JO=e10~={T5?#5*tqw{cHztGjIl% z3@>e;=;3iv3B~s?+C+fj1Mv_c@C^J!khL|)tr7;>vjSv*_XPgw_Zq59Hk`T2NVBMA z_2can9fc)V=Y)iV__i=b#Kb-TKp+MFGr(&Eh9WG+bM$5cL+~pv^h3B2MAPrxi8G5n zJp6)coLPfn7o6Ewonk*XUhUHZKj8rNQHEOm&N&=o98)c#lK%FpysAzUw#o zgz*Vx$Qr0KI(O3-&pW&PXHN|$w5}@mZ?v;HU*YP+x+(iL{i{TyA4;fwy1pZvyTF3T zf@aZj#E@B#HgcvH|Fcj5T3g0A>^5WzM~(3a4dw0daKbV0C> zeQ(=biQC9~9BJ7A+8vgeuEIx!bc4~k4)6d^HU=KsiF3T0ONB9V7^&B_+P`lA1LZli z2(QYziE9%H5BNP&vsIx~DUO9rGxWk1pmcH|V{u@=)WjKKq1uXm{CGnIe852PxT**o zR1t8+d*^LM2iJ1}>`Eg^KJ2JPKfAsv2!lbpERJppkkgbX=uK&<0JDuW;j|y>tOavL zyU@z0)N_Ylc0hTd*%q9*V>UoD0#Ck)=nXrEV6}&oB8ljrM3Y;h;u-u4zhdOtRHID~ zG6yK(F0aFH0KZpG5^E3A3XZaxY6CllXT}7s`pTdH@+%FLLvIkE8jVY&Xx?ImP6!@& zc{b-C%A@{7Q(oYOw%@1M=F(L61#fifwV}slB4)?VW`s3h>eFS1=6o5bR=9a-V zANGl+#>>4ONICIS4nTKBSLX(jk`Z$UEqd^uRmx4}Is=70JzEgJY%4@W;Z(05P`n8> z-!tImE}Qrx(r}B>D;CK9-!IRC$h-_(O zC+y)QB6&wX8F2?P1KQS>0fqzg4nL8|JZ_HZKq5_1HAFxbKX|e%ayylYsuO*I*X;nC z3F)%B0dftZEg{_uPU!{=HdAeQ=~Sc})E!8WwGWAhOC_h;%ps^4^LLk#V7gp4=fpzc z3>_FdLSUPxd7Zs1^d)ZS0DVG*adiAxRbc2F#<(RRj&C9)AahC@4)zy#Fz^jv(~&)| z6T}2`Y79WSh6n(R9-vdOd(fEKOFF28!U^>vwr<@mLK9&Tm5-jZu-FSwoN7XdU9y?% zuV)`zgsCM3=l@*bAPl!a(yED5p(zU40Xp6SxWo+1AS)ll5WxF?#^(!s6WtmH z$OBRd5=_*{J;Jn+M_{7@Lj#~Wb06;ytwc6KH68eEL01$dTQe;V4Evdwauz#Hr@4L^ zL19c@LgIf^iyVjIeP<7$vlYC-2p~f53`iG95K(x0S3wtQ0oBCt@RuPH^nR*|@)x~_ z2!TG1_pJAol{Gg_?T4Y;H)JQ_u2SBFF5y>qv_^A)QK`vcL2^FXJ33O?d(s*K{J@WJ z?$=0Hsr|#~(CrvHN)NQ@}RuQ>&SiiwjEY2$T$Nt@M$>+U`1dK1#3sx;QF760^U6-6iWU?Ls- z+i_weBU!|tue|W9XHF^+jdT(8k9rBZ^l@Sh$m?3(Q_s2ht2v?(iTCd1#Gyghvc44vOTvG905=3Jia2KRqpAF~&jSp|aeMRtTAR4Doqx_6#TxUy4 zO9OEFmXUSJp;&m7i92@e=$b|HgiknO7@2m2QYfH+(suFiEifiYd&Ji)e72_ZNV?+V zJNu21tJ71P$hE@(AEC1c4nh+{vBwJtDNM%ALs;cga^vNp;DekY<0NQq+=pA(On zXC>S5^G2*q{94750Ya!3%vsAFaE4w-Hg)c%GR(buwgO5B@Yy9PJ++lZTff^WZbLbW&)y46iHl z!x}N%@7|!2CR7ypZ1f^xgnB)5E;gU%GV(4IxK1E41UKzQD6oQw_f4wDp)BReeP9$Mc(qvfe#0PI9hKd*aF9UAasjoPGyjmPYh7}7`R3f zS}P57L9S;&D)=75nQbSQDFzgqLaTh1LeJCOe3Wc3u_}YjKmy6tF+U~_t5i*-tU@=? z@RH;D5Ny-`lWR{+azbQ7M#9zcjKG(%2an~Q;Z-iRB3wUOK*5|~#0Ef;H z=EFlievvRLz0805&WqW(lJ8(U=qpg3I97c!JJrNJzO>u?WOk6^XhHVC)uo}#42P7) zK<6QO`N`EOG1?9e_}hLF@zSX9aM3!)gV$H>Dt@F85*FFquvfQIZ2L%$Qzgr_A=9BE zWc2$mCk8I1h#7XFXj4kB6An^R+&PMQvH;*6sW6iNvytH?%B1cHhYG*{aevJY`DmODEmk z?6Pg$K=)UN>Bsc_*C&^6JYjnE3YX>{)vG7fM=^Frmsh^REn2g_)oNH_$_~4jwJR#4 zPR#zgPgeYJ$;}sIhsJBHv+mnn3|hHeq$f09>-3Lr%S^U5nm?~i$au4}!>nEVodv*7k@* zrsYsLF#ZFANzgSKhXKje0&d>S%W}fmr3o*%Y(~Tkq$vQ9USq5YjOA5HKQl}^wOu_( zL=^)#G^n#(H_`6dubxoPFd|*;Ku*7Q^>H)1{rUId*~W?rFkH_-WF^ugG`3M`#x>GS zWRKBP5ot$X{R4*O8)4ryE;KJUrP6|%lGCvkYehC z+hd6SmggRj5vfHyNn^hItjCMyTmDFk@0DxOEUHL5JpB0WWxpEb=OLEd^6XqKrOJap z@}Jkm%gJbV+SLfSHR`*FId1ps8QF2WXT0O9Y>evip|gv92ghB?VXWfA*H*)AI#c!J zGmm6C>%hNQ54@C@|71!%o0II0XFK*-ToY<(NgwdBt~Ik;ZuI?r!azq~f$zYFhM|0) zH$}!QCDjjoP3uPL5f8n@L-xn|dO4Pt-N_iNUoiG*hLc%eW1+2U{hK#CGuoq9`?MKk zI$m(>(I}o)7wY!)+<1HFS_prq>F;l-Ic6H&{xH+DJ@m)0F^)!jsU@`i&)m7$wRSP= zd_D(x#|KdjV{&&m*FB9=Puu)N#m>5C^uN8C)7+PJdo(`1n&ox?+k2!=?4;j8ugbS& zTN~0%lt(RP9rYD{^vB-+wqfmpj?vl88n$NRLCX_Hdw1^JV`x+7K9c#Le0t`V)lTm_ zi!(Jl%=-s2Eme1ASk=lNCci{ZXBEeqe!WGP^+o~?k4EvbXi3Hbn><~EdyrDfyCvX=RWWK(e))@dTOb&^RW2bwas&P z`t^);^WF06STexjcv)|cR#n-#y<+$HHJM%}(i=B|=+Ds`7-}lmd$9AC(5(Qi_h;v> z!hF@H?Jy95B-;VHfC)oQply6Q1D3;IC%gC9%^pYz-Phio#y3!*vl`|EqE8gKrVBEz z7lHcRy(Ujk2N&i8u&&58vj^0EY}!BU*6MPcs}fb%3Sl@iu=NBH+J!JtL$EEvbU2Qrkf!lRwOT_ahwMyPD+`etrgg3s;8XKU~`s65l`eAt5-BtWX)m@)sebYbZf#bTQr)Ub`8d_sozI_XhVA}Q8oZ{Xvu5Cx__JzP+p$!8hiRz}ojhZo z1s*cl?fULk|BtNifJ!?4-nOMpt+cFMm6oZgnVMV7hS^YaZ%T8ooVb8WP0f{Qxuw+H zO75Iwu5yukizCI2iim>1`=Q_O|NhT;U(V^690bJg=icW&_qop{4qIcuBD6H#Kb~f% z9t{LsFsRRYDh8=gq)Z@O@^x9DW7hjkQ6Gap27kFff`uEus(-0(m{HVrH}454Z#tbOAZ|N*tjkbst!x1wVBVS^QH-f!PLccBq!{^XEkCs+LP z@ueU0j%mXtXbr@7tt)efhTv=6gZwl+!`GtAPUT(j>E-C~J~oMkzr_V;zdn!@Pr#zU zG3b7Fms6J$Px4U)v?9_Kz;%Ej&gCAv=*o}WOY~WFj2D7$L_FG>3L?5_1v`NS(?ei( z#ebP8m-(!G%u8bzz^2~ESg_mz;QY6MdoVhi9XP|Ml=r(Oe8%xTX(zrH7O|HhgCVq92T8w3l3&wCc# z0sVO3YL^us&lT_nH2t!e+eiN6aR6qTk0>yTB@5-=-0ev^xXd1zwaVC@CamrDmC4k{ z2QcKc`yoqP z#uzRYK;cabMJCb~#|bGd7D2)cM?U87QZ4&A`B!{v>J3da7&OHUabb<#5fs zs*+L=b84d4%r&X_qgYPf=Vxzv`}FipW_ylFruFQ`2~56!xxTBn)DGNF~=DhO{S*{O{)vGW>B(c0i@4PXE@EA}?rG zX2fOxP_y{z-)gSS&;hTSCv8v3IX16v#e|NNcax`h?Pll3h{tLg? z<0gwtyf?7MD$DB>E970iwSZ|TvB2!2$a&ASCS}QdTOnQ>`eUCVp*`z87_v8}va?la zd-3>v8QG>auGtR2F97ioQ2W!B{1q@p`=XcV>1uzi z?$YbBGBWnes!^Cnq34)0C`p;r(W{M%{=dEmF}(r*=imSZNhI@_w0GHrfiSIB4$f*p z!(`nc0*Ep)5t89UPLpguO8{A`w#S^Zs2?(bKLe^A4@DEMgCGSY0T*B9$$>SUkh>=P zvE+|{JSe`HYSK(On|3ZHUHcy?=_b=)O9}_}?zQ@0$N_Rdgmu+rd?!P~d?DUO z6SM7vcIZl86csr?`fR$)P6d<-wE5cut4bSx}tg+zpf zEg(CMe3W*h_P<`+D*e?J6&l*g?&`a{CCAb<#=d>Ii=i)fcrj?6z?{!2aAn~P>YZwp zaa-jONCDgJxHIuWd;w)z1ed&?E)I!v^LBE|cCR&K(c4`MQ`LMf9F+^-HdNW`!CHoFN-7&0&;tVNm%in(uyQ z?{afgKBM3PISusA`wDujs>l1&(uy-e;`GbyKRAjd8z8qfNdhwa^U1gJr;Q_oMBGfX zKaCqWKVq+S7wlMIDzX6!0^)J+m=mZ-K_AH(bm6{50Gm(%;hA0uTic-S&97an5xc}z zj3lPW0rw6^Ax$J5&qgxC7)Vs5blf@()}ZNwM&FypXKo<*fRQawVK6UIAZG#Kf8n32 zJo6Z3Uob&{F7(9OhNl+LUPd;|&(9|U<=}bRMR4^UcxLj6*~$2S(hq2ln}GzzPvW&i z`*GX`x~g0|@aS_9UZfp zrbL1%z5WNez_{fyQ|9gls0rXSTrP6z{W-Ag7y;PSWu_AX60p)t#<>~`<`a%T0Q5W< zu#2<=v;v^#Uk9fKupMvdPD251z^YjKgC>2;nKm*l2|U2U#4JPqhsS7EVp8egwlE~X z*ib#Re|nMv{^QC_A(Zj$R-R*|%?=I`@zo^}VGN{@5NL;ekD`h}{VXOXE@}2c8Kg82 z6nHip?)O$<>L1x*E%~xjLGF;7+$nFM40tG{MI!{H-L2jt@G#f@@oGPsYZ*v;e9AkA z!1U`?CMp2GakFzDUyc9tez$>eJQsB>iJ%Ev9BKd5m|$N9z(J6qi6otls}Pp#+Bd6OdNLYr811NM{WwWC&l` z&6P7q8`Y(O)n{A3-HQ6l4{=f_+(33wC52YyuFMLk!n6a_sHmbETYAag^$t7zGQt}n zhsk^gQ%cE)?`w<%BrRRh)L%#whVV<;JgPxwB{xZ;o(jDn4a%v=V{k5a8oyOCzLx@H zRM^rGoxX;^xETg|q^oXpr35U`OY_%P&27mNvuv^!x7$7~cp`m58+K=MwltI=XeeVS zfVAN@p7py$;Q+0CW^I+BI7!8SR%31-!d5ZC_pj z7p4f1m;g>Ed=Yqy{$tZ_umI6BvnBn%WBTSmS=bgMAmTGdpuCz)u2L(oUq`IC7Qir~?>CW4diH8mMc1ASeE3 zez%S(%VaukGiNq{ljOlu;di(8h}dP&CI6?=V)mYHN^2k*m~WQ{Jo>yRcCE&H85ja_ z0lw;YR5c*vB>(q9j{lDt{IfdH7o(D){h%~~BJ+6P`A;P#$Y2OhM&vex%2*S=f7-6YOd zJp;o`{7jgXoKnjKP4Oa}`liXi_cdGWO3V2Kyu9yvhE{rAMg(_l{Kt>ke#A{CysMtL z!;p26p{E~s{cRFa`;WJBT^kHOlIK75+D}v@9QWrBr?9EMiJwV=Bsn)z@H5~y&T!*% z3!UHDV!I>8h{x{=Svg?S@29D*_I!N=w30nV_ad$?4w^*(b?1HA4*SKWqxNmltOG+; z1@$VQMf9%P&$X+ITPW5YOO4^LFJuw`;L3;%`}*e1Ohu-x2h0XkS3AQl=b-Ovd_ z6L44wKisQ?j%AEU|H4j<0-W;)Fb!Vxz#R22Va&Hs<1KBVb7P(7U<|LrAZG88jT@~= zpm?DDW8cgN0uS!{M$OnxlOX}cT`1DKH_&HL#&{!?w;Gs}jP=*QA?&7x#n<_L#45FTYcb5HL|I3xqVi~X8u~|L& zbFpG#7IC(=1@=8cHuERwgzU1e93wjAvyjNZ#|$?5vnYT7Y>gElOP!4o{%#!uL!a|J zH@(`p-To{d;GU^u(Cer(q3D3TMXirGxg4AewqZ=jP0z>;KHIu0nwWy@qM<2U>Z^WB z!KOS)LYlRVGog;TZGX%inhef({OlTl`O8gXfJ&UFr73+5(d zij1HBAy@+{tTE_`qDPiVQIYE0-z=x!AAZ-&MxS@zo$$+w3BU!QH3tTYYJ5u`fyh!h zW^3P-OlRATY9@?ZqM@PevIR9YU?m+C9KR!+y)rJQ+4Ef;eN5sxU3J~_jsP6*(ku|Kx_KNF_3;Lbv#P$?D1uC^=z9NsOp|D zU(?S|qAS$_i(PrZ6E&3MOIIDA;9H?tl707%vI-{OxMTqMp+Bh#exj17jL67F2Jc{h zIdD`-@yEn~`JKHM8NP|U4~)-#R=fCl zYpaB$^phvb*P!Ad(;Lh=MHISn;SAZg0YG{fn)jWT`DMjFrkTlReY<5U4gI30U zB-mQu4;DYgr|@9>+5ykx4BF-WOhv3EkPf<5&wZNo&l=)Ae{4CbIY89~`oV=hs}a?D zK_n=xMbVgB(#H|kaLmzl&Csbp+ZFaM`$S3<@d`M<@K+UmdC$b^-Hb=Me#@3o}!t}{Qx0xCeVbB623|d z2-FvDKbm+KEZClwoHrtOMl?Xz1BE$qs7Vo)YXUgzfmM=-801dKu}xX)Fox>ApQmI| zKq;(%9yO~uvvNcPuAU4EZYE;SZgrT*m@nhEP#(xHzJ=Ovar>+G1HH-e$4|^|JmqAo zu6$T`$p1}^Z~ha|4_VsY0qg&fB zm6nQ_mI@)-xWFl{r}<=F;86;|=X0O2pvftuJ4IT=fcUY!O3FvO5me~k7qiv7(o*DB z;LwI~up~orx$}I?=xAW+b%&X|RSzF}RM>WT+1SORU|Ya|##dmXzr6@$IvD zk>%JcRMp;|?Kn~@a>vq5Y?9ayd4CNj);r)mA}T5Y=1R|2R8syDL%;-n{_i+xnE;>S z_KqA;;WpX)ByjbBTSinDOaXj-CA9H3PIwE$AdflPFNEz5$BUe&tl}(-0Z6XU)lC%@ z*_9g@sCe|DfjS`zXnNQBHM)}Geo7+;`x1^p%LPD!I&Mqf3xqSMRSQHEw4Z?BjmAVS zf*&G}BxF@M+1cZ}u@-W;XS~Y3PodONA3o(_E}~SGk^ySBb3d&MH=Y;HDGf^C`g|09 zYv*(6-OHa4ls;?9GzSiJ9zoiAoE-lJjf7Vt7-S<$*zMgm0Z2BKS~2AB)Is?qw3uX< zzcoQv8vv+GD&}|l!#;t%Mo~C*OpMJt7FtJ7hx{2DLf0zgLiU5y&3q=#gItOeqTFI7u$MbN=WeOfy z{0F+%QVR!tb|Xj?2wXfQ5*qz|E#z`Vh2<=9yku6|w955UG=Lfe`dYLqMnWs+4r3J~ zJ_vqn^f@$9$(#m9pXg6<8(btKkqObf;TG~>cJB$4=9om^iE)cs-oF&EooaOB++vXc z!ng4iKRF2-za$J%xT;xteQ_7>v{ivFI=4=|B?ntH!;M~9TyomO>~*xU#!@IC-7Gjk z10JF4eYB=?kGx#yZu}QnG)n&y<_cSFX3-EtHahXqO1YC6W^T>YuCw|1Q#%oYe zv&JI|ZG)^C1$JYS97{XfrXl-d6JxN|c{)4{%)rndb>-1m2s>c3GyRbPMSJkNhDRTo z7Ik_uG)>(Z%0;`KKlc%+a~S#0uGj%g1PN++cIa3|P{aFlMjWho-tz$m-0sP^gbd~r zXn%TrZK$uZtaJTzlSqNg%Tv=8{w}<&eTSyZzI^f z$l@FIuOVa0K0Ps5>+k#yX)D9Q%|fAyf||*2>(>pNvoT_SIQt8b0S`i|7@Ymc0x|tlh&f`8fn2(m|149n2kFLSb8`-w&cuUm=uC*&xNU@x_x|TTyN+e~8$jYg{*>W4!DR z6E#Vco|r)=5T$|?4MneCU!i%r6zM@|4iOETACd2NVHT8iatnAzN=%6Ld6Q#svK~)r zL)K|4`SkO&>Gb%m)i`}>pp(wj;L|@it)WXw?%;O2$NaZP^p+OGY3VTeEHgLpUKD+C z6$}lWAL=646IKVv7$oU!>&y)Az#!rt04)#(@c?BEqf+=N!3v40+nraZQ8Bmzn~v;r zG;mt;7)0du!g%#?%Og7g-XZi^3vdZf$`ZF=fdc5+NqR#LeGXbokM`IuOrk|V&3o}r z+&AcHW*@rKNefE2kxXvO+*llru^a2sD4KXh7-j!~#A-_EYM|0#e0yj}hu4~w? zSf$bVkjzE}vxo>jvO%4g_xD*V97==HFK09QQ2YcHjnk|Cc)Sn6^$!d~jRr-U`!HHN zbA!76^^pao2{K;5j7oUF*Bn{diWUAO8QLUaA}f?X8QBE~KK%ba1S*mYa2|o^0tj_d z3dSUxLBkk{II*;m;At>L|E*HQo;847(tryNbPL$(lpfoflT5+8-k%}z7CuQiyO4q# z7tsVxU}yPM#|zJhojZy8HSS7;?MGpUu%7@1kq0U*;IH3P8)hYR`YEu7Vh*i&md2if zKc4n*FOvc#@_A6d!?)`vhduW|ZVLZgQ!|I147d)Int`fv;Y`;=EXEeYn%!tODpoSi<{l5Av&C9 z3QD}!r*8kVjbz%6PQk4}GQ1R~icXA;{Q@M(8 zMi;;lR5FLLfa!O=^iv^`|A=hV2mw2S)$_lMjon^k&G#YnOS_PJKkSQQZ~ieU6tG~M z1(!}nn*d`6WEtis4nio*z{IZ_w|ezS`=20=?!Kx9@3^xPsa(=f6)C)>~IZW;M&}jo#bro2Lt(`k4Pmr)8cYOLpmD@@|7DQD`4>f0V4(_D~64% zv!MzGk1^b!ZnUT_j{Ij@>63d2f}?b|k5Xf9^hrw>`sAc4nbe5)QXwdN>eeDD4R!E3 ztrzpP-dKil@kfpXkbvH2E!9W)&!S=$_QJ*B2xYi>{wJKA5sWeDVRPcyYU2MZ67cY@ z*xlNM^7WUUBvD2grbD*K#!zb;{ zAfRV>ki=}8r7?7%>9o#WH`;eyKxr*mprk*|BA>?U#BOC(#GGuFtbwsXwiX=$$`rqb;TgM;)*|?UhCAOc)ommKlm=`D!QqM@EcgL%7 zI+~iUoz@u!hL*ho^l3|%F~b72i`xpidTBL;h7n2x#<<{+;7-}Z=xC-J+$jNg%Dg}+ zxMJXM>Dz9Eez0R8 zwd>S7U=UuK85zO+CB>;Y^oE1(TM#_vC=N^c87X`7UQ1-rKxqCr>`-nJ{4psCkWys;Ez^VAFtW$a~l0g`w z)URuO00tfumn;}%IxYXagY&od;9Rc&e+b$c-;~7l zwSUxZArX~f-E}&?Ai*B+H{bFxt4crLPXw2b8v$Ow+XOK$s`ezZl6IU%>T7QUGCK>* z-a4bW!$X^ZrS+Hla^vgY!C&fS7u?&2bqmNjNQ2;4&3G!{dOD24+bf*NEzK54gDQ(? zLg-OL<%|*0id)+ZPf^V;YIj2`(+6=Xr3M25RJtEE_y58kdU*8^yz^P$mU`zaNv|mJ zC$G~vs`!4jvDi}&1i>;tt=8Ql|mf8IE`;x2d>>d5~%W^xRlGL|-O0QuZ@Wk0B=s#lw4lN&7q zmG$|=Cn!0L?RVRZjEAq+8jx#}`W8JH#IC_Ab@%u3ni~$Q?YN903<1vS%$5apXUC(+ zCUhoAId;9WI{tBOCj8jGIrfDBsrw#LNfR4!`>2WamscTYJNjb(fJ3#ccvuFo4~y%CJaK z-GjULWH9>ICQbzTr@uaW@F!(60ovzMMSG0KSQb3{#yCcXH@pW|Lm_=j8%;~5Zh~%s zDMf&SW+;A1PjA5B#UB9%q;Bo|?0NXInG)Mxz+a8B@c@D?9Iq7Mm4`~XGzqWLteT{h zgW;7$Bdo(>6_WY{K79<`%qxcY3~(BUO8Gr)2%Ls$u^03k=Q((3p@r7Fa7S0c|0 zMfxEUk zoyJI+0(?tk{M-RqSr^lqtsKf=l@Hjx#F%X+;+05an0kxlA%;31-vK2UiG%Ud7&gAq zDa(jvYzT;zI}YeMI$oeVW$*!(7C{@Ta#$TjIpMqCo(h1!!xNea7^DgFM9^1p(c1x* zQwP52+y!OPkS8{+X5MFF;x>Q@Sg=?LnYXk=$AW=x%9b6{M=gSz@!|oVln(T0hQ)!p zY#jbCOor%iXCYPWmqb}HU=bbY8%PBjsTn$Nm}Ejn-Sc*Q1Vcd=Saf(N;=n}Me_iD6 z-`~BBfygU07T{E*3 zU`uHA|8?v9KpQ?cyEXNT57QTxaJjXDxzqwIV36_$ZUU5XyehY+wltu-Dptm;^^Ac1 z^d=al$pHeZ$em*w{cv8mc(SMYmn9Fc%bx|7tRc#cUXk(P3Rw=PECW^T^OHDK;x z41E?dIV6oWPg_P7Rd+X)kQX^ASjO&Ki7le+39rztZO%4Q3}e;Cd%JA=;mF$&51VY# zAi?GQFojoOXCak1Qwr8Y$I0ckaom?b*As^ed?ZO*e^p1T)A9lUwW~I6sMaFS zZ3szQ^Qos)5iDTHf4y@UdExKcTc_f@i<*8T6E~eo_HiuRHz-r3=aUZi_RvcP6(&1O^L` z)bu4$36b$OOp?HWZ{0oZy-73w4Q;Z>>4xn}0ecnP7YuFXEMMSSygIG31wp8{g|Ap+ zmjFqy{%ZUDY7E^z#<0*=omtYoR>NUW9lci$6k_5Q6g}Duw*lN$+MIdHDpzutqzNfy z7HIvvJH#P}W74;?Q9nVi=azF~5s=b}LU6!_Mv>GmX6HTi96l0cyzQI88bB&$?F?Afo5+IcPfv2OqEPO5Th0_7}%)&w>>{!?QD9!8_3MPZZLG>Tzl zfOnZcd2a=jgN{XJa(Wn&+1${$T^%ExdiwBP=1UtGRJ8x6G6$_LW`DrNMHLKgJY8HA zIWTGj1x5yi{QmY4?x!02^EkJ{wL5!G$UQbhG(6(tyKP`p<%mruf?{u!`)q3y8N-MM zAI{QD-?6iQm-|o>2AZ1DK)X(1jMOf#Un@p)OyLHc$1RS$Ke%7$do8!PuD!^hy4e!l z5a|v3y5HcTyGZftsk4VqK2RT5;laIFrCI-8>0WX!JvB=7| zR=W1YZ~QrLQCkDH0yG}of8vDqYn@+J6yo4ehm_MG`*@wnz7peCi+xQ}&c-at%0kyf z^V_yptALBw)gFv_@z2dUs+8|H$Q~{2adBC8_L0o{F3%2?`TIV&fkX)TleaeNkCumDQ4Gw}t#kn{OU|y>DeDa?4HY z^_I_#5z?|vqO#vHEmQ-?Y0YH;s{YcU{_0xK|~3YUvYZmG;q%3-;P| z>p#`HJ4|);^4j_R{xw`Fx4vGK*7H=!&3A0>&ocvt z$d~Wl_djQWOOig6ThPaPEt-$Vxva)xAj^vFgW-HxP*a*pHJ^6iK91WxmWSVMzX^E7 zDWqlf%jRhOoW9`hU4GF`nKYgYPIEiUA@a(Vu5U+P&70XBOB&jLN#IL~Dlg=e+!ra; z?>^Te^7JO7ZXP6U-K>4=s(b!|cLQ327U*IsoY$=|5pU9Nqi>nCIAH6uI`tl7=qoZMwxz_fvo+vRYW%v- zf{l_S;=<#2=G*i~UoITG{%S9l`&83`=>>_~E+z+D(|_GrtLPoG85kyQD9C>wxn%1h zpF>g#7!i~iWZQR&|K-a#wTpiL>f7(I&hP~;E|nKuuZHJ0!Re0i2!r}DwXu=wU= ztj+4E6R^z;Dx@a9KWX#|55Z(9uh^6Iwxq$vO@XVDG*ACmZQ zU1zEFnQ$qA4`hb2y=6b47Q>(Z5|*I$Mdy-S6{|+{i`kV`&l=Mcz}Z;jAgki@-OWkc zp@bxnmMl#(W0!F7DxD~S@UX)$8Cj9oMB{hjX1d_TtQLFWzAGgSsgRtYAi-O1K58B} zW0LILB!5JnpSWTFUVwR!Bon3=dU#iGs3R2@a*JJm#*3J_&orxV5!`BNz&3ljxVCcU zS1hkBe+=7k0sb)eS@o0WavkKjxrb(ZzgqFCBnKs5AoA@`A;ftpb+ypK&E@V$Yf9&n4^~b!ULuV|b*5;4(IoUYnK0?m55m zqR4aj8w+^dH{fFnYuVr5FLOz$_CkVW!qng2fpLu+EbFa2IqO)+VrO|yj*4yPVGY9M zRSZ3G>_@r7^g7A=uBN8S_G$+( z>G1oGs6QTYcJ=i;GsY*wvMO#VEM}AFloa4$XD1aH$QvnPy)vG!pMN7@n|a{%I?Vywj zfreE$$@0O+C$5A=o%z{a%zOEij$YFp)8uXbN^w;?r$`>|$lx))YcN9h4?9$pOBwlc zQ4Z>5hkhwi{qUvWa!Y&rYrqpSX>rB5gD;YB*Rf*s^OM_}sz)vbpX@b!7ZD}A3DzOJ zefV`h%B>dX&%9gRRP0JfS~GTXlK;}BOMLn~%qM!^t@%}V8d&ca`o+e^u5bOF%<27< zr7Pkz+R_q|u5jw#i>V_Js$kgMd6NJfV^@vNGw(dv>j-Sof3>$;bwz}SFDABoU4HoW zG}wYI6;^;!^cD?-yQFZuJqaHA7js!cB3gEJVIk~x5UaP2%i%>|&?f#FE4UIksm#v- zG(>m6_lmiO;5{C$<^-)^<~GnmzkU1modct#uPXUG|Ke^;J}|@Q)N>Lcx17*Yx`kaW zj!~(pj>iS)a+|JrG#g7mz@@8N$6Nk<*|)yo{_1hInwHjy?sV}VzvFfv^IQD5foTxW zN|S6ZffnB`(rs$&VuxS#dF>`Amuuk2m2EgPI(_|@EB=s#R@}@`8~aQwlNFT9a+ZFWi zeLv#uO2=uMkL0KnAH1xZai3N2`XHN6^kx0C{WFx<>+zG_o?-Xs(n9hM27H%MAW5nI7;}u6MZAybT-t zRv#b3A4}k;Bp$3IxB*l zTt`6WxJ>l19C&q&V~dw_Keqr|KUhiN({OSTe>4#;NhdGAd|3gO)y)%=<;q|ccZ>@; z_Fq<+T6oS8o5n&ii28+j`Zi^w`hADYZgLstqjfTauPF%fm=%SUuI-N3^D`!oKfRV* z{yRENStyM6)Ipr<+BewQ%o5A!(n7}<79#h1R_i*7K7La=VTMy}`CCn9_iWsB6K+)b zvE>V4nc`mv{-dh8zBrr^hiDkM{JWPi^>q_bcN~k;}=*&1J_bbA}p0`J4yrEGm{xB6e;o-FmC@ ztF1rJ;zpn^OISZM%PlcS3{8w<_-Et;b!h72Kbin#LUAyd||N zsOA}E@$KT#r%(BrkrPCB=ZWNZqoxXAXt$Ww6WP}H*Iqz*SYV>IwrOI5{bthXq$Go3 z41{y2*rE8)lP3=gjonXPx+HN9qT2sVrJ;d+VRYcc(U7AfBcprM`}S#llXLIpOseKr zPzVR_>>>y`AsCEcYHD+CVf-bj5-ufJwv@tTT%6bq>(*Q5b>GjiUoE~}*_|`l`H2dw z023YXwsuC+ezb{+E9>hw(#Np(>{3rwm_{8s=2OxdzA)HzqAxonGXMsYR|waEqSc;0 z>PxleB{{KyXWJ9UEnF-rMygI0Bc#E z^<{HCM)60tL}Z%G_4XRFuuKLLxxjEL=fU&m7MeTV#6Et!sl55?Qm!jo=#h}Xy{*2x z*P-3)7yQ<$Uq5(ot#xHY>E-Nwg%5FJ@f! zSAPMY-FSzC^jhuNo(8RQCp}8RGaIAp*pwaU{c1=9c4|1Ibkk zG4%1b_4ON5K`C+_8z6~aI$fb`-17m3Q#r=sJNL2Cm%2J#NSI2pW1Z^#dbu;bKJJLm z-gM~Q&J^zFGViVD+Esnms;yaGfrPDNZq5dUYD9P&*KS`R`j3A<85~+G)rX|Xo?v;o zV$9weJ<;3=@p0bw)AZ!EO7CUyTz$vpR?0Kt{CU=(SD3xcXM-h@A8!GZJBjJp7>yKj ze4n;}4PPkuo6p8TvM(H;q_5Qqehb`1T)iPHn0Yit)&CsNtwfGwm+kE_yU)6L8{?h_ z)+Yl5pXG30+#PkgbHjbc*1VzcNa{qn$RW2nDK3=&e|J-)9nc!$!NaAz9k~n5UrzqHjRxnu6P(2 zMEP#@G_lCwg9VVOFZyO_cobUwt8S)~k=)9#)UkShzcM<#Td8T|=drep0iz=tjD=&s&CKQIb|q$cvdKP#>Q&2pUO|zAMZ(GBS{hHu}-g!4FPoqKpU^hw>ku z^z?1kWvB3~SCn2UifAbr7pOIBsKxId?CZANK3AQ^ZI$!3Y2OWut> ztiBTeQfuEJJjc6bT0r8Cp%@#>%I?a3esh&0>LnYrdy~08(zUVDeG-%GVfA9;Gt`$K zy{pevoKZ~UrMx`$iuD!1=wA%;rWb5O@1dgaBp_DqDsb|Lw}eAJd?OS@u&{hJ4=Hj( z{nX>JyKt~buJ$}C;=AnVXGJC9Q?{6!TqSyuNLz5$Baht+qt1!uC=wX+r}JIe-m@q2 zz5ZrAc7^;@RDYP#?U3tA80$;tI~Wk1#%C(Fp~lLof1HJ8=Pl>IvV3~c`?HRW=%tgU zlcC-?HkOB8if_Dp^6bhF+>PK1UaxwYUJ|m>d>W+JFrod|ePLV~Z6#wC)<(@!o0DG7 za3WXy(KG#y&+wu6Ki)blspC;>$ju#PkY6IF$)vl+5J()^?BgNX;q`~g9>tE`P6@D_ zOh%;Zf})NmBOA%j`YJfsN_l(u`Mm1=>u0C>v^A^;@{JEaSrOot;F`&d>Go_O^yPpH zwO>zTW3_7ZZ<(~MS4(SD-;>HBKHj?^&LDhurq69HJl_idxcF39L`P%By#O8KT!l51 zzbL@AcYh2FY?bV;H-+ka@w5!w>M;(a@16_V9t|Kw{ir9eHTGmG?5vbUq(TS*gtR*z zr9Ni&?p|A`Q{gvYYeI(Ac$7dG$P}RbRnjpXef5_0(5LGop~1l|tlkcJ36}5LVg(gC zy|C%&JES2IR`O2Qg$sX&>lDu9XwC7^0!&O)K1t%R2c?L+%7N5X-$K(fPU|B~K$EJV zlth>>K%ZtJ{>MpH3cI(pv}6%f@gu6L>|m(i7ASe1(P(oCY_rp$95>0a06*7l^&SDw zVskho5O4_l#0CXJ-N*=%!Tw-%)dmJ) zzhU8m%Qi30kUWkvL07baP_}ij+bMNZ*eJ7MeRdJLSLe&QVV@ z-)t=ABs+nPaaT%9^G&zRL(4hBgD0}%f4{i;@H9xBSrtEj{(K4cC)z4PAOarMyq9A) z@H{Zl5cPduKhPp|FkQEq$=2$aW5osFo^+J~tmIAr|H9bR#a4D}2MEt2501hjxtu&oOz(Wrc|QGfqSKd;<%W}?95|T1phD-d$OhFW zgywZdqKX*B)3H2m+J>lRt#}v}HvQUoeeIh3!kl*cjy=@bcUS(VgkvlBwFkv&Y$6Oq8W24& zxWrw(^pzO8bgd^nwF^XUcZ8YfibeBxa+B18YH%9VcL%N#>mka4R1M&ISbcS9156KJ z1Q7nW_th1xwvX`9*>>sXLu91lS%klJ;$!0hZg(g2%(~+fRcFBMJVB#=fViygd|9Bf zm^h{j;&_kCH^`cwolt81u(fd{{2PZ)9gLM5Ho2cge|~7Dw^-(QJ-)jJUHgr8wjYpW zZ&FKki0-Uo?lbv_Vd~tWR66x4iyB~VRtf2??WmmL6fEA>OT977UAA$d^*$+dOzlA<9g5e01apYz2RI%7IzKD5g7%`doEv(k4-9uq--Sf z4xHzXc>hi8!~xqR)wfw86~jBDN3JEywC#&wi}d@((W?4~?Q;0WvYO^wF5$r|ud&ZN z_vuA5d`_@6R5$;%W?!2N`3lv3m6@^;XbosEnw?kER6xc_UKjtzX+Qgq?$LUs;%yx~ z=J`PmqG&B2m8`GB)NI53PJP_1==9q9uA|IYJa>J$gC9LtPrg6%C6wK&>FtX;@}xqs zz!?k!k@g#ySlxNMV?9yh0MK!i!P;UDb#}hXqcgFD4+Ony9~}XRX3^B_n3}UeT<0lH z_q)ojpfI}J2Pcz$XhWU^X9`mN7RTH;+QzGu+&8RB3|2W{up^%&qFN7dY^mGlZkYcR zKq_AVwZT*)8^>org^GT>AZEmW*L3t~Q00_{KvXOEci+(POpVuWQW9eb3VYs}c(R+c zG&aO0*&u(o%+_000nlMCoZz*i9pr$i_Ujl`#h9we3$EpL;>0|;GE&1$JFSEBo#8Aj z!6CHZ73VH@pgo0e%%`+M5{hTrJl!efw-CS#D0FtfC^XTzOpHsESN?stUe3`-dfNX0 z@k?1MI<~6+0pg=01?cn_h&}G`o#9$o@$c6igjzI^HX!!1+z1*o+w6TL4Dcfd%gKmC z!&Pp@J)vyAf);e(!imZ<=FMpV6L(0>41fY-mtoV{cDO~^| z{~<}@df^>&^Wxv63OxDx=?(X(AIish&i=|%HUg@F)H7TZ8kuvwTS@RSt0)VTwlglX zz4?}-%EiJ9JeTt3gX5C{Zdr{iD_~O|Mqqn2<$#jXK50S-cQlX1Q192C#|k0pAz+z= z>jq8I{95xs!Pm~M_K)yF+F`?|dHsM$YwgJ-e@?MfGcA+D@kT#yD&H5fJcN^}Zpg~I zxwjdy?a9u+5(5zY&!(oy&xI)|2YKYWpTSW)a@kXRyXxKE$i0n*oh`klXI3ohTjgyO zQlIf_p5-B7Pg9s(hHhR@6`BLs6xnzwQ7DQtDCF9G{dhKM!fcG>=kjv!gcWu5Be#va zf~w7WdHV~+R4g3btGS`d(Qo!Yc{U2(s57FlK;IEmT;Grk^R1;hmvtCm%_T9sq4y&E zZx8|b_07qsriWxvt^2dCvWgqLO@3Tp+$(++M2neQTH{>DmE7F-x3B75lti9n__5&T-+^sY+&*0`zo?nhD-Pe?WM_$f z=lr#~vN1Qpk`j`1BEdP0>)^_^XUxwmzshDkREY_=fRh{v0`c@|vOx&ACwb{iZYR2> z;*QDiW~*kP5BXMBS{d)-xhcm`1qKiMt{QX>J;z={M0xaTU$xm#6p z>-ppH`m=kL`az{EpW_R@&C;Ytx3llQ`Uugw4PbSgHz?>A6n_bbArhTS8b-pe$XA(o zN_cO^KU>nHiAz{VaU#v_woz+$_McQ5@TzxN{kd({>D2Hlbf0u>l1S0?LO1(N(J!>r z(+^KD=JzaRbLJF-#+^oe-a+MHT4~9qkKj~uIYQK_Fn(^w^2fN^8mqyM(PqjCgDCm! z4uiO5Uhp>0B31dP5A1azSjjAQ*BL&Sm4hny%(|WbRE8u+mggB2DF(OR@2U>N;-6Eu zO?+p<5w-`c*x0w)06B2d>gubp_}2S}&Pt4jxM9G8>wxvf$>9e0r;$ohrArx;_x+O@ znlnO*0jw*WIk2UUI`}Xp=;9rl~p|8_T2TEWBb6YCz`_pLe<;PWaMX~YvZ?TXe)@bs| zB5;2=n_Sn%q}WbkepMfUSHC_unW9V{ zm(ub!Dv%}nQ^{B+DF^<9M86%%vU{PHHx%J_1W*}Jqzb#3;W>=9No65-w)RkZEkI^c zrrn9#_GARHOTN&2q|J&grLqRE>aQPwV*C=6vJG0?_oJnSokN8yh>ib7;8~At=SvqZ z$fVs1QC9^MY!yB{oku*+spoGzsMW}tdD)h`LS)ywh% z_bZoU0M+|tH@#}$P?e8qf4D0B3RIZ(9x2^PoUftC0WY7G>)71ye;|-9z=4A;hPM_{ zM+;@~!e)~k0xJ3NTekp4iNj7NYYM4fH$NA`?X||?^)W$iibk&B3#;*RZ=ut`Lh#v@ zm0;s~|I2ID8Yz#7TZ{XhM}E@ArL68=yyZ5i3_Hxh5n3i)E4>6o|T5q*Qx1S zD<-a3^#^U=%B*46Km9LOR99CwCOZSF6&UgH8E01Z4BPt>gksNWpHutC%3jdT;{tAO zR~eQDE#mC|;98c7jEu}hF|i2Q7}oiJ2u{iitmYXk$6z4pccR*itJr9Ek6m3|`Skt+ z1d*?soA3U`<5TtmUcS7s6Znwl0rR;%a&!s?9z1^hqHB4gRxVh*<3Jic<%+QI?6|_V znooLqdg3h?(;WU6vHwtJnQp;ASVV+%m#2?U>#KbS6m#zfdldp}PUo@8u4ha$+fzI| zWArnZ!F&A0w+afEYhq&7T|6rOGJ%1CEaj_-f{eEt_eYf=t7#t?r|fZ3x-4te#zh-M ztdaYCY1$i+WE5)@B+XMDbtq%ESh-|t`N+#{j~hF+YQNeF6GDHe{0L7<_zr3M zY;Kohn0LwNq}6U!YgMLNb6X@gYnjJ}-Bqqx+q%V)cZxytsT*+ugwQYKa~M@`fzz+3 zs{9>~#pXnL_aqwsDpDV>y$MC6zsrqnJQ?=5?YBbpqi+jwC5NL(&*#%d?CF6G%=ARM@^B!s91A8FIYOOWs;*7-tw3lu-MK!wk}u?4pwjmk)~}Quk}f z^;Ngp*2$kEPF{FDWiMH_OpaOq0jS%z^$}OOD<0;xj&xAk+RsF(##Hq0fq^LBC6QI>b~+7yio1#h^rp{?<1#~qqj{4!hQ{!V z0B8h~J}$zWE*KTZPoXj8)TW({OFOeMc`v-1<&oD#XYdHeqmuL1l{kZ8245TC? z(9OGdRyN~f@pog^54Y$WX_$;e!-( z@imRj@b~$~BRLV(YoFA+`=gB!P&IR~x85;i!NL3A{*T2r0_gg&%X3Fy3X2@ZwCn@P{ldEfLE>iY)q zUSFSf2Kpv_??6EU2z8u!01D_Z_0-$lCGt7TI8P7BeX0~n`LJC+NvKGuzPO3ff{TNE z7|nVU1CyWk*}pEjw}pNSH`mkm9Kc+X^t8vkK-aZ$^IkFJk(d~n_^r=F$RYyqF+z;X zpSo&1gdP7S|Koe{bz-+ZxYVvF%ZZ3AuqrRrb7z~S6M!upQA)J0r2rC(p=)71f3G+DJ!&ag62c zRKC^pG`E50`e8%UvOZmVX<-|KLWOhcA5nLjrd`0 zFIiO-RFXI|RyFztW1FX(%s!f`C&!qsR?|}C4ZkX|()s=brQgpicK~Yl*2o(Dz#*1< z42T8(PE2}O5b0gO_;y695%M#&DA0;uZ_$IkFZK;GiPwJe+_7OWENA^dvk@T{HGr9l z-yn=4*;o-U7I*qvYQ)s1Ufbm+O5i)^`aL_H4QMS2lV;7k9(>C1h@p=2!x^_FlZcMx zqWZrh>HzoQblp<7oi7k&ZaC;Q1VA|jO2B#A$NoyzqmFh^2)ru;ZZ7+ib6jENfW$TM z6CB*GOG$1&I~p5F25B*Iap?_>a~;OK<^$GfX#^njI7!#M1tMOcnSmz*bttbIw@A}7 z9sDjryMMW-y0m2E;78AG%7urxkeLuDY+6*7JS`<67)+R5go#HXg791!)a-gauL_kVD3uur}U!> z3%d_J<@sOzc55q}Je(9# z<1w;dOSCESgy=+Uc52G<*ArxY(grID@%{X^;nmaAvwlf6QQf9ru-|TP+nyH0-1T9t zcMmRWqx?n%^3-O+EAL7g=N`}}D(6?03cSo9)T?*fX`#Gp{2#+O`x+o|(%7tHE#EakJCiV@oCwVuHTh>x_IV%@(pZq`O%NNj&tmszYp*N^09jbZ&-KmiV4Ze z%>ZRUXzab?=WUeqj}uFXc#nfDv_^c!HUXSNhP{o+xqS~`v03+Sg~e7fSu}tVp#A+! z@+^AK!`x>M9NJY3%@}W`D?R%GFKo2m@HErRPLf{3xb4M1fyx^U?&7_pBF(FF0jRre zC@-vKqU(Va*^yzksuBeqKN^H>y9PaFw?6*n?<1^NK7m&k1868>P3N6gt~!&6ypzhV zm$Tqix8Fsl%DUc6faluL$4+3J?SFIrJh~T3w@|mzyZ;{qU0!?9*g%bBeG0q?Y@9EZ zQYZ(l>9%2URukS%W(p~gk{RBPj$=T+fgv7xo-91pG+K}=zMNxK-2Ny z;^?1ww!=INzjje$vjv-LH56J{9iOfM@z~62<@1k-)h>DTlUJhR8G&f~{3IZU<}7Q! z(m`Y86bfJY_!NyeLzFcpKjTsTOxR?U^Loj`M?+$Pqo~M$DV=kzbrH z9jDK>5qgN^4h&!lIjQ3B7yT%8s7rXxud zWYx%Q0wwjXlV^4sYpIzb92@p_wO&?_%ZS4;E4A}shITr+dHMlAar4$I^)0?CB&ozt z`iMoSVy4ADfBvU!V}$Af8Bn+{fQhipdoN+HWL$-q$&z?Vzed@%Z1(1Q1_FPk`j?Q< z&{I4?h&NeP5OQ|uVCOB!^yXd{>{{tJz?E7XlUQ}K8;jq{Jbm*z=pR0+F2KWV=l+Pg zhtkmn-$P|_KjMb>C!j`3L-}wi2!&Jj{2}Dg{ilKH2r&krGZ+RLTs0COjf8KE| zfNJYUa9hZ`8?VWp)=EfxBnCQ6^X!T*#l?@;w{#Io$=%Yfu9*ppJjTWiBh~De$(d=% z>X5 z^i7{%VSrVnoF<(g}y-ZPs|+P&sWI>)X4(V9vyU&>}Bc6wL(~u z)3pqrnvlukF#W-u#5p2x-(B6T8tICrH)4cqkK!CIP_|m>Xvy(A_-VPt^f=+GD8V_4 zuj}f$P2%H)2~M}x2I`6u%7MgW{W$uW(MHd|*&J^LHnt)QJvtp(e)fZJ4+6dgmx-`p zPjddz4o-nJ=MKzwQR>yz(2MB8ZsC8?S85aO5|n5*bT4JO=Z3(JiwCO*gTA|rcS8&E z-8~3Q-0pu=FY&)GrHb?OQ(7T)*`XA~{du$BiDx8&!NGP3fw0Wxfi1Sq(47pWAkHMc zj{QmCb$ASqMHk@~=Pn5Q7h-8&pQ+Y47A5Ug$;v*L($kCTFss2(M8mKSq|H)ULFKH5 zpKh-{d6_0QPui!2XvemL1Q!>hB2Sd(-W4FgY4#+@er|T8GM|du?)eXs@m#}KuRS@& zY$o-&YX=9CRNv_p&ZVgrSk^E8Hov~SeEFnIWHDp62^*r4wd^rM=S&78_sxNH_BQbuZJ)i_cH zfV;l9J!yOUu+{}iN)o1pmBMPp$Gq~L*^Tf*o_Ad@XU8=clEKSEq;KB))VUJHOM@ zj#x+)>Az@DP2Ctmd+%(v$MD9H-F|ZBo6$BR%daFKNYUMlvGW_I^s#iF%I3HKGvJ-% zhNx)`^z$15piAu+fgZD7%(X>OL|;6|S`wuHtr6Gh|1>J$ektGz;(p(jVUZj#Z8 z(*P*3^@j7P$*x?QC}@fDHBb{&Mvj&LDm}!cmJQj`banq%#jiKjSx10ZqKg$95fwWs zdV5dec*iCEMxzj>PA4!-05oBIp8mc-4GUyO$2HEcUHU^hP_C{|c|>kJV2oe(PbaJL zdi$t`0WfH%q!jn-*GrhUDzB4qj5M=lXCANO_av?0U|7Vo%%Z;Gc+@ z50GL2WqteFr~*#etovl_)(CM=XYw;2`;>QJqo+N0jO>7wFu8D1V?zTWNqbW^*Z|m( zoi?Wap*bx%c?-V)yY!X+qWv{mF7k#hWxbaC0G4K^E63l2Eh+w9NG&P1sSn>o{@=O7 z*zHdH^mGDmp|}SRvX;w8zk5m|BPVAv5E2@?oj!v+SAG7V2v&i}$jTZIQ1Mt2qM`I% z|D92J|4>-4F)2x>OAUD?`c-^&ZS7#N46$?_9j#<#?KMMWs|fPJgrlCke0L0eBKX-aVV@qJvIzM?)e|+dhPm&!SuWucl%q? zj-^)MBbc^Kb;oJlQO%_t3*ADnKl`AaEXEQqbm%0p*JN~_HAQ(V2$M?5(9r?6!kfO$ zkkLMPQ9Z5dCvTU&zP;tPK^K;MyYqN_npO}yliWznLWU+>U8j=q(#ZO?&ig*5BbU~s z(X3{=&&E47B%b`XQQZ2ER@+(?l3rBIwAWB*#Z`LpCyOv*g{-*^RH-X?qWWpq(pgm( zHv{ea&U8Th#nDGTZ@!EqeuF}2XiA3Ce6EC3VsrxjWRIZTrbz$Vg`N(qeYFR(pq}aS zLWqG!Evc=!(hl4oLQnz0DCVzD+Jl3N-L-|IL^n4xRl0tDc1ev9l^?pOIo`ADtgG(H zUk0^@*(?8Cnvpv!lc<2n|ol;Mh zrP%my5kAJh_=cH;X2#^-O4YRnr7V3_oU=p!$U9ZX2xO?hgTt-!#$6}qvp`#di6e2# zygI15K|!uSig|V5=QsO_KP9;Wk(VDu#X6R{;1b7Hl#Kba3cP~W`fY45L>&kky<~)D^d=JHK@zFrm9v zJ@_Z?v#or}5l!~5vPL4q6cAG;ly6~;)hSU}hYJ+g8QmWmvqi-TH_7fRC%(sn)VNkb z7!y>^J~(k}-z$v$pZFo9Y=0j zQAHP@@=qnKaG}Cpz42CW2=`TI76T8|x`HE4feeuUvK2}neAjI+ zYz$6>rD(L{zvP*)!BnquA)MYIuB5qMygd0+o}c!3JEazHbbAYd^}Re^kn|{m*!O-d zQMDpabG~A`@9_RtSvp3T>-}!Cb<9S4{btJT!zcAAFus6wpsrg=^>)jsP%CGzt@(zO z`2zRH=&$4V>b)hdHqXOp2U>w!>nR}m6%>dmsbb9IqGYL-5PdfZd6lP5u^aO^_asbz zQ$@?#ef}82EAyhh0cx7*uG-?_HGJ0a@Yp??P%DI+L$$qqGIS0cW{~K7^R*kzfN#&u zX|w9t*~($OP$f=h1VKl4WWdHqTa;=}6E1^Js{_N$n95&dPgW&)LtjW=a zAubK2;nDmU`P|T4IdioT&)6J$a0mj}ZXS_X7iZO-_5s`*4NV{bho2uCc-(|LT%OAv z4Jkplg;@o~rw{x<_<+6#h>h;|^grHug+%`4;|YVm6H9SAn-YSf4n3i!J>A;cM7n;C zFEMdjVH=&{cC7?DlRSr?qJ#NXt)#p>?!iGtiaIdGlo+;JCiyDrI604|nkcwFVoC{2 zp}ExkllJ2qvO!cmB>L;unW6mj2UrYrka~;kW zwG{FzoGk#UC*ZgU$uDFk#9CZgX`|E>5D`U$|qUOH<5Q}VS%m<&`0Y%R>`EXX+ z(w075MrJ0?=FA&OQPVnJSuPrMfvTlPyoz^!ot*oPxW8XHsA7jgCDO3aTh&yJK)Tqy*Bv`H!GJz& zsFr@@CfhfNs(|MVByF#^HhoCue*UWajbA1tNJ8q|$xlup{QFN$Dun?G#AcR24!*Qu z*#|=a0PSha%(0`^n>cW+D04{TJ~*&ujpkz)Oi1Bw(qKm%r0mTmm`_|jB!wCS1%9$f zLIh&6PgzAJA~BJve5c{0TijK(YT-Rn@co&*YDORe#QW!pTED zm-P=_heLEGojy}lJEUMU4ZuKJks9mFH|UxE{s!L)sc-LQ8{{jI{x>T&D;e_T2cG1q zV6{IARU5m5zCSp{(l9kNgrj44<5@NIi>|?57{;aQg`Wk*Pk2zrVZDKT`3+%PO(8Ub*gzu`_Rw@ zm!?XC_`H>hwCj{m5_ianmPh}b#>sr1s0Az*7m_)mwlm}88F;ADcq z)UPoP0Zi7`^Y>WmDUo5fsAvgd#ihUD=rE{2qQhJ(E~Ip4ZcM(!$TjkplUBls)eX?+l6=QaEUop3s=|2E()j+|Hm}4wo5=ll`v^<@AfT3=i`(nB7!PQC5S*= zB;s^IJmkU)A*>Dvm%fepu6xkAQ*CcLOh|F_JVe4<90>USL6%!z+w6I}?pZ-#vy*D- z)tja_}U6 z8=)jmZ%V&d#d6;n(0czlxiN z!P+J#$$*}@zX6a{UNXLF3%su^Y2U&-(KRmpHJ#&RLF25!pXp z@_<#nKT=Z%6Zt=EDF4nX#jHEy?@+p9zOKlgFfo;7*SqiGGv`$Gh!0PLj7_7iaGC;7 zlmG~DCb9Jt{1qG%91lZWE zAjY4ba5%m7?w1LFnbCqI#8#xhYF^ON`qB&MVokz7bpCr&Ffspsz@Ds5DDSVNsL9F6 zdm^;OuMt*aqs~P!#w+=;huEi-bisK_cpN&NBf&m<;a->8QVPVNm-V5a?7uQpQd(UY zUL|U-dHithD_Ti!ZuE2AnXItWO^(=n(SG#d1hBr`p@iW(;%uVblTF1LRA~)W~HESdhG*W zAD`^|H)F&6Z-75e%Y?|$h^4r>P`6$^SejPw^H>f}fj(5N(KbP;k$dMw9dN!)l^9_F z%Wn7#Y$Xe5f01r#;1Ym9@bO=Ul0W!dzqgBorxw@Wc%uFv8#P+`RQAVR>-pg}o=g10 z(85Bm1t;!67lR)Di!pF4N#)(h$D~@7?dKx)Fmf z{iy^5Dr|Kqbu@HV%5F`~-dEeCBvyV%L$j@>ssG^AR_hYtI3ya$K?d(zxfdZu8RCCa z3w!jfyRVO9aTn>@0!wEgnT_7wOSN!}ncq#>Q(ih-nCd_h*l#rlK3!fV{qkjL!f2=O zzmC@p2f=)VeLbq5ki9_q2R9t3pL+Mlg0F!>eC|j}!{Pi#^Z;xo!kopRdGNee{ zpf2cfY0XD>dI|b@FLNCH?2Y$qWcqA;1AE*8uij34R#BGn_^0LRN;bxc-?ZjXDj1jc zx>(YOR-X{bjoX}S-*5PI_{kog^LE#0cFTY5 zxM&gXzxccn*%r9p&93@jysytzu(Xk|T%Y#S_xy{pCU#7XdQ$h_mSFCx4}Tt%wQmF? z)bdRJ<~n$)aNJoOWWhJzUI$+RYgjGz`Gut)=r+)3!o>p*5dde~CCvq^cHEthrPH~K%S7!ILto4nLrZmbWVO1Wo}Dx*RR&erpWYYLZ=ShGy}Z3~;J~L%{wf(5 zS{)%PM^$1Z#kJey&HU?^Un`&hnHC;DYQZLBXX|ts!uwbr8Eyg8=g9Nd0Kd{>24hzB z0)dZP)+=d4Rb6D4mmsx5QRh$;^~-{*#YDoO5^JgDJEden7dj?+EMBi64e)#T-@NTw z?(lv?{J;Jm4+ zsRnGErze`D@C-pdWVDuA+>bgdE33dWQ#u0BRmGCiRn*d%+~NGjg|{kx+BTK$bG7Tn zm0t=wDXezai!9aB8HziXn`eSCLJ$>95tW+k;(z@7-1P$2ExYh)omenkRZbk}p5+|8 z(^DyC^>;_xRVIoYYF~R7Xd_o`d>)J&|n;RoQ(;+FTw{V`K;`QL! z?;6_zJy$|dYTHhuu$H#X2u>xK)dxMAc0FX7oGmtJVL-Czsh@r+nonZ=JLQGth!@gH z0b39b{PMibCa7HtmyFJB-iO~`+Kv7(Ok$+Hi3z7;rcS$;&)CA^Tx2*_u&^*I36;uH zugOIfIMI-q22E$nL?m8t5-^FgvwNdqmb+YDo(VV>Nz0{(>U@0g_@;An?8{~9D~8W$ zX^;Q@)zv&6r9rglZc*s0-N9u|Am74qbJVy$1xAAsU^KdQ;f0P*+yD2^`>ziuFNlFQ zS7Z$V{?>QT(My@Hzxvt!`=H`68JX~U<&EKbteS=n+uq_M(_XL|ymf4BzV5yoh!dDG z7KXkS7$?JEt@Cf!|99E{kc?z%d4hh zhO0|oFAc9Rk{}=D;(|f6$ESF5I><-eqV_cZdBa}dEg%pOsuD>r%iM54wfSUy9O%ko zH+1{+%a_4Y8X$craazFz>$8-%ry{=qXWu)JF9USiC@9Gsj_Guy?7QEA1`Z-&Z;w}A z5Ew`p8A)4k{P5w#QW(GDy!Y#XeAsGGGlU%l?QU-(KDN_R!_5yuvgjDAPYFr4 z@GY1F@&0ysSG<)pzi`2s5m!T-t^&Jh`>xVZ`6m{s3l9?@{rG%*9FUau_zb0OZ)JF^ zPvFAkSBI-NnouTBesF@@p$m_ycHi8>ciuueQ#=UpPj)xzcgmkqQ{C5~>qEF7JG~kz z&5)VR!IpiY81;)Khf<%qxf-bW{rnF&%7`mdl)XdoJO7@(p_)_IeRd=yr^yg!b!u`g zp)QPhQc?TSt?0zZjaH#LB1O{WG2v)e&IH0X`^Sq?`CH75-K+9I8~2#|V~{OADo)mETUK?Wa~TMH!5G6Gp~0Y2(IFL?K$tUp~8 zf0a^t^K9?R@3(Ltb~g?Ye=-Z1Qg}IzklAN)F7f;C9tS=k+`K3JIG_b^`^87MbKO2W zHrIT=L7b`X^?n;#!!eY?)LdP%5QElx8;j&VFH#f?cP-Fz!2?DPjYlKWjELa6fcN9DaiP~juL4L$Q{F5hI?nLnw4rwVZ55;K3Hf0l=Pd1}ER zCPt8>L`GhL!_D=F!}o#@sg?tVW7tL=7_av?b4>Q3I&F^ohI~l)EKl`ZMxG@|HcRyF zGhkuT^TvY@ku9(Gc;kg@V5S^qWoBaeQNtWMH>we?!kU`@Q}eU@{q7UEqeobr>zc~j z3@Nel8h)h1r%8qTG84S}uch)a(Iz?3_a&$4sVOY)OX0m*7I5b}=V0OB zw4EQq6EuPfvLSZ5qs=3flR6`psA$ZD|KEd9T!Bo7`u*grbjPN%d26M1R&-0ZAS_J6hC zJ$EceN$Pn_*}6g^PtRhtScdYqmX;Q`0*bqmVinf@%1xit+xzl%+aVoLC$IJOX>Ig} zQbjo(7hCZEJV1U`cXJz9eXXpo4-N(9{p9=Sc1eAaKL%gssR8su0ppa4mX@~W_U1Bd z{lAH^^a-$N<|w8Z4}cAzUHeO&hr^X?NTIsZ(>tMh^zA=I+Q(T5_V`oKQ0`wzKB0*-Z~on(QH$f9q1sGBg0Z5|I%E;YlUzU;rLprX zL&;a>ZM{(V#>Ga_zz@=i1Yd{JMyxW}WJdOvsvZ}S^HRT>GI7*%ex9N2W&dV;pa!T< zUAn2?07|%Tw^B*clOqH8QhPh!zZnJWBepOiBVGJn`{9Wtlum4WF?y}g@@}o(zSj|W zprOI9T~Hl3e@a2Ujayn{R=W(sbhImUxvyuN3VYn`d+Ya7U;8wiB&dC;U7-^wE+z*4QDDS(C*3y@^DP%F z>r)JAUKTSNq2Islw*YU9vYOi0MsIGAaKNUR(kuN2qkd1#)sr2JRuhZKKz(_Rc*GZN^;qWP39xRkV>Mw%)5U+H!)-9r1>B3U=IRQKvY8ceD?P+2j;lh$&6!=Q zn8An zvOg_me11^5^Ct;79TD;Y(+7B7t-r(zX9=1H|J|-Ga2vS`OQ^D`mTY+}!qBMC_*%Cw z@?FCRwnn{txKydmwy1W$FdBsO*Eg`ms07K?kH%YlkHTtX{_LbbA6r_)Jk7OM z%UkeqBV9jsC_g12imYBrMUV0Hx|-Lz(1&i$Yg(A93NdI~(}R{6@&rZXpigfb?6Ok=%UrZXD_Z2 z&pr~~0EjC+N6A_Wz{%$HE@?E!>_^!5*4-@dt>?978h1R0XlQnVHZ0kJwMrs)!Wx`| zzoZC{?f0=y{)l;xCo%SJrH47f7)bx4TCX6B8&5mu+}*pt8zw|1i;G$;ISdFGOHHIH zqh%E$B5Wpmfz8rlzlYgGvkyUQ@K_qdc%jLY?=rb=A>zi};;oex0>-@~d3*KY;>dUZ znD^S()f^~hmPVv6m%$qf1VFzf-tHKF=WCQde^&H-pZB}OT^|gS0O?-+B6El2!!vM{ zLg6^q%~5nVHn!ZNBJ*{ioI;OcR%*IFjr9lE&~^XKjr;n|fq@arzaR2UfsNNGlSXOa zeyvpna8OzHnzP{j`}YZ{sb?j5%UBaauNy*$XzmR2tQ!Q{8*n(Lg@pxxQ*za27E-cQ zo+T&ObgbQ>ITb;xtESKjFzbtm)^?%a(Xv;ttn6&C2k8^9!^Ut1NWbFCQ|nE1b9LQ` z55B7rvjJd>s^h`xTWG`Zu<|Qu=@k&GUvMrdDM@($nckzjZjmBUsQwsfk}r1On-W{{78aN z)opR7p~Ri?dlWe^h+D_3>sl)8`T)@i&|aC(%-8z5JQhDX9Cz~-x6v%8!8>_NW^c0o z0ZmJR7&sC>GuSr=YWGt3TfteGFhwGU+D6%#9~o<=cO3IMlJmEDcPgyquC;)niA6B!jf8=UJ3Ml-+Jc!#v5zS)3xm=;1SmTccf5mC&11-1m;X+2MVu zo6l3ZoZSYIKEadY;+`b_q4ver^)5@x7ll0`{frh>U|x4$&@;;J3c~@%WETM0h&4sY z+22Az24omTJ_O=-<42KGf|Itb!-61JYt1$4=(* zoD5hh8`gF2{TjpEouy-9y9xzo4O(hpEjg|l6Bt%3F9u=1V>G1KS7XL#Lf&p|pxhou zU2pdzI0}*&H_3vBXBPsxP_xB&AwErOklp*?lQjN~I}^$sg#)Sv1Ng689Jz5B?5&kc zvODk@6cHGZQc+TeXTuZ zw*z|+B&Octc!RflDMB{Ec%0jv&v{v#k$9~qhl9}cpQPDPOv`fBSe=l_I84bSv4$@E zejSVY-Zd+AP`Xml(Xv0ME5{O?a5{q)Yi`lY zZBCzyP4tjHQj&27K`mh!U=jOv)}0OkX}fP#*{Qc+N-gqVS_Qe!=(n)xm# z)_iYvG)PR0Y@jLRBG9ht;hZ;R3*Ra41EI@bYBS$J8q{FlR#nbp?P0zRTp_vo>df6* zPZ3(6mFol!wAZq|fto;wey*dJLJh0V(W(>!-17QBNIMvO9Jj`OOC|;Vi*?+`(nT&^ zblepxUsbc}y`z_syjOiJmsKXbUiyv@dxYnMh8S=fw40ru|UUGC35I5@bQJ~O3n^7v9K?$zHNca*!6ybTFi zxxe?n<70G?)J^_G+n>-1`S~9wu}C3qIC9GW`QMbk2{wWjSb_5R`Agzn&X_)^z?x$h zL#cOn+&3#s0@}|nGj`bfR{nv|M-p%n3)IXT9KHE3g(TL@%#1>WKO`({2c!?CZ*5uI zE;Oa^B1dEn73A5CtUErf#T~g(ZjL%x3JCsMtsG2K#BhT ztrq{xPFrWTmo&?1=(H$_O5c|Kb4Wzs^g$rp`m*t1)_aE2y7h)g5TgO|O}$CJooxs= zCh@-CMnq+5QZ=4+PV&US-4@9&LL}8b+w$uhYmAtvlx0r7M!$Kenc-XEMZtV(1jM@ ze&a2V1~HItI$!Vbk^31a)m#W^>lQIA92^f#_uW&ZomsFvjH+Dm)DpsTsbxY$jEzT5 znqL1iJ?cv0{3$oYwmpe2ymspbMdEp3MIX^k7*L`1B!ZAmpFBiVN7M4z)o* z8(h&oP|jh_D@3)n)I=(Nl38gT#KCB3NaDO=xOZ4r7&w7}cVK~z?@aT}dngr`hmVrZ zQZsJ1>Q9k<*>jH4%*x}_S=u*Y_l@1vuxn7FR$qb zIT0o0W7N_hKh75Bh67@M1U!_Jp4hJS&L__soO+k( z3I}@bteHgs@$EqLjf0>EM~JFmrS`yc7;+~66z+KhRv+W7S4ylL9L3la%X&99n)-G7Ty6afjjug>RkCtoYy3C z9q6_|*fR**AfOj*>}R)p>H1dmaArggh0NhFUqnl0~ zjRm0%7{o9GK{W-Ni3b><9Dm&6a?`@p5H4SP=*$QerTYp3iQ_#%EyU6i`qDLP z^f=P|28ioeX(Jd)&|@=Tx74jxO%kk=?92wK7bEr6Zb>eU5ii(^+Ipg5 z_ppJJKPZ2&R+x=#GQBPJ#xThah|?TuYON4|kDKO_>7}l;#<0T3YXAWJfVd5XErC z5il&LET!tOkQyFueJ>bv=C_Yd(r@|H8hHJ0SZXHx9`Nach{V#~P?~(ZW=6)CjNhMi z>Jt!`h~*1CI)RqLr@X1nZ2PLjqQWeC`kVW*dc5 zJO_ZL)>0V(lfHN>j6fhFW6f`7+SgoWUP=W5tz2b)#qM#`2Qd-ap+}o}Kuhx zJU9F%UD%!Ly6SUE%DKhF-HiYRli|~6Mz>81S4qJ)k47#6m+t1$3b3u1D>|k$>IAGa zN2aGD?;D{skjoUf(DMbCV$=#FHcB?Dz>0DsMCDH$H4gIeR|TD_C`h;M4>+=A|lc}e8up9lvNwe!NI9!F|;0PD8kc z-4rxib?;p&*7jzPyunn_`7D{cwYdKfC%vO{965^(u0ie`%9bM6th}ScPUHK*N|kQw zBmq|U!&j+2$5)dN=l;A>&}v-!WC&L$!#cThAufk+wLM!$6x~Mb=1kL;hBc> z3s6)eddA1jL5!)11~47~TXQK$V@V*Q1n#SE;m}m7m1P!qsl$t#8$~R9Xt>>9UQ9RD zD59=#EqghT8C;fK3qOI!iYq-aHu$PWU5r;}g6~*3))DeLw`koh9R5j8?D%9#<3qKP zPK8SvfGlmOs$UOU-?h}ccWgrD(f{+|O=h;@4hjK*oy2e#UD{U_s~fVpPoK1}E)cGY zh%^vE{Q0E%9Iq5Le^;CujE`R>d`E&jiR|OPI76B1NVqb7%1a zwH+Z{F&=d?A8UicmRnApQzmteq85G$48)&;e5S=Tt@66WXLD}%acn@IZl^1FR)Zbm|W!QOtK z!`b#YVK&H<6=d$tLZj6uc6#v2_a*;w4c>N&leB5ybj-PE`B%F`-DXSNtIDnC=$^1h zfw->z1N)osf@aZLBU1R#d!3PwrYY&Jw^;6(l0A0x;CxrcW`p|U>+D#j-u$Hcr(cz@ zybX1;`bPG+6<`Oo9gB5wESJhvd{h-XC|ELbAgD~Qu{=w<()IJk(_`u74&*IR9;f*@ zw=Q4kw1Ga@70zmiKGAe_fZcGpOA*z#(Us9>j~SR0m5ik3$^<5?Mi_R)Qc~GEvK4wr z<3M8KVS!ENGn9L26R9`AUATW_>fhDTu`^a046-q9u4K$Fx+86M8#KrNxZ+chLO^1< z`9&8U@qin(WtG?QL+GiH??uXV>MnrN2c3s@cG&`|t5LaG<6LaY6NvIA^JM&eqnO+dpR~|`w-}139k)HbH+u(_HL*U$GBk0QX{8^rf>vI}GEOnvPUA&}cLT39YWnx6(H*5Hgjm4pKs$aH>jw-8VPRoN zX^M@$tu2d3LlwDx`5}nX*9t21}d+=aps02Kb#nyq(7DpBB#NGk&^UTfFCS+?a5+vfWZH0Oo$pTEAKitaS1R=%)d0BdH^G>?{NQqL7 zw8$6YsLyO6V$dzMiuL&PA1gWk3J{wK#vpjvNPIF*e25o1YqNnAPV;8&71ZNCoG-`l zvfe@-PC1!0#OUCcqV}cFn+1M_{&6aAMz>Yr_W2I}&-2@> zLyVk%+B3oS`m4aL68Tfb$SEO=DvCZv+n#vS-=8aCZEth2m1VZUsumxzxcbCg!No3T zQ$3{5162Z6Azrhp>VWWl?M|iszH*@QJ^#>U%IMAi1ub_s6D&C2y;1<3QaCk@4Q3hN z_)R2?cXimM5bI#8W;pUoG&euWMcY4)&A#STee>YAf`}B!Y&ozPM*iR^(#C z2%0_%6fZVqSN;>KN;I!cGB*zu^z?`kUJRr-C8lpHdJd4{Zjt$%{$~CK5wj>Su-C+C z!Pi?^uMoS+=>y5q;ZWC<>0H&ph%`JhlAc#*{Tv>VQKjzPs8w4Q(Nm@*K1YWVz~oVs zAsH~KFKkuD0#Vc8Uj{TS_ZZ4Og42bCG1GGzVSA&IUybt04S;U*J{{~RF-lx~D~Xe~ zTYD+YS!`D6Y|0eh?5DmuX}B;6MGp?sEttb}DUMs(YLzcvI|QpW4#eypWqPL*|GhY; zj6S^Y6abVr+$kW(7f1kDpzzL8YE$M|%Sqxb6c7@YY4%?BAzveD#BTvR4tvW2f>>Yf zfQJ=!FXC?AN`XPA+-w=DZ-!n#&>;NRvUZ-hZE+pmEl+oG!v&H*VA;QJ8L;a|Gt?|m z<1qXAnndji0ksQT5N(scl=d>*Z;;`Z`yi(GVMYW5n&jH$NGt%=bB&uopKSs%pjabE zVH|)WnUY8LelsqH2iSe^2;iQ3WhPlT(!81W73@}MAI_MO1lz86@1N`rl$_BP0<+_{ zni_NPvA3Q>_zV#wDFJ+7;Uc8kf?Ux9dX7qL;S*Jn(hp z{D3UE)5_=R(}RlNFlmSJDV@%pv70Ujuc~UjDoG4{V_M{qS!J9iFST{QhMNOFfLz6z zsJps)Oyp?i|0C-yprY))c;P`5R1gFeP*Ow?rIC_SLFo{sQ@XoTL_tKl+d@D}K)Sm@ z=^h&Cjv;2gJ^KE?d)K7Q;}`RbLm#tcD@2z>B5__-R^AArR&=_}Xg+o$ zB!ph&hX{gSFlejsidX!Sf=JmbgO=eoOeu%6CqN6-k$o_ZRV!Kdev2s)tO}yk zEz64L9gAl;IXMxp+NHZ(A`=qcmzO^Pq}LG~Gt;m4-HmM68$Uim^ZF3$Ymx@O;h7j7 zcBWp&HeEiG+_=%6U~O*x7%Uj@2bzlwxt!BIPK)ud)qteM5=LhRa5EATT^p^rxvPs^ zM}Ct3O;@9$t`0gH8Zump`kO@BYdSeW1{R^VgLmnMp!Fm?1&c{IOZ04Ph!L&b-NvV^92}SF!`F_B{*j!P`FG2bYqRtOeUzwH;`KEOaLa zZEkK7egDh8MqiqloxSAUK+9u()zj;egZ>H@=KtW>^;WrnW<$LS7r zN(o!8LNF3RfOu{VS*VcxJBRfdW7e)bo3?d@Zn2lfmLI<13G92@J<}v6q4i34nfp)D8PcY+CRJs< zP+dC*E0P%L6W_z0Cx{;=|C3}5A9@FQQhgP{GAQP0ow$}HwCmJNpE{|C_pfsRsu9TU7 zn%tM|)tp3Q@LJtKl?ug70r(^Y<>jUOIxl13I4OY4T zW5z-1@iTz;DDdsyLiMtKyG7omm#6&4AO^%?G3d;|aFu%}O8OcWBQ#a#1IT%;LzF;$ zgWuRkL#z%f8l6D9pVs>nYx;pc;clDxH~<1whs#+q?>?NoRVLTQ#Xs`x8N%DCJk+mj z!tvKM0E9f&kcHdp$Powck_L8otR{Myi7`eGed4n@9k5upxC7o)b!MkBu}=-^&vw;1 zoAdl|-O{PtAC(FcjT!2{|8ln3@Wb^&99=J`%yX&Pd!ySP*6gXCTsJ^e}j`Gl8iLon*q7Q^@@A0v%$}wmVnV|6Bbc*@6S}E!czwwOvbai zPLb~_xVLZbbs|-=r&1K_yX5LeheXkbXSle&CdJy^`~6$T6_g)4AuT%xdNi~zRN4vK zRg`)3K_K~E=nx)1s}{u*Kl_#G9k)`)ObU?pM~0-O?Zxp};A0gJ7r=-4p{oZa-cV6L zj7yp!j45Wx;VWoYX}q_a<;7lH<4LB}54`82mL)w!hiG7ex&grw6i_YP;nl55d7=cD=%71|VXccx5`}FW!KW@S~8or>$3Q zZ1Aas-HG)LyyDz!s@-m2j#FRi6~ed+XlC(Cpllf+7EtgZmyap^MkJF>U75q%Y>!7W zPHvb}l1@iZ`tT=25s0GWTFZypd-Ccm@{$`TTsENZ{urd`lLnDh6@ofGUU#^wjAUrg z=rJOt%x~U2JUkhBd39zSl%(19ddMtdYjw2-W*=q9L+#4vKP4Rv}B#N>0q!o6_z5G1g8q&PtzCI8W&EJ0RNR#|& zVk|pny{y7TH|Tf+=~r4xRY-m1?%f-ZV68A{VP(<&zBJyrH*O#!=5M+o1?a8KCUSBCrx&0J z3V1$cFzLV@8Cej#{<8_yc4M~Ls=hOREZ@WB!JK_x@|~5HdDkgl+%V~)#qE*83d*P* z&rtfh-EDQr6ft2V1GVvl0*xsOyV7t$gUS znJgIXt@*en=dXNxuSQeTs!4x&;AMbNsrf+1a82>n-;6-4sHmv5tIMb>=>2=H|KYC| zfxe)a`7_@F6v9rA4>oFVOg64hR9sraRZ89P;zWk6>0;;81*oLcMH!N5UAx z(GN6=6Ut^vvyGFVF4Aa;scyhujrbAoH-qd)=kC!SAE(Q@{MD=d30mK>vuQS}T*yE% z8S!x*s?zB#x7HfYB_Av^`W$0=wgAzQpXmjub3Gay9PLgL=wmZQU~7 zP(_UTL{7N^@ls9~vDcm+V{cDlx5>E9*~cQ^gffHGZ8FRo*+Z!ft@4ba8-#?R{^@W|?cgz0LI5AhyTiC$HPJ6dP7cknAb*@1+VjYaf&7P3E;f*@o}Qe^ zqKT39az{{&uf{DU(6a8{b`~1qq1NM$%(Yut-1n#Z*kLzoa_*+v?qZq4u~N zfhO{SSy*fW`_vauF%p>r_)OT3G1rjA2bqLEHI9T*b^=MFHHr+iow5ELefIHZEL5p$ zr;AjdX?*XR*U_1L)gw~z3cMFS7prum5>^)&>xOhg4Za<$cQwXwQiewv^3!}^gW9vu zga9kwz@guWOq%S7fG!@3##b+6$4xylC=vo-3gTA)bC}YV+?>{~oards3SbEiQfT%F zyAZ18QXoa4WyoRPj|Zqu^=Ms9KK6ZLQHBJqn%uU?hPWXmvs+kajU)F4yBYvb3}%y)Ta1LBDiVx~hD$j|Rw-l-C z>n(JpSRG#fhu**@)}y$=_WuS|l7bG7R}EGM<0(YFxvy9rfT7*tVyYtyEvDAl9+{cp z8kXRa9D33W=2Q?6i?{+?Jppmc_mz$Pske8?aJduNH`$Tb%gc-lEl5>e;XtX#`peX7 z7eH(tVx^6h&4)6_fEYuHS3oW^FXKlYJJ(JQ3nU9n{uJi3m zS%YI5x}Kb5pl!q5GK~e62LDef>*u^1gQHl)L9duAx*mPKo&-}!~snBS_>pKfi=!C|v zfgczHLlo8%Ymg^Pm7QG$5mgPPXT6pULM{>OP4J_=zg1Z=?n3hoXB%R|nv(Q7PbQZO zc$3t=4v9RE5&{2qs6Q#c8&Bb1Ys&tv@l1czz^ygUn_O&UI0!8->dQQL zk;%=Sh;Na8*(xETsRHjo{}eQ+&hm-MEe2v`6OuY59@z_6psf8)Icp^qZy*e{*_{joo@x;zZ@6K=ii3LTw)Ye6MiL(v2eq8`U_oad0 zJMxkAr;l%bed46zRV}DKIIj=$^w7OOcAucJ&JBVVOz6sx(jwLAu`ZBi71<7tH#f_W*zlySGv{gSV!5dO zO*^tLV;>Qq6M>HyfAIX3NaSd-9Mz3kTu_*YZ18Yr5jqboO_2%s- zebz^Fk-u3d<#lw3`0Wiv80E+79nW|yCKoE+9dO;K3ePUctK^EiaK_qH5qQR|fAtfB z34FoB^X@$(elR<$a5j3#`269Wx!Jv;fGKF6IehOh2;FOXOYj6_n<)_>4~H!rF9Qaj z1K`1Qu~~{L->mS#SNFs5#zvdgBFi~O46vH*c8mpht^2him6YB{%3Y&lk$pR*cl+#X z?(-M7p8&WQWLxd_%$d8BnDs_w%W?X0_P`&)c1ekoh`N{R3$!;4Hm7Q^7I;|hH0aAg z5C)|NXby1_(RM#lR=)K#ly`FfVB5YnuSUMEGhXncV#dc8-=TqupAS9i&XkEuzP{aI z(nkEwEaadD@i?N&_yKcb5m(j;2rU$$hSsvElD}f!D!LJLty7=H(@oren=J1fe z7KUB$L36d3>mZHH$Q>-Eq;+Ee=rj%9ii|?+%u@O+meRqY_lh0R3lac=2FuT|4&~PN z@AY+l>ZPvF#yIbI?hwX^I)PyQW0k z!Y!yF)}g&ZlY@VgRQ)~W2DZ`IUXWX*i7|v zUTIu1$FHHGShuz1<@O^@Wo7Us!))zUGC4E+Tju4Oh`yjR8*`z z@-WSCawGivRFW+j85!VIN`&PtV<~K>FzEq-{7ZX#d)cTfjIs~)^gafr{eA3tRhaMa zJ1t+e&>70rUP9eO4Ry4()&l(W5*|qkza9M%m^%G8u&MD?$^_E~LG?94kb`RNJx7t;)eZzA%2;`J#RKyQbOz1`Z~20-FEq3zA4CL4 z$XcbifF_&AnA?s+DxsUr^4*rh$KdjFI)x2o%qIumz#w4{*QCT)tz}FW(sby_J2s{039Hu zo;mJ5O>r;EEd_j@B=mt=?=(}y?9`(2C!7R(KDu^O@@2{Kqu{7sJ>(HVr zf;^Q?V^Rzx`fcR1Kh{zFUsHPLb`^%Bi2xRcvI$1$rc-mctJ{nVr4SU|P|WHdKB7#+e3uXhe3;9&7=JTDr!iUK(Cp`_^KU{1czUFngWC0v@*q-Z;k(1}2 zb_8wiJA>p%nGy#hfdb;{mpIzWvS`vcX0I zkn^|+KGM)b2ixaQrKL2W1qFE8GgrwiybX9`JSx(v5A|7~PZ+n-2MoJfFuYSMBQ{zdkVpeKbkzAcpJsAQ1<665eti2uQR z4#2eG5pF(#<48-Z5vFptAiu|)M24(duBicld{V8tGInH5;@`f8Xwj=88>;^w(AbO{ zGV$rwaFp6lQ+Dpl!ifIC_Gcc)0iF-J?`-J$OuXHi75hD$VYpCH8mG|D4=>f7qWxy1 z(1nhth!g>%J-;H0JiG6yWwd*$c`2aK#7BQi?8@pF#f&fj!A09S6iy(_X=wKOJzYhh z`f53Q_l(}_Hpi6{aQxhaam@LHiIRYL8#7He_tDwT_$R=B5+1J7yfXI!a&_=R&o`3! zK4O%JEcx|;M=zaqy)ii`L4RJaD(Y@r=8iT_82rA z0rLIkTFGpVcPQx2Zf$okskQ7upv21+7 zwF?&#a6&=R@Q1t8=Qb?fw%!+aXKMN-QfHvj0Q7Iq0Du5%I0I5KQxzMzaV(V7Yk|1< z3yu-N*fA< z7ecgg2-pe%MC8mM2Fck!PPE%ouJs|iqY0i|NR*UYhr-gi>I@03Of^Mljjl-d*%xOX zqJ!<{kazeB87}j1ohLxjc3mv*PgBVxTmbDbN@lYZ>6fK62I8bR>&J#H-d?M8-`}rb zxtGjL!T7Tm#68E4r%fG~HcafQuF!M7vm=kl?_orqmqzA-X>`rzqtb@C? zBNJA|>ofP6-|vk!AHj@-W20;OJndAETUF7uOWAaG&t;>hA>$nwmqH+{se0-xn}EFE z9LlUYwdMPEBeZ^08UGn1tP5%ShB{o{G|+8{L!Zz!%JNeGT%(zKgR%*!Z62Eaq-5G% z1fB_Z`pfwGcwBWDnuUJh<5%+#{v=EMlkwBj&8D<`6vRTkfZ#qpJ3rSSR~0h$Iv_w8 ztV2SjB8#W$-_j#$k2g~N9)GF_xSnzQgoDCW`IOEUKBN{NB4eO-vMN5?Bi9&fBY8M0 z4jGPZE(79UV8;LqO~W zA2P3^gn&+294_v2??P)Vo!vAi3BY-Hby66gSlID$4JKEh<@*5Ea2$x1$ok%9x#Sxm z*nF<;wRT3{kRwB-XW3b(^5e7?Mof-p`xIJ!{yBXRRBU=s{8SU^0H{^yk7*89pT>;X zL(r*J)eQzoKxq7)py#>2uixI5~DOij@Ry{3~DZ6r1$j#IONQM$AiCkbDvzA(+AY z=4WVSrI6+D4;<;V2p%8kTK-K#2mZ2-Fk~&Ev=UMm1Zx&s!*8jd+Lf7OXPvXIxBSc2 zygCOz-P-;?lTHxBpIse!5+~pw$Trb^ZQ|F*TBtdO^BXvNszjcH=h2rjIv+WmeRj6b z@}utx6C>ne_STmD=aF)+Tf56eto}n*w!cMsnbK)_QRQeT*s|Nmf6=Q?KVeWLBL%nzqyuO|tq~*{J^Ec50}Y=?9cUy-#>y zy72n3Eg{-<{|=D*O5HgKK#=`9tHmM%c5Vqm9RsTm5?ee>$u2LMWi4wM?RSXLv1U9m ze$06>qT^YZrMu_5XEN?Dce(mzw}u7Y+X7*5Yh^ZoF4k<@4%_J*W>*c4+2DLY!3|;= zB8gSO4qmtX@n-4Q^>DvglW7wY5t}jsMEIU2sKz%j9peWPB2M&x8A;N~T?p z;H~&AuE$4*ddq#dKMsO|It#G&@Uk48O+6r6ZKtf;D8KgF4|#o|i_fU{%PE~5DVABU zIBp%9F_6Id*)bSO)=QM%DCfjn>^McqMp|e?pQ^Q zOimJE&sFf>2Io`_M1q|&@Bt$pse@lG(mS#J!!5lw5c8;ok=}6Ga|iFJ4&{FuPn0?p z+S+R%FyQFN8!E7~D52lC&5w0s=o?7WOpK|TF;mA=s zoE|Ne9?pDUpWCRIU8}4Xoo_VGoh% z-l9ct@lncH-#d~=rA`*-bli-vorunu;CHqyV>ZMR$m(h<6m1FGvIQ0ljB#L zY5Xe$)5*yW12*&nn;og);W}*4{2{mVry5Hwa0gWKBp68bdQt?23yc`o%a}+3GTShs zPN>t;@}_=l3}>;MwH_k3nC*|kHhVNW1k@*=l{%Vno;mZu#^#LBsiO}SKL>GselzYp zYU(GhXcP_%K)hW&5=BE0bZl&~Zo<$jz>4G-pLE7+{|0)&q48@cVIc;!{Lg&Wlb(oB z(a9ct>IdyGLY9!a%B#%NkQ@Z=k1>!B31-3f3Of{;zwz>i9M@{KoST!rX(2l1huJjl3q6KGgi z9fg}>KaXMW>()B-H08H@fu1gQnM7LY_`V9baOJTb`s7rt{AcXI-sSU$P%3f4&3Vm~ zx*9+uECg`B*Wd;Og%2dtDg5?xG;omO%FF9#ZOp=go^&%g8xaxna@W-4U6Xm?d~$MA z9OsvN-p5uKd99OP?k#dkfW>(U@*6$eJ(UueIR;-X?VD^UYk9*hvu7-8U3k=6B$9M@= z2IaVKm{z;m-3PJMPZa#WdS)C5VKPDEQ^}5fR@gmcNI_5{GU*-<2&POon0hE0bPzW^ zovT)K7mrDvrSSw6ZrmZVB`!(@o{)1cAYfmQ^@%z`9ZxR~*{#ik37j_q6_NBW53Kp{tP^X9jWiNwA2wYNMLjKIQ0**fzg z)raaHy#ag4nI6%N$A(`6+U#j)NWs>eZoD*~O4!3)UE7BZe6%@ysxNyTw@)vBFHoCI z=oumP!Q`y%R_Zt4}{LKqA zzEoG!z??hVt-5+t?H)NfxgYZCS0yu#7r}qQE5G~u-+*}IU}dZ1P7U8*tksDs@UoMH zJz{v+>FLkt=vZCR9Z)*zi2@Qh)1$xaClvehtN{qSv=%9n4E1j=Tv6L8C>+7ParW%l zhb+ZjvVR5cK%6{U`Ykvx{`K7!qdT-A6ps!M_r1a-|6`%79Q;PJ)iLkU8-LRcYB|Qp z>CSlWY&Q_fI8R6jLK!5z`yfvD!~4{|b$-?N6AW)d?PfcPt^cM%`cf}n-jq)fYD-vz zp=0dRUx|@fNPLL|fxDUxGzBb;zn%2csfMV8vr)ZiyQoOPX*ji>QglbooFRw?7* zK^`x+7H6l7jI$%BCMVUl8W_OATOnwf%Up_GsL+kKnPnsR2Th8tL!lpt0!@^cz3fg} zht!Z4VhS-Wt;8Ttd<(9Y7E24a$dRe;*K~{R?Fvo)meDZ+&Xqy=YcF&ms&?v${M!;br@4~D3;gK3c$5)j|I`Ja9`FF*V^L0iXqa5|U}_{BDQ@5bu6tkFmq! zzC(?_3lsa>62DN9hLv~Y(|-wtWB@oSJv5wS@7!&RJkUS1U3q{Dz|1LNU|%I4`czOr zqR&YSy2_4_v;tjv)JSh*pY-w%tOzVh||K%_?de^r!_14%7SoK&UUIp=C%VA`EK8E?hu#UVe$rc8c*sE>=rFd?+IMt-c; z!9idf3E$)|E&jALN+VqCzD5tOl|V#%C+-PIH+q)NYhnR-YJ@uLmcUN11<)G&0ix5F zDgL|xPUO_$;_&x906GD8;tjkrkSOu(S*(K=Eo5r+4@QeGd^#{3n$mc(g8w9TfcGZ& zF1Oo%hq=)-2%cu;;g;|2_{E8}AacRe2Cf#O%9_k# zI!f2U?LBOO7YD&0uV0E1ym@CdtI2nAEvNcBN&JN;Foz4!`u3_Y7PtQBsP@*cGUXNJ z`^@29X=ksU$jg z8*I5iHAg<0k$2cUKHHd6b`kMMfcI>WXrWcWT9-}I!}?rD#6zIFxc*tp>_pyJnFsy9 zH{WL-0P73>f9OY3_Cx0)s!B6iJNMi6dC{MW3veEXBf2N|H&@niAo~dF*x&!r@Kdp) zB^`PL0DLNS$|41b9CRN+zDW#*F<63Op3O?GaL&!xdv+jfoU6n7#K`E@`1q=|i#~zn z9?$O^&<>;Y!6XwRu!2zmL|u!Oz*;r|!(^Pf`K@Qq>H;24z4kuEtA`xW$43I&Jk7Y% z<7|-yY@;cdfCKd8bEOtuejG7mxWK{x2yPq0BkC?zR;FRMKo?3~ec%qe&8x`B1&3g& z>;1XN@Eju}hc?A61m@naLE~Q7>Q^GnbmMYO~H7 zs~V2%j_-3}Ot&7$k-idMhYdB9TAp6#fFLf z`jHwJ%x{&R(ZV6E-9UfhT-zYlC`BCVyVZOzAMfeA>P;KUv#C%s?}j!n7ja&dD6klc zSaPpktjY}Q6m`(8Ad3Qn&#N^)YL7p?Yn*M~P4(fk`X2q3zy$pY)z}#Iv*e&H`re}p zXPb>YJ*BnSDqg6R8%~tpH8cNSwitAo`!`aQ4Jc(Y?(WTowj3Gkm898FSc6Wf@)WppdU~aA=z*iBsBmkJ z<`;XkXN{xN1d1Ek|3VAM$x5oK^u(g8(MH^dF0)mR&P46;+%9N01^H$)+9O}9*joDz zujVoajr75sF5$#BG>~1VVT%f;OpQ`B<;wI)CFT;M3O`4HuIck(w6Ve97rYd6Y7YsR z6YxGM1+E-^pOb;X7ic*;9G$^Igh`-JYcK~~AcU^HF;Stap<(QT;fo}^bg8Z8*aaiM zbc#yF2n4-BARhK=RXEF`k9{y3eMMcyku`@wHJ7vY5?5?7`$)9rbsCzKT!4!Um|0rN zTF4_1VMrid$+&Gbk*8Z^4cx`-rlOzQm2u4qh}Iz4eRlYKxV6d2$l6m+M^cvv&LR-o z(k3S9C;KHe%I-e95#bY3K;oW zNN>QfIBadK+Ixv8@VtkJ*sLkg{0VCz^;gRm4lBMju6{P!hjmo21P{MKJ z(_s_)WCZ3Ubwfz>=nE}R>xTEii{4PqRMb!U5gyEMmPD7Yo@=8;A3?Jrxh%4wfrR<) z-LUd&021I-;=`sW=Zvpu-bu-A1=MU{op#eiJh-2 z+0~AEpF?XwIXT6+u7&3<~v|9(|TMVSHo8WyYaX2xjl>tfi^Gha{U zb-n57>F=qy&h@5Y>{L=|Ly;q*djqPty-3lU1^{qYnz@{J(4D;xJJm!u%Ztt5zkYkt z!+U9+hNOYxYi{lfUYp$L`3~Woqw&B%X4IcAn%nTdxrBt6qodi>OMi&dcvyzJ3LQtQ%nT<*3uBE2BwR49w8f>(uh+Fr*%*3Iiwl@2uO zT3U2PMNQkk#CUp#)#A4aVe!Bhclgo0+t|M>tP%T^@}44dRD$mF^>~|EX=`T+9b0bG zTy1?}WmRQ=ywAs;Wi$RKdCw#&TEGZ{HfsA2%{zkb8PU&gZ4NFfurn~OTs1DzFkI|z zDsWhw`RctcK|2u@5@JKfjGiH5G6G#j7W9PIMu#qukKJ{XxF@oK_aLCOx=VQ` zteLv)G`E7XqJi;VF^g=QNo-WbS38e=;&f(vL8CQm-Coh5^&f595@fCRQl}doo+{Np z#M>9Q5Qe-X`?CYvzw@f2+5$5ilPcNj#=e+&=#8qspv$6_nRJP9aVv9;jqO<)>fZGj zwk~P^5<1wmoR;UveCJ2o{#uY>Mf1k5wzjy+gGs&y2b1H)($P_k{1I`_>e53`yZn*I z+oG7B#3tjuo!8P&o_wO>|3Pn-s;Z_&=W(#mEif$Gbx_H7Edph-o+cwKnt{ds1? z^p9^02*ey1Q!@GaA;(o$mX{-9V#tC67nn$&1$?3GKL-S4nSRU?7dbM%X7oT$p4H1mYDKV2GsH61#OJ@VMDk2cF!2@ZfF$F>OqE zI7wQKe$z#_g?JOv$xw*9aXAn_C2ef-ddEPzOQR=$vBE+8fabkHOK#A+cU{KeK3nX? zw@-H#1yfZRDi%Z&jCMkz^16%%%?lP*kP2(<_L1T%)_5pd^dddzEJp&$02ON2dBuz?G3}M z0s2JVf%b25?s+CLu@!T_KFw`>1wcj^Qz}#d3dVzlDNasl+q1NMjHeEf0tyo>&%N_r zC|_$^pO7&;u+HJc?r8*q=1$3mH1;n7dX|<-FdIHDTyk#-dBDu(Y$_Wyf}XEAS(wj^ zh&VG+E$l}83DLGqixwN(_0cWvWCaQPLEKxdRvR^#u7&~37-m1CnL!dSfM<_b5w~_C z>c!ReV%j`1l5p4luqy%Vrt)6{A~&YL9%j`%xK~v*+qYNkwO4qwr;~SN>&`+JiAJLh z#c>gHpptx1y}&TAmz(%hw#f888)gBSL`Fgq^kmKlgDgqkJB*{CD{L$lxs~3k4`rTECjhtQQ zg*)UBw5#}#2?Sdst687w{2=TL zkFozHLF>f;uGP{ua6G%$BbdpE0_J0DPSOa8i0+;?1(A0U`y7hpk60!|;DBnb-SNH^ zaJ`A6Q((JL3p#&g6)WDLf|DTTp6Y9Q`t8EhoS5e3_OG-C)jMY4+67M4-ebSBNUtZ8 z!gYd`;qyb#pKxVuIHjap@dJt)Az!;_$L>qCfE6NF$Xm@^lNKg*jpgeM*^1- z`&CP*<^GqsyQ9&UOJ*Z{j%_ZiZp24X?56%gv9M+>vn+$ zpWjkJa$oZSKEjLU81;a8oBxUUT&G2>;a8s%oErKA{zIDT^ua+*?xQk;mh$-?^y;DG z#4p90(34)LO8fCcvAw-LD#Z=Oh(O56EjtZ>$Pgo;yFyw+0K25i!Mk_wPX0(-+TLsk z4Ehvarp30`)M3k?{dcjXS6Ay~GBGlKEh$k|h-2VLm5A%%hEKfE;ThVvB3DUdq(er1 z@vTw|NiIteFTrNhk6x0Xf5sVEG})#0K+p z^EjWmV-r_cX1TlYvF4W2Q;lQDAfJr@tGf2|RPXP*yq3ds;D?t$ij&FB!*ja^y_a`o zGB7hUGd%TVj%mi0L-Phau>x$NTM{9XLNuw^gwgG2DZR|~J@Uh^l^{UDQgjK{V&)qgA;R44c z-8NdxR^s+T;t&JH3w+CigV2uY26DsM=7#O%tk`eIW5ujdizybeE?PPM0ZeISsG2sH zn&X0#WmVnwV9MpGc1UvEj@Q^mYB0A^CYaSm$C}UeJXrx(d>u<&?G(LFW@{o%ce};- zl$GCZr~4#m6C-XNQw#1VQ8bH&A50#Huiue;O)nUGfcW!VZ8@q6(-TZj&vh-Ny#kXj zXG3=h7FkIu=0wiS8f~*}gIT+dTF$}=Yzbb%u+s3&752E#`5dKAhF5v$Mr>`__~)ML zVuscQsPl@8RWd!?-51EH5MhObkZgm2U|-Qo3Tzp0eDZ!SQ2g_D4+hl&CoZH^koIp# zci4psvBk<{cFAz#ARaE4POQH%@`s3{QNZAb5_*06F`R($`uTCYdwa~nJHl7P8;yR0 z=~Q$+T+geZWGG#wS9iv)Vj`gzzF`ju z`F8S43b^VvcRRW)c@tO1N;YyuHx7=YM;CX}?$g2pBED2rX=keDXxs*ikc8f^Er0%G z07a)mx(6~f;EjIG$dG(~7w+V^14uRoKCOxGFw9qjv@Q9@6`V|xpr$z+%)ZS$5L2XU zDl4@B;O_I1LnWO zRM;CX3%Ktl0Mt-UTpU~a>a0iE@-yP%ULg(2#I)F#t!^VR$KKe~TSDvM@9`6)xVWL| z9TrHt?-Ey|JABHtrKMeM32MiquNn^F_9N_jzNM?SH`wvaNx`&mORc~)EJ(-MoUYSk>S%l9a@lfEqRxx5Xqu-xjGU%F>y zkMSgFY)xg^Ce;GA`PYb-6urNHf7-oq;5LPw#l^+7KRt3v z8yX+aW|B>4oV9bCk5)B=ntch_o3jO==qZ;8kF-~~b2XwzB8i=Or^?Bc`gVZUq@oc{}lFKlB2lJW2u4fXV9az$%= zFiI$Pj8PtonqtJV>Gc-l!grA4xt=7ZGf6yUG%_^Yh6g!YrJW4uu0`i)l7G8LZ>tMu zwaZ$~X8jFR6Fcu$flIHnA}>Lb)oP`Z^~4$NS+!0vx+&~wvmkol-|(!=&0#lJ6tK2; zFfEv^xRWi~p3{Q}w=!eEsqUEIw0h=M+CM|D_Q*4=?S#jY@N^*KRG zyt}*G`uZG!lKXgbvovZEbMiZi1~=14=OgSlSvk4DhUs6RiT3JbW(W1Ps>-W;|M*m& z!e5PLgwK?TH1G~j&vF@87kl+KcJE$I4o@ zXkVDfXCr<8JKMfmfuU6UHY`9ofzi>?sB(y1D%onEVDVsYc&>KEw_W)6%}|l{<4Aa! zHb-noYtrv48ZfU~+&L*3Eoa8no<##5_Y27yB4ZZ9d($pCt~zdb?6>kcBZw$ z;#0#Sp`%D1sbir(Mul0(ipWU2GB1x+xIYkhe9()Ub3GVOnFpV-M<;y6ovT>K{`pYE zeSbvR9(~d?54qaiboq&hAo75@!JKE;sgp?)yI=>gL9mi_Ue{s>Aj*jVd+G{0zDQ+N z^0ux-pSw1dc27i3HvN-CMd^m}h&whnzc%Xoy9&u?=CHkadnc$ zXJ&Wal#}DxZtOJ_U(g(5SMYgDeWSxqWo*WWK$t1%>gM&& z&d&BCk?fEIxu5A0WRd8E8w@H-Fqes3ct0tneBT5k9?lUQ*L_MVYxa$l)7yV@pt5j}3 z{yPd$N6;qwP8QoX%H+8{Oq+8};-&ApWUcn4NeZb^6S+6^QpYTR4hns$G}(OYHFHy{ zS^270V@;8zgu4oo{8W)pqh)5ZL^-W|pWAAQZN&zp_V&jU;r}hH@weJ8*o8%>o1?9H z+hRy%DJ1-wxkb=CdCf3g&0!PX#IrCzZ}g569J_fkBa4*HqEaI!xDP)~(5nPVj{izN zi-=eWz~~2@Z0fH5qMohJCJTQ|=~u9xRyuq$fQSbYhf6Sqc5^Gcdi826+y}?J^mGZ> zdf_p%2K&n^6DMvL9*(NBkeQjAM?mE*djJr6CXfA9l8WROCR-c%bnZ{&ftv*k2*lFN zmh0E-Y+3(+fOHSYl9aTyb2taa-!!~Ts~&TMeOap&AHd*VHCchyf~yPRqV4eT(2@P> zMb6sma5QN_r&fs%1otwC-~Y2ZpK$5u4A8*0d$lXZiA znTLKp6o{}Fx1H}1E0|n{em6|tp}*3#q(gT(?AkpQQ`vKL%KzPcqz6BrE!L079-M}k z-1YSzvuT$zWU1y2#$Al>HZOYLFb!e6Z3*`1U>x=o(9c(ET-<3Zb6EW5;a#K4YcrV* zw+rM8)0}}K8v$=b>+1QlQKKt+lRGA`?>>ZvDh$=>aVTTcTk*MW)Kl2Ty%v_1{n(xo zor6}nqZA%KezcB^OwhOc+;E4mJNeJwdfXQnk<{^U$#PlAffShjBrT!0GZ{ry?->Hm$)O5;2rgdUZ@cCe z78|IYg|y!bX}$0YZG$5tnf-6yzAZ3pBmGyZw!ospUhwwbG#Q?>_;@{z4D6CuxC~c^ zuy0`CHVDxux>NF5S8jM!XqH$@-M^viuEhrjZL1}`2^a_8OB;e%$}G3+1(3JDd0UAm z%6_EUtKu8Bi&D07xHa4E5ecM@PrjmAsmiUzo{*Y9@=p?B8Km+2KtGp;A%BR;YwzC|c|=;SdS z0-*~%6_sU~ShhAOQc^ftZ(z%)|8*$Sp^=zILrooBSNBNxytN(o%SCF5BOv6uQU?AFp!|iaq`m)(hG>(58WE zD-8NYkRT|#V=u7B!TK@!!467KPA(YY&TT;&cu2$_AoP<4HGRVll2(TDhT`rxop{$! z2|CLzMMJxQyGJPU_Yfz%t%7H;-LRQP6{}{kq@7*=>P5Z|Lf|hv|}&dIbuZlaC7UmAf67^a2Y zh7-SjZPxBoTizEDVPmg3DWLG@oG4Ud*brs` zSpa!pyN%DWCbmfNfp2dA`4Va_4Ohqh*}sO!ogM*=|F-LzGi=v!0@$GsiD41K^TF3h zP?pEPzda<&vK@7lK)kAaF+Fq4*XLTLlg3LZc64-vt#Ayv0vc|@%6{e!Z?^rTY;EVf zJ+l)7>2Ii$UY$UJOd7Kh7rSGKyCd)hMFmf;mwU$d6=WxgaoFS++Tv33P*H6L!(wJW zn1OaVHnteh!)u?y<(eBWbCBK$m!N4oJV)_KQ8LqtHxX`$sWd!O*!AX?mU+OnV9T&} zD48}uQFp6UtHxVE9{@)er^i@zPJbkjd}8k-FKyF*l&93+N{3eE)xJy>~#( z|NB0E>=jg6s5Ghj3ktl3Ki-^sEDMUS!wBrq)EF}G^7&0 z>wcb$_viaN|Gd5EeLCmyd_3;^y07cH?}uaKwnAMzDD@h&v@vy?o}OM5m^6(={4V%{ zb-ObAgSKqjCN${H=ce9|m2ezfhntxbc@uA)xU%!5R2_>}W_`llNU#jybdDBtl_r%mg zHVTNBds??x&f4_k_v+>MDp8;t=;HI+9J6`zW`Q|pm&$?}^E!3vpp%nOPl9>gvk&$| z3$!glUUVq&CRn{*de+Bc=`0?h#PYTozR#~O12ds`5}mibyx|kxXL@Gp#P%-u;*5bA z$4}g8u#Cc|_tzzqm-6V5&P8#fID|d7?&8f%Etzz%S-tF@sW&(5wbA}-!SziT$W@H} zcimWGIq3^NKI7P#76o(q;PWf$qBItRr+;Pjf%VN=yLROP>#P(QE-{s&o?J*g)DU`^ z#Sdhr59w_DkzZH~SWS<6?JPQ7(^cn>9Vv~~Ft@vM_zku;kz!8r$IhDCSlphu?qfLlhCIjv&n?sQiz1c5+9N5}i&5`vQ45a{rv zYcJtTCJxXeDL*$~Ho63?U1!c>KRm=Gn^(`fX!N5q7zT-(-7`h?mK4eLUU}E%v^5v+ z8izCM7xjNmZ~io#UjD&9dvH};JH8IRbn~t6DJQ>ftZJ}CdWlp>*wcrea~^2u}OCb13E4&Nj6Oi}J?1SqBV?{6{nI>UDtY+F7}-8!i} zmbiN_4-k+SkY0t;A&4mJ>Xwc0v{}a&+}>`BAR_}n%?f4XoWuayEXT)meG=sbBJ};f zc$A%YIn~LIu)(Bga-pA7+xwNtPHmSsW#1s@YdxUq5a`MR|ESLf{L>zd~TkRYrA`9}6g`iYO@FXApa-Fkp?_NfO<(Fdc^61flRfz2Xd+7xB5Hr>2 z6ICuce=4o3P*ycQEzRvkqZ3X8{)Qu^Fc)v{m5+Ie3+K;8bBhql?O#F-U8)yB($(LQmZI_r8h=NYA;8V!V@eq4xD=NmV zsGjh}HC?LzP(yujnS&v|YeIIxr0X{UAa&7sYnp!W+ywdES0v zuG?8`Hhv?7?nNKomu9#RSfth5xqVwLMp#HF>cInPU52SNYb=Tg))A4}UA|vN7vb@R zQtqMJ7ZY8OV*(9K|C#Mr9DnK?*jMk^A#EH`pq`w54X0ov;gx-%tYGo>fVz<0uo;Sm51|iUg(wM zP`@r`49n`duW)GIo_k9~DnB(h7o&{wpQs&gC_2g8GzAEzFkJ5TiYbyodG+cozXDz? zzW4YE6R^m-+oQ*OKUKAn&%_&sOn556+kT?t?c2hp(v(VD!Y#R{v3~rE7STr8t)(Rp z6coe=5lyA9UoVE7so^RrDvBadTHsO`@*t`+b=QV{TkvjuS2Ht>XqBs81_@&tt(Mde zo+02T$N;}NL%@$8s{bXC7Z#_N|I^aHSYG46=Hq>e38Cj&IY0H8LfJe2Eq z1gA(SoHWU~Y?mR3u%OY8bV52r%*~sx7#$*^s>+ZYWt`h_aTkHGo5zhbE~KE>{M^l* zRu11NRhO#@;Lj2YZ`LVJ#5(rcT6F!DenvlgFGVlD_T z!y|1g>#NeA?>T>L%w*8qr6D|71`>_4SsdSv-T1wD{qN$fJmDnPJxuYEj_9H{C(`UM z=H`zW2lxXUiPwG;zR~Zit0N@x#}Vq2gMfKYXNv|D%g)P1jT+Y24uZL#h}&^m8^$FI z_GC=P@qb?XC7-(maZjZ7$rbo;`I|jGGrN0xh0}j&vfa2R=)7ZNpk5U?TyYn6`O5rV$~<}@Z{9hMS&sr+J6_-y`Z@GF4u-@*P!}rX@X^DBBGs_%j}|8 zul#u?Pu{001;Jh?X17={Ie7Sk!Q(H7;byw8xkYaRZN<4K7ecbmJ&gq1dBw$NH)hcu z3kO$kDC^$;;92dL=Jyu&bECIjmR@=(ooKR#RyBOWWU)DCU+UYXv%As!V1{+L-+zUW z;8(C1{k!e^AkMPcEmU{v za3cm)fu3JIW`nl1on7#%YB#BCKl7!Ea!yYdmR^FV-FFpFO8#5T2=*=n9eNZ2W3s-l zpv3F_JMn(4wc0?7h;KWlWJcK`s@}4CZ5nPPq3NVq@x2_z%1Yw0j_R%yv-e7a=aD?0 zSFhT1;;9La7zv4)NR~0_h8<@-ZNSsKYimi5hl%8j4*rcp zjLXD{6UP-f;=k&AV`2o?I5wPNP%wjMq*pcU+dKsSSX;M+trgg?VS}n2Zb-fFF)8Q! z9SwK)K{vvY{2Osi_abtb#3v`~!jphl+{sZB$GP9t4>WuR0l|vuS^oi)d<>^R1;zWs zeJCgz;t)~Jp`Vz`yHN4X@Gi%Qvp6?zXPgbU^fi7mkibfq&=o$6U+!B5-i%%e5n2o* zOn>g$rJpmxo#j7ionlxYZU&6Hetms&Xt3jOmhoF`cOnWvs7tf|U7Aqdw XAd!Da zGhW|e=zHGSC76!=AZKFn*sZOhd=L_|H2Wb!2ZP;g8b|B5jF3MBM#h4(Cow^6 zWEmTg+tBUHMuvJJeG#Q2Q~1iGv(T>|c#81^CG0I;gu-_TF$esAm2V2+t5MO@%Yii+ z@z3@16IG|1v9Ve$u*K1*FMCq^B?lB-kRklQ(b73efXG+2-Qb|aU@~ZzI$swje9JW2 zbM+2Zs`xq%&+F>Up4dBd$pwsAA%0#)JPd0Y;tPw5P$Gy|#yW>zC=E=;Bvzc3D5P?Y z>&X*NtXFOOa=3dW{tvHe!knwlV^^}1Jb!xJ(-Famg_1ja!Gc|`2C_&mIa^3n!7lCw zmnzLx3E!?Ts7abhRe%8eVeORwdDz(A@VKChdY3DzfCSebYvAPMWcPuGfXSKXbwr}O zlWZ%6u*#(jp?wTVx3A{d=7gm0e7wW9mMZYCYex8WX9z& z?@L8qO&k1Lt_qLlj&S#{y7%-#^{~SJs=y4A($Z1gn9~WedZWf7huILWhF4&>0^e8{=X$DrXAez7o^lxNq*G8~C8~s$-?TS)_GBrg8;~ z|AyA|_s?wWM}6C0L*>RlV^0xpjPtz_+Un|g=bGOGY@$Sep_C*27kDv#{NJdT$(M7& zHY$eZyR{q3C*gfVVj)hIAjzrk*dcMdNJ3ne=fwK1vp7DT705#d$qpNT7w3Ri!);Ux)%l#i05#)H3`$u4Q6${>}~3bAlKO6z5wv*?}Fcl=TT5#TvJ7cOfGP z52D}Dr6uOZvppsNfx-xye>nSCE@*6Bv{#d{lXPIriS3FK+ZQc?Px*7i#vuKk@g1M0C67)=)T-))vyfq;X^7aKJVIi zZUD|C+qk(IFuYS^k1JKFj{Rvzj}Alsga!#ij%Oh^cufPLsL{gVsOvu59~v&9Vh>)r zuQN|Ps0!wk`Y+iFN?~gNB?*s1ZtjI;LE%M>7uagS)~X}(w=Lm4Z#3AlOHxAOYBx~~ zU|QxCp~UtIJe^(3l)G>h43)cq{|#?=Iz0I#V2Z!qZ>7WG*q?-(`}i2rtzO#A06CEv zYEO8#bp>pL8^;0|5~2{STu+tGXXfWgEk1vWpqye|;Xj3TFjHT^Ji4t`?cy*%my`r@ zOjoSJp`A=Mn|nyd+5F?#wL4q(N?*D=4TZaLG?BzqV*cW&u9iYYxgzVlt=Z=N1HpYW z9Y-DemyKZQ`X8QYRPjJZrZ`SwGob7CJ8zTEws#E-s_xd)69RMLh?$EluaZ+J?Z1A~ zWha?DJ-H@gOp6nYsxGauNrB;c^#PUgujOC#W$&iCI}fr zcn~+_^pzEPEQRM>LlRp`oKO<>j3$+njMK-56~#*_H}N-HRd@1}Tj~&gQT$KRd+WMM zVtvCDg#u+`)f92n>eW>S zvlx=!X@~(ROLMvZLaEC%!=moX;RkhiBjsRx$2=@ZbT?4MGTMPZwshU!iivRo&+zZF zn>y?B7%`7w>qBoSFIgm>@$I@pthy3;KxYu^g(#u;_xz(jv*CXtx@P9(CAF8@j- z{}@?4KIS==C9zLU^_xe5OoPA`L@&y;30vXCQQy31N^?^an5NKYbdI@t2kTUY?#D+k zW3esy4u%M}J5>?tTSBPsiCc9f5+SG}%fU79xRO=ZaQU=LyJYZ)GuO(>2Du{fh$Lb~ zU0?-JExfWOl5x0_7m>FhqVLj_Mi-pHVEA(EGWw!9!poQY<3`c^R;hv_nS00KHszP1 z?9qc}sHj`FR5dOo0}E9huAMl0&K&iAaMeLn9mZghF$OEbZ4rQusvRL34ok`sTJVql z)^SBAX3U(K2iC>^Gar~k#RN)MNWahatYCY7uK8DRxv`9he;)0-7y>cOi{E+3AC3R= zU$?Gi4F0Wm-&YV#1U5QSc;F|I!s9?Emp9#1O1Cxp8U*BkFf;-Nq#{=7#A@Fw z2mOd6W$+JI8wY6?`r48RQcy0ATPWp*4teZ!tg&Z1-vBI3y_nCi%NOR6L{v~3h>zls z+t<~|CSsrs42j`{UEU{8n!&O_JV8-k7k$_{{{!|yBgi=?X7K20HwdOW^iPnRi^ZP_ zFtr8@AV)vZI;iB$8!LppGh7xzAYb>u+OW>=T}5i<=Im%Q{B*>XwTDz0@_|XUGsWKq zj#W&N_=${htYa6mV66XLF<+M~;JyFG+`Pa2(4j<~*p^s{t^|sD?Ei&(rjG#kCIKR2 z6MDrHH#%wLaLE-bt_c>ThZfBR=L|)FbZ$j4iR!M;uIY|HM7kucx)ShLUZG=04jk8SzjkR14a-9M0n zqKtO$4uc@TF0HJL<*r|T<0ZWHee;aQ6%h-I8q7Hj=CAP=QpHr$f;YCdwq_iiuNr|^ z7$pf>AxeaEC?}Y7il5nzW9T(a_P|HXX*jbyJ^e`U(shy2`qxgIGxeFIW$PYh3B7$@5n8!+QX}MR^%G6~#`%c+4wDv_b_Bj$RJ= z0lbDG&~ZKH?qMtbZ2*}b~DNv@NAcO7fxLrw(ZZJD{FJoeX#dD2c1ux zsSWaA%+)Q+{ataAbn)krNSc-z%OvNbzK_{(?bQ_ppsl)i(lc!L?lpdlWR|0LV_TRy zPM5WAhFfy9bnbzq){PDORPmvbjg5`@Ke*XGKRrFc`pqpD&xd`XOCyR_brj)=D?0Ra zcPle?7y@<{=K$=gfBv8b*qhx+`^y&5z6Qs{s*CWG?^j;BY#BqMK!mqwf`IJM-Bf)1 z2e}l{fl$Iw0g-RtY9}A4J;~W;Wp!Ra?c!e5k&0s)6h|mJd`TzG1h2Xox7)dY!V}x4 z8@j=HBr8YIE#5-BOR^C405C28+)!_COk-V|_UPRAkJw#nd3}b;3~0N-J>LdBwyLy? z&YL8h*hg}lu}priEC4d@B|r$`lA!;_A+CGrjyth_oNvF5Erqn5DA^bYjGzW&Q5?8@ z4ZIiXXuLiBx|!<+I#lJ_DoRQM44XmNXN3c9VzYRbUeTcF)(Sg4%ioPiWP&MB*53qL z*kbD3gf-6XS2+5Q?5}~6+6?*)OobF$!6H2RV~C`Anik$SM!gNFj|3ut?#zM8j+ZdB zzg~bqSjR@Gj4M2PsTu9WI`9H0hF_gn^9n`vNnf&}uEnBCj*Yc0PQ&IeAuQMk1 zEBXvKI^7}`0BgrHel)Ab1NXQt0c);G$6@;{vyoIqfW782dr*cN%#m~Dcqe)H98VtO z$k*k9=%2Np*t(k0LAIEX z&`Y06Z`K5k4t*c`y@07MP!II=iGt+_Y_C(tYeH9q(kEmv5+K-?9* zYf$9pXB;nxD3j)|T_STCBm8Iux$OXvuM@!xhsioyZy8w;QQZ+d6SI6w#2^Hun!!@W zP@ry8xhE9`Pv8$!vutQuc#jm4%>w?gckQ|1tA+?=R{p7{7E_GIiEoE<_?zm4XOHy- zb@N(#gbtwsgujR4eHrz}0hDlL(ft67js#U{Ahv70IIquMWvog@EV$O$A-1zlk~PBg zx)PD0VcTvnWa!M1!-uhDswaoWCL|;rCrDCi361wVL*>B|^zzn5%JaDr*d`(+zTrWOv(mw9`;V~g-p|>i=&lchOHV+FiJW0 zPl(<*EpKi!IW%WGthAc#bM-;KlQnAT@OB$xV?~di+eQLuKi`*>EP+=`%OVz!G)f(^ z0T&W$YUWJcz7A{cwIz%A_({L{_~h)r&T}n{&&ucO&D|m_Qeg#Vx09$g5}krTvu0%p z=Bh>O_D}13N`l**v0N_C%ML5NLcFKAz^(8X1zIM?=ch(Re8Z82jvu}p4qQ8nr*ME; zfIJ@IK*Hce%@UL$UMS&nHm`y>f~rE>6*Olsw%AsmfIPR@oL^XFIM^_FGRLu#z(wQG&TIq z7|F*_$mhws>A;N<=&=D8Y_GP*q;g*d=T%b^#$Zn=>^DM#BgRaEkL)NaJjr2%S=Ri) zEPQ_@k@$R*sdm@4I`gn{tP@W-2|9k>eW{HIzRST*!&#sCvE^0Wds?)xGhCl2%Kl}2 z{A|;}mIJ960G)DK{_c>X+2Bi=^IJX<>4!&8x_UzBgf4ZfI8OkA(3Vf1G+WH`F8Vxu zoV6YcU_=}*0e&vKS2tnwG1Y?Ma`+duD)-*qj5LX%P21mhYSFd>J_p(f=q$tPE1!do zL~2_c_9}q`)9azFZEZmc-#<)NEowga5}i@cU_oKH_z3jth1(mVJ#o@mAFtH83l;=p zg2S@3Rq(5edrz7Ve_;>80lzyo`Bbj-n_%wy%}pLOve|fSI;!{zgM>P6zxK;yTnmk)zxugsHgGU!;QW(1UF2c z_*8k;`Y4ShygRM2o(vD=)G#m)WffYSBs+>$iU-L|zJw$C=rURo219l=ecbE); zWUQpXy39+l@VXMNgMDua&gA|#m6f7QNQ9m9u86@S39Ia;mKJsEh@xxn|nuRqi?i86*xO-7^Y4*e?+o-3exj zLjsIy*u)X7IHneGVD7Z;a`#Ur?<0!53!TaMCP`%3WU$vQt=n9zTuWHFwDbn?j<*E2 zcAhio}=azpl?WP`XU@wHM4PwTq+Fzx^!J0t>9!MKsbhtwZk-2Q%( zQ8>3=mdWLcJlG*d#KE`Uo&om|jI=A44(Ht%xwEgb!($naih4hw1v-KUx6n~B!{=Kt zRLv?y?qsUj1FJ(uOsJt)m-U@ik9p;6<@RgRe?*;3dbQMcOfYMhu~@~V+a=T9AIw(d z)zsELT@d}_zO;A;Qpdmf8#bb{tOH8InHB!)%oMM_%Z=^4M(~;*f+!gZXO1 z_*H@tt=youwB8Y`Y0L0=6x%`DF-ZFX838CBvG?6pG#|mib!hEg6 znR|gP&`WvcfhqOGW@UN7zk=JgS*&!XFvlDFo+l#V?vRPht9z3iN#D6;TEIEqQ2*A&_G^`rO$~uMuS_R(>UF@HgJpM8~ zvMd_@{S2Bb5apDdNyc-DGshiA-?*@^7pU1bzjt+-p`E&XX`7R954J9Jf~6GN6>TMX zHi#Lf-Td8e_lk6ENQc+EQJp!o%&z0QewN+%@dH`dV-6xxZ?&fp)A80&aQTAkJPLR~ zbSOeExRU8Ge>V)WjCPJI7w+jv9no|Ibwt^Am+K_&4TSvYATr3Whw8CJSUTZKfjyz( z1ZaU5bM2HjZ}#Sh3kd~a5f^@N=S3{goI7il@*_(lLqoCc>%JU^Yf^GLGKF#bpG*;# z@6kFG@UqhG#MB_!hWIABc{WVz`fS+M2}XeL*$~i6#rxN} z&HPx)HNcu)mxYb-nFJ5J5fl_Tq{{ZggooadAj5_9Pb%J7~fm^?+3xB64JvpK&y4U}la4r!8%f~I#tHXxo{K*1uq0o|rpKI3*g zkL66RVT9Y+fK~y3!aMPC0)J?1mt%Y~Osy7A4trWEs=>%5<--cX0)lo>#!w56^XhpF zvx40cZj6m+_x~KvWpW<=s8{~_^G4{OZE=fEq+L`y97@C2I_QYQ>cz(B49pZhltKy#pXU#a#k5rCV5YF zZE!@`_6n7#uD)BgUqxT2)_yg$1~-p7_BhH7bY9NZ2DXrvmX=5wP^ybLdcDYK)GcH4 zehHQ4yzG;GbIzByy=4Y`sGtpii+A_oCIeWGqZZ_S{}g~eoBkE}3KGs##HGXHz~l=H zi*tN(LhkPFsvRG^7hF8l`W9Fpk+hnTD8xiW0x^(8&crMA8w|=syKrqR9-SAC_a49= z;Cw(49K@IfH}kn1A82N^`+$s>IP!;#zx;=ck4b#^-^Ls8zU&jelCqdTmyk+tPY=5P zUXzt7mV9sU(g5`xpO#!yum1DQka?M;jQD_K$?;>`C&-N1$HZKVZRR4Wy~_#Wc*xV)KJec=tYw5 zh4jMDD7W|S?IZ({S2v8$4*lI$>?BA+f`WqJ4-OH3aAX~y+`ss+0X1mM;DGpI+U@Kk zL}tU|p*|B?6nHqmgWCV{3&8I^lQ`3yg3<@g{I^#W4-!tP9;~buO#kn{;cE^y`JA>%AjbE1kbG3bq7x_zLSY{I_|K|VU zsoeLkrWM7@!9eAxmK&QVaF|!t5GfZ?_gb(at*1cik)T#1h7fR5J%FehddjpA{F z6Aw!K?DFGE;$EpVE=4_i&k7=XqNSa=qWE^abADi7)ydMlIQ{({ZQ}1+$To_9?->j5jFT12u$tuGo!V4r79pjo_MaVh_M1VMdNx z?mhgBgzQNZCXnf;yC$v<3r&u)0kKbdF5&Hd;;-05)H9jn|HpZ2XOQ8hYv|td=J*fT zEzMv?pF`$@Q$z$ZPY)esB2Z9VhY!mUq97RhM-m@+G#rc#z3Pr8)`zYcJBB4$@P2%T zxMPruiY5O)uc&M*K(hs@^?&F-&<4Ma%UP9J)fn(Kt_|i2%@*=T5)sVODIq=e-J=L9qosOu!f6!Y%T9BeE#0nz(TtWS-^(S81LL>rgZq1?b3-L|N zd@&W_Wlqj1PI;6cZjS1<3#7>J`brp~$x+o53c&!w$rL7F7< zQf&X@S?2H2cA59_ZUSp0t3BH!F-J9NgELRmDVHzDT>gQqE@jRqGlyq1X$;$EMTL=# zbg1}vG_y-`L03Y=L8%Dw)nRGi_gNNp;5SWX9|X=;_cJjR!#EWN>hiCmc-7Nd8f1u# z%)q~9XViert*wlK6oV+V;n}giT(DF?K!EAJ%70o@bCqOKSzb>^js@a^$d=}RiS zCM4!g;15K;8sgVQk;&+lcIksa5o8FOADw|l#QL$|SiqQ`6gb9D!N(Xl_R->h#^%_0 zbj<~oqxuu=HVlzPhno&9WZd(|9Rtbr`1J9oT`Vbre}^T0lLUQ_|BwVFj_>azJp7M+ zgJrMKDs<)Jg9EKgc7+BX{My}388ydg!c4J11`qP7p021rc+2zQxlNJd@*X2c7&d(P zTpDjw+TYQNg`_f+GJ=0We(y>!4Cu>|h{?7`#UUpt86eRODYpigN43pXG*px zuK7YsE0^qYY?audrdE*|qXz(WK6MdIRWupa#IhtiqnCv?6!}0l!~nw#9a=O&V9(qj z1CV5AN-1D<+LsrY#(+Rb*pJJuW4EvRf2_Cm*?2DC3s!${e>-umOjNfkB3ah4zt8Ix ziJ2o=e8;ibJ{$kext$|+4I4@$>%Q!muC{$xjmNj}d>g~~T`}1&UYNn1jkvBT^Q%<@ z5*0tFD$w&`264|EH+A_#RJa-p+bA?{A+i?)Zd!tQ13i_6%kZrEmN4UhouNjvoSEfsuS@#tjD({+zi>s&%z{DwNL81bDYh@ZUda@oH93TK>4x7=H1=#PPGlPY{uv4+!7LEf4KxTasE{oE-mxbjwjwca*XbVc`H_GM$8}b%v<*VH! zvg$H0BLs!zziqdicyv@V^fQ7r4ldZ#-+dZc70Bl??>D2&CLWY)t;G)k{h)zfFdd`73X8A4!AdjRt#UFd&mBBR<-%mSzw4Z=+(P_r>qp&GRqeR(|rXnJwp_z&C#o z%2?nS_Qrav&RZ+K{LxsnpoTrKEuq0)BytS9j1F;zwH9>W$>!~P4~8XQOc z;#Vvhj?6)|DY)jGOYQ@gaUgth>`;@z7$>tEyd?RRxWt~6jLgN)Zm)>N5_vGX|N2%zh9;CJWNvtnp&(>9qnK~@dwGe zJRSS2EYmX+B^HbI&lK{puu&5>V_K)$9c^uQML$@X`rgEQ2oR3W1Te$Mqy>%LT`2ze zyTat%*P0Y&<0COKg?CuabV{wMe^RT$VQWMldgUyhXzM+#OqA-2O@a^g&4S9W$&Th^ zQGGF#Vkf}H5R`)80t-_2@qhKmvG+^DJpx*k8niE3-nh_b@YqcZG2k=*uV&`_-gN=b z)6Cwd9S0VvOQO{Jwa-mmC+8IrdJD)OP+^y#7n|VHWpUO=9WGMA#J>0VWQt44{LPrd zz<{aH7Vs+vQ+3j0ODQA^XfE)_05=+2P+4;2HuYW+qq0<08rd-(*G-Zz>(KOSA@~t* ze9fzBXeVN}9XN34EJ{DnO4pfMJ-CgLxrK#=>(S=YD5s4deFMjJ+CY0MP5+{fF0^U* ztI@9W`BUf4{KJTHvMmyo9RZ0cUg-TOQ5eG2#M1-q`5>Hm>|!KiEVb9E9KdM|x=|e3g>T*X z2SI*O|GfipJ=;iIdnw~qp`I^rk@k&xpD-xj3}3K$0m5NqT=wuBL)wT+f3n7e?P7Ru zdbvc$ycAfsbS%qRM=2`#^h~wRjl}cy_V%ht&v`?&TH9}^my|^P^SldpV*uRu6;L>c z=)<&%M>@A|-wra|wJZ3d2+lG*djgOma}0sORNH!7HDq!sYQHQ`8k^9kqOLOT@G4SA&LnztGH-31j$YXeal+(}!p<}vkD z(sAUPy^kCoXiKUGrN_ADX9rKk$*hJ4h1R=Y?7EztC>d!WeDA}?EOHD8Tdli!|-#I9I5X{w? zyPP(EIH&;hd`erjT97j(woF3dB{8{|XY{dg;@OLP{}#muR*3f&9}1trc$Ghzdl2qF z3WIz>uWSIuB?=z`%~Ey`$I=OYp&c}RVaXL){LZ>XLjyQO;W#7*gn;Vgu0spa7e>uJ zC~S}saPCm-3dy0b8>EraOTYO(N|<${uUhn-2W!IWCX2VVU-pHajn$&`nPm50B4 z3|uf1zpi&ChHO!5Id%Avq@x5SVIk=`LE)9Q!6^SBoj7d8Plf@-F#XDCz03uhy;5Mk z`~a~ln2)3FihxSlBk#$H3EVY&D#1hVVm@?o6^VP+WI14X$uNd##yjHG_m`^Ht*>Li zpAj$|o@GU0BbB2f%NtT9s?ljCl<3kV0Y{1nrj_7RHoIe&^CImEtX~^4h7ZLodBU>> zOG-KOIxK_Q(7uKpAO2e+;O`r=;kC#0tDvJ22?Mj=O#P!hgf7 zcH^+U!bn@G4};=evPV;JN-_5`n73+k1#&P(1u&5Bb0VUf0g;bYST-@n^<8(H>{blA zv%IH+L9H8@lNy8N$v8lR5g?_|D2_msZ)j8FZ=^cy&+!%hlSR7UwTPA=&^&mrm)C17 zFFDy1ToV~u9G+u5QQ$Cj|TFu$IEjFhzT1AHG@gPoJndZgqB5f0^nLCZB0P@ zVnJy9IIvp`(PHrd&NC)Xe(k6h$Am1=>|R($p^I}I4DdZGj0p3UmX@kvK2WsuE$1qi z&5TPM6EMXm%XRxR80H&P+dyVa8$4gSDF_X4L2(Y6=cqSm2c)Z9 zrnjiVHfhM>(f1#!jn4VMnX2z*znLm8z6{Rr7|{fUZ)1k>j}cv-;2R&M=*+X%*zi3v zFvw$a7bIVeic->!Gl=QV-=KD0|8IuiSV-{qkYE=3eP!X){6^qpRxeG)z*HI<>{<2& z2kQ}fB_WDyI057O&!G%TT3>#^wW@G0v7OeAT~CEkC>C$#5^4=TNumPm%j)o&yW!WZ(Tg zGg>fzM(@Gl3A4ZY-#`ctqM<pce2~TQ6RNEti4=> zyOl1(t&N*6SW;*mjVysY8B@bmh@!uTV2e4PSby!fV*pJ3%orPvB|P0+P8ls3qek_u z$SQN&`b!U&7y`!S)i9QXJ{%|kPuBO0dh_+G6FTdy|F(|)OKtV&BUiJtvwMHePJX-n z+OV_|;#vf#;sw9gx1t~~dIOHGA2OljE;_n&djWcRnBqk_f?OHWbHd_qmF_wW4*=V| zi^vwg>_|dCSz8Pq+5ZcQfcoK&=oOjdXB$1TH+eFkhP|sSjsy2MVys)v=0Aw%zk@{ut$Ka6jcLHugI#z{pBW#lmx>U8zA(`VuT>K!f z#2k=~@nh7KV`??Sa>eQ;s-pBj;>&=SIe;7m6-h?uE_drqtu}W;eUvly?}9hxM!0H7 z(I{5TGgP{Lr+zbg9;V~@@$2#OZ+xr0`gv`Q`a-YoH3xLew~i9OQ!fjRt;+!%XTy_q zSESLE$8%}3Eu&CU(xuaBV)(VTci^?t{w|hsmkYjT)*$=XKaRcb_eS*K-1W|DOHIf=M1YJC^xYsO)RtrN3H{dP~ZP)6)LT41i{5G>`3JDZS zW(?@ifu*AQ5K;y8*K7~TECrFm(5q-n>MYUNeot`{g2e``nb3#yEC3*QUfrZ{Wd==R zDK6g0&UC0(rI8_hj)m|!m&Uix+zD*Bs>QHk9*M(=!O&<8zupO#)V5D9kjV73){=D8n zpc{v3CAAYs?KHytM+}>Sw<7qWDMpR3E$oqDL(almSptiJ=mZBK0QJq0J7G;$>x1^E z*g!yf$P{#c6*dalHAFw8YOEWK4 zRT;87dR*(5DfIa;3KAa8Rmk{;o`XHdyBc7nvX7YOrA;e1`S%JcDn0va4Uxt^8SJ=& zfhR(B7DrVNpZs40Jp)@{ zFG;|G+G^E}#7QltOoz*(2=vvfA=7Y<)ScfPh5o!YT_Tw=)aC&`8q4bzO__%Bc&le^ zDRmaDvaQ%bF7NZOwlLZx>psqEU-#?n%yi8M298QJYPd7Wb0BG)r^U^iH>qk1?}db} zxQUqkdPGgtj%(h00r-%x0zK?TLrll6iElZqb{qQTG zcBK14b$BqZ)H+dBS2q+zAhU2C_+*l6*2Hvc$Lg`UiV#dZd1xIAthy+w-)ZaR_%}mZ ziHU{(!;rS=UN7GJg!2vv`#~dqQPwJy5V$`;n9CE)nnDv_+`fOwrZFxC;jS$)X{leK zd-><0Gg`2C@4=LZIwE>;YZhgaSr2#0O#Q*50Yfx1%$A0T;pzG>pT+aC+%7Yg88a_i z-FFV4`znTtcp3aAN?AsKX+lzzc@yvIP#pBxJ;8U@t5GM2D8R`Rgd@B(#5>5z**Ooo zLAZqSI2>d^mS*(S6LibkLfDTP3%rS9-e!r24ftLa)QLqm9>{g2pVh>e7tE7Wg@EQ@ zM2JP)FfC*widO!{M6v~$m|Q~xmz8QHM!aa!6rg4J##wT(3@u$3F-W48i%cWV^h?g3 zDdcRAl0!^GNd+0aUiSc$WVk%^5>R62?Tb&3?nWF#v&JdQJRE9C<-f*9tHI=rN@EN% z>w+Jj$IvO_h{Iuk%_sjM9hrs|hPX`am1OxMXUFrzZLHbHkZzfB7e9cPTfy&h3*52G zfM2+B5_RXs0Q@D~eS~VmY*rU<(XYk3gK4A~ix9|d3|FQB1iz;7Nlb;-sVWD`_-@ql z>C}55r2|x#uL-k!!jjEOX!l-flf6njOoqGp3jQy3kp1o3;U`k^C@H6TFaB8-KNkMm z0*(~6fC^O-&wQq;#=|HL-RD6a6`$7erNSsp^b+tjY6{Z}X&^?DQ=6V>X&-TSm<-R9 zcD$R_{l&3rOy;;8tWGe`>Hk`TqOvi)_t!-Yd@glU&iZtN4j$L|9}MC6PHWS_{xWBt zO5Pd1;YoYABfaEB*WjP+nQ=qLAVM*qd|$4^r+3^LrDT%KYXybN3f&JDidq%eNf`Ku z0ST_d7mCvv?}Xz;NDeZn!jCV0Y?>ZI!g(s%`GX7*B z_}ci?zdf(oyUZ2p4HLRl+ab6z~z-Y~7`8%T2#m$T~2SIH{)X6;dkK{E& zRH?-ic(5R?BF2qT9o4nwzdr8zafcf2i@M|zvbGntksR^=I~OR*5SIPdH?|9>!?&Y? zImyvV{l`(*@%i?Gt%-hwvWkXKGApD1b`B;<(NzLP-_KBC&1v-&&)CsIcferA{d-69 z1`udulrbngd+y!kC7SdCjb5R-+%`$&=BUo*OkG&&j@#UrX0G4UTLF`}s|WQQdG}E0 zl`EQzAt5wk7{4iq7Dch&hANiSAErgJ$aKQ#sGPzbb6I7vIkxBLCvK$NflMfnRZw1}d`KbC_**O){pXd>@NR&XYDpaM{B>m5L48p!wB?Oe^lwHl;-`OD__iooBe z=VxcQ);;NY3BHH(Abq#=J)JeyM?|<_GQ3WY?ZsbiK_*&z7HBbUnFz; z)Klvz-USY&nQLuk2qx&WqiQ;SnkK1Us|A~>C{hH<bd;=W*JCd`tTvMP%ffnHVDd?ty064q|eM7-hEws>g! zFq~a1KCSaKGt~f8hB93bavwib^O&w6B2f#9e1iLc6C!P1oME~U%}DJ$vaPN#VkVz! z(?W)SY!{t2K9@VG2cET#s&s&K#&e`7_G*QZLUk{_BfZM9em}6Q8!9Bs4oiw1!`%lb zO(kdfG|9+J4=&zo@bSel1ZxKE|fbI+eU46Kf730;wiu`i2yVs$~% zTkDRBhoY_f361ytPaXEHyHrv%%In}w;fHzs<8QRj%CSeOG<)=Z=FGTJsm?&|?Ak?( z&=XS+KAGm-7UcT#G?-U3d`QIZ>;hRy6JIfEduwk>N(zS2EE33u2)$+2G%I;;eELV1 zF^I=L_Hk=Z^vYx9ZQGjA#|{V2)_us<{vJ#`QdfU+osQgVya8|c0vl3mHf54Y);X2t zu=?Ags_6VN&>_s|sA%AhR&qug8;Or}h*^qCxY_^!R#gAu*H--nZlC{s;<5+lbbncb zKrDs$&&I^u(ZG<0Xv|~Umlpgp*gLUN*)nF99$0Q^x=Xk+ck?rlnJ@p%`n*>4%WXvOUQfuGOR zJH7O+pG`dfA?6X!_U9e_ndKsj>I-EjKh;`oqu!pNayD=2n-fm=^c%k#UxCA~LqzO+ zW;0LW;x(+Oh>c2LMTDkIm^`|6SbpBRf(c7ezW8Bzbzw(Hcy?LG!ywBqV+W*m4^}qsFQKVfVOqqI( zL4?b3Jgo|VxH$1N`wYa&b18U5_-`->55VgVC=ZP}gLHCk961V%l2u8UfMNh7N2$vd)L5L%Uz>o@I7Qr{hkkHW3@WA4cL;Y!nHpa{#jOFI<k)S>Se+49}Q{ z5xtDZK@aeWsmtKIW@lp0;V!Cdu9lIB&ijvXr3TRMV~|zz`H)r3Zo_YDsgPIk`^bbA z{ysAPB1Q_I-b(QWg=hZU_fu(0`;QzQTc6pB20w&!dz-3O3(;&A@XcyA*LGX`eo~k6 z_5O(e?X(Y4lkc!BUhDPg%Z?y5Yu`ts{u?7DUOW$E5BmS{S>ahyjI%)78(+=qn`}HxOOzvWzdZXaRaorQ;cBawpSrEalHXn zY#^+dP`tmf^?rmv-+(UO6ECkVFPye<|7aHbAvffp-?rSIV!`CF&686LHW?_nF5APA zopRy6gXQd&(#g-k`{#WO7_N(pH_RuM%PG{}8?~`hOJdQ~or~qJ4KwCD{b`2F{(sLS zX7M6O(^Pwjj6P=rbP!PAEIRUY1OQ$AKYrZ9+{y~ZF^`(ba9S|Q1XEG-eY0_XM|AOC zMbd@eRo&1qyxTOiXfhZ{o!GZg3&ubV4d)UUyVNH6WiLia91>TM7lR77ot}ZZ?7T^g z@IjhEOfUQ(%M)?10Qk3BNp7GL5`-<4L0+dY1B=EZbXNI+`X)MNjz(oMNa+&n7Bu{D zXC8+G_K-`C_(nF!HqmUq>{{Gd8=v#-+ar$WN1tV8#n(4l<21U3JMlm{}6;YXa<(D9{d~R3}7SMiPaNM@eN96`?1J@pXJt{l-xywB5FF{o< zvussVjs|+P9$g(=DdZj%k22@7m)|&sho(PC zP3KNwu+IH}`$*oWT_H0{IGIrl@9jD9t}X2(Tr&=@sj}jH#1(+J1mV@xhxT{c5K@eI6s1b;4-zv&dW=Kn+`Hm)(2n14(sj1~n~ z$+Bq2q36z>wkemv%mp|YW}>JjUM}>rWe{u`xqcqqi|>``xZx8?lIRVOBc?K~T$xMV zNCLyX`rzU5+V7Cm*Z;{<q?veoXFuR0Q@*|Mv>Z*tZ z5?7}ojVNKn4#a8pF4TDfzApDnmrvsaU{Rv7X(ZY|&A)~|s&}Ao_QGE2wPdn#0r@1o zl$jF8^Aw$y^dP?fEF0+PcvvU}>oJRe*xljR$H$Dn70{wNTg}N#_q6X4{fD(WK4GiF zuWV^pSJdlPRPFe7ZR?r~`CC3|%9g}A_bq+u=!u`kt1d#ad}u#-+0B`y)?#pQOXGFZ7QQ*VILU*~{_d-41uM^ZCh`~7 z2CQFMf5}KIF8tbov-c)VQRM4%F?gSR_0riC`%)-UhrWON;6j5#IZDi*AmPKz- z<1eEQJ3ScEpKFGQL=;(;+|$Wd=S4-Fgd`8Qirh};Lx<|?)_pl#adXhab6X*v@M*Pv z$MLA%QF80$@{S#O1|_OaE({s(lL-jKJdhg6>(w-NUhL zrM`u8&dwplKuAbPgwh(j9s2uQa8t#Yx$BL722nf@BO{|1??0eQkTtmupQHox}6H+-)1Imo@UuJ%xhhp zW2$u0tu66a78uL>ZcMsVlo0jyw!pA>>vX>IFZ7h~Qc(|WEascb(rjWu- zwd4Jk+Wi#R5)-v}_pDi^CypK{-^r@cmy=9VU0E;k>Urhu`1n;VWL#Gl1RzrLVX>ay z#H>5qG9GqwJ6cZA?w?t0w_^Iz#e+Xn%_L)#7F|n_y!a8HyaP{YCm;qqW21&2y%WC7 zee9G{EI`dRm@p#4SD2nm9GLjn_|?tL(0*lPUNp4gdcqjYy@;H2V*UO_Ao5PX^_jX1 zZ4hSN@a*pw+)C4nHlc6whM7OGoni`l>RX_ut#k#Eq={MQY^7)nj*NuILIbYtFt|Wi z0__+a1#}|i3S3ao=b{l4_@gs8bMN89@0t{#-&|7g^e8NX`016}xcb~AzIA)MVLy8S zK&R%jV?Wlu`p4H*P<-DuVKiMUI#KK7C4nmTVcPCm^f?skr#i)gleGZQGc!6{8o9Ya zySc!3L()$P)|PvRUlx1s+H8}ZwaU#=KJTd7(_8VM_gZD$z_V!)UYBM^AsVlJA$O33 z=_MSO&D;O^<=r2>F_-)O&YhBPpTG04v=#0txXKcZ6D_^9Hll6TaA**f8xkxb9i%mQ3Kpwq){rVGiAy&?OgxIjI+x1F*6rvTb__lb4J-XH_Hd%Pxr3dpTuy!or z3)N>|a`3|*w{Lzb_uAWYI7W{*U%qSz+74a2_ot4&pslSPjr|dck@3vtbQX4BzN7!u z)&0uVUb!8;mV>Aeu0D!V7o;tp-fcNcL)3jN294dLzQH<8IEQxPGA}a&?KgVTo-u(-TkXv@z~!#w7DFnnDh!g*-)7xOW(cQ%UlEE z1G-KMhRhp~Ag_cwVz{j5kFJRN)rCq$^$yLLx~PTm(c6j;%e;=1;(|-0m{~MD`em(2 zwa}I-3<2l{KRZVb`KU4c$Tm!uOQPAQHH8ln6Pr;AZL-7F<7+ZY$FpSK9F}u6vkhvS z#qwHIEPLtIkznth%xx8~78GtggylReX-?Of(Cc(axfAzKtZ6qgijP;d%;x-DdqBg^ zaGrW;&7&*jp16BK_92pl)*<=w6+1?9ypNzARt>xN`F;qsxYx{R)7I*d(@#9_=$(5= zWX64JBCJ4Dh*=40j8`=^#@a>aJ#73vD5dzfLo`N%@G5c}UgWq5aF?5)R^7SU)h$g; z5@g1|NY!1PHfCoqv4z??lP<^ZipoEta8xmC8wji&36mDc-$DHt*$bm!>{0-)#m`l&sG7JYN8Z+?TCwuJ;^v-jJvdtyy!nj?>`xl3Io?75v*^(@<^$b zE`dm@autsJN%}K7EgmvNWFguU_x}7bxQYCEXsLLBx-hyB0rp~vt4%ic#+1#pSXF7? z-nfM)xt=Upb?GklDD&EQYPSRTPyQ=&10yw7)w=d=QOdOSYijBVd7*x;O$X0NQ=|Ru z)N9s&tK~bHB}O~B9$drAE#R<*aJd2(qtDk63Q7nZdm%{v6aNffUB*+a4OR%Uf3eyE zK55ljA(sL5&snn@L-G{1EywBP?fkvkAXQSQ5tORue zIA5nHb+fY%NVX4R$P}RzMU^@8=9PfTFz#;1553@Ay03ZP8Kh}8;y#NVi_1OG1|#j7 zoZ<8x^AQfA2P!fNbNMhSM&L#xXv(ZQVn$FaZL&l2o|$|s`jI3U z-mDf*!4HNx&f9F4_T?nB6y_foBsJVCrq&_3`|wS@mGu>9eKV`pd0AZCx?(Uz?s{;M z(!mxB0~L#}+?C~5JDkD|9$qTG=LC8_1epxRUs%=s|FQMu0Xe5#*bfGSOfxi8qQ)B9 z7p1fy4=QOPg$f~x(k9Y2LrkR4Sy>YHJbFZfQZKO{F|ZkCyK`ziMXQ@B4m# zdEd9Up6CAE_qorx&UIa9XHA5}%w(2Swi$wA!XuVE)VgjlfOy;BS7&!J!f% zR9#xDp~>OcF)4EceZF7*CvOA@3zqF34h3juUye^O=5ibQAp)&V=AjR2h35E~fNs0iJ+kRw}G${u|F3S3(Igi%?D?2t#+n7`Ow72z`JmqR`1I11b1n!ta>=Gq{Q<{9UC-EMFB=QQ0@M|++A=KHB*Ff*g z3U~6ASra@|z9y8U9XK9CA$pcnz%kC@6Wd09G;JAbe&lhR$FG5wk)C|t@ATAG z$Z@83TfapV_r=w?ml}mlwZm3e(+zPCc87GVJW{BB5zV}7tL33-!n#ZUhG&VPHyQ{jF^kPW0Ai~sDh8)|t-|%T z8;U(-@$&x$XC0ZVm5@(((|C>TyLMSE*Bb^X!!|(zclXbi_}~pqioRGJ3l8EFu@$wv za!tTV*8?_qa1E(})w>`B+KJ6S8$eU}HF+yLJBdPDPI}(Z7BD*5BVb!RGx9?w;4v70 ziigQdP={9+?PX@`2AR3N`L67>7#E}td^7%z*@i#r^wL%+2U0?aQP}P<-n}OaW@tBgnNPUK4?CfuBXB&eBUt@Fsk! zk+DNHyfhAabc&bnO`nIU>@rQRX^r9i?ZFN&jdflck6L5QsB`5~0hJXRJl>|0PO@EJ($ZQM z^#HxHG7^AnGch^3yewVIY!@=?rjIH}n?07(%2(n}GeniF^Rf>R9~t+tZGgi$0}QaY zQ^jA{zZI5W5FH)n+_!0XsiIPQ7j8OOYh%he-QgVgHs_SWh&7a?e;q?-nW=V?OmlXE zN=4h+)w*WK50f2Ndis@+$f2~a5q`{`Zq+Mc>X%_!D?US=QrFfPX`PZ{!P^n-( z6qA)G=9wB_60!PIX}Ap&tL;ATGMO7ay1WKl`)R-~5r&L3zsiFzoLpnYDL=Ozz4u2| z_4U@3@b0_(?!{Z=g1Wnr>LSnbCfBO5>IJq&(jcDfVbZ+iP*#(;xhw<~X;)UaDt2{f zqoRP-1ze1yHKf=queiH+D_Y?z*wnNeYR*dd8dz`;iuy2O;>&L2$RednW=*5vmTViq zF*`vwkI@iyAj5gk`sJjg_Ru`7Ck?n7)gOX)S2;ZN!w0_hmydVV9!Blh!Dk z8mwMXy&W58ajjY+hC3ZJ3c2Q*O8=~Jk$Ivajv1bOapdHe6!vk|ZdzM*KmDVx@Y4N7 z6IhzBC-wbqnb=o9d^%sn^3WlvvC^G(+EWSFiV7s4*yE)-{I#;Q(&K1Ka{0NR9G#pj znlsWLNNmcZ^znNvC}&Y&h&<4z#K@Te#+Z%(BI#-*kC&~DDFBV8o2cj}bUCvEz6b#6 z1arX6YS6zn_`46=M`lF?sLMOgV55uf#r9>R z#f^dLfqh>=4s9jmXs4sdw!<<4#V&Xld*S#f=b|x=~OHKg`SoaaP zadT&mV)Tf8d#p~~-xi5u-2m3&;-lkD=-vyBib}%sEApH)IL`h43sD|k#PPJ=oV0`Y z<);|=CPT_TmR;J6E2xgLkJMdY8x*jk+d`rij-bUArPJVo&)<~C`C~1zCxP{TT^qQL ze-<|Gy@aU!B@~ohy_;(3Da&2?9l0r%*tNk#RNvO#(RTx00hF|2HZCmFncNO`Q!vnRh*jq_^#PH?mh!T-fo2nVtn6zcn--}tr0E`vPKn0t_Rgc`Du zoG*`NV6Nbm0DP(1(8fZ)4v^T{Hht|?me$tykxBPfTzPYO=k8tBAR9(J*-)VGoODWe zr2e-u&p)MC`Uk@WL zuSsuI)M3}7_+%>XX;g8y1KPDj9cipm2pE@xs4bX-k&7xnKHs&=dh->(N2CFGjB9W4 zGPVs&@M}vSo1h>e)-qFlVm>1W^VM&~#un(yV8drMD*6&x#al%x?8L`u+YJqOBB>sD z(PC>=8o6^o1tlB`-Nk)8OE~3MmFv>;k--4or#S+Wp!tt5{Y2Zo}gv?h002n{SnUQzu+8q^?yLm-#NJF zR`tD`8g}2y$~skEsGWykY>hwiJ(k>uu4)Sc(U`L;hW9UTjfIAGbW}l4u6!MwdC&ny zlA|?t0dfzfQih5P1g*}Wo|(7xp8{x%4hs`*H`qW3Gu~Z*EkSBv>o=5;Oum948l!if zB{~rB#FQi4^m57NoQM9%7MzLuqZ#8p%!ZHeS*!O>?`)xxvsUCqXIs2)63uf8}4{`|HxSg0-=`A4309m42AWC?a z(SZ8xTH^c$TcM#E_n{pHY3-O18;3^Sfw8s5oE$=}T8XreQo85 zO6s%C7=me@s!?^5%w+Dw5-7`v3M7;j#%Pde(`kIWi; z%3Kq?0o;DcDdPM*f{y5N0n?#9xTjpGP>-Naf;JeQ{U_<#i ziJJ|)`0Yhks26Z)tEx2h0xFqp^+1-2?mBpx2bCNcy#Otb9+fAvp*ZxwtqRo~T6GKT zE8%qlTFrq@B#W$~B&@AiV@@vei(Q(h*EynP&2-hEFY3r_39CR7DHel2m~m<>v&dk( zq4B-V*!VD@QogX6vO_y%RqXTDvRFsvZC0d}lSED8w0Q4$NRIXYcNhDVX=vK?@wBNa zEqw53;U6sZu${Ulw+laQzq@&5BJOSjyieSB({bOe+P-aD9Pr`R7@I5S2b=qU$U3*p zqu4dL1BmF!ub%*R-dN|fa)IBxw9-QMA{@1M==4V2PEJk_Y^sLyh)h_x#=_qS2Fi~% z0iX)P8e4McB$8M48_F>1pE7l-Pd#dBY`g#9N{9fptA96Yw z`Qe@WG!|=y1n)9V`264Iw+iTsUA!}&P+dF*5ud*a@{{w-4ne~&t`pbKLmI}SI zMKCqIyJ8LZL!{uz+JT&SvGYLy83188fFkcjlgQA76ORSHlm!E;&^0IRB6CrLJ^FC0 zer6-Q&9l>YN zM8|0;ZN;tFj-fW`l-`9mDxx_NOXSeeW~QZs1E+)6rT3c4E;rGt-USO-A|r|cx!DUU z+BWTO2selU)SS8dIW-^fh#DTaPagu0h|`q9fVT~@(O)Mu-jAYWc%o^#TehM|Ug1&Y z>6*KdS;Lo7M(-x?bIca=~W&^`};V>*)|ABarD|v%z%S%7z9t{G}vh zfu4(8e|CTs9+mp6Ok3DsV5)Le>(Tn`H~QBe{pzLR4g2Mt)Kxye1DP9B&jIUS%kwid zs;h``4CKCq>E@IpyTE7lu7}(7cEeQzoq_WU%Lp>Hbw2_?3El1DXaXQT;-!&tVcHyj zj=WX%At1C@X5Dy&u`F#gdA(7QP4t*tQ0fjbU6~wPK&&9`;P&zvCh8E5oK`%XagoTi zfeA@&#ugSWPI+qK{04(}YvekL%~mK~x%R*xd)Bk;Oq7D;FYzj~YFKF?53+Ac+@0q!Epw)BGwHfNe{vB&_sv+wxNS5a zXJbS~u|_vewlj+1=AJJyTXQk@2T9)SK7|n!RH8x%7ndL-TG3ZEMFhVerw!$Do63C2 zp=OT(-J~C+9 zP)yf4JbgX)AQw$#rlD|;h8fzLwPP1Z{4FmJ4lEn~VNG_Kqeo>e{tE%C%Yf7F*nP&b z$K~YIwc>cHBO@<;)`M3oB~KdOC9}M|K$~s@dUYrGXU`_QM($9zyUn2T;?1~rs?q-) zy%Q*M6=ig1Lf({Ql=?`2A*)Cj=mM@GYRJK^(GhMsS=`}!n&Dx)gkF_cEE3!-)ekao z-b-HZjs~5LoTAzlNw>bHb|EO^i6{zuGnUAEd=?bavUD%?$4|cVoyf{RdO0gbSp4?( z&<1nm=^~qN5pYF0;>AsIiQa{l=poWfd1$+3^muuBJAexDY+S)=JN+Bnm}yV%kwG3p z=_f`+XmcK9Wu-Z^m7h{F<;t)9gU0{||2Wg7m$$kC5GF>hLBdtkk^72DQ~q*=tGCg2 z#jj+L*rHr$L-z0f3V-(HlHKV|7gAHDeSCcdL8sb7aZQwsL*|U`qE9`lvU}EbO3HtB z=@V{<>0E?bvNVZzq*r=PUJG?n7{?Z~`XV7{<0r!=xG7n-rXC;x z04$5i_6%abMEuF91}Lfv7o76Gk%N_c5BkMn6lv__J9qRkfDBz0)@RMdvtYqo<~d_- zC8m=A3MsO^eoD0-4diVt{jD1>SoLMy)$_S|hkp*sL+rw``8g+TH+5djI4%)qxMj7h z2)z;<_sN&|Mz5ChX8;S>F1qc64rW+(|AyO0U4Kv4;(mCv^Tlnk^!by6biD{80x2ab_425AuvGP51I$8Ka@bldX& z>p<6l5x!OiUFkaRIu|8T8tFl&sJxc0a|Z_p{ysng{1Ft7o?#}N&)b%-hanigQqFf4 zll01PgbakSihN0EPbipYbYQ;1^pClfuwJvS&De`V-=6_;VwNrN>wp6)+Xh?pQtlP+ zr~90nFYgG{ylSv0{NyJA$P{$iUt<}9cM4;+bU<;F1`N*A96JmzcBy7WTjOrsdWI%)@J_4_#b+qC=zJi|;b_R-7knag719Qw?&zOvQ9;pi=P`L)Ggc}$OPUWAr=ewQ(7xsK~ znF%4dR0RtpXk|}C=7Js!=hUXY`k|qO6Kkx|X>%_hK%QX>g}QvZi|c}ms^o)PWGl2P zYyvC70~;~^=FP%?1C}d70&%xU172$MZh`?dol(7_?aoBB^Dj`-R=#~}Nlc;yWOi?+ zIRv3w<&9Sw4+6nxZ^HotI(+>6wvyc+4ZEq79=%i!W^a_(jM_Q>i7p*2zO#r>2iGjB zPuY7KTP9Rpl%GIJc%sNNprpj^IGVF&o&Js~giks3Hf?573=BFwgj5XxuzDBr5(>Kz z^@oOrqBwu{M%4o!UST*lBc(cF*}5v{oKvF9(5Dm$`M7x_wV5~W*Y-OHl&L^Pmw7!N z>kJPf_%qMamXEJ8OuGdu-=BC!&_&nl+mowy8W-+4*mM*1vAKCB0C-6r09qt#LkE7x z!i)Zk0WMJG2uSA6b)JT&-3rNv;n~bYvOBz1*dpfa?KCA}hW_6Pat%;Ap@P>w_xXn`1!@EvSX&M$#SD z){?Hj%0A}em)uwMje@LKKqa{*0Y<=2^dqw{2}b&!--6Ngr+ucl|;XJo%MBXsW5EYfk?@k+wCX?rstTSQ8N-9Jh~D3$UGlKalg0!E%Xj2CS!*3 zryYko9nQ)ua)-MiYz)jJ0b9(eK& z7$+lXOw{PRfIkJhw7bO=_(qWftcA?iJG3QO@D1SBW@88Ye(jr1(pVgW3-MT@y6>Nq ze%_4CXxeGO%j#vD(1O80wX$8H&!RAi(ePX}%Zp!rBm_n55e$4HgA#9&14e_CtL4>Y z`ajo8DFA1hTN8zWz%udXrWljSz#lLGaK>oN=AIKv9gkogw3D!_JCM7uZD6NxD?UCI zRf@>PO|5`CPD3PP$+37Ie1qJ{&AYK{Jl27e@U5#ZfeC78n13B!<`D_A)>~kqjh_}| z{k=Q@WE&oQI^4;Vr*FqGEd9cFb0UmbczkbiaN!)iY7+mWV`bBEpq#jO@#LHA>sM#~ zi$-p+ID9TpXC^7orLM1f{s%B&ogTL zspozAuH^ zzI>MeQoxm`u=Pc-^m;n7fvsJXm4c1@Zx`H|a^eRV4UHT)tECZmT@J|PnoBRSM=;)d zLjM!|ndz@URuCz)UD}R;YM>!J-M6qQYugHV8n~h>V8T}W_2*WCN-;(*bPs<1k^|xw zdXR5hk$|nk4NtU=rG;p+Iyq4egq*k%tWuv-tW`&9GkTQWbFJB)`o5fRF&~x#9LaA| z3{ZWb6nWuvqw}3qc2%=A0sZtGo0}1WY!i>}9;K-ToQ`iH|MNp*93!F~TLn7+ zGlr<=5<|9anBFAs+!hkQFfW>Srr*c{`9FNZ^0Vw@-263o{^06f=#Hdex z8j)TEmtq__P{AlvnoL6j3mbv-bzvpuQKSRAh>3Kf;=YahvklUn``S8&;T%IYrlGD1TYYX1G zR{yIEw>me2&`IW06VmfuBID>-b)O%)d9|-oSFA81wDA0TC}dEj?i=H&e%PE> z2uG*S)WqEo1&?)xUW_2HGy6npj7EPfb!O(m>McD){0Dwr>^h`B$ek#&&1`8T;i=Ne z-srLrWvAQdvwt|Hx)eY4`6T7gNCMjaoVp-o_LP&QXpHw&?*gnV%T*2EO7eULOF+!e zZ|a0m6ZLnyQL+>Fn_j?bWYC!Xkv{)1W|o~*u2(qp`1$nC;L`Ai=$p!buu%AQ646hE z&IxS7AH?J7ksgXKfoHnpU-M6l4O zm(jd8b%ZA(jP!yXRSu-zn=I*>?DS1PD0KpeR1Dh}LFeKo{GWi-G1Vin&wDF5MK{WKYFYm|I=yAefQ#o ze%|WYRRatWo0YNBI*79^j%{^Q_Kbk%O=N#@V{LPNSy!&Zi*U6D0D`y94}SPh_v_9Q zSLmQpZoI>wJ9Jg;ZFs-|AmSGq17%w7t1j9gz*F(SP)6!#@@j$O#_?9}8%NxI-XHA8 znqMoNJ{tBr{q^vDKRL+@H335cp_3C;7FLHQIfb4J$L5bDGUbCQ-{}zfa z__p3YrBuB#IXe>Pj)QS|hr3SWxA4Wm6I;*`{Vj&qJ+rrttT}qRy#d6L9Hnajc1wS) zTC*k*Bl-lHDiCAy`W+;HXZ0fesKG%#QW%_KS+xS7Xl-AA^~*0E#ky&=gBN*NqEanW z3;IH2_3FHr*--XRkJ01XN~8ZenFxBw$eZ8v+`Ts_V%L9k%;H3CNZYB`a(DjZjAd=> z)yLPjvb=m>B^niFU=4*zeb;fzYluU}DO8wBSoHq&+=2jPdio15W;Aw#eiNmWXNo`J zFB7he(Pr7g&x2J1myPKTx^%{9r=kQ~BVhQOz*}e)@kXI(H`X*bfy0j;rQ3i@;t+m$ z|H1JF4eG+EL;&w2*EDj`sq~tlPf#G$Nt2VJ_M~iM~#pJ@s#&5i_V!&%_K2Ri!?o3R0pP z^cM~(pIv+y{>=S@b(bC8h7Rc^J-hnrbS(E!YV`2kMUl53N>h`Jo-g7j21ozpe-33| z!w+^J0?YX&0+ut;J-K{9Vrc*2q&d~jM>kIBqA>(|a9p)=ajB)wdAu<)JRm8OL0*RutO`N;(+Nk05VQE>+!!3B_h9&R1Wz}4&VfBI6qvO zRhLEWnlLr@T!svRZx;h?(rB)aA|gxmCHOQ%z30zaZU3ni;UZLf4SvdqVWZx@b*tR9 zzYbma-ZzMfec(fhN99@7NPuPI+&&2>Vt*N4rXxzaDEP(M5!sKxBkDp}0Y)^GgT3pqJUYqTf=*9kIe(1Mo#)?0-Vqz{Dl&GFlwwSu_1rN7Q8 zoZ*(ek^aA84DB8L3F_a#+EwYRGyo;yMj8*`t>@3elkTo8>HAn7G0~LmIgvH}!lJg& zlE-grWTQ3C$5W+^XW-1uF}UvJ;w##1TU(|&B!;i8#-^1`coa1^X!7!E^cot!ice3v30SQao1O8AO(xATM4M?jEw zhrwoNc%eayac{E)-dlbp`YTg;7sxq-bow^KHYx@-n>jf-TsS!g_4NNq_DqLOo0{g zMX1BC1B=P06?>mJnSZfZW&(m++ng+-9w>s)hbz_T=vv22IwS3N3sfRb zfN2C$&6LN80_CXgiP*?NdmO1rIyTJ;Nl67~VnIQ6;V<5{-jfVnw3l|bW|5-IRn@H8 z4)xDRDX49TnI1H(z0fR%cR)vYACS#nErzwKpfI;d_YfUMba2ZVl^6^FP!p}B{msn* z712YPtT>p$WT--I=n@2P;!Cbw2jxW25g5Rym!WXHLkMo$6r)f1oHMvoR|@T0P74Cn z0d96RbyY|1kFE*?y}JQ~co}p9QvjGL?F=n6iBYiZ8^K#4WJk8X8~$`j$3MS*_!c`; z_lr)bQVeE>2tp`&5Tr(=?yMpe4Dbpc=G;V;d$s&4id&_QACtESW%X6WOc0Ng$QwF0 zfmI|s_gn#CqBDJNj<3AO;@C5zQKsq7d8yH+O0)GET0>{1|AZOdiC4QS>@Qd~_kVdk z;pq}r*`_fjB_EDcl1Wx|DLeQAytoZAI*5Ue{W zmLFka`NuJV7;rR(yYgFa8D2I4o-=uQ=MAAWrkFt?gm_W=3z0a$H)Kl`2#z2Qemjbb7j^?c&MpZ{_oLE>Mt+z40B_#5_kup_lyp=om}H z*hDJH$V39#dOhv5P4m~Ql3euNDd|(c4^s?F6h&-!S$6p5&hsnuc?!z2Z&6`r|CA^a|XdBrKq8np8@kF=(NT0bVdw(PsIU51KLh0+Yh2_>@8oxQ7@{ ztqq~w|UAGWqY25^%@V~CoirWJ0zotUzWUYc<+fh_UR z&0OqpIe{R8^Ll009}Uz>#%1#IMEVWPZlc`%-3w8Ca^3GBC*K5s8SdyT$G97oRLsUk&yCpGh%p!xvLOu;wqAsPoj8GMco}4%OW&;cDNDPp-wnSZw*+K~QIWu@Z%4qJeyI~7_IF_; z!y2|aa)gm2M4W-W^e8A|lJ(6uPYqzs4%3qxh_9>$h#uUW5=}OK8^fv@lg~Kqs5vic zjq!#w%-R7Xx1!YW7gR{-5UPOEOc{PUlTQFBh#w9%kqWY^0kCF?6-!>HK0Uh?E^RI{0WE49E% zdAh#=X)o44sW^8Q!Z0f$_PE0G8v6$%wOyKdrkMIY3a(7|T*s~h5WKDZ8?R$|^5k-6 zy@_TC+vr`IcKtZRq)lD98h42k?pua(h6?@oBdo*zGBjw2N6bv0gG|Yw%aH3D3J40V zfwyG)J-raKIM90_N)3e`L%1Ap84WC@ zsJ8$+T5-VqC%PJ5O{yq-b6pe2@1)(7hvv)w%sCwrnIkeNmM-2F)!fM}3 zJ&At|8EdFgD;IN!B71slsfx)@Uw!IZqJDhxjyUHv3&c&cUytHiiZ0(%gbJ_5<(MYJ z)Z0c{@55N`!f%9??RpKGN!56V1j^BO(*lC zd5AO0$3GY8Qyuu)Uo(a?{G)qb4$uXiHveNy$*4B+2Ad_RwB=%xk=zu@kORwsrKlgr z-LaLFB{op>0NW*k>*se5920O01YI&uzjsf2VCRj)s(!;Kjnyw$8AttuGH3ZlTF;|3 zZ`{!}$Ft{F<~V*OD!(z=k9xTfilFr{wS6V$bEM$Fnl+}l|AnVtQm2cbnhwHhSR%@32a_6!gJlj+wVvd(bSUg)QWv38wdWfZ-(tg- zeRa;EX1@56XJJ$YL7h#9j1}lhXgRV&=GOGhJ%u2_MDUY2rehKO&3Obg)(l@^H70vX zhu>6Y-RD!=+ff!m)P5VbZu5pfTOfuYC^Dnln_4EsAd9N0tc(U&OHq6v3JLSF!a9tp z*6}^1tm?O`RN__@xbB88d0F}UKcm7a>xjB(yKALZ!UG{m9ob1=I`Q#G<2)X{YxKuLA&+)|Y z{-FuMsf=?Ql$flDdoc|a&?E!bO$+?gg|EAPTX0ASXgCbHN*qc6%(H*_Z^uuC8Jbj3 zu#{#gBOx}{-uAk@yG<$@DPL_o@@zCq-n6QKgU$3DM3vECBzW{+o$0j0#Ck<{MBMmJ z)BOhGXA`P|Qxr8q-j!TVoAC?gwuHUGzq*=-YjI3F(;EySt|;qw&LLZYSg0mmPs6+S zKY&`_8KaeaFoUlK1d@ktDJ)H*A(O+UpDI_hqtF^x2)AD|y(A{d{uZb9YKvWZyQw_p*{qbpN8PUs0dPfOmZ^_bqqu^sEGnz=!r4(~MTNSi-7C6W*cP&PAH%^1yzY*Xeqh>xjBJS@ZfmowX%2Ur^24~ zauq>_!Oo!76CbLK49_1XWo`WC3Ogt+%qzXaa3gGW>z(}~OywOK-pHE3E(t$-?uE*8 zCs_P~%LL3YKlj9%+cen_1jQG{J%Dh6FE!RDd8U~TX?>L=nG@=s_qVfM4;z{7o|gvQ z0zDCn|AE9sCnKcLqNk6Dzzh1(lEWRJHC5814Xc{h=%}}$x-u-7LMAsv5!Sp6aTaa% zde|$#Cs}OCBLxPf9q$q$agYN`_VzKw!KCo5dB2EaaHO%Zu|y#DNw75YN#P zHVjpLj!f`1qq#v3Vh4dnXxOpd*kVU|pZXQe=lg-NiVV6k90ST8TRjB_99cdx;1;**Q@0V)PY*;2eSWbrrrvqEqgJS_=!OSZG zwOOFqRJJ{Shp^rrkqKcRcm&yrdy5lUeF0*8Ob!8709{Z#mpU_vV-g8AserP9PQfm9 zs^Mm3tT4GIdJn^)HNjjsm38V_2H-Fr5ygWeI+i0=h)NH+8IWDk6RbL_74Of8}M(EgWop6zF@Uw||32qs4A} zbXb{}q6;6(U>MsRCHBXU&<273g!(*~P@$u`s6zmMO)3q3>wW0%%*z3T8Js4F-Cd4# z)vg|bF8)PpcWvg^`|juP4MgS8=BPn2T1H0>?j|txVPyd(>+3KZ*-0-g4;(4Ak%7`> z^&vR=m3U#Ur7dsJBZ{{BRQ^}H-B9G}xFu<(gzsjU|B(2lt!d`B=FyV0==Xp0l}~Tk z@6on7&RaKp8d%GO@WPm7bGCK7*&u&Qtg&apmo06N*N$WB=)(hBFAaOaFXI*8GP?X5 zMwdV320)Q#w%2#Ltr}87OSE|f)uigAImfgGxlu5{M2cSyLmMrrUMhM;WhsfNajwVS z1FT5}XqFs8zj(~@UkD4RvsRd-P02o5J2hA>@A^r+Ya$L;#2RDM6Og~+^0hbKgOzGsqDcLE2C6fFN-S)rQM5 z+*fXicl0dOX6$H3r;dKdqlIxfF^=JM*03y<2 zcq^|fNK!=#Qc4x~lYg-ztpB3GKI<>;JeUwz44V=HBaPMrGR>`mdD1UJihDQ!ztRqr z%+d0?17xBk4tpdtRTXJ6>kZ}m^b3*E4_22714*K+sw@NyMM4uVoWKtS??b^V>^_)U z#t87TTt(-9IA%l@EUr24VKrcH42zRIHbfLaobAKdd47Vz;A?XCKCe0eC+OZd7l&cF zfADiNG`VYrU>Ag{zD_*vM^3lvR)E3T312Y}^F!EOoh8WUsw#%KHJ>8p)LdyG!H>}u zyXZWJUs~IXGCHxN>Dh43#A)t!(-nus3#aEC=LgxZSR10-JoGFjjk_{*F#vu_#+Tl)>MGAB@ZD9 z2ZGr6v82cs*%yWv8+7ONn--M94z90sWU!QZ|J`6g&}2%YjzoP0DbW&a!T#$tX9bft zo|;qDj#ceF`TYkR)AM|8M(=|)fo_fCub(XB@I!)n)(^>i3jDe}@nPKn;4kwDIfF?$R5n@6|p&EuTm2K<2 zhW{86WP;W#3R-QYa~Poas!(1-xb!g@S5cJP!Oo-~LbaNN;f7@21Y!hsEwh<{<<^PU z9Dd&w-B1GCV5S?DJ=aTvGalxF8NhSG6{!zG7RnJTPrPn_6S}5Npz*;SUfX*Y|0{X_ zqKNQjc;P5=KZz~E55~Sk2UvB8V?Y=F6K?t}d1O6!A)BeVbw^MlWdF@dR3&_&0HGND z7Puzc5beN(VtwYvj1V%lCxU{2+B-CH+&EFf@!8pf_X}+?_zYYsv?s5>b32Ykp&1TftHUq49~MasML*~kRpTB{hIDd!rL9Z zpNd1=vR&^}Ik7h_^pY%}soZa5O@=#WJP7-(ePpKs^D9SiWkL9Eov-3tsl&f10JLV= zRd-MUQ>yE+8;mD5avE4z$*2ao&CK_}CGdq84rnMNAg1H>d<61BZTNXMycF>B)2NA( z12>>4w)jtfQPWCr{W;tV6vd_WdN$<+f!V=iM+03{SrNX2|CwN*ZT+DVX1UCbPY@sj z==W%i0s2_Of$9%_@X9hQc#r_727ve~i_wc*6&z>p{$8Lz2bb|A3eKNjg=v`4IEtM0 zH~hcBDBthg5WbAMExf#|;QG6l3>c6@nQSXh zEnByY{_{)HtmPYF2ojfD)X0v`i42IYc@~1|%|){%cE89t?aLK|4*9FeG?XCWe>T}f#e%xv5ts_?0Y6cf4d?cGmt9P{}w z3AXses%$`AXm}cF{&$JxlA%_Z@EJ@z3NyuoyCK%ut8 z1p)eUeq|QCaNY1i4kRAE9T)dXZRD$(6&lQ!l!l?biwG|;tAcy^=^h4&RKaz}p5AEaDI=72|it@-rj%)jy%yV{D!nk#%HxWoSj)G>u~ zw{=)(O|pz@YKF>!IGRMI9P=fLhY1U$wQ}GC-rZ*zV`KUI7{fdhGqPoF=arwEz4>Yl z>xc^?f5G>K~M-OF;GqXmQ*P zjLiVzZ=N3>crqEw^1j=BBny21gPh!vq3#5-Z;Px2qn2qOwppFQ_wZK%SBxP&r3>B< z*|eC`o{)U7w$PbOtr5AdA4JKSUx_hdqmT*|F!GG|+@AG7SumDu2k5R3TXCx}Y)qn? z?ZQyj4L6&?4dt)I>lP3Hn(X}P zs)aXm9$`qGD5TaPx9v(Bmebi~ZQ6>G@1YY}MQVlIM(#(YAAMsO-xOt@v45?r$q%m2 zS#t!g1FR`Q(S5Q*&M7zahh@?`71i~Ep|t}QP(Qv>LsICMyFF5@{A-BAn9sDO5~sHU zQV0}BpFLOXC6|{9vlI9wp73mH10i^}BjSiqq4cT~xL69D2Hi#iNkTR-21cP9|K>L` z8imUXaWD`sWTx0--VEdSBy|#64hRea%HCz4lwGfI2UF(xVUVK_vr+1liAbV<*zx`$ z@D8BV09DfiMV-(Kjnr89uQ20jH;BC;$1*JjtiPN zSXiLH1$uAt@xS0DRRypWm&OfpLK;FBjsaVzoId^+ z1!nt%_S_{h3r(CN`L^>*WFMLVG(H1pkqShD8**0F;Z}L!kauNrHaE^m!_4$Taq4`O z95yHV%To@1SFud&d5fMIJ-YtEnxlv3n4fjQCGM^okr^wI1RpTYPcW07W@{$Fkn>i+9>pFF&X%THaGWbgz6+16&-?yV66JB?_3MiozL(TG zJD8@qy%DOsbQ`e=<%{-a{TxD@7Uvbf+zOu*Gx9$*A**1wgmkj zBdCzwP3OGgom*vjHt+dyp$Ji;{t&I4dYLY1sruY6F&Hj4W$iA;8)D3N!wv&xB+O@D zf&|iQqzk4Q$6`5F2pziDP;1FfQM;*TFDgftudrc7Ba_?+BA#FdDZit+`J(lKa07pc zK93S6jF(Y%S!;ctU3`T~k}-=w{~#QT0BN4%pe^l8+4CoEmA)5gY60V3qoQuW2_8V9 zLuH9{JCMQlv>H=cXFyA2!JBP*mKt+>mw43%b&H#7PH#kK4j%HB(!a&Nr7@@s|L7#B zqiAzp1NoLA31!M7iq)98*HLSlBu%3vq5MkzF=z(T!O`o(xhu2f#Dc&@vMZi(9&bo< zwl!A&n(*eX5C+fh+q}^|_rI)9^@n@sE?^elBP-BOD5qLJZrG^zoA4Ak_^ny=8whqj zZg-*$?QcChS~%hSPB4D-#xC>kn%t#{P?VLvj{DuI%Xs(&d4Ms13RwlKMAT=PP9+pUdy>o{+a87|76eDr>?^lDm5-f=0&P6G> z6+$J8!Os^rE-0h%$mt<^*D_b7-`cHaGV7J!+!{+m&lcMHB89gL1DunrbmMKZwMO^5 z$GgV((O8skLZTN&R0-!L+n2|-HM+ZNE$SXLEIyWw7}~i>y-ceiA1eB-V2=^)AsVyD z4}rkO?#Ai8QXaU9gnMlvA3%ls=&yPPs(q_{n+}T2Df#ZgmB(%q-9mf{lts(^Zl)*z z!y+*_I9nNLTNLF0?4s1*-{oHo&Z2ppmX_9x(p`68JO%EO3igTa17u8qf|f+3Lu&^} z7+CEt4vz(@@xexP4dl#R7TCun7amtveRV1kV+^^7vw*y2AkU`e7TjFF+{qhpL_1+z$ zHWN0Y+b954oZCf%x~#zSZb$v(-B5-P zZ&UC)MjJ+5NE`(EP&iPZn6ak02$98gjN0?5S7`9UZ>}L7W;jf;O)!T3r%6!Su+Be6 zZQzJoEQ-c-F|YAVyaQzYLj$GmFJG_hOf6hc@>&6@MMSW-uWviNtE8O4?ay-R{+wziwV*$HbJt!rUrHC`ID9Rm09j~OB$m)2na1_|>fK=yHOo%}(AY;D% z66Z!FF61;+N`i2BEahi=)k53-kMt{3&b+|ed#wh_ZCd;vE~=|eW9K?5UV?v;T3 zJD5`914;=I7@Pl(yT=|o{rGJKtdF?vt#G`q%=$f(%!C>j_=08owDSO9GwUe)jFzKzkfsy0UtCo zyxo8?=M9$R#gXNt6#p8DAj7Tw9cDpO`<_1%d~sW>zHc8ejBn2U5x|+4)*QWrrg7?G z#IMni+c?c+JT_6o?bkEkk{M>EpMbkp*WgRSjRKmRg(AhfGzUT+0m+6*MY)_CH`>;3 zn7|6Q{8L>Wfl`i|kH;Scrf_E3j42{Tf_6*nD3FSD=UJ(VjKBZKO)+#yip-$3B^+IH z$awt?@^V93L`H)V8hAO&sdNKBkXqQRrSUmADvfUs;usu%#FsiQ=;oXP4=#eU8%CSN zUw5WV1pR*y19AY>5t%WEBfl2^{~}+ou;^aTL4v(mf7IFP7K+Y67i8M7{|LVIH-0R5 zA<$W9=B1p8ug-{6f>+=Za$!S}K)r1mtiyN;tzg0tpb+n&d|1dNg0=l-8pvJmAw`1{G_r>_cwz~C=%!@L9A8=3XeN}1DWX@oKhoDt90{3h+r>r`%g6;_gSkMOF zMuT~LqHEjMuODq}uv|KYL+%VeMtgxQ|CVWc_2PJh=1NQ88d%Ww_;09q7atfN-kPV1 zZyytn@n6OJF_x|5%XOxSxBdxkWNVV&qIpqd1hpkgSCp}QLwA%sx6sTmEjuI^73AiJ zx-Jl=DCDygqaW`@ZDw>6#A2UOOA&0ef4Kt}sgaTD_6!3=;BXQUjf-|;kfhlvRhnxI zAbYQ-EKcbj%WP~hcIVdt4vbDg7+h|TA%;1eaxH;wv)gdOBo$X`GA=;%RLoU%9_p*{ z2}6ni&z)k&Up*pW4N7-4JWLn9R zd*;nxNo{{{uE0@rjU`oUc+~6d$8lNzZmPH&6BbcXt*O5KsupkMj#l@?>#@S1eSE&v zYcup2`MS*6>2xj%&Ze;LNV*<6G@hOw9q4TaN-;YP`h47nPc>r31YegL6-QA);)h}P z|L&zBv#LmY^mQyBTD|Tc%ct8^Y+u}T=Eon188?WLXKiV@@3bY}h(CabXmt9boB-+A zAH6h*+Dj`}uH0@q+}TluH`vqB+WLz6$n}7HU@xttE^5!DA7Ru8I?vMhYT?Rt89m;V;~2wXpgZHjzo2v>3|$s}ur zaU@$w2L^?YoHKgmx}?JVnPs;OI0=kHUYrCU51=PrrAQAl`(1jJW8#<`ZRC-(U-OAmBV@S?F9K6x;ni+asFX&K3YOK<|;zB9|D9n?=3CL@b^&Ve5z< ziAn&{Yz7<*g7eko)SkmwD%|H~<_3JYCN__XA!YgQaJERz_Bw;GONCj)y?%cHhNh z3uxrrp~Lph#fd^Q!{{_p4&7*)3{go~7Q^fgo=#?HpjyNGXiJou`>@N<_d+!gwtuML zYGX|&l?TI5`%kP|4dQ!b!{8?cCCBz{1*NcJy^WG1AaClNl%2^NK}l*GF)_Y*bDou4 z-h=1I7qirVvtLYA!9@*5C68b0k^K+pUBnEU{G7V=faLNbvu?j&2bdv|@`4+DdpytEuy5d4t@a;7dg@%~D8OOzsp z`?XCCQhIR zbo+8z_z=#>;F}$b*H{N#+4{B3GiL43oz-!DvV*<#&8F=jYI!~E3H)}P7cIi^zD4gp zhIAe-jvv{(VK{N55JwxHf1dKqZ|C2t9@#Cw#qULO_4D(0PN{{J#Gny2FNeT@yg|Ew z%@bVa!T29Tcm&3IAm862%9F7kr(D5yLl;CJP}Q+Bal}tV^DepXM_3Pd!+Sj!4Ra@ zE7Qfe4_XwB9B11Ae1q!~grWw})>tI=937_*{|)*)nESP@_^51%f^!+~T_6kSE*LKv?xb3c`WYbvY_ zEo8GzB?T!gx^V#kAU{OlZNMKj2Mq(;QU}aBlXI1~C1FwR-uwc`julvGhmY(*Djet$ zUHe%P;1`}_Gftat&aprF8tf1Ci-+RGDjwS>aMX5GwPh}u0BfC+mg1M@ zksB`Da~pD2{u7lLLPpfccfXOM6ANfQyxHK#xjXM2x)`%RSQE_4g!Vzu-_=L#w}&nW z+ZA*6ybKLX+3*o^C_$mE9)o~-aYvMp8=KCVE z@OKPjfep(K$zO_XgXdva;Q6wp))k=>&n9~cR8tFVMv$nN%WVrhthGMIt&Fd7lYt5LT@+Duq zc+tD%8hD+&u5U{e|@P;>{-0!p1WA9_PKfAJwaa)k6EvJwM(+1di8d$ zyP!Uin@>BNC-5uLO*B38=+6CKlHRJD9ba4??vuWDu zsf-mE>M@ik)F=mH#>%aRHx_C|?Me(?)s+)GaOtn*?-|&HICpt_vu^|19a^0Q~Gkn_$dA+PUYuw2d-=BlA#v`ynf5qLvD^&=TteTr4+18E=7>|MLkary zKQR#RsYw-(*RSw*2b~U%Zzck2q`Cl2q9EKUV^Nx^U-gT)R}Tb!7t~VvoNTk1JIrmi zZmXIxNw5)?!6#RQIouoh=>JXZR`a#d)29CK5WInp@HHHReH+>a^D}g6|F-IL$h-EJ zs{G_UK|5u|E0s@g^mw@&>4vIQxL89H8UFodK}cSR(2<5*6XK=OAP5xaWFGrw#$>*D zFayks4x$DHanp(@gk~5DT7%q%ZhRfmoD#FmG`d)tk#daAF_iipiwT!z*|UF*Oqw(w z81zwOPGCmKj*e4m(jmkQVs#_2f;9NRtS%>B9dH?gajJFyC!84uf5#y_XrkroYX2gE zQ}yd1lr1|E)N;Jjk|?8i*LgqiP&>K5IMM`3T#^wd0P(>Fqkors7d+f7N*7em1t`KN zJ$C9aM5!=$&4yNhfK=|wa)Clt6Qy!7@Pd>^uwR&wh%}wl_~z6IQB-6@FqD6B!iH{g z`0X=?aKEU7so#a6zMDmz-)hwAh0RrGcx3m#7!ac)gred0^W&rc@^F#5wy7;=B?$-e z=b!9;mU3lEc5jet&VU65>s@wSkd*kta&K+Ek-o}^lXPFz#WT14mOc$=e`l|CP47e5 z=Vvl_Ni{IIs)`j4erZ8>GDQ;*6ufBA*1f|OOJNsbq&6Fa>GG9I=$(Ti;`ByrGO_EaD9wLKlFP5kvoK#ob@V4gh|+k9fBHvP}4^ zYSNTlQvOq2BxsG!f(gUufQ0@oGQTxJHU(ofBTWM(7;ft|wCAEa z6*WbzFI_!Hs~l}zEz6^+7@GBe*!uE-8uxemZ5D}&lw&B(gGwkvsYA$ChKfScBovh> zN|GZbW5bRLh1wA+A{8Y{n#C3w8bpI#6cuUyt>;ahbMNo&KkoHCQti+Fyw9+nwbnxh z=oqIk1IC$e-oNj>q}Y4?=kO^FviHR!uEkUWNH17%D;MR5scu zN}LexYTTY%(Ue%?CB`~dv!Qik|A)g~H@H8t<74zq=_y9@2_~yK0w2-wiD$lmF;UeS z{{G&G*8!Yp$7q3{XXHv3wY=6XV|%Sj-pnf+b9RbS;iJxLR}JSV7ns($zdNI;2qo?- zXno{KYJiQUrkKjz&N0>dv2w=DDMJ3ps}>1ay>q~HoeDVr8om-c9u8l{aA}kA(&4*c zB{PUTo5*5J58Yi#H-cu=CyIGdl)vEUu*~DoT9EBw3ubGZ1}DTHQ<2gQD>2H}(A>NG zkF)g0O59yo{qktqd#L5WqeIz_b}Q!oy9_SOwcqo9c_JVTW;KnzJJ^(t2i{JN;SDT< z3&MDQp?CN1l-~5XxRq3yXX)eg!W@9RQrq(;nf7f&A_C}XiYtDfBgp-y=clvUU$mZegq~Dd z$l#N8dEm)neelFak(PzSqK8a87hPt%X--9=U%{e>#hdHgce<}nvo{M})7E?EdgmsU zxW9hHICWR`rAw?)+NkgNkB;ijh?3}qQ7hbf6MOf@R_!f2$sL;mx8!?pU%51YkImKn zAbEoCz7b;(v5+A-AQL=cHhRL!wx7qz`u(Sys)Pqd0BIeOhE*XUIGn3BTY|sz*?^DV z4X?7mnNcNLZm3N4D*64(DaYi@k(0%kK$L&LQ&3cCw* zdfHwI62)D?6^2P>aGoG(&qL5hptGmvo(~U?c%VwBdod&ZiB(*isZ_MeJu++*v+ZU2 zp|*A8N`!W56)i2H4Mxu%IP-6|zq340J)-yG(Mc#XdU}3_mkhl;q2e2!HhTHkJiUa> zf!A8NpT9g3dF8|539Ob=()@3$mcqB=v5s@rzMIQl2V-PB?iBaYCexn7lQ)b%eeF@N ze2}rySuyj6Ta4YpH@~jk+Shva7|(6-S%(a#v{k#JZy-$(VlArX3c*1m?XxAkSbo@v zTtr4>AQWYqv}0;LFwh)Rpmn=3WcLL3RxUw+B%NSy)4mPYq)#v(C%we6RXDW@*odvi zI=(yM_Bedf^WdLK&Y*{naUPdxN3B1*RO8TF8h4^tWPnsA{d}w>)P^a!5r1!7zkZu@ zolw1KdvIb{Tq)=-Fu|Pd2~3nh-SmGE{}@bx&pMgsQNQ?X#wv)1+o}+lp+zYli;Sk= zisVV-&6kclcC4k?i%$!_BE)f}KHx9_a8KB^r`Ds~U>8ulWV-QVWBdBP_IT{Gs1L9A zSbT>iY%Z;*G9*}Xn8n}gWQLsh%V5t@_W}{c&B5CWjW?asaJ&)waC_3iy$j{9#je?2 z$B!&1h@10D=atTu(2Yrd791>9*D}umj;;StV^qZ`qn&KYr0>MB_7gvblrm2qQTp8o(uF?+{~HjlcKCE}1NV4} z30LcyNl(f~tR$uC#BkDuVmb8n!9a`^4*6$aflP~KFw)QZcuBp9Cs>sx5&8A5X`J`k zUy;I3RY8_fgFL}Ldu0}G;A<~bEFxo$aV&w1#m)_voql!|JZ^Y%w!T7%!zXKC4fV@H z!XA*S=Qx$zimw;U4g?g+t^euy6SNeTfe8d?u5j;L^yt@@=(Fr)%W@{lqwaZZ3W*E{ zGZrqDN0nAPZLF;HD!sdQ^P8K^I#HN~G<&x{v`OAHPZqH$3|ZWODD7=YNl7&@FRSgI ztXBXx>=w{TSHp~5#Bwv1TknEw`}3QrQf}xvdS)YxD_yuX5XR&FS)|EC1WDTw(R<=D zXJ5y>RLQXqEiP{Z`+_svy;9sBI1_HKO6=}8jj0+uU4n8kZOdE8*`lu&4n z3im&1omQOh(^mcCKn=*ZDUPkTfyL~@p6?(`@~aXOEi;th?SKgPXzk7-zUhqBm=Hz; z<1vdvaKLk;%r5b=#4n;z2#p7{XNnmBS&eqeMRjZn!6X`JiO;?_<>j|ZJ32ZvKn^sb zs7mV}3en4l(ylAAkg>rcln5Sdf*+Gys+SlAv?1~viAOXd zB$2AhU=3JR?9`F6GwH91gzv{>RW^Wn@_T5*YDqi%;|LM!nTWU^F!$l2gyWl;n&tr# zK6K~Zt_YGCeEHE)AGHb%l_yQICQh2P6T0HhaFeP5I_8$uyI|%_DnPV5zeL;IF`p|l zOm0PNGgAif^>PQov)z+P{oy(A&M!B?{I#Vo(8xdArr?m*`}|5*rm)1d>(>pivml`j z{-{s%^sI)4`vSmV>-(Yiko&tu1q zC83IJf`0|e&IhJj%#(s54nKq($`F@Zsz3nr<+sX#0>rp|BGC6udOT*Z&d4u%0(k(`+Q@yO%+kNXye!hBibC>nD;&j>I+&Ep$fEuQ0u^L8Nu#T6D5@H{`y z1MFUk=et~O2+L|H2qSEa+IX9RamJ!{WZAxY@#1nR2L0zz6F9O%JdneTA67Kodt`|u zuaB({y z&qIB{!Pqnu#L69P-vE)5Id*t+yf<%R+fJT1Q4?xr8FcaDRHJ-+eH)|;NnmAi&xeM} zLK#v!eXb7&n$((Ntt!Ib8$ zRGyKto~Y%s&-ysL+Lu1Cl@ypHCRTt)I@A($F$kY`u!qM?22-LG=>fntJ$E0UgAB`o zyR4S=Fa<-wYp=pzj2-F062()#J>>)U@?||f1n=~a>uIPvXW2iB3Un%W+P|bkyN$we z1#&BAp|bMHClrmjpbY3($Fq0uo|Qd6HW0uDDd9=I7bb$om?SA&v$bm_(SDV33^IWv zaDRDbQ1oHNM1jes9%!atL&G9qayt$1EcnE^w|!YBAuhg_%7b#z)qC)pqn@e3HUYM- zuI^ULLqA{pZdbzRuJMG-1;2l-!MH(g*|KGnV&85_b6w{B>u%Ia$_XWSB*=pb39tlW80vz>_1F<@V{hbW!YFxx^7L_-1>bBMJY8b1$wRix}~Ls``*oYg)Vs%>J7zx5_ zgD1>3Y?Wiq7Bhx?xT9RSy~1+Oo;&F%e3^8R&oY$5?KD_{Fo1u{Vuk13_+ttF*i=Js zX@j$111Mr4S9_lYiu+wzG&0!G+xrnw*ZAnnjNwR&5EE#L64MyWBBPbiqRv5t!B1m1 zqX-odYIXUjfA_;To8%j-_5kIgC)rAA!BinGM$n%yr1B92d|3m}4kR))scGThxaElo>HGv2i6215ml!19HlLtVdQ+oBHldSF-L6cM~>s~gi6dzphF7D_a- z*tuZ}sPkNLk(o!FW`fJLXVBM)jX7qAc@0|)&p7z01<7k!iaF|n@8BHZ1t$)Xv$>&V z7?#VDRWFmtMpp1Yt-?oa+545I}8{KI$@Gg^2NC3OXBA#P%%2?}Jv16l<}; zuOm{Yro_h^(FMNz0Wm{Vb5_3ZxYXjM@x}eO)dDMquD*0GOW&j-wh>(2XOy+0_d;&I zt+n&kEfq3)e5N0+IAOFxa>pj%AO*F#kVO8mY ze{RTuuW|<%S|TK?!7~*{#+9tt_h1-!VK*5&@y3l4txqPFy?7ysuxjGD;pTdZk>L=D z#-cG{d6INZpz%A2q!g8uc;HDaR}(v#kD6@}V*Q#dN3fXAk-N0(-o5i=hN(|NzLLCz zv&BC~c8qw+m<-~NI}7l6d|-+(!yBD{9TgbX>{DKD5f>l7-V^9xD6+RvfKS^xOUr~( z4CTDW4`T(d{`>DEMG)ij(Q7pbgw$PJea(2((ZA^~KQLQI0AW_s_h43MV z=RbmHI1Qn)w60i_Lcm-tEoC>DUDwpq>|msW3&Eg5sd0stkuHb|xAC)@?IgojkzY6QHF*<3JPn1+Mo|abkZ6q?bBt4k3e^sGtm^1&Ocb?* z6J#ii(*H);i&Cop7)8|wj#1=h6TT0RkyXwh%s1_P2ct=hXG0c6)3GQ~ywW z`sF{8LGsX>DwNA~Up_*xn&i0r{_uAc8cVnb-%^|KjEj3t9%!=-yWZ%ad{2UK4`+x9 zstW@m(a2CQK&RU8 zZW$Qbw^vSlmW%t0v2uTJ%Uv|BbzFAfuV3!ta3DUacx=n16)-N~*gK?EIB`M9IS>q5poFlu0Aq0}LKHICfDv9G(|>N>d{fcLe$Mz6=(Q)_42`$P-^v5hO#+h7#du77 zit4~0C5GU5^PTHy?dr6Gf(BB ziLAj$b6HkrAIJ*%v}~T%PWgEMkB1-pRC@TYAxwLNh{;BO)wy87f)@G1E%Gh2F`Yfq zN{LdFob^Czn)r1JQV3qWNNaJ|w_3N*&l<-0B~(9j`s!O0dg;%%}%0>qkyAm-GLu3#rUy^!MFA*sgtZ`FMTtrIr3q*W4E%K z7UKXe^aUNpPG}kb(!`^Fm-Yoau*E@g{8dW6qlffy0)q zi3tgHQzUUXBS#M(Do`c^l-3RpFJkB|^kL~ezdmRHTW*_i;NuVvhy~|oq9l{}1{~4g zKprP%j!V*oz(+=tX_#C^JI`Se;uZG0Q#Q#Bw z+aF1pHWY9Q#Sx>O6Gji`y!RT1cN?Rctq#{OXOnq# zf7`^{%amhG?-L|Khx*u{Ew#{$BCH%lG*Mt#F4&K)IO5VOvV%Ujr*&sXOk=5uMh_1p%38R zWW5E5gPsP;+z>Fs6*6EwakAL{6=!sv<3QG16GAZahtWd@D>vOmeYb7M4g^Aujm5~$8vm8-P1%vU^ z6&xDMm6Bj>VG`0n`5Ch)@*KVmUOT@9I-Bnw&U(?XR=(<}fnCanZICasn=y=V#iAQj zo~7qdk>Iu?$@@~0Ux)UZ9J)N~PCr(6M@g;1k{)*$f;qJVFou`Go^| zXo2>lTD#s}o-Mpw8JG!Gm3%BS>_nn(!I*ZHyZe2n|IDw(?l(j8z3l*K3oylK8$Q%W zSbXYK&f(tr!{@+vkxBoB>xxcd^HYayhO&V>={R2v9_O5J6Rw!e z(!2TQ$nkxGEnAN3gQ{(8Y8ul<4h(+Qgpu3_Qv7v&yomAhDHy#jYj2iyzzT4m@`S>zfOiUJdcc=0Q zkR=2tV1nRC*{MRxI%qnn3<#qQ;2~fkaobT022?neaTL-7g@g*wdxct}B_QCBrT7(N zX1W2~?bDxL=&wm53(WC1FD^fPxS58=Z2;>OM~oo6^d9($#QGN~S@fX7$phm>87(a{ z6Ahe5%?loHCf1g$^Tac7j@kpDrS1E1t38PGgu0MjWq&uHIOX8Yik{55syL0gt*n9e zo?krJ-rMc`j))4vO!b5+|I}f?^+8qG;EA@#6@FDZM|>utTH(;fI)X`iW?7`FTmLoz zX%ZkQwE=^9ep2g?O0P&a?2ii2KVTq1Skxmmt2^c~RI^OcDawxT@yre=W$3{CPXNrs-&fuO?P*nH??8DpIiGdNV(K%1@J~qdzXqfxm zyLY#NJnGb#RiqD}I z6chQYU};uRnyNx{rK`#^F#F1;nhn|6SaNE}$s@N>lLk&y9ht?Oe1=cBDUyYvqT)JF zU<&{W=E)8tgI#5 zLgLI-f$UQUKl#pkp2FrT5IjuXO4k zRCJgJb5j7-E2^rZHk>&VQ-!#(W5ztSzso1;f}F3lcoGJ{weQq!zE)bJ^K;_Tm%Ros zs5B;WBg(7~_#*MR>S}AI(SOs6y^<@PDjsm#(4oYtgSE=qhuOa?sw8fTeesn1!a~II z=pX)#1?0EeiGI-1qjpj_#_#<>;xmE-Gq9mBku*|7j|ZcaTixCDy65pjxXtblYk!t) zolHI5z+t1#q7%kpYtKeV#EWNavVX1kp5frn>&M(#bfBtTs?|Jx!#l_Ucn{(HNV4rU zl9Id|t5m|ZQ1qEH3%Ibh+~>u`MyabGd!Q`f zCsJ8ZTt6)QF~tb1s|6XFnk54xSPPr_t@^M_GaTzD7!*Z%FU;hbpaKLsg7=1o#mS1|-$)F{**zVt=NUdJmEsgy<~so4mByvvi8{ofpGcz2>jtFF*aHGH&@* zJW?ldQ|aOs+?HmwpO1=C1E_`!c8U&D18iPJI8pwoSjYE6(rK$v@h_Y+qw&(}?ODsa z>aI}G?fH#6t66opa5$t4pJnWDHdh)HMbE5{ z5c4u6*c#yY>QdOs@)agi{M!2ZLPR`tsUTlx3$!=Q*Fvo?o(AElZ`V2W4Or4-y#T6p z9%@%H_{4_(pf{O=a#*UqB+ASagn>{dsH;yQbv1BTB^_LTa{Sze>=I#&W<%yI^#YLZ zFulO;4{{rnr7Ku6lYuIO4%WiI7A2Q{-|ERwk8jCkGC;VOYOwPE0Z1Kqd;qDQwnytP z=6GW+)#;i0T~%pe`RHnj&#PG?i5@FB5EUxKNZhhlucWEy7a4-ZqjzKk|C;B#17vv( zpp?ddR@$CN>L>#NT61}Bwaa`F&nJ@O9pE1QyeQYxKU)fKmXA&HMps{cF@1)*$|O(y zl<&czbq7a=Kc;V%raArzN|gQJLDdo#4?vF1)C)lnMFkSDpMv4=NKtt}sFzEzyQcr7 z33Z}b7Hyu%{QUm4q5Wd5UE#%pP+amhL2t(L4i~?cD zZR&cTqeo#vy!sRb0}g&u**3C&D*zjLF45Ku7K6mLI(2+sp7dD*2AyEJY{xhWavW8B zG^-DM!M2>xE!_9fsQs1C&As8uaOr328+AKt% zHe9*#)IGLJw!pI#vC#&a0DV5$ZL*`!d*mg1jMl_8rp4p}{;$V9AB3`S6-RQPmz11G zf3<^sRF~;uVS!4n>d)dKUMKsAHiC?SCsFr>@?PO5XgmW^IdBO;oNXvE2wU#mpV_rV zQ9&UP02z1d#*K;?o0x>*kzkp`XJl+4{aYI{mpno(kb+W-dTIa<7=F)L`+u@XgTK92Vuc{n-P}gHtMKU-aU! z%Unzo#w~Y4XB$(xctYrfY_AnFgK(1xIpPye4{>9xzMg}|_`kjp^nzkFC?5xbG_Ii2 zYJW#u=f9=dB;4X_uM~k$U?C;vk;iXD1%nxNT|;oIf}^4m6;E5%NtM+}y#*n=iRs7C zf=e7+jpx&N1t5friw|w$V}%BLm*$07{ol5b;Ko5e6e)>BS-!ZhdoPIUY-bdlG(whT zGjl<}FU*YBU|N4kAetZ_zctg6mX<!(s<;>ovEZxw!(9~**5BH&JZ*@lsbdysj4@?O6WpCL{N6ZS`@TkfD;~ZA`*s74!3q;F6%NJS+Lq1W> z#1St0>z;h3d56IQm^wX6m)M0}%@`Se5eShZ{CY3Q{d%5=zsMcOWrVae#*<#24_^vJ znBnKYEiQ+Pg7F1^Fg%L)nM)op-njNP-s1N5S=T~3)HPb zU!D8lW5&e9=wm`=K+8+11Qvaih*5qz=3z+~mSA8jBV8wi&h>|m0CTFmfI&%oxQx?h0x0UL2de-X*!8QTw$f&dC%`^9hisx2Nj{#5EgWePya(UlGz zCxict9Xpolw)4y+eWauWb#+Z=2Xf&h^X5L3Re1T7x;&Y2&ZV4~8o`_9JLTo*neP{r ziG%HYI4S|&B0%`Q+1;0hEYGsPyK?8wI9}r)2sa4ygQR_YV=jEHWc;4YJ9o!*{#5oL zW#dV@^)*CoVPT5f(m}_ugxhcGqg^iH0~4NqxHsK+^5n^@ zR;~p&a5J)yc)-FAG2#Jvd12XC|SbDCAT}igNLR>IL?0`<#Bx_`i({m+@(C!Aa%wWhC z+jOR%W$HUHYN@`(Z{#qRf#(!4{aEfYXo4v14rhWsLQAW30y+)#S;P$hCFcU6fPsU9Bp8pNn-pSA@ZcitGK|c z0+B@qDaj<-Fj{2s&b5L01~)UfW)&3kN}qDJQsTq|5fYE72H)5h$@dAw6BYTKo-G zDkvH??8QAUPQ&^0v7(BtonYzcw(wF)-$ReQAdk`bYJ)>{T`bH_*{fE~2zEaar<-X>O+;8(p?z3xJ+i*k?+xuvYedk?=8qqrJ z>lXApGYlyLSx(DQGGMR&OuG<>qC}5BhW(OX=Wyl(ro|2t(~98Z7EuM z26$2GYCLHXgl0AHc)U}UUKI|>Y-{T)INWSp${W<65Jd|gB~$n=f0mx`_VW*!^2tXm zsn`G=ELH}^h6*eL82VK+-dD}V#_51BbJQOl;*~ojyIw+3`B9a=0#0rwo-~g^#jNZY zg^DKXy9K910v|mqW;|L8FlX7$K-Z1oN2@5@ge}em1h=I@#m`eyR1BKY^JRr#0+n>q zRB(KKd!-Gf5BQ2kC^%Avyl@0muLUN|SZ!S@?9Z$JI?(S*wbD^*%gMb_3CnQ{@bjA_rc21^8Z6Cs@_M-~Yz0 za}#hmP@7B<<*G78on%>j{{cVm`;q2T*0|nstbaBSk~n@EFx%1}9bW9+kJbbR6lEtA zXWvEvas!i}IOM>mf5u@hhoMcv=WvyLrx*CI)F7+^^&CM+R&99v(LgBDWY&X>5M|lp z>h_jmlEB`u;T!|EtyGyfCLg%Etk_A#p8Cjxs@4^2z^F3DlP_OQ{j$Lf7(WF;XT?qN zXtm!qeyEH;Mpd*}Im>MwyipS$b>66{t7~BPam2vVbruRZHoC~wY&LtY>`)(PY+dYt z8c~S`c~uU{1*p5Xftqvc8<9MxKl@XxCuKG}P{_2#h4+5-~0WHe5b-^^(K$6UkMzle$EBSkj}_9~Q{b#=625 zLE#qc5>d(hM(v_STsjVn2e?rfJA#0dq_Xvq1pv@E>a5j02QTkq&%P8oTKrlmcXgyH zgn`&zs_qhPhqgoxE!BEY>gz#Hq1!va`JlIAM*+3I50s=f<}BOXw;f*UR!+@@AA-E>zYibfC= zm6>Sp4F`UAiHY(+-T zc`D3$Hgm$6Rb?)Qs4DJ&EgYASuz}R3AyAyr4Y!)UxS_RUt}H_HKBNN`OzRku-n-i~ z+Cgm$T}V7J)CzJJI_5hs9zQ|&hi3*GTSZbgQ1(-w@v618RTiMQNkmOGQTUtN8-Y~l z1tWYjOuSsZ$*8}I0G3XL&p#ha=nWZ`NLI!0p+mUt^QZ1lKa2h>mAU^jkdIDOU)T=t#{(M%;iFTzT=XHsalt2dOwT#A3xMpc zH_p!T5Wu|4Je>JVU#~@3P0g$UH;Qa`iBj@%<0G#$<94jYL=!uYyV9uc$m$>i8S7y% zISics&y&l>RXcCnBI02@sh9y{f|bQMq}b(HE^mCHdAyxc>x-*a>I#JIiR19zqUM#E za=J2Mme@&pXl!TV(WL_kCRp5z*d^nda4)YD_RzZI z`GTMEx4+L>i?@KEyAodAysdzHR5soFaz^(RYUNUpAVPou8Ew$V>Ftp0P~Y?gN{)SVXl-2RFY|!{Jy(m>(9H34h+=vZO-diVM-A5!+bV-QB!?7 zT5gIsO{v?*vC1iGSb5Pw)5}<5e+Hf*Tr~DWaVsCN(npOA*rQ8P5Q~ku`#(Jm@IBo) z#Oo%a749wyFU1io$&DV`k-paN`^gZoi*#y=CpZCqGQp~O2mkCnx$WWp)5HB|Qj7)# zn?v>y>|l$gx!)rWovMK0`QyU#SAMbrbsfNQp83qH1O zJPo{Qy{P|k_Pf;rEYc6Gqm%OO#pKft+8UD?bgCmzJFQ?QDIiW2MXX?L8s4v$sD&;} zAdx_+$b(I3zCDgZK6=w}ejWtm3jV#f>$lFAdsH1pQzza+|7hOix`z|n?}JiMYfUs9 zL_wnJJE40Z-A|;4fg;+xTWIg&U{5WNzI9F5L@b6C@$Bvx0#?EjDYxj6*Is!rCyeO= z;DeA!gk%V}CLbmwW2GwwR)o#b8XY=YnQQnlKyI0V4$ON#0iE5 zS4@#2e)Bbo6qusdzzQeaBExe%f3u1h@mGk~&h+5Zd`zjI79Z@VP3>QU3P&%iTdE_n z9!RtFHl>#5wktNpqa!2DMH{qJBz6I(uOr+)Fv6dMa;*r(veJgelvQki$uyU;y;=(3 zKva`I4vHQgP9@)N@v}u!&n$Wb?_s5%;Ld`JXS9i+iw=x5m?DCKoI||8JO_qE@TLsy zse=Q+hJQ~z%ENrc#*Gq7`GhL0C3ZA10>y)9EVBI`D7JSVCM-R21MIvw+~Vg^cr!l7 zG3R~Uf%hs75sDX-8j%=p%xq!=OW; zz6}P)L8#jt`p+1T%-6SfmZDs#7>E?Qvw){#vP>wb+*h>_%)oWKtH*C6ab55b12?Lf z0|xuO8>wB$-FO*`js3{7vazV?nkao66WsiQf(Kjv@{>aU`kJnDom33IDO1tW1VUh` zdhvzAPfaRftFo+)TB%=G#J34Etg}sj#ZUj-1tCP%vu>2^Aq*(DxYg}2hTaVKNM=Us z3r9?uh)AB#^dp6Zg`#Jo@H^yRMR`EWd^g&VQ8 z47WHpePr$cP@#=Zg=R?%=_HeO)dBEq>fDQlk!-|ia4IK_=+2r8e-tX3xU|?NAyGDf zdih1o>C(svd^KcQSP89cDg&fiz6?H$S#D%=w2BE|EREMt>($cj1#?Te2XF#Sq8Q~B z&UMuPI@f>W#j^SK290|91b&W$Xfm@FSKA zI^KD!4e?W4b z9vXiPn^UN5{ul`$MC5rg`P2JNIGqL561W5U&M6cj46zPS7KeI=2b8J>jElXg#3*lS$bzSBa*pznpLP*AzhI zibq;GfCi6eJQS=fnOn7&bPnE6hY1tWeiWzq?4vjlHh-{EKJoA)bJUWuTlltDz*?1g zWHws_;lLQh5;HfY7$GtwUs+yW-uky(-{v8#7!f@AOoxaoV|d_;#QwVfb>z6Xb@DCu{i)kx@d|xk2A3K#&n}H`UpEn zG^dDGdlwyT02BsHfe~sn$w7Zf&gM+V(UK<}A;OKNDXc20Z}V(CN`Wge3848sk_E2{ zSLV5(Wt;r=19O z_*u&~R50}$CYM)tOI}?xloc}x=WGmk(dg}UHhg1+I3L3D_z zluc26(Et-GyeRkg!Ey7U5DiAZ37{hwsor=gqW0Ui4_<2|UlG@OIuHv3--Aeu$B#4m z3Ot|0!rzHsrb!hu+dHh3QqbVW!2O(=Z}be$25*2Nj%Nec*c7#vBy0<_A=G+h#x>Dxis zLxA7O3a6+}D;7^=?{K8H9~scR<7FH)IGB3bFg7BZ($N^a`k;e{x<})9H!GdqYg1WE{OWk3&SEh|GtnH)`N7#vA;j9st3}CQjR6Q*$WiTZ#|EZqO zt9y+JprNb(5-CPBWG684C`3aGe8N3O!>@2GZ6`MP8$pOJsEoWkH1ai6Yd=3-lznFi zD^DLA&Pim!kWO&nS>>1h5Vo}7VcL%!NWGWqd zv~?^;A0*<@XW&YaZugCT_nyZZ}WqF zitR_{LGlNod{sWd|K6BDrgt{nEN@0LnR*t`2Vwi<3Ox`&f{|o8)cR^OWwiG@rF^L6 zp0^BUYVDDdTWInpH)`OFZ&U~DW6n6x)%-H_gh!jrPMLx+dJF9he+xuunDKxV7aRhr zQ3HlG5Rty1?)nZXiQn$J{{ib1h$y_nkTV4n-a&zuUD~T375sgbt=o&124baN%(ZLV zm(a4Le5@@7Fk*%)vI8 zHF6kYJjlgsQO*1u&bF?re~Z?M8T~=!$vT88mjd)mVYkHN@+9aG)Rk@l8Bj4zwt)KO5FFfHPt^TNiXtqIeh5z+QG(;PbE+o-tx4}_&>|Crn zy3G=bz5`!rP%z9q0Za*Js{_UkE&)4N24Y5jxc@n{C|`l*aKQ}Xhij!RnaRxj)*3L} z*^NcH`@xKIf7lbLYhq!cLiI_BwhH2@jBabPcYi}TXMCpS_*`ttc!dU*At2TrL|wtk z|3p~O7C^GFg2N~(8k=1Qud)a-eUw*AK+D%qFqToaS=inv-DbJ>^B;P~3-UxchaOHT z0~Sqr!LqBCx3yv(&|i?Ff>xT29}%pN0V?x>f$Yz44E5(9_->Vz;em+2G#bxNNick_ z7{5OiM^olz5C!2Hj$mOkOg#jjTzhK6003`ai80{U9N0jJi2X;pmg<=Q-6wRnfc(u-AhhZ%)qZGQ*QF4 zC+G7IU)@!d9Z|XXEGB*y@rywHC4YjcKel6El45dpbs*A!k-j$Gw+u;WSE~H_R%rt` zLDQ-H`s{V~DB{ArV7<&PLD*(+xo#}BwpqyER$q7fbiYy_FY70O3uB^SkaLEP zW_ci>;GoI7Q`CnO7a(1|HX zT3O!ShZTRI{mm;@_@lMvZnq3USP5(*%@NFPjXm)#Ti~Z_uwlcSK?3`7}Qr>WD0>M*jVi)0)>+WsOT=T^lb3WpjG0eGHpah@CXj$oh z#0xmu*1~2Z0|@A%>STO$`BB~ZQ3eR;es(O45!9@>4ifJF)qO@k$%0JLLS^o=9Visk*RmIl;Iu!^86i1dURpEynqz^w z#&vaeK5cGavlf-4`n#gCj=Rw9kaUPAqt)v7^I|L905;B@Vr0H1TM`vO;3xyZ!X_DKd+^11m5|%?E*pxJ8DXG;ru-NPa8YiJe0+ zre~{`=F(ewm1mAyK=+ zECw(0(|<4XKEY)~QoQ#JclqXn2b)fuIKinTCXW9vAHxl#PR6c+*> zk-(b_N2ZIClSH?B?Er)tDoiwXxb|wn)hJ~g0EEscrX%nh2b&SPd{BEj5qEIRq_zXA z?N1A@m&h;`v;U=mK3wG4cDoQG5P`UEHJ-Ap{qy%L88AO)^eU+=Kt(T z`WsqLOVfnQxoE-x$F1Ve!juju*TNemJ1GkW2aAA{RCnMrJ~YVyxI`7g{U)>$BO31I z%X+Y!!G3yG^81D(j&cv+BN$P{GkWm2ZBUyK{=lVuELgpR?#*C7c4D`{Ek2)6MazTm zi>iSkuT8wFkK&VZz&sXS{ZO-yX;y_Io>qKr8dvu$ec&%)I-C^}cW}Pk%T)Lm7#}aT z4*`CnoB=I3`7B53=>-#k*&$n9M6(ABmr0z^(0r`-07@#l$*T|{IvI`b(G@rb2kZ%C z?_S(nbvnmprT7WE{=+^G>mvuvg!DytP)XE?%SjFO4tH{f!EVFAft%Tc%|T4xQkk@^ zP=+QgGf!ke=^!~1QGcmbh)H|iZ7_(qrXCq!EbdqSG<$Fs4 zpm_@u)OlawZNtam$MSbFPJ>&C02&N-3=UHMWiSRcOttwNc*n8TVvnsk zw>I3|1$O{GDXGa{Kz>4?T~YSRMesVIiJ%whR(9pVw^c+_Hd|||&1lko2JR zjA*39@PYGhYs{@%hPd0uI=n-$Y8dQa)^V@B&O6bzvej_yv8pKsQkWgPcdu`N5B*B8 z4;cv1L>|2y5jUW)>S^|>Uu0}-tkF<{zR|P+B&+ z+DB+b=V;+~m=ATycJH@%ZvQ=GSRGLd$Y-v#3a)hkyy!8u=jH}B9h@`%iRSoxl+dEg zko)Axleu#xCjfZT*aGPYst0B=6+2g4xixw6A7p=ma8S1Ycm^2)wx28V=Df|m{}SfZ zlVRlZ%C|lQGeqk9N~6$9ioQ_=ee#qvpgmS_F+NL?Hwm=*xH??ZFcO1C;>sasEB9a$ zQ;2d-LGywULn4^C_w%!H3qa1~VGep8Wh~(w-~>t$eT7tO$T6HUELxOuPX95o#COOG zhO;qm@cJGb#5sbP%RBCo^K3CZA%J8#mUwnhZo&dg<}7LYpf48~W(e}-q&g!TA&iV6&;}8E{ifIP) z<4BIlkS5SzHOV;|ehjdL9Eb`5xt~dt)j@^{mYboQ>!yV!NGId+Sw0>^u&n3zBRms$ z#bpf^7Y6#D{q~JVF9KDY%N+5L9p$Nge+?lLQ2}MON@$2=#XgIlDI#PaoB;M85&kKZ z%dH~QgEgJmC!PCA&3_0nR*UanINQ^Xo(!QiieL8J{P zZ|7R+;!)j&=^dF7k>&|~%}dT`TVd!YCyr4hlWA%=kX+Pc@U&+jlewIkw;CTXgD@32 zy5LRmjBp{omZzEUF+s?uYlylzx>p-!0kCrA&)S%z|JA)K)fH8Uq{qFKltT3LoJz!n zr`0Z)-NNYRdUt18<~d-LiaiK4`Df8g!M4{NZ9GtflWHw8AY{Zw#&? zAlH2IQu>+S*sf6l)Jh;-)BZ z=-yvPHvpe=!&}El=d2d^(qZ~!oWkV3xa@E4$|P15#Wgn~TJ1svRO(bQJU}{U6eq>S zp}3Sg*+=iRboB_}tmlz`oe%oHe5M3LCAp0nViE3HDxTdRR!|Y{b59IRJ}gH%?=t^Z z8ywwt0(zSEYWsbN%Pdyih>Mf3U4joqhBmh;xB-777bti$0QRmW-&HuSY~)X#I5F>L z;v*%rUkqbBCKBeNOuT44pq&J#W8ls)gsFF6wVoo=-_9EWUYjRJV9NKOel zLIdf4)R&Q~J43uBMYD;?oVt=-=mMFDG~4!Bm@chj%qLX@fj`_78h<{2{)VHlmpp=C zk3ytNwk~oBiCFhMtnmDYviW6T`ZK(LUujXfihBn3;~BR-EIC8mrtY?Dx}o$a9735- zzCm^nTWF6Oskf+ZICz<#4uai-SIp2se--JMg-g&68tX_s+CLlnhx&sw%3I?9AoVQ> z^XEkPW1D0k8=T~Abw%NSrdBflW113y@;TSN^-iBRYLuc#oz1|T6adlHyeum_@6wHm zpbQkeFs*fPYtT?vzf8hvlU-OhLgIh2sAP(R{p2kdkn?DZmeaso#rhWN72}8fg?-A)C<;0)gk=3A$pM-Dg11YwDt7h()aHdCRotK zDlD-PL_k07vwHAt15+Cr@*JSh2M=I8tZOZNqQNJiQ{;+C^m+ z7!J~m%Pt8;K|d{j)~s#6n}-7lUTuDJm*qjiHB`}84~nP9s-;UJOxyOBqpD)dh@@=+ zrDEhRI#}{F9gsal(_AV*BNx3#EtYIEq{Q*w--j_W7E|2AxHt%a7=avIBzC>M{WnIi z{5m}MEm$&JBG9-N3MOVMxN&2gyf~|znZP;>f*Sw1#ZQLLXN7I@YPLzIBEu6G3&NzU zisLSCg*mcuds~~q{Hbr~q5VOwuNoE$F`WbW3Ps@J|2i^U9dY!DZ)C??U6DGYov6jO zSy@@#x`T0_ObaN@-w;iRzZ|jtZ^xPT~|ns2s}_rhFp zwbpr)Tj!zqQ|~}DlZIY`(Ww=9Za%ez7;}>$AOUn2j3jTRu`GI>2bSB3MlPkLA~0aw zbI6_HHAm@g1v>KVXE=_{KnR0%n#5pkevYR=V)_j^FRzeUfv3SvuB1Z=x{i2cNL~JmM zjti1cSZkn(gLUlUgc2+`GMayl@?hC^e3FtL7)!1Kz(*pUehq|1HiZ`eeC#i}U0?6s zl0c3YKMK4`Ub`?2@w45UYV6 z7zX_@79qttB`LY(n2j3m-?vX@*Z08u z|Bg)b=gcUBi9(LUw4PuKz<97RGKT+1AMg!V^{ubAFD(q(n`XiO*536Kd|RtA486^K)Di(NNT{wU?+m__o($!^YI)T94qV(l&EA@u}F2Xh(e+1Q)2}hCkS$UAjq?< z)mVnN5JL)`y<7jPeSiCSp6D8FDg1{rvgns|b8~kufIyNNLK1b8BF0R#lvt0VhEX~D z0JDPf=!|Yj>&*FZttNI%+#m~d%yQjvcmsG@o%rc%L&UD-pSmW0o(F|HQFl0&fPV$Y z(`5I{w&@!N&}N&WON-3#;Q=nX``(aAd=tlzWd=(wq9fI5ano5p5A$G;F!UTR)TRnC zjT&kp$f~9jUIaD}iLIIKN1^WcHl-BGgeO(PBY5t=?X6( zA{zQn4h_n|2Opkemxe($BP=sFtcc!?+L!@<;&3X)W*EVdzZNDA{*Q?#hiCz%@=n8% zlCQ^&A7A*vIdc}|S4Icop_GU`^lRgz?JSvDb#T|1?uFGac=_1nHms#TumamW|514i zWvw~OKj1c07+fKc7clpN8^sR$|C}y(tqjkZ?55uuD4YDZk_)h3oN(c6SM)lhfKmh$ z&}mRsSk@K4$B-K+-cELiC+S@OaE`*Jpkrsu2E}G=Vl>`*S24g$gD)_Tpj)CB1gKkj zXX;+r{zwjqO>PBh0}Sdc_+TWEU0Sl7evew0`0X3)UQd!lB?GXcl2=a~6IijJ5x*@%*=^?6DPA}G`LTI(xLN}q)d~o z*@rQA3p+qV?JXDx0%sh!(B7F*UD7nC0lG{1Y|v{!pDf)jUAUl(xdCcq(I?s#3o*$= zY$xKAW{I2F zePO|$k*w_3OQTr%WaIFEA8t=jYSU;|V9?0ly*b0*VM$$*#}W)4hK&;1VL1E?qc%bP zhRTScb=iPUYyg=HnPQiU%8~yg_ssHWC>-doF&9feavk?(3UC(tlE`?N3&S1L$MWC4 zFQ588e1FtoRLX`8C1G^~Jc@lkOW`Y3Nyv$YlAA(fUIG?Ii#`E>16mS~@}uA@_)8E* z!@B@^Er{R-CRf|+#FuAnn(}A%_xP!!lhZlsAGLPBpGzEAPJL^x5~Nf}YRuHc*9$fW ztTuG=##^^0wn3ScwtT@#8cks7&^a?@Zadym|Ryx{dk@D8EO`(f#0p%<)9H3$N*J}ftPpG**!wJ= zhJ54zv9w-R>jen5&SU)wJfS3CgE?nNgDIu<<@);IE!3>(Yg^;7e5xMX8u0CxC+)N{ z69?-4Z}ql4aD+JP@`$k?)q?=g%7Ck!B1?m>s?4pnGp!mns+{v3#rtEOZS_eT(irfU zoy!nYS4S<5pS{l2<-ZU<&bafyC{~!i9A)M2-@m6WhB(bokftoDDoo2a@7*p<_*ty= zth@3K-NDI(gaqFqGNi0QG0xJP1T=zLHD^AL%eX)Y%bn=hbg~~%_9@!K=gBf34z{tM z1B{0T+l)$P%x4)kg*x7zr~+Js!QlwJt=d5w=1|qId7H=xAh1m8V}vAr%oLE(vDITS z*KFN7Dg4(L9EC%02lK#=6_H5g?+3IQb2G@i`Vr;$yLabQU%U^1n}bNvV8)b%AhEZ=JJHK~?WJDPah7#&Wr}TX*JLi~9-cn;ahUY;qspv7=?i1%3Ue zIy!558`{+>e;mrt>a3j++40f6XM5l8n4&En^`~unv-jUWnQ6OLUw?X!+3S|KnU;&6 z#03fVc3w$w>T93eTW7uTPN;w8Y`2kD_aFzLX=ze07zTR+6uYX>x@T66@dDS zR`$JjjLOsPYn2cdes?c!Nkijd_4VtHtgHo100;&4R@6!3BLrzgkyN7>u}#^XOx3}l zik= zK&HI@Qb{%C!z@r=Qi!9i26!MOPn!ca)Sf1M$idG5YHb6@lwd{&_R#>lM{iuOCF4q2 zeIj5r+3`FNZ0-=^-mp29f3WeEbo{~wV@do?m`Da-eNvCfe=YmM=LeBi(hQi4brhj6 zhsaI+R*9H!9G=Ga?}neuu_5=5ozBi?olvll`}(P_v?U0`hR_U<#!lqy;fM99Ap8)R zTk%|B9xGZD!ym8-3Z@@8ycRN%lEq!O(2kGG4?daGGQ+vAJ*wxcBDmYKQ(k>s9af=n zXL)CR$D7U9YUOo1+;uKnch^p?p?Gs& zz;^GTw_o3^JM{m^`pU2<)V5m$1eFpINdZAZ>F$t55fo4wq>=815kWeor3FN~JBIEM zq-*Hz8itzljPHBC?_Af}zxLiUOu!TCUiVt-UU*0IJT<{1r=dzY;7XE~c6nep{xk7i zuASCL;PsTdY>iZ$^tR)Y)iFjVMV~LV1{rUd1kc7WrXgfhxEkyf)5&%T`p+C|vKxw& zzhW4G1?BLdJjd;0=73j*hQovIbJxEdxOL+T?9Kg6j}Gw^%xPFOl^RxSDY?82qGN~I z9~D?TA)(jZ;#bneR7lg3i@>y0M)upi08SIBv&AR_nNh(@(c(9jo)I+L_Kt3y>A!1Vzb!r&lz5EBM?A(X|E zm+`7AkXjEYYS;(~^9r4gN^u!vP^80uv+@%)0Qc9|J`G?LFi0oa?e_Bk+y&(5wv#~n zZv@IXj`suIr$;u=LD`xE%%C6+(gqGzFpz2ecb^P_GMC3d9`p-T zZpZ(cNWfUF;&H+WK$IA0lm5HkUH5hVHPsDh!F`nDQm5nb6+HY5WMi`cKdg#Jr06-R zpE;ufim&tpC7S~H4KWOX2Erg72YX|zF*{2|LZWX}%f2q^G;B{4CVE6_qeRWF|oHhTVrGS+& zzV}1|Sm#iMh|0Mo!Vh$)`ap@|UX7!gpF>Y$+vI%dajr)#(Z6s{Ea`{pr>&5xVO|X>7wcu!+|OpZA}_j7S42QyX98LapN7;PH33*1?O@ z&C0Zlm+F(sm&UP87V@`+qCcsfH+jo8-)?@Nvp9h6Tpfp|mb;x;A%N73|L3r^ zxVP#y#6x9>y4Ix|nm|kYFdPe=iuHO1UfPE$SJ11+y)Sm%q{*VKm7-@^6eynf>NfD z+KJ}Z*55pQE`S+Hb=XZ*j(EBgd$QsA{K?h%K@&KGdLXJqtHVgINU+twtp!K=_)YCi zych<)g$sDcMb2_c*IK{mu^%9w6n)(9xU1zSr|5F6^`vky) zSxNI_t!GYwq8-&kWKagOL(t#&Ut7Hz@WDqX^N$Z2gDGf0A|?&M@+hpt1|2XVDBCLy z1}b0~bl~C^K>eNrls^E51OFMS*NV!oM^WD`m>?6kPXX0rJOefn)ToM4 zW>(g?e|I5fcC%tCHVEBZMde^?&;emED*7urkjg_|mCLw1KK=_A$!gVPe9-KmPdjN- z^pKh0wNJ(MJ~jTEU8;jfnH2{gByXM5MS#djLV?L3&I)9omnyU$8{+zI`MuFxI`8!L zQ;hN(hyaaxg!2NtX=&+q%Iqv#a)4-Fl#s@U z!dT39W>?oBQ!H`3X;(vLTifKZjA?HALn(FiBhQobz+~AXFe1o=kWjO^QL!1`^z8@H z9nu%KqMSZ?Lb*q_x2CD!U>kfCNpY+WsZvgZ_+$#Cy37MA87-4S0USIlp8VcEvq7i3PAdJt?lz zW(K$zIXwQi^c?)xLPyyJwlYwf$W*0VR2K(D8=#GY8LqMQpqTq1mj0*qY`Rb~?rly6}L(p`vV?>#UFBUOS4#rDmkt6?sL{1|~ zND+Ya3_8MqYEAm{Z=tKQgEAB~qzNT1nW{1Ux(6u1eyT01d($7RBv^CsgVbK?=ThzN1srd=XE^>x?s^0>%-40e@2sMx>x@h_ z6D|wAUP<431nFMf(?bC-QJrTwao@hHZblv2sd*Hi^;cEQkNRx?>M-gO73UVAd~YG= z6YrJ~r_kv~|6Mlzu5}@7Z@qA(j1PvD6u>P|+DaiaFD-(_bGR_oM`hsAz3{ z*Qt^EnVE>fd))ffeHr4(TwSZ`SN2Sc zS{vOW0A$R%O_JEi==ce{O)Y93cAdULw%_?Y<9y~~U8T9|8(xv^P$!pym`1_K4S3a? zt1TNs!mR_U_;e&9;TUqU^_PpIIw?r5n2 z%q&Xvi5i8%_kxe7>msS70%W8r_cQ_jNz`xfvLK-_76@9y2Y{U!+$3NMZ~mBGNy3Q+ zczhJqQ36ei*9X4XR8fGfczaPYj`IG4Aw3}8M{Eg-x`}^HfI)yJATMYOKv6)LrlE#s z?DlIy(vMx2jn|L94`qApD=~II!zfGX@2ikjEwBM3C-` zpdf+c3gdp{=Wm6VdT|;>u6lt2kMH$PLZK{zcc8+G?%-P zgARf>CZUZ;er$RkQbNM$zxtJZWlu>=#vO6GkAx|pJ7x`Qsn99bkwbpS@ax8J$8-86 zaqa~NKDc`}J2736?65OuLPE$!hSyO#h3#ZDOF`p>>rT#PZ{7U^?`5cmeDm#%31ZQf z;5EzT+lYDO*mGX(^}fm^S7gIh&+zv+ZIYglq7#$Lw$H9!!>+fpQ$#W4(V#4ThLAI@W^2E|%s(pgqgWIL#1Db|*hP(8F zqn_TM)Y6Y2(}?Te3ow}Ia|rVEeAXFpO`Ftoy3frTaeMxiXRqF)!6j~^HxjuaM|8U& za-2Kvu$=YgI2n5N+Hj;67yfOH1oD9$Qodvo-F5{bx%QpiiwdMV?g=5L0y>U*N+Bns z$Iyo0lU_!3(80m~M^daIHqLxtAkS={{njwf>#$V@fWs@OI4yD=HWu{v^JK~cu!4H~ z{zs$Kge!BCWC{X#+Nc6(Y=q61`7T&e+7G#Bi66=Q=1@| zk&?$kipy{<+zcEpsH6?Fn15OsP`vy{D-&$m?6c<50PGhhrYP=UcloFm*wh$b4+E_s zm{Rxa#d}kfV$tHiPQsQ8lqL_1v-pj2&;skQb6d##@7f1MXFMqX6}Q0rE>t|D3Aok%qbES=NrCV6!#`aan2e4R%lri6#I*iD+cAiA z27DAp**-tXVn4*{yT1H?lv=umZwe&PN{6pNpKGG#WZj zMjkN-V`kCP9;YH*;*2+@y00~3wqlUOoLhAh?6kBI*DUTUNVtEYyFoPAL{Rbc9fzKF zcJ}yX&2O&W!zsjT#}Kb8GDI)(vX;fYQe{`Ox}n*%H!JS=SlsHPhv$NbU_Gn{GgM~z zxE1y^*8CzFD!z6Sx~h*n{hU;DxzPRR`}8*y8Rat!Pq{wLmzFAC=e8LT6f~gEts5_<2WCJakB~cwr>+EG>gqg2Ke!V4Gt(iA zjAHdHN5AbG^bm7j8=CpagL@N@V@?g#X4-3vNLcs&C44>8Y`x$0)^L1p#&wz<>T|U# zda}A^(yM>n`5L%7)1q-m7NXM9kXp^x$5+P$$WnzTY%Jtkww7%--AK3Ha|p6RT-#;% zKm_zv!Kj4*7EKjeT3TIQDZu2y#X#~Z=y}j@e9#0-5e*Ioobd2C^(1l6kIsv-(9;X5 zXl{LfoBZ0)(vT6=Vy_c^%Gk5>D@cGPUFHAeTrjwxyIlt177oM1V5 z*~uYoIy8{*p`g!XUkC1gLoO=X4x$B|>+FprJUpeAU4)I#chRRC-Q&SGQSK?R-%=@| z6kd-({itaLXi)f$jSqhR{(a8gdw;yZN6@2E)vNg#8D$t~{uir#Anu*1;sz~0CHZrs zZlRpIIyoUBEN~!eLYpNgic>kCrL)qGj&RZh;?F-!x0Uzs_`2*gV|;n)1x6tnYSvde z_1QY43puMQr8T~oEFq9r0=4#leJ@?)Ks>noUJP=60QD_CWi0~<`(;jnrTm;tpvYkh z59;N2&&!Zc>s^ZU!CtC&a>4KY^G^EpYlm#;pJyYl;i4q!;az~kQjY||ko-&|?-Y-7 z1v|A@qQ6hO>Tkw{pzHdLx*Y#Hb;eT=c|xMDnKq!Q$#%C>us$pN4bP*Pf^P3oPa}x{ z^jE#?ebL9?WK=l2R#d?=VJ6VIM)IiU{b#|ra`eLdZye_FlMoVRv4i%?@-{GHsFTiS zbn8NfKQ`U>)MWboEq5;OC7vX{icgY0;bjj9QWo`J>@1Il8-uk;iOECc`zOWjUj!KUP{;1hvDnA0 z_07et0a#g!#8%9ZXL@P8QhZZTi&cGL&we)5*c);S4uK<{^l3E$53 z#_%K8ybRwRAD0_1wb8n-Mo%x>g(1sNa^5sKTwhQR`4)0{y7-D5_gL9Cbc?ToL7)Rk zHiFLi+F(IEe;m>)Mc}>~%=q%)iu*12nBY$-nfcpI#J${DA-GUuiR^EHzLZq-?8clA zM4mRfc6EIwr;tT`eaQLE+qZeEt6|p%{c+4bg7$5UsAU3yXq1;1s%TscBDR)0JGsol zU}iDS3!5k3a-a}jPy~a4cDOHCc#O?A3$3R9Bz=^6ijCC;E2mJAK+#QsH}m%VKD`^Y4`Hae_8T=lis*W3sA-YwocHyUK_05N4T=| zjNNQtJ93#tya5zowjD04z*VR_S$Go6!y1m=-3=D(-r2(gvV+OhuJAisTlZhv+eZM` z?6PoLr1HdpBhP>Otj!8HX8<6+1_G|pg$17mki+{Eunw>F7FNYMV#89D&Ke_IX90!Y z47$mvbn&D}gwfiWM@~`Eo{KeAIVcK$#Un18K=1ea_c0Ce1S=gjbxHPz_dqd~6aegJ zij82<-D{ECJ<|xj7epYO8zuPs$M~%`-(Fh& zlOO_v<A5;3JeS zN&p%0CGQMrc2K;MT#FR{0{#BWdVBvtEV_(VNL6kB;WMLBJif#RpN;+VWE?;1u%&&a zP!ii0y!m7E^l32aOa(TN&oK`?#i=93s1Q2huJGRZwf&g6E3pHBhs{L`#C)zVhCUDI zGO@Dap`wg_h=BleX=!OE$o-V2EJAU5!cowI}X5a9s+4|xYOJCp}V8? z5&LNb+2cjp}Z7tSS|1)NMH#!Id8(UCB z#F$i6=ph^i#dLAmqxM9O<~+*I`H-2NeLo^%;vSA;`^C|##gkc|;_aXtmPZQq>x$28 zV8tMJhkJL=JyDn+H#2i)*tO<^MZTfcen`j@9-g4ZP4n=mhi1~U9#!shJJlA0(a~`^ zJ<&`1Lvm0bDpKB;8SN)APO-lj2$z>1M5XITUV0EVHtuqI)v{yZpIS^h*v|cyP*D-= z>rZ|b(;H`0OhC0e|BG|dxlFA&wYY1p&cfg07#%Gn1vk$=pEJMFv-Z$-$2A zZ%>cq?FAOJmhg7L#G~tnwQk8zFx*XC8OqsT7`|wiy)ZZ&&=a`q=Z%Rs2orQDe4MMk zG_Yl<4D=B*x|3biYaZGJ>c9bV~U9% z$*)rinzPLL&1@N%I`}}1m?xv|BU+vYmbM&)8>Zm1jC5?u znW%)UGTAQ}FgmpTin z;1gCX(Qc=udF-s~u*v9x7QcwGl5JL|g|96lxgEAo81j{E!ZcWS4mL3S)cQ|M1MYpJ6Aocu#k+E^^t(^Y|HE{6`kxwQclV2jJooh0$CXu?jOm-JDbyg@A_e1wZLFy z^E>8|Gg@~*4^qO^lr_IVo90RX;(C`HSlF`R$AJx=SIrpntca)6Osv+DUX9C5TH20; z4;O=?=`aS$BSS;Tqf&H&=jMdQCLS1=bow?3QyUl*2L%H&F}%0>`)d>M4OD!f))tT2 zM@Gog(p*TnIlM!5PfZ5YH4jHzjG&IbAA@@=E}43MdW*R z!H;Ms8{fG)+X)C-`q{fN(0neI$Uhd58%Li#lmt6AYCm$noCx)A4@U>DuWVl~c%9%r zctBoxyRl{xXqHA6R(6UxlTlcx zX_mUsG!N!1)7c|$Zcf<(+YwrpGkddVLb(_%wf4S2k@|+E)>d(`Jj>rmmP;MY7@j{O zTY*sGbNF~YY`(Ig#lkyogF#>&?F7^xZ}r(A-^PWM(%D)(4^q36xW7eh-y2vuao1=R zpiKAXo7j8{!WxB6H;)3#DM@NbSGuuRF3-3cFAOP_QeTe!2<%^5%nYPG99Lgae30_J zcU7Zip8Cl5PG65eb4z7jb->drcvGKwyQVW3fp zrG&aJ2!;Y8P`Z|ps{yvpXW{OgTW7*IRcm>C-z-cGy=hY5TW#K`#HKaRhtcMt`kDD@|S9O@bmA)^mDGn{ckYx{*7+7vzOd`Etw|ca)+oh&6 z6YYEHFhBVL&5poHa`lvo^h5$OB{=(|5?}X$&Bsskj@=_pX2M6QR)>FTh(?TpO3Ib2 zKNHzd_R4JMc|Bez^jG_NlVnaj1tTK*h%bp5dg3Sa%I{vFPwiLBtBPlYud8mUA1?Wl z4HL+e82I$mseBcu68?D&v%DfHeej#P?nAf{RHwTLMRKjd6JteceI1FD=2hLrM{6?I{SQSNb5v$AN>dMaq z>r^bM`1!?}FD)oP7W0S+#CgCDOQ64uIz#S&6@@!2P94Z?R(T}>M)7J?7g9IhdNhs% zzFA&5;WTLbXd=iDfA1eq|EW7d8;C(pw5+N5*&eFq23WAlF~u(CrWdaSQWrlH*T1UA zW)m^I}yl(qU5ttl*;lw0b_(^6vL?XJU%n64^L7{HyCxZg21z{ zL(M@#`uf-dNq=$ly?OD6WQTay_xbbRfTNXrI7PzCt2OmS{wFGCX2GGfd0$aas{KFM zm^@eRMX1eq^|VU^$@?OY@}SWXAAH|e&qu`=58ZU;9F++`PLF_p#ir-xW#j7l#B$w+ z5=f}fNrk;xAK3bK08Q_CDNzninMm>GFIASdKlAftY(?1;{C+dpqIMUR_m~^U_cF1x ztSnwCHyU(mAAtvYdR_3k;)B{6+sUPzgyk{_Bx0gEjgK|<=$7?NY_<*f9Nq)BRG=aQ zMuMeB3nl8@8$UgOQ3F+~3cew8Om9?5N?2lIqLPi~MroDFIrJ|2E5<5-Z09_duC?H@$vD^EmY={ zi;IfB)zr8$R)>IRai)ib;mXzlfu+rNHT6R4 z++Kio@pkzNehR@3n3Yx_ups-}*x1McbofdZBM>P)J*q4a^t83`2nE0D>5~)#fZ`~l zE=h2|C-#Nv!-o$)MLRPvFmUyOsY(HUQj<^30#IMZpLB9=E({=5u3s|J(&pg?QZo#U zjMyMO;)~7@W%uXf)!v|pi2dx3gs3OaiGo>efxt)JRSD+cYVGo?NY3Bm9=f9D+)MT<~HhHiZb{Yn#j`JLKxtPIg+ z(0O~y#L~D>JV&0)p9>xsTAC;#FLI}RS_I}}(H%mOc>FJV>+8-PnRRDN>NLYp2OjkH z^{cmdj$XnfB-(`Mpy<#F$((D0oQ`1x|E z)Zv`tNNE{oN^&OM$)1L!w&Rm33)^CHZ)vW*qb}_=U3R0@xJ~gF{^MNOO7}a*n*ifb zM=39u6dC;{EFGf7t5&tR=$dc`O+=e{z>_=vtkEiwyZd!pwNvHo*ap?6{}N8y;LJzW zZ{de>rjljlI=X+*($k7Pb~bcaWsXNg=IY=1TRau?)jG1O4N6%cB2t%W6aC|+M^L8| zfusF5ZVg-%Uu_d98u(i!9aAl=_^|oKE+V?R->wKxx3q>w2E6WcI~qKrt*Gjr?x)fd z)p?($ii7r>tCaQZb3WlG*knSKgQROjR+^MI=f)YS-<4{@k7cZs>lLi;%Bx$0Qf*5P z5C05fj8wEgY^Y`5F*!hP|Kv&^?uheSB-^!8nZ+fx ztlPulag>Yvw_h-2Zq&egchRD&PAEDbV z!hC9`8ZwZFfn8qab|l8b&%B@xfL#Qo=9i#*$|k8{VHQ>kDG}aR<@Prog2o<~m;lk( z8*(go^G{?q)BL$P7Os|i^-2^3$38#b>z}X)j3dVWn+tD=6jzdtq$*}$g^%T-Th0IQ zZ4Y#;a^T6YsH$ok&AB-|pXJ>Ayr^XmaX^HJN4q=cjw9@1ybPW@Mp63<>#iU;-2#^L za({-M!?o!d;#Z1lh8BCGMN+RSF60owC z08$`&d*f&EEcQQ1JAZy;*K2mR62n>3|Of5ufI2Spf|Hr*fPoNFK*N}(*~KDQWNQvLG9XKBR&EFh44^_>0#X`%Lq@vl{uF=g(K^{pOzXE$(l zPU{($)I0iUf2NqaCiV4O50@xT_Y1BL4saM5KYSXPsp@5@G8iQPa1%{;L&hII_?9Jp zdBGFDIBll<^PTdcqKpg7u2G&FC;wrU5-zSE*9}s638=XV~r#Hk|{SEH41hX*V(8|AeJur&3 z#^d;wTKE&n*30P33k}jGMA*x4^@nGDSD<=w-gQfk&dvrH%M)`Fy{tDIX4PCn#D;?_ za(@8067-y2IpXF1V|4H2&an|xuRZgiWYNV?kS<6qp)K}xA!O^h3&i6NSKd;6QGO`- z;f=Sa-REvrdH?z=9GSci7bkgH0yQt<#<_T(UY!Z~=+qN@&c&f2-mytgq?Mb=L-&HZ z5!g@;TL1M3i|$_&xIE##E;VMip@QO-NkEYD4AFEOuV3Kbe7X=|t=06k-8kq#y#0(7 zIReU64dkeTmB&i#g7v50^V%>`KV9l{IqT0cLqf#vXBt;y?^6VS&bWXRo{-ygSHi&| zXUN^c0#l=E`yTN2{2W43lv`_K$wth9zyT+?ufODr+v^hsEc|NCrSnb}2pghDSJ>>sb(rmzyf%)#V zXef=fE#hg76R21cR`PNq43^rOJg#WBU%yE%^*F<_cO^pCqJXJq_DzimeD zqVr;}=DB}h$@tYYKbVFx8x(epE$nK#JwH*zawIm}-W^ERLmw^wPIvyc84U%6h^J;| z05$%Vla-V6y(LF6@_|@7;?MVYzprZWP^W_vHM;Nl4Z0&eCBcIS=m`Z6_Tu{2n3ythkDeWKf2GR z-#n1W2UODnLRvO0vZF^l5F>kg`w`t}6iBgtLQNgHZo1B`3FN|oadB(dV@UP}?<>A6 zN=nLspn`u#*B0CI%1U&4`f%bzt}=BPDQ6q)4r-29hAfW7^5o=xsg#DC)G(!2Dfum5_aPn{Ovt*rB3jY1TJ{1rd2{>;*Hu{l}KyqfL zkYK!}L(?)J4G1i1$ee`LuB=YITK);L2qA<;aGU#f1vep%_Zh>wexa80V;PytZu{c= zN0ov#SQ(@hm$z5&l?m4Q6*wPX$kBwhSs6GnCych7JABxfes@Pb+d0~+QGajLHifVP z9wz^Vrpd*5*Wt|HJ%=uXy?N~*0G{SWuY)^*F_+*I_Jz=mG~v7&Q9z(+)E(c8XB9?E zQiL~q?3TR>y9~rHqpoTGCHqCUp*(htnEA53ug~E~5rH@(<=fT2zF2W?GD>1(R5v_c zwVrAcR^34Ieo*6~V=%=4S5)@G31!94?R0kkpm zT_WhheqsC;$xvZ}%S(ObxAW&d(O=>{I;(7#^W_1@lgtBswmwisYxRU~t8}~KDQ^}d zxMHVgW}fh!i^ys|=SWFkjFcmwGtd^wKU>ALpeO9LbGCirqft*nwhSRaNNjk;FoHgJtpY*1f7kqsezuiFtXZ59eUkhiYn^z`*7h{WX*=yy)r z1*(6DL)ssEC>^TLJUq7&N!@vMklsCBVB&FA`GuBXC71|7D(-_Lc6))o<1&ZhnLs)P zDF)Vc_2+7>iiCaD_fg>ak@cP_7QBUx;o&00Zu4D)!Vbqc zILAs2I}LVc6T&Y&+S7x-e#JsDDh!IBCaGbi&xoZ0i-AeF&IA6r;1>g@%fV1vXDB_^ zj&qLDT@G8eyz_hbF0xLQ+n4nQQnL>A|LTUQ2I^rq;{V_ZdG*8N{-dzf&5s``71I|m z0BO@Y7#_L2EK4a6M?C^k4WUT%ss|>jD}0P80@*f-TkQJ<}zM-Oaojc2W>ik~b6;= zCYF%F?KQ{g-VVN{C3E(#k4rF!Y8*eKs%pXE05_x4K87mvz^80YTezk}g-1x?XVi{(MMAEkLB4o-XYE0YXyG zztgf7kylq+Qya62L#eW4ttB(H(85xm5~%+2N5SHSg>x`fxo6l`LQ;|w+)!K|-1$vP zDLo+8O3z=BAuurT>3%~A0T%8d_%AsXmAlo}{Swwiw9|)!$#IE%*1|*S`ly5d0~qpw z^(NuG{Q$~%2kjQYco%SQSQ8UnHR=uyciHqm-dzy8{QdIx$UIPLYYq$*e^7#-R|m$# zgb%GvPis=>zyLkaQmh-&5;~HF!QpJd1PZW`DJDrTuRGxDN?Ka&qCi(iaBCuGR&J{a zo4(II3cl^Us53dcRVp?`!~R)? zg472bzGkY(cn`wD!nz<*>afLEdZlF*vd#vgHkAERgQy3C$_ikV$ zHBu}KV~J=TOPG=3UUKK}fdV-_L$cT0&e&C!Gt%lv**j_xB|6DoNEg0lf&^?&n86dF{^U& zX~;nBhoyx7&6QoKP>PVf3ho;wEbX=kjS23Ke^}7_N6z|j==8gZ_Ti;$;9&E z!|GH<{1K~?wBMIoZ5hgK;KAbxeJ+9c=dUh$&;;D4GC|LwQWeqD@Ru=4Vf6xylelH3>UpKevVj1{jMLkL39AAZR}nB*?9k_u zG*K;pQwL=cozM(6SakAl3fkgX$(WiP6^V&?4M($!Y8}>Ixw%hQXx+NPbUT&BUwj<` zFWd?af6#jZJq2Tvaf`ckx!G!l*B7i|8Kr5Wx*Vu&8raa<)%5@{^HXJXL`kk-0(3mFD+*oWv&u2$K}Qk z05*N~A#YXUpHdd0jsR&nPvGXncc@#={00!DPVKv}%1Vo@+1~FrU4L_NN{xGcf~4~U zby0+RilAsznh&9~GwtfS)-&PR2c4aR8nt%pmvA`8jHzR3_4>=Jw`*&IAb<%|@dzP; ziMc5jczLlYb`FA}Nqw%~R_TMtYSV#D-}hP9OKT>mRv5_PY)cc>d>Cp1PEw8fQ?%XP z$5)Zzr>7QRWRt?!Wa#3OK_KF!ox?neWOuJ9I_?874g8HtFWKZLA>m)n48&iatIH88 z(FgrL_CtO^Q zyG^W%9{&Ah-F>tY%Hd#)Rgl zkNRtGZM_o_5n*nnmI?tGDN9>hpFgpIn<@J$C@ic4*cxKjKo~nv8nz7&mk?=>3to7w zV}N072!DJUo|uCB65-|bazX;9!*-t3K%lB_C`$Q>6xeA8|i9Iq@*O{V}FcM2q?&b)G%OTp+ZRf-A`)%oIP+599>=unxnL9O(TY z-VZ>SqYk3(SflqCpou1i`qiVOr+{0vW$W)Z0XGg31OksCl3Pb*6g0c|Y1ozH652-I z2Z(uRAD+h>^+eq%Ek8DQwFy@ifBP;%^C|IKIJk6apLdW+WttGlr&}$> z6V`3NKkv*ypV;ow>Dl?gu&?ybDRm0|U#-*ASkPM`8B4*HuDhAG`mWiz%w5uY6%1q* zreze9W;d5=_b@|l`j*g`)$@;OFfj!()!Ef6E9FUzQUm=3b|?$x64y;RQM(@O*R|Rq zI=RdC`P9hKQPbU-xV(Y)v`_jE58DGd4gIOFuTMAV%*`Jf>`o2#!ST`d_q|Z)1q0jT zYyTPzea<65t}u1KQ&uYONaRemLmXq+xz-qsyS!PQ-3l3qE0Cp*R)iF%#(S*WYPt;7 zI}V$d|Lu9N`rW{5;PPYM5u(;zhw$&l-+p2vYVsBjJ)?nCBYLFlU1`>QN4pY(T?ZLW z70WAaLHE)t5&I&6d%Fk0Ew^cQC$5;V5iCPr%s9lEytr!6wlE^uKitsDjTc`TDNG?I zs(Z%9&YdZZmFaC2sXi+Jg?EkyifpTVC9~FbHc_LqSG%hd}$LrU=mf>0`-WG7>dtmH0^PM5PV_tST^=O}-mW;2-=K(V$ z)wJ@a+ovfa2J)Qqm-{y%IxjVvIgFG~?HieOdO_c{$9tNzLh8#+C#;|F1(#h_V4{WW zj5XF68kksL^2(?G9DAKIoA?0D?sU

qKgx+DZx&k9JNZI+vvZA^daFuHA^DhQ~aG z@oaT-G}~Rwde6)SJ2-f)ubDn$lUnPCA{LnEM*ZT&2Ox1Y!r(NLB>)#hR%PY=E4Ufm z(vZ6xL(B@`hg!fH{U-j@q{>TN96OAh_zUP-xOvR9&in{1FE>S<+ERfIao$i-&+@}D z)$THzc~RK}ppxMNgMpYj*PE{~c@baPqZ#v1zMlzL=nMo&L z#(YS{=0F>m&Cb5;m16~Fjm`KcJAe&dC=QYi1~{8 zQ2-QMSM1-bGfKO-uommFwu4eb_C)r_mBwe=G+D=u?-nJ&M1FRAN@>kESB{5Z-Z)D& z>NE!&hWE(4%YQ14dfy5&oFhQ<#hHM=!R~@6K|8OFTl0w$3*i0RL8G~=-uZ(alnn=e7^8(q)PmD#xfT10d)vgM2ptV% zED8dF?Oikpi7#0W_4FWLg;Pzvz(x=F%^x@4xQJL;C+T~NA&hzUp-42Pw1IAZn2tfy zHDG=}%|)RO%a0{#55Tb{3s62;fa6982c&s9;MJQ@Oaf7-dy~bYzsH~Gf;naApz7E2 zac2TARXHJI3w`SMLbZD7(z)~PCa8rD7d*>^Y$@-AP!tk~xxI6*2yVZ#;K7V#+eDm2 z>~#uS^@p>z!EWnGL{YETdiwG^L)p6w_GY%a_#b{bfI_jTB183klEOX|VEwRuhzrPB z`Zwp`Vk5>BaYcO#H19|HiIq`NjQjGi?cs`w2 zGw_fA0$dF7{0_A@kCvroq$uA#D6;q-I*LDDXqwh2rI|CQ&=L>P zJ}gRthqh?f{80(H99pJP)^_@(2-Kb6-7suyyucVc98lA6-GW(=Qg#=1Y~b+3ixFc+K zcpWcT`o2LkcQ_}th9;qxkgts(RGb`xQQ5N#$yi9~KcBACf|8#+zM}=2xzAiU8#4Th zmye4TdrTZ^v4xgeZ5iA62f~k#FO+3hK%2(iC&n-thg^EeBy zE3pn;C!^a?*lgDbow;&FT)Pn;3&W@Uu-pt01TjmCeUqB7knYZ?X#>)-TK2D(I|Ku$ zX7}5Y?kw68Dzt~l4LsbmX`HmydTK7TFH$gF15BawbkeWi4#Ls(SAJbp=4O?hten)^ zKPR!bv^8gwEh)Gvx3l*QD5nI1BDl8*7S*c(zHq$>>NUYC4S3sqH4;7vhB%`QxsAiN z40k)bzfHP|&OVOh#RtP5<5QdV$`1k_ev6d&c;BWi0!pVB=k7j{k6NxerQOtpXI#6sJ}iO~UA zoG(O>=P!mE!kr}>&n7fMKL%gIKHPI9oUpW9rD`iL7i%S24(rqUKq|njQN62R?hGc# zsz~_Y*?e>U5^SyjypzuJJ;#TI_F-VetGUSKCZW~z&nY;7FK$~oZiV``B7bn+ovNZO z`Vq)35Aqw~VvST*2B8{4CQM-ht6lbZ{9b9H`(ie@nq_8W=xrdyk=H+u&kmQGr)q4)8k?RvP27F_ z>YX^!>uGg$JvDxrG4wr6)TgyRx;`=VKn$;|Jxmd#7Y~TEWz#9`0#i6SEn+d<-F2+y z`{XPvh*u2mzIkxZPQ^zZn5?6Y4N3{=ya=hG{$~yoMGe6)AJx0(T^$#InlO9m7p>KF z*!B?U=?uaL>58W+gaz>DTE{_ zJy`L+SYiT#0(8K5Q=&0Jh3V-rcY?c5#vDN|THvCm~r!Q~uFg z0T6Z6{!3BO)c5$fxc*sLx4t6(K(^3V9X2^hNpvHvNGj3Aomm71Gto(Oudx=ithT=1 zMac-fY*a&(4qG?y@H|T>z>))zu8Bg<7L3)~(9E(jLNL-j>~m&jX6j#3)ZG-tXJKyM z0($CXe?das6E>ZD!oueYC5k{#Xt#ppcpU*Oc5Qw6d-s|CgYh?nJrx8Z7W6T@NqWf3Lp}YAi>l7cUGHDL{%UI=WNj>>GKr za$~lKi~GIU=xE@27dh(S>@f~WBnLOZctNgtc*XT`f4d#vFxLlXtw#mgxsp)z8HKxh zk^75PQsIwboJJj`H8ouzdhAbbBrUa+r=ukZ*cLoQh!kpa6Zgl3rQpj{CS z2v@`y6sPi2sHh>-l2S>-Qx#=KatvU>|NPllW<6`=|9LgV4ICq3!#<$sM8YBJM{5a} z5N?m9Z!uxRO9M%wN5kUWl@5ho*GJ|cpu=Eyp@@`QkTDluS=5;lEM?Pi7srj+t^9$c z0$Gx1wPHAAg4}2z^sdUs{>L~;XFn{ynHLhajK~jR0lAW=n5q|+Amyh8?J^HX&&qFg zj)g$l^eNWcvjGFmB80-;*_L5ci6rA!_I&`s@r$bg2lVP75e+wq;un93oM%o)Uf08l z^7S@oNnq=x5)cZW{6Ui*+`P|7Fn@HHTc0W-be7w0apw)tY(2!uo;umOD^hJ5u(0;z z*YE0Z_1`#du0ykT`I+rZlo=dK;xmg2?=Pa^+jwEhy8Rs9l#qUV&})%S0PYK*0V^IL zb6iaB*Ffy39a=f*Q+G(-$M^D9&~!O3my3vfX5GMh$ju26&JwOfhrv?0O&sxi0W$Ny1h1SF4(bP!4{b zQ=KK7K3gk^zK-EdlX47D$eawPcPNYNsw*V}YzU@U=G53)+uDY2q(^oEAo^2Ze10-C zQk*Khw$_V9+XaoDK4kdjWVm~2!Lz;UIR1Bo_OUQIT>`TM6evl@Cj=h@vNpb~MBj@L zC7oz=Z-|aj4ID3+-WWVSacp5O!EG-$xjFSo03C?rFEF1s9XRYAuUO(Ub7a0Uri(e{OyNj>C$gH10j?Fa+fdi&l40dG-48V(VS4JHbWqY$pk4X|Djk?wD5Qjebwz84VxYKB` zy0`a5)N>tGzfbH9_2zZTQR}{oHyhsnHs7%Ec|OXV*Fp^qpGqgAF*Eju);9?9tx**l z5JaG)uqDumW&{=*6_v|Wtv$m^7dVKww9d3{fL;Nx>+12)YX?fPOLLwJuX+BzKwITD- z+jmOR-^c6fTHBv)N`hWXjxI?{HgMXXKO4tu*lxqpnfi&GQfCC47dDB}4@tyjRNkY( zV5TB*ga!E~Ltm0fk1kObdFZCy%Y|rVH z9N%B#W8sbM&%@GmE@k^Wst2SHmK%q!sq zP4wSN$j83a*ll~#zH#CQ(|z?O^<~eeeG1OROEsRtvlQ1mEe*NMPa1{u-@eiUbmA4X z*}q*}=fgSiWf*=(J}j34*D&Q~<{r!SC!VjD6!^7gIIi7&uwSi1(;(U(NGwXic{}=fkd}M})dB=!fB;#$0uN_9~#YYh~+AeIP zq?x1q{o{Sb*p0E5{d5NhfqBX@JGdGr`%58mK1mNe)jxb0j!rL;Vu>{$AD3anj`!lj zCtPvDR(Pw!d(X#PZL)N{z&4{<&wo;nBA^BqF^+iH5&I$&D2e4w5TMD&!xy4a1?UGl zjz9OY2^K;ZKkeOYts475kg37;E9h~5Z*TaC`EAs|t8YM%6cMhlr+NG>1t0`Q*A(;G zdWwT+Sd60RMD)Z>2NbqUAn)SgS|v*2EmU&Uwe)uZu*HGFaFe4E(bu4VhT z_gzzuBb|MVto?|TzJ`j5(a0%$iAfaPu}sR!`jdr$hX+rArXdh#pR&4oP%ufD^tWOZ z{Ir@6SU*jMaqXu)A|o60l?JK;($^tF^%(F|4HqPl58-waLEau%Q4Rf_s;a7j{(=8V zXYyY=9*{QP6+gk-`f%(?829(}iGcJM0}UxDDXOY(YM%@YC_t%dkQjoB&MpC7Ego2( zXaQ;vcNEAKemTPY;uk@_5}qNLN7t*0!wXBR50p`#0o}^P$|#(T-Sp3!Cy0n!2?e(R z$0x4~lXAm;^~$j9;rsH(8UTRC4v-RvH=2==v9z~WY>d$^2EHUfnMMFhZuf~!lN;D* zUM|_ucs|I)B-}ypB88w}3d_*w=nJ@tBKizVKlH7=FT`0~@Q2Exr}L||wS0HdMQ=!|%v1sJ<_6ZTY~d$N~-HDRcOs)&wI3%vgqy z-?MixA=NNxO@%-HwcNJ`SI&REs=+PL8ySFWs>gp)4K5?a++vlmqT>OkD7c*SvvH0P zl30ruv08-@zE~KmwdPpg7rO`j{pFX?VcT}_Ky{^yp0`g-eB#*!a_fIu6Gv;eF{i+R zaCdi|J#q@-lG<@mZ%4$D60TlLkw^7mzx?YTMTJ8lB|eM-ak;;h$L0B>t3JVX#}3KL zQd)rtDev2NA`5{oQrtNFm8z;BxUPq$QZ=i;ECGYkge8|Ta02xw*O@tB<8k9~*Ru39 zlNQFF{e&@s2z?qqq=H9Y@k;^P29$>LDvfxVT-Wd7AxUaC*ysTl% z>>v~GJM&Cq<0H|AkT(0aq48vSVhNoD&^zTIVVtnW`?YClTo-xYau6MFoKL75PnRRF zIa5J$i(fBQz!bP8;2fV530=DqlPLtFp`K*2qrfariZg30vixN4xm*^UnH{bEdbjLQ1p zTSE+a`N28Ctqm9WNrv@KL*Ad_zxqsj4w(3WT2oHl3-oqD-;X%CVf`_WSuElY5n|!m zx6giCJx4gYgzdC8+d#cU<89O8*+7hhxsD6%`8iSeQKh-MtgIctj@ac0fD5o~Qa8e& z>1MyV;aQ%Nm$8od&E*DrCYM4W#%Wi+az4S+mb=7WGi@*u6iZ8I1nwuj9kp@c7dT8? zEeK%vs~TNmpnr4FPuCGz4XiyGUzbi&lOKWwHcLGu5$rl6PgoO@4~gM2 z@L6j`-oIj;1mL8?_Gmr}`=cTw!x@^U1Uz8gcO#x`XJtJ~vwNE(v((JbL#u#Yi9%TM z)}eK0uqh`8%yf=2lJnBa6D|nL@CX^=+})&XMl{`}26_Kai>pc z9N+4s598hngRcTSeg{SwZ zXcFUo{nLcX+thu3%YRs?N`0H_&BUl&-Z@ST6&yC0kYxT{=G_Dnzf)~bBIe6OYQwr; zGC>65cIrh1ppiTwAjmqu2eDUlwXo4q3=$9K{v|sSH;Gf`06FRpKmy}CBbDmK3#@Ku zZuhWN+rv@Ws8;!HL=YK_t}<|TSk`mBAHl963uMC^sV`#IuJ^Y6K7@o)pHD4b1n^7Mnv1OKzm$$u5gebQ9ewDYLcfQER6UPA*)^}L z6?u52>gUg&;~HXOVxH_3xu?ng@dSiU)_&lK_UeGIdk5dYdA(?S*(F5NDa@#g9Dq+?rsf-rr_Y<(DITJCJ)cB2vM>^fH3?TJB>=n1Uifra}O(s}f9bRr2pikg2x1z!Xa_~n%HjABBQ$7H^1KpHC zO99E~RqjT|;X!G~&1G!F5PWZ8W6#E4JYo=$*S)|*YxZ7V-f!3~vChux(vZ~xJ#^RF zz#u@^8#;)KdKahr{B%Oqcn|0*LB7VgiP;b=LsD1>Q;a8R?QEc4t&l_d_ZJt!i@Abd zwT+UtI`Ads)v>5r(SQ|Ztj>ZV$bk5lJsAA+8KUsSK-$4GA4{k(5B5HCCR zrrXz=(Ew&VxM4e!nWy5@q?;StvuwwFD8I@BmFlBl^$;PceacNcP&2$eOaUsL5v?42 zzM+}eK=hLC<)1T7mFt>qU6Tma7!#qc=HCepCo6h>xb$9dm@WO4G7qzU4cU zJw&(NNJ*8UWWXTu+_YAf`|~)d7{;#&SV(pj1y3%MYFL!eTAoP}%ue%Tr^3$7K{BiO z;9y)}$ME#TffQx#U`bD~&`VD`^X6CUO{UXRIF4^?^5vsde$k(Yqx*zJ9ByQ@*H}*c zs)kYpyrX$fo|iJCJpk83Pu^Z9ZA|GKfa%|ybLw*0E8u|jU+z#&-S@ux2u__5^+`1= zOvmBjYL{ba<>4rUo}-iX8gRcs;u!O;;N3$2&>-ej6!VNtqJl^u42#)pUMl#U5)3}k zRaCwitVmBIM#B3?ukkaXFH8KMTx-tkMyF=u=OR>81Rkd`%jejy*T1Ri0_Tu8l;-;1 zVGqSfW+1I@Nc!y<(5!cE0jFdJJWD zE#KTL=VPO%!162?tLtW(JF@oZyYTMXz%|`~ z#rZy{t?`172A5~5Sicxg(hh*l$WvoBMAk}RuiwLEqx@~sgvaJaO5OWWeW8I%)41tB zzeQJb9N5ilGj+5DZE;y4VV-qku^|p@9v^f(4xKAT`~I(ywmD8!dJVo$2n|vL?4%K`i&G;$KwuPo=H4P3|1pCX2@1mNT%)E7Ke>eL$pBf^xULMc*&S#f?XlFBk}IvY9F{fBlMv zH1JqhDs*cO3e`p}Q@eI6x$_wX*|WUc@$J!qI&^$|=KM!NW_&=bOU9vmUZnyB2dxo| z(~I%(;Jx6`2YcZb(2atOm+d@haA!CO?d|O$V;Fg<{vrH(*5U&*7M!BPLMDyd_Fj5u z$F)<1N%f*RO3rr;&COATgoL`;hn_rz&lSrWfy7H}x~C^6O2h74u7v~{5*8NrL!y4t z6Mjv;S9EjZ&D>dDMv#?tvZZ*KM8+z}%lj768znnRNrg_QUXw#h0OC{L$<{Xg7Je

8^@vH>Rw8h58rdc>?9EfZnP@BZ07cch10g4MZh>CtaIr$R<--#V-DV0jQ z6X6Y5bECsT#TLGeb^Yk)DKKqYJ(UVHhf~iOE%?L$Sc|&n{S?&%E>0i1^DWWQ0Pmqp z4k2L^Sy~v)0n^jn@YF%taItaVypkV*fACFbzPUoDsYrU&ENOE$Mw#7KGsb9I#gGSm>3sGsIgtb zD!N2a4{+(=#$p2z#eg)f#dSjgK*kVL^J@}q5(Q9!dq1v)$bpWVq=i}>YM3$-@*P|y zx-0IIBde*JiR(_y>A6~mjDY8?n5YrC5ZHWBxX?}ck9({agOFe?A>8zzN)L6G(N5n3 z^BG#cj>cdgMquC%xD{oZ>lEGQlB*&hfg;YK0X=^F=q{rXO&FP|fjgsQiERuG1AKc2 zyOBdoiGn_%iJ8((9UxRND60VUNFufm6U(d)UhOT~#a|HPDXAn&k_Rv*h90a5g;9sJ zwX1=wNle^AhIXL2+lh^t)-KG?`k2*XyPMgEC_9V5ze5ux4)3u(I#GIWNn2SZ8F_*# zIk9%fLn8@p&Wv(m>;lwdLzTG#yapBU9id3RgPkJP3YmxdcuwJtgP`(BzxIl!V)W>P z#rXtPRYd(2=(!eYvJ!%Ma3R`_F`k1S%dzpdsXA6`OvDhpEc`hkXkUs0lJMS=NTvZD zd-dBjSz<}FDH2ymyDf4aAl`um3@s>;<8@^T>8gNJCP@KC3v?GCru^J(Ztl{69Z9Tv zG&->+#0U+$(ab{(kUxGt$XTLF=F-^N+TvOz9l8sq>*_8LW~5u4N$OP6=XCXcHk3s& zNBJgpvzylC^|h)ZB&zH}R3U=_lA_>&MsK7sgmzVYFE;J!D;9+2NzMg*t^jBV3XGcA z<@*2b@~tsVNv`(hB&3K0YYwk!e3!kLBTxfIP2?bX2D}A&q+?kv@CfJ^InL`C+fPVs zkPa$u9zTDsKaQ%OD$Mb9xzvWjk`RRXmo26EUIeAQ`U?XbNP%3No|+0thN3pADtxUX zlRc&=0gwbMy7Qs5Wa^M22V*V>8%>bp1DC-LCf&G9dp|jA*Rt)z*)e$MTB0RPav;u| z+qP$t)mV~=j=K4?Ks>4<@V#v`u%HFlVhoI?C(fjO--d~`M+ZBc*+k_S^})vAS%kK5 zy9RI`F?9F;scxViBC?=MuGRu?o1&fILUVs>A3~lWwCkuq+tF5M@S=z@Dyev~tZ2J< z9U>rUgDYJI2Nc;Zl^D8622*2J@b7M;2Tvx{yyoCz|K7#};mG8)UMnr;ZJvb;9)0DI!(Vk$|=1khk%DP@- zwHp}oXkt^%VK+Aku`SxD4i+6@Nx8t+nnZ*~SYTF!285KO+Wysy*#kPExRjL+mz=e< zA7^W3G!|tI43CN6HwBnexfZOHlWPP;5`b`ps~eu63C0KuXNY)C;t6hJ<3jtUgyxy= z)%bjL*+;whfL*Dzy`<;k_{S@?7-Ag{|3$Kz!642t?)CLtkSke-PDR;+9h*Q@gYe(5 z@z@7UZyXKjcvE?i^qeC%`l9KhakzT(lA~jr!QT?4z3j`)sZ_}VvWHJh#EpmvYUHQP zCUNs|r--*6{>vX(p3Rfb|7Qz>0&PJbHrc|7WqW?By23*{;lOekSy*L-;sSSGtqnR; z`fbc}Vs1*+bR}(c%VMINrhUdv(mzgm`vGamNtG%g+!N@+q$q?Z@tKTOq=p38h!aLn zDn@Z10S7QKGmLKlDtHRLwy-2@Yag8v$c>51q=@%3Uk5&nA9gqgjABxk#KpL@?{V5# z50;U1cYc9Q5FQF5oeS3qD!Ol9;`ueP7EYD=v-4MT_Q>5uXigFW`%Lsp(hyeiH__-* zaY=#f;ohD2@DPi0$jl{N_Pb~dMbkTaJ_yN`QHNnFru8>Taq%4xnD0R5jY19vEKvc> zG;TIuoTMI72d7>J8H4c_(+3Fn`!lYFukH8XYwRs{%td;C8_KN+=oYS72);21^B~xAZ|^M^w*0*)JXf#fXxH28IZYe9 zs=vM3;5c~4#G~eDY1-e5E?<(nVmW7;sMV8_^Hak1?z>U&Vc4g=x%1v`;rcxP5yxBR z%nWU*&o7zx5>|aE92}h5)x_t&6rjPVw15)?X%qmS)-CFX$tIyFQe44}dgg#@kjee(-#q;H=eC&PVTm)1161 z)MwM_GnUzgwkRz%-9o9I9va=p#f>-+PZ}56_67QZX+%QHa>9`;u9GD%oF=;IZ_Ja2@b3} z&}JAQl^E%Rk}%ERy=U3Z<-^RC&^2OWkaBG2hF0{T4=k*tQU&;x=J0E$D z_XD5*kiUA~$TRC~RtA18&b87VLy6ljVsG8!qz>rxJ?mu%Nu7r7W!q77LgOgJ-RS{k zG5kySzzM$!3v0ar(($iEP(!dVMBD@-(~jPV#>*#y+?)@kxXE`Z$(+vV^Tf$>iT3>7 zWbNK$B$8|C@$o|YzA%_EF)y4+>wp*U%75-N^+=_G$*sB(o|ZmBIkqTAT_ zBVy}1({Vs$nn33z1D91USZlfx`I|9AOQuZf8VsD3q<8%vT|5}k!=GONjfq>jw{h`A zPF)5G^C2aV!=4)rpOCUfe$*U>~d=8j%+-E<-m|eZ#bp;+eB& zS0RlU6ks4}X*IbjjC`Jg%(>M0hUV(Y!WXgM=_S6lG5y2j?OLX?G$6ee=J!ED+^xmE ziSicLdvm)|^;qNwewCJ&NTM>K!NHG#{N!@Bq`>kUHQ@Al>FnV!J$$L?XZtV0UAo!` z?cujssw>a{^7Pa3%VMx@il(7#db;a6{^k=R&FWNm;k)*e*I_EuVF@L z=QS;K-%$zWgPbGm@f5)39_4xIjjI5qYMY@Niv~dw#dz**;Ww{bQHej%8+}))`Ys0V zd>KHE*(YUUKsS*0~J%lC9k#TSn+5a5e^+Vsgf}( zvf0c3Z1(9l3&+C^wNpOmp~!5;og?(J8-&H>{&jegBG_&dLfhMdYnH@qC)Ndpm={kj(+* z?!f1N;iD%SU$L)0pt)_%_4+hh#1`#El*$uyXY2GLPirFQk1sj;8H{mk!-$X`(;{qk zDbo)}L+NjD*FFbiSAja&eH*4KBfz5+^-E5$>^y!ek0Jks zy-2u+^j`M>pKWfuhhd?6nuO79wLQxykR z%N^>m+JGJYEnd+jH}27YKs$(`F>9Xof+$tWmQy=YFLIvZ;LpD%1y>?#hHv2G)&QJI zk$=3G#BifkvncNy*RH9N=4ad9t)54uC7Ty~d%F(Ss_n{H+2}zw!Z%*maJfh2(<&0a z6_gVsUoqQGIxnuQ>i)Fy&Nr!T82c1rg)19dZrB0{kd5Z;z5sz2=uaaTr!!jKIVMGY z`@&wwr6`p4fMWm930v)I4h{~DT*yq7PoC5@OuP8((U!zvJRA;#KbOY`>fuP0|+W< zm0zQyHb+NookX2?dD`?0b@se@^A0%pjP5fpgio10ZxVIw8g8_xB8dshZSR1uRc8d~ zSlVd>7ZE#ri}9z?u_v%dk)Q2S2K?QCE}Ss<5_bSIlD&*$4^)t1+v~9x`K)&XT)d_Q zZhu$eC<@P3VxLb@-;;!CC#A1GzJF<*{TRnjP9Bln+Xzd~{_asbQR(O(*-C4h5x9!=6f_LCY@Upxdx%05bj$AFW=6l6n${TH3 z^X`KePf5O0TfR>8*2MEC(R$8!BD@R+qJ2VqeCKzsyoL-%Ric~J*eN$5b1G8Z1`H>7 zq`vvEVjM}Ca&lPu;zvJ@IZrG-WZXb2KC+3Wz7^iP7f}G6S3*!EOa@R}hjMykc=&ca%y?BBV$1&qekkIpOJ0?xBs2Wb0(@sGOdHSY?%lg9U!o7+7ACwH z{z=hmghHz@5~v4bE;YWA!Qnq%&GtGEJU#_jy3)A(+Ze(7+zHJXsWf)QC z(4Qu51X?3SeiccY2rDE4d~}4ICllLQwr2+#L}bu}_V)ez#yvok;+FUKKeu|9_OQsk zwJ01tuNGul+*54zHf<#ItDvy@ zCGbl!9v;%AA3RO_;-zi;*T?N55Xsx6tgLJZ9UbD)53Mg?HUr!#Zn8Ns{Kyt~2v==E zoBht0uz6m~%+#(#Ywso9&QI~scffz$wAD}7Bh}Y8+0!?hFc~>NJ{&8a8&H;*i1aWwXiS2VnS=;KUMkD8JH|rkKT_91 zPfVQ^ebKJTjDuSKw^97o-rnJlGo~b;U#F8l0lQ9mKFhp(#6}zYM*I+kIPLj zekk*3_T^tn0UJKhUvEU*sTdN6Gs&Q(+qmxrO;O2@q#bfUe?)&-U8$UmOs>>50A5Py0 z%({I8FnAIBSL>GdQ#lzK5ifxBYcTA+vcnQYqGCSV_{4ZKrSjbpu*7`=gWG>yjxC6n z?L2%30lT5Dsow0nuKUj!N-SuX^A3ifB#JGd@dX23tGc?bkiFfz?*O)zx&VGbwJ@^_1mv%ZjEvDH4T1;m$k5&(_jXr4n{cxts@v zGyoSA3_rdpg8=Y|OS3tjKYJDd(sw+TM^yJ+#AsOYmoToB@4sX3;Oh>j6Ip$4v4t4s zGXfj^cJg#^sc}-+#fwR4yeX=({1I>)t&0BE0W0144Ip$@L9^Gt{cX6firDVcY8#}v zx58NwaTC#S;AkX49il=0{+uLGeL<>Cd1_<^(RZlY>*u!u={PrGgY-^LC z>o5$vdrSt{$xW6qu&_ud`uY8x+xr$dvAmZ~LvY$wLhcBc@v@jgb(xcDQ}Iu=`B?U{ zDviD=nddi610)XFz0H_J&2>oJsJXva5l~#_IZ{ga)?WZfWPqE^GTS-uK|NJcQW{_J zoBHVza5PooINp`KirV__b34`k>d<2j4Dih6tvho<`YLA7 z?3qG&d&nv(t|A~qCbOxkFNymQzVaFMB#r&|6%deY{SjR@K&cCZ<){iX5ijyqc>&?= zBafU+X(u^|dU~}sRd9owTIK%g_%&b?ni^O?aqmVI6p*>`lW?)YwtW!hVvOkx9*i;s zaXLZ*tyyFSY#Q(R0tgRpFsn{GKV#6Xz!s;RwDqSUj9d&w7^&l(2qUj;0b%hve?k7~ z(+lj)k2qCQvH~A}kKRz|NG+-a>_lLU7=U^+y=szo|!o&-*YWU;F zEK+i4M|-)FOgY0(md=!-{2|MKD79iNio-f-RH@-7+}-!Ov==5{2VTFJim)Gz8K@6k z@*Ev}vL9Dg?)ds^-F+_(s_hvv1e8$mmskG|B2R~UyK3HyO3+qxZ6f|_@#L_8Z~TE3 z-T#lQFAtSqkt~tcs z+uzIS`B^s*`z3efo$5KXQ87F+awp9?6*TL3JQ$vJ68E2E?BXnT{FX>t%PHZxW8_KVd!W8o~$Wqo=Opl-4G5`0D`7~hikk!n`Q!=u&wJm*c zMerH)2_?M)i!fI_Ru}96P3*XRyCH8OuE4o*6EQQ<^=b`{)=DMG1IOBMEPe@8oNH{T z-W2!RSK+PkXAX293&`Irn{wku1c;UnE-J3na{QacKzPDk;HFDs^Mp-<{7XQttI;iK zaA~b<94_fzgmQ%9(-?fngr5B8A^#dEKc@MvIL&{>$rIIgE>7LV4=IWN$pvgKyd-OFj7P_Z~VC7>k^p(e?PA;ACDke2+?! zYNaADVP;A3-}vRmp$HYVK;1U~{{H>CfKht9Hy*$czY>(IMp@hvdr6^V%tZOeqYrzv z5z52dngpiLU#bJ0Qjtr}3d9wssS7THK_r1KbZi-7$CFUB&2imfke3CG@)Vs9Yr8w} zo=lIX&`_c?X^FXM(3Ll+;IhIeYGUVz22u&!IvXEqZ^BU56$;(@DwVVR#Rs3ZbrmDp zMDfz4OGY_>wK#~P3d>o7|FJJZJfz7QlZGj>>h`YbaZUb;Tf~o&+!^_U!C%tB4HWtf zbom)$S~`m9@?$L!2G75b41v)(7h{1fZxBgZ>M1>8(j613gtxyq`GA`gtH1@n@i za!W&S8MQ{KlQouuTSBj#jOe*I+#-RcPFZ@7(R)LldwVh6mT1#)2jxdc--{$}(zO|tjvs-i9#HQ>?Ty1$=p%>`izXvCQ!a^bflOWp8tWU(#9!I(jmub@LzpiE96T$T#H3*egg+-?tO}XcXja!!ee`dJAS73^etKsrG^4kcyxJ#g+yFj+|cT%6@#1< z&3n73Sr73US^gsTHAe4T%cz>_jrs?bw4jVvM4U9F}HQ$qcEKg=wztW4m801YP*bc^Q`WBh7{Q=!0g1MMM8 z$r>t17P6Qvl@7aS5giG%E@s_TfPt0F^g6?>ZX|sk#bT>ss5f-{1)!YXqk~%sZTJe{ zvF#7fmuSBotN#P{W=2M3n6PCfhrQ7n{*2suQ$yoz`vJ9n_PuiNF%fEO- zUjqiJkXt(|m8x(<2n+SIEXcOw^uH8)%(VQ@*V<{=(MjTYZl{APh^T3+QVQo5Pq zZIZbsx6e}5CgTDd&3gou9E;g{fIx5&$zD%RZrCV9f4pQx-0kMi{M}p?&aKEdmiQt`Zv-@ zK~RcAu$Tt;lfwhN)Rq@;PtuQ+L{ePH>?+Sd6c1N3w-!}T8FCxIET=n&VX+PYOs@S%rzeN0mfy|6EW*o2tUy4hPJv~>KYhjJ? zP6N8RVZJsZvY6k+!UWLnWC&< z7y28)uxA9niabnf=cfI63`r%9s;a7IYiwf6#{vJZyf za1MAc4S=76m!%dK?gZq&t4;^F0_c0Y%eP`Y^iku43;_d$E{a*rfh;5AJ6>b(7JMIk z$O1Mny$Oa+OyD%kf1%0yVB0-;d_p{YQQ6;L?}Xp*ZyI=bK~z@+hU<#&gQ;j@@#F88 zsOn+lcxV{_!tV7EV}$U3eo7wkc7rYuemYic%y1wUxp~<(w+Nv^M;dAm`*{gi2wust zk!E8eiz9-^9KNqe2^qfJo_keICUMPa7$*cOQ%Z_2;k#Ro>0s=*aSBRL<0{xQQ1X{Hd)M$*96D{td7CKg_mDiTb1jV z%QPQI+?_xFFI6HMY*5PSR*6zVDPJk4H4RE2&;HrVHD@8r9!s5Gz8Cx3Zel= z5c$Pf3)ram{ThnaBXom_!`&cjN%|*nf%Frs0{X*+8z#Sc`mr3)GOMGP-wvGxmyzL{ zm!g@w5K0(RQ@L0wiW56m}fKPINAzjQVt}Nm5ILu77*&GWDP~k4H z{0&kM$iE8y5(Axb3<|kd!_JuvJJBzSEo-3tE?3Z9osO838?!G*ddRpxW!OKQatQV?F^e5f_F8w$(PcpuW;sGKZ1;BA@AU*v+OFk>7holx8ivOmF8KLq_76#N7- zS&gpm-;H!_h0B|*plrnh)~e@p6dBtND(sB8Ut;L&wi7PdpRd^<*= zyoW>yHed(~EG_7|o>=nPQ#rk^trBfya1xxX;RFRTa2Ad7ax=~tgH|FoVimN4pnAN} zk8?{aS73N`bah?B%!Yu*L!k5rE@}(Lj_yEi&kO>q?2_&7DcSpUMuZ|+p)kE{v9je( zry1L-2^z#zYq zqFR8aSj$mTu1$!c2fh-+8_r=08!iL4o_0ri;3dT`wxmYY+}&r)K;Chv@p#?%r zQ6l@(8U*)`X*vu)@WQLn=NMx3}nr3z+Yc~EWVHa1n# z2TbD_QrQL@?mc~dedIu4!=q(#x2+m~DjWN~1h!40F`fe05{c_7_yF^tKwTnC2s}iR z&YM57yqX{yv^$Y_=!zT|CbE(6pN0p;G=4tR__+)>ey$|#!5@R4(6zp2O=rYKT!*^k z`0+xN5ZSuT&uvHL`J`2UPJ^yo7K=>I*<`r>Nf%Q`=!@3Y-OLy{k$PB6zp|P$br)6< zk#KD3MMWPU^nT4HZKVxet?Ra>S_z*u;6@sPsWd_@C1Mc3ed3h`Q3|l3aeWrsKRDFO z@-HEoc8O|a;E_W5lx?y+O^N?bN=`mK%5?4yXbCk??*TJXlk!Ck4KV|g04A6L-q$qf zLfB?;fw&N4cTywC3TwrkCQ$bn&zLxE6zMW`!5uCx>(U_JPObh7o(r8)9fZTuU-$QZ zcfsWsvU0|=R|?PQ5yp7E34!p1kNMM~d>@`t-aQG7qo65mhAYOwiHBCXzI!mHr`Wz< z8l;XLR_rpRte-Ew5GYTUv1xC%o;DD-I?8Is)yx2QbDs*}JaB{4As zlPG|$KvFHR#Lu8)^MD11?y9{zKXGy3$foxIqTAsoHIxxv+l~vTIIjPjyduP+6@o1P zZv;f~ezhLCD(-Qu=y>syF#*g?by0^*-F?|9tONW-Non_V&x{)FKgB7%ij7<|qnEX{ zb1?Z=R{Yjf)Vn6W=XY1!HHhc6qW~+I3;L4b%puMm1hfm2>7$ZAdkq+ zw=Q#0r>E=#c1?VDD0Z~0!J+};5n@rP_T==eprD)HS~G+AzyDr}ZKvy7F)KR!g*xrO zdl(bx-e+@fu#R+3)%LvJa5W3Q9_$Bzx!f+s0Nsuhw{N#tSAgkU_QGkii%*#*>_s1$g2# zwp&Lbfb}!N9S}Jk?t(1cEw|c}HeJ!EvoK$~_9BCj%tKp^5Q1~AdkR2~o&Q?*hy+;h z|1rXn-_UvMrvp)AdN+d00UytwZ%zXc@FeaRTL<)vr2Dbig9l`|7LT5!Qp#N1{Ry!L zPVPLKH~QqGEs&$d6(WHEO$T-|b8wuVw@bpx7D}A&$6s?}R(kVLL+v1bboAdx1y_F1 zknCx)zw|64V9U{f4@U$1-qmg0J9(MlT!V+|27*0JM<)l?+YhLUrdlsgOA_U!S+{n# zq4{=evE;e;Hrsn%XIIZD8uf>JyJXt#6{jWd&AryNWEPA2F+@UJL%!@{)>wZ5lpRN$ zex%s8R#yRTuE*_0KtfPy@M>V!pm2=mO}iI^sni%o=mK23GPzh705s~Z-kn>waE1%R zC3Uwgu8|TKT>WDwklust30?Ibx>1GBw@AhVY^C-I^wf@+NTzKX!BSuFFOF^ipPR3S!6h)*!eh{{X9$T+o0M^bVotV*67$O%lxGFU#V%L(liP@q z9#*oCilaZ{n+D1#i?m_tc_Cs1onZGQZb=H8nogLRb~uOY657z{b;DA*>^P2_%pn*b z6w{wKh0{Q8CK0wqL1v))@e)4nZ==N+WqbPC{M0-bHP1q^F@FIyJJWIDMM|Tl;*x2S z7Z)u0*_NxeA=zeCH#}&~e6;oJpPT_>BSOfRN5*sCv+TNUgGWgHMaC$)+o>JP*j(tK zrP_Bf>f5-jO6+hut{1!-WV{N5T-K*K5P`rqCwl$GwN#1nBQKn%Vx<>2!mOCi zZ{G}my+9aBitwB4nJGLp4VYv5TvN(tI+ed(<5pk$+g2*?UA27QnI+r?7P({R&Mdv? z$W^%KmHImtiBFz9*h*nP%2}W~)#)7Tg)FHKsCgJ2=`T%EDKe22fC%#_4+}TYDr`ca z8;B%!61cP}!BxDVtgKN-Qi!Iq0?VhL@$@xxuBl2W4!_Ey=<^C;^18^zcR(C7FskPj zN*n?+JN_CofoMM}8fb59a(!)SHgN%^7aDV8pe7kIKJppUf58JUSzjM%FN%(lKm=9+ z_JAy>)mCii#KIx%gpN0UW^SDtV7%MZ%VYw*fNrUn_+a`sxZ7?l+@y0%x_qwWGutux zV-((v}oe=u-Uth3$A#tA%3Dcd8E{+`d4$>J(e$CczVQ+0;sRh znRX$OWYi59+#o0+r1n)~Eszfc82# z7;@)f*Qg_UUgBPRVYhkbexdi9Zbpx$+pkIF4hdA(2CDI0%p~%XF(h8Hy#g zq?wdPHt8TOJ93W|-iDj|b zPvJM)C4!%AA3rC0L~!VVQFD8A)URa~*w*ffizt=WDBThEz^`xTxnen4)o7El+kv%3 zohTXjQTO#6YOVF1PZmsUnrLwO;5?<;fgfF)4n@`Ui2jCt^7YzSHIC4AV6AfZfv7ox zs)0u0RyDB(R1CM=>4=zEfM_%&cQ3CyM1#ej1q^^YvgC6nZa#?40*Y+$%n^pTr4siJQSpL8{k>1OqCq3r&D_3n zkZe0&QHjUV^`tk)#kX#?JZKSwPlPYyU;%_97&W>KUHt%ogl9r}3t~`PJ|o2KK1d8=M@T>II_&h1U!bRr-U;}OOwQ<0cxtx+qxjJg0c1pDVmvS z6Ep*XZwVBwDrW%Zd*1x{idCq;Yyf<88(xML0X^mHuNep4`#=LtypaZ0htYiMf4s2f zFBN*+b>ByWN%Ie%Di-oa-vkG5M`-Zz<5tV#wW{Sb-CEF`%yQ%iDHguZkzwB5m*)NL zgQAL>XKS?NV)7?9iJvxXTw~zE<%EdAbiae@nR5!!v~ReCmnoZA?+IK%ySr=zaTC7J@Apn zjuiBQt)_GW>Ehs3D5$AL!b15bw$%7C{t!=UYFu*cPFFN|W<`)1215kSC{Qle()|ZrVUOAd?366m5*^R7khO_2~)4CH)GO3?I@N>W=_uHL{Ly^h`O^TSHKEGQGf*u z!v|3XgWn6hz(hl`bdAVjS{#^;}a*^Xjm+v2MRbmA%crT3$ZPgwrxgeielP7;=TVK|#!wX!}r@SQaX9n{I z{fmZ`>4rk@qGsL532teY3?1-Ew(n2i=zsSRG`6re^nRDodVm%D@}%to&Ryv<$BO+0 z|9{_+p*$=Wh(02wglKF1lXGVZ=|Y($rfL&aQRx9qm`m4O8xs~D!YYG5*NGoBkk)k- zVCichsSAFXy44H8j}&=#G?-%Ke?j2r&uuxPcEoxM$Q`c!yt=T&I15)ENuRlaCr`#_ zWLPmE{_2vz_n2+z@B$Wk4l^=YGf>Oqst=7ChuTpad&BZNYQT}k+Tfcl+K83MrG14M zs3>(bKrpJ_@E$v5GHH5+V=YSAh;|@FN#;TtG-l(GV^}dd8yBYrJQpOaJQ=N!Q?Ss; zDb9A3E6)1a0d|_}T=yg6q z1M-LU9MP(^A!^H2qv@T!#Iv6F{RrFa@y%Yd4*%XZ_rA0gcf*Z7vjiRs)^;-QOkeos zFvR~&Lhv#FTd9Ax-!rv=GVbD@?xJA&*QKUZ+D>F?94&SVX>6E!{CPqBMkxza#X>L# z;Vn?7qJt;>%iNhRuZCg&%P1x0sc%w@72Fw(qs(4Y$ts^U;?q~|D>1|z}&PL?Wz9`IB(^calv7u?z-rw>or zCKLZ9TlBBsh=h?Tpz83X#R?#GaKUk-+Ge5GY&-{uS@8+PPDpf%CvFu=`gkI7uJaL> zS`R$V$P(&WhEy~#-~&A!GkY{jdG9*)uM9e!Fv|`)opk1^*?t~IqqZ*Y{(zCx+1-B#TM5gQNrhT%!Dc#+Gc^*Q8TV1|{sJsY*n<2Z_UI6E*8 zXbx+LQ}f19u>nh>R4Ilxn}p?q5Wh{MTMXA8Iscqa;vK)u8zEap>qIxi};EM zdNMe%gSF57$G05}mI1!lc!po^UelmZ3rO4zERW88kHwUJko%VO-&(C zY%@Rza%@%;4ra+qg!PY$Dy=|$JD6z}GF8{_uF1H_#){Ub-?eXJzYVSDI`Q~}Knx9w zb7Js6Qcs0AO^RyWcf^8_0Rji;Q!{YNN_#)OcU2B_p=E#h3-KW+Gb`*j>9^yIn2vY=M76y{CRUFPZaas`1f9R)bICx zBaa5<;mp%@eKSY1OirAWPS9ObvPD-ajNo~acmvynvejcPX=V@UOTAOyj{m>g@HEb< zSpQ9BeT#TO{Jkrj%?A{wocpH|@4ib_Tu>>!M{m&|SRA12Zf14{Yp{t8$ng3X4`I7y z)fTzfy`IP_6eKJp8;n8fMvj3fYML1Uzz1C;G11}f5s^3=;(8 zV#A5}n(+G1Fg+X;pFP>*PCbJurM25D5fmWj^Q}rYN(xQO{;6rCY-`a~EO!40X{LIU zkU@R)&e`otw*9QH4fNH1E*8ETp9e$#uEd07u2LI9Z-l40KAE%ui&|V}J=OtwP zbB%#u?PV6rYhCEBux*X?fu%)<9a#pn{f_KCIep(Z={W|As)!mtXlEEDeC~#NXk*`_ zM=tTc`o+L)P{1$$`4+0iJmFv@AWp% z3p=e2j9OcZ$A+IgtzW|2ts;Tb-F8dK?70OZRy3}6zp}!+jJ;iczq2j`Eh<*Zz3vfM zTyWW%wFI$Y#KJ39;qn)mVKk=o7>!25>QOx$EzCsWEl>&w2d#cGY94AE3g6rKY^c{V zmNay+F&<$I(ohS5)#52JSO+x`4FFXP*yzHL8EDWoCQY0uE4e~|Ww?jIACOhQS4(mb z*!>D5OVJ%HA%ol=J~gEJO3(Nm-y@ih|LY-|C4nU&cTn`u4lPg#;%~R3th2;Nd|4EYx}yX`uU7_jozuZ}tXDSpwR+u=B7U&9qo_(3osI zy`~HQsbw~^M;A!J9S1L%s=Z&|z6WKZb2iEhblMNd;?kN%$L9n;t?wVN`!SE+jA0o% zy~~)j2nryu9wO;j$JCe^)bPUXd_QtPi6igUffShZe{YB!hv~R#R;F6x$R@S zot+Iwm*weNX|;8CWy|h6TC=XBYFD0j-y&sYV;Vvrm%%c=RfQ!J=8j-B zDGQ${1PC1dOP{2QYj_qEN2L2&Sa00I>KM5UoFbXPE|G4D9FQ&_3W8L8{(1m12?{2} z=#XuRC4L<$B&;1fyM3Krv=b=*nr$5uNDD@WNr+IGmm#4l8ps3g6_JGh5Kna8Wj)Nt z;8F*IamM647?54)*jA&Tcw!E2C}z}vhnF;Y6k*lPqkBj4e8wNkZ%1>$gY-k#_qJ2% zG^j)6{h+Ls3gT-gwJ*KA$*skE_gMdRuKQ$^;%-mk(bEoTp8bB(V~-7k%~#hU)4^dh z{&@53$vq$2FZ8A~jOe*7FLi0{GA~(VFt}MQ_p}r>yjXlCYrMYu6$&4K z`!AtfgRnjSv*%7IbPA{7!_Mhxm@cb~6Aeq`PV~%J`!FI;p60@^?a#aPN=L5M&Yp^o3yzemyR~{mGcz-g~8LyZUd6 z1EpiN2ELGPZ>br{4f*gCsJn3W&CQ3~gKglPxU}+&vrE7_OQ-@%j&7!Vo+#<~D`2S{ za}a0&Jlm|hIZA9NlnVlW_+nlCTEF?a_?E5S0Pt+kX5(l9CMfJzIEI5#)Fik0YG?)G zQ!uE-R@Ugad!?M5Yt) zmtg~s{IRtC&VtkPqc7yBS$iFAwJ0F)=U*_|lsc4X6xRI;4k)~jo zOiC(q7!`5U_=fr#?=xdr+*y#AG19zY!}}xLU$QgK)Wv@G%+kf~EQnddxG8&w5*#Eh z0!U|O8GQF4j9xc@JsM?T`T{2!uRb1%F_y?~f+MpS=R{ zhYkkKl;*aQf498TfcwzppXv*%#6JpdVK zpHQ6W$KjWhyH~|Yyu3mzd`Vmv@9~DCnEJ0}39}p|Z=bE_l|Lx4wwd7IO~#?rDwjKr z06dRPn)&k8Z;2(@0k`l(NULOS0re0?Oz_ah%lFF!EHA(A6(N-Q^E&&C zcoa3RAvgy)Lsk#p4cg)3m*lInAwn(k3C07b;X3F>a1yjo;Z^l+$ZCcD9$$^`a_Pxc z2(2ZvP91_5neHfp#4oZNQtZ~J;g~a`DveDt!Xc;yjE=Je5#mKU-kmQtv|~(FvL>Mv zfM`J1x{lDjz|v1MCHW#~rj*ky8~ZkLJ}hZ<1N~lPHEI32pHX;YO&kMU3mH$aP20CR z7G#e9-i*1^BF>};^5V*V#wV&NqzWuPGkD{lvTblNVP2Hf35lb#O8froS<*O`y{GZ! zw*U9D!>udeGZAm+ z;Qx6$ubMoZo&SD}`tiuQujl&~+$#A@pd7;&kg9Q>_IHoW)m1X?K19zv5w?k##taxmKFRCVTX#4-7~fw_~WS8gN*l+~+Hi&!Q>z~f|WrX%Mx9AFK&{CIH;qFHS@VI+mwGPl!&)|VC` zkVB3DyVv-8#5kX{ZRel2p)95k6=ElOZ=P(X=gqhAQ+Ssx&%6iq&i(IE#qkpaILNsF zX7A+}@&|Q}J`y|KoIIKp#fsk6SakBe3;vAw3kbzMk^e43Z~D<$L&xPBx)n;e%GfIg zka!(jaW}v5ye|~)2U3n1@=yGp7G=G@EpD0f=0BSaE7`jXH4q@s_t@=7W4(=)l>&+& z@Jgj9vG^a}i;45LC_w$A%Xi3t168x&GV;~C-2ZEg%3}WoLVGb3*E2mCgh;Ja#=tu2TNlE zY*PrSQ?`Z*081=PdChS+mpT1EcFK~|iEal3s3pWx)xWX=8YCoBE6~Bw?fdo|&Zl{| zMw+k0A52F4nlNc;B2s(uXZA}WgNx~l*{OePZCX(F~qahDhV{@ivPOZ`v$%Ms(QY->&)A+;Rc3lF|`W&$a% zuD+f)**t7RpPaf{ zgoauSx3X_s zlB}+Y7epnPq_4sDA}o3}I`S4|23eXj<#2OfzHwvM17w%?=?I@NR1uB7$=n_GgG4p+ z@b9971phKxkY%X)FVMAa?#z~H2rv=yAH#}|Bn!SOiXn5K40pSzRMD=cl2Jy3^w=4q zME`jSml@d(Lyt!)9TBN&8;T4kg;{>kXJQ|->b+l+SX)yp(p>$T_U!Osguycwdr*JJ7P z6EbDbo4-^EppC#d{h_B1U**OJ8X6t%q(8iK%M?z7X<~CO{?;|Q!x3a6KG^L&TQsnq z1pgG1LN~)YgxlG4IL%t~qqwwmDh6kCW^bUbq{t=;Rw=66fWq2}Owond$YjgI&Cu;V z;>LdzqfMW&0T`N%-DPN4A%c3)DAg<7(b#~CQuG*Fwvoh3kz1Dp($>f+zOo{5_V2&R zsH_E^NwF?0seGTVS}%bz>pfU4T5f*_f3+dZ*Qmd6_8CK@%E#v}#zVSd`JW;LD(P@r z`Jt?jZEcYY-|SxotH3or{tB@;FxGc=|1p5gMg%u+t}-~aa*MHw#o4D`WG79k&h*^t zUfovN^aR7}DGAN+VL78P$;b~k4#HULa3eA(SoT9;@QdLjsz--qbbBa=@P7xs?h}Im zJUX+2X7DoQFJE3^<7T`7%_Xjbbq+Zze_aPD5HrnAFpINHLy{Zx2!VJ7r0Qed+Jfky zoyYLoPoroFtZfQ+!TlZP8@C^TGy%N`CkppQ07V+&uU7fs9%Fs{`bf;a5QJK&-q(jB zHMwJI9%0I?Ck=Xkzptzp!V?j%%DiePyxsK`Q?mF??YgNOJr<5&naC_NSY7vHhuS6! z`!#}xOI_&6vrOcM>5%yJ)@N6RVM+`Y~+)Fl1) zfseCfk)Rd^C5AJ8TYl$(2vW!}ZEQ7ijbxOSo4>n=AB7kJ@8Wj#=Pl_~;VMwF6}I7B zZxtDTB)qDrq@+2wq29cq9<$y@cFr59-QwY1v6yZmOODXNlkkjA0&-}U7vc*kL02=V z_Q%@$K}q=%q@)y;{52i45rmy-egnR;I7$(NT-7>tu+X?l)@dXc<1Ezt`0;}e8;gYo z@JT|dpg~An?@Ul)1Fn8r6G+g!lsg4<4`pnb{OByfMZ{^oaYs1zGgNhF5g~Nq(Sx%# znDaa+EU^awWrPb-A!*Vg81ZR&Ig@w8hUkD-3-joYPuCfND<*40ZE9TI?G9f< z+<{Mu$V+-zp>(STwlO9H4ZQ@zNwJgv$3ypRl@C%#ye$a6^+t!*UZPp%>ZgNyYTEd* zhtsgTPqw(ID3oNxWqz=%BKaX3tIq`*O<^=;I#x)%8Q^{0{5w*{b{pTGby*abKSJsc z+vz8Y>Nfg7I(T#RrLS27%q_usn`1Q;vmg}%`OUT;f3*+9%nY8nC)H4#RUQEpH$~-3 zA%=I*_~lk;E|Rdzg{NwtbKdop+81NsaU)P%wNkxs zAG%PF@kf>x4R)>+X23<_j?KLT>xutZ_n(W}onfTjPh=TRy6>=t3B?-L0Im}oR_B5Q zg!&8epCtlEmXg}K#2eOn|5V()p%5i0RH6KkxdG;qlS8wFNOUP9jL5joG?XhXpzsBw zQCXqT$;2n3*9P)oPsFmnAi}HXIzGEMXd^Xw-Haf8Iw4G$5}rL9CP*6*JUv4Td*7bR z%!CdR4@^`TBxJNPNMOS@1(wS9(lVX?pSc5@$fH0(piK@dPA~JE^X2%W!1Z@QKM79O z_EA4rOx!v&(^-wuTGsULZD6pQvuOWV5W7n&Z)8c^rNkg6*fu4J) zI4*xNVQAMiY5NY^A=zdE*HI)QdN$r+7%7WV?^o9c2`TkV=-?F?b4z-a)kv@s1$6Qt zg$zaVEKW`@6m0U8hqern0y-!QDKd!ltpd&jFE_aLb(?b*{<=06QuZNsm?>7lh~WsM z7pA&cAP3m=V00UTaKu_%5u+xCuCX#Kbh~A4Wu>K6`#LbQkZe+;6K|Up)&_}|aA05dsi)9lz^N)%e+w4G?1BEDZm6-c zY5=FP^;Py#9M}fTk>KQLu{u+(;9WibXv?#T@rHK9j9^bROb)BNDUP7NDIrc2$CI+@ z3|B(Jn*C4S9J`e*t$yOH;55~VEBIxNK5kdk3a$U7m%utDvdM!vB;v;|x|%G`&{ZIE zP@zEV74qWFifPv$DH^%ADkEvUgEL}nmr^_tmIMV)SJ-PtCe`Rkfq3m*PrM6V)x}M$ zLUlwN@;^bZoZ*KVUVKr7`YDvBOJTQp_9$H+L5fo99KI|d<`$*BG1kaEL$)(b z-eI%AHDJH}zvRXzq%lbO`1laHk&rm?0)_Dm#bU-s;1kEHT?-AcNwf|TuR)e33i!oH zvCFVk_CRPEtukT<$O2ij%sq}7E6`$-8J$2x{iC1S+RVCb(DxPv96Tz&6r(c4JywL} z^T~aMCW%*E9Q}oKPWo=x{iVhxxgL@%O7u46{UInQSlFDBs|tS7X!os+^$T-*f@sSU ziBUZ9IoKXKdW`?_ke^}Fhw~`>{;Q^(l=$~#oZm*@11rxebrsGi4&Gcq3QxX`sV|$e zICGx(Ga9c?s7SYWx7bwa$eX!X&iHGDdS{Dg7IQXm|DC-VJRt*8awckRD0wG-!I{Ui zF@dbt=(y;$_1$1V($K*GVJUp#YF!`5bkkkmfP7dqKx7ZS8HMTi$1$@u9Q8CD?^1t( zV*AK>{mdiCEg=<;+;TXc$QA}gB%@Al`#I`vRLTyMg3#r_!Kv2~$zhgX z@*5hc^(!Y59SZ5m_&$k!>XwB(b@_vBIcUcr^L$~UupF1e4j^3`{hD~vQ3uG*x?k7# zAiBbK%oirz&A3$K{Q5IVAjUz03B|(d&L|5Yc|0pLABZd;70_q_Z6F4~1=nS0CBmth zju+}o*)m-Ak#+}8G=D@jG)J@*2lI@h$9}`VgT~3aH5?Zo*jtN(;`^V2lEYLhRzco@ zt|gdL@YDZ%`#e?5-l1iSj>%5>5a^8KsT#f9F3Mi&`ju^gIGMKN^v7O@Z4T_c?4tc& zq!zAzpA4sUgf=E^022(yLBF+Q4uYa^<(+Q!y2~gV>`-_u8dtaFH*|e60d_qb?T3+* z-HI`Y+-YuVTIfL?I8`VG4K!PlCA(I4-LPbFAqRFA42X8X;?6bpecCZ5}S!hrnzxJv?-=4 zCP%hVmvp!cQg!b@XMBy?cQjgs6b45%4{l}^9Pqd=q@vb$gJO>9wcNEN+p&qsU2T>F zrXV@&N494iIoP>VPoW$T&Jtv+uLB-waewz#h@Ns{9;KWQd4GE5L&p!Ai&*jLYf6Ig z-0Sm;muXB3YR0XBAAO_LA+e*rQj_HALi>&0ZhO^}vc_XloSNm*eoX1iIzHkGN3t+n z{Ex9P@!x2%Y#I);5C?B;`LfKYmzV3~y>$SydqAB;@4qz9&(S$L=$0+oNj?B@` zHL>KAKoE^czfPU+&l@CA_yyO3#Wd}1MyZ$LkSA22EOytMY-@&Ho*aB0G>a^vagqtB z$HIw_C4Eh1x{pEX$x-_}8=sgFL`5waG-sQpGysm(garM%KsP-Wo7`S^Ns1b zxVq!wGf)R_XdrdZQK>yCz zmB5=YqVYl#{B6o>YLb|-AJuU6$L3y)JMHi?DKY7_klCT>U3d`@5=cm*R03_#RZNHl zI#McQXTVosZZF;z=X2K8_a(Jt2=}Pn|JD_} za8qn_q}uA~k!B-g#87sCU74p5;|9GFq-xjH)wa!7BmZPq0-18buJb`tZteFWJhyb* z3Jem7(B4LX6vD0ZtU^v3g+|#OY!579#KISPVD=;6g2V+D(?1ffQZ;ypn1~9FCo+$s+dQhzHzQ;5If%U9zQuh1Zg_7LG2wlo|$e8&@) ztI5j5_OXD*O|E;W6CkNK!eANUec>9)ZiY=j5jkN`19A#Li8z+U>Iw8;MthefXSdxUU4Cior<2G`b~;}8Z2G14H;?|5IQ_8gN8{l8VY|G-^TmnS zU?U+|Jblh6c-NTY7XOcA)^fY6D#4wa?wsv{`sEuDn+6(6c9D1Y%{0SN>JZQkYl6S2 zZ+jvMy~8@djt{%?y?g2we=;Hcd*=$@s{DG zeBB>jHE2hQ+Rtbk0d=*0z`xx!2VxY2cwS(WqK!g)gZvGAbC=B0wC})Q*$%v%y!V-0 zWOKgR^I33CCa(w%k|MHjI^m;suPR0S7iKLg8y;{{GFVA&*pJA1z*N3)T@MsmpMCt z-GCtx$dL#07~)tWVc4f;D6woyDhbR($$5vknkZj{keD|>x!nLt#KF6yEuRFtVf-z2 zI*xWm)DSR|;Ev8c?6H&};I;81x&b{LC!$<_h4`;~`RY}atDg;%q>~rJP1%N6yPU|8 ztW&U{omI>2HRnO@HDnc=3I}y2!WBcgZBy~}9iYY*$wg+eL)QSPd*I}`9fka&yX&Fxc#B86lIGcuRaGGgS?%>H_m z;sVK~2?%B*xwjx*Gt)QuMf141qqK;vCQ4X2~DQ3c+j-= zH<$m^cJnb1JRGx^UJNU0SWqWb6|iM*&zJTK-qj)*bV0?>3p`FKYHB5F!MfujjYq^E z#jH_8y9-;syvxU=K=NDc>ZA#ZjhfAj{4+|9O**!87Fa)t=WQBlQNUhwStx9BxT(2_ z_P-*>L(KU#lp1K?(AxQ%IOW{_Oj)CcRufmX?BbdSjapcZL!B*YM7dCgFhtdQ+*8oK z8ZCgjj)^#4krxB8oZ|xfHaa{MQA9CSq}BQT*x~vK8D0oXEAqJx>>Sg}CDNPxJ{fhj z9ZSj$kqS%Nc-OKLEdpUW@LOg0;*L^3LgO>|5TozlALVs->RAHgZ^yz>V@hinH)Fs4 z44-e$1UUEzSu!cGP{;ZHNDS46pBIe%V3ZgC8C*-uz7X6}My;+|eBjFAh`^)>HI~L9 z+)e2A<pLAloS*U;T*a*f|Zy6T{{=U%|qwL;8J23hdHM$G=AavUwFX>-PNuOkRE}TM=CfXrj4MQK9XuCRfQ(tTeIcDU}e>7${ z&+Ch%1}i?x`0!*>tc+bcceuVyvym9QcSusye#bKI+n646308S+?_VlCAEeKe@IG8H zVI}UsV>h?7yu!vl;s_|F6DNnpfa`TS3xmG8Z^N?VYQQ}XA3ogGchE+4!%|Vfv5z;R zuRL|QM~_iKgxU;mmVyMGu|g*4O9i`XUF~_jP~&jzZi;V})g15|#R}et2X`4d3EpsA zM`8;^1{MwIy0}AICv$31|OWT{P|y@WGSlNlgl93Cl)!I-yj$y11R6$>|B*+SslD$ufLslx~oQHJSnAwjT1+RNL{ZKS;%m4!bWM zp#62Zk9!vx7%aW`H0mIYJes-V&UAkH%Y$vV*wJn^ZXUL#eI*j629w-8G}S(T2Q~m& zT+RUUE~Ei;wIL0FUP-|U+_6*P^(#iL2&c55Bc+D_C56NHffb>;n1=0Og&5tbsXS>Q z4YT!ptfHEY7URlUMCpU9H=t1lTztprvbeWzxN$vL(lWPeJmw#h$^X}VH?6A;g94?6 zF)DyEuI5S-hAoKRRRaY8`fE|Zp6KV+Gu#f6R%R?^&%+;q3{9q|EBk!B*JDo}EC5A) zfw+6i;oIObXx?x*oRZSC%C;w|o9gj&)9$I5onNr{=O}%h2?c_jw)Yo$pH4UIUZhHI zhQ&QPk#x!^Hljh((^7U?<=kB$?d>DTYPyk@H_LBhONirg)fvL0f<>Wh77=x$nva_g zgP;>5A+D}ghEIrk*Yd+?H34uaSub0b@3Z78WF5jnC-94YXhr`I&6i%(7mis&`?A%Rkuc9 z2HSXaFf>0n6o@1xO!JVP;)pyR)ZGX?48<-#+zS+uhat$nL^SXb9|8&#CJdt zOfuw~;dKrJilTSKUj|lQKHgdcmP>#Oz~Qvx)KBF-dGeCtm&s5qPdO1tu#m08GHFtg zEw#%>znvx>*}EfTrP74`3sAnev!v~9r-%61;<`_FWMmb=nZs713&!fAk&s;&!OseJ zN}h}x63y^_W8*eR^$3Ypj4)KAF0KJ+r8agAAezvLb})f{`PSJ?m8DvCm$d3oZR-v9dj&VqYnlZ~#4*xl{EG0OE1lm>JwbQwevj3@W>mq5ecna2z zpt7%{*ug;}4GPp?#<7iQ_3^zDp(sIo8BB~NaV5|it&(4T-f+L@TRb!4j7gI-#0U$e zbysH>zyb3Nj;ok2CPm~er7Z>1E}vm^Us#dG>1b$TbYGkQukY?ub63HZxtxK%E@oFt zf@!o|N_P86;0|Y+VT55x%_o;KGbSUpB@#z?yYuLeV&jkSK`N}ySfDCRlBZ(OsKt7> z=^z~$0^BI~4xS`U#jZd3+p73IoH2u^s5oH27wM_nKTY8Mq*4}$EDh%Lhz<{_Pd94` zN8VBEzU;2`H^Do?>qWKrW`TNXNrkea6BY)HG+aV^=g`(ERek;NSmf-EQDC=1aZ?da@z0Lu!@fxU6`u%nW1><* z4{>(ffyx`pB-D3fEeCe*9XN=Pm0vnPVCa@*!5v^a*!eO_4LYNFxaQeiu#o*{PX$&bsE-Z=?Rf z{qnIAVGMW=j#|RGL)ed0=@$dUTj8_#fJ_`HXv~leK;1-`O!e-N&yJk--t3V)uTyGwJXBK);*8wzp6%`cJS92$#)1Vb%ZM6+5rR{NfZ`_&J|8b9V}`Y+_80P6Q$>3S1Ggij-+8zAxQ1P+S70s zsJReR0cqNoW{R{y6c?2e0N}ySq^m>kXmopDv!&?D&4>xTO6=;eJWMJd3mWiT%)(2+ z+<@H?iCuWVFAH)KWR)xZ@~}uiDX5KQyP@z*@h|{A?p%b(6lI_h>)qh}ClNSw8bLki zY*RR?yn2y8QX{n?ZA?uZy0{UO6en{)sPGDd&{^Jl}c ztY+EFPkB;`fYFYQ@$7@vvFz&1$D*wA^?TkX^tt!_`Mr$65;ThW_Kr+bCv^g0MTOB~ z-f&nTk%MJ25|fdNCvV{1{&OnIZhJG7{7XAzpbCCd$k_`c1^)Hx*Fi^QEb3S)d8fM6eG)uMfKX%^2|nm%Ch7dQ#8%Y8A}?2b zpWtQZVZWJ_CV?=hIwHw?N%KO1#-=yQjgjx6L5%=TJ|pofREJ8I8I<9R6t5il8Z&GW z17yHgVCUQ82#=LSM;jiNzVnVfdm<@}3Q_+>Y*?4Ul^?K0;p=aY%w@zijfh5})e9YD zu!_>a-;vof;GSvIUT&;r+QhLNIzR&(bxkcrvjP$pu%z^Q^7lf_VYd6!3dCv9xp62u z)u#%r2!6~u8+pIpB%4 zihMpFd+OM4J+nkKV7tF4z;e*V)*42iqV)wsNX;aOm62t_17v2cb1dE|=O`rM%tkvb zL;7DjZX8xr71_Wtfg?(AWa#0JR>6` zbm0d|ayv*(OagH(Y~r*t0De2YhB^!q{uPun7B|qF5Q=yr#vO}?ZK13m2X2Cd1{6}u z!R*6qBw_Y5(a36wRs_O^l{}cQX`dH08FOONK3j6Otd8xySTsA`?ej*IZMn+r6~U-; z^+gU==m_ZN%IHl8nD-@dF>{GR7PV5hl?KOzBS!K1q`+ zSQ>MRoJ9&^&2+NsH?-No1sFEduXpXBeS6{Va3+{OSm?jgkwzu%P=5E<53~S}jjKcD zS&S5xRfO#CZOlIk_K_Ae430<7Dk@}XK)${AL&03Xm%Fxa4CGqV z`p%4GnIB(nFe9opbYSP#dDeFP_rY^m5_@ou77fffWgsmuPDDC=s;KD6+T9y8I?OTs zt1p0HhRHQdP5l}>z@c{T;+pMC@d5ZB6cdzNyvEu03{t<8I@0JOI^47IHly_5JtXgO z&l+n$#hA?q%Kl}mxJGhuCTtyx5KMRZAN)WLB<~>H1L;U}VfL8N5+No^c5?ZjLtDXy zBd*z3(N+vzKNv?yT;R8FBscCejY9)HXqQY50Y`-h-ZBjXCb_r^zWu~sfO==s!aESX z!cGGui>Pd8#0*DLr~}4YE+wo}X(PK541APlf_-iJxipgY24;S7Z;6*H`ja?t48uPpU5 z>nO6YStd8J%a^k&btM$!B9Sr0=UGzHFdqG{d2azgFC3@vB+?GjI24F+2x+mjBV|gy zdTUeM!kN1z1>4n86D&TjZp%f*Sr*2(99l@z)yo}+QC7pg7)NU!L~gQHlX z)!GYdor^8{?^W8t;cAb z+H7}Jnno*}Tfn&;y!}37eR2BQL-ggW zNZ*MQC$APhZxPBJD0|~FjR_X?OTB#Q#Zolas}LyzljLc-K_i>4RSkn z#>MFJuO+>V$&T^#c37K{Yupm9x8Y9fau$2q+r9g>*MG;ZmZ_Qr@ircc%uBj!R@FZV(bE51ZlHV|5nwbVbL;CR;e(+rY#ufL;mtJ;6)yM=`ujpFVw4 z8pBu@*-*mIDb>dW`RxYPPrz_pC(#FYCXnBSva+(kFo@NOZjMCE2r{CtB7GD;1VyfL z6hILoFc!Gi-#Em`J`aQu+c}TR9;FhGkJYwpYA-(_o`S+uWF&r-s4);k)|uywL|l=+eN=prKvuTH`F zvyQ-X+_J>E-xMg?%iTS!B= zvC|pHC!-C7|J=mOp%2)iH9!Xy_!CPjmSNb6vYAHfas*{KH0)bm524@7*Q0qC!HXYM zbV~f7Y&~-6uNsZEpFe*x zexZ0@7{+@MifZrEm3Xj&&B#InoGv|8Ox)fdg63gsn`nhqa8ILu-(?CZ$i~ZsA>5KQ zTV$+}bQN9&jGr)`$21J#N0;Y29W*)H;~Be-uJXg_$1gU~O=8D; zTy(X_%Tc|GyCyLr_p_&WlGM7sjrHMb*`H@#9nR|9j)CEWTaWu3ADkhO1nqP_dVn}j zBu(SmW0P<9?LYW$IQm(TpXwK}qoj z`fDz_!LaPCtjNDti!dglC8&<@|0(FDZ91kpS#xalne~R~ETlkYx0Qkq0{AmIRn*P3 zq(Tv`1DZ?%ECPLqxrecrF?6W^z055iFZ%x)wH`tM%*SfzsvS1CRz58%@T`3b=Mzh~ zokIW@R?<@fT0P5F5vLL|jKi+O2^p>7A?y4zKn*)<{H-57etGOLxq|`@4)G|$CmWR% z79K^cfFr6QHVMISchKYEhtEZx zH}iD9JaVhjS!v!CruP)xC6=^EvK8|xar8W_HS@BU z)9tF2oSZeQ{r+|5F7YuPb~dpmgec`1%z&*^_JG@@Pb8tWf%CxrDV}+1Z2~lbbmno& zh!BzgV7-As5lU|B?f_Xggw1HX(OJ=wBfbLRRwWHhRXUap z!>rJJNe12Z^a>y-k-l^wbk$9q{?}i|?X=B<2?r?i?M%T#->o~CemtMd>0l5W zE0hf(X!q~+&Zy2LBOx22c35aK>>w&U%zinEw%oCYcGCKPfDN z)09`W_d1F`r)7uE7Ceh&$-(a$Pwr0&j>a-GK-U=4VcD)RrGYo*R$4+q{Zq?Ptgpei z5erIt&vh_Jy3Xt>UGxq33Nsu>3{Cs^C6L6T0UNnE4ILYc;;!CJi!V?ZKmP3qwlEog z>+0+0s6lGz^RxNc>yh8?w|H)w;(0bA2eeb+2$)|I@h+{Qp&_Slxxn1r?EZT<1A;GB zD2qRWf%X!WAb;E@+zzf5-nGEV9V*Rq9vOIn+T6a}JEwS1bEx9|pD5zyAm;@gUyTwldARoTNlKTvwt&KEQO;!JeA|OSB&ts*CwS@=DVhnYPxj`?@Qp>m<~sVs z`nKzOH>W;clK$_?fYQ18qpR9454ZC6_BM){< zjDdjfppOsiZ%+M{#BcG`KX~vC=UvMTq?zWUbH|taA6@80x)`&T$1_1L4iM@{Z@BpyR>kd{?$BhIZgA_zOfbgp(@=0>-6b zik%T$1!N22EUmom))jmwOgim3HN8*V8&FIN>?q>~E#lTE%i^%fp>(dZTMZZDZQ5cL z(X%aK9bNVI<|XgKaTh*`&zj?IzjF;jo9JWq3@-!b4`No0J9SpwRg&KH?r$9059dmS z$L{O0q<`F@^Y{AY!aIj(js>syzI#?NLUezGt+rUK=-M*aJ|J~mh6Z1_ez*N{DruYA zwM!RwD02Cq&HJ}9hg+0AK6rz0u!)_z>NN~50(15wl?%FB&?MW(u>v&+OkY()suJXK z(%>4bLzL|Q+3dRY6A%b*Yq|s9nz60JKGWR;i!QRP{#G^1R%*s6J!*g2!5jFU%$zw|upWRl5ChXg1e2|Zi@%+7>dYBFa1TaV4~d80hWu5a&7cjJuSf=q znzUOz%_u~{7pYfciXHUKf*78Han@g^n&&t@pnLlS7@W1b3tK|0XQH|bQ7te3#noXB`!OG9~WA36yxp=(jG8_jy zt+bBkBf$wrZT4Z;{b(#41u;vnVGia9s7xWgur zgP~6bzG#At{4oNbpyuGf6tOkd?*^JlFqHU4Z(&R3jjTnPdGqEuFiqbwKqhsdj_9cZ zXM}+Q{G}piU^H&GhV2ly(N$YJ7FLqpFbjL=;`nYRp522O$J`N$Bf~E$3dZPpxk^pK z-h5Yk@i~hw|Ecx7e>r1$?r+HTrs~qh&2~)*lG8_O4*-OCDyzhwuH8~_69179kH3lb(-4X^cN19DM8wVB)XC$rRcPzF(n>W9bx* zDLO2ImfXo<#S64#yg9}HPL|7Z#dNY0;mMMuMj}X&TtAF{FL!adqxH&w&+)eoQye#0 zO-cD}5==X4XP-PauIm%WdhMD`1H_2~5>;w~BTMJ7&gNBlbyNF?S-%>psji>-w(i-Q zOqz_BHu~7Fj@6&(Ion|fI|-ae1>woDAjFZFElKplY6z%sY9aWDhjMpd8JHfS7cWY| zsL0)!EQ-NKesqVXs3Rbcl{Az&Tmh^tjTe)L8H57BrLSQ;wa<&t`Vz90%)Gh<<^`Sc zH*S==x8w{4+wG(Ssw65Zib-+a!|t9sa+u(KsZhmu(t<1eyaodjW}GFRiV}$>ot^_c zan1G|cy=)W>cDDPenO1&f4nDqI`M=p;=!royzd00xd1Zmi}h>47M;33Jq%^`$ib^x z0l+-?ejhsT;(`J;vm0hYmxs=+A)zXa#=>z>%6R3xBf=6NEG89Za&m$Eg7IEk`|_?P zspuIivdNA!J)`A|Jb2L^ciY#eo6c6)r@uUB8A~F$Yo6a5pQ(X2JMsPS9S;l|`n%=d z-=wwbuDblnkD_${GPh3*sZ`t5ZtI=Vlk(8Jy}87BezDt^)?LeE$_#JTnHtsLZ)s_|+X`r_h-Goy^nbj2EcTqG#B`GPw=D*fv z+Hsp~PlkCk?gJSk4CLU|RufN$Hufkm>lg?ZP{LaI6UP9-?4^odJ~GR{p=k=f9j-dz z5QQ-anPj{J5s+a>n9rm&MsOe^sfC@qFp`Gcsns6=-ZK}WBf^z)?e?P<{jlct=XdZU%XK)WkcYAK-pC1Q9}Z5$z)QCvU{X}d9wB1$ z1sp#vJ>wK*3S~{iCIpz#|5U#Q@=OlWc6owIBQS&#^c2_+1wMFBXQPN~UVooYRAl7F zG`Q4RWebZQFaw^??cQN#3vb$HL`f3%YR4?dkrm0TxfZC9+{FWVECa40tl$|W)qn5y ztHD|DY9;3R=ok~P<5}yw)QzAJrxAugbgXu^;Mskxr z7a1v#x90l)ei0Xc<%$kjum*oYSLh2nh3D{muA=~L$5`*R#@7npyT7*Io3~B%Nk>ut z)^my*49hsr$6ki{Ei3eNNU*QZ9&McemdtvQ?el0BvkGP(b9H)e{$Wh*dx_#`MBgSm1ND zzLo^c>X&2a6f)osZg-E;@L}+ypa;3}#f=^2T;$h$Jzr9B?w2RqnFHS@>as&HkqdLJ~Ys1VQ>I7^@4(kn$;|L^SK~uJK5b2Vvn0$ncrI z6Rb&<0pH2jEzK9Q#4C4snwy-H%uo4Q+;3JJK3Qh^2i?YeEmq*vdEH0cbI&NB$G5_7 ze&pH-`hWJ(XR1piH7vx*_rl1kykgF2Im7n1mDAh%<`9G0!FD{^(<*aFFnR`RdgI4_qRMnX23GqG>0?Njt(~=VE9A3xN}*5pA=@ z$jG*sl7qbG?yr|xY; zh!03)>y7`ay%&{Yen1dIv-crRe^6C9(6f;RK87hJ<(e0p2I5zFXzL8gz1LALsy&yj!bRJ3yKAk!fZUsoSo5-ab2e}hzW50$@oVUJ+GDhVwCqOzV z4~+!Ig^jhI#pv0XORllm+kVTg#4v}OAB9-^y5GOKa(mnc#3InmDp;915HVG7ciLD4 z?q=95ZtXK+X`WPbmh$j;A>DFcbd+DW>8O@>3wB%{UlbwU^D^z`WR^%id5Q6rjsT1{ zAHQeyZrOzkO&n0V20UsI+v4%9sw7 zaacGJ*$mzxT0AVl1X;S$_)NyIq@>dqk7R%{d>Ck|YA;DHbqF=JKC}fnv`bX={ooHs zA$7<>3r^UH3lh3>`}Tr%<=4oBKlr$J*Q%i8039%FXcWJxBz!u9}s2wXWj%B+f3wdPfE(q@QqG?gCUg zpSQ30YMMdUp>)%}Lz`lIb~|uOc!m>&&sYl;gehy+b;vzXns&14$6dN%tWS@Y7SsQ; zv!gp@i+WQse2Uj#6wEJLmOK1D{1|RaBz;DEH zTXsElK1kq@J>l>@#j(uyv!pRENpzpyh7AlW76&RRFxhqW=%#RR>%D@+Xy&qB<&Jxg zvGO+}yve%ii$_Glu+B%h*NbyYfUgVN=Q)&q*jshlJLE7$vJGRoZ38`*ioG3Ld;u6p z)F8&ZnDOCd2QVATrXOG6kr?hAB1wl)N`6K0FeQ&=uaFU?M= zm9H}}ss;4Q=c2ie95VDahjC zP5Dw%D7Hg8H4M_K_64XLMM}Ds{3z1jdL zy`^;LSUc;=Etx;CR|L&C?)ywtzV~n-MKF}NTX;r=$9Q(co!b}C{u_vL{d+&r5K6(p zTu&*VQv*PDo5d2b##vAYXiaI+A~)r?FvHtyV$#0IcJu-Tg)M!VPl3+DhJeMphTj(v zy9YhMrF~vq?PA^x4n{ckB?f=vnUr`{W(gyb_46`cSC8qqCko#V(7okR(Xg>)aaPMt zN}DN*>GvvjCNTra%J6@!&7g@`4U++LisAd{NRz;cl4S=d(+?I~-&$n-0!TtE=0rZk zA{?ZWc&J|r(0zvPsl+NJA`VwG@B$roWIO8&inM^%Y^0C(k;2KRTtbC1xCntQu_8IZ zD4H|MV41^6Yve1mz~v!L^ti@%xR&u*xI7`%^_HEMGsj;}#OqG^x)r{_Wka zF|~o;am(FqQ{FcB=Qjc(fByLK?q{ddo;^&w=*3MgI>Xop8c|4y0t0xV4p0Pg)-N;Q z;vjBif|d$SckqblG1D$Y?6QZ1NMs3(*4B|4UX}}|2E47l1h^L#ntHEb_ljv;!W?mB zu&mH?=*v<0s!WXwItQ};+mjAQ6%Y7jy>3x|gEyh|`sK@y)5^0!rf%xg(ZME4Y1Jy2 zK{)}jF2`#O&koqhV>2GL!Ydg(V$TL2HE@f`#Z;mTQBV}(6ef>xn^;cC_J)VCfkcw1 z5t%Xetcv_f?pYyg1-aXb=X2Ns6pqIc4Z*=p8q_@%0v7Bl%)4y7qPc3riv} zX3xu?U&cK;%MxTC%wON#JL`&F-u&`r9M#}*P>%YCn{!!#9-t(1k!7lDZCz99 z4E?*2rDf@ZtoeyJ-J){t+yQ-U6}*4~7X$PJkGUPw=)009u`A_}G@F}cksKZMw#CHH z68ARkG@k3_9gWFd9@8@&1?~gxun~_oM@xhr&uoVrOYMck5)-wa@|uW12)0JQo^@?ZBak7>ua zu-o~VodXPBA*C|Mlt9Zj=?HvDl5e-)ilJlF%KL9bO{u8r&w}WI+s$?wkf!SR!P$f{ zRrkSd0RfxeUJX_zG_8l-^G*d4r+{9^5Bt();~kfHi4Zh>Y|_IipO5TelSD;)NcjQY zx@?UMc=O}yT>ZPZ@k&c6`;}EYl^oyP{lWruC{GGDCg)6N2e`kOl_pywd7|Ba`899R7Hz%L9kC=_ulPv3k)$w2rnkZgbuhLcY{SL(1^apFgnt3Ty8mo8_pS?B z=aTmr4&M2Xd^Ir!M5_N;WCtaI_Ncb~er`FA*>2=K(b}!DAYVzb12x@t+K5gCmgc!- z{(ZUbf$m$ko`tA>9fL!UBs8mz7s^;5>Emy(OL;)sopa&g5Bp<>NF1X9o3ZbaGr1lt zAZsQ(6Hr*?Gg_kCDCpf6&9ApP# z0k;!;Y#PrQQO%e!W9lLHzO8PIV}f1o0U;kfglSmf)U-G1ONc_OR^#7)7n1d-?Qe*u zg}jK`HoS}y3rUm3jVmR(1W=uxT#m#!&*(7w1YPr2ie*VhFCG7WIsi_QA&0kXkLc^x z5I!OrG|lT5%KOqom8<0auRjTGUXp`jl|E$84TLRjpQ19hD{YV2qx)4+L$|k2c3fts zo2NhG$foI|difgfuV{b#bVf4AQ2Onzi00eYeLSxf(ltL`Xk>r(Ox^Io{YGA1p3lJ= zfEqFi3Wl+MzjtvD4bhyr4fzaVVPUdkR&^q6>BJNTF(zCr(FkK5nqc_HF1-cp410s$ zdawbD$4g_6m<)IPFPaF+>hPghiZ}d5&)MyZij?+%4p@}m_s*>FclOB@AiihbSENnF9ekI0)FDsI`AKuvkLExnVT$KFWWIXvA#@JWL!`GEZ@)02;>I!Km!( z*P9lU=l5qK^;_yv6%5x##$gmdgDlyvz^F_YrHTXih5@*hl|9m~5q|LGm;gVR_DC_H z52_vkhH$J5abIp`d8>1?NQ?wKoqM2QjHG4WxR*+iuW6}`T&P@M)KYrtV5Vzzo z2ji~`9&}eYuGQ8?yO6%>+GwP%(Uc2Ai3xgKa80587|r*%{1k?6;oW~ffBrm+7_r;M z=uC)GR#GB3kcS_V$e-k=qz(`2YQOG1I1FOG69F{PpRxuZj*;1x<9UPe_#9UF1URHG z2t?erFMn%eDon`Bi*0Mh1+gHrc9?tW0FUD2!_vTrCi=;9qu)79OyNsfMQAltefXjM zx(v2=1_y_R@cI?Uu(ab3NlTx4_c86_65|2BuFK$qQ;`2<+9+P%lf~p19C;{ahm!IN zLkEX`_u$JoBGE7`y*Sx%r~Fi5m`f0jQISH(D&9{X_7U@vCIh*}{=67n1o=HKy@mZO zsc9a5u9J)3OxqRpf;WTh{0(@OVbNme>)Ydiu}mHb))PfVGvDs%Qn|PND_0E{;|78E zgaL#W*10VsLsLqtqL$$={n>cPAoSFS8LNP&7Ce%8GYt(3##ka^(g5^Ov=qo>1OO#7 zb9~J?+V`25?56ZAv3pp0Ge<*-ehXXsBY|21fBUDq0h<#fZwK}_=~k$O;D!+eXUAS_ z*d{mvsKxv;_qdI07lpLqu{panVKydoidCUF4eH!QM^yhren zy9Ssl9x$XglVx_)olRuu0v`bDRgk_;kL+>t=gv6AH=N0s~PD60(i9{{~`AI2UJbWRctEXcIS?HOnQv42>{8Vg}*iueVa&&=dSM zNYBh9 z-o4JghNf~LUz9VRzWO2p(GdsrdmQWOCCVanEIliOO^aZ@Mmi{i9pQ*hASh*ygevWx zqf_i@r4Lb@T!>B`VY?I@m|3-SWZ`qHM@Vx&$O4ug0=o-<8f0Z$hYX?+@Md=I4nGS@ zw~hH%ynRf}o#yD*SXU@H!b%0=zA%uU4lxoW1owe&=D@FeVR*4~2R4GZ1wqp?u+Dgn zzJlHay$r85D|l?2ondn|YHB9=5Mab+8e5GHo5vW9-n@CEU3b}j4qhPdm6(z@;h`~# z?ozJki8`61te67fRtQ>P3dk6B13tA;`g(e1B|$k#ia%5pdq*Jp4c0A z^I{fQzu%#mtiW9EC8O^8t6F&czy5czWRIjtZOo-7yy@xOS!)8leW`x_5SDQp zJY$qpRTKZ1R%j#t`br514f6Jy_s0Z0;3mw0&Jz1!_7w8a#_VH9TT8sas9=;naZuSKmc<}*AFo&YYo`hH&l)l5Q4s>WJ8{QX94 zuTxpL>zz;Y9}|2AaPq`zAm!JQJIANIM(6&%UZutN##JO0eBP&LV{-w-mwfbGuK}U| z@j~4G^|G)*$pnWHLl4hy!d7WPch{|7j;KsHY3gy(0F>2xMVm2LC0&S0+2rv{s`VIw zMj$`JBdpeukar6sBPr_J-ry(IehL{fBSxJU=gUdBuX6xeMsA6(0yZU8r)R3!dar6j zaQBDmVFt?q`*yluOU$IZ{_~f(cm(y9OJ;0or>Bx+1nGK~_y*j@Jtbvda2+I6Nv6WY zL4bD__~P><(GaveMVQ!Jy7tqkji3v=1q-Mf9RkNT+Gc?Ur3^1ILi1DKnnxciM7&8) zM_hjx`I&UpWCZGf6V&tVjDMZgYItOrqFB_4_x}o@eST%u-b;|OtxRD1&@K6P^pFX2 zI}Q}YOrbxpZs0_WF0_M_*24Q$zYJ2 z-vOwDR4lL~PX8~AuCNE`F36>?eg7CeOvA%mSow2sMR!+!q352r_<}iXcb?EAhUs09%=K7y1#n=Ieesu#WgwpZSTx(YGohZ5m z4q~C*!>FSJmoTgTM^z1udL<;9|9Lka47?laH*0zon44e_`$A}S_L516reRG2T<3m} zJ-k**W94@c62wDm&z?PtyLwTml!r|6T#q?wQ4u+Vdbr(1 zaQy?_QLeHqU{3S|Zfzq-G>ON%lp`L*&OYfK1a^MJ$umrvfR0W>(+l`)Ypvb8p+rCN zpL-G&TUeTx2;L?ND%yGGsF0dAYgW4aWIw28X)eI|?K$AHP8ktfsbdx!d$nx-D)?Jx^Rl9X z1TX0S$rT*0LH!TBi$*vqmBWGTu|yh^t@9UB_&2sCO6Ca2BM<@OEP2Bep<`D_xq-}9 z!u(+KFoVvFs;%(TM~h|l$6PqqFmb&9uecy_hw6dfWUXfbBkN?i|J{Jl?gaP*Z{;lMWVd*CTna zhnJU^fkr=;65(AoB{&U`^fwigbe;Tj>yyL0wlHx zjhWKaDO21<=CtZ!Qm3|U##t6iyD;tiV#J@4_>)=b)-oIT9{099U_62uy6|+@jAIaG z;S^s~5A2h%m@Ooa&BKLmCmve}qFNYxk%Z|lDG=;1%;VJIg}H&C9iT+9;mH+Wrq=uP={Qc$v3 zF;6{3bE*#Ph0L`5ap0X_Nv-G)3mEPK3}M$c)} zk;B4mOER=Y8_E59cZ0nt9wrJRMFVRslnM49ZT5J0BWO{M(Ndt>FdG4W&18r*k-(vs zVaF3=Ow>mPD&W|m5MemN7i?BS!46}Fps75%84qS61)`C|Nbq^MB|5*&Fl1&z zqyjM-?k&5au|t+_nDeuXe}l@Gd>IS@O|cao4JWCeGe_zzv0!N3cj?lXHBa#xdw))d z$l8Ebmh~^V(PmF>-$+Sm%w8=W5Adg}tiNXiza|_-mr7|nYM&GNw7B_@t0IG@z>}^% z+|)`mCuVX+pf%nWk+#>AR*LOm(6t<v&+1 zN}gnQJF9Q0n+dU=I$?aKAgy}n?eMA;!(Bliyloup1$3~xqV4PyupjL1ZQpGj zfvI}|MAG^ogQ+ah9p7MY0>VF;2ngO3kEgOsB;)|EDE`j-_cqy36_E~!af}qZ2UJgi z4?r5}K|^fId{5mMOo)-7pc`_Xq|tN=(u0jo)wE^y%p_0x1a$L3MHus7e=8q%8P9DP zV;nJ@yp1ST_>c5mS03dVeBSWuNMA>{F^7yfaN^Xb!>EzsOwAlZ8(;`#@V9K8O= zR(Rr_g9y+?eP+0S^|ZTSkV}m3>3DQy*nO0*+ep8PzFp%3&NM?=8h>~w>yxpCg*-Fw zVR=#n4oBkJ$kCun5wEo*cNp4&!q_g6I(W-Zhqxh4Q)1DvQ#3ba{?4su9TRr6a(D?t z=z9_q%7(I95kSsQ-_?=sRJ*=}f47tmeTwlV>_o^ajA74#0{bBb3ACS{N>WIiYQ?bN z%_S#FpxouoCc8w4vKjwXB#n9llZPYD0?0H?8%~}a%lO`7<6G$Q{nI8juhw&%QY5W| znSiyhwl=*1Y7kQqt(CDg7F&UJT)n|fY;lTfkdDRoM%au!;vweZ3@|nq#Z}t75F{87 z!yhg>84W#F1S~<2^Eo4WNGFjlscH~eshxh7#uzVhK4<2QDl4FczP&#xXsJ#M0`(W{ ztjDNLLDDZfq3g%T_nQNB1)Y`M%OBgRJ6AyvBO?3=;^CH=Io*GSZgzETi|^~BjqyRhDMl@81W}$}Vk$4H@!tp1vMwe1WzxrTW{^KQ!YQ*~-024m7k;E7rr-C7wc4 zFa&0+{?koOK2niVVyhZGX2M0LGUahun0G>(oKtL~%_=);LiKNp5yoEH|X={7(R8rX-I?pZU+zSI4!kD=FtpSkj%zg(x z0|U$wrI!c`*&1TTa^QlH8}9Xt3~t>VJ%%8_d@qc&r?qn`9xp_@l!g=Ea7%db?b5(IF<9~ zi)_r#*_T7C?tt63mynwshrvbQOC#OT^c>?FQ&7f!iVcQMeQXV>(I(0rSvA6sXiHSvOId-bB)brvX?B1m7!C+j z-90f@w}5o>l~o%Y-Enche#I*y-&wmnWt>QbdJpOi={$@x zT5QwRR>AE3X$Kl907OC;@K0D(u`tYQc}WVE+Pvvf(D#Le<&0mnXpzLTo%{6qEQ!{D zSHf`Z+F$2b>;oH-km1>_Xg!D0^fF!J9<7aZCUdA}EUL!5q4$Eq8N0IiJ)Lf=&J2MR ztD$TgozS}Nixkt96z_s_YQoTV;5`7y7`8>)WLs3qM+FFNgHEA%zYB<-Sly7J6RVKu z9N9?UBnkB1l`P$Q6aHV=?GWLV20Y?$7b^#Xa zU%$foe1@=0aN&sAB|}-$=5>*UVCp?|G0k7TMDYmD|I~=6DDTHdt_mG4m;^jaLM2MW zRbUZ$3ChGM#3o5%VNZq|V#mAA?h|Z=HaEx!gO@Mh(y`0SDaHmi>F?Vr@hfGZOAA*6 zV@$BXnHxZ6-UtH3@q~+5U$E(hM%JN9Kb%;yFa)vXjpF#i4{va5J30VC#h7D zYD}eUNRnyY|J#(n-}-b~tWuCu>kzc9dz+w|N%zgP!g-2OJgvTDm&9yfP?}Qu4+nbI zLs=ijs7t%_(RBD!^o+qg>J~$gTVGuaybmY? z1&s%fEUAcB+c&9`-}GavQiqrutDp{wHEisM|MBcD%mI4H&{%L3x(oKMZow|mqZJ#)zH&p7thvDvhWrQItA z2>O{>(wA|}0mRMu=mWDz)>L-h*kLmDG zW*4eXQMI$Kh{Thh2t1yj-wC-N<)vAIjCH%Gdk9MdYn*VLG!`VsN`Nmp=7tPfnYRwm zUdol=X916#J1tFMQ;)_85`2&%QF7k77#eccK`TejBycfFfk4|7EHa$Lz&gx@yb#{I z5O4;RT0_TUqmH4To4>qA=MjH>Ps2wC#G-?fod zByBC&IV6ktZ(sC*`~rryP6RM6ePr|7ff?a(at!x9 z@JzqvaXQu#zMV_w9)>JwPk$?h>f!)#zbiA}7h%V1}&p!zbtl)Ea4`wcN(S4cLV*8Jk-ZL|BmRIBG4*S z(ucNAper7_`OSZ>I6d+bT`iwyPSo-m#R?SF(*9XJ?Sbwq&^`BY%=mSx;#O2X`90=M7{pX`7m}%xFBC=C zTu_bwU`0bQ1Z#*U1^Yj!B)9HuW5;JOe&$n*C26_6a3e{ygl2;5 z^XJ+2Y7qbSYD(SBE~Hl)k3mm@#60_SkJNTI16y?>EmGKKp2*E zX456f?AkbN99Xtra{br1oSfS!W*ArI=&iXn0Nw*P7mZ~t*#;DfWKWfh6!cy^F(ffw z$m0walz8~Rxa6?z_dK14^{GOK;|*p7mt6w0UGofx%gB44keLtWgcn7pm6t>L+}TfL_EiyftI0}y?HZr?2g-*B^ySjfGH?egT-n+xP5vV&Xr2o z=RbpEcQRdZ+T(eyc#Ms`n;g!bG~|8CAZLrLHJH0Rx_`l7=a1ewTZ4-E1V=Q7*drWx zThY|iR1f;wx%+GJ-*u$1&A?3IvG}C53GXei>6-n z|N77l1MiUwD#14Vl8m(jgqh$B(g7Sssyw*3n^JJ4E-*(7!Sd9$hlXCVYe>NJv%MWp zOqJ4vVMC3paKM$@{;4bxe0y_QC#!$p4O>=HanbxukZ?V6{wf6kc2XiOK#}p005^Yn zR7J`F$jes@5x@^~E1<^&q6z!H96XB|Fup|K-sUu7zX5HIUBfnOD;;d{f5AYQHjmWe z=U%vQI6wb!2nB5x9=&t)i)UFiE>G<}fqb)8!5S_-QJ_L?*eiI-(Z_&0eTH)b?F43f zddgiO4<#X)8oH$VbfPm``SJ)fTgOYPSYI2_8t$zd zDbQpZ8Ne1$2@l0ywj{cO1g(Jalzp2=m z_JCr=ma3{cbXAx7QuyMpUD?kx`o2v7P;1&wa5E#FxqpM?NF8`NWYGEuI(Z`P1M1E6 z-2)yv;`lkD8(8tj^ZE?l;Qlf{4+a?W4Vr-v~4>ChH!v-0^_=~$oL7@JeywE_R) z8<|kJ*A!yP=BDW^Fg30eSg4dVqD)(Lo{s1BJtD9 zw;;I49xS)^0F9p2eVzS%HE9RF*($7Lu@+>$RREE0V;ZU8KulPoa^G&T;-Nv=d3h*a z1c>3e>LMd|?i7M?`^XVTim}^7g3!O`x3#nn(ioo5l7d&N;rU} zJP_sruWh(USvT-X5&rz-WBS_`{AUS_Zab7~f6`Ay5=(Mz$;Nh}!pg@G+U_ScNAR_( z{%;1MvMB#EfONotT9S zKz`%_#Uas_gbH9!drt;uh$Hp*_<#o_;ng3HAf$$-ry#@=Y@IYtj4>g{th#fj4h+V> zPa4Xegs{zvq;_RIA#qVP)v>VO3}@8RH8#>J^k&S2@Q7EBRx22&k{#_HUsFOfD};g% zW7Rz7^82A`g=~pSeAg(@TyV)wm6a}gLHjeHP=nesm>%4{OD2%|R#sL|${y2QxCW}R zdF&XA^xfg0c!3~TWezPRIXw}9n_OoInPcMQ9Ps&_$>_1k7mqAtt>I4;rRI@{H{l4C zp3T7}gQZ7p@QjTMv?6wey{8g(q>&Ra7k9m7kA$3`&ze$mFnA z&|Sa&NqjF#ZYuTP=qfqrG=4gGLi+)9Bot8Xp@$;IWI>~j@g8?c9uw~Nf5|=p#yLkH zypOH~fJcW*>^@qWED})d9%itL)&#@?00X+j z5`qCh02k~q#oA{J-NrXEb!aWaJylt}SQ6*Mt*WKJ;k~)ndi6P6KV~-oM*yV&?*Pa{lwnk4DzgYU`5_;t9Y_`zeQkNnqFiY$N(M%8F{{g1fjWIddp$(e!^-H~lz-#lZ#Lb|u9eM#VLZC6dX&K}0>8 zC@lnfJ8BmOvw|U5q^hdU#oA5J#M1Hx)s#TT$7jS%r)Z!%p`74HYYXZ+5Tomy=P-n& zV}Q=$^sE(O^0=L`px6Q~i^C>$TWs&5np*&-u6848jEPn!+6)mHQMHI!%xqH!l|UQ= zj2Q?-)&vhaysF-v*XG~v0~=okX_gR$l}r~-lyDC>oV^sQhgtcN13O#*p;yA|n=zXK z&4e)~#Qh|*83cJ@;NO<%LR&c*9JO3Ltl%iAFC%DFhL<$}jmBAe;gfM|+{v)T>kPTn z8DC9xv>-9-90%CF?D*XcWnq^_e`9|UapFK4w#}AfN#6v{X^s^-XFo(0Xcqz0U&nqA z7`=&7hr1aHMK-B!5QoMD3DU*nDTs_TQyTyLltM?^HIl)M|2mWvuiByVi822PmS-{* zBMzfWZtUsmC_dytX8@vmPMxB1H5$c^jz%(N1Q&2O-imSbRx~(G!A_tYG{cG%Yg4k) z*IB>*bmH2A4dm!q3CkChU#5KaVfxhtS(StLjK?_kiMA58k1QB|RpPyq7(u-!&Sb8W zu9Os!zMu)v*TU4ulIAc4OxOay^FRUHibfRN^kOj9oVIgUOnc5-$&%bgo3yJ77|r!!?eI$fAL=x zT#O|XH>h90_G=?_e&QKZaDEUKo05r0xlvUWsZH*Xl|74=;YF~(LjZn?2zrFTQ$(}G zfrpxvsLCUbJ~Eqb63~m*MKc#0olvaD(0pDR(Hha^Bj2Iq21J!H9-$V8*86{k`j1`U&$^&L6M(rKRO=W9J*NnPKgYjVUI04x|U7qI5XK#>Pgs0eXBd0^Xw- zh6wzqbAb8Qmo|eM3#PL-4g9airrKKdy@7EE_${m?9&EDGfX8D3jleQ%9G+<=z%bfABD)t%y{6`6p9dl~j_wB{=s|W8ev{+LFs5O--QP`DHrp=?3Kn)5&YGJer zIBO`CfJ|6`FU_4>uBBC4fyx+!624Mt`SPoFH1|^S!6~*iL*FC0%v?a;5x5;Bm!Ubt zAG={`gn}8%RZ$vZSknT4t=xr{S1E7@@t=@F)fWa$S?fAfU{ceGmz9 zGA96sLKZV_5AKo6ABhVOy;*VYq%jc4GY1F`$!5>l9`_zB7Ej!t=_7$rw*d`lvD7-8 zDJU^CjI$2_?U{>FnMops-))k|wI}(=%*pjs!Ah?Tm_-w)t>Ba!)0aao1pqMmJ z8J8#+KcI{c48!cO;y*j%y$e7{Ca~86#$2oeI#?D4L;g_R&dhKaZxXYZyHeWM+pSPG zG#T6%LDu1R6|7m9C?oXBdRQ*d9zSHuHIvj~u;-CJiPM6Pl=|y!@IlpO9D*i)b^AYe zqfQM}oaEJ4;>>fN?^ob~cM!h|d(A>U{8hu)-E>*HnkD)E~=GR;T zP#6WzQgEJKnOhNymhAWMgECmfb1=sO!Zcd|^#B4{i;)?{GA?TItWk2^z5DDNDXiVt zRLe5EXFhs>m(sKWCkS{VFUeP8LrDPVY{~|q3HO_rSh38Rxx-lp4iJ-19rWP0)%qyz zMy6z-L?#B1d_z@M4pmw4Ir!C~Jc3+VpaoEo$d$fjEr@tB&Ho+#$fQ7#&XBe>ICL4- zIVR~3S~x-K)lLEs^KQ*<{66WvWtZn6ISoGuhllCGLdb=B6|D6II>PmWA^a>+ zm~y;QIFqMN<)2k?f=lLgV&zc!Bxt^}*~A^M@!JN`!bJw*>m?2$LsNRR6|FPbP8-YB zz9(G!sy=rB*X|t@4RIcvdKP^9%+Y-nAoAh;c&;{slLn{KEwhVJm_T9E1871xG;I(c zot$o)`L?aAQ&vtc6w5#u5Z;MOzkM58wP$vlN3eF@#KcIr7|@dH5)vR|A=ddL>hLe* zO(CH?5N}=-I>yv#I0NRx#>X|P2ImU)$njZO2mZywADmkfu&AolW7>qvbnwiZJ2(E; ztx$V32cIDZbV$~kGlh=)b(c259-NCGVdXKF#3FoZJEA0M{Bt*GAdIE~ zVycgY*Kk(gm;g>~nHF3r$r^(xU}QbwK!{s+uch8?a9qvJlkA{iOuase^_1jAc!`;I zcYWNd8<$0FuVTX)m6L-}9kb$(taq(%^Bh(=11MHM_eX6}#|3*D4ZRlkj04&&Wj zk2C-QhI)N~nQQ@S6W$uxNgNo8>B+_~R5=KFx`UhIr%s4(;rw>|M*KuP3@UT z`e=)++PL5gK_L3=y_dVM0WT1-a&T3)E6JPPEJ%~zhhtSN=&O0;Cu+XP0*-!Dqci;a zJ1tJ|*a$-``!VJS!Er3G`{kA8_7vyN#1V23@PoYnV-@S@5wnk$I!wgjXNPeRg-FMU zVIX6PILJUg0J(&N`P?3~IlAbfV%HQ*A|;b^o(ZOqnV>!1w%Fl^uC*(B#NHi%^cZJ* z;JSpui_8O~jLC!j`RUS`Vl-9&eC~dNlSxmrC4>^hYtp=kJdU#@L-O*S?TxiE2mAeF zH*h_IcjBaDw$v1W%kcV#Fv|^ip_1!JeL7l%VwzZp&c0O`y$u$!+=+&A?6gf~TC0?55uDI&i zv%QaxoJQbA3TYwWr3UR8mOswSz%kWq2y6B^7`6En3<4P6zF;SeWChnJm?>!wa_SU2 z$sSL;AnQSW5_rYkoo%LRXt~V6iAY?_en~D2-Q@3Ef>i-2TfZ;I?&RFEy=!>HmjlR$ z&a9J6kE0+qwFj4oZVv`2Y`I7XgrtUdKUV?=F)}q>I&kp^$i&57A?U$WonK3!*W973 zN!C56hEgyyNMqE82m)=W0(T+U?Y})0!kRvK50h5MQ(mBmGn^jqz~MPG>f>!HoDwwS zd3o8?2-eq>v)w{sLGIV0aU2s<7y~sl`QHZQUg6GQDHOYlO2^W^b2cUhxNZ> zsF+8z36m{b67W{B0@q<5z;r{HF*!I5&T{Y+b!_6gf=2F7_agp z6ZTcI(4%Z7l4jw8o;Sy7L3uKITrPKRN`4;>0qjPP`78!B0v`ING!!qtpXdj&HW1T< zymVtDl(vC03{m_1-Atj`ht2B$DKvh7sTLBGul{UWU=a#03F%2bdw~{lN1EE&B2eD` zwYlP>0GmBJT+ePDl6nta>`hxi#jmybvk=NK=|6h%Sl4qgR^Q8=uY)`KM!i${@Q-&_ z2?^S!jU|7r$g#NP&}d(_gWQL}o+MFQNkzpncUcwv2Vl7@_gels1;jeI0wppRpYMAY zNjXltIX&qAsDbB|=%-=sc<5UyFszZD)A8IDbsf$a(ibp@*dvlV9CT@h)2+T7)7XEO zaOKf}h`xQ>i^=dB284Wy^#gbo-b_|iG_IdwZzI@g(6jX^F~0EQ+S{&K`SCq*v1q?~ z!!3mz11@a*L~<@W2>hrgW=Y3l=A__R`I?sw(BOK&WL6&AiUrqb+`G3AiFvU^6@$J6 zl1MBn{6Ni>r5*_m4zk$-q8{1f(@ae~up=~7<@fitG21w7PZ>+U8|>F4m@`AOL{I$J zn}d2evA1XL!Y^SWJnF@}e|2oqwgchUHnZP-IzrWpAD);>u%5wjGGnH6ROb0V%+`^| z9bo__6cL(=jn2RUp~1g5XFrA?khVAMmjO$Q)3tp>hvOn!e(o@St~4tB^05zYT6ipE zbgR4~+Q{E=`*kk{i0?8R;9vKoM2{k2o z;icoqWe^Wwl_d0f+KdJy1PorX@(hP!YxnM>-OmV`1+^&@$*XjuSz={SoaXlTbrajU z4)McEtCC}nWI!(sO;%j-L41mi1YSf^H$U6?-GST-VH*Ve;E{vQeo{|+)f_S<69jck z+Z6q%2b4P27XNO7BshNb3~Wc9#8i2+f|b{FW`Lc_-bUpgAGJ2G+xoyK#fJ-Ua`}q2 z5151Y&T=ts_1?ad`?4*_`9ubzH3S+6OT;(u^9p8H>HHXy;OJ~xWPPhq2aJ% zH;alsdsgrY$ccWo0({hRrt0AQOb^_ zKxNOy7Mcq_iX3H*_P`4S!8e^dpgqOfFN{2=$?20lYzgWSp7{`ze;LJv3p0Gzg<}>$ zILzzKuP{Ss{-fqr%O>M|;x^g+{{9)?N(z#_2=ex(X}%6{Ka+Oa@FeCMGb}tH8&C?R zqhFM0W2C-FA8H$bgV)VxK-Hi=_Rk5h|DvK>=h7jXqd%7M;vyrSl6f1B99%Ls4|3y$ zpkZxFBj+1ZAFe%M)?j}Vei_8H4n9cU>)7qj_i%Ka7gvfBfezE^G5WQ$XXB4kBK>S} z@ywa*u`Hc&4f!wNjEY?VLP~Yk(xM0u6r2F4yXOs(gX`Py=t-nghv|XO6m0B1y{B*( zf%Vb>32uJ}Qp`nYi2Ebuol77E; zr;CdT79D%k;^^qhq+L1u|F{?+7kFbkcpc+0wM=#R4^u9`Pz#@oG8C+dLB4&~5TY3oo&ofUM?t0={C|f+|HlNGesv=@4PH{?G~<9e8#>mU3c8_`ij3lN>;cAKL0Am&4O4d)x(kq>5 zt@mU3PMC<2%%~kpwo#`xS zJw4Ef3P0ot(>-AeyH_ujoVW?{>0#_is-h}&|M-7sbv1AYPD49LSkJDSz|t8H%?y() zw&K9o^Vq~0p8C3ofnCA3{`{FL7P}l{aJV;Ga01N8XD>`gS}8moHFZJ5F>MwRimlUm=L(*kcVt2wp}M&O>{uSp?onn!v%+d`)zA&tnK&5@9pI{?Km+l z7zdm((6>ppj)tEcQM_RxVg=UW=2ydTyM|IRs&_0x+WbIH?E+wA%nc-nDTNe2H2!8| zN|dm{oIt(6!$svQH83j+Qs~&P=@ei@(sa!y6oUmu9}e|>1jCOT{Qz^D`lzfE17JF5}b*4<#B{S}-VY%}st~VA79DQ05tc0Nj ziD{BV&!6#5%6QBOq1~*Zex&86IV>vFJ4%#s!M!}OfdxRIr%IYW9I=rHX=Mgs_k43m zR0j@#2^XpHN68?5lW`!D+baMYWAr)&b`YV24j(f`wh|tW zbe0;=Q6~uOPo@C#R~!3#v)(GFN8s6*wiM?QUw4Dee5O`Dtb z`dIb(K$ngGv-gR?domxpqon$3V>DzhF*^DJaUk2}dT<;YS3Mm@8YI{c5O-h>x;kTw zYwSWd7?U$RFiL-icO@Zyl!y)9X>k*tY1_phq(i5KzcWbi4`xxonUeBGES^1+bKLaSgfo~E(W{v&#Le4Oj^hho3C*!t8oJ%|lfR$HPW}1fAgDy_!9@0*!qm2LpP;NY8 z!o%E28G|6i32$9(2@w&|5KKEVt}t**O-)TgDvqX6S69am$f2W!k`mb!P2rfcP~OBw z0O-c=X*=yp%(@1rXPWY`7ae#{a=@RMy3pc(?74D@EzE4Shwq#Tm<~2cnAW$$sZ6 zEda)b)i#lp@#SKc?Y-}=@B*eY$fRC?o+JdTO)_b-Lfgbex4ywfAD2v;1MzDS6AcRl zgQ+>Eg0GTE9g#c;K;M9_0Gt~Bjx#G01^p5szCcGvgwc*khzMwoR+67@(jh@^4)ihM z{$Qc{c5eE}s$N4DJNn63H?06B76`8{)2dctFkoCLJ#k^s7lNQNVGxc}II)9d3Yc3^ zu>9`K%@L%HFYUnZ0q?o9&t34-pMV>jKM6opbTL*Nf?A45s3jO5s%Y1shD!;i8a1yS zg68%1cNF)huU|ceEahBcpOOwjk2xSn&| z6L4*D6*%*l=o#2(8$^%pI)=a;(Y`u49~EZo{|{@{a>pz|S#cKJi=j(BI%-mAe4tb= z-TKC|ia1Db7JI`2C_Ed$f*{gi(Mw8ws`Q;T14%XQgCq1|yoUC%|K$slQk@u;u&w4` z9}kYpr+TuVsdxVdKj^}2a)|W(b(3Of(yB)9ZNbJ;Z2YydYfr%7q&Gz4!c?(!xpMLD@!>Y=ou z2@KS}w29>I+EV%F7Q6g+kW(Q-6%EWvix*!Y5=i@#6>U0*L1cU!QujiJO|t|C{;O08 znRWYcgW1eI8LUK#ZHERDCT|HhN#6mj)UX3z#Qqr%p^g-3|2sTc+(bR#{eq- z1n4czxH}>v9+SosL9vv@>hk}{dJlN4+xLB3OGcT66d_46%P5hOMA4#BA}dOEcA`RV z*(#AJl#xv~m9k1m5wbE$k)2V(|F~{F&*%AnfA{P4d3|~$_w9XM=Q@t#JWt+r=Xco` zBq#gTnW50yEt#mFXSSoafy(vZ?jk3}+&~%}2JGQ4qOlXGDTbHtMq4c2DmWj}nVPl| zxit`;vPjZrO~^FrBv+bQ>WY&RebS~rh53f$S5%3sk3>BZe0H|~B)9G&rzX;G3nn(K zPZr>+EG`s5{x_o8cMP<_f*#NL2NrG51eIGelzkC zMucL{&lg2904cIYb~S)`nqUL6CW!REWdZJ~#w2-SAYh>}rdf>4AF=DaQFYjil{&#; zZYTc12(V%jJLKwu|3ED6NdlAB76Uty%<``I>!+q({o|)>p$ask{iV!C!&?>sg}Kl> zZDYuimhX+Y3tlMnAZ(*~@VOfgiAb1zy?R3V+K+UjRMlKo=S5HK#tgK{o zB^kVw9?5KmCrVV$kQw~)dwuJ{0}0GW`e%Ztfe1R-3o5Ye7DHG=F@!bKd>2=z~g% z#J~*dN}*wLKcV8XKH!RP=F0@Rwa~1KoC_1qk{>f4I}A97Oz|ROd>2ayr^Y_&TB~EN zAywCrxeY)d3ir&mfeZ`<$KugU=+b%#$VXMz;5N_<`OFSk2n( z2b2SK&MHnT2j5m-0I^8TqeyZEpFRe3UZe$zuk)u~T%9)E_2@&f0-RD=bn9TQ39h(` z3ZP{||G5leSR$LRO*_ctnbC2xfjX-RoQEOI8%iXh)P zrB9IB;;}bdFv~RI85f-S3bNH4UFr+4llU27YO&4`zL-Ey%SrPaUN2@p+i#-&L9yr? zv^9%p*gh?rTXzQ-pD^qo7IZU-G5Pd-#pbzlDF2CdMBIJl=-_aHY?to0 zDL4~`0azV>p4a$D1YwLeMh*%Uk?^q*=+k)WuKzcD9Dmz%x7|wa;616NxAQYwCGxS| zMvv@fhfp~)A2vaus1a(q=!`fA{e~}Klh0vI>uKO2Lqlkb4^zr}fHARTZmJet{(dx` zZk3AJm070g~#8wATi%8CbqE7=MY*`sLlUm8uc#h5O(?X zL(m`(e(28sY=ZMoY0-0f%-(L} z+5}ai5CBT8f6`hle(0)svGRj7{gZKdngKS?-}>xiTCqZh=_#I8CZ_lZLYPEWD%H#v zJ$(4WE*~Y^_-*?o7$l6O(&+)U^R2-|KTrnR&A&N8`~gyJV?n9SRW_kJF7){>Euw&B zAg#sDjQ#)aMHodZYg?AubY>+lJA{J?EG(NAhLdT6#PkFpEBYe}ZOCM-E%2v~h`V8! zv5 z$^s^#mvEUROM`BYM+rG2&5Kw-jer*)n_!9DKLekDOOcc|N5Qxslh4t8o`oh2>a=L1 z0c@9{E?=*%tqsFYlvA}~OG8&8Kz!~ZIW1l`>Hoq{uimMs{F6xya5GoGTj~)8E@tyM zAQ_NR1f2_KAnjJX*U`?j&~7!fA2sTi5&m&D`F|qCl51DDB4~$=^QcaFMUCHb4s}Ewkszh_eEQ!?7{ZL`bP*@WCk~N1BC+!&2zHrQ%bPT0n-iS`u>&{p`^4}GH#aqE1UCH zwQ2ULFq4S8=P@%q=+`H^h5p;sL{@fX*8_}S2u>vN0b>DYOhy+iav;rVzV|N~P1JBpX5_lFIN`FQ?$tW@E z07bI=9J@We$!s_RIYMvn9m`N@Sj+%CTeKH8+Rbi4A1^@zvFtSC&%IYZ%+y^yAlIyp zT_z!0iU<42Fz+Xgtl1(`IZQzlAus^3ssFagVcA?7sq=PQOd!Z(up?5wjn)`IW5)Yv zDZJ8di+_wlV*5%eIBRn_KL(sY&ti}01DHZWTB}}TQUZCz8h3zwseyg}+py9mH-2s4 zA2eJZqA_eDrE2p@S%ixEBQkKSIycH4YL}Kn#9gujg*3l~vyq|YKmkldXUa^lPn2}B z6GF5M;$zcOj#~dnk2et*iHI|S!hUF=rL6xtzy$smbKOYjZX+{ts8E!uDlOyX5J_AG z(WxTpCa5JCVkHj=4e11W4S{(t#?J>3;5T(71I!IIeq(6u!jMFw5GP$&)RuQMGqY+m zB8Y)n2s=v>9qLRa=pQ-S@#tT~LR$T=I|8!tcY-E@$ZImN4Y-JlB_^Iwi-~_43 z3sLi>&XZ{62k7>(fg$`f3C_(1f6F`l=D#xjvgepxJGgjNc^jrwYMP26 z^Ht?zIbu3)CLwed!pE&Mv+mFl5mrY?4}DQvTOsxo5kcXQL{wzsq%BLi4fyy}H^j(B zT+iG32%`-A0ZUJjm@UCrP;AL2D?nd|?%Z8A#^cj8aPERUO<4r6zf;fBb@HsD@KK>t zZhJrkN#er7!5{{x5aBwIy5ym6L|RJf3O{UJ(qfRJvUQ|K!(+~kd>>7>4`w9*v#KV; zG};Cot97kQHbNGX_s4;d{|ZjUSR$)FKo)(JtwO?rhF#Jd_6OW_CcfpY7Dyr5R;5|Kd3?DQNj2-eqTK{07TgrXjoOf@u~hbD0`%vy^nEsbJUb zXblzp{%6i6*hR2~!(hhQO{uBe!M4-Y-d;DXF0;<1_EKGFl1OZHx%&wf6+NHX6bkiK zwdOzpBl`02itQLaUBX&Csix8(^Y++)yTe0(r|jME8)Ubq_DWXPiQA7J2^N`l;)>o* z;40i$T3V`{^5MIeslAGdO2dSOhxPk`2}ffiW8<#DL9t%A%))F@{9yN2lg;Lre>@)I zT(xScoSYoLS*wO0h4MQHv^GBz9o=Vv_=Z6@H@6>lu-L8I$|?<0XkB9*4d&nL{6h#I zZPlt(pP>d)w(4(4+IoC%LlGJS=`!QS*;G^nA5>JR3U1oOSn&C?fkwFGp7@&-tEj>I zH@|&5A0fgoAkfcin!N_rum>1(@Y#GDe&xM;&po+Wc)i?c51dQz+HIyQ&YU@8@`OB( z#HsDhBYY5diS;&^%ka8><1cd9WrZOjD(2=-g4fEJYZvwS;``2jKA^6isoN*Bc{Bad zqer(MSDB5fKVJrX!4^~7iz-7yLo<-RcSI`V$;{pmRL5LYPWS~cHys@=$mTHnivO^o zilRR`heD>AQ>@fi$hO;doG3iVtK_T|kAwmntHZ{jlZ!k&JeDq7Mi1UiG~x-rKx}ku zqq%L9o)dimSmvV0yGc{@^z^JoarG3khymXrz|AGi9l$a$;aVic;Jp}st2~5;Fv-Z8 z_S0)wS&yDI+ZypB@NRCUe!n||_sg_XVT0G#;t)N)*~|0iDh_Kj@JMs7Y-?-tIs+6$ zqmhMmf(IfgGM_mZ*Wy+ZH;$!W6WhJLnwy&LKHtQ0<<;bU&!C_njXS_VH$Weo5$TzS zNm9ah&ba&ird1COM#kK0(RT)GyeAwWUJ@1+U1f&>NM(=IZ{g_|zJcoK$jOtNdf~3W zY`#HGgHv5yy%gh@yzd`NDuZ~FWzCwFp>cG8(8gt2uH5FWd=2ww9CV||^JpybMiLMn z(3&%LT0aLJpL@Z3E znn!8wcP;~uACIFATScpLBRKGY*5~%!MjCzYi5D$*tLbN;Ib*9}7d}7+Vjn-Y42)Ec zc>#*@9Mz0nJrOSu*Xav3YsK$Q>J443LURNo(#tSqhx{`YM+9%HCK@Bq1ddI~zOLJ% z4w=byZWIzyjW)vXRalDp-xgpIf8W4>ACl3mQx}wi(WF(z&(BXsPai@(?br@}F|n0E zyfUjhkrHxkHBl?7OHNO(#Q0~gws0w97A~&lFEVVn$wxcSW$I*Dw#tUrnDRe&`(>Bd zh+7#umr`83Zn$`y2^^kR@qx!LQPzgszAdPAjvQcn50vWDaLwNL@mHSAyXdT~ts|ZY zgf9PoPGfN4TqyF|X3}w=3_^-lz{x!RWU!~CzU|iwckx}j^eM?to*cnsTrC+~4XrbJ zdQw+@+w)^twjY`bTo?orgjX>XnP!5cqMn_!hxR&rDXeo>H^Y4i_CpXj82d)A0mfOWW zpz<+lNIc53YYKz!BqKCF>cku|K)-~Z{s!V?CM?ljE-WI#W~UXek*WLOM&cF>6z%~u zDFFz zQ*XXIomqN@hQxmtin)5p2X!?!=K{qgK(6 zxe=oHXAWYm-cKi7uM`<0U-RC+Y1490ne9hm4o9CGTt0I=N!!NOHYDlZHL>m6n}?=& zTjNjTZKu#*|LVLnLqTvbOT?3yYZbX`@&}a@aB^8XqE)Y7v)b9(lBbT3SmLdpUJ1n4 zs}0b1O#OLyokXV)!Pi6gJ=4?E70ThbJ|pvR)NDeEK~HXVGQVF6}t7Bd&`{zC@obhZjy)A>i~(0Pr${#nn)jdb|L%e zHqb(tsH;l&5~jJiIX(SN2nz|h28C^v)hsP;fdcwg zDgOSqlNMZ7eM5s6ri;8Vkfi$s?`Mdk-gz^T?5NE!e$rFwOQ1B6=p%Yy34NOpt$0t3T{g~g`bxqsguQ@qGeuDf)2 zRt;n^3-kJ2>v(y2sUJy2FdJ~OF-mttK;^l#qM?^)QQLWCjs%y!!u)&#B+>sK35VY@ zvN18T`YGysLp|=;og3BJz#9jmEueWefSs2&5MofaMT-_~J-(m(8j5dCw_(z}7lJRp zl6iwLDPs^vdduCn;Bk1)oJ6p}tDkagXfe4RxJgdc>3z%gBLuym=Sj7QY+A+183+EK zSaaegaz}RU+NCVMW5=3M!PnZX4no_v>rk*n=OHrQCy0%S^B>G}lr1E=KJg^ipO-W$ zqvMAbyP`7{Kl_v6%v8#t?OoR}%MYS|jRceI7;J59u!PiGqw(EtUHrQ_|JMfqdtyolRl3TaS zjSB{t6|&>Bz)e!BgJv+r5N? zBwP;6#AL`THc1(T&vn9J^?$GP{E=C z-T%3Dpn1J8GdmO~HgO2)f%1>RK})&LP=taL`}Qp(iCJ5Q;VVhEX&d6juH>%N zBpG&^2T8wVz^zZC$Ca8_VR`N6uV3NP8yZ1)dN;SX^8k;$Lk(R5ztBu&go*4Bba9Mr zQ1yR-IidsFJ)Zy)yfV6=%YN+fF+(VC_4Kz;J&hiIo$U~=1Y`QfD-?_8o=|?r0dWE% zLk}<|>$1iBN9xX>uPy)F-!HOvA?#z+=4uq)tL@Ob?mSYuJmy3)@^dwP;>sGXG}lr6 zDTN~ca<%2c5hdy^TPbFDcehGI`f58RMov!76>DWRs#;_V5H9Do@bfQ%FI6T$Zm^(w z{LRD-;^N|TD^{$s%hfzb9{&Id^|IUnFT1bh;-Y@;>DfT7RpO*^6?Pp?Z(__O)^kEl zjnb}coLiIg&*`wS36RriiIxZvq$f}aqDn&%($0s_Zd8?5R#uZLiFJY&eiA`WS$ykO zk`2K!8g`=Xk|C2s8E+iG)sLNqfcG}M%_A`#a(4gYtnIs6cdWu9aH{KVkP+oz#3-2S z#9b<(HjQ@b-;Xbm18RdVv#YyXK#S_>$55POQ=`T0vrdyWc-GWTIm6*2Cn7tEdcid#l2ymHS>w z?e6aOnuaV&l4}LkUnssVFJA_tmdy@1R}}zAVlg&087!oSM4KLNXFj3+{0}>x*^-na z!ok78e7)jl@(uEwSKtvYaxKxPg+{(67C8>C!5ajolJ_ZN;?D-u@{`5Lg{TU>y7)bgE$Zqs3ZEw~$hzqmKV> zc)0LkDe@4u)(=eRJa<>bn|r%CJ8wtswdBr$FAa`hA?egZQ!-DdRzPoD`1AyEKK-r8 z4v*^S2=^lXy}jtya1Q@&6l#uGXf&#d5Sq~DA1-UjO?BOP7OW0r5(N<)ZQsvGgTW%Z zb#%ftj>E!t<` zi(ta7=qkN<$rA4~c6M7uIsA}>?6-)l0NTA!LeD=vovL>zy~$~MU28cyAtOU>-TH9n z96rkgBdzBiiVSyv#+`R7a~a!OWY@z8pH)vPWF1R-S|q-^A&TT2S~h>r15>UjJ>A{; zyQNuK>uK(Wh6<1(g%T8mR5`YTETyDeadR^v$(Du?JF$`8;^JaSV2;F#ZG(HkeP3mV z)&gqeXjlfE*A`Qz`1^W#JW<7r7iue!@e*Y*F)@x)3B>#+$bNz5eA^9>=z6=sN0YKX zA9;KZPt+kFpkoZ+wgsJ?o$XJhlH+U0gIu7ST1eK_F%u;>ZmKn64)svbP+4^`=QMPp zOqgX8_BSbm8jmDp&8f#_AgA`{mN`JmNao2?`~v;@ZFS^h;DTSi9mR~&9Ix1GRVE=v z>Mg>kl}l=D%Zgk3LW@?kT#)82tEgbu(P}D(58|=;&Vjelin>*BF?<-w7=1l)FDf$S zGbc{a`jrb%FZ66A5;I)`UFzW=TU6Pd>aLYuBz(lWA`5M7Jwfyqj|`Y=@O0EMQwN$A2>BJq`{sz(1+s_Mx_&FN1(CHQ2ys z9_Ck3@q!kzS*GkBaODp*?!K;(SU1Q&IX3ohF3}zV+W^XZyb8YyWWNknR#r$?+xMh5 zwMOY`sHp{0<4kO{nQUYxR>Txt+DQ@x&@pZ8?NJ>k?Ci2wE=qH2M9JqI>=z_Y?pd8t zrU52yNJf;Imcu*sIoS2Hp%|KmIu&T1XfN*1NE!R#=G05n(>#kB6yfK!)uDdWLu581 zYRY~^VkL<)nGaS|kA^kJCs}FXPa%8H)d*#jE3h7&%AygrmUWP~;>)l!y>;u>lI#d} zMqdi;P1X&ne1d{1Re;4QmDlx!Np7Yo#7KPu+<#H2uCsvvL?afkFZQGH>_qUU(4H(QxlFZTOqs$-^YpeFDs*-Y_ zWMzF8eTQ%@_Qo8kBJrtyFk45oxzJ$?CVhJUDRvnLsDoP$8)5;yMUM+zf!V_iOhgLGMzP&gNW&g{Xn$&~lKsWH|he484A)Q;B4y(dj zl0pVB*maO%r6z1mnoKKKmV@;6eeq&<#6oJOkJJnk8yv@{UcP*pfmFM;Yr|ranN2MH zH#3|3PiB_zUP#|SP@tdw@%?v@SOJ_g@+^H(5r?dh0yqTfLhm!jkFQ3I{IZ$y=}TAF zqQi#|18Y1JhDcTzcp`bVQHg%%_e-jQ>*#Y%TLx@Ye~F5_s92VK{I`G4qz}|olZjUo zJuGE8u@r!u2+2z#0By0f%C5wdX_m=aBWuA$1q}&I)f?Y5U^~>dgI~`5kMntHZg{Hx zlAo@MD`-6d4{(wq#uyb0npauvpvX4dK-Zs*3I>9VU_W2qkjET&|D9w#Ls8EslCmN8 zxG=<8A!QdTL()fZgaj_~jClps*~pN?qRXx_+$*8?UGCvAulq@#7)en<0srRB-vbAz zMlu(ZmBogBIZiuwclUqFN`7@VI8;vzd~Q}mYn!A3D5R0GvCsu9EIK4K)X3>@z}>r? zylx|V3Dwc?g<<(m#6mTkC}j4zcG;H1A$HAF%_jo z{)KeXyZ-#s3Lu{VIYKh=UDfq2wCmSDh?el8oWBZP)}zcj?Z2@QHmp}$ zWHpH`OP4S2e4mR^ISZ)-3!xd%HWkVI#(WyulP6CSQ0#@tRzpM?tZmG^ zZ1HD$l;ps_)}VNabst2A){N%$V67x=ogzmughG=iHxs7}3G@aSe^wz4?M7Y>w=kHgH?5ZN;q~9t3*e+qdehDch1T*& zJRYd`-XEt1c$eHt?N(#)*BgKqQXJ%Y5dNv?Y@rG`qIQ9Vyu$olM^BDSOswukOs(C| zCs6nr-NFW$Iw_ezZE>MZs32vpNv2Nly?Y|Ps{UMtQ+cadbI_T|Y6RdlKR>?;kFuTy zpy3Gm3+Q*Gp70}GuG+XPRRjw|IY-ULsNM@on-{qI-uGeiq;?q^5{LJSqSh{OQ+sc$ z_={IY-vYWzJ;KMwd@-bkPei0YEbs39`=VvQ1;V8-yyt0DBQavO63aG@LNwPaqmU%$hHy+ye) z$ng=}uO#?SAXDoyEqQfMpBB;DNIu~M)i_poui=JqU{QnjUSSGOTJjq>h5m8485U$D z5h5{>C~@~ZK1Pr9L6_t)&CyDeKYx1p`1p{T|0`m48MqMAA3zG18G;zwT3DpC|GI!5 z#Jycyuh0ueNKJl8WSON#7RP6_NI9RQ+iq{d2l@h>%ggq5J+Lf=HwN$fA}---5j07l z&;gY=0V>-ke&M@5weLSL>xpUuKKB3AEQ&oq*{1uG(|?Si%drf5`m9|pqe$ONTwpT^FD1rX{h^iS_?L&p87XZ?UmI8KZ+Xi`!fCs`8#6&mmXhvb5Ap)N^dDHp<&3?G1FUZ=YBFE zM;aL)XSah9Q7ag`Bf_h~@!W$i;s$~qL~*;0&@SCbaUhhUI-jHS@qc0d2?fv-US3Q2 z?}mgdy5-x)hl_*Mx?QSqGW_5e02BN-i(PI_E~zdvEosd!yGJA@IJ>%jk|}~^guvy; z>Q`9E66!*kAlg)*SIsI^kbmRGeN}KM%~3@gV`X#uxKP;lZYF*WGrN@$41yV%m@2&J z*$KWm&@3Ah2%L;y&9SZan9>6ZfuR6B1x9Fyh977j1PY&s)=s;B;h2WKSVIf_{@La) z!v%f3{=AC6!}2z%vtd#_UvF_qiA(J*e0U1Y)46B)0Qrk1j0vT+;Le_<<>zyu2o)#Q zPqAuTDQ1dVgfQ# zIQ=fj^}GCS!gVf6Q$=MdkkjTbC}>vP3Eq%f^OTHjB;`io^}&K3aEP#@YlmWvG|Mc{ z)8q4}oPYSHt}bszcZaz6p}!}97EWrGOhqI2Hpl@YVaOcZL5JI_PvyN%{^h++`2EgB zWm?p^&7x16;4Y?sNI@_@3?3gFdkHB0;Su59%{L46z(cb5czg2;3#(DzgHm|b-F??T zm^=Knlx=re>!+`9Hb>AK*NZt{i|!n7Xmuc-=IA`Z>Ohgw6{z15;wPle^~5>=RTazE zyIlo4%0E0f7=TIIZC09UFsax%)lfk zALOG^tsz(4V*j5%e|~TTXX^ck0DH=3XRpwBUK&(yVDsD@1}~d8X#~QrGPPc+T*$Xf zI@h)3CT(J^-;I)QV_8>M7s*tpwL)mCFZ(91=E}^{A|p# zDMe?fR~yoORvsQP;3RrP;X;9Sgq4eHp-9h5%*rYLy_h6nDyH^Z-cxdDTXb{kXgU?yuB-LqZ<+Yh>b!fy{B*VTPXB zwtoGx$#_HLunQq15(#j(?(7v|DXF8B^fLw5p>f_RhKexpaJFJJrU;`ZCP&D?%ZbLKu3Hx6mTG>1WMy2BA$ zmmS!6_Yq&e8n|PY)@V;YU_!_~iQJm->KvjveB@NN07jvp#}u;&sfAOO3xG3pIp#%f zBVEa;S*o>HPk3Q=S9p!(fGP3>15(mZ38#iDa4?xr#Bi;k<$n+!PJt@4W76e5z-+XK z(iMOjp`U7j(XrOQK-!G%p+Mk}{LfKlbZk1Drx7anqcx8iQ#h^*s=qYGD5nZEs4`I* zM=EZNs%OUn840w8kda7KNEDf+MiH}r3&)mj^CNgdc$xUH%L9Gom@2ZoePH7a=Jnzz z>@Vi1_*Az923sWgRux=6zJt2hH&*`jah! z61W3yti12a3VLv4#0J%jJ*nYoTgNL1s!2yj>gN}gmHUxh`>?aFRKfhrMD&pUuY|Wd zsHwS{TN8(CMCiv82>fdTR1vR`CR}1cnxF`n?J9|+a-f1_tDx3!Oj98R9Eg0@_Ip4% zx)C9it$;3qWN98aTqMX2xP65KZcJU%@DjB?d8;=&N zti8|d-c3^x$aA^qG1Xrw+4VPUF1knocA$dv13&QWxvdcy4g)8!KKkuYCJJVSpYQg* zBMt%;fl5O*))Zu%tL7;B1BRGDd3*P%Tv`5q{8iY0xRp#bwkIxzA(X`Z$Jf5iXnPty zvzu^EWw_6dj9XdYmr#0>GM=sgA7Ui@-9hB+jJDu8-u>_0_{ zSG*lQ&ys+_(S}LqgJap2PtFw?pcQ0dB`{6pbx^lF4$aF5b|a8VKAch*xe&KNxA9L- zP3<=@c(`CE_fZ_yM(y0xTM-1h>YGYbVrptm^{?sasw4Ecu11nv8CXZlkHJ|!D6*+( zFf3~?<{AZOM;$~x*|2u?s#SbDcATnmadqu}HZ$Z2NkfuIKRq~xd{pPb2Q@UH&>_y= z3Q`#_Xp39*HrxydC`ZCPbUy7Tdbo~~a>}$Wy5?6CxxIwDf<+vz0{38qg%QXMt{lH~ zX&bukgJ6I~l^BcUjSY9D<>mWsyde?7^@OqU2Go%>arMHz*t24s$05t=yO3o4$2@)* z;bY*%*04rQa{QhE>(QXC2UBOxn#(wk7$jsEY3sV;+*}Se&Q8AAMdAb&1`OG}4t&Y^tMb@vm64H!dIF>pK1iZmf;&-^-8Rq^Mck*v#II`)|;2`KQ^={bmU*m2U}g+4dNrY_K!CsI!@BmC2r zxM-?|0)!c~5)|sugLLvNy$Zo?vz1;zMusFbCtm|=51$t0Sf}88qzd}p+%*rC9y&-5tY@L zafN>=(K>IwnaI3QF#gNcn;hKiw1}KKu2PZqnzw{Zk><0VZ&gh_(JHq64w}LUxG0$L z@F5@Jw5y^NT;5wr+4j2tzHHU{1B5_0s2A zc-^NB&r9Fi`DqS?$I(-#HdF28647+2cwsnpi=NVbAPCUA5uPg+JpjcsLq&;+i7aDj zX=%>iVtgbCNVkA?omxP{L_k(_OiUczkkc*T_0Ng-(iIKU zotuI9gAXc^s&ghL02=cHWJwWt%r+M+5gI#f|vW_7$e zq8t}fP@Q3@ovimm_j1R;f5Xa-J}5BqvAZPL4^9bJ45fZ zAa7RgK*mQf$fBYd9?MP?MmTY0acSuWt64E(F5J#=l9|*}5nKQ|r@`J#?8AP)3Y6|< zPP|PYKCr{`w%=gl5m`_iDGuZ6P#I?It)x%C)1UU_+W!O0jl zLo(rCEzuizce!o9eW1e%*`zef<42F^Y(Y3-XE#Iws45ln2)Rkpza@JQ9bzJ*s(OLR zvaqmF^RM)0&v=nuCH} zA)*VsuK(unYum`ex>9Xr+dKmZ1J>26HzMgWN4|N@82R)|quT&HNRvmVz+wX@0jkUA7@>9m87K^^s01S@0#ti@OE?tx1|f1?hWR=&fgr17cU-~meCNp)&6NSt|FJb{Wl)#ND`-i=K{OgfOu7<87{d9nQmADU%wUr zXmEY#5H)mSQW6rPJ*+V5)qcg1Jl5HWLkX_{P*TyE-aiVlx@81uqzb->p$Qa5WQ~LT zAus9R1$)S!&jAm=sSL$=hR*aZR%D{!ak!M+i-UJwo)nE#eIrU@`P#K>>tW#w%+)R2 zXDIE1arN_ZQmHhF>x2k|4OV$nnxq;Mr%ctjkR-N#H4_sw$au@->WKauLROQU%i{8` z2sQCs&6h5zU}l~`4Qm*3A?I|B_Z!q>w28NGui~E^9WAA>Za@N&I1Gz;5ibVvx(}A0 zn3l$2hrs}!Wjy+w$PsK85n({#;!84z{UASDb8Cj8y--Vz6@H+>guX@DEfxWKXU-7t zVhub5Jf=J@*B+t`{)^B8O)lz{!3+cAshtxDhTbz5$QhjfoRpSEBr4wbgMyeB>Ys+Y zgqdp1UFpNTKKHT(DjR3a=8pkmmq_eEjTfvup)LJYw0QhJ`S}#e(tzTJYAS+^hrVS{4+1v^8k+??z7-DkGkA>vF%Cv19(h6IeZ9lO z%v9znvX&aiv-KMGrZ;tdbOboN2tW==%4(ti-;eYLPBgdjnE1oSonR-uyuIlQh=LMi zRIF}1#NXR-uJH#bDma6S7cUYhMNJ>KKv}`0ilyk{bKH@JdC;SL=+~*zj8<5K2mFxF z?naA+D!O8W$0sK45Twf;0h0;OlGQFGB!p|5slUc*@Z5w=B;XabR0_mcg2jZ4632BV z9p6a;p?itYtFp54RZWd1Rmw^!e20`d7#MFaiZFB*wx8>}h{F3v?d6KHGCo<^+=IDP zQ1;3#h_atu*-46W;+-B6-+S{n1H=TFFuD|tNsd994|eeYp!7QMc!47ka0%BCa{K)~ z;otlUe*Oyy7{>o6VECV$BPVqJh3k*Zc1SDyXs({h7Qna*&#fmLluG8^Y0+=D1y8x| zwNA|o4)$^D-oBYYu3>R|BX`4X z>8Bq*SX@^|2VTKTxd!&G4a-YEqz`aAadYbiKhVEXI3^^mExR{H`^Br1epiL9y@E!3 zl3NF5_SAp*_;C8zM(&G0gWs|h(-s&W@n;s;zp_8;)3CPZ>&5D!0I+CiT66V)k3?+0 zr0Vnjy{fcq?r!s0$FBWK+(B|3;aiMz+N;k?_f&-l^nPjVYTE%mcIC)u2n&y;>WLE^ zv9YmMms^EyE#s6pfqcwPTU$CV>Ge`f8}{U-iGU@(frafKITMEX*t6A?bnfaa`Nkpuxpbw4PnL zxp9z5Le+=GAhq`}=;{^38!3GoaK5yQTaA-Wv^q5HNW8dj|9%id&D^cBUC1Rn4x;j~ zg@OoGv@fCs`t{x*F$(d=eY3$r0)AqoR5vJ?(9oc*CMT4*#Kys)9BC2KxkPJQd;QYM zwk1H`1F%KtlgwG&EED>^lB3T*G&fTq|KM*rzA_zGtPq=$Sto}(C3>+m!v^CT_ate* z0cau(X#?`n|EN;*zCpT)HiWx+?oh#T?Q(OzTT%<&qaG_!NX-6lt`*!Svf%Hy)K%5h zRM}l6@Kunhzx}}rxMBM|1{brB;dvoBkP|v|Uw9#p8VeSr#|kAC45A4RqEKGIoVG8& z^G&wxo7u#0f>Y0@h>s5EDkONWH>ST0F4&rtGdy^yd4AZQ=Bjn#^61Wj(_e4?mNo0F z>mQhKp6w9(IQ`v2;^k-G7^cd(uIRcWTJK#`Z92BmhtEgZ>Yb69pq23V==?oDb2j=0 zef5p@$y^J-Wv>Un+zslwbMg=kO>A6A$lWRbSE%Q6C;_uUGKUVi(`1``obz2St_!q%A8!Zz-8zxbB8d_R{clH%F$ z+MVCgO)_3lnl0{Y$DFdyvc77YkrKxI`*u7+o-JKmkNp;h7%$Eox!fb=tlU>qts!Q5 z|M#D*Odd9+@&y%PctY{JBo}{pF2iM#P+`|VJrj4b!)^ZCmwhM#+|9zXs& zlJ~-FaZ%xc?KXT%dMY&c8a%8NYdoSAS9vi#s_gRcxrZq!N$udD=bR^7R1{_`1hk** zoY&A1WsBFGS-N?1clYV0G{t~$uJs9dm&Q!!v5G>8=-hoHn3x(C;uWxA6fY75I>6CG6d^T* zQd;2iK#s~RtwB^Ire`XKed4{CGzChzVzj3+6~B}e?C>FkhAB`N%71wG&LB_69bz;T z>Ar4mWdlDUav_})f^m>V$He3*$9#KVv4g}iIUo&Wy(!Ug2ll=7wc0~4PDoVdh!)Kg z5etj0=nC*Q8>D|u4)+N5r&6=K)Dz>`haf>e1Wbhh`+ob1JFj4B#R#@K^V;{Xr_z_z z)qa|xuCs+TwIadFWW;`lO`# zo&dR8ot{eGwO0;ac}hMFMQxWtH8>XXIBd$oz_N*&fersCK z9xP3&lW^L8sPo3BVNHwl^^fO8O~mMEnIx0qbvVjsOk^*acS`%b+0yfL@ZCIkxRHZ%(POBqxe@V zdvj*blVIi*_MUlpU*ntGCzA%Bq|0^h>}+ZM^}?!mAo;t!hNI)^F!9-S*jksjv*Nrq z-^TLm^>Lk*oM%(Y8j=hTmAIRm)1OHDvv}jifT+Nl+O(HJ?)Eh6-E3HQI(7I2voI&< z$0=HuIB;y`&Pi=@M=AB z&v(>!)frsMre7;-0yXnR=lR)kolK+VRQENYxn6?xhUkWa;}bec7lWP%1Z=Zr%i4U4 z$Tw6bhAOT+*IEojrgJhE)nV80a1dICE@GwiC7fFRpe3@v2^pk$0rBIPngxRr6NN$R zs&(FfAsbmkj7dYozt|f!qK`!#dvc@s^vnzxhmb1BjwlmE>9WLQlVI#3vb%+X&BNg2 z|F|KC2ZV)bw2X1}=K}zY#cMyuB8qKA!kAY}s z^a-}LZuRPAh>=wMCkxyoaIhQDHt{O~q+YRRt4jP6H73+{REaCB!uwbf1f_Vz?j+3& zM08}t+GGSvm4&Rb?Q!}YNRhsNH7#laPKN66;DWXZ%0Ojz49P7X_LaX4At(8Fs5)Fw z022tBj`5OJyFU2`bHG2=qSuq%&OC=3<1V}t5)w3yPlzpN=IH(>=#2KPKO9^j(7&>| z?n&R^yi?)Cyt`pEogZUxCbvMS?q0uTLn5{HacNpw#u;<9%Mp_H3kcnqdQCua)`?{z z|7*mIi}41Rux@j0%GJ!42FupYKktt|@^Gl(-lDgPOKKaNRL}L83HyC|``ELXIAbN_ z7SkL#Tkp^8QM|r$Zn&L}_ebmZNiErPr{{;)_FvoIc|RyP#{E#^*EJ9KHH7b8adYz8 z;f&?dLn7h+91Z>4wu325zb?qTePCyc@L?P+4VvtEoyUnh!oF}2rnP?dCyn@ne5;xZ z3JqLgt8R?2E*cqWdTEsPFgJz1Df@}dphl8b=khC_tix_ITW9gnJ-)rx*e7)9)Ie@NB_qMmG&QavzP1BYcu3M=Ax&cuF1)< zlsmYRyZ6S-@cO{|xaFm<_6EFpV~WY)Z(ipdr@K0e;oKk1 z?N?lRo_=cG8F25xSwGIR3a+|!C0!Fuhr3~-j0L{zo|z`P(h(#`in$@WB%;fKizcr%A;W23_kAhzWv7U_~elF8V`54{b+3}%)eln zy6ECZH{bN<&(>#~?(2Bp#K!TJUQVvB;~?gd=zfxAT`3BP_Z0*P-fIn+c@-$jd7XDx zJfTekJI9}vn1~g;*AJ9e2g3*J?fc6v4O)Ub2Ydj|pyghX^QZuag#L=0)6Xsm;JXy} zp})X5X)rU)l^Mz#{1`B3I+uXCQ6NuC7ls+%K9s=TjnFH+Dyoj3ub)1U+Z&iSf#+|4 z1$K=_FD*7e{jW1L)DK@FWwBpSkk5k$m!`N_j&5l+{Q*pX+Q%3awc}F7P~j`cGpPEV zeJbt$kbzV(zA6(*G3fy=fF~jn*y!tOvKSh-2dzQ>AveU2oIZV%lxsq_@}IQ;yoXTu zt@xEx7&;jPPb6Xd3JS#s&-#$y^g9d<8=oObSGI!8ii*jDdI}B;yMd#A0e5kmnU3xS z^a^iXV(?g``M%rBlK~0h9>}X(Qe!k!@@^2#dvD`AqdmcqZN~|_NN}?kwO8@|c$t*b z^Qc5=6^O8IeY?JE!xDq@7JELWwe9@WkyLjncW?1v=`U9 zyDZc7xU$al*0LH0RSV}iKKFdvHMhmzIOcU4NyhxN%aMtmXy)=-F*r@j+Ii%#=&yPe z|M1nmFSuqZG~~7v>s(toKg{>U57W#F5)OtG?pUj}o-#b*LNn4b&s+Y59aSa6GMlEo z8>Hr2ec!nj%CEEDdN0H!pC!$T=fV3=lQXM`$@cKyQqG*rIW;oEzSduSx?}r~-_;lH zH;2>H2-l{R)ha!B`G#}2|LNqA-R5M5#zxD4qu70Xff6`0efeETPo>{zok#o<8dd+1 z{iddkVu~JbZXY>QFxDWZXg2=UB1OOC6*p#5(4EiW@u-Ww;<(=KQY%N_TzybQPn}2f zlE+y&>UugltZ2IqFs^K9E;_k#_chp~!k>{tkK)~vUs=viS1v!IdoiQpVpBzr()=kp zX6Bi?7VHhy$?Rq{G0~=>5!X*CdoX9uE%7Y8;d|3yRo?Rw&VT?h8mr-13DJ~S2PJs& zDzFI#rT9kYkv%gH(mUrIR-2-hCAXNzSIAv~Sm0MKH!Op&c+TFl}Mm6}V66IR-31L>v z^9P52d_J&upB82Xcp8l3~m{*dlaHu^ zNfRXcK7YB4e;w679V>1V_n>jx{3(G;8&yhfoek%7_n4F_sD9|~e`Q&c97Zr*aK``a z?&7zEZ5xbQ?ZP|bm1VwlR9PToL(eeJ%{$MF8I`6@HtX*i6LTAmyPVnWox~--<7Z&* zMRO_Rd5=#V9Y+g9MyGYhS>@ZFx62RxY7xAcQgqt$Lu;D-G`D6Bt=ci&bu8c8FX(0V zcuI}e*VFlbceCxSI_f!6*2$^-<&)#R?+txBqIM1cX=<`Dw2qe1pL?m@Hk17trkOjl z?oKje#oM4gZBHqVmEY_x!eyk%@U9vR*xwPpoLOn+Yph5PW7w=4gKAw+?$<*ewrkEc znaig>FS>o(ujcFxrir3K%QXR}3KI<=U)WYLd_Fth9}!vIab@Z{i_(O4WlhcDD3Q{y zHI@$|HXbw@typ_0MmtyRt&MT_(9;WI;gY2$H4hX{-4xUPG++GPJ^&Iug^TyYxx9jQ z6|P0oqmtR)8`DO8wKJc;#%F!S8slR=6M21cLE=Vr(I16BU(I0fA^vgrPPsQ-&EAa0 zP>U*l%r7op<1zh8J>ePTy+gsbZ<{nJwAE{7HOyip!Jj{6ou|mK^#!GU)@(_oV^^u} z7r^+C8ld%|91CP9zqgd=Ti?xA{CI4b4HBR-+#qy95BdwiTwYABLnkCLmmOaR2DX%; z)FaJ5f~13*^g(77brAK~Y$u@=w<{>D10o5*S%Xtcv{r^<-)%Z?pDrI8kcaiH~EWbV~ zDLMHSq!V65sR67KGMO4Hd3oMtd5(sqil1r3dPo*}c zN2F!BZAT3F85jqS>R#NfMp-U&Xhmt_QBSMOk$~!wk~ZesJ?56zW7ihhsI06Y>0mBw zWPGJ4WvI7TdDW_hn6z5Ti``Bd(d({D8803_U*uXsUm0|Ezq`BGo9alzOtq1zCZlN^ zC5>>F#S#)ttsRrEZ*8o4o1&twzCl5;kxRaT8L6>ya>=;TNptg`+wBwSyuBR~)2;XD zgfBO(1B>|da8X_TfaZ3$e7cAT(h;=B;Ni~DP-W{GMN#pzN^i#1`Hra^GBULJ`NE%v z)63t*JX1TsFc6wPe&YjzWR#wFXs9Y0@4tW80JVLpn1ZR~LA?`ogIi_u_@?YPn>z%& zlY9DN+l!T~8E@|G^Q1a_#-5*fnkQ~e`-vZa zsF2P9Soi8ul})t#^88g@Iw+0{s~d%@llSet!z)xm&wco$^Y06+J4{*ryeuwEN&9_M zsQ2-mn6wmnFDa+z?ttf4|Ef`M zBD-I1J;!F1N6J(uOH z7k#PPd9$;ZUahshzq(s#-X*s^90FvtTmY1L`@mYowU9yRR(E9IgVHC*13hSKqqcaK zzn{G3{QJ9P{$rX zuCIE{)>dzuHK1C`??6zM0MzKg;y)6%PT4QbLT2kokufa}Q4ka?%q-xRVW8?JR2r60 zi83^{ii-yh*OS~tUG84br#T)4^f|2Hk=KrnRXc&<)mO{k=LY&>kOx$lK(z7tHVM(0 zA$o}&$JHSEBdC;9bpaXO6!Z9TIXBkXplv?6`fsit_;0Qb)!N@&-P#n&C^U&-;YRsT zCU_nqRJy;Qqzn~QpB}50pf*xMI+tPp{k{IgPJC5OeRFBK-|ED~^A!*GSW$dOOH)p< z9?ARA)N>-wV={&5uK4lTloa~VrAwC=8KVaEU*|G2o9lXP-#+%xTZ~7wwVBlyzkH=A zb>SEtZKt92!^q>H8ot#Voq67IwcmV++Z7qg&9Rq1T0%xf@@MCC<%-}M6AlkRr_0RA zxlBNySkml}AnnS+Y<-ocw8qa3Nyom~YkrJAZENcuE-#bvfp=Z4c{+wzOk_XVVy#^I z%T8KbCa&Nk&7F#6z`j@&$~a5+&OX$h{84A|{fqhSGil0tr`Q`#um1D$^kpH3k3I3} zSvrEf4=xT)HRwF{`B=SrPv$Plp%pj$`X`n4<`k;-CftpAmbFIAm0iItXM}l5`Dgw^ z#~F_HC-JA3q(yDs7m&s(U{QkD+_51{bK0t z)sBgcUE^A^<{AF=4OfHy21)#1NBux!(^}ilyBCd(4;V;>v+xdidwIRs+23^`Ii{4d zVgJn|+6`B4-ZXV~Nn+*gaMhR`|KQj%&~}rN^NsDC+m4%lYmu@jTcj|5tqr(dR0KKu zIq7>LVJB|*`!i@C{~ps6iNAkI?nl7H9wQ@`V(XX$(EfPUubdr!OKlRYw=~Cx{MEre5+*Tq9kR|&NnuSdwp;wbE!6iOx92OVo{D8%IZdWUT`$8A z<;h*1R9z?EP{{HlE-3Kj%&^2B<2Y7{VwqDCag8UnMk>~t9XVhMrn0tL+- znXpHFG&I)Fqi)R4JlAqepZFFRha^BJp|vEU{=FlAX6!?z;VT{zPj2=vq7gZ{k$1XA z!~Yb^rGu^Zz1epIKmQ0!#uRG$Qs=Kq2MsK&N{8g?KL5CT(!e5XzSz5bxk1ag7E2UM z!}rN~n>w?^%@>x^yuB>6X#Dv$CftII@w$Xyhes@5Is7Eur}YeN9M~m3aYt%y|6@pd zO!iz%<|o@%XXoi8-ntlxR!FRMJ$Os|&G^O49_FO{P*sV$oT0I|3cy| z`%K3!pV|3C%W0->xPJZ>mCR!`{&upu`Olct#XmFIL0B3hXVx|1!SI{AvpahvRUh3rakr=Gr%Lo04x&{aDXdNrPV zN%0b%!!|`363j>~XiiU6`*}#t1?tb379@P?(Bizqw!zfN{l_@mKv@;dD$}R3>P3Lx zx44HaeawFA+X1^L8B^=~f-l83TXim`IdBILN+6@RlAV=n&^hyaq(`k~dBq%gQ|G1- ze}6mdmR6_PeI$6NzDJ1}2fy=h$qF4S`7qQU;Y^izPs20+Kd#;boa#S*|34a3L{Uga zMU+G;*(9N1CfSZvWUu4cDj8*kin5Zj_ueO)GPCEw5n0Ec$2sTsJbk{u-~W64yRNRQ zPx)}l`@F{eyzj^DY@vT+1IDl?xVVf(I{*Lx6-ki1qaGc75y9Oq=39dzJM8bE-hU9e zaH)l4fhj1^4GN^+EIf;A$upTM%ix!;tBXsNT2JQU$g9~B^8GZuu`ryGWg|&Cbhunn zM7W_@$q$a2Aok(WnHlU{^{`~WhQj1{>suQ?4m#db1t6dy(!K#YD9q*_jaS{fyA~tm zejP5LFX13=P2sD6r7S8xotA!R2Iw02sxALlWX8ACWe4tq1v<5p&IZ8W)y?&6NO&lzs*>pVx15QpTwgM#<~8UG*stcd;SlA776C7 ze+G$>v2DuPzIWsd9W|xv^OBGd%Nk_lDXGH889B%j40yqS(81^g`>}$tv2ip83}_%r zABXGFzX;;`)>actkiqHbQ3!*jf1l*))nZQ@z);6rfBKeA>)*%IL-at&qhfHj7VLV^ z1U&a~y}HyIPE^2YJ`KW>Y9kqNk!so?t#jv5zuXRs}u_SF`X-=kANOHD7b>+?y`_p`_{W5x@n*68TrXwPDFyvT&IY0Jo zN5=-!2*_xjaU*VOw*1U+65~Dh<2ZC7iS6&tMkV#R?w4c6#)L8MI4*mmmlL%$OItZZ`)m2Go+rk3K1(=zneoD0nW~t6i;2@mPr%ESoj2Dh z$D;N4Q1jco?ZjDiuk_!OxEwrxNTuhI8DBKJ6j374Y&s32Wme3IB+c<$-}+N)UbKAl zR21v$r(~g{N%chg^sSu!af^0ct;WqkQs83)84bGNc9{orUw^=qPG@iZa&-k?bTX$k zD4aB;sId2)1zpo$&}O&ai{f`%c|v0P-0d&Ro6%mUpTTyRjh4)V{|9}a@(b58rjjn zCi`)|?C2|;yXyuGsz;t@|GJO_H;A*M&rC*2GR52r4qc_+GatJK=Q$%U;iJEz8LGca zSXm$M=S(Q6=Ats5YA~jZrQ1F+`GcNt&7y;M!R)_5cXM^Q-Nyu@04OeC-CAt*>$b%D zseXSng|9SQ^(Bka)>rOWR92hq)Ohdi>~=Ctj+S4qoOScaSdCxinHaz%P>YJ9Qgbz6 zIkNZlJg{67Yce3Z@a)0&18?5Q?R20SS2lXg;&X~`#1nHi2F)s0YiU@P642}0zdtT* zZ~f#bsvV>FS7EK|c zJD|dd7q@dr=9vENLa&DVeoJbqJ)CnEZb$WB6=GL=5_z#sV}}o;AQ3AK7+`}+6tYbH zIUY^#-YxNSRhh?f^{pSvY@<*YoI9WLHNY>oGc>hOg;xn_ojpJ!K{Rb_p(}0$d zy0e+9_9ds|uR|qFMIm9r4O;HsqJW`7*+yAX#52+7bsuYu!NdC$@OcdRp6DDVWPFQx zf$^W8LMvm-_=Sz)xTpIw2b{aVAKPDeW0=No6(9F)^)W8Ll-aoLRg9oJhyLbim7Qj8 z$%ul>v`xItiKPp=A9e{{14RdH>Pf!e zlU8Fp-*9UwY!0q1n7M6BdVaBY*dNOOfGRrT>+K*Zm}(H19A4TeaDH9={J4i3O-_8@ z5Q9VHka?|=tmF$JpJ-E}jO%nm>Zd`H-m#o-AHOKr`%Z2?`{Q0rIbmdxhp1R%M=*oq z39r_lx>+x|8axnu(P&QQjZtC#=JnBky9UFO^08kJ6p0-|g%G@8ff%~5g)>9Gv1zkE!NN)Jbb>s!SPezv5Kse5!8AEt^Xsc>{RPYGd1r7Goo)>nemRl zJ4%YvUn0(mS8aCDn{x` z)h;c#V#s7d(72F{T=RT%0SK=j3fwLE$O!599IM_^SueDQy1XwX$7{y7-)of z_p7~K0n)=#hmRYBm+KC+VtD-W?PP*|oB)2{{)G`)i#Y+vP-!G99xVq01Sh~X(y)cn zVFl;mFgD`ytK>4o6gD|ag7x=ycjc1H*BhCFDk8(f%YxwMJ6EsSW zc0R%YEdtW@7ych3?JqBMik4c3Ex;K!s931KX z{o7IYbAs!P@@2(P53bl-PwUMKmUm%1>g04jGE$~#rhTZi$~&s{RcgYppKhT{BV>86 z#d749rw_!gOa?{+)y~%uYywv?9ebP6Ak^`J6xcF}V-9}3jl~4P|DjMGC5>{aP*jfJ z6Hah5Whf}f80OH{9ojqd`Pw8m+|ax5VoGsK-cN02y%&@X-I++ce^qVv`cA$t>PcM; zfa5YnL!aGtu}4OmB^uQEVoF&CM%>jxRe0HV|6U-GTB0p2Q!TXWrgSW&di!@aZq0hs zjXBJxa5{A#m4i9E=fh6S=^19)e)1y1wJPE5u$wqB?}=J_x1(cq5n&T2&VkVr$ia2QzWDBb?D8oS*p*~NkNu{dhV_gbf!wVlytGt-Q+x8Vl!O|8{3 zw&EnTAjVwt1C6O8l`VvWAOj_1E`hM0PHa5qU>4Y+^Cd3m@?j*p$)shC z5H9jb*Tu49$=$2VJRm5Klp-5rgt=f6AP{I)@jKQ&p%ykKM|>PEF*hK98ESb_h27Vh z<2mbt-_8sB><3a#UTIl2c!w-_LKk&8WOgQ|YzIIYgC7T@oH;(jX&AIE@!-P{!Vz*@ zHC;IT3vSkVFrYtha7;sQI7kRn_4fVyNc*M-JGBg3+;D(nkYUB*Z^`cRFojDMJdDw~ z_VmFeX2B3e^KYVhdTr3Vb^xB>q#f;;>laTYC2^>|cBlA)SL7XQha7RQ z@+>vzE^t?W=}uFB(*L@s=sLTYI;a_8Lgu53Xw1V}m{rmv-oYHkbD1OLucbXJ-Lr4s z-cR(X6=t#I-m?7(Q^$xMJenzf*wxI8_DRlYfc1dY42dzcqzbTT&#d zBw8=uMu+IK`u(uvg8kN@GvjU($%~auU+C6sf?#81r%$Lort%O!im>ci>+j^XFWY+U z%&?#xb<>V<@2KTYpl4lI@2juDrzsA$_F}kKd0kg*e`AOC)U}NE8{D2^b zKm!X#t@a=$VbaBL^FzB1+2A^xHJ!tDpYcYOINXMF8%b?Iw7KDecKte^NC zZ(D6Ion3AeH!(Z&XTEoRw5ri@z?S!{rIf)W*W}?ZC{tCH`^lTa@DG({v&Aqj$EiB7 z7}RJ}`{#X0>{9&{5gd3sMVi6mulF}fXy^U-0>s$Sm7fXgR1~9u>iL!h4}IF>sZMK$JkLNRAo=4JtgIH6uhOzKO(X>>x`@?eE{;TpS5-+^ zdy$FtlS^Z(rnI7A`Jy}*{fi}C|MBiVX7oNRKZHk4+5! za3v6>1AzcBhcC7#N8s^J>XSgT71xXhE&G(BE)?9M+FwA@CgCT_x;|CnkJYEBWG5C@ zNN}qr3N|*V`aDyO=a(a@5I({h!ycBI!^~&AyDN@n`nptBR8(_=Dl}*>6^4mZl*~!4 zeP#ItkR70wgZ0$g4zgWx&Pm1#oQV(#hU81x5h*ybu;d$xDdwU{O&`N}czLH5he~3_ zK`u~-U=f511&1JBqhW9E$-b1M;FnG@p5uVM)nAsEcj8&$ClzPF1DuanH8|3+wJ)q; zKk>h2oV`5-g^h=|0nO-U7*Dw}K$jp7Px_LKC>)KzvieJhg6{T|0f6gqnw*t*}q(CG58lKnWeU8t2HEu@vYU=QVBp?`9JpJ z*|S&va|D!j0J;tImNHir8>=~Rph6=jYGgCv|A$5%Tl1hSjrgnJb7~+n+G!WO+L)cCu7#Kv39h(PQyOuQuct`xg*8PfblFaFmIWHE7dJC_TH0w@?+&Tdx088P|8v4U8UeW$(QG;=t*FvuIm7F+% z+FG*F3-gExd}i(7;rt_<^R%rOl6X?tzzQfwR>@@;`@mMUU7foczl{&jetZ^)48r z6o&C7iC0*P8CP#yI2pxyC^V1_q~&+{vJ8K)8@b&BXIIC!F-=iqgV%pwf1xk+mWP2X z*S?!9Ws*CeQ`e%a5yUETgbgiXhl5}T%063 zb!&sJ$JVzYy#U9)yTLUmycsO6x1@YesMS=b_4xQvR34HAjq!E8EO8i9Po*fGb4%kI ztuioy>8yeg$J%d~OVtM6XP4hr7b1;nKO(sbxBO^BR-16X`bRL=@|(?r+kM1t_0yoo zP6vQ*@;n@yz+6W3I7rVWSoiXvt?U2{1;W0ZLqlC$5&c41dbinlRj0aoSIP2GI+yzS zK1Ws&egx!4U~0d!SjY}Xq$xhx`%S?@s^#BvH!HDY!KGNc*ub<$mnmk-6ct)XB}l@V zLd-Rb6=Ftm4}%XYCZA!DE@HB?vl1|rnEq(=OP8?CG6BaVn8Z_F5X?tRHvY=x>F|6@ zjm8yr7HDYpc-RSYCm#rAwgvk?gZQEu;Jo{@hF)ILw#ouGy|-hhw4bm|3>TOWEUm65 z6&0PF?TiNN(Vc}7&qoeC#ONjwoW+v~p45&rRO#j)6M@Vk4F zcAa1kKHBL{+HHj~yfk3DZXggA9e-h|vkPR2gLP+cWV+Ji$M86&c%&|s>ycIPT01mxX3Z#sR?X77gY|>+f;`zkg?}uRI*tl z4t}j#1*6pu3@z_g5g+8{A{Ac2Y9)e+f1}~tm>4Clc5z}fCv$LZO&C62)_26{VdJ}2 z6Hwkt3j1!&)>Ius1gRdW^TQ+_#-3R9yiRzN-%oxR`TA-uy8=;^$r)ihwPr%mG+mjF zF#Fg`$+(lwtzkY{YQEQa=}ndjY?O`rn@8^)OtEy*@%rfwHf1$84;sG60T1HdY@`M( zWqhSGE_)&pPG0i7@A(gZBxBNJikx&(6B+y;y^JX27G6q<2s*>{8?UemaVW~+_mUWr z{>vsYKF{S+fbua^MvkVwzOSlM+uMeD?UefVXODijDJkNRYWO>pquPQcs5u)CL=Ko$ z<>Nco7kS0oJiO>+_U5PzWE?pV4wuUj|At7)qkLP6R-uD8yUnlFEA;J>@J)9E&i=If ze2N+YvnR6+(dM}S@>nIU5!OcWFKe0N2|nW1+p!kvx`VgmjAtskm%^@gWE81dAzr*Q z$rf!n3u(d<0{`!A1NowZRY-V@e7k_V|Grlwow7{c%9+8vkKS@n5*y06~!W8S7{^9nBi z%7kMN$$EG=BBrEdGiJ~$ogG8!S6wipVaSndyQ;j{+L_xJEW~*j0TH8>aNjn7f1&5c z&_YL6HPu3GEzD!v+8&(el;|xT%V!IvhK_EiwBh&izIf>rBTqUCiJE)#y?*x4p)!-D zVK=Xh?(guM4F zB1d18ks^{57iTlw@!9L~8?4;+u^j$1oeHW3j*Cwp2TKUWX&Pu|%ZIut8p|G5_*7Ex zM&YqY9`Q}j7Vx$-u6J@Z^<@^PWsCgyH{cSbc&T>RlR;K-T=KC6&bCHoA4sLtP!0C9 zox*|<6=jhRaOu1&R{$X7gQ1GtqgBw%RMvHga-8h8YFz)~QC$-sLGh3sfZhVQJQF0v zpw0z)N(>9UVHq25*%GtzjQc)*SwgQWVD$IzR}dc*j8VMXOA9vW^Yrv~N|apc182r`!?Ov??XuZTk=O2=Gn8^3ress!0Ol zaDHLIe-sR%ZlsTU)sQDX52%ib*#xk5!rBaRr24ly?Y)OmAbSkAq6#!{8xVEE%Az(_hE4 zf&PN@B^JN?UxTbnS{6YHOSlfpgqVO#3=kh#NhH$$6~#cvvl+~z`WF%zuX3-RCk-Vl z1+UD+T4Xs|rqR^wm_la-v#0<5ytd|QE8i!#{ncE!RI`jMopvrPC{WBYBwYObS&u;N z=3WkVCEXwq>qQKS;ym`*NInSDt$rg$5EXFao;0AbY1Ea0m-ql8sZ-0XEL8-{QA1p+ zPRp>;XIUJwU|OA(?JQZ)UYXDA0jXV;q7cXu=Zbnp$V>K`u$bnE7*)6&ArY7TkL1tk zZ@`-Bu&5(U$s+yv(@t4MBa~cmi$f^YO;w8E4r=U5VWLU{Y%`^v9WYpZeVA6JzT7#R z7)lWo9tx>6wnX@`+jW9!Yd0o1?bCBZA<1$nehY?nT~>{@nEkbzRYX^|8Zwc1E%qX) z{CpGAaCZx?)i!REC&2dAKiV=Bp|eGfoM4(Ri`ItwxT;{7g z7Cq9b1w*Qu&1P}2`8!j6Ok&kb*AZVT%BU<{4QhL7JUShJoJp0z(6wHum93*)AW042 zy$x2zRtGnmF0*61JmNKBKs1Yy4of8$mT6-0|nO7lDL`y<|YR-Cdx z8`L)ZSt8T*y}vopgXDbKI=JWjMIBL?$Q~@N7UHkBup_4aQQwbyLFmrh-NgJYb`SD_ zttu4tZ_&!ZmTte5zsTA8`T^27GFkVPUUq|7cxYW=C$H%vn(C?FJ-?uyIl84I=Kz9o zDe2EX`+NYq_KdtP8|bj4r=Occz@H5ulhDFmHd|4-H4+CqrPf7m*|pVW=B7vxi+xt) zsA1ZgbrQZ*a!QTjVhOrc0$Q7}SDr&AIwGB-w)v z=6~s9*d=9N+O5U%0JG*CNw2EP{Yi$xl|b1x;4@z}H6xp~`|huf#Yo34M8Y(IlGCb+ z{U@;>$!6)NnyB7pHv99SndKE6n|Mg8;k*5184tp-$?Y1~*k1)1uv`D<_Trsn`u-sJ75bgr#oz+?hEu85u`}XZ5 zSP)fh^&?*l>ToR%Im_17S1)pIZ}v^aqlbBi^#vG7`?EJ7Z(gg7%)S%ZuHb-xYp&Zu zw)+u!yUMa6N3$n^sOcG*P>57qc;&~TW?7a$W|7DdD2k1{NBGcwl^SodYS9Z5H#%E*oOPApMKr8%uU%vZH(O#09- zP|Vo(m)!%+IF1Hw)Zu)Q2<*Ep3?Jg1cNVMmqla0mY2q4X%f8F6GGmGUMxa1R zZOwQcy6dj*tkjRZYv>*|rh^bGUEp`ow$56GBam4~lo^K+f-Y#Ii3vU#gKYLDv7FeN0)xaxj+OHftG^tRuCK$M!}aKS=3I=C`1-?o@;cg?P|QEV^!&CKrJ zZElV$D-%P!2&SF*)3;?bR>h(3OIo*~;~cG6?pjcZktS&F $;=@B`G6Ew{DQHmjo zP`0-IQCJj82$f0Mlxz@8oT)5{9PE|B=7ujc9C~!*(m&HacF>}@Y5RLboqZ@G?9QNM!fgXHx42HNKU=lLJ{#4Z9XLX~JC3Q)8PsD&^hG@)agF59)x z1}tfT=NKq_l5w3B@F)PK|HGe9;4(l6@N~yY>O$lOV2=<8&vvz@gYo7B)Iq<%U@*#k zl!dRg^FOwn>-ruOBjYu2`+mxBT5MzqpK4+pvMOe*bj_}tZUtt8%tqG{Yc+vjCL%ux zOO`ZUVt}%rR}fK$43Ar&+1Oa$PNy%kN#on3{?S|?R9pJMoN8<9;lavMTY4zUtzX^7 zbOtJ#VYx0{YB@Mkl6~8&3e$s{o#~%|9JEPjv>?YTp&Ff+?_BqoRbrXL3@e|=eYaZj zbe?;3u|F4`!x9cNAp({w5N?nq^?L^ojIOluZ6B$AA+!69IIayBUI4Tn?%#pS@uVo` z6Mc5S&j?}#EE-d4zMpMFv7SN)wR0XzYDrEQEMLK0^)XWQPVjpb>)~)wAgr<8H;y*( z%N9Y=eXr64uX!}4q~B4kFz1{u&+A44PTx1~AexJiaNmf^&TU#~{q^|0K-ret^2OtJ zHt_Cz9Zr|m1Y7^;+(K-~Gf?BLLz!3@TnCvlGr=woX1j1AN+v-3?=NZu*zT_3^pmq3 zyBm)^J|%`9FuSFjH=BKmuCPtpZI^-b@Q)Brtx#Lf=Z9qe0XqYGK3$%&icDDh*fHUPRu?~#I#2Nk zp0Q~+l+2W$F`3=lZT+K&DOSL8g{L!!$;cdkc&uJNw==&9#+)@kXK*bQchwL`*zII& z=nQ%tTtR%YnBb!)weRbEBA(T}J$|PMN8HU3RZT80S>hapuWd3}h)c4wz|aSEm+&Y!zM&_V?+`+LpIU=WpgLp=8?hvHTt4A^mwW^`=zS3#~GDBPw#BMIm^iS z3T*RMJKjAk?KU5?i0|298K|vXHp*+F2vMPDm;J+&iQl5HP!0f)9Fg!=&dIznp(OjD zq=||55VyFyN~LtMV#tvnBw}WEE)r&JqF5Vpp(7utC}7exS?K?>vW+hEloA^}I8vH> z*UM{Uw`>3@>+eJ(eT;$KQqq-=mwFP`%V86)+aU9L;?KOo673W#%_w8Wpdj|=kb;%JCQt6+oN*u^Zdf$W|R9SZhP3^ z$)Q$@%uKpZr{3lVn@Y0Zc>c%XZirpw%k9yQ%xh=jt?MeMm6(zowb2rSPc=t)4i)zw zewdi}d-4z~7_sm~2+7$9-v>I%$mC!TvPk97>c)Vn5g%4ER}LSu-L~ZGE05b3cGrsQ z!S&JeUM^Nzn0q^iU1s;QsmJhktJ)@3LssSx;?k)Xfr-rE!T;j&oUj`u zz)0~5kbP#Zuznu4kMv&m95%4N?fC3zDiRr73-hn*=PfK<_SsTv%ILi17tD@*KalBr zBe^G$3ub?l7Bdk0%Zs^6-^|958o$=9#FB5J#)&WnrP2*w{-zX@uV3UtyvRytn^?=2 z$Uav4_ZA9e%VbkkWsuDzIKsp-P=c)79WUSWNcQW_ratb}9Kmg!@8%eST6FH@xO1;X z_T;GzbVI}bSs5AM#x#v(9F{Bt$%Xy>BDeMS@|_XvOmXE{?Ei!V7aTX>%GWcuY|&9c zCW!%r?0^YY*+$ayNnqvj_%ZwC(i3NPh3+bnXvfEW8>_c{!iwyDIQdlRn!1y%g`J<7 zSB}JAlZ*RSWRLWL^A2I_^xz*ml8kH9$jE684WlE+oPZ$h7SacEk}wPXMQOtQ-V`>FMc~$K5~7N6er6zZMFdCDoacCdX3faFY|GvAW3?@o8>Ug_2|k zWTlW%>ckye??Bu2tN(4k81Tndn%UmFcTY0;%``wJ7JV49{)e2;85w`I!5X+}zMW>8F6& zHjqRBV5rmrgt<{)GO?=W=8u25SD@1Ye{c8<1%1+6$u8}2icdA}gcHDPDE`{II(AXh zMSj4%1wbe!F9t#}A!lqTfSS8E)uG6NzPx;D4+mKv)>}3)lo^*9FfptL1Q)RwEIuEs z$5Lx8+$$@VzM*y$C4;02?CQb%gHZNxM+Tg}uzTC*enx+e()6zn49;4;(HWv(YD(~q zKuomqH5>o_)-fH7T_6s|cdN~^9Zntu`zlB3XN4Q>&Xm>#AVVzf+_>&&dL+TaDnx`dE87$2aTTZx zMxNQYb@J*eRgCzJIyQ7KVP4Hw!j|a!=+UJFPrFBzX7)Ri49#W@M|=@4c&K1dFy4AY zPnA2q%P(_m#dN!+Ic}?sj5~v@3JUF%!{Rhy zcrVpJwLI*z{T{xE4|iMUv-L=SRD97_Lg!IsgIjbOW|HIN!L3rO+}WP*euvYp*cUXd zC#b$a#%KqQB#`6p?z}D{UX87|I5KC%^D80oz=sN=yEVTy3&p_C#$xfL&ZfdIMfUoq z-M_35g~L0vNz29_(k?COTuRTb#?Z(chlr;5T0agn3%KZS()NngI?nyu32+(GH;+HC z7lFAFZH;6H8?rOwj`!2T`h2j)mbrbgHLM4gWQ>z-5HuP>fgz*p3Qjn#Lp# zffAs-!#Q$2=DYrUFe?OHxc0v+Ne@;N^eWAOKk%v;%ZRAjoiRw0+!=Cq)R@XHm5{N+ zF2NCqhslkV|lG!}?9wRxPcNa!C_;x6?glWNs6cI_~9c7r03gq5d6lC=uNK zm9JxEzkio>;#VfC0{?L$gDTXTIsO9+QojR-YL+H0!KcF3;OWyFVO$tNDN-w+ZPYc$ zJGrg33Oh8tznl~lzFpenx$Qcx{HsSZ6kr_zakH^kWPn^USYtdOOS9#Lr6(Z)L|+Wvq7!Bb1GXZsnvm zz1ZI@bc&87RiOncln)NJVMmv!H7@epgE|~p_-jA4U?Nrlj8_;)VK^u z3}j1arNzC}5&FvKTzU!G3y-Cp&H^JP7Nc@GwxH{)C~7BeVdz6yq|=z7y`|0E{8}7y zJnL&dQvQkinUaa;x()14O@T4%Fg)6jlbP|BJV{>8q*?hzWSC+>=GNxYXEm;|L{7{k z=Z6XXO#1<{&^Uf8Q0+9IEGOw&a?fXqhBlreC?AE}zTu4?x!XF4_UESP>?1Vilpk2Z zB@zG<^^UxJe0PTUh3xCpe4I`oK<^>%4ZNb~vvw&)V{K!9q(8Z`+QQADwpI>_fFCghQPiosCSpTjZPyB>#Hu9gStohD=eQhxLmZ+c&6Yrh1ZGYt)C z;BfKPLcEFMop;#yt5=Cd+qB1=q=GZeQ9+7h>FHArHJvJ-YASeuS_P3vEY!Krr8j{l z-vrpD+t26F8u|qJ=~>^i5@UsFnS}G8<#26}-q3OD2;?%VgWYj44;r@gfy=wH&EgxV ztg0HoGd7<8!7ko5O85V@BZS-5+aO$zW9_2>LO{*HOcn&v-$JAP)?0omk^&B)`ppfp)wRyuGv7+eBwQcip~WlP`! zF8dI1>;|RT>+Sh@J}c`+!bdmG5U8DkZE%9p@L~sMOxBxeI8Ti?dwxLUpB!g_)*?7u z(kB41pw}A^OB8Hap{{|ZMdRDk`y=QV);2bGRl^3mc+V%TH$8^0bGNz{s4$KbKd8>9 zs$T+YmsLYUgUf<7aMJ%{T|@cm%G!L{{r`%<)ryOY2VvX$*G2+bJcYr0Hn}@UO_q2n zRV@*iq-)cyl*aMi`2_tNa1#wm6P0jXj2Uy+=HlW4bUB#dGU6>-ahbVmuLp<5wK_sZ zs>IJKnl0?pe}D8cao_xv=13U$683jB`UgMN=#0%}<__b1jjF_`O-vFvLhOpoTa8nj zsD1<-*X<|EuC-i(>ngr?ozpWbJGXzRP~x$c2zoD$hBL@7Fwi;`2MOk4|Be~CCa^Vda=ZN2gocY*IiY}Rru>BL)XgEB$g|0 z?bej|xG+b(UNmboT-yW4lYfmpI3L)~&F*bgu5?$JN!`0>-r1u@g@XwN!XpS`(hmDT* zIIm8zhPXIk(d)vxsn(2P;f7CPTL$rGSDO9d$A$~n3zqcATT9wYO{1d+=Qrkl3obZ9 zD+2s5U)ZXM`%eg!2lthjh~4Ux&UqPB#^a!H_R9q%-`ZD}X?=pK17M#VIULp78>!l? z7IR{;cdNd>vGt^g$X4jX%#6>DbB{)*{lIo5E_wai~XH*X>ny~ ztn8z==H#=VKxajV5O3h?B59?_7sGn9R!)kRmLmgt28;U&^I*lQOdhbcxoJB0yA!%V zE?V1la3|-%2w~zYen^%jPz8mzKI5ZZcA^wLx%KR1GPO7=zg(=4apq$QwX0)Koh0j+ zBJk1e^hOlwe*fyT?2c`a5Yn1RDW1FrEJhX$s@rhg?kYp1HVr#Hi)@H5b2mML==~rf zzFE~G++jGeNoQ$YvB)w5q5o2PMPE(L+as*rk}O4@u`lk~!)2wYiIN=Hi^b)|aENMV zCVD3ajBx}%SddXKQZPVq$@2dpH!ovg|Itd(7H%q$yt zjK$e$5n%N7B9<1q?lM;s3D3dhabNX=NLVa>|3Wxu6f4JVV{bo+nRNSIXWIsePKqg1 zeYf3{+;-z=*@EEgmWkoybStAlSyzc{&Gz{DBlKH;C(hVzh%N~k8padi=Mcs+X7Q-=0QJ=t)Km*@<}+v6?kigH=#B^(iFBdN`+5c}7u|zyQWd1--tI%`E6HBiFU=0XQJJrBcPAdhLjyiv% za~!q&Kzy>Qvht}Z;J)Qw7KIH#A2tMNN}y~AHZ0!IS8?g;Rg_c=OfO9;k@WQR_>a35 zcMWSx96#G?0z98I4ZS6InC`=ZZc;CS`~um)puKR4=toaWD-ZbqxWBZVDVb%2>FOUS z@=D_ZSc&WLK`CkJB|O8q`P{X&H42F&?)V=-EHXop0FXaH(qmkM3aWkr;rT=bfd2$O zmY~$3f>1G(t*`b5e2E~BKxpn*94+q=#CYwlaqdcQ15W3io!&JHVIA7P#z+;E_jE5| zRE);DQgGz=_lNEnNh$<1mBJX4oLEhux1WEoPD9d)t{lB0_<}ZN(52@94^Qv=^SAHZ z*;D4dQHUA4JoKr3V`wP8pvMBuTX~~ZS>oEu6GtxI{?Xf2OKoFwu<2HbV=0g5P{B7W zm#R5etHP~<92;NR&ieJ8Dv9lRg^&0T%i`Nr_FvP|TITYBWk#tqcF^;s`~@ttfR^+F zc$cp!m!_uBvy-#4dWMJ8=5>GZU%KZ9_zp`WASazweJZMZC*xf=4((zxaNNWzEj8dR@2SLP(rCK}Ph($EpmNfCZr4a_)K`!G zV^#~vGd)XJ#!#+PsS3BxesO({m0q|mj!zZBr36$KE&n)zOtQ7&)LocTx=6!)t_?jA zc4DDKS0eikk;i^o$0A4Q{*B}>kI~5OOPyS0obgvqJrMUojSY5IrJW{_p_H@^2pri2mC*?Fpk5$v+9(RbC z;kQ*C^QppRYt90b&#v`fIb2X0-}m05|9nyIevY^UUwX=Kos<+5L`cLp$*3dM&(RnC znQ>(>R!vwOMl8%;;I5=0{QMP9W5%waP=2em9*blTv3)Kqxe4DeD(U7_i*~kzmpMt( zqxv=ILVH;ZUZMQXog(%0Q+~rU>v46|JDbcb3;8JP^QwtCcYS=M=ND8gt>$lASYVS; z-YST>_J%hS(FBQ%Y}@+Ci~0Zs7(k>dQ}3@uZ{MyoJ)kGynV5FX;MLdt38pWO+4kSK zim0q&WMYHV^{+da#B@2nJh(;mqT+gTSD5Ni$LEsZ5~435+sqYxO)5AOfkSQSB-c+^ z-3@Q1W)zp3{oQaxDTjpNKz+ViMtXknSQxe`ej(H{%jS;fL%z0@4xiE+o5JD^8n@HJ z9};_I#-auGVkA$B7|9!Y zks0_v!!esvsHTLmLME*BLGijUdQvk=*hCc@HFKe9A#X3XaFxWMHfY8tij?0kI+9w$ zAIiv3BgGPn6t@cZ-YM)UL*=tM4}3q*D{{!`JKg3}m6jni6Xl~VDi5>Co*wp$v%l%n zk!!ITs;F~ythuG-XoZY3KGZjjVFRs)Km^Q>^Cb*k+c8^7kQU8<`jF~Hj`O6DPrhXr zq3mof#z(=@cQshkaQsQ;(rWoq-bPO$WS7~Xh_E~P8GX@kP3NDB7pcJ)E9D}b+Y^rHb%KeyISJ{1h_ugG<`m~)L=X*KWWvZxZYpb~e zv!0x-*=am@Ew-{_Np`PWis#~-!Jco~yuEMbD*!qoepxJ659;_CNVp-Mx>ceY8J8qYG)Ng{`BP0~w;>olv?8VdBXe!qiFT z8NZ5(TR_%*OWq0ZDf6j5i8yiMi=t*Y&gV846Zt0F#hYvcf^xEi#Z!;9ODC|Tr84QQ zp2UPYC9Z7LbI%JG5D=<9MhU795960Rfv68fCAj0oobZ;S0-}?$Pi}g;oWbZ1Q5*Oj z4YK+5Gevq=_4F%q`C^n)d`ej3fS?GO^`cTVGLce0jfW=3QfZo#CnHK7qyn94OCsvA zST4BmLHA>6^MeNuAeb>cMGczNenw_vqsayCKnv2D2uU&K<7feX=#c$)hs;jFjc=gG>yVcN;&;?BpB`^RD=lK~_bsspAT?sdzGtN!`7d2v_(h)}u zexS)y$QmzkXs8Y#?m}loe2tmK)2HrFBMd0AJDX3mpkW=*w)}OTJ>vy{-bKnMGBo4> z@R?z%`*SdyB6sRouDP$7KWC)4fz~c)X=%YXjv;O;TxKtAAjfvq-S7}1=4R{WuxM** zt$OfJDmeGh~GViWZ$wiU;Hjxc}4r|WB$Xh{EE}k z=k&-u9|l_Ei_1J_IJvmGEI5M3H};8o0=HoQef5TkfF17}*7i$Je&qbno~(;Yz2j#3 z?$-!?Y1*6GgXsD1_Da@#A$?c76bO==v4yuzTa>a;{moEA;o3NOEW~p|T}kx|Tf+|g z*^E8fVO-{0ezQ#90stttw{L~TS6rAevXU9Hxz&rslceZUa(dXGbO4?q_5nX&dbGpR&O`A63zII=Zt^L zoc|TOg6UjRsJECs<^L!QMXovV>573Y{e~2MX_3oBa`@ZT)#g%%KMy#70%m;E^~cTI z-sR&~+#jPwk76aM(W;y~n-w0#9Y*%8eYPHNLpBuVESl-Gtvu-ad((4fdkXEH3(vp? z6BNFjbF$3U_PUYJ?hz~NN~>SqE43)dkRA+X39moMz%!BH*<>=)b}!8{eT*Wq!XwR^ zqZJ5GEIcZIE?n@U>64Wj9gxxaJveCiBkTM3gG)=yo`hyu(kWTeYO4It^jROm*FO5C zE8g}8O*S`e??_feQ`w?fA9R$E6!CZ`EZ*z5tTqxGVDEF|EC`ucd(1f9&3gly#uHsE z1BCTN=hs^cxkAm(ohpr7xxL#pazZ6u%dB?DbdEf1aBlW(^}E<8lYKv$(up8s9&<`c z4%J9W$~cXk=kL9AgJJtv4R*xl*}Fz0`Y1;B?LRT8YF`$Z+`OoThf?`MgUQPYYxL1m z$;d?iq>QIsBcCM&4x$fyVXs4^KDtpsqfkF}Fn@J6comP~e7Rd_c6+x^qH5jAHF0E= z@SW~d!X>4~;+)PbWkJ6t-xbVyWzv?0>C`Bza@<+Jb;(?uU!u>k*uH?KpU-TFMQZjv47;bmHcr2V zxbq~J_~*fyexlp?apC}epsKXf2*fxW^5#P>h8Q$*IHZ~jL+w6%TT_#!X6=L8;u4nZ zbJ<0r6WSqv|9*K@|JdSESs4P>;OVt3lJs~fZ{je%%KzG1DsLXJKM|({>H|00JmEenLKuvL*JuWdlU=YB-DI zjPigd>B>slby*Lkf@1~Qm6zFkVoT-X!T(HVRJY1R=7RW;f8u#e0) zO6)Oq;fER44927Fz08@pMkW$wWD~KJK8|788GiFplpqpp-?D_ zB{fAQme=H&ncsu&cPuS0K{RMFA8OGW8XND&0F9{oFZ9H84-Z?}$0{ETKVSCX`y&$& zkjfk;UrB$Ait|qSW%JkUK@M;m@ zx_N%SwltQLn>xT77F^1BGL#vLoPkevM?geG>#Gf%!|xpfJ&1W23X$)rsYPEp`?Wi% zxcGE@{FL;IumLtGixq)-YPgdynF3lrj;v;;sH8+gDFD_nHT``NLcN*3?Y8;NF>Ax)&p3R`4FbNL3$3=hZST<^?STuG zs#;YXf9K@#U9jI_Zg&si=NW1+x1i&9MR+w=a2Xq6y;(mh+MHs6zj#rx#T>)AR%6gQ z_TLi!J7YO#VT$W(^#V%eS5B|#!gkp4LHnQHM|F0E3ip<9EK_@)?fG_E`kFK@-5o?z zKK~21q)(mRHC#xJC+8h$)fK+OmD%`~mTT;92TJ!qk$jet2&;Vh$-{^eBOhGK4ELVP{*krZBJI(kX=*N5v;>&;J9o8P?jB6W~{ zA)Nq3)!f~6ZD-xhzWC&($LcyO`{LeST+Rkcicd(lpzjY+k}Ygm&@bJPjZ<(7)R^-! zGF{_a+uCNoKGMJ&QL<}vbaSlFG=Fks@;bYW0X1S}QwyM2VqTN9OshM%=bYKX|1HzW zfd=!3DEGdgs+Zy6PU=MB{^PAgS4%_L z#(sO*;&4U=Or$>nbDd*JdYoJ%$96whq0>X`%(|#*LFq$5EnL&5pGE{LV#E4FH`5ab z>S;J_v@iX+*A`M#s?VmN}vR{@?iZRkA6h%upyRB_pd4Dl-|` zQTE>JlI&zf8svtAtYq&^Dl6mKdu3hoUR>_|Keylc|Ihh%&i9fd*^v@OQUBk0fm!G?a0r{{E-!pCn;U}~mhC(-V|Q!j*-SR-4Hv3$E@ znz>1H!0?PinVEqh)5ct!8d+#~wRQ7L!-B>SH|N$J6LT4)q)0Ufz<$9N&})rTnsFSm zl+jsO_fu8+PoIA7E&S>x7Xh7OU$0Hs?35{QqQ7&bi!oa@aMYm|I3Z}^(088pq4kvM zcj*sQCkw^(W*cIM1I@E|1J+ava>A4C%*`u=(-jG13kO@#EqYXU+-cl~Q58zxei zjV{3(V6UI2{?XIJYyqCdPqb|_YCfY@!^(|?Rc_zDjs6Hu5@yJN?I@k%NRU3>_ba-T zzy~r#svQn!$^o?Cdjhw&*txQb3QORcYhcY``_`gixhf`ZQtffi<}?}Et*#~Fm)Yip zV0*CBz@`?;LT&=ZJ2A>~i_rrLER0#+mtS%Ar1!7Kp{P=OW+EV}sU-Ugcel#8(>|Fl& z^XB^cguAF%NPg!Htsw8gc00%QI@55z-2lY%raEPK#GQ|?_0(&mbx#Z)vV>f@H&Bus zJBr;{MS7NdS3ly+t4eolM9@Ict)(B1w1HbgDmxB{mZtNuejdIfSw6=Cr?qtQ zy#wV6&YpN8M;^P!Y?s&hMs`E<@7!%(wvvGGY%9y2sgkn(A#3#_{4fcSSyZ+bL^zcc zLw2eMZhZt<2mZXg+z(S`Q7$g6aTSrG;2o)vy`FLbL|xsNC3CyChf)K^tB-4T-LaFb zaa7D{&6#n$ihU0zT8D;+)cO}i0SgS5E*+f$lj0H6AQ9r>0i3xDg?=MMCpXq0WL4h%Q=Ku>|0TgEbGC+M4Q)Q zS9Ftp{p(sn%7ChK{@BU?NE{}ONut}rrKURn zlc{wqe-~7BTOZ((cI}>!z>~qX&M{l zZkc)**7`D9`1IX_sw1{)o2~KTuM&4qJ(&JvJ}!i<%ywMTO#A!$1Ig$6>}(?1^`oYT zNj{5Qh}gZnrDdB`i73+>F0VihxPDjHO5$V`)_CVW&-|w&#ysTwPETiU6OxchN<1!R zdxA{GKO1kf+ZUM>69f}afZw5~))?RAQzJp;1`a@zCc3D~8+p8!6g{P`{&s!+Imrzc zR@O6YY=XOE-?JL=B~Vd7X6SyUh=3)0nahlG8XbdR7WNy~$`pwDtr#8*Iuz*GVo#q* zHB6zE??s)k?jA3$XxI#U+S*QAYvb$SKvEC?1H81GB#nc4p2U$JMn!~EV$$c9VA56B zjdAw1QBMZz9=-H0UzpBCGr78+FsQtCE=4?xN3Py}yLWwTf}t3iGXOD1rMj4MX_0jw zI67`|bL;x^_lhCp z)Av4vukVL*^%Lnb>P( zb^&WA^zL6kmzVm^R0V^hJFyvz{}tio42jhSdU|@{5R^w=4Bz!d-u61I%^6=%8=p;Q zXJ8;L8%1DOpajvhT9rKMNs|SF$Cxozmk3iDCmmjj6F4*RT5sg%Xc`Jw-** zN#7;*Y=0LpqxM(CV6- zV-FM5)!9Qvhiujw8j6bkOsZt}Ulbd#-lVvnzqCq9y>XLedGAEEX=Yk`939KHKdzi^5LRhm43my9xcTKJ?EXg&IC&FPPLNI9$8=oy}g`_Aoc$O))oxe~>rr zx{#HJmq9{F#d)eU)e8B*z$H&y&rH-E)l?3t$o#qxOCjMBTbDh&Uws;jS#lrVc*%S+ zk+w>)Udaas`b9{MVX#V}o~e z_yeRhXwa3^-k&AobL!*^~1c_gC-i3MuGuL((V(9 z^Vg_onE1p1PZ8AS<`BJou6=EOg&u@%= zvTzUC4{{lQYxZq;NWbhN|DT>L1Zy6S^Rrikj#1&r>0VvKQ>KSwSCz6ucWYv7;}eH} zcrsY2ST?UM2jT{%oX+y9w5n6EZj_QSvN4bpG*Et7ENt359BVhl-Bs(QaD3CgcPA*T z<>U!{-X#7(N;UtB6#X2kD-~vgd&=qT5+esU_XeYue)0uYo!ruCIsN;I7MZ9>@#og( zJ*eo4k>}5TO>z$C5Mhs0N$n@j^4M`kP!$)7SZKlgnaEdD(SWW$d5M>7uOzHwcULPN zF&EB3K|zXcv7iX;8alQJix>kliRDsqMJK0%ou zOz=e&rhxT8=@dd-GF2d1aD?(%+|GK|15KV!9lizD8zxd!a`#+@@>OfGrdaRS;J$H* z?=)TI#M(&TJK4sq?7BLJyLTs$Z>8K1(eegzar0@VjL+fXEYp{G)i787H<`UWd(e&;hp zmX>!}DeCAX>ZC^6Tq+ryq7MiCPB(pj4ly~go+hv0OfO|fe;vV zo!eRG(WB7va&fNY_wTQ$Unkz|q&9YT&123DF^g8NG+iJ|06l06hY=QxsHr8K#_25} zmG-%EzyBQwI^pwVV+Wd|lOpS{5Z z2~_6)D&18P&Pr#4ca;Q9dBlGS!b9Y*4SU8DU%zl!AKf`?)$e$XN2vbJ#2j}Q&6gU} z%z&eOo1>%I62-~->|`>O{nIkZf&P^u<>Yq#uvbj?Bx~x(Nn8NCG}u(SMe-@Pp5|1C z^BArhudr?Y5W`{^bn|~OCAU;;S@of%6I#jtf%n82BuP)~$Ev*cQJ3?-SC@CwsCOsr z$-jJ|_4pv9_0>4<-c2ppYAWfXG5f5}d6-vm&s5)YN>2xleK=iMw8;9o7(He%CVm^8 zzD4nfI4gCmnn^$3&hsRBKIgpDWLNy8>zBLnXh%ZD&YHZ6^5N{!K{h=zb1z!FaNsVr zcTSH)rTcc{O=0$)PILo-ezDur`D12{=)2J@+12F#EY|-_D&0LGt{=b+I{QBorzvp= zz_6%2u;S1_)?1>niz&Va4RWXQ^fl?rIahKMJ&D{IIJz5x1-k!nCt^;(mAF&n?f9)A zBjetA$$RQF<s0Zyw*p zTbDm0Crwz4-+A0^$=)AlPnUIddZqL}GHGQaihZbu_mmW+Ci#o|V-rP#*EXNN>M6U> zW*Ju0jYtf4!uY5|>)EPF<-Nj?#?Qg`>ZXvBKfZ^?)6m$+h|g(qfNsAi8?NY#Xh#$^ zsqgwt*M}R-4U(R{zSf}o+sV~=BkbsX=<%jE_EqUErv4MplQ|N9A03;B1THP{VI1vO z*D8YU-rad#gve}pTJYy#=4NFZcbdQd;^do#)x;XI6oNUMtRZc57H+4`;c@L4hK6Kt zaCYgZ>v#Gy2?8*fB@3HeZ3R{p(Y{Yx*;0i#Ol~YMyIgH0Cd4StRc?OjoIj*dD*Pbs z_`#*$6cDXKlbS`$5_sagemQ7&S0zGHIDHTkq{HdU*N+34u%p33awZ+)nvbNOdD+zm29kOHv2K6`|N;Pm@RG=0mnJ~UkX zKM>doC?rhGBBmZJMh1}jdUT}RoGH`On?X9Bxccq=Yyvlnq zJ3B4FXqLqs5zEp%yMs6@#Jccg`v!n@Z*UwPYM3n#gx8Zn1jPR;4fKJt#T^_WhU;}< zZA5yS%wKTekqp@XMz51MI9%yE7x~_Kw1liBhSg8(W|dyz+@Gvq`@30Ld->5fudlX4 zcfmX0UhUJj$MU#vcm06n1kbAH%6tE3&)kuoW-{vHkqTE67(6h@F{9r(LZX%O>0Do{ z`)f;KWIu-p0*Hm&PJ&wEwVGC7y{^Bf zPM`a4ZM|t=?`u_VL_vqYlpxnP<=%`{5EoW=7_2PML;iE={2u#7bJeSnv0uGa& zQ9FhuWu9k&YtM0xwqogZBQoXaD#?;CSpG)-mbZ;Y;IUX6A~>AJRkFY3l0J72?nTuUC?B%Vi+u zC2?<`J58HZIe?yvYYPOy3l$v79V0>A-NH3vr82yd2GLh^n%CCB@wLTjC-xp04}iJY zX;77mSXhv8kk~n+!HJ5A6vHZf_04~ftrcQWA>79C>z8m#%~ZE?_Q-6pOd;~CN3L04 z_N+bATaF=>o~G!JkiY&F*E8RoSV%CX)JPf&ioH)8em{?qgYE(K3r%zEJYI%5js}d! zdZ%n%rQ*`B4|&j0DrZ-g)2sCadvDi1@Q+@XkKiM-IQq>`QS$fCq=j;HJ+3|bdHkfa z&%>N~)iji>cIJv-$gjlFt5hpB*vupz{$^Q*+i)xVkaruK>^2tVu;YBABTt~`d~$uP z%UxFku`zUUaq!+xhjI|oY_G5PYH>m-oaXy3WI!L?$kEY3dfHfk+i%m{TmJp;@cDU- zq~zp`GTqd9dbV*BgYvIG_vZe7lnxANoRn`x(FA|uDsfpTgS5gK|C((K|U;ZYU)U{Ii!DkmhY`Yt)S zy}3!r>0kATbf1ihB+D!)=*l&RSDmEV{qs&oTl)l*LDRtZYKtD6pkwHB`MbQ_2!Wct zBUjE{g(XS^JsaD2ho0Dwn1n=TPL68iv!_p0lfVQW+`w+#y0r%PL&qG@)?sUKG7ism+-IJ5Mw#ameb+c>@zJ%w6Qgz&!%J<8JY(7_WkY0uUEh zq;BkyJO^hkx;*`&aO3W?7Q~`+8)euo?X~9evdE;yOR32YyZtp| zn03_YwGt0oY(NVG-__m`TlRn#$H4o*AWaH*t{dR-c4{9-OO%uCi!?nwqaWN%gV-1) zEiLS!#mGnn?(e#ZA3iXqoz}S4U_uiN1=ZN!e7<0DoLZ|`ICya@bT!)~FAkuzP(wpe z+w^#WLqd%ZfFi|z(=nJTb`#{%j&zMY>hFWu8O}Asb%v_%INgdWtfk!3xi4Nt`dVID z=Kzj>TVCT;YixrZ?=jH#tUIIbSwQL?YEYpLgdg6)8uSIv{2QpZvpc%W4(K$aDI?kH z^Z~sZ-|n(eyC~V%$6b58N%PcGuKKG*?K?q!BvJ3rV1{!GPVH|lWgnVo77zBucE@g@ z_<35&dV0B-;~HceDgutSgi6b3jC(ROTV6mqmy~%!B)Pm=+Bm{txBNp|;gC?-h=alu zIPa&QpRVmi2H}@3G^0i38zyr#wc^RXbMQ?ApCMbWP6{zvhONJUdC__AwL9^O!<_!A zgbE6KWTe~nYga8zs>#?{SHm<9m87Q)9+iIVmPn6%*02o?sD-7;x|;JhWqGF^ zM#U&tx~79T?aKbk)Axq+mQzIKPVmm>h%^>f_2K$DTmU_@byx$zI&rr!ddh@XV_+mm zSXg{|QN8YgFT3Q$z@P{BTuoJl_?F(^slyO?Do=1m74?^73j3oNPVi|kB6p_mpc-^; zX_)^NBBcKSEu=r%q@)5F6i68mFkV3dL0jy!>?>*1Fc!@3-l9KKb)PqJX41dS-3azy zL=zS0cS6*76(1!@9!wSUgT(m^O^luZe!}MZ`|t(d>o;zMdEL85vFmsIO$J<%6SDWb z@E(AlV;`uh(vLbiI%-6#GMq`U2EswMeN!yzWmIi~<(7*nDOX0{mv36BOYc8?pnUIe z?Lh;)bDyd%jGDU=8hU!ku;YtSAPc!;v+8ln_AD1SOFAv5+$-X2RBq8md2S>_w4+$d z^U)(lF&1?qt{yq=0M>&!d3g*LU?`*P=U1(CAwi0M{ZqnjRcyfYY=f`^-v;2zoawXvkdjgixkK-|thnkOMy`MU@&!}; z#@i4UEL>a@s9eHoLwB=9F{VZxv@&=+Hji`z*Qb)0{>ZDVZg6!B@D5Vv8(=ImQFfZz zsj#iDKT*>Q^|;DcCk}84^G3(Ot>}_(1`ebc9sYUZQ9OwU3W2$}{}=q9i4USYkS8^< zcOj5~qxv^~c|NlEqd=cbZ{v)d)QiNNbC);r(NHhuxGY6l^xkyW_SCMf5XS59wX5(t zu~F*y`s=h)`sP!qmHRQ6;e3_XCf&LdnY59)RV@qa;r*ZetIZ<1y{LrOszs;87cm;f z!y_gtypltl(1x`nOh!H}8HM>R*#4xr#E1Bx&o#*pwrkJx55AmHTQ4*j_9PuT(TyN+ zDOZubRrN4@Ags9iqu=jm?qz9{*(pWDM?A@&;iA45=Ui;nLG6m_Tz0d2P&Mbh8$9d9 zH_*%$9YLvQ7uP7*W_kM?i|!y0tl$7Lm6ht3mY9;#wT~v$i%p9c)qe;HCqoT&Y&>S( zS+CgX>+KzaQvS22sG~EH6s3;apEKeYu>W2Ga*<{ed2c(r$m1m#M{@=#FJ1O5O7J}N zSa^;T6u2?l%Y|AoBI5EGmq7HtH3~O&_9r#*4pKrM%qKo{7}FbeMm7+RDZ5jvES=?H z?JvfX)oTsgNFG*OhJHn?e6^D@% z6|z^nI{&h;>1hg$4D!ij}D9- z%wKU%$@b0dZ@I0F>B~%eLEbt<4R~o%LLF1Kuz=M-<(KkuCQZ4j4!OVL;zOK}*Ks6= zOHuoHoTkWrQ^Fg}r|Z69_EY6|sG0eBGHMdz*XAGB+ybM%&GP4?7sH9FTgtWmpfDo-w$&KI=qQsp}Nq!ZT5p2c!K$?> zwI*ON8Zrm-9Hpi2aBDkNpD(eCJFOKK>=Y6F7!M9Sj~!-yTa~0|W?~O7%C54Uk72Jw zj1Aj2)C2jVobW*H=mGK=vV|%C`)>e)A>e`{T#x-Mk;mAzswlz%@`3OY#Blr$o-JXy zQ{i-BQr!&4~^oS+==hiAHyijZwZ~>qn`1*iKy#c>U=hA7kVnePJvx zpc!xA@=E@7iG^|Z4aqMx$&dEb>wb;VWW(}{oElqJL3ezU`S8TbHv8cNVPet7jH~Ey zCC0=z%W$~ucI5%zZl$7Ua_Xa^iamD84>Ou1hZENmr?^>#-_Alf{qnqK_kdb*%PFt{ z;gnVvk|%37q;4oid*VsG#<54=1TvyIk?9+)*4S2SjW2&!f$No?>GMHqCdoF=X3}1) z*5ypQb-}DTh(|t{I4mI7HvVhlvD^WhXe7ojf0yRZ?YVwgOszk*D-i2kHoBQ5!hYL* zr*TSvWO^ook6TWzaT4FDDc1%_4FV${R8KiYz(>h(dhcoy#Mpy_O_l;Oi%B(Q%a}ig zjf5e!emE zuPLg2=)UCDpaSOZw^&$44=0vT^_Zx-!hzc_UWoPntzHT7dsfq)yX}m#KWzxCLIturTCE)}*alP|d^pA`Z@5lt zGHdG(5n8KjTe9|7-^zu<*CYA#=vVQE3e_jAYV5WKkVYF!YDnSc$r$XFGbc1BF8J(? z$X**N9krwq|H8wcflk?ZV|R*Uf0WuEr=iU=d7vjg`l`DJHS(mX8<(qIb0OTdv{+36g|{;|R5}6EqX$ zcC^(qqEtFX54)*6drYuInQFi(3;bY-}B%%IeU3N~LSNUmSl3?g{Tzj$$IdV6j6 zFQoe6&%t=jVrY2laP2e+v+$!QleJivzYo1R`uYO*hbpZ0N|??h*b~vpQX?ZH*Tvr# zqP|~XWsB~HbKo9v>Cm^FF=!W7-uaXzc}e@~8=#H9r2o~~8QitVF0l;<(gaP%PPSg& z0?kjJBuj1Kjtmp%mmkNXf)N1$^z`(-D$d;KbmCT|Y5MR#Iwhsa!f)TSY;cE%puKZF z`wX@)N@&1)zKq#MLzzUH=@SJ_anC>I78M>g#=K})`6u!G+{$5F1Du2hOGgnx&9jiRF}`$5!Iyx=xRV>GFJ-+;V;+go(TJ#i9?`!XXjA#wooXvi!q$J? z4gzasJux@l_$--~Y#liAQob?Hc{Zp6*zF$lu!luAzIoh2?g&~5d^h{mHaV5Xuzmi= z&!V0eMTIr^Zm2-*P5L5xMkPh%tg#F1xStqavf*Z;RAv|~e{B1{Zenya3b%nf4RhR$ zoORdn!XoDua7?C!5B{!u$;g8Hca7{ePJ&NSU7a^t2$&vBzP_HvPSzzOE{7hM>kkJ* z3@Tkp9`xKafJOvL05NdK#=*lBT>~vEo%|6Oq@J!9gC&D_vwyDk{Uw+7tei2f88VNP-Bv|=4nr8u>tpM>dUj3ux+2sEBV|izaO)89LJPN>*Wa>! z=83X9nj&=F)#&b>?D=D96_M^K#Mup>mKAO4X$->V|p%J)@ z>S`u%0BFrd>W$)lJ#9qdOY>3DMK+c(v|Xb=TD>xQzh(^7u&M6n3B^I0TFiWFEjO~C$UBGJbf@W!v z`}-s?zYPqPb4yg5i=cf20Hp4_f$Mr<5xCh0W*=Van{}On6Fa?QzB8yJoF@03tTZJM z!M!(1rxP{ryUzb`B)M&DOu`Z1Qj1{CwbgaY$89O_GrFZtjdS4Ry~XbRlEr0C4fUBuQJ1Q`){UAniw&P{=uzsK(KY-8Esy9r8QQu(*K4**QcdO` z(bCMvH@9~B?y^G#2?_1dUv3x?Uvxg0JHz<*bV)A5pv2w}Y2%lR;f2Ayr|WOA;|NQ$ z-{$9H3OTN}0Y~5arMD#YqSNuDEB{T(>_7p+vvXg@QP|n%9aHr$*H9unZW*)`7tWAt z0(}3R<;%GnECMOo9^cu@E-U*7Wf{h*Hⅈ+7{B3(&;JsGRPf`TnX4Uv!<80Ox<^d zi#&?z`HlM}Ouh!ldH^47>Y- zEu7|~$5D~6HZbN=LA*cnmTvL|ULzVIT~>59C9sZ>Qg;d&A+9LxMK~11;5XPn8)-hC zbnRrvKA>|-dFkvo+kj8e;0Q6a&xJ?n%#Nl`f7;Ted58ObnlEGV3rEM~$}Rom#er$; zsy80LX{U5rl z7@3*Joa>`s?Y4lLj8UZC=TnFK0&YpV(;6Tr3>Xx8&N z?RiCBnlH9WVG;_*!h*jq(q@@93>dV?8B_1~)F8QDfHSW7`2jfS z&4LqNBT7FCnwW%yGqIqtI7#-kZ_=r0eM}w5I^~qUa8)(U!Q=t+~4Bnz`gsHgndYNo=S@TjvoKA53fni(zgbC@tBCmF8VA@f4lM_qo< z{sez?zO{86r0 zv=o%q4u|O(c!dS1ynx1Xn>UdNw-p$X&BHzZ`@J;AkX1nMQq@hqG7A^`&L2!}GWeY~ zpS`NOYmYn!Q#yNuTlRFARMVKF_j6}9sfL;@%5R$uz8urHcU`q$%Ks=njC z?&!GAJL_w?)Uq}B&XcMkE>dVSc!Hc(`Yugnz^9WyS+CX>wdFE6Y%aYc+X3UdtCxQQ zeWJkAcYB+ych;>J6&c`9xE#bkWfm|L)s=GA-=g`iIUp2^ALSIjZR|Jv34@RTpznEx zra*j2(RmUrBa2o}iH+vZZFBMQXHLW*DP!Jr{g8h2@`KTYeN|2=rNjPT?i>VJA%$4- zZ@j{F;Mbl^Yrpm(eSF; zN8bi|2{@$(o<)(zP~C<}t@Ql>BJA`g`JXNn26r=5f5=x<5q z!)MeI)YUe!9;Rv}Mhj5#CdH%7x-tg98Iu+tWq$vZH}rJz=aypN$47>j)9@2@K1&M@ zj34t!NQ{q`dGpV)4iTQFYvpash*)z=)KH7Fngay!`vwl|A@skwJv$3JFOU#UgA@UT zfBpEnxomlp=V$~v^g&hr;~OWSqd143lds;;ak5`+6)0ls0N{{qW~M&A^Eo^~1_dS! zrS}pQHuKDc+E)fShq39)OU7HjZ|KCo)sKTP3^o!li=b3{zfHE-6W1h*ZGAFcNwR;C z+dD82yt1MPcIOfO*#g=o6O<&X>g;J%g;U2H#z$+`)?j8U`tYG?hhMl2LJ7;jP9iU&@@NB{} zpHW&`Lu;a}ESctCh#h^Hrx(+aDy1#yUrtH#&016R&mVj^A5yI&l;YMqwU0#FIK=3w zz!3ug7~mb$xK)|>B_#4?oV@S}67cwi)YS@lriF-2mo4KGZ**p4)Rwm`@?Y;KcBH(a z>rNyVa7(=Y6^xxU3VHSmA6#4O2NasvOqE}_Wq}q>{5Qrp`LrRl9H;N43lVhAjn$qt zT6-h6y|uM@UcO{8>ArWhsG*)MA)$VML8NE&^bF*~Z@Cxn^ZMNClFMDV{$t^K{_#w}52ov>$Diz*iL=CJyL`|K zYy3BAJ5DVA%9TXYIKmo7E&dH@PJM1i0o7ooXF3-9K~1>j!`r2cmIuA3^-gC*Z?}Nb zvURH7izEh;&`B)S1vF~sj--#q_aAHftW;nXa#sk-=5LgvvIG4&<&F-n7>^?QIVwC1 zpX@ta+i?eVPV?8Ip7tifx@qa%ISHA>OD`;sit&I5@15>2Utc@yox0-C_^~y=Z+2ndkGjQksRjqRdrd4?_BzrEgkKPwlK-DrgO1hotL!{ic`)= zK;xh({qGf$Y5k8JIy|fY<)2iRq{RX1VSQdwE7{jCZi@7{9!PHe@FsHBW@lAQ{&h#h z38*9p6i4F67*iz;axYtgUNp_+w_mm0atS;7e}1dx6FRw1(pg;eUqD&kVV;YVZIGgh z9-$-w5b%y|AytvCJfH6ZN#gk|E6|W&8_dcoKxjo8$me!fDU=hlfncPQt0a|hUOC}` za6atR)Yu!9zy61LtU~_Pt5@fZYb7-dQp+K1EH^h7gLzvoMnjZh9xPx*Att#MwHKn`Y)R>keF#8ZX2@n+P0c;(JFAvZ(!F=eVH=m46`)?z#S@ z)+Z`$(BUy-%^ZYe#!m+w_FvC8LykJjs}vM`1T%NW({#zS!xBP5Cn0Bm1*|)0SEK+C zFL~N|mXXO*vt9hXnk3>-ELrd|x5>jy0|SHe^KMf4dU>*XCMGe!*PtkYJDla9)Ucoz zUfO~&27_jDIyx0c#{${pl$2j>p8=q6b(aacm1br;?;`kf5MnUEWb)p*1YYipLiYdFpQlrdhUXPIM~sD?>@mM~QAD#A`>Z=vO3+Vb zUYRUJa!Kyu#pg&-Zf<4|508JkMKuU3gCO^_&y+3|{3FXVF}3*$0=HugG?0AbJR~F~ zR$bt!{7=rV3n+S_Y2DRU(7;O~U%;SN+YWK>kjr?F>x&lu=Kfq959~GO4u&1X0jp4o z3=0pp7%gpL`btz5Wh8U~?19~wuyJ-~p8dz|ii-pCJtGjlXgOpeb+S`Y{U5JA`ERJm zj1JO6?)(ocS^fXHT|7reN0)y7L|ApdYED;N-I`DI*B;KqKLA~JSC^GPkZ4ZBg`vbw zyfrV%CLsw9UXjDfI-62!AOp|F$!w?Bog0Ei6T&+MgcrVO?aDGw59*^k{$v@gmr>o4 zE~uQ*Qn{P$j(5WOA_{xdksN)_wD1@O;l?PHl)xUnp@t+nE6+5x&7$R_^lUwczaUdl zT{&dh`gd`xqW2C02q_Xpcih~r!EWfrH$z8QsX|~ZH4Kk?HA1}vd$Dl^MTE=BKq3)4 zfl-)@MC_&~#D8$Z)=#W)XrsqOHuTojI|+gND=B$fLnylRSwmTpMu+bggV9g8!J zqLr4Tl~-mAFmL|u&ksJp9J&@9dX=uUtcn645KT;Sd!95A5k-eplfyTX@M zd3X=KrwR@vGRI^73Gsad(IV3}xZPHjy>&36r_x;xMw)AI4io0TpmuY`4HV?!hAcHh1rAQv zW{$d7h0&J$k613J9d`Zin@w z_uy?Si%D?su^INTh+z1twe>c@xGKGUeS3(deocX`zu9j{7sZX2=5ZU*DmFQE^?L<8 zIeGRiNPJSi^%P_|!lwV0O&KYY8Js2eVaFTrr^9v2{lbNr;R>{N&S8!}|Jz>x@8o@Z z+Np=b=70ar3toiQsfT1_U$^i$y7cegeH)MDDgqH#!93(_EyfUw?fCljTsGPRNvJ0x zCXqj+EA>7m6Mttz9r<#*<^2uKS$nfzAU*ktlIzb_nI10g*n+i4OBO;HcxkK>ri~&& zzI3l&A9B7+{Phg{QqRIVQpUIe{~tG*6#tqA-1YK=1KCQSEnA}OGG`DW@)}GUT?oe% z`xEG7ziW!jB3)m|d{4p9&=E+%Pq-IEN8sKZqKef<%h?U=3cDLiaEPjOJCK?31+f%)+&~fT^maf8T zkwU8v`=ZQ;7e)1TWOVpdWBx~Or5CFCy!KC4V%QW#9evN%?*<9bOXs5RozP|o*=h|E zYT}vtDB-b-F6q^@0U;0<+Iq@-_HvbLfqJ{&7S6IT zM}mMQEXq9-cO--PjM24d4`JARBcV@hb`{s>4(Iv7fMQSnacFj+w{rtpHl~3RDiWbXoU{#?E0>LkW-SRqLKkr7zd=GI%9KfCK4k>r#f>aEl+{ly?q)tZOU3=sPB zzQ9Rh4i9oS*Gdvy7k-u)eEVcCO5QOm1x(Cuh6M_1NW3kXu!$c_kDT-cInr8{Oz+!k@@4N3@41}LA3s3d zM*K2;ZPCn-&4eTu1(`GCjdESNa_@C27^*>$IcH)7{OWl(;bToWwZ)R>7Z+y_J&U!r z{w)($Mh`!V-SY7AvVi4=q~}Ufza^N(0mYdIB}t?axw$NLv%>;Of)0L>8UR!SOB|r+N5*1Wjt)ej4Q7TZcIjJy?^!r_Sy*dOKmY9MIFl}a zMT0$R$$kT&hBJ|m&e;93tSsZy;gFzl^{$vz&qmGSy?}s=ZtlpYAOg0|pqTp%C|pU0 zd`O+799TUb(*jx(ce zd@uKh@y#cyJqusMG&O+XSk2j&TZdZK!toa`=XUOoj@eI7X?A_Z{|<1;=uImTf2NA7 z!bV@%|Cy2}y41(0lzmbWObm>#Aknj;m_OlxmAMCNs0@kNKXHu`G>Y|435ShRV=PvEsLWR zIYPUEEy^poRh6(>mpFb zYY%33RcnuC34~7X*ukhH%u{#5kqq|Q(Y_%g+&@0ZN*E-m|`36l4-hdU|N) zZ+3q|8plh(_~%Vc<$zp!=uyKwu7&%af)7q3#hKAZCq_Vw&-R-A-8b>YPN}%RU1FHud4MM?Bi~o%q z@fEOIfT-n}C5g?(ZiA;{Qx2``0-iQb4j~V- zZr`q?%XqCQvrX7Lsi;mBSUig$c@7h5g9AeKJN9iR`0G3}&WlK|KE+;Om4nbFC*Ql{ z^N>D6#}An*mI_8?K<&s8xY?*R@iJELhJ*O+x$GB38X;g71!gQSjAznVXI7nAiI#5d zqk14JozCv(mJuZ<5?MtXpH@u56!6HzS!qvQ_QQ(d6Q_zs2~|t^!jpFot1df#S}pr3 zn4_Xze4@aQ;g;G*coJtgS3)*JFDhyD-pS-XFZh=)q-WwSH_&Iw4Uf`snBufP=k5Ls8F3ZZ@|d zL=k=`|NK$7nCOdEiWnW$BSMIO9`aC!7rJ`^9hrh67}hsmr#ktMb3r+Ju%pX=MRd3! zTUiBXIjh8O)e~Ethf*q%m{}RTm6Xf^fvzt>r_cZ9`m%>%YukZkcj_|NlDg9TR}eba z9J4!;{VD{?)yGz%(wl0)ozzGuYye zsL#!%XuwUNLuWl<=XMPM?1+T}xSM5WekAZn`t9b_eN0Xc@8IxPYksxt6aWsHMCi6o z_Sw&W;yWS{RZJXA9_E-E_QU9L+ix3Q!_ts|f{(}cV6?GmS1L4pk&6Md3+OrKU)SBF&v3YfWJ z`^VU+IHE~W(wkiXmf}AgO8Dbiqhf%8KD4kdaP@=B;VwIv<^g4n-L8zSsmW07-KUnM zD^FTX>82l;jmDu&R}Go%TEZVWOswE<<|2J8B*dLJs^9vJd_T!1QyHiQJj8RLXF4t6 zw5+X<3deX+0cb`%q(9%|Qhh)b=0;woEz@M<0Q^!Wei3J7K-EYtUek&R4ZRJkdG0Is zBpiCy3;pzHnAG_j3?sgTi=PFZ)a#;HcWqhCYquogPwlza0|Cr zwAK-G!<&QzD|6!%Ed7RDvIO?$gG6R}&D>6qRNKf0SZiXps`?%7K)($GBGH%uT*$Pq zaGjTE^^4!Dh#}%cIwq_@0|2a=KH+@m@NP|>$Vei{W*5r*9x<=>{jpK7j$mSuX96N< ztUP_=zmDL?OdSz;e*q&cLoUJ<=` z)CzFJ0Oim4{`~^1dtpTZUDlJ{J>Jz7Lszhtdl^k*W3zl%4tVL&>`5}aZ+>OCDV3Ic1@VEet3l*?59_0uzzphyecLmU$a6DWOHNKU zifv^ZDY2G(zE-Zt9~4cIsn@n+bqlY`=|3a*j&5HnI&2{8*yoXKFJT}*k9%Fb=C}!v=(0@ETTaLio6@JSlEKBX`Ej7~08rw2?yFkwz?&Y_K z;UEcur|%~CY8yOx!ZkZL=f7Um$1m#bB(~Tcv$5yu%*?S7t(!Mas1v;U)7hY&@ZWRb zA~c7&0^`8IOH*n7HEjIVKHk#hH$;Hv+0$pwnx>}w>w{v>5y`GZ*NF@@_>7bR6q;ka z@m@_UZ?I)M0NA4MBwe`+&A{AjVSZkay6CtUx7^@*)Iq2f6hBgX*cbMz}Df$>f7EQf5yiJ$5sA&j!gh=Yl40yFu=;4 zr)a_Jq?OZW`oz74BIEji3q*q;htAiRYw<+?bFpwZSJx)%#%%GEA8y)D&&}~@>&@Hx z)p4Nfal&BI7Fr2bZXtAXKUVYMnSqG9lCxr&$lyCPl%u`1}9E z)px*Sz5ee%?a-j2k|fz7iL#TBtgOr;tB}f;QE3RtCXtbSgsf~*AtNOtl2NjfNMtqt z*ZrvTJ-`3+dY!($=j7q}-1q%{U+?REy{|j^5cVNQ#HpBFK%jR0VMR&R`z zS{(1GIiB9bn4)|++%5NPSA5y1(X=90*!I2Bt8!fpd>W3sv-%7=Y(Dw?#*<6~&2=Z2 ze(Eu@{q4pxY;kV>(pM2@8!-reyJxd|(T#%{`+5H3AFn67`{I*@9Z4;7O(PFOUgZ7F z@tsubHhj%vBYw_mnv3{R5NaBI8H;^u!hzyAe1ke|pZcxm=4iC~dgJB>?#sG*dQ~qHklv=dZec^^)4tD*`*1Fx%(! z^PBSqMn;-k%=EJJH%c;{;tG|+HG?m4y|GtzcD8&cyzs1GnJUxP)<(5UIq|VA9Sm^q zI4*&BBC(n;f_7wt--3wSTKIPDs%_6Qifzw?d@v+g!3Gs|AgeTAH7=nDxpj+6Q;|GX z4sR65{okfXZg-iJ{-lBf*r;lt6?Q5cPvY{ANTm#8CDW+0AC7PPnfeyNKBOT zt*xy!pZ@sj$+u>6H5HW=m_MbPnMqX0otV;Ax!wFOJYuu`R4OZFVjwIm_}0)bk2}A< zbv*kg-MXMiFH_0syQ9Wu{iX(I@$Wa9d%K;RDuHf46_me?ml~cKG=ytC<my{ID%-mo+yzV5C6=E9{)}^R>kzWjawvS@p@g;sqnw^8umn9d~ zK@rQ{`@E=#`$fP3CEG`KDWa*c)8-2lHJ_HhR62^FW|BQL&=a_Ae;|f3X?Mq}`>fuwS2kToIsDZ`iL*-3$B(xb zeUDO&#dN?09^Mh-o!{Z4|KSwf=djr0%W-S$;9PUiP+57;NNa@P=ig8|+MG*Ey?&xT zbnY?d;e$`zb?=XS%6a!;i{7WP@cGUC?X66UNJ`&NIRxU3=5?uWy4teeZ0z z0jGi!a4@}A?R;cu?(Hm?N-?DsF|xaw;_Sje=A`-FcR$EprCsT3` zOEO;Fz?CpaiVe#=@dTb;0fk31_ThsS2QeOj_62DZZlHYCEZG9|44Ord#*pCVD_~T( zia9_r@uqL`sRBbyU0tr~xt>^G-|Mi9+UoMTSD+e>P6#n8B9u9i$i{HD=vCAO|8Xjz zz(niIOxL+83t^fsyxOTJUfnwy+B^KFv(vA?Hi#}4*Tbs9g9bkVuew&ST6o*BuNR!3 zM~hBOPe(vflT0pIhwoDkCQ{*NviOM9m!(&(T;XUS-&0oxE0-OTl19GiuWW3)iy1Cn zytvL%wERU(IxarGbTeQ}YYv$Ayu8GOR_E*sNA)^xT=;&r4^9y%GTl8qAt8~P(U0E1 zyS8&rlj>vqv-zt_lg#(s4-A*Zxt$c&%~X=Ju~~_c0DB55dX>m~U!WkEcyn6s#9pJZ zuAjeVReEkouDA%+qBiT7LKD^Yb1U|X z`EA+v_54g{J?x_vr>fFioy;G`uq`nK|6K04`qGbn1z=gc^^alycl}lK>C`|cAD0*| zv4~VUpio(V@u`2<yexs-mp?O-YO*}uxdRaF^-pZ@Q(-07d&^cH+^Zd zpBqw>zBq9;Z0_+|b-65EHW3}V6s^E--}XKzl!yT3CfGa}3h9izmr zPmPfKsxX0*kgO5k1<@P^2o~ZYw1&XnOulT14C%xW;gGyM8wZCx)uOmgTRmPz-~nij>T<|Kc%UPA;y=hekL^iMx}*Dv{v`GX%CjNj*)Up9#0Jtk=7L%--D=_plHP z29Do%ys=(&MyTaXVf9HWm>-u{u*xbc-;g(OyM8b7&3agoqo^J&_}vh37+v&1-#}K$ zDY~sDJK&Pa$oRZsW;NZ^V;Ehpg(&bIw^kCX{-G3C)bqvKZ67uSMQ^(4^IJgW%1z(p zGZ{XGWoJx?#swJdY}dBw=j&@tYaJa9`7DEi`fFTJC2e-O@J9UO`g)4QEWASgvy z%<4}}#e)|)Z5wm;5`mCv`7WeDmO>s8>5q8#p=^iFsD!GlAce>gm1Sn8Vax*L7dc7OcH(I?&F$Qi zSK89)Ln1?-QMN5R$HmO&Eq592YsV_wU$O4w=dZ?{c2st>{1tMBeczL1IM9^i`#iVd zMpsw@L?F)X5Ej<5+7h7nprGJ!txuQ&eu?YKNzE(I)YjNse};5oKk@DWWPck2pP$?_ zIW@JL3@b_PC-ew56((o*<=7M$V#QtI@q(*s9~GSL_>_wp)#Bty<;V>T&6~829_4(n zHJfgIz)~h2^vRG;inYE7zRViE*%OeNoR9X?_v#m7k3 zqw$;cS6$4^*sol@O7)!beai1&b7l7JrETQekN1xsKZcejOs{)ITW@b*821s{T%=FT zh8q6`g!hk?TtQ8=eJ@yY)o78!i8t%kgYO~rOIv$8tv$~1s|-EIaW0Z)_Y?Ry_Qw0h z4chH-z#U*~WVyC%ao!B+a@9b+sa9HF!H*wput;DWZBJ2-rJb(Qp3dwR5aXikublgo zGxoct@;*!9h4ld<&y1r3EqB>xz3+RM?ygI$I3Sg=iZh&(~XUdsW0os!b1)v zbm|3*XdQ5> zw4prf@nM}uS~!8W4*3eee3lBl5xDSkVuvnpad9b!!7C_0g7`)QeBsTm&D}eiL24yU zT@furlD?tmcjWl@?9q<4zz4T#%rne-ool}0Y-z!Us*;6#^~DXkR6AT51-r+`V6g95Et(^fbhqTk;fH|8BP1rnb?R3PC#h$tNo%^@7`r2_1>fwELOyv zJ_OU#tzBCSdp2BWQ&$G(D%!E&VEUy?mr}i%0``8N{e0-?(Fpoyjg3yFYrX#27rD#i zI&^+TuWUrQ)S@}4fM2AYP?kMq8zJJ5>#nU`0cR9BJG&hDD8Dyv-arkuV3q9 ze~{M^m<5nqL0ii!a5AepI`k4Vt%`ff-PT|F{rzooTN_6;cJCJu2;0TQ4H6siS$q|* z|M=l)aq1M^?gGPrfGztbJw%#U*0^I&ww#bgql>&g`d#8A{UJm`uO(QHmUmxf$8nK1 z2PY@Z?%MZ~NbvC(_q6A$a~OLgO15R5VOOUiH{6jrG zJ+JAXJ$t`4E*_qjXQy3y+pI*^S;&op2I~7Nosyg^Pxj^0VC3~p0gBs)&a2(FO19(Y zIxDmZ7hd>USw%-jYY;=dmX;Qd1xvoiHU$_T*dxc!$+-+SBOy|X>=iTKgDuy>0kS8B z+(y;j&Q+~@{}_$8+;$Lfy(teK=vZ<8_5a*EKR-_@H0sI$lwjo)*MV}2iDB7(D=uzL zfuR6_xAG!fTV*^q;rb(bGcbOnBE}N}HQbpFCFb}M!&0Yo5|;6-(vBlJS)XZz*3%pY z`1qBo#*o^HOFT4i+pDXdJehbxw+n6$hJV)rbE(Qb5VzIkQ@*FhkAMAnvcM^{4=q&ZGVJGDKz!=Py?_#+4@6mAJ zS2RL)B6O8}($)T1rm*^f++kgw4Ywtn@6lrx7qh!l`fKoCU8}y7r|V{HkDMy~VZ|0! zkM{5`P-*V?MBl}YKOg1U0ORT~7*Ln3->`w>!B+o(^$c#3wF)mpnOFZcIP{gmoq#}r ztL0!J`(Eq~pF^{036L ze@w+llDb>OD>aqNwlj~|@XP(cMK6aO9P$vrLL?EuurT^fX&`3RB4+h=SJ5ca>$icRK$8K;lhCib9k0?W9SCe4VFuL zD{|E?Go5y?;Bcd7SbY>o^`*MH`gOU-C06*&lA_G}%KrI>Cv7xYe!y5UshO7vi+BLL zf;wG|s1B}Y<~FD>baHngHT9;XrY=WNljA2QB{-E{Wl;$~`dK|riU#cHIpWY^T(19Q zfq~KngqZo%*)P&tgzJx~c%E_fMqn0@-hCgMlhlLMqJ)E_Y zfz*8c{20tI7e<9;J07FI9CLQ^#fulWc;&J=7Hf4z9i8GDaSXtQ_CZqq}oqa@??AmyA&XOu;5iCh8Cf6gIs6FP|=kC*vLRTY9Cc}b0Ye3hX-8ifVx9DP{S(sTOUxmu6U6+Q@xX|+R5 z@j-yVa_KtmG7!hJCo&M5_~93^|72!vcn`53xq!Mh$xm@ndp?X;m;c`kpV)kL$$7FC zNlR92@WW#DXMY!C>g46;zxMnj!VAbY7swm%VKjc&trVu~Bnan)LWLiI>VNDoR|?dtb1XJ+W`_eAYfJ zC>oICapsF&BEcAc1}8yWb8__gXUN4MINjGaV|O*%p=5gSat4}V(85SFR-c5>zcfl~ zwLgQ}kpiNNxYg6s(bD3Y>?xAzB_l|t2PZs20!YH>FW;Dh>ecC|VB;*e`9Bp4%UXXGLH636QM+01z-XIZ`Q|&3L?k6TLMr5%s z-RD9SNjWDj%NtBT$F)`OYy9PGzkMB{PMuP1YbG}UCP;hC+|9~T^UXG{V5Z`6g(}Rw zFQ@v_eLIf?2J?YUP|NOE@UqyUVrU!m!7k^=v`$49WWwlcK_<*B z`G4w_=GU*u<@7qf0o(SoQVU@Dfmn3isa-9WVKD%vk_NL}jC7o#&bbglK=*3q4cK%} zzit2CG1h!r)5yplh44nyTfR-VcQ2XkagE2qd`9*1q}&>u00eD^{w5Sig=5-<)Q%iE zM71AmwkviXWdiYA`4~II9#35A`=|W!Lx>f=bqa{R5NodWM!`^(6eM$FV!{p!l4wdz zB5}2UQ^|mmpz~q~p3w;cI$tP66U*qI&}@~LXRYbK?*ZtRwxxuYj*jXTt$1UkHBFN7 zX5EVyL(}7B1I0Pk!2N_-qPCyfd54N~in+xEC2+FOAjNn{p>dt~7?()P86`EtnSM2u zVm+>>=L3?Q*sL%QvSd#!U%vcFZS5gM!0v--tu$w96Z?_D|9SABZG?IE#!4=n!-{`v z$fzh9d{$CJ8c}xpDox9-YDSy z<>qRVd|}0*!55(q9RhvEKmW)=D+it78W7cTWhp7%HIE4;q?E2Bmq4Grom7s)DD^+- z3<&Pq*U^vg!m#>!|C19a91~ToiscU-!Y%i7yKC7^@Efp_y9&C#NC-764e0Yf$^9V4 z5VDVthllX!YE9DhKGf`i} zcR9D1*e39&$PC$}=IKDGiAvJz#rh4j%Tm>2`Klcp9b=h}Bil@J1rDn&py6YX2ap@P zP^x&MRPhiq?hm+mGaPKmEDbH~x;KFewSKyDJD*<>Ny*W>zDfq3l&`TJL)OLbS{{BWreCOV2e=kw_A zNPFzXJrOu)%Y&#uk`RKu6f#Oyti#0lFQz@0Jf_56 zn7WeY1R|N7)b{O*GIX=de##|k-XvgH%Afp6UUeblsy-_x^JZP@qVUG7m%x9&QG0wg z?Af5w;lm7OfeLQ;i!Akn4r$)aiF@bD_AQvflqrV}=g&70*3rwyMrD zD&nX|IZ(%lRjGC43s@7VP!6sVsxtgM$MUCJ7Cf@Ylm4U9!DTQsa&2MOq!zMY-WUfb6c?<=AkeHHh#9|cRq&+596)c>r-*U)+k z0Di4Bsx#I>`o5%rZ8vdH*drJ9$tb6{Zi>L&1Kuvy*1U#BMt|}OI{L5_KSn^9Bx597 zhd5JO*8Tgp2s@Y?gQQZE$h!jpd)!CdmfDZrce za|p;F-E&+&8-voP$vv8(`=#t15ALkIxv%&9l8$T>S9jQq4x)FrY{23EQ;3KlG)G4~ zZ9bVffO^9e(RHDcPoq}y!{kruYx;V}Fg`9`b$(ic2X6<-eG~y7b8ggwr~mTJrSCK#ep@}I6P$c`&tdO+7!rjSqOt$ zjJeTptjtBN#S`!Ut^a(WgH&3}2m1Bvcig0-$U<>O=%WDjzz)}=e2hTuSns+EP7<*% z&z?Q2cua02&L-_NeOjb)f0DwfG9&{8SEbTk21D0=x1!>>@4sI1 z0lMh0BfvQs4%g%6@gvgH8>n%rQ52V(6QFWpf#tyimX8aKjPyaFx_z&83kTKU1qlh7 z72``kovDm9vQMT-B`a#&1qqXg@KGI|cU5cmZ$Q&|6?mS5$bBM*8A1+Gv&nx}MUg-+ z>T4i2(KBL+E4F&s)Bdpu&2*jHLZ(;f9r<4zM74kCa2VHiP%6G#(fncZR{$GXNZ3fA zsM^Fhyurcb1$qTsIuY>&KPyXPb#`%quIEVJpH+E5C`n7Xs>2ByCt#*0*O>bne%d{; zRJ|bs!}cF%lti>foFq`Cd?8f|0k}k0@vr(-q?_8+2I0CJ4HQH+<~!uL5C)%!E&O^O zUf2RKKv?+oe>#sng#U{1{>y)b)SVC#WBjVcvq8V|qV%wAOE3hxE60$Orwoa?bx8zL zH0){Y1{)Z+-+yJ3Yu(C)YnD~QqfUb&i)l}b-gk}_T<{?*)oJ0bb>;-zUIhQw_YYDhpmE#{X@*}OJ3DXALlBauCeEuY zEJj72^2>#<+A?^ytr1$s;I^3CY1J+;!^rYpowm2A9eUosH@D!Lyci}O?he%Dv}Yhk zfoy3Zp>(orT(>Vub)Ds6us`gaoC+vTbV+ftnxDzP)!8&lccB#Vm#|5ZL&uPc`1Yj1;I8 z&d6|>%5q6v{hQeT+4rB$N%bN{K;U;vOZy_Al#Eh6dG5~hlij7xDg^aeC8Il|Xlt8E zj}eqsZKP|JFly&i?bowzu&A47w^H#of_p6Wxl#RbRsb; z5$VM*xIdMvh2tlSNVtx({4Etv zxAT)y@eWHi1E?`VnuKbotFM7(XGb-_R->7%{7(=+3~x|T6VjO^8D@K8Jl|DxjxHA) z6_!>;AVNe4etOw%C8DIFLgkLwnFDbA0tnvSU2-fC!boz8*I5ph=H&_FtAoA*`Z<&Z zfgK6+>yfI1tK!!M6i{>)K8 zkGK)`U|LBsKso^i8cV!)LIel_36!+p7+RP5H*|#wg6p^zA0JOFLqo`; z`QOSgY}o>75sL@!&Yk3OBsQl&3qv%;jbGmEYE~M8XltRhrFjX6LetQY?|-)&UDw8a zNrK0U*@$1oyV22|1QSR0i!QD`y%KEQaU-KGYqm;oR71-}TBxM1+{}{Q4=|jwk`7U6 z0D|VZudPSC3G!j}oH!7tVpT^XTLmlH%-5AuLa8R!%+5gZ#L7Qy4m25wEi}T+a8!(gwkDwe4U-KYkew3Jr2lm)&h4pnEDL$S%&(i1 z4uz0qB#hU^pYA*xXsHp4WRm8Ie_54}blK0TiJn0-JKSOO6Hl=Oe_tyx?|GiP;ZslZ z`))y^{lNdm4cyeWfVM&rbaz11=DStWWfiywWE8<#_5##=L`p0#l81Yo63z%map1Jb zB|HM{<^39k?Yt|*>u!v-Qz^#;eYEQD2prm(qU8tS<@O={ub^12!`Qri z2LBN!w-up`sDj5!7nmb}jbI__eiKB(fP`s1A&5y+#RZ5T1TmoTYSbzZBpn7;VmZBE zqkUzdam9)f=*YMYew8pRl^gW%#v|F++c5dC6U?}(me zBB&RbdclxnM;F4I{nrng+f$pZCaTvTc|>hb`YPFEo`OrMY7T9Yh@YKMwv`55m<0Z^9H0=P~3v68vAn z{-13G3wuC$&3^?TLxdL3K1Sw(p)V&lH&g?vi8nEgfRK){3 z5nEl53r_CnWLc8zKj3g{n0jV1q1mAFp`__(QW-Z}OwTZyiQds3)>8{r%0Ihe7Evn` zWJ2cQro?41z2}XqH3$L+N{9;j(Xpp!03`N8(<~-%_u~6%8W`m$g*Vr2K*9H*>&+V| z5jm)-ct@me2ICv9u)~R}r3#9Q?C5>^!hfG=qkdkR8UL~W-nUP`=bTU0H(K1OXI*UI zCCjd|PWP%Wz2>!h#R<3W^_2h4>?(9UuHjFYn5eb%z_LZ0pJ$ywL-nxeIF2QrejJM)&b zj)VhD8~`u({8Bar!-Xg8>@7WnX;0dYI<}F)VwU<*Ubfcpw(^I@!O%QEWk2v+lsI@*2 zIJ=0_U2#I_pGHnj&gf{0_73`$JX7!9-L3Aom$sz5V?w)(lR2O_W_49CCPYr#E~QXX z8y;U>5m5pv2DmOOeRXgzc+qfi)VeHd0<9MXvl6Zh3!q%+K5f%gphIy%3%B8zfkFKf zNy?*hSb;DQm%8Dk#GuUAqi$yXjelyUn$RunOd z9E%z+1cSu<*1fgSsl4C>j^@d-vEkn~x8d(nT8Tlmu!O`t3v|*#kuzw6`uRBlA^u6sXw@T)$rJFHZBE(8sWFVHQ&B{6De3k;kbh)O(#>?h2KAC3zrSg zIbSz@ee0EBWanavSJ#w#MFWeUe6@(0+L>Rcn-dip7T?$B=YrIy9Us{1cizQ|MGxcH zr;B^eM&at6P=k%bizwrP*3pHhOH)!(u=Ta8KF{vVw~iJT6Z;2S|8)WOgrkCjH=@kUkk1!E4vw`sVwjW#_QBu~^ZIw(kj?hV-a;qOen8QM#TZ(K#8upN!Xb*nC{$~eq~ z`_$giok7+bUOh*!Ewvpg-|o8fbJS!$jtP60H7b>N+od15V{b;F;jskyaM*eErblgN z@r)GJtuqGHmAFW!H%L~5QkgDXn^m@$lAL_Q_F|=a8W1M#@zBgeGEh51q10RTlsY%C ztnnFqnQ+j(fg>HdFv~*0{ACnM(eRh^;hiFCo40N2Xd9{|f(BgHR-fskzl3sEg=3xO z$?T4dbehz#>6c=ole;L7dNwmN=WT7Xt{9@@C{yO19>-1*dj9a|jL~Km7M(mG;@9^R7UzJb55=8tJe&PL zLxWRI`}KsgJ9d%*_f6Lm(iz8V zpvz~&Ev*k#RaJTB(P0?CO(cs=8?)7CEX6-8j~2?Hf{d8J8Or0J%@6e$4#0qj&tXfxi~0zEus7sxxi`r_xD z+bs(a?-67u!R3fj0(3rA%+aq6)~1ht{21EXYm9h9nOin`I;FP1xov4{eQEb7_NlOp z%zf;2j{T$4$AW6Bgij{3R0&n%9CTuH#|cwTz%7Uoy(J>d8ND;wg7I}yZEo6Du)jF2zyAecI9tC_^rj~PtMQR zuD7wVNk;oZdq;9K5I;|TZGWGKB86ht#2(^H#wHvvxyTuhg9<0{XNPAihR||KNlQCa zkMSVxH7gFy;6NtdgNmYa{MlP*KB4wHZF8;6GCI|BelT*fW~0gLi?G=fAd~+9GlvfD z!QpDNgFRhm<)z9{H!0T6Qs<00PZ{#`g=exTk6_chlRWZwMRrB_%lKkn#FY0AiYm?7 z54SIZ&Ti;_^@79-p|J&!&7{mTP+;s`h{YOG3Z{)Z%14(^~@^Jh=`a;|f^ z08xiVM5I`tQ$)`A1YLWF^1>O=c5^Fn;CWyj?Jz%S84y5WD_M5+fTbX+!zv*}yi|2I zdX1sWmr00ubg!V`Xq3OdKj;4aCP?oTp`)bL`sMtOp-&Da7N7)~w=Bvr-ZJt1r~3`c zBN6ycWOFe2_mJE|BybfJJ0BFq5hZzfp(*VYFQZa$yb!F**8jLzDK*u&r+oHFvXENq zjB0J7f`9ecPz)WGV-_FBwyRJLup)H;+MY?D{XY(%gc1=g+xO-CG2py33kVtV?E2*X z9E%rm(mT`CW3}=yU}1osRH|aJ(fal4^DcaM5r>9w>=a2hz}{Rvr`BUh~CNfK;(p~86kpfq;a zt@rQEx~c}yOt{t8XQ~zussbH>gV;UmXsAMxav2c2YK2yHo$6BJNK6lC=(AHgempib zR?J?`;O+{F*P8#WjiTapj0y1?U<2bM1TVhw_^xndc(|3#VpW0B&@UQ1$>}tdACY+D zmX<^d(8Oe8c%`okP#gqbtc($0Zt%s?%!}xAO}4CDx|q5wC3byR#w4my%%lK4Y42d0 z2e#&Ow6sh~<-*h0DiX(zJJ=oiTpS@5O!peRp{`| zz(8t_0OfLX>=aZolc;R{s~*pf;5%k*{s1Yz8Z+gGFvN+HbiJRdO9@@{J(F@`GPjAA zEQFopu@Lh9lx|&xLL=8@6eZpt#;>YIoEk7w9-fJfqDiRAU}a^!1$6p>cM0Xu?^74D9z49o?rae0X|T3x zIi*{Cw4fVlMorBteQ@@BL}%$W7K+n{(a|sx>i-3__sGP{)2C0vGf8;aLSf&3sn+6W z954M!^0uvPO#by7O*|7UurzsRdl1aMI7FnR_%B?zpbWupO4=$4n`uqI{XE*{po-<| z!MN`NQ50>soe?wII~}1tBZUfZFD3YaYHK(`zdZCwX#*%d9F_{G5Gn zJ(Dc7eSy)Oca{x{1NXDJIRyNDnqH^%Sxi(hnQB!tN|I6cKB`CpjLjxy2 z+^0}@z29bh!>lcjqlZVCJUat5VgG{(CA63#K4EWC5sszBgOI5u}$82?Kx9okP<$ll8l z+`Mhmrk5>CC>PUHQWT;b9ULs4-rT%!C{9GDvPd{7)kDplf*V~?5U{f+Xw==}>%rJZ-#PZ@Xoo1nt9#b$J}N*PtE^Y5%v*a5-8%Lgr=Qf9XY(Vine)LqP30sa!>cm^U0ucu(pI z_1G5F6~W`cE5aTX#%gLpJyNVejkk9ntcOU2JWr1Ho}R2mD6rqVIBsLm3VZumZwTL6 zRaJ#E;Z^tYWiY8^{}Z@yKuCf=Gb_2|_C>5+)RF~&i;Idncl7u7D;HX~axvvCa;LHPAr-AdOAo!k3zjNb-pmh^zyK2J3JagKs%TA&}Z4%JFDO z+pEjK1rNB`gZoHaIGmq&qVQ0WcWzMY8d+g#Rl)kFsu;g^(3KjiWF7vQIYkXormU4m z3Nq5uHHW5#n^gFjS`li3GckR)Lp^P3h$X1j^7pTw%v(5pFp6XuD9h-I z@%v4nCle8U5-X&p{R^VVXYlJ1Ott3Qe`n9Bz#oIVE`{JIssVgcv}*BdTGR$^RX(}t zI`}XJsI^2376&)JKDCSzcJ+cTISyWueZ@bwQJ2DNiv;CFUgPt@+4Qr(E~J(-P@<(2 zpGGB`2lW)E4pdB*tN9r@5ok65MH+5c${B=>$d1v_1GxV0H&s)@#6`;<^#STsn==^s1^Hfh|RXJ63p&w7q$A zuMq>L>(g4(QHfA))B3F0sx^e!Y$|%(OS#AkM#_FPTMTy z*fl!kcecprWaI4wd_P;WCs+-KFYgsNVT*7J{4B}>9TylUSkAat6i$&?eepTy+yO-N z!a-uN7W&JhvC3yH-qJS|Pr{ni`d;a<&ESf^_$@sY2*nM8lxm?>YFT!YfU=V~NDa{I zo%)XCQ}F}$YrTsh&-O4W-798$sRS2JPB?yw4bpWd!Ax8$0H4Uq^+`e*^$Rxffs z)wNnnD8<`SWFiMnPL$~sVL8#?(9Xjapr-9l_+d;%T^AhpBE8KIZ3zfi7mrGCkXq~k zZwAK@O6m}jyxPScw4ym7H>!v9VH)Woxxh}$_E~0jiqaX9xmo;ck~1O`+a%? zvV{|}YKi;wJ%To0xP0E|)isJ7Jp?A(Uc&9x2Fk?L>1tGa;jo=JM1T!bpIJ*(sUIRx?P-##nk9r_*a;M8gb>`!%BqV-Z{#yzZgL=K z$$^*+`gESg5JsPPZ(ls4& z1hWeSi;(aupOb(@P1hX&?dcgA+IgsGm84iZK%NEPKRYck@?|3Sl*AYJ3N1s!dlu~} zCn#Pf`-Ftpz|+=eR_a~WF+aZd>~Y04l-3leZQ&mEyc1=Kvva{yRKzFpwxZQKzvdON zH9m&u-o+Ct|H%ysC^3InKHejYQoQ%K}?$HPq13NkJ`O@#QF-g5DAqyPj7A)j-Q!ASl*qjmo$jC4H zfcp_3OFQ#WPoTO>UPr3CpQeB44s~&-jnpH-vG$?wQm^lOg5D<4Bo!9wMxkqQW}o5` ziXAtI-ZMtF4?)97TZUe9c7vFQd_=E-fob;$JonQ!lU<;gf^Oba!ek%nHLnMj7h)Uh z>gav08l7qla2|=A_L87H%FZ{ggf;sDy)7h-+E;jC^IH>)1XR#r!3pph5Z$}?pm)glc(JK@+BJxc6wC)qIXz8GCF|T;Y5lI zfwz;54bF>Ky70e22>=&CDfHKcGxthe#-hvN)Y#J1l`(){l3JtV(t$)O75lLh-wsEA~LaQsJuTzaJS&` z(&R=!P^hY?;i88(3W^iO{Hibq*T=$ENI-}n!S3oYV%t}&VbLIbAtvJ)|Jhk4T`jg#?B z+}L?o%|{%27V5CUQrMP-gORneUB*!q7`VnVPW1FG_|cJ#K$Cy~G~;RN;!-Fz+wB;$ z&x;rO;LU!DN&8M!2_@gZuWpX|Aov5}SV?>b*D?T-XA+ZzuU}kKQbcJ``1vH}73-v% zv2%I<0pWn>n?T}E70TV_Szm0Ia4ad4#AP>}|?>%to*YG83MM|wB;xHp!#~tsP zXmQ=N&VScK9&|!U9ic$#2#?x7bwrQSud;ILinQ7sklE)%c~qfa_qpkswvkZ=sn@=J z|4w)ejy+gZ7*^@#0nI)G5EBtqw#D8

yIU9bf`4B>~2-v7NKP&jN$l@0}#hy+Hk@ z9xIW8`i-qe|IfOBmC!nNEYqTW5J|!)-?$8f@2j?@w`DHN%0|H;ZT+`zbhykDgi|AC ze?#xw$)Fw#TMV|{+ND}N{;45yNEBAt5! zZGGA2hdXoiRqOYidBcHS(cDHW%V&;M(Y@twIdT2pa)S43X>#>&`Oj@jDcw3B9S>jL zC%Wdi5+?|o(5R?%i@#^32f6=D_D*!RUbS&8L@V@iz_?L3wvlx~-V@%vJ0bn~xv6>2 z$wy1p@n-7ARnJytr5=ikb6AHR^jl50$NVq~?K4g(TPM)pH3Y7ITt2Nb zP>xn zfD^t0bolD$SO%@uxw&FO`vO0NdjkKFf`H_0Dl6MV76Y$$XaCLHFxTzv`n{dmrB7ib1L0*zBG&z)%Fo8|08enY~VGEZqvrX4i6)9&qNqZZG9k2se2NlF`Ey5I; zeOV7;S^Z}qEVJmL=bZpsq)W7SG3CQo42`M)Q-nT8frbJ#DV?FE2GW^`&f%%f7n*nw z^bCHr{ZuwqeAC5B6Jw7t(fY>5Hf>v4!tS8_Ir{=9Ye$e2;cBp{;CMc4+fhrnFu0-} z+1`y^cFVQbt#;;Zgl`>0qx>);jwwFIZ;Hfmn?KdM0h5gul6%ZRg8W5LbOb^5{6$dz zS&3zCtbHHMDaCuP`$`4ZuHNc*58byYNl8gdFu_z(*65*U$r;l@B#wxx)Gzs@z#bnN z2{|uCIzg2Bn*LmqQ!7dLs2h#aq1U}T5C}#Q0F)#FSfeV8S&tpjBlU!ojEo>P|091~ zoz6)IbH8R@{!NbH)*``A%e>O5Q3_-JP~#>qM5+J3vJ03tlwBz7xSDZK%g-pgzQ zLN$+0CBhF#h9ack1wma5vgK_jp^=;)NsA9B{;8qy@!9DMi}CvPZSoEE04-8KzqXof ziiwHkS-xV$AVIaq%Fk=oql1D(p@qAMnl71O0mD9xW>*$UXBfiY0^@Xjtjj6dfdo(u z{{>VVkBtESf;ud)Ejmc5gVmIA`J;hGc|FIjtLOlK99`2Ptfu`Uet$Lz|4}79XVKaY z2t5wd#2F0{dFac-#hC+?2^$5V$bmR(Daj*2V6LMFh+>ce;h#{9;1rWHp1gkabaHLg zlG`JEcr+yGWTLVnh2EM44aj}EWZ}`4FAJTdWbV<$cL^KVwH#c}sOHaCtPRv`kMy!>YlT~finFK3g48()3 zYAO7|gORQ+&?qD&gV#0Ek^>jp0zzs4Oq%h9>7%?nJTJ9vZ+rq#=yshHHvc&rktP>; zL6IrXKu)DZxOBxF+T~X#D{+YzDZK_z>R*KM!!uF~0gF?VjuTKwl{}ylMx}BTv=I6g zEi@>&-@W+Tuh=!m0sI22%S|_>-@Ah1YGx`^xodq?*VrwbM!qQLdVoJyb8Lu{VNY#FiN|!d4 za;&1<#jo9PY=Q9jnfZ2Ff1}%?M>^=@6jdyT=7lOL;OauOwpE$X&D2lGW$-crA{G`A zNw#P|y^IoUD`;HKgZzCo54^t;x>%`-vNoH*djn|j8GtcDl|wm$j0*i;-Me>pL$i;U z>0%l)DV?#Uo&=~8beR6oA`~SZV9#JoDey);8zd!rYPW|ENQyelLWSFc{`7o{hZ$^! z%zUbBFW}u7$+t_tk?oHQX)C+NYXGA|AuMPTzVyqJ+RN7JKv6x2J4iK2>y8kLm;@$C zVh0ILv42C8vDn|@E^Yqus(QOkrKc&fa?k8G2{*+jlr2A-M8Xr)RKO49ktn$UHeswj z!vfGW0w=}VCcGGOZnS%>utEzkiU^#-B(*U^iyO(b^8QvAHQ$)6FRWH@&lOx$<}?zz zAhKb5jBXC-DRi(!kF!NATfE3}jkg?naY~Xv2MsY%QM{zRLt1zB8~zr3Xx#w>AtTI* zf*Li8hFm0@ks>RM^;w{wFo=%JTSj=7CI_Q2v-9&7@L^-dhbgijCC^XX9qW7yNj>T0 zh=|Hbpty4YxaG{q&fZw{PKpf`k_i;mApk;rQKp+F!D}d!{-3V)DR7<*qFbP}A_X|n z=}^MX*xIU+du};bC_J~LAgdjgKcE8!ZW4Aj_GkDHkaicDi|fk&1q3?@)w>EQFHhoo z({}<*VX~c@ps7oRZ^G}lj2v}Rcb~SkZEIWp=p0;68e(7q;HpXqeS-opB|G~_J>v8( zu$AboVw%Z`X9|g&bH~*sPZqJo9s?Z-8Y0PpH0yp$HBD_@n@X&c!#zA#RS|7RKmdr^ zL_)rL{+b_NAZm#+%Sxoc-NR!L_v0!T0tD4heHr4$Q%wZWP zaqX^(6sz7vU0Avh5e^35j%Z6_m$iu^Z=$Q0A3Iuvw=zIBCgEx`30E_J6NsK?G%z$0 zNIs`G%Y9l(rT5#g`t68BxJNi56CELv$@%QWSn?nT;-p)CmFzIJfOTQcQ37<2(X?=E zuhClwR7L2`+m{^MH+)`7@Nd^*O=flo>)N%MY58am)HgNh<)J1(-N=?}klk?-W^5?T zQujXkO0C8LuMUVD3OFt3QyGcX|hysQtZ`Ob$I%FaCg2v-1ZLhBiB?-45_x>%9dxX5tGzss)ds*^ zm^b8MoC+|PU!N_zVkVljNUQ*@g5MB&3MOOd?1whRO6q}G{3o4}l9RTKa@OZ<7+C-> zkADfm@*5Wfqap@NQEYgL=){eMvTv*05$5e2)kN^z3KZ^k`^cC&>SARWAjlY-Tui|9 zF(I)mafq>EsIxmVWaQDP5EU(PG|tvzg+7)upYb2W+s@b zWeJ6=3Sfz2ogYspo?CU_yE@)?7a>y{`S?*wcvPAOI}xdwIP*Yj6`F03fBmzo&>7;E z1&zMeze-8OdSj z_?x@;7k@+ONABLG;_pc)S~)|*y3e&1TBHakRKOBB?D#FfMfe5NEYQk#fRB{4_d1OO z2QGu2bI#a zhn4W%QvKL?s_h^R4{ZH73B=A+MndB7Zbk3xm~w{3+U3C6($dmM=*PuZjd=R>;Lar? zFX$%H|L55JiNgat?h2h~`YST4qGMtnEAO1FCDt-<7KjS4gmNOMM14U{5tLzFSV|!o z!cO%wx5q|)pbozBb4M;HadzO$*8M?~-qEA#?4;M$USy4*4|RCC5hmJdPMib)nM;S} z3$hSMCL)vbFwj3R!K;{TQ}bF$lM&#dkG^c|9$Gv`YI9>>hAs(`vFV)TZuHu2fm|&W z@3!AuT{DrI@Gmx~+kx4c_+Cyc68;Xy2S%!SzGy!A-q0<5RPgacZ*l*%ElY!gt8W8r z)kK5G(dU0L5is|kIfo9se6|4Yd|fb`;18)PMnG@nZ0$x6;8ByA>PeWUNv4+oAErL!1!$ta5-ryVlDp&d zM(R-`VZRK31Ni_KV#S8Orcl(IFy+Ce-vx-Oy1 zF?4PgbHdu5kv8@$($f5IZ#X4E0?`0N=WT#%6_R)uIttk|UEqiNzzdzLBR~0ca(?ly z?4>HqkBEMrxOI!Fzxi8Z?)4+JfEI$*!NpBQNdxpOhVq)S=x5*?>Y{=!)Zs>5&?0aJ z!*w*b=EOQ7Z6l*N6Jl|p%f?6~;#c#vr>LdDrQl>!@4&GuTQE9;OmZKOlfImIVzUhY zY2myrE*D3q2@k+aqre@F{G+!LorE{&k}!T-S@PEH(2E=QW6)Lkca#L&?s?$l{H)>N zXL0zM-p(<-g1RG0*}xsRL~wn|03PFpy0j^=fW_42dl?xuj&3^vXHwfMa)@unjMva2 zh?7cS6ME9hr%s9U)xL&!B;o2c{oZ$`xT4}7F~-bt1B}=7$?8MVMI)a0;HI@j=nZmf zhrX{x3!xE$CJ@_wL+cT6`?eInN2F&uA73t7C-V(i$Dx#!gP;+uE&Cjd$NHwhfBPoa zo(qN_tu&b*K}(iSzfk?f|0*yb-#&Qt2$aL@E3zm5=GvWRND((dN?w4c=M>axO#rTP zlSWYx8N)c|)+GX-M=kAo>2e|oI*o_)oBi~S7T?|x1?+VDF>iJ3M)a*rQ{ZEYZ}+T6 zs3ryQ;Nm|UllLmtqk$*e#L~iIH_>p!_24~Vv`7fY^2rDq{;k?3g>Vp>nibvYjPNnl zVK{Ft-@olnz#ld8M)GJEK7alvT~b_(lypUE%|l~@8=Q#vdWc*f0oiU5zwp?Bs8uKy zZvaOi9vssVume$eWQfnAP}GRi!tBgy=5 zAl!DCWC!?REcfr!q2hK?r?F3-?P+BTLzg^R5bYwe8Q+wo4|*Rlt7FFq0QZ&{JuK5M z{MHE_2&N7_8p5C8X8F{S$`0ZDar@2&K6f_OTuMSHa2CrpuLM@;=IbrcZRQO z(S9^iHR$TvNV7NKKi^`_Degu`l*231ZV55V(G><@uKADfLk^B!U6%E0K517W?_GEC z;zjFT*@-;ksDlzd3q^8r#4Ox2_4`$1JxTKOv>bFEF{8kzv^>CFYl55M?9tV&2w5~f zaz9@8QE61h{D}|1O{?!i1J{W_67}Q9<2Z0+X~(VskuNBhxn+?togeGpReb ztgOszgAYk=5>7U2D_$1n;S_u{uyUt2LUK}1hOAI&XU&i$VauQZ{(M79y0{wweS@~9&HKL~oTT*BL|vNZ)tdBEl0k({K4eXS zU8C!H_j+h7AqB6#wP=P%>AQC%{bEiA2L^frd)$YjjR1gp)z8;-%>$W83k^Q<^I8#L&rqh*Yrt*+|5aLWfz zJt7S?be^OCmgTdswA4fGWXYyD$we;I)k^J&xFgjD86TPoJ9DMBr|So(k~$To_cFqQ4SiTRUNi$Sm=n^z3_XS2qW>l=_yP1dCCWiEVlVPO*0 z^1!VIFJJ)qjR;U}-K1c=er&YR>qjm@Fzf}S=FosnlGD`K_}VHi!@skACJH|?uJ&_5 z>zTq?hPeixRl-@ISfrdZA(j@7s_5PQq(X!)s@=b-F&2#(D@RLLy2AFgi!hjH4zz6$ z1Er9Ftm`*>c)b6i$1rvpg#SGa@%Nuc!E13e>Ar96d9Dp6)K6$tflAs_U#Yzf=m3V; ztz6dVK(>mNC-efU98aCHvgBb@J0H0^qNM^#5xM46A^JgNeE8-&s{=Ufl|=h5Kgps_ z1=kxO33S5zyzVFGc)en^^fBeO=T4q@kaKzS9q)_F8SS9|qh;pe&tzqrIFdQ$)*@5% zR0d#pp(-wnr&Td`rb_qQ&B#WGik|5mjCPM=$Nw!DiYOSEy^3pV<5<5Jl|bmC65#@^ z`fvp&mt~b60|zC1gVyhI%+1W`893&-i0Eub&&n1c5{02%8qr3I99xjXBaRGd~_Vd z?BUV^!K1<@G_5&N_BJ?>t22&(+9!c&NQ)h#{IbhT=zjLZkZg`8+M_& z`DI!)b|R7WO11MJ2*0E%Kj`3VR3+!;IqT8M$mxIorQ@`M;;&25P-!=!ognEv3H?=L z%FPjf`J(Bwp-6Zf>1fNu8W`Iwq7;-MW*UQ31Ep9#1GEP#kzd2KsZr5rL9?W;TYI+m zcI&AM#36|;TLuH9hy$`XB<$gMt&JL|dYLpEWK^Mn(-eU_AcO&)uSZ*C2NA#6fv2Ze zyQ%On-WTZxQ39mMQM$c@94l}nSXe4%?Efz}*azSMO!y_#p1eAsz{*2EK3C_-7eN0Y z>NZeSYh9=V30ogr%qyX-FV&&;7T3kh%H(4=;RKNA1N*xYe+Agln6~MdHp1mcdDi7V zV?3*6P_4?q7G9`#id45RXU?TxMO-y2mm@AxlOvlyaIbTGbIA5izPp$5tcydzjd zn$tv-Am6GGmI59c{Jwf2ihgFMAr~>6;=m{ICaK1;K!C(FtAgJoacn0cQ$4bSTSFOX;ErX(X zBg^OX2<2iTqxInB;l&;=qke(*0JguqW8S=IGsy0fb3X7Rq!xf(uwZil%x1Vj@oCxu zGI}k`hrw54E>NhXC#Oikxe=osZfaJ8YZ()F4RCU?D{ z?4XgshZl|PadqlpNOnW;maINP6vFc%Vvzcqm-8ps@WLSo7XYE8PBS{&!KS|YSz}F8*Pv4*Hwipd_s2_lMf`!WA=3t@n zW>Yjx$@U!xHElp3T%Geu_wX0cfHYqkoxHg(Q&>ROeOrob?Xq)xo9uYh`OoorNaglc z5AiRJ*5j|CpzYRiN(|HX?tjf;-5>yy#Zt<|Qc+>v{HJ-vswqwQOF3iws#00GnwN)F z-!A=tKQW$OvIu73@kfR?O+9B8-zNPyW^Pp9aM6}$-r&-3bFq2=pTl6HSp5sC-8|5x z(KAsP@K9mU;KC>gRV*<%5Ntg#!LLZExjEg~4w}D$@2=0B+X=KE1iot5-)$jw6PCM; z-z$!wcBs_OvO3#d`}Qp>yncHZ+C<6)IWR=b9F(sY)to`GR3dhoGiTnsJgle~?O4fr zWHS;S%uTaue{4qjdgqmjwsI_0KZ`|m^qn|b;7GJ^l%)) zUnES0VIdtFawWaNIM5dro4n$)XE(obcJBaD?Jvw|vkLM;d?UAZMA3L?EGGn==I9vr z`tlq}3}?1$V?qb;zC+YYqudM#82Z5W_~xU5M!GXWQHh(3^mKJaN5%{$@;9;=EBixB z50CIa$}=9Xhd6C)erR2=2cUHtIj_lKiVt7R4IC5eqm)!8eHoco&})`?KIhdm@St7eu!UHy4HkDh-2lB3Td@cq&0upa;rP8kX}@ zX_z%y8Y2Jrw>0ecZn|-J@uY8z`mqXR2dyhD_fS z6wL8Bzf7MIeYkAvb#^R{HOew1>KG5Dg-kDqttWko4nUb)_#DONUJ>tux_B8~XySFl znEm%QV)E29TYd#}xcVdjmLHs&7GY-9$Ru7T#pZU1b}&8i#+%1|t#Kz4Ud@UQmcs^p zgT?wlKSzbS?pKPMu0u|}A?(a1bIUOmazbtFtM{qi3glXLJE?3TEl*S~9D<2YgUDZP zMs@=FB#}jrMl5-tYqG&b4jz!4EjruhBwl4s#63W zi~DW$683xc?APtrx+TaAhISMA32y<2xYyT>32W`#AdQgHd!$Ob@;<&9OzLr5R4rC3 zjOSlGrXT1KgqhA~G~_@?=<;K813AC}kRqX(6p4V}Njy&)(=ScUH2t=f|DUmL75E3a z#52OGOA)1b1DUbnmE>*)6_2WFAp-X{77d|+EyP9T1QR>uLXdElyBIk<(Xp}bN~iwN zN3+Qe3vf)aS7OOmpl_XJ2aZ>Uc#)9+@!{gFZ!*?nbU;V2QeQ_$=lx(4J}vLdXmyXJ zpK1bO7l4eB?bnK55ypFM?T1GO=TEK^>O@{O7x;X>iwzK13jxn9CSdQ9&2=aSNFSRK zz~0F!{_m4mCoRZ63Dd^>V)>FQTOzts6RER}}A!Q3dS&NI>LX@ zHZUN?4imVBQV;O@vpEoSS%x?@20cD-cnzed07CaA0N3ll)ys%XpaSLkvtz3mu3rw6 z1y+{6n}BsKrHl#6w&KEr_!N~zd*8k&R@mGR(G@$q@7(FdK|D?#dTjva?$K1Yqpyw? zj_1(=^eEiLkIcUqBbir>2_o`&T0;Kg4fmGRfV(i zqs^$#E3-L@)Hj32$O^|>lns`$a>!KEVOX-01JsQK}#?Sx9UjNnUlVhG_kbvvmG~?b~U6< zPkYJ^BoE9%rVwetweos-REBVOU6$$3i9pp zHo9GG6J%tg8wcdWrSf9aCq_c^oZO%SR*fy;@37I@m-jEAucQ+?f>QzhLTPlSv-?lg zaJzVjx)ILp^$sLgB0ZF7`iSx6w)-V@2+$#G6oWT~LLL+ddLyT4qG95MVn2rkPaO0y zXU77{t7_WYqO5^e@d-g5LFfb{$fvlF>3!WFq(b wzIV!uHA}s{jtH!MR7gltGwy zZyI&gv*UybAwuB3s@e{?H3pfB66r5lmqkh*90iTC(Pk=e?ie6TFK>Xq|7$RpFIsnG z$;T1Coqdj%%*bLmr)483xoK}(O|pxlu*wi(hf6QRg!90RF3|y$Z{dsSIFwacj;_D)m&l zMgmvrQb_kgv&Xj^TZIv5ErhwQ8!qEk!#~izq1$47UMg%SZ)rGbLiWYzEkV|{I|hpT zU}A%iIQUFi8DsM!a4}BElR=z?q(%tlBZOT!=+ zZMSt({GrakNb&Sj?fw)(;kpJqa9>yGaQ-0X52oZBcH*$8En5H1V$D_CK$KFQ$e=pm zH&(G138f)q0Y9{j?v#sZPA%zrX2wLDTin<;HAALGQFc4j9xs}@#P#7;#9zW)w+F8D zD^;zTN;f0LYtM9DLo21n|MFW~+uRZb8jeId5Nt8xr`NiUVg^DPCa*&P5v#>pTJFC3 zd}x7`t6z`uC5LDf?en0OO=pbfNiMvtnwB85jDJKIH6_yGeVg+A=xB2EFyfH zivVSm%A>@bR61wORX0Zo3IZMrbFVf#4Rom?-iKbnI#*Mn9C*2*y*mmnBt*sVPOV*opi4*-;67V#7!EILzocsWEVf5 z$Ma)%a|zU>HgC}NQ;ZWSl>;RpJvI+LyWITEt=)DIB#?Nfb<@j}c3<8up@5+T(93jG zG5C`?wJN00>0A6O_R+ioOVkqI_Nj>SsuQm?;ed$am1jVr6{=$U| zO`$&oDC@q7z(3ZXhhG&aGY0-OKWnkeO-qofQDo{eQNsn|u4q~|M~>>6k4@T#tb{BW z#H3OGxD-heb#*yDML3vg0KvrfGoZX>4^SyE-u3#}dXW%Qh`t9RFUW@6O-#=kJkGS? z$CfJjwoo&(nRt0Re!f zzdiyJOB9;*_&P)VdBRArPV^R5{jGVFH=?Kkww^2#$Bb3-MGFof~ zd_^cP!SJS#?gtA-BxB;~tEwGZPQo5LkDMii8@sN?KrG64e!@Taq)Ws{6O!;IYiv6H zVWZQ}j#tnLGVEF$2`;MwaXCF(#n~(>EMZpyY$@rVm+SXP^BgXTlu@OVg2@i{mF&bsn)%nH85$BMY zDdW8HVC}0o+@RXKci%A?8SS&X<+qnON|!mujvf_s%_H(ck!E-kCG4?{-ouHP@eW5X zP@j8anFe|Gxo_T}kY*3gC$AEy8xn^60a6n!5R95xK>$~sK6$QCNaiW_AP>jnQDjBy zGA{RRM4Cz)WmY>b-td- zR9G6G@i_!15{pXNr`I6M!mRB*F+^EM>TUe zgoh$FS;yXh5b{duSGo0FX*~m!h0*dt%q|zEzp*w7qzp0Ip64)Fa7dIxKS~Nt3;?Fb zkL-H!f|K0h-{vwr>N1*BGqi)7F=`0DqZAepS+I}NMH!9sg4c-M^81GCzz)$`fH*@P znqwV)MQQjEcR*R4eDw-J2O+X2TFs;^2cLiHZDbYY8?LI|`6&hSmv|SP@1*j=H5F)9 z*7689zLfDq*JmGXFQK1GpkSiu?j~h4>5-qz(2zBDq%KBX_x}hNOO<2$Ecg0DgE7Z5 z<}yqg{%+(D@*|`6h`+_#m1t-%tz8C!jA%5T2OH8scMr`MGV_`yi=03c$7Zb{-O4lP zSy4QiDGp=@xxh0l^WdDfU=zR;G^B_Mu}k*B>xsObvlsAT+165$2pf{5uP@Fj3RgN1 z1etKGYE6|>GKtu6$$PP(jA&jzd+0hCci$ANjcR!GhKq+`QB=!HjkZ7?p0kvT9xNEH| zMsge_PW-nwjItpxC!xotd4(d3xCdndC@-?9V)+f=@KkOf;5-((7Acg{l)z$Ls7@4z zGp1r1Jt%#K8|wU394%%k;6Z^|bSsU)gD308Ytkdhe%zHN)Ol3jACsQ#*5KC(&s#pp z?N7^t+=SK7SXi*B5wGd?@A>*p!*R?Zb~G4}GKDcFn{GnqV-5uo{(_tfIfrmR?W6k% zDV$ZEhIm$2@Gx3>Eb%Zt+O7*n_6BH* z4~219@*2zOsk9_sngk6KvH@972~|eXU5>-0j^E#!wq&7TvJg4yCNscGTWH$nC1?t|Wtjp@V#V9UaP z9)QnEdH||de(w_wy4eC2n{h3vKA0l)8pZnD_(KL$Pa;GBncx_ zIEzOT4rvOUFDKx9ant8_Bi=4p>!GTM2reNFGi8&7jC<|jv)k*l6{@GS4Dh#bRx|2* z%B&vYs)j?zGcV0|Jp_C?uzO_>URDdk zSG{Y3-VImVj|-%x(L6oM7iIDWs?h<>*h#$;+6UV>LcEEiPJo)E5>EU`wd^=_lRn6( z{5RB5D-F>|6m*K5Rf#@u(Dex6d{-(F|L3S1ECv=l?8P2E4qxG30wBCnAG{1T_&Q;w zvoPV12N|Ib6|TuxP(;Z`qz5GlxjT3bz>w!C4W?zghL~v1ODCi+~H5$iRYz?T-Q|zJrWcz5-Y{r#XZ8lMm zj>O4N3-RFatJf$k*x_Vt#@ z?#)FdB}dZR4#Me;fZtSB1;xh~{^HacX8BfxFY`VJ@{hVs)(ZSX%nex)tb#+bE@f=A z04hc>Np?6gD*R#5(7EX0;YSp72t>F3Z-?Bl=fWFJkT?`jdDcw;zX88fm*tOff=y~e zzt$q|dl)cw1JH7Bv z-g>43+Q*Om(yx1Ss}IgNGSOSFu`$hb_olU9{(h4tMDBicsjBzwp&}u_OFCj_Xv)B$ z(4n#WnX$K6D8{_0gp;$x9yj2|jU}Lyc9A*k14h8MODGfl!BeU+7W;yIhZ1Tbue&am zzxzGGHHUi#c)Gwo%G>Pd=R;|A7S+@Rz4N5XyRl1j5d#IU?YBv&o@KH`XiZT8bMKR74ExBW~LYI*dq1E~KFK z&n7KKubD)}pB#I6xD7bwrl#D`;h*L~YmbMpH1{ukeNTg-bI{_V&wScltjTJ(=D-C{ za)T|Sa)U38W0^P~z$9I3sQh$7Fzo^3e(BQbTNkZka_8pM^$L6=wRs=bB8*9#-$C%C ziQ>h!R7$$ybJ$Q25M69P3P7ab^X|p!+mc>Xl;u1g=VNa572Gd*MMZ_TTkoR>$EYue|+_xARPbDNYM?7V%%l9#CDk{YJ zvddehPKIFfyQ2F@sLWgK@zl{m(^J2(L>*TA*I3YbZt!QxW40KHRGz*s}e?O#w!_#h9n z6teR88eR&ngIjy))Ib}yW?;!Y7)Z8eP+20G;8@KdTq}vk^#HyIeYX18vJ{GILj5pA zLLaa!fPw%xO$`lP81E#==9l=q8+qA4XV#jhR7t}dOL^aDFpiK67TQA-T3$DhJOH!W zH<(f!zf~+j@7kviQRPo$Px+AsUI@QoOMR0)lY$T=(&^C7K#D*FX6s!|)5eWa6vTV- z=}+vm;r+|p2AA;lYd4>o&R9IeKk=_8%T;*@>K0{hEI9Pm>pebtgA6jbhzG3sJ)#6K zO+d?n7S#@2RpCbAp7m;!9c5xU|Kbvi)EbUdF4e+FPsI7dLI38ZLB&}=6v+jTs0|YU zID{oc`lB(niYv29WQOvbRoh1-2^>vuMJ#4L1{fNGEhUXw0wD2CTY1I*?q3iprp%nA z*a((XqH{BXsy4?`;wHE^$~nWNddd>iIoU8McaS*e_aQ~R0^7((&D zL_f&;Xg1m21_N!;vPA73trCc{j zv<2?+6QDh-v_e-};rjjUW_~WXvX@F}cotKP!DJPel|@o)G;6_iozK{SRzrZhFEB8| zOR&CilL`r1EHMWdLz@*f0w?UtC}|UY;UBFn#$``j#92^Y*M9P|Nrh1VwH#^R!p0mR3ICrlDPtx9xa&hFmUD%w3hJE|l) zzOpo4=2?<3A2=Bx0Hd*Dat?6ukvbI(@NqMIP6@yhuf?LC<@Y;(v3v3yNW6aa6R1TR zGCUlbL90w{ao7h*-U#0AS(tWfbP5W3O!y?7O(1r4t|Kht)Tt&&QbU6fsD%p-&JSNH zHmO)7mLjRi>|x@yvGWqWlc-)Pc+6G3%veiNics_s7zGj{>}DvYM@b`YJ8i0Ebr8MB z+CO9@_eOxpr4dqGMy9M(;sg|nl^>?X?oZNS6)$)$sf($BDlvwh})gfbnp@f7^yV#+14MTAYsNd}#WJuE)v-o1u4ARgyS2hz)K3 zVYEVcf#1vg)s^yJOhHCk5KE6AD081#08QkMuX%dI#|1%lb?@)BNifB^!3%=4Lr8zU z?QgU_G+cqBSeuBGK<(wry@16Lb%{hSF2-RSB6;lKMIuGUOEaA)a^4OM3_u<6kMSIa znG4)=B*azc0VY0rm~V*CgZPCEW))Kz?|U(E9%uZY*CfcyX=!Wo2Zij?K7rxBabAht z&e0U_Ki&HW1;&9402wS|(cRO-rJ*VYOa#dvZR7A=vx#Nj>(29cE)Z0o3DWm)@Ynd~ zWmdqa-r-!ZHbcyt5%KW|zo&icYy<(w1hyLHlJtblR6qy-vPEt{QoTLaeS?(X97e+FZ-!87fS>ClMx2{0us zFe2U{LmKh`k7#^YHbeBv=#4v@&GBcd-=I^quFV{5%xE0$NUt5-arw#>v^i}D2{u9+ zh}~K2FcGCl&5f!{JWtLA{BDP7-( z$1|y0gC2&u)vjA7Byh?CH`^!Mbv7S#u{FNlFb8w|K2u56JGij_zfmhzE+fsrr+EdFwor;J*1I!Fyd)~RR2 z9~&2t-3p&%5TfQ1Y-2#>H5P`(#p9dI90g`c!8`*YO+l|-zNBP08vGP_yuXmR|4C%z z#g-tHwIGb#6xPUr2=i*;IMu0G6|xJ4;_LxOrsO_8Bw7w6b`$Rq>uEul`Q7SOB1^SL zLJjJ6&>^Rd2`0E_i%IhFi|stTXszu=LmwtUZHqxB7oL=3eH3}Eza{Gs?&#OI2=g~L z#|ImJ9%>pkV+=s$Hj*f-aa$2Rk$`FZruprmt(dZ&cD|NmLyV|VaupYvkvE13E_VL1 zB(rydcQ>9hK(2oaYR~uY8ta5YzHb6r(}|kwIhhVF4Sw`21^mViu8mk#t9O*7eSOBX z^K^0#-ITN!t!jOXdpNhHhopzVqdLqW2>Sbu2y|77*q28-gBj_Y0B({i1WyPza2Rf2 zdYRsUV>*J{QNV{tSj2vy`xd%hn*N~(ftP^q(v^%L<|C;Wa#+|_=|;~-hn5yG+Z8e8 z#&ESVB9-8atM0@RuK1qeF8O701xOCcStv1@sQNc;dqL~=?+SoALo!sd$wf0f&R;mK zdYNKE&O9g_#?A~_yRdfUH~p3WF^y%{)&14#@_1A>lHvfpzS5Z=k2rIXtA?vseB!hv z-s2&CWX&IxuS3$d)D^9*tz;Znz1^YjBiftgIHbT8J8%YRvj*Q_8|jP`5^icyea)$uGp_{_kLLk34u9Wew}(V`MYk2VmXhOpWd>U=T7kO;Sp zT)6}V;O$vKV&9MlC9%=b+ivY>DhL8G70HWdN%S=t|n`^HLeRo`%zRX0TN?uIe;gY1H4ZF33Z2C zDv8z=O(;?;UIH+@q|OH9k>`>_6O|kV)|u9tk5k_XGWj*-(a)KKVO?}mO4YFq&rcZ} z6SO;}9We0gZyl%g4kOo#1IZ~1x>f>9C)Wq+qRzAa$`9?mEhw|n6gf-g4oZ$lb_vH$ zvX;ZvP$Qs$q?+lfgOsc@7$Ympz?PCIZ$j~ zmD6U-i6i0Aen0+SZofK1BxSJ3BT>UZ9#h{q1mHlO0rs2R`+Dh8g8P0VS>dsAIZ7vU zHnab$Bx`QQ`Vl9nE1XvGiCC~n_(JPMa02i;o-BV74MhRe6()I=<`ax1MhPtM+#k!? z7RdXpaL5gDd;1zhB&n}{_hdnW*+l+*ABWgg4lwnircm$Sd>1S%y@w{GroQUsWo4=0 zz)D8v^Ic{=1B`bd;9+TqPXvRtlreImCl$a?LtN%9G(W!bbpn zkuWZw2e@=e{Pv>3j-H#>i2tdP`DZcJ#DkFGfQwO3kw_js^5?&>@PgfTK6zqfB1IC%v%H5_7KCM9tBD=9>2TJry9Q{z- z!;I$xOnz**krF?SmmiufD@#8i&<6D+*CH^!%@h!g`ft>{vMN!6-^8V)0EHuAY!%!}^S{|w(5T|bwU3wXpTK0N+5UNcgY7E(f7&$yv>+EDaYSe( z8+uK!wSph5OstU+>FjDMrf|;(B!b2_RXq*2O5L;btauQDD`Hi2<6V>a+2u3p8%P^P z>gP&~g)*Qmfik2<)7%d?JXvOc`jd%_&1S$>j06*|05IkBa#|&T{^{@uTus{!;|R0) zxX}M}PM9_~7gr?r$zE0&iKJl;0=7Qn0h|-PLvelB((P-Fc8tOpkeW?SO`_ZF5b0A@ zNO6?J+pH6c@xuX7pN(#~6Y(38qe_Y(m!ff^8TcAs9vbVQc}2MEz`M}I!l~INV}sxV z+ZC)PMdYK?tr?YaEA~Czk$Glc3dAE)g_ z&ZzG~Q~8b8qe2rWNap?A5i0SY#TXjwKEIq!N@yC!|9L=eKahyK*n9@ubo!Ieg|Ehj zM8IY`V?jOH6m&j<<70?qP52P5gPF&(!CUt`+mAqnGbhR?;s#z|E&|tHqlxm(NB$nA zrn`Jfe_LL@g7ns{$_eNzu_ounz+7|(0{JwRAOFZCN!R*pAnl{qg zJEC!7mk_WU=O!}C>EneF(JG#?DM$-HgBMnlpYeedFhIQ>4Fq+#)(d@+X>pXmXzGrX>ND1nh*HBz4r3CFLyJ`h2K0!)4 zEM&>X4rZ*fv9#pS8mw!aotWLoLz`^wgF&6@jV~Psah!Gc0~8DZy#xvII&o&J-n`%i zpNGHhho}0;Ep*s7bLI_fZ?9TloOKbsK^qao^n?3+bOQl%>b}I<&1Hr8P^zeM_Nz#0 z-nWwjhItt%fj`uv;7AtmK(>vgCOO}kK73*(@Tee*`;c;e6&)1j(y{nsw2Q;U87S8K z8D1)Ann|d-#pJBC$&`(4dw|v_jF$zBN14cLu%P7xUTm_+>;-RwdHNu6U%n? z1`~RneY81j1mmLRPAt+47||~F7gftgGQC*$GR6LqoO|gz467I;_X5=MKXF@F{TG_0 z4HnC}c z&71$)HTS9q-=@2$J>tq~eHo_!_c6=m@9w1hE80Mt+)_g=H6`!i+hxI#R6p>-WU?Yv zR~#bT*~&#AA38kZ*J>280Ui}5E2^dB% zzi$vyQA7X~k6Vu5=^Wy1^MkbkDM)zC%kbuyWV{LX^Lz2hv~C7#szO)I%E1`XyR151 zyCvgyz_{^)z!HeBiGX_-eSi+F=Rl(FJa*FcoROQ6bP4>M8M*;V)Ow@g{s7h;3O0v~AD_DV6cjauF+GsPliA*`#3VWg`wntq ztfONwipNCC~(Fbc!|pe{RI>nfRg%U^bQx5+Ay(v@G1e>uaE*n6x zksvHs^wN&j9%Do>N=)Jx)=faqPu_l|&CIq}&eP>!@T2{->?kBXk_<#eU99lU8-H2_ zYiDPNrzJyyjhI~|VO?)It%4LcZxcvNtXz!Bu_b~Ow1Q$@I4bFmtsJx-RjzB0 zWQb%#${_T|FvqTw!hh-n{Tw2HjR}8D*#%_8Z>X(ev4pfHHyPY-)ze4kS>*a8N+cVK9<5FlL=Y&Zvv(Co z?JxyiE{S&ij*LY@g4oV_WVu0PA@mRttVYJ;Brj9!gB3)cTYRyCXiD|;mW23+4`qnS zp-lXE!z0q-;)FdQ?Yi7i@0LT563BYk6+-U(d)xg06ZtQ>3_k*q%Nk{c-RD%Pel;Hg zMnl3)6PPa&tSjZvQR}gp0!Xz=egQyc&&t^xTDC#w0G;jyQzP~D|2qUpp_+Q;j#yY) zd#a#CBod7hF@nVqld_yF1)KuJ15tuDs}IXa1>HvVm8pGZ$keC&yh zArPw0SRwFoxz>V^OL?~4O<8IyJ_&dR$qBDD{kd=GxRi{*JRQE7s~v__#)&NHlZUU^4WD1PbfB1YZ*hWc}Vxd_M_wGf~6lNPj$0h7(<=JvoyQ$b_F?=18jhAZ}fo<^iBDG9n zf#yjxyTW)h>}w4;SIg7o5mO!9BcH)0ldG=(fnsv3*WuXv$J>Nvgt+hu{7$1&KtMoO z)^0|f!Ea@SfUA-aT^_Xh$i>LL*%=#PKc_s0F(F`GGVaM0lxrjhEZOZx&f;*=vp=Y( zb$A45k$QhMpFgcP5WND9o87_*j1=l{zS`)TmzcvS10E(DYZVvh`s?p_2!G004a}Y= z4N)1C2`S+9%I_zM5dq91AI;dAFcTOriO?d6Dc!wq(q3QcVd*x<*%#!zKd z%PTh{`L4feP!nCg_V3kCW_ogWDahm$rC#0gS#IOZfYj)8F?U^`X1V%?C$3z_u1A8A zTftJJ?$_R;2y|%XP?s{bu^omy$6{Ii>h4i*#LyWi0+hyO}RPd$K;R(6-zXvpo! zM}{kq!%B(8PNWtG?Xry(JP->cI4;$Xpyo!mFmNxKL;Y4y>>4g}q~X@3OljyD++3Z6 z&s|c-JvV#yY&LPnwS+o3WYn^o;1C#N*7waIFfj1ZVADFF#nBHRj+oAR4+EBD_wYit zv#2+(_9eQQ2IPQ|dW#ZWI^nJw(LZ9|Lm2N!KU$@vM&Jr<;fQ5ay`WE|^hs=>aoj&N z2;Ns51F}zNoL{SBZl3*MXIbC41-FgG=Q4icOOLFcdEdARa^ldy$(id?) zJE~{C>Km>|?;5`MHGO!vP@fa@?NHvtC5N_GoL?#8a&K?!9#xAM7=gbhgtmCJ8+G*^ z{BGGAc|qc?_5<}}rheNmj33W=B(_9#E|1SyPJVvhP@Th__65GuTo<{n)LNVBrx}JT zL<<*6n7>hK?{c+>sSS(Y((-udR1pxq!L4yr_2{+_Epd79SU%F1x;= zzJ9(#x2!$GeUO}7b`Rp)`olRQ45K#T)d|x3g^~Cf8>%%u8NRizPRyP>{;FZOq6))@ z|Nr>Ns~KSd9L`-_oVnUdnYATYG6a2P*uMSyEm!a{br&H`_yWJ8+Hp(?&hSv2o_KyY z<#EC@wAvZk0>{7c@wM;46b!@r%Wepg|3Z6xKwZK4Eb?#QYL#=wRN5d9GhX&eSOH!H z_Bfz1*bj;?VoO*KroA3ok(!Z_p$1J!OOScUM<-F-LAMax;*3nx(hnbOy74`ApI`hl zTU&y8^s2$G8EIguy zWV%LM;Arg!yV79N`|z*7vhN)@aA1z-mU_o@X)%x;nFy~5YeQ_OuSq=}WlXIZyN#gi zLMwpN^ZmDRObpWH)mppyDjv`8jd}dIv|?jCaA4%J7_Q2DH16sqfOTrhhyEV#v{6{o z!o5hmKXWtt3N%BpwYsSi_qC=4ji{DKkM05oRdfKfX@ml!_8TOe=qIlUTTN&9JfOh2 z7ah(0LQF9hh58GOOASKKa2QtUq4*{EPK4wMGqH1fM(@>-s@5`G{YT0QdBa{{Mc8c5 zGl}wE$izqVgfs!1YiVf?@V-~wa8aHpxV@qLx4T7Fn9eZ>%R3XeFK2o=VhcX-R2{$q z@1b23diA6TS?Bw+T$Kmh&U!&o(gLRq8XdWQ;EK+`Srzd;YI5lzNg{%(6c*&GhM-62 zF8?me(S6H%!=a5%4c)4jvsQq91_fljcr;V6g|QC`JaoLEoEaK zJbP4hN>ebjarduj`zBZz&Ixz%3uyE!FEHOuFG6)I{u#TmFxG!u1mix_=l1Pb=nk^y`-9_*rB%KF zEa&>vq@%MA!wO^wI-0~>pnBQbJ+bAP@mp=qjeP*UAch$l8d{ivNgnjy`@<59`}U%y zUnkx$Ua1wbvGN|Qg0$%h;{?ucUEA;c2fQ+er`DCB0l%6S7M7zez;x7CSzQ#x_icco z$-fxqaRUk8!-O|D#-OdU5TEXT;}@Lz_YHR98#Yw!n9odkkd*}~Z>ML`Li`#y;w{Bz zy0=3Ve7uxo8>ctYsh;h*ib;XCMHyCGncjzANVT~m?ak^*M_+6P#_D+XFzNZUv6 zidGi7Y>My$L?;!FS1G#{_vfmU^|>>{AoRQg{&#P5biOjQT2OT)L7{vWIDJm%U`a`d zhr{hb-*39ghMZ`A!8DInbNuE@>Xxn39we=b`P}ck$}}as!$NpYq|2ULL3g=9;;(l! zoCUsuFLKJhY6ks;xujyid+7GlxusjL892ArD-LG)l=paDwAqu=OMl9xR(HmBme=+- zrMKd(v&Xwos~nK$5QbcmQb}B`+?Hco3EgdU<-id4 z?EIu_hG6#nr4N{i?>=KMw3SXT3=cx~*8Sbh&d|Lc{=OT3vg*%N7+h$TURYDd>24%G ztz5C<74G>QCtV;HSo`JuUH{@(&FzKn(j*&i;YF@7uJOs|cp`4DT>3 zw^|WrvGqv64#R2ptwhe`A@c}7)MdWuu|IjMT2XZ2;keuFLc5Oi{`Ig|UUowEmrnH) z7w2B>s_mb4?vO8DG7vjQO42c{0Rk$)F;tbKkq6n(;Pc4gOb>X~s=~twF8hEnJbd_Y zp#xG`;dwP*4fwYsr+sa^CnTgw*Oy_Li+EA)exzkWij}vp2W{26ii%wKUl?_59Q=D> zg+AFe;NMnXJimp11+RONh=3;)h3~$=W38vbd({v%Ky8ud7UYS(XtYxmXlZG=@ah@MqGcrQM_L(Kq@BkH1m zPdkTOJdrLJ?C|~C-kw!dh5O1CxzYQz9NMAj0dpL*uqS2GAF!W(!@AiWFf0w`Kf)`k!6bMAQ++|uv1BCaKLyYl*4#e0wJTWa~4jxQg#E!eNHq`;A{ z?jo9}Fl@{!f}~sug-aAl2L)+^w9+rPkxaMm;lX&{qer*!bC22BB>Td={lh-7pe)Sj zk@2biM@M1~p}50f2_ZLuO8`=+Cr-KN0X8YNOgJ zh<}lb5H?S_R7lkh@S@^Zi;_rjF|ilgf;ep}NYXid#*7d5YHU*IhF(3>{f59<>BVPb z`I+9ku#cgGRCFc^6djs5)pDeB9xwG-zFn|r$t1=-4g!mefwpucZYcKiD|dS@-*M}x z_#9aU^ydIi%)9T_p2NrF*xlXz1y+w;crS`V(XCON^`yqS#&sZ=VX+7L3H;hN zUGSbWgw`2%!)!<;S<~}VIc~Zh50#(4LV2gq4@n})8nC^f-tLK11TOpRSQw_MRe-ia zEekSXOa@7G1#E>Hj5c{;gpg^h*wxy)1}6!fv9Wq)`E5OgCENLTzMkI^I*r-90H?7O@Zgy3&lzlj*V-S@7sJdjSYkzcW^MS=IX`NvFIg>b?8-wY~Qv zR#|VDi=``DFUvnZ`de`8eD^~@mV;#v{ZRxzPCU+pH+4gTiRy0HeCA)wcMbeaSR2H& zsO4y%ndkFD3+I&pE1b)zkjJogL?;9aL%bL74=-ihrHh)fZ{A*UF8u~LX8U2tE1%){ zrEZ{RebL$4IoYjw1qF&hQ%K(DCfv z8M09yGicH*DjRR1mCwQ@yZXF2P}_UvAx>)43nUc9Vw!`BT+SKG7hylViv?eJs!|*NJ+d=0(dLVQIiw1`V^Q=>7sV>DY_)()!aJWi9Z* zCJl&V=5P*o!`Lnz5x)!{GFqCo&P6v*^D>;x-AZrUdk<**v1PXMpHj{uGnnV;UtI26 zck@?g(4lSn^5v25c=u!A8s65`)m=lIZGlQLcNWm4Du>ScZNQKgC{HxxfF!`*ig>BwBHMQ=&V4#EPk0ulKdtN)o6W(bTC zWHp*<35joi;)7oc#L*I;?&{tN%e4akY6@}LLA_)`QF~(HrYrh8lh8!D(Al=#iX609 zAUA>;AQPeTDb&9ELl>0~`uf7J&CRR(fxlBw?VtVKdskXWgtk{^~WJoGc&l*Leipo3<&sFv?|MZo?$-iRaA5Q zC#?qy7~_?AN+D=+KJEObc4ut7)mJ>l&4$X==Fs`3vF))Q)%7 zlSPXTr8@csFQ>@abTOX_12M+VzL=?kVbQ|DvT4BUXD8I8){5XHi(=6$o{$S7Z=xAv zuc9+*I4od=695u#tP-Tpbh_-JwXHh=ALn_ry}qJ}&JSYTo@loeNmUKJrBq&HAeg`_;_ zN&ToQ=pU{?BI|E=-d=m(>SzVY9NjZ$2+@chtIagH-c@kDj4E$gtWQ;e`ZBDJ@}%`- z{a|&ksjS!*yx!qHrG7pX$H%UA4(cM*W5q`K8j|_!pTKM*eb~*l$ICFQ#Xur2gmJYW zx`dce1iEUV3if4?1q*7`_8;c{h%`r|f9kf=y@CZT$_&UceDqUk`x&+1{dQZD4LIz=0Z+D0oUoyZF7HG-Tjz9Zj z0(}kU)roE}g)QLo)oPo%nvr*M$!Q;6PNgWj{S^WwN(WoNGnAJk$fDZ@6@3G1RF*CQj1W)0zzGkUd)4%d7j?2)W_cpp^aH@RQFZ7(%7LA?os`Aypl%b&aXRU~9ZtpR${5qxb`gkTs zX2rYMd1m$ZWtCqz&aO-Toqr)NSr}*o*oW;0Xd1|>UMpeJy3ESZ{}qG-kcg3n z4BOL6;SIsno&;e|81l|m^KJnTD%~$M|LpbgZF{<9$L#Y4^@R*>mUC(_M(d0(4K&8C?Cx^$XOOo>R@fhs^lcddBk|-)gl4NJtbxu@evp3_;y9?{;M4bb9LePzR z*pfbL=FD5?X1OANzw~|gL*$SA_i$rxtSK2c1hXN11QauO+9@>{Z@5?;_lm3gIzj(2^ZHTfeu80=sC~&&78*L`gv_SN2&?JN23pGxF_qznA zeGXO_WF{uA#N?2{e=s#xU2mD%tEcN%&uWv0rTLU*|T z$Vm(L0Dy)1*lLATuAk892k;TyKN*!kwVYq`>-D`Ld8QFhGd0p((oq+J|+BRlPrE z)=^N1tDuk9Ru}xpf)Wz;z5!}kw?>kqJLUU=rQxG z_t$ip(eIdri-P%GuO#;A6i4TjxW^~i8}ePemN1oBz*o7>oZ$JEX~&NX3J}`@K#HTr zoe6|&D!tMOf#J{+Zcwc$=Y*|4dwRqER2E$R2}C845EZP;GBf>xi|2ZLygC<5=^H>% z5rzO}#tl6_c7%VBjYwy7+D0neRW&t5Jwq7&Qc=x%*x4?GGAAFl9GVS)TL_J+?sjGq z!C8>&D`fp*k}-TN(t8;J<&L6Nbh+k4r_ zQFCU^`c#gq%n(XAys0-Z+wpV2E+fIldc{DQEc68|I41PK+?o4y5j1kRA4V0!_TXJu zZJ4GDkMx;ZuN0#gkR@oA<`7wu+ft7FT_~Lct1%#-i;8klHUE_cZr8@Qrz6ZDs-$Ec z-3b|O+1+!du_v1F^bV+J6|&IhLOF0!%bp261?&t-mK3m&aPOX0{STDN4#czXDZjhh z<5w2m79SCR(vvR}mGwY}h{sPOLIpL&(qkA!$5 z9ZE>vharZ&y6>aW4w|U@j_S12x_Ms$tJBzioH{c|X||!9D8WHQmspINaYOx*g+FC^M%Vs&XwXFMo^3W!lgUUZu(HY%q0*sW?~&cKDxJ& zgOCs(|CGf_GR0VlPkV4MItO&yfah(EL|R^}R|iL+H+h6dEZ?5kg{ky>#Ymvc1w{jdQ<(?U>&b8X6jQ&mRZw2ZY>>(8``}>=Q0G zcmvxaM}Ms(I^$w+Vu96CBbQLO39besS0nwDM zK|U~7`g{x6whQ0xLMZidt0_tcN)2@`>Mkswb`y1YS4qT`)6?Qoj;ItAcvskwo0c1N zouNnr7{W7qCqVIqtwrkJC3_ZPJpg8}%8t%SyM0bu8$?{V`LovRKA*`w;UrWl!SicU zbDwy9xajnAPed9(@$~flOS*fQPsZy zvtOMn@NR!%pSw%LK3~S(2x&}-5N~c?i6E2>xUJ9@F608~SPWNjtg(k0ch`Tqvhh@- z1x7BRWG02GYr7~$yHlD`iMt%p7lZqI?aaAzwVH&{I~iFQ90?FlV5qY;Fdp-`#p%Jp zBg{;h_iL@Kok~Eu)ueBh$Luu{_CRP~GzhfVZnRr@$g*vZvEiwFh@@>vCXNxk1$=Ff z*2=Jklgy_gSb)Hy?GV96fojjIGe`anv_EoLcvb*6A{*wI%mi(>7)y!-H{#=O0WsmJ zP?|5m@_drwi;g zE_tI{dkcO*OI_VoGy}sq1c{rmv#b9><7TQ6>3>7jra{mwfw8Rv{I!}gIDyL91*4&B z-k8&M9y?@@PTu@j%k$qIplO@p3Mjr4kD#)>NY2zI)$H^*$EIl033kU75bqPKeYJDm z!N$&OQj4pV)-7sEL2M2#n-QHia|QslkC`+FcZt^|2m}7LTAQQmzIBq;p&MBSdv!-M zn?81?i3&w~s|+Rwt;!3L{pI6Z821PcHR1w8t0~xtDlZ1?*0AWi-^8@Gl~vk|kqj?8 zTe{~y(Oafo0PT*NYh(En&<$#5IV(gzi<_e?)5rz9||E%s z!qWx9VP7J6{8B;%yvjU$&k2MnvkZ0%y-(061y&QXMV%U!@SySv0Hp!g%8joF(oOA{ zg@uI(Lk`{AtD#I)RqU0 z(C)(MvIhJ-2DlgG z^8RK`L`ITjX;+yC$}PjpVd>42rxdo~UfFliOl+R*a-KlQNbIm+=ku}RhrVBNIaH|&xM-DV)iz`}XbI3m30sa$mbDE(Z~1Q>beTAr;uR+vkSLDS-wc z)`XUeB_w>|BcsCvOQuTU?@b_dI|ntZ=c~WLxff4f=SJV5rVU|PTo9i4;c<@kWkN3b zy{$&vZ4n20@(TzPk}`l~Rd9ne-;-1e_TSa<#|?i$4N{J!PUx~6M2;`RV~Ib*LEnKk zw6UXfTPcKEcEK>`3Q^Zf$(Ns8s#|AJ(Q>F6NSXzQlh0D+=Pq5+sWWg=Iqz_-sbOcj zd_yOd?RNvDr*s`Ly4br(;q}i~67^a**$i)*a|L3eM}2#QS0L@YnciRJUiXcxi#)F1 z1N1$D*`%eXyB@8$KyFe%Ldaq4@CaNiCv*MwZ2^j>;hoFy)uN9KtOlMuZp7)x<~O)# z^7iIW(Az>VDXcP(Ahj3ns70I)a%A}%w{CqIFq^dYMZSWD<+kz2F~Dw+QdDd;RBXHx zb|rKw(3?%`YB!{rT8XEE#@?v}0PDqq$K~3nJCq{A;-0J!VK{S{NX_mZpPyd(XyXE{ zFx_>*G#^+UreQDlU|8T3LD3I*a^_6KMScsBVz@S639-juZi0W+6xXfTQHL<*|IE`N zUP$HY_)T8{Zf`^)$&Ta?liAVUeI3A4giM8tF|&y%D};Ji`(o)c1Uy^cB%K^7{Z~1b z&!T0u4<9)2}&L)CxIN zGh$EPBHh4Wi{g5GI&N$u``KHpM8Bx+)Kc(cNmp5+;;I*`Z%-V^BtpB;+~O8N5AGsY zdS3zFB=`kt@1wgMJwYK*is9|r*uAv;P`0H@iM!Cwj((f+OYhi9l6V=wPt@#wmn zP;e3TWc^H%k((fB$PuI&4i7DkGvti%HNj&L@fcDdgvG<_m(vNux#@?!A+`%$jD7YUjv6K>OQJ_?rUgs zU*psS+Mr>;%BZ9;{59!W15^*nQXt=&apsJv>%e^b=_g zR*3eY(UD2pN_%e+uP(4oKd|#nwhIM$8uT=-(P$eCA35z(<6>uB+C^Y`>dz@`i?$dE zXk-F@egj=NdtMR$*My9~H?$yi?b9p%;0r`ZrE5xWh_rNYs6SEn?AbFx=rpqh_W6N- zXJmi(tmJa{+`sVoS{WVGtzSko24#%cr(2V7<4*?-YPNDsw(nfR$eiu-{^#SO;Lm`r zl$8b;jp}K)OSMTJB3xO(YgxqDZp|o-b&T8TwBDSvbnaj&=jq(R=45Xf#xU>4)WZs9EpBG!To=pD56Bb`kC$iWCVpYh**5I4mAjYKoWUSc|-f=6~}hS>QVM- z35J7J z5>VBOs>{diD8Sa2ODDi8xGhE#HPdU9*r#2 zsbQv{{Bfd@VJSd{&$>5#=~&AQQj+7Hg2PIi)PgcnQ?(s5-Ix=D1f6RjmDF-5l%pd{ zi$~9&-zkdmzanL_1os1HtCiLBju{N&+WPh5W2(Y-ky9m2l>SFdVD5yE>XdPI+00T@ zChTL_?lQ`DERLvp6E1?*ZxXrcsMjdAk<_R4;$hjpvynI%18CA=?(EJf5{#S=*hH7C zjtdxw_KMJOB{xL*+OR}b9+V2S?TrZ@enC>3|)ehqC8p6?j^gmMwsNX`25>)gcRD_bT2jCbm&fKpk(o{l7#gMn9 zVZ|F7)3)_AZPe)YwBJQwrrL{wcoj`Rkj$-DXAX**O5Nm7rw+2$zf2F5@e8gv1Qo%- zc^3l|ASh5ts7D-PeDmw&K~+t|zj-pume393;+iitmOWqj@&4G*iYug3SYsk4NfARW>6l%e!R9AjUSWa_`wfVbIO90_Yz;AM$mZ1YL1sNrEMv zHb3&eo2Q&y;Ae^20pnkg4f1i(W_{Je>m9h*z#wXJLayv7B|9a(Iz>Y}$h2uGu>cfb zD&RGyG8nKg!ZZqn;F!EC4WvUiDa)!ifw=_G6^Om3?i!uav=j%6_^6F?#^zSAl$qlg zUi7yz$EZeh#T=LC=Txl_4Mg)WtV8pgP};-Xk=tgw3zy73*a*OW{@GV+(uE z1HYX$?RazmKF4o+p^M$KZ=V(yluKaQi`f&vKsG1)_Ba(DX{^l;Dn`mpB3e7E75Kq< znJ7co5Ydt6l~3{EB((w|C6fcvA`4cp#rr=P7Ty+#4(fG(Ng?bq&+W?g3zt5<(=1N8 zcU^t!7oikm;qp{~hs14P=^uW?=zc9dOv9yiqp3D9GPkqW8tbYZYF=|sVr%{6@r+66 z9Sas2xiIePiq@K_GS>|z7@i*W ziIJdr4rIREX-Ifw_z$GUL{HFkkjoOT%8_7BPVK5}=$~>BT~# zHGmn@jjMn@hC~=#K~b*+)0qTjli*dhupyLWTuS{!ld&}L;&%V89(|Kr17VR~Lb z96CT1<|np2&VqOkW{3G)$g-?}oAozJ(7^~0ddI49G0f~%Ju@-5b{8Hm7tk5B&Vlw| zJ-a@u`6i%++_Yj4J6q%+kSsjtnyTsdLhBAJafMQ|wv=6qKHeuA#D7d!j za$L=?dTlz{#}7Vx)^cF?=07tcBhK&>$G&?i!Bg4VG^DZUo&?p)W6@7<`m(3=!+qC5 zLmAGA+C}%L;@fSKCfQqHhLT-%# zL4X4nRNdRtO#{2Cpaqn`TPV7NY@nZJap*zt*+dmVEHmiy^c(Q$%DE#-urQ#Gl9g(J zaQNiZK*Sj!oV*j{VB(kovK}#Q5n>uyB~BTZd?lp39;&VVr|8izHIgSEIyNm0cocTQ zh&AA1aVYK#B+0x(ixzZb<8K_g_IB9b9mwbQ#xMjxZ)4$YAR)~}{u4mXk`7s@0)Vbw z15OY00q0Ce2x%cFFfuIl1mO>@bSn(sgeV3=qiIyyd}HUcy27zp>mqMJy#<`gaLay` ziCOX`E^D@xp8T%dy@2hPWLbg0Gu|K|k}BCrM{iZ@)Fg~l$b38g)1QCf;1<=ssXeRN zAu5>@dnA4M@w$b8OCUL)kR^NRVk%}1*)q19LItg=OVLh+N z_(jz&(s1&|P(`vaCbbqX6hycSSYZ~P^`}1_9UTj$F9=G%Gy656t1m~Og5dIk-Jvk4 zyMRZ^B#6HRW+55*0ldj5j0nW+2|F^M7}s;EmUChy>04Y+N?ORpQ`@j-)?a=YLgr!9 z8cmXck-(R!YlXMmEP)<533~}AS&vQuAUc|!xgZw;$}T$ri!|*>PlpPeRtO`P>UNkt z0UeN!VL^5O<75$M_=n`P3JX^f;@ohL8Et{$!%DUEy%*QN2T9ohtU`!%+8s7=I~#3M zF@%sz{&~a48e}Y5Yw7jAVw)?5Z5^?L5G*9B2CcrAxa-AOi@JxV#IhhV<{x zUc3Ms#g0y_+(UH;?<1C#D7N<ge3vt_ye4? zM6}B*E~uH*OVvo%Eq7)KJy5U}n1BuKXhSV4-XRe?f*s9H^3J#-tPB9lTOouEB(*ji z{ey6Rc<$u&P<*0()y>$_JdzKY19;*mLRbW>luj2AuY^O2iwTF+QH1V3v?+2QNK@Ln zN(Eb^fPzx=kU+nr8&|JfECPkD!94n&$VTde$?ZXn5qQF`&YZrRtP1GMmq49tP}e=^ zxi?^HIDRmVDxhcWFK+CTjm38?_ZCR-(p;0=){^du4(C$3cysmklX~Zw-(}F6D#8DA zI49LBj$}gX+S5co>Q?T%5`Hsr?81>u_FV2vb5P|#!|0%VZ~{QXnJ6PorUkRz&fB0t zGuPJEcI%tSgb>RrW`#bj5!F}Y(4eb0Mj+GnG2X51KfM4lZBE;g6gIq&?|(^xw~AEH zdWXJjij9i{r>7DJLXSQ8#3QLQe(ND4Fh4y(lQU-eN-%=>6rZE%bcg+)pL(#-0yR7@ zfQA{&sWNhs`^|8ZvvODvKi%s#ZQ4br6xFTd{rg=aJVttf1|YxZs&8L+4J(tP_>4;D z`bRe_Y}CI%VSmY7I>*p=Jttbk2Lbc%}zS6!T)962@x-O zSup#5`s=@NgzC(hYwr1MRP(hM8Em&t_&g~;CIp&zPY zgv%2hwvkYY!*St53~3*vh`Hv=hlhW$IE0u66hB|>%%nj)7bGYVry>AYX_a~CCR&@; zCzNDCh+5YuSm3{AT{RxpA&dpSN5LSeVl1y5n2#-uonq(AMoH4%G z7f)CV+R$MqLkj>tzkT=4K|~7X)XO(p@N7T5McZUoJr{sPEa>dH)8sdUKJ0otsaLi= zcnz)+pYJJAyYVlGc>){cy6r(S4uoaeT}1O^>ZQjT4V;;_7(rLP0ZFpKI{E-ld+N1w zK1QUQ=R$&5y(q^WyxhEoNH6qTKEoQPB8B*S7zk16{$goS)1Il0$R zn0@&>eHAVgjlb&o+Qxt~%3!(Y2m5#~F2k)R*8wD8oQyOfs50p$Dx)=xXLV(ak{f|H zkyBMxHbL{lSZ43ZY{zgSL5GB-ftWoHRh#I?97ZM5P+8W^W@idws6Um$Tm;dvz39T6 z&5!mkVKk)tVS)k*sBYy2tD#2c0QBbP&z#wgGkOFBk4s5y!RdxRM@CK(ohb^RM(GwB zJ3Oz+n-~Lu)o61z&dJ0F(7F`>Bsmk%dGT|grJ1|=PM*c}I$0Q?uqcHOU-mU@9NNTA zrq;Wio!_yIFFJoDg^<;Q7GjIA1~vc7jQNXEuaRT#>_uk^8 z;kno1b&+dy#<~x87n)p^lM#36Hy;*i9U9qm8i%n(F|A{~l;P<8rWq~|w)JT^1=a_( zzxZr7dCdOV;LkYT-@j~z5`?CeALV}!8MA$gYV=E`z$JWq^eGEb3?>OW>$Msl$Rb zIFsv9EX*&SJGW_tBy;jJU`1A&gR&~qyYvl9&#GF+1}Zi3rh^w!+Z=QrMkF21)~ zAr=Y|JcQ;_uqoN!e7?DiVzLk2cQf=&93r&q{HJ9mJwf$ZX)Zm}=yxHNQ-b+L;YaR$ zUg@79JvFs=I1@5py2IfPFf~257FqhJrH1h;v|#_T7u2H}CD@fbNZYKlxbB5Idtm+k zV@#EmNWBVoBe+vcrGQ%ESzX*WRY-Z+UP^HJs1zf1aM_5`aUSWx@7l~h!e~B!YW=TG zYuDPU4oCiq|0QhYoUMa4UKkyYcj!RoX3as2;SzXMY2mWKI&J^t&HxFMQ&|qDv*U2D zR-o}jbq}njTxdNB+j9^-?%BI{0l7d)1Sy%0wI^iwnUF>EJC=NSIWM&kqW>tH@lQ<5 zBUqN0(Iq(V?5RpT2~s?p8X9S}5h_g9E}o{?bYfkWlR<$d1=*i!c!@H4Q82Ug)zDb`+SEn0M#e+UNw(D@T%=F@S^ zGNoSxr6T5Q4xq^nb`HQT7*CEeWu`MRHg=lBs-2<~AE?bBttPb@5%;}V)4VB)cNLcS zxeeAv>bT7zN0faNF$Es3IO>c+LRcrJaoo_kxPy&cnQLM%|jb&8+ zh6kk?%?=*~om1knGuhQ@^)n29o{WS{_xn0C!-0)9S?W`l8-!m;IH*?ifyq5{w{1mo z_2P)0bdwZHM0JKxU4ommFzV6i>_m8OHHOKQ>%Oh7`vhhmtjR$@XnPZ z>JL|FgsrK{*r89pXlX#!px=%GAuj=E7H+QDqklTs-he8Hl!}-#rz_Y%ZOxuG0LNkW zBMpAtq7BIQYe=;as1Fv8-YUEAE|5}Xi7iWvOa!WWGgc7*)V3_V4f2PY9+tXSd2HgE zz9rnCwOPk7VET_It%i)U*1?gGrIxx!pS?7Op5T7XL_aiP3~%PB0s-Wx+~zIUxJIuC zK~hP(q?5q)+>iu|plZ&=3e(&yG!&Xmk&%&?=lvQ0MjXQ6rtI!kO)fsZVzlIQr*-*Y zTGx!H4D9(ZJfzWdMv(uPd;aKl#@xU?xa$=|;UxFspmg1fH z!#jC_fdQ~1l@U-kK+*nfg6k} zXc3JT;6&uJ(7xtZ4jQ1I!NRy=i*JY=0HzpfinRfYK&b^`a^M5R!?5Z&Y|8=7%9yAl z&Weam5~cZ|D+ec%R~|9yUz~i*&@Y5Hw`MMM@TD6r&mTO?4VWv0#D~J|abfiE->7T` z7;-`9;U)Ao!2O%wun`}&c@pS45>xR1ZC()gHD3z zWSFmW;UBpl(N+RiP=z^?Hlau)a#c@D%euD5roxr?w+1RXXq|_`WiEMxUr&WPmjUyd6j;BbMB1#ps$C?Vi>zUrK;*k^4sGlCV9p25$EO z&oCA}kx>g!a+YSM&h4_9s@JSzEPAF9BfDB@IeXdysiO?qsw=__*WV`+wBlZA#8y?CV_gQps z*Vlju`O5bDuN2E~qH>2oN_oXL^^jz=*wq zr~STyk>Vv68Q_Ba0y%dmM2|iEhajWLqk1|Ii#$ikISUm2utAcw!{NzH;5lLURXBHX zk>hhZS=83~0`WY+jT;!wH5N#iH_zcUj)ZFyD`+@-c1+5K@rDNis?&6!Mp`Y)xHbc5 zC2&swemb1s-)WNS>tg&%>gwzN1tivb4Cp)3zU{tG+Jx8}5hZa!Du@L%{cI~vM)O_|brms34un$yha ztSTn}%vH1M7QIVfJ@B!(uKwfdbJ{1QVf~J_ic?$kXE!$IOzK2LhR%eLG??c5>1ga- zwKap4hV0EE%3DARk=e;?Yl4ST@7~ROutNd@rLV+*L#r~`^cRM_$jVli3#%~9MlA7s z1D*p$0R0B2hRjSu5+UNY@68NQI)DoqQ^%8du+D19r7-w}{Ma&YW8v4c>=TaaH+B^m z9zY{ZMcw}0hk2O)4=W9(?H#`j^=bFq8dlr$E+GVmbXuS_G{*6wUlY7bGtDuC8+Bsa zv=ER#CmCs@th0IWt!qe1iHEXTKE)XyPUR&+h}n^e$fb5946&r^`74Bnfu{Yhy;^+z zM&ADK^{bDjj^swYKZ|_x6)z(dj+DbQH8qQu&c}aS<{k=5zE`Wb^*sxom{6i4?NA;y zqFX8k$_X_C48b`+U%D(nrvI>K>Q<1zyi{r~PIfpg0DGeKs)6hpiG%$nX1U+D+;;c7 zMgYw*h8FgYbMY%FCLH?uXuG`QIp0 z2#QFH#mcuKw4^W%As{edv5a9NB(nuFZ_(AKMyS_JMb8sIXwV?{o-TJPhPP&SRn9eq zoam3xyc{^u!?r&XT>cv~Dz=sF>L=G`L)}bF`gpxM8Awa18D3tqe4sk?4B`DVY6N%; z1XL9W&tJSqR(mG92&7NYsya*}?ls<>5G*SzE!%cObAe)RYJ`p(hs1|2AXR1brAPBH z^%S-8qgtdG=N3?r;%NeVmk^El`J1|Mz_VD`upo2S{+)dlq6|G?vecjva(ZC3ph9Dn z=-2CDtN%|p0)i*dG)4dJf=^Zo@yS40yc&F)-JNJ{AJEYRl#R#QzyxH*6Nc|uFfjFHtr-2g zbpFP0%RB4BA~$s&g+p*rC_Ojbd530?-QJ{vpNLmiN1`%(?H%klmD|1LT5zi7m2rmu zs=izyh-)mPzUlnuE}VZAQ@ES7Z4wVCE~kZW2!g{o( zERGlB--bBeIhD>Hg>gD!T0E6ZxX#Fl>A%t|ZwN28dFw~Ts+c$kj0%aF86XiN>en|= zXY6V0@`D6qpk<=*Ani-&V4%67f`Bv|IT;Gy;sB&?tyR)s{-WWviL_>qm=n6tp3uQz z$#1rmkyJmIWRrCr05pGm&)!MQ*B)HN#7v`27FKt*p z2>Dc)JuZf~6wjV|`3!k!PW~hAj8CR7PY0q3VJd~+q#Z$IYZtblDx|!_;+BXD+j{@K zXmxMTxYB!Rvr(&drtA0YbqHR#s%V3$!ztKv)ISBMc9LpvsNYm@sQAh~QJXXczw)jY z>#-2Uo-aywAur?vn!PE!vaZLA{nyQRZ@3O{>YElcSTy+ybfIs?xW8OoHDn3+4SVSV zSc@s3z%Y@vA%iK+0$Z^eCl5acv8Za?AM?QdsB~3=!I?Ylq%VY>8*z^dK!4nUjDCMn zOp&Lc!Pr*Cc%+7ti_1o6CqR~kp2s8VG~u^JOUiKp5ytpOO38Ucaut-3m{@2r8lAj1 zZK^nW^e6@4o!UHdB!hS7N9NKluBIasaup#sjVQaUfj+dTfo+Vjx<#8(kMsvIxf9VP z)8^i!Z8^>&qMainlvQGoXMr#x!WQFRp<^o}a_MVC#U^2z20nvwlM zd@7s7K%g|+51ThWeI!R*aCxIX_a9!)n|rpKTuKI(l*1;E2W;$D8oTr7 z?yi>z=-adB{!qG>&1FtWZdQ;Nk8FT=;R|5RjRl8a$l65|lE8vzT@N5$9P z{pM3)VVnA))KWqY9&n(-;P$b&m<7|wYJXL}%c5#X%=ckfz=e+4q`Hvq)! zzRL{sTe=;sad`O6n!*WuJfG)Z4~!TXqG;8f9l6|d?$aeJ=+r)Yyib>YMOQjo?=Noc zO2dg#GUEq^{5YkVWwkLpwolVBal^QsGj-p(S?8$Dox2J%p(YmSz_M+=yn1Mj*&o}i`~ih$JFZ@k1XvEu3@@~TXhTi*#n=jZ3(G?5HX9h$vZO2 z`&EGc49Tf5)H#8@p`deI{&O)KB_m82v4K%oW9KBog3FE<+j=b!YapgmAAf`% zlI_)yKKX_M5s(l9%%UslK>v=FS8N{kX&m5 zT>wv8hL_HpIg^!hgKT0OtQWYxwHor;E=ENq13rg~b~jYU2T$lXEQ=a*^vr0z!Y>XA z942@Fp541ONZTo$AURc;jM7ujoWNNG#I-+}?r1LXn=5N1m=i_>UNk|Ck&u@5o^UDb zW2@8X(1JrLsnD=X3 z3LBDbZf^VaX?$YlqZ93ETk&A7H8|E0DpjRjom`R`wsio*Uk14$*no`JI<~q<>_%6v z%Q~;{J5K7^(qrY^B+I`>RhTli!gQ5Y&sGRdo6YZtaw`ymRR|BM&$`yu*#479BR-sK zI>7U)i#rF(;;|L%-7BwG2M(a3HK(sP>|zQmM$-yh=S-Uq`?ig9>d*I@+`($NKOT{m zNo02F#CqI+C1BmqOQld5-U#uE4m%v}4*3EbTwun{o0PhWvu45(baL9;+mV$MN+Z|< z8@H_0qOWcY-w2WtR+ax-0+Wf|VCGPB&*!4czIRUA@ByNW$<=N@V=ymm6e=49$~Hh@ z#>TZyVID{0>WELdb7ujFQ9xj}W5su8G~Qze(M2%1nqyS%Vay&bp|E=r`0rxc!F4fU`{9KL4;~aE*`5VLDBb8BJ{Crssw&n&ESplZH>u@7 z`d#O$jIhqloT*4tfZu}!yt~8kgjePKc%o+a-H;x?@EsEM(Rt6PZ?C~E4YXhf9q>zN zcAF8Bg6rlwe0X(r!tpsjj)RdOlj5>a{SBgHJ05>ZZkW)MhFynuEb@vsGbeYcPuZte zlW?~5{b>dKIEHq$!Bg=o@9z!nTJhw#O7FNRnHz5r()xDfbO)h9L-1S8Vz&hziQDvx zURO13esK6UxS+sla;K(j8lRhmhzrdoSc}n;(%C(6{{NeVFy$Cx}i;dKC|xeZ4phnEA1qf}%(}Ov*w!?@9l2-nG%N$-5blb+3NwVnEoU zlr9Ww(h*X*`xb#G#8kgsgm2t$)64T0{%HaHX+6j(pfgP1Qj{Q;9;&L^CvrsiKSGaO zUpfnj1@aCCwLo+`5gd$jJ-X~-Op`&?e~r|S$NjT;#e##dkid^0hgmF75hl*p_%jMA z7wH^wQqk)p`eGT5HOCh}d$`__`?2eo{oUw4Xnvz64$NPGj=o3hmMrl3;Y&R496y-r z(e7CpuKO<8F|L7qW_Wh-$}gP|A|{i7AiCGDZ#Lcd@{D8@%C5t!O7`G4%n8kcr0z@p0PLR8$gm(Sm-JwMQ4XcIo)(y!tB>J=+ak#3frz`>7?j&J|zX*i#K z=zzrY^Evd;)6$XIrsi;~9&g1g_O!!CEJv}EfF+U413Okm4n(J+f~nZXR$u-A-Zngx z+^WIS_S<^d3jU<_TRg62A*9q5L!^`&5lkMo{e_@&wFJoAkM)8+14+4w?3E7s>0fTy zx?v3OFol9srkM~z2xEZB)A5s_eJc?-O_orhohRNHhdC~f{vN{paC8u%U5aps11+UI zAWBi=oCfoZtck}$pfP05Am-O)27#8j*e8zauR0?f3>@CVJ{-7<^2LTZSi5 zjC`)ZYHUn8d;t)ItRwt`_ehJO4P2fjFcuu_?c{z$`>s)dlgS5@80v%pj5h!_adAiR zEfU9oUYQwVkqH4VH4B?|Sg{*&A`}ULL5m9NdCl!X@}F!RPFatvi)|eLIlb#`;+DBD z)>tC+Vwa?eA_nH>_iyzp*t%`IY5#bFE)AD<-MP6&SMB@zV92Wzm0#W%H?WbLFlS@( zD?76zYJe$NAq$rdB4J-y zOtB>W8z3)IJMgv&)TT&NC<5mqDULhPOJNlcWe|N>UG>EaH|-o1iR?f0O6G8d(Ai!T zcE^B10YwJwzlpv;N+(7;UrM$QMb;h0iT~sm{g}5L=IM7S*C6Y`%ZCg1gvbA)UALS{ z=XG>ky8J@_`+NHXnPta@n5Y~zGz5S9n|6?!hAMrRX6B`n`8g=uylHPpNOu zMN~ncUr1p>I>&XiySX)va_;#*v#43y=y+OyiQ39MBYVVUzNCPyP{UnB{H=<-6DsAx@6vNt3#)hRA0UByQIm9=`QI;Rv(mz1mE?a21LX-b{j1zLS7z1m%?l z&ZBu6F1Y{9(**;SaA9aed*kaK{MbrgPAGRW>4o`UCL1}af`NgaWWP%S7FIc-M=DXm2krr1_+375AJ`5>G&D2hv1LQGzs_$^Ff zNfch!(;tk1e2`mXi1dF zR4YBQo|q*Y`y5}%=~-NfPAF6TH}Fe5(k{TI2AxkDh zrAF}*9#guGdoN7z+XuikaDqNI@a<9^&f77%$4jFtTb1{5TvqmU*}wx+KsjZV;t+IxiP2|dnXt?R`9!16pO3w0 z!!^aWgcMfo`nZw)G1-ED%ic*jm%w~@M?S#5-nPDXP_@_x!yMkswr@#WXP++U-7>*8 ze&;pr-ThwSK8xorr^iN*Z&?1%1^ESo_cHt!Gmr~==w8twdb#i#R7McJ5u$!QS^8eP z9eeW4?7f`l$~6>^nRcnG2w9BH0EOPy%>h(;uIXg*su7^&aX0M~(GNn`{7GKkxr-Oa zYbFNv6~S(>i?%*fn2U3P!aJ+&4)a3q7F1viv zc*y2_Ft+qagb*P?AE0 zzzGIMiWwsE3B*c%T24Wur!j<3s4&H5m_LGm?#D-eHYoulaxrKuxs;p{9Q3Jfg*j1J z7&3F})W@uZ-IxTp7_-P=O1LUmMovfhMZ#L477#$&CMbIGf^~(F<@gz%g!&Z}l8#rp zZFvtovXFPg#;grwY9QO2xM;s&PG3(ONM9v4FQ9xhnQ!Kwx_YL`HDONawSwu6jRp>< z-}=6qz(+peJ=-g5?7af>q-FaK*?ci~*X_oZCs3a=#DGe}2x3yk3}W)G3ELwG;T-UTBg!8o zcsxSFHXfG%_U6iila6EjHi}-KgR%fUE?$Vr+xhwVVa+eZu4^XuH213CN%X zZ47H3CT)VMAPX}g+0%OA>m&~HJV44z;#h}c=k37 zs8ArSB8`Mb`zFc&!IjVi`vevchR6QR^WL1GMA?M)XH!IzQVN>-0KWAp3EPjB}C^}R4A*-QJ0gXC4N;Yv)ea<=~q{_KBHxk>Al#8TRG=B)eeKXb^= z%HI$Qb(MIGlCr%!7$f~l<*CX#gs)&GBsY?i)#^7v5lZej>RXo1SI)-uzrZnZxm&(ZR)H0^ z{oGW$TAv*fek7;&?Yr?C>i(!1#Q0Ufw6vl#LyPrjnWR73# z$&;})_7ty`@m31z@a-uY(!We;m-$k5Y>(1^EZr?6BRO zC{))y_o&th~AV%GG70;q*hNSwZ<<8Iyx1P8l=Z(Wfwj5(D( zDI7Z??5^+-Sb4Ye$sJ+z+>Osg;VW{j?}vp*}r zw|xPz?m=2Qq76g3SeI*?hM`lz-PCkQ=?Lu4KyDWBl=)n=INZ83%h3^iS}W?Cih^#5 z_JAD-CCTQBNzRU1|45&%-K!!NzqOcUO!E`HbGK75{2MBwlA&mH2YFcbm7-#>xa`|1 z_}{6|bVoxQW)Dr*x}7_x75$4RHu@Z918_Xin5y{(uAhu)`GD65*fS}e8jE8rk@@gs z)oC?o+3=IYW%ne2ObE2+cZy#=Y)&?DWl*OBO{yaCktLFo-PvPOs=`qs%o(9h5g zKh#^gMx8~p-U9)+FPiVQ51vn)dS*sONkzpASsO6--^7B6r$4_g1aFdn%=H;-A|KH5 zk$04h7AFZC8N&CGg6OC^UpLq@2Kj|$A8)`Vk$=P{fldwCM;cyV5CV4a$ zAuxj%^L~_6l09y=(Q^R7YX1R-BK0cJ-SSOEjy>Wq#Z4(Bo!Ea_ES-36J52eg7Fo>2 z+C*8n=GE-f|R;}seG-%ioTnqjnU|t5qWqiZqhp; zgSCsXbeblcEC506N-!xID(XOmbNkvw704vl0h&k9cNSQgeb|^w5vw%_PC~S5DPRS0 zsGi9p{CT{Ufi}1eBR%UUU@(1;`=cfrE&_1t#hA>|LhjtT8_XV!LC0~qf6Lf9tI))J zyWh{v%AXcBDV`8o`Tl6$_V8B5ps6*#hAtcHI_J1_NQm^*H}BjcxZ62N`CWSb&c}(h zbJx-(!nf$OHFk+;Uc2qYj9i?2m7jW{1)nbz%#F?1`mg7f74Dhmnp{;s`}Q_=NBPDb zsd~QhXZ^1iuE`%lL0S_w%K8?ge*V?;CdW6rPKvrTP zNV*pVD3*+w;hfd|Px=vkDP|>isL_;PaA+Q2q_cS?!Qt%0Q#>>;#SH1wmFaRCXZEA-R&q_{&IWwdf z<#&)Zt;ZZ7Q(lSiX{`2*KX>ljsG{0KnGo4r19DX(M4sEOrjpBZXXAd-Kq&*x4Ch+4 z5S#3*@d~X3@($TqJ_qx{`Yur4yEMizAG)zHhyR_D!WaaIS)gLKalGNhMAtb}OY1FO z-;>wrO&T_ID12U_*0_FXf(A17VKrbdd3Ug#C>7u=8Rc?P`E2(C1;&PEPM1`-!t!+p z-;v7eF23E?g}M0@qzp#!g2hJ%gb{mz%W|f;VRi+Q{K{-ZM!JurQ_lfg`qWF0TL>)!AFxD zZ0FaIc{6v{<8d_4^311=brrS#oOkd0x37QXb~?2TYp!bFI&|3;@RH6IXxykFXDiCu znL9ASY$E#xDmnyL1BL*%`gkCH9o9C85MJ2TS{t||j{bITJ`-Zo-EX|& zTUgK)OubXPwh&l3O@gw8Lo0e-TI@p#Aq2v}A5=(+gCj%|TdFcFuYB+zg4aLX2WhOO zt*!7o!`1LEq`MRKxFuwm&Ra5OAn$T|nY*Xxo-QxX9ogKhq8-Z&Pc%F;Jcjm9y9qO` zLQ#u^a@j-!pDk?OE~aJbKrpm?qHY#<5F3FJIuL(BVuW5^)^?)XL=C%j#2ZSqYuQl0 zog%bI_aM+B{DqNi0rCXiJ(OLpH+@YcCYdTEEWG)h`A9i88*My6h%XZ>Fjb&x+lBs4 zBWe(kezL^?uu$HB|+|B5-FU#99v?rTkj44 zdR`kXg~W|7p9Lv@!2{Wa7BT*_ zQwQZJw+}mo0#Q1kwZ*?^tAeLbI}d+n8C?8j#=H;Q76a(WIGX1*m# z5_*D`za@DyF{=?Wa(Cy8KXB*eD{h0z*G3$zUA=TMV|4uIA*OX~dka0cz=f31hp!KZ zyvzP5`R3Gf-=2}%N%E~1H)OUet$+Z*9723S1zFna+nCl@6n{?%xsV-Fq7Tm+E0h*A z;mAPH`+mg*AR~NdCnw8AiudsfGuAQwy9fSNsgP+&5;;r+uw_oK#hRi~-brjIBXc{( zKVsQPBPr-tDR~CM=L4CZ>j&JL%uT7o>1ut+xF+Ed$HSrZ%yTTmTLfO$?wvAdGc)9y z-G&8W*O;tyZ*ib5r;?V+gmqT3Dvl{qLz1vSfcE>Sb?I$FGAuqDia?Ye^m4m`0S5dD zaZ<))IKaCXXtQkDz*5+9yv*p9J6t=Q?tmT&4nw<@n01nzO5lQ6^8@q)ha~aA<&rBG zF}-1WcR&tYL&Iv8(7F}ZQqdh0Q~vTK-Y1w{j8T)Wl~H>?RN5zO0=+m`(nzvkXt6~4ff zbzTk{p?GWleoboCp{u}qxcHL}#dfTwa1cDPU;&|hoiR}T04^nauHf-&iM-R`=BT&+?`?^hEv%@R$oetJO8wsm zK6B&)l|Ud9R1Fud7RDl6YS6+T@kRw@qv)lXPcLW-1g(Q51qaRY7E_dcA{M(p-&ETG zRX9X1xV_Q*!?FOTK91$|0|Ej%&PM(E`)H7BJOnn#3DV#Kxv#ImP7A{;i?Hd9og`MFNEC)hkTn6DK5*BIf{tVg+nqiZctE5&b$$XV9W(cvVgr+enP zL7aQ`za?)Wm}C31FH!R`nQcPiY}zIK{-{Lzv}~cLx7>4*JRhK6jRj`fqVC8Sda4hwc ztUVtF8Qh`;G^_>^HSa}>!x1g!2b}2#Sx63j8dj1N;sIta8~O6GzBjj!khM_6O8rBP zEyXBA;SY&-Z6;g@{g_Gr`SN=&gaA%f!(o!j3N305fwBlIasrDmQL88ov!)2MwK-~+ zW)~!(r}e&v(E+OMvBG~{B@PM(v407K6_T(MS?QQ0FS$cp2`PQy(RR1hK_JYaicO-e zRM>xaVYH;&3LLxr$#|Uj@7Uh`Fr15D8P}TziUQpR-ElJQKmN$g8sayRQWeQDbnb6+ zdQza2H5497rRkE@y$ZS6aJjKO>pfhPFEN`6CoVo{0^Q4oJ6_hJ>5ljr7i0DnZ;%Ld z#`DE+IiHI;ibQPLFJ8WK6ErN%osKO5q?jrtTkUY3F{0=}syK*Z+RjMIZN|Cevrmp4 zZqP+bPlL8`2HtZvqqY;?kNJrMawj&@R!;Fy92kZVP%KEpy~=ua@bFIg{Lvppo_8AN zUdmeHsNs{Ntw4moVXVo(JW}Wg6$NYBJD~~V6d@nXy;%36Gpn`c{{8z+h<`y&y$z;53dW?-P~4A(LVk?}u;PC7 zeLKT*NB{8F7knyFFB%1KSadXR>$8Fm-JtD;#E@gCa(+YmtDJPAG(9-kR9M+_N<9+v zXzGbX3t9$Z-)z0DE|dPz;w7HJ&4UnU}mJ{gLA^z{}dYMa!awNmM{lylf=GOpcJ^ zblH^pfG>hh6N$h?;r0e>^T2pnqXs`5h1(^Q;#j`Fq2qw#M(nWzbtQLCWlt}9_U!hU z0l6oH&CSIi7hL$-XgBn}=1QTBkTb=Lr1AJZoN^QdvuG4@z`;!HvbD_>8X#I!qfJff zhZS%<0@|2k@G}LHATi>sK zryR}e*RIW|{{jCWWj5XN8)~AAGsa$-_(sar>HV&(wmW?*l3%%xC=ZrUen`0?y|c6K z+p7ekVU?qtEZX`vLng!)OT$@b2Nc9DmoT2QaZ9u-3Q1X`UXIf9N(G8CB4CjOW12^` zO-yQCp1*TNp>#sO%h`EW*AkQx7Q-8kT}ehoI5lI39*4a$WIwUlixEXypCOz3^pw(} zYOw*J%?(n9;Uj!fIAxy?Xl#+C4rTBFtQ=M4r{e8J?WR)3Ib)*YFxY~6H+6=ShuXw$ zI;8nY--t1d!S0`UOA8A&)Mq#>t(dU#6IGFb`LzY0(lW*6yK#@f$FpwMN_8q9}uHNHn+wqT?#^1=&soLoNWR{>131rVME(=auMkVT>s?8aY>M0P^zrg!y~Ww!UlKX31=EuHghXd)d#%;Ov{@oty?(yEY=k9S@|10g-2HEqn3pY=bkR0MPda&uHM=Rfz6?G1pWY^}l z_QH+>MgMYp4{g=Fr(PEacu zZ=JB2s0n+5T^A{GDa9f_IXTtKKwfLlIowhl%PGp=fDCL-YQvVWCd%@XOh`()@0D8k z^l1pKPqV<@VAXN0)8&I!z?1k!L63VE&Bkul&KH$ur%{GJ88tVy4_lB;Ao++}=iw;l zhkZaB^}D}lo_u2W+#aLP*;1P$t2lvj(PH6C=yo}BXg>!AdY(+Zb&#Cf)l6cnse|>$KvRlNshJMg5jQ!DAU(V%zInTv0I_!%QjY;{U_en}Fq*xBvfFi!hUY zhHSYb6d`Mtkf|ZJJ^Pl7vX7cq@*^NTOh^(Q-SSpnzOP0t|k)^0;lYX!B zDl^aT|NGBze2?e(K9kgaU7z!Fp6~5+nmOPa6xE2KJ~MKboeL-9H`bBivhbw4XdlfrC1uyTextXr(vczCV`JiY3zd@%cA1w8A@^Lrxn%&oZufQ z`V|4P-^@hBM3*gpf7t)PEpucIXRaV-*jC!i3-Ecrsj_DVXyQV0cyjyf966bNE#K8U zU1P~1cs5Z#H{}8GE-0q!^5ZHU1!gaLv+Ue?&U`nqkVm+60jFOSJ;7o8_{&aCqCJyE z$K65(uC}3b$H#&(Z_A z&E?!B^R+cJJArSL$eK2jh$-HP?5=Asnt)kt!VYZD$=Uy_@=Rt*Ki#U^*0lAxJf0#z zdE5_vCNp|09X8|Anexp#PD)wpf!>cV84hqYn4gunsT+txxJ3Qr=FRR}*Rp=E?Wx^9 zC0e%t*|9r8`q{CEt6~kCZXE0au5A19bFb|qh^Y;NkSGs`8K@{{L)R9D)T&iY<>jma*Sdx^UVrs`pD>T!%@pfe7cDJb48VZeGhF9gBj5pP%NnA~`&Gwn z2m`?;X4mkE>=6{LQ^^c)lG8hifoEg)V+;!Qc)35}`OKH?0{#?eIQru$YI?FqUnN48 z)&m;^Ikv9nrtkAJBYFX7zU+f2V+4gfcYRfmtadtuIq`%q13%?Eg$IXvmucVquN_w}0Zq8RJ@ z0F!1P6P5+dU|ZfW+irZ^FK^ttXO`(*{&!k3M_e8bx6S@>O4@yF3E7nuah1yx?$?3v z0z~-qJ=bGGv-+AE06FoaXI52fJLkCavGU}cUV3HT?2_8w@VR2yW7~><+>h>?vHbU- zqg793z#XDhoWV!r=v}b>^b+KOz@>fi`4wMz{oyJ%!JkFEjh#;~dekh+FvsM4P91{d z?Og!!BlK4L7-d|t)G$J8CH*N`nWgv`a~dO369#FR2g*m=W*ezm8W%t+d)l$I zj@Z7D3yeZ|8$QSQx4JfwKr-RRRJ-4ex_&p7hUwkrNfzxOejG$)(&B)Qb$yIPw`n9t zVTxv9lZ9;{S{Mt*^OMH5R{Hj^vg&y9R&Ef=&iNS1Jjtvb3-=TBnBvJ8=P*j687f+Z z)|(RFKA3$xi6*lrtCG~e_C9G$;gwMPSDDp%n*LMKTj~sy?N6_bO+&;q^O1TuS-7J z+#Vf}t_6`vQT#{u<~*|Y;t&bnv&Y63AZN84x`x$w%;atgMUA|aO6h{)1+QY4ct~iS9G%4*sE7B`Q77Vrk-6A(R=5$hf6IQ zZ^n6Tc)xtR>uA^+qbL^R*jQn6z3n&CzIRG#b2rz`RsO{JzFa~re{?c224s>;EFT72ZM=Hd5Ab* zwr!=!j&BNF;I7ibOV8iGtgWj#Lb+@Y<*ya2ERgVGPDV@GuwYMYrCOQ~PUvFyav zGRBkjVnGt&v#{6p{o`!hvHgyJMG}{s0;!vGml1ARTJqfgvZY~o<-J+{Np#yi!~c=p z=21jphfka`rGtP}>R>sr4@}842&P={*HO?(J=&HnwOW)B!2dgpRd8v`d$%Yc8}s-n zW*JQ?Ibl~ZwP|Q?b0ayca8f}CiuYY^8C2d)*(B0c0x@QCHM3FehqEr z%eiY=HB@Rg-CHU%ilQ{w6l*^`cJ8aj&+|sr@EPQ?F7ednM2>A%Pv5eDjUv zI79pT9H@v?+nYKs7eBvJC%NaaUh$x`UAJ1TX-Vt(O_L^aTi%Cn3GY=op=Q7*oDXfe zSVnu&wqz|I5a5@R{|a3x9mq?NkFjdd7)HOyOVf0*<@pm^(4}5i;yAOqSw4H zE_@OpD`s*JXE9#J4BtIq`Y7HAWiE9$i!*WJFbm5UG=A}ij-mNQ%PhL!0`l{we4N^# zK`6@mPgq}%rz}wdrtqk2n$W56XYEbDG%1#brGG^aIeT%Mwa%U)%X_z*(X)8n%%i_M zYbaGIyZTSfl@^<=A3fXZI@m%HIiOG-gNIRm=8SU)(Ch~fzVh~gr6EvP)4<}+sA1;w zQ!p{v7Cfe`FFP}Pz*a2Mmx~QnAdh&o?|jM$7G)37AN=7 z124yKd_%yT8v}#@1@cF~>`#sv~2E0kp3*?90 zrBR;cAZ?vs(3X(<$uXWiGh6xeXjDtz;&n+%dXA3f&iDJq$O-DSAQ!FC#lE^vjOmv< zEag}a-`RKkC`0M!tmRc(C7j z((&3(iAA(0l@*zlNo=&5SGw`7TW@E9)=8TfCKuqZRgKBZD_*LXas59>syvN5UKw63 zJ#ypG?pn%EOJX~k9_Z`4d{t}jJ3xV#a+f?lQs*MjU^6*mR+y`4Mg{~o-#9oMc#>;R zadRc@(HZfNHXo&xJIK_TxaD-oav9}@uKj?={4d%wA`QalJpNR(IP%1aaS&TzY zXj%Em<8PGnZfD^<*Sa!z6(Cbzpw&06chNHJ$dQq`BqWS&QFrZrJ6^n}g{4`f^aq2& zhYws?wq!CHB?}e=Ug9U{RO!daybSo8`_-Xf6|-k(-mu{`n1pbt6L+=O!AaD{4H)?FKpIE7uqMjp8!9&`xVzc|wLd2Jhl2V4cznQseIEm`2KNzwFIu zPcYr6h4;J3(?is9_%>tKtao*MPOwx|qxLsYh6QvV-P~%Psfqn95E+29m=R(^-6mHN zHLxSLM(ry2tAdQh*VjL~b~ig|4;^c4<>wXB>~@AbxvAcCty_aJ6~F^`z;)PBJ*{@j zrF{WXEr7t9WW1d%&7UC21K`u@B*`NjOGIgQ`9j=z3rV4MU%Xf@yODq1Jm`hysQ_`@ zL-$U@?3|I{6_9l}8GNuuH5#tjRE387EIM2u*WI3tGmWC2yZh_!^UE_+G_6}-IB3#u zisvQEl4;G>(YFp`p-pvdyt%cOy*EIr+BF*N(NhAyqwt6tB$wcf+4g>}+tD#mI#R_&!_+%TW(Iv~5(lM0& zR1f;;w3aosW!J3>e~+4YXc18CfU=iuq>sryChAU<&s#@(P<1VPa8L053^-Pl{`b^` zU0C8T$<0X`S0McBRAWRcfg*3;MJizu!hirqgjumIee5n))zGu|t6G(S*pW*OvcH=+ zX}O^?dfLmck|RbM^r~maOqc>#l#AeQ7@fm@JDUOs=CQAAG@4GGI>{4oTZQXIg91Yv z$-tSHKb1Y|w^0RK+|^gY>J-NBR1e|&r0dtW0B}%2gNVaAk&@8c@0!WY(6xI**M{rF z$FN3s^s5GK1|PlOF~CMn9pdz1J(}H-F#-91uqTa?p#wep@zG|cyv<|P^r@4B6}N$) zwGJz+jgQ@5$d6+*oU?+oU|$hCw>v_8cgd6_zn|i&duAt*F7{?40G8V-ekWy^SW7^JUZe2X&xRc22q}?}yEX|7vC4%c@fO z&0G!288b+d+_#Torj9bw)cD!jSU`rfbGK5_NSL-h{jigG#yt69H>0Xa_^>i{`)v3g zOzpqDUO~AuL#By#XLo`x2FKB_fF_5!c1iE2*fc5Zdd=%cUHcw6Gw1Ti8*aEvs~(V& zqGw{%S}|=*oI&nS8+C7Vz zl2rQDUh|#S6?&qoY1t!&OTki3K3cSeKe+&o%l5@X*SKI9!f-2=f`;O%ly|QW305?} z>KBG~pEI8?jK8sd@V)|abB9W!S?n1T_0#jgNf_&c4D0;>rgU|W69YQGb9xhzV5Qx^ zZ_Ws|GmSW-0w#xfTIZ!QofA)(J?;3#rvaItKc0>P7&vm7k2`Fyy=iL#8TOXY1kV^V z#$-d2E_Z{zvN`)QKYt6C;v^cL{nN_pm;N+8L9$n%Q-aQ)w;oYj=Vd(SYyIzxb^NpB z*lQ8XD4Ghevwoe6?JehLR3s^T23gKe{3FEJW$||zyAgIxW(i8#f`toDWcV07c^ta7 zH9qZvJ}f(EM$@~QiXARHGS9>wNO<0W+p8$Wb_LaOJkMPwajN4S-c9O?Yx=QC!zJko z2TWKOAd`gNh50s`n=+4sm1lX?@Tr-`5~$nk4BZyUx@z7I2m>aB7A@m%NDRK+zJac< zemMP5<`HR;e7=n1)a-6QT&l%=7WP%4s~Z+W?Ae1P zEOhleV2B>QdR>)<=kdA*ZB#6Al7(9<1aZNh#JPu*x8hg%9$1aymrZfk4kVY*(Ifj? zgTC!T@##piZklWn{JljV2Znw$6MjD>40+7d z->MPe#^tJ3)2C?hU){}`oc)LXQklDCZ$}Gy&9k#iPlDhKlaW~TuNWk~!7a3J*~sHt zlizC`pKcV6sPN^;lCUTJ_Hf06g}stTJN~g#@ma!c6WrG*%R1eOkkM{5XWX&aK;3e-@-EFjzMdP-Gw10`#czUKaKtqh zG;hebMd|FMH&SGE=5>6=Ju5B`en8ZUR@dpMpu+fL(iTTgiz4-oCuXW!c=98{lo3OD zSiJA0jKsmaI1xlcZ2%T%^p*X0T!cX!rH(PMgffB{^{oOzvuT{Q0^&D2ckBJerv>%~ zIeX_K67u{-v7Z6}67qac=`=H?BIV-s)QdND40m!JiG0eFqb2vL&+Zo1KaY+;u--{V zD?$|7=43v}=~5`ZI<)yXv9tbq(yvmlfkD z1K`m*u=MBD@X&7G%TVVx8iqi|Z2_j$yKU{ZUx92|^jkJOR~RNvCvzl^iY6qB8a&<; zlmI_@g!v{hL>7oXqTzbGAaCBj^B*59P=^z>lbnj=&5L1A+XWbt-HIBT-)#;$_DiKH zB}ye3yL~5re$6D7#%tiLq%_pYRn&1V>DQ`K>9=kAaw(Ncu8Zr`-9mb)hloNr*7Yi zb}!*=j12lxD^r`_oaH2_neCO-?Lm6VGZ7=Vx`&Ljbo4+g#Z3O`1 zc9{~T7`2eFoV`s1&f@2WkkK>{AS(LWqT~0S0luiMHlG+chl!(4Vi-G%xg)5S{YWY} z%bJ58gH>LB>NXcDC;o;=Tv!QW*8kk2&XQCe7qR&^v~Eygo4RXlWzm{A9&J3k8`_>) z^r!<(qdJENhK5F%-|;uR7!Vovd5UZ__E-p0Q-FX&3m@z!I%9i%UfKf8O;gI~XOF*m z@s;9x0TkoVq0KZVW5F3Z`Hx~@p-8C&W%661G^`=zu~AidG@4(HpoIB$HntfcXOKB1*X4iVh5PhG$&c>a0hlZLj>zDk)rV^fQ zaNqj15+&vxwl_$il(R>V2I%D3=yt0&@Wq1xdq(lDI`|v+f8d@zleCFkk|oBeec3jueiP!R`z%k@gl#EG!$DX)z0h+&g9Lx3 z>ue{5gk$Hz)FJwFspFlBz3;tU=JDoOLRRoZ$4xUY`pwhU(ln6%vRhSu>9hBeDNQX| z=l@t5-uH+e!W?kM)o-2G;_xAc7jLv|J5Mw&Q2{jh+bTz_t~q+zGz+!q2l170IBz#M z1$D_h21(XclCho_c<&p$F>2w7J8f!kCY0Pfie|$#9%p7%@pSc9QIZQ-lH9RB zOc=Sarw=!+>9!oXjdnUR;0MhW8!a7K5XEpx8S1_;>!KUB%^J?MlykUZxsZG0CtU$z zQF7nXx>OC{{t0dH3Ed8QIfQFXouQ)~N7=DwRx5U~yRxh$>{xYY38!gZ!F4Ll9}JJf zWZGD@C1Dwc$3pJXf(HvClf+U}Zm(zGMJ~3(RTrC+*_YYPc;qsJ?D;e7cxFFq{*GzT z=YG@ME!)hKpAI%4?Hp_CW%npNvd8_m%R<2b(z@3hqpN6cqIFKB;s=RHdRpTVvz?vo zVc^n;(`!{!3=w*BtJQW47TW^Djxe1~^4qr63LXBXEu|$_j(%&}sIa(J;kzD6Nza_c zI}C^FHY7P_%zSVWs@K?Nnw5lftz$>Pm#YmpT&C^JjsG)kx6R3~1k+@;3k(82AZRB! z;1S}$X|s^%YXD;~-UT^UP3!gN@O=lAW;M#q&!1Q(>DnB@xlG)V%x~;d(nbo|BbS^F z@d#dQ%L!RgvZ`=G zVH2U#Nvs5DOGdziGVlONm*CSh*MlEc(zZg5i~jlUOo>Uldt`d@^6wSj4vjY?vEVwJ zP|af*g(d~hA8Zs2>*2M!n%45*GO8ck!D@aWpiFnu);F;a4)rro3={JX#WY-JY5H;y z*YI!B(1}*8j)ju3)5D&TcC*XkK^*r`W+cX?j$%Jz99lB%F-ggZ1<2&X`W6oUzKRN; z0N!<5W<6vW^l6%kt^Tgm&#?`kd?tU|Z0X{~-P2Q*jLi(wT+Z6-UUk2*`N1-BGg5#Z zR!R@cdPYT_F{-Kn9+Go9Z+LMoWHcK1i`g(us?%j4W zeHS#JV&{zIPWt8%5tc?Mm4a5(gL`v)g1r#w|$rtJgPoj7&sD%4sw z|EI;1m6Y#Y+%I{6lu7%6XssM<2qo-KdC>jx&(DgE<$-z6ygsn< zymetpclFvd@v-}dIWEGdx;go!x;cq(9Yu5Uy-`>VP4tKF4evjEe4yXpH22;u1ndh> zN5I!q(R-3dnp~L0Z7V}tS&}BdU9q4C(lN;-w`l!kgH<=3rI*BdVpz5U?b%!z!^w+J zp8coM8T87eW~qEaG6aB6No@mP?T-UK>#RR4NOO64c{ZpMy{3H!(5|ubx2MkTlm3;a zE4$LUs78C{nB*$YA}P=j$;ru=XCI#@1(U0$(~D(IEGF%dk$}bP!4HUIsIBW{L#X5% znA2QjPJVB6>o?eh`!}40k`0BLsH0rE&Wz)d8M{(@9bTA&l6tc46CIwbxtL{~=gg6M zZ1V|d+ExX-ggpm2w-s<{mHGr#!9!rqppFF zA;(Pzf_P!g8D`N~-qVpxZz0?JHLupAT;L|1Ocq|+IheQ|eV5L^k$FfWD*N^Ad&zRr z+`5YWkGOHuzSi$Oao|1B>Fg_a0mfn%ATaOR>S5lJPkIeeeSOR5{N<(_hqpSIb-OcY zlK@xVEs&nJ)>#jGtdIpPqMJ2`4_Mk;4fRDvXvt!9S;^X*Ux8KT7e@cAB+cZ(q}{m_ zUSz!L(@)pd>V0{9Y#$BMSDqa>bZD4peZuE3)U)#V@Mb;Z8{phkkT#&I+~Lm>uuTZw zymIBMJ^KhpmrjQ1a-P=j!cpGWaJINaq1Wy|Q=Z)p%Jq*Ad$AQ*TS@2Gn75?uEg(8Kp)2e7V^=*BJWH(}zHd*sb+`TRPxS*j7&f@S_+lb%&p;bK%$SQPr zuWRd@b}c*_#%ngr^dH?uijBK8tlxb5t$7R%3^2u(w~M>R4Ke#Bkb__6m3rv%PP{)~ zYB0j^50-g`$FhQ)DJvH^c3!*sqg_q!o}Ah0-JTTNv=rOhgd*4rTO; z0R)>t5cJAQI(+ivohak^=-dJcI1w3&jb}g4qQt!K~v4oAAsVN}LD*#NSgY(q{032~{|@gA$Pg z3mJv^zRyq3e$76HKr311cn@yi=tVW+MZfD%yGq<}srj5q{Qb(M5er-=T`RRcJ4@mC zzOI*a=kdEoyVvy->m!;#Gl{X{HsZ2LA_94M#KosaB_U8=Q{&%LXKzNG5x$%clM$u) z?P5Vkkn)3$Jo{R)`Rk{?JCPIw2{Rtl?j$$fB!g|CR$XXZhFBVno3AoI@0?Hq1f3=4 zKB}~hvNiPjkJXwmN3hfIkELD=?DMW#2h0JTS~#hu;xky(#IG>bCHLv37?~Bix)!>! zPT7@f7CPHC9Z{16NAVxhx;z70mND6p3y_zfek&Oxq@W_ke_AUj_%D!0+lx*J{wBNvBlK?1hc}nSA^(<54*YwQ>-=3= z#L{h6L4R8802tKJ@<={v4a38!h4)*F_(BpcR5a*YpK&0$_^k#@j&Q%c2)0|9NQS(N z)bCbxZT=CGVBDoDlHmQNKeqALD$is#Sb1_Hedf9ik0zEI)h+Si;Ia?IJO*LflwM18 z-@O(xmC+V7aU3!RXG<8@ru~*3PJ2ium&>hptB>Aa7-AB`;OG0X3~0JTVY@+9P0Q@# z_rs_}xn%CUkB+;}Op)TX5PS-VvK}$u<04i-ur>cjFTS#yp{<<4U8NcASv-%$0l(Ml z+PYzT87qL-RMimIBKaD~PF;?E1kh8jKYh)&2v4PZ^vuc3Q5rsE-3mPAAXONGB25VN zxhXF`u|2UZ$C)Fx&X-MHi2xVDyAqy@`p2)(EMAc}5y5J(BPkNXB3-&L%GZSjnn9n1 z?_3^N-m-W~m=9P|G+1id)bl@gt%O)mnt}d`PuOiz%MNA4!__G*7Un@#)a+o-K*2p= zvTRmpHduSHYgj0MAmC^3vbucVzl!j_WRpGS++Wv6hgClX><$}7)W?sZnK<&CADvxV z@aFh+_vw1^EfIm)bl~Cf$9;bOf;;z|{^qOXY-2xHUHcJ;ilT3~j4f~U(-1*(J&Wds zvxU6P&AH>45Bh|zZ3pTcbw*zaSW^9H`_s@5#MZng*5AWE(qmm0MTtm!yR73XqGcpLgkR%Sv*Ua&DwOe$a+jYjJ>AFW(zmDxA%L`9`t_Si%F~hd z^?iO-sa0-$zFGW}*Lis%pu*#FFa|#K6n>etaOO?myYAJSPxW@Z%9F zI*RXg&fK|2a^|kgK0f){`mJjtbU1Ol(^IZp3&CQ-IFa$SC@%->_Wv+AzW%(qb4S0T zJ=2P=m>wnUG!%AKrQ-6`t%wC@9Q$F-@A!nc|9Y}SXX=rb6RJee@frcT5S(n>xKTThZ!T;> z(_Ra=?sqmC4wW92G3OoLN%uND4;#JR*V|`$6{3yi3#@Ta*lybg<1Y!lSHZV~z4__S zv!jN`>u)&En8Raw1WR2aa8!^Z|4!w`-Q9ZtK7aMgg*^+|I4*-I4f)ksRD52<&f=V~ zDDR}iSAwYcED`>Pf8IN#%TZKpUiDeo(JZ}e9~Yj@wCS(?uZB)Fk$w~8+(pWzL@rRo zDM5jwLK_ZIfT{p;X~;?TY5<;_7W{6z@H6%{h*BOiFPLO0{W)a^#o zR2e1?hTOb7I0z-pxAfi>r@p590QJKcEn1Y*&OwPg7!xx+0s(E+IPk=At5phe*O0xx zCL?yR0r%s%$J;jE@aV|Skq3YwhsnkjB1yGDf22-_=CQ#Ln8m(v{{8mJyQ@DdGnbM6~np+ zg3v34h-f*Vl7>(skoY3=rUsjN{ijd=y9mimJ8H~bQ95W=LYqn{9c}-}_ank|W|W@J zeb`soMK&D4w3Jou-d$%=Lg(^GB;3OwU6Ks%xf!(9*&UKl(%u7%%R5@CcfJsD zCGR21)H##8i>)g;x~G!+cSOX{h#?!BwPpV2aUOU%?-5($dFSG!D7dLxjQhiadhFsm&`UW;Xq1hS`1rU24xTYj>cZ zFnE90Zq#m)U+27sr?0zTZ0<8-g22kyx8N*-97`J_C32KTsm=1d$r081 zlnZ#c+K3D}?fCtLQEVScVu4xESN1Ilf!ZR}I*86*CVR#A1yY76neZ47v$6UG@Yna> zYx$Cq;MJ@7;cNsCS9R2UdOdQ!^qyF$H-pr9**~s4kncy-(HfV^->5YngyUU){h{vA z#(dhQv7)AcL0553=BvqsqkwY;?##^JBD>9_2!vgK`L>5}q@r(1a$QwE^(StihPw8> z1T-624VoSOf_C=hr?PEXA(KNu-=F{9ZE{g>^!V}Nx3SC#(mP_FPZj|~M$81`JwXM9 zRI{`ce+z}DLku~fll;Eve^5#&yT4imrPn}77V3J#&>*^j$>WsmD;z-y#lsKv7lwp3 z{oW2GDG?|O5K?_`z?gJxAYi$n*FALYhv$sgm0HL~f>4%BM7bpGzNH~2u#?hxlG4fWquzmNN~-Qa za7OOHld4Mde`Y{~B=p;zUfi~`6NEC9C?P+)bz4xNFmjLeWEQWwgJu4sZH_H@_O*yF zDyf;x0?W7UyLnLVGNezlJl?P=IuJ|%Y4?N-JjjZsPj$m8@5&4Se*wU`j7B6WrpHN- zO-AOIccy)sn1eQ2Jr2O!IM3cdGg(^DzSXqC1?MmKUvGxIRHt#w4n_`EdSm*$uJ@>7 z{h598qPH7aUYtM2sh68S<;!JVphqI4Op@T`B5d#?0?xPKLC#FIeJMDr4h^#ht3yN4 z7=A23Wg7YssdO*TM<$%ydoQuxy0voY#on6TLU9_fdmf+%rc4|fq#Z=<>uF4XL+(>leV4i)7 zR@b*;+=o~WcJ%fha1p(m(gVJzRTkA-o6mna^=5CgdNLUj*21)V+*)iarRbPDjy%vk z4}ZTBV2y>SBWS9FaJ7yb*ndmyDP-xFXe{FQ{aypN&{KPr+lWdUzLP5%~k{s{&YujqUduq zF)MjLgsm|Z6&7Hq2Rr(r{2mLcsrO`;zOGIwX?H=76F0SSL8>Y6wKIIYcIWQhU0CF# z-feT|&kqD}2xlRC`|JIScgRZ~;d9&l=|44+qko}l3FKv@4|^#}qse(2$*bz~;nOIKh-5uiz53tAb zwc#=TI@^`Bu~TQ*;rEfNjRwu0jNr8L&*k~yal6+HijN_~A~+>wIEcw9%uL+`LUUOA zok<_JC-FWrP79t%f&}vYx+P0R!1AxgC|6Zr>aAAdZs}<1(h55uw&|5-nTdLSAngTh59EPl4JNj4b6-`JZ87cC5Pv$j!ffMo&8w5ky)?GqV77yktNK*u|xSC4H!% z)<_e+J^ip{dU4*%m#Rr*$dI*Zq$&l7kdhvwQF<}Y*`|OByP4>L;kox0_F~fPpFUJ4 zgCpaN>MY8maJIlW+3$%AvKQLiymM!iDK;H&)A}r_7O?w%Q5D^Ow{l2aUM*$Mo#MA% zb?aUNPD2u8colgy^HSjNn47S_nDsZK-|EW2C(j+rlj?tb-x&wJTMc&}R zi+-CV6v@w(9!u{u)R4<&qPtpi@i%|>nmBp$lO6{JO}XS%I9ml%DhTH_`v59s(cDHy zF}UJ2ZPqPa(|(0Mdr6x>g>D*#5Kp)s&z!|Om4;iOTVFX@CVDi}jKnS*)ECVhy{GEl z%f$#ZY~;?q;VRi#b&7K1^NTy46n}w!{B$NLBTe9vBqY%!&H!o5b#zw_6bSXUydREr55fMocqiXt>BXJQBP2Lea+ zZow#a%3f3ytGE?iFuI7gVKAjc@aAYjva*lK{0U<;kp+oQ%D(Z@5i3?T-SgxN-Ba4~ z&CB$bI!eQTioJ~MCdvU@vxY%cd@NlX1XzoSd0F}O11so=gezy*RSep}$&kGh{1{pz z{LrBVHjz^a(GqpDo@amIVjsSpz~_I^@Z+FQ0_M~EyQZUv0}koYPS@x5EMx0z(rnu5 z7_9a9c^gT~hGT04#Nr=n+i<;E|HA01`Z&q6#f#M?bEWGibEel>STtW0L-rPp&rR>Q zb0j*~KNyj#8WK(@yUxz_W4n;c^D+dt$#m}0SiVFx5#(9&?rAq1zjERfifj1Uvr}@} zEq;|axP%XeP5~-g$C*|!F=iUy+}E!+Gx()P-4#mlZsd%}H(h;i|0MTKPk)Y_cXu9x zp$T&+lJfvw<)Ymv7?O>ReM@(-uyGS?>9ol!+PCh8sIru0ua|l*md|138A`nJq;xJ_jayZ*94E(+5c*ge{ zvO)FO?b3+AA^7HM(d?0rRO}pkwU4(Z9wl6U*X<&r|MQu(X&$wr$LX%P$rd7Wkbs_Bm&ut zQMq#lX=y&1qWNllyv=R z1bwJtKW>$01>QivG8agufORh<1*H}FtFD&`e3c=wAzAclD(5S*Jt-s|=*@bsm*@B{`PCx*Uw_)BylZ1?H{Vr1 zugv;W6hX7UJ^EZ|eu%n=xcLso5M#kj#7?|^@Z@p2ioFEL7_(t7To&F(!*J=v%|?8v zS_Eh|_2Pi*)~BG<^IWaGIO3;e8g|Hp#Xz-&FWl!FuEO94V$IMstP4fn^v}Df71|s6 z{CZ28%$&jC+(Q7q)`w4Vuvbn??8ZoP)vQ}w~H!$f{MpWhqgUdnkvf zUg351bw%f)q6)K!sC7R1 zy2jQAi=>WI4@lLp>B|SEefz=lgTBtY4PYy85EJJHhlHq3Z%U>iX|J6TXlngeH&Fsp zd79a5%+^X!U;)4}LtfYs#JTN9?iejQd4D*4Ji7xkG0JJ_KC>Wew5#qNomoKSjhda{ zyzVr58ZP&-o=hU$Nb?!7p68GTelJ=fd#FcgEjK3jPr;-j-tkG-T5VW&*fC}bYTdTI zw*Pn|Q&`rde7HlwDb3Yrf%M2NK$S>UOgHc<$x(L}%5lTxpahXsa@~f2V2k)1f22bL z3TXUqRdN-BU(RLo{S@n?gIC#?wVoEn!uvbTyJj687}jsFgK89S@y8dV_}t<8rZthq zpsAVgFKp6VEO<<*Dw?(E@%d7lND!hq80oWC%`x>vED-xL&g4lH%lN~!LlG+bZWVm8_r!ZM75L^h@#zeZq`YHtVU zD_n0{I&zS36q30_laA(p$u*^WC!1e*6-Kv`@5NMaZ|Z*T6c88KrNLi1CF5nhEn7}vJkunYcemI=P@;g z3*ge8zj9c^uhK zmnN{)qIade>vM@95&gSJ0K(>X^^SI{H*x;Lh2LCsy3kV5M9YRP-j7AnkDy6wRq`R_ z;tDh>#w91I4NLx2)8;0f5B0D#3c7M-uqee&GB1bsYE|gtOe-pK=`X1%WANsxYS!xX z204etrCw9qm55gy8^o)kC3id!c-VeV zW0lVlsD8A~n#o%uL$?G4;Y?79BO@cZ^RgNx_m7RMcp)#(LQ_tz7`;d2d}ub zVx%B{W6B>clxs2lH0|Raw#&(=Ay~=(<4z9S4J%cOASKulmG5CT9;4a;;ePD-x|5WN zfzZ9)ppnU}5HJwap%Tyxkpo(X%TdMg;(aEu^ajtZ8(W78P$VUZlM8icNJ5iNQ%oac zK5hfsU0`2R(^5LmylUf*aZt5jEGc)&D!U^^gO?<*xo=5y`_}2@rN6gWNBaXYcz$|w zKNA9e!1t9gFrkNtwrX4cjB(ke(R-O5W-6wJg{8yRUc9h>fK5Zewr>_6k1wYK zJA5LtGt98!Mv@1W9DTT2(cY+*&6M}0UfQ=y`l|e>kpku1I_}pJRSS*6V3HpbV&3l3tyi9SV63w2S$Gr*=77r8 zef8C)MK5~Q*W4FfN*)~{+uqIV1WbW4mHgRUGU7r~VFHEVCJ~b~~yDe8<-c-wGo6D~4$um*{Zs2TfYqSxxx#W8+At3^L zp?F|Q4rn2GbK)2Gh~X|V;i1|}llck_*rVbFYys-9v`4d(hFo@<@sa^hSN}||(?J$~ zrWWLno;dLUtxj0o!mg?8+@ixgQiVIm^Z20lY$H-%Sm~J76qjV^+5>t&TVkOM$y}Fn zlDSF~i-sgynrpF;tA1eJfJHPaVrOO8pCpD+N_X&M_NfW~GR#;>3KxE7ZJp-J61eCC z=HeS^t=burdGb}_uT%au=eMT69JRgI#*^Vxo8`pUmEz-GkUJ1~k{;{vbWIbB;>~=uQu$h%OV#iI6p!Iy z3!eF0rh>iPr6bueU@BqI))Tn=RXG@)`{4wAOH-+b%M+?J4fTF$scIWCPM1wJw5;K? zVoYYzi)W=tFUW5`w5h3be6%g0g(~v1jD^-6%9|gp`-B# zUAu2P$L!21@rYbfGw`U4o@}vE73@~Tl&Ksj!vVrt?c4W17wPNlnfv39!J-ZlfjeP~ z>D+n_T@5d~|9u|B8IJwT5I$ne9i7s3AXCPuE|PIJo|cs#6O)sl$6=EGPV8p&27Ve1 zlBiyJVw&eJJBb``Snn3G(;tEm1F^^2@CfGZXQb?QcqLL}-&4jhNH}GULlV`mlmFk- zq8<3(+brw1?-#!GT{1AGvjpZKCHyQ7V@VaJ4fzw%hL|NhD7@jPPp21@psrA_K)gfq zX+x&VAuGQdBEiai(V8 ztA@L=83$t z#{n2XOv(bhO6?ER3pgT&toVH%Qsr;DuKV6w4Sr5Ymvtu>SCAb z#DyvNZ2It`n=ciIh(eJ{GJ{hNv6%d45s}`ej6%g9`Vxq%eRT6?*Bf&oPfeMwt0EJN zzDO=5-l2C&oyE#KP2Lh~G?&c+mG@%^^1lrnyqWB8s@F^B0j|04f3`@!rej(UV)S;- z(_AHQh8XEq#5kIpncYxHZ+bPgsT+ae6p0JcW1l71ud}UAm0I=o*^9NgBh-)972rZF zwft-6TVfTb5~|K}%48nzV76*KE+#{Le<*qRJWXz+t|FP=KXHzbJYhl_1?5b zijT<>_QR_JV$)$ir)sJo^vy{V(8IeWU z2|J6Qr+16GasapS(FtC*Ff;a%`=~XLYO?dm(KENLissh0IV(>sqgB=1eWulLXL)pP z*B9xxPP^Gk(uo4KD^}SpGH9))OHmRnHW)2``=G7gOomv~Q-UJnFhnHcg12rmy^kp1 zgB@}7fw(^1HDeZbzN(JnqBpE+4H?4qbQYzhxqu_d`x8E~s-ZNsDx`Lv*9+W9g<+`a z7*jRD&AszqY=e;@0Dz2&`%P`;ze5c8dd0sLRG_fiOe87@HBSlaSeU;6)39_KX$ol- zmVO?I1-#i!f75xgi-stA@U^3ZDdot!$wOpEaCjWHD+x^7HQJ_GE<*>xg$^;*M0m-! zpk@V->QiE^LV|)M`liqFklC~fZH`TPvrJOEK>n&cTK;sF_sL>UlLnO|TCe0}J!5M# z5hd8|dlmbL-YP_%-rzr4^m$Uu$8CczIdY4T@uW99qcI93HrxDLAG^hn@%K;6+w7A1 zYodR7;3uS;A+!>|apnW+DpES@iP3<6^45Ntx}PtZGkUqcVz2$+zq@~-|R?t*_#i@0usjD+$ zsHU|9N(sB?Y}7}wR(M!y_kYRXF9Ndnq}m=l^4GJ6si$%pvJ=G9qJM0!m%CU5(AjLa z73Pmps46!CIV7lLGE{ZZuet%FNhbHI_>eC1`Vf+|Ec_*c+HqCIQn@#HeBQ!{|7=-B z{J2Hln=Epa`2JlDZC|`O$n-yoWY z!*H1vY=oA|#|USaGc%5#Pgm+?PHLd#2~r)2Ych{PZz`~tN2vdtoaZeA zzDl`xhn7?pzSP#U*?@hLmmg{L;%!Rc1|bPZQ_-8ClW`8XEPNN0|K8h-4z-o|(Xt=T zflbGW5-v(W&NZ3iy2m%Vk-(Ya{YMJ(P%Op(_i3!Y>o#8;!0uDzbeKnE zl1i$E2ZWzL|7rD%?hs8#^(^40qpUxoaZVi8CB_uotlpYj-D#HVKUR8ag|Kh@cjvH6eB2TN>^ zog3Idae7+sa@Lep(@bXhP z$kqD2l!#QX*luhDb|3JKjXRRUkd5I|E_>Lxzvh;rqKX-3zsm9)q*?po3{EyUrJS&= zJoOQT`4=SsDV-|YnTkPMLJU!(Bwh9TQDy_s1D>7RoYgq($>&1G*niTl6?uJB6*?2b zJXhD#ha;h-r8QCx!QG4JEqe!O*F{y~ym|9J^QKE=C1I6PY-hiTAnvpS@=PchQLGIM z-@>N^jOG0#{Y@2HI}t=4xt>*A>Bhs;&rNuqqh}ts$j4Rm>o3ll(Ck4g6rZVCGeu(3T@yGSxo=GThG<%=1TgxLTjftj~5o1km>fQ%4Vo8bg&9^5Yv-<~S^<-#66 z(~_aW&A|aksJL)*rr*M{R&jHe{JD?W_C$W0Ji5OHJVB@4^n5MTK1{0Ib-7f&{xf|{ zm(!uRRBYhOC32ZQ$;MOCfx3A;M3GQElwXsfeEX^udPdD4`Ttd8vFU(x4V!Li-a1GP zGy9V!Rb=(?X|Lr8t2rrVnmQ`nERPA4NiY(pAHX1q0t;F3_&C!Na#DOpAM|}E&}(RS zhK8D}$xu9H&B{+j6&YTx1p-v|>sP+o@i_Da1i`}l?f)SP2P(cj2_T|Tjx>TT*@p!hreTif3uxNU286vWge?_{)IZNf37#V`8YqCW2wbfVIf%y1pnM2GM znM0o)h24*SqZs_M#?tp99|t5=ud!Ml)PofZQm&a$YJvGv*8ksmfO`=~iztn>_CZ>HE}W zk`v(npm}Yd3(m$Cvy7+Idh%Eh<`3xD)l^s^br^|5xcECrjF8%T)v`I;z4FxTgB|xI zNTOtAe{ik9%-2fZ_RRd#Uc1n|&ri}-;s|JJQDC>)@inU4l29|oW6MHw>;BY^5?iD4izEAoOgfUdq-AZ zi-BBaoY*p~rbun;`FiznPNb#+`FkQd$EpY`Fq9^uPNX$)gVK=JBzxEV29iOT#5FfZ z+GB7Zsgqk|7&q{%?oMhfzo`y(hmMRM*SE?L;;uDD(fhdyezo7r9azC!` zhz5n`D55Cu#2zF3bS`LF!R@s9@ckim2IpVG<_`QVPr2Ob3JjMn zg%|JHevvT4a%ygSVNG4DDe>igGuNS&#ZmWIc_xM=Hs}U90(z-gDiq7Z0$Fls!uITu zasDWL)n>q^4l#nZ;s^G$!be)b+>>-{Q=7+vYInwD8%4 zR0pDwl1&Ioz+gaB7gb6PJb_@a!X^?9-%M;E*%`Mv!C9kLm2HNCcv}4sv21H4On|lU zfs(9FI1+mFCE!RBk(pSXw;7TE7T!@Jy6^ky+k7W#X2TZMXv){s&a8%+9s_N=>Nr_W zd-<$ziRG74c@0>F$v~*Iyb-1xRUXTR-Auli%3o&lCbjfUzj4UF)wS}YLH$+Ykx~Qp=-HZfD>O7iE=TKfXSrSoXg5-lOl_*(qCRM%Q*lfhtLd%;ki!hKiizVz(RY zgK-wb1YRl^G0re>s=Tt(92pf`sd|nglud1JHuI%uVkG~eC@=fm*JzqxB_kd!I?b@= zC7?8YO0cYcrGM~Cqf?e7^MrOMZRT6nYgeS2kB1({nHoVhWEaWN5@5^`BaQ3iRc9F$ z=vi%8eKXaQ0qItNzWoeQ8gl=3jDcF6boYgBd`!0)jk*Z8qpu8?PdK*ZJ1m&tMWmdW zsV~G=_M^lxAxHa5;B4hSmNN@TGE5x(AzT)9)(4o0HeKB9g2oGyYg5)6W24GF^+yfb zjRk7uo-LUd2y)oOTGFX`#oN%goG*KI2!zM#$@%}k4l`tBk`h*vKg2eQ0LhpF{wlXJ;NY<;*v!Dw^oUS$(0Tj~@1z>C*`4@C81m ziL&fNg^PPAnvex=U>otJqv%Z=-J@A>%YMtuqy{EeI8u};W9qFF8X2nR(-{n8GuheU zR-iYNaX_MNE9;B7Ud9Mnbs)NBIv<}z>R=XG94Ct=A|atovu#zY_C6{D@`gQ<$dYo_ z30kbTm|}S}9b<^z07gk&88Z^MpJ>Zu=aR=^ZirqB@5L`yMPFNxvEa1Y#PkzBkYv43 zHbF~?RI!}_W%jCxWbK{k(LnK{eky&G?>#59D_F2xah5+XWK zTh`JClwEM6kd)EQ^3;U?=BXiS%+3T@AIT%BDwl=d*tcay)~;krSM4=t9@h#xOJq)X zuiXjy0p}%RHsOb^RZdC!0bf>!%yDeQkNj*O&8GkaMG|dG0i)%w|DTvB?KT*DmW&)Fc!VH`QAa&^)ku(Gf19$XgFiGyQj zv@N&2wi0haFwFa0?HW5L_VWnOnw^zp=}bk{T;mXvjS&~JV$*KP^1XH~_C-2Y_ARoL z0JO{11F?cjLD8*Ii~r#3m0~K2?EO(YOT2A~+j=fIGf>ao7DQN$YJlR*KE`iy6-91m zIOwcG9UBMP^<-%L|2=aLFL&aFiQuQ{m7vI;T&lz!7T{7WiiBua6-X8LqFi=ku|2QRlOZD4 z!{8Xw!lv&lP%KpqhnP3H+O8UO11aEb`fm#G=4CO(9HPyaCn$lZs_^R~1d7G$YjaYzn%vqy;@pdL zP`ZMaQEy7D%(34P{*#g2$ zu0Axm)kvjA6^(aCC~oG^;Ky`2pxJozDS8`t!Txv8*O=~n< zA5X*~xf*hbZsv0ZUXy-=`MJuQMFL&2gNv&1E?jsEucq07zw>Nof*5d-Xk3YYEJ*@ywq39nCrfo75qcEk@Q zl<}oewA{nFKXWBC^}D?{x_sW~amIDXkRj8esFfS2mGcYTG%v*(DWFqYch_xomC>PA z`HhbmzxEk7%lPbU{7R5NTjSL9t2#~xE2!!2NE5=X_<<&q zCBqfxxq(>&$%)FuR@GgbzH)UTQJ<*78iHM*jLa_4@3ZRjc~xuOY5Li5J4PPZ55}oP zT<4vD80aViN+p3AL2ur?S>d@6LPgE^Dm}s`()KLb`^#lUN51}vqa=nw?FR3n;l-Hu zII7YJN1BlzHp{E@Gv-*ZPFfG3o|WPYxe{|r36w28PVcHU43S?FHh!8erVvcf5o@le z59%oeP%;%5ObA&EYhOQLPs-ODx;UOsT-9Fj2?D1Lm(^9crT5P8imiT5?*BfgJu6<$ z`fqM4T+veTzP$5~Svj*>2Fy=MzW_hhwndI-8`5WAw4dI&iEr@gZ)SaVePBJjW&BRs zoeoQ>l2BQLxeS%GAz!9#;ZJam@=3kx{_j2|%|^jLPPu*B^{Yni`mL%^<#)r>e#Bw-Orx`H zND->v#7NN$DDi6PrTQD$94G64@iz`fqS@eF#f!23z#G$Y;}1dl(UndN9n)WF{jkWj zTzhZBx^IvMeSiw?Wc7x!ks|CsWm?w;o+9p;xbZV{5w2$w_@isq8&iSFzL=u^F$- zm$^Is`geN+vJ2@xnS;ar4_n_IPj%n_e`ICPB0?yH%HAV6MzWL2Oh_479eWgIW`$Hn zRyh))?46aB#t~%`imXIr{a)|Wb=}|l`+fYn|G2OFzREd$-kqCFV86B7nTc4~p4-U=g55$p#)I-~b6lDJHiN3P0FL zS8q|{rT3j}|Nc&x0EmJS!J~V;32<5$K9Qb)%l6Ms5lnt@UY^!}xPUM;_ue-D>z;!4 z2AW+t3b0LQL2XUOprx&aJN>`FR0} zfOh`OAq)0}hZOOI2MuJH<)#^QTW?ywwp@q*#snF8%A(WbuDq$Z4#CA9Fijm-klo!^ zdva0%>`RCm5Fm&H#t4-Y5DAew{|zwQg4<{Iv$9^e-XiWwO9{G1_`Nzk8~0kEQE`wj zhKT~Aa~-aSqLMH@c`!Z-4i#C4fv6EGe^eGCOcFM^FrJvU`Tkcb_YmbUqD$q!VmVmA z@)a>)U19eo(L|96v(=57HQLZU6C&_MDn^3}egRX0C}#rA$B}8LvA>!RZ9r4r08nd9 z9Kk+@!kcIkCX4}w!+*DGhg{J9o`&5`)h>?vK*L84HdPfQe0)$c;LinjKuGYK@W;Uz)*eRX_uGMA)as1}I7(vh)sbicik>H`5X2DJZ@+sM zlRuN^M6nB-UKf0Jn-Lv080@wYZ5bGNjbjKMX`%d!j%+cbPVFYa0g}ELEx7Ape&>s@ z?)&Svkeh4X^x(#w>}uQGH00p5u((*^9L11ZgQNp0z=+1?1aOmJj12%ML93mu_FY+VOUT10cecOX&Z7N?1ADU?7r{% zBVRern*VAi2oNV*x>fj+_N+g83PbSI5SgI9U{fGd}uOzkM}*mlU)#BlSiQHZN2V5;{#99Cm-ga z&=x(B4gB+~CPNjC;E_)9Emd>`8fOxnpOg>6=`lQqlZTQt=ND}8Id7}Kkq!Z??dr%ge z?W6;G0s^L+Emu@xAF8U}GC2o|8v<&?V7WP{b1K26fLL!hsCZNTjfwQ(&Cgv(7P3*< zp9eY*Xdj3Pt{r(I`T;K-QJ@hr0y@KgTrG1W(F7|?Jv?ayh?Z!b0n%6g0LNhB{O((@Rs>4RF)$`F{(~^FB1;j} z-y+_SpA|gs+;AA$b`qV&pkqY9BvN$1Qc;9w*g^k**l=C8(9xpxAbMzJ6!!e`Vut$=3YrY0+y~+p<#AB}K1vJ)RFGE|0YE6|_rcUY{w8$5D7k-^Mw0=Aj>Leb z9uWOt)MSho&HnB^y58@{5D4_yYA91kuAHR(Eu!UKk;>3j) zH?u@A%wZc9uZT`V+NAK{`>lhIA#wp)we##;fz;DI@j6Iq^*`$aMvTz1%X#{Cw2f%vL zPfqbg{kOb$643=h^8mdhhCjF*I|Vbi6Cm*cd3QNl=eZ5Q*9LI5-Y5Wd7()xT)G0`n z^IJVC0@9==@(+edPL>-O$Ral=w9w*yI|`%0e?C4&+DusEBeDPFtjz+f4Pkry5`G9~ z;r7cOU`(wvs9z$%L>^<7{HhuJYr%jF1XrBfMv*O${fs{#rST}w|0sn8xvOC4@1Mt~VI##COpGK;~?iFf~1 zG@@zgO1uoFqZq-GJu}BhfQ_r3W}1_@ao@pvYU3A#4x%7gX_L@4RnQ|+ZI;7g&6{_far`c zh=B&W=rLl4(TZ52!5Xp}0{Q|)$Lk3g2;6sKgX%iK73%niL`a?+!0t)1f{_bC@MDp073ovM_6uHzqlmUGs@2;lTnUETO*4=TWDc zt*EKP0o^dF*}roE`4c*9M1MU4CKC3fYmjzMd)f+>yGBFw=4QGs6~rnW6}&c-o0 zG~^iSL!@Bn+>j1O9$*kGGI_iw@SG?SF}eXE)*4h?HDqG)zfMwDm0lS2l2sL=u||y2CCt zm6aEEe&wl_ETB@K6b;MQK0DmsJ_G7WAwW+MXSo*U_r5df-rzdgCK3HlrbT|2Hp9|4 zlMn$YLiHcyk!5rE@g8UsH?A$EZ@d=594dMB+g*(W6Lw0G&291u@JKFK>;}w=z~~Dy zCUjWL;XPMg(mH|u4IFB~DI3&*$jVQs4(KIa5RF5ccyO-xUjHWz-=e4c!#O-QB>J6liWN9Kps*8JM0wFT^a*+!jHiUgYRHX zizug1rN>7Rvl&D}L9$}#>*VHsuevzkI4t|OVhiK(e@9V~{51Xq}Ql-g3rO=x&zftyQtHXB~H?`?VrY4-24doTqJEF!xe60_>pT{W>)TYPLh*!xlAs?S(@8 z0vf5=AN|)-4aOpW+2sIq{x$5qsVKP{3&!NTkLxz+-wEIe4C6}Tj@yX#oj5^ZNMfJ8m~_uo8gl zatXx#m4Tb(nO1xWmw=1$K+=K!F!Sl~_A^-crEGrXiU2qSAHyeVAFkZp*&H7q=QTEg z?*}mbGH_QWq0JG9tE#S^C~4raxFN9TP_mBr)pv)~(%p>( z`S`Lku0Y#mD8w38YE|40q#Qc?$y_SvW55hy{`nF-3xy<51Y|xp&c{aGHj{uqIq>;2 z)ol0hU0p1}{J>)K1Lu2woRfCW&_ciaeuGa#9Xo#%(y!s#>gv=5^K9N+4~S21mA=AtR1?y9=5?TQp+9VQ5$;>2b3le3)>||0<2XPyAyB+3 z3fKr_d!=M#(AU*3@0dN*4THNG#SV{uKXkGK;P95IpJjLnCN?3Pu@Txzesv>uwA*08 zWMgZ~4~L8;d<1iW%P>p8=jSsK7`PNz(NxVxU!tJAd_Sz0WFp+TeLK^-A&5@{oe<$= zYfH-^85vVUS)Zj#v|I`%hG38o2O>URm|7d2Na-qp#cZIdDQdPwXOW#z{QNmB*ldTH zigbT31rh))yV%KGd<35V+YJR^GBH~RZ%ae6(&gq+36MiONtWZVggLPWE+(THn;=Ev zc}fWEQ^fG`&DH@u?@}o{{1eAdC#g%b!hG3pvp$lNOLLSoe|rxu3VW(7t_#hr`XORs zOGHF(U)myY9fK;%0?Kj^<{w^fx#FU7-&Cvy>NCiw-Nq&!^y_^ ztVs`-s^{)?*zILtU?V_9k{&pR#WskIgdbR(ge%b4cv9_Rp(#Jyp4_Vuc;y4u(Z80L znT|5^&C(t}ep~?8*3lQ~SF^tEqxQM4Z{Nts2s%G3K@qE=LDdOL?!EN%`;T()@pawp z&FV>aU@;oBto3;>R$kr&Mg1iQ2Xy%eXW?C=>fG7cfp-x7`8o|tOD@=u6$O~7+BUci zjDP=r6t0GysG=e#kmf_G0Z@xzrGt~d0j2pm)Tc(E48t?f)3*ViPH#b3(2;T9K6?p+ zULgBwM1{dCjCUw__UtuSNRR^q!Rtb$8!tTHtm8+HkfTF^lAqiKv=_@@E5Ro!`uyy~ z-viYvG_{4!9=UuF0RG+&S#jS{VZPbD;DE9Y{CdU%2d+slWEK`4ga4r-YFW)%US7_p zp^ZJvJzc@iH>=d`s)b%9Y@1ICDFg~8CMs6^tIOdhv4jG*n!yriS#>GwVFZ+_tF1xc zB&VcQKc2$b=~u@Ma2c~WdZ}V%0ZdQi^oIJK9uwa+C6GupGd=BPdKG>umrlM3b*FH$ zPOkPF4*20A9^mu}Ri8%Wod~?h`?*ztO|i+qgGL53-@aY-*bZr@!YYR@mud6x@Fei@ zX1_g@(l!3=n-CuAbC#nV*+oZrdDRT*>FD<9avQ*pp9g9i4_wau1lXaIv1!dWdk>D$ z(!v750DcCB%o2ACT*l_+qJdEt$->CAdiC11Yat$&E@f&bFdDgffJHda5ouN-ZQwkVgu%I27 zO})Jaxp0UOmCK>m%mft8umbK^Q{VwGscwW89jxR;uFfnh!~%)>U?2>2k}gCNPyOTE z>OhsJAdq&+R!J#n8^H-dBPuFF;gqf}E!ww{SQ3k2WMsSvb3Z;MC5zi{;3iQ_UxaIC zF|@#pPR!`6aS7=mzU&b_z67AIMRQRwF-aS}jwR#E)<0*_HGf?zRmoSv+1Z&42brD` zJ;&y@HVXJKM#2F>Z#cu*YcS*z1F%*Np4~yP-^UOHjjndCx2eMUE_4DV9|cD~jZ>eN zR#)%D#l>a5^|^8g*LKG&QlCgvpc0mK#5qiee zEAaol>FL>r9#3ie5Iw-{GX0((XB!1-732(_Q2 zWkf`LNii$bgr+a^xR8)`+NB75KUjuRpsEPJvbNR+I$ms4L`3)dt5TKVnU{GT7~t_h zcSm7T`5atUK-#W)oT#d(kVSNX#Hy%>M+*LHqCg}h$H6?4Q4r!7UpDt~8JYd!;^HB& z>rD(40<@US{yBFTG5#1RAyr7%e^=x>TJN*9+&;drFmi1fJ$<8IQ1^+qw71jDDk&+w z2-xt5Ejv8ROK_A9`tx1Td9b|4&Y}#<*90hFiQlT@Nc%}_9rwy06BF(@x`60YpALu+ z!#Q@(5A8#6tgQ#-K8_zecu))|s%ng(u)Xy!1XfR+6gaU_U& zj*p+81f!y&65=;`dJ0~uYLO?#M};^If=*k)IJ607Gi11ul9H_S;^F64-O=(8OQ>VF4o}i~vh?d0EBx?T(YC}`NJUZ1lt$;1;^Pkq4SxQdMa(-W zc&2BkrPit$79XR2rR1OFJP5)Y%+1C1G+?zLV7zs61 zRpaLy{v>w?fF?PVZ@M-o*n}=??DhPO8R!U zwvt214rqcTls2oNxc)x}=ac*>A_6N2ZlUzIA|f7LqC5_Ha>A>^Q#VbH2?A|MHRQKw z7Fr`VRFe=B)sZ6_kf@+y2{+Df0l^*_4g%>ZKmt2jtbyF6y9Bk8atKO%F@isy7CC?) zTL@>oGzZ<~%a>8vryVf8X5H4m6X_S4nR!U93N9Psgpol-!-tao+OJ=?EsnGOJ;aWW z&;g)WI0uiGF^Q6nqov~y`tWy9B4+i;#9mA3uww^Ob_Bf17oUFoZfIq79Pf$4?Gt3r z=EfHHPd45n4NQ>D{^%)3o_`EY`&afnbdgKx+S)`z1xhBN{)6;#9WoCvASKssb z`1(e&?qj-mIqfUBV@AyEz>_Ze<);h^pgXy_ht=wGa`?fkh+I%m5Q6B!1mr(_L4jn_ z5oRiYzIm;Pr>Q&u9BxBP%VT(u5Rx3^;>t2DC;?U`82o>>tAYY>5;AJ|sjP3$4T(WC zyz9-IM(9Gxaq!>(7rhxC9?|@=9drTS7mEKaeU|QBh-wUWsao$?X(K)R6X}?2K&h=%89ySO}>m5KP=dc(c6{ zpiL6uw9Y%6ozWq8%M3`RaMy6+K&=&MugJ>ELhk^UKyy`!11%RZWaH+hhrCU^n_vn+ zenTNWGxl_6-erK)$EIOHfE@vhbWI-oDuxXW2QxA< z{wY;GTie?B1Ozk<**G|&!2dm~w3HbJEA4=+=)!6&@8iG?Wynvu<^!*11U?W3zn1}t ze&={c-U29eH{p|_;k7DD0Y)jZ3PCO+9d*<)GBTCFNBrHvz~W)|ls1--Do(6bZU3oN z9i#Yj&<~7wlW9>HdOvV4oR3aH@#v0mJEv-p5@bCxTtY$u)N}+KyP&&{uI}v&(lhV{ z!e5q@G=aAr84g%zLPA1fr(f79QHjH&^0sqw5)mpe5l-OC`XrX23v&)Gn3AK1LLdju z2X$xKt6P4OlV6?NItC-~{^Io%-zO*ejSpi94=pS#$fiJ};l77=_A**$!d}6b(#3S} zpy*}v`YD^k(EwML90y!y$kTk;_n~^E>h!2OxVE;2zN934zywGbLKI-TJvR%ZQ8@TH zs{?ZQpS;)}9U0jKJvSNbBDJ~zP9*4fJ)xr{54;0Ui(Sd*1-k4;1}sn+dIaDNg_ho_ zGc*kjn&;y2it@NwRw<5{EK%scVA~}VF+t5fgzr=myc?vJ4(ANju91n!ZD7Dq15GZZ z-Cp9rLc?wlM7$D7bz)JY+l$g8tk{E> zy{ru)oE~#NuuJrytca+_BL0Km{}jC^`k@+w4vxb1g{0*EbMr(h4EmBLX|}< z%tv1?-on#-fYRA^DxCJf>rqHcWvO_)5K<)=3MT6;H-m!iZS`j*cLMxze%V0(I@Geiu%a_fd_zK}w54&+W3H?sO$M36tQ1~?49ltp? zcBva|4p6J8jPG*9eeC;o!8Vg;H~diWJm4GoL94gabmH#avbJ_xPt?a{>Qa3-N^>H)B2 zGLORrL%CF8l5-D4`)Yo4dZ{%B?pJd|hrP7qrZx-%Z}N_d@OH332wmRbG;(ETWj!p^ zPaqruIRTvOw^#`O>N)TC&ISYqqJB#u5o(pwmJmeuCNSc-IASPJKSQ_^78XVPbhKgc zE|7@ShHq1`VBrtEQD~caTU1muL3WCu4!oTSVg&-v^72^YEfnMG)Eph3We(*T@Whfp zw1%Mn0(RTM!&2J@V~z3o`C|w>Nds&R=pa<6eoR$Ll;n{hjWDv=U3z&f#gG4I~!A8H+HG z3MtLY6Fhb5)Zep0{YUBYx+-632BayNE$T7P0-S2y0xz(vHY#CANl7)LZb$f)0tk-K z14G%%EqfE)tFQzd;w~9_d3vg!Jb6>DiIK@Dj+uui4yHw^1fbustB2F?%;h11ZB zJr6vMo*-)UetVf(TE;+qOEsIMb0i=j;02hgTrGR;YYx@Y5)`t{A3wg7Y)0ipw$&Iw zXL3NF&eYF0xeTjF(iX$;sc@&k(oYn)gyCQ#l-Le0MHF;>9=iNY3Hu(50%COV-)^1m zzapbXdeM|7Jo*LUs_l6bfThDAsb?OSkYM(!vx1tf#UZUb|HTVN=)3-6AHH@H2he^P zJh-hfp>r@gd-fwSGL|Wto0}&pKnOf_m|#|4Uk~lxKEeTmD>?w9u`kxh+0d+dwXr;c zRLP&cKm?iA{-Q0+q0qTyI;YvC<0~TL;@FZ%_v{%P8#_dd#LDKdm*Dzn{V{y1n9Uwk z-@r?8F%JwH4P{g+1o-BT2nak<=63V;CI&$|DGoYffug3ShSI9Csk5^v8d2a^e*J2J z%j7#BijRhz^!GBr&)Vu?W_r(c=Je?%SXZDs&H6~$Uj`9qj7{Js0s6G0d=G*Nh|N*z z15id6{Hsoai+JYS-d+YlgAA!_icp`UsU<+8KZuB81U>~ikH!pTf3p8wRK8Ego?72{^PuI~K{x%VhNX^_CYFJ8|jXGUN~uGJ~U^RT#K+5_^~) zvRI16@pib}WH^|;Y3t}PL;PRf*xXDVx|9kX*WVQ81byr=fwN-z%(`&GsC@kQboRp_ zoH1!~Vq$%R$*9-_&;=n_^9C8ssQ*%0E{ySd?0yFD&Hh2Sg8g4Og-@Ja&`4IlRG*Og zZPo9vfB-d24_~7T6M>(Jj*4PVnggHW*@R#6$TyNh3D_k$Pz zQ|H^aZ2)QOy;T>~dvqJDEF)O=;j{oUi-?T{6bBG2_=JB-MoB2E#^E=IC_o(x)nrMsR4i(%&ThQ+2Gc*C9rx$Pli|^b>r1dmy3xO;6YO zSIw9s$sp*4RNuZ$hMo>|>CkBaaB#Ie#{J$U_?s2&__g5KNSol$WHz79SeKc7IfUQ}MJ*1@MQ!pQcOo9?lS@Lq3mbJAt{JZ};B@nnl znQUz6pmhHZe8n$X@XDUvgO+7I_$A29gp&ps4xuIrAgF{EVRjmXsL^l?YQr#=eNY<{ zp5@^1=mU*xQ7}wFk{XlE%)V2XJ8AD|$fR-^(htqTd93=ML0MK^F7|46Dt&9Y< zbgCX-wTE#1mM~Ws?gJVTnLQ^bCt4m#Y+@t)p2FV&%^?k;3D#{(A#Jqu9=-#LBI;@D zL-4xuIWjnwaRB11iEdjEbVls3loGXO=3pd_HqH*z`bvV9QkVNNU{u`xKg4}}eEyzO z+yA*jYzTGcV;-IDqXcp8D%Fpt28+>??L4gV*%{Zi|D;}cIh~pHyuU&CvJ||9@7T1@ zz@G7~?*=T(pUi7JSn!WCUm7>llg$PlU#!35;+ig=v>Rk!=F@b8^rE{)Wu*wy_x=^5 zdzO)39O-^3C}T2v56RZP*x*j!mpZXwOkeb)D&}q~xsV7xtornj`W3QEeGLw&qGumi z#nKG2#CsfJ&^R!KFDwexKihEkgV8P3^nd}ohmU@mSq%BoP&r5>JT|62cZY)mH(9>C zp?7AC&CxeAv-&Ch(epnI@0;J5{b?pwd35|!JJYt-#;K<-0!YoDXj0Z$F+^f7ulj$m znw@#Tv_c=KAMgnp49n>$;zhKPPB= zZOx;yMg+#Cvag$!c@uFvJ%JJrX`$(BhTVpC_^zHFjer0p3I=Y`tEtK7ZkvOvNG7kU(@IuIZ=YU{Y+R|KcgF~sYxCceQxPzsg_~9l%M>83!~0>j(a@826i zCWgkl9fltWUjtgogTNuV(_&C{pav@%fOU1$D6`X*g@uNaQqXbS1`J;vReIvEZ;~ko zu3$i0r0s*uC_hDb{pA!Db82gc48it=`oxJ7@b9xfp&`mv7>U?X4xpxpr8s@gReIZ zwh(G?IUK^^J3_g68wwkMwsfbt=wPYC}x2b&n>$*E~8PDKps;sf%gMiKB1*vr2=F#rwZp+jc(Mepx-J=H3tL z+6teVtep-ZDmOk*;bQb1>@)4-T{Yv@@f0>XTNL)8gW~PT+TN(8TXoTe1LIV+HCIJ+ zJZFm&-Ud06Etn1Z+FE!0EQsG{emm;5qe;rAQ`bd86H}CHLoLQwCMX`(;x=-hbvY)a znuNT5bpwICVo>g^95Px-Ah!{6u$tEuT6=%gvgDtZZcqfAsp{rLuT*W zhc7fy&q`{3+qyrHqN+)@h3#V z?v6D_XF`IAn60KNw9k&#pQZx z!_Gf+Pqcsgbuz}3!N2Mn<#j8oea9_3Tc4cq4cqnfbivQ)R6`={*Q8j3N-KE zljEJn1F7k7?XYgPy*OBzSw^OGf4{X+R$GR0MTx&Rb1c^xEwx)yzX?5hkh|mhlSOAUrh`1UcBHaZU|yFFF&05?AhC|?%v)>h6~0>@3G(Kd_tu-yva;~&ZZ6b6Se5Cm)!LOoD0y?W*v6*cOC8p*=U?=S zY+XH_%S`*|Yg~Ro7@v>xV=uoW^9A}(nmQJ3-EgkV5IIE4J%1TqeseeI!k_8G1)oyg zUt-!DVl}tE>%JU4gNg4v%r5tyN0OUH%kZ{A^xX5pXX-;8v@Pk{?eo!9cwwW9&Q3GV zq3^vJS_tRVj|JVYyQQDk?%=e>KdE^ubEm>TrKxA}?c%v(C-7~QU0$coefC+*aN?n? ztzX>PpAu+G@GHoeT)o2mQfB7U&UVLom-Sm1dmMZjS5r9vJ)_vxyzANV=}*<$O^T%+s2bo9 z=<`^e_e+ejhPnpg5cTX23ul`dJ6j<|UuwmomBce8lf@2MP9I9H+6sn;rZO{M+jcsq zAWYVEb@tW3lH*}vYA0G89H@SVSG>%3#v~Ct0r97nz3mLeR4LM?XHBC&a$aWimLQD_ zlH#$X3u;uk8ebKgJCbromftph;_#ySJQ9L_4r z%*3P|*O)90)@vK7>o``F0}coqn?LDIAGI)OVg;EKL&gGiX^yShQH&|RLXmwosArPVPE;c8f4itxr9h9 z?gJsk^^ZpFDY)NEdECBz8@L;v(BvQv&8OM-LGd$lK*>i8Fe<%!dq`K?Bh3o;OFaB8 zPlC##EE8(V>SreRT@Pjz2_8O729Pkms7QtY&?OyE$M%aO0)~Cyzh_-O?6-go_|EJPzp8xMze%q!F*(`(Uc3(|p>)llbHfs1p9 zgE&L2>jA{X4GezN($mwqz?1!&gexZ#&Ju9+fifpH03l)qEB1lKqDo3+bTmEqsfdBM zD$wljdH!U?l@bUN;0icHJJnYZ-qe@8*}w3-V5UN`;TVbkh$4fFq`RZELd)Y$?1v3! z(IDKxwi66BlQVtSa05FjlJBnlIwrgS;x+vvAWbj-=(kNo2|1eo<5E)NBUy8WJGIdt|$`X*`OiK$D2k9#u@7nHV^ zCu;~>3_ia5#8uz1WImy{czhtGKdWDQcl+J7xzWchk`Eu0D;!*T%;Qo5$~~@;dM8OK z)2~{4Za(sm)_09)no>S}VXgf%yMoBOXH$NUJX0s*-E%UVKka^0&CY!GeD1|+MN41V zM#+j}+5TQ-Z?d^h!&l1z6g>vS1BIYJN2$y!Mq)Y=x6K08rNukhk?>_c1N(@b1>v!U_SKuKQ%)Zuo%&?=)PUy*fwQ+nAYFO@~M! z!@)01W@(X=Ws+d_8ZOAGcHH$z&YrlPnd4*{>k;e;xa^?2l3Vj%+DYPqZm=kBaqI*~)JoI;5R#Nn#5?mfWqh(}i z=7e9NS!sN6aeM5WP56f)VGGpU7Is4I^fl;sqJ#jacM&~nY*+S#@oo48Dqt{=CxB z11TqpLmofQ`Mi3qOaX85`ZOOUts6}GN%qp=fGe+TB20IENT_9PK2k{ZISGc9t-x(} zyQTy;HwQ1!Wxn#GYj+1VWG!+0Z= z%EkIyOUor-mMAad5~XEqoSy|qv?AlaRYoP{9tKUvckdhWj+sv^|N0o7&7Vh@%;4P}1Y!cXmi-Ahpe#wgJGuk~ zg>-s=02zs70r<*9{9ccUmCVo6`EO1y(j4Y)7|oC@p{0?Ah!ofAY#G)iVBm ziS6cb)9MZU6Mn(2b^q{t1qJofUgY&YKM(16aujzd_z&{m@UZqFmv4UidH*a2U6iCq z`~mZbyPV`l)4ox%SyK*t5y^aJ$glc@^;-W=Ktw)uf8j{K z5KOA2+8}^m;hDJgt|M@Kb`Dd=E!2~bef?0>uD!oO)lb=NB9-$+Nuup17XI#{^LZcD zR?ZwRp;!n^e;1s@P#a=#Y0;5JuW*0W=_3*HkHVf=|9sY$)OUGuu6_AkjilHO)6A-K zzb_p5oGiE1Ai`C?!>MJ=#QAV!dTMyntez}jW5cMj1BPc&DKL%(Ei~G??Cb`$w-PXe@%PK8C_AA`Dawx3( zTmLirG^Kg5nTH9cbSDRXPM=?7t7&V?aQgS~BWi=+>`J@!og z_@VRsq9YhiAc3>gd7dIMtEiAD(iGgJUJ_uS>3gpEb{deR)sAt}J{gXlo077k!{yVF*bq>lprJnI<^%Qvmu)#!ql327mOwtW#cf{()mjD( z4Qb45umKp@+^GH8c|k3;V&u`?6sxF~;GMI(8Kd==t1a>XNeX0H8r80kl=gwx24V#<6P~CYI?Zpz#_?pU8IIU!)QBg6j4nz ziDg$&u?!ZaWGXuWQ3Oal%EmSo+fGL9}$lf!rg9;H1 zc?ihcH>C709}9ODB%>1Y`ya@A3jCN?oZ&vsmkr}PHF>cY-g;N`uw0V>S_%BdfMS#u zsTv?wI>vxS4OJ)bPreOS6cdMF6;JvcOJJgzxS|hu8*KQXh8~VjLv#3|pP$0t;SQKR zcPKwX_yxfW8e3?D%h!FhwfBN>^sTKJ@g*07?=?_rPxH#GFf$0K=zFQDnFWRT9Fqa? z+X6LtC2V4Cm5GhjPYVz4{xD~~PmwJ9__ftpn+tV*4_e!69D{0V2agP_^sa=J4rxZ& zyXvX6Y>|X`oeh>|y;yQ*ec_3%&rk>t>y`&yuJq04${#r2(+eMl?CXPWtakHgn7Q^10Uivb}ih|!A4!&!t zz-aSv&!O{g0HOv>dqY z--D9x4*aecXKI@KbHU=NSlz+?wbAl*IZV>D={U5vwDu2AMQ!d%kn>N|iCt6tU1d~R zy(m+@yFvTv(I2_kKwmB<8j8l@(VyNQKN;N;heq;Sjru!>_?pi$(N^5yb&J-Gt?A$s zlx`X~A1oi5nV{n4HmAJeyAUuVN|5Hj-1H7+`>{v2W_s1oE$dp3RKrzYg zT=|QncTP1a7yGW;sA|#vHsq77pe7-C%sJcKtSlZ>JGJk82GdD zYwlgSzBSIkvOMf94yHEh_=1mFx!61VMAl%Osg>H}>HswQ{W+Q&mwsL4NmEI`<2h4W zXjQJvB^Q4e{4z0^Tesd`DAygDrvp@LuKz?MV2&;Hq{X!o`v)sV_h7&fpKYmIlF3kb zicbq}&E*$6-_rheqpuwG`hPSY$zC%$_vOKh_xz0)TeP2_w=|`xt1oeO8R~QKDb!1G z|Lt8WT-6c7pa16WQK=*dp{;~-Pf6`_jj8U}%5VdMaA5X2_i2qQ&RLo1%U(WTwRunv z?Y_R`!OFJFQQ@i@$rD`52_aCf!0N20lG2Z*g_@7z*X{*Jr&k08#SXl5I7-2A{h;;> zPY33xD97${aKQALj9Jz`?aRy)AjZ-h?tjsuqONX_b$~by1?JYn>kPf)9Sf5l&{jB)$W(EgfEw(jrMn)b9i=7(D#N#PW)Vi@^ zGEF*{AMbWw|L9lla;C90I;P`yMvxzwm9PEt(~9{fA#o~??)a{RJQ7WfIFsU`_eG{J zp&@U3^KATMYZoD~Ti{>**<~wwU2D3V>UA}4g$y?u%;Vqg{)+91@imDmCm^j5+dZ5L z#Ak4Kc^D{q;zV&`4ve@Qt$`2vH15lnBJXBq>E8DCb~mnqTg$mWfJBn4nIr!2U8~Qi zCV}(xVdlihn7e}gg4so<3;MD_sckrelr2)&!=MD*1JMF>9HqGIj3VHC!LXCauKhbj z`<$JfO=b@y1!#s5X~t*{_)dZWH{9KU!k@j6S6$5k^sEpC7+@lO2x1^s0H=?Lb%{qZ zFb(I!KrV5P3j)=JTAz}0p5%-Pm0$~iMs9cjG{zQT%4eog+>Zz}+Pb=|1VuTyOJGm! z=HY1mQ*3q^8e;&_1@NA3ZjF`h-YN#D8e{4MTe&6}{fern!2C;TeJu1vS=l^sWC8pE z*h|$4`ZXdykq5Bdc2i~?(l0=L%mriWt3Mya#Ki1{AG1ntDcvyq{zQ?wc|ZH$a6{_N zhz-*2@>QORNG;VzbPsD)&L?UXHfQZxx~zCE`fQ5Mns8tmvs2n%x9N|(W}c8+dVEIc zE$CPW6oi-71wF-seA#o2yKL8fSUQVuWbyV)@64)xVe1WQV0YS4q-?%I302ueXGLc( zMs7dXu*~B>3%-4-i}!k3_|az5`t8Y_qTL;+A#CRRX_%CX3|@Bmu6eduI%&xa3j8so zmMV)`ad_(=B&FFcxm!neGa))e970maKsH0(0~e9x)8e*zazQl?^7|VU=mA4db)m!n z$TsvF8qAW+Ef|Z6QB%ligQxJLe&!0@gNGb2zwCr8k7~uA`6H-~f z^fc=r%h}t-4fUU$E$Hc~A=|UV?nCy-$he%!^?o_O8{{MYt_o0Ty__w>-Tpn=V7I>6 zAs8-O(*r68dI9}k%`a>+FgKx(a~^ny`^n7=Sj+6A^E?n=@Ll+4#*^#Iw03qTna`i^ zudk&en8uCXOD@LfZLQ%x(^frqz#{qjgk zDHxx!k8__4VB%iDY0aI>my+|B;kduXyhZJ zMNaAK?>oq~Z!eX5^W6g^hQf^wfQwh4%Df!!?UdFRG z{OsCTvkyICO>Py|o-&rJ78&Q6CRrpY@@74A(Q2P@(R7wl0|#Ju$5qz8$qsP>zUuwT z%2UFY#wgy*bd)!Tau0tk)77jT^|N^zBZbUg^X_MrULcRrA%#9fuKFkw;{ugz!&=Qo zZT1U}{QjRt8P+k?W^@*ta)86~$oURjw9q`4uh}`J#Cj!3z2MX9_rvttTANaXM+e_m zugtxvI~S5>-YV2v9Cqqxc&c^a5AQGT)>eiwzrF>)JC<4gjDGESx!dC0ZODl6gBKQ9 zU1OkVK%*oLH99Gr+4>#dOYgA3YMkksuIuIki?y*u6W!G4?_7$#^F>FRRY%?;;2;dC zs2BIIZ7Ss4I`XNl(52ME{C(>#T=V>e=BH z#^=9p{QQ|>WMqw9wVoj)xe zFtQ%2Jbe5(JwSp`BcrgZkWsb0zakL33cLyTl`E5asrmW&@8(u@38x6HVEC+7&__KB zoGZjcY56)a5dstj_0Mvj6^slV(hhaDwS`4S!mL8!q?#JZy?gh_5W#j#`Y_+DDG}3z z6lFAVXf_rk*Nz`%kUOsS6Jwe)j7(+4VETYKe3nQv1hh{BLsQc*KwPP2fkcJs+S_H= zVJwJdh)NRRk|E|jki+1%fs0rqOZI%<$E_a`rgfv@bLKG<(?1aRac^Ofdmx+ zUo*A)GYJMN62pSP<(1x)FGrr1>*$KMZLG+@d0TWlCx`CO8t(P0Yk@Fe2=cu0BsXbd zqSfN-)H%+dQFp?sZ6>g)Z|vqJ^=tV(@-ME)j~*>Ei$0qtk#OHMwzk%BV6e-*SDAPI z{$o&fA)Q-Ag`Qoen}uJ?uVQfZS#B<~MRihmcQ@#TDN4G76O{=+eFzZ}z;I;l^L-mHLs?KlLRoJV9*uzz;K6IXcS6jLz43ncw2zd&H z^v6f&zp1uWe710FE1isXvoR0JsvWx{zL(w0p&^ChEI0R;XKSGXe{9FdREMHEDKJoK z^PQ+-RWY4b740fM1+9FN-T71Jvcj;!Fg8D?v?q;`ae@4f%Ur#!k@;jo;)Ok#D%PJ# zgHtKOy}npGwC>~BxetRT|Ha1rDF?hq4;}ooX#Xu}qj2glYpRQbQ*mSMj8=tk#+`-( zq2EF~!e9oJCn3I>A7NUUqlUY9%(&=lwrDX2PM8kxXC|o+o3ShMfMMGWiJmmlB!711 z+`CWTFU?&gjl&48GpyYVb(&L6?PT7!whjBWPhJ;kR(92GDci1C6ngvgy7il;Cy%d_ zU}`BVFMr8tj=!N&GyKW*Id8-8d28vn%GaDknu~*^TP7fVeSh-?FP7B*{J^Kn+ysY= z$VN&Yi?;beSSAU6=zJLgAAZVNpE%)PySqaxVx=|{8F{p&#CCO)GLq~mc_`zX-o{QYEf5|_H!pRHw8M_`joy7q%Vdkqpy-PEvP(^a>cPF| z&N(7nJYds>R#;>XWVX_v$`4keNfg*HuX_s^JDNVcaJN0EbH=Hs{DAxi&fWEPsqS=6 z!o;5|-2-=_HXy;osVJ~Q`vk)o4%b?D>d^qtkIqBm;~DFV!49@xUSSF*DQ>39(k#uW zK5niL?0|^>z=c2MjJSOi5bn8dQF8Bn={KAz+dgR!tPt8_t#IIx#|@LQ7}>VT)&BOf z0bkdJZJ}f5#-F6{mH2w@F%h;IVy1d=Swje{gKNg#=f2b~{4|m7oxL<3o?>KmJnbtL zMnLylC=9k7&gE@IvK&}nZYMja^dRJ`T^ALLvx_}IolWb_JOUSROfLV$)1t<#|9I)N zKBIc#ky1aRptM8=9V_a;2N%0LwW8;RmAUx=$TPeP4XUqt=A2&uB8Q$E=F*>E^>RU5 zOV`HLAK^CB3=xdJJ3!wSYM>W$Tb|}7Ho)CbH8#Ya~pr8Y45>t z7oqdXc*bo{;miCV)CVlN+a+^YE$ zi-pR{tHGNa?r-MDy`y9gEuu=!P}+>2vCX_H(FpZW+xV_!Q>$e~X1=uKw`)4qqJf@# z76E4c*IWu0{wU~_iY$#x%!mCkM!p_{pH(E(6pvGXi{IU z(EPD{asRUx&hk~>Ha5~hTjC^&-U8#m1d49I zk73+D9ouis6RyR><=7YWl)vBQ?RaxOsJkym$E5pYibkRrhn0ahIYlbVQ!L;76 zx{r?>=&oqf^(is$9LqopD*$6|eh)j4Vs{M%MMv62%pMSQ0VC>@r%zJ}_6g2Tz%m%( zPU5UD8j&$NA0^~YN^C{U{2c% z0c&TauTNiO!K8m(2^4mTDgq!2PkQu7SjgYc?{!m_?Lw$vc7{Pg-{rw_7sLG9zo%ho zCIgre5@kV!bNzQ#Zjw<_Hn%?6zIPJv%wF$_7AgzaY)B*%drax9udgGbus2jTL_7e7 z(093qt8=Xo*l70|xP?qCEVjlc613fVm4CZgaH66TR|#k)pSZY@;mr5%k@j8p)r4JC z1VQ&*W}_|&tJ=@L5li%L>FT0y*jd%C-IykOV`uzQ{+^jJP@8HNtr`IUd(%}QGk9g} zq%5pwygKluZ36f~0&!ha9)N?DVZU~mX-0LGmllHXxKdT zxX%htT`0NC2+ga&(C~csbb(_e5I`zMM_T?LO=lSuh5CMN3_zp=6p&OT1f;uFP?Scb zySrOTML?uf6r@X}OS+{7kW@Mc7`ld#uJ@kvd;cF?%XM6b31*(>j=it#gVvX+i#rv? znwd2Yp)jBfMV@=meBqcME%|bhSMQP|V7{>gOm^kw-(LfD^-`)2Ak2yg1X9+%yNU#I z;XAg~AOYE1MU=2AHc}vc`RE}PuXGi}R|-C32n8H z^Ciy@uTT3OydCp8J*C+mX;ea(Mg{5Wp<9U2ErZmzvZ-g(1uK#9sThB z7r%DQt_RR8>hP|1G6jEj_&KUO@BQL-qZxu$yUpZ3Wm@@hyTUR4d{mm3QxxK(u zyWctpUHg)`X~z1J*3QJX#)5m;IF(~Z(juJjs2D<<5iMPIvpRZ*C+5_@+iu_5vSxNP z-&xR`6i)om>Z;9Ljhpa!;u(+0Iac>8ldF>_qS|&N;GsuNMn$+LhwD)i7_#pWYUkDD zJD)4y3Nd|&6y)H{aQpcNge|v%_Nv2k6FQ;Ug9dnO75(9K{skP&3w-WQim zB7Z#27>*N{Gz#b1JfC`JA*JjJ2es3Bm(`?j{0AMr2~Vt#&zL~4_(oO!F}HKjo`}i8 z`L1|wKxdrT+VWRKhu(9#*B( zoMjraOQ{?AdT!X*Lx(5GKGX@?R8`ALgk=_{C0gY+*9?4}Z`#?TeGMBNk@opoCAdRF zdqsJUhAcFqWeOkfiQG-2gi6A+FL|Qsl>ogcemesx9=ENA)G6{p(at@2H8D}{bW!Nj zXbFm(pK7*3k>Qd>_qAUk^u5hnvj!Y=?Ak8yDggUcD(ucircYyr+a%Vhso_qjH*%mquIQe)#CFlliqXK?ioO|}n;T?n08H{OsP!2&9pDVod zYj{W3Q+!u4rx3bv z-1>&2UIEkR&zUF%R(92hK&F@f8H~N$5lG+bDUbhdiUxz^3#DUR^rA@d4*V^*OpL-d;_36;R*cBtc4-Gtd4Kt zFldJD{kh2)FDsj#_E;xTvJ@$2 ztOe{v+@Z3{%~R1{yYRM=zB4-yqsv&?c>=U|Yd16dM>24X3Uns~ z0h;OpjfztZ5+r$=^sS(;<$%%!dZ}OXTu)3EyE*P#S@RS8&d=9?=5LRu9-NV6n z^qy0ak&^!B8+t|qBKT|9ZV=I_UeI6S(5Y|jjwBb}Yi zB`9O#VdcxAaqRV49x3W|RaP|U9-aYauzK87bVs00+l5@4i8zXD6l>8fVq zoB(EK!?5ZE@Q4;*o&68w1Gxm9H1;wWNmRapA6A~mhHP+9-92nmDk)FDh2;9(cS6C{W!wo<^P1jg5UR@K28H z$Q+LEJ4y@LH*Whdm7QlRlBJ<``}P8*#IX;YyRn4cqvHc{eA<-4PPCYV_6~dA(Sd+S zHGyLzZP41t?u0}n4vb>hHP#XMTr{PW${rp}5a4rz&fhEL4#hK|3<~J=pZ|JuGtcxs ztE%x*FQc^g-fG&%PEk$)7uk8{2lulP@9?B%f+!-jCwnjLp?feZ|8G3@C;IlIH()v~ z1RHFH(x^3fw+!h6R|XRLat3bk$7<%>b9t?u(YnmH!q-2QyKqcES`jApX*zdl>dfJ$ zjeln>|5_-?)-@-Gp`D)R)rnyjl6ANsPD$h2O zK#Gy3f)D@Xs(HcoZk8q?vWw?{=yCtE6c7DI5t{-!5bE6+1d^r`e!*Ji_iIa~RFacj zxl6nbl|Bddc`<9(;d=n0FQG5!dZF)X$(8l4{T@#_Ym&qqjeou7+HS&)g-F{0y9pQ* zY#*<&e2P{x-#g|wT1=n@oI(n0wk~Rd3ATTYm9q1gz0XRWf&&Ag>zD*49&hp}>o>=B z-Lg?Xz}C;PF|D(r3jHU%S|hOi_H zzgroooPjlq6$g#HKq40r-6_UXWVN?jHq2fm1Cr+YD)+8aZ$syj>y0NVrktr|}C#cr@a4?hrIHjB3l z#;tEcT-PpLQYtS0ikWea6vF7D1Twe61@6aA?TT>KNuM-QP%F!+fpSW=dV%U909F%lGeJz%>2lWRv=RiK(BlW6?ts6P|_> zPV(yNJ&L6{RM>ViemnT*@G(+E=S@99MpC2cx;x#8W5h5c^XHbavX3Rl#FmXIlK@6&>R-MNHBTEvGReQ?@%N>#sv2eBDxyN#D_`wD z^o{`)MHmZFYI^6v_*fU+tZU%Rt>w!FPeaxZ89<>CJf3@eaR2B9XTz&~?&NcuOY_&e z&_2fzmru8J7-89YMHDlTanranc}A ztxtzNXO7=hnQRSW!ARt>T9!0elFS5Sa9M=C$M14CQ~L)_8l!XHDR4e9cP9Lsf&bA- zkW5tZ%!6a!3&-P3;yU{|@6c)$+XtKC*`a4IzYaG~50S&Rwi#SVyg>vYTdS%O!pk*m z@PSeYnuE1(B^JUMm1>an!pF``N^uFSv!`Ayy?<~rtG`y&NR9Q+zWl5nnsQDOq<#@U z^y8pFs4nt5zKHPo9^tb4tI^tVlf%Swut^0OWXUR-;5pDd56Vq%$YA8kxrr3_q_l0+ zz7|xlpl0o^NZa7MdgrGJHLly3$iy8x9jw#GQ)zC+kKlDc-Ew!ÿ*=dF$JQR}7C z;}9k%d*jBznSaD3!j+i~SGwwp$UujN3&Z1QjF@@yZ~WZ5m0w%xi=vI=ek9FDZpw|0 z;ZkJT9RwqQ?(tPJ`OMH7;2ulStwpO2|PH}xOZv{qfSjzU%UxGf4Cy1Ml&tlEU z^u2D@P+Hk@%g?2;67x?d+$K(nYpB2jdhtT zzc!6^{X6?8bhPeD8gxbHa&y6&Fd@y|$&mR;wkDW~VPU!VxEZ-)pBxA&ZJC+gYpHVf z8+Sj;$%zQ$^!1~Sw`QFO-y^uZC}Hpl8?xfk=#1=VL7=NBfn!yAX?d)4%xmV2gs_WQ zP#Y>1v&P~dB-bn)3Y`>kXxFtT+o8cPf| z?ZzQv);gGq=kGLBiHziI4Zocv^<2S^hJtV*Lw2Cp*E@RwTSe;5or=x|lku{O#sFF- ze*T+)dkA&`k0*v9dTa#Pt@K`g%U&8xjQZOe*fL=(g(RT%Vn`P=_1&$N8#)(Kda(p6 zx`%ptiD`*(aiOqA^YYgAO#1vjTa0(wacn#ueQ|d+f{_#t7I7rh6T%3{C?W*x_Dz^i z**rHeNKRcx4Qva=kg1N$udcG)2SFP^#6D5N(k<1V7eDnSTN?mZo?!T;*_JhKE2RAK z2)r!NydolL*)b;R0?SR`v9`9%-79xjEgUAR-h>m(n!ST38&v9gqWB7Uo}@rx`R6uk zw(;hTeS*;I_po04!>IZ#A%QyPfp+tWVQ9^#{|wEbz`KTD>TnLkg>;|`0`#^2yFYNB zui>pE%J2^Ifv*y(K?N0n6@tC#n)BuViZ^4hg<()r(X9(0R|fjmV!b*%Et$bt0L_$G z_R&2`g4gij7W;olm6tO56YNDaFw{X%{^pIly1R7H4!e9OS2YH)TGEq!CKNb!Xrz+g zEG3JL=kCcq{b9O2M;rX^-$TFNoFM#@jDP?N7?HFh$btGv4Yish88v%|%;m{|01hl8 zppAY5lEJ)-$lE~uOM^>;Y^*scMRJ_64BY*vqt*cdETwz)C9X>x`T8e(Zi+h*;kQwN za8%Bs(eGe40+(y+`5EgoU+Nazg-La%-k$fLLDS;aLp#0Be(69{GP?8aki39yK0o&oz(Dy`yqB!{D; z&jTHPxYfOzdwR%k-Es$w0%B!A`+X#*m4*EVsxLq?tU#CS5*DZ?X3tNgNp8-Ef4GjR z7Fxlf6YR@KEre)-o;1Z@wB1Z%APe^nXxF#5_l`V=3u@oAg#{LTn1xA3+KZupLQ*_{ zZ_Ea_pPnV%pjrPAPellN2f{sALw@YySi&p)e$RzW<7GA!_7*L+xT~&6c<=9H9MfOr zVELY+R+5@3={W(NJM{cyoCc@2V3wEv;S8$Ox?QE4tE+Cy6C-%?vB8+9a6O2iq~ytH z@!jE%5cFAb4F-#0a4F3nsjT$**p1BG zpF~LU|5+If9xbBF<)b)X*Zgp%rV$^moHDV`3qGV4_SQP^`TU!4S!|HZFXUA->@(zK zsRwiT?<;W0p5`b>J5-GS7R6kM$rHs_C9mjl6rwd=no82a_TiUDul#M{W@;gSC4ts2 zpapN>m|y6A&&ofqeC73d#`wXww{^7r&y25VF|5Iyve82w#@{=+cR$H-JEhYo!k^aG+Ut2Cz69Q1^KWYuXT{>jK*al=w@xbJLvIjt48x0Mq$ z32HhzVfGWGgbS&6EW4AZd@#Bf_I?A z10kSwSi5E*ZmlUS!eMEBNl1u^pNf{3m(A&)Rs3xI(Apc!Eb8=(w!}iwt8`8NIcwWw z^~BZ2PoQ{)!YhrEzk?8D86!m$MqpJ1<<8t6S>l=MrTi3>dfw3Nw2vpS5fF7zuwr~* zKobtiqS?(t8P!kM+aohkGfg*__x7q2m->*AYFTux%YSG~`@q+oaCp$jbp;EvxxYd@ z*f$-k)y71@Yrnb_oRnGLr(6Tlz{^exa%a9jd=8jVh8OXyE|@cMZ1eQPbxgpyzhHX4p5IlNBt@ zY~s#q*Egdi;DXS+_#H5vgMznD`Gal7?*StPD0 z)IkwN^s?hz6c>Ds`T-ouKfr+S;J+Q7^saKE$g*CH}S z=9ue>Sm9=U#2@*`H^eKV)SdMwRWHP7p}sx1@V>qtjx(>|V@4?%;43BPnoCfWkt-84 z`JEI@DW(|MhTFQ5Az375Ca1dufs`enGN@O-jC|80ThzXFGN!l}YKhjNesepDFr>Q~dv(Nvi|_Odk-Zn(xYWo^ zm4S9ZGScPB2?#d$PjaVT!9sA-zQg8<6g)EE2O~@CSdTLt-IS}rJw&a=hLL3dx2tWZ z^Ri^a3Fo<>wmsHC>X!{|B_;fuF`6tN_RZ>6Gvwqxt>075ax(9;$9njSTe-<}@#c4a zqK4?-j7|rY;>Fpl2Mx`BF8(e#vySD%rw`)`jYbM}-=fmn>8LH-ElUKHQhH_l{Y_cU zVYA*-Rw(56%CK&KTnpp`SA&{QUpKscYa@Pscz6A1p)?}A`xk7c>?(HAUp1NGV+jeJ zHa3i-rLqw@YFs9ej%P8E1uCHW15T>n7?)VTN>^ZK<_o;dYL=^zLUmsuf!xShDmq># z7>3j#u8ns1jnH($UaF-#E=DY0%wM(i3~T%ptaKji%7}}Lo$PHPlhtNh z0GM7ds!ytLva?GY7|{9pwb659SXkPy+J#y(?S%l*^;%ehLIsSQ`SUC?1S0|q+dj`* zk&LW~1*)B}urS~R{%1t{k8^`DXOx5==!$dQBfU;QAY*4|8ejmj;Lz0tz_BKy;V>-ci+CCsM!^MirIwc2`9XiOWnlFLW03M= z!1UzRo?r=KUL^#<7$~o#@K+}%C%GYY3_zXf(`;N^#1PT&wg!F}2Kwnx0Kj&L79$<( zG~)%3TjCkM%+z}zf9$O_n~?&EEa-7saq(4{a%NzJ(58>Ew92eVf(dOK%?t;{V1b(u zj!qJ^r#iU@1^4@L%(S@p|3;R;K^>lNz@W>(o~Q9YlG4Mh0lZZT4xQE#1sQUX@P|f8 z*4W)YSV6tkuib2`a&FeU3rD7m-6Um*fGyG7grn($#i!gcuzUH+en3SUEhaKH)5Wzk z_Xf9_z#h^SQU{WNzZ5$=^N3on`h`@*9pk$p(OgX9w(&TcJp+GZ2x9AbzYF8+pPDbFfk3gL2qd1;LhwODxpy@ z9yLFIe8R@~`S?`bu|edLTqbwGpsu4L!ZJm_r6@o@1@|s%&P}fRo1vgiaU60j_v93? zm0ZGcY?Q#}YVaa}F&YHs9-3Yz=c0S9o)=s}=_%a{Zf0CNq`wfe;!fr2p3Xhk;4~f& z`466dHqDzoyEsF)xAgX&dc{^%F`TX$#y(h7u_lJ;C^pt1{1b{RAnw5-0T_!2I1PK- z>JQG}fkF-I7d)Z%orS>eMFQWQ&RL9M9qToyG(W{^n)CGOsTumYw`?8cYoBo&z0ONJ zZ6OZB!fKDoGF|MZC){g~l6sw*I&6r)3M*LnuJJUXouEn^D zY{*^4Vq}S(9kYxw+FfMQXgiN>ObmpbGkZ~ZlNegzO+mqy=Shf^!mmUTU1#X+rz>G% z_n&x1I@n96{JhtKwdjTHJXUE;v&Nxro|-jB+0ER%TNQb11y9UFuSasso4ioZ-X1`% z1q71fIiVBC?y8j~s{PfW@SJ{YkQD;#mi*&`2ZvOC$R%;U>nBomRvr33a#c*1CM>t- zV|ROc)uX*jAFAxrYC)*l%B;)i}>M4@!GVVy^?G$RV zIEzEwA0L~!x--W1ff?}axPw~;2DdE4Vx{)v^}Iw&l}tv+6AIWOt7QG1%W6z&QCoYk zMBf!|?du3X?aqaHZ4!VRV*P7^zI1ut@5;N(_!_piej#2AV3-6~q|uP2q>Czdq_4-V z+~3EEA5YHRNumNoDO{*`PeMSJyk8&Cq~fvvQ?|AF5{G0|`HP~(w4YVZN`3w<^#um| z0F;24i*VKW-0Xp6+Qt1@Uu(~Th+jEq6Q`LBd8}2}mkt+Qx9h4F9PWo#_z=!1S=@2a zM|r&f&(Y@DnT$-g&_z6eYCdxps3@en1byktcdOf?^04z-<}Q#5KZ=GzU+d3#LZ>U-2}3S;^Vr$ zZ}6E32Rxz2EJ0QeCnA9v-a ze^VHSxB49rzJ*L5R@Y_$=;{h#LQ<(A$(HK|@ zlw(b{XLDiITYIr+9 zGEpfHt1V8@zhA=|ZIptb1V6GUDTnx7fD(I2_ezFdJzUN6?pX|enZrL?&W;Crj*UH+ z{J+ynEc#7l)|b34&W_QhCMI|<)VgXmZ0|CpS6DZ)lVB>_d;EIyFb+Rrruac6e!6wT zcjvjOOYNM=Er4n4T1-eDOp-{m6y@b{C?)&$+sWR;s;ZqZZjbcg_OSD7_Nk|XOG`mUnpm%v~^9@IBqOJ3Qn!csE%m(l93cIXa~kR8ZCx16pifXT~gL z=Q5%!Zm(j{UT$u0+RVF;4pG`(a6)Z-{y1xRQS)9sZy%Bl{9liLh>0@5e z74k;jUX{Xv+?Jq?&|=q$CR9pKU~F8N*L7Wym_`9?mjr}b{aVP2{Ky;*a1`{f{oBHK zZ7-TmvF7Ezgp~-+uMXT&Y@X!PeP%ZKZk7$jhQzQgpU(sfcoh+5nvU(lP z`hlB2h2SKI$J95T_Sq0Qx}GKRpMTqjev1jILnF#a`hXi@l%?q z+fquOfVQO?#s#-N?#^WaKI3(NX27|@$4dIy`@EqN&-oha!69qNaSI_phCt(jS4ocZ ztejD#dT^@}a0qRWz9XOsC9UiIv9_WBpJ`-3V?TE)187AsQiv!iL`JThy;atqWv4kA z9E-kll8FkY%h^)o29*xUW2Ss1-abApm_Qy1dgST06cjhgX1=AVfS`o9=T;1hXqjki z{Cd$)&dbFj_ZK59(VZA-|=QkR~`I>ooG7n44!FA!H=p!(UBI(o;G4%Z{8)F~`6wA;)Qc5!@tb=P)+8 z@}zA6N7Sx01Ft(|%tx;&{D*FyK8JNUi2#d$fV#^5*4AqXdB!~0e}-9Lzzz-$URz!2 zj=LfyH6Y^hK6~wVN5@0w67Htq|5$PGa%!RhrUFqBirg2(SQsAf|A5)6tZt&T7>%2`wlJDD)ivN4(TKQQg{%WUZv}==Gu!R-Az}E@-~ap9_T2yTrg3y9@j^aCI7G5y z3DAMJo}a$fPRhm^06 zcRQhs1zvE&L#VCo>|WQRHSRr7Fx5I=*JdH0yfHZ(J5nfly2ZP*OI^uB@5eu2@_1i0 z!`EqRhIe%QcfNm-pxt{Qg{%bc_@EWr4KGBoO05)hz>`jjJERsFyZQ=c8-4 zK*aVYDXDpm7qp*7Q#IaiX;?QDvj$8z#!0-El5#9uF}??Yc5DU~*qd_3ju?m;d+39F zdaz#!#sHv2=8(K-3ln)e>7pz-R&r*uk(9vA{NsjUz|^5=dlWiwV*(vqTbuM;S5uP- zu0V!E@k+brzd#$#Y|qhlzGfKJdV=nIT~M%s9&TwNq88#Lo!1|$Dp^~+TJl9Rr{487 zzzbCy66e;h!yGDpF^PB3{7Yrk_9HVUc>dzrn(TkPsW1$hKy_<5k!TRUy;YuT^)j%fOSQWW`XfpU^=&@Sf z6Kw;5&%H}Czhq;#`j{r1k3+tB_}ODQ?OIWN;H5&IJc>R|bK_5$C}^_vi7{R+3U2Of z`6o2~knO9{+z0Ll{slbB24G8?XC&9vV`fR)=CP#r0e0Nz(Z06tPKP8$;`?@x7ygTi zrmYGrpQxp>phH$d8OnQVeKqZ!ZX-W9Lz+n^{~XNoK6KmjXH4vT_q0HuEH7{9&iP9m zN&R|y>yh0{g8>@E=k`=~@nguc;ZDb(iG$gK4vMUi6_^_quLh1SOa1;Q1~N|WjX%fY zUkE?^?Q%AjO@FS7g_%YG>4GXs=qs=I(UC8vaI5fZam6scSR@lhOzPmrJNw(7RSf=- z4u$61!8vYNcyCIv<-ih@H%;i04B9iErmac^<|Yv7>*yi6x(rm5p0k#&R)3BT+Her+ z6dNA`p1@{SOG|xx4{F#zlmXXXb}`*&F)l?_SGSSlvmDcC^)4M`U*GRY=nN6@u%ow| z@E|_kv-L|ZAJ-wI^K~Ky1rDN$EphuiuPnAvrPAerZ(Y-LH7&% z{02L8)mUCo@`WXp5GH$>2qPkF72bpOy9AI{*T(g#KeH@fg;$$pGV^ek){XGy_1lu? zUKMG+$8O{f(+{jHLZ3p5Pw3##!;<=HH3t#ar>*_9gLk0w3Km8xH#8K_yij}N=3qnM z&{8j9^H1}@c>CQvL&7)peyE@yyGsR9&H)Eg)=n-zC!Kj+z7q!%-7eME*5sl#<8zMQJ$PvUAEKC^$0w?bfX{?wf1v9D6-ujN#&BPqHgM{8|$H zQz@f8Xamn;!eHFJYpXJ&mb6s&a2>cCV6v$QXz+VH#p4jShlhXDf^$(NsnP9Vq1%-EV+>uwJLx6vuK36>Ld6kAn@OT@P|9Hh}Qp8m99 zhP23aONX`8U(K=9jT7Udw1u0Byc7HX4kD893LA*D2Iqy0JNx6_P!5{C{(EG<7$Ah8 zk@)iM%A5- zVbt?Ij!DNeJWZtobMy*+DKhAhH84=;lgEP6o7kdHLz=VbLLoha>6@@Keq!P|MAUH2 ztexPAi=T&;+Xg^}9wEpL+CHcxO4R-}VKLA(@${%@LkCDrtDGM#4O$%V(N=HbK(&iK zS*Lvs3!{!;*CeVq7U5v!(xg83@T`|)W9b6Ec9s3ch$!>ot?u6MW8Mh zV*9#i;?S)eHsN4&{gxG$CrdE7r`B_+PAR0d@W9aIh#+Z zMrn{NGb<}0aqV`LI=Jl9#WEmbYI9QLk^aT`86-m58skA$Lf4^QTxZVvcunB4LAJxA zDcG9wSgGtDl#`LTUqv;nK{y4#qSH4Y0O$So?dDcIfD0k2oD;u%%dV}4Ec(>-+1X10 z7m@M^sAD9gL7p#YKYlghsqn}&BpPr-`st_i^kjg5<6vVqdv8Ma&ZV3$Y8dGZz^I|r zg4j}Lz^Ii*ap{y@o1TtYdIV@_4A1%h9Em_1kpv1I1b2dnKfREPrv#F|EU12i0*r=) zq$j0C;@nFZ5m~;OoH;qk28#gG^u1UEDkx0X+KFCjYMKHGX0$P&@j0@8u8pe0Y|lIT z?*MyoK;z-==8D@ZSW+uWi-=E>LAAhN+x_uwGT`(+X-KsL@UwI4;2oad)@mNdQj_;= zkSeA4C@qMnuB7;ROyw@w^y%63`1STcD>L@-;`7CQj)KY6zN0ux25#Rf%`QUSyG)rk zK|VzF8x%WM+q)YNz0Pvx7w1E|NLg(92IXxA4 zKq3PxCE>#G{QY&iRf`?Aa!aD(;Vt?H-A!H-zYn-pd@G(O>}Fg zK1*^B?}Sp;Kt@fz#uklK9eF6e;?Ok;LyL9_HoXttf1%IeL8m)|4!V)=DnACrXcbuzm%+#|7QZ-Qy^S#*?NZ zT7{>RorscA%#rE!v{9qL6p*x61*4HSu`B@M#;LgDA#iFCt}pc^eCHG)PVAaYYrdoF zz^<4?E_UvM_!#K+haF|KWH)j8xIhK4+Zl$#^t!BHcqdOzLN_WlW1w`o$MmCLB^gc) zZ~>NAO)meDe7p1M-f)kq^<-g%27bAY?a)t?T9-`cWGR$hJuOsd{T?qlYob~BoMs=R zoM&D8o*2zO9!Pw!ky%$qj!$maK3otn@eX<6FAkR~Ze0G+wswIx@zd}v#P6>W=m6L1 zj|p^rP&0!Ut~6f15`42TQK=a!!8kFj0z@1z#=5nlrAKV#iMW(li16(;GUa#FzY*>E z8&5c6=TK3WLexos&z~#>*}%ZzB2Z)GD=hyBxrhiR0oc+oN>u( za3%L4R5#%ZZ8o=(6;e5Q?N_w2U$xmu8C*hueN&SaP_3 zaq8Md!9YuUORJ%7;rSnCg)K734fxPgaRU0W79_0L-Jh{6Vrn8gQ^%qkZU*gbx3tS0 zSXij@Oy~sUy#9C&<=!pOVHT;r>F&PaTA~Rh<6<|K^wHdtHLe#BnxOKGvB!|#_PKr% zd6c-zL+;|av=AIq^mL|^TUm96d*xG~v-9rlt>Tu?OikqUninp2wK9gUi$26vM&nXA z;>w5Jey6m;zN{A}_0{_CQ&300azkO&&F&tzAe4;%_KtQ0HGP~Y+xTZbI5#79?$6I+ z=8N)pqF?X`HXrI;-q_W5jIu3D?af?ReQ|4m&Bpoj^tRxZ?N36EXB?$$$rZ30YN>6+ zKc=?F?ZHc0=UT1`7yD%V-y@x_Ake3+xH!^ouJbm?bmt{n{{kNc2H$GpVJXK8zP)2zfvL``4Ko92p=`kEixermeQAJ?v5zMNR> zjADS_{{vy+GNeBbv=hn_?xQL(U73I=t>*9uE*~Du@iU&7_j6nkZUNGzvgGY>3WMyX znWl&ocXf5zzkh$jSlFKO8O9v6w?+YaKc=k;lYlWD:dF$;5ZS};{7SWlI#MTW6A zi=hXJhf7MMQE(F3#0s;q4+S^s4&+SSkVUC@}v?{{PMP)NPcjd9u_E zZHQ5FUhcmO5w+47?F|NX9{%C|D|Ya}m+{Z~-bMIL$jER!my_eIj%tUHS5Ks$XGky8 zW0?(QNQV6{bBu(f3CRnon0b(c0+i9$fFl@$+*6~?$?D0de%b+uiSs|LO6{GPpoXQ& zEtdZf>~s$^UjSXyA6y|q-I8=PG~s}0jq}53mmmg-79w^4D)3em_CODGE5Z*B$OK?D z|MJn;xUcDO#Y6?iB^&p%#IqMUS1~>*td9^V1R%e`c{;Py^YJs=B8eaoSE}|qI?;Xk z8B=xI?_p0M?cEb^tg$fWHHWw6q6Yu*_44v%>7#|%cCGS68HT2eD8YSqtrLTCOHKj# z6jyo7s$bu8kMGL_-~RA8;N{B35j4CYo;~ykh^%-u^G4+n?<@J_S8|8?6ZIz*V;+jT znM}+vGxN^}>J$iphc;R#GOgl2|LJbAwRvyyunl_hozFgh12Gx%Kc3E6OC_%=3ZVSk z-bYWw0-ps7T)^4tBK~0aH9eddrwKAChBj9M+T54}EX^uGZOi63h57Q2Qib53P}NmD z5L&hA5^-F$5g6Ux*Oo3!Z!3xnOj7G=Y#IIxD`Lwho6luE#R%g5RMpqHb5c{4TXQ~F ze}eJ~1Ck)k`8*reFOYQ}T$o=!cbmIg=j?=i?q$bhfUKkekqk~wzOx~Dxd0tSMM8sS zLFUkM@zDW`s;ZW0HfD$VkM^(cbnP&-Y}e=p4t;)y^Q462RLDjAV4a&&wz#uxM~;<> z=2jLzU(u^`Uo}2;xH$7#8*Dk;BbMj((lPZ{OnUksQ!7zscOvcwjxzqQyjpZiczdw) zk8CO8Kf^03F`5xS>ujt9_6}Y@Um#WRaibjeufK5~D4}yU&e2(`|2D}nMc_-O3*P1l z>bu4g4s~Vz!bWkkMx?6Z4(TG(*L2~$=I{FdW?!}-EisN z2B^I_-iLQ{izD0V``|=T*!Z|)drSw!Q_GSm7(l|ByN?({I&G^UB!{ttx)>Iq!B)0+ zRJ>k|>y}Th4n-^p0CFO&s_IAFusxrp(j6QEC;O7ATy<&e>PwzzC0=V&N94D_!^!GI zkI<-miW~h^#By?Mx43SAWE1aHv!IldZtO;V)xj z47C7^CnK2TJwTGDB|Qb>azrp8zV2SVJ@Llg37_SpFw&xtV%%_%s`~@`+loTxxu*Wk zbB#7eh-?$BF1aTT$_S7UJWA*uef+3>{Pmv(7}>9w1xlvbul>7D^JnMaSP68)c)u5p zhBP=RZaC;K{ebXx6A2+HS>c)qfq~=LZ{k!F*`ig_Kg1d%-;fntI1tc@Q0R)J zpf`L6Z}%Jk2e6U0Fv{QB8xi6$`W1WF7f|@HNJ#2^Av?Hm=-1cZo!GfvDTN*WD_(l~ z-N11!y{gy0fA5P4LQdf-fn4U_?}Zf=XUE>zyAhN;k5wY0qGS~nPl%~l7*?;pWSc!C z(w*3Eyc?Tc%cnGFkvJa#ndEii#>k(UYae9>eJEMfxgemAW2F9|uolWMwm|w7zlM0k-zkA2akV2hG;)hVx_F?>~OT z`%MlH$F%Rlbt*H+T4+%J0m$HhXkTA11{Zat$Q_$U1E;c_oE4yOt`Xq@FJ^J>RF0Z< zaHDM0{+p~U(hy>L!~rtE(W4OQj6%$IvZTDLtn|#=jdSNm>8TDk;HIT*0s_mbM1SA_ zAnM=db8VCc4%nc|QCH_!I<(M1<@ zBZqD;_LOecg7G~VEbr1|FZ@fb2QL3ts|M^15r(Zz)(DU+EJPr)Sdy<*vc-t~Nv_6o zbz%FIY+!t!OUvjCZL>XR)Pf8O5_l%ZX6+mo*{ymXN8bDEI3KQ-u7bbS`&maoBK97$=^v2?8T~(v|#yAgtxC-{PDr~(8#4e|H^9_87jbvioErFc5F z?@0rseuQcPdr0e|nWSkg>EhJtAFYXgZxfD3ui!OGi>Zc+l><^NUeoK?=D@BlTRVEY zFX+*;gj1^F@B&ZCSFMp)?Y?;xR4fzb^@N;u$}+#d9v`86AM5e#X|{kOz9%PdXlDEk z0u5Q;5-|_(9AKK))KvJkqO;Q+wu3GGI-}W3aBYBPhdWkDetptt7)**cS~r)Ltg4;G z);eg)hKmI$^ao7$z=ZM#Zl7B5#uD91Ic_NM^=dUP5-LkR84vt;a=2F{>)x?@srdi{ zJS;E&fgFRSuBzUoSEA8|cj8MnSkrPWF-HI_&Da=4vmHjR9nj5f^F`?9hvTy-17f|Z zeo17+kHsGRpSelV(GhoK%c{{UUYQR0!1gr@oR zrenU+?XbCgo0sPvXbg1}{G}jGnoZJRwA$(H)tgy#nzXJb&scj_N#H%c(gm5uVYm1z zu-b`8)l)Ygw+0)_gDP%e3x#&Z3oH5V#ku-ucQIjpk4=bNDp-AHU!x$ckj&v**VzfU zQ>II*s$=gXI39z56Ru?|{unVaoq<2MN#|_bbxpwuh_U5z>gP6EaeczVOihdG1q%Ur zlDB_)7B}$slq>(4>=60#!Ob`b%2?ntvMs3QfCJ2m1A0A63m4uWcxX^bO?WJ{eP?AfazeIi2 zL{!*Ffbpp-rrwG;hRdx z#~ZBEUZ|0ea`x~W3kQktR^x@fv|Kn*0vV%qXwTF8=sWNIK<-drx@u*7cy9F#i^Q#s8o1fsyeh9O6L~hT_3_ zn!9k_?bgmK{shQyAP!+OL|hTN13s5h%D?WtDje|coRG>`Rb36RRpC*?j6VP_8u%s^ z(D3XN=JZc~oc3CV;Y~wl5E-hKez5*sNC;|cwx!=@t>_aZTTRw^lbNHAtgoKJN zdtGo(sKI*$Y#f~HznPer?t_S})OBRlkU9NWCS398_!yG_?S6}slk>lj8cf9WXUM&U z>?%A=THu2+W|&!4KqB%lNN&FB4*BNj&nWfK_xZ-Yg?7H2iZXkUGDk!!%9!oy=GHox zqlQVtdR2?z4UhHoMEH%$mIzq^85=l0M@dYBgmzGN#eX8g2Qro*9>D10}2Z6J#nSSgr?fbX*pM4Gh` zOPv}C#`y-qs@P=`7v3PifWU*VPt|JvUpVfso}S~=n=if#^$j!{17dI|{Z<{vyd5Cj?SFN&z|jeRYK2ZJT1D0q)6RM zalJU_{3thZ(NGQBn--)g!mXg%xVfdE51W#1GZc=%OZ+Lwb>LURCqMEE7T^JZXw63r z7dP_;SR9NGeLzY=aQkmf-8*~C?MOxzv>P5pIcx7u0}$G{eD?1_lbW<~87c%Z|C=^e zPzD6tFI=yUNPKsU_JUhL(x93eauXtpS^Z|`t)t@Q1YQ))&mNBa>G+U$;Wr_AUu9zc ztiqi;#gQGtcCs28@vv0xFH|i>;CbG3bk*52Ai7xr^A+h|+_MKSjcwzjT~_Fso7&oU zJY8XI@=QD-sH-*_3roPfysC!lHY&EVfa|P3plGIfUq*TiYzR@l!qhGaF}vo1>Hue6 zZee}asQMZJ8ZTpq(xH14jaay<9^--+@8f_GN?okIgq0hrdU>d%%8!Q{J7*824M-0l z$q$+oR?F8qKoFK?+Q4x8^13}H&&LFy%~fYR^<14fGIL`O5|1TupxOpvBw|rWi9xEG zz%}`{HBU?oJQ|EP{hwMZp|%lKCpgBdxyA(svC2*p{-}p%2S0NlxvXhv4yklP0v9Q! z58L4KR<)G6b;&az%R^fT>kGw8)Jy>9rR+Y}ntwgdKeCECp1x2p6SetGjjV*c7aR57 zF&uQG)D<|mQcq*uVGd5?FvZq{_7|d4%Q|jpYa<6^HilP)t|JRdG;9!xjJ*C*L(qXm z*HxU29Cd1|q?81iI`>@rD?B+xXPq6|=dByOcqqVu4Ovr!&?1lF)Ot*X#qph#Tl+CF zv0;)TOxyngCGoC{OD;h7$ma|BvLG%;QnESG!}As0$mbtRkB2!29;SV4g2b-V)8i8> z(YZPy@yWb}r;6ir67fV>;KzAmP(;zK{qxlxCTM7hFr_6c#h`t}5ZS8liEYxG_)q~F zkb6(P9x4O252uAMY`0C1!a!Dl$q9bv&*r zi1X5hKVyPzgKqUP_rKbU<^M@op#bj_$(NH!tqMDmFw;ccwbgcT>N zB4i2^N4+v3y{cmM*5yxu345VaG!pAq%~9?{l@yOAj4>v{V$FdmrqRPgEU_qH?$>_$QvOk2tf?OhRi$eL-Zy zHt4i5pjGJLVyvF`XO6c|11-Z{8YunE4l@mwO+fPw42lhQlYI zZeS30v|EWnL_C(BY9S^@WTo_454{5Y)-S$38WJ}x9r?_kr*ZWso7ld4>ZsI;64N!v zfT!AX{cPnt`}pQ!ZxVR8PK-_uC4ug8-=gQ31XkX#(VfGDiY)dFfthJXGGM7!*K51Y zw4=ky57$8czZ)<-+yxj&tXO-2G!2I)4tY4Y%Ub{ZLG&9>#hX6F%FeO2LzcDV@i8%d zH9Fdv3f2OIsPp>h^{ZE}u}D*M|GEX}zB$cCs&HKT$uJmlU%?tKifV$C64>gfUi3}- zg$TJ%`C(nyXnj7I_A)LewD?vZa^>o^H{4g=5s-dnR@5A?MA;`)c-Q`QtF(@P{*>>U zG2?pi0pfKW9VJ(|ece23rw*E)^K;Ss-N5f_+h^j$eJw{T%g2`5H8lsxu6=zQ)nW`H zBHG2Hy1)R1J4#&Q1aY1WOFb-*X+r|CDTS2hCJnPvV#gDeP$&*RK4VnLdD(B3Z)&V# zzl6dc^z(l83VyQ}uOqCp=`+4H($j<@<@QU3;kY4TbLs zHaE#lX+{RnkKu!2e_EE907O-?xVfNC*Nd9ioCFE#0lsAkvL==V*`+P~t`e zL>fduI!1Sjv^1l`31f^LG2*%Q`#&%C=KkEW@nzR_p678M0oH-*3csVHccTo;aOnUn zeU2$$EMx-Z7Y9VTv{~!u+yfq_WQ`ev_DP<=ZwnXsdRIk5gZiEpSWU9B_AfNJ-^8Ud zd3s_Ux7pa(EPJ8@cz)ngm=rdmzkQ<%-0!941;X%C0LqGl#Qc(VfO(e{c5qKV2guvr ziC$i{_Jl?Iw0Eb}^#y^{H3MY6+;4xK+0`B@9zSXCF)4lR#{?t+XX@C?{e(AFzV4q3 z7SCY^M_=TNy#g5q6KCp<9p7qd62@$dlDRIYlx(-d^i;~ccbS8I?}Ha?5KfTsCIM;s z_N_sP&%1Xs$m-dqnG*w}Chc`jfysFwr#&n#)ZVYjW{gy~Rd|zxhnzO)C*k={bpFoz z?ds`1r3!Y=cCn@|`2})&u8qLNkE^>oGKdjtELu@M7>bAXo-Z_$W(NGDE_>+--2m_8 z41=3HroK-%eJ@70^s&Q|MQu~sC55}{RCO)>#HgU8$MJCrzLpU6(q@+!*jwh&cLEj@ z*NjmEV+Zij(>}Lb-kRZi!DkDjj&n-GFCtSb<2CGBq(8GML7PwmCEAHq#~%uU)`y&9 zjlS);xj-5HuPMn|Lk9j}NDjMdT!%lH6|HBtD#5@)%>My3bi=!m%qBpvcmC&Picq0+ zpx9Y(O29APt8v*`%u8g@f}gE2Mao_CnfetuQK4(ht)U+&->p7;usoKw?HV*YZM~mz zH?%SW5*z3Xk-8W$y=P{h#^brNsCq=t5GFfeDK=EhHUGgTa1Yh`wArF{l+)QIMB<=^ z$Yklk80@6rw;oXi2}sIF36O$RdQKqcI-u{#3 z+}!Clwj^3_@8LbyW61~f$S9bjt<~{%1sD2x=14T7xQk_fd^!XY67mG#;ZfFupR0WL zWqkd-E6z(K4UjX{#jklR*%TBW6D(=Tg2Mrx9~kT3a^2?IBn2nWWA@U*DEC!ofAq z5=^6jmHJvmoQsPT7-Y9K1kQ;WY=UTtuOL-EFvy7Qf|UNjN9a;oYk}iGm791&SJe71(jf=Ki}P0oWp}w~qHI1mxQ^ zOH<^zfZS5DmkCVwdn_$2MRXMu^7zt+pK@~E>(O6{ie^YoO|^wOt)tNiFPZ!ztcVpP zzn}zFl9eOBe7Toc4XE|IL*~3IK=AYuaB*3fC@I5$C$?;{mzulmb^s`k@eHD@(Y-xA z<{hDUtk$>!GwEppDiSb^bcAMVfZ^jFn0cit_CdN1MWV`<7f5fZCsR{X16geQ>wF-l zA+Z`5X?uw|nmn|oc4!irUp5dN-YpOb3iw$fNH&%Ev!+<(&l}U{9gP*Kj~33LyQNGs ztqvyLBmIag^lG$uQZTo<6qk<>A0MwBMUR-{!_JYnDv1A{Iamj5UF_hDq!+r^Tre8vE0CoJq_e&+&i z@KudVdRhKS_fKiHsMHYbp=rwnAm$>xF05B={=vBVX_}XRb>!Idk!^aXmcLFQZ?7S* zFV{7+l(*@#n5$ri77OV+n)sRB2#r!E%WvCs9V?}1<8hWgRA!%SDrsB{ZM|-T;Jgm? zj}9WZXUcYdZ3f{t_*CA8tkAE;h%VD^);f#lHGO9i9yLH{t7S37ZEWOlyV~w9T#5LA zrJt__&q&5I$NKF0dO~0qXn*=wc%m}6)jKX{#r$mo4Wg>d9=VO~Yc|#jpS}{7q1Z8>%erTP`aQAeX-{~rO>)yQMoBhc* z^1GrT)ea`@6Zs^9xuAW7{tLd3OUHesL#H;w%_<;d$ny%;CbMDf*(vn6XJXefH2?hg zu(1%a_#BRB0wv!#e!$b}b$6!xt-(AMo5CuH*@_=X%BP@55~dAZtzj!b+9Yrjcn$p# z14vXcDybYGghmPiNY9)!aW;YHTN2HmddmNnjxM)01xSwg+^sWk4E;S*6$7e3dw7@4 zT@(Amvp<4cj7xQOn;h&DX%7C z%P0CH)CPB&TC%Sty}Bu*S$o|2r?b^-u`3M!=3Qz;R9BZ6qyvKz-cMgv1$t>UHGkms zOv|pm-~t;~2*k{_2qlorztI|xD9pZ^=WlqJsV({IUP_7yOgRD)7jCS7{&cOt`=EJqP9Jp6Q^^i}HtXtLTHk}Ze*?n$ zsC9q4+0_(ymMzd1`X1pdrA#z z6ox)V|2p#$TfHO1@fN<0rx0WUEL>K;wK-P_vr?M*`8DP+($M^}q*ZvQr(;s4+wCND z5{*wA-K1Qmn_sHh@QR3p99*oTGXOR-=ha6js1#_&GK%Z=fHJkK;p|+NgWBB0k6gQ= z_F>z0#_TSVw3_<5E-y5C`o+aWE4x{u$07y>fphz#*`NXf zrB;fzT+@HTBDWs>J$|V#2Z6|#EZO;$M=8gTiAZ;1wx6e0qRT!(47U@x{QYjc<1V+i z`}f6XgGvr-*573Qrbs{*-@JYQ{e5G+xTL zl8@e%xA@Db;xtxEQ+m265VidiR`?vP0vW6Kcjb_XM619@TSkaryFM42vDVA{KCdY) zqaqhigwXu|;$|`Z6DgQ(X$GX*u$ryt$=mXr^2M4_vQ~t191TVML?Ls3XSgEZ>a&ex zw}*fH$ObmoM@Ymc+837lvsU4E>9+A}=A4RqeNGxB-=j`E^&GpGp4-mkc)#TmgHG88 zBg@iG+pom|G-mj9xTC%e#!L`SyL`F(Dyn`T^aEAFL3IU;;4(u8Lm%6W#L9bkuzXR* zdS-+ZegcM(W?l`8(})(CnUR2Ie)UT{yxez%W825`9|0#i z{tZM)m7|^%8FO2tRlhR0Si*C_ai!9&xS~5dg8`xNYZHyETZU+83%m%3vK3m5MjJn> zgZ9Y99ijdPk!e_5REcp7!eeETAxcxdkzcj>af3WWgiiWwKm_r4%d)exi{x2B!5!4m zcg&Y$L4ax_Q!wiZ-+|Z)g-t<%n6=C2svH*cPQ`O5sG1mE6y(ktfV@@3b|eM#p{J<#eF?+hh;$tkhI)d!aM$jnGbD7tTxl3ZaI4=-ipDHRrGZ6O z{~9uQ9njTu%ii7sWbEFA04?bH-k|`Uw8tjiK4;KT=QBW^j7aH)wJ3ADY!O~;&Ho0p^SN_1e zF5Cq1>rp}jA0R{I1r%x%Rp|Zm+GS3Ei1@5rKqKTt16C#$F(yr3Am?HzRA+mhekMA4 z;dROV_?fxjLJBPsxTOo&x!Bp+g(f6uz%^J{>z{+b+*7_2I&6pb`K0f2kjfMvQ`y??40!8pV2$_A3Rw5 zo%}ouJj?vMw6xE(l>lcC`(ygz@)A`ZMMW{kpQbw7;`id)w+Lhfuy?(Ad45dw>^Wl`8*%225+iG?M-TOo4l{ zzP{0KG_k-qWATmq(U>kQgOq?uS%v;C?bfWAbiIrB)%=GD{p$xrL{|;LmucLHrz!v4 z)tH=uwJ__JzUoYO9ibG>FDD_x)J`5gVE6J2S^WN zaB#q%39H1crG0;CKR*Wnp=~Zyl@(*fz>^=*w=L8rKN6ozzVY(8W2iTkcL-1m?o>mE zp~j!;p2iu>WXz`VM@KyWr4|xE0?`44<^tdLF3|}-cJv{aNYTM-LU0qrP$hIO+b~-K zxHIQilxGZfxKy6ikJkbI`@@A!!`59lMErVo%yc$$vCfTlp?E<SFI8 zHgv48m~~jF;+T7&3hC;-#SJ{bpV&Wn~5*)qV z&O{z?Iw+rVmq!_*W+N9qfA;TG{Ic;5*&6_eNE;h0D%feR8Dj~L$jQmOwQY57yT*Rl z%wjb{vU{CQE@$luH-MFl1;XV41P=mvtstv2sZMAOS^295bEI;Nl6%rdSDC#&11Xek)L7t z%1vV@ri&tP9L}zz{bHh3iMMm0pBxCeG1@0OKKVZm)vw=X*`iAJzI5}b>YqO0TxzN; zleJaI56|9QVwK?QIJo{8W+>ozgo5rqaXKMaOyywkHwFEMthl92$$BAHV{r3J8Jj#9 zouabxCuqS3eZU~1`7UPo+x@8FbmsSYBvcx^^X+C0nR7MbW|O(q)dT!^RgcODVR-s% zQ)-!F0EZLQOB#p()`z#x{R2qYOC1C-;p%0cAqNBCNjT&6CnOj*qMEn2za^#??(Ugc zSY)n<3JMx_t~=l9B4g%e_$NL$pb*y#=2ME=D6%_Sf6(is7&L7^JGb~j))qG|r404= zugwMm@#)JQ(BWe7Z{3n9b_%IBu(4tL`O`RLF$_RRw1C$3&aHqoBav-(DfeKtbWwJ>pe0qdHJBUKww0cAZ6oMFp$1oWf#EMMcG8+<0B@ zT(mz4oK^ev>(|B$N#G~Plp8mOeIQs33IscT>Sy$c8$F)P6g_MrEJG43 zG~_A;1tA2PAS+#wb;tlv=2kCHmizbDK~mGhhirpZad93X-QWgDat{am3C9a4b)DdY zk!)ybUz)CC2h1_E!aC!#Wm${|*5{@wTmN>r7h#!a`;<+DESr9PNO^Z&OZ4PYw zv)ZOtu8$SdUf!W!rm;D@25ZwOUAVS9a3$k5SOa|ujUY$YDnnD!_s6S_|JlEhFM5>f zv`snZQo>`0=FfiN7}}_!Xu-9;D_}4mvo4l3VI%NWGpM9(B7`m6+st{47C&K1Psn?Z z%Ft^QR*RyiM7Ykda~SBGsjSPOel&k`C_9d9xnLA8mpXaNUDx6uDWrGCQTaE{fNTn{ zHW^x{s!8a%d!i(+LpU++AC_2Gp;l&9q4p-Ta`&@d=%wR?)0>c2tDC3p@m+7Y?U3G? zGh+e*V0a-XV?md~uGT{LwQ#$S{{p;Pg}#ilu0J4DX`_ zUQRCBZy$L&U_~`kg@~p5+1`j7Mag6FMu6*4hlopAQ zzCLp*9BmD1#xpRO{?-5%2e(ee_e?f^Ir6@H6}^J(89x+xiLH~OzR|RNHnuq!KQ(_V z0zvuu^?MVhLp5dMF7jYrx>TNuzUzxoJCy||0x9fHlSv2<&nndXO_CK%b8wIz2rNjF zzvwR;xw{859z6ywP0=M#Ff1o#N(K-!Y@Zy@bOl@WDG<+ZZSkdCGfO;ec{Q7fPw?~B z%@(uZG?JP|1ZgFdu#872;)DM~j^Bk7h<{!L4C1>RGc;t)EQcP39!kc==ZCj!A#DG8Z!W>U%wA1#y#4Nlb z%qe?@U4%q`O;IWtvR{3TwTY63OBx(}x9eP97(bba)4kMbBkn0QAc;GUfNvfgmVFY7 zih@eLE^M5O2H-kz+cKz#mDYoDMNVyx7~X1IGCr-(^qOOW;~_D(2#5(O27ep21%Td; z&tdXurxidc6ue>HcdIA=25^YXPPkyp5)v7PTaV3MUm7<$>!pj=x33&-{M#Q}YMsVc zzIo%zR|(Pifi!P!UQPYU%iVxJih_c!*UX)rKYjc^0v;(LA0JEQsa40%Cm9p3v)y72kSP7+rV6&Dg0oFWQ?m9Vzktaz5D@f$?=(z`G z?Avz)=;9Xl?L9oIaknhRv7Ai}ZJ~N5WzX9BIt8uY^$EC)Vg@h_bIZ%YxjK1moR1!L z@6LPr`-`=-^cdu%Mc=j)m9A=iKtnzA;sGlg>pqU6^)jA;qUYk`0@uM;x^ZcdPlIk2 zxX%(>g2wm4n2I8?V?6&QaEh^)Rpm~#+Wei&fMhKMx|Nh0MbJAmyd5q~Qn))1Rt8Xd z=Dp}qFhI{iGj0hCB*MM|InBZ5AO($`*-Cn} zn6QQ1m(6Ri2Q#W$aQo+%F0@HJZv&)pUHC1R7XdW=4+D3-oXcWj5JDo91P1=Dd zI_{63kd?LxFri;yx8nT|QGX?Iqx@P`jFtZnVO7=iv7`^mV+VFC6a_Dv>5-F@;4_z5 z7N53roleDFs)N0m)9r|%w5Z!Hk3uu3rY}DO?{1yypw`w|i`+dR z&a~thF0VsQq}aqUz3uf*@zIoCCiAavsC-189ujZ^IvB(eB(ZnYZu8LJ?Nz^q(@Ugb z7{F~#BYlD^zU0nDhkb%F%wOdcchO1Idolz#yV$-ZnwK5NRHz~8;vkx41@3lmh^F4N z9pq+(Q=u(M_)@e+u&?Hx=#I454UMZ~Jm072GNG?rY9=o}R_KUK)*oXjA?F3e*H@G= z3YryM2nC{Vf*-IN)v019$$K;bU2ETc{Kb}m^0fX!T2%D*#E)Jt-19bx5)rQNbC#Y- z6SOli_*8!EzT4<&{nQ244z2rVXNEJ@c=l`>UBrC(kN0XXYJyQhxWE!TukcP>OqTau&47(QhFE&i=1^gG9tb>9K+Y13i?&>x&#{|BP#~kRk z09ue^z9!SIl5$N*7-0G5`_s<#=eplPC?WR^NI95)+kzs$iq_b2($!ij5_6kL5kLU1 zt`m1-R?+kP#CZMHol#~;8-qZAQ|5gKSovW*H|}n2@+OyOgTFsU*5E;m>i=5ouW`)+ z#z(`QyX0%mfA*N*y}^w<=*y%0a> z@Tmv!Z^kcjQoD&J`EHKg$c zyt#zK7_l21A4b*hI#Wi8pMLOtXk+*EBR7mR*lf-$^h}zLM57%;xQX5zLy&r#0s2b~iW>sg1q^r~T3+;*+C3H)fVv@xLA{qiY6|*nwsBAukPSiUbIfi&gyeAh%98YND_#Tou$m z&8HFJuenC6uFOhMfgbXnH59IGe&e|dCNLCOOGJX#rY4!pAolUYRTCAgUi-IDJ=3(w zRZmvd7U181PyGtYI@=c~3f`AV#@rm+Yh6SQ@^X0501@Qj+(H}myaf_} zew!`HfS0yGvW#8phK#oMNtVo4prKo{M230;Is1^!3+j+Zta(z8BqSIV6{m@;0eVwZ zT3TCM0UFn5Z+@_Fa;nB20o0h`hYoRtnLW%9^rg1+Qh_4M%B3?Z)ipKalaoD`Zxkk< z|MS9l66yTJR6L|Q_I7u7Z*cGSPcpKwupn%W`IWhi1gOVt*DG>zwC@rz#W5qmgo;XP zmWnELOL|p4{8Gf(LLRmQn{MD8t~i9FqC%fNeJbPE=$1GKT1;I3o_IK$^@#A+t?18$ zT2SB!6amv4j_cuR{-IM6@hFUE315#2S9hC&X02-6A%C{e#2p3f$;B7fr=<~ zN>LQBf}}h0Z!O>g4k@^e!3)q7@BRgivWV|q4hvhda+T8@e^2jmtoQR`TT$=rFN=Xo zM@L?`1uS6Z8T=j`+@G_}CfWs7_~hi|ID88TJ|6U7PtXYfK5BZQ2>v(LZ7T-{hsxkf z|F5cSvKK%B`B%^;56cU6xe*qoQ&jF^P`H;6A`zba{0$2-2lERBzjL}04v-zNp=L>5 zszZvui>4(cdj&?_TwUSyZt@CXG?pox3S2)83G?fwVcwNt;gW#pE#b2}LU9cg#PY7ED%%UwpyOJ3II}1}!%ad05T!u?**j^sT}jgTyBMkfg(pj{=O$ zv8}rBFbUj7N$T=Q3}TL&9|Q>>1Z-~x#qhGSJW34(XLBWtLYVMoLf9AuLxS@P%GZkD z6g2qmll}Q4W~z6wJ*5l`m5vlL|c8d5NG%#VEKxLTmN3ui3NtD6nQA*3)d0^>S{N5#v}izBd05`R(i` zyMpYJQs(1g)sLN9RU%kxZpcbs8r<&ZROev(`epxSP$VyHSqQR+FOd<`Udzr*Zzf7` z7;)cKwC%uwV@!)<1J`XiCnXYkyaHp%77 zi$w0@<&$ar9oJ*VvvripT^j3bc8KpeM{^6aFQU=p*R^Y0edsIKi+BCi+>+Pj9~|yf z#~WVPCBA+itHgEY!^gz%J8`Mpdl% z745-z5b&kjFO<&gmA)43&P6PJ`t=IO-eP!R6ZpPFuLRQJQ4ZLoRP%xHbAR2s6K6oY z5pgq=TI9>b&$r?KrOh2I{Th6;{)wkr>=J&tN^WZ|Fwaal_j%yEev<%^*utD={Yjaf^{~asQ#9`R4%Q0Of z-K!|)R>R015K24UVrbegYs$<@<_ZdF_!N)Gz&w zv{>~<&gPBX4`xh?@@iOoO%2zfK3KRSN@`aXk3uTt+SdD^4L{T*L^|pl+yq{sby9t) zVJpTU<1+oP>d+Oz=?sWukCHNhk={Lwu9xJht|7DS z&V2InR@K__b=G?-_8s=@I%~|1uPheli1_()N(@U|TN{nIPwC3ON-B3Q@Pw&o@sQ)C zv1!h%^H^@Xv?)BhIPH(p($t)>m)3KIGn)Dj_^%f!S#^wt1lM+7Enl~}S)`$ZG&1Nl zT3hdjd1isNc*~`a4dJL=Vxrau(5`SgGc$wxnFjTlA@j)-MUmS}V`D5&{lRx)5@lOvL z>QBNAJHLL{6&WciMw+aG)$mIYeU?+E!(pJB7R~8&LI{Ip)T_2?_|&<@dx7P2uI9pW zAU~)H)na}J-*Z5WTEzG77(^fy#qmAB6&=HzpnW@E67_c5aTo37SF4 zR35WFU_OIdBcbNr5TS02P|B~O*He4_T2|*vP$Dy~Kv6aUOJ}Lmm79}OL^G8~7z}XZ zc| zm%{Y8&B}YoLwp++uf#I(KsuX@Rle!nilU-QBt5{Xm@n<_Fa-=5QfJLW=cq0yfX`>U;|-3OiF2Yb^-)QW?Ju%8@v zf-yDmNI#&MiwU`!@E^A6iRO)^+teRFpT>sbwvB!C)LbZlQLmsYoYoC6hBKOYq0jZ2 zFidXAuE8(V#;&I8G2W-96UaEgbO;Lbmjq;qX_xDz()=dnky1PY2slspf;E^{@7%>U z6R?;yzZ^}FnHj6D(DnVzhbFYjsCKp$y%@i*v-}fXIXcpwspy&HeJy3dLlN6fO z=FzMWv>a_0FRv(YwtLXBq@8vP0RFD^*Hht<`qu{(Ut_tlDIyCQO0Gpo+`GHQE<`hl zVr!=Xb zLO|jn{J>m5A6Xq|`gJ4a+(+1Da)o|**3rJOAaX7U>I0qlVc-~Z|K8;42;)g<@7B=& z@F`(;O93|3%8^w6^|(*7jWvha?$TY|tzd8DLCQ#H8FRTt-M=4l4b2=ti6d_+rrleZ6389X5J)2{9hfKtC{vzla$<~ z5f{?#aRLk`Qe&+MaFT9)|C2&Op&O`ZuZ)WHFpYE}zb z0OnpTmLa?|qRhw@Xwl2cH7e5sR^IFX+MJdjw#_caUKZJTTloE^l3$_Tp87jVq(R#Ay}z`=+1i5f3g$r14_yWxw2Q&kT8wSy zm#>}tV4F%yl~;Ht7tNv$<|QOcezq#^Zm0qwtf1r3?D_n&41Fl$XP1Mvg7TzSPrw%` zQX|Cc?l&^6u%aI+GnX%$s~R#)kHar$a!9(GGg>cmXd2D5W( z@Y>VUaW50S6O}tgaA0v`vu5E{q^fE_c!r(zwY+* z_kT*dOQH3=&T0}&r4rH)^L%i&GiPUAy%&U}G@&t6>>t72MHHCr+yQR;x2jgVqdp~( z0?ng1SZf3G>x!K6*fPQas07R zF;_!{8v#|YJuz?^aSYYYg5MiW`A07f$x^uBAEhr=NkRBE*pvhK4WNf0Ggkx>I!7w8 z-=hw)p!p`fH3AgsP%BS2c@n(! z+c^OjV^p%9p51V&&|RolCk*h&qs=b2ebXxtktCa6#i3NSW!tfrTN)D#OIMS{d?_bI zbEFH66Yu$!tYnjbOkheFJZ~swJaq4MNi0C_mALEDatk+lWhLCr^v-4Y& z)bWrw9!+~iL%U>iUH(s6h*G=pSh9ZWIzIQWPf&Ug33tkRrY%&~XnCQ)KP;b`?V-TZ zd9!oirbsy>heM6){*-dy(VyT!G%@5wwnn(>l8?dZKnVfKeD?JbZ#kn&0370G9fCLA z9;k3P{h;SRAuIs#oi5tf8zz6I=nsu)N?e|EK5dnl4;;Zo%ka_;pN^_vwcN(9n&$u? z5L|lhd#$P}oFdcYR@Cfs$M_JCUKJ^w9ok%<_76tTa&j^AYays+kpVzPE?Lie=UsQ( z+Rc6tiq`iw`2hJtceLLnx}*&_y)bvj(F-zS&JG5RRnzO305=!{`J#+-$*;>{bJhUC z<}EA;`KXU3S1&S6!l$QT|CRzh*}nWvITTJB09aWa@HG}&wqfjO=I8%i=`^Pcc&adX z;iKGKQxUfnt?vbq29{t{xY2&G$=BRQL9qnLfs=oNkEbSP;p&-#d3j<(S*;X>8mVFz zbYNQq?#FB=`{l-oi2ZfxUbw3*@d!*=d3*n0u)or7`~eLq8!2E6P5<2?WuS>RyRZx^ zxEv8r11hlLe;Evn~^1*Pnd0 zBTAj39l(SWpNR>Bz>-Vs;XmL!rG(cbWGjU0urJcT9f7j2s9920f$EC93E#oaT1^XL z4i6?zG4&_)ej{&8m!ka+g#d2oA>t3Wgi1;`QWKIJo;ZSqm}cFj5gj@TTm&j-t=&z}i^9e?U8WnypzxxZ2;1Fl3jK;QlLt)X$dX{0COcVeaqs@$y6 z1n@8!z>{1HCohLUl$Ggt129Ul47BlcHS2&?O%f^h0T2V<+31>IU$KJo`uHo$xLzc9 z=8cFCYs11u+?ZvsGMk9EFuyYbjI#|b?T{t{WKUGEyd3XgWGz|9bSd@A9DzbQVf9oXX&I#3 zKhsE%0Zt6;`jIwqc;A|ARIVw=*;$|dCd4jl-GqGSpCJ`eHBJ8bBHhz`3}*a&0Sw5- zOM0P`NBXBx0rC}sIT9LiC4|J|nv49;RywI)nxC|fG(TqcDBBt(?ae*md4&AOg(Llw z2Jzxi+`M5fRIhrwp|%h*?~latay+@cLZU{j=_XSV$ja#Izq*O8wII^OaabxO98?4n}-45 zDR83(ULI@&M=KM3NNuO|FT)e8r`H4kyJ8yDTieFId?N3BW%1Q&(5Y%q5@(d{}HI;Y7vb`e649Un1pe@wv@; z7zZgmhT`wg>y9ta#*2z+Lmg*4G!35h9wYfdNhU-E?|&kC_1Zpw1rW-aBkMtO2CY8! zY&5~5`vU_mQIU_Y&CY>;<#)2O(^7`x*<~;mK5#lW8t?NBcoL(?C}VDc$VA}o7Wm;C z9oLY}O2`%BE^aIZ`8UyfZm!_EvH*&LjlYKM&mTWNU0=TTaq4O>kJPXl#DK}?++UED84&Y{XNQHZZCSHT%bKR?SOI( z&Chxc5=QkZOkuSuDk@s0rty1w*mk~8(XUkNNFHz`D?cb>-40b{lUG$Gby=!idS?V2 zb34Ncp?jRH9#ID#t8vE?y6ky?XBbIRH974Q_()VsBPlyO0dQ+%fshU{k_G}ow_U&j zm-}gfcCr4y)M$Tv_}bghOHlzWvv)u>;yA-CJJj8M6Uc2Tp8;~^bMVPQFsdvkH@7;D z?+$+s+%Q1rh0kYZX2xk)8|~Q>?f-sAbyZB8Bbj($VBmiQ$;H%+Bk+lm;XoNEPQ)}lcXh=VayAU5eIb-n zS7&`$2-t#^-i}Nl5cV!jk(^>=pEk8DNCBzpf2`lEOD=Q1tUp)>dS1Set3XmZfnU{^ z(YIZl=8taw9h!@7YO*Ps5|cRF;J&y(^zW^N@4r&Jqhy{#JUpxfvg#6(-^P(p@2>nQ zAijXwulOh`o&jeMoQRVg0u-K{0KyDE^_BJwFmhCjV`6*{^0QH2W{{asCIy#rx>WAB zkpFG@K<}Qb^BXup;lm80&UIm~rony^#3ac7lBcw@)uv_dumMwFG6B4F?PF!7b9QHgwTFc<%e=Wgd-&%UJ_Iq(dpC9Ql}t|>7- zgZiX`-6s&6NO0=!J@FEI1P+35yD08BMMN?0COnjv!#$v&pcfvVj&3{pM1WRw69N!Y zO?IPpC{Mct6HbY9aX!Nix=5iEE)}`J7HQB^x&v(ojy>JFB*ugV2rQqK(#+q(;C1@$ zy>Mm*${x^_0>HMN+ZmB&`(Xw2v^O9P9tD&!G3bYEebn#YbJS+B`R=d1F`3`JJclp; z0oOvzre_2AlM#>^e|pKR)-{%!``;@=sX^q7ZEjvy)0OY_)s^ph9IeYROkAs0OOMD+ z?M`KRQ}A~hBQxWIY= zY^VN0lWydn>uCbXJr_29D8lMrtjMk7Ll9-sg(~GRUNVB1*gIsjC{>LxacaSmiYeQQC)4%15P0Jh;`j@TH zpQZYCZnI(g$y-=TeNgsE9ZuE1RV5Pw(Du=lf>YDARw(ymYv z`*HWp0cd2 zvIz*Y<*JXx8x z?cCuZAUiv`3~K~?&;T~ad@p+Xb#qDUxdm$KXEXKHmGV#q?I(bqKp;CyCZR8xy-`<7 zbATye^2z0NkGJLQuc`!ghY1J)1cd4L0X7Aq!n__FBG_4<6&Li?f&zMsd@B-onzDHb zmM&_|h?!@Pdq}V+Jl&-R^zK)IgID|DT;b=B$1Y#Ge9S?Q1`Cl5rn(YB>;wpKjV@^Y zR^Ag@QZVeqfO4ds)e`3ncabX*lmXVr)a^70V9tt=Zwy2lJ#xc1r4>vA1@Hr&mP+d7 z^J%GwNI%e?=2rjlXZV`M6yDFw94qMPb5o}Im>zw;$-};Ke#}v7wtSFTYdv@ec%npY zO<3`|Ti(md3(V0T(9)ZU1$2J5Ep2@8ws&Ann+$;f;g1SVC~pejuJS=y@65dk-5g9x z{BrMB*>ot-Dp%USA^Y}?H06q1@=K^~7VV$EbzoUp=RDY5=j?a;wx;D;&(m1C`yk%V z_4L#?2asO$%5~O&8tgU(ANNmy={)u%;C$=h>0$P$frDDx;m8xf0t*A@TFI%DXB&Ae z0xc)SbqF5_B!M;%_}pIZW8m>q9lPZSb`!AgbX$>0ENJ~sIMen75d3GZ<{sd#|JT~j z_U3HKK~x-%+p)u7Z$@{ND{Mk9Eac)5<8xiFz)q*tuJ^d{X>E;`FPlrZ}N=%6r5iVvvjadr?rx)uC60bO<5npKn0GFMYb#U4nc01m))i z;%Hw^i~ik<_CBvb`Di_{kq3XTOei-V1cT=30(W>-5Z2Py&(EtmviwQ{5pvZeXW^O4 z&(cK?&JFj{^)%Du7Xb(aFdSEB*)bvA&F2a69aJGf5D#*LDCt4@XZ=(mneQu1C7;n0 z@zbdmaZ!^IsDj&qyj6f8V7Yq9DeuT<*D*xT{0n(XD?cqV#QheV{9sg@K=V95uY=>l zpLePuHZ(rh$QG4@hck4%LX@Ug%0S~PD)7@=N5Xx_(AjUupQl>l(y+$|2478 z2fyTnY-|>b59hYE8Sv)rl?Pu2iUo0x@&5uLhXwLMh3cp%a_Nllh=@=hEKM9+^DJ1f zlR3A;rm3b450BrPulM19ePNq#&B6h8l$ZDQ#86d*TnDRg3JX*B_Vz}{#85m^1D%n2u?~l< z5lCeCLewbte}M^~&i#?YmT75eAk2hN5eOmwT>-9#EF6Q4jotzOb`?kwQ+zBea--+& zj+`AbYT@aU7^IujQld$Xb4ZCMcLF*q3$Jkjq{IIbA?%#pe zla}bAyGAmnn>fWXDSCUSMU(l{ZUIAwcNZJ%RzR00&gm}Q9*@2WbTB;wNhKh)UA!_V zVDOvV@$oPSCn2O4{_FM4>}5rAN#g-}hnrlACLsu$f}?a571O0`EZrk^7l@hWt3mKt-P>N8LBn_dVY-J={_DsnwB#9`aG$v%=e6y$;_gcqO{YihGuYhiK5X*tn333XdHLrjTV5{;NWE?v?3|uv{i4cf z+j*nmX&sMR*}0CpGEP-O!MWcKm6j)Zd}nPOPg=ttkXy=kV~X?R`^3Plb088vnN8Zv zzG}4Rn_G0Zl<&gC{QSCt=InaEqdEdCzHk5hQ9v^*VnI>h70ck3o3=JbW{#oQ2>>9N<$=o&WKV$xJ}erMaLP`^J&x9mQu~zc^26 z?_qTh5`8w<+cys*c9HS*O0VdU+T`Gzx>euXw>|BsdnWc?a{>ORP@t!YeXDkXh9iZu zSL-v>hOV4EhZR7aNM}fWvSn*-d>p_NqiO{D&6k4RR>g)Sprp=GNc|lz2MQ>w8R4p@JIX@Tr#NU6Q z=V-=u9xo&Sn6P_^{b%QUd%b?oGz_#4EBSgcZ?)C`__j6k;EIht3_-#F5 zlZ#lvPndr-j&8u*bgBWBI`;BgW7^nbhx_H#DZ@;2q8b-J)kJau!P~n%=M8d5sQs?O zM9-16YtJe33P)?npqBERsasz2#^}NmnNjHlmU@v$P3&CJbNcf;Dt6~icE+Ieu2RMz z=BJibpB()6yPvw$b>qg3Wn{q)q<6Ly7q<((o|u|?-l)xry~EmO{WT*eZ1;dB`Z&+> zQx#96WEMKu5IPeF(M@T7prChRVBMb8JRG~t=f~djFv+5B`Xtc$=S2d-XV$M9E3>{l zZaX}qkQn^6I=Km(U>)6wbC$VSwVIG(T5%wkf0u5v6*l}uUaor zvnp?gAMXwUpQR_GylbBH--sIP_5WgqVdWYwfg~?)`gOACyB(W9v;E7wYNmwhM0_`+B#Qfpyf!X)9E=P~v8p zRrnA>)AHf|(L;I{^bwVXVgc7*KwZN@k| zJ^H=%Ls+N_lVADQI*o9fg;uMtZ`nX6T!S6TY5P5koI6J3Je_x9 z=jy7o<0-k`%zV75FL0k3=77W_$s+7SX<{#bi_Y1=g2$pUS>7tJpQ(Z15D+ zQ$}@-srX&{-@SY17IX(ROTz!4*365DT#PzVS63(N_Fdn@o2kKsLC{Loxth)^hV3(nf*rtZfJQuv+56@H7|C1>wPXJEy!KKb)<%+{{&48EJq` zh}a{#VYaf0)H8rK-=s2CvdX-wp z3g`zcmb-WEsIh5lYwxv9?+cAxsWW|4U*Fo@I4LGZTuT~FHQyr`FpARNk#&nl9-Ap& zC#u|P@vpT2k<@AG`w2B$=$bMtU(QazB=9#M7ZmUfH9xLE5dSW?dZ(W@ z2JH&)ZkIqfilIRzMZ>3<#d7oQcbX?v+-0E4zWmqBGZP`dx3^8nQDZ4K)5C`e4f6cT z3cL#wJMSXBh~nKWp55h_mF0)`X3Q-6L;3PLWG^%|lvO%1ygy2!u>lloIi6=2+E#39 zMwbC52O;1RD{B=QsUhzRsir=mNKMJAN;)B-+Y|E7e$Pj@=Gw`+jT&EuZb!{$ftK=+ z%RkD|uInAjyzz0_PF~-hvYvUz4?S%O;@Ts@ot;)ta0}n9uU{>1eC^tz@gWoK7gY9) zv?}qaeGs5yB6L)2%E&!yq`qh7W0(7yN*eEb$g8#&smJT2B%+~x zA`qJ8)VD}^)>>{U3-iUy%;ZKd!4b#;1U04{|FmYogW#ycQ=wr$U#?kD;Ns)c7TWM= zpo!Zl=*r**#iFIHZ=N>aJn`WTf!4u7|tDh(+3EJD4^aCtSvKL&8+V zYQamx5K;O1y8F{q`6d|AT|!#=U&8eG|x4%*Asf7ls8VUY+Iuh!MQm zO0Zf_d&M_rLAf|i|Jg>AuM2BS=ei==o1njs%+#nqXOjPpH9K}( zmQJePx0d7gmu*o_lY=&({-$(;jZosH(=69tI6bbWw`_KG1{cFZ4!CoY}RMULxtvBWpr0?E! z99n6lhwkL00y#4h#bagJH4m1Q-TG$VnX0Ctp}1kg&BF#A%+XqxV0O68l~;v@85q9$ zP&AoWtazi~^^+Uvt=MAOz9KOnk$v4a&@^^O&n52JA8yq}GUg7NW!P@4%D7tX- zvoogcIk7C8R!!2Abbap{BYFzy(gm|Texm&4QGbjG_tfuEn z>%ks|s^s;Ut*YWtm`hr{e!UtNczXsx;Gs#uzmy;Btkkr#myXzIXuQ<A)Py06~SYYU@ zrl#ge4wjxnmZ*Umi`qMKj@D{m;m04JIyL^YQM(Va-K{0wdACRPv{fvQGsGi>R9Yt` zRi!jqeCiCOfk36G>-w=utn1dRS6YyTT^-nZnQNo|SRBl|E!*e$GM(y4-w~0AkIK; z#XYcfTiMLPi$RK|Y|J|DU~x*N8x(?*ecszp>IvL|c0HQi#4Ns-Q zwu@N9oLM@H949Wu&BkT}g75}&4fQ4g0c=}d#>&WZ<~W>dqfEq7W)sDX~6zM|x^1!>w%&++p)aj%kf$`g@*eufFoQ^4`98M(y_d z_Ng37*s#abQ`E-h+~IZi7?_VhZ-jy(7Cra0iSVCnIyw%?IXTcQ?C3f4vcZ2x>Hdby zv_>M0+FAG^;L)QP-@E9rWAuj?Spg`k^@rNdc+K_@;syHsh9xJ1&_@4yPUF{XZt)bR zo2jm90*S1ktED96W>i>0h2n=lxDj{vI(&M$LVl$4%r1B#SVgSd{WbUA&lMfoZ2fkw z=GCL66iK6-7qRocvt_EVD3oVZ2$UYYd=Uji2s7lT=-tQ>o+=^4_$Z=T;P6M&sai)=4JTX#P^+L`+Mtl+FFVVuPWM z<_{RlU47fmdMPE~0p0NoneALrh*ajn^Da;g1?I7ljF%*#)mH*n-BcNFFa zUxd0J_AL?&mp@4fNXcAOF6&1XxpqtjB?@H*97iG^_an=-2fJFd76Wm7Px z{nBi-+rr4}PghSLj&{RF`52dw8chf0Rh$>EUpK8u@JV<6dYJuvW7(6OiHW4$KE-AC zeeu58*^dsNY-fOy;!{!Hu7O@ls4 zaPl8NG}c8sS62QqZ_n{#C}=o!C-G6nq2QXDZ5ZQnj*pLVSm z77?)sLHz5_VHRFCD&6A30ij)cG1wT3Y2(&2KNbA^v|isD_&tsNtt7BcR7hm$_Jz^mJJ(J<&tI3~(DK1P zQR?Y$|JLspz5hsFoMu)1rDEJV8p4?Y=(e}9sZn8_yxuXaqwe+a$iHo5Y|Mf>1@ppvfKgL3GxL<)pkI@2Tyk?8 z3+RM-caJbS+=Fa0(8jT`F}C%h&be2wvXM#QJo~O}qHTTRA{Ojzd35fAOMor&Ux< zTXuH#l3~36l4V6!(lT&-4(^{{aiQpEFz>5&x05|dv_W$ZKQ8lcC|L-q;d}Jxkt6i2 z)od{qpsA%rYq@(ru6{@-B%Lwcb!xBrbPBfk1+xfnRQmY%fCweh9lX0~Kh^^p7P%c{ z152RKDxR9kde+9~g+!8%x%-=KFc1uCKZe z*+Y+UFukHv?gfTdszP~Ay}hkQIhnCw+f{hPtu26J*UcxtX(nX0-bRyqxyocnmM!1b7u zEnG;%iZ68AO-$%bwXU*JnIDzJRG-a}B^u$E*_5~_Qb~Dd8qHA;Y1kbis%Dz|`&O+HaLUPeKi^E*SgK1`(oCUH_cQJOv9@|h6^KGnQ(4gj>%_~q zC%O;yWT=a%9k5|A_d0AOih^LhUVKsJ*-%4amlKZ_lT<&bw8#&EAsMZC(EU$|h;tBg z$SKiCs%DBqQJ$lQZ&9LgTE<4O#Mys!^vJ!1z7-@Kb&edtz~;`p-?l_LZLie!0>jGy zl4Q^5=Ir;cFBu!VY6ezMtYq(w91ynvf=Ld0D@F{llm$Nq2}jnxbAHqv?Ibp?&%b9~ zRi2F$q+x63(Jk`N_$3+=ZUvZC6#5MZVRy)~?(;Mbsyc-dlrz`;+^PTWMw!8b6iUbv zUWGezMm|s^{81mXh;q+)x#f-|aeF!6i`lD3gKN7{ntmFViofXc@i|>l?GyEqgNUh| zj6p&%QBk6QnZR6N0)NAXXEp43#$5fY>+1unFJHclC_GS4n;lKoefZt{9YW%F62Q!-xCsff_%ObSPzIPRHyHQ4#OlJ>85^nYGWRVi{Gy;OTT zn!j=d#4mT#!6r?%uPNebNx>^cRdX z^K$TqmsSkT(_=MCsvV$>B;P5%4I4Hr83+58D4jSL7~XrGnu*&={aEr?ZCxvAhtkgs0moZg?XrvN z9G$(ArX5!gV_rxm~mO3lLFc zJmt>H=uA(H`kUnZF7x;Ie|@t`vAn-@*3JE_CEfFMhbxYC4QoXPm+SXroPF`4c)V-$ z$`T5Qj{r`s_S93ovfX=1E-JaFpDk{PTe-k7|R-|^!qD-Ai7Wc)N|8K%Dv=VYe67fD_y^vqzU$T)XXE>_Nl z#2@Ur?Kt5VI}(BZ)PnZZEdS~Wrq+Uob)+^DNGG=~w{WDL9;rg8>;RyQ}TCKZ*R=5LNZmk%qJtA2jzfXN% z$V(3IKQo`~>z*p1dhG-CD0mi1@K#TpPF=wS#ig&JJ^<5DrTr!le+B=1o?L?%&?~~w6Q~YxvJPgQ1`X(nH+M`oGbM}xop31218=;!Ab*a~VJl#KVhqSmO`hN@$u;GKek z6XeXPoseK?J~TqWe`W~9z z!EGin7y@XQ9%G7Yu#)f}gw=VlX}y}#x^)atks?>C-$y*EcBHd)sxvM%6WInSDuurE z^<8+VmqG>?bUMh?!Z0MGxJ89TLLoHO!Nok(Q?^rFJ5dq}f&GjhK762Qj9+3MHP}-P z=M;q2CmOnuNlBZib2BrahyY+-UY=&0j)^m>yYiEVoeSazgov>}CM-IL_Q5>?O6FAN z77z%RKie#%pwP+z{JXAi9IopmQdWWcA6#yVQs<&aZxD~d zLh#A_w2b#Iqp!=;29Ec&H>aWcsYBX4JZu`)mx{i*;gy-S3 z64m9q+NwhVv|C%aYz^x+R{q#G>+(I8(esQ_D4dDT9!)^a}2sEOLy>uxH|2lI^^$dAXS2XYI#vYKW zKC5w;-P6;PcZ>DanmD-T0%N4Crtf|>?+HS%xB>ykJB5Yq97-NzLJ$ti4_J!3yAd=> zuafTf%{I?KCeOGQ6LSgvMZX=r{B;{Qf?a-*WWQajJ4V3=71I@0R|&QV1mP+OND-A~_eL_Dt=VV2xL&#;;RSGI)FF(4jA_d~lB|f!zyBNbF^U zEHVE*-THkJ$=$&F%jK-*r8``#l1qE>QwGjFq{z5@Il$c_lx&Z5^uOy@?2$b&71Y!PW}AZA1@EViW*3sp;Hej$U>nJ+%#w&2ex?CN+CQuetR#b zW{at$vB^oUf7%08r{&BL)fL=F;=4|PO$rjma%E-ZCF5|K$+@}5*lx+6&kfP#q|fw!J9BB5*(KvmK^lu= z-)y6A2nnC8l1Nfs@-Sz#YrpN*#YJwPPqk$DPy#YD7o9(^W&KR!$k8(xucx0sxH_C_duEooA3plNFrPG8Y$*;&nl2VD)Wz{@0HR|BcB z(0h7JZM&KVvc&U-239m~V)QL7A9Rf~Be!0I)eSE^a+_g}`=BXT=-J;JNHa>kJa#p`kpR7<(Cm7Dep*16u8_%6(z8 zAeXT+wd2pC#1C@()#Mx5li6|#iD*rIeXh4Hlv_bgV!5aLX*PA6L$|ty#wDU?5`khA z94T^IgSU_{plf6D{IuNiUL=V!D0;~(*(fSfpwa8D2&ycFcu9;*>=Xj;5@-+Xb zkN)nzGJ* z0KpxBXZdi!2_ec2ql;^=bO*LiUD|7M#i z&_6R$tETMUkcD0pm{LtEygWE9G;ZN zN6q(|d?d8hzEc7G!VqK-$RVcRwYE|L4&S+VuW}>{wTT&ua<{&Bcy9!=Xd`v=R`}TDQK7Y!D?!^J5RmdztI)4sjt5zZJqhOTk|GZx zlDgx!qcr}@Xgx!rD1YYA6bq>R{vc@}rLrb?rReIhf2PZHu_*Z{htx5dhlCkSf z=0~+%^WM4~JsK@WV4}4W61z8iey%)u2=0d{eBOTyx1_IMLoW=`t`?>hDCr{{Y5kNM26 zkkE7KV+IDH*b_P}vAcf>j45y`fNrtSjR`jm1cN!(kxoTuQSUd{5N(b~O6sfrRYa`9 zeyBJuHE$z|ThtTuSr9zRzPg#+bw81+a5FUIXX0REKa{9U8_<;$7N*lnIYeTx^L^q7 z+o1*pPrwFuE5cEQ)daL4PlmeMvMs0)5j+y>2cyW|Q|pbi9&JDx zm;w*XUTGo6$1Qd8BJ`iOCgr)08WX!Q-)vQ{ZfbhF&RLAaveH_Fd+hfEJCd_93eu^k zUNV!-HcCW}73~V9W^YEO&7zVAB3fJZ$*{@9tP&ghr*Hs6m++ss+lD5l$Hpp4(vGt_ zwN1VGvZm0sqp2vrs8oQR{X|x&(R5>i&D-}S02WQZy*~}iUD9oX^k!c#$%tvNHYi{=hbCCO7J2ah^&h&V4s$zDYN=|3b7Ux;8K zRA5bA?+bz%%q1%O{E`XNxLY>?u7mSMB2r?|v*bX@eiV?{BeW#z_3Hz~trTtp0uPvl z?diOR6cJ5OWgP7^KwODJQD08k#Mh=Xy4DK_wa}ZXy=5pnR+Vs z)H_U8)0A-cI629}D@ss3y}eJ3_f}oiD#%@~<8Zg}o8xf1yqctxRQQlJ8~dd2@eD&q z8z_D6jZpr6geAuv?S<_H#EwgdYLczH#8Ozhy|-@)FdS(!(3KS4G*L=nqluPH0z?e3NK=Oi_8t7DHjwF zKyl)L7ODM($zh-i6|-(Mc;g&TW<8O3WRDEWTc*v9zxh}(?+|h0=FQ8PyHfj(AuB{# z>{sgd%H4Ap$gC-;B?&!;?4qI{#@cx`1CdA}6%`}4w1*GXkigWr&`u7APcj_rA5mgn zv%~OhJ&*;+l@a|-*MxFYtgzy&JhCAg|37(vU>Lb}Itn7rQVEBw@*FdVA_p0BJF0MWWkKr9$()Q8dgEBgtJtKi#hHWyaLBw4UMI6 zH}qK$GlT4VD5)Li&_B~TdQ^-G5Ih7DYp%;yg)gN+Q3L02eCYVILqFd6K2$VcV5dnt zeu9ciD^?PDeCHG-_aEW%b3z;9Hdl{yi;Emb#c})gFz_xg;mF&O$U+YO_rT7D^C(&> zA$&}Fra4L(xDroXhmO&P!kD-vP3aC0i*YW*$^hCR*FQwt;pT&)A_ME!uP^U#C<})Z zX;_*`>B5m$Cs%4C?B=AOGYHL}{+ogamRW z0a{QWAh~lc0rsO-l$+aZKYo@#Pc^w7js`vc$Y|Bioy!q&BbBg78m@#nveUp^QuD8L z4sL2XQuEn)f5(4;rzUwjUu~y%yvZ3^M*KPV?k({uApp{>V8>7}p4UGJk;-|yELTva z4vDBwuiuDemB?eC7H!>!_cwx8D%nP4rt5-Y|6-H4(=2wTYQhYxEr zsP+JMG$I70i0r=rTrg)2Dm6SbbzHtmOAOf|H!3oFoGjA+`SWsfB&EH+>+q?XQWCbiGGcM%6NBk> za&jENpT3RlNb#MUYBL&2P;?O|<j5G1S8xoqZr6%c7>1;&FZ65VGTrYfaH+O`rwq~cWr7~6Y-wp}WdDRw z;u>5)E89@YS(=gm59n<$pd)N*sXRq@l!^o zL07qhEYo`t#&Hp``A~n7{?GGxt>z5pjA#q9VTjdwvYMuPP6n zr>1H%>N+iStj473y_6ItbDL+Yc%%){DHCbA25UTYh`SQblU&|^Jf~fWaRm0E7WrXu zI~AK)D^bNTmKcw|M-4#Te#-kPv?2+=@djChUQ}G%nc*RZh3pn{Dq=V4grl4cgHktM z*Ah!kPA0L_QN4eAZgqEV3r`5umL5BF+jFa0L%afk+!jqJb|{LIJMKzEhjSJ$UL1M- z`dZS>$oSCIbo%()=Baq3IY5Yalj17|t{?MA2-YIb0mHmG!^&PrChWTkgJ2EI&9Q5V z;0gD1r9Cy93@j`y8Qbwk#cuo5o0frG*+($kTcUF3z8-#!^N08>X&St0XvizlkhOTl z>4D7`v@AT*0}E=m+O`HDKy|2W#eOEzWMB+R?i$#(ZQCi(8v|p3$#6^RKR@M5EX8@l z<71)7+G)aEFx(f&7)cfAP9!e7y@nodZAtvrlDWp%;}bpe>a7PZqBoJ6V*39bTahc0 zmj61-B{#bXB2T78DzQ+|JWpn;jswwfvAz){U!b%C&k_qX*Yw7u!>TyW+{EcwU$Q3a2gI#f|n* z)@};L?klLcFq*?l1s!-PzXf^RQ-JwCzsIZ5jODBZiciH>#wGt8AJ$OPAl}lk~EOwYVvUa3!#O&UfN0G9u$VAU7^`vB>gfr znoD&U#Q^Ws#6)T8hnAK;tQaHd%1P{*8T#wu+_V5rwXuxgMl!UWInF{%x zBxO&zrOdoSVDt8~0K-d7OIum_{{8z(3=>g1cinwQ8Y-7Cuv7_}7SVaFIc>N%q;nK~ zlYG~q7@@708A5m`*bhpSy{@kE%^60ay}jmYhNy(zCGUyFu+mzNeZd_yuwMhE%~%%- zxYgCG6y5|uW4OCyHIUgreSiU|QkuRwZDeiy$Vqh?%)LEXDnRY-AuF zR_x3$gcurfVa|knOC06#iNEIfLR&Z}YyydCrFK-Y7%UDJRNLob{#r& zg~+eFo`+w}u`8>l*2v=_Secz(@c@+v%)T);++q?93tCvI!KdS?fgNI8cuttHQi_2z+ zEo`QVDB;eN4HW<<)L(_o%KaM*p<(Pc-AD{#(hZF^{USFhqGXwFox@#jYZJ9KXK zn>TM-3tiKM#BbDP0of>@4SZCiG=^IGk(aG1b5Qi&2uJ{r=?y@*Fop zvs(l!pWKp7#bb*I@K$pL-eN;@9_rkk$>g;5WszuD|msV3#bK4eRWodaC$w*|MZ9_snm6^8x)@_dofFoS~RMxB# z`>(^p$O1?sGC2|Z1RFCBN7(7OC67oF?4GteT|f4pljP&$dpk5qBNgJdk)C$Vq|Mp0 z^ux58N!a-=GP~r`Qe(DTD&AZBoeN0fq4>Wm)H(L=+qWE*m?#w+-_2{UiJ=mFEv)14 z;De3R{H!af1P_wxzUDJ%jm?=R5wPFoz4%Ey;BbdWPKY|+NKcn6QY-?10DnBm38P3I z1wQ&Rc#R+R!2@x%P5{IW$`i~S931@Z^#IJikC?01Q5OJKlYG?qDlx=@+w|{Bo=PpUGFW?s z8lU->2$7+w-ozcEz6iCP+UwkPN;>W_uoG(u08KCG58*9G;;;Z2=CdRV6-v=5)Qa6N zrwn7~R0fA}`G>zmi;IhkC@P*n)1AtH_nr0kmzB2F4?7AF<~hMo+-=n&8mw*L5evE(Egw;ItvXCufh#W zh@lk{C`Gx5yhoAeQZy0wqTQ`|f_-ua;3Ks(lB#YHefVyw49mjeyREXg669x^{{8+h175 zdIpAofNMeK3C59TAfj4;ykBIuQ@Qa|?)x3}lyB^8SRPm8pe~ zkY5w%SpUB+^gji>*$Wy}?7XJK#jL}{BDvcV8-8R~g~#M#-$cBkOa9lL?5^H4g;+!B zy(m~=xsgR!$N{$1hml{t@9*s+xPZbw_Q<~}DclHYb&!UGGZ$_uf>IxFmB4(oZD<_X zGBF*a5-%+OpH@bKlKTtWtxSUZLE(ki-HM$%cLwKvz={ni zgIorZZ4oG^`Z0-KHD_{8N}+#gl^65+-o7Xq%->fRMLbDe4~K7q7CkC%q@`JV3<%Pea|<9B9Jf^pZe-JXj06=y(Z!)PrF-AQ zBj)H)1-8Qahq?JhMFtPSoJl4EoHca-ZKjnuE0!)s|6dA~f$xQ3`3giL+FquY1mDj9 z4ujMuqxRJ+4nm#=j=PaYh*C)c83ZZYk}S9iKyV~EjGP1S2=mm2ajDyC-@Ms_&`X2h z|0R_4f%rr}jMBK~bg@+oI7Qx5k{Kpt(qx@CL6Ts$H3_Ik0O(3mv#fS+c4iz zjZJSICxlg^yGokXM^4MdIevXBOu#Q@V0gW2i5d25EWJPnelxbDNqiRsxexOb8;j2E z93fuwi_Pm;*$ebTf6omjj?MrTfUBCam>4}0RyI-z-p|S5#8ecEoSfWu!HJ293WA^U zZMJm0f)ZPio`CNKJmgn=>70FEUql1YbWAv2(P4r*uh%cn{nid&SYYF~_TtE>)Rg_m zTFB854g&3=jo@DjTVwdqy>;i7Ei{y=vOZpsxW<$La&+wQeyAv@=sm!I-SJbb%o*)0 zxPY>o<~svme8cX%?27f+s0hMOAXd1aM|v|!hqQ!rN@9oQ_lP7<(F`z`$#J0CuIa>IW(K%INjKI}?q1co~BO1&hg-1K>iiN6$e=GQMwc$^_P zO_TgT$alPE9PBSTCdRxWQ6(@IMn#q(ifE|&ssjKdtAB&eFJ49>G<(4-EHof@QjG+r zs>@u|D_6Mjl({jlx69S_nNUn(B1?$6$Ui<5+akwmC6XXWwb3o&;H|AI0GEAz@7n&q zE)*!vh!PF|OfGU8?GC5mpV;X@n&yzynV(#mV0~(;$d5}X7o=# zB3+C5|LaBj7?a4&QD4jUW&J=1Z10w)m}+W@;rA^yL2%@eH*apBChYKC6&@Z=N{dxH zcI=>4jQ^?VRcx8O$Z5{TpJ{xs+&*9nFSCN%_t-{)gbs!gB-D<{TW6FD65S-GeebOb zC3sFhrDGJ+g@k}ma=}hkp~KusyR< zR={$dp4)&U%_^4ihHI$#U`*ew&rrQd>ED`=WQ@9>01^)VfdM>0>YzrN7T@0A*VNP` z_Q4Ho6U+s29*n02bL5Q^ah3D4O zP;zi@y*AQv&o%$#o+1Xt(nOb39czH+JwEYHX7hX?puM0JA`(fCjV;=A2rG^_F(2kMCk| z?-?)e;S1L&TC@^RHv`KCm)IeT(G_2N35sh>?eLHINCdWQl7;nwEK96!H5NNTDmjggCDW%uq2@UV(B@(-TDPVj>N77#u(Tn%>2^cz-aW^nj= z)w0yQ_9FL9*t{aN)qTW}n@%WM5OZLM-;MyG>UZwmyuYc#;`Hf{gCElVaxe%Qf}dTM ztUG^;gg_O*FDX|e8^PE!19mNQ>rLRV3#>b8#y1RdGcUc+RWgH>X1p?{0pvCiT9m!L zG?BP3nq&0w^(9kTqO!7CbsY$fe^X_|TWn~bDy6jzMGDg%Jz_NR84v23`~u4q5*OD% zF6^}fuq2KVTc2H#3TAuz^j3!GKeb#z+!3k#W;vP-@XT399R3#WF*y^MUncjsMcu<$ z^*n43P`3L4cy8T?le+PUB4Bc$rmC62Td;F*h)|IP(%L1FL#UQsAO~lADz&{c238Wr zuc*sb>`|nk2YUB#Zs8>Gh#{#j-ZF^xL;pQD=E6 zcAO28e1TrWr@p?$@N3i$&CLi#Is+)b>^t*?ss9=AZb(6K!1x=`9|!1+@t|3EzMkS& z5E4>F4R4r=Nh$QExhrw;&tm~4IibPr9a)wgif6FD_t!apUD1uGprc25 z@$q6@si=(1TYIy|=ujb-)A=Fn%+=AWiyVSXjq0^3Qw`O ziB|EM8x+5t689UNx6M2XkC%+;cT)eho5{t!?d+7KLX!u!JBZ5v1;}=n!6F^CKrS~> zFQE2%>ZAzH9j=yusIv94cSa<4?(7-)Ewz3<;n06wXe*j1Ti4*d(3jK}?7CF#7`x-* z^5`>w9X{eXwEA<2F4aX_mj?++AHy7%!!x2qne_GlFT`)4X7Wny=HPK}GAB ziAjUgN{Wm{v2kX$ARphmrX_{oBxo8+w}d}4<8?;b79LJ{K4@7jF(J$RJOM(PhnoSj zU3XMSf4G7zsj#q6a(5d5K>25MY-z7A@OyiEKl@3s-wWrQUEk1f18vTOc9?3hyK`6* z_c8f2PU(68-|W6xq?rA{*ay5e%dntx^P}5)v=+&cR*9dwRVb#fWP4aB;%g>`$!T>mx11)w>njXzt<`k=nn6c#k`gT!KBUCDTU%mB zl6K4{lZ?|S6k=A)1n7fhZ^b8LA-Id4qN*&*%4P51P&T~K0p$#*0>f?1m(0LHIC$?7 z9c3(3Sui4dtAk+ZuK~9r3qZtL$BrLQVGgIGblJQ4tQx%(Lit1D+=+-QS596sT@=eZ z;rap?%|1=dw@ph`)8$xAMu&zPo0j~txMWt-_S7mm9jaJ*n7%ge`PP_nzpTup7X7=& z4_8pWn(bIN1+%$Ki8A%@&?om^?3SX-1GI^D!Q^<8pvP*FaN|ab@@iJ0QgW((zsL8A z)xCKWja1YGFDX3d=Jx9T{u|e^;_V2&hw8nb)6;ceY*?6@s@4v?xg}wcYip)2nf$V* zhRUz(akv&daS7xZ%}opximCIGMRcCiUGASzeaAF_Okjai>varsX!x3yCEEsO`ZYzI z0g<4!x3XP+i#F~_poRYrBdFn;gB|xfZlU|1{S9A)VfQ& z5vB~Zi4Nm*@C>hSIf)GJ7&uc;v?Wjt#1PxbDe;I0Q=A~%JYVC zuaZ|@r%$Wvt^A72?m@belG0}+AzrU2&kx2%Xm7n0Zgxy{2|CgHfgxRs3$CV%jm^#7 zYGGG;g@KY+q3qM894|oSMpVpYPTjD6{kx{66q-8$Gik#KKfhA+)z8>0q3Ai-+P?n5 zoH)bJN3{UkR8Cnr{WPj_%eSioVjJJTPfiB}yhWZ)rmelB!va;L)|YXQ{)Z`vtEMT2 zl&myjY7$~%wZ)zjTb|KT)L)E^Jt>~3S8@iyLaYJCVwLE5sceSrD1ch2mgATzgh!xI zV)>CZ|G?x&9kfxdcj6I=MktG*5FEXDsosC$&9<1b1-w>FRP?mjr3Z}ikMoD8wYH_%aXPK|I-4oLeo5D%Nr z=eCZRtfSscEwdZWam%fqr;WOBc9=^VcgYvplcN$;;a~1OWNqK}x0Egk@++NeQe5>ucZMPTwpiw-$&B&B8i> z{h-#`sFjN;dw5CJh|`(LUqstlc)a+?_`8oh1{ll~{%gf)Eqn)^Bauop8cYaO!i=yb#h>_C*4#}lL$<9>Mp@s#wxTVyERh9L_{lTx2uI~PzT zJEQCoBXsqKR6JxMk;0-P3mB6*@1<+Uj~=Z>cC_W$B6B@_r{6sb^GVPpX!>yii*}30 zUNHT@l5zd17kJvs5XzqSlc+|)GR~}j0@`+`Q*;z@9@bX}Owes!nb-07G)gB}Le%Zl z)S4d-zoChEvmLIx;5&R>{u9q%W1HZ|p4h&3`tq8u?h+iW`puT1qs`}4Rbfu2pSInD zpQ-ix^=Vn1&~nP#Dj%O89?CawQpT&5`A6`>aIgeVUZ?B5Idd@4WTf*4ET?0CDp2h@DeNSvQ<5>)C{$jSmqZh*!PZE-YOB!*)AcD09WtO{Kqf z6_oa!+y43dGQqd4jFguH*!lQs-s099aTI*3R@22Y!&z7*`HFXSb!nA?6?-Dz@y>%| zlye{#Cfy^R_H{$@Vv70Rb)46*>sR%pfkCt~j+Hdf>x!%b>@hKWEi#f_EG;$lt=>|2 zM2;=;^0lBQ*%kb`hT;;#BMuf|P^_GtL4z^dd)l7ZkS-WjZc2SNc-Atq6%`fPr>a@% zcE}F3WZr-;wy~kF6hv_S`tI%z|AlG)_wU}B&5%O$2X>FVLOV)6XEEjDmsn`NMupC} z;Y}#^mn2Hb@0~{;bqM=yOTn-m-+N`ZeXdI9S^~sTM`zo7*dV5a#%j*9XYf7hF^%7@ zdro{=ALq%-MWKwZ1p(>Atzg!kS8#MJ4Tw34p73bEyX`?7^H(uq*c%*?zlZf2DTrIRJ8Atk-~<;&9uOM0t6 ze!-=@pO&`27F|T*tAuhVfBw`&LUQf#$!C{8ZK6C#fSp+kPtQ$q(uoj|9VkQcX}I*5 zv#Jy}CUl^qT`_Z)-(|02K{;>}6O%NP`rO}N13pnEQopXj!NKcfhiL=$d4w?{0VR4B z=)YeFZn?+UXME4TLx-B1nDYzD%hePyp?OWpxHz_K4~3%IiLP)I9H6$w-o4SvqN2Nv z{bnox$|j7t`^*jJy>D#1dm0?LYhhtw%NK0%m=8E0bt0Bdu8py-u1IurG;xSlj*f@_ zKuDvOctv=Dnb+UWy@Hm8B$`>#_wKDdxPp$7uSl~rv%i*>RPlkA9Nf+{p{n~Nl-+Nu z{q2`9f-dt}zI!M2^zNoG%PhYf8XE1Yrx*3P%8Eki4f34G&dtqLnk}35D#-@^Af|*d zmYm$&L-~ktJFEM~*A1x(30*R?wppBj=Xek9sTTMM42uG}E_MW}3>MShx_WwIxXQJ! zclT4mB4%cRcwGmAi+>~(bN|7E%Wo|8EdgNXh~* z_D;qhN+HI!N>DJjbG78gjg2rN8DW^@K$5>S5Ch#GAjo?HmXvJZS^VxJ`blzIx4NCO zgBt;P!$cUaiUIjP<|;N~vNr|^u+LWSDO-~7i~QI&3txU8{FGYDH#fyVSWFyifQ3>> z6B4|m!`a`z+j=6~lp3vBlwW|b;yFLr@)-^-rh#OX?qzcc$fE)m4$@JQxsfuOI?Et? zM2XSxuXprvx&ZIEtjy%oq?g5;oAO%Ct*z@2_K_P=)4?~}$eJF!OZbe9&HGjcNIp`I?k(D2&t?_ZWE zM;S9g_ME+SO&R4tn5SmECzU^7{EUOc@n-bF%=0GNmj);eI4a=H_TVtv>AF^}ULB2% zyRxhK7g1zb;Cc_mE8IDa3M$V-S@2*uMKy_LY!fp`E<^^fWzZQNSTS#iL{@$7otL9z zS&_tqhWJFxmI8(?Kx5~uof_c-o}pulgKHvPE}vG1+5$I@(!ugR8L>rC5(bw``&{GFMqVb z_kTvkcpd$Z<^Bz=t+C&`OAKleE-^VaUP{ss@$bYTcDeVgKcTC8jc5pT+UGquruv;> z>4hH+{(tnkJqHB%czK`rl)~A0evUq#AH0}C^?GfxD4@1{^f=yvvNKqsjQsvP-N#;r z^8DV%rPAtuCLJ?psu~6d5$^+Pv`F-lJ}an2GLV{AuXLL~IqIXN8u@Ak=d*s(rsLVb z0X1D*Ce(-MkkJ@twp^oMM7QD~+X^&}Q*FPR@?MHp_S8KkO8Lbzk@d~8dK0D3s`N_F z`}ZQ3%)YK)Tp5W(0w(CkKyiODI=%5Mv9Xl%=jqPa*&WRWv*d4WMjm5WczBn7hwauA zxGE@wZ)-a^WM)sz_#ktE0!X7Mf{$*tqRSUYmK8#l09;S(y7M;t*&E9!6b)x>mk7Pz zvGCiaPDysU(DbG5`@A}^VUSKEbTFIvUJI_(ZdIUlezH(zC}(>jx~vDo)zCw zn`kNUH#a3NbAGqA*B##$|e+=u5i zx*rpBf==7BM&sOj8d;sI*k|;?B@|AcCC=IrkJmC>UuaMA05L?=&gIWpwQk+?A3uJW zg+xZGVdS`gaq-f83~x5+?hC;z=Cz`VpDY7$R|PC$6=Y}AkZg1XSBEXeNelh)^JiXL z#lZv~5JXryV^~Zim(E{wPp1<~QIgMHua+(8NL8BRVVIth@<#KEfrf@fah%nNv(Ym6 ztiM7myVRrponvV7=DqH2Uq!9x@sf#ExVt^1r53WYJp4HfVY3JJXOMTKAr(wmRKfh0nmisG+s_F1gpgQpDNZ_zV{JGzuHnoUK>GmLhSe& zh^Pqfdw71emS4_f9J4dHq-u9act3{8?Nh^*Fyb|NWBy~ZH9ijH(j&{-y8K2*anCTO z@-F_Lo@dqWrhxhB<9v#35kQc_liC(5DRu2xv+(%>Zo?b7s<^(s{&V|WYA}u$%s^O< zJvy_&1yy!JLdngzXBwTG_lN6y-@S76ssZDgM}IKwLsdjJBr#+~3pHmXH2(>-s3S9pLvvJj(!2xOmB+(#fwS=DoW$ z)=&s!J6I3WNkqa{fI2cY$x66)ZvzJ4V1m=&AC!IBh(#^Cb@-qo^gYmcS%sbwgxEL2 z9lz681ZIw0x>F&A`18uR6ta=`=EfR_NV0;vX@$udQ z+Ua+FUZj}eUVR);xW|Gn+SuvINgm?IOcO|HM=S%;75ojrY>++o8cND~|7A6s$MBD_ z&?5|JHc$FbyKw8)GOLa*-zMyH?-)VrEaj@L6 z<0f0t@B}Xtl2uYxb~EmSi%y##Cz*OfPC0gMgLqjMXY)YHd9p7a`aHXK;erw5bnM3F@ju(PFP<)q{aSI% z%f~up)%iYBKW0vvhP+(CVGz2lv@k32+cyEtZFIwYl=J%-_m=^or}m{D<{EiwSyR{q_c zWx4fRw(7cRVTo%lwssX>H1<=CR-8615C2!yN54sWiw& zxGin*(RkP0w4_sJm*Vb>56z%Zw0;fXqm@{d z57baodxKZVZTfz1Z;HGmw6Y5AIRqEVth0$-@EX>(mBna+&@#jVfMBxc9Ld=v`j%W+ z1vn(dXD!xbT=A5k2uk)-1;9ErTAtUgTsaxN@#HU0I$Q1FQv>w9e=3FJ5I`+wwtf2z zO!%#u(#EGJ8LGQxHXXIOzyD3M2_W_nt&0(uyR||r2oJ-$cj)__H z*T?T^2d9B(_y=g@EJJw-qD9;$%XcFXzoK)S#)fPTeDWC(ufPs<5~8i(0=?Bkv+HL{ z=eRF>;q#2cGz8^W2VnVE)RNiJKjs~lQZ>mqZ1e=i%6lLP(?mKaEzu+6gFAq9sz~h^ z|CDzCK)F7U>wD|gEn<8Wfyf5pxS-R?YhuF0wm-v)o|!hIu*uokoWKYGOlCA*ECDQy zwfd6q|FfgLKaEYJ^ud7)!2Bxg#|Y+ROh)9+Yr{T(@u`B&&+IPNCsT9t*M#s_K0kH( zbRG>7w2)LM11%S#KdGsQs(h+I|Cj++npsFhgvySR_*1fA9OY z*aUxVksDX?{=IYQghAQ2Xn+Q~9^mNYv;+7ESi8jCSrhNgo;6E}s$71SCBuBr$CiG6 z&aMT*h6f!k`CapvFqeYbOq^dMMz0fPYdVp0`31D+zUR0Y$AnE?hLubObD+>iDZ zQ%tg8lya;G4gs7T_$(rVbF#Df2<-?e z0+2V%hx*&G&0ZveI|{cD4A_i(30-jHEGj^Dk4YS9{)0)2Zr zDy)Bo_G9GbSPJxLFKq0$k?JlsphM$j07bWm^pBU3pHEm&usR8E)Ih%+%pW71Qp+5H zKbsYUE}?}fa5~||o}QlhZ{NO6I~WDX0mG{zTxRFc*Zn)3`Y_)s3dZl2B6^0ivmoLL z{aFMA0D|Q1{O~QUs_FZynSiK^iqsCzwBE1*6NW>m*R{o1uJJ};%*!1?0CGorJ0Pm= z*@D}tsq2KOnFL|VFG61t1GqM6D>{9%q82b5Md_T|=UdJd9y_#ETTf55!nfmAz_asn zE!W1aC$z~VL?qYS;-=lm$gmc-xO5w8aVPStU%eh=bZ2H(RtQ{eyK4%Fuya%CksIy} zU#OzR(xt>h&!<6Xwq%1j_M4>TOO~8LDcpwM&bT?@b|)p*`#(aet?F=4ML!Y_szGOY z?oQ)_JzoPR82GFgg2kA>oghDnWqXbjr;Kmn{J%pdID+Gk0m_(*d zeF}T03aO|Is~IMS)CSqPy&z=$?OuvTo2NJ@x3*?vALemR?cJqY{xJ3&lFI-V048l# zTvW8VGdl(@gQK!CR?+$O5NE`}`7)8ONG`Ku+KUUff=E?%b7>Ok=+$(JY-+D(cFbTj{po8LP z6``ZrKV}td?Ma}jC$ylb8^>BihhSY(DMbOqx5NMPvZ6u*rCeB0n`<-+gT&d37bh!v zf4S(F_GF;EItIM_GaZn(;>9pG;f@VLcgX`N8PN0hx-6jb(#vi!CJ`%?rcSqrTAc(6 z`q+o(QN|4C6hI4X%nlVtfL6p}i|-59ghd9>j1to3 z(B+$z+4^d!CR7Hd9SwjH05Ffv;kxipFoBDtMd8TB?LjjzF${M>Zdk-f&{~<0}2XFqDr`zQd4A7sWZt} z6f+Qm1-c&$;TVP)mgxL^E>@fC123JzNqzNdl_L3zF=&Gcve(s|6bj@Ss-0GQIXQH^ zIA^m#R6YA7LqalJg5wH2C8>c`Byg}ttSE*xP;U%|u8CY}W2|cI79J2CO(cwoOp7QU z_ylnsJNfg9%FES}O=6h@>>v*w1rm}xy@#}!lc!J52p7p|@!#HdDwNB=PZ9{P)ngXJFqx&Da9+ zV47AdEH170NXL3oMHn?AR*1=Y;fE^9^+o5yU*neX?cQMg&Il%+*y1KRhIzbsW|rZu z{x}=Bugp}4sks`#!BkC27SixOo!EkF|5nq~l$2$^oJNltWu@&_k##T&j|cxO$;9aV zyNWNW*)#&JIYi<$gIijkD7NexY{Uys(5G<}UjWt70Ssx_9#5b|q79Rolc7YOLK8Dw zTIxs8!#l+#B~IyWYXH2_*G+_8d8s?RDvG_Z7iVKZ$(1f&!|}=r)9^9k)gMN0{PI{! zxqX!qr*1LwNgFn72-*t^G&{9P+Gj40)Kz2=e=4@f@rt`XjK6h_on1m4?pK-p_3#}D z=CdV|I_NFX49!=BdTT-1C?+H;=);G=oM8|K6+r_ZHyGs^f3LDUNMXzWkSu z!BRY`*oy;W=pJLo$TMf;S;-A1q_?ZBt#6XdP8H1$l~|xo^jrIu+8m=<*eLJN;#~@@ zV=fVLl1$)3pT;z7@s=Z)&u>5(MEf)!W7!!H+>h!#c5Lq_KW+&2yN1CbAtAJ?{!+w5 zp9Dp_B8IhQM02eFc~jz2_@=R|Hg3GdgM%aqunV>jOm{nb_m0K>aw$F@c(T;4=6x-( zJ^b(ju{U1PLU0EmYSz&HbYdX{NBrW&8L}L(Knvmp&fsR{t=s#V_`A(f#Yx7<(peqT z=v4t`vALt7#}+YJ1bE5cqo&s|p?AF-7{y^v+~U!N-^<$#`!Mfs?~{_$+Et6Q2Eon@ zE3jrh*rn*KrNYYTsIRL#1HG0I5Wes%v2fTA)D9neav7ftl~W$w37wfekjakxzW+^K z7d9y%Rr`_j14+ zf=stL&gq`c4^pM&@AW%8zZ8K{R53+FC1YEDP2);CYSVA<_~?*6Mk8+|B!nV+GCHIa z@ttP?&~Mw?_hbcdfhrtxv%8q)cj?NeVa~i%gaX@%DJ4bET1~+n>K)>!^%M*LMApF) zEK319->0oaVEObe@RAJYpm@r(jNh0MQ-#4M0PuzWRlXAGjJ&{)J61e65Ypv%-!?(7 ztF@^4?PmYPC-^cwM^GN!oan$|k6eCxhb(^Q&r4EzHbwh`BF4D4CcI7KblhedfI1;T z4G)(3+s=Ik%y`{7ksMpkfqhV7adm=7=^R$;yqsM=<(H=bhOiMathq$0(w`v9Xixdm~hs3^L7Dr(>=cR=_SdQE=il1u#~$_R7S)DyDdPO_Z~!fvb<{b}NM?o0rtVeS ztf;ape0BflyB$f$c^~o&J;VXc&1rGFK}D)k3LQm#_Ut!FUvGHLY1)Uuiq+mxJksz1eHEHpB$%SY zXWD~m)BfnGaZgu`7*X=(%`SBrd$TYP{gR(bS z+DB!^43K}rQMl~gK^7|2LFoXa{BhtX4X>f9tcEJlEChF%VWGQCOPm$fwQBt|dURO4 zbzo~$#d?#JS+bkX3nHcgCO=LVj!? zz_lm3iu!+u%i>&(+gO8H1yDy6#?H#;6G0aO--_oU9FF95yVAoJ|M67?Kh)w$2d3g` z#R^9#%R~jJrcm0cK-P2#r1^HBCPsT`mV9r}3r<>|pN})fq-{P?G`3rcN!{Ee?Q7@Y z0Nb-;d8U_{2aJIE_G{(WhUt`%2oe}M`?go%;t}EV=W8`v+&&$H9c(qpwT4bIn2-oY zFw7Pz4adOiV%{y-%Tk1QG98NzXq=c)Ei;eJzn@)&Tx&4*eF1SrNZLD&Jz72Pf@s`i z#yrSN$=l8jbuKKPLaV6(FQvsCj`UkY)2BxaSWkWrHiu+gGuQF+<5Bj*=sCSjIxT0*Rs<;n4{Z1+HkJ0 ze-rW%lKj9Ht)L~`MW(l(DL1STKefS~F1yM~m4-3C$I4s+MF!eib8;b$o>BMl#h`03=i5JE&X^UXof{we3%ciZPKxE5f`~1(8Aw90bWh zo3>-Khxnojd_XWrK*g-O_>3yRXCop`-LK?$)AUJH5K!cY6~w~*JpphkF04H;Q0lN* z6c}`L*5;t{=?{EFNvwLADk5e<%VL5(nl2|NTY~CVJi19qggIrZj?OwfyaY@mP&txB zb}3+^>J}6MQAJOeoiIN?qTL-pjXZ7Qx9A}P=HB=HwLX65&#T$>*0D+2zhNB1c@t9b zUGU`L)iLHaP_1EMP$~sq=}2O@DfSKp#y0?t0IIAj$ngV+s?gqo`V{m7E3IP-2a{Fk zznixTHhGM+W*)%H^4;bzf>|vyg+2v+xN{3~stR&ynDyZaJzn>1O_wdh`(c#XEb_rL z81YUbuo^6Lh{QS!A?)Du7Qlw=*Sb2(q&GL$(XoFi?DT%MCE1j%g4r5_k1R;SE=IEK zB)zhMOw)zSmvfqjZWfDx8lIBT6-?a9B=Ays8$CZ!V^n2vKw^v}Ka72jQ{z3Tg-~YL3Y;$#}-G&C46M}*W9Shtj zT(SIX@({@8U|3Z!f)5CIhOM=EJ+TZut3Jx*qEe}2%}ipd}_Rod*slfwRl3f zx8Pt1f}3a5+A1i3RgT*KxAOHHH-@kHnuN%UJorj; zAE3${*mwW$*>O}O2VSz;w(WY-rY1a1th|dbI}{xfg#q#UV`$<`9AuGR# zRV4#Imhxo;bFwND;u7HS1`c8&Yd#NCd7dsVMSz{1WeR?hdYomusv9%0E{h8#OtMrcuhQY*bYO<`fxwK_rrTxiOLWGFNoQ@+ z!OoSmu0j2jA{Ot}r*cXKT$ z`dd)=z|PYqRm8Z-85v-3lO8&B`}V#k?QCtM99IN;g+bwv=v-3}9x~(w7A=MLiAV=n z{a^~z>j2vDj{v?O>P=AQ4A{E;kFCh|nt`LTQnO`RD}xfadAMX3|N7bBV8_x4DKtU$ zKDQm(Kl^cI3ncA^oe^ztT_m`__szDfgPc-jn13adKtzXg*7OJx?A70BcO;& zfo%uC`DJEh6%GiKD`a*Ssm6ad?8ky*kPs?-x;~2L}V8 zK#ULz4A)}ZBD8S%^p0_pwGp;cn(7)I0DbIt^Wiu;h+eHP7h`BK z-xiD1_oojJ_YWq|1soi)cW7RBh{o}RRMR?Ya2_xe=uOJ^X5WEQUI7m*5qHY?(GKUc zMZEfw)M0&j3nsu+f1I`NE#jfyKJzywveC29{KZBHDPMh%?esON0*tLB^VR>Y1^N%Lp5l%jJ5GG@gf=5b^Tj9d9?8<%C78;!1vye!#UM@a zsL9gzZT~vG`zOP_UJ&8256sOn{?|tslps0_F33zD$pnkuk?VsrTUJ=O9j|d}tHTCC zr5OSyOi|^xJ;U<%T1`4Rtkt&vs#@RY=~VbkxqSJu8A+$4+;Jb!(M{_cMvW1-F%Re% z**0cX2}*S-$=2604iAYL_~`cQ@jDC#P&ko(4&6~mnuPq8W7y(?YNNcDlZ0_gb}nGm zzufKY;bFCL!rZ^|Bh(5kCk>t>7=MPgXdVufWw+;2;GZD&)&uZv-8E-|1s+SAACGrk0zp!>7NqhT_ge!HBR2n=$08qyLs!z=AJI$O zG<|BIzl*<1?F6Ab!%+x z#~7(uk!xz?X2p$$vZbfHYgmsQW8~BFkDbOO1M)`bL(Dvq$J9y44+kgacsp7Y z*3Z5<%r7t8@43V;e+Uf8G8QoN?T~EV?NaBEkKbx0t7Y@pg&KuFASx=J1+T7x5`aDa zG7s-BZ?glFcI;nKtFJBOCDn6}p(}+`wOj;o8O8`6iklD=EpRe7YvRGvF%|@>4_++vVlAHw@ zdum@B8rHf)5J==Et|7$r@jz1`VpyF7S;rE*iXjI0l*NW8cJar2O~Omu{sp5(jP0BW zAxIGvMt8SP&$k_>pnzu|2xUpI)1T0h(B#1%B_15j`(}jE7&BJK@5405ug)|2Fg>?s zeNY)XPbSsgoeHe1IY(N`5l;eEhtHPCOmj(Xznu?tDBDq*3nKQR11;{HhrJel5*s0O z!n=w>-$S`K7j0DRuHn=Jp;N_YB!s{|mn%&lQ*y*af0`WwpXjZQ2Mh<~3$nyZHjpVM z-7R(2#3c^I;mo6!_AICDknq2K8`Or<{XcnR$PO6F7+%JMw8E^dzmSijbMzmU6o#ZX zm@B8*VPs5NzHC{FO{^40A8UkI(_IiYnKtLQ&PXAtti{x8RqhRJfF$IQzCWc)%1XVb zKmf*~{xls)3r+&TUEbc&K}xKEmeCAz;R8O5YKVuE(KBXvVH9FPf7~0Xb|2Z$3P2=S zoj7#AC9kM1Cf>xzXwdoHINx?9oN;Fn?*z@(^^%g3=?A;&?5;v-)tuGe|H%|u)b+Z5 zeg3w)FJts4jf!!-MN#2fYPSXcDyFhFc+0*cWh0sI8rs8@^D7^YHjjcMIzPsi5LHKG z7x{NsY{GS1qhK=IfNq$NF)IiH$&BiLcb2S&*l$kc^YL|xHu*bWj&~FFT*jd`S?vw zC{RR1w7u^0pGRo)=WlB(V8TTWM(iH z(Qy+MvBknetF#y`YDN!GA6gmjLjvrU-PvUk`AJgy3v31-)w74EB{^URCb_%fMd| zSF`q~N|JcZ22ikcI{uT~7?4UNUUt+V0ge@h5bS}ki3-W=Zm-k%V5o@mY_xLf~Aq8#eif^6`_71ZFsJX&c zdWc^RL8zxr@tc&@)z=b$sHv-agPkRVX_kaD2As|GPZbCwS0H(I0Nm5oM;C%O-CbbJ zkz~7DKOowqmgh}hex976dRmkGF%qZQsDoSi`otx5DZGRMADjSBefN-mh49Uwy?7q1-H<@7x$tZ^>s|Seje7@=m+S8O)Mu#lmyrYdRzP40mh%8A z!uWKN2cEFD6MvfVoQkj0f0BRX*a;cs`!(%ofR7h(*>qDSnAZ$K5m}OJ-(2V6bcXh+mP)*U&>6w05+c>7G2<5MN?M!47dT~`Q>%dftd2|eF%y%z7Js~l3jB{#P zH5}7t$wdHg{`r48e_@+L=f}WIJ+Pvu+cUpHNkIV*zPYod(E8`5Ff&wwU_=lrS)tQW z(i_S3m|O+;*)X0ae;aicVj%=3W~vCbeFQhkPA3p@RBJ^SPA9kARXBT~*xQHcWEA_@ z{t}ZlBOXO!6^nV1H7N<&zuhAkLEQVPp(CeexO7eGV>Cf!iHGU(;aP)c&c+P;=ti%k)aC2= zp`_%A*4MCEVBM(n^2FvuhGUK6c!gvPnrv?~i zgYI9Ot2atyd{9Swd-;wi&BR(7En1pJ zXI2#SfS#Xt?dHgOe86xS=JGaI(kOF7VDJT)-L|Goi>NpprVAZb61hyeSrLvBHTb5DJJeNuh%kkf%G6gHID?V(2&P-VJ4e zRh1V#Qa%rTQV8929LWXWoa>qK0cwfSUZ0Ll-}e~r#xguF`+L3Hz!Fwq zlD+3g#rOaTBR!=hyG%VUwG{VEW^yl{H;Ki;+f$M~D;Z1wt_h+4 zq}~*1Ci|Any*;LYvwWmbqV zNRSD_8aSU;;g&Xers~8}KA6!z&h`}m8xBr|(HM6UuP36C9=B3l8Pe<4u zL}bGgajUi8PJmMPcR$1RQ4*tv0gj}_U=76iUWFJ=PtoJk{r&h7E7}dQ9h`lfBgK3= zi!LD+}1H3U)r(P=oN4^8(l6`F`P1>+ zBB$xZp1qomq?)=zKvhV5d*y>(?nJ?#|Lg5g!ziDp3mBd3beNcOkJ5qLY0%bh&sZqw z*tZ&Imel){p%9Cbid@8aUXAw?ru9aAdJX=Odcw*ij20~gYA9Cj1Y*O}NC!5E1Vn^U=26_j z><)j(jebjlJTf-+)=+1hBK~^KS+l0j^Z0iA+hmeAgE57^EX)U;q}l&qr!fUkU5R0_OGq3uF-cIv4a2O6L$QJi;TJ> zMECv>JjDxobecdB0uN*}U81a$^M#Uvu`P=x>CCg02=hU(PToEEI-nZE&V1>NXv3r& z#5(kY@vVj9DL)+UY7Av8MH6m#@Bz(6xkoYX6Ur2D%O*YGBbRX-T(jF+yeZhBUD%6K zML16a!Y2kWF|43+Ibt|4b%JPHf8Ow*EiOm6#)EQ9&imU;?HT>ufRMjq3!K>8!QXZb z3O+J>=66_QXrc^*@17GsXKwnBc@j+G zv--Oa!Z^?`&zc`M({({bS(yqdBNV@ac{R_9d;1-Q)s))Tij8$IpZMI;z^R4=r0AH} zk_@lQOXP>M7GXg9Q>T7+hVd)a9ru@2>(7Tm2c%#L?8**@Q7U~J2Ae~=5HBQrFb=m* zuxD}@W3JYsWe;y;rr}_dV|b18X%b`wBd;wjntVp_=aJuoR4UK~qL_n;M-ibIprc2R z24!M|l{?V7@_80#9hPyHdL<1q>Lsh;GS|US1(hY?pCMfpVEd}BhT0xK=QI*>+!^dj zPHI~i=GawGFdOSW+~nK6eohC0LvKyI;S_aQu?`H-o#nlZ49H+XvD0z~!SM??7MafB zcw%9vX+!~W6u=^t+{j%QWsLMT=;cg7h{8A~rhitM`iU+3sMDoF6`#+!Ahrq5Lm0Or zB7**>Qd(P=UGa^^^nh3(MTE5d0X2@r-)(&u2e>}Q_%FWyU=*HQ%6wIX3j{*}G{XwG z91zb>r$p;{25Tk4#^ES}tp##9c@%deGzm-bxna2M*C(6Dt+3YVQSUQt2HYMrrTmi$ z^n?a<2$@;w_|e0X*=htd0xAv&FFbJQ`cJSd$k@7<2xO)i>p|jlhQBSx$0F>bs?i<_ z>LQUF@tr^&x=00jeH4HFNNK5R^uHBJPzyfyV(bTzpKxAd=4P+*SiA>OKFy|yWu?hO znh0Hjy3Cc_T?9FiyNesyYwe~bE~F5(MFGj^e>gTS+DMQZRyBAlPT@XxA&G-9dWy^D zDr;*Mlpr%w;r41NV@Gn2(JK<^S$-B5+?mad9zG0$3(}Yf11$_>XV>E7W*?^mIHLCe z&WgW5_OKBJUr&kWv z>bJcIcBE{!7JJpvP3G;M2muN7;ySRls>EDW7=Z8rxDdw-@^Oso9E*Y{3P*qSX%F*z zwyS3#tlxVLAlE>_p&@fQ0Jfxt`uaDoB%A7RD%qr;8Yf*ei#O0Mpj4X(sTWhb8`>go zJFG5UgBy(M`rE{#93CEyy({}Q*zIH+PwfgVsS9+84n-kSrbb#fQ+Ya)mN~;0o#t~u zlfx6svZj!fD;DByBOfWnEo{lILhO5RV^)tXxrtdRCHu$_Zt)aRZPO|6st_KY=|Hed zR(B0~a1dc!-e%OSAXQ$5QIz5Acpjy%Vt3$2Fzxr0M&oE8 z8zqs$hsgaUShoW(*YA!$6o$OxxVQw{`}>QAAO=h@8)JOlJ@_W@MqTlZ1#f^6SEUp! z{Oa}V+ej(M$-qNT(w4F4>-!wjZcylVfG^~8!SE&+Q?vOjYaHf#y?u`|FY+G7Np$X< z2LN-|oPBEtWmmO;Pra9r;O}J2SqV!czFf#bPw-!(dKqxCHL)uiN~WL4s|hNIT^U{$ z{r3|qknH3i07ID>x)B(j3-f6+An7>D7n)2 zu_8OgoC)KB3}y2~;5`naU=m6O&G+7Dr1uu=PP4&%NNoWsZzUUJ%n2&QlsqH5$~C!^ z3~V7C4H+6Y;&0vsI2zxxWy_Z4)Dd`mF4n+liNXnKGY6Zq3#yZs7*Hla_10YK6fH*8 zuz&bGz;I8x1xiOUo`-WShh>gCu~&i46D!cWk39P*kvjr3_4wTZdvMxJ*PM+Qxkh8G z!~WgrHjpL7^U$E&KmOH^dY$Ep751{4?|YT6X9I(Suf`c)JOjn5MSC08(hnh<&O!-j z<0r+u4-)yPiZ?q#$>7(vOaeTQ4=#69OX5?XxIWyylvcxmdiX@4tAnO}l>CcncGT{bnd4Q+_C4bUA?J*EUBAl%98 zXQ`7TGN8|$`(Ohv2T?t})zTEVNj4im<^YgTs(*h+Xle;ntOA}!LN4IY!k5Lx(++li zo>Wa_nGY3GlCxJsG4a`F5W^o!!q*~+Uiv_a9aW56Y<_K?qsrf>IsJ{INVxpNY{q_7 z!Ez7NmB*KQ4lzoE+!Ki0nj|#7=USW*_{mk*-!}TE@wpElF@ju~!h)O=(+}f)1PDB% z6z^*lpoqZk4^XUbmb;R>V#j-8V;nzVSpt3sl-}q)cM?VAftUlq0E`l7n$kq>#Bd#z z=XrUrUiZ!36L{|2_ENXM^a>`9O#niYpB0N01J9okOlFq^bo(uZQAE+_j&7MdAjkA# zes(q=7eOjnVO^tq8b}TJ(`vjQiUVjk!=M~%26iH3+Z~=TYaYJ^Uo06+JB9*Ul^W3} zdn){fxb*(u5-gMf(_sibq48jA=_Dz^82BUBbl14)M$&st(zCFk>NaWeu324oSwSFP z)GD1U(X!PBBM#t>to#mB$lwAZB3O!uaWECfNM$#xHXt*Fhs6Vlmx0+>P=L~~lM1pM zqcx!Hwe_x_tvqAKE1FbH`rv`(YKi+>m?{gTWM_rxoaoPAkLr0vLB|xr<*<>X(vNj^ z9leIUD@Ra;W*%_6BSZRp%%bebnOWKR97&a~7km*}-SH zRv28djw@M_;7ePeLjs0(wJoMZzS5aHig$h~c)5UEw)fA~7D!ZThZk;RMX+HF4@5QN zf16TwoA>A5f;|zLul@nyjiV&PHsE7V(Ys97vXJKtrPKk&OV(11{bA|K+G1y1nfJk zml-WL@@_QsI}gYcuQ~)c^+3vlxzS&BIeD8kWuXv3MfhK(zu+;;+ggUx@0st~2B?V&Y~^ zciqS&o^ZJ0&4h&7{UiCi#*ZJL+&L7_7Xt|=acCsz;vAPu=SHITT5I3dw3F-$eY#w6 zIk4y00iTu=m;&qJra-g|7rN%lDmEr24>mI(1N^r|6!r>_Oq4=cF(4M6?8uFXg~s(- zuPg^Jv#9PB%lZ3x+w&J0cC`jN&e@5r37HB7xU?~Uw?FCDSz+Z{ok>j}QJI?k3_Iszz`~q@2(iY4J>KbPmR_f! z5T8RlL6bndg-;7)AKCXwdYSOf?JQU}(l-NRgZ8k}GZEJ^q`oKPrKho1NINRTgO*b& zf2MzMVI}`fu9!Y#3dSdswWHLN77%EMu2X_u$SGtu6h)BC3%Vot1lES@#5UH~r%1<1 z@XN3ysD_Ky_p{$#U0ZwNLwEGz=RddpX0~qs(Ew@R)tftSg zgSw;&AFLUf6i^YF#ES_D77)l?C4zjUUk=&ea^+g)-ePsJMIq34unE1`R`8Qs0H%L| zyP_9bLDittc1*%N67mSqm&^V^G^c~MQv{9fq-cKbV6fMeZ^!_d}l`7!JL7Oh}d3y z-_$tB1fm=c-!z%PiBeo^R7Fb-Wj_E}IRhpyzZQ$>Z*vd11em~McxIml_I5XI4;)?j zJjh0Iddb)kCY@l}kY{f{f&a^`%{+m(FA{KA)>fEcNK-)8eS{BUly|Yil#pa_usaj_ zL>2=+35k}YK6*SN6D*rQh)-Qm9EfiO8TzpY&-ptz9bNIX5{#=LQ9LNfH{tpa{(8^k z&u7@Di5DYZDi&csc#@1j?%TF{>_8!6-69k_O9w$r_XMC_i(t7{6XAL_b;#4lA;scN zZ?FyBr7ljO{PpRCJKS?8az3&vxw$raoM5_`G2oW|77ra9?>-!Ql*dyU2kT2k)9R8( zV5xI~w`3{i>Hx~$L83h}UK;pbmRA8MrepDDWkiop%4+3)T|KGqY?+x+=7?CiBW+qP zb`&euVTy3EO+Z#Vwr5C*)}V5*DJR$rI<5O~)oUsyBFx~Mhuq$Nd$eStwPa!?V@9;3 z+!u$4ny;(bRL|y{$j8qj7dZN*o4OQ&bdYMDD4m8IT?4TSU|o{`Nt1ze&I-+zxf__<- zG+kaLWvCYznoM{IG$EE?(aY5c59RO6M)AZi%R`Y1u%0b=*)Bg1Hnfs$p~v4(rO|T1 z4EMm?x~f%!9CEM-ljxVCCWAs(9okB47x+c9E#-fr46ok|4b8y_40_8-+Yo|w9lCVP z)bXYV4s-?WMGx0$SrlFK?8%`ry0)*{pbf@1x+oX`L}H%L1P9~o^15Mug%pbhw>z3v ztEOEU^4QtQvbseSv=Z)VHuS$Fk8hN4QtSBg%9|;B@9+^BZd#Bq#yKXD-8kheLh&76X9J3h=)cSJ8jZiND9j{aFXY#c7SyFZ{JD&hv^8Insj#UaG74+P= zJ`DeQF7-RUxZlO}aUx+m6O;Q&R!+EV&)CDZSBBIrvU+j8QpXVHMZqD=$#`ABp-;`RYxqHL{Tat zZH$%ybFE2!dU6eg%u9P{(erNKgsKJNNdzap*bLPc0q~?xWUa^KX#_7ryHBk2r@tuq zM8fJn>>w|Zhq83%&RYSa#*E2F2_UZgEl)0>5)ZxZMK1uPBm|Gzs zqa7hWaR#ClA{}N%QysSF8I3F7Icrm?8+*I2{Q`oa5CmqncEbh|a^%Jmp93$5YGYkI zk^_+Rj#ZGm7q8M47)pg;p}of76q>Ate5v!R#lZm37>X8H{_T3&OV~s8%*6sb9~ldT zq$!3dm5tnB693{#v$-iaAZAgx)SIw@!g7U}NPy$S{yfExtQw8!NyrW^&=LOf9Ev%L zw!6gMSu@w8;-&z_tXMl3eym_%%mYY7-1QLxghGmpqmZL_B@Nu2Sylx`fS6?wLEmtVDO83{%Jn$g%m2+qsUz<02Z1gSO<-7? zneQ_}b5bnx?cYE*h9pJ7XNhn{w%7Vn z>Qz=2VdO-Tb6Z|;S@IxeB8Vo~LmkFGK1D(zNg2HhiKCb=Z~o26j+u@P0G~((sUE!W zmKH?mTSK7I^a%k(c_M#|6X1ZzY0`AHmqB2aj2>DCb2g#vFe^2`PVA+C<4$jvqw*nv^YY|c|5y&RQLKH0E1H^q>&{fyGRU1DPkmddpcdjsnjcM@k0Q`II)lk^%LY{qULuwL` z9`-yQAZTnT4yW(=1wQLxxt%(7s!}f|?GWs>+an>gBj4x_*KQqG=DK9eVZ8E=k+?zJ zV_pJ}3EO~e2irdaZ(dJK+lU_yW|#yXUM#9!obhB0r##-Aza;D@kXAiKJb6CHa9jm< zfEfwKd(9)QU>s>?=)D^vECx+bgEof%+>pHvXNe$I4M<+4RRDP=;$?xMPX0M{R;blo zWY4E#H>8;Pv9k_u`$)zIC;UhGk1(T7;hJ}L%3@?R{0^{0P~)7) zfpU56U1KWfjk8Xl%y*SVz z66g!e87!$V_9*6kK1cM*a?!L zvqP6KkiJ$h9XlV5s-pI~1hr@??XJEpZWs{vLj(jN|1g&7DDe28y}%H$Sd-5tI~Vqx zd?q;J{#cz-tW7-f{(InUMDYo4R!6!i;)B(!z%0Dz+mxNT?Evr-3KOI{D)f;S=c9Ak z$Trj)+_#MN?t-7_oZR>{2?VGYKGgd*AZ;Vc=6~5m+9|HB4oL;9&JlKzvi~@OM3t(Q ztRz>z*uKgaN04Hp#uJFr%$}tMG91gn#j!_s(R|p9oZiQ|_8Xa(jR-Nw(?xFdZ{&>> z+tdfIkgj1j>{U1`Haq^hvkTTBvIz!|#E6&&ki+t}w8TNf=T>xo_`10DlW||Gxq0UY zLO1?n8ihIXQ3Wh)NPob_#X5cvSk?@}RfwZa#!FYsVCArRiJ&@D1dks*7YUj$3F2H|htj8}4>$O4@9}D=R*m)m2@E-eJKm<8dnaFva475U zhU*ClB@k<>YEC-L#%eGH_>Je}gn*9q6FG$-z#K0KM6PyB*bI2>58NfOJMh$9!Ym$i>% zw?fO(+0}TEk_Z(Ou3f7<=!cgUiao3i*I1b*k|Z$|{enEa8u{s=faX+i)6_k^p3as} zO}8zE#q>uu3}P!uZFYr{j3%QT=>T;`pp@uDwX72^u*3}_NPJNB@OD1AcJ99K7uYcs z!R0EH!~l^XBR{&cLw64`m>O^nP!-3cYXlAuVA%OlMn}JpY@?u#+h_!CaFhhnMmxm% z-W@_W_p#+3dT^hRMIPwa9Qf#SGUjFP(%Z_nm;w&wU=_09$DX>O-)Ang0ut z32Vaq=hti=382OJc0tbH{C&lk!clvK1tpz~{S9O@IPNTW9-cPwe=Q_mJi(#Ql>Yqq zUA4QA48ph+=`Aq>IJF42xo33c08HU5SBkVLz1A1Tne(LVHz2M(U=%0O&y;KKw2rhXmqI$iD;r> zogkUn%$7E;yaECd2woKrLYqeoBaLupgJ)JC$SfLOjN+h^=5Ouwkl%aTJG|&$O`GU$ zC#Ug7Qn0+OTG90Qi6Jr^!~b+7OBqkO5$%e0C6P0(4R9L}nC8 zLKUqO9gd(j(1h{wBWCqHSbfY$C#Id1=ukblR3QdESO}%JS5eop5hF%a zr`JXW!noAZI1aAN(S*Qk5hE}c2D?rHtj4_9B9!(oShkm9N{^>sSGUqUyeU**2-=C2 z`eDwaqjw9k02}omu8)vZ!e(CPhWEE3)c`h%5OwgT#8D}d92MZ5+5v`XM_cSo;?si_ zKY#-Bd*gu}PK(C!3>HbexOw4%kyZc0^V2ap;A4G_uVq;*9pCRlpqJZw{KY`%^c^|L zteORzD7YjDG+}TiWfNU3VMbQ21FV&+w&mEfEm$>oX#gum`qo7r7U14l!Q?Dml%x-b z^w9@Qpfj2aSv+u-X{@iJI1M~|e4e?1;8>SJ1x^?m{g8=;Sk-VVhic;^%%omOT|oA^ zvBgZIKwML9>f{tl`ai*k=c8*e8Tu@bHc%s%2>F18U_Que64F+B=>xmam)){TUc|<2 zv-9VCnm-9CLRJFxR6PrR7};Zhmyu?A{m(6&VKlQ41`~2$i09-&fg6{GiSf@bxkVlL zcg!7Jw$;4CH)-DlXZb<$y89+*du{d596x@FI#~~abOvBkdE2}|ouh^(U42BN#Z47w z&~(FwbNGoAGfGeFwtS>#vl+7?>9Vks8O`%weBk0_7VF%H3sYm_tgWqSp`wB1kzQ3v z-H+|t`&4crVvyjS0^>SB8ULm^@DJ|1>*9eM2auV66ZMHAM{&){D6l05!OyYR*}!Oy z0EIm)e8#3>ig&;xxH8P@`5~R1@1_zOjo6J=_I`;(?aC3X3<#T=b9!ttvQCIRdp*cV zPZyeE3^AJ_PjSRz00PCxc^;Y-8_P!EJ=C#;qNMa`1)GJmC|fXp*mA?2zx^U%7djb&S^fjKKXZIYMt-Ga(tLrT$Nq?-iL->$w2bG;OPM`(|}Lz zxTwth91JXu`fy_+>gpBJ5{8iOww*dF@K4&K!^XmRAiKd?Zo6La{}`6}G>Ju0M3MM% zxh4lne+vt{#=|F#)lHy?9%&WfzmjRylDVoSb4&8mS)?ZJ@N)hlWi&mZLYPahwYMRZvKW?oA8=&;?f3W))8XQ$Ol~FQ zg0L-1?8%EcMxt3Z$QHGP1yqJ1oOSHc*Kgh^(*+TTj;K@a%ghAj_8^c#4ux2PDRbt$ z@um$Fe*A6uP_>XPqzu%j$&{Dw&gnnFAoH0GU4ogceD{%2F+a08oEbpx>lnyWC7+v6S%$ zYkA`XYaj(PTs*(kTlm1CPrV@EziQ>Bjqt?cu7vDnn|dJG6z~A1DwPU7r53QpysWH> zW)Z2nC`^4Seo_*&aQx}ijT4_bqgj>DcvLdqAr8eWP2Osi&G}2riz$yBL+wUQ)#CLl z`U{UE_+TBrJ$I1%r%v&sL(iPCm))~nE}$QH^_eniE<1J*Jan*`rST2L*^U&7-u(h&J>yiGPhWb#o8zTV(v(vJCmlyswRoC3M=@LWv?p zTptBtI9*}H8f4(pb^=&Vmr$NM#W1=smH!@75R!AZv&*uV*K@OPI z&nOE{BFbceqf%GC{P#_Bww(SKBE9xyc%{wzz8iWuO1Dq#4W%*KgSN7!n0zm2G%P1T z5ybY}4W9IymWPo7^k`5)y$8kpkaWZ(njmVZ!VoXPsQ?d7^pE=_C&)aAenI?YE?gut zDe{gcljf6ynfDjzM1u2C_1-{+Z}!H>Q5V+XqyTGDrfUcb0IjiG!U;rDRMD0|Q-3OA z-{6SK$6LC&S$7e<7l_bY3jOF=91BX-K*VJtMR{E+rLm=hi-bGUEWvYyi_6|am_b&d zf1@vJ3&N))-2Zvb(Qp^`@+E|;Y!oA$^5@*PVBTlsw^vQixS}_GC0N>`g7)(!LW`Pe zeRB-};v+M<3`3h7Y9_LN@B{e?U z>CxmyMw~k93Z(fb!S$@~9Pu3Pa2}J?mYn*Q$;akVI7fFYWC3JtWd{_K3`pPDN?m|8 zl@g0+0#T_fhQ^<6!G-2JdVRxHd@^2?7=2`nZKW3wFz#16dUQ!39++U-PwSrP_#X^+ zZ!u4CiGck)P(RVK<|k3w5Wl5F z5nD0ZJ&C?u`yIdY#I|=r=fk%D$0I{@)(nr1gK{eeOC1i`6vcdZt@-R-Q*YE4}=qdwNxrcf(5v-C;rM zd;=5@+jNwsVCf;FY|XSO5InMaTsnF<6zo=y6b8CWQM=ElBD$IZH*~B`qP-2DSa9tO z3Heo&JoN((om|@Dup7jJ3}I~wTJ_lCf@#OHnHO^l6WagUndOT~JBvZZiNbW6aTzSh zE*;t%k3WCD{C7$f2v9LU^fDf6zCrt!DNDNn%QbiFUfKYTiRKgc_SS9rPULdPQ%QJ= zT&$(vy@n7sby z(qi8zud!>fk0cFKhd_h6rxh+O-m$%T5=vOe4FOv{$e~0A%@vyC_@a(`nISuW?Ge z1Ul`Hhn4KbR+!=_A>U}vI6PQZ!wk+|aq^*4IgI@!!l!%laJ>I`N+c|q`b&*v^si(Z zN-{+oNq+nM*|Sl|1wznOZ0{Q%;;lnwefF&OlaIpBLO$PrHoRD5^-v-XP zdH3L&65nY>9WY6pDiAa627C*Shev>EYb?J{TCipi(mG_P-P5swWYUX9CrDH~qHiNd1)o9>u z9NqjMH24r!-bAS$RuP4+hmL)LYL8`UvDZ^R6snhCd5w35SjyQXM3i zkh}^@x~~TFtxP5Qe&@aN?C<~ozUMgJ!~4F^9_YUB>pIWhh@>>%Y=tzPCl_x@ zpKQQ=)9+`V1Rdm)r-O6zpg}m$dY`-+l{Ez6nh4YsUmI8I8N|YMb@~~N{H;<)`7<9i}M}}1JdyASTP;V8W z{}pbUOoRz$=!Er2@R!jb`VRi$gpi5!d?*`WJRT!>SjaHgiq1&^Ouz)FQCctz#S%wY z_25AQilaMkj(Y42njeL^n&aAVvAkRj9FX`P%JC{ktFDI#rEm9oZ=RtEl;UggyJOqk zG1f8im6)T51!7yFs4jVtN~k#&N)S+$Fc0AzU~!g|IU`2^OyHB#l3uD)&`0pdaiTs_ zjI@|*vN~!~WtVMM@bI5L^Q}}wW@v^#H#HU#LHW_(**(xHY!!?i59GiQ$!9>@`9XN~ zSd;8Xwi$Afg$2FMwiBKJ;nPmeg^rn=qepG1Vi2&T&xXIYHUGEwU6L6n#f*p{o8_k9 zFiCl1JUdVTNN7QX+2U%#TPkb6F{Wdo%!gi2c)BHA>hxo9^x9H;Dym%Z6QE)UD-zj4 zkn2D|*@D*1_Ni1m@@g6GYQj0Ku0w+KJK&G5y|}jP_z@u#8OLxj(FW!Cx%@*=8c{?C z8y*DjV$@6U7i!hc#%)XzK}A5`L^9;a>^*@9d7>DM$|JouVf#SFWF{Hpe=DRT213_C zkyfP`a5r?5tc|QbNPzqIU#*~iloR_Rf5F$MZm)u_?85%dWCE9(MxQ;|D1dQh0{euX zCYjQ)Nb60QP~DyfxV~X8V5Ru|;hBufJshO5yZaB)ahtRV?Tl~;hG0T-Y0Q>Qez#2r zsBU-PrYzd9UO<8Dt9a+G{2zwrPs51p`-0PJToxdj#W*Gsjtr?Zi`=>xN)Je}rqR?x z$v`;13hj`6lt7jSJV}oi%>~#=brofpijtIMizHLhQbRA5jU^DEDuIe6gh))s$Brb@aC`YJ<*u$3?cy`m**OO`E*$DzIq zI<)O`sFMkSH7QlhWDdeb5ub?wVNH_gJy@a6#J*#xJ&Zm8A;0uJ{ZZAC{vf~a$FK8i z_!pY@E!%vm?$lHD*=B=Za^g69g`8-aw;LaztX3}TT2^24_J9|f23|IcCl#fcj?b8@ zHssWRWFIMM{Lg*<1dtB07Mgn!86Ie9Aq0<9Ow^M zAb%J>aGBrhW=KCK_wrk=axm9WW08@p|8*wUSY`h*l@oDO{0AkP@%rMBAPX7g3c-{` zk<{&-Zc%uRGg5i*y%mCVn@^HK_SU=gsQ3VAS%^rb7#sU5F_k7ds6KLn6< z^D$9Or*cc&lds}KyL`I7m@=LB+P^qRmmJfN9;k4FP`Qv+-yO_J6rQiD!B-BL&A3QM zt7;8;Z&>oO_XeP=mL7|Z8?}@7OBCwp;#{uUxO;6Ue%21*o2VfU?h7dDgxWpX_cmkA#Ll-d< z;YDS^_3sr{r>Gw)Qzzn$~wzy9K zz!m=P{^wr|-^-R|B7Tj^Q-#z(h3o=)r*y{S5F~gUO9_}?E^@P{wPRQj{gC0^?e+Sr z`^FsJcS==3`qg`$`Pjh&V3XG15<@YUaq&=z*mChm9}aJ%=mtSPxp zVl*{+TE7Ii@W{0CK;f;|dJz6bpZs_tTEnBxsBmcS7_7UXLhb0=>bJUFOIml@+t=4E z+JI4=l(b(lgz`j!HUeZ#tXTSG{!+2wmiG{)A8}i!je;6f=z}TGHK*@K;I$32rxT$5 zbg*gutcg1+O4O;BJ~Qjqtp>qc`RLM$+7A#+AwP>gt<|lQJ_0BwV4KJzR4@Gl)S7g2 zuWWItwiMihTJT28pF=l zC5Y@PPJ}0b6`^9?SWJ=t(q5$eg!NtY>qkd9xt}nDmo1rj%e`*h)EMZVmaroojxH3s z;Oo1{eFr+O_Z%DB~SB)&q(H8TfEOH&9w$c>^)@QQNBjvhU$dW@;#EV)3r0g`wZ;aBp) z_EL63AmF=!2UVwiZH-p&)(GHthz-si-TyIK{_kU?mKoPS6XM+ed7CAv`Uid$9vs3o+BUqNfo7Y1 zNB~VBGXvs!IVjIj*ZUDi&DY=x0k8$W=3ncI9^k zOZMIA__p?Aat&G)NESdJp~}KDD`{_ahj}xhitMY)jlaZsH2!5irvErSU>+BS>J)y$ z{THS9cvgI<&$DF{F#w8@kRWjW2`svO!B8Nuk$2yc7n*~>YynaiO^9TA@BiV$5_ge@c3nKu2 zM4DaEnRsK?IM*Xo@ddoz1l#C#U3P20H^62>;@g-4P69g{V{t6>%q8lF9m21R)3BxP zcOi&*D1YhErMN%at6;er?-+rwkc`S&y0`iU1CGbUQFsl@8h38Dp4^T>BxAZ(@_PLz z#8OaoY(jf1JOKD~O+`cT0m4!ER^zWUstsb*W#rRkGePmpa&Ug*WLl}4HBvIi3lK&S z++S+RI_v);6AXTjmPqakenG`Y|?lJ>=RO5u5^ zc)0jIoU6`PRsEt9?V$E!m$8!TSv0!4tz8ajPTKF zVOp0z#?%MqR~om0<6&H_m+A66zj5P6M?+H^@Z&)(Wuw*f);*SD8ejy*O*kzn^r_*$ zq`^MBELlC+uQygOv%6dnVjOvOH95 z_5v(LP{HhA<=&?GG%|-lGMdnf@w4UUmJNtSVntLy&dZ%u@sndcp0Ivt@AwyoGiN>> zlPdiW&+hm=dUTeST<_q}FoC`9ap4z91>W<-rmMo&vA6i~_dVngVVtB_qMLudpdjTB ze#8VUdb;cH4*2hk^a|%*B}yZ<5&6$z#u)kg<*G{<(;(JqHfpH1&tL9dFM10A8Q?Xh zuS_2ca?AXy-A@Oonk4P7L1*O>s8AuB`7~?YC5Sk6aq!}izRUq*`xFZ zbsPeTVdT$}HkM-Eb){`23rlrUvyWAI@1F1lQzJLUm-6zSe#l=U&D^FcV!B5cQ`zWT zQ&)-|V?pc9NDMC6x)jbohQ9BzsSdl4F=g<_*@oY zOckKZy8O!TJ{E?TIsI7N#+gNKzfqaNvp)ZeQh&flUuGu#h zE|(56Qf&3|@^VzE!q}}gb0(V;^Gcuzwgy{+hq8EJnV{!+`0xaMYas-GAl>C>W^JvX zH8C015^%KreH@0WBVcfw-VQAr9sJLj!cBT|H)IA#Za=6xgW4-yuRMKn4ak~?Z8~WL zOnWJcS}+6aqd1aZRYnUJ6__mUA$FMJgGbX4{RkXPFLUev#YC?lr0psk2yLbvh{a69TzV#sK&Q&CD{fe+Y$WO?~^s)w2twuYf zYKU{Lscy|d{~P@a$!_Q{S1^#_n80?9Zu@wtq6{2LTz!3g>a7DR^Xl%kNJAVFW+NGW zR$aP!j$DqE!b-pag1PIeqlMu0V-Yv+K%bMq0w z3h0&H>97jW6dQ&Lh*%1uCRy(CB;AE77*1h)%8=3VB{2_vE7U+sDGR2K_<-Y3D6l+l z%%sUU$W|Qd(8@q9A#~Isot3#= zh0MElw&&s1GLAwFrlVyVXf99fum9o z5-Dm@=LMcN0tgzmNh<3URvPyE#QJtQUkdUHn?vgr1sk8wB}u%;rX*~XY;7X)GHh?g z{NCI92Tj1JwHo-5|Jn%PjC2=jh`eD@*T--1|Yw&kzfwb{pzrL=tIZx&lk6(N6N^>bUrLjrJEWUikX28Ne9|K&o7sRMv`%&W zH5f*{LccO;AB}C2Gc<1Ls zNBC50iW@A>58AziZ*BNlM?Yih?U5W#{eIlr;l@=hriV;b*>DoV*AjldcAAmci(Hd9 zAZo>eDonPf=ixmdb(Qw;b(U!v&LPUovHqJ-a9Lb1%M|@{qi~AmRMAVdXrI;A6>;yv z{9i~GRRT}Pmmc>Xdo_*O5EzZ@lJ5e(CmuQg9)M@**m`Zdp!0)lms?M!m#+sWEaKmw0s07 z6DGbTbetlZ(1xr=7EJWKif{g;uAd{J5Q_trDlc9*A%(D@KCncYZogI7LFl=xd66T{ zvm159!UHHFtN*50Etr-LdOhCV7=RfTs)v=-bSmlq!LFeN^%?c1Bj5?9d6=l##tn#M zHq4z*l}4C!Y}vsR2P|=PjH%K8F?hz$a=X7_!(O`(NCjzn$;vA!**jz;JG-EcgKFEL zUBGXdBAbwXvCy0o+;dK5&YU;e16`_re}`9oKSb}~KM*Yw9r|fICTqiOT03C>7p8gsB)z z8-!lz_$eC!Mc7`fu(!ucK9?dO?NhT7ECA}u$kPHW~ zjr_yOsH!@|pKvy+bW8HOVHNb&N{Khm!$RrKFH^sz_x-lez{4_|H>sq0}qNq^Js z7ahIqUPa-xCO6goyJaw!X=G5{2;{_rs>qKyy))S0jj@}_CH0#Ax_tNx~n{#9378W zCr^{I^uv`*61s|xZ|xd;a%q>Nk}2ITy)jsv1?lPP(7D#0nk`&O1mz*4gu;^*kk+*y$!YzHQA^XYGo9lwb^Iv=3QtKeguSh1Y*u0GZE+4j z0$#!lhK!+X4l~h-&A8wxXj$4{@-~8po$v(X{SeiQwh8ttn@>9k`TgF$d9xx2=LDF> zT=D?gWPFje8hwFR`vBpVVrIh zC86gJ`=d4PDRiA0sbQ*d!b3IbZXQWHP!J$>>Z-`hNJ1Qt>cIXJR`-=G*%Wzcv(M@{ zt+sAfpAnsjUs|;aaRP;z!!}^(tcXKF$>K>jnZ|vp9sIX2-|8MW9S?(-MZLphkG&)ZnGWxO$x^FMLao!pCP~ptDV$ zVC(EW1Dl!Eb*elYKc1pObR6*=sA1Pe*J(VmRm%XCBP=x3gjWqKUp#QLhX_u)I2}LI z&s)Bj_+#>bG;X8rx-&32g6m-X4~L$+8M=8}S_)mAPLShfU#A($7m;-0o=$cfiL7$#evIN_r19jW-^yR7W>hkQ+jSp?xq;y-_x>i@`yTUlv(N$B6ccj2`W(znP$-Sf!Eyc-4<4eYfu@Pvz8Un zT|dnpRgaF_xA|w#>8V;=@8dm;u+V=b*Y=q9lWwE>p@*-dx}+Qodb}SZ6%rD?cRxfZ zya)DNs~W&#G+^eW=jN{AiU}#V-#04OFsPByG{F=GV8eg4Wq!OC4MuzHNHd+e*1^Hu z#l;j~?X#o$m$%$(z$coEteQ&QiU;|D#?gZzdGS~y=pL`a6i?_{5z>|FW=Y*G3A-~$ zN`rF(K=g-a|742b^qWUL)0)M#GM^QX~@1zBa*$h!^nknpj=B zej87j;u9cR)?{Uf2isc^{rdR)ZTA?Rte5gd2LXSUiR!=tHCg)Lbg;Q zFzV@b2u7UOh~LQH#jdkR2QSVLhVz=*B-yC+N{dFym!tujK72HNr{c}T`KgbRZ<5BZ zAzx6Y)6LcRIi1NEZ9{R76y;EBw14CX^23unT1mEKnm%T|hy_>zfjDgljhcs}7$pib zk=c0rN0*++h~IRze9_tM?h#EPrhm38>!#_;Bj_w*RGMMhydsTzZ>=>R1e6`!a(3Q_ z_uxNquylURkvt89gAg=Wx4#5qI+S8QDb*SZ8m&|UBVnW;_w7f6%vka8p3r^=l@ROK zEu^<5!V==D{Jvj5%CZlmZ7p$TBG_d*U&y3GR=#FJ9;$%CG*bw?M-%`Sp518Jp-GTP zXH^~vT41aOwCw-pl5NUs{?zpCEfX&q>nhI5>XTv=ES8$jkhN%e@uXZ(@r z-6U#4kr_AxQ&%KaCXl9dEkd*ADZT%IwRid(Af&K>uS1;8rfxq(@e;W-{NiU4vbH#v zf$nFCdax8Q1k;-frw2+)rg!5ar*1l1%`{+6?qsH5ClJpQmAX57ws7HyM5&Z7VP^8X zaBt@W#bwDu^stB!oG-+B@Bi+SJ}~Q)ly|Xm4pIs(t5u?_T_*P#Oy=**+)FYV#y zOcI@}vLB~$+H3pVZ*GI^5X)!p?>)ayA#(hqS8~F{vpZUzOtEjiEf-RL_QTGglbW1x z|6EB8ucbH4Qc_xDjpXzlc5j;n=p;$k;~?9CB%`@Mm{7PL^mNsR?FNUujpS??{H#1x z{!+R`ARI2BIPz0qj3^9Xi6}9MqX&k+JmfUy1;TLFn;=ACF{7{t;~ua_C^D*t78`b5 z%cD}AWz5U_aDFfGRQVNI%`%5LD4udAGQ7TPAaz^v{`?QTvDr`H4aWE-a-4l@wA7Ib zzlsXG`^h?9G0dSY&Kuc08PmTu`1lBnav*&p{WIl|HrKjsCn-PZbXK&1QH7+bY%Y4h zyv*j17olfaX4paEfZCWtNS3?BN8%=H^B_d#%v*fsn6 z8wjFNtla>fjlZlP!oiY+Ro=Dxmm|b6ASh2lHoB}bdg%5YJ6gb@#p5J=CddgUJ_~Md z3d>*>)A*MaC*Ha92bCJ2lWVn*^mG7rXjiX)v`)XOt35w+*4v75Nt+`Jlw#AfqQjoO znA*29cdfH=(82(R57o(&`^AcB*HZ%Q!t-WN>zg+sy060}F)Zx7 z{jnQ=K2kpD)VE)PbCI7pJJj{Oz2H&q7K!3yXz@Lak^F+=9B|tARmfKqs3nH^90DXz~!)vaqqeC3;%V6?;h{>(08#{bK(BJfDM}+y zYn(77Eubw^yLGdjcmu^)zeB6my94qSX$^?-Zy#IS@s9$@YR zS`Aq!V;Aom^t>RueKZr?MJM= zbLc`AT{@y{xSEwQ;`GSV$BXdkHbu>m9S@|KW++UI#B%9k*LmL4?)6Vy%Yh7M4WZS| zi??NY#Wl(Y7u(ufnwUJ%>}$L;qkECBDLroE^xYFXT@m! zV>es+LUPv@m^`03b;o{=&l~?isw&WD;c78>6&O$_8LsTajsQsWQ|Kx5AmI!i& ze31_g@qcduY1si+V3;D=0Tqf7a9uwa{+dj9_R`fY7Xq38fjT;cD5J6FWBEY0i7XVUu z7{kHxT;-S~bAUwnEVn7U709NjWWleM?}NXpOZuB@jmdW0p$yiRLbE9mN5=<{lNP0a zx72B16ea5&3MDr9q$8BbxF%y(r#owha5vY~})5sU?W9xCS% zxp&uW9){fYDnWkXmyJ_K!_Pw!k`HM1fC-Ce-+Ec1N0IDLF9FsLR^Fhu>Aak(0hpU5 zo+j!NmP!C7Uj^mGRpfhX%3rFeZ2^v0rzLB8S4S@7+mD~y51J-JMh2GOz7{}Dy7h_k z+-Pj#v&(oQE6isZM6S6++MN5ML;czw=u#NMUbP!W0i`Elvcn>J=RbW7_uWbX!s59fF<56IrKQ~X8OixJ z^1f5_T6st`=#|p*t1jSW) z#UsH1;bga(8~lDv#<_EBXVKxS;_-yN;6>PF9yinV4K8?iTV!*BgY8#D?@lC3RcJ3wEScICQF3RgHnVznz|!#dN^!T9zmz9ehS%*3 zT+y@d=o4UmjEj4JMpLxopAU~Jy&qb3f22frhnaozkIg?H87Mw+8ircUPxG4VhNi}O zWOomWwj?v|+~>C^hJ#V(BNfHRZ?By_;*9Dt3y8Xf_SxSWhbjy>nE8$?RsqvtBc_0Z zlz*~iEKOT0voZMMOjw1^lobVI@43#R9p7JIt4y@pY;&5}_#6q+=f8*bK$bO-7<0BrHdT@4O^L=g2#z?t0cA6|)~)#hzE zXdt-qyl>|Z18~_;5x_mo)DQjdYFLFGjY9P*^7Bl1&2A{1xfFe6Tt+4BPeE?x zzKo(*g}bH13Qf>%ARoz(f1uLHQP0qDhuz$@gA~9?#1J(xPieJ??1^jw3eE&dt7M-x zBA-cqu@bumcp)zJU^jz@5Wya|1W9G-o~Et`%_j@M)TkvxmBC`4*bQ~~G{#{8OaRhe z?gf_jlF_F^sj>o{3?j6t@j{Jy`H)v2CMah9+UnFp3B%6f)j_pG#pNkUyqrT@MjIf} zpd}kPYYo6B2IOfiQ2up)vMSt)Y)oP=jo!=P`TrwzMP9 z8b_MC>YArr46!s>@Uz{q>e7SH!h3fj`k%h3kI4`+Nx7^sciX4*R#s?O(kt4Yz3%He zS*v-3gR1_aHQg$4(NvE^&+oJhQC+F+bi*#RZl1z{QucZNd0tOW=XOAoq**P)3^@Z$ z17xL>N(?a#;di-GfLhBXcs~Kq@zx>@m^R4j>H!(xEW$zc0c?c##>MVvlz!{$5xinO z9I1gN7;aDOfq5bYm>ZnYJj}Pn}xD@gSpHQze z5OI-52D8g5Vx@=DJ}hS1=DaPQ^Sh z#0geE61y2d9LP6Rk_~N+S>z7-QR9a)YF@}y#bR8r<`Fb+ZDVz23R-vB&BkT zGFAr-WUI+Jg9~(Bc=tclUWjXVQ+F6P&)>5JC6zLr9|&=!YvAK9@^fsiGMVuRdQuIxG2cT0kk9#9eeGjR_+*B1t@iBlRT9nfuu(1t10c9oomUwO%61~2mX z(H-91QDpDFWbLKnAG{(4@+OEytDD`0xfO9cSB|h8gX1M(_e0!%Baz7UkK76y8`p8p z5X!Rzhkl9C`&{C#V%LUXemeB{ZT^i&Xf^TYJ8}dNRC$>7Xjb|-U32D4va_rC`uSDc zN*A8fnYig`^$5!_{K?5sSz$ClE<(SxJy2YSN1RkLME_c_RadMS0a@NJWX<2n$UxB#bfLqqfW z6L1*|O7&nht6(6f<(0gSp-5Xc(TyAS6mbMsv5M3|!KD$azKrk*XnZJm>CwOco^*$r zf}m$)H*B|>q(LM#Tl@uKhc=t^O${8#9C=v>kvY$o%<<4CJQjVV0CuX9<)cX}BPpW6 zGgj@jE^TuBCikyncoRadFPOac+4h#(a_eF~wEa+%k9?O!gP`i2-?#s~2jAp`i>B^q zjxTDUDNx2-rXil;9DM|6;FPX?*=ENZ7Z^o#G!w6!JNwY@#~EFQWp-)uT;eWsaT(c@ z%Ag1DqrwsO8naV)4RvwxsrmZ&Q1&EMwdQhh*mL*)Icb*6$AKr(E-I02=j8H@s>|4+r`%9<>qW77)hCNjzqnft>126BZ1_psYq5%!#DOz4vM_um(1|3;}HY4YfGE4q??V2ZVf@0?7y!B0sOy zq_-f@h!CS*bP`CFw?TWv29T>SPstQdf~c9iJLl5wD9AEXLR-hNC7MfD0fQbpee*xF z4|&B%)F0m+F=;`tY?iAArsQvTe37QEX~XyFp2G4K+2M;{p&3XawPc zO%0?EY)osQ8YBAqj{ca|zxx(}F z`w;WB;8c5`JmY8w!D zMP6uMqjjIz%$WWOXgJ}{DbPp9p4H!g=9`ObdA{=eEv33I$1(D$SO>{E3lYN;;H_)t z-r6X>2O{()`aZ26VJQpSsr|=^2ON2hVGo`cKclmQ z7|Xyx=o0dqzos7*n%k9^(Oo|}qaA2A=&|$ez=8V1-8qB0jO*|{UP~AnD!4lxDm=hKsWEeS;} zg4i|N@vSaf=3fQnDyAAX=m|3!FB#yD*365DZ(X)A8o441$U)8e^BBdIt!~Tm3%(C~ z*$T}Douhm_pSK!Pj*DiGzgGWz_PPsFJoA^Jj<&Lmt7p_eJwsXM!@Y4zoH=3`pR`iiPMX5ov1VRkE4e)+SHL$@kJxCtQ@ z+z5&qt!_ue>~nyg+)X{XBB@vR;M@Tjqmk6@pv-V2iFKCR%}bfAIETJ>2yt}+6q#rd zB@p6TUk#z548wak3#N?(5D9?PjaXT0lE2yP94YuY=-8l<%Uk}FbPu6(-=<8 zwYcY%8y7%b(!c84(d_}}oW;Kc&=DL9Nbj{D#_ph6rx18PIr|Yt@lX_bA9z2j(Jt~q zISu7yVUWBYwi?eR{TEW&5;A(zSpFT$+(LTPfN(A5wgD?3RRh z*I$PbLg;(xmO4lmq)?z;W4TFpS|+lK>z0!t<~+q2vx@PF8_VJ}kYCZWcl$^EB5&Hs z%J-;oUj;wiQY#T%c;nBN-LJN!_I|L;QOZ+LDAL(tkAEX0Kj^4DG;9nM3p#0YyBg2mlCn~?Io7@S|cK&2AoM}c^^!c?}MGupp!PY%ku%oAv$zOLV69>UXP90 zb(BJ8JE52q?tmHX1Rxq&+XXb;Mim{_9`5dz80u4WO^-)lI|t}bXJ$Bb@a1AnAGiAM$|h)s8`O=oep(0CpJDb5GK{3xK;yk*Uvz%RWYuKruci0yoclPRQ8@m5 zee9qm0{0JTN+MA!7I`<%gui`#KmNg&J|& zpeD~6H|0whVkMA3x@9X~jAoc`+qMm7ir9v|M8sjXY1+KNw&WJt8i1@AI+g(eyv#w* zoklx^qe=6(p2sz=qYK#qMP*XiRBxo=$7pawn`Ncy)3N9|WZg#1r5#xlb(A0sm`RbA z2vXPR*M2+_%?O^}_2Va{J51wRTSxvk+wUO{J2JQ8f4lr)Ny2)+huYa7NoWRYmuSg3 z9Y#)1RDaQNZb=!G`|#8bG`)FvhSeX>KL6lBvdPXHzeUiVMN^))L|=;yYQDqQ3N+j{ zSGw>+-aN(lU01)B?wBeWl(p+k4@r?2RgK?0LOnC&ESigAjRj3>Odot+NXnD|<=D-S zn*T8j)75M&@|CYsU6gOy}e`b`}X$PYv-wQ1B}mPa>+?SJBFgh(TD(bZ*3|) z;0_THKPf4rKRKH5k0UWTnt*Ianv&WOni;l?2nWNI`(c^0CLUrqGXQ0>FBK?Q5K4{f z3#0)dr$L5VuR@2t+m)l(%HC=084=R^JrFV{`&ys^6b?b`PY-2PaJ}$x1A?U43orz2 zc~s;ETvcrNh7bhwH0_*1)~1Sg0Ey(q`$>34EuqvL3i)s4%}>lLYppS{;B?d|g_h)_ zM;MpyaL$(R8$JPIjYX6#N5ZoaH&1+s+5Mb~H-4`g@WQjuzs!E5l2f3*24)+p6r>js z4Y>5!?8Ua(-?4pAb^y|5<9ko-S9x_}74G0sq|?TvA7^QP;@%Z>U%a?iw2>%IDuY7d zhHG~w-jQFuZ41i(8aQm51yG`w^WjY5B+uPw^Ywvz_m}RxMH6?<_D(rPwqDJ~DKl5~ zxCdDu!$-|8Oe%=*n>Qv}Zu!^J7F9h<>~}o7q6viZKp17=D*BRS5_++2W{Ff&dmoON zMK>O+%O$uApDi}vpGZ)W^F*c*4IGeJPnSdR@xIws0r&641=_!@@fh}o^W2tcxuOKfRcn)h+OGf;C z%MXN3em+{Y7@#~}f??P_#lcsYMIdLu^Hk0I{Q0vbVdMY{Fk%)<+-^i4iQh_Hyov2K z7hgNf=jCpBSFHgh#!3Ng0UnIY0GJSn17ySLalFmk)$=$tH&sez59yfA5)db^)qfrY z+$5PNJx&n0Y7_Bep_P@vCh^PH&LG zwEJIn4Pe3((F~{csdU}P(s*vS%9Xlp0TP9m@C4b%3Z{MB+yg7^;Q*z6MzXDD3O4V5f{;ik4(FY zgQ|#wP0N{At64+Bj2W%nqnX7-<50;E8ze&bDkb`pRaZ|n^!v~7I$ZN)>-(MU@iG`_ z4YCMMu-TEfxX|{qxDRm7WglMIs>(-`43c8b-KpN41OcY{cownCO|Cd5&p=@)nCsM^ zp97bWJ&eM#*kO~4-#vNXsjP{vepc9g1rx$dHmWykPgRoMB8or@-XV&tL#Pm)2Yp)? zWz}3)skf;j0OJ35#31lQuxNsUUya`?}jIOgu1x-y*ShLyo zNG9>#xRF{*0%$_uCJ&m;oY96egrMC_1c2f?D4L-pyH>aB3eY|xO+A%EGeYIuS|6A=~Bw7BoncwYB75g z{OgYig~E6OUO#Ikw)?k%lF$NH%lvYm^|Fu6)gD0j)pin?eG&Utv zVcyYV3qzpN9RCb23@3$*pJWOZQoknf1D7(RL8%|FmLEL875V+W{Jz)xtyl&INKf7u z7I6^j<&ev-vfPJr(K3!8= zi@j(2Pzf9F5$1<)RsY$7Qn^UM9HiI?1QcLT?*{g)r=n&zImeQBGi{*yH)x2Xl=RT-scey=OY#ItEzfRSi;I<3mcY@slP5 zBc7rPpe^XWnt{StrO-x(I6{^?l;$J?+qXm%GQScD9wMc2fWCL(#UaxQG1)EIsN^YS z6V2u+jX!oT2mDt#-2s$bLJ}5s@84nw&Awk_ffH+M!^cgWE?$7~(ArVU86NO-Y_nx# z2m)9z2^%4&BLri7{YfIb>4IBQfI1!Iajdd3=~#QbDef#3$GiDj!hrM_ec?<|hvP2u z7e1ZkwO&0O#RG4%z%=7hyS$b*%4m1Il6#!~=uSP;Am|fpJvxD@*Gi74(D+=st06{n zRd-5Un5tA^B?)Rh%ue?7Y=^4Rc1qf5h!HJ4J=6JUnHJB2$VIiO!OsdoG;^+vQ^D^x zL5q9Z9j|6vt_%}chC9Hwi#N@5KenVL4i3+rTwAYLTkj%g`YNv`@IDBz4U#ULr2UOa z`(I1Ur<@C4_!Jn+*?8`)>pxkBkrm|%%)=Ch#|y{*B^|XKhIm%iKY}qO99c-By`(B` z%=j6OxvC(hYz1&mID7=y;qL@tH_hl^Yf(7Jv%tncmOQo1^OOXL@l07HDPK#~L0=%( z(CV;|G9^zUxuF3it_VVu6=gF~!3BgrsYsmF*wGDX0~N)i+Y7dg$pEE*AI=~`-ye$y z20T7yj<3wEeb`DVMwpLIxN^vdSj>;-PtDCRN@*rR2M>b;8_l-b(hh$=8`zB!Gc+5H z8dZv5HV!Vu<}e&}FQq8~s^C|T?~-r~Botzr_^9y$NO!xIzWHPQtWPbn`Ib6>*W?;e zIujhiC?5|BW)eh5=3Q&fap z1z!yFX9rtUz@o?II#*O#;BB?S_`!xQ|L;@Bo~EXknmxIjU{^4ttaL-H!R0lx$+!Qj zT6iN+oj=QsQb1vYPzf`wvsRH)xjCG!R1PGoZ1Gw|>k2UzSPEX|S8W zki}ijO0fFvBBUk*tWW7F)bG=A&KTWp2|p>pDSG^Nc+vV7_%LjDx}hR>40{nY?BQvO zYSwN7Y8KVf1zM$a*U#ErfmQfaUrOUohAU1)u{$QHNG9Xp{#%3k5=*=jTSf|+?^&fT z?Ycp*&j9Hu`{<$m3KcBczW+2dd-#u{`qD8_nKOK-UxF@cPTRHV>k{9pXj%_U7q32M zq*tkl6tEb{C|el*hn&aJ1nUF^@ul2(sJNIVjz$apYyH5o7fPkaVFrsa4e>~ z!?Hv*O@Pap+d?3|zFn84O%rkLvDTFS{`Gx_y5|+$k1`E9h6`RGVG8x6OpU}ApfL%P zzRLIxh<-!+zU%-=qq8yY|Cig87tT6L0Q>l<$TES2$n$RpjSx6Lbb-bb>d0e5V<6|r z>6df}GXTdq?m*Gl`~4|sJMz7= zu0#lHFrIvlljO&BH_k!UAUOm`S@1G}Y_l)6y&g`!W)T!!p{V&HW8bl3_imHC!PoZ! z&4HiL>Bguo{JUs983I~vegRPI4kPg~he;!T%L2sn0ahzg1bHBgil2~>&hmJ#aP-jC zRfi~~mTVj`%jr`*wW?RsGNfz@*8I2L&Phk_8HO_c>+mRhf1X<>8$7V?5DASfCEs)C zv2$|a8le|~)H|o9jr}N5pF#<5enF9$;P77Sh8B)4{D9hW!Irl+YVFX>5U_J3`XGc; z?xEgAMw!DPM{ra5tDePkKOFa{mV?~3cMet zBpI!QwSkQ_sxT8V%FtVG2wnLsY@;lraHPB5Kpbe3K38!_tCtFm@^hfyxs6f7VP8dc zO%0kJ2}u7m6Zby%N0^E6!jsD&&}11?yrWd%#UlAW{~`I?l?;QIhWEUk%F3*PwMO_3 zU|u=&@AVhZSR(G}s4pNE1HwRWHjt4}A5b|Wk`A#kj##xRItHU46Oc{)!vOnJ2nw7D zf`A-~_al`rWN>>aR~lg`np}u~G7%&}e*^QTUmb)?=p=#il{(uSnM||!aT*P;>PWIlan|aaL3zSYpagHk-RcmPyG1 z1dKw+PxTW4+Qi}y+H)sup6sMfs)V7<9>rbIQ32>I3_y_;RxrE=MipkC*jIp6@nPN~ z`Q%2%WgE&s4mD&I%1F`OD(>hP_~nM0bbH42b&4 zW*&$4WKcu_+Pmk}{*7l~WB30%&QC^;CF?>1_NRuN?Nlfp@&GS4LsOpsZgAOf{JjcM zVtry6^76D&jlaL#BS4ci9)B!GgvACkLJRIr^|i{r*XBCFfcO0IGd<3(1@98~IK4L@ zOrJA7i+*8-Y*1B*1>jTGcJ>EnREwlBZ*5{nU@6D}QQO-*UE^AEx`Xmxd>=?PsdIm) zw$R7VmfB)O)t(7HM}9d-M+(NZI@WXwAGnR9V-C*Tuc2NLKmp==`VTK5elhBF!ZKm!86JpIH8FU09uoO#de44v%Mi(-n@G!@KZIy>&k+*SnBhsv*#KN zczC856Ax!$sV4%Csz16>;r{&_=xAUkHZ?%$T7ZcUxd2%{OXsa7uP%(`kE@4TI%71; zHd=HQ*k}2U-8jd8k>C?N5zE)qA+E25R9`r`hgT{r!u$`P?sSrjlLSV50&IbTc~z4!2D@RIv;MTvwg_ktfW26KAt%R)#0Y??6~JsM)_yv-iA-QJtn#%u(@1=g;rgZtAyfX^hT)_&#x@@%n~U z4CgY;h23tU*YoY(#sbd?{1l+c8!;OhLM&YZe8y>kaDtCCL%hMwhro~Ipsdvx!6d9~^v~0X zm_bm*%uTnx)XZtTa{?q%Z|$z1iHBkGp~5*{aew+26@#edpqi8HC2SBnR}}D6jO8{o z#V$zjDt{{TQ5$LTQ@7t0S!Az!@Oe1GcJZyhRKJ`|-+Z*HE4&9=WLQ0i;i#Lxj~>iS-FPHZP3eQ za3aRBIR?Y0Nif?XSYJXR%d;i2E@D>XB7E#m$QF<}8Sot3uArbMa4?%o$(|Oi>q84S z0Lq|azoK`?C`KVZ^vz9r`nJd*@%Y!LKQ8Uf=mX~N@X1)QSzr|4G?;Pmd~vRk)$Ob# z^q*9(d`ig|*Rw>|vjtt=-WloYg>UAE_vZE@*1-xWkvlAY`G#_;lc?5{#ROqJgn*AQ z!JKs11jIc|O}K3+$)_M$VhXWC*euv^nEXQ+LXNkCdeVQf&CEK*D5Dt%CfyEnV1NG= zri{ClP#1%~HT^D5l5R3i*hEO=Q)a9`A_m50A;~+i$G?G%n*K?&SnOLrEfu>FY0Aj* zM_H9Dv!ek;G*we1{o$7Lyah?4@sKYqtK`$cKQwi1N>Xe5f@Q)OsPLfN;GBDL>S%t5 zkt!azWyO{hOqu{d`-z=^ojc8Db-y%^dY`MK>KHyKAlq-UZe=^)>uhMta@%~Hgb%L< znlopzu*X4YewL2&?EF%R!*L|t+6hz^SQZ_=0XqrNd1j<#5JUI+g$uRhs9j}1|7K~J-Koz3pNOSo_m2hq+4aQ z;B~Fo#!06xyA!~d$QDIsel6mLDCDBw%E25(I)^wzC~rESbR2XLaDhOQvmIxxr9Xfl zlEd|<1Su{gV+?xB*h&7HZ>{J(80Cj3SII=`3C(l>9CPU}B5b|fI2nI#yc@%JOpurY z_7$19_5up+1DgPtn0aYIRva+$o^|@_Sn-H+jcn0B2jCK-gHr?#B1K|0O~cGd3bsM$ zlS9fXkc3%mj*uqjAPt&SEP+M(S{UN})mab^Kd|w~aOZQb_uo-U^S4-HeY#1U9u+8M za7EyA@Vxk6Y;TF)&%T5mZ|~V)pfhIX!H$a2a@Lbj#c3RqKRbnZgdkCnFPT zlW+cWYm;RNt~d3^-mwc8z7^Gu%M+ERDUC}`3jR4!k~acoYBnVbudFzICQV!-iwhiL zdeXP?+V$kj=5CrD`v%apwwW*l=_+Psj#4mjzYM+^FIH|-o_E4nnF$jnh$EuwWxHnW zoAifi5X2jlhzcE@u}8Fsz3-9YY^lpjV^4n;#Vxwf@9A8{W~gIq20!*5jGmC)Kpsdk zCzq?rUv@{b5C@-I&tl5_^@9g0p$$HxqM}~sIAp|%!-5V+!Ue+K^fEHj@RyW(^nUL| zXh2~lut4j(k$yiQxjnplxrhR1RJ<_p) zzv9!ki^V;T2GNwAU%BN48*;Dz?;b!z_J8*P);7YF%6GiN>nwoC7~4ICvQQ5Zg^l3_ zS#6U2`~xc;LbJLyW}jo@agcMeAQkFEl5{iOSRVuCHAB}I%Yt1A+wXg(3u)=Qq!;9| za^(&8`5SL&N?7WA*l8x(zB%XkyCLiPkTi>Fd0$R2y$$n5>&euG7pybc0G4vF)M$zN z+yh+eZZq!n-qdlS0(Zys=3FKGBX|)J2oJ?Cmw%Nx~xL22mkZqS!SH*8#$lR@TH=?sc1Y z6{SaZH8@)k7ja4wdFm=X;TP|Gvebf9m(E&u8Emg<=MjujMzBgIS?J*}zjn0B%vdFg z+)^8L90*6jplLOiEbHKEiO}i&-#pOta;GaNhvtEuJy-dwRU34R~M3+{)3RPkB z{Z(B{2VFEKBd4Ntwk^&T-nL>VZTt>YW&b`stNW9CRPR%RbCB^3Xc9bcejpi@zvgZ1 z)U!R!k&Zd@-pf3YlwXHQla2KQPD<%}z<9@Z1!UoTR@orI6kkM4M7gPZ*T2o?_OPVj z`K%V;gS#WpkOAfTAKsUB@#@uTUB8BW?gC!1i^=_?QGYP}CE(d8LN_UNgNlwrMeVbS z+xil!F=Xgd9{P7~LF`LhzG?~w9LaMx*Hw~9%cKvUJ9gA4o&2%PtHb2{N8OvHHPsK3 zX`&8-tWBY^3XD$J|9cET7cmudPp1qvxUWB*kTSiCDJ%#Ly=qwgUH?_)?Hz$h10Y-w zGPK!kp29kSQs9w`3^7Hpj(glLldY$k72^!h$8C=kvKBCNq(up%ckaIc)nWiioT=8w zSCHPgJUj?Lam#SBxw8pZQ2l>tucbeI`sxTTu+gx}vH9ahjarit=`>{*96atgs?My# z>ZUpb00Z@WQb@88K?u#b#^M&os(btF+XV}5(4wrhDaYT%j@8^5FR^Q1^pDI}$~Cln#lXecri z5@je#gCey{cv7N~CZbZP6n(Gt**WL@-rxD-c;BPQzVFZataYtx*z?PZIsxkqU)3h! zSIMmpwbyxe8*7<*9Zu*C6uCU-Y(U7?*t}1#6uUBJyW2m>QJlVNYc_-o3 zpH4}*;B~?rW&=eCkw6x`GnX+7cOOJWp#|!57D&@;(nO3EFxd$R?g=EO2k*9S8Yq4& z-Isj0#CD6qPkJ-&>?_f{BJ~kP>T@Kdl+ae@Kcr}s;hdBYWtmxQZ`YZ8g1P_;BcrER zuYtnN*!;SOzyak=5a-X|tReoW#B)aQO4$6dOOjO7nBQop7;FVzP22g<6!EZn>{rWzLFr$T$_NsFWIG26*7&Q1Vsb!w%eV&#?4se zk$?0^#`Ae*7H8z^%Cm8-rK5IsWIwg^V$u#-tnK_ETJzM-*61BX1$m*{mlTTnLVP63b8!&F%^VcrN#XU>P+W)86 zTR}mRq~YI39Mq?Ho^c6Ph8CzJR)kL0=X9qR>2KNDwpYq|yen&VF#6CH{2Ma|E_f&$|OJ z8~D9g6}7+9oB4;p0Va}L(c%u$>s=$-b3T+E&#*(p+ox*;aF4hUjHyKdL)c6?pd(Md zimXr*l0vW*5hg&89&7Qhx%av7BxsG;UZ55~oscSq@==InV zRlZ2W@aVtiA8r>3*ldv%m-hO7`2EV;%`@l~_&xex0TulDWTy6L{xsRUKlGFxLEyt0 z#$u1jhLD-Q>Dg0@Hg?)oM(oPd|8Q4E%a&01Rglc*`9R4pSecpWie58pf(37;IC7r; z%i?`z3Y&8x?zG`&16>9rfua8~bjWmpIrMr2TiGFq$n+tXGWV2TPLjw;>ifkq9i}TO zZ7jj!k_s(2-`A;@#6yw{5SdnZld{!2^VG-59QlCfsXBjtoL&7^S$re|$P|d`_1Q%p zGqt^NfkhTAVSFB`k=W?nQGaI)qz2L`f_@zxwr@8q0fS63_OwCPsdG4HTe=o&q9ex& z;5AfL&M&bNq^NU(XVA{LegKw0zAJ6%hmkx?nkh8d3EFb&3CtL1pa|la;QNLoZI&PJ{C&r+7KSA+k=0DK>biuE#5AC-+ zv~&VxywuX(>9adFgVcDlKC(L*PcLOB=9jh~?_31rMX8R)fqqX~(e2TaDYg`J0pWzZ2Yviyo>lbOeBU3A+!h1?wNu!adHRX++I;l5|B?u)-)#x(q>v6YpUAoP^qMN!Br z$dyB08Tw^rV2Rn@@7V5U)! z3w!{He^Rd63=;V3;{H-5gz3W0S%J8g z>VN@w?E_A_f+J`5;OPQ*iysHU^C=uuvy(;#U}V6REI|30d#~U6T*`0-2C9D_OAr{} zuOElTX(zUQUENi^Gx1>vU11mhh`>6+(J?HAI#1BV^og2H#zLry9Jdh7ALu!Wb&pzc zTYs&P97Ri0$_S-*BpL}DtiuBz#@q}@LCRpf;`GSP68uS{N=i*|Y0sRonP2O5j{5U7 z^~IZ_ZnfsUf6tq~daLdjzVB?E+IQ~1+Ug=Tj^qTp7y&ZmPU@w6z@NrBj5zkR7XD&x z7Xq`QIGu57r@-P)a`IR-Jf(lQ@@TQ)p@8r3H?5_pc?fg?E*dN80mG{$9ZG+PwyoVhe|9NKvhF|32 zKQRAP2#ATmcv8BHv2d6tnDCuc60>EGo9fiCGG8hMThx+!-dJM4&HfHopo6SeqYMZP!@7%q<1ZbIcqZ7>UgTR6Hpk zN_jewjFKU;P;VhMZEzD6eCK0JzMM($Msnp3u~W>)Q2Hk`C}j!t==-zA}a|9pq}T7gk++Fb<Cj7;wM#XIUb1HfEYE zz%LiFLajHbjcXSWv3~}qYm)G8=IbtJ&$u}&f43Yn@4EHzft&A&`x`pT-@CQG*W?$k zhe;Z^ZyS=JiY-gpHd(nGtJsj;kayI{`;smRFC}len#7vblm)p_?MWb31mS)n(8AMO z@H~>V`0Yq0Hz67wC|zf5HPR!3G`ZkSKxdPNASnvwiD&^FA-6W)Jlo1oly66JE}_OF z`N|*trWu<5KNt`Ok_A)xX`Q2(uZ2oV$CpU1cm{C<#bRIs*JrZEFQ&Yrc%tNDJ%ovV z62SuSL7Yy;)YmDg6A)p_VhT#6us#yX#`3swJ|$UV?z7F?bkkHGVJMfZ2I?&VZ4dp! zRqEpId=5t+s0*j*Hr(6KXkvI7_b|2R~^3TQ@;lW$q zU9fvJe&yB|9{N3@D>qL@`j`~SwQcqcov!Ec>@c^+s~aNo4RNBzu=`TEBkcg9i@~m$!c|DntnN=1v2ubt8gifenf&@RAaq z3tBL6tD784Dl2Cap7LhUwHVw#$N>mulpu7AxfJP#<^yh?P!fI{@mL8@d|e6J6lM$w z$UcZLBaqKRNuxzrazZYcB5?Z>c&-|@5FER6WY;okmSusv4;at|+Pn2x)PwF*Kr5HA z^d)s*9em5`Q|0*wM?jYbw-9}!jT`0nEXtmbKo=O_2{eCyl>kDnpN>zCbgWQ(e=5uH z&487?Jg7Mz(r}^+Z4)C%WsDmKJwFL`5ZKAsx{6;xB&j$0K);c zvwL@ERQo>Y{oQT<>hcOuMTl`gW)oKu6JNFwPmK74@Wf4#1$9rEM>2KhH&(eD8+CG^ zOIzMQZ_ffd-6-uXwZ9(v?_4u~#@3&QYf&i7r2?MSWyR$rv?F-)O~0CmH23mZ%zl>F zd>ZdMcj?d~8~llVYaLOx?DDjV33%m97N{6Tov#n=O^me+~OjqSzG-<(KU6X0k9-a0l+; zqh`v>$d*U_j*#IFT_f@si;|FfF7~3Io-tsalgubcvzy0#CH5D2@(8-6z+q!?oQ?x7 zM*&1-dtZv(rE)_HLTEIke!(tQo4pZ~CqR)pD@?~|cR0V!@)Kt`A1%J3qS4stO#l>1 zRCB?jdth7VVU_?LRGogf|K=t&eP?YW8M6II!laUUlZ@BXpehaXc!KpKX^vCtqo!!# z0)cli75ucdkqW&oyVKPd6)Gr9-J-uc-R$VA@w~O-!7sFOPIjCSuidc5n4t=22>J@i zlNyC3U1^Wj*gg#8Qg)bZQ=Np&{Q50=TX0C-0CL@E!r5LqdDzg{(}H@Kz2e@vOma>*tdURknBy?pXFf4% zwREH?dz)f05~YznN<(LM&299AmTAk&As`7u({}@!g2`yT$Y}WZP0oA`C646nixRDV zi(JWCRM`g)on2fS7ERS%27eg}LK}AWc*A0DvMe}Vc#Ggw={gq_p7qAYafu_HbzxhS z^kc93`t|FXg~>khd}M1LXke<P^c$piTPsQ7xmg#rf;NSC72WQhg)< zD{7K3Kh;Ij9{NS_z!_DHID1VWxM8LwDv;6iQSaCxl5>U92|zjD?D;em+;yXyrcZuJ zTaUmfQ{*=OenP?tUkMTJMI15`>3yK&thzgtWd}a^0;8(K=pv~)get-U6W@DcVK$I(r@U45y$dF*KSDYf62{8ZsH^DRlEBSD_gtD?%zBa z&)bt%ZmHerLll%_bN0B>EULTYL)``PCM|#3FIc1i=1ETGS2xso**R@FGyhAP z~Y{kl3#gzf<_ zhS9F}95N~K?%li7nqcp9B)bLzBaU0lJo3?VUU~S?l$H<2nUaKUqKCc&MTt!Qn8O2e z6h;k;B0a%k=kocKT#b;h?bL;|)vk~yF9dpzF|cNS87n1qyS(GWn5q1|#iaSd5{egg z27NcJf}PZUZ0x5?8$`md$w{W}(R4haeSh>H;Ll8dfNVc#y*5B+nd4FajyNO8xd`hp zko9g|!puI10eI;PSPtP}^T`W}iU-hy*6q1Ahue@>k%5+Tmjkh3gAp9P=J~H-Dt8#H@f$;(XK$0p%6TzP#Iu zIjLp!Uf0oxLeQ2CfY1QIKI4Q?fL{7`PS1gWKZiN{c7K786Qu$AcK++kOnw@C-NUwf z;XCPCv6q$PCb_@{E^LVlT%)LI;iD6OMhb9fiuK7~3IwrwbziBT-m=u9+_?O2y&pQFO^t)D0pe! zCu2=W9I4;odV?l_hop7d>^;=dIK^_zf2x> z%a4zdCvog#d3h``9G;jjms_Y(iYt+m+ADM=<~!_3Dx2grW?%E~W!%thYqW#TIn{3& z#eMU~Q=kxDdPI6))n3&N7*Jnhfo7dWbi?o@5xhAVLdJ|cPP$_rww~H(MsK01$O-H0 z+$&hw0kdd?iC5B=bJ(z9K4yR^fXpBXEVCjaNQu;2!?}kM{&kW5A4Pr@Hhnie92arx zk%-nD6^R2a6n53W1Aef&(w#D+M^6RH2!Mm*XL-EO`ODt*7IqB8(E_O?Wh}C#8lvzf z?#sVW1!#OFwK1U~AtaE%Zc@^V5DF~xfOt0y(utwvF@?EzP?Q!+e>!(A9@m_D8dW}I z$UT!>#@Re+=@8%?P9bbT!!~E2=f^AIAs*@?Jat@?uoy5(LNomg`(n})$lbTVT}8Le z*S*ERbZL16@p0{#{hDCO1zV8F#mz10KMYmgeZ<$YJ_RGGCrs1r-`uu2W4dG>)R0WI z2Dsw=_kBvylyFRRXDTjCY_<@5v1de8>H0Ny-!}(%REripx>$vW8sJ>88=VLz7AEAO z$(f#a`teMn^udthDsJ@~w)uf~+$$)Z~GK~6&ZAM$zitWTmflQlFfNk&6~wfQfGaWzl8 zohTUf4EKVZd`wDTzux)zDPPsfEa}}BRqsw$Vw_e-`e&Z?&zaIl@+(SH`NQ2S^I%>GrWAh>ZWM}_6M!xf;*HOq zBp=3=3jLH@G59|Du&zT@fw_`#{5M8O{vlxljTo{jwg(ALNDF)=0U*02qqnjx>Lb~K ztyqD90hQcZ8OdA%FqCaPe<1%j6jss31~)If18t_(s`qC&%SSPBbi7WgQyNT;1O^Ia zgbRUtVBzRJ5vL}v9R`B0H(A;HYJFQ&zp!88fd?n3wrBQP%j;bFHyO`0Cs;P2j3*Z&i16|B%AhnORf2;E^n(KGcUbmRf^hf5gDZ9WGj&))2^%nbcJHg-pY*v7r=fo z9;%wGs3fSPuR?ZXK^QeLh1lXCdxXSaxS&PuFyH2*bN==%uBDc_F=G1 z2V<-e4a9?8FNG_qzDESh8K&k=`uEy?p9@?hA?*c%-qsW|(;1v~Qo%eg`0%&TQ0KzYb9E z6&P_wglnekx1`ku1|!cF0t)^~CUN7CT~6sA_E`JTqTsV?)k^y}Lnd0l(Z2_uqJ9`& z)FO>;#f(WJI%x=V2k2p#1THMpA|=t>H(Vn$ybuq)?egWx8vZSs^Ujsz1*o+sWtl$YTZ(Wd3U&K8{EWs@D&aHj7l74 zb3n^?x}xHS65KI3L2T&p2s?L95qur}Ek4)OOmT@BTZ&Zx-a$((xfz5-Y=>P39h`)4 zC+sZr_W;=wtPJ8F`*pps)%d{$46eWES$}b#manRnq2T#*YP-a8&x`4jIe^1I-RsCJ z+TGnZGb#cs zogNv<1;ZC2A8=ic^06vPn!on^h!*Ph^V>H0HF|m{ zX&1{HTTxpxNUg>#-nW4=U!S@s-~x!jKt(CJ%x1X^8-*FL*&UU7{*=rw@)=Lu><>#i zdEZahVkb|ZzJd{acvp$2f*67jn1$GoFrYPckr=L)JVJ^d?E8Mb?hZn-n3*?5uPH@0 zfwg?QgN*%1ek*87% zwN-i-4(4KxV{SNjus=HMz~y^Y;JaC{l&sBhc{Z+t8+E+@;IUKK0-GKsdE6ur5<^|Ev?eGS#OyR1KNfwEq6uhJ71tGfx7#GNG_#d+0G>utg0mQZHKYgm4?$sZ zu1dcd%Zo8ZYi;jvsq4&GPG>AWKVd;071T<7{Ksaad9P6*SU68sDI(S&-qtsQaGo#A z1xR9Mgt}+-(PzU39ioIA{d%_twcy_nGso5p?@!Y6ygZv}!zYrcTB)HIq@7RzwB*I| z>jr)VnmI_8k6XI40-}>TFl^S-VWy9;&SW*`7mL!!02tVXog`R-E>9)AI49wQyM}J< z@{hanZ`?|sfiHMbthf;R1hZ!CsVH)PgtnR1j__=y$u|$8?zGPLfvUS2*P$J`x1s8+M4TzJ)O`uDtvk6iVdFR5i+%*;dY-7VZuG!2ZMEuq zut9wf_SL<8erY#)~v}%-}Phf?(+izua$EKZ7fRp3{oFoRSVdI6omYT zh^fNvhb7EEFUSstPW9>(zgZI|EMY!MR2#8Rq;*?j7W_|uWZ$^(=T)=d& zRPz_AsGQ1r#7QpspYu8z?{Sj(CX4${(xn?kv`!Oe=zMs1Ec`oMiNZ$DDG7hFXY+#w zWSlNi=U&#}NaeSV0?h}mq)>ZsIwZlyLhN>Jk$U?vB7HdYug(}Eqdx6`R(0|{Oi$o< z()6VDIqv%PU~=8YIwd&v>eYO*&tU{F#f8AjtB_*-r|zampbd}|#gAG?PD$2lEAIHB z$?f@ZVgfQv0Gg0LvRn6$9fAgI1Hv+Tf3@^1LM>I+LdRAMdG@Gm`ID}~l7CpY&G!GtR(5<*flX_MTB7vis)qX)IVT+p>&Gk{wcm_ z2`S_8Gb#^e|CHfKCINRs2G4uT9h1WiJ2eN2W-&$WcQr1+2r77npjTT`i@5zTUxc%z z&x|n^fuQa1yf48?Mq$Zz&i!g^EE0JAOJGsN%*qI9>C?WxfqLT#tvd=jj1TC&z*jI0 zfM_cP2k7e`#YW;qFWUgx*i0x|-tWaH@Tp0aXx27T!AK9_*mB~O=smh2wr>Fu5NCn4#`kt2*Tt&>da9>i&E&uhSE48;^&nG#DH z=~T8Z7xqHx>It=YpgW|-1?n{nK3pqM82_rKVd7l=qNdi~pB|6L*0%KL)B`ZZo2rF@ z4gSHr->>H&mI)!LiIC_#L=w=d_wU~`rxL@nSq(sQ$kwFf#S6Jj3Fc(x78C6`d{^Jm zL_5YpKKH=9Btb#MCdb0R^0%r2k7catEw#WkC7j-zC*(l zw)kvF8i12JZvKUy4(ZXOw|a(y-}G%3u4rI}Cjl0jR$7dqfdXFh!Dr4~zh?366Xth@ zTdrRp{GVMY4~MN*v$rdVtaH2YYS@$2jj7FTASAR%3tH4Xb^(BmE)?MzceWG`L!Zaw z&)-lsB8aaOToes+E}-Hq%4`fx8H-sIPgKIc!U_$N5ifYjrcjHstj6Td7_FKz;HFU+6~(0ZP8@=9{EiS(bbJQY?E z`k?C4g}U>+G$#hG&;|;NHymD_6)~^f1tce+CLlQ(M+7OXjy*j(i zvb#oFN2$Fvmeic!H@HULx{(aW+K&fP!4bn=A}^1He4~K*DPUOziDC!?-0Yr7(zVth z@J`=$r%dYYsEe?nsDpS&Uk%BRf_vUvW;N+=N^o}shlC*1qSYE80fEzbSe)8}xJdAF zW?Wp{+7es=H1Xqy%u$DD1??KYeg8ha3uoURSj0Ykkxm2L_;tzII9z17Ldiw8T77;* zm?xdO$Z?4Rj|JV|_wL1K!jG4S(fr+r~4a51&YpXN|z?Ihe_-`=R)n>gV zSpdEo(~%N<`nPv6XXNC{eJiQszKeZBz+m!N2O4G425ZoKC^Fx4OZ++kY@rz=DfPVK zx#zJEy)bxQxa2he9FGQ9tgou@KpEx-6nk{!RW?=dR*Cb)m0{Z_cv_39)?=_N<1EOQ zZc$?f_NPG|NGoS;FLMT${oJ`PC!tLw?Ytev(z|JUd0e9`S}YQi8yN7#PoF)j0Coog z%AY>szNyG=D##trCE7Q>xELH9978U_=&WaaN6G-A6@u$1FhZ2`6R6+53YQ~hWZF5q zzI!|_b9EO9E5Kb@|KZfC0^=ZN+!2Iqpa7T&S`~EL%tvEs( z7E)9;UNiFy9}A*n%CFSd52j|rW-}L~kDNV1|I0ud?KBhD3WQl6Q3`V)qUr**#vV7I zdJuQ;0Q?8c5}shCV@%eBF=s(Phgpn{FuD-02~msPgS`0WDXvR^;$F6+AXcU;z{JSU zTEA}H1Qt6lKV?cjP_44flf`g!^Pq&BN~Vt9G&o~xadK+Z-hbRx@GT;{K-8qMRj~^s zy1K+%nGanN}LX;0g-@ zB!kGO`uuuWTwLtnc+ep-yYuRWd-1d}|gq+}P-Izz7&tzJa0l_7m2n;-JlYN-_ zWj_UQEy;oYV{>zfp_p$R8dByH)dw4ig(GGhLya98W>r)vI-?TA4Kq1gw0x*EeW(jG zVa=b&zyY|89a}k7_3hszQ1g7IN{}R# zoFx7a)Nt+At?)l7+duJA}*SbnGe#-+y3oH5?i{cjF-c)jEV2ac?n3v)6 zyaHWn-5qS2LYm%7DbtNAC%jS1HB+1MIOXo|y!6+ptM~t%rCHJV(n3`0;Lnfyf{OR3 zbdKPfH-SQKXJ2?N}KA;#|_;I6eoZ2HsI1I%Q)Mevua$C;jS>f3EioW zZ%ox3V*5e5m-x;zw4auf-s;upMlagcU;O&w7HXwQZf@?KtSOw_7IKz&_r-`zVu|r6 zk?)-l3FnR2fihU{hC|e{Bo8LGWb{b)yWB*#rV^#E-LE@>3p zyf2(&opZ%Qc^X&C)AyU@;dg3*JBHy#a&#Ouc29HpbfAcNxC!NPjr4(n5UVkqw`|f* zc*z9gB8nX$whW+#Q(4k7tT64n8d@MEBZULe*Ksp5t?5hTUv&ZMtOL6xtO%yaGcmxH zycm?oM;Vq86PDrb>;&whv_ z8tYR~l({NGc?qE>gyAQUdq)#kd-?E)q{&J>j(EpAG)6D$Tm)JnIY5=+-kGmCfGcDn zrdzg5(OxuE1!iwEFmy92D46Rs!ipjGB?uL1nhD-$z+bjucs&3ltOBc-MXmvVnDzDR z*E45CI<4>txRDC=ELIhwU{OkOc%jMhL7&VfV=rHRHr6F=f>11`h60?`mbw)H?|+;i zm8GD&qbEle@ZP_Bx|lQVO=T&9{n)y#!(f@u{^Z_UzQ1l@URaLG#)i~Hf~F8g3$gQ? zwlZ9Rgv5?9oiIwdbZBNakU{iIuD(KFke-eSa@o~CxX0nMq=I*BTJ&UWZIvM>kvn!y z?Ts7f-~X*TXFS)Oiz`JQi~_i;91OKm*>0g99Xq;V*G)28l|h&tbskW z1SzE$xQ-7t{GWgNQIYONdl?(wjs*F+BF#luuu$8&HVII!C%kVvHn_p$*EQU0P9 zkVb9Zabg?M|IEthS)X8)kx&aX31mIbLtF11P)>5G(y;#mRD+5EHWd76dq6HEejn?q zkk>+?PW6w1MaadKSx;k2N)`o0htbaHB!ZwqSpcj9##liSe&~QNz_yw~n#m0orDk(i z8k8ZGKHWQ{HM~=VPZcpkRMi1c7Ppn5U$qBosPCz2`PvAdYRg{QZyX$8IYevz-_nPj zZhZ+J<+2tOo^|i<&ztscd+KC6SNCyTbEt6GW>)NU_Zv>qQD~p?2?f>GwLpwJNMY<) zq$Z400AF8e>~E3;|Fa~aMnuc^!N2v{Xo0ndSJz;KKZ=ex2@7f**6NIc+!5`*v7BuX zoyC7=OS}`6(Ltv|yo1?Rn??B!aN4~Z|L)y8EYQc3=TW}{nGB*MgpeGA-}m>CXf zAl=A&rRv=aL7P=t+-eP`)FpMVtq{XwB~@GD!o5%&G03Dj5~|Cl>7wQ0w@PVhJ-t@1 zV#H6%);q_&pLBVS;WehBWoZuG^IWu`GOn;F?K6(?$4Gcb@~m|V#V z3JO9RqJ1qt@6oMYb_xG_Qt*+CH>@c5^l8TX#Mm4FZVB6;?m_9J#Vh$p!p!gFFZtj68fV?l?96&Fyu?Oo{0z zw4mub&Y1dqA#h_zPuJCD%!>l=RW{l?5atddYW`bwCyBk1$u3n!VogVlZzYOo-JLCQ zh!*6A{XOht7KVWNPIEb~4CTG3Uv;QYEYxm=o*ua(w_bqbK0!E}FPPQFUAvaZ>rZQ4 zyDeKB0d959!k|Js$AXRx2V5zk1d;*UOf)D}G zJ=IlI)-es$mi~N8w1$Let*bAJzIAOd$Gw~?#mfiRZxXm^cfHj>RHsCZy+J!AsRp~9 zAi6`Jdki=UJYuSKw%W+AT(yd1BJGRR2LXXX^BYd0eUI|L-QVXNY3s=7J->As-U6{D zWJi_pt0ZY^GCp8z>%+E|DPO>Iw3b8p#aE;(Vht15Yk#L~*CtLeVfW9v+wa^vX7$Sy zo&WjAvwN#En#XXGDyZfmaKAMnG*)Xst1`GRN@L$|>T^5XNRX1VkYIoB<#zlGG)me* za&Z#DM%az$$hOxHskfP`by+z>stm|CMltjil0%zR4h=qpSo_+B28AZDls?i!6I+*^ zc8wCO*m6mf+ZW+)u+w|6EFt~<)Bp6prG!0ncN6yT{QM2V9+qIW44>BaHv`vT$GWGb z@}?cy5h4+@x{DlTBNK`73pX6*Nyo(UT!Sd$gV{M@J!V$<_W-_wt;}a21~UlUGZfP* z=mNQg#i5Lnm7R4ft?ejJR>K?vrU#kv0AE>`1Pl5j1Y85rZv-Ks4r{Sc^v0$dW^iYs zC}h&?%?VT@KL_`Q)$GB+dn2(07S~!MU|Ajh|3`k6*U~#0I4hlD89%&_VJIMJ7p3sh zfUAwk=FL!^SRHU>fWyh}zR+Ih8Lma_+7zu5Ch-sCu-$2OkxSFIFxLpqZLX{P5uWo>v+Sx+Y&p-zBsfNend57)> zmWPbk?T(jlTJJb*s4OenR!qQSsZd8Q-V}KCtX?r5#ZrvIkw7kMKx2FId{L5*8Aee#yoF4|_ z;Q87hfp1z9aNQ$8(6oznd_WaraXCSnzvp)}b8@Sg@|V)?4ESE|Xp75W+7nFUBBf34 z+_{r17cvKl`LSgW{5EW>)%Z);&IZ?Oj2*s$*&fBMy96Z4bt*7hvYxqu?nnqua&iyJ zZvz+hG{7~6ZrBC%C0!nK|GtDWUf_nkF$qr#{(MKVipQaK`10io10APbZcgkiHe7@B z)AXqIQp%9%gp$(2qday2sw7ihVx*ccE{gB!k2e?-+7!wgZvz>CkzmLS@Qko?vc_}9 zpWd3==craQ%}%P#Vn_HlF{w=4GvCF0et8YIt^S)UAC4aqd@lWxmoFzU^NfKnLBAt%y}f^!6|yATVQ~k>WQEyjE7X?07qt&ZjWS-AzWs zeEA)o(Q~NEpNlbtgp#WGY|T47PP7J>PaATJXAm!)EGtLAT*2&@n2nOs zW`Pdc91Wo*t!r4@49X7e$cfo?bxUFc{+%QakylVu3}Fe2ghU)56CZ#`VICuagV;e7 zhMq|dG+}z(K%Xvan;8)B43Dz5_jgM#0nEbT6~WK7Sm1@gPEX!r{y2Kon5*iRfB~3c zm5qfnY>$etByYiI&6`=jaJyCpX?TVXj8K>A|)Z;dbY_W4X}Z38k;lf%>Isi`?SUW)F!UYme}q7|hL*UKl~&ImHWGm@zU za07Dtulq=>b%VI@_;LQ5EgHh-x(UpEtkj=Ml;aw_8Sn1qMka{+sf3;z9%$^JySY{R zuuWO(k>gPumk5WN>9b~SA~y+XP=Rm&`2H**8EjRwS^H_Q18+=$<^ZKqNsilx&htp_ zrF0Lc^zcCk$?Y8^%+oZIT0y1vwf8OrqNj~`tj4J^=&8}kibqVohrT#qZ2Jo^}`|}rolOPM8^g6 z!!aBBcrx@VcvDCj@-2Y)A8^IWhV@O@?V@iA4rb#oj``6)ebWy#+NHxxro6|A7}PfYvrZ{QU_S?%ApyQ z>*o1+P6K*IipB^WW%ON=&cJ0uchg;d$-3H-Jk#)T1V`Sw3FFh-@4k@FCn7VxN_hi? zAwz0A^lbzKCC9BWg}i1krZc--c(w`AU<2}ChMsFr3VTYyZbuu`3 zFXKQfW8F|+5cLI3cDrJ}LH%6=@R(U0;g3}R@n$pC-)0hqn>5gX)31CP_P4bD%EQ%9 z0Q<1fqfuP(L;(aJvFSait3` z{f_4tDxs_({A$>sYu$TJAp4+lUy~A}j?Yx1pi%NHwZrH3A6F~o7ptQ+lt5uymqep& zhReAA5sKhi-HW=ut`G6KFW}$#9XLUNM%F zc-Z2+IK<8?Ka_3H`I_w+XO%fo*d~{mlqy;5H`funuzZu1-ky@(?meP>uMMWA*W zl~ct?qv+;N7;^kL$00P8{GQYYG}7NQZc|I*KBn$W1&RFB--`s+AF_;}4eS zAO1RUIi?+Q7N?i1w=F!Q^{k{;F~!*OP-#zljvAK9N9b%Y3@SFDM+ou8Fx&=4wWH&J zYG=G;N~N38RbT&Xk4M^Ufsvb9Tp9tpoPe>h5qH(9aXq#w@t|M>JG^Fxl?O%K%tp(| zt%G=GHHok6(Agt)j9Dp|Uev@^#-MV4hXt0j7?kKF<1ImFcCe}k-{toVh&Hy1Y;lZ{ zowyQgoh5BM0FYthQ`Qh`*)t!q>Y$#EixX$o+uT0q@;$GPjX1XD`1PF2y&?AfIpgDi zNvEuexL}8MQ2GZ~vuYoJM$%X*;&?$Zm0KDevx}UujBH8C4iljBD*3)M~?@6YA92TBC}zcA9rQ zh-X0UsA~J-4HVu%+EZ$mYeD)G+P`qTD6?c|lghgHrc?O`h>=8gv~|*-$yO2jG%0S- z!E1nsFtsvA0VoVhY1Da!28cyL;eRmGFvCqvEOFtevx8h1Xvt*uzzvV*GLewcG6B?{ zLM-8fOFwD-gssKVt!6P=csJlnu#Sn4KuGY`z>nUQ(c7(wvF)W0j=*w@oCg)LAKO55 zN6%m1OYLi)okAZRh7ne4eqSHlH^=%O9v!V>{IF+9*H_&l0wMgWbH^@z^?G!%g(x?1 zDd~M7pb8TimFcJIau{N#N$9g_!trU?wNM_85P1dJVVl=so@MqU*kdY<{U*vapBL1v zFR3p)c0kXrvL^!w2C!L#>e+^L?ZrP^xIW`*o~4PDK^&zd>U_Y9S^D9OQWN}((SITF zx;jO2C%JAE$iBCr*xx zr)M+)Pl6SD>8k;qlfZ=uhQd#w3{y)j`YU=u7`#lkqSRxo-u|(l+(TUP0-O#dKdq^& zBIJl{_U4Pzbl$-m@Ep|x7djY0=^?$&BHUh%c0LH8Be|Z?Q3ULYvJ6u{5Z6o+hQskn zIB4Q*XKJn3a%F}&oA^+zj9bYHo*=`eb77u4C+4|>DmiXukNwiys5Wt-1xUx?9>blqpJ`sn7Nzm zHtf9>h_Dol&4wq4qA9v~kdlXR-BVWZD}*K0(y9*|)TM?B-^9_616H758AIpFv~69? zgAVXgMUv)hYOO4>cP8^dk|xz+iFgX2r9O(|pGWE3_bBt33sh!BtF4IG+b*MM^N*h) zzv4!0TG(CdePfNsU@l-FtD?Ahu;hmbK86P9+WPt#CVdX9aRfo!qg1}hX#~SC#}`yI zqB5i0$t9sg+kasL%}+qSr_of(BX0U1(90Je;!D|SL@8e$#*YNlOk*mwrctT^#vm4l zjSeU?7S&Qj54@>FBRXjN;xpi|2wcXhHTLIpQbEAR-(r}AlBf?^0*y4kSx5^sEZ;_C zMT0XM%H6RpW;iH4p<^pBP72pgwh*)sKtW*f*&Hfj~R2 zb(d(`IFto#l@9puh&T;75EgN+v*61#>E_Lw`bMUv`B7((sS=AsK}zDg40&(fX$&GD zK84dYQFxos2TBDatf)^Of-}nUMWOP^!y7)67~5~Y97_+}fGrro3(TclWB2Mnd@iKi z1f-PT0&8t(JF2A-9 z$D?R#KBx1mZqTmp19q2k{M+G(nZqAtn1oC_Q%g%RnS-7Kjd$;!+YpQ2JPDpre1-G8 z+_+vBZ~*eB>_1!ax4d%xiHrWENJpf0u?J=npe7pqEu@2G;;PN!08FWhrZ;@$Z} zhIkQO;8#u8DW2&=yf<|VyAcuG*)V{!iMx02wcBsIbdN!e(00+-NM!U{Fv2!VD)S@2 zOSnk;96RtbFeL@(qi>?rn{5sbYujMn=82Of5Ae%yHrFu`aPRM5-JwpXT49e?qKHNV zyFcd*N7yirp-{8Uz$_Dn_ukR*aGz0Ss1wBYWMM*=I;VV+a_d;Ti`$Ue-Ig88Z)3$w z0dOrTRqKiI0tF5^v)lVl!FTXZV4NN=<7kHy!yHV7%xE}oZ7hL+z-eO8?VaG=lWaNoxRlCL zBD~-;wr&OEC66383wnkO=hTm(xRR}7or?i}0?OPZ@Z*{^-)lF%TX3%(dy5~h`g7GX zfZ|D3NkSXY0xy_5J13Ch-F#kY9Iki@%#g=tkc4P%pB6kAor`#Ef)-8Czv1)}15N^< z27dx}0at~Aa9XjmDj%y;0d!HD2u@*_rvpNvh>QxpMX&_ef6)C*!?8ErH{l2MLn)gX zE@*pRIJbB`2^4E|X)xRGcI6J0*-pd%mfV3}B!7a>y8I$oS7Y;x08NKeFpeV1Q4&BZ zYCStG@LAd$+hg9ls#dcVBe-pHug@ZNyvJ|`K(rA5SuK*YVQBgKMgK-$puYI@$n_+a zC%L&GcUb!a)b;?nVQHHakKW#Jtj@%~`31cZ1UHpV&(fw36|=;W6QWr%I3t-eKWp8l zOkb|y7skB;ztI6_SGe=W1dcT_FD7523nm0JG zh~A`r6nw%g4{UzzCv`(|840;FhOw9xJ`ckNusH`f%ayy?c2ewM@WzFApfM z61kCKt9>H< z4`3?XhTNje7C;pCaY>$yZgks)Dg>oC*<*mLqs8Jam&5Bj4Hc-+9 zf={JBzz*m8`4jUvNhnQG#*HZ1cCo$WPLSZB;=o3Ar6}LnPG(pYcqg%Xyc;)fjzmBb zJoC`Q1*kD1H@fRTSVPADV7<$6zpp5VGz#A+}$K1dHFSY*kxMqmt}D`VN1FjQe9Gf3UFt#Gvg z71-Ekcwhjj)=CmA6!3jx&eUXJhOGnMudjv=fo!*M=i|bdMpLm<*+gQxj8-(3!zQx0 zc#uTCGLL-yd$7h0!N0&*Lw3=gc+$Di>W-zR$W26hG+B24w^P`jD0o65k@ov%lPd8l z3nc@o3I_|9af$gp9k=jTVf|ly)ADP79`-cVQG5Fx9+&d^2KF=uAK^9;1kQk+++C-A zL&|5hi+%ndWmLv&iH%HqaWEEp$H45G&K=;c5R499l2$FEbZnDv1dWEPt-%id?>bP0 zv0Kq6F0cx(c|t$YLZ~pd$Nqcm+FyLvtuptw$~+bCIraScYQY26ovU$@8U@<#S-R2@ z3oPfE@C9BJw8^!uhCe^W+>)%=xLH$#n`j6u72i$87a~cIy9l`iT%U0h9^&et2lBE9 z@ReUzRyr@ZFI<}U1c9NOPc60+RP}wt6RjY~Zb_{f45J-v*5y7?6d|<-f8YUlq~L8x zkwvWsu6EfDeRU{{*Pw??-uC3Z2_=V8w31c2ImzSOb4+xqcNKxZ z=|A(`tej|z3Z=z%$3x*94Kaoz>?bn(=n?I?F4 z+Yq5$-AlQ7^Ge&etDX)pPFNV>KZC*f!=i8lO?0m>SV!w{W3}btT#pL2JQynT(kz=3 z<@ObvTVQD7v1^weYjd@DDzYmJa>sC;w*buB;{j0Gqobc529D5sP+yUuzz$B%JEC{g z{p94P>BBC8q|aMD?LU=RNQ4+I>Lns~6c@!2o4B*b-Nw)KHc$61Yb)r+s>2S*149X) z_LQ%O?1zV|X^nJs?~S;)llUQ7%rqj(%~5AwFe+MXY^+(7OFzCi-N9^62S+MGXZx(t zQ>@T--{BqmqceWClQ4MnUUe8`^-1tfDjY@Q+ggURdu|bD*QbfzD|W6}r?i&temtUe zNs#vlXal%s(dOofE1pC&ak6CVpBk&RD{gE zr{U2}Hv!p^QPI?cayLs`YhY^cqd~l=Drg7==SxTgw8p$(CMU<`U&0%62Ub=JUyRl+GhHD#8wUEp@=SQS`$? zLMFrbA^G!?L#-nr@dCQaPGMu?ZEs6+b4B0ocU5+1w3!gDI)ju!SP0(#oHq}Ciqw^t zot5w-cK1XfyIV4_Z<*@%(fr05=O1FbFujPoU79M_V7mX|c24V!8ThnGSy#|L`1bl7 zKQ1}AKsyV6EQN0~5D8V$zUYz(4Xy*Zi9uaY=3vAO36S}p zO=^$V5T0_o@8-1rgR6rut0T31W8eL{uAtOMxJl)=(po0o%gc*QUk%T`F(j!fZ9mY( zF=|UI>0NqCu)qoH?f^))MaC55+vnCllp%hc2`^IpAE;^Wu=0Lv&cts50aQMcu>YY6 z)wFpMv37;xGfpcSUA90jg3J(ia5{lt$>m{QRMb|XgbC(pZ^X&yK+#0l)PK&XD|V$_ z_)C;>Utp2IgoxPMojFtEi;CNYtou+fWApJscw+C&$>jsfGWF>O1qkrOd0_+GjR=(~ z$PN!Cws^B9wAf@wO*AB?Q<;G8)A9NB>k)X#q%+Zgnt&)XNZ`iEWK1~ksbdVf518BM z;F0xy@Ikw(LK(VEFMdufn^dW#hgNvtb_@Xo(#TU31Uz^VA?cuuxdQBh-W@xn4sY1B zX*={^d_7vFpq%9aFvmHy3Q}(isC#mBfAumS8vlMH6Pts{zGE53@1bdCl6QH z^fKye=FZ(nCr`_P0^~dvw&YYdL3j>x$7uxi!5W+F12Z4tx0A=20YtjDOGHNpi2629 zS$BYms?cZP$Y|Xn+YYj}mDA-U>aYi=Wz0(04>6Sk%$i^|+bRcFJQnkd7MB66Soug+ zqP+uSmAK-H8UC0$r_*@Kh*9EqP#{(kd2}(VuDUq3l9KkwIA9xY@ZLInpf^{=K{L4N zngP|tR9dtFp3iEdI9t{rAsK6MA)aSz#vd;L##*Gk_OGAcgS%5*;c@oTO1=37Kl`hhC$;K_${lqAP9$Tn0sWBLQM*CuVMnuOJ1Jz$b_ zZ{$&*hsy-lmX;lyk@DZtCZ-?nSV+hS^2f$(IuH`y`pt~t(_d@yGl9Uc1Vq}NDNmx? zx(U>cv8&(iz)9W!TqC5nh`OjK0{~U@GynTydz0$qU;q55Lqh&3v zW&z^lO*J?(dDf$E!H^$r$Jxxq*?dVXZrrf)f6k`syWuDQO_pBWUvA+!#~=AST&lrt z;7d_y2d1pQ2b^WbVkLI%r-!es=QA2tJ_PQ*y)v&;~^0zl4Wc8XR8G*|+D z3gAat@)2CL;-FPiWKY^5?JD)n%wVq`(_ zkqaib6o%Ye#sV}Zrb2^=)^{M$aaC;N0BdC&{l>&$sUYEIzktv zoACgjfh-ZyS9=$@pxX_Yh~YH-=O=}2&dx?;#g?}@Xz|r+Bb%2aACTIku&v9N=^*Nv zS-;XeQSm$gurgOcr-q4JmE&qj#EL=pRKu=|gXx?O7oJli*A5AMsftbSO-q&L-*bWP z9u63nQYIW?2K+tBSnORG$-?~{Xwa;y>%=M>gEA@Oxl_oXrye!bz~!p~e0r(eQe>&} z3s8j*bd|qtXvZ+iJBOcn1x`wzMV8KkMdO#V9@h#}Tv*n%`Wo-dUbobW{C$wZ50jN8 zy5&LH_S<{gl|6aK2t%@1yEe1F2Yb?ua7bqBwm!v$XrA}#NId=6fjYINOD0gdE2Zs% zMvy|!vm3xfAAx&`Vs{I_nj9|>uy~Iy33`wMWu9R66Nr~=tQYdd;Vr6!AtsEx1IXuu zSb;q80TW?4;veEZ4So9`Pk>gkT_dAz^gX5;UY`?lNgeUN!!?2k^fWw@G@$QR0I+xP zkyi(chJiM8Wt3O*0-sC%4`>z8(Gm_j?6n}NDn*YTIWn*6_m6pKD#rJ?;i=pvi(>Ns z1n3{U28n={dus-e%T6&5yEJ~I(03(gP;))aDxWGEKb}4_gjk?2Ze3s0<8~*ug`H4n zVC|HlsCRCb-)bfIKi z*_F2Q2%PGhMJ+=`f4)0^Vp9F)!CXY-t-`bZ?IOK5z&bHGdn;N*D<0|Sh9$T;8-2m5 zWQ;KI0fZe8ZD8YPtE$fbA)+-xm6z=%e2zRF9ol1%P(~gWC6RTJ3plM#0F`&udvPz` zisjk9h26~+^lf?Yy|wqeKZLW*1L9qawY9zxVkIG%3BdN2@jFpZH`9jE=+PhpyYg^KJ0v5Lo?g8~u9$zVF}>Fi!Jf z=Pplp6IMmFV4ROkJ-_qS3x8DGn4wHFBl&kq3~gfLwjhQfwd@imPF}$Nd0x^d25}xL zUoTPC0^xP0*+H>!Db@_m|6GfBi+Kr1CfxXUwsb$G{)IP9^?Qn~pc}qezEH&co z2s9(RGYk|dvLsogZ8>eC)B-lbK<6~VBd1d=8;yu`;qPT~R6t48qJVv*gFtBw5>Ajz z%M)MtUppec|4yW3)-41Y{U1+&&IeEfrojo9E!1HpPH+HQkMn&0x7>oB#vV7g^8DY( z1Xo}h4DdOr;jR0neJ~uufdHI01rVshim(%&)RNYIf`U4=X~R~Y2YjxMM9TIF#n70O z|7A^u89CK)FZ=r3J?0zn7HDlqeOoqlaK!;yzf4@a-vhQRpJ>VU_zWg|C=X|pq0vwr zo7}JCI?+G9)T0)qN8mQICJ8ufuk7lN&-j`OEDJcMb_gPBkBKi6sa}Hy-2O{@7e-An zBP?l=hpQ0j8Hc{BfgG|0b+()JIge1#y`8Q?oQH^N2d9H1LW67usNf=?f?F%)?&5M> zb4qIc#*J6vX#-6+9+vK6Mi?NH_8`+24ylATi(VrE<2%@pyPHUkMBZ;0wZaECVIUc-Y!~on^nrv z+n_;qy+FIn+VI-4k!Z-^w*(f=5hD08;};QoP0kTpg>IMcjR_*0s3WGB!*QaS76s=Q z!MS=pdrBG6d*#p%gxS>9+t}h(P-{=jQ$Hx30rCmct!{_C2n6`V!>B3xZ_#aKJ{|D5 z==6|j+*;pzm14!Fa1+UHj^KJwwSRJ_ti(0QpG){4nP6toqDAsjJV(djzkr6-$>{VM z?C=!fe1=b4Uk%b{0u&H#qI`k$#h{#x8T-KzSW_K3dwn&)qC}-(H|c++P)iiaybhFJ z!Tu7mo~Bi!5hp1_e4^}M$oVDS;t`A^kTV#OeVRIyr*!C;bd{Hxz0kFqubR04uL&%) zZ7G*V z(bXz`_@@5!6bbF&oKEtTHEUuQ0WWPlB|D#y)nd(>i+51QvINXlk3q1?$Rif|70 zGD%D&euRggGz0@RWk`P5JB0Ji{GaE@#?Jd71pOI1bG^r@$ZqATgChJ}BmKXLOq|I4 z<>8|;VJ2s7o31f);;XW<1)a-mbMQ%r)(k?^OTavRW}*#P*TJHpA2fW(!CdynhzaC*gWYJ@uXiC^kMU-raDNuwHnd9 zsXT2+`KMEI+Qlf7WUhvVP3ULr?u^vli#2b=!ANr6zMcHTsI*A}4_;RSEdXR9xJG@k zix$etfy==!g=}4~JF1jguTI zs7^zptX9Wy!{L95OBp_OGlG6IR0y*Mjiao@uKoikfDbSISe#oI&jytX7D3=p>jNJS zRx(dH>5I{H2^&6f{OihGtuG(}-jV<%cC$jV!hXO@@D;$Ue*MqzsYk6Q2;~ZqaQ=nK zlnG%~_xin%C8N_cGqNK=ZXmD_oMeih1?J{f`)sISE4|R*^aesZ^Mco5+fw=H)$7++ z>|l=8(1l@`Gzglv zP|rJ{+Fd4yZme}R3gYx8420-ah*+ed5xs9sV4*WtegojA!dOM@7QqsLZEMXlZ-Zk? za6EBQC~D^yYS+EgM1KwlO5s0%Z9E>BsaSEJOQaMJ?^6Dmi*|lR*pE>fO?03p@j(e*!r=GQIDb@Qa+4?HD#B# z0J-#ulf8LAQlwfs7z8 zq90^&aMm#Yz}mpLaZ|vM#7qdS%5ebSLf6Un(E!Grr!t3V9}N&#-X= z8u+93=?my$DTtF-87=unm9(RYuS%vkOr1eTg`QK`z3}S^#^K7t8az4hT7r(+X^k_6 zd~xN#qeI~Trh#NixWahxoF@>_dy#o_KTV+0EP~I-v!*9v$}}Kl2kHmII*d#aS4~^T zQGeR}fq7OGnC?x!$ML4yKkTg2T7_SRx-)wBA2>t%k4RGt?jMVpH1t1h-@K0lh0O{5 zxr4xmbz9`Aojs~etEr1~ZbKHr>I7IqM9k1OQPLQp1=7DUqLr#wkKx|rz%cMWa3!x; zXIkM|3a%Ne^XFe|{ZTg%%aSgp4iU_@4L)B@h>HPGC0zb@h^Z1_a)cJ?t53zuDE0-b zc~qjRoTkHJ;)<|Z_Sbf3DTRd0^zncmMnOIG4f0Fx0amB%@G17FeQ^)@;vUM|41$7Y zJD-L;6KDOnMBYcw3R?HrQ{jr)m)AGOu%wTGX<lrV?cxRO{ zwozw-o3&``QS<^<-!jNAoS}X%1J=cOJEFhryo=01Ey&T1CYT=52B*1a4zhOr^Nq17 zT$AlFssf)si@Oggl{uY?)7Ux6I5OXSko-PBXxU)b)!o!!Ogn?WPa9p&Lk^BXVF~Ig zWkGA<=^2XG*d_LH?V3m>;5$4^aA2ru^gm`17@xKut{o5mhRX))hUU&)NbR_QDwmg> znYWg17KUX*#g9J@9Y1q@99jujGh!F;FgO4e$VKZ$SSC9_J?{mA;`fX`4F#iyw9`9N zap9tbuEF}}r!QVc;rW@Wq_;G>BtC;ixPh!c-ihv7WaJ3oz2A;sjlmC=ht6l+zlu`- z$jzV(W3oFw>OGp-cn>tz?FbzBTTn_J+yzq$w5&Y?&Fe$H_%9K^Y_bb`344C7zlL%M zG_e-`<2e-#2O8mJuTZElCM%nzr$TVPLrnA>?lTSy`2(Ictq$n2_SA`pfs^T)0)dMytwY@-x?VUY zY0<-&oJezng2F+$(M~YJ`^ozah2Zqi(B{HpFq&bu>HT9mzo+g0gw+-m9I(BL zoG7tt4BF&kXQy%q3;q6Fp^L$Qq-Uq*lmmG5c{5WFXz?JXF-l?BQxaf>B1u;cq$iqW zW}(#WH=FQ}{g$-1>EgH_B~-f}#W~~mS4Jz<;y=>M59XTWO@%MnLd9OA(X5@=Gc8vx z_6@(`dVZY1`z1KIoUAKXZi+hzvD~gP4z0jt(chkY%auf6XQoKO&Y`La78Ni%uD8YE z{v9cg`4~#b$+~>Gu`cxN#N)q=Jk%zzx&iO4%Gv`VEI?krPoy*Gts4Uc65Y!HRB>?L zxv)4%PijB3rJr77p3#PYuU9})=&~a>yIrGg=Pjxh(N^%gOwpS$2~P+>x+B_{fu;p< zkjdAl9{(P|s7LmaP}&QY3m-drlS0UE22?o)k|NK5hAyvrGR$M{#W|P58DR=djvW4~T>eA%^ctEFE4pEr7ze>4YcLq{GpRYfy$Ho;>j!j{Gw{4**S0@0AN!?d26vFYn8XR_eO}z8gx*-FlIVJ z9f1ZU4+5td{>)5(WlwWf$TG|}x}!gBFe?8NnuN>a>}Cg&tqsFWW8u)&3JDem+gY>U z*Xh-Xjp};|3|oc4c*(!qc(ZPY%^lGZ>U zqg{L&ue&tVo@MqFsw2kmp2qHa$q`YI{$)zge)r-Cs&<7u?V>x)D;>%D2mjL0Pm z1>Rr9I=FQo`270=N>*4*=`1$Wb*dl)ySPZGtCF4?J1a|NrKcC`HQq!`#dM9&Niui! zd$+`KNCrtBhdI9cZg6L?dj+g1c>lxWpfhMzFYy4dB0xzP1!o!45}$RnQ3NIcGzEv9 z4`q5GjPo?+KlVLa{}Lu+4{UM$oeP4Oml=E=@na!vEQOL9h{VN4(rzJa!mH#~-zcU| z#+~#cnto@iGu<$L@$ksrxZEimYM+C?obcAJn51S5?1t1PNeriUc~x;2ps;XFKg@bF zKIb4y%klzR0$8K=(#?F#J3r&t9y@F4xjv7 zS~GC}>d-m*6s>`D8{DhX;MuC}J&efR?q$QVrKW;?-x5X_oCxk?qd#<4|aE(18zE3cGG(%Y;B0uv} z73YzkAE3-5PvJd^3QPlu2sGDdTUu;M5gUpDP~Fncf%Y?wK_rfcY9Q%fg2Ze$kp(mr z@ding)?yGrBIHWlJbA{oyhX8vF?uy*nD^J?RCaZ@9wF)A8sIP$p$L-PJwSSCmy`zW zwWgVGAUD_t(pXcl0-O&AmNvCY-P2A&-%AZ7%*IT6fQ+#GY-zIztTe8D(`dkR30Z+2 z2*)MQ!tYXS?I-aM&QX4tjUM#-a%-jd2Mx4E3LJ;6rv!>veschC+|F0`XNK%sFkhXJ zK}7V=fw1!R5ao%>ckn+mKIR;nI$^Hn|Ay=v9x=wq$65~YBuMgv&Q}A9I6ekDN9-}X zpK~IiS3!`q^Dx+MR0%fgYkMYUm(WNcBnaj32nm0Zzb^zz@1x3B2~rha8&8m^X5-W0D04QWiCJ zBWxGXp%f!yxCU$tFAu6AtDd2AX~+b4LFFce-gc7CO}(1`;CiK*9QK95b1MlZ+sjGV zM$^?1SHa6M6)4fj1jM7UCmq7FnZ^`f%bB} z0y=Z`KWiYrIfmzPHPMogAXimkTA8);Q}<8u?cCPezE0y1zJtIf;DJ{!j`Q0>m-60l zu8bPWmG$nUETSE6_!Y^s25U~pUma=j{-ZGZh5yfBy$og)oo4U2digS$h)cITJr%h# zAb@O(|D2sLSx2X$-Jt~DARcexl~^7LG&I(~nR}V{pFVQh_%K%P9LxrCFPA8=a|mJ{Fij2LKu3N;Doj}ZYV%obzq3`koup{dy5E99o~-;sJMvq5~g`K`r<)Or0sEEuZpOk|e=UO~N!^G4(fWw{UjmO*qQ}3V+YCi=;S0%0& z%m~Ev)XdqW#K)`93u6rRXV%ve*m`+5$htAZx~xE_sC|BCvk8ftkXz72L z4mr%kqUdgn zJpNmad{RJR^so&EGgdzXjMnO8EEXjngYtQs&{okJwK5)C4J=5+TZn-$X*%VNtgUg; zKp3S=+HpPvyn1_=U$2-~-XU-jozVZkaV~jDk|i=c+-h2mkeMs65ytZ#|3ZBzya&4} z$hr|>i8wAeA|C|pLdpE^r5)@Uj6&F$F9&Q>=vdY7{*Eoihr_%6?RVPFNxFrP#} zBs|M$ZE3kK?rpgVOHQ5`0yARHlewXw9&w(Y%3O@p7|@<3@(a^_TKo*=CDSVS0<5%^ zNLU&(&v>T&qWb{RRT8Z>1T1<1vGA-Z<35DI3#C1(A4QVx0Li&TwnDT`Eo+tFeI>AI zcc8yWL=QFN8vFhhH1Pl)N^v`qhF1kVK|d(E914)y3YWa?THj59u>iy%t3NVkQ;0Ua ztcQqB-fuJlyi8K+8{AZw4I<+)dhj&GWDR9kR_8HP`A+}XE?v6xD5{7>0P_A^k^Kw+ z32f)L%nj?${d^tW6{A(RHEtVO16Y^!;51BmsprH`FwSMPX&lm#YE(d`zB$0jvgpRH$=L)hj~L5 zCeP#Uk|KlbqX=mN6R)!PxUd60Uzk*z5Jwq+s}N|Mzjy@Z9oo=VLj0rhLSyn|q6mRN z^r-FtVMlnWUdJpwR``)112bgIb@8lL+y66Bw5JlCK4bWWRol=rwKTJ&rxMy10P|5i zCoNjCf=F=8X1J7)c59(!EVIa&b?=@sLfJ}9G@wvftP5J|4J!$v%&Eb=jRq4C=)Eg2 z%Oj&eh_>F-iXU_~nO=aG$*MZcQ@~ymUJzs^wQl|TCkO#FTjb_PonuhQZ#@+r|N%|tpp%C`>@9-$*6?}Y!-cO|ZcH;2qDj5~xVNgj+ zE_}vXj1IFud1UymW#n;TqI?K(X(NUQyp*S4`S5vo#H+kI5u*d6%_U1XEB`w=AK}7M zIr=~SK<9CwnR@lp@G(M;0Ai7IxVOiBULi0xU8%ojO0`*h^Pv$=ih?Ij5C*`N?3k3k z$1J?1Ug{#U<~V0fkIuBK5Kz{Nk%g@%Z41G$vBMQVWvQ!n+Y<$ ze0|rKU1#nW_mHO>3`rRGat50NemBVAcQBwQaTQ$W-M@mXq#LIJE~Fi8-xILV+qF}bv*3{Om5a;2kwHCKw}M)ohE}qdN^QubnHe#NftE<%IQ*{W z-Mvdfpqy`Clga9O&L20v{gjBD`Q?bf{WG7wtK;F|jN38?|55Si-K%;c-;Tp7SJAM)5yz|w+{MbazB34o-+kIqH@;3izsXj zs$Ia#u0#eGfD*xN4GLGz*3i-_)3fVeI)2)S9{(?&VtRW z@`k5PdE&3nI=d5K9*Bh>ZfBz~-3P`P0(U0X!sm7<)(S)YfH;=@tu7n;TG$by)1qYyo^0{*yEWOv{PyOU&*dB89Ue;I3=G>Zc*~ozysomkl*! zYViOIDrjTJVMw5`Q-e3tw@MSc-B!S(Uw0`nPDp1?sJd+(?m4}Ndrski)MppqNWqO1yV75PFq`n7Y5CeVOR4)$ z|F~U_&M)cmkQpPK3drDCZq^o|$X(YFh}L-~1Ezz4D7n^lLdbsf>DNzrY98ef7NX#{ zEJJWZv%yKKX<3db0>s+_8*A(AP4C^5zr?)e|A(8`EYqwZK|qmj1Ijp22Qy#` z;uKOTbz<{VBV)x1;sRYCj7!XNF=Ry3DEr+rEP^qvNegx{-lyP$ph%hmFTxdujg%Ff zXax}lhs-F(8e`=d3b)c3x(^1hQH}i1y-?OfGE*h!J;Mi?%9ebau_&PwmF=Z($71$C zSgihQu=^fqg^pH0Q^llU(3F^;WE^o>-Ip*POMMaJxy3`glfOtVF~tiO~CqiPc^PghhH|70q7*C zhnFq~c$0K9$ZBTmGnD-2vE!l)eEEhqk-{vGJum;d;e|5SLe^{|NNPxG?3g8%nh(2n z*1PnEG?%YrjumONWu{bzMW>ih1X+1tI05(5biFdCqU*cx%KtMj#Qy~*%V*qz8^bK7 z;fV?*#-?wd2d3C_`hVqz^llqE78c)?a1?|D+7K&Z%!5pn+zoQ@ER+v9V*=82b=@*< zif7oN(PrQ71hgC_vv6oWm>nr3$+|cuyvUE4n16$AOvng!DK9|7euf8qcK-J{Si?z< zPU{FdJeg`r03kkri_Ch1VMikteNu^2?4ekJ(qrF+cr;DN z5k8PkShTtDZkQCp@QD)}#_5N`#|{lL5zw?;^U34yLlN&W9(kuDLtYkGla;V#cv60i z3BpJbGbzt9feSo1CoLM5uM*gZahAK-pzhNltlvx+So*g&5~4GHUVqYuBgV8 zC92xc>F}7!s>3&a#-5aJKJ2J+>iUitS}r-#R;%q(us_t z>L>t4^n$PVD3GZbxkUAZG%K4RQ&36{a}}_^jQl}6dAuNNSRbzojLFN@7apKE@2z!$K3plMS48D`t7(dXC=N)x7*)tag!3fmhH%X~A?%*3J9X?@#{# zqT=&RaCG3J6OXwBTcZM>H48WU{(vtwQos{oRqYYQc_F#9IhL877gUyn^trxvhy_dd zJzd@K)qJXjjzE2wv(M)R^3dWyQ?mcV;Lr0f{|%!F4}Py%GL95S1MjB%U`UAB`P~>00|sPpM%L<-Yu zJ!OTyDSCaAt!e*{2yzd6gOR4`G@ehbBn(||+#n?Vk#1F6J1tpb=4ZyN!SoUbmT*}R zH?5gRqfNhl-urbJD%fo%KonK>LOD#-I5RAdc7i!hOqtITR92?r92^qlC6o>e@)|zh znxpcE?GJKA?fn4U)gyBKY=OBYb>XfJiL+Q{m!FU#&&0&! z=vZaH?+8)ak};od2S6-{(LL$4h%0&p2tPmp$F9XnS6J3icx%(r4G zLyDTYF5*37hHy4&s3!72%*C1=3=f^E{8S5CaSC}BGt)pQEIRKYg=vz{?Fr+c9Gb*U zep>O8%9!rXbb`#D&-eS+#GYiR`1yGI*gf_kLcyq}U3O~l4=8DHcG~9${U{-Bg zN$3Qu8`6PXcXmY(*fs8PSk0A@usTR+CmzuM4OQPSrmpk7>Co(uOxvHBgJf}@KUyK= zGU9$j?)^GjR<1X+Z%1=;bL(C~<4IvAo6UKIqKL7rB_agt?hXJT&DK;?E6(Y)W$a24 zwm`~Gc6iK$Cv7f4vCc0xc6^$@b(Hi}l-_?j&mlKxOoGGnw$y#z4w|@F!s5zy@aJ{d znR4g+$Ec(f2Izk>7dS-O3{D3*Df$~^ipxOXr=vgstZ`%$>e~q|ER_W(g!P%gD-(r^ z`de+mnliYNlKvaCIje6iJ|&DTPd(A=GTG0R00%5uqzk`}m}eSBg!iM}E2OStz-REh znN7t;by_(0d2JE73m`bElY)LGD}u@vl%eTffX^s8@PDC4TnDoy1VGQ;O~5HAKCio{ z&7FkbK`MtF9BUBvI`n?U@*?R}yg-@-!pve)2;3c=?#&7S^{j*|Zc zq9STUdGP?G(Fm;|e~}HFH)EYi0*9uD=3fg^7zY{hvws(#&H6N`z50d~P>|hg#zP9< z{k;ZduFx3*aK$ezxQuNdN$|gdutkTSr@)O*b4;jj0haox%o zqtaSssf{GnqIS0&I`@11pLLAK>xKhd1O`}oZ?H?JeAhS5Jr~mu(g(rVct{`QUR6^= z(}j<|F8f-GiTX0K|1vV>1rpLHOg$*>t>Q3bJK(o#?~4X0rn0?QJTY<7P{@T5HJH*z z4SxMJOG!04uBIi4pHZSD+VMIRfX3@dc{0u<+!#Dhv|AxD*>rcxFIRzwavfs+j5f&Z z9uyJlLLsq%w(_c2eI^-#A%_c{b%CYEW7p<4g;?zU|i z^0gLWNi?&d!l&CUh zfM^OsvrL}65j2Y~#AevaYoCL9mvn+nuE@^qMZy!ES~~_!|7o46n>lld>_2O*mp>V| zPCd;lkt?@K{awA;jS>U?<80s;Z+drNQKt8fTX%Qv*!kd}=ij#9Fxg?$5GfEb!|ky8 zHlYL!Gb?iejU})!zB>@zQ9oOA!9<0$JsOJ;BaG_D z@6^%@#c%*?!x-DetTJ`_!HSj6&T%LN?)%;p9YAQRzw#3mmOQI=0$RJDdbH{iTCCL8rY19> zrRVf9xp&n$W(K6!);1rFh<40#dl;ZA&Q8$r50$v#q@9>3T{Z`hN41A)RIp0S=QHIT~ zbkm$RZA>$na$1+3f^q}O=y5nd^WEqIu;}Y((tcj)UCsS_G3+H)0444Zg^3Ag?Bzxw zATPvK;Y*2%K7c$Sw=TBS7KKzjK5}y%H8nxRRr>vGiPL~d8coaQ7AL_kg4xYp<1*9ZQrg}a8y*x9Q4xmJi2p`%aw}u zSE65Cddg>=unHZ2b8O3!2|KL0ohpImEVg|e?R^UHa0AUUTJZi$7K%V??tI77x{b5& z&Ey=_={#0bjl)sgl=a;bcY(adU-U%Zzi&pAaT7qJv(WN6;DJnCr*|UqZGgQYXLnk) zFPW0jTdfWg6dF~a#v)dfm#gu3eNO`H9ctV= zs%GNDL0m~iJvLoS0w;AJQv<^)M$Wa&s3IHBRaR8#q3s7%HaWV3@;%lS^wxZ$LeM~-M{h_TC3HAQJe z=IvxzY3cB>E`b6wX3W^Omt7fe6x^w)Ex&b1G06u-=s7c!vn;}Mmkl?I>aHxPcj#cT zxO$qFl0-C0sf>RDuohgw-rj!O2s{VuNWi4?7DYH{;H=@dzN1F)?B{TdoB)zd`PdiptPe zlt9@!bq4-s<$ZEyZ}u06NtR{z&bjb%AJ!YCe(g(6>|01vfM_IKf<3U-7^>M=rN^G63uhPfyPTYzCMt&bHo)g*y{W z!I$aWZJN%bi49;u-5)p`+F14mbLqkn5@N|n6x3k`0`1tp& z*nve!_}clwC+EpQGj5~o{*%4IHr%SQ$ccN)ykvr5a+LTY{x!p4ByE&qYkyU#?!fm%Z;>yua!+%!vPe* zQHHZ-joGtjPvQ$)ObP9pqSLq)R@fAt6wTuq&tJ4i411N2h$G$f|(*Ew?-~wfuVk(rRPh6+b&)^4$lQdNLO{SfE3#VMa)N_u7i|pkv zQ6o>y5$(Cp3kB*G*zs)63&~|Jt6I$d-_N9-z)q zRG>q~SFQO}BNjbN33mfkVtGDp&KOo@+n+hUcoGv-;fy7LRzFcDVGi%mp@ozU{gsv? z|LVVn_zyAGXJ%Ig#@3}LOwjHGc%*S&m7{1l>9BPMmi(Mk=ca(aSw#1|vbJ{mWbuR; z|Ij*J#RJYwi8xm;FLiSZizpWe9Zu`&tAE>Jgz70WrKM1p!`XJIW$WLFtJc=}81r&W z7K9nma12JXWP$F2?{5;5$m8`NYU#Eta5;4Ltcvx9{4;MhJ;pRsl*{79;t-Zofs$4P zd;KX@h!2U0@pBNsU*`wS1f&%?2sIg#1;?-xcIIvSEV^m-8t*HWjH$%FeLNpc2oObR z;<^i(+_Fyt+^|2-pC5(kKl$Lpla`n-UL1$xM`HW)$`uLh)pWnTag9m;k76h6mA|q2 zVA_vafLJWiO7hp_c!{K?r8SPELrn`?CN~O#HgnYWd}xtMKndNMUczc=YPt`QZ6Ahu z#Y!Xv%r3laz*@c{jH(@?r&9!vC$DKf3SVU~MSIdT%}u9H6{DHadH(Izbej6Qzpw15 zZa?NoS6(xeOS<(oxGtW@3ts-#BceLBvrv`I4tN6VuxV#gupLKBB1rJSg7npKm+x9m zz+VtL++z0yWQetO$RaZ{7ItGz?F5a!eC$rdNX5ft>c7MA46H6|+Mf6P?O)qJqJqcs z9W=eBTcW0W3{paBrH`XUzjI7~!WQ_YM!oz?E;#Gw1LS@aye{((kl?R&I$VXU$cF~E z6SLFmcx*U|PQYfzHvjnHXN+f^Zg9c{Kc!9bY<5xVSSb}99Vse`$maJ!n>UBkE`iJi zm6=R1xS(wh;#V35{!qb$OJptBR$8GvcMsx&6Y{Yg7MCi5^FuDFjL+NpNr<$z%Xi}! z9fwg*B$uoH5^fr9x8E6Uys{z|OdC!gRls+|n?Ps-?XWqxGr&*@62k~U&K}$!Mi7OrksYk6sv3yE%$Al`(K@n)zJ1M>f+(yg$^0XB-@YPr zO8dbzw(XLmt2Y)FQrcLVAZp3}I%7$);TVK~m=)!f&l-2pkq#BZ_{Jcy*3a|^ajx3E z_ZZcEM}{pS9FD^d;plc(AdTj0SFNI)X1xKQ{KQ8kC1L>p z9-2ppG@rpTqKopX&!0W}TycySacO};$53w^TQ}}NMlePzEG9lI$6AWfh@Z~4O&8&HZg~lM zX!WLDFeY%`j%osIk!BSV=C-{(;Q+l(8wWWwL1^SgTfdk>I4|_)Y`~$}TWUN{aZM=e z?GY4?ba8T0AnQeUCo}+6eE!+INyZ2(OBFbZ4tM|_AW;fnj;<3UR#b0mjbWF)z#mrp z#~9P%Otp@eShX7@EgdK6+&Y&`k_EOHgJRmsWhLDmj z@2AK25bgWZuFRYIn9BPNsF(27F7M1fNY6|DgJSw455L~u~VPm#XM95Ru+1-eK3mg1EH?=gq+P#iEieR>k}YQE2rkBzF^ zw~qynEzvGS5_@plS`rCgbDXrz|L!X^N9Kp`RAe65jGLu;y$3NXp1R|M6ykZS*yzz_ zZf<80-VJRzmiCS%gK$C_zLzi?kUscpB&s1e@lh&ExZvaKD??Wm@%dDm=4$}#Kf*4d zS146GWVP6m%Z-3KjZ1|PTTV`WvUt!DjFAzi)3iZfkyz3SRYbzlii)w#sNYGyN2jTW zN}&hhKKI-2q6U#w;twBIqto0D2MvSd=g(JPc-L6r9|I>KjFck0>%%gxy;oS12S!g z3_^OYHQ&N(A+NZ=kNYOO=_;t7L$|?6+p`T24%LIzXH0g@#ztV#0}Iqt$5is``)A-^ zAJr^pPE~028v*7Iz2ADNdXmuyc7km~%M|oA=&Y(A=mBEWgt9h?XPk^q7$GXs1-%>7 zq8`G}yXh-NxuC&B`AeH~@r$IXuF8xp*vAWrLY3ufY43KkJu)Z_OJ6wp`i@O^Bxk5_ z?r)ln`u3+NyE0<`!GjC@{I1~u|5;dA954~h^=V{aZedYCMt%!FNCsr(SqLARfHDoW zsean>Vi@9`+BwNKu>s6X&_F+m!6Qj=LTjw8MaZwQ6ZZ25A^94L zvl;i`4DkbFsTcoouQ+2mk+jiCPi~__2FOhI62 zXT!y8`EEQ#PRPm?Y}GR*y`EgIVC>aPxbxgbi5oeBC>>Gi{rRTZEaH)tkq{1Y!!?6S zkATvH^SO-ySii9M+BMzO+nBb{+|SWs2N~AH9zf!8EIy79AQZbG-Cn|*;H#v^-A&sK zpPn&wZGs{%jV>~kBh?>%Sl3ui=k+J7xMT_5pMN*7jWa>AHDYq!v6gw%>u8Cvd#4~R zD}MQE`0Y?gM4Dy>YTjS~QCd(jxFnx*e9wnfIt>B`0m(%a*R=z!dpV*58M25hbH7Awt8& zO?$67muoQv#vmDMW9Gb!C_`)S=$OPWfA(yQyp}qreGHDfdgiHTGKtbbDH!TZg=<4G zo@Rv3_QmqCmKjSX9mX|mEO(CSJ#zHu6A|->Oq7vOd6-uOOjN*wwggqR-CfDAYNT4V z1N86jJ_YeU7zadwHyBJ5FAlbX_?(RQrDM3=H??R#1vEzNdjB0-1dVSzj||JWo00Y9jJI2yi6%H|1^H$ z-fmL@O7b?N{mxv);TG;E^hikft04he2^U^Ikp#GctUkWg2MKgSNGKBMnah{8tN90@ zp&h!Hdg+x=DlZsqQscf3-y2aDVBk+7R=v;~Egh6M=2!pxQwYU<0y2b;G%}OIfd#4F zvrnOmY1anq0`N^fIICEk*Pq<7E4pPD!mOE}(w7?SD^b)lue}NuJp;J_pA0Ssr=6X3 zH9f2N#l=Pb{xi{}M7dzII0bmF;04HyyT-Wz*BB~mAqI)~@WZO{-rD2bvv@i62{?P7uJ0wepeSjzO}+5X7J#4{QnpmxO;*ZwCJOIoqv$>S?0|Tz!L{KExI-H|3!B0p>JIrfzOH zxB`*!zThe@cOd=cUzGSep-#^$2tPBLg$5zA52DLV%RUOQiy*YJtGnYbyyDVw2gN#l zbRrKQKNc;(x#LCAO%OUn33Irjh4Ndb8H{8qcs4xpQjL8n4Sgx*2pk?m)k&vw%wR(I zB|;UT`IkJv;RKr@sG5I3wK|zMRuky1@8PmY!e=+&`4oB%8APO^mX-)6Xoik+t5Vh3 zYG1aD3`gt0&bL^7?l+jbAe|Oo+}U!NS6YI5FFLDRsUK_Kv$st|4uspqhvb+rC|&23b<>E^aKr$KeN}D0L24>FT*9^Z{n1 zrV7vwSo;+#CUxFiJ#EF6)lqQDYdfBP==$m_uWxNW#Yc2=0?>z}RY0xIX`h5@u?M4W zktr@$OE5#hmY1J|4#mgE=RSNK4#DWg++MC>Hr_8J9ExZ}7Mhub4@X)%94VG0`t|p0 zt=^0*U_c%AIZk5C|NCx!dJ{p%+gN$>ilqMPxuwT=KDzo&r+sc2dX=bf+HDZkxM-gF z<>loOFUu;j&dmmHk$P+Ol?gmN7Z&Rgk1|I^L2svizHJHM6Kt813&t9TBKSyBL$7Fz zkoy~7@P-e*PrinrKn05s-H%XDa(w%r^ zqXU2gO^{4k`6?M=uvF!l>&3SlCJ)`z^!)q@xGv^fa91X{;G2^vG#YAa$3;Bvs|U)K z3($~%YF?;e_huP(uK|i7Fw-IhVC3hzmrjLyg91Kb^DW@0kI&hXat^e2?(Xh^;MWO2(Hh4N+KZ<+b@qwH$7gLk%Dkdh%1f|#ivBx{1hLWZsrih{AAe) zXW`VCfvDX$vdtVALS5L;D||6gQHQm$$SH*vkQ0D4H{DS#SMRHZnlK(X2}QkYSiRz- zuV{S{ISNC)CLmWeeiugb(=|m4d%uh);aQWgkkCikz*VTtFN=k+1th9y<{^6w}O)0M|vj*w}~;!&k@#gRp2W!F9}B?XNa@ zvhYy3`x9@H9wOJ!t}&NaT=F}$;?g5z-fUR!L`Fqf+c5B+Ak0gsvo*k$8h(wBzSB+L zF(~%&q$Lrs-TBa?4hBNoXpp5V{Oa=%z8S5CvHiIyy z1d9^0_Y`p0)E!D67fUTouXIE8WquVg37jht4iYjok5;Y~&XDfCON9|i;*Z`sX{K94 zE9&>wa`l~@WBO+4>t{rNP3+Ejvx!PSz(j>SUdHzkf`SB{boLBh+yRG3tAJ92u#@&4aMp{yM+rD zY^!GnTq-C~f1d6Qx= zQjCp_B{ziw(LzSV-n|UcfR7+y7vaBzrDNk{Q^})84M$eG*k1`+P*}G9y8CZb0T;^n zb}Syn(pdOPQ>BZDD@~QI1w=~c9BZkOCqS3hQ{_#YGDQ?+mLA}@@`~qp%iMX8l+^<@ zdVtxJs6reyrdXLRD`mXKk3_>1F(ezJZdsr@Rf@QwfA84!eLRx=y|#d(yo_In|x}#ZEr2)AD38oHmmR%|C`icbnx{k#j*S_~qgG|wqkhlT&t7^AG6q|Hv zjpOn69~2iyrL_DA8bL@s>CF2708ilZ9XoR$bZbWxg=4hXijFu1Nx`EWj zA3HX=6DHZZfO81c^Rc?4oW2+WxTHwq69`**0aNieBf~(u1Knth#Cu`yPs_;aPBGZZ zp~#LdK>>^5;vg;*HNlfaU=AG49HwyE;>Le>jhIn|10L z81bM4eyV9t|)K^0A{Zb|f1N_1~95BMv5{2M#JID^H)yJBFL4Ll6Ow zZ9wPKJIwJ{(A8nzCRN3UWuuMXg63TQGi70%!XB~$rry*DEd|<9?SDh?R!lFGAcG(# zh&WwRboMouw%!#8U zI_LbOs3qaJ73BhPax+JTV`g%i=97>pacKbxm~Xzhdal#<_&B`2!5dxr^r>7k6dBdh<2buLXP;VbA~|Biczum28NQgLLd^yvq`o{;X1N6GN% z)Rsc~u46i`&l{F;N3sBxtw=UC#)bvSut^-wA#_7~pcSb{QJwh?a`$V{mjhi9DYFtn zQ_4sf{=z5hl)d-oL9_t8{8BXG4D3PhEm5GTv=SYHu96nw%I#Wn2t=B10TB9x(U6Dm z7S&K^(7qy!ED_!P3CC>idbQ>gL@5N!7G; z1@V%(5>yfZj}pF%NJ)+Lt!d~O%)*0`arp3Ira2)ALxlgZgrU5&)Xd8ZC@t>N#i#3M z(gj3@ikE%72yF_8Kaqm)aF+I}94=lDN6{DhF>@tl<>%eDGVs>%gh~+oK+{#1f|EZ} z(3*#Xc|$iD<+5Uh{4g^b4*>nzx^nsDzW~!(u2}JkFN}jkr39UqPUnLM4|IVbFg0s1 z9D*jt zKH7^~%>!m2nf|Npxqkl+!JOmT@zxYRYoCE%g-3v;y?p|Ij)Hg~76r|~Fq?H5&zOM} zBqWFx#h2~*BTCrjjv8e0`)nJozJB0G7Cy5|R0^0(;bZ=;2vdIu{I=Ls69; z-GC3;=caM!5uT7On%m~+lQ}1w{pU5Xc28BZO>_nQu_QKxR6fZ zLYOI5@ldf_njX+SpsTIDsVO2b0p)XgTAF#&GoYk+<`|O2&fjDn!|6 zebOWjrmmPU;RR^OTuD(;GLAq=lT+0#;)Eu}RAI^3PPoYu3D-bM3aXZVk zhL=YyDx{yO$FT0EsI~F7a~%a+f^TiXX5Wz%Q2$QjqE%3sjFyl%!W=3iF?#tIchREQ zbNVUgaTznx)6>}$tC)C8qA=td{;9eC!;%pc>Ck#c8ge-1OQm(T!W*XgP6N_X)Y1!w zsdkTkn{2ex@%yMDUb{p~N2jDP^k*cz-K1&Jy#V@c!A6kNnFD7a^rv1W9P?E7VVdCh zRoxcs3yFOWZI!maK!kJFndfC?4f9sxE@*mJq&&iXL&luc32v^f`O?+QS3;`4u7S&d znpdm`5sUP7=eiI3BNSw z>ydcfrNRU&(!X0;M)9Sf9ffxe4+F>)XQk+3N+|t~vyU#lpe$XgR=xdoLz2>7Ua76Y zn6tNvrxV{&L)arxWVIl>3@#^+R9g@iUgXwvd5mQdYIk+i1DxG#s0;*W5z<-x*xzXyAFNS#W2R7@_|C zW8W%3{0GI_5$n$4S+7=6u2JnAB&+)jOtoofFcmB#PXLF)s9{mBwQpxGC{x3Zg|qx4 zJbV2xUbs`gN=}uwnA`+Cg*Z}e!)=MiB50%Icn#3rH42P9ODK@`bl%2ShZ`f!# zL|u2HFnEkS9tlguOVj%&{aTM%wAiP2C1+Bs70=Gj=KDS>E=K7>M+hvTTKYDEYdXv{ zs0b3qeT;hh$L->V`a;XJR;q7rDx7QK3vzb5&ps9P)=SUY#+*AyV|+K=*6b8ZOz8Ie za16U1nIdS*3m^T=@7Y-vO#}9TYw%8@T+s6}sG3w37F7cIX9r^*GJN>n8RIEAY=@v! zD)O-bT_Ef?#4cRl7_<^CBP{^-0477a(eQGU=5C|R5Tp|4(0Th4;8h4#u zIvHS_N+;51>A)GT%jNWR!SS;*o|qFq;-L78a6$KsaJ(wqX1cqCY2Y4S(5Q0z%P4=BxVkq+M7XsEjbNSret z88QFpBab3TucB%(eaco)(4=S<6hX3U5c~1640=hBRL2(1E9CPPBEML3!z|OkRu_$xQ>)dqRum+rHkaJ#*BAaH9M@*g6x!q7{ z#bqafD@W5U5F{byDQ>urX>f*x!ETyxJSdB7J}aF(b*incWTIzXVQ6ND77!u2!I#l1 zw$W^UYyAapHkAQR9|k|5>wVXJ*M-VTg!u7s^T}sF8todwFhQTz_nuW)D`~EdTlT>(dhv zK>o@_kX4IOrikDDP`hg5&DD#C$kJvhG_bH%g-!D+VqHA2wQho$CHI3N+**8h{{zsh zH~BP=caBC^275|OnSM)HSlE-Y9_+4gvRY6rZ~gjQDfKq4Bm?r(72>w`g}L5p38K|i zq;ct9^r3|GW-+Gy0sulX-(hHs$|RMUGY^|uP#m4OV&%#hvXcmin=@$=sa?JXEaDc_ z6F8sxTZOXkx4P(#?=3?VHIAG#)z5aVg@ZDX2l4>fk{k%b;re)fNQiPcAwjhY0aSD9 z)WZylYe^_B9AZ(dFS7aM3LNb0%fAdaqHMq_V#3+0cK=U&UF#0|Mpr>R8O+#lr)p8r zRq((^A3YFvaKVL_s%SNv0&csF3o0Cv0#+ad3l;}t|B2YD8J2HgPLdSJXIJ+e1Pjo( z1jxg3!AeM6wMa)0BvJDy3|xpb$y^V3zrF4)^=-#4(okR9^hQsz1TdxrfcS%!_;pW2 zER-7{d}sJms_)6m6Gz{^8Elcc2`J!^qehh>3J;KtEp3dTktE|^{T+-JY(LK!P4=?2 zmrf-n?qv!s9Mbk~qh=&k2BbvK)rPI@DunCnwaWo>G=NDNA8scPvCois$e>BW|IdJi zvpOL>&xWg-%dS^~81jKs(i4IIL)P+H034~)CB@BQ|g z4^z3vmF{BLQ9enH@YT-!Edg|{UNa#_4N@%)I$DX#hzb^${(d`Zmh2inbEZ}Ulh%rXugr!3 zEUw%O)(HjbWSo_9s&r`O`t;sA1S)V;t1jBnTcT!6(CY zsfRY?HFg}5*nv@9D=JRWI^um7V&dAMuS)bR1+0OFm`WwfSo!%EURn=94;EmMMIqf+ zrE!3;kPu^+kT%Qe;5XbVljmd9IEfKg@6jJ<+z@yRvUxo3r-NtiJSW30V}vS#Q}y&@ zKnutRqpe;(t!n9Dc|aict*FKvmStC-#k!QS&A3(8*8LSQkUv|~Vzd!z0npt%Mvsu(J3vvAo)y$UZFpsg?|sfBrLj14{Q=8CaA{7qD_rMBUx`vgN4BmJ6<( zr6@t#cvSZcL;4(GD|GSZTR>`O`p2M*h~A4V34tEES7`L0*l@c2hY~p6n0X38r}OiT zhpJzvYpj2(--Nr+e=r@)$!s`wfPLS4&M{??z1#$oU^%jXXr8I-*M0W|HfrrbBR@R# zqKOBR6iHmnPxEbUY#z|;D#D7j&4R{ILhY@X=@(p8+w@_aw{#41phdDGkO|ZRWlW`b zigSVz#%3`$MV;o<#w#k?Vf!8f^D^ac7WRK7J^U3J9qj=(Ec5S3G-N%5XEy1B_t#$+F{uf41_`H& z+6llo{Y3C?dvqZ*w=zK-1*c03awVaG`?1{6n&zu&&k;$Awnjsv%@|`TE0VhlU>_E? z6RJc{s+N*Zo+PQVmP8ql!dqMF4q`Yf)fAHv1bBk{bUtPMrfdp&ClZ#yd0^ z2BVb))w|ovCbvO@VGSWc?MKwzs2yd4na73b#1p8<(VQOi|N8vmAE#z*l*bbwIMmOa z#N{VHohFS*lRRK)Kh)18+=W+S8``8In)W4}>3#nn2&rzEyTs*{!d#Z$sUo-gXo<72 z3@F8)ko!g&o%{jzpa?qW0l3OuUUU248IgT`JnCf{DY;VI)&Do{-h6$!U<-awFzX22 zUBT&SfS@!U6O1H7i{erIk}H^R>L8Rq%wU9TuH=@X<1~GCVy3WAMC$bwa-+bbQYn-1 zolf(v%eRGGUo_H$Bz=d-f+kN1%!(-%ASBJXZV?`y!!B1BZrheKJT2VEh*KgWB8G{V zgzx?)(SIAp)e;~oCoeBr0K%5hlPNC-B8=KbyAzXrMGg6Q*YY*rBN1-4?F~E*k^nM0 z(E@ZSi%;lT5~kfgGOI)dj&Gkws$Rrw`T^+27`Bp(4(Q+OXiFv~s{-gDK?!73K-aqu z^N$wL1O0-#l_vz|i4K5k0i78HcV_UhUPT}w zMfGOXQaue)yLo`gI_^@Fn=lc4&m-WLZrM(eVYnN zctWd{#&BSc@B|aVXup`O5&jTVFLBen6D{%7*zAEjkWLl+C}0>DpY-_GNMT_$SnxK{ zSQjBqcm-X_Em0v^WBb8eVBJg< zyFCN_gS^s12;~=R=b$+~=cO7%vV@glEu4G}8=VHJ#O8qL%_IIzbO4+DacAu+L+X-LeeW(xz( zn7lkgX6}*18Q{|K)oKT|Hq3sS-*y79#U%Iy2jIU-A|zNb}PY+pwM7w5Mk(GGek6rG2abO{C;suu09Vl3yvlK3=8S! zA*>fc(j`jq1HL)?QI`;QSPQCG5UUAgN%d_7a}uU3vj2bBdJ}Le_qL6DNk~Fc5h0Y6 zOr^{eN|PxSWlm+DGS5VYOeuxL3K=5PmU&oa4aUSWFH@FznHQF2eb=qM-}ifu@7u@m z9M7|lXK!TP_y2!g*Lj}53t|w%o&@=B_hLvo=la?%0F{ldGL3AGMvhh;)=7UhUHPjRD`IUlwlbZ-SCAIn4nlHf-$ zqRxijVEDcW3I63miB9U>Q&##v(L3T_qnJj44_U-q0`vUZiqJXT#j-gmh1mdR5$Bx` z7vCFww$(XIljE{*oohd3dn0zEFNS%s@C=VJK@abNp=Q~vykEikGG$l8c>_kv0MI}G z{Yc$o;5zTVJXu$#w0k`BfX0uwd*!jb?GmL>Gl+xEiHNR#kljVJ%{B+_o| z31t7f%H$BQh*?tL{p=TpaOekxY~OqK?EKgM+^$KG5MaW3QXr)oej$et$1mK}A=~g| z27(D+ebE_dSo*+)znhU+HHL114id_%|1+oP8X{i762f=>e?sEX8G>idu*|^FQaQB~ zBKpeeT8F(Ln7f;5fX;hj=LuiR6&LWmOC67Fdk7|UI4U8{6fFxicY#SgEn=Ah}6crfQWFp=pox?D4PQ>Dj+^! zSU{yei+rdmNq9KWc8lW+><4s#;suS*skG&DFH=UKo^XTZdB{HUa&mGocO4ioLcuS8 z{cmx69&V{Ih@S?7QGE-ZMmJcNQMz+{_j#DfCYWjOSpT4tT3K`MbC9F;75UBcpAZma z#=GUpWq^&FmxUG)&cpjz76 z0dOp52QlS6EZPsv9T+5bzb({si23_2F8Tm1H3F$dcx_ol>(G0FHQJ!;84X~8?|Li1 zbs~pELEzoiw6`P zyW|#lau+UeL@2cqM0cZ!HSn;$KpNhA48IO;9>NmoeR=e6z(kpe%8BdH^`y@T@&UR+ zI6imhj?D1WAdiYkf$l_L^he}~heDYJkq9<0_&*n}Mkka-Ftkes@3Fw?1B_+Rlni3J?>@6qUxZ^FUMJVC8bXTLzt1jC!ZO6I7O4B|FKylc*S*RIj}EC%qL zkdo3>I=}oF!uPu+8muau{7Gne%UtF;b{2yb zKLQ-mSr6hi2!I{xMjE)xm1LkC;d?y_^=t1}3)T-j3%#?o^*OfVAn`sPV*KFBb(8QD@x#FS z%S49cLoCrcGA?s*ngHWE0prBo0^%Bkla;uT>pdhG880B+{xJB+c7Yf06KZH7fB@*n zopUd33V@LaXpo=V>H(1Q0_W=jLgjvBM8$Rdc=+xtQdy-6?8-<^6^Ylb1~YO7A@(u^ zd-NKLYEbQK5mpD{hx^a5w8;QNCh!O=ttsq9f#O&LW+XR;>=NlaHSt?7pv{^?ih>wf~MCMNXZ9$1%MS>@&z%GcIIsROVHaGX`@0 zjB;Sr1Nl7}FB3DYb5T+<6@WhflUM!i3r|5QuwfA^MfzeD(w|A@)?WlyF$jBz-nEP7 z0krlxM(`#bllf^xEQv5sNaL>?Ye+4}!w1bpKGa||+YXaFhwZV3XR22$@COw^qcNUtFg z_uVQRsWG5ZL4I5unqkC4#0P{SCJgLG39$#syQ{O0{~XA};?Qh<=|4tL>qbq+r2Xx}qu&nkfkiB#>c!RfRB!y;f4 z9)TO>;1-VgtaDXvB)IyZLEWwG6Xe{_!}Ga&p8A3>IvDAe;Q3NA0`SJ?jHEiy2tRh3 zpG@P|xDTT=plcK!o@WZOHbq7>{QkTI z^Xw_2DIT3$2LMC!&sM6JKT>s`NoJQ8=B6DF@O7FuFZj4|`yxknzp+gTuf(N&{25LB z=fyuYlX}m&97g9{vvFYk!sW78^}U$aqjO`;tAvz^^!JzU8rWS{ej?N&eBb;l$KDs5 zEQyC+#)KA7FB7&6H{Z(qekYNfmP@Va_xp><_15E;jH49qe&M==+(PLym*Vtn_JuOR z9dF(c!4bz$FZ*6tP(df+UBAm$$ptlLPdZuYaOH!)kgkUdleq`|Izb)RP$6){FD`59UIy@2kz6rr#IAG`FNM zd0*&d``uL7M97p)&8y|lE`E`z^Nfl7uvz{_-Aeb2y&VbwRt!vBXH~CW6*@pG{Q8Zu zxTg!>p+g=%YV&zr8QI`6h#Dvv_$au4vQ4e&%Y)D%TdN`&`=v+1`}fai84SU|Ube1o zkC?UY!G8)_U%fjzJWt+A&B|g7X2y|^IXU%4zvl?L8WzMaB2wB+l91H1AGs`@Yh>rw zI0gfQgU_Db5wpD8;P(eedy?K;PRLA7cUweKk_~PmK8<(a0BepsVhJ}B9FpOn@$43y z$ZZWVs)a8G!SY=lFkXRhG|ee8*YsKFb)hnvg0D#P+6*+=tQRu?4?^yhWn?l?=lK*U zIhA6{nTXvA3{TW>_Wdo>z?nJ@13@J_IMBT`_#v(BNETt|RSW!(7Sqw}9%|u&6zL9_fn|6b<*yBf!_<|T$s@I%C zyb>WoQs#JC#Gw4-e_W7n-~1nh1&3_c1ABYa$%WCabJ__RX*D44{wxH)UbHJxj)O3# zq=-l#*xW{_(E*;NVFreOaMTLk#J*1gCn#c@Q+SNrw#+Vo*90L-6;ZX28~`sbHDP)(>i?7kse4{Ky(2_8l7SSWD2pUlBF>*bF)9+%Q&LcsuK z2^>&Q%&iK?%oJq@*>`a8?PaRfdsk$y#5pgF!dxBuMG z_5p2yrbn&q_)=$k{AB!D{Xv80R?TNMjfQ`XyKJF^&&NhKr085=d3(Hnc06fwOXd=J z?7A5_$_z`5Qnx-^QyV3cy_0)JBT~uoDTVFs=ZfxdB4s71t+|`de94R=#>2{^K1($9 zA^b%1)b-7~vdtWa!z+){d|p@9x_6CWX1|`mnO~fZwhKMhsk_ra|9?@J0@50fmDj}v{f4v z&w05YNVC{3+*r~S13xXP)+v8MYTCeltjzFC^k_=hE){dtvLVm4Odbe^VYVxmDqS)(joot!dIs^IeJ^h2yIo5y6U0}efFfwN%!Zxkt-)r%(HCk zv6N?fXo7!?d<|3Fn`js|o!vB6K(XUEke-`Y0LQfpW#OE~MF*TRF` zY=c(m)r)KR>_&GA^#g2f_jt47aFo2~&AaX3`vY4KPpA>agXNBP{Zg3G@mf(|_XDU` zaPGc-jjix*VOA&&j5XXUF1#Ou*@LQFyRWd*PLY0*Aej(2Z{+15_q6rq+?Uyz{sIS6 zUZ76V0@{6QNq#>>+EW(0&}}tgqLHq%oA2MPGu^zEK!sAIY@XX5V{_Cr9DdG2apQqV zslc4e5~bd_W2^`(Fsg(3ab-lFt|1{oc%uR%8}5p$+E@4>kxFYn+X{n9bvFoL?yAV(w#C4d?IT<@Q{WLm3 znFt2hM?4^Mp^A`q;HChT7e>q+knnrZZ3MgCjnH(+%2lK)g=;En04~9Nkh&1l$#pPx zsi^z|n_Lef)E^|u2;+g$cjr4#6s(`uXhA0qXay2{T#u^Vn<3=aMhmof2v9Q^#qZ5g zADGuK1oX&TW4dOEZ)I-yxVrA zIsL`>sN`wWv3d*hOzT>VTxSt_y2zEg@tZ{z!6+-wyfz7kuKzQrm+yQ5tDZ{rE4{&_ z&*--5kB#9d9ao~99lZGo6wgDk>+fo8owF9+yt{bdC*`TYp{8qVTN^uBC4c2173CX$ z&BxIhSy`6KF6*Z^h6u}@{q^e8g=X2}cEfo1`?j*RxPcNjlhrBWA_<32E$D>96yMf8 z(y1yrirLXwtNlGZSl%5z-6c69iIK*dE`+ho!hAfow|vc`q@)5;o|JhE*R`jUs=Z&Q zIb>vh%YMX|mB>cN>I9*8RzD8JL&m`y{0DaS^AD9vO7x(4Cs!11Y})}+VXHWxXyN_^ zpOrzE4}Ct=E@8^f6)7k-m)Mc{Sl8s#%>_zmS>sEJ+q~W!C2B>U`;}mYZdi#?-;0@) zg$uY@MMKijX3V+E-tIrWOA7T8VeHDdy;YW{<@RYX!Z-1oiR;)nW-uy;LUPOV)iHW& z^tpaOlrZO|ZyG{1>$@T*pzz_+N8fhh!QGNa;pl-5ZN9EPjs^Ary2XTigqj1(s489o@mAAly zqVn7Ie)f@FULh#k+ajQBdiOx+z)N+{0@J{XBG2MV=%`XT^s8?}uAv%WcG~(h_%%1#|d!W1RcH$rs z{0FjvEUFr?ba#!lIb$+^z)n=B%)Dl2UQ`0B#Wuhyji{3!bJXN}y+L?DSVJtFoI?N3 zRv=6V{=evi?4*nm@J^i5&A+TW0Q0aO@?_|yP=pcWKG1AcrDt85+rVTjF3jSR*Ay-g z7ydX&E%miC8ULh6^^b?wrO&unQ({>+Rx{k~*B1#hH$30aV0<0umUrBeo41QjsUiPQ z@1|YG)@mkJ)klFpZ#sc7)>g2aAs~#dVb5hlFwmt3gfUj(qmgwq}f^f7{2ff^O3sCew>yNJuy z)(5k)`WIUzuY$L39bJ5Hah!*s?BsN!C%1KKNnDYbVR6lY^K{|@7}4!23CHgZ%#?M zotb6XtYc)*{8`(Tj&Z1u+IE< zYI-`?=H|s-4CUy|jLhH<1{8(l%hPP<>&b+A_4eX{RJ=g#v19w7OVMMqa&T~^KfYce z`$=@+Ozk&|rMOL}c167B1KGLWMIBP7X6H-4%koX4H?J;lu(RKm`Th3N_+TX8*X=V_ zHaA$ScVb!HwtQRaPb;?8=P(p?nEQzrz7~7-Q0_VZZSx#L&5NPNQtuopR7dCI+E@x% zY^SSP<8t;q8C7yW;|hGmlTlGu9zWW5o0lp-em)b<-sj4HGDYpEP`VFoHYQG&ZFB~m zW|2H!H;?|)<}3R`sH_Bqa!BQ0?5I4fA`nz#>zGnpN?-1`BW&(6uS=t_c|+vOyeqRp zVAI}5WV?4K#O2G4qOhEe^jlpCcpVelqFUo$eMI)+KN2fR(R{rbh)apaI zm^aTyNy2IXUIXz|OQ(4;goryj8Z=#D>l*=4J{6$^@rXj~C;L zJRRSK^C=*xu)`QGvgbahdQxRXj-K7e>f!JV^bSY%{fOzGQH5Pb7{+#|ZTKd=Y*p}0g0zci}y7+leQLxZwkG?v~O z<%t|MQ~cRv8fDW2O{*CDSFko8e%h|Cq!P$dVtdiFJ}6sot7|2rw3J?PtJBbHt^0=$ zc@Utc2-goH#9$CnGs{a!*@Mk9YKCJPuqp8%lNQ+Fs@s4mN(1v#h*!Rj1AD92J;bRD z)b8Y>q9eS#M4Zm`0I)f-g8p^Q+uPd=G+j2ZpjMU!g~+|`G!N_>bS`MB@)h7CBJf|B z$C%;yE`s(6z7T!$S6-2gk5~Z0brQlqokQ&fkU%T-Cb^h_+4b-?q19faYZODY80 z`(i0mqAOR^LKvtQfga#O1mZLz^0BOG01bF1t`&@@e=`S@Tk@KDSbjml>+0+`Ad`7l z`78W@ZOM?-J&8ooVJX9o0k!%t5S(~`iOc2zTB_SbwCb*LSp!jp5&_f>)KJ7`r3ZiC z3-GUwf4MS90vuuq?%ALN=$G1GnQDy_55OvcH`EU@`0!6kg3g>=ZUYw{f}ugM5hwZW zfPGrvJbND|NDp@>NIJQJ>e!Y3c%(l#83igoqz&%jZ^OUZX@~Uz`MMEWJrJV6(;+3r z4qmVZ$nN}Q8SwS}MVUB2o}$}^S#eto;ji-^iS{j67lWQ+ejBSTRZ*2$*fS%MxFSGZ zzo$cgz`suUiD1RuV0c^e=h2ZB2`0vaY-Q)^e~=B z-F@d|R8>>Ejh$TfTx-9sV(f-`Z`JRxcKO)Z+~LG&%Mz)@KHPD?NwhzGSl8f|?4J6F z>pwYy8de4Os?yXM|4xRf(MuucQjh!tmsW<3MW*4c7W*_-7JukPt;?{j&iX3&GvJQ6 z_Q+HnJ+`9#3VMbnW9k{9hYk*Q^b3QQO^x!W*e~%7&Z9%?CmTqL!vO;?3@%WI^U~hd zjd|G+Za&5MaPW*(ZKL-o^Gx%`W|6T0BlI+=x5g7+_lAGqj_QcJ z+}pww0|oaV@3efE-ekd=NMCsx13#w1ot7%;+?=(iUK?zr4*jl3zZ3FJe-5Z3S4Y4B z7!_R`Kk zEfWQSTk*PiU!v;;_&XiIN8#$8Ix1=T@sQ0$kBV|f_Uf)Fh3VuC*IMOy(*=(&wAF5G z7+d!)vL6+_FM56rvxC*I+!zx9V~QH8CeO!tf0UxaA!+F|Rg=S&n=&8NY2e@ie3-_g z)v4>F>uB;hPRvFl3o=rG|r{)TR!gD;DCs@>0BDu7Y78SSFq8vxscB>>#x+cNn=iJ=+hRLJ0caILrn>*|bJPK5y z8Eaa)376a03nSr%#T)g3q!uGZ79(=K)`wviE|jLj8qvmBafQ%)ay)K>z=w84^R|>usDOBSgG zrjcE46Wf)9m0^euJf8Z`{u z@)wNQ_%Er+*~hN@Qt{e0B93nr+(gy99n{M#ANJc2AF~>}yx0isShB*aCxz0BKZI~~ zUhP*1lr_xcnB9(m-rAtxS<42vtbjcv4A%VTv&vnRS#hw3h5LS=b=>!Mnw-9 z?|)&Bz-)w_fhset8EB#+Z&p9K7{XAaz z2cktN!HS#Tomvuz(lAexrEnD$ah19iFilTufp7KHuddWU(snEEvFV>5IWRjqgsSPu zkwgR+0rhZaAz;{-fMxc$^s~L-4M{|Rk^sOIL;v|k0$2PA85s^xzo+Lw%;Y>k7fK_kf@sHmO=c9x@-!VPe6LrTyOrbJHbU}8ror%Xk>d{K|j#{)|D_XF^ThgyVG zY6Djcj`Gt7Zc6#?$U$dP@2xzctxw07aJ!0!pP!H8Hb@w|m)yLJ#4EYr*)c?!0e%R} zCK~2n2xc|_8LSze1A8zAsp1q#$BxH~3V^@DAI zRX`dNZ$Wa9_#VI{ozp9BA^0#sCs+t@Ao%I_RsFuZ*3UA*iP-PK_nFKF!Y(Ye*zX}9 zYY#uvS+K2wLF4eh5j(9CxMCLp{F%Ll@F0;t57>0wE5Or_4B~2F$ubhg=R-dM8!3=r z01pr8R~hHC+@5pd=0#O`zPEPfA?*C$l+3RBO?y=fA^v>MHKFl)^7EYus78nG!0&4Tnz_Q) zj@?pQ53u1aaAbcWVXt0~KhZ{dxcyLI*>$J^#t}_@^H)1o#k%GUbJnqt%__ClJ>+a90xd$H`_;JR20@bwv{w}^3b97@65HHSmvyZ* z&5Y7P*^pZF!G@QOTxU(9C^CpYdg=DTYH9WXGNvouA(-0t9}!fi(c8g`A3#eSFd1sG zArFUqc0k@g_6ry2Wt54{26~(=y~{uOk;@Kczv6VH%@*TtA=KnlA&(k_c@1mAe%aZO zK;zv%(QP)Yj_&fcqw71P$CrmOA!I_i;w{2RFq@Xo=iEYdxI@h-?*22lt;HyTak$88 zqcw6@r8P7ZW=r;;|&mSn$zN1v&OF_dx@0)GOW~1n&@%mgt+g6uG z-ui}{v-AAMv<9u>dsBuuY8{Guz)VdPUbIN!o~NiGPF##H$Eb)Q+d ztmYxeam!b(sL=NFY)xy|JzM$W6jbyLUbjuxs1sgsTb~+hH2HhBiG2nGvPYnQJ|+`z zRFu4K0-xM%XP%t<0jDW4oV=X|%3W;)k!rh0w6#%yyJ)zoxs!3!fO!!|%U0n5xurL` z0h5hp*$$+KWN)5#i>XLD0P`p%u(rkUOheA4}!puB7$JK9tthH!htvd zwoFRbG6Hw6w~_1G`1TA%(8%K`V|(HGY6Lex^1y9e*zE(2w=mx9oo5W%!%We{?eejk zT7DqwyX{cUsTjv;IeV&ji6w3P%*8RlDEYLck+QKa%FumQt0Da}^Rr{&9&&48WgT=* z-hbpNLu6kK(og85a`Wm@`6l&8^sBux3U|J1q>tNCT%MM+M`)$;!Aq^dd+AdC3i+#| z&q~5oxA~$Qw(>@ltvoa1*`Bw$`NkXG=sXLHQ5g4JWS5s2OTyqk48$L%iIcuSl^~gS z`SB+Yx2HF6vhU2}j=J;+3*QGyyt~S^_pr#J>$*nNouTH(zs93{_4lW0Vf#qij%zeO zzz+@9R2aes8s58iB-hZHQWfI`Z1*b&JZ9E*bU)&s0qXBPhQ7P^%1Uq%w9PQ=I>;%M zRFf1NO9%TXz?Guaep|qS68G1hT%ndE_aJk2W=0bhikxBW9I^L~665G@-V4~GJIIbB zFtxi@XmWhlK@c;$kPYOpU$kFycLxXCq1Oi@VD<*sqGU-(OCsjEkS)aBo0-R!rx^b) zFLq*HA43gd!Z&)p8x9WQySWlXPxg-ax23TJkppfs?qUHuU>Fo2kO+rAu(6!~>`m_> zG4Nm|A+{DK=s|&ER*s7c3DSqJgXkz%S+MUPfCdd;uD>2M7?1n+?!DLzp8yhzPcRej zkBAuF-NVC?pkikZTug$kxZy}H{-H4JBqSr@tVI)tp{f5ny=#S9@osYxq@IvPbC>gU z^6(2kk*p8^ykMf5nB~Y1AH+Lhy74DCAtcirSF_Lz-lZS8*|l|DUnZapBs-blt@n&0-# zkfm5dnfS|&``ROqua>ePQL6gN9l6jG?(Dgqn${Xr$t>d1qxC)6Z3%np;L&UayBl^L zqoJ4&{@$HRH~dQINL7NwzC9 z*>A*5uVGW3^LRVa91U|@9_%aZ{?;ly`-QT^gEwi3`$kg8t^HVU=Tlf(2%v&Ju_E2& zZ%Js1y8<7-8gXcH4SU*orz|A}?5n~2`F_TQqJQl6`EB~n)mT%{N zum_!Gk7=s0m~-N*2QHmeXd2Wx(u#!`EvHANyf{CHLn>e3Fr1OmV#WBzk9+p4#pMgO z&dbrJl1{&A(|r^9?Nct-JT|gi(NBz5e*cA@PZd?(8qoJ56S71~kMoEb5c@0F+wKIe zE)Q}Drly}>b<>>pgu4Ffa$sEB7R>K@bC^oFefxalX-zfT+rtQ4xL)VvkF}rb-zc^Q zT}z#I@YFU(!Fe_flWAQm> zPNEnL7d#Bgd{X|=pp!;#&stBd%KOT<#-aCUhip)a7T@03$J4Vy|J|hE3j4)WD_+cb zI44Q<*rdF7wuZqt?~++Re7?z*jm>rfM%UZROU!3yrLXGg!j1bqV%zz#xKPjd+Y5?D z^>yJJ1DJUk%_=jE8|Wn9Ag=LE5mvmsm%nT6V`ihX+8M$<;XiZcYcuJjK#=j$SS~A# z`iIha7j6LF{vQ(eG-OR4|NM3k6Wr6&lSSJM^Ag6GQmLhrM%d(biLVGgRzzQL71=Ha zA%t~%WSazLj%h3trzIzIML23V*~h|3z0CSn2m{9E7+ec{s~ z-ytD5?W-Zi(dMyq8Je+1Fe+%fT2&AW!3f`*d z;Irq>H6+P<1|~r&3{1OWuU@_4Uig!MWbzOkY}V{7${3L6ZsavAvP#P5uQ^v6e5pph z9jKRi(NkS$4t%RHb8`61dEwhd>&~IIP-qNg@mvwc{7e9%ewXevx3Tppg)@;)I zErg`7^Ggx^vL+u@^3MlKE_;vX(-IN#%4e|`gOuthvTs+QbT-lP6!GZH&x*;W-0gt} z`xc&D$~+^1`nluv(9GMyj=rB`NRQcTQ=>3ha@A|@4$sH>nA57RJx6p1H759$b2>w< zf7y($U+CQ1J*Y;46+cY$xa3v!6b&WHa!Gl7y;Dktvd?8lny(J@qWGzv@uy~|Pw?|+ z6%H3>tSac?jvRVaqpcA!?_3dof%V1KO!q?YklkRfvl1*zjn*x)yqy;{q$64-aV0ZH zeU0fiO!>!}ofs#x&$!rq^B3>+b6^N2mK!_@aBpHtg+xOBwH7mWJ zU7Kk^pSV^zn89k9`P1BAskM*Hz5p6pPAR= z^}c}bOKA3!s_Kao`9w5fqWRhto(@c&1NG8&DY~=IAWx{rMb^^xQn@px_v@k|^Vylz zhYeAohdgY_Ny94~5B1Kw)oXdpZ_aGV4E}yxTVQGsRpnws`#iu?QFp73Vcb|^osja4 z&FHvdW;^!Ot@5J-b~Z($KaLO$tzf(@=Iv4EUD7#Q6en@!{#1^hWG=Qj>3rJkW3Z1e z;9^6+Ndi$3(b1-Wcsd5=y&;^x7ec~$9?(c!`Y&~hEn{+ql+S|6EsnFueiO*bdqCz^ zQ@9N*=?Q=LkySAq3c3CX08zjZSOs_9-h>=dSawKdM2a8;SC!fyms4Iq);B;{OiZ&# zqrutj#COEU{2N?!(3Q(u!&=$^=slHSaZ49zKdiK3$b@~!yRoj{ib$Y

f`<_(q`p zwNq0YbJoqpv3U2vTc)YaZ@5wfdf;w$965=8ta<7}OU$hIT2dC;b%d&D2x}r?8Pr)& zMWcCyz0(1B5EdMgYAt&BVh8-b1rV~k0V&Oadw1`SAXr&D2uoLT1HO&+ zw!6DDlI;794YS$-MkwT)nOIMkm)MGRg7xrTa~!@kA>*MWlCceuQwEX?1aebm0J_?r zr%(?^89R(l5@4N1=Ru45+r6m>$UVO+nDW^D`)65jb&@^?-V1=}h$dCV?%Pq({ z6R%;m>cImZ@OTZ^oK{nd!qLh2EYe`Tx5bCwbqS}{4_C(I;qH;Z^|Q}(|U!!+)(Ct|Ergq33!wl*bB)BBPY zoNo?xdQWL$BgcYy)L+zobd*(gaZ$pfH9nLM&8}-_G?lGw+_=FDt35KgIUX3{{DpKD zoO(c)ITq6xYSwo9pk;6&X}&JlS$L$k^U=HyKU?TAt&V9=$~VqK-OjZIAUHkpaLOan zg9CWGFanTyEnwS}tzc4U@ivi5+C;B>yl`M% zCs@4jkSA_mO>f@~rJ*jWWY1GkQ4X{2S`KUK_*wt3VnZE+FD zzm*Xr9v5Pl?yh2uQcI?~Za==+mq_RBvpPA97e+}`Y!0f$vyb<?fC^X7H})J>()e{bn@F*C#sq6 z-z{~$yfRGuLkbO>^=Q9&i=jT>q_%4Kv_iF2LesUM-o+>Rj3fY#*?pn~#ye7p@?KH` z==5$&ahES{d)~~7Ngxf;n1{HNfdLm#V3_r~G$A#PtSs%YGB?*Wc%=+)_UxnFqYOj~ zmNgw9-5{8R1%?x7lcI;of@4?j>L2_7J6e%WsDg?4In9slA1aoE%29_xiCf_Z@P!AP>yTpGq7Zs7`^kbx1-4 z5(nw!!h(YP5T-7)Lrspbsv0c9alqC9322w&09DD-Z*2{ev4&8{B5jr=P0DPgdYVDrVsDp(1K(BqzH`#pqV)&AlsoITD~3Pao_b%e<5JKwPx|g;*Y@qx!G`oQS7D1L zIR~Gi%VbYy58>y078zf724_O|a z2`qqH=<_?V^x5xVXwow3ds!OIl2{dVbBXBQD_^oEV0%{u!<4y3+%zeXwkMmP+RyZM zbbXB$ywWwlo)XrG9Yc6qUmp$SltEALYNA3_ix{7${g`Bv%rCd z=pnCEZxt+hg`LoijiID%D8vD-8U5K_ap*VG zxZ7hs5C86DA~G;DH-pR60iOOgLmwZ7;O5sFQw!wUHTl4h>j;_kgOo!`p@@)BK>a8} z=Zt%~0Bkm4(02Tws@9-Z~LmM$2?QEDzNQPMrgTeL?U~0w`j_}QtfD3gumjFxj zYRsMN5@9)C3P~E)XF&uYZKk=o89~Y`n`e3k%!U9}09pL{C+S7P-`w_h$CBN<{uL;_ zu-7zMM)BFNPdDOZU2{L(Rfb3tz>JQfif*KQvm9{6&!9 zQ?PIViTA=Vidsk{v&r(+&6BP&GY6>Wy{KQ9b4bXVHCRy{-xAY4h*DaJ zugBi)8qOg7ct&WcsBWoXh?L-!5I898%$3zV9ohByQOvElrg5aS~O&=HTjMr0tBQ7#W$}T2D{Q<;rm_&o$-h3EoB> zJmlQI@}-=JkFQwajkq!s|K;S^EKHw3JUFB-ih%fMnxOmj?pL^UX&;D=C}5{zBL|8v z37tEtYDNYpcB80|!&jsD1@FP6xZ7ls8+M3kUDj15P=?pUUVWca>s;G{_8%YBy$i{b z>*SQ<-j{tvC#I#f^-_XE-c*1H#w-<+7mV=RpSWRdAZ}5?O3v)kk#~xck5FklllAG-o6WK zcmnFlDlK}3+!AkeSLU14| z0re6lbtlvLJnR9~+Rd;S;D29AU_>k+$h!3qz->cR7jZ(LgjMjnybXf< zyOCS^Y)v;cG=dT07u=GW1ob32dVuV*NPq_rX>bwAU^o|Gk{8|T)?`ye0b|?%5!V;P zfUgt>9vG6O{PqUfsOAh=_*D;)@jtiV#d-lo>tspTCxD;0tH92Q7?M3$+aK6PpokSL>gQ@$C=t6KkUcoG8>j~~$3gf3@IHaMlw1Ce zsnl~0*<=W{!Fm{)26@caT5NZba|jx>0}I?Tpeu-ZteQjIexJQvZYo9$cVQ(*)RFS_ zA>(ac+OHS0u-~=~N%AF$U>^pFWlE)|{i`eWz3?Af3K#L2 z*^UXu?4Nb#-YlLGjlV4iecmAaF8OAz85`pi1Ze0uWhjIFmb2TAJj=wsn%&fmk6l)>B(BzXZPY3^d3;U~ z6|NagN67RNq~o4J5F}kreE-8lQdFOQSJ2k-JHrykeqdP*8P5$+h5C8=80FY*Cz(4h zB#v0lA7s13C{{)(KRB|?@yLda=cmaLX8mc#1xf(@UFc#NXk1JOPzBql@v7;YePsQjIXS@Vq}9U+i0S*r=GbvN=cd8 z`22Jr>tNVR4q3|FX$_!d+5`Ow$=L&K@K+y@+UnquEYpnSInRConD`Pv&Fb$Hf^3n&6ICJgXeuiWvK+CN|1;8lG6gDV}u?Iah5@zWn!<9&L`|! zRc86sgHVD}Wf#q|&x0KpZ0mBNpF_9-c6!t@wR2xiF{QZ(tf@Bv1CMlg=l1GDx&xS4 zp`WKbVQC=3frw&|0O$gek4Ua2>ORkdiIu92P0s()EeMc_*xlj?9GZi$nHdR5Z|=bo ze8HCs3nHG?9bf^ZCJfG!A$WnnVRm1Kt|3n!QPd}7uc%T00ub2UCk8}ocBBDDz6R{2 zH_U{E(FYMYFCz=!c)ei#dDiSfgfb7nBK{LKWRv46D!Irqd0Yi_wav4Z(-bp8b$zNr zmjn)XFHQ3Y6@93n%In*&t!wJUc^QWv8I-6R(s|Qa^=bZIc#QvH6wenjd zO)o`%lx>&}WqZTSECac^A#EWxRSa1e`R&E}XJxj}1UYFxy?nXfX1KBs35PhSiJ?%J zL$-iL8W1T7Yb$0U7<}jM-H9k0S?8G;VDaaqr;lv1fO*;j^oP<&P815Y>{0a)vL0}( zS*!w+at)dZtJSH_cdw~7jusY)Ojla3<09IY9SzJ##gma%!^7H!8u&HI{Zz5-m0{KX z?yj$;S8}V?eqac-Yfn7z8ke$W_g@f79nyMs+VF$F+ve@EtshS}Wqc`qm><+QPxl4R zxF|NM48l#BDyqU$b|d1GGquNm6sHfnhu;`okqSF3$4HGrorqD{iMBLgqkmTZJUGvLDGz5p)E*YFINrzR_6Ck%3ULH1TFpDP zz3|2R5U)zwL7iv+xGSouamo7#TUpPMk{g1V60tnw&aDL*aq)+Wn>UiVecZzLGtdc5 z@Asv{;6yW42wQTTsKHE_yqbei5O^wt1^D3N;-v+KjPej>K+eriSmn@^2P0!FD zJO*fzi-6fP%~x}9C;-2QslFghU_uQxxVJY!)@ToEO9?i?!OdKO6xo2HYktJ%H48w( z<;|9`?}h*Tb1sd_&tK@&sYZlb^}D$Z+k#Y z0vI7g5^^P&OD_y*kO9aeTggH04e!8cDsNVmWoytD-;+d}|l!1q0O ziuxe^whsn)!Dg#q@ON(Ja@D=L7ZtJ>(oXPB-0Sevz40*L*dMyp=8GRnc?}E<9u_{0 z*p6jXhAnL#a|O+OGjiZ~^1H5e=S~VN!230IRaMmrqFXm_&*92=$T`{Bw*ah*h3}B( zMgl6S6D*i^0Hg*A1x-M*>#(~jpxUk}3J6?7LI(Jo)VzGZCS`_S_6FW6{MK)0{|>iS z9Oa_=nM^E{8M-0<67$_Xb(Bv)|F1~Y)NZ9=TcIQ{?~!DSl`W$LE##hNut~PEs6u@e!iup_Xq8b zWiO2Q{P>w5;pKKGYJt=eU+p`;k>R^_&5+R8Y^ARsQ*8Yp_xbZ$>~_-D*1AGF4N9_p z)ZQHSHhPRlNS1qo=8f#qecJZHLj0IDyQJ-h<4meVrM-!Xi82mjfiQy=K242Md0#Wy z22Uq%5&GE)c!WEl|8#=+$HFi?yAlu`Y;3x=9y*OuLJ!z*tL{JULqF0qUCWzZEQ;om$6;R&hcm2xASt10 zo}>ru6#=g`>nrTPnB zBLpjtm^l<)CndFQl2CrxGTvKX#hrftd4_~}jeX1;m-S^_FWSrb31HUiPA_5DQ|fH!u+%E1n30%Kx+7Z#bLMl6vz5$ISPyYKkf zmuZDmD9H>cdNEtwc}_dq?x_SPnYC~HQINLpP!|g4;6_7US-=UJ@g9SJMJb|rymP<~ zlLJh;R31*wY2rVm)w8u=AwGyHLm5X7_LSJkHNjTlin&_|;kRB+ z@3>lio_Ak($Gsg{K;QZDr9)QX_?TQ}(2pd=CXTE;jEz@TnZRs4Wk+4DOtGX{k$PRM z7;ZvTa^O8}DikXBV`b%ywKe$x3$JzSH#4&X6|RH`h|EXqXrK-s}cF{8C znPcl4MGz4Vea%0^X9Orzc+$fSN9^J9@K7GP`U{R~OG}P8^a@M|Uob?J!_sURSC+tK zA`%*LQFMFJ&32CO5UMtanJ0JEX(jIVh$hX6!M)WAr9%tZ*7PW{s6CIDs%kK-ug?AF z;+FP^QfqYeHy%`A7Eo~DH0P|U;awa3l9qPLJo*bbO>SABP^Av4QQY@za@^ZC;9V3> zqe7{?u4(r zUna}gWf)8LEMwmDn)`m9_c`9<{o{A|9e0+Q`Ci}8a-Qes{J``c+e68G!|d%P7b6EZ zWBjhu{f*E1iWX!g42Zpr=EyLdvp}UTbC=K{nl>O7jq%ubv7)VPw;s|aZ;_| zH^h&MwwYP{`7@L36P0;X@%NIFw|51}XD|VuE2J|57G59wJy}>cNJp6= zFFnsy`koSf=!mtJ!Qk%8_Zuuf(0@*axk=NbroARL-d|4%U%5p1?B=M;Uh75%EP6H0 zy>!THXZx^*MQ@z46R7-?dwq=Q*Z7pm8GQWMt*0e zIW!yE>}cATUbtI18$NkZX-&29J>W;9mHRtO_3w0gneR?Wttn0G{PBn7chF8ZLVx1O z#F>|l7x(QL`@wm;^ZvklrE5K{Q`J+ol2r4=tkjl^s~T7MwD&$-#h!X;yd>)Gg@^Jk zO|KexzAg9dS!epK?k)XigZt@{#{<;2#qqKYPMh~XQQWxi!Z$_6*hdL1)n}USHe<_u z{dY2liwC-R&8S_&^C~qd)Ov=s4sj07bkX^h~6gjHmML6#g2?2{f z9zL1={(c?tKT~rE6}Ta9Tvw>!Y-?e0PDVf=a1RGP*d}Qdb){{t88{#~SJBz5gEGHi!yw7d07fh}D2O z!n_q$l) za1?YvrS+_3LA3cQjCV2{exV61>AH9fSBj5(S+y)GO=9%%R%oAFrkf{}&i)WF__nkx zCnqNZQ|zbM3PFW2(gxwbpCbvb{_O7Q*_$`f%&#u_24Ex-Qs#q{LD;+XU}tItNYJZw zDX@ac0kbiR)vL0yvfoU0<0-rr260H< zx=5!Zh7N_wnfaZ=m%RcfM$(=>4!m^IuKet3o7s`g&z#Nr&y29rrCt46YwvvMo5d}O zp1So*!nY;RgYJJv&Dtn<;|KTVuA7lRuJQ?MEa7%k+o?3%mx6-Pk*LAJHqYrCnuO8@ zqqlZXdG|;8sne!eS9ukfB?{EMc|Y4+MfjT=-7k;F>asE@W3RP1k3;wx{-I3LPTA?2 zhvwUrQ$tItYbwp}Q9dv4uc`J|wly7CSkGuNj;3HbRiEt6W?nBe=Tf&_{M`F6;gd zId_VGsmGu!&zrL{0^ANsuy^SF@@x+*X3}9a zH>YF3_o)87rkn!$b-6r0BET@PaPGXIg)cah?Aho!LISVm5Z-fxDtI@47c0}(|KTQq zG38Vy)1I426d^dn5_M(D+q0!%h0E0F9R$(0aHcQf$&+(3`z1%F*;BB!z0-DKbA-V} zi$AT%j0iYHudG~_!ROzqsCz96vs&yaI!%QFQo#c2h^K?+6uimvo)8;DfPQ%I-U%6j zEdfp~`PMMXRH7Iq!c=JNflkOO2ifHT``bq*GvMMH8*%Sm42(1y$%NvV>or&hi%+zV z!LyLMkFmi!v4mW!Ra}}^WvnQH+(`NK*QMmWBwVHaXtq&sc&o@sBR^Jno2`1?-GoI7 zJ#hG|gaO?Z=aowKck?;yLAO#AF+>jN4py*gGoM9=47Q-Pum`*zMn*cxiUv5|jC&uF zwb5nVwJ83xC;e&j_SLo#gJLafl^j18#wdHs_?>#UR z@A$Vysb{yDewEXRu87CMvrDJgeop?JD)sj$v&1yV7E#4j6=@?ZFTn}>+?mT~2O>2e zrS;-h zGbP)J?rDWawSh6uWR^PbmZ?uA3c?%Kv{lWy88y>s@Nj%6#NK><-X6o1VL#dL66;e< zz0);+zDJR##pZZUc)XOjuupBQ#>h3%4Kj$svMo) zI+iLvl-%B8-7;GtvacwcqFnp~uI03GO3o;Y#&DjYiGHnaifNf)gbXx@c2z_gjHF*| zxqLBav&5F))A19S^?t%i*ZSZw%`2^*F*_POyGT`9dAjP<#MfWj=#1HF|DW+WZghVe z{xi1u!@;woA@p2MW8U}_?_%0=hfunl<`Pb)%s&^c$1nNLPPU{ZB#fm@u&bk1usTA$ zCy2?~&Q2*WfA%*S8EP(Dk~5_9FkEH)j1<9K&c9;Q7GMss$QnCN95+OVQt@JifHW^L z`cDDf%ne%bT+6pXMCbM_mOjOSTaPV%Ax#{|wuM{9bUM9pVO7Zo z8!E;VPmB%^r~9PK;E*@`w)*t>^Ik~qzHxKve-~0(*LDXQnhLu@2#C*BMDHS7rBAyq zUw(f<1BP4Y0Lak)2wux6D>M49B@!O@!_uUOk00Z@5j(?RZ{v>H##r;CNA1abiOqj3 zNbLA~wP*iyV0YQefA$hdjH;nVqvlWPb1&P}ZFsIwKzlxX z_RQzuAWwQ4Y$ue8x_9@D>kX;r2V zW_Jj(;lf&Tcg>znamW=rH{cY=T@^9dvmG1t;6a?Q&81x^Br_if6pZbyvMRP*j!+(X z@XlbmTqwOKi{nT?oHf|TtjgtFzOUYpuI;7hdo*m(08Q!bP!{Kv!C@Je%Z8|yL6lOg zTUqgo^OS^N<<+N7eGgxmGg#8iVwdh2Y^j*A`SyFEsF;>wZ0=^t;Egiz_pYUb^+RpO zrvz==uEs9+u76hT5^tkIdwZI7Y%+b@RgV&?V9LwvsV9v-j-NjZEdFLNn4XpIyI-09 zIBjZDd(hZ&PkCJ#cKhu?R?SH`T^ywA(VF@u)!Ei7T~Crzn<>G_l0vs-?ohaS*4M|{ zFKz0r;W?MF@juhjVY9u+ZFaS&dDFNpXG}tRy-m{`n@;A-JnDIVA&e<|2OJ93>HgQwzf0AyrD*UwYIFBFQsA)?6I4X%Ply)`Et(m%3rk&G}$Wp;3~n##ZLaW zKNqteJ`a*H`cWY{+g39$#jI2}jLb1fJtBUu`^~hr-v=XX*cP4|3Z>nZGJI4qaeKJg zpOZ8|rKjykqfMp0z^|;TiqZaal{y;A$xfX1kRW!~W10NJs3;dQR9xZNx^{Sh;4V2? z*@YJK=Me#x%eQyFfAglwLRtCw(aGpBH*~+NckrShk0?@CqULD7eARQ{A;Spz1;A7Rnem+-vJQ z_OKE~e;xFa5AWVRE_=8IlijtB)Vu_+`Miu3!orE5o=ieDhWWZPkuakzVOma_J9FFA z#JD)-Kf*SLu-z0@GG#5QikXSdvv!k{RPUDC9EO?nna_G*8iwMOuA-K6 zn$Ds>F% zzP1=;Uy*b6o)q}ZpSbefxWBWxx!v!US0$|KDciqUS^hNBFp386&`0fOb?kTa>SX$C zEC1nDX^R`m=v~#qoT=bj&Dt0DY+DgAQ~zUNGS)B3c5tSL$9F0;*UYxQD$me6)qIDx zR*L_zp_R&!F07&Rdgc#I4LGdxYSzMooThvHQ!VA3Un%|*CHSqicAS>YY+9&aO-IXu zysORqawzlcg23mD9lZEdtT}a=$t>Z8P3cMO(s5C8l zUS8oEb_uO1_Q#Jua?LHDZ;D#HAKs-oRwL8PWLDL`Ev5Wvs}`nN0Arxg&ScGkSf(du8^}P{XX>O{pHh1+#Ro zI{WW=u}bGpUVm}Q>YRE@3ScAmi}a#@Qcd2pKh$>FJ2CMSJAXzaIV!rFBTRGaR^68p zms3wvo&7E6J0Ur6BqMi1+obbvf$-|Cif`5rEhhcbO$TZ=b(Kn_Uo*>k;@IeZdPKBi z=Z(TOn&VMtHz=k}C`>^*Z!zUxTFLR76~f1VU#kBPGb!Y}TCD6u*T!u!$E+i@B6pai zvTYxx8Q!jHSW;;E}6B+7X9u{}~`--%wYQu~B)E;jCnii62^iJ7E zowzhzcXh_Q2JcF>&T>tu5wr%*i_D}cpzvq?57dU!Za3=Z4*1K`jgu06qaRwl$w3^Ksw&rUfq zd?y$yOk-$W-QA858ZkVqts_w%C|&w}{B|KN_vdiYeUiBQj zys<*Y*2@L$&JzrK{UszMzkc49yYCs!=mEUw5~1BBFakKmC@@wXk*TnAt*k*FaN+Uz@F4Fw1fLj_1Q$hW)n z$vDd?S7mu|yhFc~u1a}XnS9G2#5jVUn3*70`|MoB`!N0#jU$Nx zrUO@`?7le8SQnbT>AEB3(9)gg-kNF>zgW-gKvVp&yM^md+xJA3=hteRhISg2)?E>n zab%|o)AL=_TN0x3N0U)Fq`dYLG`2E%DR$1n<5Gu1q^5h!^zDwF%)Q#D!YX>8;~X@mtD+lIxD*gqI$Xod059n0vQShlhJ({lJvPmokwQ z(Ux!3<4ZQDuQT(?JjNMrlF4zJQ@wUY^qfZL zJtI6luQ^HvlAJF3@U*mTqAi-!XhWA(hkVYxeDT8iT)NYJDyvQro+PJ-uV>TZ?lm3vMOsV#GN9~rA;LD z?#=r?K~KsbZf{w?F~Q}md3$fV3Vj()_|=-;si*iVpkb6aI^MX1_1wF;`r}pMj0+v2 zPED77ZERF0E1*LMwIlsYr>)+cepZ*}ZPGGXW^s7dkGH=v)+*U_8{aX% zGiQoeF0*HKC+6JI^iIjJJ6QXUHnjfp_H~ZDy;uFWTXzfE9-0gs36r8Gn&$fZTH${h z_KlrCE~WjM&c$)`%!XVxG)mlaaAwl_V$Q0Dp-N-Ds+;1=DeM-%5FLGOIdrbEnPJP{PUE)$r zzHHkM!-if8DlsnCuO%)vta*c+F*r*%9QLr8iOPQwcVxNrJM$*LFJ8kLt=^`i|HfE$ zOwVr13A$jAl9WQ8Zj12EP{>?4tp!orO$xmgz86Gv<>Oh?#mu85lL5n$Ix zISt&`AS#{$Lsf$GO?-Ft~bNO*A0zd%tHYdc)lDOBySx ziF@nR)h)0mpbCI$t~WZtC)nf7e`0Ol{Ct%ytLE?A@cr92MIKTH=70f@6+(;82i{iE zZFu2NEOiEArK+U#b#NkRtlKoIJdtD!G4E?9yOYyP|dM>QkfE~6Z}SqC+lTq<`I}tD^pOp zqStwR^5n@)@pB2ERP?hD=IKzQqYF&Lj<+T!B~|Pg{1BU-zByJK`;D|3P8?}>Yt)}YP-ffCK&Wb>

N3}1r*QCu%u-x~cGVSTb#gqr7Z8=i~ zmD2v|rrMm!?i4orgDPT=@b}Vsmw~ueme`ZBry~MEzn1hGhd<9AY-w7f7B@!wDWf+G zq&a8oWr(w;cTs=r$!C;d{~y(Hxw_V4TBn3wfZwo6qR)mZbdFigOk}&}7_anuKX|LL zMaBP`#N>U^sa$EvvF#JF{#*Ar1@|{h9>TMjim>DfvrI(D2cYde!!Ip#vABQL$nv=2j zNRh84{fW?JwV=|N%z1y8|4Zihti^|$eXFl;zEjKW%h^(ztXcbE&#Gp>>ElifqZzZI zoQ@C)Mt5I*A=m(esi(BZiA_xp9G#&WR`NI(KFjitOg>d?Q`P!$q3XvqX|t@$ z4R5 zQ%Uhu|LK8PF;B}u@r)ir!Js_`vm={It-@OvJKm?7P&prk0TS4B>~JwM(fQyv{A~NE1yKAOU|~>VT_GB<;{-FJ9Fni9sJ|=dU0OuTq?U;Ur0G0!k{ zS{xy`xyZV0=W&7)g1R?je!GWPNGeHl9@dqT1jA8wv+_&j3j6BwOfe(w)9 zE)`dpH_i(Ntz*8(EI0uiy(tCjw-<|@_FExn^;LL0J`omOuPT`R&3QZQw=5+_BYqxr zi;9B0{ChazY}V@N2k9Ru$Np<`oOiW4UTyRpJ7rEBH-OB@I7-Xc!jHz;Xc7X5EyhG%;v25L@^`A<6`}%5lC4HjDJ>7OO z@qZ3+QLsfQ1WAw`lHND) zSWm(Jv)Z3O?ML|+Eh1ymG5uOqOx&iyb95h=Grk;ol*ySWp}G-Lqw(+)LR&{&%rz5=IVIL&J5JpQ+uGiF%X6dbIwQ3O zSI1f<4XY)6|73q@Q3ic#>l7T3zRuOwZoH+nGI^?SBAhwI(J(l6%7ps#%x|VyskNf9 z|L?MOVv33#DUI=hdNnlbu_TGf4pA|G`WERWA?6v=z1w!tJlfw(QUrtC%R?7Yr!`gE z^P`$#B5b`6#U`F9Y5OqTA;0lUvg|~6Z>W}!)%`(n561M5ymbbgvt4_{H2ZrUzr`;( zSQV!IG`X8YXWY7#KikrF(I~>SgcG%Z(oSoRkGqpC=X_sdyUh|#G5Fk;O?O8&(&+D+ zfQ+-FBsW*)X}`BFlsUUWOZkYgqES3*g+KL$qn)-tY7aL19EHN6w&A!z5am_)6sdL;#aAinN4%tNO$Ak%e%}dIhGy8i4MIro1~?z{1eA(XA)lOO*=MH zX(vR6mAsNhpKgrQW^AA)8Z2~hjz3)_HxY8a==0(}{yWVanu=pN_2B-ni-~7@NcEJj zCBO$6ufr)ZDf^!P5|mzitNQm%(eGmS&q}A3cheuwQ%=#UjSg22eYQF1uj&q^(ac@n z#UdNJwJ{a(C)j8(_tKiq(c8y^lBc+iay!WR%i2ePC)_?Es@(j`}XbYSu)$YWV?eK zfiaahY!7r8on?m3-hkm-v^m+3sk<`|d1*o&uZg`0D}>tgp(`9c$QeEOCEzN`f!?+t z$;?NK3U-dwS-SKp0g)$^Yz z_CK$1cXR$`&WJYWEhd+8y-E6fTe-$5#pT56s+j>VS~d8|92V*zPB$gDqIZ z?Cc=NCM_+Ek%yrmCQR$`X^DuCA7Y8^>{k2Lhv7>Jx!U@~Y6nnv`@t2Ck)JNAuHJ!Q z5yjo)!j`B~f}nc<2E|9~4V-Dq6!z&-FgBn>(ud!c>Zv$4C1JopM04C2f$G`{d^j-W zIyybxzioo-p&426sIJ)EL; zn40-TP&%R5q2h*ASJQLp#JB!sx(!1oZXV%u6*;=cp0u8Fc^dUluJ=n>-b)Re#P6p( z&Lxi|<<+K&cs*!0WqkjtcVzr$&C}WGj=Hg5R=Rlitw;8KsdPmE(h7eh5*s8rz+Q9g zRB?hs-_Hz3=MLBH>6bo58Q#39Xz}5adsLGba5x14PnjE~dc$L<(%Dwa%ZvtI_RUhhv)^Zp z=IVUvc$f7&C^gkG!`thym!`)#j)2S4?TUMZWX8(^PZ4~}F)mk~?>pT&GAX6^U#B7g zJi6ptg)L-Uou0Yi-*dcy@{l|BNNRik{f7_ZQOtYLk>%+>#WDelGedR^RA8%Vt~a^w z7l|z`BjYQ>f8J`P_l#fvC`9<5o#X0=ERLmmP?FhtOdy6N@k(3@Sa4$Lk-4@jPo6p@ z57kPlfu5c*_IdTPSh?O%-G2Yx8+EEFI}T>xOpW&+uTFDjRuKi#DE6T3NJ&fUA3!ew zjdg{Z?tl)0!ovp-MDj35{4Z^fYqQPfzO=*t6c&i*S5a61!>^k5(TdVykDA7Id*+fe z`6N@`mXMKAljL^sV&D9a@@hVRUXj|JQJ%$OaX}K4{errk>27m~woZ)ioO5zgrjW?~ zCt}`yJkSVk$=LiE8wJz{h!Etmujhyou9{%!kC>X?ue8jtU&g&@W4Hm#`v*WAp5Y;>A`n@tfB5i27$Th>#m;DD|5=~^xO>Yi=USZ39UD-gU2i$nA7x2Gj)>-Jr;UNWPeFC}&iO^bRPMeLdQVD&RmTTZbwP0uT|w|07!(%NHJ zb)+VGeQnBm-0eqe&Idn#!m!ispy+4XQ0^=ijaYA96CCCJSX0HYp8nX8f5-oja{3B* z?6)l2?jvndw7ouXkyUAes$f!y_t44xL3_kh9Upb=z1Q}+Y>{5rw(l|CerEQn2l#IE z$9K2EygL#BDa?Hc9SJ*@(&abj_-cIvgX;dQ7UxlHfIlVheeY?v_f}%i@K*A#dcs2# zqaiI>`W`pMQaw_d-3>Ldud z`;&^xTu*`4sfFHZw8=ROvUgXfz`}W0`?g71^d}gkI}}jK|I$0e@3evwqisIvyHOqX z_V(VvuxUP5DE|nq2%;X4@}u13Dh^W$%6;xBeQ=?pJV{Sip}bcl>(Z^|qRQ=>!tdXY9rPXaUgqH=H16)<(bEyC zts-GZc1+=$+C~6g?c`It*m(o1WaMIng@j^Z+$Fc>V{`QH<_v$1AA2-W`|#zlZQ)fOL92t0NC;WtCUM(_9GJbb1d|@M!7PNu z?T`>ly>stdc=byGPIu>9}-$y%%29dX_w4$l>_Zqq+ilC0e;pQh4|Z@g1H&m|Q{X zJb|T%^W;1c^t5;U!0(Z4BJh0+eV)}}{n%zHFE6j-f(dW39s9Hkr3N0QRII=THV#-! zbE3%&`1#wr0{848QP@S#0>5R6gz%25@(u*!F;=N7xnK-~&Npvt=Z(f=n`%z2)0Oo7 z+xnTClS_rL-|V>f_SJV3>551D>(?t=+vg5= zHjf+SP1vfW4p_N)4VJ}UrvC}4aBH6!8GB|d$QUWyoqK}kuA8)QO?{;YqrX?~$yk|x z!)jZ4(L4Ug%|QwBqAm2Hit==2E>z2OG3JW4qwSE9X8im=E3956*_9zahiOiNIAVT{ zs~KoRf+H2pdJ6C7xUhz&%>1Ht_DMcS`cr+M;fogkHplfzaQ3pEQ0>W(e!ewb7UhNf z!ug2~)o(IR7R_h49SqS4HR{hVW%XB$_b;EK`^(ogx=0QT6o1Yh&GO$tQ&=~zAOI8X zFHz`{q>~Vyv>KB zMl$p_a|sGJyS{zzc{K2>rE8-TKf%y#B)Tw1gyeews?5d`5Pmhp)1 z#(Ax%gOX5~R!lQ6l$dt(^=?=gdS01x!(Jj(7r9ZHLRQ%%3?L1WY7U0!$wF5B^6=>j zC&)wVVBA{zKANeFN~HlyZ!1fF{HU3kd>OwG(QD&>sG>XUZPI`$>R!IwrhIfpANU0M z{3hq;YcVsS;?^AcEF`AobBV)Cs&0;gzNy7Tg7~PP8diuO)MjUph^X|uij|L8A}?ol z25Zd>S{`=g*c;}^=&nmVy>YW_=JYt4HT}%zwO{BHsZ*zYy|fdLze;F|a9lG~`gH2+ zQA6L;FZ)I2m5ndacfWM%{lKNuFV}ApNshVPgAoLe78~yby=mH}+62`lN$>LWa<#pR z_Qs!;`FZ1x*5?t76B4`i9`xf2k*(-Q(|(~4pnvBq??xUGnr*yy>fxL$i?F|igyW$l znN;^Zm*#@q?Bze3+T59y+K-dhbW}UV6%np!-6v4n%UhP6D+zxa7I*EMeSBhWZGBJm zFrz7s-o^LM2Fs?V#%y$0&JLS9s$1n z<a+^Xag(l$1RJX2Y8^23~{MbM<7KonT?^oYAKR zbL0XGKM+khARSTLba1F(^Hyxzdz8<|+V#EFty}F64iho5%F2gf;)?~_eFI?aHg3Cr zioq@#oWSmz?@WO7miqIEY%5Yf288^m{zXTzl`F4aFTW@-pI9mgQ+_hD#d>u|dcC18 zCMsCzqnF7DyFCRUHmPhT_bflSfBzI2VOODH6ji3KYwH5xijcXLf*k`OyU?LSidq|iK4n}QIvEm4BIqd?9Zkk4TcY>%qAYyu zv9Mp(hZ+?Vq)d4v>#e{gK-^cSE2EbhN`8Io=4}&i$3%_b{zWs7BA-M?%WzJnKQl^~ z-dU~`=~4|+&#}XCy6eux4O-Gij4L<4Q}#&8D_WVsezYTq`CPKad5!Fp*ngn$haGs; zeCy_&2wdkGeKPuj&g0NYF!AsXPwMe`nxpRf#R)^77DHaQNgi-|+ghGSfI@Y7o9e3C zQFz?nxY43>>`5s7-n#pvL&XN8S0xYh+o3s~2eTF4!!{-&%(d<^Vek9cp&=pIf)T zVU7Zd#U#vh0K9bdMVSO5WGf|b4^L$ln(HezJLXOtCKEV`YEDf}y_&HS+iq6c@40QS z2(fepb{9DaNbt{u%;2Z=iuDqUP zO&&%>V36h1Z9Z$z-{jF(qPivJ!h3R|=m8>`l9SWGT>kgpP%`f&Q3{aAR-|-L8y61# zJkU0S_^e zyxf5t11OK)a#flu*DdgJZj;9BfFe}|dw zGB?DZg2N~?4!F?RnRrHnXE8iLjw;Kkt2dze$h-MI=+uKp zD^lBpwkbQ;9Wu=K`kObjqC4GVHEXm-vgiImBm3%vZ4Fseoy3Ex2>;eLE)gasCm%i* zrS2H~QOBMLt$K6j@~xX~LyL$!r~i_!y1IIfH>TDr-kd4fl(5_XTS-Z^3;cu!a!~f5 zg%#U&_71me{%iZFn(OQJ*TR8zqtrLR&oBqXq0_=}(5YQI1p8CJ=Sot}L3Sq?`>(-@ z*&G%+y+|$JCu@57E=zgBw_5=#UM`d^wj9#~mg_r)0Ic%w-;NO@&H#?2`(}_e)1B-jKD|}fb9MqD}V3HRMQy?D>&aFs4(YIy(v(&lB=kh(5VdDkkcR6;MTowdSXzO+`~&5{w9MyH$Rq+rRdQI zwI(}nu~dRlJp!G&R59jDK8#)H>|Q{SZ;P=TW{YQ|6RILv$Bo3YYj_aXGc@)6|ID6J ze=~W$Hg8y&QEa;VpR z|Aq9_CFTYOZjME_ZelL_q;|qw!fLaYmJPZqP+yf)y)7?r~{Aia>4O}i+=rRQFeX-eSwxakcL2N8Vp|WnAJw_y1GeP>?qkO zBqM3;THWcj^!D6xCUE!iw8$5JLLmf5;l_i*3!~9>sbGgl%RYDdR2!}44 zM^p$Sz}``B;rm|z_)tlS;#&gHcF-Cu` zwGVu?!j)K!Lk80B#Cc3pG&c?TzPa4elRGSh!ZF2n zdYtUPcU?8BuC5-#G{;6wO$PZCA};&R3>AMlRo3Rpd|fPIYWDX!4Ln@CKeiPHy>n)$ z|Gypc8jT>WEWB=r#jBh1ystV#!}dhmKOdEnT;ly0;NC=b4qkq5Y^c_vhD0s0m4c-F zL)84W0&e2eO^yeiy!jGW9}>vd<*ueoX;>QkT;;Rp-$y<3-iP34clw_ym&BJxIX*F* z{q3&XABYq+nn8D>`2E{jLt-h<+0rM=mFD~5B*UwsNg>!u4?GK>{V>WtO8fESM|Al= zfrT;4EO7kF-s9(Ey)dwaoMWj!|M=>KGv^zco6EX|qL*x6C5SdrEDx_V1rh?I)zLvv zM;Y_xJ7c0k@8Ci6lP6yQZ&A6CFOpF?u#k+Ca%~ z<`O@eb1BSK-lMbn(`n*w1c-txD6}vf^RhP8oOiK_I7YK=C_U0(J1zBt_K%axCSJVw z#AJBTt+}=bd&$GHf`I}2IK*pg(M|CzkvPAmGHr+V_)EX3>Zo(L4ZE`96B2sCP@<(6 z>2lwiivEBw&q5dDi|TOKbPY-@8YWKYVCnE|4(Y%XF}}uOPL` zqj+3M3yAy^1fOwWp)jh8ZxxrLBTzbQVrqs-kv$_U;s!W737!XlRk`v$ls)sU%LNFL zY!bPh>=a6~%-YP%Y;?{w1YU3mx#OSHH{4O=aPND`*RK^2kH73heGrfg{|xx3h0$5c zQ>eG7UqU5aWNS-Dfc!v}gV_LE@|H8lFB)njC4 z7@#{ig4bka`qM-#+71Tx!B6HK1SUh$?GX?w%6|@QrZ+cBWZboB)!Ik|SO@OuSt%yg zH~;-Y^bhKueUJW6q($q}3tf$zsw$nV5_T@d(ZTw|B+Rm7VtTCJF6vwNh{wb;qeO{I zt${c4*9#LBs_1v%f^lMWZm8_q3=&$x_?Q_?X)9^S+FX(-84OZYQ`0JU4LFQd7>TUj zjee;IZTPnFi2YSK1$T5=7Wxa0udc_(jpUH)m{^?*uaIL14t&z^%p{>GR8vCkjexw~ zov+`Dk#5WV*Y|gWQo;~u$J;PqMyEnG6OxiPpgu<_KoS+?Pob#`_CpJV(02w$3QN$6 zr!M))9;~_r{nXH#H!A^kro&FN{T54&EEOXf0h0jCT3G|FLXL#rpA+ekbJnhMD@ zu>kG@_Cum|rUpPmcv!2>fu|~l2);UGUS0N{BeIiusRq3VY=B4{aJGI@}*n@7zbzPI&ydk+kzG_JUQ$MsL%(yeMCnei-D_TBexU8pK$o@0L1bb?>iQ&eX(=DCN}n+m{4oOtO%v+p?P|?&2J1#j$wzZ zBbd+?VK}VSO|V4lQKd~99`Xmr1NQ8~&pdM;*N*?6+qubPNnb*7&UsxTV2&)KbVPx; zb{`FpoT6eSGtHLPeDubWd$O)aEzm|Hvv9%tCt!Qgm!*f7ccWC%L<(D;vEe)h$&4_E zCJQ63Jcwt+Gmj|5WzI-+#{sZLdBBA zqOW!DT!K#lJv8)^>6IuMs&Ng-Y$*z;t|FiY4G$N^Uk{s+q&yoch?-6DAA_?Y79bh7 z(K@QF0dH@VMH`ZILQ;tDJeAXu>^LYp! zWYtAr&sXLlXc#yIe69NVGwGq6c}jeI5wja!Qot&Mzbcmy*Pdg}m^5`^8b+#<;21f0 zv|9%<1#XUuiTTJZE-LCFeIdvjEnT>io3C@7E~4Fu&Zng_V_Fy7&B{s-j<%n!WE!L=%MzR-@s1P^;XlT=cmTFFGrTsnaE z=gRn19tnfJ_P-~6rhXSL;VRt*v8HgbxOxrbusaX-;e^PA5=!T4Hpm?d*k9Gqkb>jn z=^r7jRPsk62eF<#N4V--6S?(E0V~#sT!^rBT}2ed!!9_>bgx2+!X$7%@W;TYCa$&ZqWG z(dhkl^quVQUDT)&%m;oy)X_bC5CgaHw<3h-S|_Hjj%zL$csiH(iH_p08va$MSKK0jSCX;){`D*ASr)(=q2!SiNu}I|4P@3M z`^mtyu9EhkK_0HkYDYM_i~5j)B_wBK&JYEL$YaXAyRPz(9uh_(R@Jx+9#f+ZTucor zXKoX75G?LFU?rpkI-vYQf&4ia7dq8%OpZH0w?uU7UXmRkW4-jeE##U$613471Y~K3 z9*RtRjp$0q`4+NYK!Don3{dD18k6{FkG}$%p&bxX`s#(ZK^mMrB-_sHxa4F#6tj!e zm;BD39^l$@#OEkNj!rWE`7^@?lRjDI7XN!Y*EYxCxmn%0U0g7}DCc}Ja;3C>1;n5`ZUaE605`J3kC|V`YX7i%Uzw6F~nl;R2o`IgvL=ALFrIo$z$^5D*%d> z8Ap%*%Z~=6q9Qqh6U0MdBYBIPG(p@7D9tCXWuS}4gkvcadi`o{gwU|$+QVI*&A&UVs{gk+Fvc(CZM6aq9C{j zD%0G-{8XS~qHH`GJ*b!$=gdM~TLm=qjyeqDNJkPO9^!Z;;yT=2Q%H#wGZTUoQh&ay zJ9%ziDaJbiG?0fq6{o1AJ$h6`rMp@Z&>;20@GT=OnsIML z88j9v$=(k&|FaVOM1TY+2GVvSgG?!nqzemvZOQJ9L`(qSf(p)md9(B1yqO*5PS&xs z_R9#l5&uO|8x(Ycl3M3+Ke1aq<4cIR<0vi;S((59ZA}I0c+|3I4k)(zuUgI&|FK>c~S}+Za^1Ov&mMgf&Kw z=0t;=NfP?&uLqAF*(0{MT97bhPJVu~e#moRNL;b)|KLI_;LlvH1@+@$!~pJvmHNGn z)WJlqR&8rMaXkVr!Xm@|9{ErE5eB&TQ}~j=vT^K_boie%3aK4QRzFxoRG5-94GAm& zmaShnZ!dw72@=P&|Lp+wp5$n*U7HV%Mqn@AIn;@FE1#m+Y46}V;QS=@a~MH$K%Cu` zWf-Mp4bU-;r2eD|cOtK=L~451F-c$-vba}V485wzO^IwaMs4IV?S{Yf>UF;U26%ar zPKcvmaPX$(nd78-&%0&%In0;kxr6=NA>bgtB?IprM|-(*8U7rDJlboMCy5H$rDnjP zJ2zZxAYGpbtA#tv=0y9)=U#4J5a6l;>Jy`IB6)yOF-m+)|Jv-*?Hc`%PvY-#<<)xV z@r_kgRBVPm*W2V2J_ zy$kZs;?MK1+I*Dkg!DS$Vx)wc$b<-}ShoFHJ_-$!u;;jSioN(e7fWh2k@+-z5NrT| z#1GsLm28*-*W)-5e{q~ACI%SDom(k*8K*1ciPms4b93Iv9h%6IJ5IRnBUxa-KQR`T zmcg@-9~9;vC-~+eTdU*JRst)}5aPK(fQQ4PDyj!5sE~xm57s5k7a!ye$j|g+Q(`@H z8C)O{5{sv>co_Y%(q(>RwXAhhL>5>wwa7dBcebP^CoiOgZiU5!EVp z9NfUruoluS*$BA|x&btg73F^8+en!_GZWOF?*wL@P`Qc^(*X1myGb?<35VMk4Z3d6 z07JsKP%I9FYZmn}w2`OMT5>Z9+!dfBA%h z0+U0Bj@Muw6HnZ`2RwY9@${?QTzUqY9vy2C!U#MD4r?u2wGz?w030IdLfc<$a=ql6 zv~bl26<})DkK$cfkS&QF21iNrS+XVhi`F7JDyjgZDWgX#Yf#Cc_5OEw2dWwpkzKpb z!6Df=F+QGj21O#RBsY!Yv4M}ru#(%a10+nOdJ=gN&pg8eQHb;qQCzk(|G$LAt!GKc zG}zmlqH~HoPVl@$FU!P2a&8BCF_%I9o%3WvgSolG+-sO?wOQUe#|+_ErOt5f6j^5D z@UPT0T*8R_-{k!3avYLW52C6C>W<3WfC?}rPh!-vwIp7NplXN()e4aMOhz}#9C#ER zVe5AcL|`umI!=Pml6)flXD(EztwuwGJiE7|;VQwi`Iq|KonT7Wk&nPKxl}TnN}aj5 znxB|jUY<)VoyEIbs+4g7Z5~R?@V{cHMa9Lui}mE;lS^8w>H!*q{x6t}%<)RuqWImY zqPJ3Tq`QR0;I_BOYE>NxAcYUm`3G0cVeqe-qXcl5vtjG&j|Gli$LT=L-ht?eMeX&XSi<{Gc@>~&G6Sel zq2Vw?{}8`-=cUFsXKs`3OkC_w!8U!(p6f?RNT?C$yfL;HID(Ugnv!@atDrznrRIa` zRF55mHuGfBBCf;BR)M{+N2`WIHdh#>eDX6N`RD)&LR(H+BUA!ypDOdpj|IYIsqf*7 z=m2j@zj)HUXjiw5j;<~XjZ&=_3hZcQvT_Y@?16bgkfdJ{dwOD}FEA-=_kC?E9uW9{ zgY3yOWZ!u$?&AgH562b+__e$5e|*6>^AtXUdqAZCZ{4xMnwP@lbyhYu&y6MF+s)jK znGRN;Yu>X;=#i|nzpib zoj<>6UNmuSd1AGw(&>NGCm%NGC6M!b!r2EG0dyymr}}R-jR&hVDK&rYtP&Sb1Mzkn zA4+y6$^IJc-_O^;D?seDO{P*+dx4z+CH}e4Z64VbcuzRTs=_!pSWNjlV$RObzmaYM zyHf&-1y3A=W>ss7xh}RFfylkbKuGU|~ z5g=fsP+!LdXx^>hU{ViuybwIe7;I4yc}kh2rvy@+#I$846~8Ql49u!zxV6am$_=;s zg+tl2k+L1T9Ag+5_a?QS@n&={`0?Zof0x@eT_EM~_3j!t9QegSgA@6VoSt#rLkc2* zF0pW(by~VE_P|lQ$|;L+7G(G#GR`u!NhksI#mN?YKe5~7L3PZX?GgPV#Fo{kbF6P& zyx6Wz?7f0IjqK8xpdF2Yo|4xS$}~|AsG^zHq%?Yps zavR+6@8#5hI9P{KQ4x*46FuI}Kq_w_RT#>Xxx#sZt9D~wdN%D_*G1WN>k7#_bqp+- zhzF(?D%QS=c@Y=)ISG=m|Fs{>{PpWB>_b|Z7>qPDHr`|cH%dwD64!}DyE9y z?!V@`W|I1TKJW8B=e3>J!##92fU*rTR-)YQ{fBz*o{JL7n)Kc~J3CJsKEboP5}YW( zi;~Ou_qS0cNs_BeGo266GMrgIwjN0B6<+`Oxyc%mSk-RT%3q!=Z~QAH90GCcL*J9N z^bcXsC0S;p3=$>V{Rt0cXcN35bC0|_tf>ru`RyAg-%}7Bs66Ru$Bh}Yp9{2+0LwE) zTgok~BqJ|`k5RRI8RR0{>-YXS?8Z8JHv$X4y8sdC#dDbr2F$xCDA%!2vYC==z0J>8 zy}>NZY+OmKUhm?YokbWCXvNbL|=^{Ud$^ zsSog>7n3%Yhgq%mXd;6)@GrWjBc(#WI)Lc(vNa7aXeHnB ztr}QvzpfLt=U;{s$?9Q1+TkX-R?9=LBtHPVs4kF$By-kmvMb0LY<_k>xw{BgLBkJ{652+dws84H?|1Y5( z|Mu7g`2vP4j}s^^dBFvoXhH0h2C604+Y(dg{3F1wrvde-M%87th zGpZy=Q*KQ`;PZKoYAybsfb{$kT>Is@EA@dif1dXQlr5Tx)H?sMUes~$i9Y$2gIPM= zzJKONBLxb3MaN(t zLd;OynYZremY$K({SEfY4z5=b^RH(Y%6PEwJ3y|n;W}|scn9U3xg)&MEJs#bb{31u$?*Nt zXSI6fkn#oEgeYL3U5qbO+KU<>Afs59^ zHD3PhzW}aLCC%4RQS>=-OmmsURU%XbS~RxyA07tIoqGzO_H6a67j_`YuLOAVZED*+ zdgq_=0_f_asP#$s6v6VJb<1YWIx30Ar?cxYU(RjHeEEayTyqK&jA9V0bK{qPLh38! z+eSOKsUEAC{JpMGR|oK*0X|p}@NF;w2bdxR0frt>h;FiK^aR5hHGbT)ip?mJ-X6F{ufW!8Z zCg}FFVp5`An+2o$Yy$h7%AxV zZ{@FV+G`xONr*3l8B)#>%24xc6ke=o~{117TJQNG2*)2lU5VNZp46xZC3kU)bF zIHNOud1+B|=d~RXc@*ax6=;Hkj(e2I=w!keGGxdLvd?saXlyHswkC~%N4m2RSxT(u z?*P2%6=NohqTs567Pz$&BS=3_-V4;Ex;60O+7H>5=5}f$)DkCDk)Tmpb*{HR(SfD_WV89;;%yQZ)F)_DaOEufhH z@`}Bb+qSZj0+83N_4!zyWYmK<$pJ6u!`CoV zW=|qSWob@El)hqjoJ+s+8=~5{|B-|aMsd7WU8Dlub%c2W{flzW3AS@{PGR9OBI4#f znH>i=9URK>{?p~Wqqrdg={zJ}?O}J#c=L+|REcC7^Jmg#fSIbT?o z>(}3?*x;)=`4-$rRSgu?R={+5{}jOB{bP@eJACTYU->!oV^w|bCFx`Fd#ZZi$Ho?e z1 zo%OGsl+`X^Q@ZiFGqG*xtqC0|k-7mNp7x}LohV}2pO@Q5Q*F&!_w|_o;yem9lQ&Zr zgV}0BvTdm@141))+kJ0D?jNw&`fSP>w)$l%)yYyGvG6%}_DtCrReE8$eT`q|YOWg9 zs&(sX2?sS`r)0Z9?Iu%vo**M!XWgO0+UJ*)ppEqx9qrb+^JtIj>tq6E>F(LHI<+cw zzrJhOUbH_zU@En3^$vO15zWwQz_lsQ-J3f(N>j;?db*2lQH;NR-&(gt!%WJa?!@m#ivJ2+6q#|0BRr#GJgvH~6;#bCfQ#2BYX{(N~L)wa#>UDHcsUl;+{ zax#CCxsz(@Sl8Vhe@Wl9-5WM+m@YV>wsukj?Zk14-8n?|vby396v&x+Z7yqf+*bFT zF%#zwM@PqX@hZCKj?Ez7a4{3FFx4*3Tx9BzVy#B1?|j8BspeM~hDYp}o~F0%AsBdr zPq@23-eMt!RKWHnyv+uAnh$GeojH9vA6-c}2k?W8RX{gt+>VAa=Q<~JzATuiieohx zfZn#`h+Cxt81DdEE*O@T6EsQk(fv0FSF&q(M7&PWpfmK@36LH1j00@RQ-m_B zt2JiwJ+7SjuWlOc>&^rF{7FkuCj4DmaRw$-6ju2pi%F80-tTi?Lu~LAqsj1njYg<; ze1MbzvA_{iUUec(xd}e8%c*)(=V8N#->+$LF$(t#AaZao^*H{(f{ez`=E0lnKe>`S zvjM_?JI}_ryzxS;cLh_!stI-N_A8$}M~mhY-ZyzYUZC;te?L;<6DEU2UH`2N!OY|M z`A4piLJLfAbkq{;YJ*wg<2%TnBbb@oD#{k|SK~8J-k*i$oHHr6-&Go&h_L<<$pJ&n z`qx&THwUge)VOiuyyCKV!8JyLG78$#a8V*Na`ax@+=pJgc+nLgS?FvnCyC?WIIMb^ z`w(0KLbFkVWbouKzc=C?HAzsS^WF~7Eq#KWoW4Qpm%ngoyD}a-W9?slIZL6E;uAQ zq9wia$Jiq!l-NJ0yywjE8L@IGDsox*46u<24kFGQl*!o6T{fv!FS#;t@gPYL2bLY;}PdtM$UD4oU^`ORe&Yk!dz9oM9Pf?>Z z>~T{gf5h^N{e~+PlU>znH$D2tAC9Q8xIkO^8jsFhVx-tjcR;&0lUor;x7TD%l0dH> z+PrzQn&4fFnc~6Ir$+_xN)Y*r^^R-YN0rARVT6-LHx?qq_T$s3`2hDWqYn&~)hz~x zr75&--hBBl-_6k_tQ7@J$p98mz{893<6M18A=xID;)-VWrtn5*z;r>vsHoxyj>g=K z-Zw1AD9xkKotr>2Fu>>1ifR|HYuRZ_dJEpb{pf;po{b|oU|Qtj0VhwMMB~$t(fm)R zQ-WX)5KLmUz=>n{%(b5Ic-8XXkfktUSfytlekh$4ul|}N!taTQIKVd4QM?v_3aft>P$BULs9^nBUL%v2d96-nf+_sA>35qX z&15^a#>bG~V3Ys?qEgu{O199hr{%@Z3los}pBFdG#qzNt`FZB@EBop!dzG=r$K^{l zavdU9orh|hJjK$qkL0h2Fh$E5<=QCYK1;WKH=aB!*CX_osT#qj;6nlE(%9S3_Ud|m zv{Cf!chYa)KFjUs+mek~KR2BnIk!QNo7oYf>A#(x?onp~1cI>J%uey%;EBv~BCaz` zQRUPE&TsVof^v%R$MxB4+AF(RUCE!n{Dy+reNn>6DEz5xz*6h8?40{jia2#F;j|(& z8xM293W{_}|tK5N?5_yq1jH^lhStc6+ zU%TG7)|~Ya?KNoH`fL`Q92Su;@1LRrBJ$r(mr%Tg0w}e;TlnMZc;M`lP=DU;Q>|`o z>tvtv{adN3W+>oK8Vr_pUCqvk-z}u9MApI)((SfTyIr3jgUf9-6CPdLyXn<}5-$~Z zQ1qIR2GqU5aBe+!ZyjF!9d9!4rFomnb`8EvH25-+fu$8ihK8}b|MDv}e)Z5SwZIKQUgk5w z{niuO+js4x-!3QSaq85|2idk**OKooPH=ty^!#7>!y;{*xHkoT(TPvam=}FE`5#X1 z8P&hWGPLQ$4$FUR;Uq*$=X0mO(?PxY|G7=C$^2>Ey?ghUeF8P3Un14H%V6nI0*xst z0ZztGe^PQDpPDuz^IM*5F!A1j3f7Gaz;(&{Gf{8bib#myOZF`Uy$K=_I5s< zbpf!1U`mR+yZ5YFcPX>7yQD;vFAP`r9sgJh-dlp|1czO_{Id{)z1f}PpS!#|8a^*o zc+vvV@p_3L=4v;pvO6W-drCTc{D@P%K| z*+7$wNZ2r8(uD>9LGW>wO2?D?Sm>I^IDFsEl(dN{LL};tVnXn z+TRgI0v7z|bRsfPrAVrt?~>j9M~{q7+Y;U<&nKa!?-%!6%_rwN3Bl5vxw7YOm-{H5 zTgcst-Qmxd=UfC%U>2~l!50?xxiIHAP1wHfMB~Z5s{3td!!O%2+JKnS!raZLF8n~j z?OGi2E*H zy!Xf0zBei*g`P8bh3EOY<-5mchS*j-${6G1WHhe=-M6aFM6lc-i^pV=HD>JC0+jXR zrGjPLzO6P#B=*Zi+Frd#-K%DmrX3d3gPFW+edAnsgLLQdO8&4JCx^GbcJ5eGbH!u| zR{ada4-e(z|EPc2O>R*JlOv)jFl1&;NlUYu$q&xsbaq&o4SKn^*N1ag@~ z(4F>Qqy9)CLP7OK!XDGb@Q_Wj5hE==p3zh;56exyz%|#QPfqkS)yCrcy`ExnI!2)R z+)UTTs-%ei&+mMX?zI(ziT~mXObAkqq>lY~rU&rFYS+?oq7nT81@;G@v{F5QB4y~J z^_L9G0){3)+1%RQOU3Mc+F^xxgLLN^RwZWT=-II1 z7f*kXpsKr8@qfb2^Xako@oKe33>!N1uBa=;e&L|AP5M_^o+e&bok?1po1nzk$h|S^ zs+NaPa2siY&IAl3OGvWIKJCo;DmU)++01DUPE?MR(6$WY1R* z2&DK(=7UO{;yk=AWJ^F5+0w(^hcl{NzMOJ(`V*8vdp}Z#^a>2eh6vF`D zPih4B*~A2g*vv=*HPtaj8xsW3IpEI6nwS4#)`jD{Hwh=a+%r>J;446}StTS&1}0k3 z(Jq@7b2rOnkCj;eU_nbF`If0%%+ZISFsPX3ZGQ4uUJ(&{>J($D@^oFYhtBdV?<#h( zU=HYH9hUa0Vdj}gyk)h6Dk`UAYG^y6TC)c=zTTWR1V%;0SuukgbE@w2Ok*e*?-|#t|2aq|}<->ca2S60?M z9-90piscERu7C-x#5W~joxcVw2$T-Yb#Y#OrZMej5!<&PHvXDheMct`o!6^)K2hU$ z5vb>L@t+ViF9z|TaITQbPQX_t?%yc89U}>>fV?`ViT_BMJb9kqy7x4NqZYYV3tBz{ z75MdP6M^Gsx?{KL1~m75C@>MfvWgVZGW}4+M|H=~P_5ScM{z!TNEz@!LhV~0%KGkd zt(!`TF@1}eRa71nLgsf%0~urHGXPborGv8TKyaz=35RLOjLXi`rcKLCT$LO!0^teG z(3HFQW*4mZn$1~QpOL`J3wUrw(D@D>#QMMmvEHcjbwPIbhlGSg3Do{ut4j!hjZe2L zrY*{w7i;sA66K@atU_H|>X|!(zzy;Q4dy-jtKJBkXenNY6(!+0J_XeSQzbpoos1+S zQ{pY|iYwwvppIS2BVaSYnswPcC1DOr+EB*P*}r{|^lg4-VmxsR1u&tmnPvNe%o%Y= zSRD&)eJ2o>u715M2B3aGe7vWlGv6cGf1tnY zqt>z6vIygNPxvDeiHuHvo`_9Ezz6OYkv08%(^xAg`&uzI9onVCNB&`A_Oi{jl)MwW zHI)`C(|`SEaede{xx!h+a{wE%kJ?!fw3iG_&J|>ISBos|<8%oA%Gx z#e!$?W}aNlF2AjU56ziAUANJIJA0(q(x;54Ayf4UVqcd1oT_-JN$rkd8p^}b2te=B z8iDS{)_Y}zW=l4E-r31^1#vaCH2{?c*&h%WEqVP@+?%#LJ z`fn`WlO34qqxDivtdWt&>a-jHG7)@;+9?o1-`At5)8mD6LrQk+=8JzM`(%I*TO%$e zGQ`M!KqMwK5(lE5v;hO2wkIIs?G*z@0bh7P>bNy`sGzzlVAWR@5!H9DI_X^nJ*xy-xZmx}6!O2R!5D1sv$$L{Byk99uQAw1)e8 zzz%HBySSAlm`O#I?uvMmYhb|&7jQk*Gk<&7TFmQE&)#aN9*^7zoMW80*1Bk=BQQ}n z_XuzgHk%_vxX(ZS3D%1~Mu|!+{K}<94U^+=E`0j@#D}J)Q6-OSh6LSU^ts0+Q*Ba* z6QCBoCUeNIzxXGgV1{uhQ8QPvG`1eBi>9T|EnbPSW4!jj2|`)zq(7 z&jaDY@(_`6{%{QXKCj|M^ADdiZDLbBDBE(w>(Po~CVBnyxz`&di1PKof$KEA-JqLr zBJ6`YQw8*OwL8YyHj zF#{ACMlKpS+q4q=$p`YMH}Ka|9*&h0%M%^v=kfuDs*32wz5%eP5~kWeuF{sPR{rK$ z@fhNHMMz4Us;d+&OD#p$7PDXt>kZw#(vT$uWsXLDXkO+;P>_4Fz2R4xzXq>Lw;~( z=Dqf$);6xBL(uti{*woM=MqtxP{q#>xAW^xq_ljKgPitpl%mQVyvfR@_ww2r=r8Gy+D5Wbq{*i+y3R_ma z<@OJFSI0U#drQfE%7;@eVWDo5HtRpKlj@+;si|tmBwTlzM)7nu; z@D{7C;BHV{o=LYXSVY{vXLIi#e>jLdZwxNEl70L3Ny>z`C=SCk#>aWws7++TV^uCg zBPBnFq9?()2=-2bt*l$d5=`QRpB}F`4=wj8t;6wr-txY2vO=PQo!vBk*AfM&FA=Ke zEzo_}V~w}$2a%L=E!Pb(;&y7U+qA0JJ?HBhSdTwo(xnwMO#RofO|ktJh~ z8zGOrwNA4yPHf0~!kaGG_`dES7t-0&JSMkZz9(;%{+b*HMitN%{_OHiGAr-#TtHb3 z$tGEmKS_F(`$tC+rFe%re@q=K>nl;+CDi?8>gEAIOrTd!vz1elmcfVb7&CuX!LUcG zX_=IqK7D#ReA@sYQdjyuXu?Nz0|lHqzBJuA1nO5%becG42OXD5#7(QgbSzu^#GGfF7k~hj{+b$%hOukZx6>70wJXvms%b)89rgXi@qsgByTT3enlPowN zCBo7|%KLq&u6T=1A#OvJPND4&okHVI9b4aDAe$CFMo+;ss06R(yj81J-4!wyI&}8% zgPo^;c?^?P1uP_A)2t%S-2USfWW}0$hYp7M~-0_uYYT$bOgec_ED733@jrx z-L$57^t`Tb;s=4Ab_l6b2ckLJ4mAWW5oGs{hg~%#_+WDMr{ya>E9?P}GJV&5{rYuT zhK7=S<{ew5WDfh64R!zNtC_l$a&^brn_=#D-f7%<>d?&dnY_zq)e&_4daoX1jh_I* zNsE;YGcPc8ujN$8V=!ljpb&a)s6Ge=Ulr`AtK|Sa1(FqUL6}DJnc0tGuk{#zpXLQZ zLZWcz`~pqC?9&l9QPbq>H|jD-yh$Ooo6a;f@||DR+}4U;_8JIq>`=Y8GeKfY4x%+9 zZ3Vv-;Wg=7EylaJxFmTVB1ys`uxto_MeF>t1f@vr6t%;nLbUEdu6)%|2)IF{>E$f>UyZ5UkXlgO1q5+wFM-}!(9UqFepVmk9?$Uhfw&fRR&So0@wVzVyL#A-w9(C~G z!4553?#m%H$G0mV5f7`g|Gt5twXHwbR)T5)rwi?@3abqVOa8Em?IKboam6Ixg<;R% z8Kc{%&?Y^@o+|!n$Pr0>7NrXs+E3}i(-e zy7sv3)k}v&7@mbaE#m+m%E5xmQ4ipT9#1%Thi;J@SYFM`P9iyRJLlUMO`l_%t+`q=!Te*_py(P(i@}g>@=lj%Htc!*p6cWs zH?DzJh=$3~YPJ7FeNtGe$ACLv_nti&Z-0X8+r?r!VJZJ}IA=24ya7*|FsOx^b@25S zN)OB222jtMp`Y@^zFW79GSP=VhR~E|=*wHTZ7Z6f{Mo!yjAH+$&NjdNgItl3h%K3F znzFcRxGi%C0VBus(WG9xheJAKfR`lSlJVqtV5K)!XF}~HG3DBt`hDMy(1_m4dJo~( z?ufrZI%|=GB`4p-WQsz;sA5g40%9aHXT80fSZY*6q9Xp+I`mH_@w_OW(5^MlcwSuQNsz3A1-cK^ zIXzy|lx;dl$`mY5cx2f`6U{4@dS%c&=1F~d2X35X(cyNy!!g5$4B5=b_ojxHw#F4p zPqf8;u-`GpjT?6m=V#vIUW)c+{wlG+Wr?a1(f=@d=E~=v{JoG|`k{16KZ{K%1rDz* z?#@PG8lh7SLuO%;h`(@Q%QqQ(U2*^)R@0D8id|ehb%wY1=yJy1t+X<=%fHFuG3xr- zJeLb877zAjFD)Lny}M5{?_=P;?jyg_y~Waw3<*S=8cY7evQ?V8?PaX}_Cw9Gev-6f zO4RW-2mG32rtqk{XO9w8eH-?yR;^mKOF$yjylJzLOQT^A?{4ilDb_3=B*f?H)vJRr z7-eP35FbV z(EIWJ(87r5>JR(fU=YIE(R0RpC@9#3?rnx7g?KTeIEYorw1&=T07kETNou@isn>hC z60BY9Ysq?IW}8!J6k?7M0i6~7&uSa)(ZiEEZavUSvFssy^{$1bWlQ}k%jd8Ey`TNU zg>g?4mW(ghp3xPf{6Ms(UYCwmH!rcCaC4;$BBy z@P5ICRW|l~zVDvIjVKUlsb4@nCFs(WW;3lBX@2c7A;~0jPWDC9oPg5ZSs-{@MCMQ& zk9rU=W&e>QZ-DVj*7Vk(S(~}U2Mry1Gs=szX7|!c&mJE~OWUs> zS9(1mve%Pv%7`-!6q5;ZLzn;Pn18W}(dRwb*u`{9^Qk0wZFbxpx(v8~3R3l3v4--D z%=JB*mt4JeZ7^_T){$e!?$Zw$yus(R1@out=(>8i5Sue$jPZ|~^X%{L+Ce!l{SH5v z&3w&syD4Cr*}E6TWiQEJZd9-N)5SK;V!8zm#LhuJB0|KN5}_7@YcwZ$>8#5_;IvHs z0?UzTCc#IxL@qv8>0XeZzmaoqN`}IU=8wA5?cT}G?g)P5;KKw{7hqXFt8s*xIgnsy z^U^5s`?n=tz6q;SKxCZu8!48aa!en8eOF88_KSZ~I7(Mv7n@c}Urb*3fiXOIAx7?s zj?E@cufGTtQ_y@JESc49rwxh8R!D~AjFx`>@a)LbVx#)?Tl3C80G`flNS9EAD}2JS z8rQE~O9fnNkBnkRm_~xf|1Gjt3p{%dbg&3;Ge;3x2Y^!aMN5h>W0y6o2sN|eUAae3G8v(768dbmsLekR*jokUIU3fg}FOXoUhAbs$ z$B(+3ewKDSRp4uRh;mn!G^^^H|M>SIZZep?b=TGD&pKg5eA#BekbYeb@;#FNM1;dY z67Gr&+3I1<=v15Fi;z!5P96}fDI8?T^vr{|O=FkuGn}~K!X)~QLnU)^63wUG-gO59 z_=&mJ_Zk;q66S<9yK3|DIo<`#_EBy1F7H?z-L_60*qr7y6upkczL$}fFr|kc=n@@$ z?AY+bw_a~YW_H8oBGkZ)L7&kM#@t1STDR$~%BxCHZsuQqwY2^I^;FhQo5faF$`z&W zjiq{9Ig_3hpDQY*O^88RmtarEeznFs$el~a|al)EG@brRc4`l&MUa7$Lx==MBQXxn!4hYrgx^=tjKKF23rXcJiZ)-8|Z?TFdHpYnauJwki8S`gb zulnXp<3@FfpEtOm(?bKrU?_}5Yj^;M7(VqWfVISHN|p;F~{um=lwP4D?rPry;<3?(K-uN~M(+L4836v!+BWeyn6-mErYkWnth} z+UE4nV%@xA{LMtv)n5{&EpjvBJI48JWpaNMI~%gc3aM!=vf;^2S*sdv<;-`+%(%CAvRQ}4=&UG`Sr3#%)-y)ke&R7U;cuJxV} z86pbiCY^mWi(+}MZ;l1m=w!o?Cf4%?70@9{umF`%v6%q$pVw?kai?r|GBZx=4QN8D z32CEq2XPH0A6NTb0ue@seSE=yfnVhB;WFnix`*RYL-Q)c~{qd9&RshibwuaVp zN{fkcK*^?g3m0|}&L)+rcuYE^Nz^Y#+}?~C8{`7YX3z~oPcue5AdkACwQ-7@v%CMm zf!>33N}iGCVV?0y&O4X>J_tg^F1VAP{kO7=KnHNjtc>*Z&HOyomIl(`6%qE3%I#oY z+QoYJa3`{mZ2+T=3KbsIH?OoJ5A{?yFLAqoowwRR#;xPjO?S{ywB+F~>H;7|K?r{| zAJGH>AL#`r()W)!NWW`vx6MK8YX!L&9yF9d!2m`Nf%~PK2}E1)s0muwnQ7ifX{P+h*YB4cU0pk(3vuuv%op=kh$lnq>aTu2D;IBzD0^&Ne#27l z5x`=XZr$2fGWnRcYnKUEYWWb%T^(zst!=9p)fXq|Qc#45xy{&W5C-J$XKWs{@mIR? zo}i*xSRZe1+-hllKw)m+4g# z48p=Z=nw}H15-X4eq8dE&3op7WIVp)mW`MV!WLos?VnI962F5!2xMAU&2uRNJqSTrkGirm7uUv<+Q@ zlTM$HyEoY3By_kn<)w0Y`{F^*&v-we&z#{<@9E9gp$w~eJF4M!f}rm(AGHAGsn+9f z^j==8b+l${D3@Mx_c*?x=Q6@L&3T|>IVaS>KYL!!}zAPMn z+_nA{%Mapalr^xOMtzY4Ie8lkB-=_RRRIH5{(0m`deI{{7Z;mWH@4eZ${RC6=}^<@`mmng&Ys`9r1)6s@0Q?%*{E{jRTXME8Uc!+XYD4oWR_vPdc+~8Rz7%rZK(FaoTyA-if*7C2F?% zaZj3_2;jsi`V^CwxDB>~K6D7U@T{W@HnMY#qsd+YE9J}VrNh;u`OjS;*85EyFe`lB zy3i~rn-y7yg4-E2NS{WGq7Q`?Kf7rH%1qvkyY+N6CZBp_cI6QiSXVihu>b)viHux% zeSLzrirPxYLPkMj)VgM(52`~f%@o7ym9F$cHf1k2uS^8hzA2^FpU$Oj{cCg2$E!;Kf47~cjner6FvwBLsGqOUrd29A8cJgxx; zA4ETY6YvBy@il_oE@@;Jd7uA)7P|xL>!0jL*YN|NXfytp%P4;Uy3Ofwa8MW>|kEW_=4`Ge}mym#)}WrqGC-i-9PcMGl`;=H;0ekv(RfnIIHFF0D> z9U&P9QzSW{1{36fuKIC6n~uLP2ejxwt+n?6k5^=!p4Z1%L#Z4+a-<0ZfP=_ccsXQh z+2%7Vad9ylNls5k%&9w;eAbBOGpT!wruLVrUaN3duCXS>XVOAmO2Mp;;B1u6S`FRW z7>33*xFA3RiD2u0H<&}0r18tX5C zQJj5!Ik=Xc1KR=_Pj{%>hd~+F`n95`kBIfZR@z$$?pds)@|h1t3K*_qX#MQjGvdnz_Z5Ax7fBrCauq{(0%gOVzVbt(V{&$V0=qm8;X+ zL8nBbA(kRHbFjeV4Mn@R3S!`Ti4cMNu~G|ED_pMiHU^-4kP-DF;u*if?B400;n362 zakoZdpqKKEw;?De>)A8&o!x!Zsjd=X@Zx{2HS|JK!|a=!E9y`i!>W1a&5wBo=jMh@ z=h{8l=ccRFY1JpK&5LJd6Sc5naR=l6aP((__rv@SgrqCHb9Cmb=5r{64jha9P%hFW z51U~*2Eect8kz$1>2^SvpR_d%<_Q*9QMrue9(TfZlCMy49C*zD4_7HVmtXwT?o3$) zgu?p86QUPyYrbr9pM{i%7!W6u%$xS_&_?Na+0vb2g~~!nVHh5up=6E+yHEE#wQ7^XLp|bpzV~%s1FSRH4&^xj~z2D(Y8SV0Kn49PY?WP>0)ew zdC~(M2ndqTlR09PWWP*|RqV;_P32b8KTq3~HDd7KO%hf!#lvU<`c;?ne%JiIzr{0o zKD&m}H=i3KED$C}dah-VBQmH)bTtWbpY{W5h>s8d#_VkFE!UmNM|dYsFyJg)-X#ow zrd-Z?DSdc{^H?&q5p@&|)FTp$@^CzE0S*bt3FOfl)&&SIVn|B|X$@aSuFu7itRX4} zlB&3W6aeYy4=TX@fol0TY`5xJn`<-lx@iRkjt_rTch&B-Us-*HCwwbFsA&S0l_885 z2m14cSDK@q8UNqu22rge!8YCwE28C+MHlOEoR)prx3X!J9qpZ9dW1xFX{GV89pEmP zP&mE$TY4prZ|@eKKes{ODMYX7-C!yIUE>*Cb`qeuT0_3j|< zx1xZ{?#_zci5^^~B_$ zlzO7|6t&et?3Tq^vM|vw(f%wlV7j5ahp~R+e(Y?N%0+009XQz`p2^8l$W&AV+Se0_ z_6|`+^smju!Tsn6s#?Aa(;?YlUhphzHO@p{dl4x%}dPy9cCOipof)l z^>06?jDdbfHM9>B4KblXlEiV<)0uqZ>yu29Q}OiMU%7a05DHFQ?!?IzRVd77GVr7rdYNYmX)j zqe{q%$%c;PG?##7nn^llU5%9)^gU=qcVUe#V1QW+7zBIx%V;YlKM5=}g*1-vZ1A}+ z!>b3GNS*xt<%aD`bSe?Eg;#0lx~Q)d@m5+o2LP;8C;}H~D_%M984hkNNZ*s9i{wS` zI7<5NyU)wlcY4ck1mois|Rq||=1h##?6!!?XdiwxgM!}vq zfYLtM3x)U5;)mqL@ZpvK{(#A#a~@5sZr{4{t76i^>UPtXgFBgJS0|?(Wjni2M$2Q{ zNSPbRx-2%p{?1GV^*HknUNmcf!=PQl~6rtNVX1mq}M8KD?HR+?T zp=g|DH);$X#;si1(;ZeW_p{y|wsq^llABkaD$1gti}xO;w-83lv{pzK0*L)lQFaAQ zZ5E9LXr((e7z#uDgJ82+v-nazAN%=v1!0kvx*dd?aWeSXu9c;}c^7$fz{U=@ESnJ^ zEAsza_PScpF)_cupd2nCDO4-bm?kOBQm-e$j9KZoZf!w-Doy30(?YH-=+T7c%C=oa z{GxyM<~7R$nN2-G!Sv!bvJo&f&&2}`LZoH$+muxRxIAlU~JoQtA*P*e~!4-f_D?-=T=(Wpc(c8cTz!MMivV$ zjml+iS1ezm`xZjL#@9R^WmLL!O-!yIO8fA~f)a_D-HI~UZqBp#m=dhIa)NDY{$#RO zp5?M1Ny{~VU`tV+HwF$F-tRQXL)2LQsTcbC6yRJBvh2cu&P)?f#ck?a-25m{?y!Z+ zITHjd>exyK$3_%5V20h+e?;zZ;4V*WS$RR*-jfyW`ry0r|3}wVLc<~>JMwyr@qabB zj$Pok45J-MjoF&^e{a;KH(#%l&9<#IAT#NAVu=3O92prI^x=xl&Fq4C6BCKw&wS3? zAK?)R)hvCvU1685T}=`AHVeP>8$Z^9_m5*r5Wsn%*wx5sNU@p^W{_tc3GgK)YUt1L zKn*Xc5e`0IK0c2nb5TBMdrY50M57{ze)YUBb#--+`Uscywa2)}9BtGXx((9NazuapY|aFkHRmyoD+Z z`RoqClY`K~CgsHO{vzhkDr1$oS;lx`3{wQ<;x9Gx9>Jq@AB0yg4NaQQVx2C{oA+f; zZ97I0y5FE?Qe~?!q>B%$FeJ?x!>1nopsPgP+hx|Lx`wixB$Wf0`3Lf*?r*E7qdq~B z&Q%bPsb(-Q9~WD=E6~<~E`~GVPk}vJ#Vq{taWZ^Zroi}?7TSN)8MW5PY|PjJ2sY_f zX-}P-Umq39bf*y%y_uE9@^^s!|Jiov#!h-yVn?nm+f|p&Qd|WDEcJ}4ICSjTxHstC zSc#yU&pVp0UtBqtms#ahX7K-;7I$*?^DBya_nHe#ZDvcUb)DU{m8dOc=eMI)L_B+^ zUYp=&i2=^ySQ1B~CqYwn3H}1-b8ojBJc35O@BzP+(6F8AST5rB_T>0Pe8X*MmN8Dl zt{pGVIyRKGO8^Hw`NU~&BV``4yfAh^PQJv?ZTeSSXj4hkg_vnG>CC^HxmGi=suj96Y*(<* zGs@(4q#01?wn!xMeq_6&Tj=#CJwlj`0;Fj()Jc_n5Dv5DM4qMku|2(TX}v?%Sv~ydZr)D_>Z}yosgwL3ARvy^IzX?Dgm2Wa3tMo9t0_I6d6ckg-BkI#T{Y6tWfAErg56(c9lXg3xzTU7IPde(OqiSX565| zW&~ZD3->QyXc!V*BH^0bQ71^n_2u{ALft2;vAzDj`n$iThVnsV5=+Yq4n`FF@M0H{ zFz7%$=mAeVckzICbW1!c$6XyJ)mB%hs@09?`Zua@x!WuKGvo&Yuv|*~nZApg$Qg_9 z&^bjWA!Si?ocR@1j2{FjW{*G<37;F4UeH`UowI3>`eS_Ys2X2%ThY;67>sj+RMx=6ZCf>Xr*WTwi)gU65uyA`f$V}q zUoPO0o?M%@(>3ZSm#*ab3EDw-vtB8`Y^j$%EeXvz(0@q8;ziMm&rRNMwa0?r)Wf8{ z65PDla!K)tisk3H1iw^%t2hHU1Tv-bju$WgIP{X~eAHDqZsldND@5AfVLk~W$)*3$ z@=3p(<(#;*XGSl&o;To9@VxEBq1bzJzlU6xG11X|3&erlX4sDX4F2wiBsk{7+UHN( zUB1KkFIpiPE5C7G@^I__1^TPGkE9%{bwqiun|n@)+BDg)2t9Qw-_87qJP2t%$rx^K zWYB}rUFCZcuvs~=aY?uYWs?UcNV=n#lB}#OsR&czN&`F}!H~XW<-)KMk*^Ci_UV5}p;sG#YKC9eX{+|#QQbAfoRcIieg0^YiXHTIBPdth6ZB*~KNE|wc z4;>om&mqzatrn!~(dU(DRjB6P4qAE};JF{M6NJ9nw357m#l!=2iO~#7X1zh|!a~=W zt-E)d%e=}NqbXK_SFDW_&qCa!lN)*7r*3nZbU0;sJ-ms7unA;1u}mRrG><07AtTQ( zl`pIK>U@*V4xi3NoQ`rqq<;ppO_j!nBBGT>byiAlAU=JGMDf(o`5WmucsupJ6nBOL z<0zWc?L_l~0XyYbFaDtzOy{DQI_>%!cr^Pzy0nbw>k4fnEn8m2feRO&6eSz=vnP^? z47c_P$N-0NO+~lk=6|=~liys(%>!LB$w?W@$=Z_55z@3?+Q4plCkwy)s_W93qyu?z zOKz?V4$Ah=eBGyJuIB2WUtKGr^Ob`#dPnZ$KI4foIBZ%2XkdH87gqWg3K5^Tm3iJ@ z&mLLWN}4)CUNR~lL-ZeYI@N{j2$t?$Iq~#?V8bU$OaCw9t^ZIi*W){OG*aJ<$dEqM z5d+Echk|SL7NR2{Z9pAJ0QUHdw`6k>ly$V5AOM{px{wB0dn_lfI6Cpnny!V=_U-D!s>=+wL*L5s*37(B~tZgU=xr79? zuid?s)t*vcfB#WmJp)UI@Z~|7olWnVrgr-Jw3>lYn8!P-PFt=SXecYCS;LVLo((_= zW%bMG7A0G6O+BUO^RLcm4Q#Sr<8DI>wyzrA@++C3Bb^??L#MLGosT-2lHvQO!)oLcyI$(p&wdd=R&8?OxV{NJaVq;<=FUHK8LSAM%D zCte4+82J{>8nmxd0m%*9tF_lqnn&}gicS*4sb6yTcTB*l#E$l@4X}>J6Dm+8E}wtz z%SdQwxd!tVUZpcBw>d%J++!q8W)C!hd*B2$$=jMFk14%#F?yp-Vm^Ae;QmQXgU9?e zUc|UN_<-7kkqJ(f?thlSbTOE19;lu}4sRMLacdw3&mIjf_rsenY7z&FPMt=f8~>vU z&t)x!u&8%q7l{6;wS~p;5ryokG8cJZ9{(mnQs;_R*?#Q{oYW&kx|#+|-l7qdGMSZ; zmKG+XCe31=)dICLxL11ipDA*|maw>>{>P)EiPhuxbtT}m*~ZA_0%q-3jwh1p}v(GhGE zfCuBVp!CT-A>`zgBoHjvxkn?V%Q?skwNlcT#TeJC*HQwJ@)p~gjH%KcG!(9P{GSuO z7Pk>_bvc&d)`GLEeJ!N8^tH!x4qmw6oXGYO8CzIoGVyH2&h8^5fMf8GA&q^f@&}`A zMZn1Bro?&k=C!Nj14)74Y4Pi?H_fzys;f8YWg1*=(he2-+=PA$KJdHg@YK~_xkf=B zBkH`}Si`gvdQYXrATn7X$lE1QaH`KrF$}ql^~AjU;PKS;q)YoWv22< z7!i~xeMiaqGS@u{a``Q&&CCS=yFf%ibed|E2nq#+_TFsAFf_~q}lb?aU6@2IH2lC1Yp zbrgbA*eM2=NVYLi3uIhehf12F#qZBNXk}~bHrmSMqYS#j#DRPcBBewQx?=Z*sxMn! z(mN-jsEVCekmZ^*g1_zSd!N9~{HPPpqNl^|cCm$QhAGwMf!Xbp3XaF!-KlgIMMFSl z)P(oK^^|^Ea4%})6~}c~wW-%kc)ljOk7jjq>S5z5=XQ+Oqkqhv5;WvTD&5FtI)q)u z`sDOk|9hq}>lUo8-m-aF3%(;vM1hcN%|2$13XbF#*^X{Is~52ZG>QW`3Gt#$>SlYQ zhRGt1gy|=g#O7;vX@*Yu_OyGT1(9^v@khP+{Q2|hKHa)mxb~R!(6T`z?I25$U_Z9h z!=>Leb4hmm{Pdz`NW6H!bnj^AulpSx{R83)kkl*TTr?KvfYFw*1Zzt+Td+KFkZ3Fh zBa*KlaF_%xBkRnX3qL&Fgu~aNfJKDqa;u`w=2i{r2m!JQ(4yiDhy!5-(}!wnhh)6o zZ0>w^5aOjQVy+@60)t#!cB$+nX4R86En~sZs2C=GvWrhLlnOq47+21mUqsm8AQ8=Y z+T{{~mm=FHOOUL3v9mxx~`pvI4z^>e=3DFqpHYhyZbKv-Zk9o-WA$0Ar))hUgNTa53sCp1T^ z&P+Jrv=d@5fM;z~Pb2fvSIYy0YlLgtxe?(%c<9;~Q!$mFMmpcH~}R%iPk%PaTJk5 z2pj^A-(Mz@3URk3mc45HPubG6vm|9ZvD)CMbKHiUc|6+8H{88N*yt#mTeFM_L#`?z z_4WZZ1UCaN(w!!_5hvzUmF8Zr4Pz;%lp35I)h>NQy`8rXh;yo6@6QiC(l&Pb_i07j zLfW>xbLV?}r|Zm^Q~W_qhw>VGog7>IaYp|TK7j4Lmz>S&MYaUPi?@TeUG?HAtpI;X zPDBoo*$_l<0YU zM1IUT5TysIAMr^nyY828eE;S59VcEJ$_g0Gb$Ymy5$8=ABLxt zPflA97qimxt*kFVu2P4+IjLw9WI&0SEh6TvuKvGIJ7KUuimocZ_&-G#&TWTaNd=GI zFFG`< zv}OKh4JA|vwCC%pTBU^xp7Ej}E$`N%&&2EJJ?7m|f`S?qZrJp0(HuX5t2b}lXhI@| zqdy57VN7bnD7+93()g7Cj;RN<1=_LVDJ83$kg&!&iNhs=q!2)9-t4sQ-o+Zu$|Vw0 zCmbfSDh74#2*co3#nTeY;+hj;C-RhIhY#1eV%a6;&J9j3sdrl<%bL_xOfGJ#p0RUP zcIhujMDmvGKXKwj4r)O=6Tx^`m^VeSAkS%A=)Ida`~pxqw9Z*p@n;quOe>Tl8VRt7 z%je-TNjK0e?%Mm1NVG5|jrOM|#x5=M6VNt(EWQW83Nx*1b&7zOI#5eT`NN$Fo8q^p zUeE~!Zqrq)J(tKq!R)2gZ4i2+B^_2656$VU#PdcH?Fb2nVQD1+l)uQyirLDihF0#L zN|Sx%of3)XHBs|d0k_#c-)6%!Lz3F7)tWHtSz>h-rW{zWS*D6FxPj!P^9Ri{{2MfA zkcMU1M3}uW4ZG>+QG$H0s(n#^%_+Q?# zau&r@@5z}yOwtYk1zD}rHTgSi)-AqskIQP<;3#KM8%)Ck1qV!}g@9VIyo+8T=JhB4 zuB|jOjq$mMk)p&|Y8{jVq8nb`H>V~G32$D}d-C|Ppx2`t1$@X(@ODbEBnVY{qLD=6 zJ3Ip{E-WP?W!xLEexsg#o=G|SN|2qU#YAFc=A{SVPu1>Kd~zb_ZBTY0me6pzsF*mQ z_1`y4z0*gr`*;W;f@JIbzV^kVr>o8{DLtkhh96gF*@o&tx~L^%Vq+bJRQ*N7z5SDu zoAy~cK}8KA#Z?arEH5mKyKfVlMVUpjWQk*>rD8Byga!acXLjv7{b*J!wc6?8qLUA2 z9o?|zV-l9bdKAGc zAjBrrd)V84QdvJ>pe;he;B-HBdzTqu+_n)pM-@w1+ifts>cSjUCq}(6Eq~6MG|DmT zL)eRjHlfQpDca{}TyY8e@b|`M>~f%H@(HCeU4AHqbLhZSq_;y)^a>5B#37Scr&wss z2c0w%&}1{(g5_nsP=O2|T~~W0lSR*^Urx}9XBT4fPnYQDPW6ML0l`~CLVaEKy)yE~ zmH9>{V;;U=0013^CL*Vh<ZRic4-Vu?Z2^bi5Qw|fiY`rfwRrLa1|g*qXCcBG!*j@| z(l8pBspc86IySGP=tVM`$SahsCBWuzr*^N{! z8_(iDeS2J%v;)njD)6;el<(JH7yo|aMZu>)UcVUDhoc45bNjmM>6+npDv2P}sjAPb zfByPLqMENxu(TQe$Bv6xcRXy$uPFxm|K2;x%64tz5)_#ph$;NoNal5?9IcEf8LyAY zUn^&=xBuzbkA7pv)?dq8<$vv>;bhdVr{m&=;NU*cKQpypUK)D#m#{-gITyflT`nvb zMy^pzC!?Tf?#`>=!UG51sEk~kY!s9`jv0f!yGd>f=gmu@eKu;f+O<>QD;O*206S4f zQ6X@BZm&pSN$hrp(CcIDNqu!aY&4aSN{gO7oem>mD*x0+6ic7pjE%W5-y4yOoCQd3 zX$|f6*yQ)P^?m`b$<396`9#c^hh}wiPRhacW=HEEQM_IN|19k!F5>}&xra-9|Ieb} z#=iHHI_x5N18DTWtZ3mBH78ct*1Qq{a2hbxsy+saUM#TygBC6rf_;ROOP6pAZz<&I-EDdSqgda91G4ljE$7;t!q;uQ zmE@LSTCxLHghqwL<)zCOg99suqrwW`v#fkDkvoN@qI%vk-u2G5pOQxsA&t)T*CZll z8~vr^A0doI-ZZOShbh3)pmnO>@3*qfY)P7dOvWFQHU;hchX-7PQw#_6H@`lf8T>B8 zI9?DXWXTA{W#oo2dq9OHdIF8)+ZsB0Rxs(9 z>FWLfL;l-&$MI)`HQVCKy?T_bfzfYmTrMseCEDWlZ&y^dr$rSVxKX!R52Id+slEIS z@s>phjh)InPrbR3Mxp;*>pse5d$1q&!Kd*&uf6@WhTW!L1yylNj_hdqd*P;)=Igpw zH&8dBkSu-K!i{g$Kiu^2@YD@2`ZWm3?9!r-!-4(QLK-Dcp)o^;f3Sqd*)^tb;Lkt* z+-^RXn4Ei+kjK9f@!X0F1>D0BYjX+$g99ZZ*hgoJ=qGukpY^@styMrtCH>R>yDT5r z%1(x1GYlMV)?D3Z*7Np??!s5mAsm`4QeE1vYx8QZJqPl~!=BDCyCiqYnb0ZCeE00x zBipsLe%Dd#?08KEp%p|!pp{(McFz6viafrH!Wu(ZV2!X1%(!=PL_wIhGO!N zrsfUstzFW#&^v5uNy`2&laGVuL?9E5W#w|KCXjCKt&87ngwS&W@d3^fI-G zn1dl4UZjng(`U@sMCh++0z;}3t~-@CeEuh@xIUF7qq+9gb|h@)&i{w4|BmaqfB*mS zN69*cjFYUS(^h1!mb0v!NM=TbvXbl(4Lh5Vt&&c=>=Y?P5-KaFQC2A-^h_$%_j){X zp6}P^^85Yqx}2}q`J(Z>KOV>ZIBx53TjGV!M#W8$KqN}Y-V%0~$t`fFA+=2L9l{EM z9k5Jei=Z;MGrY|Wuk5>(4dLw(jXrAfHdwkgJ&RucXBk1#YZ)Z`+*7X6P@=^Gb}4wtkBL1x z2iAuS@y7rn5ViHqDd!y4JV)wSkZmrH7RPdDDOw~8!$L~`xK>F;~ zqEaL}-$abjrhR*Z0k}sht!BU0_>j&{H&%+5xfS+odrxMojN&GWAxQO3(X~cnxQD$j z6^cm=As|CLykV?QI!5nprlVYcLN5#nlpap#T86_=#maecw4=P~hYuaPe%2koo~Wg4 zN7T)!WV8(U+Q}<_Cx0R|hGt(yB7XyQTvp<#JINv!m(5I31w5b-c%$wue4bwVJvRRp z?$cHxYIbF4Te-IZN0GIGNW27o=&9L=n(QP?<#RrJi(n**^Uc!jWQ-76$VRxsF8`im zId>RzW5w$^wHw{MS3&vHdi;*T^>nv9ly*4%zn}s!if~+}DgvB_9_WIB<_RJL(=m{yo&kNxZY*x&fDZeBOf! zKIO%Wnc4Ran(uqtNU^%z$tPk@Zg%!2;QO?>=F>238GpfGXe;*Y@H|J_otEXgaonN+PC}BxN zE^BnOeN9D1=~Gsmu3ALWY2ZMs#2fq0tkLx}X!H!!rnVl`dwXI!$^q+d#UPp5Q!~e_^8Wlaqu+_7T%VfE^`?8ofHGM~(5vzf;Sh|U~`0P!ApFRxvHJ^MDl(ato)2!|P? z0J5#GiBDw7?P|WsTd6k@IKCRgJ(6MIsnKNd1zM5S;oAveKk23T6 z!8*Ab%z|}#loKP$SazL+#?RKK7EMh$ctP0KwH8q3T9YjLb#|(@Y=^k1#R1!n9+iWu z(FGIJN(u2%!BJ3Di066u10)D$r?`OxIEKZ7wO4ei#iV=mthAqmO?Kc7Ugvw2o>uV0 z1ZrNn8-O?oRda(1j)i_xtm&1@w?vkv_baP`=WK z*XQ>8uMfW>eR#msaJuje76#P0T;k*w;Dr5Jz%k3&o5Hyo3bE04;H%W$x2f4%PN+%^ zocSoWIsloTo4J%(9^-=H%)iLNm+`0fn(O5hX6XKLdcr^@morc{ZM|thBzI!S0$Xo; z(K~+G_ig(j*SW`s3)bQ()Ofj#9T9}2t+JsEN0+KtHP3nt3UC)h#dX@IC4Ee6{${GQ z6;W}!4jo3K3VT?Q@^N0`-1vPSYz`1W1hDjh%)2 z?L_$79I?}st3#4sym*ll6HU7yY?^uA1vgr3TRKSTJ+tX)I|enOM8V;PLhxpj{Zvqt zTm8mX4yUa-_Vq26rHc*UVEt)*H#M{TrnNv8DMPnl{qR#8-IlWIsaE9NjP)vCh+>1% ze3c(^8>mqt|Jmuyi!NS_H}lUftmgaWG0s{T?-8Dyfd49VSnCu+vk8%N`=)snOb`=8 z5h(mw;$lC0+1QvuF9C2Tc3#awPk*`yneUbBqak2{ZS1lM7QF+InH!$XV!C&P%I}m5 z0Zo%}Zbw+h!YyS#YT`WC_g4`#4-oeBWqF%23c}Qw4Ogy$h6J&`It|i83ed}rb)l6O zXGO}^l`0j>f$iqMfbA?Xdi--#XHKt~Q6Wlfg`Q2iIdtpQXJ$8xBM3P{0BHY>H$Y9s zg8M_by^jGtPIstV5tB{Aojdld8tgTApx$5H>Bd|U*;0p@)GALy+ur?4(%%& z&POfPM$DEHRy@2RVA%U;&Od9KL0c*`#b%>{`Pgh4->j;gSCJGT@BW6{PaA^-uIu^ya5+Wa#wNkS0({Di!dje>>j(=qmGTEBiPVWBD1`NDtaTx6N2)( z%>rPP=c8agq*QGEO{p}Rx(pYbwYiy5%*mW@Pnl!p_l9fKk(Eij!Vi9#rsvykoO8R* zA8#0sd@FlSvboZ{UvD6AC1|Kz2y$dV%L%NY!P4jS?%g}ThuI=RmQwAvN=IVM{t2XniC(izx$$=5+f;uB)NlqYVfk1ott61*-J|?`j$8+sPNeY zy5fVTR!z&9sB>mDoA4QzXMr>yoS-;Q`~Q=1@R~rRe;Ar(zKnW2H|FQjVIl7}bx+&M z?>RWzH~29(wCVum8kJW}kapf*G4+X*^6gfz7+dIPz9WGA_L$QAaJ)fN3yXPV?=&s@ z(I52fgD zAJW4_H#A%SWhw=n?KRmip(jP^+DZ>^{@lnN>|vf}tBvyrX;2ZD^aB!*+f-k%r!X(= zCz9Q5x6R(21{BACb1EF7_*M~0>Jb=bbg%AY+Zv)i94-vQ@{P>F;i!;X ze%$g>a@QCyj=h|+;nn( z)>JBy-Qy z!_Y;8+AI2P#gayL=&1${mR{6R^0&~6Z^IBdpPl7lZuLs zO}+ke5;^|`a<(z%{|FGtsADTY{RUb_x+rtg*5>LdzS4le2ui6je5LY_J$puu_7NW$Sgzj#i4rG*)v7O$ zhu!}m;5Xn6c?gUP76FZJ%sZnfed*Nf>({Mo>fE@G2fD1e%$=e2f%EC&a`W|mC)F%9 z+-a9^%+YA>#vJ^rP3nP|jMA<+FsRM?ryCQGWKVqi*hKF$Pq=I0OUZB91MOcv=O*wP z;^>v|t5HQFrWQM)fI_N+@A&aM-y&q>7k~p)#UT|NIgvX2r2M6c4)OPCrWiH_g`xJx zChvtY5x(G`Mf3^yu!B;=r_y9OmnW9Is{f0Tbl$4}c5ADCaXA3PMn>P_{1y+~xKJ`n zVtxPG-(#3$d!=#bF!wPdr8Okaxkux&A`|Lw2BiTkNBI-!*m?>im!;hdoAx{;svi*S z!JZr~d5 zEii%vB1ui9XTtY z@Psz-eP2N_+=>u${C1=#pq7iJg3l#sCbT;i3RReHqxd?{7hSw~e#$xKtOG(eh@o2K zkPC92eC@4g759{^82fmFy`Y!~l=3yD7_}5#&v=%Fo^=O(kW!3Y;aSsdBBo#X`Nc8S z=EB6L%0O0yGr~E7z=o@sscbwHy_q;mU`8(de2GZoon`4;%jWIcxg9;kbFD-_L0XK$ zVH-ekYcCA5vXI;LRBxrqRLT%Szh3W(FZ?s}+oRz_ftN=FmWoATg>3o-1#y~PzcXLs zOBt>7ou`L)_m9z2Dy2L2JJ2a0=+6&2W(#^C&zAe9J82& zHwhW{SFC1ttQpy{rnPRXH513&ujneo5Q{)>n!O`ED!aQ=I>rkLl~7Cc=8BYPNuyZ= zoL^~m;Zx(DJqx$_%wxl;bi@z#LE;jlrDF;}uzV3mH{HWyv}gGsypDG;kx%xphQ84S zp)&oYIx>Ywg68HeTO{E@daUbA1`?qeP%0N5yk07sTh<88rY0ZJVkdoDR$mG7pqu?8 z_e$R8j0RCOdJ;nehBm{tQWlXp;^_{US_Ry;4efQg9OQ@uB87crVbmS4vv=gs`(uJg z`PLo5h?)w>PLDPqvev&tb}3mGV4_u01YJzF3}fv1?X5l6r>$GJ-U<*D8iB?e z1Pwqk;%;|Wa&(syB{UCzI&raE}sl(@SfLo&jLAd5vl5Pk?_p#*wnIH_wHqxIpoame3m^Q z-QJL($z1vafSWEX>!3w)J{RhaXP*}Ghb~{foWYcF;paVC$a-1RKzzq@5caKFv~ajT zuAe`aEOAgD)6g%9h-PBGq5x^_qWNtb%{i{W@tP^{hdUkj>7Vy&)pEGHEb=e-)T|0x zzLoxUYGQ)}zY{+4bujn3*!&W^UPQIC1k%Mt>?kX=8Xh2Uey+#k!vnXwa2c1swMSsu z3bA1CkYXUAML@h6{3jrW*x!vJ+B$!-iS*Yx8^Brx5ql>9i6vq)K{{j|1ih4I99D>@ zpR7B#>d0!bv+Wly>#qF( zwjh}$-S^RuCL(R>+Tu@-Jxx1lJfp?B%^YCpvU}~2)W?sf+pchySosF`ulkgEqfdKM zr;1gq`Lt=Upeb8*tb%x(A)L#FCOml9wKv`k%-21m?dzn=o=EV89S4`Ro^;7Hu=4tK z>(l^QCYz|;YhUcEJb(9|wZCs{YI(5PTD&nenp&KE#A7U!gWed%R|$|o8U4+p^xpCc zl@G(VW}5&7=MwNZz|?62^>^LV)la?qSSvWA=P`oP)X@j!BiCOM0$;JF z>~sYEI!UN8kPe^Wd<1Qk4;B;Tu|wX-vlhEJP>CZFWgS4gb%XLZQoLADmSaLrV=$)mG%fW=WdEK(={58zN_7W ze*OBz)mWlM zbQrOrs!VRzzJ2ZLwERnd$19k6nYWm<@xTxEPoQ3o~^_zCnwN`_` z`kZbEjf+*aT{INAUuq8WC99zSle?E^{)x;EVYKIU!`6E&bMB>ARP++=*33i|J;S!s zCCL}s+iawww+BDN&L}R>;PB}G*C|%Rz7sippFv`Y9Pm=3-X>nx*~FMwrE%UI2iC8~ z-Lm*u7%DFL22s-}|1>GO9f?C)h1^b3pFJ~+Cm7v?VSmSl7uyaPy(?6E;e z;DV?E4w}Y#ToS2}1ZMtBfk_k5>(BdIdc#EZEK{Z@N4-@b5)i5R61hjNf<(^*xFh0_ zXp0&}R#t{%sJvr|WZIb~wU=sKUVED|AJK~yR!E&r=)qZ8Leo-};88cMYl#FeYZs4i zAzfVt?XN9-c!KDng^H)>u1Mk`hw1%oK8u@=H4I)!eqr9on?6@aaXay9} zYvA)x9^aw6`?y$C{uN7Obwp@~&mTW7%kt97TwW3EzA;XkHTw}P^J}(_#$fd;yj0N{ z^~=t=wnolXNIzG@5+3FjfBZ4|G*FJ79h{_a{LYc#J!=!1bQ^V~+Ts#VHum$U3WNpk zn)p_XWdu*VK+nDoGtMPh zZ0F3#OGJyR!c07FYg~N2=@r!6Pga#nCbb9#04cplI);8--T!KPqM3!)m_ZQ!)I>vd#g=oK>hK~Cx^I$pu8-csm z-*C4Cs7YXB8V1l%9)IqWw9U%2)3* z>d9;)S6W7?2VTcLg$>7p@jNw}sWrCzHHpmde!+canoOf*C!xp_h|H44^no&pPg1O0 ziAy#k2bD=wn1ckxY75NKKyfmRyEMC5;94cQzo=`(0JLDi1=r^Ufj5Ku)w_J)f>y0P zARD;Up3Mzg0wN(HSgKU-hXl?``1_V+6GSha|72XJiPu*(8z71cjnlv3e>rZkO*PicQdcD;`lw}oY6B=57tgKJg)xAN}DV6)R5zEZ5m?Lq6eMeaF+ z2)fGfd7US>=-wtXrGt)QsLRe3Te!m`8o8+|J+9nw-@Pdfv6EXJ@$8Q^QIZ$z zp`F*0;nc-%^X7}X?I42AUI`PILgR^7yWt#j?yP%buGd_AnVlxZRS{^eidkOAJ7qu&?ePd28d%i&mqr-zH(G=%D6iB{xG{+m4UMmig0tI*ZyQI`uaR zM~GrOV5gvldW9K*J1K~5RHrYh8zxlkn?LQgFL~O++mUl|@yg$TT&gUVHKo)?7rS$m-vNLkw2y~02rRC@0E9J=`Ji_(_#N1t#+gNjtnO>)=UG|+1deIxU(Euy#~h+!a8XqtYgS@U4VAUaKp z_WsKtpl2SRSDF7m*$p)Sdk2Ox1&3%}&Wezurb?=Az}Z}YO8`K`EgWH1U}ywk)=q>X zmOo$DB#V7_gLUvr4gK>C5m`8!{wl~_4jM_5Y;bMXxrDthY_Mt1@2{oT^1*}}T)?r_ zOBG+g?U<3}lyhaxg7~p(8MsRNVf`a%%ZC2~ChAIDC-3qF-6$>Z3*raWjo70a=f%M9IsNs=pm}FZxJyigTox7 zXZ$mZHjyXwTY6bZdM8pf24a&!qgr!uj3E8Yr8R$ec}ljF_9k^6vdaBZx3zqmYD3me zc(Qq5?OiLk-&ZQ#$~c`cZ?b<}_N%Rp>0$rhY=u7oU{h+b3(w@r{j7P4Wu=cqOYQe_@OuJ|%fQy2`_{HhRtNLlr`c9is#Ij3 zi9a|AK9~M=ldqMV=~b&D zhn%_KFWzwDoO#MO-D-->ZzyCfOElYmW)Jo8=R!35(p7{a5*o)S+@f2z@7_Ml+!cy` zZUjdoU^%}yw^%*z^7+>)_v;CHtojfj{IZro>zmnIkIbvHdXNLL$t{hBzHec>{^8f# zsoEH^0hV3qFKu(lk&#G3-te1dx}R}22dv&en(a>Ueh2e%KHO0$*qQF|j?r#z5(&#^ zh|%5gf2qmWoDD2w3|_gcmV<$&r)a5~Yx%z7DtB#=8?WwG zTfb*ZGaP4<)_)pQDyRpYx;-Lm1XE9B0`4QwE|^AkNpVM}QR!M;m(sawV?6e(qn>xtmGQt6jt$Q{1$qjL_ky#<`aF(2u=_Oy>Z#HTa2Z+0pc#W zDzy@6=tvOW>BxACMpTlIgJ}i|Nk9>QQ-HgGtq-r(@YBnI}E@;@+(PA_^Gih}> zZSbC$n)yRH8hl?wo*-g0Agoy5{~M6?bt!ggb{YH{`dU*{pZop$_up{in)L+5d5Xb` zAE*23SP#Yz@7hc!FR~eZ8g)vgd@)Ahk6E!oJEN2vs1kdpk4VN$Hh~I*u#E)RjXDNFAzUEk!(xui8?41nCmy`*VC6~y3Knn4!*NsZ0g>CV@D3YxzbK~)qTPI z`D4PK?+MIN=J%NW%2#?~Tebt=rf9Sv6$tNXa_*0>Zv)<&)HfL$cc!Ug_9rO10Ag4{ zUH%!xiVJDxcI!XU(Xp{1K&hFU(~S$GcGIM1mpRr78TpAwfX*cO)Nd^#vtD^KUUaKmT<2WxV@>m+O*8$EO*giv9^|q9!TwI> zFY~kN^LlI1A%r5`$ZlfpIRd43mloGONKFkO$~JVWf8LZQl~sXuIdts#RNS+z8QFjBaku=g(K2pg_e2d8_=y zPmhgw?o}2!7}Ht0Pg(R?a}$$Ixi-L-$HfxG>jnXmTxUuZ3HgF;3)Y}CQPo(JCwC2% z_Z_-?L(==pPO`s*g$B25Vml=50mvJdEjnQ;a7J1?#ZO$8>LVk2jT4KQF-Ov%N@}3k z%M&m5-B2%q0!V7$bZ;x0cT(MbMkDn%LHVxN{qaFb74e^uvY9E?YN|Gd3u9lE^`CG%eR=a;u!R4g_W;PA&l{xQ(3-KC!f+~ zEa3KhxAYl5@x_@Afm3?ig27n-dppmPqH7|}Dk%@73(CEe$hT7ZB&tPT!YBQ=RqfQP z+1N#Om1bf)Hg2#d`N)O>I`_-6r>o|5lAp)a>-Xmu$OyX+;`LdHE%nDD^y;=RpFV93 zNskh~?xmNr#LptS*sbU=i9&TY+&?6LqG9>kgKLbGI>4WiPL0+yL+SKrca>gC=)D$3 zEm=MT?j2LczXe@s(8@OdV!GL>G>B)HqGI7={U02zbDY)FTXU8?XdX2xw zUsX)? zzyDeGx*CCgD(7~iAH*vw|PuF=LP<`vGG6aVF0 z!`PLPlf4V~&+hqSKX`Yadd6c6YsAOQs_|ct#1~Uws7?%Gp`+0{3$gqZW9cAKW_GT5 zqS9SSO8}x*gJI{w=z@D_b`@%+K&T0AiEzMJcx5k~1~LmDG)=y9#aXE~Q7iVwp0&;P zu5Gc#D6ePWua$n+nyWTAEPr6BOfJVYasinBjv@_Ry6QES)J(N8*I`G&j9Ifr&0y4^ zbU+&2G~HM%Ogq-%CLb2Vb=Y7CkzE+(d0iuCtJ!v!7lo^d&17)*%1R|*MQ5$Fd_$lZ zx#HNj^Oe`QCBCj>;%18BHEv^K@)){fMLg`e@ye4z?eHP!#0Xn0xsin*{~`f8)p4o1 z(S&h*zmQuHB_loIT(+uv4|M22iqth2r1zL4*_E`#!b!`vy4er`+I0Uy6(jpNROLDG zV()l_Cf(!cMT{VZN@1~SXda$rj!%?qH@OG|oroE`uDv5!*QBx&c;?=P7 zH(ggZ<}fe1-Xm0!XJ5IV9te_Jt#H=54ZnqLn5k$sY&e_1{~YL-nM}aWTB?UXY+37Q zJ`!;e|84U_lErMjP(&vl@?or_aT|0DIKNAaKCM|Fd5{-2Jsf>3l{TxI4&K}U4AG{$ zkqWl}AG1v|JFmU%8tLu%E3X1f#F*B!##Q;Z>(`xj{JyRYeoV7y#q|D{M9yM>l(zCt zNi6)NZZRLtF$@k4^_EV))P@{`Uuk(~QT`>MlT%~vh{$kl9RfyUD%k{|=J#p=7dLcw zvt;G~sqYU^N~oz=B}H+x-Sw@`3p zfVKE!HSCQN^4s-l9cE7u4PP^%DnCTr0BDT)Lfd5!iRySE|Hb< zMvJtlEYGKOi)@!*_d|t?xw4h8U%y+{5gi=P+5q!uUyQNB@Z`Hz*2$6ZHT&mY`7V`O zKhD^?pm5;z$Qeq^`imP*CJyx!`;)I@5bkVVlq0b%g%PwL4?i%uVUGC=&o|xn{@-+^RFfCHWUTI z#TPXzlBt3GS!Q2ZvAm4pr|v-Ds%+zusb8X$ z4$j0KsRrraEm)y#9gF}`P4O~6DmVy4_J(`6tZFMg2sqA|cUB$)eWl7Is*9-MoqS3z zFfz?#q4!2f7U|P_Yt7{8WD%EZ4?8%uM~_!deuc??KBuyx(Xg|Nv)taidE>%*+>x#HbygBa)C6m;x!BGa&k;!! zR*2ZtpmRU6zqGF=$&o9a%M1PTYr)<4E`P2lTpr`x*{uE8gZBo#soYID^W292=Kp%m z3q3wa5A1=rNtWA@h6Y2R$l*q2l(F?LE-wF(+Sl3(sZA=y%F@P7SKOfLm{eb!HhAzuBo13pkn+=R$#aeM zdzW;999uzhYR|?smb_kF-#BTUv{~0qq7drlk!CEo#8CDZ3ROnWj%!0%(+}4Y9IQCy z9#%Fe0LlPsU#A8N$zHV?0bre~>}~^N82$0uxRQm;b6?v1I`12Er`o_0O_<#|xZ&`Pn+sM}?gL?|f{lDXN@=hnU$bTDo6>}>Lq znG@@+k(35bt3c%5oA$EcIPs!!a9j~{nndPtu8 zc@jj91QrlKfza?9=is^W-})sy(vM#8^vO@P?{a-xm8bPt z8UE7Zd)}CGZgpI8@xWv49W2&waIiW&)}mLRmZ6>I*8&>km)}l$Md%_KNSy~|jkdE3 zdc+66T|#N5^9&l3ZcQE9VUYeQ>lA+yW8X+mcE<79XVoN?=22$fIa;e~=FMJD`$YWQ z20FFtKO5xEopVZ8@!qhc^&1ea!#Jyl?_Dl3&kz5$Y_}xI0*cvwi-GyGxz)7@%Yj{e z2b^W=LEKJ&8^GeK?Y|P@xqLrMfHWfATfRLR)K3Wh%1EHbzI9nC%d}-Nr0ao-=uwl* z3-%ME;r17cM!S-*W51>nZ&vxB?$?E8T=4O62L}?<>Q9pUAeNkn(r3I#Z-6IlJ$OZ5 zgj7|^WI;n|X+g}m`?X|BJ~w5PF?pLg&au`XUqlWUzhoNs@XqDZr2lZI{OpYRnb9F& zvVY}0C>C7@*Wlk9?mj@qocwSZ7Jt;*hW$vi*epZ)XyW_j9kN7N#+rx)z=xZ1$~eIt(7rrWO<_~xFmeDzg3Doizk8V765#M55YL7q_T+xrZ#q%Cpm^eKK>ypc<1-?MwuQQL^e%KQbBZHwJ_&W@ z%{6&mr%oMpwT|@ovLU?L>zKLn0yC+echzsrF>aG(+kDGcb1v57pUvK;ug%lB?5`QK zWN!2N=X?izeRq3tn}J&%tF<_DZltWpw9Tc_ZPs%~=lvi}iKj39O2EB#-z@Hy+4+V) zOeA39$Kw3PZtAhKc;SlGVvl;GwDk5$mg|GV?6SmDuhzc^`b^&c=55=42v%V=Q${&|T;1iFFL20eW&=}|9(S#G)=#>P-PS(%1~>392_ zpU@>)vY<)4$1Ph8*?)Q_t;QJ*JodXiQxDdpyn#01z0#N6FPW&!Cpo z7-C%2?Mp&~Th4wtpLtPaPGv>mW&Q*TesRz@M1oGU1?$>qlgSv_c`aZ(@j{^J0{USP4y z&dALCP_^>_tZgPqBUs*YyK|D+jq91j4FSs7ec*Z((IkyiYg%8hB@-EEi%>zU3y_8k zd4i*D#iP=aVp`bp2njAviUFn?^dzdnK|IvLTwtg9TlA?UW_FQUOKILIwRTAjhT>PX zkfk8#EZ^M0#Ob;eN+qL53x*64;0`%5@8~2>Td5!l0=bb*WUVFrZ_k2L+Esc3%gRPf zz_YT+krAIp%fPK3B!Dbj|6RRSx;}E45~E4yeeG9LeAWgh4>Ky$JpZ&k7RI9jHouuv zPcMvr81eQluZyOH?1AB)tP81;aBcr<F}-)erpwQe0`wZeGd)7npT zwEx+@r+Mdsg32+W-kG;54*0g`?v;a<2XdlXb@z>lAIFTBsnO4+hY8twm` zuU<9Dk<&{rhf*}g2WtSzon^J#!f<5bnj_CQYX$8yX|ydYyTT08OkfvCd(xCSxkQ)V3Ze#+BWuTTqB3MEa1V5 z-C6U@V0`&o%B06zs{d?Fna^mh_}VOBb0H~5X<1^6u*a$HIojI#M%Q8t;*%9~#jyQ| z(W4vX^C<1yvBRa!;DWi0`?Co$<`Fpt8b-5|bPT3yGZ^)q!%(HzPx)bYGdZ*d;4 zc?ja)!&Fu`KF0udK7TAC8*pQvRGyl!sqM53Q&!!W=mnxF|9bp$xthtZ+y8#IZ(s@} z7q{`#EPmgYED7+MHNlBjGz}S5m`G*l_U5`b>g3`_%@0L0)ZqQR=F`FP8{%^FxJm6$ zX5B$Bp~3kYRU|97+#_Dz>S?3Qk%$A&{Kup7Aya^#5-yn2RTR9Kx5YbeU-~q-hI+MI zw&c$K`s>b4nw)RQSJOCbe*1LumkooP=p}z!F@H&gpp^P62KMKQmI zl?pz)U#=$mVJ~zbp*EKSNVpEMSn%Od*UI)4l-lEH%PQ4qJWV56;h}w;x{_?=I(2HJ zXz~6GJc)#5N-)m-V8@A_LUc1Iw=lr7$%saImUZ<$@7{;c9UDyhlI%yzc zneA_T1k`T!jC8M!`iAkDob{ zlGQh`g0^DVu3h3|O4^6*1!dYw(Lv}FTp*3aig>zpW5uu;X7)>-pjnGv%;}%I7QsVh z0_r5~KI`{-^`2FIcn?_l6fV-cSzIMU=`P;wuBXhlMrAY7!C|_nuBXW90$#H&uxVrw z4cTaVjnyRP7Q;+UJQ?~rukbuL_l9*!ELzBx(dcyL=}swFY!(F@y@0TZc^!~zecZTkAdNS(zNtT#t9T zc1dkj^m8)~0rG22=-1+ozjG<@T1dV`JqmQ~6nvQ*-PS>DO7n0N&Qf!bYvGco_%B(Z zvzlLRehk!fqYFvzA8u`CZ5*0y0hH;9VsCuGExa`Lbk5XUr6?-*YVnY51K!Q}MAi)| zvkmL?a*08ob1%IMx7?AxhX&4K-io^@o5@Kp=N>io$Czh6MooQ5S;K9| zOBoRAVwEoHgZ~OMh?W@vNs8XhIgP#0m3p!x5tr`!XBJSp=IkVH%Pq|;Kc8)ylb50( z#vzxG3uUOpnZ?gJza(#p{H-u|slO;!r#AbYUktNK4g|*o{vI`J{PT0?&gD}hW&oEo z>@scZ>Q$HO$w}A$b<*!IFr+HVrD0zAYo1@5?tQAszEdY%yf;_y>O1P}=%E|$J$cn7 z<`ZhQ<{JSs+qP;IFGqLpLhB|U*BKZfMLA>9+3a^Ed8riDB4ii0-rS1q@H*e~g8C+d zEjF=O_SD`;v23MRMSp>*{`nm`RvrWm+f%Dk+O=zUZbzl)W}I|MnU4~?1@n#R2Yg0y zsoL+&L5no`5r;Q|(jQ8DIA#G+iK9JYeYMo19&D=r2v60vb-Aq|cIgb&2jjV5&I(L)=e(K}^{MTGf6rPBGq|AZA^mItL~AUY$yV zJHy?*t0xB>@n5wb%pdj4ITfLy1G;qIgPfgHIIzIdP9rSLr|{2!xW45EM{y+HVCi28 z3A$<05zU;}xWC9!a=M>tZbC><#EZEbRdOnKEBHRui=OTE>#)f{+6yI}($2)>)ArPPUlVO|(;JM}R$|5jzvuLb z2;V=MZMI!MWaU{Z%7D~TN4M$Ivr7yW-&N-zXpjz#KgApe3p$w_Ly4Y>Soc;zQ1j{y zI@t^yXjTWbvnjN7KQ#T$Z3Y&|9HSOzK;#P~(+^swC|`pp=?k{RW5eO6y&`F&vg2&Q z!i5b>KoyT4J{)J?R`K(K*lrvJ?BzL!)lu7AmJKvvZt95(O)+fVSa zJRY}A#-~QL*V$-5(PvDl=lmn<&0ZM-L`9em=|<(OmJ{;>N!gHpystxkp68-LkqU^zyWtQ6e`hKkV;{> zeFlooRcg0pqqFHznLLo*+8grxcZXBN?f+pFk zci~_%P?QGSr4&E=UO~BNh*EhaT{N_k=u9j&C-GX9#sGJh6sMOxiCU0DWrPUPzE${M|9yu>hIIaYz?)=>z&I37EH|H|5u}Op{->(WF8YX& zu#+c0o{OJK=e9jC(Bo0cN-bZf8eb%haZ6Pl?S7wnYzEoo(@NgA zrbd(FS`{kki*s*`?aG7o%>UxUe>RG`O5+;Pj(K^VgJk{zfl0@6?IYbbw@ohmP(f;Ar=@43@0ZqA@(j#~n%4^3@yqbOl=Z(U_c^Oqe@2J&x4`|tAEsVGmo(N##h9nEJS z_tW>j&9L#~XFdXph+^}hdb|E!GQqLDPwT~-4n7f*ikF7VvG;Q2=TSG4HI1U5{^tAR z9oWo*4qdJF9r#*nh;qpQwz1#Kl$2U|@1J&HOCgteNPVx*l@(v) zzi!_fHEOi9XH~^8MT|84%w^NbrV2{)RfI=F0DRl~91;G~?u1vEA0^x4C@nEkGLKew z2QxYgJCt)Jn-R1tjPoEE=dTjEN$=LLLs8n0S5q!TMC1@&I)Vz6($r$vB?AIbPuzed zOeM&tTgZ?>gC6oU^33|C)&Fg^2EaRu9x}|G`}!dZ0*!H;$w4zCT&`4o3s#f=iu%GH zn?po~LEC+Tm*(9w>}Drr9~h|7m1i4ljK-3D4g2)b^AmUQlecQj>b?|kItGUvRsr?1 zWQiZHj|Y@C6H5BNhb(OI;aEM?CA$1`!zDx2lvlqn_AE2{rzh!OYWUmJDhL;Dx z87^GxPP2&X+}bH-_j}I3&H4DKP4`C_9`h*k&?-7><=_9rEteTjVjo$=5aMmdE+!e6ZqjDa^a4zmT>Pt?^G2q84C1*GJkv1mj@3V zc)66(J8R;9*LlLd=Y)&UEpBuqG)GDnSAt=fjipa-ME?M-V{KP1#%&)=BNX|$1<522 z>9nDXTYQE>OTCubL38$m2K)CKYd5)rDpAM-d9;R+G}PcZ%=5=yYqzsBnk7S9)5mZ? ze$!mQXvW4~MOQ?w zJ96Z2tN%=W@gzAp`LbXquD`I8P+%>Y&aHAKC&K&NV*33P1=oQNR&saDNBC5JiQgUE zi|G+6YldvWJWop4l4#?~m6S_8;WLh(JlVw)3;ek{mFBc*T*D2ihwLg#Ny6KM3-g+D zedR|3uto$x-NoJ~kOiHaZ<@>s(ckX03oCwrgjx>l-k#3!_ML(}p0f}PxyQydHZPol zX_X#PnRJ7bAULo?w)$uguwQh~!#jk>HLB7FZ}c!ylYtwVjg+%%`z03S2 z=b^zcnWr&d4z`HifoJr!=U_Skc)bw*kj%h;+G;8>ihVEl zfBBi89KcWEd+BT7zQs-A&+LDf`R;J@?1VWE4G&~TaNPQnJ8i$PaD2dA`%8Z8?lfit z4X)6Ro_KPe#rVX8+WZI>yzDB?Ep(AO7$2fxZP8*+{EL(gYqU})(+GtH*OqW;n72y+Y!YiyO+{0$aY&B24{P!5>JSb@J)8TQu^C!ir-*tVgj_K?CU(WhV@G=WllJ%7Z8D;DW8^GS+!}?CK~;T@44m|HPDpoR?C=?s}+?jooL<=N;+*CANx$&)7y z61km}7bj_60n)|+rFJS}GW3+qp>y927?hyAx+PIL!p*>q`i!6Vxi7P>EFYAechtfQ zTlJ!Gf?4GPfb*sF_KlgqTcpgn`S$D6&B-P#SoWMY;_q4a23=@U(Qj!)X6T3DVGV2t zzR_dw8Y8YfY}ePbyg&<3W1QbnrK}^@s1&28`9)y_09kjg!lBFaa?> zPtr&hoa_X*)p<3R&h-~g`qXa3h+`~eWocg(vUn-MDd%wO8Sf93anT-1|fZjF738-H-1#mh-Xpy2V4l@#&(5P=_- zc982Fl@8arZi7Sox!x0JZGGFL2z-+WI%iFN<1s-UC*OKnl~E4X;OP1DkFo|M?wi%pgmIIm&&?uqAeaSs5uc4 zljPhJe12End$)f(cgB|aJZLxXJ-fmOpPB=1QIz9Ub{zdn)J15w91Lp72NtEZ2~2wb z%p3jFDW{mUOWh~pjv zajsGI{JWymLWJ-v>k|ZFu-$F2n3O~H&rbc(W$PGV5q2HBXqYr6RO=Y*v7z+S?^Sdd z&U~-7V-zHjrMqX!hUmEU`16p5Hi|~Z%b=&6$)XA40w_vIMFIR>K)yp&Dy@_{l(Nsl ztJnMEJkGh)6Lp+!WVZP9u&Gn07O2n0C{5S%z#G|F*|wF(XR{AC}X?zYF+{z;0n(Pn3$> zRBhh3ocq1qY@Z`(pP`nz^nNd4s zj|W|&n}9vsPW9}K>Mb@mFlqQ;C`a1%Ui{sjL6=)BIlQNW_TFxWvFa>m*zapy$xYI; zZX7?%ed(C^M!n~s?0ERu_b*9hwS03%ZT+13?YeGgarDIyE{q&IayLMrsc6GTFGc14 z0I|l-{rhKjY=CTOR82)Wj^8|aZ&d~G`7iI&;e`#z)M1&hb zk7LhI+jet=Qu>GIm#MAmp53A2Ya;106uPAsW)TaGB)0gSZ|m{nG;hZ0dR6jmB%D0` z<@i$sT6IMSw_#nFNzt+)luNuw<(E*_wJ08>(e!!rUg6(~dl{cfMYzH#g~<=eGMi*3`J)<-GyV2FO)PT6$GUDU-B-fS!p8rCP3SM?bu)WU&3 zP&VU8;mG5rk#BA0^t~yf`mto_;6PsOVzrDe$_`d0MYO3Z<_|96+m~6e2#Oc5Df?Mw zUg3QDw;5dTmc9P!PMbm~Vbrq1vnklhpU6n?hdL5`PR8ESJ#Dcyi6i|iO-u$4wJ1ES z;vp4$bHu{URTTMsYT34J+k!E7&0sLJ^jx+ju5+i(4oUUiPn$gQ-0^X}+~c+0^n1GP z`{Rli!rLFc@~HWyd$uXNPr77;wmef)#B!5XB`xG1ief+E^q~G>tL9&_IDBif-=`i{ zFTM>J+oQ+ed3p938I$lsdf5*gxDHfm!G&4w?vHsPF9=(4%oIs6hU^6ky?ycyyzM~` z=5p;fC3a*=%@z~5+aMJ9Fky5Iber@V0rGzQ=+R5hifQuZCz?dGCdg}2C&j;R6`hT} z^mWJDYR3jNvtBcepqMCjc0P6lnSo$6{`=2Ga~j8AU~(LI1`leH2&kIi51fJa%p`u! zq|eP>WZi63sd~e>(>99l>$UG^v_SDYK;#nh=H)$hoIH852PX8mGXt6*VmQ-}rvG7+ z2R9rY=MUb)O>VRO=RbFkCr%&Rnw2T!KzIuioHl(;2YH?GmB zgg)L*N(2nRo55J;b1u#1-RiigtZI=E3Bs@RqqUw!)JMavGj>QhDiHbx^st`?a%i*3 z)eL7b7iTf!4<{hg?K!&M1*sDj&FRoL4>a zs)iBp+E+8bLy^A(b{>1$c{%#9r9GPzt(YG(S_Z{M?w7A#Np2ax(9rPyql;@tK2Q7m zKv(xWw%=z=8nV9S_o2gP#ZK)Fr>$&FzElmDpmO?fXy-%G^`GY#eDjQ6cD|3(4}E;-(T|e5LT==h~D8o2@_JV-N5&Dg*GJFDMfyB_<|z z{=m6mMjJF;Zzy<);E35d;V~C?REn)Bb4b^CcPnHdX7YZfFz=lfys5>Xi$sF@cJ~(N zO4?f7ML`}aXTbTPq8c>ad;I*rv+~Rp-?&juPT}fR!c##3X%G$AY_Va{`rNy`zI-p{|w z)ccJ~K8%ie5Y%U-_ct$D{^!M<(l1<4BMRNhBpFc7iHbueUU1ZYmO>$tuPHn_$mh>a zOsdaFzAlCD_y8v(KDyxbtP6NyDr*o0`<}8u>gd z#=t%KL`L;zOV3O&IdFHpSZ!v0`A4*zN^^g^9*6FL(uLJL78N=Ceo|Rg-&v+DJ3AEA zJ>2eId|ziUt83V(=R+TzfD?LzmhOT}LhYr&LBNo}FHWaW?uUV>QiC>t^PDk4X2AP< zOK(5OZzYGfe=~thCk^Y?jpoEaq#)A}l#bP5R))@xq{*qRbdzm-fnMMDG=5*)E`;Ax zdJ(DK-SuaUHPxS=f2g1YS6oX>H^dQevP*9_ z_ZA)#RRP!UaGz+Bd`chYzc|rc= zJu0$Hrn`QYd`!YU<@qQ<;|@1pGCw}qd+#I5z6M+37dGZi7!1yF2yh{XibBr=w@ryvp8$tR zm_zoP86-WvS?O|A)O>z3A`cvfR8stYN5XUKRSIQd0~H4W#7V%r@g+^fV8VW#>AI>-FZ%x`Y=hfKad`dT%n4v}8di9QJ*xSr`77V zXk6p+n#u2EPDYN@mv7zAoURh%hA34^vVs}-UK~X+nZ*y>Pcl?rbgrTJxv|J9Rb?`F zv>D!ohP^Aogi34rH{DK4R3Dx{a&sAc)~8jGLqrikb=c?#T|v`B1J_(dLXU=q-n=w) zxIgP*XSAvQ<{ENE0XWqgjW7~7e9fXEO0T=l=Db(&!VpZBN2=Co`4r`ec^1QoC+!Q1 zfM#hbEccbbYnd_aI~jT-h{!>&)yb3jM)>ec9<-*-b%5tR(k-(G^RcT}9mAu)?>%!dyg4^@7S3OCc45YqFL{g; zLTN*QJIz~asAorCB1_i7Q=cbh>vedr%+b6{TkEx9t2Eh>qgG$PyI#lB|BqwGBZrT? zci@@llaO^K!@GD{S*^79x`^O4LAmC!CpN)-rb6KHBap{P*I)Jlf z)!erkB53GX{Y?S@a#sSYo0u*BGQ)#OSh#1QB3f3cJ?=kggYsMokM314ZF|6Jd5QkM zes4Vn;a08kOgktqRxuIr{5nb-ozY{+U1L2`vX4N@X{0$69Ca@vn53{~u|aN-KPdSo zZ_OlVycb8M6fGDvs-E>ve(QU48kfJxty(!nFNz8dzVEtxc|_+202pm5 zg^irgv^kE{7B5*~DzT#)K-t=z3)X5|Et2eT*)SJ%e=QGt4(5SW;AX1Dn?$k|4l%2S zEaD-Xn+5~bd=CxW=BLSW5b_r~0hlcA4yaeEnJfo$oRnd-Cmq{y)0j zJdn!0>l?pNG$^W5i9)4$ph2b#m5@;C6i0^UVuutB+q6$}C#6wEA)+K(lX)zf2exD@ zp;FlhrM=NMy`OdI+|T_!@9+2g)BT(Z?dyB3?^>T}W&PU5;_aVoH&{Au@>_7ub?}X( z=YjUa%!;g+w1`ez>6)7mJYe7~+_8_*q#|o(VjXS-2Y*a{Z-63|f{#4*bSPPz5a5_x zhU}Vu%*QESlV_|>lUgDZdZ2iD&sgs@(9h%oaZXeU_x1B5&zlouUSBqveX?G%WEMF0 zjZ^wF6_n$K+im(Q_~_%o!j=y_7%?KpO{tG;7;>Zo5F(&cj}Mie7{H%9XP^8@cNi{B zMLl`OnWhxgJ^EgSY>jsVc5glTTe>N8oJ}>8Nhh$Fib##uESk)NbzI2ve|oL@FgUYkmA1x zV4#c_i+u+s@Y;7G_(W*H118OpitBdC;x!k?5D19MXCcaBu*(XaM5*!Gb1Oj){Uqb)9W2U;9&49xN1oPg5yL`i#XBw)| zNudSfy9GEaIfsUP)-s+yb>;umpwoT0&)=5KyzJ?@I=m>>Jwpci#^tMboNgo*!0i3n z6l3~s5tqChOHG%}tBWg;V0pcZr;GD)S#E{MCND_WWX+kKa?`f~BA0c9G403YYQ%bh zA8@icKwz#HhqH6gBD^kqky;lP9Xl)O$>`k(?&;<~yo(UITGiY>(QW2RfB(9z@PH{a z#lisSQq*z~47XvAu{2WD#l5N2Bg=C^-6osK9)U{Ro0`K+`H-~#^k~jPqd3%SV>fl4DdaJh95+&Y7XE;8l$Qq zF4yEAELi7>eYfGY$rI|r)_4g3yfid$zxsjTGxV!JE1(%lPHBF=H#?0BOgTJX2%8_Yh5L$LjIzTT`S2ohZOcqHAhl}R@#BGRMCQRsR zd%qm#9BPGivxb~3Ct%mD;ffA(bUe3wG2A=|q`kRuLEDk4qVv&b|&4qeg%<>LB&G=ARa!XBBl{;RQ(X;~#2<2e4h!9>s7i{m^ zLjep3eo@VV@NEsEQ9Z^Of1U?=$KeiImNP(cNCP6KF+c9trr&pa?8tt7#DGlH+R|m2 zVi%x}$8oV3ea+Jz;=k{HG-+~NGIl%^DM2V7Y9gJSw}JT}k9E=Y(9+H2(m;tK1#LR7 z=?z{J^c7nHS@Av5PL(1chA^+Oqm-lY?0PR78o6C?Itx?KkTf6;NPx8=U*3fwFPRip zB%9;d-i^A040u9OEogzw2K;$U*K}M{gsv-*AtKO#+$WM-KLtH=5#oc>Q-ODh01gXa zH%q|$D%=I)2j{h`xu-iBdt9`kDj7RrsJ+FG)9UKy68tT%PJ8n>c(Rr0EN^_Pd)-telsH>SUpmHLu7@_ob+KMcYz zseO2azc92IffS!5-EDvF)*roVM(KenNx{0Er~kCKzyA^yXGyivyA$fJE-3$K1ntpZ zVWH+a!g#4jPr)gPQTTh8pI<^^;&ULkse{z#JF;{EYJONrv{9e^3t~4jbD8(qqL0hL z?alHDs`MExG`RFH`1=)z_9FugNCe=a9hH1kJ*c?6ynNbQ^ip*wru8C`4i{7pVLH4> z2M8Kfzv-h1W|jJjH8^Mv+I$Jk!_-#m+;(ZP+sb1Tt}M4g!V^mNJSkK+rqwTj(-3|B zuae`OpWb#C9F`QK+PW}q3@X75ukvI7qqi0KFrq5vacN?$e$9MZh^I!$A0%8$X#@@= z#9hO=*|*P!uLK+8WLtQn=8+`pvKbz?@e7~PINff+$dB-Y_E^}8E2iP#NZ0MgdE%WA zd|rXPUo8inc9K)dGs(AM_&dr20RSh`bOfm1A{YQLf&Tz4!8|i*;D!i+0r}T+jbsnG zo6*HcE1QF^QJ7J4TicF>A3Ah>(2_$EDB=p$CE1l2PWMqYTMRO)Xn7v@j7379{2_NswfdUKu4M z9o&d>4tkEU{dtZb{!8i6Ixx@?+NpZWaj}N`3dgGnfju8H&udFvZ9iOt9RfU! z>$PQ3^Q6FaiIH31nl_Rt%!F$P?8j@-@zDk=kmiNf!Avg`D>F!IgMH0qBL5s$x`W88 z_P9VrFFFe^fhY3+L|9VCfia?U4cHl<1R@25$wKryC_wJpsBc9fNPz9a8Q8v1l8aG| zlpvhqMw*7!6&$Mc^`eKEMKRt)8*q-N;+`}{Uf#{NzM7kHJ>as=fdd{wqw}bm4Z5Hx z%itJ`g^vw;nqM!p&OclKvmOtaxc)^Qzk<4z81a#To-R^h!e)Zua43eKFbp0@hvf0B z`(yBHdtY>AAEzzol33+*2WBolzo_ZhMTTpMF2&#CYQs@rIm>Fs%>cAVi7WWOL_f&0 z*h|EIJ<&A{@3Px#Rf^i)Lgav4^Rjm9vn$U6(5pz#OFoJC9<} zXRt8KL-X`ef`KFPh1n|y{9QMH@sQZVGv6r(c0X!vOT>R*^@UJ{*#kZ!-_eeonUL#W zCxOshEEx*|43voRMMO#wJ5*)v0_4u&Wn3qQ6r(md)HjWtzSo~B{bv$4K$qK z5DJ{!nDvi>`5L)B-5nn8?mnK)!sI9-AKl;|Qk}K_z`+Cg%bqX$S%*iNxy3xQBtW?P zRe!U#jL`!cs;{jP3&%?gcTCWF?6b`hP7!8(q>$wUG{X!|#11{*RPCU zZ#4Aet@)=n4^|WfKdRko?e!s3gBXM#(|{CWq1|js+-6^L^P?|oexx6I^{>}T7YYdr zno^-DM%GxB(cbg*QnAN}`i}Mq9#tQvz^y3t=OXdp`ANNd|c)&<)Tf z(Cv^a9Hb0>|L7tzs?va(xkU#bAB-b6;Vg7VY*8MTT{UsLcfb}wJazB2qXiuxOn9&H zH{vl13R|HysSid%v@_>?S4kVBCM5D+FGgdQjDhd{){S^D(T_))207Dw%+BO+)bz?x z4BnzQX-~$Kor5#;|EDT@<+SREJET7=cjQGK*4w$(tW!9k{B`jV!QnSA=}_3s^Lyem zy5bbR5%0q9w&cjFS|=k(>ko@0)3;ZTHGYihkz9I>qxWgEQ~!8 z0O5Rt>;G3BF+qJ_4GZ8bE{1>&_J)|tq!=)<7j&%{nhfsChgV}m?*co8%(?0eVI)3) z*tQRHIFcJSVjzio))mO3tdHtK9#v0x?Na2X%p{DYY03`>+X3)DG#4!;2{i;Ck3;~h zlm5DU$bUb2|3)NP++SNPn)OZ9%#Ah;uHRvHJ&Z;3G0I&lTD}Iaej>$wD;U7S_0+}%ZHt%{V1eRl=o&9|utA3kk9yRp) z-V|TQ%*bi4%F?H|&AG&IiypS#sxlT_O$*hlcLqS&gU*vV$-7WbfUS7~r@6#`QiRxI zQ|zmIzFnOfz?R0Bba8h$hpj&Gtdz$qBA`~e0OS;4oo%tu%vA$mT^crY0?~YNq%%Vh z7Aq!z$NFJPd64)dC)(&exGH}2c&Z@7HAW+8NjO_(18aZw=Y9R zU10)?m%&PgR^1z?k}*z1P!Xx31ZpU(&_m!a>CK(1LYP{?N({HI1mG);)?W^&?C#mG z?{K%`#=nWyKQoSYfD{8;BLb7@%fz$#c{Y>5Nx1 zx877}NIQlbz=87{fn%3qS8+5F2-Lbma1JdX>6=4hIHG}sf8v2D?}Tpi2JH~%Z}~pS z=0dck`d)Jrjk{c?{AZM8F*1?5ml1=31Mi&t^yN!(EQt}L#38TE3famHW;d2iv)ts- zm_B{bvf7T3x9V(`p0B#TtrUmg&ygF$rsI2jU*y`0;yVuc_K)byoEhA>>;;ciD@V?j z5l1daSi<%7jaOGYlQ6ccGqnc)3X}B!rTK@u$XqkYW*>D^L>Rgvl7W>Yvm$h`s$r)! zBje)ZJ&?X`k(`~~89s0YBQk)o;QIdi(Q#VL`o)kN!2Y^T46}ywvC?s^i<@7sT(N?5 zEbXTwA{tb0su%~{*27DXAwSh33O}S)Jf0pqYj%~#HhR1pP_7}@a29#F zR^#ikT`KV$Hxe82j(vgh1{-K?)I)2-lfAcY)v1RfEHXp^@s>w&Emo5S=%<^9 z)wxG%C(E#gs~jw@AbJ2Ry_?@fVt*)@qXiKdwIv66RgBmanPX8iA*$#Js3aAnIscXX%D z`DHos6{y7YFyOY-`H8@mnbR3S-5`qs?7(L^3ub^p#*8tio-U0g%FSN^JR+k%HVrcG zrS81K|4|-jhM6nf--?Y0D3;$2+#iI>VcGmshS7UO&CE0lCAV+3aAJc{-o26>OeAHR z4*XD$5_m8fw}&SW>LbfvpauNXIBXE391O(}XLJMt3}0R{c+D!B?BFHF4Yp0Z`PHQa zZ0XSGiV$tUxwxtR6fKN5L_vlS;0TW89@<=in@`dl+*2V!1r)*V6IN7v7XgSTC$nJ4 zq)C?JEkbS$lMjU3{~%@AJM^IHzebwM3@uq+JB6dNN(W%g8Ijle@^YgIEQivtA^e;c z){2?KUb0qYYVjh#DPW7Nvzv^46HyWm&q%nEznfiN|zlJ^S|!i!B8FPY!j4>bD&KRGDA;Eu+6DHCynB=bmY9P9v;maY z^xQ$>)1v#YP3NzcJDCnh9wP^q+PzC80QI=4?z8n%os%&Mm-c#?j+la?1&RFRO(wLA zPGK5BZcD3`?)}q!S&6Z-A*#>i&p*XyYJc6mbH^7tkVL;lix#n0&PCt>kU**BKt8q8 zWB{gw-%=w|Ijvf;qUB7mu)KV0DR}70F@qSMD`aprnc7zX)7f3%h(w}dkk6%WHu0C; z!n2e#S69QXW)>qtb-*6ikSZMRy%QBoAyudkFK|R}xQosvS=zKAeEM_CmReS0g`Wvf zZvJt12)ST*iK7dw8BR@nOC4Bz|4A*=@ z{GfQ%lNbQ|a6*i8L*#zo8n@*q;z6!|1(Y%c3)LNh(b|#6IYXl#Z#ZUu$i0eTnc%0U zrUDQl!~`q{h}@Frb&L`5Gva0T_I7tpIQMslG-c-D2dNum`X?kk7vwEVyZ&v*@}-C4 zrkTFnVl))rW$nSXN2+P?*TRTEx zDGC86(;^uy5@x!Tvy*-B4pL@;g*Mk}45K*<>JK%j5qxKy?&EuYjaSBE-1l?1kcj~| zv6?R^SP(mMyUfW8_)&>u6+>V4z$8Yg|KWm1pXyHgQ)}uYC>V|bmYT~6P^BHWw~P|f zWrMC9p#Ccn4oTPNYI7xvTarq7M&v(uwEhRf-TKgRuo{BDs)IU`X1Zo?6?({Bq#+55n(@3_=V(OR}!x1q=S^{(Ai0~ocN0P+TK0zdr zqX+b0-A6-~?+Xk5cvGwHr1r&pZFX!D>!%iw#hJF-1z+Aeo z?(_Q4)d6i*J`14$aE0s`vxq-p2OIFtXHlbLaAL!@fqi7TpHErdTmNtXkdx}tT3|BN z^%B@3h(farN^Zx>W^`SQZyMCL9+1=UyGb7f+2*r?_v*|DL}oOQ&BjD38jsO0BNc(m zB24RSUI}Dmj#mB3&wc)}b%&Q(o@!K*g5~C|+rx`?X=$#z%xNs)4F7%c45{$JH{rSd zHaLrHVG+oBzM$*N)||nmeS=vL<$+?}>d>d9XKL>JdG_*yK{zJSYse#zqTGWLGd*s} zHs}B?0*Q7ScL-p}nRDmjfB?Yf@=aAV6R!ZNssyAt$L)enZYVMdOt`ZEhj0Lgv~YX; zJJv%$Mt8%kA^gWFt+j%V04D#qn*R1^y!mVh#kuJu1*9=%f&!zNie zKT8Bob38LT8Wzkn)HIfm1By#18_{P&?NHtALq{8&t}*Mpj&eXZJ;Aq9T67K?{JxVm z#1^@oEtECjl9c0`47_L~Y`D&O(6QGcLVp!B^MWGez!PKz*mRAxyw)L#x*eH%ZXY6Zrbo}H8iWFh6fRtURTsRV(&0MX0+$~TI28ea-H!&EzE3)2~0tj zg+mVm!%*a=beYxvJ;m-fXz@=Y>-*Y|+U?TO!Dd#1t0IK*9Na{}KEIz{^goYB2+tBV!;@ z(S2daE!Qo+4EaJFKr^;0@luiJE+9Ps%jWVNUxOU(1;mxS7kz>cfwm$UNHwqr< z4u0(8T)Qo%s19)dN;JgwWJd#-V){!@uxJQsOqS6IMsqO0nCt%+NLEf2mfnA8L?$M; zT|3{Wzx;mw*c!!n1xCE-L`^#X_+mGsyI)W@q-gJtV>9U;$|ONYNVM(It0)W*1nY2t zN~N5YzfB)D;g+}-L%IQr_(3**yJ{fgOVNk_QbI03TcN1Vmtbs*g7p6yfS6d*t;L$; zmVQ=L1iOaS=%|2}qu{kds_?BdS^*eHZ-T=k%pHk1-Cr{EUZPzku%6o0ix=O{h@z31 zf-gi+R1RMGZR{PPsfgbV8M@!&F}t^$D)3y16T%;FAZryg`mk1!LI_)W zlBk-Z907ILOr^82S*rnZHm-+r*rQDoj(P8hZcEv1AZ_Hjc`fGlsuJ$Tzn-_v-_7_#(heou>KrPrCQ3Ss< zB^lz7O|f793@-;{0mog@(XnIAP~+d{1L}Kvoha-#7A?sd>pog1NWUg1C}p%#$xAH( zBIFDEfzi6{E-`JjuF2;YtYL~@#vGn*p|tMXAb2!R62G}3m+RT;> z;ABQrHcAmO=2=QqI>am_C5f)RgmeYn{9WKA@k<^Sc&6zWyC4pyJPb^iDWOaf%^vN_=RrdQQmp!+@u2p1`*p<~qS z%WG=E!_yeWFzW>{Qd2j}Y-3^z?XPG-q+X@9}(1YK=U}RRcE46$44zXNA3}l0tN%Y z_asj+DOGai;QGbFq&gT1SgqQn!=UoVJoX9cxE$BqygwK0evqKL=ew)Q5}fko_>#LH z0zwtCtPOIdZ4?zN;DTX08eC6Ts5Pfqc<#So2~0k?MPc&U41$?1A3~Z18iu~t0bNaq z3jJa!6P?8-C3Vf`l-<0bf1)ohYS`ez5=>6@n6CMraOeuKj3fPPAMrk1J0uA(1|5rB zik(q^;6vYu_LZ8GKp1S4AzJB%@b=l#uRU zwePTxl~aLB@Cr3eR(&yY$H%^@74tGv?R8;)4 zM}r%aIl_x!Gb?WM4h$J!M0Mqv+QLkK^UT?u?>lg@@!AXjB0Y8C1n>}*U!q-SdKVv! zx2y2s9XRz(vh@*o^~mz2R9xJ;%_nDAvw4~wvZGX(dMrK<+y0-?PX*XLl> zAw1@6D{BCVv=sS`$E(0q2mOG2>5)Y&c0V=ow_-!!keuSZEML>(hlR#$UkO1kDM*no zz??jSbpzMH*|P)F8teUzRNBen8~G7`HX7~j>!d}nIF~?}8Im%M>gaUGwP;H^XTwoQ zdNoui(YsW%1=LMgTdy}8R;&hytv>T>7C0dmzsHBVeFG>`(ne>#I&VdRG;|YI2KRJT zaw9XbsSry4m6QMI&5FSYp4GS$PmLGxQU3((5+g=2rIA#x=(1;Al|6X`mn7vt)oBO+ zI@S;wxC9e0$^*@SH&tMv;zUb&A^4aJvp4+%EZ}$*4p#aC7107g zLBelT?G0kYLa;@d(&Fw6lh1P$k8}Aw3A@2z2%tiTD|9NqG6~6bszlNz2bnOMt?2lL zl2$;GZsIfyP-lv^R5_>^FGq5558*JlnUB*p5tSfF@r^+HM`0a=paIHffH)n;s$u-(^WPDHb~b<&F-C<(4X74n4IO5@J&1<)*!M`Nfx1$|)c3|To4^f8e{ zK)ACc{!_79inesJQ7&@4c6d)F1OkX=Wp1pCv>y|65eKuYt2ubLaUSG7Lo%n}<1IqbP-$aL5f~+u z#?pzI0p9@7k#|6ZITg?%Btk6gh>Ai1epr)c8Qfl=0Xc0hhkqZt_O4T23_Ca%DT-v| zRt9G7Y;W7y)iljY_(U-;eX5c;dwIFwF<8E;r<+s!7PI;xCi7hHL>z5&*{iO!`nj5K zc0$3fKQx=ZT>ItQQ5+;~yxQ%gJ8|h5slwUQh3hb?ADMw3Zxx7m0Y=;kU2mRw5pxAyryBjRE^Jk`TUbnQOOuAVj+^= zJY`T&0y@xAL5DwM*2d}*MIRR!&yYIXig(XrG=E^h1jXs(h&cC60mDrO?JI)d+1Ft% zV|M8+mWcSUX@LJK&*o75{*QSNQlpW1mdFaRx76^*h;yFmcy9zm_+Q@ehLjRk0(M^9 zjBFP8E%FH~8ArdtVkJ1#pMU5?GS!r;6x4nNUe@D5Fm>PC)#$`JE##RW6q9Fxt(G@Y z`78R%F?q`|%ac|B!OsE_lADZM%Qu7@?2HEo{Hg!Jg=7Oe9tndBZX*3Eh(-e~CjU_a zBrO^6N0B7^6zEOJnIR%d@XEUei7o}jtOjbxYKurC9kdLux+bZ`aD&xDSmfim2Du-Y zAU2I1&NJVM>AMTkT1p{s;V4*Fn;vdOZiomfNklqE9z0;Se#w#+j}<+<)yJQfcu$fD z8y9sWV9{rAdoEgAS% z)hx@DhXF-{QzemN{T|M>`h&uDP|%M*{7D@d^q@}~Bu+N+g;2A8s6V*y3a3>@i;n^N zMtuBNk3ggs!&TcleAaa7*cuGmOChw+MH-)~1b6xygl2?&mRliWjvTPm@3qcG5AiUv3VHAP4QTw`(ZBII;9h6c2fT`#yNDE2(uf*=Sh5%<;bx&(z+vd_5~_~v`PkM$W?RTqnO z%CHzO4-KBlXznsNQR773Pf=#ocvhOCs_He5rf%RUnevS|)4dTUhide3MxMWE*l-$w zKIHUsY4cyKl=S@7tK7GH(C)z4=~+s;qkUGCvJ2(z7BMNt(do1O@J2zwKUnDVCA0NV zA!X%mg^$3ZsP?Bj9MA0dJb?K`H9Evlx!)}>S>A=!PEI$d)fL8TQBDCDTdy;6`-tyg zqR4z@I&#E_dZ=I7II<7t5l%y`EZ)7Dz!WO3*<6hAZ_2O)Aa309i)bWrlSCp4ktCFu zyT?c2#b=yDA(p^f?}qnkwiMSr80k`?AtuP3?1yQSFd^c%KW+y@LL0}xr9bnTtGsB3 zoXHH;Pp==`ch4+WFT4F{@EGrJ%WeJ7H*Pg^_R$L5d&Txzgi%Vi&Z4@Y&8u*TqMJVn zi9D3vTWLE+x@K<<`CBbzK3Si(3^&k;?zwy+_S&(#DQmZfx0D2+p<>+~&U(Q7*(5UH z)EQnS7DZU+s|w=}6eO;cfFL;b9$TwjcM32+z%ar2{^7obH-32n*+WAYDzSX13 z3|zt#-oe}?+)D|7M(e&GGgms&l+gD9@`+&CVPh5+D{cBV1#hv%<%o!BdDYd^q6Q)G z0uljO#Lft&&77IbZ#$xX8ua0w*C-x8CJa4Fs#FVFfCS7x4gD-H(jJREFyX7^AmC?; zo?csK)io>tBIRJHj0AKp#PSeYjv`^oG+JUo8C2{Xb66wj0%tQ|Okrs$(JU&zjq#3P zC)HU;A%0F{guwDUPB5@Oqgb$cDdbMXc!-AOdkyGnoHz*}R7~P36_DhzX=X-76?XcX z3ov+*4}&$>W3AH7pH%i?t&JrGC7B{q>>N?l5IvayW9+TRW$Nj^b*JZTEB_$DIOC|DhJ~GhgD6mbyJWa^|bD9=jKU7cw$?GeY|LCZe!(C1tx_3pdW&Xx5Yr3j(Irw6#KAPw2VhT=Gx^);Tv z)vrV;r?h2~q>L~R*4yptdp$R|YK5xS(ElBK#nb=#V~_K|Z}JEgcE@31K^crSm~gjD zQVd!x&b{=oR3^ICT*(DVS54=NARVJ5Lw%sdhzIm4#StJE48OrRM&TlC7VkHr8_9-^ zdArM;$bbE(A&^l0N4SJV4#F;1p&=h+cBX))B|jiQtkhb>*_3l7F>xuz*5#ImGoE~- zUJ|ykRxT2Z@yA09Crnt3bCw3+jy4#+L(WZN57Z%BA-fteRK{$5d8%iG{k6C_cVJHF z@bW_k!tj9ZB?W43VJZ>}>ydo1+{5F>!Hs6?J0L=U2XL#nDZOU!zcbVK=c7#m99b4I z$_Lq=>69aD8#zDYu}R+lJ2LryBY!heTjE3KR_&Q(B;>LHGqq}O%&*bV?KMMe8&xn`&79KPV zA7F{Dcq06%UPc3y+j4Js3J;yJ9dK|wMI|ur|M>lUcf%7R{1hTGVAqH==Q$ajh{fKH zV@Gl8Ks=T{V&*aRph4oZNF~_{qROKAn0D@RAo+3TW}O2Nem6!@e#QhD<>zQoV}rV+ zql{lw6!JxZd&RlxOmQHvv*S{roc#MC#$%ht&L97B4}r&oaNl4yq54Qz9=fx)lL>S^5VF6dH2{QaOmS_+Y2 zh#}-5&&8VQps+9XmYJ|1@3w*kS0R}QP6E~;(lT5Gn?2$VbdR%2=F4QR24NDyIo zQCJ;u1v&#W!cdZC(d~@HS7Y$2{4Iwv;V~-%P1umx>;*e20)l-)183q1d|!&t&3dRM zeUeg|5RHsIi-dwhy7uEHwl``Z;f;2b5LuRjbY~A3B&8V6lCd%2NyDaXy2TDUQNu*W zFKSV6%-kMc#JOk0QzTbzxh8KbB@D%$cfcnmVP&$zyc;Heq>jN*{2`vSLnu?TxR9kS z*MHG`sp>;p2Svz%$SEAmoOA>*NQNv|@xE@eWpJFcxk>=R-T}_0NB%!HES_W7*|VQj zk1U4=%j&>k-}Uz5TtIBB%GDc}4FAdspa}@GRNU0auY*TgOJY z(fx3m(C~Oo1$q)h+s;X~%*EV(?r{xW+`J|*!q-8%mXKAr^N_+}R+6-A@Lmk2*A~0U zO`AUbn`FZSm?E+c6@W)v4mC-`8vxgxk5Qjk(~};nHBp{}moFu#iwihf42_M7hZ189 zkso1PtmPq4n|FcI=x8UlAs|?47>Syt8$Cyb*B5}&e-AkAdUQjSVKkBuE(?~D?+KR~ z1>nNNeOi6Y_9p!k-FRqOO;c20onXfssOFe_l40j`wN~(dwg3u6401Y}o=sCm@%G>| zivz$}hu$aq>#Qs-+P=oB(_nuu>(@uB@^zoc-o?4+$KBX52GVi>H3LM{*iYX zykqDHk>~mS>vC2LqHla1Tw#yj_PWqJbP2H5y%d!6j0x8BVQ#Sq#6||cT|4%xF-+^rUnkJ&}ecSprvfr@`P6cwNFH2@Ca zj>kT1qO5@vgMW6P6z!zS4-VF=)ZN`3ZF;$1-ZDswIf9RfABZ>#8}fJ>JnF#4v!?1^5?89nD#UW3cB7@blT~X6i}a1C+gv zm8Wkd*}3Tbpf8*YnI1BgXeFy(U>k@S2Y~HJ34@xvKky4u4^mViRnAtjIwF}zi2BT( z{~V;GGw0xDh;BjBTslt$-w(hJ;c!%_XeKJ-yEHg8{K0skR|Xnc9MvtJ6o?bUmt48p z+&n7)vZnu1o28G!3;u1L2~cxJs@hzXNA1O}=Qy(n8|cv&TwPm!dyImWH+Bp}cIYD@g{D<9wITVScXb+*p`eJ=T)IXL921d#lKZOSh`fd?! zRZ3mPO!s5Te{A1oH{thvjqCHcE1H(Lx$RQ~9WB2n(OKr&`ooMb-OgGZ6}80H7OHWJ z(4c=yy9ATuXNhaicz5*6&{pAWl*{7dM;KW>NR3|D$=do3`+C#9SLkd;&)53UeBk`)Lea5@rStb;o$s!K~tH;QrPiVE{RfRKVkN+{y+$fL zxv^t;5%&Y^H0Pn{YNBIv5IvoR`UOgyGOaV*XPadIvDw}nPDwHu|oBZ~gn33n! znJRs_W^>u_&2=nadloeQ5zt6!QmNX&BQj>=j;ZH5BK@_swNPxjw z;1sbgTMlQAltPShvCO9xE_hsh7tm^Lq1D=ybBZ%?0U-Hx3QCh?FC$)!LhpKmtVnT$ z$z!V+$U2qp6JK!KzA!+-9|f9a@INjs0R}3;78VXf@;NO=wD^(r!h>?QrvWO;+X6gnvR-fqq9esWK9bHL(wv*`ysyPtoU2EmV;QF%LQe&V@BDL zNPC#~OTmjh;*EuHaFotkXxM(1%iA*7u+x!tbSbjfT)uMr< zD(y5;JX}C?`$O|1Wyo=KdNPe}d^zUmI{Z`kV@J@9-?~>&yBW?Z?zg*$D}lzSL|^`U zM;8KuN(be+v8^7O|L;I7KP|GfJj_A>P!ZjLYLSReQ4pzd8nO{qR2hHo-n|HJwP?c) z(FXKP(tku8H-HB@^DanGMcx4)6{z8#(3I2WdZ<8_xry6iPCX7U92>7!sFovqIv!7T zB?OBWdH5uhjKwD#GNuEH{eLsBzPmvc19N35BG9yjnKOXIyhdi)HQbkLtQE{oli^qB zS83FgzCSnufML9^uP-IwP?mLa#>5x6a3nA06iuEq>6$|ewn@5IX8H{{_#&eZ?To?j z=MOU+YR3jAWDip9z@Pu0w0vkiaNiW3!7D(XVWr^q{UAwpz8p1}ehjkB5EVl>{mG&E zFV~w)QqH|hxJTXRaS$ho{3%o@ z6Z)8xXdSKv@nvZ#DRr2T3EJKU%2*YjA^?L&a^ib{hH))D5z#IeIMi`N9?>SYEOsPM zhNl5-=FULv^I>7{`tRptmc^_Wba3%`;_H_HTteS?@=?_Be|`s zmM<@!7PSWbMX#bJz=Ws|YdgE9b3k&Pmts(-&P3cB`>_q`VQAdaprP6#rm#h94|jf8|*>>i9>Ve|JKeD>(W9HWA}GdLAs)9D4BI%CGmg|nF+d)!d0rvW&~ zLf{QV%`y3nc!tvFZrw4MY_n1^yVqk$5Swtusa*&ITUB723!>K5y3ixIW0uW1O#^%T zEZTn1m?Cs;xC0wd1N|MCH4qM=ybSHB1PYKnHgGripSu| z7n-08#32mZ+na+N)MI2p~MH?6z`EzS||YJBJS zQs%kH6(SXW#24J{teJWr(G@3LZw?U+q5?1mdC(G}8`5eEv32Q>*stjW^NTLd^9;9- zLUVq8rVALbnZkFWbIuNAxUOR5N20OED<@#t45@DxM`fYQKLN4C7}KElr1GmQ^=Ewt z;|y!5o&9QFW9h|Kk*rLkX(dTRVo-H7TCJ0=R&Hz92SPOYXY{_51kv zSxgWvMzS+w%L$Vo4DT3a>0v0jLs8f7OfX+kUSiIUuI|GZDLYyC^DYTZK!dc?=*1*b z(?I-c*}~^9UXaow7JQ_mwY;V)awF>jU%X~jTqzlZ2>^`#SH-1oCok2(;|^5#X@K*i zoap8C2A>+5ewVF{rD~5=OUsAc9@95p8@~!79;^CZNqW5td_S+@4lrUW-~7MzM56?Y z8t^J2v6TE_6)_`!;Dtlh_0j2glN(@sxv$y$xc0|8KG`wVbr zaq{0LJ&j}V87?tj@Fyilr3b6?Rv}lJ)_D1mv@w-3F=21h>zJY`AAqJDSy(z;bbwi; zGNc_0`ATGpLu*+Iao6yp7j`lTMkF^PTmn{b<>Z8Okoau4{R2<%9BL;3$r*NdQ<+4f z*J(=%0wQNgLCtWn(^bD&;X^!3{iMAiKaw9VQBH0=)(qrzHG0!Rq-h2Q8Rv(|itI22 zNY#5^`Vl^9`uK|LA0@;O&I!!6Z*)$I&b0K_;FRJ&vO{0cZ3slScDDbp@Wu}W zCDUD~eE%3ZQcZ&Xl8Pd#Zhk6_eH4?-`6wH$H{CVc5pE6-MAlOZ)L%Az3J;SAt9SYf zJczJjs0u?c3XIAdvcVjuforP7h(-}BST~3+4?rGVru$sve)K2G%3w*!9<|D8~G{ko16$ABbF z5JgcMdLj0UP^`(n!B3QuNPA;gS~Gq7eNhLlmX;^u+1)uCFXJKI%OPGkXt0*>59> zR^9h=(PdFvS4^7o1)DYJj3akqfrH8j96ad>;sy|3T;kvsxM`A&>?l~8IYSCXA9vw} zJVm=LkXl8WIvpZpt1=ObX>V!+5hM=S8u zR%2cQvgtyIVdDnuMA_~Fn2GRXGFvQM$W$Kxb;+#yG287I4*i{d@9wf(Epbg@ic)m* zG5wN9^&`%`@fm`NR6K}6AdtEg!alfn?>r#3gn)sifv{=y7QznJt^sD+yT9h}I=P%p z5JWxttNl@lRE#Wpk|z1I9pQgZS2? zrsYWdDn*7|WjAuTb3dYyM1h&Zr9rPN7egBVBhwZnkYB|XbET+_>o~aWF@j&E3u0uT?M@f+=n;U8E( zy}10}l%%sNF^7eO7W)+#)4QjXbwd5nAaNK57|MvUfT$)=@kw8=g?Mf44;QQhAB<$a zXzh!zJ=h}xFEQtG!?ILxV6lUHbF@AkRe!XHGC+cUZN{6-qPB~paPexLz>1Psb%>{s z-j?|#2PU5o@}jS)8p#yzo(m-&MgbP$Sv?{}fr*|J01V2*eOA=P>;`@o*3)J-&WUN> z^b``f)HZoM9dqp)SZ~J_OG~#z1;zvU9hC$A#RH|*xb-3tu(6h*s&e|qaGTg%T#t<# zuQAodf{`0#3>E|S%|gFwWhQzHiLm_!rvlD$JR{3+_aj&x9T#;3I7=HU;4!5*d$#X1 zWCSQzWTJ}I!~P6Bf(q(IPl^B#;lxU?H4qV)uTp@K{@{K*IFj=02iE1f1%K7(X+PYM z5koyajPeVI{{(vC*PhM6xVCU)Mj`jS)5|{?z!J27 z4evbd?d{WYg6Uef)YejDjsyeQ!G_!nCtRZ7?4Oe@FtZ4P6$xevwH%_6-lzLiN(}^$ z(GLDqDeXr6I`sQ?7u{ahjLCrCETU5jowecx1*R|revMdlW9~h=zvC8oQ>cZQoP%$I zA?C>Jx+KedxJG0On~el9UNPo7WFwutj)otx23IMXGZj3t9X%H0pkcKQk5l)*kewO> zin6YPLM{fB@lvc8wMPU`Din6KYc6zt%Mtigl0N_5K?l*Yrx=2O8^foq-adx#oia$t z*+fSSZXw+S3jH1+QDp+(;A3Vv0>C`EUkz^(O8|2a?MTWjtCcTKn-w+eh&RjS@NC~- zBm?`jRu>Op*4S3?SO5iS$ivZ?gg753`y!X4cjxb7r3@?%Iw4voWJS_U zwdtif$VrWLfHnf<6j=hfi1JM53b{IinHczo*#RHY4Gi&#F>N0qfS};KPjKMnHb#Qim@7Wx6?%bwQfGU-(Rj?os`vufuR1R>szt+IF z#A`}LG%w}Yq@ZO)@vcmgTZu1?hL z&Ghfv0BMXGG-wc;R7SomTTpGL?HSgv@J5Rk6gw-QX%e2p8XZsoZfG*npuJt@~?3hh3?W$WwbPuNjP`AVs zYrZL&;?2)7;3sw^jX8zwFTI+givAeocn5Yv?#L!i>!u!3(%`3Xge{6%yoMEkCTQ}c zi-z5dM6@1eu&&m$JW`6KYs(y&gJ(Qcj-BMz9&4+0!^;flQ_c^MdFa!!yGe%2A1}r^ zRIWg2or``tHVvvKG(P41&uYdT4v7MNL7Bd)49^z#uSF%uDuApQrl=UT{0eYr0FHr- zR!5sYVi)vDkMbrh>1(Nj4i=7zZA&vb7uI3PHE!o^Y!j2_%}UuOUc~Uf?DXCPF-#ImFmCwqDr@bOS4T zL`b)iVU!zyIumD4%WcXVNw1zl^Vw~Y%kIiINwvl@2-t(C&tgo!K7V2b*OWor=TIa_ z@Pbc;K%)!2>sA0^lojv^_iZtoFaqSy|MQ;VOppx>SCkqp3=BB!i|bj01sA7;&ri3Q zO;6zS%fQ8iRN*GlB~KxMMY9-EK7!iwMm=Z>=jOp^OexjY;Qb@J&nTo>NPMD0u(k9`IW-OiYj~Qt|ZR33PWhmRaJ2 z$E-&-1ApCo4E40ekMzI3QAEa&iJn|Z0Jti<{}Md52cd~UxU$;-h5afOcDp4@>P|JR zq_>Y<6)_Q+3xD0+?ANhH25bKyh@oCR0>UX^?3c|%!%@VzXuU83Z4$$tS~ki(?P-Wn^LxKp#5$5Q^ZTlu+>oJCjl7JqjuJ z(A>u&6%c@}cmfcJn>TG((K3u!xaox)(B}jMi}2Izz{+V&6ZCUdHf(7(^R5|8YvRS> z1rGb(`NluorO1SBGpvj$aOl0(9Y9NQ{{D#j&ILQMSH!f_0xl4;syXPm2r-!ljT5_F z6b207?g6A)XzA#3;2=8;QO5p%^Gv3*?A?=$^-NyK?}@z%`oQ5w&5@%hHkr@Wz|2mO z^m-%Cg_@7!|NW2NRH?H~P*6}>wl>g0cQ3D9gY>4%oY_=X^yJm6d6c9o+~*0a1|*aT z6#bNg92=w<0rn-)D;sx;Pw)m7rF!+;1d$Z`>e;jPDig4Q5wGFAddR|kJ8qQ`DF%dV zHW!mDaWU1Gk*l*qDs~s%rimTVjLqqIqE!JuZK(-0g?6zL67|m2GR#0UE5I%{iOJjp z)jVi8O#y`S7DU@Q3nMV1lG4;ReC|_Rwx(b3M(*s&!kj0-%tfJ_%+W{4wL_d$@UQ1? zFr+UQ(z58B?+*sj+Jb|=@kqT#QFtX$BylpEh60$68CnTb7sc$c>rpY%*JFR#oU-wK zc^W&hrw$lR@N1GbhmC04mJ9J5uDU=e7BnT{pA2tqv>rYBZNDFVebuT__?kQopBo8_ zG#qyRkhAjDi1HF7Oq54);+2yCwI<#h4-(MCv7?{dppLQEKuF)m?SM6Z*Zbbx(cg2R zw8@Ykr*0B(p=aLTYRbK%+vdGkBr8679QzOme6>?aZ~Z9f*n@Tsstp0U-w>>Ne`A~H z4VaIjy0Cu|E{OGzb?Vdu=!LR)SHl)4GBRk9R;3i9T{!{4YJPM!f{S1^Uj(5a$sO6A znUqeJCfZUJkaSs=CJGSu{YmOhaP;Wvpcx@E<0diuROBUT@`KCg5@~VN!8J5=D;I{M zb7cuSPfuMb%F1hCrP!y4y2>j|g$PKP(*o|~_Aq&~2gJ9L2oIyzs4hTzGaFbd&$GA*pa16ta@2dUDm{E9@*-M~6X z8DMk;59Z`|;G>iiejQmy$NBu^65h z%xqT+OSvEaK$fA!Vl&Zg+rz-mFvd*_IoG=P-dHlEVgVRnpR2)<9wi4;A=5Kxv?cKqVuJFHN2?CwWR}$Fk-I!$n_zRaLw^b7`k*tsh+6 zmUR^7ibt+HVL&iCAL~Y6!^0adOl361?Spf?JUmLy_Gzy~sD#$Md8_M4%eqWO^T4m} z-Ji2?iFBUB1ww_dB=P3W(zEh**H$Ksde@ivyVmHEVE3Myr0+RO8}7c{2vRw%3mUg8 zki0u3AUS3Fya-RLyzvt!TEGHx))mx3!&dCS4GMVI38EfET@O|m(# zQ-8Ge%cVt(@*Wg!Yoy$5Q4I>XiQimCLXq`$6*4jP8Sg^s6R&@V`YoByWR`MS#zP(FXimfH*s-P$t zPvax8d*%buDrOMs8vq|$Nks+6(ACj#GAXeryU$859pRMWhf}nzRT-Wc3|Vque_cP(_c8D8`rMQ7hgm+=2NJS zHey^};1@VJ-xLgA{>Waor{6bK$Xl4aI-k8?Y6e~|X`bh6DWkkm`u7&R~Qo1>^+wSvO852;KN(90F^^ z5B|AQRdepcM~|ka6nrc`+sEuWWx=+T4V1S^dg&Whwf;0 zWloOgg9BGOSIlG^Eb#bnvO_+pDF?BsfB&AD^=T%RWyb{KoU{bu855yY;>dzQ+yp2 zgx27(%H5}+WX|?8)7qX@*#h3x8pWWl9GaUcSVwyrug;*S%zIcDon8m$0dktx=zwug zrxXR~GuG}!bppMO-_XB*|2xoX(rcI$c1?evI9#cB2GLmEPMhYV;T}I^EP6(z3HMb zj==%w9L2u>`PJ!X@a`~q+xKL*JUwVvH~PcZiM;!MgFAObZr`K73@`tOIe4LOS2S7^ z+rxce&I9Hhhqd&l(T~}pam*}y_;3jV4oZHGIWb72tfn>%)nYjpq#CrBPPRvR#Wntc zUwjR9q2;}Sa;=5)Q%4L(|6%v#je*SzgXmEz zE7Q-P?gw}Po?z6peN5&XI8`^bPf;#(o_T&-b*@X@+immYM`BLPo2|GAktNe12Mt!J z4CNpupap>}OJS3_I<_*3P zTk!ED!iZK^?Wq}L5Zv7v9W`b0>Su#1%J2HS|77 zx_QJH#9YyJlzI}$-+2YYc|zl^tj}M+dRWT^(`}1BlqkQF(9qEHTO#)!;&7Sbn~8~> z;w={`A06*`I#f^L*{C@?fL+CTr#Ni_q<9C~?Awmh!%Jk+hv4^5x3=zz1FSXu-@<<& z3qR8@1cH3Q2Cfll6-lG-zTHOJguhZe&bhl`g_ytpCTI$0>*z#_n=@+F2|$l$ZOR(~ z2%aXPaw(J&{P}RFy^m5@`$@w60zk6m9VeN(k3BiAueV?tha`Wqj9~ zOya3MWbxnjW2<)K=`sgK7U+D=nr6JBqNZd_U$Z0E8-1SccJMFM*+U@kzUL6O+V0;T zpwNHqR>$chUO&1BW+5Dk9^Yp#Ui?{)_S$v{v&zOM1PQxVb(kps0xESCoSxISU0ExP zMi0hsAGib3%L?pm33#I7VG|erip=wz7h%45u{$QbQ!jU0)j&978?m(NI%cHTW*e&g z=Q6zSyMgKW7+$t6iYiconC%?B@ZQyZ5wioim#bqTN8i*} z&zK*O9EB^QZWMm^KA@}B39&ITT2T3=1~&nBNAF-!hh^3~2B9-_I<(FTiO3}pvo^?c zS{@w19S#x3^CHupo8k@Y*VBEen5@9lyag10Q)Ap4k8^Nxb8Px(UPAZs)TlXoJXjZN zQ}ndB48|jBH2kzox;+T^Q<$dX^soTrB5Wt|Kr%Gn)96nxCrzwlz}%li0E!T|RkzKjBOG!3g zW3>0qXM)x8C0H`>Vd$`7A4PG@@SAw7_MMJ;s&>*JY6u1>v*MkH&@^W)lQq*3I;^cBJK_NMXy#YbX zWSM=d;V^dB*}`r){~ii$)pNiX=OSNjJ_8VT377`FY&XkzhlK=cTv#KFjnUVid^Tsvfy>YaWJnn$?AYjm(TqpU~ghT7LTK|zt<>SYXAJ)lG zCQyx^2|mHPvZBIBG_gN#Fs_q3*oYBF)*f6o+41IPqFZPU8hu-458|zpZ5lHfLFzrOT<;_7fZD~V& zeM`e`Sw^-QoNoz~qLe^>jcx1;bQIr$O1=}OZ)oTQ05_Kxkk|ZO3QCt3ppC|#UhRaP zJ9cTws{SxDVDoAwh3Y=MT%cEw>#v z{vWp9Jg(<_{r`WZBr_q*kUd$m70RA6H8W%#8N13ZOBy1j<&DNV2qkM63S-GSg*MC3 z!bC-uR7T;Alth-w?|!}BnRCwP`#XP}bGv;$Gt}$#dS1`#dTbXmLH*~H;0EG7xjjb z1e=$D1rL`1pFRy*w6Ns*+)9-iG-_1HO0O*MyhOc>?!n-EK8$WFdUWR;N=Qd*H>bVy(J7?``{!kEtfckWl@OLA9DJTm# zEJN2^3ge+!I7ha(&271aN{kerclX9GN5T6T@2=)HBlHOR`eHF74eyeOg)?OI<6M&7=jkaZqi4V}o0Z6Na>4w+lHIJ{=p0jj2NS3W;eF|sy_3(?&`F(u-JqwsTSQs zJXG3{6{k4WouP1iY25~M8j)t0x%$5i)i%o)2R+N~(|l?e)s?!=Ox7Xpr~N-9gJoC? z?)NCk%2?`AEcoq;SJ5_1ejh-k+K7%U_|;W(P^tWJdrKJ_rvvxMF%yBGVty;C>{;2% zdsF)?rXFm6x3h-HIpfR0%vY(zbhWsLaw%qFmyDS-TLs%^hHsJuTxeh-Tei`-LEr{u1snh6(426ox$6d{%{CrOiZrZ$gpb}$Sdt=KzlU~+4cVZ;23;-<}*kuwY_v{6Z1 zLT@%V0(_Lc@E+f(fp|H55k3Y~3{&jgpq`Q-Z$n)sPj0EdSWjs=&&|zV9d_iyqxY;c zcBQoLVEi@jdh08m7f^zP-k)(gnKWLH4^IW1^D68On7i^4wLOQHJ9I$X2J60;fY`2e zSEevEO7k9BN%W8zcXVSQcSC0ojF;cm{r1~7k`lFzPL3h81e;ANyztO#KJ0Hs(P$cb!j^Va!Ha4*duyE?-{4>-d5(AbWVH3TV<2b58Z zFFed_VrqIb*ucKmHtyR6pA$K1C?~Aju5DU-mBZm0ebwSllc%*{-DrqyoB z$#IFeHe(}49SuZCpPoI(bZW3JH=!q%*~2v+oT;gpoL7%|*RDT_-&MK`)5hpSH@$l0 z5mAPTwtd?1@*8FM#SHG2i}tQixR{YxLq2W|+~`6I2@c#4$@m`d$vusCBVcoiX&|#9R2+p|TWA7!*9dp_1d1m8v zDaaIO=MKChM{_!C`uwd#?<9M@#HhVVB3&IiC>JB6qJ|5i=YxoR$cqQxJMxUONi--y zQj$XF?ek|1$6ge4)7e!8{e-Td-~2Ob-|#M8=e({dCNucoX|4`?UYYWDB_uN!0YXQB zXm&7M*JI@h^HcaMn$}O`O-KEZ&i8Jg$ol@FpkM^VX6b8ypbVKRf1DX+YZ~+o=CBp zV+}7`9RQrsesGxpqkG+N&vXpX+39JJXPcza(T?lXU>rGjjPkh)`YTE`*$XdKz=bHz zNNdv$+$@52{BkbScAJc#zMlyy$)ej2?E^N<0KB)IvF1d^y?Ykoi*^s*2f+7Z3;$6} z=6_8&`K%@)SqV5P{jC=%M(oyzQg85fG;RU!AJzB(S9jpTg$G`KFvZfiWt?d^=7iJ> z&i2lnn!xjI%Mgw!CL{2ok>A^rO!J>UQiA9L^N(M6l^$Yjf1=noZo#vgEd`FH_h_}^ z@35rg9#HfA+;d|br4j;w9{HZ|h7 z-ovi)T{&K%1gfW-7b;X0r#qBQjhQH(t>&c|DMs_L#W_8^f4>9e$+@wcCwEI2R>+8G zs1G6>=w~l>Zrk?inO`CWm@~BRMSReR=k(V!m6`S1F7LH_Sj3+X3Yp)PfP%hfZh0Si z{1)w%#UebGV~@p70eC(@dwBTqZ>n8p^~!j^r1uT0kHqP z$`F(tF9-#O?D=PDF-SPD+UwA|S?yKE#)pc=>t6oRUBeE7-nWlq+ttHTNt!r?$W>cd z`142UhZv2XEH+1!X@YRjsaeB@Gkp)%+b2sb`T0*Cqz~^ zwhM?Qc(tQ#?hQSwQopo^T>o6~iBW#5dJy!HleusmlYhj!I7{%iKqg@)TQ(ADY7HC6 z#r}N5h7Hn=Cnf-O$)6eU46zi04ycw+o;J&4Y}mg=1u&3S@CT0F*{+`{8+=Mq#Ews!#Pr4 zW_r3gS6^oR_Cx-mzOi zt;tr=n%%<$tV&jH{FG>>@{ShO?^pmxr{ z1$%AnQaE8sI>Z5D zk2^JOP~vF}3LO@W99hrdCGFp1j-Hb((8-z&8{)NlSg99T%Ep*{i1Df2p_&IuB#QR_ z@x6QPhnr^qSy>tIflQ4Exj!Vfx3aP^j?b zV%BFf#dz6{=3yZ{-@1NY!3Uyeg(o=~JkA%LlkuP?Q-pu@=QpqHX{a%J{OQx5P$&+&&eeFz4^8I?gT%<#28^$~I29?RDBmwRlQZGU&GssPaO?!^1WHc8c4xMk`xl zr*N7Ng7alKjwT>+qDqLtNgOHYy{0H}VeSjsrRCd;x*wI&pP5ipSQsjo|G9(ijOL`} z)N9eg?wh)LO7R7$0Dps^!P5_Q_v~wEXgCdYHyx!~DN5mKj9JD<^)zNOq-FwQbi)ip zhZ&?#n?X^Kgu8<9&?L*Uc#*@07bq>03Kyqhi3t6C9AuJ?a`o+$UR=MPb6^SbvEqbq zb?)3CYvf&)JgP339%yJSt%nP01;2OqW=2+G0sDJl9kuEMZ^8IFO5+>AsNe5xrgLP) z4LwJ?oP1n=B`8`_2}!&1`1Q8_TS7zIun9f2lx9RF=%m2N6ykzGu)-G3mgoI}ji5qW#eS@yCu#-uU#o*xK4U zCB($Mj|_m9>;vstH?9h<`1R0J!o@BIv2;S#mp*|Nz|)9P5X1d<+pj&|gdr&z>23t# z3@<(FRS*Fmb*&MiF!_Pz!lPoWH#K{W8P+ji53;q00Ho& z-*)b%^ud>zN~hmZL}^~$-cWE9=2jc~nSW-L(6|+~^M<3y)zsFV@*7`H`K_n#QYiL1 ztY%;%^HYx(M3_h1oAND%XJ&YvfW{2Sn z{fmiHP7Z|rN(WHe45~QW$-uxecl{mOEl%~NL4cnl_i%UtGT#QRQR1I^tegdlYKk&% zB!lbftfduizeT0vgh#F|=&nN}_OS?kH8I9>fB-j!)+^{Fq~puFcdyY+HUsYanXW!l zh!_??%SadFZ&zNt!aX>Jj_gdWjYZF%jhLM~M~?YzrsUA4W4P@}^;vlNb3ksqf8?j_ zK*k|BbZUzN3QSc?L5^y`>Uud^?7RNZ$X*tgTYO$xZRY*Jzb3* zKrbUoL9wR;XR2QTIuiD>wC8a}P8I2bETY9JWZhcd^TNM1A5br(`qvF~-@&-0OE1JC z=aDtF87E_6dP&{RhrA0F?pEuuUU#^!l)f4s5YRucyWIG0RrN>EGqtHCJb4qi5tqJw zSa^7IX)0q$Vp;I^`Og`@aJ}-igbFdovgkCfgJ<>c&~vFP*eqM3e0*JRj(0|{Najh+#&M7`ps%*;+0`sxG`Fik0yP=Hq;6>+JxQ!+oAKI-a3t zHcv$?VlvTm@TGpuMcgOl*>>TA5cmJFy8F|Ih+YfN(<@OqRmQn*es%n7S z^p5;d9!bfI5AG^ze6yLpeNf+A8?$x~cFUzt3KStWKZ!uGl(k4lj6E+GCa0!4KZjlv zMH&UbWESd$dQF-fDfO6kqQ_o2pR96>$a?UgBfMccNZzn`%lzKEvtLyA{6x>^{^pq7 z(m|%tGf%wfVL3tH-gDrvVX_MxZ`z z9H(!eKGnkY6f0{L$!Xbgz!xMZ##YAQSJ{V4`8ybDY~5mAzSfAIn785KSt=$KFR#bi2nh(-zG@| zk_NB+f?iE!xV%S98x!c#4-<}!KF8Tt;a2|Cp*80}1PmSH( z>imA~L3Q&!Oit&UazY6z$%IAvQ&&lva^UoVLtUGA%;4SzxYM-;-or@;<(;5t5=tg$m;m8%SHio%r1Rt+2F$1g;W=CK8+ix4RRDD z63O~XkR<>{FvB>1d+?E^TjW1hO#N(uHTrv?ZkZ8>U_!lQjkh{TKs}P;c&7iMK&tJG8<-%z` zRd#NVI*QRa(dIw2W2@}updf>xYaHcLmyZ8~i!*S}Z`>J(y)0~VXHHnb`63*rEx(Ra zAm9aTOU%CC2;j=a#o1vIBBLSOyM6E6ID4&-ay)#MloRNi@Tv%K^E_Y4RXN{w(y3`R z)4J+qpJBpH;YiOsKPHq`B+a(0X<)8?%i~1Oa_sh&vJuom0AA5s1^tc=#RpY(0rW}a zR6NC7m;jQOSO5CTl#{atdM~A8%Ll9-<;!G$YWD38$L2hjqph~ZxI%+5x^YD@{A;RLe29ZLE%wa?>}+gs@(mOO%V%)3EP z+QFS2ZT~4X<#Ok}_p)8-=lr16%8NcV$|&_o>ryoTEHnZ7 z6UcBS@z@JSCE6#8IL;Q_?}kXn1k;zk791M>8pY#pZu`F;fADu6MHS{7jeC_Kzn0jU zwCh{gx!_5#&X=R#j{9AJo%wVHQ%>KrZgc5IAI?68ojOiwQ%)wK-Y^(gH_k$WEwm(fU0#)nMt7R*fZeQT`V|lu4dLFDR+=n^atY5nM$w~@g3bdmz(8d>Mo zGQ#b|HSTd89;7wrR4Yn>kDoqCYrpQlTKlOd#}3&Uv=zDgQB#&TCgA) z*FcB>u&NM>ub}WATeaPfUE`=6@_#=Oedfq^ty>QSf<_;yWCwA(M>Gd`xN`MsWkCX@ zUa0!XfCBj7dOrJ^J#u+7CIRikErqzypN3n-xI^G=RrM}`fb`P|Koa)GBa@G8h{IQ5 zoyM8NwaZO^6>b>A{#-V8ApBb}7f`HZV_><%gY$h+gyNtOfl&@{>5!$9IMl{ZSbQUB zkOg@@abHUb=fGZI^c%yBm)h#Q5PZY&UsX`3h`UD2!%{4&*yDWP3QQh^K#$x zK}hhk6~q^D$WX7tt; z6^-$gkbZPPo=XK{MSJ8lA8VH%}Dc-r2g zs$i0*2G}%X2HVjO#Xb;06x_`|Sl4!+%J?z=r@2q&YRs3`4peU21}KIkY3(GuEqLy% zn1>!B$zu8&tR*ob6|~SMg7;(JT8rG`?Tf6zGBo1%-vrl@!Vh9ioKAaD;RwesKZ&-` zCM|97kRc(knO|y8<9gMM7V)}GuB$2>x|Z9rGG4iuOdI>eX=5+jt8eX$mBzZ_KlT1~ zPzSS{EV&ASRUe3!E8J@GRp4lDETj`B`XQNM`HNMptL)*u6Ji+C6^OqZCccQWjNx{* zK6a~b;;v$OI_1@VnUhs45D1z-DWxp7aH&fT5E~b1eSj12GqI3 zNlz9ywlQVjsjBSvYw_p!W5`COT1@T6o&FMB_|<{DOQ$Qs^um6(UWM zx0>;$lE#kw|9(?^>(E6grn8ZTZsCL2nX_g%ot=CRAiDB77!h_LA9*qjKs(H(k&pTN z1eOqVFaPP&g}cAX+X{nge`2Z{w^;erd$M4mzC0EM?e?C@vvj}NG2mzx47W16{F$wDv=En{8r7>zc`DID&=b@eSLk={jXo6 z$wP&EB(wIa71duUD?^HIZea3+wA{$2kW?1%+e4465Cf%nrr)aG(gkR9v*XLg(pFS# zC01*qO5DvP zY`n!-(Ba}SID4-YE{08*IX=*f4KykR&;*=;G?=@VY7t<(vPk$^=6tve=KR%=k{<{& zxKc2(Y|?qL^kcldMdR(cpX$y!UyC>x4M~iWwto;c`KW7x3mWeEecnOVrEo*1X&lUk z@K4fwuS=g0yQFK;Xzu$2@ffQ;AYFarRC)*XmY@M?J$lZpjmz%SmS{g;1Jw{wP8-zM zIyiSNLl!+k9B$0^71CR9eYCN$`FU^`rSj>5bJGCTR9p`qUyX_(xEIS=I=FJ2cAJ!bLkvgaFEkUMRRe^jq*7h(47gXpA3s)d{#p7q;a zmpj1c)}6<%jn8kdqnPyr<+WI(BXQRc9zMM1yv6~$x_|UYDsB9v(G~Zd_ag5;zEdZC*+CF=G}AWwCQo%PzhQt z*F849j~QU8#qK;JEdsEVU z@Hd{Zn^0Yu=!!%W5qrlthS|R#DaVhH5hH@q{d+QiihhkNFU)sIWcc)@r zF2*vE!dFV&zp2*!e-*cyou{du(=$c_#p7AX*k*Kd{)vm-;+sV4Clugr+V(f#Z^$g-4=Fo+#hny``P~01BuxkIiXVn3=73`dWe7G)vK*w9-_NGXmO*6 zD1y-Q0JXzSfJNMA&)!VT)?Euh*#iJo=j;^>OjYeY>DS^I8rI8r=sR2!BicAy8wI{!R<^QGA59HPv`bk6S zRUD2XXkfq4{PEG)^DShwd@*R56$YaJD0NDs_nSNM$V6Z((9TmiR=?2wje* z@yedf_|x{zdbu$5siw0x6WN(lRke)6C35Cq@yojHF$?O@Z4@=8`hn7;)WL&naDM^7Zq=yXv(%>N=C1y z&-+vp6tAX|mnOpP5kKbc+i0GU?fOA_0&2n;R}{C)9T+A@UgJ>^rdt~HVTGW%&F4OD zuh&}a3%a)R&X_I;!*OHhW&I}aoP#`4U~pOp>xheQdo`^s++6|1)G1Ipb`G-Df#%A0 zSJ2%K$qM7mrZhKE@+3Z8wcF+F6f8xic8ojRxOsI%GbChwG~%6%_y)W;^Mr|(v9GT- z8A*arpB`*`_aPgmKvuH{&wci`v@{9S^NJ`qIo>@F-AU)|nDrA8nqdD}d9Q)6*0??0-FRVoCTijCg=gukx2y9#o^B18 z7cTu=T``GQc^g=`0VnHjDO$9vXs^6y(bxdHw~yna8w_0~rfsWvZygpg2g3tl>o_^UTHqQzRW7cREA@cewuMI2k;{9`!|T zI}vCQxz`w9X#a1bcRtOFPA%J2xO5l~c9nXw)8}dXcwy5)!u!;#k1U~8;XLQYp+zgN zGsur$Km(u6R~Y(eB7G{lsr{${mFOpY)jHxF-n3EIDM68rE8fHnJN2Nj-x0>nIQ3$e zA!H3c<}IxC8lV-;0o63bdm)w92LC{a==jE9^K55axfyPa#R*9w@CYD^s2^(!BE5Go z1ib*ujLc2onB$j0O>1(U?#_<^I&!JAdBV{ntj>9Ay4WbF5f_~Z>sKxUtgn*jsYSg9 zaTF!wCJf>cX~*)mMUGD><>nJUk@HMfvWUZh?lYv48K?j=;`S>)gqm@Bcii@%mtg-McuNc##7SL_Wh>9H}Cfs zk3Gwku6gjpbLn(%|0n=k&k2jO%GO>klu=%BJOPq^?#EJ8%{mw(wSF_YpQ`T^?z{@^pet2SqCQ5+trC4JE2FXAD%Rc5H44ughQUCkw(x#7GuK{D%mmW3rGC{SKLxG4M-~cUfgK= zZX5R254I!|RZ4C{CdXu&{|_dde^o~-X3YSEu=02#ZAj|Rv!N2GQ#o5x!uUqDH4V%T z)iO3t0(6}_{*#$`H*!>ja1;8*p_@{9hP^wgcME~~lth~5_v;ZBu*p0{h*fduHQU4W z6kD1;xEqDOmbQE*Jfd{EV)W}EF)F8K0zl%heOPl_lr;JM_eZ{V$8(o>ImamORCK=4tialkb*{dM_jgswv{^*`J-w z?Jc>vGvO}LTs^|1e8W2P4?5-H+pPXs`u98AgZj#Hn4PW_&C4GbUt>DYc+zj~p<2qO zePW77(bdAb$!YT9cly%tG~Ub^zaoEh)bL4BlfH2B$Nx)y(Zo87>L-v5pNZ52b;eTl zJa+vrBUct8{bRFesx5-4k!Gmx@_K$5*ty7TA}$K~8ejQwj_63!qaPHOkWMm?lTIPk zO!x)Sqqgwp8b-1j%1PgBo7};`JC=yIkK?Rq1@9#x7+96aBBw_InJGmuim z8q0xj5a}SGzaMpp8$V^r6j}$X5CIPOhD!E)F28mr^3@oTN6Jg*CgTI_gPp>$p3H4E zbaPwO*w>J+ihH4VYTB{dylA(9P3QbF@W^FdC+CKLKn#cxV*sIKj zLNU0hc?A6KRbq$4V~mPhvA493yS4wv8ik4PtJL+k00T5lGH%{913_>q+L(3%S95Fb zrj*wdux|aDD<%z)@>&q!8hszDELu$KwwK(n#;X9JC5Kil(YEsyqpo}(?}~DxN{Mvu z#0KSCSS3$D&Tf(LDkikhc-RB(gA{&MYX2;!8i}s#g#o=SjP~x}7;tX1YrNmNyK{T2 zPi7!K(aZninb;3cb~2OTt!I~vv~<(Cuo`dvFa}AdAUTnhTtG3y-=4*F7xPbzxsF4B73$*AbSm&U@3j$nfMwiZf&l=w$MYKb?g zdr}o#D-|h?KVFHcyr!77HZ?zWFn+kVF&NS#8817{0McSOv2Od7Kb1mCd=*2Zu~$8k z98drzu+pBEG!00_Lvt9Gy@j0g+BGbnDa9L^)wjR8u?v`f>hCw7F8v#n{x)yK z(^m43p2N=&CudT(>!fuvX3RLXslVc=p)P0mzooJabvLBm{Xww&t7?it7@i3gtRx98 zT-UnGCC%p^k^?qxdZsMJFKW}>CFpNn`qhXQ+YX|M?;S;mBR;i)B@-v^Iy&~`cSlG6 zd8wKb6wf3X@%N1zp{T!YCtDrD0+Yh{r+TYYYK{OXq2@-1hK23CbH3rX-)d{>o6Wj? z=S~U^$*I0irhu-Cg#~BEi<*XsMW}&VVJmc+PoR_Btm+|eAg~pA7}SDw+MMN3*T@V~ z0rf0Z>)l5LdmHEG#Ez4boFGL56T7aDtH+AdWU*zj-L!&dn?fLKOVMKj0CpgJq6GLp zS%gKfj**W=!OP5%%91ITJ@p2lUF|4y`Y?4)SL)VXJe^NywdbO^F1@pQdrJFm)cZki zv$t`_b#2`6Duszt5#wHDt&W}x{Z{28iBG2Cv=&_635MD|Pg7(y&+M+?k*czmsq<}+ zo^?q@QKI*j=898F+}FCJMIeW`Hq&K`?!(Fv}8d?r~Zkrzi}W^7RR zjcw@l>(}l7s;121SB+rb>b=eEOSQO>r#j&o(nX!?s!nW_oPXh&9q^sK1IlC4FVKZd z7bdLUq&-D8Bc&@g<6_?{7klNRH%tD#*If$UQ&$^!m31OE?OSrI$zV8Gh05*?sQfTf zMSH?|IfN1>@%sxzp8W@Q=8W#wdGt==ope-|-{Msl2xW>oGXgVb2mCivi(8spmHLjb z_kZhq19?g0+>DXGA>$-NsAOlrY3_U%`&A7f+#d6DItdVy0Ham~1vBSeGy;7Ba5fZ!Nis7Y>{f}K~29^zO0)lzEIYnchlO=X=S_cR5Wj}?yoaHRYp0O z`4=5#_N(5BCa#X~FQeX+lvqc6Sonp0ZD#~hE+3lcZ*WWPW3ZR`C&uTC+=4G{I`%ht z%~>yOYOQ!3x01}0ZA;>Ea>IvK%!WP^Q4txP;^-~7OYxn~IJitD;3eu}5f|qWG$uq{ zfQfCkjDLSAGmvpdgsUp=mPJ_lkfOFoa^#4&=fLy)C!l0(UjZAK0^vHOC`#X`g-(8V z{N?xVD%%<~0GQj+>ax`|khu~9kOqM{D!Q=q0eUkcahrC(mMsxxyva% z-K8mqF<>FFbTZssdGWSGg@AWDY$&9I$sgPrkoX1UO?b=^CWS;VoA#6gV8jAuh+ zJVQVs^6ecPhJs7W&52qIuEi*XmSWqx17l-6_x%QfBJ|MS2}Enmy?bJSxM(tbk8+HV zTAk|$4jAeG)1_o?5micJ36d`HYX7Qh9=)PoZ{;`5J~QJn-%AM2ZIp8RzY6*T3M0A6 z{MeR~>EvD!vs(I0k=RONibN{~o7f+KUD0@6zZJb@#ZCxb zs26O@PONq4@Zq1H3#iAx>W5LqDc2)fhH>mc)w?y9(*vu@zy6c>V0|yylNa8O&Y%I( z|I&t?#=7}lV;b#)`ng?~tfI>{jHTGZv|k&T&B74#0YOOtg7keYQ{?5tzj;{gBTK|$s5?vbmrf?Hi4K&G!*L% zF*c$XI?M?-d9|TwZ*$frS-pLFeS0f9_E}>;*QyC*94@e!hKQK ze3;)%(?AY)P2J%>cl11yw_?9BBWqWBcF{RVG_xR^vZeXg&#JB|dxO&t3Ps@`;f+3C zXvRp#ee+S2qGH?$fi_l33qT9;WYII%R=^fUdOf`1;uzijAm9@VVR80zeoHa zAs4DkwyoEP@F@SZr_8xq!15*gD6Pyp5285a6ZVN>obYL>sb8nm+NECFaCwc)Nwau? zvOXnm^v)Y?xtH#2LkhK1%zFY!&6lxAL@?*OG5|U4=C%&W;$tpnclQs6ul?4jl_Urs z@A$prR!dip^Oiyv3=dX6M zBuZKmW|7ah-O9>pCo?i4?QI)6PdYzJQuR)bzqg^8(ae7<`ef94faU`}&qbhSYzv0x z8t@@P*xJamQ_aoArX5)RxJ5S%zqPd0C~E`ye=c9>9EgCwUn=iLlgqM(RPfa4&sm); z%HSoI)r)f5FCXT1lw)tZePRU7M$^q_cz+1r_a(x;y2r35926$)RgL<=C9d%-j~ zjXu<=2)I=IMnl(V5`ns$3UwM;3(}gWadA3DF@g`?I^xOF_%Dt5evXP;wfxg{mTn|R|qPx2x;3IQqB0|{0ls?DF z>oKEei0_FP?SK-0B@KYs?@|$&2}V9YC)Q`c(kQ&v6K>_^T5{Ob_vhTW{15CeLY+14 z)0d{wht&m*ys9fL=hAEa^SZpheG*rg9MZPcOpI|06yW0DB#t$IKq5hMI!%lTPS9Is zBta(`_wS*>R+aB?^+4L26@;0st#m zRb!rwaK%;3&dz~w!V5lElr2nXZqyZ-Kl5ZhxsPFt5^L#H{AbXdN;aO{4r*pvO(j8! z_&}9J(IidNsV0_r|MAeGnf{YiXBCqTeGFq4oPT&&rBU4IrcaS>8R3lvJ0mhR<`<%i zo%|_$8SA1rd=#zrkc=4Ir?xj6CGfp8(4~y7f4SAQG}rt`Ej4hYWYb}x4CYd`o%~%$ z(TDqGe)MU>B%7|VZ&I)B-MeeJ9r~a#QEz6(;2)fKBs8F;RULX7xQ(6FjTgA0lu!7(ktSyC~L~k5}$(V?fY+ zdF4No@D=LXQ>y3u>nn&?(?#D61^$JzPwTRS6V@`2cMd-T$%?nkJ{ zl4+EV?YLp!{_Ni+tirNB*Og=hUUgJ7_DLxzBgM%&bdcsjeE>Z*;X?8on2iC_4L0z` z#mx5F*T=nvCUEHPK0i|k0*(+920k!n$Tz--`9^*UPt!xH{rkHNH(gDvmsD^uOI4z?Dj;$o z1g#BuGm67LbZtHjlhFt1&i393j3uIk5bb6FCzV3{IasYLhGxM!fMw{1z87(Z5T;Oe zKu=k1#!y^>zag1m*_pn3x&h);5+o!uv-f8`;yw z#@(q}c06ByhV?Vr@{sRijg}&UjtoZ+OO8Qn+SN^9EfzwM2NdNPRf_jgQu526(8@`Qf2sQ6%i$Ij8@YE>DfN1~FiiCx z#tN+VZqoc}ioF#h+=tZ1K)vN%&}{GKpdXHk8dKGfQ8=Z3O5SAv&g%a_l#21MNZ}82 zrTfwxY!ylR8s*ty@KjN%{e`ARl4>P|rnP)^_lkbpU=Dr zwQzGbfMMi~W7OdfO$r)dD(XP(J4=3<+JEQuwSM5eqP@VeQ%TPWA~SA8!9ndL6Du?*6;pLJsVF%(PPpPi)195iHzv8TCWZzd1J zr$IBFS@Gt!XCMC?L5Wh042V5&X&|RZ^ClZjWZPS*JOGH-J0Z`Wn68|BzDG&iVkYac z8~_ILKwm#rXKxM*YnTXmYVd=w-0j-7UC`oZ{I$}gv1xC-3WG@96(1V8fudK^;YPK0jV^Y<6t0B|lDLT5BCv zvODqrfD#fk{71PL3l?ne-+jszBOyv;zET@H3%$H3(XzKz^=tOiboi(y@vwIBps9Og zn7;Y@zO>FZwoktA#7DHOic9&)mvaKO%RI^kLu6ne$@rRZC(58OPx1^M(K@n41LxoBEYxeY|*X%(sge>uO z?THUqj7fu7S{Y)jSdAhX`Crq7@OHe!N!cFJs(19My}3D~nK%yCfcXX+YijGd?Jh1+ zWzhn*G5yMDyp5UF;5$4ns+$bWSH?+k!QnrZAF96Omh5g$y%qYcqb-U?ai5FZ{PS4`TfvEmZq@ex7Ot@ojZSf*~+@jSCjY?yT?5n z4`9hrGBaVw@VNhS-^r7pOL^~v$3^AlD{td@riJY?je?mKpg2^@w7hCk@kG{NN_BE? z{A~5b58+4r(+ZfbLOIm4?C8Q02l0jFFF2kp9{_b#{>56MMD>c@67XNndiE- zYxCy^eri&-Ph)gi3UzpW!4|(1YBLEOs3=umE|<%cKH*_u?NoRShlE2Pe|$F*3K?np z4YH^H+xz*PQaLW8F;$r(xNw{$bA(~Ai|l<0F-~0noWDIPp>lkV?~BgoytL_i_neb4 z{`MHxP?TKZm-UQ%WqQ`?rsv+e>DkQc`O--vGYEvFRR5vt8^K_#!|5>$v_fmu$H2T_t1DfWoR&0TDt7vld8T(`OEtwO7>B}L??9khtZ21Zpqg(LJ~88Ctl>!Pl^O=>hDchV)?sd1pf1S z2@SR;=?(%i*v~QQT}f4y+b^cK#Lo)W;tevg^G@^X7vN(gN~$gfh<2Q(tk*Mlrn7_; z7-_sFgJWVU{u8?IL%|o`8P{Mv%T^MRI>ym*jf?ZOnKMl>8$v#b1iPvjEHDiHr2Q)X zSV;*{(n9~b?oX!d#rziG*pJeRZ5jk^Dzl$c*( zi~B`*P$fZQ)QdS7(k7E%i)0vw%QTaSvD4?Uo~@Rj!KF3-HB0e|%xR~`_{KMB>gE~$ zbaw8--ip~04!?xGA1+$D5m}OpDoXQ$XtZWI`?|TU)Y+u4&PfbqX!{dXS zC03*LZAYVi#eCkQ>Uu=mQ2 z)m@-M-A&`j%n;s+Uvxjcj@u!LqyLf=GTc-dPmEP64}R+X;jKvz8Xe`*S&*~kdN;2= zew~DUEu>-xpJ69c%OT`p)VY@XDi}VTZ`^YL^0{P^TQv8 zkUrdT^wD6*rc-uxHNi3^Ns_%n{nFhJ|N{_yfWT-Rk2nWa6rkt1bs zj(!URtg|jrS=L5fRp@&sP8{+kelRo)aD9_~>sCw1zsAmb_W7hO$@i@WztGKzp-;2( z4Zp@9sD$(hOv(fM=Ms!L>iNZ2Z*d(OLqA(W9cft{lPc4;v`v+gCa9MG9y{0T4c22b zlZC7TKl~ZDak7XK$!L3jna)HtKg#NpmOg0>6OV~xooKf21>+d*xMr=Ny=MDhvShwJH#mlOB?dO z+y;_>E;#GZqElajLIgq~Nj98l2yu~Gf}1ufT>|PWB5N@e&==_;^@?7_09D*qcP|_2 zR4K&?DWg}Jo_A2}gRLA7{2sb2nkgVkl;FHaHS{Ga>xjiqvrdie6gk;9-m*O%)4HOf zG*pz+9cG$+^Tl%dZ%RnsOw~iGOYUBS8$r0sgxeqlbU!w2zpBN-?y<0j=4Q#YT>pfB ziaZq~S5bU(>>{3i8e%h+KT$OraW8WV+voRzlhk?-g27VR>0KZ#*sz>^WvRjP<<)GL zd2A`N*;HgxAEDD;M7ZP{&3kV5A%9EVzU71vH1f;(^t7O05G-}LRoz3VUZm-IPSn8V z1?*^=HPApQ4il7l1F_Apen7ZpH#ao5?SMP?W&g#BQ_qYl7M2*K(eI%A3a^^dpiQ2Jk+?Z8gRArNs5mt$i z>Hep_*alb!(uQoaTin8~vO_?^B_!98I(B{Up6z7mslh)7AaUqY^>@)@wcr%zd2=6g zPK|Y`j(M)lqgsADFZ-Fk#_o@~3s+xD-9K4wVtPW=*qxKph3g8biCUtMBgf-esyFeI z3qS2bcgOq)E4i;bf@(oo(jVZF18|UwSfbnuJ&T)uZJ$SuTPl9~P*ED18m6tUB)RHa zb~TXwBpp|eySdri=%OX6wod}To#PY8NMWWl%5O4 zzs&;};R{o!1_InFd>z-eByI+CaWn{w@cpy~5q1c;8nF{lnws~nG`VUDn0yYC&2Ivf z5S-dM^6VZl>0D=lx+)=-JOx^&+QWimE^*hXC0xx zcMFe1T>oVeog1tZvmZz5>NL*aA$mG2@|)$Q)2k1?Cbk4=zJv~SIG-oF(+7XfbM~d2 zvx~VUe2Olfxsy!(D2gRr%E=D&gzKW&t-vt-iz(!Svhgjp^5Y9hlc4NGrV)vmf*-z} zh)s64QCdodrwV(4QO?xO$(27p>}}@n;ZdTO*y8Q^T1XE>gTOPn_220q4+`&uw6_*& zNqDF)p5VZB>(*_d)<$HhesC)SyP7!yBvrf7IRKg>wpe$k#ka#bAGDL>dyM>E|ms){yTsBms@3adH zNb27ZmHmnraG5AEQt1tbe(LrE_^U%5+t#wGQT47Kauo=hEn}q#X|jV|RFjY-u(C;7 zuav9no2c4ZJ3cURcr?PbhOn)22$#7U6(ad`uU_?NwvmU=#XeS6wx}?vezims5ruDK zWQJ`l*D1p_fz9SO?~N2^JpATpV4Y;4m`EU)k_ln$9%uBTuU8!!ByoEC zo@n{NzP%)~X$y{`fLxnekY3-YxPI;pJAzq-jk=8_UQVUTR1Du{BI&rcJr)>hZa#Xw zgrWJb%&5bMZv|trozF{9la)~`Sx20+`LFrd@ec{OeJego0)Et)=`p0Qx=2I(D2Rw{ z+MM@?t$|Xyk?R+=otE+NsZ%AxvnJ%{=Nt5(rPf2QExB3S1;_`>{W%3Y}z!<kC$?#9R$ftqXQMi&=4F^q8t7n?T93Hy7<5SnIOX+b8?iR>fE@A zdAdeAC+2hw=7y^~K0RD;HcP~S1`!=?J>0cVo6_r-?hv;`%L-sLAQg9~Z*~w05c-kl z_zxE<#K8LrYI~)^md4_6ig=}C-Wj0YullcMdiu$+t>MJ9yZbC!`>AW-rLunW_`gWi zYpf*R0ICFRn))346ClNMLpA#t=Bkm;0SF}QEz&w#hhX;Y%Y>y#%E->{1j#W%?0a?N zeC!l6uk!LYY3Vm_?kyGlqvV+V{-EdGw96;DGI?x=w&HR&bu zRSsdVs@8jmfp}%vlr@kLs`Nc>%Gm@JcM zmM-V|+Z)j4%c#Rxlkk>r<$r3mSapfMFCcl1b<@o=UW>HnYs*+O7z_w0#52=y)5xyO z(rfFRw5wxVYLYdS<80auT2)vRA%4^4xIPVEI!q#~EgC7VU@l*hTI5lz2$%sctb!nL zl&!mT>4G3sU7Vu?iWZpaa6_l~FMs-Uv|*oWiLQ_O@xr~nr1MBl252jnb0n_7b+d(n5-#pSyr}G)f6mVymqpwI!CF}Fw%^+ zvq((sxndAz>8dIxNv*2Z)+#qaravMoarBfggX2NBzdE&`ap{s_!-ge23^Tx?ukOTw ziI|Q;ZAZG}O5VOC+Y>irQ9(nPiofr8>;#r8)R8DKag;^FDyh;ABTWJ!grW*m^?&Mh zNuT8Xa8Ve=VEJFDiw-~-Yj5nab5#RB=-Ex3kfDiYXxp9c>sC{*lL-FGM-7~xqgjo= zGZ__FhzlZ+a-z^@`eXG{R>~O`lfL>1kWN~F_+<;2!fYqQ#4J9R{PjI^f@#tNZu-y3 zNKelfPDN7vMCQ`kK>0lRRa2$h=I<@?lGpX@Ku35fB;#?YbD?pk>Y9WFryg1i+c-w2 z0p)gRr;fOMaa(9-sNKT>(6oULxsR4|t79EIoEaAv$1t_uqL!5$u8{MDqXc}^w!>C! zhh6XI*{fFk`dq0d88foDvQ)Oj1=U;-|J1$8!ZjL7xk7sqJ6nw|MoMB4pVrV7+jZ4@ zkyP)(x4z=WV|ZO{#$UDoy;MX`#KHpp>sm=t>>A%F|h(gvR z0e@7gZ&?u#WUOqIC~NRW)ZJ^0k)rqi_^g#A+^dVn7@jvzKBJhdWU7@un{no9(L}S~ zCP^Wdw52|RxeJCZ)>UA!i@)o-_+zEyluq_-hh-p$)<9x#Rox#D_t0zYhVE#sc3kAA z(Qz#V7F4H#|7x^#mvSW03^{?H{WAjER0Vlv1e{PxXCW_^bVuIE&^bpO9VKx6i&TV1 zrDnX;R4OksFvs*)C#E3Fs)=di>1O)f@@&Xc*6i((XOqv+tmY|8xLziYLUMbDx_>8I zN!83W+7CHJfsK32-R7?UD53G6wSsmK3ctPbPs#NrYynKjxU(wbZ)aJm+}zJVeAv<@ zc2?t3qc=YH%lgR>9JnYAL3dj@yOmG+8g(U%Oliz$edV)?f&48Y z0J5ZB96CKCQ4g~;=xC`7W27$L44L&zoXLwg_FGZRmEYnZ7SQ)rrltk`#=rvCe3Bd6hh=~ zR(oIR_XHkNStrct@^&Sj0sTKa1~e3moxSiMVEbPw9%th0kxV z5L{?zZQ2-jLHH?E{>$MY9@g&i&w6G(jWhIp$h+ zi#$WURTt53syMK6P!Zg{-a;`c|EN&B5NE*PK7h>Q z3bsw6qOTmz(;Cm*Hjq=j3o|6<%}8YY#%8Gq0&qXXeCne-sG4H+&6oUQ?IhE>n@}8o zcxw0Ag0~^5GA$<+3*tk&&9=1BAM5lh)~Jby9eAxlJK;pdM$gz`QGxEk2jTwo_r;I+ z9R4kqs>E! z&@pR0@zvr%8&hMK(D;+2v2LBH^uz-ZX}zx8slWocr(Dq56T~YB? zkFv%bv}{9lrC6UZ7KCskeN|^hcK_Uq-E_7)`vbk047K+sLmTRPY?i%umc=!`Fakz9 zCx^yoeOCo)nwfJ-|D1bo%J`=ReU%>V+O5+2MbVtNjcP+xfl^s;2F=u&-6YY!w0^yM z`69O$zF6M-Vjw4Aw4Z#%YkdDlWnUUF=7wf{4;(#u-D@t%!zF-EgF#bPat9B^7OtJx zl@p%PoSw(d_>tqYkx|$tm9T*Qgp@ z<+G5--40D9c2`U(UF%v|(_9^X>a}2F15NYz%L7Ca#Y^+OWMDxYYUYaF2mmf7mj%eC zmfsrNIFYsHNu)s}JkYvpk3SqEOr#r$gzMLM00t6vym&FY&y+`LBvRp;_7%`UqG{Ja zKy<~{P(tChh!a`6kV1|QX=UTvqj(h!h@(ozq6diZ#@Ovd)k`O5x#l$3(`sV+IxORH z`TN%p+gcP81X@e0QI>$86>vA87}(>O3Ti!P74A%!+F(?q<5%0t!mV6YSnBV;F+wXmq{74k-oYeeKi<} zIwMr|)xoDMxsMdjBZxDnReXFK>1_(n*M+xk6IWKt%oCNNEGJZr`p0X1A8>EX6fjlU zZ{RdwfL_)b#oSOpudBKNxu0%8PQ2Oo_N$o6RbUUjZ~v{&Q!;5PZPR_NSM87Hoed?R zAQOgw`YJ4a;vT;m|1e;#6->mic%wKVXmiv1B7auXvJ8yZ^SGST9_8I@pxDyx&ucuo z>)p23HeJ__+OxmbVyV~daz8zmC>_bgA^fnu5cz@VI!8a}CS6sU`%3MyF6?VYp7F%k zCAX~I0?^{=&aNQM;!Y^ynAF+%$eK5e5lRG3G8Q<4g?0HW1L{m^y#WFH(JmrlGM~KHMO0^3RnKZ_48(wQ|NND zB^yHmH7z-!5}>{A#*7SIvjuSQy@Vk6+O>k%6S+4&H_G*<8{Ax?-$usQ^Voz>x5Z`! zbrGG7kf>!`w1JZSm7L=nYU-W+`$snrh)#Py{R`R9-TK z)U};L#>+G`ABLk= zv6puG%$eP{H8qm)Nph#s&cycyY!>bOIZzb>P)vTKWg3#T3er*#lFmMuThw?yabUA- zVzO^wF0G`%ToO|&GSr$YvKUZ@mJJsB2up7yL`zr32LZ#Zwj>O`9=qk1n9U%w^_SAyq|9%K2H11Q~DN@ z)*Ng4^77Fa*mPu8-S}^~L%2Gaq;3(f8H#TqP zW%8|w(|<(xioEQUxNVGJjI+O8b-egjS<`vZ4^%$6Q|{6>a=L+-PmT`^{58?a7+a9j z-u-&Y$`Ps+XiCREEURvQFJf?5J{L(SPQZUtDBU-jutBk^ z{hoQrT)OXm-K#5RR|u*Y2Pk%9K<sn+-4;Jp@&OwNIdOAjC-k!?90>wTF1`R^Ldm)MIYRBXX{`ovtY zU{c9$tedSGE=H(zL75x7(;g!LJePj2Md#r##u^g zIdk9O!%dnr;pLUjVeuU*_<$pXCg{BJdGDkHf7k5yTFNq^Lm7ZZ_;RsveYebAc&4a# zQ^jN#;u>_T9pbo$Mev8nBBH2ozj}(vcs19Gz7lbmO+=%{v{_DUi5S`>TTk(DrB57r z?XSPK-@yQ9GPldb7`OAa&vUYLiq6l)&8}bz_yZ@F+3)^DJ*H=79uRw6z=g#}v0+H2 zoIxiVmSvel?&5F$**AM|v+=SsaNdJIw|@kQ3MS^HRWZ1wUIDRDVuhOz+9;YY+6ChX zG$l!3$g_NV%7)c1;#dJkZX?HyAl6hXmO_=f=H`tV2Wlk@ix1-J8V~!vX4%YUPWi1f z-roe=x_w))PgyWp+G0tb-m5TPh}*!4gzm`rEK*~2-5{GyWqmq)v$?ax^*}YtilY=h zIKztTBv5?Oi%g$CURiAoL8X~cHD6Vw**P3MoZD)*zQk9VwO&+vNyNF_{i4ez)s&2{ zu}AOZ&VRJDU-OKs<4vs)@5yx=fK=Yn-tB>$%G<|o5jo?Y3(4L zKQLlCf@{=8$|YwIqNa8M6SX(sl0+uf23Pw>u45h2eeK#{7~63Bfc#IpI!Me{ZJ{l3 zw>Ry*z)_`?xpC1Kfke>xggj<4{(4kDD*kg={FKJOF=GqGch&Kc2tuTLzyC6pF;X%% zD6M9}Vs}Bi;uIAVbN9?*1N&JN9N3JG6QVtAj=497syjL%dYJ5cvuLJqyA4s#N-n4Q zj7ow@2$vl*j-)SmF6^tPza!R#PFVJC7D>m=uvokQ zZUTwuFmobkB-W9r>z@n^gWQ8~Vig(A*u{wf49fGC$%FQp=0d*!QhE@{6=_Ech#kP{F2ymL`w+Mp;MmxQ_02O7-J5?A#p|d2|2*F{$G@(pkxk#g1 zCp6YZSVtgAFAd)=8fc2{2qidS!_F_mAe)JaNhR?q0eW)3n=~6_-qB{L_Hq#;1Dh3+e;O>E+PHC5 z;-Rhc{pYdknFOy@OR$}D38;uSmS6sfI40P=+-S3FNT4hvq$O_Ut*T z<1VOR0^YBwy>|7+4LWGFwE9&PBP}NkxPd}v&qP$!J4t7326twQUMw3jaN2vEwlr)& z%FfH%wL_O$JAmR}4y*5IDDLT)Xue9vKWf!SHrPN_*aY!jpLxcgwu}SSe&Qbyz|0DO{S! zw!P$uH|ITFx)6MqL7u3FbXMlcpwmU3YROWJ8p;1sC`KJ6H$}X8WLDE+a@MVol$7l4{Qf;$FG_|Z6RiOz~lZwzrbf= z^39n$*OVt+Rk=f^Z`ZL`HzaoFGvqkYNA%2>?3)8)0UNswH>FodNYXwwDFTvZ11aWQ zs>1Q8fowl#{_-eDQlwIUDTv5QaBy**`VDhnS^Rqck74@K!SqTcc!6_uLyo4ovC3r2 zmh%eAf(xB5E~X4`g$l-0>Gv*}>R%0{(-|(1lO)j&Wk86j>9Vk=@`NI1cX9C+)vfrm z4zz8v2HFc&GYJNq`yc_hj^Os~+egnR;#aLCcQ*v=6$Mj9vb)6*!!PgA{5=l{khX@V zrWh1qpED{Zs8s0u-ezF1D9fRZ6M>!y&Sj`4Y7ghngS+RH`QZ=?E*0a}S&UXE2pHK*G=Jp1W zAE+w-6v$prYWy02iM8mf{&Yse-K~C$g@so!!T9v}JMqdy>6yB;2JulMej4#_bA9~T zebLJ_y(C*cJxNJtA2f@}lisNBlN8(5EnmJxa9OCXo*oXSz&0bt@n0w7mCt-O^bZQ! zMLr`l4ID0X9-Ns7FAK%g&5RT79L;!dh^m_@5*~qX=uryuSSJVu;{}9;FG1JBQS$)| z2vUs1V+_W=T}BDUe{`+pSQ*BG{9#uk`OMs)t%)pS8N)+0X`I5S(o@`CAKzg5IKf{V zCi~DxUgHa-=RQtP=Y`Ix6f-#ixdso)Fp_UlSI$fEh9$f$(BG%;U$B|sxLD8KhnFpj z{k=$MmoQ<(sOi-}QV7R7Qz6_w-ZweXkez>nG?H;rQJ7RCN~5#!2bB7t%6i=|({i$GH||R)JO+z`!C$}lG4XoRxpcAXh3@ZZ-TtLi>V^}F!hqa4GQT#a zCg$W!9okGgHT5l<2RFEP@BaNkfC%FS#KhKyh{jL_mM(p+8q7hVsHP$r*nI2n4@D)X zcQQAwyXOF#-`1UrBohkB+GW=cO3Zmw0vq@8&j<0xN5itK8(O}FNmPy0CxIHV);I75 z&Gq%UiPZ_SwY8PJE_Zi#_F)K`Y`zT))ZTZ-HlTqPh)i%9`!}{cc&oQlm#NiS;JFkVYiC2$6O6y#;0y)+nuA#j z357=!TP`VSZ0lh-`GBls-=g`h0*WnH!}d#) zKUe_+ru`a6Vjzx`4dDvDeA!Po@uU#k{NRbXb2om5MoC)E*^?(9CP`BZ$}mffDvOT> z8a>;kH!KOcM>n;R%QoZvnw1=sh!dFEE&c5eWEKBM=dfgs$Y|Zbz(B0oMSA6Yl{?}? zgM)@j3TI;7UwKfpT=*uW1vtwui&MTk0kn87#)pEKjSl+OdNj z^rgf~Zv2-O2*^pfvSe}k;PCLZtBhV%u_<>CkK~jzJ_^NEHl`+!Ogg%T7enbm5fHd< zt_AIeAa?!@9h!sh8U;;Hz|a*5oy8&|YgUGS%`kt`gf%36Dq4z){B`O-7b3o{SRo}T zDFj>PduoerV`E~h0+qP}hE-)y87_>fl4kl{yf}z*A$j0{jASlMa#xV9f_c%;Z#_DF zJ=`HWLkgrqIf8lHrwlq&o9-aW*5l4{R9WnCaoLD)eF@-Kye8y<1a~lA#d<@l@!`_o z>MeJv$px!MIEc+woQKTq%J(QJtbtIvqnm&SL6Uh&{RssuUM6;%Hxy6uPKgE6)h;>k&)=DA$0nY6DLj-!;CYa8Lcu(z%6)_pN-@$UBrj!LUDM$ z<``e}HeKBh=*y@(cF>c~PoxqrO2|0zbo8)9Mp%G6}Vcg_Fe`))1rGsUkQ ztmbp2Qrm(jOv&q4#{Km>l)7{}WM^N2X22%)_HAwi8*;~n=T+#*F4?vfkPOPNg0i=7 z`7k)YnD*SEO&ii-TzHqS5OyxR{?gYF!JNaVI52S$K;cp zu(RW*fd;=|@F40c!%gMTb=uc4a<%f%{yhv^s-vy_6w+|zWp_{}JWSb!iMwA%sQ7*> z$p&hRa2GeX3KT0VWSlK8C0GOS!Duos_Ik9Fw-D}$3?2{A2g}-?g(wMgf2zH`8s|+z zF|REQ&0=?FiC^+j1SEI(w{&)QOGyY(aw3rpV2;F<3BdU<8iPo3(TqHSr49leB?&nB zXh}(lJLW$m^`=1NFNXg|T6qOov7 z`$jNo4f8_Ltwp|Btf5|TXkCcDOcY{0m4?5UJZS>{Bvv6(bs$c(XdRW>6Dv{|ilXaz z&K=u~Nz~%uu|-jt@5PM89JbRl6yb+A(+FiWICA3pUo4+d&KH6 zo@TE*Qi#l2f4X?qpLl(@ou{`H$L%6w4~J60p>PzKNRYc`;yD-CP3? zoKRgUi6urI`%Fy(zzbMoeF9bp8q1@au0S6!*nBD#Z{~t;5Rxk0y9W_*=_nL6RBFne z)th0Fh76}~pTVtkEbp5jzSGXo_hGrml-o%1=p|z}?w+1pN)=VgJNQLJLU4*WGf_^V z38gb0_y+`Vbf>3pK>h3y9u%~In36Do1xH}mZ6#_;r`Wq7w2B>ihu2Cvle&`lZ5e{r z^iPnB};$olR0NHqTrf92`7rbv-U_T?jVMcYgZxj*k@XNmxj-4Kd4b z7v@Cx%3t1AahijgOdE_1H%id$RH}cH2^zJxz&va+GHbsiHPF@*sTQKV4f1Oz^mDox ziKV?qw>^ltl&z0Gn)FhCS>xqn8ckr}=$!K~Un_2Fvt+$q=OYS&gWwoGAjl*i{{~+O z(%6$UASNaj$(ws0?Z|&syk(w1e0)4BIZ%wO9AT77%(G0PjTOLD$*h@Uaa1|R340p; zd|c;S0<^is;y7w3cfi8yPrX~yep;W(oKHeKb8@<4s~jFm@h0zn>3);b5uz99j6JJ* z{vr<0V?)RrM#Ig7=+(&3aQVuJQ}pogrR6BvCe!0p@EJZe9mxZ+Wz3t;9WlhG`6EyN>pH`4{}e?VG^^_56JdSwHoJ+YhJ3>cw88`*q(d|1I# zqY1*XD>#8n7`}$xy}Mj5*yDP4T6lLFj=GVX%9+oLu~pi?8IO3q(}x!e3TubVjgi`) zA1A53A0=L?%JSv2P+&QiW5)24^MYAH;NNhDCyJ`ez;196Q&;=q*dyiv22>Sdm zRu}L_<|rrD)4?p19OGN&v-UvxJi_{2y@V%Ex44}kw`13?GKh;zt0AvYKuLI~NVZQG z86Th59g~MnP}o5<-=G!|5kZp0Qw$(Cc%=d97-Ti%$ER_ah*NL>S+s@Re#TXY|VJ=+d8vyqW zvD~5*-VneV`Viw&lW*>p;n1Vgn+1e~f@jZVV<%XS-pL-P)=i!s9y@p9;@vidAyNqP zY-d+j{lIuwl>27oYY4c#QMYf)v+iDhzx@CIhM&dG8L;47nLsj02RI0GWdrU|{^{)%>Mpll5=v=(Xzx2okr1UhKH7G}v_B~nxKxYXJ zAg~?t*A*-fy~GNtWM=>cRip$Ci>e>2`8jLepHUOV=0W-Oh?M)BoYoV*1;Z-ru87T8 zdMJQ6eGRpqg}$ljlIf!T;zja?SaJTd&#??4QhPxkGaK#9nzttu0hN@Xi>{csDYkXG za8J&C;0Yf326nxZc!Gr3$G@!vB3H*>pN$&QvC(wUT*PP-wuqGj*$#qZ!(B*gGg~Vv z7J!LN;6>wleeN(FJJV@H|#Bx;fM&6YdmqEhj%eq_wS$ zM1c5x=st1F^nzopsML9X{`~PLtEdP@#^ft4E%gV=5{T=@7s$Ad6EAk*!Ub)4C8Z^o z5-K^UOg$Wg*R+={(IT>PxIB7^^~>RU&OnLEO@_he=zYjO{qE~>K zGGRbG{GDuUBew_Ibc*gA}UIX2~tA(-j{P>y_)9+;^?2dR$fY=sfqLP`O)d@ ztfwjV4a)-*ghRFM;qv6@=5`(L9fy>I+hg`H>w51O4Au35pRBh2gSR%==LHnOtMe3w zwf#N$T-8L`c*I)y)|0>vY;-FKaYHhQa#lP+K!bsGyQ1;$J9 zevq9l_&xt;Sl^gvw%gPjh}+MjC3y6SBR*{9rHkmXd4U3Jd!vRlgSP2^4gj((3F{5% zn2Q3vEW~R}8P54LAB3_Q&4JlK`Meh*$lrYq0m+{cHCVMPY921`RL=Z#;bz z&|<^`bGv1`(IKI5Cmt@EW$k`^-9Py{2740g86$1L-He4L6S6{Kqvig(1veA`Aur#s zW`(!Q_L@pMeTQa+i=>T>4UYv8iwgn_0PgU-#RXw`RBr8BT@3mdy5tKqU!ON`9=mN~ zy&UQNzru=Ozo;yh9_}I_(JyN@zuCk=iG~0zh+&8%YC+vlb_EhL%5W5$NJD@P2GLJ3 z2IiY#RS>?NnwK|#cLgzZcLl&Vxf>->=$^rRL8*$GnlP$s1p2#RcZl=w)0USxR{%`< zp&s8!z|dNPPL^7*qYt|QCDIGr3n5H!mOzlZh@hioWo1L?+Yc9?pAFP3Vzrvfr-i8A z{hLvWnA9nv-88}|Xg>jKW&#HzI(*rtMc5+6iQ~9N!m{HEKszi|V9&gAWidz^0kzu( z)5V_iQFIw_E!nYW&wiA4c0zID(tR=rT^Nn8fP5?%Qp)E!1{M|~KED#hZ6KPBjSHnS z7O;@_IvYSX_^0vyHBoT42%45Q1>4TfL$&^dK5>A&bZrvHvEk8h^~#l`I6%+t0|8%Y zc2d$j95dF_bVI;mwc0qN_!q;4xE&JG(o_b>P|m00*jSWKz<)Y}^bbyobR zCa!csP%?j!uC2)~#Df!c7>6AaNfewWbHlB;jhz z*$uwSY)$@ya8yvl@KSKWf(24AFT&B1OjxiPdZ^c(#)EE0^3;? zi5)t0Xu5=pSPlFU-d!&-(_m7Kxy1S}p4!0-?@I~GSFh&s{atP42#vjR?UY`g9STz(5&+uoUEstd!K>5D$ZCjft5_j~CN>`Kp`@2D?nYKMA zVn70k^xi|0oV_rl*>3@XUlROQ_aFffv z|FK!gBp~GoDuM2%p*VkF{RA;lfV_*NYCP=Y%iZgNkdUxecK=i^(K2J#yCt}*mAccN<+(~Ef0MRxjlUxVKrIR|NH}T zEYcPwXYzduG@$TaLfP!Po4~CV9AARzv0D*O^7Hfk%~?Gme?-(C#A4Sl?B#i@j^1Lg7Aan17p1mrHg+3=2*2{^H@ z_69PUF=~U{O^%pMwY%ER0kJD#Q(nXFp4t83!_&hrIl&a@RpC$w5p26L)oLlBLs_sK zB4JWm8U;Jd()zv&%tAf=)a$qN=QC;?RrJbhFqIO3wp!aBY(;XfAl@Crjj*ktTgDLQ ztvB5CKPnu`hRN>|R00%1K{;7#aT`V9G)#`d+A6*fz!@=+`8R?q$8e*t(1zsBu#uK& zmT`<5e@zw?YR79P)8Bhyq>Hoz2*fd8ci&K$P0~0fkjqaUqf!g}y}J?71V%IPg{lAd z=7k?wZao_Hgq6f2OqJhTXRa1aB?XIg2_zmALWO}3ZBl$DN-!D~P#g_MV(=#RlM+7V z;lnPUJL5}2b^TZu9S(RzUPdmuqn$&y$~m0wjlfyT09g8)<72-*^JpDnxm(tHP)Sur zqY~9ZYMNU z+u*BZNTzy=^4H?&0i2mc>fVnc9tx_tup`li;bQ!kgNC|_Lx;*^vUjf^6mjC@ z^;$k^_%Ku>2Xl=?};aYvX^#0&8A`@ej{N=~8p~gMzNxW?*-utaweNi~ob&ozZr4Lu^tE9Ya7afC<2K3fOgkBWV$bIuVb4 z5VEWf`1SU+^3Vqd5ULL8JSaRo-2W<-I>|>c_hEGczT%Nt=I7zb(kr)J+jO8L`X{1& z^`g58RjW3$k-!=A%v_#;jZA1OOnA6nYd~sQs#AxL0DLtP^c6k#?fKbrP54?+gM(U? z-ma~EFd;3q z^m52zhnbn#gG1P=8iaBMss2bLo!$d?V1aU+w#kI9T|m(l&e}QPK-%XycFlSG?w#k| zOmq4B7{XsqK5Yxk)Sb}R#0!9bJRMWKS?nhYzDgD>L%3yF1#d!PhRoyWn3>$d4l>lHjY%_WaL!T}&nOSDo46UxNE`1MAye|$77{;ugo_s5V zEb=Hi91sJk*#Z{|Nup*;kmEet-yDEY8?Om>T%;E2zc(KB8HTk~u?<~{bMOqlt3MH-vN_C*er(byAxoNb$C-<3XDw<|Ic*e5` zSzvb_IxE+mM@<>Qw+yB`$t9>wx7$4b8LhhAqd*J}-%G%?{J}YKC8MK?iY5$I<&QFb zUs28@%=zCcu{P9?Ht1O+BFnEFQ$yM^4Ks9oY>PzXnBa+4-20+o-+x_INSzYl;-bFv zgZpIXR*@K*s`5>o!W)~RzO|OSl%V#BW8uQ9V7BSw{sfnXzD`?SL18hd%k#R4Uw)m1(4_BA zQg!Bm^XQM9$S2jQ~wH=$p>6R)sn04gUcgU}0}q$E2kR|90IDk| z0E-Kshr4?xi46d9&K-Il1DW7FU|ibr($e!Fqv*dr`U1xh(iW((z_czQOi&y857>$* zvN%Pdh*JQln+m-*W70BFxmO;@av5&=5TIBagtz?g5c&>Y7Of&W^P+FLFXhC~jJW)P z17Li630EyXSI?`N_hyb_yy!&aq#WtVobdH9)yrzj+DgmH0-UB<)E_W5KRno3qTP=q z@YMWV*yY&p?s?ltSH_FVncxmPck>IOyw1g06~GH3l@TOeOp&^4>U3Ma zDv;Vpw({?h_`YJkDj@B_gX2XfUoS?c3klUtj0JqKwg-=fkqWj4IhAw?l+0AL4Ctr` zB}PF(gx3EZCOUG9%7#ezVPqhyr=4nC8-KrnhWOkZR5)T~Vj``WjE^-rGIoa_GmmX@I`t`FiIVz^_Bqm}UY3zD$=*C)@fz?IHi`NZqxe+GlJ zsHl*aJ#mXd815!VQ$MxQ7IQF|3XGh=;Y?=&4^#_Wt^nA{I;@$al>z{JIe-!P!o)e@ zZz(sz7K@+gFO!mz2FH~t5l2`!>*g7pRq0=3W_uYQ36YCXR;c&c#P&A@@>Yy<2g%*cJ-D{?b_S;sv{HRMiWY)l{kh8PXFB z7hFGwG@-ZJmn>eqJcOIgkM-8Ijrbvw<;uk998TvBmOw^%o6#W1Lv|%nO=HsASn{_> zxEu13m$&yveYo42+D$RA!=SM6AS7Ro%#&}k@L_<^K0Z7Dn!fV;^|%EA zoKf8S^4<(xB7YKzA{M8IT?xoA(Ms>^$p;QY5fWO-Oixdjz6X`zfPjF2Oh|QgwGAdw z?ny)*|NJ^WdBOH@ajiv|nUP&=v{@vE`AjK|WJZV^USdAL|Ca6E$ii}`qQ{M2;{g|u z9}|Y*dya0<;n_JkuH(ah|Nc$>WFA%{6HMQ+DC2zy#JBMREN=a~OKdSFIu zYmbkQCc5YV2zK6*A0$b$($!_dB%~A*Aj(`sU)1*qhNVp%9pdKl59Z=`1!2Veer_$F zxOg}~7R@&!BF~qIicVG{;?QIDEiFYIItt|prMBXplMj4+j5VJ3b#+~UvYl;)?}*5# zs;Suq&dx9fWCxJU73hK|KnB1phzsD5i9BGv7ij#rflYvA9bQ7@``!XX))2TzbQP=H zoN^u=u*}Ce0C6p$-9#oNQAs&EGwI z4XVm0EMbnZ91}an8zdXCcEZvNPVKmL3lQr}t07n>gilwH)*p%RE_^d~RzW8=Q_$(h z67?A5E2N;y%Eg&1OPRGqe;+vnXi!c-98MHJh8QPNK~_wn)>fw-usVBInLl7tKA6w> zfEP=fz?ngkj-;1E(z}z6&Rmj*S8^M*h{&!|n$Ga92CSxi%^E74RAUtiik*sHaok6Q zIEcKVa8`ZK8=04GOB z1xVDV-*-&aky5110COm6$OQY!2p0gfr78*vhNPc!KKcA~M>$&R0P`(dW?`EOSFLIoV>(ok1dfS@HfIJOa`mm?^R`1^0a3_?96v0;NTsjr-s z0f#`t&-jg`0805GHMTgja(b zhdQ&ZMt=Re$|777!PE&NTWb7L7Li>lFDJM7mLsBs2o_S6x|1mKbCvha>(`g!dIbNU1^iMkBKIt93@Nb5K9ind|fE2#!q?$G0$)7#a8oCR9B7! zBk)_cerE-KvHt=;&0R8#&e(4nlf!!Ry-{M+H2B)5yZgM!ttG^R_MhD9Yng{WZ02qwvHtdc92Mf$ujB^J^*WLq zz{l)F8)xp&mK%VX_ho>s&%d|ZC(pBR;R|Rzf{_mfzh9zPR;3Z?`}e~$0DRfzy?9o1 zi^Y4;I1H!I4!IMF!csUvONL-D&NNhHd^0L*TJv>a;Y2Ovx~bw*OBm04%CM{-8@I9d z0CsWmW2O!j9FdKr7~08% z!lc|>VH5+rG<+ShmI{#$GLE>Hhf5l*d_$46vQ)!gVSa@0am&F^;KgF^C@`nk#fXSv zP;2yVWJ%F~84`pMA`2+$wMlc)4TtURCr&U(AjhsZ&N%N8A(Bu2`DbT?Gb-j+F`Ibc zVE!+zI5?~&ZktThooUV=|5>Yl_cZkH zXx*41sVhhoyXsNHC5Re0GXd}_$K~uXGNO9cstyu+J79LOJR(M5fB)7 z7(KA_913MtCpKM>`4Wty28;usA_CZ%pxUY1K+se(l+sVQi+-J8%h=Q=DGho?b?4MS3L>n@MB*j`-q;rCz{69F}090scc2<@E z&`unODr`^+1f0W_2|)v)_l+eJ_5rL)zEyQ35_-nZT`XZFh{9i0lh?ba9zK~=1}YYW zX3*xFyHAgeit@EJa&qTOCsu&pHpJuu_eNF}&jD=dpGGpcKfO{FOXsI0Gh!OTl{GjgOk(FqrYTwD1DGQle)Xbk)ykC@8fZYA z8i4@PEleRB=u=s7d&;j8Sot{I78DKsrDfIs|vB`1Ew+OfS%4 zL<{{}v(;}Q+gw0IY%%)Zc!+}4r-Qty&+&u@g29}BiS^<>@V!mhPEvKZEyCf1mM+DxNX!KO)+tVI?f`=ad;sZkgU4e? z&`Yd;77`N^;{;om;9&YdJuMTlcQI{-ORrsq8FK}rTp(Xy`%sMwg-skmPXl6{;`(B! zb-S5alhoaEiZ>yMCfU2=b%N5<-gZD3)5&7obijC;DAX`KY*DWnV<~+tJ@17L3NE0=yGIU zt{8p0+OGoxVdxvk#M+p zYYUsOLb!5Xf%Clto?b!^3~oausQ0p>f~y-a8X%9IM6=Mg4NM&L0W<1Y$zBEP;#ncE zdu1RS8Zcd!u-y$0pGH0q9CRF@=eKrXk!`$D(od#5-8y_ey2yStC!b<4@IgM8MO z*V3Are%+8|?GjutfBwT--_fOEHOpb&9xoTGA4q4Pg(wOiqyY0X7(J7cLg|EA4)o)! zv9im-!8R_<6DkI426}rJz%Gl_9yymVgenEjhyjbx!!Q(TOP|Ddu@Mm!ZIf@o-;aEP zkOg>ITOLv$Ks99U2IFhghJ3Q-)-1<)1qAwtsems5DH~Z@$}wKb z*kQzvn8b&1F>P=3%Mg2{&4_0%fy5^%KKauHXd;*2@f=WX@Dit<*#mTe!1j3Y@lcLr zT0lb&=)xV1ngT{_#iaNsISL~|Qj(E=tXC`9nDDQ7!XPtPwiAN>&iUwS@y@YiofcRs;mN!xAJAe}2g#w#_jqxnC3^NNKvzV5dMdawv3cooW?$nIY&HqLndelHk zu*evq-GD`qYqjU&nqxWyTfy=8^5ZN5Xlx0tS-jlN>L#eXQN!@^sgg-6;!K8pm6Seo zpe5k9?!Ns(5Le0L0PamI9zcjlhb!k*I%6ss043Zt&O>Jwk0}#K>^C4`pKm-yXB@n( zTC^ePSTw2D55br!VN0F2|w+La%HxJ1&R{fmuVaeVPh(N$T&`SkjdaqJR zRk;gSG3u&+<+*tnrG?d#c0aaD0S`f%ZG(V50Odd=((vc55J0S~YJk<4i+#x6bGG$3 zMiyNh)xXET{gSI5e*y-&L~F5FmJ2Lzk9-EFBQnAX_*dGFR}|M5mDF_y0jcE zqTL7rzrl)4o0eh#XEBW!khXQa?NAl#{+C=^i^nw^oaEw^4&aKgg3xtcSC*LTIl7k2 z_qyZ<;=%V^;@@V}_H`4eN=&#_a`Z@(unLU~X&Hx~K2;!+mH()=Jea-q{eFgq+a_$T zWZya5L{(w z*tir{2E}+_rgi3qqr}dwrsFe^7NowPzdgzO=z9hxuQM@RQyb$xhT@7R78hdF-K}LZ zj_MtNWd6@TrB;7`EG5ndu2vgoVT`vEjtJ#QNTDDrIaUVK8L~ti6(KT{&&{8u&jo#f zJNPi8>FD<5l6}|va;8X>Tngv4i>PViF>|AeYlzm?i_kAaXf>Mi-^Zfj*~OB?u3Hz> zR@?27fH=U#R}Be($mX-V4@*iiWLnnW+|AsjcHcw`9_5AYjVcc-!NZ_)1n>Q&ZDxGh%l=vnTtAB97p6q-b)-YNB)BmxJp8)6t2cC~bSps|dp7JqdI* z>@?SEj&;s$P?yxy)Fcxwi&6K0U>!C^3yCqaO;}q7`+F-s$TOSb`37&n?qUae6UY~Rrf5&VaybMrElEE8@RSukIxjvtBH{w@M$n&kZ`r-J zQ{F|`^g)rBNF1M$n?E@(B``*+>^6WgjU}^eY2D`{w&QcnAy8g)TwKS;etg7UxI%Gt z!wj-3_p~oRY2W}2kh-9@8!n!uO_-b_E)hhNy#rpaE(``^F%4dJy*=;k9o5TO1S>kX z*mi=Tl935z09mhMRWlA2g>`4ffByX0p-mID=o*&(QAI_Nuyv&>vq88)w);@iYV7xK zyaR65BqZoIixL!m#Ab+OasT6^!HnZaj~zRVDU;-0EkeW+@gq1L#8@M%vw$lbRC};{ z%Nj3+^mvqE34)w~ttFx#+zy+YF)_q3ib9d+y9ft-xM8x zK(4j(R&Mp#+o{s?r)l)n(3Ti0iRHQvNdnY}-C|IVF>|F0C;pW{Tjm!OboKU<0^XHx zct41ui!m_=tD792cLV9?ORJz6>C++B@q$33k?U{?&mQGuDFJA3cC=!KV_V!`eKue6V4Wa zSXDen)*wke!6wiEETrZj;sH!95YY_tAj#i57R_)h@(%y1C%{}X54LE|em>b);pFt) zepd18U>o<}sk=Nknio0Q)aiQ-#YD+>`xa#z9lGH&z*C)7=Jtc7R=9@7|EPEdqyMwkGj$6 zOz;evAt?W;YmXzOpwYyz=Um6{fnNcbABII`WRz+U8I%0+4K*(Ck`tNc!Kir|kMPEE z;Tkt!{n|^MvK2BIoFLm_VeDqb{XVsf_m7iR2Qt9sY2si&+exP6+vQur!omcE4VUoqpQ%Qn$FOD1 zQ`X=mOwUs~FZy4ZhX@?BW?g;>9UgdROe(y(PLThu!e4^EFWUY z=4PsmT2UIS+F(OoXdBN&$}_D|fJ`fI_!JUT684a;*DN9f z)N}*p$(#>O4#2}wEJtd#xDy*2f|&Nx#b+Ia+0fEo?|WYZ*)lOM`Zw@EH&qHC!J26K zL75J|;M+5}QM10Ir?=O3>hB~hTTZ#;#KnmVthJzH*yrgUh}X0R@1kM!-n#dRQ196` z>PXe$tGze2(fM+qTok^$dFQzbS9t zFxuPI*X7k(m7=Q?DkQM?Yg$iw+@A7R$91*dWQpIeJ9T=vx3yqG00b4=S_?UylPBj2 z3$Ga)h^Q)>$m1BB`df5PV|MLEj!NqHurAC5=xE$L_5US*L@5Nk@)}sVHCL|yM(bC1&Ey&W@DL|GMcr(^;fwGoujeVU7jpJD`cGz_Ew62pT6uk`};a*IU@2q1vE$ozchgbHHA==8#z(CymehAW<~7*7mEZEoRndDD6E4b| zTEL8meQWc~+n%)tPlef$T{S&Lc60UIx zAxMKAyMY{AeryWKdlIZdA770^33zfhYEOA6gHvpj z7ZPSDB+O|ja!7i4VPe^vt##JTt=In~x}?e>uOM7r#wJ+0DlS-_C7uqp&sFQE-+fDSpJkXDkA&^M~0U}zS?_r#C zcegINI6snCRSVNAAu#V(J-2~E;UG(S)lREGCrAph?S+ScX(}N04n(9PKQLpUwnWSg zF!wE3@B!fuBZ^lHA&eqoVTf5|SZ9n3iZ@Jtg2N%1!Vm|uN{T8+(2))4SWak-M2hhs zBi|v*#HIfx9`7=nxtiK*%gCjLYcv`+DXc&3Eb+oX0lOdTzr=Z&Kkuw6w0bMSHec9^Jc3zxRHC6=UhM(P8Z~F1SYfuQq#F!-(}SinA84dgU_G1sP$ilR6dJ2UPx>|T(Kp!eFu7E zDzgCEr1fyRKo_5W?=0pq$Rz<#a`nb5RVX#d2|OALVgzG%Pj8A+BDlCEGDy5mr{&QS5;U0y?o=bCq_3BV*J<_ z)LJP}>SSOFyCX_kVxl+~8%=@UCSr_c;yVIG5Ye{cL9eIi6T&9_JW0)$z51P+8g z-y+AEF!UD_%sQjp%F3yQ04oiCIbjh z-=p+B{E43oysK3{F6h4UJD+Lpdc_?pl)z-cLq zi85(D8aHIy*dT~7FcJ_5U(9F>Eh!{4kqFZ2la}zA16Pg%*OrTziS6hl z`y)~h96FSedwjW?TJH8j2)~U^qpe`k&6q?Xlbn1zA~Bn%taT88|6*2$3&O~v1`09Q zRs=Mo(}kR{PZs^!Q-boVE`vo80i*D-N$ctBggwy2fh7~YpcJ-m5Ur179ya=>0t5%VYe zv{y;?l2H4tqevWna`tOSSr7{+c8gaQErHixn0FT=HMeL_0)m=Ap&C&Z9F}xuThaLF zZqvc<{Z=Dys|LCI+H-uThQ7M;Wcuvs4?5lcc&ont*U8gG6KxBr+ZX(*Y$PAKZ=Jgg z8Mm(tP&7X`8#X3+io)jWRQ##G-3F9edHBeY`#CbQPw5=707*V+ zxy@K$S>UlG&g%!qZkZ@U9t@1ZYllwkzp6ljDS-cQAG~Y$b9A%<5eKRYBrSa?U!dnG z1}t$AWM{(QLJPz7Z`G+P8N1Zf>YnZ~8)N`=Zd1dAzMdcD?%*aWEz?6noYP zFZ`fkbGtwOeUt}H%i_R$=@466KaZZT1fd+3b(n@Kje2j||07k9l2diD_2)f>Xr*G4 z(-%(Dm$HR(ZfzReu2pg@149#;b4<6nGE!+5-|2G|Enc@!Xw$3`zdvg2TV!22H1~7r zoSgfLyJnE%jTG(IEdTJ&&B0qQzrR%aij&#;WV=o%$EAegmJYu7`oDsT8~E>N4ut3T zCp=mrSGO@CC7mL6Xn;NVN7+-oF2~Vtb?PuO>Stugh zG;qpp-)7DY)kbF@vEBFJ%KxLWSV%mCkv7+3_`?cy zx}8eZvL@CZ``oHF>TU1yVp4$wii{h|_QFJOxqAyVEP<$(cI{pfhm`H*rCuAS)j`5) zDazdy_d!jhAbysC?lUhB)6WYY-wc8a!-sRxl+qpo-{eA6A$rdjN0@P#v9qyDGebMm zbTIEpDEXjZ+Gc0>_=QP$tGe|<-4%u#p@CE~wgfScU4vpW`$rM05)!E*TE9TJq3MS%MGQ<}17S3u?nOWj@KqwRCB~ju6$)C%C<5nN)I%j1Fv>SQjP?%~l^FzmH@vYp z>U?~T164FD^3foLsssFLCfDUE)Y7+R=fzI<_W>Nj7pYIV$F;l6+yH)i1l3l0oe#%t4OSskFKpJ%#2Cjm?~0U2QIwMfc17yDKg`^%`g-KhkIC7%C0qPVDS&I~)Eo zGE&VV$2d4&U1}lQ#xrW%7yX9mq1;qwXX#%a=j2~)9zUQ(^EEhbeCJg0?Qb;Zex>;L zM*?E<7I|2?&C?x|^r9~Qv%p>J&Dm;)#!d$V?!HGc))`IYa+%pJW(!^TCAu#5Y~IDv zSI#Eaneiz<4r#riR7J3d_nW60Za`urNCig?w}88Qo|HoA0*X^fICJz_`cr|I zjjIk8uG8G=v5?NOg?Yp9glpSIo(eY&k40||t>FeOG*EyL}8p&%OZ|nN#%-pA_-J7E0n6&P2lvLOI+QJ92 zX7b8CmU22Zw-FZf2f2S}yIU+~a~mpv46-%HC?MTJN|<)*uH5<4PnWWVB0fGXcBsDN zo3=18kDHO|a51l)JMojQgk1XYXsb`h_xZCvw|=Y>eY?-McEFt{)6A~4X_7LPS}=UG zYi#H+KR+|!@ncGRyMtNI27&eKXZIK0-=xW7vVT+8=la9^OBza!q#y3wavI$PdzxkI zp2o&a7$f23U)l(n3Wxa2S%1OA8_gce>_$QN(jQ%Bqb;yePq&_m zSqj{nz(UE}CgIqNc*=Z>Jj>PP>DaiZ+k?C^p<28upFV`E)_utmd#oD4SrO z0^4ezn+xNL-z^w^EU?VDBQGQ>&N3+HP^>iDtc3DK#kZ{LJ~OMnwQ}9dc3v*7`e2u# zrA6a_hl7~8^je|7b6Pv7gPmdP-dQxc@LB!&bfkI=d%)+-iv2CgRiRmUf83len#Y&X zVY>ZVK;~HIu%CR3>(AF8f;(!;n*!E{=hu9UrCDHYIHkP%2ZfmV+ zSya~MsjvU3D3(4yHDtT1p6JM*u7wVF)?R4q@-N!=prrD>k=ftN4|!Z8xMmG}GU@KL zS}W0ehQm%EkCyXJ-)~IFFYT9Xl*01d$*3g429 zbju_|9@M-F{Kcwg>>CDudXB}^qmoorQU2&~Dq&H=Uk3~|t9Xw;dp)LtQKdSK(byLz z%(!Pt9Pw30KFoVa?QzD~6snC2C~2~YpRPsLm-|)HhaUAL!AhvMVn{s*>+Pf!bwmu6M^kALPUEH*MMg8|5GP$$D4r+*uof z(V2*=SEa#UQ|i6ZxBZ=*uzCJmb9K_&FHa$$4t{76)SafkWlKe!2}|(@Z3M6|E+ybU zL^EP;U#DLT(`PalY`%N>@l1s(gFmB_qc)9G8pr->*OPS9w)eL$JYF~#xj zbRm4(n68jUsbC2_F@Is1E<$i{9YSRY4*YZ06_UXec%v4#6^sf`qS1B*Tm{!sPSRQA zePdOn1^tKoUR(y*s`Yr^@Si@6+Y;RiV1*WUrQv9NRw*KqCsN0nTS^)p-gi(6)5ime z=_4@#9EVO-0dK1P`qs%eK=RUjr+h#rX62}37!p|uwA!30UAo1K2XTigw5EPK@h^Gk z`How#GoaB*Jlyy9^&_9pn7??DfB9n3@W8}y>VvLLMK_0v?He1Ex_Nw6E|0e7g`}sK z6Pbe~d(mY7na5UI>1B_XZ5k-sF5Z@R{0#g{cI@)x-1kTnduLAm`LaidkxVl#kdY}D zpBkOI4}$`VsKb=cxpQoP`u5AFooduq%&NCCN}PSHPAp{Vr|&20A3gL%&l}&3rra&2 z$4#!`w(`Dq`O~qq<6R$KuBQi|^jXg>?{zv3DQRt~-CrJ-z1@6d!l-g(t(6B&mlx;X~pC=CN=#Y_)s45&qV{$2Zb z{Ks>7+X}I>qrjJY&wlD`l5I3JWeixIE4sVGD&*GYdDT9eIc1~Tr!u%7JZ~$0Sm#h| z*!=pF+TXba>qmd5-N;ogENj0J)L!Uc_WjtLS#zYk2E(mV{c30xX*{W&zj_Za9`R4< zlDj?Fba#P6#>Q9N0SDgV`zve)E_Zi+X&drZwx0|`*0lL9UbnpWYHpC!tdLl~fR9rO_u4ux zPfTdHulufjAV!1V%EN7apnjjbZ+-NLk5%eDfzIN%wF&C~A6ai5)^z`d{iCRqlqe~n z7+8czrvhRjsE9~|+$z%DFchRDRFsesQMwdqVRVR;bdIhuMi1D=p6i3(=Xid{@%z`? zdycWs`+dcEzD~^w<7NJ_Fg&^TXs98}sqQzq+7noc{F_&y4n3B`8?t|h*94x~9%slY z@>FaoBzU?{<;C8_B)(Sj)7|*=_T=tkXgE1#X2&-O$ z^6ewk8r=XnTNta1; z)*>?YatzNUW&^y&BMC(}+=*xMSLiXs?GGBy4^h>Qcz7-ER7+~!L?PFrWQ7M6mLf`% zqV+{?IQ7dp>8Qvg!lcecabyQYfqPr;wkShk|*<*OKU(d^Cwf3R;+`b6u1?*!O_J zo;YkMPf3HW#l|G9$a6X8&dm|0?UyJ&_$+vCym{-Gc|en7!`>jYFpL><754V#kwdv2 z>76OkeB>G!IDiZ?SY@(cyd$8Xz#A5pBc%a8?k6BTIHJ`IHW+GXkYZYMq_{v1VFoU& zBU*41=u1SrzCj5F%acGZ>8B zys4tvEdeyi5{S-ikp`X)sB92lZ_GkxjQT{as5LGRmZQ1IkW3ze8u((D#Q+TjRwy`i zQjyKF!qMy39!0tFTYwV_#M~E8T=lw%F}I=^nE^!{B3%DjC%}=myl#bh0uzNVqp+sQ zu~3(SSuu#g?m(OY*yI-$>Y8-{Rb`RKf&F7hle_H7`lY@Olf=gd>dtUuQ zPx(rkQfs4SUKqaH1>-Kr21f>~xYW>~;f?#gDL39+c3yZ!nnktMYod1SDuluN3Hvfn z6Y)=SMqE|Hr9-lXi33)(H2enj;k+{aWgDK4%bfaErag`_uNCYPq?pVWYdGb%i31&_ zTlCcS_6YiU$oC%UpUj2A6K7gvbe zjbM*t#O%%sSS)p$^`JHI?}nkt@Y-rPIc0ds(lsHoM$e1XRU1Pd=Db8MZ^Icgh(u3D zCh^ZNyME8iXCK<#=FClFS6r`bg2@j7Z3wNd@ z#mZe-j*X4h$3M@dj%@t?%?;ap;dll8ohqER_6TvY#9CjHiO(RgsVULh2;g4nd6T+G z3W5N~Un?E>wLYOK+FiT+VMLOODnNr)WjOR2OuJGlf6z&VbEM4OuE+Fh=My5HkMYh0 z*VW^Umz&`-$5C94yXad6q+d?vyll;~zsAc1W@#{4ie{cknYZOg)v&2#O3--oO=*6Z zPUuZ@Km^L=z_h#C+;HK|m}9T!>h>+K5>&ml*>b7UHq47Mg=j^GI2^&{iM?C&p zY_Z-p>d}@QUo%dqVwMuBUqWfxZc`dhGpY`+Ut4C_P&3Qnk_u-(N%>pB!1t(tafpJG z_9=2aCjUzobuI78LguIZ>F5{cnbrYqhHvgW2AVXL>|0)TxZ!LsB+UQkMk_``JZ?b< zu3nX`&KoVFC(e$&93rox5*lJf^y0K%jEH~Jep^qqkBaJ+>a~YKNs62>m>g$I56npB zIv6`$^6t@rxE`js`fzKC!{Mbw8%`O)OsxT=Xd^{SnZitE8vhdh)e{1WoO~WqoGm=| z3L8d#*<;l{LU|cE=Keii_$ff%$$7o%W=5Hh)M4cA2oh?MTBhv}Xs$t#arEqd#Pbez zd!T1d9gI}}2lp!RDBMsB0)M0Wr*%827c7u@LW{uMfZ79dG2r$Av$9F%E{xpZ{8qNM z#!}P3fgKbaJYJjAJqZgJkjsJF`)+DDKle$%zyf8m8ZHLC%@;5V1Br%Xvm?W^0=L%q zuv~DAYwE^sjb%@|zg?Q>clrSbJ&Yt_&1gA$44i=H0O@w$P)ISJ4*>;)t=A*%SO6MvWF!d)&A2jT=LCG9$+bk)?u*cn!Y$Z2 zd`VJ|I7+CbsW4K|IR({9FvN4@1H%LqBNsI^r3RRr zfW&17scAhwfGn`0OTXay=h+nq7!QYazNum}5n!3@5EDJeL9k}sIHN>y=Y~30ujO%C zmq{~p8*)j&K7W1u#RyDjC%XuH8cQ1n3iH4go-~9gJ2M@l8 zn)pgtSg7|Gi!g(Op(vXYne_EZZ+11OluZ?NF)>gnzFGw3{_AOWC)N4$DsSH(c^yP+ zEH0^Mx9fG@Ww9g}rdLM~ZVj~m9fJlh!biMPYhZ0PbPGkn9|={v@~dJ5=xTQA3EVGz zsWd-)l$gNgId+cXsAUrlX{D-~{Qh89S9jpVi7M)_ut!r<-D6pKHm9TrqiYum3tM4Z z$J~~LVoy677h}D8h2T^{i<#qlUmmO@A z(;V*bu(V#c*e26@U6by8S{~K|wkYQQYe?UHD-TQ{RV+2-@DT%kF2Ns`vchfh>W?_R9& z8&as)htX{Ju}#=oD0)qvSQRd6L(h&+2XwBbTsHrYc38nF{nD>Kv(CqZkj)+?IhMe` zSewN!tTOku&6_==N$Lq675}<)bU*&8W~c_&t7X<|yFTna?T3MXyh}w%*2KbNVOu1f z#+BbR5g8u7hpz7^XO$BQ%+oJd58K&RvPfwf#wqBG$dAZ=%)27ISP|BJH+mn84`nSh zj_$AVHnGVc>y0gMQoKL>V@ZcHdnw>9{SCO!Dz`T1`mO0nG#+MuNanD2p}Ok#p66r9 z`UWm4X?h?*CMZdUGi8QpzUU8Tdll2A@6+&(u9{QAl@|UNF!49!6u8iYK{~s`l&+ldUP*t=DeJ8Ev6ovOac{V#jzEd;Y2vly<(_S?G1|$K!Lh zdGmZnlNd_MM^*g%Dnz~9PEa{5kWXJyC%bm?ByoL;!+}hV86FjQ#kCnm2+dMq;al?i z#Ldc)la&aCB9*P}7vfc+Tz-IqW;x6~HMCkNVsNhZ-Kp2)+A2xO&u1@p#i*&N$xs1D_rjmpf@2)QR=#MZ*>Z4? zdpsO*J#Y=VAQW;3Qk^U+Uj1=6=W=k#gEx^xwhB!YlnlAGeN8~v+mT1k$MJ|{7H`P@ z#a+yEQLo#lz5KeFy+6)!=dZb@nNWuM2n-uFZ0NYx{Tc5vojEEb_aT?(k8q6KOD|Gn zR#1`zXYE-RuSp_2689GA1t$Df;6cmOXlnO1JUM8**f z>x!3mj7My^h|KE^pHOixOXmFs4KU9NPZ3ow!+R=koepQJbf&XOl+E7laSBCQmE>Cu zMxMI77TejszEEtQS>gI>vB~(#<)_XDEx{C;QkN2{kp0LKQC_A3K zOa+boJ}Qt-R!j9W6#vCniF7lx@7|yf2<18V>)B?*_x_D?Eon6IEY;^JwPQabILxNv z(_I{q1w=2Q@x+UHQTP@WmbQV?vN~4e2Qt5Y-D(k7|6lDA#hF_KUHr3_GV7}0?Lw_5 z#rSB~f%|M8cc&1uGKS*GM0HYy1VuTAfZM%xiCY(%E(LxWChB1*dG4Ij;`bgWg1p3Q zb?sD>jP8D_CiKGqWWw8qa%bZkwQ*ZOwZg!7qTOth&^Xp1$0beDI7C(TxHo96+}3nQ z>r}Y}vD8p1x&IiD5H>c1UxPJ4A=teVEU{H=y(J<@QM3-=X+?qGqtn+M~h5XDs!4YvZoG& zJ#dl?G~NrKBUb@VBB%-nW>x`lY`OxI3O(H5RK7_`G`l-}*WFf| z5A>A5bXlD=g5j<9#2fCTLdLNDE)X8Cw!c~awFa>CrN;_+Snsdgi6{_KVB=wY=Cy)lb93+d ztAR@k_x558F}(Kouj%9MmjC9+b)I>%@~ELwkVLK*h5tYielf9 zRFj1=w@bwN{{oY=5bfMSQwE zyxdW?cVo-23hRwZiN5?4xBb31JeB5i+7G8`!ZYO`NHl2-KNd5pfttjx_s+g_#Z(+F zHw4jdCjf{WS@?u2(UcbS>B&GPd0zqs-DiAJq&n6Gnwsfm=_j5Oo*Gu<4O8x!yB-0XXL58W|PZWRpCuWlkSM^P8+{ZcJ1$@%t&2X z-W=kB@Rr*AfuqA^Ppe?49&$G>%4w?tV<6YbLSfn#n?v4Ehq1uO zjO-*oPxwd5hhZ@ru}|+dcfkWYLPmG#YlJV5<(F4I@36`h;u>g1_a(Y*D$STK&vkg} z9vvG?5^<4ry;{O-N4-7)zTN}*BqI%DVLM?$o~ z(mp{v(!Rq+rk!;8TUTrniF_#(m5pPo@>2^iY35-A9m~hNO$x??Gi9097q51lF#feX zd!3duz0z)WdNx>s((4iV?um}wM3-epspSd@W?oIw8+6Zb7>#&hz)y{bpWIJq7LJm4 zP4V1+JY2*{%D-UPm(^o3$VRs{s-*rRW7_ai?MVCrjm=PJh_QP8haUTmpRNZNPadaC zT_~a~NL%%Ka>Rlfp!y=ee8AW>AH&g2zjMDU(2>!jx}h&ssI^;0GY6x zAw^nJ`KD1sIZiz<_ICV3ie_U~6i9#SwLL^b8RmQaL}zRq7cfit$uj=^TIRH@t56iB z_TgzE-rff-<&Oh+8bbz$6EiBgjLAv8&C4`lhpX@4h1G689718J0}oEn*op*<5=O88 z-LYWP75_~2*vq-SNZAy80UHpmFOJ)!T`h?hRu4R)V_(`zT; zh6F8gM`<&F>cRaBp#vcK^Zx|udr;~BmmZY~}B?H|&C;cx|m z6$S9LyfBtP;JP6pbuw4MgNAqlYg)3%?q;xH9ee;0SdciAEC~^aTf|}o)KON2Uthim zSPlN91LlshApkvL!|-cK8L%DZ;u8RHKV|{Wsy%W5^5Enq1OEq)rYItceLX}HkD!I& zBK0fxB;I|I;R;9;1T>0R*A!%Axb~P+{}e?pLCUx}WIIF}Mgw>R5;-cubReyxzP=vG z27)}lFho`8fUK|~eNUuf{m7RugG*?b@ZgMhrw>%dh4+sbzOk*;^*gU6Fla!SPrET8 zg3_g;%6I*e)(0K<@uyD-ZLMx@b6|bsmf!wQIS%XLSw5MEG7N3xkFd@l!=_el!+Fu= z;O;Kp&AZIF%}K?&w2M zWWJ=vvhaH2`QlQ~Ui2Z9uQNHK1SKU|Fh1q%_i^iPjvwNPM|?I5ZqYJm5u{1skFu(v z+2^B>F>87ja3oHyPAA4*T)Xen%f|6jvC&~U=S{+nZp%ZbZ~I})O!e8h%cM2$c~uv& zdiEvq6tGdu*ngH4B^{I9T+#k8%dOX-x!XPvv)kPy+1FS4;`?&XO3cc7dT$@f?OsvmWGtzhXID3!;MUwjRtp#+} zA-`L5dEDYn93rF_Ct>5OjV*JZTi#OAtOaq1P$6_o1_~cA8H*u?h;+kkf>7|8uy#*~ zu$?z9W`CZMf zL6Xe%x`Sq;cXHNnqs^JhPfe`}GrL0=(nOyi98K1E5w+7dCgzDdErXX_j=Nor7$#Hj z`xmm7TJy3T`<_h3F^Jl%bV&AXz`wp=+*+0Xp%)z-IKVMKytENsfo{ybxUfX%xY47Z z#O3RmRCYA+R=xQ>z<=po_Z;lYG zX~C?$#Hv@~zdBpHp9-C3guA&?fq^};@aeDB=q}hy_GKJTkPwhLVm8w#Z`qk;i=id zH#$D|z7dv5yDdu|4B9eL*X`P9_JWC@6vdedL+aa(ded|2{i&6|<#)z-y5+*w9vE>3 z`qCKKZNN5uTub1!UQKp&P}1d+5m5AUL*we!QgS+rUNWl!Ys&k_$HpoUH3?D#E6L)@Wx{LwoT2`Y)a5yjl=f+K41I!rJKxMF z{wGR^#e8ZJEMlK6{)1!~`s?%PB=>a>g#e~w5#I_53V`woeuJ_8_;A0y_~GO4^*#THhH8vdhZ+7xD ze-nxDhSwMI@;kKMFvV9j7%N5Sb&XNMN8lvHgkcTlqWe_JJ+%mT$l=f4Bk5@fTk`ZvkvM_IMpM4W#@K8fTBAoZ44HufU zDt-D4BVpY4_mPFeBgmsmhm2MbXoSA7&v0x>V zsY{C+antIdwuHjiu^doNAmPSy-DOmt`&=v0W6Y?Spkx*OS=)i)JV#7>;7Ym@RuIO| zKWzx3N{3x}-gl^n z5$%QBVR4pOvB=)5wRZgpE_>hQ-I(pgIhx!n0x}P_hDz*z(K>$&%iI9{_B(rwqRq^e z6qd;D6|b<1K9@J%ZC=;vt5?c*&AvF_1}AFgP!gAsw*(!K2%RJQ9scWIX!Y)2Xh+VR6E3IHrwcAF+|8}x>8g+3azIQg(+;a zan4;uc6hV)Q;Aqnt+V{EVHTt^H5k<}x&e(4=f}VHGfn<1K!-b0+1eLsr9fw>^?)Ys z8;A-)%C zNp#;zyGcjWjjhr%KX-?-pYg!fSjnq|2AZ^(9~P>C*zH4}rrpE6nAW&7eAS?pR!+1# zrv4*L{#Yb0Pq+Ialy0y zGebEiK-CJ~82>?_7HQwDi%)=KuWPNE83HBm<1fGqG=Q5}L3X^@%+U|DB-(zrVM5{r zT8}-r24V!Nk(Pn?1G*6)Z+~|rOaTTi6P$C}wXYyb1oAWBH+n%tgn2JjW)HNJrNTVp zrVhBpjI*LlVyU%< zU>9tr>tcX=2sDUxNa#Us9~PmDSLU&?cAi2wIUg_T_zwb0cpQtpcpfl_?-LWlK{{A# zQAiJCF4|uZPD9Jchy)StWWgL~eAD{-eM1--sfEtD*}zZuWfxdYbjDsQ2f-VNKyfc2 zP~M;BtO#`tm>;)p-wa16>fqm_F9aX12`t?@EC!J68C2E?#2r#H;+0TF>9E4fhD1_u zfPVN8noa;MPCUe-qISBH-o_5NtZl(4Z>&ehxE`!E9!^ZhfeS<=%rZ=J7#|dr`{JlfW| z#ZOzpqEHds$6lSe6X0^8_LGD?EFP(-Vp;*W^JsY7Qd#*|zL$HW_afiNM_Jcy-+tJ# zHC+0aCYx-|xA7b&*TM5$OF+M&xXo#J?ZH{siri_;M$g%Y$9h~|cow$}U-x##%74Bh z6*}LIg~v$L#`KT((EKpLdLksz-DZAae1E@n&RT4{d!-}s9Zj?BqwZB(%Dz!fnc~UH zL~xXFUO_wW{vzF(7VVLOaHR3bQ!JcIUE?122^;rwgqJG$K987^euR6zWOA6>koK}T zfava*B`T|pXlX4C(*)4OS?^CJPYyWQ@Fo=`{U|Gp&K=1-LHsY+BH>~ny`GOScg~?J ziSt|D`=~&a!1uzqTMMNn<}~X$mokTIKisZ_DJ5Im`tSusFgN^s@s2k)I(~TYO8={TxuDXmB z4G!e6;&|d2wAOe^*8Tx4)kG@ukp@upzqTL!_M_MO><#7F(d~%0_s#VAZG_77R(vJ! zT;gJ1wT1}%=Mw#GGPRO^+KPX=pJ-{$6`=uRD?0p*%Ttw+B<3S_#xq5$MQ6lLmS_o< zdHdmLJXJ7Nj!ko%y!Pi|P70W;4Y_UT^&LvBLhdBpfD zCuwt5V8Ifx``Vj~QQAknZ%8@MU?)%yFEv+;h4ZvZHkeQ+$Ji+EPd$C|Ds9HsyJFAZ zLqb(XF$SZ69rpY^EOW(GZ|5zG@{yZqF>mkfPv(fThilq27AeS+SCN_8JTL25lGZ( zvmh(id}66|P>}hS3^?*cK_w2e^;{rR)&_#Uwzah$5U>wh5MBbp+lGuVWibKnW>}$D zZqM)}vvYC=LB0$U(Wg2!01luPAlghq%39{r6t%ED&mYVy5*;^A0RO)p=I*aQAtYz3 zekvqfa<8m&mkq3a3B~3EXWkGfVIjmpdrh5{Q7YsMverj!)$%WoxMGqEv_|+}0G`}! zB3KZCft-++{}uiGq=2pC>VMjVzUpc<^@N4N;YV-S>j|Z8$tu$$#+pfduhY*_%Cm(& zNnA`NVka9f5)G6tU#7i$**nRTu(o#1+`L`>&E31bMtyQ#C4$6|V`fCJQDA03`Pl2o zn|5*Fd0S|}={&)(*#VL^vAk!)JD&(dGre;+kv($x!XP%k5Yu{HP z^u04vF<{tvzRy&FMypi<=bejRsMbkI>R(PRNV!v}ao3*h!;p_bN>{*)vOg8a6&{j@ zONBMN^XO>F&YI}frhKt&{yy2HN==s0UCy!I_70`>rHB{jEUQaxL+mHLge-EUl1Hpw z7mggd(U*V{5h6?P6TP6=H0_*FD51A9bEotiWi9wb*rCUp(FD`#f_ZA6wwK&Z*hA@> zB_*pdqdo-u_4O@9A37Uj*HEEwDcFAt4YUvyBw~3vc5B;>wZ8W;XEMf)5;wdro~|jSzLo#&WoU zB4`%s0uX$%*!-87o2z#3t!MZd7WX*V8zJ^>C<^9tc>xUpuhC%9uQh0D}k&!-kwB zG7JqN$^vn7I1LeCLB?&BkdTm>nh#xXZ4v7tJool<9VhpqhkVE$NRBuX>^cf+-v=8$ zvH+=u@@Hh6RdtF!t~H=tiE=zz#Q2d=9Jo7?#2Cx{K}|TQ%m6dan5AuX@2A7`?NKc- zFHhA01q`usnEg=t1b;b7*pS|@dTi!DCn?BE1y5q&J>rVAhj}{I(H^Qj*3Krt>4GdL zX%BG(0d$BtB;M99pki=P4+)%!v7B)M+zrTqs^BLBW!5lITLK=~+qWO*gJE7Nsf3PA zd3-YEb?Z#Yq%4eN38ytaiDw#@e)&QwM+;xHoOdM8nNt+|w(no{WnRF_QBhf2OZKd0 zW=cN9^vYSM7ew&$|IM34k8EJ?LYRRvluj@2L@(Sow!L=ej$m2Y5~)mFLKk0R_4&|_ z-o<?L@YALK z?X1YAA@`hO{td{?7&J8Mo5ObzZoAr9#YwO1q)@ztmpn^QjJt|EBACgU?LoypV)IZ7 z%W?Nv>O5+)0$&sU0owSj;sO8CO5C#RV4E~0Opd}>x$$7h*Rk{mS#+cpuZcRmqzTl@ zpr$Jj;xvgo`dL}nTpSP$V1s|kW%UbSJd=anAeY<0!65*5`m3sHZ7jH%A3i(^u^R)1 z5OfRjcCws{s`vw#eE}_t77X^cTyp@+jF@I2D)N_!c(#n@y*c^dldSKgegnXhS08|b z%OMT%71n_=v6tX5x_9&-gor2$$?lBWu43;zdGbZ{MW|yWbST_0AgHUc!I!o@fM)~X zsYlq{E5}UikI)t9D##pQ?Xg2Nd=RRa2A)}q(-~bIHBfh;tBNYU6&sw<=W-~$v<7NVq)rnBpeE?)=%Ks$y*}TT#!D&SDyGYwuCCP9GsJ0zj-Uy)Lln=6c%*?Q-zbazO6%z#%Y!4+q=a67L_%6O2`(b+QzOCHg& z^k2bIB0dLsCZP&buE+EDp!pTpscY--wl9`E+px)V`xa~zZ5F&U53f=Nj+*<9Hf|g* zCUpF&AS^`;#(5awyNr$N=kxNI&P87w5-{`i+<9JBYf>4=^g)imW#El*ChS&?Ewuz; zM9$8%DYzLr6Wbw$gp0dXgob30OOl7jODOnM2}_5p*gfPf!SihYhwGCKy1`t!u^IMJB8 zh{$4FIVys!(ob2!@MFA8+6vN)ABunh2Q=6D5FWdv2g66;@`+jrjxOG@u*mGVngaYr zc)~#A`z{7nQ-=UAb~`3NudGb(3AaFdG~kLLyBFMsE`3aV@y$*8~Fh-bZfa6yx&OiXgCW3Mz(cHv--Ko;-Zm z{T5|B+bn5mVbL)<%A}-(K1msc<3kr9KB9a;b|;{TEXR*uLZW7-4xAL=egbSvD9Mjm zfaun*v!6m&kNjL+5bHpbr5*1pA}C0shpWOf@MP%e>uZPE#w)F{uy!uP2pUO+>-nAo z=zJ46jcnS1Wh)R$y?;LoJNwM$ap(j5&H%fd6G>QyO`=21;bef9BdowoBt_zFRMc67 zUBt)%sN!=ME*!CdE@@msNRv2oCdRjA5S3!+8KF{ z1+ms-OelHrWq;oebPVOfuG*fkl03om$fl}r(ZieKk>Ar>1kA4p`}qeI2urCt{eGn= zx3egxhfxwpbh&wKTV)0!HeU(zWGBUzdm1sR-t*@-BAXoBtVoDpOFEXZ^XvlC!KWP9 zdoJIZh?us%@ps9ql9)GyTCE zrNvxfotpCBO>om=A{*9?BATCPJYXYfdi%h$oV(ZmcHi_@C8}g;Y4VkWz2iGR$4e^p zz>MO%fXs_w=j06zQ8l#-1=Qx6kd;JyHQMQH410PatRX7<7PDU$CVp-6l=L8~D(^Ka zW>#I}@=?KhWd0?84Tuo1_qb%)NqTd>{bK^1puQb_fx#^nKKX(ga^K`KxjgoG#h-xX_`q8z2(U44~;IuB|#<~ z`{8)_M@2vDF}AT?Rl#u*B70|0^{><}}!TOO?LiSB@nZ7v>v z6j(*SAg_T9qy&3$(H}0UWR#R}LPBQXg!n$hapu;l#L5Z_I3j_;3^+xIhyk*T@H$3D z;mk^;dp%8%WBCLUGdZt33;p_^s`y`+p289)ed&z;KU4Ldntl~+4_a7KB*=@bGRTCBCX^V%xKVao~m5z?SHBSwsXgABtoj z3WBo-VqcQAAxQ?1^stKBC_|jr%<{`rFk{)~iZMSxOTQ4yU-a!;fpLVjO(k*1$7g5B zSy?%hQ{uN=+pX>G;5E^Px~?XnMBZW)R9^u_mbeD-M9zi4!aSNtIN7>Cuv9F@doHUgKtH)_mg z|Ngr8oSX#O%0_hH5Qzjkn&GW)yCzqLHFX zTa)%%H1Njt5)LsDb0UwfCEaSQ35z6d4F8bkZ?ew7zWq?=S=e1TX<*Y{uNm9Y_+*6g zi8*I;2^}2OyPI&F@305IP~r}HU~3uAG`32R+DA1@h`M_JsQfOPLpe<+Y0hagve+R& zHnq{x+Rl5ZbLMBl3gZ#Y$2+fM{8!j0WwdmWrw;M8sVsXSHbm~+|2<()e>wL7cnLCH zk;v~`>SqI?e7${}KAcys*YF|)AOdEt4sztg?}3K1rUdF%P?IPsQkUCBn4jiU@{xy7 zL(psX8WMMjc!)39hdg&3V`8y&lg(|BT<`>Dg@ztBfS3tS1IRrCsimr#8gTuj-dvYw zHW!PPK2OE^j>bOm&zTDsehUq~fOZ(9t3cjy?SrQScDb@sn7APG64*g#<4|b`%XU%$ zy#D_7`uaR&Wo2%|?}Gx4!EFoWAZ;N4<9Z|^b*L!;nor=7z5axJeci_P-3tB%mlAse zFxlk=RaC|I!)1;nO0!zPt`q^B0cUGy4a_r0QX#+)=KgmtDk&L5E?}yuD^u>TU&r~q z*I5@`?DuaXS1GJQ5sp@;9mKn5T9g1B!GQB{fSmlRpruq8gTcdmLnpd_@f;+Nfc5li zBgR-+wN@Xh<6NVP3kWCT|CP6;_LGN@2*n-=MJl+I1AwG(0&r}~Vh;7-s<8*!B9i<4 z68`vV7S1zqPoK)|E-NSZFTCX=9B@c%Q(n$j(`GA4GuYtP$iFD=0ZV7Y20$+lQRw73L)J6U51)1(#=s~z|NtNikm|_oo1b-(OKu4+yW(<*&gf3c0Mgo!A}kp($pNHi9D6+>4TU}vN)~gB3E42kO9Nk zzAp{0X_fL;?>Mj4xMf%hW_>1Yd&Q|yPmy}Oy%XgR_y&meX)-aNt%zXK-1U_F`-82_ zlA}_nz83}T-qw?6B8xN12BpWGZ849iuBtwGD{7Owl89>@vHOH%;c&OTkBrool0!iz zV`g)>8vSf)CeYi=t(?4*V&NhUTszB2Zuxkq| z4G#L%14}#>AXj!n6E)>OPcw_l?`(>jgIBo)Nv5LVAt4S!1eZk7jGsTdt&s;y`U3Pv z4FKby1-N?c;Cc#7jI?u#>Co_ENWy66@bIN>z*X&0@;Zf(rd3xr_MU#e$-e#6cS+XfUDNlLP#=wo5uIxMnJ_O1WKg%GcSR@m$->zvjCzts2m!2`L3zDI!@Hl zd68BcGq+`-xY4`%XiA+eiZO!2<_O=Kp@%#NA7F zj{UmpLe0v<3CnBx5M4*6i7u`Tys18VOaLtOiSv0`eXWw4yk zW_k8NN@3+>B0A6ZGizk@>G5V%dqWFeI56T?U&SR`f^*MKF@Z_K(>@P9w{mjCING?9 zP?=nP->1g%v2OR+s##WU5LWH9gN>l(O#a1mAs1`D^MNBT2j3?;+fs=0IWu?3uFY5A zVsl4*7MY(Pui5I($n^ASgW*m{$Rp|FzZD?(12}sBH@3~e5Y>hv8?Y@7MMQ zpWIl?7ri@2jE(vIsg;)|c>YuEF6mfWsV7T)J}1A`%@3305o2QDCzhk*eY0j|vbb~2 zk$uCuvnlm#lpnP7cRYmumX%Nr6_Ln3P6GyVrBd&;4PU6Q;NRC9RYrh6?I0UcQ zSpnoiS_ZZVLuskCvjTCf zY5#GW=?mdD+nb)kck2MGeSyZNlMe6A6gtKCM;O7X_wAeR%hRZ6N9*5VAtP!T|TyKOh`}Uu^Z1wi%X|Ck#7++2q5A@SyZrw?YV~ zTR^gaAst{ZUmy{pgjiv$TLUSi+QhZ9k#yh@t%Y2;KMjj~*$|lq<>gT%H!wnBlT%qm z1;2a_-XrXrWMyUZA(dbl(hSleSrcj!^Sus8u1&a)N-SC?M@L%xQXLp>%MY ziG98$6Jma#<#z~fcX0gP@^rM0p~o-)edZk#78m`O@|gLB;Zl9x-n?}REals4o=Y1T zG-bxH*ClUtHEJpElmLl^ho@uKEW20uxT3-6tOwQDWC~jYQ=lc8pJqpox#wek$_p;gD}+D0s{{Grv9fFhgg80dc>>hp^g`+r>-*Z<(mGXg1y~d&S=W=`My0C_D-Z!wRNZuq9 zFtyJLZrwf&GZpQ->$u_5P;vdbfkZFmW@bWEQ&ahWNFGa3ktB#5lpk<&=f?Bg*3Ws= z^B?HwMt?&JG1P9kM$R}jeV8-85p@~WfG8e$C@g^&t=zgD(|8Y%rG6TMM*I`Qp-7~r z7KoJLeE&ye4I==wixnlV~-Xf{EmO~P1uT_7#tk*AN=vO1<7(a!_5bB zYf&a74hLRrcpM=yLVo!@9t98aB{aq7R$3K{dI@A8@~`7ENOdGd#Jhk`}^gx zafa}E$hR4A!n7Me_(l-D9TqYPr=dyXxqcrRKqRxnDrryZqB@lTg((ulHIAg@K8=ly zMG|*H)yza9P762y62_~CY^{~54s~`2Jd@CIjYG~T5~yakmRc5T^5lttk^A(quiw7i zx~Z1?zYOiefhtxHN+Gl1^x3nw|JSHxS5)wV^15=tM-fg4dE#P0?MxY)z*sNo5R%kg zWQ)lyOFCF0(f(LNg9!)XSYz3RS3D6B^#$eJas7l@Y=CsrCgXZ)L6h15(<{p!ROY?8|QESz&hFkn&wcd@0y z)c*Zm{@(}cM;+k~3(OQTP<_x|5eHA$W~tBfK*m|L5ZB-1R3`W%l%n?qYPCnrsqCq%T(1xASnxDz!PAV}DmY*BBJ{zvac;}V`w~XiQiVo%5nF0|;f2L%^?d=J zZOSUxbse4Yxk+ytDvjY@Po1p36HiCYdt~Pc1d_0xo~-?hWqTq+jECETqF>q)^ZIp= zZH<&AVqaAfbC0HJjOZstz3BQPYN~8b*y=+odTX{LgmZGE+-+oT@hd2o-_sVJuTA&& z6W!4|c5BLp1h<6{`wpJEvtqZjQr?P7WO|b3DLeaG3vszL`%H>u3}j8@`eBmtvby;B zmo5>?^f~T}dP}RcWrpVJR|$Plk9b1bA7Y%z0junbyQoBaU(K|>CooL8`mC*gNpE~y z%xbq@u|M1JNThqEAKvh)%D@Ok!-d><%c+P!I{sY#(=gUU7prATS_^(+nZ=kyDptv4 zi}POj*_WmCzZK_K$UjG5q%ixlhYa{z!BY;2^55Wm#^K0BK(?tSo`jhur;ZMW{FsAv zj#Plv~lZ&B^J6GIJOLam_?5{P(IIr)Nu&@TALo2}%VQL}>llP{?UcakBZ$ReYk)n&&AWFiY6sziSINuGWonGz=RiCVF`}lgzJicp?RhMO zk0Gl|^Cl!eT}nzPJy!i}wqZHDY{C*#gah-eqk`3M*eH%>78I}nDd8xRThL)1kb3{6 z9Goh^)S@vrXA#T`hm>U)87B8(@yj>iX6iEp7&<~XwH~i|?Ui(LArO`U$lr$a^pV3) z;Sb&Bk^{f;GXT%u#^w=k9^`UY!AY`5JbvG9Q8=;IuG9$2h}5!1J+NuRa0j`#9uh9U zeGWd5{xau_aP|3nJbJ_qQI6C8$ZaSP(K|R8LKyXA*4BQaVgtz`$g>I%`zW{$?Fx&F zA^{WsHpD*`IonQzyCSL)8>~RwK`kOs>shmy1!J%i&o4 z97E89B$!(kY-|skHTav<1|$+^C`)*ziND{<%H@@Dy@6m| z285hpksP{rXt(hsDeXmE8O2aZ+r80wva1%hsES&z%XnH{TfY;f(mFeb30TWq8?26T z+wU)lmM>3H<*ntq%fX}Nk6D4uhmhQR{qCs$L)Lr1 zbD{R{<626RloA<*XpzdEWu~&4vPF{Yy;W3Hhz4b4R8o;7n^5Qo36Z@@*?W)wb$>X| z_j!K*=k+@0`}&^KF+TTwzsL2yuIn9Y<6QbIE77h`ekXt9k0_`1S?85EXI}oG-Nxp7 z_nDN`38%Sf#)Kr1vAt^g4V=vL4xfeFoQy<=t39aInmDXK@Ao{wCey43y+TSwzoML6 zaTA9<6O2a}CIb&;6?1qxGvlv}mI|((F7!8otb<4SDe%q#Q{Avh`H`-%4|5ZR>GrLP zNj;sv^op4lW=tpQMCyDMD%AcuG%3xGm~Wl6Y}(a)`sd9SnVc1?Zdr~$j_p@GG@cg` z=%t;?X5qgj=wOW zx0;Sg<=I8ed)=zkto+`SxznYF%#e`l9T(=mJg>Q&+#mJZsQVwArqmd&Uv;y!GX@>Y zKI>h6&!X92?mSVGlM}fnxq+3;PDQgiw4C@Zn=$h6`womvxv%io9CJ#^EWUB8bnu6o zt%k>&p+qsjDE@XrlkE$vpSM-G{qW;EmsUtJR`b5yyJ>K1H=Es}=TN+KX@TE->w2tY z;5`xkeAn7Uoea(c_>}0tt9T%gn3#B0Xp|?WnGef?wmR9{$3F>VXrd(>Y|iiNmr25> zdODn!kAO{XZdu_1{MOCX+x!)km4V}&Ctd4$T3YXX1M4f)UC$rns7M1}^}ruyEW$Le zFh(pU3o8rP7^VE&2Os^)?N?yP^{ceqcw6{4sFRIhKa9Pr)}pRFN2~wZ!>Q`!r_+Vr zF}sMxrCE8D0|JaQHhkB8u-FD<=U^$8-fB*NXfYRl)f^Zcj75xK>}Q?};n%%h_T$jC zY@omocIQ+?cc`Ep{9xIXV0LpeGqWFWq;o-Et^d-zWqg3COPdw|x1!?BIDA$wzJ;zOU@2lCkU#DL#wGkfqmPfq$EHmFRu~nqM??^?e zv(56+>Bx?d3Yn-o!NJX422*XfZ1-sn9(eNP^|#DAFFx*1KGoH4`-k%88kBVgp1shu zq8`@T_MRp5O8sErt&l;lb6x?rOUL*NdNNM$yMKnV)%6TXVTMaY5A2 zD2nmp!%|6APd%T7tZBR1!M2_6tD8JLEA3hsJE}vZq*QtM?+%P<1>e|K#ULcT^5Yx5 zYhJE>+N{mC47w4HHJNEwuDoyUWKx*y+oa!;eCegcui07Y$s40zji!Gsh~3Qbl60%h z4Fa{weOyiEsbHRB>Q|XhKD|t?iF8(AL`CT}(AqhKnKdPMdBogy0i)YH!&F{o}N!mo^pPE>$6ZAxUEuuN%SO|ZO(fn+n^|0|THV&g? z`!;WS{X>fe_rA>{1U8fcYn{?!O^NL01`chf+DY4jI3($wF71NKgK9gocxMhwfY<_?@eq2ya~YO8%>M5yvtUmrO6j-5IKvk{N2hosi-w(Dr+xFCzlZSDGp zG^;CxSX$d)5pnffv)XqBf7;JR3xXWl3q#v`g;zhhQa{wJ?{RDJ)AqYj%y$OkwujDo ze5<*vJeg&fjFV-!JLjhh58Pn1cjDN8|Tj$)C#EX}asmU%eJa(~q$W z7UjBD?oqjBl4glQZ8H=5si=;Pm<{$g=yM+6+NfL=vw`&&y>}A(58-WE+r-sDXA>!} zjY*w9+fnEn_f$eUZJi?1QoQAtkk|(9`~Z7eS{hmAgkX^S>2yEYdkq(18oK4Qw6tQ; zESNjo#(ugYEDJM9Iqzz>n$wDyBE-eTt(lQm3SiAg2O!$UO1BoPkr92j5SK;xJPJ-? zI_Cpxt9}zBgcl}$osSrH%{a%gmf!#SfQ^+8(ALEx2lNOkUT#Y8<$u=&eWdXU|wi(ub5P#7ozTLmoYBX|oCPg%?WfTTP2 zJyD0-zbu@N0qiB3SrNQkG>Eez2obGuej~wHUz))NQf z9AVRnl9I#dqy`?3Y;>FG|6KFx+zl;bV{UejQ4Q17dI?F2s#S3hz^I89FMS7#OV-u` zkuJZPoA)0nEWo0jw~zyny{4Ixm~jDQ+ZX1EADdhW{Vz2f0ln?I%qmNxCDXLB zMVIBxI|$DY@1HoLaTT*T632~A6ged%*o}&*xdgsl{Xjf8#j&l5Er28GlmE6f*BoYl zZ!bPoc}&AyFCTWncW6H) zoH5Gcs0!(dQY&s*n0q5u%&!xVkKc+*T6#ltG*9EaaB#@P>%sJE**|6i&a|5t7Cz1U zQQ09EIM7yU7ys}D%JftJW0)LFmUplX65a@#s6q{zRaCDK&>yuK6|IR zD@b5+y7i?#e^J=}e#PUut219a_Ri+<+O$U4h29flIk4MskoF)pV%opAkC|6IwC6U9 zx=jZA`m9cw;D(h-(?)3KKK{;|U#}ane%IQ!m~(_rzTfb2qywns%G@K=nhja%PO;3- zTx#NXrKbi)K0frJwo7NU(O5_7?6vkoX}>KpZ4$TSX^n{Sv_$#T`Nck+sEk38SPmE4 zQ)jd%I%Z|>QbN+JU!H+4Xaj}|L;qHyfMdy>Y*DglPvMNa$h%#%fWEH`QDr7zp%62C{In9%)Pk6Ps zmbiwtxuSxhR6?TQF|QWpzmg77`J0eO#$W?N|3d8?CQI4_w@5De@tO~$bD$?O%aW6q zeKQk^PqZXpH{{&g1j62ErVAw2c2)g-Klvy2Awtz&etpZUh0(3BBiTCNvLNC8?D%~F z4_C&5LJu&<41~2cbj%e!J^66ImC`?bmbijS{apRs z%lhrURp=l=Hks_=1_zI)zZ_lYBUx39aNW~^?RyXNytCh8gr2c($wR0b*bZKJj4 zY^Hl&B{+C<&JK zZ(5^um%ilC{yT&7`^U*AZP)UkwNYeePU4kygZV`ZX)76&J=8qIMn+~fY}|G=Vqsz> z(jXOz1nOr~LLSm*I}Al^(=*u!&tTXQF7>1%7Dxt=q=csME<^1D@tr$YA$~S>5aypr zNz~X0(0GnE1jkx6QgS za~{f20K8*h5+>~LPlwR@ z1i0_wz=oRo!u;$M(D40m7%tF!Rl^AZhbmUAupGQp9?p=snd&My+LD7GR|IeLBb*5v zc)f?M15fcBz%hBkW5(k12b)ap0dyz2gc?!l?c1qfE+!}=w~?Xpsf{QW`L$kEhe25} zy2KBMhGCBdEMJMRhzr&{9yMdL*fB;x z0#T9OE;*oXyQ9@lm|ExAiR=xFLu|dfWA|#4M$~h)z#^xXH--1r`eh9b<(zU-Yi|Dd z5oy;Mt2lL`^wF>Mg?V{RkOExQ@3jR~L_FMl$EY~+6IY=7wU`)1s*h!1e2}G&@L#O@ z+u1aMCEis6uJ+r7EgSuR|5l|stfxmeJUrc>FMK+~YpPTMl|6NF;JM{g^$l5X7m1A; zsg~yERM6!zxQ{)|oM?@ho$abh_F7QsZPsnn&2mXuh4Jy|RGpj08=*{zi8nLJw|J-~ zL+w4rzq$S4xD<=s5@yFO%QUKfnC8=K^S}I2HDx;V$1BYHo~c)_+uoLE;Z#pW86W7z z%l(X+k!<#}ddpBTYj&WZz2BYB+F5pkUemj^9J6J){kcIFDrY0mYw@E zrqx?FvqLy+t2F|3^&8e;5x>4O2wN3&jE20~(kM*5&4`X-H!l zn{s|ELyu=})g|Wkk~G!TymB(@uLHek+uy8z?PuV#g4P!U4iRj;^M7|J!bD&?na8Py`%rn#}@aYUC_GemP^-x-|;9%m@UaOeeEWu zPjX#$T}=iNfm=E*6<_$(;%stA)alS<_t1CkQh|lP%7X{S=G}}3YmBDlI)*|xWPB<& zNsVw5$`jYZPTG{(XO6XrQ3(alYsU-fEwf6wlp+sQ#SYcqdPPX8^?Dz~Kx?`DlV);? zH*os_uJ6CP%3Swt_L`_+`KlMPwk1nvB~_#NWeLx->)5A@e>zV^gnKS<+&t;^VG&9| zX{$~4G}fr3a~-j@eJAkc1I)T?Y=(3(@0E`{icBagbgyu3ddp8}Kx+z7}qLHz{K zS5D&qqh#wkr#xO?9V0B-vr$*yAhT^F1KlBv?GsFrphv)_)f=iW?mswoT1#us-A$XY zVUhp|B1G5)pSB%hV&_#ukHG4bn4FAseO68ZF*?$~!*hB^TwODLj0HDf!d;hP_d=a$T47L562=l*S@Z4NRmAC+{r9&D znK2@Olz1E+HbYMVlXAZK!CNx%zYk7UB9abuze>6QBPT3Dw6T)~x?A-^iA!dI89T?l zHQ#YF95^s9u+<(2L29F>X-WtJ%=zT4*Z&s{MKQy}!_NXFO$~Zxzsas+D`zVf#JUnIm|4v~)#}UKmU8<++6ULUrlcbQ z{&YZARb?&vS!Uar5*OHz8-tFL3V*MWSCsKOj|CZIipwUm)dl*=S6}+qNVq< z)J@l7-5Er7vubq64(wKI@;5oHlc;nhcGZs`s(<{&QOYw!Zt#g8+A&Bs`c?7b565lC z%Fh<+)ZF~QvzJ07xBA5%A-tAaUvFLIw&PcL@ox$0YQ9BcR!(INDWES`Lbc{;8zkA1 zmsof6`9In!J_8HNpcQ_%EBFX*`L?N4f!g=^hUZ|Ua?55yVZuc=b^rQX*HJQ7>xjhW ze7UZAoCycA^6s#L`~?9}gbrH`-lf?Rp|bwa1%(Swd$*F9ZaMOWH>wreBAbk2D}D23 zOY?l9T5|d}xrWKi{Z%WV`vv#J&vNuq+z0Sk(0u}D&8Gd6Uq4o-Zh|*)pz;a%PF-x0 z94eNXXGqgG3C4=u)g12X7xu95>v_#NoIZEXDG>gV{@A>D6jL6-bcjwp=X&&GA*ad` z#!M7j=1*TpXwDfb+7_SCox$7(QmrDN#U*WW~JYPz1|HsKhE;F)aGq3CiAP_4RyqhouSNQ=COM~=EzZ*sAK zO}o7EOL;y%zR8*-&z;rC!cp6K6#M0y@Y&Bj+qKqnB&_A5xv*2M$=A?pzypq-aIhjY z9_-j#E0m)bcq!6_dT3~9k;&JIpM9offt`+yt|=9pFSBQdOC`6QuxElT3M9B>$F$jb zz^3>F=H3n>Ct0<6Oj+)`CZo=ZhTW5vmn&3)_~(%H-Gx&j=hhJO9x6LoyOv8Kkt z)ck#c!8cC9Y9E^p*?W=f0lsT0`tI1Y$?^esm9wyrz`PVr;NFAQ`rN#?c1U?J&d$+U zmVR%f%!ATxpV9R73Cqh*?%?OAqGc+AhwmZRgUx-ZiR*;hZayFAcuxip9aBwIR1mRx zO^j)+^vU#mqo^Ib3UZa`mzi_{{Hw3W&XR}RMssAvyw}fty7BwW4tt!Z)!Mf2?fJ~7 zB7Z=zm9L&u|HOSucHq#Ur#mJh4}RB@&W)|?EF6GBm#ANCqVw%@551>no4-X>odU(`ve#1?78Y^qNo%n^d$gb*yq5n|0SL;;lLZ16 zTHJSw<|lY9EkCs1;`j#(?ru*{Yj+J_zrOD1iFLAC^}9e{JXo@Tp=3DIa5f|6mdhW~_Yds+k>93y}=_35TKlUr~G^}@tT6v2d5Qf0ZS~|e+;NuO9++CS(;04Pn?Ve8I z3zwIxZe17Y(!s?L#G7IJYb~R~U-q6cS0XCAz&X&_(?iR_!BJ=apb1?70>)yM2w}wa2@b98n46BPlhr00unj(yr88|AY^bnk^fgs{# zABo5Moe@oeN?okdBq|!RHqGqkgWabFzsb?NG)i&G3%)~S+)ifNWa&I4D-^O~hQ5o7 z7(l>NAb=%2y5Zx~3U>Cdp>z9uU|V^yX8B#WI1kj@_JKacJDEBWm0h< zJDac2m25l9j?vLIaW5tY79>tzzkXLUxge!0Iiul9PTK3uB4#s@fs@lOBd6+qOGe(j z`fa`}9Nq<$)4<{rEpYOb1M*7CvqR zZe-HwNcM>u{YXlru0Q?4T{1Sx4&UYK?>-(LDu$SD0zpID_+D3$IFa}*- zU=jG5`(GU$9B5Sj`ceO5ywU1++xTPOn_OKqW#9o3s~IuF{sNb6z1jZg3d9!G+@~MB z=h==-QZV5t%1Rc!KG=uRADH0k6k!#-8^DCv556KG>geB%&cVJV+NgpFZjg#Z`!7e3 zOIKN$hO&3zxmr}fpFgI`em8l$Dg^TL3$2#Ry>b6?%Ni+8%~=&8O&$#8(Vy9SJSA7kM982!WD>*{At;X$G;nVp@bj2mq^mY{Kz z3sCaJnoW^@0E?-C=IY6cv+TmZ1_Hwf`4+|Y5r}vmg;6Nq`4U?%k}p^ByiZM~8ACQB z+`z$!iCmLWN+JiP7cNj%6XG&hWv>7pg`UUQ8MuS_*RI|ar2RhCp&i4hCaFjnFF_=$ zjhlkfY*S^rQa3%MqB?bzev9Ox%e%N3lx;?I_)zUp8T8*2oV;bNs<|BG({!6eYAOK} zEk>TN0c@iK`SalTVP$;%&PPAB+V41+05qGe+>pl}CG=Ew!k2@F{< zefCk`yKC1y5?HAB96WfkCOvD3o6QLBKE$MJQ!!ypncClU=)Q7xWxh*+9HI=|FLY`& z#c_J7p%Uth3aNC$iwnRj`~JN^aokt)4Gx>Xc0OeWObdn{^35RHwLyO?%eSaZV)`#Z zV5SoEcnQkep$Vjrv;FNcxdE?p3z0OGl$SqxU^;0#zit!6Ogo{tqDus6r?)=!cs#Se zBH_`v=IEVFXcmiVP(E?u0U^ejw!S{QmtQxNn}`!mPxw1~!)ggG)**{BKrEr8$N-)2 z4bL>uV{>}n!+}`&eyp0uKB+kxxOU3wf)MlTpNE>+&IcAeeM>sRC7jkPC@9GCNlCFm zYqc2W0-+AX_$UxkOpt8Xl)~`zEGk$f4v*796{}owt#YRn9jsLUGgvXtA%`R{^-pQ( z5JV=iv9WUTfe><4NRF%{w{a&IODM2Jbsh`O>PVbf zMKwrw&|v=5eeM@K)X2XlcdGSTUZ^m4i}xHnSuv2hb*W4bNQJz-)2|m@3gx&!HWbbz zS7p3L?7g6%9fDt_U52@#d)~mv$h|)_?Ai0@F&{pBaMMv&C%P-^7;t9y$JF+~ke%PW z0;2H^SO;@pbI`v_MN~b!|u@N#6;D2AP!%MJkFjT*DOwaL6{;L zr+yw6w-OV)Q@UAc0H_$w?6V z94MSIV|@CA{s3)S^zVUz!0>QZWpX%;U9h&9gvE_pL<%QA|N21b>0O{H@8phkbC@2z z%qrul6A!0eLs;ATa>1Qn*!<|{Oo!drM2*$OQg$1E)TA0s^j|>V7yj|t z@v$Z(@C(VF9tn3#1lOg1XHvEL))o~Y-FH(^et4gY*rwxp&bYUz&EVC9@OXd#43{AuHU0vGr^1moJUHh#8^gZ_sNX z2ICjIN2JZu!0^}NJeoSXyUqMALx+eNC0qb*V%42vEvf|p=$Hbp#K|@n8Xu%FdR4QB zNG-oT5xh(c^|Q7Gi?ObUqZvEc_K(7QA2std*flGHy*qRM%~?WRCvB7RWlQB_#|kjA z$Sela?(I)=;6p@MU6Qhyk#S82DDe6g>edJ-q^1akbIyL8km zu3vafcZPWrvtzpgJKETcrPRdB>I}6z9=VW1|_L+4zoAfK>>xykmMj{9gF+| z!INa~F4nrs_F?n`P#}2I(d+nKvAuq6DzX3}R+~Nz4|6(q?%Z+I{Y}3sqUdwdgbZI-nr>ib z-o3kwY+sf=v=Qn(ERpoXbCnD$Y&)!>5u|>c4B$u@Me0>i9-^# zJgkhTy4Zi)E%3{hQ+pqQ0CyfPL&W2&Cn+xg*K;gW6`^0xtxpK&wpu)*yo|=rlzZ^+ zGnsQ99;=TX53BrUkZ+MzvVzLiwg_@3-UB8leBG9RfBPxd8lJX`F{D#!)FTNW8^CtX z!xfa^GPH6Heh`0S`08M8#NO*DaaCJol&CcbLC*Hz1fB>;68XpknODH}g?UFLk%?C8 zQ)Hr8b9Ot%y4@tk6C2GwfpcO#`lqtLKEtjH0$)i;Ss>+Z{)YsZt|BK~7-dC8>hdU= z7AXh_WY3&=Xr_OLx~nfK-=NjlOOA2w@cD2$}-rimiHDuNmG-*Tu zO}=yFD+n|#EG%T^IGQzLPhIMj5cNY_*J0(+n_|nBZu;6w+Amzzi4Ki4JTL zO}L1|c7}=#&Xhg#23Ws~(6cC5#dcA6$xztFELyi?S}}2`OS9r&U4P5W1qKtuU8%<| zCls4fe!S7fhrOd3n!G!3a2Jav*{g($@S8cH*q+G#DRT_`z)aXQjz3Y2X&{kX%54i2 zb*Ca~=QMP5Sn-V@m+AR!4*~h>^Y`!4G6`SkD(?|>eZO2m!9PCUQ~_*rEFCj6VVA)los1D1*S~41Cdh< z%~O7U`}R%XLUSohFB&uK2+O_BT$n12r(evvmFh>L?p`rw7cQ98-R0YICypD$aw|AY zK-LH-z|M&e-lKTo_d~n4boK4$Z6t#sy!wK=smYuBApC%Te=oa)ND3^S*(vOG$o%0~ z82o)5J;>?>b(U$dsi_aUEd5HZkPi`NSxrW0=Ew#z?`oiu(6g96Uk<1+fGWaLVdvVdS+>#l;YTcDmq=I`-CGL%OSZ zM~upXl-LbdoZ=kT1bB%nk}y+?q>^>*&|+Xb=e){ut}qhGx`PM8SvU^U}^K3WuVTcHv*BALm4eEytA z%n&uzmp1EGaPnj=0NqmzWy#O{4kE(n*RM~+^J%`Ng0Bkr4cO^&o=9a&kkI5O%qRMr z;@`Z1q00+3CCudFKx-y4BwQj`m^>u?rKtggP!tI+MN9CP7w{aP_J^mrxq9)%MMcB` z;#RJ*b5I3b#z*^q8Kh|FUzLNDYgHYGh+*M5|f0UY@ zyi0rhc7rIBW7E?$Q}+oAZvl@2ZQw!CTW>>Veur6E_tgtUZ`r*021susshBNt##rLR z{e)ZHYAx#HiqM27_$rg>aY+?f_|THe2DqOR%QLW?2?q#1Led3r>t-eK{R&xT>^3rd zDO_s6Z$w2>4*xFf6WhSrUeh{G9P`%@;jft8b9D)?c}X)fGwz+X`(Y4(&_OHIWpbSu zSYpnNMHx4=tV9gqcZ?RxK#=7|J?yPu`EF%GDM@^a@%>cDWZ2Tmig*HZKl|I3J?G#V z1>k)G@CM=lRA+TJ_aP`3q`!Jai||ggqb`Y=nc{+if@@MK3eTJLc<>7#whVwdKelg_t8ovvQ?xTiI~`ay5z8ee0Qy)Egw1pU{*G%fgR>BE-H8JG~OWF{mbj8BrGsOlWDvIOLetmx8rd9b31vz1z zOV3Xl0aJ{BR1qdZ4+}p@^fTOXvHAA@>5FD&I^YuDHDr(D=InMvmq84*tVLO6rLf%^ zn9|L&rOlk38UH_NG}e;0Uy_+&ChEE+X*6drrCsj)x1TA<5g}i7f8h#Jh*-~nU@C84 z8ZKscnkkM$)cUaMRvNG)Z`jz_8~{C@fFXh#G&D2{>L?GqytbC<{Q2{;Z{W0m7aP1W zDDB|m6O_Z#)zu}3N0*}!tGEMLR6I`e3ys}2z|nUj(kgIiJTc$FXRvtY080|~m8(ynbnVsp<^Btx4)_D9qtwFO#O?wrnmyIT!7xW} z|L)3?*4dw*r&2(XcFD!PA?-o@6gt$t`c%^XJorbbyvvqSYml~ND`e)VOleZ#jscKV z*-xj#S`_Y7jt?F@SThC@KB2LKNp*kIUEHj@z~tlmZ|T1`4Z_+O_UF%^Q zb#(6PW;xOb;dR0FWkhT5;)##UVTT(59cr)W z4^UkyUoO#xF6>O1OS=)8IG#g?PHlyfVii)SZNcp0&lebqax9_ zMX2lfL&}k_*|nr2VQ<-9sg1`PQ$F$}PPiPlk7 zRp4vL7rpJXKk@b@QE8G4Pv~URU=b4}XZOP^81S9;{}53z2D>JgOmf~TF0kQwe&R9b z0gV=jYtHF=JQKJ21Hj@%lqUE|7egi(>dzX8$;db#KQlGe z27M{qj}jp6kWb9AZoA%ns0-8>;?_lZ<*iV@CHlJ?|2gfi@%P!T`I^eFJ!C_9nvIQ()XXv%nxku2n~Gr54t)_Gy2wEm|AZ%>F9hB5 zft4z8m0D^<$(-erJa*u1{?D0*$tx-;xm6n`hfDESEgK{l z>0P%lUw_z*k>I*@_3G6r%s8_QBG5^>6!`5EM5GtsDzWjD|2pWDNuN=+_|`uFaiSD* z#6=M4EqG%_`P36>jFv&TJWlX3K>@=0amK9y$Q`n&XqAj7bwx!5Q#W`*YEom2r1}Mk zhryp)W_ECexv=(xCSX)R&%?_ZTzp5mDmuLBrPZ617=SJ_={YY$NF=u;BQZ5~GX?L3 zdIyI=A*@49`2M{C3f)QM?9&skNdT%zID-y`4=&U@cgE!#FAkGxbhuETP!vjNo)8o- z!|~^4goIN@WfQ0auOJh%Kx1VW0oaT1QAzcvSM{eRBl?JrPI&hdKKyoS zXZ7_1DQf^oCuXQKN*X;<5~?h&&!T8OQOYU?gJTvh>2fmuqOtLyvw*)TT0HoQPGTn) zd5RZtxD;a7mMuKH*${>hBQX1?^m*rAYer{wh2clphLqPfR;ynOO~L6>KwX2FyBO8K zzl^SY-3+c&?PLM_;P65Z=53>_-7=GCQ z6NjZ~E{Vg)-mOJ(7y{Uq@h$@@_7Xoge)Hjjl2rc}T?K zi%B0 zO=pm*MXcmuF5v6y+3Eh3GL{W)M$NPX6PWlRXE6qiK)4nfO2QiOkA3!fi?>7-h*N&9 z$7_OSkC7;hKb(S760BhEQwD=ED6(pN8nM_y^R2l!l`$xuZ>jK=WNF0|x&BwWN;giwR)CWCyli{NE)ruOCYXU}%< z@u@(t0*h6{l%HgQI@zGs*VmWu3r{GYc7OI&R5!{;mE**lb7On zFgPw`UE7%&CO$16d}+J9Ajxv1cTj<>Fnvy|72E9X?VXGtlT6^)MMBmgXP@|=oUO2T z(a>{D?cNbuwj>MeyWbn}`v;{u_jN^S_ccsJEnXK!Bf!_N`&9!}NQ@P4Xu-vDv1X&- zLBMZLZj5O&njZa&hJ2$LFbE?#16zu5(PCT1Il&CiORRyv@x*S^W+&Uol8-J8=&bu| zDl3;ECs&6DUUpvI_56H!8A>(9Jt(y)2HBY8?k*c-Bmvw)cpLI9tYta7y}@@!LF7{p zIKgEW&KiyVSb}6~Zq8_S{L#K^INt}R#x~N;U9%Y^d7r{Wl#DIFHi*szc#hb0^jM7y z2#*gRK185rr6-^qUKL3(6wJt8ydr$%3|t%Pa@^7q=1aH&w}a9tixXq<1CZn@rNoRP zXg5QU4v+$fAOfu^l^0%zAtn~29=vzgH+}(X{bU?&^zW{&TZEiMO!);xF%KO2p>5TM zfoj4$*fHxJrGp0=$F`8!^!bfX&uB0i|) zPaUnOw-TEdRHGHy-e-#f)$%?1i@@#ZfYlrFykr;^f*zE3oIH8b$L7e9eBVhg;+jYb z^>nzWl0%n61O&Zb_RZ_p>%dfd_`o;Y-)_^{MRrKg42oZ4W8*cc$a963X~(~GWcLZN z_@A_emX@!)crU7@g2KX`&oT~m{eB2Do2!Lwl$dU(ljC}(E)BM$TV*`6Kgk!W0lG8$ z4kMs_#J!*?Ld;Il)Hn$kJTP+tEt_&53@alHF7-QsfkXuW%+G~_`5is#`v5Me)iQ45 zy&4f|Utqx;F)(3FoOAk}zWufLy>o{WFyg)bw;7ky&|n}4${8tM-dj=mC^Fv=Y(l5> z1?=Vx@|WYgc3wsUhUtnmEZl0|fw;GU>_t3Vx;xXaL;}-))ZEczoVg$1%l!O21^y+5 zBG!=1Wsm*P&JngEfFdcgNbbUWCq9u`j(r==5KMb&la#Ej-!4tncOzB6E$+<35vsI+ z0rYfgvU_5h31Kd4Ah(cXesDF$)8m<{=+wzT9dn5d2y2$ADBU{Q>Z0!ZmdugABT38* z_OHe@;M~txv}rpijGz06*RUrp5&ZRR#Dkp1gc*=F@7N-6XT_0 zTWcZFwGpn|9xCpTkl^xzm`xY)$U9vi?KlcI1aZ{k+a5XC+dqwuKYDCSmqpUZ2jDM_ zr5TkMahLJBvOISjIH0lB$WxnK`1{ao7nKBoQB#J!o?kq#Vg3mQ&uO% zT%$O^VBdy`(F(kPJ)~q4#qWv)Yfu=a1jp< zga-m!@B`nYktnvaN6IAsk7jHrLNf4-CMKJ`u=Kyo2UbqdCtDFRB$ zB$6_)Gg_cg`>J}zG(odqck4Z{xM4_+WFg{kZKnF<7Q5@3CAK0S?F;y$LEJ9tIvQ{G z6YMP)_bADmSOHQ|QEcTkqm6;M=Gf0I=?-tI>?)*g0sk44-iTvmdaK6D5aec)MOyAo zk0r|?jfJ7Zoy_|%vw6vHPEvXI&6YM0B4lfJC17OjuPEH zJ%B=3q;Vi*?OQca`HU!e{sS!j1$7saj~9k4rn#?QwWBJvqSFt4c7HukKWCTx>^uZjH*!^k|gB-y^ZM;wcOAMugy?y^!0;oFO z@Zx96C5qE3*IB}s(MjK4P53e(lPLQs@^pDDSKS8^|4)p02C-I&hA=%*D-{?B96cr^ zK&V}!xLrU4;RFckOF~a2?8dpDWu#!Y70I-~5M({DUzJiAkdr>iCh4r6UfWV!yn5t! z8N$KQ08mw1Bb=Hw*rF!J#vr(q^hSpzmXCu5s7FV6qr_V>*F7VS1n7z1XbUNte(;BC z-eD3?*rlP;9^!&5{yz4C4}j?sLxIdR4|FOZ%-#O4ca7Sd-0zxq=*A2&iQ0IQ;pM}S zkPh5CtnPEa_T$Id8e2fS#1|cFQiII`4v-_n#$iPD{r^UFt)I*AypeWfUA6}xS~<7l z&H4T*A9c+!yZ#lRJBnZBW1Z|Q{)#p7Z^2wzJZXGgG6zJ{+=5;P7%^;;V}0u3tLLL^ z;pb{ZUUiMxL-MNU08`!=!qK@#SR3s6+2?r1h_TB8oTH3$OZJ1o58|xKRrW=M>1<^1 z7sN>l5#3O2_-8k^x%_pCosE%Rj<#JkqkU~PBllo`nG!EH)nfn z%9Js9NxoRRrDm)GzA^BH*d5k|jcM97)zz&?>#`PrOkoIr4=*}^n3bWPKc`!~dE>_G zT~S_O z+=R*0k1S2p;?w@T5hi`BN&lU}ME;IqorB#0PnL67mO*kEUg6ose)5l-bTMfI|KSEX zzNfojKh;km!vR~VtgG9F)X6XwVt2}F@vLF%2B(URX5bkT!>sVbokGMe&oOfLAx4`B zV^GwnO=xN`g6C8IhS{~DU)(ATdC&tSbilRv;FH&>wKskmM&}OHkic?Z&H;M93+HF7 z6yQeXwuMw~W!Q#{MB@Y_+2>CT4x;rcGbQO%HA4JZCcg=&iv$Tn$@Z?Hk20e#kGf1c zGNPlSEMCSDg53CE(;s%o5l@L)#nzZH5DE=RTML~S)^&7t8V63FXn<)Z48vYM`2)^L zYxFJl_8|k%(6fL=tTFP=zY2C`35^^#_7aQQcCLYc)_M|>@A&(FCMEaaptbT}yzhSC zx0FC02F)Y@2yP1C@)4b6>zlOZ0y6X~UA}zzVD%}rw*1u`K6*(Z_W~N>VZ=n)euv^n zGsjgZ@Fk(%`V%`3!XNL}Vj#!P;lYSs#`ZeH+<9ChGUS;ZPs=+q7ddZ=?GCU=_PUMj zO}6v=2wlrAL}9K!(e6mF@{$?Ti>tTe#AD{~jMmsG;eo5XXW3p5A_4#q2k#FQ9tQY6 z(1t!VyKuja2hga{e(uaGe8_x4SWL4MM7uJFlDXkv3ZqAI!x+ZY;O|csg=sS}`}gk;{j)d80RG||zXxTJX5T@& zd8t@;F()U)P>zNh{Ql2iL*D?&5saapd*IjYX0Ciw@gDdHMY9Ejm&_RxC_E$VH6%2o zHT3i$_kvN~Oj59x3B#rF>!*q`l4SIbCe>_j#7=8!hDzTJhTo4Q{VGc%|H4CAP3@fS zY%U;E5QV@gjRdt4T03V1aT~JUpZTKJfBxa9MT#3-{eO zz~Q+8){}a@14Is>wV8c0(B2uouKN45c7zAO_WAI@Q^*#s_TKkCemv}R;|9&YS)Cia zHVP{n%hew~Fqx#KnD~`(`1u;+b1T5D_NfPnA-ZZv55TZ(6oAH;<;yxdJKrA3TuFq% zbp`_Q<@eD~yMwF?Z-gYDCs+qipL!+d1G5&HrKm0u8RHi)^zuo@BpthC_w(M}gl3UU z%?Jgp_|vCWGSe6X?ST6;$zfze-PyB`P)AyVaP}7FwZyeBwU-R30cPinPp)UEBufjL zoB6A8olOFlW$g&dqonz)RaE2?$5BSt*sitY99G4o^R$pjGjRmaiBKwLnyASQP?upC@ z(Q~CaWb(DuL9kEZ;3o=+LI5oN<=lp)M`M}~&-SI|pt-_3wJgQlZdx&ng$9U?U)#T(h*O4(iH*RKobX=@o`P%kTc z{S$t53_d`V($dme3iW9a`csw~08gy;zJ7gC5P_F?5V*|W@qw28cjqu|0wRDRUgHZe zBVo(MPf|rDvLn#5+n6;BPxEmOYoTzX(0@iy+v6f+SO5?StVNlL9ks2%;}1ioWi|z2 zFma&f1O_Q9Zu@j z2W9V2L(BXYxB!Azq5ftQ6Vok}^$cX*hC+fn4yX*vJf%mMTI$gyE-z*KYi-NKPz1En zTta5yti2341?H=%0DZtFL7hB`uKl#gYf%w4e36Q{8nZL^AAk!C&V-<(KZguE0B6}g zAkdKDp?#HnTxrT!RDK`lv%w zW|eSd^!4{wW2p!6*!cI@@$0%n^|Xthe@i#<=*O@J%^TNlW`BY%eblzfxj~k81e7g5 z3=0ZEf=6?~4iEy2XXN3jZmn0zMrdz5aW#5@qE>We1gzG>KGgzeiwvI ziaT#CNGm5}qC-faPtwc5hCUujpuaWJSzYdLHk0B{>okLmYu@||BOF4$Uc zFz6*cpBOp>vPla<`Pb_9yUKC`dvyQ#bNI-SYwD#XCHEfu-m$h6ewIAe5fAahP+UXC?NbzY>liL2JwUl=<_iv*%`;h3=3~>%FV*(@mep3 z1${HwbOzmh*@a{hB-C{~@JSCrmc_MW2iTh|FR_J{D7qvrLuj8Ff3v!(Vz@){#6t9Kw4l!XVX!LT1Co$|3N8H~>(czqC-`q$FZ-mt-x z>AhBSrcgCO@NEbzhimC|R$cvR{T4hn3hSTiJGklJ{==Dw+~o76GuvweUOdsK_<_Gh z42#7t<(mIINU>rO5=D)Ko{*=yE)-+0?9b@@QIKR6qo@I0{ZZ%|QTIs!U?V)$qpng| zQG1o(KcpzXiemxMcAKmPkY@_}!q@ZWY!b}$b#Wn~)g$X{$%=*DJ2D#3iP&J%4@GfM%;CjK zIJ%2B;0@v?=7DE*TmIlH!Y+0TWo%n{)EJ_M!PXU}7Q93khu|YXk_%ZX|__ zV74X0b-(hP(Nb;y`Mf15H?uYGa`3L&$;oo8v7F?8y00K31l_y0y*hVc-VIbpRP~Qg z^OCL0T6i90{8G(PUd9E`|Az8wKT~Ux82-5Q zwGiXgrErgX2EItA&oXP!KYJU})Z&rwh?+@RS=m+=pGw=0lZ|X7iTxF_-cGhcCmBY{iOmZ+E1W?rtu{Cw}36%2&Udq>9QtM;iX)t(1RfdU$bt z&kZd=LjXy3qR6weJJ?JH01#yRz~2T%0#g{t zpnL;$m0cH6=b`luE;bM__#US$%LhIC?U;3cCmcNWg&&`go1P;OA(5yhj||bO{cf~> zg&rO41X&cBaUNv-w=IdaJqNH#z|T)H0KI`>V|6t*dD2u59zJYC=Phf2NNFB_7^4Jm z3)tK06<~vk{5h0!^c|>cHA2rQcc5erp~MC(#^^0zIsimme`lf`kgy>u8(Y~GK$j(Z z@!P~?XY0gYG%@*yW<@JzT;yp|=5O)+yHViSrp3f?*3A5dDCTKx%H@@GGN@U}EIfc+ zDnUyaYvm;8Cp(Z+HU{GbC6Y4%Do^BftN#A|8%%-79wVV+kdnOU_YQho=z$dkR6WQFrH|Ynzk(fv?sRL|2}xIF~EnP?@j;Ek=%U^33q<+7;JI&2 zOHF;rV~^Uj0x&FD7g!x`rNteSPxg)9l>qywOxM)7%2Y5x*?_R35fW@cJXq5KaS6P6 zamx11zilsII`0x52wztaR=#_I?@SremY40vX0%rY1+s>cINiy~!RbR}6)`APD%9vh z_{+PbVJApo9051<7et+yZu{%DdKkb-3aN!*u4KyYHs~ABqml2uvaw#+Yv?o6zq!5e zx4X;#H11yi-MH^8MH(Wze_wTz&kO=;Zk#Y@H`Zj3Rif1>QqI)IY@%C@5|j)#iKa{o z>{ow)=b{>EMZ}!8d~WJ(M?Z0qZoMJ5T`b zTq`zQoNh{!#<+`87%l~MtcfJ7R${$N4`Bl`41&WB)N|aJ&EqQaHp9hVuj>f5v!$c8 z7_7gNWQZ}sH?yD3@QOGpOuBUzEt237(Q=$uBCGk9+*!qJzvh5-NPw^DE+3ZEH3zM~ zFT?K{<3>5ehK2^R!d3b5&9j=Cq1`thp~xi^#CVzTF!x0nzF_v}q9RVHz2e!VTp96c zfF;FC=#X_T-wCM3dEfv81ZyB({QbwkuKBmP)qS9=QjkEIoqR^oYfY;WlO=6&kv#l- z0ibhIc2tE2-l+UW3F(+oQTl_MAi1Xed~J497z`a@__=eY;h)PAvou=Rq~q0b?~G}n zv~h^P!W%#rDX~m5B2D{9Cmx#u35Tb>O96k8xWEUNHQ{5$cmf?>oULQ-@!+Nmsg99P z*CCo!!;I7ybMp)-GK5bdl-vFrI`2&xD%8Het*Wc@C8tPcwcV%+h;DhyKFyCt{qln5 zL>)$mT6|!{f=Enx+%HiUlEpv%BF%>{UVLUY1^3Uvs;GHJbm2hjPAqzf%8`whQSbpU zla|FvaI%P@ZWFes_nDq{jb?`wvMbVUJsOkX2X__M0?r3tu_G;Sjt`-Dm4e3U9;|3+ z=areFSt8ui>_+~M7opU3_DgkL)=%<*0UU_PU8%*{SeU-Sa@nn3;mdf5vEUml0DWKbf(HJbE-PeuN(2&Pg4ebzx;c zq<&yY;fV%+sPj%^px!&b9rZM-C!M;9F*L95FIO$PVRin?sUS-9Zq|KYDiQeaf>QDe z`${>fKIVGNP^vdz#?~z!6UxT~Cy|y;*KVckOeD2#CL~PJsi|j3K+lv?f^8v1S2gn& zKwvTqN7K6Sg=~L=84KVtdJ8*y0DvKYZ#NYc6jn96bg`YOItgAEZZ(BtAyAt;9JLFi7^$!&d=TY%JDhpsL1T#X;XVb?9)|mrmB{Qa zoeckm3E!j7(bJM@{bWx;Kiva0R-c1vA?`cbFkB%$vyY(Q zBvi%ZzNHj$|I*@~FEw!eboAd4+376xNnJ;+gcW;a1rb5R;{+GudTniOEZ|DgjM+Z4 zyFUQ2rf?b0|3}uF2h^CpapO;mgrZ~!p+(wIQkF`?zVp7nzh=hFoKDYk-`9OzpY5KYOt83~L+Dzl zlH}2DfI?pLAO3B{qNWomGscdCfb?msd=S4=$Q2a9fRU3%@57+yw7zLE zvFg9?R)Ia%rJ+zZVJsdMs<8A8|Glj5PFHs{RAlr6Wq+8EVv4vuB$HduhP-}YQnH*2 zyac&6PMe!&1OMrGM3UCYKM_o__(tQHe7&O}C+)z9T|Y8POib)s)xlCb2oq_{B>sH-1cbh1PcP3f$CCAfEMKXc|dn`mp8TTIR2iMrr>{_ ze7g4|JYN!N$@D=rXWUV*Fhe}_m>jR7JYh7kaF|Qy0a>NcqJ+b+RRd-KLgNO(l&riw zP1~}dAe&B*1QzGX-ar3@31n}lHB{d&a(oDkfd?thlJy+jw5)vWsihVe*SfGrgTun6 z;{Q#Sn%49u?v7AMX~nPNz7e{)(+CBbsyaUX&K(`NUnTXMrbWUEapM2-SmK_Lz%tLg zCFnrTM?_rRw{Zak&#cD%%3GwSkTJESsHk4NX!4V;*%0~$(qyM{0UmxJR7#Z~{#oCL zbJ2T@5W_z{ft_h3R`u35FJJnibx1fL5>o$K3Qz+b(O9y6Tvu1!$=pv$-mlTuoQ3F- z#v#^P0{$HPRiYLMVH=`Pl3w^g`}_d}x3Kt&_syM9 zqq0pmWYMWd0VH=8t?|ATofEK(?2+ufA%Y)VYWi4x27%;(_SbLSB2~`%I8+zqI|!F* zQ7aD?eEs?aGL8V!w7XxBTZlO%IlEgEvIfa1Q+{5?f7U3K0!SF&i_>Ese%(_#Vr^R}F*h171P*ymV$Ip|M zjs>}B7NxuN+Qo&QqY-CCNB99}A);Rhg>9FDD>bi^xQ^8|?#W}8uG24@?EL%Y3MF4g^*c^WbE)s{y`2m^t>-u zSm0V$+*d7`3ekrWMTaPVKJ$Ig7zB@C%?dG^6KcVF3WiShU_iD3laa>BN0L|lrh{{_ zv|~S^9mgA~-U$-{2IGw89>D$V=JxVmWoGsS-#($@K;JN;@gUsXi+Jzc?5 ztrq^nix@69&pG^cGhI#Qv;`L&zMwtabQH{5b$1lBo1a#bumf-Y19Hl)?;@E60t}JJ zTIAoNvKj0jX?41#I*>qM2jeL#tAer-W$>T-4ssCP{l#6seoU;_Pb+K-%>VgOAbj0h z6udh!*yyu7{S`)tQGbU1?k=vdXG7AdWfNn??Z?j!<~~d@(jW>~T5Om|res{;UY!FO ztbi|FSw&@oTpZCzhwc@+1QQO2C?dGCl?jJ?xCD?osaEID6;r$PI2BT{y{U!~H$iXw zFi?^`vk(gB--(tB^T7()>Bv7SFk0=W^AE24+#r_|WyW9JzX*02cSKtTtz_HwgZp7E z`Rf3>>CaiXelDFj*Jt>0EabKun_TjR(;Qg0O;y%Gnt9!>8PU6qkV;;XeOm9bohYj3 z_PX^LRU*Oyxk%^xJbSp>JzT4pgJGKO)yUh%3ZG0(uBN1n1~Ca?WEFm$5H$pI$kAmS zZ9je-D(Rr*etw4aTDn*mSslcN-FpZDndfQHF$$2;T%&Fk5EVxA#}z=-VLeiS^8NTe zD`*oahqtE5*ff7NvgP17(x^0>CWkjWY*5$w^46Da7!ukbVU5R}oq7}=PnH!2<~&SL zNNaREkalGGEb&4x<@NbP>5ScL51GJ|Bdu8hI4Tk*Pn^giGNiZ*ed&Tc`5oyyVdgI*5a~%u6z6-Rol8J zrK8XKXL8n-=EM8c#)uD*l?~958$Z=9c&hOV`E$X3e{`A3m^>*3_eEnl`a-L>6$$#>SiO7#N<7dgpzDwY$N+>7lV3ptbKNhuij!v%aE zw$Ma(%+~%A`S({k)h#n+WE`NDx5r!MB;UWkUMm07o7#^*xaNj)S|oUn-@Jak``uoR zw%DcDB&))%B5ecgkkC(rCxWMSyAqD(=up!KDE5i}+NkizKXkJaBh)##namv+CZqus zhOa*pT`QL5WewTxa5;x&F;Vpi5R!3^8%t-#0hRYjS)h}I@luk*L8 z(>U(#X|8|oOD)D#D)z8c{>nP(vM6p18vdy`_&Gl4#Vvqjs*Pb7=Kdon_)bkI9Q4}61n25hfoj!SX^u*QxG2@&&R^IcIMHwdEM;gB`Qa8eDU?lHF9ePD=zBQHaF(*K8LkY=&miN?SMmz)qLl2iTpL%a1_{HfNXf^FE@D2(Z*FJ| zW5mzqDro_O<1be4xRLz*R)yf}n_3YjJuA0I7ENVnSlAVf=e+{QvR!pLW{l)e{HNt* zW${8-#Pt$SI60?TC*c4Qii^k`pxb?j5aptXOEKb%R62Ms4NXl8E+BqguWmQ<6E0f5 z`Nz!{a@@H{NntT{`Vp8ybsRB^4@c18WUb4wPqX71dwXW!k; z4d@xd(Q?E-7L!~B(AgEpY~oz_0irNe@=Y1$SNqLx82o|(FUADXv+Zk-$SH&|vnyzF zy;OcM*xW4`9^!=o(i5lAeU|-Rc;lw=(0>&}yQ>H&iDLwZ8J6I<2Ve9P0!o|Ck&DTw z@7t(~=*zaSBrl^Ib*!?F&hPIPVPqsCONTMN0bLjHG4+qHFTh`z7Td7r%25YKR?lJ- zr`tbUF!kisV*Ih#4rKN2NbPIAJS?ts&pXK(3P5YMegGjh(=Ni6I92d*-Di+`K&f7a zNd!vagX+?xqEk=rLwr#)rzD_o8(QFqZ!G`pzi3E3Y}AD0G@Rgo;&~ z^F=|I9K$>UsIeYrb~p)13vbUBH#LP`qyb8+P@w!;y#O#gA&_r?(yB5Hn|Syy694z0sBypDA~l9F_SoK}jL+1W_4@e-N9NP0kMrWN&7~qy zMx&sVF7E*zY?eQ6Ni{%;FzK~ki{g;9W3v;JbhF38$%%8M^6w`o@%rY%xdU(HT*sVR z?SXy%2)}A#%jeJKVdBiO04S{4YL5X2t3{AjTmOR_ZqRmAijgOc180&)j&h~_?%kS) zp!bPYWOR5qz{Am6#5|UXPOo;x4?>vE7QI9f8I9NV2ejrd*kRSQ<61g8O2aMJ(C)*j%}D;lI8sq6b?Sz z+76>*G32+ZnkTckO0IpsJl_H~F2DD_IpuKkmX#}3l!Xn)9(+V!k6h6ei&h|)FB~s3 z`irD62B;4mE%rFz%K!Oiufb$Q1VA7XvdPACDHKMfJ$(%0r^LSK?E^3RPEDT*K2UPV zLPmtwH*1>jcMXQgHQoZBk#1m7Yez|};Ss%QpM7RVl-y6S#_o&NVR#Eh<8y&Sagq@y zXE{O970C`ID2ZV;7+0@AE^4X#>0j4}Veh{D*vAoN9;+fiGAZNHqbr%2nYG^DbzQFo zZa?3bN-$LqbC1YQKD*p9v`=>4*Az@fzYlL>-y*baH*)C!FIPmPk6e**M`!2PVG_ny zz^2{N4MZkeAs`;TX?WG5`8o|qnwirpy#+U?JzL*t=-crGCwgiFZ4G*kh~Ozdm{Faw z@747?lMP_jO02}MAa#ETBc289K5_D*qd=~&NPTLpf{>irKfzn4IaWwixpMXDs86jD zOq~r0)dA6jVS^{|^>TX`;AAJ-t-QMWmJm>brgnDcC8>#X%-C1{zse3wda$zdRvd;p z+7lP3nzb&&dV0hS}}`Tv`v6H znFD~4Zz`surnWo@1}-gh8|+~&snEU6+8XDzj1^=*TI<&JXQ;o`k$MZ6Otx`Zaign7 z>YFdGXqTUQ-+UexFpaF-gy|hBZAT}1ERtZPGGk*WS9p=t*W96dIo!c$Oti899-%3V z?*|L4eh7OF&qpv!3_(I44rhehfUL#pQ(l55NCDng0Z6#0Y%wvWtd^K1_MnI4eYV$> zWo>(LA?CimG*O=K8!RMXHS4&bjnT57xRc{N;^^3OQBg_gH5hk3MA1}KY;0_5czrLh znTnAAm@LF?zibC^2wk4ISn?aKp(7R6a>3M|!+^YxBVM&2U*`%eA(-EGPuLrWa3*ae zPyyPlr>1F1fGiCCq|_#k|l5LAY(k1{gCcj8Np)wY?A z;~~P&PuzT%tP`r+F514YUL^|0P2a2t`uO$Wo`+G0hTDooC0H+E1SJ9(=i5t;!-3`9 zEl}%1^tKWp*$*8(=5gb(oPRZH`-GsoT#1gwKa63v>&(ImXqke=z%Y3M8{DxB7-8H2 z=c_G}j7<5nN0*)ZmGDPZX;7d5d`Y*O=&uB?(aXqd{fLUy!Ykd?rpbuUKU;9Q5(PRs zlnGHq1_6@L5{0Yd7CrBua(eCRtKBzCn!c~xzdxC;$H;6I4Y*B*1L|~loN2DC!~u zuj>9|@4D|J8K&DB%iw5Fa~ek**gT3pdiNI7y3`Zzn_tU`Gs=F0z1i~R0p+$(9h{b* zwZ!uJV>@~Ot(hL{1lXHcSxoLEgF~Ogbd3w4FM!cS;%mpnMP6IL?mK_R^#m5^De*_5 zjEfO4SMx7fNY&I&6luALb_(5iF}OGL0%-WAV~#tAPC&Yd+1^I@9?+w(8i=t}A*QfY zMj!IXO%RH!24bkwtGEk^wbyH&)&!wrOrxCQudMf|A_JuO$Corn*XsyXAfA{XX5JUSk800OLt4`!Po_c(29*?kFwSd5UCh)OS6JM=fsegm8@|^&!0DP z#TlODlmEWvg(2A9k}9!#W702oo*c?_MIJv#b|>x&;n!b>Ea0`=Zl3r^ zrRiE)v$IUa(2L_Frej*(X1G)+mNS+`P`Q@2b_5~~Q1Z6XyKefy)t@TDt3w8eJxF+( z$nV>>=!?fXUR>Se)o;e+Rwtoz|4fB_W%s7KvJz+*VBS21T2SUgPkd2a>+U}>Z9;WE zL2j|>ljg%(`W>AjQeh`g4u5#yqknlxY3Z9V2}VW((3zRZ-U8^*KwG^DlXSmy<3@>4 zVH%qLHS$a#1IufQd#g=i1f(T5Hr_fJs3h(yRu9#%S;yaa0Mk&R_;yR!7n?z8u{52d zy-qX&SK87l${Zuza3UmV0)#z&8YWhC863Z7N40w*jC=y9#k*ieR7C%DBJjvuM%nsz zyFUNoYHHV;jIn(vfe;s&uEBcS$<=-#yi>a;JMI=>coeQ@09@Lf5lP28~f)~+C^qn~;#vEr$`3QyA5=_W+mriN} zwq_sg%Jy-t_?23=)!pRl?E&^y7U5>Uw1J0k%uo7l3AL>iE z3;b~9p<>3J3t(xr06?=6W(WEPD~Rs?)^&8QBeFZemNpqiD!zGhXyuY6d38fYc#>s= zQP9Z^27$mfIN4u@?QKNy*>7%0OG;0;0AO?7L|Rgh5B5o3;6*G;mVyu3Vb z8jFPy=*Q=yVJyeJ2Pw$=q0>7H%%Ud_5A!eflST$Wma`GJE+lA-XQXZ;Kw=qNzj2|K z$6-lE9yhUH44qr!VS+VeZvd7+DLJ$KB#oFRy5bqgrYJ%Kb{Wuf20rI}>nz%2|AG*0 zZ;Y^w?#7fD28)zm#{`!(*!_U+topi2eK?S#IQ)4w5FsZg>Ea1Wg+q;;U?IxzwzEas zeV}M-Z>+{{X3BD=P5aJ>Gc8Z2zKFv=A9u!z<1h9a*v1TgEe;fdcwrkQv607~daCz} zvYyeLx6iw{ntH+V1+awE*@xzQw8xn;G#KcB#N*JW4H}sH~4y;x7hdV>^wGK2a~eQ1Ca6ixQAzf zmziTH!bn|%h=#Lt>0jkB`i|$NCK;?p#ZE#=Obj}8N-m@NHcooKS(pu}B?8=;n1eMw ze!lA!K8SZ6sU+OSjKaN_wldmIg!U&1`N*T$=-BUk$^*{cE*smk_j5&xS5@Gj&43FVqpVz zTVn(Wg-$lIo07Ib1>E~7+HvKQKgcjrd?bHVOiPA1AAMO)ZheGr{wxv2N(iQ7vQ36AlJlGFJMnVQ(M1l9 zjun{3(luw!-1r$$xl!PI*^ONWC={!0Jr^xW3dnK5Y;!T6hZ|7kmSH3hLS}IeB#wF( zB6*@~Hc0M)-7EswTq1((sQ+W!>u}7L*VNn=b|29>0tGwWd(UrLO2cl!eGe&}80(B_p1y=Z| zZiE0tkiG5f2zLl5O{m%TgvT%O`t@SBDpX_4r48`&hC>f>;`y_$GtkTJz{Fi;h~XPE z_>_%a*wuasD~oQw_mN~=Dz4c3PshLbq|1f?oLI+&M2Vh0q}_?PZ{4Cs!qARk(i*>c zggwHFvK+0b3#Vz6kxzCx>`J=QXBT@Md|27E)mJP>M6ui)1RXnZBP=r8dw1Q#Ic-~k zs%-Y{n1WLrs)oIsihc-a!yhFiJl6X)f?2ZC)AOOlsEKo?-+b{tVfvy)HTM^D;b+79 z%v*=!`vTfx!Sl#j$IWNr$z6wBcf!Pp;Wfaap9mTZBiG8L+f+6m{xVFIvAc;34HT!# zLg3fCetdoORtPRGvVGu*YXsa?O7t720F3rqazvCuAce8nBR{Hah~3Wkg<66dYONF=bb zWed6_GMA_!%Nkg2eu~`e93Yl-bSK@oVcCry zlz>b4V|ssPpx8;mP}dFzqDB@*Av)vuSOMR9dQ*@<2cT4U60EJ4h<}xJFO@L1zX7Zh z3jD;m2!ry@k^pFvM#v!CQW4NwR`e-Q0YK)gRTekZ!DwjRCV#s^v}k5M{fSqvmLOg% z@AJjwcu!T)cJl3zF%$y-GAlSg-AWluO0eq5~dDTc290c1SD>bB>7OdRTy|D=!BYBAEr8EWF$M zb~qjuP>DvIg&b`_c2EZq|73cMq9T!-8CHV_@^^fM%xDjD05V`icx^l7{hi*XETx9Y z8>gh;p=tVTseIvARj?0e(wUZAyf(n|wKtm&XP{?Fvh{v|=^1jBVw{iR-f(sjpqhFB zXB@f88uuqY-@UU;<(Lh|dzj7^6R=Tn_t&qZo=~$taZYnwA;ECrQYwa%B|Nr55Ju5~ znd2B8MF0>p_GfM7;|moB5kVPy)5X?0h2su}ue_e@HGM zD`Y2st<9-*!|$<67n$`+9vLiBbd_xrApn*Ju-@p>`SM*{J8AwY&4I9JwS9)KoDD+_%-olEIrgQNA%Ek;MRe|hIn7$d>giLh2_6TFq4 zp5D`ky&!cT+)%}%HaN*v0cXT+ZpNHe1jbtpi+c7XYxG7CSDp3^Wt3mQ%z6c)JAt9F z%!IEWgl(DJ(&GH~(V-hhf4<$rKXe-(~A&f$30Foc~DqMF8Rlw4y! z9VwISzHUd2nKS21I?@ob+#_P7`%TmwD(ziy{asTqv3&VHo60|e4K~mRjttltEQeOy zC6{XW?137-W}p?<(Rrq}Y-L#ZI9HLn4pXZcx-TXrEjLG$jAvhc{F8l zD8qr(&N+mlAqQ=OoCxE+rcS19kOSQXf)t$vrxkLOw4+61>YobMkGti5=#T?p<|P2P zfk=hfI5-FltCtJDEPOvD--ZxNd~-T`V`=uUR_SC z$7HdgfQzw_2*?Px0elG8kb!dRn+=;bu>{*xRaKxrOm6Fqq@-C{mqQ{V+-r^AU^Cs_ z8Njv*%a<=Nfmd8U8$J1WiYt2U2aSIe!q;(vx55nhh*E=Gk=2{HKka?vS)X5*|BlvQ z1>C%>2|x}+k(Y$zKtqPqD>;CZG+Jn4G~)|eg9^1B*T+Hvfev5hM)(1}PA_U7p=bPC z7~4(231+}(3-njpbwJhGdUqv)G<)La1g!*-zS!UUFo^+YD|AfGnnosm5SiU(xRzmI z|F7@uPvq9Ld7}kLXAAlJoh(t(xnn7+nQ^4+yPLWiZ#{vT8;i$W_&Vn5nN>Sc=eKW9 zd%p`R4o)jZd4Tl^*yT9Fba($Pp9qlXwuUrQcUUxl2ekqi5z_~_1Tj+LNJ~xD`}+yf zd1p%+E%ptv-)Xt#4ihI%Y=q6rz4u_ZL8$~gkg2t09|v6q?TER9#0wnYPwHLhh>yY` z49>)?sTrN3xdYMJs-VB`H!w0b7!QbX72vin^bAa(IH3*>n~y+9iE9A%ar}-Tu-WPJ z5Sx3b(>BP2cdE&>vRc0T?q>Vv(fYDIX}WDut1zr@p6-+)#YnF9J?@J9N&Cuc>pFT5 z`XKOOK6Iev@XWO%6VSiWKnlbmYy{_kBx+_+$}83l_IK+IsDPn;E^sqih#T2rkF?B> z!d2mXdnW$un=j#X4YmN?ZV_Cq#a`BoaSru&aKPf!dEnr6Bd`!ciAK!Q=ONjU7Rc-a z0mBkpgpiLIKAd2v!M6>bmTCiCl|t|hd&ZwtiPxquFMNUJi~4^ZM9!7|q_ZbZgy7sL z^yz)$LvVz@A9#@}qDP$14pj3~`#q_eJAGd3?=OLp&mv(3(F>I)=q!EY5FEFY5OxLPnH8<%R7!1 z2Ln>1#JDkI#+ZIRgqDZDXQq@h7!NHdGt}0^uIC~|NA|{#z6VPddbGvkn6&*(ih-N@ zmF+zTNQ48pRL&$Yf(ZP3DJPMB279&$DunIaDN1-)bnC?H4I+V?Yma_EBbfXXu!@=E zL;$UuARcUoj;jQOi#8~X`3UV+g_t3dG?0A5iM;I2ql@0AAGi#p&_u8Xcnbvc@fwh# zFcRm`LlB8VvwJp7LWqnSXhu!0;Z`9QoX_d+-Tx307(qJzJ;tyi1jT9sO3!FLv;-fK z^R#L{)9H0V4Rs=KoI9v@CU5h;``ea1@vqID0M1+Ga^Do5&6 z3Z#YO4Y@+EUMD>J3PPb1hDid~Ro0@K(#NCgf7M1SE9X`SkEy|0kJ)<8uAUFIq`H(?!~4!ig(+AB$LO}`U7yMlSiA(C3TFAy~gpVue`lEtDBzDYjiFo1%(aVTk?Qx1%uTR=<}ow(?AhbUIbM4%IC%Rz$qZa? zg-+UzEV4--A^K=S$xMzh+M25x+Xf9LssB^ zgSspPLSYy0j60%@3;S&QAZ%HMuHWBHas%}wzu-CMXamM21QZv{uza`F`I;IV+WJUY zUnY1zK*J+oWiQ8n?OIUbq>gDsU}!kH!HRE~`-!Hfq;P=$mz0!%)J&xzESsQmMlo|~ z%Sb5K7+K}_lT>FNw&95JYEPzG+h;D|xC3iqgpZN*fkUzUMsjil)~8@t^jCuJo#gtVA@<@6XVTqSsRFgrTAkix0rM zOvL+P3vqr205K^bKsA7+yCrNOUc!x=iR0$!$uY+^03en3KXC8C`(|CPaPJvN(ty4L zJNZp`FR{U(=8#nkAphy^dk!277ke}WMvSvE0LqO zZ}pbxV?7~HP=)z#r*5xkTq?vgx?jgKGGXZQ8nG=o+S=N!u|77S;RHRtfraQgcHwfR zfwxchtWqx{1lXAW$XPrboISl1EV@cJCv<1+Ae=))elp7MNP+UlZb=hdc z&HI3H7U60|pyr@xNDc!pLSX8xbS>P>7yh3{T|etAwjQsoqM|~DWDbI^uxjTB+<3eU z!se?QY*_s6Xq? zmks-_R=k~Zmyk^S3Sd(&3q@O3`T_Nxsj}?!TCT6jY!SuHNXFCpfyFXa^W?a;;~VOX z&klC%j|h0_xJolKtic4l%v*o}1c6Nv@;c-6ZxUk&qf|i!^Feh&Y zIhZU28$)nC_1ej_kEP$^y>a)1_xhe_vahB1JbhN7eI2G>f)92#t$%J zQ}(;ywVU#)5@y!VnZo?agJaahQwRH8=5L5PkSwa11*a6ExL$=_^TD-9$ObuCuBD{~ zD4S_Jf~Oevh=`NXjIuJ&W_{_qwNMU0cY}V}x4|{4we~O|xD}#If)X?_7gk>_&xkpq zj+QQ|$6%BgBeATm*B|Eo6Q@p9gza__cjiu<=Qfc!whnR#hkN%S-9eMCMu5vB)b&*( znFJgT)+|X#%_VeBG@H%)f@ebeBMYdS|Mh_MT*r`H$3LKzo(~cY*ZOP}oPolR?T6O2 z5eJEN^FQ!BCm+Qo{wMhOs){YCpuoGJYzMrxT(Aa<35mLHMzc`dF9D}T3T8N(f$szd zS}GgkqAx{iyIBo0$}V9p_Bg0_+`W^p#4Bw4!EG$j&RNfe4!0dTGgohMrs^Id9Z7dc z_4tpR+GGU`1Sv4EqUAaRQI8`ha4nLNl@I_Nj1rk>yLPQ2_Fg5cX*o{p>!R*YDoNL` zTXX}O!+qtY-6oOp{z^2fPC~N1U{LVy+a0CCtD~N0Vw*29Zzzs)ZW(PLgH1|lFSslI0sFMbmsAE|lwH&54vz8A%P0d?$R0eWJ&7u9FX(D&N( ze30&L0idNuHFQ64C}N6L^LpF6j`I7>#gaDw`F9tTt9C&I5M~1si(2FzsVTcG&>>o2 z)}}=a9EYjef~LUXj49Mi+xjn{{bLoOP$RMZb6;D4kza502*$V*+YnT57>)J>?c&`Z z!sinu+K76^scm#E!iF7au0)L6C}9Xpzj_vOU01M$N0jl)L=dMZoQj~!%iX2}GolWz z81U7@DJL`j{oZ8{1rOgb6Q!pFep{CL&fWfvo7D~hY$1wwcwchIFAz9MDDbbSTY+pX zjpQsKew+;^LqSQ?0KIfwsMn>3;fUxPhh1LhGjOi!-g%EwZtPdsVfqEw&||+UB|n6Y8NV zjGXgixPs^iX07=}gAGahtJ3Y>$uw+)KQ-)Kqw5drnOH3>+pF^UzZrzYvvrTuif% z?G}Bhk3s#rMR+rzE#eruHmsn=C``g#+2zi1Fe#|ROU~QLv(geudqb+pt1uuNF97B= zf(GI2Cj-#Mi?`rGFu}ruB2*~9M3Qm8iq}U}q(K}&@{gm(jk~I`!G$)=XC(Mb$U?_p zfl6$lhQ0$~SuLOU;I?9#zZ7XP`M`COg66NHXA1bHV@Qpy=zfSWzf<}3Wnq$?wnqKG z?h(jY{`PH>P)<&cB~tLY=0%L62Zd$cANm3lLnEtM#BX8-zUkGmF~W`k_tcYE2z|SlVL;+ ze+1(NZcx0bAuVwd_pwBjBnolx2>2r{$i7gxz80%wII=Z~YJafpvpU}R1oh3FcIje_cC7m^hc=_$x= z2C3CRU>a_k_bwWa(^Yq1qsM|3!kSoXsa)p+oS+1fd;%>5VEoKzLa)e3V`NUyaf|)O z(Z+E=G->P)QKp4T$}i}bLh(?egPua_i;!kD$YSbk)LX0$9JtHkE%fJjb5HBHpWTg3 z=<*sNgsnOK*$AIA7vKi4;_szK+N_*P>Mq^6^2# znB)0henk84!&L_koZ0E30Xro|mWuPXQb@_-zHr&LNn-a~P3gN!PBeQ(n5R#8#WI3zgEbf(+gWhKo2BS^4x31%2 zwa7qSp&_fd0~{vgh?FT)aThUeMf4It%1;PIO()`mt?A&gqhD)_qWi&u!(9bz|CXIy z<8~B-ATHubrn-W?oldDutmLxPIjsbo+zikugU}n&wu<2~m{Jja3bU~%(0^6p9CXFy zAk0RG*Pnxwy9!&Jo!F`p0a^f&xL=N7f1eejf=4Kjw0;-BJycxGYoufov%M`IzZ|5L z1?c5yiNBPE|1lpdDFAmF_V_%UUO;rB-uEp4{#xnVw4KS#mCP)~EgpmcM3+n5&aICG ze%QFch!+ofmS2gJ!|B3U(JJS(B&5gdbz)~<0oSk+=p0GcHno7YESrMe>5#hzCpDn8 zPs9AYUqRSdDWwcz&>i1%;2Z~ff=+wc&5aLMc5k4AJY(z3ibtDJ=@wH#-(zoI_3ir{ zAaRh;g&~tdqn9RKxJ^u#-}9WqDCz;-hW=+G7oB)$Ow0{68O5ATpF1G4S+=WS^3k@% zG#HyMUFrbV>~RwU>|Bx1kvJC)yr?;UM)OS>-dfniry(0Og!=3c-@C$gVhE?bPvFYK zQ74K1Qj7EY8Aaz29>ffG50eSucmj-GND(r82sEGdKVeuXmA#iX#M>5ojuaaZwLV)^ zn}68gVhWd%GrS6b(t}r0pLBj2pq2@{g|3%SX{? zmaJ#rARwHF4>w#ZEH2jlGj{_4>_F5vvfVm_Uj)|7aYEcBcP=`BKIJ&V851bW_Z?!1zh(TH|ODkXL{pcw$;T#YyZ2J7P0Wf9>0=}=Z zlvQdpMxAu{B~Z&RkXqGX_Uv;VdqkMU$Y?dE>zIWwjiR%h0OkS$2QqpH3Z6ctZmbK) ztfT{=npwA$rmL$5M(X=XE`nN>?WcgZn-!q9pQ^R-=Gxa@qQ+CdK*oglZVf|Zlfb*! zFnI=37X)Q}A$$@u5o$Ph5_ATaiT z*8lge>!nB0yPjP$N!8@S^g2ey%0Jynw`Pz;Yk7$y%pT>B(^~iVIA7#lLY{M4Okbpr8N*XNr~gYu1x5zp2A32V@k9@0&6-$t!H9K~*1)E18HG;GRC(a=o;-MXZv|Jo+w;<)0(^RS+=S=L zhGKUs-NrIY%Cbp@bI*};upgZOX&5ZfYZK5aBzFO;kEm;NVYu*?k;bfCYv#=J&Or?N=tqj2>Cv{>hAvj%e#F{IBLKg^~)P>`$x@MWsQlfhH4qu-@S1R;|3+f{lkCa5asLVd6BCR9h4?*$I^x0PviXl>9`tr z`gw9^Iftg2%r9)8|tplX>= zFT-efleI6gJo-D8Dj-=6&*#>LkqV64JXuMgW-;~LDAKliVp456m)1ZCrAUq z$asEzgeD*2D|61_;>Xb=9X|i8?aOCmK8(J6Q+HO^&xsqbE)>%dVPZzQdK9sTZ%GjR z_l5bE2pZ9wmN6I8BJ!`f2k(S~+=0HX1c3tjYM87yg404BY}vid(ozM>^mU{p8usDC z8}MzQJT)~%Q?Zth9Z|Q|Y z#hC6mbnC!J5}Po}CB9VA(prU&ak-a`Y%rlv7>F4uayYXdJ|m7HrgxjXQA)lORVAFS zNix3w5z5S+`sNA_?e}=%4fxKS{XkH4@t|paq7pz~#&J641gT9ZANxj2In6IF~MZXOm#QG~5`k`Tw&!tPI(Z_XIP{F;fb z44h)hfC8KU5vL|iYOilxs>M#ko%ScnSHaMHHI{qQorE%Z(5qZz=S0$EB0P2%+~@2l zI^Hbpc+g%*<;&#M)+YBCG=2Z^77+Wm8_#DBQ^Ac+ zXA^q*yBVFjffW?>dj-9nJ~ql|`m*es&J~k}FcMYA?C$RP*tiW<>CRf1+cV3`V7weU zYH}E?jeE@w?T4Tdj3=BVjn8eYtj5O_iWmojhjqpGh&hBtKOV?~VcNm-Tjm zYeV4Qv4+;6 zL}wwnmnNN0)^%-`Y459>COpXXxrf(k*$>521kN7eP6JRJ^m74wCeo!AV}CBqQ>2GL zF;jEjxbzi&{8orjBHwU4^C=&7$8k$Qx|_fP12RM*jLdKV!Mk8_a>Wo+F$)Mbl$CVn z&Vg-#-j?gu**z+4fyN2F{MUR8Wirpm^o&f|U=sAkA+3|D5g<|XqIgS)x`;%^?NfWX zIrW=JazoQcO%Z^cbh$x8hXVxVIko*6{mX|gH@~;}yNCNYkR*^>1 z%Q>*7Z#Pa{_TGMpMHuUD`J@InSvzhZQ_lh}vkE!!eA5oK1q-anym+J}0(>={A|$DW zInmgb5LBs+g68IM=u92V{Q5q)8H5koRulHXqeFy}P&6Rx7ZyWF_&bcBB)6c(HkoZ5$kIxvFVZsE-aDvT5`-e3uUZBk=BriMpeyZw#zvPwH^ zD*kPMbq_zc`WjRph{%rVhEr%xRsl|SUfTiO9t5L9p9(ruj!$P>eLIq{mR2nq+u7(@ zhL*W8^^MIrJoO(BOH&IzhS14ELQ2?D2Gp}}2Du=pkm$NYOsXsJI?==gLNo&24=;`2 zX6=tpbq`n{N$vG!}g8mVJ zYsxP$1W+vO_=azPp18<8Xk}=Oi@hb1t4L!7A`C8ToUrlD*A)wJ5aq3A&l327VT%W9 zU;{z4<{|uA!k;MzqOi;+sBB28@MR*xN$UxQCFVT7zGNUQ#nFSO3qPLlhDKwGO0(tlI)p z;5%beab|lNR%4Ygy8^dqq`L<$uE4q>R|YO-j1IzgE~054_`HcOBk2q?Tna+!5}UB= zf#&_1G?Wo0q0yue9GWB{42%haJfOt3uYDYwFEFzHI=&P}i|yQqOd2fAA_s*W@95Bk zI(KgK6g@zu|BEP**hYdcBvaldc_~5_%nK0F(-^XRatL(RCsMCsg3QKZSOK|3i+tKs z$n@$5gqR^E1MwBY@fYiYLk>upF*32b9~5*oYZ6$MUiVudl~W;a6|iv)h!0SG;7p_l5Q%$$;m+86$GEHiDtcw0RyStJb{{9gH+Nhev>f|!lWy^3z3(b8lm^^wf+%Mj@gC^@*mgEf9Yhd`tQnEXz4rGW>j7bB{>4J3TQu2lko8k^hjfk*oO0Yle4cJh>-gr5t)i8h zU$2vO4ja&!Rb;}Lj5pxr{qysy8^rkqmNr>-L#RVKPU-cgXg@dL2@1AdMITRq3g0wP zk7Prj`ijw)Vx)u`lBP#*DW)oq&IWE@ntf5IwM~;Kqx_znFqChB>h{uYI*J%YN?1K` ziu$Md>yV&6T1Et3kPf9ERk;ud#1B6l61hAQg&`$wEure(fXPS&Z0M(KhRxvg{(*U4 zZ>vuU)`nRD(qKs$*w@#QQv$0Bd~!}2I4q2CrB-Nc?Ck~5S~b(6x!m7$WJwm4Grpt&_e;@pffob zf+2KI;O9*hcs5uXY<>v|Vi2+uJvzZC9U&IKaYG<|=YVo(WTCxe-@b2?EsG(gq4DEm z>m%T#_|ZuGFth~dTp8~JXyA7-mtO7D_sUt;hR_GT^cb-KKZX}b-aMjhvmkeVt%&|~ z1KO7XiASW4Z~Z+xO_zVp4j>B0He>L0dUt>2!%=F=#EBiK$vgvNZQ_fYToy1IO894!3Sel{1b#dI8ugNo(6(^!)XQn zzGWEH5Fb1 zO7D?TA$g6iD>29beBy~P21pFoI<#@)*xGQD^`z(u`0(AH5lbeG53U@eiG|(&A&}^i zMz?eUqyP({f}UV$M_EEzhw4&?F{TD+<3Y%VVUMZz(nW)KUiyi{C;66I{I5?SfAVc# z@akq4=Did?{J%Is2#4TUdA#f`dh3_yst4js@H$8n<}0%f?a4IagDc;>`Coj^RI1doK@anBPf-I(37a;vW9N9@$v*c_Wl#lbtm9(;Ql`l+ z@=P#(XURv25=^6<;4IptSnqC!1M?!E%W41=)1uacW(>q4>@tW&j3f}(a7@L-;S7j3 zDbfg2xLbkFN?fi4$7RF62e$!%kX+6-1-yZTUnk;S)(DH6}U_1~3EsZ@nBN5ra|pKXlP1 zS_*Ti%MexEn~DvLJ}JBg$twQ@PhrZEz5BZMR2ZJcu8z)5Rz50ja|ELWK_BxVmNSEz z@Ro4dX?Z22K)nuAcf+C2LH&Tm<297*tNraFUtc4s*riLA=Di4OZloTahEpSmEl4Z7 z`IgfexJRduEDJ}gdKO$y<_^b93|>*A7Z_^S_qhJ)@B+LhJ9oCfQI^i@Gn z0!SRvDptfs(_yfhJq#FG*_u4Uw4@Hb-Sv5`#2FD>O~y~RzjkVAFkXKBOr#y)M$X$p zysw_4^OYmv9j#cKpOT_WbCnk!3!^f2WO_IrzGlpu-? zZzhH#iWnXg2spp?ZQ6LXV!rxYr2PyTdBzO&{nO(u(Eez{O8ib{1jLk>`OIt2iN!T4 zxzU_BQemtpgQiY4r`CrgVf<8eu=;1wTe62e0&Hlu26t74HlB`ELKajz4*HYibL{9av)0|I0UzTN3HftB5(8@=c>K$#14{h2s->b60=1S zmq)+)FD$w+C{%xTMImUx^T(@Aq+-4K82{&SWtfdH##~*X`$!gz#L*MGYlj zplOTqzxbQA)8)si%sPAubE1pwj;~{*;>KQP8$t=vR6$N(byGS&7?&pPHFMi98ZhBC z;-DnfV|1n!fWm;wZEYY7NgF+Jq81XPEVx(K#GQxc1LjsEppZ$paw(D4z=PcSnRJe9 z*l_3`nuOv55vRtfiENJ_Vslx%#ywCEPl;2Fz6e|jaaE?I=H_D3Bu7=^LN zVZ3jEDAQZ97u_X(;0+oW6`CaJ>-a zk^q%2bej%A6WRGZ(Rz#*KPyrJR+zRqp{c#tvqiX!en1#vYykHWfmled3GV~g*#P`v z&m_A@2nwk6VliiYq_7q;MXc3EDA&Va1vVoT=L+D+5GQCp9;qOv9=Ir}64DmFps9e` z0?^6f0|$Wk(1>cyAp$49wBmitgeOa0B+f*iaOM9WgVHe#mx;jw2YXeU3By|inuBjX zPHgTI2^odmrLi;(2~f`*ILCVsb!cYNB|3K(60K~oE%0D9vua2B3O;|f2zt2n_T#`I zjO>iI*kfebnK}=usF=Rmc*w6X4ak=2NNH$m5Aw+IM&ctM3D z>gR`+MUKd9A=)}PI3C!dxaHvSg35FvXEDo&+I%1a@FJK8O{%5T+z zuYg7n0n0XsY{1n_7&q}7gn1)$8UR&UXAuecX`QiSB~f8;X_B!!3_RW5 z73k;{X%6VEj;AcXEbfn1Z=>C`|H-L=b z6_|f{3E|<=hKM66fRCsdY4iYXjRj$Yoi%iZ8tQ8!m^gND08%CNTl^1{-tCC6EJE6w z8C_V)>a^gIK>m^9{jKRPi|Gy%A-XYpQ0vj7MVMnt(FxX~4zS zA!r7pf*@C*rhM6wyFDGO}e$57RB0U1B+tp8q6+?%Q}&!DRss z14_$DOv80-hITHtz@sgOj)ddCDhj(red0-oM~lHFEq!v-S$ZMTq`EhOcOf`;oJXnB z=piFZGx0m?W=>=zR*?x`ZQ3*r4oSY=r%@=^oImCLF++StyyZ0#eh$z}@1SdMDL^k` z_C}r(MnGPpC)muUTedwy{M9;iekA=6P*}~q-O|VJw&tPLc&c+cCiVI5bF_6pZSk&O zxni1hFC}Fvj5^r5xFxz6FE#ArfD}_3i6qA(M*~MBNE;WrY~-T5VZN-@Q6y6}1sqX| zgBNgcqW|#u_teHQRldIu28u1*_e%4{Wk#l+x&(6P;-;Obpu{Uz3JDx4DlKhl9U@VM z93JzI55!y~U5y=`anbh?-cJ3$ShZyHw)CN7+5n?pzK+21Ea>wJOj31z=O)Nkt>j_@m7->shdzWLuOEoHVN z0NVNH_BP7Xm^z|LXQ7%}z_@mkt&oT~Ty7=-nH?h@PJ)BgxRaKZfQNRYz>NMqka%~ z5db7_z*8?Kl%GgSkA<<1Pf8-^=p8ZKEV(&0Jx`y`G6Q+4SVa+x#9dqZ;4mlsNR8J>6QJVgBpZDCntUjJ&+~QX4I;;p z#-haHJtW0HC|T3iXF-(Dh#U1d`0JGw|x2(jJ_5Vn~0|#>;{fBx+ZCV zMMOH;coWD+pQMebHb6)GgN}6%$05oG4=+4gdasWxT1j-%oaDh)0Aj=bM(&vmcj^R~@M`| zJZF5-_7@9U3vrFHSpglLV2j!dRn8K{8ay(D53uxs)7u@NF?xQ?jKi5O<4*!XGN2{p zAd$r?(_>a^?q3NcW`|gXl*L}zV$Dcbgjkv^1|^tKh8igXS)4>6;yCn1zH6oWL!;39 zX7}FxQKiEa!^q-^q3-U0R=nlFZ_G9m9=V56|3>qhGli)a=x0%eKi| zu3wL|>8xQ2t{yE_^FcXK%id;9X?ADqL<3%lN=}Acij8G}n8!?Pxonl_P$4iNn#zT^ z!mY-w7AX!-+?)>?Ts%G=n9%CFEOFulR<;dOxHNvAJwSjhRP`a^o;_xk8GGsvaD{g| z(E*1WU}a7As1u>-t)8En`by8k3uFrU2jMi^R;Rxli-bfc;%Ey7=w}QQDWLDnPb|Ta z6|psoyWw}-p)rHhN@S?BmN5(-_At!zOJT=|wp$4Z2&eO+-+^2x?3Iy{_1ADfu(4V# zEgQQbSq{ft8tcfrp^7t;5^#|szJUu>41mI?clI!W#@L~dU^b#P(KC#blO<3&6X+e^ zE05<3VfRdkj&!#5&#=kd%qDgd7;KEQAKJj~s^y;O;Vm|AF6_OoQvvFOl@;Nps`EW6 zi(oQg)5+Vve}@f9+I0!)bViy>ecs}pniSlJ1(^HcRi6@^?+Jcv%l~Oatqfjf+0~+= z)#K0ArS}dMF}^%&*vP00f5!v(W&)v$qtP#n+)r4wf?;l>Ak0qAWNk=b!C1pSU`8{O zd^mrgM{!XjWfQt9PAilkGWj|be&TBI>PXID+vhbW=DFV93_RhtzidpOG`dRfp3VfS z1A1vu6|Zp&w5L+^){iVNs1UT0a|+BhEF)HD3X zzmfsuuR!lnjw|EO!=afcd<&q=6aiWbR}{f3b@abkE3R=ez0ct2zqjWst_ps2|Cui4 zA-ai@rG-Z_ODb68=h=VS|9JC3nH%APgcStgr+a=%2(Yhybcl^TR#G+}oB>$P$yLO8 z0r-2xnnA=K11yFgco$+G`8(11heM9!xZ*EgOoWY;rv_cKNsz+)@}}%Sfa>k#tF*~8 zt~r>u!&Jri`H4p@^6KXG$D)ZT2E65AYjhlIVTlNf3cF~1JrB`1ZxIk4t3|r2FU*-^ zJVfTgg*m6Dmj?cSY@K&p&wCsHKNVSJ9%N(9P30zW~l6y zD1~Uykr9eih{`cq2cbl%e$VS0_kI6p;jn`DEfV(lP=poYWLx`x^gp<2EaV2y+)R!`UjEWh&f;U z%f@ zs%!Xp7PQCfP5-8$42^sJsf)-<(3XDyy*w1rRjym$Y+R#H17-N0xGzrZb}=>bl--OQw3@Z!k~Cby=PKiDpT=GzKNWnrVuj^M zN@&9H-0?CENI+`mn#y~zVc^RcE%f6rfaoAIh3(5DcY2q+;Kx7LwSn`u_s`c`U48lC z25R{O@2L;@1qHh^VbN^arok{6qhgGzQkfJX0#(jxmz68GKog2j0h`_R8@`(K0VrRp zf|YndR1%b$ua3ci%jyZ0mNbB{Q1tujSG`5r%ehWOyx@Nr6&>eq`MBy{HPIH&2U`b5 zh#BlRglaA=-R3P@n!lO9$r(A1{!6>1*Iug9d0swrZOePQlw>LqMixB`*mOr^4bLwv z+C6M3$s$HAw+$V~{I;|b1&VZs1Yl?ul}!I>f;BddvhvWS z5>%_Qh-R1k48mSYkcWStT0MrT)t{Bn!D#;Sz8cJ|inXjyKcHzW3QW`LK2-slgj>Pn z5&~Q$OPk30voTaOhQbf)nX72gx(%Jqu@ zaa)XbqU%@5uo^f^<&6KxT?E|d?MM!~x?!2wlls5mqCq{RMY`c{pT(dCzy?H5tzBAO*;uVGoU}x zvXl*8i18QHTvl_baq1t1KRbffGKEeLaqr#2i@G+I(8*4oQ0N&fd_Mh;8&|ej=R}0$u%RT1X%*L&v24Szc|NiUGNHr%&g!Y*pWsWGUCX{md9d^s_diW0|ZV;XCPVp;?2IZc3hY1bS z>QvQf%Hz0-oHFklsH`fGfpNorH3LO?>Qgi3v+&W69WpUhwv! zYvzr)^k!HkUPUJTu9aVgRJMn)ZzO3wzDS0Kl;@3?g$V~T|3p4kb0ytNm`lOb@9ezk z3ON7aOF!|(ln!8fi!BLU<)!`pfDvl*t!fBZb>!nH83y8<{;s+e-~vsC58tEZx7&5& zSk*9C?IJ^uiUp(_0DwC&d{yVAxpQgPd7-_smKz^B5!>54{+3#`)dVx}(t~USX3CO((te(V`{A zhnfYGS2gb3)Z${HY#IOUS1^Ha%BQ)(f{>T3)L+8Q+*8z5AE8*A1Z*vt({3zA!-?Ue zjJ!tjb=tE5w(rZuYhc+svc+oBnL^dp0~)jPm{I){rQ`?(4qVrsiLYfvC?b-5hf^56 zw()gCp77a8VI4HLzJXvjAFqGemY2B#KtdIuDrrN4_Wf_ACux3}b{(Zb96JdFnj?sO zJDteeedsCBYTg0$REf-n4G&{{P?&w|FbRQm2Au4V3@m~+-0{^(X#-z7w##K zl;yDt*G(HXY#r%aT}$Th+I0X!w&|w4Ixot@etr9PFxaDQdUJE?qQ)fE{%{9Q-lIbL zs#{-Bg#!T``|1vw35{i9gkoVAuYZ?lUdX~#orYDfyV>R1OFG-Q>D+R(MaE} z_3dqWJfGr$in99;9ta0p71CKD`bMcD@5^jZ8lZ3T7l-}bIJreLr-q^bs&q@rdHGa5{ z6)}OPN=y)UXx=>-73ISw@Z%x#6GmP43Ei#~ZCW+`?T-6KtgDDR%AdBWCJUKG?kTC- z<}H?0Kgv$n&3=B<+}yS6)=;!!;$Hn4CR%VCi3N2Ixq5Q`N#Cm_b{3pCZ9lo|D(hs^ zPU(xn!G+iBD7izZ7nGyNb*|s`vZ_`$RjXwb+netGB$6C5n;up-+|ND0AnDbFaYbSW z<*Cf@(^9TD(QK1l8SI-Vwuwlaw z-Q&skoexJ=Qrz!DHA+Zs6|e9@`#fPw1VhgIcj}D&jeEavtETs~N4v1T-RB&6Ka2Es zR)`_kR%J=Cw@mcAgSH`fAi1hb~`exZ{_J#uNoY!&fUy zo105^w@}SZVNoE``V=>-siA0`6vZL0S;uZC>l=dnSQg*-OfQd`W`0K3)Wzy|}bu^PL8+Axu<~GBR{B zw5}>aS+XCvbhm)q`FC3@P34R%BI7!M4%oCK&2kwz0us?uOx~}sNaF4|rj9dYcFfut zbmab-g!%xDEF8Vw;vJK{n91p3GM#&nr*W8YxEMRNIQLwR9F$KtNpj@q{dRVDw6dRje=93J&;Y4Y2UM~A7(+}r31yyPgx3s? zEr9RZ4?XlFu=0!Nd@xuvKaQoo3_R8AGW?#?gAg(s1qxr)ZhqzL5~K0*kKr62*>or*cU$z^ZZeIz(1oZxk3WMMD)-2>H?lGkzph%`=wOk#LDF&q!zN>;FlCb6{Tfdi;xQLHZr z)v=#8b!sP-&>AtrXCkS_(6gOD{h{QJrv{7mCB-f7=K?qLvhHnABk8L!3l78kFIVDU zZ3nDM1nF447h-AsZ?NmaT5{2kvb3}e4CGP(vh572di`kGzwD$~o*k!_^5uC3Ivv5N zml&UZWV3)qUYhkeE010555jhyp^VVdP%`m8j|e6wh6y?$cm4fAqQnpmHl0G zy)XXlZ(txn<`Tx!eAml4NNT}1yT;$>-mRNzaGpd8+>*;&a<&@46wM6|sSW!Z&|JW+ z=#sGZcjQSDhYAMYd~>`I8T~d@Q-T8K>V(jv)6j=PP zkbD<@oSao9$JUZrEXlB|A{LPSD0|N(O?TDqg>=WyKkSCbjI*v{vp!Tmo5ev`ntqOT zRRoBk^1WZSrr!L{SfrFJK#(Ze6PK4?M_-ikY!p8(ThvtvDab~8 zN!~K(pM~tw6G94~gLVBYc?)pK<*hbU*2^Gz55?fIBJ{a$8Fvsk{t#ZBn+p#}}$xR`mrf}X%xa*5EpB-qe+YJ-MhpQHdZCJw(Cczo1<^=xx*2j{7!MiZ9 zOL8K^@AFHcOs6In$nTJtNJ+dC57vp24xYQ&SqLy2CLurwO1EFD7$oLhL;wRS*0Sa$ zEn+%ts@`;!{LyR8t=dLXEOr@foTB?+{@glV-VuHK&H1P0tSIS6GG$dpj~XU`@;8p6WQcJg)#aHesD^yYX3dXTVFb+!WiP$9NW)e5wF-+*c zB5P}QzP9px!@#6f;gKjiA$cxd_%v;JofobtRl5d!f(TuNwK}gtlhCxOHtf_MB7v`5 zIs#6MqF<3U5j)6#pEQKKQe5qADbl5rr%t_xW?^?Gu7s<-5@ug@{dA;iNkAfEug}<( zKDDZ&l-`wP_Of2r33940EwAX_HEvYxMwEq*?^n1k!f>wI=)t(9+{JW<(o_Po8FfrCSD4h(ZCCh(g!^03_*$G{kRUymxKbuwn2JL)5$9vD&l)c8m^nPoY9BoO(wkQ)Y09hz%D? z%tw+h={u&ad?^T5{V21J!|%2nf2rlU_fJ|?H_rZ8PlFWdz4}rlL=-aSo z8DwR#7p3uZo!N6_c^K-REdUQpBS`NLCg!4HfyVA8e+iI>KwC`;qv$r76X))N}$Cf|C%O7 z-2@P2qI=(BmvuH!kw|E=)!e_%kBwzsWi4 zk6?I9Wccp|CYOgyM`QQ~ z4)!DX6y1!pnH1)hZ=88zB_Cuf0a3sEK1`}e)A*s`mL zsb|qqmA)^@tj9d7GqP%)Ex%B}J*0Rg8_<(JS^4$e_atI-J1kUew7ac&Ew8@{ht65q0evD^34A zXmaY#ojcpnu!%Kg$yYIL^O@#!%xWX1x}V0&S0_VP&HnVo)Da}2a zoWZ`Nok-9LdWn7}79zURt*5ykL}T2ktIBX|tw(E~lC_7J4!H#qCM%kcY0gv_N}uU0 zTZv^ZRB5Ab{r&eyZr8&mHM{*+JwbU~kuxqO+>f{m_8K0DGB=34 z0og5-vc^$18&I(>29Dp9i=7p!{x0_+MfPJeb3l;265Ixi-S7U5U; zKX1#oELd1r>U>+RzUGD~htKv?TWa8hz_7pu<@uO*RHx#yXQhXP&PW=K`PVW@dbrOA zM=AN6R8yd`9X)9rL4HM-f^b2lJ>OS5?q#=kqi7vXJF%{VXeC8nz89oRpS`z>9@zLylUSD;f(xbwO;ST`!Wzh-=hvi_)41dg-+ z9FbA~D2#_x>n&a{xOcY~@zs=E3vND2=Wkfxb3U<&n6;WZZx}3c*_q~+&b+6{PHh=zhDPV8ku^iS*HDq^vw60yX^>hQa|A|rr8yRH$$jy;{#i#r z?s*Y2YYb_*w2mN?@^4hh>SEr6C9k@GVsWu1asfF-btdKHt#l+ROm68(R70>rBDN=f zcq_1s;bPK2VxK|27=|vTRWhwZpg&Z7p+%2)YOoyd+3(!d`JhjZH3=O zEUNkMs;utRnOrAToiXvWwrwfLFCb;X?#^bNHzg)K7_F4twhg~?;8CrAE-kx?ox`gF zkNT-6;w0h)STuTt2>is8{q^e7hKimmF$S{2AP)f@?yKxMlIw;%^Rc8!z!4ME8B?w9 z7a$$kF@5td5*`?F2m2AZ^3NU(TLM{o7asOj39&EzS}m>e{dp~&BuPJ#LLuQ<+L~ij zjjPIxV;v_M?i7*TRr)J_1@V?I%yj^lGG{ay-=^B3xW!tsAFP(5<%`q)moi?klDDBP0sJ!20a)Ci3IUO5A9jVPxb>eWg26@Q;GpxKOp89(_{0CLxa!{l`u@rq zh->k3$XerLnD~;Cur=S4MGUG%dEl9GCG*9aLy4J`g=HQFf8E#2!A!eFO*tKylw&XW zAj4Nm|Ja$*$jG#^%L}z4}a3lw`kmZ)Hs8e{Bc~I4mYeX?l3G#YrKt0{6Z$&ljKZgX!XwFbTNxh{fK4I^#4A{{selZ>vSeK23@wM-vw%v~CW^CnJ$;I7 zPZQ>*yyp+uKM59#k>~Q6Gu@7l!vs!{0%VZV;541@Ud*7#5D6H*Yy99HELjZLCdw=Dy~Hn?93la0pv;cjv5O@2%#@)6?$=e8 z+CfJO+9mXo2a9ojK*DSv^5a##q1XI$+A;X6rf50L73eE%884->RCxYd{(aiy)iF6` zvF+Bi-8W(Tjan8BTHa{ZxPgAFGeb_O_GWA+LIBNw>!rdwleA4e8c> zQ*?=)XQrYz`S8HiM}!3&T5y+>TZBY-DEHuoov;4jl0AhCOLC@dwFlcS1fZKqavHsl znUTQ%XXzlOK;+?&2>jlvGgKQN+RRJ#J7%5Vai|ePT25=*1o7ZFZML%3Oa+$o2mb*MUtdT zCnO}8$W}h062hP9C_4VE2LRlxmQSnVvrxHy2EdU&OZr-wgZ3n3`o(oMyhm*IZsAp) z8N{-czt&{tECeTG-E#k73l-~s-M$qDhpaSesg=K$#JEvgHgBFVEj=qem?$fLg1gq; z6q_4>s5lPPBbebipEri7A`g2tSB}^AtAtV z9K?pa;X?bMqet7@AARVZgUIln$B$1VV6vXM0b$)sN-}I@-;#O%awz`o=VP1WM^Hr-smwmd%^60;DXT`Yz<05Cm5 zU#o+P*k&4FmhKP_hl1Vy{uj2jR))>u;Kh`G%?K*9-}zgE2En*bENNf#VTas7Cq%x= z*c1{-vI?4$<|l?P}<9aDFY3W7w*#9QY)NfkT*8SQvjQcBb%$6rR=a+neVONyvCm@6#kjRC2sZNm$ z&$MNHm*>`Dc8L!jOrdiLTC#ZYBuHI;kNbbBdo5d%ZMT&F*&}hDk7Ju^ka`6)_TF>n zmY0_sgCL20&Zt?3yYC~d3|TL*0XFaH=j%x+)O~)(F5S82|DAd$jSh zDgB5Att}RohPj-Z#k?q*lhkzTpiy>oS+HP1a&Zmcu{21kBQvtD5--~v$R+kB3ReIr z%|>d6-YFi*^8!wqYF(!Jy0sWo9%DN;uXC5Z_ zGmP2`VR1Bfs>LRS^=L)SEBTmpxft=lc}`crpe;P4VK2}&6hmD?m?ox;A3HYnWT3|x z%CPC(!hsNR#`GOPpEU;Q0>=Xb*#kJPbwY;JHW=j!?(1aE70KB(+{ezJe{AacS*O|V z{fC*^wxu^EI`8?_6({p_LPfGVr@D%a)_Zc(oPW-f1I@U!Y|*)yYs#ZCo;exU^{THo z$Y55BlL=vWWL#^1f^Jxj&J%Zccd^f*l-2e*eYj`Gj(^A#Yuki36<2G&rnQxXtJD_D zm+9#z`Q;WdM{a1S<~ju(^GxDUD1sW%Ow9J^UC1-&w@+*5I?eUAOf4BYDL z>z`hrtyTE-``W0r-}`D@SIh3N{bKv3#ZkRG!}}0dR_h5LMOcr&iaRQE_9x|}a0(ow z5eb+s)O!qL=HLR>Us_tqdpoyF1FztB#cnPvQn7SmXVR9fTl>qyX(DRBdb$HI6!SBD zFH;X-rn5Y>Z{MB%JrtXpG}<>pR;&k#KAZ9U zJm{0<5b~%SG-wc60rC4=>()aaH&9dbPDDk`RMi^vQ&;M4vIE9_1j6C7VK7@8a%nfy zSh=NzwE5!M*jOvv@Ry5m^H7!AolIHSrLmIjQdC@=M@UB$+M?t>98dZvd)UUc4{e?p zd;GW}D}oD%g3U*KYB61>Jta{YHiKqCBo#o5svk=XkdTks9pgVrU?~K&3yOrHejES% zb0L@T%xJg%1@qMulaUN^Q_v<$VaUwP9BS1_SNA*>VljUkM!ET`^)`(-G1q&jdMIn7 z3yb7bnD)A#mmJw_+33y7_it0T_8+r(`N9leqpJsB{E$O+N3a#*xE-STM9+qYr&t`x zBFM!*AYgF;b9Qr@l_R+61f$L|oU+W=}s`L4x zx>x<;>>C>8C6yIf#OnMRWW&W;dfW5I2rjIrUpFacyFldfV4Nf83?IIJ<@@KyrD)KR zQincd#fFMnjs7cA%&w}|g=)aHG=tY<^30iMDod>@&r+|9a~F%dNE|Oy$z^NTuAP|? zjeA^T-p7yW+&{&E6)B$dsxy+qhwQ|?atWBUJP^gpX?9k(w{PuoU6=FWI-v^bSvvQv zOkf-S%y|2xM!!b>^>0iA9UQai+uId4-~S`MxYFK<1HIg+BBuUHUV__xtqu3(mT0%! z_Vmef~u{WaqX4j3()mu&VO6{wxbr!MuIHCuSLM%@NAl3fdT6j=(e5GTju6Ff-E} zl!MxJ`bv4AIe!7u*sU>ZwcX>YQ#sW^;(3PNru{nZ`Qx_dWI$~1`HJ3GqyxR}-zP(Z zVl?a~EmY{eupW}}S($~TAP?!m(A#`Gp`Mi=E2GI8Skk6d;XJ1KVjNVO#Vl7m4~(V3 zzoVsP;oeD3Mi2X}!`vGMjPAQL-W&L9`CRNVMiOK^%R?)t2(?H-b{orOSn=)lkVW`7 zc%@o?mqdf!H*&t@A(SkQtq83gBd)#ik1!H<=sk1Xg<<-FuZi! zmnw2PezVhl{=JUD#aH9XzQu$$bwu3$g%xT`B=AcOTCuHb=zEmczLFI7g-na6!)#$jjZjK9YZR}9@}PA8JiJc zy#q8NBO{yNRkO~;TQJ5?cTm`W%SXM|gP-u>If`fL!NK2CEZX@l(Ad}r@wg72 zEYsJFh@~jV9iz!pj;5p3vTD7+BX&scoI7!AFpz0=Objam*Jo9ELYm*UZE14h13Cf& zW^8-F4x?AEUbSz=6mL^nR_1fK=K(+^vSGQ{%(_!lVEPDvoHSI@c#lcE!j2s~=rH1* zre|e^@*ZRy9)K3~IW6SDabJz9d0F6!InNm?;MjgHE046{+LAv-58LEYrJ{F$EDn1< z?);BjM}`hhgAhNosOoMl(4X=zYrJo}WB>lsn7Hht}E6OZKjO=-V{iNbN_r7hT^UY!U;YqFtXp zj~Iw_)EWGZmu7o~G;Px41a{cI18u4Sn6YX#JB`U$50VI@8lL zR{YYeTh1s&sXvpYTfyLpicW0`N*ZmD?xm7vl!0Odyj{h=lDFL0%96_S z91hJ^tBhL8GWv>m&u{f+65HiCei5(uqlB;?vk~@Ge(qg)gcExmZm3VfZ-n+ zxVmrh$AABQ!X)f}=+K}e-xcc~2B4l=0S5}(|#K8p{tn4sm;~MR$#gs{tY|g?P z?cTX_UKefiX5UJVhJ-w@XsDRx(XEkxuqR}ig+*M6lM>e*hH_@|g`<$kiy0+0G@Pp+ zPL+BwPyhUo>RwG>efeHQfK!c^;~5LWRX1L9EAS}CVv=GR-|VDuJvV^XQ8lcPP|xK% z27%1ZV<0{mp9F~8-`Dn7I}}t%umXv~SW>_qc2gO+)#pxjD=j)LYb{mBXO;0GHDne4H>x*D|jyU`s ze#sflE|#8;N#UGmMh_Y^b;c*HZCp5UxfjvL%)YX;ZQ%vKtdG<9W?~gJnsI@V5^lhp z+5d6xzlMie#XfvDU$GuZUNtr2M@4M!GE!ITHb1D{##w@f7*->-YvIK`CC`e48O>c= zt*FgI2l?HO&u*o-j@77fue@}10Fj^cRRxgp-ZqrTdk-J#K3dCsLLJChq`4~Q0A&oP zRYul%sLGxso&&T5pNfo3n@Z^kyYa=yf8Rd)v&eq7zESip0S!9B`RZpiRZ~LG(NT%& zF{CNOBB~_Q&3ezV+h{r(1j^CVR<0jLlUq%DOaRPFliqV2x;*+e$;!&<>3>?DNfyp_Si`_*GcBvr*n1%1||(MP~xt9qKjqoUPc;kX&-;hqmM{&EFv(2 z9yw7}vxfYLA7W1*wSimdTXwNq1k}uoJ&kiYeU79pjysuR-IqGceD#<<2{A}R=N|M) z+!7wyZXNZW6C$4rdl`wg?+Ct34gQ^~uPuz?pA_wl{c$TY{a6P9HsG^zu4Q&v_T}AJ zId(pWtwzVLHO#`B#tT24ECj_Yn(`)49n-;@cZ(%zk6f>1+V1}BgHSeCos!?@(y?=< ze(W=R)HBq|*wrtfO`L1dyQ1k20&)#hiEbAV)}{kT7W2#3kWhDy#$+}o0_GV62t zX2*IANmuif}Ug|QnutRATAvstlrU_6URT~d*mPE0;o0SrR{ z;4Dz;z$zNixR`$(8yvk;;It#>Kw$*y>viQCntC}RVt*}l>pM_9d5lJpnfzLfb#<3? zAq40NZDqUMHUDsPea~2D$7~gmiphw(Z~@A?#j-Y*i7~`**+0Xh3Gvgi(P_KK_B6{p z(rgBGQ>=|lNnXSvP>LU<$=z}PIOf+SijY{k11-}d#_igo+(?V_NL z8M+T*`4rB}#LXRAyN-={XKuTf1Y$U&Cwd7;w?CBh`xMHvK3DzW;%3KbTzspjr?`>- zJQPsHZ1S8iOOF|)ujcH{>!O$?pIF%E-k5*hss89BFC zOXM72y9{j|I-=E^m{w0}D89Zp zg5ZC&k;nQA+$T88<;#Z2ER9Qra<|t@{nmeJ@}s!1ht-tcA1`~p2Cl6r^D#$QeGk0> z*tNP))jh3Z*8crj&E!-ISNFI)=S5stw7W6h@rHq*=45MY2CBKvm3i5XPSc^nP*xCW z7Q^-QU-r-6JL+jp$vY^9Y?0v!dEjFgLJ~1S)MH|j zp=g8YfJ5UPUq)Kq5?51S zu-iDZfXgnX6O<|WXEQHYtCb8h)%x(eX*N|DA@qmt_%V%7wL z4>2T>u;=WkC%`O=vt920gjsAR@Z-dM`=<`4(9)CUt6Q&Lb=}-~7&7#~u+dO_Td-pM z(xAn?7AD8;)v=ZNJSVrOa%V6wZkB-b=;=!M^(I@fkEaoE$RGlI;K_?`M z|K!OQ=1s)&z{{rnglaDxw`)yJMGLuCcq&~jNc9AoSxV)Gd)S{Rh&11_W(Xwv4sojA}-;YNDH zhR2YF2lxy2^nF7YCy=C|ms{8)F^nnDu|PnUxpN(+p-nDh|A-68+A69*1@_iFD)NnK zR_@0MJq)tp+U3`MH?y|lyPc9zV|MyD*7_hslq)6&3tSpKtYD8pHt>FK$mmd@+O}y{ znEyW$&k!zLf^H3R#E9DV3>#=jJv6n5<)I<^4Bpw6?A~1Fwy#qH9)&Da*rUi3ln+s zXqFe~HCBAbqcGrVnqrre73yTrzdv2=GhTMnwtf3fWWG*AOr{=9)yoHe5U}dT{MUXN zjasHX2}Tr|VDFg%T02HX=Kad^-wLP|n#@ATTcc=&R4lP-$^Y-M2Hu+z5{ zk1uN3YqKC-qhpPf!&spcrY13w0$Tyu>W`#5DG!nSe56#}uaV!Cd|``nL&i9@8~&1=T1&M6OpFAF*bu!&9uRT05U-0vh7ml?=;4vi7s0qQc}o3>x#f&t_|~ ziUJsvzXX+~kk}(`R@HdLm^_46jf1n-c82$1+$Vg&H_pb|oir3JV30$D-DlDn={hHi z#5dC&effwHEk@S3ewd^@98J&C#+aKwVY6afzo&9^RGnA!JC7-7n3!X1A%qj(f34QF`ad}Rw z2E4s|xYA8Ix;Le5biYC4R&-t--<+Q$`1bpii(!+Dkp~B@!Ux`8U*Db|Lvzo*1Ju-&{NcsGfDpSn-B66AGX*#VeBQffPrfXcr0OKa)i}oH z%is=*o&huq<^@Zq3u-F>p;SjlY|l*k`VjJSlWHo#_dk{t&@D$wGG?$omss0_towwJ zpX}gpZoi2#Y#kmGWDhF17QS#MLf@zPkbl&X>)QnLKLia#h@GIuHEqEu>CG<}Bdct7 zeOuFcGxqRk9izQ-S}Jbi!pWK+9H4+~lJ6n+tAh1{(<@ z*UC9lbja^tT>Y}&bRv+RLor;>w{5(eP3fatP2IpOU8c1Wm7W@<29%u_hBDkiut0>YuINvu|YFIu?iPo(GB2@w?sVNsDhY zx|jETx`uPP$lktY5d((m*gUAA1b+a7Vc?3?4LBSdLksxF@Zl&;dMC3%C>>j-)(LgJ z=PzU2Em1y1>So#XS$g41T<}4~ES9#Ux;9dG8hK>wR_S_ zkE@i`=Ms?eW45~DTZfxKC5fPb@>et)-8d%Iwzuc^zZ@SN=>XG`Pf(GnmxeDI{4&Wa^by7=X*rj#1pMNZ7+qEN#;c$MALsduS z4Bag<8JO91+oW29XlPE{>I56|82`C5Q{GTNNGO9E4}7PfE3q0A{RkwC}onC zVyG4B>q{hfGrF1qCA2?S{{*mZP1Ookn%jm8SRuq|JA3Y1O?Ur8K|`_@P4i3d+p0#~ zm`!z8*yr4LJwBpdOf#{()EAIsGR2P$rB5=|Ka#P)fB*i4y`CypuU>^CcMbBOi~eDRMDx+PJ%}~_0-Hv-P~4#+$L8Rjj#NBL0S91->=MIk%~>1T8gi>kNY;- zoVumY!FH)>)$*S|p9v3$Y{ue12$qpsp+JUW{Y zj*!EH-Hik9T41OTL1ojAaQCxk&YioIDuWo_NQqduCqykktdxQAe4&1gYPPk@{=ObC zhh4(-$c6JjM;qw>-Jezo=v8-duogreT8K!=zuQkBdY>DVe>ml^kg}b}kZ|Xt#H%%P zZ<(bSKs-G?gAi~rZ``idh5ZDfo#3NI4%}D@Xx`>yx0hN80FvLJE&0FI5Fm z`+(t&Xc`n{&Tiy8XpTXk8#_B7fE6ymd-CgPR7shPn3aRd`o)SmPDbh#|8NUE;k-+? z!_j;4=uw|sp%-zOM?iB2MUU4LTCwKrD$3dZwZ5uB{uWY{yVU)_p;PN?#+*?PZrAqa z>G5HWu6~zxr;iO5L|T}LVi5np??GE?fm+wSN7c0f6!moiM2?JN@Ixd-=cu!Y9+mGL zRxIET*rMs?gg{^nOID|7vBv9SFm%n-jT<)(Sz(&Vr{RWR2{Qm!VWTXpXY3KhQ*@PN<$zYphMdA)pi!N)y!H~Wb)1RZ(up6kc0D`oeyXPF zxVV% zM;=XfoOjTm>M$6*DMiR2nOx#S@m;F;P+&+DhKqm=t%K-c@-eqpTG7xL=T7J59sK=G zBS#{6`6x9XtA&PwOmkz`YToTSGF?^?>ZmBjX@EH*V&#IV-fB%j$mL5UeQTUf=uD;q zIVAZ*5E`80ZmJiR^M#qZ$f@#uY$~u!$qSd>9Ia0O56d_>*Z7}MiAxkYLmGeYp{ut{}cZbnU0U=1f>W*`K>IFXh?h(k~$fgZ}-rf15`3s$;{i7TvgC?be1B zDH6b7Gs4yN3gNIv8RInF7^LK5>`&sW&)tfL$P}2afFJ7=@n~tt>{Y!oeb@4X z@aP$iJ&6BA?>1Vx7g!j_E7K{cQ2ZS?#?S5b)Tqx=WgxSlFlTET$`TgO8G@m$zv^s&(a1nMJ-E6%M{gzc_DGu5x)msS>A+pR1dXY|=UOT>mIRHW5 z6n5Ums+l%o?Ju1soe+c`HNqt#Ev+)#5r?R3;o%dcHg6`G572pk3A zX4FuZwgxWE=E=Z=vOgJbaTstmf%!gv7%qSDD}90q?kQh zG`RQ1hdhwRO`0soxO4@-)0e)A&zX>rh*VTDeB+(L`Z{CLiuqI=mPImc+uq>Y_rA%E z0t_3OiH-}Kk)03&lVK$UbB~87J#6AY)+^CQ1=ja4F-X4jy zpU?+EvFPus***m9RW``E&~F6U$iPWK{C!#tMFU9Z>-!A5vH>KuMxZmN1M#`5C_8@< z(fmDtUs`0wNf0FekB@7vX6{R2Ty&6Jnp|MQ>;bi)jW^*tEF}O)>S6)VrSMb;ix#Zg z+D{mJZZDNoY8mz%VfREKw`r4(O2(T+tz43M{WK}In(zC6wd}Xxz6A&{+kcGL<<%c8 z9D&Ero2tprMyJP;y^~ts_f7ZHJ{0xsO1g7PCHA6T*NFiq67Icy<1K~FcWl)a^cZNP zX|8m1>7iEBfYoly_uPvXW7DRZsLiUof$Pj92U+;0!A#0$E?(?o!GP7mT*KO^$E+(B z$g9$^XK6$;=0HhAG*K{-Q#{Y9dr>c?RsF<9XuYi}7!rk`LCq|A9RT=%5d zk7pbldYj2BLN?MJJEk*IQ!4~I^wha?_tXCJ^?d@|_8Z9@okV;vXEVw#Tyg6@)cKtw z#F{aFBtw6PAGt{BsDmyVyydAPiw@D~P6L-7YV?6>7(JO>NPIZed_(A)N$5%#x~qCm~MEm~h`+tDjI z$JrR#{v5Y}rkhBe_?xa**6u@7{5&6ipBY%QntEJ!(Lg+Zzq0Z}!J%<>+0E~U3U`7g z&?jl0qGO$1tnjWNfHGZ}aDo~Qf7aQ8G($O=mw;OzaVmdJy#@;O;sa*?<02 z;`Q#{y?d@MtzFU=9u9rQ^EAmWWm4Co8EWC$>xNmbVs;d!!kyLfk%Fz@u0<1O+oiDw zl*1Y1W2&QLK^)a76bT1G>E{*jWF)DQd%1Vf0EP$l(=RB1V68U=JHJ_ow`oHAcI_?! zxA&W?_YbsfG0b+gLjiPqi>W{N?%wS~{zDq_zO>zN@^?nAy z1h1Ro#(B|^k#5hpEfbd?8&j5V{N~fL7S;4_ykBm=c zD!QAyKp)f|8wM1E!R3rrPYTc)leXZkuhcIERB$Y|rtI*sXBd2aPecsA<1^=0il$09 z1L1J6yVgZoSj(3zqeR!d;gtFX-pq@sVlzak#I?0mWZ%Vc-T{hPAQb=8^AE;M{KYFs zQ>i5E&mM)T>y{xig4 ztCC;Sly&Rc_0i6bW1jA1DEA!=)XvHm?b5;DC14y-JB^3@cBzq>UaN$LbUQ~iRQ`ba zqkp0}b0#lXyS`%Iu2Z2-mkTc5wll7)cfP~5+0O^JpVO$_Y9kJ$@;VWlK1(EnX`W0- z2%5}?bo3v!Jz$W(A9@&rWT*`YG4jF`J*I(+Mo=M|XVylP^GX(_H$;uAMvE-C^je z!jES))p3)>FNtq=CY6*#*icO0IyF=Ryjxs*^XAQk&2=2`z&Xl3R2r5!(?!TIjvZ|H zmQ-vMsw~mMJ#`bxhs8lsX5`p1riK_6&eBlKyOS3Hp%G%p`UW2NzFOB7`HANif50+8!{8`hr;1K$O2Tz6VoR3oMKI@3VUFO9z<1ys z7ydmK)4*e(xclxJiXP!nHY)qoWsoi~q-KMv{ZJq!Rcrnrz?&-EB8-e;=6=;&HEe7ZavYbquYTU+ z)(bl4n#|s~B%&l>n}%&E8ifcv4?cH2 z2n5?rgM+fChIBUv*tsIzO@*Wl(JC1Hed+4e{X^fx>GutPnL65?dkq}G40%bO_Z}F{J{>p1sxj6)WI+OlwA`92}uWP4H3r7Cu^f;^=Ttu&^ z&x-DhxQBo_2oj~e(a!@)79NsK$cwvZo1KM%vU~}p-KnNm)6djELc`+Ix_9YfZPZK2 zK3-H(QozT1p_uhHG8ED$zjR8kS-vRys;06J%*;{L%4vTgCjvV8J+S#7LTtMTn$t${ z+MWVwD7zI<`jpyQml3gHmA{G-e9K_^(^pIF@w1rjY{Xo327e|k;pJkYY>RIeoM~_X ztd8~xe8Xp(p0wM!4Mv;E^_`2S&H7EBKqSvf zaF;5fXSVGhQ3s{e+-`BrCG6^L=a$yymb;rD@1OT*AZc`p*D^yC21?;{Zw=TK?sDImC39q?5E$i@8>@~X!2L%#BoXTVi3%xhZ)Y|;U z?@krJlQw4T*#7E#^|IAZwlE-H4(cc#ke77{XckL|^A4RwyI z#Cs@jIGEy0S$~FHi>GPMBOgO9G*)?;hRsGg8nO-RC9Ul$MW+?$PBB>XQ#WWPU>^(q zb?{^zgjqdvKaTo|32y%kJm5~KQ~t-xPDZgTYrr#dz33x}7Llxceg+ce9^xzQq%Z}> zh@AoB_53y4lR8u2jCii5YSPSL%bFAM1dB8Pf{I8oJYKmMlZWz*R_ zC$NVvvTobh1Nx&xl6SSM@9(^lQ3Ugb7_uG&24vSqhC8LLZ0$r{VVkU5?bjR#Lt=V2 zAyN=+-W!WCV-kaIqm7MVW@BRc+=FpQ$A#Zot{Fz?yE$JtKL~gkY8OHKpX+nu0=%c* z@+uX%#jRc=_(nBHjoWDY$1WTFo(8kBQdgbISUjk5qjLQMW`oQJnd0AO(~i6G+|0Uy|-szR(@%V(i{93qJzlgyTRI9O9eVbM_lXq zEU&FAH;&+iqmONJszr+yEF(G#?g)S8D><8yd
    prr5(!~m!xu!D&Jsf2g{K&1lDcp+^%_ONh4^QS(`*@G z+^8#p2;wpU>+z3y!mc<82*L&a`x~OUqJ|>jo&$p=jQxoNZ!13fCW?7XFlFM~EJhe9 zlf`v>`Fi}#8aHw+_Zuap(JgC={Nv^PwjFw z>-@EAKi!VBcjsvp$n_xwRcu6m{d~3s8m@z7fyt~hJRf7MR5%s>6LKl_c2Wnx+3~k3 zczMM8B=`LR-E?7(Bro!ClO|2t?GG6)l2(EX_EOhsSo8tejVfT~01|Rs_uNc#gbv3G zPeKGp-D5YisOl2kQG|kBB7!7KE?!7?5vL1*g;q-j`DGk731^;*DR2kNj1Ll^I5Oge zmzJm+Nw0&53T49R^x>3TGTOsK%*y3dh*r|41!xW1L);J za!`eOq$FPD@?h^d4ymzcW`0abpP0-HXZ2&J4RlCRCKz}w+}tt$VpAn-{g%d$N~XUV z(cblVirOqkrLp$qE_i_iQ%7VWdh+HoMVFHGv4C6#36wt3YCeP@PS8hn7V22d#hKZV zl+$tWOolZY{LGa-QidKVlzYX;$+8fK^1Ci;t{@e;;7eC){D*FEMGQLMgVUz=%hQt` zTWv|iR0|ks1JC>l66s+yM<}b|y^2d3J#R`>1)~npgkuKDN+XIFf+5Wg{=nqWXQyZ{ z_v_>jgAR4BQsLQ9vqm-*F<{qEgx~%0N9h$MI94K@E{ z{TnuH5G~JwHB4Bd8;Z7>_qZ!1`$^TG#g`~{z08!QcrgU{Q@bJ-o^3_DA!CnVR6(pe zFBmToO;5AKZwV-DWrSiiy94H3%LhV%Aaq|lbI_ zr%~{FQ5HTwm@{NkiyEt6e`Wf1uzqt!6{p&ab-8!=M%@avApP^+#-aC+J6&!`tR}RB+U!}=CQUA%@d*Oy(hZSjrQ~GJoC0;0N^hudQm=-E zUA3IoY`?DWy=YPA)N$GgmqPC&hdK6ANl-KFzT{;($!bsfP>&}-6qSm-4wG*z@Cy4~ zCjBle7c@L4AB&?5dbZn7fG)k`1w@itn{e9gqfZj!ze4F(QrwsMzyeC zh!C~I2?gH#c+Nu?6PoekCRPbOEVJ=z4m ziPXcrf}9?tF(Mp_DHZu{a`9i*B1&9PQ`c&rs@yp|_w$i`2a2xtrVCDK6Q|DL#S0mp z#`V0Cs1w#Q?V#(tr>C#0${wp`0~{SIBesE0&_iqIl_l}c18jNU=k2N6M^|~8kkdGk z<^*ie2zTK5-0b9JNoWId-BSD6_SLJ6UigpR$m$n9+Tg5~#oNqe75(5Z*CjiUf+?Fb z=|BUw2FmK@S6b@k_8s-%K+Da zz4j`!w-dgy3bM1UBF zNJ^{wpn7BClFJhijkrF+BT#g!OPa*=#1@>>(T))Cn6*i3-S}1I^bm+xd3yTu=?Q*J zt!)@U3(qGxn>l;($EBPL?Va8Ql3?nLvS5y|;%J2|%HVS?vp*61)ms)KQMZIT6MT8{ zoGt;@O`Gme?DGA%}8wIxs)@cU9{hOUYj{{&(bs> zE&S|s@52iN(E!X*aEQ^v*?8xU4!|3HwPQFf?i_f;D>MwZ?KDfil0kxN<0MpAl6x8+~ zwl*Rne)GOk;|5)S5_$nIZ7*N&j0U}zVCGXwwiBo{?cBa*Tzd$1vaFgmy>gHpuR`wf zjrshwwwx&~5Jp~xL-V(NJPW}Q2`P|nY8Gg3V6(eJPI zYYJUKJ)YKgQP?gT{%Bg;*U5TO^=8kVd!K7VGx6_2WTs;(H9KQOiz7|>s+WtGDw=8% zg}`T|7B|uENwziZyUWR}{RGXc2b@>S)sJyxEj&@e?k;y^Ey3R1XV}NiKe*%V-UK9% z?9#Prz7IH#Hy0zPZM1}iyOR!98J}T#m>cJ8cHj}_wu6b!67PI~`4F-7r*}0p{rSz$ zkn@{aBnz#MB9Re)G8;oF-F9W#QEHe2Ia#Vgr|Yb{<`v;g-W1E>Or*TO8m~wQYw50Y zGvl(&D{OiGT;{TT4fZ85-%g$h?WgCQ;eLvyEy3l8`4f7_+3ZQA;s_(uOqthbes*CAL#EpfVLhWz3@&&{h zO6;PV<1R}Y{-!VTWYQDF_^V7jMXB)k5Ln zL?onh%jV5loR*WoQv=~UK>HeIY+2{?1+!682R3qV?#pGAFez0z$4{b^X3{?K22Be;Da%T5CnW~z!)m<$OV zSHeu7j+m76wU2knUy&XGao=K)nGKVfq4$`1H43l`XTlR=5Z5ef4ZTktC; zvN)9VY6wf^-r4fvX?*&(G_>mKnhch+nwJHluYmTSmvwOho#d|kE`XeHr^CP%E<`7D z0<8uEr3q`sVyx?V`CVqFTY>RX+SmA9SNh4?ny?F#e;Iz`UAKVHrMy&(J;_29`sdV z{5H7|8yA=afX5*;SdMW_Rru3u zvf$1^+Dg%x%7`-;xzC^^&^);F~f1>QsoLsPMs0&kNAa}|f zQOj;NC^6+Dk(Jm!4UA};ktj{a%T~a1VM?!W9rm!p4 zejCz6j&5#@ec2&FSq25s{p3K>N+gq()ijgoM&Efw>gXA2MAR?oE%3zo&Gn3yq3lu) zolY9QwPBCCa&L`2TF4ituc@=y=1~OKS-(HG&AO46Hp9$@6O?YiLv&s9Ve8hd7wi^X zsE-p;R=Nm8oDl7=nC5eEtRmB9XqVTUS0y;ME23I~Mm5BGj0yQaWW5PkkNMm8e-*Nxa^1VLS`8m(`cG@OS*t@F*ZH}mq>5FVjny&h(f%Uv~zaz<~ zRWcF`2P7nNEIJE|!kTV)S8W3$o;iXzp19vAVL!!&yCy9wFtIb`BN}JvtJ*rnw%qg( z!JC{}EnBBiYY72JR!K$oolIZbe>kk~kW%m=B}-9W2^OVphGA#h!C`IvCOP-G`69AN zWHPdIp~r_uQ19YHX2z904)!(!2TmtHV7HXHjzwYg;izVfqP9yW20Ds!FJL_+-Xmid zV7go=9Se=9e3B>N`TTn0tg{Teg;)wEaBHiT<7o{Hm{o3LFu2tAdi9|Ckp<{TWZSCh z?JX>Hd;(hIT18}7fkUfE13BfAuB&BTk4Q&*ni7%_Y0UlmlEqw8X%#ZcKdzEIORPO{ zPd?}H-^dAthOZh$kyySY6hN83rEFK;CwtV4yUS7i4MN+vU!ZXDrxzt`F2R3N2+Wwt zzjDF<^7lIfW%gR;?$R)))&^iRzUFzGyal6a*zcHsSoA_c1by{N^0zFqJ~adfzyl(y z7O2T_=>`@P_?3_nfJ(y;3}fc^v>jRTH1Pji)%Xh{aFJh4Gvden*;zB z(MMqr-*fBP8z?4Gk@ycge(KZ&#)=8>#R;=4%zSL;l+KYW39VZ&!LrvTY%S)_ka%CO zOu~Nd@;3CP9Bw?MXulP{GfYya>-qj6mBy9aZ6Y@H+Wp1R!M4>L=~}jMzngfhegv8uk4|mq>OJ+gXjG+CrO204`zU*8c)tG=K@`%*$hUJs|_ z*hai4^Cge?AaA3$mbP79DUpkA^OyC#yvRpF;$e1W9)+lh@7_ZR+C}XUBVbe`lWmO@ zC-ZLIMp0V&H~;$y)<=ma`R%vd%3409`4Qxkz*9teFGPI~Zkkd5zu619#HP@XqVgOh zU2;5cVt2Zm7JW%|OKtS>g3D4<3IdZ1)Xr-G8D(=1h_D+ns{X$r0F6*OwB3Xylb_@$;)sO*kIJ6A9Uyo*iq>3mDY-H=mcdQ7)#iC(mP@}ias+_4& z+sl}E;&RP{xyfL?Z?H$nc018O_te=*lvh1wvI!?}+`j`tg!n-1?rr(dUnHkU34uX` zi#SLfdTmKL`7Ir9hY#L2DMV+^Xiy$|B{*P<+%Kdnd7*Lt<-Huht@%5+n>{_ecl=m{ zcVa7iR}XwL6@*bXTg$2RPfEJyN7v97@eH2#wO)2gkU8?vWP{ct>UP^1J8SsGn$7h8 z)<8C{p97=T{&gGGpzZM{TY8Ou>%&T1%YltT=hPs`ybD<;a%h$u%k|zA84YlcM5G_= zJbrven*!ZH_vWisky{DdXd71weRcC@A4zr>ImrH@7M)_0=?xph^ z5N>GSj6v`!U+|jW^NW^{VtbBWJ`-7PH-A-V!+a4p-?wf_}2HY zm2jV-%dII#U#0zoswfJ-kh~p8T=NYD*8n!!p#kI0VgA)537D$JBT5@kqwf!tCxCL{U`ni~3>=A`W z&B2(CmrD{-kfr@_Ft4jUF*0~yB*(?annzeXocT=hm}zoC*(|K_eEm__OX^AzdpyOV z&b=~y_&3UX)HO=>H{EvqXV9|;qjluDH|nIMocnj@lzwNI^=YVl-rniEF}W#GAoL?J z`3>j#tBFna+nM@oJgPMy@q`d(5KS7teq6#16;5bIkwI#rHzJRqFfN}yMbWo_;kPGr zc@KTr$5n&>t()#A>R%Mn#=6Uat`Crg=)Ce1g`rpB*ShlZl~2o6w;J?gOeGa`{xBze zcEsYyA}c|JU#o~PBdScbt+f528IXFr5Ay?{9vDK3^4wnEre8n;XitT<4}I4w=y$M< zsF*{CPX1Xt&I)GNJ_m^nU~Pt6Pv+2DsJHw0;(UsSg9pg>U7!DM zMg-7b^mrSgKhUbTVng%-6Qek1qR-j6>vl>sJpv(GEKreo#qVo3Me0ygTT&L}AW_j1 z8Es*&u$Y_=AF;}fYlF{o;;L1jmJ~4WEWlpK`=fU0R{KkS96D5knd^ZF2g?;j&W+{0 z^y27!2PQwO9Hs^@1y*MaI#`$U7j(XA>aNAZa3XeTw`^;z?PC}M6DM%pU`o+&DJj_- zxKtOSO^~#LVQwJOQXp0(?g|M0lWa8U;-X@rqt!lR!i4y~TuK{-r)pHY7UybKN+%`f zpPKf@?cy8zVaH=)#-KQm%);*9T}TANiAu~(4P`^nn$Y#M{^vZy@>&o)8DH#yXXgJt zJs*5qKQ))kFj9Ci%l1R?lRm~Y(x5Xrg@qi>KIv`D+DoSluttjhw~ZUm?quj*ABUW{ zUZX5vTR)fAS7vLHSL>qjIVI_TjW@1u_llBj?8P<4fCrG*!LR({JGzNUriMK$L}w_7 z%6_{z(o^~s{;;-=L@sa!HNry));qQOt9`gbp)ty3r}Fv8+W(WZ7ZyW+twkZtjM~oR z0xo!S!ro%IapY@zIeq8PLe7ZtVyWMST1xV3Amc3ciXvj@>lH;66fLxD2fFu~gP6rR z{?^Fo(d8S8=eJ*acLZbh0ia?Wim&CUZ(j}NXacdHYK-Mr|F?fQ%-CgWsIS4zHyMzB z=T6IrA$_3jNU+=;TZcvEZRicM7?0(yBG}b4J`{l*n7D*+CuS&t)xs=jvOoX+-KLVX zqvPm~b32^rB;h#PAExen?6qf)s^R+UVwWNRy|l0`^U7i=ZGUb{zyGHSF)xJ3%3zj4 z9mi6Ttt~MEL*?2f*Emlo&nfhJ+h6Dfx;LhlLRrsM_(P1^Hq_NDr&*f>ILur5B;yIZ zEU@?i3g*g`nD#zf*gZ2(?Q-f_o2+y8I;J3 z7+1V9!fNFdavIJ+N+q;Tm0-M#oB#ZC>d!hFU!WcP*8@W62I1xc$k1^(0Z*e>Yjud_ z|3d$UFsZRx$R%n!c@h^&5(xy0EJq*IOSaYU%M8zRS{6X}d+oxqt#a!lA-y)HbI@Pn z?!0sT`iVblE54!B4vCViVa=%_^*2<6(BhM*drVe>eGRV6-pv6}Cp9-bqQ3_RpZPO- zrP%Exw}30#fz{{&9s~A3}O_ZHE8m4V~8l|N7Rj9inlK`EWvu6L&qTLC#HVay#!xn)%)_}578*Pgy&0coC`z(((9 zt$MS1FS{eKjib<-f*|kjpj;Zwy*J*1)s+a~x83~|Cnd+#^~FGAWq9$k*`;rKPm%$S z($hxPJ$@18>|!P2h1`MqiOq-kp!NuPPaJf`_FlUqpOOzy=;8ph0p zNPD+6@e1WFNT_=Mq~=6$3rIZ9^@6#ZYblXMt=T6KTPKx%xgY8beo5^w0L8TI6&S(~ zs3#(nm48VU_mBHT80+h-eE%v#m1;03gtId=tMWBZK+{U*?zDi4$4$f*b`DLw*YP8(jeJ-`N^}1JWVeOjJ2GW> zCIMF#5g_#!7WqS;nUUN_Dw-;wVugg(4opcUUpix(1#jwlYG-W3SCxKb#rpXp8gD`0 zEBJxS4}JI*m2=ndy~EhsA5ToojqRDTiI^~>GQ7b~Ydh;G7P9I&`U@)TO7i+1w#Bzh z;6AelsxC2_0{A)J!s6XtOJ(@dHmB##nbW=F%jtl0F+WZ;Fd4j{Q-eTfGTZEA3f7ys zQ8#7CFick~#_Gj|U}uUNGh##r*9@Fjd^o)qmrF*CfGbW9{@*jlB8+{6i1-u-YPM6! z8kz>@FTMvSbq&&o?6$UP=h4)TuKKgn+xeVz&-VJJgSq2xDbk(z8NvzOw5Ylx#FBds*Z4@fcyPK5PB zwc`l_)U$RoC4E0{yi6NW0w2`?e33m|$OxC(!Ek@8%<(e9K~ZLq4tShHFny$c3y0a_n4It5*uHH!U_DB z3RqQAiqV1+p#ELwVcxajyNgsX<$4^Sz@Lo=fv78o* z<{kIs59TiAIj@RVhX>8(Z=n3nhS(t3vYo%x^$fO)G0RZ zUjFxh;yaX*BRf#W%K|nuuvEJ6jYhbN1{M#y4SD!)$GPOkrNiyqNauIBXS1c@LN4 zh`Rs&l44>3KhnCOg`8$a7_) z=`7jlgf86(*skfNB04m#LJ=%fl4Tu<)Rh>WR2k#YE~gH9ZLxHl z(t@vuYYN0IDPqG2sfiXr2QY_4**jhgq~#eneJw>zil%JF_W5)>o5AiRBOBb65T3no z*_UqMNtZqw1S=#LYDdfx+}JaIn)mw#kEG4HL?s>RT`RC5K(pX#OVDx~qXLuLq5Ff0 z)23ZN;iP;oxL})AZ5j>FYiP1*FOlgl8KCZJlm-4EJkOT_gmA# z@b%(2-Sk~jFr>G1qpGMp#K7gRcBl!+Agq6hqRTM7zFx$WcX@Tw-vQ>x;7s+8vO(*+ zZcWrMy$B0&52C?Yj+#w#+aj-T_#Y!>sYop7-U{ga8x@UM7Cmj+0`U@G@zrM#DZ!Hw z<6YGJd&g+R1EV&?KS0Q&ZV{1^M`Cu--}A#W_*}Y6+~g@htX6poM@rwcu>7xn^X}=AkfDxWOPKEE+G1pL(E{^D zM|^tT$-=R+gsLeGjzk>2RC9F@GNZ`&g9UY`vKjv`?jGbl23&Y~Liy(;N&1i&52)^o zCH2se$|+hF;_Q;KPp7ynpE8vG&gWmsQu?UOb9$3zWC=Z`trwm-dxGcg4u>RZclqm6 zyZHh|QwhuQV#bvKw3jh^Uf6smOGw6sS^hU8+JNr~>f%)2R~Ts~+* z&F|I`^#lZMNDiLMLQZ*p5Q)W)cVXhaxAjpvyXIf zM2jI8mSf3CbS8z;;gF5x6>yecTi8z^n*fQ5hbw}48A9Iq^=i^3ha-V`7b zK)VQcaJg4jf)Q2&$^EW+HQ(kQ<(?7IcR)veJj_(!F7LB3qsx})sJ&Pl}#(g_;HNCliA;`kXX^rhBlvZv?S^MMa+~)$(yxWE)-eXmrz@Dd>L$lhw zq$i#H^UwSC54A11_h>~`Xv4-zdYLHRnZ2W6*`maaME|~Rf>~?e^9$jzxPb)@+;#@A zmc8avWa~#;CQKAF9V0^jD|CGz+8s**7l2j})pHZb9U&; zdHC>QxB**9#At%>Qw`oJ&kG1R{?WRrZHYG=xopm&nTc6~B~VT6zSc0vkT_g= zDAn9b-#D$?)R^a1v$%X*#|bi+>oY_$p>0IGVN?CdtK=EuoW$w?v@A>Tey?iwk^hL) z>#bCp!Ez`eO|HdTq`*N&f|V|0J*(g6HU`Rx)lzBRwH}drWQJ$CY3>E4sS}%eNb;QT zhN!HT+w_%lmj-nor5+-&x&rnHJT`p{U#Mu8@M*{z!cE)n;4hcef?Q5Hr3G9Ug6DT% zI*w2!+H>zw&PctuNkuh1OvLQH8V0(bAupU){U;K)3VO>of3eGrea=m8-bj4*#-|x< zX!pC1vTyaVQ1q>vU%PT;+Q>P>K*_0Z2k4JHB~ygz`wQnVcVA4*o!Io1vK^?e=JlHO zp;M+#Jwo${6$Y7R!cU;N9tIP)isp{uH1$hyVvBWi-(klmjT<*h_D-;`V5>CP#-iy| zmp8Cb^G?=kQ_;H%mIzG7(%PUsAecfvQxG-m(Av84gvLV}AFEl(3cNs?Ks_tI5a?c# z4;jInUqhjfCvx8k&eoD!Q9@GxTfu}il`RfzyWg0>I8gL1uTEu^znlvpjh5^pO^>;z zOGIc#y3w3xyl64Aw=gNEGmp^`rf=_SVd28zk{Aq^T`cmTlY~9Ld}6xkl~>RQdH$8h zGDcCDVI=5`QM%IjXavm4UG;8sR=ln#Y|!7Z)%{Lg^9WX=;J+FN<$Ul7_SoZiu#nMj z^yYCVFX5uOi?Bw5L_@r6*FuRdLfK7-@>2&iGrCfg9F{R)6`N#!Bqe|Y(mHV+Af-mH z>&_>WbDt-@2~h$Y>LlJ3r z?#QfRS-5p;6TMnG&HL#DJ~15Jn%6EH^VHL>n5fa$j-Ney_AoaFcF|QIH%b<_UiY`@ z*|$=y6vI{BgT-5Sg*}9=CCNAHdjKQ!b;3tecN}WZoQYn&3qatQmRestagzGzk>df< zEaCjb-kMMS z2O|_>r&fXOW^4^nFRabdm(lrbpXq;yJ$Ob{6faeB*fr?%|3-Mi!4^aXX4(9^PMwM{ zy*%^V<>2}1VK3%rugn@1_({ZMfyQitWaY|VlIvS^ul!F3GEOk`7BZs5ERlQ|+U~4_ z)g>srTq(FU9xC=;z>d4YXeB=9J`zG4L)+`e{W-%T)GJ1ymOp{EWkpWOeP%d7?kdpQ>Ud(~rIS7eI;%a8qc zjRnV!dwYM^vW$N9c;#N2Yf7=Gd*I|BX(WsuQ5Fqz$`Bm{q)r=~=_;;cxVFY)EN`{( zFJ0TQ(>MSZbK5%i5(7Q`p80pZ^?JN8yr`iyjLM%odw%`Apm>B!Ms3sT*3uCH+Z?){ z5nOr_h16?7?JRELzM!B{sa9_Jbt!oER_tYK-plSf+h0Ucnt^N!7hGH-mZg>@cPmy| zkB+d;;k($59mH){2uqJS=nXlm@;1UoO!U1{FYZU866tHcHf zTP#_Gr|5;r6Bbydm1Deqzmh?vHJ_dJ<_uP>5Vp` z`qMDfE}lyq$n=Azx-@7lw+i4gw{{d7tf&CAoT&42p|<>fsA-|JB>~k>%DOvjl}j5l z_XLSC`RK01Pp+A&%u_vrKZAYW3V6q}$jPV87hR&0%jutg{#nDZE%LNFf=#TjN5IF8 zV<2N*mfX5=BSZQg#GRpTi-D2NA@smxQBPdu)4))m&<0;M>HEbEMa9{zp;bg)7 ztcDHrr&>%aNO)0Nj$c2{4?Y3=eH`{L;havJlG-I&HZua-J zEe92(8}03z@SllMX##DqA49IaBm>Fr%qBXD?@nHM&b6xaW|%>!lYT)=d0wRh>5QS{ ziYv;yqB{i4h*Qlsb^|VumFzOjz4Ww>qW>dphB%Xebu&UkLLNv}2eO_v@_{!j`|Yy9 z110nVb&TsBU^1uj8*ksf4R#Bz+)5`~D0z_r{fkyc#8ga;YB1ox%vP#^N8IILar{z& ziIFIrrOlPt{(&KL7jNXJJqxvo5rUJA573^P?`cwe3&OlJ@6L%Cod_QQX8O|7hI=IN ziJ--zcNfcO1@L@kpR~bqkEVU0)+I=rhI~70a#6YLF_r&EEGcr7q2o{9=_-NgcUe}z z2wp`uJ&CpTP|Hr0qC00;uBa-Zzj{}&ETW|nuaC|z|9zWdi#!sZNTmq>@3hz&Otb&8 z_&E7qq5qx+4C}YUhc|3JaLixxM_T&QCeZ~nx!g>?dQeY*1TKJtEZE^4Omy)8PTLxZlGOH%8v={BgX?gqcx9#xd8|C4L2@eBlO=B8zRBz8&y7o3 z4!?^Xq3PN34XcgS)Fn6vwqExH7TAl7cdH^DDKzw+BJ zObfTNPh-I_1NGB^6-)Tr`W6CR^MVV*rMMnFc5Jj7vIN$i9z8y)d&l@bf=hFi7qlvr z)u*DLQAuzFf#Re(gY!U*hsVqk5!+vJ6hvKem*Vft-CuGijinRg!yOc}e^^04=#QFc zApH+W->V=Bf&OfXK9le78{olm0CD>Z+m$g0lZMEEfj+$}F2EHDOxd|pr}dTAC;+!f zG$MbEG@UUQmd~SZ7b7d9CXGb`WVPE+v1_ZIp8IF&c<#S({LFv|Rc0~!ri%<(wsC`| z-&=$uLc8#9?;br1Mb!+ZE}sF|ndG-Pml%X@CE|xdyqvm2qbd%Bm_12cSfm`y%lzVi zm@nxl#It1xk=!fz(YE~Azh_lmZ=zfx0VC$>@XKDMuecS6&sd-(_FRRLqmCWB-EZHZ z508!Yw=scXF`=LxZKkif?!rJKg-uu(7~XhrK6y-!#~4O7C5JdNGFiie8bkh-T5e|G zfWqItNWCV$?0kqkaNujMz@?xXtgXnpN+iaD^n7c|q^<6b3^P^I*Rrqu7)(XWuZ%d) z{6y2kp*TWCwB<&Q=-ZFe+>Mm9OEzCRU1*S`` zWL%$414NKM?Z2-#6BwZD#imMreAmHmWSK?Hwc@uzkiO*G610dVFgO=4F8!O=<%rRv zMlEYth_8tH_hf)rnTJ4-|3RPry&Xk`!*rwL~z%C75>CUOZv4~IwNt_z>VR$Ss9NX7q&9IG0P%yrO1v&ZRIOd z1bycGE{#ib28~N8|0E)F>^tG21ZSC4?){nQqy#oF8^2%xW{;xtDPPS~*MtXSa*wiD zlo@U9J^J~85Q%hm-V)*r9W8+DK0!UW>&FhHCy5eOu#vvyA9+1Y#tTn=E&vHiS{Tj|K2J$vp|iBBN! z(+i6kSSMimRsz%nwGg3J*ud%ce4mB!57U?UT;OU+-urk;(m%UeuF_My_Z~guboyPf zm)}q!PO`>H2##`Y9yi(Vzx}3()?FPa1>~|sy2qIP36J0uZUB40q&uF&u`EqD@LTTu zd%U5+xv~wrtZ?$E4T;Dj=VqFluX?U)dJ&z`1>cm^f84k3+U1hTwJZ-F)q?M@2uh<& z5P+#-#~)@k8SK-^XId9SS4kq1cJK?jE1=;fvG0y-6xl~~HLUAAfLc`2XRL|^&Dt{= z5s$c<1ag7hd{(Fz@lID`2`!MJsuQ3jOaeZ65b?_=&u#!afiQY}_#1u&Lr)V~aToKM zfkd?b<*OnMmx^A`1oW%Ofse!Y53a5YG3)79y6-=4#o)yqCmC9=A8!Nc7RXgVCy|)D z{y#U$Oq0-5y4i$`@}l>jAcBzf{^e2$6sG;B+wld|94vO4E;PlN^q;}uX$AM#B62&Q zb07Hqau6Pw^M$Y*OXPj1)%zU`a|Ds7S=+)u;gWDnYR?ENZX+Q(X#!3h+)IyQ$jjQO zt@TlSfN2%~ZtS6&cm1Jnv#L2wFSiUc8P5q)4@q0~kaTldG<8Y0)Cv5(4mEUk(CI(= z9Xs1zu^k@w?@$-q(e@>L*fQwza&rYO7>+&6$vfv4K|&@^pT6R9L_-t1=~JdWcxh{( zB!>bg^bxgH$U)`k-!ObU=HOgzMDzV0$JB6@HBl2o=BK9Q&-ET4T0quOxN6s|*VfYS z!TkgWkkim{tn?kUJA?$~j%gX$LRHVuL^BhUMXlY9+(KkmTb>O-|5W`<6yLnyqWJEw z8={#0RJ)cAs?0kYNsm_zl=>RuHF-%J;1vX=>ub9}_HDsj-_Ol;)wU&luzA`#uavBN z$i-q-+Tt64Ku}#qey`cWu14TY_;nqBnF>-HJsn`-h&3 z!3!l(d(3$Sk0m_8>dlp=*)j_6fZRHy#+R~=Rh8=LXf{y-wFnkElak^U`aOnm?we?n z70(-IdEZm)Zqgr>FH=- z;}Pwj@b;f>t@_OS$zVSktUahz@us~*QGSH@+qv}U$L>{$Y>qjtK#daIx%2VmE;^>G zM35DK?a&OKBoYCiB|;|Y&mX)1ud~PA1Cz=B@-QDBkG&kRsj9tDg4U5`V6?FmQ8k}^ zZfSw=)pP~YCp4ky<)bBUsF!cbhR)vy87NJsVO^l z>^RZZ6CL+lQKgeSi>)?ri7Fcxl(t^QPJmG?4*!VQmqdw&5M)?(M?J4LW(Pwi`8p%1Oi57IK;V|T`mqZ);H;}8JdU{osGEDj|j=+jS)rG zrO>q4=qF}H{QbAl`?QBTK?msUieyK#XhI`E8DxiOm&uv|zS`(F)n{?niSb$>lD4cs zi=;)1qGZnRa{HKI{=)mAVUIg2(i71dXWg7|ff2!jyM;)CObxrB-xW7fE_2^VeqQW7 zH$&bIHJuolB8C>*vEtN2z*AJw$Oj-y-B&eKCKsVGwA$T7kwI9~ zs?&gQX*VuUoygDJcBZ1uUk}bbBAm}`&zYFfI%9({fG^Ube3RQuz3>9lH=`*-Kh-h; z@#4&Kz48(W9_igRdIm>oE7?21__YE&Bq*yJ))TAB~K<$%F?U(*i&$)v!@|aSO zo`U0V&7BL3j4Ggw7f=Bm9kFP$NI{KpDa?0q&G)5L%6fXOr9s%^&?B z@`trraP>u?mrGlx#>#F(?Z%m(>IXTGdrMBc82 zvyOXkLPyay;1;pX>T=9w;*mVRP!T=lqf;nm+>v~7#AAphB{QjgxPCEf#YZMB~6 zjf;CI>16}M+byOk1xRj{fYNEu?7-^zdy8#+lg(EeKf?)0(veT@_H=j$lxS#r{Zc5)iT zYBbr~K4+S5i|C_AGgS>0);__EvLDj@NPvV^!01l5%w?~d%n9ei@1-Jq>tyqBgTNr2 zS}LZCDXlDIiv4nZjQ-v!eqU16Yqv1})NSBQaumoO+M0E9hrVehl^C|L5E9)!Kpq~a z=KNGrozs8iIsE7AHhwGLHW~z2{ibE(WFs9+i>RbB`6ySmBh6dCvA7xsm0Sd=tIW%C zT0f7F0D)_?rz5H)fa!(5p)&1?R0qqvG~8>7ZQ8TjQXWQ)$yz$30b;$EkEg(*n>)p4 zen{_48_Ou({150J#cw7AC?Q8nw9?>E5UVw5VmE5J9Dq=OJ98r)(?9hL%CU@K4^R>% z*bHHT8pYXdy7(I7F_eZ}&azf6O@&n)U$SAOce=^Ck2Yhe- zo_+L(9u2>xofY#d1KLTY$)%D1!yFG62o3A8ixy=_vlcCwE;tN?KK`g$b)=b6A%3ALMF z>d~T$02$Z-z5*Hkj!;Hu80Kol6Z6B`I$)^pWCaQQ+hwG#xLio`I3)#6R-`%#kENQf zscKjus=RC$zKdd?b^4m5p-LzQa7FL z9!^y(7?o@s`H+^p`?zYxiCL@t-|fmG3sAb{bI4lRV>dyR)=#FViV-jBrknZ}pJoP?5L>ycXmt%b3I(+z>5PLzv zTubNI?IgyGQ!w<$A7$CM{qeZCxSOYSY@j2Ws8wcF5DCeTdcw4JY&y9@XB^c!!J z0PmQiRt~PKXq%@LY)Hv^`*hLs>s!=75Qr`5BFxn;B0T)gr8aRLnIApOKS5nA>e6u5 zFqFQ~wO^3az%~?rdnYy?chZk6sgqa7Aiuu<`sVDdgGFJwcs^jvFd!*O;Zcp$rRP_1 zg#K!68$IKkYnZyPR7Q4a>KP+z$LZxgOS^UJ`J={xn=^pCI9(1jVd^U>kAeAb|Id3m z8tO+QQqn*wOc$OQaJKZB%%o?}wtZm5C{~kZW`R^&iYSjkf?}SCj3eT+_Tb17BToL= zNHL6&E$}M51?NoKrh8~({bkTMA{fyExOOwtZ~26-TvlWV>9Xow-JmUEfgtQ&P&Nsf zuJOw8O!i!bVWtkw45*3~qR%T~&>LAvq>?zc~rjSe7Me-Jj&MVSe|)@ zQ$V+%FfE^u!ksp52ERytBciM%!OULL%V2arg?@G{9rq8v_)^zK%DK*1zG6*tuo(&f zP?8NS5L0XcluH7Ah;}K*Y+i)PcO&(pr_bj^F6Pw zofx?$v1_|_cJwM1zJQ3avbZ!l`nqVx{#5N^vxfa2E7=-%_B0^*4bBCyHcG6k%ogN2 z#^o->y(`S#NR!CpZz`)0O7-zgDX36fXA&?Yc$(~vJHtLI)Y@YZAP5DQnlJ`yPeOtd z_Lgvp`p>fPR{F25()XN7Uj{MAWzy=)mM0}{HKIr$E_Ii;Bs9`bOrs|P%XeCv-Ekq} zHA#tg%!{SmHx*1$77CpsH1xAmtxMU0LOgak?p3iV1@(+dvhaDtcoEnXBhWnTis!MU z6C4s%w8A|Qm=0Dh9w6?Da(pnroapJ_i zeiqN3J(DR!&4@;)D0T<4E=s&@+^dH2SDS`UnzUeANf9hcj~f$GM=?V=cz)f8ZGZhGON;V01P5m# zE)I5vD+!G#^6K9?ih*fZAH~+K$*KlFN2VLBSzp~31th^*@Lf}C&m0BiHl1V8-DkkU z($Wd0nQT}$E+fG0dD+{AO)iZp-5UitKV_)`}!vhK{VB4?wpiWKlxN z@ZWfpwd|>QZlcwLV85oJ-Fdm8Q~2hz0z44zsBNdf;%3zMXE^g4&+|jI`n=Ve-CX4o z+F3T*u=tiL13`L0|4DPBxa$ITZ_lf~tSGpGkSvpd{tQsy3{_bYPGgtAY{~_#D#^Gg zvQe}+4r7m@uUr&3{~6^M%lIooI2yjldnLOh13?xB;3GY;xly1wosW3&@87*UM=M*H z)JjH~Hk?5L^HXP@u0-Nd=RyNqZ@1HwhQy>33NMCq#v!1C~{FGZ|bRNOsZF|3d6tyJ~D4H z(RLSHGKTZ)kwX(fKu$*=Um|9fn>&6=0%Di=#VDYH%rx_ryV}DtO@xk>BEK1{_{(+wS##w@*tOt>ND=utiieV z90Q62>QA1ruk45xX<^N*87R4@_D&+LuXia2;bX8R&k1%(athLA9nsT@@8@ty#LS!=13s_Fs#=mQ8KOMR?Ee!|*bo#~CO>;OhCB9zoo%YABe8NI3Z50_Z=qeHNCL8T z?lJ0REydRv0Rp`1TrHiG7E9rE)i#l=4z+Nu7m8LnYz0Ov13Ov-YA$WB90q0zjDf3* z9L~Z~FN9vI2}EGwh=_&=*UNUFNtzEsp5|Oq)v@CEqVL1O6p7LM!kIla2uSDC$;{0y ze%uLHinz)6c`L<02zS?fE7iX_%tz!(ckoG+EQ8+|SQsCt@Jm9rxreE8#5DBPilW#|alg<)2wnYmUo@`oY_K~AUZ}l7Q$7kNa zvQ}WvaTxqj5?<2~e>#Cy@*G3OF0O}7@#h-NsZ+5NT=zn8>eyZVN6WPTr+!@G7j;H{ zd@GFC)L++ZbqDR&gs)b>XJNCaD{;qnI)F^qOedHNr#_2{lrH@ic}1L zc}qVkhD0pz4n3=AsLZ-eFO!*`K1+*_t9V9$uC1)9m5eeVM2QlMKg(i-@e&`Iy8CU_x`#q8yNQcqBl5ipzKV z$;j}V$m36_jzx)kYCz;;Ton!@zkLigyUCXh;4Ao0#5we=x`5`>yNhfo*;{l0F%8g= zmyq)XKKPz02|oEx%W4(pcM;{Nnf6v$da)mU?^2)=8Yg~}B*SfoWkVNJ8J z+*&i>0=kW?{Hp-0=dCr=4vqm^xz?R6dk1MNjT5Mo*9+e9 z6ctSuc;uy4YuTw@r~U!mb3&X(ugzOu+ucPXhSfztFh>F)H<&i$_#{}4igTtYb4E5N zj*Ed*ERq?_2EG6gj%ocf`bi*zxY56cq1d=$K)$x}6v|l17ri<(jK5#?AN1?rzf~2j zO2VhNHyzpEeN=!&UdCW&gvYE0F~jYQaL*WIRXRm6nZzs+$_-J?C<&jPt7~|G8GR2| zeJfFF3CRvERAcS*L?Ft(qLlz=7mKa==@OFtxup8@SR%_DBWM$h#Ro;^yaXc)P6^x< zrtZzF5>E~HA~X9#kZ8sGcX|2=U!MNTuTMW)(G4vA_LHrJwY4*dM{lukR3M>Q*^}{t zK812mB-<>l03_T8MW25v-TV*gJJ%a^ls#Q>212>p_wb3cR37%0$Z%}B8Y1|akF?kG zkh1iLUw>ip@q&^raSAQH_*C|HND!sYcxc5RdHrd|G`&)?(p43XX|vh}Z@P8s_RO-j zSqb7H(V6%%Q?d2_L=PY%%JvN19MLjdOPI>1B`#Z4o>f{|C8GTdtnJJEy6NPIBzm z-=_cZKcr?*1haxF#&RS4nRkqp=SuoSSs=;p7v+s?#ND7}+~|~o(iHYUXJ1rXgA}a8 zHQSqYET({4WP;Npc;^yHemw|=Rfh1X0SK`_Ib?4F4v~8x7veE^{&jBB=3wf*3Q-N+ z{<}}=m>n{C^6oBTVMfdnaJ&b6{?YVOWYf?s@r!%1?IcR5=$M4nJtvS5orRA-zyIO5#!$*(fh}dQmM`%Fvjz@mY7l7{45W zF^_day6*fW(Idl;EO>jwDqod`U}`W*bmC1l7vjE1#{`kdP8_c~-++ z$Jd!$$KZ5rg$woR%apZjVHIydymfds;^J|7Nv_T@WK0}dP>50c%M)H@4|xe z=XU5o&kNumBpK>RI&bxdRGTyNZnw&s(e z9Uw>=jLI*-LMNR6_p92s)G@uN*THGYQUilqEb9bH+LQ|#M{@Ql6F3(@$Z&}d1WeKZ zdtHr9KB>(qEkVs{lpr_9?YhdI?|giGYdqWeFH}j;e62im6slg8c_B4fBimf7cX4fe z5kU-J#ctG8N$_NG+MF&81B=PSZ_T6!I6P{hVb?Y5|EgoqN6+pCB+F#x4sDHNz<2!@ zj{9WO+Oz+woZYF~1*KWc{P!R+?*Q^tYptXrtoBY`A}PKJ&sweuG049rzm80f8amcO zpoVcmcuz{+Bvgdh3EU$f;DiGxV*@^BqrCQ z5W!DP>XzI9BI6gnGIj~?x-Twj71l9Qlmo(W@7a^00qmcoHUs*$V-LZ666bIXch&V0 zi}l~4(N<&4nvNZ;ipJ{cXzL&&<=e-&{VsUsdK2H@$}jLZTqhzIla){^o(*<&u<|G7 z5lTsUHA_6VzeA1dK19;@WN*0y*bqBKu%1*HS=j(Ml|N%5qxdK zWq+s@7(Q-!Slx!)9uZtiQoKRSI;DE}b`@cr;RHBEHBhl8>uSrGas!hoE9kMZY)5go z>h*ARo;XpI%wj(M8g1o^vCqpqG3)*pisHOOwFy~%VQs3{gl_q8hgC|B8Q>`*2y&F)90p6mM&wyDpH6lmm+hoT>X1$a! z4pwPqL;${sfgt^u>jP$ZwK#}c5B1H5#v*;n_o`#LtuvoNzu)>%fDBJoBOk#TKBHw837l287B z9EIk>7mXCc+&uJwKA=%1F2= zoNrV4w|%I|?Sz?%{tp60hkGQ@T7ozDIzJIQ>QX(6L?v>*V@ZlD7_TtmU?}<0R8C3Bx>M6s{@I|dwIiy|h`H2PP!orr zYI{b=Xn9A{KWb~N)Q$q9AF#;sOQwsP$)W?=xW4$mRjmWpFL*6F7+^=dEy(}%8;Z3k z?9V^1{F{<&{y5GfN~JxdPx@$_Zmid%yA4KzXZ|*9HWuChVf}qU2!z6w7xUd?6L&7W zXDhpaB}i=QPqh`>0>Vi*S+emCf8m)wU(}V%KfxTJdZXG(1Q{#kSJuZiE1^Od_IgPq z5~4M|tUOzN83Mr}RMjNvm{>n>{tBM*VftNJu-spOR*uJ4snjuPQY7*>-2#G>16K84 zkAMu=pyQQc&O|ml`+xie<@cfWO|gF@zMy}WEUeF;%ae+z_+iEO$Vst!d%eBHTS-h` zxQ>pkK5M@eZE(?TmB7rJ`ztPwK$9z*dMFwro8=e7`pge;8E~Cn0;AC5+lM#|f$6NH z7du%LYXkOkduJuqIHnP}3^CVHvRc&aWN=k z{eWsvA$9=<4GHi)xw-b!y+@D!H2==-UcRU2-JWxDkfQG)D;-de2j{ZKLERYI*s#@_ zw4Aj`y=s*{6P{5PS|#wI=LA$&@uHqPAUl}8B=pMzLj}5uoUkm}SYS-^0 z#Yd9Q@ufVH%_&~HVe!FU`ul=2O_W7jL-P;-XAs@p79O*un#3U;Ah4Be^GaUARiP%9 z@LGNGl*d1T%+UfN4?#-lu|(Wt65l|8L};Zu=||qTK2T>w>=N*N%doyarL$DINyPzX zAjuTEOQkywaZps;f`BTa#l=LDBB;Z0@sx?OH_LhDXtfiJAPq)V2}<|bkJarf5`y3A z1&$Mit;VReo50hJy!*-j8NM=k{EvU+kvp$nf9kHPMFs=HT!kk^mzyEv2T#|xoqBI< z1w1LJm?l|Ni`=}EYP0V3mrTZeBpfd1Ug!pn!^iqnkZoIll0%#k&fIrE=2=Su`h1x` z;$OSz*t#`e^{KwK^(7Gf{m^eM1%-%j{%h^@bF_@->olr6(?R!KH=hLyOUF$#ZKm!i zu(PXGe?YCRdszUO<(N{?+IA|rZ9NMK*AdqensM8)Y^8iKb*A4hTEh6mM>%Q(s0Ng0 zW{vR0Psk0~IimWo;CL=oY+jZ>FEcAk-84cbl-O6#MT*>wm@t*}~8X#Qs3p3gI#Ho~s#+4cf&FyXcRc5fYiRN zWC|_#^iYK+JcRAdfoPqjWJVsD1h`><; z#?k~15+QGlA4jhnDKq2oUb0@`@RL5ZY@_v^-D7K9hH;5_65`jDqanH+n?7Y~J(D^) zB-)t=)Kz>hl|IU3nF|xjj zN|c_r3cLekP$a=)(kCQUf3EB&O7BB~4ijY{r96!sJ604=u49-XxN@x26JUT*Mf9pg zRUZE3h`6rW{p=SDVoi2;$ z2+X=7r^NptL)#m_{z{OMC?C?!XVjdJHeb)jqSMtPeu%#n+Q!?;B$UqTyW{2SZ_dH) zPE)KMst%|f{&!rIcf+|*a9$0UG&FIoNBC6!?3QRUUTO)7KVi;d`DkNYEZ!x#t{)vz zbo(P_AK1T^{wQt}``gN1TsyIvNfJ7M z*l*8cvfRQZAXlV^G%Q-{l&mJaGRxPPJ|_t@D=yE7YwRmnu_P}Q&NCk-9n#%J749MWFa(se3N9gglF@0b>9ucTS z4$!7ty!ql<0ktY#O7f?wNB|gdLFKViSV|2$$$BTYPz-aw*kRF*7*)|uO1jv;K9qR( zrjn&7P=~lyq5WLr0*E5n*f^yP>O0K^H4S9;6>+b6_hk)|3){<~}#iz|u+ryCor+TfQWI*rsAHrUQBn zqlA3#5V*i&`}Pf`#`X3EIDkAadefc5u10!;Qe~Z$MGMHJrvUPV77%EdigfFU$Z2F3 zvH%^mbTrMu=LKq&R1VsZxU*?&U$m)wlqRbTh+N5C3_itHtjLT}h}^$;*E==ixwYk5 zJci`PX#eJ)*Zvp_EWB}=?sH|6D_HoPR) zBcXW@=UbMG6~h~rM*?%Z z`h0Lf9!2~4i-dmjtuuDP+Hvp#!`*qfH9OH+feOh3rKvCKff%=SYtmlp?4SkHV$?Pg zhJYTzPOO>}bk4=Fnqlg3=YGC5d#RHfGT*>1iAmeu29jUy3fhwH9qZ3b(JiE{xK&w~ z@c}&&h@NqRq$1{Sd0mf61zYfWKdf6TStuhLi5R!QM6JRZH`?AKLUDXEw{PaA} z+%!lHOaWMoU0Lp$b(W4uwiMC9btxI8>n?q&%o@(1CSu$XkVe{l-z0E7es&RYa;GUk z4_*qZ5tGX(uZ1p0?sI;ZI&X-EN0wi=t^P1f)~toa^m&QV3zky#6;q&`mi@*;U&-4^ zKkYh?xr2b+-C0CEk&>3OmVBz|7R$0XVQ-}`a`~Jc>yp#kRP0;&Eo71;q)9VlcSmZX z45G)QqobdQK2G0yT^$35x`DZK5PGUkVd8m~Hq1GsNLAo6&FZ}05m4ytwQ0}gD*%g zYrRw=Lj+~|CeBL2EZPutt}>XqicRq@x5NyfEOm*H>T;3roWrGbPtPk}0RCi2=uXTt z5nPqrtchXz&Ez)eXa_{fE@!KPzg*4WM=@Y>n=#|Iy^n(MUx6Co=}s(bpQVOJ63^%Jh*?= z;q)t_T~D^=%Ege&DpfTXq_LsznrdUJ?VnjUw;CsxtzvnqDu&-BhUhC9od9qiL_=N0 zAg};@${)50|GFyFC$BI=fFJRM?YWfA^9Zi27C8ZdDvZp^4(JbAzb?a#@29*YoZpvB z4(E!hp72U1km`5QyOyGf6|si|d*Ok3R&l4cZC)mGE`d2Sfy2b)anc+$Aed4X9<5^% zO$u5$@5`W}O19~*kHT1~+hfu%I3=!{E-rNv|J%yG+*m6lQqm3Z+DU-i5*cwsGe^BH zo8-s^FbWIrqm? zKrw8xX8j=|(pl4@Sctb5aIcjlD~ZQ&RP`7@!tv+MK*pEU4NNrs@UX?I#aGG%38R+( zZlOrUz)+N1_dneIQ`XVcr z)u3{#6i?y4RfRH4nTu*{IC~BWIbO5fp<)fFPnOpWKAJOMBH9iGK54P4#QF>KkqDnv z$X+SWvbInO(%KHMJwmQ_QGCBCX6BeWaOv~kfOLaVi-ljz4R=`dp1Viza7W!IBE7_C zDSIG`PlEhq{|>&J73<^a*|+@CYD13-?Lz4+W6JmLFUd`wE-15elC>p7U`*^m*(QRwqO+OVo`d4eamGl? z6x=Yv?p)KR{m@CVmSsd9EvJdb!Q#f)zJ#{e6d8G~>*+wh5vQk#6vsOp3aizLQ%24Zq3q6+6C~A?sFF^g~^xont z4t~?$f)yYssjS3WFlt0aI9=xG9X(T8c9v}k@RI9)oVVKgC=+n>`SRb2w_+=Yn+dV> zx!xr`3pEL9@s?k2p>DicRCrl~p&_@i8-iQs&OyN#Ov7Z2uc!=NCSlb#ws4)aa^=cl zuX!`j+=f)ywy1B?itqGuC;jxB{L3QvqdvaW4pC9~)Y)JSKz^ev8$qKapu~ls#VVxK zEH9(~JPhGie}m}=E=t+k7Hm>ot%H6{Joi#;(DlknAd`=k%q}P|9<$)W#ep_go-!x@ z{qW|n8mZK3HEB}vQ2KO;3?+<7iW)Cx%$Rd&J9-VNAtSE!Q8c@21XtkD^R+ZzhXsQ5 zj^|l@UQcodb{WaTK%n3mpN69;mOoHVARqjv>eJqN8Kc|0%u7z%+VQ-1_DY>6f%Rv_ zzqIny?`U|cgW0dzvybw&y*f}bs{f;$#Z4xRSetIwV^6Cmlh$tXxSXHsQF7{7Ou0eb z8&hv@HJ^TVg>}J))@cih7j($!?OmMkF8cN+jOc42kni@>v(q8~)1F0PzZ zzG>(=7%h9*R8sJ>UeM;%W6K;LUA}zzCv&qQVT%m!af*iT_e?W2x(GU+{^-$->k|_W z4AFlXpZoIp^DEbpllLFxa6O$RkGF+kW`1 z9{FxP?>2MXYUnlLP%}E&Nf+J|Wn|~$>$~&NMHqy>|P58`vN7A;ox3UV`(xOR|HMp>L9Q+~3Fe>`^@l@*1dd}RT zgn4u4t_6vh#S6S>SwBO&;ogiw6N@?x7%*TK7uA_+Q}vl0h>TQb`jvglgj|2mxajxs zfBYfR^7ULaKXh*PFKYm%T(HPUOEX~a^vsckA-z`Uz*xp54*U zrFnbykF|mxx#rqjzEc1@sy;${`$84F-s#oPMZFta=*is_Ir0FCE4r_ z3o}}A_>bXD?CP|*zwDQ93P=BRH7_q>%eLViW^_t0`?+V_%JT2qzkd00gwB(C8FpSz zB3;6+v}ElpMXTisVwjty2O8R~>wIX^?;Pk}Zx=6K+!LVnUEkj}n~jN0D~90>UuW1m zkOB&iv1SWl1f50C{*!sTeI;}zEf>z5Iny%jyPyUwnRFf|thBf?r&oK)(D!s*Ft7LD z4Va#SFH%FVC7j@+_-){b4ow^{`fS-!z^mD~z5R|)3BTHzNo2+w>nG#_&B5H^nfs%@ z+0VA1Ed%^N4Pd9mv)Qcrn00d6yhY!d?d$P;=|I=qiyMfZnDOjleHws=@m}48xLEg~ zv0a@%@In(J8nPwzzol(6D%2(#0bXi9|{FnF) zy9En6JEig{=NSGj2js=<;o0M)Dj41!q}|s?4#9An1x;_>%Bju!ZThBRcJ;Mu*Dl?@ zy=ja0(@T1u%a^x0{k4wtl&rKkkD1+@i)Z|IPg~o-87H*hs{Sr*&4L=tJN(0aHkjSY zY+|RM5qkP`AMUM|D=ge{hl;tZaBA2BJ5$4!`=IjHv15dA%4$nd=D|1FJm z=|_m%yhRPobR&3|+AleJ^82^5MvWM;deY(D3pmKNpk6KJ^6i3lOaRk;yzdpG?mBq6 zzh?}+o_^r*{Hd?+HGLaN4BKTst(Gfz9?drY)xh`g-1pSpd5h|q1@%PfH;31GQTZU4 zcbNb@KlM9}SR=Nw(WDY{d3d+Fn432wA~5(Oz53r54M^MgoyJLrS{A&j`Nm(lbo2qr z_WL(53)-wB^0oyoE_0u=Tv-VT37NO3F$afT=O+TH{dYrSz8=`Ke*OC0`EMCa0>i?> z{+=(;?ksKC@`Wy9|ASMk=sL_A~GyUPD=Lg2OO7hczs2tx)l&a)3GveJ-` zP5VvF#nZFzmy4%Y>nGU_)pEANFx|EVAME-)l{4k{opI*3Em@BqZD2l_+n=JtzusJT z9jwmd=ND)R*k7WjoOrZ2xMi<3Zz^byE06i{YwS2Q1_8jtxd*oK0)EEz)rjx)ndzx=y!z`Ajr_nrL4u1=Tx?|!EKbAM6) zsXlA}K3r!|$DdC`A3HX={sR*e6W8!-hQ0Np>+Bof@U#7Id#C^)Fh(nAzE8x)_LHL) zw4+Hx+ORou+O-AH#x&OYA%%&eIYcddKxQqJL=-@D(vpW3WItj5#| zBTg+Q_%Ed7XD^fL&94Vz~QWb$CFvn`yFQ0)KCJ+<1KYh zZwa)LJmIF?e#QqUD3(O5s*Mi0pq>@s|=x?y0S8EL`ZIqMg=}LHa60Gc&x$g zil$c>uiyw!w=-~q^~<*B>6eJ7jg$mNFa&uGo)SugSfXoea#9}})PE(4kPw9{v2hbl z{+i%_bw>1D`(K&)y%=A00~-_^4H`GDm;CMh(VssDM?_#tAhy_b50SwqkU>OEjq^>uy&bWNCR`U+ zk1pYC0}lS>EIiUQaJj+M@V%%gCR!MxR-Ij3K0{wa0;~h?l5GW@g|6-mxWzb1gK)*( z_x8%d9q=+Ya(!nf5+tSloUJ#$rlukTLK}wIl>F}@L?58xH#nc$v z*svk2L|l0m|8*+$pr1F`k%4EQBvgc^9tLnyuo&Y2Tg9i52A?(?wpbUQkjxt|( zQ>~}c5YOa!d88tht|;+$ch@>ctA>mM+rOJ09ttV~K)lMG_voPaCpd`t`Kz$70V0-g z$;n?{!hJ+5@$A3m#`P}u$A3=;Nrqf%R&cN(^fGmHni>}bH<3;7ALxM*zDMfiPf;Z$ zel_*y1*I$X!^3wJ6fA?n?2!XGSQ*BCX1!~Ijhc)sR?naLBHi@9#l@vjUaqP;#Hb?v zyrN<71G5f^y1qWcuV3rYaie~%t$D$)7&r%)Vl(LW-RikNW9K^jdW8f|J6zwhvQR#K zvUq1fDEjDA(?B#D3kyHM9}#mp4D=o!yhZ7vb-UwJqQ7Y3Fvo@SJ0*)q1;tb(4zuhZ zB<}WblS)Zvi-?MHOiYMJNmZgUQcyk_n7a7>SzQX1zCr`LwwVW6^Z5^@)8jRcki$d< z<)-mTE&XweP_O59O8iJVy}t+I&o@a@i@pD3rali^_jxcBSNgrE#7q(8F+Y`t`_??S zDM`+zi;?9w6Q!l0)Vy+(TnEZIHJiCPBj5M$xW{mc*s_uveo%-#bb0j#b6aCL3D|hd+!`Lc1fWQZIYmpZnzaO~e7Z;PcpJ+Y+ zLOt-q2j$|DV44_*T&95mODVZ)P-nMM7ww%cW)gOFEL$D^t@)Caga9oe`uowo<#Z)t z%Ut95I4LbHEf_H61q2vlI-*iiA^|5BsymUVjc$)ha>q$e&&|p<4OC_FjfuJM<4ehA z6vjU_ZP7C=UKAs3I99aEj7`!0%+>YD`Z{0#0IfpxRmydD8X8+@KaW1a%u_cVw%ZPkV~8(=l<6dE=&G;ERjDeDRc``MRu&&U=%?>6#@(R3Mz zYd{4&GsIp&0T0;D`MEirR%JXfOr_Vao1}gwOWxHphkhrlSd~daNr@1J_r7n0>DLxG zs+HOyxRex>pHDE0OG}T}lb?&d&np~+##=&5N(y~ydb($b4pHJjQWp$5cnP4CpI%&C zbpNIiqe99SP21esf(}g=1_$g&*7PP@!~f9*2c!mn320; z-O~5)bs@bC?ry^+H)rQL!0*1j??4uJbmXF!O7Qnb0sR?YUojrsMN`l`0$1AIj)9G> zT;@ZCe9LQ$F zaMK8)a#q-XlE4DPcaQ=p2Q*g(b1^Rr+j%s@Hc#=Vx-7)qVsQ1mf8kV~9_Tb(s7CUznzXOj@fsRrJuF9cPF50MoQ~EhlDM4RB8}}@E_d9b z!lvqOs<2CM@6A)WrKQ|rR8(NCOEcrFFJRO&$b!z8!qPZ<kb(#j~hW2xLO^Tv-K!%{N2qNb*t?Ck6P<3A|ESS5D{`-EOTb959$ znj1Gx#lv41%HOeK7QTuPH%{h7i+E2R`dlNB7I|(IXR4(o#VWP%yUZ3R@SwDdH3fe@ zsXB*Suh~_h`Xn?yNV?{uje$+x#Nehs-&MH#=lN{@(^R6>I$6p2@JH>O zn^NVKj#*bw&ysrKY8sH6Fy1IOV9aO?5wEukjX`iH%ncsJmEZ z2WTPXJrdIQ?^C-Y>M^^!O;2y%V55$f3JIhZ3RrBz=$Za_W3fHjJuTBork9-L&)6c# zqVJL~&WF8|I&JUnFk}MQYS9F`MuM1wP6KwBc<}(CM~iHG21gp|Il(d*JMmzT#)s0w z5-ZxcE(O)G9FpHMGPx3JYER(a3SmDBv)h2+f2^HaH!L)qmDF$FUYk|yS`$|v4yu|@ zll@*jp$JPzL{Tl@rODIIA9N#y?<7~fZ*e1^7BwGBHj;f|JC&LL^ymJ+kx_L1aIOWS9(%> zAni!k-oHoUD1oz!6fEuo#P2@5|Yvrf~Gah-r1QaC)c6RP6Ar2!#GIO zg`U$l8nJaXraDf8xebgOHZY`@j6%^Z@Vs<|{RQz$HHZOE|1l$@JpU||$5tTudY~W& zH2^vRAUWdg2DRvwjQaZe76SfX83?5&3StZB=zV#wqkMgl0hB_+&eU!!|JQM0GU5l{ zf*u#TwC}BUEJ!3m?mXZPLjsJbWE85IHAo2Bm3-_0GSr2%ADyD{2PWL`4#ZHgy+z&w zKt>DED{O&rkC@{^fg8~S$Bw-f75ssiujSt|UVl{c|2|}j^!}3|jt#lw=ZfdZP0QMr z&vxB%KFq1g-D*qV;(DmCZhm(maJBE7KvnH(W199M*Z+*81G8T%i{N`I#_3P2^!%=_ zsfiZLESRIR+kSQ?3+tjDIYIzHgKrluvGG~(n=Z2#2wB|4iQB3^F?t z^UXK6g35j^%;kG|us-0Zg=(i>ngmb<1y*UvG5{`QL`0h04|#cZyN3nX+xPCX+i`mz z`d)ff`5nG~4sW7ssl7(Sd#Z~tzQFWW?4=1lQ#tZUfYjrvfhYYQ#O4< zAJZ9~;*Rq?QC4S-nn7=&$(P@{;|RHe=W4(~%E5e5F&NW{Md|C@nIMyk{>=wM6nQGs zwiOhsb<&cJPNY#AAfl*p!CjG+~ z9D?5`DCck|`EwWEj>)(YFHRY&4VWPTTOb<)ePVsqh4OWy|{2sU=?^dQcN zpQU63!CYpFB{Gsx_ioEdPioK9cklVlar}!dt2k1Zh=^Egn?l9Sx{FK7S2NgXXlTu? ztqb743#o>UBoLL?Fg#2Mw|hJ6Q>|8(mxnW__u&QE*Fc2^ zmh;zFJ9MWP$!$?PGZxJqG0(>;Ia`;u$byo+gpzGMf?IA12wLdsHz_Us&|3YwUB%>n z$!TW`8j{C6yl)@fG<9|vPbe71i9MM~45_g5LP4tD??g#UGiSF84ZucBs=H}G{)(NzwrjE8@7A7Y67}~DUp3Hct>@}4W={Durwm7 zHe#HO=W}?3qI=IEDoQ#Xc3$f*xi~w=|KMB_#mB{cf9AT3+zE;R_(R#G>f%AV027d! z0EmBQ%Ws?Y(==|Kae`@ETb;20~sO;A~lc zhO4KCz_*HlKXf4zYexbdQwZQ29Q$MDT;R@sHcH+RMS~w~y)z>yK^5+TFV_7W5FD@{ zF)|ql;TTz2MFPg%fIvH+&}RdX1vNGaIedEnx+->g^jkN9miQFS4Z``;Gcqq`S?F|u zHKKbW;~~@v)HMR0&|p?+`T#Yhy`CNy9if`}S8zcNvc?O*YA{U30G-_gRKhgBaNNB6 z3qLVn|KBYVpt6CIE=kxc`G141{pEs);zY#*k30;{kZj8YxW?018wZCL5KywZm#?@K z{f07$=q4{;<);?B3m;WJK~64-;9b$@_N?cqw(X_ zEB!(weibw8+>bVvIa?2B47@fh{s*M}PQ57UFjU{*9TG2>Q9?G80 zbJaPuHpV13!U8a?xHQCiRTdf6Jc>PkPLA}o=(P``6T=P+XGjiun&t7#TbvGXDP!7C z>p{o4zNR9|XlPU!ul8E~ZfouC7Tkz5_Wt!t8BtAP9#r^vzLNOrDc79>uevLo8{^|P zY`Q+g65+Pb<)i3fI;M^3P*9)r{`k=}iBAdhTY(w-8l08o?!?lyskJ6C@|CSczi%>JZdZ8I_A>>3G^XVB zXUv!URC=zbLO15RS52hOXhWlk`8vqI-L2NyFz`P%x^X9O_F2PD!zhY7+wp}qe)5gS?1#_C;MWkmOojck9#|TPfQ#L3!;NsG*|oKm;tM%T4G8F(Q6Hi zkMV=$!Kr#Q|IDtVJLj2QQUB7;0W`q7z5JaAvhXdH^H6uk&tu5Rsax}y4-GpYCWheP z<*Yo*QH9C=k^cVR8n>NCgP}wy^O-3`4WQXwwS^=eOm=GAx%YP&l=TkZamSP#h;kdt zBI)T1MkT2E4h@k}2qi^+dW<%5ET{^llGO(lHl6)pgh3fOFgzR!F~`9N^l;yDwl*9Z zb!XRO%OrJ9)>sjmKd;Ge(;5^gOz$Yv*VjS_YeTMjr(GNH_JDWQ_x1IK0fF-el8CU` zNe_~rK!ryD(NZ^&k!=oEWDs~ECi~~G2i^3(tpIwHt^u#qe+Fe(m5GRgk`fX2(+*MT z-|W!z3ZUNw^8wpFqv{=~{4#(pgVsJw0QxMj?F|5{wbU>$HPtP}&o}5%w$$(-rsE$W z5L~y~lZf_@0S&=jgyaKXjl87aemzFN%du#inV{)|J(WL*1jZ*O$beb&`B%gxo_4mk z>!g9E6AvF>4?*mq)4oEmmQ}{ih+J7O7%2Q8^nnI+x?3e+>(qsOYx^fLP=UYNf+iW@ z0QPimkM z|I;qwWD3t13}@rP<0#&TQuX(Ak>Fx4@7i~ zfsqF{ktleg0(a{AMk=P&)$Mjy5*G{IT^Jw89{TwE9|sTbOfN3(waG0)tAqr*kNCA* zmTw;KXlIlgQ_pFiL zyUtc>f}fj3s{Sm1tg?Sshm@@aZBYeV^3-0~x_rGY!$rM~hQIa}+Dz?DT{jhx-@u}oGcCW|fXjS_QCY3bgHs`s*>+XOO z-*h^wGyPNr=hQp=v*k#%+L#p-r$^LrxIC?tj?BJhx5_gLZns@KXhyH>j;UTcQs@z( za&*eL*%LgnTi-kQN!+hCdZU7Z=k(a{mUu-2K&NJxe>+a14xh!+Ss3;;rCulaN8RoY zq%t;-5T7ZL-t^?aYjk9T675*`Ify$V&*u|mpHzzDyFXo?JOTm~^)Do5fEWbYlLV3! z4+{;Uao8qDU$l7p)*)<9%*u*CiLZoAL7}_0fA(646oZ&bKFsi9$IC>ufCd*wuz3>$ zS5NxyUzvBNin3H-(|L`DE5faGx2H1FP4M6qL5A!Ro4PtQWUP>oqXAOMXwZ@QL;WeQ zs%ioq!18kEK^L?yU}K!3kQv4ZY-(+f&I<}ZdLNFZ!d5z=qU~;ju~Tg)6RicWr|r&0 zOiL?E*44Fa3k?;{`1p~8%XPy8&Js179uy>@zXlH=hryni9#<;r^V>|R>ovK%tggk| zdpI5_sC;6~UQLV+Se*GaM4?#ZmA!@Fjen$^M^Qd`qVt{a2MDIIpppm%$)J-GfIq

    Cb)Fvi>qW)I25G>SZ{593i$efxhm`u;0b~V_`PEf4Am^TPJ=!sY5Xwv) z2PhYaF&uiyyIaIB>d?{Abr63M2<(UPETjZ{{QR)Au(KU#Kn{>(|xvh++b+z1vX6!m1&F#`H_p7hF97M6dc3CFvecC6JNe9An#W zY-(Zyz>T9M2zT){<-y_J-i^4fe^L;fUm#0F;HdlM8q5$7h0e{rD9|Mm_UzFoIu4GQ zXkrAhrG7s38oqidueCpatensGi=q{|l!PBv^;eoh1s%x373pNL!B6w_|hDTg2r^>xS9r0rsqNeab z-T(D(3Zll+Nrz9=OFd&{e~#+)84y%TN*xKHVz!HyV)>rD$Et0d^ZS-y2oJY{$~4-MtMs57e>EV z-zk_PHNWx87iYZ=!96WBy*&FjRZklLN%r$Sn#gcQeVX*v*5?|w#>sjuU2=TQH zYT7hvUfkvEy=ErN5!DgnCDx1i-@FBL_D2l&=tz6rzwzr_R%~^4w5T%aG?jiv|NPa3 zE($Nkqu{D(EyUg}2I+;c(Z=OlDguvj9J5NBw~sz8fVjZJ1Rv4j{ps_ny*0kWja4wt z1S<~0*sknT>gtf;YeOL?kD%LO@v2BW5+xIR_3fMo^*x#?W*|uf@3!HUgCPJkrgy6V zth_0YWBWNVk$>(uzbn>%p{UPjaiRq4otfKhQc|z_2uh4X*OKi6V-G@N?nODT$}B0X zDH87(1{K}m<|Kn(3Tp@ ze*`Ybl%tiWS@4^H=M5^*ItPao1EL*kQ= z0sFFtG7!ZGF&Jn=NCglTW({=T2T%PlA^3>;?b~kS@zzLcxf>vrFhGaOgrJd0TmV5B zI(d+ml65?+biVH=AO&tm8`u^pkqosAXm|;4k@l$Xd=k{SxHQPCUV#|LrL2`-Qsv?~a!XG#4U*uS5yyMeZN9;|rJfQg|`X`0so zg<3qwvK3|{p%}1nLN5=h77j3JP(fV0^y0i9rZqT5Ae@@o8kWSDjKV^vLWr=1vGt7w zHaYO|!T*uALq0~ZZAcg5Nd89?Aa(tr!$3+#ro}kLBLH-s?~)FF!>uDHctild4JtoB zf35@MbQ21;~QFha5K{N*ZlO%fPXiWFX1jo+0By6EM_yM@9bK_jYqhQ%2l;Pyc8Oe5dA?B7NcMn7YFI+ zH+kn^vGJVb-rM4=ykO7UJFK@S8JX~z|72qm2|&~ zi_7EQWF=X44j6&pC!cFrIk8l;qQV_asAw`OLj+>klLEg=#CxsP2zEO;eP@c_*wKN1 zb#)xi`x~dU)T4?mxsP~$F#XP(H@gbjv$x&cnBCng*Bu?V)#u0nCeKXfL-rQAiFDfM zWz^8pvI{%y4UA}eXcZ7JXSGZUY^cjs&7pWdXL~eRXR4^w(l-3_L|$_`Vni7ajxKQH zekQ-&bxeL76_`a$mTg3TQi{xeB;+Udur=bS9^S4Xb4;{C;KT$rx<5WNxO#_wbV;~- zn;5wbI}B{$6Y{>eFoGAXhffFqjiAtj;dAHoj?39o*a-ro-`CVRSi!wdj-Pd8b+!FHw#|3LNJe_`6X&qb+OH;{M9qe6z~=z3@BM z7YjOgg)tu0#*Rts8w%S6#lv2TC*|R(n65iefmz)@aPP-JXm7%!zgZ7W10)M#M2|?+ zJEPrOt|_R<*>F%cn7|@Wa)G*xbe@!{uVZ2Lqs0BtM@QrDeYEBAqmUEz-31=i2@)Z1 z&zBbbo$MR^YLhoUzd@CBQoq&W+IV#$GMWJ}0zJdGSJ93*w#}!6^!Ch)l-nWkfj%|| zsyC2m4(#!WY6P$&wo6S7H>A}FX0ebkuRR0@?z<;Y`{)ZQe4H50tS;TbUJ; z>EpzGv&#MLQ5{}x$v;8BrbS@Xp4QyklgrbG>ms z02N|lQr$f&Y0G)*Ny245_a$(n>EkRY@G03jHss%S1eS23iHb(V3r%Dn{-8?<-$|f@@rj6!d{dN%|!(Z?t z)}{!xn5^^>H!v82q!sq=&-xR=A?NztBNVHzkXv-lJX`c0(p&sd4S4gYX=@S-!AmMB zZN0N1Zt9H))osdQgQL;Ar!nXyPWSM`hfW2>!H|Fe+48DMnn#^jg@wte!}QT4##1k~ z;c9Dd&w6fQfo8y7p+{lbP2vO>6_7MK`%4{C($b1dpbMmg5L#LoHh@@Gfi(#3;2CP{ zwIqQLDk^0*3`$8*3p)?Ldc)`_)$2^xrj{1_Rqt#qwd))70T|i%Z>FAk% z#^5U|w-HiX8M$>6DQ*&It21`w9ezvmO>spa)Co1Sua+yT82jraJ2c6$Hd8uvM5lIB)`u-!t|IM4huRn@1-lg#q&U zTnj=Hs9@Ef?*?erh4b;zZ#}J`pF1At5P=MlV;w zBG>Nw9oTy?0q89OLX6NLjf`A6$uZMrJRk~%-lC%&1O4l{@@?8nAOyqrk}#3rt3v#- zhK9;j8=4g`OEgHGpPttD^c>o4Bd%@1`Gda*D43pwogIIByETy>rXT`Du4EMh0&IyC z*hdFmF6>wf?mQ_kUO<{bd_f9petJ?j#QiXJ02%`cGB8WCBzBBb!~_I>5D5lQuTKbQ zGZ4QMB>cnLJdS-25064f^GxmjEsN;P+55qNfzVo+_ASmD1G)}BQWx_61&<>krL{di zX0dkdccwnzI74*^focE=Tr8kH1_j2ut(V|B885oj%zsn`b*QUCWGJ zInOKw7NDnqT3L_=(jy>A|j)JF_Fke8;9)3JhDr=W_n!y`7G;+ zX=}}t#4b4>ExuI!yjp!{1O@{OzcaQ|Koe5fi`f*l4L*m&*Z#`p>?wfGF`?>SJKogT(@+FV!1l_wFH0P1l;BhRYvYz2B+|p2; z4RaLO^tVfq=f#Bm*^kt&nPf7A`NWo7{?lYoX@IQiq45}l2eZigXNsUe0KadlBxbC4GXX+bhUim1sNobBy$&mzsr1RO?UcVI zvS|qi)%|b)GX^zuvXDMBdgj-CQ~BXp6$Uw?g(jSQdftjV?4MV#3u^>%a0ChbMzum) z0NOUq$MDCe5Y`IP_UdW!$1u#w-x-pBNvxt`R(-laQhm1041tjXaodL z&7Q0-8-|vcHSM+uM+8RVaFA#{LoTv7Lsh28vmG2AW9(NPWEH-M>N%Y$GiUQ;#WOa3 zTwXDb5Jw@&tBejb;Y$;<#KN}2@{pHtk7$Oi8x0Iw8YW8Ucl*3#1Ix<3h;DVZXSJwX zX*PW3xQ`p0W}0uuV=izgj|*8Xq!8NO^0jgYR_DIDcV7t!X{86FzBQaaI$m|;XC)@5 zQ=V{d@)LzL3=DNv+a=c2VQQjQ4Tg>NVk;-)fzKQ*S=sAq47oWxs=w|v|Jm7Fj2w5q z*!vw)tZ{{kH1{)qa4+90{cBbbFWw3D6|1zq$Z07TUa3Wu_fBn>xJ>xUmN762)d#%< z0{c>at)nlc)o=+8b6ZDeTvo~8p9Hn5%VSg9!*@3zX_E|y0&k6`yh$j)3@KsFlU8!x zFjBr>{C|~zls3CMRPc8n0nG$E!((0#aH{e6M}W*8rv{XW$KnD6u;g<%1N z2X2`57YO~u>`RgN$kNius7UTtE@Y(^FGZjGpShJ+yj{FJSwwKdd>hJ2Dx==_t~gbp8j)jnpHcdUm|Ky&*;{gk zm#?&lz01Stbqz=D=7P1aZEl;iVcb9xT@=_oEDNO}lD0k$RhQ!ZfJUobQ^yMphVqrg zoun$tHaL8F+cmz;l%MC!cvh;=c=eXmU*nSewqyO)$&b}8@H(t2?yq2l znrHmME6nP!YS}4bz5#N;zX;Y^_qmF+-aV{+8y>*E{H&PX_^hck&|_rFPqF1%<1w}# z(UAb&$f**&eDZV&OALB?dzD&ndxf`aSyACHthQ23$*U6S_BUtGPQ1#}tz`XD19N8_ z)_*Q3W2H?qGlnIk&blZpRf|jsC?g?7b$L$`cd{DRoUvsymV`u8<*vG@3|)v2q$|U| z?k&9~?W({aQe-p5$P71&({~-B&(D4UYRfZ%fik)?Ka}~S+^I~#f;1Tm?XF6o?#u|= zoSMl@-z*R@O*`63Y1)DXH@6>6M$5N(o+etc?{;&n60A*@aqBO&>xM0AIY~yUE&LhC zJIE-T73y?;LaX}(;DpgrAwoin(+zEw`BCkI{_%$jZeLn%0{qqxNG;n`Y*MQ>>6VhQ z_zPO)QVZ_g7UD93%gXbsy~qv{K{bzkj@J}+J5>%pQS)mPzW%v(YBMU@9n+_W=ewsK zJkg5_Ic&9$hlT!JI;>p$RjmGNgby@|($7A4SdCM{TdQ_t{T*S>@&^yKZ5XJEz{inP~pzD z>DVfgrv5#any08%wY)c!)sFe6OG<)wug*sc^d(CtP9>boxiHw+ww;g1-w7Ojw55

    =!s~jBg*i;EU)suJoSkbC7}+-(bupt2PS1y@=d+oArUTAkm71Fu zN@2O#-?|G!I-ye?H5Q?8UDU3VpB8oKB8wz#aQg4S{%fgdOS8N4{oe#A1mYvp(rk-M zv*qe-Qc_X`p_w7-6C_5a%3n=NO&J)cMHUKF#yltk!d~2grFMvv2uyM=Oh}H)$~UIvPxRIH77#zyFLA+#bWgx%c7?6sCQ zdvg;`WW909^i3q-IKw6;9LuZ7y)air8}$n3=I0yYFETPRP=RpgM#$2|y})G*dUg=p zb=pio8W13y4YRWawejicXM!wZ-;Ot+;b0%iLPak4HRzEW_fHJp(9i%|AQhL=LAt>{ zczQyBFBT@r9QRX3Z((iCBx58BO+iP81|+cQQgN}dV$5;B@3|U?0PDiY$f&af!+qip zPCrbc>;_ZNNIfOKCyrdG8U`S+1VlaD+uVHR)Dr<$smoN2TO8X6Iuh*h^DZj7x@&+Z z<|_AEczVD_XJ%41=wXKeJB5HVNFuve4rIzg zp6O^j4x@hTu|K@6&Tj%5Q!a^k>aXt_Tk(e_twpxvM~*tnjQ8(g^;6mH)h4#>+sth} zHtlNhit#?RVjtSSF*je9sW-*G*o+g^vDD6^tU!wCnPb%YR7M!wkI=g1*`$tVV89k*j>lI8rz|85zn*Uefwe-yx?cuk!t> zx-y-Tcri9WZ$5OfzGQ*qjhFs&@?Q7ofFwUfDRK7PWgY=8XkYb4VieTs;=CnP{&UmAfPbnI*b(e57}(Ek0HKODHH_*d+xWKRYDr1LDFr`=DSNx_VrO6tfFBjoSlC! zQ3`4a#erocP35jf$s;<6BBQHgWh}(NeVe&oPL#N*7On6c@AM6m@=zj?k@4+8PXUYr z3nP>HNL3BE26Tq9sCNfn62n?cw~Fj+NB1)mvfeL`mE1tRhCkg`SCZ?N{ids;>O?TB z284Y$!i-iTWrg!PO{j}LWC-~o}xD5u8{R0l@~N)1s@#~tOXj3VRnh8e9B8R1r%=yuSVGL&udQ~^t55X=s60OE=&5Wkk=Au2G$+$5Nf#K(pu}dSmU%j8c*1UC!YK<_WU8s|7 zJ8qNx{!QaVr|Q9$%uV--zuX!#4&-9BE0@-u`-1S__Y$@pa#ipP394qK>Vv(ek?Ir< zT7oAQh&IPuiTocb36|=Pk`r6+t(p%Jac(d%#B|#5f5X%^=kp#){VO;!NiIOX(!@aL zZ4)UIAkEu&k+i51(f?M9>EqYkfPMaxNL$}@i$3q)fpUjNX**hjb$yENMPw;f0udni zi!vE2-M+TG*0j3?@7{IvXMTp8yTpW9d8k&GOSn#KRm_`=LJj@+-CDd1-Vr#j6o#Bk= zq0YEqcZ)0m9g{hl=M^v7moMwY#Ylh%GHd5GAFjFWwUP%%nQgpzH!$(&-+!0YX>RY8 zJEuI1s5sf0_~hFW;{>8VvBeGc&lJFxLvZKjGe~6=nikRR+*yYMrq|$uT<-AlKq$12 zcoEocHr+-R--Rl&EK}17HkW-C=Y?R7@}Gt&$oApU7mL&?Y?f29Q?JKYe`bS&XRJ_o zF*Q=LzyGUS4OAg;{4_v+v#oTF7P`N$97bMt_xkEY=BdfJ70w$5xkt!C2(ZW^K`ycj z01tXbDc*;d!FQkYNuOwWg^iPKBy~(r6JZeDdNf$&IL($t3Hw+4RdcH6=Yj1aEq1rU z%-AW*M`+tudNM(j_-%qGHhX`ctNP6JCd6%1w5@+jtI_o1wL4y%cFH#qtX*2=28B`6 z>F<;eXB7?!!@~|bgK66Tj=ff%2gN!ZyQ|Z-d(Exe;`*t#;EvR-P6l@g*ya8F{cj*e zP?w48(E$qgg)cv@eObiaoz$pzD)3~s&L|1T@WHHCO8RLIfLl3ghDnLYXdyE>j|Go9 z0usW))S8myFt4gRF8MZIEu|zJqlq6@pBnhx5=> zNc}|BWdQ5GCHSFD8ys0Q%5`>m9wzoECr*jckZ9-=fI*m8jP}5_4q%zjYV5^XsHAXf z4U#l}#AUbM{p(&*pPjwh3(i{T8|a@v2w-&prZc!qaD(GVaV5k0J+5iM8#gcz{m&cc zY#u654r<|{vwYC@as6}Q$A_pgwT>4Vle&QW5IP#fRqV10>2Mr_TwGj&sB$z0jNs!I zGkIv&8wtW0_%L4}?0p%jm3g2C|LtHeCRO(kZc<2~8)^*!8B8FXzH4%GLcFB1G7<1R z2jM`xkiZ@pfRvzBus5-=cn)quda&D)flb6CPn|h#5pH)L=UtjK#TAZlAt52>{SJok zeIdjx?REE#Kab-k34J7Zu_0}V%vT;ytUDhhJqR2Bzh1%4#-;&6^@71SG|2S|eXZpG zQyXUWIQ)xgVE+h`YrtRTAQn*s(t=yr!JFv}Ihw4H=S#Jd|EX&^C^E7QypF?K;O?i& zklEn<`p=w=ig-LOU^Verg&hN^dpKkF_V*hh0|T*}<<_lRhjthU7^4T(b*ldy&{@yR zw)>l28~AG3oNat&R`_!lDr;kfM@-^h0$R$Mx^i0D9%Iw#Ny zEqq^W;)zid?~7)|P4}&v*k0CHScZaaFD)wD+CEM5k}axXKL;b(={^oa;dSoItWLAR zx236XgNEq;!T_36G*!W3HUG1RLE)nsRz>V7D~Q-W8;9)e3`<5i)HcsKa;xm^wWKmw zTkKE<@jY#`&0&7m7?EoXCGySU#^u&jT-oko=&FLscNZ!(TJ5l(ZreKN`V0fg`Cjgs zjpRzFI}@q!I+(#yCyoqwV99dF`Rm%sK#GpqP?V2w1Tj!^XsSQTtI~m%qM`zV@ zbH$olwnUYcYv>}IboED=gkPm>BgAM@_t>YZ+9ai=Iq~s3OHd!YF*ITugvd>06 z;lWE;ZkvuT8A&9<6f1h~gI-fW445n7_rk3aJ~;p@)XSHnd zzNGFd0SyfH9_O*_>kGEyVGi(j3GrWy)`!5i4=O`fb1%0l^dh6JXQtWg)_(9mW0|p9 z?Z0QIIK1xCpcXl4gU0aranYA~!b-Qlj}gYY;dX+$UUzf`pFk*c?Lm4IRq? zH53H^yPqg)Xz{sfNDw~wLR|DsjYky~hw=6^j~ezM2Y}(>$2SDw_0H`v$*Ycqflezr zWtNi$wkx3!F$=s;OFNpg9&;_(@F!ldIkgsgpLaP10M+hsDRK&nn4TVQ1l1Gy&2!-^ z9zy84^asA{d~w(wi}ue<5$Zzc+uE^;Yzg(nBU(X_VqiUh=lsTt7t4%{@3srST$V)s zex61jbS%2q)G zT(e3-H6FDk3p0FsRE9@H!9{0uMoZI4sq855RB12TGaEgeP_yC3JOrqLB8bW(#+)sW2xm&EJNtU3gF?|HO9xm#=RXmYP`=YU9rX2koTt{M>JM ztFBq_5QbLkafBD^O2HC-#R1iUkZ!yCAvHtKDC zofR)0IqtSqst7+DwwkRv+cEj!Mf0#+!M}N@h$gN>chMj?%SCh--N8rWfuxew-Cw&c8159#`P&rCJ&H8edvZRXlf##m z_6As3LBjOY*_IZy0Tq}Hl$Rzai|XeY`#|D>(1SJ@73t>c&f2ZN!Bfr?$o^xDBw}Ao zuAGDJpOS){kZ?;1js*a@AvAk1y4(S_7t)+ddv^+|dJ^S8n~@hP3_3*Jf$ub@r(@r0 zG{X)L)y5{+Cyfx?+=~~4r*pmrz$TrZP3SLo%KHUYz3(c4+X&flt5f0dH8=wyJ7Uk& zqr|%z$U5jggf4dZ;7UP=Z7elOz((~_Uuc6vkhwij#I!FO4uES{{|%4|H**E*gkG=)m)Q4 ze*6~~y|UK<^vTvvqwc-y3wpIzKFoB{m^i}jm{59sTGsjUlYn1LPA&x!e(vn((XKO* zpo+l*Amu`?+aivT;(ltYXTI7~4&z4fh9yexh6>ZJNQyX*d4KCihQA5ZU0&s;<^4pSY^&7Jd>709lrhuRSe%Ky(xzI`uTySu<^tG9pm4U}@h-cfj zb}I7*niC7_W?F_as3gQd#9Nz3SuSj z%E5zs<#@_&6=u0n9*9!hRxP9>0KYUuA2$CzNC&aZ8@8GK0m+A8dTC6#YSv(>qSk(& z9GO~MTa!ZZ4TM!|zfTsnHJ5pzry?B5QLvOOvl@k1Bn{xyk8TwGT4%GEpwg)Lg$=H* zFcC8l@gb6d5I!~M6g+wgJmS^BAaoVr9Kp-}t-2ZubX`)ic{h>Tm+OH^fHz4=Ks+0uHaA`r$V#a;vb?Of+8QHK$@+YOf zliH|cz`L3T2l2(e*(M2t3h=>mIs>HFf-JDJ#v07XkVoU&-q3IZ3@xDSoVlx{rgle8 zZZWb9Pmc+a#Za7*LM>Au6LcdfDT%}ca5A7v$Hp+H%zhi_fPzm-R@MlS=J78M1d$-4 z_Xuo_&+Y6u=&fd|T?A!kWYO8EhAsLB2Aum6@z_;C75)R|mW{nVTt$x=qv%OLNWc7i zH)_z}1h`K{1qE+dJs|eTuonyVmIXpKX8aE&aq55mL2>b0)4u-_ix3$Lsa8;5Ai^q4 zX!J3N$AK}0#N@0R=#c{;p4Yc)Yj)NApLg&-s`7Kg4_FJ4>6lMU+k63eJpg0{!4_Jv zJl&h;2ww~&zHArL_#M6gZ7A$z&Wf!A<#!3MIs%jv3%CQEBdpq8gtR`>#d{|eRaN>x zwnNUwU{x;&o!~%jnC3&UB05OF|GZ8_ye>Op%O5>0?F~q3RqHCRsL%y-5k!?`8so#y z#N_e4Z~4dST2^#oPwaVuMML9mhY)(ikzWZiPOih3_L?^l` z(9@NdO3#~pJ6|&EZezw-V@n%0$kIygKWXPv`5tutvqRo3qypW^S0!f}?o|t1_N8|S zZHUGxh>`?2`(7{4S^jiV=j6Rs;@iIW>G2?!Y-~{c3R>iRZ-LF6=%eV=)aUOL8jU_? zjDCM{KiSHwpm_OVZQP2fkimz%quvQ`G_xm#Kbb7fJ-4bVLL&2%PqChGcAioI7YR3A zfY(Rz&7@pfq89^-J(6#gDW!j#(Z;OLJm_^#4rt!tAT%LL^zRSWLjN*{WhP0bX2x&L z@Pp!$Q{@8x3qQrpzjgNtN^hjD-dZ2FEEEYz(XM(APy}3i=qW3Kkn^C*uj(_~A3S;6 zFsob>2PsP0fM@rZ@HE`f*O%N5n}MI$tmTAx9oKg@D=GBE)6;1rq9&5w(a;H-**&qL z9V6nXdADPxz^+untq2KtnQwEhgxMGg1v~k1G;t#+rb*!=RF7!YZucGG}0f|Lv z!Q3y}&8V;NtGYRnD}2yu3L@qwc^CRWp=4pt@Jx0tPhto~<8SC}Pa$p2)R0&{qFg(f zjcjTSXRtZ5zb$ksa_M8Gz4cFW3Y}gsg9J|bSz@C9Soz*PSlucH;U4$!N#Qoa1FHv! z7;-dtDPkBli0w~m8Bzp%b8=jtyRN^X23f6$8SYFh5v}8Pb@=)^c6dIrX2Z)_ce|Sq zwJ<;?9s>$2z~gtPoit~=pyM1X-9nvlG0q-wRaf^{tvcqRFF1ZJ_gr_fbj{jvH@P-y%MD4{AzY=r|qXYYlC2~GY5@I8=!&K z=f_5vT2k?OM)@&FfW1HiBln4J966cCDy6}DFGoq##~;-ZK>==iQ& z*hr-yruf5$!%)bu!$9UU$+C04mT3Zg>R7D@bE$dokjuA%(o)Ju0o$9Dl$yRY$HxK? zBkP`{PMoKFjNnH1yNC(SOE1su3JqSc0wJ2txVWev>ErVx#9n8PJLG-wKgOyq*F)HO z7K5dkp0h*(L8#+XdkNf)c;qVujH);U0Y!$B%Y9FM<^#rEfxTvvB;mT!wiBevS_UL% z1liQ3{on@*#ZDd&^(AJAtL||Sh}K#eQo5V%`daE6eJZBhc3S3ZPV(jfi@EXV4&POu zmsXp)mCSab%<{6uDVWUQM$?_Gy&uob618{M#S+=H_Pa&3_5xk}dmh^4^ZFbi#&r1oQ_5b%S4#taQ-A(h>8`bSP$?yA6UOhyBa3KJS72!;(V)w>u)+wGb2Yp45J3BGQvN7|SOybP$ zJVtu&i;QD78t`xs9xo!6nxP{trjfRg3#XaS??*;3!d$<97emCEm7HyC@FUzZw9)pO z@w!EC!QjFRp56e3Z8&%I(B3KM@`V5(kEGdl?T_IL!4-YnMsd&(!YB!y2;}kUF{u+} z7gYnoIbCzU%?UQR>MK_!p;H>!EsZ5BuS7qW1J9ORbxnQT^OMyD1PKVKHb<&XUK~eG zNB@hh_l(Da|HHqDP=uDf8=Ea0{QSzc?BLlMtI?|8qk?8zA;C*RMWu7+5X1Xo+ zS<1Hm^~twycfzL2s|A|P4Pvq}htv<`T%LO%dTE0J7-WRp(OEeg4e+iu?#2ZeC;|$$DV}yE~`^NxLxMFmTw{3M7~o!YcOs+<|b+ibND?RW?C(nqaHg zQ0edhZZQ=`lt|=k0c`D*A-2Ak5#1rr(8?U{bug3K9owWa)WU$_b?FwVT@O$4BKjNT zu5q)J6szm<*ZVc+@2Jo*7EEX;E7)ag-?;0;iau^x602J*{te_gAW*dHAM1-tW@{*W>yv!H431AG6CZ}&MI79WclUb*o+}l!)OvW1joHnE z>IbB=V3o5gEvod*WZ&p;=uK*ysg)yNhWK$`kS9LuJKt-w623e!Z?T*n^#%i7CWL$dCa1RAJnwviYpXJ+DlAGU?-~U*~TmXcFefjP)o0jps&crv60Caw?)lQZP~ zYxJ_LtX->fOego<`d}$dCFmF!3Hl7Q2~kr=$5JaOqEU_^G8N4SB}MD9Q{vz_+YP$>a4-!(2g>yia-X{YW4B2Hx$s)K&9u=HB=D8>F6tN+k^Kkx zF~i0Hhd?--aNl>2h>lXDyVr=jBqW>=hvoxlI{*;G1K^0*8qLfDKnBS8a%xS%%>but zqowE5`(?MqpDO|D6cC?jyv||}M&a)K9=*wwPzT{Z%qpQk{0uG7(a|x6CYTJ8hh|2x zU_$mnZt(%FD!aA}n0P=t%~!HUUzFP$fMpU6L0nUH2pvmuGc&WUM1k8tvdi$mNB7Q} ze)KkPp~mP=r-gW1!ColgQiRegZJKO(Mfh$g@Fk!_mebMZ<0a zHLVMwUO$%a|5^6U*#7GMUqejBXvk+;u<*)>5gQo@FJ=|=fH%+pSyM1ALNdB!%mggpQ2w4X|%oKW3RS2 zny-4^?`SVjAJbZ`#s*tp_2f$ilnejGoQ2qzaWMuRa z8QPdtT6*;5wU7`E2#7I9^3`UrPY*}QiGzQD7#@~IcSv}d2fWV)u|4;@J^6(!iQX>k z?D&L>C4b_v-Mb8U& z8pkOq1n*+9W74s7T^-CY7i$~bitL`-;kE)%}{*1@}7tm^b9>xo}5YTi;z+j=6nj-F{Ix zBBD@{_4z-<&71jw;Rb1Vo7#X7P!dxZ)GVvcc!;U*xp&1U|Mjkcw}^oy#v4@uMPtUBGo#UTk3JqW^l`68f z+y^lw9ivR;&N+DP;Qn1y*!P628Eh&&LaI>IIuc4jTKNsnE-zUAAubq2Au)D<0>f?o zH(kwT)w`!`4}MU9IG>{PO2bIr>K&vrZ^Fl>fSAP=(zd_nU5jlV^9{bsyBg$l*k`u1 zKZznEk@Br8r0f2qF6B}Pdg5$_eOlA+77SIHRPkv9m>q*m5U= z#M&*#=-@=sK$**wciuCv{gt{%@maHHHQHGlMHNHa@6zokbuH3k^IWS-}H@}a()5aj@1XQRSuJ-o?Uq)qr?yo9%>hH+~J z`nj;PS4rs_ZE7NE3(J#qt$61-2gh;H1%FeIep_a#0i2_+MB->6bIq|nRqn3?w`2q_ zPKqa|YUJg7fK@1B44*X0skONRG=UGfM$FMHkzmq*xhA5YhYP9%G`eh*WVZ1hmL9zmzI`&p=mdslsk~)l8i%-x2Cd$*&HgZ zat#Vj6K6GXHjXDrH%0^QaoCHHuoLzR@UE_y`uX9;82){=Ty{QgR{krRvDLx*;%v8x zU-&q2g|Zc-$j$IYQA?L%cyxI-+DXwFKe07=cyfufl5$^=WTz6YkTRrSZ_6UD~K?ex|EonE)Hc4&%LWY0}?UA`N5YX zTr!+&a6&mc%iaM{xCgXWuUEPdnw9^UKnf`X7dh}T!OFcI-WNcKpbv|HRVI>^3w0sj zjjbC>dTCpx*-ivWCo!=ia!FFfWkjf zqFZQ+wKgd4u9n<~^LrRL{LT*6Luw$n9`a}3YKe0(Gq3PhE067$Dgo&5@vCFEY^3CX zNA@T#_?~(6Y43j^Ao$Ng9S{%@{rN*|m_@|kz`)D)Wh|tq_`kXA++6IGUNeyFB^f4!WU_Qo$sLC&CZt zn=4)+*TTYD6Pn&f_$mU+2U3pS+S;1^M-}$+icgsY#&S+(rq8Qv&*+b!*_)l64S`jr zFdp7u6OYu5;u~_3CI@(Y$khlD280+iPy>|$^jPw6;;*|r zvkdKhdlefS7;-5s5aH0>$0^>f%nnRgeK28G--;9#m2eU@q4d;I@N85t0C)@q_iGK^+zJhl7Q8&?`-xFJKa-5^)C; zv2EMdWc=z;S$TOIr19)R4Bx%SkAt=$9t#2kZX8s6!vHZi_f$%B*ZZc{mEM)#w!acu zi#dYd7oGQh!EkK3*!Z)Kn#QjGwlR)_Z=*4-g8!a7dRfb%@NZrgnSk8s=5QtZU5_o6rRHIdugg*?rh)Uq!LSHH2Sl6%v+8xr$+2~&n` z$1f^j_#_4MnA(179g^WT@I!6YX9YP$b{ozUp%Q{qi{e*psp^feqiZ-+BDbw>E&%k4 zqBHw*=MrFETT~6$8=kNhU95--;b8W6#Nx023_i%7yt7c>t(+TxvD+?YMklQ35Q6Oc zG=pxvFb*cA@l~_$qLg&0{R^|=H)?x3!UC@4G!a)DRfsCk65}-ep9}pHW2^=c>-axU z-~4rnoGp26**RDhQOZLKYqQR^(D9;s3Fy7hyM~co(-Lrff*ulAs&8x?tvou}tF}9M z+w1$e3imhE)4A0YQu7;8dF&D~C~uU~X{w=u0xpDy;9h})!#q$1@2&$ou?FR(yF)R4 zBe>=F<%Jv+$;F2GLNKowc*Xe!2GY*0ornRY7K^W(eQx>i!R5ivU}&uTI!(aF`nI{P zV7RjGFVO3#q~_c(Kortnn{RorM%!S7ARyT=E*wv3Xq@+kzMK(FGy|&#cHiB#J+Mc| z^2i2hh37x?7^Pu_08subNzZyQ>3B!J6nS}{npCyEn?=NYjE>H4;I`tGD{x|` zYq$w(Fl?!_Nn0*pFgdbd*6c|$Wam^r%!LR}Ch6DxmwiCKA=MAmNFTQ%nQ^x)e&z&c zC1x*G_+acRQT{(yV>ExQ(K+wbVO3JW8N31ZTT(^~f@71%h@?#=bMp=#s>a5!!m<$& zA5#{zN_-`5jRpc-$`@k{qI{JN5Kf8u8~tV}-746stgO~8w)Nz;K5Ji;qWHv4k&}zF zsO{5=3ZoV~7G1K;=a!SaYHqG$b6e#eb%uI$5g==ca>~~s^#_uWyY99h(Zbxqz}={9 zI6)=}9pcZOH{VXRSV&rh z(-Q-0|Jh)r-`c8Apikkm%4Ro?5ZaObG1K{XV1}ik$hza&vX&Kzso3!TW#6mpe-tWi zGN(d>DVu5gE>%i>Y^LdX)Q4i1T0&2ded1rMe#Kn|JkKM=+15EfbKTATTJHdg+1knB z+bMV&(pAcxUJ7cBhaMJg*|RAbXZRtysHY`Mp`QfHU&!gO(YRaJFtFb3wA=Ww;*fhS zC#tRHe97Z&zPe)U}EA@|RG^`WQUlj6-I>Vp^UqL(wH&eavooA;x?(#^ z8$5Ot)Y5IIv!xUis>2=x7Jc|wId?efL|cHa`0+gV?yLjC^zLH8#S14ZOXN{y;ZT45 zDr{kwV;uSsJsGxjf7BE3ia>$|p1?V0%I_@f>>BtX<9d$2glOC($(i#vjp#Tz zQHMCfYytYimLGC=gM?*d=qCtbFV)esRgh6Jl~>z41B`JgireD=A%k^fwm{D&C`bqqqj*mQh0bnEB)9Lftdf>Q4^hzoQzH-5O+^^ zSKOTcc8aMXF8}DslZeRewY8^u(+yNj50(Hb5^@0cgrPIh4-v`0wc{9o)rhG%F1av7 zHmLxHv^{~}p5dxE1F$|aGBfeumxR=ejcHO8&@~f4@y4c+Fige&%RZr|R;Z_h(pO^n z_U%(?9EZAK_#627*OEQ5ehRv5j`?=Tt_W+R;RdztJZQcAC=1;22rNL^F9Dpm|IV2f zy{y0klz|EkJV2mC5q}V0nSulnd_b7A8i$ME$jHcWl|~SAz5>HFm6#XR-Me=~JjKMs zShR{i^mzYw7fc6q{C!47@TiJPXluu}az1dR+k}%mj7B7zw+#ng zd+^>6-@ZK<5RL9Lnx`O3%MQ}?JmEJ$kisbcP8uM;R(eyyWAxpxL3#HBR8r*^_=_TZ zjZ)ig+h=b}&Cc^i$;Hp*x0T=t<+W+k8Vi!J-5_Ul%& zqD0Ql6Ii&EMr~oBMAl_XxcPiM7HD+(-XbJF74xJWe|dy2A8)ECo?MI>r-A!BRDWks z1=-RKO-(qHWeTGk zA|X-?k@Jx|gPIysi2v^0xAu_(OMQh#@Vpxc9Za@R*&neXjZkb*LJ%*jy`>bF=_d5Otqmn0vCcV}r4n?i)aUB=lm!%#N=U!Cn z%*U@8`JE;Umi4p13|!J<@`2juJ}N#Zx?r6PH*621d{PuO^6APh&%t7qWuCEqpXyM- z>Qzz)d!l!eePhY&!u|?}zQ)JKFJ4x*zU#;Ldu`r%6{Fqc-IQ>2fvdux*GctG)CG#e zcnY+UmlqPS{e(5w%7giPMJe+}KUSpX zR7{dy+gfMsQM}iMF-$;W=g$Nlre=9t8K+HY{LE07VBzhtH!U1=F&IjE)H+GfvdR@6 zr>=f>Y@VAJw`$>8*Un^pMPAI*vcn+*PSV&{AGJ}qI9ETD`qCN03}k@p=VC`=&ub?J zNc~QcTkL}wr@xDJ>O~&>(3UYbVgV+Hb^*un&y+CrbN0w~JJ}`y-z;je{LL@&@k<|z z+z)o{-J5=bs47M|{Cjf~%uQ*$kGjA}51VJ_}7Xs9_opJxpUK#+pxv@vFGSsDB`k8z0;1_{16;-a2qmLk3F*NiA zq|}ptz{eTB@?+yW#q;OrbSG;Vf%^VHI1_^Ass@~h3Mxbf?L+}Z#Py2?3n%j@}_nRQthl#k24xRRZ&TpZ)stoxgoW%DZ zJm6T#1ePXfsWzMt%gf7fW(&}+Z|Uf`4m8EzYyJMcq-ZiM!F_ago!kDyd+d(>OZE+eaU<9d5&hr0XYnglUMiP2d zeN)K9!qTi^fcOYvnWs#5vTGsl>;u5)jDX&5=dvRpt?mLTE$Bq)4GjPq2%qmZ1%`xd zs!pOSrjaN$H8sD>HWkPgQ1rk|;Ea>c3;FTmfoXUA#~1@Ia?s#saNLEz$FJjQIAy#` z>E}qAdhz<$$g}3?E&J@5)RrFq#`Qj1vK6nsX`_zjTJU{SqlQ(mERE{#di zq2mC+m-E9E{A|3veUBe~i_xE(5+hps=&)&msF!X|*&3$i()>J2n&dbX6< zK+}0DQ^hgh)N&yfuBxKV(|pB5SP)U(zsD@SrWQ78P0ik&$&+jE*Zg#r2wlNH5I9?9 zz4A|(7F;;PHmbF4ymHZbv;-ms>D?m`RlINGnrS*jXKM(l0DC z#c4g*D{6zSTYJYu2vobfr|`{}Y|#hPOE1$!x|bWWP{S55%W{k!pXV{4(M z(Y@SoQ`5z+Qjc%2bGaduvg@{usKJK=$K_Ay^iCeGV+?dp$*FX8ly`d+Ig;I05*aR_ zLivADMd8%B&3|;2esT)y?p@U7EVtiAQE{>Ic?K{5K~Zds{g#d0w(0M18_)}TLlA#o ztEgvCaql0M4XO-OWXP`zj6H6cIu4;+HS9YQu>a1fnNxIS%%E2L&sYIv@r9$Ul0yk` zWo4)cTN>&(eaZ!NopikFsD{V>UP`bRTx>n0dQOJn=^3^0%AsTh!)Mxi5d?elJ3FaG ziFIejv}Sl&XFLbbdc9&x7f>TrgwzkdCe~OP?pRD43kZ>}6rndF(Vycdu`sUO!&!7* zYcU&M9xHou=RPxg;)!I1(Y)N^nXawvr;oSNUGU%7u3Dgt?e471H(!vO6KbF?d1QXD zvysS_M!ot+wN2n5;%6QnyO4Hb0BoPf1*;c%bs08fStkY$zZ&nL9{A3tyP+O_ZMU1- z>U^7Jqv4TaT$il=r5pXT22$f45MU2I>es9=mi>?^ap8ohQ+1#a9sjv1 zzE%~>?fLnirlwo@v&GY|KW>mh?Ygs`(02U}!g~0^&fcC2Tmg61S#=2NKe@p^$R2f+)a-HVI)dbNkD1=Z7|4kQQ3ouh-+*C;O}nl@Qzvnnys8uH10-DBEhMP97+ zD(r>2+8%s22_LOb&R%U8YXyY`R62iPj^qp({2<}hLg_JV6E2peWa$C$O+4SK z-3F$tv^HIzmnb0QB!GUVGO3}hb-07jme7NIntRXFp5OlsV^7x@1Du|noSZC-l!%|=sjRN8wLjflV6vE3kEL`d_h}w5GBQFoSAb^bTaHVPB z9{_AehQ?{L4)Grdq~Rf7Uhev&qSm$vNuHWeAJM6k&O2q*)iBjA-Q0CWB&@|9g#Wih z?d?_C(pQIfF};3xWQ2j4IXIFJ_@QIthc;miLN42|9P?K13U)|IOS^|ocN4S)i7DsW zwU6Sk*9<{p(2F4FwzULymHQC@fRmN|4cz(Ey~_?L)WZ8+T)T&dTd3BX;{NH?E;|gO zqJU0c+}lOxRgRP$7crvmnBq8}TU({kw90Ddt(P8(ch)E<<3K~4?0xAbDJe-NEDbAI zySWBTI7cT$+Cr;=mX z_x~+SiJ(Dobi_9mtD3x~r^ImY2EJf!XBhQ(h@@_X{r=4gZ~GpBqRYYn*^J!fMy#}4o%0|OQj5J0wKN)N>LB|#&8OMJrF z+S>Vfw!%`Lo`(9TbdA|+GWLI86*1U?juL$;l22T_3K7n~%7Y{0v;y?1x95;xTlm^x zzpeYo$k+va1P+;-^Ped4JH;?H7_EovQ(m;o4h?NvIOzghYEvQ646e&HN7@WFvo;Mb zv#foZO42y{_uYn=E*RNBo-?;;^${>`7|Qrj$q;Oo^{VogBauxu(5@*#JAbKPzB1SB zjY8gqdodOU8d-dHcp;>9YJhUTX5gKdh{5or4-;-wukdN|!~4^wm6Z!WMlycw01j;c z5HSEFI7m;@8Ii0XTv1WW?`(AymO+~vGHZO(s<>ncpN2yt2>u+?;GRU@;7uQGTi|EIW|q z!>97AZU(@q?~*KRM}_T%FHVd?j19pYOh6wH=C_TZ8m$&Ex%VTpX42=ZSRL#q#7==u zN0iv^gf)LXB7M!er<5iukEpm6oXAzodY`rHYzRhxi1@)ePv)byIQ1_uRI15demc7O zr;d;_*Ufr|hWYh!k+|PmA2=P$9*d2RwS05G(SW6FSuV*kj6^(O zFxAR8>7DicnT(Xc*rn=W(a{*VQs3Vy;2l@-kieV~U*+p09!4e@=gh*XnsOP_h-p^w zk%qi$OTo&Y)%`#-tm5|5lGe<;^JC$pSKom8K8MRu^xLUA<)G;Jd@o&)LyOM35&%L# z8fc_rf?D5dwS3-h9NW!T%X3GSQoKSvHSF6N&&$n}cs~;F%I0@i5M9H~gR1pqW8;r{8UN4nGN=8yMC!;u2pw3$%Ht&^EHFs{B7Ogw!2v z{b^TYIhu&Tv;X?cQ4Cre%w+QLy$e<*$gC7$vvV=ahD|0A7Y-gs!Tk0?+_ZjhVq%vA zS3zFHbdq*eK7lv1xq3$*3j^=tOSH!fJ(e*u-jUq}a7WNVm6PGaS!e&>aLWX5Ga%{t z-`xk5y%UIVr7d%ax3;Q!FCg?t+7{oixbQVSgLr)rJMiqmNzvl9w>_PV3~h~y8Et+p zBeOv}$M-m>l6rZW$FTAJTfkagg^h48>V;ZWh zx3M7siZMTO`4Y@Fbvp^1EtKBK$@_Vb!nqGAufaEIE=BIwhnsmGX;ED6pdJ#$z9#6KUGnnt{dL)BpCAFu-cOp)-3s_vdkC48 z(g)!cnuXuqp1iYMSZt)PaqBwIEDDN_MCjAhnF94`#?N*3c6LvJZ44CDyZ=RO1_g1V z`6Sse5R#h%^f`X%L#)Enrx*~;c6S{Xj}!&;SUqS6zB!;pPfkuCu;c@QO;}jhA0A+e z03MW+4EzXBAFJu_qra=*;MWZm8jrqp1nz<$ZIVuoi6Mm<*f<&I=0BfmZ!sg6|F)bI zkQV}Q6u(}xnQ&3aygWf76VTb<;T;{arJw#`T{#mSe@y=&^Rw)av3hT@j?PZICGt6J>w%NvAu;m4h_PUzhXn{c+bT`BQYHMoH z%$Tw9tloJE^kl*dt4?6%TW(KP+Bwry?caujDnypg^?j50@4na#E)uzDmd3UIwEvwZ z)%5g2B?1VumJ75?Wvy*U(DO}1<8xS#LYN8K86uoAqz6cq6@a7Q2F!FDu)!@8l7W|2 zSGoQPP!Xa7!{HR9KP-o?!}~Y?|IGFmFgfW+#+8+IrgBea2cq%I8|uWE^$apRaV9%`V!J?%RY+?ZT6U&b>i(;rFD&oJ z4X&dMQ4aIaysVd%p%X+UXvDjIr+ps3-BFo@G0~pL!(0mAUyE0R$W&JAYd!{xnb5=O zM!eO7-Yyq+%8v^7F!3W6;{Rx;#_Z$cV|^b@O&{VD5MTg!#SO3S*RQMKhJ8E$6jS4+ z=S}a+;!C^$GfCxnVn{{f{@e{~VQ=vx1=d48ZYJD`O5L&<-?&4-lm?ikhy>(l?2y;w zK21Hv&i3%AtqF0#1c)m8+)$R0)EP)1=6k)9+nE-Z-uSK$Uce597UWWG3oQWvQS2MK z-vC88GO1RJl_^bhVp3|XT0@|yte`*yir?Rhfd1c;lZ0^U3WfR9Xlr7T->n!A11L=- z50#EuzbIwn4*b1DU_E_3S6=JoJ#4L4x*f-LOrB?hQQ=lXfjaF~zyQQME-YWSoGxnj z&_ebAT78Xlgy$wPZ9U1%OskG9E6W%eS1XorE3ot1q#>RG3;bappgXu1o+zokU84@!&&+!Hy##m7myVVzU*5FninE^_1Vd!>A8-ua4Y zMm1$!xSVs`WV6u`(6(2w?vE4=vdoTza-+a-44;?(A_a{BNLc|Acp1eGGGmSeqPy2@;u%~osu0Pz&ux+Fy=-K(m z1R>h58fKBj_~tz+UWNOEN3&fUX6V~ zPwXN~mhwX90_&r-fmbi@&BU6a&Qr;X-)8B|j~ys5FtTL+p&78oVk}=fc7c2sbN~n^ zpR_bD#4RPFPF{#qr?mje^4m-a5s&cd71)pt5$Xu&OxgYo>w zJKYLpe|*v^n)d*Sp#zLBvxdLX#hD$yz>Te^g=fjLhtc#P*eNbCK9az`a)IPdddx?k z8K$77UIvV$aCJxZt^5H)gBCt(b%3{zh)2lc=Y^^ znH%SI+1ZdUeFJib97h`o3^$!jm^QX5qja12V;>c;lLsr57o~=UHI~*AzDiC`4gt0( z=+E)NYln^<#as7NNs|V^vRr1Kntr*gynJ(}N?L8P-C3~~fxV0=PC)~rH1284v8lTq1k52Qep3;8;*`MJ5dUyl{~a(Kz% zVJxkza1PvqK${_=^xH;?*{fIo&vowP{;g{e^hxFhGby(Uno-UHAuhz%*+jvlUWJLpper}=JxH|H*{bZB-1}fJz_Nx722Gi$F#Iu>TOhL`7*lMGcI}AtnKQf_PqDUbSAdpzsE#V)nFO`@9``o}}aFuNZYgulUkYFYez0 z9Xa~nO&duJ*FkV8qSIaG*Ve{*SIcSlSf9sW6_;eJ%eXCK3)X?ku06EF`FCwmzBwU39{)d)Rqs;~4 ziW9<%RsJeenKPTZbR2VTNvTd!kH3hA^{U1HDPil@R62Sb<_bSbkh~VJS(FiF<|rl? z7FT;JMU;|T?-yv4K?e<1!YirFHfGRNOLh}Ux+c=HRQOru2l>{~qLtZr{gbYTa^(sUMKXFoN>h)$4g%#rL)TG|Bk+aSx)-NAEd9VTB(ze`IC|3YZ!n2 z921k4<7yOS*B1c&0i4W0NkGR*FZ6cEo=#c{dHr9bV-D4wdkPOCC62Wz>)T-Pd;B2%K|?Z+oJ&*8eWjs z2G6}7EQKv55nAXXRrT%LZP?>>`XoyolFc(_G#Bd6$nV*EQ~e|mjA zOio3Ku50`A2x-->0LXFKak%lACDEPs>YoP#1m*+`* zWB$|CTjwVW0%(pl=e3e)7GC`SFb0uxE`^hPxyjA0<&PWUKAAOi14$f%X0uKXUk00BBAqVL6w zpcrF9)9*wxBeW2fe$&~e%?OiM5ld)}ZZu+6by-*~(|A_c@aw;C1&Pg^zxeRUJV5&q zq@tBov$vl0^ZS8Kz{CC3Ufh7e3tibTAj$dmYUD3Y`{i$>AuGQ@;S35?l(VfawH~nv2`0blwtgw~~s*TLQn z#sKb^+pz4_G^nkOYn;HDe4OJhe8oqr+5k@$ltAWB+P9u zD*QJ6TYWnk8=tY-eYQ|`hMzhXAS%GMqov)_%1ROfo1-5@fz;;WvF}sVz9;Q-%zj`8xw=R~w`kRJ^#?;~(y-!I=iS}IM=g+rbe@A=+{2J)# z0k(qk=rfdGiiHn>@6%L{R2+IOEv>b+wbx4u@!Ws@m!Ww5>({Rk0CQ?({_548^NR~~ zR5nCd;osYWJzLa^V?6RqfEz``#05_ zAQpDl)&HtI?EA!V7``b78n=c0mv)1WtT0~q5E}__?*23x^x#+O@d)Hprag(Dpcv## zdw_i&K@S56h$;U|{DSz||F4{qAAnGW#%+XXQ6xO~TTXDWWzi`kKu1WNpC2SNy}S)E zVleWhWo8CO_QDo%EiIhk|9$nnp{z|+IpL>xU!LylISe6*F^I{@LFVR)2?;qL)6y>Y zveEOAuT5%ORzzTw!-0dVe>bH?zr?BHy%yG+`0}??o*IC(!qsr}rTiqftg)wDrF>Y> z0GDLc=`6bE6EDO7Ku#S zPiw`mb(06+?Qk_hdY**QWb7RtJ;VeHhwH~PBlE| zm6UYm(xHY`d4#?e0k)5}th{6qFp*EF0PPg{#W(HHqUR3TEP^I)n>3 z))PpZkSNXR|2P8$;w&Q^;$*AX&OiJC;d+2f`>nv$;!}o{>o|aLoh?m6Q&GG2Q6V=CswE&^xkL7D9iIh}#&cmC~mdb};7R z+x2PH2ZxI!Pl?hlmv7iJW)a*SKb+&cMiee#zdy5?~LISZF5+CCgfvg zeco#&ZzSQR^!4)7kFZ%TQ1*Hum97Ij8Mv0|DD@8l3DS@BXNx4pQA9*8&vX+DF!R(` z;W`YJgyGSn7m*skqo@>q4tlaDn`0vvQxTmepnEICx1ESxh!uKuIO;MS!$Brg?OPj4 zggxDx1$kNU;1;C)vQgJ&^9d)Le-BZaX}}n-zfk|s-26@#;#j^jtYEL?ke%=3$Gne!=dS~fS#6CPW0!$+u@i~q*PXkqRClpR>+M+^XV!(a8q7^poFD& zuSfHRTsQLm1dW0b=)VPat@&WPercn!#4?fxyneVe-sjxSM&v&E`98eWwj21CmY%ID06u4P9C_oE3@hj_NAtjd_{JfX8YX6!37oJ(NAl}9 zTdKs}(LPC7lcoH0w|ZFH^gjeIUakr(ZVI0a8c;%LttuWqz8@^3va_@AxIx4g{CESv z@5q5=IF)IX3M{9p(DW4|F6xYRrWFCjYU>=-&junlcUG$^*u;*>>mq-o zIVA5VuKJgkmrIi0gir@^xTA$aR;QndzP`R@xeYa}P#?611WI#0Bi>#AnX=GBZ66v6 zQSriHK&v5q$iTo*Jq!}jdop2kLPC>kkQ^U2H1w(~fp4d7m0Trf*cEkgd}EuPUqHYR zEDANz_ZS$8Z@;IbTVM)~cXOLNDLLes*kYJ_(N)zwvI<=cK)f0qyPnAew#k|~1A7V1RP znlhcpgzjM%afvId_4y?H>{NEly8dH)_^sugUJhOFpI3K0SD%brc=BfChf=GHzPxF5 z@l+lzcW+jsE`mF5{4BIp=)NH37k13ld^P7dn1NkFGVtr*5_fTP%EOmSxlzhsc4pKd zt^XUC8NIxYH5{FLlXZYZHdJolVf`rC!)763l82X2FDlEn?(W^Z<0u1fmtfgh9w&>4 z7n7A$f(-PHxB>zqldCI)o?OOsd^SU1&*4OP^nZUYDHZoK;z0T3%Vs`xn`sNlcg6fG ztE(&)DWVr-SFpOd_Eb>_?~{3p^bDK?li80SL(funrp%v-|E|Z?tM{ItPB~xV_d-s} zBap)yF%YfUPVc>ASzxlLGb@;Fi#*8$J8o0#lg8fWt{nqeS)V%o(>xu2BJr{8R632J z(HzX!)iE0zFRz}rHrrSF9H_jPzdrc9>JXEbcBR}#gxQd{_H2d6#fIrg_ou05*`tM9r}1^Xu;;Z~t%qjUMKrXLC0em{Y&Pv%g4$IqanVwamX7V` zbMbZ5T&!eDb-&1D+kAsHulSsaUKW_SOgH@W7R<2~@(`Sl(}e&{P2D9QqRZ8ga88HpKCBSe4>71D8#VC-k0y`{dRdd=_#Z z2B;^@$*8HEGUTa86_KaH>e7PG(sDn4dG$}9o{viZica>fVfbWc^TMIwo7c2iiUNgV z2Ua`)vw`j@Nt9Lo>p*{%U{$ahb*8VsQvJ;G)6P_6hjzj6?%C_+4L~Gg`gkC=i>BrI zLaY^L9e%I9Rno0K`qkp4p&Y@S>?L>&JR%Odm56Hc6mR{&^4T6KQ&ZcUkde~U(nQoY z?~7~165aXSOxAnS?&F7w#?HS=!_kGA}M<9wl(fr@ zqv=tw&J#n93zwODZ{Ed)6PPRr4}^CzrPS0a`(G@uo_dw~H$2riITK9F&tKiiei*_;Vz1bM@Zt-Jh>My8i6W$xCO6Z zRCF{0=exB%YsoiSV98(GymNdT=V~19!?^aB4EOHYgXbSnot}>Up3ct5xcC~M%@lEI z8^rwmc=twRN$HJ(?2lVOO?_)=IoL))Nh$D%^3xp>lJ6PP=Ty^>NqN7+KQNGm9(G2+ z)%reGH$ESGMXNF4+DRBq^KclXIz7i5J+^1vS$JOLVf$gksZ zL&7HqCSea8sa7;ZE?3L%)9rxdFHwPk?fL-E0`3(4cTG5|fbi}id_ZXlia(p6S&E2^ z#7t5&D-?zZQD%*NZdbzCz;-G~Oa7FYNTt#>ICvW!;gj-(4qX<~hb6A0vzrR%HjSDlHRx}+RS{Dd}1l*)x zURXAoo0~)UV5yNaJh&rf8(f~1M_H@`uZDTspYf&xS5r`Z5_XXapO1LzLUpX6E7zTlN6t1 z^*qMlR!g{yt7CrBO6kin6I;!5^F3WEXFaA{o6X`vvL|=c@sQ#j?72a)6sg}ePbyCN ziHc;_?X#ueE5y+GIUu0AspW@@-PCYS=|fxLL?_&Tt&|#)TV^RdI}(=BGpN_u+3xro zhwRv{8b;qwq%S_=9~<1?ZjXF<(Yv=N3J&sbs9wP&pM`}C`q7Pqox_d8x0YO@#D@G6 zXHLo$l>)Ifc{!FlRn*i?>~E2+lnaHf@kOT#P8s*X9>C( zPe!_s_7IuqD7c}Q_A$U??sQevZl%i<;*iAqi(O`G zO-n&2qZ6kYgYA+#E;4&>TPXjtQy8k zYDeiN%a+%@leriP`up`sI@1aw56KZw*UV{vb{wy>P+pR&?M+0KrZ)FO5< z&0W-P7^hCY?Vo0yn<%aNrXi{1{SUso&YV4KoCl}XaK;fV zwsyn7zEiqV7;^6S}^2~(k@Nl8gy{@i|i#+_t z3^O;s+>8Ndz_v}&S`8cLfW&Vr>F7(TQCttEkid4D=H_NPVPV}nLSEXZtwolrujNIHquis}}wgd$}r?@EaQ+s?@Sa@vb>A+uun*1L>uHLwj z5gtch9F?EX28Q84Be*YM+YfoCmR zLsvHhvPo2|dJ+ZfKy_!1gzXdzj|rJMx5aWcVOnL?DSN#jxzXJu3vH*lo}S)VwMzlh z%G{g}zzAnu<{vOn)MaEmgoG#|e)j_=N5MVaM`2-M;1mofw3!wZ-ItamvEQ2Df#uVy zI}QLxObZvi4$#L>#puVVitjz$98j`PR>e@Cpq+HbB*zRc&JZ?Zr;j3lrg#LK(3xVilSnzga>Rc>?x zKcD(HPb$8&pl^+rFncCmkfPL_CrjHpU3yQ?mqY(7MQD;clj9rCh4MLr`>I85g_anU z1nV0X+@l9s^dua`E1sK3^5M*C71g1)1I{;=S=XKv^$b1*IOF?8&%(9JH=d})fG76m zQC?aa0VI_&a=0^N{$nSGv2c)V-~My=x&Y`ISYJ7>DHr{rq~%w3G^nqE4&LMVNYU;~ z>{`Ik&>V8&C(p51Y;iHkox6wnR#GyQDVY}AQ%_V@o3^L z%F21B=E+6x^z;?c)1{H!+na5ZI>JwCiU$r91A~IpFM69;E3*0>!C>SbOou!ZG~UwQ zQ20A>e1VDK>+6rD#;8R2|1kB{QBn2L+8|1Zlpsinf(nR8cZqkAtFdica4H{ zDoRLWAkrcYGc*#?T|-F6&_fOL?RoFLYkhyXmP>~?Gw1Bz-p_vGU6zHt$;!^onE4To zm^dF=^Rh1F+AL>Jf;;+egL4=AyNy?QV6LjVibg5A*cQhj5ks}Nj}m|I2o|sG z_FfZy^ZPeBu8UhUF3~Vy=rQ9>awhhw*K^ykaf1fQ0up760|R?~*<;wFx%$8d0AKhR z9~GCFCiA;H?5-tKg^A@Qcs_g1FKwV+#cZPXIcH2|4Vh9Fz$>XMw2F2;*goa6GuC&A zxy~&?CcBGJqm@26A+!G64&AWITDFa8a19L^_qCVbcTxq;91h*>JHm?p{P z)k#FZi5uO^&^_(2?UHz65%@SN#-s0KGVa5!3Pzo~;!cnicEc;F67i{u>-w!w;gu8q zD+PDg5b&G~=ZDk6k7R-vX}BNmUL@A2x}>G-l0S zlX17}-yAbX?(|f*j9y(oe%8{|8ZlO*SMdvbak3|)DU|+IK<>ns^mMI{T>&(o?ayne zVMp}0KU>YUSwiQ#G6v=Q6w%f;tHYixrebd}ZXN)GpaTMe#JQc=4pDZi^izFt1K zMFe9BW;Z-7O$1yS82NRxPbsgZU*6={hFU;-x&hP1rTP%jHQ8PfD3+@~aAd?u4hG1P z_s5KohWysi%(JgLf3JsML-UCW{#VBh%2-s=YQA3WLnc{T!frIv&DXc4C%7Ft@xX`I z31UQ^LBI`luMip%7InzPatw9>*VWC*$G^O3q(#P}KqL=)HQ!OcA0R zK=R1cG(J8~sF}~*yMMrTn$ueK?%f9;Dd-|a3CyOfG(T%v#6}&NkYDv~YB<++1$3*U zV@=MwI~?=`OhDcL^2M4`NPzx%e+*sjNMet|l`1Nf1P7C)`W;&JBum1JxauH;p?i3k zCye0fDlgyLDREDf1>`c|v}r81_=W6EVWA+{DsCr8I%b|ur0cPK1je<|3Ma#tk+7i% zkkW$^6=G`yUa~OS$bp9^2^AF;X?sd?av?Zf`K;L(i;Qij=jG*j9Icg6rS`eIx;D{k z%Dzp%yfWlNsQ7&o70zUs>TPNOyx9S3YFU&9CI-wQBB?%G_oMQzb7pEa4W|7FPZ*XF zK&8{i#GB}+MCPcT!?f>7m|J6ZeH?YfD*qbBKkUis41WLNiVoBJf6-PnO-p0^CG3lq z>!NyHBNJM9XYEKUS3!GYp*$V(7b!#6yndAzTpxOeZFp3F)E{EWW22&8WVLEnEHTd&8qx(5bivNuJ#u1*Y{uEQny3l{=Pc4LWL8#U>;2l}937kN0O^5Q64%67#{@tY{;bU;J9qt}` z24p02#V z@+j`K@hkx|NLhd5do;);CN_K%A3qIOTHh32a|V;8QIb|C>`Qr3Ld-SuYh&s4`sXNl z9b?h*q9S_x8rP@2KF2sCaAtgm#KCoa5KMaB9;edKo|CnhdzlEr^X9IU7P9+BtHLpK za=fr@K9MCA?TEc&-e6?tOX+1(_3$W4YS%CITW`^ilcsj{1N@|{cF=O>G&{XBHKWOp zfaab3t*X`@mC&r``u^ms%g1(knA%2bqukjoo^S&?^DzsdGwP^O?pcOWp5r}HO^Qw_j$-FhQ32d(>OrmwFlxcXL~ zET|oY7up+Nv720Fe}Y>r{;u$2d44_}UyRD$*o8w1gN6oI2e);(b>MK9 z);a!3(0$%|-l$J*e=6avE|M&6zmq*XcT<0EZeY4U``4ABF%*jY+c*Bhz0YDocZbyO zEvB%6(f7dDlR*5j>Zr4gcH)-QDj4tGZfxG$yO1!7HKEwttU)Bb&0oRH;Uz~CM$v8p zz^@y}ZXB*iDcSWQQcjM_jpNpfO9sV&Ms_MmeAiC^Gww;kqApSTx_zMK$&G8`b`cV# z6-^NzhlZZ`b}98+_vPcB`}FLbL?H=f{(@3IdT=n1Mxk2jpI5`&{Gi37_>@a(`0ccu z>$?^B72}$g%ikVHed7D;J!u*CG{>v!zOV>i%%`Vv|HLaQN+kWNv0giG$$~l!axP$V zM8(B3EA@}7@{J1`-jXyat*_iNz)c#VO>#edF z^eT8HpoTWC1l8jVafvTRs~bl_PAQa9Gym!?5KZ8py z(jVi(xDO-3!aiJQ=48==hV~rNh8RH6CPL7%^0OXXTW2R@382_hD=U0j;!wzMo0w$O ziTx$OH&auBwTv~LX$l0BfIDA$LT2Cu1W3z*7Mv6Z4MRghzUqKOCLDetFQI}Z$op0T z>>jD{KfSyFqz5Sd7_jaE?O_lk6i;VorTUj)H|8U_Ifb9v*zmcn4x2d0RYP@$Z2?u` z1^5>4*OnF+Yfo@T;MP)*o}OO2C>?uUytJh%=Q8j(MX!GNi-#7xCPDg;NFQqBLo23`G<`Fz$z+rD$Zn!20+ zG7SaDR@p2b^1wW@8a%Z7np>AFB2oC4wfeipBhD8Q@kn3ug_0}AEBVxUD%Qu+-ja0# zRqjzHrWktzvy~PT!qMi!e!oDKu=op#2}OTF0s-=)M;`p;Y5*2&O&a{|{>H{4@95b0IACX}H*N_`;n?lxHO7SB z=5G?}w!3@RX}GPaAvSY2L5yz)iIp2i<=pLyEb!Z-B~aCi+AK=M=<`0CGI3?!BOgdS}RC}uA_F&X&Rh_xz4{(Vl zxTx`+^Q%77k}k%65>nE9oBtHzE(?#8l4?Ca{C&G$QPy>>24Qin;3A1c+T?Zu7ryO| zYJ2TM+6dWM$0KWukk9g{q-4c8Il5+}mPusTp$vaO{&o|(^Gi+*c3b9kiVM2ch+Tj6 zc*N1Abk9&*5`IE0HjAq0(~>@G|L{TaBsN*~4JD&oylkUREh=>^7%y?ak3$7*)fsDp z^HkSDI{Uw)iX5c?fs;Y+*WVmsOnU=N)kod8wk#zp_NvSJU$e;W_WR*}{otXw`Ru$@ z_jl6F!*RB1FV`t&S_A@tgV^nBf>w_)CI9By2Rshu+5=8flapBh^||Kjee7X;Q;*if zDfN3{qkF#C@m746GP(*5`GFZ!A8ZNSk!Qa$8U=g*#rQgJMD4O>h&lv-806+ewS$#L%uf!MSlESkg5 z@sa7myM_Y)lc6l9nfS6nJ>4%-Ti<(t=eyuoIJA z5F{qFu_Jo(vi*I+ETe*!pY=To>y~Sr`h~*dI5pBP?Td69qcQ2GEp}WAD3k78#8&LA zrR5`d)@nk+mT`#r1&MQq>|@4`j6R3@rNc_9Lqzf)R%?NmEQ&b@ZJ7CO%gLyfSv4H zT}ysV6OGto)P*#2|9T$|ofdSQe{55VG zIaLxkqrO2I0cYK?m%u(w5Vh(hGT5hUG2L+0S=KiRzX2Ty2p&Sl=O^=7ccLIws?)PzVq!r^q+_Tp>tG*9d z19(RbmdD{U@YW#S%;NV%pIR~oJX>h4C z6Tp54=VJSsT}Er0CTGXVn(r$L@$B%`BSE91`m3WQlm%dUQFpPts=B78$ZbuZFq(mz z85<^zmagZ=t9Dl<;@e9b7*dlK9-2cwcQBfy6bT z1=6iUyqItl{zQ4r9{d!Tm`VAtP8}*1zOkSZ4APYzzh4`CcgX6eb%0&SXKCN38&i(nj zR%5AG<#yA@HwEK~h#SY3Azc|@wms9C%6@$6QO>}08ooN>6Q6?r-9B%j!{hDNi!y`W zwQF&=B0-2vap-op`Ss0bTUQ|Dg2N#xF78$}Jd~j9ydUri2Aw5@o$YGlY&1qh-%ywK zG+66UgD6;pNbmr2(3kd|Q@j+5-)>PzZO}z|IduE~DzlfERaI!Cy`WwLx+KAeNrMKb zp_ifq|M%t;nPvfx(J?V_Pa@ghpY`$WXlS4_K4?>p-6DGfm>-+uqg?x}@1$HpJ?MXH zM~4SpnkH+bnhyW&wZ;Vcj*V7sgc~O}igdAcw=kS#KG_KNB(bAsU5SM2bHBZUI{6e( zVxypVzjaDQsYIW>w@Of)~CJXH|%%anugo~68UP<8g_ObQxprWYa%{JPfexXX@?aCJkByOhp=xs z>*@^pGZ|Nh?C&FHd#`O@m^LgIJf$f=GFG?OmzG+qi*%wL7zl88`U#Yay=TNa1t%Aw z;|dzts#xJxNQ`O}kP(ChN72hBRR};Apneup1pul&(i1$GZ5v?B750Aq9Zck#9GsSU zCG+|y0I!zYN8zhPEs3$=Ong9U^48L~9wxR?Ymsa@lC=2sdpPmRVcGz{(f$S#cf{mv z+S9}f3dqnkyz4ol%f-|9zT0;dh&} zWrk0TThjaXTu!8aeDs*%2&a>i+Bk`oD5muG>iY6gyC&+|A><8!ZIWxgWObp%4COiRd9l z#Ba4|@&vMKh@<89n3q>%&JC_W*8sHy+$1fY{X4Q~j+nF=kD})HJM5!hT)en1`Pyb3 zIZ2oT2*5*+YZM!A-Qljc1NL!akJei>&MvCtF+-wY9Zi>P>{t6Vd84Rc4spNdef|BR z>Phkqo3rjaMSvp-`g#@CRVpgx-_uxMD^PH(Cq8p#rbkJ_ZfD#EmXFQ9UR42B+*BCV zfc3I7p@MeGP*Qp?IiR%Ey3l3mmSL57HGW0`U%|O-`Xu`!f!(2ZrG2`n-#d`hqV-4( zn}d-Hxxkl6_AGWC+|0@qfT`m2Mhq}qRSyrz=BNwU>||-zw)Aw`ZjnSwJO$;z^4peT zTDnLE=w5DovcmQ+PJa9PHDnZET1#8ok5`#EmlG|K$mg=1nRqeb_fHO)i0!PcXWU8e z|NQ-%ft`KE+jDbsPh>siPaL>3g^tQj_8EaSAngS~9Ucb4nEZ|z`Ut5Bz*0fclQ|5s z+v(X^5!jrWspGoZKRx(%Qo)h=ehLIIK(b?MBDmNBt1ncT^Mi(?W6_Vbl@*)8Oj%{o z!+zM<)LMhwuq`eObco~y!3jrM@QXFV@JNbcK8t-%+ zFx&ZAlU)rmXc_-s@~*rt6Vl^=_95g?n9mA4sh{X#?o3sRrZU`2;dQ{SWTW%#mg;jf zec!63UAQfwJc0hcES~J8K!ZQ(ZsFlbm%lrFT0zUG%U9s}o62wPiU3iNkKkPitm|hlkCp>%AL(o&gM!s0RC7R~%+03|H|jO(p~V zq4$>a3LbkDotGjDBh9n-ECBwU$`fFrrPLJA5(?p8F%pypZ2Z2W-bePyMCN;&Xv*)M@M@#o_FU@g<;5PVez01Zq1}6BWJy%+_4PE~%Z(%K z*6!e~3#W;%jSP6q=8(m}x2L}G?Y`Ip{I5MO!f?otT<}IiV&mv*O*TD8wZp0U_?16q zwfkrGgswMSifRBPNO|uezJG*LpaX_aCQiPbg{vo1buSxR@T{y7&5(wIfY$@jW4LIR zba`1CGU_*q)4t6}pjz1+76LqHBl$Aj(Dm=;Uv%PZN~CQ8C$3K6>NJooO6FsN+{BF{s&+p*UF9OPg6>8RX=aXTSOh3UHm{(f*H&~rCkd5gUTneY4_j#W*=B$Xc=;Ci|Mnb8X z0Y5nb7inS@74_lbg*fDGumIJvwCS5^43!~0TIu@xmyW}y@GK>hU%J4hg95}HSmLni zRS2`ha+B2$=?3n(@rQ+d2D^&4eOc|7$!ifO#Xhxoe6^=c_vp_aJj6-*lH};NTox0% z2zWevHYO@RN-gEUGPQ@_pd#Bw+} zW!(1|j3$Z~Apa(ph?4OgM1|14dsnXb?Qv#B1?Aja-k;X&x8{J=9sXOu z)H|N1b8+3(=;VXXxkH33s3h7R;Vq2Bt+n2b7ZZ0QeD00*ezBpA$_ft?&xgD_&Eesl zGxhK5Z5tb=vy@2?J6xeCwA=kiq8fifqQuyUTl|u=zZo6P+Gud-8uNPc_)KyN703CL zzO;FhiOTYflcehgD0#+(E-XW~tV8;+?U7hKeC&`LN7^2bx#R521lEt6`?E)I*QGS} ze$#@*G1KJa4Hs9}FByuFFg80LIEgtqS-*IT6eFtTE4T@!p|K~!2LM>Z&fgHAi_rf5 z7pG5`*VZXUegA$7*1{31t1M>bI*ZXH#<-scG7a#4Y;LyL2-;6lKY8+7F>i3j8?DxY z@rw_I#Y)VF^F$mR7vQdf{_;<(YURvD*@;2 zm;LRuF4KXZ7aN-nz>bm*zKZY}wnRU;PA{+p=eQj%cI2wInCyL-4aRb?exGT7`h$0!Us=w?53msq|dh;-P zGJpHQ_qrJ`SIRWYq>l?vlG@CItESBj6xY}Ay;)@rQ{U7!D4bPy z{@sSRuWqD)f~AB}ePHd6f_Bi>N=<cI6}Zg1BE(OIOW~kIv?&l?ULB zj_5wb-Q}{(Wjam#qx@NPazIliYw#5c`Fq~xGlhPt%ZDDxW4_u9+B{|+hf9qnYYW*~ z=Ql@fMrco`=cxy%gv-AvC=l@vFv?hTjzG*tzDkf5BM)cfub-^(o?bqiYcv+PHkOj! z*V+vO0>~Y_8M~X1F#~XaVS9J~Pta?KYrK5ne zy5WaED&Ai$hJf;#P}NJJsxLD+P80v>%&+TA83GGsBEZ~;{Pv2O1J<|W0jqqPBTq{7 zV?ZASS(G+I%`fZx9e_cXUR`~Gj3#ce%OuV7Iz=!|eEQd~2hWYK=UiN0*tivVCyHI> zQsa2VnB$th9NgNdBctmlo6%}Ot2(Rw{d9d-7c+39)YVgXZdl)ObEE1^&FeOHW;wcf z*8dG)$W!~9GGN73+mUz$uATPE6+7V8D;p${4iyZUVkS@Z*d!&dMrWl8`<>ITktPL} z^^d%@rS1I_tO!I8bJH;|5v!^O`qeC?X*O?nWsMK5SW)Kopv4VaTZ#A6@QFDn9RWU@ zrjNX1g^s4OkF^6b9{q8E$9xijXBizUIb1_-Ml#6mzla#wCccTbu=#8AJ(XrV!5tX%nJpgQyqMjkj zq#%&l^yFBudRyZX7-`XhTr*jqi*@3WaY1v2(lyY=>-7)qt{8XKmpdXq%pI4ZzQHz6 z*bu*UXa?^s;iQXXosId}zb)1FH?INi3N7uvu>uA22=3>8o>Q#}5ajbjcI%G?5mBB_ z(Ce`$A(3l$U0fuRQ{IYLScAsTZWWDSwJu?VZ2@55=Z>+6G;v;xw_F*~8^R#vxxGMo zw7S?GHcDvtkN3)gTEgoNzY)LaxF}G4a`+fx8|8^^D=Kn*h($NUZ7t;O#xp*_t_7!b zjW^{EuEM?MaQS6p)5ttYN}N|g_tZ^K#M1Kt2-?}2n^gR^GL(doC)*`Icdl_Drv{h! z^y%I}w3@PL%}0 zskH`yw3%*?9?@1;SAV@9WchA$XNRF=Y;-hzz?zJoMU^W}C#UIPdqJfLtY{eTadWa< z5oMvGW}{UYT3%m|s&HO-@R5#g7hLJCT6=qW{i3}Pd9lnM-TaAUnc+4)^@%L;>BpVvyPIS142R;pHuk1+1YW? z>S$>pYi%Xz(j4mPc?(N97DF;(uz22vsnRF40Fb+oCx(W$g4EocoTvJ-G~;==GJC#N zR4}LcY*7%Fe{oXaGdkDy2}l^6T`Hk}W(Z_wKHWyKdvMi-j<>lo)WCnUnqa@!3;H%#aGhZQRn>}MeqY6@sMa!i->RR+|_Wc>cD3d>xs7oieLXs|ov2uG` zr_HhP)DLhqZBV-fn5KPh&zQv(o-h?-4tBFqeg0i1PSk1mUD-P=N1Q(X8~}KFZw;Qc zOpQ>A#6>HfZZ)-?0tS`Z}r z{r&+R_VCy3{20?dj``V7S;*VkuULsL3)CL)}bge3_cF;zoTsmjr&Mi9ddv>(wanA~W zY*_S{!&>;+ea zVG!!XmH0y}PW@K%s4g^-ZtJTl{gOA@NWz(t>|fHTt)syF=i4_4qVbDP6IrPb8Wm28 zXf-sHXzuN7IMi>>Cc*ujCFziU@wR5hd;nQEpoN=-P9NJ}4VT!6&Ho+C5SY<}(u6wM z&FtB==sZ_hb`8V2Gmzn}-{NB~qr-oL1{w!VOd*o`1_VW$Nqf-CO z5YnY!91IYKk_v1ywJj_pVA*{g{>;A_gDW~-v!?_VZgaRdD;N2xlkK$0Odr@@4*%vg z8J1$z*6xEWGXC1EG{$ok`M^|r?q&7T>|0-wg4xB)#$Watb-C)ZK{a^T*8V_=>>vFZFbEXGi<#;bSlAe0JY%ZTM8XwDEHZKzp`z^1*fVtx4u^KokRqeVBR1=`1q1>ED28-=1F!CLx>Oe|e(&F!>mjFd zRh-9Hf8#cg2V-Pr|vKZoi1GoZ}m5-(+qmhBBSs!C4p z+412p1j^%$=1DXbrEmsu(C%+CCus;wp^~p{ZIh-&4@q-Uv-G!0^ndf5f|+)|jz;*A z!I;pZJom$mrne-l>5r*Sq&zn2F2dcYy-|kR-61AFMd+3YYlQ$;-GZn(K;$kuI@*6u z21SL6Lm{8PzoGi%p5HmF^Kd*g%ALt05&Q=7LupxMzgRkku~Yv~=ds;?Ams}EPy=m6({ZET0!u{~+o>#+yWgsY(?CZVTf zm1t~vY`SnhXz1XrdDmYC{PDs|1}2GF=$y+zqZxneDOLG;w4W%I!omHY7m7vxt*?j7 zwn&&9Y)u>*t}IJ$wDAR{eW^N6j{|>I(*6B&KJ337O;2g0-fU14m|%C{R`NrhKRED> z#>>$U0=sEr)JBb#+3K(%0_$wo(~}{TZCnP@?ty_I-?QI$)D%wkPyhbyGJS-`NV!>x zRw`oRCE*g5bHX

    6OrY@0Wc#A7d`CB!9yQOoT@(bPl0sDQMhV{Lf6AEHd`ozf?g40CyhKJ3o@0p*#aJaQOuZ!G|b~EG4UK z4_n(OPd^Zm(< zBJ;m2@(qe+X9%2Jm>7F{Oe8HH+`C6USYS-U!NEZx49(p9Xo-!a*ArRpQgHXZ10!DYp>t4L2AtA-kANDlBb6FKWH=u z#3=+;9c|37L958wZe?%Z0)jXPurVa=Vh1ZN2y4CxF|$rcA^2}DpiEgBLwmHbn$upm z9XR`g*KV+-YP4k08Y4&Ju<2CKg-Qu7%D%EVxHH)Hq(b^!Ip)tlA|G5HS>YO)s+K`- zG3I-lcmPWr$$MT>Kev}__@uW(D6iYb23lDc9=RJFp9^Z6*zi0$*}biFJ}5O-75J3S zC5-MtKe*Rcm)^DDD{19TNWfms&+ojQakiqPPM63Tqc#2HvDbQdrR(t~OGDgOJUvCJ zaofM@oJ@ggS+tiWvbkyBm2NP$E%CU_I(=Q&bWAU$H_lJ&agdi0ykc{=jH6ZtE$9Es zMy@1IblL9Tw|Gtg-^D}vW>9IF-6jPDmLz?_73KN!oP_2lW+USzo|a~Ghtf>u5AQ?Y zy?U8O_O6=Jb211wwz8vYGO=Yl(cNsvf9a23fL`zAR)DGF`j<)U$G*#HJyrTE{eRR2 zRMu$`RnQq0E(iAxaE6PdSk5#a9rKvy&aeOIR4p9An=P<%3i5sioYcn6?`d--9e$QE`R^8y*ZR1^^HB9 z#N-7d7-l&ou)N0yUE6eYLi@MhhLF1-FRJzx5-`LSKwzVCbkiEU{KQTF_$S=D^+0y_ z3Fj^2$uOAPnROz?d*q>FsQG65&4!Kf<&WbyC1VQpix*Q^985kR`M zX4rpU$6ta61AS~x?c|)+2+hdUy!d`#W>pmzFkR}v>ae9VB`0Vc50f;-nKNB8E5@44 zTezv!+{T%v_b^PWNIJfZZN-y$Zz1qy5<0ab4Sr;de=a!L zp0wY&c{FGy>CJ*i@UZ<$C=@)8o-85-STHX+~%2X9c zu8NsctsIV8J0gkWZOm!oKcrH4XV^ge4hC1h2^y2Z?~VGjS3zL4{*BAoMg$a?_?2PDtG3Mc{ zC)r*IW^Lp^_|z9}i)FpvI{f?+_#nwLWoN9cf)K$~fO0`M5ng1`rPOd3Mf3)g>B*8s z6tK@WsrBoXa=zm@-(kv#tI~vdhwys^D|D#n>q`>axI^RvO;jF!q)8YkDS`ADeXj* zhDpFqY{_tm1F!dP$k_NS2ghv7vkEr7YRU5pov-WYHLp_3V2-593WMx@OwZpk8Vw`6 zbdiLNY}Qjs@(-{n9Ls$?`BJ@ZSpD&R;45#DjXg`%*^7-2g7COW&JNf+oAuU+v82}4 z-qtes(vzN%p_pkK64GGioogX^qEA9V{zyUa)6KpDochYklG8G`2U{e4850|oqfN5Z-?#Jlb_T3 zGWjV;gAV}umX^HkgoLYOs%tjRnGj0|Oz- z>eAcCynjUw%;e(*!nw(?J3FG~X=hIsCb^%lb7>z0B46x+V)6%|a z;Xr`BmEF5CY`3!!Ivn1)>fJ#Dm{%_}k&uu;uc=u(;k)tK6u==fb^!JM{P_c<31kSg zkgRg<036jOOF6@F=tBm+D*>^wxItQi_sUg+0@hL8MGplZ?-mx>o%)**>Qq56OfgV9 zl^;oQ2}=?2p(HM#$8fYW=7D!;PrHb^FPA-~(h8P&5y6B!s?(RVxVx_4iny|vTH#1l&#j{5x4rRF|O z#JvuQ zagr|7E|ECSB(R&5qL@6msifa6elOF30hp*H2$HMlPecApV~PezQdKUW36wPj7%>cOnl zJ^1}?P7XEWQED+uf4il7}oe%c+e+c?jb3A^0JpIqi zn_iqS>a=}g0m5qr3_)vaEHEKWEiJtRHtfUC41#7tPTE$Y6O!JgVD=)GSz1bxk}?5~ zOUe!oUEWu()L~P@vzJ{?0r{`K#KskN7vPanAqD9MXTZJy7LY^NjS*_RyjJy~E$892 zx~mkx!V*ah@bJc#(;c5`8i2E=S7kjEHN74G-J@U>I03^u+9`h-cjQ?zvTe4$o8NMC z&wv!>$4_e|B_(b)_Bm+Cu3T@X3=Q=IMpBVA6@LLjT9(GHPZqe@znBWN8O0@(x1MV_VBHmA4<}jWOw@#{Ofd-m zoO$CnFGSbNAsa{;93yfw7kBN zQqARo_|E?hZ*Wq9NVU>9El?A`CdmX?WxsAosg@S~Se6Mt?g96iah%~4>v~ing&%e& z#pxjYl+Q*zSk=H?@3-G7&GXlVN+cm@i}84=ZsKsy8LuA%{m|}O+cD{r@my5^m4GJa z5nOJvxmLqy%WGcBm-F7c3{E5r-awVn(j$k#9gJK^ z?Cj(gHb8uEg~xIqZ8QHe6l3Vj%V(pgH*7-$R$oEM$=re_F|6HjZVClYL5`dG#dxT! zuTSEoVj63uZhrq|nJQA))<4Sr7!tNs3r($qV@25tRh~|;c2-+Vl0kTLv&Q~W6+ycP zy|=kj+8tv|O?)uF0g?!qzg1icc;>1aaa+6f#%%3Z<<40bTwUd8YP>xpIfZ*kW&yyK z9&P{UJ1|%}+xAJHaPK3L3y0$a1H?eNN)Y8GqM$%B9vbh2Mf#G!ZGVag=r&E@i}Cp8 ziOk>D&$!nOPak!MP%_=n(iuFPqOt!gx}PRAw0Gyd{+BK{Q7mQT_n|W2mu6{v%E-*T za5fNvhVu!J`$yh+-ZxJT#-c#nUG7_T2Ka6!ChLUeco^3>_}i>CSkk9F>*69HG*pP= zS)WjmQ{b%g{=Fk>QuRFtFb?R%?BwIERHH=TTn0}X) zt(Zb8gCMMC1Wa_4mFJ7Sv37&;a{Nuh=~D-7}C+5d=&yh<~vc z041&Se8+nbkQITr31$?V%wa5OUpzcK?tP?-)B$rI`U|C7J3EbpAHLU4pV(tlP!k0B z-F*d}JpI7dSZ=ihy8GFMg|uU(uj!izfB|T&?!%%g@h2Gl06#nqxwg6*!2u%5-fPO@ zOARy%%FmytBKc!mH$D~kfaUKzbFiv1*u_*~GSkyuguM7V0t}-cytLMh}gsLKsUk_c{jiv?aKs``|XTTtD8;m~;#_#$nGYe>?rT+E?3^m4IicrLZi|Yw)l!IP5vR z(@c;#)#BCgckGv|g8boM-ekNa2T{!Lt7oHej&(6(mi_vKdNq;4_qlSqA7A@y}G2R2)f?j5I^(g&pl1$yIv1cf}Y;gX9%_G^LIG$0nBa;^fVHi z!=|%eHEbSbWUT9D~>8&jLUAjtusAe`aQJYR;zOOg=Z4xJ(()yZfh*kGT!#$Sy>$Qkce#K99yFg1Fqq^-@>U{fKw7) zOq;epx_O{~?b-nsZ!6mNNc+ZIf`S4YjDtz95WaXu*zP}sq=4Z(vu#<~DD#{e;rtSsB+EQe;i3}N z_f{gF0R04rTo8`g-vaiV5gHHGKz~F?a~>ZzB)`*j*&FH%Jb)%kz26R90m)Pz+gbYp zUJ2R%Q3TrMhgIdkF6BdPtRwJSeQ$zsBcDz#BsY9{mq-_R%l~rv&z}z%Op7l35(Ba8 z|1k+58Q{7`GLMN4tPvoyzCS8D+QN~VQ)GK*NBPw&jO1TPLtFZAr8Py$`ELK-?yich zZ7xh}!10AC>571WCPR|&tFuQ}Lu?;uYsdar1EsGzr?R=zLMJ=e`YSVlM@Lx%8V zrx>TrvT-A~aPQu{_hrtp!HJ1TIC5wSRkIu%M@wx-PFYx3gbH&xS9)%LSKN#PzHaER zE1a1<`Bhb{gmF*r#f!_$ks`_47&w1mA=kCjr8PbWT3E1jZs>D%>3DxnS^10h|F21U zUh|nyHMQO;wY^HXG$0oP z1(P1eK(R`&$!Pd`KNiS#vQaby{~c2ApRCes5*m{|_?xlquud1rU6Ug*oUXW6IZAY9 z!F&GSdpK}&K>RMTefR`9C5$I`^?$l4ExJNm;Ie>7PN79&>2$*Lr9|y3;QQ*A()8vn*_}3>lCk;fH-@PSEQJYon0;tebv#{4uDNV z=t~jvju$3Z;lRF~)n|e~!DS{C7mG07GUNfLDN$NQ+W(wG9EHt)SwL_tVH*Sbb{-^< z#LKIqt6RxJA6_H(S4{K%-(8H+c_5qbtt$HYA$iv4=Yzm+Z_C^5bnAawb}p{zS&?dr z5#XM8Tmv^kuo21B1EZEjzs$?z;D+*_-^rs|(Hl3;60nS;DrhwtWt{$n)Va^YCTe;` z#%1#ocu&Q}F%molB|SDSS5{T2w6n3X5fU(LhYN{2JCl=wLvod9PeUpT6WrmM9?Ht6 zFZpaJMzs?_kfdI?w?V-VHXUl^9-EnpX!-6w6n^0UvGpdPSnD3rMpA~H0Yhl~w~NGVf<457(T$`BD1N@Oa@RD|Dg-t~O{?_2Btto5#E zJ#QKA>pIW9kA3W8A3Mi~6F!3%F~z@qwAg!k6LBiJP<#bnLEzJcE!S}`YCK()R>@eD zhm!5|(-Yq`9UL+m$D^X6=#uvg!>rH0D0F*{@CCVPmMY0Bmd(IRR6&9YuUp1`Onz$^ zXI6Se(?(y{pTTnAVWT#!;dg#`Lxht0-7dns@#b{eV9Td>hm6LZhWBm$wIi%!)Z-4D z$4Vml$rIJSy8`#_eV(fw2bXm4W9SZuei$-^!%(f>Z4%0p=&yF_K zcrNU?4u2?eK3mj6bFXj*Ke^S?s!X)**w{J=I}JU(y}1c z#WrkUhi7KP!)b0E(Y*Zf;cdxZ6XZ5*&F{#s29J;1qqVOwCba+{AS z#}XEn?WU#)O)E@57remY_v!=m3a0!=hn{nP!@?lKiz_^afM#Bd%|v`IaAQ=rk2|BRIJ&%VH|>+ zq}ksqk6ZMA6G)iFWZM*jQ*3JD$^ZnCz#-10b>JK4qfXKg5v!Pw(tK`mF5C*1R^l8| z#yYO8AgZB*W8Y)md1*mXP>mL-gip(_+=x+P2^qBg{v0~y?K6p7-b0tTkL`>K!JzWb zo?22>4OpT7OxR8=fDn1{qJx9ed;0(f6AD-{#!mtIcht48Z$M&(QBh%;vPJzC z6&ZOBZsQ@AFsj7_8{DGji@pq&T7#c6wma&=h57WtT?5#ESn|GHR|=PcE#?ANYor4p;FrBg4yV`V%SE-cd} zjUzp;FalM2D6>>{$BrFar}|Q%f{qoJ#3>faotpB5ZDPaWvz;hWmSN0w85V5V6?_fG z=0vE3z{Ji9h?_ma)RoWz8&QfHtMRZi`>k zRtY*giwLcqde-)FSH+_&^)UWBFHa6-Nv6vlEeejVgI6*Rm0}*7fw)V+<7CV^X}$X^PuBzv ziT6wYr)8bi%nWe!jdxqe^5I7h7+~N%pXkHX55(MgevqwAcJm^jBDRC=-z}0fV67*g z_>o8_A8OsPD74pHdF&Naq}0Q!S9uT!x*)urdWT3%Hj6MaGVYAyCS&fADkH$rr%r7k zw$dX=n}VA*8D75?SR+CvB+;Q|px(R@>gw$kch(Y4LW={=zilA;?y=uX2WQ*%!1Y?>qq=3M3+7CvV*N z{8a`J8M;?v4x5KI=DOP%Wmfs0#Xlm^+BA6!#`jZ#0UOgY_uHG((??;T0v4jT3$S@Z^ z|73J$cXdPn%!o!oQHq1o+E6MvHCd{n&HU1_wFqw49cE8VNW)=ySvjFl<>lwl+!f%4F?V!=<1Hx z##M8RIEaoETli{~V63fPTBM7?CzAi18$b*kv)=2b2~CCRyXc4|UO(Yu3_jV_ zFy(>S5;qqyM)gPKMwcZw&7PL|a_Smm8+(|lQkEaoe$GAnc;XPz)-gkBd>>z2E8vF!qu!=`T z4ePcnJ=T_Yw8(8RuDt7_AQOaZ0$6L|Tx0#{pcnX@zO|Sx?$*q)O|hz8t!|rM+BUKI z`kyai)%U&J%MVU)a`pO$hVD7m)@n-z)qjt=Uw86_Zhvd$A#R((88((4fw`iKAWNg# zCfo;G9vpkm9VTN{U4Ep{+k5(bG+o_+vL+hWNf_8yBPsGSET}FBGHuP-H2+3@WZJ8% zZ0C75YTVMbI-n=c@ndq5o6|NYGn0n)A_6+wwJUe-h~3_O?Yzs^x_&Dl9TKj{jrlmc zT_`9}>-ucdS)g^0Xll3FE84jJXO3B zZ@H#^_*;OqjEt70Wy-~ajkb|VNz38Zu`&Zm5WY;J2%TT(=C$ww04t}Spe2)KC^_(u z(Q_@wY4@*1G#DTmnVW~}TzTHj8yywJ%Eq>alnpT`y7k^SHSNDn3;cco{L;#WjxOXJp0ykX!4!|skb!P@y@MVJ7Gt{ z7UqcwT@0&bh!>vxq=ET@0|o}4$Y`yOPKa$Wk_V2uUy)+ns2Zwg&0V{Kp8Sf(a0^6I z(ET^oK<8VRs2l*%JTFJqB0uba$TTTf41;oZ>m9UIGwgjTJkgn*R63q5suM>-+288f z6VMK3<11CqzDur{%kJtrEM-ytVW9g}jzvW{3jc<~k3PXup2P!jj>Q?hnn+&l#OQEG z#Om#OdX0XAjEfctOG~q$*NWw06EhEroJiX2b#H%wtxCm z?^R4|IXtxS-tqqPWS(`3ZN*IPY=w=mV0-P;Yi+9kH@@Sk&?+lmqm#9%w%lY;`Bk2P zkkJ187Wd>?-CA4(wbm?tPthg&lq9;izn^0c5t17oVP;x#dDcH2Tm`moT|!(Mv_fdD zsjYo>k$Hv4t5>rMo850FCpS7yH`(4g(;3caVDRnO^dZ}VKgw4_v{E+?+T1E``^9ox zNat?O@cKcot9|CzM!k*e2mg5W_Viq0Ua__dCMQ!@!5PDnS?>4;MdpqCyZ}7%d03Bl z6YNdr_vwvnjXTN2v}uX!u5HCv1(_624_$pyeA6T=PDn*aD5KvuVgKC&>_<10V6M9h z;Ct%y{{1V0LqgbAt&#?{FPvnOr3I!&oC}xBql7X_Jjd6yx3|-BVXM58?!p{G4+u$gmt)LF z=nx`4#kX@!eeH`5oCIt~gj`UMBMIg+RpF~7_vq8Gpd-7kO9_Tlq>QLBN8L0ZrLSDz zwjYn@;{wJA3g_k5qVwMlZ1txc_1i8?Kz@QlUCKMG-z4j_V8eVC?w)YE;pTLPYo=9% zia%tU4!2>vEW(TO{^ZAO$PYco?Q43ZOf;>R!DN#RT_68ga`gUgjFA8S*#o<{4I!V{ zBSkRt0^hcCaa|sg(iz!yEU?D~TJf2f)p=r#;6~ZoO<1ztZ5Nrb8V{EW* zUU@kvCHE{0D& zi%>O-AYUPpv6xWN_m1x{I#mBbzwSo(k00*pe@!gHbMLMk_PP>(!TTS^kJTSwME=A- zwqHh}_Ht>Etf8hcf1oOH0Z(3FV89#{`%z8#pIEWx;pvH1wxPhF^0CWW*uZf4 z+BJ#b?c2An5f|SHAMn5$(^IETodg%DM)+bdL zIJDAo5%Hw3dKO>hxR>rk@csL;6DLkUA0t${55rW1t2{3l_!*vn1Y!{Qy$ceMtpPDu z&zd5G0~hqbraq;t%#1@vM@JK)U)7nh{TEWukt`=xtvN1u_y@L}`ohG=$4ARW2PT|N zY(w;)KWB&99xQTla#~o6eMlr6`g8>D+;Cp-@Nh?&xU4L9rLeH@$m99T?^|5K`x9yU z|6SsTE<*kLc<$vHGBzG{E&sdzgtt>h;DDoyymbEK8> z`EyQS``jawh+|>L{R)Bt9jl*JTn`EGRWKTme^s?d;^DPLFHrrElKAYePXSZLE-o9% zNESXf^Tr1!_NrbB6ieET8j=AM9FpXJw{$N|1qJ`SDEMdJfPCQhHd)^-{AhkU-fa^6 zoZIY=&Y+-(2u(hc=H{j_Ffc%D9giHFi1_LP8>+*HlhbiISrT=t0~$bjT|=0WF)i{# zmE%1`2o$$wdj&@I``5}Xg@jxnG)gH1ro>R)aezjcEhxJ7T~>vQaR8^J;g1ur7lwaU zN+HJSaG}9=vq*yzz;!?|KV=NH_TM(C{2NfT&9r36K-edOdji{!f z0W2he+4UTIUtPnAMO*bvTU!tox`$X|xC}AOALK*i-KnGyuQO+MMJ;M1@4NcRg+<%t zz=UU7!06?1;FP$81Y1FBN{XTu3c&+F3*X*9Of4-v!~q@pT6||=Jwt0durupx<2md_IY_KIZkv!V0 zW4#bm5peobO#Fq)knc`Pz_h5I#eqAOsv`ud;aNiXQ2u=F0kjU)+fkV-O5TBv$+2FHJHvHc3lM! zjxAKn5ppfJVXH`FED%bOL{BEaHce#Sp?mljK9Ih1FE7HekmO0XE<0|vNRAJVL%);} zP-#1MgjmA91HSba@HELkZAmDid7gurIXpvy^Rh8f@@K)a+Q>+ZJe@c)x7XZ!Kri$D z{e8_`j7J{tQbkF8es+N?ebTZ4SbzI+bvt>!FFiddZW?Qh3=Bl=Co5L3%~L=V6h9)x z7$HLbMMJuCcNL4CvXsSw_^nj`%#USI7B1+>8U9p0*WBbt1%^?8@!dC zBNC6)%k9>S@a`wK8Hwc?*cb4Xp8lrc?*34`{PZ`I@`#`*_>O>+AnRS|`nar&tgK2D zJ2va%)`Qs4iIe(t>!C0i1x;VY$+-aU_L^T&{I;SiMEaYd?0twOaVD`dXK`(YP1HGudH)M$^hg>a-{< zf7w{1b^6q)r(_WUH#fIk6kbJ&o(?M`N!mG&elA9-W$D@>1TjC^LO|jj)+&qCuot0F zRQFv>?T+8XLKt3SCVb@*2Rpkb+!klg5h55!oR#qjwrp^7ldUtIQ!NkIyA2!wCS=|K zW{)u@{3`C=x0BA!=SoYJNxL8ySM}HQw9Zvdw#cKzlNG69 z^VQ7KoVq$)Y(+aL2hL-#-}et4UTNTcL+Vh8&eM1c?R!N-;o1CE+xur$1IqcnaMHn^dROr2oq6X)LRS=9ST7j(34V7h>iZu&em28 zBfGz*xVXKf-$tTDvwgu6m6+MD1GK`Otc+0O-T_<1T4@6)3Kj};9eI05KR_;uV&&u% z-Lrva-Go*J>FtA%3i*s5ELEfMlxPH6vJRyf!^fAj@fvMOlZ#mLzIyxi?cv>vD6(vj zoUJX3fsvI67>6%By>fW5`0R=>tWYAT5&@GHEbfz^_}Q7}WqjD4M9P8Y44D53`2NWi zUyHYbTvivUf$M8%gl-8cl)xMFzt=W4vVqD1^|}thHTJ<^EOrmks~4MBSI?#CT=}U= ztOyqrJbZ}7Y5Nu3zb7=78&M~5`rt4Kzu=d@mm_$ISUb24;?^!%?Mqlpsl5_z3?xOR zo>NvX!5<$9*-ZG<_1;sijkF##k>Ef-kL>k$1z3QqsEQ~P%D;ITq^&6)fHIqoe-PJ$ z@}*D=FYmzFR0u7A>DZ5AV%X4?}d0?;X;@iy)>G&H0K z1gU>1WI1q&UMcACl-)Jgr?@#lJsi7vNJvN-UXT;Roy!w-??gy&EJi71isM#upsMcK znfj|kPvDw(`N|csV5oE8By)-VD9ySQpntm)W_~)ZhziXIz`!^*KfTKd?Jknb>Qlwv zR04E+kqGCjvyM3}TC@m*+U8I3d~EZTyr<+rP1AB=eR*NrFw^*4f76{qlG3jE z_Uk9sBIu#_MF7=wYfFni5Pt9t=Mo*DYQ&AJAU~J1tcd)An?;-_FIaY9?=Af>6RXEY z+gpYiSR_v;HLD^fW22(ZZx7*mg1wF}+kz4?=M%78U{fGbdXym|jMHuK;g$6c0|LI6{=T!+?d=)M?WwhumvW_<{> ztMg!sw09|Lv8M>=Qj!V^oM?HVS+Tyv6eD0;5OEMALjT4Hij|vt5#ny6yFTDxh!*=n zH8ZosCtn_*&5tNxUDD97Pdrw;arz<74T%iLwhO-{ys{G#=&#?ucfmWm@1J+J0u)uc z-<8heT7dpIMDM!OYc%}AuRdYEOk7-?wer}jtMd?pVW*Yyi427mAROL2aLS$~&k0yX z@nn~u_-*~W_m4bVsCq)nnuc0GeHAb7i&<_V_Wq%v=Y&ctDN)kfDk8#2INSQ=I}`z` zV`D{w_gr0nZl(~o!3{MOF6D>hLHz0jqjU!lVEYDLbo#j0KJoh)_d>mOK+{zg+#-Rb z$pr;(4#j?d z=U}};rg1LAnl)=6wr_xSD7Kp&VI@1($XMOTe%yMlWK4*C|F8uUV)l>EEaP0ffG|D4 zF;B^f;>BTTRQs1lO#bDOPy2t8cwzWBTt-O9?%hC`IdoAtgP)rAZhKpeZk{u~AA;fj zr?1GN2prPHt{)&;{3IF-6kJK3`+F*;xR2)iSNgahOhF)ZvaexHYVBM5g4m5nKsRQq z6=StU_0}?u?dFP%i4j@1mQXf;!n(aQ>ti!HE_h9!K7SShI=Q`HoP!CSDLuVNRYOzL zR}7lgY=~am1>N1AVujPEEj~Fjv1xpd^N_q&q8UTBy`P0qaTqtI4^5 zfr#-q&Q3}Z&@lYfmWqjqL8IY-S->4op^3*uMF}C<(B$tWZawp+YsEw=mPj`q_Z~JEW;48ige?~roNxRVp8UhK%MHavqp~5&7-{Ni;J+# z<1_r$x*ia%nKdWS(7OuXZ78k7Wbo0eKUap3r}_JL(BJkqR@g0#BTG7nv``S~Iwdj- z(h+W1b09jghW2)Q*HswDA|j8ZZAsm;xb2RtelY#B2wojrTT`P3?iU+J{!1bg81sByRV74ZF!g9_0GYF`UM-`CtWhXTO5)>ocIxPN zh7_!53{-Gxm&ehl>eGCLDFAkBe4Uh@(K!{8g6PX{3`5@q=?J90_1K% z&`38x3jTjZm)_7Pd4LpovX_wZUqOCA=zyVjau;m&PhmyRA}%{B8&7Tn#qMu<;^Bkx9cPF55)h2>f<=r}vN$2$}2 z5O##GgrLshzmuW*yt-N#lAsCAZ{EXN6uEb4p>(;FzFGDj4XOdtcDki46 zHkveQiPkqlUwVujA6&w^!)>4p(*&se~yE5(Nty!l*K5-#z~oi z;cqR#zd$L6moXBGfNFGD7aqP0iC-vhD^8Go1|ivQmZFhGz$p@ex0I~xatH#uIsH!dnffOH zW>97Mxed19mw57$o@ka?RjPm<>lz-G1uHpmr>qS7-@YsGY+OpAY)4V^1gvD6uP$v< z8p;$n2L;i~6riCSi$HEhRwDg9dbD+=k}DG`c&*lv$;oPP7*6&+WVvgbO%iWeClN~R zzOnV-c@oRCRs$WOq68c`U}(6kr~{nNeGtSK{ryeklyP=S1pXe6iGBZ2$7P*=$KzA? z^A{9E2rtaTOOamuA=4B`A4L@pf-tK1XM!|su}(tk3O2%yCR`6>QX@3i#|jT+1gX>m z>*Sy(yzuMr@;@jUH>$HH8)j^mBQRrAGl(7bFF!lhlpqP<^`VN9&qEbR-%UH{%d*?L z7E$Qw)2Fmts8sBtZnE8lJOP$YT$Na|hHbaDu29}{6fC;_C~oKwn2sl9SN@0S*K~H4 zJLTfCJy&A?ZNa}ZYoGb}wndXae@3lkOHl_(w?t#E?>pDtlO;fct$$_?5o_+bQm>gp0y zM^dE~egYRy!n!yYA7AjbjW)<)kuwrfQqMDT%7YPZc%(-0(p>6Qu<-9djqwiuToYmoQ{)S)O7=KMQUU)! z?c2AU2rMZwraH^E<{u>p3#=>4d_9K4O8Tn;&Hpbp!CZgDmUn>UQer5Oi5$#22;@WoC-;oQtc_ zUHE#{Z<|{Q)l7;9YJK{_3a(K_%Dzb&K_S@O1GnKk!Z>MaY59w{?{=}+kscI`}Szd>31>!B59PGRl99q<5uIR*Y zM$SO#TLn!5=|ZH|-q{oHkIuzo5-YCZ!Rn?q!U#1Hf*!U?jIM^_v>+sr*P4l_0oJ1u zuwiNM7m5O4Bqtq>NP5dtaXmgcR`W2ziG!^Yw*91}fo~cxF;c}AOt)MB+~rGbn#x9e zMMr^^^e?@A@V%9*2#elw6xuFQXAUL)YszRXIdR$p#AvL_$1cR3%AVRQdWQ~WM6B9! ziV@hyhHV$VSSUpukirTAQ+hlQUc1(Rxif2HI%(tC!_+JpKzH$_nX!0QHkK6f1Uou@u>8J#+*gLyOnh@l;H0D*BG z6o+&GaXC8r0?rrjb6m|3ETm@MASBWAL5h!b6u*#20W??Onp&&Am!+aU|rt5!K9NV`a zfs5QqFN_(}j=hD75T8O&Som4r*_CSbXKIiR*m!tsdxrdp$U8!HiV5*GrN~sK+Mk8r zV=f+!cB)X0ed=pWU0A8+AkET7z`}y;E}E-wP1^&T?M|qs{}b>}zXiugO#ZO({c@#c(9nT7h6V(V zldi5Y*xn@!Gy;!CRlQd)UNG}-a`e+UvaS9Z+pe^tBI9O=)ea{uX%s&Ve(fw4i7Xt6 zl{6BPka#i09C-`C%+N4SeAAJqjONQB+A_c%cuS0q5kAH^E+*#r>@hqu3Ykp{YlFqP z*xAoR1_;<0>5XCsMN1XPADGK5jjMkC{P4H-f|D*T(SpiCDK}RspI(Y~S;|!mdad^P zb)-X>xXCb%vvG5aky4O0 zk%RjM^&jjw2K=xs;>-~!+jnYfe>S~D&P1Tf+Z+_<%pM=;XJQ#$fCga9HLp&)W?tVd z-HU;?9G7b;Uz_QYFJVDJ>KYmh!1viic(e@pNJ~x4kKAo?_6vXg@&FX!GmAsr0d6+_ zEK^Ki4c`H5T2@hMY;L|4VPI&OcT$-x%mXiAVJ@VrUwV7<`k2_l#0ktI<~w4W@2>$~ z+O=y}O3FC+QN;J|&Y(Wk@!{oP++^B>LA(Lrd`2doLK{SmwDU1O0u3gITE^Rf_vonM zSP{&jbcQrTkQE?3!!e5#7v6AZV&e1#p@;i#?>-VqXwB1KYS=)j>Nx^64pj^1?f0&J zL8(O3RD2$!Ad)jxHx|z zMvLLx)epT&dPc^IO5&=t3o2x-SirioV~soL(;s9JGh(dHWqOT4#y>z@^(mR7BZ4FF zNLEKa5Kx*h>=;P8{$vBv3NFSGl;jEfF9kXpqP{>MXKgSZ4c2f8^c30SXl#i8D~4nx zowyQHu%IaDu0lo$$|y4-5kv}$>aVW(#O{;BXk9F+bLd;9Q5M#eD{R{I8C)17eVce3 zJg%T2fr>c*q?r26ea2ff$-S3vLBz;g?M3h;%84A0mhI%-)_=y1n>9?V`@G%LL#fNh5L!I9(4#f7Ek0sGIqGrIcqjx~gczCF-jSgCyfgy{} zv?<}|#8vaH{kRiEfZ&?RWF_0jh~VJGMCrnzvwJs<^uZ9WXY2{F3Awe%iwiYZWy^mt z6ciMs;DI)+$5;<4pgdV=Y3cNw9Bdaz!8RoSKoJAXr6A9RO!ZSRGUo(I00bCig4mG` z3OEd{e?o1f=yii$M)#=8uNG~U%IW8L!hL75aeH++4x!@vGczOlGO%WXlbgF6GC?g! zx}I!6$xB$!dVSW&J(7}=Sd?;BC>YWNOjDfuN|ATOh0e61^JhjJh}5L|vI?LDLU(}~ zK-uYvKO{6LJWOfMxLKrP#qpzKOrhzw06l;J-C(*J`dzhJoiR-X4v93#z#05I8eIR5 zM%Le>5fP&=hNFNcFk`VeJ*_N0Z#p!3s`dSQe!Q|OC@sfxQD2&to(@~_n(*XoEMO8? z@%y0A-%zp@N&y&bT`-k-cnZ<0a1CjQ#1#FXQ_LWHIbk&@Q`CT)89!2p*;NQO>~4(!0R7U8j91rJ@r_9yhy zw&x;-5nAgb$KnN8t+oA5O4A=rH1DaaBTak_$NkIU(Cf=?MwIis2P(VH&`wWRR<|txX~rq=6Mw zj+b93Mt;NGknQF8c*$T$sea7hnR0>rI{BkZ9nHDZkKGVd!M7jz%W5?t82_z{!JR)i z_8KhA-x;^i(4~!vqveE;#>%7I!!m4b`7lBuYk*C%3^^AxC00Vw6>XtyQi-&5RdBp*;#y5;cY-xvcUp;_tGiR@8Gze;vf@% zO!}7e_z}fG2Oa@shT6WJ_pb4eLPAZ5op;*E@KS8P!Ld5?h$)2%v(>xWODJzEp~0**0$ zH=b@MnS1x{(C618%b;lAzGFv#IG2EcEGR~_=0*7yy@Mz5kxkYzMmgpRS07#7z5ts~ zS^*GFPER=LIRcFTC;17O{n~*8j;?$&Txa-_d}oWrd+`ui_&2HD&F}o$6_}az4{rDY zCjpg2(NH(S70(|TOK&UcFnr|RUwwK*K+_PM)%QgGC$@Nl;7*cDk2lPN`k@gVEbyuD~z&8@9glixo=>9N_k7l;>Jn(&%6>gQht2JXJoXiIOF zjT9n560!m+BSJF-Lw$xF@d9YO$d!DktXu%=bq#Ip;MbEXNSPOf8H-U5(l z>_<^XS^GlK4W2a!txKLz;nG^%4Oq@pLv8d4(u*FV0NJTcrzciTn9}KqnvNf@!21Q( z1iu?WyK9wt-GtX8Hy8k*+(Iu*#v7(S%;cb91oD;+G8g`;zbJI>7!sc^Q;$gx2q8g4 zXA)A=2`H7l;Qjme{~8s4sVOnBJM*rPbca)o9^pOu;Rv0LUUNV2oD*SWp?-nJoEqDG zKQTT&1eMp$1l~w*AVl_pX^TnrwiTE{xOnkma%Sdk0Z9o7b)1fi2P0JmEq<*N7iR(x z#kO(U55ivX^YepzdZk4l{S3B5w-GB@NN4~n1E@nJKjOWJ{(o5nwre_bE+@?SqnklL z+{cF{vkPZ2HAEjbLJ0fc?$*2D`M3(c1DZrgxtf#n5{}Wkvq$9eG9d-f^MZTT!kIDL z%w7x>5xLHC<6mPk5ICS;X(9m#F7!SY@T|RkuD&C2o0h@HIRW3-*6~u8FEg^Y+z&{+ zAD}Srrk{!PJV}F0?QJqyPIGaFhYH@7+)Ux**nHr*+B>#2QVjFXT~s(ys}c8NgG zU$GOf!d^ywdA0jSMAFt_uZ^PvpA$7Hm_v0_C5Ov}c&*?k< zLMkwg9ju_VckSx`a*?70S2k?sppC2ZdHD5T3RwNP>DPUcdASoh6q*tV*I|lbajqEz3)3&Ov~x#mlu30p807sHS2phIqnJ@R>R~uaXqDEhI7{8IVj!goa{c_;n{aH zpeSzeGkPYjKfmls-2~X9V@FYqL(u$r4rV=c9Um}A{XXLHMKN42Ucn&_aa(!gYKl7Y z6_>v~*D647@MGyiGO9xk#UGyvvO>}uCn4L(JjV#l!Y1%vUq(EnZq4_Le{g2>*eZy) z)4@VIATtIB1?9#prYKq-giOH8I?zB@@!8Kg^flZxDLQ!*RFU$Qk6Cwfa$4^hWS9<% zBh~g3nEde!e3sP&SsUO0`u^;;p&hFucA$91we2cnr>GLeDSrmF>Jb0(n-oPwK6g;Y zztoJ_VK=lfl3R#M7CySRO1y=?O}5c<>uPlQYCL$03JMG?e14}@9w2}B*H6zfKGm~D zdP|&Mnwy$twzRa=g)X3uJn0@uM}p9j9;0vx_Pjz{y zs9UJ;tyWGAiwo_weRd0VZmWwurDhEtFYh$0o|?f7R5`B-BrrAmSi{Y)y=pe?==31k zg_=OQ0Fj-o7%S@&ko#fIn78qq{O%;d;#UX4>mF=HjED?r03riV5TRzn;mLzr(8ib3 z-MO;?k$ObppIW9t4kY*O5hZKbHEx{&{asj}JnaKy8$ z1PP;OJa47ak5n)SWmr8tg>gO8_n;c`PxWZOPT1099KlLGeT!+Ze7yctFzv~xm=JGY z=&z$F*1HHKAOgmoE|Kj6@v09X?2v>HE)7{q1#D3u>9JeW^O}T>!JLZko))U2qT9*b z+VJc7)Q=_SpCseMk3n=X2$u&)&&=Yd!@#&=_(Q<0US(2TL7^aeKLh1=QnPa()#uSj zf&?WP^}Kx<-gCp-G2P&|l~sPkGxge>Bf{4TG2H`rtYLvvgAAIXp}F;6AH**^YMd(VuUfWZQbdqR$r z^$gUh%?%BQoAVw#!24;h_M0~!w<&w`VJ1cfC9E}F$iip6g;(e`*vT+waHAo53~{hQ z;O#~=uB_f5gB53(p$X?LKt0>Io|zK5+^snEIYgFY>B5}WN9H-?#K>{_O|W}*tP&8Y zA6;SW(*H)(8wIPDG;~C0ju<#Px;~m)=z1H#!RkYWuWxqkIT=BorS?E4O0aX$gSNPo zXynj{JEZM@@{?jXIgah1tx(+9h_OsG0F9&l*M)WgZ4|8=x#Q$c(_Pp`kB65lCA zsa<(!*Rb_nVs^mtYY(XlHj zsgyLYdzKcZoy#YhPb5t#z1LhPaVg81rz zlqkbf9Rq_{bMJ8j9G+U|l~(cfUz_lPUCCWmRqfE%po+F2`k9n@77VtZfzu1$2J65y z1Z_d~-!nem*e>k(=@Pdq3W`mWh`~)svtwefugX0$cHi*fZF6&9L92NaP<^N;=C#Sl zSuZC8^<%}qAXM)=yB{{x5!SgHUeO$g8r<@aNSfItp#=^(}|TOCmI zR)t(e$=HrTlCMovNkYSwFZkI=H-?o}8;(+HV?TeNMgFc({f+g&2AIfC9D;CKW5?Mh<5w%L;LrqnwNPV+^!YVI7iWD%goQP6fgY?_NU<8=Zz9*_L+;mF zxJJ|87H9z-wkAv)ggHTy^T}HpL8Ap}4PDz8&IyAGIjtFJ%G3;wo9teAF#saaK8UXj zNH%$olk3*$J$V=Bk1J~4<1D-5$6Ekz@SEE-Ly~g~m4UG~F1pa3^*qhx-qT~)!w4Ub zo3Fi5pWOjpcNc7RV#4J2xh29QB~1lu;|g_PP=WyC@-FRqAfNgwX` z&dpxmeBuUh=DRab5HDycC&Ve&Oxpo|;2l(^TEOZ+)@Uh*C`GaQpk?az=hw(!H~I;-4aw?=!p-CD zNXd3wi;`eM26| zL{634Iqh)>c=v^SgDwhhpCYDu7&A_dBga__!7&H^qXTkDg45B_#RYmr}&Tl)z zi~NRAafW)U3cNmZZas90cR<#^ZEdy4!4pQMYtFU*li}IvZx8M5?21(7Re`vA z<;$4z0UtQ*t4$OKKYs=QMFxK7IQ=yr^7HfiN{7og+JozYc~Wi+6U8cx-*fy^fIx|l z%`Jfzzv98{Qq&$8sjzm`*}JzDcTrh#EikPevQ3F=f0Ul2xcK3Y%_UnaI&e=u}TI$v|&gTKLAn=cs_A8^GO8v!($xGu5G1_&=ANDyeM4pX%AEX^*Y4C__S*HFsSI+@zi(2$Jgg4;1E>b?xpBN{qd()o9lDH?xLy-g@o zU(I*+3J^S+KKw3M6V+BCFALSa2W5lYGL?xnf54XTb|d=#RA&5evB9AY!JSky3Wpz4 zRv&O7kv5TXb{-&S0?J$63k-t0QLOS2M543nWCp`Izth7VPR&{ot6wubrdZEGA6am; zmMgUd*dlC|a*MtYwQ9{+g0i&qYh6~?gCvH62(@hf{P}Y}c8FD0ZtwIup=Ki?DY>x< z@yTfS(j|rrjRV-Z+K90Or{=AzBhX8^2T4ym^QHT)=n9}hNB@*#OoZ7Fp>L2#_TklBKn-&-RK1B7A+FVGAa z`b@J)@-_rF8jUl0`R4>-rpdB-Jd7|;prY}2RuLn|QOm4B*ek%)VHqqCanTRNPs4r6 z9^udv8AbujaNyP*c*_&?w4^0JLDCYF`zs}Yl>2`CN2mt5l<{Av5qw+E+oF7uPi4U0 z)&O&sZc*W*^$pw!z0P@G?p^xF$Ja({J~fEKIMn%&Y|nf(`H|metA16FR$w|ZwF505 z+(>srDazL9W^3VJkfH^&U1hvpk(DXG?Ky+rA>;^TFvn(%9Jg~jW3OKAo0@aMKHW6= zKzW~v&bJ8=k+-7Kk%3o3RS`gk_OcHp!{yVuXwGi5|NZ+nieKtD>3}xBd1Kj3?vzHp zS`4$w*JUbViP@1*?%yB&vG1r+bx|OQUTJi1^cE>*9qM+VkzAbNAx$ zE8v>z?p{KvdDuBQy#>(xqF6WH9#L^qPEL+1dfvnzkB(mh899Yp+}ET0->WhnGld@3 z?@v_Mrd3-#_Mwi|k_Hopky9fm-~PfAL?&AV7|+??U)^x4H%;=&Szes|RlRK>T>1Mx ztmy#hCII@<^IJjd0g{^_vKTAvoY4hLh>L0-aZBci)GPxC2k?kb7 z;b9D4Vz$k?f%u77sV;OVEUB8->BcHchJ@M=ha8VXC0SUu1>7Wsx#jvdhp&75GwHl9|4}cn3 zW7Y`3O)mTH-P{<)hfkeXeuEi7It$NwHI!H+*q_Wm@oTo^-Cj)5096*oFsbc0xw6`= zZk4C*GS^$^@jO)!Bb*KH-UH;6We5lN($m-5qXmbv*S8ZA!dAFwcHM)qXkh35BD{)0 z43B6zA?1?8nev28^CLHBl$Up;S65dLK3OEte0hy#cI&cLO1JRJ`KJ+8UJj0*);2`N z*ogj{8)~Z5O*VBad%aR?@8JZ()6Wgcx7I?SxS6TeiWXE$(jxQy-h-Ci0@+VpItVk& zP3``Bz7g-gDu!b)Dn+*9&rh+=oPo9nci=&O3sBPiRLO)Jb{CcX?u|z~OMUl{c*#IfUYZv$N#}93#O; zXanPNl$WSxXJqULrUJPT*4qix(2UL+7F}Bluccf|kT|R`5-m~fO!y&D2EXv0(chuY zSo}5aD}!$i!oq##(ib?4fu!4meGfTG3c}Qf@Hv(28S$}q0Y6AJ^C6YAaGkA;*35NHYawR9vcXm_&gH=4^m@DrGWWi8R%eV=cE=*g7&VU6_IQ6BbE^slWW~}5p z`v^^iV}u((RFFcpFD1VZ3hP_1-76L@3%uCM^eC#d%uM*iLc^>Q*XhEAny50SU7 zVb!?{WLlJwnb}M#W`zAs52w$n;_%u-3kn7-kv}^ToQ5H$?M|7km>6u`i^=8n4(9<~ z1zLhS@{Wd-&rBR@Rq+=PUAxvE1>E%wM}t3qwsl=jQSFP!k($}=^?#3)f}p6bfzu=T zg;bHkeEk&PQ>59N?>=mflDx;$373*@%PEoQT0+Op$!^@Z9~YqH?C+oBCa7>>1%z{i zHgP+$6=6u|=V;DssZkUFEK=ve$4k|k7nj)#Oit&_xj?4@ruhyqf<+FgudidLJVJSk z9q+uZ(|?AYzwOv68@v=+IRi|v4Ga$EwBm~7W1a^g>4%Y?-jPtv9G7!DuS7>%g&s07 zah?5hW_-A_Q(uU79tGN}Z9X5;dwP3KwC5*F-<<41?-?~%*1KYtJ96Jq9QZF6KxIyK z8F~Zy-pkCCw+AvZ(ASRwB3IJx`+XWX0NNSpGAX&Ztj8tpE|3eqHWL9#n$*XX> z`Mc2ulGqg26Xj>CucvqOVW-a_(#!M4mPlN``!S0@y>ea|8xzA*fDYZs?+Xz8JAJ0J z$h7#DaI0po@qsiL$ryvn+B^U<8?&dedXiF7G1cAuKQaM0Q}H>x!39>GxZ67T7Cs&s zS9$pAlO62r?cbpG37a(Z#oat$Y@80Q%L~+2I~P$8bBc=(4nGWIC3`7oVzAre0HKbqi6 zVmi9OHwHHXJ$uP*+mbqUk-*mUpz0T{-ApqHcl>?ndNPW%=0L>-#lzqR+6j4@Ct2FM?0Rg=K%eqTm4ii}gUg#DU4p8M#Qrr$rwMHc z=k1>BI*PUpf-@5jD-3`kND1@~^21x`Mi1ur&SW6Cwe)(mpx(Q_#pU+w&np$_$;p27 zQ*oq3!&jN=+r82Vfnw;e7V?gt=!KF>N&oT7RI=G~4A&{QgT&^^zOb9F>@L9&Idi#rv#_NFZNG>DZCAlr;IgGq>g zo4<^DYS8EW`s-+8jKukivkNhP*^l-Ro-^BH=asDrj$q)^B8N_dL0xb~jkOVWdJ&8&N$dXA4*yfA0W3m%4w}hQEe5U)gk#hjK zSR~W&FJpFrZarP?1j6leqE415U!qcGhPKj(kU2mJ;#-em_`h~fAKnpyaacQe^w`MA zn*(tK!TI)l?9}~>IUj#%|2%YW+oswS zf7z!yxw(yQVVmsUh;T73^kZ@Xv?Ccr%L%PuKGMWfqm)eD+zRA$_v|TH!Dv?X=~+O5 zP5tUrSOnZv9)3jX)F~J?w0qcWfUqhm8^pnW|FkZV%nD)D;vNHpd=+)(kR z!m2_n;?7{^MwAXJJ{1;7_HO_y&E6lYzmWw>E&q%R3};dRMtKV6&0lB>;It}^6NvO5 z`^Gyt{K$NG?`#}HWBTpeS|=Wj_=5&B9h^fqzYjEWD>&pkIOaOc^OC)Z#h=?A(3Sbn zp<=_=?N)&=5pjW~lgXGfn(5tK zp=Y|%dnd~Dy}(9T*OU77hL#u_-4)t7IM#po} znnB)_TFXGuqQFRPs@DWH^SxwH2Ho&}vANTzv$N0ep(GBuzVIGJeaOUSuC z;tRi2;vkCrCeS}`dc86LduttN$z*U&1KP6M^^=@!rl zW_}t`x^gfvwts*VfZX+MzCO)m{5~jp%iyEh+M55TF3eh%Z+-3pTFDKnB_@B-?C1YZ&dV@4%9dNR#J#M$AN`Bw zGB5Yk36$qNd*tv{e%Vs_%wN$MIbdOtgWL&Xyzw2G%mL}?hFdoq@VOI$yDx3x{aw6${ICV?3ksqz(w|E`&Sp&o zo(8?yj9)Ue9naKkba)=^N`s42-KPdULYv|SV<*>(bgX(wBw#A-fST*#vK==7iY1u+ zCf@wlS*}amH~?L;=aWr8d`;i(%ZYmp6TIP;}* z)8@?%gDqKFP6#Wesfgs{R2TX1gZq&8nsC$!@~dmYQ3q=8$E$ZQd*1^*iXy6dwbv_! z6Rebw9>{n@tZ-HnJlu>)zD$I!LxjwKdZ}BHyihWNgAD)Htxj;}en*>Cgv#wjt33n< zdRm_Q%P)xFqKr-Z^D`@8kh7k>Z?n|327Wj;HZ;YVRyZY=@g^26CkAG=d;bj7t7^b@Z7(O9>2F^e`xufN(X$$dGaX@SZN;(HOotKTzt) z;eGgqZvg?qphNgLULzbKzGX|B{svkwY*RpV@e)qVi0o(vN92I5#%zh4p5j7jxX#X+c+U1U;%UYwYx8n0%1?F z&t;#vtj_HGz_X|rNxu*ULf90BB;{eD`nU8Y+`;V3uxY#Q67z4~9Vrwx-sm}I+bS{*ii4nkT|t~6 z!BB`LOSt_1;gJ91BLqh%eXQ{)QxIa)=gqna%rk@LT_7C-4;BMom0ZI0c zjP#cEqRMcifu`MzL7A$etnSrMEo@U?!YXADT_NDrBT{4Su?uWiUu%3lhdNV|XQ_$y zg_CYQFFA?j@4u?H>k8<_e?o#EA-ECMQZyuLp-v0le zS!*ydhKY!=j$Kr?C~2}U*`=~%NeN9UB0`24W#5SwOSa2aq@>1D3@W6H%D%No86~3n zJ&*G;_vgO9zrQ|@&)nLs^L-xg<9Kbyv18rccxT5FE+doe04`t%ZA$D@T>Cm>gWZgbI>CY5`_-=Qa)JgnMNA5J+KhAsDqX&%8T zQc`;QUa9jCp^K1Ije$-1!vhALrjo{&f6Q{5H_t*;Ee;ocJIF~ZKfAmT$kP9dPlN@G z2LR5=M`x=*x-soI6^JL{QQz|7dyTtJ4Os21q*d@o z3l!+L-<|S=_F$8Wi)5x1)gKMD zbuE%E<$*!fz#~$7T0HJJb%U$Lf~VFl+3D$5%Z+0&$~1lf<=604mCF-Va8{!^SHD_9 z4n;#P5jRs)!Qzw!9c1hRQ8tPOqIdgL^h~;rj$v;e(oykoLj2kvLv=^R?e2znMy_?H zfp=d#tOo+k^+*EPeD=h=`AYwi`x7$p0NbzIut6(Gr$B#0Ru6jKbG`crxPBKXL3h!e zyegKjrIQevO#dZyPpjc+O=sV{$UMUzCtgSeFj!sBQ0y zL?^|+PcoMIIsalexH4Y7YO0_u6+LqEGPs?F!8AN~gNPs^Y=8{gm%_m%hw%30?&JdLOQ-m=P`yL_o_` zoSEGgQ@6;UN?0BlmyyZ=(Emwpo2+}=bhk~_S;YM+6Sr?Oib!8H;wCuLfutq(JD4W< z)IJ81C&+q)IASD918jk+vb46b#jQn?&T?odw;PKGoK0byb>lD1xCgm8-=+sCX%B__ zF|DJ~v+Y0+P1hdLFt+Qb%H8D4xv$uwmK8G^9w{q59e$egrw=6<%&xmz&F`Bm=;Tn_ zduCW^zNJasI(de|njAvN6#;}ft8MMe<8!7zp>OW!Y!G-l?T`-|j=dk&9HDV-{DRg< z2c#)64W=x8zxsWj{1U%t%+D!1R*3H}(+F}m8q)COc{ zE_-#|NLDlpB0Qd4dKAe7N)T&UYbga}&MZ+e{02 z=l{9ha>%e@U$3RkadlnLcL3XF4#<@?Z<3y}_DB9TM{>)z3kOet{jgzL=c?b|$p_aK zu}S9Mn~x}oeM{2EG`3o1>GgwRyA9mgmFj2*S}=!zwjSP{ApFBqMd`L-`PTtc{7bem zfH@J&doP+%NM>>S@`_SQtd6?^G1KxV##mHSY~y)tPdLXr2=+hnqcsuN>1aGlUTzr~ zo3V;}M!$!gjL8F~Tqs{+!s&ia-Forf)q6wWl*jdyr`=))R@rj1l@K^3mmdq%OS9Cj z9{6BQ7FGFCUx2kg98I3ouGD zm+Qa$FjyEc{2&3gT=p_L_wz3k7+!xihuiN>h%X)N^Gpji(>+>VS^j5maC}&*z|O}W zx*?12a{&3$Kd66s*X~AwlMK~yO*uX3HOhErOQ7X0JbtqI0L3IY4eWZ z^}PNs!9&l?EvP7yMfY?5vENXER@K``6f#gv`MYNCyMvZE!Y}v}%upUXW4-SqEIol} z!nS0&kD~k+EoheERWSu>(BaICKlUwt-6+_k@9rmdJm`7?qn=-{3SvK=wrRusYh<)7 zh2s;`Afnf@A)7e}3zM7r8bu@d%Y{o9E{&trj8``46Q4aa90cEvl+@H7=b9mlBnLW& zc7%#?_G~aIt!J6zfw;brIFr})uQ1X*3o|d=TS*Yu=mrJ$>PAHe_fR zCihe?4*32zZtdz-R^+=-Nu)y|_N;q& z;fQkXH?-FA!~^#Bd>&|Dy2vv2LrNczsISHq<=6!8sgo6-7U=9rpPK-x2PCzS%tW_z zg!9)vOtd{nuavXD^6OzjP4Y7{7#RDK{v=u(3O+ z2*9JhJS@lD9eM7-R@QG+jFlLXbgX^_f&Cw}Lpz@Y-W#0vtK|yUr|%chmCQ<~IXy9> ze-@svXnMs63(x1aty`Tks-f8a$;Fe$JZt9x&e8-$D-x9?qr;dghOBf&Mtp*avr#l~ zBe+-4lSZLcSEt<1eH7QDGDWZ6Yd~LzIG+>8E^$R^n;1OZK{EM7Kum2ios96xI}shN zXH!TdYPW~xPdXCXneh4cdkdpbtqk>DX@7+0cIWG*$(TNU zVxRhVuF#%7x#UFI7r^>?Wzw$UbxuT0aX4{T3*W8X!QM5%xs**aOSCHFY_a}TVubA?64?jJoVgGjR+q->eJx4j!u6Sodzgq)lG{WU@`hgvQE(aT|>GCQh zx4)Z)X``&oh72D*j-6Kacqz|ruKw&GFEheOGq-Pg$J_dSHSi4&3|3kcJc`racU*w} zqhWnLWu7d>RJoCaW#z78zxOr`d-DK9ZTsftyQ@{v6AVf7<~KxXzug{MJ?)aH?M>@U zS#kY0DE)ZM_L^BUQF4tZ3jN^NwKGuNBrO8n&dWHl2M0#LDQ_^v@-I&+$MaL{N8%I_ zPy#@Yqrv@G@bGMMWH9N8+zjVK09pr~r7_hmeFp_6!aQEJfsE(j`CUZzC+Yh$q*5SfXfbs`Y|5|4;nOzZ7{;i zxMFDL^E$inM@Sa`I|!xceXkpOS82JRkd{mxyP?|RxoeaF{b!rZ&eW{i*apo;V)Vu` z^p@Ue9=-u0hOq%Nc;A~OU+A}w4jF53kUx!f@7Bi31H;T)eZ5aNN5N%a@Y?_7p+ko_ z6*LyYQbcasu|rfE1-t7g^VV-wfW=(bUwQr`^>p?Dn<*O(fF=_p@E_p_4i7 z5t#gL;qt2%sjvaSvU9T~mL_ zFO>}u%EE){{Li9ALu=&>u}M8mTF~*A*4__gS_nA~kKoQM$is4y-DI+qCmk`UYunlZ zd&uiL!t9r|FvNl`UD~qI?QPGH#iTuZmsdelYWf3dsM7?q|KKJq;%~41p6s^O!O?N9 z*<^%>kiRhn5V*FxX|OyVp1CF}$#qYDwcQ3_9A9*~R`13C9B`jZ_gkDkX5<{iq__HG zaFADCF2l%~GiJDwtoe`MoM}}Q>nM>1P=bpQZR8QOrYS&fBPCgG2TtOpp^nB3k_^)B zXht<2zK<5}U>{|t167pJ7lg2gY_cqWO65^Nj0>+T29E{`5LDa>bdBG1!HEKfTEiRC zf%MRIAkS7k6h3ZRi`FT}_RjM>=Yl}i=s~9u?K~ato|{QMLcY<%jlq4#i=Mix<>P9L zPo#fkQ0`A|Km{*?2bcY%ZymuW*ZsMk_2F6`ubn4Nnj=tqM^Q4CFRQFm{)lm01!c#N z9edS(yfQ7~!i9$mhfRO>`}G~DM}oRI#w1?o9eXitc}>pRRy4Nl>ZDzOSPSx8yttRP z)hK~b#X2SL)j_=l-2s<^J-JQ~w$c~&L}Z-N5@*UYi;l$?yS-O?cOkl29;wGKbl&HJ)K`u2V+Qs4$R+zhJf zt==`ky1VAn)8kMu&4Ys8-7#-7323cxNXq+v4trzdpzkvkMYC6@L?$A|yESmQ=J%GQ z@0cf%YcZbP$UzQ-A=a6qXUqEdjxHp_6}a(F?=<&bvBkJ=#x)?fcV}r#B|~oh`MHoW znGR`xz4Jq#h7a#mYc!PigXl zV1_}RCT^S@R}KNAOIfm21>v_X5H`hp%tg8;b|G<9zg%iGB3qea@vX z@9yFL?by01mx*xJXm|YHU3e2kx|+F}St%54UhhtsX?yYuPxIW4 z(nSKCX;!xosumBC_o9AE&RqWTty=dHKj}OD!XVt=)hnFa$G9SkC=lM|D>C6=Lh!i( zLKksjc0aQLK}-JwjMOz5+uZQfR{c|5iXF^5bhtVChtLlw(8P^kcjEZ*TO*7DS2OJ& zgZ7PF-S3NHcnP{Sh#FI;bl2rXan+zZiXd!!zNm?22^XwcQWJ6tyBO-mfKDoHM?Vi_ zw*7ANyL@0LCY?=M*zEO$gtBd_1F%6_w5le8I0XL7j^s8!;SfCp7Rt^)F+h(4)+l(d zClP{wJHGkyO*wK|A}NGeVX%!oNBEFx*j%?R?qKRAH0%UO&ripO@^<%?s&a!wMvfxf z%|0K7h)#~mm(~yPl<<_2e{OckNAII4fI>|dp{fkt=!ilN6X=BUSCb!qji=~Un~C~Q z{IPlL9ZwS(9PWX*bLi2KwhR)5?eDc4-TkcB3Er#h>8iSpsrvMoCX1fxX&W1Ia>F@9 zyq|q5zkVoVoiCo)Xp2(80z_f!c4KpEjc)`8BD?{8$Bj~#nJ9Q@MCyf?1uy_xU9 z$uo{trSesu{w{oO!TBe0$p-r|XbDXx$juGIIyQR}#E7Ay9Qbp-NBmWs>3%X77DVI0 zNSjXI@alcG-q4PF>q@eh@EsJEPjpCl>==Be~|VqN6^4m^_sE z@g>{ER>Z+|nGCtHtUP~;95xWSsI7)Lx^)M!V0W@5?6sK8AdGR^}Jk)9PWV4>HaC<8TS zk~e;p&xP6d!~7^*;4dv(MtBY24w`63UHLRqS6901-Ni8q+-0!q(v-&rN^IBlTYAiT zaB7-PMUT`7E$R#vT$`#Dv*w3g+WI0m?a2&?&fKm5bxm)|^O`qR15!YnX9jV4I<^Kl zsqa^T6m5Q9A$b?ah|!OrXPMFQ)OU(Sph>FD2H&-ga2$CERqQd(o~~cx)UCqOxQ6Ep z;0N=tNoJ1nH?)YZtE)Zo(O;{R$)ePjR&VOW6Q{X zM@VG;?TMasx>u8dT#Mt#GqZ1oD^R?x>NWHWS0HkY46+;#Q_7=0&vWDR+PCzd!9w;S zlRC5Ut_BoI{VUjUY`@`;KkguBIG{>%Yv8))JBvD(j&@`@q~9G3M_JkPg%pBwGGSF( zhBv3esd0Ic^7M`8UMhutm*Y{i7lyO^S|=Ty^4oDyA#E=^eD&z;7RlF*1>6Rsc5GI2 zReygB0cOWIXVyD@q<|pjF^*1~weN1-S`7~$K5QJ##hDVJXB(3ClQQ6^dQ}2kNxPdW z?d|aLcy0bTl#BLpn3lLT#d2NM-h{dA68GVR9dnmevt3D% zuG{ZZ;{Egs97~7O(^rcyuJwSZo=ZQSqAk>H=xee8_-eLkJtgsw7|L;u*F;+&Nn6yEr(J zYhp z#vrR;6^MPdrqOcllxeQhQ*A}g+Hv@;m!B4OM@1)&(Fh+B8MNzG@Gq<4DA*k=(DmOM-uzuF|# z^-m)w3gC$c4=OJ$EOr0SoH1w+U5>IKf_q=p)}@p3Hfcc z{#OCI9L~-P29ygfJGrc+LrLzvLhfi{bMT(tnj)_9HM5jf;0AAKET7W5bT*p2Ir0NcAB%Pj!fTIi$$ zW{-Cv8WJu)xYRgckSE>~Zk)=yV7E?LoX04VM>?^n(f3D)e#R~>bY9SIY{>JMf0_Jv zWOdpUf-Q5gkKH$(e=&>*huCw;Nl8c18%XMaITt*aT&;Em?)Xfz<|jxSX;i{JDkP8U zRz~QWy^pa^8u)blei|xV)P<4AN9D#9OWCR#b@E;Y^5mb%NlCK+aO2Wxrc;r8XkVxXpM7n6{i*4!%|)x)Wf6ReLZ@dzuCFM!1heq zp^K)Kj z6sJ-o-IovmANwYuw_~O;iZ*POAXh~WMazHqW z0BxSGK^ciE$vrGI?GTsEI#)6{8%HzW=YtqFZB$q3Ew%{W2c#U&S8z<0_AC8xj=5(~ z#a=X++UZA8E@@V$=!KsBBueRw_up?4jdD^3%z?$p^|#415ctxB#VyD&YhDz zJ04t7huWdj4(XH}^JI6r&eCHK;PK$pUy&T9IaG2W zA^n=vVw~|k+``^GPE7rbOijq&2x1lvfrYO5Rl8z@4_&%y{Y+c(QhA%jDab$HooTmj z-{Oq|ceZN~ew#BL?1@G1n_DCCcWx+vuDEvwL#=KVEa%c;5n6WVC*syp9Uojet1etX zCe1S&jW^rEjgDe0PxPQ|qMB5L1XrN4xX0x|!NK>Pd>glox%fdl0*@?~Q1>W#I?zV!1_xjM^A z3+GE4d;T>M^HNcrm%hJdp&gIP+Bkv541GV`>P`lbnb%8NlVvsj|3@X$qTju?^A77J z*0tRHPq)R&l4E?f!iO0}3k>kD94RrJ$L+H8>nBgTzSxh(kmJJnWui;WTKc)6(gIO@ z=cvy}qhk>aOt@WG3Iesr(nNsGNQOAMA)5Zl_OriA5yfpBq`mshkQ{Q;qu3E*gk<7%V$zFX#P(+*YfAcUoWxuN4 z(f`vRUG5J+cdQ{T$=9s{lkiq{bz>g?7au=ZB^I+&H57^H_zXQ_=)qkkCnOU{N4*&9DWPXUF>>e>@2SX^1zNJcpPsJb1)G1qkb8EE=?D;g zOih^uAM|kIu29tjxZo~2^Sy!H3{t1pvxmQ1qT;ktk|#52Ni)i5v+H%761$b-E?^wI znrijgZMpc2?v(7cTJ9`4;U!oOV_r`jYL(3Z{VuL#d1dKxP3hRUa8^`{WTiK{Hv5so zhNZJ)?I;Ml!w{O-+vqrHNBj?nD%m6BLM z64G;x75``Fkl+n>PNjdGTw70R@$gjoqJ6UaGK z{pL=!GKX629hj<~gK^_7H0)g?V32O)>CmB*j6C&>jq@G!?{i}A&aTs_n=K089&VHs zHxG~aS4;77f}WC)lU%FMog{inHpJN#P@RAPC;=h)+*q0q$J5*=h8}Z0rsqDnG^$=a zaA)v)FsC_*`nHjDa4wEx#FBoj;(oV@qYb;3zJQTk1V~h4@297f-1y(_&gXu-bO*coOY? z3weXkdmH>74EuS+uc)aT~_d!?f^GOs?AM=46iw2q5@pKS~QgeEuJ~R4KUNO0>3G_S? zlg9k5nmP2QY84rmbGl>k#5FmFnT|)mT^Y;H*RPQ?BTRc3Rji7M)i`xiNW@dWTg>^j z_@6K3+@;+mR}JA~jECwwzi{F6_QG+`G6cAkOf2aWRf$@69T0&s07)OoA2(jSpE#^~ z^Cp;pjbCt@`t&#S6xmR0>UQ96k;?uNpy>qW3*l|&0T#^9SkMddg?{6O+pIHucB;Ln zy{)L%jV1bxtt-M7(*F^w-D>Z;gVM2yzKK#(5{=&op_)C{HK_EBgr`jnoZOY%YIaR zXEO;6&@n#4VSHNg21!>WPa1Jw)P+=DfMWE`?gZ)#88)M@Z|<7t70%1c7fN7V3H}KY zDQL_eFxL(QEI}ezr9F*)x`1&jRU)XWGcl7}t`JELUlA@}* z?zPJ-J!xW3_q*Zw_uy&IYw*FfZ9O7>ORt9K*6@^Loxh*R;e)=`^{q|AZucSg&uQJv zp;l(gpzAgE!c&p>*2kxU@<%sD8ddMBHmBeTlU4e zVW$D2NYh3VMR!M3h|G z7k>!K|?^vxTyD zMp&(bt1Y_6PLi1>iPD-0gyiAh2JcClOQUr*b|<+7)mDf68U$?K;c4#obVko2L%$8N z_?sACd@hHM!QI*T1wX8VWKsdM;Ulp_#L?YN=Hk2FeZkhn{AZ6IXZJ;C=abcReks;i zWJeN|b}TeB5mn~1HpP1dY#aT9?VnQPc#Q=-$rG2s^cIDu^v-QXCjWwwLhJvd5>iM7 z@B{yMDjdO*&>JP1ZfPdm|9y1yb-Xh#d&+du)Mobt#ClB14Bnx}C6p~OZE6;GFZj!| z3HfqbP2nCQxZ;+BeLr)<%%e8rd(e2QU2(kyE&{EJCyTzI`@5EdtZvDNr)$da%n1$-UIiCmCFTs(;^c6XBxfZZeKsdqTHCA8yQ-zUzHyr)@ zVJragA2Z!Gm5*udia9S{^SD0T>FOm_vI0*i^0DC_nmunw38l$WuOLPk)k4~iHlvvS zBtY|xPY!NiMw5Hlam4waGuvLvT(75$C)sbD>i)Bz2@Co+dFXCDjPOlAQ9u(IdH5f^&&&h{u{3{D8s8-To7zK(CQMCN2zn zlde;^b*o0Bf=!|1`}<9Ov%~0EvA|gW_T)nfN};XQ5;vfOQq9709ApUP*&I01sJAR@ zF*UK6YjpdnUh}^tfU_{ZbRVvtU_bg{p_7Ry}bPx~jJR)6#w} zZ^7}ubcgYE4j?bJqlVWp^B5+~f08%w1Yal+b^VG@0Di56S0-MAC>DpRJJ&dWxFf>M` z;$RB6IOeyj7qH+*zY!t3ce}OKZ(+m#lz)j)=ZYB+W1khupv_zeC>Vkp*ZDy&PG}tg z)b?eOBrz66Su}}9vD{RSIq4iP zBFLgNeBekQlJCbX3(L!o5n$(#(i7@S|)tfU}o0`B^BMzXAFrFUCC z1e-5`nmU4@FLL&*S$U_EOhV&@P&JS^MR0N`vbZ zhM%-;I)d=4>4#?UgjH&f44z^IwRnmN6Yep{rLV-A*1Cwl3llm*0+Q{(#eKWr&|8=zrdTyjhe^zi3*)(CmV+_?P!Fm*dRb8fMM*5389 z(yqU z9l(8-$IzYz72=eImOC=;UF=ZjljpBos${a$^m2np+xvzx9wd_!gqocPpSe(DfH0|X z82kn9LmH($Cx$M5q3i)l+AA67oZ`kahMDC7iryFSS)jVa+tNDqtNiLE?(o|M;;yq= zehKU@dGGoCl1(HtwSY}|xV!HH=P$CVuNeC9;vlEM+bE$wl`ofJp^=0=r+fixLtQvR zI>)gRaNSS`&Mn&oHX3!@*54;K#pg#Q$Lw?b54+wzULj%%^zKIw9wY+=SH0i!KiYmO z6nCaOyNC z#lB^+;|aP@r+()t&#n!h=+e9L$(nNAkyPUf1S5&xnw^l;8683rQJ7&aWE z*R-wovm%lp7rI_))O>WHh0BSfM@J)AR81aH)3OFZaz19Ks%3@ z*EeY@JJDmjVmcBpraf)PN}Vt&V1cAoOGru2)B%&J0a0wKk9hrlQL!+ux^Q6G&e~m? zXb6XBX1wQo0>yAos^-2FHkyxp!cxn1H^W`H0JZ)Bd?wxYH&wz%v71z+3&83z=isA9 zlF*iyRx*Ow!onkSMmi+e5nyod2fo?bhq1VhDeJVyxs;7M?3j^4AjrN#h);A)3?Cl^ld&GY}dmI*4h zg*rjm)|gd;lV!X1POia_b#IF6SZg;nEW97--R7e20oV(7DzsgJ4F1)#`O*Ma&>6?u zmK#j^mYrQbX5vIwNW%nT2=6?;qlAv(ZfW#_I?RVC^Jqb;uE7Ba-?MGoD~Gj9xkxzk z^VP$0H^WmMx}ReqCpscR(i!3FnSR$d$b8$mIJQ9!vY|7M>jn}hFnP$T^*)fq_wVnj zH71B8K`AZmqAvJ=eTFo50yr5+C?0nS8bTSM*6J`<<7nW+N1n+|1>T(y_U0aqqhc9lRd)_b{x9@2BY>*?sq0e0PEe2A%bRi8Vx=CE1RBtI;Be zZrZ{kV|us56Ce)KBT1f$aNt>J=%V0q)eIB1C>#t_4Es^fMBf&%edo?dE~D;i5Z`vy zlA!=aDf$#Nf>ZZ=k&vvq%RjB+-7oY@__1;!pICfFSv~@eu4;ytV0V-+fo2wm@eUfMoY1BI0olT<-qZWZ2>|3G)N-zs!Bo$FRj zZlk>ts)A1K@;Hu|4Ce(!Z8Fwc0-vb%<4j&(bCl+M+1D<~o}!~mw<%m*HD?jRfM&}+ zg$W!RA!=w|Ovy*vzB%leY|u4j!_X`ZuiP$vMy@#6)#zZ_H&L)r2sBr^qkhme!n0?) zQ}LQsJOboc1%4(uDw5m3Kk1Nu9~zvy&~iDuhjtju(3~#0!icIqI2tOguv0lSTmJR* zzw$XJ_aIJ85kcC5c1wH0WiH8w?)xcGVpbE2@h7Ls>VgNrK|XUB8Hut`DDrJPce)BG zAga>;hhF?9==mRd(X|#0v3F}#+OoEZPBORPgR@r*QTN_QC?{R~`mR1;}guRbmLCoX2%FGRs4N{ZncdSe%Q*MM{<_ZnNmm z{p>lI0{%%h5608nuL1QAu|Gn@#3x#tF>t9yN8#yfldeW;L^`9DD$t`FMZv3cvgRE* zyib-_EMAXg(Qp>UB(y`Q$_31Luk(5szafMnt_S^+V>{coe4~zpl%_A-_K5UrAJ1 z9A6o-lC4!ovEW4I`znc*-}PO8`j^q#O{%h`nN>=;zHJB(3pWCTy!-gj4tRV`lfYG< zop1gffpWjn4t4wH_OM>EZr!>uVn2cgIlZ+_nFL3Q;m3`aR&pKZ*^*);qJ&IPmIn2& zgnKC2Hha97$ukXBo{|I)H0RePYML8?2np*OH!eoh3S6f$NxOH0vYK_66riCZ4PqOF zsT(pzyq zF&bdsze}qsQzb=_G0M>^YDdRO;iAbqqDYK+F?D1z5-l$ewwk32Do3llt}?YsVx=T$ zF+mL!_M)$q)f5TDmt`AY_tJ&cYJqPdN<&;iPA0 z&rK&BnA7i;k!FW-3j&NFZS5AJn|TLm_1-$Oa7ZuMD=J_?Fx`w zEHuATRdyOQsns{-XTRXa=6_Ed_7JgO&jpFoTZERccHRJw-JaWQbmQheC9p3(1&Rv$EX#kva*gZSjz2@%$B1mhxUZhATz)@2{|JSm!6 zCFIb716PeCzw5@0j*q`rHl+a8iRT7b-YI{3Z4=9hILBk?zaP(8Eat)39cI4o92?%L zsdyU@qOWDDRHG~Js~2QIC1ne2@7~ZZ@drqLU_}q#A=UCj5VyQ&po*BuqiMR;#GSfK z`>Z%6QIG?&m3|D!J70!u!NXoVJVBP@M_usd5PvvcT*IH_9P!Ai8NF-j7$2g|F?L4< zmG+QNfNQ8(u6!~0Xx9I0HGKaV&zp5M3QB;h=qeeGuFyhb40=IPd9Y@1xDq~Q+WuAz z7o28S>Hzv^y8^_1NG5lxPz=^T&<4+@5`_?&iM5k;yS;Z-YmQlT$gemBn)AgISWyj~S)>{V;9V+hVqO^XJZtd8GT|u(2 zLZmIraAOIt=J}RT(1UZn57^(pue0DW?F=s8hV6IyU@dPMji@B*cOw7io#&gEFNs|6 z?vHt_3oV=8oqF$No?7VN;m#inTuRr-7 zknmKo2T4FKv&p+C{1Xp;BaK$qvhc~V*vzMjUS$L+jN+^po9KiAi|Nj$l;=&5=#BIf zlY0R9Hn;T1j=2+U=gOxn>e#efP3UU9u(HJ$-yJG{9<|dci^{G6p3n15_!5Fjpi`@$ zVSGwznFFXOgEJ{cyxrbnyWKNI=_7L=7Kpwi))DA1Wlz4I1d0Ba-_R&O16;THMVY0| zbS99O)6O-Q6O^~7$>C`td{V6A8t-d-5?YBSWfDdZ&-k_FU{f(|vSc^Ioi=#D&{m*R zx^0y;deo??9d8jxJl4@M`|*3`paX{wcj%mQu}l5TA5D}N+YlVwhqTBn>4VkBoD{Z* z@uNqFQ}xc}mns1-`~4ohaPNR(eEt_rFQEsLAU-TZ6{>uWB91rp({~;z4ziZRptlVt zknmTJbb03_30Acasbi2m>Rop_SUV3+iQ|B()8XicJjk6)`1y}N=0E}F7T0j6lG%%u1VYxN)PO+wE==(d8gyTF z{-T~Q{bNPScxPwLPg9oK6YK8dar651H0ff8@9!7lWufRx8jG5}Adis9gIk+W$_0s1 zPQ()bqm&B`wI+G`I5NcB(W%2l`$_-OS6jZG-`X9P^3UMlXj+uCZL+2SKkdJ^!?B8Ii(^TG-SbkmQgjgQO}1?urTAFihnxu-1P@m_uqUq%3is1x#gS@`&(t1 z#zq0AqwGtJgq^wIY*USY6>jh9l}b;7HofBQ69RAMKo;2XclcBC-AXaI-vPEfmb=7t z%p$hScyx*+4m+!S36Pyf$FU@i0DEP>ds0;iOFl)F89I3C6~@wSyxi$Fd(8DVbu4x; z>(eJ|^bgIKf`b_e4hU;a-ec5xerlr^T;+l$(>yk9UA*@a2+HWqIRfe;T$&=5M8vMQ z1^iEytrZ|-L9`3IRf4cyz@SkRCcJTo=~L`z>@*V01JnztgGS45EXD{~$TO@)XeaRS zq!=39JT#JE%UVx)X)7zfBo!WPCG)lpUCsHEOIUWzqyu~%7HIAdv_%5j)0Q61cYJ~~ zhr?n|`oZBIlRWrsC|$vsbpK8t<}+v7_3?R|CU=knzE@S7RIj<1cWAy<^qwTd%zhZK zB?~lx1eU8*Y0hlvjXS!4!*lfLxV2X6a|VpRh3vAz7c zF5+4KZC0y>N)WuZY$FHcp2WhPouybGCWUu25O)Qc+HRzF5F+VWP@fGE9F?3d%RjG! zrJmH5ORo8MP>t4RVQOFd@*iDaJ9$g)y8D5{dUJi+N4IfOrI`( zN0M1qvJ#4-tgexBCRTvT zzw2W0l`nU{sppQuf7WW(ax&v&hQ_HwvS*MVHzY(vi<3Jt+;NQ|OO{lk9OS)wB)QVP z|KQDB^vGR`k@`=fEgCiYhue?2hr+8fE!)=DfpMMpZRUEhJ=H6I%hn%nPIp&YOdfrF z;MudgjDGr_7f|+abbR@c?~Otn4GD+F`y$;k@(<+wZ9+YA+4>;{oph{JCm}GR+;N)K z4jcaDn;+yO=pA9mQ`0m3!s7bgBH|%hc`dmtFXI$%Cz2$MzAU{M`rtm$EV1wDV4p-J zWZLH}bVYrk(y@A%CqRA6tC)`F=xt#CIFY1E9$ThPyO6d?g;}Wk3>^i5f7nQLBeSt= zg^HDkttpaBtrm5P)K0HyO2RM2d&!f(zv!nLPN|a4%8O?ljiXC8F*5Rra69Im$34q> zITIPa?&4rpr--(M%|lyG9eBt(H5%}(NjIv(Ya1XPN8ogbar_2}A?7z0dJn2pX;ZNMybwlZnW7m$8?cZjJ^g zW!!!gco+~(5Xd=IRlL_Ts{`WXNXRh>54o}9cPyJ&i2VlE@!o!K8snpsTh+fB8lhfI z-{$8RgL_Hq^5L4_xCd=ze2(eF`21?!tAETKXUv`*=WOwnCct%;^uv)mL`H`K=2g#> zE)|d9LW8cwfP--aGW(b)-Z3BpE+nJRY#d%?ZR2l~^Y@?k0U64jp=z%%Yc&1GUx_`G z<}wj%;9#p$2ceNP)vp6ls;32_$Ta2ezr(}27S*>|UXyan?wutPZsKW`LM=iZLI<1k z3q*p(0rY=$t2=X`HIrgUZ!v3BEuibiS39OO4t}qTd$`(HPf3WBye2k@O}eq7NVXVD z+bEKUeA<@7pY;t7aj`lM-=#Cdr&R~n5MD^#Tp*Ja&C6|bqux&RDWG4|Mf;)}AAR0F zikm=`={XQV-G&qr6dspoM$eX6%00zH6j5Dy$-*Vs0~$u4w<%R?)W($Hw2R$GJmeAy zt4?Wz4MqqT!TK;^ZvUZ*^`crjPl9;2(q*0@TSrYWZpFHg5n^qv5KSFYZ%qM;c+u3! zUNI4DE}nbd6{J)r1%xWggpsK6HD=Ws?>Cw(wQ53@WB!`zmPc{1Uo*hrR!3#x`edDHFhv zE0^k8YwP>fuGRN-sJm^ z88*7*=cWBxx7zuk4dMwZNH;+w8DxTK0{YeZzN+=kG9J6ro4=3&jAL>DGc+7*lNv_G zO+mF43&AG*7$wPJwojlTksVj<8U#!kEg`d8wro*-#B~fGNurblT%S9a5h+tFQ)si{ zX;ZJ^4moVye1ZR4BqETo>u6Xc*p^eDOBL~P+ESNgRT6G-|BJa$tthwbMZT<>V#&V> zc>z!*soTNu)v-XDv)k2+JtNW<#s~2uIA%$rwQi5)mJ3cwQTIz_c^y>EhAezQP^^@e z>M5rr!i8B|R=d!CDCpL;=RpW;QxY~KuteG<>0qrNyqx@o#;kfrHwBPo-p?!G~gaG(A>?`DFGKM8xWAcr99yWVw z1I6I$vKI_)*$7l?C|~ELbCkz1sv%wβ(^02u0dtY4ij3mtX=6Q^HcW6D)94%-m z`753yMtonmt4h=Vp_$RJ3b+!puZLa;gVR-`H?LnKak~K%lsFutf!rk$3Yo=*fMdwn z9+YKg0NfuVBO;P57@25zt=oRPb0E9zrx2R({+(Xh9UYN)K0E z`};M#t7rZ40pW!3Y}3-ol#rln?UD}=!d^|RmN|+(1|3}}-bn37K6**1yDNm1D(yAu z%dZnX?MjUa{-6}1n%X(L8W`)hxKX0e4_1DCV+9^Qde0YBj)O$_z!_#)*rOZm=uoB_ zONY{4*P+zZ*@a^K9}Ygfd*|SLrpl(7Hit$&9Nf|QCk+wQFWo+2F+Cs(O5IVBj)O}_ zko@^M^PzCI;W|$j5(063)MT|k>C%Hr;4q|$QqaMB#fNhZ{pO6M9F9RSC^qQ4w8Cm> z3j+i^gPfG%qIHgc@*LSF5%_?nflJ1)!1liJL^D}OeOcaIv9{1u~A$??B>8pe(P@MSawC*S459#7X>&yB9SyU*S z<}~KWS(LPBnSEO6#%1rRtA|r2elb1lyZHx~T}B4YyS2LJXU6|mVDzcce~F<8`bF8w zm**@}#Zb_YG0$cN0MKdL4EBhB`knHuBPute3I86CVoR(bzB#4jz{|Lx!g%~{g=vM= z4a*4;u-gt79`BVA z-dd~N_p9Ch!Ostac-`!II0$PK0AZ&#RX{EgQ67njc@T*xNV_nhma^DDM6QGt}U zZQZYL{8@mMc|+)i#O{gc?QSGV8vHxa|WHvx`J3sQw($kKc ze~;t$MLd-hNpjW%j)GV}<@clE+DJ|$bWi`~e?pegG6iQ!%(J7l0rV^gnW-i+a)J5U zngN4kxgvenOZf# z_?uczr1^HFw&b#5cctj%yyeOS%TgN?*`;b}K3MW+!~rQXtv5vlY%Y=TL=`&Dg84G1 zIjnUA;n81wL?xWSJUaXg9QK3;GOL4G+#fTt??ITU4;A%!u0X*=^b`+y(}0d3awEz_V5tC=t_byYjVGoYZ~9q zi0ceDhlE=?;GeaZpIf(zlxTdsAxgGro6Yht^0P$r+)n-TdHk_zX`q^&s$awUunxgb z*2~`O|K;Nm=>cSp0+~xp?dA14=vckQ;08K{bikk={VI67#l_ZVT3Rlz9PFGbYN5!S zAUB!m;pr?@yv+gLZLSnt<#VPzRP&*}8Te#KZ(ViS0s_dOuNy_DUFZVs?}h7I5+-AP z3U=F%95Et;bhy`k&F&pSJ)&k-0+qpzO0%1@h#TmxwA*MT?`@S3+G5Pkp!e;n z8xKf_@kiTzX^4W@(q65f@^v%hj_eR5LDtTn@lKR(x}zekdqPD?7Xgt0oP0fG8sFS{ zsO)ja(3om-IY(0EzP#-&g*Uo@vynmMn#6$F#{EkR%I;-`9E;MyI3#39mq;kph3G_k zQnZDGqFbS?G>`4t#~2{)PZYbI^7e-Z8nrpB=#MloHR;rN z)<1J-3_FQL#glBOJ$0UD{XjsfdJ-5eE;PJbGhhL@>(Rh!UMpQ)w}95#%250y=kuci zK{$`qV-v@wCds*53u_wsXZBvUdS61Adm{nzKjIPU)G2yvjhKaD+I@U?sKre!*~P^{ zFn0q!-G29}Phe+nGjTYSvuP5P+TkP9emVm%r#-uoW1+T~)t(+&{rKh~`)+B=fRc~%kBp-+?{u1S zY47a1`nA;9FJ5VTX)#o>lwch!-QaiFVxI~IPn)26m2YHoWkmfe2j;?jNcxj?TUj`J ziRh<@3^WVK+)_uk@NR(Yg>EE6LdcPuYd?zM8rFX_TEpH}O|X1`S;_TccLLwH_#t;f zV_CNqm79#wLON6P7kEfc1{J=9z#VS(4bk_r218Vf#}IfV=A1CYTiOgigLcU9wh)7; zA%s<|ywT4Xk);Ek0*RQzA+;j^Hy-a-(YfjS>eMLxEaRn2bbIvN#Vld|8S*pI4t@6D za=%k&BCfL!a7D-MUdaB_zoOTY|3lmk`9R@2cUC%|7cpQJ6RX(-j!EXC^rgrZ3%1u& zdjBagPtCbV*mtdDZjeOdWeEvmE|FO}UX2jr^^HIBVW{$HN?7tq0Db48+~LY z+d>x&P4{#*vqft3)a4)*o3|{k`CZ~|gT9qR)h#|D&~RR=0hO(ocLmM9TxcE0=M7V_ z&AyiZV}qv|Z_(%T)GWkup{9vWO;m;+38)nGPvldXJTTGIefYe~b=3ODKib<5|LKhBwdy_Qu3GYT?}N727I*q%Oa00%Jv~o8 zSl{8cUAL)zn{s{_9zHa5{iyx^Rz)7q%_nybJu~Y-;hTrCA8lh(3>w{jduEy5iwCIr z*hzHwYnQLU=?(C-Z6nMY4E71RB?6>F=!DjUSKHQ*x#i6ch-)TS4L^ z2*-esOKsRiDk=a2+j!z)rZ?ap}z#UJ+PM~BSi!3apTTE+60#QkA5_h3n)zBXJ%`RB9J$~Xu^q4lc zs@I)i#P(FXAfstV$Pi2Zmf2`$fmcr5@zs(>7^T9*wr$&HwOi?YpN%WD8nM5Ji^3JG z@Ju>~Rq{D9y8k0$7^a};H2npA;PQI+?mh2#%N8wU&BnT5MX@>* z5iwxD|HoZO!yJ=k^^y2O=xiG9E|0;H!AnZX)Nr#GdYoOh%r#w2u@)L1?R4{GxE;}Wa4}G%k z&p!dC_NjSd5>Xj6BSjjZd3C%7=moam;P7LIJ7)>fpH>b7uzkazQQm>cJ`I$Ot$rQ2 zC~E19m6j%tcU+cbf{w=Zc;m-VU8B?0#<>@_PEr|jjNAPAmDegigCyQ4EH$3Q@6MIA zeVLNBW#yVgiN$rRqC6rkF=^uYsHo!1!u4y{j^el4z|gMd|KM0 zc%%fBjUAgF98Dr>e`D)kYE?XL)U;_q@AEZ6z4%afXumueXhS)xv&Y(%P$^UO@p6$J zv{3w5Ozl~x4x!d($FvCwv0DDPzODL+S&b)r2vrN4)$IMvuj58P~Do75Zfu@VyaoXAriNw0NG<t{{u!{Lbi{X&>xBd11D$~ybx@}F`deu!uhHDlt%g2qbo z!4BOzI{&e0u8B#CoFZ1s%>C3zzgiTqkV^z?w-VarI$Hk>pcGA=_cCRTQ4heyMfTkK zkL>Fzek@<#eOiNHAMrZRGOaH%<)DS^FgjULfnPxKovzE%m};=Ba=J^cK|F=6 z?hz;GGBwYBp6hsN>BEVj`0ou{7S)2N69Iu8{QtZob>{sFm3G>M#;HY%nB?zry0Clo z^t-uZT)svD6*kTn0zy_A8Y_t#2p3BO*$SGA$U&^J>H5|&wJP4;3cYe^KB&1Q(rjYa zybV}I+(A2K-ngP(Z*k}j9UgEg?D+9v&6XCe$gc=M(d8r&IqBamkY%+z%HWO^_YsZw z3CN{obI?&jK4aI@H90b&)Vc8`7D0vtw6>H@TF+lR^sS|&^$>$3KYYP}TlD zx$4ruml`9|`pUvTL~?dBhvJi+*OX6k(34 zo}>i|xipKpOU3pX7K-*xsp~p_z80LJFt%A5cLmZ%O?fjtLSp3-6Ne*}1sA9_v!_Oj z*-&-}knSEXrZtN9Bv(l4fJ=YMQ8D;-RNiG1D3)`jP2Hf32dzvRR-dYUaNrlNqXkzl zHP=Poo1Wog5WlzXI+~3s+7&?AuHo0N`RrZ2q2}@&XNl&sr;5#Eq3(@q2wvk>Re97> zYKpJ7cZ|ifg2A*HqNVd;BajgA+TKczu@El{i@q3_oqRfcAnoW;K*0y?s%9UMB@ZFQ ztsgNk+_!Sg`5JY7f6akuB<6V6rqe6hq$Nl`jML=F?uQ5M1iZa3&B|Q(!nsNv<>w^b zBD$Cyhv|SVCK` zC3m-uZ5tIGog3o&_zr`1P&S_g$Dwf_pW_E-aR@@WKOw5_vo&BgY??il*Ec?in6@50 zCW!FTTextGA^&I^t6aLvTHF~___Dk};O#fKJv`YCf6#b{!AAWXBt6>F?oRM(@2d>#9{R`N@b_czf zaUz4EH`$$Ulh{dYXpfhE7%o7|V)`$6>-Oc^4KiS<9?FtUvSie`%aTyVI`S8+7cv%W zOa7J&FUj90n=}3G8<8*&)!6+EFTB`8)d3l?RTP=jYc2-wG~^A}y@o3b)^WlQ@ZL6l$O~D55aYLZKPmz9e1HYZ6Dmt9iveH+^;u}y zuH9hCE>h#C8g;vBbEP7Tpe`XrMP&9801FXI<<09|k$}YHxqK z-zC3~t?%5qBkL}Lk7I*J>~l)jxitFAc(yZ3(P>669@%e7Se3*tTt{f*w5O|{b)>*5 z8Iq2iUQLM|=nd}dYaMecLvm}Z2;Cxk7>k^T4W+vaEJ zQI4HBaeKtiPk%xUBeT$^>dV_!FS;kKtfkUKznWynuL~|3igC>-A9qB*Cwk?>irW1C zSJld*7f7XKy-Q1c-;JxzSDIfHog{K0J2YIfK6YrZQd~X6e;)IH!`-1``a+gAjNZVm zALtr5UdreWY;@wB9Y+~)BjDGk4O#t_D*?4MzQLA6G~pTOVw~LKrDQw^`qpsihFVJE zKIC=czVLoD*-B*h=oki=#n&h_n zQVx8b>=zs7?EJ2`tT9ttUf?cT-n1UF4^EaloZ&>0Q1km6!G`THI!RMg0--4uxrP59 zU2h(jW8VJ%pDAQ)*;S(=H^v?+TZ@o=-y&rxYspq3N(eKwOc=&m)~qFKN|Gg6yO6TA zm<$)8hA63gpU0W`+@Ig~@%#RD&rRy;y58seIF8qLaF1a093gQ(mV7MHXNx0A7LpjL z=;iB~v_4;JFrtmQ`PJbTY?GIS%g|;I*)DTj7F7A9eJzH1kAaMH#Xm5b>All=1atN) zQ{=#ORu9a|+=uid_I90HG#_1a-L)stI*}0*ySE%}UY}{nzV4rFgO?A=9Wlf>l#mp7 zzPgx+cN{XPaA?%BLjhvL0*MQGhZR9Wi6-Rh*oXaOQf&(K%n`EO!?+LnJMMUWY>oA( zg4%S+Moco{j9mS>vJyVEn6}KKKW&h$ywIdORRpXcp&IivbYIc8m;)SKa=OCF<|g7`CIaw?aBc5#VMfz7vb5q%zS0 zPnuhNdsi>tdp-r%aTHxR-&oPi0)ZRCU}<>@{A&*6>5`xJrp+NW2;Iw33E2Exwh5;x z3#5wB@eURhM7jY=7%H*^+?a3t^;eA;$0YsgAVnbKR3OTPj8V$!qkUK701)@#EQ1Lz_|{M9e00_WH8Geu{QJPlmYt z%q96j6wj7bG!xQU3*eEIGe1h-Pj(th>u$c0haNdp!+Q0m9cdI8J=@dMyY?!5+w-+^ zqh#I@NKfuJQQ}L!I$o)RGcSiF1eC|;_t*tmC?_#TnhfbWsU``byapAuBYbD=OzS9Y zzD`gKqQqKqi}&j)w5iL=1auSvj|>UXWKL#85IIUO|9kffie(%cg;w*T>$y|a1D(gQ z^hQ3vRrZ%t3(z`+{M_RK*Pl{ahfx|hU5#*o+3J;**NHq}?yNH=0VUHiyV__2=twdp z$CAThv<&&#o`BLh5|J^wtE`lVk5tzkl4b?V_qNDV4!4j9QG5IbgT8D3UdQJfXiS@1 z)T(lhHhuo6HowA%r8Xv6z1ogiXdCvvtx_7-J;%4e+O=+hwf_?Wi$up_4&u4VbE(zY?^KThgsrS#~uXW=8pd%VcwB4>!zh5(je~Tc^VIr`&CZ@N}`ZD=T{E zSU5ZfJ`MSdW9knGExAtUs3z+=cW?6j6WGb*?<<9a+ua45fl&KyZYwK~ck0!y)s%Q~ zh&Xj^d7yxocb}u>qB~ty3}Q&2dGsDv*mZq)c(|GxYGq$ChnWI0FyH7X9M@4(ef!#5 zLR=$7k)4~@e5Po@lfOP&-;sJC8`J2S{}^6;|4+acnST$T_eGYxBNA(opP;ond*!FT_XBep;)Db5VziGv2+9QJ+J!n_p8KXUv+ zt?q2r*gloaaam*Eq>e*AG`-jXcTntgqB9U+?*NJm=FSmlhVMsQIqCB16 zYQ;NNf;jQmZ_(+l7);0b_a;7RCZCuZ4(X_rZpux&gWPj;gCae!6>?EzQEck#JKL{* zZD4jsxXFCxiM9g67@J+hjw&Gs{3yo{#ouH}AP>O3;TE|~+O%!kC4;rv(aY0yl*HHH zrbQ+fw;%WbB1m$KmnH5^nh=bupPc$raGiG%oT?pdhUDVvPJkV{892^+B6VCa5^*L) z+vIV6WQ%q_Rwr+pZ8gY{%ZYfML~|WJYsQQ>WDT55N6TN9Xa5 z%5Y^`QCOXy)V2M+RmD|B4fg(Q=2*74yp+UeQmND&#=Ara9KvlS#%q=MhIIn_W#m=6 zVHP~&zjQTOcM|f^OpjjHe*RPnjbi`(N^3hl^?;e)qqn>|7(1tK;2w)k9men9v@f7y zt+oN!^5iw%umi%H&x5Mh{ZOR`2#6Gz|DnArV!DzWac+GF#ym4&YyPF9Y zNTK7Ou+&?345XMWnO7x&t_4x$FRP96xz-dlCpOs|4%< zhb!2ToYP;YtcxJ2r1votE$^?5W6%|WbbIj%7%S({OB&~sbJ@=3%5_$UPTuL5w!Q26 zVdgq!LEhfpvx$gsYz+Ga67uX9y~wm06?c9v+ivtid<#9vehP=b8A{YxEO?7=4cl3} z1xT%7;0jMkhw#>KU2|L%Jnsi!T-#gf2DmcciT2T5Qu3%B-pGC>dRr22!w2;1Hyec} z5lLQ;Cok?rREBIba+kbKclQI*)D`W5MUoTYxD{~)SP3VmsF4JwOFJnk453tmu@byO z(At$(U0WzYr)aO^*Qs(o$>qH0>WHjuK|2PtGZ59{DIzh#J}O4GUDoV$QIc`nSZ!Ta zEAb%08*wB^a`{VYxmemD!#Y^9nrXAV4(LnOFoLYax6}v$4^Yor!s<@2uHw#xp?u6i zy{R(VL6tE7$`$9tpBP!(p=tf74TqNOK05!jOI3T*E>^z`uDPD3c9pAafrPsmf$7lt z6damtcG|Ep57!62#wlnTU=>!n`D4k#u+rAnw&GC0@R>6Y$(H8YU)Vfz6f`&X>wf}& zm!))`?VD5{AV#5BFE6ii=`VZHs-%4LMDmJvO!hiDc~&(`XM83i-f8aYkz@^ecphT3 zJjl4K0uIhPUUd7+M3$!zJq?)`Sh$UkxK>F`lP?=I^)A+%Gqq87c$uJ%vg_&)M_mjlWWl8a2A!+3@*}bLXr`0q%p) zGQX$kY@! zlVJ6ke~%D`Evq z4L|8<7t;%iSl--iYX2M!FuM>%_ht#AQ07~!NJmhXrvO-#Rhhfo-~^-h9waK@8 z9rh2F(X62WyUi}v>r~+Zn4gleiB@$7){*OuO5jwX&+<+F1uJqbuWu99)MB;H`O9eny#gJD+#v-Kwf+*922~vHV_A{$ z%RRaH6cAfB`cBnby?_7JRAO6>%Bo8Y3R`z%8Nqy`8g}QC8q0={|8H#q^r;+#4DRs~ zOiVQYp|INtY z?z-R0vw)A^#9J&zA)R6`iU>0zMrRo2BdSfP*0y}K)Rz>>|NC8iwekL0;_Fm`<#wuuUU^CPnr*b(-Y+X26h#!|Vu|F>=Z=oZj^Dpb$Rc9F@H%720ER;ZyGGu41P8&DN z-5owOXg!z79DeWUGRy{Sk{MW56DY!z}l7_2yrT5%`wM8hIXjxhwmlH^?p{FNm~ zn)(vs!7ZcStEOTdpcqvw*HOHWg74yoyRSn3&(7WTv7NjLWB?#J%8phT1&zh>_EeO} z5kM<}(bMIYQ7+njuh0J!&7jfszHxHtGfO4tNUwD_JjPDn^4>^ahf0g7t@9~p_6t@u zs_~^)?(=^9{ZCo(_`A%&sOsKs4ai`mbTM9ooxz1Z*X6CglEpIf3s$R>xNC(}^hvJ! zC&$2P?sO6I$S{4M-;lT3;KAx7>(UjrSVM-)Gk6QQMygLp`=uofraBk;-jRj{e$RaA zVGr@qwf z6|X*A-iX{og7+pUn&o`Rl0GJ?HXl#Co|2=(Ef}!;8p9MKEu$yIJ9Gm z6k!qJYy#*@)jF51{{qTwEO~~DqP|V9MU1&Lx`#$Y-19%qxq$t_cX$keoYAuJ;~N*V z0&zhD*rg~*;FWL(t~PB?S@sPbJ_4p8#*!hmX6@#KL=p{f^0u^KHmYN(XpKzMp4U_Y zk5Kf*Z7Uum1n%CY?N5NBBB3ZENAoRXIW5_)wTJj%?b9ecfzj$Xr2NUw6h*n)wBi0+ zua-1B{!>VwVC72R;e*f2JdjdWR`dS+J4B`NuG6Gl2*(~Yh#p}>Xz01ega#s-rw%+^yMqbD;f|ZqXvJ^~j^E(A-eJUz$PoQ6Ca;E$ zwNtVrTW!(LuTy%qXt6qZTU4r3=GLqvcPN8yYl@a^8rku@s_x`@RZ`@8nV-=4$lv{9-?TQ?+(Y2KJR?ip}4rX4~J_i zSuJo5IsFj9xHP%AY*1mV)>d?!Z|GmFt94-{ngdiXKk)CF3i7FMjdyl=@fv3(bLFHC z9lrZ(&#?89#m%Q4|5P%(uy_ba4EA8=r%UD#Ax`IEVZlRcEw5cq90xtQShD zjfV`d1+^PTAl4_>ayCHl9@INt$8ouKj0igs8aP!RdQlswk~tdjxRkJyV^o36>C+d| zYAAkvWP=E0h|LpQj&dl#86?FB3^#-E1EiUU$q;KkE5%YU8@}$qFJ;3w+)4h z!gahD>NC;Zhj`MQApD^@k-N!+n<^z9%@&tH;@=Un+V{A#v-4-V`|OVhy&4pj@xE$&dQIQMnG{~NohkR_HWgmwQh zZHvA2ub56K3c)NNzBAk~j`yHxKJEF(?CSQLB(3wfhlj_BBK>hf(q+Iq-Iq}Yx=*iN zn>H$U=!x}W!>_qdS}IwcN=~0*>K>!#B5o|%f+yg%7yyNz+=9&A?FW7nFQ$+T=lMs3 zom#wTkr;`rg8MsNpd#l71hS?DtwW?~52BjEZ5zo6Sw@?$?iShJ<@)H;AJFW*Ga7%M z55Tsyj#9ddEZz7IUw0AxO^JTr(7}nmM>OR$7j+5Hy6z(!r+C%~V3@a;In7^G;sU6) zZ_&a*c#3@6Y0TPFemK;UzJIf0sr|FjKJJFe{l<;>%4JE5XiGNIDV%FJd7D~`Y2rsuR0*AQh zkAz6T&O+C-)L?kcvyb)d57#j1*Qo4LHSbnSQ^)XhigYBZR-vvsNtJwS9q8Po_jAjo zCGFnHVh^;S!D5t>rZ@$OUp`{kA;TQffcd;Zkj%;0u5-1Tj# zYz~K@$XfP#_g16kyCwr$hzLQ(;n2`Jc`WUDDANA{Cd>MXfyJnbeU5-vuuD0g@01b; zb4(U1&aJB0Yeml`*)X)yBlyqarf2-42vtAhvef0!!lnamB-SF$57H&o4e={-N z5_0MAf}aPp9r#mMd``9*{pPXwP-6lUqHa$~fpbZB96-A)nT!~U$f-!@fe;{4H-a9OJ+Fsr*2L1D8pKyEz z54-y8*)vIZcCJ{aXtMsw%M<>`J-P2o9ka=-kBB?P$5C@wPwT0d6>*ZN6FrR#U1ZQM zm12UeJW}JzO<|6}%{YpYit*|s=R+!oP-K&Q)dip<|Ebm}u>dPAvbChxFH3vZx?N>K z-~ZxzdZLq7aXr~L53U!GTG=Yc%@;`(dl->aX>Zep(h-E9((L9i1#*a<)MHoc@B@IO za}mR$kkS4k`w2Y0=)yRsI!Tup=|zPZ@EWMb># zyEK9k(V8*b={1s6Ys=08aT6BT}6MsNpbnV`!)4oQ^VXdtsu zG26i$Dhk)z&6a4dUw?&9)ZSq>m#pA~=t05UBXr;61*}F+HF6XwMnW>l-dTFU^fw-W zn1@7@45!(xc(brfeYjw9Z>RvDvirqD3aK)=T1r3d3yVR2O>Plb(BJRi`Wr3hOs!@8 zciy&%h^RaQzw%us*OkFbdC?LWUxr;W;y!L&>5AIHqi?v}wbhO&fy0vf{`D`iCp2U` zhU-|(%NwGafBbW6rc^Mq#>P7O|4H6CcFC7Z)$~{#rOIJQv^WfbIlQ@jj(>_U*Qas| z_9$Li5?AKq;nD8(Vu#DZMfwmc$@XFjj=@}2+8{FUme)kPLX~%v1$$Q8ss$P>L?O-S zY50VGQmWR$x`CHsU_3(e5O~d+6Ld*jZs4zu0)PuB1tDZ}gyjVyTnL`l@ACJUg9rP| z`>2BPQs+={{Qc=GoPt*tpMW=K$h_vISyhpsg1Hb~afVRc%vi27t+3CyhS$!_LgF)w ztJLz8&{&{aD(#fQq8FD;SLT6xIlapc&<-*F4VP<1a>j0ORFD0@eMEit-2izWGn~NL z%h;pnl~le&#CQ5GtL*I=*zQXTo_JC|RNr!`SM{=;Ic1JAz4Op^IMZ)dsHt5v0OkZi zQf(piBq#HE9zfl57#4{hcq=g>VA#F*FNso;NUGc@+?kqeGHk`lXbyj*E6TF z8BN^*M{y7#a%?C=Te{whnS?v04>0PhDMo zLZmO!SXpT@k@t?--eJ^#@0V-uD2uD?bEO-8;PauDGIfB+GQ6J87RX*g0~wxHjsze* zP7f_xW}<3&4FbxMJpk=(Y+N|ar^o2qhqV*`0LpD(=e-B1oLS@(bw`E|}^~ zYgckQn{1lNnH|_2kVyd&6AR&nzRkdj9^r8f!Qule6d_R zZr!rw0D_B2bAuL)q((}1uJp3`l76yh>0dS2=ekAbAG8tT!RG=qoXA@*Sy`77hyp)= zoyl`|0&vs1uoVlwyXhGD?1Ibis|BAsFcT zlF2C|Dmv5^Q>t+`0(7K^rOF<7l1BrxFIVnhjwIQqz}L z_=xHd;>ml6CGUJ?w`+ucfTc9%DE8kxNE?bpkm?Zfj@g#Uoj<7>_@RF9mX~CmRs%&V zUrqmFj*R(S`m4%YrtBeZL~>*0mT8n+FfqAc&gC%H>x_n$fIV1S#cxbu@vWdBJ!Syx zBt=|Vu4~p`-sij9W_iZhg>wtfR#VD%<`h_Gp!|@8^n+G=MT!VD)AbZ>D8_y{+9i9b zjwGnuOO-((b)c1170u$+IllB>Pa=B3BqQ zXT%EmwM!IDlH||Gx~o%wnz9vc)TnpQeo`(MgPY4bJ}ntW`QB-5!Rgr>_D1p#g>ST3 z)%+FRr6_~NY|xu&8oJXy;zBF5;5}d$N?CpLSypL@d85dZM@ljsM5I}f0)3#I4`e!O z$&q3EMbxr+qR_RUy|Y_q$99ejVs*4JwWeeJ|QOM9y{6MfPf(1RbNQM|+i z`IGpVS-5$ptewv5^Njc+8awYdv#Qwps`5-(GPmlMV$gB=^gExjR9d?%j*nmLm#KA3 zdds9p|7kW>FKJfQq$nM`{~*72ieP&hKWu!wg1plzBRQv+j<-u;=R*+t@m8z)74TeJ zWFX>@rEFuW*$)(x?(t)prdI)m!oh%p-<^BrbsBK5urIBjx>ec}@2F~CM)$BWck$ER z{-KVud50gW%9!z@OOjDyZq>*ruFrejJH5vVjUi}bi1LjBLqyywqg(ZH2vg2CrCIBH z(5iIFE=KT;@5nv2o>FAPD<&MS;vjRCSbpNMrLoj6GiqJDi0V{Vi|ln{p4S?3o*$Bj z9_)#>Uk<`SNBtS7JpBluk#up?CMyn;4cUfj#iUEbDG zN_Uo)`blc98hpr*xTJIkAJ7S>SO-<#NiJ=LD`!J-x%_-&Q}?x}kJ+#2+vg>L*zuyi47!khKT;HAhN(n1XLi0C{+&b?ouww2oHVKDFU{PL&;1KO4n1; zEzfYU$3@6Y6~fdsgfq3(h)C7h$avpswQ%iWjo-8OEvH>{e)M zMU^C5-(3>@9;NQzw@)2D#CsBJ@&K0{ssQ6BVCMqas`F6*$4i}$IbGlg~^r^PRnULv2ekC=UCPqLSAJ%^$Oj8<;sz;Gcn2ZKr}4#vBd%N6Z^a2!c)&E%YkP1-v}@*7Q3 zCum8Mmmm^<1b$-MZziIGb2M{QcvoxTTnNv{K5GcVEr3bTepT zsPU(zb}_C1MwZ|S6a)NImxjH2p8+X5yDC-PL=8?J@{YcSe(9asuBhOo0r&Z42fEJdi8p{N1)?;hb6Znb13?Y=CyBHw!49SvA)Td zN#ai%^j&<~$4QQmd<90JTmKz_^i&N@R8&;gYd&h z{ASgZ56m&8xO+m{0|BFdpZ;lCWEhyvNc9F`dqDRcjvp6rnRbi zfqJNzr&}pXk=Q{l1jDbi0H2>PPWycGTixg>Ngv`}yEA35@%Ea`I`OVy}7(D5~ z5W6NyeA}f?WjuZzECtC8_&rgO+|3b98Z?k?u}32#a~&nK89rXkFWjkflD&s{3Hx3K zjL;3Z2b#47@hpfyb}IM(7PL&UjY@as7Hk%XA2)t_HSIHzZcPh>Sy=S*`eA0vMRgJ4 z1AF(qnWEhyeJ_6Ntveq3up3GQm`~6&E-TOrG&s4u*embiCN)-fSCpQn9p+kp$2t|Y zLNpIDG7gDW#_B*4vt#$xwTg5lLTO>x*`ln-BaFV2JoO$g#OH6?k6r7%YU7g6F-2ncPc z+%P-zovQh^|Bjm^a(-pE;?;P>B0ApOL&JuIJlasN_n-PjmhBu{EeO}Kk>$_Ow~n|F z+-qgZZFHqSz_!rJ$Jt?h+wR}|hEw>^-$13gR6J2MjZkZ*l!9NRDD@~kAd4yS>QR~U zipFz{=kaCPdlnzH|DOKj&}4s~I6XhnVPOUfmz^G#cXYLbq+#*HK~k%gu3xH(vVc8B zPrficYN#FPy6AHZ{~eiK)R7sa<-yf+^rlAnv}Y=2f)BPBRQc7@1p(4>@rdtsP)c{p z4gXmeg)J1W55Ud@E6Vh^loRtxuC5Q4zIlmsP1V6L0@>$WgDOH|{!_$z0qFEc^f+Si zaoNF6_gu$Zj?JoOKVhlO0~_-OmJ7{x_TD#(l@Ei(?8al|tN3xC(fR3o5~cXD{_dWWrM=G$ z!1=T&tCiSlsoA{3;$b-X$&X^@<2tb>F}lmW*G2m_k;B%EyG*Wxxw>ZQsutMv(#ql{ zjA}|L?~OK{XA`j1&gxRaOwsFhUTP5}ySLSOobJnmhewXp zJW=QESUy=}${;^p02s#W@YfKi>Lm7)Hh@t)#ZUa2=m@baXsSyVH{k9Vq#F5;F>4YD zm=l$D+l%d&ym!_ah@yGw0TL8Vq9hq3H!_~cL-eDJ-QHXTKax?-m&2N`aL2RMSTTT#E|b2 zKNbnVShoT>*S6H_mF(a)Q%$DtgSqcSBWE4RMp9vb?=!A5{YiQcLaOutbz$l zIq0dLbIj-BK|xpE{wM9AxD+n#xE*Bl01`%Of37xY|78q|Jt2qF)kqH3wgxfxM6ug)wW(ozCj zf`4>2qXiZpulS8wzC3|{Ip00`_0d(AqWA4{5a^v65we0QkX!w9I>$q96FS{sO3@7n zEd7-nT$!f-_OrcTnGb(oIM=AO_34?J>Ux8%ybkt{O>R$H-lp=j?`3t1vvdQi%8SJjJ2`D}4*2ZzUR4#Xuc*i7JZx2-^SwNR zk*Zgv0*^FcS1NwDFS{LQ|5(^W*FO1%%;wTbI~@q8UX<9sk%@4^yzcH%u|UQClxBb5~z#w zxqtuFu;NPaWf4lsN%(WZ=`Qk#7vSMNxfj;g>BhBdMQ{2v&`v~-n-Wj|Jit0wCw>g~ zlawX*gL7-W79R95I7(=+*`{Hl!8zzrn2$dXV!VTxXWC_)_YPF5r(J*Ao zHedlmo>bD@e%xng#U+yIKWEILq4(LC6_=^&e@i5M9$_L9C6DX6BiSuHD90zbF1zkC ze;ntKgSKBB-g7gSOPB^Qwy_6Y1F}5u9Yt&VqjK%GlMbq(WaKn*=!0oa$Ziot`?M#s zOJ+0~e-QK5e6K{VWFvqGD)M^fB#DY!umD&jzX+@@ZQDK&UVtHCiJuGmi{N>Ku@ zWX<&6|9EYMv0@B3%TtyaONE@kc8*hW;XDLw0{#6g5>nZll3!X>3}kswmJ~?UB#i38 zZ9r^rpV5p=SDL7;vZh(3*}$x<_HP^0H|K5ooPDI{qqFhz>sK@ID;wo19ONpCdgqQs zno4qrY-8}OpU7dS9r&f~hMH6Oe)^`|Lu_L%3-O@^AROjn$_qX&$#J6uFE#H)Rvj|; zZe;3qESEC|0!O-8Vl26fCm&Z+yoMDXI-#n`pYM_+e`*9{V^TOmGKr_Ibwa1~x7?8S zPlAkG!l{(pV0k&?fb%8LKBi&mTCQieUxSOXe1|DEBm!P_qW7XU4i*Kj(PvWuWvwQM zU@(QMZEsb@;4RrB4PiQW#Ypyp2(Zq*wGnM!!KEMCzQRP=e<{2*w5o(cqMNB8n(v;I zs+%bsBVavyzSO!x0NneARi^f*9>oqHGGq)VcXt>2PO?8)cx2YSRi-bH)B53)L&V2& zvn!^8D1ue-?*cn7`LX;$yb~XTf`S6N7M{S*4)E6b6cwo*4~Xby;-9eo$46{(LOgK$ zwzph8kMqat7hiNxSgpC_jNcNthj>~%Vd zmR>P%T`(esgr${ua8BpWtsF*#j>yP$8a}DYUatl3eAzD8Lf4Z78X?oeiM|)v zjv!|s%B_T*wC>ieOnWum+Ji)D5jLHoFkyG-U}vg$jW5)c7Ek&t?4 zX5VjN-V1HX@Lk=BWeUu0To!P>v#Iym|DrX2F0P|pn4Dt{P~UW5jiQsdF5>QWtzLJ1 z|1{NffS78O?N?w*$U3h}a$s1w<1P{0J4S^uw-~5VSMgIa2XRpm0uUBC4*m zn%Uh)+XHu#NOjz$mEd0C@1_nxjTtL&O}d`eMNkrI$ygEWLpDj`X^R$1_Za)%h8GfR z*$8Xdee83S)>yP?(L%f*mDdlouCjWyY8C$_ z+u{3tJkcT&muxE$3cE2t%9tgp7Lxy*=ISd>oE17tH+L@jc=QQ|nmUTM2~ICH3Xg@7 zZ@lenxZ;+5di6$vePKvba&C04udi1mO%soab>}=DgnNo+ji`DYZUBDNU7J?CtTgpA>9xq9D|wixy>5kog$(+&XRM{ zE(4n9W}tj%nh!tOBh>B4-Mcq=dQ`YBGSbYmR9CyB(pwu)?fLQCy(8%dqNlC0_eGG_ z8UCQFj<+3b73t!Jh6TkoG2mfxot)y8iK=rm$gW$n-gn|+xX*mDth>*;_p{*iIqllF zPnST3N>SC+(_p=bsumW`n)O(O{LuR8slfCiBszS!4ZQGU4YLQmqv6ED9&;l*aTh#t zo}YLSmZRx1w0*+wrVjk#VxQBX_sM$&#h>=nRjk1f&;q~RJi>OAnL+1JT74~E%=gx> z?YRBK1LaO_7>$HR3aPj053440eJXvn>z*2KqPUfdFmLChUyF9*Yzwq^g5I%<(o|_L zG^Kj5nZze0`dMZs=_vlLG-WbCi^+r)Veg78MgJz`kZ5m!4L=}t;D71}_(yue@8Qc6 zU2Vz=ud16a5j*S=7F1l_!fOS1e}77zz+1_nWwSAx#@pW;BE!_QV0vS1Cf;F8D<{moKFaN&9pbfv}9y5D~9s$h#!o_UXroLnFWO{3VPQHE3><|XMW0Y~IISl3do?zZZ5u@{qvnHh}a7d&WX-5N0&Q-tFZ z)>#k>@m$b)Nnmw4ZD<)km#JC2un?1BRpL>Qr&cP+EB+(ZhiPtM=!Nyj7%o)Xpf_a* z3tjZ4G&x>*xx5!^t-yV$jg|HyH3Gy4p%-#iky&VK*45`%9+ZcW&8n4Y?Q@cx)r`CG zisuEuRe7}Am;7w0$&}a#S!N|5U~lZF&<1+NzL9EMB${6VXqxvY)Z53~Tl;W?-6OW= z<@K~eId1dMwFa`)N^>Wa5hc%dE8U4C6-i2Be46B(t(ihRm!;S9?n6SRio`}dB9f7) zLb?$M^d!liD1|KS*O!ZB-sHp^Pwdw(NW#58kX~3chx)`dTKYqT_r#}QeNmQWq1eR5 ze1!^S?N;pulobAj$<<+?4u`M&K5-VwTw+X#ccy0(=^b&3D<7&E5Fu>M06kG-AQ0%h z+GEN`92th(!63f!V%c*!Y6ySk-76w&BCQ8nubT}clv&; zpLQU(D>B}h?#y(uTST4^FpJwPa?C&pG>W2mzU&>C1LLXT6C@Xi_MfVih;$I2+I8Y9 z9R~|MdK#C&x_Iurd15=+A}}@~gjtD6M-B|%{JK-u%RjmNh_QEP)rn7SgR1dk{@0ON zT9VJFJl4f!-jQM77eEQP*+AW>x3#X>u9jlEMUU63zu$c+7vRb6G%3@@3{g&ss)|h4 zruxQE9-=YutQu)YbGc}DLY<0twU*0VDtM5X+FUZtl%l$V;rMAH`#@!pZ=|h?6h%-R zHLC|iPW+M$O-<(?G1Q!9j$v+fcW2Kc1epC&2?G!nFpW`WtP}?rW-%9)Pv43x@GaQ6 zs^YO7fQU1Q7c8EX)cOq?oJA2T2Z+y7HkODQI@(Rbvw&oL(rk*`py5ytkStr`jW8-6 zI_QB!U2}Y$g-%FIz~7~o7vwR>2^KaB3k}TxBmT^(N)qeg+I3nMaai#jEN9lWSOnos z$cJpLQjgIXRELrgtj0wEJ)oGu=}(GXG`Rr9f_`0eO&Kqjg+CE^t29^CiM;4Q#ixcX z_(Q)xz|-&ai5(mgs#$r_UAYLR;U%L(;rP|B>bd7gVeiO@1-LFyRA?0 z9C*;Ume2jvl5Y?c!>`iL);7Cjs52li?;UlD?6dnUBo1B`%Q{hZ_w`=$RVj++8j-E* z>Hs#~SczNqBhG3f=JUo+pZBgDJ>+V&K1FHJGTdMw*>HWP;x`G_^M}mF|0W+e=A9{7 zkGr2m$;7FfF}sc5HQwFePqQ{{sZ`q}0aM1^n}#yE9h#fz$1bs_L>1hC@glgn!n0+lc!7pcl!#glK=11-@o62v>wW1Ar>&x8%?D50FY#Q zQU-F8QhHiggr@~JXVr+K#BGKn5w$@QWpE+P|( zZ2*1HgW41@>dS&R@#G1~Iqd*st9YqgibwLz#7Mh?bj5=+zfAuBi9(AXfy#qAHA617 z=yH9HFaGCzsIqjl8TTd&ro9>llkx3U89jp}c`OfCvl%5{LZ_}^E#a6-p9+%m8lo+W#N7wI`}>(S z57bBwz;TICq!|CVX+|zQWSCb7xvTT$PM~k)Iid+KT36XJyH;$LbK4+_;Zvlo&W z*&+#In9CbRo4h+*0M+pD-Zk|u@5wDbn45N2>kr5%nWy=azTtjwDQ>KzwF9zwXLQC` zPEI=>z(XQ|kTyh`W&j>Y>7r0EF=QI@$XXJR%J5`XgnrABRenkA(VqVjzE#3Kh{vCC zWV?%Oc<$M%mE#XIAn?N&m=#P-O(!A`Bi z+?HyJ0Rf?wi(U?tSXUR>KF(G4R@iar!>N0j9^$`5eTw8f>{eRZtS~8Ck*0Ldns1d| zQTCO4ANVtEqx0;-eko!mEwD@DLQ0ho8>R)4i~=gtn67kog4 zqNiDjU7u%2M&^gBbB=*}bg=9*MV5qIPuWqtuwj!X&U4pXIxbN)T@N(Yx+M3VnxCI9 z_V%curW1rX|CVoN0Y(6lM&fQNRi^#fHxs$bh0F_QhiMy_TiPw*7R7yTj@Z6o-xM9E zCM)WgtM<9FOr}_G5JB^uR7-r2UVsH7M@otwtDHn5A5}}UM(ph|Nfl31x%gx-!|z1{ zQbgDk(gAmO2pax(3rv|oH4&iENsj@va> zEkBe#TgefL`|xBy4pcd&@JM2ZPCQ(GiPv_hGLEQocED9;idX940V)pEd zCe1b<=x%Clq?^^@-s}7O-8S)uy{$i)zpF(Q19*AYQxY-A1yhOXB<%u(5nojMrTTLw zVE*bONi`B{ba?;%{mv`vvKxo$CPv28p3jz6Q5+@~ru@Cz#?$oezd?+|ssMtCn0G|Y zfSBp}y73$T3+y&!-Z8uEJXM7PBE*z1c zmbj2t^Q&pnszxmS0q*o5g;D)5MUaEqI!f}tqt|xc9Z(m1?m*ylL&xa-xXMZih&?KiK&3kfs?!RWN&3Rqdk?B3 z%aVkPOy--A-aKBw5@*)cX*QRv&fM|zFzTWf*w8bu!FYD!T_wbf$|8MpJWbFafBrdp z?=Kviz>&h5&%)Ef|FQak0eX)cw=RO3WZ7r-Ncb+5RE1{#D_cO4-*@_U~2Zi>>(C+W{9F-1IVDq4ffNb9Z5za9IAb{nzbpYcua!B8P)@n&ntl^Crp_V zqZTLbj3346Hocyj0}a$0?IvcK?-d2#C1$5FgLS|i^5#V$OuMWuL5TZqs%To13jj-$ z&ragoVITT*Sz0(%A{8Z6d~#Z>S#ODgksHXPldP4t`E;l2^CoqaX{~77L{2P+3~$_I z7-1MJeo`rH*p#f5-^&TwwO5lm6$3BIk++=+6<=;#NtbFSr^Y+}dWPGjrZQ3;ni?8E(MjyYb|T2VgxHSl+MS+oC`^Xa4B)bDnfXY1?BlqRQsN;MMsMCT z2DdQ?r}<8VP25R=0BBVCd+M?Y(deyJ%mxBas8RU+FMWxbjMN4q;3H;?O z$ISe86-CIqEjoUe!SIR*+q)3GOy-(TleE?nk|XKkw{PAIU!Pc8xvZg+;2XUqRx{e| zeAbekDB_>S6qv(Ea%nhj{qxUM*A^Tv2yU64GZ&kZ85^dWcpOGZ0y%qTk8tH*2wg@b z^Xu0h*%z0Umwqx;g7%S1o=7G}bkZ}7&FXR~X!OW$(v$N6W{PXY#)K8vyv;-wtSM;c zKad$+vX6wUSVohj=J1KEx*GZ9l^_x^a2gb70YyFtZO0H!B!t$+_k8?B=2EuetE**e zs_|w~_)QJT%)5WFsC(@ZzoyTcQG4#}nliOtuKioM>xrNLI9#trI1NzNOsIyZL^Un^ zlus*eIp;x3tI2#n`+*xnLdIcPvPl2T$s%3(!?~OnYJg)5W&e}ocpf?AV;;~USzngR zdY!tBNc{Xgue)dGx@Mm2+%V41)fx1{9TY(J4Fb3?HV66GN$&ya)n?;e#XE{ln>!X? z({e*iBSZ`Er&OZ6kauXA;`5^uO3ttIVjh@wyn3ysmytG=Bf&^Z@nFqp#lm?VL_f5W@~sGc(_; znMI*AJ(CcAbv}g(ZVqV^6mb3j*j@CUw276thsn(9H6u{6(~po|=sa)UJV#s1SA62q z9bG~jl9{;m2TC+sRzzh7e49rS^-Uj72z3$Et{uPkZxD&-y>ZatW8GUWe}!7g83K=F z!8u~u!*IX#prcmy_&$_XJ=~y~v4RV8N2PNe`z1A1a$ECeiOppsNK*KSUEN2a+&MSq zCvvGDJbrwbo6_TZMLD?wOM6>KFJ2tBAZIj9yKFfWF|Gs;qCr%l3T{BJbx4`1Gi)57 zExc~(FNQUtv?R(b@i7VIiv1Lv;E^7c{qxA&O1h%yc7r2RX0-P3tiPuIsws0lM{f!E zV>g3(Q4b+n)I=>y%OjDIg8rc!+g8)W3T?nq`^e? zBSQgTyL^{4R*MXk1;yCe#~L6>0C+9~KZjWbW0DJI*iV2e?r#$IIwhVO2B?G6*Z(G} zLo)D-2P=u@=-59S?!yc>dgrAm@;^fJo;wY2bQA*{sP2nfV(q=R?s3p*K*0;^wvyj> zqr|p~qP0OoF$Q2D9QkKIIyt;i%4+W~%et3z?dsaNTB6C}_@z&VjMsH0VT z@9v>_PrF_#C|+!9M5l}$RMdXvLk14iFx?TOHR30|D(Lvq#ZfRk#*MN6E!&!;MEfQE z%$We&5l0}Ez|qz{)j-P4u20pKpc7o6RO~VTvtY-A3;IM8*QI?#B)}!M{6cfT(Y$&! z73t8SlvEzFNK%p3dBiIQtwb>)M@KbeqFwx7hD@8!TS$<;f1AdOi)S1ck5{F0%AoB< zHMURL)bRDEw|npO%d+Xx=YGB3yL8L`)ar+QFxeitzDt8fuk+e9D?P+);X#dbJV#J- zW^PZum07yGWFv8PB7{2UymI~jmB5xkwx3-qv{@7Y#}wZsSUP1rLiM`8*RUqBR-;A` zz?_49sTqs;@BG`AB-YX%rC8%z&t?J85S{Z!Wz4Ef1TC5`L0b*aNnXHqTgj+)3TxJD zJR=qe-;4G??h-nmUNihIi`l`4{sIG$$cS=@)?MN;3Up2`UilutVRX1IP@7;gpDn zF`&iDXB+#NnKN4zpOCYnUBX!jFjSi#v-HBTvNy$H9oRBS4IzVr(@Pg;dLO-XcgcUu z4g?R{kp3~e#^B)KlP*=Y3w^VfMF||Ao!#+GLTy7awPUf^v2!Zl*5H)H2GA7Oa_RWX zFZa2|;y)L=t66MgB)KNspIdE5kHxKaj@QvHq{?vZz|p=7Vc>S47dijZSkLq4q!hd~>X8 z7oC&-gG_nI0*jCM`nBN$OkV7%UDS(M+?3qU)L+RW;Y!qRCgU4y)N!s0bJNaK1BED8 z?Ocr~k&sI=et2ybc1v|bsGyNu$Ua!`&3E?nXpE+~pVV-EJ;uh=$yt)0!$Uj{K_DC}&5Y5q+ zO<2-@`~PLwQqkT9UfG8p0^I%GfGJ&p9HisIUVq1pW29*Es8eO#D$iEKg##pl#9{dK zu|eZ!w0g6^XXB-K&Ec(eUcdg{*WR>ovlsU_P5Wm@@6J1nb?(oHVDbQI}wStP1~*^x$eB(j`*B#hn51xFo+oC-5Hqv<&~&8Oc{) z>6mRHrBmhHhYX8PLn_WX=o6f_9F@wa|7*u~3dHtV%}cnY{A+1D@|P13&RE!=JbZYd zq$FP-ukR}g&AnSVu@p!(_>a4VwGks}5Dz-~Uqu|lGZEk>jyvEaP`#kjbSt!@?E5gD z=;GoM@%ZPZk^}hNh3F&Djx!Q{)ak}T@qC-JJ?jBEqX zgYcusjwL3EZV1v?G`vEv?VUJV*v|Y5vKo}vD+8rA9zUJ%d8r`BqGcx^y@4nTd99hg zZf_En_Xk@zPVPYA1(1cu|1-ngu?Zn<2l#D66@ptanRDb`d-@+%Y{IFyAqw`Oq^_(nP{WALYfL3iRx?EY(C^D1AOcK0O z4n1I9b2{DhlZ(<`HD>|W5L)TGAwE=LQdkoF3qL|5Sn@$~on@znGM|&m zV|$h<$|^d!L(PgyO(_b(G|f;TuU#mol*|iq_B}`fYwWx>N#xGt+w3K(OGF;MVBL5- z^7dQ+89iybS_8X*O5AWFd5$uZTUH6IVDaxjwT0&M6p=Vm%iW~5cFox3{LThAPy@JF zxeg4H@=3qCfqNyhpfW7b+Xd)9d*ye3S!tFE;&r(7o~5h5S!y~-Dwf)}SLW!L^^+*= zDM_x}-wT`FCOnLGIMubO9Z>^g*+M1?q`zDnRj$s6SXpslVW&eavnY(udp%yi%WkEs zsee&82Bpb2s2QI%ee2F)6O#%E9APZWM<%~fu>X_>kPR|JUqsT*w~S1-l}ys)W~+~IGKO?e<#9)3YBA}s~MMEFs9Z; zhY>;<6N~Kc=E4|=M4m|DNuFyLjF(xBkt?oxm6NF;XRs0@n*GbGxlw}r(h!H;COxuxzqG^SlL zq?zZYmlR3^InU%K4MY*_+iYi##fR82-aIDLWA8+*2zVq&!<|{{)%Ovg0ZZ6Rm}fh5 zkkHQRZV3lpT68h~z?uHt{ew*>3g6L$5Aj2Iu<-Aov7xq#yhE+V+^R?k)6{z^8-!fi zeNOq_hj}=aWmVpmojY%zPVj%u-$}|v{FgmV3CxtT{NcTBpg}sL9)=Q$+rale=45yNyu*}LiC+F+6_Xp-EEbk9mLt@Adl(a|9Z%AZW=_3jh&nCDlg30)ROZ` zynpU;*=0C2IQKB+5xKR1X@anel!_twkUr3kz4=kX{Qlvh-TwRExKD4wX^Oh?m0`>{h!;SXp=F};&4OzZE8P0Cev{FZJtiN z-!nvDBnhXb=Rq@P(?5xq`5<7olr_6(zy|<>uBUR-ZfOI63K=JkOa8K;VbU(mNbc*0 z+_t;jhLdO5-~$_*mdd^TnOF%A*?lxgN1}M+KAgC7^QQcmcc>K~w2QeY_QD}oP+KYhn9@t8SMDqDVbDf*59JZM`d?M?W($D)2>_{zT+)lzmh zN=6&SfBIf{jG5}dSB&oOe+IsAUII5h+Bpv}x&IYkz=QEdV);QrZ(Y(Yzf90V5r>n= z+efM3Ehlc6EFa3XCaR*H(Z+Kba4a+rK?JnOuIfSfiF=OoPLnE!f*PnDeI3qk6ju&D)U*h zXMY5Gz4&dz{meg_^<0)XeDTBSe}p%^nbJ1!K%X_$WTVbgev+iMkCpa9})yB^tS2CaU{R;W4 z$^s94qyKwD>(2GQ>C&@6QELk-$^Et9*lO#TnVZ`bogw8>tTHJ%n|S(X3v{@%`EwQr zs{24H&CZ31D41FFaqIE#r!HQc42yc^aogVDHk9dq)g)8u_fAcq10Njr;p5eB6$c8U z=d-8!DL8~SiQ9#%VE(R8;g^i}Bp@c+c!GVw6z>b_0d$qDH4#z)SWM)&zWi#X?Ec9l zeP?)h^0%s2?$LL-to$ybY4ZImTV%6tsw)}@=F|3{?UQ6*n<#qJh8Q!;Y^JqaBHeC( z1_MbxK!OEs$aS%H`ZPr%@#4mH6b0o6<64WkKvXIH@{=to4#dF^1o7UKdAqQ=qF-HJ z+P@4ghgMdmxhLhvN|t! zJ%1=$y(kaExJU^?rNbZtgz>R?&It9KHG-(l)*NcqXjt2%jF|~_G$+WA>tJFO^Na5O zWgW#rH0{QZz5Wo`aq&3L5G5!b`n+ z2^k~x441gg$8Q+Ha4wSPA8wq_5V8?z_O?%7SFnp9MY+etyD=m_6Jd5Yf&-qk|bR+NBE%cGoDD_q>4rwT^TiF#@`{I#y#*YF0BP(!+8I| z*Z5Dl4(LeYF^>E(NBA%2wWyv4XReDuoxoVB9kM#@1uPR%r=zF5V}pkQ?M4Crc*f z^w3RGAw3vH)jRN$zzTe)gy=N?P6X0Yg3wP{ed|BIo>K@pRFfc>dwM3030n9?XAVS~7Ia=zZqi z&XbO}svSIDZ|8}~hz)Ek6MYsl>+@8~h5%GliX7FqAGw(BO+Ae$PqtGGm%INTvfezP z=Dhv?evY}y|34GJ-2J?3M9sWNC-Cq)T>fr%F#W}`?`{? z#wr3247T~jsmWcnTlqqC@c7gV+cQj|BGIOCD_f@O z=|tVKua{}x>3UmN$~5!OlG+xeWCP#j2D`F)8! z=((oMdGO#4cyM@W^eZK8m6Gcz{giCFS542~H5XYBQOV!my?uK$p6xjbiAcggz3 z{Bz~N*(XNT`?+nwz#02$b?I67EN-#f!(%hk4|YlMdiefsR-Et9-`kq5o1WUeJPPfb zki*RV@#^|V9`H}bl1d%2pE$wxI0&q)o@YCVSk1A$$e|g@E1|z;*wP`Pifs>#yPvacfXUoExY`2cqus#AU ze810jR)c&3uq;V{fYYU+Ia>6X2VKgaco$aIohB!eOoFD9>7_of2j}V|)O;w>X3qqy ze}tutz3LN1e#)dPtDTD`q0bU`E%C`aQ##{J6YVAo7-EFrX@>+BbVfaF3z%y}K|nb` zkmd zrfxk8KLXAj89b$n=ft^w$ubLs4qun?*TDrA7ASD3 zHN=9_v3MlYnyyY-y=J|0pEy=i5?ldD;kabM0edu#tLVP$U>TM&nOVErKY#vD)?WO0 z`aL!s*qiEWLSf2O{~I7?N3+~jV?Hmu^1MEH25N0H0WWWUsCLJGcw>i?pbsFCw%<%p zh8~hqB7QwJ8!KTdv-!;>HE4xtpGgo=D#jwBp+)<7CNrw|Y*oC4JGZGUq&zn4`C?3s z06CerG2t=T^*5PcrEJ)h{2fTZmp*r#gm~i@(%L^tiz57@Rd=xal2Y0tLZxSn>_&#~y(%y^z z+4JT7h0DjF!~*BAE=cp`WB$(V+pk?5$sL9Yfn3-C&qUU}w$Dm3Qd*+<_fW$b>Gx&r zV4xr^2=yX@$Xb&1^o=d99uW);tAoxyA2&@PH|pJ@pME!29&?gJw#B5ABB9@m=CGC(QNx zbm3gw$?}YZgNF`{H=5kYxJH#K11B}K-g>HS)!Sjix82$Qre4sEoJEJuw7%%`YQ1fJzMO0G|M-3(tXPvC$8{o zjheDP1RE{;RW5Z*64eAE99fy0G?lO8SRE88kY=}IDFY^9=-7_>-i1h@+0E%#`$+X) z2dG6dXZifyD_Usv>VpV;YZb=|*LuJB-}TU(tbJ z13FImPSGhWi<(GCF9YJ_;onV`;>kv-Zo_Lk3NeSlq+(|R^UL(TenjjZ0duxYl0+e{ zXSY_$Ixp{_k5N@tCGUK~_Fq=unm#u&dp~f?s)r9B7FR8}B5M(7Xw1cRS-tsEN4y@p z+>F*$-TWfnq-^2JM!!YiWme|`Y+ql5WoJv#WS43o;a^t>vMl;B#=>T4xjxa`}Dx>_nZ;9UBpD3zDQ zvp4Qf858l^CiG$WaF$mpE_f89%J*CP9g6S$$hVpjvIW9GR_xJojb8h7#`Nj3{ggS! z6(}!zf8D%XoXA;(G<#IQCkj^oEK*Ti<Ipz3w;NabiKi+x@ZJ<`F{*_ZF;`gJWK@w4cTEb_4!QdJ$N* zaO2`}J&W926N@J~9_^EDmfClSUyrsnUW=-?ITpTUY(J;9NMAfb*~YlK-rbo4!6x3+ zJ4L1L!)2D)!y$~*looa*vp9yzZmTiqJAb?+N&NeIjsOUZPZw`4`%E$}%zxhb<47gV z2CJXBZ3DX%Y+l@%lHa{Bb@lY+8z_0RxudafCZ#_m&ae53k^vi!DIPJYE63^ko9cg{ z_y91D1IgQ&L&g@K`1Tu*nRNh_-}LF?Hd&O;x!a5spnwNGvC!Myyv{smOK1C^@0pv7 zRrKS40+IjS+nKZ{63L1KwDuj6y9((8BoZoZBi*|OKjCC?VD)$p<3FEHyYq$Ht1S2M#&l5E`B<93C+aw?Z=Lqcz4 zW(=>m$0#RPxGYr5ltb6*j~lPFJ;UdxpMLVl`)gOTY31e9cQ{y&dGV=r`!(-mUNKA^ z9!vdm+|qn+a|5jPCuDo6+1Wce#MHt-MbFEG%2=t zsN-&eMjsJneJ-e?yknBAN@SIV>e!me7u6nv+z|k|TR-wBCjlY9KJ#WjnP}4ZAnYHu ztuhmm?Ymz(f8xw_42WMrIAv68#eXgx)}|+KK1^{*kK+}VCWHW0j{@mj#0#{tx3;8_ zsY&LO*_Yz;iNB5-V2d>RA$t;Z8Js7WlF#q_+=8F9UVu{C4-P~Ok7K6!1{-A_zWnK;rc3rPntI4BTfrnc>NU2?` zR;M;qv@uOgO*kMvhJjmtTQEb^%L$jdaf-8%N-VBow5~lBz?szl!+bWAU5q~`(0Yn$ z5PNB_a*?bu%8>6qvS*1ji!7uzPm;4WONm&%U!sDDxFB~sGze9S0EX)B#xu9iviLe{ z(MJaMvkgty;7o!`gt!WmWOi~tciXL%M@Q-(ze^H*u1rmht2nZ6UOMp{Hx`{$%nmYC ziV+?!Tx2i-2(!d^#D1ibhO~98oR$n#oljnn?bKw2Y?aZnEBaMHjdlc`WdZegRAAa3 zFUPT;X=*j?Jlc~e+_iW_{`PU8Ft-pk6qtLeLmi&K^`SE30dxC2Kb+L0Vqs9l3rDA> zdm6=j-sBWIVq^XaD}KkAmX8+m56M-4dN|t7caHn-e^@>_mvm_I%Q|z<&oDZ0^*2Ms z;L98Q+->KNKGXL7uw3Wr8c2&H1r*y_NXGH%bh z_rPi^xnk@R~BRSJ24oB|e)hrKp*XOGO(xW=pd^Za@W#ud>&(H>|x#@6Y#KRD6&xDD=N> z${CPG5 zlT&jg?BZ-47cB;PEdLlZ%GKC?(uQP#DW?2wm@=h|gOA91{u z-_P-|MOcHQoW7q*H@w&@Q}Ll%OJewkSs-ZPnxofo+s|_EcVhR7ZXS)CQ_>pw8*Usk zv2mCXv*C!gxVPhO`j#|!-cT=U2xOp58`1RJk+w7=15vmg6w2-(c0_j2tP$TSL~6aP~YlZhk-NVE%H)iTs)y z!SHF^%4c?%{MOp=Z0j_8$FlWY7N8cJa#CzGwZ;w`wtkFkahzeR9OxX7>j#l=z&dLh z>Mv(|-e0w1SSag8f~V^8orv!10jk?~m6B`0Fl-UX#xhEwO;o{T;gZx24QV*za3Ulm=c{<2J@uW%}{+U(VcrpsSKvNA~HmfifDt zKo9?~hi|Y+>FXIO37A4h&42yu;w};`{bv9Z=rtotq&xw4+O)YEM`B4DSHQ*vA5@Ekoq7QcTDG= zemz@i=z8gMX2}tnD^Mr5WuN(!FHz9SpCo+#5#cj0i6-dIs40>qFHJ!k7Q6rFxjzqC zsULNyL&TEJj|w-Atuwn;e9AKI(3czE`o2BorI{J9m$S2I1cnr4OuNc*mb~VbVLoKp4Q%->UBDl-;|Y20%^c3coYbUgVkx;m3DlY zkWj9SV~i6C0R`zhy?}rb+&%m0`G5B8k%ToQJRe=&fPHf0_5-12{PeuGbvcV5k9%JZ zVsGfxonD+a3e(TszEeIYq+WR&6r6I&JK(O(1z)IBTCs$5$?w3CJgwVz+L$y(=ZxlC z)*4OLFP4m}ig-X`J(|D5tl01y<7zB9IOq$9$EJ-@i7Vv(rW+&sx|cONLZmqqT+<|p zgP)}sByYtzn|v0QADnV%hbSO}wm8%aY-8h6D$7$$NJUTn!q+vaYE#GS%#+;;q3t$z z=sSR(jWN@k$Q<(uc=2~9>S<$eG8H`;#Y*!(YZ`z4MG_PkGDXsy_9i>=f^e_`+qsy8 zumiqT{P>~Dy8Z`idsBlaxl~amb}{ts5HQqd)b_1gbKOIZkm?*) zqlLDlVL}ZYSTha|4@iVtlGrA_aGxX1on#A=tiG~uJfiB=hjizm<-B6{Uq0)dp(Tz% zFX}_tEaYaP*vs~Zn2FfbTHcBK^Yo0hV~GD1r7$JIF+iABNn|mIof}SSvP)GZtFz&8 zFG4s*Zfy=#c_|I}Q*F$e`c}!^i=VY=AYhO@(srgL-FZMLM~}<7tdL0=p@GY=VW)SP z)?i(hC@%Hc0S`wAj?37P;EW4N4jvTF{Wu#|Axc#f6-n!jbJ&hD-r6kQ4Ud*&*-e-0 zZVT5PkjsCsyO)}<=6O-z>*5;Xz!X2(wQbuW*=i&DK4!~KhmDnhUC7iD!Zz}4$ZUBv zZn&bA^~#!;J15dLYqAoQ^m-|m>vE zWAo=`{n0ScebMm&_l-JN{a#~Rz1hh=nGrkN6^?uUD(G?jb@B@9nzW~P5GM`y5&yfu z>m8|szy>``G>C_}XwpY!7dukkcL#OR>Ki7M4gS`%tC!o7bE%_7bqC#sX1uFMaQE-O zRn1-v{4mP&lC}3-6A!_Gx3pLqQ5{% zF35mL*i`5k^ZO#hWLGmA^WxeK0VwhDxE`A3aYTFHXZG1QyWEe6O+HrbNF1?6kt zCzfhc`a1im+mF`2OHyy?X4h}G-C)jwNEx##It>#i$ZK+kWP|+B#(&bGNj^m6?tS}W z7=-0pOD^q;-iXr|Dt<0$T3y$d_*D)YqOG1H-paWxdd=JKwc_nyiKQQCr#Sl90o#$s zHq`*+S*nlR@^3!}C3yqwoMenj693x#@9DC{Kh)xD*${#WHv4q+f7$1aWOkmH(H7J6 zg{`%eLqnMUMG&JVg<$lFMC_U6!$}Q5-K~0OF^qWl?GU=vF@8q>c89)u*GELyp}+qZ z6@%C|-u=0J$%0Uqq9=218J*~G((mErEgAacHV(!^(jxCsrSj0xtp}sbzenRl{w+$X|gHKdDF3GjAt+9j>gab3+yLBbl9e&BO0>% zA2~^UyTF#hM5;MOl%ecAyBWN*k~1k=k|h|cpSL>j{na(RVVP0onHjOPmBdEdyRY<1 zd~{%5?#=2YU*pd8mHKm|hmCvh6T991nm%Vpm3fnBL?zZ2+i!_WEIKG^ZzlJLOX zgK|qXJr@5n2`1`;ODr^e_n+_S)8B+r{OGmgxFtF!0dNAW3({zY$5U62phJ&py>2l; zmL&4xHV=*h6YDQADARukiILgv=hPNvIkZ9nV3QpK^<6uE{u@eN278T2>C=^UCX~{f z-Ih{zlDb|#L@yjS`d{{NPTJH^N%)OoK9R6J1AjVl%Djs=n5E)_4&J&XL)b3{g|-1Z zkf@$E1%YXbo-(S6*)gW7K*G8rZ(XXIHk5;|P;tT$RSuzTy2R*Tlo?|&zWPa&C%Ul&c)jFTt7N)8O< zsSI^Q(!YKc!eLNS`_KvPsqBV)>-$MpL+Uss^vYeE^1?Zh?oX_qF3_GD>z;KaxW%!w z;$OG-I5?`x#*mhOk)z;CYli#$@0~eZ&@nU<|5?k%Z?tqU=$oqb8WrCEtk=xm#gsMU&J6jKl$8!yN-@@VQ6CY~D?kc|?s7 z;j~MY+}Wlr`g`y;B)Nr?y1?`#EtQ5u3Clnh7&M(~6yZuM(Ns*_QLQh}<73yg9KqIk zwV9Ihu^?r_HZa2I$CJFiuWm;Fy1?qwj~V6VuuECQxbxu2G=k~F31Gq%_t zK@#BZ@~At`$iY{pmbgf$Hg1S1Y9;H#<+r(6pXr91TUx~Ko0o8hO17UV0%e;UP?YD| zr#|=g9Ap6K*mlj8fl6va34eq4M@JNz7T$GQmHcqbkdiq99xxJXPO<-TLRRJ_7Ry57 z(?={#iN}^r41qu~kpu_HdDYn~j!m%q3h*N#8Ax@)E1t>q3(LlAe%bP&dfVks>))>A zZsoxWd+_aG{kgZwfR!r_dHId(;s4IKteQ>n=12Qyr`~E;nAxbtm-4ShW;f{`Sn)jb z**(Ac^WXT78OC``UV~HSht8c}yA=1{EIO~3&n=rY$#Nv-O z>=IX8vvz?|d}kHt1SatCHkpImLL5V{3%hvzzGs?O^7zO9g_ww66o{!^IVeIY-feX( z`0(brw*nQrhfHdl!4sjrH$Lor@!w|{F zd@&q#C}6gD^`Yh(88ij3U-CA|D^|#jFds!blFUN7wo-P3=n11tqZRJ^`I+oRl?ZGh zM9cX(?st%b6tRdaFy8K7K2x0IsGBYl;HMumvd6iC(7!CKV*M&g>ltrt&A((?-soYM zo%(a%TKeli|sB4S9}$cd9CwZD~jWsGuw zu%Bzu*i0DAwJ|cQ6#^np8k$}q4Gm~iq)>=J%QW(zMSznb*2e~p|Am%nTXn56ZsNq4 zllND=w>Ipr@3>S3iU?LLd+A%{Y!$H>j=)HgmS#ZN_}141$3n@B-0HnR&`Gj`^@4Ou zsWH$mqbi!%wd~<9Luv6PM*GG%#lK?UmgUf988H*P^qtmf64!|2guG}v;2X>Y)B!bgX-|9d8upy z;cym`UH^AQa6Il6H5HV=+_Wu#g);9gU$3OD$7+YmSp5HD6tMCdv?FL~&Rf=TrhTiK zix;`@#1(x%&W+AW?VnN`-gqK~8DP_}E#e7{kEgmr~m&k@KTWFI) z`37lPO0k!E7cira1TYV*)3~~}sJ`7Cb(3KOCEku#1h#Yb+=_zL7962&V?OK!K_QrF zw&~{1ojQ%9t^VCB_Qp(Qpb!}1pp9ysT#pNa2{Nx^DYaiH0r zHYn-P9H7(V{a2PtiRyKPgBeYm@-olrs2?5w7v9U`moDY#Q)nJk#;=y+bwu}F!@B<# z6o20Iv1nptk(<=DU}U}O9XQT$W=*T;JzKXX_id}Ra;F<1B`Gs6`LXoKVas2SnbXO2 zeoi#wiUfi<&!>H3t~pQP)E3DWnXHHs5m>8`c8!B@H>Dg)5q9b3@+QkmfoA(^5^l$7 z+zQ9J_R{rGRRy>=B`K64RO^dsLjFIxEK(2>uB8}?pH7sOK@^*WK31&;(_={Hb=JFV zF%6@i6@u0EjuMgI2W$*WO_BGCE-jE-mMPy2;rQwadfI~$KC~DA?bqc-j`ifKQ!k`Af0oHzP3nM@PwWJVZ!X0M z%8nW~&YsR*C^;20AWnMVK&M~C#VK*2Ha0!VWf+$WR_{g$)(Hp6&rhP1fh3k_9 zV-54394FoPLTs$_r54#$udGGePqq4yUu*T`PQrwWqw(%9>zFzj*|{zowC~GUkCWR2 zPhS2q&DxamEPSH#e2NjnkRUJAo&nt4eyM)?(GPWhG0LeI6ImO)mP_t_FxLNB{=PT{ z1y^JfyB$akLXjHG76@Ryl~`smS*J-+|7C#JR^{q2`k{d5)dG4kATu>I6}8Q~Ppdi( zOsO_#OD|0BOoL5p$B3+*;?lU`>fI6`0%D`5`=8`uY`qt54_wGn&W!6UyYs?mjdXEY zc%VC>zz^h5aMeUExn;E^KFC3_-WKvYizD_=)%Byr?hINV>P5#1kN-pL?x?|v$K}+P z&6#XwT(05=P#{QwwqEM07 zY;%3PJ9NH*f8VYzQioU>w>Xw}OLGLW(6`^THb}C$%?8Kzr>z{=22k3G} z3W&C~u9CV_Y7bpjOkAdiaU1|P#K&1!w$z6-n$kO~v4AAr<@qp!7}&@lsVKVAo!S;u zaXlpBcxHjLQKVF$+?R~Cju(}PcE>n(LcmCHglyL+>AteukS0O3S8I1+SNPgb*LmJb ztLE?eta@-uMIQ}SPrRoq(ynV1&}?p>tmt|!MH!9!fsHi#k7y(LuX8UwBAcxZp0TbI z9|qQfA2nQ1y1~1om9m5?4NXAJ{B+IIQ5f3e9zWuhGH+EkcMTX@{QE9+inJfenZt$- z?fUJ^@Krc`-8DFhuq*y{z)`VllrW!-YP+^64kV-ZC|t8IG_ra7Nw>a|E2MPZ)BxCk@5t}$O!$PH5B~V$%#NyQIvQ-* z`~qUQ=?k=uj4jKOIz?6UI4L&Vz{^1uwuDvnou=fPNE9&z;sS=^&>*^!=F9%BFRZiM znXqH*?TZGjB>RjKrsS44vkMrCfxoAQ$Nvyj%FGV+gRnnEiKI`+5Wgv$WpN%#(iTAv z5!Js@Y_M{Rf#fA=RJxw#Hs+?Y5j~c3t4he;;`sOP@y)0q)zp|SDc*TaI9Q@>Pzjn3 zl`9Y=Xstt2#bAnTS?#-7Q}2pGsi360Y#9K)DqZXFc>p9j`?U*R?U1vRdXWh%z~QDOv5JP(b>cwlfDuK_K99mj41Jegc-mo7Uy2 zHx3wCKNA7KJYq(6To0o&6e1GZEHVA*@5{~#F+s7~ct`o%KHzzygf~^@m=mi|^g$+O z*;fl+Rvi#UH$cl**bDngw`qPY-pq-DipsL@0+W=hqj`-P?{kdpPzl3^@>2PWFo(<9Tr3N?Z^TSuGqZ{HAa1j~n+u!=wc zogEOI0*OA9Yq^cF_Vz@|Co&*kR7SN-ezgjcycldbVwGGPB%Q9~mMbepMtouh<3E1e z5IK>5IPEChwK!?zxn28OIQIGhxccc3zwz1WUy53oT)i!02EvcX3kYl3pCG)TQgY4g z3RzT<6Foeh&3*1bLVVU#y+|ys85`wlLS8@R?Nf)Yh6!c~&);0wah;@@NYX1JA+}-w zs{s3RgZmKu(`$9k2W8HkyLY=6(z$RqJgxcbsrs7c=HMbKDPWxIzPj#qd&s}FgPI>2 zL3=07?SIkgG5@1Unl5friPF!(9sp(Ch17H5GLW!t+x3q2HcqDc zad8#rydcz#$ zUHa;VWTwgRBg4E$%E_AuUpUhelD>2Q{y805DiJZF!7aM?z&;cS)2_O2W%sQyN)X`nBx9(^mreQ-89zeB95!uXFNxD#b(_Gk!wW z@e3}q1MZ{@o)nSj`R|3IliMtIW?p!*xr#@eKoGoz&a!$&-Ik~g1PFHabaQKUYeB-x zL_jl`l}%n`$?yRty!>Qj@BmQ5>*tiDfI-;*k(P!n~xlp~llwICtsREv>Xu4aKPyj*LeW z-X%q)&X^uAS;FGb(TF;NeJj6v?P_S`|3Xz6p#y(rJn(|t9Xuq@;3iv)NXF{;pjvBL&0isRvFL-C4`-BMy71o%J-91b}b={eG$Vk720aP(V4)Kibo=y zViEU^DD{Sjg?dK~saK*C5u5FK+$4??pCx;z~k{g*Z?!O^ymFsf#$NgISN&x8aw81U;wGewGd7lQJ?ce_7_RC4j z*902-zq`=4Ja_4V#ooTVUX;wQ|JyXCNzq%Rp_7=AQP&dHuH#Zkr=csDEeQfoF9ox* zPPSyEaS<(A+a=E^ITNX^Z6TPIz%kq?Wq~qo;j`-0=JZ{JoS@6jvHV2jX`J3sBYBHm z!tVNVWepoXd>qEEZ|b7FRsr>sQr|f#dH3_*UHp=b8T!wf4V?$B^tx zgZZEMy9_Gb7ydy7kIBB&@|4Ux(K5W$+?>hZ2)=ZNp2%*L{;iI}F?;s$16}JMx_EIi zCcS_-otZKm9OW94s!XqTPOw9hAI z8fpEA!_kpgyO6_P_Ux_#hxW%}dF5%PRp5Aj= z1|SQbL|G!IcA$&Xp@alGHIC`{@gK{jc9j<#^(>mo`jyG>trB9*S&bDq-hE2;ugM@{ zqGpx4!;s#;=nV%1h>*r7q z56Iu@wW0Lih4xpcAmkH(p9|Bw0AiC*W;AW|85QOv+dtRkPe@M2$vqqLpn0@!0gfrQ>QS-QZ#$hZB*dD7pl)4Vic4~&>Bg|GYq0p0`Zc826ze! zsYmVJYwPwYo{elRpiYf4?&J&{ApTzBT@>PHci!inIPi8}v`esz8*=fnR$Vn$>qKtDW@o3i?jo)hRABp3_id zvVg`gFx-LN)4NE5%YZ{f&$TQ4^HeEAY}E}Gii#?=uCsS=MO;xr9zBQoCMDvQsxp&a zs86dKMErTCSP{}T3ls{0wq9M8kRc#DMK#KHUC({J<}bFtOd}9O5W=SAzaW$%9Tm7twnJ9xVhZl3Cuk&-<~R4J`tKp+Mq5rHX{Tz`hRXOb|9&@S9{@F z*$AK~y`#7!`vn!R2!mAyt+Of^ItnF5r=D86DVbi6~#4??3m1D{Lazf9jW&$?o! zQVf~udImnDccz_nl9ee zwl%IkKd>ZcR;R53wd*{QpLaUKk#n>qoV;ftJHN;_(LKHN{KZ>T4#|4ijE7z>dzX=8 zN<+JngJ;g9GlZ7~vW3gp$}YA06Ukd=2=A#ODCDHO`_eHrPqi^ThbKF`qpF?_7UQWg zs)C4N$)?ROBSbY*F+aT0T2h1>6u&$pAVN}^ru<}FVGg6x= z2Yv{cl!cjoGK(KjHq?943l#XD%oEl` zl14?gjdg$y`i`tX;oOd8AaK5nlvM7qWjXZ~-7g|H%++!M2K&OfI3&I=#20Fi>0RJX zhOxvxZRFep)Z+rntL&U>_?+l0HRIHxKcDN6&gutt9~7hYUvlu{+ThsT6`z+6URs2b zQ^E}lIFe-85X4I{D@O3(aqXq6i%Ww2HQnsVxeSl84iQ=?5aWGy!h%Q z*-5r{ttrSn;LYO}E}0!G>%eQCkNTcyve6(w&Di3f;zf7Ec{yV0w5 zYO|rgL~=;f497m}Je-Ld(MgsxG_^2Wug{BpB=8SW_H#(3rZudkn7w7_lvGif0CXST zMYRYUw47N@Hj4iMMh2jqAgl+M$~dZDngmIPX$m_?kR-y5-(*i;lO{#%R!oycGbq!R zlwFjvCQ)L^eSm~86A=eT!~bQRl^mNg?|jRrh6krDGS8a?S6oxB4xH;Z z`tFuY{fe>sirXi55q8FJRiUIxo(#P#fin6aFcN`V(u#biozzW`Gl0Kd%o+*y)wN+Q z1L|)^BSnpq;pQ?Kl}!VO)UGplAAUU9`Kgy#lS9Mpon{koSvH_{s9Ew(cvwg@%cRu7 z)9XtJ#HPK2EmqzB&d#G}K-rvg(Z`RwF~OzPw)d{0echn^-W0uZ^L0879O);LKzd>Q zAGkDE*0mM$HR|C_IS4d&?A<#XTp=Jdbw;*IKV02ItN;t4$Bi93!{Bp&uWaIYTC-Zj zEmA=PbApNK`cYG|)GYlTD zHM`CrVJT;sTnsi3%8I@0=N2ZV7oa(@-wO!1HoRt9mWKRfEhtj6!`bk z9?72Nc^4y}(-kvZoC8jtIta?`(&v9&Bbzj7a&$&4?meAGkJLEcV?~`bEn2 zJmA=WzCnUX92TeMm7Q_3w`+t@zj_lMB5BF*6Yd+`THHvniqxaMK9_U>LYI!`Lyt z&MS|n1;mM9`id9cpFz4vU9=z%h|dTiHQ2*c_Fr- zGu3U-w0Sa%_Y}w))x?5Q;<5i&M2@@P`xj^9WU1A7H@EaWA4!*Us3;hPVybR_^Z0pO z+z;i%Xyyx#DV;7;!c1D2kjqRelH&M?EH~NITi2nxgvhe!QQZZDqkY~;H6KsF8^^!D z^)I1Ws>RH4TV2ohMUm-taf6?CY8Y|Lx3)`*1JMr;-Zecvq|5@%?V5Tm<{3m4$iR(B zhMOo;keEGt_H+zHa07&rU_V6fkCGx5mU0QF<~JzKgf*#a>mIAF0yy6U7!ACzxqH;U z^@eHj)M*o$c!qp2glVStO5uc0wSwAwTYp!7I*es)+j<M!S=oQp4m|0q`Vv?&5hkH9fAOaZKy0XQt(Y(I&7cH;pd3 zd6qSt^9=4M-I9aIqpmL--2^09wu!~WUOLWqbn(2n&LBe%i60J~{9_5Zm$j)UPriyc z&-c@jGL5~*v2|q+uECfX$7y9MNbE29z!Fa_P!vINHiw-?Ens4h2}#$Z1;oQ~QQ?cv ze4CNu(!RoXE`MWRk?-(zf&H&#kt{`(;O@{*XH=!kYwiV~#|Z6CE z?m(L=DBhqzp_{F(6wf0xvC*nLXTYc>{BaxaOV4`ju_e=y*zR`5zS@zaJw1z`#O7CO za~vXx@o2H4WR+@&=n1}JUzYo;I4-t!Dwjk$ReDcif9ZbCNM51Xn7F^vEFty3T}QGw zPT*eJ(-@Sqk6)E)0x9Q(>)VBm7M(;0)F5$1st)38?Dn=J#J3Kfq&OXd3s4vS$fc(Zn z*~EheleDz6N-tzg?cXj{Bwn13yjGQNFQ*GhP4>lo35YB<2a$O4uuiQ%((YF@V2t*_ zml7dnG-`KZI<(IOAPw0$*i9CT9~$#5Tq&wQ3V(j0=`#{P^kg`FM_72CYSou=1oRL& zWs$OO#lb;(S?7Y<%eyf%s-&K1_dQR!R*J1yGzsyX(ySS50F>v${_W-8v#zTpuBOzz*rGm zy#KhuG^g5B*2ChC&d9jEVE?=++K2zu4mRtv?Zrw(KOIDn*0GjJu>0&zWQUWU?D0b=lT4^i?2ou;elt$7FGdM!r8>c z2e+@Ktz$d2618jFb~evORD{*VvPi{B+&rL70Sn$!0crkNFrXJb+d|K^fDXCA#)_kp zFCt~S$4%n3#L)K9rf49ub2)u2M11~}M;GQ-Zty|qy?Umi>;s%;!IY>dBOPEBU-MGy z+-Z3wt2V1b*Mi{=la?n1?4{?5fDLe$zECG{Gu_La%AHt2xK6%ZQi~}y`v0h%u=4bGHOa|ZQ+({NSJhLupLEEMwjY20{0zkkTgPZ- z&NO7e2fA%C=iKP)9^7bmffA3$?&%_fT+r=C+G>!{9^g|+u&=GXy^CQugU30}+U(R0 z6D24N%nxl=+B-r-(nG>2IDZsX6r`QuMwx%<f=i%BJQwc~{$#|F(Z#>8|kD9PD0 z(844Ze=@Rmx3bra`!;TS!)dE!M{+MKJvj~5h-nm<-!ci_V>TEvcjh`1+NYjtRxeHR zVr|Fu>8%V-T&A)F+t0t0HDi+VwkV~(zV$a_mu!(@s^(8>iE?+h~OdUyb|7foIguvv7Vd<9uk=$W;&wIn8& zSG)<16@By2>dL8N;x+<3_YLR4qENn9I!sB9{gxBV(!`hHE4`Po8b|Z?Mw7UySE}S( z8*)^#_HqcpX1Z=#fwoqkHl)RP*oml_idhiaep$6Dx;>_zoxwgn1%h1bW0#e!QH?pxtti+ z;v4@{&noDNCxQHaogQ0Msu??SFX_~K$>dkL!I`24ineg{vJ9nDdc6o&n(d~g2v;R}zMZM&ut>6iTXp%a)$3SXpXwID zx!rkj^M!9B95d1KV9b1-@ZsgVkFpE3P&9iDJ8f9WnNm|iWI<-%wGcNk4ANFCY8C(D z1g@GhJEDCOiE5XnFe^D6y18^2Dw4zJyvVb@a%h}POE_%)f74P;W*+`OSrT197vu&v zP^2noBDeHEE;A2T`Pr=v?BzE}&J8G!VbQ{c7z69QZDy=|4rag=YpNtT%_0m00$i8E z#~^LKq%7?W4^Qdeh!f#Bj?UN2sKpHmk_pSE5IBH=>PlceifcI)07Q_;KSNBg4H+ z2h)_0zY9BB_mhRi=Cet}i2LboV`*U;B#aVDU`N$>J3PgzF&I796qjFLH_|*%70gr%)X>;V<;RI-E_%|Rg&#>J zS8L16A()K+Xw-1zmtW8Hrkxd|b!%Fg4T2VOh-K*#WA;B!nnN}U3lUs)t%c04wQHu0 zl`s@FzIt*Jeu`oB`jyZ_%n+qSN#2J~3I+V4#)q{<4vqCzXq9&Wwp{DZrM|zQrAcw5eUnkN| zNPw|{PdqDc`bdFV`*#QkuNJZo>$`;Z&_dt-P7}rjMKrsGedF3Dnl-7@&K|txyZpFd zy3M;yulnm~qmb32iPs%FRUfN#q6YL@{nss67AYmR-ym|=$jwB z70+EYVPQyZ=HJGLkq(F^=xldf zp08kS`M(*;q7|dI@dgS(dHM6pE-FF!g2$|^o6QM1)UZ+xnmv6kVJis#dk6d(8+X() zsNz>eb2#p(MWdqM<`rJMcCbc^L2FD!GlJSdClC23QvP_8_5uNVMu9CCKD`jz8DUZs z8k-ZqQA?KZ0vaQ5VKVLOrF2RC{`tZFV>wC7mn?6t{MG>!()<-e@~m!kl+VAXwtN@l zQ19Qha+)uF{j!|Zqr)6buZlp6v1NLnrm62%zurDcTkSV%<`nk0Uvz6h2SWcKg6%!E zJPT+Bb%Q_1GJzyYhu9lLw??#tdIw=}o=jzAaUY9nv8(koYG7b4{&R$v>? zF9-NCPZ+DxWOF)DQ^My&BkR2qoWpTT6vH?BQ9UL<{01)#jiL}SmtIbsvTDz*nbK00cz z?VahZl(f!BHDD-I3ExxNFGI3%z4kD%1Ak=hpH23G^L)w?K#)0cH`(!}vezBbV6_NJ6EY;W?uA(hrbBFAUm#b3ta+h?+P$12O z*iFxR%fYe+*-P83if!~G`=#+K+Ib7k=*Ii~aq**(Q(}*!76E5;nTt)6{dpA<8y}aH z)MmGkGw?gp+S5-&4{GA|W+ahe=&t|SX~(yTV(ihs%DR3BugSpFJH^Co*NXn{R#ng! zxDLAta*|}rTpD>H_-hdE1VRR!)=ur}Yua+xvrnC@n@K`|&ac+NjY*L{pqE_>IheA? z$;CLL`|7%Tcu8rzX@8evBtI6Hxk5LKnFMN6+kyRb(S0op5C4>QmSBgDfw1%3t&d(l zudSrftq)g^0PUP4TS5<}+EbS;Ebe`%E4@ufO=AR%LJ_198?DJ*Rh_ur86CdU`nm~L z!!!K*OTBqu$KuQr9lAtAsor#osN0VbK7=%p=EGz!0Bu7P#|%G2V@%RT_P3!{4PcI z5I0UYg5>2JA^b=3qYeCAf4uooi5Mi))#b0B98~|+T(@l~J!ChJ{ilD<3PQO$L|-7` zUm{%03`eXX7UN|}oBpk0c7VT4-eu{5}Pmp4YwS=$S%YQa2>slLMbWQl52Fp z4JvEV%B9EYZWYhhD`xEL-)HjCh+=ED8{4I~YgZI-=~!`=gK5C_|B+`ggy~j)kT0PV zb~}T|pFlUav(q>2h$v#x(S@G;+sJI$!0Laa*o(nOA}#B(gV(mJN{GC<99~_uuVfAY z`kJ+|d(xs?NuA%4zu`1p0+O3)b_weK^HjUkML@b2O;#~fKdt$2ggDVPVuzPZ3KM1c z!B2m0UQ-buQ7Zoh_I`|StR##EZ$u3mDr2h{p@D&AMSFfBGt}gv)4efq_b=Cf4;E%w zvHVR^R-hLjKpTU*zpACt4-=2S3iDuzFXTST6#ZmW70r35nIc(2*9G93(6ceIvQztV z#d&O*e?3`DUS0uOndzxV2NRAaV7Cg(@S;x_UF5|1GA8+UUE3E!@77iE2iWW2P6u>^ zh1;EOE!Z@d9aLh5lMsZgeQ^uFSTxNd|&Cr~Ye zq467uowuH%5_Dail>(aN+~njq?oRlJ5TSO388@@*DPO&(j4qEpH*$AHG4_Rsea{x& zwUK)%?Gsd(`A~){?Y!;?uiLb1XPHFL;X3OuVVf8nRbwYV^cE#w0AoAfTrB*vg6@wpAP57N@Uw3iMi25)vGOHy-tMOLqEpGDj>U9#gu zSpzd>ZQZEiVU=@28jEb@kdT?2zKJlzBd(X<@EED0wwepi7P{^ZY6qu56lW2wvGe@@ zPk5Dp@akkr{>DE(wE}eEbCnaGb{UCd{79pSr>I3x^WD_LZJOvZX-j0Yl7UH1+>W%e z1aBHvhzzq3Igq5b$T=wnahHkO^VWYT;A}?91adPs%=(k7-Lq)+y?wppI4QaT%r1nI z&NnbxztDtPWeF;*AwyFjl`K{Vbc-@W*hDLqjCkAFAJ zT6E7rypus65}y|kc$Q8iVnQWJ9nysxr^b!Sb5XrA9Yuf9;>j=N5$C=UO4E)&DE*r5 z-ot-MqiL-Gv{Yw;%wJeh@D$Jv;f%|Qd%Q^BHRCm&@P8~=C!ISWD|MElUk86&CknV$ zMP9u0qZ3nCXTIGIrIA_sykmgBnf5`$1P2zjt1&bt@jX#hjfOAA=BZg72A7(3(Mx~P zT6?>(U88f@Z8tyc9$u{CJSp3wY$3V8yV4`t_2ze6 zD*LPU#IIoIf%bjtMZ80CE@4Z(aDGY1iAz;$cXoMBAni24Z=ED8$Yq3nBF*i`&*&BI z1kNYO?yaOKaMvjw2k>CZy`UGATGC@1+OhWW>df4|uiV4bn72wq{+lTUpV zg`GgF<$+Fwxp;M`*b zcS2iE3?|D%w&TPfQLK;iGk})qWh*?P)7U>(*L7gIP86$AoxiOl$f$JkTK~>ql$=s& znoHPQ$s!qsS4yVF&+XWxmr=lp^QCJmLM|VusCrdaQ+r&C!jhaAzOuIkXGA|oG{Ep_ zvs51Xduhi-8yM!g-Un+tvl-Z($R&pqVW=n834C2sz?-Df36V?CZS(0qa%&ag zQ#-o_tdkSLw0|rFJXQOehjDV@ zBoHJf%2HsR++BP378cUJ1>9M3p!b5SrT0Aafh`rseCg^uv#K_+ac>>Fdt$PY(^{z0 z$dN3%Y+31P8DI$0I3YI@6gRhYlFCpegy z4VGv^#F3F)e?IL@wb2lMheTn*o8;`!qsMsKfDPwUY1OO}W{o;;at5NL{^S8f~d z5SAf!)$o4nOPX4GBnkZ|A_L$i&!>PACxx%%+`jVa$?grts|c~gN$?!?p;%Yi%t%Q) zcIJ$m+;D^%7HFJ@m*-V{gt#eK-NZ}wgW0rpI#;RMBAP#7Zbz`z0RxQu`Y7vu;39bh z$afN!Bi|=$vvkLDyzN<8Y0-1eq+abRI}!0YU0ze!zPj{FtH-4=vPliQ^1q#uI@@;l zsFmC$G4+{l)X0pEW&v}C7`3iAm3sYa=bV339b&s%d2iZtQWA0iG$DzXNL&{_+_+us zFg^fvILDHksw$akIJT~7<>LKqlKwr_$mQTQhiM{QmwD7+=dP`u9LhP7R-`Rme)PV* zdu4)gos~L%V9Gm!DIyp8cTL?6HI%p}5aI0V0?TcCQ$-mH?3N|F3&i|L=lhc=8ZM{Kc*^|kI_w{bSEkcsY})h%<6#Vg zn>tLd_A-7I-zZC(4#~;?&UA0>=m$7(c$61*)LPeEy=cH5v(9GYTN6LOm;`i-ve_U) z+U_D%Up%wSTh@?B5~tW7K0dF`E@h53a>& z@Rw!4$A>>%qhoa=@2h%?6}bt|!$IO%H-NcZsnLYfC~bz<%F^+I6P{;+!%{U~Rx7j4 zymRdC--hY!xJr>}iip?mDN8jE3NksgnzH!60V|@}2`IC)873x*L99R#Lms;KzdY60 zXT3tV>QQNIiijyWCuuo$jlgsUf zVB_1g(;9V2k7s1yTzA{s;_v+Yj%c&>!qRj1)^Edz94*VX&iB+=8rSF|m8ST^C!03w zP5D*V2ibpt;+iWEK;4Nj}weqIgO8CgB>5{T`i2qrndSBAe%M)-YQ-#elY}F3r_m-<#-I)v;RWTIzbFbbehQ z*Vdchj3^x$FmGPxtYt<405=Bo`?G~#E*&H!J$#>V)VtDKW+3kkogvrAe-o5q{ooT} zbwa(kp)AZ#9q>Gh8NlQ1J{6amJ}pW@O)u5m*tv4 z+l{O@h<=>cQFX#0;S7j>U|>EkWh`QI)N6*FZL4>zYs*=oc1eP>-&2}1&!SPrjznvW zkCH8f5*7y`{3y>yDj3;JdZ|Tdyn0^{I8Gamfl0~QP+3uGomP1vnE0_Tv%cK5x%_2- zQr6;AmpEU?=_ii7AN83u3C}r9RImnao}Sq|b1)#<5<-l9ak3jy zNnG~3bx0P)Lcrl=jWzIn9N|q&Gzm3M{wlB9n7*4MQF{3@qX?0*PjbJRn)__NR;A|U zsdPVrB| zjE=$)#S7wn46Tpm*|M=D-$(%9qB^x5C&|*vHGL2hFD_rgM13u~VZ(+nW={)P3|@8x z{AyECJEHQs9X)bH!qgUrG=Hk%z3CM+p*|kVe>=?Yktha$Nk>~Y-0f$$PUA0<0M$Y$ zA>oeP-S5Sjh>4FivxewTwH@Q_R+ODO?eIa9^i39XXfRcR^CS*AG!9srX;maiVXLXt zv6vXugyltL0UGyHry?Jo_07H`ww;6Md41aJC@t%33aQSo&o*14S%gCdZ_I9Cs+c|E zl6#eqa$wi4BiQXGoj$$kcb<@JTozX@C&X%=fQ+bVk1^9Mcy69eO3O+}B%NR0Hi47b zv2ue1Nxc`IN};jnwZEPEv^L9F%hac)ANh66Iu<+(LQCBC2H>(lvOHYZYs2om`C$1i7@{f?$!vwNvzH%6#D$|3RywoAK1QX}~h^(DNZW zcfsciMy?Ecs!SRqkP5t}%kY_3Ts@M+!hw!kNI(l*H;=e`u5IlTvF{&JK%#HSFw>%{ zVkc`$B}!R68eBD_Bm?x?KUeSJR>{c835t5pF=DU44cgt$30$3MeQ)d0ddlLd#Z?BM z;$2$#{!bs-sFj&Mz3zRcfW5uuhlK@|7ert5F$Rg@#eG6jYT8D^2q1?32pp=cbK_|X z4?rhk#UZ(P{?^T#5~ZLTmtaj6aZp-;dqJ>2Nm|a*Ke>UW@#lidK8pWGimpskbnmzl z%ZL9jZV~5TUXYjp6pv~gm-zaX#wDNWnKSvdfDKqDSGrGNU)Ilxe_*p=;bA8x2D*NV z4Qi~P4WE-m^;7k^B4unt^Fzj<*!^?;^!^#I8$(%8LM3=e5HtZmz=v?<*E!7q(#ja2>I5V!S&MMTsR<2lTsHs zR*Lo{Tqcs$74tfb$oHEwM>d;=dZDXONlbNgJ{$hiQ$7`SKLrsK6tM&4R@7O^sG>|c z&u0esChq?yIyQERXRo^U?~B*FPnjZ$-WTQ`+vL(vbrL@Sz8ap)*T1`$++`Nh0;b~e zM41|p#|Q}9HQ7Yp7M0t=>`$D3^4T3T7=zLv=UQaFJ&Y?uo(tE~kN!RHp+AtUeCPQv z%d${j^mQ?*&z_ygjG1VeB-Rx2lV*haNiuX2VQP}p?KcG=I&9girjm)~)x~v}u`(RQ zP>@`Y3U(CyJF>pwbP#!{WZ=qp6tsBdIo#;_Q7HWBt7CX!=8atE`<8h_%^Lxf6*HUv`undn`JC{O z4sqwUZPVpqY5GIm+$O6#xBkW(1+XSHCll#H<84m6=u*YVsuxEh?hj=8`epxzu{V$F zF@O8N&j=Yw7$hQl*_Uh)QDccn*_8%avXrHTP#;@m7;D)jOWCswN~zFLgi;7;F*J%% zNt7k-=W%APYkv3RcRwEYKi7B0)u=w_`#j#q@!Ae)X9(JmoUEHKSq=IDSemtFY&bDb z=N}z2)~S)dwKWkh$&|wiOV-5Lj@u9=Ja~NIe~tcEQ)(IMkW!TH>gUBB05qbun@(jl z$Wwg51tJpj`%ORY z4V6Vcr(j9NB|_Bg=m$)!=4;%3Hg_CgYP$YSZ6*2v=c7~-s(@jK3?$hDH@1P*fP@E> zn2p{IFprrn|A_h@)wCyzR_#QOEn&ax6rW`}=4DL;deFM1gSET}cYp47^pRb&ny7Oycz%@M9j-`=r-k#A(u`0>fYHD7_HvFuEF8#FHvw>w+2E zffw)hsne#l__8c?``W?{jNGIa{oc6S^tJQeT}_=fF;Hzb3i?0LDmh2t5n{S)WT-IvII@7o7zD_wEnPYw_4~M#JL%vA_HwNzBfGGP@jqFCf9=pujR7h;#(6bUv?1hBzq;G<>SteV#UO*MG>z3`M~GTtlE3%h@>C z_!sjvl~9cJrPHtWj-8y*cSB~?tD8xqz3^ehcKVsh6^N}>NB_mhB`-uU?$nN`1uZieFGeMssuGzz|<{zplKJ0BtgyYK)VJKY7**$1<<*VE> ziiysQO%LB}oL9(*z{MXU(#)K1JHCvZHeVLq#%M$&fZsxGVY;JrR_`Oer#x5M zl^yoK>;Tdh9A=SI)u1CSSSTR_43JO9)>RejD$HgGHzj{(h#yd*tkf;HsID=&4?F|j zGm>`R>*91W?)3aiMpZ5Ad{{8^ zk5P{T;t_z_;c~-}H*@TMIRbt7Z{j%WTr_zki}DiMuzAjhQ_%bCD?tOu3+XB% z82+`2@{9?Gd3SaxtDVArXr(@`YnIMjMACI_*=WhDmr?u?8RZmJclVRxtM7BSx2l7& z{;5&`!SL{@y*i zkDks9ewtJs zR-XvuGMsC)^2ivzeEHJA6I&2KNXM{G!(}uqEtnyFjMIkORWzjuE!Q2al^LLSu7;91 zxKZCqRb2_PMh1wKd^!eMLX98QVp#rPDIJsihY2B{p(Grb5~wfMs*KA7K5|vYvXUvu{CkQ4^Z^ ze^qMrSr7_CNbB@3lV~xXxV!-sr4E&%I1y#T@ZI}k(=<2nZ0`qQVG`glDObnI8p1c9 z#oFhIEFiPy&x7O@6JIL^0nY^^n1H!YPD4)PSsI}S^(vkva;F!R0%J-7y2Q(fH^3Y0 zmh1Q-Ui#6pscr58&pL_!;GJo0rdV(zlrJ z#My}Y{M)A#zt?n0-WGlphyTMSI=)#aiWMcIHZ{eg(w(2{^RBJ>XOmJFT{v-Ss;yS_ z(N*&&+pcQ!d%dAHqe!sF^S;O0uaW*)$+TjI2_%b8j#}@wp7bfJc zejz>1Mstrq3*~XcX#F@n){aiFJ$XYuPo6wsWWV~bR;*w7k`3#piVanb^x>8v zBjQ==fm+H!HXtpjUC14MOx57mRn`Wi&UlLOq2HkB#{#!oS+cD10yZtUqTF`0iV@2P zat*Lbt?o1yrwc4}Uo5GjWyE6#iD(_klx(W6<9IxFYL|?ribs1c6U2BEu*P{W*o$%p zJa>TbNW=V3ig-*dL;O&JYRZS^yBjY84DZINEH)6GK*+pL)wnl7DO7PUxr7-XWxA2; zN5uWONUXQvpLs(X$eEKUTFT}T*3b}HUUV{$qh!t&aPeiJ{&&iJ@#2Lif)dqv4cOgx z=$f9KXt6HFz9IrLlv8s3!+_T0ps;lu+G2w9%5j9hFAB)`+S-*$q_#K#&3povh>m){2;I5XY|^TOQ~>xjw$9+8x1j_Kx=6t#LA2QxPny1G(N!p z$m!skQJ5pAD<2LJBxsC^T7OwFz~a-(?XpM{F?Z^+QTzHFq*^&4F|}iE2sqGKQf_=d zyIOoJe<5p|e{e@38|3b(l+-P>_;L4~-#2$o`lI)@NdE)7=j?M&b9K9N{k$x9q!$?C zPQhN>G*PW7r5?Q6Lu4;EgWe>Y_w^|}te|Nu9nEWKcditjdFmkK!NT)6iyCL>5Rl2SJgm%HPtl31xaBF9PGSHpgJ{U@#&E8;lu+~Icf$lCkfmN9P zCV=ojR!?p+T;-2iwK_ah&$349`?ZB9k?2HxN#Xy}5O2M|mlh3lkw4K^JWfGn6T)rg zBWnXhkAkSBSXMWQ&e`YbulIqcCHl8#)B*rkm9QME{Q6VuqrE~l)x4fXpCKCgAAEF~ zUg#IcD@(^9?(60w=-PE9e-S9SIy;LZVjv;F%-6?6W@_3v;s(GFYgoG-?v0==#Fq7* zr}+)%aUfN-+#W8#{g!n@q1!}40C8FHw3nC*zkhuX;hMMGy;9&WFxceXPeDvzfe|YE70BwZ9Er-^J9~bPr)>;oNejxUllcESN;2MK7Zx zOIW-+O*V+`(^Gl;v(FWF;!QUl{*tC24@y@bj0;bZWrkT>x`4$*G_x{Jt-n?;Uit^d zR)qI%!$kaF8C@URkJmb*r6=d#>-sa}#6^x7PaE(1$=Lo!0{L6d@1pb^uddipO*IwK z-(g-7QGux*d+HSFT<$a^>K@j^AgiL9svo}L)Rf43p0<*6@koY_HdU1EFZ#6a(xtbL zZpT4GSymq*BSyqG(h-8l0BBKH(|ldri+u$5t2XPZp+o7kA2xBna!`uJ0sdUrBZ+1S z(#<#_A&T3#Z@=7yW_AiU0yI!}A6`8XH98d?AB@H`9cJUhwHnQjVX?fplAHz~Al-*F zZDMzo1h$o|RhGJbzd0|y?CbX3pcPs}vJt-(eg8I^K)mq|+n49MotYZ@e~RNY$Jt1w z#!q2AmlPy%*=Ouos^|}wqb(Ph#0I6r#Y^ak>MBlYXnP?T8{$Md z5gN*Q-^cNQar$AGzA}jo-7IP1N1&w(PrPtd> z)cmBu$T5i&=bOcAdM-)f6Gf76%|BJY+p{`~zQahsAtQ&$>&)G&2JJNv{bgNE$aKnt z^pL7R;me0NM}^###0EJr|7MmE^E;MKMK1HN=OwE^&=T2=67eeaT|$p6A^yO_UOYvF zapGu*k4Jy^8F({U`$y-LJeG5-?ApS(+oqjoo$x z-yLc8VPvD0GHeNxNuWt3@`HH%5mS9f2B9Wm_f%btqQHVo!PQ7;P^U@ft-D+)UJV79 z=uYr0FuxYi|5-Ak`RAyHKRf`9^`^Sre;3WdSYyTG;0w>w=1VW%pnW-qS`PtWYCfu# zxwtviF(~w?8ukLv@73p2wm(v1;Skj%V8B4Ny2w>zWUi^B%x4Vg`g&pCN57eIqtmeU z_5m?5>b$sC?#_!Bc9mVH%g_5za;#m;1K8G>r|$P#n5$&eZ8;L%Bleu%*?Z7j6Vk!M zWC3H>JpEy6sXzbki+Q~L+7UBaqPULBdr?WRaf>1XE zq(<~%5s@m+s@w|Yqs?56FiDsmPM20KuwN+={jNB#D}LE$6O*CDQvZ~3OiP*e28Xbe z)%&upj*S2Lrel^eZp*$^+E{l9cDuL|6_wBPb&VWTR#Z`nM~GO{=boux!`yEw?o&na zxWxS_#awWIc9CAw8%P;DJn-xDI--5T8kKXm=BiC1y#(;{*k6NfSOuJw@%Uflth|{( zmq1?7STW{Hcsu_+=ag3N&1S!^hoMaWe5#(6)O9qu#!V|Pq6r(sC8d;)`*8ZLtY5(a zYdl>3;dSQI1&NkF3s^2Fe>RQ(Q3emD2SQ%A3nPXxVZv(^8{EQEMwZrxuvY^#D>KQm z>2RJ)^vI9>21@TrWo7@hez79fC$-xk7Ub~6#o(Q%H1jaK{9Z`_guWMK4pWk>zz8#o zY9@HeT>f%iX>ut;V@M0U>Zyn9YF4{DwRI}q-nw^hhh)PMy+ZW#l1?QuNZb1PBh66J zepHzx!7V-fPElH@F3=BgOn2u@NSg7Jpqwm+>icR`(UQ)^^FS#gfWLg&vwsFc>VZK1 z+l-q!#jThm@xE`a%pz2mJ%e>K9n<0&Rb9sRhHJK~;Gb)0M?7qpkBwn7; z{ok#X?VPO>KHz(L$eG=kjfsGq6^R{3FjiDH558|m7mfiRZto1^Z$qUNkn1fh&fB+t z8_oyHQ)70arWQV{u0*vJ4G;NG&Z?L$L#hN;BadDvYVkR#idu1kJ$m%_XJEPt32~H^ znAZ%{>w+v3dG+MheGTh1WFUB|?8WhO*0SEmJxbQz5CZMb1C?^Z-}I1;iUi)PTQuRU zCvFg9nv(iS&d$CcXJ^g0fB3WF4L}Uh>9yy&dJhb(uJpRViYV_B2NP<{URAhQQatL+ zs6h%kJ#bjbHL@fcXm`aB@iJkGjmt{-LKxKoxp6{3|B7ZoE?7P43b z6|+bM$?%VXBPRX$-oF}{jC(KMNh+ck!KS7kfKUz>)i*jAzr*)AErR8PwHk;u>z3yN zZROCU{B!d$K83a3KM&17#JuD5tu&XyR+~!%k)eBjk86QbDTZo5;o~`-&-Ryv(n!2@ ztJk=0$jJ@6T`O{V&8AHTT5QyzbRytV|6`&^Cd_(ZrH}L zqR9EMZ=_j2_WVv=me&B2({m6)! zLVWU=fTCu6yxaqJa+R=Yz_mq-;`cI!ioj1$>v+Ow1SgDgV-d7j&z`P$Rz8C@hf_~G z80bb1kgb>OGp@+%;x~ra{<1j&^@ld$bXZ3HXkn^*Gre;}xQfQO8dVL`TDvnXx|h+F zXNT6WY@DRi52WodI<9<_QHGgFdZH>KpPAEEUlJ;)uM=JqbpRBx_Qr@c+&Gv! z#n*eL6d}unc^RG2WhPARs!VG`KYWZ726hC^l-*9$pz`%8v!~Afh--RVsV;cNL+m!F z>dtkDm36itTMl5rH(V@V>SWbfS9nx2Ot$@zU)A9QljAsN=QC_|Uw&}mLupmaOIOsf zOcAXqX!6wKh$sAG{&m=PTD96w9gfh>AT|6#1mK{o}+(nHqD$(9?n z+xgk2V`hE-brSMTBi+k~<~^gb(a{`WF*X-gTQ&B#-!%R4m=ZrQ2uJyz3?QEz37j3( z*K4&_*QWiK+EIEY%w=|z*m5#r?H^`j$kKaHi$<`e0{P&0&$93SERY|&(yLEQS<__E z(~)MNuIog@St&IBc`8Gss?7pYH*!1BZH7Nxd`eM&C3g?ivVaYBq($t%p8^R-DwQ-j z_tFTJAy}0N!=njWJ9s{vTAA@B^@TH)EU{#)=@}theZ1fIZ|a)Nu&RLubmwwSO1cK$ z_4#Z@O?AgD>=dB+Ka~r^^t06K3jT-_5r6aBgTego+S8vOd9C!kbl*!?$xodyyGhUf zJx7@?x?)hL-z?&o08(DucuET=kTWOh4>e8G!ZzRXD&#tPCINtU)Z5L2He3917^LK1 zHtxVuLP^N_(6&|eI}0LD^g+*F-#lV3qd^t)ok#@G_bc^uv=NKkJ9Mh|uZTF%oMypf**w}pFRA=(GpFZ_n+BgHLcTT*sER@)%JE4__ zK8c>vNzbooG-^3`7>{n~oz)>E?+&q`aNPT@Sns>=Ju!Rbz#-$pE1Jj8YCE&uur@7L zY1f-uC9tN&u-e8qs<%95()pi55fRZ}B2UHkpZIG7$3fZ?v`lNp{#4WA!CK3oS{!ND zWBcyDInBS`F0^PjY{cTx%i?{PeT=@>;{CfZqwe)xa&P608>XICRtXm0pCH?BU$(&7 z8n4iw)6aR>^ir<66GzvE5qO*r&$X9Pf|hYmWB(S^IGZl1o3}^0Wo;9XEB5T!u9H~7 zYi(^khrY{uJk|68$Gx5awz^;alk>LWG#LW25af$izT>3xYs=bd4YbEVAor~9X6=PJ zZqK4x2u+3#+@_^e*+0$TJiXjy--_t8l%%9BH15CUBq;ha%vgp+J&aGNm!4`EO{xxf#UR3YiZqK$5RBJkK-1Fxbc|5;YW8kNeySuNO(S-A<%pr?L`6k00%~h-MSyCBu59lUr>6CZ( zuTq2P!-{3VzGhaJ92Yc9$n8_jqGijLFJXr#NF#mT@4K(9^jZ3+aB{XLjI!X9-k67- z1`c#S&Ued0va=^mDIOYU^#)0&SsN`SILoc~hr$b~^$KknblMP9S-?1wyJ^Er#jfxE zQDKGoeX41n4-C{1hCB_6)L+YUlwS?}(Xe!|$eCxMTWGNt zRU>5}(B=%oru!(u$jvpL{2ZU?swA_%VCc}H$+&mR2Uuvt|5CaygC70Qxig=PQ8eE% zZ#s{B)rwhbsioiS)W&?$RQp@#%iD65g}()9E4kUkJ&Bgd447Ga4HtV7Jz0=o)BVAS zrvK6HnXXdhA=q*r#K#V}9q*`FLzpH&NQa@_Xv@hktXIW*!lX&2&c2_*E?hXa=HQpo zf>xLnx1*SxV^duzyUFj(c$!D~4VH|lsO)VI=LR$0C+B?s=DoY0cjrVEZBEs2GQL*s zi7Fnt)Ip!8ZDj75Y`W7h(=$=g4D>jD&S_NE;_Q_}LpN+>kL-o=*R(oC^YUx)l?%Jj zk_WR`(Ht;ED*KvGyg0Uz=F4JOp-&m{-`b2i8a-a5 zYp2)>m6o2~vrR$*B(psh`cfvWJP9=uiPpe{l-a%zSZPg=%OaB;@6hr)vaODizAK6{ z0QvKsI(8f>$jvfvpf={_lN|;tmZANY{Jrt#pMN%q=qqW2OuF( z0mJlEtRz_d>@MubRastjx>}Y6T#oN2$v*)CXbAyo+m{uUT|CD~<+n<+zQsRk5 z%A>(4!5fndk{bp$lu#k$*D`Y5Uozh5<a|0rKa@6GJc zuft>yj?D7&e;oRysasK&w|2DV#`LWq$);T{rq7;V6rDNZS?zrTy!^ydEIfYN`SI5-$8=AZPAck0j~J?P~(03VYN zymqa-cem&Nvvi?Q;#sMw#{ScueMqT5DfO3EjC;oTljG0{<~(_)7$2b@-xIRk0K2Iq z$vU)Dik;=UVHf3$&)L1~fJKK$9G)!HeTmP=Wmg>~USxbe{;edU&|uDN1Am6#?_0TN zO)8EZGg-#2?vZ7Om3&;yc^D9Eitto6I2j!FQh6r#0|w_O1amBneQjt2&UJqJjh{$Z ze-Na%|2d&2i+MxBdSf{D9nD9+i5CeTX-vVofPiFZPNTH4M_nAFjQO*3;Mw;tbix-S zI;z{Wnu9-oAF5~q9H<$h_3~-B;TO`>T4PmzME+uNsAK1yG^19Mgw>ab$AB;WOOjfZ zumvHR+Sa;%|0`ramqh@)v&gKCUVv4BB#lbj|Hx6vjiksC4|oeDvI$o9Z7jyoD+9&Z zAq~8Q^2cpgRTX!of!b0zvl%^KFo>hD>!0KEgL{=_H9Ap4@k{-?my7qzlZ6I4(_o~# z6HTG+JeBt@T!<~5Aq<0Qus)Q6k>etPX&N(Br}cw(y>Myq-RilKZ(e;T!p;~%pNmOz z+q;EfPPKW(zcP!4cU+puWhudNyL5DP;}S!Yl7`t1*F?-XH?cMA018C(^~^0Y+&Zce zOuEh6<@;+Gdz9-PKYjdox7z1F^roGZRQFP8)rfEnc+pS)>6;HWi2Ry=YrHwJ71!{1 zpUwSC0{^^u)61*BqT`AjVVwWx$*0G|SL-D_S3;|*hlgU-F3E}!c>DjYU18!3m$fb1 zU=B@ddt?F~G#%*cSn!LrbrQncj;u}CW3Lsp9qvh}qqO9^cOOD;SyH1)I0UyKz-=>` zG~0>=wlyWgs=G%+e!Lj3xAw9Qqq_uHaHmJ%sJ*ni`iB89NiCLdzRi8XYoEvXQ&7C@ z4uY`TjBTw6x3yyV%30E{wjmkXm7OE|zh=To@Ixrg)=o2!vbT(RaVAQ*zuqk!`Q-@0 z7Uudqg3@X|p8bcuIYAOLQL+EE_;QdhX7ww$vQI@SENJ&{L3>FkpeTbs8Fi=8IT9%f^+4QXH)_PP5pdUvW1-e1%$sm!37cCDP8x_SW#U5Y$&<5!${ zj#q=$Cv*zApS&oJRG~jU^hQ4Tmxp<9NRfrNqVuzbPifaI_cLeCd`}p-@Q+1`qTjrC+$jZZTwk2x%C{a~&4j!g5`K<1*&0~{cEo;dxf)L9_Yn5B%ERP2K zoPKViS=g#kqOGxbv*3>>FR?5KQOs?RD_^%Va)cFTMs}%sN_iGcvrA)h8+LT=Q%Q@I z#<-xbq8)+>84H4g=zsRTArj~w?S}2EHvHxC+S#rzjb8M^e+BL9V(^^wOR!S6k*&@I zb*|U2VR#E8#dsnSe8~t^ZHvk_8TS0`)o?B2tJtWLSlQZ{5|h6$%RfJV&a&6XkG{N^ zF?BmKrP?)DeObap{E1Q#Pq9OhrRc_z&miJTfB|zPvTcxc>$%T1V$N zM}&jzX>i|Gj$xX9O!OMdN;h`-`Pb@Z%kOmOU+uG3lo@GVO)+SB{_W&H4EzuF`)M#g zQ4q^6tO~FyA`=-ZZM$OG_4d7c<6m=4UiN&mBizhr#Hdj_WYex#toK%LRz=66*IrvL z>)`DDQ%_ab6wZpwrvwTw%9h@L-AysT7 zs-rtg_jtQ=vl&)6f;O8Ex*U|fk>?3(4x4>$q6?dH!iyGbl?QLATIQOR`jc)GnQ|D ze$k5|pGY!($&m_w%Owm)V=Q1Nc$EK_I2|3}*DW%*J#+Is_8Og2o*eq(;)S-XtegS3 zUh;5)^DX$aJc@}?Azq~B%jm^t5-8k%r^xpdU-4E}X`pKQayPVF!yhfoyftOi@j0G3 zD~Bq@RvA;CEiT#N;H|s%ERFJB|6(+F6t4}HcaKm>X@`f_;Tev7*nH&PUbd4o(q+>P z(}d=Bp$r5?Lq9pxQ|x{jGNk>*gy*S2t;@-|+Eu++x(jtM3$P?{x*!QO|HM5E$Z1!lloVT$y9u?;jO)CB3$iekSR`mO6?1!5&U^` z;l-dGJN6VNj^5YjTrGPo%X|?ZJ&X1nQ6Pz{r4>~bYE#Ozl_`rVvQVDBin^NgvM0mUFvw*N=sagZQ5PZH{+COsM-n(t(KD%kHe9dACnaLUp zdvLqbgzMK`AlA3-+eV2h1#{mD_cFJ4(9KYWJb({1W8G=Q#X02N3)RH@B=v??HO85&$ z@ZIP(<>{oqo5EGO%vld6wO$B06gJZ7UoO|&$pDv}|Zk%5#%?Y~!$N8Wo>)FYmhH+`R z-kCEI5s$E`jE&pSvG4x=^K*8KtiPju{s*6#OeTi)wZ7ebzR;v+t!piIffS8oY;nu* zpO(ZOY}WSr{_?5uPv-X;PI!D!bk37QuYsD zUV>%+HplPJyv4gOAu_s6ipNW8+78e&?w%S;;AuaB+yD7yT!y|M8*g9Iqc}WjhbO}= z`-&-7E#cCijwF0p*0QDG2~?xokozq_n_tAS4V;5vN(-V>9f1}nr|49)$SJN4{3l|5 z@c76@a`$Y$7+w2`!^0DtmWfOywpFX4BIulP#yO+%u(Smck!UD(o_%jNmzMLy#M9p^ z+NZsEx2vIfHGLmYhv3h!sp0EdDwZ{JS}dZ~cG#9no_Ggj1!F`-v-r)UPL%lXBo|-5s>n7W6NR$v#?Jy=(ue5@7iTT z*F%SF`HD@o6J(L4UYRI4JNcT^B=D1HMqS{>NVkaPye~RcO*1fe@#P~I$E_?&ENhzK z%!@gRT%Ce`d6bqtAse(5zr5vLdiUtD{e6c1;rBwyZbMa$t{Nkl}K%- zd%GVOPVsWQEIYupc1JkZuUF57>{GcQZEvMEfRt>>gZke}de$MJtlG(#TQt<@ek$$x+Zn-VIOC}Z@ZnTu8@Vxx<00W~M z^>)t=;>1m+Jf%P1qg%Ib-R`VKSq6wd6YFw|B~I&d7t+yAiua8zxV(<`PawQy-i{-=5YQj>29ycrt!Bz ze^IVIpZL3#oB6Zo8OCcRsfM>bD4Wi^@eyLEG^4uFKh&&KC*G-v|1`t)`E{uFGwDBV zikQvqVKjMr{kD*G_<(T)0ZC6@`|oPY>S5lqP0gt&qPAj1$h^o!%ccE0=j6ftcZt%q z?j@hv)DyR|W4y1EgW%2rDFjmLiS8V>571-pEq#tn|gYA^{Q1=MQ16v zsg0AfbIx;L1Xp9{#$?^QI$6oL<{}9OgKZznJvo9#ntnucHI-1h*PlXOd_IXNbXVcD z&7_C`pdaQ(el{>?`DL#BJ=;QKvZ5AQ3H`~cfC-;D)LT(Y(6~N3sWz@a_que2qYV__ z{t@Yn5-8*}Ct2oUDN5lU`9F2uIRY!+(NZ|oq-6tm9p~r7WtCQ3B~GR_V1UD+ z!f_4`tw`K6vq)~$Qp(vf)DE&3<#x;jrh<*o8Z?@xqq1J_*@rP^2mS*AjH8+aB(z|3 z3l@ttjKSmQ1L%>|Q5ca>l)vTGzT~fx5P$8$fcL0XWKr4li`@s*(oDtvug+kwZ0&ec zjUw7LH!|vkOnoZ^d{K9Bbl?obZFHa@C$bM&yS_ToSX!@b*O5Onj%h#T#@E0migsHo zYp)3xUT<2ZPpE$@X6tc26p@{D?J~P5`XjF=B+P`Gx52`1?XYKHFWSFfFQtU)5?jKy#@jBJB&YmMLQ{PGveH&$6~ z3xhwhR`M6UhpTa(xeomf18Aj7=GaYk)a0f$=qmZSO2vsP!ymn}_W!93M;k#4I7s^y_$e?VowrU0_ zKk>Uyxio*POtqJ4gVqRA%G{N@Mb}AW@o7&_M4Z`%O+*qwjA9n&{U(@u!AuvIBrX@T zHm?V8<#L?!7^2@6J3ZwJO_t&>EibDp^iQ!OXeOFT6-eZE@5CyDRDn*03Vh$sYouf; z_N56!j-Nh&j-!O=r`z00w)4uARk;`nq=q^8am)2zQ^PU_`bY4vD~Q1s^MdR7H5xi( zhz(5ESK_F)z|7A4-B!+t#*f}a^Xqj`e6oT9i~UKmu?&pKPV3yr(2)kk@~C-E?zZ1{ zcx83dzb-v9DY7>1OiQ}G&~aHFEj9}A;YxLNFN>cSO2ra(K=Q+PK(NfU(Z1T8dAX@w`dx)ggd+I8eg8nPB6uWYj&u zZ_ck{RqzR&kB)7e)jU{$uZ2D*IFrYhm*%$s!KyqbEc2KfnYn=6Hlk=2#bia!JNd9Z za*d)LD-`{opxtH}8Vjyf{cDx07IWszv3o7nq(OyNWK<#>sAH1?+6yHq(A{VgWvL>w zjS=q_Xo%1C&`h)5y_0GE0WBvL$a-4hj*Z86NFJ^Atx%flw6sH|D-K*#^q=w&^Kh_i zmpSi}1o2K?`n+0tk1IMWxM;D!pD88s|Mn7O5%G-gGZC}dbYZ5)DZ4wgdH?LEPuojg z)K^wH-tN)AW&Fy-uCfn>x^iT8O30piRg~3Rsl1Fba@Bp7A?MHfqqQrFoE06lXny|^ z|9v|E-Tl&)TLOpCOwGh|q@sLS6}IRvT}qh^I^g^H)&qFW?Q;2Us$$uMeiI(Pgxwep z5{VVPxm`(S#T~ZhC9_Jf&Tclm=fp@+vDC(YcVJ0_w|MIaTN2a`o{FqwUX^^ z-Rde?>X<`X8)Hs+7p9m8l^se$9tIi&nkCsOeI!+nXqF1j+9b^`=nKVM((0fBSvhe zbF`SIloPYi@DuN|`HJkhbPn=Ri!GLIk?5-e-y1lVcu|yGy-V&n5&hF!w(O2DScPqm zEq6q+1H?9iMz-4mff>Y{!fSdpwl6QIxXDGbTY#&yct8E+LH658@3DfA0cI`Nr*3~d zap5V*^}~Oymr+M&BJcKg>hPY=+`u~!<-X79rF4?_&05gt{W&O4Xc#onE@9V$La zJtU;ISpV{i>#(1ss4O+0h1I`9pTb@8bmV2$$;=pD$U@&f93=>J~7P)N~a@1=*w@2;Ezx{{@|v?Qka1 zmaSR^KlMK7RGf)7k8rQus*fd|DWr;+hQWOBFAatBW($RpgWIB$8D+> zd|7e3|LssRU$fGpCG)-w3E)$JPx2iB3sBhvPlXEPfF6K-SM3y$L>7e%Lt+#>t0zvy*u0Ih;|=6TbK*pP0=VBnaJozb>_1A5E&h++2cRgb{akVI_4{2A|#8 ze^gUKd&n8#oeiD}Tc#4$8s+L{!Qa^9oQDx*Cmn)k)Mzprzp}GwJ9zV+N7t1qw=kZ& za9qMABFBYZY()UEeK0Y7Nh*<>M_{4!ADa}7vxxqNJhlpsp zs`IhsoLcGXmMsB#XjO#l={#=Y{gzdyUtjiIdCIFllqBBPym;2u+zs=yqQ?5i(%MUs|(? zX1ug7-^;F4+n-D`0c{c%8k*}R8(;>fPqVYBWzl+;bI3`wtN!$}UgX09{)*`aH{Y65<3-O&f%Ayq>T#VjC(0=! zBV$97drDyQJ_{|M^yaLS(~`L#T4)aZz%6xbMXk~T$keBbmZkE!;@Jmp`emQh&(;up zlY~q#SaO2;SCpn-sb9r7m#w2VC|1(=_NhVDop9kLA-8KMM|mI(tJtI8>sTs%mz5k@ z8k?PYk)uSt17N)BjN?V1)-WuP1(|QXmZ9K`n%(#%KTkW_-~xCATiyia4lK@m9EAnk z7Pt7-)_nZtL&JZ{Iesd#A=^ye*_m{!-GSw5-%^x=p@$9)_AP&2<6mSDxcgMQpA}2M z)P1D!sYs6RYjFT{=bCGb-=xgHs`+lHQ9C>{(rf#QKDf!m2eR~_^Awk((Wk>i`99k;o zZQ8cA0b)vfZ{$@+$-3*EyD8zho&V#*OJu17zkjgEyu-~jevA4vwJ~bk_&6wBg$VOL zyT`#BI~acLk~8lTTqAX>yj9Ngiy*6X0zJdVd>XK~8xfuiXBkL5&7C3S?Z3zMnF~h8 zh6>Zn74avlD*X)-y1LNy-uBj^WR-wp%o|xvS(WHL?R#ZMa|i)JulbKdOHYkI37?-- zYScr~Ud<-huuGY5R}|GhBJT~VOOYy-64CGycl1_J-?`MCLFEJbZQZ}W6a0o0#YVIo z!A#J@3Eoa)U)pww*_Kwlok8?3P(w|LbT5z<9Fh*j5T?vRG?BQ9+VmOW2D%h39;JFx2888>!ak_ERY0#CJQps}0D*@9^s@U_@V5afc! zf9GInPlk6g<*m`Zy?mD(75(32hW=Tdp^ZHzc?y7QS@zz*AG$F1%O*u%+iSw5z<@#L zQ|H_cGF1m1!Y0}yKoej|F*f)!F>$Yd6+zk^_QU|V^yswO2?oBMu5y}B-u3>BoI%Hx z)GTwuX0H~%Texz<`i&cd*kEcJE4-S(uGX6G3ppZp7mirF4V3v`y^4KxU#mJf^LkYu zSEbw;>1F+sAL7r1Q4e{I4eM8ggUC<{5cpb3C+ zzTZC0?z{5yg|yAT{kDzJ78|NfxQo)v8Y;^*Y9(^_kgm$tn`km#BKnAQ&0h*dB}j|B z=cbLL1-cIQ`%di_ew5m$oZuk#UHmdIO{f>qi#Z&MV9j}3SqSr$!yJC`$oyV=@9;RK zt`{(@hp$y=#UNItFD_VKE(#N3{mjDHa~?stB95pXj<(T9$S$~$HT_%?T=YxyHaloc z0`Jax^KFY|yY9MHL^iY}h_dT~<$tYj=YqJ|&|XXD7yoI)xGPpb5eodsJ^OZHJm>2c zF1+@l6?Jv$FNei4mn{_K+L?sE zw(Z_M?ZJ{hO%1oL-9*p1yJ1Er6O-c1_me_n%9|}WHy@yN?T&XKZD*wL7(Ul6t*yDk zj>uesjB>6`)vL{6>qj8nO?xxv+OtA08twG15duAo)K>P31p7D}5FkUxQ}cp+u>njx zePytYQYSfY@hu9nDibQ0q@!GIK95Ap4=Yj@R1Bh|ImY^oE8s<{kykucId~W+$#GTS zG-!f^yMKj#PzM=d+x+tttz*ZI4LeSG5hxb=C-oj!OSI=nWG6Q+EC5$Hzv9zk2Fd$d zbhD0A1XwD2q7<&;2*8iLV4;gIJIa(3)~I2_w%9HjZMbpBbXr%u6QGGRHzyr(;Y zf?5kEM-lIq|LwAt@-2aHLLe2CoJj>H=4CwUawBcUkW>}5Apz%CN8t>+B;;}8dlczQ z0llSvkmV~*iSQIjv}(4QA-LHZ%ci}t%HaFbPl4?3gbK)Bv>=9unhcRRlifYx-bX~T zqO6idvV&_@-aPeBZs3uxrBc7N^V}=#?rFX3IWvx#Uhn}EK0;Gw;|`f^)Hy<;6`!4C zsdyO6#PKi~!XLZgpa{RAk5~`LJ#>3f*Dw8I|E#TKwfWDP zqYf*NC%Q$VYsudl64F_w^b~JA+B=IrOyK8>z(^xwD_bf$+XA&Hq680f-pWk zh4G2T&c!zLBc=a%#Sl#7&T3W^oho5VJi_s~(QI3V4(2TJ&yRn~&FO+idY z?FsrQ4{aLqld%Wl>EUz)Y42krH(ZTG-j`m~6`Du3=V7MrV5oDlCN&}RZ~m;s-z(OPn7cnDrbtH_bmY%*6Di2hVjsyIRHBJ^k<3tZ}Iv&36~WCQ3h50Fr z7Qc7GJ!MyU=j#*3jA_w%XWT0fngmG;Ex;4$%2ryZb~seQ_bQblSYq}}3sO>RqRLUx zHA?29keW)JP|lDH#~-vWuVKyd@%A*)q7kXsBKBQsgim4{C3?`0wSV+Go zosJ+ZqGdt5se@C?vxIc1DWXI|9NTq49ncI`m^@E}QbiqzUGbFGpo3Zxtn}vDxX}X8YMAUk)L1{6lgIOIm$f2d>z@zEM zALvF~o0K@sxkvS9)bANKh`SxsK1E9tI1IEl=tenb!7;|mc1j5GJ+2EDjPr>rOVgqO zx^ZK-k6YxIkKZp%?8uZ^UbsOJ)BxZ;dmd zE3#+0W>w3*K(?w{7QI-(l+)uU;olj#-#qwBPz%vw{`JAfJp;xOKlLH$I=<&2$H4(% z(?!&d0=4yVro9mZ5QN-ToHXelS+toXJtFd1Q4$Qk%e1Y+b1C!`l#R;)I*MVGW9KK_ zb}R5=dar2YMY@a%+fxAAR4Uk<5*fPC2jn59v&n(D<-b$~nL)aiCQ-}YhYQ&t{cvC0 zBv8h!TPNZDeJOn{8C2D7#M%wb|L8&Iai?JyDIdqqKkPK~ z?pr9HKcU4(AVEpXB<}aTzF3zQ4qVZw{Ym&KUP11Y&K*16xlru7Y}sW;1|1n;fXxJ0 z_ldJA(h7}v^42TN%X~j)WCz&m@L|L31UtEss@amau#dd1_73$d50GWYc(z4=-A7<- z8FNv3?b)Uj&xX-WBYZ53HAyGhZ46YUCl2=ZfcT2OE-8tETY5i}MN;cGYIB@?T4N*eS6{nGbDy|_gpwRc_&2l7At<4~HjTu8`cIm0R zurDsa*&Lju6CwV?pHG53kq#rA0gdErQ@k5C1No_?B7@%E57ITqCblQ!Fg)}=Dk_+? zqM6HuA#7JEJKbBbf+Q{I*!wCLxa}S$9K8WevSh>txudT(#n0~!nSv`dvclJ1O zToJ{_zq2@sFrM?!kc(StC%N%Q{Q;= zAv;23BfOI69rE3<5ANhLMPfZU$d^y^E3r8{PyjRLItkF_T_gk-z$agg=oYwNbZB94 zU;}q|owhpMdjC;cT+uG-)i5e!c`a-1S`ZcV7f_RB7N)nUsp zUxI8XiGd#h%&iL++$pNOn)*Mo$WiNBDDzuf#m6b>%U3YJOP|x2EDrjL)sq|?wtod5 zF2OoKoC!SO&G;es0H}s}5_Ks0ZAK&iBRoxz4w+sbs7@hckBL8~*JE>Bcb@+nOqH|1 z{~n+Je4fV1s&a4|uy^>E zhN!zI2#1H#ZSvx;XJ|gM3nIpLHR>G$7zW(8Ey{HmUe zfnB(Y^u9PGvb3`0<-SLF(_-T9S#NX%Cr;b8%ncGkEw_` zXChxKxzi&BvU;r`7m)$~Xzyj0f#4zphmU^x)Vx{ve7HbN8Zu`Bz0I!7 zep1Stpdq>HS9#f(ojoC-x{gU)f&Poa7%j_TU9FGghg*5;T1rdLYj5LDfeC+pnA3GN z%X-L=Jq`DLG`jwJvrrlIW9mg@wzj(ELZUUcU@!_32QZq6uU?}*#5dI|Y~|e!O%-j> z1Rrr+qDbKZ)HXLSEN`|~W48^FF#Q_dJFBSFi!gz~=G1ez;#JqZkDV?`&5U^T_^}Ds zu&{SAQ+g1*_hwFRDFxTSy0%j1R@t73)-E6n(zDifRg9OK(SUk_k5~^KI?tw+ykbuP z4AM4$2avhhqoa;xDnn?`{W-H{&3avPO~A)5XU`zsF|FW36w}m=Su^h28K?di)VTbr zmiFZwzr-)!DjL<5q3h0pfd_Klw>Wk7tle?Yzn6J=Qc13Umo@^$S9@`kGL_7(Qet=$ zqf%LpUl#=K`a4G$O`Lgg@=V0iCAT}AftWs}))G%Rsq`X8J|%`_f^1t;`Han;U0LQJ z%hbTv-H%+l^ymUk%fT$BG?gsU*1Q6*e%BZCu?r=!@-#GP&_?xeK#_A(UHg;KXd)9=TNqy)D=>Aza$Vs`D zIQ`r%|4eVQ{5neX1}I%9{^O>@^SS!F*Y?ihA>|y%sIouo@QYE9mj10ZerG3k+=iX9 zRMz}WE;e!k4ovSkSp@QCN);5-{`StyvCrJRBUkzg1O|w;r@NNrSu&P-e&BfCuNb5l z2Qs7f^~m3Hd>|aVoRBDWdZlz+y4GLbe1PJA>8yH2MyK1POedzITr^^xWd$ zQ(;u(uahp#Z=7;aSvrRbDPmK?g+k_&-hxxV-!t(9s53CCrcRKPS^MhoaFyIMayn+I zI#wOuD{PTf^k3Q-I;Oq;yKQsj>zRg;EnYP-aI8@Qz1AAPbHRd~xc$#5iAt2aoQt;} z=KY!u%Xb&z%+M_srJp`!8HH|h#??pF+$S*8|&y&F}ouKup zaf$mWE)CBPNfMkfSgRi`7O?TjZe#N2xViPJZSg+VzpCO4-(kZmXT5er2l#85w46|H9BCUr2FC70QAiz$f>+}++)fd4)@$0wnB8#~Mnflv z*;G_dT-?|^vrfV|$m($Eqdtq|4|~u1>~5NX>CkV83I2zE>-*xr^qbk~7t(epvSR3J z{Q1w%DJxkV^j5dpPTT? zU@o#j{C@Ak*Cjqr0xg8w(pTNX*0OwZvStjH4H_)0uK|txoA;hS(NtL zL;nYy(e^_5)C1Vxh6%kTrGhD=z;|;|ac0C{3;VoeSCmbc4joQT8PG)W5JWgC%grJF z@T%d=D#Bf%@i6}hsHOOyLzd4wS%`8J7Asw2zlLTGlBfZ1&|2;T$W5ZHZ=&FUJE zl!7y;em_8VQ-b_{vMSOTBVU;QIVPmi6yDjJOsr4%7)Dy%^lP4v_U~8QTe$*{+Lm=e z0HRhE$dY2Y$W^xtE}-rJOXnt6oxT1IMC_-M~;8V zr;N#WYNe2odV6n{efr-!UYd0N{a^+nOECuCzP?J7LJ)yc2yeP@cy!Ew?PQuzwI5>t z@QshUv!#WVL<-!BqhaEQo0il_|D`(0Kz?x`fb1NB!w%j-m00PexiG0?!`*cCE65;= zTrOhUH)rB)pvZ2zSZ2Jppaa>du?yb5UFNhaP7F-HFk-ZUQgc40Pgcn32NWk?zs)r@ ze(+0nn_9#rxX0WkmP$n@7W8UTHB-^rk8RcaC`V=tT89M-I-Un+zftBm zE^%cxBxz+6f)yNJ|BmIM%4%c=Dq6o&_;=lApvC72U{Z;c$jrCrJqqlB?l8GoPLPPN zl9i0=y97L)(Nb~BLwQspYZi19>^}2JlyEFYATJ1aDQ*M>3cHs^AnsOtHUf@a^^*j1 zx2)(eLC5Oa&1)^rRiVZL(+kWNsHQoVsxIlSXvR}wR78u2aub>n5F>HcXde~ypPb!- zeou1nmaxZQJ4Ey_7Z;b_N2``1e%S(XXCsJ!ufz^axwP?B1Ev32sH`OHrFk%$-p`JT zcxlE>-CIdX`RYSfDm{5oJyc1vcJGsgQb_X66WzP_sjGa`NgQ6j0ifKZONCGDRPw<$ zxK8WF7vyesA%EHvbcG9Tj?I!HAzZ|;Cuf*nSteX3Q7}B%V-XSKUsvinv<-8f%Q_~8 zqv)?oMlgc-kkQfcYc1nrK>0)V?Ac@f3};ME>@>xA7|9gLoMp*E2>p9`#M7(Lc134 zrIWcW=ANm5?ewXxHXeRVpCXZ!#wYzt6MiC~hnal7`?hN-7tCSu#9vhaj7(I(dXWIa zu3bxn%=%Y5MsMNuzuGZ+qj#I?4Hl)6Dnb+#Rty^W=u|&fV~r*CzNd5`rzT&tL1Fgj zIm))ut<;-IQfIib$+eYduOGOpAFl3=F913$iz}U)nuzD6OY=%Qy+Y%o(f4}2!(m^u zGld2kCiUbCk8h%4c!_XHep$s>AG6AEk}ZRUhSj>5o%@`fuyH1*NxaH5J&<)>TvZeBrS!g5y(kqM{4X( zE_=FBD?lj>E?Sw*UyKswpR|-x&=1v=8yx&F>_AP$yfJo+kEQYCJVmoNUb{x6%;%Xq z3OE-7{{qLJ0|vxR8R&;qi1GPUDWL|9r=t`j6sx@_&uZ6?tvFj`JwxCZe{9xP=^IzrOaHvAG?Kle+&I9a<$14ljR;nK8aNv6d{AYm1RrR#qr zPRtxp0Rm13haeZDu5Wnw`um6}(nt%9!G_^p3pQ-tyi@HyJzqc2l+V^uw9lY<-0hn) zxno1QaUgBfGjXGBpNb?lC-{+L3?oFn>!w-nrrP;%2gnPG&UNZ3(BSs$Rxs5v>So;5$Zpi?WcgvcJ zpwOY-voj3PqP#dBPdrv*f+qa=EhbYS>{&%{o3x5j4n)BZgo%uz`A>U+t&1tC4P=>B z_wvKQ@sRENdM(&P_7AUxQhry8G#V6fP&tFyn6-)SO#>{ziHtK39zNWCX&^h2C5L7h zhtuDKG>HjF?N<?OdZ4 z#{rM!Pzd-Zx0fm%`Th5Y+;zs$|DTi^$&o=!vpDk${tl92p|KRwHL73b?a}4O?O}NI z%|7p(nu_sJ_eEFM`UU=vc!#tT0H#Xuj&X?*dUS^wlZ)><q6@L!RyAY~SL2wyls{SAph8 z9gg_hD@LU(yR>T;uwY}Rm7Ov3duo>dFq#IjlEsesKkNAb1{E=Y-)5P08-nwqj?QM_;@M zM90l>7E8zprX&$)$&#?|&@y+%*ouv#%0Uu32#<<1EQTFakF#tKl(J z!$aK=;te>L9EPSp&L-*=+WDy11VFmUL`$H`(hd$=vg3y%abYZWB+xiU!i`S2yP%+e z(a$qIrJ(OYtUW7z=U|&?uQV0wDH>S7)Oz)IV96bpA&+B{sYT z!Xy?;8C)vpBGs~=&qI}lyVPy0f!I?(O_;K9zcHO)MS?a7+EvpH*BHi9kmZqsM*Y&%=*6k|E}9|JuZX23<>d1*bf$?VBS<Z64Yl9_I+ zPC6DJPG;jDL4nt2^MEI>9{sh_JvedzB2)c+6AqsQQF~!D8i@{y&MM?6k_03{^N*L@ zox)=yP>s3u2gu=-K0bUyNGWChV7Pkxb{y6(F70i|CzGCe^U|Ih$l*E$uK#?68wfF# z;iW~)Xj)!@mEURrV&h!Y4*=g?d^Zb%(s32gAgDFz!V0QJqqp*1kB9)d&R zWQs=RpZIt~0(j`a(6s-WMRI7Wz%e)uWfr(nMbSyh5@@}s<9uXr;tX}0q`fioU4>YO zf09iYwB-blpZsqP9ZtIa@G1=mnL2gqG?YC$^+QU5Lm@=sV|A7@StW#_^yEJqF|E>W zY~3upg5PhzwO?|pU!E_Na3#=XL^R;JnvX3YMU}?~og?=EBC zJm@39>LWdI2~e5v*!sN}aLd?aWS7i-i!qd;FVnQaxQfVvbBauLj{IX$AoD#^g`7!{ zj3y;I2gZuUXD~(FeZ%8t@X1X(37|eX8IzJ2gF&S}Mk#_}e-{}HD^P<8*hOSHQUHsU zPlPWQ5^rO4IpV2;=PzGe=a!L_S$Y84e%6Ea_Z}v_+2+HYRK6Wrn&bDfnGHqeJ2JvHu~fxJN;(@ZHrOGpT*De(Z?a>yt|nw9XjIA-s*>%zFUTS!Gd-PLA2gCiuMFH4dHdOslp6SB^oeHm$OP6R zhOk7#?e!-|dH8SPDXzz#vacF#0esBn;BVwD;|*4E)eV+eyU`S!$-m6PQBjgL3kV#lVZ!932OvEzH%q zqzg84Z)JmBvkN;PdRgcW6oEX>G2_D;zfT3QZo{M$Xq=`&cCvU4MU3Gc@JJzUAL&`; z{&+zF!eWWFznI-s;K}mK(kcBR;X}Md05u8SWGYH0rWhxPqhu9DdI|`%cmB>|DhWPl z{HL*dihurFG;72)#PC`gcpmi?9H|y8MV3=j*9|tdGSDw-ObTEeat}4f&jcKS){do& zcrG8eAM)KqaiQ@abUTKh!S_Mc*6?fJRu6+1X3~U-nRd@SR^C<|W}qG8tI04@Lq!SZ z)(z+8-bhL5h_z+0<7p6u6Jcu)nki!UqP^lbXgpAb>#wQAD}@H}p~dj=-3H)n*8(m53Zq->y&Mk7RX4x4ZjszRh31`X>?_%(Z| zy@>c*1J|_HNl$;^|i_gMsTI06ff5)7h;8N`=qUOt+cH#J1 zN%^z~wX^YLDk6nij^q?g2^8Vk0cun7fB;BAN6KnX97!nxAkQ%G_~g%Eeo}%%XB$Jr znZ{_Wq2i9|pTugKL$rSBbukw%cntUssD?Md@L~2tmb`@wc;Omm`_NkF}S@tS3Rn#YsZ2;(BHt;jIQi$PzE&-gk5)5y*vjZmsJwtnz99(2btWfo$o39LcY-u~B7Y zmw25biewL)txXJxfuL<@5JT-a2N}?Lm zNo0mEMk`;uamnpi2r*L&fM->(-sYTb-LU`(SVB?ICG6k?vjFlIqPL`qkq`3~2GG#i%1QSuoH-hT z8wkn=v7rLNqII;yF}Ny)W21k3$!{LrGuYBFeAF24NBSP$FRj5L!4t1bROYh$LqWN0 z%Hv`9ddpX>%C#O)t$Rn|QUg>EONnQ|9wTh5tfcw!c;d${oFeL~{d$R9*3LdOR--;) zXko@-aAV$0iyOrnF7CD2js0-4V#Ya4Cd~F0X01cik`gg|5df`Tq4fUcK&II%(n&AbN%*`XpbDw^NeCclbBxhl`O5pyxW$_dW>fnx{Iq zJI+Dwhm$RVNMROeoL^@vFs+J05L%Varjb@~PfZb;XaTy^;J)NV(l%BfwbtSh1JARVhiXz1wMJtcAT?rH?+Vk3J8^{f9gPWZ=YIpPj!(ZpoPB2lt7VZ) zOx)8aPY(8fAyPXG%@evIMpi%%>QK)1J5vFr8EA_eyhbRei~nl;GaY5+!9ei(~PcVRLX3hc&tzd51aKW8Y%pF-K65?x#FeWR?NSpjet%^Z9;13%s zBD@^Dsr-r0I+tNjm;q6RN;WLqg11Rgf<(^l6SdbvliDia>brovV-#d?RNBgOf?*SE zKh7~^QrhB+_*sm2AG%;I8iFE0Ub5jfz$eLTZY?Li%gyc4554I;eW(952LY7es(olj z^1MEqU`F%CD=*b2nhg`)Z|ROQbjo7zCiWqMZahL(l%&b(1E*6odxil)@|9iSF&u@H z_l!C2oCOOu&?*J0d?b(Zc=AP!(J#Mp>y|k=J9z2ug){}?&&um@0;s|u_5;1#OPz0j zZ%2x{`0#}hzd>rpUv)r7_6S5B#_}0-H+Ouvm0gQ+$`-LY_W_wBr?}|bhX`fB_JpxX zea1HVFEgsGzS0msT=s?+H{~1iA{ThN6Ik;eoNW zu?CLN#)4+8Z_bE2OQl60<#fD`Iz4HuyhdhC197ZbS%IM#CKs5;lCSOmF!JC}gsjqH zv2NX!Dm+qJguxYlz#T?7Bi2N@cyya^(2o((1salf3_2XseJBsfjwz)xHM^fPENV3xbHF%m%jGKn&PB8Ci@eY1w&lAevkF(h;$|3XF%F^0vv z7+VUWo&H2ZF~L7OA4b#h0*57se?7=u4!6jk4KrC`kRUw+=o-JJBbZAMkN*7j>Ot0E z!T(<;2Yo2RuBGN?GPA{F(E$}V!I2q9EVd6(Bs6Z~- z7?G~Lx^^UZ{PBrYvYi_fZXLBnAIii7F7FXL@-!M&fyR&P!9NA|u_%hjV~zPd|;N4qe7tM=S@0#eqtHv6G;VgOSKM*skmXB1dv>+L)8#@^DLB zEvm!?Rci)Y6B|0flP8CJv?kZ5ayX<-Ngg8n9nshdb&H#`Cg=&6hA#k*Vu07tiKZ(;^k85E%Iq32c{}n?tbeCyJQd z4~j?st0r!TuoH#BzM>aH5$>T3=2MKI`(&Bg;!(c3;oK1dc&VR=nx`L67?%CP* zdX?#>6;{FF?|SQcwvnG6IsbH%y#@-B-MzbT>MVzuJTWDh#sanyDe%yViroWV zKDi97fH`E(G2Zx6vM_VJ0yus<_|<@YGA060@`13wa{jgAeI0veZ@zEFxUo#)4Fd)U z*0)MF1pTJiaQvTIgwt9~E#lC6{5OP{4&+w$=FN_G`apbtKi@jcq0~YB>*0?O+`)Vh zwKcK3==5YqtHlDW&dC7r=>h=Y`e$gv3%~VT`~`27a}!i7Nd-r4#fwez{@|X6A$@~u zeeu@+3pcLe>6{2<7m*PDHSd)9E`gp573&PSzyk(*>B^L)cse!_e_({?u61W8NJWNh zmqWrcckQi)R^NZornOlax`D=TJzFLX6QPI}$PYb#d}wdL*u5R(;IrZ3SJbg}U=40U znghLIz#S(Z0k+MDW%$0+*S|MH-q#HEN5rpRjo*)AxQCzgS+eB;DJBM_f+g2$YipB0 z*Q8euN3%~7BaIn{I+u~J(VRJ}7MAJZ$^uBD%dt^@>W~-ErPt6W8_J4Ar$927yG=)c zwP5`Wj{94hir)2Vi@?&`cnJVw_$|p)mj&#zfnZGIJ<*b(+6mwBH0R zwlli|V}=R<=$N0A@Oy4=&7~4Ues)f#0gW+Qu8ZKQVE1*-#yf^^a7t7VXT^YCl%pdd zPF5DovWmaS26!(Z9Pgll-`F|Mz-v0;#_N9+Ch@*T0GO4X{#l3f857YOuv= zib6-fik6K4{4WOH8ZikORZXh5yeGhfqa;7_urp93oTU}58TLDDOUMDd)op)A=_>O} zS{2nXvz9Nqv3R8e1lIK+24>9`50m7fHp7B;_(83(9Bk#xqFHa~hf zUy^;@Y!0VD+zoE}MW0`Yb#j6f8|vbbXXAsNd2r`W91hJ1q;x;jE0+-sXpg)M8CVdX67kN24q_@wcI^K< z6Mg?R3rs@1kRs<<{1xGmIFk|K6%@=L5#cq3sxK2|X-_O# zkC4c79O+oZI(8n&0>7EP4zUqujueCU|9Ur>3L&jBEp1;G;{5@e@8A_1|w^Ch#Relhyidi-=`=I_PLNsAxp_EO>XfS8sY}IdJ zR>HH7KTI(D{q-h87P*W2=#>LLfNSP%FBuPbtA#iykT)^OV`LH@E(WO2Pzs172%0{P z*Q5&ru2Ky70n-$d0)g=x5@*SW!$^d91yX^oS&qOl;YC*OlQIO*?I(u9XIR^r)grj2 z!dop9!^*{PGgyX(O3;$gp%lG;S#zZ6Q)YF}LHYrx*Nj7e6cra6c8wQbj*}e>4{_F} z$3G-8#-gLa{~N}bOq9Z%4y^1e3KLi{fSqao^mj4n8?@_u5N_x|&GCuRY{IHwKKN<) z@;SY5Kdbzq)@_1S^es2yZ$d7oRYR*5W|Q}-#q|STNza-qWcXU>kd-d<$V7Nw31BRa!s&4sp;I~Ig-Z)4Ul%~B%I_AL&*6;%WRRCkQ z?4^#>r;K+DKemS)VQk=^+^CQNd0dIbxx-+wQ*9%c z5WRJpaH<8>{M@(d6qRfoAITbin318Cgr>tY{^M*G|5#w2xr0B0Uj+vT+lP6xT6RE3 z%i#YI$DnsXi%jMSk8FCpOF(+IgyW)I-xprPcx-glMD}!LF9`mQv)L+taB#@1ant9| zpG%;nn{-9w*c}KrD(3-c(PedtQE-xH8PrR!LP1PKbLkR$eAIo94x8<)E12kL*nt(2 z(01zCNVmQKM1o44SnFq?tU|cEo%&XQi|>Z1Y>eEfxn4ULesm0E6HeGXvLuf;~9+CsyxFBd1w*WE*_;`6h#I3=J0>uLgAnm52I5Qjg5WSY@0JYZ= zvvGljMx^A2QGT}_aI^?N1Kc;?3Jw;*z?gg-OjF|H{odCKY2_42GxsIfo^-&A&Xs7T zBfWyNw99Nf@qi+<=g*IL&t++w(_FLw?zxzmz7Ko><&DwK;&ziyq!Yv*6v%Wu5pdWY zL=2$1q`mw0Ef9Ih?`Lrr&#dymJ+jUQo`q`Zw2FQlyKwkDp~qk3}9|LKhc;A$9jVCA*_`9Iw)mWt%WXxj2V)RJ`m)L za4{iym4lEQR()`bTAg$OgntGTLGmu4HW+nktrz`b%!&s$2cmT?rt5oD828X(o3a!+ z^993x?7kVCz=g@^v-&nbAju>HJXx|x27jr^0CK!fIA?PnKg&iSZm-_!|0}jbQ964@~hWT_~gKiRes{tN_XbS&+SXD)3#q|_` zf5c3>7&H&xE%{@;0wD*Ul7^0`72|HK_db1x3t)a@+e_p^O}xQuC2&X$vHDw9WY2XGjIE=grw9nfLbXTvA%Gl_JX0 zV$90q90$SY{5_Mg0>ibSOA6keN-#mk&8rbY4AKVxCwvsh9~g3;sGX{%QbH~k(WP&i zH@SX9A_hQ#cZSqn){4C*TbtK_45C)|Y4 zeIof+b8@NcdNkbs?I$q%=r`fKg&Xv6soMnJ&PAK~!wPvCkh z4jecTG(dKZ4PM`7liXDH>p~k!EG#!$pD8Rik&qPp^U>371SOAM@ClKXq{Z*OnMr+j{Xsi1dn zFTa7@k1=uoFnoOaV$bQf*aiTRqidQBBM8&`?ahM8ynY54;r4^40zZ>gk|1Wfm0-}H z4OQy8e%khe70dW7Oyg^s1F815%w8QFJVO8srqJq;T+FoHiqrSt_^5?J^SVw_EynNaULMO>=1#=9}M?_5f|7-4D4BAhi*T81XgcUrEnMl2MjO&B1rG6Q@){~htcR`K ze-Z1~{chMtWLb2#7zv*t=ptwoXvW?a+oj&Jf8!)jRA#3Yx4~nZUR@NhtH3}(JhVM% zZ=ZWe?R9o;^{y}5gL#l47sbTX&#VDHf~l#}_AyRX)K|~91ERZTAodFnyqP?IB(@&n zk`D+52UJbJ^3R7?_R9x604gL=~@+lpuo?=&GVW(+A5LxS!DwcI{5 z3gZyYYk)_=V*dlH06HLNifgs&;cQyh9Oq49z?RFQ+DEEqpjo%^9Xd#LWaF&!BS}OV zQj8&=({Snpxveuq-tP>F4#-%Ovm5XjdG)30%sP%4=jT7JL zSnU>MWuEP@kPj+Im^u3UGN@QF)GB3z!Gol^meVw>g`>Yaeyna9ro+T&e#oK9GBC60 z<&l&ufJuz@gin6s_8mKv4^*df zdXStrd$g>4cxiC*VYL_DKQcR^_cr;!D-1dV{aqt1h!S3GQvxcJzr1J3uVdg5!~t_0 z^kO-T$Y{079C`(Dfbo)|gF^YBBce}^4)h->G={FYxkF2z*BGw4z_;Ti@}{aed6q_zK0C{VOO&rl&-8< zcmu7I0FL<#{#ceoqY>TLij2hN999d!T2S>3nbVwz%Af(#4N{Sy8NRc#6QXc_O<;7? z-8PedMI(p|6a-)vGtok9H#j^pEMTzJaU2a&X2`~)s1t%N;7#dh9%KMk%Mm7jjCcm* zZT~wMhfF@eD!dcM zGD^Pl%9SfhJwT}dCBynuemv0%7%#IXAA~~`BrC6QF=FoQ+x&e~3)Bk1Wz%c47x&BR zl4*GT_N`QTuU_W44vqtOiq_gru$w`5GgrheJMJT_6;G@YgN26u1iyGZ#b#(vP4@wS z!e-88t3gq3RvO5$K`#YaN9pnzv&#oy@MK7nRSwp@3CQKIAdAXPiTeR2PYKv}#74OY zG_q?O8klmXQDvyHi?A|h?C!pArL)f}DUUexkk9gTE+A2O4D3qK>40aQw}dm9dtmib^W?^k&^wR*c(ar z3xmz*Kq210i{z1vLN4OdaD=oK?(iq8zaio>7OcElS=FT{$&-AMz6HycoF@StB8oeqP8OJg)Ge zMT4v?hh(J;d>)Z>1&EX3HM(L2>_RykKWLQhq5T(*#DWw8raO^B%*$p5bIB*H;YgP` zlw&Rzj_4zcl%gX-pJLW4NylJd3RD4K37Bi$45t);K6XGfZUWfXXTiB&LsbeI+O}ao ztY98)p;dA*P?&HZtdgl#&UGr6Cpt|MK3=M#428z0y#ul;x&ZhSOz`KJ?mQ)#{K0mM zoIHwU9sCiuj0~Ip>ODt&{3X3Vt@N^vAdCNA^d4E)!y@(Sepw|@?t&ztludkPnA4Bc zoZO!>?_k5s2^7z$)&q5iQeZdU9lriJ$SRQLF_B#E*0wq}j+IGgBpA<;(cdq?evBBj z*}w~^Rf;vt)BD<@Mct~)z9XYT;=ug_ei5t2Q`mb^!}lACA#-pwQdz53 z42iWwSmI$g6!DzSc<+6XdBz<*7twtv=OT%mq9j6wkAz~u$XwpxFP!$X;T}E!{6}e` z?JnB|#s)FWm@VSM3)oA*Ll>-*6<5GMjsfwXsLo-sk_WZX6codb~# zeqw!S3gE#x3tc&yuID-(kEVi3fJ@b3T6qC3+A%-TC*ZE)^;e-7D-be2PYJP);j{}m zm;d}xB)0-RaL@qO^3PHIKK)hub-jc+5!gpcWLA_Z%W#%>C$54?Ba8pqD=*Jr z`>hNkBP-GbL-e;Khwld-eX^n_8z1e0Az8R_jZe9-&+>h)w)RUJfK7P$b7*lhc6}xe zV<38-8V{FY_bIvL*O2uXR?HKR{7NjL=p@9qxGrmBS2W3&=I{t9Q!@mnbJ7CW4~3(w zrf8CPH&lhreP~TCgE9m0ttEMw(P=fX$_zg}*!AbngozV9R{6qin>MtcNTlhDEnxDO zMwWaP)15#`hGn>usf095$!NbB;s=og3Bp%q4I?R-TpuMYL;$Y>VBN{I zAtQ3GyeAKy;D6)!=bwnlKw@S7>Eq?+;b2Wh_XN73xA!W#K;%aDMdT#Lss>o_%C#3? zsso1xDcsTvBg%)KpM7fVYs6K`Bw%D5E1~#bR|$Rs+lyfl8__@_g8*k9-w%f`!8Oq# ze@_z+5#KS!{L|eh(^*89i`p%7B|1(XQM|>v(?GpLwMPE1^dhMj`Z+YykR6QNyy`pLe}K3s z(84HZL#bQ>quqv06%dkxe@kbJz=2JmOYnpkMEBFg<=(ArQIyu zih6e*v@trANjjHj>=%?n-tB#`09^CMA$=Gu3M+bpln!zkVj@^)-x9S6mSM5xgmpJ-*w_|b=ArBZ3xa@ zASLX1H1+h42Ej+mrWApC>v~rE#`1U5TQm3UJA9zF;eKy}#!yd##=|`qHOxV+)cp|V zIVVrE;c3&+)9QjR8aA6a4GqgnPFe29N+{^_8UEni{k0|ROh`yWS8VU^+D4bbKG$on zcc#^5XJ@Bk;jBB*mZm#A|f!HfAxW#V(ySk!=67Z7Y zGo&pn7_dnWzNVnINMWcJ#JoQy%53_gWbDZ4>(`Th$gO_m(_f8^`Od9t;|Gkvz2>MT zzs%S{PWQbT-!t?T@7vpFbFp^kh3NoiJN;7Qq0`13Bg|IvG_SAGysr>p8isz6yWZ0B z;)Lr8iASd@DVdZ5(vz4e!I3^`Hj!uD^>v)>i}bLhIez90+X$pu0qvNTS59%Y9W&$J z+g&lvwzqnL%`*$B;)CKmR^)KQeh-fzK=vjz>E#)$5ppD{h`{w!{bYEX1^w7ey}x<3wbxq|)}c`l={uCNXwH4dOH>b z&)J$Y1#Qi*Ra|aA5vGi4^YeMAbj+Nbm;hU%NdPkoU=UxrcZG>b9^(6DW3=<4QXyl4?GwO_c1F?r1KMaRtGicepKOtkz@%&aZ5RGEfDpPUL;!w_z< zenxK6QDfB$_TYK?Zv^QTg=^}Xr4%w%a#51|X3v?k4BR-Anzv8Ei4q?_UXh!0<;uoK zP3O7&xp{epMp+37Jf=x&uI%PIUBh=VK7GA%gh`aes^yB~z|t9GhoRSG#R`d;Qxjia zf4$QvT0n{Gs@6p;@rc>S*i-jcU7lrNqN-9S;f``!4y(0rUFo!yC$FVM|h1 zU%w}IB4UO#FG~E)O-+9I$$NK3$HvC`yF(J4a(|g|RowM;%5=$v4X=v=UsQI>?^b65>$5($7 z_yNM;@UT257%ybamg87Xr*4!Ai+TO#%?EJruhtxLzP|1x7LHT(^<$6kzrG64!RU8* zO1gU?0Laog@L3q6i$E%Roha+z`n7@qJV5XY8vhJ?r2Q^7{C4R(a6%a%eD|My#767H z_51fF$r@^NE?q>yx6Yn1lf%)mxbYeWJ^Z9m=IH3u+L5z16}U)re4q9|jG3n>eFW>& zJLpG{l1G2oyLa!~M~CL6@JyHHui7~uC8iS*0+z+=-7jacl*W&NU~JUB>>f!RaVUMW zh=KrL(qlHx83_am3+c!!b~6m=@z-B{J=vn>#fK6k9E*fkMmH9?)rVb+lbW|f7>t)` zT<{0*?YiH2Drt%G z32wjtxpU{@0lo?>Np!ZATDV^k3oZUsbP0orDaT3t9`?)uHy0)p zLPzl8q`(@L$|YkSYAJ~K%&!!9d*jB93fy{4O`!yk6>{?A*Bez_ZrjRRmQx}0WJhZ_ zuSxNp=ymAO=yw1|vfs&yulB_?5dNe!Gr8ci3D1o(LrUXb)2p;!y=qmN=zz4aAfd8t znL+4^?a^d=hslap_>sEU>F}XL2QyQWV&MY#1TL_pCwU+*l_B(grV8h42kKdA$R2<< zh06n-xB%InVy^KAa%@zGyn}1O!UpH695L^mK^^2`2RuqtRFwJr7NheX4XjUvD#dK?Iw2T`F7gPBJK!VBF4WW2y#s@& zzTdx3gijK$@n3p59}PTZ7lnljC+)oV?q;ajbb76RfKp=zMS+XEyQIQeCVDuWAF28? zke=`0uj1krd5RNscO+SX67`+CWZ5!%5DSA6~osEr+%&%3eR)NvD zfIk(98znts&a0^$>G5Eb3A95F=sIpvA=r9%&@!j_8=_8agnOdmxDzH(h*7fVa~kOD zlLxcrk^?s|9()gaU?44xFRZVx{{h*Bi*Q)~seD^QfQa&+zkd&bdgEj`9)idc6f9gY z^2;x`R5Qj$4tmjs8)y4ctpoEkZ7Z1>A7KWZHyZh&$pQA8&(fxo**7Bq^gn_c> z%}yNPI`e>MDu)-TXVYG!%`4U4Trt&9L`gR^G-Nm2$Km^U@I$M=?HMQDvjBC0C_>|3 zpL^asYjCk~RVa|2MJW?id7Qm&_G8<=P*hHT7I&exwjH41GtZm7&rcPvFx;Ugbp|FU zTvW^8b1&q9-G^daxHzGHGIF!5@1Uz>&GbZ3XASAP9PiC+G-Mxr0*Y56nFL~tGQ32?@7*nVfirq%?gY|LSkYQ$ESk@HjD0O zKG||YFp_C^pnkt%Hqz^81f%W9c=+(d8Ic}KJ39`LwbSho`@y_o646)HRXAv~r#WqY zCnmnS7gSAg46sjsVJM7WbG_-)Cq+BUq2ShzSr4~}Zr!>y>@Fe_O6)0s%AdLq{HXf{ zS~`C_xKXOMX2t$5zF)50yiw6Jn|?XRCd$EBOg1#z9!G7fwZMN#2T1q0~-j}Ir& zr?6XVo$7@Ge7-lg{!j0t!cDQdH%4Dy-#dzff@ZmC(`YJ!6hz^rO_$dxry>!-k7@J= z=;q<`A>%MN3aO~V&;_kJ1{D$ytYcuOYHua z*47i~&w@wN5Pn=Oh%x0Q*=#>Ngx;N5Xc(XaodYFN>D~kG?oZK;iBFjD`bH5fw!ZEI ziOdyDQKJ_%HIuNa{v?GcQntD!2}O4vez?=$ETW_l;^)Dr+9h_P-*Ef(Y1ARZ3PtFH zkB^ThV)Om?@2YEd7*|ox1>K0r%9Tq&ISLvA+38GVq|^PE(nyf>#Z;|q`YTnfssC0j zebGX+=o{v*(ye5DYH8u(lE7#?e2}#CLs`MsYTrZ)B{hiSjM^}Sa<<#+pgdqd>A%t5 zekv6Nbk9|5aR<%ceaR95H|>xL_UguS&m1h2#`{N_CXJgp`Rgrgut)&I#xa8CKjHlo zD*cVO#D^Da0w!f$Pv3JfedW~xywb)muKk&bku05$mzO_flKASa=;_(%5SCQHxta@m zqs!RSu!CXw_l>%q&)9nU{{qQ@Eh)L<+#a+mqu)XJA)_j}%2eF|x=NZ{uq8Fp z)zXr}Gt!9OfC)(%sj*yE-Hu;ZPj5*{pp#R1tJ~XLUjtfSx^LdSdlnqd3lANt`hrg^ z2zzCtS^E02e_Jo7YZa|-S}skjZm(?h>0u_FIB4A6H&AS=)&vnAa?iFU7>fim-PG zJ9p0h6jn5LfXiGkWvS2Od=;$>{?g;62f+H!YCZ9q<4_CQiqA&EvD2*DJbLtK46&N` z*DFr|7Q)3m^W!XMXOWv+!6GH&`|m$|@JC`n6ckCVK$C(ezU0L}zD%tBJ{-(#n}(;0 z<`jlc>QAX(IbM{8WYmlFXyoK{t$x2E=j#f9S^mGbu+)GXp`=IeZd{X43%`9-zZ!=D zV`Do^G*iIL;ZMbqS-9_^DC&ecv|;h>&CT22%s_s}T;RXa!h(%!v+q1kpl=xmPaA)C ztiVg3N(RooWf_cAnFxrH?Yqu1wCl`?Ybux~Jv*C@%7Y&?W93}q#m7vO9;2|Kq2A?Q zi|U1Uy!fNy*8_O_0s8t!hK7p4>_v$NT8;6L-M`trw-Z0L_>?Jim)qwrSn#naKcM)< z3ymyubrm7awa72-HO z5~=U4jZ7JnHdR_0tZE{VC`FhW=_)A+mFe|P*|KGeamgvJ7YLVPxP7q{c_GLfqs|?i zi5tF+tVdsWdB0KCL4;6I;|u*4);ZiQ$hT9A4QOY>{GBr0f&p)x*$1PIIr$j+1=LS3 z4FERBbbR9CJu@&)O3oTge!Ua1GSzPB{d4eHbyA5iC6W@-_O%(8E?s(Y^4fVA2QuCB z%9Shq?^8aKxJv*JOzp?o!K~I8hiZ`MM$@of>n3MxtmkO;3A{=n06`xVE5)LQpNeLzUPRJ=sj_m zT(IPVf9ixn{Or3k<lvC z`A{WZ*)1rRZ4-)J6bI`uc5i2A9tbR>AQuNVl6X{+Qjxoxn?`ijQset;H*5$3nXjsC zU^6GIDYZzp0D}U5wYhjt1g<{P)NF@mcj4&ggNJRPd1)-Yi>g?BGj(WqO~3BfU>@&Jr=1q9&9BQ%9%zKVA!e^3Ig@T(oGB z8omuIJ}23Opj3!AzI2Qa{W%ro@Z|BwVgp0aBzu@f1laMk;oQ_V(2_DdI%3t`VLl4C zn);jaQ^k9>QejKHo;+4lLcC`}gROF9>^{e@d;RF zd^fJj-rbCr+8a>Go131M7)JNxcxGL=FcI&ZK{0t5hokmmx$)9tBRD}{4eCQ%g@m#e>p z!5oGAdtksG>j?rtj2DNXuH#2bsQz|Z9WyHsB#*=TC-DK8nvS7b`QsOwCxYT!TwGe# zzr=4D3j6Q5Ojb{t?8-2gacYs9FzZTV)lgnnw>z~AZ%ryh*ZiYK)h^UYOu$)k)>hn^ zsga(JE#KR8XNl@b^3w7kai0kfKZY%|=m0Xcg1Wjm@>z2avNQ8lpT{7~MSl_I|D>XS zUT5Cc0jyc-`ey}S4c?*`Cza&@!`{gdVxrMJd5LvLebxe}5D9z<+PQNa5JJu50jWxFws_Aav+0Y$_mg)= z-)=PTSH)z^{5w(tTKB18Aa$^s_2AK?DF_Y2Z(8^A%kP^WTa=K8s@!%_4lNRCQfu`0 zqEKr3PQf^8lm*xK&!I-f4*Rghr683@1E9gr&kvXh)%j_+RQizmR|F?Xk}|yU8)wy- z8#{mkqia-KS}KB;8Thu4l(66>K%B{J4`nJ#)VtgXY+NImSgPj#+)&IScv|!rU=b#>Fq2KPtFySpvs511LPn0EKI=8;ACAtdAc%W|&|hWQXPgK3E9$`O$jIq<&}S+q(Wo6A^o}oMF2%&8;zBa>oF|B{?x5}l!SnX+ z_OZwZFZNz|-W+M*tDo~{sDpBq?iaJFB3WHeKWr!-w+%kUt91t#Znd=Q|an?=6jBpkjy?7hmJfJ$S%?He;99Wgni1(Uh}eyAE}? z=-TeX{rt?uizg8|!I2ZhHzAZv$}N>6kk5~0Wo4m_sj%3VX&XP%<-m&3fYetOLZ8us zQytgU)h%3cDp2E=<=B{&o}&NE8B-}CvK{4LX!K|!NNUy3hlL#l_c%B*qR(vNg!w~5 zON^_k9p+9#0|!bopZzQY136rGTLiUt{5~jgvn!`^zDj_66@;m^5Ke0qGeVnNTW2~| zzj^cP*+)zP&tX)(8hcLeY#4#rJ0rn|uepBA^2UThj86C$Cun2zVa#z}9BB$zlyJxK zBpO0Mx`yx?lh@q%dGO?WeDhidzlO_WR=TsdUK_#2JvUxl#01N}4DW&a)EgJs#Tsim zs7aA@51i`gdP!N?5Ru86dB7HiaY>tgF7Q)~a9b;Ez!wcDe*Pfb=QT{Jo@|f$32XPj zEx2D&sT?r|&XIRFuG@tEINx6&Qtz;4ZY{bWi|1u!3v@y;ZqTmN^YG!|a70!aW=v!; zQP@x|h|cYTqEHy5FJWZif$7UMjO%lK4ErU*1$7=}XTNW1`VY7~ zEm^?aeC)Eyy!kkiVUo0G?dNo>*@V#BE!C(({YsMxR+gw~fUBoKJ90|m!-t!I4UVd8 zv^;wBC@^Q^#Zk@`btoaao}Gqw-u{E89d0#kG191@z=)|Sr@1{aG;~#*ZfvXF4{RsA ze`*0VEPwmf$Nj%yHX&?=A_ot0JQFBn(xat7%8;K)p zv=msd7!xH6XrzhuI4Kcl#4a}Q{-7`xsCjie(TA9r@EKzhjD082S~LJX>y7V!I^QoQ zd#H?q-_F0+ibvw1l70c)`Nz}*nTGf!7L7EY7N(^bKC#UKkb=gDV};}O!1q_88@mo zS3?T8F9&`CY>2!r*7#4QHb^5qQZ;ew+O+{dIP1l*f%$As0&RTW^=o;bkxa0AbzD!<-iqv?HCGRZKe|>R~xqoU*I~) zQv!(3Tficb$?;D7;#AC5MzsY6d|*f{16xX-j22*}^KRaJ;GuW{SCzXTJa`--qxDSK zwzs2^!2Z$?u&H{27+zA0(v?XY^24BYgU8FphCF}CElc!HFI*V$evuc~ck~8gXhy#9_DB>E%dum} z-p<-eFJwzaWB}IUg^r~ck2827B@g(TG{H!q08HfVnmYto)cB;@3DEDvGdg)jp`tuI zD(ZxFmV1=-b0Ab&5CMjKS-|aAxMy`5=QM~4gt7AY(#z`Kre1iij!=Bm^JX(un0^aMx6#NiogYHYwH zPu+3PhYMDO#e1N>PB$>4&rQftLuP?T&*X{wUw!M*!p@kW`>=VJ6N=sba4tN57%8<{ z4u#NY-MaBm9-U4_<3L2!!kdMaND=q__FGt-A*zK1?BCQtJNP(=Q}r0w;#Yy^kzI+r zA%S^*eEQ-=i+~5dK@2rFPji|qEC{?b@(DVtEZnP1dt3k~c4L4#TBDI>*CUk(>^Z*Z zC{|xav*hJj$dM21v7!rDi~Ggo~vb$njxwFDISD;|@GELDqV!>qvpLG^P6Ejg*{4G(qYk zSrILkrtr*^>(@sEaM6su(8a}n8J4zZ&YWR6Iy&k&F4IV_e{@I!O;WV_X%VCuz6p%o;@eWQFpzN8WTM*hQpba5;`Yr@L6B))i~9H&!ry+baRLOdlo*k+<9_x zXHikk-1}R9JI!2bCL*bRdHB}tN253VYA6_bKJD6z8v*~U=8L)C-qqJJbMpJP?{C}N zV+T96S9*s-Svto3Ntu}5_Owff|M|I1Pt`Z#cwPG3brY%<->Nc9*&|ctv43q-wWH>V zO9@mgO>}E&=J&N+7QS+2{D)HgKeS}_XE`4eA-G+SdedG+ZCR&%1Q~WIF{_+e^>n% z>Hr+r#=>5ZJx84i?l>YFA#CwE+MNRYz6~*gwrw*uvukrOqu631%fL{lem*uf1bzSGqM~JmFPxO-*5_(zyZ`Z5 z;oMz`1VisgJ8~y<5mV*m4e^?6>(45|;vJhe&i2J9$XisGfuw*{$*EMxD~;kXhv+yi zF?OI6qC1Yq-~eLH)dS+tf1-lKS2&#_0|L)Ohk9!4$BEZm#9AFQBER~mQKKZW;g)6* zw%HD`n$hM23tQ~x0Pz1Z3r1^HTvj$Q9X2{kN*ru~&K4kg1?;D@PA&P(1rYcbpEAZv zinmup7Szo?sK1zu`n-8ROq0&X%^UX`?0s|sN=%ZSJq`^{Fd~xX8oL9ZUl^H~m|X46 zI@C2CS~mWBR(sqx<28aq65py-k~8H|u8Ek(u$cbNhZNm}h{pnfKnn_5s@)GMITpSf zfChIJ4h<=Wy`}XFoHElq@o8e(eDYBw(Z~2|<$DYfl%7B}ne{8oL@`p!l>-LJA!|v> zh!w1eOM~oZ&u)Fw^E|ZheUQN=4#kT2n=gvvin6E zn7{$)wKvNac2~Xmr!)7@;nKI{OUvPCg;q7_541@l;qzNrx!GL>6w7yN%LvD<*sIiv zuUjxRE&gaz{_AnN8SlKrAD+wN`@C0r#{Q+=Up@s@pY|G{Fm@O!dfS z&jxF5*PEN-3+GAiH0zy{JRrxp8$&o)X{ot|)by!S`3TQZS8Sn=Wr z7PslqY&EqrA0HTNC~bMJ#=?UBC=J-KjwLM9 z<-h@ut?;DJx29gTnX~1}9$d%h?CpJE4-7nDF-|Gf=xYK-z2qPm510YM;L=Yisfn`a&mbOP7h>gFcIc&Xc6f9&0r3|nygBIVnvX$NTv}QgJ2pS7TT8(Tm)8Wl zRW@2eJj@z+Dl@Zo_4Y0U=RBwv5UJLYJ~b?pwz&hhHo}sWSaly-kZcHxghSO>|F@Ub zN1=mb!m|Ma7T317>Oe@W9@S&aMuUPId*jQFjyutW)iO!2Hyy$5nHmFw3&umo0FOa( z2B@vBwZ{_L)@7i<9r51O(=oZ489*9Hjk4Sf?*OJX7-SbGS(0@9oWEJ5zf-z9dxNYj>J*soAB+-QW0~wok{W zWVVW4MXLUn^0k|9Xv)suGhDPtio?-e^1`VFUgcxvjNWyIkA* zd@r2iaF~?hfGLF|Z76-OA2=G1r^~(ZSM_&VzbH|8JK?~B(-I3W=XA9k;`Q_3PF|F# zwdq3t%F9c|CjQPds9Bd$509C{Nrj$`e+`>dxf;?(esQ z{*xik)K}D`IwJRAW8+k`8*v@VKyX+FV?F=`mxsp17IQHJ*8-e54QwinB_-bT7Iga6 z#3Fz@&53iu74ohC+0aMGd1bqJ!{Jp9RPWIkP_WUf`ik9&g>r=dMmxJi>vTIA{XzZFH!qt(}Z30g9lE4=8U(&*3uzQ$MESao9XE+eWnf zOz^^oY63i=L-6}HJ+ekoKw1OLU>TGv&b_wJ8q*5a0_wqKk?MR1>*hNCM4votIWd}< zn3^v7uQhe?uFk*Y&9+lL3s1Cl^e0R~4t@GdBKOjX>#I&3-OYaE^r(BiWVbdS%TW1( zZKa%qx~BfAkG<^_+1^7#9h1tvv$mdnp7b{Di>vN%pUHRhyT!_v6o}cW;is|FllK_* zFsG&UcrQJ9>0hjg$3{DCFhQj3Y26P0kqoWLl;7Ek=bowUcp+8y=uo8;IOUJgN*TW> zxesQhI+*jwO%+nli;MG+W1GHwnPG?J`y|vz^u+O*neU{qtX^zl!sVhpwEfc3qT~ee zGGK;l1kBILi5nbT0QSJ?cATZD z1A+h@k+{0b0RCW!^ZRus)B8Y&k;q|V zL7dV1Np@?2x~xa%vgv3wO5x(^=T0y{@Umq^vBSf|8*wPa`y67_m?3H*8LQ59JPtUO zNtFY1d))Ku^{ah40bDOw-dLCa0_z2%GLioL=1B>I@i2vU&F-7gY`|ty2z+kwxl?yu zUUQwltBhrfwFDoz{nLI~7XsmmTusqR`Axro|y0v=r`d_hIS-So7(j9Ay+Va54FvYg5;<1-=~^ zSF<($mg&dFgpWgN_SWg<4{}@XE#KJvPBD6K+SPXk)vhPrbC*l0+G!3+&@Zk|TCu0g zKkolA_1*DQ_hJ91gfcTivKvHJ_D+;SR#x`P-ZHXf?<7&S$_OEw>=lY+JNC{xBzv6S z_3eJ1=k@FLy8pQEyK)@g^Z8uYdwM(bYZfsKT495NW@L7SGf)3ZOus+2uk6`|2}MP~0m5V+6k*C}y`AKmtJ~g861UK)XFPL2&bkAPv!jhY!h+ z?Hi%pGbTpwoyy8qfR4tD>T&)ci`iYDPT*@mwM5wpv*3Y*jN8-0eN+-A7t{wXI>jenS|H(B-0nvq96xSspJgg@F<44>|hbb!Li)rh<%*6*;B+PU{ zkdKl)AqdX*|K$dm`B*X$1N4t07y>A7UO}E75E=jY91PDNtavY;Ou;LJSq>LOZSdH? zK-1CFBg_C&aSD=wweHu5Y~m8E%Af+v0KDpAIzXTReKQ9C3=31dOoRs)#vCLVHNq2< zIh+nc@y3^;uC9t| z#rOB0hbID?MEt3ENtjt!n1Vf##T+@) zWKC!UtSJGSeI`opKZNUxnQDjyvsh%x+x03SVBpUoN+e!4Fi*n+mDpAlepmwV3!;D+ zL0k{KS3n4jFytoigS2mZ-{}juOE7G|gb}!)?luM@3OSgW9wUxdGz1& zBq2zqstoTuorI_QCR*c0yfJWp@O40SsH&myl(*ibjQ}u)5s@n_SxxVO-}uKefj|O4 zoQrTq!V-QL0^yi8=H>z+m+DChP$2{KpnLjh$=_Z90~cJ?VVB8>6#*L_DYYQ|Qd;`M zKAG1P4=|3ifMVV?m%R(%I06&2nwXgU(*wf0$*=z39?t0LgRap6JXl$L>)gEa%&T7j z_cw!`=*MC{>W;QH4bt^idTpvnroW1$$DmXtw!xmXzlBH*;- zh27pa=O6XG;}547c+jeRYRMB4er#^L+?Nevot=}(RQYE6<)p0awwbwR=2yK7foIMW zJ!^V>GaxK{tF(!)m^48?5{!v)>4u>Z5$4M~c>DXp#@}fi6yK7cb9cwjuhdW<@Q1wS zxMbbXF&zQ3+ZVjOQ+TXjQ=9h?1;jtbD1e$k=&C?D&GqZixu4XeGha4OO?G4Ew@^Qyn*R~ywGpm`GN8u+lq}hl6aa(DQ@heIq@Gqdc-O0k_|`_ zP-iO5u)D7g%Qa0kU_V#p46I}hE^kg--?Warl3DP$WweE?NmcSZ{nGIJJ$Z#vyz6_` zDhI_3T{@S_tm#w6>8CfAUd)u7ZWhn1s_GB;H|r+kayYy&u<&#CuuIuK?S=R%!{GoYW8%Ne{X^`7qOo=t}JK;vl38D@BS$1@R}wn-EGwK6KLo4xn=a9AlXl!K{!vr5 zb4gzOt7<@q{B`wc+NuLl<1q@;b3V`3S9rVTPu!yl_-o;t{-kP{88j}9hq)*el|*}v zRcN(A&TdCf*KpXGUtLgU;zK}W&Q#5q%#KJ)8Hz%th|MDxAOgEH>+ z@3cb;KAuO_m;%Wh1Bb?Hm*8(bpYJ`dCI<-+jo1*r%Tmbh{yxf5Y_ZlQhxXDXdA0-{ z_}m+{o>x|UM;Z3=Vx_%9X-NsLMQ?>KzNV&N%Y=S_am#qz%Zqpp4oCW0W52RwJUvD1 zCqBu#mJ_fiQ1$dU?;nUA$?42yao>^;Q@{02)w8^FdzfLU-@eSr>FMDqYBa!x zJhA7e>^B^CJaM4rv@KPvu;J>w^Ubq3`aI8_nc8D^%AZVvt&i#I`?L4%5j#4+y2{c_ z5gu+lahPt+ZAs;3JHPYm1cA)F zhd6R_m&GOM$rv^QE>IkM{S~>vlJ$AV7qr;$Ywdm^-?NGEL%pVT`Pb%J@AK~CQY*81 zgjO*-%QMXjidO4;=MX$4UHPfrPZW3to|yJx8pCrjn=3wlBtF>nF-z6a3A9C97FdNe ztCF}&S;#7p1l1a1lwL`A_AFGpT7FWMe*dev@5dR*mhp?^>o)!IjuqL~w$k=Gmsh%z zGYV`1N<(qPkM%E#h!l6QTB#Frvei5ejyM+0rPVe1i9^78S^QXU`pRNVW8b9U^CH#W zGDj)JKoxIqaac(MK&orH20nwZs3?Bjups0c!V`i`6u@S{%8uH#VV^>3t6-lJ0^PD6 zlz1068aO-O2EdgGdx_dP0vpx=j(FoSFd$&H@_au?K+C>`qc24F?9mD9PL&39EnErH1Y z8=IOA2{{nEOhd!LU0NWnDVYJWM)$4mnBx+Fs%mC#&a{#axIRNb!w4cG>>7a29R#hxB~r}6)mtIo~~|1DRjgdR8EfS(^c|#xD7yl4m_~6 z8ydXHh{T)>pstJ^okO4iLCD}b($U*X3a=?FJbcFWpWj4QqRdYBhVMalDI1$5Pwfvl z$EEh0t{tyv@iE6gBqQ4*`?I}z2Gwx7bL)6f$VGI*O-6l31v!vyC_3)JsToywZ&4);zW@Hd$~9K02`l>vr&Uc{_&tQe&z8@;OD?B6`nI43 z(+5#eogmp2L0R3k_*ZKF`n6vi|5|b}i07_uZqd;{1+iDjex+Y>IGIGQll!s?$?3Kf z_snb?`SQ*i8<*-@F{8GK2gmyzk9mDOi#5=L);s$r81Csn(qF0?gSKv7vO)N#$4jyW zPxxwd4;B}PX4fkpmpB*-zLwt})?zuc*h|`0i)l9<<6m`N4_`DVp7a&F#ORw8kbNV1 zbUpqC?sTkQR7cjmnTav`qdm~LBH;BKn{(H$d56D#eHYc}Ue8C8?o&ahq48rs(eGi? zXZCYBtf3uP7v7-EOTT8TOQ#EIvqo}K6n zy%C&gFVQp07MGnVlwU48BsmI+*_N4ACPOvilqOh4viw>*0 zle}=+(>0OHY31{S%GaHRs!s62;638~!&HTD^(yD;9{#wSr}&}-HpLYSzUj-$V}%{; z)mqUMeswWk)Dao|Nex0DoiAi92ALd?<}^!<_cv1o-Kt|%*wC2U#Og`+B&g9S?*8si zqq$n8>}54IPlF9zKUtgB@d*99=J__B(V!zQR;Q?8>woBhk|}tr_V;T$zB6o}el($b zj-idfzzdekerYlkQlfo)cuiP1d3YrMrhOwL<5Ab_TB>r<_Fo;klnF8Hn%iG*ht#f?yatPmzrN8Ke)F&IM72Q8?fn%U}iGJM^3 z;$3)`{I+S3Tzl+Zi%V}CFu6Bc`bxHSA}(KE@HkpmrxSY<`i3fXbltg{l0#E=cEWv* z*IaR`5NGJ;&lg~;JPRI^H*It2av#(#%+8+O9J8U=>f$IduUuLfWWN2!)Ku)IRh`Dv zT)K}NWTZI`z%uE}UR8Ip=jat4UMo6oZok117Pjn}K0pr^zx@demr-w;S8kC*$%q_# z0uT^)Y$wG@z@YmIRP{yw{o^`l=H zcv~YX;Cs5wH}YjcP@R2yZZYpM&Q^z1`&C{ikID)M9-7Uq4x!B)DgAqHZqaR$pfmEA z4rjeT*XyS={roZu%N3IZE7-NN&V2Y781G=l`1U5L(|)g(%F~(oipa#P=8~79af=j# z6emw8SXXWkt4Y>|a~xqZOS+Cf;GyXD?RT%7&s#_#F*(e+|s|pT4KyrO)HC zZVYcQ=ve;{`cs>R^3?->X(Jy24Rfo=g^iRKEE2Yo$aN&epD zi)dBt`lobZTLlc0KXPkUnuT^5Y(6T8v1hb4EITnVF{7~mH!wfn&Q{9~I3S)407fou zgUT5$%74zR8V)|b$2@yPsCj1*Xu0p4It2Y2uwJZ|AeBwI$0r8!h~x54W;1hhf?43u zJs?{7d%GfVPg&X7?Y2{Npq-cJlEu*g4g=}fE#1C(`z;Xyal#7N`Y`I-3jo?cTk+&# zGLP|D;E(>(o<#s)3|b;Wlh_DWDK&NVprZOi0s(==$RqHX%s@!3thTl`qXozS2c&@S zg&QmKMN(uWAwXc?R`X1<)W^BHHC)na3rvNp*J7sLuZ^~!ZXMqU?OR9_Q{VEo;Q?s`+7 z>kd5y1ntGcT0&iF*U)ehrHYkK&tHR|w*Ihxe*R&Bxy5R1OV{&qSAkCVw%ugLdrY#kAoyj1mU>(d_lng8(mm|aDN_I(%B;U|exDj2o? z%!~E(O-Px8bAjhtGdf}9aIQb8go_{+>y7u}%SQBNxOiI=v7YTFc5j?$mz{-tS@?f%rWL2G(sBuR|gsr!*jl<6nPfqJz;&XM$f({FZG}*zHL#3m^ zM2@W;u|&shS%&)UNtXPIiN}uw*uDSM&9QZU+9+1uZ8JHx+h+cHI!XYGflPj%aR;~B z%eQ}vTaI2Y4b8+4$C_>%QW?+lqX*veCwLW7>iRj;{@CH9au{AEs!IKPQAA6Avl%~< zS6s01(|X!I+Vz@`Gfs$7J>y>;oQ)5Z(~}1*1=tDkR<7_h&5dEw1tSbbO{_3LMCF6G z8%2lYgVoKQUsivGE7t`hZgXtG-<0Y2YVRR~^+!B0-?$eAF2Sk0HmcT6z8&=AqHklV zu@&b6u70iA5jJdaA)|JiEU)o1^xi1Po3SbETLzq0dJUa8G2_F|{%&g}pYrl2u>Be;y6Zjb&RAJJMQ#Cu=huf8tN&U+L9FZXvRr2>L+_cw zqP_%)t(m8=1BG2*(g`D_4QM&tT@@cFH&(j|16Z%n<39rJ2tgJJn;L6(2>of zgO@iC-OjFllz6hgIHB;ozShzC_Y+_hP`L~7C&n{B*NI#)2ZyWW2=vV*bc%PR9W zDX9B$R4#m&XcMMSeg<=KE0RU;l${WNvif{o#dbXZo?4_M)`VB{*phZpy#HWUP9#yG zcp)@LR9tl=t>RwnT4Wm_C86JLN>B8pdxpYwQh?xkphL_oE?$>i1hnqrl`GvZ_z{3n zMiKB{fM(5XLB9iD31h_bZT{2?0OwYq5&A%c9|cR+9C-ABo?uxC%gI43HyLW>(V1Yn zY3?tb>&$?YdI88{VNJ^h(yXR7un+;8-vmH^R0$M-f;RgqP_u}*foGx;WDw!6FZBH+ zr=|{tn}kXjk~F2CJh==69$Y4ppxiyTp1GL@w*(?{*eGYYDnI}-1pl!Kk}2${AYQ;9 z2=eo=ITCXrrYylFzGrLT8Tp5c)6vl(df@$|y%UojS$=-#l0QEN6e7^-;BL$cSb{GL zTW#p*u?SqW;wOLm-K28v4kUnJL^|J2m-IeM91{q+TY;ppWCI(+Kl9&V#m*EMw!v6V z41QQ3LSUCfuA4(~q#dz9yX3_%8+?*p0{YIXEryE%?hV#(6e!!AUa_#;Mtpjp83Y3O zJzR1ojNsOR%tJ~fV*^+avv3&P;Z-4Iekq#4YmFC)O8~4fqyn~Ph{0o$kf6s4TqvKL z0R9uWQ!w*f2EAU-<$s7HF_+ZA!`l_MR=TwcH{y*(zdX2hEvN~NzR=bdjqG+@`s~th zXMleCv8IFA&EL!nUbft?*R9`1Gh#w7aj=kG{(T`&{Y9;Neu<41RYU~s8C0$NR3Ole zJ*oNMD}(BDn`E%9&|wq1T&C%pFRf~)k+QZhAaAxc9Iw! z{Q?`e7K=n3U5>Jp?9-<%_`2k;IsZPo<8H$rpbR0z14k z>;oMr*NW;$F;o-_d1SsX<*!#gFFjl9HX);-0V(5iX2_auySx8pU?8{H z{Kpx1wRgD^O<*lXmh1ZZp4G2Izv%z&)%_2Q&;4PHM%|Hl%I8++LsI1Xg%1C8mF0H4&K@zD(-|5* zFPiZay^I2_IVJfse?C`!exIwcT6{FsJKNzk+}Jc)v8u|iE?|1Z{i;uI>g?OQ5lKNa zGz_bx_!hXI2v?ovWErgMPZ-n$tT%sOVcmjW&4!aHcU!EVK_eXF?G@@=(gSNMK^?e_ zgWVTCWmC;tyq;5XKJ!dR;u)=k+vy4sWsGxFG-)aO-V1a>RgKgY(P)Mq1`^;CW2V^{ zg@rp+wLZ(5tE~z|NM_niJ%^u&Wy3_JSPlF1ik;+?;ut^_^78~x#(TYnE-`j}&K*p5jdY@60f3-`fap$Y^ zK|xiSfCqPW;=+Mf?<92yi(K0a+?EpWgD|D^Ro3+JR`&)kI^bM9Ha2HAw5y%ZWo3C! z&&>D(8DT&5`L4YDnT@HU^*=jaCp#4Tn=KVKbpZOl;f`%h5n)C?jy^twz%YQA!N-!{ zvkY$S?|h-tYXyy;G%E6&OnGQvdJ%e&A{YIqxhvr+4**!>3}Pg_Ho*IW zi+cH5-M=l;eQs?u+R?Fi<7?CIy8C9nVJZ$R2J6wRX*9vGS;H>#GR+{s4t=|C;KLc9 zA0LNJ9WRPQi?PQ<_+4g!Q>8Qh$H$(Q<%Nvm4i2pQ`y=)}4t9kEgwz}DEunTr=?x8h zX`bu^bL|HZJ_A|{_O(eL(vKe>?f1pLDv&fd*(rLwwKHtD8hUsrYC8Cd3}_cdNlB%j zmVa|US+~PcB<%kAB8UJ}Z6RIEv+GZ#Ah6hEI|`hh-SJoQ96rK~X(8G0)K=kIWA0^U z8iA|ldeTU8xwU59e#yXR@~TiF5&r3s39BweSGC#;PKVSN_Xpja#o(9$u&3D~=?YwW zUE3?j_`K4si|^r(_6-LmzOD3PbeDnHTxd8SK6#>H*Ao<`FPS}mIeN9Ryk*JzFZb}d z-si0YMo~#r=KwdiR2PNgXLdKwyNut)mWzCF|FU^u?8aXaLqM{QeErUZxM3`c zn6zYn5C;2IvC`>ZQV}T2Sd->-E=tJy)aY&0JKEp3ua}P|KCV6xZfa`#bAhqi6ytZQ zH|a%sbjn2cIx38?F-auTaohFo<7k3W>WD4ioltj*_@<`;B{bqO+Q@dIsvOmxh_aKC z;PfC3QYLBy+Gdl$IarZ7SSRnE-NUPZLS5yqBNB~1c!4rwq}>R$dim9_9# z5_|dJ8j#3=slV%kBmF2wJHYQW4RJalAr%LB;P}-3Viq3uWJdznO8D!fZ-L1O{M7%v ztpKyXYkL&APEJOA4Y*#9{mo1!B(1$RmKRJA9-f|Ahy(^Qs^+SBSXL52TpjoB9eWt? zyG-CX0OJWbPk^unZ9Pu;nD?0ic+p_jV4>B6TtFazuOq)REM8G65y0(T^j+YGAt@E! zj3i)I`;6cd{W30&ISeruIEy4YU;P5lYh{VUl;rZW=ChYT_lnp)y?uP{bwz#q0)tNJ z-BNJydl!+Dsis5p#67S$F$n@i0W5tMn7Tj^T`Jt$1Cm)tDwYN6xK0hM*(>hW*4D)~ z!(7Wd%>rUtg*w47NFi>YI2FjZ03%NXVsidE@&DW5R|&E}0fxxhJvN!32gTkE4ZPC0bFcU;0)mt+uQ&D{65l|#dew2h+H5mb9ub_?$JBQ0c?$yO$% z`s;PH{$!;7gv7vOhONT3nmDg))s?ENI~f^I*b)lMYsV+(0VPu?HV%AFTsDRE<$`o_ zet5Q$Q3%ow9WYln$6cChzyi@ggxde%ptgmcs(>6u!S6q4FocZWFGG5zdig6tiCtP) z({Hu?`WQ!bWK-@*C;AtKu6U~RmEO`P)$g$13f|gVXK^@(-yWr7AGU@vX3vF6DMECX zoJ%QHQG+k}&6~>Z?=TA7^esP<6WVuEL|QgGSry1GEtr=0>8H}_R!-ix%KOs4e)f*F zHQ8TJoaPpZ%|WeL(=N&x9Sf<0PJU?l5!(8t(19wer)t2S;I57Bc!Z)AQmwRjXX&hB3kUi3_gs z0_>a>SaIA(<<@g9hFf!1NeAkbA8zMmU}lF|Xa06v7Cv!W-MM&hdEvq7-y2l*xBEUV zq3PjYd1z2mGYlT5epF0MzZ4w3fO!}>jPI{X9Md7Zit)Tmg6+MeH<3<^Pc4#m+UpUO z%B{y}ak@R{1mlG8&fYWB9U!Q+NXI+YayrKN_c)He82`L74RhPl=~tBW`>(i_+`c}y zNwAnpDN|1Ps0?e%ruwP$4h}}{RY$u_o2l^*WJq!wIp=4)es~xu#Byi5!UIIi5;EtM7K)G`--!U1t)tVj$6+t$Lp5|g`rc*yKac)XgT5P zx4NMk$%QK1maPGYCD#MtUB_96Vdgmb;^V@bN!yB1qemh$!58+I6W#+)aTb*_6&D^v zeWP*o{kn)3UKK&wsu+j+m7`u?&s#uWDJT*lt@X>YYyIXb&vT--Lf zWzV``x>VG2_x-P|Jk4l{hDUAPj9ld_2a=vroSvK{7r5K+-bi#CspOX0^OVu`>3Q)2 z4=O{5J-oz>mwGyiJ)D9vFuD$pn)=>XgDzDH9ySW)5<1=|?Y3Z!)gNHXk-DI~diJg;LoYrc1H)_10Q^)`Ycou>3 zk=b>hg;e+1tsb#0kYUJ}&3Cl#QyUh5#&@bGU&Mn4zu`zQVC!)lsa{i^xbc{`SPvzr0-Vb?*|V?!jF%K<|6hc69(Ls5KCap> z%NWzSC?4{V3mfLNrEA>*B(Wloh1Xt=PQ!yS)alm**b2^h| z?sFz%S?Llyo{>D>VpSZEAC%0>^@BW7JrrjQx01tYu{8NbJgbXfyHQX3P+L3J^^ZmS z{uiMfqprF4g}$FBRO1Xo1O2S8?5hcPmdxyJ^VmyQWf(RSmQ;CFpu|oEa5<-x^blN$ zMud~5iAw3V;kYaHr~AaHI~!~*rGE0V11UdAwp9cwrh&mM4e8;2Lu^Cbxkx4ltqG`WRn*nZY%RbG0|}pQ-0wz)f$5I{@qboco?Wwwni@Hh3rtoD z;&g~4B%u0!>wiUQbo9X#PydCgzLY>RJd=e;5g9QO%M%C6e6p*e^~C|Xm5PL}q9U$7 zebi(-Y5!@oGlXX#<*3+krdhg9+rr1EZj_fe5{DvgWMitqp<+`EMS1Cx^!q-?J~%XLzad?4`}I)y_cZ;)xNU=BA4W*0vZr`ez< zaRYK7kX%b9BI{!^)3F?b`2kFF^nV#@z$l1h#z`we1|bwcryNPk3MQndI|vSSnFPw} zgIE^Cd}onrBsbj$56&Zf@KrTBTq)AmvO&8*5TOIXvftAkCe9APlPRGA!W(e;Nq+}2 zAM&=q0Nx61&Zb+UIX{1@h#$Oj za}ejlLvyRhK@1F%5L5EfG+Xv048&^zIQWRAUs?}dEcE&%@fg2=_%mV&s0s&DVH-ev znSGYGb|K@owctr9l6q9S6Nr#ZVYy4bJNYgi{2xB5JaF3I)M;(Q)m}UL&S%;;UeE@| zPoTk+Yif?+@$9XxuGc6OKK|txx+jq$E+J3WR%f#sI~ch^kIJ`;X&^-*WT)*IB~IR@ zo!LYAryt3sum_h-#dv<=F7?=f?r(wC9>tWX0Dd*L0s{%)1xOjbo@LGHQBYF&NRGfL zOyHZV!;~9nIV~zO+mGkou`Ot5JE9Z$gYw!P=c}L~Z+4vjHm1bb4JtBFJj^S(JeMsJ zYHVqNyrfmYsh*bfMsh1PcE%r!l*|ElyISM|WYk$9A^b?R*FwTv``%-ZDtxuaLe0Is zZL<|JG80b6`;WmALQ7i54(0_QCxybX=4#H zH3`E6t2_SJKYJfHiTI_anB89*uQ&8I)#mTsLI0{b9T_dGJcH`}8voPyq&;{lH4L4z zt@=(M2UX>Ci@G$O!3I6PgC|_|4WTDygvMLe+yHC=1p`i(j=-;2s3k|BsGhfHTU(Qd z-5%T87 z@iU+~L&yk_bdyP!InK!q`q>x(tl*V&--eyTb9Rvol}ZbyW3=7r327dVg_Ohf)eg43 zdW_M74N0TMSkQToY~mZCm0zJy0HZr%)p&ueEm_%`6^iX>#1inHtuyp?il>vC#wp<1V^HdE!w|Zed z&bT)uqpe7WM12l3w=f{M{Ccumpmfx|pjMzKLe8NTvsN}`l-aioq5#&YhMOR~d>}st zA<|+76k706Z+bDKUEJt=^8Kouv(q0lw7`hXP*l6rY01E>=RJTv0N;J6m|Q_TI^`RC z+<o<(e)lB=YISsw2|#xP=mo=fs>)3QV67c z6}G=moP^29E)cI>^+nbq=Ipizifl984lFmZ=%dvO=fls9{IzGfuS8fe;*Kag{+v8S z>EP$oIs-=VQ+v=eu2PY?ZE=s9+0E)hTRlmSv-rU-UF)?9V#m~d-FF(SeYG-0LNI3b zT-n|O@rVspFW{x6VdRMfj{ep!EN6h06UGe?6d;F>GZCW0jy)M`bLWbiL* z95N+Yp7y%~2f^e1*|Av*B5fge6QQU@DQMVIn6!0t#-x>j0&#S9CNCSbE`RX$WgPq^k16)-Ut~2K~mc0om{2>Gsv*3|X86 zsK=f}t?qbIQc;Z|BJ4@n<&ngA+!9EKmbt;#z=LK`jNMa=T!cJV*n`Cz!6|pH1fnb6 zrZQdwfAVigJOrtLT4<#U(W|iBJtU}e9TsTtzc@>;L)F+U!1Lx?kVs{`rj6#htswjQ zU-IwK(ba3MHEl1#L)d`}0U85(!zj!vqc8K}O&#>OYg;BBeuM-MB!K2(viS4ca0CC# z*9i%d!|ipgsVMJ3$dm%@0I)_O>O=EkW?>b@S}kI!@|cP}tKFfZYITT}OOb zv%nrTfg$f+7nn*C;6?;{)xQ*JBx1hEkd%meLv#eBm?R|o)3N6Yr zbz*-Dv^%18y$alq8q2Jt_3a^m`GTQ$pIhoytS>(Cn45y{^7bRAn7)MBN@YUaCs`<7cg;($H1ou40|MKk(V(fImurP`z6qQrdVn;My{ z6C(u8n?CDgFsH8l`;Q;(3_5@!Jr4 zg?#K@Z#lsE3bJIewS332)4H|G(-BXujD41O=vJkUHe>8jQPTCLc=lpH&iw7OvZT`ho2^2P^q< zS#MXmVql;P+H1XQ(u?Xye>9y~!*o^qjVyc9V-+E)o|@?-?p;o#iQz3p$Cz@h zmwatcyD4PIddpkjbGNV0N%ipClos!;wyh+Q(rue;10Mm+pY_i=ogXFhSr9lLIHNst z0l}fuETHLox{-R&A!QFRmFo|WJav=7Ek%zi*3%usoGWNXKW`jS4k<(Zo17Z7SO&fj z7m-KnBTm%!?nSrnV*I42p)kPSF0nzy8|@kX`)@Ob}K z?Ql&*N5uMZWa?;9e(KJ}Q z&Elc7L85(Xj`klejDK$VN#pHKcb#Sgi1^!gzwh*grJ=VvQ%*m?*F--&Xx|;#=ZR7v z!u*x&bCKcoRn<*jBW!yyyc-wd#+r80T{e?8D;%UlpDVN|326)b$OZ$6go?kE#B0SmKm_(d|;J77%*^)@0h ziX7F`gu^5qDHfS-3{W~MHWDIJ#Vjl>5suBwq#B|{2f*2A1a+O}E`EuS?*Vnp;Q(P0 zYr;tdb{QnOjjp$j!WGH~jNscv;s!?T;2Um(&YbD#pH8ZHEWD{;eFAdEWbVx(Ed{t; z$3WW*%CTEvkCJ%KfFHCOVrqUIdU<*xac|(J3sUd8o*f=$X({G=EDSprcNp%ubJHM{ zn%}!d77liy|FYV^?6qjVJzELOm&cz@RwUS+>Dq_wvwpH>hVXVl>MRhVmAFn&0RUg% zDuNq-hY)5+{{%bOG`HUYAtbH*7+OZ5ppybFo*5{(YK4}h>DgwJB;-+)t#JEmnU-<# zOM#-GCSBlUcdhfTLL>=_DM5boJ>!!~Ng2yrF)B`4d)sv3xT%Mj`brzEH(MB9)2}7w zKP+pP;P<^uqA|BMtX8uwF@&Au;X^xtgS6-7x0;)64f*W07oGHX-wJY`2Q9ANnwa~i zTf6A`lb9_`5{=<4T`DUL5}V>?bH%vl=9=b67w)K7Vi1$-p`hiEZ`5Qvc zo;>2mVvM2SD+rOx$S+x6&TbsppGh5mukoOEPvKtL!jdOe?E-B+dPqL)Xndcmk`M(0 zP%BrAWo7DYjM!fDhmQMhqZT9DhBH2zZa(R|MbVmtGLM{0_u^NB;P`vSu{&cg*%S8A z^2pHKkS$~vjVax|JvMlg9*EPkavo0-I_5RrNBIe9=Bd2X?_cnF(cy9R+E`_`FHY0E z#?`_aO;+0%4OnKL^qjo*8aaX7^TducNtgZM8a+de)pSuOgGwlKONak zj_-bjy|T*beb#U_%0E&8jok#2-I9OtC@8bw#H$f5Fc6%VXrUkHm%o@oZG}zXx?Kh_IY`E`{!^BU8le){gu_&yvKo}zbCAZ zJ^a3>j6P4bzq2pN4GIL-oC-M+AQLc7nn+GpAL>_@g|V3zTx#d)D%J(kE6f!W_u@J~ zCe?VVC!W}+eGgUY7k$MZC(mOM@5;I)j-$_O?TK!TD^%_=qqDm*Pe3yhoGX%<`q!S% z-dGqIDijK?&F5B6&3)`szVV%M!k@%e2e{n*Gx>bR%Xsl|pXP+a&|!ryUY)dK^E4TL zPDMN(AiH(nUabXynuh?~@OXZHe$s0oz@*|fM}nHOnjV7u1U$kQk$$Xv}EktQebZFElzJZ zTCUx_AuBW78s1F!3gn#s`UgOEkZ@ti8PJ@F*WicG2fIaNhiq&T)x>q^3IWK|5enOo zq#P2^XtF#V&j()WTLGv7e1MOY06Uid0v$%DrD?(N*{jt?NR zl#u`UYb{BHh2?IF*VfeMJpJ2r4X8MaKJ^pCgeR5Mno^;Rd9e=Xk;IY&%}nx8`Zg!s3-z zSl9+5@V~5EnkobH+8W=Z!q(*S2TkBln=grgqBYI;(5h5mEbLna z5M$7(7O^+w#|WWTl~s8Dpi6{vdeUQ#7|{bQIU$=t*x^r5!?ibnfQQ%Wm8*@`^=m<{ zqf2X}EOS$jv(K^c{2W~QYX1Ga;Hl4VweZ2c((;uRb2Iab)M3L?^D@ghpQa}J_J+s^ zTFzZ7>^q#sv2y+4`PI*xj-=9D^iEEyjp$mtleRIPZX2>xvEet{z?>KKKpL6JpLvvN zZ6$^xRVmhHcubNM<^^q^sgDYU7&L{d>g47Qy~pr!r5_w()ICj4NGwS4O>3(&+ss9( zj>_xo@o-cOjVE-1jOR`YR+bj~cNrL|)(PK!T|MaESU*ktM3QDRxl`dn3aJ$5qqoZ0 z8Sf-{H5EoEdG1DO73WWjJo|8B)Bi5f4_^XZhV?t+|EbRngNr&(yG{&-z5K(Nw-Vp? zE0yx)I_s>P)~!F@zZm(>MqffDV@G28()ezw$4N+cJKACz-q|B z?*FYpl|-i2M{PiS73*$HF(zsFDTGRW-nDM>aa97gw*GW{b*!+VP~`#d(+JlN@0O5q zpQYg;9Rsh7ldTRI+i4`nTRz#je*me&@85O;>@_al7ipFwHa9CeDhHkB=R56+h6C|k zJq~{dvfZ*R?ySHUgNQh|o?o;5%D9p0 zK!1&+$_gFN_x!ir^Ly@dW)AyKlhX_ym+5~cICz+v|Abj$1(-^dgLoF0hIsW7Sd_zjc}-9ZfvS{$4MU$ zs$$H}L_QSY^WQ(-?u{-%P)hI{14{fq#^zm0P?Ez{PlE=Vb?)cS5E&Q?baXVtrd@?< zT$Pu+H*a15W7vE%Jph?^GyNgqo}B}Iqa3>4S@;JIofM%$$W7bz&N8kq32Jz*cQs=+X4)eHC@dw}qq2nG-o6k;KH8sO=bng4!x zABr|J=xHn`#AzUl*Q7=`A2QZI<)gW>xo_QqAj@FAo^At3R)adbno*ipBOPx*vW+x# zaYA%dZvdHU!%rxY(bVIHg!%sqID7*^=2^K9sxDfAgA#!4XgfGWAa68jG6ABzGEaOj zS*FTS)7eh!6b9-;EeaxAx8!dQ#%3$NMVf!YkXgD-ck zz|iw|5NO#B88;Bc{!J7%Q=aow+FC{%@E6oM1!LEi{rIFJ<@ z=W+*57!;^IXCRsHUyv&dgx?ocArLC?zZ67I$ zfevqm_}tz=NO+~j)ljzvoTuyA$;K<{=%V!gIas5F86N-n%XJnteo~Mx@_Tx`!uL&b ztm|rYMT4?jZ~43dT72D~%wXc3-2BjI19Q^^SW6jhR}ZWlxwnmZrmUx1*cj+HNM`Sq zEY|&`m!8Pg7q`DD9)vu9 zW^;7%R+n^Arox-M4C~!FueMKId0OteO}1!Y_*wSO2d$jFl3l;y;q;^34nEl5RM^ah z?9hdUlb=3fobPxX8%LVP>Q=I6+Uyg5PZ$`F7pXQquNRz;Iuzvd^1d6c6>ys{&RnV>Kz;&;m0 z>Ch=DvGWn2l>n!~MT-6duWw^>x;B!eK6%B((wj#dH`)(PEo>y^W@gf~>&408^L-Wt zBCzJazYUF=&q)Ecd3I~dI7xux90XfUD5t^aB$z`0_7XB;COn#&4Ozj*al;@q0*9K~ zTUXXQXYYGRv&Wyu+kKQ^UP&yE3npAmgLQnX)vj8Gzm=L`p^MhAySI6^#_M+0L`KEO zc(R4+^7e*MSKhFpDG>Y%ozerwd^^|`gNvwZ;tV`I-Z$Hd#VYza6a50Hm`+>jpmJ5X zbm0bKVw*2Bl2s&ux0#jt>@q8I$g$kMO%BPvNNjIAa;@KdoL#KKj7*IeAy0|`KJQJ< zu8=+tfjMvBcq9Jrkn)Dkk9KV*CtjGUpc|1BG(l$U5Eu=4;Qtq@XtXqc?JL{8k7}+z z@QgsIfed^31ny(m>?82;G_$12x-HJn2LYG}c9QOY-4|dcmHqz7 z0e0A=OWp(oD<}7!0NL~uh+SyP3m`VsQOOS`L~}w!=Ug!)ncV?9F;a^Vb^u*RP>TRD zaYF6j++BxI*uIQ-#K03894vj~if1J0J7{2Ug8qXaJ+-m$3Lsp0dZ$WufEDT3>ZnCs z1Pc$;U!3THr`T<6gt_X*1FYE zK&GU>O9+i{kft>=XX?`=gv`VRH8r#lCIN|#1)BRKrm-;B%^>rmOacg}Vf{oBWCkD; zM2ZLnc40)W1<7&iDfAE;4gn_rTIfn4KiA(P?Rm&>4cQJEh@~LumkH|9izg%VAZZWE zAf0_||ABsD`Gt1LwV3twfrrY>dFp4CmF3hm1>Hw-K7CrR=#{LEhJPmB)7eT;Kz zxS;%}jf9fzk2<83nuiMCwvU+)spXZ?+i#Nv-zAWeLKjKR%ny{>7!pyXefYf!p?Iob zK`oO*W7dya|DK*XGt}UAdfKD!zJf~{8&Dnd^k`4Ma;4dIE1>FFgDS@NWyqqaYtG57 zzKZ376;}7<(<8l&k&`3sLp_xLmkAdy2DEezV_0$UeM!1^Y^Xo~E_dz}%Th)`krwy1> z)63#GwtQ1M#w~!AvDu(c-QnEB3Z85;ICQ_mroR7g#;WwgO-=FKd>*tAYvxl&t7C`$ zN}@~0@s@!Rj2dfO6^ad0*guK<#|zw6`jePgzm!&MZf-5(ofRKT+W~0T z7R6CY_YLdMVLXe{)0e68ltahI(B|lLj!}V(5qlkk zsRHfxlLf7yb^>ifJrSrH-~=pjUN|qlr$==ES*fNF8QGH@3ppEjL}jlLkhPx=DccSF zl8tka8pmPg_)O!OXfE1!bAL=|&uKmRsII4^Qc?N-+QE(p{8P7Sr5wy(GAEkAK%xXf_H|D%IqQ(uD~U%Sq0opJHF&}7y>AfdwE zS0cT?_UgQBFL9t^)4k!g<`W*`>%hOaLy<_vV1*LxZHR7io<9#uFvc+s&?Mkm?yx7Y zu(K2V%XNhV2yye1m9B_EU}`JuSm%Ok!21V{L~V11S-#hJcvOnNx%^MiD@0wfr4ZTZ z{0zf`2^B%tV0Y`F$+Z3ZL3xr%z9QQ!L@;wO4e*vjh z-@A5^4pBRBAHoI;F~KZn<>lpnB25zr-UOH${APi6;1jAWImn35BMAeK9!39?AI5#L zn%meQfh+>;J1}WbadA}`G6ASB1}7NgAaaF44G{}cEC`tg->jii2fCUXm~O;(B6;Hf zY=bbf?J7itS-$K-vMfam2%wnGHFY&IY@K>dIJu@?qMR$E0%5ool_UY8bkY(&);$gbVSYNaC5!tNwEnN5$Ahm0o zU*q_nJ<7Lx)(-GoN5?lZlN&DYhkvVrv|Z^BV#`1F-n}Bi z7xvC@%FjqPm&I5(SgPw|8a{lff6`k~|MkLPFtPq_zs-)%pM4V*fw2w0VUsqj(v8Xu zKOq&VJs+n6GxQwJj74Ql&1Dgha{bAo$-BfexP&2dW93WWTuy5{J=x3bztHgbwLE7G zV{gmW-1Gf|-rnh{Q@b0!Aa%qsm#~h$?=WaXLA!FpGPwbt&NEfvedp3ZTifAY!f&Uy zzbo**>^4lG8vFVV=QM1NLV~KmV73HiXZ=Pw#tEJyERe!_)oKwy|861w5v(n?S@b!9+9Pw_~kZ9;C%%bbVfmQo>yoO;C1}EH2Yp3M(^W193nc@&F5CW zpZf?;nZAbi{4irKI+afI*<33ZllX;!Q;9)?<3*rEciXlavNJa|KaQ@06>(hnP2`mh z=NVAu0dWzqT%B^BGvu`R%8(^LQk@4|01iwvun{3pO)3@fU3mDD#N0m{or#C|F6dCa z0&_a`f62CAlHn=?(rseIcYl-lwfsWHEp4cKf+}r)I7etrpFVwx=u!Y3kp*O9jE^1% zI<+8Wn1+XfHQdt33E?FF)AG=WvJA>3Hvj^e_p+Y(a8gG8B`aY8sR^d2Uq zis{!py9q(5KVSiYG371v;AZrXkKanFL?OjyP_&_(tIiIwc*GD*_re46prCuLcXnKM zzyfwX+pA7fX}W#Bt9||9|$R6R2N|KU25+QqUZrQRDQf6*sC)qn>Ws{XnviF|% z|NQp*z5n+;j;H5%j^nA^ z>Hb*rx=y74lxCQVFG3m)?VGr`I2&-Y15Z(~UWBziAXtc@-DN5j=|RW>upB8#DJ#<; zceb;uhqS2XCBC4;J9>JIun!J{&z1BDD9}j00a(El8UEv)ql~gWue$b4Z|P>OrFqfe zl$EuFc|z>W)ROpm6Nvrj@mHGUl?Wv(=YaPsYit}-AZf04P7m$L^0$*AV5&iqIqGA6 z$ib3+#+Ewh!*lvBLtLCOLbm__TQI%Zxse!HBF0c6pIE^2^Z&n^=h<@IPB^XbJC^8)2BD) z4p$>g`x&}s`1@Zx$>=NM$?$DG>JVtT#P8ZQAn3F+vW4P^1#uyyA|l(%hrQl7BLm}C1eypTd&~{v3=9 z;HTbEQHf+0f^8o)2Rry;BhM7T(K1}@>;$kvJNw)#7yN!+<^KX22V36;LAj1pT8M6O zQPFb{92n14gPs*ZNH{N+XlFwBuPQv2i;I3C-$@6SZUnj2Lx}?FLzzuo)u%=PcR-1( zyiwsm%LiMpvq;+kMh;KGYd*;Io(TH1YWSdU7b+|8{3GgyT_-O+2Sdk2MI;mfGFtux zkwWKU<4OLX$P>^xFpG+oH#naAj}bQiKeIisJmP_nUkKpDK-w$0Gq@fjHN)I4*aBH+ zf~$NJq_l`FBf-D`K0OB$!jr$W+{ibLNok*`T}g11k@4-{0}BA~>`U*H^785gFOik4 zZ4lUuOi=G}-KRJ>JU__8ef`z)j*y~a{8z(f2E(Ia6GO;yx4V~~h?URITX7rvwR>PD z=%%W61IWw0t&%@x=Qt7skd1W^2_5EcYvYLaek~8Lx|Zhz_}OV^2+R**xJJt@OT*BW z2Vzr%BWiU5-Nk)JnH59IJL+HbEZ3TY?!R`K@_k*3U5=p5c?LL;+S9PF`GtWtomC;R zHpv_9;zXCF>T2g&ga1@A@zRK&tu)8hMqN+akIf%^^Fb;8^T}2rqd743m9yHl{G!`7leU+D`qn%R^%*Fl9w#uPetdv8=z_lvSxBhE6 zD%aQ@j&4D}%U?c3L=M+i-&aNEhqB}=m2~-HFK`dJaH1E z6zQb{RO;4R`3CrK0wyM7`~de0LyNqU?b7%7IvkI!RW?)?nSVSP4e3`7_pZBxrh-V* zP;5T<`0?ZC139zJHJo=J4NP=Mq)`(evQiO9k@^5*X_C17ezZcz7P)7-3%p0HqG` z(a(kJmY>1?S(YliR-GcFb6}Vv3tU{tCk!h9wk^2Hg6t^x!xa+h8!_uu*SdDdgE?Sl z1>u+iC6F}=j*=7%+?{9VJ?oyN$b%a-ESEkZp|tKrT?|qlWz1jt%~XK;OwGZvsEnW= zrf}~Yj+WU%aBgQ?o8sf{(V`$8cq`yW2_9ch$hUTOUfQaIhs`9Ai;yRp`R~d~81SR6 zbfv^Gn!CBW4&Ri7Xshc;qzNh5{_02x#uD{H-fXpd>G19Mpn7tFoOI1k@j-_=Zf+Hy z7$AIj^7#mtmZD(yt70w0;|wz4j%M zFg1t+f0!^IVc5Khg99E97|=|@P?tDC^nGXPrS5JpFjagB^9Iu|Q44L~P%a3in?}Vz5Adr2dDrDRXE<-rCbqI+P#T2a?h+4&VWjZQHT3h$(#fgUe}#L zo;7t{LB2i7Q;MXP^4>#_rJwPrPUCgXYW=>M8m}p^T1NW?e$v$+$Nyg2TpoFXk19Ih zEjD9_Gka#ULy3NLaOZS10(`^Z?#Ib%W~{z+X_niRhFSe1fBO}%4)2OA+rLi&pijUvkJab0pvEKNh6j)cDZVe>XFl=nv z)mJ#GXq57tkL8JBzGcf17q2~0IZ`rK2gPIi5Q;};%zvXu?T>Lb+ zkXNC9*NS0VOm1$(O4vn5eL$MvmE>{a)ck*X93WDzowDRGyJ2UZ z(xPImA7pu;4+2@gk7+VBe?VI#GoLJMi5q0}W|Fn(0S_I7rVW&u{p zYvBA#jpTP~K-{E*SRxn$sE#EinUAqr(9#2m5X@K4 zBI!cVT7yqEi4X*HbQ|Lmvux}i9wHzDIhcP#T+j%x`8jV=>CGD+j=X^%{g5WY;Xi4? z8`gbywILdQkVq8+pI{%*&-EQXOHoo*o@!hJ2!aJNvwpdqR%hLJNvwyQn8SXr zC~%F2-)@LKJ^8nKsc>8hofC$&FfFZL2p>`juFC;-+3H{+h~&aP{EF-gSy0?CCMUW8fX$6hVgGn7I6ee)Jz!E$nVB^JXz;He}7Gadu% z)rBfM!-CvC0}u=T@r@83YwF~=a1CZrP)Oab*{fbNfU0@S2#QSTXiSVd&fhxC8>9oV z_sJu-(u2jEWu7Bec2eO}&s80YBCRDeA5Fy+)A;7hL1#VXFBw(UiF{sF$pzKc`X!AA zIAbcRznt=tYHNuhZ?eWoHg8~hv>d~8)Dcf}Bhz=kUwzLlI<>mR=t+A#jhO`po1qnz^?HxwOt44pv|FC})uWfg><0$_q9@TEdAGMm+7e8iM3jg@rbu=RUE~(CcbWW$c*K&3IJ(b;@RuD|t_ZHBPU|P2~HpSFbw` zUFi64{zjwnE1%h*Ev&gBGcQG;shL!rKV{^p*9-P3%yc+8lzxyevgr439?|iC5u9xA)R4U7( zSfx&r*2YZlDx}&5r7Jx8;&B{*HMFm;wOo-A%Qciss!9{SJo!q=mWr#XnXo@er_1!Z zuuY?Mp9D* zBw)Vh)<#L2I8`!0Gak!#329)IqTzn*QK}G%Cjk$1cw0~7A@~&%f-(oj$HU<-e07E* zlGwM7nO69A*m4;+MUN>YcEqd3gwpwA?t5B$@`!X`P z^va`}Yg1FN!8VkLsvP9Y7+VE}KtY)x(kaHf+1|aV3B<9Ly)PRskL=8WZVZIx^1v_+LhD8O63Ps z!6Cu7pQB+a{Pin{Bkd6+o;^r^$BBHN>YkpSB5vE9NP=t6ZH@>KDiNSJInX}d-rl%+ zN^oilA!l$3DCazAxl+rKzjjQV6p`YZT-1oaD!85It311HlR6GRe^PwnfD}pKPJbd= zM-)8&%E8HL8WQ$o6%|7=zYH#I#N>ONg`2~->Rph0c3~k^AQ@9r?a4@ON^Y*j_m{v& z2lt7te}4-lC8bE081jYpuogR-vmT!?bw%wJCSUq)#Z+-$Lr#v&3wyL_Fj#HUb%ql= zWRjQK;8QwIdwM%{3c9f4io(9>0Y~Xq6EzxhgU_C=K(cPUq1HrH>7pjrE=BcUP%sCC zhfmcud*diu46VC(CF2p^u$g-_m?`imjRWghSZMxObC_N8z2yidL4R<-Y3fJtZv*|P zSZ5kxt9!Ao*_XFsnom}3i^C?Jn}-aVVy^WYdYarv<&*k19ZflY@xjp%v5}ib_4!Vc zKi(c*A6ZXw68JJFRrB7fZgKc=_C=mH?lJm>iTj1_s1`^hPN^46WT-s*xJ3g@nyBk&zK|<}F>; zmz@%Kc63D?&5X8$++2g#VlH=izPxjc%_;2jnK=IJ@NnsAPRvl*#)j0&T=E1M97;CS z@Z4MyGBXoPoS91sp@mZN3VZ6`neXm)2%Z|cS9Wh>H<;K=&F#@xe-3o#>(}Uk;Gp@1 z^>*ZY>AA>4cXXYc)_*?waZ5}Dwu4ySW_rNNesXhaDmFx$X;r7ReB{kEua%@{H*RK8 zgtAqrbcQVD^etpog&9Ytz!rdB)D9o0`Mm1~bICI> z!=$=ZKgzIoXUCiZO83=n_JK;?w{MiCR^@wZG{}Kck^+W`fq?-=>+an~W<_N(Vrs5C z5N4O@w7aa?lQxZFO$4m7rz|V$K86(~I0np&*tp^4Q(~i6c2TWHFm_$+uc)lFJ=lC1 zOxfe`q9Dk+3ZjuI*{-tSXjHirm`*DPHr$#=nLJg(b}7Nng8P^<{VL{TrxAa-U}i{I z*rl)|;o-de{J^o!c%op_%>u1Fp5}8trsRbc% zc-JqUhfJFL^{VhlQX+r?fH7&3$5|S}-kf{pE@?4TWMRn_^?B3LTJO z-^UJPu-C+?uAVl20^}J9Y8ljwJUKoR*En&yG64*j1Kc&skB}x&vs;{R0Gp`wjg3fV zHVVnG-CZXng3;8RO4xsQXQxrUX7kei(Gl6^0~Xrgq$Ebr*G#F`9$tc^IF<_1MGA-r z0`4Hi#L2320c0OAQUQF!!V>rAml@mXwsNdk^UIgq`{R3^7b>b6UqF<)!gIl0lFHSV zd-_$|y))$|r+=+GCl7bttISg2xt<`eau9W`x{f-t}xb)r9ajb*x)KzCJp z;!WVAy|TdH$tT12e}$*4X}j#t%!DN-QvaDH8Eeb_{232BgHB-BIIP{PhNBzSBvrMm zMcw|lRUCU*<5>_C>amaQwW5Kx*R9x z)#m9k+1H~l##;)u5&8(bl~9|jH**SJ6caQ=J4NtL?3IwKT3rS93JhhnE!Pl9tovQ+t@BGpCUi zJ)?|uIMj76GHmwfsoj|X6AYcIA-?%d5O(ZOk#&WcjybP6HCQnntZBYlm`S!z?8K9i z`At##J}4-FJyr}^Qc#2?__$kB{p7e8GE{7zIFmD6s$ySm+cKDdUK5Km50s8vW?Br} z?@;O7=s{nX?OO@|cXUc>UMpK2b$Cf+#iVDDF@!ZyI$y|?c#<&oFGGYQsC{x5^5 zKHyzh{%-5y!B_vDK(zUQ>Dj00(Qb|@`RUfR;;afo)qKf=_?s9ijaae?dK=MXW^QT| zKk?>4`$gNC%6US`0E99dCaK zpBZdHF7`v=AgzMDPB5L2bVK8nmzMTpwon%q!KhaR8G|7lOnmVj z%AVCM6g_>wGKSUobg%(1mm6A-f`i!0wM(t0Amkhq2aLuN>xosn(1OCkzL%nmfvhZU zZqUeJY@yV*Idb)lLH1E35NNXDjVzB<(txRABgGJ6Gyz;@upq^RB?bUh3A;`})uZ*ZucDy==? z1~ECIj`2i|CjfAD_#9A};%Mnf38hZ=4MF}wceU}ykLM?cJCx}xn1xlix{%et23%vQ z85zy9uCP;s?eq-Y#6A+KCjOl^$n+biRT{LdtNp{nNma%sCYX>HGqtpI^F9w|;ZtcT zO~#|ddeC<$Uc7kmf{01+gLGLfE!xv*-qSOX$n(*55QUs?3Z<9eyvJYP;F&OkK8S4t z+4fT~#1l1ls>ExK7<|#6z4tcWo$%^avHc@5>DuuwulE6(g^auP3-e!@*=BLT)c*I^ zu(h?Zr6#CVFMs?veiS`(|E8MX9Uint@(2`oO$Vn6bj?{bS{WSu7CLR8MiFIY5!xKP ztbEEZ7^a7SKWcE}`Pydx@uR`x8%)Bjhrb8Esc4TZe51#K967S30neat(vW#~hM%96 zD27VcHmbJD%bgL&9#ypxlEb)FtDC1M6Zw;Y(h|a$JsXbnC*i{5Ll^v~r}or!M*W`U zj5qib?*(|S^yhT!Cu6s`{&u)GaBKSbI_}5UYEIF|#uP)!bXQOB3p>q{>aozdWa-f9 z>CY6{+K;VGziA$Qd24u@t=YLT%WIXg-=>YnW4_Ec^BvpXSe)S2@_8K_UzGo&j&Fhn z=lPob_^Ey5h<(`H z+KG!Y$vP9Z0N1Se49=v@xuN1E`{R+?OAW?!9?5H)v+`Bz^#;END|ZQYclsw|YduDv z{+;ikP0K)oo2i!0d(U^w=b*3r4p$vf+Gj?WpKDG1V1E);R7AEu6?ln{q~~bR*nZM| zqOCEEE@H3JX*EH5@*5;@yM&L1lM(|EM_gU~WodQs+a#D2O}Ft{M=W9u?KWp><7*F; zu0MFd7<$~8nx^`5BNLvFEU7*c-nKix>pnH2O0-`-sdq7UV~#UNoPT;QN!U5(-ihNS<%$W ztE%hw@82Q%#~AX31tv^J23ZL8*$b7c!sBIcK75#%(#P)b(*`a?ob)JMg6EHa&gE#= zHNn>}s3pv0?ixHSJr^-?^&Bjpvz4&i6fv&O2Bo6R=e~#VOQ$#vel_72o)glosrc8~QE+m~LdJy=vhzTbO8v_uEwCY54LmsPyJkk>xYAH?%gE2E(84 zMGP=YjPH0RIn-v8qz{+c&bC;3V248RcnY@&bKtks_;oCLPm-^%>L)h|po`DsBt^WI z6&dDQ_1VjHodhJt>1Mj7iT5&Rig{FI~D>~jSvsZnS zE(kelt@n^^C190b@;_IpSaR>%r(JT|ht?Ama{RzC933cA&FCwCDT!+6fy<%ROA^+_ z9=1=+&2J+9B=6p_7=WJ{2#u|Q`zhkI{K4LW|D(-ZCqpcILEn8KI6#Pzi_H8ssNRE4%%?J<#&uVMV18!{Ed zfNBS9Ybiftvc6!2?xtQjB`{;5)dxBe=S_Eytq0KDf|nWvB_)eNc7eGWrE&r|3bHe-9_&F)(ODh6lVotiIS!bwbmO`q6`(1U9GT_!bf@4daJtiL9ix>b*- z4-5|nPfn)f6SK3}#n#kFfhmfP>ej}_6oh{AIr8<-f)L@{E1!FRz=DJ)`5J_kf$^KK>(<}<(Qkr`F@SgUKrk~^Pft&;-_amCVb6;OQ#i!;a^-fMEP|IOh zSus{sEt>inLs;fke5Jm7Ws1+egr>wI_u_37o{U;>MAdZ0j9XR4kT6pnwa^*i)w#^T20 zYoe*CFN697XJ2iES##B)kDD6|V@jjJ!poqT=S_o&!@)N_slY>?}8& zQmMtIN?aVX;(AL>@Q^Zs5t@#TjSL3|Zhsc`zs@eubaUy_;(xrr$!lFl$fSxXE~c?7 zzkppr=hZ8Fy<01}&=)CwX#d&sC6QG^Ii3s-exSFnv{WFb%Zf#e?cz9Y5+B`>&6@m8 zt3Lw`qgysLQ(+T&0W(-ViNAaAeSY@gG%wH&eF?dF&D^J1Pljd|~wm~DBG2rX|^Gv}9 zMdK^-!@*ibV=GQP`5S5mP$Vz7uDTtR9Jo9ldtGQPeSGv-eG><}SwD9!f=nKT!FBT` zy`jv;)g-lDw1Zht6JKnL>*+gOUo&}(pWhezGn|Pvb;nBBEMFp_t8l&VhK{Hm@s@f= z(dlDoNAhOPL%aa^q)x^?^V#zx(K1a`LyoxuTr=E{{L7+_7h z#v6wVom2W;pYJRV-+RFSeAj|^e_vu$AuH3gVXC1&GiS+ddV=ko{)ySB_maookDR$W zR{$xI$APT!QVO~waZvf3+xmM@Z&3n_JY$c|R;8kn-TD!2e{0X1;|FWE67oGQiM|z@ z&o*tFLx*a6eJf^2l&C!#GtSjHV5FrB!|}!*=H~#tR%mxM zOy*&z^x+{rCy!NqGn0Bs&XVoKP|Qua^*yw-=aH4v-l)X!dSL5*Alju2J3M0E0Esby zOvlV*GBR2%Mt7im(4&__s%%oEAu2^s;%NEmoY=)TuS<#g`x*AzYTN73*tK%xvzq;> z8T6*Np-b7fMvBnPT-^59h5OAL#xDQdKG#X6)7SP&6p$ePWDy6cT7q`jGEzo;`?pXB zFJ_Xt<#u^_@o?P4xm{-Bo4>+I{+pL@HZOuUv4dTwD@#G{?oAwK+YyOT9y3(4=gixs zUD97Kuw4S{N7B-`iposwV`}urwv6G}ip#|BOd7~%iItfyEqjt~J8Adm^aTg}R1+Mx zVA}mtqWqS+$6Qv$brr0Z77J*jle7INxOg)qn9vu_>0S(yl%{4HjVraIPC8kN+25Rt zg=WQfgkir!J>rF0s5IuC>?G47NpUCMfF9jL24lfz&XbQc1;kEWo-1Y1?i)$rw7=PH z4qwY}N=ep%E&}OqLy~o6$46#^bcJbNPf2ypOC(HKE7@L)GiPGN_x#HnMW8dcxw56Q zexD5=_v`NO^`_ErnWdZn=TG~Se^!|?o_Autr(mZ7$S;0rwx7`USH@0F$=o;3Cs{l0 ze}&=_U}k5#sCSyg=k!6vaf;E3=Jp%C%Gt`cTQ}V%eOpN5z2D`IaJE*Ejex>MBN;U~J*#--)qxg=qTer@=v9Y9;nNd8sIARdeh-}Ir;vD!ibt@lT zWF(c+fKv@2C077#vLEP7k_-Xu>8AV*=ru5}5Qi`VUzuO+6#y2(6iWG-Rv!)|=)+u= z@=6nm0eAvKmlxpj)qD0D3y>)A&z=n~xeqoleQxn6R4=qL^Q&GSxZj2?-t&>%e+3j# z82WIn#;X=`5e)^9i4!EkleDmwZ?y8>a}rTVf=xUwP$U6bnO)s2!V*n_V6z33E=MS?&-OjZ3#dbEMw#D!0F`*YY=ts4o%haPSYEyuw&OOwg+gr?!3LzwGe}_JunhkHWC(AKO_ucDs67(Xi4P_P zpY25Gc{&up8{v$9y(d*4mC_|2W(cHwicm1x>{V1$T&DbKT)5ekG=Vz8b2I@)5u@V} z`mT*IOp?*3<@0f2rw|rX;B|6H1g{e%SPcGhDL}_o?OSkgSxAp{If3Ubro&jy%S#Gq z>(7g|V4eofHvq9}eRssV(&G^sDQS~=elB94n^mnYB{V`3uawC5KVP&2$L2$ww>QLP=*~_anku zTK36Ht);67J7F>!$Rq9Huc`U6g(D>NCgK{Gw$L#>iFHl+6#`#s?6&gR&QrU_MkT0e zw1@t9`?XkMhQ>xRn}uFa9B-Ilw5@fKU-p_#P{2J-4<6A=d9=-z*X{3T+U_n(RqapH zh&bg^e{Vp?&orAup<2n@dIa9Y926-jYsrWI@KyF9jwPWbV`@#UD4~C!#C3ZpOB)cVQF4_xG8P|8N{M{7twQ z7B=0rb?x8^lk)0(*Od~%%i36eA_c5|WTD#AQNK7`FG9fRg}mIaP?pF~@7K~q98{-M zUjFO5R#i%}cDnOw^g3F^B(|VodO28xzT9KoD@q{J*_%V}RC>YXs5UtOHU&BG9vt6V zVt%*b=^xAnQ0zpG9>(5ZPh9bb-oU%M3l}Hzrl)sB?^LNi|Ecu5&_JzV>4Oe-kJCPn zi=MkSkIdrxU%#oU%xtL?UtVfn+B2{6T8p7tPu=a$=XtG{569vqJQihG4gHBQm=fP* z=*=OpM$kE?<9_|wdNa?mHP{@`?&Fyop% zY+!YFeVI-wD;kg+5g*@leFE#wb9h=N341Rp&zj3&<@TO$V@t{6g*-#!;sTG3_EvW$ zld%VzpYrc3Cx?K$1yPKd+l26DRC1auc$_Ini;)!$6X1}!Kb0n5!!tH&ZzN=rKm^R_ zC=ZW-q?=l(|LOWA2bvXt~=uV8?Z!UJUmEcd%AmT zzTpFG0S*Z+cF)MYvx)Z}J5D9FeTxi+n1JNgeRLOE2;&&oOgPgu?nb?Kyz{E)m!ySF z|FEe8<2}akhqv~7h7u=~o2+YHFIYM=33WOQ;TMfBP;;klFz)|20ysK_Y+!m4D@EAG zq&zv1Z_#i$6>T#7$GJx7bhHEja3|?^|zeRzKwBw=N@J(KHA;w4Ux@|vhC4L z*FO4M_-i-nU8@oBJZ>VU0VW-0#n7im4HcWm`n4iBsc99Uc0qBP>Z+>Gfq?(E_4C?B zHgM<>lT8?zIuVN(u;;hS?3dHOAcvZ1d(IbBaz89gPTxfyYU-h|A^@0_Y$>fseu%KO zlvJRJP<&)^a>Vww!-8y2^Z)>-1l*|JsJ2-owMtJfsZdQ#EdaJFkA?#Tj0=a;iK^?x3hCe2{VRfiiG#!;NXDB9x+G+Ej2IW@Xij1tRUA%=U0C)N9CK~;JWN*z7uV~ z&`eMBnY{QVVI&`dofES}g(fvwmq7shTlHH;DJY1EiibIeV3 z8(950I6IS-g#wuXUL3fPPA)Bp7vRE}3}3kpw?3m!Z|@*nGlLsF3Ng@}pUulNh9{Z) zN>=mg7%})-giu^e^JhM1YK@kdNXlR9ws+&zx$M2UnLoK#%iGM0)(izyEl|g+2ZLh) zh?csZrk|hRq)aR^@s`Z1g(c`R?2s`GmQ|+a0;#2gLi6T0%yYq~2K-l`%}wV2vq=^k zH3j((fz|GY8#7GF%>{Eq`&<+oG&2*>SIG&N#leM*_IvnerN*l~6upE)_ z*sJUvpC~XNJd>*}Ojx^rmjc%9c7j5*gt`L_a^W#IO$-1f0}0sFr#9dcVlw7ztt*p{ zjrOsFz$R=v|YxubxL9$J-gPHj{V!O__?Mm6h3^JduS@ z+hcizM*Q>I?0xJyS=K%v5(y$If!W!wyCkJ4FQqq^)+Tz_?uVdUTrPMjKZ22nN6+I5 zdV?kdppxRYEf~%(`F27#AmX`3~!UFMW^;#aFM8xY&bT45ID73g(@x1u=EkIY^^N`e= z=W{pNcin~L^VGbRjkU0c5TRbSTkf^f-$;(nd^3kY{fmTmhtU3RH%wh8M>NyT(t2T^ zn3E#dSnPy9cB9ouk-I9cNGfkZ_GR-Vq2DVH-LAam9;|xMfcO^`tAcIzK< zDbX{@Ns>X~M`X%UU$A{H7W)I;QA)!{^kp9V>^vSecuM^Yay)eOvmqa%AG1nO`48X1 zmE$qLwKKIjrJX6^OcPf6yJzI3O!C3T{kfJ|g5w42Qqn^=!@w(ZaN^qZM`fMcz7w|L z+)0Txe5Aun{Utuc&jj;{jPW#0!8wE|1MPl^ZMf86m2rPgiycoEa}v2fD#2QNnRu3* zP-ZP!ICXw2zU=8h=;q&*gvt;k|BmkV5Z5Hy)s@+r}&;L@*jokpNS|( zvWE}+$A!w}>}NTEhQ24dhfGvEMI|NBhK_p=b0sC)!=KlvaL7Vs0r>mM%Brcru=%m> z%I_Z}E-p+5z!LikoYwIO1wnscu#&CS9D(KHT~kv|J=fVwrh>Exnkz{fhzE*A5&OTP z^Km_=)VzA3(71pM^3iiK6EA)`MkX$SpAFw@lSjP>#6d>go&(3}G!Tw-nW% zQ)~G$4n<7wd4=y}JPk9I)YS#b#aLrUJOkgyL-LDhjR~T9_Y~ua^9Brml@r**euziM z^p0UF1>n9eZZ{10=Dz~`1BhtK)%SYaPMXY7ZJ#!t)baiz4sflxw#X|Qb)nz7XzMk_PI||@$8*01p6?tQA+GnY zu9MP_Ong$1PA0Q*uK#5a&~M@U^C?k2jxxUglEH^L;`BFj8SJT2g(QyYWZkwC=b(J~ z$az0l!mXAGnY^cuA8>TfF4-{tl)R-;&Orc73#5v#stL5~QgcdNT;EoyLx&4r*gBcK{RU6hERS$tSv%)vyiMR9j7Fw~+Fyi{rvdCoB6_4jAMO_-OM~gLR zD5aD?){=|n)jL8j8LXB}JK0_ni+{-HtepaXT=^@tN}Jr|LMmfV1q|p{4VD_@eJ%GY zHa>l(V%T$kzFHt_gJYIR=wY(JXQIOF(R5HxK%3NzGh!e^4(o>hjz{#s6Qzo~pD{zE z9p?gNy%{Ew=LH%G#ksj$9w$rsfjUwLJm-)KfGLOvUYCc5M|Xe!Z0<8vRb~+pje6A? zUO~Z&u&Bz+&bAu>Sx*r3-F7c`nCytki8qGCQY8O*ohZwdF0k&{p19<(lRnq zLFOo0qQQmSG@upcHMkM?249Ap!I1EK*JV_cAIuH)_4SYS_1AL2z3RnOeMaS~tQp{P z7~98>$vNcMSndP--eaqV6BHbL#laP}Qotrl&CBc1M2wjcj|J{)6g~IA_%oylsCwY4 z5XITiGQZ>2Q&w2`6rlh{fj|iuVoMk;$XvKU;QMcRCNefQ1_HIW|FgMk=(Arxs>GME z8E#&#B?Fq=HFb5c35L-{G1_QaP*`}TdT-pf|3(r@Q7!6fxGVA!GwY^rot7vK|tWFkej2X|3@_pPmY zAyXOQT=#e#Mgh$lq5HVFx+>X-vDjq*Wq5MTdBU-r@hTVB1=#3+YfE533G&I1BWVZl zIw}&RJ)Sr?2!dW)=P%(uz8&V{5|9Vx2B8ThBD%JAlK=-`sG-8}t@JG;fP$a()v+EIZ2SHbBD4F}IOm z4>_7YGxhWPvY>4~zV-5hDG<1309(VkgYqM*Io)CFCGa^qvi~{D->!KvNHpZkSjBkO zowPU3-C8}mTCDLghapbGB%4Ke1n zvp+8Z?5R_41m^)Is0VEh$i*IOXw2t6Q&U4MxG6u!T?U*R(^PI;c!h>n4;4_EKhJ`p zSehv6HZDxb%OiRHy6Qt*(5PZ+Pf!ze$XyJppeA@^{Ag@Usj3>)tb|x|(4(dFDb;Mb z3%U+l2o0?`spMgW0)x`1n3y6C51WNU)So>|$;dEy@QD34R!1oQJ5{7As1J#wT3NHf zS%mZ;4j`1G;$qQepu&TCWvA@VH5S^eFJE{-JEx?oU2eyA@w`7OSiWg+P+f3!e*P^h zAw>s0NCUwC2@?moxm|@+6iP3kI{{l=yMyBzuM@X+a0qA_baHlXn6e2nJK2)~(Awzm z@X!X@F%@pm|6l~EQi0O8R<%Xy=I(xX=WYn~)UZji|HCHtrC`%(U6Abw#dnClZ=2Ql zhB_n~3}$~|Xn1YheNA$Edpl_HM$CD{8U1fpvKGn3Ac7|V)4YFAt-=k!j$OG92Kl&S zu7t@0aI4|KyLs)r{~1`u13#ap-+6fmH|D`@6$-!?K3@PJK0^BRa$a5@#tl-+`2__2 zJ*EB)Q+L3|%FfPC=S!rk5%iNo-(C_McNybx19Ji3l{FBo!rMXZ07n40tcu4vj|9&E z1P>Jame$rf`0@#LdB8&pw2KXT*8lW@D1ne3(?*1~HRrPO`|-TP)444dPfwcc9B~|+ zqdi`TaPuyDZ91yKC8SfkhX6S8vi}P)CJ>GXjiy(uPo3?sOTR!@^O4!|avZ;^-4%F0 zzSGk$;QQ^EWv9CSb?W;4myK^Oaq{rU+i%T8ok1whh$k^{qW?<>;VmdA;N;?R2&bc^ zg}db}Y)^=uhSdUFn@g+c99Zr+4*-J%(PfzyYY|0Jbm)jBl0r$x5OvW3p#$g|2Ecp{ z2cfJf6il`UvQhVe9rs1Il0b$WN5X>C3Ho-6@hZ(}Bf*#pMUo4(9amWvxb&(ngL6*j zRvpl>a&s?$E#P~R;d!vH2yTLf5pqBD!5tUWR99CA_SbZqw_$`TvzvW`ENTd4$cIAT z2eCeT{1_*&VT%4{%!7Z=BI?= zk>h6sFAQ$PH%)NoLT=ahK77+!25iVi3vu!Du#keSJOn%{@5a>Hn810w1LdgD6GVeO+A! zQ2F)@4T(%{A(?_@w%4Yor^~9Z#t~Re)P&&?l72TX3|<5(4Uz`{>$a=Nz9tpsJNVY+ zJv~o8wZX$n2nRPP)F|q~zpmmV2H4eFd4v}NQu1?d$6LDW^m9QoE@}?^jANH(`<#S? zz~n$^5>Hpx2JR&UG(Gh?_1p$nIeeQ8`}T%yT=3SsiGE|Xy` zx>hujRo6K;h7ZnNh#wF8)#|y~m3VhG>`aznkd3VPyEouI4%g~>r}&zPIeQclK%9tl z@P8m=zraAbJhg@w1tR9K8Urs~f4?ktc{evP2tHh;*UnP|5+oxm-JOcNs_t86{kpv!VUE7ab5}%nuB06_u<1%JEUfDTxNO zCt#6M#(j;YRU=Og7N?r3ArTRQYB@2SJuNT>_msiO10B`_HfwHHmOTO5FsM2d{jj@y z`{4tp3Vd3$oj_pKjk7xf_c~BFUK3;g4O>XsrJcj;j$AQ8rulQ`r$%^$|M`>y4 zT>V-INdeeHNfcIvoSfucTbX?e>%EG+GU-nclGW#m&Q6G%vONQ{Gu^tYzwB#pgJ8aW zJ7$Cy^J*G!W*At>3`$4t?tm~2D3!!jjIT{E16U&W=n*M+?-A-YJ?CQ<5TF8?$>ag> zi{KDl`P-W=@3CS&3_DJZs&&~5@1jiV2KP3OUAej0jb8Ro0(J3>PVM+&ZoZ}fo%V!C8*NUJN%hFlou~kt?t!Myn(<2yPfwh(2rPHUS*Xru*a!+ZKrOP zIp)AB?dJ<c|%~uC91#Im;A*)BO_3?0B%P=mmT8& zP##Za2<=#CD{D_SU9+xrUo=`-cqir)q{CxU_~0ERe_W*I(FuYBDq;>DC^V^lu#(_| z=yxE`6T(LX9|bfgqW|}BLo@y!LeI?}>OFq!4M!2o+FH%PxM`3&e=gyNM{XGA4f-pShA7b+U&!yes0KVxtSyfHZ3ETKrjSshJ+ zT?i{UBP%4DPr zVyxL9;+4Q2Z*`j$U(-?m<)3Np;L_No2$28%JKGHn4QvniYmla6@;j~|EP@x|o^S!D z*WvvLxEBC>qctzq;?nqfCyozAg(Am?_IetI5}q#qz(7Mt9{`nSG&7WFXsiBHqDe?* zOTfBQKE1sB3exL{mf+&RJqL!0->Y1=)(5n$%XQi+J&&9ikC3<)7_CV_KX~K+Z%jS~ z&u4A#>>xDxe~WsrgKJ2209P6KScK~qU^4_?@FWQSI$fC=HOp+S0_Cvb=pZ;?eX1nIVf_wCvag|Kl3_PY)%tg?KW3K>227lMp@-B%Q5(_S+A-U9^dmqeRfJPnUhZpLZ*6h9xTeP&b zHNZW|_DEh(08u=+d}@^l4rAa{w?%>W( zCH0g-3Az2dmv-itmmLN`q2!}kXng;JnZqIa5%3p$9%V#vl2CLQLD9z)L_~B2m1e-@ z2j$8+d}(nJLElRMLf=ayL;f~7nK%2cg2FZM2*jXLQ`K@HN)DB*ATB0G4n7e78W~zz zS{vx@RV){J)6WqTMA&|k|e^0o*Sh91=k%8pnZg80FPq|-l&ZTJ;n_z-l2JGwf92#0v$%q%Ib@$Roy8B74`P@ z6+o0y%j|L#I2FN5i*mitEuIf_#=N|0Jf2#RmX3Sa3mA87(yp1J{uuAAI$q;mkt@wx zGm+K~1TkTtPQZz1a8O#~3SZB5FLo=w->muM2ybs{XLIxWUTy9939Qr{oUgbWE`z-A z`S)C`#{=Omr2v`7bJZK*fwtd_K2`uDerQSI-*c(heU&gO@5o9J`LVFEoi*u9upK}$ zNKOtNq1Dv*@0i92Lj{??ENEsmtXsOkhtd35?kJcs#CkGUh64mjGN53W%uA;A5 zq>=bg$53G3X!YSCJN%qrN{SwbG&n9Z9CRX7P_;m%u~aw%;o)lw_-flwAvzMj{c0vacyZ zNij*XW(%dwE~!M8jFP32Bq<_99!Zm}lCo4nvM&+-=N>)p`}=P`KJWW{o+V@E{$BUF z&UMbY1jc5BG^rG0aQ4&=R8DS%(M6CTX={i4i){JBgC(snd4;~7nuqAzkaxxVMsOC>#%LN+b-1*x?4q%S zMb}^g02@jm!bf!gtqExk$Ur{=r#1eFiI7laZBvd6kB+vNoMDw6YrdV9b_h=$A@MUZ zb8@)RA*5$z0Utnfj-)Tdv8%~;3Vos0RpI^`Ku@Vgkcx_mFs9j^eaZ|LrS01x2{Ue@E_qv5hZ`0vDwdR% zo+5MKTGiDN%hua#uZjy!G68{$TI#(2z}{HT_ecI)VK+iu%_?Qsh=3YjzGd=&(!4S< zMVoH)_74v3+<986y?t(U+{z1k4_KdrB|1ZlmzyEo&!NIR(CxRso zKItP?16?W3v|Ho4_3N?XTgo;wBo{fNSFT-yKc0o<=gI9jBv`s#tDO)|Lp&qfY4sE? z`NKyY9n}vXzIBd_FO0Jq=&m#DoJZ{;rdI#{(P`&JKofCqB0|v9ycBR8~kx^6uohmPUJNAj*oXlF+P4= zmsyFNSA?F>&8U=Rae5G%`wPGV$Tjc9Hy^kH5pw`?37@pirAxsTbwoNb-ftT_0g=Z8 z2s(sWCA%@!TZpJ94J5qLe%+VCR#;N!Q18+7RL+wkS1|%8oCn%>A{#^6lLHqs&grc| z)yTp%p7ZG1FRIUiROq7KHffD>pd%5$*kqT8DBbnum4DrtcT5%(aJW4A+yfb7hn~y( zIaw=7q3zyV9EuyCM7ebDOSRA02hjGmv2iQ?8~$lZU01?VI3fj9Eyjk2zeXr*xpML1 z!mYsCo^tsw!iQC70MTwEUf7}o-p|P~rr6g}fMSmvxlKd+ zj;C^Sd(-5^gnD*UD(sDf$kxD*e}+D0Ne`Sh7#%Ka03~@g7E|MQF9;O?Xic=z1#dJD6&>hHnk2RwVae;P5tz18pjME~0 z)vH&#F>%;-jJ!}hSgq(*W*IQI{Oi~G8T!BRLBh&KKFxiciJh6Kkh?YSaIl8Y1s(Zudg?o zH-A2AwB)mC-c@dHlJu(eDk?Vh9D{fUpcy$68ykBrxeHPE)+`2Kn& ze}tCtaopfW>wFe(;bt5bW1FioHEhb{SV2r1He6~xET@>A>U(SExfCF7t#f_{46*c& zbxyHTSqR)kT{ky_VD9}huZ^(Eu1COuXb#lhptU;p$;R4MU6$ZM%;ak5_c;h(aO2 zfcT5IB0=oF1z~XMIDsg^p@`Dp1i(5=vQ?IG>q|fn2n@_mY`~{JH#Mz5bD|D{w^Lan zK8+w@oak6zH#a$yq4F>Obk&C#bAe1JkHQb3Zk^j$MSGu$;H}PjlG;PQsvZ#5$^(G_ zN&Cs)i&x+FY&Fg+gD-Wcl^unDE!+3XnD5Cu{FV<443H&FO$kpP_%l=PQVx{&UhN$#Po~|?uJ8lcDss_(B&{Z6){xc zns-J)j+b%n2kv+upMvzq$cm7@YT9>AxyWWClZm}@E;#K52z;xj9@>O>t-zWXCQ#H+ z#MJ;QM>7w`I}v&Lk&MQsz+w~0m>12>lNm!$%$R}iIa_1T+rTW79X9x-S+~w>&O5dO zIDVrVipCix6$?jWq4}k>#CPvZFcskEyy*QV{`6@UIBr&BY{w%ZCUAhvUulyLFC~8OAG|5JtPh4$v6eayYF&-#W3>mFv zNFscne^U-21#x*TwX>;2|EvrHOPMzAFx{&9;X289m_;mbP zWfX`WabJPk?Q{n9$Po!>XI`?~;r$3si$mQ+f^Mzi{kwO|x6~?cv2%Wn-;``<*Q}|k z#V8av-)>+{`_23ubXRxxEr6g=>|D{!Na}`fz*uZS?uQMw5Vt(Cl)=SHpu0maM#2Dq z)ZM#-&sEXn(0b63#x1RXC0hEBKfF4LK42-CGyT;0udk5a2%rf?6H@UBl3I4IRzM+B4 zEaoI7CGp&LO-9rwd7jwO}CQ}s*DXR+WvhT8B~BY%v=@`QxBG6&WVk#I!x5lmX`6=QwS)Z z{W6)QNL?n$;v`C*^leT6NGdh8qlTQ+%WT8%+N+j3L*2AnM@JpeZEb$yfY?wmO9ROmas;y)M5U!^@4tac*OTN8*Gm1tC?2JKDh~PXjg#=e@}4a3 zzNK2J40v*IR`Fjb6${gTc6C*0KWGuhln_6PoN~*PC$~NaA)TwHOdcExkDy>!-5ta2 zmB8*;GOdDr1~TZ7QtP@1LzJ-H>y5?wjT@n>TL&=$sjAnei%(z_DkCHFXNS0GK7RSP zI7@ccQ&M-N<`S}%tT-4RijIjn8*Af>2q8+`$d_wOMHd%iH`vpgj~>-4Iy*Zj4Ogmx z^QYPcfJv^&GbgLI=*o2caE)rCbxnlZMGjg6w4Ho3fB84`TqP@W|TLNb<7Q3?2G z>yMwRwhsUvySv+|rBsR}XfA9YTTa|bOR$lGS<4Ro3pisG{sPYN&sWYMDJ-NgnA(%LpP7r~XW(o3TK6oojUoSug+ zMYdK3gFv#B#-u8P?dXJDoE}*J6%FY1&f(&jt)+YR?+-Ino#I0+L1VOF!67t8XQ!vY z8OTu2Z-^I~sBgr6N z2QNzP;#)M|#MBgx^hMkdv0A#nwqDb|c~LaB8ewSjH*U=4mN}98;hZORTZz5dPYxDu(R)=BhVsw@(@<}^KKsS5ac7|%AFlz zy-GyYZtLrddi;2+!#z;KCJNtA=7W&Jt}CNq#m%_H5lVTL46xd}y4Z2aI-YE(ub+o# zK0;nP*E`DVNyfN;3T)R9ZEbU9;J<7|dhQ!b^yVl)VT$hGcLR)OO*ivIXo(r~Ap8k; z`S`>HF2*v8XKvsAAvKloRJkdf-1-#s0Z^QVI^v2o%#+2@!oPhbHg@|l-2F}mb*UZq zv8ouKCf$;mB~$SRws~P;;U7Ic?u|-HN|5`nx0V)-e@Cc#Kpf*5AQ*h)SDBXOg44y`Jf@+xAhKqb|6M2Tcy4gfIdUISIAg(-6JoohXW2GIs$791waCT zc_m{3jKEk`is3E6kQ%$`g;`ZLvdtx*j$HCP6y_9@j893i7*=MImdd;TW_7}8F#A-5 zBND;q!)IQ7!dOWI^+VB|Ot-X`$XfD=Bbsyap6L0dMD|M@w#;{h0@%( z=FP#%6%6)Z!XLU>O&Do0v9n_dWNR3{7`G~>A$xt--}JYtUj0v1En{B)Yy$&SL@Lun zrx4W!qWBp_8KK25pZiZ5VRI<(oN^*3NuT>bc8*#l#N$Rc{3&i)b9sF9f&HZv@6=*0w? z1Yx6SN$4@}fM22@JF&L#sckV#o)|8pso#g^cQ-=xAr)cbR|t z5iynxi=2Z)Xoo?!(K@Lwx8I+9*77ak4u)l93}6OCCOfH2PHfThzsHs6U%2MkYFUS= zQF(Q_<}tzOemF_-v`OetPD146TO^E1|K_8&b@OiOmNMiRVZzV!c1G)u1_w)~pfEn8 zq(&i?+tIQ6dQ(eavPpJRlusg+x}^r95JppzR3@fN7fvGBatDFRBv$b^ku2es3jZtV>2V!BuXcx372cpM@!gt~9A95w_;l7}%2+&{2Xz`z zg`=ZH7o2=*FoYJ6uuFNzjvP#*kP6OO5D82}P>3UX10+Uy95W~VwCwing3wdYEfFHl zr12M0ztaCdX%kZg)Geo($mgWu=fy0Ti@!uPhQw}7@g107TU$rq;zWo>e>9qCtR*@G z@{RX;Xk=GHDdaBt^0Q-iWx-hDe*2{auX z7>C3*m6w-e1Mnius$arP+v>9IQ@nOo!&6UxUTl%WsQ5)~?TPnC=Bu54dFTv-=HY{G zG8REJL2KALrguQShd1+?p3*k{+a~OZ2A!v9c^^e8V{)R~XgDWMEl-!7c>X&r*B%w2 zyM8UU@|r$9NbZ2I=8(KR@gAVN|H14H`luIQz8pY%wI17F0II+zcs7EP85peIsCzw> ziC&4kGV(@zytp1lm&P;NRU4LvkrQj0VTL~J1s?JQth05*r=Z_eVM|eYZJ$dxC8%Mj zV#_cI0KD2(i6(^T5}-6Zm$9Jn-MhKZT4XaZ4-0w6#n9UWm4Lv-3l~aB_tS(-MG*ln zFR#?1B7dz_QIhl!Pi+1~`yA?6RaFI?xf&NLaN4#?BpSXuFG62?56PcQE*=nzNeSQ= zJ)4o8jl~{!H0A&H2W|zYUib7B8)h{1R2CY*PnH(Q+H>>f8eCJDTIoADNFjx473tai z(b1CJWWjKpTQf3Ht&Nut>wJIo9|W*g`1KWC!~oye8o48A(nuB=wJKP$5oUmYlT#l( zqOhZ!F%FBc&BYuz`s&sBSqqcp5ye1sbTscQXO8nrb7Hdj0)J1gjOnvxGHYaTd`-Qq zA;S7Is9qa{GUxf9ZKHc-V{Fl1d3PpyCDe|Lj*3c2Ev!`@dVM;31R;>H3yl97K%8OHzs75>X{Ne9k~JyKD?E&A)gkcnq*+} z>A!?(6KTLCzpy_guUkY!jiP9Wjbzt09*D%dtg~X4^|!~9h;mJroJ&qtk#8QTt83VY z8ZUp0%FFf{j&O8Tkomgy?KCql2PgoPjP2z{uCkf%7i9w`eLGw#udCI=@?xB*F+4dIxlTk7e(zTUW! zi;E@EU|}lb$K8g;#?mie0`+<%LRV%K z@lVH1v^(J#9>H@lSB>jS@Dsl=XQ*cjzU`RkGX;QndZY-o7AmMjnh=2mmvc4?AWbUK z3#v7V%w1Yq0)&1k#rP=#G=h!V|x-_-KQ*g(U z;Q$WqSy?-*zKn|t!JMam0&*JWSG*fJu(b9a-L*`%!1Ut3xfx1saU21T^DSSWWYVVm z`0KBMg*R1Zmot@vOHQa=v1}Q1I*i*QqcOPo2Cz3AT2mp2hrs%LNQWT9NP| zIT3v-jJ$%NY2R3yn=gNk+@AhU;HY4l06wE&gg&>F8~|7$kZ6U@T*Ao(1U&!YZUcdj zjbE=kyD5~YkOIOU83Cfyb-$R5D1AC+?%0X zo)G87>fZ|pxD^l;h`@NNzOhlMl8k!2ycB?1ZZDQU4ba4RGSew3N*_208*A)>N4OB| zEbqXkN~l(-#qQz*z&qv2JiEK?>{4mPOIZl_|8m3t0}HyVgToyXN3D~Uk>Pr@G5^%> zL6PiK{DC1-pjjdBi>>{EU2f;)dKCEQ_(YPO-0xV>P8aNYx zJvqBEt7=CN6cVrkYull-kg~;5iRb9{%i#5_ME)4n6U_k>5JL&VHQQi2*olu3&)))T$u(O6_y!cyDucM?-HDI5i@&u_U(Y zFae=te;|-aYI^#sZ)qQ*-#O+Pjwtbfp6eeS6?UyF!8jFn8Hn)M${5d>s&jx_vFx8J zbWs~nhpi=DR|V@yp2ezcL>sFl35^9B+%s1liI9N0b+@iA*6&p&=1q?EbpQ)F@er^} z(Ffq7LlmfZA3?)ahvSWtEiht0n@HXzhkI#h3jnzic3+~+g&hJiC01~BZ!9ssB8(;J z&kDJiZVtc?TjOk@f2rn`bS< z*uETXj;4-IXn9pRlS)9thwd#tW5Up_3+-Y_Ny+AH1?&zXdfmuL1uqfgqJhI_?7LH^ zf1@gE#J7m{33yhYOEmr=!ZMlV3J~n@;lsO0g8Bk?^e6}rF*OSd4t6;v_UnwmlL=T# zk58fGs^R=sD}Du*h87`_q^%)@7`De14e&Z3bax|Emqa)~^TriHcf~aj{&Yo9+EO?h zu=x&w=I_20U?f1q)aWaC#KkUaAqk!&4QEP7N|u6wBhe}1icoA58fxRl6D0*n#6BUj z|1l=q5j6ka-d@>B8$fi3<2-b2aA26ZoC`UHPGij1&~$JAMUvDGVWA{%?uT&n4j3F@ z=*{s&PhFiDT$Z9=C$@DH(!{}E@ok1fx?6}G2?1~hEX{#)Ki!0A2>{<8Tnceh^;Kf!2|Tkt*;g0q5Pf|__I zLSIi`L`LGCJ=jHbY3uuWx;i=}GY-?HUUJ<-%QhE$4yLVj$#@i?8pJc4n1@;8>tt16 zVnosot6pdbl%vw8ujojixLkxP^qmFidO@{@K$6HF!#KS8?a6bM)gfsMu=^9oHiaph zUAvaR5Xpwl8T@gKZQ|loq8KoRKO(dO-I?O<=ABZ{`CS2=grg0`0fibZ*}0G&8Ul1&mx;B1Qwi6UKUIMQTN5+zPjsTYySXE#h z9HTcPKryi9x3aRG~GQjCj2_Gyw;r1aCec{&ABJt{k(J(P-7s`KxxT};o zDJd!M&ikrEehQS6Zcl)9%(@z4s1g`aUwg{eE5qGM!HMGjai{5galf!^OV1cEUL=g7{Bpz?Am_GoZ3PUG?!sFF$!9z8ba zATTS`LT){NJA71^uz-ns>dCCR|An~>zxwcJnVC9fV3#1z$BN1hF=IK1=6wb~srEnc z5zL;tlaU4~7U2eTV+>fxfD*jv+yX0EH965Pw25G%&CKxVxBuP*opd**d}PBiNuqMc zrHdGrrt~OVk`hJS5&NH?QiPhTf(Nnpw2z>wfU=mgKa%o+2X1g$hrSZqvpDf=Vp4;# z7uLv@k?ao@fw0sL5djB+V1=C!y-L4Bq^5RZFGY=c;&s3+<9_MK*;hOU zbv4%1E+En=>rL@|6DM|21eU07;nEj@R1fAC4L`qUOlb@VIGyX=In&7(m6erS`}#Hl z;>L$0DVuXB7d}6N`^MQKp9x`~{Mdmahn1AA*G71`aUZ3mr|&k~k0r8*DW`^fss#&+ zB_^B~%Pe|+{yeOqB`OdN9QDj945|lnnARh_Z&nBrjp(mrV=kUCoIP}0=Y z)Z3?qTLI&N`lhBZN4y*F&aIRHIiHvq0nJBfH+&JLVNOjNKj&`=Aohf(-#vp zo%h0dx2rFZ%v?yq!yf{hvnkj2hBk+kjb^A zq*nW3UfmR-rXlNFq*LhY0|zdpYTk@oLVUKd;RMu9uS$yZr$Vgb?0`j==!jiu4?t?A5VjBBJs4 zQNCNAxL01q@+~?j3aNzqfyc4I-0?8Ids%(AwrNgSl5J zCY2I8TH=8BlHi@g97&|_Cm|qXCjR*GRs^EWC(50`;XqGf6pBZE&$4-Q$a@?Y1wnXWK}z!{2Q$vREovE4a)(z zOTMX)&FWy9hkTF3H{LZTxz3mtzf|kYg=qr=dc@tB+-z|CuiT1JZU^@YJ!Fd+F|nW} zEPS$%Y+9$xU;9mNp6|N;n@zHwd;kd>cc-2ue6(YWk1 z_5Bx$#Z-aZBHs;;i3gTa8oNBe?K!=L-HkfT2LS+7uvCR{1c0rlKm@a4y zzcZ1lq~wLkcxs0rq!HPB;Lw!)~<;SF$_0Fj%wBV5fbT`jj!mTr{WIMCvUV^{O)`PlAFyjbnJ4 zxFVXHn;8uaFWKQ>eU0HgsETy5XqSk)uF(GA+71=Cb`r)z7&X%X$Czna$c7VMUZ9-ipCliA>iDyPMqbG4z}BNKf5 zr|MkuVmopuTJ|1Jgi)hG1LqZ>GTDPee;i(CIR zSMS@^iRd|*`-j{wW33p;yac%O&KxttZa*lZ;8{R#i|(iGA;$@hn*-3u-!#b}v130U z2CX5q6^+6<4t)SREYM0IKB`ai7KFo63JLgGfCFxsY$ItLQ_^Q`|M9-8%r2prm08#1<7;P6PuRcO zMF0|}SFu3@pn;x4CSL4F79#9x&m7diPb$WbUK~6PcDL6Q>?;w|JNvIDx@GigP=**E zhct=j_R0f9ze@bxl5E`iD?u%I#tjF;v`*Fqk6Cx1P|O~Tax{qn#+>$4ByX(SpkRkVkAe~Baorr(8DT9Pp$uoi&D zE0q0LOklr(c}V%o{yn52ey-&3nwkaQ;n{1uGC&PtpP1+b7HwT!4GW8v=q7R@8{JIR zL{yiN@`zv0h%Z@-f)9+mXT=J;$0)-=JdzlC%_Hi+>&od zHj4=(+S|LTcH96j8G;KaLj)Lu(A;7z7POK|i3Y=-GX$~LjgqVc+Cyx|dP&GEJG5Nr z+~2&al|{6Y7W6;>8;N-d2|^pJo?PNh<%&p!po84I#1+eIsi^nN!z$p>MtgyXTROb= zxSUB6LfKMPk4DHyj4Ulfk&6V0>#h1g`kue4f-tC-iQ2Xj;!v%!pX_01$P4VXhp0| z_KJw@+qaXU8qqVoUd45&75LpG z8DbQhfZsU$=_W<}BRFB*2n$*jvNJOuV)j}d{LCn+<=$tXrSO8Qv59`e2 z$gE?m#S89l=}ZVqhyxEhYstY~4HkA*0<)|py5S4J3b_Ko2(s&u+n#(;ev=U`qnuy+ zFA_NQWCarmXbzkf!rx~b5DM_iV$g`OM@Z!q?#|=KHSwDRQB)*|C@BeGXof^B4N&M9 zRJ43938B$(WC&wR0<{|PUSf-k*7)z=OU4B*md#<}RcYVh>>UD2DMw#>gV(}0XdZsX zAhsxJ9GwJjHtZ)o`-sX7_u?>3l=VfWG~o0 z9>gb9CAwx|NW6(1Bd#qt2i&1c+lY0T@>{pU{-b54YhZxYZ86e}aR@S&MeOqJ<>ju? z1C^o)IdJ-8%q*p^7ABv^Ln$pSy;D>3eERa`%P)k7hr9ZJA!9(x8Gn7$5Gx1<32>VA z6LMsu+a|xf;tH~HW7&FhLUsP=g0m;I0;3CN@m#{_#~qFAG19Tbb`owJ8EQ#QkKToQ z!5qjWclRTmExQ&q`um3Ei=kSv8XcIk6Lk*nEjl8*q-WotMU`#3DlRE`2*X1=OD1i# zhHWnScc;zY#LF*6P(s4vCR?89>mWLejkz%>xBivv55j3KK>wK+2d@**x^iTAxV1O` zixw=x%v^*f8^0w?A%hsPo)O%IsuP?1TWVJOf@CX^|0)V-gG5tE57~VAU)suLBYApg zp^HY%)HzE;Jh^~c^*Gni&PnYAmE!`j91)^gYzH4kK4Bn>%?{YjxAV}U_Ula=;_?8t zaO|}M*haXUSmGJ%sEi^$VAkhavav8tF9zi==DYMit=1rax^wP_Poq;yUjBatU2Qev zc(lX|&_BT>U7~m&i?cBCa`+nX4=7e_Itjv@i7A-Doqbk41y*jxa6_Od9emJ z8uhDk3-v_ zX@{1RNtoOJVXj5Qk5W!Dpt3q1J?i>F3;PQXei|LZ`#ns|uz$u~jcdT1fQ?+~Ue?eC zRpHaZY*2(vV>n3$OxRjv(>ECB|B&b^fl6E5X7)gU4~$1l1N1vXkXb+i6zmA~Jgnsc zHlU@67lA2iEr49OBhDddt4@Ey@f5sT&ZCbC#{6;JKupc}pnUGhanE6S`t&I??)-ICH@4h29SB@CXU-f{SGSG-5x{i7^l?@e%8WA@ z36sumZrxuNfLlL+Nm)&6)nPb`FUb|ZUXp*NZ>&?maVBYz@{c^-Re0z>E*du5xpwu3 z2rS>>ASxR~KepYqKr#ikz7m|39ZWSfadFvXes|v)sCtM5a4u#Dpp!mdZ^|}XhjX%} zU6O%Di*#AIG+1%Tw7KZLmXMxRalO6aoGi{96rj(-xWEchZVUoiBc9@{emx1^9h~u9 z_fKH1zkj!%huU^sM*uGdju4{A%>BFX_iwgQ-CoGBNvsQcpZO3Q!hIJn3(Sj|7s$h0 zxWSN2Y0AJgzRByC6_j+97Gw!9*jK5K31=owXkgk!)effJB@ZdgU7b%L#rEp^eAS8P z2^k7t2rG^utQk}La{fHDyph#@h|bsv)G?X`4hxoO1=+q`)Vlq+ddQeD55;e1M#A&_ zPa~?%&Uu!$cy`EsPrR^j^kqPOKYX>%tDk{e9oii@vkz~*K{)r=W6fTWowWD-sD#u} zgMuBHtHzk;my82CnEQY@AlqEV%xCoZh3kc>9F*i{TmeBYN=6C=-;&JHXUfI(V|z>R z{q3j3AtyoI{Ig$g-p$$3AKeoAVY78tnPFziAjt@8f|wHG^!p8aR67p(8OwXZLw@W* zrxDy7_6t(tQ2hjsm#u#L`Lk>$z-8g|L=T9A7?N&AW>5bSzo3nSa7%REI>)&&|2eg*{*O}| z+qHfFtjl1NZq`>AU+AtB-R)1KYQT?Ue#E73)x7)$5assh3cQSrEe^8gowQxqEPSw;8``cDJnAR<#BTV z&8}O&c5U>}PcHv*0aa=+AtEno&z`GH_o{Or_yum<*BDAG9fQsgg_Qr%t<}d|FY!?a=S?4Hr=oI^u@|RAZyOJa9%?mM*Sho zmGHq3y}0mfM%LM55MD#2CG}KtzPI;%4@f+i5v34mtzZhNnci8$$svERSbL!RBIP)? zV_iwvu|Z7qEr5s1kN0`uUImH|CPwfw;OUl8VRB0MfF$G;R{kNE{}JPuf3*T^C`yoL zL02dGbM#DSS!7rk0$ByUW!Skx&{A|P4IF2M+=iJCL(psg$jDVm{%ApY3WZ3NWt-Ym ztB~m;CdP^pULjocvtKlR<{^Eq)%>@#F8x)-EUmWgMao7W9hp5BqDXCoQv+(yGr1y6 z&v?sE%-wHz72`re5O>Y>H<#XO$H6$TJ>1t<3e^l&JtMP z7{w~aHTTf7;3Zo-zZC@g3wN8l3X^0#Yz>=2ic$EG7v|Q~m}rphdU_mSITDFk3v{_T z_$!b|&;pll?A~RGa#*-!UE|DGtiYT7ma{HlPk- zVo%0ck`z|%8Y>iB&!a{N>)L+54+>|Z=vJy^l~2SrGy~kkaCB1hP@CZW0he_MTUZbT z5Ekj>>EUq@b%rMm^1E`R{UG~}xv-chOo$D;I~#`{*qf^X?!d6uuV1rQznxL?*m@QC zwE&QV7!YUk;8?fB=@Frp)#EXIbY$r6lPu1yuT9MAAEqmqo){X)qn0v69QV0#nxpLp zIQz@Qe>H^wmE?0dV!V0Yq^e6k1eP-VbsH8W3-t#ifh9Zy7dYykd-Eh-bt~9@dOQjU z^dny%k733gpNU72n7kRvVJw$mae*357ZAGQ#{b)L5cJ@|12TDyu_k12z~php4h|M1 zgq$WYmhiR!22lXyYcN}cko&AGFa8-4dpE#+AQ$KKKY)St?CDKPL;r;@ea^zOmvOLZ zi#pCI;^vbdTl+Zo{~dD*6j)o za>=v87H8;Df*_J*3R&Jb<{6ZD&4J(@-+20>Xg;Vxf32}&GhdqwRUm`&Im-tUg@VI% z7GBHFCzynSPS|pi=QIc>7|X-rJHF4Z1LXajogRaTFH7f`yodMzOK?EDzUiz z2R;VkXAQs|N6M<(aR%w(uU(S_cosN;$3g?jC%8AZw+?~fA>rR(*7lFp;BBFS<8D4t z906q$GqNkYX!PTwIInB=X;8C(fn5M@ZrtVPn;Y>d@-a+R2yU5H~w#r zrRdVQl<=HuU+=noI$I}_GfSge8e_G{WJ}J~4R!i&3IWOCH3*YzQ}*oU!?Ex`mdDO5p$cD4)mh|P-h--_@8r+7C#lO>=CH76Q`OAgqv7tFEi7ft&3@EsgyO*d z_<}}xs=Oexxp&lcno}6S+w*8Eg&+3sd*+LYo|{A7JgWiJ5L}3$8W?rJbEhruU3huO zl|)>Sz2x8qcA*O5(jXD$xsBj(agqu54XJmX89}d~;ASCGDR^W4O(-;N0aV2%i@9iu z>nw;%6CS#4#v>eqUppmO{sbCQULuJNq+J{Ohc}km+U!W{kxNmW2%N^e3qeG@@n?eV zCD9pJSgg&&=92KqNzxnt>wE#Ed_ZH+T?0>u#tZ*?ZqrvdS+!H-EKaUQ`%LI>kmeyhN+0wwwE zQk9(~T0@whKNu(enU#hHm9mGZ22i7cV`EDbVE8 zpZRIGZ>yzvKUGvmNg6x~hYbll$I*%H#6c#KLAZz3D?e2+27`HmY>?Z}1&7IG$(F&M^xs-;RkU+#*7e1I2IQT@VuP~6F2jxBWzhL~MAq1ELE8>mPu5yGeiP;xPFuqq z|5)v$j6k>?EWnMPUl2V!??tbT+!I*WbT??;3ajO%Xm!hHKNu zkCsUGq<#8wBRK6M-XJWn)FiysHR+o2p5knTw5=u2ixVQE&Vcjk+$R}hi@NnE{P&DQ z-<#riogvRNs&P{Gar5N2ZDL!rt)sk^?e-pX3-hK=4)^^0?z#U8^mt=EHoD`^v2KA! znOPeg=D&p9X&$%BJgJ?B4R)(v_7CiTazo(-&$2;#Z)~ORetz(jG`F$1Dc^itRq8d)`MNq{gOgJz z#eqh43{0{}nij}Lw;(vkPeU5lW}^+ZgHY6Z)L}#X(Le`?Xgl@R9UM3bE+|*Dr_SNi zRD?#G^lK!JT|+~Iq;MJG?kWKScKDNr zPhGSI7|I1OJY_kI03>>LLD`M$54)*tl|3Wvf6cDS!nIc>0j}o(&!CTU46frgzU4jm zmHqid_xkXtsKp}$xRT#sAJ$257ZFV)@;VUyRN6VDUl0Y>8x#|9iwjHf&K7d+8W!eI zV%3SiRAi!2?w==IW>hW(9?E&9-aFNxvvkP!yruHJ>X7kB;TV}^DyL43mg+J zu$Qz+)}tluzygNFH!NSTiO^yNkBgB3CXai6&CreSm$h+iy;Jx5U1duDA+^a8&E5k@ zVMONW3w#4`{PuR!&+c4nipMk`*{r950X8E zeEy)nA&a>w9ctMF(GU>{0d3N_yoEa43dAK+Xe7M{JO~g>Rvx7BxI!ZrAAQM^-6Z&E z_k`1ia06-zNrA=d%%4mi@`7|b5}p6W4H{5sZTc>MRCLkRs>-~QA^Qz5+%Ks**4%rt zt?<}E(*vS()pu{-t_iPu_YS36RQss3l0#$6 z{Mm5;SV7qCw-UPBrM_<iyW`Y2*MY#PJeOJ- z7LrVhAE;RDU|PO%_3BDge)7rb;j18uK`TeE1s=YlxC1Q^^J_~msmb=0&9jxWJ*hynNn;3m4K_ zb^~bPIA$PGZdpalG4rgeVGVdr^lbs0VU+2goMK5@Ed?H*WF^ET=q1713-MSALh0{W zKXI9Zw&GjOSL&8m2R9vrqUuLq-`->Jw~|ztax*I5lDUBgE_N3#(98xek&+SH+>88< z&fAnrx89=hR+A5qS`&H-BMGFQ!pfX!i6E@fH2oXuC#|@(b!GGgkuQH{wi8by~b&)9TWg20u2j=RWV|=>02WjpH}kyBZ6efwZ{9scx`A9 z)Rrk38QST<=%mEF8{+0Jx`)dekuYo|!dCle@bpoSVeuY~b{ zLoXV(V9^&)m<`k!s;U|DrA$vx_K_$J6DZ=3PEAb_-4l|lw|pDI=o$?b0Fxd|7UeT@KTUm5?FMyVI+8V|L@)bbCUgaRf#+UcF>=kZa6=Kw9X@U37L_sA0Ak?Po( zvtpa*BCLv5vMTaj292@lG0dS$U|%iR=W`B63aMZCuKW0*4n;0e~d+qEiKnc@f**h9X-lEk#A~Y!HztQwKxGqGEtRhQ+Guz8gh|p#(q*!E$T&ddQguvk0yI;^?4lL z4t(bQlXq;Ip2G8vB8EjfR?Pdm0Al#(P!-(t)Ii8gu%XKDb7Sj`%KPEDfDpmoe@B8r z=aorBNZ!T$fk`ctO-o(XnRpgg<$ETg-Etyr0@ea%O?1W6NkR#E!w>%{i7X&C0`T7eU=tsN~Y8*L49{pUp0ww!Ej9XaE;#%G0NohvedA z@J^x+0C#)6aT=eI!5v=CiyH4Gn|Ri)6gi(|M7YP|Dt@?tRsgiucd<6@K7hK-b8#Yk)@4sPr3{3-6qwnh@XbZ5zsul30 z`$>NO-a`VSq08-aE%={V$!cp$$CJ{Ie`drtpJmhL&1cYNyUpp5UNQdLb`$@h2|pfV zN*u5;*4LcHurz}>5D?#cu<^k52WJYkQq_EYC+-%@avP&}+60*ztn=(2dH^P1Q&^2_ z&-57X#a$RMzLMiI{<>07Py^7{h>SajC*D7ODT}#B`c?|1ykj?=|C@)BFh4_&3~}FLl=#?B?Jtc9yj-1zN`#J^xMroR$NFB zH81Y^Z6iwm&_49r1v#TB=7UIHX+_T$)_l zSq4k)c}wI0k?3aTA-xA>*uk0UsnS2qD=;c~0LIQjHIOd7C}6Tz!CP54W%}2%z6C9Y zsQG-lr&O@?t8F0%C+DIORARibyOEx;^q31lfOn>l#u^ z@nsy&oXf_^$NcTSBHvr?fJ!`AT1Q4_6aDwfjgV>8T+w8&SUG8dTaX0M00^fyYH4Ya zWspvgxK`L)lolX78S=~2d2<1UNL-pz0J3#SK2hCMc3opWlngwF+k_%-$~^exq12%D zHdou2FFYb*zKqasImic)LH+)&d5>;o;Q&f#sW!9<0W(uxolOF?1b??N(ke$Vdu zjGIH|aFIFH>r?p_Aond4I6CDT)^qyb2Mu^>lL5BZV84NMDaVo}w%*?R(&6KI#m;&Y z%r=s%7iAVBQ2;WE*h{#fzeCiuK0OD?03oDs&KOLi5+in&s~hEX{AX@$(Dzc_XsG+y z&z?Q2elIgKbS#XETXPK_hgs_uPe8HikKZ_EBwlKH*pbFu7-N^aimF7 zRCLi^G*-wBR&&qIefaPJMzjG-#X?=kHlI`Dn;@@EI|aT>7+S6}jAnoLB?hP6#^_4* zPybn$>;?U&mOO-URxR5>SA5Zlfa29y+JV8x;-{ROEkIvDU2+MLw57~!Gy*lt2?wy{ zm?SY{p<)UpL}-vFWB&3A!IU}{70r`8qV!jH>Xc`!618!_Ug82he_?lLZE__5<)wrK z<h z1fmew-)haGlDrg83%0z(HBRFx$NU3u3n$ZC=!wO>#!Ctd(H6YtK{E)9sq8zv`(9pt zz8w_1Jf>B#Kv85(M{$7lJ?p0)?1a=KQN)L-4E&KA{C>*9_;d$^IJk?JK?)eoDwxMp zlmTrL`7f-Enqf4-GFnWGFDHcChc(N5e$PMtTb&wx53U7rJS$P`!5mOYgYewB_0WSe zUtsLBKik~9{QC=*OYa^uI_EF$0QP%gB`fTJ8KLEaaPETCo^7_r2xp7yl zLPu1!ScCdb*=KMbc2sbaSQy-Ue8%xV8C@k(k^jnUFkhVK!!+(GYjrd$`uY*S? zu`mb)y>ca7S`PIJbxAAzh=YTeg2G-C+?&3pfGXqPVh@m+!Oc!QMQEiVL=(6yZx3lX z@nSNYaI`jWCXV42JD-bQza3IVbuuD@Plg>CPq)md!;i_o-d@g*WCerYg@`XiS3`mY z_#>QlW9saVuJ{k6i`~BLSWNen4}~{nxWJhjNHyj|PnwE#F^0)1{#^kW@ggFk6op3P zKe-seHV27{C2ou0Y_*gg9)8G5BvXv$E!~s{l`CDmJk$yL40ec8C`rHZVg5R(6~PV- z^vz87b<=HezIH_kzXT;G$2_p`ACsU2_%5>YkOI^Os}yeGE7!y6aflZJRn_3AX=(X! z{n*&#l#5a#TxF%p_xWw+bW&2@uPnS2+`}V5El8tq&*K{?bY#E`h7?>{+n=3)Qjl(d zGAg{o_f{lSb^}i2?$Wf`9euy~iMfL)yzV z%-cn+586!{`8VD95u1<@4&IAx$!U^<3HSNKXuB)ukxJhmmeYoUFBgiRIfQ}1hg5HA zYFc+!!SyGpq);N{PWV!(ECdkSw&7*2zunaYY5=0#)}i0^WQ2s@25lh!P36cp+U@!>FkTsu;J;Fj%gzj^l!L9Dc>>8;2yyelO83~*C>OdPE zKYq{U%kW~7_MbdGeCMW5CYA3ejvDF$w<9D`A-^Z<(f;VeSTsEl705O=T|`(77!FB{ zib&P$EuVkW?;QBnnvN}^qM=7I7%PR6>s-7|b|qnC0a3Lxf`1%G?@fYUA>nX(YjtG> zvN9e320pkRi$|Y`7hs4rYGU}>0;PD*OpO(slsGSgQV#hSXxc?%4fFoN1|tZ!Hw8}r z;Ywq!7$c8mn+@K+ zbqkI*KFs_~kD(^Lz?k{{-HWsZxX~eUXvgv!$9ov%qV!*7p2^nIQeIwOG$62YT?W!z zi<~zQ(Ew5`B|uGJor!u~Tq^ZLt6Raj%`0T3LRCn=(G%mNg|l=aznq!(cb zMX=lVyM@a!(1$U()SHBV#_5UF;sM!w;y`!53lit z!QZFG5$l04{E?fTDDY$(jH~QA1_j6ICRP-rBMA#?P9tV%wZ=O|;#CC%1@D{+Kr4*8 zwx}zI_#-xTu3*v8(b+bJ?2$ts(Gg*NTYz+^XuT@7DqlY!bxxWGHz}se?HCnrOq5E% zB?q+w$Y$yCu3+hRsQ4YLXU2EEb=3ef@Y#RN8a)v(@$^1KaKmAG9wCFVzhXeIf#B#L z6C;olU&j2l%sRSx|KB?L=-)c}df#PQdCo=751a9TeQrPZ;Tekw=yp^qpvSzEh$;S) zJu_1L#G9WX{uhpdK)BUNiav%b7=iq=SZnwKnc4C1q_sSG*)f1iKK%wlSOy2ejt{D5 z&|r1YkPxzw>u+e!{&&5zuG*4CCT?6Nqx4mff^z)6K?@&CLb< zT-Z|XMp016Dz5ZbSq)2M4&qFzs)Fen5cPuTyA$c>=O+QzD(&Fb8)qGz^99Yb_NWVl zZUx|kIzb-09Mchc?okb7PRSvXKCIOp%bXI&hy3QcA1AQtLr z^m{9-BW~Bu+$Ic74NyJ$RQ5bDsM<%96jp)IFsY1L&C6XxO2Y8i*y21yqiY{6)i5!+ zyeYtICEyssi~2sQqK)6WbGIvO?_Q!=;?Ks!-dDnTK`j^#AhNKyO?0V zP1#fD!Plk&cNPbRq;&xslAg)&j{@MFA`Up2SKagxWFgxWma2@ipvrO_O>1kj1%*kG zt_Jz85sz&|(g}atcC{vMkftO|8^l3T2IT%E#1PL+@}LihGmv(~{bzY^>!Y?{>2_-{ zI>l3?`;Gk3g1j&CQ>bk*+2{LboYR+~fM-qk z2U0fim694eyOhvgE6DgFM;K>O$Md~?urQkb^XNzP!TOqywF)bv^%hhl!IRv&R8c2uA z_4K^}3F_g&2OL!Q@c?L=S;}*&EXStSw3$kcCYeJxBj6*9h=&g9kg#&IHj%Y!*fhMf28S;A43?zi zA#ALpV1?9r#g#RWO6FVMnK&gve+8|{1E&viSiGQl550>myt%m#`K?>WR>7t?K0dxO za1~S}zu^4vu!_MFmbFXZg+D(KBW2in@|n+FO=?h8Qx*$xCIS*%yCSKKxgVHa$u25i z3vNl?`q!RFzfXwOq>5WWvO<;w)ngck`)9@0WX74wh}oXdfFGCmJ93H3EMJUjt1&tO zI6dB-mEGzRym$a&hBlrh&~ABDK+<2 z63ZL*lII<5=eKmD+N6mXR`A6a0zF>sy)l&<@!Au0%+MMIbg@n787S_5L%SV@ZP95DD`}%as~~ zixiPB9jenEs`Y|a5Pp&?U3IF<@NymZ`K1^P<$mD6ct5#v-doYrB)g4GK31FfxVKr1`kUTbhrBM(|$*??Wt z-1|_4@`Ll#O&pTLbPzYw_*4K(t>7!N0E;XIzZMc)s*AHna(wF{MLjhAaZ`tQqip69xc)3EpFyFP2Zr^S3YsNsfn>!33%qq9D(Ql)&xYar$#SN;p~ zcK06OBdDaoFlJuzW53Cc>_9d(HQ+znJhF0=#*ORTU^R)MzdM4IX3D(|X{`AqEAqj+ zYob8TYcwbBS&YZae(4?-Dm%E3K@HpK>rbWKn199gE_T<;a$r+Z>>^(>W2jer^EFx+z>|z=CQ!dkNuh>g!f@CN#3m@1$WeyAl-Ep9(9HMvB(XqQb zQnxY?LSI2;eMoK+WsF0uwjs5d%Xf60Vm*2Ciy>c@$36SR*QjZ~VI^Bbh*n=lA6hw; z^qCEOvD`DuuiUkd}5=Z;Gqy;pDB2HRuyXVG?{O$Y3*x?0`%eJO7!n<|MZg!;l)9zIVdN zapP8R==amcsYB?j<>AdhOGiSy4hQM%Oe&LUZK@HDIgBVgqqaG!V;eBYcu*pSi zmr8oilh%g+L@YD*=MmteRnar7rxUn^%Tkm1JA`UKWN_VnBJ*-3lSuJX8>w_VP^uu0 zt}HM@ZB3tcf0v%#=8x<#xzfiy_h-c!0Jww-HfGMU!to`}&VnAgB>;Q>eU{ zl)~;Eoa1i<3Vxye_Mq%qlqwj^YA^lCtq=p(;)p9Z;T%P8PlK=FonHMj`dV42*oL{? zowO8j>u-NrIArL6l$%7mvAvB<20M&kv&ARt?0oRrbfYa5 z&HM5f8C{IRz$zF90?Jv*a97`lsAAmM*$}1kiAnj1(H}_(TSD(S14hO6f|lW4v0>NC$xld4&nzuEZQHS9 z$IOJKENY-34k#QWD_t9HuVF`AX23irC-v@@DwWQ3+9617OY&Z3KAMt-=7J$A5$Y%4 zC-{-^*BY#V4YoM5y|Tvd z8hV5~sj0bNQ(D;2Z-q6Q^qLA1_V=R@JIW6SU9 zunk`l-(GL?%(-)GBgH1sqiAaqa(&p!YE62%zj{7P&u3cKM^{aj{`5-rlz{K3sEr## z40c}4xWtV@82H;A$LG2B3X5SHpdDCv<4XaktMXDmNtCPJqs!K^@yO--yBSr{yn;Ll zjK9_WN@4oECw5Fo!|nd6)_7gp>Cticza{oGGbbwcx8mcJ{p_3bYX|nFUy}#`dKlMZ z31t-@a0Ot}s2(^9fvvN83W>v&hV8JNPparPJhF$&y!%fOa{LT^z|A}rXU54V4?E8_ zJ$pY)C)l8VJ-O?>xOVAy9AoX@25Qv>ER|K1{C=tS$ZTa@$Ew#VkM=!!)P+F&mqe3Dizg$F9;l7WUjAU-?m1(9d6f5}gLSQ%`b%MHs@nnI8fXtO`Y5XQWOr(2c|_!3k4o6j_K0 z*E4x*Elw@0E={MrtZE7g@8cQqxoliI*|B&mxfdMJxRE=rX!GoPs14ejs?jp@^=ru^ zp_4ciOS#_1qe)|2eO7(-+&HwJ3>THbfb1lDd^|s#q};Y`t&T%LBq{9OcWvCf`P=PZ z^{kfPdAs62QYGfpZYCxnm*O+qn54_4n5b8wdL{z1rA12S)O+=6EX zxttdLiqJT-nbEuLvh2G$GzdY5lT$^S&uO-!)=KXwYLKn(H^87*o7U?upmYg1)XtP$ zaz|3{Km9;wfjU4o@LIlbjeRYY$V_7$D>4iW@%?G_Opoh-lKVY_`hY-3%Lmh=rff|A zQcK@w03+JU#7rGm3fTNBw4FovYO<~gI0*2~A}K2?E0g(aWvc78ji1AX6a=DC?seLa zva>ngWX}({&J`CIQFkK8;P!#g{F^7TllwOJJ=qKKQ0E8g#HOCv33|~?ViLqCty$m2-}J?6%8d8T zRFyH9&3Smh{4t%Rh9V2XoGwm9T0(=ZT$M=7JJr8)s_{{3KpCAh^ej}pBZTDurX}Mo z4I4Msa!vl)9aIMj=><(_x4n~K<4>Nm?W2ZUo*O>|0 z8`itG^-|LMOG)+hLb?d-$&G7rkw&r{_H1pQz!toQkp~a{bqEdF;memT2t+DJl~fR4 zXq7}aqBNh9Vny?LYaxtyMo~|VE-RX~JkhGqD@)%tuJUw)z+Weq_78qUN)1iC5fO$Hj^t|GII;_IpvW3xgUHPU+(+W?0zTQ$V zWICxtwz^i$xKD{NL* zBT&bDOP!RCWwd7f=i&$Om+QFzFVr1L!1_mr7ip%vFwH2ScHGWC1cHQIu znUem4(Z4Spo?g+7e)ME<&ApyrQ0HGXY}gPXN&PieOJ8z(16Vd3L0KN80}K zvL}ZN9+JOa36?}C*51CeJeBiRS$YC4@+&At7Wb`ZQt|Icbu6uF&?i_PGg zdZ9cOlU-)duIEaRbPeoSkqMWS_))HGqu44y#8}s(35ivjcXc%iot1PR%yZ208vWK; zXC7^~W(q*<&ed9`rAO1_JX(;J*t=u&AeHWLAa^!6wtL8AM-%Guh{jTm2B?AMQmme&?ni@k7)b)j!6X`>W<9`QC>HeW4r~jN@P&Ugy z(P_TulS0R@4e8H}ZD?+02?!)zW98QBozsmPI~p}!8MqzR)zZ*AbPTW4ZZX|cKG_q%X`9M;}0PSb+tb!>nyP9Duo@4nDIw1vB zZu#{K_{3yif+2l&_br2V+k8=!Oyg;1PoQHU%w_yB`sEBbQBhA`o1WL};Lqop#qO=t z*`oj|IG_tmjf?{D;<1GOLTXZe(JuuSmt&eQ$25p(W6`2GTkW_TKH!stOW8%AEcESX z>ic8kx<$cGqZviI4JU9%);MuSmwPs+vwKxg6p?y+&!|DF9*JKHw6hvjz_T)q?n6F$ zY#NpPigN}I9U4^d>r;As699?LBZnm^NRy4ttC-f@r?PRjAziTl*nWLIo}XDxTUsUc zC7$LJeRm9jiLcnCBaxfmOmyy%5b-&Kldu|f%X?Hfigxr~SiAO(?DFs{j!S!?p#j7N z9~aF0lO4#+=7dPhB6nN`%nKR63SQPaa<)-9aGy8_ARmj_ChVa>HhVz}_UofY@i0qS{X*f{g0qkWM5QzzyRHSz&6n0SGk5BK4mKjOJp?A4;f-TxFaa!J!k%) z(FV4+vg_8bAD;CJ{E2*UsNY^!Q?xNF7UdNGld)-&4v{oN-Te2bXqE#`%QJ5+Adm9S zmt{fI5eN@N6KV39@X(c42SHUyQakU*VMUk1S)0^wPj(HFgTm0g_da}m1eV>7U!RY9 zRq?v2$86X(_5$zzYE~?%)o13mYEXTx?eg#@xEt|8jckl-0~-vUv1)5xLK?Sj*DmXr z-)8{j3l7n?ckbTxJrX`hl{b(%E5abe1h>PYDvpSnL>TM{6oNC7y;r zA2PxFl@{mDdSPs%@^=Rb;)CKljXVCPk0#I-$Ag@Bh?e?Bi|To&SMotC4Q*2A@A&5u zyoYTg;xt>TAWanQzy1igi0@xscD-A*>N1)5F}Q5wXjaTGD9FrC+0W1SeQOd@f7z6T zd?oqA6Ni8x=k>@>@wUuE*c5R*)inp!Am_YM;8wK(YKNL~IPqU_I9as@3^Y}+%YRd( zR2eG-VmjEaX_-_x7PZw2v9mr_Fy?=4zAE9gW_@5chOATepFUo!pcgt1lfQVhOct8#@&56( zl@wLOG2Rn^wFR+&G>Ij|c?}^%UDglJtHQAfoHsSsN>80eqDUa3`^{SeoB|!92U=NG z77=dj_w1C`VjCgi0A49ftN1xAspgO&h;60Kp;aN@DC^j3iM>D2!`pZMhd?k+Bx zkVADyjTk!5WDly|t+qo(j7S77RW5p1F8l~%qW+AG+)sZ$Qk_<}cZ;ow>r^WLyKI}~ z)Qc?iZnm4yS4=B^6#FIEvIS`1gb7d9UPOP_o8;eXB>c-!-L!R;dY#%pc6Gz6G|I!) zkt?esSn)B~##(nNRi<^*XYccU6x$Ya2e==%uv>Dm;j6-*UFDi&y1p{Up*TC+V13g* zdJmyd_=M#r)!UqyG56K^FBb(Fc+XHTITT3mjCNscKvpr2jKX~%{JN55<;3+s*QH$a z3qd-7En4%82cAQh$2cdk{#^9+7c5#N3!weIM@+>RP0;lU{1K~O^l8sGQ(jt1l%!AW z-12Fu$B^GewC0xKYt*`RkLRE4qen&TG{o@ofd{wOSI}Ts;e1y0`}wY2Y%iDr>ZcmN znp1c6SINF*w9(dnEOQKxtBU=X3{5Jr-(%Yywj>8Xnzm9!tF6A;%6(L7TVuVjVMrYK z!$TbQs6>&54tr5Camxqile>&J)d^e<-${&~_PzRy^kqcR6+tP4|0GQ^_WP^Lf6&xb z73Og#DoARnW~P*7q^MxbkYt0le6P{}V}Mn3{C zfYonKz9|c^ z*XH|};aqY4n1w|9H4Zww)d99s=(QnFW`0}%h|R{llieIx-GFL4z|fY zi_nF5MPsLn(A4KkN=o)6%qdRgNz33vxFh zZd;aR{G>VZmnptEl|AZ%^%3E;K|%0WF^vk*)Vt~Xf4ode=YA|G0)^6ud{B+$zqCge z=TE_cPa4|4mw$t>%Tr%{sjHqq-tq=C5)0TSvRtU%m(TTO0AW%ISk;fSHy)I#w69M5 zLFg17ko@zlHSm`SX-zG&JJ3DsdhV^|+V$$*tAZ;Rg8}>QySg5Wj8x>xj*bx*rsi2! zG&injJ`M$^K&BY?&Mf$_U8qpYS+m|XDZLB$>bQH;rkoq*h@8d|A|CQ!Kb203x1K7Na<5uHb&XRWg8|P zOdPO!oT!>l^e`X3!Dxv0($0@ujBprSPADx(II^Z4qeIia3#+qPo@}It{ z7^I43^`M@a+2rwC4a>kB5?LAH%!A5cl8!Y z(SRSjKfZqt&A)^1XL((ANW|DhbtaL3Pi%7?t5t|69GTICPP*5SLM5QN2(CyKRhm}1 zJ&UY7dQdxp%G#%dRVzEDn^y)Ij3iZ|TIMd-^qx>4(HDPeO`A5&(#)7Gvfn>uOak!j zgJh$@=V8Fhij+J^efja@hM3vvMK)xk|6c>VZD;Z|vsG2(4VJ*#oyr!!c8hJ!n6D;LefM zVxo>#q8L6Di-SjQUJ;^k^A-z7OA;t@ZaX(>>s7{FhHIE7y^zWOIz^yW7QZb4oB)cV zs{X6K2M}uI-5{w+T8&z@rm^bBzKD*z+-G;C_t!q@*@I);kvd4a+WHL6zTkR>V}9`D zUFMh>iky_DoXGX08#=1!_5_^>1(vwHMgkEWcccTbvU`=|4g00vMm-+4xE%}{1Mr|@ zW=ntSVp6(liQRD>~&$Yf5pBMQs zoI&=-kGihdvo`-dbi<1|4enQ{+^a%cb#^KnE9q~_u&Cgsbf;Im&!yc8Ht=PQah0Cc ziC`R+oUjNL4+57t`{WRV4Ova3!ZLQ zv(~*%)=B^?L9d_ze-qG(*@HQrCRxhfh4fde%xIu4BQJ*+ZP)5W9+m*X5%%0kk9BJx z4Qact@m1r0=lYy*|h(nEn-& z);cU2)fNCH66aG98P4X-cAYpYmr1A;eaeqtCG87hp19LG^pMSI@q0g=`prLOZ&0*) z53j_Q>F)1UzaJ&UeaQU6VC(VujeEz3D0tM>gJ~_hrWo|SF@)Vovdn&X0x_4h-`ji_^F_Pur8hiMQ|KXT>&MR5E3IzRjQGZ<%4m!X78Q;Rc>_22@ zU!qHGb5P5ic^!yNaw8?aVE|rfkqotM9byW4bfx!+ciKqLcHR4DU+A5;POa-|1=F^m z)goqX+%Zr`SNd!M8RU1k@VzkM(wCkv@?bT3h9L}Umx|Up(<5XL? zr}5Eg2@C3W#Yrq+sJNTWcn}3lt5F)~`z%h$*p z8PJ52y1qo*|0E_%j8JECwi&UH78s+B72af#j_tgnIDQpJz|emb7*WJ6!Tt*w~p+ zF1>FgMi!W{>_Afd(K|nDv2Iw~QIETRwcd&j&M4C_FI&fY;K0aV9(~8k_rTdOzFR!a z3{^c4jGgcOSEVjnzr!{5I?z^a?9_eN3Uv1qE$f6>alRQ%e-yneJ^rXll`6?dwCPl$I>TF!43kZ@A=)3?DHf>ug26(4NbO%|vv}hL}zq z%*US=IGv@aG`eM<` z%9A70$s$y|&Gqa)N}>fO!MOmg?Zc75M;$#XOMlLmRMxp?!Z=A&(jmKKG)rpaSW-7A zphjGqMD&=V@xv=l>E-T+O&FixXO@*9y{5cU+?4GmQ!~=bW~KDz<4}lEJu9?1u{VJahw( zHZei9bV~R`J(Yb>U`5cUuE*G70PVVcXNO``d-zXYgKLa-10=w&t}Ubs#k13L?R`JM z*U7Th3tOtQX%d>P?1neq&WGBLY0LF3eEV<3Hr@Q$uTujwi|_I+kU4LXpc^aN8H&V1 zdsqp$ogen*?dDdn=~8T?Ite8q0VJ^^2AXv+iG(_e1s=vRZWOwm=Kc^4Dx+t-@d5L2 zw1||n{jlkS1`n=t15~@_>Xo^=!FtJ{BVrF0LA(>vL!}jb26EZISVIBY<~ehWDm%$UBA`s)H~ni`=tE+x1LZ>`%>ms znCxT5O-=>x?N{ub?8)yeim$?=?JnB|(!wEE$CbHZ>K`NZ)U_%6GFLN%9`yBN#f99{ z)vF8$+7jQ7Bp$^~mG<(#Zv2R>e4iXqrA~f^DZ3BowSGHYyS0c<;Tv@dO`wHx1ZCX;CW!_p;WZgr?{pk4BJY7FNOesD5Tl35= zLaLIkpM)16KgJ?3;TTi5@H^2jk~d+@+QHC7U>PfH1`oci3{AjLlKLR20MjBLGy>xH zwKzkuS-XDya5w*zmo*slm0929ZRV|>@q2e{r^0)g$TG~>*fEN0S=NS(3XDWJp@%#V z-Kgs)W>!&Qd7b-OI%kM(O|Vx?RNcCFS3iQ1+DKFxG7zf{7zN^<9a}%;bnTic?gX)l zOO{S?+3YiPd_8m0+j*(TP6 zfrF;FSjcEvxLWt!4Ww>jd1z%#}sdx94WUk@YL zM!$7P4L++YkDX6C!s&@J|N45;E;87N#sfVs*^|=CU}gdD7ckx9Ae|o89m`|1Nn&Lb zhSea?VbTa?971*?0ly7m(hdddt$X=pAmjm)!W5rc=LqJF0wn>RH}J|vWM z%x)fo#jvni@qe&-juJ1M*{o+7;1hBKISbFQy@CM>k5gooCCVs+c zLYOsT7mEH_ej+auhdey;yCPBWkrN#=9-IHq+rNJKnUe|2b#m}II;Cz4&%a=Xx>T0% z)D5B6v0ua5)|DblaJr-ES*CsFo|y$TY_D}-rvj^#71QSPIR$P#GuKxxzuFO%J$#D& z=I4uzFc=(SgvtMEOs6kT?9g>$)AcT=C7FUwt#*B_W6{wAu_N6^&AkE7O~M*v$; zwD`pegTdq;YvR7X()Ls|JK|d!8tMQUxX$+W-swq=9f}Nb&Wm$x9@0e_AM4W@705|8 zQRIAxPSENJj0d}8;87V#a_O&&#z0Mhz>|H9@kq`tzT#BlB$dDyR zkO^;^^DRE@QtpQV5Qx~G^h3cam;X_9#ql*2FL5nn^St_Djp}4J^+cBR**;^aQ-zzi z($iWLq83Fa*ZvIviR4@qrdNc1M+;N!UHV;M0~Ai39&-PQLtH`rGb!HyrFC3I>`L%N zMB2&973hLkg(W+1r%ZXdHHimJJcXbS0oMDg3CBMMxb?@gmnChVcO*TO21Ph`H#ehc zK$_^k=l*lIk>k3^TL8~oekgw~CoCzO_<4^q6`j;sn{wpeWBtjMWe0WpJ&h&1D@>kt z*?3i5o$Ffs|DLlqrS9{Da==X0)^xAG!?k`yRg`?&k?{2&OZ6b?nV_>|VNF&>X_Uc| zN+rO*Drr`%y<(z2J?qn#d>=qy4(r3xJr^oJI%uysrx&mVe;6)cP}f{nU-VF_YlpS% zJV5^w&lpbsPZ*g7?kPN)jwGXO-VzuN#W5Yx4pqwa6RuCX;|wQT)q|~U{a_AQ@SG)i zhBaXNB@^PsoVWw}x<_~)WR=a7()=l|v+2taWmqVRZROA|4{xv1ZGu#GV^~;6-+{Br zLe)5~;bu#!geSF=6dY$G_;zhd&6VCPk#M@7NlKpM#~g2ovEio?c@sS)6gR$Oj>%Ip219bZZt!4 z{d(vGK7^1i`5u>0`Cidrc9@2j7SE(jM8TCJ=@5z(<#VmHHk}AcnlYmdT0_hKL_;`L z6Jv>$bD3t!r^m&m@^4K2)-Wp=U_<5tj> zBR$%=rO5YzJ3Ug#&lx)uT05K5 zfB!ua@a1qOwTs84Wt%o{xB2(_S|nS$J8i#tzUFXMh6iUFg-sn-7~Rc(U${t2;DLh& zwbI%ljemS2e7^oJt1(zP_{vNP0j*#4AP^oiFHh^KdGp<`zx z8{X*k6Y*TJjk-=>62<@t13!&=oQ{Lph> zU8;b(qLP{M%==A!37Od0chPa~`^f*?_dzdfaMvHqg9M{GTjUcKLEkmX5?-m@Dh9e{G+P;yMHUet+8V(A|-FUAA z$_%b%t2bxSvzEO=Oa&&5=~AaGQzs=XW6w^hs)pB^d-OjzW<&Ex1=Qad=9YT3`OJ;Y zGS^9*HCpKFe6|mt!k}QfZuqeF4ukDHZi-d!S3+szBibj$s+Yb5Ej+x&6U3dOj%u9v z|4>H-28PXCgFFL?E}6B!`R}~S;ruD~JsRre7WFskXkc)O(cxiEnJx>=SPmy(eE>*h z=vcB*rdm`D^1^b{~=8f`46xAh&0ar1h9FsU1t*l~BU+*dWiCkH)nh|CmZ!N&nz8nZ|=9b8Br0N@&gvCM0>t zWDp@XC#Y@1dd#fJqT0%+mD{J@#d6K0X*t`(vtDqJ(lxY{!*1ZI)D! zSZ?#9lg>gN=&7@Xb^4h%i??w^VX^drOp`&T+ne)=Akz*EC1GaRuyqIIWJF|@&HK2B zH1qj1)TlZsnb~Ayu;*<)Zi^IPN|8w`qzs5_uD+yb?kjYFcohT8*+(_w z(f7;hNDL+&T>S1%CpAk-&5Fv}|6MVgaWd^Xt$9=CYtH*w9s*&PjWCt=^HtW8Q7^C2 zvERGhy_3%QsFTAZ2hasyEc~ZQEF7mDR!d9sQ$&NJ#Yp1xeGvUf(hW^2pF>}C=;PkKa zE>1=XVY;%-EFGtc84~$Gl3x@XtQ=a<{f^PIVA8cRsW%gD*{Ik> z(rT@zr)TTW7tX$zlPGRzIkm-p=IphQ$$twU-HQ76x^(QgWoz5N80D*pEV_?HPrDM5@zwW+>be^L+++9;@ZvejMj9CAsGf8>7@5$5)1g%$pUUVEFONZvk7g z4T+nXRaywOu9UO19qcYy559JHJo!jA->Junf||>qkE4N_V5MoGMPGfcSI*)c$tlbV zFB`PgkN-|d(Rq046jkr_Klsaimh8i41NkGD~&DC^8e88f0@`)uRY`JR|fyfWX)Dz-QcukZ{?>^GX^A5S1}VTiZ@9zNw^%rP%XEPWMd z%VTSdxLF)`(4Mi6kE8GcENxnIU^3@^W%KhdKG(jWO@JzYrlKRYFJd=NIR!PU-)YiI z$9J04@J$UAt*vq5j(>FmkV+xHI`~R33z*D7ODM|HQf7cbVKl8QJ@%QSt)36Eqa-e8(iS;QD)QJ8MBrF>bT;=Z`W=b z1qvVjHCcFr=Bdu;B(}ZG3}32jCOj16v_Wkk%X?g*@@V5X)up&-@k3K}GBFHZ3GLVw z2)VX+r@&404}>+6L_zprvGoz!A5m6h|14`)*)^h>&nTV20e^eJTs@)NZ#jRsBIn2b z9u*a}?7CC6*}e7DrSeGm=BJLc$bF2QMgf9zgVP_Q|DXH*VSC~{i~cM^eKZY=3aW|B z9NE08yarvTyy~t|N0c#A@y+8uYDlrywWt64(7jXdhh}a2YU$YB-1_H4_3J8uT*ZR8 zyw}^$XSiC5ZGQp(CYRU9YBIls-Xmc{{LN~&J7MP6QP-&$3=)B(ws zmYmf@xC6{zD)RR!YMWROG2q9Acb`6OV>`3nHcTToman0KA4$3aFm(tQ@!@~d#RCE( z4)yEGPOAWYl`ihS9<07>xAgnWaYCMhqRw2DzIdRpQR?Kr0QDlrw|8>V6N_YazPoXM zqb)pX*Z_jic3#cQ8G1GeGKywmIHKM_YH>7?_)^6}7$}z;VQAVwr8WVeQ&i z^v6nz(ZN?@48iaTk$3$BJTm_L!p=CTfE6;jiY2OQ7gtFGS)R3Rl60!7Bq(1kZ4+*5 zg07o`6p(0ku+ERg^w3XiR~pHc1MYGL`P1(SK0uRW!w6W1VOx3=O#?WSf>Vd2kQ{ur% zPU6lGxrx&#{D(pi*du5r zFIO9oo{w(@1snI-{Ckyt4^movl|f2Bv|J;d_nRwCelW}Ev1wY!%X2rT47bn@JVQ|yxfu--xP-Q3d z@FRtP{SJv5H3w~g_eU?2DLw=8hN4sa-=tCd(gxmda|w+e5VofFU-Vo9i!E25JVXWM zXuhZM1Ow0pt>3&e#GqJFpF-n$$EOGVysNKQ$dMIX7Q#spQsRS*5UODE1#cnhCQa>o zx00VjCBMD3?c6(b>2iaq`&EH?!ty87Bu*)U!QJyqFWtyx2AFhaLy+@sdKQ}70DF+{ z14bT6bK)e*`2873bkSYSRcjVd7Pz-2Mj<%Jc}d*DJ1^19{L`i;!WslLhznEh5bh#R zoQR^QaU%*6bp0Tj5z!cmBhQhV*KL>DO(*)&s?*@6{P;S7aWsC!GaZ@+4Tuf8I3t|+ z=&Rry^^79D0qTTi_wLN?-Ct3jQa{nVz6ppp(F-Ugq4d`my1<$~$9dzpNB&t*SzX(0 z)+`Gg!nQ|_9(`lTa^Q)44zgApjN|yir)MG&e79dMbu zp95flFf2B3r3rO^Jy@k{3Au_7zm`d90t0u{?$x!u-FFfo(Wa`7-5FB*>fGRe{&m&u zDm^ta)1EK^fj_aC3P)`b`QYAQnWK85TSxYx*>i7`gNH~5e@#UGWzd;`bMQJ)MTrT< zsyYzMyZbG*k0MRW3FW;4maaxiSBwe(u*xFY6Hf&@P`j|ru#JA$VnRL$m9FCoGxg;{ z?)tJL3gWhkzf)7`H$c$QL2hJ>I!AsBU%WAM%Zis@!smw*rS1OWTs>&GRuJjBAL$f- zpb*5|8t5};P8ZTr&B=Uj!jde60Q{t{(W-qdJotGGk%;9n75$E$o>R5|1Z9uTllIB-~Y6Mj%}v+O=;YkZ)g|g zc=^uwPWAbV5~cfVM9;EWs)t`wS$@NP8CQ=nQn75E z&vD~O4sbBg^XW*tEHY46q5)UnJl$&GL+pFCVRyLJC82T$W%@E$L4ZB)C77;(!29kd z1kEc_6HbaGVGRm)PJ8nu^-}H^@&|Q~9yv0kA&cF1eC~?|2H8}v?4P1|mp!AsJo)6V z(10lJt;Xxb5pqqXyS}-(-&zq^K~_hF4Q(p{^|Ba4zfJ3q**t?51!vLd5FPu^oofuOD*hAoA&IvYaYowsrL6#SL>=akR9}_n$|ERA;z;_Zoa?n0ZE{)S zNm9N}d4n2cpjJB66)vC3=+fvLPZWK>p}XCR3b~ni-rfG$CflHI$btt)U$0)xQox0{ z=>5^4M6G#;MOhTmvNb?Ivb7eAiQ0?wg)(*I5m((xQbI)eE$Tj1O8@^&Hr4!$#wB+a z_tdiA`DF$qQMoYXR|)~l@$;RY;_(49fV)by$CgNDOX=7z9YZ-yY$_e^a%nNJsN~-> zq=`r-jv!1&yAQb4i5fAeQ@P8|Oo*?1pR@hz4jSj+u%5B@>X(8)yh|3Hy!bnq++i+p z&pWl}LM~BCEwZd6&XnjOM|03ox`~MG7+N;3Rq6J{`-VZvcFn2}?`cq74y4ndQpj&< zlMVAH3q>5hm}i-}u|Pu@2X(g2Y)p^ z($`@;YO+n+8__vvq*g3Sz&?)n^mYORiH;B`HGKXQEXQK=;0dbz9$xgv>or{=qn$gq z!>~Xa9%oN{`?Zw~Ct5@J^1^GOOltLzUQ_TX&%Kp5=_9}5+0$`_)RJKky58U~`u3(K zUN&dq@<4P~Vl;Grentn)qvg3Lg5T0HXvykTbySp1*)&Bq0keMfa|elN_Wj7t#}3hU zbpuC|c`Y>x_0rHnU(_cAT1b9;$&%LU6Y_;<7><0rU|s?DoVDj!O5v23dY5b<(Y>cWTR!wRKA)*{0p0kKZ+B{P@qN?n%5fdz zJYoM&>x-NO;wu`HLo?P*M27@p#IQn3fmUfTQl3BW)M+hZJUfAos{GM)D{YuvHfhj` zh^pX!5mC8V_F}I3KSflS`WRVG^qRuvLR>3R_A|^_({TVCgy2_kUFWkAwlYG z94v_!dCFC$kS}&#)h?{+V|usnttvwl9au6fXFoVN>_O{o+sB5ymG2*)w0^#2up1J=OM-s zyDjm1(a~in6zAod-0&gJDdpn6l7=&N$nn>?e#Skl0@l*!jlbVq&tAJ}-|DQp`B$Uq zi8~i?HxP$#T^O6bLgasXSyIDf&v3EuIT0v=S%QhN|1klrnQRQYc14UTm;GC6BgSF= zpzL-yo4_b_b`q@170}bjrhWI^7xYwFJ}7Cs(fL_Wc!CE@emzYhf5<^uTI|&H&YHIW zuVC^@LD{lFE^wZQN7pdBx+!kE4j!vRNat228?+>%ju|{axfJP{}aJCFIzZOZI?N4c<_D|XBX&!7~1cyQHSXU%geSdcs(;1mK z9*G!4S2o`GM}I^`s7loH9o}&!SA1UWhg3_Md{ZcEPwKzOX82_VK#BR22>S+@Wv3md<9j-|zA-HXYWx6Y}h8^GYv$u$C+~(zt!{=wH(U0?M?|^XRT)AH96J z0hGy6JT=s;a2cGr-zW;M$GA5kTx?i-p&U#XXBt@gOa8!jU;H;`mz*Hu$OrAg!xw3$ z2WA5-Mn6ydKbGG0d|!VgCww5k3f=9ptTh#c7-gW8LK!JnIbQ$uNC0Uj6)!1x*MP_e zml=b}eQsk2KF4CwsSsmw@{}!h>~1oIuMWPY**|$vhhFhyRR}mr^d~)ml66T(bR_du zQiaPeJ%G|J#p-mQj2<#rIFmG#JTMmH`{#p-$~ndB^9 zm-{{&)h3S_p^Xmx^SJQb`Tq{nLo<0p?kZq01#&9$>PgOSlaOz|PgnECAcmF{;#=>e z!L=MwmV)nma+sSx#$Y_7N`^!$QvxIqYYTWE_pC6Y(64abD^kd= z`sD=Df=TR;(mqI31@D!u8C z(zK~+%9QZ(E8H3Ulv_N)&=Js2(T@-te;mz98J9g}+cE3-E)(Pw@ zR;qq8`0gjzM@D01wU_NsD4pisE*gs)bpscK^`|})nWq(`9=8W+p*g)b;HFG3bM~Fz z*RwC};|&$G>hro%OvIH%hKCR z5eX?{5P8sHU%d~Id&eyPwvawFd2%y8{0M_l(I+qP;iFu_ zf2y!Zc|wX{u&H&}^THcYbAZ5}kie|GvO=toov7X3ema zO-h=JYk6duf$7eC`_ax%=0+_UDy+Qm*j+PLPG}LoX6A*vpl$5DEEA2;JjevZpr>5o z%n8r2)G!OWei9Be&HqLF-H#?NCa_}Yxp(Lra(EN&M!;!?%Yl;hh z%DEqR6vSRpKjq!ufdn6H$&n1muE!cxU)AUq!~6!~47Atv!2-UXtqiSt_FQ1?u+kt@ zOm*kaPizK2Ie6#*)h2@zw`ehmFmZ8Z+*2V9H*LPb9(T`G_Fa?DSlMsvHOs5OoMt3b z@KbOdjZClae6VH9NJGRe%9N!R;fMna1S{O1&HQC7Lu^+)2B1LTg9BO z1T;H6J=u~#e3!2%oAtX^*Qua^fH_dHudzMVLA$MgR8V29Ilz}WH3PygX>9BDU;h{JzeoR(dY7}9xGuJFd! zT+6O-t%Kz&MXn@%)2$o&bF%lxyxUSTpZ+!5r_%_-TkBihqlhPi>=`vR^UAKR=$3EP zQV*iOiMUNNL?pcp5t^j`a87w$@1m@3Sj#>-F1?Jliy(3_lxP-p3|T?bg=S=dr^%jC z{TB9a+I4a_=YxWptX(^m)Aif84AP0IWnbbJW0E2E3@`XQJeEvVN#^B4)Yw~UOoFu$ z1>|a@#U82@eOYB!ke6qJlVa2D>2;C@D9b>D4$>B|J9<@*nfIgc4)>xdn;WAFu4vw| zEL>dozPPp0(~nE~NDxR7*RE!f;G|SQ34QIsbOQw#F;?+_ek6a+g6(<9QD22}?=f?| zQ0|kZWby!=tnjH52Bs?%;M5!v#;zHu@IA~^O!T%2No7fQ>$^@o3wk5zls@ikvAXnQ!_A;U*|9Df?%@p@JTNx-A5nNM%&o=vaddnukt|K1 zp3tqW0FsFvOaIEaBhdj0a;cI9p!-)*A9yDK++z>^7BS(0|9(6C-T#b!A~k96wu1UM z%XCb0=g$3a9g_&xWohL568?b?y{@R*KXk5bbpQg*(X?B)Wu)7HK*3)d{9UQbKTXeg z_p|FQplX-N=k$wz9mVw(8SdA?aE!Zl@m`%z+E?qQgozNVd^@MCVYsEAuDKJy@pHLDkz%N~^RCzE{j3 z#m6mmt#=P^r?B{rj*iLkO<7_vf}iXMU}oSFw<%mpkMYZiJXK#i9;Eat>bF=rGy;!A zNu|fO>9Tnc(AM-l7&7kkR{SL6Ie`+0|!5S!Lv({sB@PM@m!4tMcFFOT07M zJUEzhc(VROS( zQNWL$LfkR8QvKWB979okkr!NBvh+4?T{RtKn_6d)b$eM>S}8)I%$+~?TI*H=QEcGV zY0`uV?c33}R$_dX(^_RmNALJJj*;V+^my4Vxv^8;Q;c_olQlLHKtvrWKZVSR7HN?c zrt;S=_y=UyBVO{=<*ny2P@l?#%pf?H-o*7D324|QNtKQip-<&vWi}pdZw9ntERVda4Um-di&J- z^g>!MbZR@TPLKRPS;vkYyXoBi7IqaXTQt?MxY78~ z;a`Q9Kg?NF&wBK$$yak@9yzQ@bMwdv4xW+PJiRM|f=hGaj`Ui6)$m;!J+xnfe|^7@ zEZ$(?KZhkOtCE$YWJIL8jyZkl=XDlc^f|TT%62mIP=fKUZgT+fltA5XETgHrKPBpL zyNQ)T4Rp~f?Td)00L%9-m6blS$bDxA6i%P_?ImkxV?fXq3lfK@Oh~m=weHnxtgZUyj*T1l z9bSz%iSS0b_ALIG#43@+d$tF;)aTX_YG%Q@${MHVya?fW*Bso&`c;tyqv+0^4Gp1* zEJwdG3*~+9?ftbSPQ=3r#4uc7yRz?y5vsf@Pz1*h)AsEDY_|ARE;|-k#t2cXd#Wp! z5qwfBFOx1219b=EWn_WJj4{#s_Gti)uHZHg>_V0V?c=_Jc#W2W>d5*a+Ieh%nl{Eo zD1D?5y{F$1Ji#>FlBa35uWwT^_XAsv{>7DH_0jBw>Vd`f8N?(?S;aFEmWO0OSDNKs zVY9bCV0U|;3EOk9=w)K;vnWdUFI_6J!LbjrUh5&biAQOT}FdcEDI3{gB@KG2?*E#erM& zt09B0veI$z0QVUaRzcpt&v|+FZhH9XJ1@)aGfUdeV1kH=i5b`BLR}jGOTtQet-0R) zT^jJMc75BMgUF%9c#yKM$p6487~)ng_N9RhH! z`@zvij<^lZS{Q7zRy=&Zla_Y%MC7{9D1{^ohcT>)zWO2mn|-a6PD>zK<~%ZK^VF%CcW-k%8`@;@+rSN&SYrC;Oocv)ER%Ppd$-p*$h@ox_Ec(7BgG9$mBH6%u z{paba-$6bu=UX1~K@sO`-q;G8jr6B@sj7TZ0`KR%xLw*J$kLXDy&w)FhineufK?mH z#T_a%JK)fvy&g*>a6`5s(=JuguN#SmiKjC488^Svv}u)P1`1tV9>mC4IOZ2E1)Q*h zZ3lI70TN@@Fzk^+zH_d=-~e)lWsZpY2AEHY$oRn9kH8 zJugLX%pQOLU1hO7C)i%vu==VA|A`dW_3I6^ex+KLvfn#p`WX{NpggRDc}mbDM`IJ~ zb`JjCbT+koGSBBtyT(?Dntf1nsGtZTFP3H1Y6=yUL0QRga#M8;X7?=A5{Uk>Jh|6t zNA|H+Vyc23Wp78X^z*0BpMDJGQ-yp8(RSi`^{q!Q6)>0b)yey+@klO5nx z#M77d-gz4u82-A@X&fJy#eg7(>yRU^u^qy$JK3c!FFJFx*1c0aimAyf5P&R)FMp0% zXpGs!(vmvy>w3C95?1!G<7bIhPx$pLA;;%(CoSJSxZ|6C9Io<@f@+3n+~pOnKS=Pw zPVEr)JbL={@ZlcayTeOY`v2&96S$o7um3+}D4PZC)|c1n~{vP73%cCw8vTUu12 zlr4LxM3&1EvU61;I|)seQPzUw|9`<&NyqIZl6 zHKmV!2@&WRgGLSz#z63N`>j*yDxFhfpUehguf5s4JNUkKK5yN|?c2NloERAOD0sQ| z_-W8#(D1fkA5jDF8B>T{G=Ny*MRQ&@XDUV|71>O|RdLfqmA0IbSuoiu4GYbrf4tPs z$u4gdk3og?*Y{@S?nusTPaPI1duF4#4x!5KL>Al+>0l!37n`}wp6!y4A!#X6vdrcu zTZ=7N^l-nteeM@yka{f%%pQL>fAE~xEdVsNE8-v-A>>_8xwC0%J;5kxLRCdj7g7_Q zmwAO{E|S733b%M~uLnh;rsAoW-v7?DudNR>Gx6wwa8wpDfEl`YIfTm7^EGz90ofuV z+2?-oJGL{9C@=1MKPurL&kKL+;!L8JBo#u5>&Y?X34MHt&qCbc0Gcw;-kV7hfiXd2S~;r>@<=P6w7+fnJrm8$X}5QM;pBa z1wNq}m)+h}94=*0AC6SE_CRt(r{R!?)bl-bmBS!4X7tfv-P7p+S$~Ybx1=IL&3WKV zEc&R(Tng~cKZ(`2l~j5EP9+xiGNGND*O9X#zk6isTgf(~=77O>-ns1073m`#9oMKo zfB4|v`t~m$f_zQqO>6|+Lvj4@o@G>jpL6bXUviWP0JCE=M8cGm<~fTJr*z2ms;pnM zN0vn*=ij6k0Y@eM@CLy7jX)tVJ!;adEzIwJ#Am~3c?P}05?$)A&%VQlE!=W4FD&|? z9Uat&CkhQkCnqmV?*7e!rlp|2x&Q3h;jB3%Xx>T-;dp#nyMDd+Bh=xh4l-6ul=EI| z)|4rBsurfEfy$9&+I5N8Q-S?XHG~2ugQ>OXKB|$@rlwZcN3!Dg;Qnk`bA;6=#v@#N zsdu}-neWyBVcFgc%v>ZJ7QPm49=$gul1)w%M8@8M6U-4)Aorg{lw!gQNK8yTcrp2J zx;OOIYPyp|NnBXuV@MetN8`F$oC6+WZvPmY9Qknm6s?roqI(~AVI@7%2-lsrZ{6wx zAMSE&0C&-C*5=3#Mue^|UPvoIUM67s_6yzEA3+&>$TeA3bAD?Oay{a7hYTNnZTTI+ zCzV%vEvljO!PlOO@a;#lOV5;*pRO(cicC6FIK07f%eLJyW^9=A^=M$~ENbE?LH-~B z@hfaj7Yc{QSMNwOk08=?`PwD^#9RaFLyF!+5aRL*LS*00A(RkIe-sj}h)(J(XJUdn zCo**=6-0EW4G2SjkD`47E=x(>&MKfQ12}rjB{k<{@on9T$)-w-Xt(Ie%^m#8+Cv*P ze#W``O(8PXT*qtilnI20cS)xOCqflJ)46ff5D}*W=(=_~4n13m#23%;Rs+h9Z-E&j z(iAfWmV8j=X_B;OuK8W6vNfrTYLJhCd5`%#Je;Jp@84eAOe?; zsCnp;w81EyxvyjXrmtK9U1&TlcG)-4qX&Wuh>zI=m{N}31)ma@isfF9Y(81&=L6xx0y68;pDayPFY<~Hu$C7xw5 zSFc^uk(t}-Di$ z#A0`cBvOrM?FVC$eu<}1Me&zjb?v&oel=j`5a_Bai^cb}Mm)+|RH9{uQzKJ}TBC;i&;1S8%<~a@4 z?PuTzXmOk@Wiqfvyaytda2>J7ButSRE5!m+g!Bbj)cT3DXNLsq401*?BCAC)N=A`b2%e`Jhc+@jRFsmqjm{RErw2mv})S&0mCrI%Z=KV7Wls z$`_ME6Vieb()3wkOtjcbo*&yHZuoNC?L#ZJoz#QHl7WrgJMyo{uYF+i8?5PPFo_vR zK+G&0IP>Nuv<_Oo{tuYPp*3pM2-fXuFcm?A->O=l>DNcMF$%#EFazudvuK?9ERU+v zXGPtYKpk~q;f}!mM=xGn{=nPGshUtQ>0ce~^0s2TQJqGC2rU(9Wi^KXoR92^!A506 ztlD_5{5siR&2;87cB|v#k>;>$vwUr)4>Cil@*p=L(_+)-b`}=GfbIR}@d7<8G9Ct~ zH*e(mmPtkb?j?Z?sasXEvY*6OMju&wL!h};^-`oqy{O!Hi6SNJ2w`oX1FyGNH_>S) zEh9S5`?DCDt_bVSXrSxZ&xEfGWJ)9;*b;W<)pZCZ#BT+SX>ORzzRD}bN=lrLy-Xjx zVy^UPAvG|whC}GeTLJA0V*Z+V&a0!%knNQ5kkM7Q)N9|MbBBr&%L+B33f>(KBScd? zwr-et0&f#*-}bcj{^$Eo9^AELjjf4cwYZ&Kc|3!OxFwvaMM_`0rB9c$OtZrP&!?(p;Jn{h6{t5Rh5 zyVGc|W+YaoV&}N^`*WR_x{1jBiM2Hy? zsy;x!h4XW{OSOqIe(Q7srjz~AERuLYzEEElQ6#VihI%QVJ#)wqR#V|Np8j@!ABRw+ z@^p9DV*gHSl>PlDw;}FCZZQ8d32{K?xe?`-#vQ+&lQZvrLi7l7Lg@={O*=8tCI5CW zOqFZwi%+!!bihM+L|?M~#xVxH0DseE zj?C4H!L{0M8=(4{?-#hg4*^Dm3IjC{%YVn_72Z<&FvKd6Ndvxq`NF_54phe(zqX`n9r?iDz*MIyw~s71>*8N3$DAyz9#t!o?xV)vj1^A)7$q=d&JUP+TC0nkyK~T$_Z997O z)7Ja{MQYXzk~6vIH&WB8Ag_~D(t!@%QTlVPH2S-Kejv=D^X&P%%5iO{v`=&-*$+}; zt?irBqCbgF|+Es7H?;Y{Z;s^=08p z7ib$X`2wptX48TI-s;}8y#n2j5GXJxXvc$Gt^#0l;6v(We*Tn2FB10cc(wQ32 zN3^Ru(%IRUS=9DWpFdvw?ClU*@MK3|X32TvG63!VKcw1;l(7k=?G2GJ7u17bAf*S4 z-m~Kk$x2_7ZG^kC{Rz>D?m{oU$6$j>GS?(~mlLrg(ZrGb)F;W_cG*djnOWs2N*4)t z#)#3c=;tmvI@WXyOTB&brY$SEwynWH~3Q>^cJX0?r zLpFV09~30ftazqw#J96EI0JMfvGjb3D#Oj{)!~@9SDH%By|+%Kp^f6yW_rCOXSzON zSc%K){yUCLt0&`#xzt2nVXX!YH{;(?ExGG_PA0u5Iiojuc4E8kdiwTzMrtZ$Kbcf8 z;l&uqQbZG=OtCyZx!dA@RYVLwp+nUDNi-1vh2qq6oJ(epWQAkjE^7{FNMM$mRnE_d zh+veVLdVMf$gZObxaef}fT*999`WzL|2|&gdGz6XM(QqaFaN=0rJTQUrz?JQeZ;=F zRX%?2Kv?+z?{Qsz1Q=U6fn3`vuU@0(Sk_G}j{oX}_t-g=szGX}rSY<_ZS;%U`(Lj4 z2=IUvP(~Y8xnTkPU=aD3F035Dh zbOfy^GIWn`tuYtw&V~|wPlpNA90yAS z?o<(wV@(7895Oh#g4e8x`p0;<71EOH%*rE)I z(PK3GV_gD4V4nsI7;uHFasw8%CZpWgPb1j3OD`G?{*Z|5B52l0EG2$TU#WJ5QL+B{ zv16utgBrb4W|8J}V9;)zA3t;bh99;kU=Q&7MyDo$=0{kE`Y|KEv2u6`_>(qQHl%Pt z>++lyG3@jXx^ZIsP?9t+LXRL*Wxhq&B#isNvFwfcMqZy?CHiTJKl>- zNUKtIAa`yXA2XgREh*r5%r>B4NP7HudZmXme!zHNGxlp4%KgHKS`vY!RAjg7#9+H3Y?^r;+48ot93Nf|NwF=kc;T;K@k}%Hpk?0`?=y0&P~! z<=qQhyS9!5yW%Iu`Ps>-1>)hWZWcRmQx z>n&&W<NEzRf=ycvb_GI*CZNud4lGVK?vh}J)8rT*1vv3pQP!h&3^xWB2%i^B8Y z$b3;HPm68ZtJk<8QJGUuj3g!CMZU+DbhL@DUcaum-(-ywUHlc4UoM&>M_PllR+UhT zhJXYBAyq~dr+rz$&yvsaJoBnK=(M^!x^yR2l-5k)$YWL+m7jf`8}AvoyUCiK{}WVn zlE@#)%lv$x15xRvyEDzr>*;IO60WA2(|M=!=+UjrszFYGt-u>bW<9wpWJ)gR^JFAn zx)2zK(|B!05C}S}4CHwji!?-~hf$_A)Tc+ND<6VYAksH%)osFSax}75A@0^ zSzv-q50GSErFfp7_u7i~0jDt^5T!6L@4k|{1oKjPj^65K2l)f!d~$d3!QBirj_;wA zcUd--pMBp2?L&lj>C=`uyZpnN&RyH@a8mnv#^#S6Xw?w>iCE%w$VDw=O)q(6TGD)+ zyiw;rXOh-*d|}}ca!!#!?|93>agaMZpNTx`vu47J<1g?^vxBRdnOQ&3%Pfjr_djlz zTQ#YFC<3#FV#4;FuObXW$BR_H z_M+s@HJzu>?_QDofz)1wS~cs}4;BM2QnfmpL#92T+ZHhfKy>e|ZynY{#a-Dya3?0k z8$Mr4QEmR&$AHq9GW%6&C+ShYdrCR^HS=F?rd2O?oSa)Sx1rya9))-BvCG2t^`gm3 zrkPlc?LTQWszd>)&>HQ3aFJ9-Ieq}>x;XzmB)JuO^zz#UEj3&Ozuva<@g=xG@zt1L zU|eI*PeUR#hkL+1GvgQ%$WEm|Ppj(m_$&An%cbA-saT%*hD8ug%O0Od`vCT`nh%@^ z0_{BegkZVjui8D|Eja&T!FkKBT??KH;Eh*-MiNdsNwdHQn&4_7IGu!fJ9Al zcXvr;ZE~fr!Zc;z{N}YQ00<7}c~K zN^~6D*iM9d^hHpPuCh{mhRyr-;PWsvn7mdEx~}gmH?#8E6N2!;?aWB`!vc~oUOGtZNk|$hBk{s;)*Bm-%W0J_6kl9Hy??4xQ)EFJ{JAm` z0kDnKc$`j{VpRt$7Yv+9#OKoq;5pi$->_snCo^X3<4N6bo~)+$a(n?V;=V1Vu|u)T zhoY8fH`hZ`eAKn}&|$+=i^vuM60tJATi>Uh$(puin;cZgbZs#z4Pw6{(tsSiR?#Ey zVSyrM=6m?wG#U0{mfee4RrFJ)$jo{g3dhSEjbtKi;Itlw`t~b0WthoY@+$&nYMa#} ztwvT|b&^apUFI7jk`FupktGlz)q=%M=STaNk8wb5T*a}E!46n721Hda1j?dYP;PLP zk1Iqm>;bqSY0`!13u{Sl>Cx)@(VE~*a4<^FC2(OJmdxGLPd}L*+i)#*{8IQug)S$; zzt9Aahp1kUq2NE&j#f`rVHM4^o0y4sSb^V13?E(zSPFz7<>Ptaf1KClFFtQ{svV)K zeDY6g4nPwHpp-Q1R*@{1m`f?rh|88PWvK-|s71<2ry;=Ucu zvH2vtxVS7*=9~0mR3z|JWN^{n`N}ah@5PZm)-tN*DO5qMj#pRe%^LG22%ES)%Cr-q zJ9i#7*0X{)r3aVgBoKwNEVDJMoJeVQ@ql+=Jb&}*lfE4=dbLV?NchX?>$l71AnW)4 zK;E5Z7=g7ZUI;xvS}WwwY{%;#+`FSSPo?8@9Ti{TV@(as_u_%VF}D(x=!_B)zD|m& z;?sijbdw%ELQVJ3Px$dgb63_zoA?o^IgytETleB6%w4XcW;*c z6pC(RD;KMd9TytHz1oL={`@&|;f9`nG$`i#oX0~K`C*gRakxqPhcDrkqbwo3q?T%! z*o3g)p|I)!;3)aJJW+!$_<$!1#1ipvmxPm1^!8`fTiZ_*Ya2`*hP2|K>=?#lOaq~6 zhgdwq>svi?KLJT&KXCT(4)yc#DcSdv#uj@)g_po*$nC`{Cl8G%gQUDC7ou}j#Ji2A$Pj^XV#VaYOudR zTloadrju5>KX$pghYXT2NdIcu^2Tvz-UTm!D3XKjxniF27A(5#d6R1Xpa-cVl*)n^ zIw=EwH>AbcCBr?SW<*azFVn7VUH)F@%?=1DCrygV>gdE`Dtz!2*;rM2N*LyLKd8^r zK=cQqKJvZU-xMUh!wx>cH1eLlW*$c$tq_Zq2_TrM`>~?l;josVnM@2g-a$5DI=MUe z0Ij!<82n@j8b2`7RzKUY)hWIO$`FX{NOmV+OG9*DJl0g%_gxy?X9;#HK@sJ&D z8?FDN(?Jp-+o@NMzuyUAvy3?wxq&-B+1}Qg17leowLx z)e+WStYa+~oMd9I@9fU%5`|M?Oem}C%4`&?;_h}K8R-Xoq9cC&G&C$Zl&YaWG zsrAUb(lh=w6!SS)Mpef;PtBH>7!@?y7-$}f&_`exmJz7EA*3ZV0ZO)|oI0M~>1Ude zTfRAA;Z>Tvx3T3X-Xp6EW9yKjorQ_;FekA39IM~kU_XKC_a8pon>7asc=v|eVojrN zq+)(1u{eq~jBev0GOd4PHY0XLx4&T{iap@Ru`HIEZtD94_{aoJ7>FJ2@l2s8$fbg1=KmNa&YM^s>l_lW+hlgkp+@I|NtuWav_se+28MqH7S{s* z6=fC#ruJIXm;QG~{?d49`?t;b z;8TtjL|11NE%o@cZ)gpCp)~KZKqtXvBQkAqQW8+3c@X>TlMqtejM|tPd%=U?dl2cb zhinuePapXYze1s2(<$Gj{s5An%A=4QWbzhNU z935^^d2MayN3v!Y+GGW{&%qxO+mdxSX2`g4N3)*I?i?4K4@np+$n-L z%UNf@SJ~W8B7i_i=q(kuW>ujGAGz=HjHsD9H?y~2Sn&qhL(l4;%JujnBvSVlf zqA--9ZHnCmZddl2Cb&|H`_5SpCM}s(En>G^_)$v|w?xXXXYQ_d+SoI!&g%Hn)vvtW zWPiPBl;yMsjs+$y^h*NQI=AV`mA+CLnGBb)>B)~&(1V6g_G2d6zU={+3apXeXhiTS)_nv6SGc2EK<9;h{OW3G#^+A}fOTZ4uiaQ~%*YEoO5-|sxCM5XI8kWd5JJ`8J^*2u!47GfU;h?5onYWE!rrOlXH z0Zm0hPJTODpzLW#b8OReL|#G+4u_bgboH}OdReSHDd&Lcnv;L%*no1qAvSsU#Viw@ z_SavPl!DpenSPMD6X2~n3+LHE*Blp=G5mqZ-duz{mprNj7rT@s1X~E1)#?0W?~l_h zU9|O9F!dq)h1=4v9cbQ%-yV7X{HRomR;^z3dH?})5Tu%OJUkyT#=7uLZ6X}th%yAt z{AY!w$r?kzQ}TW6Xe^Y$q*O#yPej08sJ0Ek>+QpFEF3F?IgY>O1toR`j^BI% z#tUBQ7GHEQ_=j^SBZF!#aWd>V61m3R26PyxiJ~-`UTzVTNa_w*=Xa(j7W3QxZ3%z@ zcKZCuH9Vh*pZUTj;X}UqhnRAtTaQNd%M%eDT@j={5$-WV3bKNW!>+71 zJiU{$K}-iGznX z2pN5Bp#q0y_iNEzOWFndRo(ivvAz|Ow7%iq7tL$p+N`x=yyG(C@7j1QN0*_ZQ|iTPb<25CI)j2dkq9qiqKUpR) z8Ri~(*YK@V8MOhH`Jn2){cv+8 zaQ>+9!q~Kvtu5j+gb|g@j-$h;>K+-Np65|SF}Aa+qYPEg|CFKXIoRHOP@3q)=f~1G z$u7;LfHh5oONVOF>)AXijh%U_R5}X=9SJ0oNhoc}n9-x9SJ7`DWPXQ%23}aj^fF>d z*nQS}-hcAs&?8^})j?p~7Z&M)&#PS(C3@aTybPQ|dh6&ejK%yHKvd#RWd_=!W*ftA zDX>Wd2LuU*|K~cAlUfk>PbY;&>eI(>1NN|32ZdX-%Iv9TmX~F$D28Q6UFO(It6SF6 z*X)EVRB2Ey0k5p7+Bf?R>YJ|anM3Y3`3y29sp@P8QA@jdvj8qaG*@N3EuSv6Us+q!7+g8@d*7jU%R%DtInr}< zeDBRVRvuGh?QA?W<+?uEE%kN(z-hb6jd?d_epbq(N-_KhW`8Je z(P46RtrdxheaJ9w^%Hv-gazAs+nJA>;^^dL2j4}_ipi7ih}bR=Ii<#;kDXWaV-ZEG zr3O=#30*+x*`v;867G&9bMNRCmxxXkMS)SP5EU6%Fi?jY>QxrCDXcy3YI=yN1I9f* zc;-Am<(_PPLqFNHj3L8bc&PTU) zsbBl<%7moI@k5C z24zmAvEh0Uajb3^PVewG2?%bOrIrm$&Xa#aKpc*3yFZb{L|Wo_HGt(k?z zfFbG}{epLftUZB%P5HQCYCZ5?a&ThRbFGYX**4q^Wt`Ui=g*HmoqKxp7B3pSf4IE- zT*v5w#7FO0l0t?kR^iI|4<9_pdAn`=mj-p6wb3+p2Cq?Rz-$*Lbk(;vK+OZWQu+Dq zoIcR?nBFSwUN@j1ZV)n_`^&b{#mfNg)P(o1p2_VFp1*_Mvi=^My3w6Hix)L zewgHClD!z=9&hRZ`X`j~o#kH`y2!(S!E%Ml&|~TdemYPCp~;(8w;Z(IS{N_wW%(cvw2-p~H>>RRL&bV@qS0v!4s z3I7@6dkpw{<#L-;{ z5yE1uh;&N*6q5hSTOaL>-vx>t#n-tZ6KBf$3Sf{$r`C*66|oP6Kxvq5gH1iM;< zO6bcXXJ>tM3DJ+>VAx6)kyRI+r+AjKK-h`V9z~I53xnS7L6u|D4(ACFh2WScE=u~_ z@Se+}vY1hI9Q$r^K*zK@I6Q0F3=CmRRFqJdXoQ>5L? z<~f@2=<>EVp~mhzc1(}-)S>w-5~c52%~K_??$Peorp3-bXRe>pLx#LmvrR#1Ziyjq z2~ry4_iBl*)|0&kmUYi9?cjXwE8$sEV8M4!cor@YmB9WCj_Z6<0mvPx^a*RKEG0a_ zZ!32*as7XF1_BoJTrZBZ1AHxF#z+T779YDU;Gv_vRYwuHkBQKM@xg69f4JFlRTml+1ec9;hd0m8Nt=v+2} z>7j#C+k&a?)o2&3vQOB=DYT zjv3cW7aCv}qVhr&=#6!owx4xyd~jU?ZH-Yfy~*E_!!>o4WD(uzo zg^H`w;8r+k=E?a>t*0zyPlcjArO9mI?O>(*n;R%t((Wp%_=8yhr|oS6O?q<=qN7%i z7&QvipJw(A7cxl}o7Fs8$EPPZQkvPs0WX%vCwxZ}w($*pG5bOAnwNH4Nx#Z7TNd4iq(E575p4H~%Q zOZAI?ZD*j*wLoaey2dugdY=7pk{|nPn$%rizE0H!0O+%5y@l4t$0sV#l9g(d#eP?D zoI*~OdX={i;mZd=Q67*&U!#BS?C>W#U8wnF^P0|AKal>;>{!#i5oYZAl2C|jHF8Sc z8KKblO= zdtGG4``Qgx2ljc~GuAL-N>ZQ0RpSb%jFnlny8e0va;q9i^7p7wPi9{aSt@(gJ@Q*o zR3h^ztg8Xp+veEj)j}MN+N-1}zN8D_u~tI%cS#u$!%Mq2N*ikl+9nqb!Rzk{l-?tV z?Lo_}VPR{qN0)lT&@K`wH#Tog)F3o}pr2b%cxJDSU(IqP`ePC*P%Yydy7F?x^FM1m zb^RP)@bulz-3vc~I`hGcmY1A)MFEhA*^=|zxo)*zAD#5Ch_f;xnSyxxrMbahB8B2j zz>unyT=|CRLg1+lcD0s0L#l=aKWyy`ZoGF_NSpx!2dd)!0tO&zg>Gz1I+yARv|oB> zT11WDaO!l=*eGtIfte?s%vRe^f&EVkXu77BY)Kw7W~Z7A-^8Y?nwsULU*_b}QYVgK zOjew~ZN^lDsT2#nAG3Gu*wJlME|l$PB+=3c_p&ysY0aG``p)q1@JKIFVv_xvUC(Sy zP1-30H>!LH6^R>AD(-p`G@%0gKabbaLPPuM+1bfB)I7(3t;80e6cU}Y*aa~eRqrG$ zB>HzZ&>*}!pZ@}af-WsCJXkxH*Yz;9#Z=+g)>do=!o?LnN8hO z#kj`xcm{8$C0c%EZ6lJ=OJ6d&X~7vb!p_c4f!hm}@k{x~K9TK(scpjOZF%LjU3x%H zl>Xf+n}+)*pM0G=?x$tHp2cY;3-IsEYnySi}2+2tR4X!Hr+H(GKTRoY4hfNSpZ}&5pg=sL5$4yIN+{(xEge9eEYfs zSBT(~Lwobi9hd4ZA3l8wz$Ka&+gN|#dJ(P6u`=!!(n?zaW)mh}M$*qEbK%11MbG;Y z>OgpL`C%N|wxhO+g##Mw;1BS(k(ZX9h;(_`A}6^<74Nyo@2hfj{8z0mFZ(f$qYaAm z79@cH+Xfvlgy@MOWD#01!4#wSbP>OyV7WXw!}80Ly-1F(;@FP>do&KPPHI*6i@+1g5)npJFxAFWLIFV}SU8+~qo;^Gn-aiMXihr5_q>|@NOrioi5Cm1BvYvt) zh~cLUH*h8P3))8rfef3%(TM~h3daKEGre0w{f%{jH6>!i-_?jVN3w@{OIOivZWaPc zxk`?nuj59OAP5qd%bpacxM`i`WGXNFE>e2amOU?(3PdgWt-&`L5#s8f9Q!(V{lzQu zmQMGs+e$y6M)07-cTaKyQ%Z7^gt(M^heMUwpbHcPLp5Q4vwpj2uLO!0+mQ!y$ZIqa zl4Pt1TJQ{uVcR&t72^n9PNsX$=L%E12~k87htLiQbD&+xf^Lvdw3TPo87B)u2XhTn zj3h1zM6^A`ZE?l6b)RKq)QHb$+#z^nsa_A6BzQ_%qjtADC?vzhlNd@y{59ynLPKwz z|3u9qZ&7T;rTVK@V!zy>wpCJM534s}=S%;<&0@q};{@K}Ev_W=B(JZTVxd4_Pz)QoW$WvZ)KKH- za9v6|I;$;!R81&`^H}eRQbeg%M}@WUw{>db5zRoaVogJ}QCqup4auy|VvhO(VWk7e z`xSe08Rj87S#W>)3xtX&;;J|rgi8rMO}HZjTkTbWZmZmvEE(Lr>uer(R^WWv7jKrM z(cGQ(dioUjbYaXY74x*^F6vZ`9ZyxHqmdkid5=3RfV=452VKL3bI-F#sia<2HiX!mGKqn`%IV;$f{Bx90N-j&k zq51wtx%zM56*eO1T(~Z5C?U4`Y2B;V=1J-b79uC+ z$T&mP1md-#FMZ`RLLgb?<Qc*5Mp4QdUH@p&xBB=h`L8t%>V&NX)2)l`srV%KJx^Q?7}=&yOSqyxZ7&#+jJ z7Qz0##2hmmK0Y^pZ^9J$TQ(+7IzhW#LE#ZjpWag8T*hU@IP-|eocdCmaTXgq*%1SH z$DeEp3sbATtQ*6pSb|C02BojXT{N;aAN0K5pK`hG@J}wj*vpM<2HVs4Oadax=Pd2X zM_ps%um`y~eM={^+ZMB8pWOL6E^pAO(QTM&@JKz_c{IMhUB?+b=d9Ie4$QqJ9U`u) zRbfNzxXxg?T(sW zybBsaj6jxkoZrlxJ)4$%CDPKoTbDCThrifhN$8iU&dK|BfI@Xrdbp#_0qyKGsZh0pWb-;b7Gad+Cz zjh}l5|Q|jY$J~WT=4e8hHX-BqtWMzT2ocdONCO{L&1J+I(DorivkKp z1tq!;9$W*g(7(8`530CHkPJeO)}N^92V(Ug*LZ1lnHRdHWa9UHU*HH`nhAkL8zCr$)FV{u`!Po?TL3YA>; z?_&fbN%f2$WVdk&TdR;%Lyrv42jr5Kt|QVLi*AGbCly=A7Ve(}?Qn^fPNiWrdCQC| zj+2D8Ax6XYN*C;g3Jw-)EVp_$nS)~CARF84fB*##f(jH z&FLISa8ya>AX}6QS{u^hYz8r+hz>}VRkt!W9^G0Y+T0W1{S_HW%HmCg3PnHyBh)=5iZhmjY_u1_WPA~tJU%9sPRgZSfDh0nv|F|bu(A!6k%qeUA zP;qf6+dt2(l7Jl`!o}oZ^XHJ(8o`o-WDDhfi@0> zSHy#w9Y$^CM@!Vhv8N66u3LOa*fYN)gBBG~ilEnzywlfFnez3>M{7eNCFrU&uE|L& zqgD`FP*4zmZ29@IZAl{8KbWx(-{pL^dqk&k$5Hwuv~$AA9D{`Eo67i% z^vf{({+}y8!LYSCyaybCj*<90=FB;&@F*Bc`RTXc6PUh{&}@N<(AuCnbXX5LeHkLI zU~Ef$`)CT>E8-+Q0DEAc!T8ikl^V||$jVRCNqr?3aI!3^^~DamEc!yGPa38b4j zVF{0f#0BlwnJd6C0tT79^JHHJXK4bTN4>rqjtpCj{)c}4_@TW&r~!F3rE}Z+e7mhv zfx^g(aOkr7${sawds9YXQzaZ(89grlJ%T+PxK!2D-%BU|dcXANpsqR%)I?%)Hp9G` zisVZ?d3+!6CoPw-te0;l4t_DKf>Fz6Q8pbcE%8wu$Ra`_?n?oUKLJ_hi%22^{V2&v ztLf~%gCh3FAFI_c1})47x{WzLp`3VYBqu-Hce#g$ELgaFGtdNfNYd}sXIU{T!OTNU z!q(9ErY=hXcHmh`yw2JSmxye<4!Aj2>jNgNK%Jpoed{<58B@1*a~&VW{L*&{$#FcZ zB2To9nANdk$6V|im}Y;yjLF6po(j+%Q^X2-tMfP|#6_S0Ak~ZX>C5n39sE2%iL#S~ z9RypbYONmfoC%X87{b>l%S?27(0~iNX4$>F@u|7cHG&~zpY{BMvAgf@7J|@;pwbpX zLZD~}4ts=}zI-^?#(u$qlVE1qXfaNmJ}nhX`nbI9pz@P|>n?2z4~!b#w{HcS=ths) zk!HhjUTQNh@XOMkA~5GGDY-6e$B6_QO_-$aE_8Yj$avYfhup7gN>DlyhZU zyJZFO_7L+tSu!Sewq7fTqfscyk1sYdsxKfzn62z^%g!E=MYV#a*sa_3l0U45$jM;x zwY0d7sS~gEuD7mLKg4aTc3bmjX=yFY9o$JG`#&R~loTlQ@q>rQ&(z=X{-1mgw8kCY zPSij}(6%XpFjfAd7w8t3-fL>N-6?BfI&0#F)QWl!{OK>*PY@A~*_0_-^JYeay!!ga zRQHW(XlK_>_g8~jMn;umpZvP@+NjOwmGG;nIpP~e|95XBI72(iEyU-LnMy`WGfU^p z4t1;XGq|n{bubY|sup4WWDgh|qKii4kB)tZ)uF+K|Bi0Nh6JNk=1vmc4rHhZ!1*%7 z2+n5>ITP@ZgrSJHQ#K|cF-#lpqso^|p6v9y-`7C=wdYU>12wfM zz(`{0mxr`$o&Q|{NJ(CTOvBZ{_a?zE;b)si9bAOyj%>|V2gj2-45pjYac-3_zavn3 zef)Ma?P8jnl}sbt%;(p2`i(f~-!++buI z7JKJ^uUe5YAfMtezeLfPZ_TeLAsFQ>{~y+qmTN>Zm~1 zNhd~RRvW&=fE@zMAnt?234wW#IW*Y1cNUf zT;+z^qP7HY_DQhQ)?yMGCMAyX(eJ@ZYp5UOhpqCbR4@w4{G{&uHH1@6{eiSRU46JrYwmh6&8at67y-+*>N3yR3S=iav?%h#?OVCmu@y19}vRJH7*TJN^n_NH; zecK|m?-P;Gj6K`zDnk#l?AJV;J_Wn_5?0%J&eN65HKUyVT?1m5BBq6?+wQ;Tv|+!! zG~JI5^7ihqnUiYrjFb8{`{%{`AM!<80*<&pkG*V?HnrM)TL1r6&G0V0LL|))X&_Y= z&GB9g^Ego)vr&S|?_PaTo|L;A&;X(OP#e0>oHIwR5`f$o21yi-)GIA5ya;O zw+QHvR8;gp;<2eG#{m3V6|--jHtD{!2kL~&6FEqI7Ss#~mMuT?<_$?@3a_7Bl{kiu zgmEG~nZypz)bXs-P`p)Zs)^O|%8jm0TLTlfaE<{16sMOB5K;!=v71+0EMBtY@J#>9 z3A6rT8{Q%0W~$_!UDn4OYWfd;er0EZC}-ls(=ntmm?iQPuGcYQ2| z-`gD*+!%6J`oN(!h|_T!T;*Cl@acP#)7*M5x!u(P7YCE?eCg7q$d?!GQk?dtOc49* zvy9^Xc?`?6ss32i$CuUG<_@*QrAUbrmEbB0tBzBhVl;eAJ1+irkzVn9bdKJWY4?&p zsR{ZdajAQ7h{pD}xkYxKtbMn(E>;) zh7~icKBi|h;73P|h>2C?uGpHn4ir+qtQnkYvC%m_$njPmtQYLk2k;%wyW$yf@if24I6caly zbhd){KqJfDY(Hz(VfEY-BXjO{$#8n_8fiq7jLT_OqW9aoW8HRq+DYgyz{VtjMSpd- z)f;%sWZU;O@Hus3JcDRk7C3<m(WII1%n(NIr(;Yw;y^248ruyphC4e!Z6P^<9$q z`Z%p$HSeql`jIhB)yx0#3aQq~#NnA-L+I{*Z|dD09V^2?k^fa^^P0@N?u!=f0d&Y5 z6>S9L`LDI%HIiH7k_;0D?L+x`%N3C)&reEL$T~_{9G*=2*lUjtCyGI z!+%=?{fY!tMcwH9{I^(PnIlf*fAVK1o6K34)j|gU7&SxsZ z{=8+K!mzCDi;jNDu&sofa(bc00&D=2fcg%5$dX!d=>(?Ry+~M$pqJ-Oacene%1T<% zOPxhSt~VE};WT`}*jk3?v)xblEax;6`+5S5F}nUb8xokfjv*4GF7Jkm7Y9k?xWHpx zd$UD#qn^|G*~NY=XL4lBXG553COT{-ryz5xfVdcJDvjEzDncpknV&ks_T8Ew^A5Xh ze5^Bjey96PTFF-sMSCovCU{1wKHbKmy}{OnE3X_p@PyfsQB9pSCnK(*aM8j64~zvH zBougfs)XwEyOi`xldub7d;nKs#tB!94-`0wi%+K?ahANK5hGr6QaHl+vw_leIKuBy zIAGUsnN_~xts6HIc%YV!ul(=f4KR?t)~RFrGAvKnyN6=hx0^@$dswc^*ds> z@lW(OvgwTH{K4U4$F51(?fi+0<%+4dVaJZDBnF&)c2t7h(I~g3r^J;&EPF#y`MQEL z(8JwH>e}*^LjJNt!q&mu7z#}hjP;EEIvX){sZc?f2;GnIzp`ID0a4k#v%PLD$E04su)I8)3sE z3fz1kZ`QUpV^-0xPu-4!_BvP-cK`050LOd&4@qv$q}V5ln^UiWwUgc4V<%2HFKP#e zQtEt2zdEJwbaE&qZMaEw=S+`+oTkFuYCtQXCVML|7#vtG&o5fILqCKKs2nk9Q^JS1 zO`680COV5~SOCxb)dxz0%y?cjDc?^%`?T~&-XZmM+u?Ky2M!%_P_Ym|8zRQ3?ohry zuz1!0R{nIG!b16NYmjBKC&9YMVUU>iTb~>lXO_%BW z`^cKa(R2!(e0?2T58JzCpwN-f&vJ*%hrnIPrOb+)5sfx&-Kwa=di2;iD^5`+$ia2) zk4HS}M*B^_3$FHoagxgG+<#=an;$_hdw%ZSo)bK%uie}$ksEVzqeiw~v-5mXHzF43 zwD-2wdxaB&)j>=scOGXul%)ffi!UkEJ%t<^CN1tqmWB0-zImY_XV{?k;0bx(YiW)a zGPsc^CIysXVGg(j$RkobJa|AM`>>C9K$G!`B{1{4?{K07*1ve&6sFKqgqP z;C9~-5@^R!9Z|jAHq)v*JI|>_6OfsTo>%rC^(1J)ErRb_P8r`pxtk_CpM4T{uBsP| zp~8WwYvK8>ixUtrxl6>_74JS|1E}db+KW8nHBFiLe*ha=^y2!Gnm2cmCIs6KP6tT{ zb(=YJXnu2tMA_{4VPsQTrza*v`jPR=oX}8+^adT?!rJnnC@+IA*dNS)-Mv;u0v~$S z>ZbgHK?lR6dO2@Hm;QRxUgy9FL*!%WE9_fLgav-k`bWJYFFF?@j{?M=kOr{e=jX@& z+&hjfR?<2WDQBVK*AOZ;!xSKiTCp?(VxnZ;c)y#=10>9{?owY#4R*Wm_J4)B8Doz0 zp(WskmHl^H$R_)al-?iFq+koNf-Rh;;*kU(;C*uwkL7sKA?HMF`5+gg6@^%;@bdE| zuVzIgce!G7dEx&hB5U>R_M#Wo{+DwG=~idKf$G!Ziy1$b>jzvYi$y2UWQ2$O43)d$ z@-ds51sC|RwaYt6mO4tZ!Z(Z1=nHL~`W=zY+YL@p=BW~nj-0*F!_pp{LH!it4wUiO zklmSWp2vChaC>Z9YKAyNL=27cW)Ike3C244AwPv!CateA`Z4ShZ9po}@3s(#wqaKMmk`|&X& zY_uTDB*Hc9OhkbBBgFBt*&W8zF`E(o-vISJZE_Y}E{pP@I*d>q*thReO3Zll_B_xo zR~plKZrHlDa~mWLmq6YRWx;am)!b!5V~si~^}_;ST?s=9Fwq%?U<@OA>KNDsvLneS z&>V%2kaZP@##^r*Wblb$N%kCM-QC?|o4<&@MTh^$l`nK$m<4M=fY z38C)6@g4idZ51<%xFAlC{6OXnnbiq4%cHdDi^-PZ_E@dh9>i4101s)*%Va$eHAsw0 zwubiiIBF^1Lni*0C!h+h2g!0Qk8-gd)?Ues0yH}!ry+qI(bn@P2KL#E(PxJfAH%w* zHT^$SWNY5(_J>eMZtL!N9b~AK7-rat3^@G!&n`m#ARC9RY&+9Ve>$w6(6CwAP7*A8 z^5nVuSr)X%syU@}@Q`<~9F~@m3Z<6vo#hy$oXf520V;b{c+I$#=C_zaUvjmo*|ta+ z^~;+?cX8eiRajlKcFH4hFfz~m_N29?46E`nMIw{fbAT96**bFg$dNbiECJTb@-bPR zqEGw60JMa8- zG`=xp1S#y^_j}2T6qLU7LdW@UzRW{Eh!E%GCy8dUc{3Ad@&Bya5rh7#Y!DlW0;Mt{ zuc5uN^ZO7NAsizNpwBmd`>LeZE1HIfAM!;F8PQDCkVB9JP&f~JFj7ChxTIZnxu^}V zIhsTQ$T)Xe@7-S`*DZyR)d&%O3N z>qgY1lQ7$;zxR5JQ70^CO#l9BHRQ6lx3$QwL{290ivbqX9>drO#Lqiru3-db(A%Ip zxVDDm4*LIBCBGs{_V&yqM|(aS`aG_8@D?vt~r~=kR$n0@Qaf`{P67{3*0cZ zo0|{cFOWb*;E^Wd@&xEWLgboDA;iG8n*?43-iSC z+*`gc=9UqY4;|!K-IamnqSltLh^u#SYTA2u;R2Ut$7s>-F_%hIjo;_OrLr2mEUBij zv6NNiwWh1!yg%&jmNLO%I=eHql!6>H>A}pLQHvW4>e=A>+RIPswRNx?Hre}|))=EH zRW25Y&`YxA8W|h+VNwKmUil`-@V_$=B{SvA8wYVgAwUQShZTzgm{-CSL`BE>T%E6c z$FUsQU*tGD(tEN5O2oa?-YdZY0qa@2m4^d_G(}(Z8$^j&EtJkz z4)a?|tAJl$kx$nSN2sCx=Al`dm#LpUz{H~hJ40w5 zQ&6^NbzB)^RPDx|+wg8QabO$RqSbxGSQi&lGpEgx1O<|&6OV)_8QrmXR+J)MPd~_t zU7T{hW6qtc%N{Q|gGINI1nD*GzEW{ja14QmWhH$@)8Mu+-h%S0x9!*wgkDB`6@mMc zvGdMjf2(S6sOd+t=qXp=yylNzyl4W4?fdQ9H;3;ZN*gllvi^gv&u9MfWmkk)H@H&| z%LTw$zry{i-8^^@4LZlf4tYb*M#=lG)5|f;PeQ}Z`o{m(lDU0Dfa9Fms(*(`@BHez zbs9Tshg9M|wX^HeyPHI9*#EHqTb78KFrmtQK~ z{P*8$4nP8bE!nvrz&%O6%z~VW3bhA|FB@)N`30CuJomuTJuX=aEb-!9Xcu{^5t^Il zGce=)zGs_g=gdlm%5ApqR181`NIBW<>#VV6pU~GhY*+Kpt%;T51h|y#jLdP(#lHrW z1XHXx^}xZ!)St-jG^6Jw@5GMO{srf zA8#~YBfRa&Uok3A;!kefx}}W-#p4*`B3MZMzdyh5wUAH7TfaZ=R~1ctItdQ4+QUQR z{q&v&Br8ZXnV2dL-Wyl0Hh z!=^!QE&Ww+V(8h^rzQFJ(z3b6rE&9z4L5lrG~}lbw$j3~Udo;R{rj)DJb?%*XN8j6 z9&Fx+%3C(ziJMb=@MVOuOxyuQrVc~fJhB-k1Kzc3mYq9yj(*Mci)=!PT|Uz7(Z=E% z(h(8dqGWm!?uxZN*VuMNf_?7)ILD5Tx5oIy;JDiZzsi%N6&@dPe%V(u-lXq?>H!SP zfJ#2pAvCRq?Oa=1Iv1S8)|U1TVnErhWD=EwwvEM+`;_H?`2KNNOs5mI)u1yoH8Z>C zT{o(xEc|1A#6b_>+Dk1qKq~>U{`&Pb+4jl{LRkGK4lzV-(WCSyG(4 z-e-cW?l}8owoFEQQfFoqerKx)dX4Hoibcv$KDo}jH*e-`f6*ExYS+ad>!RapbZRc~ z!3CLn*DsA};%xBY!-u|XqEKviX){b}-TJgxsAL8o+n*o^5FKnwxVHSS={KRH{qNPE z`|QlxckhXC`p8$;vUuBXjYirSkaLZ4-(A{fE-6>@kOr(lS$^1SbLDBdot3= z4?`Mt(7Pe2U7$gL*-lKjT;BF(R4}qi*o`3)ic|bSLU91hXFoj^H0!6}z{<F z6NtHSb8g*Xa<`IDrA9Z7s7ai!N!0O;^`<98EUcZAe9yVs1<#n0igs3W{s}wv*e(A> zLc;%}>p$Rn-v9r9JX?0ko+Xtdgsep96(YOL6mhcmYAGuXqY$!^mk}=`dnY0?QzYX^ z2@NA53Dy64f0lFJpa13i>(b@C&v_q*SFhLOd4JsRxBG2ex6Q|%Kpgkcz3Rb}rR!b` zg`vTHd${akC34-VN7I_nlvc*X!~no)!)bxd-pj#ZbK?7)9Lb*q7dnkc z*AsMP;Okym>v<_t8-l&PmW7#Qmx9)mIOoOTgyGLcmd0JHpcgb-n6?4rWdWtZ@z&GJ!9H8!*gFn|>6G9?oUh+^|@hAEfHlMScN%T%@^ zNI8^nIZb4_E^A_ADqPtMXZ;!Ut=qP}Ng7G{mVq_eIp>#O26B&<*}sTla1BqpTzuUftnh6lks%NawT|=!$MFN56UCe3&@I@R+;s|1n4E zlyPHSBkc-EU#F^+DsV-1^v|pu)5W`( z$>H@WA#;{~iKcUdoa9Y){JQl$)?T3nrS&eg)k)Y9I_dH27dzb=e{5)dD|1z5SJkkI zB!b|uJK>=r4)q6D`;}*bB$QfFAp;8Xku-19W>(+%WC0n#>SrVvq2L|iv$wznN(XUN zoaUvsh>!P+b%FD4avIZ8%6^(v@mH(ODGUmdvdzv@Rf^ti+_`ft;Nro$;WKBmQr2y4dk+~BG(6g?Xr+JV-{PTzDpcXQKWSCyfl>{iPFV3} zJ=%7mOkDoVFl@((G>pWwCU6Ph)Ky-)Y~-fRR-wwC|l{;=cSf9z6}8i$f(g#auGId-1r&RLnVf@ zwuqe^^4PW$Jm}ukK;1>>$CVV96BAfXtw>aDr77|%F3cBd9vcc_C`-f%PUXJhL?WVb zv0e3*aSxzx8`wTi88}AOFmoaqKO7;pi ze2Rz9i6uEsQ_nOuvw1BT5Gl)=Da)M>fCG*l8^~OGrHhe)2cZY+eSW7m>z!}Z+7ram&-K-sf}MTaB#)r4;-a`# zXQ-5RlMnUJH2iQo@cISg8jhI@t`ya)l`w7`ey2Ch*PQI7a&>imXtwt@4We+ME?j7U z%m%aw2yfT-Jt4v(T9Oks{8hA1GXP@8%Ib& z?#Jhy_Ag-r!BYU5ZC!e_^=)gjMH3#sLL!2F#aMZlK}K`{fS7-Q^o`{Z&U|i_qZ@!u zU!xIrd`o@x(0X0}x$|~@R6=C(^fTzxaY_3SvVM1HD0uWARC-gMSI<|C-nx&YI`Ca0 z&EIGUhQ|XTW>cRSF59{^S@Hj6z5lTGkL_l=%H{f-%rEbqf=njIE2=x{F&YZZ@)*qg zU=sU8l@lh?xRYi;bGSe4YnKMNgpesFR^HosSQsERp_+kIk|v=60Nv> z6=#t2MX|EVz7-pr23Za-?Ojh^5phcj^31c-3h7xkJOJWQJl9N2QiC(b@TUgx+d3_Z z*3yDiDCA|yve>9I%;XS);v}Mpxtb{4!|WS1XrMUkf9uv*i)|(7bq|SN>2)%5arWaw z;rW~BLzP58ndY`P1>IBuuxO@q*d}3Nq4!;TA2NCF9_0kRP6xP~zQF7Jgu#Pr2yuwx ztn633Ss3KzM;DLJGp(fsJu~&ioM>T0hELuAQG@jyMuYEaCTHG$@Zct#j2}l@|LI(G z5}HxM)z5SRrImB-IJZ+SOb<#J)O4Vvanp)HzHaYiKaCVcv6J5?` zzlKU6mIALb$6;8zmk*Ci2~lku0OxMA%GF=6V1dLNmTSsTbz<711JeqMzTIM{fYFa= z(e^LlILk?H(}oobdqZT2f~V(su+8u-F0d*#E@{=dXEv~0RQxHk0it0J&r7>LP0f=I ziFvp8Puna>1?0&sVT?}wsqi+~d_q9Ew~w zh(eM>VfQm{%^;$_J@CpeWzBIxDtquwneVTzi%dFYY;0VK<(!NMiQT!d{IsJCKX;mV z@mLVM=PV&3Q|EJjFH^&~Fg3MD35Ws_`;WQ79fy^lK}22>|23OS-4LS=ByAThxyN|@;N=Wx=Ii4&}3zDwZhMX6>T)Kfu*biRenxWB!VsA1%a-|#n!IIH5{ z3m#rQ${5?CefwcUHJSQ%=Rfqke8_PVC%-zIPyl zKPz#H8U6$QlDQJ^TSc5^#qdjPhW_hC(%FgrA!;ZNmAkr+8v0kDD|;oJ#Ux~c@Zf#7 z#1sMdQmh@r#L_2i8j(Fd#lKz=FnTci7P)}*h#kc;r{x+i@EJ*RMmPKIj?h8Hf<48j z_n)3bWp2aSO>oq8>}j>V(#y~DER7RRpUgD;aF&~SXK%^NqmrMCad;=dj>(ZWfgMw= zzPw9KKGuF=(2&bg${$ZXo!QASt@xh?eH$AcH)vU7VC}V|&zq$mjeZMkBzO;16^-zo zef!?*su%bqYaz|Zs5Y}jz)d+`()4-X5vNdr3u0EctVGa97ypX>8eJw~ApB0TEy>QF zuAmj*wR_H=A6;6_PoVmtxT4xosyU=$s`)^KaN?PM{n4Km;tb}&{s%^-Hn&AHPAMuS ziSPMqZu_F!b#K+R`8Pmv*P5-R0g@%E>Abr+7hE?&wad&jVMi2E8XqZCyDxs7UR}T3 zgZY~W3sZvDG6_0fTD-1maC*V3LsutSHBRlV+LT4kkDT;JUzV;33}j^x@tB)|?qoF6 z_GvF4gDP#>)iv=MqqHg;uG6Y~vh~22i&j#eXnWP3_ne5{RR?@+IOt8~#@~A?$!8my zGp#PlY4+Lc_yQXnU*(Kv09i|zj6)OyP75ovQ0mOr1RvzO*+*T?U#3#2Wt_vuY?R0} zq-EvC81gHJAkA_lt^^)MiBPp(+nwKE`Qb2(80~O2q~U3o>JHv?iCI|;)h0Io6Eh zH*prbN^}5|9=dO$AI0TooE&utMuF%tJf8mD@zu$Zdw$;SDTiQrAUO@#p!OE998S%@~!OcFcbDO*5 z#=4#nIeF1U6oT{tv!3<1AZFUh>I+4srpv2 z@19!)J&hmachiYD*DHEvlgl+f^=jJ8`*2PM>?Lp=apq;QXws&QjjQXsaaN|o?d|t9 ztY~g&xfUqxs_Ut!r2M^IbV(0k^{LF6l7A?787XlBxJ;pVfvGlReX%(OhTw6iF&y;L zwIrNf;Vn#`cL!>Fjp&^O28pk_DBlws0lu>_B4~qdDB+ukk4UvkO$|>3BTy2KiP9Z9 zS@z;DKGqF+^YUMFx8k6v?!|%GPWd;AQ##e`rN7e7z{H?y*}&J!sys02mT<6v-QdJ? z$=RNsFMSU9@XL;U`ZfOWkiH8Bn{U58Mx~mob*+J31Bc*4RSv4(6&7Aw?A*myr_PVZ zpS_Qs8RZuC{LT);EtkH3c4tg>y5wV4e-+89d_T&n_RQ$}l=rXJ#b0U8u?T$m zKhHQw%jX{!l&4|9XRxOvUFy5`^tiM z_jr$LaEO!*8cn=Kn`kQ?->oSrPU75BLRwM5?Mbt{y z^ipv9uvcx5Q#O^(CwMV_{W!stb&vI zz5T5HYUDn1Fc>afiI|IH`R8|SLM`S2m;11y1!Cup_r2_j+8abhG?2YIlwF3DBPMOG z@=C2Gc}9wT&I2;EFfXd#sL_Z#(&3ay?L#IbCCwc+x}3Mk?qBLG7&u^n#QZ7pKokQP za0ST~$L=>BKZ?y9CJFjnpmxViur67{fXvSD%KY6d!f46W$B%zLn!H__|B|o3I^||t zeX8L;cgoXpeqBr~Y7Be4QETR%;}gbjZ_v;wT9J8IjwsD(ooYdzOryj$q4gqeMK;-6 zKQFhBc7k2~+}jUL$5yzmcBuS(xBk;dfmhQjExVOBGTIsXDb}6jsyZGScdiC1@MVMZ zU7IBSvKhb3j z_E=EyWfO$ZJn5<$yIi*IZ8bgCefj#eJ$@U?4wmazeehuBI;%&zzNgUFuu}YsJyaxl z1bWAe!ipDOdbwhw1h$C}E4D?0rujs_s)@fv#G^wq+a>EdldPrW)8x8Qrj5qG@#;{* zB`nCe5$T*_XEfq*yHIow!aU~_7?dSYIK1M~(cV#4C%<;L3=3{rX{2s~irXOXLOHjh zF_SOSB_3WqwWrsQTBueUo7(C5ifN=K19P>KU1dLb7I@(}3JDl5;nCq}JyKHXQzizrN2ZU$)GC zoW>OLCK2QHtG%W{>IKRiH-jCV2)?XF>1^nuB+ecZXqB=mQE@IpYCL+>hM15Q*B*Ad zCbm~xfxGj^R(*{7Cf(Oz4=8uHgmfjtg2xhiJZM|`@qVntYSE&xM0FWvR$qmSotRYf zTifPK&yQ$`z_j*~`)crlg_Rr)#qFuoz&sexz+}wAhV|=T1rZgiyiuRHcp9TWkWe?T zC(C3uP`)yUtSwk^zWax3yY##Kyjf15%x{BV=QZ>SZ4~sOYtJdx@?a(`*L^L&+C+0N z=uvufU8C1Q&;lCg2Clc(OEM2wu*Y_1)sH8u2mkV{l{wAkb5@JypWj(q??&3h)~K2Dy9q&k*F*EOMMt&FQCCpWbqW*F{J!lrFx8VHI6QilGi%YZpt(B?pppiWA(Gv}e5QCGzdv)ArHdv3!OY|O(X_UR1 zwcIXIh4!~PksX1I0k-Oa9rs;FBvEvEp!Aojr!E9q4qgz=0eY8PS0}+7N|kJ0_w0=J zNQ3i=ojIAtCvH=Ivv1r`DM;sTwrr`)7ge)-$PQ!P*?A)8{nrA;9j2YJ@syia;KG;_ z95P&96nM2ujH-Q!t{fe3r#EMiYqQ`@vo#B%cZp}U?ribhr^y9Bz;pPiM zyc>T0eCVyQa=1&rB2%@?Crre7Tgn0V{28Wz1BLzsKj> zB5lLZuI{dimFnJ1{~lavO0({1+MkD4NwC{!Sbg7xuY0z2Z+7z4nh8s)mThuQKd44j zZqC~hV+O~I3$?jw`$5iPaCZLnUbtMK^{r|i23W%AU~9R|DG3~Vu@95Yg<@Z`Z{Mml zD|H7Q=g4r6#pL9dbL&~{3)=c05k5bSdm0Yr^?zFruEv~VU(Ows*^h#`+uIPILxZQ7 z(PNSp1K<=c-AIe=J(*fH7b7EUo;3Zsppnj@f3q6`dUBvDuADgFdQHfT_&JiOqx3)a zvvfV%ZhgObjq{+031U@&IKf_<5L~WFfD_trY+b55%zC%e8aVzn;D80k0HJ^I-gTI3Nx^XsVQs2*R(FRtYZ z<`7LKDC6b8+zx125de#SqD7NiP5;G-bOyb9cD`AcivOJN_-1*RxG~#Rr(zFoKHlSQ z;9FbAn!@k)SJ%vWtVER^#A$k@;f~-&k+*JtYJYm&4^=;dK<$kWv_7t=^RVsc<*BK< z<)6>d|M5+y`H&*B+lR}q>@Cu5A@+e$v)EhBo@-Q%`~b!w(T&2p6Z-i*-}kN?N{nw9 z{r6c~>Q$>K?kZkI1sRv)`hKg8#vAAtvz6XiaFN7})N$x;Uf9%zBxNZv;xC>*PyJdT zC;LEPTZRk}hvNL=BIo_`Nim8~#QWv-3djxgJ`NvgV&--Y4h=QoRQR^sbbFu1cwE;V zu2^N3@E{divLd3;yu2C@ju>&~g5xpU=%&q@?Z%g^a{02Tudc<##f5z5@Rn@8sDs3y z%qJ`U!RYTukl$ov96t64({?p~-kW&aBcVUMYaK3b>Yo{}P;mlwTOY7yLjwUPIr|Tx1qUJ92kH{h3zbBuFUyFX(~GAjG2DI)#m*ZASfACbFMagUIo zLQM+CLpGfXBPV4%oGf9jW)o>hhvhaPzG>|!tOWg_*hz%O=U;`JK1nv1Tr|mVc{me6 z6LQguuCgrN4Ywz{C@&JW&5DgSw6MM)I-pm1n|*c*ulDKjGD73IHX-!ws?w6r|D?66 z6_W7!v-*Tj;~o!g$DgU4aPRiOhm-PucfIl6J@PN5!8ZOwF6-DTB|P?;ulD&KZ>~MN zHQ24}$;|$$_STzrY7d%kG&!j1@&}hHFH9+O^H$1_{46p`f!~NcXJN7iE!!ZA8z;bp zI{i3_zcl!EDE(OX!p7!{fL=tcCfPdQWTD5zEbf{(nR!d{1kN2veS*<4kk4MG*-2qF zc=V&KUu|d`{5B(FIdn-Lo*VHkADMz@GJ^^g)I&->%Fb@VE7g67fR=K>;3?*6)DkwIHcLm~?L!JgQ{0rh2nL${_)WAB zef}JI!a-~t+BseRu`cLsW)k4Pq`xH1TLM>6N!l`~(u}4?j5YIw-S+k1u)o^3*TKqG zjIhcd(BJM3jI*i6Oy)x|Vl?q!ym?I%GeQAzIMk4^dLMFmaW|u888D3i?LMmZN4fcg zPfn_;#+ls`44W~x0c7`RVD7lS{e^63C6}DA%jV#!`*8Y&;GBhgnlXS}))i-Ozwasn zc?lEfNW`NwT&PzW+TszZ zqZ8E#!vKWWEoxOh_)}s5nR6@bSC1wPQuqNJk_jk05|;|a3t)H>jqs_R>-kZI49!-+ z7rIJb24i@;D?e7m&yQ{a-4&$+T+FEXQG$-sm&w_8J>|ZISeQRwzE*x+mc1*}WqZY! za|^n)-Zb@WgEO_yP8oB;@^+l3z5=6aN(HhLaxWo4^6FK_$!dqfpwEl z_d0i8@A+?s%HPkn_#CctvhZ1>;-1hqMR>#5A+$68bTGWuk$lMYT~CcUb9^j&gnKov z=>5%Z?mVC!aCJz^p%D>~a4PLuAEEE>#Dy;IQr^FF`|a-nxdHm`|L8U_5jXQG@)BF0 zDAo?jlTv3Fn2iMs14qvi(u@DH$>gxCg@%uwQI_UBHD+wV0 zJ|uLytf+OrEVl$uibncm8E%xv0D}d#q{WpRO!xvrwFRk%L;u?D=Ey40#Hg}P>=86~ zk`ToYr75DW=CZ@Oe`jfZmeP`Dn}>HeCeeP82~T|AyULXFBxS98m`I%Y)+)2AQRmjc$(2tFs{Aa?MRf~Wy~X+ z>LDT9L?E_(bLA9Rc{&nTKT&fF|EQ;E1BGpox6}DR(sZP?68?#m<%9Ws_~S~x)SvTj zWLYnL)wOBY%fU8&`hmmTwLLzBHS=?e8ds)9J8vL zQBLhlbbQM(GVi!wx5H`A4shF)knGhn+isna7->B2_|l6+zk&mCj}{dbC4Y)cm`Z*C zy}SKVDhyWlvJaI9;Xv~Hy+zOO-?--P_;kI$5$Tnv&NNPBlCJfNglvndytQ~B4jfpO zO=ag6uY&wtGkm~wrc%qhrdZf7W{{^q05qz zPAn2TxDJs02{p|MPG)M8@C%8p?B7?lp?Upkc)kxv!ixTF4{MeANAZ(_w~yDb$s4hg z#hF^h%>3u8HlV6Z`e}Fd>+(HC;OBl8`w z@&YI8)_qp(pnG=Ax-wnfBo}vpP`Ul_%=*Jm=`{XiojS3_iXqnZ8f*)cvskta+^Bk5 zVeRY+zm%tFFvlOrUb7xka}Ie8sE}w&BzC1Vao$idvHX@$vd9z?y7P>z(d5q zH2CLpF5z6fHRXb)Sjb%a<<8iMHJ4vBemVP`ChSAu3V)XuH{$CVZJKuT*nar2lCK0n zDZI6S>#P0MS80PC7O~s23c9x}AioJ+$=>Mb-PWC2ykgqaPRHM>QVvxb`CAU>%Q za&>y_$0L8c7g$u$0v5YI)%?Npd}D*a$=rwf$wz6+Xxf|7N|D4?rQ4_qTJ(=M(JFVH zyUD-kG0YhOm&8V`mx8XU5)O(wHK^GwNJnuqV4?}e2_jp;aPO#Gk0>z+wC}-0%25nn z*^LtEB7PNbe%m8fuF=#oF20xf$Fj?{aD`gUzD~?2FUCVLI_{Z2J7vybm3ejOeM0Z% zo^tTP1wcs(Yn(KWxtnB~zO2^7p)`No36~0@eGUmRUjx$l=PPD6fm@5*h9!4d@3a-0 zzGeQdtICHP#;&^Gpl;~G%T}x87rG35Gw73G_XgW5y6&pj{Z#t#4$oF~>@msh!yDg* z20G>o62r%ws&p+uzkJh_UdJQq1h1U)&GYhQli|=Hq}gKhk~EY9!M%H%pW;~%z^b?~ zy7*iokhpBC*{nB*&hvxHECCJ_s}RL)f><9|VWo?(aV(SKGOmjRkN6^jFeAvP!lYD> zsoue3)G^0v63lX9G{G?*OQcuvaC58QG{|q|6#xG1pZ!{XE4?>XX3Na=s?skKyY*~l zSYn@#e3E>!P|W1}g5#a!O-ZxQd5Y~jJxto%-&yLv+rd?5I+Y6D`Gog{W;Lqb8vJ|7 z?et0p+Y#LjafpbBh|iqhM}bb7aGz_QXW+|uUy~KwpR(ij0OW)Jbq;t?s13O@XvyoX zjR+;bI}$^aMsz#|Ce`n+=HENei_@)E3K?b(>!1DqGSj;Qn224VTPtY9IX09JjRnBtNOoew z3@;I#Rd)HZ;kYnM+~x#zTn56pZkZGChw9YTg+W3FLn6W)r zS4fWG)vH(cMY3TQKTBH5H&L6pUO+(W2)Dr@dC#(p_#sU!-D-ige7MX_h5>970Wi+2 z|EdzfBl5%rBc#BVeCd{-++fLS%kzm{*7qKgs+?uJEe?K7zP=8gotCE*?!ZBowH2yb z=$6?DX^{n;|7rLoHAdIi(Yrua!v}U5_0-tua2uRaOKwvnxp65)xwKO!Olo z2@q1>td~)qN zqM$0q>=pUw+5co=6}UXW|1mbHa3+q7d>`_?X8E$5U;a$}7k3NmIeSK>NWcA6v4GdF zx+gmD+02pNv)9C`y?Ttn&veS0jw_gVNUoapJi3yO%~=ui??U*z_thK-i%2=GLh@%r zp}^Vzq@qFL6_xn{ftG%n8raq@;mt3+OLNWDxLH)>v}AQc{rj83D^$Q{{`sxhV-t+2 z#izgz((@PhO6vYXgaO$+?9r(6%&WYi%7Uo|cdig@`y6Y1IX}g-|DZw0hnz(6V!`sU ze+`4cHp_cy$+3`oPcadU^eKFQ#MwFTSE2qGX|g+&<7lIE`X5Za47Wcwk>Br^<+X%J z1~Iz8uDnjv$Ppt(#Pz_yeY00_bM-XY24m?uVZwywZ_ZGqy>Jl_p3$<&-z>P3hyv1R zwvt21{8fUj3<6v6E_ho{)J|SsZW{x2v0PG3F41L&hKXT&Zy6Cz)Dk*V#7mNdFFBQrzTp2Tr>@P7$NJ0K$#lHZ#U$N10AhZOy-5 z(vL3MO3HNbk1#?ZIlO#JBGvoah~9)LYk65(CO1dJkWZB zYc-x}UQY6N(5fdbOJU#;#L5*dsJv2YaT21&S#jm|=A`h!5e;*`$yRUR<>2AP$()aVbIWxPeJy zY7+#g=C`vZX|y7%5AC(j1(**bHeKCS{w{sZ>feaBDprC+t$Oxs%6@NuuxE|kmhNRm zs7H*3&yxkUbZpd=fHb5Wnx;W{uTnT=SAKo!4iDhaPwD~B8Yq10@-17a`@cpE4B6G8 zUF+6W39;f8DE_Zxt&X{On2kvr4!TUa-P?LBHQ zb2wx)bHKT)`_j)Sh1=i{ZkQIc@6m)?m%D}-?wFfB6pUUBxv;a+-46yHTEc12+JSi5GMpIvI@nW zRyt`>CG(H=HFL&8EaijW`NeF4ypG{_{Gwt%;+ZjEwyJ4)Z z{5IFk%+m5m_I^5&ANg(Cw(X10^uJN)>=4%a%rV&RbZx@WMb6ch+2-uolhtwFhVj%% zIH3yvTV=SOK55dVy^6XhzDIl7*yj5BWfor=FVHy4mw>7^=IdasDE5Z5*5X73k5sP- z3*|Kw{X-t0ux>=(ly9jHFe)nK0uV;Epm0S$(rXfk*DtBdj{85}I~QAIoymBn;Du0X z2vYE!tZxu#-p+Yt#W1V%T}a7{>{h+c%DN_Fn(StF*cpkMm{F*X?pl9?!L0HX`7~WO zcclP^4TbQ|)e3N1)qPN8vbbqt@s;K|s6IPnfQkK7?$RzkL@k{;nMK*{nV#!!EAlJ7o1g?m8EhUAMZNafBEY z$Wiz7{M9=DHvVqN%`hImwZNj}n9E;vEf@csn--w^a*{Vbgf1x*=p7x~^lzV~>LJ|ErJsRsQF&pJnH@%zKP=nL{d_t^@s} zVte*EFyt!JJfG6p;WV5M4!SJKhIUzu5=}u!R>mqykjw1wspsidMd?-{VIa{Ig%hCg;d@(fg~jFUnFLyg_P$2_xmJ8fkZq}?DT z*8-f0Mg8(vm*q**9nJ=v8F_fuo;NDJ!A@5eSFyD6<5{yvf3<|xp8<6J^s6eCoitfp z(m@&yj2bdzNZg8_N~e8ewPb`+PubuPh-5>+d8By`AS>ekZha)*gL6$}uOBjyM{%sS z&z>DAB>Pxy2BDDz)?mBv?}#S3Zp1Bt|4n_}e!Po|8uuYRT_KYQJ$XoEa{8KfSFUov zE9@@|70qFh*bNy%NNx$ug_o*H;im0Y|4SQ;j+=pMFMk#3lpN!FDvkdvbhBp_e-jIK z*Fv)N0OBv9I;-@mLwW-3&b7@+#?$C35-RtagF+=Pk;Sk>6c<)X4)k6f(x63ss2}f{ z@%F9gGgk~BX3iLNjhiF)M{c^U^SQA{MMA}=sc)NDa+m=+#Iw`-r-VONYUXt{|79x# zD$r~;Pw(+AfXiEyfUu&9x(lnO*gHYQRn`_f_WbHlg9>6OIouY zDGIe<@j`VuMF8k~*z|DI-R(~9S^C(o>X4mootxHb5@6ywtz+Fjo5HFt@Tqjg@m%@a zXMb3|^vQi43m&YvP4S3PnRjL#G+&MgjxF0~g%3r~yORUm%-oJ zoK;wm>xArEdYUnrn}>#bzca|X@;*8y`vp&k<_u9`7);W*JxkYq)t}N z!9{V@JIK5w|U7z z5;smIn@{djhW_y6uh#~u`x76h)-Wcm!LFwFG?$fzwiLlw>RyNw9SUrq3T*3_QrGjk z)pYZ2uws z9dkq5mUi9V5_gKG(7p$oNy+~e08!K^5Wh@f;8)aQ$1Bpmy1GUy#$$Mj zjIvr5gR#KL8K-KhzV#IZ0|}HvAOxD5Vz5vnZk0bJ{{ zdlR|}qf7k?2>J7^WF z{y5^u=@(R7X>hh~UC_0a%@M|ShmT(i%*@SK;fN~t<3Im0RP?J@(WF9b;Jq#uKii7!_L2}b-bUR3kEd-Gp7N+&_QsZ-|lWlvBrDg z_T<-#I;hPOl9MAe#@?SoZA)!uh9eQ=x)8G5sV&R>Q`VW@)nAj1psn%%-Epw7=CV@X z|LMbrRi<|>Z>mG*!;gjI?=XjL&5Fd}%AI!%av1Wvc)3Y!Bn_2C@|eeY->_wik?8am ziab)udzOxrasZWO@^5dSxQAZ?ZV>_YD2*`zh-s~OVT#)X%d{#3^8Vpjus&7^rM09T z=#3GBIDyB-N^gGvBbgZJqVcclJ_}4uNK=)X=q5BGqD+a`WgvF*8mOA{mKzlPMrBG= z-7jN#(|Z2QY>~D*Yua?#)(EJ`X|K)wQ-V@Qu?dB^WSi7+n;r7Xp>YesR3qNfGP3Vw zHRl$U0wpm%{uYnQ`72sXY=;gXzRv2#`X~HG9PYaR_~*fHreX|q`z~0}5&rIm3rrzT zX+MP_oiu$)H>4}2U&GusL4yMi`FulrtIwCOM^xtBYgcbL^TBBsTlMl}KRHSMPS&Mr z)M17!_y%rTYZ5x?>P!8a*}rbK&(!u`Z=<)#(Yt}q!gYJxAFfl7lx@HF&O$maI_k&l zC($-IrmBLG@`QtPdNa)JlEX^i$0($}WF)Vao?36Z0<| zI30jVrv?j9u5)UQer>HOTTEqUcd#{v%e%iN!RWE3lcV+<)H_S+oOXU`c#q6^(Oq zE+2^p$J2X5xcP=#2w&-h&%%BVEz^Fl*{fADt480%c%akzwtnia1E2f7*F@Cqm&Ky6+3$_ zZAhQA>C+}&yhuB&tbDexKZso~WE1)!eThwgPqZ$#-`>jl)ZTJlfzo7zPTo}a{k!ca%x_n26RFs0=?v1&si^#7k8)#` z1;9^$^-zrX*j!B$02T>9AtYVYW1YL;cLLH+WG%5k9KrzgN#l!6C~ z?PI?7UV?JFk75|`>63_Gj{|AQ92ietF9HwR1VC#6q&@MG!3=1_@ci%JUEueyhaT3o{Wd!d+n34%`sjrT zKZU0pTY$f!D8-VGTy{iMw^GZDJv(V;e>5vj_c`bRLb!~w{4b-hGpofhTc(&4_*AL5 z61xBOr%#`Td`GtmU@ehNtG+pX%kwC#R$N|{Kx>oKX=ZA{l=aW-p%f6^x8EC!BBm&z z>K^XqpZSsI)Y;({?)m6whi-q0JiHp&WpLQP(bIN&yv(I9@Rx^&hZ1Kq?^;g%NrLc| zEE7GDL=9TByQvl|gLU~?P%<#ySK=>|NN1Nanv)@jI08r+zK(_=*aHUVz{h?nNC6GkB*xG zn#;dlv&=p%o;@3IF5QhUxw{^js9Xf`lof?peb@i3(?1&JSDFWw0e?`vw^r=wYHMn| zPrNsJX2;Jd_Hw+{UG|kKCQOJRU$sKjqV9`otR43pThe&fq@D>U{9V>p3!is&o#*13 zXI;laJ2~iRilsb-m*@KQ$zqxap(V__Ho14_nNKAX_v-9(t)p1ug@=bPD<0(UM4YnZ zYkC#tB@dVbVGl4;nc$72+X=J-O9JWo(=x*^aKS@p#U`6b7=@CMmf`7re>;EBSfpR^--5I_wh_qgAm`gn^WnkZSn>@&BrCaH{`s!0 zV>QGlzWw^8DpSv??GpvXF3f)gS*VPqPqXQa^l+$MJ5|z&0_LvkFdjunrNdJKN?K{! zL8X-*d30<`=q?`);if2_^f2WQg_|1=VA|7sce4b#(a(tUH-OK3gYsb?MVu~hl0WMX zy$WngFky=be*ZV*qiuN$CYX0>XZ50t79EUTEj;TX({}axF*ok`L*6%tL`;%rn2%AL zIJE21Wt3fg19fF=zxHX!gq+wJxB7tQGJZl1RCbCa3VI2j{I9=ihk$;NE;c@{+cq{Z zM2gTH(S!2{tz^1(=N72WXVGi=bPXW(S(#9IXB_6xU}N4Sxx75*Zc$Fi1G>KH`uD%k zhd~lyMR;W{qmk@Q22MVm%QbKeoqF}UC`X+DHhd{9(K+dTc@D;RVqSXRGKXSz02#qh zk+oe$ALGRLd9Y_7i#Bq5DgG)YUsG#y)j;vi*xj}Lr`^9j^ogDM$F0Yc{o6L^Mr8bS zm$^|jXWEy5ddlCKx7=u`-z8<)KW^p1ES4U8-{@93zj~LwROXj%7IyH?X?T6{`EnEmo@HdO_=N?}3K|>-=lHU`X3ltR!`r0e>C`P_g-v z#A(WNDJP^UHYy@L5W)zMu-Ij>YwUvTkp)0fpXLl^GX4ckX=N~|amW)1C)3vPZ2_{U zF>d>5i0UePY0-`)?G9pLoBD8%b=eVey!UGGGg^3Y)GJLZ+uB$znAW`;Quh;E8%AD= z`Obw{8)$wyRJ}i_3-TJ|0BLDytv=4_unn$AY4WKj()ot#Tvz*)RX=N=`xTDzt~{GH zAf2#5R!iF?4Q9yJtzY#Asm+A%1xA0kMEiE*z|NhG1$#Tx-NFJW?v+)mZ1B9K9;R9n zG?f0Wen@a|t;AC=KI&&A1Q-ZjZ2drd=lybKAtp=Z@AP+%V;p>t(IqDIdfLz=XiJc$ zhCO!w{-ME@;v!F=3kM%`8c#1SIvj4lhi=dMSj4;12D=;0)Ms&i!DVi=U1()=2fAX0 zp@01}Mp+_#CPiiF&<+NH3+ZCuIx2o0?uD!U58Np6zo$F8voA6dz0ky=9bT4E(>_ev?zhpu>(%I@@LH{9(-^OhfCn6ov6?e{jv-s23h(ij2^g>U zzbJJtYzGb;kwh!cdn1HjmIg{=;_mVZ1{qk6e=(TH5)TvrY%h7AbM#ZEotwg@aV7H&%QW)ksEvk@@&7|W?voT{Y2kYm ziKau3+^;uhw!%0Fg=~li>9~|i#JqJk`uQJt_x6{w)>Ry%r9_6-?=-J#F0x?UloSI) zorDjZj4G20taAK1)vtEBQoFuf^H}4H7e-dM#x-i3-{=4JNA4ty*AICB<-eK2FfdV~1QF+_P0d~Gz~>2=r&^+x)%!&CKOHe+;+ z_wXQA1Y|g{nG#8Q`-uiBJ_fPh{Na@ydylsf@{V)mOG}pdtF1|ABVb!IYS=J+B%(GU zw33ax%`eYFKtB>24}1h_;<=2HA;tF(<^j~gvXCrFn0v@k7BfoLn_s+ezIe zdWS}A9>Hd|L7vMqGBU;N0szA=X^|=H@^3C}?o+;fFMo$42vy&{m*%rxFzp!q)bLodw##hH|RGqr7*)%j{W}kV1+S(N#lvfV) z_O^et(&|Agdat>^ZRAY5bKyy8HtAQpUcBWx#lMkGnXqMRUtPL6_tmFsp9B^V&k9CPOsf1jiD6g-|IA_k{{6xG|g-sCb15%uNEjTICO<^lsu z4JdzY+t%l04lDl`c6)Fi_B>`E#lLl|ytE<-i9`)MEwc1vw=AjPuxz0IMfu{xg{_oG z4hi$SRVJ0>JH!c^=Otrf3UpUK|3(O)L@HOY>(tTcN=-JLy=}uVvOWH6=7_P5 zq;*l;y6$=qEZqoZ81H*@1Z_nk$~Q(r(Tjc7^(|N8Df+i0OnF3wRgFtLvh95MiG%u+ zaH17e_QQw4iR`EQEM9yz*ivQQNw?n9cubl`-mP?O%82RH|1zz+Cj0zZ7U_tYbs%E7 zx?46CuOmU#YIHO1a{{}7i5%)faGijw=cL8XUAa}H3AkJ<4qQ{ty%5Hn1ALL!pkp!d z8y@Z6;byuOrm7(e{vh!mLcXVoU#A5_*iW+hKCJd#ym+xDo4WyzQ-6bg&1+p{{;`i@ zL|pyS)2pjUI`ML;F=k{t*Pbj;gqMc5Uo9FC^|hx@>kAB^nB?K%E|tR&#XY72l30+I z?{_R;tok2lh~%KFA1PgS~*A&5IcML-Q$O*vh7#a2|XJ@JA0 zpF}>rF@2-TlyI+yEYwqnZfX*7U8C2LmipoJ^G_7-mxP6nN|}3lM^87@9{8^QiOK7A z3e&y+sXtd*n3QmgMD0{LM`qpdD?UHv)|0gbX5&oj`%E(%%AYTh8)oBY4*ZYGlmbQb z;q&K#B?h)g5op3iitsu|*xgJk1w*@Ywi6oh&h~-=EpV3NHZE9|U7bQYFh45S1TQbg zEIM;p7t{(RMT>wwSR!gTk6GN9`qW296Lv_t^-d{LJ7Yp{6)9~D*ZEKOt*~s zzR)7YW8j>~QtngL+xRTOiyP|Tu>YG7?YUd6X<P$h{%>yc8kuj-oz z;aIDLDebmv@_f2ePA6=o;c+vhLFB)wJuJTtL!su-q2;e~Mv?I%BhZl6qzZ~0v^8@6 zdv=VT$;bj^LzeaK7F<&Gs#VNIiWV;3N?Q%J#UGbWw?fOR&3Lg?P(ohtdPr@Z4+5pa zP7JDD_x~mL8LS~M`H#M4XB}Jfan%}X-v#5dZLgnnuhTfCm2s^dRDyiF_STH?+x2YT z(~Z-v(QCt`%B-KaeCuw1*Dc1)(^WxLwRiN4aZA#<`pYZfJ-?OdGg=HYz6o)bpd?RqMcX zeSBg$evSx5zk?&^WWrzU`hI)I2IL{TlE>|Q2B}WaJ7}iUB2O@j`x>v>B=5ZtoRcfc zsN?f9!Y5FZ{w2VA^j`O1EF0-+_%0Eo{Xgh_y2axk=)vU1y3X%<`~8LM`nI4w$9eSR zAUwRj=9KrC$jbjsv1tB#Pg@2L%-+GF)fwyzVE_o6KXDroHzGF@oB}t4_MR3qiYm``Wgg>Ta54|>=dIDdE08bTi}u@yMvy&(#tDp9aTGF8 z&&?-gQ&x)_&kN4rs%?SMuckfGxkr=CUQrW<$1p#qd%KPRE_&cSaIOx2+EH;COs<+d zO0_})4VH5fxyM;c>CSKS+n3$C3y|h-p8)@)ax_0L>( z-k`MKo#(#pB3({`X@}{ZXtZO#80y~i>itin@2L-hD@CPyCnWf*pZmAhS>0xLO!C}@ zUmlL&ofi5EDlqllSW|{V6w(5y4w%EPoqEfKRF9PQYIW+?bxDiy6>u-{63N^30!sRC z7>b%bFF?^`pGt+gYM(HNNn3Kfs5xTg^&|UmWcdS_?=yUAe#X~btgn9R(qUU44_X^w z#$FB8pHT#eBhJpQ^ZF>VO^W!iMfY!}=N`xp8t<sS_|EKUy&b6%fls*_+( zpZ2>j$ETlIJzF7O`<)l*Q~kv+V|42Fi(nz}rQqBwSzEu^?x3LDX?F>-WX7*naH*n} zSV!UU=k1Emw6JhWA4gV$b*Ae z`H(}Cd7lPM%$#9WYP?x1ipvfqdj$iFME}uzZn`qN^A1^%vx&Ffn`UNipng_cxtVn2 zj#KhasP!1hnwX4nliaxS`|HnNCzBhs!C~1Z&n;rwcB8*4Zr1PG#*?n>U(+b zti=xi$5)hjrHK;ju@Vv2rB1zyU>~5`trhy$PdZPKhIGs&m7GcOxPmjuUVe@dDq9B$ zfkkV+CRj%YW0U^zI5&J;*2W+Kd`-lpJX|JGyy;B+X45O?V4E!-;uTs3crk3V(vm!) z#3Dpg*k_gO)Xxlh;_2S?u3bYI`=C^0BRO4)G_?I6UN-Jgd*8(mHWdrhB>GW?-W7?K zqky?_%WRd0v-iKkc69()9lQ#MCpv}fn-hI=iyq>qyDKewe zV8CgvZ2mP)wR<$pprc8^>Yd~QRcP1njn^-;h|8hNyYqFORyfWLUS!V%!~{XZTk>kD zA0wlKGfY#g&_|z{zBzodN^cgnHYNIf02kUNJzdWC6QaPt(m9_g!O7fU9B*O0mAO6D zLR<)s9UGi<_s*RdkxeKLDGPW>5X5gHc`Fg}(8a(RIlsVS7o9~P$W@ca)Nmh<6Y!QT zY8d`3B{tT-TfA{MVN@l=$oxQjKao(=nn zI{!WT^GwTSm1?wWYH>NUN&cGkef1m+ZOgsfxai@&$!Sgtq}i(%UAI5c1CSVsGm3vV z+b@iED6K5C2^3a;-h@FAL;y`iD-gX@xQ&g2g{WeV7o6m0o_XkkqzYva%J=F*^Pr3i z#m@=18P@X`Dp#~w%jNIT1W|9-uKwbzLl@{sf75^CzOk1-ztzuE?K{8+M}H`T$$%WL zSGl?rn)I`8tc^9>lhv+41iiSpybf7?B<^H2I%F{h!VF@_``dLjyJnv>ZT)J_Ur=6I zbpF?XFid&_`2FmtTgl7r_fCDWbC=zJ)M1=cmC_dIZv<_%TDP0x4jqD)xetC10wFme zG{6?46(G+yXmq`$uwsv35tbYg98V%kRn!9TnNY;HVr+x4VPE~>lj&zrjBdq>PPL&m zi%N{Qg_4tpwQK-FuHJ_-2LT&8%sTbV?eY-S92Juo3zPFpf1T9;Rmu$QBHWXWQDLTc z#V{jJ3Dssh*%(Hc<{}w3`LUPxk;?aKX?_!tkV(j^;*q$#cEV{5$*{Y=;ut6iGFNF5 zXQQ*?AfiXpj<#A@s1dyxJXPTk2d_V{>Hh;Vkw!S;VwZYaEZ^p)UM*9!tFL~khK4qA zRL}TJ*q5^kYHb?!l9$$rI)iM#g&1Wo1veI=3)lXNjRCN8{`0(|dBXEbf+jC-RX{!I zEc~Zdp^y0eynWz<_PeEMw|aTxy1|Yz1)a;i+kWP*}d@I@a4>(X#Tr-$y#CL-*L;Uwt{NdE1m}*B>O^Rox1h29&{8?3&%4c;VQbp(wIL6IwZd=>VXw!79y)gQ4;QY;6D-Fb439g`V!Ot-$ z&~Kel+v#_D+ojo-8Y=e1rZQ@=56+>BFQqNCFeY_nYxWF+F{@g|eyy+wPgFCgE&B_> ztEIVSlC(PNUO3T1QMS(={bn?DO~7_2DfVJ&%@G)3f7w{qW{m`dh>8$rQrH?-zfO#8 zSF?9$*L$HRlAd~)P?qw@fb54htYX}GN1J^abGy1-*%|Yk_9U0i)j_PoM_3s5!s`^6 z=OHn|*=-16o$=%dn~)T8H2%f1Z}9J9GG6XyHh{@O$*4aJH>i^rpQQo!@&*-$%#76xj-IQVx|a;+X&5IJG(c(n(HLY5Ewigk&CeU?Ly5K z0KLeK75sOe`W@Ro5uz)mF7*|;fpmn^{k5T-o06{) zdLe&1dSu7Ey^@=pL%y@ANxACfX4gMcJHtXTYVd89(rTdmQoYrd%bR;=)wg-!FYsTc zk?r|^x~#u4&#LFWb$-L{E_XLzLm_VCFmwihaMcu29@zmL(zFQl(EOvDm>gen%QfI` zz1@3?{H`Oy6Q)WgeNx#hcCW&iIX|>LOnnz#s?di+QmE4G99F=jg;;W+Ee$7mphKlX zFK2tZ^FLZ%U}KyOufn+`=PUf1EMxg^rkZb98s+-h^XiTK%7N8^82;qE7O`_F>5rck z9r<41n&DLT-X~Wt20AvUcvqGD$=0n~uM{6AO0G@wT10}>LAPMMGz$MFd1+3OyWdVWAA#$OIOSZ7 zw&nw)RJ8Xwf)Ch+>$A@(lcoEU*RTlsAqNT_s<)Uk$|EL$Ctr&1RF#j4Hk(`lGue-4 zR=`1m4Em90`Wk)y-QayR9y)P%?$i#M`5iAuH6hd-A66clK216?no!R5aN)utmBk8A z49`rto$vdbdv;A$KnoMEKdvDLXC@BOvg*vpU0bXP7gEiwBDWdo~J z!uXb#E>!6tawWb;i@ z3s|B#%wZ`h^hUW5RCB9#TT>*w zfAv-XY;i5o{JP5qvE`NDUk36^Di7#W)ww%F#o&l`O1$Pv@Y>#8URRQb!# z1#(J%{W+zUxvdt<|F1SnZ2vm;#QI_9%bgS;ZpypX~esttznO zN*bTBS)%91Ihxhd6{)O3<%cb-iES;8`1H7-zH6!~FtI*UB+YAxZ)0R)EK)E$FzhbdRb^*UD~OgSlHaj-Te zBLy!Iidi?sNI$2vw5qd8HX_`BKb|lB*SJq%+6}+r?a=0KN1ymUsP=eMTi4SL>yNzw z(d*OC{3Tk$E7X&;-KqC4|8}ah%KJ*KSy&#);{3}iVhLY!P)A>wzdv=d>alS7t5&Jf z+;(HQ`L~nw1(t4~dmS>a(a9*#9#M7W0>=>}M;e(__ier7{sFNzV5Jc`@7=uyNo9+% z58{Kg45L^8vpe=0HEOG76ilW`v6Wliogdf!ANvPgbcfBH0oEif!2BlDpi96iKND_0 zXExM#q3I)V5yMuKK8igD0utHF4rqa3;o7U>q9uyu#u+9KNddDgR;fC^MnZDlr}&EI zoOhuIZ4=XPe{-b47`l^y1zE!P@)b7WvdpGgDVvyJm(92-&dQP!$nR7Iff(6Vv`y=q z_6X!y9{LGL9z)}bKhs=Paw}zuVQ2&PP~ z_g2dtJ-|p|+<~V0ytU9p6iovytAqF~<-S({ho}k<78%y|ThK&7&SHW}kuw92E2-Aj z(Ty^icE2`owCeTwavi+Ax~`ZtX5^8`tcm*4r25S%^W`9%Gf}-u2Pld>E%}cGI~pl z6JX{-i(S)$|K=LOZ`Fiu(YM@xYLgAX64HV|a#X-(h9O}_hJN@+ZfJ09I^{RqTosR( z_omx?o9{0(@&Eelz2{RH^P?5^G(rC_zMR^tP%{>ry{Sb3RcEe#5wl%*ZkVPp|At5a zZ3$+eQ(b(5eQ*cF>s@zHr7v|dt^;xADPU{^LA0@zH?u4kZ zo}QPoz3sJt)!r5!q())BQ=LpAP1t?c>ud@N?%AxGZ?TK2s7bjo$&zx^gQY#R!W!;5 z@)=?xUtzyTciIh&1-hG(ZHJDH7FVd z_Gd_0hh`{fahjxmh_{zx&2n1YRbq1rMcmS_kuSs-M`j7R zXxhHnIYI~*Y&xRjUH%b~350%nP?X(M&4u0ud%`;GJq6~O5Uu81Dd>Aq=;E6 z%)9W(r!WHgmqxVU?7uW3JemJ7HU5w59Oy+s)oc2vpkB^YRZ`qG0v8p9uYJ^G()n(g z?u8Fj=IN8QFi6e4_wm*P_jwPexwW6s`6K%^C_Z<>=fGhi;pO)ut3us&rnRFC8AF-c z2~7)0^!)kl|Hs#Rfc3n;@&DN}B4zJX>Kug35E>{mdzM17RYsDmqOwwErSw6_A>)K7 zkr4+cQ8G&zMW|@d{5|eZIp=(T|Lgbf>T-R*$EZG^*ZY3m_w#-Z{iUxiZv>^T5=XT} z9F<5JN&*Ie2TNFF^$E9Y^YXwZ6bycfE0mqfE3+q$GMGGeH$Ks`nI5^5yv6qJw%Do( zN-(=ut4T^KvWLN05 z=>shK!o3LZT;kkQgSnMYV9n!y#@?H|2?ZdgF^X`TuLmD~=xd#Sg(deStT_qf32Dep zA;x^4U@znAsrFvGN!EMk11ea5RI=?y@7cY(+U6UFi?2c@)_{YKuT9o#Bq-2cPr0^P zIKuy+%l3YG6|J=rL4}kSk5EW>;-pMzMuGe}=4V+=8ha%14Ll0rM2%0p8sJijNVm%B z-7^MIRxZ$vDK3~~pC!#==*d*1SjWTvzIQL(a1pET47yc%J{V9(t?lD$&zJXiU5oS* z+P|byKhDMwkjGLE4Y8_mkN$7c{j4i{>X&QC&v*x}1YCo;(`u_g$l;#y+O{ddkNF5h z;MA52Q>IK23u7U>`K!lZo7GB~6e7MxZf$k>z~$Uy@i({b_X@MDtgpn3PH<)4$(?(h zFIrP~?SQ411{{sQ?Wq3(ZUxUWX{<{vOG?VzNyDB8KNEveyYD4sN*8xPu2ol=o9Gk} z0U{nz>POhcO_Y$p!LTeNDJW2@Sn znT~N?={Jezgs_hJLpfng)syDayrFvC^mtr9(eovv1fXPNtETvQr6KhcuAW}g(WJ5oxvjOf_qOY#FtMU#NNEO9ExepgKTlfNS8L(+ zW@TlK6Tr4|%_e}YD`;nrH3p-QIA9_e@4L1J8$^lPqbVGE4`W^ctCRnU1W~ZWn>b|( zkxKTDQ>L)%frBf7qfR{oy29xbRI)%qY2XdhL?C4w1zWS&AHa0JoUr%3waypVe{+F znq%x%aed?1Z;QR2e7AeL_&L`TWAemCvkoC$1ZSODcWagom0eb7ebd~IwMCF9NTnAbM)K=^op>wz2lS*n1|E55@U<1|$vOYtFs%mLsmOeM* zdQ(6xJyol@+=`OuF)2Qi=Mq|-*m52C{BU&XcLR$i5lb46rc1`QnNJpDUG7uX#S117 z-<3brYHv0yH@tiP54SY$8abVzDo6B228!II6W0WNRh0vKzH5x{#$cE};;e0Ix>=;I zcQ`|xmR{Gh>ct*ZiRinO_bh$)?wt^%Pj)Wv+sQ{;oDSD@tFZy-^n1Szy1k$0mEOHO zBB`71*xk23uGrFGA&w~$)28%#naO*jJZLOOwkuc1W>4%hHUxnobnNe?7v2BR7@?_Q z5NuVzOX(#%wRgqCwOfE2J$rT_d_O_NP${iyy0E3ZyY4)$o9I3SGeoYC4oJuobxs_NlF-R4^WZGuhtx8_}@oi`i80i|3*nC3aK zr1a0m)=qs*@D>CfyEts3{dU^Xw$~+y}S4rkyeVu-+%31e2k?dD6i@WMs)e* zesRv>Zrkg-mZ%GO)GZD7PUi`v~O*i3{6|Bad6RZEH?; z3Sg|Rcxz3g!~12f8N>S97VB@feu6Y)^`<~g+S~`b^TrKoN40p{MMQ|vYrZQJPPyS zo&#u@5su*foX^Z(CvW^zNqryKZddpvq-Vi+fSior~+gsd&NmHL*&@yYPF85#Ji@sp}s{ZKj)(N}j&8KyXhKvGIhhcl0wjvp+k%U zM)NS>n6)AN5?e;4p`r}kCoy=EDq<3xF3!gyU(svoE%xpp3%GYqS{Ykk2l)kxrU^pGL5S6y|Um~_uid77*c#QT!#PWe2$Ux z+5gx147xkBz>|Xno-}A>acha>j3@fd8Lv0Dx*Ue3Rg!X!&AnvAE-((stUz> zm%X15-|9Pteg7*B46;7JdU(fX+^3k)uE`AH!5t;}&41s&zu7ur?c})8&ek*uvp(y1 z`LiIf1K#y!reJL;1m4oct!q$rw{y3gautREs~;tVH36QZG~CHC_9_I;?m|lp^IeM> z6Z&%vJXORyY()q-1#F_$3>jkD>}8#aL`4oUtHw!X@Omo%TCuZVx zTGGIUHouv!tLFEdCRItWQGD!^_pB=nGL&-l2Eb(-T$ZmPRNXTA0=J*<_sbaO7c&6V z3E8tCwO*0xYq<`Ygh+f)tj%lZ=`1|;|9Iv>kcoy2H2Zr1TMR_kyzsb1ju!r2dpl<2g#JZ^=?YGXeXrtHviI}bt1;BRv_ZT1(UJez;Kc*tj5ILLx{*l*{D5wOE zCfM$lFSMQj>mj}S7K%(pKg)|QBiDD zF|v0H5tm|IPWKw#wQ=1C6Ibl5bax|Nd^W?Fn=wupKL_vj^Q*2rwfx!&GC?y$8|`YS zkKPO_z;JkMpSCLAE)f69M@f2KdOBg?tlCn8QnJlyZNuS(wMncIzhIR-X`k-Zeyjd4 z3|2EdrW39##e%=h?+37_Yy=bHV4uU1zsTcP}Q($}{bbzquQPY~~RL>@L=vjjg9Srl#{@#GY9 zfl#59(7w>tny4o(20OUSl9Bf7#Fd;}PZl8HTw527xaG@U7S#z(oAjL1rw3FP(ba5n zvAsU|OGaThr3aaCZ{Y1gy#4!V1a6<=T~DnTct=O8-UQ+@ZZNQDd@EhS?NeTy59c1WGwWE&)_mi}YG@r$Lx^LD$XD=LJ+}JeV;#hs0rj_8J;M{kluK1Cx|KOp z*Ex^BGx#tE&1}3otE&L2>>VwE(<|c@>N$p{UcXhVP*-Z^V#*eOvZJX9=6IWZi|W8) zDR7mmMFyz+MUvx?yZc~Ipg!tP%Gcw`3)wH=ydeA4C3ay224ZMls z#nLpw(9z2gUJW4R5&PZqeB6N=Hb}|<(xaNkLPyw%bpG|y<{BP#`OhM{oD_q@kJ@&* z$eCNvl}|1Zof$S5Y7@P+Ac|X~OAd5dJY1=Q5gsaxuJHwM)@6wU14q@e&pKJ0M#Aio zY^q-LK1*QcudGm0VYR1fX}A{tV&3c3X<~t2%&^BAKi*Ggq_9<{YTxh4`fwuO#L)vT zgLQr8i2zx$re42p`z_$!+&JaD1leXt&nEAwkV35d&SWbX0DnP$u|{^NCy1_LK1wiS$Dq+2=W7}`H_s4pV)Qb_F9dN zqWgPeKw#PTz)NP;Zr=k8pz6aa|1G0@){UxB{=!!OXyswGf3{AW%+DldKmcb^REjU>fBHhJnKK`3&6R)6x?`tKHE7hs ztGz7BNgEEjdwt@$zBT5x~uDz@UPj~u-Dt%9j19xcgDc`nC3t- z8DXmVTwW_uSCkT%

    -us@)%R`(yxU@M?=ruVpdb@8epGo``4Sb3&_c z`bvdP(3s4loXPRYz&B3{4a*LKrv3x+V;fk>Re3*l(do$pAb0O}VN(i19B$1Pk$}sl zI5<&Z0FW#rPMz9T@mfzBtx7OoPj86Q8Bfy!e#s;`mwub;iUDx?itwD2G`z)E7#SOn zH0w2~vd7e$J8iNnj)j1v>W=QgVM{X{aXJ@y7~7>d+0HBFVdyd4NIJleSYyvZS9Cv0 zy;*9GIrxek-gSMb_R1#7%rJt}Ba48jqWHyq-lClr6{rBD$%u;bK0h~{6roUTy1C3- z@r<@IwYrinkaJ91OL4X@ajdZwWjU19 z@N%5aHn4XadeWf}qDT5UiDg~huZ{m#Lqu*eL9<*760KRjE)P>4V?V|m;l=1F0gUAXeOG=`zh9lTAT%F2(2DwIhdnU&;L!n1q67TR?}U(zJub~V=*i{ zzR2tNKe`$Vi#1=$Ic1eR6s7lB>Ri(?_M4O$uZ>#zl>YYLh}Sg$)NTGUQ$LAe;+W&b z?iavshc0mVjbsO)rsJ)azk-AfEqfz-`!I}gsX*bWY;&Yu8an=XjgW|GWY-Ocrwto4 zcmvm<((ofBgd7HSt9STj53VRRRHhTK&5$f78Z>>4t=DhI$2)IOv)XRJo*XMO|S5rjZt`ILnGUQl;BeW$^|${wHOF%Nv&+!ptco~ zw@=5y!jyg(T-u|)F^%J#yqsF#n*tM;3lw9QyV+-ZoFFxiSiSI6`60g;r3O;>kWd2CdoNOt||LRk?v5m!Z-ddl@KZ-`*{p*n4#*~0A|Sv96#8-v;`=& z&>au_R)|mxZz%JVgW5HMeS4Oz5^t?;Ulbk;-B2ba>id3CLq!YrD<-MUX%eqPzgfa& zLl*Q(cPQ$J4^W|8s-p^BH7Y*i1d3o)Z)QHJNDHktcZ6Gj2~=ZzDvp(vxespbC*=L8>t_iu5Vloo%ZpB@k znl^)aoOA_J5$(_(FMB`1l`7pq>eXjAzz8vSM8%O^#&IOO)zl7XskXJ#yE5?x8PSY= zTM-5dV(VGyvuD4BFT*e`CJh}USbwKeE^XIGkTr80lhc1_q`9jUdP%4cIXYY1>}v%N zzWOH=s|S>;!p*yvOKRGZC8s4$=kM=@tiK9nNso#^R0o3x4{ndcQS&`M_5n~FvJVt&uG@%ofjRWW^(V}X7_0Gv&+#$ z2O$XJI8j>+`c(gIEA8sgR^G1ttF7$6drfntG~MdJ)gITkO*K3BZqzOfp`@nY{^OwE z-o2i@t)h-r`hD@OKy6VnNsc2_;8G!Mdb}$G=k?g~>{y6+Rhq+K7`$v^((_#QSi|^N z`X>4e!Y+n|OJyMHnVC_Wm>V5K_3I4)ZeegC!%#^ClmTjj`fdkm89mrZj3EjB-OY7W zonhIknNGn^H&Ks1Yqa0}r*kb87=3a=+i<_gbuOs>xPZQ3mEu$;XETPW=3QpySfjJ_ zj^rf$JEU(MGJCSS8mrMO_J#8~*;QhIvS~}Yx9DK;7$YZ5tW@v7Gnf>kES_1T3AvM; zLd!Ca2vXW^BSzVN0UE+1X?y|TM|^CW!(Ye>gmtiyA#EoIb>m6Br|d)p#!sFR>prMb z?Wg54kXcsta^cMh_XK?Rir-@O*T3!$qVGzUBB=98(XU2`j zD{cn9!xF3QZ{2C<3hoBP1-B4jm{|V~k|);Z6fJFPX!E)?$IY5`T8OlBz`bPf8O04P za48Bhr-=Of=o8O^&8Dtyazrk6D80q&O)KH~nFhpD#HG?T=aGUq$pA_!t~VQagE+93 zKvFy0E+q-<{SD-_cxMCL8eV@XnI(+R~JD4RM z6E?)kc)}RH_|;`~d&3k>T|tVJ9|0DEf6l1FKOOANQV$W0!-wseHP?4ccpdF!uDNK; z$^oIfwKa8|AH5qTIEMu8x}MFJw9(A<<@1f*#^g8gzvm&c!u>YxR?3yT1a2*UMiz$h zQpZ{Ys(n0vV(-7lbu?>zA)%aiK^!RkH2x1A`?a(>fkI`)>eaumBpr*Bsi}s?mWWue zl$ofWn8XhG2%~!M{4D(5h}v8?wUJTCxa>z5ct>5*~#k-%{@pAT;Ta zzkdx2KiPEFn}coP%CUoFJTia1HVje8?oPwSA>`W&*20psVtiqgSsc^6t&7XlWYD+m zSBmTuQ$IlA2DcO&nrQo8tz?fXCYROytS)mED?VKtt@12-jkDS7^#IW4PM$lbF7{-> zcwmK^ZiPGn_vzr{r%$(Fe&N~XQZ((W1k*AZd4w$R_R_k=fpsMD{e9Gd=Ik$gc-~>C z*0U^J1%fMXsxTW+Cc8ae2YW^AKQY;ph4p{4>23?a=`;G*eP8y^eeXZObZJml@QcH| zgvOn(+&S#OVV(uEx&~E(aeZP99l8*zeT%)@(wf9<1AQR$qb;|Em8&r4qMUTvZxG}( zz>K+KVGkg#RZ@3W^}a9I6JRu3*a;tJflkucWIA^7JmkIYN?~Ck^n0`PIA-(aRRq#q zV&P0r^q1W#x?0A4@hPm;d2OWk`ofmWw)yaRCd3Nd>&BzW)B5^qM~0;=O)7 zkw^9J4K>RU1GK@Vxiw1NW5Z*SW+ADpZ`u2T4Dh`x&UDfWn7L@tqPqo8S-=flV2^e7 zExgb-$d)qt;+%LjY)^t~_=S0OLH%2*lt(ncaeger^SY3 zX4}1@Em;m|BG=KNg$d6at6Sx>)4c^Mb>`Co{KZJKP%mguD@IZV#)KHRbxJF#54lbp z8o46!UCWf)%}d;nUF9e9oi{DnhO+>7sy+~e-t)lvHLE);UVMuM8vW4E;x`7tfv?!V zFYfd5e;7}4A~j$Bs-Ld^?BAPx>117+dQUF=uSI8iOkL5c%ZN7{HXl@8ntl2xBrW2X zsrmE8%?@nW6H{}R?QbedD#`~R%On|6bbl`|jh)$@6 zrhNuwtTgSPPHe)zru|3v35B#yojSxhT%Ov>s)6Cbq(=P^sB)Vt;>9nRI57OakELJ? z37ER&OGXI|=~%aAG8~EbEnfCoOMRaMRfq)fz4H|c3TG8GVc?>cL=P#cB10E0{Isa3 zDzhc3GPdQ`wUq2bo*@}OBaQxF_Fw%K&0LR7xhh=DZcwu?7wwq3&)WkptRz1*~cjpGp}?jw@Ds1X=0-HiavFHs<%@AJXF2bj?qsoyI(oq zv8es(OhJU_~=)#%Qvdf9E(n*Q5w&#L`4+G56#f`_Fs z%Llxj;JMrE#PJhXjOQdxeYN`GxaQMB5A+VqY}YCdEo!?}i=+SeqX7f?&U?bY;c(DDJ#QzyFcI zn&rurj>X>M_4Qp=t$W2dE2~iJ8qG2*+X>wUo3YBIMfDtW+#gE0SJoxC`jU9wks>I@BXGYwq@BoS+0t*-=c!X;`R?J?6bALTR%fdP zCd;i+_M!I|3|IwMM;p{WYPqsiQ2ye$6L`Pwo0-Fh*LM3hGj<+|Po}4=S!;nu0MQTdAyGXPbiq9lmtz|I<`)S%f#RMlX7+^|I)WJtnzXjvXr@>f9$`g%ChR!m4JbI6;f8 zljNhAjZ6zD1cyVfxmaJCfS&5;boA-UewZKmQ`|9nyGn)H-k>f)3O(D{685L#TWR^! zX0ADL;lc$Sou6-+n9sp`7trg=w9}LK?A?2>enCB;M`%6Ras)pugnB`W@;W$yX@8Wt zS9GbUnZP5*CvS4B(+upDE$l>Yca5} z@tKfK%gtnoA6Uy^#Ws=-7(YC%nW6!s4p_!wt%MK)0bo%JTgBMBVDM7=Lo@eu$0;L~f%CxYkgs3A zHe1MG(^)F(!;9YDssaaR?cKM}($es|EI%cV=u0EXu|Un;X5VPD$ApA24b+c`jvgo? z#_?RK`p64^6uw->dcIZROja%8Tm9^G7Z$9bW%(2o1a317)hCS3$G*~?f7aCDvq zUEVpwN|TjsYZ3uy)?7cX0*nHF{!ma|c~F(VsNVr)&2RH-8Pz7{vx@S+JHPfCupy%T z&YSTUrkds3xD9=CceA5gv{u89x`#gZ>oK2FD;)z9{gJ|71D0)5MPt_6xA1s+ljo9@ z3YeUS%m0!wuktPUM>PcuLqffAI=EwVIs>D&gHz0>LwzKlV(0AK=hZF&e8T+C_;M+2 zAEu*IV3sh*32kUC4JUl&&|4GDwEgeRZ6a%beO0T)2+(dYeaZzGxteGIRswbq>zep` zLf89b{Mk7)r=gjq4ctl(*E8L$r=h2iB&(H}-qA=)+v< zy%4Blk}0Wg|xQBGN5tGs&=|bJ=7a4-3lx znJ}@p4?8;7%q(`{k|hmW=UdO3^)S1)*N3x^EvkYx|5|jHvY7jH#Pbsd|g= zq=^9wLC&nd(tI=-km1_T4}oX1Q}8-Hhv}llNe5lolVT_X@#N6AH6S+}4F$b+rnQ&X zhr@z&@SC<}^ow(#Kv=C`Jf0JKb+st;Y>OPPIdICD8*c!eX#Gdl>LnPgV)?HQ*f9Tm zWirLXdH&ZrTyveh^?pRO>l`hg-^avmifH#N&%(BqUlW(NpGQd^ssN$g>YmW62XK`; zwjzJ5)q2C$R%}^xY-NS8=G)O1xZFxi?OB>P?|V|a6_Oq#p-PeX#b%MeL0+$Rn9o>* zC(=W)JWdYA<`KZ=L4(1^pV1U<2EUJFSZ?y9@$SW;^*7^}Z(-qOHx}@a+YF1SI&Yr$ z-^eDCh4T=np1WUuQOhPwLKS$S6(bQPvVII`w#5SLv6ur;;!tuW$=6_nzBSQ_?gyZ) zMT0F1+8!CFS&JPKZ7fq01=q!+FlfyDn(jXHx7|QVJloFBZu-mY0#_R2bR1}HUOdTO zvn{SYX5jisya8Ls#0MVFY=UB6oXNK37Ukz3W^HWuo0b*cYoN@J5Cq(Uq$oAETkqbJ zP9ViOtXD50{m3*8(9$VWI_LB4z5VuGRXwqNs9rr$gUDrVGij11;s{Gi^W7{-b(7#D z!ku!8!{dmlLfCxNq$b7OM7Mza+kcN=vDVeq!^ps3?D!^HimYRL^-z>Qni07WOm=h7 z5JSn_1pTR-Ey1p^T{;48Xdi{hR(hf2snUHM$zHkFzVmoo_Hou`LtEHhW*I^<7=yy@J)|a-{b!^#`56oRKMG zxYFg2;a@&ZN`GvH+(^TqhuWhe3cDUv~d+Y1gd)j(>DfYZB`MfTJ?x6ce zm!6AsLBnhz1Urz0x0W^^8hN&s{XJt2Qu+v{Q-{=6ZwvF1>VpstNr#nR^d@L%#dx+2Dkuk_q z>8VNdyltmiQha;s^Ti*k=et~N59L;;sxwG|>98v`)jBeG;Umr6qeu+Bot1Us5i)z< zJzMN#i$1QB<;}_Ozftc) ze1Zzne<->U8cJEhQt|4S;R`J}g0?9}w*v0nvq-mYrGkyC{~lylVt63vY!(FId#Z+} zn&vmiAmdg#b~bbJA5%BRmcBRW-l9PhKXrA_tFVF~ojp2r=FC3+u5}7nkf5w%;17WJ zIbv=Bk~`L#9aix3PL32a1i^=Ea)%7gk#24KC*dX)|xxKqe8DJsp<9AjXLM1Y|_k%j2^l` zcggE3O>q@$v1t|i4wObtwnG%n2?i9(4JHavFYuy~aT3vxPNlbnF>Qh9_p+JFrD3Z- zW&}h3c2wkbxB0(@FUE^H^Q@18ysnGm8Z~G@S95KCQ6GB2`US|HBs?sXmu(Mi%2Go-HT};Ob!t+e>}l<-`>5Uk&%;56Y_WE>eX`~yxu46 zc1=FzsT|`DJ}sm-Q5MrqpVeQi=5)@|b_=7-P`3aaMlC8~Jy4K&acUO`Oa`YJY36)$ zF5)RtuUqQcHUR51XhJ%=iow4T{0P@=N;apIqK?DcD4(GfcR`(yrG z+edLC=G?h+3vv<~m$jpi)2}J-RyuVfWk_%IYL+u zoy`PD+d8=H*)e_{#f1FpTw+dLqqj)dg-=)__f9mQ7ui^g2L2f<0*F{-GPg_u1M_XG$ z0fcjyoOf41QBXlAueaz!ACkOOP)-1S%jBt=bMUbhjkSmO?AeP z7!Ok#0?QA#gq=5O*5zs+=rYc!r(!;jm3Q@t+ zGe^K3g5*y;w`o)s)3?Hdm&y=;c51{AcMGe~hzNtldaKv0`5ELAay08$@|+U3cww}a zrg*T~#<%O*tiJKjcN#ELMp!nR78-9N%%p_IW@a6!yEs$GeD^VE(5laL#4zkcO=_Ev zoRkEs>*+#b%m;tkheq`g|2?CTF|&h**+Cg8|d+_c^MH&Ibczx9@hvG=3(0I6dYR z@x(DKQ5Tx{b9!Q4v33Ay6QZnXC6!@g;^K%caax<s|Dcw*tqLd&|*5M@D?U$k2?W+mqQ zlBlfT{!dX2Tk>c}>rU;S_uyfV>hXE#WJSByJvHVH_tu9ZFN-W7^`ulDdD&e+e;x^c zVBM;m)Ey9gZVJD8l_N-HTX`{q=XU0V78^jSJ%AAeQ%GPYq>@+qpk|KuZyhXG5(b!x4c2FzF%Q|m~gUXPY#?I^_jqk&H| zC!1TnfrYiap9hwSZVTEPy>nB}SE(zwBRHIV9(edcKh`}@>A)~}=w)U?MmE(Lc*|ay zhYzyrJBOH8w!|^P!R=(gw);&n9~j5O2r4c~Al#~U7Q6$n8K>{svEyES(;lZdq+Fa$ zU1{@`cQ#e^D~&VD+uJfjMPL#n{n0V4X(mO3h9NvDN2g}lt zmd)1CfSEj$Q<@H@d-Kd1{jOd2?)gxF^I?X}WOwg+J*tGCZnJ98)&cPiAi-h4)=LYM zf2+F3WA`ic#K4D38BFPB@cOqq`sQX1t~i*Y`DBmOzg4F{Qb=74FfTUmiUS28!_D_j zs?u%p5|0|AS9}w4N2}JY6JNYAdVj5p3IL1{W0@3vmZJv>~K0Tbn1$2wIg z>|AvHlDBII`Ibr=n&>bNX?2&}mvBU&-*}(Qc*z5WxpKW-nJM+}(~jlQ&X45gL?k_w z9-#YO``NQMnBMZv*h7o@=}~A`v;1OkF!8Jw zHGB4utaUS2SaG!@mHl@R{zTKSlBCo(da74iq1EZJ$%sQs`Ec2+}WD)j(iX5D6+ka8UP% zP|VpRi(9ZQIm`@PyOgsTj>%5K#!ty^Dj7t+ki6V4F};c4kj;l}0+x`pWQ~|Vc!!%j zw)!(WCnp}hh=(=FYRZOLip>(B4ptLgxu-qcnO9MM=<7EvQ|CW38kxdCeRl58Oc94+ z9BxPYn);aGco4`%0L>N$UCU?A>&w4~oWqMi^`!MuC#NP*Cs0nBc4&OSsWrF(v>&0m zQbyFrD4AE4Mb+S@pjSp!ONXqY#QSQ>a@%XlcGv3tUJR;czN)cSzy(c};a7EY$C@uV zF#p~1{RjPUFkrE!x-F9?v^m}jHv#swbvyYO(`gFZ6j2iUpzvVl zrlX_fGg@e6wD4zD7MhyYkecm%G8jKW?)qy(%VzpAcw*#h(}c46igqf2hDfk5sXXPXbGqY-Y`{zY&`YOz-e!(`U!^Zc-VjwUm=oW*EXDW5pmn9!+T_mo2|{s7Ji&h(~2ZNjms`}^-zQdMdOOx4Pf zYC7s6+&DZ$$unrmj2YYL&+cw@CEVE)(CjT5T-(gG!Qkrr?apVw6VXq{h2l zh{}EAy!-nct2I+J=j9VO7=jDEUI2iI1P(r2)qvG5Dk_R$fH`}ikTKc(HJNbx5`je$ zTsZa5NOleF*X8d9TTS5D)g{Z^e~$;tfrF!L2VHvE5V=vuhE|_@(gxuGDgwzuamMEI z{aB*_YS%dDjRO=vnifZ0pWCyej*j{GfP<63S=so6i$g08fpNCK}_e9L|z(nnHu{SIUmk)(2)_S7?^h6>`BJWk-92T`tCK7%M^agpNh1$eX{&*Cs~ zkf6D=&3qFA?cf$~=#QWN?h|nIeYP@;0wkWhN2R>Q!trfLci!X||37Aevj@6WK#auH zRO9#G*|tC&k^J8VwM~^vD^VE8((2iuk$gpP-fo;(Z5<7US9n2hnCNm{22NP&@9VpP z0w|DDbe&Cta4J9a>N%e2r2bH|>f#SD{~=!$N9fw>H@|N9tY@a#Y0Il>rH=+} z+U2^$VA|xU|48O%j8YyDKI&_%31q{bB$yQ@lkI?z2Xz+EQRq zZO(N8(D#Dp{GIOr@9o1n$u$l;@k9*0?OR?_QgQbl*q-%VWpnc#+qYlOE^w;WN70Y6 z8s`e`36f*dO7ZL>>hqZdyK|BjM!@!wXw?QgZhh~z^wNwI$6akYD1PuHcSwSZK0YnP ze*FJ}s-s8mxHVC;5ke-c6v74#!8{`lMojv0k3nIdwyG@(>BCgw zuk;=9aJ)UPfsr8GI7dTlanJqz_n#7_fuu#XNdFpRtu5}Ja;mxfw1OZa{`v^p3~uTW zCWn-d#Qfzt!<031O23qxTbmt2KDRK_oYQW5$?`L4-r=6jo?F=slQk&xKk39)v$_9c z+J)K_a|S>2<{%g-ansYvzoda@ZY+wI!*oZ(jShIMeHL7y8RgUX?F6Xv^8q|wR~*;S*{*j!-LJBg5%!#7gt&NQprQp zM+a$|H>h2MY2nXi*QRLJg3Aq5q=fw^loih_*cM;Sek7ODyLZ~UojU`eT1SAIf}dKo z4=vbM=HcNntz0!@2K^+Dga0QV8Rx|P%dMa6g)X$D?v#XCnf@z63QFmV*A{&D8(#*kPRc~SxFcSM0 zo{!(!+(a@71>k+Tj8MmR@7}GSFgTh7sHx}YdLNzGt1VqO!~x87CO%t}k=fJG@SLnu zXI$(e>r{t6kc-3&1{Qf^5!3)Fwu82)t+Q5_b4-qyM3~S^?Rm+-AhQ=h7=Cp-mYc|l zto$e*Bz>uUjjG*hOu(;k-{EQRueWOjj$GA1%g3tu;%@JBb!IQ{8MoS*l9-VxLl^YCr4y^t}53qrEJ1wV9?qc;Y=Dkod zqMXkEnAg39==brpIuQe10#chTh;$Qwt7zz{IL};v^XBjiw3EsT>*F<-7N0C7X%iB_ zr$(7+z7$N5c4iJSHD7x7>LpHk{GstZJ?o5VqUwJj>MIpyb7Sp9hiiL9JnD#g!*6>(Qct5Db-o z>ZcTgQ~hgZ7Tv=egBc~Mm@9zBt+Z%_x(s#zuytc-T_~lKDO4T>z)*vmKUrj@Sqmm! z+=fmlSt+Lgt!BOY(ja88q%ko-VvQ1=>t^pB9|<`+Fb?A(3)otQi)PPOL7g%=Vx`+s z82-4V`|Yv31}>l)X&n*I;!*n;?!aR}v`>(>(YJH$k@)Gk6(mn#4XCj)roTo4G=v*P9IzCp(>5XPY2cBUY3$a2E}Bn9jkzdB)&09RPheV*$#< z{*{CN)>(C8z_ZU>YW#k2pVl;?_!|#;|6SK#6`G!yX+?DGpyktpmyh(B-oUAy$$}k} zy1rsv7n^G{$M@{eqCp1B8kR-%u5S5g4kB_#IJPU-{~HXZeosKCUV*;`r%S)^QlH7J zP~ATt^liJ@%kI^#K^i|QnkJZzuf8nlkZ3Qdrw7mW%b@o^%S517yK_D9afK*o>#z#^ zsl33ZVy|f;^;+*MKQ48G{yNVCk6ScHV$bD=?Ga_{38HA~0&+QVYjv>U>_q zbqZE?q;k1tVgdv;nsz|k{TLSXier2XaEU~{YGC51}_Dg81dn(3%6r#iYk=+`9= z0xH}opPE%&sB8Z-hn0J&UY>^)Z{okI_A5*8`%GQ^q4x2L=OqtI?oT+p>L*F?06xqwx zgiK<^H{F_;H##Ckd@NBMU3jsU#S4{o15+lDYhJej17@^Hg4C-W)=}#qaMZY~ua~md z&oG6DByMsTK}V(#%h&Yx$s<}G;EZCRuNAz}>s|e_fz)s z=$6wdU=yk4PEI{xRe8Dt4U7wXXs?t+gAU3urD~Qwe34T${9Pu0flqAP-QQoN$Hz~U zPnPTWZ2vII2OT}){^otoygq^CC3~VM3y*bMy*>c<4%uSWiMet#tUayRD76voYYr|joe2f`AUzyGT#XYN|Q`%|7& zSw=DnC}F7G;L(mha?{VQtwVohT$*dDpLxHjxgX`sjJ)_fSYfD6{PtzK;EmGI!yTF)sQCZFl$zy;$=T zJtI;mKEZ^>EE;k77jJ5UG4zvXDF$j+4$`Qdh!}vK+FxI&M=~jaLc~sM-C)bKik9{! zw4%Yr>%yNhTGQlq;Spk#10$|W;Sf7%MB;J86ORgN~8*-6XC zUM_dpf+d%16)+H`z?<*hiw0B2)1gPzySj6~m)>+}>?e!#stqwQsZOeh4v5z{z(7#X zFMCq1f+4)6l;37F)X_c?^%-`Q?agQM(7>v(R8CAz=!zK|{T^MI+66s&Bg6ta#}xb5 zz+UExw|0$L#Sbeh9ge`a;MH1+7VnV~k~;c^2FN7Mp_h`XwRGvyTZWb1?8aT5Y?Ggw z?R{x!>D^kQ*Q@+cw3KbArU+VOkm(iqY(`$N)Yq)ug97r?XU0yD&eQN=*%&&{4t@2d z^E}IBBgP{&LE&AB`e2^IVq0Ga=oMo|J!FI|Nbs(xu(n z4RSqN$fGVpLXW2CgK?1q1aTj?mQ&7wF>`c0cWbU@0*|HVi_Gt&UTQ+^^J}8My9@XI z3Fyp6waxO5aOAjNH2$u`vr3j=zl_aAChIe!a=Tf%)oZJzUc9EGF#P|Rdak-O76}U@ z?Qs5N1bf()@mw&Ph!ZEWRJKg9eCS?$`!pH3odW(t)xbXlV zPYY=@D4{MapY|8#(~3RBv0TX#Xb;B|wL(!1g?>*&|3EL@@)3a6Jv62>NYj-^>9;wZ zV;Tp1rxT^1cUB(7@#D3WqY@AEE;gz4VnA!z2m6+6sQKpk{2U<44dhdEDvW(sb?Fj-4PsrMoyalAZvxqH*0g-O(JI4q53RyQlZ6wF0>;8+S)C2! zXb3Td`C~7->a3!3&llTf=CH4XTPxp2%m^z@d?Z`)FyjJLckMOia;E)05A4d30Ufo-x08u>HG@0ANY`0v z>Ai;!&AF2ymYyj@{z8dOFhAfeeJQRoZz+EZ*D*|}`R=F=9wTV!uwlcJtbyZc2`;Os zT9D;fldiA9mv${$3_<-bywlD95~51c+Y|j%2f}0P$iCzU(E@`l=-Ty%he0^-Bd}BT zwd=He+WkA#ucM`k+w>Gl=pC9)NtEc(H1XAP8d_T~1q{@oLvQ>MdxjO4ot+k~=JCjN z-CKBIrbd_v8O-00tvqU6gQ*mI)MsA`7c3zfOS~%j1oCUraEG$!z7yitfB*K)<3)A> znu_+YHUf6%jLn|`JCQCTc{;rxmQ77Gtp2Fqw)#LQd7bM%T*Hw-)2-pk9P_sS5G_R- z#9c|@JR2GX{MM+X_2|4sAoxGFt9azx^GM2^DL_!Z||@$y?Wwg|LRqn!kIMvGOAbynwmLV&Y5#$9A0vqBl9sW zw3=Y+lEKT0FSYwKXC6pdT_O{Ax#r_8)L_8mdhzzX@f^=dYs7`G!iB$ryXuD6My?to zq}gG3`qNDR#cL21)v~#Pb%U$J-b6tapTl4_ust*F+n_O>jY~coHfYeG?m9M0qH2YX zqkbRb4ic3+yqe$QH=H4=6j8g*H5RN%`SH+73hMdgmGF~Qztn?LU0Du6F)Bm*BNMdMx{ds>V}hX!K`ik_j=mN^7k`*pqZ(D$F771uXt1U0)>y%j*eND^ zZiA`cipvZN9@LC*tn7t_A>I<+Cr40(Y1exQTM^z7ScbV14A9Y>hu4Y$%vkEuJv ztoirtf70b$U)<*=Jd#1DlBKDpaqQLCquZ4kO{@mO7tr`bz_EURho#wHzupnxFco)H+5TJbC@Vq~ zdOrxNf>QrmK2j_tJk`|v$fGjD80&s#!k&qaamQW3*N+Z zgVe4u`w9hyh$dgIrO9}s}qAYop%x~Tm2cmRbT;Ja_Va?RJ^BhWrSWpW~nj{Y8Qc}s|^ZR|! zptgwB;uLMTn`a)-ugCdp#rgwR%!z9mhB=r`o3j9t&OA5W^cOvv``=||-xXG4v;x|4 zl6mmNdnsAcWhvk7&b{YR6+MEWmi>KgtceL45xgKuMVw^L}5!&3l$zmyCN7Whzh zrf+W>!Fc{*EFCI!Xz`uS5+!xh_z0T5k(Q=;TEd5>58x|8v4=bwr3>~V%lw47JwwrD zDM_a37+S#C-S=n)qq}l7u5SMO)z{sUSn@}QY;sToyOOLj8I-Q!Un()hg$KI?uZgi{ zsjM2KL>4&jk0=IM=XQ}j*E^3js-Q)ZIHlavvv5;{g>5N0Z`LcbSFc`mS8|CYq^Vnc zC@OJ;=D_X0=C4e7NG43E5feNz@^Sk)l)I!P?R-H#ksbEm$c1YSTrIX{s#onVo;cx^ zdFY54PDetNCprfrRNzCgo94^Aw6CLX)rP+5I_UyMXJcRU3jZK41emo)c1s!|<`-;~ zLiPUI^HRcb*Z4FgI(vs)`*p8-^xg0zMh!ZYmx{{q zwQcW?wHV@5eOZ|6EMBsIxg=MwI)Wrb=iO;7fPM(@oS+eLT1VtL;zjR2Vn6>{#&pR0 z%0YwGus9Ak@3VgtOeK&C@UzGlt=qM`!(Wvru}90>r$CQFx%7Qy16ps;mL^6Gg1qd7 zgfFiC4FV?6Z4GpOwu96Vak9ltg!ImR=I-XuJIoT826YrB= ziW!i@-Hbz~^H&7F8l^p!K>`_w@4kJ7e=TWhzA+`+lPS8L-1G(an;8XL$ney+tqdd; z#Wof7`Jz>)veTbXPhTBBng)M@NvCcMCEILnsH4xa2{HSLb1lC#{K+cD-SnU?DzsZv zbJ^KwmVx^($4@?Ve$Rs+?3A72Mj=*iBcV>u9JK&|gg%NOB1Y6C>tDeoaI7d_ z7b@na5NfE|S-{z%g1TAQU!?i}>`!9Pz&& zNRo!Ydt#S9bL7zTP_Loi>3%+t>I|_ou+Ne>n!cz$E!3@o@Q40Ul2)-3@kMAzK!A_L zWiVx2C@y9%2x31BZB=`S9VdiQ47oyLk3pkg$P9cqT$>rhqSp*dP%g1h(Oej`d0v z?U@(I^Z!G8_JgQim+H81P|u#5xDMyq+jG;Z5HGWxHrZCijmi$k|Do*+O-HU3lO>m4 za%n`JXl)gw+O`?2BoXZmt}K{wd809aU?{eIduPXm@?u!gCr0aPB;+-|-BIRw{(g@N zYh)=Jh0k9tjC2}OKXI9CHa@XbXPmNF@PW*q8l<8jbUSVC&uL@mZYcs(s13tOvYlu=qw^B`6<^E`?XOUe0mO7F~ev0iO@` zNO@pe7P|k?p@$ovu;jc!zzr#swwtZ#C9! zIK`oL(>~^7Eyz_2lGfA#(3Whdxk^eEh95^h1YIPCqPdNm`YJUYzCGF2K)dg#-x3|C z>Y(x^=cdYhYq7v9BT%7N?WjJ^W42*Xv%esnXvSN#&ZbLru1!7S&&wRZ^`ARgzoV&ulU)S#Ff1D@Tsh zSb8Zi$J1;ANciQoMQ0vyKl$y^y@+U?6QtFnhvW2ZIdwJ<1!#sTB4m;5lb=ScOC=&5 za#Q^oX2tW^wkvn=^rPEyS7xyTRG)r&)vEO>K9?VDe9spY*ZrftG?!6nVFV?ixf?mk z)Y(GGNZZT$H!nw1J#i#^P-baW+zL%%qoadDdVD-)QuGi#AkUa0?JZXc6Kr!=9yU2y zIM%=S@Np8f;o+i7US96WiHYi~+>fwy8{aAsrl!~&dp4X&mHt5^d5$A%agWP&ELQ+y zwTCq#Fgi_+SKfVee3JBZb!iI-i{Ss*74)V--eWb z4Dp5b^?O8io{QwyvDUek->wxCJ-6T-Fh0VEgZ3f<`1PUPhz+JyuZ&7Xv(}KR4s!P> zjZ~E}MMiVuJ7@(w{%Bq2+k_^*L5Ka@&ii`?%1}`ycga|ZV(5=qSDUd~Ly4_dEZ>`$ zOm%3ilDjx>6IaAr|FOpDaW(m3n)@mK99MyABU5nrps;$M%!=ZAn+@xF z!@eNWbbZ>6VQ>Gq)(%@GOejPD{`>EJd$i9BB^1pHpt8yaNTJLdXW=8>6D5+J5SS-jg=icEuhHD$3lsN-E{Sw}FruWe_Klnc6O-qEc0HQ7ipu!A^?%r!Y|N96oK3S)+!pxk)8N100tAw7K zb)D`(0)vI6ne))BYtxW9?P&vgj;gu+_UlWn2BnqC^DUrvYQ;!1=Y3R12ZzS|(Dzt0Za&Hp&1QKi_WEGTH-qnEyezDK z_81*o-VUgm$xCDrA$R#YvKKTbk@%8VkYcm7?XETK1AOuJ*PrCp8uNUSMBzE6WYn{0 zjf0gOWi!-Fb3VM$+>nZ&A-3_o8J)ypQmcReHFs>DfOJC{6o%@Pk3mkYY4|B&(oH_O z)`*Skua6JSSQaT9M9M7ZD96RThs-#)yP8{#D{^TPJY?>c^^Mibtb;dfx-#7V$Nlq~ zYCda5^GyrD5L@;#_M~Y@Wo*pGM#eveI2kFjdcsl?cuQ_*TovonJ4)6Ucqkv=h_aHZ zu<$jhir83An|5NeJD8)CPo;u?8Ep}GFI4*rfS$Wx1hF(qD!v`zt!E5{Efk}aahjg- ztG%`KkhjCZFemiu1!zDUkg7(TZ**T2kN%Pxe=7&j7EZgIcIX8WEm9!5i} z6kb;%Ti{4mD?Z6}4iKnh^q5AV!0Uwm50~F}u5{T`FhxPZigG)J1eZ z13ZM!+I8w6entcJFK1-X<{Q@&ZmK`U`$E*4mCX7KtE}#CAC=8fjQbC58No-`QC^qC z1p5o2gpy|=Wd-eQY}B@Sg5spUPm_zwnqA2mW|sIKLS0K=lo&Rv#9(`kk9QrB6&rA_7A|{!C(#Yg;fUFz+=M!`5Sc;NnEx?Lz$cG9tQOG z8-Lq?ajG(J=jX8$HONl?rPKR;ktR=X)T*=n>WmcV4XJ*bNf-4a4raY zHLOwu0kvKP4~Z$F#;E+LbTrlcQgydyw9WAn%wBOh*%%vJC4I>(_T!DqjlOhhx934- zhx;k80X^aANp_=>xk=@WS8B`tp(-1JNGGTAU`Ys}QedV8<%ff^L8W7TEW|qTXVVT- zWKxoS*j|etvfR^u%(%3(Btp}QYtn@9Z*859TGM2w`MXbb^vUQm_0_H1i0>Z8*_B6+p!jLWixX9%qRN&u;q3ub^CX zO{(9lOWmtGfncnG7fes!h64>;=h&J)Z@!^xyTj%!E_f}<4W&^@ny#Wh1DOopgpu~* z(wjV9OYy@Ha9H&AkQ;0hu(W1zVn={Y6F;jzT*>f4!Qp951s20Eokom26RW)X5EF&k zFbkj|wm6wF0VInrhiFq=3EubN&b|Zgbj~4BQTI3sW(V0bNXEI?Stte4>q~n)u>Zb2 zW-@=1dmY=SAAu?BBS2)}`Cn&YIRA5`a=1~}P(33h^Rrgf#wc(5>I@AEsMSoTpPNtQ zINxQD*{f^8I<-GW^uB&H<*%JRGrv4LSEKK;RL!M-ojG;tX2p;86*a)f?0PD)?0|jo z*Aj}u!UO_W`C2n;%9qOUW1qnmSuz?5k&bkU%I6x7b}MoN^+@$KJX6;h90>Z({&^QwtuBPIS7dD z;d;?}tCwO*RtJArAD4|PVADA<>-1|AYrSm_8afOLWG=rC!42%+Bly9MU7Gz#PY~F# z*+OhT)Z~1!awWR#-rRtvaSweI;=3=TP&S-X+g=$`usE-ztOYIij~_qa*(*s*;tesD zgKQLB{`VN^_C0UBcZ~_o?5>ybujzyHCLmH)+a8?8@8hhjIka5d-e|34mMUm}WZAq`nZB1uZ55Taz4 zA||P@f;*|Es7q$2$5JULspP}s9fti8cR!}rT=ZpvbW7EH(?LZl#i5Qn-(KoG!}!+5P)930xN8nwEJe!Uzx?!5}{JL#iXm z&8rxqX($tUn>sas;~*7{0NT*Mcc^sttHl>lROoUvm!s~!j@Ts=rRV;G2mgFft7L8W zq7+{{#dm7JW0&|4M1I5D#38_-bSIW`x%pIJd;Bn4TdjAP+aPXIe{!I6?>>E|!wbp($XNSZ zDYMH}_gLYSS(l;__dBf0RW%Qo2d`b2$!i`CSJ8)ibh_vr(2Twb#DLK)Yka%it1 zm)Y3mrx8Y=1UTiC9L)U0EFjVK<^CMa6fyEiv#2op(0>!8))&D4d$ul@s5)$2rPD7hTpNg|`bFAawM6sgCuk-huw|5fJz5_z z@WI=`om^9;TzxJ-kAAmq;%%jW!5kyL#Ii3~CA8e=&EUQ->uYv(pIBB(kt11^H(e9p zMygjKV4mO&=k*KYTGvT z(tjp=6^oi)a9*Rf{jWjzSR2k=iKY2vuk{))zW$iU7>&t^PXCCx|IDTdDdYjr1%ME) zvVIt%e}10tLmoa4W>?VKbRmGsw82jM(Ln&;`n@#-xoRIk)Ol&4qH zANUvcW;xlyC)P;Tf};1O@BnmrtHj;aUu15x!T9N~`nN815isRo1X2FzT zHBSg;WSVPP>&T6M=Iq&H2~09CxEaI&UNtdo1#~`zY+YGM@r%#lh5{ONI)jy;L>Pd? z!R_Hn1NO0Jr`d_7^y;eXT>cQ@;Var7AV7zjCI&o^M3c`!Eq?*q!u=!rYTg_ynLVyM z-}=~W?g67+BYKs$w9-sib!FwW!jeTYoZo{p%=A$QWwEZ! ztfA|;L?a1-VqHZx){X;K8HcmpZw(7GjcTvS)vr|8kN80$nzpEi_MYMh$Z^HJYs=V_ zptO$A&~c4f`cL#9&2DYzR`hEK9ss!NMxLYX5NLOpRpD*eq-gcv+FidcBACFV`YrdG z%(wKeKrSq<_R$-E*9}_*>suH|B|q1e3o@B_+6dk5lp|OW=GJJh=qx-7C`}+>O-m)* zU|B&zZB|g79=6Vqo|TKqZ02yQH8Ybo&lSZfU`YqR)`v)AfQtF&!`Ovf9fGOCE9nrn zKEUlPa0gtQdI?-(*-P-D;YLA~EFQa^Bc7k1-$6PC1`8{b7RjsW%b_#3Z(Qey7_C2k zJSo4}@eMl`MG}k#9Hy&V+pkgn(9?Uypps`8sLdPPpx?}&j^H$s#=9t<^^TlzPDj$o zreYn8D5T~K^seRXA*I<~4y4Or4$&a1s+T;H655g8Tl z8U_X^?4px4@9v;;Vpm$t(avJ1!W9z4wS7E+e@sVNqS6!Tm%`_hP$@d*k8BQKI+wq8 zKL4<{od3NX2sx-x19uLt>Y}JZ1PHHfGjf=dMLdp1B#_$uH@o$Tv1KYkl_nO!0j) zTEKV}$tjKMXKD&uDPlH5t*y(kw$Z|aES9n=%jrpWBR$3G-w8%I+d%);qYuS^w6 zB@LxhT!KPhXrd;qRipmZ4rlTfns)f?VAiUgP|`sxX3dVydJl7C4H^;k_vQs@xs8tj zhKLz%{@5{NZa*yPRB8?!2L>wK>~BuS6V~Qrf%FRjJ94~}(@UZ7HQ?ywd@gz(dHC>1}O1LF(~u#@v-`f{oWyzv+MR=q_@BR9E{&@!sM03$NLxoeo70E)NAidPORahSrN?BICf%J#TM{Lm1gNbt{>IvUsEvn91KOY~{{^j4NhtDl*L-nH->{g`2_&J+B zg&SJGUOf==CR?+{Y1Wh-4pzlzi&^HwXxBWKcRx_yn3WK&pmoKw4Q!BotL8;mK{B$B zA748nt@*d*EJ&@FbXLQbakx0y?ee3f0m+^)X$LOMJ$@y9T%@ljRd?vAiIidMxG;+1cgy zW;1t6nl=2iH+>2wwyWx0brla&Wc^;h3Xck`(kBh=`=nq^Mq9ra9gTFyoT9F7I`eW; zlN}r6E#fj(y5uSOA20L%-SG9R%+ zb=@bO>OY&=L2*OW7QnT_k%^3;frqxVCacA-ku0r2Q~ z$ki;x15kQenou0YHFD_C0125GdU{gSw3N$)xG(**$!#4CL>Yl5wEQC7CuW0wM>&&kJP7A$Tq_TW8x_Nb63 z@sVC$@@xrz%F-VbIC0s7_fb$D{;&JUyRd$dkR|MJ2;@)yip=`&nG{Ed|DDOY`%+H*wNgo+D-^~&Pg&28vM$!~_Yob%aE(OhCWFs`U5 zY+DiejSn7Yf{ipRmFE$ba2?)G_h)#H(9q5|JgG0H5S-a*4+|Kx?CtGW9?)~r9v0&P zhfCzZcUy5!uspFrShiKu$1(j?;ZWoaY2dss8I0fQ`r>)Qvfz70M7dFi=g#)qPd#50 zKD}L#!N0>(pgI@adEcLBCy}w|f(tJeJq%Aby&y?U)81I+8>Tyv5o!$s(o{r1tnWga zBn#7tIT0q!D34`ev+I+p@pU-ksGs@Rk06<~Y*U+s-l`dT>vzZJMLi{EkrqyPq1^7x z-ZgJqYxT;!K5m2g_EpkjUtuyoecJ6mWM%uJFO+|32{e_v{%9Y!hKG0Wtd%;J0QC>8 z2j};1rD4r}=sVk)HLvp3n@)1N6C{IuPF4&6|dZE z(T_nW76nUnrW)oe4UNG64EsdA{RRWU=BP^+I2rDg>#tn7(m7(3t!=VEQD_|seQb4g zM+;w!ef3&JKoT(w!wc{9D=7Qq)%jnV$dlm(UhF}wRAD~>sAnzq?EUwG4{)h3E75LM za8&cBujGDLoWGk{u;$CVR16|()V%zte<$^B!?&^z1?iFgzCY+*ES*?7P#34pMns)3 z_)n7_BDF|n6EQgvzcq2K4$%KwY)9_i-S+_Ux3;$SWg$WwQrL-t15EdR@&qV)=m>vI z-0Zp+DLw5bEl;KhJHRSB8fs}+hbTGAyZcPixXF`ZCaYMGANK-yQo?B4(BM1Uba_+_ ziV|2gAv_PF zi9>oYL^wE2#!+c?8Jc)-Ue2onX8q%y>B7R#kM`*YmHK2vl4=fhS;=0$AAR^3Hek(l z504iN${k)@=epLr$xpbF_qA?LJ8tIx!KZYxR(!;0lvkUy>dFo_GJU@pJ^wdnaf zsEgm{R(wpHA896tQ>*`*iI~<}mB09|9=GM)g5CPEy1UVP5q7++N3nCkDcnfB8L?i# z%GHNJEiHO6Ix97lsc;(VvZ9Xum0+$jYJ%$2g_41A?b@2VY?pUz?`TTt7M(r}pfSs6 zAusdhi*pYCJn>atV9GuGP-BKr0FN{0&K(ykFp?9TT)LF*r$u7-#TWP?!+F&#B*LV_ zb{8E_)+y6W%RWp2OsduK(gl=u@yu8|F zp`!c!72sA6Pt)}*(}jo1a~-{BfKeWYL7L>AB^iNwrhI&KGX_QBjcQI2BGDMH$*f_x zpVvD1`CZxrX%`1>a=^+z28Rx64a zam-heS-`@fgw)`yM9tfL$BS7`RCnHgs9Ou&87gPy`}7Eqduz+;S>^3@b$_sqm&g(Q zAQRJ*%tsX39oxHn153f9TtKsFKWnBzGiZtTR8CkeI_Ebu5h^Z0D`U^gn7H-+=y8&y zFT`2%;9+zwm|sh+k}bkP2Ca+!YxjRjV^d{mtYnJ>)%JN*P5D2%ZezXhLA6Ti+VbZf zlQU*(llr+{M!zgGW%0VXpiVqI;;B$pGQ4ON7oKQ{a6EU9#=na9Jfhfon85yuwWuW6 zf%5}pI0`|O)H^&h|Qn5Kn=@$Mj9+RJn)m+;dObJODL}E;bKz`$Xk!?4ICM@wlMtY4Nv{gZZBQBoKib zJR4vv{Vz;lSHEe|bm|?9bdxMonfX~FtyAd78~?7(%0)$K(x6I4KkcvGK;|-J|IfExcp;(NsM<_8PC(BF$7j` z+rRbVXJ7EiK>w)Qd8{YoX!eZP096fny*6gRmAf=;Y-5e|?Q$e@t_7YV)4ng{Df6i5 zK|C)A39g6-2|)AH!RRI`?L;>3{SVJ>xQyt@JyjxO#e@_s<)ET}$h>vQ%fbWBQa>ja zy4_oUvToZQ2eoGm*#(1X_j))%IZzVZqruZ8Pn*GC=^F-(Pn4WIYWK7GNy!CiZM;Gr z3jJ|3<1B!zfrwiD>o~7J2DUy-kIe)*%LR)K*+2#@J@JV74)}njqb~G?_Yn_6pf>8 z(GBKhWC!Wwmxu&P_>d4qasglKRa$@Pc5Z0B`tHi#Si_KKV~xz|fR$zx8v>ehj5{^I zJw=Onb=|G`3=YMa8%%=8nXV6`j{$SI7%K~@tD|~Gjggo zw!h(WTF`%i3okyqljuMHL{!utAG{o}c4ZyKuV{qe*brsHx`Vb1NCe*K|HtwXOSQgT44#!RzV%{ch`Vl|k zq3EDhmPDbSHkgSF6VQNTkyJ{B0f=Avdj2MEGR77W-tidYamQ80Sc^1xZ&n?ih00fY z*Z=dF97 zlbtY-6gl_8ve3PW!Z`NH#~qO7Ed?WBH%AW>|5%)~IJijBi`OU4%`aG*|5HsPSyH?* zBO_|(pcZK%9qR>L$>7Iwm&r0|lR{s}$v#XEtUu2{A|{b=Wm4h$MH>N=FlGydQ{05x z0vj47zFz!r`mkaDoVm0a4t>VkHAye)EE{pZQLWXz%40?(E0fJ=*FNKb8$0P=Tq*Hk zFQP=QcduRva`x=0_y#U7Y&Aozn>h?!$nUQ7wcq4XP52w)6W-17yd6H_ zGgS-vLG2nh7W$uK@H*~DOZ+&LY}1h=)d8N;Jl;5E@Z7So`R(|91ofPB=qPG@g`kg=lG(F&u~{xA0vz~>hHA14m9`@mR|m{;4Q?Fu=@X0Vo54q zXsO7Upx4^1U%+;p478?b4np>9(C>+$qDTM!-7Ed*mA;JJ{Jejvb`%DXOP8?Wa4$B) z?~g1Qa>rH~+E5DsKViXStVIjGD^burU*6{}%D-Fu2$x8>1%dD{s}OpX!t?4Szd~$A z`&xbI&3`>@#tf?{e-FduL9-<(WA((6k56h#)fXDc1)ezx1Sk@21D-}U%n55g1rure zPhCsR!{Ly#qA!)!yE4@-f9{jB4H+(Azik3P64M*WtbryUpcs;WDi`7d*37r_#Tt2! z$^7bM2dghTi6gjp>0Ft*Gd)XBH^>+hHYHh~$F-6*FWs&c$BEJOu-Wr?oK^8#2>q$8 zhjD#<2LkN2e?HIpI%GBT-)bzeXf@<*5TmUH#P78|+4dXJ+XrrAwHqkpAi}SbX6Mh9tOF`9Z74+ z61Vm{yPLdg)P?0`g(-JG|B^O0XMnx^)>A8%;T{QW$So%)6dp5%Ys|CNyw5fs&Km=u zq2ISf^51|O$a!N$V?1lt(QDlIoka|9uCp1C-bf2|D)2q=K;u3>W0i(B0dK%7ZA-(& z)Nau8+5!5Dxr~64>j~~`GfOeb#YGp$K(wJM;Pder8#bLc-U>Ms^2q#ZKGi>a4*kN7 zdcQQ;V@!Y(kdXC}7!C?t*NMKis6gP0z@1^cqjJAh0`*ssCG|Q1yC5d4qHp5!MzoQw z&;9xX_d}Hsvh!CzrHn~(QMa723OpysR?N|D? zQ)DP6geBR_8sX_GSAN}R%>|opx<+4J7Wbl=@kxE`#RxKh%##CSPX9S|YP%0%4gB7; zQ+W3=Hg2{!JMzQ{&%w^(r+Bg8QvH6-YnlD*hwpTMz@-+x=#bqOGOnERsJ#SoU8e@= zt^1go1_>9B$iuqJ?#$@<62b6wY2oxbREL5LyUsw#IT74lc{0WNh0iyVMX)46lBi(~ zu23Kj_VaUUz+*YI9Bp75304NXI)A+xb#yzvdyC#_iZS`Or~FBdW8ppjPu!h?;`{qo z3{R}!Xp3zFJ#g9t*MISN-JnLwQ@W<|l#ct~DLpsXO#5rCKVjMj1sKk4d`>CVTp1cY zoYYiNz?b+}EMbB3;?dMAhy-(fmGqf2dXZW+H8mTnfn%PB?%AY&0b=leKr?1Zhh;2a z!6RzZzI0YK05dQS84#VNv=TpiBL(7;9dfw7E!L|1!edSqJZ9O$Ii7)g@`%BwyY=hW zN(>l;HhQSNVrb7@*Iu2W^Ikb%#h_z>ilLQm;SFlfBI$u!t^96D{Wd*aj9NN(ZHrd} zqe-1w$sP{h`#R7G2SJ)RO2lbfM0yBY*=GjmzDj;oT@7F8Yu5R7)tIL`Wr7}~C~Dng zRP>CPN1p{7XA7ybsNjz(I#dh53P!8Obt=P5g{Wuw(3|95BUoMDfP^Zep=jovFy_7W4+cY|%p4~;yseA18(fY?THaf=7t;J!Qj$rMo zt#6w)Z7}_BmWiJS8&{C=ec6W_yHC(cIu>!@7OBbJA zVv3GT0M*+^ai{-P@|+IQWunA|Jn_U`{_6BTR`V~qFs~2s%e^gm)!OP2duEgB`R8yP1vQ(nq5%x zQ>;it*zm;zG(YjV#Vv~~y|_*Jg0UF=6i-P};$uKo(W{vy^Hd<-o-|?!Oywyl(U(=*ZMB**0t-aMV*YMe^|V1!e;-?fv(Sqwh+&Qm*-8? zV868HlaFwSC{RrKZ+qHCHL$PE249`&IpGWX>r3Br>Wkr?FZ%}y`A!ojTS7KOj`2Uk zaJNkhE9Yrv#FJuv?iPR`7~ArBHa%NCqmq0`S(-v z(#+I=Z3?wh+oEVpCya>xuY>j;pE}Nr1~4KI`R?T!Pd2c&|IEop67$E0IB-(2BC7wM zHHrd`6V+__+mh6qH_4ao&6s=Ty}De*s#kuaS8^nWR2@k-*Kv)N>!u*xB_Ev{EUW(a zbYdtoPxV`Iaj`+ry$#~g1h;5kLIRF54k~6yMX@uGwVcfETzNQ}Ugt+RF5`kwkC@q} zsJ!fNAiS{zBcJp<(Z4O#kh2ORJ_a)n*(Y7UJ~%cDRTEyM(ziSH2$~3%Vfk^PDIMR< z>8rbf9ElB{l9Fd9|vRU0?|61 ztK2MUV>qSTX8zUp#Y)@7xim~XW5EZQ6Vl(Zww^Y~4Y_$CIdZJ%Pb2IYaRk^PdX{e3 zpNxNEcn3tX)(<-&>BqmE|K5NU1|B3!1BnHb@S3|B8BNbNngD3p;rUNI+LecBZ$pV@ zkFO~IiPdn#)~xorVfA@Z=R>)PJjZfH#lJEtG4m_ae(qZ`<>96JYFvl1Bc&wT=dRXB znCj+MN1Vx(KXRPcVfm4*(&+a^J0QA1e^`=OldRuk=QRdj7LwR}eSZy)em(h&n}(2G ze>q^au!DXOS(OH)fqaDCaQ)Xk>m|M-4qwXbY1g3qN8LPb6| zZGIjD6*izoo-YLSvOm$e(mzaJ4V%ODIAVwjedJ(=&ISf4@QPk6_~lb}>R%&p&Eq_+ zNI=vWSHCglP_??rI%yR)qloU>eSU8{o+i|V-#Lg2ge{q{LcWPDIH2cqbEBKD3hQvo zg#`v!8P$I1X5TT8-Ke$%mt-1$I~=olWD?HKos00f&h+p6B?&^Ox)xjvPNeh^BLUS|?x` zwoDUZTpLy-G=VP9P$w}DBH}3JJCn11naZ#9-4Gn$tS`j5Doc^iez2jcUSS%_Hg!W~ zoBCkhpDI#*XY;208<_lx&c+U^R&l>y7~r+H*j4}2?KP~q6sQ@`2gVba4tS!c$NZM)rbGZM^`%O*1%Ik;?-55az@bhR~U?QXD8znO;hdK9)V_)Gw+$1LNS z|DK0F_W7a=yZnOloY>PFppNo>!j!69NL_%nE1?xwTa8@??<{4MzAGdOWK@;ZQT<74 z)rS*2K$MIL9a(M-#gR8~c6kn!i@n?g`k{`Q9Jd9l8Yiy=a}>i6v2-ZqvQ9!Aar^7~n07_lAD|6-du|NK34?tq zMmrL`VBCzu5#COPC+i zBKEGFnb(yKmCBnVM45RPGS5O=%@ci=>=vYDG|zbaxaGH9daRJ!F8^GuL9LU0`?O*A zD$^5|Hiwrx9nOekVxeR}16Nu#dW|K-X~}b6!C311;`A%IddM<=UokVK`EI4<$lCWO zoVRb^zUJ7mF6i6TnMohSNDr!d*_SE3MQ2uZ1A{BI%;_U8xOv^&grVU5{wfPn7V*kR|cm za_+llYwklb8d+5c*drce{eKlXK>i&kYpMG89(x)G&I-^EL3;Q3T*uK#hXeCVY*bd^ zYX%P6tYdyvm<)DyJw9TG4?Oi-t-zclkUYr{ye0T+M}d`zco`qnuU|jd_25L(oVF3# zXtR0`-WMl&JYSVF2zCJ0x}TSxW>?QM*u=s9%mwP|L%$&aX$Opobfd=AS7tE-=KrRY zm9$n**}nO1-tzv>|6j;Y(-uiz+_7V(zbisbP|Q71ZC5HJJ&RcF%t@?Y7*M0l>+A-0 z5kZ6G@I{|FqY1dWpt}7C!YVBH?%07Dnf-<;;u9A`74fMCI_r>-rQLX~&Qhf>i9@LzQ}zw$!}rO=$72u-jWO4^Xoscq=$n0>CX zU2^i7;$&~HA(DNm1fs%=yIoN7iji?3J>cD}w+p|&cdiJ!desskqqvdD$_;o|D4V}%JFGQ)#m^Kmm=%1_$WzmG~ zN^BOh#J@*7NQk9mfw^r8a0w+Q?A_634v?3H{0Ajxp!Pi|u`8fQ*)H=YcB9$5yCrIL zs6u^eSnD@>iB2q}CPN+-=LhRC$Z=3=2J(gvA350Qt7UVo?)v&KaxQDiwhnrmxIsL| z^J}n;O(4Mx8-e^J@)EZA03|cO$Q1n!Ub)Cy%%VQ}_HWx-y*6Jw5v44jl3sehT~*vG zwJQ70?Qz9zZpZo5<0@w9j?QqRrag{G>3l(akYRu{^^sH_fMenfuVgBu!U+33jAj_x5sdPO6IA%w<U*{8_u;qyLfVgO!Vbl9TMX`pVj=Da?e3+RBM@EJF2ekojdL1 z>ZaY#%QFj8Dwxg9%o;?br9|e+sQ>Ra%E`LY&AI&h{>Se>eoPiNv-{FH&>}0xdYN}@#VMPr7UC+TWp(o`F=ES=! zhx^Tf*QPw#zXrz%YUUazs;MgQ3)HCvJ%zFl#T!Jn`0Bk6PHbvAb@Oi-fv!%zzUBh1 zIzeC&|ArZV{%JEz>nA#XG41)6XEJr+LZc6@2Th##sBNheLl%_r-*ssFlmVM;*{^@i zvC=$;7LCn=(jyY=1-*^+IX0OufK`+7^W7eLF~p0noe=5Uw@AMQ>{C~|2p={f*!YGY zkQ0%%DGTcuZBUqeGNGmcg2@>~!GkY<{-;mjUzUO^Gg?n&rAGMT)`mk^3dz+P>g*Rt zp^1#({aanTkpi*@olM=It;+Xc!7qeCRIY*nDaABT%NWX5VW$7dDgO2exj4*NsN-+=uA9hQNA`^x4@W; z1AcHm9A!47M(RnJMgZ)`76F$o{`fe#=EMEl@#+LI^DF-`Gq&V=;dG&afLes^+~8s> zkW<&7RadSIlo22Iw;;{`F`>dXvEkj*Z^=mQRU&ppQ82#m0Y@p_MR6IT_|_!lxJaL zs!xdcQ^t2J#)6nd`%VM@UjcE-ecR2V^>;>XBr!4A;L_#KpHE3*qVSkLSIjK9lc+E$ z@hwbT7QVu@aVvXIs}?mhYxFj0Z;}%jcI~#|&Av+_mVLV9pFZXNr(*Mx4+*_?&Nyy! z`$acMd`ALWve0S0Gjin|pOXcRbUYl4x^#IqYROl>sIgtmBDxgYZvXV|*x~R|Ha0hR zUt@bUi=_&K*0@Zt@SDpnD!qt+Ift-3)w8MhC=niH*)7-)1s&`&Tfo3Xw5=HnHG)QV zqMh`6hfB)*P^NBK+85yorY23AcBw_w1*3lsG%APiZO6uRVRsY>pYZ)OimnGiZzLr; z-$?MFvw(0ca=q@M9{)HWA9vgyT@5AfbE$4s!^-Bl#!uvVznaQF8RX;d231DfH9c8! zQKPg@JB{=Ydez+CE!p35TO*6K?dP(6>JxQ2w%bW>ZgCpZ?HdONcCUfV|dlOqajT zPt_bRQArLCTEP{7Nn$BAwTo`hnw#|xp?n%>6X0gbRvi`UXB6K5K!chi?%}*Y24%6Z zr29vVvVZ0Ko8pnEfDG^4xY2mK-eO8)JEVW<+HnvBY44lw403agJ#$9?I-siK`S9?r z++L_HoeC_Pf3EbCzC1?L7UGr2|E;4Gid9*sW(__vrVBfMzHaa8m3J_u_UeHWJc5JN z-jT^!4javNM%B1fA2FXOo#rRzSwj=7vgZlTd4F>^YX3=5>IHoo*QObeDbsp3lIIDR zmdEX*#--=MkQ5^#1>57E*ic*A>4r5#dp(pm&Awj-2IOQi>k5s{+VE)6{H8~b~*h>sb{2%3mtzv{qq-FbY zk!c&aLIyq9?Jh8BEtaQ)`k$*dDlaQh;%tyuu9#tE(5yi%A<3}8>H3Zjb{+T6_8>(5 z{cJ+!egBNz_Co!4GV@W=Z!KLTmRrDYRhfH zc2`f8*3h5o*1qA{$&)7wF};UYLI4vr$dklMWXV>WWj<@HI^;B5ca4!WcAXL$^AUrr zwz7EAdNs87jE;(|@px*0_u!E1d=j)ga^JoeYg2C=F8vW+){^kW6y854gB6GK%8Ojw zV_PyOQ4K;pAok0$0GY_;8CJbIKw|~Yu)md-)|{-_m6M{bOWTB@bR1w-AXmo9qxxYu zf#~ytwBT;wPr3GsowSj>P^6S2-hUT;EMU-IDd;ay5Qt1KPF6l&vViv6Z_K`$%%bAs z;;tv3BnBuAc~+9+wA}puLsWzRT(^LYup%wbArsdS!>$$1SQ5UbVHJ0I#f}|2)*-3+ z5*u>$pO^FWuS5MWF$i$IAtWmNyw_wxTMGp6t2fl>;?NMydmHZ0ITtLX{hG8{mUn)k z7j=GHB0dmwg@aBO(xS-#7vFrORg4o`ZTj}3M-qx3Og=T?e7nBsR(RZB1yrbauHV96 zh{Lo`l&$ED2g`!;)cx*jqXtV2O>D$*_Z#et!J{%3e=`)S!Gm zwGwW}9LjIH+wQ@G2f7v(5qVKY(+ThaBKTu7?hQ3$QM&8t)49jiJ_yvmaOA!_n~!Pu zVJ(Ovome=ISo=i$e1E9)hq1Qz+;#fd#oxYJhHU>j=Hd3)KHawqqH;w|+N9cghzA-zE2^-*<@G&zz^5okDIMiRt z^wdCM8LbXpwz7)8?jE}?lW~2+;8)vIP({_FDlugR~Nq;j&!zT_bjiJ?xFAF z>Q|&4VB`f^+^}(@?R#$(Mx``_YVxJ{TB@hcz77}|SPdvIbR1R6bk zu4-{Q5s(S}(H7rv$F+nc!Yxr#uS1v)cQVflqE%~B2%ha}%2RYz@`EV`?Z58LW<_;7 z!ou0^SpKVG9}&gFA6WnN;?hH?)&Lz|Coov}a8XQGn%!QybZJ(bM0^xHAD^i!mQ`K? zL{`2O&8Jfc6-Q4;E7?%VrcwE7GOSt6RV;bBdHu+WTaEN|!`5E7UY6ggq{Fjon=5*8 z=hcLon*1nc=Ih675`yx>y zx`ReS9GyTi6bWa$CPBZ`^sF9vcy(IdHV+spdsg?ng)btG2VLB0H6-$hzAkzsNsaB> zSChT^Y;SMx!FTp@%~Xm=?%i|pSN0( z41IUunJ~TX96(;od$t>hCk=8ZUgK|9fPHWjb#0o=d3t^&VS+#k3;&r;<+H;~{Ngj6 zm&9diZl+!ST#{JP27K4Rd@((1+&v=C`eAg`Y zc%to`{p4(2RiCKCGE0mxSYzbM-ONI<<{?F>{n0T9wXO@Ck|mb{S0j4@=@wy3Uv)FO z`msDO_5;Nbm1S^!YpZE}mKqiZMpr{gjvkYE>7#C~mr*8(h-@%vcK*U!H!Yrf9vWcO z5A;_bqY_)YkcUKzt)Y@%F7PjX;RAV=ckkilK$l7W{{A;yl4{nhx$2%>H*zUuO~sX# zI|=bhz54ZQ9@0(cy{#5{>)?^7i7|uiS}L+o1sJls+1y5b{g&2rqBN3xVP#;=eg~@P@JAuQJm;Hqh7Hnl1qD6s%x^h%d;6h2S$w>)$bO=zJxa)fz3Xho*=r& zoCo(f&x&-a?ktJK+@1N3O8I0&|B{^6H5FNXFt`%7D0`V8AK-3L)Lc3{WQ(4S1X^T1 zbmSDrV4Ik1(x*@Jx^)fwc0g!+g{4QMrRFuo8C%#L2^$0Q6Y2A%c?o(MYh}oH=Y3fyDNEsQ-`Vob5e%nwrJHSo4+FcHh1&P?%}Rwe5NlxWOOmo%UXNFHl8s$~Bqy zVuhoZNo!@?_OWBvYIX>+uwQjFsk`!TaqE z7hn7_wVC}1JJpd!G~XeM`xLIe@t@m1axS;|mj~*X4-Hwvakl=WLx(|jVGm~*wLFam zM}(#)MeGFcS8C-htQC??XgHXzIGrc($U48IAlxk6yG@{LTbmG&s(nY$rA&pqzN(2v z!X;od8qc9RjT$uyuKyL~JdrX@X+un+UMaWghh4nl93A|iwGbqqRMy4}#)^nUdi3~w zZ#q>~)lil_zv(>csc^f9LP{=>1Onr4?RBqalw6clBl`?DBiWGmU9v{k{YzXZm=VO# z>xk@L9*@3sH9!A|;*xiviwqlPwKu_Ah}@DG{NaPAAqZ0qL6X4{HDkfko0qrb0N_tr zi%UD`!mxv)kD~oF&M@R2nKXNLl#3_k;1JuTQYJAa&CkCZ**WieT7*SguIzzG`5CXH z2Z!jD5i``jWDZSR!vrS-eF%1`t|59a2|pf!`-LJL{rHmXrM0{KHEhHweV*NQETU06 z&idyUk1yr*Dy5pFY~5C@wsRyCvJiM6ab7l|XDzuT=*CRmKaUo6l_Ev^2P;MujpBJgvnn42rg`nQvk%vtdFW6Rd z|JIV}sCjPO7A5HZ)0tEJcfWg<@Ih9bcBu) z^wWA{nn~jXQ=DOC{6y~(REh*!B*r+L)EQ>H%y!Og_tO*5J9q5ZKx6CFZMzM3kLU=` z%Dm(|sFwr?Lkk_734$XKJyeEu9Xe{*-@ZV)3bKIcBT>$Y;aL}FJrF(#9?Q`7O<$LT z08WJ#S$1NC;VW9Kprtu#f`oLNbY`ePj$t2}t2Gy#{7WUk*URmD=#hkLBjv;Zw}Dys z(N=kXz&_?z83H;cx$yS!EqNwjQDD&V?df#XV1RSt7<=Tt=Qw3XoT?C}uU8hrO;v?( zmUV4qA@rGYZrvfj2}E&kpVQFCoAHGS)GU8%dpXcet_j!(vY^kdM0h`7uq1W z59{qAlSK5eXr}y z+qa1dhN%-OEo0ddqv2#9>tE;c6@x)Jc^w&R4Rv&MnuT--194=6sTyLuZ9-r&vVswd zn}w!YPjbUpgMPyNZ|YaOB&dG_o$`{5%-sM3VN# zHBj?llLG21M>&6cUZfhPIjbl=9SjUMm(+vKGnjUnF|ro$wC^WF+8Mw^E85+>3;&@I zk5LXRZ>Lq_H)Ho)5l3Sy!&$eRTjd=YwGd(E&bx;1l6D#d)Yv;|;r8?J>*Vo(uRQVO zGjL`mszHFR0CnjzSv4tJ$RR^sKj!$K=FT9Kqw*NCBkRxEd|e1$23H~cVtzmC+J6mO zi=t2bfEUb0W-U@u6wtr-!2N8)h6c&r=>DCeLsNqSa(1R`mrj+S4AfU!)&)2XS-jX- z%-JE=-ijrQlNqRMb`rscIa!5yd-Tg~=L~j_d31hXu?Jv0#-CS|0~Rf!thby7MefXa z@ZgX(^Dwk%5II8|cHy+q^jTZv7z*pTdC(dmAw6|gTLH`;oYzt_h9m9rVV+5i-vg&~ z`KunH;f~v8O%m9fXjVLxyBLSw>is%vTeQa-wm0j^^!S!o4PKv;=s_-*+3T#P9`3#K zt~c50O>x9EYn5ULADh@`FW;m)=gYE_JPEjnY+tAUjgeZ64=>B*TFeq;dHIjLjQ$pf zcV$%$DnS1M*QB`7vn#LP&YKgE#fn4N(2xU1(suQ#EemKInRbo_$vMfIi?*$^J79;W z$%q&^z-*ADD;?52vV?3k;Rw;$lI*+cn<#MVk*>iY`K!zA=hZY)jmhK^ zHml-Lx|{fBd=4|LYiMS6LVI%5xJwlpw%hb<#C66p%&8-wF)-Nxl1cZ?Fk<7sJ}L4D z$_<*gfNRjjM{hu8&19xkzvHu(3xi>%X*$el_+{`o6PV7zqDw3Z)Q_RvhjBmj_5N`o8N-XUR49 z@5I4`u13tf;vT4629;+HbO!fWmz8&K2Cr>mUv%rAe=G@HR8)HjAxesH+|d(HqT=Vy z?xC$4wlzQ(nf&CcL&yFXiUPsK!v#KM?=k@Ra6F7Q%P>Q6I36}gR7I21_0G@?!Y%G! zaNxUm~uMM_m{V$qwnkWrq0ko4$RUBF7}Me2$yCX5|_P@i~#Gn zdMhvB$uza&7p`T@pdxntlw`J+{I&J?P;WOlu zngiJvLj^Ud~Wj_ePxv*PfHL8N8G?Z4FOL0*U?~OqxmsTI( zif1N43A}UFZU$uE#uz|Y)zcEu4x~0Q=|K5P&+rgG7z`Rdf5W2AqCix;>_g z>IuJex|pVkTxY_mc1p)s9X7x)<}^5y{%>O{WG@H_<7S>|)$D;r@{n@+I9_!RTRZ-- zLjG5mk%><>hZo-HHesZdron}jU)uA2ZgtH)xMJ5?w#ANd3u>Ze+FCiu4hCM6d)WHm zXnCZR0n9cbYHHFflF^R8Z99`Tz=!F#fwjkX8}^ho5r)XArBfQ9t(JAO+|QF>x&Yz` zXwY&0gwU;9Bi%MU!t1n0LkQ{VZezNQc(eFx`#%&D#!v*=tlf}`O5POE10V4DyY1|k zfAvYw#x5|E(oo}IU?b!T`S0F^q=@k(g)G@cgKcvu-%F!TnU~IPz;sUaWh+?lX%UU+ z8!MKD=+$kcV>SS;%ZnMBoq15k#fP)VJTWPD*i=p>#zRwk?$S)H-$uRmo7=lO0LgC> z^b%^EJseGv%Tv7y{D+0qh|a2iIHSeiagHZDC;Ay5?zTtICZr<%E$BEE^oy(U?!#rJ zj}B$Ly;|0$vHh8&QBh+ZpQx>1U-!jv)0&n#I^#Bw(Xq3gmEob2C)F8;gTn9$mX;7n zy}p*Y627L~-AU0?OneJsNRUlv3iGEBR8j!fEy_OsCROf>M;0!E9OkdZ+XC|Y>e*N_ z3M=sJ45DZZ3=Gh{g{l(9003shMBs4-R6tGLJsa{)zTCa|pwe!1z_^;{_&GmE@ z0o(NICRUU$yRoQ%G9y~En0JqD2`j?RvlVPUg+|7Y5iz#Hnd4=Kb(IKzX_U@8 zo($DVBqd3FZOfkiXC5(JtDHyLR9?j0avpURVnp&Di(2Le7Mw=Qr!9;2_1qR(-8GC4 z4WR%3AUN>jDcitKPzNBe1ts4kF>Ec8S{bFL3QDfnHP+5<--Mmsw0aDnJNwKHzdXih z3X5U4xpPBPm^!7PvrYm*oG>%yk(iS%eTt;7^6cR?uqB0JHMOO#&lGu#nAZd^k9z>Y zhr}Kc9+Y{u+IW5F5Isemcl2%>fI#pis>vzjaTWQ>Q!4}Z$?!u_@n{gQ?gLX;^A%u5 zc%!=>>2DXJ`CX}aX$4dW4q4-nV(8Md4}Vzw54`Th=PQm6Dnw>b^hdm=>(nTZt8+i5 z?=}PXvi+lv(oONTB`mD;p2eULJ!uUA5LM`5(~2o;9iRds_o8b%v?w01JlSPypO{A{ zv`3wc3G8I=c@c3nZT;aRVoFbY?YU{NrEUC>x@NzY*jO3JugbWl@6Iu6>szFO?7YT7 zMPSE1?l?lMA}=rSerDzY?K~_dTu!Tbn>znu^WAV)#IeywPXYVz2z41PV9Y%LTgx}K zScd~w)~6Qs#I{ z2fuKj)0HXvr*@`e2kh)22qti@{97W|X&vQ8Pj~LNlr-aQ zH?RFzWMz-WwuY)H9CuvA*Pl%d>-PEKd&5082pfYAFc#+(<)?2c#gI)BlB)%}{S9K1 z>^S1Z+gKwua;{USs==EJnnVn^+CDgil7k_0$xqN2$xh47%Nx=KG-gD-TB}jXIAluP z08m87#`bG~PVvdYS*R(-p6!3q0|qf9Vv$mR=n$h`vhRjlWu3}{2MP=b0fw}gcMkK+ z5%wAGvEOsIjS}-_{js?$`Q(FO>VU!LptB820ag|5X@QhCaW*O(v6KM$ZC|q6d4|Ls z;a2ngb~`z3s#kJr=?Q&&WwXrd^x&?h$_U@|z)5I1p3489y=l&H_PC_ooVK*!6<{BNvV>7HT1LTh% zKai0gHpkHpwfZ?pa;+d2JP9IbWU z%IK@?>!HB#TREH*qA1fBX4FB`A9?T4zf>&Cl5J^U-!eaL-xn7*QAb3abbJ)greJ@s z$%Pqor{=?_TeGj_9{LB8={f?XnXtD9=nf3gtGZBE3Vs;i=1MD+{16YVVV2Py=(C_e z2ZbfO3>-W5ubl&$!mxmC(~c;@Nhv>n`6+o|cec!SdG>n2GzLMxM2p+cuQ;d#w_tl2 zrrAoh?Tshb3Uhr2@89Y*KJcm@fw;&7Vdn&uLR>KPFJC_JXh3*~Ugz9BT>*hcXe+cO zM2{PBt&&=!Q0Q_LyC7%|I-Ty88{1Sj>@Uu$CaE7A;Mn69-j=uD_ePl!|AM@X4G|6j z0f)pw6q_--Ou1*r9k2PjZoNcy-r`HdMbzjK?KsP})lf!1JN=eyRJlK1-e0vp-pq`l zk(HITo(6V6h@SN?3MIjKhxU!x+TNxKC1oBwA$H%Rmfq%Zl)s2UWQcKV%GutT1^uv$ zAwXlvld``-S{q?7mAOIN*4tDwPuXX2N;R9Er#i|Wz&b!3CLDd|?L ziPJ(rC0YAm7h-wR9D<4z!P72lr3=J!w>5Z*z0sEwY=h^|H)Kb}zvtEKFiQ~l++WTE z^>U(wK(sUHkWl_$4ax>S_c1>*D%xr>kGg@y9HXCr<7d6?0x&Cj)q1$HzGTCpMe8Sp zFje$FPcJMQ?lkW%d>4R%+?l(B?>9|87g*9;?bMIg4JM-71Upg8Uw|Kn2o!o~rQcEl z&z!_Ti1qgwX*S|2+c)mGMuYxS^0_vFDyH##uG(gA0(x&Szy`-(7$jTOrF~oHo6DBvNW-NUam<1dUEU0#noeeD* zLmSOv-g%)54AW`>^;b}BkZz0Deygvn;D6z^uh(*8M`i5`r^;7w~; zl_vZ}!z(iWyq{GykpG+9O<4weRh7XtGf&Fd(RBD^Vy0{c!{47w3NHToaHzRsB~PJ2 zg9bNZnG9Xt-aFX7VPK$ZFGzR{7~;ApFHs2t<0n0i@!*SgnK&=1g zy29`P{4(@LJ9gitlnN6T_J!7-4K7=it2u{miY(;FCBe0uq`Gd~U35XkeeH-rk1!=U z`3s_Wh!0F_Ur@7gvH`oajEwz99(fuMoRt#{QHhuw+?YCK$y!}{pX0B`ED0J3I>A(E zKi#XRj|1#Lv5p{r&GCJH{$O(UiKe0CNZz5DecehzswBlS})o?N{4tHxNV&)##apo$yL)$w z0xT=l!(20z2|;vAz*)9&{*b+dP|#|=gQGT@XXrLsy^r7jC$BouOF60`c>R{NHgGGH zBXE~8c4UE)9#ypN(tz*LeIQ; z&3d&lbOZaAZ~`T^LfFf7-l3Y92=vFw2(Y;7@KfsgEt;ka~UgrytQ?X107nu>J#yglDSY-Bz<~ z*{ph`#ZN0u3tAVvP_$yfvN?s&;FU*kGg;*#&=DCUs7xOF=5IG*{T^a52*zF9xz{}x51s$JautGoV7tas9g?)gj&V!c0G!=lcw}mV7yaK@ zG|JsvP&D7Q!{;exvxmK#XJUEBD`E%^G}J(LBYvM))ql+k&tcW84}N;&$kF@mVmC<# zylS*YXC!-FuOkig47W0}@%OiYvAlcB*JqLx%R6h9x3MR>D&>XxVev&4HdZ+!_H0&H z19KD)tG;un2g*dCjJLYmHxMHMl_z<*){9yDbGV}!f3F^7S*0LvdbBLe}2pYPXD4w~wfY@{rKUR5Q~s;U!Q z_-v84vmdCbs=V<^#)Q#ZVbh4RQS%}Zta07M=i=W^G3rkjV zbybvvXFTQUBlu9n3kmV_UwJx7*6#yz#vKfR2CwWzI4>0`TN8|P` zT299qG=JEzdbE#S^&2KPMPCdy0E@10I>ndu8eOL&J{X$9Yy$KIFx1qM>IZ8O9*MLJ z+XD{0!k0^>=QuH8B_>w&hiN*q+=g^O8seEZ+iO*?rn-cGBsM*Fvv+i#o@tK!^?ZS# z!7o%wuXX3T#!NH}pU`L52+wZ=?K}Oa2jra-79>5OW#887N_y*v#tQw{*cnLZb9}hL zlna_*P`SMt*t;#r7)ONax`QyP&Oe)jG)&*!q4$!LM~+-Zc=g?n6!Usep+#n69fdw5 z`k>}`v;q8X$omazRF6y@!UK4k>`Pbzz;Co7WvuOb0B~3G!<( zh_!=PO+ouqFa7RHEb#W!%AK566{q@8aG$^7!5d+v<2PbhB)PP-f2(^PQ`WIeRJ z&Q`ZolFBh|Tu$!IZ3{k<`@`oW!p_RpZ#sklLt1{RAG*E+oa?@A zU)tNuNMvNC5*mcEM=DZQLI_a_4J0bcEJ>51)J;W%R4N)~LsAJPdxf&2tp9m^RnPst z&--^c?&mn3Tit%Y-{*UMuIs$c!Fw7&6vR?_tb1qy2TWQc%9$f8tjNJQ330tsHk?9*NT4(@kWttkiS=l4{tHdFq~GZCdK+)f*W);x(6f$sx3E-u_QEM%@$!1OGc0t5LtWUbcu|E%2%^Fc z5k9P$O?y^i#VG=?PZ!TQUK#f?bU6UkfL?%v20Kdvr$2UKOd2CuS6M0e?q9O15-$MX zV4+!gC;dgEM_2u93yywfQ}c7>u`f`5lwhfL1F#h|@KBk=-DGOsrO)cU8vxZ)b_or$ zefTZmAsX2KGwwq}e>DcQ?v9Wkhs$VGvU+VdsH7_v@H zcwt0op&W=pRWzzZ5<&QRS-Dg)W`%kPT|z8KjJWHwxFpI?kqOY!sa_kvv>3oxlAtnK z%!Fj8_&H=6{5Ug+99T*w0dv~zq!=nu1OQMXyfc$*yaTTIEcxd%j@VGH!rp zMluHE0gUI&2KAjal3gnn4Z7u{{WCG$xqrhjj~yxyAT~zy)Kc_yuM!-ecu{!VZ^J$c zyjND^lEhEQyn?wPC?PB?0ErymTzUp#)dY!aW|1*5v(P`<$bw_-?QL`aGkAM(doKEH zr9ew$LC9ktFcrrn7o&>C4@4l3LvOMXKgwlC=&at5G&cM=CAB>&mxaXiIPhOdg*W1} z7eQyR9n`Smg#Fb7T(@?B>OdxfiNK4DlP715Ee31JL&n-;5)1kY^bFGtG`R#ZuLFm{ z?8fkO>R3zG;=-5-)GWjTLp!FcE1JExAjJdUBgv#~Z8+%5IEu6pfHwMAm_$|cG#wwWNEWGFcEyTy_JW+4*-*@Rz49#+gd;lF!wd4LDwrQ zur8l?Q13UaUgvfD3wjw6Ou>fuu)Sujs2&F?dy3V38~{05-aks-VYrns?tsu5@(a0V zfv_~3s=urU;W4oBavK@)%JV4Q(oFORuQgvywzyI0TCthWjoO42^{_ya&1k z4&yqmA&9#m8E*5$tIk;Sr{4oYIGxU)f(N00__Cd7`VvMf;2!Mp`w;r{L-7&VPcNLN zyTRJDCo=P}{>okWYhh7dfF9fu%?~d`C9MBptCnXUT7_p(qKrft_vZ4mZRe6bV_vSzJGEKC%B!4{Vp^g>;m(1g|pi4?84(T6Ug|Ls<;9-6?mG z^qjbKMRDoUv8#Wv?t;`0YKrV_z-;y4<_ym{7$q9 zF+>A;R(~QAo3pG|-yQ@?@(eO2#()==;V-OLy9!LjHPp9KJy6xV(3H8 z&E3bD7m0_s25t4+Nj|usQzVbn?E2+2YfwA$z?85)@4bZueump0bkbOQlvKmdftQ*QE-%d_$N_VDVcI=C51)VylrazeT zG}Ul2s>kWxz31-X9Rq%|3nC=qH|#Q>|g=$FvZ+qe3xp4$1)a^(^g;w2Xt5q*1BB)C#}uRgFpLARgPO-FTtoIdWhR6Rsd!I_Elx0#iMIU=atv{( zGRAP0=f%YLi!8n^oz(m|8|*#odr9(%aW`H-r&Z}8 zLw-3-{Ff9r_|^H1c8nsf2IB_I)+DoI%<7x{2SAT^!%TR}<38?i%E!Q<%M|V_KJ?u`u8#f-ECic|@ z{;4;GrU~Ow$|);l-0mPnvNftCw$qt2ds1EF^x$1^5}*!XZI%slR|&v94UC^-`u}h+ zj!rFoaC8C4_^J!80T6? zynUQJ^5bKcEB6f!zjapJ8Y(GBVK!)f@u|4)f|Z;Nj$Oiw=5JU-VH1D!UG9DC&_L#2 zIA^;ncz(b`9~f$KV)qrOkhh4D%+-}*5V`uh>6^I_wg>_EBdO1vw1G@vfvmL)OhF>0 zB582wFVX5M5B_>Jyzsbt;8WA>jBL*}K=7Ee1aT+A!w|;A1Wk?@C^F z@rC{AC$5aM6IN?U=p)~RhuOk)EDz5gcLttT3XMmr#{*;va81ak29Kq26`H5X6?qCR zSRUrtKI5_WIf8~-?S<1raqG~n5B3N{l}e{BVu2n9WvD@Yc&yd0aJ&|56`*qpofP?7XpwvhmO>JOmACbm6BHS20f3B>>afU z5TG9i{RTE6+-*}?Hl#EH16tM8JP8jMX~?ldi*|nkS~XTbsUYNEFoV42GlUl*vwJwrw&t z1#{_c4)GSPVmYp?Rstu{x|gRUf{a{Qg1r~N=3kF$hMfvA3`AC(B)1jcp_RnBefBjd zZY^dUD6{-gpk8F)aKen4k*_fu;1$TGPBAk?Gy&zzqkP#&dv6AQrGZblvIXxH^{{Y8 zVFT{>qCq;R;OS5;Y&)?FPutU{=#Ou_;K1CKRaX~qH`7v3zzVz~B4V~0I=$Dha%H*7zA zRPlLvF~t1gozsz3H!>@RAN2uv1GcNy|C&O6b7I1O#&1up!yU+M#z5`t`u?b+yW?=& z$4$-VCW*@+b&op?T@m8K(8hA_!`wrR7>Gzv4illzCj|Nt<}%F)va18*ocv_%W1&$+ z$~D|HQR%28F19=j!Qltck#hrtqsHNi1nj_;4QYxH_*IOoczhJE{bT{X(x=$uDa@46*ye_gZctT10AB8^djiJ))Nv3Cp-7pA6BJ~# zr)NJAS6{Mq--(}^I#715ot^I^so#ienSgXSoRc{C6YL<5paP~E@s)^y=7`xhpJ@?+ zi(`a;9UqD7nCm?{e=-g5YTv8W)lwpg3OUt4E9E6FhTHe6pf}92e6+7^4tzRUFiD^h zJpE$S1lLG#Zh*%?-48d^Gtd8_X}HPp zX{R)(rx-6AaeZrV*)n%bf8_1#E}uYi3kWLu8eOk<->*k!*X`*p?+~&TpORr1@`uTV zM{=eS<_qu=+v;LgB{$qNIAvG4HC&_;1b5fXPmTO1OFY64~u3pPEH*vT+3%hILn{4-;D zu2y`-jm7BP<4c{pDjZ{lslAo<<_22T^7X6H2c%Ht1t7O`K1?pL4GXA@WW`?;Mjp#} zwAaXq#0@~*j2I&=t=o6*oQma?bif2M8+dg6nX9+i5`e~xENR$<{W6taV7)X4?-}I>fZSBV%Wb*A=fhM1|^k$Pa<*i9{pl! z;)jDz=bCG3ZtLpRHgSryjPfD?5uXJ`5|sU=81zOvvBGEsHMRzJe`j3C5*qPTL}_8R zzTWb9W)V}NV{IL4i3E5x7;HV=)T;zNY$$%G91?W?wzy|CHj+s-lYuX`p$y9xumTBA z1QS&Np1+U*u^xdI>d_ZgSV$Z6;94OaDAdAikfgFBfeY^q9!7thYny_Nh0O^%M z0G~fs$6phZ-Cvz`(Db*dLfz#ryLB`){FQwNI)!o4?^>v`a=ijZuWqG-4yOEqe;s_7 zH;2Kf4)xad;m`HTUEW#3GY2v2;v`Byf$Irk{_ng67~1v@V3a1~DNqcWmIEV0K?D-` z(*-Gt7lZ`XqiWzpBX9BI$1bd0;A>w6eTc1c<{QPL4bi0rrEd?AunfwUE1NXm%M&|w zb&3?H(PI*ML4bIhL8yC6bbJ?00AQ zZDlBv{1s=_Ea_^`&VFpX#5B#52%&;$6Jg;mGB7Qo@bi(_jPMv*VF1JgFT?#dDln?H z@XS!Ev9@}?zN+|m8{Dw-fcb;8EPK{0j7!U0K3+*qUQeXe8q8Qb539dFM+894$}e9| zkkJMp7E7!cVEv}NABXQfAZSR7jX&ZFf!Ch3=1SDVwTPo|bxn!GpA?oXY6s+b%SqK6 z^LE(Dlax0PD(p9)9g|}_VtZt;vLK$7c$}xh{muOq9U%JY2otYjSYyp;avih;3>D3K+?D@lexf95+ThgLSB1gbM(%KU7p%%0eoM zmV(P5Zd_$b<-ls`51o^sL{MgN(YW76j2=D_ZkWsm!)O*4NF&LUo`+jk8gEZK^oLFGh6fhSGKCv-*BOhL~h0OTdHGz6e4oyRa( zd>pws#q*a}GqQq0ED!=j=pue@E6xD4K&avuK6ZhK%OKP8_R(esGA)73P0w7;zL?>} zZ0U*`76Gl9N}>2ki4Nr9L1~4~vC05uXfSf*BFY_Qx*5e@p2C7~Fx#CO784f;b6f+O z3hI`z?y&Vpqcu_|1%9sS)uyr84vHesGmHknciL8+eJI51MI^}IAV28HB4%Mo@!*f7 zI;CiI06>i=ii?2DEmn)U0exoSnX<6+aB6|Z((zN-&0yOv2ok|IEuMNm3Z)*SVb%8! zcEN)L*|}y$*7tVup=X*e{=o$>4fgR!TDo3mUKmUp^8dky{@+XlVC3ri_VwLu;++i5 z3#Eg3^~A>|Z@k%wT-s|lZnT{6`1jeAr&Bpi=)cdVVAxo1=(5#LmFpkS_r*X;VuW*j z)}sr`D3=&w7aSm1b#NU54J8n)3XrI^k}(iX2MlFzY>m-<6l&aNXPF=%v8V2hVgYtc#%qZrW*Fdh}VLjCk|^*L()3bWpY+b`}E4o@}SZUSv2HIeRV& zvdwGoNE*WiCt&Kx#l*s$QqN)t3PEx8(n82%Dr0Lsu%t{Or(UpWq?3g<`~yb49Bh2* z&H&Cr#r|oAnMKI0*!PUgAsE;>+?R40go%^ zsNwdVg#!v7f*eK@1-OOyu}k+^eS~=`EJ>(|Mmq>MUwpPkclNAh zI6_O4%j9;JDCQ8>g%N4sR8!rD?p|I8~Puo?t{b-{8{V=;=!05 zTuUE1C$J;2ZZt|p7f1Pt0BPDRGMu>=TRuQCx>Om=B)U>Yf2E)&Oaek;7zPbA1Q#F} zg1d&L$m0inxorZPT!U7<^21oXRieoT&U%tS8fGH)?Ea~IjQ9scbZ|rX3$U!$gSW(T z(Ijz|9bwt?%y!x;G>N#OUev=x3iqs&f{%^LN35Yq>SqLMD<4S?QPLjc0OY~8!%TP9 zuLN)t;)p=A1ws%aTI1K@0rUke7FNVwUsC}}BVpm|-j~s@8oj@lH@d5xzJAAVHfDeD zU{C;~KX&1bm=ikjPw5MfEk%C0iz6m%j0vJJ-QgyVQ~9?aWg~}zMRs*7o*R>=KPL$# zR$(w_UD%3_r~J+~|0|n%#l=^|L(cwq#Trewn{e0%z&`Wl7MVGDhonODjxJbxPE(E_ zGHfVXW}t;uHc2cc1`AB!e_`HV-PpI51?@TI4zEl&;cyu(VI3xDhSrMgXuLdtgH6D( zLcG4SAUrMrncNdBcI}kuQMU>ml3_nuj&Yj+BspeTRbC&qz=wg;S|F*1M1 zM0k`!MiHX*=Fy`^5s{J5V6Y#{X~{5dxv+vJMXq*Qt5r`AIjw zc8|fJ3a0RX6Ja#)@jdy0SwZ;39%v_#S$mvsI40;h!ZV*Wx=T4@2}nj*ZxOeZJ@U)N zQ&~_e3&uFu_}3qL$Xi-ca+P-pq(2V)520!#HyG@v0Gz1*SY)0?vWJjQ*vgS%7I%1% z<(WZjaL;di)a{)RNH){x2rQ z2E|Lk@M|#Ky!lvMT%75{ln%VtSO*5a?xT$}XkvFfyXwl`{ChBVVFu&J14@jCOi5Vc z9fGNiK6FoB&>KG4$`>5!tb14TLJn1r9vpD#3FjOvjI#yscvlA!1F{E^^c~(d zG`TW+o&5^0VEp~ zdSKIH`3>T+a7i#=V`ZTZ7NcRvC;V+1$&MT)?cle!H}+( zwVil+VBCKZWLQarbZr_7{I>9hX`)MpqXH(DW){N_a|H!w-88@bv2e0*2r2c*J!bLw zwMI)MC7%LR9{4mb=U<&*$!p}#lLrh`Gv7`fLdy>G6x#45+~~Pzj0LU-FcY*OUP-0S z0O=;7&9K^SW5WW^BFfX&_yH81tdch z4_0*8umnQexV$k-iv=U^ELvDl5oYMn$Iw1sQa!|v-ui!W7JUDyu5RnIMbF9@-;kpN z0+k@uexl8YgMSamM+bIhxKni zi(EPzOYQ)W%fhF3YEx_)-lF5w!|#W{0B)9hYUjq0I|rN|k31RaOL2$dmsn%v$f$V$ zXbCf$9x~uSOGbX}IR?NXkB&{f4*|*mhf)L}fV;*V&*%IzmYW${!iCn0uqeklf=i$o1 za!dnGvXmir-weXhw*G&H;cAtrT``r-oq$OO%P9t`{6`qw~6&m9^DqAq2%9J~#{* z5D5-q7Rf7iePS4J`pZ-Z#sQqpT>Ae9Tl(XPu&^3JKOv|W(~Ii9@EL;OvqUWx}%>l8}q3Gku)WjX!nL4PQv3!okM!5tMxNE4FxDU^` z;4M+3K9eZ*%s}1tqPVyKxMV{Y)?Bq+o=~EpN|j~&%UqZSYM>k2a+Z2uC6@*G!7vkx z1kqVnr<5#U&l3zU;M|yQCIgG*=b)NMmLUQ9w=FS+YR?@&KHtwV#JYbU%%oFDwdvU|h%JSIzLA8NK)(m+NoW58v-*DP@5_S6!*Fse8=L^8VLF(U8qJ#iUNcDu%lcOknD{ zQc7Qk#S!y6ZpURXrl_<-$sNT9#BQJ|+ErFo_R5XKQfDUC4^J%E&hS{dRZh+wFD__m zx;5jdg!HV%9gJNn*h0NNKrU#Fhg$0w&~6MrXc*c8O7+|j=&%TNnz8Z#Weu0iBS1l# z3eo7aeei?`_cxqdVl7x2yL60F9g24>&+%F?g%aWMFq1jQZ2Zh1)+P^Wv}=$&8+e5Q z1_W9~)=DjVQ_AiZqj14NGoSkVKLToODgAC&pC`U?-xg8nV?+qVZTHNhTH5^~h zgA?1vZuVBe<`Xi=^f8o35G|id$^qEFK7;5|ODBpO?J%O`LrDRgfmy>Aln307Jb!## z!SjLy%l#WsUWG>M{q_hiqtRLsRS||9?vY7QfMYpG+J*JnDd@QfpeHSM<0nW6koGd> zU?cen4F0?auxv$R1c;5;4wJ`d*Meky9iRDScOIf2G!jdxhroJwvw|kJi&|=Gq}Rg z3kK~3hqDp@*#Vrp;o?lpLKjhqeFwj-AYYMItRAT6h;vs((fLip4(;mdk_tD+-W+}r zo-um3L3jSgo`Ktib^Z0Swp83vz{?+(G{ZmT7@0popLSYs5*V**e_WBcWWe#6*JiYO{xoL#A<3zag!W{D*Nd{~V3wiv;R zYlSqA(0q;spbtW$N*?nK7SK+UE`*5AjICjBD^PBj0DrKwJon z_mm6TbV|f<-@ zFs;|3NF;ZWL{Hh8hY;NXbl{CmBG9BjNq|~IYi5r!J6WEDi5 zjWK(olEI$HE4P=!wp-JXUnCVB3=S#sj~b$KYHQleClKVT>SJbv5ccxNUg=bDT;VYO z%w7)P+aQa9%^}n&_+SunN&O-BAQ~FHeq119t$^r$>jKO_@Mox!u+D|`C_}%1faMi< zW*Ej`J!hzJVWrnZM!opXg7eL4V|hjf9qc&m7*|ElHe&LHF^b54oKtySD%DRYmn{1 zr^Otnvl4!lJHS6Vc<@o23D9s1Kg^u8ToGEr!hO;Bk5IjT>wL}CydMZJC^KRW=0T2l zzlg@j#m(Kt=LydC0=O#EUK@B(?5Y%fm;+&afOGAaxz#xblr9gnCZ?!Yy+euyUH?hC19lMa0@S{`lHfP+=ecPZQ#y78ySi7Mv%!C z3&@dc-lFe_4#!MJ(ZP`}_&5-Zv)x!ePLC)RXl}OA&^&LouK>L{yfc1J?hhea1L(6s zuo~jvu}TTtn!snM335UIX|!;4T|#<%+Ty&Bj6x%is;L!u2h_gB0h(Nnpm^H~+|)j& z$>kgA7b7w9m}kzM=ML#ob5xbVN<-J_`DO!`o%j-)C|c{hXpb*z1DJlptQ>Gv8(I4D zXeHdi8pt@T&s&Ar3tu&VwfnxEjm;WCgvX}2SU&rtU2)-?JIIxtcIiG&rRoMAB&mMLV{)jkMN7%0QF8%W?DTP4z>dzbqc_}(-O zg1v)pSWbZb4PMsYZDOi}+(pl<1={OW@9U$Ip7xitL={6oGKGoVfW?jv4koX zFx}Dqaw&R@kS5@VV>(9j#%>s0g$AJj#Ho^$U>>H(a|3AdwMWSRSZZ^@sqi?cL0#c8my z7jZ~uOuBVlK{qFhR?b5L+FnQyH|PhI}@lK}Cmv(FvN}ogofMAf95;XtWZD zKpw^%rmT|LQa)z<5evpO(ZX$!2du(?qw7HBmMh0m59~&bUvOOz^Itpq#(I!YH}FD@&)U1;Xxkg^?(r^_*AwaxbET@ zJx!x&7rFpWI`-0NlowGqL1a!c2snibF~8&kc;c|J2t?Pput{e9CT!j?mf%`eOsfTv zCmv6-!m?$xm}`W4ZpXkcm`vKRhjMH`rYNbDjNF8DZ{(PB$hh@^WZ)yp`7wcs{iCBx-zI z?KV`eRGi7YqZ44YNw5-xSCV8K0i*dGFhln+=@MYn!FXR)!2o~fQtB757Xq4_YU#3T z*X6WY?z~CAEuFA9aNlKP!vg4k4WqT;f;aVC(t`cjR5E}RsN>`eZy8*7`13cDFp1H} z&5e!6F7?F1rH>dXCa-oxYo=a#@?`aOy}P8jKMt7;+4_MZ#c($!PM-tUvaUDvT{R3U z4VPLB|Ix7I$}6w$x&k>jCAjZ|z?xaP;(RRX|*%vHyh0 z>Pm?dRa=g0SFftM`ORTL>2}4;P(+(_%%sZ_YPTWNCSM`Q4Lc zf30+^SAP8Zpwna}RtIQmrJZ*!*6nHl9UGP|adE^SLV4@{rF-csoQ%UT++Rh(1$W4M z9U$=4qeJqDy5LKrs6tE>Gcv_FXG~(us1~8)a?VgnGDpRQ_1pP6K6@t94aQh#V@Bcr zM4-z$uVg@Lse~VFeRR&FD0pNGIY7C113V#q0fXB=|GJkDC7kJyU$h`HG@LJa7BCjJ zjpD>ikBEqn3Wxh8%&yh;@83_UZAJ!&X`p{k?MJcOIuo2?7#8BtI_8hGqf!qIclgcz z=>gtb7q(R*3l?n8ACdXbL5(vT>sL#gXW7IYYZDu*v)dA;(~pGcYcMM$qs)I6*0w!3 zhQBUzKU#9I=dPutZM3{&P9qJ7T2yl^6OogIM_t!$PF#fnsvel{Op+_Z{S|Uh1(-n? z+2RM(4liAR$a>g0oGZX;8?UEUaor5c^V8Ozgh{^90)a~*p`jHu5%Fih@8AUi0gC(V zS!{Xi*GhOdGL#PVu@If)dAj#-;FPb6US5lN@|+7mXmWA^eOTmlL{OChBR9;jfYmN& z8u&qw7C;w&nE}d+Eip58uG{t=VUTNTVm->ym(mhNJgx$)xT(U^x6_p(z)Wa+8%u^hYNtz8UG>x17*d!k8%^W@Sxch$Ee`OU@!ik-aGXR z^hn{!0IZIM%QIC~RRxc9uwq>R-jUbtoJTLfor4Z%%(&bAvd5x$$6KSLK9i2rBK@Q0PsU!0^jgqMF6LYQGdlCyZSSVWwmxPGpg64##Km|+c5ULuA z*?^5%Qm}tm)aG}5-x)Dd2c4%ZFXhV+(M^xrN z;$C$R-m!>RU2iD;52|?$Mry!iQSML1s6UXJiZ}SKL}wuWm+=8CP^JF?IL@0)O^XM5 zW}q*j^a8R4YW&Mq8-f#fdc1*T4m5I z9imE{X592rj*)892jMXzO{w((IUygm& zIN=lH0087oxTuAmGD1|RK{1%+3SjeHpO=GQ7bN0}X^KW5S>8<_`r9iw<@+7R{J3$y zgXPlm?k`6v;qf|w4OPuuo+6!FX&}Ir^j>u~0w8~5tjoQIcFJEgNWYg^)i4?hq_s>| zaxt96$CC$`?(pRURka@D-3nnL2Gdpg{q9|m#hi`IE#5yk*FLZUutg`3rg%655M4+Y zh$>h@2%aSPssoGsF}knsFnQ}@WQws$T!kGeX2WX_9D594>jO~ynO}k?C)gVlYuNCq z57#^M+|yDNPAEga+EQ>ZH-4Z{Y*6OZ$SL>7nfG6Fv6pv)rGQj_;wHwE0{l0hVhF}| zV9SdU2WuJXy<-nr;=NrCj5aALiLdxAR7FolhdX%5(-{T~{y4UB4o#kaumGC=S%06~ z=>I;qq?HwMdCU2<@>1qNjsI+l^vPmMt^G3*;Leoul|LVoYYtMIe(p zHnf$EJx_kDfz833-wFVt03G7Qf2OU*=>hD(glTy`sh-W~E%?&bZSEf&CT_d?qc89r z8k@iV?D5AaOm00Foxv1Xde7pvTTfO=`#$h?iQpsLN70n9?uwYLxvd|)*#y>6pLh^6 zx8WHBrh7ri2~`pMS?=zzn4hKq&xxOh2Tp z4DAmv&XtFL?JevvD2C+31{c|dRR?x6`+S);5MR*squcdw7L*69Kp_YCXtuqeWM4Y5 zaQ+DSg#|Lf9tH5D?*~ySOBl60&G97?+`{6P4ND2pPe4K+%*)B_ zim1QpwsA@9ZwNg3*Cw;nvW3P+VI1Ds)}3`>WIT-pN>rs}C2bLtQj9djhR8?MLg_g(ln?%p!I>!!V6uv}vGI&E z-?3rCY+Sz;l9Kt0EbdOh+XNl7`!~!R2>Ypbbe^_t?A-pbk38lN6!!DCa|N7FPmm|_J8r2MJk4T>KwIL9v@d*dK-RkSP3i|@_iDQjr|arV(V)~$hsxZW1-Gg z4~}RZV13fMO5=_7d#oUFSa)1L4{DB7KJ2Q2!O8=bg?@iUjWq^U3_k!3ZOKvjLI#d)o#D+ICkkLo-#>=s^h|y%Y91rk z6L;4pna*EuKlu$3Y^%Q{E4^e%0qkVBX1>5~9=zHa5d2V)1y;eNyLO*{Qn(Dt7ql+* zK<=^n!UN#+3o|?1X&GGz%-{@-jD%rXano`7q|yI&e!ijL<@P+dYHD0In&s4Cm76e)HIa@r>T)#hXP=Z2B6UFjfF;T?+!K z`)#DT;*M)p*%`I;x3ksQ+Mp$tmKlsD7?GJ{js}rT%mNtHVDtj@(MkZX595m$=e5|Id~tSBZu zurhE9Hm$TZnz`~M#PQ4$-2uo}2*1GY?zb<)pqKq)C6VxW>)VvTjQB_5EQ+JU8QRyLA^9T!qUym>@-Y?i0BQhCVlrk?E>8D37>02`1LGpH z@NuD4xr0<`w`W0dZy{G9VbOzcb$hX&6`W>m+(0ZG^`YNE=W!o*{6v?wzG_G7Bv-az&Im$`N$tE%vIso(n zNyD?dCs#CX#Ru53@qXU9e7Wig5oFCh#X_f{3-214z%Vp=$rwy&feLdSBj<)pLSYB0 zd%ghG9ne{akrD21sS#h6D{wdT&awjL-U8l0*qDt%N2Y>2Ag$v78+y7hY~k<>#HeW! zmw4sg7DHXm8u9}Gd&gG2&~ z<|4ihB4pE^K+;NnUp<_2@z$e*ASU z7)W$zSB}Q~1rCB*CK3{>8x}kkViOcMZj}WcUgUT8c52UIkC+lE!GSl~=hxSSf#T7j z4O;Ijp${R5>&_k7ckGTc!8-oyUz+934V_Bi>(^WijR1%sp6kFkVIUrXT~9{JMv)dh z#+r8W27ixx`fIF~Xl)4`+EOx#{r_~5c9O@ymS=Wa4BW@=p4CPI%yza^+wp`+VFt|`V^8%eLz5b8oa1bbcuD(gmS{Oz(zVm zg7MM7_g{s)t0(St8X|_f#t{ncTzQcW{c@LMf+<_62svVlqZLpr9$P$J^ zfVv2-oug()iVC!h6z_q{HKEcK)Lv)<@IZ8C3t>16eg9PxHudCazQ(4t?=no|$qWhi z8ts6^%3!wH36BS+Y~#ryJ}asEPW%7o;7T%0Yy@xQ7_FK;IR3Z9$tlgq+Hmg#bY(ks zhLlx9b@_O7WDp_pFcCW4Mw2DR=n9!tVHBE7h>^w_`aNSudL##mGHO!^u?|)&0Ys!yxfcn($sYb z1GowR3_+@=U9i4J0MksKxPSPTc3I3~0pqb#P(|w){xcLj4a#BizWw_}R8+RwNVCQ) zKb=Jp*CvQL!`2*-P#l8cHj@HAZ$hx~ku__U22S%-x{tc9Z!lj(WE|OJK!hUYj71WG zn`D%L$auyP$)(Wi1!aXng~FWR85QDXTNm>jeb%{}Q$p|bkmLGB6sHi76ptf%d#6MC z&rBqUUlw6yfx?Z#+9^-&7ar42+9E`C$dFwZg>_Q*8smz65g}cmpK|X51`T4~T-1@2 z-a*K8Ry6n|-@V(r&JJ>NC3MW#eS}!R1DuGI6XML9u?b*0-=;IHYf$kh0srsyrjWWb?wj7lJyZaE0!M63|lPuKGSD?!_jCi}UKtfm-d z|GedRlJE)Bh9%XvQ4?%I?Ac%rQgk;a^s*Fs?L5prOHyw47$B2N^;gTapsZ1*1E4}m z3t!sSa@gG(lYwfAxaDIFjQ(Jrl`hg$>b+8)fK)}!GzeLiF8j?D8Y=J`noMm8x0?xA z*w2s8oXd5hk1koP_UL9I36u#4=ET9CnO2&lB(EbEMp}vq`=>ENX-H%tV zN4|s9$Zi$E*$|~1#yS9!dFlbigCa+Xa%9RMwIh;ebnl}vYr-z?EaX-iMZZxGgTOm4 zZTl+<0nOb1_1yyCQREYy@4->h)-^bcVre>-6>+uMJliL;PW2Ypm1z1?fqTP;P!I-b zfj*0inf!NMfDgh*E^$-E#x@wm6?TSC-^Y9WdwHL7D)o$W{j8kGK)IEPzgkX($yN6$;d~k)6L_LE}FKYfn#L1OrnO-cLRMYJuPI zV>dA@TCmu{Q?l2h0Npn(HR=eWm1E2t{UyD(VvPl5p$b<2*KZ{J^UpTKxbM8HxEAX$ zs%)@=+X`w9o^Z6N(XQE36=UCh-Alz6q~HFRPaKVnBW)k!enA`8yv`P9RUcJ$O6Fo1 z*vtgDgIQ}1W-VZmFol~OV`WUnX;9le^$XQ}4^RD&|7rtB`w0w9+8O$4ggi>g%voDL z>I3co8F67w9y3UXZxI=!V@uTTg{|xlI49E}Oi?A@J=$^GQj{VZW6|uz)?gn~zFENQ zHsps;oXFEm3ECF?{(8?0Teh8j3{(&|)A0+PQlk%U-RjcAqwQY~mY5MJjD(f@#c3-O7lu#L6H{@fNurjqVcU|7O1&RLBy6RiixsI@m zV-0Az{B4)7pUt8fkJ=aCxpTOxxTn3RUGbp9k1ulR=^jm&&a>5VeN;+wD2^eTDahi{ zt!Q3S{YHHW17tm7vr^rs?)BwWJfIbCTOhR>utMCmYdg}mt9ygw1g2OXx@!AdAZyY_ z7<`-h+`=is0c`5+oZQ?*oH5QsCH2mV5jWx+&&tZm#>wdqUYPM0Y~z>0xo%tR-M}!* zk&zJsX>VFuTX*C-O%Xo%an;Wh?Q3ou#RAxLQWey$Wd&zPo<2<*X4x6zSrDLreA+m@ z-l=?Q*s8Ak++yeJ>$^VWcyHd9FY`FX4UkZL@#bRWy@E-%J}e|`qDpbexWk$vYEDi! zBIAT}EzQh$(F?6>MAgAxI4iI|_eRXl89C%DmGlrz)`W=@6{SSfwU;A9)WrO%p<(O= z|H4LhXx@)(=HcQB*zrD|{{0j_EBqnM4M{G^N0MDq(b8V#;`Vw+xzS=A_#PM-DBp>> z6c#>L3?3y>GD!G!>}ZcRDf84k^Pc>mev&HHves4%jQz!T5TD;1f#R}z=(1dVjc)TwUj z_5oq4ctJ2sbZAyBOETA}|BerH#zl0WnDGmfd?5qu?C%$P*X+PM2}Z`id++Ae>#=lp zbv*+Fx)j?*;SJ59n=T7#; z9{e~H@Xg3SN{mN7PL)v#Q+joye>} z|ILXe#1{k`C;k`SQ8IyoH$|Mijt0O-QN)V_y_PRkSud6b4!DH2c3~l>py0{!ax+|M z7g|ZE3GjTvq)AFk<(OaQvxulDh<*lB-$Hf)w-zbAkGv6>u|RRZsA>&bI;aYEn0wcs z3=8ARl30|Kl9G`7E+9-;BqC_DU~=w}fKylPcC4B+?b!ZBZQs6~zyK;DAyKgD0Nz8` z^vvG9d-us{5o~f(a1$q`AurKXY@742(wcjdHm^WVn;|LBo>6kzN z_%^ct2dEgndC6J*uGjL#ZxyQ~42qXJv(z90V`|}4VNq?YlQ4ieoRfY3`SVX z&4)im$Ho?0NW>QF?A+OY#dbl!DG4L|AP@}meIQuJk4D%RVhqKMzy}_RL&0Pg+3hd6 z&|FTlO)Z~V!Y-#o8vpCp12!XbJg}w3NkU6=im-hWOzhFU#+tAGggcmxn|s;y;xCxT zjT6>ief+qxu3jW!3SgWI%UtGoDp#G##~HBi1yK|d*=M*t!Is{ZIx{dw&BsR>IQpc@ zgr#PHM+P9spS(m^xiZu4sl}Gk z@~l;}ckGyLXjUF}Z~06Ge*dpss}96ysUMBy6y2-*vQ4+JFuwI|P1E^)BcDwEnX^A# z`TjvY3eL8$NuvAJXHg1h?j z0yd7jE+{fV=U~o?=ULWvv%Q=5y?&kaTt|qz=2ssW)nb8`$^IqfrAr>}{-PaQCM!!0(xTWRmG+oOH@w`Wxsj&tV7N-lJPMZr4HfjgY@=2__e|wiuQy_59dq2v$G~nERvMj zscm-+zB}Ut@nR-s+I&o6nK1d0(yH90$FPD@pOPBT(athfkG|^t<7i{Q^S7>eP2c}c%-(L^p(gtqynJi6+stp=@_neT%ab-r@FyvXx|4|Di0tfaP$v14 z4Pzxin7N>^g992G*ISQ}9Us3Tc9xKk5L08)(Yv9|yd%zvIK={B|4tj#=yZfR$z7ZQ zLtrRNFVptJ0oR`ZpiW0BWyshA|G#R`f$xjmT}|8C1GZUWNGpO zOA-697L^x|EpG2;?o%YvC8g<#wU%iH;S58}CcoeYSh4Ud;s9sw+uM~>g~uCPTNg|B zt6E8j>g?UCpee%c^5hbP3xG8hmrBb;0kuxIe{iP#c$k8fmDL_Mw*NO?tBP{aIkWC=WU_dD4=L-5(_R2u)!ZL28*c&`+bpaNPGQS+TKBR0*j|-P*+yR z0wqNeIpjj_gRo)z1HUI7Jq31omsjNC` zC~S>{PH)3aDd+sa(S36+uakBR?AA2Cd(&u7_Ov|n*+#yJvz#-Ni|_5u-uSwbH+bSI z+xh#sKUwzIKR(GJ*x8!s*p=ef_{q}Yf$WU;DHq0SUEsIlCSnUtXgD-24-Mb9-`TC* zjPIo2qP(9yo1J#HAAHo=%T=G*Y}@t8MV_6vMed5($j@Q{7y;bJZ-H@pD4nr)sv zMPj)H{ZB?MQ&!Z4`#nD5HzYcF*)sR^f@A7}ACu>WuO3z8tF0Y=C2_Ts|IWb!f+M{N z(T98mw{6=!@bNxJXZ5Jcc;%R$!Sc0hx46};nfmU%5ZXWsr;J0z8zN5C1a|m1^6&o^ zGX8wqgP%h8cB_PppR8}TkB@Il#3ccxmVFDkEOPS>1%*ue(fmv{r6NNaJ6<=Cp_HbJw)LQR2A% z_-RYWfHTzx9FK02&VzXk6eTd)OtU3R1vgRcmhhNQj^IBSLHVnX`%l4sFz>HwbMu_No-Tujl zg?Ghn0Z?qGY}l|tLQ}i2B~Nw+!bW!A+j<)%2EUY^5iTgcSG>ocAx#+2XVIQ*d zr1|lNCjrPW2N;0MX587A?x!!meQOB%&3xHY9R75xo!rKY^%tD!#)Bz&=w37~Bv}2m z_g0@nBs=-4JM=+kX9j_Q;n!t(LI20{V%|pivXOk!XCObpq>R*74ifrg+9HMFctS=rTId zt#mruvT94vy-(kF*$&NkIBVO=ulpwX{IZy&6#Xz|c5PSYp1z2~pG7Cnx0e0!W38y3 zhS%Jxs{IqRvkui)ui`(uvOh(jQpsKD^N1t+^Mr~Rk4?$)w#9zasMMsjW}0$6 zOKtr4?ZbZK{PFCM+$m_N6~1RyU2*eRZ1w6ZCT-s=Hn=46yJqd1E97f-_{$?x+nnQi z`wOmYYK3x;{d$6FZ$-Y1&7NO-C(=v5eBfNoS=0DWStGx^Cw@#)3_xQtxbs>mKX}}S zE-4xZhp)Cewy#b6Q^xmC7K+PvD<5?!bH05qF{%A&xbI$eYagSd*K}4qt@iVLvPI=v z)#bqRR!X}L3|_i4>2c?CsiDE@18@vLr$K| z2X)tA`!V$+SQ!b6iRCiAM|Zb6T$LAYBhYCn0s&3jCA;lmW{&yLaY5xx3^=1h0b%U6 zO9zj7t&Mz-(s91(vs=fvN(;K*zuowVILp8NqYO)gJnIQ3O}oDh;oEAyN&yNNMa_IOHDu*xcrk!tlen0g zj}qL|SSECW_jCo*Q+q0~QVMAFSLKO~jLeaKl$i-zcU!|YF$^~+z%d8lN`Er!M?!8b z{uaoe&Fc>fqI$ymPZ;CI(?<72GV7+Kii6I)U_twf+WNXWS+8#U-|fdEa7DmdG28e3 z`*%-ln4UGS@^$_JAcXcy*;VfD{3}+hAkAKCYUMl(Mn_ zl+b+aJ@1K6iS4yRA123q%>Q5ShyQIlP&JOPS-*aKvgKFR@S#c%HyGc}$thabkbC2J zJ}kB1P$jq;G_2xh#lFiY+jf<2c(o3*KHtTV^jjqsqASkwJmC$`Tav3WpMP-hp9swq z`KLD=&Gz}Sf1eWjeRhZ2?o-}DSBzRaTdd@#y)ji!{4k#7QybSn@%`eOpMFVe$~Ub} zEEX=~>*{}UzG62-n9BFdke)mqWrY_H!WsOeYoX_{>!5(|d(gVqTm6zwug#z5_Eo+;^H8uP>c$S!ms${Wh+m-$B{Mjukv(=l$-28J*z39Boo&$TG zI*VLm=6!Iv)0BV0;&IRVd*6=l@9v3wURs)6{PI1*n4>sR8Ccegt{ektg4E}k2h6xF z6kA@`&Q>_ZvBTFBuN#?q6Jg=A{T5lk13b3!>C$?TnK^TF(1s+;5yJ3yclSDw&<&_V zcH8p+;fpPYPnCoeH``&sj-P*WGa218$q6xmU*hGXel}T|?tCFE`%_c>9~Dy=ZZISU`;d$*LL%S-hhoaHx(8YZ7)SS zth(ghH#NJ|wK)k?%*@LAqIe{19OQ;fh4?uzfa%kGeMk@<98++5(Y?L8HGS5sco(e~ zoc`N*vHD7Fg{Nd-VYkyq-kH{t22G8mGB;R%|-2p z+C$A`jvI*kZ<0P{ATIx?x<@=hS5nPuHf>LWIVE1VQou4ebpWKb2^BJ_K#RF}@nS+2 zyRQ>v6ZeRbsj1@+GiT>p8gK5Ss{zK7-da~*|MkKV^eSSZNviYl9&H3}4S>xND@pS*PJm8Tq$II)BqNeXqy?Ib*&%XBQEfN_P z)g2e)RP4!I9yLoWd{cKLC$!xA;6LM9IxnHH9-=BT{ zNPl(lcoyCnGpsyox#ldMaXdV{PjN=op%qwc>@A*~xuo$H!%l^eKRXHIsQdi~Sbg=N~kTmdl%*@>q@=A+ruy)#1Yf1?ds7)zGkoU%hdoZQmT8`!$2r z#kmr(N&KBXBP}{SM#k@#opb88UsCy~%+_w|;McvA9uL$>@t&BmQUGAhE3Uyc!(HK_ z`1LONix2`VbFICU`Dp(iU2g$Y<=*y29RE<6O6Uzwh^*GjnE`J9$fVtphP)`QN+QB#92f}C? z8qUEz!Kn~5X#d`6o(VnJt{e+WO;bmTV7acd70dk#x7hST9JovCLw}I){iMa)>_;d8 z#|1Uy{_<)xWUIEeHVJOM)Azt78Z6`>~gT0QnGc;RY52-M8^QwjenNqKU+v5olUaO~GIK*x{8G7e|dmxz_K@A?p+( zo&ztY4+MQl4(;vlBg&`XurT_YeY85z6bz3nl%)^FtT00=C?Y%D|K4f%Cgw$IDMtMR-*L(-r!_J#$2RihRlQ&^gsR%S||_7o{ZdF9eUa^Bx@PpdB6A zz-&RUDk3>b<-VI6c-!D4!Feez?8_IDijxCYRCQfYGMJ@S$MK}%p4*QA?)s^Z5ET`b zBJ5;^`{D7XI7Wt%mD-_gWVNzk5%_>at4W`(p?z1n&vQh4YO@Ud8%=I z4ODaO)(yWinc`oMNFIMD!yb-!_y=sGy~aYRn4n|*1r92j&3(ds>_I(8Uece_(D^#1 z6Fc1feI=Cl90qfyax^$CjWQ?)10N3!``bT2l)=Xoc7qnzxV($kkox=+*KqS2wti_! z#PG(aNOK3rF``nu7bCa(hArhiUBxct=(0W6t+U%5ikQY=^h6@k?dekiuVhVuPFTMf8fTy7Z09S8fbmFjPIxPT9-ldM@o~ zNHN`JfzRr(=U-}A+j@KLaGuki(pqY#4s@izNh7&p5T~QK4YD^R#|d0Ra>*$qw}25r z78Oz^Nr4`?4H_=|&U{NB70D<}PXapXH6Z*Gu6qN!@wbY0sefX6{_jP9K*_++mcR)* z3K}p+0R?Dpq*fn=xX6H=iMt*LX}n0JUT>cM>Tq>%_jXF_SEy3~dR&G%8GFs-dw~E> zrB+Un*ghk($ut5=Z``I(Dk&*J#vl;1f6L^C1qK<%FqpCTM#wk@>eZ~J)m7!= zt$;a>bT+Fi3jQQSnx@wmeb5JkrsfQQ9Y9a9-H{D;gclB&Oo?-zd!#*N>OqX-EHJ&C zkO~J%0fU|f2>b`Z$~|&&zOt={-@3)t`SX<_(|w@ohNvhl5Xp9r<^0L`JRmzB6xasu z`mI5zy$158$7=pWSOB2ZTM--7<}Q zKpcXU7s3?t7oe8(HJp_};&?#VKvEuH0svtf=7(c2OmLZiOLro24vuI(Cqc+px_Z@5 z@ZozQ9v&W~zz$Z)tF!dy>Ucq>h;+}jF~W1Y37xjn{-6K%1`4=TINaaM)kA%KuL}rf z!a5@DQX=-|O{4S`Y7Y|TK*Pvn-H6)SX95B@YIwb|*h?WHC&3{!$(xFF;Y7JQuKutI zJ<`bV4-G|iXB@)#cqMC&b8~~%e)Ac_!s6w~A7_KZw(8%orjmsO@`S_3a_8Bsthh&i z$c_{7nKLGFH?N}d^ZgDvx7@D6$av@ZbB=|@nDNb0eEdkmmEqfRaxTK0JE2rrk94^d zfHhiKVX+#aJvYYaI`LA}?@v`Vj>vZ4&vb?461T$hq)bk=1_or*H=w8+zcIW3t@Ghw zpTt9J-n=K=eUraF=I$0|-aDtEfdgz70Lf>R@n+Psa$=?ydT4+5vUiT=`e3m{JQchR zvwRx4zu)IR^m$FfRZ-bMEj5aJf$F>%XP)uMvHI_K-`}S{B7ply#}%qD3_0&Sy%$$c z@@H;y--;*1;W`(5;z*s*G)HW1)wZZ1KWW?gy7F*yuYLm80zw^0s@&0w{*8yV94;s8P=7!#NE9eEIGJos-U;37Rc=wc$(!RZ~2*WZDv! zd*cj~fnU!`*_Haauk4&Z$9VZxdd@=}c&a23xKV2G+cX|^BNG&}w>UlDGKAk(n-P>5 zP4oSti(3_pr@{2k5t~ajI?W7|*Uk2u_+EMTsA35`JP2dc<8^wvLA?t+fSCHe!cLyecVg_bu)puTs^1j2NqQ z9&7W_8j*)LxBZsI}Doht4fgoN*y+JZa`{ zH^v^R=Vc>SO25kTdm1JvETODhpP!?CwI*DW5 zH`{Ahm`yz}iwJl8%+b1#FmxM57BwFcJEmMno5%&alG)UYI!7N+ZBO3e)k)6MO#8-7 zt7McMX&t2x(4=GFH=stfpKOw}SH@mIt>M3&XkshsFC2E~`8ZLt?1rk|Dx~;2 z)3-!1^LnrK@X)g`NYCtcZ^pTj15I*K+PCUM2X{)Y8upy!^kVkv>==qI`=y@b&V$#W z+0&UxOgUt?Pw&sRXT0<2MtzH3s_UcftMLmU1VwmDUVc2!JEt; zmO?2ID)CC>l31IY+nK+(AQS8m+%QX}38{T>4hu8W$^>8i+_NO97j;))+!3AxDdZ>m zL$^{7@KK1>5?7Cn*<>7Q0yRz1Xf)d$nS(muYY_p~^XHj=;S^9zID#fcz>vC;b6rVI z?JXj|l7urkJEX~-&EdK%(Bs~Yj7%ujCu|>>0N&7?0X!9ehBX&I%;)8MHM~T~isDHK zu5%9cE!z!zxOl^X^8f$?6s4_SFrdKzMHey(wgNHm`@jU!ws#&aib}0XtBsE&SGUECPS~7 zOXq$mnFhKNqoBONb#G)fH|@*MBZ<&g8?{oWr9?k|q-GzeW-yKiNmY0s9bGRlMQ~QQ zaRX`cR=Vk^KUeqp-hBEQ)W~X(Olm^}+Jl_QQw>$BV^-9<_2&eEeGEn`PIN>2JKtC5 zp*=6dLL{rroV09f?zhgdEtd79UXdZ2b6EyUF}T@zt4C2+KO*ndB!Dna zY1`)~S*kZO(r?ZT2;bGo>G0I_%A#Dwdi+#7=~`D>BYJV?&o=FcN>4gLXuND$JFC;` z>BjdCD@rUAa%@b#EsASH&sh`W8DL)YRoSz(N5S`>mb(1;MSHv5XieRWiaP-gr$6sJ zbQAfVtQBhQn^yUCLo6gXN$h1GxfqM`lrioPs(!8R5UCsyb9U_3=zaBv6{DrwjAFZb zx&PQ2*(Q#~Mbu|pAMjW?>Adl>c~Co;4lgukmTQZ4?p5YL9gh`zVJ^vC;*y)iZodDzRz5|3r7lfZmDkzwCALo= z-XOElHtB}s1IZzjt}A8wY((A9)oZm^s}G!p_2sYn4S#>3b1=z?asFs7apiYUbwEjBX4Nfx2meCnb@JHugpFJK7gu-OeXg~A6;~? zdKXYCS=)D&(NNaS)gd_Ga145dTM;vR9^5JJ+$69O3;tcYOB}OE_@AAPD{O0aF)!mG zvxELZxx{=0ITwT3R+Q>d5b0P``6@SV$Lk90Ysr;Ad8REg>)jZqSZmLscSo~A8jm)5 zU?!sa1jZ)&8(hn$Zk(_m^Hjd1rB&*4-|W@?v9=}9!OHts!lg5C)niWlS9e=PzNg4dW6fGl$P!%}Q07VFO(#@7wIO#oGe>&n*CJ8>VY(u~tDR29 zs2sr;2vhufwG9<9HnJ{gX?*QP_fH%YjEI-?z5IWK$}DMRe3A9@4O^)x>F#@Ywz%(J z`;ObWQMAa2MaGnISd_HVBmQp@u|~a(cgu~gSS>u{Iyuhic5&%D@+Emd%lLh2(1!(P zfYFJ3P!#Z=miV=vPW#=|8jGdgCIZD*>dYgyNe%h_!WCTX&)VcE&s92F4$-w5A>}Dc zTke|a54o#b0i>Ju3wgy{PNTTqUt~jNv88A3gwp=#5Id2e3HDhE`uO-CB+Hvra?e{( zWC@th@x{eZ5S1zf?$uu)B9a5s5=cv}zkvU=I*f+joKH{24M~FshoNQK4jiS1M|PoGvc ztw2Kx+JYpCbQXW%GZ9U=u(;~w3H4<5^d5!)>Z!dd=W4WGjYoR-<731i2^@?-4-~+V zf{z)Wi0J#bYs@A$e)kpN$)7(PN4mgrwHwY8XsohemvUJ{fyQfXZOu+k#pL(r++4VE zGR`wxHg)a1!F=`3>Y^-Q{GFqtB+kw)6ggx~@Ejo+$+#yVFp%ZOjb>yIKiHC#H>N%h zjKRyLUQ*cWk=~N0j^G6e0@-Uc3yXi`1b8DD{t^*qaB&g812CJxn+QMha|_cc2|GUk zn(#9^I(ln}E?gc8Z9ttG3gYV@?4%m{U-j}pcP$38tz`?ION7W4E|jVElYKWk`u;~K zVD@{J2RH@D|1$>;Zvr*~?lr9>lW1{G@b_t&!bV$C_Uo(}JPnvl5H}S(n|KMJI)tr; ziir8*joE&-wX}eBULXc4?Tw6#=nv>&EAG{;bn96ftel=UnL8`nNCX%pe4zP__S@e9 zL599U(gQRfv$812(2n-(!D6j`4E}NkuO%h;PQeukQvxE|&OQKi0|FG>Iz2BD2Alyr zQUS7S8k&F-de)v40)PGPwKvidR0zeAlb()?5NI%yJoUtv_+i2a-ArU4?6_*vE)I_a z^`<&luaTvE0EJe{zn5 zK`W@xk`0AA*=t&KIo^KYIcs5I{bZ_vN*R1UA$|Hx37a8~j)a`rJ0gUH35Sq=ts4<= zVRs7AU1*ObJnM6dVBn~0PiE^J$SfROX-__%t+Rg)RvkI0z?t-FNJ$*>ev(C!`bplV zd>!9;$BMPcO5|11EhUUK=*qx!x82<8K^uLYcqEAcdt^#^GKbEsGiS|+*R!qfN;kc$ zoL9|;`2dscyq5%&(^5{tGm}og+25nb-DfmZG$m}GHY_j3P<3>LY3}vAiG62{YIKDH zcCu{EQ1an1S9ynr&G}?;9c9?Dv!dLW)vuy(26~|(zECdd$t_{g8rRx_&Ee&^^fItwJrM<`dJ8F%w~wgx`sJoHINmju3A z5*0tj-gC+m+mvz@-qvX8tKI^&Fbn+A5M?EQ1@Ecc<0W&o$>>wbh=nsl`#dq$w8Nqv zOeuIr?&Yj82XWvRa6m z>?63ixtBWRQ#*INetjDrK3}mv3t%w@a|wMAeS9e74!I*DB}YBB?Hew^zLdTXP;4l9VbUVEBT34g!bSPWlk|PE&?wa}W|B-a~59>CW;YKalp7MtOZzX)c6rXRp;RoMtsm%(!FN}D&upWW~ zOR9qLoxYHE?Vty|u8)FsZ<=6k_?={tsFC2sm8OgsHuDng98aIgn(7j(zx?PzV-3J8 zGMOmRLPn5${*EaE;d64&>y)TxB~lLV9{X60^)(ykxv)gcrD&>vYN1*NBP!-CTHH(4 z7QbWRA3G)MGWg;;j%#R^I8=mzb!Iu>o`c`S3JbVF4 zG`?~Woj-rsc_9k@srL7}*8!6=nFtTHbXO6Z5TivI&PovYAoE=}pCLiXiV=FLkb(z% z2X0MGuh_M_rwEY@6kGO>1gMW)`^k~#-7@{?iEXvxyoXqm@_wIYS(DlLO=(M3X-3re&&{zps5Vn^rgK6a03J}w0(iP z?R3jj*jbT|aY)Y2UjR^eEVd@HTv$g>>oXdQywa6zaBK>82tKh4@rxD zPl0}upZe|Fnf&MmK!@Lg#x0P@gT@}5oSk$1)xp8|AgHYF z=&-HskVRy2sy8lyC#@Dh5<}jn4%5@OjKu3l-%?|Z{OOTJ=@$loLvitVe?P7v#8F+ za9D@CRUU7^;LouZjnO%SQcRc*38!N-JaTcK-~F( z)V^&ha7N6DUO`&AFUcsBGF;Ze;;e_~M^J)-y^Ie~{=+BQ=g0QjMO@q;e6`m8-m|?j z;!fHj>3;FNtnI3YnujC!n5uhiS;BIH++0Lo*H+{? zsolZq0IF;wqv3jxR}5~1w37XW7Z2abW-SalOEl*;4WMVPY?t$A=WcdYK3h6z{6~FM zDnAqzwdqSPK7jX_l>9rp1Z>x7?8P`e=adYrKFGG*<7o2H*YnM9+PQpj&iJR8Sts2i z{lQli{FhY0>hNZ-ki0sB+g8iHd|BUiYtW+c^5n z58i*So+0#R(U7zcC)PN6TFRfwfB_|aZu)Ys^j`RpC)=pHR{-YZ>$>IhF@M5 z-kGE8Y<2``AX&NaC7kGrD`cCUM_(yQpU;>dS{%c&=y4MIV6PFa9={wb?x#~d`K6RRACXpx1z`ZgtV(ev;H{hYl-ZTA=H0H@etzvgv{ z@jn?kOdP~&ckb+3;^WVYty(;Kbm9Bpir0>agYS|S+Wa#e&`yd?6Rxh|ltVMK-xlXu z3BFq%7d$)X+c77TC*)P=wRw{erPyS7yU(_&Qe0x-a2rI*t(Y-q$<7JkmHZ|@yrq4x zV72Z>c;2KAb6Iat{j|GvdSYvj9p%?%lw7J_%yT|Ai@Li9B||eL+Pf-QS*kEs8=G4{ zk-af(I8sIsC&WX|7mqSEwcb10H0pLA8_kurSdi8!FQRz#Nblk6B+&b%cZFI0J#8Ug zQgm(F8vSHAvk5?|Pv1bE)lpG+mf3#khiq(>z-5u=GkcTaoG^Rg23fj4R&)&|JVd+U zh&pPRCEUGU-H7RsJ^G1XP{dIs!q9Z;!61ZPxYR18>a!GJ8dnwQ`D-~yK?vtR8llgT zrK3&5;*0aOockSaw%2(GYr0R%@mZdaMQ2Nt^@X(@5*j{l?~JFrYQK?_i1JR$ws?6f zYusXD+Re5YR@=9)F*N3)>nHtwLyn(Yt$AZ&D@BPz^@D{L-@CTH7n(fjQTW{Y{OjJ{ zUH&9Ysf3oLXq5AUPo_9v8oehjxAxAYH0%KS^h#{IS?suaa>~!%^i*P4MDH1U`To89 zKe3zSA{`s|?EFT9C$lYvMKqtSuaCB4V{udcBQj2%$>DOs_jye$#m5n#Y#DnPDeg4K+>DSaML_v8z2+Z&!u7 z^mz5kLfdP_Pn;!5~o)r>06g3X9hGQSBP|uXUje#>#yV%AMkuQdYMMkkf zXfP6+;pkYHxdLgJ!4VM|#}|KpN3|48sQ4T~4DWrP |`{4{QMM9 z#`hgyiT2sm)d*kTl?yhDi;Fqmult8xH_DHT@leW9*M6>qPxHskw$4w9nibAi5S}Bd zKs62MtpHzH7^FF=fOM!oFV)9$1Qk@58d9S`E{TZ%I-|F~2(${2R!YuV&U$P6p%*yR zXFlA;&u0Rsrn&iz#Y1gP%@82OMQ?s#s{RH2A|;1Fqv3Sw&qxl)vTR?lR$4=yx$WglduxDkT*9^4^ha|@n|<102}{`My3 z;2S9^D?_BTKL%midV8N0bOMXL1-Wy_qlY#BX{x%0t=wyG!vli43mA#~r(p*NZUyy8 zpk8_kv3l@(B|6;6#h$ewc|vrXx=)|R!Bpojjs!Q7Fp3Amn_JNItSun`w2{6oGg?ny z*n#n+^&lNc1NswjSY6(MlOPj4ZjJ*lphG*N!*4PR3Kvr0{w)cn$UO$ z{F95HN&kNL|NT$cQ%%htmqRccy)bxB6hxZWl&$HL?F6W~4sv#PNW{L5kYu0dAA6!q zOiXAs=0$(sw6BCoLxXt~jXsy+B|4bBjA$2;0bpq~nG}WGbwE4D*db2QVxggRz=9eN z7vi4k!&VWZz(jhphcs6I^nlQW4`*jZH#{RHq1kOINGd5{>|}@CYrmxh<=v|;=(SV8 zxiqGK7PYr*J2wUTE5E+08r?!z5#kOhCdk$ymCoP?kNaBrm@W*Ui}bX+Fc=sJ@I~I; zLy(eNR&n)iM`luqeiH74>%$IzI{$Mn2N?!?>=~-J zy@@bhWrPo66Cy(PF{hf|d1wjZrifTv)XC9YBKUQ{ZY*O3gs-^Mf{Z$}EP<5|76-!^oMrlsnp^<7MPj{55k zFE0YO9CB3k1nLJ*z47|yI$7{OW$g1CwsjwsFI3NHGZshhV?4y0S4J5d8}<$6n4%}h z=W8vBC$+onk2-|xjze=@i{Dz)daP%0Rvbxx(Zvo>NpYi$O(G*j85h0y@QgH=7uQ*N z9yX*fJ$nyy2rvRi@^3*1GkJSdfN$<(WS0 zjljFJar@RpZdZ3Iy>OPd@2l!6zQ9j8_(5NCs9DvnFq&2yF3U<1nmX=rIgT-tR(7gw zj&Zp(r}TT)dUC^|QpEB=0U^+N&sw$`9yPJi3CSBxHRg(R08c_rzJB6u{B3sY4 zQm>=qzqTG_Feo6!sYmY%zf+eJF?C&g6fH#x2RD2Xpk`6De*0&^@z7X%6DisSh`(fF zl(}X1_)M(wRsPNAWmRB0ZlrEGt@;;nwZE7Xl|N!0oOI$|^v8rJqN5=f%MCbTaYGS7 z=2U9;y#2h8Ms+{A!+}^av+c@#wI+&}Ibz~9V@BF=?1y9HNbe+`5718@mfEzrbx}RM zIE%vghIp&3|HBc%e8Sj{nos&|#e;v&p=zQl^S6aAa7o_sY&)BDGGskpC=Uz=kTfjM z`z|fsyd;pi;*|x%h@b1@h;NCGpALN<={-Gov+d6zE=kpOL6)qt(+N-friN4c;qg1y zSL?qc=DgYF-8Up)0fnyy$zr6ZV%Hb>1F;dRjS`p8^>B1XE&u1$A0DnA+(vI?$&Jm- z@as@RUKk*4IMEvKDySoTH)B3?tt?zAvHxu8TYE=8qV5e97n+o1;;(S1QolbG}ufZ`t3=9VVK!CE1w+Ad{z@T>_ zHa9>iAjO&+N(Yo20cJsJE>t3qshXqmS+96|j%HU?F%s2vo^$sP!>MHRff$^Jul&g- z@hgiB6%@CG-FK`I*yz6L?)x}+HsYJD{C;{uAb|MmkA-TaZR^*3P;ZYkqh=06e;=3} z?pxf?1`#sfgKzW<){>UHB{??t}e((-bU zzIUnFSXw&h@uuGeeLJuctzRyXk}4=FhW*EwY)s!PFtOz~YyaSR#WihPGH?SXvHSEU zcHo8u)x`u6^o3jwb&*{_8)WMy%ogCL7rTzpuhhD;`f8u?UqmSrJwdB7Nntb}OqprulW7cZWg zg1-#nkB9~~$z__KUX5+Bhm0%r5ePB>I&7%vqW@M7IDbG?0j9I~0b@vOhtScuTh_GU z6Ibe`|CN5Uw&MZ`LdzFFGffWjLGx{BoZ_PhIgkPGGi6CPFd5tf*~C03Qt7x@5s6d zBn})@$p=Zb|JOhw+}+{z&h+QK>tuHJ8`IaXU3c*z)VM zwzHuTr$=)RkbUVnJ^gZ~)pfNpcQZ)ppiA-SXzpEJmtK`PJz}(gbCUs89YO(?p5h<~QsBlB9Vf#tr2i6ogO3 zV^jCxUJB)qo9jJzK+#qV7IZ$nK8D_yQ}GM2+}6b$UdqZh+c9IW-d~|AAB<3gdmLaH zsGXI{peI?4Vw{1mNr6X{w&^afGG2e4F!gk~TtnK&CrGc-78TMKClYLuH z4jg!$-z10f^rZ;K_8j;|PkJ>`w+F(t5O$+}plI%lVsmo;BjJC-)rucAxNCzn|s zUMweE-&j%G^Doo1;&7yzute+mq9xH65kQ1`ymhQ#{U=sKGrUIcb$hokW%5xA*XKdO zPa9=@a|S!l7B`Fb!*{Dc>9-f}#ic2Qx1ybQrCiUKkMCo-g6fq_df)L(8y+jq$qeL5 zBf$P`Qybisl$_f~ghR%T)K~q0#{RnPe1U5v*8iep(TNlDsE#ukZ7B&VGEx2Y{JANs z+@UV`0>q<-Z^p3N$cUdtT(;X4T2-C~^XHx-y;`ysNGJH+D}*ZBy?-XHcsOUBx?cyk zlC48T>0!%P7%UFg3A4-hO$okil+7_FBwX~#!D^dqyEHdE_+`f3e8NW?Rk~@^ha4pb0$YS{lH3F59C%sJL}qp_d1I1e`Eya zuE02FW1~nq2m9qq_oN%+=5|cNYgahkxr*{X>@IS16jYo{NUr36Dl3D2;Sv5APmgO) zEBdQE@@HnO_hxv?XZzp$g$$}9qRvC_kUJ1=^e8gA#c+`6tQx=d5p{nj!Da%E?VKEo zVeHr|Km{9>ynf!pZdSgO<9WPFn4wi|$+N#-YLxhvxnJZMc{_`2;Z{W&1w z*2`|pH#T)y$PB?nKZ5VGJbjtlUqT%W0$O$B58soh8JmhRnmDs1CY0X1DE(+u9R^^@ zi$Wyt-j}`qVNu?Go%O}9pGq_5#Ei*O#%=jOWq1kD-HBza{cP5$o^{(^T@Vn1{L*2S zd8?I_d1;XFw>sF}+c9A@s^=zFSkYK*$+@~4^}Xf5AWd7}iL^5Y(5iy^BRNAeRp%s6 z$BP%6{_Iz-5jo)~k0;4-^pRYO7Y)HN7*U?B1&K>)2YI1EQqOV(EN;w>;uc`ea^xc= zf9H9%X2C5ZCq6g}a~WrT!61*zMtVpXS#h@&vvu z$-o8Ause;6u%1WVJqtg|$}Q1-JN3LoP}TH*bpKxZGZO{^w2-+O0)ot;@UEC)pe2x0 z8k@=%iN$|#=9@J%f263j)z$g!82U#t_x%3t52mO%1}P+40y5n}G%vd|3|0k*j>;|r zT;kV0JuJ~d4`R8c^Hx(H_`-+!&p$sx$fLTxK)x{?nE89BLcF|W;N|iyF5Y$x#AiNM zRZ%f;@>Ew;BmyNu(!bWr``od{rzgjdqL+ODrA=4)e!Zv_u!N6~TdyJ*EO5|Mp=Ct@ zAypqA#MZWj&x(So-zZ)$52?t8mb`NTGLUW++tL7^Pri9q8o3k%ZSao6>1)y!fcvsC z5JZe!Xo>$LCE!bodHWr4HLLE=#7;q`ikV|Pm0p}L`j~?X4jN5s4UG(5R!=3(3jaZO)#L}G9f9T5oszxOudph#D58O@&GJaGz_ z2?A}M(oFRY9?sL*mx(u@Zh}Ytr>*0vrD_%b$g@#tK&lI+62hIaUT07$QQh94cs{H6 zU1H@IrNZ2EdP{@>828xStDX;-;&m_B_RQfbj(8`N_b26n>GrOf%;Yl(s=7g@Hu?1# z3-cJ=zIc5xFj-zEzl8~HnH$fA5N3 zehbQ4C#fc*JhLw?Z}&D`;209%-D!R#JX|UnE`L5&^3RI%%gq=gI3|5t7^>FQsiewU zehP|+{Bu^it2d9ej}{N3-xmd^QfB_6kaUF>#hEy2+t%d8y?vV9-B^5>}+CG=dUz* zCnt$d%EgkaIUq{_q$ay@zu%5&cPqe<8nLC`h#4{V@jB=pAR}~j;yT@E$w+74qfObGpDxI-dMVViz>j$#_i+p@v z?53OD?q2it$~jb!N(czK_1b;Zje`ck%NjUK{x0>C);ab zA5pRhv3AD9Qm;3c3)i#Ot^)I+9= zC9U4Bvy&^)`Wuvj2~1AaVkUOUX9|SmS%h-vQJrc7K6u58WKq-x`8nzzWmo>-ecJfm zLs4C9CzETI6tN;$_nzOEje76ieIwnD-STy5G4OQU)d`Y%@2g*74`(?u9rE~d&M3@KHHW8S2eNE0g)`(|;Y?d!(2!Xhl zHlU%QK@7&MpIA(&K_y=B^W>y`xtR61X~HftGP3#rqpPCg4Vhm6c7p;y{u?Brs*A9* zzw&ba@p)o(l@t<|4k9!kJ@WY*(DWt$0VMRYu?0Pejf=aBz3ERB2`NWe!=s}sNw?*r zAmg!M+7s3je+*{($12dx-z}}IkcKc#N10SI?IJ52jIu0*K|jb*Puv#Sk!)n~hx8J7 zWh_zEPz&_T9fa;N_mLa76T*$}y6?slUj_vll*yN$3I)rt{F~?qg1_$Ls-FVy4E7+) z?c2}4aYm3IdJTf-YN5hHv(NMlq^Q>XEyi&hK~UzohY&LQ3w&RZ`5!7P8H#Pkp7pR^ zyVkkA{XrjaYQ($%PH8O}*#Oxc z7sVy;FgF}NL#!1PS9C6k_BJ`ZmcMgHqzOC%1nm+39LGYaMw2EVah zG|Jyz$Eao;_%uybuRY2A1WY{VQ(l{KA1#oStUDDVqTavfj*w86-OPPrN}gx>W>wCs z65XDpq){Vdsla!WGb{(BrR7#AnYF8dSh#P2PlvlHch^(kX_>$!W~b@mM1D4WCa?Y& zhfKI#i6-Nmx#0TeeA=8@?3;_P62d)<;Ast7;i8t8 zX+cV0gFvm_S1t{DRcc-Jt1qM|cc2;l49>h;b1ex~PQJPh76FRucPy1O^j+(4Ta5Sb z=Suo&dar#-wOl)B8{W>m!%+nobaAkY-)42 z@Vj9NjpsiL@KEi;UBg+WV_nuUW6$fkESK1VFXwSBJHh+H>5t;k zhpnS?YO{jXwiQj@lk~v}Y0ti^IVb#dD3TjETB6TflILo#aBothEXDu26I5W$o#Aj> zPeFkZL-a91iraf7tD@ZdxT)j#kxHk3tiGf*LgPblJ)vEx=jpQ_?nmcwyq{W~uKUW> zN=nAG6jJtftp{i>m5!z!;tb18&2b6FidV7NME)iy`*clANX)^q=i^cdfHs-FU z@L|!r{bshaKJuhet93Yz4-BsBp9aMNpP3q^vw^wX-r~`77ZwfaFR4 zprG`Mio?nIv^19b`g&mqoVtYQYN6ahCRHVF0Wy%*ppj?4AcAGysG&Z;e+g9J;cz#5f;r$W4ViHFHxxMEw9J5_`dQB%h)MuP@*tvRhDAW& z5_C=cjovi+d8vtX`g%3ICz^e;puq&~QA-HoYz7U}*PqMF@7UN_bm3TBOkjJ@;i~`Q z#W#mDpr$rV0~~C;8mHHu8YRB{TlKAfg7`WogH&m0*V!NwyMMtx6TZX72^ymFkbw;` zQQC?!h~638NdE@^798)ZfZn;&T)_PK^ZcwzIhLE1Hb7kdr>eetP2izfB!oyoq3K_Q z@Sl9ZzN?U^J7|}NZq6sGkR^yXr<%;hji;Amkl3rBz`$RpG|0Y#PKW{aX_TIH|VjDki7mzULxoez(H)!`h^xTBXcRpqU!$t6;Nvy zeA1uWf@=V|7K(D4;KF#vt&8-)EpR}#nh|*&Z;AUZ0{iCdXf*v#)@IQhI|BiHq=-wq zx8&UW#WHV*2{oLQ8ww%&D+5IyO9tP*C{bupad%|oPTZ}wvD}$A(Xv)Wr7ttf(fVtA?WUOw#^Ohi6 z5%vNK!sLv~I(Tmb*d{6YnBzge0(Tz^TSko$AqE&djbr~PqO=(eZ8Q?^aBUQHd;oIuNd6_(a;Hd>h~zBEoGPA>3M&o z)SU75jCKEJIuSMT$l-(%Hi;CrCXNS=_HzH72BebnCnN*`;^K$d#+&KCdw=G;Z8JB! zPk%hFW1=()3{R@I5QIG zhDB%+Z!!2h9@rLP6n)8ruwi$>T=je zjN+CM^%$aid#u$P1=1-p5h-fc*m7>G&peiOQYj*@y+cxhwS-B?yN}5!SMl<*z+9aS z_!-++83C%qz38Iyv0YJvGv)IVX4`BRgjqj}v8EmaRieEdm09P>kaYeNnvW}A<{MnsT;mem&l zTP{}PR6`H(_@6%^5GoZU(HGr1GQt2dUugXf%v^yAB~XDXcz9F*O=L11FXC_=I^m$1 zYlPYDGifgd0Pc zvQZD0ZrR&hXBPi3S0v>x#E*i4f}Dk;{D2IPsa|eckD4gm*R#l0)F=QhyaDlv$Ab&sVag1 znFCDs165UuxA%912flutNEClVwlQ7ew#^QzWNURMFv!B`R@cz5^C91lHMXVgjpJi1 zj5Ymvg@s7bZ+01^^1HaW2(vQsmBB3^UgbDAm?%&6Aqd+T%9JDT<6)}d_MQuwkxNOB z02NCfV*|dv9CBU6O=5qU!d^~xr!!`ZYE3MYSuTpa-GYuQN zt5pwk%Cu}gn#P7uNE{ZPM75@@lA=mVzG7t|8c34lu9VZR4Fyr)ijF4k{t`Qf7)p4j zU~%8`B7RA5*mqlOy|+&&mHX?E*XLP}tfsZTv(3>~x0m;7oP39It=n_V&KaNeOz84K zdhQ4J#LY!p8wH-GIVburaMQojv3h%Ly2kc12nE39Fjjn9tQZf4A6Gv*OUog~Ywr}< zW2)o1G{h)ZW#RA^g@Z!flap3^yc1`gbDwm8cQr-ZZCY))qi8~^rmCnw=nnI7@(>CY zt`k%#)$X5|cd0A1(Xck`wm<7$%}D#$)`^8n=d&`NHw5E!lH!{#g}pB)$Hv5opD65B zq;p^;7*yW{3>6+3Y>J8=?3sMlsnqgw-z~$~f)sIW-G10{my3%X+(ZF?geXyZULp`q zS)-WP@~WY-NGa#oxJ!`MNjoH<;tS-NCh+I<8tf$bX-hGIpZ6@pGz#@p9xQ0;R6GsW z&{w(*dS1x7T^X}&NRilG+Zh@FC#Rj`<+Or=L-$3C2_f^62zO2q*KhRYP9Ni6{Cp)R zPA~eMgkou!#}i~Qod-`&CN%WoUzIxKtacyDrgl!!jR_mGpU&`vE{`%U&n#MwIl*x2 zgFY1t*p}F9v&}VvFv#+E7`}#+jw0tF*gawNbIa9+ySouXn*ij)!Zj!VwkBDULD(^Z z!tA2Zp_nafxAoj-ZVVngyq3AN-v9q_^&aq4_woO?QmDwP>=mI>B73ipj6%rFD6;qF z*fS$4%1T6%mF!JOva`prW$$@7=YD^?e*gddyYC)d>v73E-_PegUa!{^+O2Y*ug+3D zdt%pDiIaSVnbvC(L*_%rsjeZekNoeKsj%^+kh?J1Hh~Ct`?S^kKp&E{oLu|-v?^{@ zU`LnL*;3q5?TzW)yoFX$2%*jA-;0bE{H7-+G%L|BV%PkVw zB{%cLU%y=UO3m#!3{mIgPH-45G7@HEv6-(mlik4(K6wVoM!t2cLo<7Dxd_lV)jPcp z?7=9oLMf5{f75YG-!CE449H7-4+ecksEd(sB*0%MEy3m+vWQ{;rG}(5Am47MM~@zT z#i>KMBTSnBpk@yl2%NtYdGNLK^pvXr!k zQ%~W`L$)GC>26yO0~^3Uui83w<^NJ|sHqj(zMf+-HMF(Oe{$LqtS#_e6XIZ&kxxW| zkcP*7+}RCTGqZtoG6QJ>Uqu*W{$Y}YgoeJ>NHN2vIdm9z>cRgB-7a+hH;H;@OdSs+ zo>@oXSw{@J*372+|Agq4=w2ZrkYrg_)PD4+0SR>&-c5nZ+zaUB$r3)a0Js$N;^O6{ zK$aa3g~Z~e7Ut%PeA7VAU}BQ`dlR}`UNWlN`hGn_DSv|D5T0rbR(%=v@(Sn@5NW}S zoWn5**!d&U0tp|Hs6iI9Flc7Lq7nC)>)|j%5ARo!TdgQhx^%eWyl|hLknlXME!p*h@W`hOI*4b78YONk{#fc4^cx9h;(G# zH#T-M6v}jEvWfHp!6j3x+?n}o?}Zd=B&4;&6?N2i)q0HjMM;#-7_P$o$24}q(s8Gp zSyHmSqUdOP`R2V*?ZY`w6i{Za&_*?A`p!h%bcet2Bid8DhVz&_wvU=#p)Yum%wq6` zZrf>fty1E}#Y@s{RMH;~@4kGHeOHFr-KySUIKIoC`LOS3gxzaC+(|B4C{Ky7rWE5j z1rY%a6{tC43xgVTr2l*)eh9nTjN$8ntD&+ZJj%9FaNyXe_pesY_-wc1Vr(EUQ|r)mqcjalsgojfjul9 zUbU<~&kNf-S$VWE-R_cAriGu(Rkf+OHw*O#$0t72uh}KbE6DEeODtAgSM-&bSaTuX z)0_M`6QBMa006ni(~*013Z>u!iH&n4gV?yN$`nzKnQ^y-(UR)O3tsf4x;B3^J_)~3 zLR%Tkk1DN;l<3;+Z@xS{x)no%I$E^2bh)H3nqzho9|Lm)gP#%O7!{c=qlR28#G=4ur z?)K>)1w>`E3qeo(5B5?^gigw=FFn_4uy~5ffja_bttzyFYp0Yh1Ik-tO>Kg5`RWK+OtUK!BBkVQJ0B?5tE zCzyRxaFsU+q|fNIK)R&koDzhlpfx!_X#@`}myxXzAuWqq1dI$|#PgxBuv^m&vu6V| z$E(cD_2IAnnV<_nR>catRPek*sMN=+AXxf03_9?ZfGRKu37TAD1TbK4ue$PkzBc$% z=6_G>GV}6C;LJBEI-zQX!1L6J(L!Ans%Z*d-Q5Xa4>86A?^eSQVTi;8)o7%jpv)kRJ62~JUt>TKYSs*6I|_gs4EMEZJ=?; zUH+ToZ2lR>Ef;A?FuHTFq_D5h|2Kk{Ey&3U??+v9zE%o3x{*y?t?-$bl$90#0p1<=U$O-nLVW@;80DzBf9?$&i8*AflwToaec|lYfdooZTsRirZ%-;E=~jH zbxw$Uh3@OgjH~Asgy%yO_Tr@=Oi(4@LZ1Sa*z=P#@w0@4rbs?$SQxd7&qrWfdF`z< zfW7p|F7Tls1W`^=F)aS-{{kqOJ>?VM;tZENp^G%m_0dZ*b=124Cc6FpK<`Hx$O7qb zWnhSI%NC%x&jxRCBi?19GOm5tqmX`=2H#ITr+7G|i|M`9MB0z_X2p{gdYS~+#BG|r z-;^BAQqe+>vuA4UYMewb@=-C~;q#zXct1cZJYv~VP*!rU&s2|0i2%pYy?1m%8L^5+ zUGQ98=D)EpzwgXfldquq*#D%JTTFD+(L>*RHSbzE_8!k|-^7?wk?nQa?Iu-_xi*$` zMIvEdB$7#lQ1e%rE?!vn;v+s}7fpISbn@%^fP^tNnmW^)qsVBP#@f>W8M0rl(2oRx zj=-Cy#<4={Cy7l1DepbWVB`O8Nx&`|+>W0D+Di)5#yg_K1qw>~vBDExIBaF*`}N}S zu^|^w__(_dCD46bI68a!w%o^B$#qB?V*Hp z$nBEU*`wdzc@2u;?9?NQr2^l`L+{&rt9pU_R_sG1-Y^40A3xu(J_}_#_1aoAU8o0G z#bMDeiiel;I>rsnKyxSt$rwR&f|X^Vx!P7_wyhGSl9;$u4EfW8y=!pDh7flA`0t{I zf;v@-f%DZ@Os3)bMFu_V5~{GnZf_Tvdn-4B{`hm`g(M9<4ttDF3x9@ItUt-+=d5}X z`D=^-GW!B}Tv49#_lsJ3l!k|HU-~->hUr*eB@MjUSO9Y#yb<%AZ&uR}13SY+ zIesbIWaqyu%r$Kj{LO&m#DWAVqL?xL*p=ai6Wwo-6Fx_V-ig}aqb)7#!`(*T8mi1B z=#P*pKDSm7zgZjJ^f|ogQqP8;_*9XhX?=6d+Q+QVG*#G2dND%%Kg>)0(wU13;V(=e z0)HB?z=^_6Sxp0Nu*`&vNH9N(xo^~K8tLgp!jtuMd%h#Xc5+tn5)Dl_#BhBCYJsL3 zcsi8RBp4ub(egGnc{%#}`Y$1T zLgYD3AVhP9Ko{}5WoxXxOd?s2vAC=654}dy4>GCILONNn}=$OLMB-jqTChp zwMJ2!A}nTRi!~>E6SWMmd}9$7e#igB5rjf(<5i)FiPWKzOe~qpiB%l`<&)9-Ln-*{ z_4RU(?W}+KWI({$VM)4z<~9Y9JC$D?@z)jiNH05jwhWdP}$Feyo)?jHhX zQnQE}DW22(ZwClqe#rn)BLJuR%7)BNUKf5yK*(0JvxQ3l=2KdyWHD0)%p)io@tj5W z%KiAQ2D*27o`gGNKDbaw`9z zsU09Zk@pVKp$cV zfni|u!6$DGI#N>d>Y8gOq_INDN}txXz3#<~IYIV=m`<;uM-ixn?3tdfRlu}_;5$7{ z;zXK(V%W`~M*rQjn8RT~+YUzYnQ2Giu&_cqZ;Pq{Jkx^N=$h?P5`N>Dg+38xh2)37 zYx6bcSpt44rOH~3oiOavoRW>tcHkmYI`a^OUypH(h%eM%FNn; zgDdV_y5&0U=tx@f%agXcUvrKSLB;pnE?TBJ#S=uV($7@!87FLlRBfJ?_}Eu5qELH6 zhQleAzqmynMo%Q@u-ZK`-oKd*0uAFYNj2BbTNx(x3UVl`BwbxKY=yM7v*2OJlLGg5sjbV3_t}D4 zPlA>L-zwB@t_Kk*6OEIT+`XdxPap)Ydb`|JE7!9uXk7l^QQJCnLPQ>7wj;nbamO$Ce|_)6L(v_h`8HB*2=M+05?W9!UM9Phfs2?gpL9aIz6=_I6>4zG)_ z$ll`OB1V$=9oN~q%H3;-=JluxYyt zMuYe9AD1L=CtLs|D17CDIR%CVq3St72q^5ym%(L>;QdLdoPHp2TJi}Bj`kb=uk`z& zshLP;M3=c?H6BsHH z>cQWY6|;V|Bmok*qZXZ@gAE2?;8?Xs-`ALo1ra#@m2l$lNa_562!0qr0do~%A%Uu* zsH!TjuRr$rgeg<=29n5;7*zssSU|Herxy{t{(mPEL@Z5hgW4vQxDDg%H*c*$RO0Lk zgsaWXS06Gmayv?GdJw}m{)JwDAec%Z6w~4+`!68`&+0}51zgMacg(t^0l@{aWByaQ z`ql+C0{EcaUTGi{{+;?`Jvb&N=&=M?0Dh;g9(#;?wOqTj^-%Zdqd|L3${_KZ2FJ90 z`^9%0o#evPKk{YlJ?J&^n}WMqSY9NUw)heAQ5o7rgZ3o0mUWeEf#(>gzxcYUOSc1# za?vf(EiiLn%U?NsfBDv4*C3bOv$D%o;YSw2A{FitA1v43<4S8fvB_WHcs`;|Xm2?9 zf%j5vWr_69GBKF7>&KnmlskR<{TlU{b#Q9Z?)J=RTK?+3)6O3!qpNn00*w<{7EK?2 z&)IAcRLz)@uFA1(Il8a2A75K=_nT!bi^vvCoV_=sYZoQpU2k`hN`xjUeQFvhF`7nX zu?BEDUZi(FfBIB`(Q-ui=JPiX9*nLSmVW(OW#jyvs$74Y^iS_me4Nbhk=-n;Pw_nB2-|df!&Xh0Acj#tjdK({g{g}eTVfU2)uRGil zXFWGJK579PM|RaQn-ZUOoCRQEgX9_o-^4JC`nAPZNd~4aqxQ$?tM?64`*-&zw7gno zfEeVuSFDv#rrnizN-F2Ooo@h)A3WC-{H8_`Oa=|3_b8H4hWx8UC#?cV{em&tR(N+@c;2R2u@F)XHSG6`Jf2%#`$mUQs%Gw zH_L`L`}@8|y%O3SP|?6Z;Gdt=kJb+E^`wzA@*XTgE#?m$JkR{R8q3#dt?z+ILpPOo+li3jNaM9iqq5A8g}?8d3-wLE}3bXbdla|pso_R zw+_egST@FuVu%8Fj&ai7uhq1c>9T7y`OCD5HuX zvsdeM7o5GoV+>`Fc0*JrNJ<_Ic6s{WfcqJAa`0sRr&NU0It4%lSp{H$X2=TDN}2R- z5C4Ayh8-Xq6N0~7%drKtfoBLra5Apb?ojSRij2v%mu)|PN@;5T`JCbxMyxC)g=gis zGC?T>I(a70CQjLOZD)X(=U(0#Kaq+UAJu?U1u`a>iR7-m(+>m3DX4xy2*(t+z&3mT zQ;)^(xWitvoD8F9vX)w$Mb7=Ds}(&8IjIA?n<~Kt}|8U*>H6OgQzYE46%I1tH*WrQ!+r9AH4sFgcMSE z!5s!nrs00W+@omXu3@mD?kdgnMA)x?3 z(*J--0B!oI(*%G|Z!{PFhLnJO~*> z*leK$&d>wH9_UP7rKORE6eVx;L4YzeC9uV%?jHoYfTQ3V+juZCnS!-ya9Ud0 z|97Uak)^uZoTG6W4{~e-;543EvJ|C4c&-AYatdDWC?>Q~Z<1aLp#jlhVWj~hmOU|l zu+4tV%k%Q>{>-U1FavE2L~d|$t-R;1tkhi_dfFDdc=AL`+s$OpdZBBXkzrgsRVi;E z1Ij=p z`j1%1DN1qW3L&O(E83l-v6D~WgznC%%0CO(iXyM!O3X@Wt?^zIj{eqZc8~O;kN8r9 zn8rIV4NT)LBdixROM@Jb)-eix-W?dc3E4j1#|uN>)9!6n$oK8r*)(fozF~okeRmwN z?)nmgpk`=o-P>fwn_U>iVL+TzWutj1YMBF+w?TGQ|06$YYJGB|lfF65u?udp#@{!T z^~4+I9l-!MVtvQdlic|mHMz+kX#aVk@`$>a@2gDqfe9Gn^@rU^@w$gI z1HQ=OhwGs^nzIU0!Zc7-BsBv>k3+AVUC1Sq=wR9}a7n99sbJ|%cj!bj297GMWt3*- z;!ip#WUz*L=_`I*>Xypxk!Ajd-OtT<6kaNV5fN4xF}IBwYBo-+wxBGgh$C3ikw9U-N9d>CyPqsyylB{*X)SV?kq@m+laP=Y`u6Ms96wF879O)d z$=yBR9uOKb?%K`Ng)|R<88Rohxp{*y%@p!cC$|?h$yj1G552$YS5Tz+?7jO(l7kI@ zBf^LMk0hs9mVb)qQMv0XXz@OOCTeSI>%0&_OmjOS5>7EtrTx?#gisU6W&hDkvKAo1 zTX}%N6#v@YEhYQ}XRrWr2G>EJc#j2rKV=8`R;GGxKQOgt%l%IwWafG_I?aX49<=N* zlnrQI#z{ev`m>stvI_m%-rn@0A0mk*|KmHs`O>1P4UIH_DOtwd;(OA+eEGE^DE@yA zV`zE*#k4tRRlp{81E%cvlC$K`RbA_s(-6u{=M)LWG>gZmP_zu@JEVFsvso zFo;PM_bUBo|4knp8+!@J%wk}$o&xF1Y5%8_lzrs9R~DT`I4`YiD%8Z}5jlK^tT!-4 zhq_=-;Zaah76H?Z;~u{w6w@GLclR7g-H0)!h9NZ6p93m6nz~#!-~5uGL<`6DSOn?c zHjh&I7Sgo<0dD}!1^)K?wS6wC%a@DZo+BmA=J5-PmA-!CMkC1f(g&BH?1JnBT9nW} zif3Kt6~b)y0qgb2 zW~Zgyj1qf`~fB?9TB%3Z2KZR#0YZhUmBLyu9F+4(dY~HNW#|!`n z4Z7q=4pqRf04G&Sg^f6!JcNow)#~Hyd6(ruECqD&slLglMs- zk?AWM<9}WgS^Jv$8=&)Qg-j$@e$G%TU?u~-wGJVTn%L4uH}4c^+nN{qE@xFXN6I2b znXqR3WmBriL!EDIKX$Ol$e~m9mcG}D`7L3mkQ#%pPfQQXZeDG~@98n*ys>n&mMpG! zF2vZ{(CB$Bc2jF~U%@>43QRTL{{lW;i1Kr+XXti0hh!ajdjiUTU|{V~3`*r^kY#ni z;!#_418iTp^yJ29Uw-Zaa`3jow$8eAwSt?Rs9aHTW3`G0wgHdKosCNVc_C$5=r0Q4 zh1AK4LXM~6*u94(Py&N>DH7t%C%2B)zOO@I+g?wv1QUwO_64=-$*RChPXb7xhU5h> zZndU3Qz}()=xMc=T+~~Cx+8ID*R98>xtsXMAgKJ))X)1j3GtAWRVb?Kyr3_vSE)0R!Ct&+=m>%tX;jUyGf(nPy3|{ym&NT?X0ZJ5@*($q${qX;deG&|n zK*u9?8rTjAc^?$!f;lH9qRVZ(|Zjfsm!}l`L9{_FFM1i@#Fi8zGQkD;B~hBh7jqgHHi)X&LAPWm{wVs zy1=+n#pTVLH~&exX-`1k0i-`27bxOG71_R3gX{wBiNltbln8Ugl5@f=M(LWDdOA$9 zBimjh3nfT^LdL$|b@ztCVC?(Qkxquo!p=@GH^Yg}jI*S)bUI#5f@hf{GBUF1%i~eBlP|_`zf#DO97( z|1eTIWvwods);2fm-!7)-avkd-6Alp!WalYKR;hJ?8}37r*?4jAm}R9J23101yi5G zUlieI{u^bRWx@oe3Dk}(AgW{B-rsKp3Er^@q}NV%gG_%*rLez^vUXF za~jqef3!>?(EJ+56O|}>++mkN((=Uj7Zir56AA!0jg0UeY%~E@z1cmqd2PBkCGE>e zZnEVM39wzpfO~z_BEh_J9a12V4+_9H>B}vXD1y<_doxsZ z<-~Xgv+*(zmcKnUcg?|4xwclF>j_1K)4~@#Jk;R>0a#{zt4J>164|>X%uPfg$N~Yq zrj5lCx9b1A8E_IC;dOa-Cf{d*pF;;EtV#ugPp7T7SM4*uaj0d$0P==Y)qD~v5pCg= zLfpiRIr#be8Fd*O@#6Yez$5goLG{6uK@hIl(@f5rQdAsr5e8uEC0OBWgl zx^wFSRMAx5F|#YpiI1Cjb3vHY$msaGOH1AM#+XrpoTh8uLGjlnbaD?ezot^V| zu)1so#wa+UH1cluZJ%5UcUM%2Dy+mJc@%gbzJ0?-?!&oXwWG_s@&mXK7|!#U%KM3P zH{Y&1?6{2NRDd`d@JHu-N*SVOQXvtm$a?q`pJ1-C3nIWPs2y?BFQu2Lt~FYfJl;S6 z3FDfbKZhdBZr0J+*q|2&kK9(3n*OD$5^w9)y)8rVU6_5huJ9YLHj17~7aFwFMiEOK zFf&PF-tBH`MMWvmOL(I-4Z-1amAs=w8DN>`o)tjq88mFHznG6A2!bKV0Z0NG6`+jc zz^!Ygt?ugLOJIiYSv>YkTHZVF+WhCujLKX*ssgrhrM}Ir$EZE5sMr@wCmj}jH)HWe z#%-2#f*YFm`Ff-^!zoPZ3LPS6hT5`LY)jepi!m$fwd*y-80#+f3c(~ZQ;yOi5`ipQ zu2TXsnvc%Jao*rNw`-VfA%AyapRnCO{CfxErKEBSSN)5#gS zZAdz*^C>bE+aSd2({7et*gwD*T6Gzh?c^jnO>0*gY1$`S*J{qsa*gQ5ZlL7+1+SZo z72bL(f?Mc1-q}{v0B(D#*Kk3?xGO{K*o~Z8*t>51Wk*~0U#6AsW4IXew7+)#nU`)a zRTI&vb?C8fO=mw%1`lMd2s{ z>4bES_eSlivk0Vv8PjM02AKRlRNYbq38~fnb>X~G43LZuyTpV)x_Ww#)YW-61iKT# z`Y!JAy`Z4WMuy`mtpS0hp@h0heVB#&E^77S&Jm+3G~J1Jq&-=>5wg(W{=v*2IX=FV znBYA$V|Gvb;2;&~5aM(l9Ey}*{L~`~@Pm2#{aeC^oAGzaJgSCt7Bl%_z8=VRU)BXi z-_Qsxf3U9*SU+jU4xYIJ4Uoqv?eM zl9TDnFzhPtg9Opul(75bWvj}rs-hFLQ{wPfeefWjgP6RBQ^UYuwe%t^CevHHdNO)4 z3JOS(;0gsMpV>X(%Xo+#Yj4z!m5WO`#93}e8)j`>S0XdtrlgPb5uHCDcbx`# zNg}^`HhL|6Hd5Bb$_+Pvy}5EcXEaY*>1}7n1o>iQzn`~$Xdk0uvV)=2_349F$W*qMMhpAt6N+8mQnfs#|Ws3JU8#yA*PIa$jvu2Od@~{Ti{K)lNuV-%e2DmE=TCd`n}v6OefnfkX+$iXx0zy%Hg+mr$~y?JAfG9y{1Ie3mp z_j#?Whk0h#y=eU@y?IO@Ckx#ce-!1ECaG5!G>lFpX3A5dXF3`_J5Hb{{`AA9Xva2I}T{{%|AiTy&l)8N6t`K0#+< z)ANzj`*7~U9on=LA`^( zwJ*)!mC-`IWqb z_fh=v!A4iDyv>VT9VcOf&q8{UIPV!|TO(^fqcFxJzg9Ji_>=77DJsW;SKsqSqtV{W z5A;Rcmx~DFMUs9CowT_ek8XQC+J6`wdQa_w|8zQ9uao<>#2#K-d}U$TY24niVKR1B zq*9O32QPp2CxiYN?rH1V>t~dq-;3}1x}K52VSW%u<9UuSt`w$JU4Nk{EQv#BZyMPx ze3;!}7|IJ=&vf3a*qEy`lIu%REfTQt_#K;S%K4puKZ`4cGdi-iI=GSB_F}oU&5N79 z`{VjM?-{6+NHRqf_b29rTkRI!VR&C(rgEy*d} zubASB>GJIU5OnpMz=-0KK;M|+uHfZEx6Mtpt&m%-I#YIhiYaMX_UfFMU>XvfIRF7O?Fodn-4S7_`ndVQsi`TrgfbpMPOF*z zJMLO9C?yY%E_TOXjHOAlC(Q<$4T2dk4XEoH@YmF##$r@3pI2z9;)O zJLvqYw70c#FEjbA`nmi z5UkxKsXpFnKiDg)i;_HymuxUP={K4xz#SEAW-y%)tygbfH1wFejJSD05O^8Ma|7`r z48>?*I*)wbJ$0vY`y&8&HwK~ppFVv;8a?>vvGMVNU6MF@Chv_?*lkH1B|I9q00`j7 zJ{UxuXak|Ht|gSoHw<|ja1jYPOrL?~28c!xFi83cEv`UP)LUmA)q;a96VDe9FvxdbG0$-%ltUvfxG(bzPA%N^_QI zfJTX#hljkrX4@v;uqFa`5QBs0H;%ElXZ=7xUcK2u2dTL?WAez!Uj>cZwt`A>Gt%hz zG>2L$y?w0;?5Pm=BWjBS6#z9tjFEks+&2dj5PPQZUq-Q~bNT;Py&{ zis3F(Q4x0fU_p}mhf~*T^QQ--^J$JE211UR6f`tUGY6ZNa&pwL5kJ}ugW4o`pT8_Mj#V&Cb*Jma zbU1E57sX=NFOV_eMEL|`)(oI~SyWSSW`<OrI%J*_1pT)?xhiEOzTE ztzC)a^I=!-!*$XDL98)>oC=A@7E@N5a1sT1!WH)DNGIEw8K#BwT#t>xl#1~X__`gQ zo9B%Bz~@nlI~*+E?iT3(^PAcl6Z-7oA?Ns86z!7vO2KmYYW&pJgqJqkOc^?LQ!|E@ z=rly&Pyv7IL(H$^!&g7sJjiQn-<$t(Pb(-OfFgFg*}L+xT))3p|tksJ0)-OTyEOzR+c~Xc?9I;}xzt=*J-?S?be(2_yXM zrn6RM3>|JFkF+NuLg-fA$BJli5Aox!$fq9zX99T<>Yr3n_`j-!4$gW@L4~9t66f?+ zH1@~}UTqC1@fcajD!N$I#B_uiH`}KRS(nb|G_n&6^eum+Jc#Q4>qB<;?xjZ>vXe}fegmIF#z#Wuf2qEn zN8f9e6}`?Hf0_qF7cSc=eCeHC*N@%fD*N4YWM?L~aW3cY8BNIaezf>|!gr@^)V`|l zlwjw8wEa27WXV$(^_Xr%j*$lJ3aaB4WFHyzM33VZ`1Veko+u|XM*Mxja_1g(`gY%k zg2C{CrQ5I5zDW0_5E6bmmL<_Sn%_`mk1GDNwrD0lN{Hkr+$yGV?6u z(+ED9xr~5{rjVlufiV%Im{b8eWt>mkMe#|yZWiAOvHS5Ua1kK^2hgq2Zb3y>TfP!2f2C4V-z|9p)hx9N?oOHlG&zNrO)oD$?#&~S3X z&INWl&y?j^uZxM%i;I)qzGPu-J$t4`E+bJU-A^kwxMOef%v5L{`^TOPoRu3Jd%Uuu zq7)2bVI+KW0J0SPAwc7E|B`|?2f283`<#qvhOM(2*|2jZOHHn1g z-o2~QqREyG51hHkLWG*LHC9@G;cqhqZ4%1l%^x>~5i zbtkFAwk&ixFE0;YAb9tTG}(^@hV!4)&t>#*R?rZKNL$f%Y$yjP%Bi(jmY6^eN1PMK znqIB9FsP?Qz7Ez(kVuoUEqLMAgLt4~g83?K0DO++t4v0Kv>!}|clQwXdEf#dJ$Prb zHQVOCRwYPuz`ubkI zawu3-g|uB5XA_Q@pBeMA`}OrYhF~#02wAN-OV z?D5G7q5kvl$O4s+$(#9I*J(V^F>kcc0j8)+Ez1l!SpNM=HRZDe{Lde1Z(Z{2vt$n073 z6x_@yre#j%G!nS#(+&>UJG4yt=)2~sXC{2yR*SLkTVCFa{5Uz2A!TUDFJLng`mFuq zs}zQq?uYoabhm`uH?DLh=TIhSbNYcF;IfP?VcPQ3k?&+Mm$(SrQZm;n%90ZCp+aNs zxVY1R1mcQ|51d>t@b4H19nhjvdZ6d9X~O!eeN=v0Axq(4UNC!Rq(tkuy>4td-0jHu zR+buy=>nB%eeb89M4=v1`3&0eaaTS8T}W-ZaQ^&K_mIUYJT?wO54OPqR1IyJt;_Um zo7c&L?|v19_rVnZS3`!#moF8Af@wZz-Rv2Dq2S^|<=8ba^ZXIz%7E8$jt4h8{LvW; z63gl>wf2oQH`1n&XM}jkXbt~3R=3z{AzDiH0Ar%=M&o5!F9Xsw!xt^%({N4}9->Ic z6u7V43s>fWhufYirJl(nLbXx`=!U+t+BT!GD|e&^DP@` zr)PtHG5cRbs^Enq6&cK}*3h;JkoeUS@q@zwAH{an`k>jToR_4mR0Q*x>LTBM@sjLY zTF;*Oi!x+XpZLCV#(OuZ{cmvG6U<#AB0-tMhmzfo(GObxJmXQrSdpI% zqS@?4b2slvVvZ)9&es>AFV7sWIc9Rb3){HA@9D6od(`b%cH!~8sLwa8D2p$e9hNH1+_#%=d-wBv@zK=|TZg)ge6NI9FHN0er#}51*2{lwOFn)nKL?jarrrjK^DV5Uc>qH<-^%f$3#b9Yd0Y8i82Wi3`RzTKCWde3Dy(CJX~##u;m zra0a!yB5>oY$PwwCV_Q7iA`Wq0hm=i(@D>x<4q=1)Y%K{;I<})_r9?|_@SUH_{j6F zYRZ@9B2%|1M{%9(qXW*(VRJfi%DX$;>cALg-&?KnrxP6Eu^Mo-9^s?8KqcSsbfY0p zkKtgY*m11(9EPp2D*}$~`<7cO%Wno1f?u)K2pouw1-XA|koZ5IBD8N}GElnd{Q|+o$-o^&UEv z%)M$~#Q*L-V}6tDlK4(!1Xrr=-gQJ<^F z4t~xllgoQAOKQokXEW>(b7KqcZIBGi{J?1cNI7zOmIqIq&yHYRkbF)(0A(M6@b2h{ z5LJL!$Ri_VJw1;KCDPzJf?jrGt!iEAy=SVJ2Ss{%dLm@~7D}Mqi-B(gVZH`iRUK_@ z4Q{pD;C7bUAe4MLgjKdOg+z0e7A}o)*LgF2VG><)sN{WiHd; zxww`P=to&aL}>Z>NmXx)L65Djp)obn)O0)}4T&9q_PZ915%j+Lu}2}h@BUt&_%S~; z(KR(SaYd@x+XqfJVDkU``E($u1euk%xOlny=2e7wyt%+Z+YW8)80{e3Yd>Fyh=JLU z@b2^9^@rP}o#Tj$56tKu{(poX8yeoG$2EZVHxZodHB0!#M_oDym z5|~X_xvmC`+a~-O%+m`6PVYN)@~DeIGLCbQlJZl&Z}HVpBn25ELHO9U-THbdWCjnb zV%}@4b{P*jKlEFs_gj)vQy=_%1~IcnAgGq5H#ak51{Tki$g|IjX`UB9+1fqXR0QQC z6br6aj-u2hx+GxLhstH*D26G2+gtLZ`Cw0ve~cuS`qr)9<&if^*C47Z3~D%!fL#r3 z?a=x8=l47GzQF|Nt~6O}LIOt)NHmSX;{yW+JL4i9dx!!na}xLSYJI0wU>N1GJ-=B^ zdi5RHn#z4#yE;2VfHR4Fy%t>LXHF4bq?a-LXBS-8~69gqAxoQB|6yFp2YXS17AX3R=GWZJBNiST0 zl%(oY8Jn9OL&(7L-!J*x;bPVeICnOda*aM#SF3#8-x2EWn;~m$Xz=tt-W2(&PZK#| zn60zF&s0tLC*UmQ!-pF;kn$lS!WB=-#YGSb?_aUsax??e2zQr_3gn;9XPd+y)=E&> zPOQXonevjLNI&-|=bVu^G{e`*J0c#;FS#mNM{sz^AIOzk&+YpvJi$oSV`av65e93-ZdEB&iyQiSb z9FE$_+=9|c3>v4Tuf4M;;;=Dk1LrEH2+eKYm2cQiLp@mI{ON@WLgp++}wND<2d@10|`1bi+eT{G8 zf}3{vt!!us=P`?l9TBZPxN5P?ycXr;E4ms}O87CO1VT zNO+v;XOCaIo6AoWzKN=2GI_1|gg#2px0V9Jj9)nOOA6Wb4cLU{VH)&FvJ_bjPZE^R zsJ+HYIM)X5Yu4Ct7Vv*k^4~@Aklr2qL0YNomiw63S0ASS_N=uy_`x1z6g<-NENIPa zn*V{^!KrThMHNi!<@&ZI(FEIg4GzSnivO-LW2?7@62c7(f(2RP1V%(^*vw8}X= zqr-?`XJ*Zs>SfD>)Y6ODa|b&b`^&a|UTMa4$6NQH+7$0`A|vnlTmMcOc@|)RFXk+@ zdLJe*4QVUh7ZWZ{G``7j>vqq@f7@vYWeripR@3*(Jc{T$G`0RBL;-kKgMs)QB6SUF zfYL2IL+QIS%zlZn-2blVgPw!L6jS4K>}=c-G9#5|loc7h%`2|=QlY+W({YN~Q_f{u zcH0*-4CEUMS$9g;5#rezSd-Ohar!+*tI*k3?Ci!f6G_1<;?JA1#(oasa(cYKC;UiR z8L^((Ah^*+yG3*QyW$fOa_zdxvL>xhS|$T6ZE*L&NFaE+$<2OSf83N*=gd6}i>VXu z6`Q2ESmH1Q0z$v+<;7~kwz&5Xesd< zoqs-Vx)PoloJ3LivUvU;ejWDAuG4Q|;~-E~O%@a<%Q5Dr^Y4YccbBhfCg%k^#p0j6 zQ9`j<%z6iPH)&%h`_c8unVCTFS521Ho#bWq%!3m=9FG5>;9!zvCjzi|Lq_qz-;5Y$ zB2sMh-a0x4PHwn7NSC*xIKZF@i)ICgMA3mnUO6yx&UdEVH08az9uLM@a0z@YE!Fy( zBII!TUwSWkl4I-N65vdfeBk|f_WSJpx45|#U{?P!=Vv^ei68_o^1-+k4fOX1Lp~y? zbbcnjlvjccf!SDh_nn_{a(udfoqJ8$N!Puuee7v^UVOhrclD=!ECUr)D5MucF*Iq? z17LY$kjr*kF#5cFA4V`ZG{i}RIXZHr=;z_Qb&F)7OR_w%02nJEaSDS$*=cRsB-l2O zNf>c(Y*&-hih|ws(UT|FqjSG}nZ0l~;<2`N6QMJl1!dfKa+t^-!R;>7F6JSk! z|Mm*deKvKS$niaA5O(~rdRG@XJD?t$JtxN(v4ehoo{0L$OkFN!#<2pNh=ya;*Ha36 zV4+@6hZUhy?$cOYS#j2L`;?u{DkgUP3JqafPfK)-1(L>XuS7Q6jNx?hvtgg^?CM(S z6GHbAMixC5m+S}vjq3~6W9E|S6Mxe6_xDf1$iu-ZE~m!=QGG%t!~HC?wTOraHq-dF zAYl7?G?Kvt|9`NAnWfH)>IVn*oX2(=nRW-hLfrO+PJQN2*U6KGNWGmyk^xiyAKK~Hb=L%T|f$YtckVqx)@D2q-QRB%JJKB8Z>UYOQH1O!qG+@U6h zS^%kBkzcgJ#pVhh$`44OuXOR=dm@}~U?pvpOVZFF@}re-GMI`2`tCk5T1!7>A`3f| zeO^G@&R^k$Eah`8{keA2uqBQ#!fsK(pYoaA(#R%Y6%}O!zt!azF9Mm+;4)Hh*c9Bv zVzeFpcH8Jz^ael!@O1!o!&9yA!iZQqQ@k$YW>CrXgiQLfYB5bV?{;?X9HVhn6p>f% zksLZu-HHI%F&eC#kAu+1S=4T8#J)b^>N!%Lp40;cYQX1hfhp2cy-Yz z>ngve3x8)d9sAAsW2+;{J)MAak|VqHoxZvu#c+Q0JY{r#q7g(6*9WUZLxz!+!;7hd z_7m>OmF#CiWmWX6C2J?xH4VukxKFY>EVVuvAh@!0_MeZ)4t+a%P z8&8J)#CNa8pDtSKnm2K5IrDw6B25;x_u;7!>7#A^);9f zS$$6-tRwouPu%2<+iST}YNL|{Ik7sZ%)F$XpNz@y-Y1Uhav$zM%T=TOa{iY;Mr{fI zJ2DC7MZZ6n*K^m0tVX3bfb)2GO{KhiogkkfR|cA2ovKx+;aP=7kXw=Umgp(n3|fPZ zD{cE1|7f_#INlLBc4z#h*8c$-SJ(E*(MxckU60Ht+NydHrhMMhtaFr#@i!AWKR>lb zLaW%G`8Fy4B!($p0j_I@ttKmreEMUbenG|17dK5`m8zyLw}oWXV-fpZr8ND}FyNWH#(7aJ-a*r50XY^Lw(aVdOoLv6W zrRo3}mTw<);kaa)@YbDZqt4R_uuU4_Txh0eRAfEJ{#r;@ADCQ?W-5Reu}l54OXr_X zR4X1&625Rh*?pn*8Y5vhwnmd1081b2eqU^wQ54u&${BmwsqolUc&Ct8#=5sTyK)xC z^0qg+_YemY$X4sCm5^mmc_=wXGCxjwbGI+y&!S3Jl z^p4@nQp=mtp1#qMV;;i@*ov~1aH znc@8I&=hurWxc}Hw>1q6bpDTbGI3(l9Ed@{F#mGDu5P$~eZR|8x5BKM(NgsfS_ltl z^PW7Rb_PJT+w8C2Q@Elil<`+sCqlA>9n;C^G@8F3_R@&^e*Pt;`Od9aMqvkc!? z^5w6<9=EN*a{*caSe0;1OEnHmOjW)7b3no~S)}{jEg7gpnM$+v-Fcm=LRNtf}VaS^6UnjAxd!Rb5)lhJ*@aCo&Pt z8;uYT=Qbh%!w+GoQ*WJv9i`&SmoI-`S0~yr8KWVPek^-2GT2(}ae#OLof{bi<>$** zp4~sgk31$Qt3mDaM%$T_{ccIY@?l)qyNc$F+?M%6NPV9Fu%uhN_Kd3LY=M;gS?l#| zJFan#O*S>UCf&Zx9T;fwBzJOOk~R-uZMATXXn8q?J9pxYU8M?sS*((u>foe1+5Fg& zOJVl0o3U2#hVtN}Nzb3JzjdpCO5J4&u396OZR~B0C#dG-E7oH(uj1*{4br!kD=JF! z2`o1KJbEsBx7}VRC+3ucEBy3i#(oqY&sY_= zN7w)Uec>lh^gCUakNnW(mHxyOI4k`m#>L1}BOiV2w>JjE6B*e*6*DF;Y%QWvOEz6D znNnyld)hjN|v^CrWSrdZW~K#?{?YlAjDVK)LiA~i|2QB->!NQCoJ}P2q>+l zi^Q>$x#Nm=7{+}#FPAWEvR2>a;uD;(u&Cdm(>}_3f&XAz>mK{r_9LoBYi3G*wOGH< zcFtM!`HUVUI)?_ln3=8je&k~<5{xlFu|&SL$x+J9Y1&-&xx*dqARwJ zy44-u0b}3$rrv${-lDtBm{~-oo+m&{xVX?0+76b`qqX<)f6z6a#2rjE-WIW3L)GW; zB{6yvWrp+!`z3EPRxr}hVXQsk8zFIbSR23R?%gXzcKiG4)V@hn7CS0$*(W*q>u##g zcC*{7!oMn;)*1fVIi@4DmKwuyEt=zwlgjA2TbYYTvx`_1syqe8>p4GYoei*H1fp`RLx``?^fC(4@1cAuR;Rq)J5PC?HfR8gX(9?A|NI8&% zG=906S=KQ-sB2?lV$dkeZfbmMv71C+?!V4fi9v-#R=_L~r08&8PN zm}9V`h)Iw84hHqyyDteqccd+X??U3}T5-mXPq&GNqbzI^f0|h8rnHY7;pZqXEA#02 z5u~1c5m}ypX8a{gGKxFTH{0H5-Og5Z-Z-|}qSDMkEeHC*+-9+EdL^R z!|m9Dfb0Q`%s&X(?xrpF6V2xE^ zSJWqZSSRZHny@_-8@XT193+CzxV%0VoxXJ;~3+WT7^vF@DCU`!45i% zbiAg2sgj{bvdlsTKwPY>v>KgZW@N%z=lts4{(h9jM<&I5!LmOA zokYv-dCwbsFdqHnF3^8p-(C%oxpvto6KQK%3){{;Nm(*j-6j)!#_i&=&Sfbk-fz*& zTrR0NH$zSRE%wbzJo>qp?S=#>;ixw`FWyVrT>LZojwAijz;5vmTHiY3?-#z7X1FZ( zT;aY>GKFHgiDz}fU_ttC>)l*WJ)L_?Qm)9IZLhe!MS)?zd+*PBhsmaio}ZO&V(0Am zzIxm<&QMP-fiMHC>eTN$K%eiX_-z(%_|Y`tIpA8OZ!TuN%XgLm+BK7s_}%ygk3Z!L z{`dp6VX`CU2YD5Gj)=Xf-Tmwqn2(qZ$z4#ruH-B*D5GelBo!_>BR(mRe6U(`uYSH- z+g`fmu5y+TC=L3Tu(T>FQzk8^GyI#KZ$4 zpX%FKTE1#4@XW%*t9kKv1aen)!EuF^vj@}Hvay*01E2|O=KPrusY6~Sr zMS3jckWbu0ExT4$rd%n-bCF#m@CXo*5*+YFQB{lR`UVDyF>3F!Y1|$YUaOjAkZ~_P zJ>o-HxclY}fjbbXh)NaaT>-!Xbt6Kgq0b2mfhzSMU`e^$i4-wPx6Cn3PpAnwA-I!mwn_wR=Vn3*k!=R0>7$uH3wDVsA?7Ac z%ROR|rQcdM)6^8dY3TwF50Cft^}FwWIU=++Ksl+$V%NIGe?D1??aSbnYdTQ>5jQcG zOn?1Z$Uuj%2eRg#n(#nm4R#BMIl>LMVBwxhS(e%)7!2^-sqNUZWocSk8nF3i_%}cg zc;T5?l$SZqZ@C9vOwENLXabFF^e=g)x%dQ61vVXzQI zj{fCoEU_Zu$v-n+Y?PIuQSC)kgp?}+*&W~bAB^5bk6pcc_w6f+PH2HKZ*_v#*wq%F zsJuKW+m@${oPo;!BOz$3#&yd6a?6_X12-tIEqi9gc*1HdzYVBT#-I=HsAHEbb*wA8 z$KpA39OG!x%io@6&hH-1dT#)GK!q5P_Z%IW%M$&qj*JcMJ@a7D&!BhV%ab#_QO@H( zJF{Qzx%%_@64uFTPe&K=jAv)F=;>~cYoBa5SJ%V1@HW(MM}yi}$~j`g%m0)bP})@M z?d87z2juPe?)pL>8`^fVtX+Er{z4OPALwqYV31!R9 zo$NNHrKK>=-3Umt_u{cTZfr%JrY3%EhApz6vsXA%0 ze^}q^_qs?EZznc1mkt&}>T8P?THEH{J%9e3ulLu2z1XB9Yl7=A6gw;Lq@}6p5Rp!V zda}k<)m0E66Om9)T*WjR5|WPvE8>smTX}Y#1jp6)ezV#^g!IGx;T5==rFOTo6xYXR z&Vn=$&~xff`!tBY&|9|xA+HUh$YO}$|28n-h!h<}DLKDCRa8_Ig5=hoo>7seL98Ue zP_(MDHJol2BDP80IP{?5e}KHO@>?<+cCGuclu1(FX-{khQ|8O~OK;vR>233!e32F( zq(T{2b(%cO$Kx*6HnewdG`+NOHE&e5>qJ+00+)YA`R2y21-qQ&j@ZwgQ~03f%!}mX z9;)21>bLf$Q1U3LYRWq;p%LC5(XUn7DoyWaN|;Ty$18vP!~oUj_2`&8@2BM~cbNcv z_~WtzHm!>(EgMYbB*a$&UaAV2>aT8-;>ecNydkrg?l!OynOUKY4+}ln7py;MU~bF( z>=G+R`@I95w{LI4TfhQUG@A|9H-POd0(h|dJ3ITg2@g^rh6|^KZ6#v$72L=8&{^ON zy*2SJ97V1!9a-9EWE8Le0?8Fbz09%O0?>BFG-+nZ_629>Z2jrq1DQlXjm1K=9<}g zQS`~tlJ@ZdSE*(G@ZDB^w9ZM)%{_Xfro}1Gq4Q$TA3i$$XHNXVbIi5ThoLJ+F32S{Dg#4Hz%IAO2RQg*uu#`O{>p~e^Gg9aQdwP>Stj$JP$a6fu) zSh4|rt5&@*6rRTbDhE416&p|f{OK_zn1=;~AdVM8`=?}Jlv80v?yMPF z&H^K*WSzEw%E}iXZ6&;Z_ovzoFw*HiciD;?dMKww4zhqSMsCO&vjHCPxD!i?RR*KrO*Hgm8r;@w$7SllX5#fYk*QN`&P zBH$udIdS5%S(G?uAbBmo59GTVW6lW72PNI}<%`2d_|G?OB2O01B0a zqJ9_kZzz03-R|Y}G5#_+^yH%~!MT@nr1AK(k56|jLIVanv1+u2^yusjwiQlJLmeCE zMr9uw+pirUg02HQlPg$asH%MDj8%BTRIFxC5IIo)fS}3mYwG}!U1%J+ee~q%aE|0( z4Ha|6YhA=-rD~kP9F?T_1?s;$H@nTIB>sS~ArmWp=upW{j`Q#-z@ZPf@_cr~PG9IuqRkJLngUmE*V)(}1G_H;8oQzS%h4~xlP7Xd!%C21S zdZL5<#n&a-d`m>AmOU$pZ$9%gJ}t!TU%8$0?X%DC%UZ~Xe7TsSX8KPvybkGHZX zfKcy;4lyStCHc?nDZg}O1$!5S4q;bKf)POk!a^d6$AhY#yiZS$?WujcaAUgOHUy}I5$KH$sDTl=x z4<9+AR)+n{vfR7rAPhA&Zr{9dcyM6gQmN^MW?R=6jE9E@27ZiaIRsZ*TtbIyV>8gr zzWC&>b=de^+LBZzeao&@>^fI49&M5!a^PL?Vmc^u@ld->Ok-AsEak=rgv7q7k)}|B z9lyL;n|sC&ri{Y&U{>78pp43~?|E^)FFJVr7p^-#Sijq&)U!kVzyZgJZ%5R2Ae<2r zmyoUG@!^Ml13@;Qc%!t1A}cB?pjsL==5$cg_KIlPWCL*5CPO>cnY288M; zuA+487zh3qztK?#V}w0OAouE_4?8*1w2zr%8QxPf!U$1A1rTB9&b+JldBbkxKI*=BF;Ug}?rkV*_?Q;abknkR4(19_VF+iExBYB5dnb#8%xy1_nZn zBn<{gFPKQfGIP8VVr~4mZUj288nztrsZ$yhUI=g(pz{vbczg~+2q;=q*Ei7tvT1Xt znNsVoBEY~$U^8d(Z!MosEixlnf;?kJ60rwK1%E6~IR-s9u0J8oxPE{t-#%=Gow+j1 z9}60%q6Iq&XAYVLt0xx)aO`%#*Ur*IJ~H~xY>sd1NJPC9SKW$x;bCres{;?BM3bD3 zg9iikAEc+3U|KWR_%eeNQrpnLpH`iqU`)oM>joKqi05gxWg}|vr7Kp1i!@YB!Asiu zlf~lnY3G&*Y8e-PXo3-+xfK-|Svn&^(YlkJtt>-9}?o>8FNJ|N~~YP#)Yzdjyur#gI+j^ z2XJJR!}yF80kIKCoHt{aqLmPYP0%}Eu&}Y-9Xh?Sp2fgp`qLd!O)@ZTKztViATCCw*3(EBEQs#N^~s-h|4E7u%0NT?sJHf74RB zA#uh%-oM50HF%vx$JU@NK|r^fa}V$ViEq4|$u82j%x$w7y+oFd2!Q{fdUS6{CzO(LAzkXGQ1W~#9 zB_)qK$+9EC>hY8ZKTqOmo}~*w+#rT2R55DVs#RsErlHZ%QWq;BNQ#ZAw!pi+oPDR! zn%>I;S{SWr6ZfMfZp({HdVKdT1K>ty86Qc^l*GMxNT_*N@ZTq=ESKF_qlhRL-rs4F z@_f2#?OJ3J4=8E{FBFPg0Gg$wWXcQ-jcr@pkWI^oyjI1(C3i(gglqvhQR3=Z}72T0#4vVtWHV^l0=|FYYT!G zs4wam`{U(uX?emLsMUiLEQ8~#6%0Lehn7-41nbcE9SM%l-zFusZoP!`B?Yh+A;Q8Ux8s(kAV-U>L`)cS zZT+{o=Yw<4xvfwHlwelP%WUs=A>$09g$vT3QI!|s79iSCup#?W#KgjY%rJ9v2jO|l z0>W__+9D3!P(d6STJ8lu|9)IAyvSW+OVvtXE617?Y@FrZlX`kP!QqJ}^y$gh3u5Y+ z1GVVs>F;G^tSz634GR*#xM`E1po&htr5I2s(o9liQ`6H|BFinz0PQTP(?4Ebjn|of zln$)oEWn9<1WA;BhMt-?CL$7r+*Bi$4oHsAEKs?)TO{c`j7*xTmo!tA92`c>qRu@t zgs=Ip&u4ZbRN6QNH3~XhXqC9?#&B~&)sT;Ewv|(zH(#GTubwz1~Cwk&A127huHh{ZM*CC7#82__Dr9!O-4W|8d$>`cT ze5CLWI@q@tq5IH2bcpW7XZa3M{HWkC^bvpYy~+E{7c}^Yl{$YM6=4HT<9VG*3wLs! zX&-dE&KN8O*BC0Z*!>mMh21lg*vL4+~|v z|FFoP9(sW_V`~R6D+5Ga2b^Wk_nJM8S1w=P#LK%7iCb%JB<))F!T}!^7G|!Y+~ZBQ z=U|^s<=qnGr0&OyBE}gwf8z7;w5Y19?=vyEqrVO97LvVAY3%1cg|M+ar!UJHI@==Y z;J}tT`)$qRYGgoU1mmf?`As;Px;WvrcA z!_2&igDJ;x%JK->H!L_%JA?OlA4r=Lfly}4l7^5%F1{{>@aP4#)mALvCEr*e8CG(>nTq`$z9;I5~9E&a1KjkXIXI{ zvm3e5qYovczg$p*btxVLesS!q#;D^^aF9aLDz%1+T45tv@%1h-kfD=M&ceb%-e|N= z`H=u+fXg|8V)$tLls*bhwrI!-*mfGNOsbVCT2@ewoE7NPHs}6EJP;XPeS-eg#{rdF z_z_22RbFZEpSrH&)w60ZInW;xw5t~$qfneijxc$LHK9V|y}S*KF#QD107=Us?uOIQ zw0wk7kVsi7_a~CfeM`$N(52H3F`Ceo1P-yVMdXY!fFLigyeev+Jo-~m6dJE+Oq=!qeQ(7HdzcVou5$%%k77;->I(zhPZ?dk391{@H+ha>bH*q@Mq znVA`tgW)nBuEg_zZ5f?TpJfnPpUFg4U=gfZQ#l^03SkZ`)aH?r(kEqG3A_m~XQc6) zW&q>|42s8V(%RmLd(JU%j>9rG5z(@1!%jTO=KS1&Ne^ zb9=x9plp8r2F$PgrnpVncSmC1{pTZbluG!Jo>9JkXc6V2zL-e#mjJgIhgA7_!pg(b zeH*OB$--g;Va(w-VIIIORdRV5t8CqS(wC5=Cy=pxG5XWUD_^>DWeE@z0t$)#>(QAp zMc2uQOP07*P;K+IF!HZny_&H4S*~B-a@io!S1abkvBd_doujin?O4M-+ z65IegN3%l7n~vKLLObP{@KC{V`}e0MVr%Q`7aAKI<5bkvLw9cnnZe>5gB?n!$(%E> z{np?e!el#!hl9bdb+kTnHqF4|Ty*e1E3!Bfvl6qiv_&nQ_?h^7JL{A?n zyXQZbK)(s@oD@C^?>1B^9NDfs-`$qad+}w$EZlHGW?NK)^r)RO7meZT#{L=iivj2V z{PD`Q;5c^d*rBw;BuHZ{09s=>kHo`1A^4HlQV0>kIFlO=a)34!Sc>7Ihj(QOcH;0& zkT{L8H`{C%YjK0Kc;DK(Jq@o(C99vZIG%ohP7DkTvaZL#YoN+X#Qw)?plg4oT&oKc zaDwv|bOL`Fz>L5f6K?kQJ4rZJWzMn59oXiH%oDC7jG`}|KPNove-myA3Pfug)9&57 zRT_RL_@w~E!Co=P7L}!~G6^2tWr5Lf46dxCTHZrj=m6|UZ3j|8=2=3>j?ow`2dQIM zzJ=RDjB~VL*gLo!TBfC#BtSBfb=F6ZHl43RitpRFJ*=c1JbAKXso=Og8o4s?v0wCv zkr@^3wYNR3H<o%J<>lv_tAzQ2{a` zkjhmc|dk8SLfd6%rjy1^;t32Yo3y zsjzwz7TGY*K`|ZXGn3J-(m0CM%H=CpUIrF|r^g1lHze4TC{wIJQRN!oeLEL{pk-lee5qwe!3BkDO6u+8N+8&VOx{8S@YU{E}esS@&IIdmDz2^i% zv2@w8P?5Eaxd1GlJbkKe4PhJU8;;?=k@EbRF7+@iP0`pmLFL^SjtpCg^*|c7r&XgS zys8fjg79$1pkBlA#Mk(fEjVQEgFtKw25e3OHx({LcNYzL6-?F7I zt7^qXIy|T-xNT6W+%~&jDoQxHud&!n0(%*E-5CBRdJOOl%1O0@1OCw5E8g%H#+`=* zNUheOdg94DecB9bp?Kr_k0AiyKp7rL!W?X61amMjIEY;ORK6SlxSJ&;^in82Tk$JN zx%4rEoo(^P%R;oHd;IA`qp}S$!hRW^_iWmS@VX1ryjQ7~6+llN$NRJ^&yFmtwa#(- zQ7s9MP-P3!)!ca%nH_+Haw+(5VH?&ZVM%Oufzu38crFa}?wNT0@qe{u6cS7-Ue1 zJvKEpg44<-fQn z#+UP(!L)RAX(<@jo4#xDb5;{Hq8sJZ{o?z35JnJl4G<;OZ)gx93U5LJt)Q+bCljsp z#jUe1x$A6k&k1~QpnUj5z2Sb5U-ji;gY^{x%cE<^9aX)>-s2$kzP7gWX~wQ~8G0*- zIdbLV;@f7R9g@eI1+>msTNjbGgi!0BfnMSbQFd|~+bvhf#h|zs36FPS8${-XHLcBg z36}A-b&k3<=zToRT92A-iRg|7+LK|ASauNge!A`HCZ3uqM099r1>e098Mz367Wo3T zmR&%4(c#9)dmH4>y6*yoM^ZiN3iqId#o{|6AQ|n&`wt()`kavHPzs=o7)!>Bnq&fx-oPRSIMbDySRkR0c)0DQHejd{U#8Iy83!Y z(zv?foz0eh3Zy(NsmId4eEr(x**?uJSS5hrmL)1WdTFxGRtZr6co0Ld zf#eW!7Uf)q$S=~_BRXqaBOq9UW@wg4F2?Yc1cwwCVQ-`TsPR`Y$D>A<^ETi?^;4N^ zN=IY7_38^kZ^37S{0`H>snKpQfe%o>^L6oBn=2@g85SD7obHW^gur?Vk;FNTc-hdYmf{;y(I8`iwu+4;q!8JM1^3Sjta951R7y}Py?-PR^ z_!WAb&%yPJlYL43p{g=C-ozq>(L(O+Fh_SMFe1v ztGW8Vqzkm@G)J42nf3ixTbb#2JVF-2aCOnIK3h3g&>95Z0OjtG8|S2NA(jN6z%9C( zE;Vq=r&fdm5JX|P{`Be7%{wxi21*lJXl(AhzPq4DAKdvlLXZ-!c`O=rnqYre59>E^ zdi_>(I~dI)KhTU=N))l>Ku|D5DeC~w<<1#i*z*vV78(mh6&3#$Z!8)hX`oYc2gTH} zWj5*VU5d^D4GjvB90H`Vd;$uW=rA#1d=9c=v>5!96D?*H<{wVD(A5m|x%TbKp!dyJ zU73Y@EDN6TcUPvEQKd-%m*t(zfX;gb+xC#nYe}K~=Ob{rM1E&n8!Zk@zG~@G+wX$wlC7Lc@`M1- zaMPUlgZh4R^)77EJc)Q~Pzsmzx9!*gNwQq*-0F#X!i7F)um&*u21gblKz_Iw99yB( z!d>E!I4zP0keIZp2I}h909W^rL)skt;pNtGH$4M`Z;cG6@MTenc<7LM7veRNBS11hDkn+XXUNtMvWDX&58P&#`y=dJ{9N1T~?byTf! z&y$l=L@tH>-B%i{&&qJK;-%19$HS3^`D}W+xV_z1wMvF-6&TjaYSXNh(yr3CxM!CQ zL8%9@4QuJ#zduG{n@1(kY9j5rQ(taZY5Xg3Tc8ju zOJQc{LHz<Yozh426~mcV-qzlZ2#~=Ef)TY(OxGFr;8ps(w{XO*vp+ER7PJ2O5GQ^dr^C z(NJ5PKDB;X#@?n3dWZXaq4E zUZRjc2f@Oa)ZJARO-bye6<8ZFPKj5^j)KC%tAO>emcx?S>Kow6nw{_uw_@=_pC<;r zg4&PYkZJxZTA2-;I&VhSrZL@QwyWYlKJfPQ=br}UF=(SPH^p!%%Ez0S8W|fq@N@^+=!z$x zDgrN9{;g8e*TgylBv5fuZqee!i3J5Gbgl;ntGK#KOs2w{x*LX2a@pexEj}TDy@%*G zkjVUXnxX(F6DVyQ1(z50Wd4KjZ)MtlCzWb>H!G8urMAMe@gHC}{*@g~joouLV)>#4 zIYLi9#yP0~%ag>PwSWgd&9re>eo^@1&@*>OqbG9-U;whP*+I_wjEdLu5 z1)gIbQ%Vlw!7p?CZE#46nDHGW7- zLv`dzYib5b0h+}dCfwECO>kW+(033=5!S2p6Bymmx&u3L zj3$&^M7#)~G_lJ8jlln3D(NTZeC$|E%N}Du0LW5DjmOfIRDi!UTN^loFDod;T@WuI zI;-P+uObobUtu?Lkol=ZX#+RPML^C^pn^hg+z`-$j&RHx6D15LdwqNgTh2;M0{Xzd zrFCc>!#v3Gux;)NosmH8yM_12@z0ba;$ghspTbJBM_#|2G_ZV9oj>;Z1bo~hrR>JH z@7+_?Il5&Hf#uxdF8+6HExAlkU=u?nx&Fbv{MMg#gt!9Pc56xqc*`rKDkIB2nE6lp zYGuV@Lw+i#q;d45x7XAf|GzPXuO%%?cXiTmZ3H4yc?mzl{3&p|D@_J5g(uwX3h z)7M{j^XAPGgCj@QVnQ?5;D3n%St*6z;GY5fbJH9b8N1vsuFOtS35Ez!L#CmQ4B_7U9 zLc4VXMJ1e~_g^t0J>r6bPqck;dQGDt4%p4uOqPRv;T_SXgH9C6lfJ&bb#-HK2F^DK z^XC2bagUjhoAE#GXJa7T!AI(p6=v^3BF|CqHPVR%Mjz64y+UT<=T9EAaX5!|laLmZ z6;42BEO;FgwSjIB@HgzH-h)!?EO9psr(#Zjes%@H?46x|oS%%HHyM)b-t5!CJ)~CS za#PG~3Z@+9;R!qD=?QP%4{aTt1&Yjz4a?fwHxf(?2?mGncOnNR-4U-6@&SnTGYv0a z{z%aZ!qRF~{&$k8s={Op6MO9U0CgB2wGLyUGIX>*?afn)q*|IU>i0Wc@BrZiNG$v_ zbna!iSR>j{IuP{;{E^yTQ{Rr>9Ii+NyT7|Cp4}XP*1jtvx9y^G(rCqxAaP*yfc=C> zPumDWM#S@{t;A`*Z>Q1D603toVk|=KzaEmPoAh+qc=cXY7ZG)2o@Va%1YBO0rQ0jBff!1 z{DxbAk3-1`up&$gb|P|s9dUq3#|l3!#CiuBf6-xU!fUfX6rBCbYK`wuZ2%D1V5MX#g9zQ|RDoKQ=TsZ{@)IL-Y#~e)&)+@k8GBrSan#yhLOZ znns$XEY$FpfImLC#f4dpLoNsR#Ffm<;gB1{eSNQRmGDg`4HMoSvCVHSvfVtVp6hDi z@8IshD7B$(j0S~i=TB^0aq&EP9t#C`?5JK?3B`yXGB?y3f8mM1=?NEMk-H9p5N)W6 z)4^hIG&DO4r+A>%Jf*5ksriOG0g8^$4~-8`toR2d2$h7$q!_O*xElG68Fb|2)y5Wf zzyg;i+kK6lUP5kRSx`t|EiX#l!CfY`7y6dh6i&544ivYT72(I&K+ zqG^$G{u*%i7}6xUCx5Wg0|s%8ZIzg}RO~f-V>HL_%VhxQBQBt;L8=K+w!l2jCqb7#-K{^F#VcK8iz z9|v~*L0Y3~?NPu@kRz}VoIdEWifD)^+6u4k_3a%#sys}njq^0#;M3KNFCk(hYnwDv zb#;v4#RFZ=rz?cIi^#BPV<3JQ?}55e;v4bPl9VqJFZLYbR-)ZOd)hGQ`Q2599NujG z?F)Up=OpwO)_2071ojF%hHTI99`Kdm6QNW!q3KnZ0(UtSL~3UcbBe{=;ZWSNxZi4PGXfj7 z8f=qY`Q{-M@x^)>1_TYm5*HK923`3_CULov>46u5Y^pg{jSwXyAp6&Y-W~E_j{*0f zGA@Jw?J{eSFiQ8-pC2`!9|nUTCapiw68ztTY&L(8!+nJ5}h`ST+$)#3UauL z3)&89X{n$rkV@W2>>Pk50ADi=IDbz4N6s+W=7~|3Wq|D0f)^nRlHg$3VSO{v7y?)U zQZ`Kgj0jt@@(g!%&kE(9D+Ff6%%;}32Qy|dUMuW3MBFZ(v2Hb7(dY#4vY=RU;Dq?2T3XZHh0s`(h}Z#TYW{bH~*`z{;)UmsQsCHcaU~#Ub$M0 zcO1_^7aVSJ?_&H0j+q@=39Pgps{(|U6mA3-w%o36|!b#yDlfs zFBllq%WxlK(4Ynd-PjuHMM#Jf&-v1jz4O9bPmsr*g#PO@GvJ+i)}V47;6R@EV!w6^y6?liB1BS{An&X?t2`zWS1SBXF^xN?X$6s1O*sx4!YX(v#zd1=AxDC-?l=pNGvDT_&v!qO(0IcsH!hT&@+I{<(ojXWnk5y zMq=pLi2Dqk@)uWBX0+1C82)_bmMq{ji7J2$d`ch?B9UYqVQ4$|z}`+o)Hwj79lW8xf%0xY z?7nC&mgKiviV;f(ybMP_m5-F>&S;{K}PJDhqk*l9|T8@@tXEIKqe0I04`EL^e*GiZ@nxF&886D$t-h?FeD zhf7i^NJ?Y|PEEafm9)hCcvI1Uygr@aEzpSH*%6>2#6>#LFtiD7-b{}vNYTnFP5%Wj z28-&OrDZ@rDPpF8%^O(*w;dh&4Y6NC*C>oh1ec#UZO^QEh>VxZ|&> zm$e3KO#(zBfMi!5$|qp*GECaQKeUnv&lVIA&~y4U=OiG7TAX8&xc11hmWP8iP!<|9 z%mQkJM7E7q>Ddp~3*0u}z=NQmAnoRgSxk_wEx`mE#lr!VB(XCREcN~SLNt2_YEcy0 z;;HwfEq3-@f&)($cY(VN^T69}!&Npm3dIeMzz8sViT7AZ;Kuzm{z0+7{=dCQ<$I`6M@&$;usyZh%9x z-$5Op@tR9%8gMrfVR1hUNTKJq57|X>>@lT`GR}ve0Lt-0hY$C-CZNNGc_j&Hcxtv9ZS2@_PdIaIt z-eev3?YmLL@-dvOVPhjoSCA_Ug%LFn8Xq4760-=9Ho6NVw2aVw__sVeh7kv%Podr# zS`SQs2z=1dCc7}%2Aper^d0+Ev4NI5W4ViE@kUy2LGLrtT0+J2L$tmSz?I&F?x{Dy zHwYs7Dl^ulu!*mdb1ka|*p+g}= zqEiE5fJ1i~BaI7|Y__wr6K-?sX4L#hqLz5k1!FNix88#z>5)-Uyu7?huy55t5Ag`E z9+$L>YC2N;x^Y?2{PG-(OJF~O6&?ZnIM6^Uwr*7;NX6aXT7{opYU7>R8CxyDVMCNU zD^FnfB`$a?J$z|@EW{qlz3ZsP%pisnO)4fC@>rq0L~0$>PuG!u%7Z05$I1?m-=(pc z&;E8C!z^&-K-l5v1z%;J;X}uUT@T??CoStIj5cJ$(It@>4bpOz10)!`;;s&CW!|q0 z54*&bO^G7icxecSa+i&n`8;eFjtRV>{uAvI%jZ$yE9f?n{#s6>x{?wTM?lRMm;38d zLuAo1Z(e#)N>3<~bUPv}ER2$Prhv248qGp@{^yA)ZVfNVthOtU4xyDbJJiY6n?PO zhU7eXa+@${vAD;0o3eZ4@Q@*G-wrbIXP?>eivSS3P%cX?g8)My#(z;>({~>LyRAdK zJe;W{+yBs^^Jo`}>^8b_KL-122D%L_isOr6ko0!tp=YHxkljNCM#G_KMa0Goh1qdI zICPaP%l+|qqXalmf{|RfZ~@Ut;}8c-O?3yuLCc$lH<&~x$qtc2^!H;m?TCx=tjxEF zC1*UJHTN(sL~4c2{yy(R3nVP~c*ewRicf6fZ#4^6fVc+GF~Ji}GvI_Z_MTXaId&kc z_m-Te1OWtLz{tYO^$4Em{E5cnNr~?F;!wPB!;HkX|5*%jhz@ur+1-c+@ZeV$N?WP9 zDDewGo5W6QDUee^@TPg9V<%WrH2jAQ4Yv~=JZ+_-P#A)%r8N`ecUWnVGm4kUIqajz zCakz1F&5?6cO{y#IWhH4eLtTWUSi$VppMXg%ox|SsiHpw)%5YjxlCF}jg{3_LQ<2G zjp|ZNnEuYrco}8&BrVKAm=j7J(Y^zh6)fAhA5-pUf#=v23584OBsDNFGQblgEqEIm z17X=i69XfxBJS?3iWP{CQ7sF$KG_+F=K=^gZDj>!3L*<&%p|)m0T2*A18Zv8e>KNg zGK{xNh!O&Nb29`am;;p_?_{Ohr=&FcW?SZ7^lT9UeQn}vmiKGDa07klPNQJ0> zlYH-?Es^R(R*9}UuVJX^*^nS-J8&`D6ZD{grW?F!xrIUT6|m*ds3^$}&sy+USw1tr zwH@2|SmfHU9+H-qlX2ojke*~%_jJ_( zKP4$Ed-gQ*)~ui-;D(LOHlPk$8-Y_j`Ul)FbFHiDTn#TYlCjk|#g805+yUJ2k|vj; zm=RI+5q@(q9r>M2O?d?M$QCb_P{3iBlgX-vxAwU$kR{|Lgjucu{1i57Y4t<|UL!JDLuqD;*jOfX28NUG^XegH zAeCEUl>aK;oNkagn1kr(Albnfh6uA~qJl|xFo}f|!5E?oPD2r8 zty?t?8^LIc_eAXskV4EFz$+48kGISFPI1N6qL!7M@y&p~krcVog>KG)CL?-mx;jNe2N z`wo_aSk0g-%AF=z>bTGp1U~TIH?wiai;)0f;uEibl+)u!ARfrtch^Q zyqh@|Z2|Iu!NX6lj>`h^k1i*`PTrv1u^3v<`O~$?($hG=k z^eg%f`!d4i@$&AgnZUDz23pWAjW#AWEw7*6yX+PJIcALH8V-AETMQ9#zMNn`W;IC- z)?_CR@nwXmEyC65Ka9;J+%~g7IA~$DBs)>8J^kLjSNN`h*x~0%x)8*$(@ZKU3!oeY z>(Sd2OkvW-W1nroNKC?wh}<9&CMas)bj8(Bb{(3&*U(M@o1x>@m_phyk@;AWxeWjp zZA68p3BB-JwL2=$W?5CS^t{L}m&~2uL*pa#9-MsuSbtw;Q&k8WYRX($0!?0D0`&QbsWFc zg$4j>(nIlkF)8eA**}we%knz_P2#^N;EEkIwt$KtkT6a$r%Yn-hel7`8s>4dNr|0s znOq|<9mMrv&+**jUfYGoI3_GG0OL4*Y84$APJUE4gVz8gf=^hjWL@16iiD+3+0C*Y zn9EfQhkS>w(?XD4Tvocz#s8Oq{5}f?CY`kL?@W6sT0|NZmZkUL%PjBCqllTnD7A8m znVNb_1zT{r5T{UxvP~Y(NWAAkir5=?!SF9h;J;n@95)+Q0ATYJmK9hq)CcYW2sDLc%~~Rz zM)Wk6mbC4)w7mC!Sd)qtLR*LSZl%BctvQxv2>h@0__by2(%J{_&CkBsaD{65cJ=cX zx+qqZ25$R80DBjR0|P?fH8v8li~z*SV(kl;MXG-Z?QQta(t%Ms%LjwqWj{X;+kF*T z`yl^mxPRp1OW&9JHbC2n{D%x%;EWTt>ZWW0I3YUvo!oB_<&cRwoLnDnum*&Lglw|f zrwd&W{HO52aiJ#xgiQ>=)SQ$Q0VHmnc`JEw-2kfV?L%8OExo7@|HnSKNdwZTeQJR7 zaxoKJGRlnF44MVW#zrK&>P*3B3G1#N zvjB}v4!YDs4D-+i3r=iU1mz?Vv5{?&5I$5)Z8wb#laC_oAn#Ts2Fr$#O4QIk?G>4hqAAcI1E{`pXiLe=V!D&C(E;jF zYM9HpPb6_;Js1OsT-Lc)Gpn(&(GNWl@lFG=;e9a2(1+&$P@262LX=Q;AcIq(DfmGn zL1lCYu~$(6aI)1b+~lD;Cy<2)AGbQ*%aE4hs3aUz2J%1qsl}ns~Xt8CFPQ+@vd%IGRd=4g`N8mp&3Ta!xik8M=0q1y?!hn*Y?>p~D(cOf2-_ZDDKv5fmZYSg`L@73 z{uR=-47$#PfTKU}W!3Ky{h>%;!3;gnsIZmHheOUEcpo`z9247@RQu?TaIG#BsNv)_ z_*#>&Q z;EBjCUD|eE!esr3ZKsg+eE&;{TZ)lJoTuiRSin1*3cO8JORtW`c`PvJYN7R-+@EhY zTta1c%4wv}lRRdM{85lyJ+-~CqJGSCrXIF;dnhiby_(I1%I+p`Tfr^8Mw%x=7 zoSNWs?OdC?KkHeAivWD_{1PW0(cg^%e(n6#n-E3vJ(gjiOuqXMzCeFsn1R=ZYzM%h z!*9twKBs|I>BI*W?9<3Ij3v%Q!$Poo(+q6>g*1i`TRYduWy3;KY2I<^;Bu(u)L^7Q zwvu(uHCUYl%)l?*Z!_u_v7ssHQo|!6E`#%+Y4uSSNo+vUQ%APia9E@*ZEV6ne0&nQ zIPtKM2@Qvb(-((MU_LJ3=i*B?X!Ukji*Z_t-Q0ZQ9O4XUJmin_lGjVSB-j{UcPm`l zv=p?rCXW{jBlMYZH~@(E&gsc3 z^M;nR7;6&=!v?Qjym@vrpL06hTlb+^)*V)o-A=~1rnAOx$P*(-4c@OSvXlROma!)dvs}Jn1H{cVSWT8 zq+x!12w+-+AMf^^JBsOFdV9@K23mzTl!8v@XWb|9j}G4tI(!gG5b@;SzHTWtKY_}` zqqiZscNkq%37J9?r=aS82)I&N85!|+pT*!Y)Yp#=xLOel&0QxP1K6vcx&*xUK4kL- zr1fb7NFk&k*{K1<**y!$bLF0m;01}0V`qpzapAkZIH9w?N?~WMKE5F{?9^7jTOqe3 z`aWGb1IUty;(%n_Y$QD`Vb}dbMfD)2(?xJbk24>WQ4GK$)&0#h>haq||A)Ib&y2o}mw*+ZA^y8a^gw*4A z8EO8tPpz_QkH#`lgN*-?>@N)5DBu1r93felmEJcrh~NDp&dEs2x`Dz!sb^poqu#3@ zxmIs!MI3r&d>1SR=Ttp+P7;G!@6(q|bg=WlBZv0$>*yWC^(MS0OE**sHZ#w&B5%bwKYg&r;|8hY z`O*JoH~oKXoe5mcdHeT6iL5E42uWp|MD~<&iYSD!HlZ4_Om>nK*(xQPQlbbU5tX**eI3kw-_P@$m)F1h^&d-}^Zi}l>-ub0>Wr8`#~8)t+^#Ei znMTtmzqE_H)pP6Ky?b3IPaa?tX*>qGwE!uAUsDjmiQMdR<*kLjW$yQwJ!7b_!~?XP z54DR_rG=aC%dWss;X}*4&H2w$u+zNDFpUa|XW$amGLb%%**V%Y*OD4&*HZm* zTCjume}!d4zNnk_rf!;HGq}l?2t+Fr1zN=;YkU0p#<7-v0HBXd2&7F=0n!0S%>JE2?y(|CI!{$U!pLFe(zmry#O4f7 zJCR|bd)Z-eD+YT0$-h#5GDKC}&II&U1?l0V=V7Fg{(RRXf8+NnYR(1UAL!#3TTypr zWzj!(K<<40d zmHW@U>6@qJCO!>&f&3RGk5(l0H9rEkr9hdc8tW|9Q4g}d2u12&id`+q`25#VBQ!X#~HmBAw3Zr zvlcD7GbfjKOH_A+qH3YOTDJ%;g1tQ7yX+mb)TGUKu4ZOb^2$hKN3ko>-+%gRwKv}m zX<&joT+S+gi;a=MgVsfX^=1;BzOWiM?Z|z#wf_68S-f__bzOSj&()Xl0#c0SIXgS; zk2pPZMyCoAVk~<1j*g{oVTI@AVJMrUJ-}JM2>u!>i?h0m6_hc?0i6*PaY% zRQ<;cOv*r2PdnCCXfh}k`l6!1PUg++tju*}YX)lpdtS`&7I;;5vjf zCw%UMI!_E@`Qy0I8J7KLF8xt?2a%Dk$L6b@%^GW8YFtwOd->`?x~L9z?AXC>n9n<1 z4fOPqVdEq-p?ACObw~hvq~)dmt;mxY-WOS2RxGx~&g1doB!|i!M`}B}<)ydhqP1HxE}u?39|~_$bMO{!!5VXDK?7A;y-RkpS&S~M^p(<0Z#p*H}gsDl!H!bkjrC*k1g zs0g13;{!^ioeLqG`5&I!x_fywLXaVA7GsNdesSh|S?;s^)l0c#VEci76(^POt))*k zcYJ?-U65D(ugXHd+u4`VW|ABfh(DH<-UB}8ft?apw2?>dUfW*Nt`)aYC^`ev#V{?9 z<>dFWavb<1ACEiH@QE#!?BnAp$XS@QG1q(Yy@G;*Qd;Z}`^g^q_vJfZ$fUxK-JT52 z!br=@Ex1Fvd^LOV+lJzC=}i!J7kt;%Rda&(cQgXe9#SnajelNi*=;iFE*gLKk1BUr zRtX#G>}-#UBqKL-Nm5Oeaol>@Zfv^k`QG0N64=!NsZU;ixr`@q9- z=BzFN3RTb(xkBmNA#*Wp{#)bYcs?w)hKIM^vB55tiA2nC=X1T)*5u-PfLBOtF`u@VF7#2n~kh=vpmU@NzLG&PEQjIXKJjTfqFB~JLlgV9!fV#HX z+j1U0)cr_-XJbp*@ufQ2E;E5{C&HQYz9Ni|@s7FsQfOEX|EAY{BU4_}SXQtgS9j@f zwP8>@k|lpTM-?Z=KPaz~4Wd4*?$XuObnqAEsYj`!n>-m>z8O|oqZEa0c`BSTZ`qTc z=nBw<6gio~)F2`Ss~}2?3R1^)Bj49RVM|0~npM609H{Fw>U$|s3Ee-{@aT0mG1;W^ z^zsIzr>I+$2{rlow5Q$QvE8(>lY_1wR4HNGtt05BtvDyN~P&N+ckR zyoA=ff)sm@W+D-jc#CWxCdrVsJ6w>EJuYVa5z_tRD(W8OC+1PUVy3N^tDc*rzT$D=J3Ddzq9wI;77G97Ab%2}`N$%=v!_(bcl zmQ9*8dEgmortleXwfN;r?TCWq0>M3bGHcZ_+~u>bHmp6?S-EZU_up@%j&+;I(JxFG zZnO?Sf6crFovaqa6h5gcNh(kH4pY_^F2GWs$iRr4H=hQzy;^iC$Sxgcgiw2=A=?kd zbcL#0e0J(xb&g)VIDK)ri5jp>=c#7Bzc|i$>StU$ZtMub#^JW!lIvaGj=#*s;9B^A z@bvXG1!d*s;d57G>H%OiH;H`tv&ZpY?%vxTVeB|?)%%0WHh}T)T#ZI4l0Vd$qbp8k z@O-l3;+lUlclXs4R0YN8+GPsek>zi5uhJv8jwe;OleziWaU3N5h2G~3BrO{mYy0Nu zu{rdl2ROjLzWr4qw}DJrOT>)`E9b?3KAvPwXM?QG&e^#c&g-hb9_xsVA3-&P8@VM7 z_AOe%<^3eKT+O&^*I9f5oohl>5REh)o=z724+{Mix>kv%Yt{W8-#B1Q5)D>Ly06Ud%laJ7@N6cSI!B zQ=M#!v1#iW8cw>WD)n?iqEO)N_we!>=_23_$l2f{alxSX-O^yv5U2#5hOqrP(JxN;qGPh!rU%P#7hegmosVazJ@J0YdlFJyR%Q{_XC zPFV!#yyG^p7vDv7fcc%|WHtZFXKf!{T$wrNS2Ft0Td=r0Idi8XaxGy(1oQkJ`*PXf z{h(-jhQXAI0}8w=^g^EdshP;8cJWLPsd)0~k7y;iD9SddDn4UzRdqY=- z0UkIs$o5Xri~qTMcRfK8-R2<{o!v#DY$@Hh2R!EHtJ;>%VOEAM_>pz{fl1L%+fqf+ zB$EH~nyOBY5w2fSL7D(U@k1+7#-t7ea~awEy*YJ*EqbyDKExNuLS#=wOPRIHDn?{w zmWD6N_Cvd^L2hrkT8J;3;MY)Xf6``~nzFl&x!Wd27~kWF&!gj8Kj}ud3oqUvMCa7< z3&QNuiq+?GnlW2|;hJ|z&nr&coAY_gmx*)b%5o-7qE?wW88?eNR(`xrKGpl(g)O@{ zw&KFojR#u8JgzlL0S)mr(?Z%iJn3i)Mi~~j{i{DBu=N}p zM7BoCZN2-B55Mr8n^+{w`*x}G8@QNG0JWG+VekuM$1BjR^&g&hV@jg35SbSI^a)z) z?p!;wz$*Q7>m)&hw$Ll>0YjKn7+eth?ySdm;!5p6CK} z`((~sBdxg+#!p~IUD|i4Kg`8Nd-ly;^xN8(SR53iH@#;Y3)|}p0TPke96z7gUJWRE zlYFX2pDpX8`$(!y0qh22;7nm&8+mPk~GqzS)jQ&&0G$I39l zpM`MMAL9Z#eAwVCLBAg01QqEvwd;Tp6a1meC47qi--n#b!+GLwj3cQWVm|>%lf}*b zLx&XZJ<}>o-hJ3Lzq*?8CAXNX`4PdzIebi?3POT5f*IW?08w$DIa3Y(ldfeXWGmZv zOt?o2AAXT2D5K}TcJv^TLo=S$LIHo#l?%{zB*^O+xt{K?7Y$U$^mgjKf7F;ACIjHv z&$ArNYDI#ptRa$!?cS}kD&{X-I60J=U8C!PacrV%-Aw&XlkIeZF^w`reogEPVnK|9yN>r0;g7A&7p! zz<3g61ff4SzBTD!m20OR|CDq2>Xj=eA?vHN18Uk5+2$HYwlns7IRfX_Z0?Th2uoD~ z&1A%UP6BejE1+QNCSA0wNoADPBTOXErZIhhvjzdtX{n5HI^T1VGf%mGx6_pj)0vUi z98U-ynV172tJ{y|lZD$u4ZLAw|$Z2;VB6cgkig72XMxFcioiMYz z@2tY*1Jn@gx7xA6>?{YpyZ)U&&Ym3>ad6v0hvh`e4g7VC{ZF7?a`G+`%0%pQqrU$nZUZJ5&1Lu*{k&#`2~CxjkPatII}x*)e7sm3<76TtRk?E$t=XE$Y|01^q^;PH%|qK-34N4}Vb0y}ZT%oz4360+)M5tm*@@M(EJ!e>;~Bs?H@U>oW}uFO4qN z&9F6bq`TxQi#GN|+OTa_R@FVaVGLklL#Fu(8&a>W}@ajz8_3$(LrlkAC-8=mJ&{4crhu!8{w!pC*}w z2Lt#i;R;2dIH;bu{lkV3DG(akREUie;;o?c0gm@VI|QNref332Il{H?o4C0<048YW7k7@F)YGZEtx(D>R+~#3~ZNG z0rC)19F_dRW3F<*_Pu+L?s}92qQgF_uzl*quV0tV_5Ee{?rvtyT)Z4dkDd@(RaJ#5 zyTf1QKHO;%is^FW&p)+ZW-BarBc^-8!F>-wJHmBWU1>$B2W^?X3soRrws2UVd)x7l% zx*JHdMi*tnN~+;5pl~9RIt6NLn-Q4F$KBhqtjMY%V>+VCw!4Bx_EhNt8Zje4jwP;G z4WyEiVk>i9TP9q+)sG{enS&e2I)o1i&&fml&jh-M36z>u(xj0Q7zwK0kZD9VV>-B| zzGnl&fAbE5H!#On9Frx~WXPjVtw`*XWlZAB1}@H99f-kAMXD`y?K-i6$+`Cpb zEr-ExlFF;;_x9=@n*eL#!)`u~k@NfYRk%`RzOZ&E{yKYx{yTeqf5ATd{pS8z7C}LF zdB;XRZZefOiu@~HO9U^LpJLhIA37Dl_^T8JIge{MfmkQ_qtvCMxuZMr&kX>C5CKCq zBGo0g3QF_oy{;wIRV8&9JosT;Rf~T7rLx(et}38ztpC7&-mG8@ZBM>W%Y9qT&MQWY zVE4k01J`^$>v|EMP4>aQe!XB_2Je!W@=un0qBMH87z-GZ@G{@L{DHgq_GA^p`#A%t zS+Yfn89p#FYp=u(a$7uG4Aq%zsVFl^G!4KIwaqv^;Xgabp)lK(4qHoeI zV4!x2vi@P5zVc2h0|T(QrhDIHuC@S((T30M{rAPWUS99!cAtt9(4?vMrM_EIm3k~t z(;D@DpRuEik@pA+1iR`%w~}=+(zx;F<{idAI^0YRNV=?Q%|nId3jSpQ)Te>q?CNdl zZR}`AZu(e{EKa4!hop_t34PHEBFQXGIv8g18&<2{tJ_q+YtXP^L*3<<+`>f}jE*NT zLc_G=aVGzZx3#6tCNp(eGAd*0bBmvBtgM=TnNjOj7C!yKKws|t`fjtye^hSq+uvB#&Js!s$`Ah+$Mi?V;XfB)`M8_l|RZ$)i%xSO3_bT|3Wswu&C zAl7pvH`CskH5PpM7<+VgguJ)~p8MZ@yO#_ZQU0GB3*ztV<1RJB!L0@@1 z=HX&=(o;H0FmazgOKuiTBMBe%7@H*B+f6m7jL+$?Ymg(np9KiO?c2Ary^oGRqH5bc z`-FY|xWzs$Tl3cPcg%+{^3z-(CG21gqZN@dT)=yu@Yxh_5b-}V{ho4#wzE4J#fOAL z7wCGaZ%xD;w9qcXm?=e*t~iyN#kt2{$+;iFUo^wI3kIJ4(<)#0R0#9`BKOko zUwqOoC!grBqZ$Zge4n1PygHuUvej0hd-|_l<_nk1KQ*N{Zb&$j87Kb!=O6dH#8vBp zo_85Es2!b`=+>Tz-;_J%-Gez%l{K^OqYjQfdUUi!y#TUHB~ush;N2`&a~TX57c}H; zb(J;fwI6Kibo85oP)ss9Q1o6n+RZoYM|08C4AW%=x z;w|ZNl=mI%U3SV-=BEgx5PW%rcr zk)XUWu)l1$SN=j=8ka|zj#C?2dxw~85iX`L zvq5Si^wBq1J=gnORrSp>O~;7dy?d`kVrcPG9D`~9y{)%oc>2Daa`uI%VbmvZO)`d~pC`~!~W?>e>9*4V;llv;$=`~O zfo7e;3c}2NyTtT5lPiBHpyD0^0SS59Kj6QCn6;`TmX#lV|5nW%zE$(RBhS5X`8!Sc z|AjpY`n5fc5>VRg@Dk`=+7_<)&qE%@vNbt`tV)3lPUJ!p&bhtM7S-d7^5X=cFkAVN zBINVA?1(G5%*}-hq^GD_UUxvMB1-X!kJ&SxO(WZp0QGQeO)5_Gi>7W9>x~jt0PyjH z@Z4`{A|Wk06=4OpIUJz=JwLkO`Dj=$H&<#_RNUDUM#ESc6};D4hJ{QZ8rlaZD)(4M51Ss8{aFZ1rX z9qy!M$LAh$D%E{fgQG21AbEA!8~IaSVmmpf+bBt@h6TYIltWt?Nw*m{rkj=yF!KVj zk&+_7j56$WzKLTi_yOH`?%&e(EnOOc;eTb1f`eDR`4nM{{Hi5V8-5|^to1!xnx8*Y z%}pbJ#YN6CANgfYdUo+F6$Aqt*?Pum2-CSYGj}v?)=Z?MA=9|Tz!Xc<;)2s0099Ds zg6;)k>(kVz?c3Mk21h*51SxO|3kthG|D>h{B7uWVaecJAOHEgXGm|ScAV5x!r4*>x-Sc=1F@e<3**E7+e93=TabLsv10LNmo zJ?zH*7BKPmeI+5~+pDpPJ}P;>{2L0>xWq_Bmu_pd@LsZVa!hu=&%<$flr3pm1Y}1NM~hWB4P+m|#gh^rX!nWt6Ib=E{d6v=+n!6`Pnu-ZwQFigTzB3i#3@B-Q;jHVWEAzBP~2b0W|3qv@OpbihU_E@@krfq6;7adG+-I$f51z*lKN2 zr2Q{DcS_bv;9b7;EJY-uE_T1J0zbsC_h&|QV`dy_1b7ax)EpG4>E@39W&_)Wea|d~ zEWd~hDl@Ak-U zl&jqFJ`q-Ue%!(XBm@B|PrN4qw^6qgoo~CJs<_9ji8$uo+UM-~x(313I-aRqpxEd^ z;#h=%XY&BcfYu)|s>ys7I9@yJs87-ias#fZA6U5pP*>Vy>yMN*vb)r+TV@SJk3A`r zMF6SlosD**(}4{@+gfzS8=tJGmEsO3N+N3JsmZawnwu(D@*xVS*k?Cc(&Z1|}$Z$HGB*UYJXx+EcgNdjPh-$QSjHEcjp;TN=jb$odRlar7QHq*zCzv6Ji zA2_F!yo=9|4C<*t-P5>VC6T(Ke5?Yq-ucYMuX)G7J$iA=YG2x#UXX%fK!h);R+GAL zgQE@51-LAq-v)pOJX@%~S3q-DxVdXCEHpA24@D`k9Yx*3%F60hf5>3O!Kw6=(b;D@ z?-_|^RgkT*YcTp+&5?@v?PaIRoXs&9{S9)zvbzfxPLSC_&fU$h=2sIwrDQL}|D&}u z=Bis&zH-drt5kJWS^cD}_gm~{LzHgtd7J8LOX@Zu#;cL09pMYMx3}3fBkb*GaaJp` zlWH}>i#YWDbu}w#ygCuN$D!}~b#Aq(Ep3tToXlu)Nd3yPM$oiqL;*FrBLau2*ghY4 z;yet(UZ5!Ltn;U*^rTI_Mz})ZG&Enry!39Z(EwOZRH{q?OOF5XM;l3ibaDz!_OzS? z!N?!qZ}n+^(}T9`4;c_)YS40Uk_}>1NC@ zmj>SM#FKe&Oi$g6sBe6qTD+?Twir+r^zX6f*H525xw9Cr-VGk|t%gH?KR;vK@kwXu z>1b~}3td)W*^jsaTeceP8m1_lme=fAuQpqDbt)SJEna$etWCGAz3N4O=?S!H-gAM8 zRL#5JA0Z=q5-$)HQ2U@rU+Z19+gf0|Q=Ompz}M0}jkrU56#c;Mx@>T3263BS?N>#= z4b<|jbwB>;w8OoETTd1dPEJEw99Fw8Wo9x*Z82;h<87+@m(t%`ui|1+FUx-Y1U=N# z(|b2{02;Rixfl9HXO9jt^N}`y`ZiwF45!e%3cvJXxyU0XO`<9M!zFu!LNSv?D!Xg~ z{D*D*ZS1~*PnOpBz-7pCuY2?2AM?B@GBa;|m3M^{g^gfM5y4gxH)Hy`Z`gsSsb_ux z9eh7^GMTXCnbbQXNBP^U_L_7v3m5kMR^WptEmM{at@dM|mLI zeZyJm^N#KkMS4|_hx0LM&ct{s50XmnN{d>m>ecJxOwh6>g<29@{?);h2h^>kL6NRT zR47bG78XU^I^0F(Wsh6B^zIHh#x=+Cp=S??jC{u<(pp7U%4FdMfm#Z#EW+8f6tcY6 zBd6f@k#B3P``c0<@o_K6OPso(w|*p0Wa@dQOBhq%2|W5VBJb}%S=5L;#khiHgIgB_ z58ge8zK4Zuq4WoT#xoLm;Lct!&#rUE>h~+~CqGXcr-%s-3$tIcqz?>sC(#UiJ=9Aj z^(EJP6*ZfQvU6{3mlvZ@bP`KyG3{Ho?rPHL5>E5})!>Zp z*94rUkkjqfe{-U9=;p=4Q4N& zqLX$0uX*uc|NM%}ofKhAU&1D|3{+nM&+ZqLs^@=Sok8wC9eCWM|5wj=!LjB{_L~(2 zP~7aqG+bUobveb%`C{1xHPbH2nT5Q;ANH%Uy7DTYrs>Db@$i;xaM{8=4%==Jb0XWz z(yV9CYqTNJ*~JYaj+{6#ru0!GHDWqmQ%SE_n-5p$I-%l}@M}IE_eG1!1}(>Rf>vn* zmoEBB&4Ha)fnJ%8-LLrm`=2u}Fd9mBLH+UHdKdXxsfqzT*Y{g`c#7S>J^LQ_J4Si2 zwpGzp9}RT0wbeSM#M&R5wcv0)SRye&h#1r74I_P=3WWg^IzL2zz5E?|hHARN= zvb<*c@kuCpWb@aOpXdq65KO77eO$L!r)TI%dNQHrv(CnotUSZNva3P6mSM!8NwcO> zhv73)9y-*J9_Z^l8VPO6a^F?RrnVq8Bt8j!qOEM@X6PC3qFFQTuyc?&R{qsf3|a4_zTh##G5)kzWg8-tXhhU|HsDZ$0MVyMjg5JQxysCQ zYfX?zj}njQQ=a+($2!m`L^}|x4vZhV@b9R>QUqUApV2LuE+_JGS!jc1%lr+O7 z`-=g7cIN(GW>a6FyfoRgdGj-b=vPp-YM8)Jj`?jo4ES(J@WF#W$LEZWP-lO@-dp`& z(QUMzfj4zzL}7XMlTsfvKqViBy!w$?BT=NoA5GpG|1h}ka(9MG0Ty`DqLxDeK^ZuJ zl&tks=pGZJ0X(?->!{YGRlo%#q8;Jq8}bs=Dlbp$E+A;eGS|-XDLT*qvFBNR{y? zauZkSz|14csSO(N+?5|F4Wsq(UE7X9*~61shecM;;wPfDB%3zIwV1$Q%vNG$MpBeC zr;!HiJs6pQz^<#`7dDpMio`@Kkn+HA_F{r%3)z>jg2vz`#x>#kzo-EEaMz(*?xx?C zG?+7khSXD&WC5D6S;+j8KR8C9l0WrRPXCmQ7t0B?fV9qEQCE3q1SrIp?nOInW6I2N zp+!sU7E=5saF zd|p1rtbHf%bJG_eH&o*yL1U=3hPz(=h4v-=2Mp+9YC2-(@25(AtYEbQm2gcN4ZA6H zf>PaGLN6J-uAG1}AM~JgJolAR&jqi`XSUMTXyO?0jxM(Q+tzCPXLK1u{-91sP?{pb zm~69d^MoZuJ6_CQU6C^Gb*_9}fEJxavkr8Uwi!WbBM`K>?;~QwtZ`Q1KM*#8CX$fr zGGOm52~@Lya@8%>OlYHBD-_|k>SvdE?lxi~KWhf4aU1?Y1c!CK7uvAIz^9&Fhe z^1QenfoiNYa-$V@w>s_JXE(_Yh}QF<1i*Mgo?7qN&N`%1bMKneBJ19WDnQZ(LfF%y z#Rb_@dT=SWvI=L_s>;#f3k&|W5jC`&K6Y#igbu|^_thcQ*6_^xjwkWYg3@RND0Cja z&l1AF=(vt5jam*0Y|S*Kaq!y_m#@D4g!uREyYR0sv24gCv@YN&;RD+ZU8V{}O zMF-iu{lK-P;As)S8do*>=-&d2U9Nm?Ae7~a$AsTX%&lsCs2g|szm;5&;-C`5L~_Q= z&ht>n?D^R5LzpWol4yO@1^?(RbF;HCy*rZAkWI&dI#BwwX&Vx_2`G!HV;p|tY)N{S zED<_p1Xvx=&@k+hot6<#y}!B3V))GNM*2<*3Qbg;8SV@ zO?EuCtEs66U7vI{Q<^nxT4A}l%-TVj@5xBV=pO^?>N3{ZIh6^Bb9PO=i@qr~pWlA7 zckw8grc6G1^ypQzv0CBjF|p~ejf}^50ov^7dDd$6r>3m)4B3OgOJ`G%*)Znto3lU6 zk4it6^QOY4uBgt;dhelw2jQVx5Zuy#ud|!O)*rsxnKYxmS~3~^&z4lW_VsHA(+(|$ zttzQ@`;2f`!c-@A|9Qp-%KukV*SeU-o+){K*U`UJ4As=$*akvRB~XMy#e=95;MP)G zBbbjwh&ZHhF(^hwOFz5K&ucqpxda6Tx=q{A{K&7VWxa@u zgpuDug172;QTZ}>o_8Y!6MN~02C(w^|S>5k&@7Wm(oouW8_4>YF&y)F2u86J$jpB*lBb zt{X$?c2u>MzbNppI|>irOiukOIH;}M*|i2u)JoD^5p6BF>0Gu|JRq#02*0#&ot*Zk34gXl z%OT-`w7K;%>?Xj#iQru}9$3ucBfUv3u;6CW(O{TZd;a|S$+Zdpx3aN(bUj8^qmqLA zvQ!$uCqmx$?|)xBWy*bnA4Z&q%5H9G6@*_SYB})|D)isBv(lvlTO*4Sp9rb~Pi!LD zUre*eeTAYJ8L@D+Ir9P^ECfFZo*Gstb5mrR6HH6Ai2yWBd<Z$s7smoAK6QpuSyoMV&UDd^7f4k8zO=R`^rjpFeV>F#;vqto-?aHH2!~iv5g1#}Mzw7UqGfHFeyz zoUtL7Yd%d+Eu527S33l-8+PY;dD8>(d?1J#Zy9ot5+;f*#zW~}GO?J;s;Sh2VaUJ5 zrtzRqM&mSlZa$DX_wmmNZJXSlq?8k!{V1D+8MxfIc~h4iP*2}^8|JbshtHM=uhFP7-*CgBcJt>SRZbc=F1h%X zLdQvXPbbeUAY;+ko$Wp zMBV3Od)a@8{^ND_Qd6C<^n%j4JqNQ&+sjg4@QlC_pwB?ny764il$*El0h0iD+)stR zY`+7-NM)vD_~88!?=#X(V@KcO?0!t88C3>N!WTU7KH>e0@sC20 zE8%v7fAzMS(OlcPC6)TTPpMM*3rZ}&7#07S{NY0r>JTOQS#0Khwak(%Sy@LYiiRJ{b6k>M3JoS%SCKc#Z)P|Ie0o9H60jE+>pBF15&iJWmQ$ zLGlOxhw#@9`y%POl_M<-uU27wNAsX*yIkMX;kWcto4cR4s*O@`u}ysWcX!VzWeY#f zIdl3nqR>NLTc-mjGZ<{0oOq7yqVKXsbbLZ%nI_fcBPaWmgVE7yBALj~k9IwZ1seq`e0+AW=+nuUgYI}> zyx#1()NpH`2gYX=#uzfjN{@apyZ98ip-3+!MU11mCCkWEwktoR`fNtnT5=wtb*Tr>(CTprDHuV=32kqJ)!{8=<(z5 z+S=N4pFet$xQiX^i_xy}v|fjUjrY6;opm3hqju*@i(;eMQoFH8y!eRGpn2T; zNOHBH=(HRgY7+@mlO>QUSBkibX7+A0%3fhvDty0SCM*ydeU2r$*O0Y}*~ajOW;YJF z7}+pohFoAbB~<9mIa9tNg?u&hJPD_tReb3yFm9ApQP_>+t-bMB?=bzP9#-n> z2f595bJDwF`Rd%~GsIdj9H(-yzp+8ux-HHzmv{g~i|MqJW9GtnbWPylS7+3Ho=hZA z(j4+f#Af7ltv9j=lIZfy11g)Cx+j0gxC2@mqMH1mJXwacrI|7)k@~DICh5tOce)NC z=?`_-)hJVryXAKbdT<)1;!{Inf&9)4OneJMCc$%%G4Gr&5*NZydA`(5VkSF^!5?OW z?oHgt)Gx1|i8e(;+9&>FuX_w`-2K?ymo&BNzcoHa_1QOMW%C9N8mQ`eU2#B7T=e-d zin8uBHzB9W7x&G&cp6nZcfZviId2HUyT)guerV5qaWfXsC;5Nss;|j|$1|;R<4K?w z=4tgqz6qRZe!S5x(-VD44-RrL-Q&ceCWB8m>ISjAEY2+6S@Xk%Wf?_F6-c3my|rcC z!b0Ah_k(8RDp84*aRr=XfBjeA#fx|x73?g3emDB2XP+lT>lf7x*)tAt7bB>WJ>}ia znq?KrN$lufQ4gz?bulvFwNge$w*Jgh+q!^D6Tx>#W3V}eNhE1C{zcTww*abA|srRegQid(Qm@E4I%%VTr@}qp!xpYhGG9RF2 z-p8CP+eU$w<5bZP`F^N$ZsPXUw?9-|IOow7o zmHEBTFs&YHr?4D@wh^)Zy0o;k{}FBNPVB^PoKO|)U-z3sObQ2G&|6u%{qHMluZrf+ zUcK`x&99vMZTxK8zEFP>*Uiy)lyDS>OHL98f zFy{)KoaoGNueAzgjlw*(kI0370_4yA@#Qr#;Sbv0b}b!D48t-l&Jx*LhdJ|3@kgP1 z+I}%vkc2k<1ltE?vEkXX6KD3J`t30z`dN4lKEXRJ*`IMiI> zG<$mue5tk0=)zmm%o1!)_naX>a{sxeT1y(Pf1h|f_E1AbnQ!yW@24#z;V9a8-C;yC z?77|X^UoRa%#iRG;)&uLy5xq$(Y?Kj<9D8ju$W$&DB+{7GShuM+zxKxnxeBFbKBW+ z47`RcR9N*o*K_bLs#S79>2(jN9>I@-gRV!>M(U=2%5yGH1A2xR+o){Qv12$-R?_Qm z{xg9Ifp~1Y&jZ0n0=J$HY)42?UfT7lxAmBN8jZ4x15gmH1bs;NSD+lSE?xu-UAw#w z$HAAn=Evj@OGhgTZ4gG2(Q_{I)H)6kTqG+{x|&V(f=ZVoshk}@EpV-V(0 zxqL4IM<)|v3*39_`IKn5Y@1IE7f({_cm{CkCPk!^AsdL$0)U`TYTq~9@{5ni#Oy>d z5q_d&Ez9^^TU8?WrjwbGvhhK(=@_JdJ)e(0C!vny9e~fLuWGa&KBT38b?FG~N@HEJ zxgQutqJI0YtbgS;70(m*>BQH$Qt?+v^I=p-McH?U^y5eNnO}ORf6u{M+Aen-n`ax{ zuJG3-z6sx}w;R*gY z+jwqc&Yrzj(k>8x>{YhNzN7Bkknjo7RRJN$A&)96TSokD_dPuUz+|*%Z25{HLJmlj zoC>o*2B^X;>{AqR*+SKX99k|79|MmHEo^sb6zd;AstZf6Z#s_Su4%((4H`Aq?D)_~ z3tI(kNr~bddO>RTn5@^w=)|$#dU~|dnr0Y2V)3o!KK?hIEG3&oqAtVPy+y!{WLE>E z9KyBu3WY(naY;nG9wy+itGOnJ=>EwH{EWpTs`2ap}F*tmDMu< zwl{TB=q|xkL(wU^JJTEArfrs5cu-zw`KhrYwCz+jO5CmpIhBW4rU&}Ux=H78-Hqzz z3Qvy`N!W0crdh_M8ibN|C&G|d6i%eyit*zkqCF3ka2)guYnQs!2kN2OZUULA+n&zH zg)ocQUTEb1+BY5E=YN7MRAQ}=RPS>I=qr!}d>S;lZ6n54z4q-9SQX7b_dP=<6DFvu z18EmZ)?GaA1J%CC0+|=adD4P`eM8z*?kCtpnl8G6-h7L2pLF| z?-5}mvQ6}6K_;7cWA_bSu7&z`oAKOxKT+2ZimHS2VaYs%3nhkPLqOAA1{NNlplE37ChOpa$&M47{Cii zyh>OBQA-LJDa}bA$~(5;62=IS^0@ZX0K6s%{t}7>&O~5(Njt8pZ&sn5Mpq%J(LYE5 z0~%j}+g-KWlHM_4g=BUS*{ah@XVp~DdwRtlW3A1;7@~=jS$d^6-C&^G2oJ0i%V{C6*u3mQ0e-pfD7;0M6w^159o_SZq=!rH`WC^`wi+i7U32W zOByzhhi)-h$!HAPL}rxzwP(!X5BU4{+!td6-G=>7R3?>K5FPA$6)B6%kav0Og*^5- z@E^s84-UBYZrE)Q55yg!+r!+we=FpG7LB&<=g(DDi{Ycice1$Flf~p41oySEiCVgH zLnFvo>9Pdtl(2v1s)|Qi|Mnm>)K)6@Tdq&p*FiFPePEE z6bG&x`+n9R_BP)=?!gvOip>tw(@PRup@Bz?8FTx*UtKARldVMNqYF6@F_o~ z0qKo9>&665R7uDQGzT>a`<@2G-OAEZ9s4>#f)EdvLhrtu7^;$@JOvLzz&k3dP?j$+ zC(XWUt2IiR?ZxP_LLIEtMi!W39DDHK0a~v;f`;1L+SUuO-g*WUCl%f91KYdT*LfLX zuB35svCQCJCL8&WPYIOgclCh)W4?jbZlkfN}q z37D|Bu7Mh4ClI_QQ*tp`47})Orsn8~4SY}H5LKB(D~q$zsOkkZ zOIZ2!|4}J^F^X2j2G`!>dckpM-wPSG>e=TZSkjIxGwj$nfrq^E<#wT&L2hgIV!y>8 z_z!l6w3bCnqFE@n3Y^m5tOr9caH1jIv)vskk|~>;7IdMEXR-)dTjnIZnDY8zw|mzK#m3YX*u+ z=E?jv1)_krz7;!}AaKG)7U#Cm-kyYZ{Q#@IT>9!Y75S0D=T;ldN(u~r9+m2`qmwvs z+7DcmeE70|jjyCfj8y3A$@YqplF47gTK4Sv5|zQK_^wgQ@goX29c8MonHQFdV6NnV ziBbJ=Q>Qj|{CuWVv?Ey0)lIZI@N@tSNpb_)Lcf0wlfFu5jBiZlCrT6z5Wnac<8t%f ztng6ja&OI%l0Rd}ueS&IhApJch8PN~)MvHyTDHTqjm)rom5r;2VS<3(B=fZ+TmE14 zPhGB#cBM!nX`(3NFI~dls0oIQFRfPcbIM8Em#O|=ML%nUN^N^N?aUYF#CEf3 zS@l%UL)!4Sx($BM(d>N5e*%-hSdHYtKl+(!umC<4ZN#%uP#dg}VGMV|p!9^~wm_LJ zTa~HjY5jJdL-!@pH~k&#nRp6zN)J$0e0c_cfjveFK=dBjXgOueZS^zrfZb&}{RwR; z;0Oq4{YE^YnRV4;``dNWM_%E9?M)ONuEj+3Kv}5LR`mEmoE!0(hHTmbrzj9*^!UZZ{^rL&`tM{YO}68*H~3^j?s6Otclp2vmWVOyMrBsa@?FlZ8p# z?`CETj=|O0XHvY+@qfGhk~ObO)P#4YK#!;%Pni7d%8@8_+5c{p97Uw?m?eG7$w!$ei`&|5n}VG?#K;lhN_ph@8{IjB-NFNEdhq&8fp#Z z(jji=D@nHyq>(PB;lbf0lG zns&t^VJRpKzhV87`l%UCVBXf&hP3Mu^Fi4b+#f?I)3jb_IhBilz57yE0C3Bq%OP2by(uqpeHNCvRR@o9jM*WXFq)| zm{r(srP?m8KBVjvcb#l36bE3|%Qb<2*rzpm=I=!WqJ|;{us@ESugHu?pk%cNkRXgr z&o(!P1e7o$&zw1Pn2V^9!k?f$A<*P;Kwa+PsU>Ls);}G%H7L)G8ZP|3Hr0>r}9M6h~a~s-MzB7$faVXY&0nuox0Oa4+3! z2CKjw8x32U+WAq*s_ug~wP4c5)5!wU?6@j8Hd&u8?bMBa_{v#+`s8Tb<-ja2Sp9WH zEm|#na9Ts;D$2%B!3`vu69G#Kpv;4>4z<*4M-b|WijIG|^Ve`uR;;ALh|*+81usNl7d z@$%b&3{2E*L8pF6r=ew%R;W67ovX~!{}nE3d5YCnlAA3^)p2MV?k0j8#`Xjn7m@y;N#J0f{QAlze~3RC?0<;Kl7!Ev z33nEg<#1Gwu)R+BMV#~qAe58>=BOU?;F#;JYGX5Tm6ai`Pks4GqpCV&%!_x&h$z7D zUtu<+38*IDaE^Djx4C00kfjCvb7DbJC;R;*!KAC(FXS=-hoJ%Mc=r<*AV>d{Dk!7> z-dlI(4f@se(nSU|(Pz7e>;=aMMTi^(h0ac{7$hr*ZySWC_kGnL7Y_I;A-_IMycDTx z5*nQuTW=WCl`3aiia&!n?eJCpoZpGR!!03z=^?tPqP)DU;LD#d5!pl0C=J`S3pT%8 zQJ<^uf9cc8j&cJGovfERr>mTNFZX<%Fu{2&XF2sPme7|Q$*RzoD?Wn_hrQxKNE=Ox zZbMHSF@8V{xQN6`m1my6Popit8(8y<9@K~)eK=w;5J;WWr`F}+3RWUL2l zPNGSpQ99LP(0rF~3!RcY=T~lFY+3_fmbCphz|mvg>o=u75QB~O-r`Cp;qK6s#QEzk zJjQ>m#}yH`V2n+KTioU^o&G_+)o}$M23qd<*51g92^Bv8gCgSy#FpTJsY`$OFIQ2E zMHdDXl=_?usPj+#fa!F;9&~SOmQa0+h-8XluoOqKRSA4pddunGL?C)0; z@}g1Pj;N@PC)ePOmG=|4NHv5)$v3)R==D|cQcIFo3L*6$F}z852U{=dWoU;YW0$Q~ zD#;eo0^l%;VG|K^N$MQNybGa>012im4xQq-_`j#guIz7+@fyIZeXYEG@B&!f?>|P#wr$4$zRsw{knid8O9u1 zKwRgNH#8pCGcukm*3{U@(NYD|-EuYSBv?DEp~FiaC9MBC0Xwz+oVZtEVru%KWXJXJ z#J!^kNlL-CF|^@{BS+Sy7i>=-{Ofnol_S*YfW)P@mDRD>g#`ZGS`uW^sKjGGn1zL{z}pp2D?*OO9zJ+5pV9L9nn#v%RSK6c zoO}0>M(+-lS3z#xk;?q}(H0^_ z)>Enji^V6p%Vvw^;xn^+B ztK*9eC1)q4%wmwXz7<;S_%(12_5d#J*=I@v%M?R7ESkn6=)>4}6C@~yv@)WGMsw4Z zcB4T=7ro#sqf`bPK)hE8v9R+&@-*n+L2HNl360ZlR^6kDj zj+3?-jThdlZ6ey5a@h~@m2N0U9XV9&u4Sb_%9yLH4SH#87tavp7t|CDQ1pqqwb*CC zqOan_qdo(4fT{u`)HP}G$N^C$-yO%UDVe5N7YWety}f$L%#s36qkxRtw;kB6yJ^#Y z8$WJm0A9hrw8pf#;O1046lb=CLq`rDKOT!&n6k4spPrO)<8aO`Y;fy%RvhVt`CX6l z>Qm}njjZy+Kab~A;8gM!ENQsb|2$XAG6f?xM0Z5GKk=SeV@NO%$wK0)jvr38`$dl- zQ!-ihTd(;&Hl1`7Fbiw|T)G-=Uv=ky&hZb*vqjmC+HuV*>28>pWq(MgzpqW6Jo&Jm z1hIfcjng+$=1T&e9^7VLgZ^3!$E0~RK)d^4YWY+D3mDREiH06Ipc2^^)r8~V?v&F4 zTajuAt?U`|_Xs!h^236YHgCRl?sPhsuez={{mmOUmiC<<#Q1nc6`D)uMl@`&&2*G_ zjx1^uf|uDR^a7}TG~$~cla8%*i=#@PTZ)w(?^};~Ux=`>RkPWfw*MlWprjT7*BHh?;0#PcBMGl5Py zzvzGpZTz~yV!zcYqilQ=7W7x(0f-pUoU9!qJ2?$GJY1!#lphH)y&AZYsu2U6rV9WgZ4+==oqZiuK{D9g3^K!QDwWH z6+i7kXygXsuaR!4auzd8zToU3X`4U=fs;J1Mx84oJq=Tf**scp!L^&ybRtd(*>o44 zcGu9=e{8ejH(RF{XlQ8Ioo=~(prDP5;gqHa#F59saMfcTW$VeDHAY(9_?;o>^>|8y z2qA}G@wd+mdPz9y#YSu0|567@A z*)|B1^UruQaZ>u`U$;zd-j6%OIw_mT1r?tDl&p-FGabupV|91bMVgaVHqkllN|3|S{uzmUEl2lpTuz%?4 zL$SxF#cPRo3*(U2Q(|rHc<^w{Lc3YM>JKd_G0F_M_4mDfLj-U>oy3H1xh%LQn+UIz zDEI;pqI~1#xh57C+9JOtfV86Nb#)Qz`c=I-XB~Bzc`gKmu|2i#f1B)J0U0fKw`Qia zQH2pX{|9EwS+K?M)GYmFgL0KI2;iNL*{*-kX6VG^SIkN1UHv&VqYrrWi3%SyZl?K7H@`-r@@m3%_L0tkrfO#vzPwE(NQKi(2gb<^Q{- z#d!sfU`VkuA#|R~VW&vMHYr6oLOgI|qH0ZgfB!;%)ejxiBzF;M1cpnKaN2UXoa+VK zXK0dyZ|&?0UUn<0fg*C;*9@YnNskX>%S7bpa!;DF$jA?WWCaD)f6I+0NeQj{5tqfn zw$eSlvef%LEWJHN-|KD#Hl%?liohTa$KnA#>DbPB6y0q|)pj*(;?5)6D97=vZ(t8@ zg=?MKS|091%$|{-9$XnlG{^U$w`TX{-BC$e>i=iQ$}slfL=(ryB}WhrLrTo;Z0QnV z=_~#ZU2g)GbN>DRXB%0kL}W=)cG7~7vJ{m>)`X%$)>Mp2M7F4;vS!aVJ|ZDPWvzrX z*`^W4shCN_sy8Y9Vz^!@nT6iF8} zUnm{AWAQir_;m5bhOZ|n)x?9nFX%X}9;>AqDzq`;w($RPRzG7@Xbj!ZUNe73MOu;Y z3f!q`oDXC+{kh2Bd9C`o9#|H>&>9I8Y9(7*8PK5BDe}QzO zF4o2n)qZPJCP!)M#6N_a@@@&U&Kpu7*~l_tsAqk~ng0pLjf!%9P_<7b(T4s;{7U#Z z6-EliPziHW2)Y_i0#{_bM-Z*(@_l@Kgs!?IX5CqFp-e;mR*`?!xktU;RkDQ*+ZkwdL*&a819a6$)A*M}>ko1dz zUAig&^KakC79Fs-z9;d(3J#PFTAJ-PtJvMlnjOJ{+=XJcUX&roiOqH&e|)=Vl3MZ_8%_n zjEBg(c@rg)$$qV&i^II3=q8!C|0;QQdHEUDD2anOCCpEc=gt67l*B-RrG^?b?$3t^ z3B)fhWuA5}2wX)9V^R9tqSFmOzA6SFL5=l&?@r&{|2LDd#h_HnGwow zF`HkHKIf)qhgBQcj@+Ws`d>kX{7PRgrwiR&j>|%lu&%oqHU1w@g^nh`Nv5mO*&5;~ z6PP*xR4~wOcw{@Z4PJ|W`xb8!lf5gj+bxR^k8f|W>tEO5^3v{Lzn=~D&ng3ql2kI> zkj71rb9_+!AcNleQa-t_>ON^{3+MAQq~(7wl*-k4%KbUS`c# zCVr+)kO*=RK6s=FsxVlQLEG}v>10M0NQ-Umd6PwF3mZl5g%~YqE z@wK9luGI}@kgZg;KiUf%=s`*hW}>>0YXSpC$i+@(Ffu^Y6{hbB7-9oxPrJ!_PjhGI z9B*^PP=So&Vh)>y;mUz6&V!~k08@UI%+Y7ota{Q3kp)(Ml3}^mn1rl zgw~thHxo0Qc=;)>Etf&LgD<&|%z`dP6D&$WtNv64aFAHH(M|lOVkzT>f6I_6;}8+@ zE1!s6kFL}-t50BxHS~jUL|KS2D3NI<$g%wzu~lZ>N$~hV)|)Pk5*!_Ayb3Y^+uN8 z7X!0ksV?w(X-v2v4W4K?yTyyjk>Lx>uwBXWI!b&a;R%5n+lF+x)+jAomIlCXR_uLl zWow&467!w8Bjlm}`E^bDm`lM=eqQgd7v?Lj_fGRB_K3DLdb8y2Y6*y5buYM{<_7Xm%3y^RhMU7 z#p>|3yrX~R8p=L4&CXf_1JaQ*E`en#E5PmxTqb+)c1={XQcbjFo0iB8NISS2M{wJr z)2Fwn<)PonoR;=~b43$n8m(Q-mEm7C;q~6E3BNnn)D`zDP|r zmzG)9Zk-D*tr(Bv9o>Ponyg3KN^}%*z}s;E=d(zO*&7|Pj^!;=f)hhSH`NuhKW$-n zD!Vp0dkfoCZ}>lnqG=X9jg0A#m}{nsHipsRnt@#Y5`7@ia|InbMIK@PWEJ+CqRN4> zb`sy!Zh)_erUEcTrb|28HfyiBc@<;XMigYSFG&+2k#k+uFeeiDdZaA$)ikC_66&~> z+?$f&_oC&P`LFWl$x;{HPK}3#u2WPR6xh8|dsYGsdC{&qI; z+~4FUSM>R_pMoo5n&eglFfS@`b4hgUc(zp020)N({u40Ej+K6V6n5{6F^ez#AWJhG z%NI^2)61Gow&hjDeDXc_zf{7(r_jWto-jB1P*Pf&%k37AaxTa5W*Dm>cj!57u?SLc zkn9&|RE6wL;)JoWcK?VjNFfdP|FCi&(Es9a{f+g%Nkk$XT;GArgP|^ox`T5PoHwvcLDQO?7k4L5mAX)=M-o(s3OSQ z5{7YwT$mrfbI+bZA^x1#_aU=gUAwg%eEH>K5qzb-G5QY!Im^+Tuxch^0S|&~coEdB zOXt@Dbi7}&61aT$g|IbOu_$)ya*3cKWB)1> z_|fF`QC24b7piCcfX4`K?o?2YHK-Pjak1 zJiR z>Zj)q``N+s_o$hQaaAMXXPW5mxlM(Egc1VtXsH`nJwGy&l|42uH~AocMkW3c%Qggwk;4`7dY;WTV~Yix4fOKS8OQBRnU>w=+6`hx+4$k z&zu?{b`#XoK}ktw{>yi=X}WXgwZDFE5h7|V2Z#TBf42DCvn2#yFwJ1-=LI*m;0~1M zZ~px71=8UVXGn)LmCXmta$Q3kHf$*AamaSZ&+Jim{{gj7dK{yz5Jag+0A=su&LUIy zk2QNQc6tzF>~UsBf$}c^lB|nrj{UxO7RNT0d;%mg06J&GbQ+-}*zDlvmw=J<+1q#T zqAp#Uswsg6@HA9QmDWAh2`qE^ZM~tjz~clSzZ}OMu<$qEdNyz!Uxg@6p=rc!qz@vQ zT{_16`d#PPZjk|hfBP2vA%joyz{ryljkRi3m3K;K7L(B$BZkR1?r~qd$}$e02)v!0 z5V|bo!MMm*=`{6)3JoL(6D_{2-;WPB(n`=?qGEEz2p0liG$^A%L=ah zp0;JlmFm5{rX9|lNg8ZjHC5G3%OH|#cFgqXp?DH4djSn|g~?hK&Ct}jKe(5Rh@L1) z8Wy>gT8dMOUpxsc-qo=}$i6OpP6RRQrXRC_Z8%qTyj!uDawNMIofmJOlKH8mW)~8F zH6<^Akc{jvlVs(rv|h4DfQ?FJv;7%k?YJyKO39fp`{+v}3smcZ!mE>+lildiO+?vh z0lY6kgo;Fi&E&oVl!9$h40)$RE{7m%BMXA??*~=`cV@dP@~gQ&U{WSOGBhxdbR=^jK<&?amL0-x#(DScTUj2mDR8(AJG~6G9nBgI z2bvK?pVIh?*5d;JD_qZYFK5J}d=(NPsj7T|#Y5r#*mtWzlTqjHS$5ulj`rx1@E)QE zz=pKf(^dmc?5Qzs2!5%!5!sQ%{i{d?0ALXBy271-VwB9rt!i88@iW_DSe6Z;CW{v? zT=)F1IS;3_eN5R5Ll=w?w?-)Ma6V9NX@fNkp&ZMhninr#Y$DlQIWNy#`>?dJ^>A}N z!{WAe3lrXGmPsfW8_?gz*6f#-ogVD%I{TrZpmg5+7Pl9yGVeTl?%cyj6ZgM&AWV}D zCA7P0dwUrLC;Z*O#J#Bzl4b#Gj#J$j{w>4nQ)!YwhZRBc*`*9FtdsCOK(0 zUaUYSrMIb?jwvZ4e;o*k&E*xvf!&sZnv@v@J2Ip&b?VUuGzch--IDCAs^%KbsaNP9 zGOp=G&$y!AM7@`Yzq}Q^5`BH8jm5XdR8=f)_-6KITxM_38wL5)unuvO$J?;z?gl-x z^$2sF9kmNv`eQ?sdDTYix-i>NqrgpM{eLR)770p3L}x9Q!GaP<62NX77vBvp|5pc@ zJM`tKf|^{$moJCZ<}xZGk9KZh(ey4;Lo@D08FFe{o9~lj;1JHzB}vW|mkmoCN4=bb zw!$8F)w=kU2jDgI!gd~GJ1B{GRuQPcv%Z%rfkXY8i_En2mn7~;a@|5XBTO77*BW*I zgLYF4Ko#v<10YJK`5*J2tzw7MKVR=>Kc$`TayGWsK;3v!!iaGBuU(9o;d189@ymMJ z6Pr?Ie9!%J0<4d=eRzS|tdLYT)Be#L4Lq=Wo#(6;BYqkBTXq;gMQAh^NTT#OC{U0QPOx$H`*)yj%XLp;oI*i)(tf>- zwNY|G1a;o!Zzc>_os6zeyZ7uFA+0qm?ZPS>y-A35dDs=PG&5#|Fby@bu~6E&bt};> zCfP&HuP37gNa6nv*C{d_VyEK4fFE00v19+9|L^LTCE8OO?-J(Jo@6Vw$Jw-r{Kl3U z!hfkvJ+pwx#1g6bDP$OctMlId`|n#F+Emwuv7JnG5r#`MpTZ^l_6buMLRNDI5Hf27N_Fly~n>wUGnKkZ{k{>=2s0>vc%B}()`_IaOU6# zZbszycKkZzh~E7m$`Z<;WCMpq-o|~?$F*bVV3*Ysgj}7-W%*@OsOUBvZDOe|oG(6%ec%=DB zrIDw9YO}1}f8}1LrR^WwTFR^boPD*UK$#i$@2Bvpy=QFr=lP}`?pO1K5BN~K6=SpU zv;jOps$;w#rVk<$ApYT}7P5Hs#k~vf_0yxwSVvhlof*XOar3MzD?Ugy8;b>w2{-5RB7!W5vBBP>kX-?q?R&Xz}^n_WfIbs9%eV&rl?3a{_|W z=Zru`QyY|Sw^ddzpozjc`0*SIpV|oa7&p4kNXL`)l+Ha1#^5LcFZIC}$Zg&hwPWkn zz8_Af!uZQQ0)JdG^*B3!u~w2jub@#!8C}BMW#4Ou{=OzFJTeklW8RBZx{qQO_1`o0 zoUcEho&u`;5%SIpoq*h5{|(zMxy>A3>BzuW=WBpSyKX;irY)5%9@@k|{}k}8e9Wf6fc=fH-I;f6K$X|O22|sH z_lth*Kit8gEm8B8mqUxB&P#8QIq?H+A$KxM2pz9^vJhOM==zae-R$D?-dSw@VV`6W z%J5%$jqWSESRmt(u654BX77Osm05GNVml-6FVXdfh!Q-#omPIxkbUWv9L@J3RHGbB zZN>438G$PHCTw&34-X|{LU`B3_^FIn{M`%7a987Jc~Xv(?Zb6wNF{$*mg+ddh2A^< zZiz8&)doj8MP+(BjjmA&T%{|9mg>>g537D7u3Z3#Bk?nh=7A9Rlm(x_^+R?NRzlbz z?B`v{n_Sg@MF_B8kJCFb)VjL5DkG!t;YGc9amlOryKKJ&r+ePd4FxEN-=a5=8t8S= z-KxND=Y`l!L8!G==flygGmBruzj>(Q2Lj?U-_6oo=TBV1a2O4Djvlz_IX(L|m(EdE z2MO!a@834$u;YXy%gOT%LA0XDrb)X4WV%oihrH@(@y(2JwIw)6=&f7k^ics*NPH7X zqoAiVNH+wxP+X7;!&wL>)~l!;1a)sXVs11K7f!j~yXkTBA0H{N6fj$qyzD|W{RE$S zVd1j?z<@ewESP7U<6htw`1jS{R#=4tU4pQVoB`76Y+`Ku{^9Fa2iPuy=&bP1ziM(D zomtU)coj$D&|}ZdQ?(nf#M&n=svM2=Vf4JBh~QajR2rR%$Ycoy6^lg-*Wx`w-|r)EBu2eX{!_$#_wy^U6YWXVro2TW zSLADbThk#5C40n;mTT2X30QLUVUaZC0WWSC*E`nc)tAmI7cT52KP@FC*sI4;XzI@| zue7F19l^}*R9e!9K(Y&qUv920sII7Psb07Lb4<8SB~+K> z6p;YY?o6-i2f1vxJ6FG4ymNsJ=1$e+Q<&$G1q1|PECiwy5!)r%U<)RA==a|(Yo^qG zQ1p^fAba0H2uI<;iyznW5${%CgasCA8*5g)NN-a&V$5QWDZs<#Mp<0TV$@o5BGAp~ za6h%B#_cf^x7^ASh}@UE!p97CI>osG&t~&k9i{k}i^&A2UG`$6p?K&kdyS6bqn1-~ zKaP>J9;YAm-Pzm`mLyobnuN{SAqJQA zwUU=IEbToPJ}tmM-n;;UlVnT5W(QO~j7AM~8~5UnVh@rS&ZL+{N62H5?&o!4BXhM~ z)+N>Fmgl?M?fY=~dqLaZ!>s0*H?XL)-D;S{oItfrE-emkvMPPH@AJco)UTm?&1Nny ze)-|cQs>m}`Vl#|w0_s>wzl5d#(|C3-y5ek@o{<8Ymekf>s4FLZ{Ksuyjj97jyU%vO$+AI6%ZTWvPH^SZ!x|o_M2$G~uYac8dA0>T|PyL(c<+A1I2pPA=jK>ltFqqHSCaPsa zhN&PQ+{AU~Gu`ZK6n6##W8teZi{DY{(Fz8Q8iHz4NWzwl*Ec=!ujVLFy}va6{>4Az z@6=m-DUlf- z0e#R=M@LN<-Q0))?tO@dk$4%XM!hHeuoB~1U71n+?PFe3I$Z?2-9=dN1b>S7T~JN< zr5x)~Wz9wr!Zth&3&GzhCC}DvzMYOGN)|K03&l<*nwS*+&@PIleR^uGTCM-%k8#0U zwrp0UYb-l%#*11986ImFd-KguRz-diCQHYaI!28_Axy@Y(7V<|{R@W!s`Daj$`S_TWZs@`PV`*u*YiFu< z)cUT*#!nCTr@zdNTgoW7YuB!=k8rj%ii^BBu0>eEm1UpQVSaVvu$U!Oe7R!45(xJd zMUfqp5`OXC0%ajNn>TLX4m^_NSLWH?#9O7ucI#uV=1CutSEwmT+T}4F^Q03bSjg(o zZ{N-z(QMgLl8Xxcj^3l!)X4>o)8;hq$K-0TfTW!7-{X%=9zQ-OqO>9-uBeC(JvS6! zqS*d8$Q$Cn;NW13qU?acIb%;uWMw+P?$2KV5fYRv%Lu1BMh}6th(K$xcRJZ)jahS` znR>$!Wb`Kg^US`18UDv&Gu_rX#^kT)s+ztBJr-D@3kvs?5=^hxQNi(K`(Efis zdD6e(iK?f}TScU=V>S~L1W&K&03R=uT3GRe3CM^QO$Y>TtA@?n7k&&m4oe68B|+o5 zQ{}Pe9smBq|4P~USvP+Hwy|#Oh>%jdckgB_=P~Hgxwv=Z^>rmdMRE|7Z=4Idz;Tg{ z9HPhED+*=x#_NZ#0Y#yY+9WnqB>s0>Z}ww(NE;;aV)=-butwO{b?esM-nGyp2&Xt1 z2Df6+9P#du8dq)DkstBDwca*t=Sf%-h4o z%4^k$Teq&h&kdEbgDEK^MHOYZ|F8)idcfMXv#m!e5kZmp>Q$c;v**mYL&~Y;91gUw zLPsR}2HxP#w%aQ>%#!SOeSVy8^fb77Q7w=ombNuGurmFt*Y07%h8dprCoaA!-@j_q zczKtBm(l65Z6-O{Io2`-@+v`ddD7HVPc*U(=SGd|CYBqcw#jcpQyfn1$XczE*|S;m z=26&sm~ZxYB#t~h3`U3^rEc7xX2&x%5!lf*;keZ);qeg9GJT2b_F7i+w~!Xn>GQ;*>rr9E{u7^YI+_OwObTGWFnsj0~<#@NXj zxODE>!thwVeGdyKUB7-Ef?$VV40P2R#fZh3qESlNO$V`Fg9(ww2XY zD}zHr&*{Hy`l6i$)%t9s0bw$H%aiL-iSE+4{g#35g~d58N<(MP)PhIN|E3@4_NGa* zX34j1tuxl_(4pk%v4n)_4-Rr+O60Cde_M0Y3^M)!BX>nchCkwop259xheti{byDa} zm(E@{p;O5cdjHp7+aK-hH8lW~zNK%Xqg?Et*cKCxKu%&HW?PTQ8(e?bH69D*uH0DJ z4<*`HcCWcwj?^QhYkT|7Ae1KWY|Mk6Fh%l^>I$xY)pW7xkRc((#hzS7_v1~?%sR5k zZQO~8j}?~`@-m>^IpOR@-$IfYwxygLXTc3(Lg)U`@rqxWL7y3y^8_9; zN8(ex=-*d;x~j|w>lgN^oe_H1_Zn>2R|-K`=!9-9pT?S~_`7%9J{T->ft}LRq+UyG z-Phs2l0`mA2Bx}s z<;amaMQ$cy7ihi+{bxC?Fa>K6xlBXk3@${+5Vh08-P)CVA8_B$!+*@#u&1A~2g^c; zUHkX{QW&i|43*j||LT>yBf_qJEGXC+{IYc8ro8izjDl}DNBGcLqP0Oaw>zRtoqr)U z>jVFG9xvxehK!=sW>^ zTA3j+jG|A26*~G{(iTa$6080A6W^Q%73&6Q9fXOaZPfgPLUSDAh!H*hjS5vK)9;)g zmp-aypOUsy>eLkqd0OTs{oiBthi`1Rc44uld6U!)${n1WdR6x`W#a|!z_qK@KMDr^ zxPsgoPb4x9*++1f2hF^Vf|$T;u@arXr~y zf{MObIw-lEITD-07KRf!4nz{kv(*1fz15r{MPh2&jHuS25!?|uxL{i?7<<6ff-^*c zqzUJzbLh6sS{KcYPQ+KZD=()2#}fw+DiQv@?&ce%!q4%nWbKFQNzeU^MGrnuf-w6( zFG#&c&V{0n@1Y&J$@X|eu#?}*7GaeyoFcx&OpTn%)h(SIFi`B1KmXiZSy?%Cz$7i- z4mdCjxP*3eZxq%Q#SB+yE}hcP6K9f9RyQIoFLU)^U0n+vuWx%DV@54qdOq*mh2zKP z+U`+*l9dJfxQVQ6?vKv{D=NstBLP<@w{>X;Iy@Ln-A_O#cn|WnFElrI&!zZ@DgF^Z zXCX$vJD^;7?DpzwYTD-nlgS;ml(EVIinmJ6*JUWexd&FJEa(081v@A+rv2S|%-FFmOP97heE9IWKOIj#W?a}DbbaR5V5ew}roVLCey40@UGs0& zB0J*2V29s;-12MJhAw=5p&3anK{>nba{Vp=OSI~Z)Pf(%H2YlO?HKMhSttrzpi+NU z%A$aP03QB8sKTK~$NLQIRMujciflSa`>`ow5kfL37lZ%Vbx`at%x;rRzuKm#c=n5W zU8E(lIMMgX>MMtTv+rP-eQ>0C-sqlMjEk*pjNn8KA2ldf~SlZ6NBkr znd+|v7>K_C0eCWl!jM%NAC^Wjb2C3)!?cVxb0Z}>Y0{31v8+VJVbchgTI&VkhC?ct z@XCHA#X|&ew)+O@KB1r^QY<96EMva2mVaSoN)Z@;pcqpQa?`U}d#k=iSiv)V!;FRm zhw43%RF25V#-yy)9+B{fh|Cu+dTqIKMS9bL8iqd(9XzPTpKN8|vlzx4Ia2HNhy{~_krE2DlRxMg+Yg~OA{Dt=pett)E_Ju`zQey{bYdaZA zj_9g#tMTM-h)f^Jgw2US%RZQ6G{-!BQM%@R3HS9uqCUqrQC7ajCM)mHo8JElSCy8i zEF_@T*f!c7`}Xg@f%MYJo*6oYpW7zNvL;|T1Kq>BX-ch9C^D8dHru*3RSgRUw>dri zlC~5(BD8IfnSY&&PXu-%g$8-ZlLqzc&!neAeO^y6l@Eb#v8>&Z&6auI2gaYcg<0g# z#2&M4x*VO@L$yttK!iV27Zofqo*8RT3ahkZW*J7rm+jA=TS)xv#+QprjptSxAZ6hf zTezpcNX=+XWI~2Y)IcLsQxV%Ds9u&*0yjnIC?O~IMp=F|aZxOC`UwWLPWPwp@@Lp$ zx-sAxA=`e=3$~e(g4Pk1RasfN&~tckKgtISD02;4D6u|yJ*J!6TQ~m;6mlyP-J65u zyDPP^I5Fezc33U{gJttUaV?IH4*6d4=7?f)IN{3FTvBOkG=U>RN;-|o)yx7s)Y4{P zBvx4y{j#!lc17k*1gRjpqK2kCWYRS*(uo-=LCQ9BwB!AnF!Gvyp{?QiH!Wo)yjotJF1r0+RRxTRv=g|RWCxztyvuGrs1>@(xajxn<~ zIUbzYL-x91adU`QQ8L_0j8uU7}5Y27Z=R~2S9)fj&_QCMmMNR{KCA|o(%AcMsQ+M@{e*E^!xwT zsaTbHioUtEfW=Jb^yltBE|RFsk=BjdyLa!!sw?Js2N;QTb2BsTlrkcxuvB#P6F$k{ z{J6l|`8dA!8hhNxjZYzkE(i!@tTJ>g4f%V(US3Ajj z4Wc*r=<(u2e7ybszF&XG(#c`jknCJ!-x{*$SJUrGcn}P9I!7g+rgz~A40o`z%M389 zikW;;wSD_w=FO>#%=-0fpa7IU$0peOSDkzKVXqvJWal+}qwm#jU969`P#h3E1a`$= zZCpY|-u(0`a86%_8fbIk&qgMcKAHiNEKdo#};2}ts#kZ2E65iu3d!F89H4EhpY`#(KaRPuUpyL2iV`LoA5 zvSYBq-W4`|{LS-j?c#E)zTX?rn_QoOTd$RlY)G#TIS1Gy+Cv=R>-o-ERX^YvW)Vb0 z3%t`N?=f=$T(L)jAbLy^yhjuk3yhKm2d4d|-~1VL}US&jAlVg{kdJ+P`;idr&Q_s%eua zPJGS2t}y*pWqWw&(%4g-ez~kc*u`!w8$-yEJvco2C5aFKWy6OKT^*d}ck})G-%}S1 ztm(e;`cPI3xJy$|U!CzQE52BMB|z&nMI`)B*}kEwS~S^zXJF+NOUpplg>eqNQ)G(v z@ivOV*6H$afOfGDFv=*zX-9T2qS4>_Af5iq(65sV`1R#s!Xe}UInaUOk;4yN6Y(8g zEJvz$dpfP~A`bJR$zHr+e(azQ#Y<}SrDy%sbTbl=O{(Sg?iz{rs2XzX_ge+{n|#|j zmR(J@<0X%W$-&jysPx$J;~vn9)32^tnNmXELKw`9mZh)M!U}v?d!Z;r$tfw?W@b*u z%!@PKjL?-SH*2=??S%lXYZ!hP_P^;IOTOl4_0mXAilX<=4_)}QD!lI<_*YT69VBp+dR_OQ zotMSlMhcmRx`0kI?$#@s^t{{W`(+1~u4Wo+jMlFOYuFk zC|G-jN7N=11gkE6L^BT*u+92t-g5aF5Y54T$RaGHpoXT$*t|?Wisavk(}otR@K2qd zxb0+aA$+vc6M5eeqlhh;x70DMj!DJ6Kd<_mv4c)0{TZqBOAsZA)6em)_Ho_-Vg@yc z2y;G#j+ov7T4tM~DPt1~5TeeQU8HiZy-KLtLi?eNxG~#s+Jkj_ymZ z&8Hi`>c?f*8)q2tBn>8IVo*edPBBtfRPuk!_qlEqrkD9)X?W4yF4J|i3-){{d0!QI z)a=;94`Ll;h};w>$_(VlfC3Tv8f9vj0X0%@-70KjTy_MCgjOg}EINda^ET0m4M_0* z(pm`x)1ICxWh$)nfL0KNXLQN8Z(?6|FIhf#@L+w?XivP&%S(8it)pGtf5OAdd;{Eg z*7JGeWH(SEP}Tc%ROPL~-NPL~%!IJk%+hkW^`f2E1^{R^*EN`yZ$jyOwnGp*KuG)% z&>hENy$!a6hVEpb@`Ha6RoV)pKoD-})#jMk4l5=oCS0+n;%X<=-qzgB;T0vV8aF@= z@wSOkD(Za<=G_PSmrWlxPDM7032+NKt4skG0HFSx(t*)Th!Xd#pWWE1d&|b}mk#0= zIgeL8R}51D1?B__R-3StTOhHpml9k&%zx^yLH=BF%9GRd0y}`VR zPu1t%iutwc+bzS*^JY!kt2(UCt9|COtx{%p*fz6I*=e_LtGNH2jvN&5G=8z4pWIRQ zr~!hnqL~R6)Rw84!y(Z@c}MbB6a%ixGt09`5|0Kq9kws;^tq+wqQ8T8p~>7FM=j4m zg2k+oonTfK1bJ>+_1(pXCdm&cQ7PS-8+(_tU(6w^R9;)1IRY6JFR5h>=O(Y1FZ~J!?L6qckjOGz$J522@`to zI_CKChHQ0pFBuRs<4P9|oNlppTB<(|tzNCfQoyv*yR9=Uqzg;@dVjDWso#9LKExtz zg*aL8Lar78^PLz#x@c^adET{-#Je!>>10_cT>}m|!;V6!DX5M*u;3(I*T6HQ_ z(PxW>qqq9CM~_+_J9exu_`%7)xR7D3dv-4C0Lb51B?e!Iz=@AFOwk|0B^s?m!YqpRXr!dOXheWBfYLVPP~R9dhXYp z5R+JBGK&LV~vQB?|5jg1mzYl;pU`9seyH+A!YPN7| zK1UG8hw$dk_MS5Fp&J=g5aCWdO%uqOVsBg8b!dZKS*gFZ=8S;^4@MbjZ-$Tw zjv)SKL25^Ub}PtlN(%_s1_FNQtZtV=t_T;$*sH!kHUQDR9D01`tHubH_KBUj1pSA#j7e=VL(khECS zjFbGUP3ec!OX@#Ipd(!m0e)vqW(B@K%J2E(k3Y1wkP@K0^KMDXm|!sc%*T&)Pb}i# zHeR1|%CkjSHMGcjj$!JTF@LHr_}W9;!I=Cs3hp> zB<>2FusQZe$4h%{0K^kMid~L*ue@kB$Y#}0OAkp)vy!wo$hfce$0whRWnmvitg!rc zn=B{SY->7oM7>?RyDk$R zF~hVSme5h510x~otw&M?#NRww3hV71UwR`m;LQ?aS;8p@%SW0!>|=R9bfr-I{4!9S z4-3y)VRvz>y*~ov{HPE4SHETZ<}pz$Mj7XOH2Ce5^0s9;t07zv<^tR0{(6!2bYE)7 zRdO;IP+CiT{`&MednSFgW$b{1&^9zwqZJbH3;)@W3w!qOA4&YONSOIB80hamuip}X zFR$&E@Y^-S zUMd!JRTUL2V6N<9KhNZXqzc>U-RW~Um9ywpnOgwy*0Vl9gONEWp6N3{Yf1QEUtK&S zJU^1%6b-A2j@eXpV7uB}_)Z$`Yf5oWJWqRyk-3rDsmZ>EYcZ~LZ=`z3%YVrIckLUd z_ZmF79Y(9`d3i_dWg%B}xl?sew%+?wtk^*jVL3m@=Z0_bN?K=Gi_>MEc{X$j1CYjq zw38%NNKf-d8^QIC{dmRUtIO%8)8GiS`}Hdc`Kp?kqYAsjdvPzm z4{NdTffal5V`X_U7=)I_@}}~D+8x8!Jo@Kw;uFwoC_s5K$t686Wz|!XET`qKd`2Cc zXx?OkAj9ojw@zKL)Xk30(s)&6{3tIsB|P9aVH$3FJvTj#amm2(%5S|%ETz~4=IMiw z!J%7W(N?2PrjWD)-<19e{Zi^>E=<3|>kWf)l6{g{5+eMjZkNPHT`An zcP4s*nklQRrgUMARBPEb$f?uF!o|g4)q%X0mr+&AAEJ2R;Y1gf#@sig{NtMN;Q8HXzdWG5& z%E4`mkVDUQa}Fm}Y@4%ZrIrS5rU+UKzU+anO+k#pz*ww^SisgJ z*95fR{ikXb590~Ifw0p^^*teq)2F{!P5B8SLM2LY~r37CF#zDn;@aJ8E(%bNtAP$@ccq` zea9f?4fl}N%PKHI5Z~-o6{k|;mVw-`uaY+;!Y0k_0>cQGlxF^i*lP-v4C`0%zv0$(Uq#Ivf8N9tWTOVS#?bAzZ}`f1;JTy7r(!4#&*%#_+vy zN$?9Dl+WA0e-88+@$^+`Pv^%CS9y0eR&vU2-KtNz=bX4iR+wkHZR_>R+BZjCR?Vq6 z9u}r5e;OC`J`eumNThfes}fB0B4R$!Wy;NT0u89}ao|qH>sX31hW-$B7;Q0PO#ctR z0fy4G`5gWfJg*3HI0YuqU9{li+1o6Fn^-=K-Qhef|9piFmeS&s)1)>&+OJN*BW4k8 z?Q(wsz&h@`kyzCI(qWZM?F)nK*V2K#&{r~9aKYz$;VjAJ+7(&s=Wc@>uBgjQ&OFF4 zwQN%yye%gs@wZ>7Qiys8K4dwnG+yy2;>hVsy+CqtDrkxraPr>1?WC(a+jh?u#@s`M zX+P+@7_v)LC%8)AefF4h61lEGU|#uK{sn=l{ywp4rNL5K10mgPR!_%p${=E!G%Rlp z{g6~*@eka67_CS2vgOMq&lWer zHW>LF)7?eHFH^?%oNrO4B0U&QA(`e^%`S{OuOj4x{mGILuK`USa~M5Hr?6W47@-Z} zX`55KeF(WU)~&s&>h)iK$@_{xLN|X8+pvnD<9azIO;4Fv%-}0Kh>z@GaGJpCg6+It z_&+w~B9^53CjFg#378=-vSTd^nPzETRWLhi9Y)bD`I>>ZkB4m8l7oIWJQ9Nzfjd4g z!UC0);Jd=R6koery7K*U{#mw5nY4X(njkHP%-3U`Y9l3zM zkd?BO3Wu7Bd+bmJr^3j$aJsd5(0#OusTFVdIjM2lV-H?5V%Zb76y)SJml@EoP`9FX z*WelR@g@EVkCkp5^@g|h?zTWb>8fmxueCis!rS|M4e97`>VG6DFJsmm(4!=Qw1+Ra zSsgg}BulzcrL5zqRDMokP}9sV@!;k}2XGzmz~sKW$wgKtkT|jO`j?3NN8h}eC1Pfm z)6s~=d)@i8ag*Duj~pMs&qA~+I~Sh=m=y_2r%u;bTz4A=jBx|CGm8LvpIBhjR?t3^ zQ3{`EZ4Z{WRxmHWHF~v}S9rU*IbCZ`2&#ftA-~?rT6B2VuHO>NTZ6FKk6YcMAh}`r z#Lg+8Gva9y7|tFKbm@a^XtJT?YZ-rG0|2!xx{IaHZafU4?Z>JR?_1IX^TC=AALfPr zMPOZk--{aL2C(w-u>t^AAztc4i10rHAz_rOd@|2*wgz@pVU5K z6c7)hz&Z!YFB)a>IuR-qW>^2nR;gd-+BvI*6xfZSrsbCJmYOHMEvuMCTWx=Z@qn!W zh{$WG`r>7HFT+;`GD*Yej!7{;zg1*joZ|42<7C~ErDfr#f|E&Tn@1{uTc)RJ9S(Gg zx^(p62b1||EC4Gvz^iMQrahbMoA}@}ak(wk)pJk@sxOe{9RK*9%bD#AciV)=R#fSQ z={ZOe=*^n~VLgEgy40;*Q<4#8&9@%(Y$(gRXl~w5;(5(vIrWwmzP`xp98YTe@$H)t z7D=5r{6ec5c8Qw0=fu42NUTWBa`ZZsH?8SU6N<5%%a2bvhz!dBFqf-V(sclU!{ zO?jczhu#Fc=$1fv3cKL-TeWhg^8+Ted0Qz5t>4Bja4=!kld$lci!YA;2z^1UcOwPK zGBO6iR}?Ex$B!O6c5det+73(MnRO~W>6!^kG3erJNJ~R`l-vcz$<%VLNqh`D^ILDa zJ(n6IBHf(nt*@OccnGnu(fVk(pn(XS4`&*G$}sz{Jom+djMOh#h3S~Go6|6W5lmea zll3V%e-UGUWm%bQ8H?GRpySMOJ!1!SQqEqsWtex#JLI_|>&$c?uJUbu_MS=!0QwROHpYuT1bNr{ zlJbkbpJ_TpM(tnmJR3Ht_Y;4xY*wm2`qV5*$Sgu-B4@vi^NFCx2M*gq%^jwGhzRjsIyg328yE0uciTGt|wt9&WUPeuKH57?p7QD@4R` zzSUorMFRDvc>@J6Oc;}Qlc8P*0J%8^HiAf|R)`8VVM_%=hnqcWvLI#f{u|{xj)diRO_Y&?A~i_s5!+)o&XD zn`qu-nhYul)bphMx7{U&k)pKZ(h)Unf(zd8)QG?h4&^kRoX?9Xk-K+yhs@XBO&wO8 zo#(Wv%v`S4-{xSokX;~98s4(0L)J3l*)3bSJJ8P>dC1L$uVSUwc;$wzaZCH+PZCJpv+vIx?qne-+F0>Z96{B#%`W z9z^rDN`jI% zX7MlP`jjwbdL7Yvvij|sq~}YbX??h$_+lE9ealsa$7XVOao351B)Z>#0f&zqalPLq zoD4ic7C@#bX%dnZns#Z;+=y`T(2*E|vbVEXrC?|Wta?EMMAk<;LMGPQ@1X)RFTkBR z4L$Yump40Jzn&?~r9$+z#Q2sZno;j$5Bg)1SGWrTxLvFw?e~#U;{W556Y0vXHY4+< zi5H8Z1<&)QI0$XzMy=4F)t(e}(&M>rZES5fivYg%q=X~-Ie6j1cyJmzxaxVkvD z;KE|k2z24Og(1Bwt38fPKFJ@W6JZ4WrJ<&p>Gj){83s@0#~J7rbp$v8&JX|ElMEG_ z=VYW$yq+duTFMyv!`trh{bM*ccF(A%?2G2Z5lgtjj+B{Z|GvN5}oan;f%gqKn8vu>La5r_Z24 zZAD~7s1pKW^KYI!;KQiwN7Tuq7WF49#&&Zq}=3t;{yFz|{9Ixfo?(L^wD3x1RZ*0qP#2A%d9o zAT*Aai!D5iRGoVFUUdIm3!Un@vG#@O1F{c-*^D1hv&T$yO#f$y?;t8K&%%;g*N11} z;?#xGcSSXDUT|LX*RIDh4F~qd*5pt>r(V6SYJWYC=*8eDJ{vqFUAI}kjqWn&_Pu-C zj9N$T+BFm$f*O-=G;QcmrT#Vh<$&uN!tiJgI8-+ya+|27-T(m?5HVI zcDlA0&4Wm$i8z|w%O}3FNB0S;y-}v~en{17iN{q#VvnJwL)pLTN6D1fznSdefjgk{ z;RqBD!E>P)Q||Z^MG4PzW_KNiuyo5q;{@-b7zs79==vD@F~x-T%SI4jtz-vcoPYLY zXOlbg0R!@?q#|rofL#nDyOp0one5sSxZlga_s6A!c87o83C1)%$xnlk$@vuLiZ*WV zwjGXlQScl)el*K#iGhPdH(V9q+1=(168?tUL}P+amH>clgNlsngcXn>DCtVZUp{=; zU9wC=O4uKNY(0{)U=YME-`po9<;b~3eG%C$qE-J+nzq8~gN|W1+kNo-`(bF=<$7;{ z@7ylRb;WxC*=aQd{FVF%4aG%2@_ZpQTP|16PnzT%8gq%V-b^f@cl0z6Ehxe>@7rzB zGa^@vMA>i$WJCbgfv&_Z2_9p$2=WvYqk*pP5T*uEKi;z_j6wo36LspilLNc(-rVcU>@ ztS%OuN$NMQz~NqA=UmN7Eo!+msFTWfU$-Q?8HRC%T_c$Jn*e$W@{FI!0XaIHqV(A?~B$+>#gVWp&;fW5%+Ll*3(P5l;vapVLRKC7mFE}KmcjUW=AW`8S z)O@BBre*VUtuP(IvTl|!T{)odNf$q<0>K125_a&Pff-Ec^Sr&E=B~`U1c0Y^95kyR zL^XHDjrnobAE8%=E?gLmDC73dO9cVGd@z4#DWjFV6Q$pOnFrcC! z%dtN+E+KQ}Wb-8P=0UuO3_tZU3LKpMYqJdc4E@QN5T;bim=q@oRl(+0jBiu+_l^0$ zts*f6!C4o`i51kIo;$3Q2H+d&GZ&Utjqlm5TVcHaMz>fD7c>?8qM7%8`!{-WAXh_CIvl{Ceje{NJIPVdo0^>MD+*;>{{vka%SJ5Cnnx6Q1cdXt%##u{8}C*%l!an zfG~6Pw`<6jFouhWM+lkVM9;gOR_hYAUQPkBcsI8Fws&6{Miea6I~^d@!Rxo{&i^q6 zgTjI8%C5w%>Xsmkt^6?7Kl1S^K|%ldDpOaSk*RLAXC%R@_i`nv9@G?Tr))y^KWc!6d#xg7aKut=^9(w5oTsd8R66EYbX2h7)=h;DSS z-6!_go@MBRAHbZj2?>{qopbTIe4svh5U2e^R+$~eE zAacS*g*i>&EyVVGJo&MUd9(eZ6=`yqPi+E+DGk4gheFg`eY}UtV@H!>My0 z<+|Kmc8-qC*$WuEiY*u-;6~mt+j5loxA|~83ZDs6A-Q{66STFf5dtFtHvaTOiG_eQ zcG*69TV7@*8teVxSFsDJMrq-vKdM(`nxiVHxBc7@p2@Gh*-^irDqh zeB0SWB9OZ(0IMx0(OWn(yNSl2*frK(whsVMYg|20Z#5yDH1>?TH@=SiHWC25kkr$M zE?2wZkx)STyAOvchVI(Gy!%B2Cuxb84RX`(Lt06)j?KQWYLEAzyY$! zgYPLGYchW4^wSvW>ZW{zM zk%jTs#b)~_vn-VcgnX{RxMRn@qRm8+Z6cEsmonrpFJw=w3$9`3;| z7oRgqo9lszgzI-5zi%*y9qaG+(yP_Ip8yA;!4IZt!R)KI5N+1EI=y=vG5lK+{mlkS zd$dl&NovpDpRsZ=#YE&04%XJ21*KX>kF8~7B+d!vQ&+B3Y(H=lf;@uF;hlM}-gTxc zIYdLmh?}tV2aQtehbpRx4n*oG08^fR!oXOBQ=po;N45U2;vr4E|NarQN|;92qS5m? z<7@_XHXIu@kjRA28f+=KyKSA@VyvmMr=#S1^duDa&{?9~8r#=*LCr$5o&T6eLIJ4L zBo?b{*HOoe&f$a--|V5p#FsAnz>#Th#qy>(sZ=U%?Eo3dLwe*1WQPGYy|z4_U9=A4tpiIjw# z%5h)7I|}a)F3{@M?S0&VTxTLo6soKh=!3svx$A?WO1nFl7aZ*ySY*A#(0*E4N|N5m z;|K4#Q!V^z8x4)?X4Pe5L%ob3NWOnPP#vteNB$`a{6pr8SsaPTu|bDwL%*LFF5X!? zR&+Caq7?CXO-1fQ&rzMU&CWu21lX^Aq5qDBT0_yj$Yy$x7XcLg^|-zG-EqZkgln(C zrCzxVZ}|Gzt;4K+QWHxDx{m1R=qdd>h>i46{QiNvCoxIChWnJ= z;(hh?J3V3P(i)WBXr7Wk?M|3Y4@=hC%}0;6*$)QLmF=IlmZQGaMf-;jRN?Mh1XaTI z@-)Hc9AH&o;O!|#j~o#KHRn4RmP~jfZw498`wa{j5uOKO^9r{+x)|bn?zCR0iY{IB zk4CvrND^q`cPP+u`0UQc#ZeUmjnaVzP31 zq*GAi;4Jq#bmD}SLG$L#Q@GyVHo{u9mW;z3a!SG-uuAnT*tqi{b4{UQ%3k3wq<%-g z!aCH3*-Y6+D|k1!ZMG_wX#LR6tc$Ci48J+cnmVe~M_@pj8WbRZP0Mj)Jvx@BR|nQI z=j7$-AnctRmx%GUh6fy0{FMpJ<45^2`%{JkHuG`aL|k%)hvAZ6k5#E}J>^0r$0+>K zgz@9aQSBiv!)%$MDALp`pARdNz{4g_R>wA`(x%OO1ifRq$BIJzE9>F03HK#$psF(8 zth2GPJ&@hMzKpU2#C>98@da_BnS z^+Ou1p>_8hRkRuU6i8)Cdoz@*T!B1-g}&;{EMWpV+h_%(Jz+x^#& z%lxDF1A9Y7A59ELK&Y|5@#Di6&SYEi*W&3_mP?lO73nZjVO#Q*q9sh0HI(jMbDKAA zJRIvit7^2K$Zw|J_fP2R89^SAXdl$Fgo9&S1|2E21i3{J!6x-U)py=)@OtWq+u)FC z#Va?epN2OV$U7!xY&1@1K#FFhaHP0a=`n3{Z{~bYE~sO!V^+FBq zj_a?oe_Cz5pepG;;RjrT4Ma>@@Z%}~f}vkmEF5RvggwvgDOWoAyWBCiFhn8mszP)> zI)w$GLGS)-r0uW5Fu}|Bc=7OIyDj6Pf2@2S6<4*j*jwQ%*_HPRa}w8;6Cz%1*dv49t%k~=|k3wqU|{AQyA z+(W4ImJWIuy`?h}l9P`c6FkyKJ?QOGmr}3IUz4%nB6sC!jrnl}WRLM)7Mjx}2Jy2C zfJ;VLLf3B%gdXNp>Il0i0^F?IEyALaqX~mX1_nUm{rHgKhwvSBA*o8GMT;CxgF7RG zoQ9jZ+0*C|A*-gpE&r+XO+c*t4S}PsN9DZ<8_o6rdnOS*wWy}7@ zwU)QncDS~*M*~y)R!vXMFS7B5zMEeYo~V#O5DcES!wPjC>6&!5H+W+t5t;0y(7!(a zHkdjuDtv$-tpNj;r=*PlJ#DUO5{N{8yn%iJIN>^eFtRJs*@~FPTO3Q_MT9v#njsNf z`Zdz)QN*G#Kiq`Ekj;cvUXd6gv~+dXH*@{RHH8A8z!{(a8@`3aEcjiQ8qxK`(m@ojZOh`B8|pKt<+^Q;4A<5?S)Ou z2u-K`D`Y@Ij#Ws)O6+9LHY9xiy@$;5lFUwyKY^OF6B3aD$H#S(oP9DQHpejs5ns+S2#Kc5}7ExNKG4jFyUm9?k7&@*sKuQcSF?v3jEkMs#4dO5WFzOSqsvC#! zOspmNyrM%wLow^d*6F93b{eaR=iVktFj1ckx(-XoN7wjF1NbSiQw14H&T^I*5Ci?Zx+UBUXvnLOsYr{El0} zjKfOlUPQ4l{%%G_Y1*fz5{klEWQ(XzG}%rvQvjYQz*t|AN)iA+aB{8AO7m`KZKaIU z4g0le+P6*9bCGuuLV^vz4b^ByGCW_*%_EZN;lo^y%0dgA&^5F5&C#j8iLcya{u28^ z>M~D4K`B>%r~O!}?I3QEpJGGX@>;QBq=)U0sH}c4^)e}hU+Pjv3pcKR`NIR=TmGG~ zd3$UuJyH7hWGMxF5ia+(x!4~tq4&!C&lxkQyzGt*$IhlH9jEm4s)6$8^Fa&3HEP)i z+{Mw*J_iYZ`a7-j{<)(^kG{!7!xS?(H?IvGtoUhZr-d?MVs^$f8@>prT4IR~_*YF8 zZ=a})W$DSm$t$Y+-lM81^ih*0Ss8Q~;@eUdgMuw&8Kw`Q_b2n+s_iTG>WO-gxle(z zQtJi4U9jM!{S}YgU(TLAOJ?+VjR-1Pn4UEYF}N~rc`&;3X>=WbH0vyf76nLC6u>Q{ zt&!Z2w=2GqO03pd_N#~c=h-z=az^^ML9S{^mw1@5We50I=5};m#Pcf+ z7=vXnzQf%%UDAvV48pU=%)1VPoC*SP>s>$EV=-`L_qY4}{{7Zq^j&PaKsu%Et46Cz z3$2hye){rde{8HR4qs6;5e8^jpt0$;8LBRc?g)1PB@nw{Kd33BQ;Zni`W;lm9?825 zS11;9XL8}G%RJ9f>zyTd=eO+aslg&9KwZXy66=qmCSaWv$W!IciXjuFInwRqJD91D z8@0skmu4ncs!E`_=ZYpMCbvwa1E6i;4)7#^F|)0W3<19>d6HF+ zJW%r!mQ}f>;5hGW-4gh|1VCxU^ozR)GbYecaSDm+q%UEJ?!*vXvjGbMp+qd8#C@~m z-d96!5_qo9IXBDA?fv%WJ#c@N_|{(f;YC3~lTXf2$#>|;iWBk%Eay`nc=X@J^)lCK z!fRoxh&bSuEGaztoF-U8{xAqQ+Kj4-W|Nq#@=lG*1vwC4?T6>-5kK#I`FnW>2IkD? z4$-xE(W{n&{iOdyblZq-J^g1ofv()oy|zWrG6DB+6} zedB^ptm4@aX}7HMkN zug{mdK0ohJBLVc)i#i=1p2U2$)Na?|;JOE31e|ey1?%_utj7T++04W{0qV&0H0BA#H3w}lT30}H> zY_9En)8+;S1?|t!X<+bja#!`)efXa8BC2OtSiCpSdj;0zT2-Fcz90=fsz?@C{B9Pk zH@dihYv3qe)@fd}M=xA>)QouW{xz~%|0j_uTe=E%s_08;FU%V18!#8m{2vq04q^ke ziV$IQc!uS#q&CT46RCQK-C_a%J|Y*2jns~2hI=nPdBIJ?7?AscRI3QofbG7dIRebORjE7&NH;_PW~euuQ1rUmv7Oc7^3OV!7Sl|T@;Q&Qe7_2{yU zKq|c-GfqFThEms=!f@1VHS;t;Iw_09H3a`jR}dM9&YVs}1;G#~ zt34WY^q&@z<%B;INKQZt%SlJ4j5?OaZ+RskVX;XEILv{UL-d0vRLXGe25qDWdaUjx z!lnvxOA_9#UT9@CdiBVhE?8w441EcPk6CkOM*OSk8R$Ovk(Wl*KZ8pSs>#0evZse7 z#k_56@aBq#E2KmSy$7?VlTN$7jTk_`vBquZCt|XR&^AGv|9D~xIv~&euV1T0zp+8p zEA}UL6zFvIs=ACy9&Nh8hj+ePtH15~H*a#joOavn(I&R_X60uQeO9GSUIMT1A?M?g-`LE z(kN_%NSAWIIO7`F$BQDnA#WU+oc;Uv=dwruU>(w>PD(TO7vUxd(^ z#EFu&b11%Y?7@bwIjI#AKXUq!Ra^altK*p|yPIo|4ihr_U}g1Mf`w7@1gJh&i5&$^ z%HqIT_ooFHyb;;Z*uqw4usB+RDulPUOl zAFE)oQea2vbak??8LoaT-w~I4yw(goBvt z)t0FPN3@m8p+jgi#c(`MyJ)^pfCtcO@mFSDg0K?1H6cx$%PO(EiNavGwpzVFRbaJrOo0wKUdRv)79`kvapZQFwC zsvj8z-9ppf#_6asBu#G7p6yE%Y-+m=4)vS2i^x{Pv$<>98I_#)f6sSXem=rwxM)c* zSxX`*9?6o|7rKx+mvrr#`|*Q}c&s$)1thE~HU+HZ&=ZR%e7OpC6J+4r5^9Et_n&a8WpnBJ&YnKK6;O>eanoOH*+=-1V*)B1yreh-bb&STYhqcQ-Y^Nf zjoa-`B!~x>t3vUo>9z4wIhL${cCU`@_rKEE;Uh+TF4~9-3ewG8!F=WoS)?8?6R{Xy zBO%^y9|kezOIa+BT6wThiD*CNnXv9RMoE6$B$^M7@1j@+YW^opJyAfSIigSE*O%PE z-6>pRvgr8m)c$W;{p-A5<85VkM-zys*sxo`(gM|?2c~`28gZQrAIMYka#WU+ilf%l z$mKu{u_x(eplsq)<-S*YNi;A#&lp^1=)XDn-d^F3&bqO^v83bbAM>JScNb5gLdP(% zMIHHQS)qrJRJb^K;Qap;;zh6C?l)2;+JtO6WT+Yw-SI!8I2i(a!l#9D`_ADzZvTM=y)Bu}x55l5A8U--!xY{u**5bD0%{2ruT+prwtJ z1oVG*XnrlHul@^b@@o?|Ra!AL7OUy9Qb9ROBob#XUFfis$R|Nf{7R#g%P+%93bCse zauZ?5bsB%_L+z>Q3l5{SpQLZA?31?E9OO951r-(OotiHi&i z-p0rBn6fQD?{MxcI6g0ioX4Up;?6>>>hfm^vVk}S@WxJ$8xT3h$I(lEo*axn+H z!U41};O@K8s18Py=crDy5hJ!z@I>!J2)B8rBc5hv?oph`X~-@^;oz@9qyIqL?6YAb8EwiFvKCnVc4JM<)Q8fpEfH}%DT*5$ zxSv5kh3q4?JBgbCdNE>V;4xoPdUa&gx9(N>@m0%bs9N3I$!(7cUS3D56~r!M7?NDb z+?o320ANgvZ|vPF0-d(T66_0mCS5ZAk5tMJ78b6OR1FlYR-Yg%-@kvqz;8U?MN7fH z2qx-OR#rxO>WX<>%%G%UfPK9I!&Q{204^vJ)&1>Imc{TONIAl@F8$4sSVWoKfT5 zT^mRm{Bh@9H71@V8JRzV9zT25u8!e?W9I6`yP$WdeKED=fP(!$XRz7B#;~L04xp}h z{7*vv(yd(^eEV=R#9&Xy_PdP*H3MKhsNFv8?His&YRA|TnCWK$XmU#2WZN-#dKS12 z9O>x%AoHXCYuBw4m7Xo%3(T>4@j2LXJb5Oz9F%`|RxSAO;d=9tmF{fQK5NNClZ<6~ zs>St|vKwCq`q)|jf%)37#a8b*wwL5y(E)`-Msu$Y&@SBsyTl1qX6rHF}54T zQR9Qr+DS@L+XmMqg9TIG9+28uE4nD6X|?)c8f~6brMI(Hn$-Xpbh@?3VJtdPP>S>j z&%MOe$cv=<(8>{&tQ-mb4z0QIO5zn>$A^Rrkth1subj;QJ+}q_lm|X#cS3*D*79d* z23`Z}Wc@gBeXy%rCxb&{FGgv%zE;tDtIG1Mlq&Wxj*fDi>xOr{Ww>3;9Sz_=|%6 z+wV6ILVa=711sS7M{mc4Xc4{ogu;vi0AvVYMZz3q6QBmkl-7kLzqM=Sq{T=NsoNSn$W?Y?ZWH@{rB zL|UlOsTyC5igIhNfOQ(CogRYIjILyH(I#C1Fn9~(Hs?W^=EKwIrA!hDj63{$wgbcg zUFI7)`RwxaG(^pmet|nhVpF(F-mYz`^MdFU>{xZ_KrAKn))B&1VLx>q4YSL|m9+6A zuC|{RG$z(|UAlxg1fn|rNYDz%TV~U|Z2q65Sn-?F$}&3k?z=4sR@YIdOO}`a*1Af3 z@iMk3A~uJA9^czC|s3}x@%2%1%{MUIef83xgBTe2<;>+f9;Y#94 znQeyrt!uHh>}~BL$5e^R9B*83w!y7C3nm4Zjzh2{cf5lEuOCzLB7rNy0y>g=F07v{ zsL;DCuvR=Si7UkO>qbja0+9_{;YaJ#aVpDMiWT8!EZQ{PhKZmufbQE_<-&9Q?OFMG zXa_+ADWpGDw5bzdl1$3XWvt2)v8yoebVT#STP7;Sn!U9(+A#cLzC8>D$@zP+m-i`~ zZbByTP~K{IiUc6h6;c^|JLq0Dt7|r3;afuY6&3fm9!Rn$%Zf0P{b zqfD?4=h=j5e60fb)6ZZnfd7$=*|!;moVPg5VvpxR&=?3wh(B#xl8a}|EyObtz270Q zwO{z(7JWnrv>Szz&suK6+o`EL38L!O-uW&GAvu-p-Rrtld3)Ggg0?V-8+P#E!3{qy z*mE;e1AhY^&EP5YIkQoH9G}CVh2un&UG%fQ)tDbkq+F{{yleS}KV* zi*XFD1{rB=(EVzTN|-H%@Cmt#Yq@g+iLA+YljHwuOWh<`_UoMzsa${g>0fCn9B}3Q zvXJ?65mET0sHAlJPfc9B9-25*=@K2F(8N8ok`~}f_%5|l7QO|b<{1Sdp8WMIbfy(! ze63#(e!pLgZ_ebWJ3nNtQeL{jS=Y;^f3G&(qn5!7^F z^jCCi;*xlv{`fbNZ0mEoug~tUyqt};dkhv$S#No|H@uCc^BsV2v_i! zZ98{XJgM%<)x{4(T4NGkupY=*FR8YYt0chyTn4)R)4>QV;cV2^u@Tuf`mZQ`Ld~ z01o*e4tZko)dvNvA9^H;=v}c-bI51N_|9u6QLjAV<>2_*%g#Te8Vn^jsqRLJ)9p_( zTImv2IF|TuGz>QV5}4u@8oY#jcO_|neOf1 z9Y(`lQBhIlyG)F>$;OzygO5WI-RS9Yy9>~sB89oQPEjZ!Ve;sR47AIpFDkFiJ!5Xm z8+M5hJVEg9?nbK%OM=OK@9}e3UVvK^(RSW3r%s%xyQQl~HuLqFIhQ9TG6ax?G*|%) z@o6`Kx%p}<1qb3@MLk$-aR}hAcKv2usX99C5011I1+CL-{tHG^r|yH(@^D=`C{QlG zj2LIf#8L(t0Oxgk&<3`K}J-~aKGzrm)!$9>`rpiSFa^Wk0``Iwy)VnqVV}2z1zjZ_-@AM ze%#oP$RClFy-ru`raxC#o2NYKHKAj?0T|VP<-)`#dpx*aSNUD+60$D#wf!`ArIGRA z$yO;d@EaZPKXv2AVRY1>J@7u28;r>dN3eJ)QOGdrJwJA|QK8-H9f;KlYLWV>XJmyi zMa1|L#M4Gn{SaFv1TfY*ysaY|&Uo84kp#tYJ51Gs|<^e{^Q$&d=z0t$DL% zH{ProbvE}zccax+LTEsR-P z?LLlYu5$AA17&%B$z|}2H5)n~w-|uRwsR!@`zviZ z(P5zpo!P+{2OC#XSbm-a9*J$0pkGn{ zleY0KwYqXLA%o7TSxq^t2;cd}v62t~Y)^co97{-Z%7cLzWZcIcc7`8WW0d@4OYGHY z$2U`6=f9(hsUMxkT)!7*@xq%snnpQf@35fw%m@{DPqV-=pOe}lffD(G%p(kT_hXms zsYwG<4h}awvPWOnc@A$c9#-w(adedadG1^yF}K!}j92T515E$^Q5EMv(sxLHA_J|T z=)L*GFLv!|Px;9h;h7E+kO9mcu$(iUi=FlxwS2jBa8nX4J$t-h8msX`Bo^jfND1|!p>u((H^WTv!ZR;q*zJfn7WF2hp7W|>ZwetU5V8hdFsMDgL zhXJ3+iUd+?0`iuQ($oJ}xi&C+eL)^gOz36}PNxq~n=7rNL_Hh)!(z)6^{%S^zYjTo z65F5AefHd66O8MhKQ(JP;eV@#bbg$h+(*6o(kB0@F0qe3ekk1l>mZSoR%fH5B_$P= zLM6%(2@+=WV^y=fa-?emn`O%k*=Y$D+Z_{Q03w%~KQS5#3EWOwG#XF&EJaDL5abMK zcoTYJFC&K>!0VW|aIiS~YN_jUeaqf?yjuw=14Z%GWz{>TodPnE$9fAJA)uW{Hj;qr zit}09X2Sq}JkV5a0`~zZLI5lxyIHqqNf)$#j6c2&^9f~?2YbGjdac5brRAY*tt9lI>peofBN#HKD?|hwoEFwC` zb<$i4&lNpoB)5bq@b;0}r8)pSK~BLQRS(cDJ&@kD739i-U0(S9LxDJgv)}(40#Suw zUUK2Nzi`uCyvCM^_I85p=Ze8qpAQ_>RQfjJsQE4Mgd^w6{(BYW`tZntd61`? zeW9Jxy5VDx#&Pn-%bbx2+SoB2y-X;b>AIXI*xR(WFRIXvr#bC)vJqR%vO;Yd!7?m zd|k{ZRI z#mv`PAAVmor7gHG z1w?!pGgqFTqJ^A}vpNB7&GX_aqZ*(?5biQSlOY7?|pg{_AvHPYL8 zXGR_M?sLojY5L&=Gp`D_CW==DvM$iY;XA|Fe?>2^XnxjE-HInotJSXdJ!d$faO29_ z@e>{A!*(1IF;$S3&O^6_$9->vwZICbp&_?7b9#Qzh~T;Zs}iD*+@d;#{JwYyCs)&d;pt2d!)BINz38aYjiQ=j-_SnG0Ms zFyB7xT9vSAle(|Z#@d=SEeZjw_%dOO2`nHOiDl{&S@6bFfcsiX(&|1ofj9jtln!5v2v!$LdS5;&z)liS04nP$bH^QVZU9Fs0 ziUSYgrW_)BZ4sC*kx_0je9H5br-g+Z|{$*Y^*ePulTsjv5@o&oirQ$KGdgJC+3v<@V@ zH2M5rg0X!aksBjGfl7I@=oOD!<&w{aKM!!lABy`T4#)}lHI-DQtZ2Jj@izjd8gFZ6 zt67+T(2j4_rem){B zZIf{PDCklGcNNa%eowiac<0FF@xUOl(M-L*LExVi5~ciaD%Qs#EB18Y-H|Zs{P6fh zI&LgRl0pW+N>8ueQXkEeEDtR1=DPiFGh>N1KRz(y$omxlR07yme@hy9DX8;3~DI)i4a^uGI=uNoPr zZ|+xLy|`If?hN<5<@;lL^3!D+NfLA8=`&|;K{abE>g5Z(;OmWqtNevVycBt0X&Kcb zc%HQ$;#_1G#9OaG?|~(5@;wf>g^khG=oY%2eW@1Pn(u7PSL{sl z^9!Q%2a8Mlxg8_BeaC$s4WK}yoJ+#S{a_x>8#0jBfU(Hef!e}8Lvrt;iWa&8bkbxv z>>cOby9(`n-pCX*77ePywsLZf9&a$AH%~d0?5F4#grpR$5 z5wb7;4j$!_VkhWpuFfK-)VfMcPTVoP>SM@BxSOr2lDuref@|3^i?Jq2VM0SABd=-p z_BxWEi~i$YRHludRK#LZ$ST_BOs2%YCveVJPw&P`!Jv_qc>omAsK z^ht>Q?Ap&Z4mAJYKoepfnO%~B(k*A3pB^$iCSGbvM1P2AFlG&Qy>=ueS|L?;3F$VA zV;_YrdvNBWd5l|tdVT(@Oh>(VX0t=?%BO2jUNOo0Sx~#bhpDvt<64eh;Lw(R*G#B; ze&g5{Pq#-DbW|MtF4?W$D-M3kzekwW-#XjVd|vk9b;(DN!6x3i*jROuxo<$rQQIuH zx7*sG|Av89O>B)mJektlIlG`sGIm44*vVl_+`_Z# zkqskk;9A*EQS#Bc?;2JuR;z13JMK$gkL%sLchQ@jUG4{kU zrRcCmd2bVz9x~%@U|)5|Mii+x;qJtT6z8~89JCScH| zp1wqH5%Rn5aZR-7myacdSUhin>1wX!nU`G7KTFj0E)#=WQw|78APwf0YfAt59mlN6 z*{CQdOSOE)7scXgY;5opZ_C@Q%|~Qy>SmO4nS;-@vS?h?-1x~=$C+X=o&5PEDByrc z?rYxLewHyF+_`_H}X{7di5l38V&tkZ7djy8RuwvKDCxr zcRtPQ*H$s_4*;)GfHh`|>|b5TTQ!)@hg^rakBey$MG`LE2~j@t&y=_X762QFOfBkQ z)mWbFpE|Pn6<3CYM)QVB7L-q_q0%#IRB$6%k01M=(@@ooH!Xi#u=bDFZ%0*Av1xc( z)$Y~!95=lQb(@`Oa^uc|`vVmUvz(?ejmfOVxaT#a-txlafEJyuj%;>%%-&(#*WKFk z&Z*X((Q|g8U9pa)dxO|U5%mThtY2@z*ab>k2KJie_&oq6_P;JCo(s}feET0LL!2NS zETVnihIOH-5Ma3j({~3tm9P=`G{&B!*Py^RgkqXGb0=scK$*I(Zt>X*XXjmQ14+gA zzSrO>KPt(O`!@F2cET;i`$E=*)miR)gvorz+y@>msa#xCc<%wCh*&2rxk+x7^?P20 zkC=Un8;MVEvp9eepe1_#`Qa=_6Flp&ncWtf zjWuFn*8Td_`lL}#@{dx=?gu-B-&?(U^^}=2dzM#+F^>}0>FIsy_REaaeMMK5`_^N? zlh-6kvgB*3?X1lP4kHfa*fMKc5d_7LBa#9ehSTF$fpb+=hNt5}g|Jejs&kfy|F~SE z$5L?7AOdXWB~OQV+7uyw6mBjj_T3P*?i!)dxtZR-X=oUQX*k}66JgUmFxRw>5=JCR zFYjoenrydf)q`xONs6uzn}~c?4DtbXa&VZ;JC>FOGbGj`v*g=t`GQ8ZaE-4&5}pV0 z$*;eR(4x#fecJD_EDL!_u{^^3ELkwMox?ZvBPdB=aGGy}M)bo27mW`?k$Rnxk)c1- z-o)t3P^wJD>&QNBn)}IV>a=MsrFM<^S?A=O7%9Wz)SqArYA;@^;fET#V*yID2`V+P z04rsLk~kpe2z=#j3vM2n%>Bl1WYRQ-hNpX_zEntb?tQyJhOcFKUd$*)KM&y~djM2BSiDfte_1rbZK@T*+PKJnQo8M%GZ| zM&eOKwUR(xeBsHFKN|LWa-+KJDibz75+y#_YZt2bh{+kn;o#_6X^mw`nKH#)Vhkf; zE?Fhj1q-$28`ZcBnq*>rRCEuDq~4t7LQg;wahfVxMHr6m9=9A`Vh; z*6QkI#?9cuSiN(qzQYjcY96J$rSob>N6n-j6iO+=sueZYIek&2(Ys^o_{zoec3a+l z!|?Q*wvoFe@Ec?XXZt!*at(UqOy-1o%`lzewXE>u|{$C;A80&@_kXA znz<={?7qC^+3~}A`nqrVQ=|2iy$duv>a2b2-?`-e`@t8krUqZQnzy0qkAml`?mMsR zIpqFd119MjJ(kbj|sDsh=b1S3fr~jT_=)8Vy`m1BRpR^uf zk4_mZJFaMXBIXG!BUp3}>bv$;SI=@D^N>=n=tV8k4Ej62)+EL_xa)x1`K5~T_WXRm z$E=DH96soB&qm6eXTA>8r#}w*_G8SOlcYXY6Qof4z9t{P*yRn{4o+>jrNx6L135j> zkp^R&#gt~=z5Mc21*KzWd>OT}BKGj~>E2oW`Lal>m-H?Fv=uE?wg-USD6(D9`AArn zW^mxki75!X?!3`VYOTJU7iO{Suhv}e*@PDf=dVIJzWm@ zCiG{AjvZ^e9;)mEiYBv_q2cwQ!{NJg>&3tJ!3Z%!_j1?5%JqrjgYfhFpXkO$JylnB zxpG!*&8*??g9{gU$F}p#Jz{gVWtVY!x z*Q1Z__ZWTi@JpYFg>3_Ngy=uB{KH&*$mp$KwE8|&k9_b~?(3B2pp&8zLlpPAAc=i5 zg8jqauWqUsYVz@M(nW}>HLxDvYsw-R4c{Lhklmsp>o#J9W#aT!pBW?gziaqaU5bo6ia!<6 zMseN1?$X3Vqz|OU1@C1Z4XAsIwf?wfs00=DJ7lUGTv@K+EFHuEqHs}GXOTMATy0tw z08lxpYEE&JP4&_#3Xoe^(NxTh&lAp$aEdfhz9>=Z2Q(ED4FJ@%O991{>(;Gn zp{g?VPxzEuocS7&d|Q`ajPsyDyHh^w$Hwe*b@hKxYQf#S+hb@{ayw_!A$EjWtM9eq zsgP6x#3a5O(D6&J7>^ZQ18H8I2pZWCwtGirnmMK-jFeueIvLF`uX;0DRM!ge;IS#x zN4I@oF{FkIG`fxzWizwGcv1OXM@O{Ot|Qx}{20PwMyzG;7nx0)J{G?%df8pQ(OB2` zE}X_&URQ_L(}IHD`1RxF*(utyJZ;uBY0@N%qeeL?)s61GDVCOdP(qA4VYT((0xA35 z@WtlZa1EF#$qmzPy&ZqcR?1SB9l6UYu<~+8PR6+(*H!53IT@gD7Y35B(tarSKF$l; zPMD9P%+#u5Z1e(Vxz<0LY#aF&9VjIS2M60{T7xx0F(e|j&;3;0v(dy6AySbe_7Tms zu#P-`R=6|&CnI=D-q(gsx^EJ*9rSsFysuP-yS7wGndQ}_m& zWzpvF_iM1bWtX$wiOET-?=Nl0*)zsSxBc4k{{O5#y6joq1GUC`K3*_(f1BQ+ksc54 z`*?Oob|rKy&$6e)Z7BjvreB|lSCXAtp?>D_fp#x|VdzHOk z|F88FB#`07dSk>pg9@bNg{tqLPi&={LJ_@9hrrX09XlqZH(f7h2QqkO{aY`fK_Yku zGczWCsRgMBl9zQlJg(*Pk&_S0TU!Gz1WA%E11kwCG75?UoSLk;QY{8WEt-)T;J`IC zSu2Gn^yXC&cQO(46o?u`_f@Hznp3ZjLX-VP9TOoltyD8veFEl&0T3|M<+_#@WQyjZ zvor4Q7_Vo~5-uS+Z=d11P}*SxV`gwWvcdqKPdIB;6B7%YnYQ7$kRJ+2=%9 z*jwJ7CQ+_Ca!*2OCp%r=Qnzx`SR=gs~fAV@%d5p#B(FcALlw5JAw63B1qYvB2Ok5rD@{@1Kr*#`!WG0lP zCh>_}E5CP$u2W1@Z>{t?xH(i_aSYksfpjKO#KybMTm9;z7xM;R3qRxA6%Vy?oMJ5n z&J#DgcXCTvtu zyd*iliht5v!e(k0T`$%O%8v|nKAh6SGj~{sl7U)TR#8;_ybUYH|Mus3-Pt=^r_I)A zH(e2I6f_XJQJOghyMF!H{@xPDQ8zzEn=zA;u<-5f{3`Wj2MN7t4WDTC+`1q5OZ@jI z;c~iw#UTOdsGMT1yns|VP9>>}fpCgIHkp3K8PQ`cCh5fL%GB!dE4>AdV^6+W6=ubw z%ITP$lk8_D`#0Gl3jJQP$LPtBJQ9<~xDqjgJq&Lg_HD#~e3v=2z@s+xB-p_-5+GVJ zhwaL(#ef0lB#l64AdmmxeZ}CZ+oQ1?GH>4^RvrVj=2F1!>0z^at7dq5V}3^l99pMR z_hrtak9O{DhKfv?#7lV*<&}H=&87iO@& zQ!mMdEuYs^i7OB7bm((f8FwaW6I+JgLwcbh86N{YaxbILbFVi9JkKS6$pDUe-y#y~ z=ftY1q)diAp({ywcO(NS34DDBxeCZfGF3ZZ-=xgT4cqefd6|`We~z5pSFRa`qPwcz z9<{KH)bDh4qKU~q?WQUEoVBmrs)zGc)l!>-5tXdLl;ze9u9Ykgvy4eVYz z-Mo1J6#oJybTGQ2c%6QJ%lpk-m3#4vlYyuOzDCZ0_9b*@E?|SisoLm`KGkG*j~VDj zxck}oHh!p{!)1k%SLzz-ZS8iup{Hd^c2O2c@4nX9aA!+Z%gVXK5qqQ4pGRJ=` zJbLqO;ux>SOLk8FJl?34)#~gu&m#0{_0kNFR6y0fjhDVxXa#-!XP>%uZn27m^^|O@ z@TNCThm1BV+4lU+*5?b8B3*}tMozHSdC+axjGcpjWK;|dfp)-81<2?T$q9urWlZAq z>q~!Yw<_zJa^;T#x1FyG+UCne=mr45f57Ta-dH1}KS%}ssd@b;KfiW>rc!Fp0VCIc z?nGGG^1JSZh1bKXhZsd2SpmY}v-g`adi3(7Gif4o8mf>=`qjdb{`p zx=cKjVC9^jt`S-;lnX*&-`Ox%L@ECJLiH#>MjUW{#S}6nBi#!;Yu5WEi+&Gn@rc0I z_T!9=)dvll5vHdX0=ZI2mf|f?)!J$=ao>{b{``x-5yPZ%`UL-LlAV zeH_sF&w<1C+Duur_Dr+WZQr*zI`vNH6~+)S5}(ICtk9|e`)`$Em|2CQ(!@=!Fxgz7 z%|$eYc-yoN=u{6NHCmG6c@`XC{F%AAfqq*s9I_lG(or4-k``SUFl z-XnC?*l2lRcGU2q{rIGi-L1O%N|DRTfB8WF)+u!{Cdf3*445E6r+lAal8l_G&lWXTXgi>B$kx4yWgjC4cI++WKXpPaGfv+-M^S7CK5-;I*_@y{V&GyJkCT{#`cfnCJP^i4_G5lJv&!Oj**{qDML(gv*H6 zed4#~@y4+iU3~g`uLJloWV$OhWt^^fSdhf!i_uf+xxhyIzZ=TKBoAKi-WyLB6dPSg z0sKK6sGQfBD|}CGdpM9=8(0#k@xk^yUHKSI+UwYO38a7sapl``{ z1%9x-R%C{<;w5`Of7-E&-&QPXvS*}P(s{q0uRLsQe>Bti9Ej*y$2a}E?fbnd z$F@&8v47Z$ImxN1NzBT(B3`XTagh{bXF+g;H#?63nxr#w1~F-~FT1@f`Mpk41x1qKCxyj;#{VJr1f$h1yMSROmEcLX0LBf zWKOc6&p@FbL*6Q9&7ioR0oDxVD8A6Hw-%L+A^f66wHlXVsQxk9Y+=Qd>+Is7?eU6EszV_CI92cTO_6Dc$gS3e8e{&)sx&r>$1o?O_uJ*_7-Q_SI+0 z2ON#!;j8tJP_IQBLR)Aw7=5km0&ykIHy*^$0WW+*9le!LZ1e^@6_S0TYdyjEs4edl zn2hgZ?mAIIw9*j$ai5&tK zV#B0wzkVxcJFX*Stc}Ek$8FkZ5#0Fjax|t6G=DVM()CoqtE*)P z9P!K4S6KPh&5jSRW9W{nDP;*M@>^f6Z0qLw+^Kk}{h)6@T$!em=Z=pzIy>Vg$hftx zrx&%*uvVOhwXVLZdYG3BWku2j48wrdAvQC(eO>rg-+%ga;4xW4GVxMqH9gE)cK=_$ zG}<0Y_4(Rgf41?w_W^|qzNY53oEqhoetK2c;f)%v>4m0UR;RRTtCJRTd4;yPFzVzD zd)?5Yd3klzc5f8SpqrV}gCTDW^S0H`+t~W{o+n4UTKbILQCNAU3%+P-(Tac1D4 zLHF-A{7WwF&#%1VE8pp0?`oTm*2kDo5t6OLehfu%1N11VW_)aEFJ26Ncjafmvv%eqz^F>hIu-M*`m%EIL` z+H2Cun@F4rKV5#kE0VARB+#)>G!FAIw2EVb<9wsEfmti+)k>Sl*#aAo7ADbV@-MeK zi^8pQ=gu`&!W(nQI?fred{E$GGt7P^1ut|ajW_sxYXkLwO@1Rcf+d7$e9>iqlpuIL z&vSHk5W{8>67t6>Qnu@v8O7S9spIKO?n(VNo2tUF&5P7c|7dxRV;Uc~VnBpO3UiLN zWYiclP4*BzeI)~X+aG+9MG@!Do%9chvod%M?j&)=PMtBM-ho|yn^lo=l~ss-?VHO8lT8R$Yx;ac zqer!<18tQhZFEC6TkeNrWJf9{Y5<;a0=gymJ7N?j8T-iEt{80NI%>Z zW1Tgag881#&Bh6)kZ(oW`TXx!U3xKOL!vQf@FUusY`ulwX*R9gs+{i?ywT7HV1P;z zu-Es>^##`?!LwxGuNOxCSaQ$(rq{4BlP-EkWqoZN?pjMl#l7{^*)bsW!IFtTNRm8qLzE?1L!VmL_)m zA@L+%_w{LW*E^0uZ6dUj^6UA`dLo!2yDwmUkAe%NXx&!LX^Mwj-hyUP&5xcr`@Ki|qY$T?4pW0Ldw^Rne}Av@JS|kyz?F*%ME? zWT38PEN5tLdieCtoHsykG(D(_G-(6N5H)|e9rfaCh#NjO4u`@6`4uiB07jqLUDBo+ zvS0hFuReE;@yY|>QP@sOdZ$0hFa3$XX=b-wj%l`fwb_>!F{_ihh5lw9v|!W+tL}L%HRLL3WM~^*yXJOzyuVT1OB(*? z`;6XNaJ9#RvPObjtGHVRwvD~iu-%hJIY}S(wGAJ4afaslGs}|i-%q5`A**ym{BPF_ z%Cd=6)auh`{}6uz`C;)0OK{&nl4nF;GHy>N=VO6)pyMLYpbmol;@fqZaB} zWA;*cKjn78@qDm8;7TxCYpD?&^`1y^s43}y1UF$h>a2q&%{pdd! zQYWxzMT@kIV%o|S)?4bT_?AcAc8TZ7v@afEisZt;(9)c!H>pNXE*qgF>4+VjT5gY$ zw`DMK)2gE%dP9~_fHW2;?Aju5#%~zS6>1^lHFoP*e7 z4#l(L*rm@ljOQGF4TY`UfRaJ5h3w---7pc;YRw6+@@V$!{L&jsE_iyqS{YKD7jh^$ zZ`e{Vu4fD3V1Gz)GWI|TfAIH^rz+PA8 z>RUvuI%+0j;D3Kpr9h{OCxMou@b@ zi419Po_reIk8A{m+)J1|)|&H-YWl9P0N`ju?ko-z=Mo87S`azA_EYKypc; z72j?<&t@V~jEGo!INyi+D5`i}t;YFoA2fLq3RFP>URPJGkH}p!)PcQQRJtgm8yWHg zNCcg5AhjR@bal97?F(bkL-39vyaR8CdqkzG{eVBlO_Mw z9hI8~mf;(ru)}|vN}5Y|Hk1*jgM&_X!IY1Ju-T!zny2z|bML>11K18qPFe4KWMqn7 zNa{|Ef)01;y+T|r&Let&?2UBN5vR4+u{f{9>r=d~n|h-y>>?t>K8hOdMQRA$33wL0 znBD3dP>vxH6tGq$S39Vr%&B2NeSR7{1H_L5 z(Ae5uZTdD^J#@nX&XnZ2TGvX-4K_%K?A^S? zTJzuZ&2L|x8~mw1tnk6x9bMPAQ*UIdtjUvm2Ap;0{fByq6 z;O)Pb6?oki`BGj2*-u>DoY**CD{v4AibcVKd$(T3+$KusBV?mt@e{-eZ~%^*=lK_0 zmcI7tD@hKkD%?2|HB9X9BccHSB-zy!@jx&hjV zNu+R7Tvysn_f-OAlN;0zK-RS+YTWDcVFtj5>x3iR(#L`TA=SK(i(C1bc zp4aNoeRAf>h6Cm4_;?`|S&Bv}tR6MA!>kKTyM^X~Hq?si9s6k$K^UZ+{x8VyE>CNl_zKhR&|O+fL(9v!r7_zT+6`9M1a&%!|5v_ zozHEcn_`LjGT8Ke`n2AC`rKmHmWaVOf1%+KeHA9%{XR)(gUDT?E;Og?SqlKM&Ga~$ zC`50~I*+DjfxPtblWOF@%IUH3S|diN;U`0vH<0IxZJ!I|KQOY5Q!>4PxNqIsWX+qnMQx7k2_e~eMq>9RR1Itf}c&b)Cn7($Y?zZS~^m^G5)x(9~-jT<- z@Xx#d7#@1EHNkjbBh@CiJ)Z8!&}a*AM5p8pB6`I+dyAuPv6-e0ra4^xSzW5X)vsfy zR9LC*LlpK~*}ctrJep%ahOA8RM-gEzU)~#mVXYpPwUiL}q0z|V6_UguvG|en09Hg2 zjaNQc4#j3oM1OPR>y8qr&0}b7)KcU7g$^MG*cHL+&WyYr<0sWbF%k`YQBr16PqZPg zt0vB*(It{&`dG%aW1&=CE;=1@>OhkfV-vM9w9M6vP2rYJPfXKZ^mic(X4hpckxk25 zFRIXWAYq!_0^(V00LG4oNkI6Ra7fcB+$R`Ek?VQFl>GN73x@=qaP3U>)k%DxWwM2LTib{8FOZNsr+>&Bl{ zNsH0PNLqMxP``eA@@jj$kAzwk0Rz$PW-Kt?#@8j&shZqc@F*_{o^)1U2MFlybIW`AjSsGlr-sR*hVcv~-*#%WHc zywY1X9Fd~jGxN|xyYzE!@&CK%Q6Pc>i%8qn$f+K@dbOK<8mJjs25sa|UnYafID)Mt zwvj>H;RsQe64itSFxC5A<1hvch$$)BQS`}!$OOILZ{rIqn$0fLPO9$sx$+9`ph!yL zFRnWW@fqt#k3Dnd%t2wg=X~X)QAZ`u8E8?F=iE_EO}bqLCvDs&jLSeX`&5U77E=q& zRz4jOU4*ba?o^MpM`ZT$MGOlyGyeWW2`G(@jt=^`sTsjPz7a%WOrO4=Eq=5V+>0X$ zgcIVnk^Qn|*SQ3(Q%c{bb!QB5IKz}KzD!k>G_$|L!w-2kt#{Kj+#igrAtB)3h8eb~ z!=O>PzE+xnHmdxqFc%vlB_ShQ@N252ix0vfr;*@&+dRQw<0Nfe~`O5T~ zt~?R(O)!3=NcN1a`9L)lkPSyB8^C}%lB)p%y+S58DhUzS=B^M3FE)e(FD|_ z*vtS!&Zz;xlGmCkiC;?Xu6pP)hv>Nyf6tbdkgwPCZc0i5KQ-rxjtwsSM8G!9;x6IG zcTl^vN-_(hXx>2>wsmb_evZ7l&7$7n@Ex!^#UJgp0_h5P`W=>qvtdwR-v$12FTeb> z>gv`31%HA%qoD?Xz4(-7?Y9+!{LNrzbUJrzQZM*)z(N2@8O_L(>vt-N0s#H^@}^~T z{@q^qliYS8UH@2oYkR#5wQu%&eL7^6(U-{ZXNk!o!uu93$?qqA4^rh$+g-W)arFFq`7jbD@{m7U^<|)@G!$`hW5=zGGu7}*4uN9T zhR`oxUR>w#s4T`Xu!K>Zrjqs&u_VGmx~frz(L|=FD!Q|5%3Jmh_A$}u$-T=Iga&Mv zs3jDcJk$5)DATDt9o4yn#+iZBNO-jf{Hq@(&I9&j%QqyT+(HN!*s7%id#WU-l2~}Z zx$-EsUdGlXYpWoxQqXs{1r*nD zuFFl%De;C~-FE|Mm(UF0u*~df`Dk@~y=k1Yo-o?c1m>1avCXO#LR$bfE7ClDr$)7h zk|Y{S5oGRsqlHRc!{2{DU`_oiyZMeo=cv|Z30?K(#&-AYIZBe#Qu)u!c;MQgqBG25 ze`5E{a+<*r2>P>FrZeg$#qWvmlT8C@&Hnw{L|VfwiL@sCW?sn~g~~;xjuHTh$fOw1 zw4UMJ5i0`+$HekY9hA`7$UuVLGv-UcvjsIxhep=cITxRnIT^?nqi_WJF8=Gq-OOq; z$HAS@@lde0;oTM33B{dbGPeglA58w4re7M%kpU;AXmIA*y=CuO(vbxPOz=P7pd}>z zM2-CC#*?M%yrXMZ7?V-a?8t=8D@@I9d%kmamP;pe)$yOnLn^WDA%6!tY&dRgyDBVc z*SKu8=pQJ`esVlZ6h;Em4sD=@7or^fdvT+>e4Wyb zA@2O-rq{Uz#+#VeONRW6FfX2yKe*OA<)OCJF( zLub&Pt#)%?B7H7A^a2z`3_=puvgz}iDWPM%@FaF^utn;Kvy4s|>H25fB4Rn2!qTdM z41V}xQy zsrjF@P5_l^J{wn+`!}uKtma~AboZFvGyhJT$M2qi>0f?*+3{?+LOG*#Z>;U_gAc21 z-`Kt1TqoPDcl9gu%V=*MNL_0FxNM6yM~$6*8=9IGzp@KFbg1Vt3#{hNfQu0hKiAzZ z`eBUsT|zc}^>-R`8QzR#vMPIIRc+&A4o@#Z0`8U%mra{evQhXbuB9aFIKI8ru<~X6 zk2w~Dd>C{z{}o))MuXt;c?Ysi{;JtxCfZGtH8*6$ZV^QWo!?1qLoibw`=b%*%ogCo zW1O9%U5?t^pG7LE#I-AE7X+iM6FlyVTRakXI??6|$J^`IKHzrAXkV%x=9l_LJo>O- zyJ1@8z;efvk3Kk=^=<;*FNxg+Zq-*UF3dhFpdNbJoW{oXDRzTQ6T$G^)Bsae?+vU)>KLY)yhgpbK2n*>wFg|_hd?adxx zCof-aOOUb@psMlJ56yo$L#q|y6G=F?K-nEw%9Y|IUj_y-om1pm?@t-;IevyTv;i9m_<3@#x=MnLYdyWfY% z23uOJGSeP5Y@_8sAI;G6F({zfTWnEf(mnHIfBxmmm$MOxnrFtxin5YO+p zyubgy?5&i@NJYaYiO9Z0mgyJZnlSIi*oDh<1N+DE|O3D9u z-EqF>|MT!T-*Hymecqq3M22_)v{vQ zN2<*4Id*n4<1gyg;Z~Ikstc8fCI{RbUv@4mbB7a0_gwn-BN(B7S(lWae|E44jsl)* zz*9N(Qs`B(3`8GtX;!!Ydx6}a5L`Qbt=ZQpi4+lsnGcK@KkI1|jb(GpGrGuOvOZ@w zuZ!{;F&~H^d31=*@m(!K-k*Mxc5b*RIdj}1ZQUO&x_ryA^DY1F-Wj!js^RT0<8f^m zT~g-Ni1!@tCuhUDx^$f~_`rf_Es*u`+tbD5I=!Acz-IwULuhlhJW{#TkwJo0{J5U$ zVxc2qgwm`wG>>|FPfhLhfKLR3)JK`a!TV9wBJ|h?M0~xMW!D)Sq>-o#r40)0U$@7F z3l~I`rp(E(=csFrLLw~9_Q_y%c#f1pO*VUOG}tkLh?L!A(nCHv*bG9Yq8+ucOO}jl zMdzRP1GUt}qWAy3-5Yv-HuUV$eg)arx_BNz!=-^>##NRO>G2sgZ1Js6tCR}%D*OvU z=<$itP$N^h4Ox~nu7ELbJP<+Ly2%*I$GYo=nM(;UKVE;E*|T4F42|OnWr{5PUJqb~ z{6~V0|FIgQ?;ptf1@J;5Yp^k#Jvf)Q=w%Ly&@1(yob@qYAwPAnravEob(31s{H(8+ zx8tI{3DHfQlC34;54L`U6A-l|$f1%kM)d49L1{U|i`pr9SNjSUWeM4!?52TBCn{mU z6z2uZb6bcMO*MBDIp_9PLlmYLSX9J5bbMjvnVL?qUtg@{yb~58=7tFu$KcjLyy@z` z&D+r*cOdtNm6JT)VtD03*^N>`Qstnw3S7bu9PUhFe||zyBT^>NiM@CTr?ymd3ZtVY zWnrO~+GsYGR5QP1QQZo-|9`ZUo1cfwyG*9kuZ_P=o1eFee|QY25^I*eY!;Y8mtoUQ zs|zdguOq|xJG$+|_bdKAGR>N8ysXwF_let@HHBrlp|vXp%s9WIJw)u4HTpjPnYeP+ z>1}Bi7@Apl@6DdR_0&_*ADQH}@xArAUT>}(?ccJM z?I7ie(Xn~9`bQC zDRr0(4SZL~*bDu0CNpILdVx!f_}Ad)_jGTVR)OoOo!!6tN$!q-rJDokMwTf?;@}}n zq_gwJ#INCo`GF+Pce43=1Uj!nB5Xlv{>FtY4Plo^DcTUkt5}(%#MvE`I#3bA|K^FY zbb?y}Z+CyY97!LE4DC{rfT}lC_TB2eAq#|2={AA9-tgof1C%NzEGC1VnyGsk5|V?i zn0ek(gkB9x_YdO5utB%R#}i7H-l3lKJx%Y}*kbYH)$i`K9S+p7}O`@1!00$7ASd|GN<;$E;xuHyf3AXv7u;eP@H>#uX*o(NQm3zP2`?~|Wx}jjswfgw^^9@=s zRO6;S@uo&yntJK&^6OcpAYk9|fcdR#s@>7Yrd=dgmJyk{Z(kDG5o8;-NAhSoSjcuNjn|m1jY^%l%8bBq4 zY7Bf;*6;~0qU6w@74vX0oc16FmKDWL*0fZAm|TW=37A%Ds8rX|zLG48;z+u-L<MuiO&~-g6EOZR% z4An~8OO=n!G(#q(i4$M_{E_t2=n>i*;ME-fH`)PSfhnVpraJVfF{n;({s+Z!A6k{R zW0oJ_rx=tYsii5QBkL{XQ5lMo`ABzLq#D4`mj>MKW^Y!N>T+P4p4%Wu7_F|h+j*Od zeJ=c9uj}9w7&ZbPvQKRcTJP2Z19dCh6b@f1+G`dJw9U-CA1XU&T2Yqhzk-oRi4c|UoLI3-^NNuaqdea z{h$Czq|g1PDmk;%E?g?gnkv=2JlL;_%Bmd*tx9l0omk7DtGadXLc9{F&ICKZ2%<&! zID3(`af?>1t|8#C?bCS2m~|5!NohWnGE7RRl>ao4Mmza6ww)n?b&*W}Heg zhAiXHdT-lS-?q1#0kq?>W5%L3Ua&UqX+tWUf>X6ljryac{6|bhqXWQX{6m#=BX%%i zn!tiBr+dtgOn34u>|Mskz2xgVYraj(ehf5Q8ns4LzBaE!>5i;A2MWa(;+|FypMLCf z%;nE>BL?Zat*NI{v(OEAYLji$ZgMo_S8BIAkE9AuVLDvgRb$KYpv3eoh>fEC8`NQRrca6$rV+?&NqtwvY)E0(yV`_>nkZ=^KDC z=?Sf@TD~9XfDyXiiaz@x6^A2%wEtM>(#^!=%8yn0+Q{zKQI`{RW{e^RirebOv(P#N z_&EXmHxrVQ+du&Bu9}IvBy+~)i$_OwRV0%Bh7OH&Ea1Qze5kaWHZ9n`m4?REyB*V7 zSDJVzdjNdc5W$S}A4yLZ+p)a58>&b( z2(R^aKmBm-i7J%MZp2xiTzPduGm6BmSy$Tzj->@LEXG0;O6&oy1Q6E2-|wy>nL?IJ z(NHo~W=KygM4)!S!}+g=sHm!_SSkihRQJ;2FtNwK8St}c09tZ(k5{?OToMIsPm>HF zm012UGPoq(?dpteOZtzis|GD)!2m?w)q7R%R%{@z+W9}wtg3~Xua6)YshE*nhp;kB zv37-w$=$tDKdAMXhzTY+xm_y+50=7ves0)1&(_(aZ|8?^_4eb8j1h`M$5A?98`$-0uc!ml&*l*3 zNrbxpQdsMcNue;6n|MRkdbpZrxT2!sRc-w+?FZma+D1l3MQfUjrs=ZR#qS?FLc>`; zpVjTvy+cDJ-mSfMYEyy({0reCTxX(RvE;l#HSK)J7{{i>J8!)dV!}VXaz!qLYD|UA zrFrBrXyyqM+^_pEt1AFk5ecFLvjjc&wYG?mpP z(@Diyn$H@E#Sx$yHgV9ZLn@U%;F z;+|Ib@W`yZHmb-i@!xHHvau%E?v;p8k9taZSGz6~eBWN|uOu>lZxcsa{hpVK(mQIX zkKSk!JL%|o57ZNyY8s}e9>LGBNV1~?7qvgzCx?NaY||(S9`x&cB7$+*`rmK zthwn^+pfz*!+kq@Y%|g-`tn6t74mUOZ<+L&iSFL|b_J26Hj+;RjXIzilBP|$(3hH5 zi+n;P*AhMM8G_iMZ2%~T^GMx;2LiV~fZ-802 zG21w=QkRO-(<0EyUD`}vo7a7UH^I z!3?(I#a1nU#NRA4g9Xuqc7=>UXp`va zFLR5eH<{lKF*Q2{d@` zW8?CgOm+XR5$M2wuMzL>(^9sk3&bI}9jry^^l`;UT!an(hE1sD0|99kgOfpS&T zENR}QqkadQ0mPTYXxaT$uuDWGrG;neP9ac4ED9_7;LADv?af_b!w5j`!xJ#{Nw z)4Ra^jFZ_pgLA}Ac*TuuHMF_Oc;d9dn3iLT+0iAr_&#}#`<8&=*!v9X43L4dXefou8RVN z8`0#be%L;QTHSj0Ha=cx+bn1YvMG{ux*R+J`v&$?S-YlpFL}} zB;hY=Uw;00`Lo~K*e57?*y0h8b==Std#f{L90&0Yjem+{IC_SDkZ8O_C4d#c^Hws0 z6)~3>=&D6cYMH&lG(e}nyEkCf$X{V&jpEC2uN3!N`(0BmlV?R1Q|HXl=LaN80;WLS z@WW*y>@_i2_T=XsLL*o~MMjd;+oSMFa%T35KGe-u-!XMGr5aTygLw?v!qVeF0zW|y zo&SPJ;j^!Ny1GxNBX%g;L=*KiQ+vgN(*`NtF8#{;EA@z}MisRPb>h9>_**7JJOYX#5#=+jucSy)9>qxR>1S8 zhkIpN?wnL!2q;KPAB0xcpkmtYd@dhFcVsu+cD|67KtL3VoM}O zsekYoied|s)f7BFmpDeVO!nE2($ax9Y?pQ1FwrRN6+x-~S);pFj#9*+EF42~`R+|G zCUI*49+T^(qJnc!AXX8OjT!bJ?d8j^+E}d*XtZ_SLQ1e54vHN1VgZNLeF=J4Mxc9W#wgk3ZFc& zNm)6zs||`lN~U*!2)){TSi2YL7~o&qO6pdy=dHD|o#8M>ty}ktT2KwaVRF`bdM!FZW$_*vq;~(JMaJE!yh&gslo}Cvv6pVMJFr@a zf?vPF%7&@cc-9h+^n{kv`@dEyqP()}G`tnCeS}Hj#ZK~hkZ;B=@V zcjY??TF6#2<*f5Q`=3d%u}$UCXAk}Z!h_h@i@oEIJXM-IrMYllfY-%q*Aywl|6HPT zPZ$|*s5ls6UbyRk)p^G&ZM5$eGNtK}QFZ_EM^uk&EIHj%y3tf;kj3Ahd3UM0R|jw@ zjxDID^4W*BL)3fc$tvt?12;ZJjxzPBj(W{48F|;y;J1qn32CJmHtb?ro&nVU4KzIf z`e+E(K0t zD21tpVgBKpX?0eD*+MgEOD6qA-j>SnkvTp&yuKF`k`a&I{B5C&q|uGa%w8stHN_+I zibJZWc7=sk3R3|{m6XC7g5c;eGCc0<=T|#2qH5fd80L!p!xW|se+6%4&DO_sK zH$~A-WrN0L@au?Y8P(Pe-KI602s@OU^_TSt5O_Z|BU_`JaWI{JgdM2}WjEIKg#%2!OMYU>t7Y;3Nw~|VTBh-> za!xb{>=F?2-;c1n!zlP)cQn2Dcle|j-&+4J!^mf&>&7*n=N9Q=`gxZy2dczu?KaxC zQ|tKawRNh7nwN~2RyF4O+~!+($1_VCr^V=X5i$n2K$?KbI8R~{c;+PE_Ba`UsD zyXzTE>ebe$Ef=J8hhXDZ<^n^qANUw@1m&HWH9mjV5+N&l=fK&s4{c95FyTbH*kuai zl75UfNjXp0Hghp~ec=h_EmV+i;-TG&_7i=R)M}ooEQ{v87WcVuu_y4Qf+&y?;~WnP zfrE*iszClxk**dxLVPk<1y*SShoA}k?MkdM=+$cTprxW>VdVyX?eqxjVm)rPUW>vk zX0k&NN_{2&XU0;#l7Af16ee;V{3fR9=oY-xvR+qClTaijOL;R22Z42GY>xfc4!<&j zM7rT3k>K)fKWh^u(yd7+r<*T$>RwuKD7)_w;+=R!%v-odq1g(VLVd?^Kg}}svC|>J z3PW}q3HMmIR|7!|?K`7K`ySbYe6q>t0|K39rTEUI@OK$0{Y-a*-OIW=AaRy zBAH6C!D^e@OKI69@Ds^nKXChqs{~3_x#5N1;KC)%u!>U)FLvVu@{Rt=sT-Mm`Funs z4kxr-Il2`F|B=s%l*#kt(l$F~Z2$FZ@OrdLP+`n6=iDcRHZ#xaPi{PO>19xDSn<=B zk^4eJ{Wt+jEpXL?mg}`||2}HwnlALl3RF<}B{1pIqPT6v%zHu?=yjhTkz#)Z0JKoh z90?Ek!3$On#%Thtv1W)wyZ_8rNsI|Iq+SGFk$5wBe@-=2XLorVyP%m!g1t!&@!<-7 zJvr(7ANPvXyv~qFtVt^aAKRQy%r3GSh|Em65XX>ez#*8fmduJl*yD9B9I^HNA#)C0 z-eZR47P~E_%9AwwkvaB7VE=_?QUN;brYtM zNFtKCoixQ0BI%MXCvG_>z-eziy+OpSzMjp_NEY+pnKMJDEWm9lQoe=35_=}7%b|29 zITxnoKp>?5$dkVn{xB&$FZ{@n5tR!=pS241zH(b|`>NbGwU)Lwv5?!C99Zl19Z(wy zz>8+O^YN}z&(z3I`q|&FvznJSIv)lKDa@X5r(-Pk$va~g+?CQ_?QV8>Spk8UHmEL2lqwBwiX_ARcU*a7j2k}Uwxyck z>z?+x2jE1c!)K;@q1TimX4Ky-f@z;UL!hA~yF&a6;|I8YX(w)qENS*aen58v&!)7R!r}A-1fI>=c8Je=vt8FS( z#8IJ~q((z_debdwx>?BQ-$w)wx3H)I+vT6bTr_Cm$EG_(NuKIs>)dLz;&P9&bNhcu znZz5n^xe4Ld&Vq|`1@~7;$hT{I)4KZWN1UG({{Oun|8#5x}>rFHeR@c|B#D1wpHBO zvosj~W3{m%Z(tJeB!GMQjkqC-%jub!+V{?#KXs}gYGyBZ7~I~<7zU=O4u5`#nu#qc zkC+izw2z$S1Rx-}xqCzhf__ar3R?_87RaY9kyyt&12Lg=*a6LpMrmTX(dF|yE?0?6 zhY8e=l4BdY^xoVd-Z^{4_X?I^r_9?uG%lT7a7g?sJWL;W-Ad#dNf=1UmuCVJxAV2a zM95q^cNB-C2&|3;qT+rsfR5PG!1-!*XDKhmK>KoSN$UjL_s@nyXf*o%tcjvsV~7LZ z_y^CQce@ooPet@PP|DYM^~w2|wsvhlz5#<+9u)BPIxSM4rp_IzxSU0jBN^2A`&=vz zw9iO$ie*k6hTdRBpBXzfnxy0bsS{r`XJ8^f2^RlxHIAo*U}FYqVnvOJy{OzRen*!a zaDrC2wx+^?(3W*qU+?^9=@%1rvHwpUd;Z`LSLKpiHCtJ`d^qb?ur{}bp@j$8Bi7BQ zIZr&??eeE@?@{}F@@A0o^Vj<~dxVC{rFf>AczN%vi*(Fo^L$v}wxlbZF)by`^73+%;G} zuhMODEeN-ZjGq*Td@A*RY%;_YyN zYk+?yuD)=0cYf4TjFq&7D`h=TI3W%rom!eX)6Ddj#d>e!V`wv$fbpgLINp^8^UGko zVl`e+Bj+HPw|q9^k-3W6uO!On%BI_I2M-xi9k}}SviR`n!#UKvIC+-Vr#UuQHdx=4 z&J(_rkvV@T|rC*PrLpo*;k(JD! zgmNG((npE z5%r3T1@c~V0U(~}5PuhFIhwIN93q&YRb#^89UZ+#=ggl(JiYh~|LygJ3L1B@?dB7k zHq}URfe}-ZU%{=f>JgBYk!O$|gz@RE+@GACqh%Z2jO`Y20_pFj3z&?A zfk2UdoXptZ+-g3?N6-jZ-T=mmxj1@|T0xfIfUu)ShmOaoBk>#I>AKs!uZ;bVv}vdm zBnhJsWDfBC=CEIXr582D@ua-xp!M3tA}PWoqMm9&5jcFD*{hJ21qrhYdE-bY)0jqM z6klsGzAZtSW7!!a9&f_{P_mi;U3Ell^X=QW_H87(q;uz?lo8@ZKVi{KqGMk4Sx6L$Ql$iC^2N^g?9tL zL5-p>tUIRPh>d*c_qvc+*}F zA}fvgtpKx9Rp76~1Nu+6ZB<1ug_cK+m%m+ypxag>M=k|CJKg;d)??-hDR*?;=ygg7 zZ1E|>$tt~~RE8Yk==_!h&im_bOTSNd8p@RjOfLeBClDCYMJGx?97|2{l53BBG1ed~ z5Dif>L8aH1V#33F>+p3ob*ZYU{l!c2vDjyX4iXWu1%{^GI3)~^XV!)D=f~OD^syuM zE#@{68eFaZPmn&-WuZGE>^Nd8;4zXvw#DKY=Q`D%pq}O|w=7tuK`bOnQJ*>b^;|_* zzQf07U-3d)o-$7Oo`vLo_fL8g8m-1@pm;&k)qI;4%NgLTJ*U z+~oO_P`_B-U`DG7C)N*|I`)9YWl2(m<6M`_2rem1U%odZevcYT$#i+q@@ovA#*hGuHW#Sa{>^>M!`=|?4qMV?$P-Z_kv?Xb2v(0wKnR!7TiEyos73_S zKB;5%ySXiIQ^LvDFz!xp6EA%@w$IPv*X5m46$`qg;OPA!akVh8{C*G=crCxAipB#^ zhr4k4xGa7{K^GNNw|r#u!lRlZfa$-HCDcTLHq0w&_@ALHtz$p!)mh~y>psrB>gR16 zU>P;gHbkea&`mM_4}yK<*o1Wp`)y0|3N?S$!M#0@cdAq{@-YG(O4m2CcX5#zf8*%{-!D;=fOax7F9L!qK%*y zcsowfjLN>fo|l}mOK#*|oIZI@Do+;M^JCPP+ZX#8hXAXHde`4@p8#cmFTFF_&(c~| z)hFuBcLPw^EV4kzZ!O9)xTX%0?jG5*S1&u<&<}#vIrWFC7KQVu(hNH6$lDua7M~gL zYn|y)R)nn9;q4TY6q=x#qo3P3YMkwi8NNtv2Hu!hb!3cW-PmMYA)OPPE{dY(M4_f< zZoRX8VU}3!GHPGS*`=6$XHi^j)6FxmHf%ThVr6N0naBS6Qa%7cwTT_Zcn{QUgny~| zy-l-uDd=U~8;Ew4=&og-GAVIH9;HtqaL`idLM30p1_r*=$d$C4#LX$b;Xq~cR<7X& zKey^r(2?-*$Cn_a{oGbPbvAL9YMVSq+n*rH5pBD+hP82lTcX$i((d1HQZU8CqrXS- ziJ65{TBmjxq*m?X?YG(4G|BItjBTASO?qr6Oz818i)pAdiF)I~htq19Ya!IRH0u=_IpStsbbvKmf66vg(J4wVdOcS>?wtCy9 zR3$^5qmBd+Z{~ShpiT%81Uk*Gtxy~qs#Zg(iARFt@-eaGg#f2>M>MVEX5wk)+3y}z zOhQJL+}wWObP*ETL>0Jg?2$p&35h?~s*JMA_)-ChLnB%cT=t@eGE$*xP&3`gptj)d zM>2Y)R%o`v)J-iGWHm(eG5BbU>G+@h_2z6T>Vv9>RB8r&dxvUqt%eX3%=;I zS)V>z0X=kxkZs`8au`v405scwR_6cQV_GK>832le(`47S50!>t5Y7gT8ik!btHnPF z1Y9+*pGQS=l-t8#O_3CG^L)@=ygL9lH}u@W!-t1NqB4+aD@1d>qdy!&ZQ;UwO9qS$ zDY)?RN0)XUXdczm{=qwR1t8`6QZy5hlk)yHD%EqF!OstcR(N))p!O(MjF+Fe_b1_g z=r5KM*(aL_X7AJi|1oKJ8upqNTH$)6K#~xEBbcnW#qzyL02<24)7w+tNsotsQ8p+O zF<)Qt8-H9<46{R)z$r1)nAznH#nG;e#y(Knpx$zhxBsX;_0#nG1qyM*j=6{x$uCKq8{Tm_5*%*9cqAqCH2A6(XEnHs8!^116wZ{pi2_sDV z^u2>O`rg(bx5xh4kZmnHx9zfGjEnBg*4w=YKY>E&zcB9AEH_Q9cM~7D?I0C;)vB{c z$}H{d+Cn9|w$18FI)-wIS+nN4WTh|VHLbn4OL4eXXB%a53D=DOce%@9er7s1InT6@ z7O*z>hS1?mUawYNsQw5MQ!-1ZjKgAXI(Tr_4Z|{G*nelkzi9OllPB1{JFU$X<(6MJ zpgVMCsX)0rCkiti1iUoGbd^f7)F49FkVB<-U{UG&=;Vx!GKV3>pzT2xrPGh@-`~J| zEV$F&PAjmD%c=YAUmB}C8mOp9C-Ec!%RchKdpy=LN0APJWcd%sBXd!{43qG%W(^xR zCM4a;Da1YKT`{>&htHm!)at3T8FACPZQI%nvrR&~Wpd}Ae?oR;80fnqh9LQA#`xPQ zDf_;)%X=^vBQ_sM%8^_a7$n9R?6-J3?BUVC&)lvj%@ce;w|2ivmnffx8AiP>8ip0X z5H2C&*2Us}-6wJn=ymBbMgZjKd0#GCu2}Ik(m59hph|-)*euy~Jv%;a7k01k&1!uW zjL5s%*Y(=Kj3S;fq<8)g(I>t@cW;aC_6% z2#NX4L5iL}S*-@M-!Fy9gvyGdDQXU$k8G1(Ha?MF??if^sAA+S|5gGeO$j|7-|9d5 zlEKf9-QJSrm)!m1*Ob;?-h)G`4RktEEAw<8wwS>Y<8QUD`fGFQyqK+wSWxt^iWskN zG|4YyNMVN7vaM8rpT=IEfEMCr746 zN~fa@JlRBexn2`*hGkJIu+|*(RLb}4`$_|xUGS*MaJ1J27n?kDt=E8x*NxeuO0VYv zBEP08dJv`Fk{Ali!qT>`Sj0Op@X`$=G!%F{5ezT;c%o9^fj98|3HHL;d{uiv#c-=m0uG)#}*sl(i0#C@3FWCCh?`iK82~Zxx@3GENS|;dRJVNtgQSmSDU1 zQc3iC(@3`&VxHoiQ!QxHLbUvBhdnO#)34WAD2FxEg@T+vVi7+#r5fvV#C@mO(pMWB zi*k9Nf0}#NstXKb`p4f=6MSf#XwBcHnzw2s@i6}7df}2|G-;C75su~Z2M7x8aXlT6 zf*_W#JJLn>&Vxck69X*>Y7? z92{83*PXuly3UmgE*D>-VcG#LG(ReuB3YAt*C6CEkKeWX)2ip~{B{4MSsZg#?zm96 zFqhHpH5%Nsgg+f8wq3Kn(Fv`pY6QI2{L0b&4In#y-g?jY^z2=#Ew!|}ef%0mcF7*= zW}Ox#JxlX8zSetfwH+(B+vPRUc+~k}(lY-Prld}U>>@G7*37cf3eWvRpm@m4=T* z-N?P+{QUeMH|#krrqfug4K$VwAQ=TXm|j74i$H>Jc07IZMEV0c=^s`qM=unO2SSbN*%7XE@uKbck?bq%^YiAeVd0mNu=nTB zpDFm*GCn0m!+Y>9rfgVCgILll1au5nJm^st)D_5hteSq1HKXde<+(eK?5WCyz9r9h z@#yqp?fdj`8y1%awF1ReuYc&+@h@o}5mnI&LbHN^GIi_x40=2XpajqkA#yyKN@~){ z2u#(knEXq*IVA!RA^UJ(z`%+b6EWyx_vp?{8~h#ynV~Eq-Fr1&19CiPE2;d^{P3h% z1hq7Dd+1@$z!TqHtk9*Vq{P^S7qCI~suqQ5-RAH~dht@G` z5sC8X^iY0exS)O;p-w6Ted`O-{ApFvikY4!?E3^ybfK}{#N+g$Zp~fth0^%-E5!#-!MbWswC;5D=*~dQlgIYwc#eL-jXL_MDs;Z-8gZ`N48++RIBwDUJ z=?aIg`!?Fv@<*d+IX{XzD&+dizV!Zj-;r$(PP$l8YdJS0n4{&!Le9%<01M|w<@H$E z1-CE_LdA2r@qmhT3}{=f`TqDnNEr!;TWOx5F9jf=FCfO zhzc2g&UyUkBa0x;W8{IyAG|-s?C)OwDQ?cf6)QA(K}nkb;f!`cCpaJ?XeSnPTP(~k zM87V*HC3mlC7*lUqvPvxNxv$)){A0*E2JaZj~TXnDr^`uDV)MJ8-jqqk@W5Uy2q7O zJJtr1{LEx$oPwNCqyVN#gg&hcdhni)ZBql{t(%b!r(KnA?aa&iG0!E^0;5M%3vK7x zAT;Gxv+XJtFH79hf)7OK1?8hX6iVW>b}TPYrVBcJZp67gm$1d&Ja6spbk^X3q?itr z`pBe-6HOO94Nj1xbtIL37>uSqHB?WGXIm1U!3hFz6HPGf)4nPj!y$6M)X)pl?t#+@ z>T!^g-k=acs=ba=K=U&DN0UU`0f%$-!0QU$2F?^2@&{PoA#>-x-M(;(_uy$tUIv-V z=N_p;1TKN-tTSlmn3V}6yvQk1eYxMqVH%Dy`veLpP^(Gm-MeCXFk}=!`?vo~A#mFq z@#c5btj7>mv!f3B;I{PJQMIrxYx;l2w%r6?u-^$6F-)3K|M=rjY;5t-JnpSSv^jH~<@C%^bQQRQ3REAs9m^R<$G@f` zd?5DCQ7OhDG+^?jjwuh>hMrWUk^)+X5)5d6(&v{pxVwc##jTZ)ip*{Qj&46i8(djM zyeLnKqQ{NAW=l?HY|WXSabmJzn5#kB!5f%lCkTCC3ERGqBb*!{|hnH#-x&gRT+c4NASva0T=`+GMmbdtd! z7CVjt>qXtcw+dw21?ZCa zO~iX6d;vb&zV&1)C9zqupSDD$031pHLkK8J42cT#eX4&6Hz_C#44Fwv9>qCQMc zuTaO%i9%Kn#w2Rp?o;2P{6R9ME;4o-64ZX5u2JdU+_HRdgEyNhib!JKy?v#~=^f9? z7{djp#LXx?Nh*Jy?=mFxA7z1-B$YP_E0C`NL-n3LT1urI1JK1n*3?OSa4@Ie2Wq~M zZ0szu1*YzQ)t>}tXjA``tF#&;vf!Mx3+vSbeG)L$NGD!mM4ft#@(J%M3V~Je(+T&O zptGEUhFs+)RJhLYh_w6(;4U_+UPW(W3nUw;-{I&V1Qc9-y=24(C^ykkx;_hb)Zvo( zot(9d9R~jE3ga%>qy%wOYIrF>S#t%02%VI>AHr*AQ&>_()(|q3vi9)+Oc`vwVzX^m zOtkT}8#iioL_ZR;9FlTfew-ZQU*MV{?k&k%L?@=U>9~j!@T(O@Gm-Otm{AsJVw{y3)4e~>!z~8f3diy@`uwkhc zmEK&dVMV?+QNvjM{ysSw!zAx&v*3ldt^;3IR1_sFje!JOdI2|l^z@m4`Y7$S4yW~g zdly>(0Y{HsR{woNp}aJTxa$|-XLN62ae~`i{h*?W^>llV-$(&tC%^mpRww-XDs=+t zsT2P7r&nkl{o|1Ci**}Zsz2+mI@a)+i^b<*ap@baGd{j_R$zvE`r&e5& zC%kll(%;B_ZfREz>`0S*jgAhF#~+{Ao0p@g&qf5DWWH&^09be5WP@=^Kh}25mM+irB`)_C{ykzk4U%H1Vf$qVP|RS)*Tr>v6_~ zRj(I&mIqHMaq8p6yPDO73{V2 zxzYgs9kD_+C$1&E<(*rpCm!IXQ7a^(0}muXp6F{{gkYfB&R5}(y^^ql{^cKI*4dpP zy66(|9;m)&hl2Olsq3r&ubeNNU*vHnV@lc zybYPI0`mMjv(ci~Xj3jUUj#y!y(Wx@X%&0w&Y++UBfEr67JAevmfyH|E_cjmRzsCL zz${oq(g-@wj5?Y5>6=tJyd2JIQ0?S#S8e7KCcmlMhuIDMlzZML^--6qOG^G&v_3s6 zIm540^ChL?VM(2Ox=+(RLyzs!IXcwlh?Y&$=mpBFO24eIgPHQb)zh~5v)aDyebC>i zs8n@!FQq$pW??;rX<&oi?xm4y8z@V=Jw62A!^YUo_fs4Eq^xo6gMD{SBt`@6)<%K@ zeR?w3Vr0l_>{(h}%;oPZ=OS(trKbeT9rr9s?Io!=EDA46V+jbO6T{F3szbOCCf(zj}xGFPuF z6i)#60kuKi@S8s1KxVPwQ;()QVM6Fob|E7((}(-2Apb?XfZ+78iz-GaV#Zcg*xNDH z*|i8OQ~KUP=EB=)Hegj(CRca~*3NxpzAJu|vNf2|RT9nbt9d11=4Z^XGQ3Y^t0a%S zcvmd>3RytxOH5{jb(c&0@#ue;w@5u?~HV=UCj)%+3!)_1<2^>K7V<86@fMs?uQ zrGosm@8Gdj^AXdYYAtCF927f13^Qs#?!Z@5l!J?o}wYG+kaUqt<7@ze4Q zmy^Fx!azf!aK8prX5r=MySxPLiB$~<7Q`D5#= z<3Vlkj?3t=Etc1+vi>S=@ywf}vL$a<*ccFdjRoAX(nMK_!>%<>uEoXQfg?rhET+b+ z8w~|CTXmt|tVH=|017ck-oVad8N3-UH{-_f ziQTnJouv1}M~*0}mZk@=BoYiizyC`(QZ7(9Qg`XAgEF1t?mj(|5T?3GKY!#8$thoK zy!Y9$m6YD~5A{pE?Y)6A)T>UhTeD`3J}}Jm<><=D(}ylU!?q+9T<$-Al+1~u=$<=H zEWq6}D|*JOUPjC4qT}_hGgdYVBEpaZMT)o&1<9IE`Ktjvlyz>U7yOOg&0FnQ7v6=b zXXB(3`->Mwgoj(*;Mte-p5(*cBXszvlp$MOKvzEZNnosz651>P=&~znX)WPl_Oyyo zRUxMd5!9K}P1U`kQp(%$aV{>FwUfu}?e641uuQkwo$sR1v?0N$T^8@31NkAfX|KCj z`ldc8kd;7RqW?hrVl=@dEH^38FuywR+--kCz0UKXk`bBGzU4jO(l^)y5RFt*5zTC)Y0)6yv((*I~_uIhWB}j4UylBiv0P@v(#S3DGRpT-#?z! zZAqm@)m0*Y+}AJ0wV*g_Y43y$l+!!zX|H;%<-{q@6Wf{(-Z)pig<`&WwCH#m7I_ncWhe`Nn;zm5}X%4A1r}@ouMM3zrcNfl!u;e8|?tgrdnEaX3ZLS-*^G? zB5;bUjAT!>=UYfc7T%(&oUPhG2@}kYQv!k@R$mM(0!cu?py-%zF+9b7FudNK5c9T= z{&6V!dH2PB&z>tkmsj`9uVG?Rf5X-d)dP$-8k-#RuKwoX<>Rr5XLFyeIsZBFXIkqg zE0f}MrakRLRzCFrOXsLHow(`XKQ}{mZfaLvzA%gwErsAvU6()T@N;+XLUrP&fbCvuE2Gy9(B} zO8qONWztWs&+(st+gz{}W98HwWfliYaQg;tQKm*`&C%I|ke8a_#HnYX_oZt<=z@UI zZP{zM+pPci!{BL+Gh_mxg>hdTwl2kYg;7!$gRqi7UPp|>^SNpzogA_zF(U(bnMZ5) zjSad5YfX1GyRbaB5rzR%@Hl*>-)@#Ps#|vu!y_V|qg-z-@Af5@4ODE==*isVpI0`# zOj>@yo9p%GTcjt>RG6*?(k6-PN|fbI?gO=^O^=Ra!D`&Atf!@;^S1 z4EQY0;m#XQ^V5>B6_AYaQ4BT&@ji1||E}Q%-C10tB565_H82^PatD%&iI?s9W=$WE zR5#Anwu|qU&70BbT}`b%{}H7QVF+0tRh-}`u#~(C@B4Fj!~;E3(}|C~_uL(;e;3T} z3@G~_#&vgj?yHziS@Gh1mKO&{O*#ye$$NH>6BbJOdU+Q$9hDPmICIOsju{c&2~(Md z$PGnLS^qd=p+I{j-}``opIHip)^di1YkOu^^yn#0eED_i)-9Z}nWtLr96Qo|8KV5w zdg=!~A^w<;!d?*dX!pgMEn0MFndEvh*gNMUWKI13wuJ9U8?yn_A@rFL#x`pRXq&#zjWqVK2vn*(utB4*%vVL6cpVZfZ&YkkU@h=-)!sp z_?f@vyqHW;#7-p=a$|{Ole7>iOU3e=1%tTA56lOLX*03drlW1s+xqiyoPM`%YNQ;* z9Qf1wW;35KZvt^=fY-8ywB zWh(TeDE`Xiiv6X|2+_pENp zKiwmT7)rHfKx${^tTW+J0tb#AJGq z3;)$iPA2U5C>*nJO2oF;Gc4-L4Zsz4rp3!PO;RPDd#c?N{iWB+pI;oV6Y~lN!PawVSR{fR_nza82pv97Ze~MZSzWWCq zLxYLoI#U>&dmc-vv;@nbaRPa&2yyH26a;Al)h$Z(F-&sU*T`B^a{M5MZ;9oHlf-3L zTKjD~{rvQ_G*!YjeQOSSwYH>xB)LAhcSE<_HhB4=H+~42G5>DyNj>!}XcJnC!7b`< zSjUWAlaQ{_pP-ORi!)|E|J<^=?TiQCQU5-FktB+G=%O=E#1(AXVqVXV>bs)PbKkON z+Q;@>@MsnU*sU7+VI}t|7XgUcJ1QJScV01!jDjU2|F)q@^tbNc5qdoP{1zAX5D$-w zgO7Vx0JSQ}lj9Op23ocnq_z_xg{12P%MF9ViB4$M&+XyDg%ih(Q}6ft8+RFdm+IX zuLH*Rx9;+|9cDoR`Y85&+Hf+b!Ri%i*UtRRKe7I zpjOsq|Coy|bN|X|rZA0R(27*=tHPk~b$_Q3JO>f(Zvtm}>~iANAOW@CXxjKge*WqO zVa;2&jvCiO|D}Ib^YZ)r5RM7^;AytQ`D;Dathw-X^EuG19tQ%v9YT$_itQp&i?hN8 z;$}MB-915mqC043_$PG0%fsg0@7j=FB)D*j6TEiulZ@^FanCVxeMh!Oggp!Q*9XpB`+CG-MU35Y2V~1wv8unU7aMRKY z`RfwotDxA4DKO{T6ltO;|JVbEZB2jm_!IQT_<|j&X=)zfyV73`P?q%LTT6(%XFB-j zP;j8T4_(&U4)2mP>5#hu1A9%g_VIR?D^o)1PIztFbnw`*5r=s4h-=fvfO3wBd-s}? z^hN^B69cs~x8n%?t@;wm%)!0=LGoYe=*w8yB-MnMC-Nk)a(dniqKahKKqU|S7-qA2gx<$7tvyk(}49z znfMS#m+LIpKc@D^%PPtSO5<{uFNZSn;)m=0JH)vfe2Z)QWpqIK)2jcg%wOebRr@M> zs0(Ye&VAX?Y_X4f`q5m6a|+##AJT_DW3QA3UO%*A=o3T~Z8h|(72X|k*J|>HK4OE5 z5FIJl!lCJ>EOYLL`ib#DgpmuUEC<`&v}KDOxeGDh60u2p^XFz^!;#p0EdA56Y*bt- zE6hbMfrSgmPW}a`u=|3-&MhLfs30(@!YlZwS|VKmbCB#2UgbYA@AKzjym`2}HI*jp z9_@**|{?V;zS#|JQl)w_gH9m?!dNL+2!QZmftRA(<;^~72)vYzl zUi^7;2u;Gy&apO4tp;Zg2!C?UZ*+OZa4fX|8B&v)j$)YV@p`c~QL{{breWD=$-MW) ze1z!`a1fl}X*4gU6ZRyS38w<0)0qIvji*kX+P8g#LT3(z6_q(T+1V}@!>3GXj$sVm zuv0p(-)^3j?tb(1^e^GI0zaAbK>5<0Xq>hsxWeZ8oYS*N;aI>Ov&8zghKT8t7c>v{rhu(5x<(J?Nf)Xn(Yx~^1uy2%MW^aUdhuWD= z3mIc1&m|XGTry%iFCe<+s;bZWc;{?}BaM4Gl616sNA>iVl~j~159zbg>DG-CuV24@ z9Jja)b7w@wGwA8K1@CBZx|bI})m%NTyva~?#^b9L)kRjO4Q~_weB@Zg@-1KnXdiIB z$HtvpK9xXH@-JVXHS;~j{(|%2jE$qCv7E~TPxXU_h8c@9pOZw!^P0Q_+$Ow^^bHMf zY<%yX@REaf@n_{4X4qRFX3w5o`r$;hnvTxNaVGjf^%>qaY}mkQHjOGH!GJ0bQX^K4 zI!SSyxEkV1-a5nm@tfGey^q}gb1wcmJtiVc$544o$RCdMmUFT2f9Im!$!k^9SLoGk zaf-^<#YGk!t^DrY8`o*t*#>mU1?WU03Wf3fb8GIO)v8rn^6HXmMXqOML}cbwk~+8Y z{=0MBp)rCQd6*^gC!K6T2`F?2$*$9=(Sh{*HlX}aui_VA6}ikO7V%6G0l>xZG!*F^ zwExcF;EqpqFD53o#`skcXJwkgZZD-@nYE1I&99>&iJy;7c9)+>)Ch+sE%PX8^bh`S zX|J)?n_yqT$m8NB%hh1Vt)`rXf?FCc)-QDn2+Q<{9<7K5-OPOK5YvA5rkkFlRxVoo zW%(|5BI&vQuVH>ZN;{n{B&Z0oYX+40ai3)F4Ytupjg;_YGc&6!a0Hg-q@sAktvI5C zl_S*D6p-RZ&z!;A)IkMfFP)b~lxP^DNCrMNO3ZOPRlOz%?&r9m2h@8!z@ zZ?kwljXQK84BzDLA3KA&@GHb(&pum+ZnZkP;qolBK)ZLOWc6Cm!gq7Nd6hbpiOX@=>7>@XJY8av zs?2e_qU((7qoTYGC|3KFdwrU|=d!n6+8tDHZhzR8CIscfzJNM5u{tz+lGc&GRv4%y zw4EH!S<+knvx17&jcAFw?VmnoOd|+-oKqdsg&{-!Tk*E5=b%}$UPaEs21(OWeJbOm z$U$@is8<#h$zKON!Z^b3sdn$#0(INBpFieDoWb}Y)&YVqgab@mx&+FtqGxsZ(u{?j162y0e)I{?1yIZ-hb=DY2W{{M;;8X71onkvEAAFAkq=nb0QSivh)1THin36+it^fK%iJbZV)UuEuIBf zBhVCLg6Q6(T4p$x6kBj(gxzC?y`Q0dz&vDoIGJpu z@5eyM?>hDCugRUx-Vl>XaMn>a@lj=kd8duH##Ht|V$G|6wzIRdO;Pn~)uiH0iD*6V z?@@#qPA78Kvln;i*K3GDohkeL zH@x&HZwRUC7Kz&*{qn?a`&Ucq9CfA=^pehFFRlP-s;Orfz&OcVPu-0Q74F#+!0~v3!3~`B)TU;}s{aCdc%-(If+9vE2 zTnQiYxNc2%{X{bJdwKr3x56U0!lKtYI>6be0gkK^Y{CK>R(5xOg6dx ziP|g_RaDA&W9_u3h=Sr1zgd5oZmO=X!W~9_QT+)g^JZWz;~VM~l@rA{a^^G*bzHX& z9SqfXy;M^UI&d6;>9DEwqb8jIU=G@v>VOQ+K#Id69++ZFwuubCB~am>_eBD8Nz#KtiZSJ?_)n9tN4y4}wYJoiFtt!>vAR^+UG8k6#R$t)L_ z$De|-pCZVHuYAulzgW5(lU~l-{QJ4T8G1^t0JrBayR6kI39bUD@|zu8p`t{p!uA!+ zy0mrM-Q?Xn1}W;rKkmPI-8o%>fp{FoCxSTfQoAXo+Bz#W^H~~RI12i3F z7%}O@QXGP!wrt+2)$R|BF8Cl7g9fcnE+RU$Ezb|OS+RZ54BU4>G(#ccl?OKGEVIW_ zN>z}ZqJbJoC0eV?%mf#8%0_#7#sP1Xty2~|Od1(H@%;2I5=IG!P|v8XC&|z9*b)mk zB9?cO(2Pu;^$desGQbO1I=>>POjSQ6_Pa5oM)}l=t zi7AUb%cR7|Kew7HaQ^TmOS*BvDM<{T|E9MU*^2?S<${#CMu~({J#juqB!iaCM`X-7P^*U-Ssu6XVt1%vUok>p~4QRkO#%p{I`&w#k= zaHOn}UlY{=_dtN?g;lolTudT7gG=^iomycZyWusv4kCWS_-syjq(sJ%U>myP$)BUs z-mfq)$a(i`?BSlegoJZPE9Epx=JpdOYQ? z(x~@6SSrv3&Qx#fY5xwUjbJQn6gk6}=EOM4jL8h^s6RBV0l{_&#H*a1-(W3IJ#f#C zgE*x6)^HNk{kCme@}L|Lk1<~{>1P%mmDAzFHW|`u_zs4i2RBM9A^MDA&KgxghFHH3H#9YG<-_mc(%>OZ?bdarCbp0#qr=62L z16Z-_C;8JFHf(quIPJ4}b%}ZS4yX94?|q0ZA9|tQwg6ny!mBzV-wSWH0UTQyWQZAj z|5AEmV8sA_ZUgC%#9gvWWtdx#+fOa?EpE){V|CK18#QSHxTXhop_YydOiX#enLno; zR|Hx7i=V%JwmAD3+D=fvnV$O65BD!AfXsrayl&lZWrc0LRJJje_hGoHezT@{TyN9R zKTh;8sIpjX5tW=ewwSiUg+!=-bD?=_qIl;eG?UHlZoMz7xF^CdjSmKsyfS8)n^Oh{ zj%1zcx>U_ShvgHD$?;sol zZyCDyw)qV1+7N=mNjXzJ?S7MH%?^3=(z&Lp`f33*Hwo3@Pg&FdcLX|6JbNu;mbSjks^n zX1Skl*Q%8(vLeN^FXtXTz`V#IPT2qxKz2^?hDANxRS8lrT35B2V@G0Usw-Qj9ALEu z^11i%j$dC0Kr}%B*t;g3Dqc8THJ2d$-7pSUFO)5~va%bTWK|=yLkwpIcZqdmPgJmX z1tU5(r&Ts{(WTiT6)5)*irz1t_Q@TqM!L-%rrg|N*voijHjFohj9RJV z9voy{$ONG(&$jloK5S`endCE)KS6u2zwyCoE9@K`B5bi;LJ#P?|9!W$)iD35icx(s zgQ2zcnK8So#Ps;M%`MBpquY#&Yu>eZ3$B?ctHnaHlo3_txKvbljlkzt#0e)=w*sU|ed<>lT~X-2yKAUx%$=Y!(nHs4MBqjy{OuKk4k zd>%IWkujqz8@__)77WUKa_f{W$cj)O9LH{YIq|VCc+H=XZ6sIjOdS#Q`zLTF$qZtw zHs!J(Bodjp@rcu=!El4(`9FTzZblpCr&@4}MDMlvHW*Xi_gZ ztX}>4i_wnW zBfLQURQ3e9u}H}420UG5aW19d3T+iqhifl$tJs!qbI2oFTyyH-FbiO=;=r zt5dYL_7;F*j=Ci`|877r`nSTUTuJbbN;Eb${sc$$y~NozzmuBIrD-HCrCFP(H&ZST z<{`us+(FGNxloF6`H?V%7RJVfsJpB8>Eo3SyGzPf`RF}0-dRm_5jfFJyQpcyUYB(458mB*s$mY>P9u=GL+R}9pgi{->KSxNVzVc{MRci; z-ukWHN6S68y10?ej#=+y?-iY$UDy#s(2|O|(tN0D&>XdfwfQCZ>*lRnxia085XhMi zKPe~wplC|KbwACFxjKN4{hz{|KcgBoY$zB{C*r6slwR%J^7^{5iYHtChs*ZLLO1L1 zM!$WUe6L}ZMz^NQ!y_NogdDl$s1&uYp`^n!wO`zv*6&vP^=^}-C)8?8+b!4qV*fPP zulJ-cJhxcTHraPJtd=|p>UNF`kSO&Y;scJ|80ngvEfxz{RY;Hk6Z)m@02qD-v0pOP zHlSt4j(Y*;BoIwR?49+ThQ5m72O;JTR#f5FI6G3C1t;4RT`d`|;mzKVEyo9YLv1bJ zgjs(?2FwZx#3%@;33G!WXqY)`ogBH6{nA0oh6ypRxGti7GN- z(RE?bwbSxVsSn;as)K6v!pONxEVRd#+H)bvJO<}`YA&k?5;~n%Mb_U3%Ives$Y}4F ziEd7TG?T#tNJXvS`v<(dy8AATocqk@GU*kLJs5gALgeyb%vr$>;ZTtMfJsR(2VeC(%DNEFxg_Itt%XRUX1T5%Wm97&z8Q-P(~lZLwyIT|Y-wi( zg>bxOF|~d~QQerUEvyNhdSo+q_#6;4umj6T zp>LO~aRAHM<5V857<*tw)xk{+X2{32>g~gx!0}Uu$pA0T(~J!dH@#(eKRfeu_B%vO zW9g5m4QY1k+qa?oJZDhQ%kZo5nw>~aJ{_GS%Y)pk6dA|BgHf+@sPX#*UYEK%_Jf<- z1gfX$5nEPtk7VwFdV0-qV~-H6Mwe-8`z_XHGi=(T#kFTHrP?^_36bQl`r8)W-Jg5@ zvtHeQr53$DzyaG^taAHyc1e4~R<~NVNpg6tcs=qLThEFI2m2k4Tlu24v=ZSEt@kLIzyWb< zu$hK8JAx}nU<>MmUG923J9PZfTcHJH9eDHR&4%|e1mww8ShMef09DRZQElZogbrig zuj+66tGC=Md8fsHcAH;M8cX~5?PG@1Gch)9CBik2F8Z6B&S>-c{_^$3&iGDl;DZ-6 z&RJ6~UgqoXC00%#3gPvwd59Z_bSv%xd4x1O=-d8!wgFGiyC<2WBg>r4!0f%fz+I>@ z-q(ju$Ub8EV;3_Sc&KRA(2j_~j-+2H*G>Cymo@|F-cyp?!sCI*p!V5P_Vc%Mn<@b> zou7T_X7I4xZ*E0KMP1V}x}TgpL*0W;W|{lBASTG%kHA2kxjzb|@dYng_+0L^Y1*xu zG-=`x>aln4r_Fs-cPxnAm|m!t%&LSyM1v>Ix)cv*asrA9I|pVrMbI6oC8koE4pH66 zWBd?b<=sh>!S0FGyk}E0`|Bg2Z>nGMX@*r7fUS2?7My2V8_Fj zp87O=NZKb7vn)Kw3pt zL(yPcmy%5{_iUg3=AWYY(i+G71dSN4LyA2i#{E{$m;Xe}__}YzA##cw{CaD)+L}3H z(5f%GLfJzDYUq@Nyk)<<&HpZ@owHU^Sy-1FRosiOXu`SM)~MJ24gJ8ljv_^al;1W+ zy?_^`>6wthQh$d|%^OHU4rbJ@T3T)SEO2*x9yKW_x*G~4JaAVqf}1S2vr9lDmf7CR z%`NF3w?rP`*pUQaHD0JuD4bl+Cgn=F3Sl>g5}1$quL}G&ws^aY*MQs0Q}g57w=)Jd z^Q`7AKxf0hVO+Sv*R56B{i78fEgsdGE8%^{#+G3_B&UMZEQd!0zyFh!)nrDSC6Op7 zcwWkPAt+Pr(W5S0h4BU8FLW9Q5`f*qQ|HjUrn0ED;|b8GksPqMclE&b zj#5acznV`ykdl)p>-Nvk*Pdb=n$Cl{%k`|)ez3gp!Jb;f=?xjfqV0;JCX3_sw3Bl< z!FyGOy;K9W75Bi79myh7uc@fLmKB4AVN_T~>N*3}Vq@e`b}7&DUcPKIX{^h5T#;hC zV_%pv66jRMCLCgCjRpplk+@GE%z)4fX0bj~KI|rBXx+M(Qc+l5d@9R=7fncBgx$rb z-n^C7;?#yEq5G;5@>4#(I4+DcKMp5PR0mNt*7q4dDxmTF;ECamv zPmeN1iR;+1O&c6#6N>t3L?v9b{RDWH*Oj^v;bE2}s_Qle_fAx(J zWVFAKAnL!3V-cb;*1*sFG0;C7G>&0&HvwI9Mp9%2max)yfV(PO073#cj{fejESzZb z_Zn^%cRM&c`rpIS)%2~*rFy;HLL;e7QE4T2Slm_Jbc>k%XhO+%wC{ej@n&414aHj zH9+_6x{u4)$K>5Ci$b?w)(TP$`g>XII>;Ce5byL}k?T7Q3)hOn6 zYUpovu3NwUga4AuRk^Vmym5})KjnGxoEwe9}nTVv5tYHVp zoE@6}AbIDXpkUnV;RS|Rg1HkKho}~h1S=Ea()o7mM0|+7y_1HZb|9Ukj|XOiwnk&~ zH<+^qQf+-oeV8XBJdF7?e81d1-1(@d9&v>0!K zT09iz|Cplp;L?(ON*37p&Ye5&lV%3E@R?9vfiq{#>ZM|6XxJTV`muoq`ClYEj+ipe zud5?51c_CjbK2!ohhiWcQ4^YGVH?T_X{{XFfF+00wWuElFvH&4v}U(%lI7qNCr{ok z-25!Vb=x*cHy7+KZ0(zA*HBiYr?ZSiE8|p}xwK)!=j$>I9ubfIH-{V71*=2cS~INB zlAX@^)gq*O@c^L`ar`aWlwNBTl6<|YZx$clQ$7|!5{W_=@O0TTedHyugdF;u{?GO$ zmn4@JKWK;G{eM@AF8H5H(YP)0*r!@=Sa!?FS?Tfkw=(3&*Vn1@FO63Go!N;MxA%p1 zvN~na((_2;3D35i@O}LmfU57DaD%r)&HAB7mpm*N#L{AcJ{Jf!CZbIT-xn8WU#d-} z3z+b369<{axZvzMdPF3g;a?w53r~DJ;;qRBS3NSo{6TbC`}{01 z;33$Q2|SXvoAd1GFsL^y?}Di9VC(=35-QmE5v_A0y>!f?GOt|mu-1+(k7}-j6mHB! z>x&xLtt~8q|GM;nJXQgvSWS0$mG2bw&rncRVGa0i4C-nq$9mm+b6sQf z=k+!6rui+VC`1}4{q(;rZ>?NT0s34PfFv85BZW^_M~p5ZjiA+*@AGm9M?^Tf3+=(`@$t6TMYTTd<%L0{Hi)Tv68npp0E+VXt#n{+faG zZ&$4|_}cAa+`#U}r^h=DEV@vW7$4nl)LSLR%fPx|?eNklOWokqww9h5txS5?z4&aa z&yN<;XLKrW180=FRN7oV{*>iLTm?s5%73d{_x^>MQyJfdd2{ffk&dn@V+yI2B}G;G}X=94E!O(?|~gIb+6p$o(8bJ~Gg z-kG(ud*Qkfl09o;a&qP1RFoQQ3#7LyU?arLa$>jmFPB>jMkKGmZTaysaUl=G{4g<6 z78iO&QDi}w)#Aq~o{_wrnFaymqjp=q@CRG6*qx)J=V+sUaf?ai;~0n#0Ac?qLeE@x zSD{Y!JHK@WedmZCocymQyCRn$(+=|}S<2cWzP)#PW6A`?k0MnxGp-dvabp51kIvv0jk$`E@U z4H0%FykXrrNVY;Rk8LxK*GcT;giG4{N1=&B0=k&jUNmuma{7VhT}h9?sk5ofENWeKm zLgkbvn-w41(bjuzFsI2v3`}lzj##+hzy31$ZhtOAv0jLm8J!VxIH3BPA9H_-&ADAs zJp)8;<9^*=+1DUK(HoqUbU>^AMIbBoDVEyAt54EH9c3+W?f>EcSz7-dedp zKv#?lk-CTsY30g4U6Z}xLM0c%W7EcsZ5T)N$zZV6jgD@wfH+ZFSw<$OjxM%`y`*>! zQ&?>!jz-X|h3L*2_xfTdNAb76=)cheT>+ zS@4!ils+Lz`WCsVFIFB83L0a&$HU{*4=rZ$pY#1X^N#`&Id8FU$BlzWL;}U_W3}}V zd9x_gfo%VRC}`YIQP8PP)6KKc28%X!(zyIi@ckkS%E_sEJdEd8WI>1fFL=kiTLrs3 zPbisr&S1L&ADCy`PsD+6Xv-F*?c28-{Rn6Sr5TagRG|s&J0j#5=ak&AjP9HAmv-}6fq?i9EV`viIub5rY}&b3ChD-nADJ|DYJyxt zpP>-wpxnt=BAN-pWCYgi<{sAW5UZ<1AaI_W3_bRGoS2;jJr#E|%mEd+af_4-X&F6hN=XvjgRHbi?_VRe`JQwGfTf_JFV+X1#(c)xa=-=!RQNCe0Ax^K z*J!GKS7IHK+J^Ioy}W{_`@G$~Z{JQ@-(rijju~OFnh9@VFEZn2E_JULR&_VmF0Z_U zAQZ=OQLk>rO4vxK?vN}6(*eDE-$cB|6J(I@$i*Lyu>BUF%d-L^zf?#cDn>d~T#rf- z?R|Up%v5g_y9Z`XqB)o+qY@>=0peZzoN8D)6C`hKlFd&KfO&U!RRP5$`19?WT#LoB zhazeCy0LQaW}$-@Q|4w0otU4yoR@&8pABa{N2;P;>eLLkpvbb+%-brI25kw#ywUJ0k z{=^BdRUvsd)rdhK&0XKmT@^7CGtipZeGf(_42U)8C$@Z5Ah-Xgi{uL9(M(%<~= z8}J@Y_Vd2gb!$#}7Wh3vmxYwf^BZq<#PYl(H+hXVSQnpG<1Twx`mIxXafayfL=s|y(^V< z9V2<3rI`>}kg=5818h*zZ9MRIZnnxf+(A3W2XAFOu#p5=o?2ngKvAS2LBZHb-~qDu zY#*QRSCdn08q}{(QYT%-Z6+Dycaa+agKJ+}E=)tyFKCw!oA4LZy;wP??5{Due?r{Y z$M60oz^^1}rHQc`oxg~oSRVQW8aGNe7#klv-Ca4Y!G>pNK2H%YPa#Fq9LeoO45q+T z2sxXHkdL8jGbsZ+g5K$fpk;xdu9^x>;N@X%FTK)1MqO3)Nj=#~R^Oa{7=9Qt#aC%8 z+k?V5sJ)akcJ{S(7k0H$_8fqk|3c;(C#RUfi{7*l8cZ@Ao77tG zC<`113{7i;xIo11At51A500VpmhG6?SZ}|Z?isjbpe4H!WEP9q=biLK6-Dg_Oc8$# zw9koI{{R;NMi(q$Gds%%|JkJMT}pj0SO{16vT4iXcoJZXn{T`E>G37jg1Hw0UIqS2 z%}rX5Fb)UUVb4YkLo)9K4eRsoCNOhZr%S!2;xE1Iz+~ab4OsyLVk^=fBwuQQ>J*BU?)uE8wrUl z?-c!>OfL~0%gP#$K0~=fhwqP@4Wuf0q#q2P0rBpfZ%ZP{OWg4=CJ?ca!kn2r(_&M3Z?%_1>5OO3Qfa&mTT~IHADW z;Oh+u9z_#!wX#{gEyPU zt$PEZDvnL{0Rvp#5BDO8ouTbZ&U~L${7I$fXGtXfZmI>X9~Aa8IQbVtuq^C++55Nd z(7_$R6U=EWniBkvE+Crhdi+>cZaNRek3!10t1z{d%7-S^+jc_<3XgxtqdwffX4-#P z9p@+sXX{#2wAn)X4EYStpFjVrfbaD`B2Za;*Y;D)@QlUtgb9n-&2cYZ))!y)ZeBCq zJ0z&hA8Oy>4-n7tD_=rpR5+|2>RLN`RQ1>;mX^oK#=oo~d6-P6t_(Q6kJ{QGyW>O| zG2Pf$RlHqdVYH7cx0794}wUw6dpBjKD6brnrOXXP+4Gt=wkQr(g}?-~@q3nr+CS5BR(M^Kf2 z*tXqPVy2U*Tara4OM(bFzkaL8Xyhk}ehN_`k(x1J-}-Sc(-E(qtQG2&OuZF}uCHHLNXGr2nG%HaC$iGh**XyKO`s>PFCzCLBKkfiz7B@IL07hy3|r%v4N2$bte7YoboEp zEJ9_%pK^g-vP?cWr+hQpF9}%9g-}d~=ip~|Mh^9V0I9i8PlScp-%VuBlKK&H2?AU~ zelNqGgONp{Xl3f(DVn3UY1u>IJ2)wX7YI_HI;g=y)8j5i+^~tf31s!yAjYF>LojCiRJ;Ou zYWv}R=^#nPP^kFwoF;P5;<~@i@@Tj9L5g)Z&F^KaaenqHJMIiBc{!IDpv9|VzB_-( zbZ>d)%Fu#sc$nGvImO0a-tv~6LoDjV%e%e}IqdK6fwoM*%5w~}iYg{LvE1=s*LxIe zqh!vu$r$e(=5XCl%z?MN>8;xX7LIiH_yNWmw2>eP~{abaC^qkhgJzT2Q} zSVW(TKVSRkt~9MtV9HHP+rAYWnd%;mszE8`C6s*|zml~pz%xeUoC8>yL;oPjf)qn>KyG+tjbJaZDVsVT{!rDJ54Jy(K$p>=%k!L(*^{gI=`R&?<&?Ez-$Ja^G~;8- zw7|!Y9&KYUplTqs@Dl8l6Oj#4)p6bmd+Dlq>sh3ShUR)Pwh*i!WM%kVq8WaTEkk%` zInsioqaE)i@@-0n9w0>TxM_{cLjqG|nG^AbCFSRHn${qq11DoX^p%-4`(T)rG;YS360%}d05ml?K^i~=SzUfPGrcP z#7J`Y?ijp~I-}IW&@EiP z6KRq$ z{`1LR|Bje(lr!fqV=xn9pCXJtxKZGzDAb=jBT)~pL z730IE1VzUsJa|c7DSvJW4XS_Z!^KLtIm5MUBhMf9_TCN$p7O4Y1+?u?{SMhMqTc~N z7Nlha%c4BHOwtuw`4|mRE0J|lurH&i4`6b3 zK(DZ6H8Qqdy>=}>?AC^;hs{3`Lgk;v~=az^}#7JwEu@YUs+ z|C7HJzv4q9wjI&TcIxE+xB?`HktGweVzy1#`&zYZw}SU8z9RNBtR#8;yODOt@vot) z9LE26QupF|#{#Pjlxz0B)^~QtHg0!DZ3n)VTH()X-~PZmzzE8Ehe1fSH7c{IH5ch~Ri1uT9sYbu6h4UFPr5x~<7Lq$L`L%Q zBzFmHa;Em7s7%|31-~)62ybO+nctvV*C0ZpZ{Mbnxgb40CV290!OXx&ox|Lzs``D1 zkhf2t_EyN*5aQRLa?%YLg^$qiX~*ep#_JLOs9GT{;JHo zV5(Q-X0=w0gN0fM2!~nalk1{*m-FOmKQrzJ$ zOuMTVAp@Nu7OV?%A`%LnU(KfwZkN2HdC4hEy%WcId{h>IO5s(zuz>O#p&^gIT40nSf z!DaBEK_bp+#B3G55q^<&ao!9~9XRcpl)0xX7Cc=3-yw6_r&W#Jxwd$svOh-E-y+Eo zEcyX0hf-iB&-OpJ$`|oT1_ae$|si>*geq-U$V*KebQNNst zIbz85Z1wM>-swXKo1R;<~6p1eHLdk(GWF?+_0tpb0~jhu92O_+9YDx?x8&Z(zx`k!HI zDA}qw8+I1*j?WPzXD~B1!y(Nb4!*B}tO5I&+qs`bo&!SbQfk=EXDv_p8Tzp4f2U&* zbVd=Z1#@js)CY)ydHvxAbV(?1ZO@ExXJBLE@zgSPA|Fb0PdwlCyc~(<+iR=K&xC3N z{!65`No6FWWj=Ye`o@cuT?Zc6Gu_SVX@317UV$zKUUVoPCj;p6vUxb^#4u5BrsEOe zWj7SRpQ2iP{%nV%y41bU3DEdtx1h4MK&!-qzNN;i{MxIxNqKSBL4vGo7rQ%7UtuWI93tsLuOQPYLOz z!)pCTs`d}5<>DgaaAl)C?|Hg&>*bHn-;i&)kwOuh2ScUjXPN9s9TN^XFM5L^oT@M+u8)3I(y z=8m7=`Gr@d1NS9h$g8sdW;v?EoMGg0bz>Lo_qNKa|A*^y;;@|8Z~WdFk^-?(PO`z)n=^$ zl82w`t?;Pi%wqk7b(SXMbPP;Lf5Yah7cU3Zm8+V}PZMgdZV@4SabfS&MvWUwGzIzX zHh{x?9O6Z{Cjrq;3o)T=69_J~af7Ht3{#KY18NN93oUZ$DTA$;g^19aeZw-xVzq;r z_BvJLeb4L+;ZNs@Et%gjrt<{N#9tPydC{v+&C(7rSt~(1O&9wVJ0j#ac3S?K|&zw_73wu>iuo%FrVZ%?^>o&+Dx_l<3NXL&~x5y22OY&t$g!fE({6y*cf5w+*$i;*> zM~l}2|fHmy;b3ETzp?Cnr@G+mU#k>th{ z>rGWPmri!MkT!f8Wy&un2EL*hGUWyyr{od65|GNWlt9I#zBAKyCqFyA02_X;FV+lS zTHb5J|6g|5xevdRe090W+UYnzV9gr3=1ohpI46l_BeQy1bl3ZqgKt<4UWCDq@#Pk-(<_cmQ=n}$ zSe!JlaGa8MAGlq~98yd#8_iJ&U{yYx0C)`-*TDX2&vCN;z}E_(iLNm?BBHgpPx>F- z#YQb7%=-_V&vtR+HTST8&O>%Gb(2L`RUbWh&>e2A^Vx^;SOntiL5k2q@&IQHJhUDO zDpXYXT>KPRH_mV$HQ#mXLQY1;E^?TpTbn^sa{(a8>g8)Z6--i#V6w7t>O!Yah@>AV zfVj^PP>36}J=pMxbzYG_{I@A_h9p5=UwjEpq<3q#q(m_#fN+^krfiCi1pkv%ExMU*czfS<~3(JdNbxzR)?_t2cXT!KN-o(V)(4nQWV<`MN&VB<^ z8cxQ-9U%6JIdbd8D2_W~1{X$6i!FN$vMXL1NlhEtsXP?OC5V_F(?`~t?8O6%qPxcD z{jXxP`?=``4%=y5Wu-~O&K4S)ZqreB!-1j&oI^?W(2lBo``XU^QEN|7Lj|4sVI6#u zhJL$os~)>BP@O8$=|o;8N~+=PrQTiQ^=j9t!yC}G<^d0Caf2sZ^EW}#V|VI=dELJWnh43=mVyYXRyRO=8u2VuF(B=pP~WOGmVYKH_s9Y|vZm@Si>M4fUW z*?ke~B4R&aD%D=S8qgf>URBm@V@(#j@SUCnf}4uB#Sxv z)T33Nzs35pMu3~IL^pDl`iCKZMMicbB_w*G85zERU)WkZ_qDRd>lITXYEh}(d{4W6 z#S1&|r(v06cnFv7cV!io(CI-fWN3$Tltd}fER6jADdZrw(oFmRCcg}S2; ztXMr#SNFcv9D^apBP0J<*Y7ta);)O?Y2*+YjRp!7jGT>tQ|$eR_*-?xL1#%y9iMr4 zhp>qnVv`Yz)`!i_iCcRyAnsdjzTL(>yZbC)NdnFeC-wHJ-~28U2tQ*-b^f8vj#_)y ztrw*mK#nAUb=-qYv-r&m4%2W0W4A%K0b@I5XM*XRmpbG5^Ut#`i-TOycz_s-g$ozn z9mdlmhYTWtk!+`PK%gZ|JQN^Sg7i4H?rv8#-lFafC^d$0P{{t_0|pIJMxZml^=Qpq zOldUhq;4qA->u-iFcn$cac|2Xou6=Vf7h;~Lep_y^1r~Nm)eJpy;ZC zswol4Z`VNdiuvJ@PnY)X8}jJkI2^4VF#ol4YsG?UU=VOip8O-fo_vjeMP&K$ARftI z20ih6d97D#WydWk8bK$Wn>IMSBxztxRCH04WM-{K?xTzkMq`A z;1av_E(}rIAL{{)fBHvE&jI>NxR$6gmR9^gl_F>U6M?W13&FN6#w58S2SY84SITYf#Aa9L3vo5#(skrnGPL&Ds+Lz)S3bbrdz34=J`nDM_6 z%5~}~_%s5IeG;p5^!^Yv{j#gqt|>wT;Y!b+I7Fs1UXvo9ifkgDQ4rWiuqkI>Sv7Xu z#~8w(UCQs7Z}52A0th`Rff!K+SOpftBLqLY`F$S9s{F8P*S?ay#k&7S*c%0B-34#8CK;Keo<_|83moF7HnwMIPB{kDJq8!5g=^x`w2G+hz3_n&T6j zQ}X7-I$VYSPljUWoA5l`110_z9UC2IH75f-wZ5}t3v0M)>}{veu9foZ@6b$ z3?WSlnZR99Njf=?GZ4|E1nqDj{Wl!r{J%S@QtKz?&j22g{G27rk10tyxy-LZu*8|$G4fX9YefHpyu+p|64>!aD&gYEUpCyyr7Qb8<`Gt$#7X&Zm@_t1s6-Z>6G6}(|2;zj&DYu2rkzPod|Z1Q+Lu%e=k z{O7;aGC;7#mCWQ|YKDszomiYHQOc|}3iTlp?VnM~5yd0n3Wd}FWo6Fq!WaoFx?X&$ z(z*iYK)c<0pwkse!n!pR(ERv%SicS@AkznL*6Jpfhaw@$hm2-wXohTLR5p^Uc~`s z+Yzo<@JXJ3pa)ez5Il17t8J=G9jEDw-uPLE!)(N#0{nGYITw+e*J0XvFKbmFAJ7UU z<*0ttLhYdz9)DcKe0fj&!oZ?)rVwNWOSUe4Eq2^pUPK}fI%X8U3ino6*oFg~;TubO zqhQSLJP(tXv_RuP88uin6{zKi$8UXSKVLVkn{wK6+PsH46zt6!)`LA#m1p!8r%T%j zr39qbLx!)3a$r3?}hew7>c zH*=@0(9BC*Wdcd_uF1*_aqfJ-X7rQXDEr6#+BLzlX!-t;l@~%%@U+4O6^2dg*Ae?j z%&>rpqr_y6T&#Gc4GgMxiAtvbVz?#wGB*ANy(j5=JoWz;d#mE(ZUckH{LS8P2S(Lx zaPH8)mnA<(c#>|WUS5_f@$q5!KO9!yBAiD7&Ri>s^U^wffe8k9jx>r3voG`5z2wQ~ zpfE1vwpZkp8=ti*{BbVbh6s+XNQ^iz$1|DZiv6gKAn!Q`khD4BcH0dmPX@xu$E%e^ z0Q{o4cx6|BS5@b~(mUZ%6 z&30^=cl2m`i5=XTjrj+iPsz%`99iH#KsFOh=f!WXJfAb(S1lz z6w0E1EhGp*G}u@{Y^A3+!9LEVdr;Xd-gZP5$u^V@jFO0M)+=v^>MG~?F&ok@R59&{ z1U`H&-03c3^f9f_$9Sq1Wqj@kI!Cy^WIfpJKXv-F@K4i2qf7TXWnmP{T6>C9X9)#3 zev(Jv~umDY8o%7zPPvVtd0H0)0)2j zUtMI``I@IaUxBz64F0*Ws-9Ahn(M2;Zz>Ms#c|ZD?>p0Ecw^oy+o{Su3oVBJ*>`)B zzE;;=>oq}F-Xm+fGq4|s^6d)S$ddz}`gH7tY zd>~Z22OuM7d&>7RpeJ@s>VL+d9zhVp;rKy&T21GHL>a^2vDv2XJfFWx1YJD-j0>kE z{qFz>rbr#VN;KacL&kv4~1cXQ^k*4{#BAnhTgM~MX_-Fh$JSiOVEKJv!kCdJKb z#vHq{h$5nrn3cu%3dn$A%JOc;#OtB&d2LqQLc{79u(i!h-dZxj=k(Ln?Z_hSM&R#1pX?7K?nxZJ(TyTt}a@Eo^z3Y*3rBH+jAk{*bMb3q7JX%zpyi`*6IL_>f zlyfPm`KcCj*xw^)*!{(qp1(;8791|US^YZl4k%6>#e=Ftvn)Dn{(|D&^xEmjy?XGT zKLeyJyX@b;f7A2RN=UXauaLy9d!ZbCz)x6bULuQ`P2|S^C=S-kTaY8}LHU}{=-*zJ z>cJmZp7`t0`87>dgO+I&YIg5tQfGmZPUCOW!*pI0Qpu~e@$AKm zTA=2#D8L(`0P2pO-v3Eo{Dp@Vx&j{aE^0bQegDY}}ePnsrA@)BTc+KO{is$dT8N ziq`LrN#=F8%*d-Z`Z|1aTO-W~mYcflF`N8z*aqYxnW>yIl42iEt&O+_Im`2N-)k`7 zCnwkkWb_Jp%M(dcQUV^Y^JaWNl6)yRPB7%Jl>U5(Y_BA?M~?N{&#N|$4NWh8T1VNr zrMv))7@sS1`;yyylcxFSr5$GD%Y#0V?&dMcZn|MK1xNXNcJGzX>V4cdQpkuLs&3tt_tv45 zBIX39dmN|Bc{|MxpVMcV3xZt(>ZqF2+#@w!1KWqJZBAY1O@LnRfa)IwgQB(5&Eg_% zFci_^yK?2qxnHd&~rrQ_-670im%N2>4LSNg_atCF$e_ioIJ%C z#QB$xt2Qx*l-h}yPDbE!&q$#JpR{Y1Hxd}HE+3Tm0_oD(#!GP1c3u3k<&ZRDFOoTj zw@R*@g#*)49_p^U4E$ic99doaVsLsdmSGB&(v62C1)T^gXfzh z=3BZgNLn6u!!n`|CKToXCSofqI_A{Zs>^H!$cY2$kYX`o#Lr%kefaUlL+ zrrO_q5id)!$@$NhDGvk!hpj3&43w`RXlR>RJ;G&j7y_!h35XX>Lk|^I@6Ndee?XwH zYG^-*5#o8V;VF9#7NP?_DtbW#84e;6D@& z;`iUrD%e^&UN~O)ze?l6;|idmnmo0)--18qe;E;RN1>>E$@5X<*Qa6an$7oXeAxbE z3q;r#Oz$}_sv)V8n(-cZ>MZ)X`xgK8%NOEV{pRw}m0`KDXV?45xSID*}Z<6*)@{sSRE=C%Y0 zW>|8A7?_)OM~lmhpX(?TWsIT{SAkwF$`Nv16w|~xB242th;F2x;?CeN<07xxVUa^n z?RiIG{SZ;YBg`FtQc5kqg-}(b;7z$dj|)5S*{6l8gOyke`DN$? zo5Zyqw_>$dezMB7*0htnbEFcv?SwHoIPoOsFutOAkIDO@B12~9+-or>$ZFCKnCx-J zts)tmND(Z?ulhC*sLoQKy$Qk&Xg21QH#&(g46RAHeQtIH9&1LP{* z>)=5V%qMT|#g&v?U{tbkJ3Xsh{?9gt-KHqw;GHhg{D`vD=IENiUAdKwI+D9%wlX7%SZ4y>}}J ztzU%^7w7eDcV+&?In}=-Rf>DP22|sx^ekuiV{1*iBwETCrR@1oaRDg*x&V58_HUs0 z2ael;$10qRsM0bhno@V=t2^z981!z|)@lNg_SD)?+SVhT$^bF6faUZ)`k@e=qa?5h zll%Cu%P54IZX`l5aR^LBHv~15q$w1%KRbGK1D1w^CW|MKc?EC2Jmg3pvSN6LNgQfI z3upp#Eh|83$VtsSE@bJH_*zyrITAuq%xc*^&=n;$L9mkt52)!S(tU6{A)&Xobt%sx!m%+O)~{Vls@G`9;iK->b|O`QzEjvRSk!j3N^wS* zW%oJ~G1|Ay+pRkQXTY#Ff}r&2+ED0wE_5-%=-#Utl6guFr;~iOiY&S)t8lN^eBvn* z9<@l+^t79Z_Ib=Qrz5}`JjVWB8~f?$b!HcJH4B^7^>@$k0AgWBoUW}@V+;(6e$H`8 zY;q4?>+H@$+BcOcRdn*B!u20#W@^m4*HnqwzT(GJgRjlm_JySvit7@VeOUxXa%y4O z5d+!>pQp=lg7xXMC!=K9RAxHxtF>7_dSVTOIVgyml;8Tpx)C-uHjGb|k4rZ2jf*l? z+xE%z+%y9-ONyhBxQ&GFuq?rV;8a1Pp>)J>5Bfsu}^hLiG4HzZf?Kc_lD?O!tdm89U1~_NRH_IEBaWqbfKJED)Jmv z>i)!l+%>E2)+8R=4*4PFC-8S2JQDI8L&!q?bM>o9R1#hXKR9=;u}A@Zd%3I1@gmA~ zs5}Wv=6A67JO29#MjGzZczR%sQx;#sU?nPvnzF^WM`LD0qn2TFmKe=R245$U!GaEU zNwY;Hm0#kwiBhwSLl-^1G~goUXNBNb88$0>L*)vUpjK&tI^~Y19)0ehROtvm$Dz-G zb;O3F9BX2t;)c=IVVuc)Z}mKV`DE4N7V|&x+u3=S>p|g zGX;dfVb*5SkZWu!#9R>hs1$Sc$nn@%|JU1+i+&kLc-6$$sP0*&$cTEh`HNm3PJVTF z(r*6Bs-Fx0hgV^iS{r50+NgUa1FfoR^BmF2e}nxzZ998lv@$-KH@4$@sCLO9J6C1E zobc`M`&b<%Y!RShhPioB=@TFs3GTf#x>@;!TrVy(qloPldr#F>cHw!=W z2IS#47(IRZf!c3#Ruo5rW*0?dWJ)Ow8ZJc#V+1wiI&9DkF$K+usbPuXCsJ|Yy)Mcv z`8lBsL5%s`0yb{Lr{`PGwa1p3}4Nr zq1}|7-53HYimIyy$lY0jl1W|w@w%%kARNy!Qtz0T_zUlGkzdjO%+s`(R=8&fR2NFI zWnnE~EKw)QU+tU9>+$1V+#-3A*ydeDfb1(wmwY)kUOz_Bk`gp$>DnvPbB2|;E>Kmq zpOLd{{Ngk9uPXI#bF>k^BTr`{kYe?ODW{a46Zr5g%H>DYuxq<-p%jtujtcy+#`rm2 zlF%gpzi!-aCkP|yf568xA(BuZv1P-)e6~Kko{b)gNc<=q%)pkbfVJI%-e{L`sLSjs zqZSfk&Uy#W=FWfe;~}H|cdiZNLY4xtlIu(aoAb$&{OMmw_RK#KEX3rE%DhB7bcBhN z_Hf}-=1iB}9O);O-t%7_irRE29=$zN-oLTVc7R))$M70V|PAvTCikRe-=D4q{`lu?G*ZR2;Yk1$* zAsbYo6(Hs>%Yte1I(u1Hg;vvh>806!Q*{wq{%)kF!Pi7$m?RL2g)OOY5E0H?^5{yI zKB!8uwYOhCBK1&UVD;m&H5eQwSy%*Ty}tpc7Z^Kj>l z*OCW>epi<~zfnhm@@+&@gpmrvtXiA2ref8k&lW;Wn&kbulRx+Hp?HeOmT%FE@GWt! zW4*4vBWOTpDi^}yzS}(b_=mzFl3%Du3GcpleBNILBmAfS8>t*+vDz67@vw{QERGER zg2M>WG7rm`tRICw`NqNn9=Ye=mTU}Y!h^$?&uF;?6fsf)1SPeuxRzI|=8Z5Iccfms zU1tOcYO#2g+nEoS7jCjW@NkCHcw5`PVzCGOoUBDI34q;8Y8`zdnyuln+$D~xIOloi zIOb&WajAT7&U~)x)2nCCI~PVTB$Zf{XUfVi()vDX%d^N&vkjNG_?A+r?*ngC9K%e!Z#&E{$4Bk(g>B*zjvWX<&X32 zM{afhbP0lC08`+qMvij%t(rdl@4s}qcMm%D1$8wy!?N>;(-z-x*LZD3j{Ze`J)Q-0 zB^+4CO8g9ZfF8Pm%+6Us$b~~`fcr5*^?Qxzq8t6Yj%WN^3X}vhie&A+G^kQjAy8(S zm7Ip=@S)&g=M+N*4NHNWE4h_L#47;l0}mykgZKoBA3ON=KXUKF+ZXv6s_a5q!-v0n z+_ERqqhka0tXUWo(Xp}9$b@AY6(2cpP{B@Swq?(neR}8fke9WMk%7S^B;mI>cuw1X zl&6jTS3=dcT0WWueyO73F&?qWRJMPE9D1mlDh%x`mVkPvrmF@;FJfrH3p2*S;fwcv zf(aJlK9e7wth0}lW9LDp{FlV1E4$aR=|qZ?FnH;b_+yEKae5yE>@97W2ukx$F&%ye zSiNO?SSJWyot~mDYf8;SI~GNOFk!LH9a%A`M^w~aqg~61IkH3>DgmUqBMt?{`U_cC zvjJx%dbJ=UWaJll(%syulw|oGAf`h3Dz3Zk3${G8&~wuDKaN@vU-GOqI=$<$dVL0W zU+ebIXYad4NXNE9O|H0YG5FX@Qp*&2;Sc+f59V<=OOmLRcqZ^EN>$MQ{@=K&m{rI? z{?@|nWvEJsR8=C~p~@iJVp!E)raRce)?D-=u7bp`x984FZF^2x*0xb>&lvLd0|iBE zy;bws$H{;lO@`@ns33()R_J~iirwBUAja$SkD~a=c_?GB;dj;6Hf044%kjdO{!;L8 z=^$ilE=eA*<)zXHoI?;aASSf(>!;|IEsDw_&F+cPVe9&HE)*|6s{4AvkVOa#q=Av8 z=ah(W0n#h4zPO%Wu+JfObX(6bM%aTMN6=mqT>L5hXruYn>8@x`lgiL5KD0*4?y=iZ5B;&T~L(nA=*&!@6F!&%HQZwMdZ@ zC0D`0ZAmHs93TzGrBCZ9LA~rc-qj9IYZc`aJQ_KLy6!g?Uia*7*i!uIsoYthTnE$D z;@fpo^|=VjBq4^X#4_ah109Wnv~$?{`S~@)3}Dn=wU`Q) z-C&`akv=+r%8U4~9IEt=5|S8|VIpl&C2Fcc`ZZ|K76~|XhT@$+WX$G!c2N?PkHlod z@-JDu+KXVuTsBl4oiLHbLFA57RZOttc7>kQ{Y@$!ac*@o`^TD_=gxZ#L@1LeexFZh ze|#o#hzXFMwaFVF`}08hRKDr$kViP_-Y5=`LSXGb@O%i*g%O_ z0AJyBH7>;4jBx6C?Y_LQ!a)~mNTHmTaJS&^m zW?7t0)q0&ZYTLHB*Bw`D#=+fPTdir+vDcfS!>2y^GJn&$)Q{!+*Ttpc8~0jc)v=Rd zr-&_+G+TOG*EShErcNj8Hy4Xq=sljR*}L~jos=JI7p(19c6#|6o&L|({PAq93#3Nk z`%wmeSx-2)9h*xqX}~iwYGet@?b?;zt=|0ZbyIV5b8Um}K~ay>(oVEbynTDr1=A*z z@-X0SP+L`fBMP;Rof1|-H+wGKn1FU_RBGr+I)KrJzLB~Lt3@K)KFf2`8|_j2 zBi*~L@74Y2(pM4bhP8}%Ks%)sz-2iAFs*+dpu{agPdlP`9mW=0g4&x?u% z419KvSRSmg4P_H=s}pWcDg_Md=t-0A#uOx{rY7P)rU z5Vb9iNhq-FQsO?(oIh{Cutt!$5E5XV4JQ~i@kE1#DoIitqOG)jDMQvm_}fs?R~_a> zc?`>&V>S^#lq)neGWfc8(y1{-B#Dn9T1+<39UGsG@dv(UL36L)BQ`eH)@Y(;gOGV$ z(;#pb4s1AKY`%Ypwmj(<-F4iT%B;XfJ^18cq6ZAjfNOB7f?Ubo;mU(CYnf#$keHP@4G< zkBmh0%0pR@3u#0B*{L(--@g4Hy3PbH=e+y-_AOhMA`}UUtjSi^qU>QXX^NOcgtD|L zyHvL9l`~8r~(#z1jrIi@{0uMT( zy23f9u9>oWL>z~rV@-&GhTIU3D!JIuQnGC@E`CVx*N69_CKUdS0#G38Lfu3^>j+%c z8yQM&FLe6$H7RvozjqypH{;8-sZ^!qlIE$bN(Wt%Ymlqd#^(Oh9{sM`X_L48lznS{ zU8y@_y|3!sP!28ybDCmiGT%EPws&UPHKq83_BeI0ZEVj}v*5zWrMLCb#y zyCo!GgXlQJn>BA9iWCD`LD_)?hOzdI`am?g6l%Ce`ZRvU%xNhFkrIRdr>FL?ly+4=}UNCE5VCq22d57%J zyHtGcPY}YNC(&fQ&W=uBP~-^yc}c=(xh#DGKy*%oc=`}g)_&ctQ@+*YZ>3Eirfs z93_kdzb?D-6-Qpj>U!zek@@C+HH!xG1les=}kVW8l>ripFP}K27~Qvgh#O>?!JmS}xqmVfk{|F%E~&x6eZd zxiYva1X8qe9(F`RwyP?NMy40%y$92gp_|uQU|BP3wD9t(cD>VoF5rS_T!aipdO(Xu zY1pV^?)dFeRMa0Nn~4?^(oVhkAG0Zk;`GU_Dj)pi6SfLjEg}T?Q*aWK4+FpmwuOX* zgtlMAN^v^5dSTjMqw#K5l!a99OXg-pENA^`IG&MH?LK|@&R=NenIlYmm_AS_oUI#5G=&hOgQ0>OX%<6SI+bj-&9M1>wW zbm;S=YN$NPomIr0Wzd9oM{*Y#yw`glx(9YGw2fjQ8NImB21o-dRzWF;fH>Q0pR_Cu zMExn3I=dw_kTI$5vwtRVV!Nh3WlNdS!E9TTVCBr>LRn)+F}9HWz>Pu9W+opDq4tj* zYFPDH+>(5?_X;_cyrOLAwjl4giFliA&)i=k65P>?)eM<3a9|v}bm>9Kc?yXw7yP@A z&o2x&Nhh|Zg_^I^{I}fK{ZTAgGuSDYq$Urn#i--6?pwKxXm$MK|0vmm9HMh~Q3Dj37oQ6Fs2WER&K-5#1j=3G~{>WMvq z1GeE*p$!9yZAVSb=1{o0%@lbbB)oaYj-E>fE(<@xgo2>Q)d;aDzb2|8&shUo?TJPF zm*j$z(c8zT)jqo_e0W5gjP}>5l^!tK*t6o{v7((VQ`OAo6fp(XQG znlzanv?wqKqRC;Gs>kc@~p00Ybx?3F2jw~D=l6F z83IAW*GekZku@Q>`z7>`l2qtg&$6F5_t7={Vlkt}JNc|LOar2mBfb z4xaRRjI(xMCw^W9!!gjV&sSI%p(a0ny|AN>@`=^AqV41x*+#^Z+5S4R=GykWF1GNm z|3;jMeKRtf27?m$_|0su_6$}?1^5Jrr@T8^Ix1RqUv&u9u z*f}Dw?BnoOs`~TR8TJU+)^SdUJmY4!j0s4*iDGo4fXTl+ekxk+isya3HgGLpT@^t- z{btwz*;?Ur7LAzj^(eXvOKiQ^d?A=L3iwRgRY^`_GnTFOfV;%P`wYV_FT7@FG9?EU8t*siCU~S$L9$c0N;=^!&2~D!8{yJOgta4`PyZ6u`Z*sM6 z=c8)?SiE~|S6lm)%qp`DqRwY66@J{>I{50$NF3K=`mZ5{qg}fdr&}vxGm1a@l&&fo z<^7^$^18{T6i(DZnHrO;$&$7=-ze}*PF zWcGo?SD`O*1A37l5^(w_;UFgi4)3msX?z!dIOPs1kmqH?ZC=fb-ASed?Y@F5hF&7kX;!JMFt{ZlEk4Ts|rDH&f`6-SFc_p zo_!uKoF8qYK+6*jG82!1tF!g&T1x_J8R_KiH^x*(V?d48-MPUUfO0Fqi1XwomMduK zSFomCs^lx-3Q^P4Kd$X2EEjnL&ye-57~LE}l~OQ5qQ;~=-XYOSerHZN~JX)>~?=wL@4 zUrL5M%_`i z?+t!T7_qU>zfTzHTW;|98|{vW=!;%J0t+&E{vq2p%qWRnkX!Vmtx;)#FKm?jLS38{VbIY+~>moB;)UB&(Poj@w2i1}t zL9m5KP4Z|Sk6ySioq|x&j@Vlo^Rv?m;`!yCs7+t@HvZG&wAPA1@?WO=vDsxyLFe~v z*2H$Al&yAn9l5*1G}a7lm+DGcTpc?_%&fpNiTCS#VDjO|`8>Lp-r2-tAV|U=?Hmj= z#L+t~V9N;^FwuKwyT?(xWo=^Hw%v5w?NX4evX`>O;(haNyg)_sAANne`ocThvUmH-|yp{VWiws(v{p-aRc1N%A46U&!E zSZ_h2iD%p+7`kpa@Mv*FqTYkMwbnDoM<A0~^ zVcZq!XSa^I#ZsJkn=2uW9Lws39jgJ51{|I@;IQ@Ak7cBulm_X^&rui&PiHyl$5d?BVfORyoljTu zR)4uKQ(~p3{~FQZ!Il+ypO@0LDR~MsD+@5~g&!m%V%RVZsF1wt@#6^hk)2**-&9=9 zCqQv5^FL~){(XlEG(bPnQ=>UTA4~j%9#*lUv#7qpjQitjOHr%#D_f-$^VZlmn@v8Rl%S`i>xPhOmR8X z@Ah7kKnx#xHMM=4&5Y=(*_}x?cdu7NhYm&BqTZSf*qQ{W${E;_WNUFRlT=@%WG~L9 zubx!tfx7EiQPHOdL$}+P2Dr7ji6HxFe!kQKCt^6|sdCH=BRW<X^$ev89I3zmDQ8 zln1z5ds~a(VBn&(%bh+M6}^4C3+Q?tY<*CR@hY>>;N}I-eVSENiA-T%k^1dp+|#^A zu&{Z@Ki&TF^{Z`0*ktiIBgHU6xAA2e*ntR758VCn9T(xKWkF$v7tu(|V*7rNHTxYiJKT_by?Z_JkiI_AQ(yvt0i zx_?alGSlu@ylY~6$rSHeMd^5Hb;dB21uX(Z7)>~qcphTx@Z1y4M8`7J%hN)zp$O|8 zM9E2h2-?cE4soBYP+kBLkYM8b`t?$}Mdtyxw4-d3nqZ`1;fr-Ra3vL_+49GKvh>uu zFo*F7VO&9eJ)thkV+`W+q`<*%cOg=PQ* zZ9AEQL5tRfxf{P+Wt2YEnWEE%6DqlN4w7n$YPU|EIydVFHWYwpCB!nawW9_mzB53F zY=htmW+gjHh2jVpc_OzGWE=6`+B7qhYR+xA%h}rLYL7zPJTA{63Q6o`8YX!3MHT^$ z8{=G>XG#Q}t|Sl{y&a|!mUbV}{uUG^t@~K+gByQbm~oa3YKTc_I+!>EQ@6^(5U?X3 zH_~k}##B!UX%^<0j{Mee5;U5UASg3RDFzxP-9sGCCFf}g^WhXU*Hx?T>ETjptx@aU zr%z}jgLo+hbLn=Z@dU6g#Er>X3s8KV>N5qw5w&slmH6tW`v6Z_cs3C z1U2I%i=hNICu78${Cw$6Y3baO7HM0?i8F-`$&?HD8UJzG7g)2Nlc!9Puo}{H9!>0k zl2>f&^We7mg+hupn;3+U0{+LGgC#Na!7^tXov7K}esL7(a`?mfM&y^c#>?Cv%b;X_ zC_$m;5jKXwEg0zOq}!1js;E*r*Hi&-bHaAoocZ?k!Y`eb#Y8y*{GqNRjN~&a2uQNM z20{2Pd1Tc}Yqg%1z*xl6v~6a}RJ#-B8E5bAmoi_mgaCKhGBF7p%Cex0mm{*yP^gjV z@GHpE#{HeO-d^<)JwInbTKh+m3~*v(IuymV9GS2u1V?R3d~mxdp^1An%LhED!s+v`(_sNZyo=C zY-M2#)fXL7EqJ?v<9L=Mw7n@{jQ-zy&A>YLW*^uRCJKq-Fn3F3?8+@+2U18QdGStnI#(Sn){S z^4FQq12E3_Nh{aAj0mI?jJBX(eTZ0Fnx~m2CS^AWF`)GrFMXJ(iqc1XYpSuM@JK>p z*3wyiCp@e0d^QeDEnKs#8>foa=Ni5UEL7mqTeBDN`?4c6cX(nlqc0Z)FVs|j(4hUv z+f7s+Tev2^or_`L(oHQ!jB;|iuzq3Wa*7oeJBxerI<>U=ifL$3iI5A|SUl?+5o4qP2>$hN6pS%(T1X<&*iSmH*(f>^Q?{}>1FwZ(R zCpF)sWs#O=9c|JxhfU_ULB;J@cH3qguJ(lkZU-vtgUXAG7Nq4XoB!oC&HoR*YO31L z=70DqzbQwPhxKFt-qZ5Q1M^jWlRH1K={3w}_TTKmg5Q)VqD;|N)6H}lJb3oJ3z^yM zI&ILfp$JZ6a0>WJz#nG8GAD%>fB@5iX7?T;l}8TfG#120tLoi0(}6ELqw-`8Y8N12 zG5(2GBYQopl@c@Jn%V>%Mb~D4*CxM*MEE#QN;{&oy}XVU!?NX25LFY@rBilUN9~{%pJvj(FvgWy^l? z8>G>UDwgxgozIlYB<-)0@B-Z6OW#Z|n8vtm@TK-!eV*7uidCcjxG4bv?=YHury>4T63&h%q-#(9q~PNNqag&=ul>FGuQ z-soiMPuWVfazqzJQ-bT&XPrh1F>zvRf~4(|2sc{s>fWivMIF%pFTljpsAqqT zN70cY=bE$dVn6ad7K2Udp5a$sv-YB9Aa=~Y_tn?oRO)e{7vRyv_i zG4};roAO8|WAj&K`bY9ltIt-6oB5AZ64W}?xX}X-w zB4R^1&fwq*uN3;DE`0p>CywTnCaE#xMvMJ^$5l9Z=@00)%~o89$;i$=-ERE2aZM(S zxj2LZq<4#6erv?r?A#Hc@pp z5W`<5lWyJ8h7l4sXZi#7_o7m(A;QlN9+h*73yn+9)ZP8t#a+;{NG=!9;#+@o^;V0J z6t1C8HfI^l#peJdMf3S^kHd>2tAFkOPk+NMQWU0jTG0B%rAhNnnRlfq z3^+VV){4GM9h{re>@9q`TrCpLsQF|!SF7}zqJ3^2Sd!a17*t2nwYe+PT*dSzz5Myl zmE02YIoe10F3;P5gAp!R;)w~qJcc_5MAh*dvU_%u)Tdc8n>tlwe9Un5b;7=Y`fLJe z@Tji*ke7r4slXDx7+^1Hi*~>*fN3@y_0661hE!JyugAM< z4L9zMkA)pW)-p$P>awE~P2NtCy6~T)VDRzdI0JPifOBZanAq+_A*5kx_x)ZS;sEQe z!M*$#i$_QRY3Z6VW3CThC(XF5dSrL=K@m(a@VlOPJJBQ^=1=bybapZmYQ5v79r-*& z!erNt(o_-MB?4iU#q{Z+CmHobySK5GYXrgstb2Fz5dy88?_HQh$$YG_bDXIu=F(|A zeW72vz?2;s_PkuvzF)g{GU4j#+CK<(c(us)>eZ{ie#ZK?}zF1h1?e!hmXa`ZE~y7!Th?O z{0cn~#DC$pVU78_qWI&zA*i7E_&1JM=(%&d%c|*mU}@1&aLZgVo&|Q!y;Pe*sQh{? z!*KojsrEZ#bj(3AKtwYfNfs+9ei^>tuA#YqzZ&f-^l^TE-5N-=bpL`KTZ@%jP8G3A z+w#vx|5>kLwUk4GzQ-@|h6+_s#xcJ~H2XH|eVsQ}+js0Rt;t+|{$qEqXt0l|n{)qO z6S)dlP;(K~h2UpqF{X3Eimf$Lx6-ook$1$nLpkGI&zlmGy_d0Ho4789N5Vl(=^()e zUth|ly35qSjo2BV4r(HVV5ipKG|=Lq--9*|l0Ie@EC6GLESXRy?*{0B2VpXQe>p0u zEhpjqiv@KpJ;-1M6Wu_z!ks9Dq!b-QJSOultW{T4{iEHd@^WtQf_PC$!1zJ{zaKHf zW~y7f^|V%&OD{P-@d3e_Z=09Z>|cK!1TIY}f31U-mdCrqH86^ub^f1`bzAG}$aZr0 z9iv&NX5xAhaaVX4mjx z!yfKlA9stk@5xRJSY0GbU_sXZ!Gi2PB@TfG!3U3XGGNy;fpyxVL6Eh8K+0_gv6h3& z$LT@QrYV(6uQ_)!`49#k`$VG&Qqo2f@?!{oS_vBeaOvOQe*K8mw$m`?bNkV^z170F zYI?nKF8lb=l&=3ON|P{|gaNleFW{Bx?( zK3Xnv3(rFY9O*iAHq!-upnmW{$13SoA?o7L@+G+}IZ|{+crbfCdEni)es6nT{PyKX z+oKYg_3Y+vkz?`eXzyB)`TDB&jqSDcdyGuq&!H3H2tBt`39I2`=n>(;!U##4Wg5!A zcWQ={nATGEbp4Dj$EZj}fmx@Ep`jXgo6}jYbe?p{8ptRrUqv-yE<<}giuVCPm>so+ zmtb_n(JQb4BPgz^ZN4Ecr|46k@gIZ_uNAC6UAQ!lkN0O+RFgQCvWaXVNjMh5^cZQt z99`O(m^+LtEMj@w&e~r`>IV{x^AG2}wSQG79$odJprFkWeNTiUK@bQiyJU@-|IGz6 zLVray5Z-R;x_H&pw)C^e!GBeBehCaHz#NK<`rQN9DM28>)atFOsmC~pNke#j4K_VH zlDK@#m^vJ#yyc5oMadwS^ToZjz8z^YJ}XC!szsh2*FAd-`d{{}ykg`DJp9$3+bI)vNpNgMq^a-&tzt+jxw@E*@s(?csbh_#3(a)pqT4YYqTg$av@4 z3<4SVD9>6nhvIo=#K9+yYh&N^%PhOTJ>?rgnq&NRTwhZy5j}0#ZZs8#sm#0OUks6y zvh2M+X199Xx>{al8f}wL97kG6XUkEfma^z&#Im|BN2Y8O1sqq1`tGHk6u{U)1$w%= zEy=G=i9~gU@y5bp;-pD8SwUW}o3`6v@u)MpDspp2mLOT&;KDR~4sB!F+-?!hcY`=B{ctx7Mf@FS8#4amS~B*YP%sh8k_;JLALld-fb0%kE4HJ?5g1r4Pl_v*jeH zAWF4LT=lXQnOF!Sn{0S#y&QCsgi%?yW5rM|q1#oTlShs@k&&9^OzRDAlh2^ zW3DCY;@VL6oGm5@%vig`^MU_vbPmT%7vYTP^ZbGcTGsT3o`|xzch7!|8R)+12rY{a-+`;yP`foVa{nAHvJf zchn0DP@ahXyY71z?qYQ!`vyV}?B3n4SMJ@LTrah%DLX?E` zT>Ea7al4Aj(D5Ul0zoI=m6f&L?PEWmZIvry!vYT+=-$1lYW#T|Zm358Ue>%#Ep#h& z&~nOB@bwGJZrDZsfr-k3VECs|U5U98bJ2)(meK^d(3cL)+@=w0qp=A{r&Ysqq9;UkeaL5;k#^R{XmT6iLjSxDY)HJ6mq9 zk)^i%)X^v!cn%T!=uvDNQTFUx#-5#ZBRzfjE$6O1di0=+1Q&gL>E$cvLFbCrMhgH+ zqqF-D88YNpC0jt`9KO_U@~C3FW+L`BufJo}mi-qCoCF8zYYv@OFDnfIurZU8x_fI%?4)IB1Bt@359gIp zh-ri-g%4tp0e7~*n1`0GVSS|TEsoko|CahA7{JM9^|53OIJ2&rUMR`-8J@WptOlDj z>Fyb0gchu8scZiAQ!70HYQ)=hTDxlUW=cY@KnHLGk+*g9?IpxRsmzyWh7*mKO}$N< zKidVS+EK2ynB8`G!p3iA5Fh1+MpPO~7lEYA$S^Rg`rVBdBu@ zhDFZB(YSVD|MB9ddtG*&iBb@x#q{jaqodH&X{SqtMy_mn_VQ(yTd7UA8*Ko2Vw^=% z*!V=o(2@}nX-E-rNj(zxfE04nk9p-9v`TrF{SB6rS0NjMr86ruO3-Db7j4|ciDh3G z%+L;CJ*#X7e3E7(cq6oj2+O!#T%+D*ouxr?l6`ve^`Vc4X~WJn2krFz*Xvh1 zeeJMiySG}eJAZD*SYv)|n8G6N5jdBaIi$jcOHLV%h|jMh(~aD&6Bwsd3W+RKVRV`& z6PIg1S%c2v*>o=2Y1{uiZ|~E@>TiiL?Wzpu+s`AHV!Ku|p`Z}QQE9S0d5i>p78Xr^ z?j^wukP+h@SgKeOB`7hcL@$%l&A3hETHikHcqV&FgM?R3li)SzgG8N-mabYy(keUV-wV?)`LO2on&*@M%81YLE}TxB z60)?miM~)Gk|0q(EWlV)2o9aEMyBsHF&IfPYLlOmVkTqRqfdwdhSFrTi6~?Ls^Q&k zZYmhMge?m_5FFf`O+dPPihQCFmTNBBi4?550#WYnN?T9rX(V}HLn?Tq6=kepIS!Ey~NiK%z zH(Dz4U`p|a9T(unr*ZyhQRyJ&V%Ko2@g-!5uXEx9n>lXJ@uxCF{kDxlg_02FZDhq_ z0m=U&0`p$Uc@hQX+7s7?e`;R_U#0(H=up?u>B-%-1A0NqvutbTJ%U8ek%7dS6bG6Z zb0x8gKeqVOc07i#qDA3*W@C_&9syjc%}1fTuoP~buCS$X7c%bXrAy_C>l~Z+e-xqIVRrG!nTZ zYgmTZ#oReJ`X=up63LRsU+#SV`t`bU$;&k&WH}>-hbI}OoAbaMv@xkXU8{DjTIMFB zXWo(KDCI}isUr62ee9Baxj96CT^8Dv)g*t;tt%L5LyoFkbX+f1M9_}?{rBU^gI_C( zKgw#Es;W7<_qNu93Ezd*3SK z2BfEV3t%kPl!nXW%LOah3@D1cCTt zmMqazY6nTUCeW?v*;&I+r<)d(i1=|tpkvkr(`g#%veXNm-rxD;QE(?4-aTIbCET{7 zip`rxQNF&j()v>%P`jeS_wud;h|?`PyzXAR(q(&9*pwU$ohW%#0Av%NjT^RL5P*auYhyV@^9Y zvyQ*2>o%AIxyRwke&J$wLu-IwWhax&;ttm$ur?vFs|r0_J=5Ac{LqY3(@$ac?9oqn zvI}&+I7O64lSV~zs$wGtZwpu4BzMPJl6y_O*Ph_uw3x01_i1X#aq|%+hB4Vjm@DK2 z@YISmo`&H8c4VajIu*_>fc=8e8vI#p{SR`;D{6ZW_iJJk7 za;k&|Ktfvnx&Gm?W5%pN%q_bmeYSL&dSuuZAbPQwW;ve#s0S^>KA*m*=Lrw1JXCf^%(@ za%D}O;7;mddqH#Vbe0Ol^QK0ImK$;|js|JO`yF_2$>DD-U3i~|!zqK){Mf1iFsFW| zdw?mxkYdt;1hro}zh5)MW}@%$(Jfp_LAiAFufHZJKA%O-ZnN&=eni#_UTJ^k-u~yM zOIcTi2lI@xgU=Bd2d&;xYfcQU7vFblh#73cq=84o=DMla!&F~h3vONr!y*1JWO zcu`W4#M_rtPup!Ga4=fgnfCy}Wcl&v?P@JZE#@}nTAmrsR3Uj!G<&Ii7!O3!`F7~L z*Bx(KbyVkm@*z#6f_{jcxM}w&sowB>L%pg_`R;2-jr!u;YR}rm%a_mgBZc77$qq3i zSSJsj8J}Hb$+RbdRYXv3W{mQsjx~aSTykzi7v9Xs{iGAejza zzLvFs*v@C)li&rJw8T^vBbLfTfsJHSrPcWLMmg9CWD_vPsl{6O>V@lp_ZnIqsI~6~=Btu`)y@Dg_zxva%SPfr^cbr{h;=pA6K~F-URaK1- z0-_=UaDAMU!(xau(~^{~vYY_xCd&43!i0$UOlW!NRf#EL9k_r_62t-pD*GT2pCmnW zEc)0!G^^josGI3KbvB}ThhCGj7E{ikei3aCZ8H%`{|HYuwem5Gk4}i4vN@{)O-yc{ z8M!Mlailoo7$$GaR(c^uzB;qv3lEKOxJ&q4ZLjyP1^dm3urLU7#Ei#5s2`^7#xjB< zMBnlz1TBu%s4D}psqb(@GvXrW2Y0@N=o(D2cSc-zu7 z3~oOxqky9AtkPNHW# z$U=my4Tb-wkRLQWV1-~tjM>NR4w&S~W-jSYrG6+AP;4e{Q|rzM!i~o|wzv;fgK)OU z5J^qI7GD;$+uKONaOX*IdeXZ3S0g|9GHhy(7-9Y0O{uMI*~r7g1ADu86Wf=J?BBmO z#xdWd(1&3Ua2L~mvCOyVa}SqW*zoNl$)R}{_3bZZ_+yQOMh*+Hly66J{fE}yA;77% z@~_;IFP8mejaAYwXKcB3B42c*t`~<-P2LlIx6zSZmaE!CL@&9ohcZe}Z5E#t3$yCs zb9+(R`s6@vcDB~>+C(C^^1GXCmBSe0xoA&>Q04iKnPp?Miw2Rn(Ir!YGdeK+V96vu zu{mupQ0!GW`_9-{6C{cpFjf_z5qJ&FzgxxI9>y~O5qSTvvZ5f0h}yiqUS9xTJnD&* zLc12j^EN~Y9<5I@UGaxwR`hYz0r-P(M?7W`&fjVwyz?zIcNWNd^CWe#+dZ##i z7m`XSR;Z>HspliBTyoOR%dBvyXdKE06nV1U+sBd&kVE}8N>C0LEWq*1d#p`*m!xh( zRI3$F*IOqr%_QTg2?+_@PHKmsA6cO)k=Vc&gG%xVdxWl?yQ;{8!3>P*)Jc|zbcKac z(#8w}>vVZgo4B)PZM0cns~*w+XZobi_GW(kji2(9+MP3xbBM93yTr zsSB**($j70dKxbMF<8Lt2y3LVhu3IPv>=Cg*cgWA1-EXs=Y;%Dcy={Mc$_?mLR_4g zFmIH!ny)?4|JoaH#RVxe))Ga*64N1>TN@uLp^!H+TwU$4K>E#7mycBtvyXZ}JdTxn@I=tv8aiV=+g|DRBr z7#VW01OTxg=24dIPl~G%0QwTL!-lgzW~HZ^P+ry?jfrVb2y*xl*cY#_86lU4WXvVA z$Az>zGIFr6qQDZU4W=!j zbIt-zaoiCuaz^;Jla1$ETAo8>Pm68s$V4W6$K#HKgKexPgGmYl$TK&&?#^V|BPhAf z{o3wXt0>-Iq@JAYx~Y?5DMECg^E9dZHmeOM8G~U)H1iDPc_e zlR2Hcb+%VKUhFKb0J}ha0zyKxfLy!l;#gSy%px+YH^=R@JO&a0WECY%4Lusd1JzW0 zB&9-Jf-m@d<(x_XAU*KJokc1K679SD9%-#={E27JbYNY2F6G#xGmf>BX~D)S=F@W` z@m+6+2<=F{k}gS|K>+~i))CHZlXjlK~pDI;q&WdBM*CyT^?hP>QOg&WVhk zyXab6b{_Gz*8uii>9CME^t<_b@68}BW;6y~HHxlb*BY2rcv;DuC^L0$ zUXzsq&C9H|k%eYcP6R5xO#UG)EkIl_<=e-yQV+1Bc7$p3r~S$^m1!*E#yLF6(I)XS zGlTUuQKbz{62J(a@;hu~=(1%SkCwN0&Yb*~fi?(TO!6~UGAZ|Ce>K;;t6$sPLe^2%d#GgiLuh|g7U`Ox!=7BB8ek19?fWouMB zJzyEmzS)|1&|5Gf#l75PXSAhJ(j!|j|YkYF&Y&wn@yxN_^@w3qEO zE(x7ed|j+}noK~m0j+)4F48I%hKCNU@%{0PKaC~Q@_N#qj2L5ZrR%VMKd6M zTYGymi)V4ZaQsD6+VhE=|ao7{yaDCEJ@M zlBR&J*Lix*fusR0UC+(68D;C%mTHNVNEs#JAkZX)9>d~TrNgq3%cXg5d>K~d!3V=8 z9^}IkEPbPD5;qGTpmG?Re{07NvSw*=J= zkOJ8grHF8*`=#0q)G@bQQS^MfHIug}zfG6D{P#E*@(DU~4h1YewUf=sYOPvr6mdmP z`lqQ!)R!7)5RzInsIIF1QtW^}D<@6zJE~JATIs~9uf6aKKhZ=*dV_`BE@^)D2kt+3 zpmz%nI4*2#L2dfLQruv9RCC8&h<2bt3AtHj5t1Co$^+YkgP8e%IqP>PMA3+@1H;>_ zd&KLo1vm0!_14{s*gB3Y;|A|I;856;MdS-eW(ewA)TMh|c=QDmjUBw4d~?`@R{MP7 zwj)PHT0{3qV1xCQTrf2Yx&#h#nj8mRl1;^%f*!EvD;fjDgWY(fy8q`mR7}Tpd3X}v z5DGQxLVk8ety`C6sMrw^$wMR3RloAV?V7$%Uxx2YV!G6e8*JK;c^ZIGt-p{70Yl}l ze1#5v{!?Vsd-B-)-6q>O4V`srrH;*i*v7GCr{5J4=8GB*Nb@;a{{3X}F$se%6`Knf zvc6{#owdDolVMGpSRO9U$f_X3T=IU9IMySRnTC5P#w@kBa05`SvTRaf^bQ`39b{iW zcydYv`VD%n@3~y0Pt@N2@^^@AI;BwiS2K`5qfK1#$3TAJYg{@l+qOM!Kk7aTML7kN zOg}iKtKafkS^EWkhiiD9)Ruy%TqCmCsOr(JnKRUr2;rKuVq{&#n1#{V%=h6ZNiW1P(P!?Zrjk8| z@?TiwMFS^>&PGeURj1OuK`@1B+_xU2hzFhRhUhK82m?cj~Y20{|uRU0dhJl zsOQeDnDW4o+RT$NTj^&OW8%F-KDNQX}(qSYJ{TA~ck%Dk}qOI**28o^gw#)`W?>;~~Y!w1xa zd@cX8^pyLJ5pW1Gw@$>ZAD#;!EH9Df_O=9`V5;uz9X6BOPN;P0VPXoMV%}kh&P(^I zDhujul}PEu10X4W;p{4wJ+25TBda`o_Y>mtJ@Hi5HY&YkZ5r329cQwlT{g9~)n+7+ zwB0W)CS^RLS%9&E9I7$egpf%Yx1BZbW|F*RdgA~pABm}`=rCEc7jiw)5PY`&4)?_& zwV9R}&3_fZ+~?LKX+#85_epVwLKc*0wMOXLK~IX!PGmGT8}nf~2UPsfK<&>hnMxr9 z$iV5{FDtOPq~sSr?glD{r4^&B%E3F`q$S39GO{TBG{`!REPBM0sG1!KVi*5X)>_}- zCzE#I0GCbK1Uo%=4@%4X%ajU#9sQGKi!WZ5Cv%WzT1v`Q;x+4avwVyq;&Iv{6jL|s z4VrM2*HGVtVP}jqtrQE4nVGk;IU*8K<`yNN#U`-?cBc(JY-8NIul3u8>nS*0hI-r2 z$KkSl?!fg)|9oEU;?hoF)42=kP*Yos_*@IYm+eMX*BH83`q3XuLI7!sXn$9tZ(XdF z4rGU68#JPD=;5NC`ue{FW!$$n0LfEjm`*8^I0*0{pGRC4yZ!yOk%uG-?#i-c$IR4_ zG8~4;f>A5}{pIn+lY>Zb z`+0eJ$(SX<%nq1!N26s^AKOD)yH#%yRSb5ci;}-RbLKvTYklROV5O{J65;4HkD=!v z?}3hniD(nCd8Qdh;dK(qz3pz_D;u?D`0|Tg-DdO9`kWtkZ2my4_l*RMq$6wN`|$wQ zYHWqA30cLo#<&SixT^e83YGHNwXf@C14zxyV*qwWeS&C zn3#Xf?-K;B`}4ZM*@*IRf(x1EsjW+jwZ|ID`AL7E_~Y>Ep|) zHO5BO)Uyy3!pZmqpx~A4QPe4;PaXYiRl61~4O$Pyn0pi|F+#AA(YNuysUZ2{E{Gmi zmw=vz%{yKdhfBAv?>$*MC%sZrPyb z!S*F|PVV9bky*HLdJ0CgfW6&uT1Pf>fJQM}h)@;MWbr3+GqbCd8rP*`d~*&9Yxkc8zo3xo@r^VH|Smx1&5oU`TpuX`(O&j8VV&O$Amn zni4gZ{lK>8%c%$^rlyU#I%PK*M%IA5$ln>I)I5|yUIa$@^LEP^NBp-AI z&fY`zU~?|jvI86JyMRa#Y&2c)qO654n;|9X(z)}f^Ovod(b`}ZVj7dJ5OZfZabLT9Byo;Y&bnn(}K94Y6LY;qI!)Q%p=)r?CQK=hRCAy*BYNo8aZlr936t*-mu!JkY zJHUje#!j6xryWZ$&MmqB*OzAF3pR*M74gs-KiQO4g8`}h`oHvZU}utju7A>f&UUk= zaObi~KxrcaTq@P7F4d|@OjW`(g47%U2f{R2YDBoD?8fv>(k59kJyT;b=1RhK$XQpKtrV`yNonVw!!k2I^Rx1k!6$Stpmd@V$!2a91}m)-og0+BxCNL(MhI6M$=sp*C>qDlW+QBMq{Yw|fy zfpT%VEUxYnykuI+nUe6C?+t$+8vYZPE~ycdKTRpa z{8G?*m;K$LA0tgzUbDNS$;EdNrfl*sv+%`K*V;}~V0M1UR?^|zxI`tlHhRGz?SS84 z)qOHUs1%YW-Cf(#14>E4Pb3koX53|&hq3XK{I{-0rjo?2ed81xIWX%`f(6-(S-zCz zk6@pUMH{T!fRj;}kwBe4rKg&j0{w?K%Zq)-y1xSMVG+d9HFcY1AH1a8DPaN?qiiOQ^9|LEB}kX09Xa@kdmll+%N1-q zsK~AU=0*5wfg44Rn{4BU4l(!aYlWehr`1W64tSt{>!)Ij$wKcImH@!_y;7Rk8i6 zVm~t-Kxo3-@%x#COX?r^v}iu}rcP4wU>X`t(Bn$55IZVEC7X~C^<(^YvQ!~_F(_}T zMpuQ3I70=0l=TDn+x2Pl`C;!b$FmD6G$kK^SGTH9olz@SmTnwcRGb=S7$5S;XiLgo zh%DI!tkgY>{QR`8b&GD=5yHlD%Urqt>2=m7B>%Oy?_;NTcE{XPA%mbb!%m;Rv#kHR z*2D$e-*r}i5Kmc?`#I3N}3YiczUYbuAFBp_{%TvvY2}R z-&)bjKv>qLi)*UEzSf z#IQC;Dj>2b>;hB}pT#|nVP}yiRVM>3&<98rDI~ox&)#45_aE(tzJ7BmFal8R}5AK#juAnL0 zdUWOlyU47wh94sd0lX9ElGvcu=55;;l6@Kv94|tZZzY3S5dh5N@4u1t$h)1jybDLK zT^pTUX{PeKwYFl+;GsjeGp8R73$uHaj7=FUOG@Il3^b9lpr=g~jC%B!eO$O%SY+CM zno@fdnT&oN;#5-`pkskBKT3}EGS71HL;zua_l~_^O6NXq;zZw2H4-K(gi|oY(M6L`r-*ez!iq7Xxmz^OC;##lw`9LtHPiKEf>SGEvIxRaD^)-|N6*FiQ6R zwQgNs)DV_&*fHdhl{C>P9!qB9Ml||v3|bu3o>%@8LYLTCqlF}r&FiPsRe<61bNP1* zI(=}9pSz4L^~D-{RZ~xtu6({a{iXFH6K#$Y0c%f{vM9UA_GG2VqRsl<-Q$G-Q1<<; zLX)t+-dVOs+agn!_D52NU6&dLislL8t&6L-k+~3IubCV9>>C z>8BxNPn|5nD>{7m+-Z7iXN(+==*il-b7=cws8hcMwzl=U9qxcEYGmby*6<=P%I6>< zXP!Jd{Zu`ju9hBQniCt;ukZTAz4@LVFnBjOWg8LTMJIE^oFifpWAyY<7q*b_YSJIY zF|~Owt)EHo2~>&FM?I2`Skg&V^~bf)8X@~iPE^L#HHz)?4MCHTJVp&i8JLb8yUBK4 zlc!n6%?KdOTiw7-pFl=xKa!Da&kiM=3Y>M?6sX*UvrID?>&L+vn!~^}ixU^Eb|k z^+!gaE+|iYhaX`7;h$KKQ7&Fe>zZ&~#l(*fyZ6sqp*`x(_QOE7a&h+LPHPkfRDf@Yz)dI z+-ib{4xbd=@(hj|MM|Klcg}-*TRPj7J=Rk!{WZp^8a?PJVf6N83!HHo$Y#_feD}!45Z%dhh7{I{LP;Uv6*23%7 zZ$){A#*nn&Jr2-Rc+WQhR;4p%k1rjn>0%fmQ^7QC9xz?A{+F^#+G%MOeX0H4UepGw zU+oJGV-K`fwdXElG{7G&!5oPoj@_1HEYc|;fRXi&2js9~+4gR_2jd+Ssfem&mq*g( z!*w~>vMX)wCMP;S7>1Cd72bskEYf*KdbLpcz1Qp-I+V^zJl=Gt)MJ}K`LrYxr>Hvq zI%;es0rI@Ca14D>V=n3WoZ9R`b-y=E&YG#d_{tzSY9no|@s&|nxTxL@Q&Sj-GF{eD zSdd9FgqF=O!2aL7=PQOta4h11gNEZjmX==Q-XVJwKfD2TBBcYO;(STgBP-$_EUAwi zjs}}X|JUJ&K1RB_GE(sz$t;;~-mV%}<1&LwXoGctW}xid45Qd70O?F?2FEHvjDdhE zBC%UWmzJ@ZV?_cx0aG>PWVo&Uu{L(#PshD$f_veY!;g8rC2w{O*{yji@Qi5|;Jmz` z9#eyr;5VdyA8)(2S0h94O4UN_w%-BhR+&eawJ1a!NspYL)3Jbt5eDN_0(bbi*rxjL zaf>%fh+4?H^7e9|EG#zE#9c*K3bD&VMeP9tTA}Yp!xel#SJiFY>HD@E1^K}R!ocCP zRjS+J$rX7lYHPS-+%fxE@S9s<@I@!omk4}*tP3`Gt5~;>dR*O4i;KTx$X2*_fKjfG zv$K2kYqsapyN@4b9dgHSiA%fusvU3`>r~Plj=`+MC@fa zKe)blEN@-Ghs~s-a&*r$x?SbaD|tspj_B*EP9}x?pRuoqaL+* z2ODzoltQ5$XzG;75vgf7l}n)p+#eUMi_ALH4jJ|K#$!xWrbst5Xg>(-=*4!k*+=U6 z`B_7TF}+uK-2{$g_R`mFugJT{jbxavTVpMF%L+{id`yZDNtY}{3bP#AJ802D{dmk*XOt9Q5?J9vN|g|3@- zC)kZHemL)I4jHhI}2<6*;mxIU#< zn_yCpQR)8K+TTDxpBCTA{$#Wr6#{Kan;jie?IN>>I%-c{$y*-n&@cXHr-y>s=zK*Ez{)n~5hfp|FmB1}%-G-Zx9{9}3kIY)SIbXn z1U8?H@0!s{of^&Pe~|*5g(e5@3@RGANyq%XbDH2ww--)dI__BM@j+9qx-xPq_m-)q zHviElWj;HSshmnaOlW>$^Um{)75!rxvy=75vQWtOEwRG1%A2d-R)4E5*cymDP6FG-x_#{>cZ1rA z&712P3>tJhUy{(Dt$5d9(0dm|w#0G$VxWD6n!pjEuOdox1Fek+)7sHQyAihnBmW0l z&!P>otI5DbCHG$DKVydUo#Zk~hyDBZ?epExEXIN1g!s`rZe zN5fG|gyr4bv8nx6S&(HlubZXDB4!ouezfVk8`rOkjQX9izu9^gPs!F2i-}6oAjMsz z6GXlt^aw651veu0trMV5VaaMlb>`;VLrEjL=>X_lEnpH--ZlazK{&53`(U4jn&%l+ zi`G|haTgnBC0GUCeo2%Yq15^v(FqaD=IDHOHVzE3MKIk5PxzTZ=a`vh4VtzY@QHAp zg_;?vdj`7^upqtn_QuaGkMW6$srRyBT|qeIOVQin$1=ZxSF}hRQF_hPT*-gv+bQw# zxF=Rl{~zt>d&AWgakpaN2~?C*L#pO~PD9ozWi49-`ISiX-xRoaaw7h9| zLYtgRTRux4Q>}(bJdu!d39TL7`+8eoB$Pr=1xD)q)PsM(ER*)A&VBTz6u{4j8~#U{ zbin?iqglsr&=PO?>V2FT-W=0jpy^Ro=<#^qOz7@ zESZLb6uFpPV}IW}gq#_)HoFzMcJu~-r@Z@x$M^1CJ+#xHZ8&NY@l`rk=>UU1KOFLg zvZxfQ+>`VWD}mo3W<37^xS3EC@Wh$ z66>WLT2CYf0{4i<66%s>%=rvHPnz^1Z)4sB=LDm)wKr{03Wd?<%%v#j>(bK?Q55-5 zrItYv$F$5)TgmK8!Am}W_NaEP@c}T2|9eJLl+2%D?%w(b++*6>+O0b1ct%kqiEEoGShJj87}AHnzbpj7 zv%I@M5ghgEQS+t+8jizDsqG3dxvI>P+;=iaT&fSFow3xNQm-J-{=2H2U6QMPs8~P438{#F<&py1aozeRtIE|!yM)0p%{>M=33g87(5|Mxb zRb=0BDT02gUS}?;($)_&%b3J`JiZS3nRxtNR=1;YFzA3v(=cmGl+%Y7cbdJP-@R#3 z_(QDsRRd~hl}C5F$5=n0;i{XU>#v(&T)~hjKW~Ec$cpVHwu&?3OMUM3gg%`LO)O^L zMgeWa4ui&FO822X@34lMf5JM614t}0 zLg*rrI|5_K+E1nYK%KzE5;5csS-w2s$@>d8U2F+;0fA6I;1$nJf&lYcA$;(v*~+-o zyLNs*d9!IReJdQZcbY$jMGEF3#%CBimT3`4CNGIxt$aRfiFSDGUBfnQlGn7J*p^uz zjhrC|RRVy$(l0T^L*_%DxQE7J_wJQh;jSY+%lcZDmdqzHeZy|k+}STL9?Wq|p|MwK z8zSsxb7K+LIp!|OIH}<`CYKyY7!In0fz0U%xY3_k<7w#e;(8!G% zqTuMs1Ycr>chBhA)65JcfBzxoIALv2XbzIXrv)S*b(!_-+}xjreX*{-j>Mo>k{!9a zc(uPe`}FdqODiK#K@mF?91=P4_~+Nrd5=zHRnIuFnLspMObbu)5_{dv&OWqlMO_I+ z3H>x&v1Zn%zlkRrEQwkIod3wZb?iLx1kxpQ=*>2$U$0&Ym%=Gl?Dsc1H*c|xr2w-V zdVFl*rByXlhU@j_*W^D}yqk%4_2OUQpyWhLY9qNSs_dq7a|d4ma{RlmcPw>qXaF>+ zE7bkJW_ymkWTskBO4E*S%lyur_>)9P_^af(aStyL6j-(pzZzza?n+W;SB--^-PTd)1svvIX2 zogouw85i(pcJR{&boHWQwE9tzAwA=rn~w;YLDQr}eHeS6Eg?h$K~GVqYwg8=D@8xa z_1Di5I)3-wJ;?cQyDnPY>_P)exVnsD1uzyt4AC8J2Q7T~iTe*(uq6lNI^Z^}l?z*E z^x6VZDJ|v>^{tp}xF!(7R^Loj*u@g~b^U?ezyRW%tjgqFhqZnqZGv5m)$C7cEdbf3 zo*st@NoBQHHOGB-4Ku-%*o3_4f}hhl9^B1p8r=RmgVGi)iZTak6U!%`PW`}_=)>Wa zY+;FULt572fLR>ppWn%o#gFtvHHs6+(;Ob0=1)YAvLD0wz^I-&&A~+GxN^=vYvH(l zau}Qd5NH0Hb^*iKt6`QtM-IJag{JP%?neq05X1=nQB@| zjsxb6jpAZ?Npmk+dI@W7w=?!|xB)zCp8;ExF8ltK5EZ=^6DFzdOD|aNtLm&6grv!` z%23$`SiNF6-NLXQbB|IriQUKS^3WOZS9Fou`ubg4dcWPoKzj+tCKNS#@j*=Aez1BAcXd>q&+?BQ|DZ`{ z^2LTXh0tj#N_pl3=(O{J2Pbu2agIY3fM|BH+OSl13^!S1v%r*)jlb*NR2}_c0V+hu z;1?ovw6trts-yq#0B2oQ(8~Vw$l7iFAd}mVhkU0PUZV>41l}QfOi!%xzT@{n4@zpc zs@RuP_9KNy0eabNipyi>!{I~-A0Z2@LzYC+AOLv&@rzVJ-Yi>*gN4eG`8=xh&&%I> zPLcc@QbvBVW?(6frf%z4b28`>HTI|zfjH|pu0Mm5SCMGTI97kjeeR<%Wkb2|QReoy zwq6xc@RdXG-?>aj5eIfiK~}+i;kaqz^Oi#`K22AJkxu+9DV}r8FAmbQMrtF9Zf>L- zXIar=-Vq++^eiXyH`Q0dd4-KK+F{G{hRquxf;f%++ntC2=IJ%mnnTlhYT?ttz_ibX z|KiT5GpuLFg+M6+IYFNkO+H~h-}|J*M|ZOCM`%;}L30KNe~f)~(Q zBr|~Hz#%~@xq&%rRN!dB?b{CMSZqfEi%2Ze;}dfB#-+950c7q1*9(-#z$QPGD^7K{wFt<_8`>32}1qVIM zmM&nh{k%6{Z&2)!tA@^lpX97jy$GK$FevZX+VHK+FIFsC{Zm%)e5S{^e$s90#E)*@ zEgvVP|Ewkpa!T)xVz`Sg1KRCDo*m~RP;hfA@tv^ykE)R)pD+IS^y4xCU>|`xY3S8~ zgx-E_fk***{W4F#+fS>^$kC%er00b{qgT}yiJ|0p5v~G?zCMpq-O^{hOxHbgX0%i3 zRT(cIcZ%Sa3%_fgIDR}CO;#kja+P7MaX4QidZnv{G+0am4bc4OjxS2rRV+9-s_SQY z9~4X)Z2qaILl9q%-at@`%!F0t|0#q#ej1wuSVQ66*z&^TH)D57A3T~1Uy{P&v`Zy_ zeDZ~7I>pa{QyZb7(DZj*A_yah)VuM_hD8flT{jkn4gts&=Ij+sWUdbKtsmuoXEmoX z56!5yk|}!bs}A*HtE`4!^E5wutyiYwdGIVFqb={<0s;a&wWe2tssHOJY4l+?jP}s> zxqo;*lAsVfA4KN0B<_ezz;blx1*kgK??9|b^(;A<)`xRLWiZF;pSscgH2AN*W~!|8 z^9u-&RUz`vjQQflh)#!(+&(jaUHHi9AWKwcOf7?q6ZWOy`^@m3dd#&g46=AL@wLqp zQX^!%a2I$$D0!7vh7ga9pxhnj(8n4qE3I~N!0mnz;J7_vaGi=?zmkz;EpXD9-*4}w z`3nF#DgY42InLn-QD%{FESjo#wNa1_t-5uq&szzdOL&LOX@r&CH^~B%iFbKPfqPBW zN|b-p1{JKDRxbk!pv8U#kv4o7aJI?Cu#>-6;u+wFYXve1H{UDZ5 zqkw<@Cx7>*BfnONfwk?K;VDzjA~pZsF~sOIIz0x$i&6H};m(=;Zok^vYxR(MxK@sL)GtifgX@|xHkOcuJcOEpjhRz`vF-mSI*LM(~>z%-J&t- zLxox=ECE*XM@tBlU2B-VX|#;s$SNWISkUh7!Ap1cZGt(t7Ap12RtCYJWR9vTgnjY4jH0_gvg;QF359^hz7lB7u_ce22Z4C&1-hGkwK#jUqvYSbjhC|`}~h=&M;K@ zEQRi=jJoW?J-C|<5D3f5RklT0lo1+sKVwGUf8s#+Pt53@o6SNsL9_XR^ujBtV!WE8 ztqgZ}$lsV>{mr>a(GQl)T{`Q9*~1}5*LI~1?Sb1pwpYdDYeW9*absxT$GbkvnH&B3 z_VLHBy9O9WtW-v{y56ZW;qKrr7p;`-{HklH*V739^lD7E3d`aJFRc%aI+$J&R$Nii z(>o3Hl72#l3n?4=h%to%^S6NH>bnnSKSSEI1kR3JP+&tcPjkv{ ze5{McXUyn<_FWU!neISLY4A>ps~%t9mzG}QM3q1x?-Btdh(o0;Rc`0F8QPUTtMtPM ziP*D{q06}RbfOIP%;_&_S!AwOxs-XQxQ}8kTnN~fa|x>p;cGJuvWQ5cku-Es6n!1t zY3o$C?oAcf*Icdm=<}`A-fN*5^Tlnb&*J;XCQrv#SyPOZ6NmAjx$=FRS}oEE)Q8{% zCaN=?6Weg+dk-kxD3_ZJOdpCc*;hkTk+$CTukAyO2p*|39h`i@;>C?R?)A^e%QNV- zb?|~?Q=r|kEBkM2H1sciFPwhEq%DKnp|)z{U~k%jNq0=cDR<8^icvUA?6zb8wKAJ| zy&;X#_Tm#t*UNHgW1nmJgct~GTKQVdEOzC-AE9>2#8gLXu$|r7sr1^N9E?`a0{LL& ziwSgE&Y9VFWSkWMjZTj?5#QOMySd1<3%_+%*d9c34D-z7%09-Mug#FEnn8cp-k=sjd2B=Y23d^7*py*aNJ z@3!Bm%CWF@b}rP-+B$eYI1Ju^@&S47Z~X!SG&7p$W>)3eJr~SXUV2K$5_J`j>$>bn z$;-z5Gz=qZA3t)mxrrs_%rF^WN_;y9Z25SJmkl%b=jdc{-C{=HUR(emPv=tcL=b7D z%yRO6eONOKjGz@C3M1K>)a%!;3v-5!%j({_LCYKtHw1HM>Rny_2dQ5t$AqbLCYnB* zqsrdByUZ$9&*~x49NHoeZ|{J=N1?m4E%*Q#HgaL>9d&73pz6?O9qJH#+VeCxI1TW_ zRH`869<3v>hPM$I#>eo>a037^oY}L_!2J%~!}**I<>N`ipn`8deXMCFK2lLpYyUyk zABof*+vwHn)~#FO+eoP6BsbR{bLY)VX02z~K{y*uLx{kX-u)u{_a8ZOecBIz zl}PVTe7WKj#w)@6Mw<`kEq0{yBa#ibZEl8*V(%MT$=)lz$Js6;62-j!zYwd! zRBf$IwY4@Thb9gKRpFUUJbu4m&z%=767IkFtKO$^fr#~F!a4+?)*PQ$$9jSG&MOYx zd%67L^3R)YE32Sjt&7l4^%qank4)`zty`RcBWAXD__J4mUJ9U73ybhVPVI4TgpT3zT*2bsGS^VdITs3y^s~DKFzy* zn0NONg>6=JzFVHlCxxkp&lD@;=R?w6d8ml1`!efgWtcSS&BWc)q2hx@?xIDDR`XQb zHHo;k8*fv(VxM*mn-zq#%K7Gw-wj>tO#328q%PBl1j6JO~_N$LF#kY{HQBsa#`?cufBZSNSA!GJuIzk6@{E@d-KgZs?jj{Z%%7P z~Kvk3Jn0rl(1F;Jed^gca)0u#S5J!-YN2j8oO)fGSy95n&e~ zDNnim`q%*XxRdE(ZH#E3r%Rf4NAldu3PI{oZZ}K{lD4R>_6KT}+(?-T=UY&7zZ#POV7)Bhn=YKsJHRq7f-7!I z9{EYP2-udwvlGSvTxi%=yWxd5Cq%i_ULQ4U)-2~lUyW@&X0ETQcJyfkFxUI;JPOt% ztdrC_J3us4g^taBzD`-d=NBKny1mcKhux9fOA(J6XWs^8X!ziEaXLvGN>dS8j>`st_}Q6Oa&O91)Xyr9Q1?lsuH2BY_^4<>1^ThpjjeKVgkjhH|UWa$wR zit8>5X7p$#XLt6jMhwa3G`u-?_T^aiF@39gB$&DXmYYSlql1UjY~uvZfA-91n2qV4 z{Ra+U57wM6`2E*f?e9kW7fua$rW3eN(i|Wuv_X&c8yk;rXP97s8*U==;-hq+^2sv>B`T z&{lNqfW@sEGrV$G55=$pv9ZnY(Jt;iyoTC|6DQd8l`$8)zH(!U)6FzcAB`pe`KqdNtMhHl_qec!iNW-W}Iy*)}>@$OnB{Pn`Y z*fD0rzGJS(AWkR&MZr-o++F|g!TYyat+15vr~37SOwu#k4RMwg{zWi_F&OoUs09ZB0K#|pNx)u18D9IY^i!%3X=rS~Vzuf0Bp z@$`&aczZY?N9MzlT0Wvfzn(pH>9h;ajWDxlIe-3qf%j!LHp(9!^2=MdnyJwfOXN5r zIzy;39zp1bz%a?91v?yXY>}ERxi`ahUH+q6eA2aa-ncWFz9LP<+*>}LG0WoXojz7! zTlnIIzlLUGnbBRI&4~w7rj9OAr{^M5>gB65dQ_HmyEQ7`BV|J~4(p;vs)9c%hk@uu z--_o&6g>B~Jo@%kz|{!?jBsrbJXiD9``k?on`XCf@a!r@KXYp?Z7c3?IsrManB}x7 z1RA&B-dfUWmMl5vBN>d0_vHW_m2^yLpUb{fFQiaHaE4>wZX^j3BXRF-*{!TUZ{%?85GO z!e&WX%|H=A&+0gBbnRv?=XFZj?P*lrasC z^GREK#qcde;Ww>>EhVDoNn6{ZwpG`HWQtUN4antoErBTkq#y!g zAY$LZ`Tzn<>CjGUSVx19iI;baZ@ZRY5sSa^`a)q-ijLF^Q6r`o7!ChA4L=%Qwg9%b&22ER{FeT5)jB_<*+OVE#qOp{~mhHR@NJqCdUP?veBDH>KE> zj6P&Q_Mt5mE1YdfBzkFrs~Rw;S=S_LkJQSV-Qre)pib$Y`5vXp8ibROQ0wC7`73-u z7YhP=<(_JCxCdFMi0P0uemiscPH&|3gn0O}-ziw6`)%2>WxZF6mn@OcLT1IQPUKCS z)Oz*mrPHHvIOb%!DfQ%0Z<#xb=Io$r3*nk{kihQQz59}7*KXbXfY1i-Z(!0nV$TEy zI8wUTi^{3H```#ide|_Z@(J#`uWVT`G@$^3<>L{4?EKKcLSw(f3vGbKZz=xI%a{&2 zHLcKk;n+X_{B!*28UFd9g>X?)eIWD=n%$K}>+3#sYW_A$tIQb=*kzjHUQ-n35<5s~A z6cL?i1Xfdf25f1jV=;($Mua{ZkyN;0dyTy6#ybT`3?8t?lJQ1V?f4I!UNvh6P(gq0 zJAUxs^#gOXbpj2jgWP+EpQDEIn>$hAGl2z8gKN+k0rh4;C$6UHiIAQ~|K6SQP*G+(CSr31sW|5ji1-z}-{IILrb+p1RTdbGtXqFDPhjW2Q*IiMS%g?emayWBL)+8Or6; zsfZp$;;oq z{Y?!3oReOE(X%x79iA?VbVDnvyn<}}+l_=Vh`U<*Dl49+Kv#5mWu@cpV#{aT84R+Q z<6mJ>d04mou-?l~sI4SFY})s)75{Vk7j9P=u5viM#^G>_(cyJKH+a~`$Ie~x{IB!X zQv2@uv+nZsVGk~IZq)nzR81tA)dTHo(6UMY`=O=PhM{Y`3zYV{!Yc=A{Sp=>cs#?d z#;e8M7oOEZl&;1LE~kB_Z1a_Yd(3rJz^GXe>=7>7u2Bbu=$KsCQ`^vu!SXX$p{N1?qJXu^U-@oh<-wRCl>diGFcwZC>9J|sEbYe(7@rv-HSa|pMi@h zah~r5ZG2wk>vaARIwvYeiWlCxf9UoRxV)!2QpYnkWi$JoDJ zg!-I#5SbckRJf8N1*vX0%l^X$W(Lvg@=tkCW(Fs%s4UCD2sr76vw^HJb8-K*@b9DgqFz0kub*%c$bt!=!KURs}Umo`VK$GVq)_|UMuj^RQ*pW^!$E?l_s@ZqF=ZfiLA1wh2M;5QQ*I8Q~HNaYhp-#undT*tts+k(>t*ughIv$=zI;2G{2KcDE)ostC% zzZVk}>c{RT#}qClf5EPc(OgHOPzT?0-quoQWPLCjGK0lV)uP_Y0#7?4bS$(A2%(D|IbO@rX^ud^#nB_cXTqNm6KnwLg`DrN*kG9#HnQegF z`x4mKy3w6lJ~mkC8mdBmB2}a^<<#Af{}ew38lat#iQis-?j=|=2W zKbSjg%%<7E9w2=7?(UHm_o+2;={dj6jlU9h1{vylG7$@z8E`6<2uyck?D3_TFU~D8 z2mNumxnnKeZ2_AFHuBxdwKrwFCAVNJ#J4;-HsH2~Yu|BH%Dj0<{s+!rreyB72(Xef zeicX~QpfC6j(5hDY`Wo@6B^10J8K&o-`#y|(BhE`wn}J#p1O4yQm~R9korB}N9@v^Yb`eyzkTSMotGDG*Fb4M z=^maS>D9&J@bRMaT#N#y;Hy4l=t0DvtRC1G-BQfp2cve7k$o*u7JJ7vY^&!?_o zL90bOk4}ln#%o)>dHQtiI44gHY30-GouE6Wrd5Q~*ZO=41QaNI@|k%+Pe5pdB)D=K z5K43ab=KOfNmIIZkgYp$Vkafgu>TgV%+L4GprHOLe>ftSWyDfWrrfl<>h_L3D&wV+ z5PPQF`#Q_e&_uuuic()dg;)8~W80XXfYT5uH7NJG45rF8=m|q&)ln z**S5UiJDkzD1nm(t+%&BJ|*NKrE|5c+Gy~wGy?)!c1^EnkXhtR{Njs);leAc*1X(3 zy1H!~T%^*kdYR@7+V9|Msz*~{eSE&zPGC-ia<6}sFaHqu@kD2n9jcKYZNMICo=rG< z^ys(Z`>v*ZU%p`cgZDpKY`@T}B$|WMEaqYr!kLgg8%&aydfM3LJ3f4CvZs|{MDRV= zQ&sJ6(aADnl*@q9lDuFc;|3@;oT+RKUCc~5KQk;owaE1>Zb@#`!$KL_$AJd?l`?I< zH;~QMn>QPo8V*Hoz{yHGVy zqH1eh7}+{_;3@R!aTU*YqN-N&GO`S}@NO&)Ts`xsaPNMf--}Mx<`

    @eH30t! zMynza6is-xGkc3Dt;;N@0pBXTr_5Jt5mdw9wMevWDVUjuEgjkpeVHVsI$`k`S5PUAnDxQX*TC8_tR@$~zjKu=@(KnpYD^_~p<-f6O1)MbDI4(nE zLgJ;`bIUvoo1OrlGqGzr2d%Zbnb=j5ngdbL%*OT+SZ}>@m=A3=-eente!Zp^~B)-~g zdP1eM$o@rrz5Ju=lIp5GXPRL8{N}0kyVP|u+fF-MEzql{uW8rm2W^JS(j+H&@18x0 z`Z8G<EQJ!v~E!glS_!do`2iSg#a`^h!Xl^x@qMisziDUjQ! z;-TqtuW#wd>CzdSs^or!hK4S0eu!X^e5zK#%;VKI+w9S9U6|Q~7(C8gaE4U<&c?<` zpq1l4u!&u_FjZpOgEO-kYqLtl0{`3(eJLuWs6ncrLW`&+OcuW!UDn`LGRGcYD`NIj zsYEj9;1QuxPanG9p>_%q)O|ygp_B5m>g}|;qc{bUCnF3;lvsA@qIELv-StCOOMQIe z>5wfVGN*NK+CndzBO2&^6yL{62m{B1@I|RWQM9$sU!Z}zQqZ&~CbE5Q9k}C|dtL6% zk*>_Sr4Xl*b)pI1D~sf`N0vI>TgRr+XV3a?t~J!7>iA#M`%fW{P!KAvBQTmla3~7r zIv`-aF+gygPsUXT=?d=A+`=cQ*%$p$+BzTaVpR{bq8!;RSwHncMUQsu2F1I?lgmqP>vX zKWUb!o;#?Wby{+=2KYyzw{C6VrhH5c92zf|#?nuVI{`cnv~HvO62)e}dvlUO5jB3F zr8X6Mn>_nn11Dq-oWrz5+}dRISmgK3&CNxm0gGMtQ>>zGCjkB7mKEw)+8ZT7BFiu- zF17zIgADbWQ-|@xEjrvWi$OIi_P4weG^n2Q&YNa0tw`jxMXd?^%&joQNZ1_nyK_FS zbWZPhFf&C2b=(y$HtUFugfR` zTOlvPC1dE)%ap(0qhM_l@!{0g>)^xjQA(n~hGsq2oqbH(Ph(t1uP#`ZGu`jDzKdVV zqQCk0aZA{4$fwQ{Jwa3Zq2G(M@$oHj4ieB*Q@T}*uKo9BDmRrqAIyWHvLQ#sVhK3i zvR9Mxri;ulVr1i%Ed%+6vs1a+<-QI0^!FX&FIDx-wjM0lZt)-fR!*pm03UV870VMx zkJ=(UQ#RA-(~I`B>8E)!{!pyG^XV)^GsZeNR6qK0WS^Sbhpgk;TfZ%{l(svKh5(spWGZ@Dnb6{7(E8wvs2ueb8ET?^ zl^bG<&PR*#!zWMrLEmsm_pTj$r*p@S?ymX5Wta`w)gJ&uqYxq|xW^v0DdX5=Ax>kx_p#9y- z;WWu*IMO0;g=&Xfo57jWPe=<&3)2=tQqP{PlSB|~@}PCDp@)whOJbE)S`e5}DA{Ua zLSB<*bPyt!q_~Hv|9$r3FtJ~7QA15t`0Thk?C3q2o=maeb@2vpqr|k~9d&n7&ul8> z?Ufb{AdslZHorI6#W2Xuuw5|o-KTE0mQL9*wgK1y5SMdkYn|W`AW%|7VT+ie&9np$ zEcd(KS=RUuBM_0ocIn$PNvwgb#xIoUF725Iwo}_9hhwnk|7v@_IS+Kd1j0mB?<}9Fv6?Um2stDTT?) zryYq(rAQI~C&~t`g|Y8L=j^of=?fUVez?`QyZSQboAH(jWr6eS&heeGLih|+Q_zy$ zC3prsUZ2fs6+AG^Y~Btv#4a?fa==Q|nS?swz`EjvS{@clw2FK)jO?uid7#rnAIGX5 zkeIOduLXXES(9kus)o3XVnn6{mAkXw<(!}Qz*JQd`K8{E{}J}~@dZB`F~BM1wwyLB z8sE!Df%FYsY>U2#LC5A1!Dn4F4QI@pxhE>>jy~e!5>>VCU|Co|qJL=KIDbY@bw1X@IF)ecf?*x~8sTl}o$< zB4l-Uyc|uOJgT97izu>FTnSN80bjl>v@{;-1zg8d2XlTN9SMnUj+kDZ-r8o#=26h= zGKnA(+_HY9{#Xjoe9Rvp7P?6;)JRo)c3^vs2{2K9!%FzqrMqOXH83G*@y?+wGmGFI z-Y&SARRsXKKZp&z+jGV5p#QKf7{k7u*h|U`2($6TGglAKsiTw0slR}uw^|}>C^U-COI?CN7w4k3LVU| z031`-a1^sP7dAZBgt(-2GHi@f5$^mr`K}!Zau=PI3>C;8O#n^4};?vgq9z!{+>^^Z2hLqRM9ET%G==F;Ww zhMR9Z-Ns(uU1n3Y+LQlSA9`Pyv8)DaQ=xOew&;`S;pJ5)g(j0;XVbv8hBFnqLz!Fv z^Jb)^>l4*@rLN&ZFG?OkS8cz>7KiP;75@I=!-pEq%yH)K0m3w#HDct*)=dKze}7K@ z2yE7YIXkW1Kn#XBdHHMI^beI|^I;32D1G#A=tvqQE5dK?J;7F;kqCni6y<4VHsvn) zD|`MQ`yJv+FKd>TmpAU%*-MWl`g7z;i}Vn2`AEY>ilxn-5gXjsDHPuzU?edB>5+!} zTANk8ah2|CCXnB+muW1IBqlaC@#f8WBVGaKgZW?@@EfqAY}JXm8%;q%S0{GQkK3B? zi}70esNVsH!@aX+R#(N+MgFVy+4(Q#vkV_Gm{0Pz6Yo(6>T-T&W@ozwJtg!E1qfj| zG`)Wzu)-m$%X#-Y5?!&d!GI-e0hHdENk$U+sy((E2CBX zB08`KQs`iS52S+83WD+g?P=}$Wh_mwP_NreY?IQw)Nvc8Q2LT9yE?&rqjJrcuyM%R z3}@OD#ldaa4iN^bcII>8;YEt5wXhr51li{!`-Ds!H?DotPZr{?Z zmf@<rmCGI}aZ|+`U0-!-H5{jAH(*@gej=)43eIz7=-$DfFfT#C@%+DB1_I zR=7aRzxbqUj|-mfjS>)CFE75gcHO!s?4ZK33$=ht$b;^t>@xn;%8QzYJMJPWd{sKZ z9o$*9X4~l1r+|_!1<38g1xFxhDpv5am5%A}A{Ea2*+H1BOhCU{C7|~lnf@LKplMwW zq08*?7U_r0ivZBEfB^HJwydWU*ctv9LB{|(l2+uJmU5XpLm3$U_Yzmw5;cvlts43W zz#f$)@^$@P!?NCH)i>}t=^oTrmA>Boc~%eY!=uTIt-|&TOBd&fp7T|7?`q`}sNN?% zdaH`v9XUVl3biAzl4EAgCwjIfgLKGQy)LI7xNTba_vSkzIy4k%POjIhE&$oyUqKgY z5e?1=NMsIyV6dD-rOfu+n~;#Je-XjE zi96rm^7f~(v&zwm7HHs*AxS`MDCO*~etyAMGMz6=;zTU|2?_C|%RMcZTxJJc2sWfU z`#j3dR^mGqWf?F_Dbf;csh0)}@UvLLK8gGC@@Pcfh^Mb#YtgHahw#uDjGRx*d2Cp` ze{?4eB4=9x|EC|lSW{uFH&oZ(v1=+Pn>%7(?_-2e&}cEk>#ovj%G+0U)Cf)J{v^X% zK^|xDRC8)6P8`_-tirB*`gs@Y&4?T?c5u1-Lz`vhPo_iezGTs&E5hbpU+1MsIJ}qF zc^`(w|F$~ulJg3^ztOn%F*T%3-~7DTadr_sJCsP`bWhWz=kb!idGiKkhOQm3l^lPO zY59ux}*v^|q%9BpUpZY=L~MwL zP?sJMHhV^NNHI1r)SJK+NE$~1}APEKLAh27N1+)NH`GeC}d0cz-h7P!0Xv@Sdth0Ghc zaj&50{UE-bd_Imb>qX(C*A4_v8u=DK8Z$A=#AGB|n{~z0_I&S_@cFX7H>< z+AL3qL?7L~UaeY9GhgeD9G)qn3v1&}od)l7v)@Qe+UU^@*|*Lj1dYmR&<6`|J$S!A zv?rl$oY5n_vT6MT!*nf-w+BUzSXwPLlo0hq^hgwptcR9 z60)=f`gBH}Bl{r{N3AILtnLE$y==H_U6H~i z4_c!PrQA}f|NFY6uBN=C&bfj{z2!eQ!>tTtXyjQ3!}dZea%);+^c@BOn#4%Kwa@~6 zTO6#hGCI1~l9KFIpw@vt)78}g5f}E(OMs&n8^souOblU$ySrAq)iY=@AW+ddZ!K)#z$rGeaE~C4t}S@cO==nYk_+{@~LQdl17=-X_Dk{q`t}+aCvk z0qqmSW@^(}0~H@;R2kS-UHR4z;1cMq&fCI&-Ia8!)o1~T*x_f&l)UyQ=2sD6DrTUi zzE7dgtIvUrtAwE#`RmW|K`vb-fu55Xv()kU*eJ`iTH-H@_|J~m`TnoLM$!pKjLe#T zM!BoHPUgP5)6d{A)?Ls2PejJmu5Ft^j`&51w9LLcwE2ik~(AwuTx$4iW#* z8JQH#`xEUF>_GHxyF|u6q2ZJA)aTp_C6T~AyGe1-hyVq)hYDo)Ohc6uG60=iOtpqeT3T8T0fM!hPv=wwYGWi|{}4Dc-GrMxSH;3$^@RyCdNi2RJ(O zY`2YDUl%@kB2YK1q0Z^OerP|`FB)*Wwr$tWf17LoDV7%eIh(g!RV+351C~q7Ak=8f zLoR?xa}*+8DJ))U7%Uz&XcgaW(AEsBs7>~PbrA=mgV{L@Ek zOcP&a^P1;~+DHCh&yPNjPqj83I+_9{nQlzoPI>3{?e54_bULrk`xKL&oV+&BL1}L- zwNS~cR{<&{^QMTeK>JrovyH4ik~59P?1h_7a`)v-tUE+N#-}wD#f7cS+R=pdV$uj>~3}lqdjGzuNsT$8a>c#VAFcdw|t6r7(O{M)u zcX~NF*)DXAGG#olju;7J>XmMSNI*{7ed!xVogRC0VMIu3@1VqNRJyF6&d`ZK!Sxnt z!R3lbE4YWD2{NJoH;-5Dq+x9%NMIl!?GJ78b|r#k%Fe=2EN zX%Iy9VD1m$9aOcv^t`90t`aP93h7>0<=S>I3n9*g6)|%Du$SJ(xqy=BJYz%ej8U#T zGQah|<^J1OE{BTy`G?Y{+mT%=aW&q@uEaKDaA0FM;@S#z}02QDq zey$$PJULxR=in=hqoH#=$c$7&q|Ch?z*)Y(x7OE2IU%7oXK1F_P z+^Er7w(OPE)L9#%+C=061(u=#)11zu!Th=8`~#EC{-iDf@$%bWc2A9uhJ6$Uw|8$3 z)%re9-i~U07q5zsi!0rFa@pFqe8c~~icA5N@?X5DJ$!gZ`xA3js&upsRSE}9FETV- zcsKdm$x)iMYF%QvX{>b-CK?Q%k$8=IY#)+TvhGh}5=5LJ>X!RVL=pB4^62&^z1oP) zRJN8N{j;guZIX;6(dvr#2d*I{7R-!0zzlJ%7+&b^`yHEgcA^ZHg-uVR8z7jV76(+c zMkTec$=v(KBH(eR4M?O`G(luk1!-=>dBZMRP@LY<`xw}Vv`%uA!yS`DokE|Wl3k0^ zj6D?b^j>u2-<&KsFjeHJ@~S77Y9jpTmFbbaAIG*gH2FAh*xgTy?M#EA_?xq#@fk;n zh*JiM7`|=k6A&G=#CB-JwL_ReR1BAB5pp(cq$mgY&q)P9l87TTOCsc~fe2Tnj^W>v z=Xw_2;>K&eb@1zCk(e=&26VdhLKoApUltcmhoab!LA&Q)dBPbEU~_OOmP^MtkCEE% zKYU2S(VO~uSv$js_R@)x6_a`QZd)b!m4#H3D;+IkuQ&X#G_?D+Cx_7csVJ?z7C{ne zERILpy^BDP;y&E|Bkmtt(^b^nnkmqca%=RyrX8pQZ%C!p=j*o`D*-U5>agIhZ~gew zz`Y0AsGKLhOu@a8|MF!g@!34-%T)O*p8KJCFN(z`NauAL^!~A{PrgOcx72%q8pFVwC{{q9X=@VP%ebv@N z{wE}FcjuJ4R3qCBGVS-uz=Z#gI5u zE*;D@(y9G%UN&}w?tnePr<=8;^Jfs8{m`L-Dy^HUpKg?S3B>!>qesS48o*^)Yy@wk z!(C5`i}$gJ2w$yu*Z%NoDocwb?!dqhOujh_7n!Z5Z;@$6fhz$avCm5RMW(4OtF zT^afh@pA(VeAa-bS}#Ko;_+gCh|$wNm`ZB0CUqHEG>bZ{VCcN2Wpq zU7tHgVLE&3)~$UU{YX1W7FGRqWy)Y%f+}|PU5om2c?kCf<%)CJlVlY?+fwJF+!|ci z*j}uc<(!@gjGTa+p^!lv5H^}ZnlIQz6r0F8ficaON;AThOB`5&ZPaykQhw(t6?Gmx zer$KW3YVlqWIhfV*2{9kXqt6p6vxI@Yn4M!?;QpqKK0jx1B6c{PRoM5D|4c=c-w}>~vp`hnadkX^TGfm>m zu(x=Z8;7m&^T@9=bUL*BN za)SircSb2ESE^2#ZRaJn99x))H^aqr6w^T@C%2^vDinDB!#x_#X3*o+AMZ zU^c9cTeX^+{g7ycMk08_jI4XwS|-L4p;AIqIGry(=KOgY?U6r3A6&@aCd|V=x95Xw zY;8x588g%1)4V#vK5U{cAooIJ=%c3#`{0`GhV<>9xVSj=LcT(N3D-C-+=I^lRr%8~ z3SFkE{9E!Zp{wm@x|jeZ5&HC)o!7pyaRvV-GH1O{cpzxRjFrI08$Z7nVUX1dr|zG= z?qS=N>N!0~t5gO0@8(@rPdR)Bi3mv^I2Uu$I1aZ^g^ODvBW= zHANx&RUo}v+1VYG%z~}z=h`!#Rvz79{GJiYDMby^GS$8BAe(Cx>=%FJH(A3wIZZe?Acs4NCC zJ@Ae?K1oIRYNo^!wlU{cR!)wpKq}lrLm;of`h^GIS4AuOW4_sQBaLffc;Tb) z6NO)Q_EJ-Gvex_hIiT1sT^VW@l7q*70ya~@9u(L0E3YE z_pilPlpfiIGFGE{jT$X?ft0Jz#Jx_S z1`4MVal<%sM-0Db%A{yzi`et$CG?$$1eN%WP50;0{Wwr$>Gg0&ur771) zBCnHESGiD5t-97*CVC;a$w|dsga{LA1{Aa~6X=p{*jmZO+UK#kfu1GB&%m|~5{pAA zt|iC>80qie;DQCA-6=1nyr;uehEX2McyX243MaTMELcA6jy=nG=b9H42WqHkenlG>GZ&{wV|HHFY2KE$FgQa9dRsTY20?jyOSDswHRCJ zlF*b6|63XiW{fyhwu>0_iuFP9q??4~tp*xBxW18y^(Ac)z1;G*bAMYYt}rwbd>?2e zbS%bNR_lw60u#_37eLk`^+4FB@}o$iP7_mX~}% zslyN9;lm%D!;xdZH_}SvbyJouwmNciLBHb-dJWqe@nu-gzB@lYzrXXf)!ZjxIoV&V zLh8Mm)y`0F_=lGEj{U~^K1ut{wZq)id+w-5y%>2i<6^&XB#qunPc%qQ?&3PXSHy+N z%pb!kSqq;2Zu!MpM{De?gSA7tTJ*QG`)%T+z|waP(_fD6Wjw{Qk)`E{Hs$6nQH{o( zxVt+dqQ=>?6aQK|so?2;g-1;LXBnUTE?#s=3Y#0#_V2t07nc|B3jF*a>)FPa@3-F^ z>li-~QBz}6vzNOauIaQWS*M{{z5AjAS*M~OKAs0qq&fJHha)FH_TN}f*SK+WbBji` z>MXqSwc>lN`Uc<2!EICYE^M!=Pu2R*KbJVi`<++O*TIYCjNCEmH?qD+bx6#uyXBvl;TB%*MZ^T^1FX z-?=ji$+2r3c3ABVwl+5Qe_8HGoO(%8?twr2X7UadfHy${{EbL9K%(ZeY*}1~Sna?A z5=zaj@GU)1a+zRks;R_sd)s#;RX=;U40*x?B8Q=ZdtHKht&MqZGT)ViouPC?NNO4a z_*G7RQ&6yq$SWz=|eLnsN}7G=@g zEbZVs@C-zcEz4if)fjsm^?47+$pcPEM%S-bred5^?#88g@iPs6r|y*K%zJZ$$-&ZhY3{Q&*SM#_$mq(0ri|R z_ob5-~oZRPoNTJMi}u^lpw4p=<0XPW&D=d6!) zN9-+Z9{D-dpv$TYm7inAk!jMSB-f{U%a#Xpa>DM-v30mzjcI(Hl_6VwYD}9pe8v0~ zLx(nbmgf1}>(}<)#-?{`JbjwC?8&FWZ7=$~EdKFwoO9`$3p>9|9?K_d)-zY{{t~N- z8J(BBUaz;nbK=By4i5FS=01#%f1hr@?DgC$C*wcuHP~OsJgWz%&Q+W}xhu*0Z6izf z_=M@xmoZc>>$mvj)28WiT;KTo;f{CroyY6lnC!SO`&8??7b|WKrWK!gwZeQf$I6e7 z7f&xJ8d!VB&fjt`Dh5>Md*@AB`F2_R<=Gh;YEza^SbuY8P@U;PmEH|)M{TahA087| z;gYzra*JI+_0p*DE?3UY(^>t%Wn4&Y=I4bhc;WDSojQ7%k6(Z8_qt1`>%ZlFbumt; zd|WZ2L+|!mIjnm9M;d+WUWWgW9oq&kdb!~H&9G~=D3gp=bhIzpkP`0q!%8i`Wy!ky zL)Rt`+?irOF1pd2imQ!U=qsr;1^&HrN8I??A6-49>|tSB>&DZeQMBr_`*`gU-2O1F%Lpgpmn`3$ ztgWe|bxd_m1s0l?0c%JbsM0iHeqGB3q&jgM|FFb+0fl%aOE!x*HgiI+qtiLOZ;H6-MS&S-csU9Sq)0)G=D zJc_>H(|T54^xKr~=%u5z?-!32IS&-KklB9P}6E0{_Dfr#Ql4~(!|`+>KUgFeE}X&onSd) zHyT)V6>+!BxM|?8ASuF>W>*H-5Inx*B9aCPX_YJjr`1M!egelbHm8=rvEAZItMh^J zo0ag$;O|%QZ`EEIFqv3lJs=^UI9hTP69sb#{i-T$2^>azi*awO%wSc0xbpgqhh;aW zxa6cxseIt*;aRqh_dL8vCDRI(FJ*>g{9&B2l!pnA9c8>R;>}Xxvzc}coxCb-nJqWIbUf~6O4G)r`|&*c=pl=tni=U+?f zKZ=gL@Nj*GRk!4vs79ZIehf^EzSz>LsJP;a!HL55R@eHE8}~^!t4pUjiA|<#dD1Vh z*_Zd`cXMW6j=t!#BsUy9rTv>J4<2mz zobe;A`Cs}JZ=Vb5-yu#$gJ!c&+K#6d&))C2 z`7q(X4;FK;;R@8F;v}L^zxCk3oS<1VXSSvTVkKbHFc%O4efjQY(@7)}>>Fu?jZyPQ zGemA>*sIsLTXBr5YDBXPhuEwv&R-SPLmH8IQ9-o}LHJ}=&3LGM-0TLl1lay<=#nWI zET2^jZrecQi}MpI4p`8W2J(xhp@nn~tB;UN@eBheXb4DQjx&P7TA1^@b&NQOj9>$McWING8mfWTc|^KoT*c?a73Mu$vE*>#Pqq9Co=^)4HN!4E@sBZRVb_JB)!F zBufq0(7S}LaD2>y)(z_R&pMD8>1DM{eIcx2@$V|iQjzoq9FTndtGE5!WozfxmcI~duSvFgQDmlBiD8KBz zHwVc;2W8;h6nG8fczx zCq(3|a&Tz-ryM4sdfC^qRNp@ThpO`cr~3c@{)q~iQT7OxmJzZ!MN!Ep6|%`FJK2sy z%1Bl!$x1THCVLzblB{HpV`O9Xv2dcR-e`FuPrP>8Mjp>I|8-IN>F03gf% z{%D7U>2fEE0u&D5*W{gU-|+qH2yYa)&fvfL;bFLD97YrW6x0F|9859)qvpyd_S*Uh zr60VT%{UMuu#p?DFvKBF@;y`?G!jg5V?7+9(HO`7zGS+Klz5tG8MAutobfARZ2nVY z!N%Y6dMupdj42P6T-^093cbGAXLDK0RqDBzO2Ew_MpRbb($`A&6kts$hlZ~StQa4h z4`OyoOPlN(4(T&93&#*G1>-bBN@|`z_t;y{j8i|OOcTP)0ai=`Bavv^zRtb6b^ETz z`zn3seq~=9=3U>QppmgTg>u{l*|O%2bT@Z*u193{@S{^xe49{+AL5a1=lmtZFPV_r zUq)a!$39e0ixYo+J@S(tjpy)T)`|038>3#62Wv@-#mG*g9`t5hv`ijqMhSdS-%0twISS4p7B2|lJHHTU}Qpn zH*b%&q9Xs;+valEWN5E2IsI$CsZfvIjZ&(i&5L-~HOOp~dlxoWki1rdr;3nP0PhP- zNxk=`#Ib;bFKmV^9X-U?^{DdX!v8EI5PLD47n87lN04L5-qZ1y95rC$1`;^0tr5?~ z^RVVY(jB0!6HbzeLG;$Je1x~(`^rjg(1#*R1@mHgZB^j5fmQ#!s;X)Rp}S%ODv_B7;D>EbIvUEcgQeL8?&p zfc_5zTiBoU&hUnYvxfFv6;wD7#rf#Q+Oc=E{|{4Fego6Cj?p@-bssuD@WqrwxhwDi zED^*X&jlTuZHnns(EBceyDaq2)yv9Dc$~Q1XPEV#+?x>rr8bmkfZ!hBKMVQFFm@Jv z%|tk9^&#dL9I^k}87S~z=>a>dQ*gW-AijZ;5|-u-pmO+@V^6+sn0+48xUl!U5mUj92%Nf4^)d4z^p5LgJFxxc!ws#8=(K%W){q{ zCeX9p#3DNgE7;2KKX5>K!vk>@!J712>dkaGRACdLCNqvysHS9cxB_beFxvxRz zhrASU$snUvGaf+9gOtT3vO^B2yhw&lC|F?rAx^9>Up}4;b+K`khZYMx9bNs)6Pk!- z92sMQTHa-8SRM+z^(t-Hdmz#}VqI9%d0-V62!Uj!lXcGru;v%i2rnI@~4C9Up#rDPTX!rn?9*< zBdM>;|7gVlRS0CO!^6Ws@{`GOeAi|J1Ec>;j_T8JZ||h5yIIq$^F`$m@`(vM%S`;k zYljcZ$9^`_+RPuxEVmr-oR0Ex{9rBqQ+@sU16m^%R7JQ$-Nt&7Ixj^_W)-*a>GWVv zPlU7IBtxDl#{2Rb*Ykyid&EA?H~S}s@R8Whi#fdDnWn#L<=g< zTzn(O$o+_noNPNYI{nS!HPSq1Ui%qzP=@fP;EkmYh}jttGvS7#gd^o4Pt8$wnQ>oy zmASy#*OJxH&Mgc`85N8|Cnxl0x);VOIyd#Y429^eWqN&F)l!M;rn+^ zRa>ZWl9xkD-pY4p<2QVlYO?EdO^F{;EBz|C^Kt&vb>DG$>W~6dpJ&pX+K2v(+iegK z`%2kCpB?*Un*xDq;>~#`BA~ONM%6)acJa~b(#;7w$Dztc!$R%+EF1slY?}QU`K$_dP75#A{rlna z23O?vqyQxp^q(IpD!eb3>F7YJ%O;(sGa3!EOuhm=9`cuqw`zKceZI z2>cRoe(JDP0YnwMy{Cx?2lzlv2U0ii{xpSy(M7&` zMhpTo{QZTEhQks=GBXS6)O?AZheA95+yeIrhz2kH1XC|ahZaIPSfOyGhha7J@j84H zFq%QMHelb--sp2FC!B}CU=(m4Xq&(xK;5nip3et)AONE;>_5O*dra`K zd?ttpXq#YP0?Sx-#FPlbt3XKhL_uW#D57Wlj)T&S`7 zAOI+`e*;qPGboDc;3)q1^Ir}B-=Ah5T#%i&C|2gyxa<}7rd1bV7{F8wEY)xX{TKHI z5}ptk`y<`ic9U#vpupZ3&2}bsSLCCE*TF#<3qA$>Uw$EL9T2xYuG#cd(HqgQ)?R*t z1<_D?eRP!)nV&)5^MnV5aMa+7UF0LIvhScs#B87ta9qx-|i_? zK#9Ftp|9s}J;(ff0kQ<6yZb#m0N8W*;DoTRBfuW`>h*WeoiS&fqS0ZncCN3* zWLlIA9*aRaR~M37@X}4>)wqOHqL|zB^HXs%V`Vt4U`;gfG8W5NyKPBh^V940%w!7r z?}stZzPSASv;J!dcdp(@1SR@zzwwix5h$zN;1q4<2VoTdyM&FX9Wkg`zz3)u@b4vR zqDgDq{PTLY5LWhQ=()Re2M;~KB8xwq~8}CMmYs>!N2F1Q7Qbmmck~^%k(C1B(!K);Y*Lb zh0MLN=`>Y4KUBt3mzT{~B_leKGwcI|w84N9utfYZj=z>gO{+QolUG)%yg61+4eo`WJU7O^`R-j8TX1ok3C z03bpNV4Vt!h-Bh3s^J5Igx|s?{v13`$SCA@f&na=kWm$U?WO?+qYPPr1rUXD69Xsj zhnC~Vj-g;B_{1g_X#&BR2?L%?4hR*3y6uwM7Nl^&Ox^Bxlr?dV=mVjQ0^Ktr?@sO3 zdJu_w*DsR*6votOWcc>o$prXDhHHHLR3HQ#iwEw~oJC z2V2&d`ymt@8NgjHg)>(I=odil82V?{eDUJQ`Lpj9;1$j@t`G_ZvvS=(!h>BFOf+D9 zc|hxYYisKOgTqVsSCP@!F@#o`n0O8lKw|48<8CmYUFVl3KPb6}i2F-Td&ZH$5@?$v z)P<3yeS;p%(Le|cD)2)SKz>b3>({_f{=<$iT>hO8?3;QyY3axPLq!7)2>4K4J-DF= z8X{zM4J+J+QI*5;G$5x1r2-@$!;3$q19Fu-?rc>eNQTlPwq z1xG9bB8+7@91{w^H{-&DkgEK2+#bPUKhu%Y3>O;F1esldk`+mDl2V9a1Ia?pn2I6_ z25zu)Q#|%s1>-2etpsWSIF}Ix4osX$dBU%+BWni4tIyA$<9g+Cdp=^&^IHvjZ^MBC z1Sf){jBXXNS-kRpmv0{md-X&9`8RG`BS+lH}{}%^)~9l1rxgm=j;0Vj)H7}=3$8@ZeCyC z6K?sFuU?m=<+jm6ET4eDQ6ZsfSV1zpzFrM*<&`S_yrLo**lmT-Fe_LT_QOVtqc@N7 zS>{ZgyA2PI`}gl&0Yya~2IubXr>TNiC6*gWb1K2{)cX0o@PRo{Qev{a4B_2h>04U5 zQ+78D;c&C>o6|MR54t-99GD9qK2#Sv7)snPiiu~7j@1j9m|*|i`_Q6z0F2+#=Zx4L z%tl8AZ*wBpwhN~ z=J7aIe~aX|&{>Q9NG0eSeO6p~UqELfdhqS~NT*l!7{^>#v;q36#_Yxge(6`GsTt)5 zOxk?jGT3}KxvyJMQFrz`V_;#(ZO;$I*O)&UKR2YSr+u;PKj*l0*K9C&uu8PEl-vB5hgzyfpnmkhOED_T zTur{$^sJlBD0x0)GwIPY+wh**9X>6*>De>-{)+V_;lcSY1@|102e|w!aG8N~&owH& zkF2fm6Ekx4C5*sU=8XRs`z<%tyD7y zJ;ynED$^Ya_Yq%r}DmKp@LoEY^dvZKt~dCm4_ zH8fq8j2qc%=l_zX#LE`Ukdkh+v2UJuhv(?pL@z5Oky1%*DmN-_@H?6$sJKe!t*DBsouSi(mi)7}n4#E=4UPmdH zl8HDSc=zaWC@KxJIoFij97=`6^ZDm{pGAB}cDCxQw*3DQ)uQU|mneIm;O^WIUz%JO zSnV{;bT`OYxoa^%-*wenl>Jux5!}EY-QYvtUZb@(xg65wUbE9s)}s1um8|Pxch&5} zCEP%kb0|%(zasw%PXowp#&{ixB#LNk-nC`$9V?Kt{r*gpNJ zBa{94#S4=Hy#DO_#!cn6GUw$e(~Kb-%oHKE5Nyln&Nja!pXsTccxZ& z$K_BGb_|N79(JOsQAcc5>IVAq({?@FaIHrl%1@)E*z9pq0#2hkDg&cskmisS=nNR+`k#SfB%)DQ@^fCs7(ETSMw5u@T4gjx^XD)ZtCd?pE~0KYGQ~k zuz(_S;)LuC7;Q9w7zVLI_=3M}JXS{qiXS3q#faY=0d!>!3`{uys{FgsA^G{3n3&V; z)ssG~;Ew@`7?S#{0US-Z(*A)@Hef%4WXC`X7P29N;oWIyVbw+R!MF;VK%LHQ+5gUb z?yUYwMqsT57l=VM1gBBgIok zo}X_E`aZDXh!`%fATdJgTRs+(y6+r7nhaJF2e8F9^H)L66d1b{X5Au?H8t?WQ9Ss= z5$_R1U#5m-gitHgtr>CjRr5OK?H}Tk35lU3L>z2gJ z(wqFfQL~WMYt@mgoX8^)42~^C%?iy|h`o5t|MgE$#|d1i2w{!Lx`#)5R}!Rb znP59!zh?2j!&%|`6!H+X6qTW$KWm3r`}^l@FL};PW}>wM;0Q73Psqxe{e-5hrHD6( z`^T*JJ9>J&TTZ|mH>OaG19ts=+EH~mhNIh4;%tSs9kD!V*hPC02%}Vv=-1DE7Yu^~ z3mKyo{5qpzueH2|qcR_BG1?3nyin zIb1N7(Vq!Q{{~;e$X6U`=)#eWZptB)AI4;*hS1gwe{BEfeLm*O>q}K-%NzMAMNtCe zB*-?~+1|q9rkA&Nd-9v!mzQ5zR<79~W4&g_=P~66s|ef?n4j?Cgqe{gv+>q7zOl+* z%;#??QCg3Z^3b<1{rtOsW;kLzCsYD7YI<+@NVV7c^Hnr!z9&e~_3Bqtn2*bNhAhJ6 zJh32tieqPz;2iqEVtBU#5&PMt2G~=u3-}ROr!(yoKuO)vP z^FIBiTzt5~Va}?|wq!8cbAn%TwCY*YDtW8tq4)__v&4wkW=ZrMQ?tWBqs<)6GdpfE zUMhm4c{BZcBG);&Gzz=EqwPhbNUjY`=4S1S9KF&+^UN;q@q%V|f@cBG^zjh$HD(U< z3fZ!_>W;EveB}|@8qd4BIw`(FsO=kW8pBOo1}l#kjFk^2 zhXcAW8e4Sa6Sg)G9s>nuqWs#RQkblnoA%L`q{@DYucjqy@ba*wk(ZQF0Z&&aT^ za~~NtE^szG=f4_zf|jU^=fp~H@}lS+C9`@71+E(-V*P}Cy;ki*jUa}VR6iV zH%7LnWjj!_15vn@CWTW+($G4>!fDrChSwH~4=-=HgZqu|G@;s!zSp$o$kt+lY|di! z#@#@Y0cNC97iIl>pC2|5IW7?r3E3k<9K1*75g09OCbBc#+z;v%8erWeGkK zt)A_59|=pINrAW7rN_!T_%F?w9XeQjiV4;Bns0kMMbc%g`9Z=sx66;_cYUxqnsOn7H1xo9arY_4;}LL#FlL zQcDW6_!zw6X_$Z(wZXg8kt79rGcx%eu9aAHFGp^*^+q+(b4C|aA7CVjWu~5IlDIZ`4LQI^tHaRb zl_-RCtwznqnsPn;>skkzpnH1-2#~Y0UVF!5S00X5dxM-zF16KMVhn=XnY7qwUPecc z=`u`%G{7N8>w_Ms&q^J*YPV_#7BCV6b^5%gGK|7OO*)PyPofdmR$6*GWilc21VH>D zsVHQo7Qh2pN&>K-y?+$;nw)*Ty(*=3f-%J~ZS%0RNqPPH!fc0=a3da%AD<4ZWuyz8 zZP}IiA;WpF*6HG9RZ@*E1uG<$#9FPPr;KhJX8GnaitK@@Ex;XPLm*!U zzzc(-pz&I06C*IHRZmuHU&fdKe;0)C9N!>fA?%<3A0EUqXoRi6O~>TsVP$1iXweWX z!rqkd0zt-1wmfbR21hZX<#+=~CWVca#(P97FfxqQ?L!ci7KX6MPWD}r|2~zb_vLkH zQ_4R+Hta^0mUP65u1b%>D(W8w-N`AhK@xFlZe*J^LjG6(i-9m07=BDor?Ujt2vBSf zi-1c%2I?YSw7)S65TN=?9eFy6pQl%7=hr(5V08GA+CIn2J0f^ z>Y4eiJUa%_OF&K%U-qML0?rk2){tQzF#&8A>h>1IZdSXNL>L2(ja!z6+T}kv$h!!{ z@(pm=#~~3yFo$UQ|1NvjP>tUs51pFdS@uk^=7vx-|m zhQ;I;W|ZASb2PK^)F`;c`m46+K_SLFGV(@HS;M#LaQ0oyQ@CyD+uBHdheX8*kUO7( zYe;{c7C#Y7mAdRdS6j7Q-X6Dr^)*C zm;2Tzh-1MPbsC;)H=R9SK7zWvfE2G)S4eKIb~t707oG8-87-!Hj9gj_AAL{7NI}Z| z)0K?W1o75K^DDdhD$VJnXKFv_IFnS&r=92{_axbzt9@8c=sz^~zJWhm-1THalQ|j` zkXB^_13As}W<)9t|M;+`)j-OoyQ6v{QBQ@8*Gx%Ea~;D>ds#1)js@Xx0amznRkx{F zqIya1W5KmLM}w5k$-+-X)RaP^l-~}K_X}fZD+@}2`>`B3!C`O5DAVFMQ*KIXYU@+k zt)y@9Xj2W_?%Cg*K!I;|PEmg1mT#t^(J+a{i83q1?9NdA9_91GI{ZplS=wqJ??jtj zw%Vzq;CDTQe~ngz;R(A(7}Fiq+KHwfjAQ_Q9x?Hg7$eMS`y3^&Z^YVheS{5C2DN^Jp-$Li;7!)5_rllqpZ6Y(F$ei>yJFNF`g0FZFcp2j z4}JRd7fP*H`%t={{&KtF6pS8R=1qE1hb^~_$l?Y1I{=M{n~U5 zY(jP^45;;mv~x9^JwYU#r<>lfYJL1btVpZ+x|^X!%A|&wS+6*NWLCoN*!~+P)1u6a zRLB!YHwdkG6SdJW4H1uYNje+Rgy+jw0$nzA->2P-nh3v>RoV0OSE7R7X1+du@r6`k z-@}#BtqyuIC7R1#f@dCZ!zdWG*6&lR`D8oZ42gdmQ8Hpe^%E2Bhb->gm_MzMMUWU2 z573-q^*T_wLEPgfzq>S}z`2#zE3~+nP+;h&>wEfemD52gp=$HuL0#Xnqp}^Z88>&x)o2gRa`Imuf+{U+6W$F*MvNF>QQ37_haeV_qwg;ZST-#G~NijwfLmeN7O0#v|59i z_zTXGk?q+`uI>6%+f+L!@W#(R-N0vgBmpI99@b;Okmw*x^c1Z^3@XY{?1-2v|u#5Q6at2gwptw2Mkj8^B=l&<{jtkicp4<Q8UwrNz4b2-SYy~8I6A^+;o1LK;TwR!>oWD@C?c|D z54TF~kawF#@_Ag``O1$;M(BWJj%iCq1P0-5Ck|^Vz_+bopIuCI;R31EiXJE?~Yh$o)F4iBN_1QqRNL?O93fsH8%N^9UR`MZ-@RZ{8q#VKYV_>{TJN(u49ii&9C>^( zR~01KuyZ~#Fi-;R=2tvG+K&J&OZm`J5TXfH#!#AaS z{&RK}3YFsbhy$J?0c^FqZiht)mG7osWNVx~EYjxtLIh7(E9~w4wsX7y+ZxG4G$l)h z%HPcitwIlD2U_6StepU&m;3V@O2HusZr|y@{_8gn z!yLcyjp%rP`3mjaynCme@$#@+aek@f zXZrfH@si7y`?K2aP!Op^SI`o+u4A`OYLoQykXL*>{0U2~9LtiPahGGkSpDM3Bgy#z z*PMgf4*IQB*+-wgZ&0e}qw5`^E+7G2wD2u8sX&*I3{%QufBQiJ}+FX zA9(boZ=+har0>B>Q*G>adsV{u6Zc(ehPhjZTqnx(Ef(g22Q|aqzGB(mz8V;QJ$U-F z_OP7c^HQ7>F70NB8oDaf8Mnsp@z%I+q-Q|_CMCo1Lcf{pBY1Q8KMa|{+@gGq@z{7$ zVEb!zN{?m(1Mix!uzG_pr&zi(DjFKp0^7COU;zT=K4qfoCC6O$1X+PAm+JKovyAoQ zR>z6_sN4FU^~-%2;IvSY$&nPaSk&Ls10ZDftaW3$*g>9;knL?~w6@#&qoVEP#0Pa! zGRgO105^CmejFa!ocvP81L-&>L^+u7+4stI_%*uCoY6HU&%U+k%lm#w0KKtUhV^Ay zTvR4Dq>#>CS}v1H{!)5q{*JG2_CW)^??Xo0*wy9oVzh0CN~Ol*m5Gbx5v#;0EX9+* zsMt4>xi_D=VYH@L!(j+M@F3J#vygZ@4)R2kDxpI2ODI_BEDb+sY8K!~TgoR6&drJC z7`oBQ%jehSr;uh`C~=5ui=kl9xaUm_145|d~VZv#n{*9%# zcQ2V;=|ZOui9bJkL9BnJBP?`8qQPQh16zzrp~&#`MQ$9CukbGA^**~~RsSab$p(35 zB?%SahOLZTa~?&(JnPfqO4pmsO|AYMnad}&8tHNO=t?q|_wQ8V;-jL-=lRm*-BAJW zKCpizr=#X6Hh~SLZx%wo_6RF;-29SzaJVJw+wcVoouA`_nNecZr$&q=oJZ0q9Uqn5 zqQBTOp~eI_s_mnk_eNFE4a?YfqG!|=Hx!JOeK_*?4=wEU-T!%=h5h2yK}DLzc;@AL zyjehc)y$K*3iOolnFnJTKfP1%8sruYkra%Y((sXH;~b4hbuaIB@ptS>Vb%hnQ_3moLLYREfAyYzLRT ziAj>ODOkDy^I-vWQJrCRQ0gG0l2gB518E2V59U8!#Z=g7tbc*EJb+g~P8$ZC29Q60 z0IM9{-5JK?APGW-4+xDE>_r+mS#uEcqibyZVrfZ(Z3|(U^G}pvG%J&cf!HlF2_RaAR^*7Hvo9SUZ z*nqNUy=kVNc2nZ+r`t-IsP>UbOj6_7}x%j-675Ok3b5cKS<^ufQh;Rq;LfoOYiHo zfF8=jprr~Rbg)VXBb8CqgYJXL`5LhR>CYW7wLjrno!V>St48h7WF*yzRTTP{dHj70|=>|@qes)YHA_tBqSjiWZ|%R z+V2Wm93(jcEC@PD!iVMKYG;Z;U9olG&uy@%tbj6W9R6yiRs<=HWM`+KQ{21~qd{{D z@YG`_|m@5G)O-Y3%-d7#BG4`45`baJJMMX%Lf6m%#XY?!?Xgw zZwc=vB@v*){P%~ns+fsS0F3;NiE^N@)InoPYP*1yvtqT_ zOd%NHruuM8-`bN>Ap0nJ_dRSI{u)%t9Epih(fLUtZTaqAIC1jeWLi|I`R-%p`c>>+ zb2<~A?a0S^^j?D{%!VEI?_bBJED^9NPtc^(0_g2aXDX|5J7LVy?%6J=Ou7={j@~QC z6j1+-f3a|_>or?)TXF0c+8Pl^34A2-O5P2CEV^E z9nzb8EJh3zWEYq!@Hl+%OfvC^th3BMRPA071JaJ?*Ysi|IABDKjI`3zmA9^5KMtz3 zcL6D6>3wHj7l^LGX5=aIknp4>Z-a9(!&QC3loj-?l{lPIZSSBOWhccux-}iHlRis) ztv9tPexNYTHWKUYtq2OTR({HIgKc!;Ie*94n89X0f-n7Y4MDW7d27&BrG(WPv-Slu zAd4TQC5$z9vhfGiZ!s_YbDcZhfC3313l;E$LO+WZwD;TNC;9Hs7ov z^QJbpsjTh!`C4&!#K>X$h9(K+V`>Hq3ZbbIgYr|)^7BjZ$x#apSCF}nfvAkMzWYQt z#g|bLo4o#w624ORQI_6^sohkbPkax1HuX01aCa`4(JwiO<;?04j&iR2=XmE=?ux+_)plgVRMP=QLQo;>yJ$CXIWyu@w zWY;;afxh1COHiIHSgP?QE|`gkV4tq9)1U(Q6m7;wJVQ1qYf+T9fuBDoW7=U@1)@cT zj|41l@xXHovAu4!uQmr+cUITj5iTV)%X=*|t@{*X_U5vw1vHe}v3nZ){(F-Q!#I58 zriZYS^6TX_x5k$z0+CN|m8Zs~PYV?H9ce`A_+$nCcD2!_@rFjk8oNyys3L4<>p1k^>6cWDZ(wemA#~pgJCy6Zz%AFPRb21-vs(Yv{|h?w8WktgNc6oBaEf z>KLqo7@N#_2sPHNYD6;GCE8DP;AMn^w-TW*oI^ol##Ff$98#4t!)vIzkJp^GcRFkh zPL$F-4>wx=#-FJ`|ZO)bc^n(K{FsPgG&rcJlY-^l+tZ-1u#tT}z*R-_e6gYKqD?}b=4;EEg zL6Y?@2dCgudBu#pXL#e>!uo@XW`6Z+J?<6LApUZ%Tw#&ou}ws%R02Mo|{L;E+PpQ;Li9R@{1 zvsoAM;srw*U5`&OK#N=|f?Hofz5{stZB9HEykE{K>%13Fr{0c2&RcvT?(_Tgp(5iv)04aP&5QW(&J)@Wn1BLP)3~CYH_jS0?bH;2Q{L z3$_3|ss*UI?_!|r0O^YZ+Y!O13vH-pI3L9XVwZI_Ix%aD4%Sg zZ;&X5Oauf57|?J4#`S7BaB6|`tM@}1UCsz?3`VY=W>fneutWT1@?$Y zxamb}>+}{G6%`c()`JAArxlt!Ws&3tI?yCMq3RpOHf>+iqExQ;>C*n^Blhoe_U$=n zu{C!SFH!B|5YfqN_!+Nqa>!{|?ovwP$V+1t=UVT>B_(_)RBCjHUS{w3-%Mex(vmWc zvB{`7P+aO_MhO0KHeK_xj~RIRo?X4}bErGRJHcUXZ!NaaB51rMieL1yQNriGjhf}J zh9rBMg9<{Tj?@`9&MQYn;BZHvy9}$V%lo#Pl*C=V(R-)Z70tpu_d?1xxvtzba+-(w z)ES?BXyU+QFYjwUjtx_fmV*Rt)cX=dfA^(l+#T!9-8GH#=2Y7-Wj@k?(os4b$&5Cp zQdA7=TI=fKhw8_6z7>qTaxU*tf`aXG7$3g9y%)_ja*E+AEh3s50KW+dqG_AK%!wyO z)(c1NpN!Rc6aS*-Y?H{?xJ70@vCUabHMjSjF+R?08sfAhF+2HKt#3xoZO;rGd=g2m z;>{CDp40QKRUT82Kx7j>iKUR9!SOeHcm!v)X!Y~r+7P2PzYKp$_zbybf8cpcM@%N1 zw%fhd7=C$M0Xoys9a=x(ksGC+t1r!muy4K&WX^A=-?eicpL7VfEb`zl82oBuyt^7% zyC6TxC61~5&{yq^I>xegexpz~XUDO+Wu{o0{ewi!Ju_2s>|cZ3pieBJH?Xxf%7?== zOcPSYZTUrKFU$IZYItn}^GMDkgqdTfdbf?!OfoL6wj1phg{M#_Q?^*;h;p1IQ}o^jr1^oj z)yJ=z>Ay1xNpw5G-q=3NdE>1tlMYFT>f5chKvtjps6q}J9pC}2Y&AXeY9Ir1DTATCaM)ZaJxP9LQVX966VzP(Q{7PXjcqPuH? zH3J8o1ePzVx!$eg%0^eEr$*$khK~;;H;-a!*DP?v!hE;h75I9Jl{(0Sg*pnrHDy2Z zvjpJxUAGGaLrgC}J7;LPO6oJ{=v8eq z)y+1y;y!%*cwlV?8+5@wa;!6z>Ig^K#ZSLKR2T+=m~d|kHvqyyG%M>SqKf zY3B?BK>w|(0a`lOrM}f^_`vRw(&fA#iBeI$z*1jNSB0F>w5vCy7i7)+ZTGSG-bQ{H^*wPoQzVbfGO5Ge zjT|gsfa9yKKa_S8^ML-jGbih8y8#aqp0=Cs+pbJ@!#KKSWTA43k?;Fl?~b%*_vKhV zJLTNH#&a$^CPA$CdltV}-L~+EPu=Y2DfU^EJBW(89H8NEGi&$jvwOMsUTsUMR3G=t zU=HKlCpj549ewQ;x}~|MHgdnEAAV$LiWDZ+FnUN3pG)gSp*~ za38y@Ec#LSKWZ5fg%m>UdK3a7b|vp02anK=7)oZkEGZn;IL-qeV?-GwH$rD*># zI+yu3u9^S;oy-DO3mMu-gYcvzzGSZny0oxHCO(DxGAlDKQ?s+7pz7z*a`A8}pNv@xf4dIcOtfm><<8f7mEX$#4z z&bs}Ec}~(AAQ%usVbOyDkL|1IWO#N9UxLkT*((_T74G;02YPYx!7N6AJ%aFB>Fgy4 z?NO(@Koc9sLFBfDsRUk3n={LOv(mXgCPQ}wGRnxfXicX|%asJo=%{HM zj}Px)2Jy>tm==I6fe3_HBZ~iuZ`F$N1=!<&vF$6UgU1PY+_eol!G;ESnIm?6AoQZp zrj0vo$28O(DhYG&j3JgdSXtdDU0GhP)6-D~rmojMAhBCLN<1fcf4vIQ1)-L)z731R zD2Pq&DvUy^;D1*%{FHMFjidYYDL7F{yp}VfzrWV4!C^=SY9g-$6HhHI#m$9^Y;oM? zp!CmpD}M(HkswZ$g6|PAyf^qfRwVrAxvr@;SzAH5{)!G6@tPxP*qqxBW0D*d^4rE6 z4&?k<5b!VGzqGV@a!6(D&+)HA&K=!xTZ^qgGS1wo+lPFkD|o^0X32-^V5rUjc_e56t`t)v2_;nOe~hoZ0wBQNSKZYV`H35$}`Dc zDp|a^vSVCLAq%-w)NI9U?<$N%VPEvc+1o3+tY`h@6(Ll|reb+HW`bl#-&z!zc;?qC z9A!IMPKDw;xV_u}H)2%8#(m6^x}-h7)xno|r>wPtHXlhUmGL~Ap{bLvQ$Kx`5MP@! z>j?gKWw@C&07~)a`iyHVQ;BCURsAgTw#bdAS(xDPTsFUM%S7X~s^exsSa;bvWVSaE zNnR;$;fG57$AaVv&&j$q;-qu~>MxDDdZeIt89~lhBW7T0gtz2YP&R(FJ1z|_i`AFM zid#iVmll4y6{*N^ePinTe(X;lq1Dvhc<*%; z>FW(jI#IPU)Eq{`+oev+6t;#_Jf>55x(0(&~?Tj`nt=HJfIRSAn zy0!(AwuVeU{hs*xOj1gd;l=gz#?3|B_2xn>*=?UO|EgQR$)!tD{_F zdwLzUJv{g%r5M*r#f;eZ10LAz#%5#J&CmeUX(i>w`pLqrMd&plM9$D*#OK#U;;U)D z9c!^yBPzv1rSTN8*}oC8FK@14#JV_4LrZ6eeU5GWiB(rG9A-K8EQs3I)HrxwB;VP~ z4hv2|wd$Gi$V!@Ea1bwj)EyA2(Ac)WLZLo<7+z`O36k=h0C{9t2mY;}3I~O}CDcoZjY7+` zxBexFKZ}dAQ05T7dof)SEo<)MQ_@u`Q?jVl@;z%`$eJe!9XKYiyh%o#y~|stZ#HtE z0VQ;p8Kz;kkBgugKmA_CSHef|yKb7I)qg*cDAeoMVk}Wdcj5-vq&f`vY`PwG>&H8L z5{*xPnag7`Wr_=uPdI7##Yse*PStjIu5~Ma17qQv<$X|5(ChOK6FF4g-(~g9Y^$f< z53SO0Lw<&skMmzu(yXGrzYv~|bA>Q;5Y;QcI)A7-VS#@2;B1>t#`MHI>Qhxxt+APd z!l%ZO+YKIL^cC!jnpEcnx~m#HjR%4&h2jUMl%DV&=Iq`YlSk3MT}Z{|+#Io1}5ZwS41c*-<+=5wse;_K80inxP z5o*Ihj2O$daPLXSj1_F@a}fG`+BFk8q+F1I9PP>GDBGTm8%)~YugU=Wvv>dJ4A%V& zC!L))Bi@4CLNearyJ<>v*Tr)-_;V1Pwcf`@Cy3EI)i3L`xu@p)#kKxqK; zX5bwWuh80MKU)*VR^SEEln66%nh99xnpsy5lvo-ii$%gIaR+G^hk~8 z;c)2fy;_kr8_t6OsRJTnL6@ryNFI@BuyOeEMO`{U;maN}zQ5ccR}b*x{c!7@(=##( zk3y3;`hpoJ!Zp zCdAAl^E}kxx>(#~XQ~?F0>633T1kXCuJsCBg;1ml2+PR4sm0TBrGldxDLw82SG67_ zHOE+$jfGBH=A@<~IeE=k3egPEHwg~F9ex5!pfP|`G(Dwbo87)n%T3o zVOW*>{Ud%UT}?$J!0&8M;N`3RP`r2e`!uYCvi3ygsq~_Ek-I`FsttAw*X&ysJOMf zVqkowS?%0C%0-t7-g5>L+Ge%>H5pO8)Kb3*8yC;rJAyJZ>)v7JUtWJzhsQ_HH^8}e zC;R7&=DR`Bkl!`FTGY%%95C*(-1D+6UX*c8-mj?GfRWRP;_kTcDpoJbsBCUyDuv`w zdF8x|9IwO3!|fM_yQYpK3J-p-6AackrK0yd)RIElUK(4VqA`_MDgAEMQbN<)Mqh?M zUJ2awd%*pg>4faVHZiL?e)5mUjH5F}zHjJCk zO$INTxVU>w{g{uq=un=SntKg|@VzSc59wm2Q6!lifAqs zmrWTgey`jqr8whI1$kSVwO=v3fb6*f5rp!?Ba6AU{xmd#6B7*0BlU7yQ~@xfKU?lw zHeU^&=3~HNsed>RZ{(q}lt9(4*Mx;FT#92?MQDlNYK!CDfnWGj%yM$H%X@1#<@~%Z zJbY++NwYN@Y~pk!CFqOx^VxE4({xI^rxc@_mPRVPCL{iYF!G^5coDpPfC~Dn!Uf`3 z;M$C4p&M*X>snj+opH(?XIzyf+&^^C{Qg#jF+OF6B-W%F z};Cfxp1yXF*~AYR#u?5vgZDN^JX{rlnsW^4sWSqQKOv0Z^}AGtNU z`t5wi7`X-@k^sUd>OUjlN{=2DIZ_op0q(ffsFFuBglzsR!yz7YrU=gea1L>so|gNP z$k1!fl~t1So8@@~a0Wsr5ig)$jZeeApa7)jptIYaK~utDs>XOK{*ITIoT}PDP%Rki zRPh{eKZm|vSJDwbbt(X;zc;0y8pJ{T*Ugt(NF+{ry7g64voo*1wnt)ei!*Fo4#u-d z-iv=nYnc9FzUZj|(G9+rbLj@b8R@oLKqqqOHt?#pow46RgMjg z?K&$j&&9{bms1Jics(j(%Wne%@pt&Yzh#p2LBfl`LZCkRvM<+&h9;y|WLfxW^!QT? zPF_V11^~ud_rSGf1#3_jcrtI^7uM`pqf$v>4*RyZ-~DNHVfvR>%JQz}=H}v}RlvYO zz6c1D>K$5%a)&_#7g$rXArgB0=?eznf`hy-r}FLFw>PCKA%4xBucwfNaxPDq<7gN zi91l&6Okl~78w|kZr3yW2LJ{0EpM3)Ef*J(OALu>cVVc~(5-`Zlokt6;*Gx%4C`%k zs6)zublV~m85!xdP}scOAV6k;TY25&8r&$H!I!`X068A!>%&O=mU%r@Wv%A_d#MDC z?YcJ@ztPqfx)EffTsHY|G!8mJIY=a5<+Ef zLLy}EY}tFytYnr@*`Z-(XI*=gtn3{!FWK`N|L6UFKIebFzyCS^bAFvvr*z$Wz2DE* z^Z9r_h7eCz60YvxV-89d@#*+adFUl{@ZSx}Z}T^8Q_szvwqEYcI(_cuvsL6Nok81XhP5t( zOlF6;Llg1>BA7RQk(LO7+HksZoqd0H@%`X4_s^a8|MU^0tY79HDMjvdUT8U@$u`re zX7Rnk6VyxHLRjqQYrue4boS&_g3XcOf+_Osi1;13@hLA68JWn|l;_P5@{qW=Sp#dH zo_ZITRaoZc;2_r44iA5av{@!CXT*uuguwxgkkoi1Vif3Kqd78jElfH@%$IOS*vort z9&=_JO?SvY|M4SvYvof{iH+vxe(x{v?Ey?~_He5La}uF;c6$RYS;S6fd*lZO@}1F@ zvt|3EX3VF;0)N*u=5v#g1IaT`MN`uX1rBy*&XQjI{MmijcE^&1fB1?tv+vp0KGkbr zj8f)=-q-q>ZZ8;TYrTw-8Em*t0R(gJ)T}V6>3FY0&3MM7v`{X)#kDt0&bxYYxw_}^ zydSH*{o2NZiW-#pD(ti2sWY_YJPhB!3J!{pXC!J$yq%)6RD5*Q4Y*B;{~1v+(g!Oh z#&CxXyiW+{&|8MnjF_u(e7{~7DXMV(tw1TZrp6uWg$JT>31L(A z!pJo-Yp3P<=-AxCBsN{syld9b?zEYadS6c9+WqNmcbp)Zs>i!Y!UNYn+qa1fT1eV! z($ka=SGJ6`28F$|y&fOH#3(u9>jK?H^ISCEGjlrb#t=Fz7%ccSVaO%D8J*)=yo7D;Aa-X%~Hjf{ZH zya@m&zzT8C_@0K1jr|+;Dyud!Y{rnu+aAY=2?+b_e8p55s3VwY7&SGrBI5>!H|5us z-*<8Y101?vAoDgtDpCyj+}=$ffrF*^@eKul%sS#2J0QrQaL)>J(3&O1k)S`)LgYG7 z%bD2OJNdE5OfA4dA6#g|weoTM64#bNFs)NJT7y&K3mWTi@Cxy-h6Wpn%%1;vTRQyt z7gRXVY~0pENiUaImZIfhTW5&Fc0m&GI9Rk!TE77b6sWDfefx^`pbnQ(CPFV#;o;#O z(~+6p6uWX4{M)h$yN?AHVs8M}RxlF>W}csmiav(cnXB>=h5%BgWe8U4vMgl&OEK86 z!gfrMfu5et?&OO3&I0p(DoR1Tk71fl?ND#m%XZbY<5 zsj8BuHM}yxJQb_n^!rEk?ekka;93IqBvOT11LC5WKnYKk_6hPzLZ&c|9%{W#>^Z5R-V?I?TVB0xj@(8x6>`qw=%lp!v)UoaDeO{ zRlWuXb1ixFUeEpn9nN+bRmos!!4M~~*wIQZEmws%9|xPr=yc%AODZW5`s0!P z7uMFvR2|r$b}lG7`F##mlAdmWf5E`t)GJK-S@~LbN_cfrbMx`Y3?$uks;L`I+qjJV zb-m={^@7W&pCdMUu!NyiZRo}K#or2D8rg+rQFzguG0%g|Z^=UQ{2ZQtB z8AI3r5HoN#@1fi~yKLpu6Ud<6H_Dp=iVK^*N80w+ zJYbO#Eh>73UIDX=`XceTw!RXQFidmWzU60d%hDR*R$;q;?7JUoP=R41&_E$^|#$EC9h>0#;w$MSIAUysWgcs^(bLx2l=Nx&; zPeM&y0izCCcei5a0>`pjeTMIM^Or^$wD_Xyr z&=GI;%Rgc!-A!lf9w(;fkaR6GNwFW}`(j*thkT2fT$jJMzpPgdvr2*Gs+#e^u)a1# z-0F4V{GA<8aV+~WQFY)LPzeG(qOdSNMC0~?lum=VOJm8~xmlAjx;N?RtdmSDsFeTh_ z-ZI67To~KVpKCv4SQ9TXC)4)S=CC3KT21HunZ&2BHJsWT6bFWAaI~h2#j)_Yu-%(k zs_{QRrS;nBe#hEi_BfTjb;NiE2jS)WXfwIt#%|5ui2L593o|4MY-KxNr)ozosHhO+ zJNy9q>k}73@e>|eMBjK`^U2W+X3Pf-s6uFXjb!dGmfQKn=SM16F>iE`#l-ZR&+H_V z4Gn3UpN3YQP+DT*=O^Hh(D_`iPzGozi#?|_@^MA&uWw3SJ@5Vx9QT2m4}&7Xor?&X zzO!>honnfcb6iNz!-Nd+&49R-7`wddO?~Cc^1i`h3A$6%BT$mpqmqV=qvB>{{^)ik z_gZC|3uUT*^u&>}Q^<_0fX1RnjB2M~YTQ>r539__`DXSlpWA}3dj1wCL^?f9dlVh1 zlKnv$_oKd5s*FTsu4HS30!!aUS=#pWK|S6@8WEp&_nzx4qkCx8ABNjYebJbk@|>w~ zr!xBmlaW3B^C6fX<{?&S;{fCRlo4>e!Xzu(xd{uQ2=E1uFj9H=kbu4lUQ)c23$PaS zBP2UBe|-Ng_4skDvJo1bM!$h8va(T1eE;*JGyu<_l?MqEe1T9x5yF<=AW^RRyxyNJ zk0(SU-Cl%|w!*rjlGseA|DUvBJNk>%A&QS4eIapp^Wki$B>(Hz_-S9$5`c7ODv5&$ zuMd9Wt-M8X>G+H(I~?je{BL{u_7{-7sMK*m`h_l-Ayf6a2slo+0->Rhj)>Vj$BgPC zW8g_iVGC2vOt~HxW@EBL!iO8L_6&kN;4w+1?f!z9`!}zx4@cqVIyMdtAALLUsTVM5 z)ev?!E*#DKAWM}A98UlVt0WGC&wm+k`=8nOpNic1n}f3)rx6!%Pe6MQ6~GwW$YA5H z?GTSiBAW6X8*#mTdmYP7?0iW3=x{av1glNQc*c^iXwrZg(R5&+2d)4WLz@D|)uwcw zhV*zr^~jSD#+=`H5(?Cke;T_}4j?XJ%)cE3LdqbC^l2EL=*NOg4M5)*Nh(WnS6AQr zOlh^X86Z54w|l_aYtB?Wu;*$0NMRw}+PH;yaePAqF>3hT59S;F5{eJ2gNC(sPNyj; zO%aCG;g>G&z@t}nbGz>4AtbfWJ|-m%8v&)pqp+~P3Uwr&Dqhbh;Ha0>!G5QAlLqjh zs+j%p?w9!HE zs`W%%yw866bi-*EQ>Cq9>(2ggzo{uTHtrXGakL-qz4(|o0;l(>6{okwrH_S#Z9UpO zA4;T&p};=6;~znNw~p8OgKu$Ha#H2*2no@+D{L>1PscBX?VCsoY!?SkHCEUTfj%H% zYU>zvx1i<>*iCeL$eB1}pIaBf%;~*ubB*{>sQ=#z`-wkN{`Dh9DY1d)B9|#C4LeF) zmwxn(eF;xXvzQ5+s;J7}+#3n6zH@q-tAi9Bom;qj*LD*|4s68Q`Vr{kN#FEN3mv)K2y@<7j)=dVCkw# z=!@;;xGgeRUzt|TFMTd3P))ebhl@I@D_1NZrWO{nDk7*7jGdJpY6`f6YZ`EeOBsa} zOQdhHZD5+(b41Yn-v#G?PxSw(>Hmvg%)*A_`S4H9avibaMm(RhhJEP$#Pv~&#Quyd ztl6j7>fzdCfMt`tBc8{4sfGpbnKA?@{*mf8TnKA$YA)^*sogaeW~7AVsVz^laZw(||U^b4%*WKYyPnK{or7)NGiQg3bfI?u_z>URb9mXXmuK7KsQjaJ=r6%NYP2_7xc zBMvV6CdWWu&~pHF3fhvUrGp*wQWEsR((L9e8i8PBBmg?Ax0Zg@&6iOY3W^*9$xA(Z zGP0qlOyoz^&K9${n2}`4OGKrmt9a!_{^;%gaT~!TG4$Gv(*Vm@7?Yt5${O-mzY};U zBl{nclvs}$1BD}Sz%#W((8rC4LJYYB;Bxf1 zi15qk)vKITBUYUC6}8UU!=Jm00}@|uZx@*zzocyPI$lwT8WvOaI6VxA-#rYlM=khj z+?WGGTgBnj#@tiKnFG(Z=eYAADU}lxDyhocY|4lIsc%*@z;Pxn!CSQ#!a z9es&Akyrk?wB_=psN{L`B5#Dpl3LW_UdOB99Wi2Blo=ZylH;Uop8kG_y`_+&Wz#Ki zoSrBDDZLYU5cPw%E5vy_VK7(Ju9Kp!mE(@L7p`{FUb)z4orJ&Pt8W zfA9>%yXH2N&T{u!UnJ4qP#N$V{uh_?|L0RWsaVOWOF3^Po_6h+6xB;2*b@!o+DtEc zc!1$dLZ-YPBOYFwT3T+-6V!S4CzU8w zI^YF+JrxNbY?-EE)>Rt49+6V2PE@ix8=Ek7?tzTIEU)rCXr%4hHUmde){Ue~Tb;Id zzaB2=P#AoXp53!~V>yx~+&~@O;}qThj2a*X$p?>djK-_6P4V{k{~}LAJ^Tc}jS&g5 za?f~FcZGUjv;W~h1m%?l9#JV570btpYD#RcfAL-9F3-j}zzKNeprEiRqo|0B01Ge{ z9zvYh$oZX}-V8ZS%@kyv=>wkhK=DgZ_#xD==?|ZNReCGz6|~~e6(g_v&-|x*l8s)< z8Kf2u4HrInwl&Xa+=l^;uu{VhYZnJpTpQg)?vEk;x8v&^fWwt1lF7p~${pzuZe5zgIeYR&@x>h{j5snD7%b zBIC--6$c01fSd*1msl94^v+`Bi)c3-#GB-)ZBd46{~@_8%d0#6mc)Af z`iH6!xBR0hFazCx%hN=J8j)w>b*2L#0(o}KV0&~+>*u3d7YlZ59C|gq@knJ)=7~FSi57 zfUJal=|&ce!s- z1St-A`uh4_RJ71e`OHql;Hz~1D^rBdfju^&D>~3?@H{hNV&x*dfN=Yf_${!IrmBbHI-I7$_9Fqt z&~9B5?`gkY-2Iem#vhHVyw&V_ZnRAr*wjM1%wOe6F2%fY%wSg-cxYzVOXq$1-EegH zx@XqttL=5M##uy3NxH(rF9>L%Z}$!61f`_V=TAZ3^v&(d6JyBYvziS&7id~Lw&2Tk z$2l`nd!=-KxZE$DDyBbfh5pEZ?t;-Rs8=9WaJsG<_t%E2o`>6KHFdMYfYZI=JF_DV z$ib!K+qWll;$&nb`jgonyS4OZCo$qx=+TR#IpvK#T*zm?vt8#jIx`s^+tOW#jYYa< zzki)P5?X$*1zBpX#`OGoDyI<=?W?cGEN}j&M|{H5!$xNTJZ&h?cO zWO6|LxwcVc+wJ1k{jnl+N)3qs0`aKMtCK6U7k!@Er@@SeC?Aa7491!o+?{86 zW}lT+*cVG8uYwkz=+(Ox3^&p7$gSQl9p7Lk7W7~gRvw{rN)L*<6&FU#`$)nm1S5yLRP(1B#Aj1LRv+-hm7>D$o=% z4vyR~zXxPG5@5LRSsW9RnZCog%!BL4@isgbgbX>fyuhb6g19~m@1C6^oOTOb@;Cn^ z7z(`IKOq#`?z|3B(G1&y1l0BbAa1T#&6KW!x`;7m87XCpI+_^j=^>Ul?vHDn-#XTL z8Gi;LKrLXl3?YBJ?yieJrKF!9rL27Kbb+j5*r+KPUyYPK{H_v!T~tk--}&O#{*N-@SWxyq)tfVe@c4Zi0t}PXmJHoM)SHwWdJs`PHQ5 z;&D1v_KzPwHixv0SGqhrJTMPy3a^G2+_11=ZV%JyO4ncqxEeNoWojDN8C!U~meM>k z`>9xK#(~@qp^a88pYfSnV9Q4U#ogBbUCS(hD@Gxz>=!^r;IU2vA=EKRJ6l^#S5!o$ z4&+Itb$l4q5t#JHFvv^;p+wFjUtbZ7o)%m%vzNR?88BuiU(`SSx<9Ixmz|Bn=(mYA zXwxEF2TD;u;49XjvxLSo=V;}_*z_1Ypcf@)x4iztSOKuMr4t_knZ59R1`E^PKRcof zBp^1BhQQiT=~+^FV8_@&WDH1x9CgfO;cG!w2tII&Xyy3%V9$!#AA^bnG{ox_Q}Dj= zKr}BVJJZAsE*{Z;7n+cZU^e7hZGdsRhNr~;GWx$!f_BzwQ6jL0z(Mp){%j!$JiM!r zyV!x3lcVwbPM}cEo!&H&YmP{vsG-Hxp=?3_8Vbbk73y$E2>*N9ey|Z3v9?x(>IUsF zU|bbfxvE^3tN_y6zC}!~QVvWFCFo z5hs4kUTOF4smHsWB@*CilOuRQOaeJ%@7@EW3JU<~C=Dmmz|7nZEgwmdD;U!7Qj-Zz%#%7hSnBHutcSMq))$cQ<;lV*3h2Gu5+VcAz z;=8EvhY#M?3F0SQ!^>Lp5EJ)axi>h|5HS3OMRa?dP|G)tiH*Xj_##i~er|f5Oxj+? z&=1f#>ZL6GvFP<7K_@sCe+-%$4%uE`aYnq5T@yMi*Gp5Y1qR4o96fy)M$n;Oeg;9e zJ24DTEG+aJl5cTY#67#aCm^A!!;TuMxlcoHOO1!OCD;$QQ2Pr!bZl)x8#R%n)K zm3C*uQO&^eP*uRAzrJnw=2Nw$+&}qD6ixIF+k%|N=EPJrAjx~D+lA1W$FVP{>I$e zQrrS|8^_17;(DId{uA395vg<@6EE8i?b;DbsG^v+;=L$vmrX<}5n4sB~gawfqZ{2aiPK z`ro!oTHcf9aYlON`d2oOmZ$66=C`6*UohIRyz2Z|EHNAkT6C~976db7bMug~Ju(+#^ zXVTN70Zw}K_?ICYSQGt*!Ur2KK9G;kuB?D=NKz?Hm{P!DvabrF^SHrl0OT0ccubhe zkmH!Zs@x3ZhTVnsb~#9F?ElkbuJKO9lNjEB5;IS|kY#~YQpAwFnvTwkni^rq2>Fm3 z=AWGjls^A({F3MC1SjC=v@$M30xXp_V0!E=^z=N2q~ZQQ>#cc^0BZLp#}e4;yM_Xn z!otH#e*hx`sFJYH19l!^_?tHr!&dL1mcNo`2-dM18wzdG|24t5k!ExLOm}}E)AH59D(DXEMj9g3J z&gN#O88)+QacSvm&HcT-Fb$3{&=^9&kAL;bq+Cf>_5xTK-~~UmW~BnA2;J01JvIU& z^&m#8?})DC3!D;k;(k>2_U#GhK9CLt0d$+YyX8{m=AXfaTCz|J)dgxk4EJ#60dNm7 zAYZCYS>5l6fdm@bi(xBUCecDKc#iW;wi4>pnrTy0#TU%26!>gxU21T>=vs;|L zj)#6N$kCDgit;qoPNeqY$(hk>I+pJzaT2r_2c}&_SlesP z_Rv#*R5{?Ub$CQ@VXeI9l>YW@@{$eZtCzSY@X#&4dS{+8n0=$0cV1W|^kAqA{=NMT z|K;AV4o$E6iv%gzb~FjObu9+WNtd$XJuSWgN${nviYNr6%v({3Kl;IcKkx7T=6`BD z&03B$RCTip`Y2&ey^kjz^_pftB@U*xZFctFFa!g-tSf1Silag_;*9fFUh}KSBIKVM zY3lL!!n41Hw-@v9->=3@{&7LTauEBbZUsCIn$6>HqTDBJHCcypTtP*XVZ!t+$H>+0 zdJP$*7;!Sr`NPT?#)!+rL35@plbNCncpycPTj#h<)Pd?$(0(E7m(4FWG*~|0lXkIe z9e?6e#^To>6P3ygr6WkUNdGujr?2V_N!s}pqxBO`D%mjJfYtuGS>&tv@d`*a4CnoL zI>iq(^<0zMEw5DZnS%~SlBS~_3q*}e`*e+F+!yt3Mq|h60<*-9I8O$huUcH-Oded34Cd{qjA+zqd3IDVt zrg!Q3F1VkP@pdqJJPwaV4)(fc4y+#QU|Sb%NG|rHEl21p*hvinM$lRD`A+k1k1UR5 ztR1Q1v#iy1-KYG&+8;f$F-`992mgnKgG@V6+1KWl5XGcRP55y-QoWI35UmS!SSdk0 z2FT1mc@4Fq{`>%QBX%93 z(K0M1BpMLjQU=64LSkS;w{~=tt!{5`N0YPZrX*kU{cUSscw`$MN*a>ha87aQ0fucy z0`N}mlka)it-GOADMGHSYwKeINN6&t(~6y|4ZOU(2raBbqZJ8Tm?R`JH02Xs)4GCW z=$rGJ^L6mzt0V}IR$g=RL(U482=1 zT9$M8u8_&2M^P)`=7Xy26r&A+4gFLpKyhvaGMbzknY6+=#Ier8i|1Lr`yP%Vtpc5< z_oT0PjeB>nV2J@ViEL|V%GG*pF9!z)tA5eIG`!J8B3O+OQ@S~*u0lN}14tkwmiAW8%Ai%sQ9Ea3Hh06koLbLfwV6&} zu>cdoEc>$@L%2yiY#u=<(SR|Bf%2m$f*eK-B8@(NVo;Q zGKe zEOo!0IX5LL4(fj`CdM=1#)Cd|r7LV@=yRvN*Jt-*xtb=fMxF-g@$^Y?Jra88Mo;Ir zC-y8S@`Z=K`4^e1&La?o;r>W6@d*;sFQk$q2U836o`no8sxS1c4-2p*^C*#5uGcP1 z*cPH1>jz!Ry2|gnvn)NIMRsT2N+%V;XB1BAmmVu>8Y>H5ziaW5?VG{8nd^bU(@S9% zOGf9%BSgEerPs17EbMVx>P)s8Bu;|GjB39f;tuZs(`W{(r)MspFvVnA5_HSE|Qc7px<|FLFQX|av2~6;* zhZ}4(*-uV=GeS&(SMS!KygT98y(H4VLbW#I_%IFwfd|6T&3xnB)h=;y@&f&94vFp(s!x7B|Ebg> z`cr1_9V-;g<0xAc;@`8n1t%_me)qC*XUoru@TixtWCX0eJt15?Q|;MSx8swn)qCla zIc(Yzw@i~@C82^$2_9$BZJ2`%G)=`yEd4cOyi+ra6q?fa`vHY$ZFPx(b6<47|Eb>o z;mofrwY}-toK=mo%IhkPZErd_YkBC(M5$OFj$|@g)Y72Oy&j&YMit{U&-wZ44=Nnk zYv{9=c~v*va}Gezi4hWD^#-OA=~isz1Wcu!v#0Ex_2^xby7GkBOj?}8!NH-R23kx^ zJ~$@mytsG)%|l%cuq(t4+uw8cS5&+N^jEtpZz%CEU}MZpiHV5|jv-hJg!uHr=(snw$yNc4w9MDLf!_7h%Bmyp(c{MqgCo29mLwkw3JdQmo;<`~NmICC znwY5AhXMCBsNnq3FQP4gqKDDGIPa}$W-oyY82}M4pi3VZ4d#Ua z;ztE$SK#fh^WL$eA_E>ieT5z-ujA_~Ko-D>d(eQg%Yhu*4F7#X)y`R=_XGG=on_52~I(|0aix}>w`C(a1NdXBT)f1~22GAv|_caYXpj=pFQ zoN2G?xg-0ExffIT(|}eyzE%?wk_N5|CJb*u35Q!LjDY#d;Yr%!*eC+$N%l-$-Da`@8NDy@b!KMzKBG_a|0Rh@};Q~Zrm)VgIuudT3JTzwfUnpslAw!lVa^4eWpl2$#jf zWw&sPN{6(;jgiwo3Hk-wlJksqX=$hEg0-=PC`Tmk9|~uWt0qGnsOYVgWmU8J_AAz> zXJ|PVvfqs7Z)cYH&d#2Q?!94N`VNxMD-SGt!p=@NQrOp$MD_rYe`>|H>>td$i!wHr zRadtv?zOEDJ~`~T*5{ubFpGHA7gp$Co+%q178 zq?1>ZBg*;f@zs?tW7DZk<)hTYTjiQ2r|%6!o|Z^3{taoBe)&wuO9vZRz<%0MjFdS^0FA=Fvc>Ow>oP!+R)>an`C)h<-ckG2Y8(S;I zD}_*SP5!zH4;xonS}NtM3>}}AZgzhru&9!UL*vt1P51A+hDrB7dL-Qm)(*}M8oj;b z%sE{^g=S7v)OGj#J86>Wo8C(%eQ|MGY+rcvH9LbYci6O2C-%M{jwHirhc0~F#;h>{6H5A z&xPzM>x%G5dm%bvP<;e}L8O)lLtF#^9@tx3Tc`qy3Ms*B}fE;9&bRt?Wt{Dkl4wE+9^;3IA z*p<8?Z#AQ0F05N4oQiSwi0!-0&m(Et1a^EdD>Bp7)bTy4)mdwlA0@ONQ$@Qb>j z=U=)f>gvv5Mh+K;^jLp*Fx5{{JDj$8*OI@Uu8pT>Ub^nZWFnQRlFvNS#w=jd=5%aK zeTe=Mq80P~NqSSy+k-W8|LP)z@N)-|ux$~ddhTp38>byvX5!Ngi-cYkf6)tBWqt@Z zeOtyhFA@9ranG|)cG~m9?@ax6n{OFR`+6b*l{ZYasE*Cu^)4Ep3**vrwP$$7 zC`Kim*TsU_K*&v*7e$OeIz()Vnb-3TI%$u47F=BZs`O}z=_!LpEonU>NHkD5;(_``aTb<{IZJvyi zq2sN*C&9|<68@~55KvLewDa*K>^4WD)+|%#`t6ypBbAUG|4qHAb6?o;lY$zbuw|=uXW_-o{y_zZ1;dgP_l_Tk0P3pz{4tU%xT$ zU};HY*?V>ggx%tlrIj9!5B>eA)nt)8#06w>*_ zS?Wd7{?3>4wrSMV`v_f+p3sS{kuhvU)IBmxxG*WK9-X>#(`m~&sIO=9ZolIPk>S&T z$By|cRY$!la@#!<-#kmOHwwgGe18-&OTS7OzOpLV;9hy*QgxPVi@>kMAt~!fT@(w5 zjWRSl7MywfHLV+l??Pwwi{wwR2}1MJ#f&&{k*s=&eu%7D-o4+P|rH9vGIG@FYt9Y*BZ-DQV3{RjTvw zJkpDf4q>EMK5XJ?mGRD9W=(`LSl54&7|mm!4- z*WXZIpB5~ALX}haV@AQF`s4(IlC8}&7#m~!5H@Dw?7{LP^^HB?Fc{~y_bDlR?$F=M zfbFp?%PTcF;je?+t|%;`!OmFL$43~BL}k{@4e=(izQmv#aQwgmWLY%aC(03U=1 zNv0sZ!r_sHr6sc;Z*K8Nza&Lra4$@D|c=%T)@U6Cnv{9FZ~Sf8x18;iB>sG{Tc!Ejv2x$TlZ65&t^e%H4)L;!~1h{;T=TZ%r|$H-eZn$Sb7L| zP|reU1OzYt%-h-9%S9dK%&e`gMIVQQA2ay-*k7ZdrT}ka2r=f&2Sx$#qYJZh5<}z3 zW~Io>kA3zpC{9mLkA5}5xqw~c*a(RYqY0D0ys?;bYX2tei8Fyyo<)gHG&m@!vlvS& zWB@XyW@H2)SWHZXGLQMsRWPUkmV3-GFIG^;yvg_1H{Zg0wAUNM4BbzG?+^@Yi`^+! zuuo?;Uw#9W`yYw73xr|a{4-r_LXIp(8OZmbl~Q>4aK-(n9~j#t-7zLsQR!uljU_d` zOifJ_)hDMGLx>q!6cxc3H=x)alxmokMpjPlm!pILGiv+;***A*~=ZBm@+wrh_A% z(pk%jZcx`Z7n0+iPE_8twpo3j5Pqt0+j-Wgop?+BQFy^@k1tBgwr{aFoAWw z87kM)c0$J&ZGIZsRr8b8Pj0Yz8fF|EB+qSibfA)XmyVXQH=~Smuwi1PAtm1?Xpc!~ zZhH{@cPdj?cdc4W9!AK?)BP%EIGl@qgU)4sZH!#>)a^oZi(!_kwbQ8&5iwn-ynKY* z%-JjX=Udr}u`P z+d<01Lkw(&!WD<^b3=&h)BF>A7{qNV^v;Ws$jCj&Gv%eDr_I#L&wBDj=Un2@D(|ex zEbXz??t7PN`k*4S;i=jK$f37SDEV$KjW9fT$C1b}I8vpz;0P<2!NrNUJa4^68RR;Z z7)6hdmFUhV?(-0;YTsyy=uHb%3BBE}auA-MLBSG<$a_pAY<_9*>JL?L-79@xRDE

    |f8po~ZxH1QvHj z$WiZu`R0zr27&h+e+17C+)@ua+z*xATE0E}>$I2Pu^vHlv0Kxg{hMCok4d>xKCqlf zI(wS;eUD)Mg4g24<*ojG^>_g{>FmL|R*{icCo5d={6gSOS)+gOln~D$j@~>(h z)WimY=WB|=(#aC*{Mir(#mr}1#G9$S4vAs<=06XlHq(AzLzzH{{`ujL!hC4WZbspIIn({lX1mkDbrsx%X0 zXE%zu2}kYb>FM1%Y|O1v!b0`wF(nS7$f!`x>+o@bF=OysDwWM6gEz@lbHG0}Hy>P0 zCb;a{j`XjOyLcrGB@V3zB~*DtrwViM+2XKxRjP(cFsy7#bN(GMw}2A;KqPN z5@YUoRuzwk+FG$e8!qtoh4^ftB7-l_izb#fNdn)-0L`sp*%3IX)DTNU0i;=M1jW@K zq*r!?V3F3ADijc}81^~O255a@-va6(rznDvTh8lS{)6JXqXGM6&qhnGUmhvaG@ENR zgdMCCgs|fV2SwF-rht*#fvTasU=kiQgmHmCH(b+%goqA@in*Jcq=IlSWAgw+?&nY6 zxa6v)JrZm?lIOf{M<6Uh)IgiWb-8&dqnL-bSm`4eZW^}1#{&?M!%Z)3o^#v4pk_a< zlAjV2t#jl;3Aeo^7iwuK2$a`u8i6EGeW}M_{qoO3PP!eN*|3p@6jX_$XS1LgozNIe zrV<_m`Sa%*BGa#)S7%qw-$nH4B99-dAxIzjWE=|dv|o`pTvPMgI>x>C#-p${_ai-Z z#AVKxF#}zatO%Xcd_ljHU)1+B-H0~(pHYD@M%TXe$viQRw+vqX8tsau8YhBOcmQwz zF-z)+)x~iOMT66QqhkAd>C~;sz$9s=)tO$c83h5j$IRS&!r^8~_>^OeqQ15e+*YjA zR88Wribvzt`DQu3gso7`ozU)gOAE%Z3{>5(yw}7LlKI2TO9r=GdJ^B;-ATGsvcPzm zj7sLpDxs$|Hy-FVDs|P~e$xuQ(@Vo$<+s0HOSEu=QB1~pA`_TgxI8|RnN6RWGbdo;6Se!&N z0KA!%mG#Z{pa}UFgl2F0-b4#y2P5FCL$*^nvM49#0^rGDM9~wYF6Qm)TNbb}3`Tmt zavp|k>xD-1oARRNaKfZbvH@=u@(FkhHpn;Mv*wVfe^io!mZErsp}xL8LXbsTE(5^% z>^`-T4pwl-6V=sQIpIK!4vtHIC{S~5&7DFmy0Hhvq=3)ve~V2x-# z1Rjo}r@I4g>EjrqFM#P`-!4*L3At%aCr3MEl#~x<+(SvX>w(Y$Tv9v?mg@t38Z3fa zFbt3X+41mNjNfxil{mk_Z!a8KSI6q_4`Y?ReYhW=gM-It*%BGJG2!Pv$uIVFyVujT z)3^q(vyb}~tzz-$lM_pXq<1x4Mu5-6hfy!li5T0%{-r-uA$l==Ed2wZ9V>~*cQqYV}S9zUc93F1mOq?J5 zV(M=QkG-(aZf5RF^L=B_X@AmH&DvIdmhrq(7XBsh+*hvHeyNu+!XK^mo}T2-qGiNw zZT{JJR*!!lZb23epcZGRN23iAGj>W`TzruDE}8oTr0g>A6HuP%dK;};fAs#!#I9XF zv>N@^I_oJZLI^@{EdU$TjCzn6{u+Is5xW7CYh6zlO?;`gP;8Cg2N=+2UIafj8U*ro z-m6Zni{y^HwG$kE-mzqg@{Mv@4&(j{@VdB}L&mi7Q4e`4J`f4IGOnf*M#f1U4U>0?S}l*X$u zo$#*Fw9T=A(fuYqzau&F@s!#{7kk?$O!F@+GKSKD<&o|mNC3uc1V_oW$Fopi%^7No zdm&RHAz3UP-TUMlUUZ$1Os&iz9kkxRj5s``+~~6%jx8!~LkVf;P%9tW7fO zNv&t+g!st!DU?7i7|4hNCJ3Ey`6b{^T11&L@d&E`ZM0Mq^qN^so->JawHrSfJ+EIk zguIXJx55>H2mZhuGOTskwTP4r+C#~=8@vU7j^mv@B!E^YCrfuB48-;L{&Zbl9+B_< zr(pjh=}^{OTIR?>hq%|(twtY+R}pe zqRvzu((cn+;+!qtmUf0$)CIgy2!P`Ood>~+1&_4j-a z303VX4Tdp4vvDz<<6yvy?tnHL90;Kq^vg(l)FV#pwTDXV6!l^BPUcVru>tUVK2wex zC1V9ON7G=mFFGer>@wDU-`74Puahx!+MXgzpkIL|5PYVCZx$;BXJ%Y;WPH2t#m?bM zySMPpwgs;|J#4}ZV1=mY>$vO!EaD>D;9 z@NRw1%cJ=b|D6da@j6g>Fmh=JtqZ-890KpatBV}K-=Hf*Z98mV|6t(gfX`O z01I{y9X5=B9WoeOLQmEQ4%v7%VbUaPXp0)5%?=i6Wm{fvu?$lbl3WEZ{N zb%)n|RRicNE#2LvSPB?mW`&%ZI}V zV))Go3)Yk;nM%WJ3kz*EXP6b>-UyR<`RY53Ct0ea?BGigo1Sq+u}4v1=Fm?Zp0e}q zvRXo~`Y&E-1|uUQhJ4Of3coit9J8(E!XYOU>~Kq7Kew{5X!`dID<~==e(+%JAuYIX zu|FH4BBP)PU9)SEB>?UY!-r<}hc+0`i>lXHt{xtiH-+DPH?O=Lt^=+J zf*xyB?^DLE#K7xQpYxRNd;x5DOrB!)%-ttnD8OMcT$ZIo%{*JX*A+|AMG@ay5|$1h zK71Hi^7r$@1O^ET3KE)VfWNT9rvXeTQb7&)e$h_pKj?hgXH)UbX%?H?m|x6$6BsdH z&A*f5UwFIXHYe zdj-V4*GMu+N=su~_NEhwXM9wdWCKy+`F>PxYTkT#d*P7?yCQfR_vZ zxC4no?vop=tSsPmSR_L#tsnrNT(D=qqpGgnk_XiI4!{m(*Vpa;Cy6k07jvGOTRAp? zoa}IaLUgivAgEmqcXF~7pV*@r+I>v95EWf(;vsyWLbUT*NF?gm-~jsc&hOq=G4Alj z=QVI8DJ?~KKb?~mD9|B`2F&xtsE$hhi9b$-r83oG_fCh%hWXQxB0AWw9*k=-{We`x zVk|JAey*ItN~kEhzmE-eq%jbkLk_kG(()1QpJQZMz)JYRZ^yt#p+I|!WvF*FRsKkD zK{QpcTYqnPfoqmSG=1 zW=HC0=@x#P#pzMw|Do&6Gga3(W!GzhP~hSeXVP)b*<~pIcL|ww;a2se>io> z>(Qy5;n}a7Ey_1Ly?ILeyo#P$;Rh!jH)%RX+J4hq>R}V}B%h>Psmx^ENKbJ7`SX%W z;hCwRMyBWVaP`uyt9JSrL-F?LpFXg`*R6rlQk`FMV_t0Fh0p}s6O zE#sK?KKyjm10s_jog0|adnbxn*-y3HP7D7neS4Fk#hwsRzGE8#T}+eKC?;9UtlheG zMXssOa=Ws~kgXNFS@7u#)*q&SEP`q+>f~$77KSs~z=Prex zVO(Kf`@3nMz>dPDlRq0@JPHq+c!He<7!whF`YLzf=dptaWrKnk9pw~3)Z97kkoob= zmzU8``s}f8bH~T_RU5;#LssA08KdSWZJYCaSQizvlnayiS;zZXf+b=84YTeo&aA5i0$ZsAgz z)?k$|k*;Vn5$iwsHhl%;9kw2;y+2xx7I@x2f071f-OI`+=hm~(P$_)y%2wz3@cMr! z@Y)hK-nC%jXI{55GU{O1$B$u(?Y~@kHk=zSbM%{cCqvxFvFWDnrpnB({ksjZ%HY?l^6Mqe*Sd23%k-A0X0ggIVR8;remt`u z!x3hB`}(VPO&M_KZV;!T(K2E63$<(dS!nAR!{g)i?PA5;vwe-*yi1Y=AE)2uYMs`G z1C++J+80(Mqyj5jPM(i`!ZXv=sIK(7b(-Q-RJHYh;)=v&yk>$i+3CEeJ~&0}bXLxJ z{iO3KqoEQ#o%ZNbQ}t-25TWSfs;<(yD}nP}I^F0s(|Y2{z(~G`-iquSVh2ij|ot<4(Jzp5YKF%b0oqkw} z1agjWg^4&l(PJ>LIDdQp^yYghX=!%iL|G#%^#zy;#+E)5HSAIzEclM#SLO{H4m7+c z7-f}{vvaSV&+PhP&$-OIiO>92_&OG_3p~0O8F`6KROkK>+%~(!R><06giOVsu}uS4 z@90PCyrX}d7`Q^wt&So~cBl~@q+QwQRa7gBDIjSXgz9F%Y2w}+V`!3plIq9k16B_z zpAS|+S1(@KO>j4?S5>Y5ns41E0ki#JBf7w+pnYvQo9iR4UGu}7o7V?0(Rr2ga>BLz zv=06CERc`~8z&|v$c&uk3Yoe9RJY!*&8agFkb66vSE*ypq{e^sB$hiQ-LFk#xS|Ye zgi<=)h{cnz4N76KZ+YE#)v6W?4Lm{4Dd;e!5O#r^k9oCe#s0HrH!hAfkS|$@3_6>2 z%mQEKCS_4w&vlkH+$$<5K)js7z7l};9OGF-;ST>CLQal5co}ppgy;yZ2l*v`{3LEpq{-^F8vV3}4J0ReF z=2p{^hf5s4OdT!rgve)@XON1JB&07fT){wC01InxNQRgTY`XJUjGw%ib8<9|%Z^;Syp&5$u4QE8zJFyUEh}5y(@t;K7k?z9 z;K@J9xg14~)VY}F{U~&I@9yj!Z(B9ixG-$0c;5cxSkpRoF}8CbWL8VSZ|`|et&Hj=Om9tib7)y0FE~Cim4F zNr!i@?9L3!+mvS(cZJ-%eEQ+eO}D@4s)&85!m*y*v(MFEPQ`MkeApocr-x#CQHNK# zdC|*R9B)6=OS$ZEn{7_y@oKT~8!6gX-h7jL*y4OUHEr1YSR|EU-G&wNb-Sxt4_JLy zN;UlcawQ7H9)tEZx76s7IzA^_T%~ZLyKiOd%GLLN$G<)zux~ko8`*yGk;t-#R$GK5%v|Z@1o%JF8h6(770eE9I&*%( zsI6S?_an8F@cJBkm{68Xi?&7Duqe4eby=v1g}&2;r@ zUUlspvLr=cd<%l-SvbE2xc#0Ja1HQEIFh85zDGxgR&)O`!WxFHSv}>s1`dUhNtY+s zUS!@Y-@<+?47cF+U%%t^FjH)u#cEZ!=3pjsZmt`)LwUF zU)+OjdPQ`5_v*wcw6F8r^hMd$=38~z{+NvHYbqj#o_#p#cbi{SNN7q$oR@1d1BRZE zkPst%gXd4-KSPQy(q5jOq8T_3H(py_8^2N|7aq;mR$SOy)IGfEDM-}w7uXf_v$00) zu}dBxc@8w2uamrI&ahA@#w=`oOG;0=hmqN@IAs^#f4n6PjWVBb8|O;2Wn1kwvkNd*dV67TBIJh9l;!Tu|p|ok3{V7uwBRFC5S42adSs-YbUmF zfHQjRIsKsG2Pm=nrY3Cjt&XVCu0LPhbDkL&+Wq_6;F=*#6^t|6KTP0$01+sG4Xu)A zUh!azEbM(BDJW9x&=r|EIRm1iSiO#;0}|1SJ5hR@flKHw%g1dYb)y1H$vIU|a{lRYEn7ouh+>2cm^(l5Y|`7H=SckeI;f!fE$5~Q)hk77`+FLz<*6663E z78Su)pKy2`7uQ+zR@Y(lLiF!Yk_@Xg;HmcG*kmJ_+5wg#Vwfr|OHcCxuD<|DE? zIy=K~&tekcy% z<8$+4ZNrwe>$h&b6bCW5J~U6T`X*9*!NS>N*$mv4sRJs?%0#Iyc_nvMSX9*aB%=l= zbl@-846>++TK)_1bz6Y=u8?B)z{Uw(8$6fRBbGpFKzR5*GWPo{GV(Eu zyjP=OJ$7o zia)2-)2r$;k>0nDh0ja6`rXN~vqEKUj8%KGT9p?Gw;OYnMRNN;PQ0W46yIwmK~q)s z0J)gw?IlARqBJi(jkD6&y!_d-hwn`7l~&L##l4K%`Q5^G`5#$`KSGY-nqW`X|6I_c zYn(2*WH>qATfNLflm(ORN1Se%(F>3Scc)vl!uBlY*!hu_dx-ywApb|UCC!Nj$A8kY zu9FmbRrJF{ss%zM*qJK3l<}FD3YQbQS^6ys$5*f8E?k@T!HI?@(w7Ez;L)<|mQQ>K zT|5fDFL&p0Mp($l!X8?q>x^5@oI&pOcwI;FqcDr99tYWgfb*(UrQa13{=vbVjj>GH zin@rO$~NMV6Uy6hL>bziRhpwnWWd6o3Z-2Gyq7NNFfj6XbtwSS)=E_5&>|$3m_`_x zlykrDj@-BV@LlfM$YDI6 z6|DNhlxa#HIdd2c%})IG311OI=|(v{%X4qukr(3Dg+`Xr+$A!~ z>Koiem^cEdi~Es;#l=^O>Lgs$OncK-_~_A_p90E!2fe*3^rVVl+-u*K5?KX4A0y*p zn>M{A)`_Ekc5dSOLsm+(3BiH@=ICTqK)tS`z5ON3S{h5VwZ&Y&{;bN}RqKxG=2TrV za_|jqm6qWdSjopG_ll^tJ7cVF6b@o2$mGU$u=1Ce&LV@KNFydlj=YG!8O;h)%`1(D`itJ6pn z_y2NIa2b97E?zd7C_t~Se0c*R^U)@lU(&hjYm{?UGfir`vRKa1wzTOKbp|=Jn7`y_ zUwL%l*ssTVa>=?3ib+Z$wcB;QcZ0J#KAq!^F~{FGUwqic!g?nEhVaY*YTZe4VsLC| zo@=T6{oN9p0M{1#ZOqy+4h+?v>YrvRrcH8My4$-nF(a>Oo zI@mO%&~}@MrM}S}Pv>K%Yr-devyxh_vaq~SdUmevBwQ$n=S@vc-fJ2g6N72xJ;7}l zzaxHP6h&wW{KCS^;GHoDy}`&U5gUy8<=!eLC8~!)zvxQJYK=|BmF>9SXr4o zwp1e11azT4(*`xftAjAy23C{diZnl2E;cAZp^!KCT`VT`^sv$$K%jk@YCV5oSXeDq zj~-#OhWI5YCH3asPC3X%sDHQ9)74B3v7!XTWEQyV)R-eSo{oXa$rpWneOIRlSXf{N zE&D-c6N^AKXKdrL@H+EWw2Kr`QF=0)fq>sgc$mWvuUWl%S<(TOnJpaa*9T*_fLAeS zURzt+3yNBs(=jRcXU56k6a1fNH*lNd62eB$^RtgmW*XjE|xoRy%XK^H?T zP5bw&oH=v1K?YYH#%@qb9M5_pVs9)Ntaqu)Pnn?&IBvC|WkYMD|M#)-M*7?RqIFLi z_rE!V5d`0t@duLcRByZfD=pS+`EGfsrI^JGRilm3TvljgJGOYr3XO7d7OGibq}#*K zyhn`v!F*A_y{Qw^{D4F8qj|P{^3Ux6XUjbzx|(0<1^As2S^eCaVQ`n$zjL*x;KaXg zfre(nbBJz3z^?>%o++QR0Y_!{^nz(wn7=RCoq6(d^u+09zut&P{`z3s!qG4`js*Jv z6S)T>zKM_Wxc8A=#`4FMI--WpN-KaTh32aH>A(94h;`SMn$FM9m3^| zF`Y+tU9V=^MGhpQ6JUhVYs9NPJ`L);jtPo;27Vn1>m45qiO* zhY!1vA^>ImDgHr*Pt}FIm@-HRSM5q@9IhT7spK@pVmk9b2d~GWI8Mn&5mQIE^RU*U z3S$V&92}=w+lHo#@8#qa4$Z9K%F)t(kgG3#_Q46|ncKQzM^0%P>Py&`p{Zi5dGk>>*}Z4kTP0W`;JUTJMK2w$m;^km47z9Cn6m2dw`1RJS)y2q3Bvj?Q?1F z)ww3+LEwn~yiZDe_Usw?JdZIvWq$$A{W}>M40m;InGty$!+YK@nl`4iT1DQt5eVKR zB`4>l#d=6mh^;??jYsC*9{9mzwY9Ygd(-^!USb+f&LcL(`$BJrw!y&wl80>SwU zFI$hkxqc+9X#nX*%KXe<7kozIPClezbZo4xd8xeTVvEJGLkn1wcyrP`E_Ke?kNHh$ z&DI*T(f#?pGu#(=$OIua)+e9e+`j5>-+kb0#tYkvMwzN`;6AjvHD4U}`6)xd#6zU_ zU^2SzJftSfaokRNp$T;y_yYzEuGcgF4?D_Ga-WEb_1x4bam&JxTak-7089sxJ8s|D z#p3E}D*P7rqn=8msDU=C0b@2EsDsahL2QyGqTZ#GV?gfBBe1hZ)S=(}^5aJsF6tg! zuQPI(Ma1v_8e3PZHDbpzy1ZUcnb?L+mS7nh8()>O!AoLsBq$(YkG6Jbh3C&s1=!Uu zP?LPw-o)bo^&nUyBP;7|8;T6?| z0xL^%HN3%#Sl7T5m})7;(yKD+_q4XObfbCAmV;#jgrc$%Unt)1ERyT7>-Q$S`}TGwd!q2=Vj~y~DcL1}eOP{@ zmXn??g3tA&-|G%e+>v4jVT$0!{N;mD*REBIw}1EmZ}BU&WIEGcL*7ttJK)p7oXsn+*)1vgis#SbycD7wiy#4Lq z1qga-7_Noe$K7@(O7Dv#r40ZzDxAD97-?Zk_TnLY_pbIHR;|`u-*U{d?B8zrzfXS1 zHY}a~)^Wr4)^0}{8u(JqpJ(19VVGQe^yIbeV*Go8wpM!jIV?XK`4?U;B22SCcDb*N zkHu)7Y6;hJulwvEz`jy|nPF_f#iye@w`eBO=~L3Mby!k=ztmji!emYb&$@N~xB2fT zPZcyUCzC^HcI^M>=nd`SbrI*Dz{Pnvy2c)gBk?!L+DD~J z0$}ZJy0!|&Ty_0*>hFUn34kZ8K!mzcObohFWk^a21x;UaSy_jrERq9I7BN|4hI10P z{mdFqPtU7T7Z%S~AiQAYgl~>+K>gU)5ZsUfYKW|b@SM%*Nza92iww!#TcPXj?TxCN z+-kK^KulmQb8t;|mmBDjtIf<(bK`R%P&XDA%^y5=tbN!K{%?5QmM0ykidZoijv%%C z=LllJ?kL1g2pZxddQea>PK;mJN=rOT#Q24AE@t@LvP*yj+p)CC8eJ}?$}H{d{x)oQ zBCqPP?8kokb?~EQmtgSj34Wu-64`vUst3^!kuf}aBGI9P)H8xHSZ!6|HG8|&>gYIT zmM`NA+aflNc@@Gohz$&Vig_zI#MeLL1X{K~y&KJ@?Wb9P1d0A(fNPVokxnO0bTdQz z=GyXK&abOiKf?Jngd_6pn+=qPm0+vCP6K6Q%=>E8X0~#G$IVC^;t;H;&BFK6lY;oE4K6&*fkT|K$NDhK)Dt~^b;hEeVG;k$J zv18i*%GuBP#~!yb-5ncW@8Xe)6RX6re!rI2|38`kdu}ZG_r+*v=oL}6s4Xe{a~HbA zJ4%mSLU0vAQdF(I==E<|!SVB6HmLjCmc1j=cGN9vp25}pXpNhGZ?Nn64Fdrh!Am9W z8ig3X(>+=f7~#FW_0oFSPHwj3!GkDu?!{~JJ~1|S9qffi8t$Ut(xb6juclhxjZ^1`s4Gm&jXS{)5r^F| zt>GduiySHFjJVw{xfnQ2jVUPDP9|$U{}|x4uz;5w zq|6?azATcH0#E5fJ$>A%$nKOQrbMg4n}?NbKT2;ta{meDHnbI*mJpjI8|J7)P$-KX z2hLCbI+Z$rH=2flZ)*e=ng{fEVgf!;Eu>jwDH-n#?NNsL#-E*N;q($2r|X0&NH@sKQMKA z8SnRAN+jbVq*TvpHTw^ldLVAwNv(-+9B1OrD`x5*JBGv2XL zjCbBHh`x30U0oeLT%?SoE&1OpYQ!a@A_*Q@*(Tyl$9CN-FjDFtE=?s~t^@yTZzxwpUFBNSb)Jcfi+^X??$ zZi$M6b6ufl=xzUO^)54e{Z6}H*ZhLfLm5O$$KQI7M`tNElgFIc1vvuETG2;qh-iNP z{(^{pShj2#a^S8IWf3zyg>C%rL}N6`YHgC2Gb4vpws5v$`cl# zD#97Ca!S|KG#o8lA0Yhe!3%1`8^y(yv1bak47ztXvN+yQgGuAYh$lKAfB}&n!^4O* z!u4v2gibDb5{d>pw4te2T3&R61(_2gJVi_!s**3Z#2OtthK&k4c91Z&=V;KS zchJjyz@u$uW(Ex>C9koe;VDuro({bI&QkplM+CgjgCUR!md_F;C4eJiZWrQ?W2}1Z zEg1rnSaMq0mBU3B1`o=K3L6Z-11&z;@w0x!<@XnDm@VTE4-5|n!{`mI1B9R!vivHh zdRL_mf5c*R%tP+`YU~s+E>Dm1|45r3(#BozHP~bF&dHY^WN~CtQq#RvjEp?@UNZ=! zQt=`p-6%_9aY1{89$8ik#!AXfp%GLquG1@dd3mcTf>lTD?Ikhjd&CJs74ym~k5-UI zPTj#likqAJh*Mx-;02{Bl{bSI5UskLYzdS$7`j9j-qr+hjEr(sG(dz_`H8zqOkCWo z6+Uuv98vRJpF40puahu7@p5(ZA-vNFKUHrNSs`#INqo~LVng5_Zd!&d$oj)r>o~@ydV9PA3$8n4G!S8Fk9? z$Dd0#l$2-~e_zG26yJbf?PwIgpO?MJYiARB_h0gLoZ9yD{{rf1E(rg$?{;jym%aqV zfK)#_FJeOkNiGx*56@UD-1B(R@zlO<15Ou|lx!Yi%fL=ra%$w|JMIk&`vPas;WZIx~MA4bQ=Yi{nazK;Yg~2XC*ZcYSde@aC5Kp@$q?Qkxoz?M*WPQ zxu9@8eDr9j5krm!N4E;39&ZYi@X!g+U^qEE%>}V(k*HV6YE7xANIbj~L<*)3)J#cl zgfIXQYzi$sEK-!upFih6hkx=~%ZvaS1QixmHU07J8`axe5xfVEY5jR;(qVn*?BrYI zBCj&8LmA6&U%4B`(>!T%ATvbJl7FQl(B8|&jSs2ab-_D|AyAPrh zs@8hyzN_2OEs~_+Tew;v7@q!z2Mr8JpPMa5@Zhc8NMj6LL!#ciW-jDSUV$N!@WptZ z1l{?t=V$zQr8h4rU9GaacAfY$Gj2ZQqJ;qllE#3$!=xP>sZT%vWLU8x*?c3|#Y1jx zvK4c`_NTgCz>P^kYqfQ zwbez8w# zWE{Q|JsqV=U;mofF>q<$I*QqlB}WpQkjs7QttBnZ%Xsj%;45x&>tuq?hZ=Sf+6)F5-82 zrj;2Y3!?7pqJ%KzvsVv$_H;LfdN43DQjx{iyX!3O2H>|;GjRt4ko6fkIfq*yo$)hV z)(osa3NUMqv0l#g>!Ym^_>I4%FUj$;ACGX*igwU-&ObX}YHYXM3o2k!Eil{=U2@TB zsT=1ZfXA=kW8&HMWYUG=VAa>xN3xchySnl+wwhM(U{jT(=>Ee`ySr~|Z{g>EGP*aQ zK6&}_<(N|Ix_$TVKD{=pLZtpFA>V{dOxKivv)iX8MA^AbQ88>^INwGqTutI>0xqw+ zy23YjPc1=K`Te5V zLBIt^baC#bOG}`HEo0iC_^{OE?ppyg*0Qp)@@)|THu7FG?CZ9g=(k{f_*$xQlb+H2wWO?&dgHo3q;^K`@M)+RZlM*vrYIGrWIna_VVvX{nKoN zOHhW&SLkw z{PR5^pruD}#P|&6geVmED~6YmD1v7wgq|7yML;`jDNUl1PZ+kCt)OS(yLKQBi-x%X z?_YoVw8hT!&><3+vm+%c8bmP3Xq}&}TsYReZ5T`(F&5xc>T6BTKeG97IWdt>-gRV8 z?)cS}lQnKG@%@ZhC$x?H$WHfhefWjP-oU(jfE zwV$R&KtfWDS^in|?}LMx2e~P2l7BpIBqeRYE;n6W-HXuAo{!_JgUDEgGa44A-hZ&9 zq{N0h1a|$bYqdlk2jEiWQ}UkUtg5OS86Q6^$svnicugEBx9qorBr2YREwbRXr8H;_ z18G`lSeR|Q>7HDli_iX1+O`uz^dV}gI5VG?(b0*Kvk)N^7~L%^s}VHEY&IdFg}C^(qIF->b7975m^ zF|pQui|KMe!0E}c!9xl1Zg0Nq9=>-ti#Rw(;^fKGyZ8~w8zr=tT{R$u0pF;j$D^C& zGcDtXn$psCzO|Iz#>vP`hf`5sbXQ{Bz5LUR*5nQM_&R~WugC9K^0GwAP0jB|Gft)E z4Ysb?v-rDLRk4d_LQ+w@3qgCT>go(KcAbpft${wcFQ!nF`fZ1XhVGwf;?r@^p)uL9 zOyR;;=8EYr>x)kp8j-2IxjLUY|NT4LTP>xhMyi;a+uHmA5sEvZh?{y(F9W&f59I>8 z30}W{&!ETt*_P-`KC?wW=H~n<8_UYe_3*rb$t`}@+_&HsJ2<8eY*$=d-1)(upQr_d)Ps3^ zp}w!VS&w8-i;51EZ}6Cztihild0})Ce~z!erM=K`BOyV5$IT3+5zT*D36zk7Q8S;X}hP!t0vJ?m5r&)ZQJr@w7j>kOf+H*+J@A;pHf5HFE-oNF* zCX;@vnkeOyX2_Ga0#97IG=ezWB@ zyMcj$)ckxsz5`lXw~^ZmDq~IKzJ4q4*&KaSQ?5U@C()wA94;FZ@9yGmGq&tkxhPpyn?;@_1N0=cfncPv3ih z&s-B%$r8|E2+-#}{drxy+7lo)6b8TE@vb*-Qr@2)e3ORS{KY|=ycfefscq1!zt7!_@ceo^++faJQKp)Dyzd@r21JwehmFQZy3-4x#iinhqPuHVJilHdu z8$~7hWJ``cZWKX5LE6jwyLJh%M0#ZA<<;RY*>*aD$v%(ix%oBx1ip}ulx1fMjRb-y zi(p8WKXLvJ<~UnYa>&m z=JEg;_>SwbP8L$9D2EV*5=)*Aq^gN~C+(1yM#BD40)g^w-`>;AY3J$518olyx<_6e z4%0U7(4h29nAZRfX!TH#h%Eg*Hl`nfH%4DmxqnVXRFnfrDs|~CRL9rN&B>hpSFSV^ z;r)t)%s{T}Lc|^tDF~>3O{03h_;Pys(O;kUt8Nhx*qbK(Llpf3?&)9fMy>z;{ku(x zI?ANM=B--|TMTlISEM#)9Hcm?;kg|7_3PQx)MI0)G)TL?7nJ$6Ffx+m+O=z9Ry{pE z2<|c5$g*WE-buWoxJ#p~frwyiHbnbSZ6VbI5E1nIIn4>MuVf-ZGczEdQ!7e{T-{K0 z54Aw*9KnhlNUXYP7GodLD42aiMLEbyF%m;311kVnGMV8;y;*7g{g?Ni$O|{aw14?+ zuE6>~S_gH`j1D8<4t5%W>%Jppl~ z6#?}*G1so$E-gK#*A9DZlR#W53M|=R?((y>mhxvh>lZ((P~Q?d{Fgh~L08YsI?qkC+c{{n ztD=mt!eFxu{AmQtj8Kf-y!P*3g{D<@i9!Q52#4`01XaTM5L9~<*F!B@zI^n*9B<1j zG4)G4RjAw7t)ql`KU{9$}D1e6JvnKL|t3^)b{g0Mn{2+g1XM2 zx!o53a?VDu2rY{gTw27!my3PQh+GZ*so?go*^&z5v^;nv&%8Q8BgXWdd*lBb6X}L1 zfR%H25^yP^0Z!I7*VKr@;=_~-6x6Mjs1REX zvRc1*i3$r}2|a&Shcwr)FLqD=9*`L(xmi97-al3)%c9gS76@e(m1(NKj`J9;T_7KFDi!({US=JC35ZPkO7v01gzNxA;R#hJ zAUL>+G&%hZY;Im4;^4){-1i+d3(DO>vZ+f(=9!HQt?Q1n^EJWiM*WrMdT!~|GBGeb z0pUdQ|8ZCdpa^QpZgcnW7{wCtR8U8#OW>K#PfvGXS%%^Qa)%eD5kd!=L1BkZoy0; z5?={kHYPSR+*EWRC;j_{95@d7>|62#5AX*|2R=O8FzJYMe^qcDp+N%rWs|1;!fQ)W zO=>V{9ydEup9_+vW_En+bl;sYP_0n*HxZf!Btx;>CClc~$w|Wy`sK@8Rw&!)A3tvU z+k`~mU;r+Y4M2Pi#_<9{lZgT7u0ifAqWkV>W?GgECjEi4`(8hP%Jfc2$xA;g7x++e z3y#Ld#}jB@I&cuSt`7<8dD++sbR`4pS1Y*vq5}{K0ttCj?qq!$7!4Ywg}H)Hrzh}4 zf&jVyvSM?}&P=q2c6ND-7ylOtYX74br?XoRXQ93us<01_bqSVzRd^WPZ+3!|9SFRig#wUu0%(66W5 zzERHNjgJWnX1YZ%)VA;;t0}G4k(NB{5-Az)Fhkjvz0D>XM>fOhVVMxr-A3A!)?>-6e+~8n=f}R30>6)0FE7EAIuV*0s zKTz-}5PZnramWg+3J9n#pPriHmy%)y%NJn{`-IIot#eJ5(iFrJI5{~vHyrw~;f2D|s$m@=d>Ytxhg3 zeduO_j830UZ``o;?iI=M8?L6N8_+iO3=e1N@Lv|vm_jxa#}4187ZFj3mUC_Dp!O|t z`)ew1AN=v-(X0uHmf$nzr`&ad#SL7fsvgJ>Lev3oWgZm2`N}N+gsbZjz4-Z07Xysn zc6akv{Fyj*!O3ak5O?TuSZitkZqAh1E#e<$<|mWODejv{t38TjC}wW%dpevBD?oEY zexhK5m(4qz^6Kziw&^74aLD4dc!^!m!gzMFp_YFs>QI%R#sFf zlD8B|z|L)}E+fIv7$du*af~c4kAnL_d%h*fW$q)ca%L8mb%w_8VGa z2p`AP1*GH4-@e)S`T3Q#IpS!`DtmgC_0Br0sL*0k51MlQcLbF7^;G%)vbdijmI7)O z<=AteEV{xcKtO|4$v@y5s3jmF1qD2Y{c`C)>*6z6 z_Ur00BNvGybnHh*F|jyJO;2yJpi*L+TUzL{TE)Yi}-PlIFmI(>jX7_4xu(bhwMUW9@2*KwZ z)}#jOVT=N5M#sv7Nn%078Aa98ZvnLs4qr+PF39o^Xq-nNn0{!P{NC=5u3hb)vV|2kS>=-_leN9z>U&@8C_3qdqqO)D`LoS2HoCk`CdKZG7wi0%_EBbZSwDMnM>p_6g=Anf9YO z20#(1cw~V6NvS{+G9XL~kgSY3nkR(R!h8V(-XFYF$!6jySJv_Hgg}!eAg>VgJ+7&8 zN5>P8)3H5IOymSsgj|^J%B8MqKn+fpk(E7Lr1S8-c2adh9$U%>Ou>jFpw7P5_V=$} zK-a5Kk-J;{cH%~?L+M;AEf|><*J89~9m2?P`8JwmXP^ln4-i^WL_)*QX&~9!yF*ek`TfEg8pzwg$NK1%4S;ikM$zcX z?XAxxj&{<99xFO`>zH(&AZ2olFzE^&?@lWZkbMW>lF=31&|2u&$;l{Rv8TT`QQqw` z;E{=a=$etCnOiV0pMZ7OuB=i>Bj`^!4KpvVLKgs1D8X-Q8!JUR-wj^qSWtJyi%jT# zze2ZC?&>fZoMu&Pa(V#ohWZy{l$wnv_xjs}Sl=7MoQfGzeU&DQM!647mMV zY(&IGU^z@1%WdOjY3?O+PF>!!FVhpFqkT~}^FRFUXz?b$T%{4kA4H+*`88M^-2A;Y zJvYA5%cQAYx!^yF!PC>SvF!z|4~=p@`cTkO?EiNbI{%%8wL6{rAbQ~A${s#-eu{&b zhv+NRIXK7Y5Wc71;xjk8kHC-P$B#RF$JP2Cn~z0fgx1f`&+l>_;fH5QxXt81=kE;* zqZ=d^>}uQE){#90grA3UQtS;^8{M*HClPIh38n<&pFe-@Lo)z9?6CU&{xSggeB>cP zThu2_tDu4cFWla@iUv?4iLyi78&Dj`5kRuCv#8cNOUpgD@1Yk*po%~z)0tAHQguXe zQt|8r1qu!i5C47oDRpF~7n>-yn!cDTIr$W`E+enPxd5_3ExL6yH;-0G26o&mUmgHm z+a^Taq7K@b*|(ut)xfnkKiH@pI1o;F44J+OJc%&w^P&z2wh`2dt!8w~mrEScp%?;I zp&s1Hs999b{s@bR^tPG$#H?rBALaM)qY-HtKU>_nO1eVsas>V&xp{95wSQ$wYAVOi zc0B*$=|?-@BHAV=cQ4~R8cFS?4Lw-Y$7A2k4l>^oAM12Bh}1)~1z zOEUL8)UvVWCwuqd?GN!C#T;yo*HUm^UG|abxg%$wu_2l_f>h%*W>LyvZ%v}`^PhNE z2ZHwL`}c7vYuVVghi`;FcNmRHEiM$0I0r=;Wpki4%wa#Y^W*^!1UbTfOO7bYKrotc z#?`AgTHz3W{>e*mIEIVB@I?a+4Mapc`3>hJ-A8+uk?!E#yVEY^MMXOyhmxsK@6*i9 z$D)N+06SD;c7T1H5GL}juTOOu3TY?<8o>J7>AJcD%_7>!C)PTF0c&-!KN=#M?U0+B z8@N8l4^7;Q4=H|e_lOdDYL<_OD^5$K=8cK%P)Tt|osCS^AJG!_z0CX!OVM|m;iRX;I9E=OFMy=jKKX%>Gg8_Uf1M3~}`aii0vuebL;mNpRF znH!iwHmTM)r-=39xd&o1eVx#MlRUAQhKG~6z7UXS4ojP`@*$uxNu3ub9sE@fv|_^+ zbY;P0eF|!5HEyqdca%rK2@v7~Uo)sW`%{|-&hTpMV9EOc%gr7XJ#z`A}eZcpYg>>~SdYdZ(MA(4mfQ zBvt`n&SGc$(%;|zUfSl?COki2cilQZe3ksdO>J~*s6R<)} zC$^mc8xXWSiKqq*4ffOKDPzvKB;j4-D1@#zH8TU9%eZDNy%%VUlZ)$8czC$nrL~!0 zBEUKy(+kEFHRKJ=KugJjqDl|UJrKUKubLocx4Q-reMM#9qoY;x~Qq?fu!$=w2$CmBIF*pE^#427qIj&?Qj$V?PVXy zS55=8UNww0U8KiNIW5f{86Tf@kTS4cUOohXD_J=z}H)Qfa1G_TN$=bX-D?6Kd z_;5_hm%ct<=t{-kMnAy{O197kxoM>zg9QQViIt-wc>I?eG@^9%^(_i2X=(5;pZ}a+ z1oQYs=ebp(@_!D%%g&MyKejt5O1N*or9;1$`NTF(_Qv84@3&rz%V~ju23#RE7}34Q zjbEQXRqiB)QEepoz*zJvA6 z@`Mb;+JloYe*tqGxljD(jxi{hI4RHj@Dz$q-0b7*N=u-0(E|`okCoJZzqOGHIFD{_oO}ne@iPuW19i+5OPOywG zW=ckX|HewC_3X3j;C&|?#=K&RYEDIkiDcNKcRgJ5>{@@^bv60pu#KP(5|ENQSgzZD z9>Bfs>(`8wN4PU$-b8F#!NlZ`bJt+;7yW?b1p1+nJh3u{tZ~ybc|#ZE;4e5-qyI2& z#bp}AVATvffAWL|gydnZ!pFFiFp#0buBz(QRpG_3W{tr{4wmK@gw90P6dR01K{Lca z>GIQrZW%!2-0$ph@}>3-__dU!yw6%RC&0yeg)Nqfr8 zg2A7vU@uX6W55;-;%vju*Zkwmbu@YCR|$Z}jaug~Smc1zk$7`Uu>UX~QH+iN+F-@R zA?d`ofI?1Y0oh=^<8qo>zfNCZxei_D!InjCVpor5YGqUkRP$??;EP zkOqkGMY2@SM5p{ux>LcYf$;<<>D`zP{-1}!WUHGFH!lA2t&4jITiKDa^H3i2m;>L}J)45dYoqI5# z#MfR!Plpm=AjuJFP84b!mB(JQkBq@d@i{m+be>(P9_vcSlfucz=iIR3OdwCq+qZ}E z^UlmDs!-fd%~)6Ak7+{G+~m|$70JY5Nk7z=RLqr}W8JF3ilS`aUmuAEvVSG2x4C(0 z%GwPZ@($jt5kxM<|C9vKJVFwP0vOK7&~1S-)w8y)`fLw`*3GcI;r)+X#Q%R!z(G?} zySLRFckhlLP>H-p08rf}NO2GaASvN`KG*{Kn2b0dgkE|hG$f=uX<<$j#1h9i=EF$z zbVB`y2JM9xAdq!M8Cc4r8yYkc6upc_*Dn;JdB+aSKRGvPfy%IPOEt8O!rW{Oqz}$R z+zJyXbBgLT7lAX$q@S0Um&`D}1A8bjkS{VLLONa&3ZJL|7$}^drBI5T$h+6K0>CGD zzT;Bcz;-+U8usU2>H~wmFSK5VNBw*E1#15}lr3WZQ;)inu>tZj9y{?g_ni>pVWhnI zQa1U}GJ5({$jfj(+7in|xwv_Fe9<1X7Z1Kd;px2j^jS>f^0^R+`ED&Cfx@;QBRw!aj6BDz=*fSe zPzKl@QJZKVQ>km1A6ZNs-wr;AI1!0s7TnB>$HxI7iIg8cSbqx*WA~FwiwtM~&A$rx z=z!7!7ffg3=^j3y-XOaO%*@PSL`S3z+$NF!(P$Dc+&J;V;dI`7y_sz(94lIsr zm*zh}yrVCBkuXFgLsIf4x}VY#(eE;UMqCKbXVSH6*Fr+LbCBCX6&NJw%jdW8;cuU@ zsIp)<{9p@N-GgEh!^%?53K9uVpxa#f?@7y(=t}q4Ypo&IK60b^@ISD;v@tVV+U5Cs zHC|-)aU6BxeghRDE}U9?bN&Vzt2`ZM^q3I03HgJsmYiWN%?)*G_SVgV6XWA2Ei9;d zSlH~Hmq2~!N_2PP=J+QJ;1Tu1%g)Zh@f4e>vrS8#w-8Rv(b%#R75apyr?FmRQBR5@2D-yXgZk6f#)c8yYmS*W6Ah4j z6@boR&G89HI(o*&A$VP1Jl4keJ{e8H-k#$90RcRFAbf*EL4fjuqeV~#h&*})Xz%aG zBgju^c*TNn*UJ>$-8$glVfC_8cArI(LDm(NR(=L@I5KxY7yFUgd|#IM>$aR*ME{(Q zgJ=TCNAM~i;YCU*GIZNk1uIF}*{jf+O}@C(w6(XduM6K$*R?S3qMG_66C5$=N}{9A zs$pmWpDWVg2WY@9^G`;0~YJ_Aa7lwr#*e3|9oE!s`nWW}|=j@JYZ2 zE2^-A5hNhuibmz=|A`Q;3MIO&boHmUIX8_o?fJUB4|Jr72o|kWRhKWI&D}sGE`XP zK5j*DPf%c>UPx(4$u=38tC-ZuV6$Y|xM@=r_??tTXt+Szkwef+Tyh5%Tr|U)+uMVo z$7txW(Jg^b;|f26cp8QB;J{y{9t3BQJIrW03A>*D+$n(vCcbIz;zb&#XJ|DChla>L z;?&32XXp9r6X|EJg(28~Vu|9kBKW$l97+6iRPSQ;y&y7*xMS8`AG( zGyu+Ib3Evu$8%2J{qo>2ZNrk$OPH@UiYqxKNg3RnHs zk4>Bu%0B4PQ)u07CW@^v&4&tt$7hJ{uc!;73?LD@t3~<;MIosDZLDK43V=*}>pa`> zp#O6is{TxoM}Ji?QOJ1AbN>ltmZRNyFht1PUc1%vT!@ls31A18MZE4AX zhPDbmz=2JC#mT(8or!XeQRG-_VtNDohdyxE?7?&ox*F^Y>)Ws}MGt59Ft7+Bvkavf zVUklup3nL~7H!;@#BxIBVc+Qm8{~g-@pbtCKw%gSm}t{_UzEiNBHwb;ix9l zF|eY;5!dVYf=caY#)!BdKSR*ZB#1D?Tm!Jvhc*}EuPb32e3G^5=YG5qV_%z#yf^=Q z?)wmV=hmpvAND^bu%f}d(7`-j%1mre&&iPR@J&|OyXII?3ClLFZ9h1HizmJ|3zm6K zv%<#n`+9Ue2M)Cu;Y-AYk9q04O6h7FBn-pgmF*=0HzX&uxV^-_E4DAJ-J=9kNC^|%k+$Niu z@Y)%+CPHe%>)JJ)zsUZhb^YN;Ns;ARzrH~BbhSuva69w_GMiI6;HFBYb^}WhB7S8< z{VgzZ?VkI@#l&-2088;BPsN}-ourau1>7o+GAC%HNG4@%jaHs8l$GdIA0)+RW zg=f4Y(Fn5y-xb}`aGIntW_oGZ*vMA}ym_KK6R){Z}QzXb3iK zxfoF-e{nq?L-K^qf&U$?7CosSZgeRH9I_C6y5M ze_nT-^Zot*zkiSOI16>(pZh(o>$NWR^b|kzA%YX+uB(hw3KO4`GUb( zp)FKZH;_gnC`tq|X0}x2Oi-WPGlRz_X_eSK>agfd2=_2=-c$wZZ4@>}I3r4Sf`V(F znRaS~E!A5O8hYU8A{M36DZ}?vpQ38C6Sxg$%%oYfbTUNJO;N!lg1CqC&2z6FJ2v0C zZ?hO(A~F(Ic{Eqm^5~!Ztt7-kO1BdoT|Lv|*R^W6_0p0nvGk~z0>&3T*45FufrYf7 zla~MZ*YDnq6aw_mUAvA@<}-Vy5obX_A{G4g`}dnt!dRM_Hh^>4N^?f|M};u z(F7BV(Yv*ShplUmxXI@)TA-noKirhi(e|82C8?`HIp~Y`{zDBf#0qU^v{B?LPIZ}} zj*8=-m7b3Osej^&^FaGPQ<){K4j|{?*b~#gId%AfN`X}=MLc!MCW}zFrF~sjQW6;? z-e>&xlNYTgYboBKkgn2UQ2K!<`;PN|z>D07Qma*DBI?rw#P8xYY#-2B;l5mY;@lx3 z{~=LX{0k4?-@biYZgWar-Fr3%{-*lP%F3dQ$Sv4+Sz@LJw9Ylcnr8MI%TM(Z1lOOZqjx*1~zYQuFf%2 zqfX;z;F4h{cph1r+9;9R&% zrBT}55W_WCOPrsszm=Ze99z?6kJ6J?l4XG0K5EUk&+h0P-b-mSpXJ393!}6dXJ&Q_ z>xklOFXk|8qS{rTk~RV^d!&=o8O23H6h2NlvJ%j16Chr4-W&6z#hybTGoseDQ=v7_ zY`Zsj)NtW*`}VDlj4|%ez6AqE%$Xxx!m+R|n>Wi|5Yx%I3A?q$ImvB0{Q=Nu(SKsk zUQ`Hb^KqaW*mrTAjKC1*oR33oYh2>DP+LSTNZwuMv6EXv3C>gY7nbg)zBh~i(z|2F zYTOZTH+Os}7Z|TlG(IVkbtQ-D;hyK`$_2#@Ar@nP#(ryZHdgebv? zq_lB64J#nVFp$v|8Sj=2q~8+7#FZRxuY#vsbuPfyrsk-B$R?;#{{(@i!O~Pm$hhJa z5%k7Q!703H#VbNk^V_eJ z-r-ke)1huvzw1KjZ@uLly0_oS(!J8X6IBPwYZAKyg%}o~31H52^Q3%gPsY&OXx7x4`eR*Ini;~8 ziV@es5!b$2i4Zg+Brwd!Cw~@bKOF>@o8GI$dW{-Q-HthB`t*$1Lu#klzQFa>GF880 z-!xm2jOI-Vs(9Q{vF^?L_m2JyF{(6(!RA{v z)+g91QN02bgix52^jQd=E`>L0+*nZAE&9IWX`Cc=D+~=IBC0*+{k)Uj;Dgh>msrdC zr@;^$ErSZ@cPhnIpmQ7I?QPAD3PUOa(GBtV{)q0Sr>83Ztqyeye-4)7B8saIZ%1yq z3qof(M!@5Lb+u#E(MAIvOpmnQ0z*6VFbZSdkZW$gp8Kiyf^tuS{ood!Btp|0+-w(& z%6ih~Q|jq-cI`1qO{iA_Z(0QnNB(5XNrU5a8A|6_4gZRb6@%2b$A4!~*?3Xlh3~On zi0A}qsDzN91%K*=18ShGklxW(f&8f+Km4~&K z;@ck=W})*@lDd1GC)nJx406hTQ@%v$0{Fk%A|0NH+B0cR6xB`I4UddpqM!Vnuj4n! zj_`Uv>tm-QA^TE#K&*}1lNvi=1s=XnMTweuIPq>>ySCnT4oD)ewDQm!J3Y}`SYKIn zbpNM|2sHHc^py2=)(^~n;z0#{_(aS@t4j_}P6=qd>Du34TV`_4nFd#~vUxBsA9IRJ z6h>&EN>v+#^UZ0kDO^<&ant`v3cj%rnF2C3R*7dztM`B@WqQ1Ln z%+{Sd8z&6t(c}HW_KEOQoLZ6qZpCEWR_Jh4$)y^q>K&Z(`gJ#KqY+^)mFouzd4~GT z{RvGdtmWKqpI_>PD7HZMx^Tmb>#G%;nsa+PWnHy6>qxsNk3fRy_|>Pxgd(&^D6&(@ z{A>BCDee5ad@d>3EI5-U&29&bCzfU}D?~RCY*Yg!d)T^9*Xz_}2ZQ7LuLa;0!b*WD zw}{WD*f7jD&~wRE3Z;R@UUs864j0`EHpaS<@)w+vn5<`=Sz&$X6mCsDqG=Xxs;dtR#ujLH4g*|}Uy)>-FFAplJpQ2x|V zsvPX>UMp6m!rEd5zJ(W~&wv4TD}zMSo)gzbciQQtvI}}p+=$%Q%tGf?H)c1FoIH8= zxbXKVbYvw6;erPuHk%)SrL-d{s)=>Hye!E^7wTG?rb#VoRs(E#RW`e7`MY)#J%i%M zVeahM;BbTNiyFSBABLsmOv#0E%wqLR~q}(iSh|Q}*%EP^eQ$Mc1mcGiFsBDvbWE@)6Bjm-<9j2%N*s^zEu@a!S>iq{c*a8};TD zV!q%TEcIR3>wlDT^X%;GZhsIWf{&ZmR41g9>k%+ouENq4J?N5bQu236#ulP4f(EIk zP?W(}bhjUMtmvR-Z}f%S#{mC-I?%U$y{TDoEu&-11?aaiyS4pl8PAO#yI0V8D7dLt zF456O8(nGR@#$H$QKLsIyV>wPTc2#OjvS2~*>)Wo%LdTl4;#GP%C7_Goel%7pU2yM zTT{!U1|>_vP!U^{-ssUxJQMT;>KBkBu^5yh=i0EumzTZSZT^#<6PjmgN7a;VZ&HG5 zRIiRI+GK^m{KP3<_;$6fNUQ_PevUvf;p4JwS;>{5LsetCf~a6Y)fEJP@Zn|td@|R7 zCG>_4T@t+n7F|@Mvf;*z@=a#%G0pQ-*on~4`E&|WbIod5#aV57XusEi8<%+{>n%d3 z=w;n_M|wQuR7&Ba3=S3HGm$TOHtDRWH6NxPJinx@O!qCSR~m;Y-S>)t4Y3DT`IVwo z>!9X=cD@Gf@EY?+Rh+`{l$CQ(ydpvRG^0^ffs<8L-zd6OvAEc}?4zA@eH4$^DaNbT zG_HV2&zy7byywM0i_knPb*bM?EWDJZ{iAZ>t>mlh_`ec{uERDpu2#EeZL>#NhF!E2 zmzupky5;cmpItB3Z`4Sd3Kuq<=}&qvn$8icR()MPtWL}&0+S`~X@Fb7cB_6tDWWx$ z+?=ZVz6q*#1PN^`v5hUfa&{4vlL)_10KRuVKUw}*9#cZ=Pl zjsb9J$vWml*0~z%m&`C?N686ZCI>>!6+lHkL(i7Ig zC+Rxng%?bp>6B0C@Whz#GdO+`72Du7UnjO-xnjk=)fQSSxu=0IopzgF&r?7=?6x_{!L`siZ9(>O0(!q0V8u9VNFbF9l zeD+N$w~lqMWImn)Do(*>+7r__z!0lur7B)uR1fZhOAS)|Ao(BvSeT6@Dk#r?N`1Je z_oqCk-0bVIj$_k=1agH23Oq`V&%uqgTy-SsZe7I{bK!2q-@8+Y>2>Sc3vJ7j3) zoWqF2D7juQD_m#ZJlirsV2Gf^H$Bb@Th z^V+H6KZH+&co&Fv*Dfy&^kw3HejGd{8heq1Xm;QEVO*77joyn_T%}jTFcf-l&;~C% zy#xFAUz6Y%?#*s*IlGv9n%cBgtMK0yj)%>(Q|kY_?%lE;<+wCIHz73L_s%eK2_KNTfl!lZF;&eCk1ZyGRV3 zoc22K!)P3DwXj)J&Bi9zeRVHaY&K(oSEiY*f1IAaNvJ6RT%9gK!}x&Z#JF1iF}BsE z@UXDq9v%ZZKHBczp7iYlS8bKmO2yjz_HukmKPt-NSHAU)$J+>dE2{!rop6Udl2Fdh za+)x4Eh0!cuX)rR0q)r>j%y$voF!^N3@I@p*&z^1S>tT%AoLQar|TIKk$z*XB`wq2 zc(K?=?A8vg_&D~sMiqoJ#Tf`k?}KH!jiE_NZ^6P{5GyyVeRK7Yg}i1i8E)%t88ixK zi6!-n?{rDqM~j!DsFJB|Rgo-r>Dv=IpTvUi>Uz#6+-@i`#k(O{GvL*6ZyhvqHprp6 z!Yfb_vcD@y=a*L&9#Z^?20JXQn}GEPl=w}lVdIoYgiu7p8Ba=oR|*0CdUfTf&RT&J zsLthVNiq2X3?Op>NO~wX7Oehz6;|&*f(>bLXz*gUAE|flG@!noy(@Q5dXTQjruFrg zle8jg&zGB7`rc+edvIe=o^euV{xI-#B2+`zT7oJzrK_gdE+c3UU~>AT zVsM?P`H3Cs78@SE7t2x2M-PB~&S;p&NopZmTt!kC%pFL(mJGaqwc@)2l%Vcgx@}qS zlGD6*chyPXvoUftM}wOS`YNLxy5$}^W3bPur-m;_{T*s*WvGKv#LscxtN*=dgr$5n z*q}V@Z?F0Huvh$h*hhBLtkz!|S^dI-a+Qx9Rz(@{6;mgXkaK%w?%HioS7j$BJR5P( znGbnY(Tgp%8xK+x3W$ytJz+s{SAYTeNGQ`kL|md;TlS_IRLb*b3rz0G@&p@Uikgz2 zM~Sbb?92>Sjyk5w0dB$!H#l1;samM6FG?Z@jl9glp|{5p^|>Eyc+R*HBQ`2Jx>E9B z5vqp;+*U^`aaO$N5a|Gp9_^@XJn;kHeRHAD+P?20>tUQb^y#xSC%Re8UvJFQFz|Lj zo4?X4WPi@)>o;$XM3HdM`K@!SYhPAp#om8cDp#-mK7g9hgz!>@I`|fC^h;!2{dWfhOf@qt4i~htasB!?aC1!^H}a!4ZW$Pv&*0QTt5H&7qyJ45EdtaR?%!=N& zL*F|ooHdX~LajkSKEqGR&KZ5eQr||nqkBE67^VZF$4$fXqhy)l4EywH3d7Z%BSs_N z{g*P|uDxGZ#mpT$cM?oGX_DK%cjuD+!c}jCUeElT-fNB1#k{c+5C6Ek@=NknQuj^rPXK0v?Ts424*qqY2qe=7bhmJ@K=> zlR9#*Y^OTYwWrW!7j7crZd|zaV*DT2lRo|UR@~~+%qnG?5Bx!5EiP_Dl-&tvUC-<< ztFI!CcV@`8Ro;8j^b$986Q7Pg!w&ZUlgY& zW`r9kW>Z%D{Q2`M349abtqG{IrQr>38tCO!pv`@QqU!T1f+HwB90SMr`dCMbzegmC zcP?-hJ_D(9vw!v;Q3Pi=wkahqEGptL)>_jG1hJXjLj$8v-RgcHn#LKF7S}wx_G#0! zc(Y-*^=;zkBOR67n^Hdc0?h^dZBeFr%t4e72?GWW3@rcipo}{@G<2 z%a#*a;<2tT>*FO*3ODJ}MR%Mu4GEJ3#3Z<1RM|y3Fg(epGOsrKD~ux3?2uK2TC~){ z==T)P;sbTKo$kvWOjnS6%daW65NyH5$W;9HLLBN zLcUwsKS>-Tr?|=STe6-T%)ct2BwzmJ_fNH3ul#nw`wkm#8{VYn_pa`r^;FZ=)GN7v zBjZR=S)w_;qL^QP@_+!RM0Z#4(lIXw62hH!{3 z0WH!#eaiD2_P0nUsP~&vuoZu9YjW=z$`t(?K7M|jq~?e|RetGc!k(NywHA3=2c*V))NsS(6 zWJsc_&nyZm(Z7rh@=jU|7WqQtW}qEvrO+NITKx*AFR^{Aw?qZ7J8#iY|8a;?O03Agdxawy|d}G##sxr>06|>m1FY zYg!Vop`(2)A%gVN-P+0XLCwjeQJ(7=Sy|p3_qz5X0hhir?Z^Mxu4cSn62L4C%rG}o zQ&VAJ?eqWhxuWC|DVlzo9_>mG0WGamN1~M=jUBBS(&u2$6FuTul~n;(j--0B|u_eE%zm^nlyq52t$>!_D4^j?vT$?~ z)XvK6ga(iI1Zo^cSU{@&-+8FY_}D52t(M`fh=ry;Aj7E6b~j$6AG_vznjjZE_O0e6Z-UCa{k`Hmq}GAcgixpqoV)h&26#D5?;N==6S zSdX_LXgTIh973rZHPhB15zOXYsQ_>GqzRpr8v^(SCw=A|i&)j)# z_N;@OTd8)}sG#8PBwQ4q*=?iBdtc0z(U3;=NgVJi=DIY9t1hw%9$sleLi~7_&8=?Yy@a^<ANrUpc+~-@_jrWfhjz|4Z8)v+HdFl2Y$N!U-jSNjU>Vc3jWe#Bm6=mxxTZm1- z0}-ZYS!{06l^)8fyDOO8Z<(dfiMYoPmb=BIzH^ANTLD(4B%(5VAica;{wY{OGx_pp zqde&2|L35;x2Ed$a6W${Lxlj6%=j)Oz2$7QD1N(k`B8A-EdBV^DUJqt55lK>B`)*X zyLYj3TP2m&+qfQ4(u>G-KaqOwc}HMHKY3pu04NU*i`Is#GIwp+!IXiM)QZ4}MFEaM z$JLt4QB=fqs7jJl8XG`9^?;Nnu)Wf|Bj#C!epUKV++DHF_w+X~pxIJIfh4%3WKgZp z*3wcT%2rJnr5$Q*TDQ*Za6qFSMjl`;B;wN|NIE!1auGox&Q}UcLXOcH><eJv@)#a#X*#L%f;g9brmYv*uDh&5pFbVSOJBIevl zu1tQ3W{Q&jmz`?C1eP6Fuv$t!n53$@`UwJbrgLx-ssY}7!NI+K&6Gg^%OFhMTPq9= z4d(-{>V&wO!+M@QJ5kU0EQ%#cKlC~JEBVwo_Hh6COHzfzcIxe2EftYtYo{)qJBy=6 zK=H}Vs4OiGp37RNLC77D^|iUrj$40R=uu-Zcs~2Krv)ZefhozFjTG))ykL)t%?r{? z7T)<$F~KC7%o>SL5vb_KDDx$VGs$R_m}g@B4BlZpD0;q4Qv$bij(74n!D=qpzkJ#> z4bn+1!9MgW5ud^hDnuU9p?B{ICrpax13tsynj(&RT8e)dMo_$+sT(?LJ(LtvoODwj zNE{+glDZq8?P`eEx+;)u@6+6cDrY%Yabj{cwQ76^iJNoerAsZO%*UgwR*H;O%KS|P zf{sg8R_0GocK(ha92=#qb$Vq7R@FLMQs}I|K6fIEzude`uVuq4qke@K2G^G&KjrY2 zkH3@$5flD3nlGt+uy)$e>W3Qp+K;B%a!gd=5nYrG;f%YqZ!qnlX z-9i*jli-P4a=Kw^{#t2A3>2%csy`Ka-s*erjuL&jrB|1xw0ba3H+lO*`s& zha{u0#^j`aSfaSTrdIx#aOtlx2n5?o$U23s@cvvNS#ce{&;^N3uqgmF^$p>TBO_lO zZK0(|FeCRD830j`u4f4M;C#o(;XjbOVj>eUA|iUlOoO^b8*6A_&6Ti0JcK{rom6mn zR}AfR4*N)R@w%_VOqif=a&HmlLcK3y5U=O&_U@Mo;mRqZ+YL|SWxVfL6-l-~RDz81@|3u;$!g|j6OHCPK8ol(@3=p2Z z)EYcx$E!7J<9CQo9eQ(j=kb)l4Hkc2v<4jyEUd;+Y3D(CP(4mjN_9~0b?DON_Pm*B z8`W|vH%u?D%Nch(JUqEzNROU9%{lg+DWO~dWrPel4M-}eBaBJv+E&N^dhxIbDzj1GsRCk*GQ6Y@WqSc~?05Fwmt@qE+g>@J@` zFb7};qE=#`rBWE|gneOmINZLSAsix4gH~=;oQkS{fY>Id+pPV=t^HC$VddDro|riL z!~$i(v-5Wxp;S8yoeADKFmO?mxC0EEQtEDeURo`Qe>0Rjz$5O^D>YGABA-xPmrl2M z(b3>U0r3VlXTSfXCozE*<@OIZ$~$5irX-Qn&uIwTt`TXY!W4SIx?-K#mW0^ zP8>geBqpX6yE?!DGX{OW9D#O$BkwLQtm-NVCH zw#3rs8?D}b_>dsH?!kj+Y|m{`1)OX$^s5S9Vw_M0m{FA!;T*6GiH-+Pkpw^vKxSQ7 zcGM$SWE0#jVA0AvHKEBzeIInvKbmCtF&Kat7~%X3(`qnIQp!-Bn{^rPANKH#v_2k)RmkrbrBC zabD2rs`bZQKuQs=+t*M3OC@({&Xq5Id_@TJWk}pH*xpGi6u?!U2jH0LQ(O>-*Hsuv z9m(={s#^X!MqV00i19eTJ#&r8=&?JE-Z|h$KcA#SFn;{nj7fyIJohxva@A#CG1v161#`rA zpswB9EN&1|&df|NzMJG`e>C_%EmkJuK-`37Fz@#c^L5X!73Tnxv8;$`2-t7MCDod8 zgqc`aH0FlU&~Jj!H=d(vP3gnw$}PNOR437wqTD*QXSa1>X9J<81q z+(~_4wDC;}<@Eoa{_b)4R?*5m!^+<#fPF#Yh_YYoY(`?MpH-Fqw5W>OR-W?)d1E__ zJ2=&|!8#Q~RJ0V#pEGTvNGqVGw5VqOb;t3%26Y1peyKuV1=T<>97?rpaEYbggqkQ32szlymfahmYGh4` zgo#VN`NsH)k74O=7GAMoq?^coTt<&h+=voks#1yYD~UY4B;!^|Mnly&lyYKyXwaa+ z>|L+^vD5EHiZl3e&kSy1>FKHAU5pO}BoUbcGs^~~-P`-meKYY$L1US^U4_Ho)(PA1 zlQRu%w0KlRV2>n0V-r0Jzda^x_d|Owg5GBl?E3c}g?D%!of62zN<#0VHle>G1(tL+f>5PKre1nMWMGDcqRnPORs-V=+H@Vfk z>?%+=G~j+UD5Vk&9^f)!uiMg~pdj@;UWLI?^lCuklCS!!+4XRyW{dy%rHxENSoxDf z_^ea50jbC5e1k_~o3W)gO>yhP>CcCKR2e>1ubwf0&A<&gD6b6#SAxULoa5Z5r63J= z@7^`Eb}%-{uRDMKe3{n3t)Q?5%fTYlBeU2?;RVGdC?BQPic4v4mLX{zg8C3|M)cg7 z;FuMn2Cd}ibjr!bEmnlp{pOuJ|l&U(E5ZYHIM0XB(&iW!ax}C;U?A zqxTPUmuV8L>aaq&#=X3(l$ld0O-BC~9%tOnYOE}$?526vZnxcf;If^wAM6}ql7H^T zm6A{Wdi+>0an!N2kRsx+Fmq8p?{Oe>1xqt?pa0poNGqdr-Req6|K|s1AK!e*vXCJ% zV(wm6Op~1#Fk$F*o67IgMfdt>&eb)oz)oH+&@h?8N|2FTj6KVJ8JN7_1~iw;r)S2c z8;vO3hA&*$1)N!?!*V}LALeLiWaQ1ib7%-Lnw6C$zG}G#!Xch&h>S`<7;VXLB~9w} z0RM`_9kOSpMKo&I@EP+jh$P%!76#BRqNKuKmw(;wsoMny}b^dOb@@q zLspBR24Db@M9w${L)l&-wGhxic0mpBP#%$P&O274_)<`#i7;5;*+~RP-%xt*OiDh6 ztg{h}zw@?1GS5TAeqjs&DcF(N5Mxj{&?Bz9X%C=OAO$zD#`jM4@03}-rmU=N<(j4n zQ|M^X0;41{cqd+jw9%Rr#EYsYgeUe<=`e8Mti5eb@(1GIXd*dTr5-;w8aRY*Ne8k6 z8G1a#25(mzVG;>?sGsch{+AL@{Q^44L+gx9mXNUdj7&_Q!dW~jQRP?{4oqKULDn3S z18IKZ|2PTiEd4L=M`Mf8CJXLVAwHL~oCWXU!t0gk#^=%kEgNd5i?J|}B6k0%$PJGNQ^s(C(juOm2vQiIl^ zcWBNpk4fttqdpA7wX6IJR*}*tZrJG3-#0+P!0A+#eEFZ8R=)a(+ZwYEElo^%=|0MJ z3Fs7h;>2^Ls-GDu+Df7|@$GaRyW46Q)|cuvYDl>3wm&*$02FUTKOz~SO0(3jblLpL z0hi;vQNGG2gg(95!swY1K`9HxzGjK7=#6PrMR&jG1 zD}XFPZaB~f6A#IiYY_0gdSn+9?wR&*F59TQ(WDVE_)pOWZ?B~1(5qYk+I~TSfn1DF z3(EVYWuwPv@bj}iY+TscwSRioXTiVnHuZv1 z&d%7=BGmPtSz5Ex+nmDu-1~K*Mpb1cQ%NiH9}{3jfyOA>xVfq0&rg)gU_OYE!Kvl8 zG#e5$WwunNc#x`v6yK_}q~wTDrFU>t3lJ89Rau;!T+mS~qr=$UF9PeU zj4?CI|5mTpsNLFpj&;D`roB&J;{+Hbp;5GdQ^#Omh3BQ*Qxv>Z6JLZ+Y$AXf{YEP-ni@RJ#lSB4HlDSw(8Y)x^BA+)KF2cQQlsu~ALm3|?OL}Szo z1WqWC|FFwL){?9<#o^1&_G9^*ywBT@{;?m)FWj(zJf{#)Qh-4H+BL`g8TwkxG9k@U97yOt-zE(7W! z^Bv7Inq>irPuw25uF34Wm^NlGm00N84AeWjFvD%y(86T51F`Ybw}X77c(*>dpZ1Oe z91(nH<#tBV@-cE{HQ4FN5?#m`1F~yoS+W6W@_YFaXj-kxaWX$HnRC~%bqceOaysT3 zdTS`1SB_CLNS&$=%el8%%pr)u;6W=bCP{h+*>*t}oBL0ZWuqh_v4N2Jexbg1Qd)@N z&_>AII@j@!ZKJ7~$b3w6e)iE3WEh~bJNfZcP|m8EO>1%uo!Yb2%gPuVgwjm>5kooU zict1!X-C4=B<7?!MeDODZP$T*n!|2Wx2ee;&_2F{)}^$E54X@6oAv|)AsX-`3JGvk zTEX*1!wRLTA<>~7_}h;|#wHdg@R=wvAZkS!sxrLkX7|86$&DTK1SAD3OGd zc&Oa>R~3}hQ>kMpcUkZ&ZozfV6-BV}dMLQ{`oyMqV_)NdsUNy=3nb_k&HhK0z9`T7Y2bI9D)&L_|H^ zyN^ZGql*ebggd&rQ(scJ%)kQk>*#adk6dkl#W6sr5+H@p;DcbOb;Qk zOL2jDRa{P{BjdR@5|jZLsfBcny6TUcv$V7f^KthQJdkj!Q1(aF2Jn!~`QPc6^bjZo zvJZ~A&rd{Z4coNW0SLQp{ravZR$BY?;#>-U^#!SL!R9+sBD@{Vo&S6lR_{jxb`ha| z?t^iNa|SbnHlt(e1bt0#c(r&J6^LYGkd#FX5+N91cOsB00zXsS0!Y^kSX#oXJ`hl) zpk{v4^juAj5_a|1t3|cZ08*{{1Y+uCs{%5XQ~(w-x_RD(ig#__QXyf_2;2Ynax^m{ z6`*eR`Ho~>?qEsW{5c+p5&yktN+oH4VtDvE{J0N)EFn_{Uvw~8aymXl7;?&mZT)yv3`5Mmqdv0sIU+)MhjQv885EL5fP3I2`g!qcepJ_(6(Y?r>2BR~! zvkRjplI(vd`0n;Vwqm~!f65Q>#iAa4aT&du1ce{T7*uxt0FC@*gc_JW-$!I9M~~me zPwRfKhN{s)I(cP+E)rN&Jwca@47A7I5VsZ+Ns!Uqz5Eo9t9aBoneW~yCMt8jH{$QSK`DMz{nO{O8x$*F zz5Xrk`75}5+<4_(;o6!SZ|sKlm$Qt9mfuD&u}kfBHEc@vUzyRzLIUKw%%0sw(!U7s z!{IkU&&8EYc2XDp1Vk9zHKs8LxR3?T>&k;{t9ubC-2uE)+BQ;m9$$27L*;zh?yfyf z%gTz}%A4ydUacGWv*rquo#)B+5F;|Dr>g|hb4zdDw{M?M@`L7TYMUttB$-8`ND=Zr z4S6j<1H~DD`c@2xr)OW<88EJyYRmyzRm#Z`?F|hKq^|g=Fg?Yet1KQ?*~xD-y7-wh zy%DO!W)rY1xTFxmL^UtI?$!)O8c%L!j&l=5g1x|`l0g|cT7L&&wK9f*M6q_CpTJDu zjT=eSj#VsETA{1w7JnO+f0~>KwSd*+MV+2?VO#Bc>P(k(jt zQ}uwk5p%|zEZjI0Q-eiUnW zF0jZ=MvXej-jJ~mP-@d3pg{ZPZee1h)KRkp@m=m-a-*Sgf@grDh5WCgY1-Ja1Y;ugD*z zlt!=4WF`6ZP|lqEz4VzNF)<%Poe6MQ^`QV-ph);2oZLft zX!t7H;|3={*Q@*qr??o($R5Op`diF-H8}iBh|{9SUB`SOu1hHcQd73ai(>t*&wW<( z%h#o))%BVz{z8i#zxP1ks%Zww?U9ldctX(8l3u(2^ce8Nh;8@$!-q5buf>q`I%viF zYoj&SZl=bcoYvbI4vrn$Dmll+wvNzD7o#xffO}@q|S`!tBCQ}PF^^Sx$aE?fgj-K)=z~< zX3Ul%j2>#x2ctiz{PB=;6a2kBai<8LVF=i2{~!eNDp%@4-vx4!`X(G?_@vybkV%1b9TU;sOcdf;B`kx9;5*V^B z5_u~wEkdXMuP!akb>+YE8&qS4iJ;lT(EeYTD8GHXP0H#ispG~WrF}f!+&5Y>X;6Jt z|2uVy|LK&JUKBeQr2_ApRQz5UOnQ$3J(Ibxzf*c7V(J}nA80*crL?!b>YPC_ycOtP zAbio}2>B=*g@D3%Kwt@RQ_9sNPo3J7X1AcYo}#-1&C>Qh$jG=ONy(CU%kIMY+*r61 z5;uX3_}NkOgh=A~+qE(*Z+ zj8FM@!vBnO3%qZ-HjQgQJIcVCU*e#Wi#)5P%nf$8oe4QkPy}L%lZcqy#m5Cj^&2!j zMb##85A9`knTGF{l)mq!0Zl24qMsZNtFng%RDz;!Ll~dTU)otDV$=ib{i$%cab0a1 zsfI@gl?ssZVzB+Qheqd6WGtw$-PG>;qSz|a;uFUDu2fn`^xega1kh5S{k(T~mc5hH z)a|7nLm>7;7*gq4OO&`SG@z-KD{WcM98Z+{o3k7*&eUwv#+M|50VVs5(qbhW{`ZWc zpPx+fHNYAu2b%c4v?eP z|E!AXE%YoKWKs9aIt})`rxI}uSt`aR}A9sIL=UnimoVi{c z(BJqg=M9}BNCa57n>`)&;3Xp(lc0E_B`_IP!+bIIc6PNOeP8TPia&#c9g8b*!GDw4 zy;RjYon0Dn)JZ75QurxS=O-pwlb>QGtoW9??(vn;%)*d~U4wI-RFO>_JNEax<4P6X z$owu8+Q82E#}`H~L2@F2zQG^A&LJILye#$N4y8vm^ z2??o9%+;lDp1~{t!N=kb!HyY|N zb3)VKta7BLs`?6UuY^e3_5T+ZdcJ+YujT)!rpJ*mC$)>|$047VwN~-o@V4DGbjAtD z_|*bjBwmU2ud*IFkE~ZZy3ZaQd;mVrnX|hd`NqwD0S@Fs2!0_MhB6J&lhR071LS?Ip56=DYyDy_`jH+& zMmhfpfXWm{BQ2M(2)I~;QUlD--}0Qkz6<6W#B_TSiL50%=cY&y%uJ6*pdB(j<_C?$ zoiQI_wa*8{tw+<}pgzl=wP;aSwrX4Ty-9D(oimxDH21T9PNGEi`4vd`7(A_JORp05#rj5g@VW5C$K}^%&Ti-kzp3tC{s+B z#XGL?bXhSHqe<#>d!#rb8}j&rm_P1;{*+AWrCcn+U= zFKz9bxFQQK%**e~RSb)8DvWs_#RD1p(}0Kld$J?`#mI5(SJVHCzZwuJ$j#EmmkHF| zC{vAE_<6l{BG`&C3bQ5^(~T>QUN>eB%VA%`mjahN^)pO#kkk$|MjU?!T{TH=h>K%m zyzjbme_Vs&MaPEUyd}7j{zts{=YZ{YMf+?WnaY^~7 z)m3T)#!tQ9!m$gj>xUwZ z$={88_wKE7jWCZim=(g=-AU0>4Ia3>e&rYE$}bkp6Ysy6tKVAUBNMb$#fgkxNrpiF zm9lxb5Y0p~Kp-i4`Dh`+@$P-~>qhg*-W91F;fQgL**$V0x?hYY#S3avu!wOOGE#JN zNN+((f<6F6ZKYoANc9NtBzHm3^QNjXZ1uV{2lJOLvy4Db-)YdG*~Brw?b~_>o>#(e z4qBZ;%|bl|@cojbZseFTUJ6xrkV4r4t@b5_BjtGf0CUdEwd+iLvq}n@v~PbU)5q?w z@Z4yXut&>!_U_$;;M`qZ<59*CoKhQc?3r=x6+>?LW_Ad<{twM0i zMhLR*jr<*gaL0;8kK#0FIX^^b-YK44AxO>|ExA!V#mlbCZtX1nrC5aW-RoK0W*QM8 zVlXr-G8Qp184bmiM_a7660OlXPT{FF=1}kCPDhb$04OaHx%xv>@17SN6qAgbO|3d|XZz z#W@zVu90}55Z#nln&hw{|`F?cYu>%b+_K`~6&^HJe&QIh=^$j0qpY zse{rT(lcBt41O^LT8b?R(h0l0YACu1zLG91y{NDJRSyRMA*EiZ&e-t}56@TzWg$5M zYm61tZDfwf{Bf1YURT9nV3*P=11L_hHH8XECVZpuLoH{(u;gs*cY>zTU zd=9Ji9ufZ-z_oOB8vsfMd+qQ_N`@n3%RGmZP-^GklBM>yLEX%4!U|GR`FAdnrQ;&~ z*6NLp0FSo|3`okVTJ&AIJbU@_#E8~fulkZp+cJA~T+^Nl=^0zLZaw({z@N@c6rnOq z(eA}c;9qzl>>d;LnwdENL+cFzVY6w*a2jWV{(-{E+l)sm~xCn|Zti>m_$5xQK&CECp% zAtrs|%Sv6X$$N=&5-K1kCnBs)ojQ3;CnA92`>Qnp9!j}1aMJRPO=B)Yq6yVxkn*^H z>qb;K+mQT)?{QoKu(U6AUNsejVS*m8toljFEIqE;1?pC8zjH)&itfnqYGsY}0J)8s zw+`?oy6IREH2zq@RPbj}=3E70@F198@h7$>2-Z`3UfU2_vy{_JZtgj~aeUx_2g0*{ zHP)opF-8(|h+PQRB)*SIz|Aw5qEu>KUSCtw^5R|YN=bGDJP9wN$mBGff`$?`Quc%B zbLY;IUHUP)gO;KeL%s;@5rN)IM>^sYG06P>XJ&g(q z%@*{0A6S*o@7%N3we`Ba_4~{`A&CyNjyIguXj*xj2Ro~A>NDmuPuT|o5W%4AN+d5p zb8U%g1)g+7QWGME;hMtiAvn>31+E?cr!OEd7^LsF;^=S zt#IpM3ssD;#U$;^DJ9W|%2pFJklA#SPyQDb5COO{D?DZzFe#r2mCN5G&xT#Lq5i)~ zY+-fdrbp83)OP&CeK6I`B!6&7yvBf|M*jbl6i(%Bi(l$QHF*7h2NY=^m7EF$QkHj2 z^aC)ZBn&_^@-U-GTr8<_G%lmtH)+`L2k;C^{M#Ogpi_t=~_< z+c)lb>1{VkdSU|nhxzp!P>*2{OkC!EdhFCb2G0Oqn29bswfuF-RYdJP^#Ogr!HPi| zs%cixM-0oHF$u6|gt764fg39P&=8_AVG!xOD6C@9s3hKJM%D22^c4L>$wv{WlH>!t zZ@QM^lzSpLeLl#_uL!q(*6}ExC6ogwyZw}|TCO7{<%$-^=`Fk;UBNB zAA>8y$_FkiEPgt6ySK~TXQz+!_3E&t!zQ1;Uh6v)&q)7X@yw;aug;=psIyX+mrgAm z>Y)DQ_?B509>wO=L7v2Nz{Kr2H2TZQ@9zVnoz~HZQDn>$3c}EB?{E0k`toC4 zWmg{8ulJ7DV>?=heK_2#-4?H;Kh88L+PH62&w;g7L=Ggt<5!p(<{G!cGp3V)flCZf z6u2f)bLpMEl($C3g;#`vTe{&T!Z08e?}ar8fZfsc$h#`w>0(L>n7w#nN*dZh$3g7Xro| zU6leUJoi%e(P^=_jaq1>S?Z|6=S$J`_<>sF)27vS#y7yzG_jdh1`|4u$>40`V`+8A z6)T<>N`RbFRpQ`)?c5h6VP1?^FkX~N+qtT0bw1kU0@RtfK;H@C=mwT@WsZK=g5gJcj}*5LgrS5{3p+?@`P_fl2@ zB}}Z%lcV8LhYxFt@bl7ug-e{Br<09$?1;8W{#)vKU?4RaS7VUlCNEn#v$AaB)#KAn z3HC_4RaB|#I-c|*pW@vrOlY~aF4ZLe%lj3B2=zL4*|438pwCDrwFA&}kc1^6Er9Hy z<+zm_f>)q8vSK`diRNv@9JCbJ?{+D!qR#>9h$||^riB1&6ax_9K7$D{SLVx8jB(XF z_p;@7Y~Zh54`~y-etG6Jv}p8})xsGIwRX(znn5YG;r)FRUVQpWVCWa*ABuG*mk=t` zYaU5|lmDupL*&WS2&3#SpASuvy`lX2>IFVD)~?e%)<=_{N6N@38`@LJ#FOE)cvwsOExOkM zNDYpC#@W01*8Dhe(31&o9zm{#AQHEc@nx90Y&tpa#0?;frTIy!q1lm)J_Ye$Uf_xi z0je>FB&~>Y#AnnpDm$Tm&CIr6Tkvu8sfSu1u&}L?naeXnMe+75ra|MWQ`Kc6GbILG zatpG8syCCug0cl#0$+U91Qv61_5D6$uI$>?6m6+Qox6VEZ%pgh+vaM@(2%GsN4SVY zn)nv_y8gQTS~e7=V&ar3S5}OyIiR9f%Z3qd_YNhgGj~SkcEr+Ni^-QmH?gu<&#N>{gE>J4F9XMhHA3Bopu2z?0+=Vb=?hHf+ z{^{OFHLU@%LdfpVm*e8vanx3C?%jE8GXXqfpZ3ZSf8WO^-TSDg{ixrtp{0KHLr*WK z1Pt*gNjmy%B6yAr+8&i`qP<<2W+CN?;v@x_#5uO?&cXc7Cg@qfD_`~9Hufy3eg85lsD&R^D%(+6KoDb9!+ zb?Tw(p`OZnLH^cL2k0AN!B$kH%mA^h=y<<{>&JeoX{Rnv{#~+Z&Pw~0GdHAX=_A_{ zks1?jpHUoR0|ig@oVD3K_TJ(a7iR5{EYT}htRkL*!Geyu%}~+B#R*<62Jn}G=#~K@ zL`Kw+F>Tjf16Ge^m?9+^phedVaSguwTV1x-+;cI}NIO}q1N2X=dP5I`xZ;K4f{6;+ z2F`gu5TlqCxGC2BEX^Qarg}h3SAIM`Z+4OfkfC^-iB)g+g>XM)9FqS7?}I*g9s=sR z)i_)P?n1sic!`tKmMq7G(GH(iNv#K~#`Uzmh3_M4*0oHI<=_yGg zkJ(OFT$Nln?R3*UTMKS5aY@Q{_j!Mi+rwusqg0pnP5mP}i};xnVO)YP5!GWhBfFLf zmw5y|vg!PshOXrHV_uq$5W|C495d|+9fSYk^C~;LF;*>G^hIGJYt~+8Z+(%M64iZ# z@ec=@zz|e(E7Ur{=71qZWI;o=l@$y_p+BfYO>T zS&Q!5AB@~F|5ElOK6vDA8P;uVm{iD07+|y6%TBdw>fW-;6^&I`X)-ZsJ#X~I*xfbp zH1k(<*6CGi5OS2`ZsXCV4O3)K47Vg*hdQFMpL6jK^T?tH<+XnIE7+R;e7a;bv47}E z+dk^(KdR2$brouFJKw2w0uXLa#G~KwTy=(xRU%z!tFv1c?C6?z%sI#3E9s_V!{u>X z_Dreb^8COomEryg2}bWFdt8DF19b!k2S54x&Ze^dL5oWr4kX8gS?b7ORf*Lnt<)+q zENp$6oynRtSFSB{TST?WVeV?TwBZ%I8LMyrinD`QhUU9=CM}1Kl#ptfu59&=(UaT> z6il_cxHqE}b5)<&)-f@u&aG{HVsi4jXza!9 zhj{S$*1VqVb&ErnFo8ft(JInS*+VaH%i+cXqeu_iy~C8^zsG@OG~LYw>M@(N5be?W<^kG+Zr@^5)RW zA0qFGEy9iM7(2Kz;Dh9ne>|+C~=e>Ee7=m){H`mnJHAO3WV@m^{ zK#C_);Iz7tutS2a<#w3*TAf|mt%z+0t+dQ;w-w2%%u%X0&eWtP$=caRuSane1GTi< z`7S9s%h6z4>QQ_*wl|u_)C)@4YUBA$qLLeDb6RhMEWC0qJW7|#W6E~O*2Zzxy}LA2 zjX6(KIGD7)N#noVycjg5lTps!@zq!l;9uB+RZ^)1xN`l?MK!dhx zqu^PMcs^7P*A_;1f6JS+zTdwgm6t}E<0p-B;gaBvx;?tf0a`F!%rV9jn^jiMB}o>_ZFq+O6Jxoqq4-49BEUy z?0R1hLn^1tOU|0a$>EvY0Qei@Qfwx28U{-+#E*cIP1MSl?0*b5IX{DQ@ur==PV2`K z{f}hNs+(2|e@rVpj3nQs_?tl(-#?9Qam_dH5FUV}^0F^q>XW;EQigU8?%%(2w|f2# z2Y$wag2}(WdsA4r((dC6os?=w_GIv;>83Maykh4?;-$WSd)=0rnxnxA6t`1LeMb`M zAR9y+g?cPFs=5u~gQ>!Sxh$nWj#^{XqGiX7XMnf-nsNd;@fdzT3>m<^6LiYd31Hvx znT0QYnq9Wc>+aFTs{wgKTWxoC{G^zES#pf3Dp2o$CGVvK)Q)i>&rB zyFVWaOX!nJ4!GHb33*c3LzB0?__CltBIA~8s0 zZMFSL{dhP+F~Cc2%W_`f(k5sA=}0YRXamM}kKk6-n>G ze^TpP1*{EQRt%6Y9v7Mc@k^9*YS(7|+2V&J38vj!-rY#rBn2~m03^L+NM&f~tGzaE z^p5O|TNGKZ1~W(?Mg#RMrQnf^7az^brUrpe)nRA!G~mqmy)An6UsSr!FZ$Ky`=>Cg zBK}|uQbJz#;B`+O@Qxq>%ueat;lEAs3Qm;gAvyX51qFtOeEoLrynXw;rPYOK$^J5n z_&uNdNsU1hIRel;u1*h|I%cNraQDLepp{*;J~vGs-|&9eoZ>D?G}H+j=dj}$-7XjS z&8vW1ET~(`*4sf${rFfQ2^;J_@FW!qQII-OIRDpkRD5oMTwueS0{Y6@LKQK|FEpDO zMH0{;89#=m0i&mcTZ&m%_EHRUbzGzh%ZeU7_gq3}iV9L`M}X~0n*jlX<#WJ$aG~`O zVKNtCBpe4;BXWRhYV+c)IDPCK90C$~)n~ImOk{M>2-O%B&>>_AOhOo;WJ};SKp`79 z$ocZ+ORr_jn>Q%G>}hX5P8xBJtB}x8_5PlKRy>TYzmqMJV=OGi+A85c>>MCc#SUY=g!2_VvX-M_bcSDvUwt5{ZJDYE%A`9Xd4h`$X;$!J^W1GTYDIz9^e_}4v=kJ63>u^`lJHvsH zRqYx4?T5K?TNnhHm-xLl09HXnQQ{HzqR8n`{Kx*iHDf;3leJy-6?czFAl!D;Q)=%b z2F%Cd2ObJeDFP>zKA)Ba3p=uK>C4}+)%Wv?8p_JbfA#pr{f{VCHv1=NB?x_qrOMX2=rFVei~s4wte~$gRgAY4kJ~$CU)i%H}ZdfDd4&bk-Cn#;yV7$w^7W`RH4> zY4h?iH?&BH5ZBM4(GaH_tvV9UFMG=Y&pU`t?4sk$YYR*w7NKg8$`E^cgQlE zygf7|q$UBt?fRuIieJ#@^2eCfiY<=>y&`zE?zk7h)E&h6tC@eq$wb7`4!F8{W}^;8 zDD%^I?4TQLPT#ScK`hkzN~eX8{EUCA{bz&IOoGsBj6IRM^9Z>HytktlFHUREzQU62 z_kFd@z)-gClf%!VzVrL;Jp7O@(6b(6(U38HNm_`0K5u}_i<;VRa{DDs)lvUam@6^M z887fqyj7LzEnO}mRSaRkB2jH_#6)-NzAKBiFI znAg}6j|Xqa#O>yiOL?gN8t+If!y8fPwv=L3o;XL_w&HEQvE|R@8l$uh%IHBn4ThZ; ziJI${eQkIFK8_wPIgj4nD$YaQj|%x^hL4xma5Pnja@;F|y^=;|IvNBIUE3N-htKyr zF%XzzH}0mB=6|Eq;K6-!DJV9482oV2lF&;`xWQ@xVwInGAa&8B;Di=4A;7`lA9R=AZT?Yg+4u%+~$iy6B( z@W7J*isY_`*KFh{B_rTvb^6-qOCrWwRX_A{aql#ZEV+$ke4(;^yzMP0~@Juupq*f2v8B;0>6W+)*Ay8 zVgwV|6b6`q3NeomLmf-&)<@mnfuTgETs%vjT<(~`#k~!oA@ga~fW1hS$YD2ZLY|>I z$=hn@^&?k~E?6<8nbrg%BwkJo`WZVSKY5{Nmul-t*)$$&EhvpZu|I~|P(c$15Od8k zKN(#GLKt<-ajT3M+YJSXVCveq_E^S_ojKD;E}SXuf1(?w0PJSZQOi9?L2qPOzUJeD z2Wrr@oO~U-rQMKz;qIX7>*ig?5K5IRVF|oWe`rqW8S=uk*ihx z`@mMA?FoAksHNig_5$GMlPW?1 z`dmuNMyfG=xm;QTc@_2hR6Q#D0|OTZl9nBD`BKpdstVB-dBj-{AE@|7O2kfzEYqH> zr_QfdHdT%D`D{Cjvrce^gswZL4qHHEUF2q_-zZdtUq+x|pdYejl zdKJb-nppbl*HmNVsN1TvZVg82$b3hZV7N-NN)LM>xtB3@+!Ue@v9SrLy4dZyvsMvFWb?v1R+zKc+OeY+|Bv?q;0n z=vBvfKBmWxu-t(=udZ;p!Ae3oo~JaYE{s;T?PLftSG`Z^v~~qL&<>M+VjKa0pzx&rZ%BPktWV+_4ktQED@Q%x*=J^ z5C}_v-0t1$akz{C=0!{U{_J5M3-fT{wvcBL4pOju(gZE73Hz9sb2&-@#fMn%*@%(6 z{^)3q^AmPjIl!@C=g^|IO)om;~|p+%8p zNWor17mLaN!`GXD<(z-<-^s4Al$52hB)jY(l2G=2mxP#VB&moLS*OKXcBRO^g{dT3 zY%M5hEQtuE>`5rvp4a)N8NdJkd7iGh%v{EDci-R7=bZCC?*(C+XlX)7W>w+Nm4xgo z^RrLLQAk@ym6z+dZE&O(Id34e5@}Dlt>=4pZ^$L&~B|GU|{ulF}TR*JPPAD<`;=rsf+J_-S#P{quY&jET zA_wL>ABv8i)xCGq&1?M{_;$kg4Uzl!`!LH~10q!*a!F{w5*L?#3qifbUJ>4S`<6{o zOVH=8i?i}g07Bu@Qe~U8@}f9^Uv?QYK@EN?s6g%qH&TH_#f{^XT0A zBxnQ(v-*o)b7XH^2=Lo_K9G?-W7VKwe5p_s1c!xv*;!0=6b7;wbTYceu72ZXF^W5$ z^MSs!Ynr5yAOwsE>^m}V7>fmfWuA}aoFMkSsgE8(Jkj}P5A_e^PC8niWmqbXj$_Dy z6pMkrt0V}8eIVtei&8_reM9@Y#p*>qJU=`V(97sB@jFFh^k>RU(yO@VgP#cRJeP*% z>#kl&i7+jf{8ORIhQr|)^rxi*+ZqgUt|0j1(fH6*%zD&Dn+2YfZi}bx)3==D=#Ad& zKQSz1>)Aey7F014z>H!}kCbkPhP7o5VA18+Ts87E<3U_(I^#NSJw9q-Ca`=(!)bN~ zh7{XAj*PdugaQSS8YevHSB+$30sy zTJb3ku3r{y@T%a3!3pBiJQ(38Ag~HOvfX)CHFfl>faC5%X3J=3i zFE8b>2R+C2XxnSw1}~QHU2Y(qq9HRCes2f~LcDf$o&&n9QaH~XS&9eLu?a>Vv+&%z zz978cnOJvhP+VU*Y8d{rARH(H=+`G9!j%Fu_0%i(7XKBI;Og^nfx(A|BEp>z<1(9< z%2Gkj46bu|?qh?beq{U`ztj87YZZmr8`gKer(1z)7j@UL_q7UbO^&ferC$`oqHx$5 zxHx>+yzz#V5f&keTIk)IH&sr3wHtoL?_mQ7unKDO7T9WLHlP zE5xojJ@)EoeXF(_lC2~;GlEO#4);~$9v$^m9X>mgIdcxYEILn70T5H2q#n5?DQPqt z3@d!jSX#Q&b#!#}DTR2i)vu}V?H-MP^En@S)W|f@+GSohh$2!}Qr>5^M>_%X#PJ{* zs5Qerbi*BXkne{wQ$x16QN8t6@cdNVlq9V|ZRC)GKe>~T?)G56%)-S69!-6oP|bja z`KL~tm|!t`3J?WGXje>4&%!9fwkX??aZYc9`TYWG7RWmL{sHbqr7~u`uC623YlX_T zU|)(=9~vC4oc2x6gRsl$j*YzPc6Hxl`zmOPxt}5+9KR$VBOuSz;g4^zL0n4R2YFd| z@fe?tEtAw6PkT|?U0&8T%)D^+QGC(bvGw>oom;3!kBM`*y{Gg{sjez+ijaNc*5|9- zyLYb$FKS0De?09eAz(D}IntkQ+k)W%UL}q}TWPF$x~qZF{#Z;4dX2eMGo1Ik@#c8L zQeyxp@Nz*Y?zy$C0?b%N>U_SAy8cL(OiVk6hYUb{#??A0K+Rv8dPIX@WY5j9-)Mdd zQvnRDOoSsALJ%7EHw9&NSBTTWFSR5Dv@-yNT?4ZdUNRqWGaJ{)4$d)LW)8 z$jasjNF;ftwbN|gPs#Vf*$h{u>>q4FSqi8(Yu2eth7N~B6p>q64D6Cd6OU4%TaO;P z%5Jd&@N$^Yr%me>`)TR`LU3T45rV{HE*ebHHeHz?*bXYB2aHK^zdjn3)!J2Oz3auId$ z`D>bKi(7NLwU58E<%73T^83tn>9Orh15Z@Bu z)l$OmttHv;24ZpvsBPK4ZgM-!0Zz{dtpUoWcxuU6*|Q*@EryTtA1Y+&uBYkXGJdwu zRPay?oj^u*+jbRaV6QlOk& z#x-GB_ig7KQ3-QhdUQh31bXGHr$2DH*k|m3<|_;Ca5hFyy%;f5|8E@zClIHXyj{{~ z8}=%8iT??5N%p&~(Gubkn3ToWv;YX$|LPpBv6`P|mw%lN`07+DSXJxpRDrzXuKstamlH|f8wr@xJTno zQ!egn;x1Wq`RhNV3W|LfD=YRbRhfpB#pE7%3M|>D^_umOm1bh1*OM074U=~=`ya%~ z0+2|AgO=yFvTMbnmm;jyuetB-y-!=(gM|9&<@eNb4cnU=QY9g?q3*bc=pY(%!>f(_ z>;lv%+*Y*9v&4L#sE0;yT~xRGwj;}9>Ly^_^y4ekM~xjjlqd$HkFQ@#_46-FK7_n$ z)0U38{xHpJj~w#FMWBVojgQ+dN&_ioo~bk1z~j>8%fB}b4nh0?GDj^ruOPzH3@%wL z_gRkt zU-UZiXJ2Tft=$@Dev^~m@5v!4aPFdRIBD9nnj`$O&sPwdtG6`|pbUa-i1;$jBX2j; zib!3Xzq=Q_3>Ts}8%vva=mbWS%I z-YR&)0#G=A9^ma|k$n4I+dCn%QY@ZiwV_N?Zg4@up3kP0F4hQuko&H|-OpY8f#3Ul zoX3xbFq&Gm-)j^4b^-g>waJB^^UG-d_|)4_yRI!rILNC) zWvl!OuHKWj<>$6=tXqSmY#e~@-=8dwO1DdQDP<$+X=T}X_m59K6Q!}*ILzWe^HEW~ zmK)oSOYb zmz7VOo{G&(}`f}|uNmKJU$McsMNiriXlBQX%28oLm-?5o-Ia_1Vp@59y?gFa!HV&o$2 z7D6@IlsGv+dBSUsze0(_;xm$d3%5{D3R)JQpwk)fVVV_of4l8~g(#y9)O{0=BrS@f zd7^BC99Cwb3&^t=(AYHo-q&isarSk6*m}IpTtGU4Bz#Za};0? z^RCZN@m_WQ`mTchHgDQg!*5r8k{8gBVg{J=@09CW$35*H!g4{w=tb1^?X#L0a5)kYOtn zOMu3MX_qrst`Av=e+(;rQ#4=Kdvn=K(DL+%Mf*X2)$E9?!Nt_)F?7z{KPKSYxxtA9 zEYB26sP6l=r_2V>l4e_W6L!Bx9iDhXV15*}K>Y5X3L$(o@dh^ zF}EA}UapMIN;_PM`fAk&Bu`rGQ1YyM+kSP_Q`g=!fBRuqa{cXoen0b)RJv-tcfjY* z+B#=u^eVZ^*!;P@-|{Jz@3FQ-l$iEGNbFO`J#9dUbw}xl&nXVMT??Z4E1jJ)5{E;ek!JPuYs=v&kRd)VvmdGo@^rMFEE7Lr}zMHx8=FNLFvNsDK69 z>ch!ZT>5zjw&~r}&utJ*PDb>QhNn2Wt+7U+Zsb#3G_&BtA|kU5-&oK)nTI8yuj943 zR}mw8JOAhJ>iT_uuAfhw7%@R;fw-F8jJ*Bjjx<bpalL8VO9)mWKn1s7{8%t@q|csi8z+;yx2p8(G*8#&D&j24 zsyF3@#c!!rOQnDq_T}p5#j3GO`7$?<+znc&jtkHk5a|1^=E_+BzVJm39X@>Hh4cGS zaEQ1XT{cEKLg#&pc8&w+_Q9CmSMIcwlw2F^^i@?hZQYuMP5sh~!_w#z;LjX+(qhyo zg{=D@6Kly-|Huzx&f=f0Fm@|xCcg)!oQ^(ee7U!?|IkA(3|9Sn7@)P!fUwQPC&GX= zYu2QFME?pga}x_hlyF_f50+z)k{wbB?HcIc+1XWTtcLG7+@R}@!qIaQ7kWtXSkz_s z*`ZcmyI(iXiz6kO)di@=Zaa(~Xj&Xf4Y<2M0k%w@Hcjb|WJZ>lT$VJsy?RCtVI zdJd0wx7CDlTJ9^hz3@{RI_FaTX-o^u`Er~7ocHT0tk<8~>X;vIJAs*P^ctzR1KBfZ zSVd?Sm?!r%Y!wrWZhk-8MO?{6eh2!Cu44bkCz#Sg??J^V1Yhp*fQ+H0U+}Hh&usm_0ypRp`jY^qAd7DomjbGQ<@Do{%Pe)2&CF7T9tW1wc%H+OrE z_zr_ABf!ppMM!LJ@L8TiXL6@Q>tM4Kc8ek@c3pG_=cIa&ST=q$YJsd)(x%*tKSb0^ zcmTt#s353()46l!s^6ST9X~&vN+HYnt#=mmjEGLQ1Ju|SG_N~!s|u`&AANEw`0UB6 z>JAL5T&v5!lLulPjf)(m0>~s~ZrQViP|zjRlMf6cyjJ@o8p_n+m|Hn@?5kX^r@joH zaN?Zn-ukQlY|#)=i0w1`898d3n`Nr(=XCKjyJoOQ!{&I|f|JwOkhAVGTJdFg{!(FL zAyE8W!_n#ahYv!^dLCO|9NTE~cphADAd|;uwD0t4{2O;kr&S0QBER9E3FJsuB4#CD zc{7xw_5Jqj^<$!1Yjo(|U7ZIC>cgS9Rn6aM7Ga$G>5c6IH`B9cGuDp%d_`>sW!@jy1R z{Noy5001Z2qu{o(FXJk_HRsaD? zrio&2D~J(V^l?*ha^&aDut0cDE{JTq4@}PfXgzzQfV>UY4_Bd=s}S+^UpNzu8+2ds z$gXlzYv`{8EhK%y3O(MYUAuyQsXDLUzP$!QkA~~@Ibjq_9|lBRO|P}av9Peosl8?r zoY1GwG(fN5`je(i*#MSkW)BFCbb7_vvwiyZ^+LSWFRA84@DfLm?FR`5sB z$;VILzgw8B;WYNh#6e2*4iftgmDR(oS?(fMHUEA^c14lTqs*1w|8)00{#?Gmo#gFp zffb=Y{@OG#ZTqbE@!3yKuSNZ78+dp5<_vc?e!pR6O56N}etlMJq`nNlKXYC9)fK7T zjfc~WRpLspT3Cc;{;>P$$s%h&tNb-T`IwB#&inRic}gS;qylB-S2=aiD_;6~t@oHKfL))!~CY}(UiWH>Jx|b*{fT}kR4)LsZthJ4( zv)l?7X=-T&qhCg|Q?G938tu&o#}Tw4>fQeRoADRjefP9@59U3_1Z&C#?sIetk1@vC z(N`v$hL{~n6;2Bvm*b_CXuVM|)~Z!&`g7X21fEH&^NI6p65g5y&WESf{oCiy8nQ@G z6d@lUE}4T{I8bF33684<3i#KIl)c#3@K~`zFPX7&}&LhqWW1 ziYU_f2H(3}zW$95EDMI$%kajZ`_lhcWC!ouS)F61EJBjNPJX}h3im4U!pJKLXGbk# zJ?Hk(!aHdpGoZ1f--s{hXBzezW+d#Tmkd436RTdW;&U6y34wDT$tuiixF(vXpKSIe zQ=2tuvW6YW#oUpn9$W(w8d&;e`*pk5(&fTW6qe$We&4?*DZ_YZ1jNOVQDnmW_f-cU zv>_Piv7e;$%)ab^yS?nnY0h{1pG~1wWY^aESFk&vQvG3@KjFmW#MwC;v$hU3CF5mZ zpX#?)J}ry4uh4!xr-@MPpy(ZmK5>0_HjDZILm?~Cx3!O1E&1NzWG#GjZths;drIyW zpRinQA9IrmN=2>_4r`>s@@D ze9kmjISnjM*7`*OfNX~IfC|qd-wM0|Y&WQBYmVV&G!Q06 znN90`ooJ4IQl^7{soqP9#)C!1je|3afLPh5c5UBC-Ss4_VhNP1_mxJLmBJtsUGve_ zv7Qh`ylW2*TMc3^5y^|o8mh=UYGLskI8Mv-Yi~Y&bg?~o(>d0(mp{9xP23Hxr$1E@ zz?StnMnnA8b^u&|=yRH)ld7={8?}M9iK%+fNT8 zvMdR8`JUR7r*jS{3!=f214*W)9}hsYVG#(cB)1c@QT|mFS*{+&AXG3B6>e_&n=JnY z4v&|8%Lpt44Eg<|!Hi9nERo-ZDB^DYVanHZ1Dhv(BmDkbisKaLkMrcDNe!8oAoAot zUdek1m>@*~SDD?>f#TWlKac}b_F5-2hOM}6pN4Oa4GD`KyJVB0 zD7<;hZe=m%@jyS{ul-^6-}7rL62X{a;<0ajW%UgCvKk&b0ij7@eMRr_G2HaQnh=N2^I&q@k>M&EQ zMw*($s#F2Va3aFpx#-8cvyd9AwLR2_jA%l( zR@DWQ8X)UE)PS<5gVF}P<1_#mokkW$-}-wWK%tOlB�d=h4k}52Ssf3>YMp{l2HP z|KMz%%faKuZ4HJX(yD!O@l5=Xsim3&N1-5wFp*H;-7D9+&+A!8Jj(aGHsGEFAXidU zqobY;YY}qv=+M+^=B(8~NayGV7d8|tD;U(|T{fT>vQkabukQP9 z@6&DWthYyRb7eh%JT>c6PkXk@$?v^vuMWT^0Iyf4<9Vq08CO*`Qmtf(K`NFIGgxA1 zB{HyJBVH2_*KpyJn3X~h9$@8rozoWA7d9vuoU#vZ+N8-<_=%d~Foe3#)mj6dT!Q*` zY;Mmp)6sU4%#}7}_W!8{90Yw3&dkB7 zrv~bE!4U8r9T;1t-;EF5(E|T3rz=`NHlpvtE3I_RhqJ!34wa;J%JJ5oL9vUT_*>2k zcPforR>36vAMI;=vC6N{ed4UT_38!VrqFi#h}7Fx_fkG7uE@i{QZlRt8TR4*I%@S&I zeoykAn}SYAm+N)W^cn(;Ll7dwsO5hYPlqCoJXj!XDz=frjAEJ?tF3}M_#%-=oOinS z^^glCPuau@Y0L{G5;SKW%NAWg$$>M0aB;Wuix}>o!)~hg1O`oYJ)U;a`<)N{%b5PB zLhbh(e4@xLj53N|K`#A&q68rTIusU*lRb~Dgn8jedu4vuBjB4CEXsV$6=JR4NHiRb z_pcoB#1i-g%2KV-ugZ`fBAe3;zj4F#PP>*7A~wY4AY19_na(*`e_pt`BR!N^ zGDI?k9_W>5ki04r;W_^F&H1E^0=2%(atT5u|TsD`|Sk$kN#1qqUt& z0LG2T28WMs99*3MbZ7VS(!$0>zuy<*l8;X}Vl+AErk3Te;{pz4$1Y})OcI;6(j`!= z>p`j^K`(uD6h<&|e~R)DvZ99%%WleIS92b;_*FS=bR!Iq3~=?eUjU8q5ls)^BW_wO zB&CnJH0#XNc?=PPB?XL!=q>-fuos3S7 zVe$UYJ865Ap65St2FVP;Gg3DHdH`Mz0f`pu5yOe)MW0^S6~ZbIxtTz#c!jt)e-T2I zR%qP)%5h-h)Fmq1V645PrWU=qAj4kOGx)^r{{*yGG37R$uRE8ULsI3$DSav|W;#-_%9+z+ zg*JUJEMJ+kuc>3oFz(>x+9?78%`nP$OF zVEollQxgZdehX38G}`8s_%+GW)`Js(QX`)x1b$R-ytSt3mV_~o(Kw-N|8d`ggQt*( z872Mf;5E_hRSh#AyYXnGtclI4E`hCrzug+|;p_R8te$F0>`=@=Up zU@XMFyRz}lk=2RQ;%6ftfcV2)H+^^u5L#Av0mdjWYiex1x4DH@4gJ9gYCe(@|7;-^@PDD`zFu$lP&&`ngi>1DzD z?>Qt?xRrrh+v!H=DJ7~Z)}zOvM-^{Qn81Pq98D;?N!GdV{(NU_QDx??Vy`XTbHSn(f6Op+ebE(M znQ0tBM08qOC$??E>ZD(fWWT#nOb+iJ90GUP+0S2VEAN;1UTZyYXT>Iy1O5+dZW<>M zEwvbyfa2UOvn}8e=&hI#QZ};fdG(6q@AK|hlVe$?>YsIJU|{B56Fm!B* zdp12wxIL7ej=Bs(Le_luZhx1O`;Q;{kQO2?7A*OQBh%Lg5!kwY+cvqs60^ASjaL59 z1ZHcn$0zYCHbwK;`6xReWa7YiMNNE+p1nV+$7YWj;Z4w>pp*u^Kekd;pe3NIh-gb0 z=ciBi%8Mx;8^Vw*B1(neG(T()OP@~hw5IL@^G#SrV18tRIlG0dBNzx%{)lbI$5}q0 z${aXpM%qIcB|}6Y9!5kOYNHG0LI7M0FTl*6|1HBkd+HFoZ>qCc&NII*TDb|%p*O0^ z{uWO%p){40mR3Ikpwx>Cnpbn@g?R##$$9q85d)5yh1e*(!CJKzzbkYKp7tkM?x0~h z`))8YyeLQ+Sa>n#BVFO1z`&{Zh1s>fmG15@**6<<5G*Q0vGH}8Y-v4Vk3V=*S`|fv zB+lUaQtfbSCw+SUNt+;<80M})_Kk8`j#xO*J{ME$x23Qt5*Tz~0X<*hMv5cia&In; zPf;z^^%pSdpge~hw^lE@w9YJ%{ULfaYTG(E4v%^y7t}XY7jep*kcOJpV%>o>h~X8m z>ev=G4j%VndfM?8s|a080ci6&30+mvDL5`0i&85H8b7N@BKAZAK2|Q~HV&Kw<%{a! z3oB=KYm47xFQ>IVu2SP?;rVa*$QWGL{P15*7at2s?&h6XYX4i$miNyUjXeew(~2ip z)=5A25P9KqcAKAx7am7)pUn1Jf7mZt6i-7)guz3@n`1V5LK}nr?;}7E)S8{|xME2p zT6Fn!1nWsh$MeA>-lDgn_=-Ts{Y&aFRJ6haiz)>$7dRV5PD=HH#6wu>%L0dG&UX2` z@poG|@=ZMcKRzPKgj<84)RegyhxqHOFxV@CL?%IylnqHzJ$6qN8eyLVe4RAUdc^N!=H!m;?u?TIrs6ev3ehpmt}f(5;k7geD=M+8|UAYf#o{V z!cO!yD7Zwp&y2rsv@Y-8AoUbja9m#oKJfr)7@tf@6X~FkKd<+Odc>=0WoWoJ;j5_% z%ihx$gBqyFV)=VL)9hX1GKMFF;^_dE;_=E3{7afXBln`aZ(R7@`ZF&GaAE%8<;=7- zUaya^X~c)5D$pOy#Ic30hV@r{8=wmveTchzNtBIV@CM-w{9H8TO>~~>@xA?sg)hEc zU@QAS+C9p+`5BjIDO)8ZL(-?Omm<14a}5WZJ8q|cZKu2}V5E`DSE1{h*k5#gzpUl1 zDc*-+dpbX?+&uIv@?NL>0Xa{1{b6j|d|&+foqakq^ez6Jdh(p%wP8y(xk5JH1Y7-9 z7SD$${JB|)VZlSwvC@s|Q8Hk@I)ZV?^X`kS%`HSS?NPp}#G^d3yj|CY%ytqNCWD2{ zZ4G0Bexdq3jak`@ZVIYj2bs(X*O;?-@%U$}@}3P4gIM-(CRVzR5N~}zaK&2pTX9}& zma-5(=;BN#Dq>?y2G;VSWHHd9%f+gSPG6Lpa^lc#11Vi$^5YK*-q5H;i>+LJt{Ftl zFyGWc(6o=aTDm{@#U0;{3o@or>QwEBu2bDm?Hc=|SGV_Vw>P?|-}54wrs;wkWhpYH zoWEd*arqPLR2u7KPvEfXjTG-jy4Pj|~aeUsv=;+@?xrQUdW4DxMy5{fk0XrZO zGUl%y-@V&_b5s533J(aAK9OTo8lanH+5zcIY&d4@*q4{{Vy=tFQ*(YE1Z^-qTkWA6 zAAb6D&Ps>#*tS-x5ga@1E*4jKD|9Xp##cj(Ni5H5ZyOyYvotC8-2%uJB`FV9wME}x#^y{*j!p|wEO;2^-PPBTaRDW?d!%%8T3r-iTC<9Ian@2c4 zUj}Xd3&dnC%90m+R+BnwxPM|}eip}X+ zBY4Kop=2Q0Gd?S@C;+Hb&d+~%)bObE@-l$SW@&y2q0>7rFDr@siAZI1Z@QU~F9W2T zdWzp;VBp8|o2UG_NWo=k=~m-|d!Sc#9obW$S8>hA$k@c5!ZT~}ifHzq(wT|eZ_(4l zHMN#I-noSfMA=IEuGVeZm{}ZOHb3HtBaN;ZOS}Wej}Hg^Nuv{|8m&h<6xkc6g>I?E zBKqoL8Fl9**y`(MsD9mPe8(R1&C{aRjR4^^3cr7h66WisbLO8 zgM5&{?YU1Vl=+?mORmn_TEI{OL8K9GlI=zI7cA~VD+g2ThypV^Ap=(R<+|S@qLMZ6{!@{&8nLzL?w5t*eKoeH47|St09!W!nrZjBq*m~mF z*s?p}S>~*HDTi<&yQG0G`EK9d8eWK<4is8UXicmWA~;LsSV zsL--eJBE)NryY=qP?8*I=Cuf%&WsY<@w8&@wmeaPU!pP+<%(&?J)pPu&= zL}{>7gXAMX@4_o!7g+BJMlQq%=cd-}Pn>MdV*TzNJB}RP7_u1PXLn1kfIK(1&6pPj zkZkO@D*Fh3iI_Zi*mzr0RAef-4})uY@+Pm0f1nRT=0ANL7iS(b>bFBFIg9V_hb8*c3R9Kh*cp)*+amKvnzDr&= z;PAB?{qgpq2moDHtnk+R8&w>SZ|MasUO%W@Xzaf(mLrokC%CA+<7yDQ__^U7)0SC) zUS;y@TbJNzCHv z+03FdxV{P&KFBL$&Gmf-cH*qSRkXFQF(fh25PZ=+S+7Mtq$OjJS#!e6Vj2z|R^6S! zCNKd&;$M5TnclA5yFce_PapHZ%jo8Ne}@6zYs?JZt@L96-Db}AvY)}J=sMKnSLSpN zRH35cae1$385qi@RIqGYO(Twa9mpD|?523Y@iC~4er1W}fKJoP{MgTZS1YG!q5ki& zT)_6=hJHu^rsxWV_AwIi?|rts)=yeA-WaW4+e9}^$mwjBCho_Kh#}akwwY=31Sm(a z{Oc8Lpymnz^&0Ts_#U6DA?~$_jc6F@PuWs%U9ynxv+2StaUHF5t&utXpweh&L7s{z zdOy}wZ5}LIWXBULEEOCb{qzpPB@lo%+1mM0;GR8RBF>|JW7HFQfjXmPz{1&rs099| z9v!qht+9lO!IWC$YAG4Fq6%Q+cx?6%xCL+!Sddxva!$wV1EI=Kcf;Rv+QbO!mc}1| z2gPs|6_&UAaNHq4-E}&IXM8uY#DGhHZnn&tD{@!Yy02qLoMA(8ckR9N_G z#7|P;aHBgKiU=yWdczlo;^N0w{rDxcdt3F$tyqQXx*KI_js%_H8E@2ZCrvl>hvKWV z-9Yh_=EK_r@{r^YV0pVqJ8=Y&mD=pF5@ff-6bdExZ$lh2sB|6|UtJr4#V^I@3WH_m zGy!v{D=<4$Rd6(4&jTQI+0Vw6IM&@Fd7}LlI4Z1utgc`95IIQL(eZO-XWD@pa=jr)1UAHP>WFz-hYE*8ctbQ%gSX zN{BHnuos30*_y#ZJi^wnckgyv|LE3OQ@q%a{oU~2H__5FptKnKHwEPqdF+UsVRRgd zR~4ouFw0v7xwTuN-7{rWRmp2<(6njC_H`ruFxHX-k_CtRKx}MKx$(1qfb!rayuzSu zu(kEkEBJQUz4_+|qGg|y)*SUksU@ZG%@*fkHDzxKIqv!$wEu$ zqcC5A*|GL94P*x`MO^WQ|4G*FSF*p&zENoP>Lpw)=`-x(X^cAQKH?DbDh{H2T)*+Y z!ROxf>wJP4$5^GfDWsMVJ2bGcW+r5`DDord0VWrqykTsRphZf08QG_t>^1po)nh@+2|ZWpNAC@#delGQ2n#o3XWFmpwe#;PbW z9K1iE-_l!iGr51}@y-dUTge15)ZfwoeR`|pEu|T#u|qX4QW1*tqem)3#0BM z&P>f>OW4;byB3SFE@?A4%J@-y`ppKe!)5S&I@!-*?CuW zwedILjg=K6b2As!24rB5@>tVI;T5n8?%^C1t9KW;rf>Ahm5o!GficgK=fP=}TBoaiEhp(S+{p>v}l7u35Cqgjo9UT>f+qEf7$7 z_~_A-dxeJZJK5l4{_w6}i0meoZKruhYp=_c*hVXmuM^aj)Tj(jV z9IsT3U)qO$1L3$$KaG$rC%nM73(P@$Il^5GEL|S6>)26 z*2|Y7F<}E{dC#q?l$m8Oy(`O1XO{)aZzpQu?#T&Z8QQT+H+f^<^JVA;nd(kNWri;{3wRpRqq5>#H`p*hCt_N`kDz{q$^>z6eTyC_bAd=!E`TDTQ` zYDEd4J8su4y%;{12tX8Pb~SgNNy6u7SePbFux@Ze z{fqPs{OQkSTKYhoPW}hh#0n>|WGuxar=t1diWa$aD6fv7V1C|;5r+~`JZRt5J24TLvC)( zGrLB0*;X70NrBp)dv|#ctSDx?=U12AOdIC`JjnH&$BTD6eD(#v-q=1KIL_VaRXb}f zoGt0IKqUJ=4Wu(f$%R=$^|3>Z*MzSwTq%Q|9GQeY$(78?@gvm*(q8jqUSn+x~izKd@d>} zNO6R=$RKvRc;~KNx~LP`a?g~IJthsg61zu!gUd(IxEQnT*`(!1#vk6s;y6m7CD?Dy z^5u`_#sctL#JD#hkivwMgEs4t#X7Hyu~Y@R*lb7sP9B8JLazW!f0j+Z;?7eDzYC!A z)SmICs!S+>2?2qDf8ipzx|gXHN3BEromB&wZop?>8ue76$k!U#(?hmDkO<%_T8c2f z6k%>4)I3y+AeE7LPN3#z!E*}(Z@)l>{;LB9&TGk zu!q$mi!1Hd70J6_{7(0z(VCIE_7&YH~J>mD){MT@^~GZ=WZeda&YstYdKD z6q+vVHXj@e+_ftwHLn?;)qh_FWBl~k033waC3*wP{&z$xL|Oo2`i7y0W*peDQB7&g zKHaZu%yw0-B|jT68N{6`S$x>F*1g`(PLG4@J?hY4$HPGh&lU$Uk}q%^$ z^XISsZ+KW{dD*fqUq;vANs*?3?D4W;AwWgq^wXle$a7|wks^>{6jTIKyvJgawX0n% zAgPzs7#ZywZEPk{B2Y*cdW~p4=xo|NE^J{PS~SBq2{jaF;zsq>t&ke`=-E@qnh-+d zR*i^An;Xjz0u(8okFZTJ$&%SZFtN)UaRnWD#BXak^By3bY-&XnQkzp^9*f@`C+YdS zcbBhtd)@fw%@PD3KK`2Gu-H~tgPUA@-4&+SUtmR|5NRp$zJ1?!f^PHe#Mzu=rVS>A zW^oTE?Mh9^{RQD*$UbNr@E+^ITPMxgdZE0Wu$U=?FXNRbD{D(EeZq6Xu!=@Yw3XP9 zj((N#}8=rA3J^q4{!MMjl?8t4Dspr zhW;t8^-Q3hT0;0`9*W!JoOW|Rw=hca8bEaQo^Wx0qLFpJYTi!3<1Fzc9pshxij9|N z$}LvH&E3Ymkd6(Ekv?m!0sz2&7-#(bhJM}~U2|I~E3BP;9@Dw?9T?!1afd|o>jpzr zpkX*)iiO3>6vTIy)4!>@Zv#w~L}JUkKRa?9+bPZtd9C}>R^5q0G63sBv6Ijtn#%ml zyDr+DM`!>elnfN#_Md7&$8WAX0uX3d4EnNu`KgD>h>r|24LQW?WY%^*Zewk2#eX#S zu>3cTQ|f`OF(kcU(q)ABRaWxdt`HZ)!EBcV?bgz|U{YJVt2AQMa^F-oVmfKjSxPDK z(lYRfuJL4Q(aVS}-G%w3vvB4MHIMzdU2f?xZ$LWYBDyIbFuh7jdLaPlFqF*D__778 zLLT;4Z0QFFUb*wP=)_X9VZ(4|oqfJ>F8GKDsl>nU>3lV4a{n(L?yt-V9udw1N9MKx z&*r>@EiX=;jWG`!GvC{q+CCVoiQw)g91lLzN18W=K=qecXKD;sSSR{C_zF4af|9L= z;N6au7SF#hkx0#U5u!+k_+Y7{iqp%>d7kqK=M zO1Qa~!Awp>7Gn$_3fhI4b2_F?LE0g@3@i9aP`;A8!k!1Tt560IV+;16I>~Sutd>oi zJLK9kjNCioZ>}Dx%dZ2ujnlczbG|#e%TsJn_pGZH)z`rZ>Zon6!2Z+xytKyhPJq7D z9x<_t%0(QKA&1r(X%@KPKS!+BPkAlk^Eo-`#L5n+VCY>pertNg6M658*u6iM=VQni zO|lL(ntEa~rm$LjhDTA9_!kF^c<#{~`W1Y?or4f|&0QoICgFOqmv%<`;^Z9$yGRUrb<& z2hC_Rg5xMQSNxN{aH>(iQ^^_Mj;2K=7ps;cZ>k9TShl<(ABqCgbn{<%mra`!%b~+j zb~>KR+mlxaePx38?QKs%xY1OjI5xo2%k9lybI ztgK5wQiU!j`aX-I)ta(Wr;t*{Zia(N(T__m!OOML$9P=OZgpzf7^0m-{erh|w?n!= zm@`IOnEe=oT;+_FeLc&Nrnb-D_TW$T2yr{H**U%FFaA?X`LaC}n2fA#7Os>GCWO>C zF%(V-)|o$F4;bt`S<=mI8u;G!!M#MM0A}!&MG#Osuf2Bd8tAr8)5fY-z?gjZ z@2`u-1eV_osP=L~2+f(Kdar>uZvST*H~X!IIARTPa5!tFgX>)To7Y56$XZ(2m~e@T z{*TA5bKfSBmzdEz8PA5t3iSzLcE|!j+?MC-DtROxHgGn@xxeq;cTIG3me}6dPH=L8 zMVJVcBPMC!c3ly+MPPZ8xcj4pZUdFKZ|?S{d>IgzF3+njJ_v7DLkG zLY>FHr2;{QSQWRj%N;zYXpY7Hx^O_l)L>zz;Xi;{gyX@BrWVcvd~q-scJlJ&W(v8V z>NpQl1UNw@Ik;GpB%LGSHM`O zR3-2bzcNP*!=`UNBr~K4S?{r&>Fmf&UB*6ZNdto=q0BaP+a#Y1Ztv^@!{p zTesFnc8*Ju-CalcKpKXZScrpj z8cii~1~yOb?+v1l@fXG|R}>^^P*ji=bb^7U`4t)nCSkP*3<*?1nrTz#o^P*RItq@$ zgbBXrZQoeO+N7)+*Urg+)vS2@h&AL=7$lG^aHt$QzDFSi_-JF8p#}S@s8>;;Wv2kNT(qLc=`UdxKN|H4i%`b2nqSb5MZe=v7P?kPos z+R-N@s|jiqR`96%s<0(e+i^EkGC(4mN@pcssTV(26*ySE_N%jLbDq?hUGlK|H<`|J z(l0cM`IPpjX~N>#^#70waF%rlB^bs`qXxS*g~ZqDsjv>T|+Io&Y^j=Z|I?`Y&~QyHw!%= zW5kXfl-$LGoEv%zCs!JKw>XR%*~gSrZCW?;@|z9fDZ71~@gT~%#Ip$6vOlSm$w(k& z5vn0af+7tCND)$LLivEWMRRm3oKD9x{(@;W?iC~fDQ_--z5<<{^VMX-elF`G-+_0^k+Q{Vw8F1f6@*8a1t zzkH+g|3)9*zLnxgp@R3NFUa$mGgFj6LNRFc_R@-{X@{+(M6$cN6$hk*yUrG#Eu1aQ zaMDTTh8}5}U1r_qF$QEB4YLjCx7P8dnAtM|vZA=m=$<_SP%s~#(S89DV@8gY$i*Ag zBZk-Fu%GY#s&rIT$mo#wyvco6m0Wxo+V+!`Who58EwUB5nURp>W~I`yV@IF7L8jGI zeB4}DYu9}>Airs^_S>E@G~KMQZSwYD8T|hcpQvQ42MNoC8NIx+a#*T zzcWse!8t@>Kc4kTP^(EIwn4iJgDH+f6D$MN3~ z8(SW*DhMl08IA~vJ7cvbP`+8Y_sN*VCve<= zhQxrq?`x3-7H5`+6wUaLOIeM@mlG}3JK?2pZ))}GlRCHnQXo@wKJ)pla6l4*lA7V` z2LRNGaIWk8FA2Kg2^LdOaGkla2=@gurjr>)Wn?y&umVPJ--%#T^m(rwSq@-^{lX>T zl4*fH$R>eoV6+4mTE}`Re$8|5xEPJ0>S9e^G%WDFFe8ejK$c(=1 z6$E3eB*H_z5%piUCXpv0>oeTCo`!Tl=eEW>@d4eBtie;0G7yz0xNqbxy3h6@9w#Sv z#8?*|W>`0fuk34;Zj?p1*gkiApeD^?7mM-tAqSrOP0-|Rt)u9pBx#Uf^9N`xngNKqk zp@St6#aenJoTj%_Xpi{!fPYbIJ1n5bpo?#~!}bLbiV#hC2Yl@Q=n4m7ZsEtL0kg+_ z36wAvNEc>j&Yc@iwn^}^0ytN!1;jcACu+EP92131ZLGVZQ>E!=r{?<~1lZ%-W*stN z@L4=+|F1{i@JW1e3vT4Uc<`x{Ty?L`ow;~%kF(SN5aErlzy_uLW=s68TjsJ0yhRmK z!bYGIYCE=BZeWg1Y;>+N14^~hC)bo&vPF~9PnpR>8iy}Vt*L9iysV!=*_Bx(w2JcfDV9${IG`zLRQxOIq?R01fjSWb9k=#XTMdYc@@b*x z*ky<61_8wY?dZ10U~LK_OGiK(dI%~OSzGmrIruO5BQbzB0ZkCtNr9Env3Gf&ymJD}p|#TkUq4r66Ye43M)pibvgZP!b{U$5+XFowj7nZ_;B1qKkH z!p_YRpI+`cO@6@y5#*lzzCA%otiAotgi!wj8z(|$ha+OfUd_HYebaF8_9p+l)rre9p>w`4uMXY|o&AGfns||P!`yD%U>LKh7OK9@sGJo_9 zgiH*DO*&TgJIcQcts|a@j*Y1_i~8n};ej26fIj3)rZseGC0Bj$zu{+fQeO9}7__^j z3_BJ7vAu`R%5J}nSkhScbNh}emk*V^IP>a8U^<{C8YxXRZ(3f)9vjOpF2{}>Ay~M+ z;b>a8JfmFqH6?D|4*Jd@E34q^x+^IY zVsA(iNA~}f!`V>37j+8<>5@W^@(2rhXr&9Uh;5hc@q zZicOv`GHmQ$h#VmdPzON8h0qL5{5H}`<|_?LIDF+(vM0Xa|bryU%Y2%k!y{tCXx*5 zpU-2uLb1M(>wc@u0_;KO;X+CqYo%H`kd0qN=Bi0aW~vf2G*>XHiTFn|X-ZPZJ?BaA zp#PE^d(-O;=|MAWVuKob!*G%X01Mi*xlhn4$eY<)eN|1!D?aDh@JR8J0Kpx53vX31 zv9&$^}N@4HvS-lz{i;V|Ko-I)!3O>pOuyTs#n8<80nc+E8fidy;de zeJreaR_&{7Cy%3Y$)MykEq(IrOH*t^M!Z}w&SBK42XKWdTyy%bTyi10LZLdW1v##0GhR|?*YoeX`@1|B)AXWG2Q858HTlV-`fY(h*RXqPv<0%l5#z{6dAYDbvVnYxz$tPFS3NJx(@R;Ga)GaA~C`U12+?)(iwZ@@vcG97#O)nPiJ~OKJW+kmEd%z4?c*V~i_bqwD)Z$+ zawih_!GfLQGe0=@yanVF$X6KFitYuK>6PTQR##E>aTe`&Slq;FZWV%aqg5r3=00J`JYo9u#1yBz z<6_K)x%EqrA-Zd6#(DE3fIQo*yiih>ID>FxmRuTutcg;zZtCqL$H8Y%s9gX0H}AaM zFVggFl4{m#iZLA}rIm7!5|5A{aA@BBwconmmJq-nCPN4Q{`a-WN2TvT=V3*5(3;n0?`UaKqTB~^ zS@Zm4`8&i6l3aO=i;Qszy4JSCVF*DR=_)d!TqlLF>Jfb#d+lvA_3MF}Y~)=;=#Ur= z%~kLfaDXZAPgU4T0pxYkI*|8YC_%)Jbnj;mpf=yl$T&UylVU6~yZoI;ex`ABIM_S| zCz$Jl9W&wYA@{srV(%J{s+4PfjR*wWw24l8xyIBqucNVXa{!LdMM8~j(!}xcYrn+p z8$3O;kO+S6i#e3B%_mOO=HdS%tE-=Z2xuuD3rCJ|&~MYGx||6KBm|&c9}$szXApXQ zQ-X1VckSP=PnRVOV>E6PEp9uYg@V<$%?P#(~bR+~TA4l1H>nZ;& zU=dg9(c^q;qLY}@u&9un@sP&NFrFIWmADfU(^aeQg{8M?nZY_#8z#pxkBN~kJz>2k z@cab`kV;3oY}_#=eaWBlB=hLEOG776N_dLIe(Cx1=bZ{6Zw;o`M({bX$cEb^uKvIh z6%n!{!`~ki$8fJQhwA}LAj}BIP4I}UH;v|lZ~oV>k5XBYo6=N&=No={nUk5G3xPcj`s0w$jUpuc^Ty3vx2~bT)AVS;`cPsaNUI4!E-ebE z1wWxAb9~9TVO!3AB0Oz^U!Y!9Yu6A*V8Q`u?S$xcq?JYju^FrrJ}-Ca`DFviPL(oO&StFmDZO;Lr`g=sa8=H`8beF-_=n0OHF>BRhuVxl1PjOhS-7|$- znB_RH|M}t=6%`Q{a859pu04EMSP@jo{uW(%kUuAWZ)%!->A&>9M<2dLtVky5?q#=e zgxf*``k!-h_GI*;*&|-|8|p9$u78{$5tI=hFuRf_5S|;s@~p-=?p-0c!w~jI>~${Ss`^8XpHV@ zmwX`EK(zSmEb#&!`Bl&wI?#w^M!J{xmCdq!8>1Th%IZhnfI{`VP7j|x)o*q=GE)Ec zQ74SwIq^6q9-VBraG`01A!wt~L$@4=h|oT38SVFgSKDahkWs#Thy3RR==cxlDcN!y zoYV0YqDc{_{w+qe_ZQr`ec+YNiqHZ!nyj%Om-Z)lEHXLw8lCeM8k!J8uYmo8fb_4=6t^g{Hcm3?}^AyTaJ=+)!W6{SvXxiNVc6Dwc_iooUdH!GkiO5v}f-E zqDGc8h|=T8fr|o(AjN!6>9#V2G#x)) z?O1m1cmTPfL8C^0c&-nd>=j%qH2t^jH>(Y5(6ZLrB`bftS{NQ-Z+CwBjxzP#zhM=l z)UbhBulzlFKFFQqyvuN4aC@|8O(bClS*^{}EH#rfDzA_1r3&uG}p%QR~R68My<80xyIQbKS@g7GZwE?+8iljwQHZBH2CkD z(&4!Y_$yydrshW~0&Ufk_5K;P^>oW=n#;zu$8$Ge{M2oobfdnEOqkvStu=J{Rx?+J z|NXa9YO~!`J;^bZesnU^wj#(a{Wh~pt#KM+a?k3#i0Fb257?x#V_jEXw6%Qt#LZq2 z?9yrQeyd+@r~8C16*(uFJ~p8%O-xj_=oqeTL3C7b`;lK+6<<$FTQ*4hs;-kBpA7!S zJ4>H#1-#TTY(~iG(oy4AsA#G$9 zAD?&y0DsguI&mm|^zWA+A&dd(;aYF69ii|Aac7;8#j>O~w#;`HH34P%qc}lDe#ayH zxt{iDps#&zYs1*|arO>4@i2z$cz`9Iq32pJt*3O_BBh%#24MogV?gZidI<}{Z{_5w zu15Gf_)jlOkH0RnriU|rx_N^Jt2sV?&Kd*9BSl-t(m!V5)}0MfV?`#1@naQWm^w#h z7_p$eF4f8;XE*d@LVcxi?{qBe%xI?P-@cJs=mNnq<7knchWsa%E( zWPe{GZ;?&@JPyWP)_f?BstsD*verLe-|P;3kn?y~&NwE1g;$tvaVPi3lLfFriu>i< z#o`9e*ugVr>>t`sMPy66quZm0)77BZbLSe+$nSbUdntNknOZ!IvA;5}Hb@D}VBspz3SD5=yFKB9fqL3}AM3g02vR4wKL5ip_DN89N zN+=~sOpy?xq%292RL|>Nao_X%J8yul6afdAz7M|t37i<>xf&6|LXR@YKRD+s>I6)|~ewb=R< z1J;7#S`OxDfXI{RNZ%3klpbW?kt4f6ylWru8iM7z2-e{*Lt`^CG1<|!XlYrOHsZ|>UpMwf!6!YqQs`iQ z^iozohOuqt=+;UoNyX^^4r0jkJ^`7Hn>HP*Vqlhn*4JlXM5ud!xD5Cvd{PpL0{5(+ z$BX`(gY#mQ#<21n3=wgGTYwu^n3?+M(Jz8yM$GO9pL_M>HOtdYD9g9$0#Gp(`t43E z6V5({f$c)lZ4n4fiQ~#VU%KO$5z3Gk=hCeA`S`fwl_Jh7u+qY_ns(5-kyGh$+cr|K zS*?8kB+Da34ws(J$>L)<>2kX}G@r5mJuh>H-XA|>9E?rD+H!Mq%`VrctL1v+@w(ot z%KzBNbqg<6rN+nC7Q(-8`^V9EG=Ah1!Z2Ro*pKg&Lk{W7L5=@M>FCW#g#x+@ywZ1( z-6Qtx@TX;C$hhMdd(lv~ChDSKSNy5*xg$d?sSAukkBmZ0zH8lVkv*)@373b1OFLpi z*KGcSm-DyR?OF4>E1$o(sXlzY(7x5;=PY>xbXXTfrx9;~n@O;QBFpX&GlxaR-S&(6 zi6K`u?Y1n9;@}}dR6a33eCIA0KUtuhAZ+?5b&`JBHo=Erqs+pd25#y{z~e7X*|ORwO{nB=fbM0xA*Nq(Ajd}kBjiw z78T3)%P!@^KE|4z!o@Z%GqZOHTf;t-mBFAZw{Ppyn@o&^k0d_CtTy5-zm>prHwzpL zvj-HoH?HyPfEZ+rk}%`^fJ@bMvSeEvRJjr_%&rsg*7C=kV+4ytkB;q9Pix~3jas@f zvohEj;5unfxOl`h@ezZY77^T;1YvBxrgu?Jdv+>zTEVK}T$g=X>^!K>O$1DX68B;5 zC4w2mG4_FNRINl;E~D@=-z7t)so5@D7P)R2>wnX3$qRAIzk_ilQP7ipvv9BYoYd1_ z4s!zt?wh#poW8#E3O(rO*GNZ4XGe48G>4w`YNBn~OZCs}pySk)R8>IMEi~)2gOWU& z`2p|&wc@4mb6mUaQe&f{C@~zSAr|*deV5LLru%j8l_2H?)+)ZqWpn&PF8Sq__Nxm_ zA^9+K=ROb3Vl;!vT$i75Rzz7Kz$G>!exZW0QTX_P``>ls#XLkIUi4_1PEy9Wnil>u zDv>iIm@qkE$>aFnuRV(~+yQr@x4!}UJo_@cf^oLCeYnS7vPC$({Y@_ggwXCcgbz374zf}H3kYHIVco!+uZCKzNc;P*J>lJ`4)9lWFEv|(6&WTa ztt&QPLHhv|P(*%tM=m|%>ntIL*}sr$sC7x1O+NBxx$G*-W21gQk1GM78?O9vk5XUtzJBD&s~H{oznFChvArJ1WVxnc~ntb(((i zBVmoOYw=w-r1QqF;-%Yil7YtM{PHOqH*9b!JY0xN%WLj`Ucsd(iF=pd<5*vU$jX+0~=0IF#tUmF@CdJak<~1b-d=;1i5srneTScl^Qmy zUzK^HMPQl`Y8&)W3wS$8#efyQ7@4DD@muK&H<@^2VKpmRd>RnNLfEc7+&7#i)`sP5 z8pvKorkI+BJs#~=G;F5Lwzo}fALR4P&$Prj^X-Onv1dgtovwJ}4iJG5nf_iS4t1)T3*+W!9i#@_1-<=2y{DY;3#n(L46flCcDxu~g zk zpNvp&8G#JzLx0rBS@=;dCjL&?AN$w}(6w4ej?@^h|H;a^>r(dh&@yS_dh^g08QOPY z3T~_f=UB|7&RJaV^E`vL8XP_K$Ms#-y?t%`?_iGxZ$M?j*r455XJjHD>E*qBf|>p3 zrq?NpaAc29Z+FLwZW7I&S&P?neW7!s4>mSDcmk8mOs(O=+qG-FxBH{=`Y5d@OmGV| z6Z%WarjAQWV0-x1`2@hx;H0WUhZ84G(4S#XrDQSe4?BDNd0#h1SEudzhF;<=03{#r zzPP#ik|7lv!E;ri4yO4>L(oUv9Bw{+`h)$}?I!6@p2=)B_2}^KORqIiqRP@!nt}oR zG_=2$lA2^WjR#ByPIoGu*<3uj*?8fi+Mv7RRG{_nl?Xfb)^4Wm2((K-;g!xp`{2)| zx3D6_?8prlsHe}MRf#7d44nRD{s$7h8Z%552EwMyDQ&D49xd8qJ|q^`{zYp@qX)m} zGHTS^BVD`hQsaBnJCM{Z>l<>RhJ&k&RSfEFRnl!nqOsLAzq6YD%d$IooQ$HElHLBl z&rP~*<`Y+yAsQOnIk24PF)B+eyzbt*^#=oxfvxTP0RQa1^=i`4^d6Dj(P@;+YJ7}5 zemsNW_Wf6w=#vh8VTGgM`dn8rdG1ymf?vN(Scf1=P$q#~hHF0Jwg+9Vjlv(F)yNro z<_+Z3j8^$oi_(na`nxF(UPL#2r~I(Kw>u6Sp5Tqbck+XC=^x!b|I@eQqNIim8*XEd zicL-TbV3V6@fJNYmj5D5_fN4^7MHA_{0BfA82zTpoz}Sd;)^H!jqwM%c;Sr$s9m#c zm5~Bl7kH1sQ#Dj_d6tRKECR!MZd=cY-6pz%*Bu0&y@o>|V@MLXf5e|G;X- zA~i*bo_^818Bv1P<>MnO8|EXZ$gZZzt?p?IVK@qC8uv?$5@E99QyayfR&Q&wrOsEw zBSz0tcOKE#aG6-ye?r57y*l3NP1pj&f0|IEZM%OKnh*>PmCxp4gVLOwyQ*Z?%W$%LpgDoL80}1>8pA6{bqO z(xk^G0aYzJb=rfT$7bYp?H_v4>r4BCYFS3Is&ZJ+U2Vi?;yk{8zoNlv$o!wY7PV?n z91adOt-F~dF&^jnM*hBT2hTG(DU7%eAKsTf3`QHxYVmp{_4<#p1GO*jD}2Q}ERg8# zab-~}mM>o}ZAISd0b4=TGFK`|@WFKWHQKxH-kp@G#gs{o+1>4BV~7r6H6h#st81g6 zcbv=np{%}yz8*4@S329?E0*)-(Uii>=P*!lqBYui%Dw@%qurj?*Iwo3?xaNo27UP^ z>40XbfBq&asb4fbY}bD5 z`tK5S@F>j%T@K=%4*VeY&odaLVa|)Z19QPnYKHAP?uZmU6>fb4NLY`FG(up}$6qktM|EZ9Dt zJ+~QMmOb9FHE|`25+Pi=AON=XbZ3pROP+t9ouQz_*f`z+TRZtw7-RePDK%}n*X+|} z2)ihKUIAMer{Ma@TlyW3o>w0}T>m@Dy$}hDbQvxL#dX=Gri43M4)4Vs#w3-g@g+^qO5eNBGFP;1_k-3ek@RyDnwZL@v2UCZ4lkav74s!RFA)^9{uw2Hz*UtS&6Y7q@N<}b&QnvH z4LeP82NJk3Jbr8t4Fb_Cp`@&Dg|=d_(L&8UhO{|}Lr>gur^flb-ka!mskaTfcXb}d zDPlTONzzRo&$3h^BjHe+J%^Z@v=OE z${fPjb@$uYYp>qF-zCmqh`0b5%#YD}cRhNmmJ-g6G?5f;weJ7g+1T~ZJChQ5Hh$r{ zPj=w8n*Q7TeEt)O+yAtMzq#+((bqLgG(XnuZoDVSQjg{wd_t!{yE*bJP6J)(P8M@> zixF7977YA|hetzpfXhq!(X-8h+xxkqG<(tf`D08=c~8a3yHj7q!~W}+fW;?SdSdP2 zgs*(9S06^*0qPf!nzginS>QH51wKS6%$o3o|QaUXz(E*#Wg zJXv;%$R3g&8S)Op*`7!@WoG7Mvi{4P`l8^1s#E(wngdU$c!L2UA+BgdKj-wV&?jXL zRj+#CaT@^uW*(jwE)1;QyoFa_!+P~LQ_+z8oivB0hAEYbp~0J3yj6=ED@R91qa#*Q z0{WoZw8sl^!1GEFCuU!XqF<9bDdI5RE_NfcUT5ixN$TB%M^!!C4ib7``zYwxlwoh- zU9E&ThdyRfB=hVd{?y2rh(>JOGm8wAc*Ko*HP-e8_=iL`t6l7dvgSY@Q_F{sggnfZ zlW0qm`Fd&ZG!=*ypv}gP#?&L~_dU(}eWKl$5stt@YQu#yF;Z)5eX} zpE{=h^N)mhhCk(Q1n|FXojOrJIg3$EIBUK;ZG@c{w&>3B^e-B#&D-f4MXcL*f?fU12Ydhc@Ykmn7~VTw!|Rf9wg1X3Fpq8lj!Ne3tk-Pwd?OQXKq5I zWk|7Gz8A&b690{I*VKNftUisO2SWN^>EQmg)~;PESe%`{(f0U-Te{S1R;va!m20$T zE*qWo@Mr$-WE0z<=gXbCHHbE$ZK8#jS8+vs%3UZ8$32GaCvn zaR4ERNLhhQsMUG*=z6lu+3=g}tZp!;tU7PyV25qvJOh_rFB*HR%T7bnlR3@2{xGi5 zre)~&k4arLY{#58)je;N;*=j?6gMSgr|ZY6N}Kc*>1VWFy;G-8FCvM?MvQmSv%n6@ zRNwXsfg?XJNjr7jR+N^5?^2#bc0p1g)LCMYy&S3nY7Xt_CeRy}_iIXSebB7ok8t&G!Nz8<% zcaye(+1>ao$_W0#KL@72?v4KR!>5db5;#YH4y4UUD=0xwB;gA|=E!w2pp_j@ee~ps zXyMEx-czC=ALOY9WKOYfqj5d#fnGj9GfF~BVGLm0_-@rzgLPBY%wMqkB0p3>#*p(^ zQ;3TZZnNwOUp$8nloB&Tvm!8Fb_9^tWqbat6~si;l0_G@)QA-S%M+elOI*PBi~)XQ zX)2tg2eyaqJ4_b6ESbQV7%W?kxweQm!^6Y1O8pY=vn95aLM8UQCp7dY`x6-Y?_+ds z*%TnQJcqP@ZmwHa@wBW9^uV4r2a)Wql+yAZx*=ZSoPiAO1Vc!y+{2Hwdy-x`Rw!>FAa^= zj<{d0T;17l+s=Xm3jR&n!|+w6_p0mCk(*;rCxk`5_#U>hS&mhwgzjqE%HLxj?<`%v zae9bS)3fb94j(#lm95kJe?B{=-AFyY#owcjh4avVlAUUV%F+mO`!-k|wy-qCFKDhK zThx~&T;u93~WN1_EbD)=j*ROYEz%v%>FD?x%iH>Yd%Ke1KS5k2b zKYgU5`=0A-|Ke&FfAh2BHf)7LfT%ZOSr`Sc|pzmu!$WT0} zLKrssQnW4uX^YsK034AX2>2DSO^HzYJxs)E48E+6n25KJ3p7G>qkRNWSvI~B9zcVB z0|{812|HF2t~8Im=w^g$g5{T3$;)44fFXv0Ddd&dNl~mGCmq(mudia@n_ZUg(n3Eu z?rW+pQv)N~j9#8O<_LkL`c+*QwmSr=j~*)>pe%&1#P}1C1G3Kjac502OsLOwe83i_ z(D!$Nd;LB}#^ZgA!Ica=-hH-e|75ww@B99ef5i6x(=~LXhLd7|m32htegb;?rH$enxB+c6O;Jd)4G9TgPR9C5TFe zhtrP)Fs5iE)zz*>6RTeAKApj1WIyCiA%KSgwkBv^VlOxM3IKQ}|6(^?Kn~Nw#pN7h zS3&;RjPJ>R!*xR5%)r5e)k1LP`Lbb+TEOCS%-vO-JBi>7k*Hu~x?S_mRrAuS51!Yf z@LB!v_!5Q}Ki-S0XL$;#6k_TL#^4^*2qsL!lH zJ!TEEy>#<##wz`Y8_y#8beLVxxM|0{X}=C#;Ge^l6n{fn`IjNqD=g2puFTY`6?bM8 z`-2{)%57SOwmJ4T-XO3h5cucvSeKYOjZNEV)LWIV7qX@@X)^c_`@!&?i{z-G40ztt zzU;@3*DxH#0%<{HWkJc$$x8?J`HG z(MquOZcH)UXUky}hsjBk?#}*1g5WL?zZ?`=2Zq|j03yIJ!QA$WHITTF7uY;!>iMt( zB^}I_W^QYen-cp#!9rjPkHK+NVD|OPHU;ud3ww(}NKBUw9<0Nsqv#LeIcK04 z$b9bHj$FjjA8N@7k%S}^EJ>rQsDHxQ6TruI?Z;=jFd|zU_dGqQVn~jm|4?_ozcTF( ze&||92^BO8n785!lQ4Nf)n%OYS|X+sDRA(&_vwA91Ck#QM>Z1um`(H60+R8Fvo?^- zU5!qiJCA1xra=87Kf@ohyfe23@hCH>1$#)KtjP;`c!edz)-EtN)f00LVJ0FILOCbY zX^wm1eZ`gu=zs%f9oca0`&@1(yO`qcJ$rJXZQ_n?b9>j=NJX@toSwVyJ9X-$2s_~+%4ckpQp^qbgerq&d==9QTchy#@U*z2j7MbcBsdMu(it#%fU(gvrlTy zs7UFa;*XL8H%f)=%Yk##uKQh0`8t_`Og*A6e31L(Mtd4ksg!KR@0lzUHA?5hN{1vt349E#lVH*elN@f&MtZ@(wdFuOIl+7`@US7r}UP8%!?0zseG z^w;|A9Qy`mukq*B>iv6-G}Q*H5LUCYWljkT&llAsfFmX@$3m_)@;R&K+~eN6qY*FD zgKAfdYiLoCt>f__xP`vH$GFpV_Jo#f_16u=dR)E&O2b!#%)wTguio=d`ZjS@yT0P; zx;7b6!+NTjH*2N*&Nt$?{X?~okQ%kF*fuLXWpE*+Rors(&=qpjVrpAbU&W%%6X%7Z z*YOPzwLcv-891^@2d&L8<0pulk|d6y+uZ>#?fDR|oT8GFc=*3t{aq@}&iE*<-gZ?* zovXsSwR)R!?;fk4+ASxANe=L8DxW zpl;?FELu=xw19aR#!;(`fBw9q5D;-xkg#Gw{b+hX{g9)9Z%8gW_mmf?A53&Cm4P#Zf#FRGIkdT6Fo!4TqE~9Ds$dMy4$bV6Kf0F*!6U>zuO9{n@ zZTkJIbH#x?$1hznH7(}B1oS$dcEPMZl#LQb?f*cdY|Gux$NXpj1PZXD zI6J?7F?V9RhmkJ6_)HxUrxNVwLy*(%0Tez~aMAE^^?7VJEU=)ee|e97h3ez=cg+ad zJl4x%R{p6O!y8mry7f5?epIil-WWLNhv_`~qxJ0Zf3dPk_G>vNb$jhmEn1o6tS+86 zbhFaT#->+ol;58id)T#!#gCJ*e;YCv0Va{()UR2Z0f);S`Gm+9gh&B!ma+yu z9LOJxLS19y<`gvaKIg|QR5LcVvkSHJ$_LM;s>MBjK3AGbU7AVgqXNL$;%_HhNV{m5 zcA;~kj4!Pyss=f47;H}JEd!lXHP!Xout%Rpls0@Mo;uyzvp(wUeY;4ebBuIxf%29_ zJFZS;-UME!$4N{$HNO=Xk42HkEawB}N2%{g=C zH0#izUsmUI1^v&h>83(v{El0@B--*8HPI>ElGqkuHqW&%)gwXIkwk3#yr?NiHvXgG zM@&NuXw$zsSJc%Rt(>+>P;y%F#Y3S54v6_)Q|^hAYCl$RtilK7-Fv8Kvvlb#K0LcU ze@0!sDu~wjS+fr7B^^AC{x(x<b=|^>Kc#oaYHo5F?xbED_dRTX2R7z7Ss+wwfDG-)Uu_EJUky z&sZ_sNKY@hKBR^6MmJt5#KMN~S1v9Wny}0vFwoW*ep^Foi{!C{=DvQtRepRqUMD$N zCs(gpC7)GIlj^4M)JxbKYH}L(5K@rH|sj(w?;P0k1sJ-fH2lJkNR}nT>Y=Ax69hjzf^PP zQ^l$4wy5@lAq#Xlcz;%$Z~gI0T8Q$;;IW7A6u2C7zB4<~u79;&>ZQn2ZgV2F|b5FBa2CPyW&pE9oaI;V9#QP7U2R8tvYk=9J0n*u&GoGHFc<=|BL!M z{JcSJoI}z`>_ql3AyPc>v{u%{pLA&Z)?PEpXsJ*=^;~*)E%?cXijv26cgHsok_M9z zaE!IBtsdS}-4#^2%*w1DfdSIQ_ghuRpDc8w2QFO*d^&3&>~5q9~1U7f7$!?O$$_W++(UvyTah-9M=Y5J2@5$Mw26hnt z@3(SxQ$lj)a{mxK4iBaKiSw!uH6V8ndGV1gC)>1zu*g1@WS>Mrhbcz5*w53Bzvw$a zIn9+z@-~t)grp-2G`UBlAB9-d@qGo#yFVnvTfqdxj_G5=ZffQU2vx+F`suw-^nWw!}=5KDPb}n*nZ)6$o&eoY? z+UQ$*qjdXPO5d;kt(<$}R=WYC)JBy&SlVHUm*3@yhKBMZ1!26Q&-f!(TM`hv(D;9T zuy+y1KNEYhQfZg>@b2{4@6o}d`hu+_m{y?`Vj0HsZaGooQ)V@L_GEk^>^SLpbP$9V zn_DjB`hw4SNc|b>a!<)8px0~?IlXn7tE`LA=1Me3@Hg69+$b1SCdOInUqlBfK_5gC zSX^`30<(x>lXB$)fPv(4QOLVMf#X~2jt;2XQMmx^g zsGK$YzrXuE%m?&wIhke=*gbXrKL0m^wY7tZfiCyGdcNs}iuGO0o1Ai~#P|)|U}mqB zhu9>FV@g$NRi(d4d&Hu`F#hYWlXC`|uK+jG^t8#eq)&#B23>CP6#Wp?O+G%s^Uv&% zem%3)MhT7Mn;4q_ZC>+nj$XT1r=prx7t5}f8eI43I`A*{KBRw?=xV#a6ZAWAv?D1x zn?5hHz*Y#Qgm*ekte2nP&$zu)^fl(f=R7wz+G<7>?&|{JjUk?KXvOtR3`l*tyiw6N zy3K;Q2(Cxgvr8Zz9dtG!-ssez+w2!SXqJnpg2Z6<#(cpWlzD#f)t7%~pU&bzMd5qz z%78c)XRb{;>cP^g*rx2YyVtI8N1H8=^VCr$_x4U)=^jw;7t`-R3gV37sL_$%a{VS9 z@rC1}nBHJm_`^ic$~yzx4~#El^z-cWMO(UPQO@qm1u#<~TvF~oZM1`I)kZETX(Z{3 z$}RSe*Z&#=n*_%Gz=E=HCDAGih|<6R{a3kNA>jW9zl47va`YY8Sb04J8c}$f(z_qV#Uovh@K5sPqMW`?wSrIFsde$pys;atShH!x7g^E`@+?L5Ds3u?# zs{s&%*}c)C+Z25+H|@CJGn^P_-5T&&^oERXIlKc36(R7@EgrqRYq8Y`rF=tDr6-rr zSkV@EOi4}*hN?!-41Ds3@dp{rfRJdkXwEb)16GM12*+b#FY*2?nKL2kH)`t|WNt&dA4sQrM0IY>^K27LsO8QqIW2pc_`rUw$aBYt7nx_KtSCSt6}-!dQ_ zx&(Jyu1d;ln{`9lyyhG2K^Q$<7vg;)ym%>1s$*WvhnS^LGGs86up zbJejJQ#^CtyaDY2b{3VqPj7JY#9J|ML2v!=3IR8CtGFqe4or9Li+IM9=n)AogZ`x6 zkxf>37}%wBKxQkd}Krvs(Z0MobT>9x!JvL`kZUu%}Da z`|>~;Xf|ow`0boY0gFSC8Vl|b&sx&E{*ZJuhXV^z4wUgNB`$hn~{^Z;0_CaNih`%McboMy5s5n zB~xhHwA9i~oq&AuA9nXGyDdSbjBx992gE49!zX*5{hH~EycA?a)tv9?B)hLAXR$+ zE=g_R8%>xn;pMZnJvHpYkJ}_}Kc(8je0=R$cGEU?Q8RD8=KPxRlazizlU~KMoR?gw zhCowP)jyZS`DM@B-P>SToRmDuJk`i_`I(l(VzW-k9HF*7n=gftpfLUuiy;4gGqoz$2+kTkdeGb3Bg%8Rr_#X;wuJ2Rjlx+|#q1{{hOGae&(#pFqVj1?kht%fM z8a1k7n`84dSC(8}o(zQ9yjbp>~A~qnoQs}8vRpU;4ThbKu zFkO`{AWq6MxlJ*9<*rhhFiw96{CjZW@FSS3G6A&upiN7H;De0yF7?6-9t~I$a(!7= zet8Y+HAqJ39s)kReXjQG_wHDmJJ-*xbSg-9D_*VZ=il&W<2_-sdm_Q$Z3S;LKFgVaDm!Cf zOVPR^3Vi*utk_s0R0!=;P@M`2Lgo*5P+>6(hi-uEO$8`XhCm1V;qBHG8iPBmZbaY^ zjPuBL8oENtj>TS7M0@#}#kG7pZa(k11+g0G%xuznxTcpH4K^w@_1T6Op+b?U!vc3Y zM=QGM>(|?ub#9jfi;W$aLYs~qBQqsgKz0kj^~y#W$OP|fCn93FtWb3<3jfykz|AEw0pxmNn3l=5 z?g8;Mj=kIe>$pK-+_AskC?Q^O-9LEnpi4Rnc8NWWIi|IRtHC#S&(BU%JW$}udX1^= zFI%eF6B_IN6&ozf&HF5E74#?{r1{i*wSbe|!5+S?Tju=eYgwOOy>4J95K5ZkWy^W9 z-Td5tmoJOGlDJFwOy-7>FH5qA$L1-g>A@bpEs(~MQ_BcUntjR62>$y`-^v&i>PU-# z?#<7L;T)bF@R@JsUXqw2c93%nXX?tX-m#|AuW>Bs3LGF36=#_`203n*{TBS-%nhY= zE&Q9VefL1sx1y1LP$-SYwGCy)rdbpa#xRk2)~l&FZ3#zi@);%7BlBV*kX_Wbl9V!hf0~EFHgm`A5Ym%L2WU zZ47XrZ^;l(ZY2l{pdB?3hv#kolna+i0g~p@CSkR^;#?gdfAUXUnH-Wc@@rV6PD@Qq z%`30=@cnhn)O&-$3cyJAO^W%ZM_gFK)CF~3K{5eCc-NSTLyk3SV^n_EP4&R)k0Y8X zt1GV```oMfjnaPAo#E&`X-P+_fez)MT37TMw>%e-rD+GszYFdNI{D*Q*4Kj}g6^y#{+iVxXhnfF^0tm%!6!(51o16<0` zNFOkPPc)cYiO|`<=UhpLJ#0j+l1DYz<(jrBXIP9`q-*q}o zyaJO`QsSX;1;rQgsjQ2JWID*)$Vi#h zz^m7`fuBA2WgvR7YG#hfJJb8Oo zLLvlg>~(loEAELu^o_g|xWjn}8uSQwOGK37Y$Z_hNJ-QO_S_o$VlP|Al=@1WskWPP z)tu|{!Almn#&UamS6FCfvs#zx_~0o8J`IE3?~ zY_Qf>Q}bc}Q3q}mL|w)YBG<2vL!TLYRsYz0`gG6se>@z~mi}s*PQ`XyDHZk$ z;=7C~lU~w0PlWK9v%r!7irDQ1MJU+4y?Q08?m3D`Q#{itFq)UaRJkPo-~_*#Gd-ZT zW!Nk@SO~3Z>Fzr>E!OXBk9sxX)WLSK>r~E#Ed6Rto%Dx6zs-PQDLPgx8%j80MVmJd zq&yKl0T6W0iT7LA%BsgDD)@ZICk95hS=ZX-{@F=u#pWps+^o|GxH*31N*l($X0;Y1 zEMv(nt;)T7_qq@I^iI&`e*L`D86m_&E#_Er$`}aGcZ81fBk#8VA`4To$s9D{--x0) zc_CwGz8}HhUhfVaaD0Jb1sO=F!Bz=f@wIIqsh(hLp2_RUOmO0d%?&5ksAHXe!+u%dgFl0a&%2(}^)ZfU?q$bEMdJUJ>)2`p4IokoDQj%m;r`wf73Ks( z{f6DC$g+4z;2?r>jz?!MILBC0v3Xxlt|a7a_T^NhU?P`uI zHI1Z_l$0r3;LbBI^lm@EB(O=qNqEqYt-K1Pd_C+p1>gPKFnB);FU%Cs?1B_U8ryhy%*@Am6>XRWB~T!pw74)57W!w5({E@? zJ=gj9`42$}ON8VjJFXPf(+HuuwV&pa-NV_JhT$dUPV>BvwCQ^~j>S@KaNGoipNk}9 zlbL%`FQNl#&}y$42Z!+2{wwDJD#`K;c6WFblK#59;|aXVmJ`MW^@Cij+bqbX#{nDg zLKq<9=@}F&)!h>SY(3A6A-Mz8jle84t9$hq#mjQ<3=yD)FTt1%4g(CLcX6mY{ z3nt#5*+uQnlEFUH&CPe=KjK^V%%0nv{)9l`^-1^c6ux_R11Z4bp_~cI7POHuCrtG9 zgMwtwK;Iayy+KJFN+Hi~Upnho*;4$&>sI@A?es`SgD?P<Y9P^yLRffu$|X7ZMfRzWzXCa+Xnx6W6Qv4cAs;OL?{l{n`>9jG*Ns%?dw`Pe>{t@ z{rP1>o0^ZCcU9}zwWf7jR^ab;?NI!g*dD*L>PF%~TS0s{c zXT)|^!d_)qRg)K@j}fw^jU5jG7zgJ7XMoY$nMW3NR(fLRUf74i#SZ7;6_nX9tEfHG z2c}1dnV(gh3fz-gU(;>pbMA1kU_9NmrBrO$yEj!Yp<6)H zmq-F2FtN5uhfYrBp-Y(h9X>@J5#Wnlv(VI$py0Q!Zzl8T$`TiH*eW1Xw-AO?v%mBt>ugXb(DrOtM zRLGj;-Cof6-vII=95?c1s?OlSTj{g!+JfhBAJ0Wd5bzc^S#h)$7BPDIf3t`upD{}R z{CObiNSoqyLy=;@fBmypODGw^pkgqCE(D(r3VKOi@5uX8tMR?+Uz<#3b3+O%k`$P1 zL+&s2w*?lE9V{KbQ-^!*Ukwi%r44g`TBECB#^tIew1tbEBOCY|IIBIJeEv5F#2|i; zuj2bf^wW3FFEWT2wrP2%gzY|cqump=eL~gx{OBRyB>kg#P@Su43A3AeI_LIy*3f`Q z7{CjWzPByawVF-yQ+F-gC4qHaDhlSI0qmcS3mh$=8^^u&BdafBnkod}h#I6<|LpCt_`M2XL_{ZG30R;$F{|lKDh& z;DR+Ft#frL97-(p8xl$^n`yu{h*7%qgh-1Z#4fwE5qDcI!JSaX)yQ2$?v}URC7?UX z`*GhgG2`zf&J2?xuVh!bb(lMS`aZjn_%MNfT<&jq4gV}Y^NEWB)8wy0pSCp3p}Bf1 z{ICEyvg)FQ44$v1Rz9emr-<{X_p0wTShQa`IC?XCEdxm$qo-GsSE6vZkrq|68zv8t}cCzmSHhYy;J zUA$xoW)d(Z%picthAilcqFF>#&9lGDlC+YWNGP#FvUT)jl_2~|4q@&6UKhh=^@D-0 zpjxFkY0qAEZdq;J!x!tvUB1>dKkNE@lyM*&aMz35+OJ)?GLEjB zK{{cbe|dR-mJ8A*?inT#KXmI2D>vvzj3mE<5CzuN&A32@EDvN~QXTCwAgUq^_NGXy zpiGCtLEQsap^pMd%-CNW9`ZriF(+ zoZ_mZU;fghxYspw+w;bh5WVo&8Z?tm1+6!Qg%4xjva%pU-6n1# zA}nTG8AhKv#x&$3r|2{i$pK)ENfi_g3jLkhf7fUDU}F59Jg*WXgrFb1Ny0V-B;K?{R0H5r_-KLLhC2kKae8fExoaX=!9Cqdg;;wwzAs= z%cVoYRxjd$Y3Qsi%oHl1dN4fV08T`eacMoEE9Pk;`IL-groRXIS}XTk_m+xskuV>S5uq2^dS^}gs;e82RlhLB#%;8icZ$Lh z;ar1Ls^R|mjfcwG0Mn-3pe31FFp^Jno186`?R(Lp{KJQ;?T~@VLRJiR?kpA>qNi`) z-)nBvl`E1o{#(VsPHfKL#v+Lt(EdEpS==I60GL1J!V*~4GwbP7?h=;z+0L=dRtqgH z^-Z&I9^ISYOuGvoge*96qFOCokBedZckkZqOBX=w;K57#irlPyXxqh+lGWtm61Z3r z_WkViq^Zk(tRI}$;9(X-bw0%BR{r&rAj7HZ==h)aX>tl0^3&5zmBd4L=6TbB6o->6 zu9z3X&1o6c%pmM5Q1qxqN~Ch`5}$F15NO3j7xq8=FD{-2WU2L2%x!G)*d@$ku!bD5mKeM~_DepOG z`LT6VyDEEKEXoL3X?J$m&qf}P4^?Yr61`T~cT$`+^}d_xG_JgGzpl=(rYDxX{%`W#MMp|IaXIy)mKMb z_4`WT3rZ=4g0R)!*Rd4f8}SidJ^o9Z2|gm)Ov}bQU8*Y9g0S@U+hhG|-TL(s$Rz!x zSi#Y^3xxMUaBUte(V{=p|CYOjC9)fDGGmh#(8mUgnnf9JS(gHyp1I-0a;ZhjkE0hDEk1qOz--7 zO;S<2vh5H|E@he6ERw9N(XRj0ch;w3V|Q%duJ!eri<Ig-a z(rg#5bUqJuz@^4O&xofkE$#0-!boL7*sOnf&0IW7dgPDsbBq-cVAl&BMnBG8<3H@s zr!QvbLPPIajW&k9DmtJCI)jzf7r+vN*NdMMcmpW*LHV1c!JLo*S1@yjv+zBCt;;U z(_aNm@el1rv^6X+B(L%G>AbU9?@OF1E*!R_1>V{|_XpMJ{=9{Chrzlg!y74kbrebF zJ9B%dcTO>_s!vwu|JV#>`npc_91w85DEXptglB_EcU0BF8!C^vwXc5Tu|2l0e|JGU zvE{2{S8LBLl$|N;$%o&WE^+f&^|!$@FkYcO(O?B@a`%YxGS#JWh;K%74xrfXq2X6( zJh@>c*SJBC^&{vL*;xGiR8@K21Rm-Y5^qOWe$QF!9EwL@VbW|E#(F}B-0|>%1D`<+jD5_8e5C;seq@|U@eE^Qwf5_soflHtq<;N?$RL~rba}yI zqk}@+?2{vzs3<~haMt9xO3v+}-^BKZ-;v-?RYRnEQ)nBjCFi%|;}>sv1T~gHNumMd z_D5jZ(FK;iq$^J!tX3|B!^StYJMXd@%x8yyDB$ebl-x@&qa-6IJHCU-FUQVvkXnrQ zvpEF84n4SU-~7jQz96pv*XoB3+itl;WSto6%0kEL^2qxM3MWDGXW9->D4TmZ)c;oY zzkC274zKA<)J!w~D8@KUp%+KeD|YSEXVM8Y%Wg8&^TgbLdM}<^P#ki_F*)`UTRYhr z+zW&uuEPbr}{*9h{;ItB4;4au|5Rlm(^_U{9B9f2*kA|41^x?HnU4D@~bZWKQ`3KP@1 zqw z62^i_kpyM^W+?XD`*8muSV@fiW;tF!b}uC?q-^w>rwC-mjHV6RoPA3q8o; z?cI2{@%qX8dhNR6T+#=SXz^-3D{*vGaJ$-iRowf!X=KGH-*-I5!oau8eylVGe_Yww z8yoM}OS10Y<3N&C6H9y*v8&{aty?X7-JvOB!vUG~)h>DSQ zs8O1}m>ytulk#>>WYnd|S^XwexlP{rzZ?RBM?cVwak@3=O-DZ-Z5xgLYuCZVVZzj!Fe%5VMXnl7yyYu1iZyPsP7NYle@ zU}uWPA9Yu-DQ?Z&_1gbzA3^~_Ui*r{cmxEoAZLV91n=%41RT_(D@-JbB%W9GV`jar zUD5~4y}BKqoA#<1dqm|4-Qi$4Ao}-W|Eiz&U0eYmNF32otfH_3g@y0RB-P)e_Tr>< z;(P^BNZ|-5){iU&$t^(dQi!SQBJO#KQWN$F7#!YZR#hqWq@A~5KQ(OKBB zz|JMgAUxqO2B9YyG=tT9w}#x_zO=M-Y_sj!t({6?#luB}pVTcKqe^&%wL5nvfnrib zuT#T@46atXbOw$ViiGz1MFRqYX(AWrwCrp8Il8>2Shj)s28~-wU$bM!j_+>{OjDUU zt{C@mvCNHBRIX+<#J-Syjbdv+q=fi$7?!MoxU{)fFRUR{Cn@Rul&D z(=wZazcQTr3)-i~y$KaCaS??&dT8C#Cr@0w*rMuZp3Wdm%^o2z0vW6C^={X1#6=1& z1CCTm@Y*vVJ;swz%i4A5(82Cd>NOf`ArbTD+btJh>au-j8kUZ)zn|^3>)}rzdxa~< zlG#UeerJH#J8raHcIwBR(ll$0CPgDIoA?;^@a^%2r<3GilU~Zo+Pt&dL6>PjW*wQP zF&Ac#?1iH5kLrwL4Na)!P+l}dluY$Ml(qM=(tuL8G{JF7wvgjs3h)l(srZk;<4S_S zeII|5RQkEmrV8-BmXV53F~!b|bB0|rO^NCMI%i9vAIn1~KWE3+KHwXwQ0!&%3}Ph~ z>>j-W*4%J=__6-i_DhZ>Ptbo;Xri>{?3(fBo8x-t4yu-xU2l5!VGTtMbMp=Yfa&yZ zms0D-o0heW3o`I42hyB+D%(2l?Ik0XwVsQzCan!EO^+?r{R%aMUUYH}Z7(L;jd@-h zwp^?GTQ*%VuWA=HCm3de_(W@(b_JtY2p#Tw$>Vt^rAg<4DoPoaJtk!Jy}2PEIWj9@ zEYUlil_cKTJJ`{#b?(Q>g)^ID?^52L#aECsqlDOi$RfzZPR#(q9Qf*-j2ltt_gHwX zSi5%SK<;MLSF*fjl$FganQv%87f)G|Ej!&si@J*6Hxe>vQVy@Xd)-7i1yuwfuWLGw zzK|{i=defH5+2rT*E z8OG*4s%qutm-$b{>>CBSO~1ZE04Q`40(K`T9B(qiYeOb$&?B%+zB5gar8t4I`?_v% z1>Q!7vuDpLI5T2~J21UagjZK|K0Eh^d>5j}LA&#{YzP}~dOI1-LM_;{QRU2*vE1Xv z@Hpn>_QGE%9(J0(oRK={=gh>fmAz{C5&ZL-bQ+ns=mHa1r;$$LqnQkqq8;so0n(Mh zC)`xT;k13$MO`iJ7+Oe5VjW?|@&SGSk~|BlY>~)~f;T&$$M`#!G+#dbE-3ZDz zcX-U_t~leM&%NLrci7TvmlJk1*i&ucVQVWs)OwKTX#NV10jF)cST-`8)t}~HssU^1 z+_s6DnuqM-@|^z?fo(>o9x?18oe*7oIJIE7=!^)r)}MrPvS8crlkAq$XXTU=#qi&+ z-Z$|V1AeAMbRvR!_T@jJn1he;9Y|0;YPG5E^&Y#_j$Ir4`eX9SMn^(rs50J-ukLBD zV{V_{tP4^gCK1=Pty<6J74tR1<72CC`IM7PE#NG=ZSjO+jQ1~^u5ViQ8zpH;N%Qzak++Z@zm ztPkp)J?PrO-J7d+8!ASdpZJ=P&k;URU41F$NF=3(e?mMk7vqg|zfv(VrkdEP63{aV z^S-x7edC+;Ui@SI4Z|t>-26P#tRsSpikqqT(lF5CSVTzj72yw`8vF24Bb z4Bp&y!`=z-B&Fm%AJDDa_B(m*2bB<+^Ipt4TV?(au}TxZZPI~+hS=xNH9DT-wy^^F znA# z@7*ur|EXA})ZxpumTKl371ge<1Ff7|t{$ZHH;$+Rs8#TDCd+&n_{NJyG7g=D8bpBdT zL{LDn7+HV)sxL;?GiIzC?cJiefpY53+RxT$?>tujnEe?2^j_TzHw6yI{Msbr?rV@7e0&nDZ=h zXq>o6RMP=wp;2S~7~5v0&IG{@JB{qM^vA+Tb484zBK=^c9xxD9=|NUj-*n(S#>Op} zNYy@&BAF*_1f?B0`Se!^B#gY6;v5=gQL0F^qGe-mac%|c=^s96GLhz5I6n#I9sCXD zGw;a)w!;m2bT!_??aT})&JO@5b)gv5VDDvjsQ{Z9h>=Fqr%U!G6WYm|3|(9;&0y` zbJ7?k$h%17#xLoPn{9vWxSy_m_3EJTc@fKrO9a>2!M-)74Z$`Y1e_l%mH}d&fOBmDLYZkTU(A z43jsBJym&KHRC6Q`PdaItVA7D;EI4xhF8ScQ~bqI!msXSeMS)iDa;7no+EUrpx+xX zP8dC*!e**I6^`ApZ*XOUpg=j?yJa|&>0GMGRA

    ZoNx0BBO#v2QiG01!Ln{*VL|$g)OAlH zV^)}3frY;glJrQB9l(e~3Pl{4_Vnzf@;$J>170L%xXZygmyt~7+>4T31|(Y z{H&tL?IU`!mDO>9yaIITc@`i~zqFJESRK(eb!+&*7<~n>cGzib7Q-qqxq&quYDK`n zN08<)T-H6U0|PV^5lGgcOOp5rv}DMR1cHANk^l(iwns1mbU)>mbQ zAyO6JLr)kz!-hH`P*KqGs(o8auW~d9z>qp4B!n4f>(~GU9w^R4&1(wP)quhQp9eI z_mb|`rk_Z{mvZo@#eQ^}3pswF`aX$kq+73oMbQm-)xA{2KHIq-j$sd#Z4LYf@Y?78 za4X>tKce~Tv-~tT{;of>K9S(`*TOx@DVNC$Z$^PMS^MR;AJ}N}MBvXIKhIoz);@a$ z!-c;`((l7`Qs#d4^=lI1Ni6JNPMsZNlDNmtZtXoQGoU)^coM-MU|oZR+-K _GB zP0`l&knvQjfZC4yho0lR`!@8|IQ<|8Kbd1SxGhh3l*4wv`R%>GgFUFRfW7Z*kg%33 zVx(Q+$_p50P9EU5fYX@5y@x~eonX>BoL2~_$%FPz%>$CIVT~_4toywt4ub;&fzovf zz^Z+Hz>5)vJ*WWR|I`rZ02@a{x{VknSW`P8UZ65Eg_s}5faM$nJ{KUxoJZoCkYNu2 zy^b&%q5Qov*zUsGFyT#4CWWnS4{X3;55wnuObOMb0Ja1F$FUPE66)N*-=pm51~+s^0DioLtw6xcOlNiuJzCq< z+1WJBfF&Ame~brUsjpjG@L2J74H(L9W;G6sgScT&ev20VUqsCh^~NXX!v_>FoNPZz z5dxV2*vy!6({jU(na84;2%C_eejZ`#!IlO3&*;Fclddil4HAH4!b;*Q!1w@|O*#mr2{clsh&>tUr}sxJ3czEo(>-Q^$d#6?+n+x4 zy_}_i-Q^LM2I3K9tIT}~Y&9U^3mjFf)?xu2Maaf_h33~^hh38fRs(3JQsa>lW%5E* z@l($P*i9lA1ZFa_5p!69A$(@Uzvplk=q*>NtMv#FgEtU2vh1LhMDN041Sy|q&JY0Z zqh$Z-&ZQ%#LZBZt`hY|v4R~UA4B(Eps-90h&IP6hXa#|IUNRI=YbW*$xVmAN$PEU> z%3^?)016YZk^I8)#vH__f)mJ*sshr4Ccx|Uk5B%;(1~&y)e<=a%gQ_&mk+U6 z#FhfQgEqW{F-(Z*6yTWunNHmhowX8$#qYnSg&<0`S&S>};75v@CPWo5%Oz=DSqOEQ zdDyNu7gp-aOr%By#xo$ukkA2`GkFhA;C_5fRh1OKkv9#N)@D^#2fMgL|?jh$U%n?^r6EI zXCE1?k~#i-?UAOUj7(D=WY&m$(VcjPjDGFyCxFsvA3E16Ye3nN_?D`&CPN z5x|%<&tyRVp`IX!jn6W?>oCs+RaPF9nV*{y%ok!vSkw@J!x$d0D?l>`&sA`e98qM= zhN25|?rAaI5<$J3(iP~xd#K*EiCZQqb{^> z(f~>eUVm^goZG=wk-(v9>s?@0m@)9dK&Q|38OGbAeLDzZkrh9PQTF9P&ZP)Et-!2C zSkplqu2q+$7nqi$S^rcQO$Mk6?ECXLsjv}Nmyy4R^iNP_{HIu%nrs&32XA(jDVP$i zo@?BOT?nGmSM&jT0Mcq8JWt3g7STZn2g7C0fE`1z5A6De(waj%n7K8d2y@K?*`3vp z%uLBl_#PB5M70gO05TtNFJ?d>HIQ1U`Y5#PLS;JI>Vu#ii~Yb1KCdT{b>q{gd#{HO zSq0#80k^W)`s87a3af@oM=RP2=_ndN^MoZsDnZQ%hO5S@%F2&z!C6Xq=N_TcBbc{of8T{3=2q7^{f{MK|ogC@OP`i^cx+=0@Dc zwXURI)cVwu0tDTE5M=mCx~yBO8P=dH&NoxP0K3Wo9OA>0dR>TqzYx3ClyplCg%XV^ zP5bm9htmN&mF~EiA0-GijEoL~Rz7a+&8`EAg)swb9w)BK#?nYy5ziXdAq zMNmx0!+Xm8{p3XWt;F5=2b-=|C=^xR`!iFvLR?0i6^fPh!^GKR6pg9HMr$Wso((I@ z|0!9f7?E6$K6mQV9U_!8i}GKrA5VkH@t?2W%tqajI#lU2LX`RBO4pXT-}LXvfy?4`H8g?7T{Qc{qe%x3q7uRlKXKv+ zx%!VA4nj}-=y61^1~0sd<|ty-38=0TKj*%)jU>YO0}?&L_``$)q?HWyLdGFDrVC~o z#Q5Ku5LuEV(Q5%z^5D8MVId@EW@7RTmNHXdCvfE12IgE~{D1za$3liYSe6`?fu#WW z4ky3PxGIWxg~E{d0g;PCHvy&z?*p^G$5PmOLs1%Y{D%<3?WqN-Qp8)-e($@-NpHI~ zCUeg>;MfY*O2DQ?D26Zuv54vtCi9QSfbRypln7NENa3lS@lXpf+pylDOHRs&1dmU! zxZvbHbpl+?wE@IU0{9xZ!JwsqrPrwwT1|d>@TP$%i3Ey6N}e1@&wz0b#rCaW+PXK(x;x|+QmI7}*I>Z+dD&R$s=KX-ebySq# zU9AJaK20{}Qolj21rl{cRb>LK zWr3LYFSxZ}P6KiOrOyC}oe>wmcT~i{#Z!4lhCq9E0&p;LkQ*LxlB$4F#0I{y(72}V z7cTaLrVixh$l3~i3H!p93koN!k3n7$q9uXV2P*R@=qg&^n3Q^-U`Y)B0I1CW6ftOF zP4%frJSw=;!2IAcA1kJR+a;$3tOB`2MP0bKj(pZAULNYH=4Ic-CXM}y-oa60TBi;K zHH(^?IS%oi=uVE)Fde3r-hY*SUwvz8z-Ve8F$5KLcDxJ!!O<^a>k)meZ>NRKVrBY$ zsZRpneb%0}Y?KU|o!up`fz9GT?I$$r`_IpyO_v`&+f|_AyvrbSpZD7adU1Wst0Tpz zj}-M?{esec8-XwEmxH~H=4STt-3yzCRXQ9K0CBu6g*v@qDD<^P-1fIL!z#JI>6w7Thu4EU*mA1F?3f@Tf&dra%G+c?%|?Z&E6B_-xS^P$`St#Ekzg#7NqANidIh35RvpBvI}tOR%6 zlo%Y8eXv$lR73#~VKBvu3|?4$LA=;RVW&Kum=xm22btsfNOnRGvjDJA;cQYZ(shE7 z4msKo3D;;I0M{Vmbc9eYpB!K}{D(QQ_M9&=X&?oQh?LCCYsNOPU<6h^0^dUugn(E8 zX>o~&+%D=e1LtgbNeNjK<`0qrUzl?PJ<_mpyM5i^{(VMZg8%ADQiYut1qj#n*B!%; zg~BFU1>94rbrc2wVFA#m7AWak#Q;&kSqydwb)D*%-j99ODSg%|V&tzt@YDk1N_hwb_?q`z;z@xzqhbnQB(9a= z6rG&Rum7h9Kh7ZERpobrF_mfmzAOOa>#8vKRnIh>wV=b%(mB5opyA^9y}%uu(Y)SUhb8 zZ6(0*NczxcAcL4d4RUm4_l-D#(LG|^!>b2wZV?fN!*eQ-1qMt>7NA)Od)NY%2ayuz z;ps3R!F)nbPY>y`>Yies!5qZRb1I7FUxvM!f;QSR6`c8j_Kxrrz%L@eo(_%zkk9~i z#;Se;A&w(iv)H5*=5$c5f?Xy=Conx>1&tvhF+~Q!jq*+~{)Tbvg*AL?M~EnGV=y-wwBj z5W&U&cJWb80E(#qUk;XvU4w&?OC@j!fY-UIuI^yI4L2>O$7}i@J~04-Ai0;CbpHJL zPiA&>TDCeVD=RY=Ye>yF8$v=pQ+LEJ=HA6b`cggIw8@g~>FNrXalfQ;($6S=VP!cDzj1ax%FGabb$8eu%3S>c|BSy=ou&nJ$pt&y&I z;<9qRvw~eD>hC)|Ck|)~47{QehWsZd?}`WI5Om{)K7vBW$Md_meJ*(T?5x|; z8mG(956|V8x3Y)smX>p>1_>t;BJA-Et!{>NE>?fMZTbfq)g4Q*QDbmUYXa4hC#o7 z+q<~ij88rSn<%m`hEina=2L|n;9f}uI3UbVWS(Bdv0|+!c<*=AVfO^wJ0HVey?OKb z=7SMfAjyNgwV}0@!Dhfy>s2Q{w70wa6>=}(FK1?qBxMXHmrEJze*V1EZ3q{15{Kli zy4E);2m7OVz8K7ssJEs#-@Q|G_F~ovdV2QPKTrY!GPZ-`hxIiQrpx4ABmE&?mpolW z-O?&^s9-VLAHfhiy@;an|Pq3f-&B3#8zC%rOez>#jMU{$j<4<;9TjXp(PA z!{d`?JGTy&7yCbK`m+CCti<&FCXdq~e zeAHpDQN>2C>50VeO#J@M59=zR9MZ|j=|ksiG9oPi?;O=$;`X*Ytk5CqngNnk5wyUj z-=n%$=#h3Lsr$+uh&*Bs50}d?7K7$k4BXshY~K~Y`-=Q$Wo7j6@&YLV8W}BE&hnoU zSYy6@w-18BfU8uRR`q}a-g#TQc0u=CK0?Aoh$KnV$&-2e?p^>E%MHS^y5{Cn8?!Qx zmi4uDb;EEzng_!+ZM~aKV>R4Ux}s;(7=nV@GqdJ_osRtPsj0HT!?<~$C0CEl%;WH# zsfodJ|AdK&J>-~puLQ-vaHgznWTRX6n8+G6;-jVgnR2T*B~1xb#;nA|lHs3udLMgi z%kS*2_v}dV@Q`0NoC}+qlbUkNwdIoY9e&={W-)6wz4rHOQ-!OMBh!?{{$u#&ja6QK znc0gFn&?4;$JNzI-rn$nH)X54rBX~R31K(bElUiX_V!^1i#fK~n%bSkl&`aYL?M!uh=Jk6 z?(X1O?q)i004eC7ew?V}8MQ&5lw=K88>$%*+=$n$W{z$p%2)j9>g$J_H|0Q)P7i57 z7kF)yyee4$=y81VwKi3y;~2SfK-NXC5CG@|<AXobt7tlg{pxT2lwgA1)$YHGKsM62|5Wprz;BKW?&?|A;`jR4EHwvTBs z4tyLvo%+;bWWG@x7qqfSQAKxLQ*%GM4&$m$WpL}3G;lKd?iIzo*W&$9@qyVlN=f_0 z;OMS()m!Ka~s5 zxj!cDW)N6o(LoI&H}_fZ7^!+GN#t7ek5^~l(*@cTHa5^>O1`+Bfgv)^%XOdcP7Qb% zQWJx0o_=#%J_)rphG&2E=FMf4>%LvDH3hQ3hlFLn@6F79>x?F*`{e}%@-M=->iV+N zQK*>D6i`ViW$NY)#`J|V(Cc6S{$+`Z(s>0(2U3iCdOWzATO`FOY|P%7-Nay($%ltrH}#6J&-J#U|8KMhV)WI76^(7ftWZW#Iy`# zwa>H_4o@Nd-##oL5U~Zq7o^P)u?PAhNJA0lrA;w#Jq6R?@!|oaXvD*E*V)Z27?{;S z3f(LQ4FAZy17J8Xk3qPy3c1p7!SRy=J=y<{OqkjOg;CC3ScKV^# zoRZh+Sa4qaSDX}NKOsDCQy_aHJ}=OfDOjcr!ORRCsooFs<%awr*^9mMH8-Nwpyeqm z)4j8?!|9FTN-cnF^Z;@1q7n`?>+1Zx>vL$TCt)$V|GTj1^e+!}perL3yVQ8lTD-d) zH@v7Nx>Z(L3FS=F=Di0v#DF$B=d>X7< zjylGD#Ow=%+A3m*Y7n?K92`oWcey-iKt%z57eNTA5WHf>=Om@b6Er0dRRDyd!f1Gs z;}!Va0=qKIGwL!|ygV!boFGpc%tX5aypf4ExcTsT_Wky*VOd8?HCFk9%T!|r$ro`et25tj=Jxio zVCU3Iofhw%cad%=E`qz2Z4Yq6NyE5~mr?5y+nJt0Dk&sjceZeFFRT*h4Sez@Z+V|kpLV<%CMg&)Ehiai`tGuc`w$eHQ`_7x{?jiDMV z%xUdfQ#X};uf53+XL0f!zbk8xud%W*nqjt6p>BOjrR-@T8`N?o`^6>OP=&*u^j+Le zgKD4ScGV95MC~(K6vT@-N5z(r>E}=?;P>RP`vbo$t%b3A^73xGJARRMC3b0ObxcoM z-PqyC1MGIG)Z9WhlWeui{AKJ=O-@}f#c0~-tOp#^Y?5o#_-%)I7&owo-+%q0_1m^k zU%&5HZ2O(N==kyJ!TgCbTrm?saCtSO@{@*b;7muu;d@L@PP@i5FtF|Fjh_jE68D*~ z@(RvymmYsLrl_cKAA1nisSr638RDs3=9*3Dw;@p)_2$-D&+5|Bq&YcDE@Q9G+t~-} zm1EQ6H8@`KNX}Bq@{y%q{e#7=oUk3Gi4|#KK)sIDjfjmkB^~0BeErGKj%qU+tvvBb zA)MoObExp(>O=QZQokMd>$^*ZvPDHN21p7^C0e{znSCi^-jz&n;@3H}5-(i!s`Tta zpMT#<%`g+ZGinf~fSX!3FvkBiqlZWA&Fd(k-FSC#@eBZ#X?(W}Ez2q*uu}g~CwFJY zJ{J_Qj^TZ|A~*%$_Dp#F{TbuDrhqRbP*gOn9DRD=M&kQ7`Z3a@;}kuufE^1|O+xI3 zLSAr4NPFzITwH&yTq-_{b5~w4adu1{9&gIkDn`jUw}8V0U}sK{DO_s|B0o=xj?Viw zDYCSazAo5%U~uc!dEd1Fa6BJik!^v=Mgjk{sBW$D3n<&YxSXy?0&KIIq$>(ua*;CBi3d2lF~vbj1Cov8S+@M zg@^kNt5eOZ&ZIs|W8!Gh%pm)?kR*J*b=UGig=pE<9??hi#j6%Q8F zmH6w!5^(smPfp+Ppxx(jGOuc<|BfCx>+L|hVzEJO=$3O2SAG7Rf2-7E;~}reP2_!b zgc{FCv&N=koKc6Xf{q@6nI|T;Zdvw82h8pUw$=C7gf=dbXq{cbKTZGTTDEt7FU;i|l<4B<@4 z>sj;4nWepNi@&~EwuNENOs^o4Q25ytz?vR_^G3b?VHR81#hhX=6FGQWbtJ86S%8CU zT9=?gpA>)S6mRn?b|-+k!WH+zM-#XTYnaj+y?}*0e%p|=5Wc^(KOMIQ^fK==;~zu^sqe5$94(OG`nnCM%x0an?T-g@H2Z8Fm>FZJNIkGO_|wTEcam>Blm9)+rB zWfDaAizw`e}mrMR@a?9<3NQ>;$CT zx#u>HRL7~7SDP)wozXx1q2=9e61W%snX?UuSB&^IsvVT+=-*3YhB;*!*3q+II^+xb?^6Km1vVHF*nS9Q*|wi@ecoO~_D_$^`3N{=dt_7;$#ycZ4Lt0+)W<5^FN z8nvW$p4^%R{uDYn^5R7r{7Tiq$|GeJ=3J*9OiFVN&W1%ECwbqcdI8vt5xXINy3ass zNSSav_S-*w|MjQeUcVoJbgIs+se_#{#cSWYs&?I0RVMN+* z(5~e`cH<`A4bnO%@^A|bD`OH^N?<#^T=nK*QaI2nn)9l+Oq4rdLXL;W=KaiUUh+db zNPDX7S3yYGu<1j5QX%@zXR3)Q(@0i{pKJI1Tv_$>f@dvZnYPB`NBbN;e@y;Y7K80| zVdmM_Mt{7UMk?>_{_0#gb0&q2r5zp`7)QPsuLi*hc(bv2pGkIfEUy82SV^}p)c5n= zrBwW0JDl57m<1ZsZbFTo_#ZC;;832N@*@rVhskM4c+w4%m#rRNgIgRzcuGkD8b-$` zTpRrGJJMftg*F4ESRDHKszi>Pf934r55E-z%d{k#&J=#6LAX#+Q*TtK5TfE8Z9EF5ulEb3;jYKnX)3DSUW*89+u-* z<=y4oaPgZ#s7+0IIp5u7&fBTXZ+rF^D0K;zxtV~$s`=nzxf-vTlF8&XCeYK|=e+6}umh>Qpi z(&9Nid+YcLoA--&MQ#Z>k$&kk2Yt{Pp z&?|825%Z@@T?VbWO z71fA3uvJCY`N*1s)gH{15VtXZ|MflFi~=x}JhFqusBxxL9wdauz#M>3%Msh8`LH2` zJ^;eb#6$;swjUiG06Q|xbg(~tBnlJ}u%{Hz|5FOYA6D^`a&ZZBPhpvr>IC-Ipx-8! zXik}LufOWeQIOZUHZwW_8|*XE(kwv5>luIr`*ZMk_wdqQxymW({J5iietyT0Mj7QI3o@F5#IoDuWD;= zDY-W00eJ|Bv`cZS3RZ;;J zfnL)B#xcY*XOR+M-foikdce|=+_==N>#`xYL3Re4&SR7uBhL+B#iO2nYXbO(sZ*;K zAkjfG+D=2Ri0qqAU>}0O&HxQWOAn;w z=XZ_&b_Cta$3t(fi!ggkfT|3z5 z!K?@ETvw+11mhQ&pV0IH`CuI^NUj|X8DLTg`mlwC##d)L6C@&6>6As!L&)|Nj2MDt zGiEk6uiVFc7&|)B3>^O3pj&bSd=8|LGk{Zr;V*=Hf&UF;-6AezAt7X|^k-Qh?*Y*z z|Kk$_BoFotdGJ4Y{rX{0G!L2adu9}1x@(2#5lGu4smASx3oS$OmOlp z#TKNKcb*ZVp76O#2hqbz^piYQkCWt^oel*vQ6Sl(jAb zup5Yy8+IpOF&fLb6s?S8H+Lk>suhEu8I|vT&to>JTmIyXqo9DKG~L&;v-24&nm_j@ zIs}L1<+FoXKd}kUIgYV|(eD&htniuWoKHP{s)VP611j>qk<>G&zs1KW-Pmu>Fy6gC z5R_w}eU{H|cPBj~g9H1A=ZB2B&0c~T^b5BLP>?n>sen662Tkhk z>x1RxKFF6MpWAEr{Tpy?HhJu&wss!3oOi$@6@{tJ{R6AYnEwxtW%^U1Cav#!Jg{Z1 zjdnKH6>DgGTo3^69;;QgI?H~avobB!;50sUN;LA@Iwy7&KiaZWgSPB)d3qW^P)|=j zK;73{{AB;Gu3XzcaNXT5&717e=A^8AVDX6*C@qu`Y!@NEehBci!T#6nKe8ZqKAB}( zP6|M2-uoQQec3Y9%u;{jH({6>+v^bX{`(`qq;zq)4@O-M5C*hJ`uVcn%bStgL^%SW z=e8snQc_ZoV<3Tn)EqYK5A?AAJsA&?^>^=#yjQ&>ZW@g~gZtWtO7<=FTrTbj*#1F~ zTzwBWr68=MI~e9XTT<8fcyj}*w$Rxyn~`~`DMfaF!ttl!$mzffyLOGTbsi)9@bgQ9!)Up`=2&Fy zPoGz6Res}ERIfYnOVR*thYV+kR&vxFjMI+d(cx<+SjrV5?)5|#)?8C;{xum@y1b}n za_c-meV1RbRG{wwq*uL*qqrk)%jV`clH*f`8?yal#MeYt6B;yftmQnB7gG-2O;Ft1 zViWBUKE1rV>2)6?+&KH{G^+f*y3Xz1=c~Te$KScl%q`!XDpaTn{yQA3h$ceOQ?aa7 zQt?kj%0~nYeOs@x7#}SlG{n|eHB{%Ku5=yGY_m}|Gn&e;dXC5xexVIn3wAV|&B(NH zOZ+Q-hr|x}18LpC*zk)xefS?&>v_tRCiXR7d9bctA?+fc11 z=gu#;#28zoq*u@l;oO=&3~f@Z%O}3M1%@nIX$2TZJ{44#rOPf56`LDE!pq$3c%V~) zwH2^EB2z45s|Aexa_SfQ)Ly?Wa&I)%M?oCSaiqCiQQX&;J9|vKX7-Db4>5iW&lmx? zLVIk7oVDoTVATPbm~ia7SEQtn`E9BV+%z114u8Fze{k^L?|M_DuZzVUh064FMq}T& z$emI0z)zn>GDWmM0w}kwIS*I&GNh5=WLnvIs!Rh9NE=_Q;)JQ!skrK3X0)I$Ebd z4vcnid!TZfzHyP3&fOH}ZMCBI1(0={yd*`>{qM8e2~e?r@KC{;AY5^^^%>Ac_7paK ziTz<6Ch@0)&Ncbz2beW9u;XN*O34z7OTFXwYkt;Fx2BgpbsosQ)75JI%Z~9D8(|D`FQeLemzc7Vp-ffQCz_} zcDPbZ<4mAEwjp42S<&JX3jnsHk@xSofbsC{wEgm=Bo=H9{(Weg8X70n&@iIrOAC}S z^`+4(|Ile~-?Fn~6t|TWU-~%y<%{$eeOF@qmhmF@A>o?>;caq<0G*vkS&YYJ7{9%@ z20}gF3SyhuW;L7tv`mK@1EJ&0Ns;6CKGDDS%O+I$ad&VlGMoC8%#*_O4ybI{<<_&< zT(PO1)dvmP3NLDxNYW~=j1J@|c9_NZveB-}oOf8Ub(m|e;~Z8t8(pL0dsgrumMmE- zcd!P3Nz~7SYBv)9;u*|3VjpvJR~n5E`|MC)S5627FjW-OlWt&wiyR^2qh1b%ET%&z zW4i|X#Hv?C1SSV+I_lB*r}g>KRJd8cuLv5L`(Q}w$Nb55pcv*=ZL?@4?a!o^LObD$ zl|;-yg62tu97bh9BMn}+SBxQp0&$KR9yY>m!2X{NQ7g7^`h|hrI(I`7xJyDm>FDU_ z5jVIA?jrxWSD~awB*wBJa}zP125S8^#*XiH5H|?XFtBgm{|)_3z8@ZUaW76&S@bRV z5FoVJ7Vr!LR#M=r)(6a)Ix5gzd(S`?sRnztBG<`Fj=t!WF7zqY6BgtM{3w=k~?DuqZoP8Vl-bx<14^3+xO+X)$-`faD8=sl_}2AbJ0po<0Nmr8lx*+%9gJu{=8) zl9`F&>$)x)bwuZynK1|c*fmevH}BtvfV!Wl7woWJLeC#(tRl(;N@!70@51lD99#a1 zmVpBXo4hv@r~y(dz`_PN@+3g}KoV&*5Q7ZxxC3(__pX)(aN}oXC5QPTV8^qAJ}u6A zTqqj`5uvP2Vvd-dnY7f_ujp!fkCq(h`LfSg&GlAIBA4TvMn3DbZi^;owMfqG1qT7~qMDg)Wa!Ud841vPIo9Ud+f0iJS zTiDg*4|ZW-*YzRKVRyzA{M?~SRr{YtJ8FFqjOM@v2*7s;G{32*cQIZ@hy|P#z!hX4 z4>l;?iL!f|M7D7!2g@*Itl}PZKYq~Oy~{4?voG|uaM9_&?R@o46>Az!4=73OfCFaj z7Z~z4G>4Ds*5DF+&@wy9W4lGb@*1fW7#}RaqpLP3UoBV;?N$Y)yU$l0Pd!C*s*X^% zOE;~6N2uvshWC6`_58HvU+xU6&)1i#++R|t0M7YxbJR=k=Zdf2m&B1m9}^SCxG~qK z6jS@u^>sJd>iKdi~7uvfQ=n zG{^H*)p!}@RUUt$q8nIqN227MSs8$ z)+{JA@gXbp6|A$_Lmz`w&U!;ORe!L;bR6AV0 z@*=|RS$H^2e0-VY)33*zoKg?An;7W?eJVrxUf zG$hMC1%gO@s{qerl6%IB!2Rm=a>6bSx4|7WSxi0-wvF8Ls$j~xZ^-~I-JsEmy z8+1)4dU{OjTU!ZHHw}hDOT<&OUp*sW)zbRf6tlc^>L7(f`fa3ULicHE)b@6>Zb=&% z4%a{KsC$9mUh&D3m|@e9$x(In?7A;{UYVu@wZn#-8#Cpm-RES&m`z`7ZyTMWFFt)S zi*_LQu3|U+K-bMkX>EG?<+quoSr^GlO1^f6d_HV-?CEJ}A);baggef3@-%ldy^v6D zeyN1&Z92vuKkmYJD>4Pyfw27k+zGzUfq@e{BeMkB+RjYO^C<;79!a!xty=9I3|x zSeXk798(y%wM#-`VhC7R%3eeYG-cgZp+4r{-_OyOP|C?DklXz6dYXY~MP?>Rg80Z4 zb#;lbutD9T@8q{LB#}!61OVumsDB?*VB|D@*(tA18p#86x%2M?cAkCuMB?e`QRY#= z=Cw1UBQ#or{mU}b<`j^Y2JBlYXf4Q)lbH&hcR8-{cyokgAbV|x+-a_AXotu@sg%U~Y?8_JV?xq^ zXTK*IhNTU{`t?l?OFI~dQ2oBl>D^Rc&`WU4k}tg}NlVm7F!n?2|5DwTp60>>0tlda!C(`^xLxbJGh7 z!Kd>gBXjWX5pIV}Cu{k>GbEc(3f3GL+1dO_6|UrfA1QtF!8xm(#m09n+NnBAhoo3s{pUtS<&fd)OavF<3K7_?*I$5P<6I&Di z?PmUA+1r7iWhX>N`ntobR*maL@chKlS#etHO9B$qcmqFM5cAImFiq!||+2?NwlGR>Oag(!W6Zo*-XhZgQT&A5Fr;~fDhA*2FX$ZfXh3FWM zkpO{HcCA*Im^h3eq@g~?z;$;PJ#=TxZ{n(yJZ|^IsHe?|l}ZPNzklODd}?W&Y%&5` z9(Z2K-%vLwurm-pv9d4&C1(7WMd!qLUHXa6)*sP9Wq4e}+#=o9vZA86ZbdIEKfk%9 z^(5Dslv6MS1-S35?#?8f+T2=SA8{ADl;nl-SSgoelfBSe5ij0;+;4puh6txzgfgB0e&0t;<^Gl5@~5>)W{%w)Q64zi)z zJ40Qps;TLxpUZg#Rxhrvr+e&8-;A3U7FoP$puW0OSjY-&KLRhWU+c0>cCY{10yHe! zhOR+pL|K|Q*ZrL~6>>`D+uYLo-s#W`W%a@(>$(BsPql*{YhLTU6Ae_+`R)!JsE13s z>L55HcXt;lf=0DI=n0$5y=z;=GH#C>+QO9PA6g}R`QnMiI!l)g%g4sfO(2!*Tq7k^ zvR}U_K)c4+*HAFbbork3BE6Q%59rTb{0aUOqi za8v(xaQ1y(r}Po<|G#L)`GaoWD(h$4zTz z$zMn5gj9JsUu3;!ZZ7x8bmz=PD?YGri!6|l_M~R7N-!TNEfNLxRC25&hSJ=e4Jc}p ze|~Toe!IZNc=uk3=}g>2>0-g=splx;^h@t`@#b`283$;WOm7vy}3jx|Q2Z)2C!jgGFXKcl@Czfic~2Nta;yTl*~enjfB# z4Bo{rMsgKFl0{W({WV=9gWpwdN8fz=w%xqGU32>?1Xy-uWryVbPL2GWxI5-LtVPW- zv;2)u(7uBFGP`EiK&#pV9dcc^iiSZ)$&ll_wYjANA7*li{MBBtH>*n54R)iJw;p`2 z){XdT;OgFb!f`YEF_Yfh*$ejy1e->81kN0Ow8X@vw*-#4FqV-GT^-Y<(u`UD9>%Ns zz=;>-Khz&klPnX^;ujiZ@Y-BuOLW=gBfsXO1YuNduYRG{@MMSDWjSs`?l86qcY80Y zmpv-vq4st0Rome)3+MwS^aI7e@;a~OxDyD*j#-qE4Al?TqlP0mV4KV(Q*iQ|yu3%1 z_mb}{;pXNqCW+fP(odr+^4PsL<^??>l#&P~;Y?Y>=n(BmZlb#rlj@}N!+rl zvI0kD_n4k%A*I}dRNs1D+dopC68DQuH%xgfJJHDmxI(K;q29^N|qsjCiI|$!&IhAW~qY$EZjxEr%$SRZ3R6O^@%* zPFb-Ub8)c&o0tb0Zyf{PC$!(&zO+=hwxrBdx1K?)k-;aj%tPnTInV%4uf!ClxRePdt1V!hXZa}Kke?CquNu-KU zpMmVQCa{@z1jm+3k)5+hvf^OA3DNOC7s!AI)s`ql2NJ3Dzd(DBN=}Xlr3-Pm%@2>; zNimRf3JM7+dR1ETzXNfSRZoUh!QH^9=b#)43lF#ALbeb2B_;P9Pni^@=4p@VHcmh3 z%T)riKrk&vEV{G4fxr@g5;4Y2G0rJ&;Ib>cX9?k`SoDAZu8_%*uNV1Nhf?@!Y~At5$dVmtD5JX($L^_byd zjp?SA7I;x@Zf>u55y+#pStW?-3+`%!Ef8=XBL?3MfC0Kj6F{j0W~t2ECGe3ZfPP;j z3l2Hpk?zu~uu42qJ37h%h*r_Vaacjb3Yt8>_d6Pmj)WxYYtvwA#w97~vCFE=bQjF6 zd%7o0#hyR;`}Z$m54^eEP4oxx;{^6weql8G=~#GIMX!U?7m^JJ?zLH~Cn+cjz}gnv zyKX&w<5LwB)I?qS`bw83jF6UA?(Xi(qJe^+CpPvFEBC8cpAU;Y1u7m0y{NXgfB&lK z0{1=$v(o2?%cfmXsQ^hP`2ImCRhE>I0vjb6z!ET^zWCRBWn*!06yvw8{{zPg3x6ze(XsER-{N~iZ7m8^Q)`fJ4h+68U%q6{E^aS-e#86H-tF)1Mjh-3sJe(g@WvkQ zoL%eDmRQ4XtRwwzjPkgE;f01-G|z1-R#v5@u^Qr@PcrI#hb^3?`A$=MaO~GXx%(4s zK}|DpjEu~xV1CZ>{?hLlqAM3TunXBH!hXhv$6}iv`M1!47=h;Q-DTm$D#w&W+Fmok zK(3~rz`^7l-yi`E&`r=Fg1=RdS;wf41e^2BQd2lg1Tf}(y zSkTf6)$NF5CA%?UEp&8;8A43J@aNA8pB&f2ls}}f)qMs+R7cCCR)hCX8Ma5?%2gg# zga4oJ;%uFqglyQqL*$I%PHzm=b7^TjNWgja&NJR%UQ)B}>dc7w&HN1siLqVhi3#_e zAsOtG%VSN1pTUT(mn$Be0Xaj~S`At#!nz13PvF&3msUBCQft{+PN~{CS;#`cdZs0E z`tK&>eKJ=K$P53F4$cyBuOxHYP^W&Bc>zyZ{BFuNkzX}LUgTAD{`2oUr>KGlKe43= zlA96JuJq*Fb%d;s7%oz-B@OnM90ZTG$Xx_c_VyEYk&m(v?y7WHIwlLJ9-_JV`ddZ@ z8I(5pZ|re%a`2nm{JgG_ZFY*cT-MNIwX^x%PBc^F#6;PMltvW9P8Tix?(xhU#Ct~N z4UopUiF$*^BMSwAd<|#3$ZpWmE|w$)u1#B5ZFclTNipi1AYA5yMB)0yRoCdz31nlj zYR`OS#x>>7s7#){4^<@2@*vJaADiZqZ-M6WYYG zE`%2;m~nF6;;~Nl<+d>D#3jZ;fANB$)g(om!a?);k~f%+d2_)q2o00Sumzn=<43sg zr?#vVo%Zu>98cDonzNmqR{DsD-C~tSreIy~dTe3c_hV&KJKK%N&C|)Oe=Y_D=3Lft zI8XKmWf3hW^O{v}=i^NNb!K1w`h-LUy@kF-vqK6*igkm~h8rC1pAVo2a`)jC7+=BwVoKO~g$t z%_QK?yDxWVp<@;Qm1s|@z~AeLogd4`*vS5Zu|KAZ`|6|C0zQI4HmAi}J~^|&eXq*3 z*nvy+Zx}Q_8*b-<86w}U^8<~ypHs8{k;vVcG!eJiSU1uValw4A5J+d`h9XYn#(3#2 zV}0|`ShkbIV9NV@)LP?Rt)iAys^RPd5?;>yQx7`w%k<8CmcanM2@7n7!MjkIoK88q z$lJaTO|aA5hfalh7H3wd9p$k|@~vTxgA&z)1s}eH)qCmWL|6ULy}b~M56^kL*m=l* zDZzBJP*u@PQyxcfC*?);i-XO}1G{sZbuA9*p^Sw=lopQB|`}2_x#*Fr^w~F;eJ7CEk#QdD~O;iYQo-=dr%AeMEqT7!@VUzht_(RC(Ym zI%0@&a+$-K-FJJ<%s7$hLr~O%y`tAG4j~UHFZk~l`W(3HchI8no!H)S9C%zx_SwBe zD|PEru8}T-M9Vf!6169x5VNN{(}(5muwzC7ECrYP``F&ok>{6J_-v^KIwM(rth>!= zqgn3_Uo`UAr{Fb*@!rW<*Ghg9Zr?cBRJSalYVLkfw5nH1s=l|RBCEp_eKb|>xDesQ zOyv&eD`NI=*Y=$~f6tTkre$n)V}CZHy1N)c<0MnW%4(slMPzaxCRodb+AiZ!t{G}V z5ePTqgVx5YcoZLmdfi_f&8mTzwR2r}7Tn>iZSj{u3fHUW+r#jF_SdXhn_IW=Zw-uT z8I38V+F43aE_PUbkSMH;i{Nu$psITJ`vJA+QJJ#tOpp|n(6jz+5N~^3*b@DxrP}(f z)(Xwmqi7ZZ*?zOeYhk?oC0D&^!PN%2FQuhCaCE*>*SHU~Ie$=}CLLYku*tT-{6@X> zAHkv5#jNkcz7)3ehC5X$F0O9?>V$dMyc;|$`uj_`v2l!4`*@hhks=0$<;B&x78p9@ z&$q@*=;t$H==6^^>LZCJI;nJsl+@y!3-d8RJrjfHx{*`()absB%Y)?j^UblZ!7$m{ z{VA~!RcXdxc}s$WekkKqTsGwK(Z=nL`?WAH5~S2!Qdfx%RgBeT+Sh{ai<(#~BJnj1Rao=;vE9z@k*l$a^w0{rskZ(Uf}%)=ITVY@MK zdkk3=eb{1mQ@j6vnlfZ@WU#7xz{zC>f3; z`5GJljf?L^f#+^5qw!-*US6_=ir02421XbB^N}#SihimRG4f29U3lm)x|I9LT#GdOojsV%N?R{bTyc1}Ctnd1kg2if^)?v#$*HUUNLZGWb zAs{pN797ZWmZ|t;&h{sM0{TODtVuMp8}-IjeuZroJ^Dsdi?SK2jUZuvvfR`&c9-(O z5mxL1>tWsF573tnR1JL3$o$%~A8yw?&)drJs~7cgeFlZ}?>Dh*jAqRO+gKC_xOig# z)E#50dUfw;vp0iiwKKe`!=k8rS`-rjoIHd^j>DTbq)2HUh^Bno4e+Yr;5fQQG|d$g z+({EcNrlh6;C|mztgQy|!^1aF!k#LHsl$DJ1M}8D2x%gOA%LnG;Xi;|yV zTZGVjo8LpcFr*6S)J;xm$tKXvbVrI3bWTqv(|uM{R_0u>eKa>MX?O8Z9`_X~gj;|? z`iX|-Gkc^f8UFgLr*Vv!7vFk+;3+{eRjkg8Of!e36+$VS0cS={@S+HN*F_L;We?!z z14ml|mUkdRG#+_DQUI*;@87@sM$@xq%TnXQsAqM35y>4qh^+5;Ng~Khgv0r^4`w1# z*>$jYgcy>RL3ywDUw(;7OW!oNSVd8V&8of;mz$>^78-=OlKl9Qc6b6QvRphoEzoK; zE5>Jl?;8-|-xfJYaFGe7lf&pqKHWwwzklCz-uifffbNAu|HHf~*TBj*nRwR^m>M}B($*_>R0zL?t(XDiJ3&>(fVI6Qq?82a1LhpxJ%xCD80XWff z*9?}pZ9o>$H)pyz4IOZX<@uLwR8)N~_#y=*@naun`(dwQzLu3Vz=veXC5dZmC-rTE z)rSL!zbEJC7LHbz-~lh^U<+(*uC^^jv$=ys8v(X&@SP*ki{^P6LTtY4?f*H?L`i;g@$%AgbKir!@z3MSOhJ(EK=&%% z4=@Eo7e`orzdnjmi$co?3;;zH6+eKJI5;^mi74RmB;Y~k3U3lX!K1-`RqshgyCVwN z|5+8q$8a%V@ow-V?Z=?7z-^d*xvUoi&-N0~#QgiMHmJ-YGG<8^Mx`52%sw{%HC%-9 zJ*1Tho?DV5zLi7-hl@0PNbydlRpsFyzX5X;1i>fFV`u{34qprVh>BvzLI07jwHizz z;I}KVuppDsR(igFj*V5^9Lv$;seR&slI;EtJQ)1aOUFPK`CDR$(h)u*om564U%R0M*y`wAU$sBE5zoSm3z1kfZEYK=Z9>l51MR(Rfj#%oglzPJB3-JO z)4prl?)|g#^9NHo8x{OQ$j(*p+T&y2-s)-vSINkmGOgV~k$vM&BDFQ~bN{**WBGP< z<6mqm5KwM?EH4j?iss~)o2{BL_%%K*fvs*m_Ql6}1}`u(59K)NDoAgF?iC&_gL3Gck=bG&>?}>^WPbS%y5j6-wMHsCcYjZypEiO`jhO z=P@vtW)4}`agai%EHSrTT@&BEIk zZc6?^75pC+7vHeCIC)~jH*|a_p}%cVWbnC%2jiz;e~UB$r+>3?CUiadMZgT8^7inT z2K|#iygogls}!WGeuVVVhHE1fE9(mjN1GUy6VaiaRQL+xNA9H7);Nj6m+$);bTzTi zo1kTaDFBU;7qVOCd57BqttP_HkV}+7tHF7j=EwHg^(EVVd_n4vR3+HE!V0}FZlO>q z;ooZ6KOI)7rKi7=i3b(j1@~GlSGq~)4-NHt#M;cKcJ_&PZv5o^)k@en;2ZUI?9s=n=E`Mcj{HSR?v zAB!`*6=W8fZjHpe_&M6zSyEJlf=~*dZ0?gnWA9MiZ(u=1(OWh8(CHPA<7v2hKJ-(C zsuz@v_!j6j!D{hv&vvEk@rZyf zY~ScB`+SG_PBnyZG0=3rQp`Rw5srdI=V^DE1*h=%z252Y&p#Tw`#P|~Jw(VvgdK44 zGFqHB+pXSwRA0d7yjR|ej;c4;`<26WOg(NUkon;GTP^I;L7e5z|1369VbZ&*sccNS z!2ahZhdlaR+qDu{exKrHvsp#es2Lr$qN2b@`u*j3%!~eEl&1D)e~;NTok$P>G4f^6 zd;&2Q@4-}W^X1nzDM|6}a?1Yf>M2=Gt^ZN5cC8%&0kyl}@o%+JMcPdI{eE6Wm$Oy9 zIOEn<3fuWMJK4C>q|(s3J!zVy{;z)~zW5CizlUz4Eyx%P-#PNl+i5|3_>T<2*Mg)- zohvsCWzv`Uq4(FSQ4zB!i;3}XnQ?aFp(wW!`wCMX%Lyl{Xto%8wt&BL!EBieIavd! z2r$3Elqz7XjJt}>kzqNl6q3V91a~FB4cUR_FrtS3DLIi`vQ}8v3;H`L_Dq6d%M6S}}R| z-(ThU2m3cz>vf13u)oh{Sanu~#eBmFO`NnlG-xNg#!o5=_{fPGE_v=lrGIifGhu$= z?;AKsthdlX?Qxc~aq0GzZfxOSw=ovp)bqn%MfFZPk2E_KT9 zW4h^O|3}tAlEyU}H4!Jvui}hdJv|by#2!k%)9Yzo5E=BT5ApY+`|9&X9{m@2F6e-5 zjGe=Vobo;hW__P^-Sa;`OA6z${3E;5-dBBxD>8^jBP7)3Lo+qGJba^aZiffQc2Aw^ z?NKIlUwtb~{o2UA)JVX;!h`>)eo`0+8nfd*f)~X{Sc?od5Wud)s-o5my?ib63Ujdu z8__{@wu4(1qnQkP3d8GrhpKW5`MYgJCB;q=n{_7ZWFkPFl+APaqt=5<_C zDY#i5h5;L>^tP%yckVoQOql^SA}Aa+k|Co^JN{b&QdEllFEc9(e`0(8)C2HL z3O%py0b*QjwE4z&S^}=n1hT2Xy!3QQx$e~OfU$(U2T?#`vWS4&2H1Z8X*8hQ`w$J$ zrhlXbp5K85Gq6-!0V1E#Q39_d=-Z@%6lkGXw}W*La>)dzkEA(sv(Xb|_JQEZtgjLc zed(F&%SJBfG!%4o*&dNRd@4mv5b!B8lWt?Wp6vSSoNaAwExo~{Yz)$ST7X*?4By}4 z_gfZdULqpIdiRfyF$2E~_{?Mb`5?1?phW;-^efP5eqJSRHF|QJh$sNs1WNm*zKGKL zThTQ&{F8MrX9W0M_l$Bb*`e<^hn@il4=TAu>eV)>ktr1Lz@E@K7=nvycGso+;(V(X zA4Iz2W_MwsnEgUqo1zkArT{qAXb26(qjdduyL9Op98&*1nc#2wDlN*D9-CU_Lq2z1Zccgz@@#1yjxw z5TKh{-t9QR!YRwbE~a}J*?r-;>tX9-@C;^?v%zdeI0r`y%O;_uoQXLOcBKyPzrp%7 zWVQ+&4T~~YB^3zWz$k!c8&d;Yk{C^$XwiJZ4<9T`-i9({j)D}%uJ`IFp=&0v+i13v z%7pRmT?)<)EO?-U59@0e^$63^I*=NenN=`Rox^nX-%bDH{Sjrck>#y-luw?dUp_*a z-EJ)VIZUfz0RS;m60+W7MJ}$9*5euc%Z6I(hTB1d0f7Pxcm1}PPnRXw-zD1M*NY-7 zrxPDkZGfoKWrIU8|D+L|vI9wD2tQlj(E@?6fB;qmBDlRNi~>hbpF>cX2_zpgo>^y4 zDRbS6K>Dd!`7yZ)Br{;k`6(hp>ywpo>-9@hkL3Y!6_t-i?%sRdTz-Ck8TrKRfv|`f z6_qyQb$yxh#6P3`bf*=oy82_sPy(+>Ho4R}{Kh`55mb3vx+$Sjl(&WKK~u4^{*_|* zxOQ8qc>&vd6vUn0SEP9g>J$iZiOmvy2}D%1M`!5shBNa_rIPKrj`481K9nb(ZJD7-x&d|UYY#38x0@X=YhH}dq7>UH35q1w zO~#x}jhgtD zC~0gti8J2sedfj_Gn0JNko5ljd6HN=_G?$gi?}}f_7&45CO+}DdEU(T1){w^{rV4# zB6y<;yWSz><*r?JXPBjf1DX-=GrVC2}vr~UDd#;-c} z0nP_D_j#*NEY*%bI)ohHs;eJ#-1R?Cz?FTfe)0*t#M^|~9#$K5%yLvw87o zZcyaewhz1sGWa=Ko)G37Y);^T981+jRwP`A2W4(9>}Idvp$d$t@;0H^8zH%g=m=AN z=lqL%X~3})G)niPU3bL1Nl1gHyW@+)KXQ)PE82eWwKB7Jc7@tb@BGM;>2gTsD*@?h z`av~}F`fxYJ$h8YXDi-N@QnP`IC+49T)M@+e~Dqi?7xTOfkaenmmB-1sEHzMlW6N+0V?)D?ZQ$gbxZQ1aKJs+cq;Qivyw98?J<$>PtI@x&0gEUcDeeOW++; zJd@3q_VOYGX)l7>W7@#LXyP((e1N(8c&3w~JTNZaZU@SKm71);K<}%Ff;aSDS6bSy zb~meftn6fB&S2Z&%v4~WGQVu%AEW!4eAI4B%elHX*rSvCGWlz}&BSMw3N^^Jy=ir| ze>}6VZ~SPAIM+9sQYO_??91_Vg7jhB)RbVHryx>9wB@PdZT~+fuFp4#wzg`6s5?^c z@yJj)TuZxFuVC-^y|&gmJ0bIHNTfgSn}Z#$$6QOZ7SHUNzINIO1iJb^MegqOF;P|n<3losF7N2gWD{afK&529A*XL2e}sOKPAI_gm2cl9pp38wu-ax;t8 z5#IXqT`_7oT)y3+eS|#}q-~6i+Tm5=l}gvsNf-FS31wuO*W2Vz zer@bmUct@8yX9>g_2=rCwsLOb9Djb6{$+NzH3mYRKVENBP4S}}mx8UaIov44*WY_r z7C)U$Wp|?9<0sP*alo9c^}OfX-SjCuN}y7MKc0GxL$S7qst+xNe)C(bQ#qneNAb#& zEzrl1;aX#rw76tNwSB;F-hnHKob@pD;PZ|362?~*WdUrJfDpk0AIyo3^p4oFpC*z~gE z1}j)UkurOgc(@Jfk27;s1L;<`w!S|lkWE_qc({H~VBjXNbZ(`XP4%n|pL)Gph04wb z7*PNn^-88-NE;$Y*`bEe&;i3QFy^^dYJeV90?C2J>IYZnKT{=4&CEz0F8zF<1SkPR z1u)TvKy|cl3rVUh71GDMdwU;>*gS=8z%)JbC%0iY3f@%?6rf^ZOih`bOr0QkR$WzG z_I;u;H_x$kF6o>5i-Ek_+S=yj-f6DfkZ<3p)is8AgMbQ}EhI(1LaD{-`Sr=|$p>8$ zd!B}-rXK-k#ON+50wNq3(qTdbY{*O#+g@Zv-jqF``r!Q0W%frW+j zSvme2W2`J4sJ~5JA%UW+=S#h-9Tx!=6&rV9Jn)()-^ZbUpiyUzR?&@)h~St#O{~$X zbGhyBfAF3FC>tL`RLJv#MSo&IHCFE6=~>_MZ7x1DCkGn>JHEPMf?yNnc*;5D^^Fzm z?_jaU(Wt4Yi2A#E_b1d$5TZ2?x+SF<$mP%nbnwqnDlJhVXtxxeLnH_C;bDp|<7on8 zR>-N;TN}xF(nN?@EG!tQ0BBX$;h1< z`^5MrjC#{EUc*HAgE`6IbtkeB<&y&3EKE!LP|z+5K^y514FzTgCAJQ)0>Crl(*fkzP;cXoYaf*F9wXq9E6`+0T42sTF5qEdsNv_0iyg{;{%0 zH>J@Mc-osr)ef93CtZ-d#1ojZM%wTpJa)v;X=oyY8!bS{ETm^*l3zEAP&}L#06{Yg z%aqF5Z#4-<+w)%o-@}=_15uiih0@enmX^-`Rx8%WUiymF6^)FV%$0OTcJ8usjeM_9 z3I3#EbM93Cf!6QjOhbQ7MJi*lZN^aR9o45`<8U>nt-GhLA{RR@{^C3Hu=LK}G&AEi zwcJozt#-c+N>gTye;J!^U%>gHPQvsP^qozjWX0-f$D`G@ud(nd)NK3|^aQCz(;t8N zI-G5nV6HvD@Jg6pE9ZI7tX0{*(B7QV2z{b`-_IQ4*u zL>7Z)l#!|t@%~2=M;(3eS>(k(>8P-Ta`n?090X~Wq#+fEHcp=NKW1PtxM1d$Cd()a z44>yZUHI)~IoUr;WRzE)Xz?EJ@CW!BV2YXZ1g7e7yJ;n%qehr~=iw1dS+_t2n%yug znUN)uJ<<&k`Zb!dIm5yVsl;zs-{roJ5VM@wjg#0=Zlz|#8+1NX{IEpiy!W^9iFgy(_VA&^7A&Ai8_DKMW`KXZsr`9 ztxwF*Ur^61k6C;|>$AKI_Q8By{de?C%Jd6UY%(eBgM$FRyrpzHUpyuYWAmSTlx|(R za~ufMeI3{B;KyoAzkX8Rm$IBBkxbC+UWhaK@WAVSIEAuRLw?WSp?=jcF%AR@0`XE( zOr@Xv$Y48d!*aD!W-a;Aqntn9Yj<)JaX_LrUNP$DnVvEiAAD5ib?Fi{KfW;$6P#Qd zZwBTpw)*sVZ}=QCz8xew{GMAW^OYwsHI{I7-a~8>gQ53rEHDi-Yc({%U;{|ars~@5 zG%yiIMpsHx5K^&~n)H-$Bn!Tg^aP__+m}Z@of4Im9qRe>)i0H^&12m6Ue;)zc155* zeH4g~gX5Voo`e9UC0@Ui!@n0-3OvCyECm=hZuG01!EY92Zk+3T!X3*yh<0XIz8Nq) zAc%B_$*Tzmq=?L*v9i*fH|C$`WMm{9)q<4iHhGx2tw`H>WOcPt)VyTQ_;z4m@RCI7 z^lAKCKC2hGj155@{S+#7UsS`lSUO6d%7Pq6QKKjsg)G%zv^FP=0=D8z6)$hcJMk-F>QGVDPB*--bo({$+O}rv z>m>cP4Llin!bw5Tie8~e1)=4z&ht)@l2x|R=vCG}Uk`;Ty>sEmEpK{n3t4R;xj7C} z`e(rjhO~)#%Vw!V)?bT+RzKG4nqa=r*f=$s5a| z!ALLL;pC<+N$f|S_js99g?a03pm;<@kl^aUhUw?(nook#!tnTP5NM(q+eTs36U)7^ZF!@=1T;0OcO6z5_3ugM)*S^h%9nh_Z&6UhWqvd>{5nK0dw|r*|VG zvx$Bh6(eaeGv3!xcii@OcOMMm>i^~7%{rHZt$vRCSL$?l@sPDqybPi#gp{)wl+BD( zBvADHd?-kX571F$ESiR9hUaq$adFV-%C|I&dF`}<=U78yPe_zHgy>=C6^w<9$mK|$ zz$^T2x-JMPXSZWmwZE*!SU>`z6s0RVBpr_CG&~g&N{X&5FZavKGf(42|4`Tr^Pz|c zpA@e%YzWtq($q}ovjg&JCpcPtAIU^FwX_&?jVo(wheF{sya&9W437?M^bbd7R#xp0 z^=0OY3=0Yk{jC%Xn>_)Rwc+Vh{ zMnXcRlB-FP_0IntSG9=3KQpkhf`R57iup_A@2=$KG!N`GvN`EeuVC9@X89fE?QZE>%wSwmaY_rAU>+ZLuP;?B+~DsgLT zw<4p0#wp0w3zeqM&MYOL{a|3@>-%`!fA3!I<|8|M&hHfT_;BHlwGco5zs?sNJWG0`h5kQBiMO7a zTWvzf9Olkk;7gS_n4#^0xOR=p`=_KN9}^qb3zZ+oZAEA>iyF!pA65K$Lss^HfW-8B zIXNiK7JJj|`K*%#LP7%A3L8#p*=Sl6A91+6pYD|TLmTc>*NGUrRC01cTWc@BMZ~-l z$wV9enLu-86+1)x2I?7!aXECvLVqo%zP`G*{e7`{uGn;&O*IS!0il-nYxx%*zNta9 z^RC#|baYndXr&bD&FSHCk}V4VBA4L7?U&@o*0zcdW})+kIx?=XXsAv1kxQ=U z9b?M6O~~KkV@0b%g+x~Km>I6$9qWiD1J1b>`m@kd*~LveY=x@^1cFoZtYW@WMmKP+ z?l~l(DXn6cVR-9)(xt`t>Qi#Nhx^&t?Z)SDwXCeJsVcbX@twW#$q?72D`GsgHFbsRa&(X#*L zrIK_WRAm7^}Q-92{%l%C=N ze`}xB<@1de1z!bpy^xCH&JCD&PDGYBUiNR(3Y(1$hl%Ct*ep1kf!KyGH)+&$e==t< zBf~1YVDq!Cx&+WfWC^qBY&%Go^otU3|{Q2Rr2}SE?m8x!M;LiPh|lv2{%aiPXmddL^y#jvhl&xe zf?0~qZwhSBO#ZaO>Sty@w&Z}&)hO>`$FdPxvRibn$2x~)IH&jFlr5m8d*;5t%w6o_ zq%#hrmH}M6=lA=(IdJKs_nswoR>$iv(W1nb|0TO_AP*l(XleNN{Rf%UwL4z8WjxD% z^*k4LmI3y1fAc}>pa@lX07fcpAEb|i1^id; z01raF)Ce2AJ)(=y2X9Q(5+kG4VmWGlFPjsBE!uT|3FO871e2TpTm>m19yAm*gZMR9 z#l`G*iHT_i1QL24v$08ecnCt!E9c4?Hd-er|MFfJv_L~FL72&BJC6?B>mY>!Tx5Xe z?b|-V#B55g5I6_^n}Ei~Mk3AU@Ft6gW1slVn>Q*opeqv+5}LJ9DZ5SZoDCxO^h(sr zZJ$9D4XH9g`saIfS-@`&2*{mKY{$mJYJ!yta)hu5QJ^h@gZQ$UTUK=R+0UJMJvsk~ z`U>CP@-#PJ;gJ>)m6lP{%DC2u-HgM||Nj*fKCh^d{Em6n9!5KCPeohSSEiAwOmYLv3jRIo>RF4lB z7+wUkkm(?+tYpELy+A~#<#BEK1~KJ<01nB`d)6QcgMpm`6;HI%8UU7nF6(mSJ`uIY zpBOe>xehDzB>#4UW>EIQFhfPh)^_)E(IfEA=|gp5YMzGq2uOwP5dGI=kh63wMQwT$ z9*;4>7GNy05UfnjeFk}>9gtX_j+(^@VnN8XhwE3v94>hh77!H9H~FAtXJ>yt8~FK? z0K!~apq`BUL(79Lb&nBVqchy57}6owJ8vDpf^7GRzE7#0>#Uw>I#{`kQI zpJ*O-b_KI=9C#Ass}n{k#I#6T0ZBqA=f7@a`MC!|Ml|D0CWm2%-s#>VKW_s)R3f%L zy}c!#=T0_14!-Ro-=s9C?G%qPtJ3HfTjfN=H%0Z2p zzy!j$tO;d!I1aI&m&w@~v*h$Wzi<$Qy^h|o4g(Iu_izk_!$%K4w)xS^m8(8=%BvQw zp^=r9v@PK0cT?qmPU-4;;HfIc)NuzB<_>2SUA-0`sy_T3fAvahVXb{Bx=@^Pe)eW#vN?CG96q+U6T-<6BhPtG5xR;+ulW z2r%;Q97TJ3*c7ugT-aggcqp#OM}MVVV*?dOvFAA|dtB>!)#|1d$E+c@>-JOwi`(?Y zMZTV;tid`8n7)5VA4@_JwmBIPa4X#UA@#wJo@zL(LgqE`BxESvd;ZX~(i!lrRLKZg#~UaHEgrFPt+#X;zbq0XDp)aUaJBYtwh)jC1` z^(7kPjPuJM1_{4a6yg|aVHVZXgyk;FYsGZ_Ohk)L-g}yp9xX4gMiqj|Ks@q#2K${R ze$=0~iX1kpj0}nnjU11B-|!Xuxx?LG>SKimq+E(~S~6=ju^M?iGxtPB>z`~{K{>*| z9qh$*+CWhCKwW*M_4jYtj2e_Vj~_E{sRxN3ghfU9^whkiqwASyur(x@c3D70e5v>2Qt+?+(6zoIFneNzr zc6L(U*gT_qXrH>}2|<6TBV{u{PDJ}QVOs9*W|cM30|tM63xRUmnKmol%uMo=vs~e~ z3eP1Zh*@=sI)3Ywkf;}nI&IX=m2;x4&B$MZr7NwjF7!x;cv~4NEt0&B*xrGGkI}4b z)`2N0^CmKf$CFJhIV3 zF@v0X%uO4g+)e%8x$aK_R@O`=ffBf{pl3BYy7fLzcR3al{SE7PO}EXWaBb za}KvSt9S!abIe=vmz+A-*%uWC1RdL1%C~QzqTKJ19Mz=AA{Su8VUNYd+G3>}i797# zDh?V^hjgYaZ`sz0OIMZgpVeomXCjCk6>c8zYWh`&BS$|y7Zgrl(*lZu9>fPgf95U?!7PS z+oOxltv{E|$KBm=E-N-Wyj?!M>1)Rp^?lx{zHhm>GCS@p-#Q-?+zaE!Xq$oIRBTO! z+D{RF!QG&j^%OnqTDUxBQ@%xlgzg-gi&U)d?CrIb8uucV9~xIM{MUoU2;^k{=xo>@ zcxE~FMJ!`#svo$4m#~B=9Mks5H#9RtNJ$Y!g_RYvzb`Axk$a~As}WI9TmT8}#iFCL zvmq!<{^R=|e;iOyz(OaV90DF7$H<5cO*+K> zlB!(#H6x=+AoOv5XRcL+GG(wcS;)N@LhRJvm<_D1>FA#wtRk-zi#*dlcUTmGL`GCz zzAOO)-dbX0@Ne(Cjlf^DWr}nW4-QhOF}A?Mhnzul2RH&%jdyTH6h&I}W7X?jO+8>ANH@>d_DY-YPGL#B<0$ zWH;`)J6Y#S2#F!!6pzUb>|LCPEXEfw{sDX!e1V?X)25`~ezyTpbcRlp05s?K~k>LEn~kTh}dfv{M^uM0jW zA$D{h>TuX-jqgV#CE>Bg2s|`&@)3OxTcNc2`lM*8d-tB9Ph<0kLl7WDm}Wb?eA)f^ zN(T(`P{O~uWjm$l=CV0iBco8{Xim$4WptLp4)SCK2X`^3dUHPrJiRbBqa1QzNDV4g*ryM zeze=|sn{?udTCi%Z$U-%q!D<;P?t%ls6-AFmOy4)T5SUB*|)CaWYM48P;kK0!u~8A zEEqo)1qB76ltD8wS=GytfGYF`_})!jK>_xA?CQNJNq>4)7OhAM>n$Px&;hOBzvcrq z@S#gV+zQJe)J`y7d$8?*>ho8Ut7%Wao@@c%|C@T-yJKvp3Q z#5Y1*&dR1LbFK+Ygns^104(79R#KM@IaK|0C}^OD4TpmXy!@%v)#^RoFG#@)k9@=a z*K2NRX&K<5>!SQMI~2$wH>HO_Cdtk@ET#PQf{zKUfQ;ZKyV_J}%43~yzRM3yh^EDx z?Eq!+cLi|?GF=loytITxaXt$AqA`&x)-U|?v}HJgJ|*}LA9 zVM}OteRlSn#^mC!_qn54sXkuYPx)|lnM0$a-^It4@()%5Ck&hh($9mPARNBi~QKZCmuM+@nS#<9PpNnsu6n$NK@I zGj+Qe;9>%C2+cw*Vb2LqvgpS9zIEy`r(q7?Q5=GzvYb*1Ykx}$8|~agd`HqQPr+Gx zVXOkWU3ZF~eHE+b8KXIJA`c7a9)jJrYsg!%Lv)}bUs&z+v`szRKMr+GNs<2koj-y3&cRtC)FxebrGX%&O8w*rgc7Hq2sjFupAX^W^q&kqQMy=1v&en!V^_X){ zk<(MM5sNxh3_IZw*X_rK)yrwQgDjGp<2Bsib$yHXCV@+!a`Yn6lwlbMtwJ9GmD>%_ z^(tsFkIaLSTtX%CXGhny2idp5atDPh#y`#V~yQ)#U4+hx3DRNl6jU zM^Pc#1bw@(Vd`LYpt$slPzUKPfql{Sm~u>BZriyvrcc;3(^&^cd#PE+1wD>3X&O`l zpN}U+HNNw=T1gDusMUD+5+~V{=T>JpF%ugzN=)ytx&U&n$!_Jb(W*1m!qEr6w z$jE3bLh5Tms*bm%dx8H)&0)(OMY-l*mHHepyEicLVmz*(eB**xoI^j_V3)?z@E%1; zz^|u$lKuOZmS%`8Z4qh)PcwOJB&26ty4N>&ZAo-2T;>l z_4HbNydxiX^gD=MSd;QqJ*7Y>#0|f1)zxXwmB(0Gd1F&>b3#`JQ523Zy7H~&D9_a8 zm6L*9dO6yhwLF8fbX4F!d-}N7%Rdx;mxLNhZfnBskb|e?<*M!QpHQi_+ltp;ybJCq zQ|_q#@Z4jsWEVwR$uCY-nb^+7NfR$RUqs#WehquPbfP89(##3a{AZ;IOJy{kv)(5n zig(XS&wRb$eJyhXK|)%xyFXZlsWp1CyU@I7XS?y2L*oj}ZhCrp2)J@`43RSa6whec zjHE zP*Oz-Jqt9eAn5buViLdOO~?j2_%`^}#|H&&4j+-L6EqLulpYb69crF;lP@F{D`eqb z1eK^MiX|ky7uzq(30|EnJbYh(?&ju(T$BJZUTU!d++`qgfNwse(~7*%B}hm}nx>{G z{QYU9TEIexbg})13<*$e&&*IE?`n6DBix3G zJ?7VLa+N9cO=?&9&#u^FM8Fp{wX(`s9GjTvW$uObiS4{31Y^;%v4K5T7YWx_`2p8F zxbmpMn09q_^)V#G&)3&ZfLq__*YtGQ-kzOCGP66Y&o@>Og{F_Gp#Wz_OQwLd9_k5r z&5-qviVpPtL&L)yWxnZ$OIE|M-0YJ(lRoO(Y8)bNKh?Y z?g1sn;vjZ5HKV1am5qC1{C7f(n&2*SuLWbD;XPSdS@@oL8AQ+~Xu~i-#k`2=Yj+!j zv~^#65v5+?YHx1`rA}bvP9>21Ul-}TlC1pSK7x9(?BM7K@{9@L$7ko~tEfl!zcp;} z!v-HB^kLthx8KD^*9N!f4S#=s1|}wPpvR0)MB#`8!wzkBejfF|69>p7u>ETo=-}|t zGC3e%t=pcIY@IR0aC(|`4~6rI63d?om#NaV+nxRWS~h1}QffeA-WH3j2cFAgnCm~T64m9mO5 z$kqcBWn((&aoMfMkI{fqDWaGk-LnD<+-r@( zsu;E?XTTKzg)CL|d7=Xv8V@t`fxcc7YVaes=Z~4jK^BQZ;#TtJ4esG7z_L|VJ)9hW zP5he7e~Pj^Fd!V&W6O8IR|*%h26y)Z)Qy&8Nc)=?> zw#l7>8@DpUG5rReuk9J~!l|UocQ0mwvnRqs-Woaj2qmSR>&W--7BHW9dT`|ugilY( z?Ypm6yo^cYsIED%ixTOTjW$27OP}u3Q^?ScPnR;7%(Uh^aprVJPr=ttt5u#K&wlD0 z^-P}l(!&l{WjmPf&gJXyutxj}c08p|nU=SK1?%8^ET-#ylHT`*neG;a#;5*-;oZfU zQ3VaA%d74|>*rox{9b2_6r3jf;4J*Ha~O40&%*ld*~^!CDp&P=HsR04EPq?N#G8-W zgog(j$ZL%ca4G%4m6Ni zTmwmTCnxGkoyW9;9e=0#zh|^tS@tMoSQ;9}NH0r7vxtB38BYjs7iPkd;O@{G;!*Ao zt@tNaHD&r|%jTr`#e!A$9qS*90Z2Kf-bT!qLyJoJUssp>1OXEP=~lJlNdm7vej0zJ z8Mq=PAC3)ggq8KOZ&~zf^cEHZIY(IK`fbS3+lN1-4R3`0_u9N~Kz-}-a-{{eUr)i< zC$A`c3te~w$66v(6yZ?ekoWJ~pQ%(2l;Dhc@gp1lz||brh`;oRwK^hvwcz$ z-qvT`5#j+ur;0;wUdiK`(5q5R;65a>7HT!ZS&=r_HpDi>@pPEfj?C{fQ?c_~3ki2> zpBpSvS{+Joe9{{vl1ENDxGEY=Fypq4+SB8Ac?6^f^$KbGv|`xbTmH4d5KJyMrt%56j(=ms zsKr)N(`P49uW?|EkB^4{kynev!0-Z3w0f}~%2>81F|6ME;<<68!YMQMAlC9TguFmn zk=W=qB*s8I<4;sZXP67a_!w5Xm&Mi>kAB8+TM!|?GU4&~jt2A+p65qM1>e837mxsr z5A7%n>y=%RR}b8gx7BH2t%DB9>cF>3AA*E*@Hgm{lv{5IOA4{ z!n9rV297dqpU7VUc+P?LxcCnt7*~L-4V174mhixiu<`WtoCn4nlt#K;XoPRy zy^He6%huKp7z6KA%DzubOn6-F(qK@4kP_C+3=R$sc}<8{3HY$AR%In6s>h!#A>R_R zNuybb^u>Ubjr#0UM2leb+zZHSzqsI7y@8D6DM*2aMJ^^RjLbMUI{E;(ci2!fILA8gUkAXrqp*0Hi?%yDf5 ztH|hb^*12RAJ5t9AKNGvp#U!oEbF&T23`D0OLJcO9{Kw#f#F^8WGZG)`)B>?hZ$Qx z->=)9epspLVu?f;uVFq_q{W2^`Y#BNj(;-^tZKUj5mn@}@zH5~joQq5I?GscmSY=yn5xp1N-^M=pva2Ve9YJtj z{T??zl5-bU1uwpxeotfW{oOxhx3@?PIz0>zE$4`Z)UblkXq-<;`x-Efo^&^w7=)6R z{q>*G@x`GU4H6|!{oRz5W)`*)(GHymwzoEh4?gR>Pt`?#Zbq=bS1#u4j8;)e(!G&G z@gpjc$jj>I=Rkao>q`!8SYbY`8Vt?)cACe~(4gXjB{lUDjq8>2%{|BL7mcE>t`y&s zpMD+*Bk;%{z4d(G#pJ~;!`2T)b5d5My+&;F3#@o)|L7o9h;MxL5JNsfGgs7}4D z&2l*rj0fe=;aN=w339hwzf@D_Cd{fYxPkQEv78i9zW&brd;kuYhnqW#t(ccm zRA13i=*1|86%VFm?NEU90gpni*I#3YZv5f;g_(^l2L7k~zk7>>TEk>Ck5YcRH>&zD zxvO{v7ClO~W7tx4eZ^7BB`jHKXQ&~$Kkwhv%}Qp73fpYE-6P{|*@{j4T}gK%{@e=E zXuRlD-7qj=p;0>dp8^qPLp9wWL9RWds-S4wdUFPl3qXI61E`03J7~mefvqBs|h14-N zh7SahF@sH~FxV4PL01zO7pF1{dsQ54Y$W9$zI^wN*kPpk;lqa^F?typndtZL--4C$ zUW0c>n?7420FuwaOzx9eQxpG%`N@+u7!f6S!>2ArV0&qz4542yr-s zt>?sRfl2VTUoY&AVON0ro3En^9|J50LVCot`dQ% z{KRGZZg5p5jvla(`uYz^Nl5|jiR|O!^Vf!C;U6m|DDYobhG=o<+d+qw-qMm9Pe4vy zk~4-4S_}|>@9nl;E5YNVSV~s5`@>V^s)>q>oQw=)wNl-j;)0mT%`EP~);T{I&P=LSggz`~Tgq!ey$ZiY57dpWxNwtf}NI~BCG zUw#F};ty!djxpd30!{`8*er~_2N&#E40|vTomf|qBg9|~jlf_NFgib!EN`POkB*H| z!GhQXE3RvDk{A~1e~ypGlU|1bkVQlUg>eu<{(!BjX^!Rl{Jp${MBr$n0Bm>R;DXpX zKOedY+12IH=RZA{2BE!(h{ShqN0|DYL7E8U6?n8Q4I8E+-e~$)MTD$Jcj43jKf7a9 zIP~=O_X|r)hki(V{|K-$-rnAbHV*6^V`DLQ{4=2CL<8r>zinLfXp4_3A-f5p$(&zb zzXR7QpZn2d&xq24CAepmgNbq*P+tM{Y9mO6b`Z&+8LsOWZ}_*)kisZf?eNAJg9 zzub4aI+}X0QQ6wc851)`hlg!#^NL(q@~>vWkDUhIWn)}iwLdIg z$n)dm9dN5&tz;On91H8{d>ml|s$DPe4Sw2;C$MG;h(c%C_uc#DHm}NK`odKF8^0#h zqe4kMH&j?53I-3P(}@WKqU{Q^9%Eyf=dgIz5q|NGp{N)#N(**+Mkz1{z8!*v7l_rz zU3vRfj`dy~-!L@1E)=M)E^a(m8qu8iHACXDg2pztPHND6=be=oJ~jHVo8NdZtab=` z{M_&X(7g9%lKO_Acea6*{&Vvo)Nc3J>r=|r(^PGJ@4U$fTG)tu%c-etin0UcwY&%h ztkK4Wh{IB_WF(cFJm;^3e{yL?J6qoh%kV!^h3^S|Bs#LGS*a+Qry3tiGD-x$+Oa6` z1sT0TN8e8M0Ed1eyAB8i(UCI9R{k#EMQqi5uA5~ePJ!=$0E^mPo{3g1g@vV&9zC=C zw2&ND(3!v*IC`krJc-qM!6x~YQ7Hn{3z zQ}G?XVpgYU{9+A#Uxg+!#ypz~0QldlwhHw-9ykB3jodu<WgIlpN~a2_o&X=tE%VhL(XW{JS}%nOAEXP6quXtabh4;c8 zBQogFHRuOVQFBOP(|bZPgXMd&>*#T}Xh{EEiAGBCPCm)3ZN!=(%k-EUaWRINYNp;^DHgT9>A_zr8D2QOX2kMzy zhhDJ!so_M;tx|&ugJ8iF7SbVNLFZOyVR1gcb#l=N{N-9N+wz};ZsHmxB)z>F} zL8PBfPfyPSENN|y5`)ys%d1GE@>c8h6(29Jm|R4Ee}7qdd3*MY*s2NMna=Ou;Z&<% z=dw*42TF+uST4D_7jUH_wo^Ktu$coPa9%+{=3i)&;6%mtED5IyAyZ=fwzLP!C@d{4 zEyn8L^g^jyw-z0KRyuxp*Z!K4mzNlX$hm)^w|O5QFM;||f->gVzvN~t)!HqfC0AAb z_K;GAT@^w^4MWN$Fu6ma6PK5dQmKdS8=!nCKvWE~NjflfW#r~^rO!PwivpV%q#KR= zy|}oj8YS*%?uw&VGXqdEiGk}F8#EJkj*g(QE7S@IN%DJHI$>*nu z4Vjowzuy&~Zr{}0tZ!r_hTHcQu+WHWql5Uuc0aV?F^OZoBpyc^kdUM42>G%k*VWf& ztzoQQ}T))9EuzEYW6ddkTEinTbX>0ifr8T$(40{mr6bpPXf6YrLA(vln=0G>sZw@ zCd!!DTo?r6og^d};~5kgTb93lIyrMl z%iWtBM};O)Hi0FuKnVEVWMl#?&tAzYBqMS2B(_1XE9<&clAEo9HDwFRJvR?6glg+; zS(I8F#8F6pAb!8I6hyqYN`c)YUR%jdKK2X)PeLM7Ai7|q6v7{#FpCjBsF-DP(Z#QF zI^A{$2N{-($IfHe`wY9O`Q?gB?ijO^h^~EFQ=H^xtZo=)QZ3uBNzb9T8Snh%W7lx+ z*-oZcMDbyY=(ClEaaOPbXAq(36nCh5;?<%F$&oKvNXV!yb9M-~H@!;U;LEHh4EyFE zmu0h?JJPKnzxd35WyJ;@Qyr>{NqwT8Ek%9!8TWvB1Vc|8vhC30xwKFsb{A?cHMiF6 z>`_XtT1AWRz~L>a&uch2)yODQ;12q6(~D1)D3m9fX^;R=+K>hw!sqJ0Uj~vl5ZNvM zHO`><5E4R8Bkz(<#%>shK4C=zN+j4zW<_q&0_2s3=I`nPqb7l5Dp3hX^v=-&KZa5DpF}cz7!;pY7gs`Wg7h z+xA9>tCZPV+oIVqE&=VNniXV{obMKXj-9;5L?Bi>*2Ei!S^xKQMTi7RU}iS2WY?-P zGw*>wPes$gSHe^O7U5Ypku}bPNB8N6m%*!U_H|EWde1B`rBkzjZ|LV(aTBFKrvCJ^ z==9HSe8>xn!PQ}AesZe1)G9eO=|m{y-O;Qz1w65RS##@o6tdi8{Zz|CWTaQxkMPa@ zVHJH-{6+LaxvcBia&TI2G?JEqD2`tDk=hE!e5vW60%jP-ExYSRoP;W=>YJ+eR@3Xp zT7|b#D86lnXP8ntIF$6;zB*M`{mLuNkbcZJl`mo%wsy*;q;D8g@aK)EQ)u&z2u36H z=V5$!uC51}at72ruNxLhN(3P_m;RDg4$xTJjSak~yVE@qB_j>pTyx0Jt*vRd&yQS& ztHG@J4|YYWclm3x+vQ=Q%4CNKh3R+xx9OBq!pSKa!qbUa<w0PF5P`YRs#QHD%r%sN! zi}VJ>CP{Cpgr!te9)K&`>YAHVSXjtua^r=xbO-2e z%-+EN1>@Ji*w|PJZ@=e5s4FW@6y2~_RIDFs`!~kp-(pBa6aruy3ae*YJ<(7%SlqzR z6~L%f(|2$Q5^6vslH^>F|IEn=*x|D`FQxKR#00kp6H@7-p#6#@gLn9;avLvGxAwRPV zHpE8ptgNiNU-X66yABq7!BJ7U%ZFjJ@YfmG0z&+t;o8pU-J7~~b`LS` zNvge9gGxsWA|jZ8SMkGvBhh0Q@!qS!+Oi97sT?Z;gk}8(5D}}n5MY4+`v~i2gLMl* zVM0J^!CWQ)!nEKcms8eLID`OB+5(c!&=cUq0ov^4_{Z21(0qcHRu$A+9k7Omt&*Mr zoCOi@IV><$CnZhH%x=RVfkC|tX;9#pRM&G)oI#@k6;j2i7kLud#2*z+{rnjeM(|aY zcW3nJa`{e#tR!K?y3$%`VvTfd-Y7Y@5n?0OqByD+4T5 zz$x{F-cr0pU3&(iCGriO?xCUK)-{Cgl*ZDkMy0rIC9GXU5*gmMwRLAW3%%=}{Bkd0 zI}{Mps+v9R?Wjx!tnGuzPhaKlq>_{0$K}MOR>5ENsVwlZm6qnz{25%x|Da6$<;$c; z_gw@$YSfnv<)aTZu5pLQn&evT{b64}=?9j)8eF z=&xl?F0E$cSlcs$IKnR;wmlf88|+cEZ5g7ccclv>Q`N&=RF^8F#9)f@Sr&I&U%e}Z%rs`)}j?T=6K{DCRoI8JEELC+*$DEHMpi{7J-`{DT%&Bf*_;va# z9v#Ie#}^#5wa4Yz=DllF4TR1w9WJsqJnQS~Zfa;`o_5IN%^sUb56;gQskezG-X0I4DaJJ?{W`LU+Gn0qq>WVA2o7ufXg;bx0u6cQ>c*-Zvh zWxsB2ZFOB;o^w*^z91w;SbnO^a2WqfKr*vZ*%si`goTe7s2~_iTUxGC@p$0qk%bOI z9L)xhBXDvyl!v@cpPKmfYk1`6PioUxD*Y1{7MAoJ&d1-sy=DQFx9nE{&544KLSlm! z0}CFF(&3VSp(p@MVOWg!|UGHw9zlOC`rM9UlEp#c6TsOCX zFe_6vq0T(w;`$ix;_6E4g`q#BUGESO|BQ`I(Vol zT?AfMRdFN>c*Mu|0LY2rYdr$rry4cs?zDyAC zct%)}DFAn72PN=kdZ0Re5 z;Hvy4lUE1>OgE$~>~gywJmKZ#&2Hg>5(MDvJqVEL?C*aeaj!w%-JO@w+SvFe0fBim z2`Gob^D_?@2w{qb_L=5~=`6@eD@_M`_6LQcAYsby+gQZgF(8i%i{JFR9)VlK>9ZHq zBPj`qTRYPkyY-kD82+WDr6BA9NR98offJ-X5B~+bSGmbI;>AJ4zsAGH6EcCHTg<`$ z+8=6j+KKD`)GgS53?p6!yi&x=D1$sMNLTKUs5db(@(&Ka&3}EdkLZCty}UxK{aG@h zbefr&A$a8oW!Cl8aVw-E=-ER~0|8rL7eo;iEaaFUr>v zEH_@{CHvWouS)T|X|2rHU#$y#e%f^|%~Xr!a~YR^K#2Iu)l>Hd>#? z2e6XIN=#JF{vJ*F71(m%K5@~rpYn)q5>gX3UoPWea0@#W{k1AHtT~>4P&fT`+3fxg zl>JXT{CO@IXbon_A;EV3)zwmxW?$nm;tpuMPamf|6V$6ys5MMFIU6yQn58!?X=fpI z9QG~qtj99;u(a9f87rnX4bNNWi7pu==-LkEvKd{RZ|7rcjddLuD^C0e!aY13YMlgf zz$`&@)HVXWNA%W9kbc=WKsrq*q61l#qLzh!Pp?Z`M|Ptg%QP9s=+;k=^i z8rKn8v0(%FPB0mVs1_nr(4k_Qvl8}oNc0uokos)wk%|ws)wwJ z7)9z2CuIl;)ZTsg&~DBZ9JN}f{d1P=)3clx#s7Tx^bdsjlmD*>`m#_dc4z3R@6a6n zf`{CfJE@)e?VX|z_|G?fO6NGo$JbCXm57=1=!ks8pB;6sFgEKyG^k7IETI@sE}lkK z_|?K6>5h@H9!e3+8xrtBpHu3DZH->Mcb;CT)8^F%TX-*mrJ zwP5s0YgM{!HRRP+QwOQanSiR) zpSr`XjWBj1q5@;%)xeJcqb;*7E?_IL!nssa7bcBR%s7uo{CyE2^nHC&n&dFD`<}uF z8rNzl&qjc&wzErb4T5h1>l~)l+fXVAiGVC7eJ5z;_ir=MuiylL6@pPmSve~w9@~lN z#R95qBr8<;1q8sO5eB*L?zd_a-?Dg0-OWK6ImYmXkt?%tZ+Wd+OTQ%9gw-<ya40{~O+o6emGVeHn#f7U>Zb54a3e{|$B-5_3@8x$A_ zXf65gIQ|PXyu3;AJFwbysBmZvibhOThek)mzJ7w*cW`)EJxbMK)&}y8FVC2svK3>d zq@+x=`n0P2i2;F3dsmlH+u{A&lm6dKI)5dSPv_OuxhXiC0D%^oD5cZ?{G&#s!jxtN zFcQqCT!ajWN%L_0y!`xjSaLEM(*qWz69&LI#KZ+T@d%q|N<{^m)oc?Hm#y^qk%p$` zVtFrlCnENNCU9g#e-rec9F3co1ljxty4g@OrlC&3_w4=+5k6^s7k@D$xSf z!q?{cjUEaB8C#=)IFKMrvY5Zy5X0CG&1x_b*Z(&+g;voYCj_>ba5n)W81UDZ`1bbp zcP~?5b%5Z|H64t|B(R%emapHJ;IOu~u6wgjgGnJ6zT0w+3Cd<_#O@YG?OBC|GV6P= znk*?PL3EP{$ZQj^UlAo596<0pvn+b~0pi*$bBfQ^1opLT0lgQyK2H%2e@$1jJONpx z!`w8R`JQ{-tarUH4{2l*j@r?B+ebfS{UNk3p99(9qsMm)*ONYNQa|dFTVY-7$bnb_ zIWJ5s*S8rNJ15_sEUxGJnXj9s2?V8*pR!7Lc)W32Npzgw7eCrHO}!IOx4HoXLJGcD zd1YiXGhPr;uUzGhx7UU1J8BBd{?)W7H8(3T0M5QyjgfZ7dcalg)=9whfgVps&}j|R zWoI;k(0Fq^3N5HJbELE>Ciwez-TLMFX~CuwgLl)roZt~{gA}CCBM{PS2x+#}n+p!- zMk{J6RSSE)^2+9*_<__cZOugI%?n7N%2aRixk?NX@OYrd?&>=1Ww$Da+t6ry-9kh3 z#baGhGSK4n6MB(<-BOhpybAZIKa{)vwfahAMJ!Xz}cp_v<1wh7|ux z>NMJaGHtcmCnWJF8wfrNtVLbfXiy>@wfLAT5K{vLOEC&nVLea?)Chnm$nrjm}9u7kH|@{ zSBT?GDu@!Aw2uvY%w=e0Z~&6Re>8y#yi5s3@3R+s%^v$EWG&Z%5z^=9TGVu3RNHZq za?8unt*r}sqrdg#fta;-Ju5CFH4O6s78ZJu2K5H6uE#dN&D<+nCdik2FzFc>irB9n zdg`-?iHBH)gZ~(?xeDg?TAADs`iwADOYlCuXf9(x^9LLjkygz#goFHbJ`D}i`g8EZ z6mN{@0SspsKy*pR;CSvL@XY%DVKpx8q`}CeU#yPZtFHjZg$OTFKjxatGYO0DowC&v zGt9bhMgrsZ^?9pXaO>yp?T?~&{MsQcBY1I7MWxDcsi%@|O2gRxw-oZ>n>_>DxGfY> zW7GIvbmlJlszzt0gLy>ai#Z%k0cg)$xiw`Qq?`s!j<(gbLJC*Myt5ArB5KxB%&q(( z&#pCEud+K#6^RyIMq&5xfPj8Xy2+*ZuAYyVS6eGW&EoHYR&gWze}+xeF^H%XTfLGF zHlIu9uob84Kb=>Qzbq4ntC!4UHyzvrd24KkZkob1qTWPyEp7sz9@nz;EwMFkv9~_wq!tzGI*#Si1Kz-gl+ICBV7& zGRb#k1Cvg`xW2TbJDQnbD~Ct6-wYNLMqa(T06J)Sp%$kltm;xp32} zcXN4Zo>q+#TZ@>mj@!aOZR- zG@gG^sX^Z{TRhSBPK<&Hu?99{M@F*;1jVg64KF;lcej51yvMeB&g=* z(8T)MqQ;3kIdL$~3=jKSj*nK3V^;+goeLv+_(yZh5cmWG0@9c{1DGoxANK~{BZ1gY ztj?(^Efa(+7sO2vhx%=z#fuNnS|KSfdY>>d##LCqhJ0t?0p{i6 zs4rmro}GG_o&6{d#V8ULv3vxlR4gnN*->r;nNlOX0#cfSJzHCRFLgM4`^G(0c%;?ZtKMW^z_wcY^=(u?SZ9BrOqR|6dvO)fR43lq3 zoQvZ!jt&mlW4h6a}w3fGHfGy*1VZX<6dXy}j4Ei5{AW*T#jOc=osZ($8b z1#G8)&GFKuwc#wKQ&`-?GGrl`fcXC(TMYK2dc;QG`*fMWYNi1XJf#kX7WUkv&}|!l zs0u;aN2u+dY4HlO`lfHfUHAL~4$BYBtRE-GA44!ZAH*?1>UXu< zQUn(`LZhhQv<&l&MIE^LX_?cbZ;azuV z7@BKH_e?}&JX!230wN)A400d-x6-taRpGEpZDtWOdk*qnpv**pgxP`478JTKQ@1Ep z$7W{kZ0O4K2VvXhcCrw=_5lYLt=I_du))s)LnIZCTY?ZlA< z$!m9Jl$G7QKG(Z@HX0rwsPwuraiYSQ=`eDAYfJRy06FAGQy&xlBkzM8r%KnM*30FO z?0HUx)C7*JJEAohGbnIA8x(8|Wp2UlCff)8W#!;4tb%(tAb&jO=;{jMFV!0k-gm5G zA^p2lDy_Jd*S&|a2J02xO;+(y8m~t;W9mbX7dxaz@ zZ{Iox9^9{dHcOw*u`-ozxYI9|O}qreQ_M(x-hg@AAQpw>i@>c_#pHq}iR8%-?@9?M zhRi;xbdPnSu#@9N%vqv;Dqr4(~u(+~Lb#d190@5nU+w>f5+YQ*u4^r|~T4z8jzAK+1(qn!)i3}T@ zfTyY_OY1+_T8|@~GE43u8Ku&>h%{e^td6g2yT3@d zEI_d2ZPz_(e|@?8vi?v2(pNWB&a4r=`Mh8y;q#vJo6xYf4D#|@i^;KSd8CNJO-Tf!0 z8cRrOQ0HI1nZ{y{5=r2f?+oZ5=o=j+=Ku3{>uhs_7b58&PPZ^7Nj6=%6I%Ta4@1Z< z`>3h5%WLOaf@#zE{UFscarqqD2fHnGWPE19U^U$rD*4+Dp(T^=VRHd(RqQ`d{{%rV zf)c&b7!M2(cR<>l;gW>8?1Xwbk02oUk~krzs113(FWPe;r4b9{e<@`PVEaK4Zp1F7 z{XgXlWjHs)y#b_ktW@ZZfB=3xLo`12bUix>30G>?2mtsYFYqbbrdFd1DNu=-{v7fZ zKtX_QwR@FAaq$eI#o~4;!o~=+Yj=LM_m$ZZ8^7ir*pDn0wwFKBF4V8mbaab9A%_8k z-%O+9-k25vq!Zut@v)rD^rt~2ca%ItYPveRX*J7_wz5)j!_%S4tS=Qla=&TSGK*Wn z95A&GCLe&Z4oTTpQcA`ejcqV){!1!p(S;eY_6=*iD@?!sw{Mx-pcV1$Ht7)hqFV6s zEMJ85Z0jyU#3wmwSaIXDV&dh1hxh%XCfe!Ik)I>*K^O2tYmM|R&b?X802k}D)n?&B zuQrdL-cLRt{AvjE2l@{wtZH(c}7$ zU(5Hu?av?I*?kB~zA@XRdr{cG!p1iK&V-8I)>w;pAD~<2!m6xZ^SO*DNT~H)rb=|@ zyZsZ^1eRVMYHP)hd}{PK$CkHD`1z5SOd6igNy*!2z{3ujAnv0>&I6}V{K{2+0UWEy zEm?ipIO`LyCG)DEVg6T;ycBAV75Py^R|iA(uzaoh>BMpSVLU^r_=cJALaF*+?WoI)5ekptas3)t;^|*zDw~Z~cY*F_nPBJaHt=|H_N^Mo;`f1inu3 z%*5r6xXKEL!eA~Pd<%mS16xE$IMVwjE`lEXTM36o?^V)Ez2r-h}K2x_0u;@)XB9r|nd47FoO#hY?@C`Yi)z6q(@m8_Lr z4nGra+{dA8nc)B^x@l)<9= z;%Lg&%$9&BCL2ruV1`ou+(&*LL8H!q4`}?9W9E*b?i3C`pVf9lr{^Z?vLsQAxO2CAGE%yd^!eF2y zOu)*LKBaPt>UHUD0s;acxy-{{Pi1!b>qi(<(H%~q&WMg(riY2>p>9(>t)Mx*BDxDJ zXN2M#1QQz@7O-!52;N^5B6J_X%mDi(C^i-sh_?%yo9HyklCBt0R1OYnBwS>qq`~TJ z`U-F|Ux2ipswQB)BqSu5L6qqmF>&#STxt)tAVz=$5i9lbWjMZ84$=2i0s6gjm}RV6)$8x$sr~tW|m!y+M%FtV-p#VuQ(J7M3!w%lLUa=Akt~G z#qI*4uiR#aS6%bV@d9|mAVUcx7B|o^)2mc851s>57@dHr%l{5;;P2YU=PnhgFA1#? zMvf79HvzQPgyqlT4yz(S`XLi((J%a_^9kSufy85y|(3)*4L1-OkkD4sp9Ygyc+ zZ zfR-UTG4W|Nt;~H8M8e6L3gUiAF-;RRr6|oJXJ>Uk#KpnM3LE5=@Z~Y}%(^%reEAQ= z0s>M=oh?G2Yu1yKC8?&<&nyy-aG+vk^+0epG%BZIBlRo|`Jk_ z2Ok;v*=rUsS>6J-w_gLVVzQCEyyL~e4~F`~mdm3D@WID;tDwyTC+C&H>Ac^l{woRoKAF2YubF2Qw*!=lg7jMrQ3Nk{XE$geSndqlOGX2$$*@AHNHQFZ=V@ z>)lyxV0d-(1xJ{sr3Ax%upuT1t!J@&mQZMxSVN4OLx=M>qiKDeLK!6L9hvo z(aektbeL%EHnHEN@bPr+o9r@Rg*!pyWm*gFuU`@K`!^BE0uxnF@0FMD0Jjy>u!j&< zzUetRB;w+XeSLZGoWOH;_a9ttdPA;s=za7{6+t*#N2nD0q<*oo$iC!;0XH%gwQFYW z$0$%X7HPSntPYl<$Vg0IJXIz&J8p+?`NiC{k%b)_%o{fpwU@iYn*R(-{&vHD75urvy`jc!fv#16=Uo#%_XfPGH6_Ho*%C&9-|=<5?6pT>9~DD-NW!4aV@sC;O=1z_p(2+6O1IIe-51- z+Ce#Mdc=i+|ASRR{dmso;?T!*cxDzIn0qku z#4+`j^OLWm-r{F`ZPtTa`Z21chI|ySI-RW+`KN1Aph{MsWyKBrCN`qpt4|odNO`d9 zT9K4*YahRxnsHR-z!uO_PVU}tHuP&iTvnLqD&Y#yV9 z?_1{QkY10l4hkxhEr8!0DnHVV`I_U*_3K}a`(;l?ICM^2XV}24+%0w%+>~SAT-|AqwKQyu4*q%@GxK)_rqxVhlL*j-1UN zXG%5`u=Rj_>WK=}9WQ7g$`J#=&0!tP0+52OY4te?dfD_hl%k`9RNuB##OE4YQ`22hWi%WZ zR`%yEe!rcsX_iv^5_SoclspeCuxV)AqfUv~xwyo&llho*?$2cj*-{w7XTtmX+!!8O z>_h8s+l3ju_rgd{x|wa-hKET_Ola~7H}4DXmz3b2>|X2o2A~V!)Uaw&^opJJ`h<7N z`?%u?U9PK=@~VUvYx&KunubM2dTY062QarXw70d5JAU`v)QVPb7+!!931Hqke^R_yu-OPltIPso=Kh*6~*a&SeR|UyK0<}&`>WZb7Yu(X9vTw%z^aH{4tL`b8iVD=5#f|yUd!;}mO($Lz4E<~r5YfpoO z7$jf3_%x(;bah_4H!K;T-ZD*qj*oGDotX4xcx)`{%NLj5t6Y?P=Qs@wfu*hRaQR!m z&-JQjGU#CNy6JX(y)(Cmi(6!9qwA-#*X_uU9=RF7G8w{agqc2Ol$4MlTIuQfyH(YP z4_Sw0Y_${>0hKHrQN?i&3Gq$ey%TIZ+HMaHck&f^C<$%>x#TB!shr~C^{>Ym_(*vi z^lUxE#5f$EWXrt;00aCnYr21*KSQApY|bGOBvJ5P(F=TizbL_B*;|RmvHa^=RTB3l&oJn2VTWkx8z+ zk&Cu^nL7AfWs!DFRjHYaA0Q1Wj7940o}g6XHZPE@qp-^;SB*Ml{_*75;M60)WK~U^ zulCW!$f;jf|3qFJLhf|XJy?3@MkCU58nnnOBTp=uCD#Foj>sX!kB2S?OC9H@b$`4C zwQp?C^GxA<_pV{7edZ_fR8sGlGO3VHAzCl$3l8(zM4kHUsaA$JE0l5X_Lrs32T7aV zd}>c__KIHH5#43SZ9aVE#f~naXWk-$#cZ>@PxUI;gVAW9iMh###9IEs1}=y;pJuqFWLWP-VsukyO8KQ;e0V*- zi4Nt4jHdm177Czcbp@{G#+Lp>IQi;T(Oc;-W^kybms%pFd=r2N-Mdh}duvw2-Z*W) z22BR6X?H`OE1KYu<6#M5d%oB4yPmk%*!`*KnSYDVDK1;*ScBoEqj%k+b7Q+~2MxVv zdgu8ZWr?>tmYPsA8m%6Q+n{}5*d+eSv(8>)R}Ud_Jg%%0X}pFsw{#4oX!@V(RrFi= zZ-mIrPJ?RY@3t;AuH(7cRTJZcH4W7btkzQu$fNljs;vh2Dj~hLC}{hrmlXAcTP2|N z^wV71h{M6hvzGH2Gtc8jLwOnvof+S*E>ZWrc42$PEWPV9e8H=e)LC~(-R&6k5p-u2 zA|Q~t*m0Es$L*apr{{We*s#5mdGu(yD~a>U&bj)zmlu~@%Ev; z9#Asc>m679)z#DpfI`T=HQW5DyYca76*>eI`eI$)M?`IWd~SKXR4=Hqa=$JtT+Z}s zJfkS^md?)410bb>7&gviauqMcl;fkKu)sYv{A2%6;I+LyD?S6^Ik|rypepVLh&V;) zXld~k-!&cns+`|-84I>L!=ls0n3x!`ZXMVTM1y$_FkN@%2xN<(IZ&J$7ziI9&CV?; zksEb!Cc6~h>t9@12?Tm20+Inn?r!+2_?%ais+!so>^$t8oVvh9jYC9qC*WPDZBF`+ zbuc@5)3m?SJE*XkYZ0($f5IjRQ+Md_x&hJ=Fi~we4gv-=^swPxzn3?`Pz4ER*xmm| zhcPKiaQpxxHzmlFj(5B+U$O}a`DhPf78DfNK@5uGz{tpbSmdm*QzhX8I)(D%*8tWV zyeXIvEt~*;eMegaY%R_6Z^-QLRS?a?RZbl8459G@xeA1_`MUsW#i{=#TZ&u|ILxU9 z1axc@Yiqf{<0zO@DiqS^h6g=)VQCBT;hQkHsL!k?QTg~9Urb&1IW`s-Z)b0Z^$~*Y zKV(u+P|&kRJPWoVonX{Eh)nENp@Rc&3nd_YM`j7IPyr8eGIga zz~Tf11?;fG1rrM)M+3rbL5o9sIBA{<4na8l3qOGL8-rH5Xq@%6HG~CFj1@=%(Dk@S zQ&A;dRcSH;Ts#yk^NOATB853ay_g0Lj)0Dd$>-TROeFOtH`bWg*k4!@&=Mp5D0hHo z4|Yn?v9W&O@y{(ORCt07Nx|SKbaQvFxc2+}p!ii-QW7z|Qjx4-1QcCVRCMsFomO2# zL%8@Ay3gkh!qEPsP6{TV*I{{|eqDI;)|nSno@~jys-J6jVjkunY%r_)Q~m<uHN{7VV)-%4v6o39ghs#nqqtNb_IfEG`am)%If39)&kt zU0oKI8;BR!ksci015&C+m7L0G*MK}a;mSlW@1cmOziS7!%QeY8J13Qe z(raGPCk?vV(?zaUF9WfJv~Maa^Bir{qaqy~V+kCxLmGGBQZJYEl^0EHza5;P29k`U z;Ou2NWZ_ImJhE_cdKV0jdjn|{r>8z6eTJ4h+MVCJ4^nmqy1TpJY~xofVL25b&N`NF}>3Z=zQKAHjY-U+^FYC)EN&Cs0PpzNUi;xxP> z4LOx)nCCkmdwo5p{oL>kA5wa84LVORIsX!EP(Mw&&7j$0?O$OUpu}r%OGLsS$#DmH7t2IdDp+&JRpYMI`%C4us}!4oGp*9T9ZnWaw#GF2DMh9l)n6#md|HsFYP>>r zb{5{?+Rha87Hb*$j9s0T8vgmJM?>_aGq=&F_2%(zEx$7vE-um4?zL=>rF?!02d?Zv z^O~4O-O!C+zxc}Smsl5EwKRVDUSI9<4Ib{t#*ObMta)ViH|e=)^!Lx6Y}?v~7|3rL zdNy5*g;&fNv@A( zu*Y!{u&KQs5OAS;HD#kPId+#WHt+0PY{MVyM?C-Ph{6jnMS?KX3{3ylqW*WBiF z@oySxR#i{HJ=)d^;F^^Zm6Q|`JV{5Q@Kj*gXh=(Wi3tRO{4qLYkE;4diu7%d%O}>ir-vNs z@f*Vu68x3mAGvw~(uW5=LGak9chuTT*-gg%o)3rjlZ`^91&?|2Z2nfcM}}LXN>6^n zKYcq{;ToLp#0uIUS${h)V6m_-MDB51A|}!y{NY0t3gD~Ud-);qiuBLvD#@QE3e#r0 zD>w=Zl!>6g&Kv&7@;LDP>8G^>HJkne{AUY`-E;s#;OuE&yIA$1F+4viry%~i%;tG8 z3{z9G`?uy)Rh;IpFVZaCHzy~a*#GWx*S9-XF=W7+g~o4LN>1<*#v~c(=;A`>t6^Rj z7ZW6;|Hs#xKvUheQR5m#$`}dB5Yiw+rA#G5DpN^9W>FL(WUS00QJE4s8hDZ+MS~$T zMVwGX#$+l(#*E*-)%(8B|NpJ^S-sC%Z_ksCbAI=I?Q8FS?dzf87ut3H>$!z)Eo)x5 zL^nUvY^8PbBfa_s9;sKC%u}}(e<`>ZRn1!(BL+5R zQ!bQb?q$F9e5=u1CBL+*zJlY|{4_7AlX5aW6aItRU7n}sYG#|Rw()$q)qlib2?Ouy&&RHhf4Wm?sRz{9zR@CV^-$QHma8B@ zdG7dt0@ zOB!>`TW>RPE@;T($i zN;9N-Gx6X7Tt@o|JFLYLV!|?ijg6JSUP5N(`5x8vkpXg-*2}aQ&15-a#cg!Cd!4N7 z@P5}yy~lNYuum0jeF^evqRo?rSHZS9dkwGGRAR5? zD1wHNpviphg?`bGkboCR?%H_>C?)rk_`oZn4tg?|&AdG|GWa>?~8bDjbH&ZLCfD=!2`H%w(ELskBnOc|BR`9vL z-ZwpenjT}9Byr8m-2AdFmmU)AzkKf&vA@(`)vqF@|a) zhY30$?Y#?AO{(7B3TcL~BrLCcoPD8>Owk}S(L40W9c8gpl01R1^WOX?SnalE&6=9= zkf+G1!LmQME;<6rYzC~t5;n!sWBRb8uP^A`yL931?P6l(FdM}W^dQ?|1QguBS1Xu# zOvn2h;(o+ET8`hw}3nf6~-afzF zT-iKJ&_4*22c{Z%py9t4f9S?q#0at{-H&;cYbdMx<@4v0l%o}TNb3iXU~NN?N50N)X^s3c6B z#WbEYKxsM1=6LCnvbA-x6diN;M&MMu$%pN@>#=qYu9+#d%Rv>=aWXC%V}sl@cxsT- zxqaIiMI{y0$8{aBXt_;pd^;?iio~O0VdXyuB>}RgG~Y9c zgE3x&kM9Ot%4^Upa@hZ9Toxk1z(W4M4=$A9N$&!amV~R7-#g8Mxb`@AEX*uOjI_Gv zP#bpL&(Q!;Hb2+KnrRy|^wQf{?A@nN48~rsIITY1?r59;xnYNKT3P#j=c;iz*;EXN zh&lgQRrt)(IPAm)n}Onibnnfxsd;(yv};rSD;=wB7F@jdFk00_cC{q*gw98KSC)Nz zD6jXd#K7+BE63B?dQ-dSX9|Yet70Tt-h2)LVK_Fi_Qqtwd<%oakLviJfkj0*Bh78C zML@@_>(-gq+!^dL4?W+km|^DE7Cn$^n4)6Z(d@b~*L+Aby<~2t+S+D-cieII!Pr=O zn2e_8*4&v3-(D_kKlb}cL2%hWC+0zhrL~XN@ZNvpK4xB2qU8TlnlX27B95*pJt&A9 z6}9No(?Z~g*Zgdvyp`|XvB|jjcRp4K4S-QFq%N}3Vyeg6$oHO%6(H~XvXL%EYEz2$ z1-B|-{Re_}aEh-s$X*`$%E)c{bK6&TcKX@5wwi?6+yoWi>^jZQop=ZLdCiXw^!C=K zU&W%pfP{IeB{I%cyY-ADo3i?|l2dO_l?@pkDExiV!o!{mltWUyRlw(6$`vejC3Zd1 z-Qk9_KZoavxx!~kmsxnaFb%3BX#UVJ#6@>%`xL%L6?BjsV<++l&MKCyFFD}BAL@w_REM_X%{ePVF? z+jMVFRzE*=&C?1}^~~V^GHp(ZP$=<60_UHpkI7LjzKZ3h;?t|JU^*@N=z;89%gu;R zAKnG&Bi48CZp|=>YfFFQ>}S>_VGJH42XO$KMYf+mXl13)@^!nod$xe{U|c9#rE4)U zOHZbM3iO(A_*qnGX2xIMQ9Lj*VBA(B=H=j7@1@AdxOMiM@CbI{biT?yYif#!YvWRd zd2=DJ$v3&$GZ3cljQ~fR;xD>=CvR$2&LC%{z|PvWxE}WHF#eSOHAN@Zq~}A^RJRyw zNBgF)j*|F}hHtW2XC~HlMit~($j-mNM{$IbwY<`^5kL_%3yHTR)9ZTal+Ve#Fu}*Bernml8J1@zl z3%|sgq&;6VfMc0FzIR`&z~)k}=4?Ncudn;GkG$2Jxq16A>yPFfdb_-J{t2_+L~%EK zeBv@0?dP|2t8Vhh*P$CmLlq5|UNX+wE$m)K+NZZy=3U<4epvhybGFRe;A9&6MzM}!#?`8xH+Q?Ft@awd`|!}A zVUILVlEduY_U4LMz6jMoDC&L2#fQA)-wsda6z6-caGm*iyy)V+LA&ynNBVR5wHlh< zYt+=_lhW?q9^10Rh}%nV^!r^?!~i!qcF*xcmWZDzH1L=e)|_>pUz(qvophnDW~e~R zsC6w3VbVLx6*OGWVf<#UF(;iySmSo*hkGhgdnSWdZkL^x^G-Xf>hNTKe@V|BoX-dI zQ^B?K3VYhz5`QwRu@-i+Zw%)>T~;j?5Vq@5$kzPd^P*#u44Zx`I690z3FzhRzM7>P zxa`xM$&y)0*-55!$TL0j_NyBvRlI4^pKEP8cSj-8+kHoBeAiWz%U*KxQxj^5b7cz# z!A&)L?>|U+=4t42oc!gM{N(cTnNjN}1Bzv|E;qejv&sE5PoO0PaH!J( z>t|y8CzDN0XT}aJos;vCo(V8`>Ggd5ToOa%MMmeh{X^ya_E%{SJ8tZlvx{esoNIrw zJ!8_XZKVyxlV!3-KU=nkPw7g3KhJP2sISiW%1ehI^1te&lDAo}-_;|$ zrFz{e3R{m;LBY}|5qw@-uXLLItz28@p5sq9Q!dq3=k|BeE#d}ONzqcLl{(cAIb}u za2~Ph4c*gvMgM7h#$4_7H=Z-Q zpb)*Mig}eXm9Mj#_I-TFnlBbB4L^*u^N;9SXKX3V z$sga|$HJdf*U;c+JpU_-(r7&2ZM;D_^Cgxyw7kBw8Z@c0jm<#xz?M6QOG-*MA%kQ0 z?%n=J>{oI%EdWgg4B-lk6AUwIEDm5KgW1})u;H&_;9j^MU?H90J--*s>GtRA=vRQh z9xD1W-BdFiQXR^*#~$W;ZQF)_V6tqpXH{liQasnWFu5>q>~zU<&Q^wxZwUaQdw)%z z(?0HfPOFgiPnkk$OL#;-7xHm3hv3E;Z1(vT| zNf!R>6pwLZo%vYhH2PiL<9BzIVlxKXF}aoI%c-7AwmLj|gf=$7?LGSr!$5e8M&}(@ zBEzH{ispDdWZBQjBK=fyfW80Q;F18OoZjvFbJVW+;mgKr(~AYX)zUCokY05EQw zpU6Q5UvIBw*=*f26X&-rtx^2T8Yn@7}$&)>4|MPYT`QLvR?@Kx;i|;2t`1ToG%$F<_TS zwZ2caRvL1WW5i4)*|zmm_uy0A;^N|E7k}6FNN$G3V`ImMdn=`+#@j5^wK#H+tzKG`-T^`P}Q)@!%aB9(!Jv zd-=kuRT+10{tP>bP-7G&EES`Dt`FPpb|xm1fGJnaite%nxi1Y3Vc0aSrTghde*OWd zkbL|_oE;q>InwOxhKlcOZAkv7U+MU9Mw+)}Ds;xiT)zJM6rX0_+I=91v%m3;dk! zO^fVUP<6D~%$;|h=#<}gYo?D&MCd*_c;dja{$1yL&-MAWD6QN|>*!kL>^tCC|5jXu z&-t*ej%Mhq74B{;?fZ_cdk5CDzJA%(CmjPf;Q$<{LN7O3YugcdM76j7sUm;cwM_=PZ~OYb&HbEZ+iNIv~)5P zOD9qZ^Ne1$VOJYtU*8?1{T$S+u~*Oq5zjAh+Gx5>;$v~Z_KM1sgLs7Dm&na~n(34> z^q{r8KqU0hy3gsKDGLm_P_QhZ zCpmpeij`V3oBH~J%b?UIg@Pr!cHPrTVH+OJLhqKJk>UEvW->m033>it>S69O=i+_}Sp9?s|P-Ob;t9S`a?USGE}HEj?a!{&46_Tl4c@3su}U9RbRxjMH{ z@YY=W&&zQlLvQbWm-iVQJtbuJAB1&;hNgf0NsrOF6GhMR2bU23?`hiVynAyQ&Qbv< zqgf4?8C{@pv^mcq`fGqnt@pXwdO)u+r5(1Jf>&u>8?M}$f9w(Wdu}_;l}T=4kDjC# zT^}qdv^?sOk+Ikl@*;6uTi9*iw768VUyR=toll3QXTH3Zt61_#w2M1VR_ea;>9eb2 zMP+JRYeVM;4^(_B_W72|tz*x4@k&z8_-$8_!>_28Vop3L21&&XvhLc^9zP%8d;qi$XMJTWPg3(~dj zT2UM*kHnhiI1a~^Wj4HThA-JKKW$0JOQ{@jAA|cmEoYiE8%^3UBUtfHzaynV?z7MD zn0AyrX)I%UuK!dd|6=!HL|N^pq{ctJe}4&VY>dwkuZ#5PSbPDp-upVn+4;`c*jz7|eP7_4(pR#EKRr45E$Zj+HvwdAq6;5< zlkx0XEI~h9r}i_`aP#nFTGeb!)+yVgb(B77-QyeAuNPxkvf#VGz~1h$$;qy^g}F9y zX=#xX@M9$Dgf5HU++}Y15_mPb^DqN@%rv(}?i7&NjUZF>|LL?v@4GL~b}7tvJs9J_ zQUx|_Wmas)N{F_Z)=8|UH5ALGepZP;h@=#_@9os0m^kl6M6i$$S4D$Ft#>98oXyr!8{)9^3t{3(J91vUD8(8L zc(nIDt%-q_4MdhMG7)O$u?(*gbZFf@jmIP-c8B51E7+P6`YK-ksR}}CAb~SPe=Q-| zeW291p1*iOCyotFfG}4237UbAB9NXpi%=U5x%XMmpO?7gkMdwht4FC(N$M5$Ps5VL z8#H|pY%%uju9=#iJ{---94?O&18eps85sk+COl&h>xB>K{v{KYYOu20A8W)2%msGT zJpCCtzmVF=M(h{kRd=^WKZ;CC+kqdiL-WXy_b70A90l)C$u1bKbke^91)InEM2w?;G+CaN?>*h~sYoarLUUt*R4zgOh z=2jhND9lsts55Nz#hMwZQsVh)yUw59X8OkI7kyjosc5r_h@gbz&mXE;oGStc0JwwnL}5WuG#$c87Eu z9Upbv$qAPU;jhV76gGuR{cNxyfJNT??nmGEDu=STy>$iB&yEU_phD`$kNmMAx!A{q zWwhsQ+>^CMF0Im2}Nva6cS7c8qu~_&zl?1%cY-mT?)h zpsp3T84+;}X(PzPrOSNuNC+-6--0BC)eHzj< zxll|6RK^F-az?rALv^)!{RLawlJ@pZZ1ijQP)r4`n?deiSi4r^ostSjail@|YCK*> zgAF~KFp+?S`{-d$YKiMJX~vy~72&;V*c?yEMyezAYuLN)i)7FfbG4wc;qyKTv_1Bz z?i^N(zdvYXv=L#_n$~4lgJ3lLf>@bwb!Lf zyU373@F_h#6Lev?fAoJ%O!y<+Vp<5!4biPznVEJ@zbr+M=<%yk*lkiCF=udN)80PZ z{h;TlR5&)@DE)*72U8X5*X0xZ{a2EtD;>6tG;H)McH6av;8-8I&B?5O5{Vj9v!gZI zMn-8lM^kjnFwDEs`vwxlM9(!hHD!kRp1B>Kpo)vIwfTen_U+rH-6ys|cG=T^(NFOz za@z$31*-xT6_5)_$%a)4*T@4k|3`UHr?4nt2`(V>w65-MzuULjk{iSpZp7)JZBB9F zjg5<|vf0TmkXY9@+;GykPwu?|7F=K+Metp4;DdUMR2&Rgd*De!aHq}ZG(RMH5qC-` zk%6Fs?YYoWYmkkJnyR~dG_R%p1vY_Y5YV)a9lM53ucg4-czu>aB!)?_Cx~A}B+%TW z!g_pil3c(@Nkfz6HWVq#hz2^j;T(Qp6+~b4&;gv{GDL%g1@GgT%BS?}z9(NcY5rW+Sf!pTgvknFmwR};6_ zu@-SH(3S!_%fon~RAZBIhEfbOO>}H5X{=FqC#g+MTT1L&bLmi%34|#6@qlB)272V! zVJ?F%lT5&1N_k>pf;BqG(JDeDnHe}jCy?l zZw?~(*s#xUyQyBC_{{on{kd%NAm+8fxSC!njfP?Vr=f zlSqB_`gQOZ{io~@ijYQrO9U_>N!_Hrxmjk*ibh}$xVyH9dPKrS#=Mq02d88iZ}1w+kVafwJCRMPu@$&S1tLT2EA|4_>sKU42yw4MyJvKjnlc$KvVE(jzIVB}S?w0tOX!OkuW2(7Rs?$*+e_2w{%aoC(czpH=d zcaJt!l;EN68@D9Ky8H$YmVHNvzQ8QKhf7$JS5oM{ABpEH_ujw-4+ab34Vcx z^iyjTmB!$P>xUe$tg0$re!){&S%%mxL8;UXoipPvmKpNp;#Zcqi0(-G(mzM7(~Z*= zcsHVcP>+_19{>B=B`b71hR>WSy2?@Zpi^VnO1A)EeWUq%)9)(0&5iS7G$AK;5RhZ?IasEM~l^0>NW`g`0Rn>Cs)muKWLHWfwECClO%tqaa zXArx3kcp8^_q7Iv6w2RFVp7tuOCV!tJpg2M|*R?PvWqi`1ExLpf*9EROG z+A`qGF1|9P*joRCE$&6IwJK-gEyVqG_%?HB6h3jN+YVi2Yxp9u*$w=t+A%D8cG|}S z)qHdC7B>2A#`%VnkidL)BqzgHn0Q)8#}^5T$eLU&4qS=o{{i*+xSKb3CM6V7tJ((! zuEWxhc$yTl8@f+Qv66gvxZ|L93s$neaT)r)oGN9B?+UwXaQt?_Wc8St-&7<&lB{F5 z$#GnhUELt_8Fr4N*S)!@PPG6A&!0;z`c;7%B18oEF~;zul$xYU zMPx#74R#^`Bis_%%1@!9g(B=E#w|HYc?$;K&s-*86kif6@e1KLg?L{G>#VA$!lW7y zbYN&G90DS2^pOpGU8u!c1_mUq6Oda+N5=$ThRxU_8=(KxAHU$e`0?}SUS}Bmt%JY3 zaalJp*geJXwpU?VTh?nV$u*^JlRy!Ihm<~=5riEzS+4NC>$=#+&gwnISa`loNqO(} zb3(}M>#PVjo1H1>xu24v8o2PY?E|{qP_`|%!nH-!Yg^|};~SBee3)D9ukk?2=V&Rd z9XmZcyOhwp_=PFxIuZNU@Jnop4=B&Jrebf~SUV;f6>V`7V<%zBUAufbG%s{n0{EwG z%S&c7(B#naVb|7B?&w+^sLfRx`Xnk0%d*mp3wD&?qafJf%9SgCtPdY<#s;lxj0&?% zc-!6%549s~Wa?8+KG@Rtbj=)j>tO-;kITmTnLj=s>3#c(+S-iavMC0&dov*+jCAJC z->WzQ>7JSFf(wkyMovtSGQqS}RJ7>tDZK|CI=S8T^yzy}l{cwqf3D)iT2mh2I#6p) zz@CxsAMRa8m&E4`4jpINDTV|NyiL^-N1$$X2BUhlQhXg@ZnynFXfDHGXlXCo5NcHqCJ z%2wU;E9;*M~q^GCX($?k+-fol|UD{SD zg5%$XWn)EqFXBT013+>72J}82yaJsv{NAPaAsQ6c)G*@qP{h*d4InUzBn5Dg@#f(e z26|~gl9?R*ngJKJb-og7HP*5q0#0$JGQTTD37;K!0Vo@F2H#=R!3}9IyNp>q85e`c zM3a^ExpN;42n_-TzPP+x^;%W_?q6{q+;w8r(3~ote|U^+p$dA&w*}>$B+F7pq%@yp zWRSR4yuYhod3+CjQuF>j>kAxIQ}f~=p1$r~d-foBeHo55ry~zOZ$Tj;ehG<-Up(ev znFR1_PtUpIpHGP10dyor8yKqa5| z*!5u>FA+P*zWJYRX$KQ;7DFdMU-bO>7)p_`-Q2??p2KwsLWgc{L5XCh)>B+3ME@2^ zse4CX(Aw!f;&wftB)A!e4~AY6B#iSHC}cc@YxC+d+F^{jlU(1|%jxigArtp4mcaPw z!4I`|qimp@`$SbQ-L+d05n5VWR|W?!@Hr#T7!zD=*Mfp7eSeY-b+`?Yyu)Dz=tFuF zvbQXs`}}f}7wZ3YtbV6WuNnp_L5{Cv=>0F_=(fTd_VxM-;u`s-NMdr-Z&Bo{(%B1xlg0f?JkA zMz6VPwXb5KZ58GR9v)M>O~;vprWK@(w(UO}EY_Bmq`AStY{&RD#<~&v_omg2hlhuB zN;{16Ls0ZcR=uvSE{lciE;-j>8ql%UTiNI_&v7W4S>2KpUxzM#5%!H6Kh%6RHKeRy}#wJD3k;0N@3t4exQn zt2@iW7qOqf)t_Bm#qKSf6coyyngou}CEgK+JIaA>@yV>h61-v|D?!9zLDW*pOimUd zH&9Yu9uK)>05z0q7UC`N?&9eY*tc&3ed$;%dmkbKu!E7M6tfAN-}wish{6Qt;E*Y@ z8==bBH)R08aOiMY`8B!9qN1e$Wmf=#NbA@~@IRgnhoM$EZNnW=WW5y9D8qrj|9(OJ zVMk*LgO`hvBTGmb05jybRg8>Q3fnmSz<&K1Y}lVf4Vj*ws;lU}h-DUdLyNKBK0ZF4 zbzcE5j*B`^Lj%vDqq0(_%`>H9ktUeQf1lCZ72)QBF( z72sD{MpouL**fN)kC=<>?|-d&`ece!p-VoipYg02-BIsj2Xit)h`{XZ zY?d>JU`VvKwZ(GAL?$daECeN;={`BgIrA(HPZbCo(w5U@;z`HO3?5_}#MOG@s$Wlr zY_@M-X9m`l?0$eEEbL@FW`;Mhwzd}b=%aj}&@=haGIri`X^qJ&-L(-y=s^vQ8_Da{ zQLex!ow^wm_=s$C(q`@hy@zI?5W#Bjl}0c2jsTp{5^q*ofUGPltEytaQBr)9ljCx1 z(hB2II8BCcoYe#II!vh)WmpiS>pknHLS)%P65on)hJE>>uicCi`A(DF8WbtYo!R{C zZ+Io&l2q5)+WR?1c6MTDpd7%uAYj52gbW6cZg1*Jv9wsS6N@E8?i>z!mY6b#yo=sJ zb_Id3>(_6IaJikLqoV_QVirnn=yM`*5E$+ebvToRjM|9aX-*p5R6X&Z^r;`~u@v49 zfRp|w;HZ?8)LW36sOY%#S)LYvUz2Bi$XSdZ3zk^dvEFc|tzu%pS7o4}9ng|dr!T(Z zZq7b$zOYrDOP~hPgik*T{%&az^Vt}N4hE*=Yjo}FE4fn zi|n@fv~2>-eGp=D7yY+rtVu$O5?J8dD8UX`-J<9+W@!&4PaqV=>qI_4`ppv2qPtzvt>NsUJQ1pJ}7ZmpMzL_r<1fJ zul=-Q6MqDIO)-kZ5Ui#a#6?Iflhfn+$V{W;=X1%)`9Q71m-xyi9H zq_QsCz5C-NSA>IBiVnpYQvHJ=M;t;D$-8ppN*WZlwdC@Cq<&Y(vu$ivg?*Ika%ydr z?#YwF=fY<5W8PswI1}1A5}%JcNHSfNmHk@XFeygh4v12isF`_r84=7(C$ZZLmC<#$ z{i+B%H}^K}-BLJs`^e@Sw$Vj=^F)c0`u-Wj2$;@6H*YDycv>db04B}kL7M90@kTO= z?Bj4M-Kter(E9Yod_YyHh30@F8Wa4qbh}+;7dw-vUWtoiC0kY>^+JpBM zcY27&@Serw?joCojb20hU{_sToey@@lFM&>k9fqfl*L&|4vsxf%h@quH4yp)D~Te6 zIQdzqeNGr#Ngo^RLO;I@*DpD6DbJrjZ?8=}=r*^REMX^P*(q*26%LRVR`wVs*kYl?1=|7@oGYG&$tN%3%jEtRxtYGvYU7oO&PZ^!=*-CL=S zjk4Lbee=5gQuvFlS9bjn`vWI>7+?XrJ=;+6Ak@G%JoBKW$JGf}g z@xLSYRx(qMS6W_qRaq5pICmL+{$h9LIToHAt$qAB>soHaz;8U=3gW# zmCCnBqJj!8DJ@ltzUKrHOZTb1AEW%#{y=bQU2r-zW|$WH4!+08C_dLhRbEfQ+$&G( z`pDP@fv%lL8dUbiq)&G+OYn-nqF`(P*`zNR9mt5Hl*6Vs%uFVH=0erNFuNjb|`tUwC* zD$@VId#9+aO-&}YHV&7eR=WGLsWwcf)d4Y=HU=RrzlI$pk#sM`nESAUOYz%pA2z_IuvlJ^*d=v|}=+WG#P?kc{py**jM5?m(c70o0 z*pDB!xMD6<6*=OYqR^=-Di%HGJB3)X^cR{Qn0%28sh&@1DE2d=oPl61NM zHR!^^!a}ezemFyIDf~W=p1?Kxt0dvyAHWJ3q)3&$SOZ-jQW%V1#QXK>lk*?Ww9j|N zle3gz>DW_`YzP69a%KD!ImhvB26oj5RH_aKXTne649M#Giy6bUZ&g}k9IK$p-}jt& zhoQB^RJM!xZ)MivoRBp;WRnRHAm4m1RlsAq?=ks3;ipkIA(dm$`9_}mq?x_FR$#(j z$CcO+POejwm-6OjuJG`1!a!QR;!X0lO1barswk-2VlDGzjt#Lu8|QZ_iS_bd_PaP2?GIx=cw z<0@fcVd7NSe5cIw! zg)v)!Fu(9}`f5YeFT!_5s#7lr(d&y9`ZjNRAI8H z?*W@Z<=il?7abf-Pd*;EQi~I!>+nvlH8wU5k`SQueMiePh3yDRT`PIBu<{m_IV5|N zt%1Sa|3;1D7n0sx#8l7h&XRGqk@$ct&bnNsM*YZ~?9(oSm^;cG7D;TGDr%*<+q?{1gc zmA^+%iOai;me*sGXbaJEpbh0te6iUkx{dGu%QSX?FeJ8EaO2xdoj;y>?X(4|sj!>F zKIoV36F*{x4`dKp8JI(ALnxE`)G!}mEWlt_}WP=@GD9I9(%}jl84qxWfvA6$XohT^bVflibf|Vr`~ld61F55 z;CQ7P7m!5}gyevBX2iS?vwag;HS)#S(wk}Dp;&cKhU4(VW7k3+K1OX^nD^A)h82V4 zFpyUY84Ve*OecFE2JR;5U`I|owNf`rW_T4A#Po`X^8WnosEqTlG2tp$u@)N z9%-^l=o>#yEW;oVJ$5K6u_^*uq0_wu@^W)m<-rTq2)IwXlU-jfFI5Z z-dj2>BS}gVxK&9`60{Ml6yw}|fDVR(#h<70QCpw}ey$}cTfBd23)jtLway0P~ z^g6wMoW_fYL_jy>KV>D{VR(ckXB$FQrX)W#CT4>BcXf9sbp$*H@)2dd7SZ1!iN%Al z=3fWn@y&fp{vHyFRR;|NFx~L%j2T&C?LK+o$JaL}$IQ-wOUj?>Ty5K!ah3J?5xgr5 zWMYt9`A7>Po;M7O5=hz?aVrp21fMh6kj1Ba{5VjiEFK^i#mUXcG`&1}W_BdZo6S@H zLxKXMot+)L&PRLadrLvOkUoiAYQE$>Lnvbg>}*NfcUrB&9RPr^Zr#1Aii-WnI~3-F zICc=K3-mEERGA7P26r~GjUePiy`CHkXk03I*6Rlc1|%Ev5j-9U);uXE=Vai+$B&Ul z&a?9URh7waL*FtND}+FSDm)e;qVlv1lci9oxXB?#--0xE$nhyTMg|64Wb6~aO?ZcB z1@HNgfj9r8rG1(yz+W9sU={vb<@2Dwa04B>y+&OMBb|gkM31NV12jx>a`MgGw!L^n zu_%Uupz8OFJim%kjj)6S3sFo*aEkC*9=XXJJ_7cRS#GPFEmT$lAD447bNWN~(-FOs zJoNa*iw2PdUgJEsb#S>+ufKWori-+Jz=6w8F;kuDAT2>gOM37?$u0(%iJ)Haa7nyd z8;Sp;=J~w-Vw*jdHh@nLe4EfQKoI0z4&!#yjxsoP$`o7XfG~DWJ3iOJ{T2X2O5*lg zl{f|FAOAI>st*hOzb9175Dj{KNb(LLXz}dpq0@){qLf)%tA&(F=!48^-2^~(9VP_> zo7d! zU*uqHC5|raJX-qt^r0$AdxU6AQQGHMO% z0Iaww6D=pVyvKS}5C1<+xH)#~k-ix%mM>sc&rTEtIx&7bV0&m7<_q(4iySv8Bglic zGE6*gz77)xaPokUITZ-Kvh$XvJn|FWP4M`k#Du@^2H^rt?(5bP0!87s!hLO2RE3sa z7_$WA0KCBvG)nQd3biDZhIKFQ#>Qf?AvTO2M@2T5_~q46*;(S>=z=xx@CR-x8Eh*E z!!~-HInHg2ANPE5c+1Z-)>vVd0Od?FI{`y9=v|f&Lnf*faU}HLy_|~iYC-~lZrcgV zz&68|Owd+YENoo`R$m2=#A(^#F5v~3wWCzYRFBDWYUEw-bXYw%OT)xIi-Qy zlk5)=`QF`KYL+jx8EB`!6x}#k3q<#fPhun3v6tQp`9>bNRrKCC{|w11NQ@~)kM3lr z&`Q@Ld>0Hp1a}ND?8>vuI#XQ)&W`@!u`LJAC`o+&aNDvl|5>QcGTVIXfln1lm0#Hy%iL3))&i` zE;TuSe!IGa#d8SRc%*N}$KxaO7yrDPo?a2d*)1(C4AJ*S@O@N4iHbnlrbAuDn@=g< zCnMuPMslhEGGG9WBs%Y5%=T- zFohaZ-0pu41FIkDBOVkIX?O9<;)k^2b<$bjDzxjB&7H1;%2n! zn@Vt$(L!9sGo&#!56Fhdau#jVPmol(6}-5y?kT}xVWCVEPZP?^V)%f~g$re*7C#Z`EGzLCGYL`45UtG9plGD?@`(LAlz%Dto)b}QaUPm}V zE7!&8t_!u{!g>EIn}WW&bEiSj^2p+oE|t0HGTK1E(4Vie8(D&XYeG1+ zYY_$6)BJirIzEp0NCSu7T0W}M9ay>X)R*H!+FW(vj3iorvdJ1dPytL22DZWPjDQd9 zElC8R7udOz32!K+s0N`JXrQVd96g2rY`911wfTHzCWpEp1j6%^Fg!<{s)S$+7)(eD zen4k5~x1BqIj@K=EAS!agF59c6!=T-?_ye(+ip_{Xq z0geujjorrYvJ)h<@#+3ai({g9NDw(54?jJ)#lhesxZ9@Iu-67ifHG0h<$EUkH6oMm z-dgKLsB2unfL360S0IPu8rYWJr|1EQ#Cz@9QW9K;I^pHzRX6DMJTsG-*k<&Ip%LcF z-I&QaDTO-B4LdkApIcjY?`B8$Nby=gZ=m+!7VcoU-fxKPMm2GDbzLvTNEzuL7$C@R zr8w!3g0Bjf0k6nt}q3}n~x^H$1&>Y>9J~j5GaV@BH?rJ{ixllScUz9 z1dAdHZY**;+G-%Py21RIK!)_Rw6$=>H*8(Hng%b2_#)v;TON}qjk|%g237b+tVJM0 zYHqG!{r9ZW6r^P&0#*`yB@-owrG~z^w2+hiHltGA6f!;~`|!igQBD@(zJzrQ;BLJm z59n?9d)~ISwOxtfe2j0|l6(0$+g%x#9WdCS@d@0xb=$TQ&?|o+k)NL$fkcTpIUIOr zl{7WOL^dJ;W>GSQAukaM5HG0C-qUMo&;w(cMD7w*RaFoGf64)+?-8_390>9@TFXDA zzI^?fA8#^_1^Qx1Eq|%w=*8yc^&P;K)w_^_%;M#^UpToMj*j ziiXc0W(kL`qH)WABR>OsPf64G1RmNTyPy$r|NbWOs3v$$#E9UESP8%jiwI1|r?5H( zxEcw~j7nmg66D<3p*H;Gj?A&e(RFG#QOoHVx-_)P08H`29I4$G_Y$RJvelh&ks!j! zg3|GaR|+!<0U1sX*kgq`I=~x6Sz%$4pabxoPxvSZa7QAy{mDp#v-;QQ=w4=VW_3<2b4WVeYP#FvD8bQRKYz9w<^~=OKwS<_I|(L!;E7;ANpV|0kY}h2PUz}o9gA2zo^`cu*F55dGrJeF`?Ju4U+i24+!&+hK7ok7E}4T*(R`1N;R&8 z5C#yyur)soTmVH7$dtv=Or{#;=@a3f9=>CoUS)IhB$175Y$ZL@N)}S!hsbbB;=_l^ zY<3#pHAl5%Pbh0~{t3c4~oyJ#gYg!|~6%#KjN5KMZKv0pQtFBxQ=X05~1ZgRg=> zkEkm<05c5Z5dTTv+|Y2RTx78%sIN|C+^;0KCf@r$3QD7c(E2}zkYB=W_u;)qE#BVL z=sy}5cxEkY)9~*oAr-W#D1KwX+8bq@8SqK2BHX*=MHiQHaCJyiWBzw66kPs@g>uDt zL_sKr0Oc1A>;@YsOuKR⋙Z$W%yIWNGw_{4)vF?Q=m&xiV}uBKTctTt{Ffi@)i=3 zYd^x^*pB7}A{SlOtq&j}?cp!ePJ9EjK}L^J2}tV@Pqwa+Y)pO;h;%rbd{F2%e+UT) zhrbBE<*mseh#%3IkYYxV5)t93axRP9)*7IP7$rOs>SZ+~C$#Tdf0Dv`5&^iG&;X$Oje;?BWnL5*Hqy;BUtII)1 z6oRjAIsntCBL09(V0_<)0)FW5VMRT?#@l&9u?)aWqzfrEGcz-R?4b@~Q~EEq%ETo! z`sJ#%|6a%MAx7S?G_1ybT4g1tWWffHHaQ@Da@VtRa!BVz2eaG1jt5+tbZ8#&t?S$` zNJPMqh?9ZVdIe72GRs4Cr`_FUq&BRTAC-L{R`^)+)ekR%`Ac|c=+6&qAgNxB>oQ!ve^ z6i5g>JT(La3s!|-d>gDAtKh>_q1(#>`b z(9~i`1_L6S;86tfC$=iW1RN+6dkn!D-QRx^)+3M}SehIR^rv=v4QFsr5ZT-8FgxYI zHvwUX4KG+w(Bk7{+t;t5&?o}UNbQ6G{$S{CuO(Vc)M0ft00lDoMbgk#XF_?9^_aTZ zRxKp}a{=#DFk z#Lt3i)@~v0d6z+7%ytwy@>BY{ajeKEf);ly67QfXfbDUw+^UMsYy@9yD^izKT=bUp z_3hnaE^xLJbrn|!znq+BRgoUF65={(om%PUwK$Rl<0YUaMv?>@T3SNTt9Jq@n5Pkf z9ek{kipsBPQ#Q~Qz8aI4Hq&6F38nE!G&A!O;yxkwTX1!}+}*A-L1&`j!6o85TUJKK z+Z#MGV`U{Qz+^zP1bZ22pFtx}77k}*XGi{zlzNB96~&zx?qba?$R2=y>3a>2qdtMt zB7)MN(A<=ivn|bFwf-8eF*(?#t+H8?f>W*!FFH%s+dYm zH~C;a$GG5esZ(cR)Q8k4?0Kw(8Z4Nu{~~yWeq}O@0RnK7qcNsT!Ia31U6SzZz|2NB zGCS)5c9v_OI=|1yJ}2m2#l%zSaObce1PQ=}b-=sUwzh1MIHZLDleH-Vq#4G_+tZL#*;j30Xoe#kUO~n4h=;`#{ob2r2 zFUM0D!+S0NrCZcM1S4i=6b;@>uR=rDkUU8ctj{)mI?{{diI<~HjF2;mxxBZ?3Q4-!psS$ zkS`E>0B-WJ4=X_BARmRRT!}SY8-IB@3=_V%GqhVuOIHBVfA?JR_sS17K=HstU=S(R zNl9J*x11-%vxb3z2=#xa_TV(3f~e~4UG*nV5OO*)zK<$C`rDl^mKKKV|0*!;wuA>F zUs9XxH?3q2H{F};+Io+HOy=mO?uQu=4S}C4u_wbMlGEnJg@*tp5l8geMDW4`M5etu z0Skz$hPZ(_clFr=AVOhL@{=b0F1k9k7<)qH;IbmVs@kt#x0e8utp=;i{!5Y{su@0U z($V?qf&C%{49`SSiN$wMZ*sN6O$mdbm#ONL2BLuxLO5if9TT;ljhj0J2KhuXh=$XL zNkyV@TbP?K!{daKK{W37l~Z(pYho~7u11&T(}Tq&L2zQi@v|xuJ#7jDqicL)0;DkPL&OsJ77Y1Tg=kxq(20F*`<>vN$hY%Liu!R}qYG1F*niQI0Ga zGQkR055m*`#6CoJ=?4z~R`rZHs+iyd0lB>>&FFc0%E68$j=or}*$$lOMMptV(Rp5IQ&7#9O@_#hD^bc{}_u9WVG8jK+L z`#^_WA*RQyyI%Oglt2H*1B4D5xXm9m(hu)<%S(rY>sQM}z&{J00k%GRczuuOLl9@Z zuIQZ~lKc`&F)q!P_hcyz?i#R6i{{8gj2V#d``U9aC{wR54YYT431D_W$@t0Br%1e> z8BaR}(wHc0H>iL4T}lrbbj2JK23=9S!C=W<@&sx7XVTSLrBInE!)Sm$18N zYofat%K&ze!MS}l^TmsM4{ZPRD392Q#3!-szd|2uBU$5%&MkPiZkyCxh19XK^lJGn zRnRFVf|6!ks8 z&rk^67&8-!scC69+Ab|eD+hxVwZsDac*HkA=WD{8J|XnM4E|jfOG!`oz>!Nv9$_#ic(6paY@r%xB7hZk5zs$q=M-^x;KL93 zwK2b-fEU*oWqtbIUS*uF>m&vuFeK!*2vbWf=oOa#OPi<;J8`hHtICe zrms_?VY>&yRazkm_Kp}ypTli}dtSKP-M>N)KR?QDB6bEql6}Ex(T1XifU@;?%*P`^ zX`5p2@wVXoN@8T}5eE{34XO5DXC8pzAr~aksx9_hR*5e@RqJ!+8tgCH3!#QP0&;S4 zd^xFyaH#O%DCz2ciuMaGDk>uLBJnvwTIl_gdDgFAWn*K5xtxQ2c&v1F*`bY!Lz3#> z2YwP*RLRhgJ7G>C03A~c_8_qVB>JFLhu?~s2BWY2p1V;GL&2{TN6A4@_N73N&C zA$i$vqhsduLpzBydt*E9{D1&j*lAbbiu_-9lkxz5Odv(l{A0EY%|Fx%9UV422;Ff^Q29Sxod-PE``iDur=l$l360YtX^E0V zhjxl~B%vWKN}8gS(y~g?I8GXJ8Y;AeqH-$IP|~5IAr(UY&+AL)ckaJOkKcXnb4p*I z&wE_gYh7jc`YSHA-H!w^PChIDTm5ys_LfJDY7|$h!w!W*TUiU+*x*g!Pz({SA z>46AUd_X7XLi`b%cyVx}^QJOskO>~btm~)$fX9-4orjJOwAa+BQ`zovJen}?V~yD&?K=)X@Og_> z39VbFVIG@NxAIQ6ZHqEmiEu&aIgt;Vn-8a4oG`O|*xJ(Zlb25Bl4a|2>`t1nYSlmq zx7!A(aW>4so9{+Dd;B^OH4%iHcrB0}5lefwrk-w)!2%eLL)xZbK(y@4)EUvKX# zvW&R-+$v|TerZfEec~T?hc|;mOU~-IG%QP1Z`bEb)a0JWEWAyDo%lRU5lEyDy4nqQ z#5|t9M~U6nW%TUdPM_}<(R`nuBKw*GJ1QzlH)%C;4&hRfguY&*oD2Qf#|e>$oWS8O zJ`LN&=c||Xx`l*@8|cvf5_i+1a#Rf%Fh&1#t9G46CMdO5hI&kkrKW%;{2 z%fHW6C>_b!VY5r+&~NBX){nGHlE)+E$2Jx)CP7KQNsrmGc{82)NJsWG<*7Nf7zF!t(Q~z zk00l*)!#84#`*0fpcw@&uC{x!iHXT8r2RiSU8eBoFYNZ0{}pa1PSn_4vxzX_);X2EJzaZ`)=vV#J0A{3P&a(gViauVl`G{(^DMLm0A@_Ma{l~^i6zAS$kF$NldWR|sv8KLdPN9S3e8l14&c*oo+}2$<^FaR>VokIvI;YGW7i5JgYW{Q|v} zQGr$fX7S)zuD67k6ji66Px1p16HC%dp=*M_rH;aNJ(mw7zK#cQ%G-F@^I$hRZ2F~G zv=gpMY!K|&M;y7uxDVtap9gYK`_^oGEuaO8GPH8ty^P!@|q$g zmWUz+ybuh7F!pZ@IV zhm*&T*B8=n!GaUrPfzYCmpNk?=KTHL%JktgJ}>e(H0y;E4)n!iO7|I(VON5TH$5}6 z+qNBsNiD4gVzib9Hj(%>fDWa@Cr>uPjmZF+i)J(X>`9S6g;VPWUB02Ike{351)X@O$tp3 zJHz=FcKI}TwSdm5s@Dg1c^ibh>3_~I{vVuQ01MTzV35p)AFFRF9VOu^)tbrwD!$Su zU$>cCbBj6NSsF8bsZs~k-L)>+oGo~=s~6xUUDn#q*BgX4N*K^ySpZ5N(z*4QP0=*_ zq2xlLH`<4V@S1+@346*fXss8zJ2Z!htoF^*0OmPH=Pq10xJNkwYC>)>E^0dLkssiyhrQPwkD=*$P*9>l8%&wrhJ68U@ z5FNc+qowA}$UpzwXnnHvQ`%2(kX4oD<)(XeZ&OuTxn}*oqf`N_hxQ}rDBmxr8eUR zomv2z=rw0J41KFawstgOGHS&i^=j^dVPRZyRMDE< zx{dz2?vbMy?%KECh6)muZ*gqH9Sgw_fgY-TTXHU>X@4=~08tSAuwr}m_O9M5skcS% z<^4a^m;RKqZ8z2Ip5sjiy4723_cdj!uaD#FrAnbWLM|f76#J79XRA5I1+t|4>4zJApqKpX!i8mDRp+DM@>Zj8vTm zNdSg;%kQ$$zB|d98@u|e1_zn7DF<5K*L_dW?`XGBbV{BI)tg{~G#PQZ}q$UH4$GDTBQGWd02!$SfdMgZ4WUq#pVgSSa%vuq{_LgY?q+n;!bFObZ zko{BoeEqNAOpW}@A7yQ|y8S0j`x%WB=pX1Kf27u)iUv!dkf)U0I45gy4(n)7)f_3{ zV^U`?KW_O6h8Rt_z`XLxp5MPe?$O70?@<2@`{eFutL~2cJ){1?u>+4}8q*$%$sk*+ z1lCz92KO^S&=SNka9~>oQ)c}3qjA%=>t67>{Cez47r`)s;OMu#t=0gqgkL}h2-tYk z!lC{eK8RT1NC=#803n!&JZQA$bq=Y8M8Wqd-(QrxI%>iFxP1T|B>Z7VvD!lVJRU>d zrwVPTII5KGLfgVj9a8lFVl&RZUwb_VW?JAHqwt{95S8o>Z}x0r=BepPuJ=u~Vu>Px z#52{;ZYVJZVa|oWs;0CK!k5za)9q2l9J+j3n9X_fRA4q4xC65}$rt%32RppK=!Pdu z(0xDc&WY%=XT{@KU8Et`>DFU<^}XexkjALh|L=7(3zfX$D_C!k#wMTSe$P|g9;!l>Vslu^pTxnSB$cTxBvF}lo>M&yKH&7(8JZut!3*z$Qs2(q7aFF{P2Nge+G;k z>NWS;hB_vo70Qr?Z{lugVOo1z&@v(U-i@7)pmiwYCWzi1KM5xpr;8xUh|1U{;Zygv zRze^vQA-j7N+BjVD}J$~|9rbN{`DT-S+_-A`ok9Lw~upvggOLyj9N>ue}Mn=vX8{r z`H&VG8Tqcsk`m5)MfENm#h{@>6FwXG>O5p><-|#oz?3w$otb&14b9vA3U|~|9ETC- z&Q0`@TL|{^25{MC_$Psi&X_L(2|&$PGUNV5t}`9 zfcSq>3fc~@su-)^ai2J_6iyR8q=Ex-AOP6A*Z;U{g_Hul3uc*Um5UoFg>$H{#I5+q z?}t(7V&DO?-9%zcX{(zl21|-cQc{!yUwl1AaAmw5g-gZ6`<)5)1wLyFauz;~A1yC6 zs4Jcmdvh>PZq>ut(=x~hgbW>n#xngZ3=LDjBLtCNO*j-nfu>k3|HaHS?T$QlB8JZo z_!-{PXvIvb7GNns05mk#-!He|kPu$98UWxh$UW4+ByuuoSZ%A_`M^#&u4Y2Xm}=Eh zz|T$p13zz!194V->CeBU=>I+Qf=x}ohQCn?9jr7~50i>ST?#w_E!C(9F$fgU9E_bU z90s}xwojcY7s?-Va-}svl2;9yh0edbU$lDEn_MwF*n#BGn2qX+H`f^T301N%E6E9Lbf+JQxOITyFdQ@X`p6 z8 zGw0Rjk_ME~M+kFi)ud6Qpa-50z8;Wh#Gylm%o6}@{>a;=SYyCLi2Z0ikB*p6U!N;? zwquACo-5w(?e4H}p)VZc0gHW(6X^=y%?ueedURJq!;098BknP!7suRhP(Dgeg*xA%{4rD{GJKQ-(0h4H{nGCr#TMwJIG@sy)E$nqp?e=t~Rr*`Y z#9o8X{Kubvwn#+Z&86f0j$#5X-&G-&c;-LK%xoU>H~M;7S>#TdHqD1BNsKDDACEJ= zeGGOH9R`s#Wq}1K@HBa&n6vArx_HzKZRqYm@(6IS2=$39ss;}uCr=}?bh~zoG^J|F zLd3L*BTY=s)}UzZS0eip;kBaQL+E9&`1zpI3m*Mi-Iae`yqF8~b_ovb54#tNup9mo z#6nUU3|ISm@`LQ`oz+TZNB!Uhoqq3%58&F}yG?3BEesD|R^A z(P5wMzD}I(x#}+iRYy4qF3qtN%Ir8FMmI}SLD{kA!fc=&h2(SHk4P1D@^(<@O3wC; z)KnMx)m%FsA}?2_TZf0Y{y(#=&&s4b01|35g)tYI(pmH0RMk!aJ-^A%Z}New3u84O zZU(}dCw^Byp28SE_Jm!|PBn7Fe4lcXwV1D2)9!!_j`vxye7OddimR(@3kQR^9pB44 zyZb#zEq)WHsx)Zu;1Ne&eV$@RVuo+#tToCUv?6a8O5#C<9ZMAOma$_^*bTy9HVT<> z_K!czA6>wgzkU1N%`PDhVhSe00#QCA3(rikfVSI&tZ47vy$z$G0-4lub-Zygq2Ldm zKku~-xX#Zr!~QRUq^Dn8>^qO;i`U_wOBY&_-#O{T*hcqjArzsPyjk8Wc%oE|Omc~P zy_e!s28~?$G?ngsl;=s=J^dpVqPV^ev;gFwxORbP`(90>EBG^3=Nt*ez1J%%$cV$Katneb%H^Eo~vXX3$noH zoRC?>j7rWK#E|n}6*gS|c2*lDhqT~&$W_V7X*&h$9wak*prxrR!1YDW$9bDQ-Rdlj z?cvLpZCg!8k`cDGr3Ea7)u*R+WRFzr>>AvKny`2KmQ2)bvgqkwzL`XdMZ#<;rmL*F zT}!fSC8+{;^}lkOfB-%kM2sLvGiF3(uId*+dO3iLlnI4#V}qYk$yLiVU+!MriJPKE z)$i|r->>5&3&qZ}PZes$@rwjfRDW4%ho*1*)!DRK)uf(yeyucDHCd^yzKIoxo3R!L zP|L(|p@V9{yR$mY_dIDjaLsoYPRCc02pR=&8n*674Aa&$a?u4QmUlyab%_}S?Xd$r9M%(e(jnNZ(?#pW!vus%ZIQeMNQXYf!Xa_;&3Do6acorM5NvE z@w=Y9W&;vkU%m8v#X!d1JzYW(rE%tvdDN(XNIbyNs6|1Rx2v2@L?AHiOZjHMt)x2h zcWnhIME)e%%k92hokiX?B5sJ+CfFEqHiu4ynH$CQkU8JW9$YfaTGg=W>(B^MjE<)#egXGnf@Rdsx#@HrQTDB~{@k}Q?2<34jk^Pt7YoLUxQQkK(EG_IE z49yN*VUrxQ*ql6Ef)W=E9C*>9WAUJeQ>?8ud-q=SsJNdJau^xy1dsbBM#)!#`OtMr z2rpqQ9$kP8XlIf@i~gJRnY&<73zTz;o(45sr8jOW9CrGU!7ee^AFx2t|_wXI3)QL#w%vBR4FB>%a)C8 zV1mLO60EwHx!ob8Sf5h3W_RF>r#|o2Wl~MA;wW|({n-C7m@U5B|H9DrzFstcTZK+; zkEOq3qAQ)YAt3!Z-XyIQ=vSMj|m{&9YYAO_8y^yRny={>k)t``_NDVEMq{Oq+ zYTLWYx6pQVk>FhG7PeONmH;3Ov_}jxZ(>Ot20xgXa1IZj)@|C{AUT3NY0u>)?TMPm zJ4|g!H{R<1vlO%EIzTdEGT45vEuDxd`dPGpBG7bP4xRk-XO;1qlFE$9mlOPdj_tC3 z`O&W4hg1o*wRtw`>_}_#kC`jnXO_D!*!?`FQIGo`8O6;;LU~RTO$>+Se9KW85+O%# z+pc5B*3Ohy&r|Q*OLKX?pkF)tF?v^cNpHV-xEc-_vW;CWJ(=F|b2IyURqTizuYk*Gs`pT)Uu)0O(gsaz!i^6EkTd z%kRgl3VSbdu?W!W9eM2N(X-cE1vP5c%*@_ii=kGxY&l2yqu*wQM9yMgf=dNFr5GuT z-AabpqgHSyx&sDiM=qh*qU-FuJPcYnAodgW-SU+yDfC}OoNJLbkZ?idZZLl4@42fe z6+SlzyZHqYgZ7`k?v^d04X!R@oj$v#6o3A_%MTT|@!NG`c0_*~^C^@-W5JBTO^pY2 zRn6))LS?*-jW5w;L7w&Nucu<4%gMVGkfqDP6+vZBKfHU_W1Na+lZA^G?ejZtd&<(n zMJ_mjn1cqcMm1<6(vZ_Lo6zDlIT0}xM#thddR9*NUEP|-9T0ceRri*(S!agbCZ>sC z0HpL=Cr17?XGr;%(jvPUw^05a7?7T}=D8)5ETx!L%jthtO)SU0xDE_|-*MvcR9aUD6saD!}0FU|9Q%^^4f zH>ijVUS0F7^|)d^M?KztM5#O;EvnGh?uH<`d1;0 zTVQj(ar}U_&d{Oif;tfeHBGd?;=9EScO`3W6-H$srnsZ;rT~b>m*uI4c%9M+-_k=* zZ{n9uRc}&ST8JfJ7C-ue4RdyF&xH!&iQEnJd1Dac`ut>A=BW`huB?(GetWbv zNxg(_QpcqPrW&MGR2Em+wWwQ*O%@PLOorvcx1NKC3=yUWB@yV|2-Z54LF&%_?V2KE zG@#JhM90hVi!rhMVCNISXnHgfdC$AW9e{`7^K7@HtOTy+BShK2oW5DHVBWmilJSg4 zEBjqOz9ol@Qmi7X2zuRu_c)#=?;)kIcp6$&>(evJu9v8h#m{v_kLr6K6E#qm=Vwp0 z%m|gw<)0ElJijkTeSuiP*lEuqgvUsJQp!gn$dyfby~M))gW<4Y+PQr9l6ZfnV}#O| z$|k)`T`EDUhYuh2_cb?N>9}@n2fCMQ82Qr{PogP-q{X6tX;z(UY-6+0qqPSZ6zEKM z&~Gn7x5zEt-gjEwHnSQxE*GVcXgcjrJwM68e|F)+<0V79h8N$o*E_<&JCPebCS=7y z%EJdLLI`mGWg4ivxw?)(rxOqma4FmQ5X=M9J1m?8mm(&yn~Lg@!-uaGuDQoak^{HQ zjdE@;g{ge_jT+|fWwjyCvW)ccSXi!z=dDTsx`TUrDRC@T+kiJR3tZk9iHO3e9;CAn|j)IB)1+t2A&SL^17SC?`w zO0qxF`hNeF?GnCa^U!U-uCa};Pfxf$DdN_(H-#aQEmYID7XRHn=XB2P*BP>q+4lk^ zO$#}6NLcM6kY`x8;kTE6w>V`j4NNJ0O=|nTKT1nQBz$}Me%bT-0|s1o)URFYyyqJS zn9%XX9=ey|Lq$SBmG3(sBHf31IIpTI!m1b~k>tL{#lx6Qh+}#H~^R86YCsTOd-C>HeEBQB3_-P&w#SX?&ZJ7)0vQL5Vc zF>N|=bEAO6uPoOriaLk185^`GYF7~pz&g4m&WMqX#`so5G+;Ync2pDZG4Takq~q; zT&)o)%_xH%JFiIKx(4x-)j3U~ap04$nPu4Tt2^QvL{wVv89)pnL8dWLeY7!v?KA+8g%k_j#7_Y}~%m*V+f1uJf&^=z81PYio@gci-RA-1NI7 zp}>V6i2CP82x9w{pR&{3Q>GF#5d66G>F*Z`$+ZdO=XwMW-g8p>XPOObL*~1}S7ff! z`O5dgY$bBGyWSC*P74?6uDK80vbTqFT-w{W-BJC!f{nPo_-={b8&Lihlr)r1uh0iZ z0?b^wGJWFRl$4yoV>N>e*HyHGiwdyiyoE!vAH#(*uU|W1gkn~l6h(!U6k|r2))i|r zC3wG+3rM8u)@^J4?wV7-aQM`zZG}~$9YbwAuoG;`cFxGC8Lv0C<&=H^d>|A2LeBCk*13A~so(kukWid-cemUziZ92x6Yh2-^j-1w)^p*D4*ebs zE&5)^*XM2AA6qYXX*lcolhivXkI1M|V}ap(YE{4%E`_uCjPc+a@vgTwe@FY$8iD6E zQoi~wJm(28gR5JEJK*;6;`?NV;e0~9*~Xa#EShmB!MtzL)Y428+D7{yodJ|={Ij$U z=f)f7z}Av;{uBM_C$B1XN}_p{z`vireUr>$zjwX5f$u?cy+v#W=`$1iR4baG`IZ#b z_xK#(%#O~^EuCow0HY7+wzdfgN>jgb`Lg{OlZ<(JW()ZE)|u;NhJjS5oUMsZ1y>Ia zZVUjWwkPXQ&<>;0paTx^<3zZCs^_@=*Jy-E+&O}bp`nop7gT8uNsTtH2l7KBEJP35 z6I2T^@dN$*{Ufj>PK@~$J0N&EEK)l)wcU>j=V$vG%@{kjStsLlO-HCGDRJVXD825j zte#!!MqK1@bO2eNv6n8TPPYzk{uos_YunsOswj!V?$E*L$Iq56S}ZO9zR1;(IkD7# z?#AOxzjomS*B*SaVa54IC8P(X=3O@0Ykd;+n3>rFo-0V%{gld9?Yv- zZ^SipH?Ft>W1ay{;G0P*8=~pa*DOL6jwWD$<(V;XRBVB~H~r7{R%_R8mmgXdR`{Dn zBkgPT&p%vQQE`@Ve)k(sc44)84c+~q;9E@gM^ubJ$I{RCY(3ENybd*5Ks4&bP*3CI zv3C91iL@CEdQ8Tj31cGCHF%RO4VQ@vMsRk>-^>+n3|{n4BUAf)e!kBTTjoyeF`{y| zPdPQaYk2Gey&#G4q$3&go>L2L(Lt`9ChxLh+WR}sT7vJ(Bf76wjOGTkGdjT<(+M!_qZCyPq@<5JAXuReY1J4x>@At=3} zX><(@du0d-$!U3i&7?48b%VpuGxx)c+d8v5O+od!z}a5!D$y`X*i328*3SKg4eOn8 zW=3pnx+odG!L4@0`lMWJ5^W!~!_QA5&YxUzjqppso!6F0K3Ro-L`N4L4Bi${8CW)W z72I-eZmvsv=OhP%SiK-i62s43Hrr1ZySkz@deEW3z~NM4ocn$4dOqV#YuOo$h@jL< zd6JR;dtnMIBpU3LG&#w(r$+A=Wa*If?wbdll4J<5c~EvCA$k3Q%WQ*2!`IWejgbko zDO+-Q%Q%&~0v1ecqO3fMI>FM{@raxUD2Ne=-BOgW*wAUxKGdoFl z*svWmG91-o-V2UH&yxQ}R}$y>K!$6+TSkF&3UTSStjg5wA9{5M<%oS_w29kka>{z> z0{>hTP>6E<{QP{*LwMmYTz#+8rsQDolGAXd~bt<=44S&Sy9;_2ZoWQeI8`nR z)+6ku7cVNaOC)}&y309bP2-m2Zq5ZLUl}_UjZV_U%(4$BhFAYPQ~k_Kc3o?byWWGk z%s4#4RKKrf+ za>~{%_}DFi12{Qe-|}r)nr~dS@L`A&vrh!X78Rs3`U)`1Q7cz|ziRaTrC0N+@`S4G zzP?qn+D#zY0<_Cu=tVt{B6){2%!$Do4M{ah!bCK5LE89JHeBcbwk)ItBH7`}OQv{B zp23AAP3ulMfQjMa7)cJ?7UfG7LWkvN7BLN9wF%LpYmb;)^9daWY`U(r!>sGbU;ae}nMadbgJ5)x!F_u5N`0R_GTGz&Uw;M0-GeGl+3VdV7_bs>cf3nD{m2gJ zbZ&X~j9g$cT_dADvQA*H)7Dsrm#mN^$dxO{IDr(nWxEU^Fc#ro%Ov!p$gMtoObLxAkdQnt_8tpeZGMpdPKKsb|^Q|df+pGgaRhZn{bM_*W86t@>R-(t_^n#Y^q0p^oM4;n z;uBk~BZTeSa_eQM02= zoA}v&RI2$)Oh4V78L{y|M2pInIxORDv48ZiD% z5;8Kag*~5O6Bg_jA>(t?N%g5tM=(Z4amJ!Y6odiE=kQY(UOeP<(}hVH&9OD@H;g$5 zlUYOg-=gELUS$ZD0UATIC^}JLkg6*3jms|1nGYr+=DDoBUbNfrs@EjlJk#LM7;Pr7 znIP-8Jf0Ar6lkd<@rtw87S(&X#5eg*v2)XdimY2|I!PDui%bp{lLndh@q1t>qkTg< zx-DO>f07i5Ty&02$Pr)lGxJsQY8tN1ay7Bl3b;v{_9*X1-GFreP6Gv3qX z-uCeL5o+8fz1#^o&zXduPS!7ReP10gE0dJqQ>R)dfp)9Z-H^W_S%;Y>m+($II+{lT z2@(MGpzujf&RmR5sV+N7uhKxvDjtB6tEToto7S(>XVz68)_%-0qId=ErzelEicfs? z>M#xBntr=`yBEE#r_-&Q_JC8?zfB6>a+1#JK%bd<<(4G_B|8nooR_^;`NA^xVSp=Z zHDp>!s-u25Mgisr1*}c-U=0g0y8GZkW&8q7x*eU$ER<`7F_Sr+ zO)1x;O2^k_2J4%LtTlkHl;}UVotu2@yELpV@LSCJ_DW}&O2W-@_unEe@9yW%{pcmv zLp@#n-r>_rA*7H2LqXh|{j?uwC0FlZW|(=zbpqF?xm$Op9617+nJuea7u6&Yqr2dv;%N2t^G$w=VIu-SxkJK_gp{AqL|l&+cE+2oYuJTn3fcliN}iT>2Ce7rLx-BN z7Roke?Fvd*!w(-6uxkKy&p_vUc~YLXKi7(X8HEamdLHSa8MPp#Lh4RB(wiu4Q{QuB z?)R}UI7|)(pDDl=5-h*&yEx=?5y+r9Y_HAUF=q)_6u*00Qp1fBF6G*+gA%#EaO;jm zS522Zq>6g!RW)-WIvi<<$esx`J_Oppe2xvEC(+nUx8SIa@3be1Xc+Mz23r1|_L$?~ ztjD+a?rUbSz|{!oiQu6@tQLO{G+hRuy;t8Fv2wqmd6d+c;{t8!)IV>#gT3ChXHTbi zyB52Zfus`w4ASyCfR|K(vh08XQ48p8*nf5mLlozyjIeLS1`UYjj>nhrrcsO%TT|wi ziK#j2;>pKOKr2G&V4FC*bcn}nm_E+~C_R`ee-{A3{w4e7S0#t%(dxLVseL29BZ^|bHx2hv-reY$e`Mm{hq;&A^ZKb6UX8kEeH*p+)@B`V!1HS?eEkRo zs6^yZYtS0#>HK9(jf({6=Bk&T-O3K4lDWy5iuYgdc*tx27<^EnJlzw{JU7Y(BS5jpa6>si1;pAia0Vvye86qJL zY95$IW5KI#O#jj3*wHO8{g!$w5w@C`Y&4wfc>T-g&%-HMB#I#eh2U9-wDv?V)>0x> z`2n)cvRDKQ21~k~>|fmjrL5|&Bn&ITX#*Y|~?}-Xh`gz%V8CB-m8vTvtxmt9-2#|me26L!ouP%IiUFm5U)iS z%P%^_tC{yPA5AD&SNi}PCx$HdSazU;v4s8dtEe)$G z3v>7S0CXrP>rY}{rv43fXI#a+?$1{ZUK}mSXGRG3bS}2KJuZJ&y}Dk9`U|4m3XE?Y zmJf-nWhW)tf_C!`^W%ExsHT@=YVgK`U_U7oPMrB9hScU;R}86H{74ol5F-KSk(aKNi2UjdPs zz-SLoj9V~lrHf0`s!vtrRr6RG;If;oPclgvZ>g4vRxZ;T%?b6tGsbt6oYt zseMqG{mU1(C$kS%Vp{ov-|<6|&xPc`_dE;+YyMtAZi|%pc{84#{8W#EwDiNpms-xd zlz>jvrN{+{=&icj$3<8Idho5d>tmjUhK8P9QD7}R*)EqHz4ki-+=g76{c?q`3r^S@ z^mtn5`8e)3dZ|=j9pIohY4oC!xYKc7_*c#D(+@~Z3N~9%lKU@gc4;(3qR&}4@N-33 zab_pRNpGV`CJ*e^#&>TQ@eI&?_KHM}9p9!!${bJ>uZp6rNAp1RkT@;%thHOYfBPM@ zz@rJ7KiY{F8jQTa=nXn^?Ye4aCp~nIVgp4?toU~M zSJU&?gFpUh_6tAw+C{%-y}wR3bL?~|U}$62^o=K0xUDxkHk^)JqDA3pEc$Fa4FZkW zK6=%sS${kZwZ5Q(ei4zJ%DNx#y3s~&`$n;cHph;jJRwrKR%CiC)tz>vBpSY4SKgc8 zv-x8T5{M^_W*4QH-G?2OpV7(b{Z-@mn%NRyPSFO{=PvTosyTM^e#>Qetn8t!o!6;^ z{H!f=%gz$0hOXgm;NqXuXGaLYpjMoBimVJ+5|>g_Q|pnG^rV9mQZnzQ2U|*8d;G_<{QHZ3$w?KpAzbl134NHjjzqpBDuZv@wYDw z>1*=IjPJ}%j(tTU_h3Y2T2VnXLyi;KPO9XS&?_($i1b?9LAo|EY|WIbGMx1oVoe8!B<{oSST_Y5R(HRe>94Bv$t3rxOtomW-w;Z7%RvG~Bqnfi0kw`wf+} z7~sj6;{CR#tdHwwe4=m$WY`tuyyo%C0#j0v1}#WX0-%H||J{k~6TM^6#?&D<*+E>b zKc3nl=8?CL)@OT~-GeI>@Q1RlPFlS9wBFH?-h*DJEW%rIN%wVRD(mu}t z$W(+qVF!JZUxR6ZElofL94UGKFRrvD7*9|8SGcFY?%=_`01n|-uRbq)`XmT6M+Uq? z{RpPfy-V5&PDT(+iH8G!U*g#;^>$KHgX+qRx5}WELtBql*LwD!o`^K=W5tVq!wVdb zpXQc~DW}#mHAod}0$h8q09LuEYWPVrqzf7rw6%BR%Mq zUBTa?qlYLV6es>k4X|Lh?ml{TPTn{PSL3v61n(6N`| z&z@C5_(OvhRFdRRbQ5Y48jVC-Wc8G{)Mnq^z1Dgd|6d4B25-h!exJ_Ob0=`|I_a_? zrmB8qCfs@oU3z*}+%~3=3Jho7vwF|Gz63hXuvdd}8}ID0J3nRe|8x(y0bPYH|rrV_=j^P5Vs^b z*lg;LkrpY}V9geB)?5MWsw!7mSb=@B)^Kv2DT|D4&E+D}(p>I%eFFo{Aw!;O`j6wE zym7F|l%9#rm6a1gL{+bG>X^Dg`P5CYLn1v zT~1AFlbzjcPs-O>B{?5IHWRTy?zbH^+ongM2<8UP`tWZ|^k2J{9$2U7Xf!o#&+cF2 zxb&Fkq&{mDN%?g!)S1NT!V4+*r z5eXAz|9#&NPwLX2u-Zs_h25pt{WKO^2y>DKe;>3r#RLM62 z@#GUmJ7@L@+S7Z+MvcJAkFnPeRl*cer5`)mMny#fl)wzyG{6xPARaYsQ4V~Q6^E9mR_1h zoXQsV$|yG#n~e&20r23SL_z*z>)4Elk06CCr4sK1nK-O=LAm+<-ukIf985t9;b_*Wl3q~+JC{*Ead z73kXSR1q3fe>K_iVMW3(VC_l{%AA)rTTZ)1JfGm>(GJXzX8l(UtnR_*j z`9L=jgveBpkgJ#0@)Mw3x``+*(U!1R5O$STt@J?gHqQzm)ykaFCd1SB5o)p9yT-SMb?C5izWY;5o`so~^duCQ#a9GQf_Y$KB zb$s9vcEkBW2^Qw&yY$z2eR>=qD0EY#wRG3x`=(UYt0{q@@~{$3L8 zaPLZV(7>#oBUH3rPTG)=VfJwLAtTRiLP#aA`r?mzWt(1)?&W#mbEb^7x^JbL@(KV)ahec- zc*$XXvbV&@(^5NxofPqw&~tmY`4kG7DhDMDuf@#j2m@3!U+VrU#!6)19|A&hAm_Tc z6z)hub*MRbuo|TyMNIT~li=Uz#XuITi!r9Xx)*#}tAi-E7cunIWQapVjWeWu_1$r< zvD_?Qi)Lm1uORC({y%CQyUl9`g*#VN_W2HWR%QFK=!bVl^a==Z`>iYcr>%uu5nEP< zkxBCsVmUpCV+84m_{IXnm+%UA93tD4y{Se_YT!_4{?fCgWBmpV5n&w{*dh~`=kT$mv+e5c2qS~5{ZM_s+BA6 z?A;u7J+*8D2X;`v)iA&-`IF0*O^l-Znfoa_PVczwrCA4!e}r^MdzCUIb?s~>n~D02 zTO#AQLg~p^w53b?A=j}!GiwKxvUDJT;W6uOgUQ_@nUQ!2Z^`k8W2EuhziphJ)1pNK z*2rjMI8m1!#L6;8eZu6)_A5isV}iu?I^_G-N+3CFYn@29K`EqH@Bz7yC`I@Yj##_>)?)euB;sl?(O<5<=%9{qi`|F{yV0H${Q_MW zOZmOj0%axV*Hgy_pW2X9R$KEfO80(ijHeTTR8xf&F9Hx?qRznbSr?hvfo(NKv1ceQ zh;MW-*!lsSpl^I>Z5HVN`j=I6^@0$(2&7}^`Qz)3q@JxB)9Z@xhFa}n)-ZpU$B;T- zJ>p82FE<}LX7vU728j4kX4tc z$E^Nc`}ccGgK!?y=}1h>!@|L1mkf;Q(N^4WDZAKGCttcL6_+TVTcRkx0Qc=<#9Bvm z1Wps-%-A6`kPaonoEy96(u%tpOjDErVHk=9XV^-f}SaC%kpBQ--Y$fK?g8kv?jw&5H_Fkw$ZMp>~+2T7^ZevyI5`|NXYCPZv zpM7cR=efXY$$Ja?(unJQL|%?48CDitL*E!0XTiY+AQlpsP*&;ZlWhTxM8PY zil{fUUOrotmU{d*M&*5}+a-8`qE%sh-^_$VXf|I#dCDk=OZDxKv+ZTF)&`;&wmeK< z`<{4&HEXUMZPU3k4A!&$_%%TGC7jMI!ZpJV435-B5sHtD$jM7=l7jM+-caj2j}4gZ z?p~i{qaB>LoRbR8tK^XuH#ar7lt~p=>Y>ta&N%dev2! z7*n%Or_^ZBJZ3BI7T4^IfA5y$;K={rWV#gxr+j|UU3bvna-9@hqaK~wYgdF$Su(7T zhKAB?&u^{Q{VZF6C}P_;Xp>iHmW5!f z-?d2@{^=gtkQyneKR15#I*?frlNuT#S}Pt+76&roQh3qC;Olc_gGO`bNcU1%#S0_& z66sN*Ez#A;2*s&$gF@QryzQ6l7bLk-{HjgbJ%itIoZhuzZPER)Hx=~>%ZE8BySqL4jVhDWWLkwr z(GpcB;+9FOX{X@)Lyf(%3(JF`NCh-ONipp>H_Ujp1r~$hU&@M>pil>b7q@dl$^A`e zsZs)!S0r1Lgf!;10ApHTFkeV^8df!X<z_t; zBk5;A{$+S7OV-9e-BBNL_(%_r%UQvVZQG%m7TeKI9v)Gk;D`2M;)vh9Th(F_USv@~ z_E1$#yU3Aak#V|?*UXofDdbPr)70FKQROD?Nyaeth-4s1F8Y*nXMSboYPxOI7*2D7 z)tf^lW>(j)f6~y>!eBGqmN~Mo5>UbF?}cg*g4KE&qk{9tm%RSxpL+D-tOy|lKbck+ zWwvJpC3|}_&Kab+=F9om{-OWQF-T&>norR((}kIr6dg=xyS_=QviGKAV^eG!Fyxfm zr*W&u5_M(92U3l+sgpg#k0atk9+9ZKF0*zey-+07cklj=gjxnbisng_V;RJ$h(Nre zIILMisL^h;>f%O-UN~jijR{i>ozin@7>g9{M(dA1vLzg4c zla=>3M6OSxy!vNF?6iWyLQ|%C`$d}f=^wH%?&z0^t~Beh8P>R||6@Iq9cF@>m_?*V&6+%UGd|mi-lnE`GRH>FlioTy^;vy-+CTcD zI0WgXobnNq4v!_J?KYRO)aqIJwW+zF#-^pYw>~zp5wd`?!e>wrHCsn{gbH&2JJKAj z$aqE)h`@kNPH2X2InK3D&xNZhA!`x$4M|}G@!R@pG;G(Q!`??Ip#EtzQeRkebWul- zKX#fwrfV344(>7Yd+!qNfMduzrl@-9K5TIBUUWzTceTGCvNjYbmkc?i<&O$0N?#&- z5M*8GTfOPmu-lQ*-zPFLDaoyJ@a`=G3=G6TIZ-OK`z>2$ksVU)97E#n?XKZwj&LE31~bAs`T(ocZM%b^p#($vQ#2o{ zidn{Iqch~H;a3s`twY?$0xJrne|>97-r;K8nGpWeN@&P+icu}o7jyLH@Sv1A4^zS;oSU&!3l4WagsUns+KHsws1- z+dK=s4la%zTftBYQv7Q-AokWA{Pu-pUyvvOr#x@bvSlq4;sCuzM2M}ut8YWgh%*aKfKDroO& za}qE~Jf8TE=AGiY0oUKpb<@AN#HiSP#CN;p%Uc0c2iQ{SSuRX|N{XEMZK`CiGw!-z zTA~|}L?cw8kp0F5<&0+)6dcd9#okl3%A4DXsC#gs{x07Q09B;c}^U z%$UM<^dnRq;Okk}!PU^#?tKWOX(Wgp9?ty5Hx=0~+=YUkQVxC`JpUM*o>x+R$&hMB zyj>A^GWRl<5gZY}AT7Q;uhs&nX7v*J=8t}#p8c;vd%AS%?3z2r_Ve1%N1s<&A8~%( zeN`)@GhMIQ>_{B@$#@j8R6EcIwl}W)+7f{NmInapy3(8mYs>yQ;4*FQC?v*4RTXYR zr3sN32OyD9@uf-pkN3aPaNZ4k5HD^XoWIhzzZ?yAEC~`A(4Ra{EM!T56?&(aX6EbD z8F3R!Y|GgvUIn~D3M+*iOzhmqf%zVZnrd$=Mf3&zsesE6<)Q*<#S8=vSZzcG;ORT5 z`Njl0UEpA+h7{SXSPYa*f&ir-vZIh;gJs;T4p_Eu2Y@R(YWTp;Tnh@WN-@pMw* zh z)8~dpy*q&TmU4<|i+&Lgt?iWlMM7&b?XQD+)B0>*4KPYTz$V=H)B;|Cj~Phy^Lkj1 z{7E8+evZF~(q~*2#1-S^%3 z*Dq~9Q*rlHdOH5+?zxerT(ghPHPX!<`}TdieM7NLZ{+&Pv3k!fEvC<)Mdu9q^;_TB zt3q}3&JH;`&epfkLQ7yLcTx9+%__mIIxy^u1JTl|PHaj3=310X*1 zC36VSM|`%1)>vv?TocoHozO6K$cpT>KlkA{WA2s)Tvds-EG+&5S)0z$9lkD>sOv?~ zyov2wzI?gQH^K6FQ$)gqSQ*qBi+mr*`bT+r;!{>WqO*ll(&qan8?mTm&%D0GWK6i9 zb19j29>)FEr>ZYT<$+3wkdwt%#<^ijr-0FQPUlzC2GAXy`Q>0xZ3@b77$|`+b z-7aeP*Us8i85kQGK}N*DuU(bA(weC4sCw~$JTFahpdvm_s&;7MDKVkxCXY@#5Fcj%9yDJ>)oH%m2vVG#m|MhB^=Z{txIR~C3F0YG` z(JXJ%C2dKt-(iLq(n1=ZI|=n{cS%i5#heoREP*sD2~&H`hpS6XIsT-3R!`MRVsJWg z4LE*xZ==L&Ity@sf5+A+uw#3%w+%BiT-|K5%S5r^L`mH?Ppw*J#;({MHEhj3S~6&5BMcZGfus8=k3 zsomUxqRtdO8uw*tUdshvii$?SZBkgepNiS1Mz#$Qfx704_>Q0j*;JY;W%U+XYx$oobp2qbn(dmbW4dN=_*uJF8=sGWvQetHzV)om7nSaju<#2g zfR^z2M(kDv7%;J{4H0`7>Geo!T_ci4vm5tjV#f5cb(|LHG-68n6_gmw*kR?&-9FBuIpCTcb~62bj0W{b8+8?@x5 zQAPR7s@qaio+03sbq&;q>cCh;#JoHn`?m8Ox)Mi->|~cNWHFhLTX=7-h6wo}x}ZCg z+ir{6D0)t2TH{wp%lGaWyvXeW?X{GH-uj)zGlBDz2-pqN2-Q}DdXur`2n9Oy54k&J z$d}rdYXAqe*K3Z?^u`V`m7K|IfflVY)R-eyDS_snaDF%WsLh!o9CiU z$iF|DQ_&FJqJC&Za zFegL+6OE`5I^P?cmND%>az?|A8f9Y}R}5&o=;W;UwZ0U#5wWrE8PBMwa1HF>jB`79 z_FyDYt?7=fl6vlIj5b71$F?0+6=x}axFRqHRBN3&?F3Jwas%iD6HqicSaWh9Xg=}s2 z<0i<~PTou!_)X{y&g?C;aW>~_l>iW~t0)@XfzgFD<((@B=?_F#;|hdY@%d)+0Eg4? zG@`|Wm`-%)+vd%uPyR3q(ohHd0H`A7YYXzDpn$Yt9G^Hxg=d(z2;i3R3!)$PCoZE# zea)SfxFn%cCxahwcUYJ||J=rDz_<__btoH202J55@#8emAbX zp+l$Zov1*%LJgOCx1eo$fuZ}Cy4)%O!Jx`h2ELv_G`vt-hy`nx7JR#kP+TPF-6FOe zXFQ`oqF=xMbu`}TaZk3^-M#N$EIep7!|! z*i&2$i=SWGz`dIA zU50D|>wa`u)SKJB!J$+jf)^rHU1ho|WrxDb^Ru)o>bXP@ z3IzaSQ7gI@k8QHe&+ir~R+2C6dmb+ae(Bm`Xj|lTa}%9k&FL`j122-ap^I38eTB0%;QCvy?s$);C>xGVC&p2xQ2B;NlBb=P*;S8QWF?U2qi*z{hd35b`cKc7K| zED2DOiatBS*W7^riY^duT1#i}Wnh5aeOuWm1GWpcC3!lM8`9Rt9!ocqfV9;&$35M< z`UqO?(a{m}>H-*Dr9oZ->{O)cl>1$S3lJC$eQNNDdVD@PsGw$6 z|M>cDy1&1V&v0r&h>H5h0+i?fL&OO(F}ip$XLD~wMwUElo`G_6$icA{`KWl@ifitx zPa72XRoaYy^K~=>_WZ|N>ZW~T(x#Ii8e0a}6>q6c?;R62ENz|^nw>9%9w;MfE6a01 zWlx^86H#>sqATGr+&%O;wWXYQBAtbV8X%NL&CS^%T(IU!(M97jQl&#H{ZA>NA z6d_qls3xf-3YkI(m9(h;<9y1vzrXA6a^2V5GnRUP-mlj==eZqZ0J^rL-q-l#dWLQ^ zSK34sW@a~i9R?OEc3JMMgpsiBPFmRvQLHG*n`QmaRyVY{d+(&}_uGmbF5vL~ecLGV^eK@LXQZp|-l$NiuG* z>f_LE-Nrdq$zEz;UYXooPcFO3rpjCJ5{Z5{B&C>X40lz3+ z<}l#uzyP!uW5x|04kSeqybBoS1Wc*b7v7JZF~iF;IAs|vbf4+%G)BThtbBQ^fuZrK z6bpWpYxADrt7t##khxQmsJ5x6?g@tpFCs zngT?%UOB4~Tma9^aL1jhJcG3gmZi-QoSR57R3CND|*} z;2#87zljf76s__#KDNmk@xk}dAz99+NX($smmCk>(wr#<&EnzgC8Z&=BYOWjXgVP_ zZGYf?<674DFY7q9Fp20-Q1Vw#EKGfh{$>^+wNIe7yK}9rdy$^=u;D9*K}86dUu({4 z<(>NQEMML+*8%8h$4V3(^=9ekjtn|aE-tDkUnTMrVUWH&uW$gglMNrTmX>&sZmghB zoM;LAu^V@iVgI2++f4cQ+40=26~#o5GB!=NK{98h-a0-mE)YisGk0rST$&zx_vRIO z<^8^B(aG1(R6b=?B5Z414bL{y;2-v5}L6}1Mg z=@*q*t;aMj&KCucQPv=Q?v}2SoF~YJ^XvX16L_zGHrePoSV^^;1f?Yr;;-IefLq?8J+b< zji*k{)f=zT>Aocls9gw6(?jlRY$`j3Nm7xC`*JaMtDUv zAdJf}j=wA5(pkMBI99miC)?nq19SJx45N94B}PU2w<$|JNpousk&`#FZ{~R#VNR9l z%%bAr`M){6fN_*D_dCLCx0(jO}_n)ElP=3fE{l>emwlRUfsGE-KsYZA3hwp z&~2tFMV@5sT$PPAl3Z(t^gW%FTB)i^*Buk3A{Gc@4k}2yi2^Saq6(bFr{pWFgvQ8` z{)q;`H)vA;{56L%llVSrnYfRKtuX!2NN{-Sk?xL`lbrYi5B)mwtN5cC~9|IO)zz~aF%yNXyNQWMp?*_X3bhqt7pOBr3?{L2PqZaXJe9$)~Xq&Hmv^I+y`pS%p<77^`xSa{i(Uq)iKu2d}LWr zIB_w>b&{#+lWSHd)?7mGBYYD*)4BbpAQe>>@1k0wH>PDIx^&SVk)lnK?1XFUB7Tkd=jxBxIDrQx9fO3} zo19QCJ3;8-PUN-`o%mH)LZDK}sR2s{YlK_#!ppiOF_}T0?qKB;8iy6!s4(f?$zepF zfx{da)F$C(f}u;w(VG`n4xG*2XW?VXl{YQ;_;Dg2da{v`0l*fZI)L2@=V!dU5+nm9!bC-oNIh`P!J1KB za2{JVe*;nwt_qH`BUD_TjJY9kqlG?n(7vH^G$qgH!7aU7GWH+qgBaSAaDtek2Fi*K@ zQ}Em4qZS36i8Ci9cWg-TM?9Ac$tpV|FbQot#yM z4r^s_4Seu)nrZO&Cm>eAo`v&tAznfBM?@|vLLvpPf!IIY8AB|Rkd+%c)V%pAqab{x zH5QcJ)^^b7?|GxgyZ@kkpJbTlPjC|E4wh9oQ$!NPEOc#rYyET9BM)0W;^;RAn#iJq zhih9stJ~83g<9|Y>5Vvi$G61~qDs&3deX8koeo1(d8HYC2{Ur%Wn^KZM+ss0pouk zr@`I_unveqVx2v&n1q`e{p#qG>9ZUU&sj~U?%b{{=&;IJfI%1pd8v_A%r6byE#D)) zq}kiuwXt_>dnI)As8G!g@E#`&A3XTcUmbQLN5`q!4HXqbA&Uggo2ia}5t_di=88{G z@Oj2dO!;PMZo5O6v-Usy&Nm%w9y2%2z;Y_Xn*=V1{h7PzRiwVeLtyTN0Guz~t>}~J zi;E5)dcO|?wwQnfF?1u_R-xfc+d&cAI_(^^OIus8e;({1k|(>H;cz!@SvbP7pf5pid869!k-Xso629qY^ni!VJPLUdrB~s9!od^$#T6(bLgT zN601A8@n$7iF}`TR#kNuVmK-3uvzHjLMsc4v4fr0tjV_>ni)V{OfU^f8*sTSOD;+} zZ&YxZ!Iu!wiPnX0r&7J=5H1*`E&FTK6-ycbD9ZjHtP&Zzm$4O(x3cPql0LC#v2!qc zG6tLfy-bBPD`ZZVJA@I&Qu}`x;YKiFzja%(B;r|h>(lLusVz(922KfuBz8AVp)jeGK zy)=jMvoQn2?!1$0R9m*(gk9kkbCq{FQ=397C`vj@r-3|KY6v6gW>dD?N!QGUi>%_F zIkz5wDe+=6f|JNO=&*VR{;{q;Ggn1}Lx{$tph$gd;WW zW@_rJV-`*Zz}o!5-rM~tMJ5%#KJTXji^`1(AZ+-YnE;kW55gh)?MADrqfNJLq7XKy zE}d!+q29T39HJYcdLpufYD5A{@Ylujf4Au%aD3Sa+8nOqK2tQlr78MRG)e_^H5 zWun0i`wtnCYS;W1mBfA0E~BEJiHUFm=1jupmR(1@tSq;=AD1*n17%8vB9mbq_brJ9 zOVgK)#7!XIS4LGW+mv_`5Rzu`6uCP9!49L+f$RJ9=>v?FADy)hfFH3!Z4@fIiY`^H zuySPc^iRKEZtm?rw$vhpCW<(%hh|rPHjF(!*^nKr_z|wx_e|X;yN~|%bm#z7Od3Hd zu}_-xJ+pLR-tXsUOU{Vwp+Tk`wgUY?dGT_YV@Uy3OEd_*wDG$OW~Qlr>MY@X$u#=y zT~H{~RIeo?D!c*3_#QA~?TqC9q3dh~6Me|o&Y3ObvdWZILy8CyVHi-91e$^1d7(F_ zK<|o#$K{CU`a4|TJmpOIvd#*W@CG35%G|i-VY8G8J9eO{V^TYO{{^!?`ZX%GMOKKjD7t+AO+W+0PvG}Kn z6t$c-G5r;NmsRs+EYNMyX#{r3g}yTr`xAD;ZNKUnaYGB+1y(O4D*$n2;`Mx#&!QgF z_AlC?O0qIj+vJb7#K_WzIJG(A5iI?$C>Zzb^AAmKfgBj=v!r3P_fEMgP>E%xV@dr*OcC~M8?@PKLi4!SHX z=3kzcGzCSXOPXk9b=Ej!$Qfm{^fHePC02>B;wfkGp5L|5mUTbgeiKjK#52;ic%6#I z;8H$XW=1oK9Fs-#3SSD1<7wnKY>(P3W=;M|QD9$aQah3P)#Hi-pdpE>Y+l9`Knsbf z;e-ioJWyM{9X3bRVjH~fgbgDh12mwRQOSOx^M96KTg>j=7huCnPp=&RQ_YXWVa_{h z&zFXO>%!cvBNybheBENafn|f%O08U9FK--z0S9>t35hZ#{Xq{x6p1iTw9es{-TUw+ zNYB9GXKRIO0Uf}PIPZ(=4sNUbeiiG=ZOF$vl4>V;1pMCAp1LN2r?Fl{)A&9Vprn7B zR>ls*pTgLrQ2)^^$Mdpr(yUViAY`uz$MipdkV!*Gb^yZRZLSa=4dxw)3n3VcnzwO_ zaZ8WMYJ7{{pTYHzQA%q_?tNEs8V9qI@-E#kH!tyLjw>C#lSW^=usM5t;BUS6vsw(> z@Xko|C#Wbk9lvYk6LK@_VSfG0>%bWaPjYf{dgf`r$yEqy`SY9Ebf3bcPbOp&1g0=l zV=(p5%~?6mXMC581g4}#xD`z!Z4NCM`aJ+!=L`5G@27N7WTI%z(g^P{@zjjL@eM17 zmL5KEpeU*IWa?X4#mhMeb0*|Vvqz8570Me9B9{{+e&(#chv>vhx47w3@!=-MEnRxX zG>AYcnk)T&tuRK+zzD3cZo{sN-Q$ekdWz3w=vd()$klyOYUORrBq9Sb876<6!TH0g zh;P}=7vZNsxbRPEYWH+rEgUfUngdl592iXi+sE+|24AwaveFnf?1=IFkY_K5eS8m= zIIC4)+2?nz&6NPS2vM7JOuO(6sW=!3!(Tf+4nia+>Cg~?Uq+U&|nG%ub0WS8htG?H^`O%IIVEDYBl5@WH zc`Gl^w@k}S&6%#nz}a^1=EEO_idALgnif3|7|tXLTGY0*K@x6c^>xLMYSc?%idyLS zlhyK@K0otEEURw9gj3@$?`Y`he!y{3Hx2Iv{PV{SWx9s)>0RG2a(3TCr#}aDZ9II< z$ByK-(0o`vm{ffR($Xn^T00b*KY!~SJZ@lJF-1# zDVPn&``^dd8s(h*4?Kxne4!F7fl6eDDS;!5WtT9VNXap zR#w?UwWP1NKO?rKmZIWkj=GWb(C#mLMOS!9+)dm_uY@fSY8#`Y#jL;PJgnT%uXWR^ z>Lok=Hwi@{IB%*>tk`s)eJm~C6CA7s$|Vg3s)>jdk5FFkLOKfq*`4W=9>?x$WoOJE z+o@uc*{+~aH_gH}#5xf}d;Ek}$Cu<75KhFHg4V>{RJ~KDyRxUo*H_QlEa5tOTzOrj z8!yL=snV*I#;m`h4-ty3H(hkK_&~sk4dP~NUg9mCAG?vrNEk#K4#sk$^Lemc+KMRV z?wexslH?1af%iS&6{GS$+1E_bH0OouH|V+?BG>`9_npr{1sxe>@WI^S!Sry;b(00> zysh>hW3o&VUG2XTCfTD=iYs}MiWNp_7dnA?-oi@=A*t?T0`^O_w?$EhY>q z3GC|%K9jab6r^~&$~L&RFvc^+xJtOHOHB6r^6WJ)3ksSklD*Nd{G)6>$*8s^Om~;T zt7_`UpEyU+wvn|6ql#DJl%h3a2GLM@Tb1NEX%lHuRNIp?&x^q&8iUeCch(F{3^zf` z`cmL_@qM9bORp#||MXyB>`1$HJhwbfDYkTaNUK0ywHF zu>5ZG-~qP)5Kgh7xBxx67FNu?vbv9)a^9dHJC}AR+a9pL{6~Qr@(#^NY%~H@t1jzI z{GCstI$1x98J`#@^#C|Tt87&wd7^#$PT6iy^Z-pHUK$VvUrAHpQr!wEkuB-AECq=t zgClXp?tS~(W)!>jrus@~nsXiA_OBkM>zhA5Hwa6LaI%h)XbMyAMR1{^g2X%K8F$n z07BrUS!6r}Rb=;Gy&7S_Rx_H^>@ID-!i$U6bX-WDu8~2JJ-O7v)SNyG^$9C~VW>uU zH<~#*J+Ju#8!8dwgrerKXfY1pT8&8Vs3bHnD&8mW*I3n*nZPo!iXtNyKe)q!UhjBM zp-LDCbwaw&QdBEX{(19&7LM^spXN*`6v~N?m+#Cw3(>O&QW(zv{eyeW*&VIW zU!c^LTprUxg5L<75eeb&;imCL_tRxl4+FX)`Kt=U)h(9N3K-qPU!ZGLO5WAie`;ti zt~~|-%>t~KFjp;YNt5FDOBTQWU)|IbyHKKQHNK-oFqm1*1c7Tc%(rJc{f)FjBx9Qf z;i++S{IbQZKdZweshBuJ)p&>%@JNeU%SWR3?z+5^uu2J0ahDL48t|eNIX$hHfBoHyKeFcCZPGpYkHxRr~Sl)!SLPi%|Aty(L`bdZ*JnsoKgCY!w z7md%HIa0Z}{X{iL&zEUxuLf|;;+i9mfxa?-*`9ZY*Hv4u&+}2%veT&Yl8?WCzcPO+ z`y~8XNMFbF?UmUD=2CIw1QI91**Innr5&Z#DHKG(93cSjefb?f{Koi239&X2B|Ke) zR+oF@K~ktnJ@+EGSZXxkYaF8q-7~7PL09|t>(>lEt;_f8Ln7AQ$?7n@rPm*n-kZP^ zBMRp4AaDTej<<5ztm9_(&modBM_nhz?2LHhWJ34&K{Nj^ohJ9k)g>RM8+GQUfG7KHgXe@KiQo* zfw=3a)A(pP0p4?6T`MPT*nukV|5;txz}mWC#IQ-}ws<&DEggCc!pI6QPibcl+Ej6h zrutP)qa|SuXR77foGRg5=VzUJsC#x{P5#N!DNEbS5XGjk;Tx+e_1L=e?YnQ@bCT6W zNW*AqxS%FKYVwpReh1<|D|T2mY?$|Y4B=i&P(QEonsoO-xqRPidS z^$U)bJvIJ&f1;gBAqt?O1$Wu7xQdmxkoXT>u0nAMVubEKscmW_rabRG&|q{+{ige@ z;{}(XE>qKQPh)$$2%WeFS<-Fp7>punfxPpQ$3F8xQc_-|ih*^?IND|^*{s}aHRKg0-(gpsE9@DuB}0LEmi3e>qZh%&Q~T!$;JB1GXmVqEMo zYlO=oB19$m3wEh97yjb!%%Gz6kzfz=O)I)F97vryz4HdzcS+bs0qS_u;U5W_l8Gy^ zv2jl^a&9Gj=GLp^L3CK?6J9}mz9}qhTY#>8vq?~7!o;Uh8q^he+6+c_{u2XM6&)AZ zC=DZ)wj01)n#S|9o<9Bb>E#5rtIfm_Z@G_GxQD+XXu3okdks=au(xu*#4Xf#%ZbDl=&-od!Rbx}6Jn@PHuNb#|LojsSBJR^!N@?88Pv8RK(NUQT_+Si!VS$wL3 z1W%nZWed|Uff6P)f%&3zdzes~foKP~!y^~0Qw)K8+ZAsrdS=P)tdGz1$n}`=F6!CX zOvJI=n7jaL3eJixk8oCx7$0SUtC6wcu)$o><0R4ayp}b$2^s9K%6>!c#@Go={~%zG zT_%{S#2Y5KD8C3zoK@MU-gtt(z9uS zHOjSlV;cTv0k=LD?kOr?q!8K8_lu24Tj3-B_P_xGz_*DNXhlCQt*x}Ym|D{EF7*Nq zRwNnY5iyMT3=9t2`w~0Hzl8~PMQmi3xdomvp=X82vY?wdWfvNJUoY>MWM095rW02w zkxdLYyYqjHOO&!kAYQ@M^mX52t3s=rLI0~I8x*oDekhAsymXw&p}52CtB4p_cHy9x zm#m0V=-_}|Md_Vr<8coVBT%#OIYM22)8>8q_93&FMy2!qOpX+kQeuZx?+<=uW(F4{I0p<#+yKJ3UN>9X!~EzFC6SSkFEL+$h+N;oW|nRerI@>ae@ke= z`v>E~X-Yn!3`XRmURszVTe=B;U*S9-*MvBpC2;c0nQnS|nJX6RqeEsFiz1ptR`nPe zM8UMQ0)jd67^y)1>yq9`X6XI}FIDWkG&Y7ee=sRWKQoH1h7WtH4aa5@@UqcPt?u2r z)xos>>N4seuKrPliyOf&i~S&?a4)_lXQ%6C_)O@CK79DFIE1O1-+N?Go6K-|9f=Mi zsf@q|j6G;-usS8Z%}sReKdghST3U45EU>{V4xc39!nQ4U_6ONP$hlLpZM;+F^FbWk zv#x5dU1E@J-#TL@TE9N}$T$>js|wQlCj{gNfvgI)M;y!(>GdRp8RV@a3B)MgD#DA!jaZO>;iM~H!7lRJl+fBn|R${_)noQC{oestPuio z#`AT)@JU!b1)xbDh5Y>}?)}mb*nnyz7S?908aVLqx1pyK>;Y}>+T1WE4IY$iqCl9u zt<}@LNGR#kF$GHu<+*eDF@@c0R7_l8rdL*m|wT=VMpAvzwU0E-hXw1vzOJ0&RN-61Ji60TRXKE}J@J`^Uk%VVy;$eHv1 zdKuXos#xd(`%&V(KP^N$ka2w_J7P!=yi)R*fJfONOv4k=g(!B*Ps>}-}q z@HGfG0i>X2E4yP^%0j03Qlp-+RE~tTQTg>UqiR8Pfg}i&9IJ3$0rk!5P{w-9uFTV* z4u1v*OSIm@4j1aUZ3vAe#{ojgljevcCJRco-}$25!SgZ+*{9Vybs}a#*|(Kfh|r(J z3Z2eFPr^;Fw|fhwe| ziV4b{f$YA|uDmR|guR_d_w}7re-no!>uyoi@5C56R{e#}JWj{*L1!`mC|Q0(H{%hz z%ZQ%8G%20PDPnFo1Hp8nPt(?@2X33p#zAg*UWLsm%dW?bKvx3DdRw9fCr@&En-*!M zG`w)U?ouq3#P50g(PXS`L1p~spG~lP#6Y^GFG1(`cJ+M z!gCS^w?f#(d~HX(2LZ*Dkn{_AZQkTEgTg=<+JI*ZnANMLSEuDGR*)38?X(&s1Je-D zU&?&htRsSobd@}QGiK?tvkfej`GoKw@bvASv*0u@cdp$Q_NFW&GR!#;W2$ zS~-ceT3nBNwx6Wb zQ_mvpX5HNHkKV{~cCo#~Jy<3fIGn3dc!){(u+DdkXWK(V2O>?tB0P*4O*}Z#Q$`XJ z(RI)u6&}N3LHbl_DyfHtsUF}5{%en(QAQmkk(JAXpRsX||HL`#0m$u_M!b_ygp-H3 zQK~*+KwLEnyb47G5S4k{>@SkT|Jg3NQTPJf<Mu@}0{4uC)}$K5os9YgVt8MvvdkFg6db5un>j zF8m)0W`{W#cjLVP>(X>9CX zveG6&FAPC1g7VL8j!l}u`e{?03CPr?>(9%}GjrgtHUQ~Hrewb+$9>sGCefEj-2ku|BnkLAR6)rH2sc%EDbi8Sb%(qI1_rl zM+vvo4R8}_qw7Zu5tRY-h-d{fw^p+t1Q-R~&WSy5$4p~?2*LcaVLlzE5^eFo2Ubmh zN+Z&pjplJK+gK+F4D_9dN`E`J2x8yLta{9yo80xUT?Bd>@%A(@-Ol zrvVD2Yoq-aux(@#TkdECnXkz(6}I-Umf7O68)}U4!0!FdZ`-=HE3hdh;(nrhV)`+( zk)^~m+5!3|yk^9l^iIma7j+8z2;(oi{{puex2#VvALH($M1OQ05@$1I={i0M5p)gz zNSHMcF;}9c(XT~|NTU*kDji~vKbXfw-zjIT$ARf&47P5%?b|U4S!E#O1%krrFLie1 z%eu520PdcicezYYnj{yVIgRYXyP>!+iptj-6u&%1+A=g{8|Fu=OFY{pV7{cT9qQIT z@rZ9|0>4_qG#d@^U)rDeViM3b_KikpPE-PQabfu3qTdOwFpb?9v*7ro(`B<{Mbp*j z=>C@rePDZqM35Cbk^6^8{thH3^whD^!l9N|p-vT*oW7_3w~{*s&2B3C#{<%6@zQw< zht*?T%TIJvejA20Ll(j@TsykAN}@UnS5TISl5t9+zY^)@e~Zl0Z85Qfa?pqb{PFxr z5ZBE9g9o=llYLbGtOkKkGXP1<&u`UFDvB=sdEuJkfPyBn^lSJI3f-DQ3fOWxTXoGAwWQ2Tq{0Y3eRgKmW1lcTX57v@LSr!4QyT3Rtpnmdi@tu#)pj?)1wU@mq#ePXaTOxD zvCBR-#AWSTaw(hQDTwkj4_$L5@AK=m68DH}mI3F<=)(jqfMO&jth-a3JchFrQ8i@e zn-4yTR&rn@Ei4j4cqCS%l@q7)b0^bVab~~fsgmfYJ-v-ugAs)dpAp+YM=DfT@G47)!eEnrQuO`LcN)v(jVxJ$ zyPnv1Yf8+zEcfL+-&kEFtK?F$vijM&K;iA8yOZ#>X;-~Kt2M=1pl*;&uLVnkY@(%i zA3kod8j%dxM^cE8dH*7w^o2({nlf;~e)n1*!=4OrPgWJgRyBjguA3SYi`K9vFR(f~ zB_9|M-q!NAwuYZ7Jre3)RNEKRKx?FH-F$xXX=-ogT5dtIsyqYpR50Aa(q*kBCx_#t zTC>aPaeDew8!iu|O>J;u-)W!BXU;i~{J)uR)au=qd%jMIQa8k||> zicC#92VQZ?+h*iCrdnS!27{E=Rvw{_NbKu-Xv)}EzJDO3%w)grrsTFR4!Es0O*?lW zjiv)}^Ke`Jh!ZE2c>ki@VtEmnuS9HT{lVJXDW#wUgU*pO(t{M+e@dfS?;utfUgW5! z^b?Xse=a8KS4WeeHgYAgxMJxaG`KK~H{OOOUJC`==IB3-gVM0*;Mf2>SyEV2xLtyf zt+;-=*$B&yZ-f?;Rru%PxIzO>6VM>Y0IO1$IuZi&?e&4_+##A>+b1H2u*|663-y^% zyTJCkls?2MX6TrVzjGc~Q22Z8em;=ct9zH}(r1Fo!q ztqd@DtwAT@LAUk5hEJb9V_Rrw=!w||uix>?$~J(^4l7<{MutJcw&SH9?^tufRP?-& zg9ro++Y-9q!CL}g5a$p}lKgJNup~R0dbkDjd%%-ezg>bt;V2n8w-Z=o67(yp$V==0 z=tlMK(v9kL8NU8!H+t!2n`N~_p^gLOW@`hN-2B5{84P`|Pd7bv?dZl+>7r{y$pHg; zac)SI3ue!URzpC?=LdfND`H-Z%TN*j&zP|&$V^L(1o~IvX`Vg%Ag7ES6E|3t=0Omh zS?(1Epnws7G{sP`@@a;9O-}d1^cPjYW#7oYZF**AAI((ali1C*3wCTCMYuVRX2EFo z8x>pe`lTRm=gpmG&&`QAgpOZF+l_5;@7V7%M z$WV4Ze!MowdJMu*zO+r6hEXgMQzo^&!P&rQofy3*MaSPRNQB@#lE+cA}& zCfrQ8<5P9j>eS`3r7o&T9Mo}!_f$sduc@n4BUn*87`G1SW(mMJ_Q%?dnf3yKb`k-q z2%I?sL?`y(b~*YrqV&d%P0x#Z3>!A*NoAMlhVrH=LD4y+=idjX60Zx&yet#322k^e zskADwK9DQkO-zKA`S)qwd5)VdUAlC`@)fq26YLlJx@Iwx%L~GQ&z);FZDQe6Z%rA5 zB*7#2ZLYd{_u0_ z;500AFWKK1YY;KPTNAbROXdomit|38+a;hWMPDP{x_%1mIU#jn#Lj@UqJo~At&h7y zw^;vcP3?Y(8aKQ|C3)8Q*|K&J&0uO$rni%Cw#R5Pe(5t7-?~*z>g#zuY^v0lS^71_vFi@XS%a60C5@7GcIFTr09EvVOE(+4)@ zJYClw=atP}0y#u!Z3Z_evH%FEE#9@KX=g9PR!BFbT=AeXuiFo7h0~IPMKG90fzLT# zKff)Eb(BX^plGnm({s8b=*4BpPrR+y@@w}ZdTd?HDe_aEfAT-we2V|8za&6Oe)!F4 z_5E4Pf>RSKe7^UrFfd3OkQCp1?!sXygRCUYkv>rD|F;*1J$c7-;#`SB+`(>*5>!Pz zSRW3dB|+^yCkkIJpdvDf250@dAbArG)!}+sL&kupa;{Vv+Nd*|-)0j%-d09-(xT^m z;PzpQ;4O~RR?C^aTYp0>lEUV8+tUO~4Fxlj{Wdg$qX9 zkaXPS*k3oFUp4?;`n|Y;wpw{j>(;9W!n2K*Y+eW*4&ZcAcvyaM&d^~G{1>9ov&^M6 zmBR&$lMqMWx%yqv$7sP9X{kihi_LQ)vakiW4U_=WB@I|Odzu(gG)&6~M$X|M9!BF1 z+f@Hdw>d&WU7fgt^1anR>-|FgdBlkFbn8WwWzMZ1l(vgbOuk#+lHIEQ7M^7nyWCY66oU~#d%|LnLFSS)#Q3->rj+fPhNF6Ewps?Y1$7YRA6n;F zh~86lB!>qsOmaInU`6@5aI}j+QVq^@^E0C^>h9gW8}iLBl{c! zvy#8q!^)TK9q1Z^5n%0O=xWBE&LMdF>Pn1e%CduLxK|`jd)#@Cdue?5Kz<#0hr_R+ z14IS0uDWT>SxcOUln5Y4ULf`|_1jm%rehbE5x{Dfqbl^QBHBY?F{88HeFAO^`;b!q zFQuhBC|ZjbYw)GeL|FCdUwdv-uFB?J$E6n@{};CV#`xm;ezpHOu^$)Wwi(&OpNy(3 zci-s|V6^Dqk*PzQ(PW7}WtXV`qRzWvY#`Wn3q;C8I_~opqKwVZ_Ui1w~tlL@0zhxr-MK~L9Fd-Hf2H}v(?F%m<1+N zW#Ej9=B#$b$jfOGzK&4#s$0g3n>VZ{n;AJp(#|0vYuvPHtNq_Tpqi#5y>EjKANswe zs>l3Wh%@+21A=;JYCbfpr$cuqhZ79O+}7T<<+hhomLb8HD1rT>42l+Tzl&>(&gyDB;JB1(24ZN4)wqWe0U2P1m8PKK>~Iey#^;qRTd z@7|3k2A+S<>Z0QNOK6+DctO#UAW%=9F+P52UAVk~s6QWxt!zNw zURO0q8L5wyL0Q#i=xtYmx|4=ng6`*%!rU53KU$Wm5T{)W6C{V&ZmaXTKfh8gW+$tih)5Kl{jo|EaG znVC7{>DnD0Z%t@4(;ZNk}{DgQ`v9c(X3(LfmDMG z=G6b1HFx(0J^knPJdkjq+t=d!y3rTjh88v5w#CykKa4*!^U3q)M?KAQ4iF>?9Em6d zE72ax{;8Ol=O2gH+|+Ps7Ff?|n%2D0yGa;DJJ0H9lOW&`WQy7<(OZ}!qf&vuH(*koP#3`9n50&f)ba_%kTM4_TI`}C3|c?hEdh!g9Yqs zwf}v4DkRrerr$Q4d<60~eZY)CU103#f-xelS(k;REh2eSiuAj5B)M&>`gY=q8Yhg? zNo8>OaAC&7$0N=otb6f3GlGp->tE)AT+&33XMRuqZ0JG5pShNY0VKBN(LF=&t;|lM zmBL`Q_U$T3nSe5(Kg!qclsJr%!>w{8jqU^+VZYugDU~b9kGfj4Uvwp?xsAGN_zmJr zqf(E!dVwq~Pu>Y1&rTl8OzAKK^&g@{VB}r+UkFB>?R?or6q* zB!7&LAKQ9jLD@({Z*g_h9?3Vv-n{<2PVvj5{Z4vP~rr%V+L{T{+Ardh0)Zc$Aj4ovB zH8{I|7#$CsP7_ir9D?Y)CHVxF>Xu{3{ODG=2Dp9%@QP<1WHo@6R@)eJpaywkz)#G+ zi^KE5!YWB99G8&25H&AOGfV#yk}h8TJ*PTx;&20<0_HJkP_x)RDpG);w5CSC1{%5P z6F<3G)9{(4?J=K?A4{Xu|JFMNs(gwT5fRXh2+w!Iyu&BoOvp>3m0I9gq*N4NoOhfd z!fwT=eWUf~)K#L|yN-D%Hc4J_Bt0h>AVHL!-EwY3l%9n7Ls5mCLUOA?zFU--J)}ZP zyZDrrJhP*J_5<*UlqXMm2lc07aq6A?NTin~o}wfQajve~zijzioL=9 zSxQQsh|Gt{NySq@Qvll3rS~s=!j}}bp~CrPa<@ZM8c;-WZ$GgCOdKEjK{at0jF+2*;K8=7WQw#{dwF>k-LmgX!Y!AY!wEP&oA42^J#>R=wx_}Z$ z_z6dc_9CaiS|uh3Ipx*Ta-tf?Ew5tSLj`q;*^-%WV*ikM?b_R zi&?;}3itpT6%J813SPWhZ?ON<@+hj$Py%ISU+M(X#i3%VeOM4Fi=S(nME7auZu5Q2+Yp1A3|yn;L=kbp zF!eu;;xk2~*yBR>&qi_SWv3rUE^%tN4nxDo>-_oF>3hRygV*JUx7oA|T=n5?Hpr9c z+zFNP9vYKgP$DWUn3wptj4h-9DXb->vDKbHNb_PytAsZcG;x^v(R#ceZ$5lDH1uA( zPMv-wV@TH4ovZ95avD%_=)$O{zC)}(5m?{E_P6t-Qs7IdBrxE}(9hAp3kb#$d-v9C z8{^AsD@F)BReuYXFRDsJ;ZK+Q-)7k^qVtvrX(Q8^l0+}TBifwcv1|-I(j*}G;5rm8 z)nTs3&WY`E2|XlK85?t&Ofhg;ULIXmpnJus0q?0P3uj2dvlAKfV#Hstnr(dSK3;oM zDxSiwO?GO>)Ol50TE|Zxo`|Wj-CE1c>dLRW}8)##m^XLNZa2U&d`$y@N zY8u{RX#ZedfGu_HcMCt3<{)j)k|o%TrIY5*54bVASQk^vN;Z~H_I7O+PV}*mp`5zw zd~%sjxXglrb|-Ie=>wi1D|5KU!`Iu1P3rQhQx|RXQ(?#+WY;?$du*CP#3XM`N~~0z z8Z@Ivk42yv)2?>jlATI&=0cgz^+1 z_YCVATH1_d;koSIboPnvr{S_jxyzV4M7~-oz(XmsB_(ZG)i^&Bs}Qe*-LOW*`5PXl zhDO&qvZ6aMZUUHr&!CwoIYjpM6ucrD2z$8EiyX~2Iyydb9(a^F%MI6*+dkwD>^9~n zx`LZf1Zv(tyj?p5A$&b0Z^F;+ukFh{wOV|Y#CcFz{|wjJ)+C}>rmT(!kIdaXRK+}S z$-|ulJ&K@!AjuQKpZ|dU#M-J@ctJ6*tLhCjA@PNZg}rzfLM~(!h?6d zk@MRNcV+75z${nfq|MGvJ9-OPH(-To6F5`wpbkKRQFHl&#Vti&y(n4?t~BlnEy&PK zV~91(&?3xf{Um0#pwbZgq!Nv9T%q%oA z+JfI;!aDiqpN;uyKsOG@wEcdb8wSo;yeP6m3cwdDev`?S!86`M%UMS9vR=j|jZ78ec6OXYh>|eW)!Pf7 zMcmDICdopBMHv^e))_;w>!`FTPE@ zlpWcpgqpjG*rTguXY>(k3QGkU8G_b?GaIoa>$FXoS`3 zhv?wj%NbBCHtfbjw zzfy0d-mR-EVt*TZrv9@-O)m$v9$2|h`FEu+&Kq{BAF0~3Xswd>qK?`f{$XLF$nr^O z+jOLD+xokBS9gfz5;^cWy94fwBLQS;$wISXKF62hH=`)T4a;$Lr*_Qy&fE)@x9Wak z8Z8FfG5S>$Ml%P~!o8qWXw9Jr^Y-lkq_$S%u zJTHWtTVN5jABV4bL-!M!P7m<537JZ*v5Ta|rj!7zXC!*aHxP-|WrNhF@bv}?&mK31pj%#Sq-Cb==$-4z;Y~sz+UnpNU;c9GBjIZQPUAZYbF1Dcs3&fc7fcuX z)2_wDF#6uIZ8-1*Bb=94w~CR(KHk8*DUmnY?PHQs&rSbK1-NB7IN==|u5QB(*FWPp z0K_@Gs1Lcc+RgR;;zpAl0qxkNCG!ak%tbO7#wD01(e3FTJ9gB=>M%VLibCxXBWy}d z@tpH(hA$kJ%rtg2e-dlSe6$UQh@phFT_!Z3Zb5|rL7SZNlTJoQmu=gJ{6ipCVtm}^ zCid66H)s^X;#w{&E+Ahbz~lqt0yxrHm5fcKB(&MX|D9{(xPliz!VZEDGZG0uveRS! zLP$1=B?)5Fmpsrdh#rl6VXN69g~)1Y3BP|wz^R@Z`NI&467!Q8GKdhaJJ!cHY$v|t zzsbvLAE3VMxTi$~gdFc}{=D4?e6&K(B1#!n+J`F@HNZ(gBl=Nb#fzp|4ke=*Yap8W zDNzIZ`mttHrq-eyd#AmxDsQ^>{Gm^)DsuE~dW=fjrMF;BvJ}6j20vFkIJSP)yjJz$ zkr#PpT<1SO3^`$YOKU_^m6UnAznUyrzCV7C+rl8tMR%6ny>YNXy@t&ip4DpiJ!jja z+@Ag79-5dg+@M@B(k0d5ue#y7uaXb1_D-wYuIE4cLpN9FcEtehtZOYi)RlEH~78&5RyWgQd|0|dvKET#(_n1$`GmT~x(6OqU z?VdRc7w!Y$WW<>W!g(&uMo?ZxrYAN~kGN*7atCSn{mxiKcH0dp;^~hc-^n<(t}kdL z1Z33bqQt`qk?Ypp%kT4x;%qvwzyux9;rFr8$gLUM+xXb5FsdGHtJ7he;j#-Tb>)4x zv3sYhZx&*5wdVyyc3U1jT!?fMs3DH8bH2R0g z-oYk8!Q@{!ld{=#%D>-$V#J%3W$>I?g^_K%pKYqYq%e~Crj64lXdub8Y-;NkS}*n_ zO(#2O9_X%caM*EoF*!su1sx4};UzBp>bkZF#sqglxUt1M=4r9h?)=c(tx_I_nQFn1 z$5Rj&ep`!T5x~Cjuwj?HY(I~97gCYX=@ zvFy=A39i2jU6$#wtK&4(a&{dFOnSk-nTj$%VZxEU?d)HzMgYU7e8;=T)UC;>k1GFw zD9HT7j@ekqA`YiNszU4@mO~j2=|5Io_qmrKs10is!XsbO0`#( zd1pRf8dI=GiO{5UhA!Zammi6$dt+m-2h6PV?axD>oM+vNH*T_USPu?I#Ff}F><{-E z;ve#}o+gd&KjOz$S1-Zx)t7AeRCufYGi?s(~ymzj2lKCOPV$K1Vvd&6es z=d{{Y=I9?bGwV8fWa}wyUi{l^+mV3}l|HtZw{p$eXJ0Q*sZ-x&)I6h3uTPFYF>#x= zXVa6%Z2mlSZINpf-?0Vx0+N*eFt1umCl-Mm9VkY?-UFVE5*7}v(JRCz2Z#FUksb%p zrP8KN5AEk!^DeLK1~~eOcW_<^T14)5|I~)fQgH}~WbPXWeJ?_fd8{jY067Z-(&n(I zc{0cYN9Hc+2-7nsYQi2%Eq>ZC6Aa$K$tmJP$sE`!u$6@y%X$o(+AYuwNwX|7q$_)B zLj!B~aqx5y{s(cX! z`wLuVs7k`gxaE^igB=neci#s6(3VI0G1UUeL92HDH>^U!7DgxR2Mh+`^&CZ zsUZqh%)a1>mn%Egjqw$*kQ6`)LbwYlb|kC~EH3|6Altv=LT!D<890QpUr~DPIdhIY zU&6G){x)@5ocfyGC+U3u1GKgi|5C7y6T~rQ4u`n5DG9oixL>mgbOYRqKFLF0Z!LlJ z7In)G--Q}3#M-yRB)T;6;lqb|WtmK|gz^lA{a9Myb(~E)J2->D8?tu8a6q8M+l~{$ z?J%AzE?_C0IjEmMB#oB#3S0M}%^u4p0>Fye6Dvu$1fa^U@`{-r%Vbu0iA;AAlxy%u zcv1nsLDK)Dyc4y+B~h2+=HfDk6uSQ98$>031%a4D7pT8{k4h`$(WAOGSs*}Tm&#qZ z75GKWh>)N3tXm4QY761~fS~uG4u>|W9B9-1Y}@2EiE&lsBHl`}>G?C^N^fL_x$)%M zVq4r$B_Sn>NdK-SL$EVrdZ%y^QSZX@k_$w%c*et1W_jM=8Q{@bL2`K)&Qlx`S zN3RgeMEPafN!+tSjD@Zk0-+OE9|}Zzgu7o|!{>&G`-+#Q*XnT>8BPVobgQn&3D0YW zIpwxUB`@k-b#JDuJc*YQSCL270WA4)SZ&mYDZFV_V6^;v#w-wp4NgZvr(Kl zuH&#sBV-R{>j@Tjb^Gbp=-Gc<)V!!5=R+;i& zB?rH#h;a`=5m$8G11WZVbq2F$9Z(R$nC=E`4lu$d_kQ%0bE%{>H^%<=YP?q`Qfy`I z0{NvTk^lu6m7f@0ZS?u;*NJS)48CbjaU{y7{#lI_O~dhQQj}eOrKY^fUq3!?22WEb zA^TZz#z*vZxpgGW$Fd=nBITN5-sy+ALzCtR$-QEQ?Ig%0OL&%g7`2b+h zwZHou!!3gaK?3uSIBcJk^pa-*J`}|s(L0EO8X)Gi=fQ{_Jy@-6+xYd{Jm$+53mJ`q zXf?#AxsBj96Hg=&&DMqEX3tc7X|7TOX_ofBQ4n&G#S+HZ_9qt+K z0l{QR-R{^UU_BiiA3>CUYQ~5d+5X3a0(QcFPw2(%g7n(MHDFIgL2_m75qx7$kyKPt z6_sd>q`g1>xsIv&+ii^|%<|Eh8mjN(FKfX$L8(o@19fW${VWfB)>w$}X1zgdJ^t1m~|!wp(> zF-;l?Ovn;pj7Pa{YDxiS@NfhvHI?w6-SAz6hjzs! ziW8Q=u8X#o<>!zhy(FbUdS zzD6RE{E$>fh-@ra=4;kYlDV;AJvbYMyuiai?mMTvUwGHvcb3c!LDpV9y-u&Wz@A;Y zXVfiQCT`>4RbmeoRqA>pJkLCQ5v(N-xrgBGqFqNKTzWQ6`31r1EvQ+`$Ri7ONU9fI zH*(S|ZO2@g-ItqqH&Wy;J$s5)1?X4ZDxz##JlT|@I0Y3d zs{2}7i1&hj>vTnw0vpYo7_AG8%TTNkOY%1iN}j$mU_Nb+U2{z(iKXPI{c(T2t;fAE zyokB5!ncXV3;sw-uGBx8-=FnwkC2O8^>DbISyRz(p8(eM-mLm~=bf?W+}X1z4x82# zKN$JzC8hYh-D`9x^b_kGYU9Mcn&noNhYnM%D3EY{y#vF3y61Gd{Q2vDp(Kv=J4Cj+ zJ0tsBh`rHjHa}lD|h7x#sQRlBYjNsZ&eIBBvwrufkD=6_l(bhaT{P`h1!4Q5r(D@403{V(o8bq2!MEWJ}D-9JL{o6qfh2kiZV@| zYS(Pac#V?cV%9rPHpz88p4(JeIi3$re3o!Za!Idr%0fmeyQjcm$J56$v}N0_kZ3dj zSY3Q@qo$u&!vSB3RM4#d@ZnFo9iPSb0?s1~5v_tE$z$M%x(^z(o0%Uj(LT$e1@SAz&dk zwfF0C($gElXP%i^wkuR)jwRSWWCbi**61j{eY@;ynahv0MPK>p>({t%-!6yfjv57n zpuPFLM}R-HDhe|!#*0f$F`Izfw8Wv?wY{GE+al*$!1M(8=j2Cjx}#ZKAnD|tmgxF3 z5$@!S-_PrD{0Pc>@v06VzSaaE+*YCU;_Rp?L#mt*Ovti8n*JkIS*u9AW40#bv-DO; z9RrO8(T>0B&Iv+!2{J6BVU`?e5J1w0uKZEy`s?f0GQ@G`XUYQXB)#4meyiwdzwvH< z$Q#6a__-|qIA-p#lLyqmn}MaQkCxsWNL=#b=r%ib3d8??#J=Szx^e_@@M9^>t5 zsOp0uWouPwBx;Ak@#E|MANkkLxbG#_aakWj6%dV)`A-$tvN4Vq2DsD~LxYIk{qNQD zoG@$csKq&3_j%l#q4mjW!rc9h_NXZx>Z&yiEl=OE32q{! z>GAv)(@y^O;i6_p&)dhB)0}o{ zNf2+g0A?at_p$qGRNW~;<$0xrc027+qqKquHEBF$f`-DF!|T^rS-I`O?b)-(j#U!g zi!1(>T&Rq|K>hFcs<&%*L&CL5g(4DGc6>Q}AK?%=SK@$|~@$zwYa@ z-vM!Meq@`QHGJV`CTcTTRVnzksOc>)EGjR`!tn=vf^2AUU9XnFk_rbwTH4amC&>H- z3OiTvP~y}HB*Zzn4c&fP zbu`BQ4-f+i(fUcp0(XZFJG7U28N_$f2+zGxfAO{FpNeu--ozWdn0lz4`z(`Rrj4HW zTgXhM&(~K>{^B&rJ9cE{vrZ9ZXHLocSwofne{8)6SkL?W$8TgEGn>p*_HM{LNJO@i zQItw(XelbnRtS|@5oH__Dn-&TDk%s5yNkI+k&z_mGataVI%DjPRGuBWtYuHnNPiwn}*K z+?SqF8^t{)w|ugw{j|DxKGkqPy#&5m(h_X%{otT5R}Y;(_N~sq$`$P1QBpq~2ihYTONl;HIG=6>&~Mbp3p=0{)Unjp`(0sb-=9-2;B8y)eD&z_c+1#1784RSXn zr$5T0^ENzoY~VYjcc(9wCr|{04KZ0fJ`9mv`rRMS>6A2S!n>y$WiQbz`w*RRA*Z4d zd4PZHbATksp&A&Db3OsELiTHI(1k@VJoWJB5@-e<8X_BSTwHQ)^dFkiC3Lo2A5=fq z5iBj;$3!8}#<9D_`p1&$Rx8Bu4Rl0{vdcnPjV;P^?ykQ~LnamX;sTD1cwF~jM7;$G?lH-F zdZ%xJ`{KQfk|-feWb7`fb5xaDrXA#nw=X_!1InkbBX)aRNdiscub&>Q)=7KEyHrL$ z!b+8HJ=jJrUzv}i=k!@(YpdttqP2YJ)80GSjR2`Rj;0fu@ZPiaypKp;h>YYPn$PbK zN_UYb(QV|&PS^|pU}!*{zbDC(NcnP)Qx2=MW0UVnP1{#Lzt>7w;#Jw!Uh7fH?E&-3 zCv{a#$vZs5%sIGe!H(Zt#m~L-me;8IyQ-?=U%o$m<@*x*9}l#@32UFfFY_-G&p};> z)*X?eTC;CAjdR=fIV4(icyX|oT7^fKv}Yap|N0pP>yxYo8B4?m(cm#zTw_GWZ(yP4 z7T;5g=Rz(?WDT9d^w2Zx&Ym2v({go1nC{UG zD4oz^>fyXa3xTIyi$)fI5)iboDbpW1?OH7|P1rrgI-Q=}s*7A;l0-wz2f;ZU1&)=2 z7Ie=m&;zWKr&ATkloI8T;LNX7YS=u6V z8&mydDSkc;$`igaAxXYJTx%WmY8Kvo7$2Wp0YLKuw7o7T z2UqsQjXi!Z=O;~BVpg^!C|$cIBEf+Jm1!X^^_N9`e5G!r4?Jkluv24|FAgjJ8WGSs z({D*&L1}h(Ri|0+!xCP+zY%9yedIz%o%lb~pytxiotAV6ykgOBQ#aMn^Y;fY9i8z? z>9ot#J)BM$1v&P5sejPy?MyizRoNxnV>J>jaw6A$0Czx*7Hai=5v0BiAb$#`Z*kbc1e| zq$w7E*XAtk$(o7~p8J@{hQB6$TZRB@8ayDqk?}`<5-rTM@bXUM^al55mk)Bb*e^80 zX4*mJfrB&Xx>Ik^$iu&hifos6l`2L-%$qN=w0zDi8ntOS)r)b7g8fo{EY@7MDX=tdquPxv2uNxIK4@vLb8<8 zF*`SRJI8mfY`l!#6JlbFv4dRjQ+xLAEohnUNaQmwLo)3YJ}3-uBT>^UX@sp<$Y6-< z2wnGFc5r>1;_Q2U`qAUZ6%drxE**3oY-6N7dT3U37hq4}>zoV}0!hlm)oRyGEnE$$ zr&q+%E?LKnIWY_MYIFp9P$3XDUsruLVIYxKvN;ct`60`}G81V_&{~TPscC$!5|ScH z@~0N?g^=P9?ap!KivmZ0^|=;mnd}{Z=X6-uSl5xKr}g+uaIZO$x*JniU22Z230x^K z+*@xj+T$tW0UA4P+TCHR`y2Q>pi5;NjD?xm^P?$!=bfvM!@%U(=qY(L+)9_6 zeM46#M{Wdesa;tZ9tAQNt>y-3m_w-7jEahiuAc$vJm z_UGc_mbxYDLNP7;kBPmNOGjauJtJ9GEXkZ#JDPbOz$&VNrt$p ztFg;AZR){y&Ygg%Nb$A(NQRmI)#R3+noe2xaLMY*Hei+s*J_Q@Ia0^Xd;93>KlN(b z%F_N-wN*J&Q$+7jeUtN=9qV%@u{^*V)&3@5qglmy6jfS4g7qd((fn-gmbog8VFqQ|^h_ENCU!j7gQdxm4Q;$*WxS zu-Or=h?akpdi^^8*;kj_SU}LE%8&MKRaBz0(4Z&-2U|UzVus^Uc5EmhMN*&yQQ>pk zL{MJxa{Ck*Y4P__Ep1~^#^4$VnA{fSxxk^?+Uz7E=pU0!xEz>wN! z1M7*&XE-cf51%>JZJhXBC8&*rQ8QF*FW057E67M&vp8f_S_v>p>J6?$T;UU7h@v6~ z+wYm^#5*Ts8Nml44T4#R5xV4%_k`OTprRY|1~B%V3l4`P=DH(J{xF9EfirT+l%Fc8 zF;$uUQxuO1cS{|0cMm#mxmFZfy7?EEH$b10Z*#nbABh1Z4N2oyf{^Tv$L$&@nCU@Z zp!|rP$7vQ{+Qz`mfVhqXTg1CVvIxc3#m^?lGyN{0DgH$sH%1>TV}6mR1?`{DE!xPL z)y>jM-TQ^eCEv}PHeFNd>ZA%GCdwo7CO>v+NZ1G^leBnsu#@PwXOepDzNwzOc!?(# zq20df22nOk8V?_q^n}ldE5X)zx85E4G(+vFwVofyH-Kzls-vWSJpE;a>_y=u{dh8Z z%2DP?vA8_~HhJ;-wR#U_zxRm1qGI0E!1)u2o_E1}RVQ<<$0!f08p(u_K|8)4UQ0Rs zUy(i`+HtknZeeaNt54%g3VuKOmYBCCE7ZlM#3ozM-Oa5lS(O-3fVKjI)(lQ%kVU~> z+A3+}YHG>%?m7RFZu*6_8$362z>tji?fft=m_Z*$Y>n)A6)XX+esfx5?k7N7_9-V6|(-_HMb^jVIL_adtp*$4)29 z;~SizBPPyTL1=`B-!dv>y-d?lKh_Z0x~_5hV!x`6Z@_>7HN>Shitu1CS24mJFwcyD zRRJB5K0%@3LQH}NIWE~UgP?strH+Cl*`)`m8(cxo`8hl3XDgb)buIpT&t9LuQgrg9_0xv1XJ(Moc5R%X;~PM}?kOk}`0R4QuLOMFs7Pd>E( zt>9NbK67=f+ivQA&PrR`Mig^vE&E04fp@CBgY;}$?B=Hxaa@Jm-s`sdP@Gof@JDRq z>G@#2astHYt=^qyWADCo>ne9b`6GjVhZ7SQd}~K$U}J6r&+tN+<#ka}gNSuVEZ&D~ zMs|qw>-XtEbaS5>ER0Y=4I~SkrnOhi{)T)aHGg2uLqsjHia_*Y71HDJTzq&(qSK`* zqxA-#oe-pRDSR1eUiE-GO+fI+f5(y_+ur-^nXQwxolGePFXHk(%zCfh zw{=5kl`6J-Glz3d!~+y5^nz$nHYhR^Him>e>vObv#kK1A!HXh8-BP_P*3Z1VBhbui za|gz|c;n6=8=LCve$%%4DC;#zaWm@qTSc87d9F)S&BpD%EV))D6PI`H8l^)n_cuJ4 z-lE^~9%t23YtQZ4euLYcjJNJs$;8X^3*oY5`Xhnzd2aYO&rWEYIB1I?)Ai#O5oA&> zj1y=ypq%kL`B6=lF^RC*%TG(6jIgWdeBg=HnXw>`2}ueD z@6xjWZi5JlcheTLR;RT~mz*|yrj~0qp#DVFgSa!uFC@azI=tlDHwh(YlSi9H7clY9&QrfO2{)qK8$;b49@fVCcvTXp@)$ zjrxN596w=z))bd2COd!7pg3=gXR;zjB#kM&5lZUrD>%f|kH0ahj1MO$R@;WgF{*6k|kU|9+@csUb29G6F`Q z$luHZJlDb$B1Oy%hI)h3Nj|&g(?SmTQw#2HQLzhfpcd2-1kg1G1%P(AMxYg2(+E7- zIe%(JK-G3K5!cmBOZ|sMF=;nIzQ;H1wfwU7(EiHk%v4m|r8 ztyiH9r3Z?3K>OFF8Ni9%NE}{S-~bgXm1lUPhs9%v7*hN&hI)f@-#F!IG=MC z==Xp5>C<*VH%Y!6K77^o=82tAnqd0>Bln6g`j*YUEQW~KQJm*X`PCa{G&6=D(9D|> zQnR$)F|%_M&-|*T%4D?bRl)Xihjik1Y^iEAS&q1H_SEXi6}lH(v8+pOyrWixywpAHctM6%ei)*lZeWYw zek8#-P{vGNt|}duWU5`^wECWD2A(iQbJE!7zx(&qRQk_3yC!{1egmEqu3W*JWe*@z z%>Y=Y6;F!Om+tK|_eyDf_wEH5A)9k-V||HvC-&~;uAnw#?eKT;o@M)wV9}ka6o)Ud$<7 zfe~$np2>cX_9UdJAFs6*Podl`9v?C?vQB@Cf3QRoBl*sB6=>aV1oN5oQcaOnI5Il* z4>kq#6h^RIHz4&${EL2scl(yCQ4kXa?y7SJ7gl`YjmJG4q2vnq|yWi(-Hx zvOVWh{>3@`|57ktcv4uh@`rxT@!&Urco!=}yhrFUkrY*t&%$ml z+uqBdIaOK_k_DHN`_6+6nB;Tdj2r_Bf$}Q5e)Vz4@4EG$G55|YYb6A{X{$rZ6#qWg z?@Unh&NnJPCswa6ROO&-ZSwc;xheJ1{fVoBpKtBlxyjh(9a)lPVw7^BUB|qz&og@O zYes$>1OZ^O}xr$b5NYuSkSwWfb@$kgGu zb7UthmJxiTQgQx&e517jMsSS8@~s$@I(I&wb?fF$v5+AwTl@CSWNH7Z^A%8o+m!%{ z(=o{;BWON;U69q$58ix`d?0iQaWz??`Y%)qf9TMeGa1s_=@}YsOP-~fEgqFTiX8@f z9Gk||R0A7=Sh(^9-k%1Ikr2bBHFz;mWp+MI}@t0LG4&nGy@=DLEpG7Z(7gpO__5Qw|Z&Uq|GGO}?$~m$aMq*gko~{sTk3y_dN5tikzcI901~b#m$K&zdyp3rjK-aXJ@uq8DD3 z?SoKokFAkaUFQ*b+bp9abBv@kvWF@*P|ZLLzv6IXozo<y#9agNP-|wU zZBXuqG->5t4iU@^BZg2l*1}hVZ|8{Igd#2*NG_wCzxMz@$H zB22m_KehS_c<1#wHodR6$#i^&5-dTZ8J_|xD!1Ccef#2?>h~jD#9o{=UG(fFBnO=~ zWp=Gv!m7M?luWYO&6@-$r*|aRf#%?*HF}Kx!>4v;*8(IkF=M~}^OERZ$(U!zI`H$m zrt)XoN*6b`xKk2(RF%K;>LS8`X<{C|e(joU1XmoUlkD6={(mYTLGcKWZ)Y#lIPU`s zdQQ?IiBeJD$=#M@lY>E3gNdI5K&_l2JG+lIg=?E2pn*8#r*?F;@3Q`K4Vv1j&aL>{ zHxWTKZHg3KLgY}lJWY66T~aiEE%yzYmNVFg2#`xZKCx9JbbpM-e+?r?KDp&zU8&yu zvvT}^-*BDEoK~NFVyCw#w$Kn|{*q2RYiKrA$sBI0cWH-;(MAu|-0bw%D#?6?=2Qdu z#BN(fM|iSBQ381ApK9HUOXPjXlSr#-jLt!#MM|<}6Ryx^-=`9ZOtVUFpR=h*g7!hc zk#_YOkmCi%Dh-%Su@TzrbQ75rKcn9-`zNiy+?oM?USk%plf)fzN%91$Lr3s`hAX>X zveZ(2(jLH;nKBD$iM$sQ*X{{fbZp`vX9mv6SyG~T0JrT{XR?#NK9mobN`n?fOxO4d%6^V^m9+TtJ-^}_Gco@;k-hm360@%bQ z^fy@dMD0<)--k}0zN3vno+o=_vR?5>X%)cKb1v39`-)f6BW3A4`#C^MVY1TX8eU0t& zQ57r1Gp8aet^nXTUXeCsKfBY{iY=HTH7z_YrcqoIpr;jmt?AE$O}hs!L^Mj5;-5ui zngoKDTc~yyxf+6oyAse$Rn`Lf`<8k3=FVpFK5Xl`Jb&<1D)uY?MB65yc9T3jH zS$>hc58kW5z%3MbWbvEh3)1kMCnR*N#^>I>bAR#Z-0TL88UZ83R3dvmc=X)wHq21X zV4dgv4Jso7KBRzYJNL^?Y25qj0WbTWzsH-L8}luzPY-axjKj$fluUm_J2q>Pe?Go} zAl$3bPTbwGzuDWPH*Pvi?NhvB#f8ZYt!5x0lFm#KaJA5r^}^`k4hs!UWP;Y*>X72=30YED zkwKQJE+*3dVzu}krc(!AS!L?v$qgpHoWt zk5*v#zps)-X^cI3E*sFAVwsdgtu$?DP^c1C`>lstR0mjND zKdop2HBf}>k9$0|97Kb9dC`J;dU%QD-XdFdQ3;bCyxHc-x9Z^K8P9G+yHhgm_+K>* zQTIS)afJ}skL^eBdr3Z(LjP;KOtJ}XWSCy<9ypb8%i-m<2052LhY*1+8ArsjPX2q4 zlMAl30sCJC)hHUisY8yqrb%&;tNQbox9^+O=+dv>vVNh;4OsHr_49{de4WjbqitO1 zq^|N%FVqe)Ei2*NLkSaK>as5;T5m9R8$~pb)E}~(4S|x3+!BNP_LeJ5A&H;Xx!`=xsCaoirJM9@nZt`MuWV~^ zak`SnzdKbv`KXWXG%fDUm^5iL`Ct{{JeBzI*Enp zMA>zwo`uE5T7G+|)gc`n9IbGa?`2rxVqi+uLqtS2LVmS~_d4r9HW(N-TH$9Gy&^onw- zeS%Z*LGjMR0X z>$K!=nw3UC!6MtPK0kzhYpk*DFg{b2PON^R$`x$*CEbpB9@^Y{f9y7zL|7)LxosM|@eo9oCbn3g$W zZH!;;oR)7s`Rn{b%YmlxIRm;vqkfp+@!;S-J@&sB>56P=PS<&ss(oBSbP4E+T>f^( z+P4c35?nuh;=R*V0V%c zCwlA)4)zPLZfY!>4|8nmezpw=9K86>b16z`rL9$)gJ-rvyBh0A-wvhAD(G$lTBRQ) znkwhQ`SWcES|TF_l)&zuN?(f4Xn!@U6iWp3xn#P688WD2pRUWX5H0sRxG8v1dMcVyhy{H9!Cmb=I1xtAm6U;DR3U);rK-g%ak{B_;JR7DVBe zGRItO#C$F~4FVN+0dq)7F3N%U6u#dV%1nBE`CSDM#KI1v@ZHY!HKmq{+{Cp&s|LXZ z)22;ZLrml6XSLH$0&C|y%*fc0q3hIR=FFM$2~&U&{^=ouXsgj64s1rvcA8%=rf zE}%i9b`ha@QeXDA`Q)QCs^H018a2h&1SP29>gG@Hj+M)iHtrhfh|-%2&zc|EHLTU~ zlCXbQogA}pm(IDwW&O@ie@0R-CUYOi$Q*0#BZ-W~U~ogDMRX4oa6$t?xOTBw?(pt@ zPd3V_vM$Q(;%w>iSPby9@KDR&e=>OXwW!;LhshGFj?sZ`Rk;E{# zL%`n8AKEX*wLmzLP59dqcQ_FUS9=6J%|}%hw?Mx7)B@!p#N58U*`Cbdi!8`_A*N$? zcPA~3iV~yOFzB(8l9D7=!x-1{&s;T_rBEtGC3`z;R5M_Abqesm>7xm~>02dvhmu<( z**sG0D6WQM#$2%-uiktNxmGL)$U12HtaLc-2!gy*Y{a9%i|)L0!dYfVMf6AS9zC94 z>YLe!_LGX1S*z26Fr&+|=mZvKIJJ~{K!KxA; z&h&(uDp2Mpz_03n`@3HNx{<-rub}hx%0=Xvoa*eYah?t=~Y%p+wZ$`!B5Y%ksAd4b&V@MYrRh0X$_#Mv`8k?>>l-KKY zQ{8)*97(i&gx45Hu(^nMfEHxj`0IzQO-dRCSfEe`_m0y@sGTFYYbyet;q*98>t4g%sq$wh>^c@%CYF2bQiqH(r3SKgQ(NadZ7rap@ zrdlIO7}Vak3{MbK$D$HJr3Va{Htl5jq@yzy(qKy@h=L`;ORPQ6gV1d$GP0aKYsB#3 zvMEjxTc@8PbS^dDE{{2Gg8#LG_}VI@ac~WqtvGmuYKvIX(#mQ}gM3+Eju2;2;Ryl~ zn^JX;?Ke0r^>8#7Lj`3K<&lPGm*s00B;Cn^cuWNJx-ZN|#T7WJ=7IwNikMjD8 zRhigI-U~@KFF}V3U!4<4I~*Ojd%`aMcYM*-w?(tOCQh9?p0(W>Z)#8j!hkf_@q`JuRF|QwAHqT!TVnN*L zOP48C%~#+*6ja`g@fbG{R(-!|fgah99s%HCVd+?S4Fo7vu8sH#&1ZVt+1$w@uWR1? zIo)eRgWp}>KQ7yY98TDD_-N68LD&>MJ_4b=Rqdxe`iOr3?QH z2)tH{i?OsghiI~Bf?^*Lgdj*a*@pzp&8J{-^@!`4A>1y8+@Y=m2WBqR{_me&yBUNO z<|xTodq)DI9`~DqUkcONjkArszvoC-{+wU1^I@xsENS2+#))bfDH0*Em}-yK2yGWY z5C#Y8F+PD5_GvY#oV(5n70iZH%I3gxXsMMnLUUfx#mr`Qw;lN-nCpG`szimc@ zf(eOvkxp$zu$qCOk}{ltrV+vgyXRz4vyH|7e2Q%;U@dnoSw{UdLjWsF8zNl6K)Haf zz-XC8m6I$eBq&R<_bS>OAngVd$Ds$3^JK@g)#$(!`Sw()4B*m5UON=xG#tSTgFXZ# zD99t25~ll0ojx6cnH3OK;a~v%kuF=b7d`e{FsBmm^S7yKNQAbgre*2y?oJZLEfb)k zjPa+-P_*$x_G8J)VoqGtz%!XUCm^qqGz|0srB_mr;8L<9fu`Er=|074HQ3_Dy?e7y zS*nYbT2kD;UEl>AhNXbUaNfBk)sdxS&pyp_K|8`)TEQY@sT`}bqXLP%2Er?|_f2g; zQ=c>6YXuHQ#tXNOf!?`6ytl}|h-bGfox@*7`I|qoWp&r5!qI`0)}|~sR3u~bLL?cR zlbZNBw;;!o&z1buD(~+fH4x=eAz6^Jxob~#OJ_3o*^zd8Lkyq4I6ECzud1qw@5Hqo zeLO!;bBq{T_2ck;IBom1^>^PlrN`Z|8>cp3_2pik@`TW6k|XIt1|YP|4;Fhgl#nfwdz{@T#-@f`cJqY8tp}uQ!jJBRes*OAn5hrZ42Rhr7Gd zvBNp+w5BYre+fTK0ervz3J+>DY=oRoAoMi;_UJL<{HiHwOjTok~^YwG_*~FCQUXG2=}3K z+L`Psd4jtiD*BgfqKUP!MS!>!IU3T-G1CU6yu@Qj*?lYq3$ym2Ihkyh<^6uW(fK)$ z5cl0^)-$fL)O9b5i*FaDf|7$2m2GZW9FhN!N-xn=8dK+H=d|k#aH2N zGm^*x8X(;{ySXlXUCTOark+czU+hr1nmN7d%WD1q@i8sl^SofYOJ?qchq^x%s=9n{ zr9;Ou*4#MUv!%WpQEZ#<*=x)G80Ug1LY_ZxFlS8=E$_A2rmJ5spZw&&MlO=m^W#5% zJpb&bcCaS`L2=IzWj)|&ICcf@#CfrCsi56E4<2knK;YW6SG=#XOAW#1V+miD`H2NA zAB%r^7SskcrWa`@aA`Kg)Su->Z=Qv5rfD|6oKLDue^BVdb`4ttEAXHba*tMg_?oWo6_;_qrhqD0Td?QgFTVJAuO=X<7^SUE0GwrW+o)O z^yHo zt5V}62U?}Ah;>o(k3-HOQKN_va5*L1%TW7*Y^KV<%fVF0(w8jTh@y#3itglm7MznW zYkDtdXV0cO2|FkI=od*k{)Z=VEs#6JD_5`oLSS@0i!^Z7cXdu?9yA+NASM&Is7K-` z-;Qjirq7#y(AD#_h@HWB@|kQ4U@-`T<>-aW_{6#xeErz=YE{Gq=}iqM=C4$&CxVjg zs_ShfOHx7BC5$XSg>NKNFDfKZ>6&l7KbSg~|I~99p^%P_Yzk;gcKnNh^eaHQ_iPG0 zIWI7*maN0uuz$-$H5050pq(J3<^Nf5@`*d&#x@9DL;Y8CFhhRGok`mp_sM%n-9JFz z_U-y~$&{_)(-KWZX^p>;!ecXD=n1xb^}@r%@E|JT_!MosM;k;xI02ire4Rdp7*ar^ z@bIflj=gs@Zq!H*u#Zl(z_MvCCdDdcbw$y|ZTFad$+2rvWfbI`Jl7gfjoP`}(@K9z zZrRVp*WyZ_1cg8-|KY{9_G)DpxBkydaqhy*LB9)@$IT{L&re+5_T5uE2gz&7poM!r zdpvIY?t9~h&GOdQ?_ZGEM-=WW3KxhLPGVmpav?rN2qaEp73Dk4tCQ|TIC`r52Plr6 z@63+Xitzt=09SrdCDcGkBYHOPrOZxYg+vx(9Batyk=bcR71SqA+(xYMekYg%7$)Uh ziWBoGRQj0MxK)H)gUepiII5@FjX|Ev=J%=wZL46>d06H#anjewgDDUnhl#4sVJc8A zmhBzr<|LZH_@lrOY|#_Ln1;u&dtT?O?0*e`yV)(5y~4>(1pQ*5!L$iRJ^|qGW8i`e zz36rvH!*xyr-!l))dzSfJ`W+6z)xblkqw&Uxq%;)=Z*@=-wtj-7zgNkgviNAXFG@^>OjhY`;>)`xw*cp}V70q?qQ5H$wc$j`&GUWRApDJ3_Ry^zs zZlXDSY60n36A-aqmsV9zcAP4BD6SSUgdaWnnoR7Le-S7oCMC$FXiqyAGK#o_zZKYJ zj{^(V8$X_&Ty@|;WlFRCIZw72D8AlIcA2`*F>!UH`7QEgDW0C%-}-0kMUB1CzxMAV zRN?Sx$`Qpe=KnPL8@k3~ThmR)P7yWXvmFhS<#FdH+X@3W?Q9i#`O$>_OYYx^H#m!P znJBs)UX|}nqx4518<31@N$>eL>(Y;bD(lv-k7RkPl(E?I;%av-f+^thei*X>E-yG2 zY>#k7Q^{9;SKA$3CVzw|unj>2%! zDqVvsKWC+TH$(Sh^z$6t^9L+2Kgop4oY*}0L;B=jPotF2+q{4K*3Xy2h)zfuF3U2H zi~r756=n`CSIVcTjiZWKY$nTYh}&+kaOGo0Cy6;BAFl;@WbPwboFMiJ|h zl74M_eJJPN!-q@GFD(X}YDO|f55KmOb8!3iuJlE-*<#VZ|Bj5;bH%>}k(6c)Owwdj z$gmJJQr++oe{&<{u(bknxfvNlIx6CsC>!@0(oHdNk3R8cGAHTQ4PwMuuP6((59uil zw{ewq;`g(&ACD09URH=?|F=LO1#QpA`W^JP>$5J8lziA!Ny-00nvBTVgS)A}SWxVK zt7u}N9oaa56IVd2P0QLhsKFdZzD`F{PjbsD-j%rL3{g{i6_I!_H52Y$eEzUZt3F1` zb52i^-=&qG1#4X+D^|^%HB0k-y1Gw(0MK#Tr*!s*E&Z#yzXLZEuk+p9+2w$Rr zdg;+!IiZb#zt{^|F?-`vWQ6hgua8DlJ$Dxpw(MUz}Tc7Xix_AOrVOY8T!Fx;nU?D#rO=~J=e868zsClkk!%O1AP zyXq40`>+)H9H?1+&b+PG%scV7L1=%^RYSHBL6+&rfA}z=hV}PGD+g$9eA&YuKj4s$ zs)p5|>#rXLqa-aS{QdHw9oWVSKE7_oiI)5Bx~QZ1;p3EE#7>J#Kz6%UmSi`1eA^1< zjbR{J>}|;W z#&vVg{ZyqJNli@^MOTC?t9gwaBwRcjZBX>=YAOk|Tw{r&S7P|7wB#zyE_|Tgkg!t-Pil9zlW65n1^(3l0^1`f! zH%)JZ_(`#rR6c65Y7Xg{w`$o^7Cf}R6|Gh8e=A<=ba$SQJt8V2uLiTMgem$o(Cfc@ z|9&4((8m`N5;}rw*Kg9K^}d?}N6=L=7eUVL$$}x*wKp|Az`ola0CfWf4qPY@IV8qk zhFVGO8r7ApJKV*eB4L-#_Wa)4&~(%QO-&^O|FTkrx|@yZpEsTnm5M-kH0O5+jNU-w zm-CSD7i+EaEe_=@g!iEL6FtMot0Vg(|3sJKc3f+wVq+dr*T;kPLRnfXn}YyXeyVgV zq!D?fjit5DuDN=|PF39~k^|!21&B|0*P{;dC0#Kvm^~hUu zk;Y$EvEYjMErv*K1y{>)kxF#pghA~1TU!~kWjO=ln+<4Ez6_`4{Vxj%9g*F!w-6Xr zxz*lmXy&TQSXqq@s)mk|6-c#b_ITeu zzfam=y|5YPrM0|3@~)iC>@~Cr(0#+nf$C&JY+`=0-Cnfz%;Veku>;zGQ96RaiWbIS zD|QLiGSa%%$ptsr%PNg(Eg^qE!;ck-@3QY0pn^TqV0&$fz)z7Nuvgb69D9x|@8Pzs z6_CT)A-dyEinSx;N=X*H&@3U@Q;2eCCn`KLJ#1z8tv;4$#&LNS)D%(U;hWGY?Haj|x+C*}am?BsvRYya0 zyln4KsVFa(jS)mg0&&5WZ4(qLCZ?6_vO^^EaXSpQfVW_r{53yjN2uL{UPJbQUBusb z%WGYp>XsiT`9nxz((<3#*x1OQVmMx#W-^mB1kZ|(I&;(E@QhgBrs#va{Q8v!rmEp1n1>h8G^<*&3w0a2H6q__4rZI1&Br=Ar&+2sbE+ z0(QJ}MXuX-y>VlxqrU9U!qupoGEQ?WifCe=)ID~{7(mwt1c-*LIF^@)lzGr$^oyCt+KCir{M(n$CX7BR&QSbAP zOm{r+2lX?2emd>XNlQ(c>@%X_563qznrYnbpZoVdvLlp-cN+L2sNUE)oj>|w4Lttj z-(hgNYz(^G^H+TEWw3+Hs!d`F$EQ4|*jZCR!y(qTh=dlqlB5fiQGhp>}R7ZYvLkLDB!To%8h zTF_}cv}Pe8xcfB)o>BuiS@=o{c3Z{EH|wz%QKG|k1wejPM@@b7k$_2l(cEuC~+ z1+&53E){LK5V<)9p`|L)1ww!5~9&JX;O1>!w=G)|Ya7Foqmle|un zrNh{1Alj!>vF6wR`ghhz-SYCkHeQK!vj?l^(qLRmXrt}6y6kgT*e*$#0YXROk=3#rzjin!Z&Ae@;%at#<8|u`fy(vp=VG5AklCZ_Y*~&!6Rs_Zfaok0oMg zXkO5sd$&=T%(=L{CnMa_aNbz}>7Bkaer21uH}F+Us|!k#27YT9P+7R$%p=TW@8oN+IBe zXHw*q^F|Dg(8NK{6@~|ZfAA)Le4ue@JF^bsrfsPv=#kTb-}NcIC&?A|W`DIAmNIbB z!p@E85N#J&*ZJ1x^X45YrJKM|6<>ip2S=a$ttHD(G%WgP32phAPP?lXGCGS@~DX~CdoCNQX)$NTv z`IPbbDU>iaRf`5%!kvum7<|Ys5wfF>M)dJ2WbwyGd$0AE&CX_J92mBliA-Ws49D9~ zB*Slg(~*CBd5wZ3iO3~gVdn8E3vstMiBB_x2$WK0Y-|&r#oAZiFc|)q>y-WOp=I4n z3hSw!3z*?~!2$*jO2{0|_u+9BWL`Zq=ZMJ23r1!8CrBxajeX#{0C%Q{sqtq`c8Hqo zky+^Sz3`yPcv1LuXx%#hY=Nd`G@zCugj``|O7$acq6OwHUu(YW&I1Sfs-dt{0+W$R zhXNuowXzH=#RWn%)vp_koYZU}hsCx5=I$P{A+qvd%Pqe~d3gP7m6cGi^;gD8^Bz0h zjP`GtJFbsX&C4}AOb=M1YVAF+yFvHW--{CpmWRY&{N?@lz2}3EU%nZ14hVkGR;&Hu z9nrOFg@&&Ct5&b;!!sWG40iLlictPl~dEAAUnYmyBBKSOh|m&1c&cIFIY)EsCA>!^ojL#g8#wkukRJ_`H? znxNLqxtpnkf?@&w?YxiInEa{+als8s9{T(E##Vg)4o%*aTX^8w{6heid4ODNrW_Br zUyvTOSyp6AP~xuJv8k_}c$h-ZpU*aVUsyPvh@#|KdS?m>oEK_*#XAE0V;`BM5!HHB zQk#ZrQkrSnL3K)iX=rGu!I^}be?w&e|0&tq7oPulDJ0{3tr`d-g^Mq#Xr6wA@ryl1 zFS+geSB{w8Yd0;=cx&rk!NI`}=r7XVlZ+4?HeN|SEZGBdA9ol_uA@ewg!cECUSojQ z&s7A=qrlO#M-o||7F?o6k0hemxbv}V=y?{~N-F7AxAjaT7*g~x&>ga@DqfabK`teo z8K)e~YhIi4fFyc|T4-dB+f}ZG%OJJ4$cuU~^Lr_OA1xl&r`({(j0&u(LI;)o2CN@i zyPtNG7v;%i;W`?ss?BdtZ}MSF@tPg~eRDVct>c(G`1HR}(uyz5+WGNAU)J7TZ~KVO zYo;%6i|}Li%8>)MS5EzYcHxWR*D8iQHYv)0D*Es!rXg7MZp~{6o|QvQOiUC*)PYJT zN5|32mi3cx|D_X6=gg@OK1V=CeM22xl)T7h5M%ylBGE|9&d2N)`oDdN{a0cEdC>DQ z_$Ev81TAm-J9q9#_OyHC=Xc!4>b|9vpXUVyo28d}v0x$KyDY)seYzbZcQ%5<=jOV4 z#WySmwF4(s^F0!N@}we0^~91`2AqjE=);YE1hX$+avn zhFDH4lRgv_tn2^Tg(l(QN^}M^fxgp$)@%M*aJ-dzw{EiID_`G<-%F(6 zX=%#Aw@1Q!HxrrFeG%PpHS|zUI88%}41+L*bQ4){ZE1NVdDct$E!dDP*gP0ONEKo%&Z!PV zX23E~%N>ZMd3+2X7d*($o^*rJ`rfx(y?f#Y6q!A+2A!^Sa4b!W-A53M(AVvoB;av_2X~udIhKDZ_Hkku z3`mdfCN|4ns!{cXtwpbTaFkcS6T^lKdCp)FTvS?6#hJY+A@&Seb4!Bx-QTXh-v+{h zCw3b@=)9-C3+xmfGOmh^`eA;z7q03!Xn!LvnKeYVvKts#M~99bpY^IfRdCD5AXh#a zyeT~WYTLp47F0H6@KXx7wWjH;PWuXP+I*-!OnUcx!*BoTUYqUS+#+^k+zZ>voo{;Z zC(dYE4S2EYYO|9S%OAGSuRFWm<9Tx=W|WCeVu{oQHyn$XY2?TSh69F)a9?!rgl7iy znYJ3tgi+M^^_b5*V9-a6c~jPebc&LtU?VGwXNKFnLo~S+unWV`J{!7D!MwR5ipD$Ylf1lt0V_c{VY_?sdd(?Qrby-_i}m6FmNyYUMTa%`^|-MyZ3L@Vi?xu|L*#yO`PTfg{Iow4 z7Ma<9J4EdbDoYrus~b7pucM8<{oZYmL}YK~Ai7s?SFcW;%Xljd+eWpQ(_m6iV-2ME z=%sz)E?!h+%tC1@sUoDgYAr2#Rn>PkmfSyNSg;DR%d5+mC+3JcSUGHve*JnKDAX3a zQor90(rJostVf?dn%P_^XW4y4Tk-&gdvek6Vy?Tr zF(JlH5-3^TH1Qopk3p!=xc}X$=Bbv0kd2{fePklitTAKOCC~a+cs*tx9r)I46nS45 z_O4vNKJM>9kF~%2h&*#<>Vzht*#n0Rk!*Ei&H1ZeB$Vc^pBIsD(m2Y!MQO~XJ>%`e z$WIad&Fa;aN6z%#JrTWsS6yAVd42iWHhPqTr~YMuRPZ(n0=3pXFzaRvrH*H>-eGak6m`@=Lk3tqZf3=NGcs42@Z z1AB4AnJz;CNoF(!cSG{AvT}=e?!GCvC>)-OK^M^XK+}lC#Pt1GN;z_G^-BnrJ(06RrAK$aM z_Rv+vS({6*JXL;aW$fFQ*S-l;IuEtM3}{w0a?|Mol+>f`!Dt>{bm@I3WR(4B0l--9 zE=mm1q8^JzD7X<~l{p~8gf(%06NE=wbG?p-4PD8pE=L&+=XTha?jobRX@kI(S`oye>fvG^g z-GCC!dIk1nmSfr*xZ|0npH5Wc7Al9P+IASXM|wu)K~z%CA2ISG-_-&sf;WrvN&kA` zO=#tR?@L;zpP+u!0ZHLkm^OcY-xJqw++YFhgDd2!GFAG|qxWGnBA!LJd-v|Lq7?K8 z+!Xa&U~$`7d{9iW#Qcc2QD49Aj~_pj)DEhkA5+f`s$nL+iCoRm_^;s1dh6!H1zg3c zt8^^NPHth$(HwU5k`=jbo0E#PEk$UE=AI?CYm)kyhj5t;$@wt2y)Ri|7sfdcq0y6- z*<4EI>oU)gw;Ir4Yj$>aLqm&Sgr!qXH!~D$%_dH19esinxrK>7K1Fr#c)?zLUxW}z zgurOdG*(M$@4$hMduA$6K0wwA%My;TrcGU=QO-H~Igy?4?(e%Bz_;<)v_Ry2GGB4AT)TVsRmNXwubsFPM8}IfIbhjL zLX+AdqnPtF*f=ytd+5+D@N@=!`}MnHU69i}ClJ;PlxUJQH#u2%n}m|qfUah1-&lim z7I<-aVEPJ`N z|AI#w^whX*XhQ&~@0zHv#ffW;1Qk^&TZ7_Ewlg25T`A)toM2#}yl;!uB*-)PQS(kn zh#DG>C7E#7$Yy0l7IWR>K#Y+#wE+ec@q}CYj%a#C9JT}7o;%eqpjlq6+-IX^8O^Chv~%YgYRr|Cl!Bv= zStB?ncr)ao?W3sXIVWGfG)E?2U=8)gdas;zpqo_1N%>xkctl@ybc!><@g2BD)`#E; z@(&CMXaUVVkm@%2gm>zLr7`m$XzR=9brG~gM0_F-pQvB;EI+u-FM{2Kew!Yz^19uk zLG2`Y6R9^mPtWue`Z-x6OB$kP`^M7hqbNOks$ai;RYMi+-5T!jf(_Nj#P6TMfrb~d zMsB(CKzZMS1dJ_ySr0o5YIYo0xqy zp4fIAef9Z}+{P<52Zep?GbEV#K`x%6F1&hK6^oI$r^=FyojlbLkCR$AUTSZjf>kJz zrCZ_i_5rdhi}qJU*s{P;H1#YzNJzE}kTvw&^0L<184cib@Z+-gNk46d3>((aaQsX^ z$_ENdQF)rE5Pv}2$LE~IA(#2-W*5^C!pJr<5tzgYtKG*mF}PwTmlQ6(m1Um~yP2RD z;*T;J(CI~Y?a^bxAo{`eX_iqZEye|Qns~JMaI#N&*b-?oh2I)IdTpE$%Q29eY<ya;@NFNq|R;Lkk0D2%!IM7CC*^^{w z_d3U-CPOFeG1*&Z4RexhPSiM#kqWsRPQgSqZ9qF#R$kt~P|Jg12;JTWq!^plZP~VM{l$wHr#(xD1(021X1yu~)5pnY2Jq{D(NIHT;1x28=+4CC`#YMN zc0SV}Ad7UR6!;r&Sd^{V%*<*jFZ5zVX$1$dN92<0)H4y>;@o(_%aiK=SiX8Cgvy+A zOES0p>dRcZcN>x`tKuT>1=n$CVz6Oolqkx#t`gmWMwF3eLHWTJ8Sf7S2eXbg^!%NP zmbWl12>fEucjU-}rQLmw8F+doUXRQr4!C@nzU5e?Rt{fZ8EpMdnKgaic9-{(L5PPX zZ~(|q-Gw|dDb3WcKgnaKvRIBm;>Qj!a|;KD&p;DMfMKIJO-fCh?#OocqI(1@#r-lS zER%?WWjCGUn4gqex30T;VjWNb=Gi;OzjH8y25K(5mUD6R{ieQKw~s*?udcGbpljxi z_kaB!x6(E1W&Rpj>fsEPH`Q?!X=nONDbJT;fE=7K0q+fJZMY=aLv`~_gX2<}@U@j~Yi=LI#VpkKUAW?cp zVyR<{Hh)dDtPDPI0Q_z(ka(aC;0EYZ?O!W`)>Y*d8O?A8qe3djXIMV#KEf_YLFFK#mKus#mWbYWgVM;EsN`UJJ9&ZJ!?V{kwbhar%!Yta^jp zj1>+J$-HU_Y3s3n_5PI~XCJ>uYjEZI#)ez{h5(+B+j^O|Mq@-%b|x@ENqmvgk#6q8 zfG7M17GBPLPZxce2BYukdP&cpKVKsq{fH5(jqZd^)@T7pq8@Yh?Adm|OhVHT))I#! zTr)@2>#ZxVR-_;_@1GwP8XFmDoaj3@YD4zPC3kwG1uZKxZQql@kABd|OjWf)TuHR$ z26jJxlv7t^S;#T=;tBM(=SPIn8a^}e3JO}`0_nghJ9J%_aH8JwXxwR4opU*usGcYkM^nb8_(VHJ*%_-IBd}mzfg`8UrSo6m&gCzuXaDJCQ0rVe23lx zLv>!fDXMv--bJ&LRkMSZX!j0CJBvM-%ngHkv2C_$z~K}qa$oL_Bru#$R95r&>z6Rh zh~*U7gVM18|1?Xioi{rF{9F%`w8k1BJ~aRG@D6$Uilw{Wh!1g3@3R4#VVWEwJ&?>% z@b|P{lGBUGE%1GnsAD`q6c-%&6Rsy&MsQC-HUiurYL~D{xH99{uU=8qC-5=uvw=kt ztrYFvdgRZbey`?4CL|n4SN^y&Iy!os;8RVS1fTxaiwtlO1qqXh6V?n$Oc5qy^|m1C zR`S%Gw2_l-M_xYG!dXBMxfo*rLPjr5od>uw;Rbx@oJ%Vs4{6?mx~*4usYt}4a_RCB z&imn8dkxpUs1hhaf6trOp4T_>?ZBp!@8GFGhw2{~7S>I6L-+9eNRCS#AdQVLUL@Tf z$eM+^o+aU?4Gb5_T#fdn?tv-YPuM?>Z7e}yGcq5e$cHdH@v3kJVI#oLUH0C>Kk3H4 z$?H67YFa!OKt^391EyDCPf$63kEiG^YTbL-hQv)Q%I4%P&sgE&GH%(j(^>P^M=p|p z^ccY93l|(52R66h*(GD`22ens1jvk- zwWJn9o*l$;eqtIKDHZ}@;L8(0tbaN zn3=@QP@FDqCb3-;FRSi?FG+U$2G4{xm9H9Wm;@20yr3OuT{@)uj9$B^g=e9vCU2;) z|1xlmE!1^>P5vQnYqG8}jXB0>{(-Q2{_2(H>rw(Q6<<-U^#dAgjOf;5S?5;L*DblJ zz4`Xo(9YRSfB%WVz8}h{YKkw-Tg@v?x@>OLru}9;ocyYR{x82xKeyxEmjxGw1-E+e z?8#J6PEtb3KE)TR`K}jb0ef|252LNSR&BZu>HLH?!A6z;9Yj|$T@-`X5fA{nI@P6S zb9sFE%v6UCoo5!6#$IDYiO<8&B>HRKiU%7cvmmwZ7wv0VS*<#+E>;$pzPukkl@|c( z*pk-1)!54gPi3G81=&c@7=&4ycU?nungb`obQYEIb4lHFaCknewTDJP8p}p{EIsxF z=#Mk8_QVnmblzJ7jl!>Aaw}3g5;;P^HvD9)tcY`k{S!B|S+7Ygf>sWr8J#$0a2m}F zzS$Y%g*V@|$!eyQuT!&n9e-jL@L>|i4LnIrA^_u;&J$!3AJ=a{QBDeX;nt6mBLM@) z2x9mjAic zq8*#&kbyLqsCRE%gwdo=g23U z*vaG4QTa1;0RA31HHbM_k3EKAXZ*B56-4tThO5OUBHXK zLXG27=-#DWW^s9WaFZH}p#Kf0_A|`kconU1qsEhe4!_FZhZyj^d-v|#DfQA4ENc+b zxke0K1&89$AvQq%vROaO?X5OlTMDu`w%0Y-Z>aq`gRwJU+v-)Tnsw+f{B>#9sP4cT z93Pz>ubp%i+bER%D$&geHgzPH46l(Me_M8=4Q55zdXNl*5u8?Jz#d_F(^W%3-^Fo_ zj=-Pgxb>)D0kgIJs>A&%tOo6uxF#e-NH3Gk2K;#W4Y>zr>;~x!FW$Tv@>Uz52nFC8 zthM@KqY*~44sw@Md0822oc8~6c5ylIUsCmVQ;jmRsN3z%{l+iLlP@Ts^O>>W7PnUQ z&F=I&8#;9V)ziN@EWKKNie_FJ-6Yt)?K(&6j8qW#1|_S^I@GiN?$_c)P5Z^~?T5dA z^TwMH6j>vnLl1mgHodK)uX}jjzB3G;^oiAZN{sL6S}fG%x&&!RJbQ9eSJ`05|5R)I zFY!av&idiDr$4aV3IVB5l>@S?uIo)n>^#oTh;A&hMoO8;gwn3)ays!X>2MNz3^j=JzbI!*Nx^h>Z5BVzB~mPPT11Ro=~-O`0_E&!0Xo zg5KltOR64i%?tYQdZHbv#^2>OufRk}-iNs8{}##PCU?V^ou9`@k9N+bm@sBqGsn(hu*o@ zlVBu)#%yfD<7DX$LmueG9qxEXk`=aoXN;4L$rWpt7~KvT)fuLNtlxYI?UiWt0lPmk zwgco}<4)iRL6So}2K{rp84|5DM?7(l24LnSW7oHz=7hZ}Kui;Iq&O+c-;f47dv=)h9bL@F~Bu*Q!|Lk2LDiU))kpmb$i~a{YtjCbGYPMLwju4LRH{`NH(&?cs*Avm>;gCxus> z9s7E_!;0>ijtgYN2k2!P_E^HI{vWp91Fq-&{r}J2Ga@6T;vgk&k(H6rGBP7sAxb1O zDl>J;XdoFmNlwSCgdE9e2uW6U6ooPp6{Y_7>s_7i_xrp3yPezT_W7Jo^6ve9J)hTg zJ=QhD=|K*F z29C9dn=DHp3Lh>4Eu|uZ49gZBJL)3*^#sjJDVgH)(C^cV)vG<_N{pFt>Z;;=H(oA( z6t%fQ1}mm^vaadjlAS-%Z)?pzX7vZDPbk(6Q>I5uUePDbh*d?18*Qz8WIc`N$xVGl*hIv2MmY2oGP6gJ1uBmUa-~dQ{ zIjZiN9Ujyak;KrF*R=)=9N2KIVSglUrl0H39Dtd+SdJM}&%LZ*4D#IRmvbo2d@P*} z46KG3gzoR-+$m3h_xjd{s@OSz&wwkv_uZCz1st&9kmDR*wTy634J4njA_D}`;F?(b z!4Pc)t^pJJ$hRwg&UAZ)+#3d2OH5%GT*qt9oHk)X=h&iSfUMLh_N3--p$&vIonx^7 zjd1OdqBSJ%dY&aOO8&C zE7v~OBk`VGS@l3AO6KAC>g3M`bU>u^~y{PELjySh~1jRWJ^1O0jhU zv%qHWuXwcqd7m&^K|$S_#HN^;HIz=eNelM8EQkD*j&$my#uK4X&~bW{+~{=2=Mm9o zuxkylqy6=IrHP#n_^P(0DO{-RWI16p`pEtkI6~CHP{^5!9DcbYeK)!p!>Xf}44Mh9 zkOXXdCx!uo33rC&jThdT5jQ>eC|s~EX=igx%TFDyh0S(JdTsq4Jyz~%~e&mFjUF`1CyUOs8X;%ILHX@Kx7wuLN4whPifrX+Rc}MsRJLX z1>Rf@upavN-}RM*1QQgyYx?DS+|_|l6RIjP>pc_BD4bVECM#6o0VTVz(AkH979oSs ze*>O=XTh6Lq|n6LKVYm47a#7*)Sy*7BQGznPI#mxGFJ*kGBr-u*4)xE5liuaX>&X+ z9Tyr!e$Y<0&}T`X1nStt&MR2cN`>F%>#Ox&BK`V4ZOs~P(LC&Evdt9ggO8pFo3)Eo zBPp8_XDk-^gLkdC4#+Lkk|im9=HC3wx$gqKk9aO8cDZUS(k)V0nRgH*{;bMoZY=;*$$+=^!z793p2HHjqPNxX+KR;$z>$S%cE1z*xU=(8RV9Zb_^ zOn?U*_;`uSOhkI7r~mYJ<&|8o;>YX??Zr)h=Neb2BUO64`G(=!#I)IbQY?~+eT9by zfnpc#?th9b2@GR^@l)&tJeR@y$;rcB;k_`K@?-n$;sCNw!WXoMNh_SF%8@z&rtLOx zz*)X(0%zR7?3k57`A_GbafboCBq#Cyxu3vUyqRbJoNsb!YESj5xhsO9%V6Qr>UI!@ zgHX#O3a#LD`#_s|DjS*RL@jHkH4gd9(6X;1NoSGn5A)O3bo+H4#fe$BFn0-N2=tQk zRx<@C4(vLxY?&n4B3+_E6NG{|Wi&iXPvhtkK#PW{W`K1FG44lWVy0M`j~z=*R8#d@ z=nD|Wr6G%OT~0v@(ni*tWElduC?l_0?gMnU1Z+i5IoRg@9r&hwzKrVn1_m{8mi5`> zc<0`|=Asq#iEpGLrcS{EhW!(pOjxkN=e#^sEd9Qy4i6f+)P*osJHux)hYODe6wx+r#$v1NRpz z4_HEgZDt+!OF5z_k&xhPbARo%`+zJFG&zjtBWm$_56zi@At8E3p3Xp;%%Qb)G^3hH zTiB~tptOZ0rK9Y+)~#1h1SS{(%s)c!5MA$!Y1!_!(5SdUd`1`Wh{%|tRHRmhqlZ2v zLv4#R5!{CUG5d6~!N<_G(ZY(}g+iz549?GSMbc%re(<5UPgFlN=yYs^GHFA~Q>f97Xuaw`aB|}=t4(fwY)js( zGafj)iG;AVuuQq}xzG0<>IK8M4hn3T=M=?euH#>Hvg`PO!IQ-=YQo= zjt#GrQEp&~oMQ*4k@(FegqOo;9$a(E)Tx?CxFPv#KCr%Z=S~7s8&Ax<)3YwGZ4v)7 zb=i>#Uu)FRbalgTMa>>LZQ8zA$^5K3Uq%B5UdOroEe?+O|DLa(DsdWB?HC#H{W1{u z3XXrmUSUu$=upX_05|Bw0AC)eIw+b@dP=hxZ3dq%*jc6nkyVQ`dL z0{{JJMx?i{g^l2m#8)9rCo_wfVR!D_S+k{&cyMR{&6l8~bL{b(sH2uLGIB^Py3;@ob9V9;p> z#|0{|b=$TA^~uVH4y^CNxw>7)J!~2}*WN}>Qs^IzmQS3sCal|w4yXRDSEq_tPPcj8 zd7w(w|7h_$A$8(T+CZb>(g0fBxHkax#WGb@69nL1l6Qfq9@mt928oofv$8gWh@-g( zU=k-7W5<^(rA-MFv@u5b2ld54*VKPG8)Hii*}Fs_X6$OTadK{U-#%5C>q>%%?Bn18 zjD1XNM4_4k)I}b^@jW5_Qi@+aeypS4B(;@LsMAk8YG^j^IfK3@31ouv3sFm2 z*SW8rzN+z)dN*C{0l2J0hWW$r#OFa}N!&8k6(WVR#GL3)&T?UkcI|p}-?6Fy-8bhT zl1rEoS)@)3Z&6Q9pE=WvH%(&Np=FUz%_~lIL!%@eoT(VRWY2iu&7sH*MBIbRM?ZrL zmn9=}r(`r6_2JW}JvPy`RfM*B`}WS*k&C06V#^BZzf6NxDK&;i@xYZU3S#X!OQGl{ zteQ1yBwI1s-@Mz0hK3Of#hVhjV4Bs^-}3`8*;U#^-P5z6(M78IwjjS3QLTZ#jPwMz zloam|-@{D9pw#Q`+}UibPGmZM{rc*de^T5n2fNd+v~=&ZY2iM00x%{@?A89G#V$ZI zSi!a*IPkpJ@%0j=52a?RD+EBhsBVai(Cn)Ln=Wv!$gaa*B)TtjALtzV`g2tNlJ*bp z&UO^%z8f_Bbcxjs@P$ntuq(vA9q{Urqen+t%|vjPyT18T6@ENPuSa5It>xFfG3d+X z8-*J*gQIX*iV7#|@KcssN=j(j6C?e~XWDu@fUPL1P+=usy?Ry8DK96dF32k%E>q+Y z`b`u}dWSr|8c=x=U(mN8!i~E1vq2SxZlc+EM||TWGB3i4WE1$hyU`5j$V{XA;Eye(+1A-|-R5(qdzy{_$&+w=aBbN5*2Y@ym7Q7C4DhiFTdZtuE- zw)gAb|0o-Otjx{L=RY{nfHy({X+%w|4Bu|c?5y^D$IhLaj1~YSV=XKaKnzc1U3zaJ zj~L8_Rty8wbyhT5bRN{QQ>R@V@083BmoE=S@z{37&lj^2B(9iDxVG&B?nVn^&+h_l z{1g()52X;3)_w=Z6X@4NmgVD62mPmZr~d6?kRLGMYoD{R^iGWlV;sy0ZxfXCYs%!w zuSwU}ZyV;LV2h_x>Rwo=8tC{rI6vq@e0)eoTl;5{GsrybZ=|Xx?a`ps`NJkp4%}#G z677(+?jlLbL7f$OU?6;y8#zw%v2omAN)80D`_d^cyL70D(e_@F=!mXx8*WMxIO>t) zgN^dg%J#hTv|f^B&2w=eeJ}_qCc!-qA1?Xf>#pP1kB?&{dsM>^&ef%L1qaL@$xHJ<6KoQ-?5y1MPlKe)xlQ@4xLljeh)9b5>8 zaXx(ZqJ%Z$OYaRd=soaOPacXltlx?GJH*)I)x2vi!5XHfrrctC9!fMI9)6$1-lMEp zTQREZIpFlTbHlgwqvar0d*>*ZOhDnI*oY?daGeAL4|}}#p+M>#jOL|p9ixRD2mpEP z5@s#Vk*d_jM-ZuRa&mMcP!TmS8yUS|o=ZtlQPH&CE5@8pe)`nf;QZeC?lW6;>a;9< z@E|(E(MK=VsfVI_)Y7aqqeySSA8;)AqE+2!sU}I+Ue|PB6 z0Xh8EiUQ;EdX(AU_dV0}+GT_Em-ifZaKPNi_?T7b4lbf|2vcjYC>R8TCE7t2iNLcs^=2FnOApK>-8S+6+)r@b`or>W$EjH&Y zQ+>)sj3EDfnf2#9q`k;Ez#ceCV{K>B)0cldQ+_M!SHsk%-0nVg3qdmGUEe*c4>-B5 zZlZ*#TDsbjfw?XI=Yja)^3bm|i9SQYGANR@(pwi!Lef!Ql58xYGHun>BiBOkbVS%R z6ri?npY4e#HRyK#0=T`kWuuoR%+U`z6K@;my6S#mH$E0d&TN9VblItgH1D!{8hxma zI)1JhzrTQOTyoC#6mBn)=)2*I0qsCe#OYA@_3a8H);KraHlhocUEEIWB|h=HXbXXj zV=i7XGx^>F97Cy>c=1UVrei4Ns+(X?dD{p7A$=m0LwrSPeD>>WThOkhn1?)Hy>jJN znj!d{dn`T`E(1P~ty+6RvY~$d_3a+1}(4T~FGPZnWFTIF+;6vqktf*tx>8=s5Buo_GHlj*cfm9%cyJ+q0*(u)T(7)(bW*$~FLK zE?lzVd|}RJIFb(`1ju05@Ac(wAf^KrHdXKD2DV0~Tz|+5MpPR4vm*YaDStADT7g5@ zMVCT_gZfZF{86rmiv$&g<>06x5Vs)LX(u6@%0=Aw@6S8EXWzcALZ1H<(TC(*x)(rR ztyPkw#t<>J-LZflGnh*mGg(C*`}p3ytMh+^@SP;jwxkQ2UoeBoKzug+iZgP0K;N+J zFesUGn8i8dL|v;C<}NJ=M&ClG3L1VrdwB39ozF87&2D{rFy|y< z7Q`@Pm%!N_bQ$)2FPJYun_VJK*~SIC4DoUtCV(D>{|K0Px?5!!Z+FcoYt-aiRZ7pI z1{ovF8@tomR=Sy&Er0 zZpGq)7Y^I58eGG-{itrDfM|DYglESdYqtLTJSp0{y2blH2;TP^3eOoUW@*LtD!MME!Qvhh*ZGOX{ecBgKLAChM@3vL7C#IbQAWk;n02-{t#7!Zz3#M>27B}rH68+=c0+p?t=y%qb{mYo(ywxQwFIX&|FFsG|_ z8#leIX$?`nfuGF_NS;!Z1ezBWf{LO|kf-A;Aw0kuL<5o`fzG5&+9rn@EzY{~NW5ds zgC}xFQ5Qy&Mf=_Lci3|x)FhUB z-F*2f%c^Ro*)#Xk$*bM^9QOLYyp>Y0Wag{$+VOYuTmPPnZ{WrPm{AYz@_I!~Pms}O ztEYN+^l7$z!guqQpde9IU9y?DAyc)N3YZEMMLZf;C!g6%Yn*PM2mC0c*%6cP;gvvK z9iTeX-xM#kRCN;+`!^6CYs(t+!E3*Y)hvp{w_Ed^qfi!!2pln`&-}ebMXUKA;RuOR z*o1B|p=D|3fD|^AXh;0`{{6`1%Qj;*PEB<^eP(ixE_`fuf9;=Zu)kMPs$;%m1$ths zY!T;~j#cnp0$G?ZA%ZIU;^8r8^pG1!E8|}LZ)O)fxjT0TWR@;vS*R4rP89{6=#ktO zA6@KVSGjXrY=eneZ9T)s5Q@hmsvSd$<$+)O*Udz_OIf=xz%cdh-O&*5M<#2T!Bw{U zGrpG-LDCY-yUAPm%md!_`n-$7Q9X&M`kovovLQtDdOBWv)HO7c;7)qoU7#|DB`TsP ze!F7m(8W#gpJ`{aQen91&Ar&tM8Y8Xsfa2P^TgZ7TFMlM=?c0o9Ur&61UJ2>hP)Rv zPG!j+W%US2(iu`wTwI(CsUhit%$SYF{>;HLm7bB&Edqh2cY~3JGV=p#&360ITFK+C zSu_69Ea;#sW#7!nfv+cz|G{tFRKI+nYljdVIaWH#dwvCf>d3!Z?`n~S0OXmj6G_HO0 zeP@owq*l2=@e#jU-|R;l#Qk(2yC2GXGI3&; z?&*2XN)7bRQCqT!RR8P533C}@09fc`w^>C5KBO)7rW0*ow%`wz?}VCu6ojW;E!7rr zzFdTO618GrmjvjtaBTjJv^Yctf!@jbgALcNrxOFX-xS7D*JfL;>E~JNxQD35FN<=+ z-V3fS4x-&m^6f$5bWeAbyhyW=q3C9<^YfmTEpa6)L2`E_P)!1f06528nkDK11Qj0T zxgJSuChs$F$;*rgnYAwwmzg@L|-+5C>YRN1|H zRGg;M*vciLVt#%#c&fX1wB$C{R%gj=gOuUt^Af*oaCVMNbbuEV?=psa-5j&#sjX=s z)ruSHd|scjT6k_K2Js*ApzGRB>#;|a$C+iRnaD9m47+}TdApnXzrdM5(s+v-(Blx6 z+JsxW?0#%r*ab-jE}-daG$a|)XbW!ehk%nh-JAGLhu%V3x9=ep`z+Zw%DEkDOJe2? zgjdcx>-p`GE!8eLzKOr@(xb_~=8Oy8_GYxSk^ny4rYm@NA0Fg7z=+k6Wbc=vPZH0+ zyo~Zf;|f(e5Vf>2%RR9Bk_?L6y6ksm#Sr(>=b2fD*1Wm~g)UR6JpKGXGHzdwDj zz0Mcj#oF~qqt&A?-mtz_^tif2Jl;8NdG(y{xYb7IMsS@%K(AV-o7&g_065-0mNfT8 z-nq|p|DoTmf%~V8g z`p(FIs(e}RxI^AeCIFgY1mB;PpWjOz|Hg3B{O!!o6sNGq^v2xjX|8+efvh49{%EX- z)-gP@0#t*3N}hcw0P~UjvQu;deNYO(eOQ=@4D;LY&>S%-Nl9kfVbP+q%q2l}~=;UlW3od6byIYN~q68cfk| zaDe|at|5+xc+l%xuok>(wng(OBfeemQnItRadLbzn05;eWR}(CKB#%Wh4n7)(ka_U zT+54HJ%@c!pkvDKgr@Y#pkaZfD0rS(u7(bG_wA)?o_YZZrwr)T%Qv@J5y5OAD$aUM zJFNZiA{MEmatkoo&jS~t-|EkID|D zl+0LFI88IEH}Vtz?SA00l3W2xa!%A}m@hk_>?Hk8$>?jeZ;!vOWnhv!AxD~Q;IsOd zt$+|1i~1!e6vlXG_J4XslxPF=^>;UG`aG5gGi88|j%Vi24y-WQ!ZIjRcxKB9?Mib& zK159?Fb!j>H;}vOg$Ey2ibgxW>~{QX1k7whJNPavJ|Cy0T5_R`(;t$aoFAr~0nDvf zD4E2EN2Gs8fG*hv7RJf}u=83lI(`1S*@KsOJvG(Rs;6{|IjuE}-F-RTGapL4IMtzs z_swT8t71m-N6Q&^L@`eyDmji7J5c7EzQnsMb}jmi#M(z^Dg(`rZw_Ah} zIjIDdR#J+o(HoTFFOcFZG^>U@^j1O2xKozv4stqk0^V&%$& zF`eAD8vp(<9(Pp#4CEJrSfwAvUVD8-oUDa|K2m4W$a%()O=@-XPM!4FD((#gBzhC}P};O-(NL8`)jH8# zP_eZ|`TJw{A7P}F@tR1v<$ju)uG&nR&@ATVIa8-in~#H)#CnTw#l_ogdhb(=#XGww z+GtSd+) zZUQ2oDAcV8%c_X{mZzvJ22C3_%DG^p>e-1;jEGnebly3Q*FeY_Wk?!u}dzWHqpa9BMExo&3Z3oqb#r> z@6m;S3O!V06UdV>Sa!Ntdzq|9Y=eSATOvrVU)ywBqeWZ zr6wR4BrI(e!U{=@59f0`UAFZ@Rt%j*W%zIG>dMDo(T3FvMDX=2WLgxlg z3r`qpDWJlyuh*|WDbeqTXZ&|kzCW{5nVPm{)QlsI-X3(QvX7hCxePF8qOYnZDGLn3 z>cwh4No}zxuWp|KP8)tm#Bfwis!*4GDQE9wxE{DqSc{yClAHs}*jD6ly1$`e7S@1a zDAZdu+;(+q(Wu`0=I7o$RPrnDO!}io3r@3Y?=`dWlZ(zr&pPsZa_e4`ru@2{L5-4# z8XtkyB|)a)?H>b0sSi_W%qmX*QT9Z<`tab~klhahT~#YVDbLYI`(%i>7acuGUFIcO zkrZRJTI9oH#%6~eAF~MYY6c*qLp8D|KbsA~*ilcN-qgyn87pRpW-$#2q^KAki0*23 z?441_)Dwr!jU)13qbYf-PZz1RZY>EKp!CuH_%Y3r%4|QSCFo3bEOE@_XKBrcg20nF zeDPSQF13dQLy2hR`Wd4;+WdapJG`BOJ(5~8!PSC=aCnD;9$S^uln8RO!0-u8)B zR`UuZ7oS*)*R-Ua)#G{}vuZ%GV}{x9z_}1BM9`ln{jSXsfdNEbH@6|YGd#kcFzLOz zb&YbOQYuf+5v}jx((GVY!n|(z>k=2bP^-vV{(W8#-G!-KJoOtlZl&sZax}K_Z-s?i zA45OHOjN;Xnam?X19WQj&}G)v`|Zdn+fLw;SA&sSn31p3Piyr<%#6a<0&L}Z21<+A zsr6MD<5#a+=5_NhmL|y)#J=VKOLo~HfA8M9#nVs@)$}_VH6&sQCFy>bxXMTU}w5WRw*>K(9LFM`UAhdmA+wxqNpG^9?62Y>KT`H?tB`)Xl$W zA*3_hz~U0_+M24*+EgM_`I+&Zu&n2}vMvq~kauU>g2LirmVGp2Ey~i za&O)$TpROc(KKC4Z1{r@%jqSgkYWh=G-75bU^^JDgpmX68fKuL-;Ccw;%;g__-#jy zm8?Tm#9t!JhE%>!r-!#}VHrmCq#a`W${{^tYd?ZQ|b6+*>pX0|8+GR`dH#D1~j&Rp*&648GQ3ij|e7 z`oHWNAqtF+kYEHJMXnVe$^b2dwW3F@jg%#cF5=Rza&a&Ew5Eg#A%gCQ#GN+5ns5Dc z!><%>K~7fBry-O}iE&)+?A#*A(UnQ$2x?I)39}<(3>T4|ICUtnp{$2i=FXu+ygJ!f zLMUX%e5BrN2qm1ZiCZ-fFsaszwP|`Zz5eUtIj=ytNkL_~(?W(@BnU0RAk@x)1IdnW zgvTt@oQ%}p^mL7iGIz}uDin;@M0S{%oBJd>fR(HTWN(h$yR*9Aj{_IS|B$S@t>G5O zAH0E37kvS?7Qo!(Et zl>I%e;M+GqT$uNq9K%|s)u+O~M-SI?7w-+w8pz_MbO)br%dbDy?|*UN*PlZw2OP%Z zg>gMUb3Bw^=?*TdcF$k5b&y7}@0%rs`bmv-()~ECwjW56!<#c<-E_lK7u2}yOCS>q zjvN1OpgbHzR3YLJDd4E~$Nj1lVodmEYIkJUgy_Q%Q1zNgjD+;-2xVP9fi#q)E_0v% zqtQA&JOdEF;J2L#02gcIb|*q z%n$5HIB`pY`5ia0@|M;@tI1*}b}p0Ny<4UJt@PpKrY=q?qBj%xx6f^KYFAB)w8>~@ zPC}I%5j&OU3V}uUJ-5^=%0fKOs4RTfcX5%$i%z%Stx#?w8rj#9{HpWkr&dw;Chgj` zJ!`)=7}eL0j4L@cxgmM+P3T4>G)+m`U1+N!@o%K+y}0OHi;@5dv^hGu(Dz-t_U&)5 zNJlpcq+>Jq2bE%LYT0{V*KXamCx_Y%KQ&c0s}B|- z_syI2ZD!5dw%{W{1YqR4MVY)oiF9gfm~e@5nCbqWm42RuYO+gxJ^9OXiefkLhJQEn>2a$RZl*+|h4o5N8KkUv;bP z0jdMC$K{wveR96*ktQk54IJUo2c1^Kx|k;wl~?FlI)NXcW0AK-$(M+!g(sW$WBH1x zSL>4XWWnl&4KD^GAyyjey~Z}il^`HVD!fyP%{0xKQ>Rb2?{h{xx`I_o;19_r#2`L(kY@lsB|K34?hYmHVS8r%& zXL#1m66*%@=JJWVR%DK18`oLC9;gwtDEa;Szr(IDRvJDJGbMVXHUZeHZjzE=c_OoJ>ViG@Q-AkWvEv9`k26CK17_KV%o?r_qYS|H_GKb+oE5K7Tm!)Q# z{aMH(s>@PkmYhTAqb0IjImJ1@5ojpr;7Bs=Dr{T+t_is!!e>(3%*U#^4gvVSxV9kl z?ZwZz9PCIU%n6Hm_S@|N!e_=F(C5Nm=LsmRonIr(&F{ofBE9AXY-w=y$5@ptEMZY(K<2@6DS{`lk>g~7-wI1&W0c19I)%7TnA_ldf4Ri&V$h;7%RW*^e)b;2A=&6Ij^zm z-0gZT82HyGU)M^)EATRD$JNN(T4>2k#{3+GLs{awb=BMfUQFo2k4!Fd`1$P}lIsAa z2<+`mzkP;(1vOa`^DOk~VnnPwHFdOI-9+|7hK_yH4P3{x2KCKGEyLzelcVGJ5SR=Z znC9jEAyjAXPpw88!j7>Wat{1bYhl;f9uJ9 zwoc@;V72rMn`&jXncG&qBHufAq1mw&NyY&S$IqO30C|dsWzjKIUjbDx4M=ug#yiZ%g^I`YwHASz$%QHKmTZ$TAMc8B_JGp48`?yoB3cys{RW= zT7`uI+X~Y_PzID7`9aIldiwMrJE#a@mxnK2IF2=Wf?fG!w_z|5eU$7`3iN&DiAI%Q zCv&U|!ulcQU$+}iS1hV4szB*`A+P2f?HD=@_}`zFNy1a?i}HP&lo@a6f+gcxXj>xU4xE{llbbt!px$Hve#q7M(wAQZraExR z_9dMvXte2$+&+pngyECgV!niIOJ!&ZD3bsk-#qO5-NBh0r{tSW2>^NPlnq%)a zR6oD%QWqZe&pU0N6FH^)x_Nl{sv6aQW{{AE52txAUo*zWvBUMk8l?^{(uH$#G^CtJ6b?QPxu0!_C%7s>VTf)fjIE_c zwE`-8&7LU344xaxgjj9wxE_vZh;oT0)S08=Lf7*Kz^J6GFfCR;w#kf2y_fU2+ZaDL&czl5X5f;( z>encw@&HpgvXhUFcF~U~Q+fOF?TwN63jxVQVmxr*-5F|8ogfC8uX9QtQI#kv=+ZX_ zScafk=wR&LAv^p2PbdBor9^Y$(jYK0!o^Ef6c>m);#tq;#rgm2ugAwM%U<69cG8?_ z;Os%hu47S$d$wF(qzH-fec&Od`%7h4FIClOO~|>^r^gvIWqp`|-tL9F;1B3NmZNV< z8N4YZfc^{3HH=HIon4lWTDLA%bd~EWw&jE&;8Ox{n4T`5H~;Pl96>jw6LE@^er0Yx z6$Fm44!N}c50NP{O6I)y*{MaGIwg*-^bG}+hI7#(wt7q%)hsG}E7D8rSHhayQ z;`Z_TPd!MLhb)-n0asb?bn!&BoYmzjW>8#4fV0rSdCP z<@CMEvr5TGesQSPP4BN)&rj%oalqFucED)1zXx`8U;C=rscR)4chs3t6+`YLaW~pc z-hB8(1S>mDAIwXCNr#k0^u%VX8q^wEBL5l0AknWU)(+OhQoyz*(^ISiBE$wc&a0_} z7xBwz#W0m~J*>FD&?)*4dKcPr|7TRhx%zFDWF%4(nLQ)G4H}^aR>m71eqyrCk+qnC zzF7ZjYe&uoQ31)^_mJUX3xuHhZ<{J-W!n4P^nB!iO7dLi^NAF9FxWXOS5CH?zvWIW zYqYK5Gl`W-@qB1M%lMy&h-(~C?HEo< z`JL(B0HtPX-81rjK(V(9dY=k_E3$Y0U_KZm)QPcm(@l80X}k74#6P$2Q{j)w&~#=D z)0Ze2@Cv?5(~s*zu^?JwwiP~YRn>)cMbHO8gp6%h(+UVUf}0L-*w0L40sK?o4+F>6w!gG!~bt?r-} z01&XJev|E)eGmJCUE`pZ7}347QDr~VJkAGY)KQTfFC1xg!YXh1;6Yrxhli~9WEJ9T z33mB9zT*3Mks9D^PJj9G_vkMlxLp_xvWw#@hRC}C0H5;0qq;~$+0M8zK5t=hUWo;g z7i+yX_5P=>Y`FU$x@_y*x_$K@8Pu(^u5=&Yjj+&bTYBq;zgh@qZ=bWux83s4)y@Cy zI^=cd3$N-ZB$bOuPMbK_M>ePsM7COxKBf~a>4-T4OCi#wepVVRZ(jjG0Vor5zg7kV zSOEx#63N6Mb+ygx4K)}Wv>xPve#-Wz88c?w6MQ<8L|mS9so7Uoe%G;8y!{Pyt=&i+ zIqe&MiX@Mob%ZLGRaBhS)EJbO;)~$@YPHwL>*Q=izII(cL12>}@!YZSk6#Pu0ffp~ z_X3%VeiO?J-c2&d-8iCEq)WoPT+W_XM8JoNge!E>+d`yfZ#UFGTXs5SHFNEBMbg|_ zYfJh2m2vUb3g2#}(;z#Eo~$A}U1xGJz(Dtx*`7Q{%qz^2(;cUgbO8%a&(@i8!or+& zr*hu8#%VxH`-{BeN!dMg`z6CpP0eOl&w0kh%ceW0Nx5?W%LXXItdx5^lp=#3QQFK3 zn3HiT8e|YUxfNgQ<;UfpmvaA0BSPM}KRTB|T&z^k3+lf&M0qeCC`FRWRDk^&qAfW+ zE1>M|6@Z>iw0H&2wsZ_ydu6er9SQOFlajlCU#-EuCGG&$&s$%{d$JP{t`m7}HT_#W zJ&#Ze_usc3@$qn}ZEB);#D7;Q7X$CeZgb!} z{NjIlN2i4h-A4t8Bt~RJRCDi@73FR|*?|m?hEKIzYoUIVQ1kSXec?L7$gIu}ica=7 zdF-@dLkCV3>B2|CuKX{QECDK(C&53qlip=+i0g+VDziVM7V(o9q_D>E54e1w%|5sG zh#Djv0G9mG8B@Bb%CEiON2`i^a{j{=pPF($96Hz9{r-4D@Xyl|27a}zT~%BB;<8&; z2d{-1T~8do)3S?`XouDAW@iOZZw|HdOmdE0eDlTRmow0>@$rwe63&(+8VIO5$D+oI zW`UfHM3jI+$&kKxZ#PJYT_e;_y^0nCAF+tb0?kCA@AK!+VHWobu~4OT zIe>;=IJ^jKKW%Uj_x-kxDC(@shb*su8Y9C=K1nOI2#C8RJk7ep{K{;1O<-zsgQl#j zMqe1DuUJZG`&W|Wvc8mm{(`7#uCqQd>rdNQDoFfxV`xigUWFGO3bP5=;2mPTfUMOb*`?2Lb-@2HuX(6WspTWt4b-NsI{YMKD!(EVmQAoN_+l zmn^v8+E`q)!?HKySJr#8SydRIznGNBD0HtcGz;|JAg zL;WKO?#C{mr_pwU>Oi4$N9UEoEOG~B) zM~hlpn*d7SE7?fvliLdQjYQ{2e_xeEZ8w{uCA=KBk;ILa9QXLuZA0m&3pwlZM;|RX z^~&Sb<#j@O@{yG{WgR!r%lYa3xBoY#OSJd9q%MlaJ1WC+BeoTG_hS)Ge zB-doF{-rctm11f3L?~fet@*O|M6_jP#R8sg5wpl|1XUjK&0-1w4rU+i7>N?+{jIxY zxEu#DEFt_!-24a#j2xH;i`A*Kg*n;CNGem{Sq%7z+j5DUEE??d-+PIKk zOKnFMIHd?um_*XvFM9AoQrOhV#$nq%XJ9pMAMwzmqN6*9x|DEp;I?4~!40@d1>401 zqqJkv$QBTSZ~1HeXR-v@KrZ`<<`s4fy`b+rd!X}d^XMDgtHAD+5|_Y0i)VlL8Ze+O zbA280QZcL5_~Vp0vGaBv;JfD=55maumFz{UZGTvEH=uzHG&L#kkI$Zs>amNqr zO%sO#-f880&YgxrA$V_bb7Ss9 zGZbsXFGNcXJ9f-;3D3G{gOZXKR?O{K>Af;t3RDh<6@B=frS9x2FXqj?56KNhAaUX| zkqP2o-J?XSG=}sz7dx932Fg2~cUM-=it@b?-n-aaM<#;x75D{NFVdUu2{AC|lb|vy=?|>AkPEpRWhk~CtZY%7Ozc1b1 z1pfTW@1p0X@r#(?W>@^mC~I%3XrfcTlTB5BHZa&uSoIu`DM8o5a@!O3AgfXFCc1ns zb`P(j096CGhMQxPZQ5bL##;57US4VY1occLhjPQO564&kJTz_URI}56-Lf-~+-(aJ zMJcU4W3qNXavLNzX_J>@5c4*YRqpasfCf(vy!q!EJa7U@k)Zy+ASopgc)e1qqa+qd z)1>=fP|&!_^c44uWEyD;=@(wlE`K-s>zRsQX9leK+<~{tcKy(TJeyrC`WwAuNsscP zWff}izaE{DZSX$1;s1@+MGu?2Uw&$xl3&+spuUj|^k0p&OM1lWNSKv;rFTepRQJEz zU;o&a4(+$R-2MK9$?&}TOMa~i?aKGmP_a-Mx43Qr^DUZkv z%!!vEt;~}|JLw-Q`#2@_G=C5gw~-{bF8+OxN|o~D>C<(O(~|9Tg!xT6zuet?S-60` zJ9q4O_w#9ozJQ2I0V_h1*P1t1-z995p&f`p)F}wGxSMs)KVbUCRLgK6&U%zCK6OcM zG7U2JD0dun{Ero=HU4bfQ!2eusx>`kNo!f7qqE)!%1pdXy^5{?kV?=r2gwN<2s}%Z zCTU}c`(1fnz{z;MlIo)j*hB)&tvI>IqjTF}<6K@1Rcj{!6HI#IYk@%1e-m zGJMKNn_?d>3*M3tpU5@AH{bA9?WQv%LlQn2qCwR6=Ul19P!sMR-R(a zT-WrQnBRq_LMFhbjDn@1s>*Xh)!6f)W8c@4Uv{<#-NLl>+rsy_>f>I^w}trY{#||G zatv12sg@XPmc$Nty`_G$e4hdnulC(q@e@|vYiCgsxmsm@-J|cj$KT}fP&z|)J|%^T z778^~L$ie*n3}~18adz#oucqzUG^I;d)2t!_V^CQ7#t9dw9pAyI(W3)(NsvEB)%;K z%}+?sgzt~(ssxaY5KjZh&UJS~*=xJ{(+ty7fOj+{Bff!Kc9U-Y@FR*dzjWl^nL>JGaB7suOTB#P$p+&)2}4#6OcQ*xC+>C1F+qdt7^} zn6_yI3qu7N0?;Q~hk(tzV=e|A$u9^B@{+~Vv)OaTqHS%LqUz`VX3-9H-vPFRR8Czp zeB7Ei%F;F;bG*jS8%&f}y|Tc3?f)9AjDr6V-4^?G59WI93LX5HdWSK#xAg~(UV79S zVe!*5y-(Fx^s#Pa-|Mwj^mMWyI$fze_4xH5M9U|TyI1K!N2d~dLU?8|=@elZc8eC72QOlKiwqwjZbj-Mt^}Uq z=z(j??f~@22lB-7&6d)~XJ+eZO{31zOzEYpL=3@=RiBeGmLN7K;SRT*4;iw)Zso#_ zHIe-j>CsjjKT5@z2)>Oot=! zkECn~(o1(I=_+K#dIJigh!l@D^uO{sS-xsj>p(A)hE1A`vbXOpF>jO%u>>&?wyCfL zXFrqsWTP6pnfi)lNrj%Zf2+CAE`%dl=AN{9G&izuG(mGMT4kRNDz(f~) zQKLYI*<~jyY`Ab%Zi>afDZ9f?Wrn<8pra8FLCXGx>25c^V6Dk!mRF1KE8<@wCkXaR zsD0Ero0B6(Ebj&*9&k7tOOz6vL!txKo@ZFzkhd#h$&x5K0c#X$$aL}mtow~budaA^ z%e=yqs6ye|kcAz5x!f>ZNo_Q3nlo6X{e4WtoCk`=%nr}viAmondJuNF-MwRcaBYVw zOBrzy2`CmKgU0XPzyA+L^dP^C_5-5nFVZB;rgFk{n?5;(%HFNybJu8>ep z-P3z6ubsNb4_$MdaQ+l{j1DW5p8cq_nNKMmIVEJP|mDsO4!GoQU<4 z5T6ZuH&-=UH+QOA>Ku${Z*pO8oVu6S*vl$y>KL@Sv*p%Hzq~Kik}W^=X&tLUDNT~t zJ?l5?_V+Z+<-Y`v>>f8Xf81EL>T$%RZ@-*81aD2F{k>TP8DnVLQTVm?A6` zR~2gdJNrjB>YIw1nvx^dQLRSH$XL-}3mtne{^f$g)MU(<8!0J^r$%3n zU@;itk-tJiuiH2{IVGWJf@NB`DdoK-5nk1cJ3O2H&*F}SLjrJDFbjwvWxoTKBf_9q zYgBYHw~J<88&%b=>Uk?=O9FpMBQuk}+Ry|BpY!v%xL+4UcMQ6CFDqg6WW{gb)c>=s zkVggXG5lR$9rs-}6CSuLJ22=J$97APD-(zi%DZQ{Sjgj5K3M5liBjYLe{#ib5N5;) zF!ppr!;e3(ytLHXTAiLo(nQFv5Uy^;4A)t*$76l*X}fFRktK=6bWV>5L>YieBdnm! zkQQtLa5^&CZoI96Ow+dd84x6|eqCD0#K2nF9-;u zS`M`1n);7!Q@_P4e&op2ESAUUiw6S?5MiSuySMyBPJ$=-V&~u;4oj9?r|A+ActJ3l zH}Twn8PrJe%Fj&8{kw*7zw|7lUw33P6lFv=cofq19 zflB&XmR96m*MkceO(RI@;Yp=k0i&S(OeIR@MW=fQS-Dreh_GoT<#%&+>o#pF^ES!r zL;_j|qt-suV{(KNznnBXcAbxU?aKf!_jFn-fhff1Ad9bfw_~1_x>sKO(EyH^S{vb; z$pxq%9&K0PoAMs^MHky;kRUzLqrq zw(OuuUT2gNx$xTHUf)(^{ccc=T1EYD!Irh8%OnoH>-A&xhy`hFF{z{vS=hXzkC#`@ z-|W!XdeLaayEj?`Z|O5_Z+w?--4K$ogSQk#dd!NP(DONzHk~05tJJ=97M>j5A@06}U*qMK3Wz0+19) zM)Lg6+dl!i)W9o*^M2M@;ONa-}J=QJL*C-HJOx?&!Qc4U^|l zUKoNsN#rMps;(v`F4+6v{t2S#=3t27;P=e{qa#FTUh5rmQG$!H2LRaA4S2@!{HO<` z(TH^EQbUTSFR}<$MvEZ( z75t9K^M%AtbRbqxPWqYFsY;<^q;iL+K9@*s?a&KM;}gNxIoA+%J{hXmW3g)caz$L@-Uxj?9@~Utg<5oJ?8Pda-ijFkmP>_`KSrqS$U z%@&S@{_JBl7uV=LJwLl~ozk||wTLW;ALFV|*soCy_cu0o+k;SYv}Jr#?N3MTJMP~w zcQF$F{VUJAH^(G-BXuHaUV7gMz?Gef!btmve_~P7X5#v}F*;|zeYF@SQ8wDKwePkK z&{86mwm{hI=x6WeVZ-8LP=+5eH>0^`8^R*pb-;m#1F#exdPA2mqet+7lB8OgCRa`RBM2fLM8-w(c-li#7kMU%FEG3&}o z`UsQX4}k*h$=wh0=WDvga8x_6%#Xg-|B{bSBS{)RJYJ~+PCwX}^i5`i$}sWigTH(m zDCu{V3kUJn(H_(+`EV(Z_f&o-E}Xb;Zd|;?w)^t#p9uCL2dMI^?=t%rUeQ0ch^Ici zL%+;NH5PB`?NzWc>sO=dZ~1O!<5s=f)5})U-A?Z7J8$rz3C4<@GnXFyyHmS1%Hp)U zMO*_+Vv*f$F#&~WwZ{}9-a7%=SRPkyOEqsJ+@Xl#?_QB8RncokMD)o(A?(b+DZl%( zP(Au=j9monm(OuJB3sSr?&vudO*OFCF0O4%kw9jfW5dy}@}~sn6rV*E!dYhpenoUOYP0 zfs?(>j@=LCFMYavt(t&ElsP{|>K!^oC#TOdeI?Dt`Qja$qB{Qva za@d*muCDV$#fZ`=ZFc4`A#5%$f5Uq|AgZbGkU%kj(|s5Wm5bepMW6noV*Z!A6~P0F z%M0TD&u(>V)2hqXQ7@hx?fiFrrzfeG9-fMcON+>TQ22Y-=T}3H$2lyGS#`n3e2R&k zpXIR0&AZe#S#8&?w*Sxa*PjkHy*4C8>#OTfjlp+gU-h-KRU50*ciQRVUq2d^zxwj< zVJjJxBaVfKfBU&fEs%@HH0L&#Adls(Ag`PK_5=n_f4PN!-7(j!rO{th0U}&%ttOG= zA?aIKN+$KtTv3N#19kl0k5+Xf4rl{Q14VsnQ3YjmU{YiQSjSVRPEmj{Ub$x{!%c0& z$VH_lO-<6JpCi1IVGPaPiP+ZB?1}1(D3$x|U}5!{JU7gG>!Q3_g*3bc><%zf%iZw(eT(w4 z!tpSBFXvi|a34ZltmyMju`eDMzL$@W%k-UVVqpI$eWnk5nbosV(2)Ie!Z+ppETIIU z{gH>XaG=7FDAE{E$sQ%~uj^!xVh`Z#CF4D<8Hr=E}vVkmWdBv@S{+`g1`7}&4hf>pg% zxPcbyA4v6=y3g5)w3Gl}XwXWtm!4jz94hWwjvJQ+>my#Eu=29QM~DYrY#Ta_ zf)aUY^)vMH&=TsCh6ohL24&L~o$9N+c==N3AMr(o+r4d!o4b!fp|!K(S4B!}6%va` zXXQ!nLf=e#N-Po5%LU`}kcWsz^{&KA?*E*{^ zDBA7!zvs9xS{sh=h+4cVeP@Hl1P^z(MWQlNy}9H2WS9zsJ~7XwWwW#I!X+C z5`3ray2d+3*8-Fx8e(}<{VJq9z2>yXeH+|(a_&VRL}!~OsEn=u_3EYNdcBI?{Ex5m zeQIpVe|xUIY%=C`T9dzXRkBx#?=$zVAM5(I4QXNfdk2T_%aMx?H78#A?DcI`c}lc2 z`Y5xM!Gp75e~}l;-itq;mSsV(^OCY)39lt2Y@JS1KZdc5>8In~1ybl}d9-9$E~xnJ zQ4w;xm)X%ux)FfW{JvfExpOF0J)6{;709`yz!rS*sF)cQw3PW?gwqJA2Pk%t*?<{N zXPtWo4UN$VVQ{d&z1Pd+ZJFSqX|uqQpOEa~>wQy$Zdogvhra%1m2D%sOu}TqG_ZKi zB1O9e7SU%hYOvwl=39B#*|W_#OYX;$5B=_o0C&Uj^yz_t&4-U(pgkG>Nyg z_O)L_ohim>l^_SaAq~)$$fNo;bGQ{+FlqJ(6YcdFiOG?SJkaNk!K0ej7KhS5>8s#Y z5BLYn5+N|fr&asWGkU4X0p={9DYFGb07XBFph+oc`D^5Mu(+~f-No+r?G^qG$7#fo zJ{joL1Trfi{CdukC1zH|3p)7J`Qf^{DYqu1Sa6Du^WvU!!9^DQlmRQa`qi+h=S=jP3;=&9}Sb)KHHc5Pb;I=Yku!LUoT zLI?>B5ckMe)R5uL`CzwHo-M9$i}CT+F7Va5?hAz@;al~}PDVmzsXC=IJU`@8<{uyJ&veefT)?9Fr|JK z=IHhQ(9iXMpLOwnE2q1e{~ufD z0oU{X_WxwdiX+F2WXniJ_Q)t@OH!diDyxVh95Ygs*-*AaCFx{Wq)f@Avb5U)S|oSMmxlB4RneQkpbBGf&+lTLq%_0LQEuy(k$J zxk%o(C0MH##{Q@rOq18{K{8un1grfDuaNMF<_8Wv6>I*qPd-z!bg?8Sz|2%uRn*J`%Y0By^CE96CIKjTf5};mK*psvM59(N+^_f#= zvuGmVk5@XPR+A6YJN1b|9pR3X9KuyZbj*{g2IkI4Jq8A*|F*C zyiXixUaN*I1?InYF5344O}{eAzM>U#wp1c*skyO(hcFWeqW(Q-H;-Q{VN=vVt4Dt? z#jy48pGD1!+NI57f8-mtzm65;sHQApF))QrhTqoDel(9h0++#gH<&7&VEP`9z?~iH z4P0M21lCM*WtgcaefYpn>w2nQ-cPqJx;Fw)eecoaNb&zrQkBh=_p#Q<>(tqdM~@n? zy@Q9v*!bG4(F;)`05h|5nxB-BYfkOC8@mb~_1}Me3l{!aX-|#?nsSlN@vdZdYxA>w zsiw_o-e0Z!kzRx6_pi$Fit09J!0~Nv^JZ@Hm&7q zJ!W^;?_CxyJN3+C>y7RQdj(ocAdc>!#W&Qq_Itbiv-9>b7L1&I_4T(S58bXivZ>gA zWiy&c+WI^h(yU!OFYG%_5!keJ;8Th%|EPj>Z6o0Cwmd-|WYO)B58wp!0#~QsErN`n z%M^~T(bp^>FfbnZiJ~`p@Njpw3Hl+Kb_r-RzBA%ILAGP~3Tc6D_tV0Pd1TJ|il~Qc z?G`MkLuYt{UbXH1O&)=SO=+Sl>fGoHLpq@$oosrXkK#MH#2zp=H}gI#oU_LoMJ6oM>xam)5RR#~Ngn>^_>C+DKn`>+}n9f3PT0 zBzRzIti4#>`#BLQOGrWKwC+d6st8Ha;vy#L{zgZPGBp({9lUD@&2DvmU%I<+f}!C^ zPT0WruV0Tu`>fW-YDhsP@3sJn%#Fw2qfDR?zIxAH-9MP%1=}IIx?8vi4s0@Zm=bRZ%E7rQP_R*`KM-(Hezc$?_|TJlGM7v9%H=PCMS*%;;8*2fsp zV3@t*g7WV{#6%GHC2h3xlF~0<cDaTZe0+2lD}zl+B4e@^^~dF0$;+yw|tN7 zYa2KwsXv4I9&$Jx60&C}y0|&6TKnvVT_!s|FMAJ|JnEbI#s+zAvI&~q$%b0k;0Xir z?c78)`XMKD8NzJCci$HFpWb7iF$RXLMg)9tp(lwzy*;~WL^eoKt}b0}1GrjQ47mmJ z*eX2Ox$(_gSe5xEMj`YLxi7k!WzK6>hRC8d*4mIz0@z=8q71lN?B{&8*9v~YB&Hl! z&ce=#v#p~0LtwS}IJw8UK@XcRI%2=sHNsM-`V?lVg%7O$XCKSWu9xS{-rL9N!0KbJ z4>zybG|jHfp{*TO^(DM*Kzpx)|MeZX<#^LfJ1c(@Znkf)Z|FQTGD{-vbJ@MSZ1tW? z-+I7;;(n(iYT>}fGIGWTN?hqSovSmt{UK_w%8H`(p0fJ{c=<){uHhHTiWe_p=^a$) zXb=E1tZ3s11V)O{%SPL+B_W^0O|)NE_HIDL0=(6CPDFVFt#$ztz2z3gm zF9%CZV&~@7WNC1>2Y~flbUT0|F1n>=4Ie%3TDnFV+ESmLbQ!irqrTjseMhVGbMDHs zW1gWz?7v4i)bn@mw5Dk|nc{5PuhV=+k9`81{m~%7!9AA6!V<;=P6rT1R!&2jCDSsB zp~5lQyZ2U*jRgz3P_!3+PCtA=?jiOYS3g5MLXk)-1)*#ko(E`Qnw6wuTOk?Lq|Y-~t!QmaxSbiI^X}8(mbQ9ZNAg zmXW;I_Bjz8viCDDrreQiR#|LKfJyGg%N;xu1SmArqJt!pjFHP*mjJ)dZfCfE!h&Bl zV#1do+69rbLUAOnm3T9JRQ+11rn!JVJ-OyOZ7Q33;`t`OW&3zo{Y|iiK-3~J5reLr zIq36WXU-Xjlg1xxL*R|V!ZpAfO`0%>3BrM${+Pa`IsA>&($ccs$&)9ot}JEpW8#B1 zjdNhotNQV2RHIGPR_j>%4kjrity2S~e(TOYgls! z?<{AArNsB%CtYk&)oUNUdUf`lb6@zwAp*oV*Qagt9)$Ft*pe*3$YOdvK#-(oKPp#k z(+k7Y#Zz0eOV_RmV$3T}UthVW(QJ?L|DeZ36X%f&e}rM_28mY`E(cI>-?4TSChe0Q zXHUTfSZvW~aY_vHB0z7CMW*IF+7BN-yl7uOWQ}Ti)^HoAq1=KBiyH|J^NmJ-xkGSN zD^@FhG{2`jm4sN=XN{B?inLF3T=@CJ^U+ARC)o4vOFvuicl{;)2t@2mmIxC3O3Jf= zub4R*=ES!GD1gZ$`}?yI;V0QB6cFG^+wCc4C>>qhd9+h5gU`|&t$H6gBR33FmfqDa z+t`vwdpVL)WOfy_kHK`o%AwzZgcJuJ@M1Ze!$n-z(RI6(sD7BK2I~FUPt_Fk;* zw~SyU@->s~XA>{PZ~tPN+1_rQO1*l!cb+~To=bpJbCIActZEn#r_*|ffN3e zCG%F{>z@ykjCux3ZadfZKU$;O1JTfoqRDrSYnpPHX-&FIS~##PE1APig@&fSDMyrf z>%=gJ5&twLL(b@Zm~#620q*R%4qP?bpwB)0;@}$QX}2uchb-b=Zl>hD?B3wVtIw&a zsqCSuSp^r$gqRm|y|5k;=>lzJe~R9n^MPxh?lX%_R&h{s zop*iUCd${P(iIJEcJX|Y!2+H!CVsUo?A32(o2@@!*1@1^N8W}EyLi5x`0w%YwM5Za zPI8LOK7d2=o?Z=Y+79R7x0YWM5lgCy`8VUMpF{Oll{_^Do41p|)+-sDy_HX2JD&YKy z{N64P;2fvSnw7HPwSgv18p##=^ywe7HEjLm)JWA5Zd6wFChy(s$RIOKJILSPw|h$5 zORPuq9WQ7=TZkSQR#VnlXTOSI+7N*8bXsMrJ%z7Xi?9>eg7tX4rnNqQ0KOeAFXKDktUs681rLTysK$1>#+oRROi(GetZ_H0iQ zVZ=n;wQF52^r97xZV`wipdD41B&@>#VBkYtuDVz+sW!jCs!w5U6gC7r(wL-&nP{G5I-{X@u4P(Y1qi)EwU#<`_sJ|h07$6%BR zy+EP~Ni}!RCUwo!%?$K_0tW?7*}b)rF<(V!tiZ0r3P21{1B|8!?cV8;uMN=H%K3Jd z=TH<~6m0O2oWRGM*fB^rD1c<@iTo9p{khRnp7fUx-{Fk~TZoGr#dQ(AEEwhroyk0M zy2{(Et_k?-yBy&G&!er;=sYv(hM-}`FQ5zk`u+RAB=UadQa$q1$ubh-mN9GQds9Va zn`G*O$m3Ro^Wu~7o2u#)Ng6^NBgzKmbwhWCD8)J$=`E_b>7y+p&3dtcW)z$Ue8M?Umz<}+I5oIIS<5_YIu9ix{l69ze z>0ZCB-6K1!V*jDjyc+F|LK}RHKQnjI=D)KV`&YvoZ9Qimum0!pY8~wdI9j~e+Q>6H zWALJ}CFknz+G_GDXl-PZlK`DjJB^RcTh_p^qy6P$*VO0*M1_H+#aG9=9rUTZG_i%D zN%86Tyqr$#f8mO;?qzaSi!-jYAqdq&m(C#qK7pmVo&;&*L7ERZ@52Khnv|eA;r8hKAAx zgscRnN%J1@;NHDFr|{FCoFW5OqQJFbep|96@I~Ig_v~M<(LI*YZJO^ES!;Q5$+CH_CAu@uU0sQD^6%cQZNmtlI&-=F z%F05Xm)mEEWp@fvQi8+!iB*WWq}gqoY{CbIO!>#i{}S;VfhoLWMF5j2SFVn171i-}m35Khl1 z4sEDegtF4Ig4^voBHnPeT^FRV_irtp$7ruVU@P+^kNVkq>f_0QTNS7k}ip==YgEZWH zvchjFCZFY2G%J@#>_bwoQ{ci7$33HpYVn%-YbE?@+E(~}(#{U`i6Xj4(!WUN?(?K; zw{G=-7l{FTq3b_@g%HJPFATMaL=bqfYWWAf00=B~PSH{dy5^`i;f(~8UcGviam-FO zGBCQ>u)|_%CIH1%moUYYhTB6J3{&S5lH{GWFnD@>dB z4nc^;2wN)Xu%iu*=bG8BIz1;8N`3y{$L0h1-ha(J$R%(YKy%r=LOwCz<-nPAQcj5mJGe6&)B z6&;FxHB5giQ@AA9vGO58mbX!pMG_TWG0?f9+~QqyqgjfXjSb85nb|&OmVQ)%x7s9F znjAkWowgNC3>GiAv{uB2D?%1$Us=&zLVlgDak;Fe@AV~!Mt2_d41@j@V zM+6tF;~IIw$!m?$*}2T%5`W0|ned1*ESsSMdfyq=uiszX4WpYLFFgp%*N9RY&~Do+ z6IfeuY_#PZ@d!4ArxQpw?8^^m>Fih0&3!u6#sN>bJ>9l7Av}mw<9a8Fu6Fs0&G&Vc zAztJyrq;UX_fG3WkN9W1b}hhRZ)%^M^-z+yiONC}XTI2(4g#{Bi1dHQeDnzX?`^?t z#L2E32X@5j0>U+Y`t1$eT`UKS^K3PP)z)Mbn0`!(^|R~rg}Mu)Sr}v@nNo&o3tfo1AkDgd$x z7$oEVfbZ+*4p4}IKE@{|T4U2G8d9{D_Y{VQ&y+=vuhS<8pw~eVjfVXd$yjQCJ z7lZXLi;j#k|GVX#QPFd?RsP%M|KA|%QDqNxf+nU#j~|j{b?#JyDKSetK9_EIkY4k> zQO@+jdd6zE$7^a7=*(AJcG|t+8k-8sliht`yF}6{5$j_R_m~yEf1ijb^~38^!EsNY zS|)BT8NFhKj(C-Yz7K_JDgSt@kGSGt1m3x=d=EiLr4GA5HIzeG#``M>Hq) zg9&P--N6>d9vf+45xDd9D@#mGHqT<*zamA_^zsvIg|Dzy?NLX4K+D}|ZeWEYOlQwl zfpg>xY_sj4QLumaZlT!pGW`AgMq(hNYwzf+eZB?t4dnYdc&K&QsZ({yYXEGC?g`=2 zD&1;zaNC6(3kaXF7-!9{0wrR3sC?x4*XPMu>4y=@WG&<{rAXXpI7>94@jxx%?bhzU@0lZE6>(Y zqU#cH8!Xai-#$t1)mwbN2KOei`saZAst&ZHBxH@Vw?AXzU3_i%GeP5m0s>UROda;v zG>r~}!gzts=0!;`EfL?a;@DqWN|S+j*(qMrZL#nHVPP%AgHNT>8$OYFnD@GFWLB@! zb$KuDP8vRD{CJPFU4|voMvPFRHH$B%F0P%Yc8-J>K`(&DOdm_O9z9V>k$>{`v~}+r zG7xt4Xb$r$a#XA%SYb*xW{`kJmezI1l6HYh<^|A^Q?xoq(Fwc8rA}j37DAgu=g`}k z!4eP4U?ITWDJJ;7Z{{Y$^vOdMWlIN?TzzWo9KVL;r%owHDuV0O5t3K;M zH)AqNk)Oy8S(Ea@nU6g==fhJ2 zgWE^u+qMR>8Q);at#&V?xFd2S(}&$;r$pC9tyvPKh~|ELahGT2qY+qAfOsbZS3Cgq zP$;-q%RM>dUD@E~0L4&c6a4?0Iej`UX**UTo-d48Oc0@puyOL|&3aWpTlTeJ_>cPB ztaPPU;UI2+1Mh+rm=WriwxArz7h63m|KeKSmM7bG;R8^p$14C9SrF+uNL5Be9z0Mg z>oEF1%#XVPHQ`)pA%E=QO;a=9c=Uz7WosYX>HNEQGnqdEHBLJxF($f~IPp+V8VKbbj{R%Rh*&t7=chlvt0!h7e~ zPo_Gp8*v;HB5usx(4n5*)r%VSlp*|<;}r^7%t2uXX$Qh`9i4ia|A~)5GnW}qpk2^e zoGuMurShKy0bPUb-=5jg$y&`b;Z1bpvHX<-L!?i>vcm9V`_gbInFzuVAby-Hw>QLu z)=&;MCb3(47p(IO3^YKI0PE_-+1D>E{`AQJR0)EFLiIXlbIBwmocx|%A{yb1O^)DS zcA`BnvPUBrIAFvl|6Xb;;;LI|MRcip*74JQRI?Jf2iT$6OY#e)s)LtmpU}`)h*V$) z$9;BmAC#jB_({GFs)9Zm+i3f7s6ac8OEnU)C5m%XRI0 zgf=jpH{(NOle4=HroQ?bo3hNZdH+|zI-91wurjIN;ogKT3E4|LzN{R#H+*pU9PQ_e zof0<|PD*Uoa<|8;b{6Zky+K3&ehM1ux)@AEO#8UWXYbtgqu|>YWXWPH76G>2n25i4 zfZ&~#m~yPcfHg_9i9_>nc>1H#RSTfuKsMA`{?W2l4WP|9eyipCf~&096iwp#%5SI4 z0uCO$&MHq?fN}N5Ma?opvi0$v3RnK1OP4MRvmi6!s_~mzb0!%%rrgQF)7c!I5ukfy zS@Cv7H##aj(=(UPF3vjQ!{>$V>!aHeuxL3W-OfeHY72r#^EUhWHdPXJr68TLVf0h1 z1N|NqGy!Z+(?+kgGq(IiTMsVu&yeH=p{%4HP#g-BazHaS7`bvtN8DdD$vKl}_P4kYoM zIlGwyog;N>%-FGsgNM4%S}~1vIgL1PF5}_aCX*(1Vsxda>+^(U3vb@>!F^RT9^XYz zQVbf#-{SbO^8Gjc(KBZ3yMKYa9003rUM*gG!4(=M@@JpVGxNYkPMpeX{ebeXRg2TF zEg6xg49XaKw($14C6?XzN69q~DV&>rdU_8wddwpgys=;DC}&z(-{o9iMg5{#FxDN5 z8A6lPHjs`GQk6mIV}}Ki!rV`Ag3q z7cZx4Uusm$u(HU9i2+}JJ6)#bc^cKl>~gOkvzotH&`TLk8~A_I`|S(oxT~z48rW>a z!2=CcvzMgDTzk?q|7D#4CtgIUA493Nr+-|?DoqV1{y}+(i^<&Cd&8eh$sR96GzQ@E zq92vv$7i8#7z!wfG4t@`Yn9FeW=#VQiE?Nurw+^Mj?Y!07^n?uy9`}7d2&-+nUY}` zaWb<|CTf}T&L&;hMu_&2u&_oL!L7k{8%cOOQ&*nK*do;Yi*adP2M~@o=YZV~)Zc_` zW$48op9&_gSbZ-q^zy2^Hj#@vBOM(XpU%FE2zoU zb6Mff#vTKYtTn{ppf7TLG(aA*1BM%|qn6gYeb0uM6(p7P&&*_gXhp{rVOTiiuFwRC zjLsFOCi3TTtmnCV->^;7} z4v3w&43XU-ijg%Q(EiE0A4VmRW-rN=1OS`KZYQ#KvcG?}2Cf8GykMTQj(jkeS^osR zT1+W}vu^?}Y5Lgoti^V*Hf457o{z^tlGlM0WSE_tW~uA{cbIk=(=UsucAO4?{sv4Zkg-dr%&B)R8&ly zG)aV70!1Nk3oHX;l0g4@jT^fx8|2U5IX)2u!AZ@tOWoXB6LccC2klJ3lW`}eN^U1u zEN)nbv=b{`VCV(QG-tKz^d7y(8kYnk4=2ar`2Ib6Zss302tUo0tk|~40K$&rK62s~ zqsk&x=G+yR3pPoOIWhmrL^sRErjzWx(H&qdVimBsXFU}Im`_^$jT@K~5kT_u&xTuD zX7^dI%23NBSKM>pz=yq^=YxV}{P!Q>usx8k}`!i76=EIN5(${-i<-6*1 zH9MiSamb8%U#537t9ZNq=Djnkg5EwD+HaHpIz6v08ktTAPV$=SYFx7{)I+sgOXfE9 zbzf`r@891#`sh(*>NexZp0&^O>oXVqf8@rjT z2tGQn_EgCfq}j3=K&H0Jp-3^n`0|xgdmF<>Afo?DPLdY_|A=#jZpMNt*KKr z0Lp%H>%}N_`E5|nSD)^hWN=%y<8jNN+g5XcXI7|R{>UPZceAY{>2=!L*7witx7db| zYo|$<=5!_jGjG|!o%057w_TpjIGAZ4v{1(=yangyQRhzjJZ8IWhm^S>(qy!{^s z2`Z#tJpTIiA_NJB_Oj~=M7O)DegFaia;l8%>51bM+CAH!#Jt%B$5sqQbksXg;_8=1 z|4fi&rlFq%^Ym8P6WfKEjXtfMlxk~<^Wj`l6xQY@A%T)DdI%X6DSu>~L_1q)%xgF#Xa!)xC9y_1~+{Sy*C9+UV=SMnqS_HKq_TmhYoXZls8h@C|fvU=LJNo*ey`@%ZinQ7cE|_V5Rr7 zx^B{+8g-l1s-ZYjw4aCG1S$LU2j7=TR(OFJmM0%5Ygi*=?R@w|L=Y(jJC)|rG^~y8 zbX1?_u?bOLe)+#mp0o2|lI#&X`Dz3Rtn5>MNGdlaZV z8}l;1)uu)HUlcczHnen-DE9K@{ENobkjFpg6bEh0(kqIB7BgI>x`O%icMiK9SpDk^ zmfD>)T&^@%wbssqz0$iRR6L8n6tU>m`j`d1&JE0{rGG;C!^&xKnH9t5q<6hid(D}g zq9dE{B&t`^(9Il_Z*@J;BiXOMQsvqqThp6agclnDf53D~85pnjcds5jHsactty{D6 zX$B_QFe96!2c6$KeK-(e>G!IS0&Nc%%0r_PF>0xJ5lxD z#isZPY!)AXI4sO06bmxGoeHluHv8$sL_J0)D3@6y-)33f(4_d??LN1!tHe|Bu&=j; zFL;t@yOuaOZ0b;N>B^@o=+}iEka6^M$aHxMRR~3ME4cR0xwMtht5&Z*SX$#<4&Ayu z)pX&Z7i1ljNITImG-M5*Wy2;-CM1Gneyp|p96_NF;;r1G*WG?ce<~;sL31~q-MXT^ z>3(@)qalM=oyfGQaxu_#H*C}>dRZ(^dfYD>nws&;pQ|0_eM*6j+gSR*ZY090G1U0u z7id5S#H=&^XEN(ErOWd*2inGKHXkYf`p)%~p>2Em9cZWoe!-F6-MxjXek6DpSHi@P zvj!<4BS-}7(Vw(M!GmR|yn59Q{*9kVuene^j(x`JYc$vOkyc{UQiv@|)6*98XQ-DA z>wdc`kFG`LyA81qaLVwO6RVGni((j?k!3Bz*snRvKlXQ1jAq>vqy8E(sfF^;86Jmh zu6C$*P)Ehra5+-SgQ?0Vdnp?)sB!m4X>1qI4X1yU&55ktw8Mz&k@eSZRO>o{``vB0h4lShIb{4$0+^dE|kFLXeLx+Dwb-_qWvk4Vz30T3_*bCWFmj z+bQMP%b?C{)1h6DjJ+#S*Z?-U*3%4Bo&Y3Cj*B-U5uE(DPEIP>{8Q@{F@+7vm}~4!Kb{v_m_vsldr4$`GiS4`!j5ZWTqdm;LbEnTlv*(P%K2R zVIsK8+Dc0$Kx@7W#a=*!;wgJ+EI}wmL>|R~k_~1-BbH(m`2&{8K;+7|Zt;QF#^W3{ zM>aXr9HT~^Cwk=x6E-m_G~8~viSHDJMoiiJ!4Z=2C~NNX@~(Z!MS3Cm*}ndW#-yS- zB;-v;=OOS@i1YJ;)~#Ew9Rouj0%gmMgb^BaVA zBcsLCD>(R8a3_6nC$D?(5EJS9|N2sVTWr@yR(Lq7FU(&`q2^xjIBdB@e4w+0|nayPCle*ELz^o zXCLos4&exvgcd*x);MKc^$Dz+*Rog}_uBz}E%OZyhubXd zb*VV-rFwZ+4JVfYK|5`bFW*~o_To7k_M#JEQu#HviD3nz1vX(Bh_fUme-~OT(E*^Z z4<$}pJ}rs6HedWWr8J%SU4kT7CZN_P|6tkz8jRWmzg)c4@0}Jv7?3mm?q&|e4j`DI zli1gVQnS0f?qWhSoWps$JF zj)ljj0X!m_8r+T}PTjq6_ihREA;RxHT_=ItjDLfSaU-(Gw324gT1@zS+34)x)GxT^ zF#n&gp?o5ECK@hh&h+Y5=mYuJ)?hDZyaOU7oD|688x~?Syve+So=R6@nODKX7V@AG7Vsyoz9M^*vj?3lc@G+1RyYb zJU}^xw50}X|MX)Qeo3DgX8KsSQ`Srwu2_-0oipioXLlI!ar84_PitfQdVFx4@i%DBE+Q`Q&T%IR=?thBl=DMp|4_%a6e48iDG32@z3dmg@o zCq+m^_~Z6>>XI=VGciNu$kC%WnE~jLF1I?Yhyt4gD-!kw`C+11)d~y!OP{8=0vw79 zux*n@jZ#`J2x?lZhTsqY^E})4zfdvOf`}vX-`}yFYSx{%1;#Vo=lsyFihdupbZJrG zV+w`H=m4gC(}E8j%yPVNMJ>T~S07=hn5jWsa<`P>7U(#9c(C(F+bZK(mo6W#?a0R$ zrdjgvuZ1R)rcIrSZ8OhYUOjl<1bcw|A?g}V&335|Pvl+D)MfOXH0)a~ijM+?Un$#p zudGNp6T}-#>~isUVg~sUy_>t1`{chWPYm0m^b0;;+etz=@b1^nNkhIO)TNt!@f3t_6y7T}*hGI|N)m0y$K`{3&H;PLk6?^JW1#rLkbU4l)HZns%9S%R;fvrNR=@-Vb1O1lX3%L1w6CcD}XLLayIO5_}NojW|91(YuF;He@jYU(JwEzIx>f zPuh?^0YzE{f|?g&_D9mpm-t+Y$k@1PSDya)q@0p*5M>|5XUOWEz> zi>WNivc$)I<*HQ^qz7@_+i#bGJmyV3-)oey_x-zfYeGw&CXBIoX@HUvf)b$yjj z1GwCep$4TF2VOoCYM-omiHy8!AIq$TsX+F7Grf!=1_gB3cKq8wcE3oLid;6Pyee)b zjH88Hp3R&xW!~p2%`p@KS6rRn*=<@?VN+=T&YFfuP@sfvw)SGlT8QB9JvKX9Onx61mK9;F5LBqvM=KBqTt}VF7fhTv}ZzJNN zG>#iPw$7Z#9{Dv!HVqBxUvPJ2EMVjdYQHF2^Pjv&4T=JaZt^-}&l|5_fr74-^DBd$ z_KD?IC`1(!{Gg|fylEoAU%Ph5ob>sUnXodtUzM7w@h5=bkTw5uxAoRf_e;)@4AP44 zD-E^KU?ExE4{uy?|HFsAqz=uy->^KoG#d;iRFqV+sK`0Au%@5+HlSwwitZ{PiSeVF z)}sF*cb?3v7Iik=cvG@ETs)YxF%%hoN_YJ+dvw!UvM8Dl0?QjNNY~eu6ejVKu{()} z%Gpc3ngI4!zZ5e}HLdGesLtwRrJB_~T&)@+#D_m6vsSIkqnoN`c_`)N@2UUR`}2Xp z|IP++ljoJr|9JNoBq~Ub8y?CctgAkKa)#s}Bjepv*qy(Ay<`?n!Z3E{6@w00dKa=$ z1PZhqkCcL>g4Yzyx25-@r6foIHuxsoBN#!{NfP;CvF!d9WO}eDb-^2BsWV+41Ox;K zc?pc3b~b4a>1$Aw(0cKYu-RfT&<^)^WPf6Yoet@?Y~U)Xk;CH$6=aIEz9+bvB=2@8 zFK94^W4OP!We=MkZY{}M)UGUi);q-k#F{;jyLEc^?%o~3J@WhI8V`nFz5JzPW`l9QHNb_Q6H-e%>mE8@#z$a;SvzA1Rx;FCj_aB86kZz{n$&aec!5zo*dGCnJlRHDV zXGVbZM(k7TtVgZG40x~Gc|n_KQs%OBZWUW7ZlD|KLHA&n0QXBcuwv2(XP+7zta8bT>&lj{SJm8n?UK&+-?)b2rg-w zc?X2U=3;siRiDBuuncHe!rpN31Tb8)Qt_-SD>0mLay;jt{8;*_f-G}66AHzzV78$p zUl=-MD>NI5XU(?m&3()}DRPUWV?5jL-CbQ^UcLOkPE26L-gdLgkXFHo&D1k3QfHx_zu9E?W^B}A+6snlt=988Uis)P`^%6KikH;4&Vcph z#dQ&s064yLdcy2R_mE)>`s!=&k8kSr;v#o_PfJbBkt?uk@LKDOxy^i>;k*R3BZ@G= z9CS{C@v&L9;6cMXd{{)mln^0zv`-#g#eV9}xlA&|MOZNAh&aKUL^THa2f}GdD?;#dUE!|>JlKQq_0z21X(p}>oBFx~WTt}0{e}&NP~^t>=WzH8{hFQ$8h!!FN+~HRq>~9iuoJsu z$jM`vD_=M_%W!}YyKrCkB37S|79Pp<>uub?tY)6(w}UsA?VD12T2-M8Og^(0-%3v} zSeU1Hs?{#hk)hl@LgAsWWT}Dsv7(vX{*ZSY1us8gJaXRm{^2&z($bQxDF)nW3`BFT zEQRd>68H7>r6fE*ytx`NFbGeC;imPkW#ful$)+qQ3C|7!7Zn#i0c#mqA zGNRl7^=bTe-dS;|Bs`*WNqh$0y+`G%y$@HdN-r;6|6>3QOS~InB?#x`b5}f78Z?MQ zW4rv*t0{(epR*&Z5wMF(#kXN|j-NSyero8`*!Aj=ytoIV9xvaXd>Kuyk^cyE5jX5~ zMunN)I=I8JGs6{)_>0vpqw6_oMgHi`sW3LQ>n+_r7op_fMM$7EpO1NTW?)5RuJhSK zZ9ij(;@$VEOkEcqJbc(qvNt{yTvejmigVQSw1LK=sY-^gO`#R)9IK9^l(XG?O)bsTuYNeEkW*(B`Gc z>STi!GQ7Msk&%%;yLaa#Uoz;!9=UB`v9s@^t?*)kRiNy?z8e9QBObs#g7@7_^HPu2 z$UHK+8_%%97Jy%J_x}A|vX+v_FNn7Ub+5$Wz`KviCaa_CqIrt#K_phf2GLdL)z7m= zPq9pt65)rKq;ha&?!WdxHKV*=nY$T@0k>je*RxoT?NFO#jJUXl{G#E*)d9!=^&3wd zSM)E_wgkM-e&w8m$^g7`Gur*)E4Lga3>Zf?p61n7d76xNeaSzNP^f>KZ%rQAOU*NH zP^%Ab&C8ftfw=Pq{htjSa>KCvdy=LAGqU!Xp@9aLc{&qjNpL9|Ba_sf1pcgMDWK?fYOO2e%H7mnJo^}xe;3C`Z@;FxTFwsez<%rUCt_3_h@Nz z6r)f7|26tJZVSo#)wiiC9bReMx}8mbMG^8u*!ODODb>QFj;r=w%%{U;@W^?Uam;s% zCAO(z-qshWJGQ|b@2qb*2LfB(c0sQUm^Fx5n|X7#4>&`3zi%sKA0iN^A4e&;F{Ny4O?V1KP0AB9D|%myhnZ&;QvY`f_%*RSF4zA|#+|n@y0d z?@s(Lm(N`vG5}x~&fHK-U*9itUOgqj-xMxM=3Jt?_I39eLV=zPtw|Hj;gk3)Xxwv; ziV*RBW$F_YSM)|8C@g|M&~W?9=7`-Tq&PFvGYaJk#A#z5yBA->3wB+PEPM);)i}-Q zlQs;qh;C+7kgRr{u;b7-(4D@Cx3G|)ddLC!V}P>un@OZ z)eoP=mwt`hz~859^Ya%(P%k`q09#G-38V6kPo(*3TI;U@)!z0lHFwR_GB@XqCQ1qrTY>$CQ<_bu|${$8oj(B;amoK!3(#Bu}9?g#1h0*HfbZWqxF8Gsb> zpW@AF*Xa7@r~hrBrwH{%cWpzu3lu4)o4C`{PKEkwl4hMNpQiNp2luKCiZXA6_9R93 z6rj0?0uTOsS!@9?C$cg65x(bh0-?={+?tuOJs;s{$CQWrvP1mS7Ceb@Z^O73SQ1oI z2?5Vhf=if`s6H2{c$&F%9SR%cSdp*G%O)v=-l*V75|Uln#%^l5|7pHO#$)nP_NI2O zy@87Ccc-cT)+|R%%Ra|1;NDS$ykRtEr@Hi+zK?L@Ax4lGwe*!MR>*hh>rXNm5h;@8 zE%}7P6QH6RMV{lPd=<%@RNr{5yd;4X`UL*-*?6OtCGm2ch4mfoMtJm+Ms(%r9HN_d* z*Dj2}H3tlsxPAFb%8zdZQN0Y)S%DA6q%ApFdi3NT&2YXU${1QarR$$Q>0rW@wahiy zEqm;K>A5LK3%?iY>&~sXHsP#;%N{4z89>5-R(GNgmz;AgJc~dRzDR)f@b3*9H#V}T zfIW8@h{|ad&s`nL?dlIXHNe1Sd1(||^F}eWh@@igUcxcn)-En>fk%;1Q33^`ctwzP zvO%|Rz=_iRbX+zp2OT)5OGcoWK}8>35pAaFj84TXhE92&%F22ZlOglfef#-@qumz$umli^y6h~;d$MO1ByYck;u*J9r;DI#rNhTafF@*5 z>cutf(A(12Pq(j^7Bim2be;wC5{1#eBgw=`o0NFhAW5_mf=5qKc z?5FPbw#se5-M{QZ&d9YTX#+rMS{A|-)KgKpiH^*19T@}0JQ?HtKgc+W6Jmoo^jmRT z0;Cn5ebldHpe0e&F*W(8WmpDsqX?o6m(vERlXdjv%c2wSid1LG#oYG~BWjYje#k7q z-#?C_Mc%Wj0)%p)`0>)%?KW6NNxZ_qDf7<_9;!1wPB8k?pBRr>>hMKBjGVLX^`ngu zPN9#{Y@`O7B?Utk4Q7qVxVuMvs>=)oVj#htAO=K%YUa+rwVxRA6ugMfIhn7DiVlc| zSwuZN@UU}v1H;b3ZeAf^rWB}6S{_lcxl#l&{aLfvY#ZUFj=JeAzzsSEncEXpg< zzZlMA&YuiLIlKpwo78n@K|=O+PAY7TYM5gKwKSy4_Di}lcG9)I1IE=F$-TGx9(hp4 zi`p)%QmgWEeNqd{$>!Vwe) zBxS6xA zTTw3(r-TcY0A#~GUy(bJcp%#nORn@g6ODgVw()(fEGtUL4^diH{9+6PUI#tB2PcyN z55!f5oTr<@5JAU}?BL-iGw(c%Js`6(R4M3Im>-#A0#;rc_?x6v(_|%}n0R1u*j;T) z*lBB=LcR&>i_m=Y36Wa$NP4(yVy3~FAt9%sx_WvV!-iQ&o*A$mFHm7-x_Y%um6<78 zE6V(P`&5N}$*MD{K|m_!%z`ZY=mezUAWXzEwe+(y;XZj~6I&wGQYgAvD@X0oLPsIH zN(98l>b9-BfY>+V;vRPMRZ^N6Zk0fS#&OXl$E=Ipz#R{}E==tG`&-DydvbNv(%T6S z)!E?8l_Vb~iBTi1tzTR#FKXSisVqZdO4<=coi+NhedkWucMaQPH<95(Jd~mS*TeD9 ztD{MAs5s=ddbN0vee2`_nh5E7V10qkdR9flPVLBLYzuKGGq?CgUG;%0_x$1i<9~cg z15ZipJMTAijp<|9!eWr2P8~ar+-hr}*-}WJo(e%hNdx97f#s>n32PchVlfJ31L$B}HKFfm^~}G|PG#;;&wSU~|BL0|j21R~h_&8Xs=}{OAxgXL4!(q5`?~X6*HA>Q+KmYwLi;mh z-rHoZ(J0Z zBEbeBqm8V~LR$#BAw`XL!iN0mzZ3{QKteD3*iZu;#k=`x&JE zYOyf}jp*ohO#J)=U2a;z^TX8Ff2UoK2De~bp4kWmJeIK*_FgM0c7@9 zhV7$YQGQJIoSzYQQ$XjA{*!*2{WAh`8_umJM32^f`}|z}n$s3sW=(?ry$u;Vp>H(n zxc%EMp$ia<)NPuAQRhcY1R9p3K;F3=g}gjl$^8QDvrc1yF|3q4q|Z@--_2c2XDiD^ zvdZ6no<&h5`iT3|<+b4tW)10eb%GZmVFI6pxOk~DR*%6BnVz}~b=IhbateaiCl0SH zZi4(FtNdUUZh6dh$`69+#o42)dp|#^6HaqjcG}<_x%Aw6UO`=fj63IYt3k?Kol1fj z;l0vpd81r8df|fU+Bw#UfTvHly}#P=OZq;GajRBcw0-;XWo`HfDGg~$_BU(OhBMJb zh7@7q6J>4h#EJV6)umf1C|8gPV?eQ zleK}NY=--2-u&_(d|&L838~W)v>Pi)$D$Lv?;dpTEp*a^#tXGkAz>44!T4(W_~+&o zy59pq!$%k_UHYZuI0Ufb0nca8=3l%L1U3%0-_acLYWmZPejTEhPBv#%>|Q*m_;u*) z$%L*k@w9w@fVUdZGNdd!uN?CS4PWOd9s$yGNNvcP!+?)Gw{sM~ghBpt9WY_AFWT?% z5~Pyn?xx{~EklHiQ>TU8AGqRG(}HWNs!DU|*@UVf14Z-nRnjTxLL^wOdn-j#6WyCP zozEQK4?n3(9>H7ISn@5?L;Y-{K*SH-d9)3B5eu7y$!2Dj-f@AYpJnw~AiGl1Xm+ivIaK55mc!Q;HoK)fB;VxN9@_8c zAjXNT>gfDg%mZlT#P8fWKppx^68je^{iZ7e!qppW%G1H)EluVFG~Z~tg`x{+mz06y z8lA3Jcx<~++C!pOe~6ss;r%z}_nn!inJGy1++>-*Yqv9SK9NFI@|Os$kxfMzKseIVH}0?I zBqSzkK)7HcZ)`m7!Y5IOdU+YzgI`N@f)Yt7zP`;2wXlprs0t_9h$GKlxC8g#>_zY3 zN<*t8(RZ$@G|*MS%0-#KD#Nv^$`G*h z_sUS1`|qV+heKuWycMmVHdhcTzGE`u8#c7L>ruR>vyDF+S)y zllP{ItyugVGA1977!M`_M!J)@-Sm@5vx&8r8fr494_|{rBfj{t3=%+FmRDV#a}6qP ze_0~+CL|n3QD>V$3J_~9(6S=1lH(yn9`B4@$=HpsMKYUA8eXBPa0lQ>6@kpjBn^!M zxv=Pxm$OvK1H_aq4UQc-E!>@Ip2X^X`ji?_$3Vg}!2gMqWMcdx)ABX1~m%HxNp~EA&1M*jv2n=2(yk2{xcR zg^U+|k#^OzjFT$={69I9Xg=nTFMrQp%hYT)L!AsE5QFqi<4r*$@w1<|U#pH3jg0Yz zl%u5;4{^?+D)}P`hmf74V?4sCXW#p6Z+ir+f{rvW^0fyUTcRu(KjxD)T!C){>O+({ zGu&hFneki6B}fT;ymC-Gd{2pmrwuj>v}^Xh;1+EOJl+e6dLKtOsje`35%tTJWw~dH z;cMk*3SK1Q2oMLy0a&&j5nC+Xfz~N`@$4!f-F?GEP6nOtp6-o&DlpF6w9_%Nm#||L z{hoieZ5t16q8UZ;BaCefQ+KT)Yo3v6F?>S)RHf^(+FqV`z}zmpIs*GH-5KYol#wpw6@a@#pcs^>D z$;69>o)x?sZ*FS>f@Ge!;%pi+c*)H|gVJf%rN7UgZ^pHTXV=SQR}@Y{rZ)4?M$ih! zkH^1kS-y*CPLZ%nkpK=}PSM1X73|sO^1z=tc%VxAbQY}pZ5vH~65!EwFdU)lB3uvx zB5;Z5-Y`RX4;ITsPqGj?W2UCzOg^|Hdmq|Gp0Lj0J$Z~R%AeLII3mO!AbqrD*L)^; zJ`SUx*$_RNWk`?*4N}V*FKtmLyszP-;A zEBXxHwt?gdtOp&I!2Gj~YHmOw)mVL4!}aTbta2--OQ32>jP>gq)gmuX(j${f!yZd+yeX8?k z*(TP$eMwh`gjAdi@I$#bN5GI1bctQ4lBsLxh&Y)8u^mTU*6tU2-4MYn88z zgE43I@b6=o-$@Npu5y-fK5nF?MICs?Jt71t$tc7FBUK>Nbc%U*VMjxUT2FJVV6EIc!vq`qT+a`?pZHh zu%OQHicignpRW`TsP;+c#au}Pm*pr`B06a)>CAJO4=D7 zKd|JXvJyAQC?JNsXCy7kl2nN0iHQ*7bmNlc67wy;jj^$@2PB~rnot_;r};^+RD(sp z9?Rmfjc7N7{F;AP{b+4ld7$wg>!?9=jdF3hu=8}X8mFJ~+Vf{V&eB_A0z(Us)%0=7 z{DljTage+0S#$k%;620GwK1% zC_;ZR_T%pBIqm4TNn|wA6F^C?W+xunL2?r|>6b3XbAa=mNsCB@z?gYauq?1U{rdMW ziE{_~{Q%>tcc$|;G(9AkjKW8Gyz<&|3)Z=tc<%xFK)CEZTVW?u#;O!BWSCDF=Lb`i zLeK=CM4*U-pb|Cm$cF$`BfgCe9h88FtQP|UAiUB(=|o_+%*I@v=DIp{H{eBFOIr4| ztwsjTI5xh^X|iwjCkNsMXVusFW$Z^{vyoxRD9E|5d#cmI51IP&lmISljXo{`JYa0F zTluacoQp@+k%GJr{JgTgTYn)hz8&eYZ%5HFJFA{rS_y<-9>0KwLDCmJV01&A+takK zaWc?mz{Nf|5}(8I_GBkfh*A{&5f`s0Zz*)+-_V6KSLR=nwpT;Lk1)6!bNh}^sD)>I z)W4J2LQSns)d{fE@pmnD9RSFMESBCV`a^cPGteSWMa%FhA7eTz<_-Qw>krYH*i{OC zRQ;n*!Ef2Y8cx?f1{q%N#IU^Oe30Q=??ZE%|J|ZNZ6X;6XzuLQiZmAqN>oH{lkJ7N zEFp6dqu#&AvP z68UewV;=!cj$y(FwO~uj1y;=dUrS%uV`6hRSB#N}$C0S+H*;9DXe3%jLY0=>wIK>i zBC3sz4NOeD6mFNFkG^7V&T6EYtjotqQ#PUXI}#vTHP0x-V=)%}9!$P|feWo>-L4w5 z4^|90gKt%q!nHoXj}Kz5Bf*%tWkEtKO87EQ?EdAYzMvoD@qto@KoEZ)!dO|ju!>d>BLaaO zK62#wmNKXi!39OUh;qw#oVUV+LTlU>0+LzcJj=C~hZH*UZRHQ`dh~!aPqO+&ny2hN zoqV$>-T#qga(c_wt=2$GLl$<`;10~eUOR#ke*8TyPWqC^pW63VJl)YxP%KbC8dgmZ zRIU{BTH&3z4S-qs$e5_9@=?>wld6p7jU+2>SSOtTx=D$#-@m^OS4`c`zfe}E_HtHU z`F)cfs`^KC;O{7Fa>Q1X1v*Uv@(`Bf{eeRG&whT-i=JDg4S%)|c3oH&1||;ij?cVU ziE#K@_Uy;)F$=Os;)?uEn3{A(2$k1;f8A9iSqsv-@w7t8lj7I?0b=eAu;fS(9_9V% zR9sq`1l;z4NqvWSi^b*jqayZ4($@ARBRD+6%0G<$KY|e#yIsQ^b@=Sr#<+s4-@*8^ z+Hcd?9DB$p7S;>BttSBW~GuFyMg z!m2X)b!Ton!PQ$2YG{AkaxX+!QV(7Lx`#}SkK9}Tvc>>Be) zPu_aAC((6SF6wzN$Vw4_36*FVnx*o*5ndUN)$tq+D*f6cM9*r@^uHI7*5%e+8YFor1V#_7VHh;MYAgA!13x-47u-Y+i&BkQbZ0J?=FmnIeuRhvJG%l&y+Q zE8A|&MfH=~X6RO|*5ceENL(j8e_4Z#8&DC*$W3NJpKCL}S-^PGSg~0-G5ujRgO{;U zT>BoCHZlLJxFnu!R$XWeRE#ZSRr}1@vBAUcc~hl(U)n@0-@fa~6vv5-FP!kN&^(>J z@?}tv3X{3Mq9x?deB)P-M1oG+kN8~?e-y|{U^H5%(^~IF7{H@8@=i_xRY=Bai^n$Ac5xkVk7D){)|09eFjHl;)^vaj(O@lFbozG<)}s|SmNJLozHLESR+Ypsxw@Au zIh*D^nnfxh_a%Y!j?!`5jeIg9k#C`@+Obu7YO2TxlQ0GkGe9cx|gcbzN*OQ)kIZm1?tHf=rh0kfqea$i30DV^HdO(E+A>`yd-fE|C_^BP4AK`~BjQ#>K2rV+5B)Cz`qxGa(~Ha9Xv< zQZp?Pq)1jZ4II=%YXp>{pc^wP_|;=ajoN7GjXh-tBaBBb)ktIp`wAaHS+x(eu&BZn!tj$2@47riSdR6>s9Lz40-OMDlc7SPUunB0M#S zIpT`O6UQ2yjkDD@%x_X7J^RPwJ?u>OmtW2OXG0{Y!7mzV=FOkYK zOihi39jKuMRtL?%V15tQM{*~pN){OF2Smakx)Eyx2eKzcTz#^ZsZY$;jRi4^N7#k! zNC;5K8cYZuP|@%R7#6uYKl5YSw;$H0XdKBp$I#IN6tMX7^@LJU^|5`7`m#Fhm&v!D zqOOGJm;8p*qD#G8vpO$L)hG9$ma=`_hvj}Aufo|*nY{hORwMl%YA2QxVT{CS8LD;Q z!tMW$uQvhfIbZ+(vvV*cB3V;m5+m7{ELmEu6NQjUWUCAkvJ9eZMUjZ2juA;Ek+M}2 zVzNX;6lIAdB&Gh3`yIylUcc+#HP^Y$nPd4Z@7L?T@8@=Z#O75&Hn2npl8?txIm=`u z*?~ao&FRwZzPisofM_g7H?!PAmbg?doQGmtwwRHw?5G7f~5W2C394o7>jWK1qy1Aj! ztSQ=7(RL6H(;YL`WI$A^Z}R>7d#1d8`&NQZe2%_YaD8=AahSKa?@E{itee{~UYJZr zT))ff@kp@%e;dnL3CDZJbML`_WG*%1}uiz3vKQ1(|8WK6NPfFy5 z4H7UKLM>v=h}pP=ov54p;93VB4cx-=xX#U0`)>u25Ob5V?uWsg;#j3J|i~$;?grP?iubz5zx=5`qEPIiGU)obm1fmEB3YN3~#}K zM!DL|S*$vAZc4Ulqj=7N;$kGQdaD?+RzJVmg2!MM!LSBlou<=h_8|a3_iBoHK=+5@q6T>agQJSNSj9IZc^x6Pr znCrIkqGi_!E!omBDf}mj?HLo05n*)Leb^thHp&t@Ca+kg_V9%REKd-@_XG?)_m7+c z>bjJ#bM2-ww?m8rxV3a9loR(-4br*Zte77XAA?()yy~@VT)%@IQ^eHAJe)z|81BE1 zHY}uBGR8LKp6a&}G@|LB?k=_+73CI9h3Ay7k7};5I+o0T1#R=xxTU)D*xpqZj|q&) zFF4E)g&u79+_^6sY93Pjj;H4jLrZV5-QvX9rnSLpOe|zsSAH_E$T+SkI3R}c^!V+Z zKyEIyVYL9->UGwRXT4E<>Y=XmT(LpZ|xT z;NW2$U6)oP!u_yzYO_7a9Oavm1uqs)aVa$-jONv_VZ%NHC%aGg7Ny6FwrrUWA&HXn zg@nkrx4G>-?iu3DB+;yG+rdRfRRh^1h#m^Bp;_kCJ>F!j6Y0-@@p*5L@5d&jzfgx- z&Lz+f-~XMKuU(wU_x}-6=Wy7f32*Bg8(SU?^K`Wxd+-z1c$z_FnRx4suRg9|_^%5h zy+INBzLszhCd%TZs`4b?ltBXRAPA(zd0f`2+RP-OV^ND<71#sM(M%T97<3L@rpjrU8l>nb&$aBmL)sr{1^1ej z4SJHm`Q`+RQJ&cZPrc-QS(O8i@F~ewE|Hi51fafp^C_C|PZ6rBtE#R=%ry1QYaBgH zZKC&X=?o9C9DtlMQs}!~{RH$EQ90SK2KCX?Ysk>Bc+5tQuPirFY%)<%s&Kcz^dFMp zhR#HyaArkK(f-Ja9$1DK8!WDnv-0dg;Ki9Ioh`}DJg54Cv&=*Hgh+$29W;BghnE(Q zyvdod81sY+B(HhUvd$s*`kXuD3RafhZYtohp`lpXed8TGxp%gayD!@@3eOlON$0_& z=^HpIbZ6JCvDJBqBBbywGxI#cvN5g%-r9wG_wHS6V6srG!mW2!5oxh&eMQf<(NDz zD(dl#UcA>E%7ky?n?{~}kw~zZQ>$wcybn;&=@AnO2oc>1P$vtgY1n%_>WjyZo{>+K zbDHFw^P@#OuP0-UVO-hIPW!K${s$fCxuWx&cgt7i%i3i9p?J4m>Pl$4S+4Jm+P6zs z^0Hatmsfg9U$=?OdAbkjY-x1GDh$}vQF=Hq5MLV5IQC{wzi!)#^aS$NAE!(!ct)ow zv2v_cXtepo0>xs2jG=o@p~oT3_U*C%Z8{9yMn2}LI6mAok9j{Vj?Xf)v#V3VkuHto zK$~xVVIiVbgp?u{UNmI!W&3ga_BUV0+e-$K$lkcZwLb#83zIr4$r-R-P9h3o1sb5S8?kMycKw<_ zf}4S|rY!K@woOb0u(cmA72P8fn$YVepk_0^3^o>Ox89KnUHdnoJ#u~NtPXG?iumoLr4~2qFmTmD zvnug|=3$PPwFbmvlTTY+Bf$Y-dKNDp?phb08wkJe9T*O@()KfWi%6aQ?l7BIu!X}XOo1gHQMZjo;lB;VBMKI1<1 z-8h;adZ@$ms+*7ZC-wfL;A@_z?yQ`ht9CXu9_mnU_sBZgN6T-GcYZVYwu|Es&y25S zk8j66v|JGT_PE)|@l9J8JB)5_T>n^M(@!cUH%d3HUXe0loOR7j&&$qqk33d1+0ycW z;bPa*2{U?}1pQ}{I)cCoNlQrCKCKI27S|%(nuF1TA z`ful^8M*Xrzd*2T%Smvr<(GCPrIy(qMojxb{rc_E3^=P&Lq)~Jc-SN_q#phH^<&U= zyv(0qa3Fi;0rkG;SHE)emZzmb18enEZ9PeLwjbnPZfbV(*+w&xf+X6bYvc6J>o2as zzcywbZTI}3=yi-qls}VsKv%@`u@56%K5a?<;x!5meCEs)*LI(q@UWj*@$pvyDXBlG z1}{E%u);EOFiI_WhuH+`OfXvfr&xLwP=C-ltEW(17g9`nuUT<`kkZ0la@ZmsrTYCZ@v+jJaX< z%5m~$FN-WhLm3uhqtDwzBna$ zKFsKR&}D6{e>8a-PbwI%+Q&7^zX#R(?dM5euN^Vo8n{c(QgKZh+I}x_fZNt4Lf1zo8`sB$4ll+1X5F6Boxn9Vp~zpmLWl zS<;+Yo;Yy-9?jD?hlh7&E46)9EC4v9P|763iGD@}IXMNP$wecICi9wLlacqIzOdP; z&*L1Ux~mr@4l|pI2hYAu>(>9#(|g=Hq0R4;Chh6{j|qu%T^sAyMIQ6w&70kKP(h}J zIhzG}dp8t4oG7085-16miEPetrK^zLOkgjvp9Br(jTNq60N@UaBa^+8!B(~MDfc9P z*GAf4rw^BcPfhQ{Q5EY<04}ktkT7ZXir50FYrC#)Nb+x~s#*iO)MxA0f<$j%dhpCY zE=K=zmDJ^poXOA=CvFr*fr_CCaI<+hm0KFdWdxVFgdXxxO)>0|G40IRv$PhQ^U{KA zm;v6m7cViQx#(0LEzWZd*^vq+5i`yFaUWK0r_UP_&Dc|)1P@)Qu7`eNt%0X8CDH^?aD;xg-0|!cO{m}(1 zz_YXrS`*~-D{5u3GQ6V6XM%gMCg*$ zBLJQp8nDanhX#jzd)kKT87V#S*t@P)R#w;}*RhL-e-O`gD_wdO&2I=76@Ra9c}ZBv zxGqg;aDl8U;%8Z|0yU~S~9}d zMoGj=9uUOmT<;#;3~KD1aMYO0UvbUIssjl`1_in9{?$0<426T$#Vxty?T!Px@|3;b z5Ym7Aue&9eu5dPqe6l-q&lbmd?-PS;BLliNNKZ+9ES5a&GWGCORLw$5wyJWr6{eH{pXhQwberU(SwJ=6d$GW9Ie%ucHek92STnZ9B&UYC7 zoKhjOAo{LO7>Vf0d(9g+1H2N9!6Y3$40pu-%o29#k^&({!Kh63lhc)O3s?@Umr)M5 z^A2KlZ||L@0L~~8W;y8Gymt_dd=!`;W13ei_b}S64XMKodyr=#lBKl%H;h2o?QVxz z^s)&?QfD}HZ}ybEV*u9|qEKs|I$*zL;&0JVcAJ%;&g(1JO~egK@aToRcYl|o62wut z;dpKWDg<#FydAhA81Ptb1#FjZ3vT%k7%7LDaLN_KdO<2P-9nyDcAf5iifbGwfCYGq zYuwJT%Y6KJ;nKnunwolJTZOn|#_z20@cwmzOR=X<=YoY8o=at?cvIj$TlJh+94S~!L@b96sW`{HD4k3& zTFo`e$+*b0=Ws6AttV3Mp)M{xt2^uu84d{05Dsl;5MxVxw{gRN9FFIkN-*TFdSDv( zuS>;AXKMU1*Y6@t)#Dw0UzMLemZikgelNPeC4TJyG)9|#X?qFLnbuK^1DZ>hE}h?Q zfElUr{oLPciw*{pEUp`#!UHbR7t)Y|e=5vcuWYXdjD!xF#SQCbQ)GdFB+ii4+q%sH)Ak+Z%42R2b|`DlptP!Ijw&%{EbdM#lirAe*BcfpY0YM* zF#30Vu^q%q&nl#DhhXUt;I?5$n37>;Y&e4}2^W^mQOG;Y7wYSYx}D7?stMPwUw>A7 z!+?g2n4;Pe@ycODg|(aqJ0`>& zsbdFX(bL8wtkwT!DaVH!yq)P0*9WgV)>NVS6Z33gDR77yPr^p&N76|7)bs{JcFAVT zHJ3l_YucVI{-Jza!Z$dkvev{(K2E-E#{^4Raz|1RKAq^Uo%CA+QTV#B=g!>3(UBE| zNnbx{GS**jhhB$ZOxYt=>h0xp1Koi2L_%-l<{BlD{Ulp`7A$zPYdaNeHeCTSJDUuP zXHK6OuC!;@t5*+CpZ<&21#H@ik1tz)4b4w$N&j2C_}#kaAx93$7E2x%_WRM*W>m~| zcD{_ZK+*D5H^)xJsBS=TCAaqQYARXQ&53Hq(I?H8jY~=w%GE07R-`!U>%mHu_geZ7 z@BXQGkHjYheMAOu+=<^IDOFa-Hdr0oon0_+I>J>k4m%mrR@fGo-)DO?UonpN+cEH z0AKVtLDg9*_R5tN>-sF+JC;qPLmBiXVA2M-1uhKWstXqT=6n|tG^;EyC4Vt9c(6xZ zuujy_Q=a`HJ`YUkCVp$^TO_L4@%F+HhwhwV==|I%;kOS(N0<0DelWK<|==L-aVJee9~LUmH~^pN8mz_<}@O|2T1#^J!xFFar<8#UY9< zWM#273zoKHHz9}KRFy8o(CAEY;Z*12Q;uhhht1l=_sd2K5wGbX#9eRTJC$+&p21_I~dGiN@D z$kc4r%4uo09OQi`PoC`mT<27Cl)Y)Vz-Aoi(2IOEAMg+6<-LRRVTmY)H@+yBUT)?2^XJ=}O8aQ6-Gu9e>!Lcn zQn0qBe`*}Rj5Lbgx`LByICgFToPa9*i#R7hc-l{=q_yKKB%yK5rcJ3e{}tfaRQ5|6 zqMGsHcud*9kz$aRKM|!F6)W!c-MH66W0-M_YXcSA(yg;XORvt&4zodE62V!B3D8w{Yi7<^YZTr+1c5$ilhQ*kH6uSiZ^jNC!w*?S5Nb@h-Hx%W9 zh$z^OsH^LE<8|kddDMhuix-PY_DOE8)gqdoV3Q+4$|um5Of z?Mn=ieZ=jT&l(sshPDBU8f9zycvtx>u0yus$S~dLrI%BDV8V3#w!P22eLn>8Cpn3H z0W+C~uQ~vX?cBRp>}h7Bj zBpi&aw`5JU?+PcUE;6zdKT!5*AIGcOfH00~*B5}E!Nbv&&8c_AiO*>M5G>xD_~6KB zFn6NPnII!=&--3q|C><9L>Sm1us?Y!cS+Tcn&F=!$0cl$&W(RXo-w!HZFBh1nK_*J zYjV{~_I8Y!bobo*g$LPXl*;xFR*ZL4Oz`x)73_i82-M{B(*FdTZ^fHLGJ_p@-bNB1 z$8njp$f!IgTJ}94GsHcu>P(k~{zx+R^LKefEp~IejE70w_Dbj&>_`Qx2Iq{3bFgEK z7m2h;uc91TGI5OE?9m~N}LR*yF!adDHC5I!ny#b7O)6bKbMu;Pj`wU_=X zamV;0diS2SWXYr-hefF_Tt}iCKGHFMH@B2z&6$lIy9r9eeDfGfVC{Y@K6D{4V;jv< z2+{(2Fvn}sP*XRkv_q9;VJ&EnwXYGzKt82grZuk}35a+Gqe#M(=aE6uIx@gYoP z2k+Xw+vL^h)29Wb)OX!>-SjAzVSG7Keaxh`_*TCSLhMk$9nOSfc^z`T^jl1Q3 zO7-QS8KaL|h2o}elT?*si~z&U&xgES;QRC&U9W^2t>#yBH4y+_ibu0R?knzbp`E;bhO0NWhE!^pM~ATL3^TDQKQ`1)g4<^xW# zG++%HtVO*RsWH}odIN^T6g4{GaP!ak)eSD569V)h-jYY`V#F!!^ti6uetNG4Pr!SCB3dI$A$^@Sru3 z^uZ;kb#S#w%-7txs!IG5AAshXjM045tpM7519{(;`v)i<=LxI!P;S8##!a5wS>jn_ z3FhoyZLV@5$Z~JQKl=s?#Y%f+Grwx=aVxVF?%h_JnzK7iwrNh^e0|=4{R_wUE^;q> zuOoOfva{$^I=jWs{#I?bPc0+iCDAlK;SV13q2X_+=UCuZd2rINJLxu%9u=1acl(QD zXmFZf&Ck8XpC|G}^hWDCn8*A0`Kd`Oz~|imk>6-N7PtNOIKjP>Ed6CDbgG0~4oOV3 zmHjXq=;m1eq8^DjhhEA=qNE5RWSd;Z)w{UpmM$Q-Bo`OW;$_PMSj?36hOSo@2q;rN zT3eQ&`xMlYLBlCagMqyoBSzC{Z@Kv;QfKnb1KAb-{sd|hdVgv0M!_s zjo=K)ywbL9x^vNw%F-I)8xIA%G2^hM^V73Xauh@G0rQ;*OyUUX|{ zhWagBv}jJPcRE*Y+_(_|S~iQbFrszf(X_1rylfz1k&mY(t^m-*E7;MxV?xe!Z|K-y<&=%=vDJ zXQZ*Q-$xLw6gu2oPaHbnFwJGJ1o+;ro5C@|7xF>$jO;(xVIJ5ndkj8C6m8l!AH|Hc zbuKPZ&!)XBN1P-(sM$=c3WK8VKHWw`q@5OTjz>g@Ob@-9^??rcDTzDp1O)|2k_nD| zeOn$BT4&$DYVe&^xFNfb@qnR3@hv=7;{p$t`p1u*v;xJeE1t~`_17IHm}oPg-Mcn} zr3&abq~z0+g=Q>EvWp;P~;Xn6QSNjW9hTIo&%De9)MA zPj900<4}yv6O=eQx|!6OZ$CC&m**o~ss3@=3o!Bfs)V(HUQT`f`jT?;?#)@r7L7tD z9DT>Fn+H;7JaMAmlzT6i2)|Oy26YWsA_LuNfr9bKtS{+cty?#HEy&pW_kk1Lpa{lL z`b-{2Lfp#8k!5&k($Ps+PWx3BcvjY%J-aF^F74`7hEv%TZVyfj$}RGVmE~{ZUc7kG z^fk&1vb(!e-wG;PdweP!CJ&EwJop=y8TMM7&plmgtha$!6>2p2&J`OAi^gyn+OvNR zY=}EV#t=G^y~DhNX~j|8AMQ_xUqDaQ^efLwuX4^XD1qOkD%nefH;%Wlxw_HOn|P4z z)_Iog%=D)NE%K!L+7K>2XqMAsgjf`d;S*ps*5S^Uc8sQ-Ys>eJ*b?ZK_cqJ0b6l`s zE9-3m(p8D^L2NhF!=t=jfGqYJJ-VKl0iQqb9eyuU0c-pXbv^N`!7F4vS#w=7U;zbt z9U}s%m6DAgmTr+!G5p!3xDeg{?->TQzDL!#PfrX06(LM0hz%J~(s(u^J1n_xkky#cI( zRhj2KO2e`(Yph`up{!rN=@B?tCj2Nmu#y1?DsbRzXA1@_tKC%xfc~99K|VCjywn?I zz2`pfs;B4YbMN@ndL+wYK1=sQ)0Fl#S>+@i-2;(2fyPa1XD7I;iB#W0r*Lt`$0 zjZ2&_TD>u^K-QI@3QEgyR!@tS|5@d16*k;lr0gA<^s7Vlz;D=>KZ4eBoq~n!e{Xwg zZ1KByAjVQ!=uP{#{M%5t2WEZtk5sdd2*H{4hvFgNN4-nOZ!H(=Gu8?mmj2ju|Ig*+ zr0+nLAHO7h`?m3&zN9Ki{5HP)mlth%UrL?ze7b4W>C-9XfJ(I8)LStz>@b(*s0^Sp z@gL5s=AmwsC_7!^Vxn`pL*63Z>c+MNgXLwd#jy94IB%V-Q#Z)b%M!g$G&50~l% zcGT0;gKZI!Km;Nq0dj+aHisRd4Hly;3quZ{m~d1s8c^ufk8P(iy2Qs@w*lU|@z7Z{ zeHRm(xLS*!CGFe*Yfr1Y9Rn4vfuySL^zRb$7TLRL#yXD6q%O@xKM-T{X4f@xl(op8 zkpv#oBQ|UPjG55*mtSOvlth+j+Z?ivrQ9P_i*}yqd6Ex;4`-*Z?_u_Ff#j2 zWiS20>@%~hn4MD*Ub(MQVj}&U(+F_*tAQ~F`LXUZuBL+Bi3W|b!aqnBN_H)EXZr@{ ze;7HjS@9Rrs)t@t5puh=TlPbJmge>Cs2up|sKn=4|I~@ST2OT+Th+_UKjuMYaSM-H zvHSP8l?_GmJ>D9M9;1$g7V`wtsu)z|Tpx+pTSOkbt1Z15OVKh{e2nx&`*ANK6D@`} ztrZnzb1d^6x*rw_UhiLOC)ti9NHqXWZmD!yn7Ht?Ur*cb-$1ZZ0vBb;i=qiwl=QW( zEc&XlT=ej?#D*X@Ztl#?nG8C8W_l-^FlHc9;;$^&!jnkkFMX7~<*4YVuV3qlu|#6M zEG;cTtoCA!k%6IK=If7;FMIb?*aBzO{JiScT{J2J#HqK3R5YeIqjqg+*lE@MXLY^( z;=*HM6hN8AW00<`Ba6$si+ zc7c1w6LHY$n4Cayt}?*QTpF}s(W33R5VX2@M)GzG=Xsb}P4rqb`Hf4W?QOS>-CrV&e)#3(i7`*r~idFjRH+|7di_8TzZ#xg^# z=$mF!r#`gapBl7h&rm)Y_@H%2SGop3>T)5AI_zm-;he~RQ6oW#OQzjb^X)>UPZx9qc-ow|(#QdxWc>_cvHL)i=D z_z{7FR#i++ zw)i?>xzC>cNwW^~a3ML_VQsU2RVk^9SFh3viMLLZGx6<5f6k!SqrVR>G3IwaZBJ{v z*2~Gn1O8oq^)IfeJGL^9$tF^sN-KjbOC}_Q4mV6z&*^Gl(2UuE%pW!9Z`=^Uw+-uU z%KCc4!QNid{OJts-q3k}PBp5mvD>tFLAG}?NV&lH0*G3{NaGBVAhkU{-R26~g^6FX zQ!){Mf0KTJMqXd>2;kyAQLpxoXbB|jg$Xf#*FG=x&P+XEL}LQgdGNq3Oi`vxrE;F6 z%t2pGabC1&w3C=g5=tX3lZV%bnF$Z4p2?8=2S>4T{wUz~{jI4Qs8T1;_K)oDZzw0dK@r7cUMgjxJZu?6M+T zl#BX9e+|j$ZD=?q1xbCcv*AET0_G9x_GRXXp-eKMU)+)8N-X$g?&j5V3|lLj6lV2z ztEbpladQA}9Tu*JUnmoYCDVJTAQl-xlfzmZQ;f3KB z27NMgaV;4V z#6UMRnWsWVY=&OP?ccR~KdtJhij8x=3CMjTD7i~RA z?}yIf#_#Q6&awzlN#K>)tXJ?I`Ntn0N4H<*;&PFPlwo_LPTUL&Gerj(VRbD0>*Z6yNK7Z}C zwlqXlKLYg`Ag7slF+bl9uB=TiIMKvAl~{9_n>}|CG;HBZJ5WnRDj_2s=mb#iLpU&? zQx1bON>FRU7m>6`Ob7W9e;1vV+5W?H zaI-(4?+|(%-~2iOj=IdeKoGLRii zFWLJooY9yt${0@dS7-fCQxxqwy+sjz>z`>rV`Ea7j^}Hixh@A5rqI_05w_;*h@pVo zR58h{Zp;^j-N`JStNR^YfJr*~5?U3txUTeY_sOmyY%-@lo^iiql18 zw_sV9)6!msMg|iRRF;jr^)lq3v@;DWmN0??z#?LFEI%T#dKf5tzXpj^pHN3!x@&T7 z_CLS+Yin3R{65JNIYdV!7rf9=mY196Wf?J$(xXMJ}QfMhx&YeJF_ag zsi0&4u2}GNgz}(Ugxq^AZ5A+qQ|yE7wk2PhvZ=wzur+mb=y9ukt8%u5VRgeJB|>5D z_1{_>J&x!H#{^$8C2957gdD%`RRas=WjQk4*8hBDjI4eR6}y(#_>Qr$|M-VDHUC>f z5_vjX28n1u$L~Gby~|K+7zz!-rYZoIaOa43F9{RUgM-B3V!ar$BGeq$ubZzdeY8v= z9#QOH0DaK+^@t+Muom3UIJVtqo;`O?7MkO38bf4~31{NFxJxv^G}PM(G+Fs6YYwb3 zSs&P_xDmQ#;uiy~`<&)zVjT0KU;q9cd9%vWy|ZUqS67X!_oRaPK#Mz=4v(HC2jum~ z6@;&v6BVz7BKqZ+sfeJ79Ro6nNc7x_6<=14KD>Ro=Zw~$zb?N+Xh4{uVM|;Sk_}`- z=Dh$}i0DmIK=6Q5kSe9U#17``7FB(i@vBj=j)d?9BURdQCn6#OOG9zjeVH=hLbVbV zbqTN=0LaMUvV=Q`LKt{>B+Q7QTVSdVbwLy)8(s?Z8@O7@KyA~uZNX=?TN7w2*5R2^ zWt8we%f4hnjhVUZP5UeZ*p{#WBr|%#o_x+?%kJg|)6Ojw`>W%2HlWeLt05e_=ti^w ztesaLLjjpAC!hW&?FPRlLe~6&uJwqdy(cjUYx3+&6%7?Y20|bcyWU){s+t}SpUE>2 z%2JiO>t|hckfa9U5v9|bby@X%xI&;rMWMva*>#QRiE1fiOEC8K)51rEO8=>fmL?;@ z*b~U}65WM%?{=xTT}$3SH;XPgN5wrAw7!E)1f4c_NY5eNzrKA0#I?NL8mAhZ^{4a| z3d|0I2#?kI+p&?6lqZY_>Mi-2brg%*l(D%Y~qilDNeMPihwb_CKpxb~7Iv^JP~3$;AtgD5UFZtGis8z|Dq^0xy` zMjS%N$mVx79^hWKbBJi=5>cXLNaO-Vs1HTq{mizTFoiVHEH3pHy(^+esnacex?N?S zgjK#ID<+XsMpbfiOO!rK6rycQ`RK?0{(BwdTbUw9_N>TjKR;zyOrs5(Hifczz75G1 zGZAf$>vASI?hZWHm=uZ@I63-+1>?-t(v#T_>eEM;0c7*h$Kh#3Ix2MHvk+@77juAK zw{8-<_2kKu-y$_$UVd!kSW|^d>TOZ?@ibVz8((4x5E87Xs>J* z1=QPPK0Gd}P2Y|!r05)wk+tj&r$_H$n=n)% z9*rIc-KTe_!sB ze09!=jptl0X{fM>xGe*j#8QoUMHjga6s+^f;5r-x*;6$6#57qb_e3=Dl%N5}9G#Ie z8WXyxt6N#J;jeGHMq8`K-!5X8ijv@}UFID=5~G2&XiejN0jh#?VbK(O8mob(BoX0H zHY&G+!Lg46`t_Sh*Yg=s*cBT}hy=FGy&cxS03;ZOs3tY7X)50pSIeVajql8LDIvJe zr_QjU%nB716?9O&ey%)}oxOK;lL9<39LZi6otvzmo|)H5Ly>^_#f8Q!kwYL0zJj#f z_jZLFF}FSOJ=$Ss8+=q?z}mfG5NHstsd@PaF-znDHTl|r2IJ5UJO2yQFKcC)Cna(f zPV~PgP8@K}k}d|+sXpMsym`M8_S@;ercd}6`E5e}JSXOGEqZIwBj$#O9=U?cAKM+4 z&E0oE!)4`TlXerk^``$}ko0Sl1C{7t$z%eqJmgi`9@!*Jdv6nr0*Ip&|u?gaYNeaSg1Yow# zha8S1BxDz^1k|n@7iC<81Rxc5M!WZ%Ide#V7)_Rp-b9?-_=I$N%la#D2h$@Ha3pKj z14gAYfwa0*7)8LXPpPFGjEn?ChuniW^6Fbd=!mmVw3Mj!Pu<1aEqtf{4DvZnDX-qyl%%<&zvR}{%fXR zRTdpwjf}6&hZX1T?H2MPIn^*ncIkn=r6&`E9i%ndh0U?_7ZRU@6lDWSBcIFyRBlKe zp3;XY>xuD{A(D-qR5yC^H7Ca{N~`I^s5C5@>qh?(8-^XJ== zpHIj1GBA5h<=47zW0qU24hA+ylo~cIliLJWBl~NcD|%<96iDLy-kmMpd|fsDNvUIS z%m)mSy|R$#@YeAag%LRRc7Kvb1WZkJn<+G9Euz5-r&(> z{4L5epkq;^7&NnVujG+JvTm=FCxnO(LST z0qp9W-JAUDas`V3_}E?3S5nY15!0_Zm^!+nvnSIRBzxx+y&gSA%K$w!VjPX?cg^}A zfD{6=i$kaOK4a(KcG5n2J@u)jd?~9-?Q>2!Gz6fg^Xv90 zkSo*R|2Q9^6m<$-KFZEk@6tA7(^elJVreE$D4#te4rUGJl7mMc6z%!HdxW7@tO0TOI@M4*u)MxzTH(oSSYxpPPjLb8hB34Oh<1 zf8vhi!zT2ke}*pe>djY1+)3`S$V5o=g9s(VXc<^J7fMc|;y8f@;ETNOv+p1fwyZB@ zI<|WQgho+%bZTEre}HC?K7)=6xa_5&VtBm@V62b#9VX9r!FKYo#LIs#sj>y*LND9! ze+^q{1KhB|dseMByP_L1lF&lQ=FL5WX$>*|4oG%?(uGMFyy61x>oeC$SGRA;-t2dq z)Wb_PgKMrie*DeoeIvt;9C?Z#Q&uzYC@>J>4W9{DHAbr#EwijQq09b>*}H z?xwP|Yp&Cfgq3AVbSh)Nuld|D-O0P3*r$nu-?YGAn2W1pwG^?znUOadNSrTXDH)Ui zJ~`*%*}+9GUR>a*5$che0Z`ttdv{CM%G2<~Kq*k>``x5*?+ z`xki#lR_|c6rx*rT^Q!)4M(1$ee4ZHRTp0O=()3Ja|Jej|K7Lj-gr=WlUny_b*u5g zV;i+2(}LCHI9M;DKJc3ymma9CB9;jxUJK(kI}OABESMk;hV}_oAsJ5M_L1?__fA0G zkly-(CG=SAz2b(%VTNX4^*eVOK2S4g7{h&rx7HE;3VkwtdVG^H*Gx@JHfQymG-_1J z=*Tx}#|?qH6^}mV+bH6NgghCf0R<_8d429`aV$+)M4&r_%SJ|+n@$t7 z;ff_d2}N;JXQFtaTtpV6AGoUL{d3EDAgEAE0<#O!uInaq1NPEmi=dKOQ?Y2!zFllN zV`Yuq^YR~JG2M#jg-3HjM2nU!J7L|TGN`9n7<-RkX*}J2vJ2-7zXl^Rc@}|Ccs0KqMw2{y#K_F&!(>^jM{#Z)PX5!}T!Z$iFKZ#doM~WkvnBhh zg+0mrdULRf2DS6TF>1+qav|1FQ?`wtom~dn9?=v3vpJ zTI}c3SFk}#@C~;fpQDc;VvQT`)xQVL0(3zg?#PY6>@q5J{Pl8N>=@-Qgk+6jRRpc~ zRM&1QDhv<(R(&y~UOsQo=O&jfn`$;t!)5>Y4c!90>(s0ryLUTSl_F-7PmJb8W5TK_ zC<^T7|JLf=+eZVV2V@L)`{R$@K7R~&6qG3ge8YwkKo#w#ts-H2pqK5rgxprN)e6}w z)&Qv`5G-BWFGJ}Kw=SXqW=5%}shgjH8g|eCJISl6%&8*dGHlx8w6q4O0Y+dtig^2` zW1t?#HNI~n6(lUn-|qI`3f3@)?PwDkvY8!1nASJuv%q}cu&W2A~D93l9*9L$EVJ?OGcPwo!{hIQVaVu2;y|(`hdiD5U&}(kHz#Ri? zs%D(JV`#Xx@v-H#K9V3$dYBmkL{eDot5EXWyx0i+{3&M z#_f(>yUrc;S{8e-J!SmVsd||rU0|SgxL5m5wK7(rQOwON=6Jr~sR7`hvDB!@6eEY+ zV}h9ZHh9WMevk*;h6a|5EurAnBxP;|zmltydb$)yX_8yN<*9(_}zDl4U5bE6ohr=5J5R4$rH=W0U z5rI5gfy|HQ2XpsPJD8dkhZ}9zEb7N>0~_KXI7AN%m$g?xeSl%Ic%*w9N2JkmJGji| zhTV6;w@}(BCXT+Z|3FoeD|n^AN5`bSts3ABP)v3cLlRrL3;676$+Q zo`QG%bTXXkpvN7*cvj2E!=l01zx&D8Xp@=ziB-t)$f1$b;Vv^p@(Si2$M0zg16 zMON7+14X+HDO+`-OU49}-!mRO*rLF}HlV&QF9^X67^VjkM8*IA$X%u> zKTeq4$^ZEgdSQbD>HRy{Jm8J}FljM*AbB|gFqJZY;3=GCl9Muw(6ESzrWmi!oarYL z0dj2^3LLLlENnfqI_m^{WBKD=w=PlrFVR_Q_hzn#9b(hklmp`$@^AkGbI{WBUT3QI ztGjxA?oN}oPtqP>7WWQTCzhVT0sU8_x{v{l&@%T7&uogJanin(+c0-E9Tn=5;%SDL z^dhUCwm_vur-E1rPASrcu6n8&O9XGlHUuzUF;gu8TD0(f)a8h@>3DhNv0qLatCh=x^+i{|Ff z&WnGw`bV#}Vki0kJq>vsl;hB(V%^nEM<@L%YC}=>vIMqUX8D&d67ijT)3dT9Ch*>E zPgdvVy?c6DkW6taO3y=1zi?#Nz%F z8zWPcfo>-=%6l^OdNKKK9G@*8LANe2wF4Fb7UNr@(aK;8YbLj(IF(^Wu8pB}0vWyH z;^x*167VTDF-PxL8Zo+76CT1_bnZK2mb@@> zFpeHUkwMTVf|N^6&B<2q|z*PgVO5&LnsMck_WoL~Nn^tLaVG+a$!yz>~c2T zSk@#GLY7tiK?^SZhvF_#*3#+Msn(!WQyyaWWpeY$#G525`zR1lhA2^%Ap(jGqDC-W zQ6usgBYop*gf&CXK#dUciQ?FBm#>^LfB zRgL;)t7hvq^3i;1cUkxMiJUS;Ys-Th3Jwvmj94TD*kYGc%J`)tb7&Wk5~W}yNq&F_ zP@GZ1p7)yfMKPM6Ms-_sLT>s{zjAK~{DrW=Cvy%t46+oMa;h#N;lfrU{d9=RHr9?rq6rx>+b-8=#tx5? z{Kfz+JR-f0jO!xVigQ&S5jhR8N(os)(#zhv7`V2#RPPU+qA~X45KFYBb%4GROzjn@ zCzW`0$ArjHy{e^h+TZDS1>Y68O6O1+i%7ailO5gj)qi@ikS-{wjl#3bLMrU=EdbiN zE_yzk{E$E%1y250UW|)xqs{SAfYAb1Sz9;ooBQhZeI(v`k|V+MM&|xDi~0cvsm!@J z2OU51jw8<8j6pkLa=8Sp23gD(VK$MZwFhC4k?#;o_GIa;!o-(7nhdjE%vpdk*#sU4 zKl7rr#FFbBL1RlyJPU0e-h!MJDLiAcC_Uo9D`un6w zLz~uip_ptXaq{myUVpptnKKFM#KP;pzklFG6|3-MgD_p~;Y7d`=m)|doE!ildY; zz^<7D1a_Z(_D`ab%*(67w{C6BGrR%((Ij&s4sb4P>p^kZ;w>RRM7$;NZm)@42jwvrg={G&<^;kIAe^6$A!}~CXL^7nU{Y9ZW?k3 zq$5=lgYky*s9#`Q1tOu4)}@Rw~8AFaI~BS;Ih3&MfL)za|D= z(T#gMYP$hADI~5~S4>Al-$kO4Ic2bx*6=Vx`?+M}D^mqbzli@QYWZH~T1am_u;|gb zfPx7x{fI}96%R*`lY_#EGte<8s9q+*@2rKZkR{vxLD64F2tqV1z9zE_a}g$*x?E`n z+F^U=@!2wi0s?ei!N;nJTEX5_Cz!L%^JDhu^0lj1=@q~jPQ}pQiB*JK-~SHgX~_xT z{``B_{Ct#haOFDv^NA3$S99}aVaOU(cjz#a195u2aHG*D)&?PFWANlp%Q1yQWU17v zSI;a}N6VlWY2kFcJWh63QwcocHJs#&o4emRIH57UPpq*Vkk(rfb|vF?WVhz$i_iJE zBpL*ONq<_ukh%B*fTMgLaLC=;9q``@@DTmFj46ioEH-~)^wcS>RZCWjV7w+cGUk+| zzEaYB12H?XGk$r+(3o{lQt}RcJrdXEL>}KiWL0bQECxD?P%C*9+lufx&{^3W&n)z4 ze*0-&UajhEI)ruUFUrT-Thm(7j1I5fv5NH&`3Ddc*2BgF2cD~AI_Af7@2#pFj2S;4 z4EO(?Ge-4N8mq|`Bh>Dn+@khir4OUTHfxEyUk7|EhZsXfBGv+)B(g18 z5o__putRh!L0Cf%jWc(ijzWu<=WG7r3o|b6wQkq)N8v)S=uhtR#8Wv~-R0Lc0As*qKGI@Rg~SFRW9<3! zUu%XRdHD*67sXG{o*mDR75#wT=?16hY0!e`JcKuNg@5dEHOV3{UfkY-h9#9EKmo;y z$twHn2%;pjz%nec%BE>utuL!`uyIdpLu{Al;w;9+iZ8Dt9n(|R=EPkCL>tK^6L;Cz zu!sppB2N*T?t7zG(UGf&FO|kMHRj$xcg?@u5@MP%TS0-#;x5emW4k9YPgX9x6~Z23 z$IJ`-jaCPKGgj{d%Fl5yS%5l0vYw(kew`aiQ^_s`ZGU2e|z_1&>etrXc#+m>Ec!T=?4L>s-nn!bN;8jx^Tne$LYCd z{Lt&^ZI;w!M7RtbIGDdoSf)f|BTTDNw&*$_lE84btJO|lgJDkw@I{Gbb-8No&O=^^ zw~1WS%oViRz$eAbCliB$uFd>S=zsBQ2Nb$`1DJxCARf9RPm~~pjF4#vODs8M3^B5L zg$HxWy?L>JU%eVwT>b`n57oJeUjf|)TddWY*wn9C7{9m1RD);mS+6LPln|x*_18$3 zz!iIPs@5YRlIYG|USZcTg@WdtZ#!@)oKr4iZu@Bq5qv<739(1<*|BR^?a7nx)iE~>md6CqRYu{Ij?CS_ zz-m=RFgAV~Y8DX|gW~NQEL$&eV4wnWvo&a@`gK_K{&;#RKKy;IVx*DL zLs1CDS%&-FSw7#m#H49$6uPY?hly7lb0CxrXsufHyP4+Y7#R~YNjk3e1mDRB&Qc&{ zBO0L?0i7FVXmuhi`O#U2Pkm4Q+dn=3=k-U0pBz0}=Od_zq5>vUE1S^@+k~XXn9vQk zgVd44#cE1pOn3AmXfD?)Uc!;MV|+&C`D8M3ius8Th<~VD>0_q^?tJ{%_S-5CN;Z=? z2}AULve+s#H1DSfp^zi%N`@fC@3C;s4;_|&y`ME#8QB`#prt_|j$RQpIhEtmD$m&o{SLKp@~$ zxWdd39@`E*0V!f4@{K@&Cr(tzS^sV4yO4Yn5*z#bEyHkBmi#-}$n2B$_bte3smqEA z@}GZpOzH9KuV|QBaL@%pDZHBf%2JvP39@gct*xE(3 zL5`~d>8deT$L=~hbjg_Qy`uFs906?M?H_Xd>mJH`@mLfnu@fPIM6Pz&f z^jJcIv}aeFIid4bUM=X{x$|t4I0<2+jZ03@xFULoN)CSSLc|$R_AFc}1S9maO@efl zgp!n!x`yq!2&PYOOTTzMaAVR~EEwPgd>3~5NLppSRrPa9k^6(DT5=pBKD)Jy86?NB zt=h;@quy6^&8T9#&Nd7RCa}7J_a1ImUqkvcGf~E)NBT#(fjFvdCAV}#9~gMJfxV9| zzu-Az@4=Q*jxRE*rp)8KUZsaphbnZ;8T&h#vz+pkhQ1^!%Sb>Pa{6e@n&Cg8$agjUT}n?xw3UJ+$uU{-+M5 zEU8_$u4GOiej8*SR<+VpGC07=s0#i)AKl6$4GQ=C6Bf)VV>4@NJior45&meB2%xaa zuOm6j86WCJqsrnYM9q5_`0r!%kYFvTGYZgq#-$cty{MrH*1APdiQ;Z&!;Ej~K(-wS z2psL09&^vZ+1XomYSXzY4iw~P$$;(pfq=F^S3hND$8|?a{^)Z?xNbYndI&P_dH1_-q^DaL zAU14Sbq-Ajf5u3TTUH4InZ=*qJ=K`5rmYpYRalmlpPO=wF<(vWt_CPf!6Sv*B{Q<5 z@>5Y0)GE=o?K37J&=)qJ1okG~)j&|VD3LNf@vJKK)Y*RY?yItGV+@@!EIHomk7-@v z>a7Wlx`>ypA8j#mwtyZ@GqnC+EtV?*NlJuS$)`^#kOs>=yJ!5TCov-8z7UdJ`sxq_GyxTv<{iD?i00Q45~>y=pz zI)d<~%I}gmlgcm7IG%i8x(eh5zMuPl@T&0ef|z^URU7CVOl}v@p>@*L-Dr{I<-Zoc zkx3_CzFIE$jsO^m;|yoh3;X7$;ef(V`;sD|L`d124!_+y-X-T*>EZ$C9mM1S#1!=@ zKV1{jNHS?}Xqe2yFaaL!8z@=7bDd7J{!k~6btCOAcUR-{M}D*8&bO-Z_syGq_w884 zM{@FXe*By-Cq>*Xcs`j`mywhm!U<>E+k*%A3cXKH37n)fzAH_tYAZTKwvA1j{CNc3 zj)ZdakO5Y~M)gPw_Uzi4qsO?Bk(Jf;C=hxsVdU8x&&&39N^@?}JJ2W$g2}J#%zU`u ziD;A!C!d&Q9pqZ2)D-SuUwT*W*`Q&!tZ3xX0gpWwD+iDK-o_8Z!afvEaSAESM5zY5;jIx|*%Y*A`0{gD45 zC=I3%_%u?$@f<7~b#ro;=Dhs%-O<7T8s8{%)eM7whFsPBFTqANlTI4ALE4K&-lZhp!sIKQ zsTd3ikFz&DZOGUq$l!9Cjw#;)j5*=Q=33m^wPVK`-ls_?IN52y%(Jrk%$H;v#ttQe z1k47(z$T81d5XPZdobO9Sm~lPI{#`J_b>IU0UL(7MC829odLwzpOO+aJGI*Z%(wpF zFeGOx3-NfGvxY9%j2d9r@4xTvx7xF6qk0b}rL-xL_pa^wGmJ&*(WOrBJ^D?DwjD)} z10sLLX5qr_pbovlee6##wHiE)1l%DYmq3RwzH4P|5P&kFXx)wr>1P`|Pxg&*D>f8A zB}y(?pYzmx!k7Po>&t2%vBnYJ@;Rb3{h1?dCRiNX`^+Lxe6tZc4}kPOLL3==F`c$* zlz-2W5QW8L@4>}&pbtYx13GbUcD^?Qr9AzDe??LfCH(nR(x#GRhFTtOB{7Lcle^H5 zmdlv9FxgG3kZEX;c-jb!O)yW(7gj|fhQLdhsCStj!HcY;WY1X;x5e+_xr%m>m>xXh zxZ{T(8Yk({Hk)Z1!96}+bRJLbKt|pfNI?Je;9J#Mbe%HhvqD5R2P5)nKP?4*iu)4P z>-lG;j8+=DekMw8GKbl=q*;0<3z~$-h^5xC`_VcR@GEtiHJmKMb;P=`RnCPT1Ii33O7je^@AIPlLXuSlZ*K0iRAg8|BplBQrGc&dlurb zrtb&hhZ-$hG^aB<6zcyO&#FX)`iry`@y=StA)Fujd>@!2jR_Ty;NBGsOp;kfDOi0f zW9Pq0!JnhuCoi!^B*T>7gh2~9I8LF$k{`Ca=s(D;5jVH&Du9jV=YGVcOFZVn zjw6aNdBn9Y!>dTX($wR4_@HIH^1yW1BX7rxm-fDeBbEhvGZag&1|`n9xqG(-KrTLtjT4RGyy58%WESMTrj7h_zwVHyh{-WM#REo&=@^$)WZV%}T{G~|Bi{rmPcU|fK&hl%?EdM8`u`lz0Q(%d}tNqoMIg;0q zVT%tIq%dVe!e8C4{#e8@j>`n=N|5y@!^-7~EZlu$@^o<-6Kch7MQG7OVQ;^b2Mq{` zp6KlKGjge^!>38S?8OV?+o~;e(U+Zvvv9qXh~yugW)S45#J2#Li||12LXbV8U* zuspI>s~zjTNL%CcCI~l_D<-dv+qNCz8giA#3UmMOC25fLQ*nA*QEqzhT~H+daPpO- z*IY99SwRX=H>6jDV?e)F7Z`h*_)Xzfub9~8koVgdr*}{k(C@6 zJ`zn15IriGE3!42SPn^a0O-+FwEOHNRDjIkKau+ZjGUTsP~weIGZ1>HPAD!rdwaut z8F_y!! zUarxepn621A}R`WL0haBe;6sPN=ggKNm)2wm!07|5lm*)3&+X|glaXxuE+wXh9~tZ zJ*DLg&FH{=vd7eDR_mnVk&w*hbs%hO{JPdI2(X z`58(HnO=+vh#ptp3{$1I_s+lZrt8DrAwOL}3(#>AV-F29j?BO(QrH zyu-v~iS)bA=9DCE$}G=Lu^O^76WZkO5B(Wg8vcL!G1gW&+weP zZdaHrvX8u~!0JeWCu}+?=Oot@*4!Hqy0b$)N)avkOy06EBa3;6`2*cUk|EhpPh^Ik z3*-&Q^1*E#p)XMkNZ2PnnoATz34sQtYmD1oImQxZpGZ!L)%lGOoo7 zO)z*gf5=oQ38m=6%|Rrv*B?}4#O6*)cy^yJ+JwL1*@4KD)XN=)xC*gdH8(E-5@^xE zGAynT_8aM+ld6ZiH3>hDm*BEb2Ciy!z2@LLa*ej_qa2tydjoyk@s!E}9)xUau)zvj zSX?gaV=j!p`w*I^BV*HabG~nR6%|h=?s_7C_@8b#XfGq9T^Q*}Pwd}zS3u1UH5QhF zWPa`I3@Oyac}Q!p z5hos!Yogd9%OJ@(lZm7%7l&aVu<5ec7L7W@AEOnn-CW?khml0o?l}-ALEKZ_c+0(p`kT@5lO*b~DdV zk#sDIBXI@#_V$_Bf+~^*!RB{)Z43+$+7o4p2UPQMC(RM;(c|WR*ONp*BW^0!=Miol zNW@fOo_Z1uCUg`n83|WAI*HMrgo%IoBJgLSV;r{TyzA&dWf5NRUi9{Agg#|Op8kr6 zy`UFq0Kk6cF9^h4N>(o6iGfjD9C-XEo((}|QW$3cn>?3AOPTSzNs!RrA9qutxF`OZ zY_2~Qn{QfOxq4N@+`#+wo;2ekf7>W9-z4342wfV9BEwWZbgJ`X{bizrgA$i)kv>|C zEologt;NVCnN6-Ic^X6)dgTg#3wm_;g^)FK)3ds0El~=BEJgh``vFJ@U7r%NK!wdE z;t0w@G5KOs52S24iPx?z1&)vX=a%|I`|wg>Aqydn^#3N7&r;V=&jtNgv1V<0m#(F!1m~J;DUw?Q%P^ z&}HRz`n*HSj$2|=`j%_^vB;E*QkotsU>OzsGlbSNgSLWL&b4_uAdW+B5kKC2R4Skc zH@H?Wl>CY6fffuO4_PLH+>icW>U!<~hT)3emX2RU3{eE;O3~K`@FLKEi|lR3PFyUH zf_zOASt)3g5cVVCt%PWdVkp5SOVkmG2AlMkkhGmzT^6te%WL)?InqG)C~6UTUS{(% zpQg;(h!VbakrJZv_qbe_{=ZGx@6lzpe0P^N@2vRo(-6Qf)-%%L++dh8^MQ_(EbB5} z7~_y$6YuXf)WVCN0)w8J1A=M(M$Ll)nra{bK3U~XY=6W=g+J`3J55~5UWC@bbWZFg z_qP0o;1?4aSXO#6&0sCaE0RHvSs-T|^obywm?CB;rar z;K--H76tP1-UdVtAX_FF2~#9y3(r@eZeXXp{1-;mMzep@AY3(l{Y0D-Owc8PsUd5I z%(8I*!Lp8NCH~jU2^4~0Ko9%sAgV_g#Coz1I_J@&ZSgGIXi!fccQR1Hv2Y#)6B;mH zAZEO11&{ukoK(j?Rs2*A**@a_BTkS15ujIqZinw5`z0hVzYh3mOu{1{bZoe@l-jDG zSJ;TOY}_HFoyJ~KOmT`|ALAs!tw$tZC^0P$Pu47BNIHz*70mJnY|%P_fKVlfcKUTf zToG~e+v6BzM;E*K%)sglwTR~SjZ+~yKNPCCk~g-e&W&pk#sya%)(-}fyrfX;6b$Jl zaN^cRhAwt}`xc6h?Ltzwn$$LM?*tbH0JvoXUUTvW;i#bJ;}B>>X94Sw;EqK8dr_R& z6ASc&l*Wgv=1MT+pa)Q~wH*wP4}_UUEl|TR z^DR$$uw$9hgms_^!&;Z&l?}B3Qe!{S5GGMORjULFHd0^?V^|C(jEm1Q0>YD3AK7QD zN#i7?h}RPz#9xChw$+8VB7~6fPjSi9uQM+ed;pvV5-8jklF!vWB_N0;oV=LW&nv%< z|4Yi(QKa$uy8oqGYz#Pa@BNcrCu&|46BS}@IzARAe8v(l(P|+rl59f)&xZgdOkr*P zi1FgMF>y_A?|%On39c|19*ASX&al6GO{T4T)d*9K!cDPK{VPn|@u={t1-KHRBE1j| zbA(qc&K+FD9cw@LQWFdn@+%o}4YK|x=x%2gLp1tQGJFxsN*=&AGWgWjI~lpGZkS`l zsytXYEH3s#YkF2uu@{)gO^kQoZDj{MjE}~01q@)a@&TpnM2_ZfK$o?c= zy0^0n$a!3aOP&mRI1}ici4S%rSA-6=ycGR^?I4+Q2^OjG+9c*Zqb&n)9qxnLtc7RsuYLY5My3%cHvex3Bn}RZrD6ks7>b&n1 zzzB<89`QY0i28xa5TRAsNTq|Idz*DWAZNO9Iza)Uo zQK|@HJTCC+*EIrWu*~@)C%BCB}2E#X$ZrbjA0zWVSf!L7m6as#(!geDJm{~>;H2kW>3qg z{B6821F?G0>{!)MYP*4JX&2wwaKUjxKyG*`Nw~|JJcrPE4~To=b$qRlAnb*?e&yUP zd=0EvZH_H9WaR{9KM6%xg>?!pS1kVA|Io|7xf-H$%H`LRKl`sDtB0df;?!XZf)uKk z!eqkDj-p$E|1UbWPWU*Ha3Qg=%%@rK9w<5#63V6$AtuT+N){+K+3oq)2AeltY${i2Qq1oxv88lUL{~S0s76@$jFlYVE#08 zWYmJtxf^es<-vyYoytyr*v~@3>X14F{2PJCIc@ciucDrG0|2tD(;y2BVLypID(V0o zD-f!nY71&AG^hwpSx%yJg$zEBrR)oHp0HSG5ECoNkv1%L*R1}MqLU$5$&YiQaA^Op zH`Y>oLPB&v=aLL{QX6-}NAp5iO^tB~pA88ihZ#Bb+()Q9pytAS8z|C|Slv$#6@$kH zwtb*zP_IxmS#WP@izTD!UA>wth#_i`nLN}B^w|PC^7Ivl#*se+rGK?9(rdp>CSjSC zo!ym>zv!t5P(iOL{29F|#drhY3t>mvB-}w23%{Y36)>8dA{-S-UO+fqg9_yOuZh4- zNl7d{quCZ~(k@Lpk^s_bk~fA1{%`q!b0WMVwoVmKl9r^K_7;w(IHB(WcnObt$*Whd zU{7~-B|2H;kWYUmzXjz-j0h2PHI(+a-}v7kVr`TAf8wwc_pN+axqdn%g#FlJ;Q99U z0^R4*$9`@;kcbJ5XwFEO`{fFGavVr%T!JjpN(GHy3BE>sC282m`QeWhcmM|-nT_`C zo9;S;AU#YSVT%jHi|j&|q|tle`D1xP*r`unL;>lLy7=G$s~wtj%7lVMGBTn30lxME zj7DcniM%6@cBsF6#wI@jj3h!{N5_Y{z=%nF)hYyqI(-PW_5}?B4XX;5428%b>#I>v zPU5RUe|9;DgPWTho$=-EY(BMLmeGQz!BUuylg9TDRxV(Ez;Qj~Kg4?olQ-45%ZxvY32$lQ?bU&16Y?33474%n8|B-xjz?MwLDR(Xk&|lCqR|BSlt+@EW zpS0mSB;tR{`UK4OH+*vSKo;RfAk$nXav5(!!;Qlrsec2^GYR$qicKQUAsyD1w%f3F z?JIoxL|}-9$P8mYGH()wq8?l1J$XVhQ7l&o7GO0wjPE!(WFRN7_V<3weej(cRU8vG zfOhU}lCo2aYdLXyqCX?P`|2;LIvGcdjKm`Gwer%JK7OgAF-9zhxTFMIzb_y-={!J} z9t5J8)tQK!Akt)ypU&!-Q-GvN0mH3T_h-2xN(e+7LD#OGGcFk>8%;jYMlKN77$UUD zV1yPMS~gr2U;~0dUnCd} zM-dXSt6L{&jcVyP!%MiR`C`G_L}FhjTn{S2J$MVWpR*WTch|KADy z&+x_Q?ndbLBbJ|oQ+NJ zj6@tWx{t>&PMTi--X@u=v4ivRqR<4BH@(nh56N#O z=@9n%l5{wRe|Py>OOs^OUi`6%9D2(@IBw+xjd-b_?L-KeM`&5KjvhF0Ab_%;!Fv0j z(PbTm0Kst#FC;-5Gr2CVH*m}ph`gqzM)laSrp$F4P$2ohR1z8$7EBEDkVrY?W4CZ6 zy4TU)-?X73BmsEv0>(AgKW;)NK$3$b>^9v1CyY0)=Bn*9vFt!yt2w zvSp53_`&fa<=1sTcg(K=c-@Yx7I7AFrxf+j3W`$JG|qP}UTm03^2tDhxOB09ac+aOr+(0X z_o6af2CIer4=iw``2LxFkPDasmiTo&KBO-Szbn}(MQr$Z^;hCV z0YYMcJ)E?ngyZCFe3z=(y!pVR)-s?oK~lfB3~xwYQtTutY6HxKi~)0>kc-YMA#wQ% zcYP@~5x_MikQc>G_rB*nEBeQ~vIRhyWduKDe6QQBL}f_ z0T6gy&jYw03`|%J{tJ2FCM9F64P&^@VLc(=c=f#aFmRAc4C?5*2A670nhrZHqBBjk zIR)Qg!5-cUS0Qkkl-}P{!R0x_AYPxk}oG@kk8iW91Yz_ zbz4b^e=DbT`FbQsqGk&00AdPpE*f;}Q>5P^k^&Nb1E?wx2kUwvhFy}rIOZcvMbE-g zN!o(($vRRpzDOx3FcBC!eoisw5LRZxE)e{$dd8>V)Lw|Iq45V=K=dqMX2-Ng0q&3j z#m^7e`o*p-|2b0>2~BnK*;;xTgKH#j$GUc8Al*G1z;iVD8~rrBZF(uEdm@aD0t<>;}U zu`O`mTSH)IYz2sSpgZIzo3{sH=pQ*aA zD4v9`0`m{3z0boFMF$?g#WNpMtcY&*`LfI^O>~@*aJUfnO|LZRlCfG11u^q!N{S#* zoTeYZ??bh&tduAtMGhJQ;G6nRGTXP8Tw)Uj7OU_zax&QQmxzeS(|v)bAE0%H26*?W zQ(v=va3f<@`2J;O$ zWeCN*jz*Q9H*lo{q#QFOuPsNHDMxos^Gq+CqP(o38(BlUkfQtZmtR%W#w*n$@HU~B zM!)O(`X#kL6&|Q6Xlv+XH*HZ>A-~bz|KeQ#;L#y^-jCKlv`eoH3H;}u@YVLKv5;0CK~|YPDMUdEogP#+ zL(G~GI8c*~{A-rbwmkkvcfSN5lH zS*q=o_?-*AF4tuLO-UrQNY;XXuZl7FP0ZeY7D?e9d9V-o`lp$ z3B_n2K)CGE#h{?~)F-!{$JiFJv>i>scJB%MNC`e|Ju;XZKBWpWZQ9NgMRdtU(P1`Y z9tv>jb)mx;)CEe9ULwc3;jXhc4uO0qApOSossDn~2c0ssIB6Yd zuUB6e;EpQF-eb6?~^d0sp=uD{le<+RF*O4hNp; zETjPt_AW!h{c5jF9RmZ#Lc_P0kgj^D?boWY%%H@uq#btbP~{Zv3yOArCC-GgyzEkJ zT%1|Sg{54l)T)}$8a#47`KhyW1HKLNGqpnOVa|v))4?h>7Le}=qrVQ&dp0OyG6W80 z7IBJdrvDZ(##I=~2$7e%`x4T1&~iYY884J{gfy3g8w-)&+c9m`p>@?Tk-6R zxv)6So<+ji*tEqOaVGFPOnA_!4QFmP)xup%VPr}#GG-F&*3`lx5bbWHM7)uQ0uPB3 zLgR%yE29@C9qa-p2}7A~&g<|hfrje7a!Bb&0^zSeS69kmmS=ur{cKXhT1Js=I_xsN#2FK4EHnP>^O% zAe}4prm~+BUh&HDLfn&ND|g|M*+3_d60Gx2pFxyx&mB#rnpRy7Rx=c09^w-vuh{0751 zy?r=dA?K^BcqNT7#1SN}CC`bf4RIzoiE?i)t(4#dP(buCc60N%fXIg3u3yt!r}mOJ zT*7BNQvhU!Ks9%ZuqQu$%<58jPBRKGFl~$m`3qtXEQV<)9N~no5)qmp=Q!8(hX?ok zn!bsq3sT~)pFg!aSNx|sRXy1~KF+`!xc<}sJyq?jr*BdHTf=z2d}yFj!jTS&%~Na- zdi`>`Z#~`|eG3mR60W)RS`VP#%qb#|y>nQD(RD~;<4O~}>_k%n-lcgI{s7<+8!^rd z%|j)>YTdd^AZ^@4poqb}w=RtJt(3~0mD^tYW+gL5+A*3+J<`0Meff)9f5c{44~`h3 zNdbuHhVf&ud@s5{0LHx`d)jcqh|Rc#?>$NqU@0M5>StVss$8Lk3P7{q_8bTth}7F= z;f?3h)3<357JaWCfv>(Fj!G%@Dn|U+zs{xaEMAao!|Uf2#fbT$n`wq<={o$1~$o=`2|$mE(~ZW5P~>EFyRp@%Zd z%O0ke1J)TI3qJPPH+bca=;%~js!LQ{kM8N~=HzsKdHHqNOmL(Cftc8tE&bi|NB=2D z$4DGL^XV}n>LKsykW8O8To?$|Ehm%h>24J?VkYgPwMcO$oIYuidN8kgj5nXa3-S7h zrKB;W*ibn%{eaU$Bjr8A?5%M=yo$u2h)u|;&K=-jAOm^3Tc@!Mk`WVI+c(dCqOD_w zD~dUnuW?10I1#$y&4q5Z;s;twbdUkDu^&RBB!9QtVoIQb|VyUgl(bdQV@bvW-egiENTGGL^^_kb4ud@+Xc0?g16P znCu=xlS68xuD41ecb@<&^iKNb-`haMq1~xEq7xWhO=M=mCmveA1_f$ks7T)HsqO1!>MTVny7 z*3qDUYY~p9Z`w;G-oVoV`NjYFX-(rNeCz&mm>P9iR&?$pIZMQaE^B^oJTAko5)<3P31+5HU^Ows&A$CYPQD>%@Vk~rKJzB{a;v;dVkj81j-bI!Ib| zR@Oy?DFo}SIw>bBOZLzYx`SW>{Ulc#68;)cXfw~VXogt%AhAB7HOqy*c%sUZ$q9jA zZcv{w;*R>g-as9F6G)%To4EYdKSbwjC8HBij~)5fEWEZL%zVCrR=j8UfP`e#SC@}Z zodd|rka~tzEdw(N_|I%J)J+mWAi8l0M5HSKF`!-?uLPa|`c3-ItOm065zfJf`hi&u zIQL#VyNtcX5JwY&9($KC8j^=kUR9Atqj#~4^4ojCmf7XM{m6m?^)PMCY|meP-GI># zr_WS+;`T%4AcPfHduuJPqh%(_M>K4iPj_u3ZmsQSe;T5v;K6qb5ah_=!?z>xTECmT zI-rT$62S(U!|$}uDQ$p`Q}?k&s8;1p)yXWJ@ZPI}ycU~mY;9GL#0~B1lh2q%Ly%x9 zg1XioKDqYU3rQ)dgzPiF0u9i05>x`VtP=6?79~6Z-F$n_zWO8fA}~CFv>o^F2eE~PhbR1=r{dM#w|n;z6g=JSFfO9lQ7TnD zgsfZn5M{dfqEY<5_`scHJnNQB$0Esq6k^;;~qR3W}jDKvyJ4R|Hnw z{+mDR`+p%MmHN-A0=0iA-Y((GH1t>su!@ozWv-A186PviE8wj=dhL||H#?E4*QmDV zR>`O5%~a=U=;oY%56DMK_Ou+!eB?LOYyWXUfP>29oArexBcIsb@cS|+JL~h?#pWD) zm%7bn9cpSeK^#-aygI%Zd==D4WC4V_ctq_aIRtBX85w`S_ds^<=-!^D)9Dlf&cvt z>a(UcQ`v2U-8578b90!y7QSTZhDffbqWUFMS5Y&(E&eK3Qfr~HAm32@!qT2ScP2b$ zN5T}p`PXuNTF$1Esk;Op3;?>vj?bNYmtEa2C@Q~``MTK2<)La9;H7rcM3w-?KEev% zEX&q9&ir)+I!}Dp)0qRf>oGsxiv1stoHt+-1zND&-m*5wO5!WHCw5ig8t4Fj=V+boz`lvouN zg_Npte5fI9Gep6tubCGuhM@?DBM=f(&9+}}*FabmUGp{bW`3u)#UKzfFw)zFAto$4 zB5k1{&QvC!w3G?}zN^^K($h$w1(%m?h8hrrQlZWnll~wzl|%-ch$_A99nChcWQKn7 z^vw@VtZZ!M`WCzSVvw`QhKjR#;UmU%bR7U}uuydst_YyG84ZWT$R_6%pw6HV+oh)V z`l{kG^R>(HTvTx^gvHh@$F>?JN+tggRtC5h_id&f3oIj8gZ>qLY6?IMxIY@EMC~wB zpiGxFH(yJxz0X0wlYCgtsF}*%-MwOQ46p@R9bAD`i$r-0ChxRslako}^&on5oz468mAreI%3m4$>(?Wa$-t{BpDYJo)@YUiWEQh_s9n0R~7o32GFL9Q8^ z2h`%8&HO4)TcJaTI3mXkOa@sjpw78U;umtrY`DyE$D5!Ha80}!%_#yv4IYPgA=~sG z)1tiIEB2#wmZAcz41j9U``|B3KMH-7cv&&{T|jpREX?|6En^zun&D_f+Q)nOK1*>h zxX3>ux9LNE40(Ugn5HuW19p(6QB&#aFCPuq@J`{V%9GfEnVABmw>pcR+uB!XDYdMg zFvKPC$yNARy`gH?i^1Yjw zcKzq?J*8Qf(p;NxW229CZ?}COJw5)4I$)LeyorTnmFeEVaLsb6qih70^51p)2-5;? zc&uc?@OqCrq29l;*A(U+6l`3vM=DQYv_VQrtC9B#40#^~Wg`&yY*v1W4ce1@`At`< zkxXh+G~QyM&+%df#$8LxO#r8xbEc*|&`iH-Yx6id*0(&WvJ9fZBUlp+S&N$jDio?e z^2>X~mVr#zV~W-zy9- zQ`-5*qg0YWgm3RtR?iek?naT%a4<8I_`&|!=LiBh?dGzkr>0I{UBStM_j{e+6%2cQ zo7YR|p=k%j`TCNlhX>G^V9j#l_(NwxeSkSrO+WUSG|nV}DKjhG`u4-1pHUcD z;9Mj0s3R+%b|NYuPBXE#1vPNhl2Zr*B+_%0D9&qyF3uCs3d#Qc^sx&_og~S#fVMRR zShIXE)sZxJtPwlt;@)dMBFnaGb|PBy5SlQ23M94Si%VgP60TU%da!RZQs{2A*b5%z zt!+nJ+dDTpiLby$a@v84V~YK{dA&-aF|q|JcHBopF!1dgDbOFyD0ft&>;x8Pzi7!B z4vg-w~s~?O+9@6d?%(#ar8R!#+v_} zCW(Ur0>`@NC$*`Nf7v%x!lEMgHY7UT=I+o?=YqhAc|TzxZQ1E}hUO~tp8LzGSOsa! zOWG=f6&2qup<-cSq@~lYyIei-EtLgnLVb_TaxEE5ZEVtbe)SJ%nE(8@09$Td(#r6k zPC+H{V1d~e>eYdoLJoPKGI!73iGN{YQSqZ^d0_Z`gNGM#&PRc_Cb}IgS}^1x?|Svp zwW|LY*>ueX9EgCL#MFf)g|arDXZCDf)01+;s1%8iZ?}GTLo0c%e`lwEVWC{`hksC3 zlFZ3XqO!g8L-`4ewOF@9R-l#s+7uYR0Xkfa3K<>H?LY^5 zXK8rq5`Y9cB&UHj)Z#sRmXNum>nhVf)m8ui?z)^%BdT^VZ-4<&O>^Y0Lu-IMI{kBA zYsfe|@$Ls->@(8wIe>E0|Kb*R;NZbidEoTKL7PNA=DqTj^t)TH4NBuW!p!~cqj&?+ zD}YVZsYS3dxS3weuw*p_IkxWLM*wKynv_iU9Pv;9MnsZi@kb$zR>jx%WlQ-Ugti{h z*Iy3VHi1{6wL7Qeck~zcRumI}NttVt>MJqspkFLKqO)xiI1rR^j4JBfHL$XwZ~h^Ruqib_q#d0d{o1{8D8VMj#X{_`Qx-qWqYiItA>^;sN9iWk9cvoIIK39DjH- zWtUig|DZ7S4LVIGQl$|3g4yH*4AbtSsAyo=1^xZ z>F%W?!r~TyT|q?&o>k)ej_u&eu#5xukJ%d$HN7K8N^B9~e zZ#za*z#clafKb6G z0G_Hkb2R-J(#uWC(Kv}aeOAEjVzC@XJ(Ic(R8*wro}E2I&>;grNXYQe+m5F2dwX7( zZ#=7nDt zg!mf2=t5D(5*_qE{@1TxUo{J93>J&iV;0EH%e!@t>EC089^U1Mj}O3}md2SGi<#+x zWc%?&fPcKazH{62Pl1l+xM=cQ>_0tlbb6YB7EKy{vjd-8*uIC)EN)n}Y6%_@ zhEbg)92-^0@}c%<^y*&!5G)zlKSt8nbwzY4VPqt0E{Z1F?Ve=cW+q@PBJ41tw@^D1 zC>H4!*uZypRB?P{ZavU$ObAFLtJBQ7k#b?0!EEtV%ZUuED7&EF#t?CZXoY-IqiX!$h; z?owQ{41Pbq*$><;dZIh|8$dhEbGqBXA>(xQ-hnz3=O(l5?w+uLv)xRSSRcHGjV;HF zkzM4r!}bQmu!iEs{%5##5DKES_>uBxWy9D-d%iy-6mGs#C)qbm(kKx~;ZkUrY>Y4r z0vf;xWLkY$5RKi1UEOCeMcROkcp$P~9;Ot{h&2_r*88UpXzirj6G@@jCJ_2dU5lbORHwVh4Mk!1J{vc&y37J!y`Sfnzl-**hB4`5yuBszwiu|XAl;^*x(9*Vne*X6t$J$>p{&e#{rB6c4nrcTZvyrbK zC;3mS%}NR0Aje=1QGHrmAqdpH#&!tKaHJnVy`FZy>C8%cT9UCq_+e-m+=}ob5?L8w zA^0t;OoJNB8Yf4Q;JXKgF8Rdxct*6AxUa20TH$FxpK2(!kGGLol7IC@yk_{sNCFp@p~@TKg~41wVjuO3XCaji!3uA$#8Y-2^S zJGlQR`j^VyQL-TmsfuoyPoLhrlzE88%gYNF)gB+87xLI4LKUO&8kf5)GR>N=gH1f8 z0ygmsh?pynv!6rE)%W&RjXu31P6>uzY(pIlR97p^ohmJ)3wXPLYjt;EdLV%{buF0@lKHYKI~& z6%z3`IH`r<5iL5#(LSkdc!Lp^%#6F1gt9dH%9II>(Y;1!t*7og4Y$`b_EJxj?>>eCSz@C>QBIM?~xsjDpj?U zzuT0tOr&j3J04Wn@eCapYiuv)#>=V}xS=|H6i`Vd&I#BCL6-+Ze4FKi>ln#XU=-v8 zm3zV3kVeUj(zvTuiG#GzbpUJ(xx#78l=b^kj*`vFnKyAc-74eX_5@!Q2Jmb%+|0oQ zs>MKg=6_IaCXoFaQVk;?^95?+gq@zBaf@))qRK|=ETaD;Bmi&|YRpq)y)Vnr&0J`0 zY?Qn|%ScNN-gdVDEB-2JXo#s0Sw(eTbCW6zPMCv%aLbq(qwqk|2c(<|IZjuu^8R~* zfYC-#_{0M1oeGM1Pc+1zo`W=3(5mo1r^=E3sn+@b{Qa1{7n{`SD10gR9CmWCvoFKh zItSP#6Cz4^ZHOIR0bcvEK_`4#1gnJXmmoEz(Bu>59&P2{14UMQSL@SmMT8(;0Nmoz zq^m?w4_TSpO#(Fs2ai_fK7MQhlMghRS(66YCQC4x;POg~{8V^`4xBEwY$Q^KLNat= z79Vtnz7ogG+WCsjE2z6*P#JVbeopX*=oqZjecZ0fA;YU(_O2eitya5Jds^D&7L@^X znE+EO(3lB*oladJs&+T0D*Z2ryLszYvpn3%2xA9^^TiXNc*N2pz~FJ)zdER5ag^+e zK}}|ZUf3T}27=;*K)(UJ+1!ttlam3b{D@9tFr;?WG&C{s@fndRsj243(HEDL>=n=T zMZ1R){PktQ17ycJa54zE?S2p8q6ZI7Fe3B@1o1^i>gww!&{Pp!faU4qoK%1-UV};y z`tDghfNmciuqu65G}m+hj=3y%ydX&*z}t{}(?BTzWIj82#YYQoe5N~$__)?h2#l&? z@dmQ@QZ$%(KZl?Hn*@q9F9}}Gh&N_~a>*KUx+C{H&8hJ($oktFVlr_alNy(ow z*C0ywsC%+_O9^8majD|sgpk>Ec6xRe-d5ZFNsMA(YxAstt_u1FtRA~+D5g%JD2RR3 zikDVDxfUNEhW8vep^lZ(4{ zrM##nS+!8WkbU)d3-~9X8Qu*qEUpRb?{fbfwU9^*o0^(j%c?Y} zVydXOZ+8@z=6NAK;+pyB(b4T`RgSyP92fG^Pgxnq`AW-T$)t(*07K&Z&ZdTU0hd1R zb@wEq=%~ob#hOjKaOmnWBz@Z+ ze7X9nhGyp8J$r=Kt@ARq7kPDE)M#X6t)c= zxp}j7MOkoNTE@^`<0RH!{T=4xwk%mb@$udZzvW6^**Z&uRp99upQjv6wFk?9z_2^eEqV$ zWl37YKL%7iwHAE@Zb{JBF`0Zbp{^IFTe~(wiBpARJgVdH93zFP7TEpnQ%gRl5`}y(HyXR~s&UR;XHr?kY?3ZVQABQFM!?f(0Bh|@2EG?8^3SSk8S zQWgBJ>HkN70GUVRh(WIsNIJAH<gsGyAbkz`kk-N-P9ZwrJ% zx47`#usfCFD4wJ#HV-K zvIk$;lIqpo9+zxir3-sUZacmK(y$VgcxdfqG}M2UI{J`zB7vah55rJnJlBT;=AdbmzW=OXN|q^gRZHB^loslDoeV!;-Rf&Tl51zzB>%_9Ol zwj#3_{-Xf@lBqS(K5xKlH1iCVnqXc`MIZsCZVrDVJHkKH$!rKI6|QyKjT0Ui;_lJYEmC4t`l|}oScM<78h8*e^xx(R@pte zaC=oo+CbAfy7N^U90CI0*KLss1n?L7kx z=ZYru6K33lrgOCForjIse8)2^1UBiYrl;^cI=uOn@T1z2#kqQ1h7pDv(u#^`_!>;z zb}B8fsqHD`-T8cp@|>x{&(B#cPg#=c%NFK+r{;(G(Gq5Azooi;bi1s($cEzClk%4? z>E})4=1U9`G>D>@q(t7rsEz62Yy-&}M`zOtP0bRbM z{zZJ;VDGy-8|P+PSBbB?FFn~^=IJr}WBuR+OTE*@Td65;JyT9@u1M*;pVpx)Tg*0* zc>U?Pv$Uh%<&1{KIq!Y4i$<4QD!qoT>_A218kf6LzozGNPVm6H$G60DSk1=Is%D?A zY_p|f6FT18r~LTHue3cwKbBQc2WBQkTw7$N*3M{nbmUxiH&^?R7ZsK8XXes3s=Na> zS9;!TXpHvD;94HLzgLN;#Jz29L-*+fNjuHLsDl6T+6JxlU(;LS#Lmr^&?so7``cDY zUwTwo_VMe(w%=FwWM|8879>=5bPYdBNoMf%8b7s}ZR7n2~O^$fu#;krU^5-)%6sHT;rC?q`x$ z;1kC30sE$x#!JuZiCNO?3zt4VF>h+IFqfB=F&2ON`zM2e=?Nd!I8KwrB(sgnxo_4q z8+8U04VfuCGYB1=v)s%U=ePP$;Z(=QnTTCedkQ1Au-M#S%JHk))pAV;P0kL-vlF=$ z&xV6dC#cm?yShg@TAK^zzr10-)v_#2*rvR$*eY2dW!e71KD#bPkCD*RCC@elmb82_ zYpdjZ`0}On@QWLPlbweeLW(E!x$KH%+Zug>@+(iDl~MPc23L1%sw+V1-0vgOgD*e% zDvsm4?A`Thm*VvY53~~Gv;!Z#^t$lmiF{RF(L^e{$mv)0ems$)21Tx3J^@j!kz9IV z0dzDQ*$a9`v)-s`{UgI-c=yw0QA3m%o2< zYnhyfID2IzTXrIp(vb*2QHtc+vdduo^S5aiFY+?;eD|w=!p)IaaA34-Lq%_GteHm2 z@)jRYg|2HS&W$vv3!6G99!}b`NxP?@_+WX`VnXO!t%(FlxKqx)wo1xEtbSeEBdhF3 zFSnLE`ChqDTf5A9OAeK`wh!zQ3PqEjS#|Xzi;}c7(ky&*%tj5c74g+YMnR9oc?*Se zZMB0NIKN$J&QR3NNl4f`H`gEf)c1wq(AlvqXizfszeXA_%2WU9i*P!~{|`j0xuX81 z%a(tp0W>b!64!pxOj9wf>yrsH+OW+`8{Ti>~20SG&{K3aB(vu_qgJ^f>>RK9FNp(cRT56 zjiVm9Wfw}fE5(;6c?z@l)GmvdUE$jq?Dct4+u$df3BWYRVx{5CV4t4|2sT=S!1u-{Z?Eo{86>~M9}QRYf5Q*z8gy%&NxLK=AT0q z=00W8TpYN|?`pMhz_|PG?g=5es=52FZM(Z7&tztg9j>}br(A43v$u_zZrL&wz?_J6 zun=8$1wa~<$dGcY9CES`7^RFw zF_4FW6+xLqU7PNyNr$E=Gatje!|Z*K5bP12sZgcYPtx4JdMy93a1If+3n#sYir|nFaocGh~Q`?5%gPBU*1xX_4U^@ z1RR7RfP4yM6)iBMhC{%zG2KI~0ryM^0E4c+iMiH3I3~!3g@`@RK%qym3F|0K_FfrX zh3IS;*>gw(SFh`-FLf#&THYEmQNy*iKKQiMu+VPJ`C=our-?#pCz;N6>!Zn@ea#z&ch;m?>CyiFTlaab{($S$GTiF7V)fuC&>qG?x-pb}qX_2G? za)7kH0~!Z1HsnCRy?A!u`v#%hdv$G#rDH7v;v+@S68SBdwH z>b@&>Ee=h6&kY6`QSLpwM^i(I3V|XPJ?hH)&)eR724RlNTsI46NSJWtt(}jo=QG`h zC(2oT*F=4GvNN|jH@)oR~pONCnZX6 z5u2A@T6gW1{0n9b{0EBn?XEZxrT$%L=y+GjwAmLU4)evutn=0{=R@X$AE2H#oOQaB z>pbOP=&8lOFuhmAXwjPOBX#hjX60+4l0(itoj-TI-_q#ye1G-rjW1aby&oEB4{q#t zdn4e&HQ#Em(#ob~*zZBgc;FPt?!r4MWTGb&7d;AMG)9 zvdat&c0p zw0A`5k4&F+GMSHJf90&<`|Rm-l>KYJ>vWcZ?NeKq29!QOzIorn65+)>CjX2*`m+(o zjQu2B-6w#>F>37N^mY7fQapF+eMn_sKuT&W(jaSOT{qKj@O?_{C4bI0fHaRcj2G7Z z_~BDFF+DRRL@*|mQZ0|D5j~v~XL;pkMD`!(Q1>wcsG*}n6Ui<7%D`!9-h0ox-VuFEzCL506>S zL>x5}O@g99_{Wc%1cZkDq0ivv_JOkJ%kUAvY1c(}26#`pmvnYoHYq%NeyS~RX!%1U z(e)`wTz=CB*x%j~E~$&(R#NZuyOQTSf}*6A=EMzBUkV*kD)pzyIc%%Paw;v6M^4OG ztUPE%;!T-)cTG9UaVBVjh1WZu@}vLMaB$35`5H~B-i1BLDC9_*uUR3bV6!3rki!v1 z^5#g-&#H)gmODFkIsU@2t;;w*7c2L?YQr1p=#voeINXbCs-{-|>BHZEZo(Pb^g(q`opmYwfTv}Ek+ z38lVzuc@of?_sk|wEJ=2)sZi}85Q}NY?qz~I7QwI+J~#UuNiL(NK_JyPOr1>FvTiW z2xN$F8DU}34^8Ne$O0_PI(8E;=D>_4HqHmF3DcQob{M zR#h$twfwV_wS_tWxmc9i@?EOhJ|){1ZE3 zu!-}X)59%Txk+@De;N@4#ARp4g!#UxA}|H~66y8Nl}~{jfey8_<@%+ba2X=49KH^8 zh&h+nu)4mUq&4Zbeo2DVp$r5ry>}!z+AEMMh&4z0^ z-%{21R-s3s@t)0Ldv_}mF^PH(kao#OCf!9XCu*)^I?Tp~?a-P)u1(yvuWJ*H&CQc5 zhLJgb7EHdHHaLOTX4{3a^`#ctY6T(Cb+2Em>C{4tPHap>Rtb3imAut*DuT|VJE5TS zim2v5H!^R+N`jaOxjX%{u!;g~fk2dX`SLc#+>z;X#9NEE=&ucEYz}k*OjR(FfDQ{s z$&VP_le`@}(}}eXq|aSDR|{Q$6Ntiud*oGbC4Xze5-LQv5_kbd;B!jN4*xdfp6rVg zP7Jc{_FZ17i99TJt(R7^eO)dr%xi1++WuH!Z z-{P{elDu_>I}(U|b4Gne5)0MGkF}_tN;f4dmrKO578V|q^?ON+%U1UEM)uBGqmZ{z zkB&WnhPt5hbtR-;{JJz{}_ zx?$5lKiSSr6r@Mj97pvn%Xcsu8Nam@STo@qw!QSg_Phn>pHVz{UAN9U+RZa9%g7k2 zc%PDbM5pCWq?~-5$uG-oQ?mwTuhlkOyLM}8Fhi)S+UA(DyTuny3(KDY@QrE}E>2Z4 zpphuvYq#tC3Ir6dU|~5^Ww9SXaOt~|4@1l+blmy)dZZp?&<)0}rEK{v)-6smZ;lmO-sA zpG4F<3OY{P6mY38jI+P=wx_XIyuYAZShIL#xwDR+!*-!gy^%1NfWlkby*g$O$LRzM zvDh|uzVNObGqpDjsS5bz>SfQ*sOmx6QBrNiu4&1+yYn#13vQn@&!+HF@8i)2W{13v zeUa|@eQ5PaMWx-V{-|>+vSicO8P9caZ?>RuYddJg(4E_~biznSTtae}@guIfueUbW z+w7X}&A0M6A$98Hmhy|nLc3E2{3PuaYe(CIGfclqxt!z>w$gq|*Ix3uhP8rS1sI!^ z_O{o0D-aN(U0lHOmZR+T{*dE~5@QS6@0)`Uv7A)*_KA|WG!uMXbd4uF-hqLK#d^)z zjf*;R&*B81&Q!Xp6|a)L-|978tUb~-_iE1X<$1a!L9x>;`2nFRE8p{QZgp9IrKPcTE-r9WNDqfVhmN9^Ypog0&3&$3&+l$ZS2?CLF>H<-2Bmd}fZ&gY z8JSXXCikCn%TiK)4BiA0FKs^(SWv*Gd2qa;BV(7356jqCTy*r|l+RJjmX40MeHYo5 zvu)if-&`Ok7h7swm2c4D+y@YE>h1qy>MX#jOxG^F6$OJ*Bm^W(6huNoBm`6p6f7i^ z7F1fgMHG}&F+dtrL_k2gOTZwdMH=bu-v9cXbLRYW&0N5#%@s+pD$nTl2$w1zkW^mTJf7o zOsws${@P-7(85HCG3kCx%*J%nS+oH%+DS&FrtV5hYa7Z(b*{Lbz(wbf^8V>0-A8P% z_^mwx0uF9iFD)UlG|_wZxR5X#f-P+iIT^P9JKSm14J#tD_U{tuLwiRJF|x`uWh#|ppxCWT(uzT<#O9Eq=n; zCOf%$*IH4F=l-}5b0%doJ`6qHzjn_xJIhq8;hOl(t zdH<2QV#9qbB%BuiytczX-1Ien-WdMD$_f|qPJpH3bzy0~0}IA!;5R*6F3>?gfaGuz z32%r6^^%@^Kx+9^X4132ub)a z_y8axfFegEWa9Lo?f{EZh%l>og|QVZKX`k;uKD%L3(gQ=7H{9&@D3(HqM!vFRJb}H z6c(QFwe&9;L@E|`@0x~&QZEdOxrS&E~KW@h8$QOvvr@0 zkvUale#W`GW6kEE&3L}oaoa*>2tQ z=XcSoehU&6*E60ZS69BzpXcuMMpY)dhOUQ(Rj{hT;t@cUh-?S#_(9kd76zgj&21W` zt5cGbsluI}Q&~t)x~>ltcNCp1Flc%uokhtfb-<=`{QIv;_QW-AR$5J!co$ysz73P%f4UlFKCdI)i`}lZCbX?X zCTAz*>80qtFq`BJK3;B3yA^V$w0>wC=v8D(VW)Is&18#Ez5_MscHH3UKCySIE8k~W zm7co{3bHV|V`bGmAH(81Z_!XtQJy9fvuDwYYbs7;Jqaa7F*71jejb|oTK4NX z!aXm@sJ)%cI>`~@Sst4vJ{w?nCg%_7KBTBJY}t~gQ`zS%kl<9!mm#%xRn^orNh@UL zT=4Q9KM*;gN$#*vJ1zA&|KO?SS6yU}U&GlJbIebMvgaf@=x0xJMQPDJ%vWA^iAav$ z^+apO{UdgBS>J8_rN7J;-0B{DdZvlQ#=bCoE>QJu;_H|LoqMbTn3?jrw_YcA^&e__ znc6+$x;G*=x8VJ);(mL=pCM}6SJHl9u6ifFY?X{vz2B2R6L)hR71o~Bn9f_wTnN_h zy~(_1EhRUSPB^~8?A&DM%85#$Cj0c}#o${>Hn;jytrBHefBdo-p~|RLZ`=6tc~L3( zTru7HjN&g|tN8<$waBry9ijgChmO>V^|7DpMA1EVNtTG|9?5H$YM*#{T@1uBwf7rM zb8d@K{qn%@w8cb(O#X74HlJ#zxN*Vn-={}@+r#IcK)TIUVr=vaU3gRIekwcx`)PuS9(QXz#b#$6oi?CPiMpNbO$kD2O&? zW7|4W%FLEz!nQoVB(tWs?USbS1LBRd+-mYe$<=`l&aF+6Y4>NE>j=N% ztIeZq#R^QDMH_qPkmDI=d+hoRWi%c#@>X&4#ufB+$6k4qlp~diq^Vw6TdsrALoFh4G z)(noV=>1m9P$<~>aKuQ-yen@nR>lI<6Aff(T>UNzq`g@=^(spxtnGep9ffyTPz#Gr zN4Rb$Yaz!#Mt1^N@0V9yD~|KeW-pKndnc2IGk&Zi-P&m_zA{(%K-Y{@b1QGm6Rr9~ z%jd|uw~fi#G|$RBOJWx)+BYeV_~T2LAto9nv0GJ4^h zq`20@Q%5c&ep?qH+ccU49L(HEi#wXDQka7IfMUj$Ey=R~;=r)Kbh7*H{=Pns>zei( zF-BZEqDn`?_#_}CR19l4f#Ld(uO8vcibIc)illUfWoi_ z{D723aIn-*jKDNSAmQBBIXl@JS@c#DwI}smd99)I9UJgZ$yePDi0dtZ4t)|O*t|>}990Nt3t_-Ys-hzmV&c>>5S?riLEsPp7y{je4aY8g z^1t^UmFLq)r^X6sX!tM$DtLcN(0P+}K~Vu|!(tB_HY~j!0jR}=8l~ZG9?V2U8{amp z#$aN`+~Bw1coy^o0Iq;@zg2n@yH30?n>HM^zI|wFYU%+z)!8{zUW$Kudx@$*7X9m;_<$#YRtO8J;qtv<8frE7>O|Z$n&EH34lD@UD|vJ#_wJ%{C%o#d1UF z@Hc%SV8Ox}8J?U>ZQFMwzPVWyz~;|{gFeQh<)M!-tZ4WVHM$Y54(3DX@;-e^*qI2m zngCy`+`!Hm$)SfC^Ucg{e-~_Ba=@NJqQu`A+%W5kEQL~bO)nC5-hYNRq^D+PPA4wP zZYmLg-RcB_p2*CqrSYX&pjr)PF@y!qcL@G_wmh|%nhx(Mue;~JwB&RnC!GSBYxU)^ z7GdV?Pu82|+wVHmFp#E8?vdFJW1Eb@x55*2$;rp*w{^v*B_=0J({gIM_Ze))24W*= zW$N|W=A0(F6^oD6D}3EOyl8(dTo`|?73S_Qtsa|}7#g}cf9WnS6W_vmD2?jN)YnuR z{m4s+JFFe+-nV2r*$*cr9@|~Kv+2`lVRvav1eS3Fm$v3FMA4h&3NB^tz)VKirs!NB z|8&SnOWSra^cTsM96eM$Umv@BrBCd9+8nvtTOrdq=W^Rx+xYIu9?l~ybBlDlwh+!t znFU!%Qs>uaPAemZ-kpDLiQF9{O=J|5b|z%3R~Q$k^rY+O-ghhFQ|VY6fkda-;{qJC z;)J&rA4iiUnig3eK5XvlE1**`$#8FSC@#F+-Nz|-h|*~t9ThoyyQ$l+3>oQd z{Hk*wWw!fCZXiMNi_ZGYfSG{R`Q;YIPh^vbSFdU+NMIBAk*@;Z+Q@};OAIA zy^fU7aP!PDjfY>|qsce%hhEHY6rX5%Nk3o|meI&Ck^IN7i?~v!|5{rd%d)sd%u^rT zzI*_!VJlXFajE(J4ognPOY?-SNdBTR*;XN>;XVYk32&?55i9F5r8ceS8HRE5 z10#z!k4lrrXh`=je=7F%|2#3osh2uOcVmy-KwoV+b&&ezEdfqGh{!+asSJ6g{cG-A9s^ zU3W*uR5uM6LB|X&Q1cjLdg9>hR(mGrR_4^Q+ncRbmI;%M2aG(N58H&jvYYM^Ss4D5 z$Tj*ZI_H|#+dBWFjYk5v42NJY*a_v_YG^s5AfYCEeCHvaue2Fv1wQuEevt_mw!C{@ z)Ikn&msvfi9@Bp*mc{tjQMLmYU&-EYm{wOF>Iptz;j-z<+M^^c~L>aL!x;yOC_Vy#rG6_n56v!ln!CJ=WFY5 zWCxZr3EuvQv+O=HUiv%V|0P0FsL@E6+?9vE5+b0TjKHta4o81)?<+?(E$_yueM*tq zvG@Ycd4A1Y@w*m+Le7IAIpKQG>g|grMj-gEMHc`@1m#FI8M*{aLKkm`!RFC3{3EPh z_3~x8*2O!7W4eK$-SRL9ZI_p??8ug9mg4@+mP3_#!&m6v}6hT?ak{`>h#cklGL=cjsK2@B|l= zrlL}0>&_RKA=K8EP^-ZIx`)W>BpNbvAmCa8EGe`?HLFL8H1KY_ zxsB+$Ly59l8`OY}iXbq8dwbM07}!Fnb*DPUHZf&hBoZSbH^f_0PYs7!Fuk!S@oB2~Xg+y2f3}$FW#Vn+t+51~>Oz zd|LVJ>x;fHQiXl=FzxX$rTEqhgjOkHT@(JUD-`I2P+x`#hNypB7_%+#Moul^ltPSm zj5i;x=uF^Kdj4xFDtFoADF4B0%MCK8_$=k~ZIVA1w7m!!S z#cf7d5OVU^)no`<73}!kIUTF~DB5|9FKo&0UuVHXpMdAjRj@NiFfCZM;dyusK6bi?41HYrP>|VpoPV{97KOBH?<5L`Xn~iuADWX{dlp@1 zsiwA>@LC#wk7|oLdEmemrVNkdFttvN_1T3>g7swpdDfq!zu z-QzfaY$E4^V~BA1xqkMtD>rY7@!vdjEkn#K%r_x=M5ZZm{>%9zm6tLSwGjA?5aB;P zb%ea#r|`t+aEMQzJKd|23gZQd-Q3)B6Z07crR^@q>1Ab0#$$YaXK@E8G^(9t`H7er zO2Y>9;xq8H`<2Eql>Vu_02QQ=7xzibCU9CgeQB6~{O=~^%GPTc`~SCC(SP&&VAifo zv6q#aosSDe_Rt-=kz_)9W4&?x=t!JW#M6TcndKFAS8LZ?@;rw3|yy9==?1ZQ%xHtN=%N->EN4i(h}cRuJU7tua^F^-F)cl|a$=@eOgU&;6u( zZ#FrTM!loVd|GvIDS<6jNl$hjA$O4NF6(vK3W?&ZDnSXANH+j7r_X+?51$ca#O0Cz zSg}q5_w$)a6!QKTsP$^j1oa1ijILuB_Yp+$sVUu3+&sX)B!qlQ0Dwm@M(A;;A~&aYx~Og-|5Sr5C4y?t*clY66EX7;_dcXlfg3HU{S9=4yW3 z_}cIRa+7~a3!o^h4_A^;HT($9mhVP+JU`Dr>V5L&s!H zw9a|e&Bzg}32tt;oQGlVxOKxfgZVoDHbq5n?!wLaFxvRNAK*$vIVm0stQn)7U)2= zQ9S;29G(w;0?`-5J_T^wSUjYdiV;?aHyG>5)}v*|6^U(*N={y$^G{2zhmxGQP2&4N zC$mD2IwR#nBIS3GSnW=0(){a+F_pe2*c@a$AZj`w`cyFt0dL|Hg>dXoAg0s&agWBm z+|#z*o(c|4;ck|+tSq&j-%T}oa&nn3pgItEqQVyYnNqGzL3i;O0F+5p5JLZp}nm>dOPf`jgRR>8@H4#?}XvKRnPsJJ`5|eH8bX5JXayGR;lZg@vTV z{G~(4pC0?}L%fep9*2(xV1iC)91FbxN)vdh&=N~5EHyL$bSF$a^ zyCg{Fxw*MV`;KfA3PEj<6Nsn_J)GpnK#&0aMAzC>Te}S4* z(jQf)Lln@@A>4Xc(t3!B6r)5}|FgNEgHu$rOrmYPvxqbcXAW!4uR}X$r4w_zDHsB3 ztNT4XkAEal%_OHU2vTsi8$??=V-rVc&)(Q&m`zIZ4kCsly z0yatZSZae+baK3N{c?*#V2RWj$JGZ#zx!xHdQ@f()~d8h4LQHx4z>JI>uFcc3x+$3 zj@;E|TN8h0^F2`kP(?O}2etjYhkr{elQrLID~DVa=C2;#rc38mR5*Q|`F4TLbJrZX zn;D#>-CA`f;j4U04!z=YV~14|R{ktgOiV4Gw{}P@`)>3sV2NDGxNBhenbGL==Qk48 z3fwaoZ&)>p93ryzm}^~OdUVbyucdO6_||fDE!W(< zIrYMU(6Ey4FDa(Gy}UJQ8}`)=y6~!G7ByJU*n^iq(E()PFYebyrdND?sfNDa(yuof z6|U{>{-Vr7ImG2WpHqZa(XoSVAm*}QrcHwJLY2GZaQ|HcgZm~D#?}4R8HQ=|qtk44 z&H#40s=%k2>0sR{l0SWM6E(F}+(FTW1wcg4`8v@28u|5SYou~UwtJa-qw?8{pBe_G z|9wORwY+zI$NS&w5dO8MvJhhi#r*rvkB;~Gv~MmfHlbK0OfAn^#V;MaaCYCf5x3&4 z>3-4^wD&JkG{tPyJ1@V$-<0swR`WOd;}+R+wRPXMM*ViD>S|<4i^zyJt6xUsmy1>C z>m|1EO@^xCfgf5Q*~T4Rf|E8D(0Jk*YdH+=)w^I2JIl(H4J*H*Aj-VaHQWeK7FM+(p>apzf+~i)Y zw*j6~^HxTX9WgxT%lG!-d_!m%F#aW3*g5y9k6Sw}|xKE$T48xs0aGx`3^8`|hxOfn(qt3p*_kXr& zvQwAbHtqrZHtTHP#zwTF5~69M>nbmx5h!ptx`bz_4k9LY^KF0W)0;MAGrEP)diW6? zw=v_~K|=U8;!5(x3tRJAvy(!H-@Si-*XJ3F#9Gi(W&$GZ;OMB94E%Xf{nd<(=9{>) z&m#K@iILX;5H6x@q)>b2QRpV_It8Rt3530G#{v{u0%+}-$8>>6JPQS3Crv~!@FN53 z?ha0;iFL&91h*mj)2E9u<-kJ7qq7>As0YNg(XKz#7PbRQfK`f`RWml;+q`aRm`DHC zEdr52pve&vj!@=Fonzdz0rhc2yyi(phYK43Y%v`T|Gq7`8F-|eBf`9pzmk))ii_FK zDB|GT_2rA3Lq%ldTvCSE&PcJ=s8fDJWEs1N)?<}V2&;{{236Vap{eY3_pAGL7Vo{j zw9Lqq9+9OMWop*k8Ea{Zu(zyMLBZVa64ISJv(djPkkDdJqa%qQr}H^%WY(nFVL#FT z13n&`gflVal+EF0va-Ss4*IwNecK*$Y1!vm_fysyfagNhJB=C#=3G#4fdGA^so^b= z6C!W@QWQ%IYg9R9-KC=|63-V2{2G?Zz?+;;uXCK=IL=gLAlx(TK3#v7sKl&D#;(lM z+=D;Y+F~7=F&%0|?0RUB9csv-F5^7WLtSnimT=ME#I!s=efKEGs-w(HaXXEoiqjdx z@|Lf}Gp(yf{Hcr=YS+gv%Qio(JS>~gaaQ^EO(b6L zWimUNR@%eZY+T-Aa$xy&vJ<(#n$ymg)9I-Pprgz}L6`fN|BdtN_XQneO#E+p9}|v! z5s!#bdw`k!kwz2chMINh4I3*1^G;;AxeCi#7Zsk+iHwyFaZ0ex?og%7ylBn8CR$Uc zQ>de-cHDvTg5;#y`(>^hhFV4gn-`Q1UDXa;lg*Ugwx^@RG1=6;z+RFgwX2c2);17(rBOPYZqUG zvy|vfBSzLOT2h{2MZ_fWMxlxj6>mW8psGh;=u*pv*?6D&)x#JGrv( zle=Y+wBi+o3+ZCUYyEBr(^%H1aPNWI7q6N31{DKW1CXpDj5M%=`!c->B@sm!>gudZ zOO&5F04ya`J9dNF6^wnkd3jx1!WX!p; zOV~7nYuGS~?HZO+IGcUV6N6&kzo!ID@2Be1X50;k?a!IFj@!-%Nkhf~afcLS91*Au zg6dy3yO$3$%6jmL`gVw%AWPqYsUK>{v4xe|@&?7dc)SvqiO>MaP275Q4GoWQ(?}@` zqlQw3+ML~lxFr{=Fz0{ypDj9Y4^(R)zVH~4+WUBtLJ_B5i zDH2vwcdjfRdPzMrzth4SN)!}pEeGQHM7lN2PDVb zl8q&0CiT;F@u>8f9Od=!!Hk|BbeFL6G-`ZZow>rIblzoP>&8uL{fpFz+QvS7)Dba> zt?cJ1dffh;__O^{*(mVk@9uVv7g_j?g~zQLpPyKe6!^F%7LgTY`NEN8(3l-sCwkQ2 zh~A;P=gU|7I&Zym)H1W_ct1r852l~(xt<3wekOdCi^<2KZ+z<22VT>Sq@y$~3Mk(Z zq&KXO=8D{IU7Bz^VeCPw0%g`oH`!a(N6+M>Y4sTK8C@rL*nb*peVlwdcC5HO@uvK6 z#_|rSb(}|59S%Ndl5Vv*vHw~0geK#;=SIrp%w3_rQcvf;Mh;x@vy(ady3etuwr1-) z-OkLCi~6ya9CKmO^Cmnd?&pq86g5{ju1>BitpBxX)LM$PWVy@dhxNsZ3Tkb`d1LDw zkY*P8f`t$rn4HiN{uq~2({j-Ls>IY>T%L39@B^hmmm5(sdygD5_E_0$+%e|MNfuvs z;lk=L{f*oO7A@y%UU+N}*qT2-Zf;|}OGiN=6&g0J_ToiqEP1@&5Kx=8Fz@mn!Y_UB zU>d8xF#EHw57MN|R0o#wg9E8^$APxBc93!(r?^dyr{t6$8lDz3pH9nLhjA1tC1OQR80@28$oRaG5DEP9CC>Zeol5paN+ z4Ni>_O@rTmUCWUBXzj0|W^$7fWMvrfpJl;|yUaX0W<0X%4~HImv-F3=`0H*}9vrF? zZ$9eDAE5TRHRTqa!RR%9onoL`WB;YX!pHoFC1)jDbpWbmM-w|CVBpA*&;-{S{jGW( z`OX4@_UY?>9+5!75{@@eH9j*PL%gK~Rr795TCTeq8}))WEJP74OPLmXP$6>Z+O>Cn z*U?*sd94UKMGPZ!oifP%z>L_x+xlW=P7cwqf^xY_<=Hto79jCL z&VIe+ccnHaX?Zw1LEdUW`L4}>!MM*_l|PSDbV=x^eDO8y0$+ts%YZ?K`)MbsHgVw|?L!n+w_0od0~ z4rJ5-+yRu{UmIG7o9#J;cvn zhHoy=Al}kJXbM0_Y7#n~c~Y3!!mB-lgV})eZAgYtNZ@M&8F$hl{FQ0mwH_~(h-pev z5>bcI)z?=hYw(7TuX*E+;}V+jlQ$nKilJ@^r~Vm5MRuD@vQUqRr={ObPnK-wp6NxX z`n%qX?K~nBM2(T-#Jzi;%dDTp;Or+J(x}v7VPnG`odHiN_2{Rk#2YH$6B=PagZYP-Jutc}udGZPiG1Y0 z@uPxx*h9RczNLGs(R&aD-VESE;AV3sAHjDE7IC76sjEx2{~xO1-*>8bO}n%}e(U#h z^KZnUGchrF+%|nh-5T@mR%bdkt>!bgi-i9)H@^l|RP?GI< z!=>fX<}P14>BsEJ{-L2`(}#>rTAW*dy(OL^6l z#np3+LIpbmhgp+;?g9ra)Y4vMy}RME>D@TB!=taNm$(Dt}ia;Pa6~)YqXYL+0kztvTNkVx(Tp*Xn!nsLe3!y$qGUCyi{ z0ZR8KQ^NN}rCE=Z>s*f2ow-PQSZo-_kZ@~yGib^o$5J+Hq~DOUwRKGp!u~$=_{&Vo znC&EOz`$_mHm6r^&9PZ5{&~-7Sc+CUZ08Rb_5m5%+BxwO`>>uUL;H^M=@eEc>VvQ2 zA`;0lh7QIjn&!{V7r0&5oIHg*9hwx~W|_d06L(qJhA7Df5)n}Qcx|jXua%kidUhQ^ zo&Pz-^*y3rR{g&dJ?MQVTe?_Uke`m+h8h`j8*S~``d##r+upisj;m58Of5=oLTaW? zdX<@$`OUy^eWLAGqt&*u>FSk`Y-&j{`!KiBn?kA*v@ZE>zh`F__1;M8+^yuOIXpPz z?zZiwy4{d{Mv0A$4G~85+jZQ70A%N)wHIW;(zVrP9Rq{8km@O2qLK^6We|U{_iUJq zDOG_aVyzyiy8~|p44+>xL#u7&0v49&Cc-<5f}t>@nDerrRzajlvYXUJKSp7pcDMJ(XZ;|VZP3!v z+OdOrM^Xi-=npn-6|cB^4{XdmqM|B#ZdV;09f@>E`a5<@IJfQ$o~SMRCn8&or3v@3 zV>_XBv6l0b=mEMw@YdXg%xksp0bh-)}F^jx=kML)FypFi$bpR@}6(21DfjAf{|AkH?vqny$(D<^>t!EC(n$)0Hd@m__OwjAcYGAOK`*ay~uSd^cqF zfqez1;KL-4AdL8Sup%jYmLH9-NWOvtnyd?2*x$zrlClhJY;hLs0BuPCV@@m-g#EFP z0VN+qkT8Y}(9M8kH-Ev%t-qT9b)wW0PhAy^XUv$r?su^)|O%M|c6Af50mG*oINttbXB9nnk&f-!S9>;Gu6U#~Vc#axB) z-Q0Nk@#*0X>5H>3939iD=R>L8J2#mk&#=-%oaaGxB@mgURsaT9FS1nX!Y@ z)y%KY^g_6Iu6LoK$%~#7WR<^pH-Ou^ASdTf{9LD6>DG+p&Y#ixq(Y>M{xbQF2~;PG z$5Mv+QY;?+EvnEj`F=~w)A4|ObK@pCYM%{54pYtnE4B?S&F+mupDr!O?szk;a^Q_N zb*qd!_jp$Pb!kPVu|a=}4zWyw74yV6uKe{et`5xS4u=QiE0drmZ18XxlE^02Xm-%N z>~1YkIqfJimoHmwsQp$WkDR=nWk_~%G+zmV{Za%ItowQ{AUw`(_dj}m`i~dL)sQye|`KC11Jzt>I z8To|hw%4vvhO7SZft zrZ0M8Wb!<R>KQyK3J8ITB z?yB_&138i&LR+?C_DK9RR{U(wqWHbNoW$zBTt;R0%FgtFe}%muj@#)Q{~eqmJ?>EG zX?~ykghl-L6&j9AiC405qzJLHwJVHTN*P67KP}iF@&@=x-*@Hf&~xWz5MJ_d^1t=m zt>5w06RP;~Zu6$yLuN0{;XFe%dA{;iLnwcQ zFwE(?B4`vD2Dt=W1 z)HIaxhX^f*aRsSilUP$*TjZ6Xp0l*QV8#r()7^t`{RX>C2< zq#hHAW$BGbxZ3*wa$PbVQao= z$#vy4 z4djG8NDE37(Tr(p8)G4-e}i}?zIsKJgMmBAkyd~j9%8l~9i@?WU14VUf7S;*LsMz< zsw5*H5x3*V6~@er`IP6A0d-AFt8HfJZSI_>Vrg|e`fHhDra-~n-a%b*I#-xC?puP| zQE^*&d8y1yNykuyE|y_&e`cV{X304@<%#>}?4F*nO1#ikSMuqZ)PnW8`~m4FimXIX$lIarS@34`OW&nCF$l)Rv5Tb zX9li#N&guL%@;}K&vi>)6OhU}z*#c9zPW?WBry)*p6T&@va!nRNDJ#ZV90R4&p2it zW<;~caj%ft zVI+z&ZJo5KQ=YuzW*VO>3E5X=9%vGqX5(sff!U>Cu=ue*?@^s5&Y|JjRjH>o{_RcL zXEd2;_t!8;w$AbesEs&JS*iE+_Ep|nS(_gmN(bMz-Pu;U@zd-H@CST+juy=8T7YRT z6IyZ9Q^c@}FGQwb6GH(xt*?xU>QPDjl<042yQev{g*vR=?GZ^+=~<9I z5VP&a>u@xHwWf-1~w3LBEC)tz6h z)wR_W6{Upf-FxzEae#uNd`jvSO;DhYc(0?e_R*)y(~6R$qX&d?2LdgA(N}}{m6$ij zipjK0y)YHkzhI{oxlvGQet2}TEC7B-P_2>07Q`GE+XE}1xc$^#lz=Feem{+Dj6Y0+ zpymn3FJLFAee2dn=n}*R6a4e|ZVQY>6Z7+~kxcuKqADIA!hB`HLZHKx3k}EwVm5T# z$&=}RXc5s1W7Nb?BsS9QzWTC>fj8-U%(9O(prxn}xHq~q-M$^f%@su##V}~>momd_ zijHy+w%(gIH8e!R%fZA{Ev@rdkq@7NN={bRPE1hi*R8$}I`>d+ZZ8~{sLVUbmUV;n z<+)NXn$?O-E?in+#)Cwvu#5NLL0M~S5y0NWD}31gstC{`80Esc|Cae+{*xFNrXfPL z^;cCY8X6dha=u$9_g}`m^Vh2ZY{@@mGeLKM*42ST`qe87cow4E(MOXH^&=&np~#q+a#;g?9<%`=ijnfk4F!eTs3M1g zQuYx?8Lkv*jh5#4?1?1Ci#8=#ePBR8v7e2{r=hl1@YeLc-c!3@o+A<#0LaaXKZ$(c z?w$+)9tKC03CpC4BF_%p5)T@y5!tWe zI)C{+Pz2&LQ9tn)5o`3Yf(t%T-0#Gpgs53SuVxoFCn4$5{hw3zBwlhf+x(*=e29(R zImg;%jvXvFII)9+n_s5#qO2Rjog(7;a?a?x6&E)htLSS{)u7A8?iEP(=^H&RQBl7O1!Oq<>(L$%XW8+7zl|1e+}hGL zxS%jNTz*GkN4Y8}ujjlWfsuPuZZq_PDZP^;lN2O)2$f%`zH=vk}pw(&j{ zG!uO?jh!juZ6~Zp#p#RM%a;LN(OSd0gc_5oLjaUuND?x{bVE zuhpM!RA#}#AgZYRDJ7)?&td0)664e9+HQM5AN^A8O3U-|(A$_Mf+oj)?0Mo^MejYv zcDo*lbLTtmM{-w|pgs7XbJc)#DNX5r;=tFVr`~a;4V$vH>Nf773vo`jRoj0}x^8uU zYo}YB-pdH>!2du*JE&X#Uad4d->^j^0URl{`ePhT%*rrKiqnn zDJoCZtTfl_9&0LTG=-*|Zrn>*Psz5nwxWjh(nv=^xYq;eJ+Mj;24Tnprz|Z~Bs4Kl z2^&^vLnTaAF6F@tnS|RqS_seY`i~!^-C_7J-k!R|R?uc7aLMLhsGi~s3|Da1Bwf8k_KnQz|hXT zMNBN?Y-JFA)Nu^oI5QfH6W|n4%~a0#_E>+a1!C%PfnHk79{kNKL zN)pZl6p8-udgSQ|uvXA*bYtGfmMjB%Sg6zr$m^0K&;x>ht^GHEk1Z$=Wh%X%l7Dw) zNX9{I9#XZpKL)2HkW`*QzKMkW^0@6qJ=xcRfh+{A0bd5urDkoNR^|*m0}gN0%&Y_L z49T;HE}TlTu)NF4%HsE2F30zR_})XFrWFomLFu4jbr{-^9+jEkzahFSokw>>0SN#t z9q`6jJ!myVPdw4(jQAPMHoNdxR5#N?zzkDf%uMoD|8jT4+As~*C)YAPMO_OYT9`&g zjPX$Nv<|Zoz@LO>8YZG`C8TWzCv?$W4R!T<0bi+01gP<70_C#$*K2BwCG5bXl@t}} zY)=W6z?q9Lnzs-70N7cma<~*}vBLaceE=9O9vHV2ac_d0P=zv1bI2S_Qc@#}=LuwZ zP|MfSuFc%p1WVn(VE=#pB(hps;Y3k!PXY?3;83kw$kT@b6HDoutbN3LM-l}bk5G;Z zNH_i_nv%=vhKfOK`P-RB47y35Kfj4E{^kwpCZ0<~8x{PS%CT4X%t2U{|4$&!djJ34 zeeaVZxHN7QXws5+4x1=|32Y!y|Fg2nlqJnSah&hP%US;}+({iiNoH-Q*QSsTrvlDn z%mTbXoAE}IDlL^)rsK8R*yRzP62~ZZmJml16EQ@joz`aFq_EO!N3Xf!4@HuCVm7T$ zAbK4gr>KAXpvK}Nh4Tf)nXVCIFZf7_MjCKPJ&N8msjV`PPRRaxw4HRq`3~F1sbCOQ zyyMNzMP8n9ZG!Z-sbxTfHc@{HK~vS7H<4K&i4#_i?^2XUrSDKhvwjfK?lF}fTfA_6 z_MJ)f@20Xg-Q2pJVu$v*@3)|0@AuL4zwz#BWGqbdZxTHFh7VncyF%_TdbX#J&w^vt z!tvMJfng5kUF)fioKJPhkx4jM@60Bs0_8Y&a=t-)s_MekzS(Mq^AfY04!zqF>?(0@ z^Z@XQYGs9vnAz!=Bu1LljZVv46i5;D803r0e&u@M(vJ`G_8(O|^V2y-3g)ZII7sOd zGUaF9pFc~5|I*3nW%jB&=fmRCTQigWO8)cjg|1(>WS7|CBk%cOvZ=tJiwJ_}k9r>= zgs!w!V0a=lzD@= z-qm#xPMzW-F5CQQ*!TgzTU(OHD~6WZ;x{6~F_aAd>IqAgxrfsJdraeJB$w;`kHWlf zjh8wo&b~I?s&0F94z#Q*8J@Hu4Pv^?+n=3%S#!$P`(zB14av;RcOSPpOGVSoZM^?W zw~Z>ZYDfC+Ios%zNW;Le9z8yaR<AC^Cd?<_yk5R~^_hurpKu-C{zR{g7fy{3 zY3Da-fyG{oyhM@M5c4v*C1*P6!%tzel@4_tuU##-j_6;xdO~@dn7MpTTUuN8X!`N9 zwhlsO=*89V3mpwr+#gWUD;A({@ciOW$DDNVg43fCy7_s&_pRq2SdmByw>RE{@@EC$F|K#_fO5tyx?JA z^??`T=+VcC%XssjeL<7=7#Di)Ul>WD8o%aYV?DdLxCq@dpNEN!?Z@ijbpz6x5>^9G zZ_BBXi1Or+5Wb8}wRSrg81!sb$pEd;Z-@+6@HvdbwzH0BY7~O3q{GaaDC5%?kV>2)&*KHhL#V+s8T8-ebxIa@6$;?mv~UP#K|%kAdR(%{#mdndt-6 z$A|Os4&2xmReR;?RbbRzkx2|W;pmvtau3%N`$HgR^k6#~1v#C?pdyg_(&>{u)mRZ*EX~}xg!*%W%40BJ z_xSe6$3#R)I9-ixq`slr6rf_Eoc9+|PIr{$0apY(FnGWgqIZ>mr@sdEosKB@>jnQa zu}Q{~4Uc`|vXZR35rm3yFnE`?iJG4HtpbWOe<$PJjIwsHJ!zA*sy8`?OiEZ`jf*|Z~i=qijso}P~iH} zBYc5ea{M==blr^$qrnQ<=O&$qh?bx{JAy>mHt9U0q~Mz?G@Z6B!HBvV~l z8?MEf@zbf<*|b*DvpfBL*ndk;{LV_g#(={-eBV;*SQ}ed z)`{-!{lPBdBS$+|7Q1Do#(uuLoGK~e`J=9Nr~mftD<>_&)_4#CoU#u%#;%mCQdF{K59c@FdwQ4+(aF=R(V#BrXn6r|; z_MNb?)I%&Af9D8fR+-wKj=1vf#=-m>CI>_3!Y4M{y8UYZUhfhj?G$RhJSAXl9ez7P!u$812dl{}? zzwTd{a*-O-6w%cP0@bO>-@k9L%eaIDm`l4OIPv310ib-YmvZ57!ieL}wFw@stPk|B zxi*Pg57j?(H2HJyn5c0x34*|-6`Vah{2Y$Ee%!f@9r|t2O9a~i5xv`o3U3=4dLsW) zD*+kR`6BJvldtyh@hR5x^YJ|-4oWswpLT{#Dk^>>_UtsF>W4XctH}p{{9Ineg!JHs z>9IB%Ts15GvaVddyuW_YQ!`Nkb&ouv&kiW<+UnpHZDo1yUbmwtWkXZb9x<`&JA#kf zemXU}=Z+lEz-d;Quq z$Tjj&#sg{_WD7w}pD3c@jQCWQt9=!e4B`}dlb84kd$_aprg!f?N00iGI5;7-oF+Ho zRSb|G&*Is!h3`ftz3W=PtKUvxgU5c??wC7_w~9ZTndXJJA|~9$3v0lr+}!c}K5p76M?ZJtz1-@n|8M zJ~P@KwqifyPk1^jg2m`CHcR%$Y(s~+AAFr%1#80vPeVd{-n>~eTCuaYCp?+JS-Cz& z6VtbNKdS${2e+wxx)yiT++r|RX@-+*K|xtki005A8k3XdwqaSL7lP#?R<7*oEi1c- z&o_lrc*mSl!x7WFL+t;@)Op8qz5nlDizJniA}h)$DI;WNHf+ks3Rxi}BfC^qs8p2f zP}zHBl#nQ7kIagY5pQJvuGjgT^S%At>YsB;yuDw~=VM&g{pzrn`Q5IY`rOBNwU~xt z_s?C~u0=4%lkdD|)j6uk^S$eBiB;b!C<~M^UuL|5!gBTVfLRo*Z(Ap2Ih>37JQozU zaN63sX<(+`^O?2HFQ!47q|yFzal`WE$)1k-`fWV?*8~bu)lL#C8T|jCgz)g;K2<5w zdTtXv`}9@bt05lC=RRcaX1tMkBsz9O(LT-Mt&jFT(tXnwHzvo4t~^_gF>%C8w99LI zJ_?tTOce`P@GyULnf`&#{E$$mt$lOw_;f(y+&!)2{q=3eT^Z)#m%VD;*vR^p{@nWI zR-$z;dz<$>qt_dUgbG{7l{s6yg|C+XQr^t2y8xTEaN|H)xqh$y(B}dE0Iu3rdS!{n z0e~NtZ!p~`A71`qRY`6idg+SSn!Cm+#f;}>%jIcC;kFd9Ydy&bhyU3MsSBk_>B zuulH)?B`%v%Oe@te=D+`azzS`dwYyNV+o74?V)IQ@_vq zuHy-Y`<1e_ja+;cc>SfEsVzNv5FHy!p+ z{dvYqasyVNU=C(P#C4^2aB9P5+svW3B4Ba;I{f$V&;K8%f1Zf_0^kdKhIrxm!k4)^W8+NJ+E7=KU4=a zvED!bb#{K9z98*Qtaq4Vl5&PlNKZ@*r%csj@+>!jU;WdOZ5YQcOKxE78Of8>qHADq z^!bNfCk7TDTGW5uyYCMr6_t4lNr30rG1WH2PAc8Fk+0C*e75xFT#A0qGW6V4wY5r# z_dhNw1|n z#1a$JQbi}+Sa1LSjE)ktI4=`^NEx}PTd-^l#xim-PvqQdWQ{@R^-gra%iQ%_)u&H8 zh={fn)#%zjC;RE?zeuLEp*{HA?}j)FB0v2$)&@3CO6>M98*h8P)W28}fbVYJbMoAl z)~f}Ui|53i%{PP9LOmKQl06m{SD_ay(MHeq9iahe8gRr2e+mw~XMkZoaplh{7>!lPyu~@}lSNBI(;@_|#0UWE1SuW2 z$2vtN&MjgFe^5fAcXEG{41~{0FoY6s$JNgRtJDO46p=60O-<^;3t`s9zix=*3LIQp z9H?8Ft_%nF(Pu^S$dmKr)jIqeSCvnUEEA|;5dL0($4}7kFT@&te zBWHjlAwhx(3lBAItoFsTt=!RBnOv@`R^^Wza@cSUdtL4@;62&v=KHbtA;%&&dZqR8EHl;L zxw$%Ci)8~<*7^5y=6q-Ojka`mZ%jZqba>hK;*m|<+;bj3y{QcX4S|UOxQhbEZ|kj5pN_A%F}PyH?dzg_!Pa}Bw%MoCp2y+cMhq$F*J${_sgrgkyA)MT=`ttF zMF+*h9WD%h>6X*>9G8<3EAD=tF)CRyx^Zy&YPZnn!Q$LS{@~m<6i-GEIe0d@zj57I zQ$Bolg`;SLV&|48p?h7|7O$&3lj^1~yZL=n#cHIj_ta7QnDm6)UyMiNJWFm={XP)7 zIc#3^Tx-%tWGV&7gCeW72d1vSyx}F)nVDSK4P(d zUOt*lwzJM><(&7Lx^xMr`1;bQZ-w|^VXSmViL;e;okml2dpo(StCX|e3$a=EDpIJ@ zCIB-T0dY5xPtJCnF(2M0d-{v^T)gC(cx7OH#P1wa`Rv!;^J7U?758vufr>OiDwQVXm+r-FNZR`qtcdj0J^1}wysd3##gvN9(_+g@m)Yx%|N90E zJ#+mv9s7UPc;aZ%V0*cb)cG|0i!o*^kJj7eA4|472`vBM)p*lL)xV_bCbZ@=>P6EN zy&T79uxG0mDdfU_>aVUHGAk=PH7R}?)Uhs<#A+p1y+{?8)0MtjRd15KSZ3rMxhZaA z?eGnkof~A7qxG(du0N7$4@#}$XLxuy`oJd&T{0UUhUw><{>G8l3O_Cw?y5}Pb@PC? z%lqF=K_>lv6>;yB$eP;|=*mrY96gn&OR-oo)u-!0E`72ye`Io+f^|>Zh4y=S)1-9s zkt?Ck&OLvZB5ivAyH8g~2SZlnx;Wcml2Abs0TwJ5~sye zxNDNFw2wT&`4!#0l>-16!%H=gu(}NCsJ3L(}z{>s3QT!-SQ6 z?p7RniLMtf?ydAccK=caLqWl%425SJCLWb+hhk=2COH*Y2ZT_WWf`T79b&y_Mvr+DZydvs-{DymK;LRpJj}0wH zh?c&75R}M7nQ9I@;tNB0xgB@kPkPwlvbgSirqH_Otc|U0H7a)`2lI=)YOx0OU+Loy z|FUx+yb-yT;CaiOHZ$X}v;RzWZqsSZ8q{JXgG@G-JBj=RSl*479O6fXyq9mZRD7@& zCLp@Bmco+PzLnlxTUm@&-99ok73Sc((u--uBOW0pzGN**DC;!Sv$HjxUwLH$ZVccK zix1-B910tIdk;*%o!f-~%DO8<_5GcGK)WTMyJNi>lMuJ*{u+E91H-rOG#=$QD@6%{ zLSG4;L80Bdm**l5eZ0OpW?R|eIdEo;uwr5;M>HcNEu3?=+sOuD+fh z45aa>Q_hw$;p~YCC-(0ovQ3$oK13)hDkwyKli^IvNlQ%J9Ow0ilDL#+AxRkt747$n zi7~@ZERY8pYuMW#gMtPSS&|8sJ6(P*B8IRHVO|E?`}7G+=iM$=beWD(4`QwgzrfhEq1!|(6Xy5H=6SM#IW<@)-F z41@WfxT^<6MO@9+SJ%4ASNM05kNAZjJ!<>T!N-}()RZmgk9*H#xAB7p&rH>|rMk#w z;nPwRqd%suh4`)%#YzXYxBm_d=?#%tJxKNWvs9C=PRDKErGTSd*+WOiNJ=WvKgD1D z4!EQ8US@qnQhO!bx3PAabOw-k;lYu44^Azu;MpEq6Z3|rBl4?h_v#KfC|vH&o${Hz zO$N#-?Z(V*U4<-VlS`LMXa0_soINY%H?aD%!}?`UmV>t~IRoF=>vL~$mD%;o@BP$Z zu;Q&^Z!h7qkauCVX`_!d=k53FjMu)sK@M(RbMs|mq4bm#mWd9QnsQH<6g@AmeI9Q1 zsEBqjFON;f=jOEHEc z%PVY0?ljJctQqU}-8bKONbcXiD$x`##PP$Q@5O3XbfRReMf+|u(}_N9|6gfwsl(BW zg7K{jCgGJS*_0;h^GAs5ki=a6+U~LYh^R6RbLU)9vM*^*fbaEfN=e1LVhj(Sndz%f zS;!a(cU(~z7-{ZdD(KoM+UF>m?B48K`Stv*0^b8xS10m4yq2bH441?XWT^i>+RIc> z{B~tm|Av}iX%A^;`*qf~1$*Z7oS#{v&wFm_o}XRst$*dwccLn(v@VFjN_S+eDQjf6 zLyVxo`dbodWZ1*Fp-HBx`tMv#Q&EDf7SH#TX%{lkD9RA(Rpy;(fF3EsAbjNG!+^vQ zf!A;US_(28m-fS7QS>a!93bIM<1?Q`Rg5S-B2M`y7Cq8Y%zWdW&qxWCinz+3KzVP8u z{|8SD$;g@uZww8}7}ut4JS00>u=M9mVF8uNn&jS+NaceOo6Id(zaJim4NCv+C9HG9 z?OIvsH0yZ7rR@3NJWDFSSx=Y=@1Gr0+V;pdA}pAK+R-uZma&yW-HBb-a-6d7dB)RS z-BCvI%y5~H;+7_><7~kTjz8Dd!_Ka{9{RF&kdDl8q=CXZW4dab?Q_i`hZP?-t7ha| zXx^Pq+TGHfQaw>=yxchYsNRC7%p{=PH|F*+8Kdn7&dhEt%Amf)yi~bQ(Vv2$T!3XE zVwrX8Nl)+XUw8dT9Cun=j&vKi`>A7xa)lJ#ALDXA+a1x_qi1-nZ6|N~MpTGW>2$nJ zZtguh9bj)~N5DpjRX3uZQ;K!cY6wF;+_8Ioe4a{0z0(5Xzzo;e(&}83qT&W_$;;;E z{@r<2%26>t=4NK9(4G+TWX;u&4>^IP5`Fmi@tr1_jXrt@8yj$gBeU(gb6eCdUMKd> znwk%-Y2yNRvke2d6Br&I$uGdAhDJ0C^n#I5C_GJ?b{Gd_7ft(4ZfbFFZE3kh$E_AL>`++;kDD?1w?`E}zE#f&^9r(Uu?)?+@Lg z{<1e3wx!#*Z^yS-?_7fB#=%4bO`}@CNKZX4i&4B|Uh)sy40u1S&k5D03 z*CWi*ZyA&$S>;;euClX9EbOlfle$-4zL!f?D=8&q%cTrfGNf;TKBv6YQKXhsQL)0B zCfDL7bN%>n3H;SvM|FLWxyCOn;yO&~R3E~OV24inb_InyM~Tmn*5G%K{MdEpiH8q0 zLnkMnTXnoxSSYOf^yHS<&D^qB(yc8efUZ((-+ALsOw1FT@-DsOUNf>^smhg*93_0M zKOM=pE}cT@S)Bxwj(lLHU*FxmOD;Zomr7dv+5^9BOYxw8JqSr z-f@?iVpn?Y`@1Vn8$MF^l{U$&=pR{MsdV7v<=rD75bNlicT0r}wnOe6JDxvya?w9O ze+R1gPFlSZEzWaO>KoK6i&;uutJ3h0l97e6df#bHrF!_79`vGmgz+j`rEWTW_!uH6 zt7<}$nCT&z97tQ`d%8-`uk` zK21J+cZDZY;g=2^*)-k@qU)_*SOp#UT4L<#ZYHx; z!K13@%MR)v8O5HN+uOHEjYy@p?Xdg%OYW6j`f~659br%T?NdfPN=8mX>uz-dD#|kf zzAq^*Ts=H~-r&KU>cMek1C@z1mJ3}=;j;|B>hJ1X8J_Pye&vPx{iF?77SReXKbxR+ z$LmFkp3dB?$xlZ#FO`f**!(SP!R8<ekOipSYrm^=JpRI=1x(+hoaldT!{I&a)dD*`K(X%R(OPBwSz=-PZLnXKPe=0Kdc>%N9a3dl&OMvj9W_xyROhSt~n>FG=_Q*`1pGCV71 zzeqf+j@3RBz>3iDyNmq|w<{~R43Bu9Fu!-8ug@-Ci&sOtH7r{6sMD%@#jn2YWb1$9 zMuxs$k*azc^zYz0eR$`&|8+1#6Z81IX*Y{rGVPvUn@Z&FVP87AK6KOCOp-=qV4v%k ztN9&n?2FfVQQfLeG+e1Ft&E|<^kv@=mBAp@z^`jO<=RwaAIr3YH+tN+YNp*D93Ca} zPTho^tRpekI3QtaZ*{jM`n^hw@UfUrwL7xb1Zmj37V}@XPOOC-F80mYdc@$c9dkxY zNBrH2yt2Fs75?g{h#wg8EeQ^jamjs566&&iW!8Cju+5>ATltMgTcAm%)K2c>5?x}i zW^VZYo@Ub%gP=z3zfn1Sd-$an#>ShI#8Sx@`RSwk z9mt}dj)j***KNG)Z>sj7U#N!u*8O)u=RY;n>`VAC2vG!#&x2Rkn|S&7c0!j?_mK09 z!YqcOau_Mp-e;t>5D@`4-VAgL`1~DI1aPH80XAqbu*`sg`fb?Jqo0CQ!~u(VW@e_= z%cG0{&Jn%%V&#TZPmc$A8(|BFX<}w>F5-j)lZ~;lF?e=cur}ohW|Q{X?@&%+0Ix0w z>qklV88Q#=#l^gTwy*TW<{*SETS=G(!jT{@WjHS6Oid5o_74!b{7fkTv~_4sN=rSy z#DH+9zrXYLeC!_Ctj_AoCmi3aQnh$;P$|b$aUe#CC}`y6{drv>c_bkCDrd3nN=ErGm8j{>uFk4MD$rwSzB%8zOE zcgXH>1ilM$o{(XLQ7XE-7c1zuwBmxNrKNRTnlaT*2M3=RY0Uqs*1chI2vcOJJ=x$; zPlTTyFL~D{+lhTgmrX4!?gAk~a)jO;}sYySf&3=VCE>8ReiP7pu9)KxAfmIyTP*UAE+OipZuy zp2B@BEAuRN?b=1W{K8B&R0+E%ARE{0wH8iHPQKge8FWt{dryKJ23U#wSkk3JT(kgB z;d{Caj+X4b5nXNVh*zI&)wbBe^NoW0!NZ3w(=VUH>#CP;O#@r~w&X`Y=*hJ%TsTAk z4riDj@X6`uJbjg+lLlMh5hNfqjS}ORLgZm%nSS`wy|YXJuF#Sa$o>8|){p^Q*#8+GDax^_%jFP&=T|-OjHnHbL76bj_+*~!Htd`#|KERP;ZAZ6t6O`aQ zJUpjmWjCSvQy6(gE}x+zpVSiNg8@q2n%>S_L+Dirhtjoy3VT=8*P>W65U)HTC@>D|jv?L1;KcsYI{*DUHy=6jp|_{>!HF@39 zrQn^(TKl%P4%fk=%FmN-Ms9ALh>03Lr^^L3uw73+B_3k3<*ck}mZ=?aHkmwSx>mt= z8|Q9C1blmNofJJ4W-+ZaGU>_aE_mThc@Ew0&|#nbM!!p?S*3D~|3+@p;Q3MP@^-~r zckMPCy7$%9++HG+$_C-+#7A0KpxN~DbO!#S0;-Kr^PtFq>u?C&fr6I;~VKrsTS;_V| z`&l*~28M5}9vI92Msk=4j&>U2oV!FucETlxtz-4G;p*t}af%p+F-bPBiSk2}e>HdX zFqPVN6?E7>TyL+Ibr`v+r++kOPA2klQ@GXxw`6*<2R8-TZfjQIlDM;~KF7V(G+*_G z?Zfc*tgou!$BK>a-tN6<+55QOLT~+=aH5WlBFQC-Xsm35iV&EC2?4FtO z=@@_GfwjLS2zie@9$k6w^y0))+{QCMwpDgZy}9fgR|{yZnksI~tmNgT%-#N?wgqoE zQ5Nl8S6+r63<)vNFnj@o6}jp9+;NwqZplfHGrl4@C1{%IB|dfFBTo*$v$XWl}Vp}zKc z*ENxBw*tneFFeqjcP-4-IFj5x+zr3ZzJD7ER@Q-6RU-#}Ml6yRUJCum+uHPOrvbO^ zi`9$bJxmf0Jx&x2xJadnjeH)+^m{_8id-$PHk$oCz&+GD9jDkEM=rOsjgpxRLW1lc zCJeidj0RcfFH}Qlhl>KvA&Lia1Ff41FkD6>UjsMP9{RQopZm+AxR7q#xWRtvls>(T z$2<=v2nfkO^vTYSFfIaa^5EGsn_wcH4z>frLWF)*C4!7`Cpk2xO!(Xncnp3K_+hYe zVKpt{<=VcPxF-l|t%wNcT#VvVQ|x;Qks~ezhhP^Mmmz#POAAGn)ZpV2t}o0PP8>Ua zQz5uCi?QFYUND3> z6fJv{h&|K4)OSgr)h;`!q?W++biOMV76%U<+KM?x0QA1x@ro>4;eGn- zSrFDOlzB`{&>0YN&-nN%P!_jQWy6x@6IRQPtqEjK`**4bCr}~&Ts)|eY**G{HUtZnxEkX-A)bmk$Tb77HLu_mq_-|202YfhAoY({P#CMmE zG+(>Ac&b0|k*uJM9-wkr*VQzJGF|J>WK zunb`%pBM$Y1Ky|Z-|kz#;Nj!@P~p82-x|dIZU=Gzz_Pjd+aFiqVv6f%7 zJEJfNCSJ$`gqf)E3tKL}3I~=@2c;w$sLzx@$x)1+* zj6BB{>+ORlByr$AS$DBER(~ric$L=rzWpL~Xf5Rd(R+iWPR_xpYb&X?ucgd6U&K&k zJ`-|zb9Ab&`);^Qh5yF-bZ@a|(X}d4s;YC%n~O|vpbIM~-kdLst*qqC&F!z^j_Z5Z zX)?;moytHWUAl2&Y{h#g*~^#o(LRy&esQZ;*8L{7_{kltJMR0;FGhkBdbm?4ndKfo zz7eiTnsuteN(Dp9UwW8IlWCa9EOn+z>}K>?ZruiD!0V+>U)35^9CEg4n0T`=UpRYq zraDFSU|0UR9uE^WwGJ^KFU~J7ct0HXc?bq+Lwx6f6{7+4gG8gYvf#*Pz;Gh?*+LWk zi;Ydza0Ivk8!H>vI7EvoaP&yp<=AL%H}-b9GQqz#NFfj<{f*jpt#jYQXZrSqeV@+b zYZE`jHGY|tn)?F*?eME<@rt|W@_deVnG^ezzyIb>- zw*S3gksB^!m)mxGNqM}i-+SF|y?(Qb32R{9v_<@+&%g@nlMGt)W7(_Uth>tYhtDQ* z%F>P8-N=`+a{+Kbth&&Fjpw?fa3s*tU?Xbu?$`Gh=n^ z++~bBW-@SCuCC~xm3kbqAk+TX_jy{}7USh1I~9Y;jmFu>eAVgJLe-sT zU8)j0IZwC&OjWnz{=$dwuxlS6Jz!rO+mhe0C}m7T*0-6NhV+%*J)lIo(^*ROwboz( zkYMYTC4Qd!N^R9G5AUn8-jhwT3W?Wn9nig0>@04$#_TjM`U?SRQ_coHLVY=ZHX7wL zCa(xTE}Ad-a+;xBnx!>z>mYFM(-9f~U&2>Wn*n>3;;76sK}j`*or zt^4rO3#RMQ1e&ax(O7=D!S+y+FXnu56$RU&q2EPT!&WlNHNRR30HmkN(SK^zEzqhW8Tc!1b`|$5Y-Gnsx0PV_>yS?1332EQF zX70KbQ+2Gm^KVWf7d2IqzeyFg@rQiMgipYLE23c!)3kqqOf1zt6N*0Zg*_&#a~E6x z#0~VfPm64rlT5W&Bz#ry!}~7Xxi;damN2NZm+W%Kb6)o!+*B){Niw2dHsgKb-Zo^$ zYZk*o8`5OWZj6pAAI!gdY5TpY3$=C_zmpMh($I@}PtFInmNb7;nRC=AnAGQ7z-*PLwkSvTrZ;tk~XdwL>#jH@pKf?1G7z`6rY8Sys-f5 zRx(>W-pX;Ac1vgm>X|Y6oAO;O&f76@GD$aIH>;DV6x^3)5_mvb25VSnE5+iP@7X4z zamHd^lb+B-KwP`J7AI9@#?d-KLfs$=SF_?Q*fQ0KghS_=xP&8~hUv&}cG=f2WPDkX zb-+*qu0b|FpW3+47fAOw&xA#(<_?aI1`vEYD3*&V*N#1Z?m?a@2FSiA4h||MI{kRu zVS{kSq!56&b)-!^TDgurdZ!7%px@fU5L67{542;kDV-AlQLU-auguA8I2V^zES)y> zaDoA&3hNG)-)OK2z&)*Hn+$Gh!kPf-b4nCMi2dG+-<%BWfcs#qTyD=&$RPxGQNlKl z6c}3oOuLRCT$ATLFd5^p^rg0)u-fraHp3z~B(V zvLG{{nyOd-Hu3i;ju@ccA^z|fNovs_N)Zgsre{_2u?Xl&Sk?XR7`u1>Rd*-GIFib>>Lqc18Yij4^eEu_;jf8W3F|tG>s7Qg4hWgw3+RE(UyDi!V5NaP^A3nY$rAVg)B#aLw}~vWSN)jG~giwbl`b<2+pb6A|mR7 zo}L|8M6g~GonHALjHpMZH&v+87W^U#wQ?_s`}=t5>r-iIABF81rg61b7p|*3e_8vO_X}}P zLk`UOu&~~(!kLN9tJx^evEk_nul2vrO?`n^P`kXbZb(*@Yq1HXhVH$2_n0pz8Fxrx zLfO-^>xL`S;9&gw@Aa-x;_cgphB7sd9LxKu?(EEsOlEl-o1^37;@FXd9of27%&bRP z*IM|?S1vJdV?%+bpuKqUxAnEDI~f^!y}TxAXoi9Vw-=eRZD{LtZ4LeDtu1lbf{K>b zV|@PS_qM_D_RR>upw2H6eb=pjTI0bc^-tD4U0s)fa=@Eaw0Ge&i4?v%$#ji*YAR%6 z(9*y1)ZZQS49g`e4x zidFd+KTMN};XwIoI@gG{nH?^-Oz!kLOX(V&6T&>XMXPrc7WfpR2i&)czq9I@v(GvX z6V$2?%bPz3{+S0iY`2v-DyClO@Aiu71$Xc6UT5kjEj;m(MY}8($D8}xC{g2tT`Qb< z_H)O+xR!|tFDxF0-q(E_SjqI+wbY^;eQ$n=+xLu%4_(iJg0ZjT3Avi@``gKD%y@8f^SLBJa>)H8-JAX1zV)ErCz@1JLIvao-VZ9zhVOUj zsTWfZt>Bpb=cATCY{$Pk2q+03ar+HU_*Jxf7ZmEy|vs`~0(n;=m5lP{EUAWvkgfUP`v!Dn2~ zCtr>FuB=`3zm)a5l)6CiGZ`5qH5LP*Ol>1%r(bJb{!**a*u7tT(4(N{GcIIRd)>-) ziN=aQ5qF7un<9oV^voyUimDKHY!1KamG^$WEF$uo(423nIVohOShbf>TROTZH&-(e~Z;eYwPt^Ca zzH@$?kjuDi;e*CL{qIDH`!9+-&Bqt-mOf(0KY2Cb0`KMNtNwn^rlP)ozTM<+dh*kX zDdg<9fQCbkR{)Hh$(z|Z;WSva>Fu9SM?ptE(Eg$}1WMy1=m=oAJ5e+5A%%-GfFSud zG$_BH6)2EYC7o7|tRwRUs@DahfvXk7Ox5M*2{Uf#M z#rGI1C#uAv$m2xs4%5zEUL6n&05H7#v+)_NrO0*I`W5V-oZp6-fz6`?E0|^J3jA;1 z=3Dj+ZhZ3iG5#S#`jL{=rKI+&P79OznvF0eVygZGZYFvF3?RdM(EZW`PW6#P6bwfs zC7B7$wxVK_)CWia%&?@;;NiZ{OtRZ7_lRLDaF;m-~hrHuFNiOPJ8nPyieo& z{QSUShz1Gp>>k=blRorUy5Zo_Ff_EgD&sv6qEv^beg^k{LV5W&yKW@x=HS*WE}=G= zl~Ep^oV>rh?4j-XDk78Sp`N$5)Rz~h%w2zvQ{u)Y<|t^%N3PT*#V_9ReB0lffl-f* zjSWFs&A((HHv#+ zCaxEd?V+W;4L`=rIv0FfhQDlamt#Z$r5MzstWs>t?lBz7uQwkqvwH{*ZoIx3VO5xs~hJ z@~PyPkIBgBcQE?=iC8in@X?LjH2Q^fpXPqhEb|``41n~qtM*YbEbX$on)JS?-yfN$AzzFV`_^BzncayHvEn)HOv{*-4Y6w!WLbH zqJ4iH7E}tQh0bkh;;ohuFzbI3tCl8Mva_GoO`$=3`fqfkmjvzk_AxcSSJ{s^PxR2; zYLREVAf#t}Fk*18;M?is_vgdnyXvS+e znS^8_O3G6N<`tV@!pV5@*dhMFJ5W&w5+TEwbT)X=L`xNI!Taiw@o=*KgZ-wG%Z@&!S6$9*nSAN5SqhcV1O>Zvg^%XjobKiP2PDZ zLnOb^d+@Ln1aNtwtnMpz+5!o>^b$>%F^=Ee9m{m29E5>mU52x73r@i908d-rfDhs0)BI+ zzke*W%W&G`tOc8c7M!ZF8{$NsS)yl_dPsP99gfc0A1)8LU%Yss@Ee{cID?o*2o)Os zCT^SOs*bB4?lTHOekUj<9%gq-bG)Uk?a)=cBa|Zo?>0}6hywwyM`LEEGN7$wEY~7Hb0M71QamDV_ zu()gfB6$Er`)@H%Ao|$1%lFhmVh+ugij1pklgtoD%UO4=>Vr~JEYPTJ$xPFFeFCHQ z54E+=L#ZM3{@;hk)51c2SfSBi-lg)sY$fViTd{Ya=*aAd5?NJRmYSk_@kF)^OhZ;l zrKPD1KcTj&US}2-K<|T~I=Kr6{!(I74j&b}w6sr^&){0S%NNNf;a_xE{HA(F^F;6M zQva&pukpxeOH5(e^q1A&t(Qj|nRW;ZUOyfgKP;>6+uXd=l4bSjrrN(J1v(Wj9To2X zzCNNxbM51F4tLNYBl~`y@o72T$;n=rriXggS5jOr=Tlq|`KmU#{@uZK*5!0#&*gK@ z2|+cN64$b!ZX(EH8axVj%R-eniI))(ben`M8y_4h$^l-2s44K!gByH{w;zaMG^Zn< zMF+lrP~HOC#sA4P32Pth*sn4(MQZqsO7B1xulBLC_1(KoIFzfPwN%aF-Y6p+0d{tF zhZ@6_p2OW&=r9l#F1@^6Jmw3%jOPL$9(4l2L%8$T_bl(=_nY@J#&QlTMUTsN;A<0N?re<9d)_ z;zOdF{6{YU>%ewO%7;YB0=I}Ioi%I}j4+@?@^CVxqwHQQVN{m9P3kto#>FMLDTE1qswjr>Rf2gYih<*!*$Y-#1@Akm$Er!>F-=u^eWbjDIfN6ad#$A3+{B9qTQQE0bBT@C#ebKV z<@h6``)y_S5UX@v?Ub`Gx|ehyk%YE2WU54(;QIk*%+AAe_I9IEWJE;7t8bl)v@|63 zJzFuD7=d2I+TQ-Ra|{X%bDaE-*h18V@e)}(IP9zOn4aU>0sRCxC*XTh;;i=0YU(L^^xZO{#PO3P{%|lQk zPG~GN8+&>Pl_sm>BN`455e!NGd_VUOsiVKYU$5vI%SK&b?JZ;q48zMWTsF!;Lvst? z0<1b`A2VWEL(Pqu57-^JD3XvYhVumHk$lfRl+piNG(fO9I5{aPSWjCXk3ZM<_9h*e zAv8uS(;?t-;#3yE7Q|TnzxJKu;qBW>T|JlHwK@;xJb^G%!Yvl66m^q%xBejJU-S1a zoucIENN##-f)tPKCj0%j+te*d0zjA>iic2POY%5AQD(A*3W#}IVP-eI%OE-r-g z^2E(Q->t5tW-Tu-^Kaqzd1QVypmwFICcwPYW*$q)tz=zdZ?b0!wg=@Uj?ZUPt(4v@FaK9FD9oGjrREC$PisoNAVCGUSr$JU zuE&X=X!1C<9};nY4;62Hs`R$=V#Pf>i(@mj!~HiL8L59r(LHYLKJ@!aDRhX|JQH86 ze)Vc<@SGCZ9LoCS&Wb6HFT|0_Vrj?8Sxvuh>n4ILwv&7d-rt8B9MZLT=n200!qm5; zM|%3Yx&oy<7cK*ul3h^nFTAF`aTG}0bO%texpbXK}g?Ir`(eLZ(&b&__v$Z40?_7gyKdp7MdC%Jh(k!@Uc}*E2KXg2q3Ta3L%!^ARK`iG`7okqXb6^UzEM z_h2BHT~q|BjR*EHHhZ?dYtwms0pQ-Jka9ahqXI*7Vtou+C_p3+H3st$5K0^syutbg z284xcWU9+d@=^U2V(E%amXLNjW|IU2leqsH)6clNim1hiTQHaaj6*DYKfZFno|w=e zVIhVMT>u`H=Pcoyzm4v}OD&IxBum6BK}3WZ=eY;7OLzm^f@7?8s{;0FAWOns93hs+ ziV-ZTGh|NO?iK_OgMh?|h-_JukIzv|)_MouR&sK3iXOyQ(0p;2>WqMoQG9cb3th|> z|96)QB_STe3mkJ|G22u9RR?yC=U5fYHTCrw@x>l4bzK$)PndwOp-Owu(7kajpLFql zH-HbfRbFy#-wa+Q5nUCJ7jJ-sC*dBBKgguQ&0I%%TfM)NvBS!hs1oP_Tyt;_;4^QF zeg4NU+~VXo5*_%;muRHCR|4*mQ-##ZhyV>q-Z62)Z$Xb$1YQfpK z9T_9cAu0^YsjWkJ$x+-)FS3&8z-Fn*SUJxuPh{=($Zl1j$8;GCs+L0sF_6XaqX;e& z?^7TKuo)l}eGii7Y3b-R4Gr&uG@#8VD!LB?ne3=~C!6mOWtso_nv7u17rY&a@xp}I zeDq7q$NG9%99TB`aHwO6V1&&*@7KeL(qN)gL036n`8+Z*08brdUgEmDhX>k`R7VAq zh*uFe+O8J7g7o3=r#l`KYsPv828($qKOWRy0lwt}8XuJ$IPWo|xSvKja6>%Cx|00H z^u)(MPy%3D5om^39$E|eCi%2>6uf-)7h^qLYjEo6A!k}5fv_14GkWUtr zlU*$0Tg#1)2WRQ$-HQtqP{7+4(g3I_Q@UU~BbZW}MeKfv^QmBIH{awJy%_gmDh z_uzyg)Q8yWprEZQ_8`3(EAW1Q`2@l6LrejY_W%Lk(8PqqGG+J{7%9-$D5|N2@jgX9 zCxU`0^Twe`n8C6pn&tQ=6y96GGuWN{sJf#z2^JAxub=ne)~B@;^1(7@2ui;9s6jqI zMMf!N+YWMZS<0z4exID=yDIvRO@xxfw1@g4xIR##93QWq6~^TQUNGU>wd|a*nHN5J zGN=b44Z0MEN9@}_HZ;fszxyUFZN=y~G#M!LqL7clV=20heaG`}4oH=K410*o>#VGO zq}N;gh!qS5DZ4-jvy2-8srbB))FbPBMEo2CaEqP_f`cnpUSi`A%C?(L`U5K2e;PF6 zJV6!qzw-o?05EnfXUE2@k;B7u{Z!o}SUf^c647V?=R%eU{Y`UEkNC^Fqx3t#rh)7D zbW%%IU0p!k)+@76XFP<21IP)hu-(p1PO>=Aag)LVC3K#VI9Z5~7QK#^gt{|q-`?T( zCSv(o)!)D4*fAq(>t?wLEuYVGxnB#*1GH=NWE>Y@sPC0S7 zxLmR>@W>-=%(9K(K*>LuY-5M4q}xiI>#10kKwv_R16bsxefNHC?k9O4YV7 zeKIg@W>jt9)V(?=(`v5whl_*h7%BPK(oZ)=;JqK6g<7?-!q=L@|Q3v{xWe1nD*f|+C(sxd5JtO zu%hutOoA^2z1oe>j-DW~J`$A*W);Uxb=ug})z$mT+?v`l-@JKr;=p*u;+<3s4v0#F z#7{1%!<9V(c3@!h)_zGf@nLASQCp{aRaoy@!S zbkis%#vo@17FY-}W)*}AiO1|pRgYt+L5fSZ1OpyO-YmbNVpFyjD5$spMPao6N@#h8#ikbO7+85%bM2K+W+*In1pFcTp zLawi`6Zs^lLD(Z^*Z*i>gu%hhUDpQPu>$nQknbF7e6zB;24d1byG0=0ewt$E=#^DR zj4chzJ=)v&j~?|`OFbYu`t2zZYa<2?8%EoWPj7m75CUxATO!$oY*3{H_23)B#Dgdu zc{n*5k~=2|${b##piFzoV`5_a5z>zIf&)J$Uln0Mp;@-LeoYekB0SHi5+pHl#zBCi z4}qXSQXN&6a?Vb$4zvvb4QS|1_C}In6q-t{syHKaCNk}!Dz&=fR~ zg!|B30kUdBdnW@6n|%L0DA4}bdLT!#d$)h>Ry0k-+&(oG9m?e@7yK;$=M%$%vD)0| zzvqQXI8A!I;vczZW>R>;E%2T&TRWr@YbW#dwf_D7tcN;jc;;>6CQ3`X2lrL#-GO1c z*7~Iq-Dw!~2#DF4Gj#`LI{MFtl|Hh(AZdNV;OAaR%87N6Yhqf5m|QCjJ>9zs_f~oz z`RNcRV%N)Q!~1x6`ilRTz^6SG;rbT}&mVewijLCZ++vI!mQnxBa)E9UbLRNp@q`0v ze@&RDu%UjIJUUu+_V(?~UVnc?60OAd>kUgg^_cMNV=7G`S@x+W zUJ-_%8s+;io&?lhEI)Q0^9Mj2->#qc%EB`6odj@K>NuK-uxf zJf66PhS3pX<4^mxyj(HD(Flzz9E->w^QZqstI39+14zH7S0k~e0Dl2{d0I#HvCu>lX5=T3pfOApZ zGw>eJ7NBXy+M7UYK8lV$dbO;y^!ocz0fMlAt|qYyX3HU5eyVc99RHXzDVP5IXecp> z86>iHo;*1m46XuR!E=fC#}rY)aN-F7ctEfw)3`hsMNQt^!xjxq&0DBe0BsBz1{PTl z#r><%xkO+yWDMw;hDXwbxI)M4*L6GX-_gB(YG5IESPY3$Pv5988X12ez|>>-?wic zQ3wbN-*}vtv_4-UBG8{D19;I00pUAxWZaE~z$pRQOh*@%jnpa4-hqL*wV|_Gx7nkN zBBKS&$+HeNN6liJfwL>f44b0Z>`_PISwi}nkugkZj|ME(_B{>9cuRkO7+$EPJZ;I^ z<2cUoR;IO3RH#0D>=-rLHOpC)GwFIcw>uSaeDdj`tiq;^@O@%(AidHe83G~6|DIPw^pO~@|5d_+{@L;Kj|tr zVBz;khY~wbchzTi;=7FlanVn{NY40e#ryxC3D$b`cmA78THnvr%^hK-4*#XRW!7JQ z;(=!gea)$pP|_(u097h};{k+fPWkupB3Xn5jH4>}_W01eP>9@~oxZD0@L8iUO*hu0 ziVYRuwTzKG2u2@qPQbeI&p83!RZnlP>MVgj{SZV)R4^gsgx84ylHa-jHlINFl6*QP za1G!j<+2-k3&kpiYa_VNQuF|D#5DY#uJO&gaam4kaBE}FNH9+kJaZ)&8%rXN5>sN) zn$przN4$|S36=flf`_=5fH$h8ZfO=8G6(zX!!u2XfAB9>R@^Wy3)p-;gb`ut>-OPW zIpy412W{p_((^u9e1u&ZZyf@O5^RL!D+HGsuWNm7J?^h*@SGm~cpX(|P|rV=zFgA? z1TKlrC*wW;urbNtvpQiGqS=U@8rHvrA?f`i49saHp zplKK}hDMvuKRguKK*Puwgkfg18IeT+K;|L3e1OXT!En``L*I*?kR0shuSOM;nlP@z zewN`|9iCD-4UGqP^h3RgFA{0-SOhmd?Z#RNw7$$3(qR0nUJ@Zf;YF);am zbe(xv&HLB)<2cB0NQQ)vlrd96sDoXUip*mql4Q;lWlAcQp$tVib|hnx%w?=hNkk%r zA_<8IP0wp(IOl$z`|0w>`CY%Fz4!O?S!=zgMfpxF>T_=#G^LJHjcV0~1ghGcrhXXo z>a}qi(8E2QK?aJoty;k;G~C6V|C2y)>Cd%8P`h#*TcL}y6n>QhiyjN21M=q4^n8b?8sKDU$>Ki zS$ew}n{TOPU+URc*JG^viTaVH)Ay#%@|_hrQup$rT6!~g+|-#@qqOvk{=xhIwB|dT zt)J28dmy9ikRinOUt;5_>8%ljIG|16y!leq{ewWrD&rm`8JOc_S?KFqC*pw<%dmGx z&NAuOub(`@_~b5TY1`B7N#|zk{_UmBojVJ;>8nwx5yp3mmk1{Y-D#jth#bP$Qm93= zF%~MfY9ZD$eX>AsA%2l+g^vR+<4t~^o2zGfNRhe`fjJ*BKR?e$CT64Hbw-{2=oT2t z9j6uH3A7BpwrJ|mf5sd(eHW*DoNak&C3j0=Pq=V-GQ;(XkKFgE(>UDK(F5 z4FB(iGk9!`D@qp;Snl(J0s0a77%SZp&9-#X)MBBaTzxSceSCCoJBY;&@TK>nSH7>m z7I|}v3T9Sv!(_pdC1y|*2g`~I6zbcE^GSW?&u?= z;Pslov;%0dU&lGk)7RGr{NS;>cBYTX2^Tg4JER?FGZ>xt>tR4UiaE?pB%cx;o^m$> z;Up2e%101=FjN2?jzxH2P#dc&cgDo@9204qWZzfW1d*oDuFivbwavllPA8>#+2!)hu2h^>T+PUT#pE5L0HxF+690@Y3V zY{6$xM#vszXp)1;MklzL#>F@_8r9)J?c8hMzZ;B+^6%$YWBa%%zhgeVst<0zxazNy zsVG`W5Bql8LWBJ8p`+G%d71ggj+h%|dWnSJn$X5HuwE~Ksn^;@ir})7;nJZm-@Wq# ztgs`XkzlLvuRNP}vuF3!*FDp+y`*__r>Ex@&XR}7((<-Ph@}GZpAqcq)t2R}{r&$M zSqkKhgZCR5 z?!6VIU%PSKJ!jlCsM|8gvU-;;T^crS96XJm2{ZQYRmso&gxrVaml;E?wcBQY`0!ph zY{ttEv~VNeGPw(~I^pQ{uF*|^j73uv`HDiORcG{6lWg;&3Hd}(^Xg8>MlwZEuqM5&sQucwm?PC)c6`EF(^xDTvDFv} z!jCQi% z3tXQc+a*6Xve{t-Li3Srk1Tu>*Sk+2siFKt^BuWn+mA#<3`sq8;>5I;ZMO!FIo$KS zL$jt$Z`j*us{5L?!3WjK=j_?DcIc|^=_m|tas#c~zP+)HbG%QZ>{n;D59t7+e*OCO z`jsHVoF|6k#*a4-Y@`<3Zb1FbTeq&2KB2R0*_OQN0b_V*tGTQi*2}=)7v4gMydo)~3Aqj})<`-o4w{cq^2l~>QT_rq<_7%V7huYe;AEb@z z2Tnh3oEN=66G-&{T`QjVUinRGx@>AWg(i?@&$F|hz|&j(+ot^orCt-2X8iBC>M%3@ zuT+I%#PQQ7-m2OlD6heI2_)qG0{#SU<8K60eOK4pLY?9N6cI(uwNrieuniCaG&w^6tnM1C1(XqH)W@K%KMerxGpl!$E6Lbls{J*7 zR`OL&F|L+chxM2}CAEk(4W^U@gd_WgzWI|rEGX85CKV9*kG%e8S2{Yj(|gon5M83_ z92#oN5TnsluQN^k%%bzdd4XKa5-MJeyX-nSdg8w$bV-M~^ez9%yC}?S@HB378Z`a4 zZ{M1|T`oc(1apEpy`U3J0GUrHQ#1#<=c3~oyoOLtID1g}T}Nd+WCLYJoLe*Aw|K#n zQKk$?*xs>2hgx!cBz7s~;X^~^o+vPO zAM{^p8XE1a2HFpIcQ=;kZKT`LGiVPVot&jkgQ0W%_KDA-85a&7G>1^yHyS+~Y#8Z7 zU(RNrOAzHgLY9P_n5W*?rgq)gXbuzqcHJPmd#rpHNDlPd%s#7Z7qN|kuMzy}Tk=VJ z=;xx?k^ii?H)Q?5)xQ!B3|I z(FH=%>^Jvv+>P&|HHWQ@4;Z*}R$L3b?=pvWv8Z2dH;O18sU8zgkpoCb_^`D{7_M|r zaV{xbJKa59E5_^ZxJpJ5jSqtfZ_mq^C7Pem=ztci7d<*T36~a+?$Ai+KoA1|+5es< z%|CRlf{Si1=EnH$Tk`Fj9jDD9yTtPkL2k{j-un-LYf~{UMl|u2UygO`YZz;5&+=Vg zR=^SbXL!3x!oDAKGhMOhmGoL`DnERfFB^?uY__#?htQ|t<4zb77&HST?J}BI8(JrT zZsWA}9DSrD-Q2zxNy{g(6WqHbZ=P%a@b2B!G|=|^W~Hk_R0N~o5aGBCnnuc|;7C=H z={LL0d2I0L(W4<-bk+Jx4X5ZTZ##cxK@Wn?uRYGoDA%aZ5=JBQ@RVvnoP9RB|PoSiaKH3ylSd$&u-UKfQ?VZg~O zmDOh#TwUT{w)IkuTK^{8$ z22@nwydiC;alN+rxRQfj#8nTqN03hT|K@n%$vIcuK#4jX?q21mR1&)&I4yRhYQWpb!jF{xmk%UCsB@oh7;(dHj~i`X;~0X&+TK_S$#uRk-`4!pi6t=rX|4nha=iX9;Is)ekd!Io|UVYu(#%CIL6?WTTVP3Q_&hiXwoCe`d$NaqK zWMz}#Xji(#eZAshlJAY`W9rA88^6vBs3NbinZvCyM<=Z39~`o|5%An`IGq$#9yK`E^XQB7aXgtIfx4- zv?OZAj2V(HEkhYH0X@9~cDH7CN?i&=vgu64Z`p|E`Edug{(Ev3O_if#?z`=;RGI(V zHIzke5q~eh=86s+eDdmzZ5x=NYw>tjBKj9gKMsX01nC>6yxnORjloZ8_&?AL$%~~* z-@SWRwSScq`fZ^yPn_sJ@+~(JRF}fp`G`kQiX$|m*udZ9=i7`JG0n#$X#N*(nvF?k z=2rp8(|p$u-U8eK$F4z4%YMsL!cwl=cdDKJ@?~2m>Njh715Af^UM;b@S|FCRa?xMthso;bg9{KBm*+?{*NlsbZuEyD#ZvkXaRT-Sz3slsQq~>-@CUT(sM#L+eJ5dej0Tob zG6!-=V6)bEnA=^`518Tj>)C&Q1et~fH$D~ z1(A{Rsi?Ze#sge%y-1cyrCXaOO|6d}PQMTxC z_Zl!8ZG0WulKxU_QCbca9njYkev4qU&u9HD_76ne$+gn!Re4-tL|E9>x9)?cPW{K= zM*S+tS=2vot9q_vRs+93!7w5qG~*$&mt%}4`D|F&{{5y6hgw)@J~SWfr5ntr&3v8p zHlstgZfd+*-cM~;8?F)7LrdFrd;3wSOPDT!N!D^++Q>S+Wq@aHaHgBg)+}SI~cH&ZLQ1e2L5{Fy&UU&{P1Df`$SDY zGxjdkl$=~tWY1r|T+fOUE;c*PZWouQI|W7D*KN$3e_6vVefFoS{@N7;hqx$e%#e_( zYAGH$O0B*nS#Y<|@2rwljOyJP$ssqLcm=Ah9VEQJJl}*AHtfN(vFt-DR(PwLn3$+W zM^E}W>>8JT7{u@!+*v{-VZ~*jRy`42hZn-bmC&;MLk%RGBl{tsN1q}w$D|n51@xk- z#uEu|^321h*SySYkV5SA>f@8HW-|raK(;Q8Jdd}7CZ=Z1nwz_Je3SI_X^ZYbmM;A! zjUN3cDjU^kZ54-r;RuduuDvucrooEu;~I8dwI`tg(Z{kP39v2v5n5o(aWzQuSM9OF zI99LD%ff$V{IF20TSt6QTQtWFIg$X)%5g3(MLRR=ZGA*3x_xXY8^C?i2J0P{4rlDx zjnlC8ac;tydWK9Su7@B+tK1C~91NNXaH1n7PjrJ#1^_A^j+~q$t6mWC zjL@q!j|`0-0Xo~$J$9)KCwscf#nJ}3$$Ju!*Vb>*qQBjyqd;m~LPF*^0S|^@U8Mih zdYC(F&YXB~OdDI(du+nV+BX%Q&McoZ^9ioI<56netqf85zlt{pGcPLMXx*9W-I?SDqSi&_is6sE~_i zxCL{JnLhVH2iD|97FD;;@)axE7#Og?!OJXJ)Cym+MuuC00h}~YRrC9Uo(FKxpzkm! z1|u1MtVSq`t}s&Zmd^bW+aBm=$-4>c4KUx@gi5q*hJhyUi2sjD#2!qI>g()=JYgND z>@Ke)6#UZrJ1Gl=s*!4R`}W&n<1BUH62^`Jqee9>DE>9ZATwA1%I&cDQRrA63CYd? zUcmC$3h@IsIydehtr~X(>bs{g^glTL5X=98E0jK3v&n0Sru`h1TeQK_&~c|}_?eCD z*SBvoo)iSDIf?~?0(b880u+F-+%=6G8j=51-rC;gD@`(J{75u$yVKrkEN~b!cKrDEPM6Q2roMYjzcN;CPMZC%- zTF|a=VyeBKdt-u%VS%+C#=6q}6w}O)l#@{5K_e46LWRsLdaQ67N zyY?MZTv&J+hUVKvQv)-`(dPYK8*(2i&!FOu8$GhyAWEYJQ^R<1`Vt-{RrBQzuV8W70%I zXowL-BJZSJd3s*S%q$PuG}FyZ2B4V#fas~~9w>hoy@akyp0&~5nlLV-!g`lsUEX=x znYEXvc8xD!Mx@C_YqLPPr8S+|GSpZ9;DD2i-ARWZOiK>!zbyag>8{X7sy$9l`nhut zUS!Ux+wc>_tN_)XuNE}kp9$-BNZ%&FGT~w^v)3& zswej9ot$tqqW{CJdfe@z_UO^$KASZ@=?Z27-;vj#nG)%n0h=DD@%PA-v>f2?C+bbe zatAvzgK=hIHZEF!xwOnJ{C?*vAg$;_edt^4(of82Fhel{82oCb%z|G5)U7G)A`N}m z3^J6j-@MuVX~%9Fb!0OE&&#JPPvQXH1iKKk3@aY&rG>aqRw(NbxZKgU{!BOagzS(0 zEQ)fBJ>oodrfEOqVMi;SDS5+Fsx6Inv%fEaF{Gj*gY~I-yUP;hBb% z+tswnGPt;OSnTeFjSRsj2h^ahBe%A>!rvp!eN|k z92nZ|r$Coh3R#9__0_7gexR&P{&C2btynQgf5)cbrxf>ueGmkXvM{s!B1LE_9~Khm zhdVWA5BiN^E7qlrjg5T14o4jCyn+n zjQN`~5dJ8n@6!4j8eVuYR6%1KG;Zuk)YNkv5@pAj6PmJO{_p)G{H8f5QYQlX(?2*? z4pHaM1*DI z-yRa;ckkW^8)pttr%;mtIG~yvxzrSE%U4Vjw+kMsULjALY^f{E7*4CL{FO}3Yo&_WhdsduR_k&5J#biTzG>*UYKa;UQzD@g zWq0#j!oSi4M-j-6UBEB?fhV4uXWPcwUPi_&wrC$P1H`EPAH~dWqNiOT?tG>U4;F@F z-wEn$viMCy`gF^63A)+cidlwBdQGI(XE)E$VS!_7{8X#dh!jb>mJZ#4&r?VN)h^2x}7Vd92pg|bT$U+UCp zemY>l`7>uOWo1oq8Pu^BlMa1UAl z^;1>;wHBun`>2mDaN7wFW#Co7iw?2r2lORL z*C&h2+T7)Vwb zC>YbapLu|;-@+vdY|XlZs8Dk=&e-`N?v-^FG%2wt3@c@~+)eWrFGhCjN+fhxboBjH z_wP%$dP?JChwn7Cz_QhT#DaMMm{&R;i9EW6Ip!$&%f^l1|G)t zY@tg<*ulZAw{&T7>pA&6B4~#r#;?SZ2*UAdW4O_CHmP&lNfII@Rv<~@oNnvpS6eL^ zg`BYuU%n*3HbD&hb#{{efQM~_aN_s*K{!2rT#HlGdQKvOObIlcJz<#85g=w4>J3P4 zSf8!7`7TDIMvc0ZmNr)DD<&qfU2^v_MMl4bf|1STsIYae&t17-;lc(nyMluOx?jil zG%%3f9fv~xqO$~i;BNzitk<#QVVgTRV40RA$dYNdlU4U&?8W;3Dwqd%IV3#p7pR`{ z^*W1q!w|=foT$I!(}Y@nsu);g`*^$UJ9oAtb~%jD*d7jvksZez>DqUgp7JsT##JC0 z)9mG|-xmbJ+gk{QTa(j8x;ISil!mLizp@NVsWee;nbAX|4r?w=xff*68Z)MPSW54G za{yVjA#dTFH2pk_U(OoH?+e`ZGycIcd&c+d+}sVEEmA!F+z$R$40|Q?Vf4W;#^o{N zdhryq3AAH^;xL|6QJJ)3&Y#Xtb?*3hIlj=S_aCrAx4VD!-|)L;_M z_iEKj1+YpCG*J1a zCIEgELaViJy0MMZ(_)ZYxlfgJHJFK?Ebdae7> zB(M8CrU+JYB$TE@q&2-Ot|G7xw1Ctm+4at9;Yzy-HFnVc5s>xfv&u`q@i7sH@WDA4xZn{;zLW_T3_G4__L>Z!J>C@@As&UKX>eyzQ}6;mkO4y zJ%d!)p4st2nt{JqdqRc|DRY;}?f;SqSK|rInqxn_pxE$-nNV!XHhm2Y8Uh20E^r87 z1iV_KctAgb>GWh?%I2P^#CzydWaS=(UGoV8l0=$`%U~spkhG`<=4vUm^GGh>L?zuh z56eox`{hsb*a>&whrH-#pU3^;_NAgJN){qc{qhpys4ZQuM#ZD3 zljDS<&V$x>WU1u4(W`?jUAxh|;~1-tkVT8w18 zvV+g6EJCiX+9oDh;+P>i+QXx%nhN}}O-Wuwo2_OA#s3tqFxcK%)Nk{$3>$UBf{eE7 z8a2A@9p|ZQTgka;_({%9vo{+nU191HL?}I-GAc@qISI@8?Vzi(duOk%Ob=>oRdjTC zyUhTDc4KrJ#oBHHqYRonb~(_@v**uS+lf zp7JZ##c?FJcF8+j1@fdONjN^KC*Cv=|42GeETwp5+2=GQ`Bg{og#1KlpYm`;#eP5w zE@X3kj-|{GSgBJBjQ-WwF2B&*3}BTmlH8_hc{d;i>1LBw9i zO59;PKf$ft=0^Y@^rQkO%jQ$EJOMA^PXJ>1^1??!x14CBLx{Okl? zTV?j_e4Cq_f1L;~uYF61mqurGB-zN8o$NJS^iN6 z??!L$kJX-o>H+Y*0Gx8uA7*WxaK0d}_WEfn-G-_8H3DJ;DulZ37LSM&2Eh}fOtPMl} zP-XcJ*DnJfNH+1@%b{`P5&`NdnWP0eOlmhhx!*#E>^&?5iFo?(Vf7H}#I)c}w%FU5 zl9F^gxL<)RLmp&0YE5hA2tw%9RIE>vX8Lw) z-(GEU?QOSSRxUsp$cAQd*JEMMTo*hP#*#S!~;>vLf;BIL@8hIGzae zxl7!w73&s}X%=nYn!Z6y`xb47UdD2Yc$#;!9yFopf|v<@lyT!vnJt8LdT{UF+MmvL z0?6N9_&A=WB`~#nD|eQ0;%;>dFxU%1L_S9oSf{egdF~R+uB}~_@-4cevi`r%fl_rJ zNoB|r5&+9eCdHwM5@CyMBq1fF-OOIchtFHua7aiswUlbzGylWrS{XFoX8mi_GDukd zs9ozCQv;3Ew8@du|082gLe{%Rr~ob$CNyJ3Wb)mI4;_5Q;_<>w6_1b#NVSWdr~O9U zN>X)X1F%vxXI@rZUD<0l@suUud1+k3!#RvxbvC7-T-P?a7qtOq8;bB~{(=RUgL}@d zjCh|0ttAGrrE#BHVDeYx3j7b2=Wt8bl4CU#FE@OH3dZNb*qjfRRYWh%9cH!v-MO!k zmi4D=H5k6RzT%R%xuUvW^T!7tjLADSXX2c!vL)}_-i!+Qv}d=`=mXa83Np`UX*HkV zI{)q5i4$kkuTtahQ4QfLteA${z@*E1mVlnVwA-k>mlJwv_1?6yO+!n~$RwE^?LH@zK`CbW=zaE&*{+jRVy zmuhz?jZmeAezCE!XZILiO-)VpO2E8nUC`Iyldqp&AFCq)yLb1?wJpxr^1vw`5vigl zqIJ%RwNjz8AATmA&RjWxyd`OB5^7iDgF&B&TKbX=+9p#}8=x0@Wx;FKRFPmK7vn!w z(A-DSQk2`8gMr$yGFeDjEfaVA^)Jkezjbb08>QX?2r?ZVomM~@OpR-=wbW+XL8(T(bh(D;mjrH zpXO;^Cd}mc<*rG~n1D2zHB*0p1+kTthIW(X-;3Jw7U~`1RFSSbLa9(hKY{^zo&$(p_{+<&v1Zezm*GBk-x+#`mKuTJs~CHQB4-SFw!jZH*Iiawclm>D-Og99 zT=|DZl>E7$x9nmkwa}~{lX%jpe7^qvaZ8?p`#>aWW3WVozsP3b@AIl1++LUm&}o;Y zOZ#^Igr2)HnLkfus@cNx7B=SZ_Iq~uwMD)X_x(-B<|gjV;qLvZH+C&+_>%uhelYq%UqeYALlHcN8Q-vC7n(bXT_IeEQ$RWX7tV^3FE#Vy$(Lawv zix|K~He}jMv!%Nt#@otLH!gHb5y~IaY--)3XL|jZ?%xFAoj$3?D%wOq`D* z2EXcYyYBfsUSu!jzVg`r39#X*qxo5+Ku(o9+qF zWt7!n45bKXz$K>`pEsO;7Cv@?ZamPxN77{4AM99-ttO?Xrh5EYyYY~nPOd)^CnOcd zx}?FSPt~8w-s{rRQk@~W#=KJk?7J4cFDMucHR5s5Yj?V{q4*ua9d;J={6hs-gyemf zm!CW6UDAku?|xho7lb6&F?FnaYW(eFk_0%NU($KZu}OXMW)2OG6}5eKlYoutXD85N zyw)78!XkpLx^+?}qD;L)5mUral|SvQl{CxV-rhQQoY3xEGoiKhTnBJ4j&#h?@7+V=_n~TD-*DKC+7WzH zE9cA%;~3P!Yguwti)(oBs#O=mGKnB5xLBbD)wsaC$L_PTb`9NKB0?PP;7J7#<_MGq zVyyAVbI=7tczN(_2qVu`SM~OT@Lo5MGI+ne9I6pmy(ex%t1_QLewShVQ5F@9~!Ri!BhZ%hL0laTX8T^oQLtT)3Mlw!VNKd~BoGav`uQfLL`@dkc z8i#|&F;;fV@C#W_X$cF69JMM!vHPLhUKso31J?tbx*E z_jXCW%s8V)_(1flp3_f#DlSfZ{W`)v1GBx2{U5}yRSj!$jk~mFL;2<@u-n@`iF> z(NeQ1Sy{3$)024*Q3`*c4wpw7rBve%rdR)){W8^S)R=Pq7=H~SP!PDP$n=`D{Z002 zD}x;7&#%vKKA6kles@;;(7R8A=9fgT6i1I6)7Pdr2A10PHEwE>Ftj%WuI2JhkDo0Snwn=CEf!M^}PjkM}YDjD|3hH|vZ5ERLvOvz5 zZ~mJ%Q)4FV9d)NHc}w(~fqPd&m(cu$J!FG12R3h@nm8Bx87A0I8qs}Ml66mLW!0^g zU3ds6hOjezoQ8$=8-RkHm#brGcou<1O7BjcER{M2K}_u__*ujA8aklWn1) z`c@jjBLoKP`B98~>J z75+9Xo<={frJQ%ET8L*Uz9FT^7=M^ zeTN-Nn=TODp8o_Ar?WEx2pV(!+<|s6B+gJ*w>lc2MGIGL+DGd{z`=8kAL&4SJ~fq` zG#)ooZpr&6_zY#;qNK~V3M#2pvu6Dkxgl?74$@gK;31xC#?1izVzMfsOpd#Lb_(sQ zeZYPs-QUQ#y3y&ru;s!&xcJ?;MU9coN^s)upWL?NF+Tp}oj)O%*^Aric&DV;NI&f) zV8{N}u6=o!&#Wju76Iz+B?qB_n$<=6S08^C-eSgs@#4}lSTUZK1{Iuj$!fZkOH3mB zR818HM=xG%X;S)O-D~&0Dj&ED*u)8SrDM|_81Z0Rmf-5DQwNcQU`;Q&Q@os9DS5A$ zU$~NKZ-aWI)vs^WXh3~*;H&A6FyIWV?~!(_C0q}f46yQ@ud|>03xGJfK6v=>1#~$* z(BcW*|_a8C7(GqGOZ& zm=mFTzrM-=haC>l)Byep`zku|k_6^R=5k{hH0KnZhbhgnz zO8px}oKym1?D7j9y8E&1qY4#?vJFAN9+#)C1~KQ?$AzJoBJ>Fy`_(5;o;+BP*B_69 z9pPiLzf)uk6l%+J73s7Dh=YJhTQaZ9phbH|7r2B{#n|v#0|dvOn-fhIDQ3-?!rm73 zL|iMFlu7ixKtRI=en}2JD8w_%a=UVC|KvFXtgi(H26`~sU%h&D&{~)GNl?!Qp^u&g zU4OtpN=qzSwMXy1n5UUrSX?1CiUsr8FI)`NRo6aq$$*oYdNVX@TGTUMDy}~Fm&=>b z_5w8$skO1B=iGlVVHwySpL*FhGi~qLe6aT8>CXO9WT8rrZmB48KmlTX@w9M>UDZ;G z(0N>)cd<=Y!% z()R^wJwf=kb>NgKEq@#$&QaHX{YtH;L=JBUQ?Clxi)@pdd)1xH4SyHyH8V344C|8aQky0&j!qwDh7wY9Z_Yb{jK>~(JSW4xmhaR+kxSHO*&hniw zW7+#WQHaxATpA*zc1d+MY{p$G2aMlq2ou!f)AL%ECs%wkQNEhoC2l79R6V70x73Rj z0TakK2)lpqjpv~G#^fh&RjmlFSD~**i`f6sZ&Ny zNHHoskBwqGA%r$SxD(h;HMZ-5E31QgpIy^8=xf>7!w(v?4RRN>7!zjs_xvt{kJ18g zqg(qu*l$p{*a9?3M@OMFFxZmrY$ifC?2~#{8m_ZxMc3oC4C^H2>!a?Ma;0 zVf%YEK7RSaY5+r8dCA4$eosj*T~BnId~m{*;*o@Yd7ho!X26Ryf!QUnn2SawS+7Ya zLZUBEo$s`)mI^*d8Cgtnt|Z$X;(^j4=vJ!X#uD~ZL|yw+u88R>@;eF zyrtXqC)e6$@lqN1?}0H;XR_S8w>5xkB7@mpuF@%z~) zrT>Q5E^4&tT>w;)zZK|!Rs%it>He-GK{Nic6WjwNpjfkj8DJPWNOL4<1V=Mz7 znNKaUaoqh~znYKYzYNHt(ff-;)iOB&&n|2K=Poi3^R4DHR0jW+* z_ogBpBJk+4qk)eTFLwH(iM{Nj(?3958@YvNkeYozP9;Twz?q&do_Xkb1gxpgm%=KS zu3r5E9Q4J4)8p^(pHfQTYME0=ihfaKT(w`;;0`~m*_&x@Xxap(-n?v~9Q0GpK@pep zpsPYOAAzF3&5SMa3n*4|Lr$RQH0~jw0+XMrdhA6otrwe2%I6zdSM66(fyT6pDE5b# zbz#u(N}yBNn$xQIAv!OUic)nzYE=*IMch%6(vi+x!`d3qF6 zoQwW^3<(W%wS#Se1(o?3Ra((dL2>X21;>h|>pu`3$sAkSp^>N;a6>?ajmNrG3!Rd< zlxM5lN?k4G2s5ZBp-;dG{5dc?HOmvZK3aJU5H3bfcVA3f?bw0O`Z+p2LN!)pXR%X{ z!MQ4U!t$h!zn|?by9AO-actP1eh=jDfO9iCDy9Yk*ZnI;pU+nJTa&lT{Wov%JVP09 zg5BTu{&&DhnFLv-^7}8e^85BF@WSd~FGtO!J-ML=Mw4_VwPJU_?DTPCDVro=+-SMd_Xj^$y!Cu7E1&q49&w zU+P< zb6Mv^u;>;mZWWveTf3&4SUJz{SGd@B-{^@GJ8I^C{@$lzX^$tA=;vH(6s`t|O=>{VWi{^zQg~+jQ($d*RthYh8*?a80Ewqnk6_ zUBXu6Th<3(#-raOjTHC_5zF&SQ9yE z9(C!*8u+BX^VT_BM4+DHE+>eTQxEU!lVdMjROxrouqUl@Wl1@IzL~J)4|rH5OvgrB znMYugXJsu$tpN8KIy!Nwm&01xX_)1Q2jft!8Sc~zokw z#=Px-h>W67A_;#GfUSEvch!MhW}gi|&J8dDl&3&(8TX0j;9rqUgX7~mMNX#8&}M%K zckzWLYkmzPUXK1J#&zl%WZ6~IPnpOV6gWpb@rVkLC`~x}u(05UfB#_@8c#C5xjJDsb4 z1CaJJlKP0mUz`d7C$Ce|n0kb(W?M>>C@BcSQN+b%+Mt}JCh2`((V<^JQLKV;2l4q8 zk$TAX{_dq3%v;2YZf})-TV`VJNDZptqEt>&A>`FF_7e6T)uGt!-9kMOx;?AP`;@0u ze4iqdTPm9~3h9$XX*_jZP3-I;!)z#vM+szr2qC(U7{-)@^S9ni2h?&*<4ZY`cd->-EZq^Gynq%K+n zdgz|O2ZHVN0laKmi$U}9>F?2jbR+1T>0`eKhf0u#*bQ^C;>uz*SKNQg8QdsE$_ao~ zaz|%gTVa&+7l#KCY$9PnmCrfX2uZF(H~WU~LMe?Q~P9=yOgl@pGiFm!4pdf%y zU#la5-WqoY+XlJqGw+$#qjesyM|ga(K9-xwEXd|(FC zz$2PtKIKU*eRW^5a6z;7+ST-dKaj4BD__6%7I7l}Y2-`hY2Iul4u8`*OpmCVUR^h`FIboKbZju_BFXaNZ!%W;=u{S7w?rD*3D<^Q(Z+d>u zcZ}2>>sd*`JuB;uU@en-RZ*FE%2|Z43U4om17Iq$7CYyE!5aUy=>9NCv1J8U?>EqxQjCOQGF-FVoH!nC3HsCf zCuny>_eLMKnPapCrX7tIJw~z(-rqAgg_gW~+}5u9!nRDI5G~IE3)CLun4tEXz`v(m)p zmBFqg&x(>&6_KxdzMsJb%Y8NoFYLyE6z84`rsbV>EBmqFTf(m=PJGfa?S7uiqcaQI zM84EkNj3?H^-W9~Qft^AgtN9)$_j$s_a3fXd@F|@;?|KtD>)}D2vNX%5|gNocsPifON{F zz5k|sskXz05=JVX=-#yp%@FV(%*XDsgn;vuUQ?L~RON7Zo z!`^oCrSaG3+>viyV^>xeC&MgirW`D~ZcmH&jbEz#&FUVM=Uz<1 zp+DeUkUjs8h_K6yNyU*qSPoLxre{aM)E!bn zf&C~a{D9Pyf=u^DaYC2Ru{kaCvP?-%fQH-K)wTK)Q%vrjRX$F19^9K-+Nqgec8GqC zG{$<5y{ro3iX0l~<8T<69YP`Zdo%hu6u8rCAD^A3FPn@|3!04|A9R1F>a*on-(c=- zjS`WGhdo+rHf%^dnX6u{3i9_eNMiTh>e0PBdeC0)`gqwA(u8~Ik4ioV;Zp-$*I16j zZnB`byR2`4yA=YUR&LhS3G21x#&lfDt8YwSmp4@m+LG8-ILi6%p`piyPK&tFoG?4W`fteg_?kLGO=% zqYHRe1@*9jomb|-0Rz@JzJ30D4U^fdEA_3SqoTUW#^zBY4~`Nx-DG5g*gtsR=)ENp zViW`VB|O_-ixmbNiN|CS#YpSKKR_P}V4-SWtqWUykpGTeMjH>wkHL``;P6K5lNV=Qx^7J8ic|E+7BIt|{B=xH{BBjoscv zY}&L*UNj=`XMq{VbN)hUE858yXz2#cn->|D2|G-m)l_;dz2cHp81kV!Fg4COFvD<@{Kb7$NYmmCWkpLpg)K<^HRKo_-WL7Z<} zzGG;?V_|dXE`&r=0q{8dg4L z(E%`YvNxGI1Jhp}Fz+jO?{2>DDx=84m$k5wkW4%HaEcLo2iN-Zj%i{D zh5$G)qeKy@{Lf)p+v@i91Pf6s5b7P`zg{JImoME!6DHqd^~~mVM&7R*aHCtO^S#l^ z;?Lps!~)l8-@d(6lTB$UD}ufTe?C{do2y!M*2j*$-8@{#>)Tf~_vpR$ZeS zHTqTDHp{V~HMGth1w2moTI9eC45-rh0r20qvzuNW0T}7yX8iPr*HOT;zU(M{hO%23 zjFtAU=Dk(+Zyk9-EvYdT_AAHTv^b}d+~G$0i-%0e&n@RQTeL6*7W~QAJbhYrCt0m= zZzedi7Uk?7#B74T|p(^Y=6vuC8X)$M)~e1PzO~Hk-EX zG8B39#fKmGNR=VBs&f0DW$zVu_g>JVe*N-fo@r7iS}ZX;qaoOlWL>cbD*?LfT8yaH zx7c%rVX^-Tk9#3Fmd_hbnQR-E zd{6!*i$MSDZlF%^TQ3lL`cEK_9Vum7OVchAKbgS2q@kr1dMf;Rkpo1!@|D*-r*ez? zGhYu57ozsmY$I89nKezL`RP_2>trq^xpC>Z-ef8ykH!VS|oZ9 z?tGih{rgW2?rPAMBv8rpvFVJ_NGSfc+U=$JL!qNY{n3h+Gv4!Uku_7W-W-GC6Q}|D zVJW7uI9A)NhY$4}L|O>Qek;G$mmezI&;-y&GG=8-A(=CLjeLAN&Dk*fz#jL3_V(Gc z(^eoTLnS5kr7r7T;vOoJ*aDmUG`I`Ug#7be%vh{W0Ha#@JVdeO<~OV?ga zRxkeMmAT>RT$TkiCAPBs=3=sF#R+4-xM+ZGoP*WF&38d&!3t19xWD`)w9{43y5s0f zi`UH>po%C)G$P3y+2>!Y4jYKx|A${ha0sCLXuBQT(iFg4x z@V89o;-^LYHUlEl+8}(vbTabszOC0MqiE+l7;E-2m91@cVTl_DO(@U!c(#+BY1j5Q z7^W6`6wdZ1_6$f0hvIVc6glhlj*jW*if7x>sO4(_F;XyBV^lpb<7dtf$k2h(IJKi= zN-ihMTncSD(;|0nwjk8^*NIf_=T}mTw|_%K{rEE2&?)CncfMn7li__w{$Tti287&z zzHfdO5oVk8oh6+m2UL#C`||};4R^KpOt>RPhqsTS)>(OcxNzet!wA_0@#Z_05_Zd= zDKpX2?0xVnU3I2Mr@xPhl}J4`%h}Iszz?(geUWi2Vi_CMpIzL8_LBw0>&$AjM+B^a#Th z87NqiN@%Zy<+!&cQDVRo9oHrlXo;6(<_lw!S#SF2YF?$t$^FGhSrU>tru%s;G(t7D>Li*^_nnDl2pX) znRe{(n8WgpbGje^&bMp*>%oVG@u$75RN*x2uX|@?VQc;}fLl?PKX7VM!2;ct=&_LF zOECDNd9egN_{f5auoTPjnLB31JT5MufaMDsp-un(ZZ{+Qq7pvUscNSufk8m zTiqS$pV{qvs*RV|E8o9|^;y~f(e)YKy9CFE(v@>J`Ja0_SJZK98Nn(uMs#=g$C4sb z6iu0kpKQ7yo%#`H+|KS?P#a{k!ekNqzwZ`lB-kbjzGOwhg0nH$Dz&%x>bZDIcezw* z<(9m?bg`o3V&Be~cdm+!PH@jHt%4mOiaKn&S(vF`vnr10wG8>*0b3^NZ(w|N3&(77 zO=-GO$MR*SuHh$?I}O_I_0B3e9rAYCv7bTqDe38X5si4*F}H0sSpJR_5UpY5g|Mf`6btscY`g55oRKVuTImW{4Pui^qu#J z6la6py}f{xdRyt7<{MXjyrGAr^b)}1>xg1daxRD=R#uA_j|yIcnpAp5g7VlF&oS%U z^LX{pmg->6a`NfZi5UGe*0$o#9#ctJ!T)z4Ybw13J4Q>N_E>ErkXn@v1K z_IgC3V~>T6>QI;k_OQVsUR5LSs?`@)2(u0U>PvSOmDN%KW{#q@Wsu+M7nSGdY`E+G z0vi}96x(CX!cy`kT*TVJtQ5N*Lqy6lOeusT#wccag{L9k4Kdy|mzC7qi}$K&2e1Fx zsf_E?*v1VPurvum1|0^FEJcT6ooK(k)A+T#$e|0Lwo+3|n!L8vB}W705Ahp-p!FWT zEVfR!`61Zh;b4%ypzq;Wi`oVFPVEv?GDC3WWD_P|WCqv3ZI&4a{d(HHhG_HXFgX2& z9r%xtt`~#Yojm3+@t$kO9A4ZHptm?PD3bI*G>OB`F_E@lH$(8l6@v{llUBG*rX|4# z9X`jlMkPx3`MdIfTZumTo=5Db@Q)b@9fwJI;ij?2MvfX)4I?~) z`7>S9Z@*w_73L^yRauiGV-AP8lhK0?V^7i_86Kn;L)9R5$v%06rV*4$)F7>zZYhEJ zk<6UMi(|sZ(35e#P@>j`VNjYm0&zD8-|L;UCvlnc(C+AZ43?Zu$lzCizQ3N7v^W*K zXMRPw8)@)o3~o{G`E(UAT$wu@{<-nFsOS=cVJGKKJ)^<&h8lc9CQmtKSZGNUFGbeW zrPNY?hdrgs`NVTG@&-aQox8A~Kp=s(%5+C`84$k;rpO2_z#lmWVBG-Lyt{uh!`m+> zrT{lV_~L=Rg540OvR*?KLo~`k+Q7R9y<4krE0gP4g@QO3SvQYX)w7ez6LRJDx9&*=2o{c`4mC2 z0ci_zPTmG9qFc!M_ICVD5%y?lXlyQ8>gUiBp)Bn%TD`cagS##EPwUYqxI4q0G79Y` zBjS9V<|#e3x)7iXx+t-FNaQA|1-x3#{N=l8(?&u&QSas)v&|g*XXwROo!QqS9FnN^ z>C$ZG0DZ44*x%;{Rp-bN%Tb_${gMY3{dHtaM=4YX+%8ZHf8l!E;Ee$9YY(We_zWLs zk3VqUVlBXn*H&mP&`34k#2mf!yDOM3sPqrGTOOVQ8^!&pFliFrGk$z}vx05_A(9&d zgViE!&c9VFr3lCqIBLy*ux`cDrKj(ottdvEh1^DJSY-kG{5j-A$wP)O%&9%{3R!wQ zPwq_~!=SBmV*RNh7^%AT?)~-QlxX^umFPh~6cz1Gm|m%8I3#n{&oAZK^*yxz?HRkJ zxNDusN)p|7xW&nqdfQj2QwUj?iz<={@rn(`R_LI05Q_^Jp72nexyf{6V>?1vMEh*o z>nv8%DX8d5jjPHNHe$rqk)O8im`tiGt<2n~^ZBGCU_vCTVGdE_L=5m^-l~#jyK>J- zXs%3S06*5Zc#?&DqGz$#?0emf?wQo+$KBi+H|WFDuPv-j=#07_Y=e%)z)PPJg1hwi zRk^%Pnk^M(fGDz)F)@9dL{c+w;M$=kgNN-_fQ}OraDB#;17le)qb5$IN?D(#Ye0Fq zI~4PfQA39gT{m#==5}gs5gZ*B!P<@*I&9croWTn3>ksf?!N_zv^k=ndqVO7Nc(6&5 zTjPy?nv{QS<7*Zcp+zSFU262@39HitGo!Tkeb`h(1Ms*H8oqb;ZX5Ik5s*(yh#G{$ zsDP?$jDK;`mS8;hvd?41rrf5@9}H%Z{cK$T97`EqtpnPyiuM^%8UT6oQ^|Dn!doM4 zpc4%ET(>xnNnJX0urSbS-ds+*PRU6IrUW!s_XJ zPO-rn$A@6kJe}9TQEdCHnM>v)x6&l|U{fldBWDT~>OJ$c&xll6et-coZ+l{X-c5ORldN;2my*?4P7T$#yc2{Nx&hP8kSLvd-W8|E?*HniM-eamKHdFgg{( zU2`l>u>gpNuBwb*@CuesKAS+U(POPD`z+Pq{zsZBd$Jt#W8Y6X{2=aQ`Jo-;qca%E zXQhAE*hhvhtIkr+eobAcKUk9t_~T32$E+Z~%KAFTRp;4ahqoQpdPVs;i+P|)AHZ%{ za`p^pL^%V;r$>@QwxVp_EMdk$A>1pU8Re%hmVJ4$?^be1-wlictZlgz z50E|>hccVf>_56PyC9PIShliyrTBrZa8P!A7DOv3 z^?RK=V?5XI_y4=P=9=dja(Ca~&*z-;-j3>yt~xhrs-W=T)y<_y^*KDu zBiSFTS4SpAE#tKb;x6?WydJV5bcvJhrmW)the+>?Oj9GAE>7C{vT^+1;&9I`SeKn$ z1K99-ZX@?a?tDtGfkO?8a0N?&=wo>tKaxW%BngiWk_9||qFqfZLEf$qsN1FYD@7w}*RlU&ITx5>@oX;R6_SI1r{jBO`XQcz$G@N1Vl2eT=WBEF>$T zpLaZp7B7FWYq^&tsLnakEN1zm0+e7(?D@&&u95bKQ4nG^c*CSX$w_FV&XB@UTLVm4SYclPF9VRf)$8F)JZ*9pEg79G z0Em&xOASj@MZ*k6KPWCbgs8i|QjN*wz$o8YU(w+tR4p~o?>XZb37y@IHSD|aw_(ov zYrz7AY5IkQ^-%D1(I0zp2g57D}_x5WRZ_3=t7KiW0^UkOstPde6SLiPydH9}Z8%(gdVHHZ4 zL|atJ=Vp)ZbD$HjCvEaCR>6zh--PAYR6n#F|Iy5p4qvTx@Iflv2>gfE6afc z0-_D6zE4GadjaZ$Z~hL{`^GKy z|FvSkZ$|Gs=^K{qI_7#B$9#qenslU_1GiC&qLyNO$3(4(T@2VOd*Ia zN7hN&-4i5UmY1ZjDgAi-K7u}w-EBA+^}n>SaL+mk6Ga0v^bUuuQVaWNlLj~^mH+6u za|y<|@7|3>IJ5lNi-UoIffkJHUG(3iX_iJAm8D#%2W*F%U(CQB&mTT)%teUa?X_op z#y!9O=0_H7rge1ls*SIYIiQYOVlMdiJWj2&k_WmqSRF*~LG+-z70=}@@fheU&i*-N z%B#oIhq0JcNiM6#>tIR&1(9`qV^>hMMHb2Dd#>y`&p7*D=uOj%d-mxw&dTa!>Y7CX zEB&Vi??OVueU24KV!UZq9A?_OxU_|4mSBXgt9^0gQT=m5vvpy1aF6a_5E_35IWb4D z;&X+nUl^~ei*qEK0=&L1t-F3LqaYe8_XjPa&NE5iCRMWvb$)(vS1u2be_&v2|1~9! zeN{h~wcf^~>wo@y z=+y+E*YMn`%2~}r9tBXPlTZ*7)i%FtK*vq!g62uSTSNUWff>ibZzDFqE9B=zW*L{p zi?6Mh_^o`xlTisd=B4k%Pa}}yH7r%laV@?vO}KkkmQ;N6ouE+cxxkdgr^|y*fRcDi zi!GyeNaobJWLDGy=X+&Wb(?(`wG_5W=;WKQ0@wlR1?Y}wWkW0yHP-0;X9sj{L<69? zrw+bfz?}R%u;%WmvRWBJ{2iF%q8G!{fhHt^+|n}V`2D0uk0gO};<*`5tU~uZg`fa8 z-GQ6$%HYHiN(wCa#_uPq{R21D@0Gu_YTf>}>eacYbA#tMR*iSwR_0ixsFcWN;U58^ z(J?6QL#EfHedFDoq-L`}1^Q5U90rN;@w{`wG_&Ai)}Xs^x6epC5m-AY<>TKx{K#y_ zpMk>V0_Gd~H|@W8BX6Y~BDzIyf3`4f2KZ{m1UJ7^gFym36U_aF3w}=@*RLO12~`2^ zT4hi*vAMT?$`T{}7I)$etn2^yl0C%!Fxr5`=@27Di*Lgn zq2xz5d&5|=oDq7|X|ywExY=J9Qv@p4)*0Z{N>e9kySw{3{S%%b`5u%c9%?12ERv7C zZhDUyVfyLy@zu3JLh@qg3N=JMV~ve>2W92uB?KHQG?d+FY_(E`9WYGUt~k9KK8aNo zE4PbmlKoGJAxH|;syK$`XLz@eSa^hAVKWEy?{9pmZ$V*>NquAq2?=?T z_G*!*+++1X-x251P_MO2NxObs1^%$nh8wRc0gk^vFo#whG3cLBc+maw@5iok7dRGt z?Py@&;zw`k0VPGUd0(@Cl52TE9?3Ke-F>cXcy-?+3F)Cr4$LyIrZ7QLWvsV3L3<{0Yv7gybL2v;F+}Gv2Tb zmC{%+?DQpWv#pT=b6ew}Me}}rNpcbEBjJy+iujxz^D8S)!!HiYA^eF_d|~3`df<1^ zunc$Es(Y9F&yn<4$%a_Id~k2dMSH!HOf~XcrmtThOO_BV!FitRI`5k$^G_i3nhg)LUs z7hSZgnjF@#axj}u*TgLSw^CNi(x@`F3hpVp)4MxpQf;VM0-Eg}3#VdWgGmgMX%uLK zoE&gK3XI5vJ)Uzj2{$XDd@&0$d&rVn;KFyd84F1rTR+Z!?wExfnZ}LRa@^PeqI5SI z8aT%l(f`5o=C+J38>o>y6(2i&(Lu$;7pb@0lhA&>Tyvy1`F2hp|U~QuT*z7i`QX-5GC#tFjujxql}M!BL76#V0fv+3=Ftzi7D^%Xc#On${fDr zk|lKlbiQNln4ZyDXis6Y#=5wawJClg7&S@}_pj&j(xXRwEFGZeOL~?B_yGWU%tWyu zu%yc@Y+j&68r)!qmGNryBa8h%4zYd{{)HN9x&GnyN4O5<5Jg3;dGuC1AgJ(}>>8Z# z44a<&0EBp%y6o7EiB7G75u`!S+shV8DKKqwG zJHM`2g8Wr*gQ@WGR0_*QToHJVEOVmz)GM9mA&?;L%{zLM`o`}O@sfc-{E~CGtE&7W zj`MBz|9XtyHA7JU_ zsS?5smA>TO19E0-ZZ^Dd2mOc|#>y>J)*K5f2nG&#v= z1bBQj7^iATK9G5Id$h}|-wx%nx`dbFdrFk107{G#zb2>3on1Sj~(c(KmWAI-sQm+3JyUlD-ZH+~c0^ zHCMy=u+s7*=s11_h2^_3&pyhQpBbh=ol-*q4fpYpiR^o~YW|P}RGHx)?ev^i%BR9~ z83g`wp5SUqUv!cK;}lyop7w9wrp*}EaGLLVw}G*bP_|amrs6i3eeTSe23V2?5;iPJ zWC%~Y?c0;yTs}&|K8S%GY=!nV7ALpkiw(0!Ywj5Z@(y-tyu`|CG#jJ#E`@1qD}iNB zP9?|hC$sD#;EPR1k|^=hcJ_L)jn8TBTy=PkYS&)se3d5(|AiyPZEZ$XZT zICE9*IZ`5y+%e7c;K!3LDwG0l8MA8^nbZm)+I04nR#liFI`=<%bfjrLJ{W>2PSjCo z8T4viPnOu66!a`D(mS@Udm5yGNR}Tyf8OI{Ut%th@$|3oa9cE(O`gQMQv-(fPGxZ- z)Oe>+J(!N_a3d%|_XBxhF0*+5e;PG7rbxpB)Vt$G+r3*=phP%SKozHtR~hs8am1I5 z0(16nswRspt_eERUs76+is#msyQgfP-G@+_+S13VHg4Rn>)D%VU~Yh_ZpO_JDBDYT zVRyhp7r&@jLa{^0H8(7E@rxmS1q3Xd`#{PR<87Mi)!^HklERMn^Tpd zy>WN%uBD>|^Hvn0QGh0=Tr9owva@zlw|)9;2QO1$9Z6#Yo0myR8_0rv{VvhpCiS!) zf;3_~B*f1DDYQFPM=3N1oP%Gw4Mg$ra{OUHI{*_Il*TRh&j!botmeR)YqW|_vt)}$ z@!VmpBJO{QswX6Y<74q@+s>UWBBvF7{Tj1iz^GGC^bc+oFd_rd3g!(~${P78KHX1B zGSN~xq3mqcqVqmo-Pzw5Z>t<1Nd4`c4Uy|4U5+PmInnV=ZnKFz-hXO6%~{UQ3tx`Q z3P>WK)}I3c0}H+-^&-1+GwKmzYMQsJtBoJi1pSxaO!{t|xX0@#ENx~dfFutts!&tx0gXYA_xE-JlLJ14)0@cFbpdshc~#8Z zs?C~dFHH46wB(IbI#MilXZ;v42#~;-g;Ws{x%vY7y9Jr9}Pkp8{7u zJ0NMHG!V#a9Ug<1yn20O$zuyEHp~P%==~gq#85k*JC56762p)-wN!fHP`HwjeGW|8 z>t^9h!`4m-;h@VQ2%JO=zmyZqekdv%KVim`q_S@Ne6H@W(BvWKq8w{!8Ju*1aZ2{2 ztv6MW27vz*8!x+HM1K((`Qz{rmix7Ij!es^`g?6Yt|+`rO9{WvpgZM{KZ@>TU4iQY zXt>Q?yJ)teN|P4SA#s`eKQLpy;$K;`zkr(i$Pr ziB!X_mlwsimYahs!Q;7T!aO}StfGcMgz!HyAgsJobsR|@P;U1ir(&=bVLH*}-+N+A z)ahhf|9G%JTNdzA$V#HAZ79$HxbZUQ@-1$ff`Hl~w)$RL z3YSrlme9R+RoQdS{Bi?PMtdtr5h03FYCqTkR^xVu*8%i(`0{)mtpbscQ)JFrYvka* z7WocbRH>>~Z^_yRd1S0Wu>p9DW(@oXtu&kjjKf~)6!ZeGamDW;Pq zOjwI39_UVtO?=#ALut=@&ADoddN9eKB&C&~UZ_41E{@h$*%(T<%97ZiEQAEmfHA45 zv;0gIg*q@@sJF#7NEt&?!jsuW%9wo@xEr9oN~JjizvDd$cwtmmA1#99&ZD zpi+9J5EovFj|c*j;N#{)^j+=*MW*-tU8k^;HIucKwX6JpYS)SXv4zMA*oEtSyXE*? z-B#}I_Tu)1kw!Y2UNzA{(*zPFygo?X0^cd`ny604HMJG_1!{pN9L3%^5ZV@@-w{!* zcu$+wExz{6obW;aoq91MDF_m=e`;zQ*dxi7drZ!;L>ahzO<8PHx-yEC1(~bRsCWR6 zwgK(f%w?v3aQkd^Qh6k=h%Di^L%C3-0J7rXcAJ^*0dBxIvuFDRy`F?97h0(6+#aO+ z!EKFYPm(glh^jd0;Jx9&B+)%*bMNsLUV5U}$SSTlLVn^}kohX=7_*OS0Wt0uB<}zQ+)582mYx+(`1-u1VWbGUol9| zgL1-xH)Fz5mcn2Q9d;869{|)C6vYL^T(Rb%!xF95WD<$!+v7P*C~$HWbMy2@&NlWP z@H*xMg@B-<1?H7>uyt71?KbN^xf^k~9>jQ{vUTj@?lZX*V5q3O_Iup?0iCcp_nc`(z z4j1QG108vK=^C@0o`<#Y zCL4ph5p53$=#jr*N0jAv1D)6BQmnHS*ZX|^vbc(h&G0IB6<|6NKgFwV7m5(5BU&*g zRFkX3wWy0-+sfLyGntI769;Q)?Tz2CIEdfm<}pNqyh$;QQ4~h;I?;vpIZ51_Xd)5H z9SBMN$X@8vCY>^vx#prTXmY|YuZ3_~=hxG=!kEPmm9;(?Bc#!M2mw8lM!58i9`iq@1>gj(On3e0^ z7zfx%l|6ki5E7LE&Cl&iWznjUlPjk(20aN{Qj8uQGi*dM1_lD$YXtsSsB&k7Hs+ z8!r`^^o|{L2#JN_1`{3GWxI@V?Z>D)*MmyK`*0~KWDAA1VCX8pRu6#8VU%UJPEa4v zv$(EDB9e!&5X~zbiBu(#1(($?rzA6()Hwv$8!I71GHV=d2PW2^A3Bq@13O3zqmoDe zJqPI#+f*}>}Jn0Z3wlPn-P76PaL0RIDLc*cUn zR0Cfb_Gt^tTr1KmvMs7EyqK~Ke}%Zg(p?I6yj!&b1eP)#cKEMnIu1kN-@H^q1`VV9@Hr^KA) zf^ii4(~F7gA;z>R%MV+x1ccvvE7Hl)5gVqvms3<(*yytoT-h4j$lddw$uw)@*&>xu zyd11RHVPw+4_H2o%hfMr%a&TSs)5Vf6xuKoNSk?Qv&q0cx90&UgiEH0q%ylVNi1a$ za4jzh3=(Jpx&r}lPXC7Cf1T**} zX!{VcSM0UkkQKm%AfiqLZ!2eB2D^e_j{dR)I0l((Olp_$MyvszBJQ~NkZWh7flQ|CBLa+qaeq9&2N zCuOfuB`cWtVJ_Br_|Lh2{Wa3mNMHX=iGJnDV~J|e$&*c<@Mg&6K|n7)ayPetdtm*_ z)TFc7HTZAv`81cp`~k2{YoE|n!;%?aObQ6eTDEEhP6Ej&APJ6!l(xSXcerKMhs8@v zGgE4&u3ej@H6m#X;t8zX@@1jr;mW4fFJAYOX~%D)>(a~l5efFk{WOkb;pi7L%ccuYeJjVq{h{!X-1}N z_^G(2DUN$y~p$WdE+x%_jRqP zay@BJX>4Bt_{L#x=bma8%DqR^2ee{)?=mSP%9IQHN2>0`{i5{SYE^0cN)9LDB&oEP z_lI~x(Uj7^2Fbvx^YQO$#G7u~w8_s4W8~LQFYgzRafVM*_hH2a52&9rdlLD->xIwO zJCVvz*iOXLgJb7b-a-V~Ezu7fxs#WjR7q3Q&L_u? z8>d_+Mx2r}Bjaz4qi^>ShK5?noNh!%exad%nC>Li7Y&3{l&3{aaJ0b=S@k8zZ3X2B z1@+!!XH@k>tpdFSuINxy!(VyC6E%t4J_vZvRz{7u|M z@+kp<9_X0zB378=vgG0M%FivhbMI`ki~(<+-(|)>7B-s&Oj1~$M}OQ5)%{-lJI77c?ScW z^pwl*`Ff`r85r!a_<&{z_=yqj6i<;dB6AzfoKpS!ZZ#;uYsM?3pHhsSuw?^&&T<|T zLAf$BGylT8(L~cfHWNWVLYchQD`tzBpZVVw8wQxri<3w zRR5GSKnl3I4NK=B1(RzHXyKb#LL15K%pU^;VqT(+dW46V$lCYE_XKea<}>}2xr}V6mQcW^wux#R171KGqO_LA`MHtf zC@j3FCov{q2H)RNJ$@me78uhNL==ejj_0Cgk$pA;2Ns<12;`#;oETKCYSjRZKD&tt zU<(1d5rf9|lPDv8b_6mc40^QP1jK?!8zrFIK*z!`9B?El75i^lJ>HE+N|HMl|Lk*lg-mTy^ zVG98z4Yyc=!vw{}>#ts+(dvP+y1(U^9(YnKkFcxUe5%2-VYM(>4VJ9U_^7VgUlGtJCzrK6c z;12uR4mEGqx>l?*JR@T)!b5-0N$qZ)=>>Bvi$3^J-SYEYqoy(GW*}^oee{9>&W$ z)~4i|berT%kSV{dbrUXaOc-_pvWD0Ip}B3{ii6kR|4-U&;tY|PXVS@&TJ+n^~SNg~9+_^CEb^SsT z&I8A-f`-K1+(@EuC|$-&k>6Q=(~0GRBpqYOj}L6V!KBjNxQAD79_oBln=BmiH?Tg8}?2VGT)~P3TUzOy^vnGScJ)z-`TE_y<;OWx<^ z9^Ma~B`qN+*o&^6cDozIQWP;`Dk(79E9S6sx`wBda+D#Pn>K|Lvbh4~kbHo4cKJPB zE1=c^1~>6SB24Bd#kKBn!A`@V3n|Lto&6b_p7ZYA5cA_5RH~?`OjL|AT{bk9OaO@y zoCZB8=P4+Fqaa7Iqn{;zaHD9gt(a+)jxsWxWHGG>x~wGnjdyI=$5PeZOLmY!AC=#Meo#Q)B}gKO8lchmOf?*x4ISvtU@Lb{^yZ`sHn=ntlndv)xFE35->nXEHiYY z!^|uoM2Rs4!D!Oip|6q_`1n7v-<<i{SvYVcj}UiH509lCxN=Ql?3ouJM@ zdj`L!RSGmb9Mi(wrHYYJxwYBik}q#j{4!jOP!s`bjO4^g`2T`O;%UOY0bG+6)U0XK zzr+@ZZVDD&q1KpL>J6Q@B&ff*aO~L3Gvi3$i2Ct$mhpziqEs}mC^AoJyoBuB1@ec= zGe(qGTi;Pd1)u>ZKE%R$qRcmr8n4$Jt0=r|bcw1O{fBx~z9#YUk0ALE8BT?I%y{4)%C#gPR zAv#Vxts<@=;TVk#h~9!b*>!)&4c)|CZruzX48tOZ2ZXlzWN4&qW=aO(Wf}`QOLhhM zHcBb+Uu47R1IJ)Q;i=?{J|oX1VJ+9JpP%2)1v^iyoF1_MpcQc*tnK8S9scBfaDuWd zsKTm5m~H&G$8%fM`TrA-{5O8kHf>wKAFvXT6w7iRbxCX+-WR2+xRUZ#xYmj6pe$~G z|4(rX{2vY=`k6|j*-Q7`nAhUCL_^JYy1Q`t2ecsd!z5)&@JG5ImJc~R_V)Ic$TZ~h zZi8YX#f$@WFSu9`cRGYy4I$lT87RBs)un%zWpt=m_88zzq(i1q**{huw)W9` zaVsx8u&eC=-|ajQXhCIj)cB8}^Aa(QCS%0Aa07VOQ7-il?C_{e`~*|cS$=pO*N23% z8u#whX+vr6iw{|E^;cMGyZH(XMCDdX)U6@9p(J#a2S2 z;>B}yt>~EywRW8uy>ZKy-_laOzLjx&U*w(WuASd9_}sa?T0b5%iR?}FB`$2M6_(!0 zq+YM0}My*Q&kha zCGHlYRyG<3eS(}Lbx(c3fT?~9i1sjcbsbR&kTe*=8%5JyTbVkrljx++dgJ|Okllur z@EA2*)Ot^>R)73X?5Z+Vy07^IhnxS+ah;%gE5(h!I5^~XE6Mf9$*Cj8GQhpq;f|Rz z|KH13BD&tXEoUy#`?tcTQGET-kY8=(i?xBHJr<7gy^?(U&I5(w+y&jcYQA-K6uYLP zhLo)*;2VFj6~Neb>C)R0^NB|)=KlRjF&hV_BMR!r!UM!5sgGZn`A+sF5*?4Usi4+FuIR1CRtO5Z~I=DSx z7Ss$H1;x2Y!1Wts#D9LdGTq%~{Tkdt5U3$@Z`PE)vVofSpZM3I@fd{1`6>QIB}X@| zZm9BVo}_`IfZEf|eN~g{H*!|#;9?0krqHZ^K^6$VmH0or8SV0XhY~Ii;L(lA3u5GZbgS_?KL=7*3}Ty)_9|#3#do)A~3RBp+r}Yv*aSjzF^tXdH6R zkLR*%us#uVhYA-P6h`s)0R_tk1N}DbY?!3~E?!K&h0A9h-V(+9a>FE3M#UyOIQ`!a z`Ofwu%p2jQ-^@x&nb~V#~#oQH^gVnR3~aaWb-hq>l(pu*ql)8q@-ho}V*b z6T~^4^~Wk~=jnLM;&Gx0>Czo_o4t(H9zDmWwZAcuTIGB3gA%ED%KJJn2-C|&j>Oa{cbdk9f!zD$69zGGr?rGqKpS)?=e=d7?lzKqAuiK8H+<(*SCdR1a7@#jAy< zZ==P$GJx=?2l^m4ljUfQ;mYIoK3*p?KKeV3nMl^^%%1%r zXgK@^AZW?atYi_Hod|n?E6X zH^~3Ct;(g>-DmlJm5g_2S`ZrtVtdlY9eh5&}x`qWNR3Ya=VH&S;HyR^!*Hyk}S&xkb- z3~rL|`5633Rja~wb1hksy8#qbk*#u@qecx4~6=)r%NJft1x zj%4=eYaUozg^yPOkVFruDHcajzLR%P@>CR|6Y(H=%x`Fs;U#OBTonqZ&fKbMt$v=osAxr@mSh0fE7WyxOfDJd~>1;r0&)(u2c&@Q7eF{2)K&|5Yxr(}ii_l!^D<{HmiwmFDudjM~ zdPr5tGBP|ow9D1GUvVaZEhq%7v~yn+30f)ZO2lR+Su)-;C){GA3YX)ef)Td5dF;Hq zpHOwG*Wg2+y)`ugf{VW7Rn+srArHK0N4y19a{L+M&$up9R*lHMwfPE+N8XRwg|C4e z#+Ht)O!H`>=@-YWF}~1YYOtwDy85X`?xf5sSpy39W-M$7k(-eVpO0(BxHpky9BJZ8 zN8iH^rsI8J%rQlS`VsPA^*v8qK?f^hyl zY9IyvHUkcQCInOVulP8A*5&~bUQq%TYi_t<+yS`{?}L;%?C_HgK3pModXCnyQ>X45 zqZ1q|D_0C5(dm*JYujbO(_dpBPf%@P`$ z5X*1{m6SnFRv{>HNkQc_$s6KJgPW*rJ$b*}B%v`ZT8$AgU z>9Un;^nl+;F*q4wx%Cp_B*lCn=ICv$?r_zzbl!B*GV)V5!~?Ozw_%m!+bXfST>X}A4`bVfhEbm>wLJGRwrw0!r6q)V~K8^5aI z)!8rtd<$ik!El3MaCbQ;Sitt>O7BK+o`OjJOWLD+zCCY>KE8LVdU4?wC-8i`{jziJ z(XL@mlA&ECbZT&BW~r7xFVOvLgBojTPaM+sWNqg=%5po?ZOk7ObLFuEo(y!@R|!rY zO?@$clvXN#?De<&3?i=muF{xr=6F;)u@K5H+^BJN{8ImgTOsUWZv*fc*kmOcH*U(5 zhnL!I>#C8=Ss4qC2R6KghBNsr5kxzA?lguBX~ie>yNdFYEVTN~(K)EyHJ27ZNickV zVhAwp@nrAP4YTvO9t1D!r=yc~X>wrFTrz=B^%A2TSy#CX<#!j2eTU!WCA)KPm8|;9 zcj{)z2$L#izX<~WNjVh;t_6Mg>vi4niBMmP8Xjp`Dq2f2OkT* z*R}}UPF4qTW>##w^PYjkG|gWq$>K^v(5k)og~m zehZ%k_v%J}IQ|UH19!gIRv_huis*ph8;p3J|DR|lILNZh6Cqzj#ETGX3ya#EWxS*< zZFgxpkEe^^UK%G#GNHkdyVo%Z4?{3(oDl`hRwwC7p!*R6=1$yeNw<{eZq}NHYo;-c zU0Gdv`CaL>%1x|M1rBUi9cJ)nljhC8-e{F0p&!bzP3clRHZ7RlCC)$el|Jgob2M^8 zwR@#Rtsa)XCF}gp3uJvqJ@ar$zdL+1`DAdg7&fv@)43+MTey7uf9_=MGI?B*7+*#2 z{KTr+6NKLc{~v4hq;Y>ez4pwbg{@8o1k@B=3|a;nC$U8*_CF;Fsgj_`|LWYeD?M5@ zN?4!CMi*xg>8|vwmma?lcCv%{cua8S8IQs>nr4GOe<`Zbc{un6?T7SCdkmk}Qvq$4 zrK*b`mukkZL1`oE%%Km82XI?qHOI?jkQ-8aRAV1AZ^w8wgA)f59I7UC9i5%8?PUXW z2pw-a>%xLMEaB8z=cXwz$d&KG10hOSvJbl1k*1e+?n>_ieR@zQ4ATV z?;iw5G5vBQh-=y#Q|l{^n_>1j>X&S+lKo0ZNi1yO%h2l}%3XWoFSZJ13>)+bei$iD zu6Y5vT+^-bY=SLEsV<={rXJ-Ch`6m zZJ!ra+K`Yt52S0ugj8cT8Fgan#DfpD#oSI`Co$nx!rtKg!34|4O`zT9ovvkL3Rj;5 zx5t1T!t_U#)5l>-NB?bS#kWSrdrl*iXP>rQ{V~h64oiYH$fk69N(w!??0)oL6h7d} zb?kcUwZzs|aw_rf+|2q@k%wNpA&9z%9Xe)ctIl&ZikmlYuGyM2)(LxWtv2x4nDKT| zj@A!Notjz4yV+9hU~JXs=scdIMZjRSm9~4O^I6VhU^anr-kv^%pBRhA9&ulno%d49 zfhkSN8HI#|SP#{@k2;p*Hht4c6DOhutVdEol0oC@gxO0LyWe+mP>>TzLI^Kbsfioa z*r@4ces1N0dD92~_jcBO{hApBGX1k0$$K*3!&Z4>l7F9U=xFq=Rx7G~DDU zphUWLap;^mBVtRj4Ec(SjraZr1(l2tdoY!u?WU7-;tTk3WN@Zdn8G2R-zU7cnc1i14c;jPwlXxl!`tj+2*mgESK2lR zZLkBNCcem!14KzG$!JswQPxwY;O9Eh(;21cSS+%#w7kI^+BLG4|(J>Nj;y zwADLVQ9R||>l1qoEHk=cX&|e+zHF(r&?ur2aw&ef5;(J4o0<`K09^!|aVh;6<`<`q zeM`*bq9f>IdAW0_6vL>f_GmV*7esplFUb-cG8E_XY6H)H`%0lLn(ZK#7d2*0wyd}}3GWa(Gr|_+ zoA?o5%{9>_*DAhBqs{sGeHUzcjuVNe3Rz#SvvB_u!9m-$Z~tKR(N1WZQeY0~N8PZuj~MpIc>=2*62(Oxd>#QtN$XFr&qsI{1;{OF}i zrl$28H5&i3X^o<~iqOgM$l*Pno{nd#B-Vd|OOs1v+ougxeFs$?#)F9$Pj-!Il_D^d zOSLkPRdY`cZaW~WN+fjXVUJ{AwRmGnI>vLHtHcYazFe$(h?gb0F+IJs6WMS6J<4tt zUtx&8Ris=i2n+ZWvHI1@=CwG^pZ)Z4j+0@n3`?m<>dSK z&@-V{XJ+Qw{L#FIvGJ&5y?V?wZ!+?0^$R|EjXVO|uOGIoW4L+pzEeLU97fxyw|(&4 zX#^|5ctrVvv*4es0_rC*Adz00wS0cj_ zc@9C+<7u1Ltv`dw3W8Not{HNm4rC(C2CGYZJmt=$dI+JcLO@VU!kITP5*yo_ya)7c z+`RebD_5?}Na!5~dGzY=%Cs9OVf#Fl6ns{)$tH096I1Uj*D@glf18AZ6VrBj4hDQw zBNC2)&`5dz5MG*aVE=xarK*s-_6kj}0}u*0;`)}P&NesSNlVzu#Hr6=!&=149U6?j z(kV0DHT>JR3AQ%iT`Xy^i9S4Z8ohIzGn*-gBZ5WPgSviIa|;c z)MSoojy8BDSwItecj+=VK&w!9Pn%D$>&Kx5$cf{T_ zSm?SI8oD;J$P#HDaGgU+nDbL;na5UX&nG0!x9{{EzcNa`HFp*e?k0W-rdHjR8t>2r zM-)6522|(sc6sJm@D}-xV9vV54%rNbB17%jbJG45H#5gQXCiws;;1FTJsPb_OG z0Jn=5XYXy$!n`wAM$HYQTs{vSp;1gHJp~~bE{?e20VatSTKGc8z~|-_{0edK%cgl} z^k<0n6z1f^-=&u#woDD?_*LgtRvzVc{0o8#$ED2>my6q6B7*Nb5b@c;<#QF#zVdPA z-UT%h^pxOIV99|0HhroZba-(7mklv<-P3X3k(Fv_sN`y_e(TY3?#jCTW_;*3tDU3u zileqK*2Pw=9=s@Xv)`fDP1mY>tBtuF>|GwzbYxIgT>SXCt(%QrZThez=xEU&W6rpg z@0h?$^)1)DVkGgZxvn(n+V=K%Kw( zBlEey6ojoD@Be{^Up6KMQKLsPYW<0@y9&9z)4U3acY6SVxUApA&0WFw7v+f0ET~e@7AoVG_E4^ZN6>n zRCBiM_}DZL{_|7_jvsKjai>F(v`B@Ha17ZFJ|LVrZ=0yIoNN$fVClQ*^TS$?wQ%r+ z9+K`73alWSI66_2C{0Khkp{~G2_PmA!CFKIEDUE7X*=AxDT*G6ZejV&u|Er*Y8xdX zUL$Sl$--!>9>}03=<9oJGR^!HFm%f>5I(AWN~4TpoFygzJ9>G1uZH21;d8*xNS0qe z1JfbCvHD=5H+d3#hpC5y9Z5UCDBdgECikDs{IGF@2M?a^+c9&$W`~|na*G_+L~qQB zrnNaRUk6;`%SLst(eSH24@R7u%T7u9zuo<@LUr%a!``yK?|}ocHD4+EE<^J`-r58! z5D90Y$r~X)S7JVn$U^Yv7I3Ef$c@AEHN4%@A!h5SH{Dz za=P&DsvhpTJ37`TmtTR-2guRvYMR80bDTMW4@%ekD*h9Y+<`w30&{B%LTDzRTek}1 zH?(8B4|Rc6R{WF5L{5iUz0oEDgka!r~6aKyN$G+(sry1GydEfq`e`$2Z z$I!w3ih2+IbX#2|^3Ph%XSm+#bc~s{soMx`KbI$NUrfdf%F*l$g`#xclX=Txf$mRZ z@7xiTc|ME)tMGi!xZaCUW5*s0zZiWy<6h%S$MR~9gZ|z@n=A%dS(o~aa&qfAGGB?_ zJ8+=uQ}~j(u+O-uLL{b4wkoo%6qClG_-7Xc^Z+6%FFNNsmsKC9sjQ<(dP7hvrC7#! zWM`-mwuXY~i~51SKy2&+#o|29ciM7uok*J4gFSl6lwEB@+_Mf>YPhGh4GRsO3tGsB zD5n}=kt6-)3S`0^E#Ji_m!76IG&I$nq-k_iR=S&bmc$V^OM@)l#OX`0{8EiY=lv+z zX!7Kv(b48pw{73<^VO%&iixjt2A>{R^xeIi1Ev}qtGB8}?lkl9fD^|Cp^(jd6!ACs z9xj{#UHlI5I7m)uoLv_cp48WNU7XJXh{<_YmJ~o^a!LDVC%1r)kytx0{^>n?^oU{o z!?Vt7+5?BiqWG9|`Q<+SE@Je>D|L78hoUp^TQIqsJL(?Fe+bKgQh?^Juge@ZyYdFf zdU6@&!LdZ{HRCL4n_?+tP$#Tq19^krzT401xiWj%U>Ug)92u3AaS5Rw-oUzn5ZSR! z3~oh5MTtlvuk=eNcCI|#0i82F=ytjQ-1jwV{G}UlX`Uia$Dxy1XN=ZF z;Bw3zT|#-GW(nhLAaUZ~&bH(p;rlFPRxkd+lM1r$`51VLNuo9n@F65c1p>^1q4vo> z0+B81k@B=As>A8L(EQ7XN~6f%8)$v@n}!VHgb4te@gY!Z6LK3nai;4?Oyh+MU&6Cp zj=(qy6~rlrNnX_bzhx%zf54<`?wROX>_(4H#oR7G3W>Cy`0 zi>cLZEn?O^Rc-Tpob|geN4>Y1ZBbhr_lB?Q!*vg-?mg3WEFi35L$$7^dR}4;gaEpn29VMD20uOH`0! zYlEFmc7L~G1rHLh(epVDNTH65I(pe5Xy7I6eaxFjP9M*q!(lSAIiBK{5AF;5A_VN? z_u;N>wzKsQQ7K}Q?oSE0n8I{GF3#P9+tWXgwhmb#Pwb_yMkl@ z5+8_TCxd!G;at4pK_oo%-a>)~L{3Euemy}W>&7Fc)qu|{FkA4_bTs$2S?I*Wum(@ zk8ut$7iGGu9+$$-iI3WlI(uEvX@p`2vN;Op>CJ%t-L*N%_UClX&li7)uaNzCca!43 z{&t?V2L1DcG_ZeL1B-F*szo+_8#u1`;G9olUdG?kVD;MM?NPCxU2XeWwF)`5;@ReP zb<)fVHGL!N1YgaIGnk;Brq7?+Rr0#)_1j4nu8%wR8Nt0G-b;wb@h&bAlY_IPsFRWv zE@9=z*H9WUM~EJNKo@T(xF^V?Z^_Z+T?`-DhNX%onoWDtW*dU((uE0;o0yu~Q>JTK z31^K{>gmP%y}`5dA_^J`%>PuBEixykhV9(5>ng64egIMp`NL{UXqo19dK(yR1OP$r z^kx8$mMoo+=m#{vL;Td@+eiffKH@XJC8k7VX$vO54d;69ET%-_nO3CqV(~6O-}%%E z-9NR%6RXGja2gFCe)$fb0emkzgF>E?B})?sqZ9o(RM}&5(LS~^OY6P`ppkp{=a8(m zkEb?0N zr0MiGAXi#zudR*E9e^fuGT-l+-=w!-m+NRu%;n!Q8j&;}URGY!NVDS^VmaZUWWzR7 zGkSEx`_UUR(K;fK-bU@Xxya%0E!0QKB@ASpsi<%(k0dGTn+s4e$|T9L?L9F39K3Ga zPz$`rQT=T2`y+2`%g5vWWn~8y{K6p!vI(q=3-_EKA)h2lhVAZdVOp9Q3I}C<*5qHvM{<0ghS-wpUqY_}liVAjdbV8E=!>;s zpc)KG0s{4%duueCzw&YM?NmX{zLqylDo3|o96ARva}knzm(yJQ@o zo??8B+iRUA9$NM~T)|~Dof#X~pBfPtw`Icze_SaQo)_X)^yP~^R3cWvghd~Qs1F;~ z2EB{eqo>RAbLXNSQqU%k8@LealXx{2`yDB5z}5>{LnalN`xV*$D|Vvqo@_G<9uf3a zLPnv~eo!sktmVQmpE3Hp+H=xU^RdI3b z1nF-n-?0rpL9U1okB<95|%9j!NGA0hwylznLt`3`{IP5CSWzSY~X}( z^kO)iJlm$afH}SvED~6}OX*Ft>jVup4QoVx195k4swQ1J7-7z~pl&8jin$VymdM== z4Lmt~GB=+L)I^h{G#GD_lXH2@#EEhBTmz?_p1uzCjU4uQSEqHYz5ae){hga0F1NI? z*}olLE2ZP!MSDBe#>DIS7!SV)HjFy@O$6@mGM{ZlsFClB@Jb*BI(W~SSkp(OD-xt= z!`#G$Ca938z79Txx7E!z@WhE4vM*UvQ?v65f>D_cKiM|k9qUC&Wp|{K%JRP-dgq(~ zrWj~w!l0xJK0R-v;iU&p8Fc!zf~JP@Vt&P%xrJ$wNo^fL%`<)r&$<|wv-}M#jqFnA z%R8QbzRS&Z2=7~Ffccz<8rqhx{0f$jI7SYPKS(z4KF*of0)XzdnK-G z_h)Thw{G_RPCI)cu(e<7_2^t zC?vxTa)qY@f3_{pb}g^fW1s$Mq);MgA);$-W{>f;)xJT2vvh${*EPF(k zJ(mw)D`k{cS05F%Osf+kl9Ygno!$bjFzR(;7EzHvB0ex-<2)B#Mh&rvkSSspPiJd- z84;se9|&9L>gCjrBsi*)7rnqR%L#EZOJ%#J=!t0uE#>xLUGgX!5e|PV|J%1KG~)L! zy56jEJ~*}wl1H)5(bV0VNyHM@1K|9l?5MNM-~&9Er9-wC&ZB~Q+QIqk)ju)W(&7RK z<^b^yTjD#Ysj11P=PseBT6GMUn;^IaukF(ybCC73>lvY1CjyTgA{uDIdq7B;bgEXX zwow1;-%Kc*MMTVcHkL}yBNT3LYD)(nB7m3YG@*B%;|uqD3l~rJk2QN}4!G&XkMNXH z6OVoA9W~?bP3uYn*}uOklykSfeI>Ea@kQj7Fh@H5LSO)KfLqsmElt1kZTS-Y|}g`G78BZ!1=_bUxsdbXJFtv_~qeq zMSkI^`zG1@R8s+_dn@BJ-2v}-=TJ_nWP%FjeG?_HVBYhsAq&|0G_*(-M zV%z@P54*kC_-hRf zM(-5s<`u(AS{DyGVcLWBUkz!%oyb}EJiSx=Norv=xiue5e3Q{=p=s5!!bz`cd)#W* zb*m$<3^|WNn_O%%LTlRuq$@tzY zRsNN{Xn5`$euyT1`50X8N9%8zz`H>$=xhmb_z-lfcHOWb7#?*|N8NfzS5$fB8>7 z`&k5HWt(-#Q#b{IwAdc#xlpZ1lg}6jM8i8To|_I`r$FAiXYKZ4W)sVSoauQttBqQRN06#zefUQ)K0A?X}TP3rCqXWc*){X z=WGDDq)&jky*(2XDpx{)kM8)!)e8#?Cs>(i+XWsPgpCOzYT^#po!Jd}ih;(0g?Q#g zQJa(hfXcxoI#JByQ%*Szxi;)7;XHz;g~GbCnf?A=x2ysDm`pGXq~=qLvD~pr7y3@q zj#Ksw?u>!~jQtPzgTZS+FMj%hdm)XD+df#4E3o^AG@%rU#!Mds9?ipNp za0VUK)nA{9{+qNZvJW>AJ%(DYEkm}6;mqexpKb;Q=}3)&1Q5~l*BfoKmJP;cv7Pza zpu~l~oq)CPF`JJG1D03xA-tAfSpO2AkE{UJ9ra>6{!!i#!eY8)s~p+K#K*o+?9&PK z+k+}8V_^P1v6GL4BFN_F?4+r=h_f6Xm)mL6=o$)Oaf1AyUtm(hv2CEf)ExgvkxLY?RqYvOoBV zuK0|Y$=oFfgDpuRPP+$}wtQYsvX; z&q{Y>*UMs#&U8;(^k2fPhF67Stv>(0hTU6cH4hl&RNS(rrjN1~+c7EkYLFE1HP`vf z-(`s-Jf6p9w9Nc&Z)RM5iSgoU?_1Yw{Nc>43sV~G-(=bN)6Jv>M%?w1a}JOPNHA|T z(j{rRqsqQal9u)0vMdm!S3p3lx@ocm$%G(*#C$DCfjFha2(~(JDO`qxMnKxs*^yP0 z+`AwpY4amw7phCZB?jBQsm?cuZS%*A?Hn_fo0S*4R!pPZiiCJ6&eR(-4lw2heU2vbIa+H45~t~3+iv$OZRmVFDEG-lo1yLVe9OgxKP zs>_bckr5F{m=;V{B*4uqx4e8mz_VaH5G<;t&Y|QKBF3E&pwQZbnB_Asdy*BgwUR?v zY-gd*8$%Y!{^J3Ep2}x4d-(0YjZxmQ^*2zrVf@4v+3i^u%ENe+AkMepJhw0gSD)l# zv9O?XV>%1@lW0oNU7p~4?m?RBMv`sD&=~k4U37rL$}()o{x%}D+pqkzP&C4^05X^v za=!SGsw5$)j1LioxsuAsW%ve%Fg=7C1HA9kh)gHVBU`v<-q?K&;~f-tFkv&~4bVx0 zIrH9c2CNopig&3FAKs|(<6|8TxbNH<%ax5+U~&JH)HbX?-A;#j#m3Jv|3k^bS8+}{ zE1&HQ2*1y~9S|^XDk~}-PW@Qm6%Dk2cDGwFvGD~FgW^|x{P>Yh;=p4d|04moZ42mfX5FXN*@^Wdf9~cj2d#4!Zny zyyWG4+yBGXnZWg!wtYW4GnUfWlD)+ml8DHXC6x@uShJK$*^;Gf*@YApX;Idi#*#82 zN|ux)NeG2#Lx_r`)cZaDVdj3{*T?61p8L6Hl7H8EoyYN8`_Jy9n4F74{xdo6t;n8r z;Rmp7+C?-Eh zsau@4h_ehilEB%UwO{Je2zvVP;l;=Z8&C`WHsYvnpE1myL^3;+bbR9RBELXMvJ=a8 z8}BqTNn)8YW&g2hrN~HX5h04N?xFbT0KOsgBfvQ~3HP*wn$))L&4-NPnI{FIqerin zzoLZ9pS5|^sy@*RW&=-~nHZ+$=Fgebzh|f)x@Daf+Mp^^;&`TUAM?%`@36So5c1ZG04YjQmxZM#6T&VES1s=aw+c9kk~1J9e#MH? zhkWM3V-bU5w91b<4Q^R0aI=!)egpdS>Db08);OK-XOu35B4rmGeC<~D*Vn(Omogst zop$3i4WZ#BOhSqbCaHl^1C%m6eQf=U68TD0Egf_%*%2bJ_O2V%+$KMd^h) z+n}IviVLI0h06N3b`P1*GqY}y^1-aVc4{#xYp*H*tE9sJ0#@%${$~Jwes4mP#Rq!! znWo-*_5nA$pnK(ipV272vAZ;(-FUsGW`{n195{X3B4x`9dMUO~TMoWB_-)rWt+x)n z`)l%u{$qdnxNUS^wCXvY8G0kyvX=Sityfc5kKb6Qfx_AF`Jamj)q%H8;oY3C#H?h^k|61hM<&5XK>;Sfh; z#Kccuw{A$rab3>2mwST-4dzBnWg+e{sdK8X%ckZvcPWB-o&(MP<6D-jBm@Y#up=Ny zgmkI$7>kfP^6lBe_1Q_Q-7-zjrlKjr4dOMO24-1iMeCvUfB`bE=$4^)Ib>h8$KN*` z>#x}kyO5gQE9Lk{QEQ6=#$`lWpd>#DX?TAXWtANMq&;@|LJCQ;0reeId-Uoxib0o4 z;bFg4lpeew9>`0*u)Y%L4X}PIF)MRo1+fX$sf{ z6!U3ry_nBGiLi# zhXo593!nG%Z^uZ$j<_ksM%4jm2Z!t4-uDelpiIOTP2O=pu?w~{U~Vg7Z>7Khf!AVw zLZTA!wW^iyizun)ZW;f61~{nflB zL7j51&b}O2_}h;~@`%CB7fY!rYz8k}>|FWf<3F+6M<3ltCD)V_TY;E6phigl+}`-? z+^iSIE6O_FYQN&T+Mb3FbjvPwI9OGxRZ`E1X}7p;_~gKa zpxLR0yJbdu9UA>pVnAgf+TP zr3@UzFP;k_UW9kT7gH{y&{}3ZcyJh&mIf~W2U*4?qX3lUkUByakr$#H-d5+1lmH>A zSUTw7yi=i}4X{8JZ!PZVeZGHUV&eR9FQ7aEdZDK%Gzjk&U6$m* za6Bas=IG;BJMifV{6vPRPU_a}+r?)J*s%U+_$jO(70Hvnz1?By%i4i8jwZCJ0dGo7 zPXpCT1OI=6_k3+Pmg;ZExO14^&|51y z)KVQ^Yt@=l9NZhVUHeb5U$OmPs;?1uT!z)$5Wme zKoo-|mX7_TU=DwPgmpZWgmpYSNg5Y;m3QsgGk(DHwRN}&ABJ1T{(|3)SKqW0bG;5M zTB5WkbfdC0;&VE2&;yO3<&5+*z}=HTn`aI`?}_nuuY2&fwj=u4{#{g(8(c4TD7+ZR zhzQ8$Euxo@Wak@5j^dx3&HO`28NIVZo7WZs3G+aE5!uQ1-e>gATbZbp z^#=tOT<@AuI6hjb?H(z5TD#*#q~;-3L(lxH*2UT0ET=tL`KaYr+*2P)Lj-ij~Uaa-{sOjbLLDN z*CqTVP^=1}2@E<0=PJ8a$jo!-uTW0|-pe?Qb}imXxu3`i(Wrv0AB=Rd96efE=$_d> z-G`cFLFCi4v|0fv^?UaYZxyriRUUsNO1T>!k$pG7nG#K7G1|usPHdCHrhMyh;PQ zM&S|`Mkz^2FgCUjp96`y6Emq0E9eMPPDy|mj6iUGbEAl(*1t@`pL>(Qnt@6zX^mFl2Iryev_@h3LEn<1K;hKAINYW z$Tu6Uo+{Bz2&dvIg&PArs@$49+Ss@wW;j+?wDt#&Z~1ZA8JG6Nnxv;!7rQ;b&pz&1 zm9aQD$iRXjnh}LmV+~LE0ZS|6#a3_1HV56VgZ!PD0 zrQLcbbZDT~AWuTl-WAWQj87WZKwJH<_fbdwPTkqJ;(n7QZcoO%v+K}qTk!OarAs>; z8}+5*qs(>w9b(;q#KdqfdnRj1^Fl-@tX$X88E3t(FHNT}x!B;RR2^iF_0lxWhIflx zOUdXn*Xnh+g2$aSI5=M2U|HR2V&pN94eAkHuoG1c2uVGWQ@mq}6d@onYei-5NJA1KRW&khT`5&erVgIgOiEMbEaJ zJ-ZobIY`_ONH~jwbM&USo4Y)cYM?I6w_ZrNAeSsFhcW><4jF4 z>@?-k5zE5r4$hHHk37y}MPmaG)6~}H@zRo3*ue3j1F3pu-Sg_J68kq|DpCu_*~`#6 zeY(F4yY3`eQjMmQvQp9YG)ub`oP$CiP%s}DyuG7(U%HQ$&0K_N6PfolM&kKqd46!A zAgk== z`ow(7&`Q;Jsc3l59lC9FcGe#}cH%&jRSVMR1%K7SV;|>*j`t<($lF7h<1sNLhvGmKGv%Cx< z_?rA@b5HOAg|-FGj@N)L#=T4Y5e(%$!lYu2zpncjAZk3*2@6c$dq3&3}aqOvZX z{dd(JzJaZ=hOBXNDsh$4iS@buVi+-wAlS3+G%};PMG`QX zZ$ZGB($ve-HVT5O|8k5RI#9A*8^-6>ol1G$vi_iz`^FZP>bg zbD(dDm!@P?gsx4&c*&cVI+9Kb1CTfp0g;Hx?-#V@G!v!-!YDM!*4D-+XH$sIO@=Mb zw{JW1^Y9e-y`)=t-aC%D2+HuWS|d&{7C}8m^%f3Ix7T4TN|1`9(Rs$HjO>WyuOW6< zsGm?Xp?>ny8c@nxdvCWL1%6qTPwk^_`NW?!s~T!iWpFn@bkSr9SF!ZENnU`olygRO z3R|IffjpZw-UQq65XBZ_XxE1#<=(e1@ApLS zJiuu&UT{^0~0JE9P0q>&VK*kQy?}8Dmj`6k~h3)lGYT ze?K!p#YMcxyHZr$g3*48c`qrcvL+<2R#R%;yt%CXK$7OkYrsPQIECRn8THRUbm5sL zXK1Ik_(`T(=}68bBOl2;PrC*G+4VP3aUy4tuGQq%HxG&<0!$c`x+VS^pJbFM9nrtA ziaTb;z2Lo-*V-_3Z~F2BS?-eo7JRlC4(E_FG9TMdu$o2dBesap0klU>OioJrcp+y{ zt|dVDgeg4+&lhPA(87xY%Ruy+cy1BF@ABzR&}04+`2#P2HI6|zP`4%*O!0XYCGbPXyI07uW7)Fz;J)nbO+cWCtgT>9096$hi(5|Iz?EzTCGp5 z2!>_pplj1S)dB>Pch`!?>+2}-j$ueg31m?b(HHuFwaYQ<-aXw@vx5yM5tL@lAJlb{ zS9BuwEWUo%UcGK`pnQ^%O#nt`M;9dQu;l+UP=aF7J(-^F=dvIDui#E^-!43F)PBf= z)po@fcfMM|_DgJuiA0Iw>G7n(QnSr=iWNA6v}(q2_d7a99BauMiY?HExC{cUJxEI! zpvmp_X*$HislX1k`*P}RI-UQSF+N}^{vC^5*p%aLeX z#tZg>8q;1pLqLtzka*=9uMZ%()zJx^SN8(0krt2n^N;5a%`2Ej`7q)n0qLF<{5&Ok zDeR|;L#Ha|t{Mglct-+h*fYULkQhdPi~3|diOjZM*Y#%>MJ&BMXLS$nf?qAw-lT|J zcu`lS-1wzKZGYZ~U%jWlSNiu4IB)#rue~ZSHV=+nsAO(j63j5_bn57`#Et{fb?1#t zc58y<|K#q{Q4`OcpJCz7Yj8JQk~+ObD=*DD`RC-xqHe`>;M4bMX(lWP@AHP+uh+^> z%3aT>aGhH$w9-Ie+b2=KN(Z2z1g;;lYk?eY;qcy`{yCc0ZAT^#o|t)yh?bEyO+$_E%hG$$CCrM#& zMJ{{Z)F}IXD2Oa3Pxi5XyL2Glg2a5yvvZ2AyQNm?+`a9Wwda8ML$f%Z#-x?(VXe7D zUfheHQ~Y(i4H1AG=RNnKMmcY(m?%Lny`B)t$C%%V`|l46&*O%sN7KaYyBEyH2XaM{ zRwhb}FIwJ5UA;9#*37o-+o*Mu`gJ(xd5-22B``2Bu;=6YbRrCr1A-}{jdye$BdvU{ zAbS(JV4SPJmC^IuZBk@(r|#W$_ms3ZpVFzRvpkD@Q*YjCw0F<@vK`O)oud8s&d} z@Vx-=>%D?jcWSsPF?joG`B9O)&AQ$#xEZrQ@+ZFew;!MPSnb3}AUUK23epnclMIVN z5cn^cjWrr&yN!m8QH%60XS@vQc&_tfgMZ`_7g z{S6QHeq?d#%7IyX1~*knn=vEo)9@|LicBV$*66?i4@V427Rb~!{Io&S zB4CgqD3{NWK80toJ)*;^z=8Nmjr7!ZTTkQdjekN~2KVV1odn3`dwBF$(5pDy8zRC2 z|K1gaq5S;U8G}2(I(Nc_55}T!yQ0kn$(dW6`xbf*oK<09D{hSOOE;);Rp7H{?3R0c ze6}FMkC;N3$vwS*-8XE4A1@T8yJi%sc)xph&$c&o&b~Kr-Mc4I*x@o*g${NL*t9HC zuT1<5Q)8F}>|A&Q+hNR)@q|b;Bm1)@9JuGVm zj&l(G( zhxw6#WYoft3y49B-8`l@QD}Agn8jWzW^)1>lcM87`Fas_!V$QVI&~Xkf6^mp{-a0h ztd3ou@XR|4#l`QaikZ$uW2`#X2LFRpPVl$ylUIUHo~K=ey+c-Ba+l`cM2@vsQt3@J zi+0E04T-~uN*0)2a-+Qd(S90behb9hJT^hd(6Gt6$2+acxO+EynNHtcz05=Oj4cKp z9MAf0qG*k{yfzwy^_eAeDt|8xDS5YaGS#CLt@7sve`B>-9768k4$y~2FhRc^P2d%pNFhf!dug+2K5VZO|e}KGae<}(rFAlv| z_=yu3531oj{$~f-t0sJFyuXM$Flo={5Wenmw#)wmsCLa-ZaMDAfIQ{8_dk2)jP|Uz zF2_TTVy~q`?K>}Pu-?9&{^;_gcAI~y1M^!yn>*s?N!M9>e_Za3FF6Yf?>FhfKl93B zT~1RJdm;f^cH~=x#{JA*OB$LJtiuD~E-dXEi4CLKc}D+LQfmqzK*p@bLwRFDtX)k` zcDR)RnEaBb?kekl$LQBz$lfL`tK=`6osg19pyG|H%Pp`l@07BaE@Y1Jv(XJX`uf$}(Xz#Aw2o2GK@+;jFwGOrhX*{==z6zbF868YD}_%fgO zuI=Fci6_FteMOQ2)}uK|ZS|B9U;~vSJ27phHS%t<4WScF-f({k_>8f{dYo1qG?*k<6O%emx?)(FHpa!pxhG|% zbCN&cK(d-QZ^XnhLMNoe4c;AcV~CW7azOp>F~G<3lSyjN}_xnBWBL(^*${ zV&oEL(wDy%+_(E$uWsF8$0uHW@x^Hqkr2Z@9hw{PrU5p|BK0;})drQi$m~B}C~9Kf zp+=T3`2eV^qLal%3(><5J>bT2C;rq8*6OsiTPQDsXzk_I_k6IO7Vx@B0y8S;+CH=b zuJjg=QTC_Yhnpvc`E*lB%cRs)Oi!H}y6Ru!ttm~|hbM-C-szo(|zjd7r$jKgiMN}ubV?6 zBiprO$5n6?F_-Q|L@l}!{}5(AvEn1(_3rjKi<*w@NEJtBY5ntfp+Ln9e#rAJQB{aj zE-7Mcc9P>Q{AgK6Ju>5dVaJ1PoEmv>CgK9-fm;XO+yLKI>Ck~7Td9E05kSXii4tqk z2xL$|9a6>45ni1eA5Qw@gHqzo<_00A<}=gmk41cv+Ok!QDOeHq{s3jFmkm@0)Z2TXmth3CKY*pN zv0UB?u#~;zJ6IYZF(r-rp_v?Ibd zvP}CrJhDtc%ufnb&C#PAH}}&{W4&I;;-t8gguq7&&$SW+>bo959Zu;#q@!IG)i7I; z>~v7%G%dJQas1h2QYaM(O+rK zuCg@sWB+xcq?q%O&v;q;L-4j*A_Y)M>7li&j**eth?yJIT>4hIeJb-BeD9y>^q=%> zx$9WcW*#LT2zRV-Uwn$zI(ibDlSqa!Y4R0ZvCv2B+si4THa8Y zj>BorcT(E%?qRGMj{k6KTl75~ww=oN2J(QIb_#czM!l zrd|R=1V!Es1It#}0G5u45vPW40S8K?74&Pa`n)-%4!e`r^XFW2h&B;Nnh|nl)G`rzuz0KADDYwaU3HkzkhMd}5*8~j;IMPh z{%3AHZ8%w=RZv(2UCP3R5GzpUcWj?OE)y5{|o^fQLqZ;C&xkucVub$km)zhu4}`8?+0$8j;j( z=`d2S0h1!(K59_ED_q_KACJJ!-#E6OspB zHUGxJb1H+I>NTC`cxJ#dt4ie;IYUG98Shb*Ed`s4utDxEIribM-+J%?S(^BBDUF3@etqHce7$fA93hEX|qiKGpX zYqWj(yP=t?PP447k69ZcZAIUYfdSiQKMo-99vud5{ZH&mJZ>Y0khRRrBrAA)=t#6G zQcqKymHVk_N(_5t(jx#KToPoJ-T(06Omfjp82$_G^u9L*n)pA!fXx;%En%|WQxT(3 z-|^_(J(DXEZ9+_DquODZi-jpmPn_F6LaD3O<0B$eZ)e72W6EXtpBghTAi&r&aa<~q8?UHN`aw+d` zz%i-inIBDZa}J$$Sb!7wvJtC4*^5VXk8{3QAxX24t1a0{vV>4N0pWHWse9oqf=qpo z<<1n+?KjMA4ohe^C~N%f3vc_OIEqZGOE;g!fJpSlnsV=h8w4!K7YQIK-s5HuYbL+57U%z zV353-mf=5_wbH+T0HfYtW;0^&QpIM?O2a5M;=A?wPWTADZ>xK)^jW_9#ntRBHOJ_b zd&3UKT1j|2cA|{fcb-@Zve@EiZj2-U=0V^Ty_@jSSTThmGFSj3qGp z=#*RSUZ29(LB#eIrZcVP?K^Zx6u&u4L35(4bhQGd)`su7hLD5wWYR{9`z0#_o%H_= zgs!jtwhD{LiqTG~nig@b6z0klv);g?WKBF)v%FU-5(?QY(K9JFON7hxIO>fJX1G}H)Z-D!~`g-|JD`ZeG zxNTI^PRM#iWZbcHyqEQ#uI$1R$R7wRBBuqBoRlk^d0C$^E16DOT`03KxwDr**& zf10od@X8#fzIpHXAEIvIIiGK)?<(b#Q2Jh09v0mGO)<3PJ0|)9UC1U|sZ(Q+ZZqd<_vOZl|KiEI?gM5LD-GV=R^bc5z8X0n-DV3!~wsx0`EU!L#xK zVfB1tVht8RsXOHMF5kRnbmb()=tawxazkB;N#FX`)6KnU>IHHjQU&> z4?;;Spa*>;)2_+4*F!WA8SKMH8C?f+JXdUuUWU!v+~rQmDAJGc;lv|;dM+HGg}BfN zjzKJgYAJ51pnI(BywiCRZA^^%aXsD4L92jS6gEYEFVw$ot!wU*nh{X^`7^E8-i9Cm z9;G|{y{;T(C*9&P^CLRaeYX@m1{-jbCNMM ztggM@3#U~Rrya3UWGYce0Vae_`0)P}qrdy9DJL*+Bx1m`HY~Rt zmfnME3?2}LgZ}Y>l`9DY^i8-0P7B%@%4<(tfKfm`WaP{yqoBLoaUxFZ#e^ov8%`bfFet-ys< zZHP)`?%o2W;aea>1TOOAJ@)HK-%sR}#eXDknvAQ0JV%w9mjF`r=6nupir(r)F1L_N za{ZAPp@jrmHa$?DHq9+Oxw?MZ<{`0Ts;esGD0lMf!3MS<;VCB$nbOSzZuysz5~pGw z$!(-Nk{u5vy^dwfmn_#A#1deXyyf&uX#$%YQD_>`Ev+Rh9}WlE;O&aS**3Q@O`7s) zSbyHUCzEI%5h5Q;__oNun|G=eiWb6jEra?PJ4ZgZoIbsK*p0-*)MTp`#!7_JZ{;2} zvFA#9L>B`1XMTGQ5iT4nYva{MR=6}h%T!Zp@qtNEwXnpWLJPJ}!n#htZ^^#&*9hVo zC_)Lk@Fs{a>}vT3B{B-vavc&gTxRL0r2K_lRHU{d38bs(S6}fq^Oc3j#l>RU=x3Tw zZqD*SWHBjuCut0Y1VlOeV)-Col^Woa6UU18g;&~T&SU4F)z6$`rW+i8BOK;}YH`Ay!ha~^*f?7 zcgzCs3r%ly+)FhyEx@m`ynXA|8#*>Dc~kbq1$_syvu4x|<(05E1D5dNw4m}tCd#`Q z%GpAxJ{Y>nk13i|leyTnXZ{%%FOcTj!opvXm$_WqGlLGKtsN^3=3YX8uRv*x-}63T zqpI;Q?t0hFKaN)zO=z9ND0wS!GAZ%8{z4iAx|d%%T{B?5PgeQn+p(b_o@oyrXolhA zLj1J5(6zH4)S-wJq@3iKbm>y@Jx8K{G`k!rhlJzeuDS5I&OnY|MqS=E`si?keH=ng zAlUQOcXd1a{|@_3pzsK28u*lskIx6+0+TkKbtczv*Dn`kb+e~%#Z248W&$H?sKQszw+(g32m;%{G>@U-iS2P>?MJevv=YXkB%;n2+8(#kUrU$IV zovH7mwXpAU*rc{f@Db4L`fcxAYJ>opO3N3n)n%?X8tK1VIasDt$gqm&aKg_M!SbYj z{Tegir#;_Lsf)3j0jDsEF5a0NSOc4M}1!W4F(B?eQZHzK~0&FnHd0HD{QREQz zjhd0{*qtQ) z2zoS;qzz*d2DscVnwyzri}+Ku)_Q8i+OL?o%km^XFT?~VOxJ;ba?2e2Hmu|>idV5Z zd|XA91p?BS{5AFucw~sx$DKF8rME_3%gae0Q$|;cEwK`HmIr<(w;bAYr=EJ~5|{ed zgs#j_{3gU}H%vP;w%KF7H=ws99d*KZC+sN!VyUlpIB0V*6EuL=KH5;C1=uYa9}LQt zu<3YxBy9-r#97_v*;LWIEIj*o4h}KV)*U%-ORh7adYtwG{pcA;vMBs0y;qW9JgH6V z*274qWg!TzX!9$5bFAwSiEsfn1}`cVrlyidy<<-yNlhBa6v9IzJl{ijzDUo53b9#d z!^aje^l~5Dj{pg|I=YwG;{t1%X_+iSOe%&oD2J>`*fazKp7<-ee_nGJEXHYIu#qH# zY_XwTB;JJBa!%EJx&5v(A=PjSY@dAx4f;Ag{Ld2)z`Zxu#9uNF7i2|SbY5lwP;kMi zps6@IH*Vb6N~f@(U^}278mMA5(;rvWvL5_nn8#j{03gF=VPGGYJgJ=Fw|x}OLq)|T^t zhaGIEkI6-(ZJgV(uF|U2r~Yc&Ix(cRYtLrsw=)Bjz~Uv%Nd%xGxGt29mm`EgrQlPO z%TRbcTi2x;;vwj@w3A7zUGH`4P~wZaZ|-?IL5Vr}P!?C+PHnf%=w+$>znc+-_v4a@ zK;0fk-@M0et*0zRzo7vFHR$$`_c-zoc(xB&8Eq7Azb1sMs)+QH2E?mIRUez_J9P2< zO+`ggu#=uY1Q|Kq+cqsX!3Y?*>GsX$<82=0jxLzeL;J`9lT95jn0@K@JRl{t0l1`& z%(}2wm`-PHGY-0e%nFOv52)4eL;TBdWsOgurxNoSGP%Tp30s&!gKcZMszf0z5&&zb=b*%5e@%Mc zCEMiaJ#b@n*_|~5s>L(MMI1HT+riZ%=t ziu~w%_wut`!X;E1I181APHH_k4gN)u)^>WF42#DgG$JyE(|$YWP-()ZO`C9kYGi67 z{}A4z?}GrxK?gC(>SI*VDEKp2@{%FW80sWwqXG1Q?0u3ER+SQbyGHqBJY~wmz>10i z8Rm;8w5cKX+6u9U!^F2+qv%TjLrIV3^jX?&bfjdFPzazA#jNUgPdWA=!nc{RtN)PT z-+1O_5xSJ|LJ{xF#B_ybV+-suI+un@GUP*`a;urX0XGq+&V}G{#yX;x)>FHo=&ezN z+SLfnzK7a*PEt)fi8hRPy!hfekO`3Gw7`OiSIkSy6fZ7HE0E$>e*Y5>eo@?WSRpj3 zs+mpyi5FU<(!1)1nKm)A+;uKvB z!lDPz-hygLG#FByuw{uh?6?rW%suf>kf3lr%EvWn{(N28EH+c>>3%;+!$u{#Os2)a zjOPIv!0TJcyUxbJ+T4K~UxxO}w1ogSk{KIfD&}4X2hH8hQ<2*4} zye}d;hrgCxPKH>6%iksw+abMJjDt|h^xt0pPzB=YC)mvMsj$)p!CjwrL;36Nn3butDIH- zjez)}8@=vxJ(5_pMLXkH!!=JED_oI^>z&3klBNWytotH0q$lrX&PU(Lf0w(w8<@6S z5?N``(UiPc)@6BfgiJQBYZ!g!18+-$x4iF5wr+K$6N7dVEou2)Bz_~WxnieB2mH{o z>5p5bDKuxC#VriyB?SgMZc3?L7FCeon>;XR&VU%$_5WW=J^eZ}GeD@q==<5y{k#BN zY|%CB!pEyQ)y5_1AGXvf$Q(UT+dxGnt?|^>xm)w*`uq0&chIam?0>dBqyJ8t*OF&5 z!WxIaP21Dt`TZu@Pp2yGykxC;0zsc&(&2K&`@tqT2~0kL&?yvDyojk>HRKZ}At_Np zzcue2YwVYb=hapBTzxTb^8@@yjqZ$l#_t)2BwTIC-w`E{?=lkc-0g^B^A#87Bd(8_ zD)+mp2w5d5f|{ut{%k{Fd}qk$Z*_Sk&o|fYbwz3hk}T0_SPU)S65OnKAblxbqfNQv zc*7&A@b0^>02F0755TdF*8H;y@*}TG%R-}H&6r7?$1MB6+qU_{En7@$RadmhkLU#K zsa@QYnb*sa$Ztis+sG^4&l?dC8g^>drTv-JP6yaZC@ItI54$Z2X(=gxym0^9j5-$n z3*@zA-+#&Qj^f5?_qT1+=Bl_6 z`0@-*^SG!bT=i27PrNuEKxSqIm3;o3fD~lAXoGgGh~ze^9Dg3u-0S5d!Y1E3urvQG zhdb%mqtx~}2MhVClDfk*Ty;QUqDM!)@X?u#u0^k+1cVjDlZw^ci`#8k3+JZz_rH1P z&Ng`Up4!WMduI&cktwd*={)MU`l0FFJaEqi#HG7~;BV_#>EgM8PynJJJ6m-wUXF-X zr{EvLyb;xwuEb`PmY&`v47K^jj7}=wkfSL#?J%_QrKdV!t?00iL1~z(T+OvNSz({+FVJuQ_IgZVFl=>&}IdSNtBTa6=yL zI8nc9K;0NQY_s_*OWjf%JqF`y09(xda_@_Y^%^(U|M2N1h3wb9o zoPLzmP2pJL^JZE`AC+eoALLEI1cDz)P>YyCz$!rgT*7s!H+k(vrWle*Q#E^FaVs`^ z9%0w_g(!Fg`yn8yrBw99g0YC_5dLuR(dZrd{={p@y}f+-At=XvUG_Lvr6<}cu32O2IRDo;5b?p zG@LAC;e^}EEsLd>2zJ}UJkQmD2Q$!chUQhTjDIWCZYj?#K%7W&-8aE ztxvMi^112ftzX9*Ml#{<-n~05Nw=M-T5`B6&?WOkPY)WVSHz%dnjpk_U6&@jS%Fb8 z7)G0Rc^?WZx09f%E})q!ucyV>qN9WE>(*KSG(&CQ#LIM&0%qm|AH_;tQBIf?rE`wMk`?0^1Ff#bD# zFU`^Qliv!W9T4i*&~$@dMXb9uk}?e;-P`x;vTV|I)uw?<+Fm`{NlZi>!U4sLPgchS zy8e6Wa*zIxyu~Q_x}II@16QseZlqWeqmP4vuAMYs1EXw&p;^D8#WA}PN-;a`1p35m ztOz&;^PGooRySnLV5Evy(cEg7Ul6(u!QY?u*ZARX;-;#LtvWV0>S$`bh2?pW8#^Q-gbh z`)=vLH^^!+i~xYfNj{{*#Y-zU@2w#(%Vv=` z$J?6eatsbh=xlu`1yv274@65Ey$dpZ*>8A{UcUL&+*&sTN!nuP}w$o2PGv%cN? zG#wcSJ&UAQA_0E>pBUuQwVOK1&s|1NhSM{d@6wCgqXA=m`-!;-?9!ykC$}}X++M|% z4gTbrGcmDiLc1EmA?x9hj!OVeZvIzd-R91-pIxEG;pT@YWDi-VbooSwD`!&5i(5eg zra81-)o@_?piT4=#Dq565WIT*Qy+&utwl8@NU!o1qa{6K)|TsX`-vmQ>f(UV|BpEO z{Ni=T!x0>MiqD!NuHR(gqwunyr2B|1Rr_N|#!DO&tfzoi1i=km6@CByWaeX++x>qz3psvs{*1ZB7k9k+ zLsT==BHNd^T=L96os?M5WnI&x1=iMeF)aDq2Vz2v%q3TvFL`{V6)14enD!f9#leu0 z*yK9a-Xgpc&0oT=6KGP&3t6XNTbm5<~!5SzyDlk=Pyx~;iL+; zYSjSbnj8=&qbhUlBL-IqY+_$RjE6^l?Tbhb6f2A@d{^gWe~S2lM{0ZzJOzXWe)w1& z>ePNeG|PYWWJHFRtpB}pGku#?FBc{^Y~R#8X6;OTXqp&*Av*b1S_86saTmr&sgpvdf+o>mY&OrjukE0Kuyi97438*i7-Q`dSAkvIKy0I z$pnmuU)*atN0ciMY%V>FeH;*WuM|QF|L^RQ5}xCw_YHXzjm?`t9!-Y8-AlX+PARjb zEKm4BRwqa5Zz4W_crlW%Dco~{b@75li*E22M7^i60@c!5;;-9Rtn=%IatN=`Xi3h( z;(QI|5XYN>1RSn>A$9Z;>tcgr5RfjL3=`@5;os|7e%r|O@Ps+kJy1j+ zP!^Y|cZmU#&T&pg8ebNn19rVOOwRq}!BFfOm)@4Xa^HmwNQr;nAnl*C zicq+Hk{57c$oR-;_=eoo{djwj&x*D))=%7$f_J$q zBmQMf+QVF3jr6U`%M1~hVvd0C#HW8 z5cYeeRoSiG)6kl<;}w5IFwak)KflDj4(7RK;_Eg8c1pV`@YG1j@;iYp3+|KaUUFVK zp-iNR<-*$X$f;5;50JWN|NhVSGHh*Zu2JO*1pVK%;iUYGTooAaGek-whTb8jKJtoh z9=PQs30dsI0s>KD8Q|R{^-xmb$`cQ6+1&4UQ#=a`_Y5jQ{Hu|}RS4QDVF;2~Tqq&; z{K@dbHiM63z9}l&o@DLy-D9gHY_{<3i)xq$U%+^gCyRvo16$us02puVXK+uNO=j21 zrau(~YYRKyX{>~zaK^-_ct{`@u8TOv`n?0|tsocHwbKar<=JuF+jpO#kmN7p8-1Kw zISJ%K%t+B5{+cYSlEa(`C`4FI``Bh}@X88#3gn@HKc8OAi}Y-w1ok9FpU9iVan3_@ z`Jv-Gxg;=%131=%^TP|Fc)vJ9eSirqXYORSLa+1U%4=j?GF`#(h3(%Bhr4lKx|3+wm9lxzCWx@}FW=ZLF(?o!;2-P;Ep-1p za^?o#tiC_bpk2`a9?A2x=h(G7cX)pMXNAo3)$T@0r&sQlcC2QqwNZ&YGOv;&HRtc;+#^e1Y+6!e8NG2W&nF zua6u$G)E>-W-N&!jjQ_9+oa?UE+s3X^MyDd`(^u&Xs4p0Rt}`Aj3BA9NB%jD|6kU1r=)jkmJlX_M8J%eJbQ+l&*0>xzkqWaH zaueVHJa&#@P=8MFbw=O9$fmf|MiN>M4L|6hk*nphwZACIZshOZ9Unvwo?othG={yy zXi%Pu#Ry5v{%(G2N{WlG@x~D!hjc-TCK&64RpjH6!qa)_8!P zo)mn5Br3k}9DYWN9|R5=IIuOZ0I!fxmcf_n`=u2gux~pbsprFWci=%^R+g6lLO9?4{XbNf92xLHDx*tQ`Ps0T z?z0X&aNuUnP>ld-4gpB8VtfB_4M{v+g`F%ZG6%iwE$yGn>)M+CTKqdlQ%PvSg z`usZju`q=};q&9Sxn>w8Z2vl}^`@9SkCI!g`bw6R7Bd!1MB-azdgFMUM6wClSxT$FUInKRC(-hrwsYSAKQx5RXJ#QcUnmZZW)S zXp9z30yCQ^Vc*^);WqEI8&a1k#&V(8if7W516EmYNm04e`uKL@-^a%RKB!1tk{vX} ze+G70u_%F;5=&oSLVg2Mbds68XI@&_BqMzG7-nYq2Sk~m%l$yfmrvlNH4mAEQRz0@ zH?Dt|>XrttKdmF|*{t5{iq;j00G+z8K4j2(R zV*CsXGnr_rh~;*Pfs*pgd4Pvs;)J!8)7rjy|9&*~Gx;omce-#_Mx!IE&C4KxVA5kj zTv#H81qo$!7EEUB``xt|_8LmLvGAHqrm;H1l>sfkr;7471WT2uLU^hgB^qKhhE8X9 zKev1W2x%DGDZ#%F9L@6a|nxlMg5jw@OVK zMP)da8G5cfpR1=~`A#xx$W>&m)(Ud&M%$FY5o>JEuRo^Z9~;0)ZAdN)afYHt==JN4V_a zv?X2Oo(s_k;?Q5X&$r)p=51wyN841O1OS$if|&%qcdu{sx%Py}Hudi#EP=SG&ToHy zkKG78uSWl>cKiMm3;aot*aHW;VgvMJtW3JxTHe!G!~xesOl#=zUSYzNVFy*z8nsAi zfp^Vm+Uye#?7?(9gZqa}4jPHK6P*=nz)_~SylBXH$-`Ih z5=VJ$%P^ip>n^XQ%lAIcvY?utIer^VpXhjmJiv>QybP(3SO9tViFsADH^5o1-oH-} zOeQn4u-G3GNz6I*$px+I7Kthi#;&DM^~!ZOR>YT%#)g5AtGh>o1`TXKeIo@~(ku8? zFuv#Ic{hHMBxl3VCgI0ndQc^R5O4~44l(SJiS=U8WG(Lu0|hmB*d_ev(d)fqRl^$y)rs}ritMl_1uQg@HqgyHpS z4Ig##kz+!H$J3_=r@q4#lx{ms7Lc>2g9nP5GRr2lX znmel-w$xZU!g=WF9vQbQ_T?B@bkWcbjhPL%$ZzBa-d{^}baXK8Lh~cTB|p4tYO?Cy z>xkuKU#Ae;Oe=kj?+PcU>xiX*@V8}WXSeycf9?*t)fxyF;ea%YRU5wLrtWY6tE#&w@ZE0)Bz_9i@-I;ocJf?x2oV-2BB-? zy?2PnNF?JSrwNe)u*;-YmmXEikr*-J$-dxg)kqmEQgF$9($!t|BvREt+&ADvyK= zK0Ei^W;Vr<5)pQe9T@*AE9l$#^zkq}N=)^!D(Yz1s>g;*&h5&%5NW9i5ceFWT#0%q4O-Xt7;U$a8K78$Iu4G@t=|JE+&8=;+7!(K5lZrj9Q!3+e^Z^dc`$bHVs& zoHTg9LdbD~!fvU$*@%uI$sqB(kwqwea7k6;)v!qIYt;e@c^_R~H<)X|MnlzLJFgaq zCGp@D&D6r$OQh2-Z2EW~Uk>Mo^gUj4dZDq) z!=q_Ha3Yav;t&9VNylhmgUURAj9;95O@{rbX$`pOfCvaj4dsrpAXSfay2JHP3J6&83bne2y9G4USp= zP3^{w8*DOP=HwKx8B5OvI)L}@-}n1Br`zTd9}M7aK;oBZgOJYr6@Re$(k!L6ds|}R z4!hBfXPo>Jb-7toY?09&38?tIPvz}KyV(Q&W0~Kpn*E^O&!cBOz+uFYG=05x`ru8Pm%mWDaKBBFoE{ds6ue}fv_FlqSoG(vV%I%e zK?xrkG*ui)dB+~^BhiE(3P|^kC}3yNf=%bA<2L9FS{kYug0`5}Jd2 zim-cjdSY+LwT+>Q0jD%t+={~YJGdh`OrUS%m}bHOsiT1 zBT`&kd>2*6LzrY`Rnz4?KC)U?HwsYXh?Vk8QXTg^}Pc_T&?fCnhI z?-^z0%lILKFrnuDA6&q#*(ciY`gD4^%Kt6O{kF*IUk4j2+26EsdW9~7G;mkIc%V62iBsQWyliNnYt zc^-6^eE&86+Nl2|73LcH{8I0-+vlsE{fERnX5hA~zv4izFNZI-z4U3Q+h@nhww~l1 zIMQq;OR|^NO)`*(sK17J&-Ou+`-~xY07mgHpStGrTNKrGN>_h7_Am;ACU3~27cWjf zDL!oWee~To1MQ02fGy0rv{HON3Q$yD`PG%F)wll+ZW0-__~{^#JBE{;FL7#!k(N4^ zUodSgqD6oKwuPNhoF6%8+Gp7Jb5jK zEL&_JHNALR-^;b?KTiFUvEO`DNJeJSiz@*|&2yqJT$m%57=0G;)6rTpfGf9ZHGq?q z#&jw&R;f7jyEnnr)ksmQhCs}hZN{KZW0Ft$T_Is*Ns z;nka?Ii+%-9GQK&UHiCJt%WRXkCIcEYRdg~kQynGNbqC_lgWUWfWl6lt}a;r+(=&K zg9fSP<>e{tlCb-}eOshV$%#A9rD>6Q4*pzZTS=+!Gm%QbN=ESE2PGcye27dAvi2yz z>68;IuFdC~h_FdfYd^!W54LFiY8&K*SSBCeo)>XQR=%9FE|KS1?~|q;P3vfAI5Vh^ zUMPJ}0VQLQ?VwRRkKa!#_4|6gZa7>80tP47gZVLXpreJ@?|pi(1lLiWJnD%+*D0 zjg2S$@qF;6ffGQO#x}6obbNm>PcS)D0xg1!mr_ctOu4MXJWO zj>wj6TFA8R)#V|poEVWwCeqw}`qIUVkW8Mu)7F))Mpe0!nF0~O#FPBQW+I*;(l79S zQM*yA}CJz<$-l1BcY<_lIp>GzU^MJ_~YTEW3SV9yDs8gZe|kr9OV;; zh-MTMcq3nPid;}*9acTJTxR`wf!NCT%3@_c5_M)w{0=UDNeB^*yG)+{Zn z&Z~VM5h%iZb(-n$J=k>-Y#rKBKXuynN({IIHe?& znT3r7Bt}I9>at17(dF$*?V5YiTgd!hTItN(^ESNJlA%sO&b9eTPtx^w8y|W-#--%L zn+H82rZ5_!OHDpt|Cto+t0>ArJ-6_3k8v`95tAcv;X-Rs%HwR$Hx{>#g1R}Pm- z!%bI;uInw}WOzjk(Qa0q&ga3wT?+GtJAM3x+mh9?qHv0SW{9UgaleeNhO0|&+H`nr z!3afJ+c2P=xI}|PABzTHmP~Pd?H#lF)2;k<9f7_=^m5%H*aW)^;~nGpJiuD$#-v9n zDNR6FNZ|cy%u+7&Y!w`o$>^?AR~Ij{3t`dBU?9A@`*J3=h_v$q5YrECp9Fx1D#gtA54;xd<@ z+8N1dYpiV#5Y#nyoPtf@V03Qud2ibczT-L1Btsk|`{n4vUehL;owI3B8JN}dBZ5*!R)n@6aiWlQtd%F*TMIJJ&?#k&s>-7mq zIobbdhhAe+F3rsZqGM!DIs4piGj`Q6vtSH!4@(I5_-^x4EL(?#g`r38qTEEVMq<*? zeU^|>a|0(OR#&-0-`Ma89Fl;Pw7tpiupb%5s4y$hujfP1J{iM+Lh$sQsH3YJ!NZ{p zfdDJOzvR?{pUQf{eW$&->i@N*6V-6@Ck8IDMdjsB)dGMUIr+w11S=2}3=6{Q z&|*saZsZ0LzVDV29328! zM2tepMt96pP1_uvwY?wjNnk()sc@N@jYVUu6>>beIt*(jd&b3Hy=)Q>Pe#Z%Wr{3W z*)oPK;vQSg{AK^Ovdkt+Q=uyKx?g@7ihG&D>x`+4tQgmG;9ks+u2=B>{jPwj%F0KJ zRt~Z7^}XnD?A{x1m(aIJLdJa9dyv5)I9S_WE99uWDF&j086RXAIQ{plu&hSf^kp$dm z=VI#7==VO(pKTmF#7*m|ZoTxbZ@G8w?CD-!UV8(L5@C0z8o^s|b8{=4G34i;>tG2w zO4tBo3k>513?7s6H7tL04T!592gpL#2)N3oRVNO35&cnxcn*=4GT|O+Lt?eO@y0CS z&loRlAV5%Mv%x7#h>TK(y)RNZ3A`u)md{sctQ>yevS@52X!s#O4Wr=HvWo7FWVs+* z%Fuiej$-Eh4;MDp8;e_|xJh`n#YfaSe8@cg%x#?;)#WH@M?Zf8j=zCk;Q-eah@@fq zwk=yEz>Vo%tca-}uAaAVs?$NkM3{be(J@F?@XRrN>cI4tc^~#cWM!;_L3{XUs;jT` z$u6OQ$sHsdpPie6ez?0{octjZz;g-OSK<@OSc8It$*LYoH+8GgK@S*W)j89*TCH)c zVGA(}M{4ICL$;uk%LyZWyu(76N7N$ncu9s9MID&oYHL zUO*^;P)wQ+GmnMsBgQ+-WefwI8zrsS&1*D*pMjU|M!}nJQoL7e+u`6fROW;>5^bLs zhE%e zHgDNNUSm*KwTtPT0PS!!E(qio!~$*a+s5(%xSsZN9hJ@yc3@;~@2Y0}J7|Mov+dfo z+kT%8aRV>G?}??&v>`h2BO^eFv%RHIU~-#tY~esdL&J4k@OWw5eHG6=?DIKue(XPm z`fZmjUyhepGqA%}&g|`K3=uV+i@6Q^K=yQxO!B$QV3&$6hHr|~P6tcCO4gcNKmNY* zh;_KYFdMdPxlZ3pWAVpM*$If@c1ZDGz)Fx>jhey;=jC!mf6^n+KPh{c@ z>|6crwGB`(xRk}hp!i+Czc#}KhW1W74hUsg(%<`F5(Ln_SoG}DWp)1X1(Q=b+!|*a zWL0XcU%OV>ZHnoQP-YR>K1%dA4Lzrp%KvYYpB{cTeioD*O&aL@GUJE^;`<v8g{L$%H35l+G0(wBod3&re|N>{8pJUb zY{cSy!##fy&sZzvW14)ClXPaLd7e-CsN+V{pVZ6U{_-lZerh|O=`47tq??8rK(%-0claAN-a(=UAt@54H}Ty9oMnvyKDT z&`MkA4)(+2-NeLX`;Hwy+n?Xo7C0ZpDTvXAI0lH$J$tGW)foD*9LS9Y=JwsY^&a9p zBSSxa*S9y0)ZoS%$(f?5zCI0dyJR&Zg`0k9j=?;f?#xNEaM0QSn3o-npy5zRbF|yp z|1a9em+IWP^W_uq8}o!Z!Fc6S$za1e3vbIQ$rIDk`H0->O80^+l_>ga$%VO(Jnwcc zkKQ;dBmSg+*WJn*XJqtcwZ9`~GQudS%%m-o6mrYCL+*ZYoI->8)HIkFqn^rnuePo+ zLbjya34F|Gq$JRre-_uDn!(M(;|jQ0_N!OAHr)cBKqA5PZX?*b#o%t;Hd(&b*vFe!JJmZNY+7jPE%5 z_T8DgyzGOH*wwOk4$bF+iNReU)fsmP`l_`IYNzAke#Vr8YPgn(wXCGZyZSn_A8(n& z^%{@xe9q^Y_?A!9$jHbrYf=BwsJ{V}3qw4CE@ACg z-T%{IJ%@)`u*-k}r58-G` z!0`ha*0X6JatmBn+uqvRTJnjpjSrenkqr&)9DC22>5Jb?| zym|Az6Vwzl!J;IsNU>}ko^W%xhD>^obI|M3MaJYc%7CKF;JA7ho8d`U(`5ozmv{zk z*tZFB80NtgJFF;lx(Zb&vvWx83AwS6zp4g*9HZIlnP*89jXiI4EqSFn z|Ke#vaefQwX=L+cb(q?d3N;~&m5*3{!Or>IfWpoHF2@&K9pw7&v6~kEOPfa>a#!2* zUOq=-d9}H~C=ULEyG-8ab7xC$dtbGG-gNWYSGozo&?Vmras3caS@*1j#`t4#)urxv zpX&+?jEb{e$Bu0{ATzIK@YGoh*np0_F*PIb)eB%D;dnCj=d|9UYw7@p5KiAnmbtdw zS?d1wn*kuv%O6MDE^{_j$U zFbsg>9Qvtc%MVvf#modukLAROWAuUO;9MU1m3ggdJLdC!TZj9YeNb88TC!%MD4RcDx&s)fE&1+zLy)T{*);xLw1== zQG}az0`uaf_{kYfk{RjOR|n+0L0lkq!{9;|2!n)kXOcGP*0tnhVWtAn+a2d(W`c$A znXAsk2&b|LJzL{&p4=4LkF1IcEQSNncF&;>F?(HRfYHeh|N4=;NJ8Kdjtby0VZz;G ze!aIT=ezQ9xh+_mL$PsTBIs?bd+*t^!BM7gxVOE1>k^tf(EkI8eZascF|oC3J-U;2 zqq;n>>)8E;`bL};=LK{nP+NYuw`1@Y|L9_N);bLpZb~IGykz*cd5z7iMk$s077|<( zTRy11!is+n<`HzC_Vnt=p$5*JW7tUQlMDPq&%s2(R`OQ%wv3jRN5Cq-Cl%39{vj^= zBMiJyPfD6=V1=j39Ec?_n=^o$9*JbH_Yg$Zxx9pVW z^e=LG&+7PuoT0}$`Tg}~WcrOfUE40Ss!<&@S}po#MDx%8_#P--^7fS~`pqOH>8`rE z>;s=UOowxhx*4yl* zc-(cqmikw><+`$~;_MBjvYwuHJm8H}H|Y1jLS!h*Ok>7kIRX}~7k7e$qdySm;+v^F zrnGIBF@rH$HPBdO#<^hg0d;r(-5b28M#o*(^YYEt?;%TCSWHf6l}e)8^owDW56Z^NmUbz+Q0z^ALJZJ)K% zh)q}6aqvGDk34=k{`@STpWbZIS+(`q97BFMDCcnIBW+S}VgH;f^$mfyEW=H{QJ;@O*YZm?SzLZn?9kTv|Uayw;@riNOuOwk$Q^V(l|o zHbEk&U!dNU0cxh_z)+>j578{3i2`B*sVh4qZCza$oLPtQh+yd(_?R~C(VnHdR(!2A z7FmY`Im&3Z&)Z|`scFaQjI@B;l!_qnFcR-BorBvBXA}s6E7mMa!RgV~T~;V8FU={G z{ANzh6NC02*aPsIxn6R<_g3OhFc|W9N}e>9usv)~4o#l;f%kzQTMXXVPq~E6J~^ij z;KZlY!n2GlLa0ROyO!k`JT@S@TaNKGs7y+od-rY*sRt=T`6&P9OX@RkAg7YHRype7 zz9ysI2Z~t0isj3*E>v>;!|z|oGQ6?>`a0sw_vqBT&5H9Ib?y4J6Ml@?%zExc7UJ^s zoEl$CA7}AfERYnJ3SG*Z>ZV}Vafd`~$-QLdy>9|sOa~2wKez4VX&{T1S(oy zCql3$0DsWUYbk)#haP4#`SOj`j(O-D#^`itp|+!>J>j2B=GV8K0Xl1=le= zH=)Ybi*byxmHX#KZ~w9%VOq``S2`Dt+9Q%c80LBkH*L8GRHkE-ovuP@I=TGQLUo(WZUp{PRfx(vbmHC{ z@tIfhodVvnhJjb#6xdSPgo@ix-<=7-K*dOI9{aSdljZY+JNpbC&$G9_?u5@lQQOMv zXX5`=gAJLWIf})!8Hl>Bu~)`D9Hgr0>)+0Ib@$hmk(!NS&N&i7FZD5Smf0_v@vK!Z zXp+ooQu&yrt4EE}v*1LYx86 z#M!)8uYJiIU+EG&gyb=_z@GkJ{Nk0PpQwMJ^jI=!cH}00`PKB!3ic!D_5L8)9wsIK z0wyIl3<4t9c&q`3VOCzMw$aH-3-*g~v@_+=f)Qd6C8k$Mr^bVyez zFQ@0MX+E!2V=?h~UI)G=hNj;ZdcyIdVPH?aH1h|wJ2NIcb{o0WCQ+#tdevi!PSLz$ zz3V>c_CL;nS1$bY4Nl$QG;;0o^LEi$ZSxndm|Inp!}&B&J<v_}kc@ z7Z3z}pOFXOx8u$Dj?eI5hE&jF@3l(|HBjD!roj=EK|@Tz_HM|*zC(DMx`P`p2hEkl zW!$WOy<0yI*_Kq%4qiKKc+laertsKZ7gq%iFG10jNF`b~S$imEId}_|o2rbEgl!P1%-t|aHV?8+s2X_g)FqE!21wJ=8e%m zi_rWkxeoSpyPWo0dp79aMd@(rR6mQJs0!%GaWRWJg}Dz&DT;jG$&-I#!NI(;SW-~J zOqOXyQ8u$tH_{FCI~-w+25cj%@3C8dlm%SRo{FD87c>}lS;{__UT8i`;AR>|b4o65 zb3B5qjKO9@u1H3do4lBGQ_JTYq$2Z|jApptF7tnQs-24C|J^OH-A;es;o3z5K6 zW*cY7)IrBUaP{I-p5=egO9=WH8To4eEp9|ARvo64#ji`}mKzAX8514d`}rWJyUI`} zrpzzGPG7fAL6O(Vc*U4E%YB6Y37m8no77Q(?$D6kg1=d;85{;h{A&zaiWAx^7GQZX z4AGG?dB(rm&reS)WG#u-(Z(vbn1|oHxc)Q`Mpy{ z*^vh=aEYSn4yO%ONm0AL56^qWF3ULWsTk^NY6mSJneHSek07eFVJU zqRR&mbw&VD2JE>R&r)~z!8?Ckyg2h1upYwW;b5H$D!<--{rXSK+&LlVi(8o)SNI*; zN4Kwotn-CMbQc!mu=WERKkY-2}{fZnB$UQ7x8~r{f)&&x_u-P;i_1qfglERoSa6Bzl~94{IJD6l&qD`jqnp(iwKPtM*2xKG2Th0N6`x4KqDuL zFSEjGzXtl}L1N7je`0O|V7tg4fICedzPl#6WP3liJ4eK9DqtCV^e7BB`)y?ioeW(wIt#IZ;vTo?VuTmynkTq}C_*)fnN&t>wxPir-YX5&QEyGJXw@|YT zZ+WkUo8gnfi{gT$TNpdq{k3n|MRAy}`EY#rG0knvw-(d*gyIY4cc_Ue)Fx`Y_-54+N9JFT1R0o6cDvXG0(>dYNulbJ%-j5!?GRq=EZ!X}|X_8dHTCP75KNX(I@?a&!{(}!dYFPwr)37kiZ zo-E>J(C@HS@hkm+fKJM|94=<&46=wO0>Ctep$cR{O^FzA z4HF?UE@_ubf7W9ixeA&mfsC8tD5HEjcZmKGD9cRQdy2knhY2NyZD32cy-%>S601B} z%nTb=hkE)uwDQoG_yc;db>@Eixl5ONoQg$s&C^v$*ocUT(9TcdFU>q1rhEO~y;jT& zfzMarY$2T~wYZ(8W>`@=$~{)t9YHZ8BkuNifD<;m7d*)>#%??=jAo>NyiuCrA@x|3 z=-6Xoxa{}8kI|e-VT*hH+6kZddjPnn%F@D!!XEom$b~Rdf47ps%OS-*wT_s2rN#$K zrV)`-fR;B~GH$Jtt}rfv&3E2s+e)EV5vEPIhZ*{Q4bxqpp4*YJ>&`yMxM_lG ziKvbaGb(C9Qi)?>?#q{uS9%ZOU`KZCskS4xCLNTJKbT^l2BmMV>PqpWw@mO0tvw=6 z>m+`a7g!0V{OZFp7|JC>^LJ0A9J3P@nb9+KJdGHp4n8ag=jH0h%IuD*_WOkNNlArz=)XLYZRD8YOv#b-X=nC1E-A}*Ob2k zx@9YI`GY>Coc!e!UFS&?h{*y#|$qHpne$^)>VACd1 zh#R?4i~6c6M)EcSJ}Slg;sD_9U%loQ6cl@QLC7~&4?nOcYLCH>Rm+;i{y(&5;D{>^ z9xPecDsV>ffJMp}&&G<$r1Y_Y51dvz(s4#Dk{8B>Sswv`5b;^15Y8_{u>rQ_CHZV+I1nFF_)?p^YFee6sft|U2eM7KS47i$X8L9to zoqMHAH^Kc=W3^OM@E|9VAA7;&>VpSu#GHlUpy>1GS00{PT6+gC=#->zOv|-2H`hMm zsy1u%U8CfxFW)PiL8;HQvm$h={TO7?_`}LcD(-GDGXw8@imdZXn4x$s*GhYTig}q`C429R7q4G#$T&6jsQelVG0)^L?yd?ue`7Xv}nG*QIx76D{ zBAzzW>QwV)DsSGrfl2Qbj}$;jX5r+S)u>&Q;LF1qHViKo#}g-~KOr|;(IN!e2l_w8 zy@vO6{*V@#GtnfY4*Q*!FJ>}YeHc;_VwSvEY?e58BgNn6J&m}q=)g<%#Pl#I0p)o? zxV=zfh4Tzpb#u?caxq8XmNVnjC_E7WNkS9NX~%zT39W4_R)lm|(m*w6(BSo(H!)LB z(5#$wdzM6xFWO7_tMLzAcdZ(Qua3Ick|pA^ERnuCY!Kk-^8n{~SLU}VdZq7-6OAn) zC^Kew-|Bhs328MD+9ZG_V9OTEa{(7F3__87lH?5tgz1Zh2R`|dZYwh()lQk1#>u_P zP6A@|I2Nc=r|vzpYy0-gzMVpTWx>DmZD>9BG~aGJ#}H-dYXZ22u4&9do!f-Dt*Y4|%$)gbO@5uM zQHLimVbrZr$bqO-h7C9O4EgU8{fEuhKV-e6Uy@gab$pR`m)df*V?W`!?2I{d#q99N zaQ}KG*Ra!Az9ft!Ida@Do`0^eC*5nysipcS26%jR<3t&|HZC1LX8Zd88m~0V?P&7- z_-S95_%3vFS9ik_U171r&v1Ubesf*hmVfRVbaT)$Z|@g8&H}WtWz|s{z=MKd4|!v5 z2PZvkZHCZxY+>x)QRrbVhfL@KRl-#4BL#|ZxGN$u1m2x>bsfimnS`slRbw52k<-Lu z)c|{Y2DoeD%N6kEH9C$aB1omrgM6x@QadFYFU||nJpo9P!j4R=tUDm?wY$jqK-A2E zkDcxpu~!%?G79GMSc=*Zz6>7JH4x^gQKN{fso4)3>#ZGL2S%nKp#5Vrd)RB`8+`3Q z>1fB&3zuCUK76RFzRkbW-ea_A65xXYeOqT9SXpq@1#m@o^AzuXL$;kCxRHInwK zL($tz{YJm5yikZ!v%kXt5lIiC9(xt#0c zyp#DZLv~)7BMI#A3g?&X!-@i{v?G=K-3Q9u$XbxCuo)Kg{8xZ+uDj@$MuMu~Oe~T= zOr&78jb3+cD7ONAy)xQR40MpxkJSwE1@hIa-0%B@&7o`zl_X*Wc1&}dxgk?m+Y%Y{Qu+gnC!T0{AU=wC-G0D+yYw0v??>?vTKXr{u&Jm1e(C@VL1-$H z3ZYC4(kl0cEQ6GJ$BctAK%zoH=y!8Zz-UbZ4B?d+W?W^FX~)fm4`9d&)`nC^_Pll_ zPr0)MuvOgUWyv9*9+H)j%z+1mm;Qm>F_%avDeLc!#p? zLXOD6d-ZpQMT^>Fr}znWU}9pYkq3YMAUp`;xX9u3NeAz+I0sV35o*|xdp|Q!j5&O_ z7b?1lHS#6|ik6{(5^(~~Y<^K1Y8u$eFZA0>jC%AtjW|Cp!(h@#9pF_=Ho_3ip2 zn_DhN(t0opVVU~;qA}cDRiNKfdwmRg_Ecqs0D}3ZB|iY?2FlDh@u8lJ z&mI!Vy$@YYsaTTIlvk3MK8`6Fh@<+nTe!w3q;+!mWc7f`&$P_@#K9Qb9+JwFYnUT3q$bC>7c`-*K!3%Jtzrb{AxTWP5+oRFZ zqhO5&y!s=vhINw7&LGh9raOgIQ!;ZIgw(A@#ex;KKunSto&;@N9;#We%T29bR0+_Vq< zdvVkMD_woOWQah!^s4--p=t8fJv4i1MNGZ@_n_a8s%uE2dvb@w@c+trWD7waDi~?jE{~(KX zQBN`7=`(iA1H-aMY5W6(5KNV$aKg0&BX5CD-?heCY+n$RvH)CS+14c_TTi%dk@U{G zMSX~j77OW0ZDxrj4NoNi;kC!I%nAz|k{{X`?&6b#*~;t;$M-dYa1#?@upmD$7seEi zuad?4e6wQ|7Ok+IrV3ICXE#wXq>XVR6c;w}cO7TvJv)zi2ary-95lHIM%1y)F5FkdQqv zBJ0!T$>GeS2q(cXLW@d&@-B%@G^y^R;pc$&mwhb1;8c390lRj-@Zv>Cg$pv zTTD9`r~^9iWClK-7_FyOH_KqLKb!YM$-X~_SipQg=T#VXNUp6TMs7pX(jEH!tfx0h zc+EF%-wy7owj+T8zl>ne?_WFasGDo6mT+rrw^%79ufI4u9%9s%nT}wYxZR zLyIrkM(D_of|sB|IRajC=Pf+fbv%q>3v%Mqo()9&1?r3yu@44Fnp3_l;>t(URZ_t? zQF5rN69@0UJ8{{H73>!*2!Q$tWy;bgIEAGae!BciulCFP#>Yl-IvlL0?#LQ_w`khb zZ{wNP;-=vb<2KhqF)%elv>QZaN7O%I`1xwL8~cfYbb7BISC>Vchs!EcS=!|$1hMsm zZc>E9zq0jf%MR-5)q2V&_}K;EAvHjOz#py%1ye$}e%X1mYxD&G9x966ap+pSuPJ(B|Zz;(K(-_rlDFrqeH0 z`8HC4$HdUo#U>K{|dv*F9wXtj4|8wn`8{XYB*qE_0oDwxRoE=K7Pb^^;; zC-&iaLbp1DJ*vEfnd2r56m%o&VyI&7K3f)vOmxLxitS@Q^P9vI@ysrLwhPruVPf@G z7$l+j>=ni8_uRRn$%o^?)-3Lx{_Fht_B_ZaKZ5%{I6{*^Z8#^MuqOAz8S|v|I#PB? z4Z}^aFX7B_@rVP%bvx!a2wuFD;B7+396mhhbhy7#ZZ~M)z`wBskzA^0y%a+L%w@-+ z@VYBv48F~Hck@=zYVhDmVPJiJVFtGlZVRlW^vzP83>gl01Yn^R?>;W*uH;+ePpDk@ z_sN^c1*BUxOO$o=q$um0DWY~I#_=AxBi{!FU1c*!FCLGM*leFi11Jx|IC^=td*nR`c(faC|n3fI!RKzuA z9S%a^Ts#_O`(a4drJ<}5i$^K?ug{ebGn5%H^PXjieL1g%q`N`R8MmNn zYZ?fQ5H5v8lIn0R@oRba;bvyWWmP>+QzMkNM31w$;L)cPefq??6SxurHRh0N&fJ=E zh()J9ap3z3%{%76jQHxTh14YRR`0fluTIm_);90wSK+z)ZOXHm%k@vxs8`!phy3+m zAu_sXkr+I>ckiB?ssug35FJlg@I!ECh{#^L2rLqhnbPb^@29 zm%Uc|LQW>Ds-5>G6KLe7*ojtF&6vux;~?={mKn8D4B^O0`V{9|`xE!4GQ z2!am0U+nn^c!kl5yrLt|&M+qhz(a~SaU$)NGOUEnp34Wr^*Y+xFIYGsi>+4_xylnM zAMVCoJx`@}03_v9y4`!x%KKiPC!;Dh!(KPTz0jVrGuL9(A*>ASJ{h@OccDa=hF@|S znHPoR0c9|B*EsY={DT-C3F+%HH-6;26$TYxAQF`cCA;F!b%$x22{2hYXrx)%Fr4aE zd~V7LZN!Qd6@zwf-nK18a?jxItbXHk1`9H!nl-QS@Yu=XFRD?)urzcJA6s`NBd#R& zV$P5c)v?Bcn^Y1m2{CLhre^<*wPw`uX>p-KQ<^FFDr$OpKazZT+zqCF+5$JGaRBZ;Beg)ABX&1VU<4k9^;ow1OgRV2KQq$$nFnncJu8}geR-#G`w5cWQxop~r`z?_PZd-pL)8Iw#; zxpaS}GmB|1_Vo`IHYkd`@)>Ne7%#3&*AB6%R7i}<^#mQc73}EN{@?BRjmW}ghY%pc zh6tR7oZ_dR)3i})ZxY>G<%;r(id^DsUN{7K-o{d1WIo1ebrLd5MIX)+f(Mm;Drq)6 z07~oWMGvRuo^K=c9d54!vt$q7$W~cVPeD`jP_<`YBD@&=la}Ov=d%jStxGGe12z$)r z_P{GPLo)h@O^7n*V9dJY)Ab=;e+-nw3vTNjacP1P1Ov$p9;b8CJRIc2K7%Bh4Jpbb{LL}OS70H+ z-h~F;kCiVs*_f?C^;T}){G+gyl+aed>s-zx3%d>5SQ?>w_w4s`s z?@;1QFsBW`Gr`*XQnf2<7uz!~>TvL}W%|#KTmwN0mklIQ%pGbSRA2@4_-XUZGZ>hF zCDa!m6$FOG=i7sos_g0M{;JM9^%y_zxtz*VgRI*&xD@bd#GhCdf~D4eq+Gqw{o59P z!;QIc!E(OT_1~9msm6&s)u-9iMJQ+++WBTj{Pwg%#_k?d>te45Ksj?lwQxn&?b&Kh zjt$VQl0St+ypg`XgK6onO>=hc zjv8Ce-;F&1Lq^~?NaMDSmt%7r%vY_RJzd8r&H=26vfQUcF(5FqG3ztdB)m+SatKee zLk1!@*BC%{1?++Mij{9vX%;ZTTHy5L;lpIzRZAl$A62Sfsf9;2&f=2wQV%pDK@M@b zkO_q-j_-?@mM zld@CCTmNTfh+aTb`iDQAiHj5YfwI=Z^_SHjW2NPSsyq|vRh9clHo@F{nN=75{*6r* zOpfdPWqLkCXr(~+Rv#^R8e)DMLk(~Ri5fwxAOsaUs?e7Z1~8D299V`nd+-`eUQbWo z3jfC<_W_6D!tqc?J-~amtt^`@za+2~5Kk}cIN|+V;ljX|DyIa5lEC8dsB?LL(DG4Z z?PGfnR^wG&vv#dwXtxkW@_HX534J@>-pm~I66}=GE3h0xEAexY)j?0b;1{I#E?d{q za^x@&iszwDm*u8BpiJA?74b_+1nYt=Hk$*X>kuK-{{)m}iaI}@sPWl4wWyzc@m2~) z5-_EP6cKVD_*>oD%BKpk)5&0rCY>06EA|e<53GbzY;~kYfv_4#fM+Cl%;A}5*ovpk zzuoPH-ONq_X1i)V8}jobuU)Y+C%!i6#;jx2RR`Q6omc(;s&C&l0c~t|U=c zMkag(#*`&4#h|q_uOMt$2~#v+Uv@V82B6F;0jz>#N0%Mp6q}f5IbBtL) zwYtQ#TK{mWA@n!N)lqV)ryaMeLM_d661L5rFVtr%1AeF??;gw$a$l4o@V)ltE?ou@q)WO+bQo4c^e;Tppch* zD~Q>rD$Z!kR9fw|ZJYo1bNl^F8nB3i@AJ3Ttj=>EkX$DM=;@byj9F%n+xXf>tecqu z1!|BP5MMWuz~X;*9jtQv^y%HBFv@-Lr8t*8>#nZqsg%}+ut-O-m#{sGYdl92g(cW6 zG+q)?V#=CjBT*zMyvTRGHh(jMGByQz4>3-N!Jt~*#@CkVS3KHzO>0mVKoJ%pqt`iX zWwR*WD#&NW>7BI!U9-zBA-OR-kd`DflnkDy* znC^6fbf+Yzt!AT-5H_|&{W>TX1E(SInlZ!ZdLu;uMsJTbQqlx(QN5GWy+@DJ##X^l zp^y=<)Jx3=N;S-|13a8dz?aLeQ3mzsv^~l{E(k3Q;0BYeN!4wNh!7V^;S{V#=apb1S?RawZ*mNX!x z^0Z)f^(OIn+LQK>hf2DVJ9nzzKbN|O^tFHMCf9>A$hN_fVvU4uu_X&Yq%J%#Ig0-2 zQ8IOa1uIu^bn|vqaDSxQg&HaP(Yyk4T@xgp@v(zE7d~Ylx*lgE+md(nns`DAFH^!+ z;d$Z{Vzr>kT>mS;L~#MW82MFHrw&n)pVL$K-}w9qmMPyu zP}ePag>-xjgrUMA{;Dct zuuc2+_{vF0--MEnA04{sxtFIV`-^HeDhazT)STFg9nM~v-F(#9AH_7NDU6oE`;<3wLB ztNJ}Ic0lu}vWWm+|7B2#A+rvrgk+#8>wep6Q8U^IuRv+`pa~z-Wfw3FP??yMfk&|D zM!{8*nxDuVE^w0Z5wX~2nDzbee63?!cBnKUxhO8VbVC-QYiT5*c1fN|xusi@JH;M| zKfpdj;`oYu3#{2_^6ssHtwyf!di8Om(&kjI*j!^RJGuRHmm%G{-Rzaoti}XP0=nRf zPnDSv!B*;$eJUsCT&auC?Jdk@b8)Pazcz3h6$LlXUxb@UjF49E-qsUJii$4tF1em4 z(9)`)*ZP7wjQIB+QE-!TSvEh7*2?7af#q~3JpTgwDoWaxVZSmci^7tB|{E_2;}C1I@aAYYH77()A*7geycK&6jG#C`+HLV@V}W&O~P! zPl5B5IU;63qnG=yE>3}Y5)!0!ac0bbJfba z*Yvz`dJ-nh$r%%(jvR{wFAZRnrI{mH&Im~fbFP?&boxTd48GSNgDZ$S%6s{8s|C-v!)1{fX(UhwuI^u%A7gja zRme~zM=%AZNV|?5UtYU%Pg_zL#Z3$gvG&zp2btW?9roG5FjU8$9?9_h>Dc1JAIBVC z{P@x~v5LoU>ZcBI7?ObN6Nch7?M(iLzm6W&5nBSooV{~}EfzEK;1#}2f`)T3n{EWH zBk(c0?=QadfBFk?FIxcZ{_qE=`J2OEdfpZh0qv!Qf*V5iZ(`5V#<(sDReTOw0&2B& z<}}#GI;ygn8M3e=EdqBFd$^cogqGID3^U|IU*pBI_?ZLK|6(UBUK4^1MV^{x7%ExK zCcfqOUm4TrGM%x0Ssfl-D1ruRMe&#AsF0a-6^r2{K%X*$xGJ+HiwLu*eNV5s0AYYP z@NZf5H2ag}O`x><5$rXqY*0adzMgt1OvS-^#wTFjVp8Ol zR9a~YdMGA@qe&!ZdNeb;+EiD2KEZ{X0LCn~sP24Y&qJc1`aAtM-od9O+nOm zHnz4-o>l|(^=Z32)R$$WW)JvQqYOF-Bn6WHU-c_}H>Ypf@sV&1zE?v2p)d*nbV13n z;|e7G?guJ~6qh%jKCmUi-Wz;1vqPd9Goo@;rUb*H+B-{j}4p|Eg&a ziex9t0xO23^PX^}uReR`d$V8)ng)D|=~38dc6&(tC-g#rH7u8P*xG}ZNK63!{Ii8K z5}PFMbq*wfO(8vaqDU=}R?|*Fj2CyI{LM!azJesk%K_2L?l}J|LR>CGy=Ogkj&26Y z3bU@2ofFE=n9zNo6y^K4wePm^mhgR<*w8PG`%sm%RcfKhrDf`tiZ2nqj>c~TpPhEOcjucH&&N7!7~NU#*_VRvc$SJ zhE1S91sbdoXIh}Ek5^2As+0%Jukz8WcuJDwF1RF*Ej)b3iCyd}aoZLK8JY}r8;kyi zhB=Tsgd4W37ijgFV$ulW(ElX|g}$_x5)n21ki zTo_}ppYLDxPorR)9{QSq51P4o{hH$^+n`zZMwuHlVBafih-J&l*5v0smvj`U2TTW# zUusVJR-)o*Md+fX3DRV3QJ65q`xK)NIv|cQzh7HV6^uU1aZ$JVP}#uk^Rl`pOiqwf zX%N&}#c=J4pZkwi6t4VsoXriK&)xV>j-FFLyh`q#)@-zRO7>#kPPc!mc&Z`Bf1Mo9 zG&(n)n1COvNHutBF&CSJ&+ngYW=l4~~$bvKBU@EZ8{oBu<3 zj=5z>RLP=mkV)fJq$5+U@l|PZ!6iivg`-Hdy&(Yu_FgMzlidOuCl4DUAzc9A_G5ST z7zR(+wmZPx!iE0?cX(7;cIkg$P{0p%brr!6tqoeiSW)9K;NIUU+;hmgn(&+{Nrb|P z?}XUQI;Crrs5;RP00VG~j}q+xA~+zv4RP?N5!Zppz1{EchJHfUEv&j+!gvHz*j|*C z4eO+XQi!!!&N-+tAj_x1LkVVJXRXQcACPT+pF$}ysBod0zHt->6}V_^$W%kiqz03A z12rl1r5vqykNdN`x8the!Z+rHGd=gUSg}nqcd_WAS6)%o=EgAD7O--`10v4 zZ-uw_$!tB}(*QiFkH}QUS`Zdz57`T5?&fZ%ga-FjQ|Pd#G|bmQozdC4UBtHgSy{b| z!R&?$J|Z%&|IdOVTBJqT0p@g>K4DKL9gKrhWHx3I_f+T3! zy%!7#!q|~T7D2mmK1JzP9yOqscFM_lw|=4`J*m{Fe&_BV`oW>o ze-!E#o@*K6hoji;-+#%N0=c6SiXafHUhB7?lg2oq4J> zYy914>5?TQ#*GWWV0sr%x-jz>y1aiUY!p&$i|sW%xQvMlMU zmYNM%UL$LeNjGA|W{1({TV24{MK3O9+&Fz+A>~JR#wDIJ))GnVL|nr_Iy+E>kM!(r z91h=C=24Talup3?p9|8loNS9fufg1_0xr-E+X+ch+j9sz45$qfHa33pxIv9{mSXaj5_%KLEDp}KhFvrQc7z0-nT zwdPN_vZPr3gN8WE8b&vaaSOAzwtbB{1EB6jMPVXY ztSt*L*Op0*J^ZT{ih5LBDV5a~{?)x%bFLf^lrWpT1GEt_)98%4Ai=xjen{aF>TK@3$xncpIPFF@$@{!(-x?>P(X3;0zOopzsY zrQBYi9W?XF2<jUo4XO?}P5ckP;Vr!=qtl zT;7YalNw+E7v)Z5q?jsEJzqf=q9@GdF|#{P9t%G=*m`ge=YD&OB&g&F8IWFxv8XTr zoH^>L|rBnAf^N(|dilC&h!|e9cy|>S^;B)huJdf@IJ^85(#W-x7&qZ?I~W%kYt(NsYuJL88J*Uk9iu){GSX+GcU8 zR!Wah(~m=B8xhakDs){&NhrF$g%Zh57bI4($k>*3{ASvzOve5uv8^;|Xp@a(7w@tf z2QzLlYSOuLCslDyGA8g(%;g4@R`(jcdhF=aPbGxmRdCrQw@2#lRK40nZ~JZ#ACb9n zO7eGx=$&J)h+&u=&|5P>)7lxxx}J{-FEuasqRD>#NZM&*5)|Y?efD>#Fjp>va{0y(9y9x4k1TSXB;JxqC#U3 zWH)7&tZ{5zu;2>BNy&{wkK{1V!9jeD0-wKu{wr3oZf-XI)|&zXt}#}6lzo^NgRs)0 z#+>5!poAc^9~(6i8ozRJ0T;5vt8qazv0xt)G!m;&`;iqCiZ#=jTPp@%am+cfah7}e zZ(Na(Rf6v8El>AEDqu~8yjL?Q2@vT_gju5oHYr++8=BJTTAg<7Hh^7p~ zJVZEl1uC<499!|LxIaglf9%TkUev_j;;ao81!$O0Zo zpFVv$KNPZy-0*hx5!~zoTN^%CgD)j>0Qc-|C&$H_T;6g9QzYh?I|i9>z&wPo$rx;4 zaJ@J*pIfq4cVw8c-;oQ0x{^#;rr@tcW0Bf7&JXJO3IqpDoMnpax8#~ag$`B z;H<1!LaJH_5U4D6TIDCfjZ({&yik0a5^Cs5+p)t$(4dTy^{NM3Uv}QKArf*+Tk%s) zT~d5GH2t9TuoA(i)7H45GSsq3%~|Air8l=)*il5Y1)p-rR3E?iJ>1PmQD)KS$h-mF z5D_5bp=`1-%q*V4jzRfh9PnM~E_$SDu9bIZRH^j;5^JgfdH6Ht%DoT*w{cw`Zvkc#^h#1;ExJ5}X zGiJQ?TIhw&01#B92DSx=c}Xr1s~~KUF1X&>ff7kjmY<|~& zIp%4o^f!AFJ;?HzdC;)E3P&?VU_21*~chiDd+RS8t{0DIzwX^dDB%nCm`aQ;yS8PjNL5`5 z!Kq}$8H7B%?BF_%umG=(PfUnAxKDOE3si==@2%=cN6a> z9x|i9&Olkg0{7;tI|e$~O~d*dA%~$mId*_E^BpRfH3dboG_!u8NdF?78coe1U8cTP zg{nk%#*=@L4TRV$(Qnkh^0|GuP?I1ZKzxxD7hd#+Kxup`)8?jIkd2aD3`@=>_dqx# zB3`;(J8ECjJfIPpqGtQ8=I?eb$9yR(NO!saADLM1z2KVWP0AY7p)GH$@C86yUD8JD z+f}Y;;{X1&{R7A}$#M>W*WlCOKqkTnxz~(@kN_(ld@1LwPd-8vX#ub{0>oMWbEuf<#4z+>_+oJkK>a`sJ3H>lQf_LoN1%&AyEp9^4uGor2`GWLyOP%*6 z{GS10jg-s=GY5C}9KzB!LfZO!s>7-2t)`Sm(-)_nFT(bjUi53qZb1E4wW*v(J4}t-E4&nRj-ON7c!C2 zyS^r;qEWTcNQ`aY;T30}0;}Xye?UP>kKBhnx??|G`$k&YNQk6s(nSalK*V5qw&3elw%afRu90JubY$;b##pwUypp9~ zD@eH!*|=8yi^m+;pQkL#7&a!&@5jn_J{@Ln*B_SC*ADxxORYpUqV<3bIJIkYP< zFe%-9v6D=nKGyYDLg7JpHirGWel)p#T*Ip#EZlkDwBoJ)k%Z(mH+Ik+R>Ev1F@5)a zu(_`vb|+qze#F7xi3GaVnCc2$cTo2FUv`dRW69Vn5+#fSCE~#HAbTQql@W;f1u&gC z`S|26IAcLwY1E|2f5t`k6PfR7Gm1zy)Jw3=wLz*Via@0wRxW)iQ(59&On$ zzarwKYopTH&69?l?F%E|&IN;y+vJbbf8OdJWKX{*3ej-C^2W-wJl2y%EL!3C=KWY& zjp-#BL$-H6%6?3=qlgXOp6N&)mQt$gQ?kWjLILyY@8M}rFf&0CGcgQ5KBdpRuupAOz=LuQdrNL+ zx83hrWKL&;#!#ly>h_dLlh93Kz+_BevM_T{ktR-%m#BT4iAaRjY$OP4q(`sZ*<))E z){mbr)hl{+;l?;R9y6>{=uWOpgTMXt^Vvk>2Zvmcje-sZPV@0e7`Nb+CVdY1LXsJa zIycMFpnWkCaY104K{Y@!Xbrl)h$Jiv1-u!`$kM+`7cW0so3cZ;(6E89LyKn(UYF@ufAlYsiAR=qZ8?xc#$Vm zEF?0<40}}#8NchU0;Eo&+TdG465o(xkmJcQ zvBm47X=J4J1CTj@qbEMQtAlDXCs%|2o;&xb`b*y_P0-RRhbD@h!Z6rH^c&scs~xns?PRqN z)7@LOdF@fL>`qgQ=NmC9N|UojYxBb71M~BFXzid~_*|^Ak zO6BC|>o5IUu)bQ(s_&_a=!R<9?>04ZPYExgcC-{k6-zT6)xMyGVmrqyesfPVhITOa zBqp5RN}nYFabY4M*um9IXy3g3nsSB*GxxXOk{s1x_mRVg50Is!j0iynChSyLqgpH) z0~mN^7mm)=?j3M5^sJca`V-jC%fYT@w@*#({IC;f1D^>&CUf2(^UsHBEaR9`WYM8W zw!SwsY9cpRHnqi$kxEt83z3U^@yLFQ5L2(7RwKd?{bs6nWT+N#fOa58|BZ0ox6m{( zDk?SdSL+!w_HhcsL>o@D;~daq%qnE!f|4mV{7~y4pX(CRdWi zij{J?$Oh+vdx2!sihmr6*peC0v!STg%;4~{y4VF;MaU{_Bwo3dR<2x}E{Fat-mxv~GCKfoAx=t%C z+!)E)HwTh>V1Z*Daa~f9pKzC`F(-~VRIH|gd*#ZEUt3wmGUMdRVhz0ckY@Y#n;2Pn z?aU$_^8O^@>dwE0XMRbjIf9vsMf@H_(@yK++S_TzDi z#|10Re-pz(xS=S879swRig6U#05ScCsA89{*GNEO(D~pW@S|aGPoS^5Roo=1QQ| zYOG3;vu~$Mn@W+&=MLz-pS=X{>1qI@(f^>PCE^AviomX!9jIa7f`?WxV8BLym$d{lkX?|VZ6RAZ0_>Fh#zCG zUZ3N&lQ1WdbFNHMx>JEOndCPWd|H~>Y{y_InFq-)(ny5Lgs9EstAQ$<9kIajcuF5> z-#(wHF&%rHDCjKpFU_s^k`XdQ)=ktl?YQmRx2vbio&~tVBK92u5eWA~4~)O<9ri{H zM%5cOY}lo9=b>yAH>WxzQE{{-D(pyEW{3jou4b$8Y63bc^0NO43n0o+pig>(l{g7v znR%0$?=HU32rA~4Jd=}uV@DU4tz*K?A$np4%d6OGNFwbzZQ!Qxmj*p~DE`I*daLuo zg*Ey<_Y24TIU9T;spWCYtdQc;-q~JQ+VlV8>rKFV&cFWu>}Kq1QnIyJGm^5TELoyt z8*9p%Buj-BDSI`RRJKUjjj<$6h|1EAgpf5_6hc}kMg1P~8vRN7>&(S@Uar+3Ur5zb%VsJm*Ol_cAG>_m6$qKC^K22xI=;oj`AS+v>K@vN5K%-- zhhcdEGF;XM0&6pICu+5XcaQq$_oHIR5GES@;Z0LiDy_mt{_j3MM8W{kX_=<5+64dD z-Mz)yKbX}p6B#cdVGH}E4-|_fl#=?}C)TN5d&P$*eNPSQ@=4kp$_8%-dD8RHW!kXhK9x*^vuFI zV`0#^e(d1ARcuhVGxI~S-d~m56k>@A2ag6RusY0~xEh+zeyf+cy4QIO!w{Mbe)LO6 zU9{*(>rfB$GVGgG$wj7IF$t#HgtST5+F4wkVfo|7w{X*r#}yteWUp29AEtMS`Dbi` z(2n}54<)FNw;UdIv^BVf0$LIRTeY9*epj&%6DCY9-A=4mv*Pm3M;1SP z^28h00~It9{%(|C#*~|{t_?)0$*5N21;~Hg@kePQK0l+;fK~Hc*jdG8PiD@uhkQZn z1Jyh%CBizJM?_9(J8l*cLafoRdCAuj^U$JZHNYzW6*=c$zi#=dZfI2%lGmSyobj>7 zLItZ*`{3f4ft&j_v$8g&DV+V{_h0Rs77rhOXmDyg?_>3s-(;u%M#tV?NtZI_;h!yj zRE0a|S(_hRqxv~iW9YuvUBCS~w0W&IDs|h`Y7w+F|Al(Q*=-Hj9fUS^@>jGbT#ixuC4L3pqbI9E59_g^*6uyWk%0&1qB&7F(ygpOYi<@ zQWzhpebmL!P}5l?{GlbbE-jnsD3<7d{zX zyb;C{Nd(7!nZoiwAv^9-(7Hp1-YN&ov@kd`@*0p_lKA+k{_I16zid$28sFtqq3a#j zD`HKF{%h8V`_w61eX-uLKC>I*u%>2ww(-#Ax`DDGI0fe(3qlW-jvW)ZH8ey#De&no z?_g0rNcVG*+Ew|=Whc$5VvWyr@g#haZ=&(Cw0ycy8Iy&u_7K2F6j|s?lXh)0dC9WX4RQ*-HTP+|> z8gLPz(6q+UNel4OaKr2K%XtK>J;H47cOlBbC4P;yg&tdLvF)d}^;E%nG#ADwq0n_h|O*`<~}*kAM2fwa(qW$ zjVHsMd@l@RP>1Y2nPr*sndfH8A+6YY#&aPX&+dTKQwCOOPRlGQvx|2;Nl*78p6@k__uQKqdM9x%m)xDKCqP;A!p`>%4}g?7E@Ac64)mB! zSGf&Ft*d)+dM{ngLLEX-53e8GQ8nl!yKLP(#f6;qx%6q!3I0~>tT5~m{?ja-MI^cT zZv%>kIDu2ex8ul>9{g=ViWm0ex>{Xc&&WO^?Jt=`_p?3I^8)Cvf(EVnb;IYp$ez6V zYb(BnsU#bMo(o2zr>8gOJyHq5Csl_(n6(5FGW!L!_Mc+@gxb!%mFLjg@f7$6lCIHe zs@q4QaJ}|u;6931kkQ)qkan7~8T&CVm}HO>b_rSbn z6y58`l!m*7lS}XKN8nKK9}^y}d&+R92PIk^p_&DE%9+T7q+o#pX0oA+qWwF#K&@Rn zZp#cL#M(L}r=M2=5s%7P zz|cH9g4X0ZG-7M#OxC3E!@a)PC^AoHIe)sLIj-IH{^x;0f=K(CleVGLvC)5i%h)4_ zNAZ>0q~m{a%g6w5!av4%YHTUXdJ!ew7~Pf3`jSEWHWq7R;&uEehLp6 zy?%Ojfk8oW2Cv*VUc)^JGc14211Q%8YJ4=!+NVF>9lSZ&8=D!eD0hecAlr5$fL~Ub ze@H&MpqDek6W%%96*Vh7ER$)|_8xPX&V>!i^yEz92-z2as>r_JBWDQF=}7RxwA_hc z4_BT0h{bHO*Nu)yXK9k=!$KZ^VC&o2n%dgvb#B;%*K)bo31C(Hl60JO3tBdBj@|J_ z-l4F^aC_NOsZ4flM)RKRGH>4S6B-#gC3zek^mgwRJ~XuXFGW zP8|KF#bM4YGi8c_WwB_LF}*<|PAR(C70*~gi)#V;SkJsDy-VD#63JKg_4#t^$PmU& z$@$~MAT6vJmpUW&vatg_>8crqUqgiGv-D3W1U}+}=Hs*s?5?IBf>kw2u~r$5 zvVxt7)ib@$8X9#Okthmm8XTR5A2BG+TB0TyXVfx_2eNHpkM%TFduQH-=O1Cm?3Et$ zMxKRZBp^xh6UJ|8(X|`jUm`4ae(;?SvD-P%3Wz2(GCeM?F0?7FfjYIt3rJR(QrTW1 zH2KUg{v*B6XfG4CuTgnJBO}Yo)LG_cW{1GP6vCsq30f;dkHV5xxCSyvCM_|fuC1fb zl=uZ09e^v;HH5iz<1nW(UF+189Dihq+@&J-oyA5pcHrhq$EWn*mJ)!(c-5c|e2-;; zsG_=P(as-4GZ7oE&6%Pylp)) zwRio4NZMNcS~xi~_Sgiy5{uW4dF}2z4pcn|B6De3R$S0a4ILLq}@O8?6N~wQEB`5{1UdBrP6n_ zhZ`oHTKUWndm?eCQ5@av+kL)pF-ts;fkFCWrxwwTB4##$SqnMJCjywA^KyINt-(^` z_FKf5cdfPmX~~vf`}Oa1^67zh6-o79-klr%l*>wCVjlRw@aU8t6EQ&m`q~-+gMv!b zscY8}2@94lXGCS8*x6G~e}yVPLhC8GiP=^Gg}sgc&6^{EAip_JO{Rw(yfh z8%YBl_Y83qC}OUGT?|J@BwdFuUR<3}Va0+0l&Roh_I15Wy`f6;kx^bFjG!(zZL(+m zBJ(Vf=R%pBgXCRWM$n{}D(Oowd4SB9ks0y92@Hhe#J@`ka^3MuqecO0lU)ig>qI63 z_6SFfQ@2-tB0NSDSPcS8=jZ3=ZZZis_z|YjtCyJ+Il4iOY>?BK;NzEUeru=jxq|AJ zECr4cUkKg)Gd(CA!jF=LDlC@JCgCjd+CoxYPDn?m~J&%If_nPY^`tg2jvMFzIkE~HR7OeBP1f0JR& zse}`L|9x|AZf?3!jJB4RsH?d>-D7G3V)y6a0?Ci5(D(_9yDTp!=h)<2@ODnlx}TGv z=-fe(e9bu`kx$CX2HXuAxRLkFYbhH83egQ|*6tavPmHzf!x>8otnJ?VNps~*B@I!> zZdP!?Y7XLyPhU=7t|_lfi>}>igWC92j(J>`X_r30ZcXpzR#gjuV@m4NW_IrTP`PtV zaM<+q)vi;LuP&wo|Ub{7>0t!uznpnuiW)v`n0Y;>D#KqT-Sm8V_7Iz=e)CQ zq(HbRe5>AD;$%U)pE%8Wt>)t3GT5eay4|8PP8GL^KFiM8(5+;Qnz4IXbNvI`xBfD4 z#>$m{A1j<{Vd1-NaTII1J==P>)w!J6n9D;ElhaW-=-8!8>4(D{`GsbSE5#KG$P%D_ z))|61<`bx?{>{^P?Rs^{%w(PfKDg5n0jwg=%FN7!=Gx4sp_0A6>fPWWSi;t?FDuJn zf>r6EFhRO-tZj8R?C0rmh7<782k#y55O97gW7qAS-i9w9K5Q+{CbZdJOF1QR{6x*z zinG_0jbzqEF@=5nc+ih_?eA}e9410t2O^aUNww@X`%1}f>vXnj$ej;|hNjLvfK*Wk zD;+x)lu!WS%+Wj?K6-S-nX~L$0EPiXR@h!_Y&clm5jDgMt_E0PNF7A(GkE@*Cd}uY zeF-_-@LTcjsgwu_c9tU>^yv|%#|sQ`N441LnuOmsDmv`2{ewz9{D97A4=@YOTVcP? z_8jGbLgN3dB(eRGK*gMAm|$Lj-_&^B9i(xwGX&V3aZ&fmJ_+|Ws_pf0lyI0B)$y~$;`}(CIK)~@ zp?X)R^Mr~X@;PQHPO<`+ol<-;4V4 zn_90eE5KQQ{jzVQ<63ujF(na?9j5ix`E>u%+Z3i>tNeY>C|gd3$W>3M3$2ms8h^1& z|ABCmnu2)nsfOwI?JHq)0-JN0u1t<{oW*?%D_oS!k6F4-gqPn$RUb2$jECGP0@yD$ zB58afZt{2D8DFZ2!W9N+m=;45V*&8VrtRj7$BX?1mcLp%xa1fcZ;dJ)uOA&b3rnrK z!#?jzALZ(jLn*T!n5N^&yaT?3BMeZz_*Q4ukP`0blwuq7E@KZChRfv zJ0wj)|Dp=!#i!>hA@WlM*Y5vzlz6$!MTU^ph)j7QZw)i&sinNB;|;&w%ir$nn~nD0 z|HJ}n5wPaS6)VnfOHS$l;w@IzUgZrsx~~IKQyf3;Quz-eZ;qS$oF9I|^y%GVEX>D` z7v7~tg%zwk*e90z_F1y;sSR8q9DyZpfgmKj+g5>cn9Yg+!{?uWZ2BC|QX!LF@sN*( z7__g0`RMSS*;xQg5GiQlow26V`R2Oju1)x1dNFP_AvQoF6)K$bsM6_7#QZ~!MtoUW z86zMSmF{^2tH}dt(Rc`VRcY3wN%F1*mHYPX8;Ucw`1f#DR7wkpi-vqQal(Y_rQtaa zypyyT8WD=P%bkfrleJ7!GF-ECM2S9LzJmTo&``5akGjv4FP0={F=3<2LFM2UDVzeZ zHvgGqb5t+QyOjQQ>eLM&b&Uw7P}ubo_AJTr$=%4OB*oM}UrDJGK5z^$n!KnqDvE)>n@J!n36_?fX6b5&o zE|6X!rVS`v(7KiW+=R7MN!!(J>wcS4d~;4*UB@AfYnovGk9zli_vq!ueb*b6E%2%D zT%kST@xUz(J<vnkmsyZ)O3dtai^XLrxAn#`Q-Y^Ji>x#anPfMjm8Lp0^ma^UDH+DwNrTaR>PW+3#BjjJ8$47cK!&=^iD=kKKDrS{^Ds6wo)t$YMV(n#gumwv?BedOhNVu!Uw-lH?pM06azOco`WN*Ex7|8YH+RE=yTp-Sve@? z79U%E%D#p1kDT*mwi73|P{e*D%|@C=OOSbQ*D~fah7iB*ZCo#aL%COcy!19dKj0zK zOM0`-&_qmUAD(dN*fA5w#!Jx$a&glqZl9%__Fdvv*qJzdrxG$ucR}}m;T4VvqD=Kl zuawobS82wYW*j{GX9H4e%nh^3*sLSR5)Sm(vA?3jmE2K5ZHe|ts1Vx<;3Zb(9He#B z=;FNhnlzV!OcU*L=kgF{KN#(!2(k^TRNu0BvuOg*;)R@j%}$&+u>z$5Jiw2~x4WU< zb#!z*PsWZ{UhOUxeSbKxr7X+~w91c9kGmdT|BWzuj(4_5hs?5g*BL%kR(dL}{4Sw? z5x?u1aHD8-(T0>bGB=Z)LT}~&kgdosH4Q5{rm7wl9}P^+Pff}C8P7U)WX?-$yCs&% ztDFUV;f)3z@4TCTGRKyON4bKS$zPJb^eDY2I|;O4)fIC3`8n&SdDQpvPEs`+KVGmp z@N>t@ZQTcnM+yGw)JU`d|JKM)@b|&`8Yws|-7Sj;x!{?Su2DA1T~;%@^Z~<-MSEBF0M)bA16iGPvOc zjg+UfYiw!d8hnJ#!)bOWqEC^mDUchM4wkwnL4OFm1F&C5Sd8K~f^xlFqMY0mi*1=4 z`^1#hmdP;$9I=W8Ve^(PyUZWt){4g{0c#L%6`7cP_}H9emwbc5w2qO^bYmJp4+Ouc zAv|=f$gX#PCFDHFm(3F)$GCvUM-}z8)Zs^mki1(@@BO=XnNC|;A*CFbz?lS>Sy*1` zP+l6Ckx`=c65cXNnlNG}Oq#R-$Hh^?-ew?ct8hVl`azoDqCvCSi(_|^U{X9nobHU= zyPd+vxA4oRJ3^2%Gcy}w2Mr}}P4i|BKgw5mz3RyX3%!P+@hThBEKzc$*aSQ*E-psN z-feP^q6Nr_C064%BO{$tKpY=lUF+x;?U4|gmM>lqCZwZHzl!$^b^}j?_FCG|JY&Fi z#-%ZCnKbeZ5V>X5f0bY**bJQ-rk;Br_S9bJ)f~k5 zwUV02bCpF9WD(h(s0D9azbG$;)oWiOte23UtN9WQ^yK;TLm0z*7$-kgW)V2KPJbrkJaSbN+hTx-gz7GbGFp) zcJk6xL>4MUf}S^R5jkbtxJIIiiC|ZI1OFT7oqe3S0le}EuYa*uy`N^07VRy*G#xe4 zrzu^RwnHX}NiAG2@!YdfpE~~;3T0J3gu>d*P%l!7Ys3zC5zcre{=MuK;W0F)qi>Oz z0+Q1Yx){JIk25iI=ACyqP;gDq>fKmUP=l^Sd9&|PuwC$tVWt!V86h6#KYS$_lnV2T zgE`D4cZwTu_S6}Nfrez#;sG^%9uN=^hU;?O1`TjhdbERs^=TUgIA>LBrKxs5!gRFm z9eo`BFZso>!MUKHeKQBSj%PPkX!`n?8Uh*W<@s zr!j)j5YsM)Y5sgqkEC$6!_Z=HaQ0rrB#q!^7Rx=@k729XYA;x{h$-)ZrV~HLJQ3%< zs`_DVS|<|t6v_chE~;i^Z|*Xv!Et>4@y}@Mpowf|og3c$cmpvsQOD=r{8dopRSD}5>LBHWv!iH*3#LW+DyIN|4sM8|$XgqAWTc`OBULBQy0w$6mzh=eJF zPm6XLFa9I3G0aVv3Tw0Cn=oSracJ*VRPRWpJwPvyPFFHxK79D#`rL3Oiz3h|2GD&D zg_cUv_98*qhC`RZ$u(=>O&r#Tl2e~8$PXeo)}6bpM?7>^^Lw>pP_S?ELx zpcOWp0IjLjKx=)tOs9;EGe(-+oyN}qU^w_8oVBfB8sU*|)o8~zizT+g)ev+n<;qk0 zYF`V^CAQw)w8P5BX&Dz`OCbFNfOd}C`#^Zwj7^O}$>EvjISs0=YJLqU@YN_QKr4WH zRtTiw?oRw_Uga7BP209@6S9AmvuQ_{cI|tS^y_LDPfh7z4{HJinuE)^YtQ;O4PUi7 zFN9P7)}niRd8F}r3x7~fQ zQW89^&RtC3ux*TITwS-v?~;)K2nkaxW{#Q8#V8aDJCXlf(oqtk8UIVh>}8kgxIBLQ zQOuj5QqI<2{u6VJDI5M^-DmPwBozdO=Iz`)X&?jlA3xp#$nG5>t8Fp|>E{v_lxP8% zv9x6B)RO5_LWEg{4OnYLhr@u;T>M7ZCdHK_^A-mUyGK1vrka+VKym^Oww@U!>@oB} zI3^sx5Sm)k;!7yiRMK2zpW*e4kN3cUsH5+iTwYXel=fl!jvZsI8YrcztLC-@^~-w%9A41qu*AQ%vPx((IGKwL1_|9PRQe7Y^#1%SvRB|ji@Hp-65=;! zka(01o$@NG$Hq;Xq+IqT_yC?Osvba}mvDX150Bg7+oWkzIxS*CgUy@QCBNaw-+!-6 zNG!~Qph%RFZr+Z)Euv3@Rta}DsnsUKEl=%N5AfZ$b7!Z??Ta`!tNa~Up{H9Vj&zD; zm2vO>{m!Sj5N;$UhM#MsSwQBzfSRXHbvIp0@{eJnsa5K-OG2mRkodb%5mL2VMJ6%c ze1@Yv;9(;&pytNb4{i`Vf_0+!X8+cXzdrLDW*8jgn;k`p3B{AYeCq*p?v%~`v}q#f z`~ThSlW8TPAL4+YvOssPqvHb(XKOi(|C62s7|?^j2Pa-czwa{6fZ?%Y@mk7GQZNTF zuGov89D3Th_5S2deau;vwQ=o4HD&M1oviD90(J~6p{yHO_? zH|rmDUA?s#Xl=@(+UvGjj4n)WxjnJ>c{Nqltvtajf~L`tw8YZ&}nQ@ zP;8mLWZAL=AQfun+7kx9AIzA(GXt^BwGk=v!tNZ{a_<~!JZ0U9ryt4EAI*T^u?0c) z%YJ;1upc*WE40jy?P;TP-TPhS+Ayu-@Op;R634Sx#w1tF1tOI!vVGosWA~mu?Jlnp zdpQ#%yeD{{+3)lkcvs1=qAGjMr~-YAsn~!H`u<8=Q*$oOA!Z!-Fj#0btj*cf7&z*? zB`=GpZtlGA??IHL`Vcose3DG?mmVJuJ3~9A>{;!rHxvOPd*$DFIw98#_8! zt8-^@IQ)0Ebp0|~i*hbAWd9|PMSgJE}?SBWTqR*u(eTUhqvtDNt& z;SospLAz=5;uAs(AYdd=ZYBA90dbNGpDTCteVYOgh$T}dfK$`NcqtZ6B*U>8Uk!N&ULLJ|4e+7*wkv!k>Ql zdmx43zlNh+_^v~yY}|)H;m|QuS+&Z9D=PG8qX~xx z=rk;EZfzC94!Gv4q&|!?t?DW)fA^t}^B?}>P_x33+1E8_R_CsGpyS+&Kfw21Uz_-s zmA@=7DZkxe;bYeDpYM?3qWiMP1P1BfCT~;l%4=*qk|ssEFfGq8NSuJVHD#n%5on?_TeqLB)oIWm zx4RxsqEV623TFvKBUL%=!A=`+83Z-)O#(%^JkP4z3#1-^d zTbAhXzX^r07sD3C?hUJ3)d1Ilupt;%>vJ~z%lV@O&W>4hXa=2=!cp_yz1Qb^_w3n- zYI);~svqNFXPmX^l;?efS|oCJ9(=~Ye)m^$&9J$ErR@362#gRrticy9T*&=U(PO_V zADG5`7IaMTP`s|)Psen44z4!B;)2#jfRtc9UOx~OOB>8IC5jBxC`}XV40%d(3`J_< z@erVKDKea?=O#i;y7uhXcDu&~Q)SJDZ!lB7Y6e>vUr$(bR zHoo)i41J)eZf9RQA1#$WGyk@IT#)UpbO<<9D5W!}PTj)_#EN!t<$#Wux86zoZNHNw z(Z743X=;x6q?(-S7{Z9dGaPHjI#t%s7#j%}0Hu+1*Gx;f=a&MZb;FstbDLVJR*h&= z3%Ty}PluL*a3P_pE`d-b01jvGg7;#%5M>`tEr}F3!hP;p+Jxi)#u5oaRL}mompn@T z5RskLVHEM#EUvB2osIo40yxuUY|-)vs`kze1kw;b6&vcN&*exkH^TIaDakMRYH`Tv z#rs>{96l?}I~G1iE_dqHq&Rzj?Vrcd#3cOsqci_$R`HIVGh0)}X|l3UDW{_q_v9A6RZ=X0Ns?tz5m{y&6HE8tCt`_g#~3jA#qT7P*Om z2)&L>tB<8dP~=jchKLV>xXnh}WvV;wpudnrs!6wR-wx>Bq}EVZ*T@mY$Cf_7==!F@ zVR^B<+gtXDbr>ss-`*}^^i*;INr}CJNPzaKv9z-Oe%iPZ3p}B=|7jhuPYLt*QGUs4 z2KBjr8bwXa%2~J(K=bipOfb;uV$p_&FJSVTA}TazE%m(z zx9Woj55zozp$S~|HQT$U6_hQg3-B5Q#_raw+fatbL>;$9@DxEz7zwVG?3;zl2z{si zNnfWsk4q9ZP|S+z77hh-k|8`N>iJ3(Ub2n^8W{4`dE;Ai{!|)HIWI8T3=>L9S61^$ zEpU|3R!K}qaC@{wfB!7{LvfslfRr!93`7$){|l)mpK!?e0+=Kjfu=QtQs)6}hnp1O zu7u931!XHHe^%~ zi70L+jBikHts_185e2T45gfN3E7^zIcQYtY*YmP<$(Hi*@=@lcWEJG<4}G~kv%>Z#>SmHcc$rob)UwQp^Li- zg%GuIyju-oO&UyWPt}w+d3hKqm_OL|V9sx^SzS%P<`KOu`Ryv+AYOwlq65szx}blc ziI@UXL}u$3@Gi3J+bTYskN?9k&5Fob4GoyS6C<8BOQ;t#k7u`?l7&Axl>@j~<7Qe| zG^+-1Ixx~gl+}?{Eji73f1n!G(=FoD}n4YR;tXAX;Yd1KMYdK{n)&9K<(`hs_OTYxngKGQu(*JlFu7mK|N1;glR@66(+4z#!P+MMlPs z!a@g>oVWJfx(eM!qFvOkl~q_QU+z}i0A1g8&AJ?ng{KQ}B2o6>67%hD70x6yQwM9U+DwGP$-zb)=Ydy?mrZ+LP>@KIK@VV< zy_)erY<2hnoyuPo>cGm>$fgX=aCF%~89_P1$;v!l!MrLdE6jZBATZ7!)a9GG=XY8V z_iLj=#a~o~{RsQN3D}V)!>E}P{xmKO0gRj5m2Eyr2cFvNQN2z`{+0$;pIseH%M)*kkb+66{NMiD)9_W7lWJ30sm+xS#e2pR|! zIrkqtsExH6p_4NVPCim7Pm#WEAg&Oi#?}WdTligN*ra)vLrP}j)YD&fSI8*93KGJP z)taWe6ou2YX2A)UdQdJ2J*-pgAY4=t!NN%|F~O&HuM;9sfFT zc2)H^Ab-#8#77<1+!(_6;U)IJ8o1qY8KCF$w72V+y4{p|-7lNFbfwa~jqVN?7Va}1 zZIaW!uF|wYiD3rar?96yOGX+=Wz1NummtpMNyS=Mm8Tn&|8}Y$#4QOx_aAu29Wb^#7btcplNx|U!8Yur2 zoXrU1fOQ16J3bMxH;FhV9XHi2o$zeNSVws85NM1evRfgEn!Mj&8nI5U zFW_y6xJa%j;q1lPM#LeHo!v<+laVD!Vl@uHLZgO!3&>yiQn|)8s@*s{{M@4`=1W>hbNa ziga7OCs$VWMC>Hp>I)PSwE9genlx_w8ja=%@Bq0EY1KP->U6!kN)Ur}ZI+887cP7= z&Qp2Xnd$3sQ17hw&?z-L<`)Hrr&+VuWtT>rnsRFw!L$G2mVYq#zufZ3xwf`rbelG5 z@}{6p^dvT{3!*>)_RbC+n(;Bd^waflCoMjAs07QW;pRilGCh0j*rvR9w&M{{vr=;y zu;U7{DX%5ty`y}pIj+og8eiz!pe0Sj?I|M}oE3sW)6jV2xngSs?IO_MwIem~zqGp9 zMXtLINJVc*l5Ymydf&=-dnb*9EpUWG3VjDJMYlIGU<~4mN0*BaR^EwCH3z4dkrfis z#(!CL>>0-giccv}>d$t!DgOg#CTf{4C(DM&^~0=yc#;Pl*3cG^KliJ(^(%ub*NSQi z>YKuQt()rHU$J(rn|2yj=e#vFUwN5-{r>Nn)oHGpvo+c@rd{yghtH*NC^u@9a3*Tk zeyiR|qu1XVgAnVaK_=8p`abp zHZXYKzUzGKTNQebZQE9k+as$4s)4u#e55rg^Q;W1N_=BpLcb~dMUWcHGt4SNLBJA9 zoBR$6Y;1A;EXa9nW&a>7LZfdvt#mAhctP5fEe};He0o0jK5MAi&puBjm_?2$#)6tn zj8T?uWdsACjkI5O3X)#Evexw| zT-7Z4Ab?b|OL^%&8_bAi={9ZFtkiqdBAz^@;yR;sybv)ZnELhWIVu6BU#cUx>{y19 zM$-`xQ!-&pm(qy3j}J(Ozj`;sxX-E0TehG!dAPa4lU^c3?YT0bHfKtcer<;Mr;^$)S*0xiCuhP=wXzu12fUqNUs-U zq@2=>rzKF4?1sIT^0JBL4ZPvFnFvByWq!L7V58Zs8_|jJxO4Z&3k;J9w7=`a3`^a+d)O(9cf`F zG;)&J$@7dfBG8Kx2X(2&o9);|Xm7xWNiv{vo|qkXKP1#Ac}a0nK4O|1nNLpBHpuQ* z$3Mx3ha*b|#%iHR&4EUNt9hhZaul-(9IB1Dv@H*h9`%smI2vH=9r%k_;M_=d_kHH; zwVwztoh9o4%Lc}?*&4`L8W@!*usB;gD{|Cqub4$*VE6piY0PWjH4FN-hQ%(Nzc>gQL`?a`l<{ym@> zy&hU@1|w=6S2yd%Jq#a$>WVH3!8yjTrnIyVM24Me}p53Jw{&43W4?4 ztvAJeSkSUqMnl(X3HQ-})gRiSAolG1+i&;YsJswDB%(eMR7_ujy(!%$XWQL0DXk64@h!ESnLxoR3LV;#2*QPTjeBlTF{<)JnUSsJDhQ>|a=txG~ZmhKenw zhMf#&eV@w!DAD7*nG42$J*oW4sn$^7y&Lg>}@KN63l}9a>kgbn#;rrwY~l{P{cbuFfZpn=yliBU9F_`LMu5V+`v1 zV&AT)VT`5p2?I##5uciIxkdOeg0%9!ixZ9z@=l+=Jr`%5?t&o$G)qziXQ6 z42ME}O^UYbNjh$Y0}vbo=p|cekY5}noSa^6*F$J-651EDT-&kBfs`#=Pe;PSCQd># zUH0~^&PwpvKRyyLnyzn;`XV={#Kk>z9+N0)d~v}tJ7;&{<_7l zyT)d>W&M`&1Bfd2m3X)vz70k1A2kVr=^{0*&*3B*V%MXv z2{lBbC#^0$31?EpJms@7gjxU1laaaOLUXghWOA#vbD=i0= z{?VqV$gj2^@7=bM$i<l=eS<&|pbUtimVr{~G0?)D2 z#hmf-N~|8Uw1A7r0nts@&B$nh>lTW$Bh^X*{R(Im78 zK7FBD@6U_+tvd2lNcV{nDd%!`ljS^a#A)X&{T@Oi^xL%yqqi=j3Gpy9H#dEcif~Ts zpdvdErfT*EsTFV@x{7>kf{9fUO$PR~ia6ABlTInzz{PJ}v)GXNjNep^_70`{JX6pW z^G?{F{KZ%jwZq3dX)BY!Ar3!R1bh+CH!)1eHLd}SwEwsUxJtNPHPZVJu$0`tvS94l zsUxcQDs3?$8Gm3SMe^@%T755PNgrHA`S4JtPH`3~|TtCh2hNX4Li8TRV zzSLA~epk{r7n6Ub%}FVL*CNcTTf6-1+J!4@Tb7n<-y5V`&uZ+)AFY*Ij&Kh%9KKzX zy83&c<-dx9?A3%9$~(Nr)L(dT{yndl+&-qD`Q@$C&#HOqs=!WtHpE zI=lw}klCk0V_GiZK>-W8K&hQQ$l&uCpI5x1_mOcZc(m|;hw zarGy0Be6vj8D`4J3AyLF_=uFK);sfONN0xLd0d_jc{uP`W%`^6y;0Du5nfG)9iJ4r z(xZzKiWUA(Kk@Tfem%NSs1 z9K`qZz5jhR!+3I)SM{9}{b}vROOb0UF6NMtBq9##2EvMILWzKz@L_@{&j}j_oe^D% zWN#`-1&!-(jJ*gZ#YcUkw{KPI0Rya^#+{LvRn9(Rkxod6;Q>sKktxpAYv17_lkR+^ zWYeQu!lq;Kf>hIF#*9F*J-$x8%*>AUjkb4IDeB z0VyYwt&c3mTq@s=lo=XCI*jpB&Mr!YQ@$U$DVWhEP?CkutQDDMioJuRh z*sL>#?TFK6#$tYOX^3eXqgPi+s0qat&G3XC)|CU-*36XU0sKOC8Pug5)(-~Rt5TN( zqBLN1QVg(DzpoKA*g%#wPbE!~9VH=_j!(!+6|aftM~m)Vw)(9_N`uguefH(&r&@>4 z_-{{ax$$(wztYTPle3e$47u6yU5mvK8baX{QN*0Y7g@!US%C`~$a7)J1+*|W6GM-v3eD;A1* z!G%a4D~7Nw3M~hdB~7W$oevt6QQPIql@n=xbx#yFg?s8>xYhTS-WpM;Ozqa%e%&_- z%;^vl`pKoqCo@(Kpp2S-{qLWT-kZDi_H8kly_DN2m6|TB1CZ1| zj1BsXH8s7#C5MVO=X`JUQ;3$cp3DnFjq(dv=eccWR$2(0;CK5MujJq?j~MXbs*JpR zQ|I{}WXVm{=~p?PP{`IWM(PlpA+91cl+J_pc57IhP4YTsU`&ZT6V~!Ch0KZmBIj&g zp%xAyx){Ii()Jd-7n!NTv(f2F9}#|I_sQkZA;!W)qyLXEsg!%va#XAgjT;$W&D~^s zQ5a#_+$hX~t~Ym1I{?H7IZq>k=ge4htChEHeJGHMJifNrI+7K^V4L)XvWB^Vf^)0P zEhCmc6kuDrGlUjHxC4)U;=_U%xi8em22dJumk%EhyaxFR^`Cz*YNhgz`zrq}tJ5qt zEf4@nS_ODQKmB@Z;axRnd*&Aq5Sv|C)53;Q;XwVX2@N1%amI_*eE7QXAFP-zxjI~~ zxw`t@h}9HR+q<+E3a*xAWY%Yu6-w|&5*W7-7L`sdP5rXL$JWRtK-%TX?}hptF0Sb6 zFTW+1FE;vD?AapPZ;rL*R5r3>mDd?jV#$k<$w13{dd{9drb+3>Td~Uf8p6_Gm0k$;p|F+1ay-b;wPnY(B`tQ1 zP51D&LF2|wZjavfA!Oc*!EoflTE2jt3MQ>*W#qA^gcT~2758R$6IOeTgp<_q9L|O9Q&A(=tT%4p}HNr zzle^PFs#{o0;47~(*lpl?p2iwXkl_ob@d%0K?83}q%b-V(&?3OUkN)NFKdr)^=OyK zyc{up{EpbL^XYv9%_Li%PMRGL{0z|uLeO15_PE<=PhW%m8GnCX)uh>wgVZ1fHj(Nw z-tPUhmy~ce)~o@p^ilKk+P`n#iUApSxBOY6%QVQqz3btVd%-rh?ezx-^-EUJ<6WcK z-pq5ibn7rU)#9FIm+BcKIoVV}yq$2`IzwrFQoz%Xt7R2MdG~h%xKIQlV~EfX4RJwC zMN_c|xls(4)jmyt)K%QsTO>S7gFl&Q1Od~qFkf+)YbJ*#|1p474&?NIf1zB z>^8ite7~wS8*15cFx3{hYgZql`;FjnLdK%ny7W^_wc$oK6+QTu*o^tlVrlT|BuDOk zknE*cE6KZu{}Ocd`H4+o-hVh-5RzZ7)qQu}19C=VN?rH;CdN$IeO zLCqe=*S~3dF*Ea?hI$k8BhT%34{jel%4}UT|Nb7LFVh+N<#D6=8TldCJM{_HKH@rW z#e*Fc;&3dY^s8_5%M7BApN~|3h}aY@eYVpO7#Q z5C(s!%aVQtLEyS^)i(?7)k`@mlxeX!(62r5uOwi%$73>3xADMrh8ua|`td6_9wYy} zkIcvtY6uC-a**>+HvYBmEvM#(gcb@fk$IG5=ab{bt%UYDXi$3!tAecrgrIw(NS%7? z??78knT803Y* zg_WV}c$aDKKYonIQYrhq?eeKp+hQA?eLsst|B2LhT*z>Wo7nbbr$Vu~(qIlMVR5V-ZIEE~a(aNvt7iUC3Zb6COQ zZn)vMY$NCmPjHdJGvQ!SL7^6niGQJ4Ri6x=Mm1~x;oX zt|M9&)s-X^y{Er-Yb~-L_OPwE&y@8Kn5(g@RhY_nNA2dWW0$5GL67&wJ@PGQX?mmF^77!@GybQ%=9m39&(+)J1lHgk-fnz5v}5ap`tw{KY)Om_ zkS***zwS?LKJ_tKc5IT8Pv;m+PUbrN*>U)fDFyB!44pQ<#YSc3CONSajA# zZDyANE8jL3BOKGoPZ4gDyEli*s`JCyX!(i(m|r}u;lQ&ujpXo31Qw$Tg8erC7eVel z@P#J58@+WrvX1!rJNZxIOHhUZO7^NMtL2AbB|vur`kHcvEFD=yNbmUVyV#;|X*fVh z6q^MZbV=4DLE_dM6dS_DDGj#X9`^)&G@lfAf-2Tx)5h%7M_d@%HP*S-6+|-$!iM+ zlpE5cc5pT0zIhYet3yWqHP+F;$cXA;UeeYoN~?~`*Ya=X))n`@r%479Y=y{=?bVCa z@yhgfAP|mwnf-RgnF|*lWCppLn);Y6_Etlrl0{GEKa=}ftnmS?(oe(6W#gwAf&Yyp zpW}-tE82Kici1>9;_B+)-i!TbTtdRFU1mn?!@%IOmX?R@Ove!p%&r!cUl>&9nd8+5 zckZ(PBv`qa5V0vr?th~n>ns7<=~-MG@2+0Q`<@SfDlHgYZ_Wd)>%Zyx?8!9!+jsojIX5fI2X{K@J>m5EZciGR9`*ot<~qG!O9!!PZCh2=vt9b%8Qj*?ck9+z8nnU`gICvWDvJC5;KFXm+(xyv z)W78G+l;;OA00i)hMi%tEkwg*OTD8J=JNwXNcD~YU6m-nGOWN3PjXH7+QIW<)lW6i zc`VQ+SXU+;pjFzPd(YXp&VBdbIK1uFOXyz&o5DBfcAhoM3D9YA=L!3d&5FIoNFAJD zwZ1*Y z;kW0%5=OyyE!gm5cn@2Fh-L(LWs{_Eyn;k^Wk{m?#2GWXtB}suRa0{${KV&yDaW4I zbSVqif~Y5LBFEFrD;Bk4-psPl%DM3-5jqL-jwPJ`6}YPmm1S4QhH5sfEnqQTbzVz> zI6+gc2MyHu8=M$ppm9Y}EM&GoLP^^9M~)4Ki(FF}VGg!$#A$}plJ$&(@b+749+b&tvF2V(HS} zoJ>iM5{@FPHJVDeA1}6D^Hf5EumMiRTz=d4m#*7IAoIj7Vjhb>n&P6mjz{1lTgJGn zV1?uQ&PbTvbR38^-^R=_v9fWS{OO%wO4|3RVtcQdQE#-m5QNwE*J zHb2GKh$HG_U^U^RVWyoXg0}cs-XZz>eSY1rPIZXQ-j~KL?pscMI)Bt338_o=zd?JF zb=22LkU6Dc&Ev|sIfblGyRbLqAWT4}(QJ013TcN(GfYf2-OgR>2Y^cPBLPHI&|&D{ zcL=KzIJRVeRel}^6OSm#2~cbsnXHZ;Iy6)USFu3@#UsF}mf@Nq3iCczxvOrc+JaQk zkNh9pILvQIBo+vbc_S5)H_j6ffMCV~7-J~3NL(NceUIL%o;`cUT2~e+U&;kHP^=R6K%ITuxY!f}OU#fpBWM}X z`3AO3v4+$RReASARu}LJ`b_TPCaesIB5L{vR(1|!zzgqwNSR?ym?rt3zjwq(mVHGm z=7p(;rm^pl)HXHs{93Xp!eOm6sC?kBBxu5%5>QSMaBh7?zFvTz8arG>pmO@vE!IwS zbkr4ZmDjIdU%FC-2T*dsD84~{dCw$)c)Owdq-rb$M^BdWK~ zz|jp2uU_=AJ%=s6gG(yu-P;v}>lTrZiVUX`X7eVr-hfE^VX8!M50eag_zBoWsjHB(B-%PzaViNp$vyEy7b(E z=7&Zs)#dJB;6LUg=V$Flaxigvn07{(6E71F{!byzPdT`I>%S|Xw`Vt(ojPwgeDj+Y zsr$z^vv~G7r%LCgv=jN_UYEZ)DDV>$ck>P+%j{2zLoQY3uWyPy1E@-neRrGa8 z;urN>H?Rkfn^6uKm+}&BciBYT_84uAvB`OCVj2_AeERe_N!Jt825gRulz(*I3x`$Q z#sE|l?>RR@&8fxqV~E3xw6w1C+ou|&b?Ml4Yy9G^o_7ocF-GRLmxT|zasRO4hcQ+R z=)T>F+9@v?X-att!Ns}LC1RbftEybt~%mL|t? zaTlThk-%X!8av7I{7^-BTX9iQ*}gZMQI?6TSaOB)>lxueK;wm9x<@}okB;lndUgy0 z?Pn1>g8zY`wr`!YGs5e$WBKhsm8D)7(nba|pKYg-UM3_aijR+;fx%$=$)~0~iE`}+ z&G5%~<5=q1aBj;S*9g4q0C(N zwKUsar}9#uAynd~*tTH|td~x@)|lYu@CI4BE;KU)U_ks8RKqrOkYjW}&-R z2nC<9bT_Pgq3!C+!oM0_Z_-5cn-D#7X0il>{~DlnpusxIJ*#uD^`xtfn>EYE<(&&E zL=g@|L2V5fuB++P1d0rF>1taBmMyV%2VX8~$9|#4`F7W?xa15ok=ug2;lgFy{^{oRrzp*-zReGz8$TY9>4)5!m!B&HYt zHRMXiZC={+ypkC5(yaLV5j#z#rR#phbu4vREu(w&%dU^^{%V5_ zwBpcLvT1(#R{v^Aozadg`!*561Wc>>1@J4Dfu6fmC(r{V5PH4yo!ySWm*r5?xLy;T{2c zepg)zyTI0E1S?O}0H%~7MT{D_XXD+dGU#@BOFtGh)-&c6%Y=WCb(zQ%TFiz`nush8 zVy@pdGT(CDyXHiiot--PA*26-F`=;4(ipm84aGU4%IKVthR^YTWnH85VF&2aM4s-aP^=im4 zAUZ*t*!%AN)QW$>x#n4IOwslZvM-;%1&OM5!xYyvUrlP&=NFArZVB5=UtjA~wHjRRx}PRAi^OiFt~t zEMJ0~-I(_PTT)D;v)XH-PQ(dpqarb##ydG;&hnf8&<=&$KmKI)s`Dl$8tv88On67= zyF2W#Ha~ga_B^6fiGrgt|6?tdAUU(|$9slt4O@!Ukylvau64yxCvpRpWk`DJ;h9y*q>QPtzZkhT&+`}9< zqp|g%=(kWUS(JS)PxaD+U%6H;t-S9G z(%>5|NGMzVL+<0lE4j0(`t~cvZNb7?4kMcdO~Nv!8$*+qmkVO>rJO_GH1Q|^Ot^4w zIK5#_xXEwP-Ai+FG?2$(W6>GN7uiw=kEx5EciUL0_hU_>&Fq2`LJiOQhj|v-@x79Sr^J^l8l|+ECdZpPA8h{*iCVh-K02_|N1r zuIiI}9*35o!Or6+O+Nid1ggqwzUbnOq|eIdJ-a74iL~Sk$BqiVX{(oYwIC)|qES9& zXK&*W9CG%R`%!^9-!CowU12cHZ8p?vUQ zl7k_@1>9&!3SI29(-|aeBWJ}h_u|EiZVw~>WUIw~aY~WVQY|bC0F>!P)Ok#s@o(tV zty_|L_vqM?&09_yPfPtBVJaQVXb8u3=jJww0_lYox(QgU=*KU9Tg^Q0tccHglhZ-r z8zy8z|D7j!C;{7)dren5J5@LjGx#eoMoRI5f@PGie?VMJ_5&ZKS1WvK9R3g? zXBUnnPg$lDk~>+IQygN`KV8GZouyV6I{dN6A2j)?feT9fNdV@9_`?%K*!h_ndK|^} z1^&n6(EEZ26=^ZsGL{F$eu+>5h&=D{!|e#1nUKLmF z{`>#l57JE=`ABJwa^1!j&q8KT`QfQN)pz{Us;C(s&!--4bY}Ojot}*Lne)ncd1{|s zJ0_fN{{G~q2X)$Jp8e4Gc8B$ZvR}SGKOsY-U{<%I2%g{}F^{$VDX4i>`PBd>{NT0Y zsnT?8nV15$U4l{tXc0g0!)bYIc5|^GLG%1z`byT&qQdd(Iy&myTnUI3{r+OTFYppT zJ)~JA0A;T!X|9td`5*K9e}tV2T+jCx|1&m>TxOWun=*-A-R?)Ns@%3l;oCfQtJOa-^~2>|34o89(!!R-{#i$^Lf9|Ij`GE z=6&0Qq?ALNnah zYwIZX1MoCC)uD(HNC?N9teUf5eaoFlJVdI&!PLV+)%Nm>E$c>6v<`~-B3vl3XaYA-MHXso1IPe zV0O_QF+#kNVge!>{C>^4 zM1OwZ^HkY*i=eKR%^syObgW=Lo`n9#*{&0XZ+VKzd?||msuj{*ug)&{cc7d+?&L2$ zriG0=IB&D>gJc$6MH}oknKf#8jM*La#g_*H*DiihUw7zKi-YeIM%y`#S+CT{$+4Z% z=C6Lqt2St~@4L5#eg81(lXqg)#Tm!fzp4LtvMynC;}Z+|cl~4Lm?uvW z(XX;n^P7!hLs@}t1}qGZYKu@wM&KlgSkVly`%twr;mwK}gdvl4Z7nNmkK0SW887Ek zV$w`yy(GGxfh+zP5ua@V5dhY>U__DYZ_Kh=HfwNT`r&E4X|AEI={psz5UYe4mm(nT zMI5>%aUtr%!je&gR)ahX8#}YCwDb~hWc5S6h~M!0DYBUhZz)mWe`gMjD{%U|-?M^( zJgzqHDhoPvTp``u!}r&bg(N%(76}Bi+O~_%I}H{q7~so+rO>A1NMwScem@w^6_etN zDDC8P3>0X37j}t_i5c5wlXi{q(&%g<8aPJ>x|L0tHti}G5UV0(-I^j`yF2<<(pKah zXP6Yk2wFHKf32o?;u6ArkD5Ro0s+INhIQ)h-)Kyjkr`JmB8iRj!W>z&Qf(QWu;s?Q;uH-cxD!TY{gr_{oy^2eTIQKR@3qRN|v5_ z@8f~dVz7_+tw)488?^~k&^KVm6v>jUk_K&`672S|%M4kcZLL2uQg2k@iWE+6MA@B0hNaXefKm z4B{TVx-_@^)j++_RS&mor9g3;h8eX0%Y6*Ba!A3Grn{QO=1@u4vK!SY4M(A3gkorAII%al!`RH8M zNr!m-{WvF&60Zn6Xo_5q%@~C1@(K#o>!ma_* zs>R&DZ%nH>H(0NrFSFY?;YewsOT{RT*(>0eg0_IK>~fOC-7R)tHCKOc+*D_k{WAKg z?^!aOwaC4_4Xpn)^2z?D5o4%~rc~78^OJC(`@$=b{UjNuwK2{xi6j6O#Jf{DIa@_{ zoRim*^VJP@7G*S|rD@w1z6MhlA<;DT46mG_D#|d68)$IGHQZKAbI2-_=vxT=fBAMO zoHP$enq62gr1nUIuU2IaQMlZiHf`Fo+2upk3*|1vh>Otw!_-|{K9Ht_`A?27h9m!W zz}X1>ye0m5S4@T%IVuX3$ijZ(qGw86?w|!{_l_Q4uo^14RnbohxK6M91ho#0hVjgs zH*P#Fo&0W!EHjX-2fv?Q;p770^6t}qPZeU(M96cA>o^Jenn%OxPSW)8UZSPClN27fU^D8i1;Z)SH89~R9nBWleJHxkOg+~I{eU;nyOBP{-3*zJY6 z-Ob!O@t_(^Z^@K@^IIm0K$QYz=?U0_fFVcQmV{)HvLd={d7vn-;Xv;H?}60#pFLwl zqiLO6e^&Qq39B;wCcUWX|I^myo2C@Ic`he3z_mPml7vUfI*fZf2K(~ozt$4Yx5nj1 zU$|-VaLF+=3KelkL%!&m7bX$3F&qDhb9npqZ{vuN5Rj`0Y`vLZcIE*@Gbzqn5%6w{ z0e_Raa6iF)bjSU^8%6HU?Wq7~Im@l;-_+OCk3(0l_9+BH$QZD;f8wyaN|>d)NzDOUYLM`8^tTSlCS?l%+6pSvl{g6K5$dd$FP}(7I(C{y z5LOut;KFN+ixN!ZNBcafyIYg)-52hEV?e)M^448hNs@AisBgLLx*-B%fs<+ATIC=n z_?y8>7IE%Q(dKep$yyX}=j}1fjL;rsPW!fiOkZQzRZDsmNU+^yNK#4m-rryT=t!&j8j|AZ=i9jbQKp$#0O8kI(^6P-4*za3q8_uoDU;KDdCF za=UfOR?;|>XaIiz`!W~a@2}`Zzd1`A;TQ{AkZ!%1{ zoyLpfq}_M;q;{L@d@SAcJg&IK6P6xd-@8fYS0;=rgKIz8PVwoC{}gTN33fp0-^XQE zR?RvNG}(QZrz2Ufk_5Puf&fC*CA_##|Ng6|Hda$zAuDGTd5SwRPd@RPp}Xg!Q=TUm z9lySlZ7cvZRA%(q)}&j(ah8$?mt5A5tqBuJ0kX$I5ABYv_rj`NZcmv!iTA~r_!$5a zqLnrSc2g}D{x9o7teYWMKIMUc8zsDt~C4ND2lzvb0og%pu`}Poa9CFm)tC zV_@1PG`d^6IG4$Z{QMQLzTc65o8Pv{e>z)zmv1yy(U0CZIQ!zpBl`W7=iKaiO^vxm zY<#X2nR67Pb$B2VkV;d3V$iKPuH}ab?%k;~4X$rS@n@A2W&j5$eiot9vJR;Pqm>@x za@J61F|g1-5Ou2{E?^cUCa;rE`ZQ53^2;64c9CzK?t|nn`sd z4R;-!-(`WE+4J6pA%8k<@4h^0kuF~ODh7^r_06YC3v^p|(^2ev_yzcMq)d2qV+sZ% zEVO4gBhgA1m*g;Fi4c9)qV@9{mV1h)``SDb7x*n?SuCGQPE@un&fF^imKF~KP~3?F zKPxLMj4gUmzmU(j6Iw=MxbfFq57WLrM3_oAXEX?=wA9Mnj2gi%7L4=~^v-vJn%v@H zZ(cTkEjxVX&aXZKCcbtojWIgPEj9`A~gUV`J=kEqdFp=8F80O z-`r-hEp6`n`JN!oi;*rQx`CyF;}D+af{`?CSdX@n=A@QC%sC~vI*BnHP|tfFfoD2P zuKx(Kq@T$BK+8x{wR0Fryv;5LX8m3B;e*)c6ZQ1c2tNVCo3JXrpAtz~8?%P*&Wg-= zC5f$wi+fejN5Y(r6;)6^*ODS*IKU9!=q5z-#+60w$F@OEsRTp*7zWAmx9%8XpgJ1j z$w-vY^PUJ{DC-RIX9nOG`De4>cT66kfxvTo-=KAl^E5jq6sNzA^Qy$a9Pkx!+)F4#ug0>V0ex|D+Fa@&Z(^^h z-QMZYkCEXJK72X4mmbZcMs*;Nh8se#VDgj>+XqVP45f^KAo@Z*1<01)o8SkToLFo` zCS#XMy7sG>h<-)b3+!tKnuE>nP=)Jbx)a&LDb6H2rab2A<*7>`)0ub)K+--x$@Nzj zkupkvt`=Wdsizr4&!b4D-n*$V3U`mVLalXXOkP2r>}SF4b)Rl~Jo^q4R z{*@ERK>uk~FE`BJ1tZ9$Tl3y|*Up`ZP!8t{&wsVY1iV)k)Rim|fU^c4ni&G#=^r09Cn~q~e`N-kanb4hM`ob8Xm0-wjEtoy zukf zsLIb@N8`K#`~SPtLDhR(y6v@$!66R^V?#OCJILr@ijh_0^HqB-fShD=113h3JMY6m zFJM6v>4^?KFjmJi$Nlz&9zCvqi%pV#4nc7d`Omcn09Znd3ogp3U7l9Z7etvQ=dHzr z9Z45%Pb@(#=h6e)h*z=I2tF)`{(SfPDtE=tmQ)Uo^_ihJB|a9B049ZTUxuqN1kEqo zy+$TaeuSiq-;_5+S>x_Q%{xuk-7VN zpIJx*mq^lba|bE*W_x=-z3X8NFa_wN=u97c%m^eTS`CN(4#hgH6hkpRIll5c)}ed% zUEoO&kRcG1M*vD@l45qLf$t7ST)Tr^Y?rL}1;oy6Sfnni8~$L&iS!s!Kz_rc%(p5E zpLb0h+F}IY$a4H(Ax7}>>z zeyh7gZ?Z_te0>vc&#Y?sO5A zkYydATd7yDEO@k0N>1}d0^ky$JF0q25qMPkJ zo_`I^!908ZScmVvyX>G*u#bu*Vtf4o#JnIt*%TNUDC8u8Cl~2ce;zti?IbbZ5^A-4 zRF-5AHW;0LXz4MhTMJiT3=A}-qP^?Y>#g5tqWAhi9>HMA#baYr(ywZE-+hxo_&I#6 zpE>Lj_VXuL<*n_*?arpQPTbJ{Ktq9 z%LnK35ZAjGd~F6Sx**|G`_2~) z7&IseP>~4brPC%xm_-`DMU?Us;G|U!20~f!gzind^vnT%Ss_UC5eQ=TfPNjHIIEx! z|3po_i7qPg9xo%MGIl8g;w6Qa+`wj2Y|5q3dHx^fNg zr_ecP2r)wwD#7x+$0sa&ONdGb0a4wFPTfTJ1UYR`dth4WnVA)LJ|Y7bfi~sltPZR8 zZj~o(2R(J(aQhH{4)so-#=`XStlc0}GNsU>_CrK;MB++_7tS9#x!1_CZQc^#GQ3Mj zecr~_ZQ8WyVc1tS3$U$KQ4hA9VG?Ga_>cU%H*d*g5+${_jfPSW*hI;Ssdiom=u7UQ zL|-G8<(p~KWQkD~D&SU) z>-pzqR8LRi;NRKy5xJ3rhqrC}@g%7xWG5Uka$Ymu5OoIt`V$^}A}e~03Co-SjDd#h z<*7JO5bkvk1!f7rEs6$S>FNGO2FxUMR*m5sg>o`>e^=0a-N%YKOdU4UHOKw7pNkRk z(o~hMSPf-?I(KoD`RZ@6Vj}ytJWZ|8H)k*{3X7mpe4&P#fdu7jJ3+ZO0s+ zX|SxDpEED8nsOkMk;KqwI}~0&c%yE|^1>W4sSUrY8Fp5`?4nZdx9`6Ta$l~p!{Wy1 z42$jQ?rAJXPaNg}yV8@9tSQOa_b3=NWXqy+gP41ghok-@1yak|?n4BcFP@yI+2(PV zKJMvbgr6yUP}B8z2j^DaEY1fFfM!*z_HVRahQOqxNx`$sZ5Aw8Ad~0p#sToJw32oc zyZWogYLGI_m0#nR~qB}xdO#1@sVMs)-z=k@e9djZ+WWOcB-N)|*> zE(l=xQTt8R1x92uGzva-bGb2OLm7>kILWxPovPm*&nyEfVao(dlYVxub0zYNYHe<0 z+my~*KhZ4gN4t}FnelR=xT?u>;D`v{N&V7K9jCV_5qCJM$eCs3sDgu!V?ke1~8$;r*7Phi>nTA(5+jy=%;>jq_X-lr?+IuK+_L3HDUWxeTOs!*xUaqVBIaDHkU;&z3^pI zTbx+AX4)56PkQg!t6MI){;pL25tPWeo}TM(-0IoC+}?ZSjbrR9yvp;D{xf;geGrWG z(a}RlqgobS^?{g@UvOVccg4vs10mSg4UOXM0F5-h;t$bRacX@Z)noCG--Se%`rk zVrUi{*_f1Mb6IbLqs>dp$~IyLP^6m>H(8kC`=eRkniE+iYG!=;QEgP&xn^&!g`AV7bBy1IwK-V?il0I5}KLoF>Wj~zeGB&l1-AC5*}a1B+) zzn=_Mqhe4yi-Mgi-rI#ynRu=%c576~RuNNZ!NK&A4uvU~pg-~o3)RO*ooo*!$w8w| zZjU*1s0)53rn=;VUK68@`(vz;6~;-nBTmz~<9r-X#{2Hq`JCC(7n%L+O@Ka`;+7}e zv1HD?MDKgmp4f*bV-KD(E5=?7fF_l@i=|s29~g6`1&U2bbt zl|WGOkc!^^9vN0NE4W0x1v$m(RT(ULl7myjjD8eDo1SLI`E?&vSX$yekF8jl+RjH$ z-Xl?*n`H`6_Hu>5ZrfIMS=-0J7Z(0xE|Iwg;qg8EQUgZ2@8tWNH}otTheTFB*gDo> zNnoJ-gUD>=Ep+gmsw%w5V&PVzE(45$`#&2zR#i1>8~rL_E(EmY`4q*Qn3%{Q1W0x{ zCugnoR}En!dmdqJ<1TH;4dbCcO?XrIx9?BKXUeM4Pbu>9oIpXGtarbg%2gXkaRK zueW7o;}Jynk5k`x;zVyrAOz!eAY3hvd_}l$KAL9x3_I7!@RIbEUmKdADwpsXHP99LiY7H0k`+o>Z|7%6@CNbxLA$ z?hjqiWiw0!K!Q9vihYH3yxVSZ#L7fBnI^> zI!87XXCAx>)rjJ6qaaNq1`mAe=+D@2n3HlvqIK9Gsy(dg7S2nCim#c%#ZoeG;PQ#f zyDN%bp!MD13t6>J(POrx=+RHh4hfoEt)A*pjf1kZaq){H~mOabS@wEBNr%yL1@Gg(%1L$4Rd(AsM=rI>*Y>d^!!G!V?l_n%* z1%3F!6O%N@0{_Z@STzPon*3cuVaCL2U6OYCGSw0g&#_>{+DRJEq@`Yl_U!lrM#ef##w)>_{d5&rc-C;C5U zY{Qz(pJr6~he2e(k`~tI!aDN|_r&drp}tVp0Ka7@Oy=GVp*A5D9|fOppZe zzASCDXjrwnADTF;Z+i$<=}r|Z8W0W8Cz#zRV+1sl)t%^WcHl3KTnEv{!C6vQTWuJy z&0?qEhc3yO@PH^mtTlq8R)mL#uXJ$m(F~H92&R0mlP`S$F3fd){<#Njr`Cs}L<^5a z;c;lObh*4*MU4vrn;iE^r%l(9064K-(=~;^9 z3wqI{&PUA7pTP;E-vqDOBC2j=T;6vXu24_i4DLVVs>B?zQ51l6@(Y+hT;jVkD|@$9 zl6gZORn7naG_~@ng_UPNW_-4V0S~;p<{1{^;eMH(U?+&Czj6BER;-fj#@t^?KV+p1gIM{`JQ) zqS9Z?^pp)Z%BgzDZGcg{CAPLtPQ?bjHxiUjVfB81{t68eBIBdUQaeH_{^m;L#ZLbC z?a}5x@g3t>Z8Ww`uL>AZZlMa#o$8AFe&XSWjahsh5{CGiRT+J&mb(1ksN904Aod z7&s6~T@i44+QgvEn>Y8w1~O>j`8GB%9}=}%?(%VOkuTki)J9;Qts+pxTvYpNg*(l{ z;dj$Chu|8WJM^VM9T;`Yr!B2#s^#9hYL8%CT0Aq=8qKrCr5prY(f8l3JEUN_eD6+k z=?f-}O=+t2D%*C~)gZu$R>R3kn$GKomw50O-mjS1rU{VIRBKx;%e4RuPGyBVjwPm5 zpOk_h@|!(wq0Z?uhEUHXck1omtMB|RpS)~Uo@G82z*RO3iu~{OlPAAZP}2U)pJ6W3 z4)W4t1;&p@3iCZ>%8gLlZhtitJHXX#F~pZ67+4}{n(F?KkW|WkEvy#C0}LB5KAN#d zwi|gmKaL)~s7q{YtPD1=@cRb0&u&bw2mp~B`H_ml51H{khI|1Uc5-~eT5Bz0Tw+_o z2z(>GYuEWCb4+RM$KtN!hsf$YurT9M5dgXfgDzvylG02oAG+@I>qnoVl!%7v_EPVJG-!B+#QN> zPp<4p3IwrPtP@Kl5rG2g4N26Q@`FyW+z3XMiAD7!WN6P(qZY91&qsoCoZb~|J68DO z#db`k(puxSs$(BdJ~hFpf?E(xQ+0G&?x(Hd(qvKx4GlBSL~~~&@P*6hp{cogp60Y4 z8OlyZMz)m~4Yy0?(B(lm%q2F24fhXUe+mWRmTU-<7A<-Gqs+{u(N%Ofkl8*u2*KW1 z3isgm4CvjS+H~Hf;)b>F#*K{|?6ITmUS~XWg~JbAp(Q=$=dWJF9IbF;{qyB#n)cwE z0;ogtIjc(MN9uL#wZ^MYjOn#+*@u@etT2lw^T^~Aj{)3H?^oa`ltJF(xda(}_4xF98XJQ2gPC_is*1Fm&54 zU0Q;ccgje&g6tRrfyE(FI&b&D0bwR|hu(hw2P7ijW+MtE(NjVP0Op^ zwFVRrP;B5b7bC`R7A;-xX#EY^Lht0kJkO=m%n}m)oR+U#xiZnVEy4tcCg6h8*IB=kRik)v}9AqYy zENL!j{THY-@ZKJ*DA}^cpo0yX>x<2 zZ$rA!JS}i^5-)U8Z8y+RalU;*tzl?^6ZZ9tw6oGwIdkUB9}y8Vq!IS2SxXR^44ODw zT1RK=6gnI_d{`FYVVP?}!`beJf!@1fLbSZ%y6XoKOJ+vW<;)fs*?PCkZqJYme5XHv z)v8mD&twsBhPVvF!>7-jA7kA<)tev+!N|VceijhUg2t&?(>HqtmYK;Z=e~jYoV;A=@m`QE-(iF_$@&}KmMy7 zU&-B?y_NzEGYHi2JFUp%*N>cgli&})A~|d=lXbny08n?NMV}tqKfUz9qJNGZe~got zTMhVpD)saX|6IDbN$N+lVL$TUA}fA6yz*6@neF7ceb?;mLmY+&G%|nSZ86xFYAbVA zynlV}4J3+s`3I0S^qlYE{{~L~(@x7@It4sUi|OIgO(b`eWHnIG0ryHuN@QO#a&foB zX**x=DiduiqIc6&TP>`*Ey#kdZ;TO;16gza{Q2E%4FqF|G$Q>5>Q;vG{nMDfJvie6 z9?V0RBPo`YmhpfP-kvY#6VSonb=4k98{I|T)2FJ7ei~Wy=FOl^?tj`3b~}XbWwzbo z#p~N;?#Q^fe)U&-BiDtOFQX0fj`a7>39VVslKok9hH3^a-#m=lNV`AC+SHg`*PU2L zc`+?*#bxbcW|;?}>u%x0H8acTquOl4Y{UM@{dm0>BCWxh$tr`NcHiYWaT9W%%NZ08 ze0EVIK=+MeERNoiSpK2HhRbB0f$Q_<&$n^_qLXXu2Z45q0REai%1{`B4a8crdu5=g z>#53gp2XT@zN6bh!!kzDmF1>jP^_$2NlA_bycq)QlJl%;ykS4MM2*+apO3M$Eb2IP z-D^_Zc}qz$8&S9lpo_a|G0eQ<$8#(wm-JYihXPHo2#chsn9MSyNPLpN;E@s~(XnHT z<$(7J7(Qswpkzxoeo;$zJAe;KsswbRRP3tm!gmMP!(PrqMRi*()~A;=<3~eWoC2=x zr}zXPJz5Z~X+8t?HEic-BrC~#JuFnlwqaZ2fRoNHE~>I?K~HZ5xLiRVK*^L=tp1S?}FS(0_XYo_}=B+`14FXFi=FvSLZ?W@*zjEWVm^JrQV;J)-Q8n zTi4eXjaE~>fWm^!?3T04awsJ{Et7Zke>4mmS3#;QQs0~#h2dxHufOgoSo03fLn6id zr%uE#E-_1=!A`L8Phh9JyVXOePQodAXe~srPdyMeZrH)D8Pq~DC5L@FjE1T;tVE?F8uMNUiaHn-()P&PYkNRWhp%zQ0r52_}bcFX_qdU?l$qG3s81Ml4F$e-|UOa_iQusfWhgopRymr!uFJD76`u zf7;3G`{^pSU=Nrgx4C$$QSd?IXW-%Mbc-D=47&P{U=fJClIWj4+2uplyf#Sz4JXF$ zdmA|+NS3Qc29rP06w+b~maCAZa3Zp92%N}#vc*VydwUiY?Z1DOSiffU^b)0)l5BOB z&x7&ZyUV1%(4y01>(&hAeJRDMhBhGW#gHe58L3f$rX}y)@ zsBsO>ADZ#a_6gf=D+ zhac*sbI|G=`qH=15WSlj%>QyKMA)vM2vS1M5rr~!I1iC=V7KD)jPc7z-R+8dVkM1{ zKG&tPDEQS>qvELwa*Hj-N(0p_uof{8uG`izIN}r$x+i>%d{g~Fab-A zZh|XBvSA4pq?5OsiUOd~pb?F9X%#W!3s&FnTOHLeHY#XNAl@?C@$pJ#ShZPWpInct z?njMgGsQ?7^^{o+DqOu^8ZA@*cRM5L(s<07{rJqK|DP zN(>ZLD+pxN(gz5{*L^7NO}vw51q>Znu$=MzCgutJl%I*Opc&j21DD4=ceCS-TSTgn z$BzBkGp4YxaM?)x#MOmJezF%<|B}Dx!{6!3yAB;XWIn|-S9EM<@V|!NId9&yN!9>> zwVA$xQ73n0)5M_ams!i$Hjr+$_byM%Zokjtx>p@LbeLid+ApYNpY)d+s2MQmO_i%t^7LN_Nb`yI zYNx9{vK?M1{hdpy7!@!a7)M*|@H%O6f!2yGANDD31+=bWUzFFqr>|WcXclM5J2w(d zjufAR+h6`Z?#LMOzlnOjCK1%Ih2_Z+?VB{>%wFUULS?C4=)kZ8f8Vx42aD*GjC8IG zSrA3<{N%}%EubJO(MKj=SCBP=VDEsWWaQ=o&VtOKtPrL_QE4y^FxDI)U@TPgy zKYcoV!3@a^8?Fs?(f_gGtNi@N|76cK<$n;@+pV2&4@a9z3?G5&o>TGr$X?k`z?z!w`#RF7sKjm!1A)zecHZZEwrXSVyc z^=J`QfmU*7UI~q>o_u_!g6#)qBROTzy_}q!QjvYIB>3;0LC~EGiJs-Dk-8u%x(eQQE@6@>a znS^@43Lc-;7d0SU?N6e7V_pEu3FO>ifa$jr<$(D#v%rb(VlE449Ut?+I1fF|!@Nu!-F)78qnj9uws1BtGtGLK zXXJ7upx^prUyy-(|G)(|f-n0lM&ikNoye}(Obw1QLL-q_Fm!pW_F+*`H>svce0KD| z>kk|_;CkfxKoq-woIh{UCF|b3b?E=bWdSr*nma=| z%Rq0)EL;$oy^3qso~S}h3idG78e)!xxaZ!1{~GtNk3UVij9;+nqgislhY3K`wB5*y3mi%^vw zDCrV)@%{>r$D7NFWtjZ*WuVn<$Cs5*bVfyyBoT{^4p2pTq2w9 zaUR%bwoUYILKFo@cU`e!h1iSHt9G#Alnc;UUz=7x?Rov6?48`*V0m>6OrGuP@s~-g z22wJg0a^0bS0WU+CAa%itk#lH)v;@ zLXrsgkM}&*Zg%cHgD^zNAD?1PMC=&JP~B$LUmky2+JU=NmWFVfsN=$Bv&Gq{f-y~X z;K1o}o1Tg|Eq-i=>DEW!y0%=MLUxrQ+oNQAw%qKI?hmS(A085sf@>prg*HQA;7> z$F*r9?WioWH7c68?E+EpV->{@ZI~jzVC(Z&7oKtx_Zo_nMKYGzQcy_>yBCa}qCcgt z&x^UJw}=2|lX#cR7au&=@C}ZxiD6XwRPOlp&zwE`pf4vb{R})GR*p9GLR?4hpF1ot zF@gWt1ta0KcI^yU5l-Kt;%Z^!~P zyyCQbG4l#ZR0prRl9o2Z!#w|r6g>6dD2HY)9Z%~?f&BF4XRD*~Kh0*}kxj)^Va|#B z=3u|a)9=?o53wX-F<3S|wMK7$mFd%`i<#stWZqPjym<5qBJD=6apBC^aLxx9eO+~$ z`*yJp%H9dY{D)RnZUZSDi(umLFrMfnFQy+HRMW*(uTLZRfILaJScUS%%)u=+$s zvHZ7Wy(*H4ppe^B2zt*Chw;U1q~QG73f0Wl*H>(zmcK>MQXvGfW6I%?bMB0Zprhxo z3rXk_zfgk%zQ4~lx6Y&5KCV$tE-q`?YRayuD_31A1%mYULe9Hj+J+&%61k0vM@pb5 z75QB#4ESzfZ+ig)qp+a}%6i#m8y3g7()HaxwJgKS&NRX(L@*V7vs`w0)w$j2U0mTZe8h+;J3VRd zOrKodEbBt@fb@}uRyN(GE$uWGCsZmv0oe<|JL%ZFOx@ofaJ~;?AMu1%73Uf1q{%1G zc$|qz7jGXQ$=VwKG!oooN+M^N1OwziYV0Xd_DL?k6u}fc3GcD{psg0$(hW?3A3FF3vM)?r!jg0?^PU`?S^Dg<5{0JJE|zVz7BX=lG2)un4!Pz$k4v|bpZI3}x* zh$>Rm9SY-D5GpGwk4J|}@`QD0SH}73*VxJ6XI{BaU+L!2yM~Zx_4&&&+~&Gh;ksx= zJLz|4YSZCyUp0p)|^Fdo52kfIx0@A zD!v#RY*BX{4-O{Rrirk}vLN7nZvDqaxQnw_g)vNH>rteZGv>?sSxg=MZM32W@eO)^ z}Y<+nB$Zmd}p_(bHL%`HxF>P2B=KXL{6NB}NL1Ob~*x5yPMDw!_HrG9I89U`5}ECh1c zZEH1^_mcU1o->LUuYwWHl zE)-_t?-@Y&iz0-B6Yp#SCLR^NztXF*a%&g~QNQTpCge=?n z)alc#NWNb%3S&6!2z!p-M-ZGyC%Ma6v}Wn2;2-RX+u%rsEaldg6b`xe$43onC}rU% z-J2{JZ{cJ$JobzIIqBvzZ+2DmpNfM`FsxIy_67Z4w{5kuwRUej(R#`?`tQNC<?gK*>EEl0jFiPI74mTy_+I4c+#8U+S*Il)+R!j zYx8Vu`pGt3_J+;>cyO;*^sm^w&HdXb(NsxUTS~7g>d3n?Tw**Bd=s?RgLpIrnS~?o zT@zLRWi~cf=shw2_c)o9n8^FIvF2K@(!$ivLfR;-g=oISb8^nFcI8IRb}iASNILWAQ{`|qb-y4m3y+kntL-oc zV62NyztZHsQWVQ!>;BX>$GQf*Mo!_+vrD&HsW{h<_*zNuoIcGBCUHuPYW9Dc(*~pK z`^4<+Op3U!E|DboB~H?6msWY2XW-pZP>&;)tN2#!RJ8gHD{VG+7kEmB5XM0fQ-x?|f3_+HSnhkC)ln>M@?71`iE?(k+!F zPi}rmh?*GWdvsGng()A5s5K;{#lI4kHRk45)!(2mr*xBL(U2G;SP5fJw-TA^jxL+p zrU~3Z6X?V44mrcoWl)ATve&j{@r5peQ%3EWvtYdO%cA~kTK>~G^3ysQ{0AvnKdejJ zw%Lh?-nVyJP=&&6LMSAjDK8PLMOaVdJ^0bx^y1aSo!<5k>74H|O)vsj5=OB{9pl48 zLi+KNs_nQ72`wlP>`?QffS+i#C9SajLsmks+$xSG8CtTWrc*}t&e+&t+*BTD&?B{Y z6#P@t5f9lq`myV9Fk^+hgpvDk$#B1-d@?v~LUoZii_d5!br-Y}k=nS^w&tVh)T#Ci zx7X%5J8S!oa_@rn?e?#6haN^y(-b{CU^vnKKF6Ie7O_bSiX7FS9{|B05Eiqy`Zrq?@J479bNMg@j@L1Vq;rrd5FyMoH_Kq4UUm_cDJpf5`^+0{x_<6h@Mh1Uy zK4p2|@)CoKyj9?Bs@;;h)2J{2#07LcK_6~HM4HG(ZgM1OFn7>Ur2Um#Bze3LdHIRT z3L+1NZ{>VX-M+w|8p~N^UH>tme$J{3rEZc(!yoTSak2K-#TLO7W8ro<3o;6VMSM%r zc*S`?KNWzJF*L5I1o5>PuDcL>}WoW%%?z(5+pu3UvY zeucPz=3Ruwy}wV_WaZEnfgT%}IoN>wbvv-7h$;mr8RkJQzD>tCJButR`!w6*IbpPv zx*tqnESdX6F2L`t({zI3fb|M`tO#Q&ZOGG4IzOmtYW%r1^&huHM7#<0{c(#F9t%lq zQfwvp^UuHM1=E;Q@Yx8F{mi+w=u*r(?>qeeKTf{OulN7swB`GZ8NF1G&aiH)c|Ur^ z=vx80YRgRLuvh`bi7aLLCB!mm*=>0sbjKtr`4?D9ytQbfwY94~7p_V67H*iG65rXN zNz?-$ICS_~LiW^%EZGSvvZ!P0aN)bMXY{RPU;Elkm^8_ZpXKdQ%mY5Ny7)VaWzR)F ze}`&>7bK4)%>%}rRa7nm8kpC+7rcJGQ_^H;G!sd;HBMm5ue`ZHkDW61#Iq`RQ@Yjn zNz87t#r5T?ueKJGQYgsA(=#kDlmMT>dp_F3PK1Pg@MniahH>54% zELZf8n8+z#K-G5K5Tkny1(RZq8mEvn*&YFE!S1fe1J;Fj!vcN4N6QD1rEcAal0o)4 z&$=F-z6BN?(sg{W`4F*K&^Cff^ti_@Lj`Zi%u&4|dk-IR9(<3HvR$K#85!;1VJ|~A zEuE~MARQhpjJMk^(G}?GE_}RLGwkaoO`y~lTRl%`cWqw(4k1M%iDD|FnKbOQVxr^) zalwfu8^RkY#13bZ2pU^{g(T4IziEOA!Ew41n})Lv&lJnvDlZk$7psEM0%YiDP1iH3 z@dr=;e9l@mZ~nxxsoPALR&yP`RpbWO+j&^*TuTKJK2Zyqb@dA-enewerf2o}j8A~pJWI+A;Yx=<~ z+k`YEJyN%J?F#HrRopNprK;0H!jXvu6#3VY*w`$KHzcz(`m$cZb3Be#974x8Twk)D zIXxr8EM^*SGQANBO^sgu!xiUiWXz*0KWMbL{ioB%UY`ar=_<{AyB@A?=nNHqH=p%r zRFmkp#YOzL6TV-MIqC07fP(6fA%=HuU%*g_%Kj?G9pRYGZk&`5WiC|7cfT{Oira9d zqhliUM$$(=NL|QD@t7Hx;Fbo`0%0@vdDu>@D|8-`Lu+cs1@j4|c!IITY(!2r+vq`v z((m5wbx$G)R8($Qgk4IG0DU@M5cQFTp?m!O`*pI>^c(bG^s&-& z;_x>9T8Zo6@6v~w(gsN~v}m5vu3yh{3_mRQ1B(`AclvLT2BOWm*QMXF8L%>+RF0wh zJGJ~q0%~oot$svdBcS~mWjnkHz~#)r28HGs$Qa?1TO-v__Pk~q;|hvFx_wz_-6(<- zlRz$In~i*Q&MAj<|B8$(4azB?ATX2<5Q*OLncAC^En0ulff2(Z2O3ScX5pr*ayP7c z_AQ)*q^U{LJ58Ea>K$yi=8f?p+q7LDTk{=1&D#&=02So} zRFsO;KqSuCVV6sAW2KGK$^;KvQzs|mRbfrvR_d4-8y^IGJzYXR8S|Z*V26Bavf=c! zG{3BlQ(%WEyexx$_1nj2f^KQYC-dBqBTG`sy5`FQuobzw{f+;=II#~v;WY&_@;{5@CPIKCC24ck9S1w=1 z4Y66|e6j@v`hon58N;IoKyk{=aORA0D6-%uqRwjqfd;g?qc5h|IrHSBFT$%r)n1&rpT&?+tTlpdv zy+*b)fefek39_E2r6KY0^}au7(xhVLBJL{mw|?r!3zJ}+qd`uqo*wk zK7oEFkK7$ov!3)Oh~rElo4t!rKud~}H6NYfeuv=BKm0I86qWmT?_M0!O)V$n5<9-9QR(#?w3KUE>aar(ef60Apc%2M zZOe?JffADrYYaThhxp)jf}1L3g=?e)-ZQ0&IX2NegZHtcWcbp%g)GGyM&yx1pq?4j zvm9_!hxd*IOF2nZP`+qsNZ^_wdVVA00k zV{^{yDM=^;2^{(Zgp&W+d>GAU-)wbG<{0*Ad2O=_MyV4$ueVHGPG2Ph;ww1BtTixo zzFA@EUT<*$N=~aHMx4f9M@I+Yjb;qgHoIwvh!EW*;RVU@T?4OVXLq9VvIiWX+SA;Z z7YkGynTW;B`xKsbcZf4BIKun#o@OI_fyi0R4 z){i3F9k&kQ$R4smR5)iIO9COgSPR4p$$Ry*WC61RDVv8oC{JZtkXT4eGFS*aD;N;- z33R#2_!d0j*0y&<$if2&P!p+zzVy@lMNhqZaMwobQI|8&+2h}dtYi+8fDp8HiRLS0 zA1NU=JTucUCSSbPmkH9o@8qv5QMnW|#-Fw}bc`^|`#5|D#6$NGVFEA0nYcJSbah_K zzZ-S-_x5fNp9gmIU%qS6Cl8&UV81*O*#I2c9{fyOd+r^*T1qghO?+JK%*_k-K08!Q zer2yzZ-yU)wCZL@voC>+gL?{S(vHc$pP{PCD{yG1+V_#iM4*SoAZ2TQ(`Z?#iG>1b zuG&dV?)jul9(wDm$7LX=5s6Xge!jQu27bMh8%V1j+KK2tH&VE7uyw(?^r_@~?iyV6 z8$TtIP?}_|3fqn(90**R76MNO%l2;b(Pgs6kK0(W1)$#Q6!W5hiiQWh6v8TITj@TL ze^WNZ*kP0g2zm($LHJHKCW&dTM~}rOZ!$A7HljikhqXntb4n5KVDg(ObLM2dzre!3 zQm{=mv(4igiFWPI?bSnE7G|)s-I0c1)0-=Z3w!Q~#u6+_z7v2;qnzJ^fC5_6NHP^} znR)Wi(Y+1FW5@w`J4A9s^YWKsz37ip1S5@Y-*x-H1e8@O3Mq5wkIlZKj~5kjY2EEA zaiP-$1P(4!w^x+Fr!JosASzDns*}%JjrsRPYWn=9?3=D@<-e^Zf6lPVu`SMSNm-fo zr(ih9g!4&m-KTjpG$ceoGR5*i_}#=)hGfrtEr-#){(>>Hj^v+{qdVet^EdHd=#U8= zXwV@iTEu;cIrdV%By%@SF)OL3B(6Rk%_Cs?Mt8ADa(_(Du5QlvaDd#OXs&3mp}efbXx|2R0e=#iTEO#nrfjwJ>ql&E!(H(x;hsaOLfn;=*a zrKz%G+DWUU4P236_WF;wxViNT&+gKru1EsjG#tflX%nnB%>d_2z z|0*L0SYKswQB`H-^s#+E`~ZRrEX8{vY=Ah%CrsF^NN|8q6B!-j1JY{J8FtmBih7LK z8y(5ME(hEKdD#)0wqaMHxREMo*5HD)yn3nrI%(1t^Ovt*-?G#EE)$iOBmzDhczl<& zt?j810bM$F>}{yxrw@pn&ITcY9Yx%Hj%yOUXYNb~d;75yCwffVw$=6oMVR5kC~Kfl zI>0%532-PVe*IcZt7O=T+kghx-Yp*EKLbcZWG{Y2E9{kTT{@%d}dtN)(K z%qp)(1sqrP;Bd~AfICS?zl z_u#{?PzT7h%(8$EHlS|Jl$DqI99>383f8Kw33VSrjG^YPa62lZaHC>HS4o3k%O0h@ zn%^Mr+dv*KmNN-9Ytg*<@0eMS`A(r3-QRz%DyQyPgL3Qk?H-b+I~bi#C&TQHQbgFe zpu|$IgUzR^=UlwF9$$Hpe&9U@%uXdDESt6h0u@11b8)uH;3OUvc?P)duatRHg^*tIBlWFY> z6(lP%Fd237&fbFg0DM|cLABxU7xaCz>RfS^19v*%+Ecj7(X`Sns^24?-ZI6x*IEQ6 za-{e*ppl3ZRYD5;w{d9i_hCqaS)NH;zt}yxs9C7g5n_k-3>lOT6ctszPO zota6pbZ%nl!*vD%x-4hK-mNe1)nrXm5)GNoSj8kLY7&*${9;2f;e+iCdVTZuCgqkb zQH?hBe!#blfUg%6cPiHwdn(o90y;`2kd96Gv*W&Kj|Em(<<(q?>v<1b(P#Xm+F51; z^r4@6R8m*%6G*!o-EcG^KoZZt`GaH49{AlRhRx5%hrc4TYV6|0c^%zba}BlY+3YLO zg*~(cR1ii+_ws5GPsyMM1iWiK{gwDwQ5DDW{lT|g+rr-2bgSCD-`p1w`_LJm!8=hk zPq|ho-LIaW*~!2Dy3Asa<-g4ss03*KlBp~4QDB-vTaxsj18G+}Q0oh>Kz~+JXSP$Q zU(4yd^Uzg^0Kf3Da(zpS*a{chcQH#~W^Lg6{KZw@l+3;JGAZ zNu$Fj&J_qBXouWv6%hOj96cnTkoS; z_7Q3g+fy^DXQzy|N;qs=19OSI$p+2nM0~I^^(8-EMa@8Dso>$ST&aIF5wVMao05w| z{wL#4d*PP^nG`b~erW=a)*-ymAF_jKu~Wuc+<}cvp*CLfiD#a3F84F7J8V%$;XU{~d?Z z+%U@_8X8E=HBu4v6g zI7h0AuXpM|E`Ax>6f0)}My>T=I=uD3&YDaFiY3b;+DUYfGj1_n*p?6?o$h4s;-@mr zaIh5fB)X>RaeMls4U}&HD0Nr$u&-sH6PcZKR_IkC7tWtQazdYFIzSNOE=K4jCyo#e zqrjqzIb|0uZyov_Mu!|=$-R`K0zf@}(W2ag5n;3Xl76#wc8Jj>sKU^5$e%c&=H_~# z5DAwvGBPT2d*EDQ5iB7sOC04)K481qmW~wJK+>AZ)3lp`)cBY%TheODm z5HGqTWz*Z{`TE$T^U@nZg?HxTW*h)p*?+h3S6}_j+O>57K95uoyEH_^%gjh_VY3ZC zhiB2XvKq8acW|ffZtGZ(Dr?wdDi`8eU1_eL%V8C=&-aI_0{5?5k9C z7f`#+qqAG)*^S>0YA0d-8Aq%2NKuo33wekCl}oQ4;p?5gn&fcB@-AdRJ7`y+a&H{G zp}S+b-{7-x--rdTBDIhQV0JH=v-k&u_>mSC=$G78iqDP zutMgVf;h@0*(nEKY#nk3DF9*VJx-GLXA-O#odjY=CL;eIkes$mL+xl4aLE%nr+)AQ z!b1$GOn-ip^qYJE{{2{3%;^%HHO?Oc@)id!JWnK(>dBhwYDph^`|h20@B96w>WUC9 zYSAiA4Ybap#98min@QyA-3c%lvXl!D3G5hGSDn!(bcx;!Vm-IuDmPC$J}9racpC-Wlz0}<6!I)zA5_*y(P@tf zo<_2I;|`zPupzX+Lkb<&vlJCD331!Cd5l@;scfR(Y}xv9zNO^}^OwcNeGI?zTTbQg zWrhe3vu<)bkC)?6_J~8}A5xEIb&J9b1yiTfbW+rFytN8&l%Mhm9CZ|7$zjp3?vyA z@j>2qKi1YtH1Lzr_0_w~kNP3*khpq~&Y!@of=9gG%YN7Q?2xs`=0i}_yiiu^h>iOi z!snW7pO;9CrT0WPcJBV&G(ZIUqm1z1Uou?)#nZ?3x+jh*V#{4?Ud8?r@~ZxOXCy#8 zFL{;kF)>SQ8Pw8^gI-D(X#&9W1cWdS2@g*c(FF1#v#4fvqvt3!Qc}{_8#}9>erMc& zpj3|XMZzb5QoG71jr`vB#qvmFPW3iX(jH2wqfjTG;gbe$8uGMkSzoWA+iEUeRPtZH zE;xNhEA1yw-z%JX?9>s%v}Pp^SBabcp-Y$VZ3rJZ?e~i&FwwW0P5;VIogCD82JsWB zefv(~SZ?gm%t*Nd=lX5toJHUfkF(O#6)t6Q-qC3;arj}Rcvf-ir2ln+1y3xAys>+0 zaOZ~Fi*+SrDW_Rtzs2P;S?ip(f=ZpcO6M;icla(5V-3!}+n4+cz8fNcSH#%fI;4G{ zwP;u_mQJ=v5cLzej9ZHa0)ESYDzI&#xn{OVKZ(H_-21}UkF}B(;Ifja!0aL#olWQ? zG;_j2Ldd7OddxC-P|?emUer6uq@Zpjf}RqBx4h!~nvxCaIQ|H*Vfatjbzv70W;u=a zlO`o?TGcK+MZ@KA-m__#ZNd6-3wZfsx%&VkdfR~wK4Qc|;6xcrNxQ`pC;ivd%avn&SC~yf#dxDvm z2B5)Ga49YAjc27Bt+EVS1msM9fqsiO_<8SLPbgm5H@NSm$pGCaYuL3sj^~0_K-yBe zn_UelNa>-LTK?Exza_MezZ#ZggF?b#Obj9nqG8i+yY6%@4+miuAzrC1T2E9X61MN; zblB;}y&J{j6ZXt}4{o zJto=Gch9CGH_9!{PS{H=pe6tyct~*Wt;u{rr1;=px0{4cNcX(x!|e|V|MicS3zh%lKR`zBEE`>($FUlekuFAHPrmN0!wWND4uE^ViacD`0TQa zTth;M=X}eVC^&tf>smCucvvlG=hDRkWS-D%V=z*vi$M!dS(hs-iP(f2)j4JSM7#%A zL@blK4N^t0v+D<~ttCkb?IWeCvx75CGrL*-m}$QurW73H3X_&ZFe+wSV2DnJ>EgPSziIj!Ip-8GB(DouV6ms9ZlwHh2>DKN~B<3 zBS((xz4yRYi~Z>(o3dA?Hg^lCitVRF*2%1Z9_Af+{nd&dHF|rdgMl$d7>Zf{lh?>C zJdwebWWkghF!yk@7rR>^F$A80+UiLd!G<1_Utr%Evv~0zBvtE%)5d;c)+JhO?b;+o zv>357pIqhthpzX4%lYsA$0HJ=Wt0XbEup0)p%PJ{rLx-6Fe1`KDzr-~qe!A*G^wa4 z?Ii6bBvF!(DCz$^S3c|h{{9{wpWAITuIv4JopYY+V1gC(m<2p{{%=4BCMzYNqA5fJ zf^`RS*P1{JkMNU)9W@=~>e~_DG_d`PWMnwem?Ko>Z_!wt{~}g^H*irxs&Sez1!_X6 zkHrJJFOhOEfzD{76%v%Q+|5C!E&f9?W-@PDU2^yYl)^-^LFpDZ*w#()M|RGhcK;6d z6TZmv?9RXPSmd>P8U1HMav!f&U`SnbC2X^wu@m79bOqtssjE~}C`y=G&|WuWdPUhdr2pk#!Fl$61XQo4oGriUQZBc&J6 z()@!8Zx99o{l^(qIJh^B-s*-k?*ihHZu)8uw*5)X5M2i&_hkd+Lz*ago+Al_@6K-n< zjt-~qf=q3dB3g1Bsg3qX`vR%vt8@0fCtAK@G^&K!+d*-Jr%f~&NkyPo(k~XfA${Rj z-eRT$O_k7pYaz|1Asvt;-n`&a73+7n#_|8kdwOu|Q&Jte^NMIkXK>uFael#yrI``u zMOkF#(q*BJ@1Ws8o}RDL9Yh@O*wL1%G)?{z9Zo4h1tv+qEx!xcv33gu zL&;g#Ys4u0Io5i+UQtzN*(UF%3R z2x1Df?J=aGa}FgT=C`3CAt*hZA8|2}O&TSLbYw7Z29Vl$@b}7daT@S0qOwznA1Dtv z-;r;tt4j?9r5YHIBL8+bvCs`^%J@Ea$X==z$GTq#$N;FW8QBRy7~?zM1tBS^z^01~ zpIdDzXcgQJl&``LWi^cdNU=pvjrj4usCfbWf`o{F+sHM)9Y=>on?-T;n=T_&5FW_A z)t4pz95%=Q9JUeX?7W$m|EX(dL>7!X%9T&%?Fvo7bkYahu+`*U=kFSS*oB#fc5yI__2Mky#Nm(1`@_@o4NDnf0eWJL&O2enem3) zUeJ_(y}=$k8A@u@jCmn$#vwY>1(X6?%D$MC%WnshgpY;Tp-)h@K)QtQamu=TX2$5s z@fIW91Zx#}F+du}t)^UuL4U6vthnTL@RWADvM!@;w27JNz?uL6W2Ak-0S2BDZU#_@ z=$Ny1q8ON99*s?yH-A3aupzc{M1msLaEK_YF*DI7B@o}JYNF5nfStF&Dam7Kz@<)8gAI}_Op|ta!B|sTZ5Jl>OOIUhCm7%>5pn+)jie&*kJ&1LQj0h}@Zm;*o zU9rbbO3MFe3d;mLez;PHXRyqn|o{2E!dM z6SYcNS!m_TxnP{PCK<+!Jf%IA*Ul>HsKE8nRM2_PyI z*QO;Jb+G7#69Knp!Z`cNm*NEu0+Lk(B#Z)kGVQ{ezZF+jZvN5^rit}bq04kmgy(9= zf@r#86mh4{|1bLQSNm0VH+%|WIC0j90F)r)_EQ_v2|#rS^t9H|%xFN5`5Ar>?PV$| z3#hg&5x%Gd2`3Pp6e|PD*T&nP)3L!|Kjityu=zs+0^l-tSzUM2)y)kx{!B>HgYN|x z(}B)w96NSD;S;3y0917HM%jT^{)P_mA4>ksnCe0<3>xIF)6`s>WT>}o)1MQ9Mg>!x zIC3|+aY^DVI(-+!69vU)xIgM1E|5Qhm@|&oam>*jM^*{^2u7|tIxU61cjF9uUBtlQ zckm+o{Ud(R;J;=cQQ7Yy<{k!*xHxNFO_`!cHgLQe87$dCWeEVvA6Pc+B`Q3XZ>n)v zd?%)S`xkgELc_aCL#!a&OYQ0IevFVtC}HC~W}%1&2C`933YP;OSCT#fyq4ij6Qc!= zdpw8cQqC5+$QjGv0AtEo>VJg~4LIAn7scYRIN(>Xfn5tM-*~gmf`rv24{&h)a< zN}%DR9a-pOF4RMU9sruxJ+IhG@{^?{2;MU*KvzQ^;!39ry4&V|G932AE119GTP0_}v zaFpIIuGn&+){a7Ku^~w92S2J2(hh~t9R6Y({QkWJl1_^M#rHv$dE-%5rsvpiux9)h z_dh8P;Q1tuKSP8mo>r)JZQBk6pZTaq7-yB$fp22NXOo+CuTch($af`O3Ry#6DF%1KPyRO( z&pF{I$JnQ{P5sYllz~zUXu-eTt|@)HM4y1Jz?IGj7@G~?4ZlF|t>Dhi_-7aS!QVrl z1Q?+N8$%nMeIB2{An@!=&X1A+0y}l{Yp(ozU&@)14gYlq#oS6d;K5yh!x^=2q>3*X zy5fUaS3=4HfNGYH!1z;r$L^xnrnRXMmGFQjU5PB)CDrxF#bZXN3WxHqh`Y2Q2ZXS8 z*JVWN;Q129#spc*EoBoJr^xX<0G#kwqoJz*#5WbZh&d`b+4Qi8Nvd%ij%=7_{3E(6 zq;Q=R_v2TL8u57i=bG_pP)n9rLmR)1B*eJHm(Y|)a4D^;ah$xC=>)}34{Sud7$li{ zpZ45bh6=}E-N82*S0C8th}kZD5Y#(5&45L$81v58YmG~ND8iW)>8kZE{k9J!go3&f zwuK{F5tkqGgZzMfnCSA1Dv!E+>&7Cn&wz@wodEq&GL{7Aqlhb4f?(a%UWSGc9&H4^ z>uP$##e@wpDSqn)f}M14BA}-f1%+Z!%VqSKNK6HxhO3U_0?1~7T&x7!Al^vgV!Ve` zk&tv74Pz2zDL`+>d-ymKX(4{A9BL9793J-HeV5v3%9zHAoKTs$K_slgYy-{g8rLc@ zn1>;GhhqdWQ?Qal^P)!)DthmS%8pXfC#VbHQ68U_(MwX`3H0S4D@#3saX&y62+Kqu zEC;F{KMq#;EhSAUC`^>G0Ce1tg_M6^kHG}|lIZUs&ahp5@85?89@q0faldC%TWhNb ztraD~nj(bDAYU0~#2LQoNY>6 zHKBF)N28vIjuu!68qj$QyB;Y#`&*cX(a#4798Q-qFu$V^W{d<xz6a6um_Qw-U`*+CtsK61wKzPd{R)yp2L0W$&7>TKYzdG)9iAJ}w*(4r- zy}6Hc!$UkUYsY8@?r>!^9Vv@_-ph>4CI{- zKl_?rpC41X!wRfURt)`XZ-{|5MWC^h9=(v+6%vX`M5J78W&3K zJy!jTfo+l#8$)P>P;J#9I?Zw=X=x5Laal_@XNIp9gZm!3PlNYW@J|<&i&do0U#u$fSDNE z1#xglsgt+!0AG+67E3qUKbrcm!?-!;uo!N4>x~NTKw%LP_8QnGBdHs~4&1}u{hE-f zfR&jXQ7-?^nTB_0p|GDcyWEW&{Kx^HTMEnYV~E(~7gV3#b17d3cpy+(py59r+e!d^ zr`#_TO|{X>zCf(gZ7K=igeIg5bfZyuS5yw6_y-rJYfO@g1Qw;G-<`Ww)G$9vxg?kT<~I z@WlHlUC5)Ci;uzKP{pVxQoR>7hW3&>wq|OVSR=2BhDMN-f33}V0tYP08&atL-x0Z2 zBFvR@UlB;s zka=;|jFP)>3@l#UIJzIlK^>Wc_hs_8lCwN-78ArBkK=ko+>qIYUTI0z-TaXmc(l+M zy>hC{#_PV>9$9rX?Dbp`0HCJiIg;5hQr*^esk)aHZL9a#(0r)3Q!n80vcTRD(wYnL z21Ow+4U-S!%4vj-43q@Pk)s_wAh&`pnqHOYYTJ}h|$}JHyl-;SaW&#Z?!$WM^u1lS|E4k*Dr=2D4r4o;ZH6? zASQlNCgyB*_O`dH;T-@U%8;+XYQDRRA(@uffB}Yf17Ums-+oI*dzQRDx0H&-jU-=e59>cHd$^pwB?#oCM z>^UR<8LjI<&Y7-O`@SXMJxFeS{A(h^`|#{C^+!-+sErKaMO?pb-6Y@-#jTK)bIzSR z6=}oTm18$hkPrRI$qd)gyaXc}q&S)zHdIDGBaa_IOP2C+JYcbua(wiBMtJI(6Zaz$u24PNH3rTc7cd0--)=x`LB1di z_}aaHAYt@l)|<5IsMG)!_W^2{VguHPl2V$#H9n)$1jSZ_d}kHLHBHUayL4PT@rZ)Y z3;ek%2{vdX|Dj?3GVDAfCpVXR(D1PkYT`vi%A(JyS);hpS_{A=778JrPV3P;k%9hYy&=fGj-E(CpQ`8(41zzhx=z)X+cvNaL;CmoR z(QCZbV3dLbh8foSUp_UoTM>|07_^@l7Y{ztli!St_l+eHgDslhZ;}}uCfP8AJshK+ zzBCUrX*W_-ppDdovnnAQa(clvHzbZxV>vXWZT}axiG-~ZjLq24f#xo%9x49!C^SPq z>=7T7W4m?<{84gZV>OhVkO*P4L*+*h#fJXYA%$q=EqxYqcmL02lgwm-c?-es6t zo^Dw)M*KHESq?#1IZzrNYue|#gHbFkq*h-(N%cM|Jny;`as+G0n89^m*7c+nEEl&>0o9RT{fRncLlUR5PpT# zy9P!Bl*Pge;UwMf*)v=25}$Wx+Y38+hMJzUB@aH9%_Vr#QG}6LYf3ROuD^ffB+P?T z6irkN7pHDj-dDrxA?&p15r4Z!*P8t5LIBR~l{wjPli zSx|#ZLbkE#&Yo!}khwxR3bSW-~n!VL^DF$aAk>vuxs6 zcve8&3s43>0m=U8wAyv}*Nq9?=ETD1r`{|70XRR_U`8KA7YBx{%-$5+40Br1g#E2Xy%z2)(cjFf>eM0T#X1A zqy*!-h`2bmc^SA^2E4f|PxSAQOz&Tuctmi@l%N_VToevoODBvEhynfqk`N2#WGCg2 z8i1bp_F`{D1`u=t4Vo{lae}1Z;HI8-FFLm(oN0DSc9spl1kVZ7qEM0*lj9aWq7YUn zyrmTi_p`IX{GQ=v@}eAFM2hJKYTJLS+4rp`Of8ibdNuY9)qOhbp6h^c9QfL%MWI%9 zLHq4!tf+@!&UltCHy4++j(kQ#ef`zUOhf(68#k_9yY__6Sym=eIN>7Ewg*N*U13wh z(f9B(GEy=@4UlPapL^ihL|XoUW`bg#lF$zlR!h6@mMq?J28-HJOi8u-XJa2JvUL=) zaW7HyA-)1Qs-c>}hb^_bCu@eqrszG9L@vfqr~t;q0bE7%8|eN)gbK(+LELQK-1D!^I3q$z+06dkw0g@`k6W!&~mVZzmwsK%Z0r@Z6^fYIp*sbBqN47YWc44dMM=5kWdpFCzKa?=~4i^vD(!g z9RjqjBu#ikd>sdzT-zYY1wKrPx-IYftbkdA*DD6@AJ=hv3&^BH!0Mu_U$}oDB|Rpj zi#@uic^POEu?GqPYwS?KU|k7Ru9})tNU#Yz?y7Y|XOa?BMx-VY3%H;HmPa3-MUr?plby(rw z(M3FJB$RpAjhTL{ZIP8#5L&P(SD2^WE$f1xG z)zy9V*;9Z-7(@b*;g;#8UN(Z6Z9u_Wx9WX{OP%(n!!3J`)>2(oJmo4^rT(&G%WR55q7JeiLlbEK8mG6Wci{Iyd!5-ztK}GWY<_+z1KISsAZDP* z0qwAfkr)c6J;0Wz70o~nu)HGyzuhD9elmV|m)7AZ5G$;viNf^<>TaUDLE*T~LH)Am~n(Qd-M=%2brv=g^AsdJ8)BLcG zWcUyP3o_DdfH)P=suM%`D_GWfo%Z3unit&H2Ze!f~q8|X6|8(IF) zdISRwPup_(f5n+~NAJv@;12{*fe+#Bre$Z#z5<-agbm_GY3T7sCW;ulXvQ7$AO0Zq ze;j>vpPwAV0c}ut%(t2rYxTEI#rZ?8aLxtK7R8UCs33Xkie1${dZd6w;&EpTg9FfD zkoOh^?{PzeOVwSl|5`x1@jfnY#EJS0i7uvb9k=f*u1z?P$f}G%fH6vu4-ucuriojD zT(daSxUR$GRosy8rLLiYVR4C7ZVL$@X$Q|Q?;If6I06J#S*54lNJyYft&KrsOr5|q zCud!Zx>BWOVPOIEq7ZUeCL{(U6yjc3E7%I9LIg9UFNb$fYwOlf04+uDUAiaugvwX& zE~pBuIh>dRgGspw!pP1SdqrYhUJ2+T99VT@;|nGZh%bP80Lo(dhQW{em}|jd6sJMA z2IP+_A%LB0IKlde=l8)rMxC=|u)mW+vw8)ltD1g^RfZ7KLPfUO9$*kUM=c65ftCd^ zpij|M?r*34LTw`ZYl}9f-nuwEkpf) zDvt0jo#Pi`@I|AZ&!JV6?(U=0;^uR6`=IPWW~6J8%Z{bznaMv#Mj#B!my?!0f$xHN zNK&lfjR#_uC@jjUd#yGiD>ppAOR^tS?Fwk?+9ltX+a(0y4Ba2zL2I3CE1E zO#I85u@~oQaXe0pC}T@nTc74xwWrpyB#|@LwMNjz*$J*R@e+D(#3fyyu)JF-3^6Ri z;6(ww;OCTS(`up}4#YtUjl+)wf{nf4>466KWGdA_1YQ>fpD~Pz0AI)jn_aF&zI!BI z;~4(r#b3Y=1u@pifYJWwS~eiGqG9kZNZh2WjMUm!xP$Y=lOhjw*fUcv@u1MM> ztAvhAboFQyL>1(47^H3Kq5)Vn;0yd8-jl_9l=uKu(TG9m(wi3a{6^k02q@Zr5ct7% z1#SaOE#MQ?|C%K!b5W+HnL;e~cbtQKoMD-00?>u>NMF0y>Px=(wEtRA@`ZO?d7lG9 zk;4xmdEsG#7P5Na|l5)&&;o_+)`1tWh`ctoGH zU5LloXVi&QU)(9y8#U`rGD2~MpgP!}B?Yh?spBUosj#w)H&G~P_Ro*W z-TxmeH8%ZBLX~?xn6|H^v$NCQ8#%(PxQk>HFM5J`q|MMNW4*^9KqR>4w7-Alh~gaB zXwgHQ>8fpuk@o_uj#zDC^J8=tdhEv63k7y!N7_7qOew7Z_G(f1kTD8Q@R@xMN+wd5 zfZ0;64`5LR8L^A&v=n29;dWXwO?M&KedD-8WT3in?SrGME3GLAHv^8&k@J4AIqYASwkH$pXvbABy9|D}Bl z00Vj=F}uRuJb&WgJ$U)ASoZW~M{WO=wv;Xuu;}WQ`T3yPlfJK$lRCxJJ>?QVc+!c4 zpBb>mgxf$-aRM*4rRAH`FRgOx?BpqEleC@M8pwARrACsjY+&{cP0C=!i~MF`wE>p{ zKrVPi1?fE$8I1x8@|a{`cfH4EqGPli!vl;i`e>&$*p5w`o}vEc zt*hP*q-O&lzl1at+7NoFZW}K}`J>qGKU=_RDJ2*62A4XZH(@K4aLlk%!08naqQ8ME zLG|6GR?9uP z5*O_Z+>2rL_$tUXqEL`2hq;4*6{DhDkQC?Qre;JV{RJ!~?MqIv!4*l4AJF%7^2p`{ z*gRZYW@3e(>#c?_?cag9`{H#$s%xoz47?yX8adw5LAdv57czf=>mXvPnA;b|PNtwm zWSN7SR(|J|)`wk-x_1AhMa9^d%%NAZa&xBo zeesI*-%yj&^YZx3j-2kx!{!y|C7?H0XKKoe0nKFC^^*k;3r1gDcfJs0qF}*1v93q* zbl{;!cb~>wy5u0iC2IzGz`Y6}KWKuU>&siqSJu?{n@HI~w|N7KcIqh8OT;qDF_37M0o=YPNAlR2x@U@>4h>T@{CR3ReO$%|r~kyS zU2XKA-w73}f3d{h@nDV}ep_`B@Ac+!GoJUVpR<+tX9FmignJ$ejw@Lo0Xv6ox)F`f z#k}U3ovpR?=?#L%r+}?z1}k`e2-yducmp^vSwNtn$*s3O35SY&A?eEzaSCI?z06Gh zlVt-$78=Ljq z0&MyN<+<>CMn-Vp27Fvwi(Y(gMK=uaXbWzNq`_e%C+E(`lm450g}NQQpi z2eN^qlZ6q?RRp_?sSE$?X@ufKsfQ@ zWHLuPLWX%=TczMbO8g*JUf+!K(r?7(3Sm4ysAY>61n7ts0ckJ{&ifRz+Fb#0ggT|E z*DnJE!&V%ooSn1R9_p7A6PpI$*(})c7d%RusCmF(uX%L1Uy`w6fxeX$i0^ICZA$=^rE)>YIoY2TnU3j^VZCLX6Md?j+RQZ5ita( zjqxLd1>t4V@VqaI?=~0+5D?*Yi#Vuf*H1dKwM`n@9ej9v<+vPVl1Izgv(Oz}Rz9qC z1FASug6I&|#?|=WzP;>+NB);_^jjL=dH$TW__HQbT4)tEbsDafQ9zIa7i#m3Lra-8 z=|Eb)on@16{?DPLUooBMjQZC(RcGZ@pJiz}9)v$)%V9}#EMwuvqd+;5FxWzgV7V;d z?Aen#qI|;o;JzcI?eJgb~4ut9z?_(pyKkKZj;y+F8vf$?VAT0hf?9 zXKrma9ATi!m~iwZE6I5a41eU&B_?{X0A1k9ve{*6DU3?HLF%|SSU9MPXdPXo4O)7N zrvUz`B{J`eIk9ZO_hBZm+2!V@0$X4vPs5L5X*L2*QF9zFR~6xkfhEx=BBG9UL(>|; zkZJDj?v2Cws8qm%%mVE{QZf`8j6=hpb_F?o{P>YX;mEKLis?Lz@vZUaF>FBD#J6L| zj?`Uw^C+lVPmhG=cxDV+jka$mW9b9LR03C!Is8rs{K%}mKTd}VHM%b}-@pIPm9=HJ zYQO##XHwa}$R6=GxA+$$tI|%4NA?}Q{iOE8@a_ru7(aKWYret=F;k%ToY%g zMEI=cDJXPNF!2-XXw=2$#OLH3FB2>Rq2!^q=jUDrhnZ?|>>3(HLhfvT4jykHo)M@x z=U_>eWTn)yWk}KX+oo*mAb;KCF)F2i1^}b~9LbpXN3sN%@MZmTbY)}ZkGwdsSe>;N zW<54RjR&h|`}@GQtqzS81a7H}t{DXgrfUwhH}9=%Y<#I~5?y8@W!$*e*7j6smjzle z$+N{jJ=AQgFZ#lAC*DVyCU}o^VpWe9TNW(Dm4EB@Z5v%xQWArXrj;vJZf_=`y+aTs zFQj#+bQ(QVonhNo9sQVb@7}%VyAN_VL#znk<|Kr0HXZ14ki(#Czvck)_n;RSf^aw? zHDaR(5v#FYBm1JGqfs|yUjV!?;r`r_@5$$!!@qO+_JD0c0=O9=&wwt$Km86?5F;xH z52NF&L042bgA3DESyXA(kFI<(=xTP(+7XPQMP&cSkC}7Io9Qx=X1gxchX{ruAJ4{j z2wg^-(wJ}#n%HaQxhwI2+D2AaRY|+_8ZKKemk+!bn-&6%6p=2G9=46!L;Ao-=U&&NkMmW7^f2`z9*0r zUX{WSPkvh%P8K$n+VHTTpT*pK(ooXK^b_JKfO5Xbx+^0Jniqszg)?X(;MV3=R@a~2 zDv5#y6!;y)eVh87xw*MXh=39P`pB`Dc#N@Fp~AVwAy#wswXJ6m1KxNnRS|0=Ahmje z<=Tb?f@_Oyx^U=N@bqgL1-p<`Qsy&=)4c!D_d4(9*H6HgPU5;3brEnbuJj~MQsN7vuwSl zXgut0kzX(4o~T1#D~h*%8tyss26)9Rp)^F{YS1^dR)Tq+QamP~WWrCN!ainkmE9!M z#=*pXSH@!YBck7n#Ys?%wHcc{F^Z#YP6hk8wv(G0emytKDaksf-@X?96}~= zp^garIVShYtYv}EtQDU_hZ11B`ZW$nC{xqy+0_wdGtt$iuCcPRQrYn5p9Idpg9oDU zOhABv`*P59A+*8+-`mXEAgA+mCNr&Wfkye~6yjIhF?X_=(P#hbH%lT|WM(M|pXPAl z37!;(77S+rN=ahEOe?`-3q_QJm{h(K!Pe5&W(A6~Ib$!ogJmNQU?J`e{B6lT^j<=% zR`CgkZ8RkSuqW1izoFRPo`XuX1q7-OlZY+G*op>WQ!{w;Xs^3SJfz54=r4^;n~>)U zJBv9E>-CXGymuj?NJAJjznX`MX;XW9AgnA|owsYT0FfUQcIxyjGE!mBg6j8XtLF_j z?uc_gJuhKEzDyfa5w2^9hk(D7ssQ@hdl?yw=K2ocP0>U?Cfo!7l_e?x1DgffI&uU& zO_dEPtFqOeG-6US+Pp4CRuLf%BRrVmV7%tqFD#a zGU%#XI=>aMC!Am)Tpn}=N%DY^pI-Xpo_pQ2qBaSh1&XjofMZ22UJ2lp>^tIh z!Zw~pBO!$FHk%kcQB-h&cRoMxjOXo#57Y6OG)`$n>xc{N+Pv!v7D7^W5R(Gchh4C< z`#??r<<0>eksQ3DPvKjP?1okk&!9@D{&kQaxRZg}*Eg56xAUV9Dh=e7EBPoBi}5X` z9Yo21imD(xTDA4)9IzUKz$!evw=y9Rr#Ku+aRjeH>wCKgS%t0>BDzWT*x|#6m#x>( zq74GZrnvY)f0~{13pB>bT)p~=QBP0LP3Mr)p(n8!O#u*?i3_fPdPV=hXvilq?_~7i z%9ZvUIfAoWHpV@}7lYKj3Yp);*6w>Dg5Jb$6QXdYOg{PLZhpSk^R{UDeO%OjeEKvS zcPXkVyGDYQ0aC^AP#g6Ea|%H}SGRhOI%~`;Jfjps2jnvH_rgb}W@f_T;;fj)q!knd zud)6a1+UL7l7>Cx)K+opF+irjSnA| zgG;2O!sl2B3_=Kw(Dr;F$e21g`gww>e*WtqmT9h!+56|8PW%C;PRTHbRhW8XLRT(WjM#TGdOFG7kgxbk&n-5qxi<6U5zHA@DMrLntU2yIS$ zgRJffG&m|mh-1Q9Ja2f6o;Ccdd)V*;eX3;NVP|J^2ZsQ>EDkcc)8<1!jlK;FEG7GZ zeNzk(E@n-PnmeJ${hzV*`cd^wsOgAG!N;D2@>DT`CyMMyXt4R3Mm1klPYmel*S4BQ z#l^AXEm@|uGP}8@B`tp(w|nbMTejDOuBpywg3-#Zax2h!v_~U88CA?OE!t@BfdhAM z*p&#Pg@Iwg_)17fNNYB_J+wyfBZs9ghprGqoP3J`JEWxMy5|#b-t?!7UQ6C<7nTCz z+@|JY5`1ADXGZDrh_guV#qbXG;1saGczRe;GBUWKLs+k&!JP6E%6mj(oy0SGL3nX1 zAPIl`3w6oJ^j_>hAB2{o2ooM1+kkWDECn`W9X&#hkDJ`7o-t)I^7PnHC?qQ#ot$Xr zhzdJT5ePSWb`c4@v=2uHDD2ebQ7KQ$N|xh00`@M1ub=CzrV~7-3#t6!x?IfPF?QI+ z#U>J4`EWVB{`fKV#u=?Rv?*9S2Mq9v%iViosI!!t2i1?&?_cHZ?U#+9tr7f;H#!$g z)DsB2;DkDXi-RDTO!P72@W95^v#_+1A>QjSE3dDc_xRAaZ|~iTr!;-d7rgo)Tc*)_ zxa|4T6)U#q%$~==arKIMS#*gs8xD@C3%Z(U87sdF!ou7^8%4;gVB&)70JL&KzJ+xa z=UGi^mGJpZhR|6<@`ft77jfVxIt2UYZ2)Nekly^I=K1r3Ji-EPp9DfE zA>5_~azLU2mT9hM6|$BOt?VWe6x}(ry={AU%=Tx7juqFS)t9&BN7U9a4ouk;3L{{^^0ACZ?v_;a3P$~XaDDfYM9>rpA)Jn@N&A4<|;q;ao8_r00%<*m(hz@g&CM3@xTT5 zi6kMQ01H`GlaV4h1nm@3F43Hu4gn?7#5aaFH`0YV8#PZA9UUDlk$0om`E?qL$X@3S zA9sDAp z*}%|nN|Ir+ROcmg^#~+-0&I||6b@qrp?ULcpx2Ul0a%G!l6*<|aAL*pP}K)8Q(<#& zkP($amF+LM08s$y!uqu!qXLX<@D|!hNXnXUot*-HW$RC;?yos7g@1kUtW$Vygmv5Y{9Lk5>BQS(9*O;8|*P zd4>xTVE_&?$+bv|6hODWZque2*0M3J3ZQ$@MwjCAmsWda9E-O_&BlebP9V=YXQ$2; zmAM9qI_n_#16B)<$gLN_7CflWls^L@vlecLS~~-* zJ6UQ6Bor8nvpD9W4Rs=t8>wR;EiiGl?%jdgXbKC9F!-={@7^t*BQ(*fYPM+#bDmbe z_T?_4vE6Fp(dC2L3nT5`G1DD8G`2@2n*HfX13`oA0(fcs!GDJM7Z*@A6hyys`LVY? ze0#6*j8_09s2|Qnb4Vj5$m!sfXL+M4pW-%oKYDiwW({bXEE}+O7QOgR zz>klC{r~8W+OyTntRne-W@ho?5Y@?`tCp?T%JjA1xxB_(5HcLeN|4{Qu@fzrgOIFj zD`vj*V2P)eGSn!dHBtMVjUZ;yBLhWCmXec~FNR#fv?;f|*>>+8sbdpedwJ1n4B= z&iHV`-Mb->bwgGgn0NS?%Wwsz2@3~@Rqveu9D?q`e!5w>)d;JF^TcF$;Xsc2@N(3i znRt`I!&(7VJFj$+W0}ML{kDHPvZ zUU^(=xO|MS1O?5)GnaVdOqpf`I*N}U9jTFFPuvoB=f)YnpEyik8&q74S&kPIN@@M=M`SQQxr|=+=Rcr!!OX>P7>m85cb+H4EV6l2y%b z;hnY4eiqZBgG1JiUK_@yI3vOZFlXqrI zDD5nKZn$t>gwCt?4$Rz@gDX$AZQK`8@mRX*p4e>m88h&_&DM)mP0~qjt)1lY-IQgw z`)wX|p=r9hmS>*IcB|aqzuNBvQ+{i|_6_H^Z|jBRGUmCs2ymQDkY3r~|L76tWaj=? zANDC*r_G~z+ zq>ly@SeZ*bcH0bB5erxYE?9og4) zz!S{(DIe)Mr%7fA+K`j7AC$_yw6t1%P3|3#Ey74`;sU&npljU)idxcaK49EaJno`_ z`~3cTi2YFRjM~|7siB_%M6lha1+91*h=PMTJD1@CT#PFmBB?Y{mjZ-YLtbp|=;*Ph z&6~R~p&0K#|QcXHJ6eE%fbnJ{v{U(KbPI+Mwsx!W~kof#wfd z3=1b+tE7n@f_F=Tz63zcMRR7DdSliDTZK-J2<1OhK0=*lw^%VQ!O*m`ML$h`jxY2w z@xv~jeKHH;^{n5nD=v*m zKRIgnw()wF-jr<-Jh?0?(pf7fcUJQsjlJ-^R$%8s-fbB?4ZcCQmhM}(As~K0u10gA zv0mcA?-`S7484_NOYYrT?x=bDD(^BE;Uig_uO3L9HglC#^Kr8$B~L%*XScMp`ks5} zRv(#tHRJY7wj#PQ8=u)8ylg*9L(b>7Vmu4DjTH{-t2?yho6aoS-*-xI%Y46gEYWYB zrn_(1e9(nuc=v31F=^gBGyC=8V{2*T|W_2weZm}7MUab|>H*nax>vq|8&jTi}H?Nrb zrf$o8JH@rK%YXeIPW$oAV7+($uDlCxl5QqCC=34$y#%g29~1g7Bt~}8p+h&$1T1A{ zGBNr5`Ag;bt>L$}o3)-=AK(SaUQp|7o92#OBcHiTgX42^ceP#PaNZ+1vp0-yN3vAN z+qL)4CM4dPzTK-z>yRJEESqJ|rTyc=-LDEfxj*{kz{$SQrw9ADs_Fe+9b=@}{5VKp zndSMO#_PiO^?n{O2pNlW4;6_0Y3%Of)6Mo=$jUsSZS|}7{r4k2KU>wr@@Scr)#}Ww zM+t|H*y~;W)Me${6USco zwPsp-6O%-YPVkBXozgqIOv|cL8=9Zt&ab?X(=|E}_2l`zfrL>gJ~eQ1!z1$67!c7p zo4*hMmNf7+n=$Mo`~1@VzG)vE)m;ek_;fh^9G_zH=f~5U4!zUiOjyA3wfFtOdyX-y zcDp;xeo{0V_f10a{%oZD=!z^ciobU+t#iOq^ECHvHwg~l707)zXoRuk4R$j?9$tD? zO#H`}${nt+eK)7(-hVmcY1;w+;k^A?ejNVnouFQyZq&VfQYZ1}^r}ler=#}FF7})C zazR_w#g#MB2EvJ9(1_e`A!Kr4m+6AS4BiK?N0tYR;p`HN154cP)Rfz|>0-n-qxh#~ zBRySRw2$~TM2HnC0Aa8ckF?9uGBVV8KR`DK68E0yz%p_1!iUG@w|XwZ@N8jaMI&-5 zf|&gRX3lEv-#kz(@ks3FV#i+5h(zt?(d`BXMUsg9agYd4yZP|pPMh#spyt2jy-d)K zAo8Yha33fQNM7Xji>WR=gLW+PY$#ghEbb~L(a24l+}uUUa!_ml;>(&-TTI$1f~rJ4 z-W^n;8i{!!3wO1+p9L|C3|REIs-WQUx11(sX7&=Y@yjr7gP+zkHI1oAhq2Ne@@o0C zW!pHB&5j*E-2*2zh1Ws9iUKeD-U);WwVaR$8JU)JJEuW{htBwU&%w(V0Y<=i%AT|H z(A{;S5d`l;S_P5Bk?_iHg20h%rlL~cIqZybg9)LeAtxQyzLa*<8d$lMRK9zU?^A^q zhW;*?-`2KuFpw*OjZVq#(rh7W$(LcB=`(3_RT zyGBic(6-SX_*8dlzMKx-!5FgMoa z`>Ybb+xxc%PIgZ&%G;h2gwU4kz;#{upH5zNO^dv5}gZs5q10 z%FcxAQh9p)&ar0&ZKb{zzUX9m^1RA?tIWn?eZ#r(;?6UA8m+cVXY%n#adFZLVcy&Aj{Q!igZhNAGw3nVInB`>0><{Wn;_%r$?z-?5iD zEKCPoR$f2pp?Te@O|R?O6)l^X%5L))B37SWJE9VFb-rbBwubiNQx*Af-@5ZjukiNG zQs1;Jp$Dr~()xNo*ezC7XGuuDIw>muL+l)@+QV{LJk>vpVz!6W8~8HsU$HUetZ}~u?b>&W2ifmjySDQy)5F?$U19%P)>$HS0a(k< zj*ov9fBW{ROwLXgW8++DMa4;1d45@&7ye%M!0XYBzNUp6j3P&-|8|oUw=3MdVpmDX zns+#{v+mtr_(QWjCML`EV_HU8*{%NjP6iP?)7am2eXW^acrUJoA21~rfx*pJzG*E8 zVqeMuJ}VY`A}Gjm?cK9yr$e^EbflEh>JgJccKH79oyU75UwRHNc>5(k!l>LDDkq62 zEg9>7e2v@NRm8q{#4+xIXfY;7&+HEpLf81GPwzI#s$fppo)@2-e0qqV)=6Q(!faQj zW2UAve}9Qu%Ix(k_ND7$PVdnjUrw~`$tY}7U2%Ha8&&~kgovd51y3^D_e|_-Ry<*R#j}< zkpEDmMD+%on`9+4B3OWZ0Kw_uRO(}RcUZbx^bYX31iP8lTv5XpKi0P4j zkqW-#E_(OK0V>^g-bqt2J)+XuIoa&-KL@CgfW&xVpSkc(LGxvN1s<&I1D$#WN!(k} zW@GU22R>`O8VC@)SkhJ{Y`}H2R*`mabTs3RVh2{zfb8+feFr1oDO4#02d(e7B1$tt zH0YVgS!+8|GL@G4yWGKstzd{af!(M~pE4OLfqQq1h9DUC0}zW#=~(-c`zsX``~Z;j z_U1;cM4}=@KHCDe%~}Jv4s!sq32t?4nS5?kTh6ws5XBLyDw*_2_F4MBDxO{0N^(1eFq6)^-m#N#fwRcC|bP> z=>9JDMq#{N0E8!?nxH13(I-7$iGGB!_(5*&i`8R7B2ksJG$OKd638+DCKH;yKuB6Q zQgW-Uttb|DkMvjG0eb~6-C`xB=-}HEj03L^N()y0omLmZ&k3An+xG3#VssQlLV$2G zPf#4)1yI2PeLYeWKrQX2zY2%xPtPeQFr?Hbyf2ahvJE%iN4Y)0k&z7#R^xr{V|)1H z&615G&OuHhT*`WJ%c1L9^5a{bxa#oELgqDhbVN%J#I9z~$sB0+yS~oAh|lZzQqJ`f z4P)8;y`jP@^4Vq{eRd+|c37)||H}ubzP%UDs&un@%Ql0na$BfLa0}nVw0AW^&ujT& z+GClwo!9p*8W6db7Js`l+iB>^#x*-m58d@2YWx1C>BECxPAt)0&U-3s&NcNM)mIii zs#BUif91lWGi^&;y=&KPe9D~n?nUhj-{MEg8&?~h2|D2S_`u_0{)~hD=7JUJBQ;mt z3iN)h=Dp$+ZqerD?K<|>Dr%Fp($8V*cGaUdYmQ!V;@Y`p=ZW?Q`i|opMA|2h)O6K0 zK0i4?%Xi~NZ}-n!**!D=NYWIsEib1sRb4Jetj5d50tmTmdx-8-5%tJ%=H`0F*eu* zD|WE2@+)1~51u>o{>j7RT<ua(sq-AfN; zoG#7^3>wI}iV<82SmeJ{A0mgdfse)Hzk z3oB??pDHEAhPVOWQ>R#V7VP%T%=}dIa_t{x3-S!uzqdCAXbJUC0cUzx9x+CyBwoHW~3{THX++f91$VvhC!UKW~& z3Y$&X4?N^lx%#93V;~3D8>WQ|H!H2W^rTpGp=BDIjm_NNTEZ0+d?_NFLBRvs+b9T)~;f#ygX;`ax1X!ZOz7>gu9G^7Xj#c!AupM@Bk5 za7~*sIM`irR7`J<<NjrO$LV=IEeItiX!D%dHGpU zv-)scuI-O1_3B>%tvdeo9ds|UCItI;N-U3}oxO!EEvB2daXKa4x>ZO5mzOV}B`Zox z7eaY}e{$_cS(yox9w(tL?#|i_#Ef=_z~Vl3XDO6N@W$c)-`XXJxJj_7>RS$2?((^O zi}Rb8q+cSj=hI0~8Etz@f1#K*QD}GAn&2Y7~;Yp$G7Q zU<@g6lum-$8q`ml}ju%#@sB#LY?t z7l7ddz;+;UA>h^;l~MQOgnX#e6XMeTDH@q~VBW4kDAxA?iVa`%#Dsb+rG*~-VSJ%SoYA5yY0=&~@W7x1>Z9P)VQ(`N-DLk9O z#9QOm@!NrQhA1Nbzv?f6(gFnbdSKx0i>OO~!2J71NN*-q~F^4E<2J|_91PRo1u z&5{aZj|@2$U84@GjOrg+pQj%?b^8)8TQ%GB2BGyD$Cp+d?%R67Xl738fSU3yfsYQ8 zSVY|p4)qk3*RUWm<9S0@d{VB!)U(~U%2Kvx=N9O{n?OMIRVJbQBEZk?%1aDF3w0ltr9 zcv$R`?KRfvPnG;S_bz)8)8HEGb)8%1&y0<|b`%sZzkXVv(cW(B8N+K=HRr|@l|5n7 zPh$hnR+eLU>bvOr;9J4AGT+CQXSc06ce-7`M0Fd z+T1R2Vxnb?wfK+5yOANu?r|6^)W$s~p)UA4j)sLTH;-O3tL3|(W=#*=nSsYTXKgU{ zc@VOam3L3<&!ain8a#4RqA}(_H$RS6>EzAc?|DpWMP*Xa$%5i*Y3ed-GsSN#Xkwqg z=t$+5u5tLGJFGvqS4?8^IQDfC`}w{A-cDA%{KkXP_C5~NoFjc&w0soK?(CA?uz`Ed zJhjycAx&TJ7Z#shXLx?faLJZ;Jw096@zx9@=Mk5@1C{x|zSzhs9xJeI%hr8-zjOJv zQTOA+`?(u$uL}N%Ae!#C^<5EL^goaE33-+n88@DK8sNPSToV&bW**;LCOf|Tc>U0o ziV6jfanE_*hFnIx-xTCV9q}0dJ;uSBJ1rgRaLT=<_yZs}4YMWZp4B8U1AvLip@N}C zBOB8_d#0`V{eXYR{baA-o&O(IZy8qQ`gQRx1PLWn6zNn@KvFstFg6HCE7F1rNJ_UL z9V!CSAkqzjbVxUdba%7pMRVrk-tYgM^I>1x%Ouu%?t9KL#&5W}aqO*kG=wTNN^Y$j zosK)TfYy_4VfRy!DmOEe;&d|vKkL~H0UceYc>G95w5t7LV(+En!`R1<((3)MS?lV4 z#K|u?Itd5~K@u16WIG%kuk}1Yjx-dPO~$pYEy_dF+{YSUvEv*X>7w}^GEdqoHXq!& za-}uTX622_iExSF?eueW&K5N2N&)pXet1S9J;I31;u#Cr*?|>|3^Y=4C{w|zfEg$R z;)SQ+CwP=($toxynj)Y@ipZax{r{XM{fN^91_!VzDS*>t>N|pC;^c(&6&vE=1~+UL zB5Q+;5BW;CfdtUI8pAonhU%%F+ zTOqdH>Hyaoh$@;wx5Hq;1q-K@QN3^fXPbWgJ5>)cK!(k>BC$1>WU?>e;f#O%!20^{ z!T;EM#Qgv&+X{0yM+?ER1*|h;2F7nsfWRPVHS60d-rwF%2t*?vxR`(+%)l=!4C$@a zAP@i(Ka$~r2Y=8CA1tdNIgpl?27)Nya$p4MmHt5*2yz;H^P!_D_6sjS>0m$kTVG{( z0A^k|I$u;eUgL?BdrJi|ARro%&(F_DIE2F{4aUo`IdE23+Qou31nLW!V7Cb=BFYhH zyI_KvwYlLr>EB0oDGM$kQ@`e{xZe)n1w{#VSM)HeLS&D14Gub3Qo-y+3{wke*L4jI zhJ!OqFTcUrqJH2KgJ10- z-6iQI0fq2gX(^#t&(xFZ~scjtru9%+Q-J)f<=d`wh?FesRfLAlg6j0A0oAE@esgC?lq!?FHgnqEkq{?PNa zvWGYRlwEas7CROBBJ9$`^+x#jG`b{?8OYPq`!ovY1zFifYPf1h73%5X?q_x~yZPsP zS{i|QWf%^Mt9aH!kKFgc)6;gR`n4;oj_aaPt{rr*b-(${RbTv-MI=6H)i;qz$`@d{ z(ym1wwHJ>Wf^Ax1zmwiSUcB#iRkojSoOZ~))M0I$`WZ@ZrXEPrRi*L%3#lEKNh}6i zv=C4IlcrClTLs~p2*!o)L75c&m1O=cg)DDgq~`oMzw`>dS2_mUofpT#TF?4>X7snq zt+@@-ef}A_$%!-FA6YKAC4c3mk_>f3b%$h)6^VwB@w&+y_~<72wDF(M_&pw9-qy)E zt}~4}-N!Yf1zhOIHgakQ0Rl7LHC@vdGZcdO>=J$;Fb*8D1^*65|z*Jl1mjEr7H2Maqb=soVD8+n4q47T)pw6lk1@=TJEq zWw&?4{Vjieda+!cA+)szCOd4m;5syM4wm|>$r(L|ltYUceWIrH{#)|7_uiz;bEitl zXjK7?g3(Z0M_rxQbTn=3goxp#ewlUZ)5tn1)VLETbxzLRxG#j9nwm_ahiRNu2h5~2 zG)m4pJw(Vu`>oIa#;=du?;SV(%g6jTX7hkX-+BgQjk>{-vwN%ts$C1qQ+mYUJN%?l zlqAC7Rf0EMD`ue^nk{@ZG}&^-QTs|Wm<_LHOw^VtQ4?1nQ3Ulvz-Ic7gc(Ya@cr|X zWolUC>)^Gm4TS~_ z=#VV&<}AdOyL;m{)BvbUxxPWnt0nZ#;fto)S{xXDhF?+H{NHmKqLTLw4U-G1fDDIl z%b-dq$8roz{eT{ZFc2NeVnbJ4Sfre^$8CQ~)O0N4pzy4jsVXhQxGpo783oGk%l(6e zbH{U>)_;oQ1-b~J48!z&+OtCDsnOCWU}Y4j*B{E=B!C97XoUaSU#|?veIMA@zyZ-` zmI>B@>Z7rnC{#!)6hXXVAmcohd8_&4+_`;0H&;?JPTQ5AZ>7C>J7Y&6{ad-XtWy|uuc4|&cNYtm`d>6dU;D$l^%b(eq>+ge)h?rGvYZ9@-gQ^0b=Is@TDsTQ#RtP%u z>IB*&;t)il@@mJB+eB7PPmV={pMb#%vUsfp@R-tI_Xb@G{PI+*tAkB`pnpLG<6q4` zi7uo`K8l_!ByOgsC(VxW>7n*sF5<9SKM}hbwJREjbJg!7^ON5(?J_iwYH9zg-U?EuB-fnv97d0t}>(D~jmJtmRmUr(|LXSTlsl!+TlaHE~t}Z>S?S6pfa}$o) z{SCeT1AbVM0@PDiUmx_UX306YyUezPvhKo!05tA+GDgPXcaZu*45wAS?$ag4@O>m* z7Kjxsfx;_jaFt|MezEce)S5?+YG8w>GR#Bh2}DT5?`HFMnHCAHH#?vCV(e zcaZgAJ)7v~FY)_wqz}%^|2BB>aCIAF`oEFt2&~Nr2N|4!_c+u$J}pA~3-AwqLH4GG zua78@30>v%B$AW_1${y?gT+r=#0wJWonpi*X|3m1J*62T&0n{Y%c4I~6^7Xz>=L!6 zW&FfgJ=H4F{_B9Vm!4AQfEVp|s&Ij__1eq{q1?{Kp9tAW(T?ISo4|IWOI2e(yZG1d z?fE(z?zn@F`2w6jrn@%k5={0T$-eql70sOJ^J4r&59VDKoPG=ZY48@P>+ZWP^WfcOR6gDGAtmmCr8@3B<@ zA_G?q=)@M}-megfbGU?ZzZ{V`%DwvIG)jbj`L^3t?ObVthHfo&WtO!at>)x0gL9q( z4sT4^!kR6XM%EWHGrJl{<2m1ty_79l!&Y~=Kgv<&HKlEuA%DKNlWEr_yr z%$PX4!t)2YYfm^Qk#}vMV4m48#Byn9?6|}r!rs5*_BmA9FtPi;KSn00kA86DyTPK) zz>*5oP8x7kc|}JH&3Dq9)sWWNJ6v^vKTsR1hTnTPzROG&}OMs!x0TVJG-ht^Yz~VD0zEjb9}uDCFg_ zff~rTwEo8xUFvU`5sJ0@5R7;GYD7^c(wF#b?oT`-nRjNg#~G>XHhs-ZWS)L4@opg* z+$7!=Ki1XnSg~^t(WhZ}z9LL-7E#kYibBjP(QojyP7)*WOAqClGRy=LlGSjuTxc|e zzt%7f(u#)S$;$m!9QWUf5&pqvdw;b5yjUDZ`&yvh!W*np7iYScNo1 z5oETgZ$>`^Y~A%xq-;`xhtKOe>+}a{2jJB&J|*Amr+VX2Zq*nnT^e`n9_{s*gIVTb zs%wv{u8Yo*m@FI}Rg{6#Wd!=1o?~47q8gBkj!8B?%^j`D$mP0UH;QyUTthVl)D6NVMy%>rmdj=y*kN;BTpH)w<1(c!(^|5SH4LEuOY+g<2l z`Igs>A~bm4_<$SM%9GRj{>_hGky9x%=1;+c)gR77C>DMCOdSd?VHXE=yRSL(u;64v z@MZ{ET5e-YfQ^dkLkr@}|5!FsM3KyGpCVhAq?dl|aFQl)*FD$YwfMhw4KNgANX&)`@GN zX(3C_xz;0Dvb$s;9>wx)mVo*O>G&e@o_(f>w#&f$%$V7<2daceIAc@^xpysWi=m3S zH6z5=*Bx>*h;qkp3IFd+!Sx~v0;j&<0fZdW4;VfW$2!DO!FdfJ9m6&LQIxiu@WuOOA7aCe%tdT1UWGKHiFz#|Bmz*{R2Xa5_| zd$2qCkF2aE5H{h~_S$p?Xma3Asi~8)-umO$`0LkuRkOEmKgcnA26uMvL{OxG>lqL=fSJ$Dg?8V6)3w-2y5h zlD`DIM`gd_8#6O+NqWW8-KwS$I@pIm-UASiG)Vl{!Xm+h{5iy4k>DMKNAi~;N0kSD z{xWq1e1;h>c=5oU20@%dlznd~95$#hSb&e_D`ZP!N&bD}f0KcmCTubQXp>bPCxoKC z8q(w$8%qOXM=dhNf`#cnWCn#pU2N~pxIth`lRI z>ccLvBH1!Wc!jj>sm34<97b$xv?=mM59~{rN=7#wo>alXlAmvQ>(1MIva;7r-v)6x z6^o@P{B%iJgVTtSNeUnS4D5bmW4TkF6xOy!9y63z!(RmjxRhTae%^oRgAG%Rw2V+) zqme}^Ommul#Z#wk_vzE)t+}?#5KSvO;(7M02;6M2@t`6kvYA>p3qF7Gh3?DmLR~W_ z#wIID^IO>N>B`v~f0SJo;M&(q93na$Krav`b^ExKC}tQhEi|tMYHV`5EMNfQkk{+~;jCSnS*$WzCqwE#8iltSeaY2j> z>umLWRTk@O#gWdMM>v+mxEP;W+p4txjlkQ#r<|?C=H7>t^f=C>b#C zycmo1!Buw#mnTLI*D=G0>R?)X9l)RvV#Mnv~*}+l)Avn>#}k10GLX@KT@xZi>P(A znMl6P#%KIA4Y;y)ZtD~%=cYxh?mWCEmJ9Nvl$65spPKdFv;ZaK_OCh`UBg}7 zfrOQ1Y2TnbQVtGmXP4$R_nnOW^Pc$0|VdHE>P9h)t$q_D&|^ciV&)aQ^zhjE9EmbktwlCrbfKb!v0X(N7RKu$?1<0+ii85y=9#gtmdUZPx1D1Iv=!=|KU{AD3eSeW{26l>1) zCz1N+&CT2NgzPR}C}`SQ;%{!vkkX9CnQ)5_f_0UBGo8=>?v*nnhKm26?7wsXvTBB! z?rUGd(wS#}6!F!YyNQtIR?`BTieq%N3ess8QIpv}?{DsnOu3i=e*Pkz%RB6HTSK@F zelHG+h`2fKz(NSC^S%-q;st}Svd^E%igr<4ZdIhy)5UufBa*A*u|X_xzE8s`<`q!x z=G7hbM`#_XH|fhJ^I%NPt$OYSX$Q5FQtNXSTmOg_6o5f8|Htf9{8`Kddh$!{U_KM2 z#OYV;grnWp=ICGW)9!R`?e{dJr43_iBt@K056*dgAmU)CR2&Qj6P5SL(Ke_`Y9mJT za^k0_7<~3;)BWf@MkW$Q@hWQ6#{P`!V5O7L@`}htE8C{vyEN3=+UnrRaTa`n>R~H8 z-PXt)wRrMGiSD|(e_EOpYzFd_4Sf3JLR`??;6XlKw~M$;bH_bPJ@p+5D%E(z zk4!xp-|ijL?p>wR$=NGiKQ_88`o2#=P1U9x>v~nihmiqR`p{<8aBCYOkdA(BMR@A1 z47=fN4V<%MJF8w=vjDH#{k}CAuP{$WE1wZP=ya_r>`$k=`;kiBzpb1Hu9qYac5l9~ z`o!^mz)`Qq8jI`hn$wk*Z(mrQkJ^O{5B_@Mbp3y=g!!JbUkf|fWd%La^73J~3Ono( z7paJk`Mp&b>w=hmTJB16EcdM{qq&Q?>vtN=eC?en+Yh2ds?JGjO5t(1#Cq&}g?qMb z`r8?=fAZ|m+7^PV_uj|FxsV)|HMIqdffY`9rY4@V zswDZ?P2~%3I0sGK>?D{f_t(rh88~tfX4yC zeF4DDohty<6}c8Gh;HKS?37KMZ7&4mzf#~pflG=qw@UcpwQH+>FTLBo$>cy6mz?3j z`dEti2t<$d;j#p=AUZ`Aoce~Pr0-rAy&tQ+X`5NnZzD5z-q-W$$4d-S2T>nN56PBS z%0nWMOec7g-J7V&q?zj);9`O>u3(S$#pd6@Z4EH4^z`q?M{go6)o|Yds1hh5U-{Qg zJ5QJ?5$KYMx|SA^MD-_{TVmnlagqSwu!fP{tK3$so_L+tTvT%gD`PMQ3i6hIh-&VQ{PYogg4fdQ;B(VqXrMFwq zmt=t|irlI>AnVuI=%T}WLkbPPx4+q(PPmSNg#aABgbxf2Z$gd;KQb`(H4}^iQ&TPz zZ8v5DR9G9JI{;XX*#+#7TR}v20_GN%Xc0RO&{*ih{=vGn&eG7-WN>o0i*g4<1t5=9 z*6J@p4j-{yQ&az8@nayx1CkjK)mYieLvexaSJRex=Jjdjo~9-h;xYrDV!9TP#JHUH zOu>ofc0b+;Y+v6J6Ga$aL(mtA&64+=WM*Ya1OFV>M|{QkkPB5&xeAomrQzqC^Ya-e z?~niv>}5Z^Mgc|y;q!v;WSBq)fcAiWaM8N>*7Ht;5}4k!~aVD}yk{su6D zh{vUD2>G)PR6%>$4^l#kFbLv*3=OTz=$(Ig`K2lRX^4?9W_H%qdF~0klt%9G*+FBe z0Cz~{;&aI(xP?#g7*=scyQ(->Lj@s%9(2emP7eVc@U_7O7u2P^oLHhS@J0B4(Hd$J zCf{fviVA4~q`@Rp8;CWCSoZf8&ICe^3VhEY`0Wkf;2%GB#2)n4*W)1BcOYvIAHi?< zyYrZWvFE9d4(v31|D`&@iv}hL_oopX3*sO|d2}s!b}b;-g#$Op8JDF#hD}KzZ(U^X zRq`W;Cr)LSNf$V)>jd0V~rWLZoFy)Cw^ zE%E8mdw~hLdJ>7*g#1rqEaUK|7O{P&lc^RV`;NYAs#igIb9aEkW&U7mgR5z8P5G~6 z{@d!cmBnkvT=6qeA~-*#-ii_vGuxvpEkqB#&%Y=kk#NR6!Yte#i3k_xO~cq6?ez*; z(CZkU#5ufjva&l8MP zWR5n*Si4V`o}6Ot?Sp!sm^*>d=%1c8>fa6w8ha0C?I;9SCeFmw)>c3=+!!>d+Fy`Z zsB26)bN+~kotD!$pu(wy|3{_;M#H+D$nxvMETTUR0XOY;`|L-{ZmK*p(*NeO5EtzM zO1@9xRQK*C+@9ROKkOU}qUu}twf44+jP7{DQ-74F4)ri(yf%Onmw4aO#CUi!RiXh~ zadk%LSeG^f_p+TECF^s`t)!I_>+JhC&6(AGJku|CYJg<&b!0cbjuGy}hNn7gEXwV) zlO}gxh%N|D%3EPH+Pw?Z1|2kBo9cXd9+b7+P~wt(j%Y4w8*615gswremGKY2Ql6&tc^0pUwGM{&sMco9bmcJdHEQ zTkm$gB^EsBi=x7mhu{QkB#R0W@>d#kN_}(j2wm4OXT|06iEY}BZ7*~;`A%9@FTPW5 zPjqy;UDf1RdAYT+m$z_DkLBVKCkaLr%f^P0T3E&1py|S=FDf3v#OaO2$|)LCU2btf zO>`Klo!z0~=a4dQ*lbdmg%Mf?1J^%YI|&O;=1Q*tO*E-B1$c zgt03|ezs6m-uEs%v-p_h_x|L?M=L_aS4@?F$^iGd|-+b ze^=V83aZ0mWQyHfO1(IVHg}q6`iFqP(0(O56K=>a)+)}|ee(Rduuzy9x6QfJ?*(I+ySmt!`Td-b|0RNcp7YN5NW`RCt_Q@ezl!_lv|W%^zOJl2?tfjU z!mBqmK&48`(ecIV=u$|CL|m6J-s?A)XLbij^G(#rghf;_Z`ssJW>#_|YTlrl-eCg& z@P+19DaA;TUCVDG0FMUUeW{_+SJTt~Yz|bjRvo#(O%>U#IfFhXC_*xpe*tAWJewUO zvCu?ky#Zz!lBK=>S^as$#rCiLe_@u*2{1XwIrBVtyHM?9Y$CGJuL%i3P4C{+0TSb* zmX+ zOIwTrwbAo%U~||+J>fR`K6q6CT1V}5|1xBFBdjf}AhKO|!Hh5}Tg8r@lbYQPG z`UhN>&2!j(e3#FUuUN=oivtFeOV=@%D!lWhPClHC-Cwnf)V=$z!u=_LaFpy)P?z;mXlE{?N_O)5{*BFLl^!L(epM7Vq8M zEHAWJ54aoDrN#V-$e7WM#OI`#t5>6|kZnU#;z&)4WpuGp^jQ|Ers?KNabbgpAI{fw0S2#<7vq_mxf+=m6! ziW%OxafDY>6Vi+&$~r4Y-L8+5(0-s2Q#VPJcz>C(J2Yjv)@HG?qcz(YHofV_ z>E-lM%)}ug>K=8H3?yZjom_Gp?G^{VzY?LLAD3e$fF~> z)j=B5F-J-*EuNw_LmuLSf_G1qel;Y8*q#boM(BU^C@mFgZx_}eFoO*v38-BC+mwqY zz2$e}!SMUJ_Vp6?V(7o|ZdjXw{pF#Hix}Kl!`d=4GAzK14n3s=@GEO;TK0QG+o-D( z4Y}3YqTiZEn3-2=$FLoqtgX>lY^Siv$12eXjw?d|Q{v=Hz)aIc=nOi%N3y|q(hD4^ z-^sH1AoCh(f#n`B?Z}$<&I5i6(8K`ADC6i@R*(a|!U#DP0)+{gvRZ15(Qx}j8Yo-0 zV8cou+Vc9`Ar_D_3@U8_zyNr*otaIQkPr3x`NNLi?B6p_#=_SMr#-2=PyM=a&IFkX=Fo*!d z^(!;DYrz1>xWLYsM#yX6E!D6=MB+R>?2^$Q7cnTbEGj~34$8^~G3iE~F@6vheEMsP z0P!v%y^^grykXKkb%vW>yg13M$JeMRefSHJWofAb82O=75G<NK#(COj|Cf4h(=VrFXoplFFyjNTEj5vtKBp! zC#TPg@iDwmSXvH69zObL2089516(jdAVKa2(v9CBQ!aHwKjg^QpYlb#+x_*2u<8cD z@iQ9$9{tCNdbt5%1pps?Q-YkkBZT@hpLy;igGm&$SD=Z2J4n)|XxYp8B>yiC3ZYp{ z&d>AbjEfY}PE>TGQrGH8N$|3))YNDWjU0d?y>zJnrR9iXJ>!5ai5N@H{)W)}LF zAYuZ_v7(zBa-#&NE5-i-+q@KSAw>l6vA_-Z%S;OG7g+wn5*chl&)mT6`{yBKFz_iB zyhT!Pb5(6cxgNb`j&oXn-$*cko>TD;V3U$-Pq}AaosvRh(obC%MCCGJlAWtw!6I?? z!u{fPeiBEkBjk)hMn+2q*uUgf$3dmfb868NW635jrcLpPup-@-&vle63~q{#A72|R z*_5vC!b17`M{S-SwP=)d%Lw@$DTcODz+3%7^|6(vK{VJ3MR)%Qy(m-h2Tr1Uo2pAL zq9p#kEIxdA5wOFvSPk8SgMM)Ahs}XXl90sfJo8glRuf(~db4wHTtf;AmcfS4DfsR+ zUw?LQ%*CBJxCexDNkv|J%$)B1jOZRrYd(P?mLboQUrN<;-nrhB z;Zb%Y^3;78aOm7^Md2AUs01644=hz0gxB4w8sGZKTzOfK|7Fg7j`q1k{D6S|g3rqG zs)*0jPZ0y$A6joBsMA?4esU?r84-^miQlL?%D`93e5x}nW@|j(>ali)+uGeOd2p5^ zzvMo!vb<E_41`M!&dj=Shr-kgM**ZrpSVns#ux z8N}KPL_Z^$LLV+0hpEvqLHc<_f#rOUY%5?VDL z&V~+S!d;jB^k5mfKOLY6im{1HaVd*LHI{3# zOq@ZaoY?Ku`Wzy~Mvr&~`5aC8u#0ZDpMR9>kFLti<_<# zfnO%K)Icj>z_t> zyf$5}uX1q!`)yRK>&CroxF7x;FS~$3r$F-CQQeW|I+Svt_V9P?Gh=^ia<-eAdy(lTm zK%?dsUkgua;LVKLy3lwbZcKU?#?x zJm*poP-7Ie#)5lO(TqZ+%LB!P+RmF-va*TtZA>O;mN4k9I)jkd4_ObJmLck>*K$Q2 z!ZOew0GoMYAL45XiT5_EPL3whyZh#*>CV1NBn(nfiR8PU5Wq9#9^Blov5j&gc#3SI zr|JV9B_%wIixyMx38r=VMGiG9pd#Q46QNxe7#_}{p-H8G;;68HH1Z1qG#@?$4OUB# zBq#f0VMCY(iH|<|XE=xFQCCL?VOg2~J)8T9_}SV1wUVN7900-S<_IekE?=Qc)Ap7) zt>t?mF)F6Vs~&+J&&5)LyP;=VEA`6h`ae8g%>itla;!dbj7?p+ERtQ4*OeDYvSuuw z$Ju1$mRncA??{mr1ftLd zQ3IDBeD4zI)6X^hyz04)WRdrlULV^|7;DH;YOr+dyvFK06CgtAuIb}eKf10V6nRcp z@maU~PN7Vz2mkvYl;F*zY16r|ZXJEiVRvvB3fYQnN!q6)7i+-@8m#LI)^N}MVapCq4cZCwtEb1pUQx^2> zW)J#5=>NdK=9yE#aTOa?>~>Y?rDe0B6{Lw3!)a19?oC)s{@m?6!-Q|`QncJt#K>}l z8g$WdB+=(PNg@;w<$0l6hbSTla?l%-&A>l2TKXKz8>{~9mE(=+^P=S@LewsAzpOmI zOK4B?&&dhdC{yTTP{1#URj%6*5t-D^QO=B4$V702?T82>ns?QU40dQRBoK`|?5b-B zNX~M3q4KAj#986dEy(t#LLHJ=Lfym!*~)+uJOv4%qYjE?_@}2?dw&0FNR$#C94=rn zehz#)Vmk4xEWmp`Kx)2zQCmR#L7*Tx`Qq){efY%dPFC*G0-0aXSP6ptl&n0F)C*nL z*`RjUnVjPGxFYDorhposaKX?}QkLP3BmQM^aavg7+0O*sxO$xpw@Ccths1m*&5+kx zCEiQex?+a3nL`c6%4P(uAEagV_8LsC-SJFHBIGr)_w2H(n`=w<^}Fn@cNN`RVEE8t zu2l`^e2+A-R0ot2KzT=rOSwo(J;a)k8b4-c|GED`mldB6SJGud=n8lU@XmD1CV9^y z)z;L6_o_4IX*l555NY0FtDAV4E(9?=n(*Pyd5_*Q<**t)r{%d&nuGMt#RGq zivQZ>+|DL5nAd4&R7K|Ms;!M!z%L-dXC>hU`^F}Q9!f2+6?0((6kIM>3XfWJmJ)VGez=vTcml6cc z!B5WzfbEAo(mc3;Hz$HHlCu0FK&(GPPY-;oZ-fe_EEhUyAkjd)ySp4Z3$R}pbCZFH zmol-gQ(9I)Y7EoGH*ht|$r+77-|POqeVE^z6OVvE69kXBAqOpdwuI+X=K=mcuV3p& zM$k|;d@=1hP?R7chqRJ3x3nOL4YqzKScfAXM}Qeo@EXyxvzxxqQD=C0i%*~s{WCgR zLgT&~k1gb{3tK~(pL}z=<8>2?kp6tD)aZ+`bJq!SCm_umQ%KC5=kY$EikU$wRO{=H^H`78|>8Xp4H(!@Cw;XXqE zTM}X(aCts7{m;TiA?mHy4KqPM%;0^+O;h`A6;vE0i-B`0SO{#d$k ze2hDeDa*(pvRD@zAGeyC|fSa%3Jv+;Kd~7X#<4S|e1f0{CD3$^zj+qm> z-V*N~Wyz~YP)GB}Lhc@0kQ#8L*V#2D&pA06+~v20-dEPel?gYq_uxa%(s6sw*CQ5h z?iI!+#szs;_B%w~!>q8I$G$?!_A&CA(?kfLA`%y_KK|BBHMjQGuOr%Ja^A?hzn|mA zjW5tu&j?pEjPcp2=-0@l8Miadtb*8s?O1L@|7%;@*DzSXw)>T#irqF>2vU%+TSt2H z6iPviPpfoQn|LUg__oPsC-s4KY9~*IE06PR*DPn}_#`1$l&R)G&)$xurY5SQ9ssZ) zn$e2%<^_gN&71Y4+@PmlVcz}J1iX`rmoMXmbL0;3 z08z`*PgbEf=WmV3%v&s(Gk*x%fP2P8lAFj!HsYU6fCn)|@a7fs*GrKc=nmhdYMvwH z;W6H9<2?V`SZP+Rqcf-Ocd`$>2W*3cNbqLH#m{JE+dSs*3x2#rQ=!)+@PI0RpISU? zpZM_L*-V(a(bPMKBliKctceTW-q}rJaM5-5%>04{E5~Uuxag#$94;cK<>tVN;j3sI zi3XQM*d6|35O2y;GeCp75zDjcc9X1wkI|w8p`_GU z@7>st+wD>53yvAQ^5-f)7V5i}6|yXgJDm9KI#Uf(M#=G=@~#R9L{`+r>dw8s;1b6H z`9`pkxYHL=cJO_}YGOBwnB4dArxpR1#rm$|;v&%)UylE%@QpR~*#d$5dA$SKds4+~ zRd5AiI8w?lFkRAp4{4I!Qu&8$<#gW@mujqjQth>FYCDvpU9pGzd37ut%M=|<32p_G z_>5R|kXm$sw-cYlPFfO02M~wu-Wl)*c!Bd1baF-7qgcVGzDR7E&h14Hn!oyp<06=l z<7{YWz@-Oon8_J~^WgA+HR7jG<&gb-o~x&aVs46b&L0B@bQDy=>)-^2MU!htE?A51 zzfQx3|9L0hB!K|~Qrji((xnf;*uF6z{TOj{kSfTbt)0%@BMoO4Y8MW-u;gx-?-1YE zE0zX>6PTDFY{VZGRd0X1`MP4`SB-VdyM@U+O8pdIc>R6UxOi3DG?Cz%`yzE7)Tr>ENDrcobKLLX&Oa!58ZAUE@}!}K%uw{yzhjw zTYADp9IQw^{ml2kl=QpzgKr3@K`eS4dQH+Q&f6KrP;(dbed_4gogSRtG^e*m_O-827VBQqY^0F{cKmP{s zUCM}HI{Fx<8W8W=Dzlxt&4xTvDls{r7tKSGnhCB5mtpOl^^BK7PcInU^^nH82=7zq zE`Nf$>X55rKgAeKqdBl(O`dMT>~^S39ZbiGlTeZav)D`UfF)+idRJ87l9Ru4JMD@k zodg-`R@>pBFgQDD3XO*9Iy=d!iNSXPGf7a1Lm}mxH+2zow(wpa9HwNI0Pya0y~CZJ zca?DBuI!sMzl0L%Am<`4zZ+haanwQ?2NoorY-?Hij0h!eis_7cAH9$+?6@E(`=@ z5y=Auyt66$wIZP`d8;l72~q`dAtE+)+7IyD1D`dZ9V};?hZ%xIATenrge@hd1uonx zaJ|eAuRe9e-&<>IfPk1J<0nr&_v_jvz8@}X>6&B?PhCQ7Z7t_pF9W+q)kcTrnZ)8{oLL{pEkQ2qN%DOr~Uew__3f?Opk{%A!{2ZY7Q`dzGo)i=Ct==#uc>4&lWh0g$o8X&P#=wB5bBO|GlAnEU1^~krT z+w;szBaoi#8QkSQSEvjw6NjmdckA)D{-gBZ>qVm{5@`hga$*= z{e4><3n!ZwK@bE0YO7}oYeP+TCZ5&hgZg*m#*3)oq5cmiZoyKoLOia*Ld5_cUz7uH zAeYqV=>QLS>Mm@2+GzjThK2e+Anc*KyxE_BM73y?UWk3nWLF5XXq2LpuBmx+a)g6|dF_$# z$AH9Ej|sqrc%EG+Y01TYLln51Hxm7(7OcT4 zKk);5&aPp0t5tncFK&g|8?`$*wC&hj1)d4C^>bR0w;r*pho?ReG&rG3r)H+a;f0p{}izi-ziRx?jGoMa{*t)P53D4g!E#32u}{VCoOP`|-`_VJ%NI7@-- zLowfJQ-8RtjZd^Dk#6f&eO`>Z=2Rf$nd5Z#1{7I2Xh7mU6fjcUSrKQ3$$>~yZWTg10D~)edw5AWbx~s9&x&d+m@DA_{3Suy7pEQ64 zg(yg4fk8)fUj6VSR91)uptt@daUbmC6`iN{OUaV`URT`?d)cXlzq|)8n9SQd*KME* z7K{#4D=QZPktZTapsXeo`;Hs@7 zAsO>Jms>^IMX9@+Dv___1kfMTkzyjG+UYMRzo{vyKyEzv55tUWA>jMzuCKeQJ`Fe-mxMgXhCZ zloOe-*eVx%iuB3Zp~qF_zp{Zf8wzfqoa&{V98OYZCIYz7g@t;SpGaXr0n(mmWyN7_`t_Mi7P;^SMzN-)qnx78yFIV%KC1e}iTnh}soIFNYoV|xnkdw{ru5bH+G z2yHe@W-830Bm@LK@?0}2Ha2|lo1WPq?7JuPe3tllGH;uRP!Ei2Op*9@0{dVJa^Ia9DJZ$JyFnHk2V1;|b9QOO76Z5k>%Iw1(08Z1S| z5kUJk7_V@E3Sf^M4x&4&&|5)GMn;ykT>!(M ze*KuAgs~g{&nB}{sbgcKx|S--4CO$~z-&?*gCXt5RU(Q^r~^=4w|gif^9C|ypI}my zpBr9>$iA*oit8`V`wysqQX=I;oIa|9!C;!8KCFoJ4>?fsp@vNn8W4b|51+?9g~0Zi z&B^ukoa4gcVp`IZXDq@n@j~}c0umfFN2)==4x*xieG_Wb{PoD8p#zaKklxdOR9U*0 zS^zx=R`Ct!$&8De-;DatC*uL#ry!}pkoEF}Xio4UM9-m$092y){~BZ(8@&(R5h{c9 z8$s)+b2DtYZx9vOY5m-+l}zvj2=P=M80ELVTy{I2VMioj*gQ0MWIuFut#dEtgcD?^ zr?AzpScqCCxCbJoSRqZ~u1-g57S6k>!mdXnvL9Q>ArCZ#sSatdo47;OZBPlCy}hW` z>TnSbiNtE#^Sxgec{h(H@br4V>A;R(klzJeX<|Q5;T9AcM$&YY)mdZ`2Glo_xGMg6(8QQJyG|`=r zYEwFOgE`LLUgK8V(OxzVs=gtNk#+fFfjGxHgW-BvEu{U40f&Kw<$S!=W^C{HydWGn z8PxrD)#;A|-Ofa=o$WU{2aPx%%(Z2qr&_~k_m@)V@(T19W*p8=N|B4oob-;2Oo#pH zL4csW#jd-)8)nSpBiN5NLu&U&Cgw9{pE}J1U1HZhzGyb46v!ibZ4mPV z@ee$BtlM6@cYzfO9UsqE9K7I;C=4G{TSU15di)w|?u--qgTYFF?_WYK*m3G7PQwE+ z<4zk@g5KV%13!|I&_|k4!;sNTC|3K5syqi%sLz{;?Ar$|GGVqUGVL#h8L~ zsJoz)@6OW9-u#l7pFso1c;FpE-4!F^+)EE}vUk1m8#ZPyxB9KEKg>;P`cQXuMSLWB zm|TC{p!ZVVD0WK!vcw8_J*j}_G1=ZEKp)!5u=HiB9&ccyJ!g*cr)FI5$uQtjq%Z>~ zw$E(uD9Mk4Lk72lZw-Y9x%%3c0bFtLLg%$(3p<`Hh<+`#;^A<$xfuJ_Ut9&m8Nbxi zpVE;dKLq?V{CVjn6vI1AgD&OiFK!uJoyfSSDTUd+dEM>QeW0UC<=lhLn!#Tv61NJ} zdgWF$2e?etU0d(%yt<7-MM?)Vp_!+W;|C-aq)wYFym;!@TNT5JG`{g5-d3XKb1JP?8eg2}^wJQ}st6ecF?4H49XYo^UA=s#2-+Baa3sU2brmPqI!_EY_d+OTZi*Rbf zL%oe-QA(Yqwc-cNTj*9UY~0QULzn4<>jQ@u$Gr-rAZ}28`}X2)V|2KhCtNerL_>N8 z25Ot=x|+;gNl{uv@gcCU1XG9ruJianG+bPhBRFOPIL&UasO825f*g^&IXDcPch1VI zzOUkAJ<^r?&a^AePG8hFhEGrVbR{egRBXd91`ttIsLm5CE)BinOh{~U-8@*pMvNgD z)VU{X=GIVSK><5he+BW!6dT)Xv7UCY#{qfoX5P0U^wlQ3Mp&)vfuVBqTo^TF@MRR{vWZ+nNL|wtaC|Pa=&qY{qNI!;^=fP^>UC^-TVe26- zt|n-yPvh*IrLlq}-Jvtla?bU5fAjuJ!#@4PcbY}qfwO#}Ft~W;a9jt02VX%#tg>rj zA`U$^+d?L2XGY4bP~1HTz^-dLE@4=_g{Rv;Nys4AUEv_#DmuN;Nw6_&V(oow`9Ix> zr%G|p{9pO{@6SCLgz)b4_;CwNa4}-#PYvlO#72H9G(_ldoI@Et?5={3OI=Z!37K0c zX)2$tO>#)~Y@$cZBlO40*8Qf&oq69#h9nUYVuKca!K6PezIh#ha+fZ3k6E5ZoAdMl zf(=HA4E@qUBHkgAkge1zJZNmW0Rs5D_LOJYme*M7+Ra)6(+HQ7_!>d(T!~b!bwb8!QOO>aAso`w2|c zol|ftx++R4v#?U^Eq{ZiS#?B><|j%VLcaZ6%@pA>4(9= zu1|C97tOc7%#{mdDB4$ANpf?;Q&i^yaAi|HGUr9!Eb031ntR=D`^_O#z?m)|^G*RQ zORx8B6mO#1C%B2I#tB!fyL}pNkY??lu*>PJrq~|%RkKL_PTzOkl-u2e2i8W+OhR$? zr&2811ADBP<8>lp9FAjwCo=}59jyh%Vc41_Wz?q5Y}B|9Tv*EA#TWK`g^opV`(M0*3Gv3>aP1gak-*~1eH3cwViIvsyj;g^?p@>Kwh zOsGDKLs$TU5V2i#hb4P~+0t(gjpwY)c&n>l7$2<06{%pQZ#b)LYk1alsXNE-2^;@T8&NYV}?Lq~-+7 znc?z%v$a+ZvTIE2tEg{D0rX^cBv;l~zd)d|9G_?`Bvbub^*wLzGC>R7n}I>->ZB?S z_79TXP~ij6v~?^PQpN}b<6LaLhi8ldQ3UEuk{ttzg~j*7}3FU)v(3VSYC?p}48dXx$sRf<7(1@s_YqFwhbd?mn&mohis z4`;RqjmVpsa=~!mhceu~`$bhUcccya=dXKSU>}igOuXhsnl=*3tKKtOC8= z`e-du9?RpA4e{7G`)LBmy;rBoHhbbzeki0;*} zs;j95M_UI1k_DB@CvR`RuKD3&@+t& zPZoT3Q%!Vk^)N>*Qi?MeLnZ|Q7tzod$jia2YHn^Wt8BdK!b|-&q(K(^Wn6~-hptCn zX-J{-+AyEm7)UJKZ$2O9-}E93gYQ$A%U1{t8?}E;)FZn803zWQ^X(LXm?~I~jfnxp zYT1=#>+2)nw*pfZ$a}y zJ$QioU1>K|V0mc?=*gG5M7IgS3i<}J&yotc=>HohE~$cXA3kBA@k`0c5yBVH6KI<_ z0PUB^cfyzfD})T(KDX}NsYG)JGcz+I;0|d01d+=V2r`tpm3x%hj!QR@863D)?|~u$ z9USQ-h`@7S!<+X%H`U=ARmEW&gMH91hQBkg2w`s=nU-}D+M?h^;-<9Q}XD`s=8w z)A#!yKY)~^f=YuRp&}^Vpdd(hH%PaHbVwP5AR!IX-CarwN=TQ4ba#i}>&(37vwrXO z`(xIuab~DPTPPbt*aZ$67hw^G2kph66lOmXJZ|mk`rSMT>$fs*v)~{0 zKZ$nM^2;Ie%qJ#ZcW_EEHTClbQTdJJUcq`#tKq~b zl?}6iuiR9&WvzvNne^N1yhqwO}}7=$G)lE#c}h8A*$lt(+=c3oF!9AwsZQf zj-sJ6yx707v2@~yAkzQ!Jv^ivY`S@xYTRM}L2f7jW?;j;HKwjl}4y_Yv(tk36S1&$r* z)#dh9jh1#jfGtT&AR{`E*<-n{vF}=Ua&>M};5gp&Dn5LHlL&Egv5!7hJJ{SOv;~go zdkkt!KTq5f{Smmku!$EBThn4|C7*)p;TF58qL z5^PRk{YW=cCa@OKl-FhJq5Nn^)$HHP z^?XkcKi0ozyT?B{zPAJ!!KB&i+WJP$9sWDW)0YxWy_pY_&&!FVnbAT1wS*pyhjGEh ze1Z4q?uS=_ufyd!;PQ^k#GuXX_|vPcxQ}Z(Ev#f?H}2u!j^?duY5VyX$MbS~r7eG! z(J+aF89*>+l3JCkso!W989NwE(~}7QFkIZL81rE;Fo#MkSoKT4)=b((b3~k9_v~jw zp9{Vl1L^fXf0u+daH@T;7}NCk*C_0rZ=FX&z?Ar2%yD_pvljVr*GKl%d)qx91y7dO z=uS=#uSH#)@xL@`Z=D!@R&sS%f+8GV^IJ3LzBMWWHmL`^u2vNr+mn`^Npy}c8M{Ld@Sv&oax=i3IvRD1!;2JVs9oX{w`0ky1POjnw2CL2rFljmD5j+%p{F$z6xsEHm zv9ptQ$BnF~2ZJh$%g|;?Fyqrq49HCAEkVggMghX81g``1YSzF*;L_<>B=PShtJl{j zmdJ4!C@%y+P%Eed!Z?As(c51jU>ZYF@q@$8tCYrl{10mye@|XNq>P#7N4@?+RZ&H( z^q99pAhsAf|WOrWjF)L+cT-^Q9PSD)%p=4oxhMvu~_IAZR=#N28+5M|+6<&`AfBz1J znYvq-6#&(+ZnZ2K&kJnFY7K_I5F;?Nz3O8?LZlt$#c#HK>}Q3M(x@|xlJNTCOmuu) z+#FDtIsh#6y)q#6HdK6mj#sjwamQGovV1E>n~{+LVL|V&U=1VgB{DNJ11B&$OqLN- z`=+S3!2TmuS5rem`)ii)Ic8g*JBw~z0K6jeAoaic(~kIRS`HmFq3Zfxw;-*1Z|o)O zs$22c&`?H7komh0A8^zPH2h-q+_8bS06)F!NT`M3@z!ss1O&!Qa*;jr4Xp5oP}~MM z*C+w};PjLZ!UHMjE}uI61*Pipg?DkefL{mm>U}rpjk!ld^Yr0OBBI42USvhKyfIcd z9yri=-1hF#_d)g;-XVH_ZtVR~idEwGMZT3Y2c zX(CHsN(}uIf7ALwvH*h-LMS|5XBpCsl}3ewfL)iY|vizjROh9ta76f4^IP`$4- z*=J%%*#%0Ra*Mm!tVYA$l$IV`*0Ra=_zCs0HW0q~mF|2VR#sAi9gM&DVqkyCEmyD! zEf5?D&~Q-J&F1G<<$9=^>uo9}1z5=l9|G_ni1BWf8EcVFR+%VN={L9rydS8W+wWt1 zi0jV!!?91YQuPP)gXvA@mFUO|mZx~X5A?gfsm7Jy_ek65t`+LxpnR^>j~_-B7H|86 z5;Dp{nwvQp?aTG#owfo8@8==E!9ZhB#ySSSXEL`>K@L|yxAxQwk^-AS|_T!0i2PT zu66$*%i&%xE8H>hfvYAj9tp$RBk^3WkA6>Zb|(ky|9rUT}@ePME-t$nvI^YGUShm04O#-__!MYIWN{SZM+uM%}%V%w|0MTp$- z#b?GVF{3fKG!lhtvt52?d-=8H+cgXZmA|ze@3}6ny3yZv6tKN95-izD$bDSRiGfgg zIc(hd@Xqe94BZp!BX3VG58*3L3+^k|NBGg>cnHL@;)tvFIfZM`dT+MSp*0GXPV`zR z|Cy%Y0`ca>otEYEuYwwf@sCaC%{6#FzZj?vXQGO=Siat`L|4h3aAic2D4}IFOl7iD zkIr-9@J8&Rand~*kiPqwJ^L?5mbRAbXTftqe2S0BL_#M96je3} z*chUfzjL^}2q~3~WUq8PjuLf}t}`F56;gThz~DaB6lfZ^8`|0U_OcDf{~g{e_u|Ua>dR zZ4oa%J^6f?G>hxO8Yb=>)N>r+X`8?NohIY`UW*|a`q}Dw!mXIDT19-@1dg4>3}+OBoX1YX0Ttx~ z#xKn~#M_8 z*D};}SqR62=p-aMncJOj`QY|3A-}-oAg&mT`ep^KwCqzN7Q0x7H|{>OiK%DL>V7uh zB&?$mafY==ma#3TzAtcpx1aiDzofXB{_3z8yT;8CZ(>64xonR#(*t*WS=r*$f(^He z17&6bfsE0&M@O~=I&Mmt4)s+Unwl^~yZ}`ZuuBdQR%)tSt z{nYu7nAucKDU@^MO?TDgO0J2;MJmxbd_3GQZ;7D8`8BY2T}nDd!g&7S_ilGGdBIyR z#;@#r>fbvqU%xe?tI!|dah%byx;~2O+oR^w13SVjrk_X%E>NtYEL)~a*l`KhwC_m7 z`%2t%=NnI#P+i?MuwX!V^kE%!uR6o0u-7w`{_CUaU4>}QP;<}To1!0dM-NL;+ht>{ z6ezC}HjnVsi_560XJo^nzMGe2% z`A<)R$}eib0*8=sO34?JdV_R12gD-esNoMmN5RUK-3<7E5I2S7U&aC};D}94yPsoY zok`FT&H{0gi6yH^{||WO>s<<`*r{4}#O~d^3r2l7r9XPGE&$2|DQ|%(L8^*@&^;JP zPtdu#a}M%xZmy&uNUG#DH8o5ZVInKoNphVlzf{P>!0b?a*rkwGQ9;~%b@nifS~%?J z$fb50?BNiLdWIaoUM8Er1!OnlNazerrB84g(+z*dBSKl=@3If{~Njq+Uy)*<0v%xfj28NTzaq*Z`G`K9_ z{nLH+Kx z@o`e9!a=@=M~ZYmaL;%WZ8x2R7ii2E6gMNNZw8<+qsz&^Pj7cwAIo^uR_Rd=_Huj^ z6D8e*?BwezfYo=Ja6dfkkz!Yz(=pS}As$0h`i+j5p z<;%O@(a(JTCS1ptIGdJ<)+^!aU*QN*O<%`EWT}aha6Wf$`q`+6;vL)2@kD)z+qq)L z{wkXo;agkyB*sVJ#@~iAvh5UIzrXbr*SC|jg1Y1}-ms*nd+m$dwx#Ue`XP^sz&d%h zaXI@yYdu5j4Ut+{zz?S1VJ3V@7~5E@o)7A?U72?#>S@+{lbtJxdEYG}5V0cU0_7$c zLUmObKKzXo)VkXKv9EhNzjqZy?mcV53Ko(RF$r@|+Ku?6yL+@t|9S7g{IQ0SuJO}q z!k%GD^P(+ZU)CexOm>UU4MKzo-U(>S5q0H_xxIF65r1EBqOZ!3#azr>3msuD3(?KH z68OnxKx8(}y{1!}-^jd>-9sGA+#>Don-F_zq zLl^3az^+chnf2oW9Spf23r7(YQhY{fXO8HIkhe~63Whd%E;Uz{_Ec?^BXo-u{>G0& zXoF9~)ro3p_aut5E@ATP_>*-uIT1UU63!l1S2{mzq}rNc6)YZHS51GtA3!m*3_d#4?Emn;2U)5VCdA6SEOeg}!CD|S~_xy*In zT0C|RUy)8E^znHZLOP#QhWkk}TR;@y>%2a+KHW8)e*eQdTYsO-NHIY_(^X*UlVKq} z_2rW$8V>>tgqxe}nFZGW8ISIY&d@sF{BK(NZ+0pK1GCiLIP96z($Xjx% zx({PL@d>Jf8ynwsY?5)yS<=KraqytkcULKr9v!LB$;nn3mV;Cn#TUZCxm#hENM`zT zr$1+^Kr=b2V#C#~tX-E7{Km2wwv`oK=sNY!?_@D~E#sP&BFr}- zl{68G=@jy*QQt?{E z;TkTxt&5s?*323bk^OM2dpW+HO__}{S^^5R7J&cKe!%*=<=XE77Chxhn z|2N`3#fTSt+wRH1`rf?6nws+S5a9H6n5Qs&+4f~*-WoYPJQeT2VAffo>so$w3Q={6 z0KQlE(A!WpsIc7};dD^8-uP|h>)w8af3rxRRYIX3ALxPT71jCp7^5Q$mreEBQ(!ee zdLg9+h-Gnb?pY$Xm(2oayc0l%{>DRgM z{9QPK&nWm^aN!9Zbx~KxcX6amktI7B4$hKDEE1+SeRuwVeWi8MgS?$oPZ`2CbDKR8 z7w|~;zsr7y=w|l~vNOh*NBlu+mZz!7W&P$HNQ&2TuOq#!(+pQr9h=T%cHrx58?1;Z2)@#xP`b61m8hOxvNzeFw3e0@ zvV{2JXwojHAgcN=$U$IwZGjbM$j6U_p#f1?Mm!W@Q)gK<8a`iEz)-+#H{JGn#tY&< zEcZR}#j^*I?oOo81j#a*r&dnaf?Lxvw6(Noh#x$`C2RW}7l)*J+coTB09zm$!^+Oq z7Hr7Na2vcN*xlU)L-VC~ln6=hfv1b1si}DKpkUU44%&gInY?R*q^xXahK;rL?7~7u z&11h*U(kV+txe21$bb!TI7ZttFCr(04_;9wHz6URSA9}g0pFy+I13R_=GbPC>3Dc} zR??oILHSrZENLAa9R8t5oczJr_+5Xe+V=M|n?lno+?bdcEF2sNDoI0=q@6KT@qLEn zAH)zBH_EU?KLKVAMsYMSvPOAAs~eJ#3cYgfmZZps-_Wmr$#tNlHn-2U2CFvVEZg&Z z=CUE1_S~06T92W5Z=sRqzX#I^EQ^4M(y4` zs97S8S=*tRsK{jPOqejC!{1GF4Xi& zyNM`cnI1*|#V>mAYsI2x2?3rC`i>MbJAYCfZ*cWtT9ft&g{9v&ZKm>LyyA~j>Kv!u z`jTI6M`qh8-2AQ~;Z|hD{tK(0?wmU9w;MZuisv0q=%l++I5{hXSNgIF`8E+H?p_>? z?~pEE;}gvz-4GZclfSYfqm7jg=TBk$8uynz-$q{29k0rcFZ@Syrf7emXDZeytmlvpaj!eXDAKJ}tm(rec*%?W;&rV|I_VKnhy7Q#u0Lmz z%TJ3kBVQFy$b3+IL*4t1eyc_hF7uf|gWQ@|TcIc~Q^UiNk7dVm1_Ki!Nh8JYZK?WF(la7^} zvFLg87U&;gDx3qLiAW&%qEb+}85<98!1+tov4ayI-Y3TV(JHmfZ~6GLoUVsYeSW|0 z`qH(l9Kf<|(%O)-5c_u6mUKaP;DWqx?q_cR1h3%XLL@{XYTb zH`CEmJ6v{m9s|+qY#FVrPlm&73rrF+3^M&ctHWEL1Y~Ci-n(2%*o7`W?c#~i-d>g8 zcH;L`3{^tFYyT;3L@wv}A;-y0bc( zX3)v=&B7_;oqzH>><^J6LtQ;>0rXF>Z0xoZM4!v+==OY_Grt|u*WQjx5$4zSndPDM zv_pV{W{t6%C0TZG%Y)C&`AOnS&rSZS)(#lk6>pYSk6ev~Yc70v^1<`In{JY4RL1CR zM-yg^Pc1R+jgO@2A$ypIuC|lb_9}^;wp)FXJodaN8u&RxD*8D;Gxe4$Ry{C2;L|yB zO;GWLR0wRGCodb4<0Wy+%8o;9imD?#mahYx8DO-~Vb)#CH?`_{!p$uq^Yf;d#P(CK z#+pKoc&xNE&k*v?mIS+^H}&;sYwNWyr1GG|F=O^myOGmtoKh2MGTW(~woK`V1{S%l zHiqWkGYU*roXA|Z647vQ7N6~IH?ZG^mG0aCeW`(sni^Vf3Tvd}z25G;>0*(A0nVBgg2lSuui;%tEbZbZ1L$mRD8oFkl}uqz4<3c`4=;Ns@Si_|-S=1u6=Xy^+zAls>$ zyON_rLqPq#5fjt8OjRyiQk{KWXn14V zT9f&9OY6Br&ay*BTAE*(5DEg$GyvkQ<~|Q+gnZGfWn6CVYVT%!h66k}LCekhXaxn6 zBJB(ed|@dR`PDfb8}?Pm4%4T84GI%&?}(m&2mh6@d_OOI0kDe7m=eT8fa;S9m>){Y zfn&B<7TuP!!}TtWd*%Uv>`6>YQq*JddJRpd82U5c{!+yEEz>=DqR8^Gk=fhZ8!0x_ z${pW?ofIe`-DlabeKYO|A?7;Le|KZegLQ0oNATZ|gMLxI(Hy%jP3vC?9BZFc7?C{ZWR_5 z?k;?i#~)wTh6~^0#2eQGk+@wjDemp>#Dk~ehP+V1Y8#Z=QHX9Xz;_2VIyRSowLzFOJD9RWZ|B;DOx&K?Bm3g5f(#0*V zJWx>!6jg_d%0jI5Yir}a$>1ajj5poH63Z#D4h|1r+vFS_iW*6m(RZc^FNbqv#p(BC z&Qvq&@beFE?fcy>z3wu-a15=26~MwWH;O{R({nJ;Q-(st5H14pqnI;Z+_Gs+Q&l#k zk$prAJcdI|n@(09qIMk&sji+N^C%5``{qJcZG2#^U0XM_x@I}x0`JWDcs9wp`UJk@ z%ZrN=s2w07x5}!_BB-GJ6mD*{E)KfyR7HQDY8wt~z-*uJwsgfJ8`~No+Td^$1C%%~0U?_Fo{~@9`X}TLsWRUhu+2a6Le&Jrse6Pdqd3&!Zm6F^ogQ>D< z3-iSDITlcBA~GH&$?Put&GPFSk$at;R&Z-|8AmZv@dMmEojCTtuCD}rENB0xY~Kjeg_HhR4!_z-^gv z!fW9Q9whN)zmz8Z3am2U&LoG81Z$mH8#C1LWpD*zba;&}Z?RG(1yfsW3%C5VHkc?_ z=RqlRa>lmb(zzj*{8Pf+{n2%t1ZcJi48tEjN)|l-b7RQ4sIi63Yw>9FL4k$?YGvi0 z(wDZq9lz2)5E9LgvF(VONro2cQ83bne*LBZ`C`wT@a`J@EFG?Bdcu2BZFqVe~4KZ@Mp=t?#z&60rC^N z|1Y0WUy3*$jBnqn9!M#HML8%v9s2>b-nq&Drl*PRvQpiOck6ptvdR3oZR|wU)L|Wm zb`!)9L#n@y00u(K`&*2&3oqZF?l;~bntz|eWL8tNq9(0)OVBu-gY$-s>i?OV@)ZB7 zg#G`l$^H?3R&X`_sidWQ{82#K($FAD-AbD)rd?aloY@?6AmDeIQr1$t6&hN{&Udy| zUw;CBS^j_}-%DoAFMR!<($bE?+41ly322zfuc_S`8@E`dd1+@{Ks`aVnZkTCa>2Yyi@}{Z>LLm_^%Fz5v zoS|26$c;K**0x~dzZ;`2_g{G`1()&OpUgJp64x4?{8ZFyyHlTmKj@&Xe50DPQr-D* z<3nVK*2ej{YUL!loMY$4D6M;C-`v*$AF;XEP%{Pn2b=}P-Bx0%dUQn>$tYw=7|RhO zI|(aqN@}M|2GabBHgswnX$fA-<=oy=31Zk6A+@Gs@a*9IW|l}48NT>urjYaEs0=$q?L3%tJ0&2Fq5vS;?wzVmy;8l|qDk01rSM_&xwVZCj< zBqJ{s*CD>T%l6pd>y*y)eO=P3s)56gutGwt@&$CxOTN)2zB337)a0}gJe1&JP|l9| z+Wp==`S`dx2&@6gp09|Pe)5K7WJn>tc2duXhy|c6^&OAY*a$x1GQG<+LRnh6b#i%( zYdz&k1;63XpFaXpD@T!@ZmxYjbi+WTi4*as78)Z6apQhW169b-*w{ZcHC3-#_7?x^ z>CPDv#IShMHc6Uj?;l~_!Nov(s8;Y5T>VVnE};z55F)`~rCKj`xRwOiT2v@elKQv< zR(v?3^}-9uE_iqp0&TCMqO7g28=7U;zUZD0NV|JjOBO4gxI%pQ=DUTnj;|v7&QtT^ zpi9vIcGSDM0LLDnnz9xS)n|)A2KoaKtEhEp2m-@3c$D>=`~{T0+e62Zlpa{MLqg83 zetw`?f|x2HRz}~*suQ1SPyF*YkML>0_g!N@hf0sk^nw~g|7OZR5x`Wl@L{3^7(Ts& zAJHePMdN4E(l{=g6M^yZWKu#f5du1q@W$>*(Ij1FY zmP{O~wsz0+A6OHHMnF~mg4(;@G zyu75qw)za~5CD=M_8XxZ&aS!@LVcFI1?JOc=u1*ALP5+fjqv;Fwm@O!<$07r5PXN} z65%7pAFxD2L3HbCtcEEK`H$gf@Eu54JygB$T!P@<6;<$#D|CxOPooeAx3WK82$dii z46p8%s(;}nRe((uqy`|d`~|)41kIJ(1r)BD~s(>r=LLY#7eYiypSs!r191bsFPNUGQ*11cm!Ul7N9--D z@k7Q&%1^#E)W4_BOs!1esW#V&LrCs{Ku0&?Pu<(&0%7`j=A=x*Qjjl#yt|>IB1$*~ z>l$(!%e!-GBsdPS0|5a+-dBrGnOruOnWvSb8J(445qkG0Kj%?fuN$Jk>|wc{T*H8P z9VYFpyl<;&FvTVuI(hUK#F+U#J0*QS7*eb++!%h^a8k2x%IYMZEpaKyFIpAO|J=UG z^pjg5HJ^g*2Tqn{QJ>6hw1|h|f7umVdlH``w#D+qt4OLyWzkkcEr7{TGmy5OloNv9rDyq?$iz&5IlTG;lF30iT z#&EQX`t9K(A$$c8d#H4$kB_3Kr(Pd+?kbDf83i)6_ViFpLNQ|yo*BJkU?6l(1UR||(+rxthx+#AQr-wAPVXw+IukUH#9k3JG zEPd6sdyc%lC-OwFcID@oLs_83SRE%Gw5_xPlZ*;10>rSHsIaTJeL#`k7#IHEUh1D2 zhKlvA3HyJ)XaAdF9{cXNDy6?hL)#1~7>c&D9FVhCA+6)!vov3UGO3uAqjXLi0ZE`pNsx8;w=ahF8IPCVf_CSi}03hT}v{U#&%JwO_djp;b) zGzyIgp?=Qj#kzb!9(*c?L01Ceo>bI$CTmI8CdgFytqETLvpiWhc!XDtzpN1U z1o1Y2HtKi2=y)Svo1L`im?!m5FGpNlSr5B=mv7}Ld(dTWoJ31}VmO*7jJE_dd1rfQV~ckE9Q z{Yy*MNWa#=lPB@$;_t&;P(m3R!+tkv)2V0an#vuScp2N+py}&d!SepaUX2cm2J`LS z6f==~H}zXT%iMtdCUZeS6#qRfO;VUa(BZCF-`2GO&m4%Y*%eM?hrq7>nYcJ*)8!F9 zVs@euOt`mTxS+UjCzC3N?i_ueYhd4!sVou)W~MO2PC`Hx$E zKxSHwHYa}#(>GGJb#}JGV>VqpbijBW%PUYWyLEIl_T~!#I0@%Fk9&779lm}4js>k2 z#15xQCC+UI669Vx9sYN?4}Y6 zQ&39Ekr8J+xOSg_Sv^tMM=w%cLBSWg|6E;N4}J(?z6bPS?LMGv<`0JA=On;fvq(iu zcw7uSiDa#V6R4_n0E8i2;iazrw11H7|M64~%&J8?wa>cJ2Uj&pjnJ{bUw&55)eVL4 zTo-zwNvVcLl{KlSmnx3=O{@teJ-sk!#7HYGRovnS4-N=HkUC`^xI;x12Db2ScMkB@ zBqt}|`~Ki59V#lS{n3V|q=+l*o&ZPl{)b=;u=g$X%B-F@f&RH0= z*()mEfWnf9=xBN~ASgr27+9Mi6J$$Lqmp$#l^)icxIlK72!K7$U^a5zE|c=%v4rRs5I zO>zld?&Wl*2+l-_gpWsP#OP<0fL1K&&-S?eS39115yPnt^;8aC%YupLPc26+Ep6V0 z3iky>e^{#eWfQ7%akim3M9k6UC1KWYU~P!NNI(e+>2Nl~AxI+V%cP~p4C*rV{q{w) z|4|aV-i`P}*}hlZXG;gw)py-bH^Giv zKYWf9@`n@lbKs&ugvzzT)lRuXOuwr}Bj9|O9)Vydr~BznuVdzH^-R?5;q}Oe^cLZS z{A2Fs+u;j$7T?G=_jE+a+om#?%USSNL6A&8Tx8*uPew}Y85OxU39e;*w?uhk_R^(@ zF1a%)j z?){HY;RXjf2f=gN8zTk0fnMijx4i94Opqq1#p%5APvP&cukL$4_oDHj#x#G;%00C` zuadUurS3!7>}Svuuebw_NhBUnN2gSWKR&^Y$6J89Gkke|vpN*ICO!cLCbjoyh9TX= zu3*tQ(%n|k$8KET;TgsozhtY4gIoSjJ3%1X+3(AR2$(k0{fKbth}hA9h6WWVk9+VX zUAM$}9Tzs47Q?o(t6N#g#~KA`9#j9vp^J+A`+w%C*Fg zA)#@RY!qQA1%-=A3|MILX4x(nUp6D_)IO@w)wpS!A19_X_1JAwZ6rztyNgfTz|pf)iviM)Ws zHd4KX$q1=>qacbKZoVgC+@m+ar|y03TGQa45gUvC>Q#oH_U|bUoIM*`TcDYVa&XwA zAYgs_%H$;_$yr%b+Am*|A2G<2H_AYclhZRQR(g5wdjx7mh7KTP!=)Ix&CJ-D^#q>% zd7Kib89X%wk9osfRr2Uv$TgfCP4Y(`bbSIGT_{=9-pQ`OKl2~512Q{BE+{Y%1#Y>4 za45g%hL;$*t!Fna7?PYs%)+LqK+o$ss-No)Iq4JVS~LQ}td)+0{IUltD^8{t%T4TG zQ&Z1wh-Cdjz#;KCj#)!%%`|CX#?gVv7+Ra4QtA)b>>?ZB=()$pXgpQp;BFmv1Q#j8 z@iG(GViL4Jy;%8`UQ$tU@0)5c56I(DU&Z0*fzmIUr{_Zkc6RdE1$w%=g`Fd~wzzn? zqZBN$`PnhJw#0|heNA+4K>^Rn%}rZ?ha)nM6ue=bXIvW{ee1b^74SMPDQua2mSJF!PY(ISC@;0w-Q5lF6Nz5sYoU+X z*pRVsST!(q0mQEz`f3ix9z1|E=WTen#H&|jN)bJ9ipnJMMPG{ z11bfoxIV%v1YqBRqYKEpHZk#6=+h!%kolUUnEs-~v85zqSqF?3U_pCY{C%PV@yF_k zA3Rq>Ljg1LnZ}ibM%Axmk0OLoSKBY+D06 z{0_LFAwr+N06fFlcZ|u1PrObh=cX2v*7fczTQM+*HTZtSwt!*j@M5K0%oSPSA!@ZQ!_c-jVG;p2t_E0|aS!*E|JAqNSCUqOvmGvQPgVTs%BOGc$?P^2$p3rlNl@E2PsL>{tE6;-QzW36mPXb9^X@9^17@(%{%Hqx=(r zGTz}0Rg3-0Q)U6^l zyMK)=cJq(Yc~YTBb#*odW9L&x{iisX$`!%_1XbJ?+09Wk$0%}Mj@pr2b0h}yee8zo zhH0vTjLmg{u&I!ou&H>|@n}C-B8QR(pCOffA-ps2(fyhOt9Tu6`GUkA*iszs_E*?3 zQ|3-|zdy?={cZC5# z6LW}HSvG%J@lfrGTgy=2;yv<+uQTajn0(8};;|;Z@05D#xwfvo;_gaCRzGydHhuF< zXPNHgH)`U^Gxw(C<0B1{c)X26)>Ctq=Q!L(rFO;ZulA}tqvWBzN=CB!&8#m^FpA!S-Dr~b};y>NdY&y+)Vej8_N!6~c;aiW{-wq7yIax_( zO4^tliV9sa-6bm!{jq-UaEk3I+QxBafs6Qn0mNhx!R za~?ik2xUFdXi;Iy%CEXn&5c@NO-k2)=iL9U@y~ISxQGAyhyVSnmd9U5)YhEn*B4U5 zmQ0hi=w6VM)4kh;k9RdzX5zk0mMbcB#w2Bi@me&RjFiqsHm5K;X4^uidyd5STPjd8 zwUwJMU9Lf&h0Y8)?^oPqhqRv|W1mF@WAcSlP^(j~a7GI5mu(uAthg?pC<#`H^M^co znV7!Lry4bc3mVP}E?EG<@@NJl+KtA$IemwyzUl#VLXGulugQ-Gcp4M(lN|dE{B7Ev zeN}mVO~x4$;Z(KvF;?Z>fny*TE5u$U=IxO=td4U0bTgJoIWlW`a&0@U_4$sBJXn1^ zXS{v!OPtvJ)9&L;1#jlD+C)4YRAA7Rc5*m9@$SK{Wh=u%A3_+t^Pr|#bn?3==sa&8 zz)Dh+nk3j4n&6#+RxZSsvren_CL-eCRvwpJSNz8qwTGu0+ji(yR_q13^|v9sDlRGc zZh@aA_5+}@h^$5m{NR~POH09YI8~pm@s^9`D_r$%OAZ7D;f78!Sef@NcK@}NmR@y> z1*I@Eka4*p9e4Z?4^g7QT%c8X9huPtN*4-ZcvvO)1tTNMg9i_QRP`P<-O+~iNj=FVGscyPD0{*BDw=i`}uW#Z3+v(v@s zaevb1SogZiR#esBRsyaF6d`?qYCK#L9vABySuzfTE7gK69YK8C zH;L3+r}b+6tsNYmT%F7lNIh0zXJ<(Exspu65ZUajU%y6f(1EvRW@tL%fl2$udv|XS^+Wr!*VV3%pk)S-t^VMi zyyUX}Xl!Q}J~gEy37WoEcp>T3cZ^I-1XXx;#=<;a>_gTtLxa?DvMmS5QsASa9O_Mg zQBTsMg$}OAOcuh>w2NodX;k_c>V6W1ytE?Km6YDVap>ydaXjgL{wXn$f`dzGn;qgB zhoB2|LIQXZwH9RfNXI$EtMelp0oNVh$mO-Q_i()ytbCh_4N5wI9&>cAkXrzfoU1r= z!*msoS0DQ(+;!IEn9L} z1wbUD+kFjOlTa9X_ggeJ*$wzZkK--Lf#~SyoT8#2Kn7E3qk}2|=N~3`Jzsr$NjmScq;)_>kko^hq@}ghxh@4#E;@Bil5@N}Hy}X^kDv!|>~_`4 z62M(uN-OqTI0hYDeUUXgV3V=mUjpv72SkYd?E9abAef z90bAl4EEf$dJeGXmUZ04#V#u^ci8>&Txyz*{H3z8GQBpQ9K5;;IyxcHO$u89r43K=bzTT=sfCeBw?F33k$Yc}&10tJDP6nm;d z!UIoyfTh+e=21G;q^dsb90gZ>=TEFhr~XIO<#>l z4b03i!Gr@Nt>hK#8{YlCg;X1`0t_Yprwo9Yps^R%cL$&m;O&cVbYyv`JX2jZAq(Xs zt~;|9g_Aj~81oJzx4};j&F+Wd4<1gQ>!iSHz{--31D~r{%D;;76JveL%*=I7O(W_v zn@P5v%i}==0xo>Y2X-l5YTR1!XtI4m{o;ewW}42IxkxH71)tN4!YzVN_o??}@40KQ zU*Zu@92m?0{yKe-&LJ4|b1^>K7xQQU*BvLK@N!tGn{2T@7(@%VL#GTaQ4 zQ-K~%^H2Du^Pt-gxM-5h#-*6GOl$K~?~T;qXQV{0#RdJ`ZP^QF^pjJ+6bPEk@)krz z98FUU-IjEwio-xk141f8+|j%$g?C~jM#zRQbF3#wSGS9L?E*p*+&yM(S*U%QsDfMS z2lhYMMX{^-JVD33t=k^6_RCb~m3BSNO3WJ80Yz+?d!Mf@XaL=r>v@>!QV|~M&rD8x zZ|5bCvGygS>zZCQ*O{T<%Cv0{*VlSVk@sEN-&yvq$uILB-xra#!CVwD*lq``W~npsl%OnVD7A`78!7%Z?j-U5*8F3^`WJLl0>TJDfVO&yL`VPdPm~ChD zviG?TC?vwd!e@Bj{qI-fpTE6v`QBXk|0f1qirbew>P>oxiU7z)T4g1Usp(o$kdo36 zX2#KR^|PmbW9E89bT3-I0)FRx76S_p#UVcD;$k5&nzXQRo-qd80$_;Unz=*H854aN z?(jJ6m}&4{>0v70`--6ow3Yh1mindpAFqGdcSEhj#4?Ld4N|CP~H_W%A?MgY5iI1XU4dqx?39X!ANx6iDJoX_!XP zo|aNEb>*~IE%$v%c<8bx1@;wQyO34o%POkd=B}=r6wIAet_x!$c)(Ou#d)|pQTNSi zeI)OyH--faFiHD-oP_xO^i|12vlx$tHy@X8B`DOfWTfTgXJU*CCW)U(dDI_}#Kphw z)5O5Q&?q;>JKFmF0iI@E$It*VR_n<4?*&p_N-!zuJ-UVHvz*=+cw~InVbJ90<|)y7 z$b;|gtrq2EYD&oLGOLjXHzYHCI{Y;0H~ z)!<@9Q3NI?rc@obpE=bX_aZ!Z4E4KjSI7T9w*EVu>;8ZLz+d(jnIXw2%HD)zk8C1) zlw=dKh3u7`O;Kb=R`%XyWN%55y~_OEpV#}kzQ5n+IDUUz$8}uQCB1sRo{#(Eyr1Xo z!UA0}+u$k~hr$bWOEq^54Gl@?=!%egKA@x*psOfB4Djekf`Bmi#~Zzu{&`?I zgh*@xnP*H`Sj%f3A1`8M>QfK}pfrghnad>UWN=`BNKI@96U?D2Ld+zdJ!dCJPD|~0 zK}Rl}oSb@4j>XQA7;%M{nh=3%K#C^*yN-?yvcYGPkN%x3NCx2{V`FYUzPFfbs>;f3 zpx^J0y@U4wfaD%*`mU40g5$M>k0`7Fg1?J@RYwRIpsB~XfQT)_7^`~R@bTki_;X4@ zxMY|YRve*WVg9A1Tq0wh?X3aFss@K1?T($m00&{G{d-S*ESzzX{%+LupHaQNN-oc}Hp;r68K5)xy2T|W$@LHr z5^7VM21u$SF<;g%#$YDBp_-oGMM)Zb2M~w~5F`2S;pN4|&#!%#h>@fcQp_|V#8>0` zOzK?NAYF0rm2m^E>#P7kh1fJa2Q?l%K zf{p+A%PevW1{zQq+L}$>aX67vwnYiPWeC8+0bVplpWgF`k(Ct}LNc%xpvk8EfAP4p zKoI|*J>vtZgVhgi|0KBlmtGa)V+W!^Oc+_~gJA#>_$w+fLfWpoJ^0@hhyM0|F36S^ z?S?L32|`a_!PG{LVWZ5R76lyP6clzO__Em}A)Qk$!gQ<6lK;3Hp+5pASxy-?Y4!EO zG>Nd@UoW^<7`mEC8znSfPl%fMw2lUSjWnPK!y15-jSN@`+m1La0)+P7J_SB-MF74KB*=*jKl5eE(QII{jr4&)nYcpDNp842rhU0s$FdSyT z6GnKnRpl$RE51%h@!g62{5ut0mg{cB{@6W?@Vd6U}|25dJj+$iW%I>96bo-F=4 zVnHkSS*QI9DNDn0=Gn=jhx0C#>YwPMM0R~fzDbwH*%j4}8L6o(*k>L;F*ya~i09tr zT?bvIU2n#Wvcp1Z?uS0(^ZA40=7xu?Hq#b{snb{5y3;wXM64VM-1W1PQjR+I7>j?W z^uZzRmY0}HS8^_AW9Y{y|8lo;G>ESz%#(}tDKpJl8L2byWuls&%q&#had(0Jj6iT0 zp7NV7chqe(AWMOIQB!ldXZ2G3hG9x3x!9<;0y@YV@BOXvayuP7=C(IQ>Du^46`XSBc*k6ffp4qC$eO#~D?Q8MwbF_zS7kE%SeE%+Nw+75Ty5FbQbBkbF9BOAXE%Y-$m#nz5VBS{SoSg0_N|z zGgOxE)KLvdW8(u>Fo>Xf48Ogst0UIdRv52tY|!%~VhR4mM+VTB*VIDo?KO+QWsm8J zKe~R$K+FU>Nn$f*8Qzb~=|0->LN(9M8Uh>OpIP(23ptNCQx55WO8mb+_|o^$se3_e zi8?FvpY~!5bUXi4FPVZ+n|<7u@zUhLFXO5;+E>}HSRBX)r(g~U`!y=Obp|Q4FY3xN ztL}{(ojSWB5{)vOUID%umQz;Nk1@R*qM0-Nyc~tOJ>`TYADHBsz7l!xp_T9cuWeTH zP)9YKy_VZj`m3gE1a0)h^BkXkU44o?hg^K(6SNhbisCw87mbLD&=c<2Szu#Bc5MK_ z1P~IFU6HuuB}}Mqux{U2S>={fqDtf4KWBV__Ii0KyxeMtXPaL1?MFQqI$`c>6;Hel zJ~WnbdxbNpdwUm5+1*e{#pL4pb-uf|2Ws%wLc#r9*oiwf!wCY-X9@MNaE4>(XMt=( z2pHM{?COj*+uPR%I725T(;KZ{;ldOXH`vz~Pw-tndzjB%6=-1q&rL}1tzWw$EUUro zh2FfZF>V$d{OjuSHC<2a7u5JIMf)lb9#FDCqK}Y@D%p3QE51j@#f6`x^vxS|C{8$Q zbTHD=(D0AD{EM%+yIX-Vw1^2ht{JKiUklh!uFazb^sOub{lS@W389q+Vbylg7J>fy z4{Ga;2b}2OUW?1rm!)O}N1kfBus?|0eLhqqB#>wxgKhA|mrmTaKmeQ}GCE`hUVYAb zJ7o1*%!4=6^B0?*{s*cHI&n{asN-hyfcvUGjwg#)=GQeB@PDPoB?B0%LXvd~X*7g$ zbs2GXsYIQgWXK;$aW$RtqGMq6)VZO+29})*x38gz{91+gjG*Qs#p=*Uz}4NSWy@#z z_;C=ll`p~m1$4G+a54rXd}x5-%Kd_oO>&;G6q}=-=|7C&N+1 z>H*G(pn?M505_80yO5IrPDb|k@=06RXlKpXT*ej$ei^JjVrEX}AZ6GynM*{8+(K{P zjs+DS;qC5yxeMdM9q6y5fV0nPzkdqs4q%69g{%Dz{MnWd#!b2a;lbm%Zv{z2qUKTt zLV@t`@UmyCsxGb?7+fBnp61AfSMZ+;?@vEC_K+4t@4W#bB%G0M_kmqdP!Rchp7UJi zfFTW~sv3Alf%?Cv2}1-Nm(Y0*?^mbgP9h{=JY8ki2AGd5ek2Hpn1w{CcON#YMGP0+42jzt-HnIT@%GCUk7cJjY#!uG__zp=@y zcgA8bpB{IU4jh9=1U;-V{??w*1}Gq@8$@MWk6Jf;IUU zmyno4NnT#7rdH98WiWlHhzo7udk=cg6A&!_{K-Gkz-;#=MP^foN3NU}T_MD_{E`{f z-R%Uq1MW8k_Sp)|ZxVWP}Zb?%v|`BU6y1_zBl;rI;aAMK_SnM{l;=^iR_xaIkCM_;7xC@NM&XTADU zqgF{Iul}$6$59;G(`$gMn)vG`=Io#(fZo7JxBXp6tA*Fpy52;`rOG9Tzv98$j~_uy zH$>0x%7W+zQ+uXnV`03fJ+p$Mgp!ekJUb1K>4~ZcN5kf%=j*oHhG(KuS=|yHM1%>^ zpJ!$o|880b0<$V1xmrmA2WQ$^+rcp1@_6mz`V6!~CiJFnxkdqgl+(l){rr5`X7tXM zVat;&mVz18!y}Bk4Gb+U!GM5V3z?CVnqR-%44X>MGqZ(g>d){ZzG+=+yf~W7>T)Op zm1-MS*laZ4j<#&xO_oBN^qNQxi0P9wCG)Box3Y5|uV0BKTzPUM2D(b%>ub1Hy9h)- z>gmZbCI}4RPmBNWJpAW$q&IQd3;sWsRpf`Q+J8@G0&Q4EI#9TKjLKvNm@VhGGC1JC z2bM~T^}2DL3+9~P1#3>k6Xi@~v_jDjK4-jpo%)eED=p3AmeOU<-SyHd(ClUs z=)4y91>7B>;c-+62DQW2zz=|rs(D^Y4{mNI!7-yJb}aiuG*&3`JKv%**TM+y=AlrU zs;d^q1Uc3j3dgis%F^ypPBQ1kN+?OFNkwD^n>7B8=K*Aq{gXd!NeR}B=RGd9@QL5s zf(m8Ore|ijh)awz=3mz5;{S;pDJ^?jj3dyjqUgzCJJYXYAMaF-YjibLWQ&@*g7vAf z0aau_P7|)v7hlr-p#l-dRPekz;sEDkW{h)2JT`*h8{myE*|ZFucHT@Vo`UNQ65WKa?|<{Db1)HP5qJHNW_X2*ix^@^PFtB1xaTo< zHy6Pr*J8={y|))0icfDfRhN}DLsVb=;omzfckgET|Fm*4^*5-tzYf6As;cm)WiL?= zw)Lzz$L|W99cB`YQgdsY2jgu!$Lou?)(Z*?+rWVa?yjK;g|;69nmAKX5HE(TF5Vs* zHt2_A5k$Y}qsUrKxi5OSnGu3T;LSf}OiGA-wJ+(f`H7noDjl#d9X6#wJ$N|XQUJGs z6|wDX9R)n?<4$2C+XAP(z5T9*0`0LECovTS^udQDi`VJ|{ByCh{Sj$Zg_sso;F3~6 z5YdFKr53WGvnjgExCtIew2qHcxg0l#;=|r{8E)ZL zRxJ`A>@inH$Hy%IHovfwzlD;PlH!K6-gQ@B7OP7!U2QFaII+n$xqA74`2ezh zMkHFmU_Nv$Lx5kygo~@j94u&Hg0njCj#Yhc(VM^~?4odd{uT^aK*^L=|GGeu#!P50 zB_;Lh)hhgsAHE-r`WOf$I0EyDM&aR0tG%}W*r22^=@In;w;4BrT#wz08!{rYuw zLxWMAuAw2}?b{xtdalT=7l=o&!q?SDF2>-%hYc}(*@;bnCp&w76&@`6;j$GoI~58S zNk&F=Aa-{5_QLOs4jS?IURXsiL2VYm{^d+f8E)KvEty$C8cc^+;VL2eqsN{UH~ex@ z38ZzFjZIZsN?daU+W8db6Jb$ehwFL)o5+QmWdNLqX04cK{K4CLj)0J=tkRTid(!{< z`tvMl20}sjSR8K6b5#pog6*|-f@M_|hCdv~fWwEoGK*hEMk-D|T6@*9y}kYR{{BlS zJEo(fqyPMb_~l%DR#psxNX3%f+u3Oer?}nSzzz>PLI&7*Baj0K3MwgS<1FNitWk?N zGb2I4%}v#kv;Xc2*Y7=faubFHX1`CLKEZ~3=-A`+E#d$F7Wl#cS>Pqr<^I#Tr>1Z# zEIk(X*Q8L*B}SLF{Vu5Ro0lChpQ1Sb7k<&XQ&!jq<$z~o=%vZe`Az~Z^jz+g^b;>po^>?`%?eAaKN z1h9X5lPhEGRpprT+)qg5dSas8ME&2A`**w-mdtPHax%9QGY(DDeETachcWd>%3IE`Mw<&_7qX9!b$6Wg4L=VB+>w8QWOeLdwuN>it^GAp z8|NRhA0*}Z)QXLrIy=m9u{5J(ws<~lq>cc_WkG(Nh2}kEzi#L+ zVpGsy4|}ih*w5pT5PvK5;GJ;v4fm~4=i(|q83XB?A4?Qtir<#97|KoVf8ez1se2F( zz#q#8-@{`uKF&8X6jF&|6mCx9D4_ZDY(>d0 z>_(v^1K^nFI)P0&Vl_Rt@Y4zs4_VEXPfu7<+hR5;Bu*XcTkT}Tng0l^1wx`C{f=&~ zMjE_Z!YcW9Cg6Z-JwJPB9Wx8&`T)snVZj&d4MPhXYCz9S*q1EoIXU@5L`h93THnB- zZo=hT`fYYri0DMVKWi%r0MXYKFHWD)G@c2{tq3y4;u=&bl-hv@P)7dquUzrK#GKQ9 z8A{Nz8)cqXTdo`1y=tKJ^we*jno?Lj`TycB;Umah&tRSSe`Xh4%vO}SX)`d-6f;U7 zxeB6$ZbH&3B+n{-kWWp2PE7K6`Tiz=s}+fu=RH>Ofj~C$!3VwvBPGIPaj{%g7Y+_1 zn1hEtY<8tYP(?Uxn^yJmHeF<1=LyGtaWjh!3j@PL-am}Yi2G?=1apJavWWJB-aEVn zZ$2BK{3V5gUt!+5b?VG>j^{@$=F+_U$<^ZG!YJ@X>8#Vw9|c0A-;!Rx&s)u18l^Ej zSMiW!f4ndMd9+2)+-yeksd`O#5qA2;PDwb&=rXDGr5N?6rn%DBZf56INf$!uLJHSa zxCy)?0CP>sjgR?}&ptod+BUhhWo6iu7th9GlsKNNbK)ZV%=Pnwvj-_mM3RGo=B@3+ z>SnbV7xlqMZEgzZVBDlgs&E#R-o9TzwzhV`^0qEparqGZNvtORPL4t`r3Z!%kX5a{ zmb;uXm}{mT`J+WRTHB&_5^<&tfm+tysL*qHQ zzbfI##iM#&=?g8`^$|;++2oU~L}R}2aD5<~S8I>?>{(*LIy|$N7X?)%f#W777`Ja1 zgeWQEVbsvXY<0fY2U+ zAONPYY*HJi7VdaR%*)f7B8Q8bmbMRz{7Y}|u$E0QNMlBvdNw*biJAI*H9Y*ea~R;K z4zdaAy2_cXtVm0pPf72QUQklOvAa0CPR^?jWlh+g}MxaWmd7PEOamOi4y& zW+DM68R2~QQs|cr1ccr4Dp&+U^WdYyfrR|`#x%I{LiNnOPSm}dpye|&Gn>MFFTb=j z`9uPWU*NhL=arO=2Vu9einrkr#*N-rq^?8eQ#~dsiU^hltR5IUkvfpWV`9R%d%BGQ zA``a1Lomd&H1qeK52M9u=h?6(AvaY@fQa|1#$ zGBSj;g-982vfq18_WVnW%$M4()xDLmffzCXI3o9b(B_f6f9teCd3U8ctBj|oD8euL zJ=Z`hD=Yi`FNaw-2qZF!@iyDbz}-ynI-h-}0{nu-9e3w1;xLDGKO+l}LrilpgC*3% zmcDIbLa&DsQ)<4uuTR?A`WDL;rhQ5rfEO-ty zwwhwNUAlXES~A7Gnpd@aYGH&$VB{}4%-JU6ki}tLE+8+cxQyGY=Y1pELN4`*YsGcI zWTy3AwS(57-XC7imbSJ)!;}B4$AQrxC_&(m3EDrAaw7n027w`v#>9IE3mC5c(4dW^ z72!pv5x=DM5^zTl5#+X?5k^>`NUzWNVRaLfeH*s{qmzV|He%?ld|OKk+WmL-fqZ2D z(@qKh2S4zwdso_c>r={t^IyG&V{%c^grg3K@e%Ygvq>=Cgt6U+04Ju`sf~zH~s2rOsExkcP zUy?Ew^jm}h9yxGjyrsxxvIz`}fqhvph_S8p-{0tz^xJ7AxSp^NjbYj%V|Aagz`qqi zf}ht&z|~0orm~(L2~!%R)xX<#_L{J6|TwUL{ zEBgLdD-*?YxYXRcx$*nw&(2@p-hUk)#zJxPm@TF`yxXre)&atS^GeKp{)A5-*qhOq~!qZ=)P+pN=AfMCmIkRPT=t$Y(AMe9@p}3xFQN~|E4EzI~BrbSW z68QcO3yP+-GW8pmq(`NR6dCq2uf0jIesnXT!0yXg9}BB$VeHtlJ+bmlzM#iJ=+_VZ z+$oNh^0wkfptT(N@_V%N$xpfi?JAa^-p8avdyKuk@m|5QV#^SS*sN6kZr>%cJ zTHt-UEge3*{*aLoHtnZgYnYC?`;LJ{_CWM!vdJ*pYSJegdCks0j*rSD_?|+z z&c-IWb7f>WfBm*>-74S{>vkaPga5Zi`*%3O=nuIjrx_`K zany82uVx!}vzK~^08=XcuE|TMWw*uM6-{B`*Ui1X>Kd-(jei(vqNB~1H{RW#fAi*W zp|ol99)tQ-_d=+7OcR>-&zg5gaO>uUIq#=S!NTO{Eo0edd}?-S^v~%JCXWxEIR1X^ zZF(0qQL&09XjjLoXz=jL(%DfLTk*5LB)!+E;aBm(BMi7L`&H9L6H6;knRa(+cIGeU zUcO%~%`e`_n83LwU|Z*`A<4vt(eEekX;@L3yg0r(_(s&E){KwAt)5-6v}H)m!E}A( ziFVQk(VWFqQi*x_IEkY+BZS%oF~OKv0DE03Nm}|czU+9M_9&xTne3eyAI%tVXTPr7 zqor=Y%R`fIPG?57zeSJC&&hQLF$T?Z9Wz|EwJ0C*(|UZz*YNq}hvP~VNr`zSxmbWY z-R?LIv~u|VfuoeDr9zA)i(u;k)240*+RiU!gYEUb)vt!%h*b{QHpE1@xno6d7@dk> z6(3;u`PTR_&$ZK}R`4^Ut2v zd#9sE5-+Vwa-b1bVZ!Gc@_*uE5o>*t9r$C}L&|)$UoHfDeBk#>UfW4Wd=7zO-%x!4 zd`h<)x2kCBCTdccEG^2Iebya`&pr8wqbPQx>plWy_WOKEdt;MHN9NHZx`&Sk+e+)5 z@zf+dQq8?faV1PbK>}$3PQuVEhxlBYAXZyfm*#Xub^ldGgvM-88qz^N)o_Uz#d=9q%~+*|qPl z;jr1%RnLXL$je)DUETYF)7BNjq4e!t9oett6*l3xx1Tc}{$|MNn@`95KY&1Cu z^?^7vm9v=z0Zf_EL$7ZU93fSZx^DSpEsGUkMZ;8*LI>cr|@Ax z*j=c%FVAA<(|GH`@1hAbZ8GE}0e?g7X@9?$mHmT_1!vd*S`LW5tPjq~M z*LW1uLMUm{pQa01F2##oWl>5F(eNenZyV08t|7G^ti)G&oBc7&L*9*zJ%Ig(_T@*8 z+=Cs9GLIfHgOPCUP6N6O!oAfs+h+$2ia$(J-~BN7CjN%i%RNGZ^Ywq zaji%RoZSNf0|YlV53n-O^Ny#0N~ST@YhDg{_>wM4qHCLT4G|EnRGLd*(=s*Z70W6n z6od_h6UY@xnK?PvlQ@lFdyafd+MuHNBR)IvG>{|O!J&vuGl7fu?d&+J9OgNn&pqNz zK|+z8QCjDXagtwylaeS;LE)-#-+43RiLC90-pGPoLa}t~Aq-96GGz4tD+j1FTdrvk z7eLUaE^cmu(4XE^@tXi#hXAgacLEAmWmRv)E1D};kd2x9X6-7N8;2Y=81lPjXEWkB zZ`~TU`V7lE=p-ke_c^+X^d)W^e*-iYFelpzOB=&sw-Nh7G>{yUE42bnT3YeGG<>1i zLsrP218|W$r+Z(RSXmVcX5}CQiWpdtZW7Q@vc8Sy_f)vYQ9l3Yl%YJq<@7%~>4S~j?CFbhW@CeYmpTR}>8y=*cs z8}yvJia=e^r}_2+=)LG*EO}|1nc-M#_~`NDMX+ihyvl$23cF*Oz+$0eb9931<>dvu zQ*$W0gO=~-JNqMQOgC;|BM8hhwj5osqoQEA4V085suOhnt^gO80I?^KbA0J5$4br! zVcO4b8?6Xk6uhO0TNFUR{0BpR{HqY*Qh=8+F$u|4{Bsgu<3mIS-VnL(-!pM?5_=u~ zZULg%4wUMc!)8d>_f*(^u09OhvOz7A0xj?wgE|!&8rlrb*D|v^eR`yF9=Zi>p_?$uDubEvm2m^E$eujSbb|Mqg_Jn^(EVUWW1M z$HtMYE|2U#yM0!A2DFO9T#fFeP*3g}USgp3v4VCR5Zw^Tn{(KBGx8K(X>BAkfq%ls zl1EeV&;CUtn_WEr#Nk(NvTn{N|9aYZvPnHLMYCQ+gd_}M_m$Yl0#XRLbnkMSBw086 zLcDWkmgBhI4;kAe-;Ojx`HBOViR7n%zl=XAmqvxEW1TfWkZq7!#{kMZaqQ`qUD!63 zrqMz!#$-(yi);?1wT%&uTfHZ#O5#Rb{Z|0#=nTtG;ys?&tG-RL+i^C?`^fz4dHFWs z`sb?XaXtC3DZ{?v`=#qvMX!!hFkieGwLbm0Ls(O6l>O9+ossfy>zMTMG_h~Oi<4w! z@+6K1-4H;x#GgJuh5ATf623DZco}9Rr!nj`-xP47OFmA7N^z2%I9k+En#*5NZlP59dV?#z()YDXxocLN_+&(5XB zozN`@_g{H8x)fk$c{^)C6V4%L{jL(7phII}tl$YA_n)d;F<7+GBH`5IuB zoi`vdhC}$zpUXu?Uih~kq^m!CK+DLKXEUDVrwZ4-sKH+kWc;o*oj>|DdZt&lfQvA- zo>xmgt&NVhhaH3n9Kl_EZ)BMglE#zZ0hzQ9KTGoL2i*hDn7=1mR}kF*p(DQHJrCv! z+(}Uc9@orF)e_Rt;dS&&J87Uhzhm0LBq=u|Lj~e0*&~OQmD{Xe9sn_4TO;-V*D;EI zuYcPkUgiG?V4zf!-oHN~>|%xXPUQ}R)5|;mOSQI*CVYj}u&=;-NxaP!2{RLR2_JzbUJv+>SM8k$00-=|qxRtBSo2IzXG zU?+i(<=eVCjGJmdn0U&iG&AKed{22E(bGZNet> zo+EJJnRFt~F0gX8apGv_Deoi#Zcn29{R>6p%!KIVIn8DiHF|k|h;_y@i zcEo5X`>TWZV6gn7;^H6QBoN6#_;q0AZTTA_mMfoF6$||pFxgL7mCP~@=aTexwiN$) zPPh;ZX^!~cHI4|3`YhtzDcaK6Mi+d8efu-!PphSzcV$-qgaUrtJ8fPnuWJNR)zRNy zjNfo`X{cerg29-SWEorQHj@juYM6_$VbYfR%JC9p;iB38SdVf&yIDfYt2xi=ouKXj zmCaC#pEWphDf%rQVdTThPZ;3?%dXIZGs1_~aTmJr#s;Q4_p(ceVo{ z^Alc9?RIOFZz?_!dY!W>F)0<{JH?Ok#JT*NBA_}wF|EduRv3z*wMOe zo~xP{C-tWwmCLKO@W_mHEhfxdBzC-16cti5BJ(JR~O^Y4HmlM+E z3})9W-RU8Jy|06HMFRN;>Q1?R>&ej3Z9!4QghdEM!jbj!do`-;`t)1T6jwI_N*jk~ zGz_$UZ>Ve!2=TzYv=>*;VKD5tZAzTPz9Xytis=4*z5LQG8FNOmt{B=tu7j%~{{Fbj zUAhe7;zo~Oq+PvZX?FT&7Uaz!klTR(hCGM>aOMvUkW*82z$?LmOfy!@P1kp?1qX=y zErK*+;^uYG&k;fh|pJEoyw|JTv>jyz=+VXrD&*b*TQcZOyf z9OCc!8(By5F)cmuc~Hb4_QgYt=5SNsWqdqJLIPdP-4UCI$1oxB(=zOuM9c!K_Imcl^}0GuDbwDb*q{W^7xq2T>}{h1n#ov)L@ zcn%J0rm!xQH8*F3QX1?PLpoLNl>5j;1Uayz?*Q$9*xYTvp9a$#jCmob4uE${|cvrg+K2KYE}2w-aPX!{aP@Q_4OLBmc>Q@M8hmab^3jJh-?Y_mabw zUNR1nw;}!g{r9esR2CKzJl|Bmni&`vX!ht)?jwSLcQBMM!ML9w1`AFwFUzTMc-AlM zxC^^)XB#q9%0^Jen$881gAp7&O*tjew-OMte6?iEOaHfk@`CJAB0-(Kt!<=E;e(cr z4p}we=FC0Uuy$YAUk_{G}lqaFe&#SBQy; z5p(xD)r@ecSX_XxcHCY8JMiI<3^=&iklilgVVikIg3U58plyIg5YZU%nk<;IqeDCG z4s50`3#Xz`NCY}pK9c%gC-yxiBm)jelpdsUruG`cJ}si^DHD7v@d4`XBO__*eYgf( z=j4ozgMeBYsPM4dCRCuU`}|o$Uq2j4LXnea1Ks8^H?ypszW%>#6mZ8;msOF`UxM#3 z1dua5QGJkVVmGd*b~#?S#4QbnuXG|(Yy^oeXr_-fa5!oQ|08Vsp$hA!pw@r8s0K!I zeCP;&0Z_Y9pcMa)i~^$h_;GDAFEEDSBKc2pnomFgE9%_^>c1scTgLe)KUQ#7$r@Kv zKa^CF4UgEMp^i4cS-^gR8_b z==QZ42W<-;4GvH`?_fILXv+?g?4?Tt@N1!diWk6FG2pAtJ$<@U|1AzZFq2SnCQ?Hbx%sF z_7Oe(-OLzLrYegLd=pHp$pAUBFDvjkQ8&1KV6wmau{q|hs+Qh+SpLh&1rPfi|M_DI z&0TH7a{GaoIv3;B;v9D$p?UjMFNfVcMgB;!xC+XJ9gRK@FCBbue;GVgFv|R!2!NH# zE6K|UVe~b%F8nS)+@_wLMc3A@gkES(cru~osCx$dMRMg(_viA_&(*UfmDh3ycc#fO z;5d%*wCq?|PQ@wEpzP{I@cq%`PmIe_q_SG0U=U}y5W2Uq7gy^LEjxMUsVy-Y-MDW< zGdxJaXq9>gvG z53wLVDRIt^4wG0=o-nSx&uu`lC^%x@Eg68pbDrdED+STKe`O@WP ze>!$!_OY{#p9@UD(G9t@`sSTn(d=o~#r<}iYlb8F4Zvrf_{c&@E0hVVC@DDX5oyrG z*8|88UTx#Lbb;9gVIeq|emSiur7)j2Cz~7^w|#^6qLWh}kI(VFupe8{B}s;fI>wK;}JwXAtG&__e~dqN++RVczL@ANnoVEQ_b+yGolN@geNQ zcnv;Sfo(j^FY#1y|7ljjyRY&PaK2Iy%ZjESi0>Ciz2@s8Eq(_Jl{Zb11tqGqNCfkGNnla4^k9?Q=H}J zOMv{-*q9U==^_JH3_!Zydq1MC^9`6``03&XP!3hsc={ua&H4++l=F^x`J-z9t!@x! z`hVmLM9JZU=zpWK9csje>b!YL%fPh}*Pgdvw37*f(E1qaHswInLC!`=IGpM=`eF75XR0vnq$`0RM}YBd{p_Tkd$xHgQ> zZs10zb17=;nKKPbmtnjcsmmn;jJD> zRb{%a*zGkI%VQ}MXPb{JCHQ$uZ`5nI0xP2mawQ!VAnrS?HpMyLQ-u}k31C=L&Ne}Kc7qp-FW z-_hNQQuF1bc))3?)K}Rnj8{f@rR+eRqhyQTHiYVVe}{>KQ5@aYj}LA=_3Ah(K^P)P zl^fjrvW0a5|EgkuqGCQbZP6k)d~v%0nsE7bKDZq&?O~EX>6XXElETW%gW9B`Q+}E; z`yVx@2G}xCAB#@9>!8DU@gT3)4($c!$X4%Q8w*P@D_akC_8jR>!F%+b+2>kj*{u&t zxWrC=TW!001Fn~%=eVS*ip-$;UBS#?|NYRFK98~rvL#R(K?1}jBGSW zz>;US`IL}A6Q)(qNXbR--*lc@tci<8a!2y_Bp+4bgN!)pU+>{a<|uwpQUrGcVtT_D zU+y;LeH@er!nR?34b8J3zvZL?M$szX1o(&JW23GgZbWF+Y}9)gPc(RCP90)Mg6x2Y zx1ylHs%wxWhaqTB7X@wT*qxp1pH>8}gUKBA&8+gn2Ye)f2sSrzQ4#{rekQn9pPt?* zJ~S|z&Dr&$i3%3?`eD|NtL(k2co{l|G0{nvmXt`Sbl+6@QbyoO{xYPeqQZfxSt+H3 zpDKimNiMQEl$43JB{D89js%{JkPQ>BDmsd!6(R;)eu88JVkP+eoC~@SQIyI;d$C_H zkYQ_Z5Po{t21f9tnZN!F9W5>XaV3R?&;fDNU*9X6RgX5ND=;Nx_av~3ls-ltOc1YD z-NGTW|1K9Ob&AdO6fR&ayqi^*d9b%PqHG`+O@{~hM{OM)#a&L5#z>fQVOP>nsYnKP z!j=`yAQ46;*W3O3HJ5?u2Y4x^x!;o&Pz*xN36=%~p$F=D!YM>L8Zwx0KP>M3MZRb7 zy0QL6oFb4neJDiu42ES#r~-$Ff>I;!k`N97@c+d>MRKSAg!BxQMj=$>pn0`)VN#@L ze9|>l?(KDum;bq_z{_zRHt-O%6P?5{`(tiph3n_%_sG&xX&1gn9{Y4>4OKgMC2RqAMB!V+hUf?KYac8`LmlpJ#Je(6mry-lOjRT zF_K^!2%#W8&ho)cA-tEF-&a;Neix&5_(FiV1ULuDuaTgC(@A*iarz+`zUu0&>N9-k zanRNV(hCXF-GE%F?c(mzM%#9ga9P~L+aUxUaN%~csHWy#cxrGi3tl+6Hu}< zRH?#)`0FM3J0qu{xDKWT$iPCN_@oP3CjggC2PskJL+gJZ)-TaynJ?!6AOd+KHbK!_ zan^ITKp>|Na`sG)>R8(*vFQYY(bk=qEoQEE2;gE6R(G#JCyBDLtNnF)K8(y%6qcB>j-io4wrk8M|B4&2PAzN(FKn zfB?4Z z{V)~)AF9%0qXQ=BtSUO^#cpbHp@-(q?ndL_-|OKAxmValANWubeR0GIn(w=O;*ASJ zA8fa$NyKcG?KB9iBQ~F&GX=AWyM}kM)8tgt74D^~&sx~ckSX8P0s#=;m^7MZe0uC_ zfjVE!&E8%sQToK8J+XRwBh+9f2Re%GPEt&qIzsB)IK@2{2?Pit~TWa&NsfT{Vzf#4|eEb)XZdn|vhImwaNj+{tJ=eWPJh zSLah-aW(y#08Rz7zh67+^S|r0fB%Jg zAyaZ49-IG8X8-(aU&g%#V|X~T9Y)$N*q?)}M|^N#2x~03<)bSnMK;H)VK^cOI*mgOZBx zE8>CI{O)&rx%(c%IS-#c1VhwU^*d0i66Crr*D^SmtJ@TmO;Z&d9KNjk$3naEHs?i> zKUA>&v|ZEGPkP+_!mI46Tt^L)`@8ogskd^0y z5bq>?`^3H3@1srACuOqV^}_CF%~CTH$w9PU*Rl&j7Ax85aL6Jv7sE)Gb-oD1ka*Sm z*y7p$?vonMvE@&yX5&zPJF@Wo306}WCvL_%krozlm=B9%Do*C4c_^6)bIvW@IlV-? zc9{vWYyH0Bbhb<@gJ$8}{1z4?(IN_Wv|f1au;lmJ$#^kQ%^FI0T2UR*<~(b-ei=z3 zgDxEMs?@tn?uYHm{=?SMRVh>J0x^ZeY=cnqEAJz_SI)UBG_L&|7>o!UKjLc5Sguu= zIYE8Xu_}U!PUsdMIi;!Q14eN1sMnqc} zfRJ*svn3fRS;1fnOl2tRvQ)sQM>&nVvLFzIXBuC<&`GcCJ)zy$xnzcH1 z*p!2<@Mf1fo_OFd04xE}r*%l9di-q;abOE7z6Udl%SfJ$;QUm0q(V(0+oVHVSF1zr zn6KuWBQ$29W@m>57y)HcTI!pg@$?^qL)nuUm#b6IWm#ESd5`vy8m!~l*uvsg*VhSb zZ2msM$K1&O@IeaF8Y~W@`5N2%znh+Xz+{8^0$Ix&I=NqXBfv-`>&D*7le6(#-oAKy`pQh8prS&hsg0 zX%;(69Uv8c{p;)F?Y#hQ&f#HAMWpis5L3_*QKLqx4P_Kk55K@fngc^OG4U@fN*#E9 zL?E=LM&t6?=)61@u+^dgQFFK6mIT62kSmFX;*NhfQ;?rQp&)_M)6-MqdnP(HO)<>| zIp3E-L1w$(djNnz8)`}BLI_o+^@SkI4x3m6qdGe~5yciVHZLF?4cJhOfq`8?h*H@> z^0W;~$Ph#mjJUk=xPbwg&(hl#f974qJY%>V#=7c_f<|&(&D3q-*d#fkM~GZq{haA# zjx4-N`1ts&9xys!rm4I%$Eyp0j5AD_31Wh2QNUM-hY-y@6}b1pXenJTjd}rN1yYOV zVge6Wh{Qm5i3~6+AZpvZTu^1R5$5lYCMG8KZ)X=wv~ePGKc8!AY9bnT?I{e7{b@om zk@w(1{*?zPo}l?kgJF62xi=h9+>%fp1`!-no0C6((Ejt-?IWZL_zwOBQ~oUkY>IBO z^8k$<7*n7OF(4!3>i_gs9B8D_d-#y;Am-mB4#Ur^|3*Ggp;a^V!{IpG9yd8cHZvbS zh?)Wv60T$B7PI9}79^)<9lrBKL{kTeaOSF}>jmtsMnKR;vBs^K&b!f>L;mJXt@~M< z(K>~vq_l-5ULEJ6gBU&SVVvCqv!Ps#H9>%(O0X_Ldr?}-GFEam=kt9>@pP+)H zA%{-mpC!s(3af{rA}LEI7pX6zXyuQruRVAm_xkRVRlv?pmj?H+m*LNx{0>-En8YOy3(3=RP^W1?cw9f>PMn z$qvME)^R8o!KMI`1H-h{0L|VvjmPyg$CXAdYJdBPet2rx1tfu(c*YHC`YYlz^??A)M;l` zdnSTQ!#xshr{+1)N7+B<-&O+_QKb#ESEwSGpWde8bC=ye6TRGMis@{hP&nFL<;){J zA?9%3mzRXSEobHKJWPI%dT>FM&nYP(J2=Rwam>L*C~{?v3o2a$?9IC4Ee)!o2 z1+4326IJMNMnj@vRGj3BV=o5>EfHp_nm9#hfz~X_%J14Z713Otvg^9R{(322xp~kv zx6S&P@87Lq%LsLX25WVSRQpFe>z90w*(gJZSCBf)gap>cyY4cuI5qa-pi{ehb;9AW z3bqU~{VB^q5Elt9)a@gu=fc3iMpP0ti4ru(LA~_R#$0bvW~^!As)*SSJ>D0&iSiJ8_kOY~TNx7S-id zRWVLaedBXWO-%A){eLm__8GvZpXEd0^w2~%Mw&_psEzdvBDjQ={>2=4{Q1g)w!(pT z&uef{q0w$FD}A?F=0AOMMJ&7tNmXxZQ$U9xgSL8*AkrkZgt$sc!Tk=&On4d_aq~Zd z(VT=yPLB~jWOT%MQnT~vx9nx*0^QI%U}Hn^@F1VfN=}}P%E(Xx4B@Z!X}~oPTMvZk3su<@7`^51Ea4m!;3sCg`9_GdvN+f z@uCCq=;ZWTItS`?UB?nDvzP9PPtnxE>fZlp^NZM8G)O+1ZMFHwrg#QEGtx*IdGXQ9 z6>PQvcKzsGLzr`<(cIh`TkS&RKY#2ZKYRS~L8XA>(7;Myk7e^jE9y`p$#MS^W_+Hn zDtX=m3orx&rQhPloJcw0yQHZsoHUy$jZ}?@F;C7K4l!;$Lh6(XGuZumT~jXpMUF3< z&tWZR0PJQ)rMxB(aF?4lwfP`kI?(IIwj`t&KNEVo|< zP2P54rN?y_lhD+%ak^;KW@zb;Bm$EVyr)_K5{PnKY{h~jQ`-I{h3^R?FR$i`1Sb0R z%Vf7-WzD|?A}}>IH-NPRx)tQzJ)Dk5&*Aq%aKa#ZgTPi(muj~Ahzjo62L;9`;7)g? z@abw84@y4cBQh-cQ@8y6QCQ;bC+7XsA;!fPK~hjI@hdidB=jS^oKI zcFv^$YhO!3N(AUJHJ%5z@`z1LOo|L^FZ0vbsekEMY=$oZ6cx!Hucfe?3tL;I?o$#ib^+F= za=p(E#zE^Attfd|fh{4?W|A&R>$W))c6#cq=y|-q1}@+LdINr8fRz(iTj#cl042N= zf))o8Z(0VRt_8rVw-i5r{)~iQJZ)m2HXb768pg()Jr$5ehbmaY8_%}GtYCR-Hs9!z zP=ByK-qzM8eV2&l2IxMg;OR_QXaS{*t6~2#9-}x4CM+{}rQkyyCnhZ|4ZFo>WtF3b z38CEF@auuYai25)BSeieeUG31!!CdP@*K>ZOVIcPh$vu^L2-w25<6OifY=-#$$kiY zLjdeR0GhZ#yHeaprl{Uy&-Bsg(P~#OOfzt)JXh=)`7R>r&p?h3UP()sNELPgamw@T zN5AnmT@#ZS81k+Q3F$5#u~QPn-%x3V?KsX#1Hrvs;~7g4zw`Y3>`PfYC3 zMwgTVW(BULSK}&UV@c?sqijk8Sy`2pO6e97Zc$cbCL|#vdq(!2|Kq2g`?=ru{a+uS`aI7q zuIoI{-*+6x@jbpv<*ScQiUuE{+{kd8fu5eB>FOPUhXPx6C#JmZiJP;_edJcPuZn%& zCUzNyAL6q`1$IqgQtZ`Jac3u_PjxM3gzg&5*!Ik0C6~vB(%6H1bF|9cVT{cOy0%-d zRlhX4pkN%b)FK(wqJmkd%F(44+_+{%Cp)Pq4}H*(TYyJQ3%|kS=xBcKoc)r2ahr)I zL26Obhklb?oHly?;zenQS8utZlF|t5Ut;3olr=R}R3K)F;Ole?1vHA)^dlpqX^K|z zFEoK{u5`-gz!!wH#|0b{<60#V zqfoW6;XLqi=icmYQJM{N(I%FIYboaImflk?%`GijPVrBSQ)A8UWfQZX^3HQU3yz$e zrxg@uF^YIF(^6%ycd5c-8^jXY(hgr}gG~=citiHsy0*0RfI?P}OGwgKwMOW4cSNet z#c6{=xwDBYUmIBE2lMOgWm!v4KWvzj_~O{drzhRMJGMyLE_%T3n&LxkVltG19cve) zolk3DQgD37px8k{aWH=3!~D#lqsr?zrr6#W8p2=cW_i}RT$osXgr@st*EQFvp@`%9 zmic2nu3t>PdEFb$`&kn>JnWEJmv8>|L*>vC#tVIHuUxJ$hwOX&?81+VitSDBYp8EM zWsQoGziNGgi>F}Ii4!?GSrV$kNk0^vqGvo>6w{MAX}b9ASm!RJQ4Lo)p4Zl;%DN?M zes(9-%J5eW7liF%(k&>~zgVfig5~q0OdtGFQ_rFTArBb77_)^p4`80=t z;zm)~7PqJRRMCv%`nC<9E+od8DsrBm(48#?b=D2uyney@;Z<;(e^fUe4m0bd= zy#s7m&%}CIUM4V1a@F?jS`V-ERA_zA&1Io@<8+N@&$$xkm$^%?ZHEtRaGsfRK3y3- zg{4k3*RBao?t)?~D_alJDMt7<%Q;m1vcD)ygDqBkBH~j|UtXqMv4TZGVH&FJs}4@1 z2T~u_m+jl+Q+V&yD@NN8IpoE(N7q3fq--fb5Xqlk;k8=i>DT8Q9u`|E>!OVnD5 z=g%o){|Yqz+#=+@oq3!fc% znxtn1kEr8uCy@t2zEws^t@SDNzzE9eMZHiJ2=G59+nG+^Hg>Bw&Us7Lbhfr`!wALx zOWq85c~M`?S_Tz6lkow%E26%$rZvO5dmBb<-(0Tbpz_#O3)<)=ipJZX8uPhznKsrZ zE|dmc6+AO~{mcU)1Ap)?-)d^S)06@@p2BC-hf(Nl|J1YnWJ?x!sxL4rsjY~T@&rI5 zIY3Fp!NGx)yVh-fM2VVi$<1g9PzF6#X*iB`mp<}u);2NO;o{=LD z4}-mwFjH;Mc=4jEW|xK$i=X0G5%@PBEfSeD9k}wWN}`C8QOnfMZZmOb(LH+9OVxBB zy2h*!>y~)ZARP<)n)T%Pj{8wjeh}z;rNK*)h@>zEE6m4g0E2F0HrQ_ie+c_35dZ+@ zNluQw=2@u1=I7@-*NOO1)6sPdw|#^qd(bRGcQ zX3eJ%U%p28d-=0T8jeZQW|{nzjUfrK>gRN|jg3AX#Q_~~qY^8_@G%r(KG^K0Y!wn$ z*UzwJ3$LUiR%mzu9}&A7Q>Q5q3MpGiq-PwgRj?oZsb*aBY!7)O_#cFaOVIAm=;z_% zBl9A|lik;({jnKcMR*X)(3hq;m*j2cEfYZd8Nz@@dd*LsBDeH`!NCXoUbpa*AIZ7P zB4H1Y_i2Hw$~(XJd&1>3T^M)XO-aF)xI*G;3Y*c8?fcX&3q_aB`XwbLQ7a$aBjg@H zK>=oPdir~@=q;XT5LXeEOP!s@j`{cH;{a{Q%intb7E?~1*>^n$8UeFePYvoRt0mQO zi;A9#RZy_vqgtu{SlT-_mQUFg>q2#M-4;-B4-O3}>FKcpB)PDc=}Ue6jJmtQ)8pIV z^Yc3`;=OdfjWE`NO*qPz`fnJj@)f;Ot2!&m$n~u6ysAyIXfCnm^#pBz_xkplm z<4z(Hh3VwD7$ybgT6ZwNikB)Fv2Wp3@2yN%LR#eE&;7v`N>+P1jqi9Y=Wdz$8UC+i zK;1QmTCzMksc?Q~$bG0f@oYJE+^?oEzF6+?@>f)D?ixwS*(tZdsTUr@-J#95k}5Wh z)v11Vc_E~=WAh_PA*%g7^>yb8lVV0}j?8~8&E-7$>s9#QGeOz&qF(N0j*P#;Jd;?v zFKW!Yh?$+g?_&<+fqDVR9=5`LIlTi0yTKm}Xo78HlP}ny#@P)nvnucbgcCH!JW4;GLp?@t&Q2-Ute$n{^nVe9u( zF>jzZJtn;JAT0e@6kpU*7t(#g}0#Lqg+qLV;C*k`>D(dQrSZpZ4!Nl|d@vp_K1(7o( zG!$~hW4%K|oc2)wR&g6fb>7{o>azoYAJid;i`wy83gel^moMA4d@A*oF_?Pg?(QUW z=P+4!2wiJyed=ZGD>{7YR82;IEp)8dp@_o^gQtAxC*D=6Xc-R7HKJ?iK1V5;o?L9j z&CTy7Y-uA+)_5hWzmz}f(gcH2cY|L zquZ8=9Y|jA3akJ2Q1A?a>}^;z6vl6&LsU^w!9s(@qW*we9v)<$2dtbkGBY(!97bi% zWg`|76Jug!6Z$Ofqo04Q}^=kXEDR%?G#rm?c z)yE!uzQ2#AYI-quDM@wS*6wei+Jv^Wqz^+=j$J)HAI-%Y1p^?l;E#4$X=P=_{8cib z>>+D`+|t7(!=))pS;tYzsa>o$OJ;18wsVEQEs7A*c{x^q4rk}I)zYu6ib{#oxr<3s z(=Xk1dF2n-tXUcr+On6$J$j6bpI;?sFT(rPMHlwLW>@PadCj1$j*iyC{N&@1m(u9u zeqo^ezh70i_c6W4e5Xh2`O3Q2;vypV^fE_JoUY@pT^xV;ywqPwVW?ECaGk#QKpsk5 znkhAk8@!ntTL(M%Y`MY(UaRM@Ywy^MXe)VAAhi5;$LZJSZlX81e*LYMngqOYv9WEd z%hJNsUKT$?J$lOdi6r;J+=cX|+BU0E{fw*IaRX5MaM#Yx%@Ko9m$_f*SFc@5@RrWX z&L&+m9w4QJGl|-4pjF&wtD_~hkMSJCI43$U78bW$$0g!ZrlOK!`lRnD23(0p21)_- z1V4ZO_WuMCH>ths7!-|p{#=1g&Uum=3q9u3o$s`^o-fUS1&7BriJHsv3QUdEy7S(h8*V|BE7*jIJ^YuqPl`Oh+B)x ziL{s?v<_+_lfXX2DAj_$wzl>z3@{o*giuFL;}`9-YL<5N2&~>4QI2)#9-ZI2%V1-d z4BNlIPQKb%z6GoePNTHrs6qlcAf~3KO)W3$)>BiHU|IG3`;J=VSO!kcrH0%(@MC~m z+QG|o?3uzyF6x-TkdT57DO$t>w8K;$!wnM{qadb&P!`$V3#vK^$I@V5Pft&P;{`?* zoaZ5*Cr&3yN^3#dkRb`5Lc>&jc&V|luso?T3k(bM#?RN7aa*k>WRxrA=eiVZyNPlI z)_Vl81cZGV9^U)v)hnJfR4jva)vPj3<0aiv?I{<_o@m=TF@`%3nZAZb!1K44%2@`> z?(4jM9bZx0}X$*|18;`&JVB{(xq2lmW_aY6T>48-Wi}k$K((WWH${Sdf6M5j;r|wye z)ot_Yr5u7p$sJz{`V!z<7ud=TVYcRf4>_7msYS&Fff$%o*G-d z!;dso0@p-se|(D z>S=|}q)-ZkuE%YC_$>+95hgZB+eLAE1Wx1mo@}&&x5S*gXTAB>et;x!c8MPGl$&3H zk!xYT`KxfX#I;wlut?OoVqsATeDIA$qnPL1R3vPN9w4BDs!Fc?@xu>U%?P8}v{OqR zM>nruZDVUYf{}Kg;4S`$2Pi|ozrPm$xli`t3C-*x>AR$SLqQxuqr5UXP_%f<4HHPKCwKLR|c-+ z$Ckd;J$sh7BY8+}HfkFcT>b*TwYRq$s~GJP7T$WAkfHEv{>Ea&j7O z&0D3sTSg{IElut?juJ+_cg7u2QsS0<+;f8O>~EuBUQsA1A(~6hd&U0VeMz<=Qqo9k z3GPL3+1Vw}6}}#-M2V}d6l>P5<%N-=y$cpPd3m)e92mcV_ha2z!hofS5f_`An@Jpb zTGxN^=;UFH{Xdp=B^>jsGoxK~Vt+;MGg_nl`rNu~{YqO-b@Qx_6qF3o{vs=|U*V10 zf)nxeO1J!RSXsF=qu5c6Z<9Z#oIeKIjg5`B@$%kjJQcqA$B!RD$>?lUm5LpD?B~z>YQA{K7NNW0t6D8v69H7eEA=<3%hRJx&(~}_0n_znc-7USW9U#9KB>4OYQBg|Ens77I%54 zufOgJibID6r^T1<*8NpeSBL$DF>v%tK0_8_0!1dPBJLVj#63Bm*~FlKjGmHQ&zYGS z6w(eSSUFriws3JhD;<9|=D(9^dG3HGl&NA zoPXM&TwdLu!`gbqaRGx!{$$=euYuf`nLlx`r;~Uy(!zY7Xq#HZ(R?NI;q4+el@9 z0PDenMjZUEyi7%8j96z;oO6lt@3(JNiaJCypE1Ic{!3NW_PE00#<~JqY(K-!mo3Gs zR;?oN0~|vHj9xKXw&c*nPfV?&{$?i1x=I~GCl$(Xuc^HF+LuGu7=y+GC6vzBM~oC= zmQ6%c9Hv{T8W?U zw}EkwZ=(V?eB4>Z%x|C?pzd>H+`9Gd^t1!k!TY4aG7F6 zfK&?ij)!b@9PI3cgxy$}oj5Hf@9*bV2xOfL#WF4Sgxl-5ox9KLR z#itjHSdPcnkOk=m2GKhGgM*5|lzihNeI7o1_}zF819+&pIP4D(KYq-C)`|nYk-V-h z%huhx1#6N;o??)X*9h0!(c`|LzU*3>EEC1@mK6d*LJxhh=A8x$QNq67j{)0}BDwaR z%Z$xmCk4Vb2Bxxba1?=V;1$Kbbn~W+z?M^`mMC2=EG;gG6@Nq7b_)U)3L*bP_PFsF z+rz#Sh<=#ZT+)jUVn+?ih}#H8=m^nM&g5*%pX5OF1j)KDVBaT7FHfym;=wV5=MI9 z5WbkDkw5$FsHmt~XI7H~<+LlZdi>YAmG`5q4!Y&6plJQ|YrVU>I5o|gFU_)4K1XH^ zO-0P+N005v&vz&nE)|lTxlD2Jau0oY`0V+##l^YVDfe7TEO?VzYVb_a5hOezx>?!5 zQ7nK>IGaw3Pp1{>+xfM+`WC`&2VCBQ-tQIJriEX+55Mh7kYrD_fd2U-nV{x>IsQBpsPWN-3Uye6N zDjHYn+dq{8dX&@mmYp;iXie0Dy&Kc5i~vRS*CpA%S~p>2Xt?)s1)H?X^l?oqGqZg7 z_@S(z{j0{`c|Gg}(RL4fpp%=w0ZX1_eH|UDMD6~oY9B*Ycmb+;$SutPjF?`C9l(h= z5A=)-7W?)^8X&F0?s1#xXQG3f@6&L%>0Ot4N?;2B+~JROa!*oIwG!kIgK@0j+;hjJ zt=A7F!f}X={4u_|PgC+YN zbB!(M({yeSnc>g z?2!bPnkl&_+putbsoMaGf3j2o)3(f7f!PRskM>`t9c;-JswLyPfq^Ry(I3~-^TSV$ ziI2b1l*@^1hSe7slDKqlW3|G*Fx4G9cD#|Kn-DNdYp{H4FERsk_)A?KzrR5m%@eu~ zWB^dv+=xWPGYH}|RleYIC(yoxm__&@3!5eR@dqn2Ji!P8!Me@iq5$oqm1P47cv+(N=|Ikr8sDfnO_I0PP<}d&Ey) z#p;VX6WT~5;lOXf!NFuj_2{S#Y7TrwTx(W99fJu3V5z7F@zAVc6%UqYpj?5RoMGGJ zU!D>mJ4=OUDrEZ>hCm~1ar}sawC^m@E`|->SB_nOSyV=@}UFd`?AGJp>a}RHCTlN?L!F@lm1WA?TwozddfEx zyxI*1}Cghy0Rx{t|GacFXAX#pk<&`#`)4RP>R4$yoZdBD%#Lf zu6Xg{1-j9dii+mxL$9?{spGu7dc~7~R>Ylbm0K)6qV?!5TjBSiY9>@y(cZap zCpu{2qeR&@RZ0i@tqr)7G^}C=0U5A{6;4ivGs4=-9R__jSe_P0Y#IKJ9)2B z9PbZip~Ec$SZa8SwC;Zp$><_Cf4I<>h7`zVXZg+3$T`12fn&k9j&|L;z@2^zfHDJp zJKEa^QW)mY?GE==aa&7^h}+Sc7#==oZl0iOT5$*%^%#y1yMLEvh96!?UWo77t^kgc9{oGOEb%o?z; zg9e1;tH*UjFc2QAsHlhr_Phwo1re4)xv9xX_3yS=?RNNOwK4~w37CXIXs~>!D0)O( z8FdraV(AXqQL}UXhKS>5>ivs~IL60)KNvE!_VnDnPp@Ho>F#zUQ9o9rCnp?{(#@p`s*mKFX+}L%R<%IU7!+Lt+2M*i?b8$00 zeV2zvQ}X$nwDG5v40|dhVJ;$C_@7*_&^iCjX6-go-H=!K>+bRmP}@w_Jn`Wx(va(F ze;IAl<$p-{>gioiPEyp4IBTP9;?WeKu&)`# zdUa<3(j};gVq^=82e6tWn*tHZ5_=0^hD?K1dG_U+q8 zplv@^@d91!$RkS|va+N-7t>Shu(j;58$d+Co zsxxrEgN;cLb#KR^--2}CJK-elI(stV)albk%~_7Zu_*>VCa>5eh$$B^IfIqM})v$jJ=l;u=+UpDkvlE+YW%kaBI&u?+YQNL<#Us-O3g|o!4{|# zq3IUp4bw+AhMI=~?!!ezibUIvp}vmSXHT8Fg9$@iXUtI=qIjPllpiWU>5!FPoHJGG z2y>X2+*|{4goF60>sKTGB4;aGEY8oE>#8SR#O(){rAz`Cv|EAv^pc5yhR6}uw6(U* z!{>QNr+U2~olJ)NQjS_S5MiJLRulUHfIO+|FZ5*VS@Px!2MLZ5O7lsXE$A6>YQPwS zqDDK6nEo^?iw!6;>MrMB?{Rn$vJg#J{?`4qnU`#n;Ccr@`&o82`=4wN8y6gSgg;2U zkm~X0NM6Tja$rP4C0_P5Zdf-`0M4^LbI4(jKVNg50YC7qO?4F*a_82iyAldr|el9se9wh3g+>FH`W5H~sr%a}fQCz8SC6=MM2Cf;V)$&{pq{ zxQWpRax7stsl2IhZffWwRz4#^s8{n`hHqtt^VA;L@0dUMk!eqdjH;9Eltx|3r;e+F zYVLWTmAktfEt|v*tK5aHAcurwo$b*VlTFe_Mlm|h)5F`rE8IfTVh)xRf1&A(?d8jO zy#*#}0q_*vrD5&dAa;mw22cgS>_+?FoB?7VH;!j0!WcQ5sAlJxw)zyB%Es_%UcLIp zxT&gYMQT0PzcGewtmpJo<5!--~6?65U&#}by4JZ^5QPFzAvuS2q;P^8= zq&`$`B0`4Efk}y#Asu1keDryo0bw3zXHv>S*&s5NW!po$RY;$jP*K5wGN2)ddC!Fl zZCC1iyu1#otJA_u{pAj3wcF@K;r06L`SXHHOOxU(eru3EVJOO2S?asMVJ;j7aEwkv zqq5Xr_4l0u8DnDV)mcLV(k(QzX79Tx}{ z=HXMSZfx9e=zfLpU{4Rf22(ivU9SZtr>Bd9ed$^!56h$1v@$X>;1Ld0N5HbYQADJ? zqM|!lGW;szo^uE7?bDZO1Gnpedda$p4vX`WWai_{nRUUX;}vorAvO4|Wk3lo)>7`x z7KM|E-p@Sz#m>#^%>fwVDGc7p45flI{w;Zo(3YHj+&`lR$_9BDFMmq!0jmRJUpU7UJ2_P(Tm-$Yew# z6!acq1W>jfV7AIaeA$5Vqidr@AGp_2J{?&VP!R)0(}kZq+v8$=L`{1kEl0vr3TY9r zLq|-2Y>*^#eLTcEz#WD|%PDoAUlnlrpB#$P6@&2 zz}J@=y9jTTU+zmQ%1Np*6P1>h?p!CY^&09bVgGyg+TEH(u#kd}8FT;PcXJnjd^B69H3&ZDhy>E(iwqH|u^JRd`!loBb>^v8cUw z&*KjQ=HZc{p{QPlw1O}sR{Rtg3g8e}2?%Kkb8*dG6pOH4T6D#;LI}wuz0-->+}pRW z02!Ax7!K^Gq!a}?+H4g1_`*{)k}K45dTBfO2oDdBC)IsCbinsGkXe>rL4i2Uq-?-@ zr}8+T>H;MJHYK#VuByI`vIYDaY!O5Thp+&h6N@F}1|zj2g&uhw@6evEf6aTov_w&kKX&+cu!@I27jQCX%E(r%>y}H7>@p|?W5<2 zz5A5Cn4hd^JJGnHm1tR)YQr|f+bb$kteCBsY6`WK#P9$+5E$dckh&o0E-CoCTkcO(7Dw%m|0ka`qt~% zxIjz~j5+i-fi*Tsw1oh%t%bLz*876Qf?7`X%-!zpOQo{wDd2ndXpW!r2{uj;U4|~k zE9E%(&p0~HWY9Rign-+M$T(c*Ci8$_{zP$#%jc=ZM#mKg@fjp7oB*1^5B{SXfzoJkdLF4B<=QGJ3blxBoYpMBy5|m5rn=Xa_5ZyIXu4P zSgBa12pIu=T(jo9Sj4;N-P}luO6wSSOKd)mG?BS z_z3)(jBDT-1#Hiiq5<7G>dGA;pD~Dfr4A&r0xBqUM6H8^vVTPC!Z={R5$2H#%RS{j z1kml3Pmk}Qr>BQk-*2_cTA22s^nlu6og?j^lM&8@o}BD2VPR#pcz_XH^Y3qNPEDzD z14zTCp+o^!q?cn-#f2%8(a@l_s>F2}2L$NrnkM6Cba+kt80YtCwo@;bUTtF6~Q z^-_6$Pqe!~d3Majd?o;koCgFy4X?(?XV%?VNl1i~SPy6YFk04?1-t{TYe$A+OdiIK z3Ep}%{8BTz05rtP%F3if#C3sEV;|vJYY#|CS&w!q<3xt~Y4Ujo1kiy=!FDSlpENv| zAU!3ThQ2u9`5@qJM}V9P9}Ss`z-dhej5(8E1*6N3k@*>0>vISKIEa?3ieP!-J2`pX zx~ZW-z=-8{62s%Clc+7WYxcAe>poxC>{zMPdO=6`6++Fn#{D1}-smP0e$x=6vBG># zQn2nO)&H3@VLH2@qr7y94;tK99(kgT9q8)11E6E4qvd7eG~K}HXrVYPhQEOA%)-3k z9Eil~(XzYa-XU=a3DKcXluu^_dkWvkZAg}Uaci-}6*CZ?5fKrpr6wt$Y&sBeHi!0m`dL#&73zRNj?vYv|P&wJAv}b1#F?mx#}OvC`i9HnR)Nq5>52xxKl2CuCw8~96j>(cQotq5;{XSh^1Rm1zW3zGHt5bv9{Crcp!?WW z_OQ@eLxV0nB7$M-Rz4$vxy?9rXz>4a>deg$Pcf|iaJ%0EGYd`dKwKKjVbDe)jGuQwSNLtD&EY*}ZiGHc#EfG?~V%Evi^z{i%6&nL={2cQqhu zGttLAe~SPBC!*4jk{LWtM4NUg^^B%LuA2o&G-Qpc545xp8zn9E$|4Ad!aoF>dXEw}df_xk|vZ_05! zn*d=okt2ZWyT6aQRQWfVPZ}QdTW1s9Q9~gSJFNd&HJ75BxMSC@kBC$TGefT+j>h_H zT(w*+?kT7tmql-Q`lLBTX;5S4|M;QzbsqbsHxRBj_SwU@8Z-F2Zi82|tZVm4$+>34d_iqA7g=WpUx~7Jg**0jl&&WLZ3cv!6x$tg+BX^)4<}iw> z6F?9Hb^H7+3R4(FfL+}$%tTEroxY*Gb4gZjoKx1w? zxluf%9XQG#wQO_AJKqv;M zStS#bzrad9{L}DZ_i63y>@2B_QP3zB8E@j>v4hId(UBZ+%0u-1paMN1gZYC^hH1wi zIe3r)IlMp@m1=%|zOm}X=pwNr5JBJBMrCe1nN-vmfYW6t-|Idx+9SAkv0pUG&mwXQ z6B89=+2ACFF!%x)j_R; zeCx5hgr_sZt}p0$Y>ipHoplgV%$kU(n1Rb7VO-VQL=eh842@>{5C79pJKJ$Dd3YLh6O48&&Uj!7<>7q=kc6k~v5%i}%Qf*}O!n_GGo?~eQ^Cb}0jTns0Emp#G(7Fg4#5x3CBU+K zo!>ijpt-=1$1dm}TV^DQZ%3N$Zc8N!46$$^0StwPvH76wfKaQDG;J;ydPC90fIhwj zoC(wf$)OJO)5+ydnCL^WM?))1K1M=Azw`1|wZ9-U0aAp46P!6B z;105Fw)zP25H*xoam2#>JS;1s4<&YsoEDLxp`$axl&x5C3{+4fC=bh+T_$_@xJeMJz$o>pTDjo=|0!(x z|D3{%1$IJ<=gj`xg-w5FzwS+ni(7|Bv53*YWd)RT%$wYp{yR_L+I8SkDl^H1;s7ty z#TgnS(1f4?U5>_hKk#B(MOrcJ=@T$|Z{LAftsy%@iO!Jz%2uD%Aa9CV7JjrOhfVT$ zgC)bm#JkBJO}r6;6uR}Gef0%l&@ZB2^MoAtUK(!6GE#z-Iw<*ol;mPwg1EAPvM$uU zb`1*+QZc6x4OWo112NO?+BHn28^kJ1Vk_TjfUF+dywU)fFmI}Vi2q#rC^|{gDB^;j zYi`vB6y##L$9^9}IkYlF$aX$i2g%mN2JyhW;cwHgk8Lr&gQ6u!7q;C%2>5T z7^HLp1Bc-vnRM^*3TXGR0+CFh5tUOHn3V63`2dwPJ}NFr6yK;EOZRz$ViJU2R5ww( z$XFy4rIb=c+L`GgG?hcI-KF7}-e4xiMue8AD6< z^jr(!*KREq8%k&&^FMsJ@x15?r^z$DjH`LERTX4MB0c>!jcqn85CMT^#k;~6Y!*4E zbVmB&rG7A}&L;p1o!Ac89Zm6%kKctQjCtK(L@a27cq8f zBQj&7!Jj@6e?9Y#c_2mcCrgqyvwy_;)ZrHtf}5cIfJ9++XnW!0!y-l&mU~G#v+Hx` zM}3}e{{#sLnc6mjs*6@;#2x#>5o*4C`LeBc^ZGlTYsT=j<0P}TxXmTXYaGo2k`FY5 z%=H;2LfMbhQwW%o-Ry8SCJ#w7k&&U3aLvxH^;?|Q_bkU$`Gs7GASg@P$iEmn0^*10 z;_(aZ!|S9WA3^tw^RTS($Ep5+IIs%mm$|yi3biV4`3C_1=pxyjSY?=TV&NmS7lT5z zP~FZPSlD&v@Cy9*u%pIM?G!0Yc6GI|wjR-9oz(+9XpGI`WUxpZ^G3{`ckbjB7c0ez z3}IRPTB80Kuz&FIp%E$esHr1EdIW`p(!)A;fFjU*eB}UnE-|_ETnu!uf|61hicmNx zwc#TLh5PtyK)0G9XQ4z^n$LZs0K-CrIk1E(4Y|kn5u>IAx00Nkh=7deJlq%5$D$s` zR$%Wd;V~h3jJODj@k(vNAt$HzT8ow#(Rqp%4s5UCFSJKQKkXZ5B7hDDQIHkgm}}R_ zDooj63mQnMWb_ZKVJe|)RDig-T|~t0>;CF~kj8*9V5Cp-Owu_5u0lzJuBPC&v=yjr z>NU3{UMD3*-;X{vSy@@BA?&90@ni}oWV=MLOsJ`*EXPfl=)Y5M{`SleP(lg2UV4yg z%K~W4AfIhiVne3*wP9bsoJ1>_GLs(Q>l?C@rnKBF?O9gVIn86Ls<%}NmEV}U5D}1o zK+rcr6LN6u(c-dETyrb0Tlx&4*sH5AswnuY(o^CgLN)3YkX-j53uyHQ2*N?+K{}28 zUXTNYm}L=yo4^^2MiN*(sKY{`x)j&;`cShUKrv->lkC_5)^2@?EBrcw9ww9U+!FL> z-%bIju@FL|?)RW|MXtd5UU-t;T2>I> z%FWHy(l0*xr!9O-fT)4*$<{+Ju0bz>c^HU}w#f^Jn;}`FT@-=NTX=93&FKZGF2$y1 zvUXQ6sw#*~5N6?@S5-kf`@=s!k^HskXx#A+Eoyh(ZueB=Y<~8j{Zl zcFc7=G;kwjy^a<1_z;TvV&U#aeSLjlJ{NanBA^oRG#uJaB{Vp_g>nTjaTMu{b!cZk zqS_kymELk-U@0EeG?L#4uoa@INT~bvtrgWZVfuXgnt$MNBjOiRHr+F4ywSHnIFFvu zarK%thVWHGV{%u=88I3B*=?L|=3uqRmbdE&O3#;^D|)aV^)Ua7zgMA`0P^(#0ayn! zBt`9oK1iYNAc|q0cKq~d{#r=r2o`OyLkg6`UI~dE{w-#EC(`BM2SUVj-yL-6DG3aSTIKO@4=O^WZ(jcv~D>+O9bnIm73n48-YYu!AjuC4Ch)n$8?qP4AU1%!thOhHk3=ikh{ z{%j;PegGFxtjG9tg3fjgCL0w5pJiwiQo>~G4n_Sr1*^oDFV{kIi!X$}4h}{)cA!fT zTpdoTlTWIHy)iRHUt;|elauWz(%L0wS_p)T;J)}|X)Tvh4dx=jD>t5t>4YW=kM7sF z%0hYuKYZfbp#ITU<{JxVU)KYq4wjy4_ima4QI_|XHU134?F|e611+_9y?crNw;nf0 z>EOW+8O2baEx$~hz^@m?X}yGg!aP+F3R$8WOB#B-2Hqv4nI>gT|NR?`W8Fkr3yhM} zj)t9Z?BP!Tla4U7<~#YXT)J-S2SWG7p1i@z&+g0$d=u-MMcrn)7$KAJx)xk|8@~uE z0%mxZ3(4t4JnUq&5%W&TwP0v*!NWdeq^52n&FENO@to{*4jC=noxd$5mvg;021K~d4Ogap-uns49E<2;YvNzp>Y59;iX~A_TPEt>=9DMlIW(S%rOmfEJvK;;JI^=s;08O5Je(x6Kz-WbrgfB8;$cL z9zocyld`me0^o{1|!1RKwDi}fXAc2e!5Du4| zvXMkczJ!u4=^XY3ZHH4PYWLu;Z)BhzMS|X1=5ysR5)0<-kjePx-<9O}F6?Re@ng3J zJ)B_%2Z`eY7=2$9a@9 z2Xd=r<}#dleji74iz`7rCF!*w<^k3at79Z=WX~tl22!x%_;J0v%TBa%!(9&z4#(FJ zS&WGpxdq@ZaD>>>Z?=|^NBF^myU6~z)=SIvscaLj8bEJf#h7V8sTgW`^ri=&4pTON zG9_ki(}2DXDaM=YNkRAqZOlOKIP@+xR0L2#6P1o-VgEpZdI18&fE5 zvwi$76E&x48&B@W%|si!e8Mmh35ftiJqc-V-x?BXquob{?RtAVm=?n0pgFx;W45KA zERP2kLzpVkX$}nqs6~ji;aVD6K+^6>%TCsf632b!YDs?lMWP+FtedXj#-rZJ$6%$g zs#r3F?L_3h46R(P*xHlWwluvy!rx+RTPY5FwHuuHzOw>MKxVk?~m5+QN2nCbQ@dMu`vNp6No7WdUzjz|P2J!IWye#}z)Z%~j+KEoG znE|B+Sa(w3as2g@7@W`nkc@8Z1oUXcQk_j|zPz`0$6{af;yT0!C9FA+=j7);OU_m@ z=qUc=Xg>W#>9seA9ZJysrvF7C(nCYPS6G-UE>S1>FbKsEq6P#!tGpZM4U{D2=bb^W zll@Z5EaF}3fEBsQc#8h8hg-HxsMlGdo*{iBgys zj5U8eI|s)Daykj!F(WmWiMf$d2_Bx?bF3X$iSfX<1N6%5H5+?-C74lwRuQNrBhbh+ z2-NB&wm?V82%BbENy*jTIyH4%^yMiAknbDN^kP&b9ZDE@t7#I|;~{;04g*^$f{20! z%Qs6?`ANt@tkY7BGZ7}hJk*6jL&+2nXuLsEdpX;zj)_%DKu2^Kb6W1?WUtf;5+N>z zyMneyWkIHT8`4f{TH1SQ7!QSwHY2hp9ndAnugDxKDDZ$xbsZfh8ylNuM~?=~j*}A# zX*Xv~Qj(Q!j(e=;<;(npF#>`VQCZXiaVgATx==9RMT8GJ2UI7h#q;5kYQ>c-TPtYD zXdU%Kp96!1Qg<%~?7_1W)hKF8v3^iE(3ZtOmPv zkY$kr1f|8Dyz^MWQJvxEs9y|$8l-8zCsR4?n%mIQAqj?j!;s7J*mJhn!(MrSe{DFD z$uqCpFsg?f8dMzURT77{{uG8jJDdKxCb zB5PKcTp^?lywOtm_!{`%+vfj+0Ywt~CP)%UNfY55dy}!Q!xbk`Y^?PDnB(LCGk}8? zF;xRbE&)T|x*Uz)X;E*d4A-g}Kd?BUPtW0f!b069Qpb|%Jsdw|3ZFEf`P>2m>p?7F##8;jjcZ{z=q`|y zZ(tyU&N2%hYdb&xT4-^5mvU0?70>=_rabTYz7xG=AcCE@xJqX75wOpxdV05$gH zcnSgmF~Pxq4Vnc}OwYnck&OQTy5!xS5nhE3EOV! z&zJV!U~K$_GYN^Y+i^=41LQ?QvjQXXld9i4+};90?FGhb@94^;r60xnS1E!V5|`=v zUt_}KgIF|iPEU`wB??f-cbx`r6m9^%xDgrw3KgQMHF!4&e=#!ff(e~XVPRoF%_H8< zovL_}nR!-tlFVjNsl{QcgVZa-E_ou_a1v!ME-cK0XDEg|hJcn84&D9W<4SnavPWK# zsGGJP&D|&^*O_%56fsp^%eiz70#9RPweu!)On@dJEyowM3u+}(55YvE zcK>Zz1D%+hlBw>`-rf&5qK0Ga)!;M?Tqm}mW4@@Os378q`T*Q3NeTFHb+6-33c#cz z07L^ApAiQEl1ZJx=e93!{DTcap_!Cb)U5}i76%qzATviQp)+j?Aj2VeJ)`>yGXz+D z)}Jfhkd-mm35Z3}(C`^`sfmD}uP?ktD1b@_eSSF}fkHFEAQz68ZeNA3Hmh(wi!1!K z=bvQCP*o7#P%r*30YU7Fs7w|WicS0P&>?O6YpyW);n*0V!~oiD9Q_6qD7jhBWv+ z6hlU^JVEtGh6?!{b9J^1q9=I0`I$=hBEq8rF;oJn12PFzrxp~tq{7nGMRvm=ruytJ zRkVkYHDRcTi?E+WBJDC(E)JKO(0xC~w=yz%VsO%W9y%h3D>E`ON*;a60ZfCNsVo~0 zE2w(G1p_~e+4<3)dArHc%Y9bxg`!p-(873z@#niSZ_Uiic#I$rA+uqzBAb+As$jRY z;o%;=jaM)XrD$Z-v36;R01?ZkFIZOe&CJcfl@rMMtwKYFV#FM{ThPvBmp7xzthm(p zC?w=I@o`%jh-Z8)5JijB=>0fc#FxpnjX0OUKf!YB%t{JxZ*Ss>bW%Y5y@FK|MA4}D z?*v7Y;R5VK>%zm~W=WS+xZBDAJR;i;1T z!+UVrh~EsXp}f=3K@x#quFxhE=wnGK2EG>J&|5>(Gu0pA% z^W%HpnqEg0I@KS$69F7DtUt7-@es;Up1Q{rLT(1l*IRD(s{Omz_1U^RXeUGIQ?H%U7l&+>}jLs#Gp*5&&Dja^T;P{H5u?%rDd_;f|aR2KqP(uR6KeXlHo zQj8!cpre*gsO0^ahDX-s0`uh6=0S-Sv6 z;THTpnZJ)HmK{ibb}jvbuy#s72Ll4T)Vk-+Iy$;y6uU%t1khFm9{^KYSaAN{`~l?X zxb#eDGaikOU{ON4jwb?`N(rI#d>Mnh#;%Sd3{F*u)$pV&dnnY_^3`$zxIhzq0b{Vx*oiMLE|SNRmi+)vEfEp-e%X%w zvi2#WEY~iP(i-(5&n9sg19WJ(&jwekafM+%7@|28U&*z=6ai&z0sLt$#vcp`31U5! z2L%Oek5DBY0qFC&bN~iWR2Tx0aGTlh{C?Rv0f7LsQe_UCMoDKHPsq{h*Hz5O0eF#Y z0FQ?tYYgyJ53r&vj{ynvl?hWuyD;Zkw|1ck`PIv>?o4Ozb2I~6bxkg3r4q!%a?D{m9M^E700m6ZpYZn=hi;=4~)~!Z# zYBryjE)vlq*o51n_fP~8VW5Oqn5v1TrJ|0`cck7vbLhrlfuVp=FQm>Qy<)f|q1Ggp zR0GA5_%(Ztil~=I$ss(4xj@OId$Ar+n*leG;IKMuY@<#9BlM<#u4D#B>QEeFxM?LC zW^7Ypz%_vx4xhZH6Q@y28B{i-luvh_g3BE<`iM`6Rm-^)g^5kBzkVNT)i*kBxnc+? zhN!%S`M55ldhl5Ga6=zM)Cc;j+K+cnNI~d~#L5sdh(;7a zbSyF`n3avKfPffN-DEC-37;)8Wp4ccdrYlHFNOSDc2{&*`dd~T45HWHb{G!n@f-zW zK@KAXbfh6LbVaBo#qjlC4cPy2oE%W7xnkbG*ZmqI0vQA;XQ7r}7REwwSBClp(<1&D zYL{&PKQJy<6na3*gZ(HjZOv7X1`(Pae<>v(*t9{=I1MwQ&JEqq!TA$oR#bDzmqb{| zbzGjGQj-o z!_dSA8D-^K+uIjIkP~H=YxxY^(=)7AN5^QGh~mj9`>|j2uaIS>0l~4V>gy#34?wvK zMZ_Uh)z7zO?UBWh`*@-&p&5)w@z#i{f)f|+9m2n-D+L&DE8G8~%Rm&lA5AXuCw^Af@e&A65qKB=2b$nji$5U{c-*7e*?nhIN(=QgVuqhGa{&(Z+f!Mmhccp z?D5)(q}hVJDg~O-7-%GZXH&7deOW{#@H>m5+>PkJ%;$p!39>w5DI6yRZ7c$gR?y9o z$AA&Aj`j!A@BMG`hY!EGQnyE6{IG)h)7)H6BA*0dR&M=wGVb1+^LZ)a&=p^LPKPq&KoYQVpzz6@GMp2bvqy5GIHmAK71SSU2e*|-^<-+)O;!uwtCd?BP zHx9y;%qpW37b^pOK{%<;pFi)+b$du4BQ+XAg#Pmq8F3i@4|9W)Z?f@rM*NAJ7v34s zGf2ubUlrL$&{KRIG7O}qre^30^96E7i2~#Qv2`ABIq&cP&*s>2j3h+aAxV)k5>W`p ztc>gt712=Ek&qQiLqs`QNs^@OhJ>u7g(9m$R95|;*Bxhk|GyqR&T-D^Q1|_Lzpv|h ztqUtuitqbu!5iSzMY}d-G1+u1Nq{_&3`A8lqR4YndRd-VqLPAt58j^Ct_L8ZgbYw} z(U{!$$|wV%{&Zab=iyedEqELjEL_-hF~dAuAX}@>xLygE$XkwcV5_8pOd2 zi?^fZX8=}23+&N}jRP!`U;zOlF1+rpr8VZ>#WI>`l;+~F3q8|In3jmN1^J6Vg-guo z_hTlH?x(&#{b}_^lf&zV9@^Q}YmJjrWpyQS*i0%Jcq0XJqEp6LBzi<;?dy|veKe`> zjN2OK_gVb(9zyt1x=RRH2|U1V760(KAPEOB>#Z;r7Y9VOjKq59XH?Sg^*~iBNXj05pUhV4&%jD)}NReBz@9k%2 zQR=-fE#224@W+JX4~6l-Ot10C&(H6DezJ0WTY~N@;zKixNiN{#ARarpzB}Iqa7g=sY@GE8KlmU4MRAllzkI(Q+{5@xwO^XPkIRtAH+GJ*c15miKK{vMhW^fD0xknLYkUT{Vg+5| zMXf(W0}1A{;dtt%ALqw=#!e3eR+>#pk z{`r8XRe(~m%{zV1##I{IpBbqY`_r?3qO-dOkr*NdH`gy8&lmVf>@_i5B~vzezbz~r z&KnCJ35-N1ijAZi^TC9GceNB+g?|5|Rme*t1nk#Gu<7OcPoZ8+HV=%BRvylZG`{`I zOj{bMzMDKeq)&6L_)@?(y0g!-*yN<7%M#*7oYo_81OnAc3Iz7NL{6WEdqIA(H#r_8 z(%hby7}&(Z!lJx9Z-++Z40Kw!32;@fDd_F>fRHU(>h}dkl~eCEDB`t{FqgB_PBMe; z7$0T~0WLjCgf4_Kls@ZOKWl6HOPUs;nTkviFKfoQMr4X8t`1x0E$oBgp}O$_DGpDt z>YKjv&v-4}yZ`_{BO`fU=-SjdeQnyrn@zTvQD@A}Dsc7JM^(uGEx&$m1KsvmVSE-Q15fva0bfT-TnJZ6`=&J zKYFy_$UH2t``$%2!Wj={h94JM@X8jv1?Z8^I~J-?z!R-TAl2*Brzx*5s^!Y&on2jc z(>AiW&%HAQH!ot7wrzVlX#1Jj+=+^NQ|22GMA4;l;9lGTp>~jh4wwtlivQdX9Tmx4BQ&Zyy?c5tAf+Us1qc2DyUe!S~Y*G^-6HmmaHHX+hqvQn@$KUoQB zM271|&|Wtm%=f_yF@j^RSyTkxG~cJNu4vUdLj};V$ab&{!OWetvKEn5R()WDO%P_-2{kc%q;0aR1&> zP&>2!KVXqCN?lo?vn2}bhz3G1qo13pF)q$N$=gjcNTF?P;~%qa=~rzlufT&hEDS4; z4i)j=t%O$N>14+S8U8d__Kguk69r1KAD z+(n=Yx}Cu3fHvh6%28nm>Z7NpS2TbZ$=KGmXwQ#$)UX{u(>YYLPtps>vE5y;Y9bF0 zq9T%^hkzQP$VKo;PmA*VXz%QAWc&<+(!mL#`3Ou$we)+lxWI^l#+>jDb*o<*ZEgcX zCPOx-00+@VAa4WL^5t18+d*CdQnQ7t8FcezPtRw?+uObpBN$X8I~v5_9IEg)ix#k6 z@m~B?V zfbcw|Ae2oGE~l1fAmd@#pInkpxy5#KceX*KTaAP5G_+~*5nfajB#|-k9rzZBlY6>y znsNS4zDhhaKl3K#G%dd37fVy0#NukjoTg*_}V z|5;(-(#eT`BR|$hFI-TeuD2>PJl(qQJc-c|n^1?==6m_AgtOCjf0iOilsk+n9&5cU zQ&2PBJN`?YHd0&jcStzau$qfPst+mZ)~ZfV2_d(i#;^a{8!%;QFn7aj9v7(so47|e zuY?W8p*f46mbU>9bf$zt@qT?M|6mj02#+CtA|5&3CvpL=GXh}7y(Imq#nk31J! z^h^I`0rt&O;BBrh&>^G3_yn@|XQI;Qzv#{S+MImf4eXpqgrz~X<29k8C@viQi}!d- z9u2i_d~Gzmir4|&nzfgxSb41p4DJ3?|5poX9g`|~Poqj}0R*7CnQNxw#CsxR1pvq9 zDDFoQ`7i^!ybWSSMvvBS!RJ4Jt(T&21^!YcMJ0oO_3HCxo&l5LDu0y<;(hJfO__)7 zLqb`O9ozi?p^gk2;D*V{+cdY4G$2Wg)^Up>r3WR;(1Ql8L2^*p@yX#PLx*17)2+;KUr_gAK3Q(l>y|X>GZrgU9qj}v9`TMqo3%YH~ z?iG!~ys+?3DyPA^q*L@ZvY$?}n=LGw;tHD;@z}b*#C5^NFxSBcL1$I#ws+K~>owWu zo4vcwCHH}skk_4!iZWdUU8^0^L7$R4>fC))Pxb2615k3AtD_=`0TP_tRwSqF2^i0B z$;4s0NJ=F3In*P|j-wH*D!L%Td{b+HgU5$8QB%WXzJq-eXreO;%ht=A1~R($Cm#Hp zh!MM}?+1;9H=JR=w4oA5JumW(;RXqCrz6e@Wb~yX2&15%-bW2Sf`vI95DE6h@>81r zidlA72{KUXDXA;ZN$te=K8uLhp$7IlQA~}$6aE>j zkB@8IZfgZ{AkfD^LFnf)(dMFrZJ47>Mb2S=_a)6u)^FJl)?ZBX| zbbqQ`#PnxZ0aV0)Um+G)I_c-=DeRJJ|HiCD;kYuR;$us2m52(+I8vJGAtm312b2bI zh;)+mmJ&0EoOli#dvTBdr$1zSWeskQi6mu@)lZ6-L*=-oA;*`n!Ei21oZtlGGL0V|Cvqx-KZMqm$xM;>; zOXiyC{t)sXCYOssWmODKEm)Z#N$j1w@!DF#G$DjHFkUP5@>Bj};zS6k(fbTaM>eU+ zdkUlcG|IduqFPbb)4Z6?y5Olyh6|O|?)$rpzP_n17rHOGx!;L6If#scw(DAE_9Sdi zvO4RrPk`j4(@@9~sQ?zIn|q@gBkpsK&~R=UUZ|bBcdKX4&s_0HjG^DYe!Yo7M&VX_ zN3;Q8Ci84Sg4$7Z^$u?w)IU|6vW!u09gAnZZ{G>Xm?lCO|GE~XgZA27c$Y^GlR&xT z?-v9~>_OED=9|~M{$T@n7`~~?p$qRKU}m)FIH8-m6PplfQkl;>8G(4uq?8z>k1%co zL({M-4SA<_%0Hk~gu% zJFc}tO=$UFHKG0cx(0w&KR?ns?SK7Z{YS^l+I_K*_EEq1&fVT^ge0i~hfogk0mkTt^eRISRqi^QuC$Vd$aDx2r*-2+D6Do5#l6>c5o zzeH4x*QZsbN224o*uUZjN}}Y)_wOr%bOgN*xP6N8MfO%c!Jb)O5l|D~v;qW)0zacW zN!qj7%>fN@vR(bBH_G43iBl1MhLu%z_w;)8>tEwlY}&SM;ob15Tv{|BHHRp@ zf3Et30<33uMw!emq_6A_@Q{$~eSMl{^p)TezFDZb;kfQ33#nj;+kJ!q%-hd!SCCmU z0+)}~n2Uuc65>5x&bdR|`a!}=mR7B_CoW0f zo8Q>r@Udbk<@0gC9oIIDK~?$sCQMlXsGU2!&hO)&%D#fN zsbx^z&3jBcJ2G^l@kiDrTp_|_X2i%gdflb_Leyyy)YG%$PY02s^BHB|$2xnGgcEgQjWue#wRJj-Z`+%yL^`XGO9xZ3T>|B9y;Fham|Ym(6Fi5>Q@e6qXDT&DX+; z$9*ez2;l{i!F7JjvSq!*QzY6bIvsoC%J#yp#;+MDTL0vRSwD5byaFhjO1T0e+bif=Yo~9KS93f7DRvJTxBu$__fGtuuxmFST#k7MS+}s_z6U_5rKKwF zC4~uGku~4#TQkaBYvK{OGL}CXd;K{g!U=6uRMtzF!NPZmDVcp|eotRGU=hc9TVi~8 zXe=m(zv(%$X{od;8Mdjkbwwt^-M?Pn;U(Bg{KsNM5~JcfR+Dvc87tnjZ3sF9l(l;r zysN^Rm46YXj2+{IX-{x--lzE;JM7N}x#7Ls^Q(Q6{Ge`rUFf|exmGTZl{_%m(`<`h zofL)^tWNivniAgrU2t8{0qI4tcQa)(8JWGLfZv#g?1GcMGa#VHjNY9|ZWkaIR~|8y z^Rrfx%z^yy-J8fhR1MIm=jSOYcY=%((G=&`*NFVTmHT5N+8s+<&;)3A;dpBWo7LsZ z(6l3xR?CUila#z(4fJb@!itudzV>HuHNpQ&9`VXvg{dAlaiR!A_yQ=C8Jn)(PkCH+ zS`l2m;3jgU(FxKnYjn`x>*F&zfu}`gv91j>BGFQc?5Tanj!2#I%}AhIaywHxa~wkS z&YUg;nN>z`?i{F76tilWgDcz(W7TBb8u zizFZ~i8PncQIcXBU@IWnE4aU;^>g#UZm9!rYgp^s^ogQKU$#kE6J9FMs9UXwPmI6~zAEht_fEzq!~>hNrrf$8<~U_tDmSf--^oUt+sH%ez@G?&$ie zxVP&3JPXgUZOs1Lg^D==)oc+1PCFoT4Vw{J9{U?0T-OeWmQquE}bCaC=^K!CvLx@Xep>b&dlS>i`N4UJ!8%F zZaByTP zmN@^psPgde%FNa0^Ob-p+)|9CX<1o49*F`NEZw2q+-CnriUSbeZz1FL>6ClH2y>re ziEh4_cE*2Su_>w}5o;Vf_Ga%}dcT>(Irw7h%ljWJg5%6i`RWY=O z(f*6k8r37IZyiNJ40WgbmP{8wh?*e7axiGno2aOC8}|J5tamPtvAG^-ucIL;V>&7t zQLo)jEggW&?-xrOXlu0BkXR{Y7DMz1n@cilR`)f%BQusbN=1DL1#U2A)csL&=W6g$c8fXHe6^i< z36U6A3w4`jXbfp0ky@hcr-9VXJRKnc6wp1`wd`Yo!HpXsTE-M0NwgnVe^EKm_{GbY zwSR?#u@Z04pw3k*@f7pyL)z8l-^=a%^{XrW+)-%vME_V_$TRj;`Yfk0xAP&FYBosljkrr!d zZ7+-U*}WTZT}0^-y7-^~qw^ZOmv6YHbL39dIGdmQ5?u%ljv`$uIbbW@Hlbt;t@V$8 zzTs5@RLb3Y1I#`!)br*PU|rT<-cSU?6YvLp{yuwEZibF(h$nq_EpC6-@K*dF7Oyto zI`DBjeiVRu;8($v|1?+r5d6%W9a)JI+7}wp*k(qJ5noOOEm53Q*|lr>ZihmY@N@H2 z-_H^*B8#hZOF4-=$vNj!A2MK3&pQdvGe)msJE=&rf{*1gi8cz0oBbHTIi}8tFH59n zh~A(rK3iF%d%x}?q?tVH>rnXz1CcO;jQjYbwh}Y6sQjyf2=coyCyTxp#NI2jw-$vG z@uvtbI|Mq&iaQEz{F-ly^lWNFqD^zzbu^57Nbl`V{zX_7NaCX5i{&~M8V7^$Arx7T` z-8*NL&LCiAfp=Lt-^befNIG{1&%La!Vp|LzH~Ny!T_aqj=Kz`;wMcjpRKyC9Vz|y{ zu|g2#;%^fKqVo9ktPK<5D?O7UB#ne85%PNYQa`Jum&<#JZau-)vL)6}16Qg9z07veo;_BFb^?B~Y8qjBz=tt~gW zym=62SA5u_+g`~-ha^`mI?zqQ*XsY5ug$kR5qajHHrcq5=GBy1FBJNxs~%xW{kVh`w>5y~ z(6L~cYql5(;1&26vc$VnXC=~(J|)?>75vWQOYTPTKT4Y?7T- zQtX2T`*29ea8=ex-1JRw!$gh}XG`9*S#EXOi8ZI6iWkR}%(sK7o%*B0K%i(OYh}&f zQ?KWa>ie0|4thkkVMaVcsDuBEhUxK%85S1nNZvICxv{zDFo4~LJIViBm>D zu~iy(gm|Ry$Hmg4;*uM#3J-!{O$E^Q z4z1aSD*3OvEA7?j6?cE|XPHMFvsF!=Lt*x4ei@wYqRI7$5+x`?Z=OaOn?3X{EiO** zZ?)!~OF+PA5bQYyq)mHVtk_^^Mp)q-)fO#YHpw+wun08b5SfFDb>iV|4mB2joV(q$ zr-4C0fl6mn))KQxvMS6rA-mG0ErSEx1W7DnHK3^);~tR5_b4Q(MnC=|amnIVRo*hP zHU&p-CCuRX(ENLid7gisnmWpBRiqq*T7_}LO+xzIM# z1ymi%>y@l3#EwNRV+#A(1vkc5c3>5Qx#GjQKKH$FpR(3QTMTgsVJJc(dFoA_V!S^T z96USgg+#5Q#N}4OoP7PMD2KiOC$3B_r-NXCKRl_D6bM^;fN$o3*{MJ~$W*S%;^9x9KYMx(Z(#IQ7=Gp&De{u| zN?bNA6;bVq9V%|y)-+6rH@TnoxNIxkC`(Gr7QP!ReeOHP6a`M1WJ4yW2s;$)q*OFY zW^z0wR@b}+3#1-j{L~gRXogOW+c$5nCziutUBClrco6R@dM$QIJas#{jS9=EnEEu? zVMjwlXTvXQhXi|)os4YL4lB9;Tpq4pxBN(x?fHmT^hoa5x(w?sNNw1a2g3Z(Jv(+!vi@^>j`Ods>w07zH z_rAJ8Gw%QKPg>IM6yz@AppZqdtgg?7HbHm1+t0_RCFcYh=QcE{JVr7yNWS&snojE( z;}899t?4c+kb4s+Cw-`Y&fkpKQzR{+lH9d#-#mx5+TgKi6WR-Zo#+q81e*cGpvy>lEmDcriJ#f5ZPU<0_oa2XIKaR-+w-d-~9uRj846oy2C z(uTjyx#TWAE%asCdDEB7)e>YVc~1~Qx1G5qg--1;Rejcl{yzTx_0fnk5YImu8{0+z zTl(mb`Y-3=vD97qWFvKVz+%d=g!20JTYRLxZc4trzF3IF-vi;d@==>MZA9h7P+)O& zMb0vE*LJ47Sgv=V*hpWsk@RjnzRfy2G^P;@j=-pV?b-WRrUT|(eK2oaFE`t7$HKh( zV#xB`eSSwv|BW^7w5hOP|JulGX#&UPwO!_g;?bJkhrkjE^xhDutEaaaK|74i)u}E* zp*lMo&zL@LT<`h%aqD2Cn0Ab#2oH9QdDGU|gCUzXY3S-!gjw808 zUlPuqb(~vm4a;)gnyWUdBS;^#&4nHkL&|+r!`*$%LE~Qi`}A4ItIRl>GZ$aF*uKQ4pGQiXLkRTL=ONc5dr%(-)=L9HZ7vg;j}KZRQ#6m zlmn;6Xe|)M?^Ah$?O#?9?Xq{-<8QsGD_v~2T7<|yY{G^ zk0XKnwj@lLk(+OY%?S&PaTJAS;nkgUXouS`+DS!{vr63<8y70wUb z%5Sj-L^inOg>lX){2niyYb(jRIIf)4hCZ7+mBEZbJV-1fGTLy<{<^v?U~lnmU7RG7 z?#qB1vn6dbW67rEIrfI3i)Vj7MPw#=0{NVDJh1Xbgyqiy9=xA~?e&DQ@CaU?yPx}7 z!gG0>K3ZPz;Rs#Di}dlHOLas|@1J?JQ}5Po(RS}t14V0H^553_-N%djr~l-mrtILoz;U^`82;;@A(lH>Yko;tu{&mlDsCKea9epIzuMA!09;#YOK~{<eJ z4_d>iS63{v<Ij}6&j;1hPNR&xFE zIh{HEHV<(+h!4`M?z09IvzG>u&BYc<*4726)WDnP4*$TlId^Q#FU#icX#nROSehnOOHjJ-1!dB z{9MAdNda0G&q=X6s^i~?ef-hqBHFn5w`8vvGtAY>Tq?vXdJcx zwHo8`6{J{y(a3Jcv@uxAXY`(JIqCWNoy0s#>l52!IzkBobFRA2a1KF86hJ^ZM@}mR zqn(%!#BH^H&O}SgZSEB%LAO6J@Q6KtUaQ^&r|q7e^)Tig#PG4A$JgOJWdtR9k7&2p zl1o5t1hX+?s;`cp;Z~e82##kOu*oue`)1JgL+8$YF?vVz&ukMaLTN_roXaJqE0<+o zc%d2{6?Mfndf1_-HMit+(a~wHz2EG?C%5Ix?;f2|L#IUegP%77Kppq0>|9N8wz09{ zsM1(zi1~%B$Q=v}EZ1K@uyE$g!8$LCZ^x2dvL%6{fK11`OGDETVgz*Ph{Q~wGb^bl z5fgkLTNcseFo<{>dnQds-VqZppM9-{$j3X4%HPjLN0UVq+gbp0{_B-_k#P^p>NaY$ zY0~kPhP-6+yPhndpB6ZUi{JP&&()IL`0SqHH8lMiQjHuk8Y!&>C7+Gd*0pDoU2{0t z93}3ZJo~`hiF|h~(nw%P?Twc*ew9@A6Qq~oE0YxN{bsi-gPc#6f0L{=z{f#662;!q zpUa2eaB*_dYGKT>+9&K30(F7c+qX{DC5aY2*LTXBukXAHOWIs?@iu5TtXCTDmXe;@ z8LAbD+Iy~Y< zmXyCsl|PEr(pE^kD*pFG$$vWj@Xd)z8g=@^!K~ii`1+y;H3d25`^t*kN(WhTh&WDE zUZ%cFN?{ol*{9r1#^FC0H$>qE0NgCJ#Fo-1)RjZlg9q2=#a9x6wvk>Mwd=se=z9Ps zn}zN^Fg~og*NjKKGbcY@qWiee(PHVF;-r>w=dIv&<>$4^~1U<{vIaLukcdbE%!f({rZ^*ou1PQpM+9KB0k1!d~w%ur+o_e75{4_YqJ2 zhW2%}QXCu{Bs$0T&S;Wt>Y=zN_trCV5?h4=$0WM@Oy=1{Y$jW?XoN$pIwNahDk<4w zvjl`$5HFyuz$m&Ia-*Ey#jmGO;`b$sIcj0&X0?@KEVmOyJd=MULB&6BV^L?WQ=M~) zyv|QsF4@^@_@t0_z6k_i{!+|gOx4Xo)t8ZsEPEL!9l_AWq*#Gvub6rVUV)K8mP3QP z!?h*Z99;JFMtUD5XrIx0HohE+d3k|2_RGK6`PG_%S-jA`LFZn*#s=%;pF%ioX~1kWB`l_C6^Zqh7Qoc?t_Z!!nuF!SzvnpodJnL>XD4+lJf! zxnsk&bfx0eLv`_JY9uR_c}5JszaQrF`TqU;QVB$#Iq?z!_}ssP&df^NtudO_Bcueq zIN`NpM;?Qr80X(KZRp4@6cnZ%7CqJ0*Y|X<{FW>K;ihMUc_Uq24dimB0i9n6iAl(g zmigp3FV+DtiG09?ueaDSwyjt{aw{uV>2?j!;GhbS5tv7*fq%n^HOlwq8i;Qeb&8|FP( zl1@Z^q$4yk`$8K4x&*JJ-Ca{LsC=mvf-uL}Z2f)7b0D!}GYk7VpP^PBM)y9Eg$(k9 zmAmXqqKb0$Y=T{}7(|-|$gq*&+Bo^u8n%!=MQ{V9)Z`%<)u-p*QInJIH#5Jf=)%FN z7It35QmFE$8ycSve5NVa240|o)vm6pd?(WMYz)hq3^pi<#UL#UeOP%UG#jGQvSUlV zK{?>x()EjY9{DOpK@GFdrCgjmnQScsfxF^!iqT{%ok~Y-ZTm*|W1_Ko6Bq!TIG z&RMxiaWM?dX>aGe(e03C=AH6FMr+xoOkft|(6F;!9|ux3DBY5V!&L&{DDJMwYl?0k z2@B8We>Szho8^0<@c7#t_2G- z_+U|#v1eAMLN6tUljfIJ^W&r$;fgH)Y_YC?a?(~bFF}TtKL(&E8w^7B7mb#5Vbl@R zJzEwOflvdrpAWF9vzAhZ7-HP|iN^UUOnM|o(IC$3-O4)Vl$qu*WM`~?-2|npR)e_j z;MUKU>j6M1!R$K&dR~3D{8o|!edmyqlZKHHQ+XT;yhV$Ir7FLURy`({fB`FS&b4i% zR2K^vY=BdCh>yLVs<=hKx{V!%eAmKTD;g!P@VL@eY6|(3{(t3Dr4emf{xc+}^{78C zX{YM*9WQ4cf54DzU~T`GQT66IP8*JOmZl7VRaz*EIEEUY#-43%ckbNT24TzU_fOYF zPJ+DQN@ZyG`5a05Q^b__1Ojh;!!3v!8Km3q^R2#1Vc_UM1tAXXhYoVW(J>1BkENDR z?`;2MA}P3G|Jucdm6o#M32WX+-&?U_Yr;Fb@y2YsN3uY-@`F)@$B6HN7aN0L@VlVq z(qBx+DjRj;+0kCJ=6lV@`&)4B^NwL?AFvo+kps53_ug-;U`A)k~H%;EtCa zWy9aTUNNBj%LnbO{-0^K#VNi>Of`!3x{N%jt1wo^{U}i~tbU)rZX%CD#gAfl)O6+( ziQ2u+s;Z_C(iYmzGiaVT(W^2_e?uzBO|;9`mw*4UCO~L8$?-?C9mmqCz*X>m^(niu zU>cQ-7q!{aM8?cn(5=l&;M59hJq@dBOMBv&`n49;jy+OJCEMvX@8+34zJ zrYQk|c%pfKY_ra<kv2V+k=E|k#Y@fC6xt8Wa-qeNoV4I+ zqegAfnKiN*&(2(THc0yvZ|(FEag8o;_k(E-$az2F?PP6K+$VX#bR zQ;@Gn=xlhr&pn!@sV;5ScV5!!x4P~>S~h8#*8P8KXyWZ(OTML)I6y#Dg?=FeYjZoRy`(dzP%VVCnqS1ewoRJ~?hrJB{6C?zQ+ z&8hs-YtFc&bxJV}s{NL6w(<{6t*d9OX8m>YSWQin4pU6lnbzm~Z|I-8sj<=bCffU5 zvU{9!Dw&6rWy(o+SCb$kcQyAfDKkP-FD%K>X0*#=BEId1orv5Gge>pu8lIqJQ#U=( zU70-4DIv5A;k7ntA>9C`qG|)Dwm+i#e4JBAGg(N5V@b6YNGokpQT=% zjXX9KzqRhhdI}viRn((fP~u#$g9M zb_0rx@^P=Gdicz!X`;C~Ms$nB6dJ~b>(Qz~J|ILP;p5`jh}7ep>Z(3jT|?u>asS*~ z`+(b6F`G5F-E1xpq%+=KeoxD4TC>`vim%6Ctq9rn|!V*BHm7S*B4p zPSgtZ-nS3^g$FwxgK`c?85S8M`DgjAT!Fr3p{?V}G;w8t^21y6S3VIU7`peW|FdN7 zik1Cwh|Y~mx_!TmUl^og-I$|!u13Wp?|o7FUAyL&{ z)>c*xVZyjvNBI_|iZTeHhAb&EZrGP`+=HhK49wK`whQkQOiUea@Tsv_cA#>ZfjiAQ zlhN5HMWOuSw=`izS>&now|0nwVNpy0r@Nwl|CD2ZUl@qPDa+;E5^O}uUSzEFMwH6M z;@Qk}EZp)bqZ8mc=r=<9%wb7f$DMB-exy>ca>jcjRtq+e8@>9)3~ z#$GCmhpDtZ#G37inec)BwQe80cW?YzP@6?qvM}_77<;;A7B?&eFfm?<$z0qX6qIHf zhs&TRMW6+t<1?4LLGtB{+Pn%uMcsmP)k*i3 zb#IskN2iY_C>aE#yyDX)WLCrTl9NphC`AgyscCBH`}fP!Rs%=*wKraE_@RuuTP*!z zQDFAFT3&uW#hKez8W4*~lcani=l}BQLmuO_&omYhWYsyO5C1bfkIFAepusww{zOrg z#`54V0`zog7;~6=?|eWaY}Y2bJgM-!hG{EJE|)m$Tog4e*!bRQP)X{ygF|0BZ){g| z!vN0U=&-m0+MyVU-1DeE`gM-FDlkUySI{T{zdgq{O>OIOZ5@ux);KFyPCbY5OFW6v zc8>;glKc}wiY|UK$Sh_?gu|j?OmtD_^t_4!DMz8fb^GeW^FfD;yMwv^I_e2uDh-Os ztyFyEJIVGEu(J!6!>Sux3bJL1h`=$ON>q!C+8y+@ti7>3`C0Ml*_R)9CJ)ii@2bCi zTZ6%^3hN}8DAg6ympD__Di!+`+hG^Nt^(sM8f7T>H}-0IdJH?pnGn8&Ks&GK}8dpdra{_a)a z>BE&qjlZ49T5d3Kj(x2r&3~IZu8WD%gVHfdzRR^HC#@YBQr)7`$@p0FyMHwwSLf@M zv0DD#+r*s&%re}2pqOas-Z&0gZ&T)IOC|{CMo0gLjFt!pFMWT}OD+9+7jUZ}*N^q$ zzZDI%DA^%#S6q!+=KZpFRV|TRUX-Ojf9dSm)kTs*Ko8qSt<^rn|D5QH4@GFqa;l-&O1_gj)-!6#$xQ`i854W) z;K7;Om7Pu@!WS#SB34c@)%tYW(chWA_SB3^TF3rG_Dk2lGx_DbT)*q0la%nQML<-t zQ&X>XkcZ4(5SKJig=c#7?W>Fo^YaDI(gpLF`bF8u=y_i^n+WheQj~0?UDi_C^B4e{ zY%Hkr3YELWY0wmUdZ!Lzq!Hb8OpMc$7Z^1-^1}BV99N#8-}dR3(#d4*yeb>d!;L(f zZh+b>JL`z}{CH19;)2!yOW3=7?8(wg3CDWinnOp=qVQ&+!5P@>Iw(4N<&CduRW&q& zkcddOpo|g_taIadDI~F31|1IDIPy|XpFSPaxJN`-`1FwR$%Xd42s#>WGxmJ#_495+ zTzztm`zxX3+b8wd>>_(LxP}(8jzzPH)1T7LPoBJ0+WBEbW*U3G+}^$}D6C1YK&24_ zkj)*+Oikns&R`J-#{LX~RR36A*S{gwYrB~E*Q;cV5NCNlf!iEN6vXiuPgHYUF`L7j(rVr@IO z@*xi+D-$05Nf&?`c`v(%Uz~XT*_jbwwC~M9NiXaXe<{XxkZ$Jy#2@_#U97f?>Ux7X*vOsOQ!^LzuH};_F;-A6#$m3RwAf-mQv7S#Da}NpF=}}6 zHGpAUh+1AJs$q@!Fv!b#5t$t{`a^&#QqFL_vq+`}T-}oLHyua{*$L1^8Ju6pA1Jx5 z2&^^CbXVaUmNE)2IIKx%kp4;*6(L2efzIfqUZ4wi&Ik}d1&>U9jnR4?Q_)^v)?NY>^I*80cz!|DA6@K=Z)KH4X%oY37PkIPJ?C z$P^M?NBeHhGAB8QD9qteNWY3&rVWGy;utn{90OhAntTee(<7d|xZbcdQw{h6^OH{z6gtph%9EU%yk{|-}k9ZsWu zYE@kJ#!LjI9<{9Puch5R()MJ>NoT#Le7NOfYv@+Lky5DLK(jo@tZOyb8rhED-uUCN z*_(}a5B=V>Men{16{~xRl@sb)hSLPl?k390eGAh$FEXfAlKTw881Pw!A~wv*sBU1Z zo&Yin>deiqzZkK|FNiF9*|9F($?vb!X?GB&_kq$UOwzDMR{#q&%nAmH8->_1|J0W! z*t}LvnASDHeYX8yuy{pje_qm&EpMmbu-THcgJ+EpC|LtQY4RY6(xT&y*D71T?hXqg z2a8GbTK6_CarL&|P*>@`cN7{C*=dIaf3sKSUSQ&LifVR~j5-&Qq@TlyK;qOC&S;nQXBvRIcgEy%YX$CBr0OI{Yz z@Ma;wWqlY6(1{#6d~M}`80^G{PM^Npx%AK1g@ag;Aaq}bVV$+ymcmtLUHUwj>Z{!E zTy(|~kzoppV0R`1Eeu-qt(?u-EQ)!ulKU)TX{Ig~WN4ed=aFwA-)s&P>jfYG;`f(oA zuU3>?21Y(6@xwu$kc)z7N%0iMt{}?2fNzCP0D_E{pRB#4&fBx`@oi2pNGyn>`UGUY zzg%L|FaCZ1G48(3+nn(~f4`;K82|yU(_$$-K~hqZ0(c{^HZ5wg(rLFJQVKnrrCtfF0oWy{QACG1v4NQlDNi(3|MGqcsbqwEk}4Cx)xry zsxjO0rLR+Kx(5JHUN{i_mxZ*PwD*X{kgNQ2F;lFD!HmCFi_cmW-uF}jYK(|SO1Fya2|tZ~(lJ>jrCwivo-Xm3m4IyjUx&(X_*TN60NkS>wZEzCEOw6LB5Ti5MCHPH|d z1}gJn6sU^9YQA@U{bri}0}}w;bL|_Bhl|Pvg05*(=JcWC7EGhYvBdY)a3WyHNei9g{48< zSB0d{i_WxK)V6I~!^rFsLs~qLui;UYP*IyfJs%|j2an7|_9;9q-EAz}eOStr91@RD zLO84S#-HLPg^R_s3N9Lj0-21mpiEUy}xCX9k0dE z0ss~4o=fjetgPq`@73}j4)TsUZ&gP0S1ZKP+5l%W`4V%3d|f{I7??IhGQK;2*?t5Nx^?ua0Cy z%<2~Wn)^uH^XPxsG9#O=kiLfgaAhv!Q+C)RF(nV+VrgF#CgXG(u^y#Mz0`AIiFau3 zR~x2#5MvM%FglePMYBOn2;8O59WmR4|M-45Bb}v5B&l2Ugh(fnu9=1(6I`voH)W#R zSoUKO87wG-_(a#!AY{CLebd)-B{V`1IJqqe{wJ}dS%|t{8`@w&{KOJ)tUKTwdnd%l zVw}7=DH9cfPzcy=BwF2aa+SuG2Z|fGz_2c|%N@|xcy2^0q7aOr7CF`RQ!Hvi0wXq& znaJm~ll)=`o6$0JTmG%7j@9PkIn16Qt6+bTYW|c@8q3r z!>)T3*@YbjR}XkMG9)qUvcbZZGg}=nF1g!j+K3acf}VPXM$VmiZ{50_dIb${?i!^} zY2KyMLTTv&rRvR>RrFVG+tv$-AH&`?g^&?2k4&^L_rfJci&Ooc?R{HRmPrPM$h(;v zpB5KaHZls-k^8YId#iYkMN#vWA4h;AuD>w8Qm{bEhU<(4D{Irm zJ?9>U*IoWdgeLspK?`4=LyJK_B$0CpJe@Abiw(QiZ44u`_FHPn9upqvfGDh=lDOaU zraAPrX9C_|1a6GgsiUuiE}q7|E=$d7AAT1Ap!9R7^#uPhi*!`RobW2Rn^>DX<3?W$(le6cdh(=SyfDlD1rrdO%70fKVEwOwPLpgI?1opQ1uWdT<;Dm@F*QW)IB2PO{q z+^-G8xOo%5mE3qbFWvSp3kwf#>BWiRY-)@eR+PZt!4DTKn}w0uaitu%Jui|?CB`dG z4JLt?sJV~l9UgERt5Ex2)58${l2|Xndbr%d3Zo)*-YjTlQ8p`BzLVCfcpB|yMD5NG#)u>bAZ|} zRfpP_kB9;|4MI?x4uTtqE4rxmi&l<{g|;KBy9M5StrU+Mc$d%KR_s(JQ? zmtS+OvVM?H0z~3J^jk`&E?J-OmU2O5mIsyF-3ABE0+?lyQGwEUp^^Dhl={?>Y@>YE*( zPAeVpsMEg3)6dFcylcU|uG~-gB9anB^wv%j?6ANHL=rdn<@kLQjI{F>&Wp$xBSFVB=7cUZew1yzbdwcI+#qqjEuWg4JZGK@{ zLNpKl?%o+$yHZ|AUgn3?{<#W*Ed8x$qPE=mt~PHvFM}LcN!r@x<@M$Jk?#{aud>rq zg-9Plu)DG6kr&0qjbkje7F`(R?XBzL7iJVYDr9#TPZw~O7G7~{?2q(+$VN~V4!+EM zdlJ?F+?!92&McIp!W&(nx2ph9K>ozPT5ydCPr4^57E&jVT&-X~ORYqVhvJ>K=$88s zV>bMKtj-DovWHq&>@p5}wZ?$vne-ILg%4V*scpwJwk>FXJ~R+>hpaCQ{;@lE?)>?A z8yXuvbvf(9#;-(hC0W%n|0D|-Xr#%x*t0Qt9$VXnGqBR!TIJMnkA{{O43cNrA>ejj zq@V4;8D;Pl5*u%$&ha|~0WrYsWocBeAJL>oi29g6vT2hhy^lM&PPMn!W^4_#Asmg{ zxt?;`5;g~nKv3)O{ki^7^;InqO`#|g{i6uKgaO@psSCRXG0&DAc0 z!0UA#5Em&ec0XgK(_rjC9G=bM^yze~T#XnTAr>wKl&ubgv|uv{pMZGWEwdf3-kZK3 zicpSkPBB3W1gB|P&|wp$;3=aFwz-LA#9vZm&MzBmTzuo%a?DqUsmnySP?t5mJJ8@# zml?pb3h?lM)TSq(>mwaWj&UX02_|DMRDkVq#~<=nKEdywMYa3Y^L`o+f(or58;wG( z;+ehWjR6342v}mf+;DK&`}mS-zpWQw=n#@RkXlXe*Iq#bH8sT~gHqutzPOw>20cZG z;@rQ;@IT#ke8_m37etI^L3@T}{59jgYfhH+|N@|~KrY}whWOFjBBylrW3mtQbB z7C-yVom1n(1P)`|9&R&ZO`usw`rF}oj95FKObV*A7vSPGPE_ImbU@qr?YZDI~ zCe4RwEBFb-@WF8x-GaL~_u~nKRFBos5-5s67M)@&$OvfvZ^s{YJ0@2d*+_f3{2sRh zBf@h40tTsEMzDk#nX84gYnJaE1jRoD3u<7Wii?{F?BeCc{z?!62QG1UJ zc$bZ+dDEuaqAsM$-9JTGj_l@I5cNcVOuSj|?#(}W=ukC&tDwE6E({pFqT_hE zLA@5He<&*(L1Gpjur(EWS1ds-hG1TFSfZoCoMBY|;y8}y0s|HjTQmz^)e_SkI(^Bk z6$9j{QwJQU&jh;@PK}gt4ijnoo-p<>q)*OPCf76wj zYYAY)AP+(bsl_|U*Rv{2Q0_%he&6D4O`6n{^2nA3<%dzzdh*UBxcBS$=ZU%gR-Xs1;ogmOG@QTmxY#8ds65qEABYcJf6!| zHqb@JD<}HaE!Xfbcf?hZmg$iAAN1>D>m4`$m1CP4@vh777Y3TlxTMZb8KpCa_aIt% z6Y=={>1EX${!nfeuwrCekDQ@p=i6rI*__Ymw~5*GOvk-R+s9wXy0s6_A}WQnBcC86 zhFM$ZEbhE%JGdbTuLzsBs5puku-o37d42=f#VqhPu%w-6w%xve7@>%uQYf>n$8sA8 z`HX{RxOpN@ehyZmrbDqQ$D01S+^Djk^76CQ6>hJV>n$fKS5i8juW;y$20%jou;&F_ zs-sl?`e+bi?G2L_#mPMFG*S$0xBaZc=jZ7ufnZ6SupPU6Nd3IpWEcVY2HG$e&r(@v zzEtI9inj$~3G@XPJ@I4EJs)PT^NT_3*_^;X$STmY4ZHfJFpmVt!K7UlHN^PgE2eW* zGov?qh^v4-ZxeYbm-rIRl)(m&S`j< za|24()Kx-6D7l~TRUP&8Rwo?rx9%xpvs$}$9?5f__o>%Q=d;26t%-|04ZMJpM0?CP zB5(yFZLP#f>k1Nc!e?BP5m(%R50Cw!7Lz*Az@W!~@H*$?Lz4$0$xYKUG*lZE^38IC zapZAgM|8%RnSDws>*3dy9Ea7aud@zsxwOo@GsmBLjA{P+EpGacgDI|HJXMy8s;-*x!9Suv z7Oa>YwsU&_q|vYP=-dvUBtxIjjVC4@Px%PX@h&H>4LC!yBmQjz8sBw@(`z@=_fi9u z%qE&%JMMmcbs*)Uey==OA-+@!6;^nY9TD~tK^>hpM$8s}e|)KaDLwob_bt~%--Q-O znuWAIy!!KsJ=n^Xw3X_242gTT;QE`(*DC7ie42W9kS)r!otbR{Ry;K+)3aT%qodZ{ z##@8;Olclwy!n!<_WlN3+xJ#dRe?BZvPS`Mi#G#Z>OhfoBe+ad$hvi0$0F#EgZzP72uUbg zrDk<|eS=-wqP`+DVP9{rPoZvV{8^s6M=pDFrzpisH+5d*Xz!`IH9TGo`dwB&>=ezW zO4*#KX%a7O(UX-8m^RT*-<}3}g55K9Q)n*`5j>g87ha~@oP=db7W$A?V9^t08e{P$ z6bXRWVy)mLvjDxvv7_W1bMlrXy`F_QULq=?t?@MyP5^tJ>^`9xdrH`y(TsL8)XEyi zKj%_y)kk{@zj!|K^N&PMf_DNK9PRt25G9r({o~}xK*tyqonwxQ2@)1mC-JqxbTXwG zG!^DC>CEA@ihsl{)olaGY=|!fgI%u}V{M%G|~{59H~QmY!^eNO)l9U9SZb!rQ3b9 zRiFS+b{m^6I3;O7gfj@*k4Y+~4VUSKM!zWmkcnK+v)Qhr-7#6QzU?n`hE;9#AWVi> zqMo{5T&ld33QoK7vzzYIC#@j-;fGEV6NWkduh1&lf!HX~i ztp**MkDOL*#FXEf7n+}5Wl&o)!SYeV-+mhkkR$bA-#*>N>1RbL7Tmv}(SMJ#ic#3V zBW=Ib3GXdC~O2)|FnF zcOT9gwlJ{^U%#tunDHjf77)8)zX^Cdh#Pr8V3C)~BrZ^iik57Q1w(H#;n44%Ks+$X zZC}P2Cy9xPmYo%qhw`9KdRt=SjK%=`m#Vl|J#@M=YlU(C{v`{E!ALxt zIqiTl2{2qUZK8Mb9GT}c2D@zMPope62-#Htn{0=>KIlo~kF{yTdCBzX?=1KI-m;|) zOW+J+z0B{wB~b)C-%_Pf4Xia?#JG$&WQexkVv%YfCjzHU+l}t}r|@ucn+4i%!N8{QxLv$ohIj%q^9W>!_&#P_u;yIy^|B>u zm9v^(YvH28!&gBocbs%$N{3nMehpy`g2iNsOyh*Pt(lGwueGGrl9`IOL(t9_=bsm= z{Y;hD(5S_&18MK@?Z;pIscCu!23duD?`M+1MGS4rhdKGqjXyTx{I#?Smx@GMk@%9s zjvcelXlE_j+F40?vQ3J`1Hsdf=^#ycX^?{c2$~;SnNJMPIbU6>)S%Kw^GzFEDmo+^ zCH{W^dkwQ~dBwM4G#|#FnXJFR9_x-%5k+oOr)C6!0hiNWO8BO&`^sV0AR~yUpGCx< zfi63a@folP|4k_%(@>kp$ty?F*{q&2ETJR|9?AfQ}CALyg zv7g#PS$PBgznC9CY{Rd8QCj0+Ftmo##@bun4Xyjq@o}bq6T`F}CqC&gMoO5ygMHRs zIlyuHVgCiY<_1*z*OH}?)iZXD%2^JVT$^^<;j3$?U7MoPDR=MoUuRlp*|1WL>TCDS z>OO7L$@5uS!{(JP>f#mX`^xF1r96V=?JD!!FQzrTDJy#DBdk=CX29{vb(WeM_`2P! zN52?l#dKU|gx{ZW=_->+C$lnV$~j_=4hR{cd5cd>{^O4hyyaVuIwN+6k2P z6DWUd=pOIY^1h7Wg`HPh(SzwpC^J}5f@G(e&dmj1X+nrJNiKo?Xh(st5pykjliv$` zUs5KfOz{Qc>KE1ZQHsimZ|ecKdi8JM|75wMgYHx$L;>JuuwVG4pfN1Hrpr~#Wd*r+ z5KWDnI`y@sBU%RBzJ6XOlqB{2!V1%g_3PI!iQyZ4O0SO&MEK`7B6TK@3LS_yT?(33 zOG6vjXURu`p7PwC7R$w-C~-e5?>d$O?anGz8RIoNuIhyLA7KSR-_j?VD-b_j_h{#~ z8BtTcd%Gqws!BtU<~D^8Z(;T7O^AEK(X@MZwHDW-S*!tpdSa6X>6@dSp?b7Li*^JN z9>D7IN4!NIkWBFp%aAZjE>8FE4qt9Lr!k33YH5-Gc~4*nk}L-%g1@eB|9<6*x%7&b zi*S!EDIH_o84^##dXnuyR4-q6?b-74C(?0LFg{>emkn(MI@r3{E)l-zK+$Kj1L21c zUkiy?AT@a5f>X3Go(cR=pI$r6`{V)E=~FV8e>4_bw`UT`YPtm94Q^6#{`N!&Vo+3v zGi|Yelake_DrK*u>#lf{4r8tRkxSfB&6ZN_1n=jC$f zBy7sZCYQce6025U+_c$xF_J=ld#OR#SbyAHi8CBUGRC8V4_>Mqz>BF|RsUEvDra1D z%v&=Y+!Odu4eXHB(*M$c7%#hO8!ey+TL6L_zlg4srVRPa9lJC8<*@pgX?mD>*NoAu zr>o!IIPBmSi`3VHeiIYbtJGVc|D)roS#H{P?LTmIQp>r-I%@Ig4Atw{gqjv%!c5Jd?h=SBO}4Cwx3);rFb17e|)_&cE6|=;eL=D zu`}n`0VEh2ZTaj1*1(1K8==#aBu#L3cV=nP>9G zbkowZgC(H2GNlC!Ti5lYAzCRU@%k`_h!y1OwqW&lJn z`Z8GbNe_kXk@5_dCQb87;bT!}@0M;T>H-=2N52dTZFi+)-p^(^$6ww704w-`mwPot zD@Q+s5UA_O^JVenO3O|f)msr6zv;@bmiL-|zWMvAPW^AzIYH8WCaf-GWsSIogVOJtMN*M^K z5Ab#0ZvOmEa6+ceoA*9c*i#It&_2HfymruMdIIrjp7ulJC9N^VMnSIzu>@(?Z#VXS z=aJ2Xq9RwqJGte*l_V2V&;vPSEA|IOqODx^tSwMIwq~6LE?b;KujVIzMM}^TdBs=& z!?)BNKlZ)YgMsh1>`<==O8hl3PV7^#jA2h}y>G2O_}hJV%|D{3r9mA!Af8#ix` z393PrRXx_(ubq5b*y>8Mj_^y^1#-po$rm?nDmDFie6?rKc720*gQv%0W^zsatvo9H z4slp<&ul(+z&a$@j%MenS@J0RMR=ts@IHu+lC8tyHG>eQT2hBJD8kFvpk3HeAm^Y10^*c_%BtDIPC1udZbJL zoN=diw>mOv?X`k??G!-rJll)EXZ7}&R=l-~M)0iJsso1jd|BmXI_>!| zulUVAFP0~KI=1*ur!Pz1cIq{IyKVlHb3>-=aGPJH|MiKk)uZ{TiZW-#YBY-d!9V3bWL_o>@leFi00zrChUzHMk71a`Rj}y z9zBj2()5Wv!h+K`u0Pr#nUvq^&N4OK&e=&an!Lt=q6J+icXl<{ZGERCP|~uwNbBEF zAE(Xr!VSHR`AFHPyd@(0%5UqQ#k!i!m~~>tK&LNfpBB~gd5jI59lxKF-(YRb!4-tT z3D;xq&S#$0m>w-|;WA%VYYT5~Z9KKRxy7qdAKNn!u-uIU@;fN$8o?0}e>});89PPK zk-mZwVti#X4>S+vo09lh=BexmA+>Ji2~+Ha+4rQL$%350=O7F4_0g;W(3+%Ax3ugL z8Tp)-Z!|6H(smD)>Yz+A+P~d=w4tHf@$!m?=8p|CjgS5=a%7?J18qz53$fvHs6KEa zh#OI23t?bOou?b6Klzb*`}%#EPRPg?KuRrc&)mcm4fG_@v_ z7EH9DvI)nY8!RjZw;jJ)Kk(yflGQj@*VwxeE=4_P&Dwg#l~^_fjv+7G(X$pfc?J+E%%>{$A71Ty4b>$M%0b&z{|^EGtViVjV}Au3$}fp_B4MMT*`hPon5BZyZVN*zr*(i6^?{~)- zj|??WIjs{kqE7@3Yq`UA^84af71|%Z*=u7P7p(JO{keWRADl+no$$|`a5^d~<7nQ( zv17Noc{~%l%%$zI_<@M;Tk2vP<0LE3LKgg70dSgMh2}cwvoPqB^qvhRY&|3k9Y7-R z6M$)?agq&4VA9@P87JW@^gdZ$QIU=B*ta2NooG6QZ?UKi5rC6!<7|X6a#zY@!s7lf zX(@u8|H<0o<~E;FErtlJfKG8XkDNR?;Zu||rO&DOb}R4pkw7)^vgk=k=~wuXTS4r> z>DdjO=Pc>NXJYk|9hv1~|43jTu7w?+(?Qx+@b`zheEV`VYvSpdh>Tk^#C}^-(u`9q zy`TNF2M=nFZ1$xncgZom%OW%GuDr*AD8;i5dA}pus}1@pTcXuNZkdORp|KdBtzNAM z05s>m{k=LPo6VVbYN&0lGtk4B_?tC8?DyLR|L=w@boTlBYE5gqQj7A%-%?3KI5za% z`ra$m@yt<77tqi$^8z;BUWj6&AxBcI?NU=q9%Xf5lYlgVAAFexj-DH|ueeqj&{TFi zd7^r84=^H3$hpwt1W4*9kbL~DTQ8=#epM4;$Aqzl@8f=DT$GiGHADwUL~DlzK}~9j zH3+0Jo*?<5h;Cr+zozi{V0#4Jnp%)T*Sj;Q=-&f+4!{ere?Z}|h77Kf$|0oE@tONNKRa2 zZvMLy?Y_=HkCX5196XEbPVqB|Q|Y<0%gkH#$`5zaYu4sv4ejWDCe?=tdXVf#z9RM# zTWApi_L79^$*e7^DvPQF&T(VU zuwq4D4v-)q7N<|0YREntxr%BxpEy{6AZ^p;&MgcTI9^@)gFmK&_P2^>7P1Yo}Q%{S+|SkJ+rB` zfoe~?ZnQV{}hfZ9O-sR_q%*rT*sE$*ua~(M&}w3-zam&;>DV5q^c`td7IqqUvb`` zpp7{Z06qd1rIfb4R#I3w{CnOMirqdS-Y6fKsuJD?W#Z<|uo&9`nwqt-`@St(8}%AY zpN#<3cv*#ayA8euKhg#dNZPuje;ekCY75H$!FYTA1b7mx&~@(h_05KznL5wn0`s?U z`N`H${oBzopy_V&)Tj6AebH0!BR-k`I(4phbossB|3s;LTyo%Ljrt#b7To*1+GmHF z%DV9R!`t+~kyW(S;!3AOhiz->Ux`l{^74uCzNOuUwF`f_b3yBr33X=1)rIM(d|OAq z8(5?hiR3~UL3R~Tiels&D!wmw%_?8@Ydl)IyMm0(vLIFX-vuFPMnximO4VbI(QsCT z%LQ%T;SqcVX$qwb1h{`1Ccyysm2VVRN>RPalw0>}X42nKfyQEGhUv?zP%$F{iV%#k z>QUw>`Z=z}F`hNswrdC7wJSp>>x8xqH{oyOhOowH@A$#f+nrL_a~PT`*u}56mT21q zDUGnU4mC7-w)y5ueEnMq)uAf_AgOSc#wYMFCI8&_j5-nrKpz=xO|VTKivn+=+Im}w zOO;iVC1j;fiG?u-7VET~W@_5w}0Hb2gSl&ol8A$s(fq#Tr$(oAEqEEho^VjC$ zZh>ry9`4+Vmp4p?uhA=A)dTmtwrvNWN;a~$eP_3Uk=DcepE)d-B~%DeH^9!j%3*tD|n zL6HEt07Jg9Cuj+}=<5CvJOcs-mpe?lVLVu46udD5r^k!VT*p~Ra4~=XY160Iik?^X zz`k#6DypJ92Z%*EQJB{Smto-0Kw(*@x&B_xunt0K{$F|`0)im*km=oi@*;}Tp!k8R zABm2BzT^M|3_;+L{u=ErD6}&e;WyECu=)#sMS|}c^M${_K$=_Mwwr^4LZe)2zMuL*>n{aSIac?jFUY2zvJUrE;2*)ed28wm^grh6Z9EnwoIE^%dKoa4U+Z_x_=H}g)>;nYPWD*{~e}XBu6&Jt4R71RVYItX-7|0FbzWH-=IZhJ_J-WB6GGO-s7baiB^b*kV|m9p#_J`)nfSK#XlI)=#x3zLxt6)j zCN>v7VEdF!ovvBy z9f^u+gZszWBgLov?Pz`ebCx_kv*K3Nxb;J54P;Y)*RGJZe^L3z*NWB_%96;-=A7B( zM+Am=Ii;o>sgz#^W_y^HrtnS8&v$%%88!=-6^E3^CQ#v$mAQb8%zsjV)Oq%$XSn?- znTEUe?{8hZ_SvY;Ket;szm%l)L9?KO}#PZ(mi@aR?>z?@!%IYQ!juWGCmH z^Dpbq;IjAw6MpVjIjEZZ0F3@98|aoEHo-8bbg|OMMVT&}R?yB4T|A~deA@oF!U5_# z-}Y`~Jb1A>z;6zY8fwsCujUm+9G z+kZ9iK&WnsfB*EhC{3HyTD@vfuw;#*N$xUyxY^|u9lQ1H*$~fy`*!RpP^Ak1N>?k) zqsU176iK-2>*=s}!qKJZ-{CSO>*o1Ad3C|-rnppI9WP#z;OaD=C}@YX zU5{7L+@?(UVZjho(L*(>1q*Pzy}*2%tFluOS%#N{+Ln} zb5W%+e?@}uUgwz|leDvjFj8h;v_1gSf4MlBiPl&#h}Nq%^by^3_cl>#s7giZ0L?C7 z^*T7ZdN$AA_KVLOzn-Ut(asQDB3vBD{|*UB{!)1Y=^`u-xJIeiN%EQK3x(m(yut0A z2Kv#_fQ}4DDg8b%h_#TiM!T!`=%WUMnovg++V_s(AI*8bg3ooT&$fLODl(mo?$2hd zT{{p=X{%TP$mRhw>~8TuC0XAJotutW0)jDW8#jOci=F0e+P81+84}}_Si(1V?`Rvr zlw|G6)P}Bh+H%+ceR7*6mVq7x!zY}Pn&;3lW&f6E6DfxZdl0cUW)lSa=ZA|;qq;Z$ z)PFs*dvoM`o2o7ID9y={DLuctOtyQcJBlsnIVUR7>SX&<#z81=g0!@C~@@uAI{OrLGd3A7*j~#X{fMw zt)=jJ9hRND<=+Z7)RP!3xO3a_F}@<;k6c%fu%4Qk-6~f~Jb2Z{;F`W-sI2^O<`+Iu z%DtDyiT^uPHf%a|{>ZgTOrmD7{Z;n({s~%ic0;7qzKnjBV@z_pKN)KszIXfE=bXqd z#d*zCM}IqO(7AQVrmFq-2E$5*$rd-fs(eaH?SrxUqlOc2`gBU|4_A2W|KDKjkt0VN zmbGOhm%F~QJlPeq%z^pBCUC7PbM2y|GlNCta$Y7;1}1&}7HtB(T8!&N!G6Bnl3~K2 zJV#9-tBs97SgOpK;Z5>pDGHN(BM!^DZBtNUGK?Pco@s5}OAd5d`%cxUE@RH~(26LFxba<6u=Tp1SH=!pY!>J1So3E7_pm^;hU|9l*Wa6BCR{Itn9F8*c{7jICB$s?;Lsn zLt?sAg~10K2hnP<$q!KIe_$5e#PMp*XGa%boIZfcG&D6Aj}7$6e17XzZQYykodxj)YUs&Ig?Vt}jUg*-1f}FrHj#7oCjPVwI=IJfd!!;f{Qp zZKVj-A@HL5x)LhUu^2)L7)w!?Pb*F&i#bZoJT2Z;cvld|pyEkr%%$Z8!u-Jj!Y~3) z1D2P6{-&+ns&M#^HLOgOa1Mk5^z~0V3Lf@rRbS1jtezAen6)QyF&5n$s!t-Pj00p= zwZ*VXwmrU9N??j&G3IzrH^k5u4caTba{J^~GTVx-I_QeQ&V+PPUeay@FRd#qJul|e ziHYL&wyGw_AyquA^7A(*E>;zj472rO$@PUpyMv=39o4K`4E16#U^5sj}v=tK+R0|f`{nFC) z!vyvOdAPf%+tYjh0juWcW{WSnaOmyx<4#fu<5{1{{@Y@Cb@Qe`sPzihj^mRetklJN zHbC^{01gUgzdAlsSFY?YE$TNK;`@I1G-R}hK zcR$p8FC==9bBnRNpA*Ue=Xly$~g7T0|(CDG zcdpnob{k^u>O+Rt>z`&5$GsN^3|1zW|D)Q*l$WL(1_slrqVlyL6yn3xD6>#yrEO;S z?e0VWSuuJ~(B7I78n{$Jv7F#!3g6$Q!E&c&Gh9}(=g*crA6OE7`}KP{Cz53j{WP?%;{6^2VP&|ZYMxf|7e8Zc zuDY#BhxK7~uZEvKbVv!rfO59-5r8l=l;Dr79ffo%-N}Ia;8~r!bh%lm6}`VV|C@3I zPb5M?ltrhRtsu&t*xUD@1&`bwbD$CCvNOZZ-&yK)^vpVfP#i40Z*djQgX?jk6`FMbo_Oo{H4B0N$S zq2!cIxo~qme;lbcfXA%)8pzrn+WiQH|1QwYxn-%{AZ2_gnnj^(4EikYF05GRU2(x( zh|4;+QyZf(2q$qaLhGS;?G*t_J^YsOP_a|EU%VsN(EV*1Y*2|EL~etGlQPDGov~yX z#~K4D*581xwNRbYVyq8Y9&`eKx*|=-`0~K2{sOnochzG!-2b%_tJ90ppM3N*tEWv4zDUEP&xFAhs zGVi>w>Lew?8G#U7DkZ#Hmbcv#2@_Wg(qzm3=+Wo@Ys>#$O;u&j6yz$^m}s?e!~H7L z(*p*q*~k@xotI8q&$kS+d5_8(!;7f?h&lz=7Z7m$lad9je1#y&i^8(v4aI8H+Ei!cweK;?dslfaychT9p2Pvl zN|uhjmbTx2?W^1T9K+vK)YW$h^#!131NxCOe*)Y?xdc<2dHI^{NER2YxgfDtcDB6t z5r=L`&Vyr!)%YoKxj-{hRaRAL)~HF7!TCI4<0nrZ3;htd#FyI>HNkdrp$A$C=+48z zEYp2K|7AGlv_Yw}7ByhFIX9w!rNjsZ>@Cy9MVLGa4326j(GEz?#nUOr`-^Enb#`<> z1g0rLR>u)|Ax_?a6{khv?JSQlc9G3I|C(^r&eiP!Leh0RskPY_QD+qNJ~MF;&e4+0&cfDr&F-Zy0X zu3hm0?q>s~$oyI5GEcH@<(SD#=eBIAp4+x&zm<>3>E(Q=O3k! zE28`SO-LytZy>*8eR#XD1Nb#w?O&}skGAg&U~Q7zQuA{(kTae(ED70uu(!Q5(< z7HON+1YSNW?2OdMkKN@i`<7&}WXVftBU{DEyp2lI7Pj^x91w5Pj4XfLm1{zLV-#jN zQH`EFAZSg|Gu|^V*_k<-Q}u5s2UFW--pYA9a5Sj?&>0OLBwm}hUq6|p#D#K$(mMD3 ztQA^dCs(gKP{iIn87`+R=KZQ)kJ+;q&xif|H2i9#G3cSX-AEVwj*L2NabJ4>c0DZG zZdDz4=>p=BQ+*Cp*<@;}?Yub}&PPkv9LtpSj zzZDO&97B@YZ@hhbJlq8+o|PubK;_hEQ~n$e&o)v#T5>c@hLg`&SrJ>MkgFXlnS+#6 zq;=3a!)JmULZs%O*=NG7T;MvDOhiKWGbgd4JW^ZLH^H5Mxm`3&Sjh!`l4G}GUp4_ju0h`5h z<>`)+`J_P9@ze%%Fxs0CY0SJp=ZINBi?rM%|82B5dp&iAZs454Ij~JJ$Spz<=7~7@ zta9kC@TY~c*<;2+T#w?Bh0KAlHGb1z+XF5Pw0q?i{wO_t8vq*VE>#g`w)kb6n8^C< zv2|*YnhEh%;#>qX_`vo_ML)d#LqqQ^aeOsK*yy70f>v5pKD;WVr2PgMJ2$sA&3aq3 zY3M9EwWULeKNXv?Kuy2Q`ww03^1nU2o2`>H2eD53VFFGF?T|vfX^;se4heXV{t?^Z z+=!>8EqC00{j5gTuIRfva_=hf7B+wnC$0$7&CImhk`qx;5K;frHnIMl4J+r+A@~tF zHq{ii9~%Yu=*+H>Upq2eaTU`?j)CO^`C(lzKl7yOY?C)D16R|2+BgsEmU0CT#%r*L z$D|k6904f8itA>+n)}tgTs9=YQ{N2J>rEL)4W|Ti@W8X4HGy;&rq{0@5IIpen(K|9 zk8@JzE3I}sfG2ams&4Jt_x{VmQAtrpxh-@k-+c9rxw$tUg_-R_yn3ayeH}S=Ne|1F zvNs)z)!)d8+oSFH{1;BR&YFQc`v0tvz%~B66Bo~J(awFq)&*iTloSQ8W|>yqj*&lm zN96Q=&K-{EIfbZS^KRYZ(XnH~rkAUJ=|Oe{EqIp2w(F?V5Sij~e5MGGk7TkT`B z_Y-|}baL85T8sQXu+(Dai5X04_t6AS+?;{X)8oPTDFRbYIQrmdzsq55J9Joe;KIa^ z)azMQ&%50K0hisIq+<-Rf@y(uN(~04te&T?i12Gw{=Vb;P#m#VgGw3$pf6F$Ga8bA z%Ox09)UTf85yJAue!}A&)M47^h4zh`@)fz^t|c?dJB|`@DD$UJmVrTBgt+s4mtZv` zlr1c^Ybty1(WXTfRl{5RW5~RCskMg*e_LoSpoNQ8tXT1671dCB1Nc}biIZ%R@9XI5 z;<*3mulNNJKE!+%1E7Ic5_LtBU_S@21CmR|&WWn2OLgiuYSdi!itVyxq-h7bomkp! zjG>`>9LR}xPN$E06dH-M6-5?134tzy%N}Hjc8)qi9`bEAI_Io2&ii(37xcaqrxk=o z$_sRR=tNdCjOKWaWISDf@1g7lh6$Jq?+EDmcp$c>sHEL0zGaCVj*lPJT?Q2lK$ME8a9zx}I%DDUMs1}v`D=PAJ^w_mZ!GAUQH2LvS zuTh3N7QufsSnRgL-=HBB24}TPPX=wb^Z(!-runf+%?(dFHtg8gc>c|+jjW$vY}`tF zPP5bY#3mzU+_lUvVE=yixmQ-dMA0R27Xr8ygJ3kKX0Sewpw&TiJ}o>LdMHEYe8!Il zbxP**E%uxlIzlEf2^6PrI^N6rWG>u(Cd|!b65LFdqnNLl_&X^@ z&6UKAjPN;+-^-hDIY@7#a~dMtHF$H#gz;ouKb;m!*alm--B!CK>@)Guh1+JBn9u># z97{U#ylUwACoNgGe*Iu^mB5L5{BAFym5e5DN!EFTta$9%JNI!IeDLm9$mj^#H{G>J zEduR_8U&5@@rFMZ^3X~UxT9mm#B?Ki9;|eZ4T}Mu)E@j9%a3ZS(FBvki4Txm(iEu{ z9%<(}mPu5|plq|<#0sST$YujGx}`KyZqj5d|2pmB54Y<{Pj}jLGGf1fZU^N{;vndu zFSlQG;Oi=?LPytUUX{d6%c!OsXj|&SUA(!|H@mq9I6UG`yFg=QJ%lR3c1YM*+Wt1F zwf8-8@@PPCV&#WW%;sfH!DCn~mgmfQ)~KuI`T=6j5363`w6tq@<^Fi(#C-8f6OihJ zHbgl1<2hMF_q2kV!ZwfPkO0bs>66CMk%pw=<>M2-(w%IGjKb9f30(ix_O>V@$iudp z`B@!Y5hwlTn?c4up+^l5&=7i>4t(XQ9UBr`f18_g&YMYgt*B^D9!m|bxy|q8_4{_O zFnhM-AIGF7(Nq5S@u1jeu+o1;P35}`T1%`uu?F9Rq&Owuw|{DHw~1*Vcp-b#)B1r; zOu8D6(J6~tUeje+hmoenEoYvN3_e+~-TnYzAm6^b81Y~`t$NJK9#M$g_P|BqKe0z1 z6k5$gX#OYj&+rC(iDhV8xO8cP2?3AQjQ7)wZ0U43cED56xdvOwq|YQbWlGB+HnuE` zx3?7t=%<@?#{Mr|naV)fHn98?0=5eK7#hg?59`i`3?byurydsO$vlv-o+^-d?3n43e$j)np9EXMiI8Fv#C?Uv2{b`h?I5Zu5| zEl$VAszdCZec2(;(b@KduAbiIZ^KTLA1wR2t{<(zIArDh+h^8vyZ7zuHmye%MMvT> zjvebZZ5UfBC1x;O zLKh&4Kvq27zUQ!E9cT)KVg3j^@ZDG|43Eu?v=@pU*ds=G2Uo8&p)&w5lxSwe#`gOT z&H)&a{|pAdE^n~tEodsl8ttor57n4i8~)4?AuUt9zc@1S?Ky+Wss9)g zanZ`NLlaqSM%j+kxrw^zEL%KA0}x^{LSmu1BpNJAfDo{<+yCYPQ3k7glNo=4Ki8}|u+8O21TqN0 zwkW)JghrNm`NSHSHO#zRa>XQ0rJR=*ZjS#*ou_c=#-z`ynH*MY5Zd%9^O}yzJPn8| zLGcskq67w`92zz{rz@}RR5P<(h|@T7(JSY&;b03vQs2FG)?~dW_;Bt0^sGhA)nwZu zqM{B@{oy^Wh;ZxJ@&59q|DHnq{sdnSu()sRsO>@1RZrasq^iBLcXS&t$(VO#Lw3~Py=dCQ zlDNL`3`b_wcFAvj+wFUy-{@xRd_qRg?B1o7hVyczz!?E+9^vpHck;_**UIpCouL`L z&n(9cCmnXgABeKx^f|+V9*uez1?ns_0gK+#$4!!Gz^jFYq#OY;{@&v+5_i>b*9rI` z06ud3_;|)SajO8wV89885TleKs~fp~&o|2$QP`Ebmy(f@*FL{=y+qS$*)E)UDveG# zxU46PF;;=f)03i4B>B>aYfjK;%>E5nQrFC>N?~uu3PTv!2#c@L2gd`0(RrzMTd6LM z3*+eVnPiD~$#Mh)+Yh+`o6~?FlE)j#$pf@ABJj>wTO546C7`D8VgOE_Bcvo4X2wM7 z4cE$1{g2o{Yeq6A;(sH~%Z|2-c$qCvD-b`v&U zCq+`Eh#16) z59k5svB8#64(-hWFobS#OylD`jBPk|y$89av^#{Cx;pH5PQ2bQUa~EIBgf;U+Wi84w>@_}P4=@a?v46)DDEcXJ;GF-Q&7 zG6*rUdYWU)Lb3LL3Sc}#IG!IEHg-JP;purBfY_qHL5h`JJJ^1z3b;yoz_k*SoK+&P zW|F=1kf#4EAV5d`t{+!$a}r6#OUS%%-6}GJ4VvV-LQ@nG@3aZf$ku4pugUdOo%ujM z=)tN8Q;TCJVZVZz7t3D%ThdbOe}?WwyY@mn&CtE-sg>U#^-d}FQ?`YE`I@4nm((!s zRd>`2ZDef_^ijbo`}bhxbQ~J~6msd$k3FrXCoG<=rtTKBvx~&H_^qV&9?5 z;8QQ_`fJZBjq0imsj`-8CN3fpr_86<&4IbxmOFfkT<#DQbcKgZW#3xq4z3`E zP&}zX+v>yn)9}FBHz~f=74jI(=sG*cT1sfpIIhE@=V`aKiZBiO=D&Xeevpy{VtRUY zT;F|M8p#PL0Ci|ZFE4(p9vH_WJjHHPpFCGzW7E9gC!*i8F}PyNUIo)7Jqv)wqShI-_)2*9|O+b;YchU}|qv4z*}xtJlnvugu|E zhaq4+f1I}%JK-M(hctX99(5wz3kk7!5(eus9>;>S5{o$Ruc-`-Y&HQNKRj54R_F(d z`pfByuQ`Tt1asGNv{=*d`HKEJ@N5>m0WAEuumU1DS1cOF(~WNDk%O25dp#h6QV}%7 zV4_Ysufl`WR52|^WGpmmwsomzo`NYt|MUGG_u<2bDEA5Ww?rk#BJM$LAeC9OwAum- zc7i76*Egpn_({r>Cp}J{HVNMv?{?L#Q+=7cS~ubcg&VR}&Y#Xr6?j>`6K#XLS2SAz z1pDsyuVCIQ24f0Ue0a0y`!MZ*c4}OCPN*HoZ2Yc-e1*fuw?RtHiTr{FG<59boa-V1 z<|-6$lZGg4PC9ILhGq722J0I4RZ&^z1#?XFGB*`r3`uXD8#3DdKiHd%|7CA})D&u( zYvt9u5nk6yth~iu2-kzlh0(9+O2=E@=<~w&k%~?eb5F&K zgV^`|`(d*b>ni<8*?m2qD#$6@9@Or!yT!|0711rfJ?ghz`AVl%ni`P<+BWyRHfdWw zyRi0Ndf!W)yzMRm2Y5oNa#w0OMZ;uJ7&tnYwoq_2Ss{$(LKpHJ;~!jP#y>?Y19ap# zYyr=$E;J?8up8YJL|E}+U;T(ztQaF=SxC_H1j6IQQ1t_Dudw!p#wP@1x~o7UIQwNt-eo$jiYX|rURY6#@3Pu72;fh_!3TgB5S4Be zYfy@5bKQ%zH$aD4dn@K($dz>^-6U03vP=;)cKYo%BfqJi#!@XL5SG~`eMtuGn8XU} z%`aZSencjD_U&``)u~(e6)wfERyQ*`Ltk0-EqAyW;`J{y2w;d31oV$nGuJ&^eMG+v z>?wR{XvjL(PYsG}(75q!R6BI4Rg)Xv0l^neVMRB3P3yjew#FS0-|7by48DGF)r%3tC+kXcdz)G$-5SLm!aVM zxqz$JMl1TW2@R91A>_jxx-Tm1R4Qa`}M|6gL7dqQIpAoq52dtn&A|7~|e!-nR| z=QNv~r}bfRL7G}qwerDN?x`GS6qMs+k(4&X7gV4n=m5CEO9{Q)>Gs z1c0-csKsPV3U|6*P`3pg(p1%cZ!B%paZm2#F)+O9J+kTxrl$|-ENs)LZb&ZB^~FOv z?_ZY{wNn!9D~!nGd_A_>95~5)7${bt@EwJaLD?CS#?U;ZD6ED~Ed4jL-v>z3Axngc1KuL7T&rDqf>`gJ~GWqJZ`D-2FRUY|&DS69<_% z1*IFI%j59o=!nfbl;ENK#1eDEgw8Y=VUv=~iv; z);Z!lpZ8=oSn462oF?Kh2Yte1Id!VYp|Wvvm$K9i)(StTEWJgR z(ADK#;6Fku{!Cq@OYw_bE{PP1k3Eij$23G^{`ey;AZU^)^FgL#(Xf!HY0qx|o#qwP zn3DK*)jd7s+{@km(No><>Qbj~j|X)gtg4c-DPwQz8x@ZKohbKb%`6_G5&rqHa(bJW zmE)C5Z-y)9E7ZAtpLA-`r_6Zajs9zV#%ZiPwmp5M$@=>H;&$xb8c34TOUl-5n(=ok zlO|H4MZz!9)}`l|7N96uIxi8cYHltA3pxHBwb zOyVJ|Io(4AFXVRo&S-DVpsy>+3d(L|Oj3@1UJxW52)*ajw*AcY^H}SZW*plL*ZI61 zZ#U0touhW@xK6>T+>-_>(<&#TKW5FCR`Euxct|3vEUb4eTm1}gbDT>PE%~zCckODm zHz6ES^4+|s11$C!&qjJU#S63I>!)QQUqK8B`pW-0;!~X93RrRkhlM3C%ivT>YHE!W z2_1|lO_D+82FtdC`*7p|yI*oWGutC`tEQ1$#==JS@QBhnU_5@joaWtre(zp6eDa4I zK)?Qi1A5jpH>OV*lOGB~#?yOGE1gjK5sscSFc(Jn2ha@G{rB#cX_XDAc=nzKj;>P@ z7rLt?g&^n4SZmNZWI zFRy*gh*!$LfTC_WLCB{!%rN{ku5r_*NAQG}jekIZ)9cGLlmZpOGEa_ern2WQCR9)z zt}oTO!4Du&BpQH(c1e|UVfaaIFhO9r*mUgN+4}LnJN2A~f|85_taAs($=KJh8+JsjhEtd>a$r>F2>>)*ST3 zysYtm|6iji?k#73>sQ-sm^t1r9}SNE*6xkP*RdK|!Sx$#sk$6}HX?J=>F-Ts2xA|R zO`o(WD5xTAhgd?blF%+e7Er7z8<(wQkqO4)6M8rWtl8zu%X*^yvl&vp z`AP$2WidEG{#9q(>9DY-Fb~F1$F41Y-6ej%5dC&EqBeGxoeAJ?7Z1K5{NEZ`nwrTOhROi`AE1eUrx|vpAb_$8UOT zCZ1SkQeDc9!jac&(x8FqKX3MOmCF&^$p+Z2X!7_zX+*AoSeo>v(qOV%oLDhDVTh5p8#u}fE`!N; z@sITY3T-%kjG^saoFv78fK~%Y$Smp%4FR%J;{OA<9sn}@N&dUF{069Hm0V{4Q_Xour5n}?L^Wvg?os6!^tG-Xa zRP5wUzxcg|+Dv7&N99pR?^K5#uh(z4)OU2L`H#Wi zHPdx8G^X5c+C}ZVU4t#EE)%yO-Z^R0>-*Yp10ex6Z_;t-Ta~23yLawD9LKl%-EjHzkbUB-q$?z(yhn7@f?+)qYF!}Msy;!m6I(KR6CO5>NpkCVZuC9201!dm2O!9%7T;Of^&6mPyJ3AcDzhtW2Or zG`t8GVln^BOWrIm)8)&1b8)}^T=W`B(!Row8ZtwbrB-g>urh26)QyyempiHU(T|Oj zgN931H~NJorQ#8MGb)M&h*R0~HFBueqsf=imrp-3al9sC9_YQV06^{gXp^ZyAJpno z^uf22L^CT_w~dWU;bj)RjASA+Z0Mb0i>HWCi+Sd#d`VFxq!19IA_>SWTk71!(OG-Q zJQ}4N|NPSown8gwe@z$%hP2>;D2b2L5QC^2&+p-<9looGVY{=Ds@4n5m1^)F*j|Lq zO~%-sB0#srGUYdpS?1gP@x6?^*|jZZ51=!0Swai8Z=H;)qDsWojT@!Kes{z3bW>-} z?2|%sj^v7d8^_1t`^N=|@QnwBFKFdpi>kqJ0e~>P#?6@V@alVuGOSqWxzGFIsmf!K zyS$xYSP5CQA~>8m9Vtj^Q01)>!yH|@MMB{JdHne8g#xbeaoo64vGio4T5WP1pO<+d z12h(^OCfb;w!@{S;VAsLjE%1?)wx4dhH}dtOXgiXJheL`03%n!pR?BvIAFWJ6#N^f zqtg58fdg%1xr!|NfPFdZP>B`1XuoXI?1F($2Q|5Q>s!Dq36Ch+(?Vs#t!DS1j`;87 z6{}_7^AC$^CbqO&ca{n;&Jb3>{B=JViPZX}kJ3g$&s3 zTfSMZ*R|PYmVk^$H;BjX3%Ob>K5(@x9~Rzu9!cRj}~ zx3p|P_>4`3{tHW}&)1=cgc_YoTzms1i(O8F%4ExLRmeV%aD@8lwQfYIH(aseET;%F zyTibj{eZShPMAAk!KKG~nbXp5*s$L1k}I9paXU@ z&kAm}-MT~Z0LJboZIf*zMd$N8gJHuwxs-7w>;2_o)o}#y19eMyy5oURZ`b}fpLrIA zBD2rFG*$uPGKVuaROwskX0C-7hVS#WzOKRBPR#A>o{ zRT4&&}sgXeCcqi#i2EBts0`sF1P^G^L*td;O*zhi49&XVTLUN0yQE^CvB}hCZU*pQ^!chqG z_fIt}xkokP#h+ZkwJrgFV_jTgcF!^VwBB2DB-u8sY^A;bek=-I6Dy|rz>hXh7w>U* zADRqRZ0Fb`*@E~>Ga7L76!jS@=beto@rMro?>uOiVz?`0d*J8EcNr7TxmLME#svK_ zTyJyuRy$+UL05jG3?=@iWNn<2tMNZNfM;!nm;QIG40GNrOmpe({zvxxYu_gA7YH(Q zq)O{XRj;0(jnEH1k=IR7YrwroTd$UoB`NONc+khwUbHV2G!aCBYQsUAGk$!1iAxsr z(fd<=?S?el6e|TySvuZvUoah@$nF{EbH^ogp6pj?6!-@!IVKity$nid6Ki|v9|%`m^<7WBR(OY6Uytkn8yj((xGQ;y z-@O1~7c0C#?QzxEmnK&A=7FvwMvvZHkkwHM9|KZh`gGr8i`T+!iO^!^F&Q>wNITx_ z&*kZcYq}bVp}hp*g)9M?5fBEq0ou7e02a9Ths|@L5ixF5-JQl_E(JuGG5z`sn(8YqxFd*2@k-UUNGE#+CL(zLYJjv0P7u&Fb-&s1~b_8Q#Tr z&lG2&9R$-j|AfN5XaD}U-i2xhlFZq+-pliQ1i@hH?AhJBw@CR$O%#w8*E+3JsHBA4 z3&pL~-kt9AmT*ZkXTdrN2tWL)=tP<&kBc~|acA_Agqh3)1+HH`Yz?k-#VRH9qRRgl zGF+!`58eL=KP{*4 z$H$%_*$15dceFa&uXF3uUCfP5kEVyoQruw7;i=cOPu1W4q4mS(F>VYrp=Po3975AE zwGrkLjwWW;%)Uj^qRwaME&Yq<#Me)gfp}UQ>wmy zwFVtNZ-?^vkmf?@Z-dkBnTIFq)Cdd=+=p>q1iyRFrhO}j95Aj%F1rP(Z}s~3xj8xY zCuAE4;hUF1A?D5Q%rqfy&N(O%*V!D z*i@m%W}cGC+E@3{r!jyxl6`Z~&nqM(BpbS?b*J~ofbaqSB)I)TFD4yWfo|NL!W zv_DIK7PvPTo*QV%3S;8FUMl;yzu8Qu$nLpdHn>7<+QFLBRZ@yPckWCO3x6dg2=nYK zgWG$sMGsp|QRvw>FmojKK5p1&)=OW9TEGvOq%xMb3VCrt!vq3sb^?Z+fva zP!p&2XO=B^7Nv&Tz{atue};Mdu7lZY2b-@GKNG_opH1JJKWnWdFS#ANCg%D_(esckuGl zG=i&MTfhp8Fd6_>KpiImOO#6w#kh&oZ20lY#&6ci9u=^9PW1Y;X#-j}62h&vw(lparBy*gc0h*~n!^V+p*%~KEmmJf=|8Y=uf*r{%MaH$@b6GZh>q$71P zFvwiehF4egnk+fT6*4J{K7Q%a<9AnH_0lv#Ou07=1+lE_MtThOMQ=xzh;>$DAx`Em zb(vI>%r@QMHs<=ZYa2oXVUJSqZzk`GOuBM}=3N>N7JjQuFOn<@;ibcbqrZ>nuX+Nb z93BzB@Zza_v(f}`2x=s{yXf-0;JORBtljD>&i7Zbay|!TcTiaw8aw$4+8B-XYr=yddhI!H#~w@!6_DS^9!dJvc~{rI{N&6h!eZY; zY9|H=G}nJ6&)K$OoVVun`F#^C=|nd@_-7P+EZMGwB5%v}N#Fc7J$4)GheS)NI>CUd zdlMYV$be};gI^0V2UDtRei3tWt%H+=%>n8AWzj)7K~sYE9ZZL>D&;<`>+ZXAVw^Ja zzPhk&7nVrAMotU55>^}mmG}SX(Zb-{htbRu4Ke5oPr7&S1x#_q08DeUdZw3RiopyE zxS{#&YXcRPsNoO1%NZKL*0xgGU#{Zjf%sG5Na0vhS&yPLZ z5`#!r&x~eHQbpoDL4DotJ+;vtX3}yVPXlgop&rf|6rv7H*)^R_Vilv zcw|T4Au3D&m~)G^-3`BLOveRcw>W}_PTc7M2{nyW?*|Us0WZZQ`qHJxxt}M^N#yTf zm^#|lHY^mU>m4vkTeGs;K_Y3FmBc=q2bfZ+zUQ$Zy@ zUnqXFM%;iDAuL>RREIJVkWWw_h6^1;xtUkyv3>ifAAZrt;csG5RGV{A(&1?cXdGIE zIuN3@=Sr*>j+0D2_6yF>k^mM+W9_rqdx)OZV2yQAPsU`;bLP$Eh8kuDU9Y#%Ug2+wHqxiKWHQPnY-E6 zdGgC8+(bVrS}YF@Ct~Qo(W77`cNU&|0T#*e-EUvcj5>AZ%({O~c>QXz?HPOPx*fR%`zc)T9>)>=Sz#ccsR(q@?rvaFub zgw3eZzI`u9kR()!zFdycI5A2P_QAS!QNZwBX?7rFJUV1fi_ODz(;~?EO{!02y?XV! zK-_!uErRfCCajv%qcUyPY^!4S%ye}fuBdz~J8v0Pg8|J-bvTGe-#u1IHma|z!j!|? zCD0uz10dq6G<~|gnPZ25iJ0bEx369%Ul}?MI<4@O%HGc%C$XB~+r&VuSr?eWK`n-v zo?yvIr^L0~n_$nw{R(%-^O87?u3Dq}o-(v)cu(Q|77_{$CE2KM@#Z#`^XDsN5iT)( z^IB#k5^ff-+giA3PhVOnF_ZcLR2H%X6@;uEh^Oq)sZB~4P^%c+I}{ZA^VqQstHVy& z_Te{6Hn!l2t97y&OGI6|nz|Mi! z!Km!P6U1?rizIbD4>&#P4SHD8MRuT~4H`PU_niX!S5U3pJw03O-P+PrmSUl)dGjjf zbSqknQwH7+CcU`;n9(<>uG0gQM4q5%9wp+9xd3S!F9+;Y@iRzB7y+B^X$QxuW3a=K z1pJl8^G`)roVm2LEqK!y{e&6aKmkXdaDTJw_QLCHvYfZE1TcBdupz;DlS9-vcN@&Y zhD!!B%*FE+%X5Qg^f-r^U-K3%Sp1|dFDv2^UjvZN5M$Oj;)Ir+np9-kR|M2F|NY0>|RsT81b}V zI7nza9nVitvY@0{E*AJTvwpv(*yDfz$TuRpkUS$_`;{vfH&&yBi=6HK8da9djvbs$ zvK?%X&O8O|p4WQHl#2W3N(zE>`}UnWRW;o|FmScym-g!F+TA?|qF37eca?GD*Q8kS zOm6hJ)eZ&|F|eT*+5If2SPDdENDX7e+F;hMiy1itMX^;eyNAZE!F7g^@li9M0*ik8 zw${J+sf5Kdv@Owv`-PTPvGaNToOcWe7N?kQr??@q;r6p!aXYm5Qx9l&3MUC&-L9uj zrCGKgv{Gobo4ogQhwX6_I9iyl`$We@nv0WxPhW|_qM^pY+nDUh5_FVpGM)et!*+b^&^3_=cAd)|IY_%fZg)R4|vTJg)1)3;(EUHt0yd1pVG z39{aPpWnl@JIx#v(amX=s~$pH-hJ?3@4hXPwh8>W_H%g;4OI)|geN{9i*FsPn>`N< zi>m;5M?pp@tp;=e?j^NUz!4fpn=4f>dy}qaeJ?Aj3@g!_%$rLhAhM2JHvR(R64gDy zk}~I187pps)f^8ZMO3)N z&b`?K$08yTp^_!c8B~R&*WQ`Q>MQL0bZQJ5- zdzLX|W8S3`FjKdLa_+|mM64RNDvEK`Ay~d894+oOC1)@?X#9ke` zi)>(rW8p=|JnlR2wOA|i=R5a3IGai)-1wAG8oIe-c@K3570;r|9W5)*IB$P3b)FIYh3~Ae-+Gh-;%Uyc&0nZf=o+{U! z*OTX0*2pAjIA)CU-6dEuAU`+hO>+ypN<|w0`X!67Nji~GS3I~;t!Lq^&Nb?jr z|5sr3Be{$B&;K-`sk*@}Vg3a6q&i4|pYz9I&wTj|-+09!Y}lC|o7$b`L!9N-8QL%U zXh5X^in&DW3ITZm`E-uL-FIDUX2TAa{h?QO#YAs6nU6S#t^*^(xQl>6bUQNcKUl>k z@+?jy*z7**GH)D3nzR0<^>CQ=K+?}bis?CB&_eH`Wt^`#%EqVl(b>fpD^5}|IO#ts z7;MsUdJV(K7=>uIB!U;EGt|(j8XvntCzSSvmLzl@n+z$BZDhtY6KMQhhb0dQ2}e?W~1ZL0$Douj(KA#1+h#^%RK$*TOUZ&GUy=nf9Ke|fgZB#iJnRMTl@ zW<|$E_rM9lM{wZ@bzy0*-!XeEw`V|(=Pv4vj45hQzdxVAVi5Z(7xC_~@ThzzWxHr5B z^j`bY_|#kUbwuKY2Be{XQr#En<|A0}-3%P&=$5lbQyB+zi7k3auTB_IAI)7M-u&kM zgEwW17sEBe(_eqW>+j-;IApbwl8govCUA#Mzo(oMCsBI%$))&B3+bF*rN!>;hU-2T zw1KvvYcU{)OcY?~pxhbb>l=9s(kUAZ3YWbiCVo+T6Kea_)Qi%A@Q+dA2V3&wif&w| zZPE~DTU*`!{Wo#c#9PW+Gbw5^VYcE!6cki6Eq*K4!B}>Q1x{vEIOu4-`nKr3mz^<5 zQiJ1QV$5mnr!QDXF(WmMCUD-{wKX)hWyFi~aM8o&kg%I@rX)M_-Me=pAf4UozS@^E z)Y&*l(>IL;AH{S1MQ;4A(qvx_Xo&M?#ZG!@XE+x!cQ2viAv^L`GvpZDOQ1O*{EQbTpLuH9{qhgD9< z^oUfQ==_vcj-$ZIrcOgZju^J-A%{)7nPL?e)m}jvsoO$P)Ps*85U=DDN>?DpG-Jk( zZ*pi~%mA^k1J4y)6d%IBQtH>P4b!F?@3~jJIkjs&8NyX8MWJ>ic^2oE8Plz@Ao_mF zMo*gb;D3%-2MJe(cWQBMbp5YiEHBDygEDf&h z-sblw{v=+}E%wkkNNnzpj{`Mlsq!}sy{xdCgk>#Z(VZXJ))jPt<*dL}r@c6l zm6(sbH?D8rd?sFylIKtt@+I$9Rc>P~Y4T(z!GCH@$Q%WX_J zEel-H=zmC|2Pv!GxV#(IYGOjetmgZe{S*7$j+&ZxURl$S0ugjLd!81J0itKZ%UBU* zAw!B4V}})8u5J;v7!HavCY+e`9JqN=N3#JS&laTCr-xf7ElcH`3GT0bsHg*1D=gh; zSUx<3Nd+`e;KE5+t#($?v{T?S-f8gMB*9@q8!Oi`Xcu!!?9!@4I z0=wq!T@IA8XeSD$C6EW7AWE-?6c=^lOe?q^OP9W>?zeU*=pX&%z4)#;&hm^!cTDK8 zKPQx>0j*H?vvj}YIc&+Xjmer|WF$s6LV`Fa@>^_O9(B%t(V>yPiv~A|U>^*Uy&{ej zc<>y_0M@zlu3}S6{aaQcj8)iT^g_Py`Glt7`%poE<|_bw$E-@M$!y=7=lm+sGiB?~BIpp4wE9XD zPSTGLm6NbII(BMOxwP%pRBgS+;u(2h{lP~kz81aJY0!1b80U5Qntdrv*z4T{BWt4- zl)4WL>;*nH(w9=j7-OYCaKq=FhKu$RPXrySHw`;KwH?82K`Ou<5Rx}SvU(}7dDpJ0 zZuiUS`(8rq7PFSNTD%I3Vp;r=T$#<98}TN!Nwr*Es*PO0pv#v5ua%)qE|mj85b29HL68d0#&)?1sn-%0`%_QjiZn&b)sf}}oL zvo$?zy$T);L4VsqikG9j-(k*e&?vSYM)^~f*rtj%#HEr`D)dS>RP@TkO3HsR5ab3Z zmcllw7K?c(PYrlH{b0Oj{ZJZe?p_^5aziX zxUoXk$}#CoD*$XGYlvaPw(M;j9{>wYj(Q(3&+T&pMw^X zL0|~`_-8eFdi0TkSs@zt)gCnH-NBrH9ny(PJ-wn0+RlJXJv8QAl)PUz=%(Ra?~?NM z1uC)Z-(|Oehn;+N%vr-tKXt;+FVJG<%vJ%h8_%$>zk7#!ibb^Qp`8X~pjck32&1d* zGhZ3#XhTKWz^bFiSUATfy{vB)&FXs#3D>H+U(*1#@xcDB$xyl^Jgtjri3WSDKNxt{*6ygea6oi*jd)p5l4p|u0DA32@4i8?h%->qu+xfs?{>XfmrFV}n?2sfhlpw)XpKq4>e0@fJ6B`kC;VQT zw(3c7D{(4CgcLX`O}t9!2qClA&@AlE-A83NAa34Q{I$R5xz6ibe9+akCQM=oc?#;) z56IEdaRq1^L5Mq9dygh$AUVgq>wnSF_cqrA&&O=DsawoiI-tc10h7Dz25bDUrcPzg z@WZr!?jK@P+;!7(*OB$~&ot}~REeWwni=-EUYjkma`#H$1{E%_IG>6&pDC#4$_G=S zb3f{0jYzpxty-&R{5f#|69)QC|5M&V_jotzn0hcPD;jpAFKq{}n}-tgf{H;vf96K+ zpA%tWY0E_0D`TF~v}}bD8u-|3DQY4$oEF(ktEJQ)mGeh0jZ~TF;CE_4oAIv$vp;c)n-htf(=HpHBrgEU%k851s&yCM?*j_OFPo6F+ z?r`Af5zbu!A%J{bYi#yRV00c4MZc)VQS3F;^;6%UjH?mgV{C>e4spdVl+(C4n842! zPcIveQ7h=dgf9t)=T1%OW2Y#%q<)$YdyBp5VPrB z(xdD$-yS=h%2q&rAk>PtGBYzrcHQl;G;)l~4lEp)|D(T8X$vmhzb6tViO@+lll&b1 znrx=pwrzdC$pd?({h}bQ834&nHj01-E3`e53O;;TA;ye1ZsZNsYp(Pr+I00;y+8NP zwqg^mUZ8=Vtl{hY&6d_bohL#3-lD>iE$#PLhL-SL83 zhULo7g8m4-lsCBcwx8hQK1Q0s?|300g>1-=C{Dj%EDl&=(1ReQEt8Rx7u!6AM#|8|jaDOZyBY+vM#2*>X!}Md;x71)7m|jpD$#E{ouj=a~D99!A|hZ zA3mw?8fgj`_l2Ul3^(y%l7zY9$@9OXFJz0mf_SC5jXyQem0n+<5cq1b+q}0=Ih%}> zqX)voHo?3?5ZJ$_ml*85y^T)u1h+q<3uE5oGjyOkq!aGn|IzK*&F8RllZ8PGJ+N&$ zJ6%i9{Bg2EBhvW$3QfrO$CT@C}XsAZC(_6U$S=ITA#Q_Fp z57=t)mIxrCrL|T+$a*h~>v}3G_kQfGd85`|YaqFDUFz?i0&RhFJ43GD1vjt8kakt2 zc7d?cM_0FE+S~y8UaU{pR`1(a4+h6@zk>6%vvb!Ch7{T{?1Jb(cm=}08tT>mO=F#XL(svK zJ6^Us-?4X^Mxq79FXrzqhjmiERG(J6%LC=TMun4s??jNPTUllaETi)d`Zh%yzwqGe zR+d=A{Ln({m@%x#v4LfLcT}J`?MQ7ZTi~Cs#RUqn4_1X&=dBuZV$wY7@Yj^63VUG0 z(dG1&u$`(iJ#_8u^`0o2<0nqs=#xk44gwiz0#B_uk4z=?DgcO}_~oqzdp8aY3rf}V zAAVvBe5d36FMKIY8LEWnam4D2rFm)Hckqm+{&P0@n^llM6nX!sz3}DrmqA$UQj2vP zdZ~YXi{2!59Aa0|27IG#WI6D{O|#%ZcL2yl>FBWgVfsjb20-ma!IH3*c4fjO^?WgL z#0qm-(fYbdAQhvQEjx33%ICgfr-hJ!cXII;Ty^h2!xqeR!^Yy_8U0Qlm*4V* z;#t_j+hi5pvcz>gb*^wk&LR^SJ1SST=~w{#NyqHw?!M*`M@_<}uXf))2F#Y%Tx5

    +R$B~DiNyknLe@6!EF%@AM>y&?VwXnJ5h_Rq7Ut%?0K|p)+q-GR z+0=lEfHK=$8YY@^m}K`H3>@paV%CXC?Ez~U5oF1XFoC69EyFdD(E%(qx%X%gZwSBm z@ZsB*YXgL`kESKp%64D)2!+55yXfTTKl+DIOT^bf_-yoKO%E-d54{4??_f0=JM;3P zbS}rq*u{W|;_n@5TGp3^IA8K`vM{_vy|6oGx?9}qfdL0mGc81`yKS;+oHI*;O(m~f z(XSjx_Vw}kn4?#jx;T24LF9jn`byIv_IMs~G>+QZ&5-{QFq&2V4ztwz{|i1YPPtp@ z09S8m@XS<*yB}w2a8{~5O2UCA6sY5;m-b14JaUo(2zWCrFeIgaZ^y!f*ZJw<2U)VS zs#z=v6{h}=50%&dXI18LQK{(_4g2RN&Gj^3T^g9{pSW(HXQ))C$!3I`_S zj`pNpFs8vyd?^m)Vwmiq{0dNsag6@Ckz0H)bq4>?E~R11mMw=RXzPA1PP5wLSc2V9fKMP!gNJJr9j_$YbkXU^J3@m>2|&wm zukczJms6?^NOFpE#Dh0)x*XHB?e?Agp^g^p4}4ks5JMxVq1)2U^ts2OnN>B8ztgOw zYO(t2*d@j`zjBQh>}D>fyJpl55w(Li+JiS%%|VO5a8)d%lUE!nC{v0<)w=b;WI{UR zP_@LGRC>pD2#9#N!3*e=e&>Tw8w;=0XA8>d zVQI!Glh6LGcuf=1w116O;al;r75OZfLwsmB))M?fkz2y<%(iYf?U(eL6Q#~UOYE-b z7!wktp#Au;?wfEm>^GRiVa_sZtM*SvnSNZfVw9BtCI!8lUz>Zq!SP91krT}DnULBo zxA5Yb5}6g-h-J+`9*%|tOHtXJ8m#SR(0bZHkM=z+6nvb0HRw{=3^;48kwsG%QWF0?>z9383GUei;mX#=?7)Ik4|q<6pHB;S1d1 zW{3U@Hm&pwvQv3>PQhy=j2NaB`)!_^STh!HAR&yvTj1&36Z-I~v<;&%EkVsJ8%N*A z*v9iscj(k$WBsKBia~S!&;{TdUUfjrkF~US*@L%^J5t;L{FfJB~pqa#7w1(jdsR=!%BZ_R|jWq`%+yD*AkSyYH?8Lvw<02i}1>n`9{q>@k0S2d3c~ z8tWzoIS)*F_)xji3HwJ6AF8)ze<-}kZ4dW%fr=n1Bgq~{#MQA&V!I_lsc?IVHx$}G zuQp_rsN`a4G8oY?A>b3yLh~-?mw=N9`R>EBmW&GGx-$Nc4h1Hd(A9-ByI2cqGe#!E z&P-v6_Bl_|PRk0$`4@hm$&+9s303IQ^1yl!DE$#5{@}{-%sc3Yu|jo0ZMt-MZn@uK zOm8kgh;vM7seGTc*pJ+k94W@Ks`;#~Ges>R%zMkvGqyBRl0A9sm#wO!MB>K{2pN)d zP%XB?ZB$Z|o@=AN*(eT~Ni+MJ^Q`;pc~hf)d}P1X+4wuPu{oQ!Cx@Huu?u#ZmtL^0 z@ncNIRLiyB4m+VaX0?l}z;*mwT* zwcmd0h`)gNMMnPG-TsRhHryNGP<^yqG|dcRtucRVEig#nJmpTlsZ4Hi*+InfUrdAb zm7`^AIyXHWw1?DkOiLF8TbOWHrbu{u3e(i@n8t)|0RHSJ>R@O20LS!t-SA4V(#;9>wd; z_Zr!WY|TGGkXeA4(lqD2dv9Ueum=I2%wXFoe(a^0u(uQy6{TF@CtU!tKc8HY0QH}Eb>-@2C+h3>D!Qu7ji{6dGl5TxmS=vP- zaX!Bg3tQp;df$@pSJS^kVptvaE>a#GE_y(bUh_m;hBS!-D)k5z>h=5uB;d5kpYlL^~ z)_gBtew6FvEwDaV0mX9kc`6u>VHN@uo1t(Rx8>#Kxn7X4BTmCfKfh3v4k%;6T1P52 zfVlPWKO&FV9A$5mYMsvD$A%&=-{vCkk^fVAJJicHPIgdNk_pB!$L83cCWvoX)5ty% z0<|u5D$n>`N>RuD3Wbisj#>n&%!QQMT4TB&yD@LS;~&+lH-sg?+UyW~97U^8wKfyh zrcsUuCdRv&<@#)$!#e~V0-tKI8Yr0gm!V(cB}RmSvSmp+1ef=@W4P|}Yntfm=WUGYb0Eya&Fwl5 z)oPT|c=txG$>7j2o+530u(Eg;xczPoLkxMb&rx^|fN6Gbb}xJd-m2~S!U~w<@h4VE zuS1o73{vVo*1bWF+u4<=GIko>6TSlC1u+_XloOoSVEl{8#dQ<=$U*s|tI`J}l@a^f z1&sc6HdU?v_+KT=(u$KizAdyJ-ur@{U6^66?Aw%{-wbL#saunny%qs?!lS;65-^(l zgr;FI3lG3?#DR1|b5sx`+AaF5eh-v5(Q=UAjbvF533ld`q zZJ9aGXTWq&I0#_k?rR1EP~5fhtuv;FuX(j6>HTd0(1^`Z^CG$IJO{5gnkF5dgxc%r zJ#!lB4I5X?P5o1@;v6EY3#0sEjd;koBJxG$F|bD(#*;8OV;0V7C+IQ)>G`WyKUg*c zzpv6X)!lyM z`Q>9o5cTgh7iy!(EP#uv?Y{k1jC6oDt(}tUW}wm2c0zdwYwCD^jk}y&c&BX_a>)_` zQfqwn3aTHYS(PUNs#sg}oshbS!l)9Rk)#(eFkGB0=}bk+Xcm>R{B?eQuih=K!$3Dk zu{9}jub~9XR2PgC-I>jLO!yU|zr23@Ij`5w^3faPqCrp4Nh8kr8;ie<7 zm7sQ@ycqu<$@*r*jIwIWjDlW*yKKXwgx7T_!vt<#=VfjiOdS?WI$S->S&Q?ZhrzM; zeVm$F+sml40E$!`w%!B9LFeVohN}2}EY70n=^H-V$9{of6E4RCyl?pOm#C$cFa6UUG2vE)+N`l%=I3^FlnQro2>zahNA$)i0g z>ah2CnGDF>v|Y9XLm~fzU#VtBIXn~jtgIO@1+j9py#l8DT|(W;@y^KFRJBZoYz7`- z&Qr!ED=Olb zP`nP98`k|Y+QMZ28z_@MKl(clc^_Z%x3gx`!*BRjm4~-JCOn=$erBgC#6GLb>iFJH z^Yl7dy{P7M>wZ_LNy0pgs`%0i($eU1EW)In4%0l()mKwR?LVP0j-NPj@H?^E=VAhr zRX$Z%k2S1t2!uNl3^GN>*-%lS?%V#xJcc@6cXau0@Zzc$EVZ(rIj-9mr?3*koe#|3 zi}xk<$*K^RisC^G#>nn?bEZ2a{L-Di`FB@n-@6mp_kgXPUDS#prTNd&pijo$p`qT% zfN*HkIi{h(TDujxme4ydYw6U z4tl_$WU0gG{`GfLh~~d;(N6qBhW00Hvl(5S33#UIc8Jq+|ZZw;go|JJ%$dQJ-)@v-U4lxoWY`95=7{GpAnTfVl^#U`rd^*c=r<@j`aE%Fo|xU@+%)Khu=o z(0reeM(+attGR6c*NV3aCsTv$tsf>mUliR%H-3-7@QYDzbSx$96-cq>P#bZ*IL(0s072a>I*achBQ_ma_AVCpCH9_m#(s{*pqNMW zB_S17k8Cz>`~>QDSuG= z_DVGHePrkX|Dey3I7|PXJK%lGT0MNT-NDU6FMK16r;nD_tIJ~4{yqN^_+vR;uVg#3 zN2>dqXxbDnu4_6OVFK5)WTw4-bDH+H17JNO?0JE%t>JKS!D0te_Rut!FZJZAMMq+i zz-LW79~8=JK=y8)DH5WIVx5wr+}XF}5H1q5%h6ZqKG|4p-dy&?-yh$J{f>5a6Sg~; zjC0ZOKNS?@?R%^}%Ow~uqW~~@)e@gP(a8yRrt1)_R080>dlUqO?KG497&nuG@-d>W z-7&>U}Q)Y$*O82{fR}vKj$B#HKP%rm;Fw&TC^zp z=8jG^t9r+nyuH=L+xdEC=Ft3&9rHh;0X!ZNFc<*gVJ9ew1jTbkOC~GZkDkXNf z;d(OI24;dmf>S;lHYY zeAws;Clev4=#_syc#iqO8P?){$CTGICQPUgYOqL)iKOFbCm?j=N!{Ef9S3~g((I*g zvY8Bv_8g#${U(BJtPs3-i{jSrfk&QX-~v z4V+hHb2CWru40wKj5241`RU)2CvRx;ZZTyNPt(O(bFW4ku7On+ zXn{xu&Q`NUv#K~)Rl^$MoI|shZ*D9ctceqMVR;3>Pfl(^`oZTtc}dv;z1uR2z6kou~K^g0}0W^A!|a*#IqD$Y8}2RCBdbKWGM$1 zV$K{j)U76m_B+J%qj<1yQ9%3i=3QSF^~(7trO|$r@7AhBZh8RcPrvkH=}+*<5;Z;S zS^vefbz;YcKhTf@ll%vl>qig1Gw*1r?@YzHnaU~p{832(nfy3jORIf6SowPaTme4v z&Ho%GBi}_e4=%U;dcjuNE`Im$(@~SHJO>KBi02L?nAHjxM5v8KG^|S5zk)7NcB1F{ z#6xkAGz2IJEYWUZ?a8ix(S%se7*_Dkv_D`YY3zG}A6IAHsXuz9jZLc`w8r524S3?1 zNnW*Zc61c^B6S=T--uLDSoLkPxM{a8Gm%9YYQ#RWN0W0lpuh3iWQhF4A8h$Mt5mAo zGL!t`gq2F%p1Pm%bzM^@WMWr{hg2s{m@#fRWbtuoo}Y#>ESu_ofT-JjJ0=&2+2n92 zHQEg}vU{yT)xP*ocFxpHa3C>68T-l$DzXKfS_O5VRW{E{^TvA?f$!7dFN$s)mH5_5 zoCULk@<&9zWN#?*9gT_S5&K=c7K)fN(q)`@Ww5%(=7}y6TCYS^5E|<2sk811?V6A_ zu#0h;3E7eV^{3zDve2jWZ$B-C&s$z-Od2TIAOwky?YA6!QmzW1vEl2-6ARPXSkW+H zn}>c=q6onHQkQ$&nD>0+*_wOz?K|W+deo>L+3nSHa=bC*At>Cfcj1_-)#5)vruAte zG4I{WV%Yx+u{HQCXfbYzz?uX8$fFtYNAvhtZMBq~< zEVT4P%+r8DzS|Wr<0h>=lmcW5kK~u*)vPxDrQHaK3hSk;^jjavG61f1tQnI4+tz}w zv2dv`;9KZBDT@&^74E567bHH@k2F0^$I&kpdcB&cE<;L5=-PL_c zu$=(aK+fG(g;$2nAnOt9RAPQf+wRXE@&m-ixP`wFeN}u@-b)RbhsXS~qX@hxn1ogL zYM&bBaq-Eg@+9N&Gdfin4~*zhN9Fp?9VKBku?qkBkzRLjz2B5))_s`SGTzSrTJWjM zpZ+VwfLQfF%~nBU<$QTB8a@2{nkd8d_{y_E*bWl(xu|H+JBXoQX#o{@cCtp^kHM|7 zV!dK;gYk&9MZuHU9ws4Zp2bug~V5w(=#D!M3E(5MXJx848H9H97Gk zN7fMk+yKuhvwRaJ=@sbMX(nBzVg_8}l^W2QesaG??D;v3n-d;-#L=#D=KJ$hGleh5 z)%bjmb$`Iu+>sO&5kl|+dneaqwsAH5RF}i(z#8XBvUtZO<}VZa9<_|4-X<6e9Owht z=j|@A$^aPrd)h2kCLjZ^JDm*)gZ{+=XK^qfpMs{Sp8xszjX%nWvoZQYr*rYsMGwp` zL@G{s&bhpTdF)BV#sRGx+4NCZ|0r^m&vfXdBr%o(pow$9$&PdTLd=73UV|_xZ_CiJ zem8kC#rs`ZS^26?#!X;f_GGt;Gwt$EqeVRdCj^n<%{7!;NWNrhtCrdBS0nJbGo@^qj@X@wHv6Y z6u+_D6$FASv2N^jhs~PRntKswB=_<$FapR2f_QNQoH%*%zm5TiA&(673K%C>%+6*z zSn-*exg^u9HJ}T_1l0ivQ7ZGj@71ecy&~}()A;Fk;uhw&bH9>mQTDdWs3o3y35Jxb&zcL8yKUO9_U=pJ{a+A^YDwSCV~M!@@ey;c z#=nPj+*FxMy>S&BiXQK4WyyQ64?OY|Hv$O zMOmQf+h>bxY~YHN5B@kR$YVdkt-51h+Wfiw#@oyxg9p!=)auga%5M$IzJ+*W%By*Q z2vrSN@y3{(yf|GS*AuQR0PfqZ?b;c-$R>Hl!XdPrJYosU=a{c$bB+e=t{Gs}U&XEl zHfW3yalV>Dff~$iP+7xAVJ?ClF3Ls(X<(eR$af5Phd+Un2`P3vfDR3)ayg=6dO1xGEts0u4Gt~C`6oKa3Iy62yeBBewi6+{s|;}+n(!D^kHqAWkh z7wmc%$}VDaubt}Ind@IdSZB}n@_o#LjU|0hpNq{6Cr)(r%HND849fXE@0|X6fZK}~ zVquEnXi8@v`vHCXs<(!}>`*)KI{a383j7Ql5%!2bPDDh;vZ1^tyae!d#PAQui-J_> zmNiH)NLDNzq;+mJvlf7E3vrJfQtqi2;Uy%Yh|Px&9SZ*voB&|>)A4G~I=lZo2raO$ z!ue4xaQi&_w(-HuvjG94e*lrJMIbh&akVJS9Mhk06_p$tsdm=SuZF-)8nJpy4zcm$ zWEHm2NX$nDOe&Ew*OH~$`h+X8vu@T*pn^?3RSzV%A3eHN5l10gwSyn}>E(Y%AirxSNPw0|T|O=O;fOHcF@Z z)bP2#I!%WdkN?BauiLBJB>M>JS;gBHCsP89_^sI|H$L21cXk#Wk=I8i+@)n-{q^H~ z@f+SImJ+yUJFrJr;K+lPu(qPra6I?+&GYB&d46_bKNI>Y|A`kehuUI*f$rRQ8op$e z<(ZX>lO9iJcYIxh*&Ng?=xa{jc#fgHTr^AWw)JhZQAS1$n8w>mdkDVWkpTL6-BJLn zhSTjsd~G99h8P*v9v}SCbiip`K%ie?`@H5`%gif=-ks4=x;iQxt}G%HS~Rc-A;CR# z=%=M+FEGtYx!>4dA6i-=95}Ir+1w0>+% zQ}J?OUlijMYIhKj3?9+Pq4v!V!9o1|Mpax=IbH)(5|{QDk7j@&6VLcyo_UCEpYYk& zG2^}#>Kcd2#0B<+Oy^?ILqjyjJ&3MOTuvzFoo_Bb(}vB zZt4?^yCLbHF*=6jr`VWZ{HMEun99k!ZLeNn98}yXqTCvrX_m}8!IBteWJ>`AW|T8m z={Xx1`6FpDE}{YAHT*Bv!w(OwR2^t9n>TxE-tZvzD)tV#6$t@uymN;bemesFQ+VA| zHhXailj2r|ktAkh49!`dc@`LIgn!aNwa5**aY^Fz#7;ye{Ud316zioL>MdN3!@>gPIKL%I|qR*vEpZrqJZ*Gh-6st)N+_(RR z{%c?m^<7>4{lqa=zFyfH!Q_wlmj(@v{TBbCdSD~MdWyF=4QBXPdce+vrIF&BGZ__r zbMFbqm)|VO5!A${aUG?NYv()E$!@cccj3p!GpBy$KMyZ-V+V$}6-IS#^cvdhyCQq7 z`G!BA9`AZ?k7a`4Cf+a6u!u{>cQY$*+afFw9j{x(OVl|R*NKpj6k~osH74y#B)Bq! zd1Fz=j#?hl(L%*V({5mBxEtdF94e}#c)HVk?xt!Uaj#ELpEgY$W%K)18~sn7Y!8Mm zx*DlrqFy_-raqiEW-jt1?;F3*&oBP=I+;#Ubok<4fNrzBp^G?~65>Lz%mdcgXTeQs zkIg~{MF7x*DqiuZbNI5PDlZu8SK#PA#fGKsR`yYHRSM^Y2$RCrxgEtQ@rUM9Jl%($ z4dI=^-Rinv*^$drdWH->@^KOU>(`<*b1%(mCc$P+q0eY&Hi>0LrO3kh`t|BL?4<-X zdGNltv+Yq#nP3Q0gK`JDx*j@ooaFd!O*{Mf*e7LXI*U>!D9AGYmB?eYSKcJX0+YPi zt&R4x`Nx(o#wkAl4irb{Ye|pv=$n8iOi-&q_E;8+qbu?n=9*^R_XZ>pRycnAA+OP^ z%*E~-+jASzy24n1mrPGj0+5xb`Z2r!Ng|G(4>z2af56# z<_bXl{mBAB6OHG;wv*v-OvTc$kn_<|j^<81Qhzlx-|*&wd@dfCCwY z)s-k87HP$O3O2d+oPCovagi3^9g~0Qee1CWJxMaS1Ks<>l)1e6;u~E&vH%t+mXZiN zsxiiG%UEUi>Ki6O07S!$jN-S=3G1NV2!@|98-HgLbN-Er`&Jrr;Q*^6Itdh7Zn5!? zJ`c-#IBiWtV(!1O-qyJ}%_ZQ9$8u;zn>D`6TJ;!VTE!FDzS5|HzKvU9dGU`k#^1AT zzb-TVj$F#$+D$%fd`oBL$b0Seb_ieJ?*6M|%^m6&Z10&qRBy*sSZp#!UAy+wxSvOy z=sld&hVB6bvUp+2SgW()V|xiEyS1Vsp+d-k5RV&TN!8n{l>9!BHs5o7@&Tu5%Ibh) z!@{Z%WKF2a|C`4Y@$ZStOy=xO*+44PR!sSO>%JP!QWh-@4|nyXt1LofF#Gn-5vbhD zEXc0rGgJ5nvfuoUGyZQ3g$3=k!tWdYUp*;qjpCxHq#cpyz%g*57fAtv*ps5&_g{iM zKYqM;PLPO!9vfWJ@jz4@TJ($^^z`P!#A4Ro5fD}J^lkZH0|vM{=B58f3oLedJZ>@X zn2iMCN8r?@OUoXYDartbk`%pc2J%G2f-sOew{Kap2S{Ve)Tw}m7V&o$qq_ldG{9$X zmRF`Fj+Ju!g`4j5l#UZGgJ#)fq%MJRbYM)ttp4l6F9H_fBzS(>;QW>`GoVL)7_yXW z3D^P(l7`f-s!AkTW%Ce;S8tpc#-p=8cRJelI%Eunnd%iw1yfe-!kvdkKbl)b0zsv> ze_J+4iFP6jaw^5>4wTSK^cx{h6WrKf4l>7jx~FEn9uCl#W8+67%8|i!p8A?j45U-1 zb~+2?4Wu*X(k9Zce!`5Ng~1=DiUQT$y-t5_2eOlR2*d*&l^Ks{_?irUE#FRc$ok>ah8xVm z&=qo)Br7Sxl?GNU4@$*84om?wl4DhMxcCRWd$TKQ{i;3Ha0Y3Lt?e53xz&^;eglVn z*=UT>jv9ovGc?Cuv8~R&QLv!8af|mEmQKHZuWF)Q__cAr<>;zfH|1?z8GE8dpEQMc z&o17KAqR$U-BZ+|&Fl;MQ;v_dav#VfAl{NcQAQpsF+!PiCcX1|QotduJ4&jM8H*ZO zsu6;;jcaV{+jNib+Rn;yyGn0gZ4}Bf`6FU(+Wh_qB-IcH|HpHcaNYao-5W%n6>X=M z)<8^Ts&RgF6CVtL$EbZQBRsKn+^UNH*fk=dzW=4rdgCAGc$!mn*lh_l?%~z{O{vh~ zd%iB^r3%Ww31wQ)1%<1Lm@UwHA)5O#E^3vPRTE4l_a|fCi`_=pycrRbxFr{PeO0=UO0P zx-Qz{Cp|WsGk<|Y^_g~NN_x95eWRj|)Gf0~SA}^)UbP7f5F7~>1%v&+0CXe=hEGnK z?b-Y@fPjY%DSuaUcJ6GM`E;CI-T(eum3zh|LnX`wEwRW;A2?%z-(<xOCx;~ScCu|RqG*`9YKi@_#Udmd*0qx$%HGJX--c>TQYm{6Ga@^(JnWF_k*_>UN3|JBp2CfqZ!;?xK# zR~&jK6m}*(~+z;`(_J_6n2l>eY`=xt6)Ha1*24K}TZUXGH|`Iw?9%oxy|CmTWG{ z!w|ITQ6-*c;zUrF^dgNG75Awhb)Deo?Khk*g|JRB>5EsHKc5GM_M zq9rDH*-3YGcZan8gW8pZ(D7* z-Ir3!lS-WF?BtYdo8k}{au#Gw<}@yf(FF$Pfn@R>|IVF2ZjXLd7s$E}N~&joTec=m zE)Y=G)#9uHbb>ERtM?$QqXYcKHtWJ>PqSmFqP}&5U)8Rop+X`6`0Kxz_IWsZ+4mz< zYny`kaCgv0#N=st@BM8Bt*KdXt*NT@N!&m_J(=`Yxw!aVGx2(b2#5IHI)Jg zkNW?PXo_nG;t#mrRI%9(_3RdvHNwsc;OY%N^_>ZS>Q7|9`<-WVs z6lc@mnbf=vX|Bpbh^oEv+J_0;bYikePjUAhy{RO;cG z*DjchL7-sIV&rwXyRNR9ussxhW7&0uQ;jA_K@LnO=LXTHgjHmA-rN~mAaO`z#%L3a zdWIwMPNiMlb?euQKRy>errjMnR3pR)(>>LM;M&4xO3TU5&1J62-h9>--IG54@@(MR zXt7j+4Y>8p8|OWWJ=o~cx0O2-%(onxFWEC1788LRS!CX#f+pe(0$ow!W|j?wb-^2m zIn6@Qw|{WfVpf|SId|#zIm>mGce7~1>bfBFp?5UHXJDz^qIx?J`@cI^B>5R zHpr_1sw6gsqsNSi8E(*Nz#4S3D17djEq{DYUlIM09Vg6}4^R4VL&d&htP6K_9zte; za7&0eeQ|jOcDP(55hRqol#*5@d=7{R=?$B{O7=WL>`xjsVj~SmSw#Vsqf}X0(NF9D zqJ)vFRr>wv*m~4sJ_l#@wyQ{e+b>gz)vdgH0io@l^fdf>$+FG%B<=5OuaU}Sg95J3 z^lPFxR6h=m$-W@_-wsvD=}C5-8Y~|5{C$z?Pp;x0M;q=)D!z8n&VHk|l9)Fg=s5Z# zT;9gh&L$fp%~5Au7T(s@x7m=yS0kkr;h5`$LsF}uC1|8XzV6TpKFAoB-Ypp zAKShOh}^gQU>9qhih`04Rg8qaPQ5eAV&mgS&6?%0unYzU+Kd2mD~UhBr~x6xKr^A=M?rQOd9zA-OqM@RfQE%V=y0Lh00|wcipLu>1-N(H^n}4YhFW3(UK4lsHZs?Ot5yCORoP zm0jNL^vs4An*XMTz7pDb;_NFcS_%e4VQTJQSdW9Up&)zr!XW^g)KYj}k;%nxG25nY zkj9?ndZ95U{7hazbUX5Eu2RPkfK?JL=xUKdUnEwCz)W2@=Q!A0mxwpcgV%NPUE?;l z!wSw`!*J8kPHRq}rb^Z6-~UkbKufydsvQ&_$s?esuvt>VRlxOPM1+`z(p%gSR{<}t zT$dE_CV`hJEJfQJA>%HPAIMi-6pTwtVaYpt>JINVrX74D+x_y;@{eAP(WlV{M8BU_ z#gPi>ucPBYfsHZax_UycA)Q_vztQN}kgkJA)!agUK}FnwXn;7Lp5bg3k^5*mJ1otj*24Q)oD(ZIFY)oI|bKlZjhL996CLW%?ck z5HGbxKB})?r}=x+*X;bX7q{-O-jt?W2Tz?={I3Fom5=8=y3%rziXKBbj1SMeKQ{&V@5`XWoJrzjwG~x^~SrlV9U~RYf2yvdEcpcW-DXw|=z1KlK z(fjBUPWE*{-br7Iao;jkfdu|3+RCIjXI@JB$ti`BGDa>~pbi3|NF`+#Fdxhhmpz-@6cr*$uU)f=uf5T( zanq*uI3hBA+LT9L@n=Z~PRA*5RP3|BVm@Zx9YJZ zM1L6y!`TOFe~mP?&?7hIt+t7Los&~Hc6cGgMhXPJI(*`Ocyooe2%PKHGhl~RH5qby zCS;U$sUjZgMD)tVz>}tD+vEcS!nmuC4Y5`~i&{i{U!&h-8zVuWqru)X_wov+*nbF= zp;BECV^>^N#KDxc`EBe=p|c{&hMKuS&DkM+4-+a@k%+_iRM6@>a>y3v#67deUy<)Y z!P02e>h3~|U&mo4tm@f9uR(C&bp)D;ye@Ku?_co(8d{Z5t zCUh32^o22fXHsdPVPVTD*MQ;)BmEA!$Uz5V$(%+?FeFXC@ysh(4M` zb08cz#EOH>=-?Fjv9cP5bZByQhR}5)=JZ-V+zfRk3oJ(VTH{|oW2-3>gJ5RH9{7Pl z>`_>3AI*{L52EtwcaOS`p5UUv!L2Ufz0hV)ZCnqO23tmd`gB*{#}LXnVO!6bbi~M3 z?B(_;QvYUb5U~AHvYCCd+3W|e-PGe&W%~oRce1FD1nAME4pm1@tj$pnekW?=0Q@V2 zExZBBeimM|MgJ#}wg$|hDtsPuz1UCxeAa=cclTXr%>#t-vh-A(QI;Fq{SOAwj4B#sI$@tY=`#a;; zV)CH%=!tgG#zlVx>Xf=S(KD-{hGLxJ&wzB9GT~HiU9N{J)5SKylr3#-%{3=dC)C-{ z^JN_U)O?;|VT(~CEU-!#KVjei{T3`!0bq>L&b)jnTiKp5T6Z+|EEyX6{*(d7fhWGggSdB3kBUGV{DKM!9xYp`L(2|5j!N9_U+m29;+r}5#!RZMdbH8&KGj^lYR z*=^g$x)+pppL0KVqPokW_O--z`1S%;{#bgfEiWx7*;(Ad_1+0|j)_Z6D$V-0qL2;K zJQ#P^mJ{{kBci^4(r$K-&jDPLjwK1Bn{jDVo_pd6i<vI)q;v=V|iihooH{ zdd5tAC((jGJ~TRHez+zp>mMKed{hdfqpvkAGdpHILZKv`F|46c|KO~EZgxwx%+6$u zUk(8LCNED2!Dq1^b*9N}Db_oqwK6iUtZa=4!wH7K^z)dRLGz5kKA7?+)M=Kg2KC+Z zGs46IW0-H^7`OWyIW1rt;*om6onQj^u`&s%!w

    l@CYe5B9v4jvnsr&#ea8MxNOL zSm`);@Zb)$wj~&8=*=!17nae$@8N)6hiN!`KAH5d{H)O{YC!d2sQ(itbkp)W6SoE` zKX0hUe4i2=jdt=B8~5nd%Ur*#qS|rXQrIqD5b`X?LkxJ}%7>(E;~_0AD@`k1jd2k% zZXec#osOhq3n`vpGeEeF%h%=P|XtY;RyB(Go3Cg7~XB2LS!Y!|1+8nM&ZQI?B& zX4x1vjc_j0$LyGa2;o_AYWW{7@kA8~IC7qHDh$;zcVwe0X(uohHHjbQHmHZhK6} zvERW}Ux*6-LX`j==f6I)AFnxr=)gDY2O_{hnpyjA)gIk3jPN1Y?5crb_;et0nly29 z1!{mr5ms)pwRxJSgh4?XA<1zU-xO0r{LBV8L9}{rb!|rJGmY~gtZ1AjO~Kjdb)P!D zV2jmZo@UsiDzYz(NGVPustm@J?zp!x(~;ahXHnuK&eM_vyc;U;@@30zz&GppFAD%P z!SaMZ+@Wq$lUNsCL|nAapYC}uRt#t&GhGr|;qXDWGC(I$$CXdlZr`>|U>V>ssLt^d zhpGEij&7)oy6O6YOAf1eXyA|qt!dZyXg#_%Kzelj0@W|=54oY>bVA&(1C@9aaqKSE zFuq8US007wMy(ZAl73+53#&n*Y93;?!r=VeA)cz2CXbdX+UtwOnp&Ud*IjtwHxjY=9`Qq(kc*?FccL49`i1X9Nxy1+Jrd~sW^t4QxDOZDxew}^ui#}0v zCr)Pv0&W6E5;OIEZtiXP5}#6+cw2utnGHlQL!WS1Kk_4?e;VgzmUsKkp3eSse%wSK zV^ZFw_MW|PBN+7KlNr;Q;}XH^hxup}vd8_q|6{Y@MxBn%9p!7>e-3Ax2~%TiM@MXY zaJIX8hZh=4Zf_$@gG6|-0X%7y8(!^)2YmZrXG$YK#RyR?%+Q*kUS6Jum zZd7uFnTdpz-Dd2u1UDQ$e4D|UT1rBr6SePKt3B^{3#A!v*tBUU1hOUMWk-5#*wm{@_ynkV||vs)Y|@ffbPO-YH8g8h6;M#AdP zM;`pj?kbDs-p$j~vuGpiMu`^21FERR)^ILMjqTv5DNmO6Cm|Q*zJ8-qKB=rHw=rv_ z@Cu15my{FRXmP zWai74g13>sHj^8~Cs$2rNYUF7ym0AGrM>1xn%so5$775)8+iZD*hTI|!Otf!;{z^0 z$sHMej$Q|5ww8xQY)u_6BfJgos%uuIFt^oGjSxO#=KC*Q%GX#cTb*DSwD#;ES+@^`*!VtQ*V zmuZ)7FHp_E{LVC27xYI;)c{%^Uey}w9e>NMwcdp=s0=az)X7!%=!qpGvT;he(fIN3 z`CFn#kaZ014o&tJ9ye)J`ns14Z0>i8vKw}HT;n}UJoKR{7@`2{=kCtS4FX{O_t$eU}8?9|K0L)taxkY*pND=yuIfFYuEjha%2@-?X zeYg(jX%dX%o-~e`n@f9i-nZEvQJBO7v0+-BOf!vF+z#4~+6S#wj*j0+M@mGT3M_yN z*RTm^x%&|f);PDka+tyT=cl3jyJki(MNaG5^nCgf6BraK(q_Kl{XHMGncVF#6Fn8? z!qa=s87bqvHW>O9I^dWk!)26*!aJK66FtdZ(Z3f=)Wq0gtQwfw~ zlDqJUwXTu3ZjYJKRlVekO`{9DiOnI3veV6&o3JjSZ7@7^cRib^7r=c!?%Xx-8U!f9U~50zdTZqaQCRK`otAMJZ0v%TE2nBiL*>X-HYd(R_kh~$S9|#7VH0fQ zOWqAUP=b#;E{gUn*kAA%i&GCo`ZSvcyZ+$)W!DV9=uc6SG0m;NmcVw!*wUJ}j%~qu z+gjPA#hV!j!Gn4>w(-tWuU6~GT}a!>`bh{ipFKCVrW1zqYdwU(2JsCe&n` zRnS5@6Z#%70O67$5t@B(k+H?3MISBoXahg5x}sU54Bc&AzO=P?tbKcyrz7~xkCuF?+xL!Kvbc^%n!ph@JWmT+JI zZe0mQVqAgYV-Po{N3*)@eVjqV4c+bsv0-U|xQW@oDp{~OCXQPP2@`KFdWO+*JE2oa z@*6Px^av8+3AsJ%(RRtAMGmbdL&RMGs;isv0Vb#Z*<_~hFk$K7Dc$Il`z)699Xs0P zB|TlCT|4Ri`7_ROuC43W+9p2r#yzyPr}^|RjAre)V8KnTBe9Frf+PC2ldG@QT3lGh z92zwU;?j&6GjJQWOg>d%BCMZ_7cZ7Lz1HHXlv&PLLMWJ=b<4jEoXhWA4a^$yjryJX z!Du*bOjLQ1$I_?*6%k$V)P#CxF9ZXFEo_fMJY+Gm3Os2Q3@bEnhdYOYl6BXA;Pt0w z3(E@NM%=_Fbt~y}Dm4)8C=A0;_gfP&)UPb9tkBbGa((B6 zvEONkmxF@^N`iCbzK+{|6t6X@Vkf9r+yloF2Zp*PtgJQ+$jUjZ!~vFQNT#`wkyHQq zF(Vr)3m`&j5uYox_r@DXI6|rbCrRl6zw~psx2NZSG`9wGVP#%?9&elI@rCw~%jWm? zFksCfFX_aug@uh=yZy%Xr~w7tsKz>*%XGjs4c0=3m`EIBez4rn;EJyN>ZHNPb7 z;7xx1`dQ6dw%pAcyJLqA^DDN;ZR5{lGr%XCm*EYSiw~Dg{dg~}`qA;amDPB4W8T}k zwb+;LHtfv8UyUZhl^6FTe~exHJzH~P`O8~1UA>*d=TDHnvd@fpwQRCnU+iKix}D>_ zaglE!^gxA8m28P~+V0he76*jr~{g{NS>mt}r+N-oz9}V|;NQbUHx8x2=l75_lj{Vcwi# zIT52M_cCn54Bbrgc)s@SD+|?I9+K8+e7_KQc9rkX%q>_ma-Fs>3WD84Owy;I-(3tuR%1LX(V)M2Dn-(y~5ioFA&zv1QBY7vdQD2e) z<70bOkNx(1KuGpj2imtS>Bf!Ab}Wm7xO$LR(gkVlkZyM9-{0yOb&Sf-we1TYd=we-{_DL{+s~Z)Js*=yLrJolCC~=c zY}Ke!hnX8rZH%-SymS>iz~>UP1_f85Yna)Vg&5-Xfan%*PS^{S!!Aws4A@*L2a1$+ zkKhj)=cx^e|M*Srfaicu|GKX2Ye71&Ct6R@cYPj8^Gy?oKodA0KbpEn#~0oGa`xYU z)93i_-Me>gsn+F>+;adV*V^W=u58YVyVKZ_k?(-ucty{gK7|g0^;D$xffV{SajwcP z3OQHhj2{pY@BBuER)p>MXu3fJ=ySt>^Z(;FQ=y!(bdEgIIB~G$ATRb3+oc)Aa_^LR zcQh4zTfe{yifqA`-%!F$+h{(SDT&efvdlSP$&w`t>D)ol_qWdoQ&p~pWef$1PhxA# z%UUgt*NmxN5^5w(VW>*u?XwHF|F$4gE!Xe5R)JQ?!QrD|y!E?bF^lO54-5zxFIBD4 z$a88da~145=F5-g-S@_Y(9LlWd=UvvZ4IdX1APF3ylG{}fPG60@MY%WWm%n^0K|ZHgDV35t#^RYFLjny0JrJ=QJzp239eWMcwHu5K{16 zF+3ErUW~d0s+d*y7RypD@JSpY!W;o1Z{%t<+3?Bc(*C*QY!m6k5yjR8`n)o}DqQJ5 zD7#Ot?=f{udTYS?NgN9g9C(}1QBy->9`78Q6t^6!y4!hI*?6b7iZ?=8XV1;9K4_55 zF_*+mhB%uJzB83XK1%#pAd0={gTW4$r?V*qxEtr@@C>5Uj-~0DnJW4XuTB`%Uxm^- zGK+Enp=VrU)2-N>-JzfwGuO&oM++;{#Ctq}MAR*Gd>~HT-iq&L#Q73pKYa?_?Z@Z&+@ z51!5gE#g!i3h>HKri3xTZsq6?m{Tcl8=wVuOZ(pY>svd#$*sL@3%WijO^WK267_e0 zz0Gv`MJ7Y=3ZGe+gC(^5RYndZpt^{r`^~uUNA)@PYX-895x2o@i~3RNYJi1=tjUSaOro*k0$04P? z#pFA&-JyGrUs%NUVZ{8Ck>pEE>_J#VD%1s5q0`Tk<13seyjkystn%rfA|nCcm8Y)l zQ^he8+eNC@b?NG?ju~nmoX-va)w%Mkip=9oL}G~9Ky64Di$VE;+=motL`}D*ia9J~ zVZ~uu{<{6aSgg}0v4s1VD!(A(+FF{tjS$=W=K*!V9!${Ap2DW^lSivV8-mW%*Y($3 zH}_$3a;H|ygJ(LYN4ibmhQUHmH5pt|unW)$hdcau(k}Xas>dKqNJ{YOW`x7wM7X<= zbtF_G=9s;bnEWKv(mx@}>An`p4Y)AzBMzUEKyL^ShmUbt=7{#*AjwDXK?~G^-k*BG%0< zmz7+tpZz~S1ht<>~`e@u_|P*#PYp`mRmOeM?`2 z&A+qcQ-2?+Q3vK#^yqeFyT!_a0d%8_GNUrGw7d?+&BOWFTC$K>ih4noq9Vx?tk6KK zh=+uo7slpeBd%Xxg7ilsKCjPKH497;D=7>Ajs<>=y{m@wXHTEQ#oe(9KyWggScL#t ziO>v^>~N~N?=iLrkh_z5?b4;j>|O&{*xx7R*+U22U2*f8Z7^E-R(2V)?T3POYSLCC zE9Wauq{%yL7TVptSfP?2QU$)Rifda6)ht3b!arw8lOi;sm9~`%D%<6TW z(CIz9WJtw90Zg}Ky6dkiVzVR%TR8;{Rg(?_*CIn+FhVg#5DRhl{fHXlhAP!iQW|SK zd`hH4-orDZjWn>gbv)VkNJ0sqOU4=89UU^94%|2Qy#>51%qF)m z0tQ667duHI-M7udf=w1{DoW>r6IyukG#Tt1VRHvl8#)F?i0}X!ypMPUnQw;s&oeHt zN5)Du7xn(fj~^GNHb!j(up?Ou$D>~F-SRv`Eq5w&#~XdV78au{ZsI+vR>-Wz{FNV` z{7ait)t09;^o6DdFf(@T@_jhhKW(*0v0@*z1kK+`y<(DLZ$((t*WQ?<5Osimdem!$UxLFPPT~1F3f_cZ%vR<1g)mgdE}@^W3Fk|^(&z-z(S3A zx~B$SyvP=o2GG=6bNVGtMKDU~!y0hiQ2_X8mTO{DhTw|e!jWB_h@vnHElDxgYA#p@ z%K_T}%iP}3pQ%V>_gr$*6C8Be)o#m(HSYWrjAElcmm$u{x#GRQX3e{5zk8Lgd_N8l z2`z(hh~m_(Sh4NLsRJToBHt!^idD|+j-mZD$>MkAp&HXXW+dfb*6BJeuc8WPGX|-g z{$#~Dk&$p=?<*7({TrSWP%{f%YH9boXxq8=?7G(dyj*rc(_g$?GSK)7bCE=I(T5%A z);1xOxfe^yq^n+EH)C{?eN}u|DjHpEb*VNy)WlB3FRsjL3fq8fa5y{e49;1$0rinI z3?AgDrG5xS1dxpp$v-8CfdNC`Sa{uEkS6 zJiwp{ZPmrx8$hjZt~XXvb!)6VHlLYNGF$>exqE;b$28yKv8RgLBvsD%3qY$n#&iEE z#{K(4NRlvg&97?%HZ4Ni+;V-Ht;aJC>_cbsnU0*PQYR3f>Wx|P^(+h<*)`sgYxn00PVGG3{o8sy_Fe~L^{Qi zcH`;OQ!Tv~o0&CY-shX|kWn@orjne%Cef!<4OG`eg79OG_7A2?Vzhf->yw7$>+LMNC_kT4gi6 zSy9NPXrm0yp2vx=CS`y_IDTIdt`8G%r6a>8bm`NlnOFy@%~l~?pR$q?!RKxY8|$F% zoE*9NBPnq-x@l8`i7zB>go#DNY(RggBv2{^VQsJ5j;U3LGDi5ooRC@8S%7cAf!g~` zHQP;C+$*OiWUuAow|2!s88T+KOreL-H0^Ih-=3jbgdW9+4-InEVll@1hSx90GH75Q zF8M2m*hL{`3CPLqyMKNJ(}-dKVNjG>tfKj;F`yfv;+0wYV`opAq{3@d+Bl-K#UC3P zPG~H%DCSP_uh<_!&Z0U)=(>5j>9l`{T*xBrG`RQ2%_LpTftL9JVze9HOcGlJR>jPg z9K@8qo;)teV=rp&^qa%Ng?D(^m~hJfTF$EwxecsgR%JzI<#wK-)IU&E5l=hY(0|Vc z=2zqN8=%6p#4p{Wn<6z7@L0OY$hU2JMatsjxkO@xA*UzMrz; zB{Gg{XhezeOsB@bK`WcgTzfVbBON^U3i+OOiN7Z65A>W)suj+zf2d>r!63KV3b4<=0P=FjpmN!m zoVU#K7~}6w!?JsJuWvEaI4oH3o8q*!ur8?7Vs$*X8f9(O``psaF&(cx)bH2c1*p*B zw|bNQS~A5{}6iu9sIVwfha4wAyh8qf5dsS#_w4@RAvcGYu}#1crtA`9B?(pm(rVR$hfJ=%Th$fbzG zDTS$Nz`agkFcZV1o-rD#7CjoJ7d7u~wz2*ZbAJujh1X+qx;O8ipuhc3^>&bt0Iypg z-rspXVJ7{jgaiQ;PA9?Y@#$sY-YQyqY{b$y8RuJ#&39_NFqsTeJ_)&;?Y32_xdQ$* zp1@5-p4@{$i?6=TC+9q2vLo0hUnGZuAv@f|M!h!^Vw^zZySP5vw6{#>NqSqhGcX_JBB>Yww%i zkoZJV%s7imQ@U-8 zYXIvtb@2xDO?(n9Tj72jF^G+M^X$FH(P7JO+|+!yzil*?_i{cBGR+U7Cx3s#-ohg3 zMuw4_;qYclFtaCx4_A`GW2bZS#p@zbWTnytktk+F9Q>A^-4Y2~AHv`CtWE|Kfe6K?o6<) z1}Y})Fh1X+M15#YYT4Rf-+kG8TYjXUedEzYZ?R#ZNW=!h0TY@A{{(zE=$=z>A%hlsAfg;> zFXL@MMB;npzQ$0(9LW(r5ft=bY9OXA2qsN4lPfHKGSO8m(Y6q_~tv$M7xGb7oJw}&Lm7`8MBQi76sdQfFC>rusZ;^Y*RcwFt{R32zW z^LJqH-Y({Cnv$SXlkE`Kp*IrVLH zrTUy(-@HKOFuz%=PUa_S9`Ds)^cJ<;I&2O8`Q;y$dn9Ab&p(d8JB;605?7e_YROSY z)!SF*tZ636rq9j7mubgD%<5#8CT@9;iqc%C#`;5Lt0IBNCXKC8^cE5&pS4{lYS+aD z$x|eb9K=-;!Y(kZvFoP`klgKg$3SrF6R0E`DN>LF$fginV320#5EY0r5de&H}ii|M>clWk&R+TG22{xuq6EgH5LhGlQM z$n(wAvLYg5stzjJuE#z4WhxKd1lZ2y&=;-k1uswz!ix`f+g}V9Gtje4S2X1pHUsbR zE(~*U2r!Oo6@)NXqCnn%uc?IYcL%n#;~BkYjA5*l9c|>PFF2T+lQV~ZSag4UK?36s ze=1Ik;AdVB^OmKBE1`)XFyd9tH+hx{%%nwdy6+>?gN^ zc}r#G&gN}a_Y`3=$rLQwT?<@=R=O^>Xs3LMpCloS&7Ol1Vr)cFx%$sRW6P4`bKlk= zY^<4#rDqCzk-UKq2xq)DQZBJ z!do1<`o;CCX=)rr8~c)|yVeT4O4Z;^Xy&Nwtxy$;a>%6bznc0jtI5W;zpfldaVin0hU)js2G#F$!v&E)y ziJJ>g#8$l4>7=giU&p*vvwFYA^e}szvEk0kn?j+bOt+Ps`nL}1{A7$lBM3bYNcslG z{gDK75rV5GWoK|D;at2aVh#oK3`zpaYl3ihL!Lo0mix7$l+n^PBAQkp|D7d%tB;_{ z#^=X~ersxee!c`iNtBiQpQk@?&B$r=Rx?`DX!D(yEuUnXj2Tmp{IEWQ2fw{QdU{!Q z{-2LFxmPux@lLG^w!532x)t6hMER7E)N$qL&s1bF5IidLUNloU7rk)^*EC;az%F@o zxW@g(XU0ukSBd1oo1|x3$BQt`$EQi^r;TUvj4+>3MEmvoofZ(*z=$F3TS2s>qxxS* zPAXIWrj;sh*vgB)IK#JDj}{FCIiYl^5b*s7{U_}?s6)o$nQY8h@&4&XY+3d9><;dG z*i6>@_G(2RdWR@qP#c`ApGx9-5$qEKm1;3_X8p}~`o14rT_T)Y98(Z3-6&^;dnafX z2&Z9XdC|;x^JAmA2l4g@(uF#tF~V`JIXza}^05u}8-LZvLH~$iSnlC_o>41F7Ig3F zVB9e7QSM<9awez^Dc0m3|O$E-Zx31HLf~O%38Lw2>`r=M8R%}P6(pd~;;iRqSmzIpJ zsI#-32zde7f(+RBrYt`X{}rrRBTgn))r&#h)Ybav>;Dp%(f~^q970)Td}^=gVBjZy zjw*==|G;&1BgJjX&)UV`S9^&&7b^`#`Pt8;TH#8<3>Xf1fVd7h(xX!IyJ~3MoZW|S zh=|0$2E`xgc&g2j7SC_UB0=p{@uzD2uGhDAgF(SZs{QW%hbNoD=UywroAjQ?*f#xrt){g4d&o{c$*Y;6UhX-({g7;H0!q zAq{9(H}B63Qb4LY7m%V5dl#qIn_~&*#^?9?0wIGh@KF1O9XWNMD@5myi*yDH-rHYK_yfKTDf3J4K&QsnD#cN&x9~BP z!|YMtRs$x859D}1Pt>(ZsC>vyREC=3RMP+z_i>^Yea-nfI(jR;Ol%dO$gygXu0S%Tl8i&GcyO zWvoDZ6|VlN4S`8Sjtyh2Qv2ZK?Zga{jUhUU7A=aTb6J@Cwzlw|G9&6!a_wWJxU2|`MwQP{C@ z#wRDz0^U)azx!o_iXSA6owEGf>l<1WJ6-pAN`_+^jk4MCSC&yz@_GOHI=0;}pxPh! zCXzlNJFu2=^Ak69v`79vsP_3T9KYS0_rzsS?47xGtK~T*CCOID&L34#wlGY~ds0K@ zWO0F#2fDpo9b14q3F;F)WX5q`xp>A|Z*t8MYDd~XmNoT;UbxEjEdfvG&c#lRz`%|M z`}WxQ%-^b!6+rhvt8qk#Njum^d(QO%IKoGQo1)jT4q?nAkJj{%qZ9Kxh(JUP+CP=E zhlfl)p2!o_^~it;e!b$R+s4wmlKt8HLg~`I7C39jlfHnBWz0VN3LIeDg&lABwr~P^ zhrF+|tWjY=3a27W5*?uUVp1ez5t%Iu5E3k>okyMg@^fCYzpqtq!C7Q+wndME={7b# z(7oFnnKgK^#%$v-KHn;cYX1^>om0YgUW?GYXsE~S+hM;Q1Q5hmM8sjg@=GNBf_2!5 zi&Gz#w?njpm_%Yv75c}S*OzrSEdMxi=MbCR_grHio`tW6hDdw=bwR2?#NffdQ&3kv z-y~~rM1vOVNQ%95EQMe0Y-an;_Rw@(kg}?yPNY#zLvn?TvxwvXuIP+f(RY}xU2^2X zCXE`2F=MtE$8E6zlSEv_ z`CAh4U_lQ?Yr2f=++2>ld&&y!q!F)|3}jqttf+rA6@_qJR8-W=2+6vm_KEIzdyscE zf?IuoOr)Sdhrix9frMGc7Ont~@rkvX5D5(dL-FI@4=y)*#w^r;o*alq;wAuoUuzua z_&yq3{<@{0pu;D3pO<#3sflSo^rLDabzAPmcR$ml`k9RCta)F}wypXSf462sL&Ls( z)ROvX@F`&6y}kWjELvRq<6QM^P%JMffq}8+S{-raQyjzy(JBQxcdJx61t#o8(=GRf zEVFeCSjt`z5VhItG*&rVWB?+FM&%9-n{@~gqreVx`QIv9MK%m1vOu$cL)gDfBVxT*dXpm0S^jXvywBMvCz?^yee;~CS7A(+$3s6PoYn95IV84t zGq_FCz;4~{oolZ>)a{e8ZubLLy^BjyA3k|VSI8l@kWDAM;&W6q_dKS};;PfSut2vR zA6vww2r|{^@kT#7dXXlS#IBs9xu0v(b%2iDee)(DAPVt55W4BUzrIbKK0OQl-cKKZ zEHmCZN!tqFx`30<-V$b3f>%=ZDG%@AYfj&SOA8;jf_7YSa zka*@1<4Fjii4@R3G0!8NSy>0wY+^rRxrF*7OXiA@7_mA_{sVeScs4eY69fABJx6n80c&mhQeM>r8bzubQ3DLOjTeZ|wnk(%iCK;T?ImHslk8Asil|0ELTkdVkPserqb$4*cw zfXziqJK9(WcYJ?pud|J8erk2G&HkF-Cf{k+_C@4zWv0M?z7kLVmv}Grxu;Yo%B*SZ zlLMU%Iv!nB)EXWsX5ae@?gc%yIFyr?jm@9`usSGDz|Y zWfe{I_xH}MJ~_}QqoKqEEqS3N@sM^)KPA@hDGAzSw~3JC(1W5XISw!s-jfd*BWWkF z($J?)rx40Y0Cd9F9HFOJSR+dT4a#zT(Y+Anl)8%GR;&VBSkQ~^WvjP$OHstb*Rt_Q z!&G;`S$7?LafZbM7o%`~x1iwUl`}tFaE2NCUgZ>bRRTybEDrnDgG1C(#z_2#JuTYh z(P<{UcZ&iiThRV37=dE*JUe>>?a>}m?jaAteR_n$WrW52$(6`ZxvSgmVs|nb)_ngm zJ*-S}62yM!FD#+e^=0*O0go=(q^d}^fEJ)oQH%~&dfDwRy4Hp*sfra6Q zvtuRcRQhCAi5BO5Kfkdz*+-uZyvj@Z;;u1x)kZ(E{b4h4>i}O(Prw5QEh@x3io%=s z?rm9!np4NRjenilZ(Vz-tB=^z;jc4JyP4~QI}!e zVZYhp-2(B&Za=@qn9h2P$&o2KzfJbiL4y9z@qYsJC6cqM?p2n2mq@G?t*i8{Ji3Oi z<>xpmP<@KIy~Nk7H=rnFoU-hv--Q9nywj&a(jg<~HwmGifh$UzZ?iCNWk^_9*1g?b zoKhPIX$qP+diLxS#&Hgv;WvO9z=!If$2~jrToWAvXdoqlL0#8maiA4{L+`~Y;wvcgw~@|TogqEh%yN>Bgx z*tR|R!}?oF>u>cbaJ5L&^6R!(&8=t~o$A%uXEv7F%{Q%Av&D@A)kU55b>}Dl^JjtE z%xrmLmz|em8>i(vUad$y^vH#CeXmyiH8tubXy5+(`?#Q4z*chl#N~{8?%~EG5-%h7 zgrY-5*sfRy$g1$w2pMPwWD^u0p#m)xi6?Yg;@i`zQL_OE@UhhgV_e&MZ_sg8Ny}x| zb(q&^#wOAhWCfYE9?2Mt4w5WvLhjOrJR31KF|&iG1y0DoV^TZqzFw&9e}!`0RG zoTs5qZF+5|y`kFdy8sLhF876*kWd9!NCj3>ky=@1YZ&R7j8It7wi5a(3k@b1eF%TV zzXPnfuGcdChFkw>7iV;~xUg{nTL>tZqehNgpZ*6B2$H<@jWcdo+1jd!*@xXO5!#c_ zPeDQ2RiI3B1?vC1@)Wn?kfE)epnOfZ$-SPqN#2xZ$CkN+kg0iKZP$Gb+1^& z>Hr_|D5>u`u6bgX@PU_1%^fci4KI#2_~ueNw+!e%88ym5qPaWB#7dBX7vO zYX5t8;_mo)Y0cDjoNi`R?x^}|b60T?B^W=OyCG)i?TMoU1KHm$I{;;6l&apK5HJ`t z@!ShN9oszzynq20ql;?c%S2(zz+@EeFR28!Q9C%Lxa^83nEniHWR))yYb(o`F5^Xn z_Xy*`6R(d1qognnDo98871b{{D%P86t!J}2u&DSKws2+FpPbMULY}#Y!#MpC+r9w~ zZ%?^00h9;o|IzQU_E*Lpa&R*9G>dw=gE-!E>a))MRo_8U&_=1=yqbV%{s$z4CFB3mOhoU2(QZe;%f%l{|> z$=SN6St%_1vtxFX^gEn5Og4hDH%J1ezE6LBA?rXkeiN5MI8r;pq!o@d5wwWvm2p@| zEK(XixF+s}A`Q#3WKG1w4Tn1e^x$5vDwT-uu6G>ZiC3i=$Uqg;qq8OTG`~P$lK=?y zc%ZztbwP?mSok$pgcZt1>OUoW9N?xr7WLMZxHg2J9;KMzw>aC~ zK*b9jaT0=Hs&iFn24#B1sz*g-P5G6QZ?ll>;Ek9t6)s35t7C*8a4Eva|(V@0lm;`U$bW#Luj@rx zvMOGY_%TV_jKVHEA$gdLmve2>D6@am(tn)`XD}6Sdhp=E^q_;(81eLB(zIDhtjHp4hSBAp zwg<9udn>~Zij8?T{r)aT>=^^jYn89sIjL&*c^Vj~?&G{=2v6vfW)Rhnb4>P6!F4PMONfn)wE*$r zvz=S|DF^5KhFO=6?SH2mDW*P#{dzo+2f8KX*I%au)>5@sjgB_V?u@favt~-7>?W73 z*EP=qdO~!*sc91I`S{XaX3aftpr}f=II^@3G3tJ;W~Hn}?5kjSpYEcq>e934IDud> zHos)A$)Q8dyq;^(86%2#6|zHD2aHUTHTZ32^g56lM9}F@3NQJpndgGD)IVv62J*v) zO;zk<;G1?O#VtE@X!KLtomCdCV{`D|-BZTKbLB=x6@ymjZRmY|PzH$60Lnvbo2If) zkD|f*9+vob^msT0_yI<5k^F5OQ8>QXXnN4f9V1p1IZ7LQ^Ua9z-}`H}=)C5r&bn%# z>+4OE-YQi)lG*7a%CiR^YPD&IwcWM2rjt?VB#oODvv#LZX@}_VA18>e`a40uY)BIF zdbHlc2Gt-~-{ItR*WrFqNQ~XN-;{)t%w#tl_S5O#?;q|@x%#zv)Io@*EaYUuwp)jA zLxbzf`pxUJZ65F{c{^{*2?sKm9?9&^31|J8%Go0}lP2x4+-r-AZuiXjcOISXB)BUq z2k;kdS5V<2ttnBn-`X|6_1G=$+2OL{?jqBf8PKU7*qpy_*F~21Wv&Xd?;cgwu3QWL z`k;<&s}!mQXnx5w+hCd|Y617n|o8R#qS$A$zm5G&N zFCEl1w8#Vn1zv%KiZB%&G~JA8 zTOa;yHpp*0Hb}E6NAmQq0x?TQE=ET{n=HT{LQ;->jY8AGCbae35E6etX)b7f{4CRh zg{5WB3*fYFQ`Ya|rlcs+1ojOdK(HWm9V^Qv?!n6H*}CVce~|nf7}#7A9yU@g0bsil z^tp0K-BzlqKGeI3=ubtL4`?m_EJh_1>!o|04+*U3`_c~u=<4RoD# zf>I}m+DRTgy?75L)`KmKLr)cSuP2dEf2XCf`FqQU4EPC z!(F=6r~NjZtbgv&?kzV=w@lj9$!Jh7yC#yvl5l;nd#5I96)ot|M;n{BB8x_R0^GwE zx0ihoMKTea63XUZM*NBlj3X&Q``%3ef0Utvrfk6U1Z*jS*RTQWtwrPL)8b_%a`UEy zJMaMlS-320XX?x@4Uu}?HmWFWnp9K#GmlGkc+Q~Z;E?g1Xbhkn)vVE0-L$cmT8CdWH z%INp1xchOtd~UGYNj4rYA;fo}U-EG;`7uOCL7(b2J~u*+JGKjsNAyxZFf-W^#xuER z@u_<`OPMKODh$hvL$z@CZI6-EAk5$pyE|jS%>VMnI8ywA4LgRdcusT<29`n=nm< zL%(a1qw9Ye&H=3GO#P9ngigqn_&%+e$%xa_$)d+6O=%j2jBi~<0DSWFGw@AvxC<)q za>*7WHMB7w4@Q@7OmC|@AA^mw_7p|t%kq$X1R63>i#EW_EIYqPQZJXEW_NTug!K`D zgnlrwXJYAteUr{p-;wPoi-@7rs=ARDNcaU3f24^OSL31Q(Bq#CtLXI?1_F$sg6%wd zl>ODMbUH{Jl3;2Q-{3?Z`LUf^4sSN`siFGvM(X~9PBxh`d)cI$eR3OW`R<-*G-$<{ z4eH(R{x{d}s?rjzn$_?g-vM!**OV+AReQ~Vrde?>u7UU57%}URQNL5=E}=Q^4VRc3 zXD>e1c*2R<7QG}hxf~|Me{2XEQ&_t#tcDnS=x=M%bfkGB5-8;#zkFEq_r?J8DrenU z-Q5&lj_Cda!5cS`nkAukkR-i$jASQ_S9z@0wRR89(SC3~rQT?a&&@y)saW1a*uzwy zHFW$pNteYvrafoyN@@j1A&98T7zhvgvC|FpMsstRWFZOvs}JtK`S#-= zKtn3T3HT%_C=X}G^*F=hon%+bW>NC{%;?OnnPIt-RtOy=WA}xwq$aV#c+u(sI}v(` z+9yKqF9OmO915|s7X4P+N)bjE!Yq_=fN5kl;E7>jnOV^ULMmWmc=Fq{J-lW_GEX)^ z9*uz(ijF>535LZA)7qm>74FYRZ6bk%#K#nFdA&x27qDS9wz6$%f1{8|e=h>*wqGWcGMh1mgqE{PrH6)@s?IskLcS%fn_)xta zxo{8ua<>LImwlZlKzkT8Wf=l$|Mb<6Gvd2c#&3dx5R$L_(h^GFM zDtXV3yY<8c$lXqaOEjjA=kIvm34_`2X7x-ZJY07H*RN@ty0TZVF4!(Iqd$>3EKK!A zTZrJ5JMxaFKV)EzF8o$r=+=_r%72?ZckZi8A2MkLxlx7Hys_$Xs^8v4sz-|Z`W)^_Y&o2j?I^A>9j`mvh&SJWdRdGWd_Z;FlA%wZERo=SFQi}oQ zZk)aC#EH$g68kbZUEr}6cK?v$SB`>Y5s4v$*@kZ2BB6DA^%d2iTL;~3vQP-OBmqsF znjw-^IZ$Xp(=L~oowoVz6a{^H=zG74CJ1r22}rB8sV6E~B-J;r7w%w)6?M6RL9YwW z)I6F%l{24gM2t?Fz+eJ4mZN@>cn0(Kx%lE_bqP}UX9v!lJ^M=XoJv)rCo+q^B#zNF z-Kmzs6HZbdtt%9$b1IjLEVny#%I;|njWSW`{dv!$I)!%6l~s$Pg|oG-{A;;1?f68| zvh95O8nRiWoI|c&UG>bZ zxvvc}sq}(i!WM|sQaP*s$-GvrR|^+ji_WqBL4MkQ`9@;POLn%5vanDl;d(m=?gaM_ z0l<5bsR=w)6_=5_;=$JE{32QZVr9pD4+m3!I!{z^6_0Xxv9`ZsKCO%V3tL z=TOP<3H959Ns1)BRgVgP{j>%L0vDD@g4j~HFm-?2JdP5fsq&{L_QeGPvLjtFD>TkU z+Pts^sr`?WAC9=e#<~BAZQA0292OhAD%hv!LPHIpGEgHKE~M=tu~gh`w065_SawlFavIHg#(@fCi!5Jnyqd`nuSdswzw zo5pN*m9!e$$}hJ5TZcpcw=PeQ{tm8t<1|$R2$%o^*Ax*AdwmP@$~yhNU_n)oZ!5iz z9{}80G`K@2s^qranka!WkY(W02*fX$d&c5ei>tG=CALP~l9DcR+xW@^p2nDDy~CgF z0STP;>QQ)~Kge5l8&{7b{kzJU!5OK#F^ZT- zXmb}Wt=E20$4Ru{Nkzx<;llUAHSZD{zjG#j>Tw?qb7O(bAV!xG$xEV+fsd36uj5UssmYLRw3~Ivq5%L zj~sqZ8k>)UWdThk>yAeuw4v}?lN}|z%jJ0H71X6UN*z z%{SPaHLHyR&q7;M^Vjhor0R*G2-c0{-;G?#xC5?V8oT6*6IV8AYHa~3dZ8UwY2Nqv9+Iy+k$8H1I^SL`9s zt0DqgS$7>MjsV%4CrhVi%t(WD`DBAyvDNlJmtRpBDj|{lLcW2s6V1G_)1J&)Zbm{lwwEiuA62-QB>+UcMCDC`xy*lzw#YI{?uNKE_FVCx z?igdk#D1@b*;Uq-itBxlzg9o3&Zg&U0ssE|O1jhpIi`I%BtVO=3cM;K0$y5^whpRDB4y7x*IPqq$`yIk$m{u3qEznyEfpd}yIiBN zvN=7sM3nR1x~5sk^ffeU3-e1CplKTV`wnk!p=p*bjB2a|zc`GOUl|(t zh`QrKh(8c_xHk3>>F&}x?6+S8t1A0YnXCKs?fYG04L!j>nF2-TjGNcoinO5oUM?{+ zTCfP^Je$HcixH&!axLC85tWc`5i|?3jCkP0yzLXe3IO?^B}!+$g;pu_&zUS%aocF{UIOVp2{c;!{60wGQauDUw9+i5OGa zNOl{#IJpsoHd)3ZYG8zx#M5QVJ|DF&s>3|v?;^X+*RRj~_N~FXwcuh^LZg#s&iwk+ z)jl|@prENj1&IHv>7{A$SxXAeU%fg-w+TA|@U6*&dxmI#GF_)g-aA9J2CkI3^!rBT zAJ?b5-vptT3ErZI{{Vt2T38=1*-Xlm0DsT4P8Ms`t0{r^WHlrANzxmXOfr`Voiglk z08KHWqm8*6g*nW`7LN*B+$SNm*ON>wevP6xF+DAGDUJwx3@b zb0 zeSKOgCza=^t@J;c>O=jNuuf!qtgKEA@!&y{cvb{Q8!^Dv-@A=R`h|p)23MBpSge1; zr`U;3Lczkg&%wDbiOUWSOH#NlEucD>x`=`%ZScjT7+7#^J!=92rM zi=RuPHb#VegQVutTpeQ2$TTWAgu86KSh-QEY=v_``OD}sR~i^tG8+K4`s{(z~q0|HQ}+P1X5 ziqK$q=zt3sZqsq25}rjRxj$}{b238?Z_9Q^u;_yF?*s@{6;&3-X$;OR*GG$ly}2rU zardrWBs#zgo(|(pSx5f8#bScVJ{A|Fj(6LJ9hEcW8SiV&udKnvFZn@w?HrgMIlzZ8lQS`ret3-v)!ig#_2RrDwWCdUZE3aVU~a8!k6Xq%W&>>d{H&k+eRqq2>%~;|^UH0+ zM6Ws(d)jT1={02M>_rbFf9twir@@rJe(!GBCf>iM@!stX*R74Pwo}pSqVDUpvX@;@ zs!-VcAY0o7JSF)nv9i3WsONY3wy`DDid#jH2a_+8&yAPOlcNYqGLEw$rAqWPh}})R zOBvo|;To&Y@@~5|W5O3@l3HcSW`(^Pt!kCMI*PE*7Lq4StG!cUK@-r}$LH&o=sOS$ zAd8Y}q0k<2Gm1+ED~f{w?H5;x!L;XFFiS5r8_#$!$m^V!d|erNc!tHAB(uI8S&!HQpjAn>{0q@W@!Z@?LX@rezcYo zdqifb%%wdm`@Dcf%G=-4!vYY>6btYvR_B1TQ+h~h$o0i)e?KGG<)-lPUkAL0&qE~$ z^=&AROTF{if2q|`O3{Me445XJF^bP!(#Rl~lK&7?<^uO^9-uNMB@kCjj_aliFE$nbjc@we~1vHqM*) zZ&rMVAV8XJM`D?cYh=BoML#l96@ame#H7Ah_F#OMv1caLS43rXxz11W=}YyK`g;GL zb8pel0rO!eqxHgED7*%`O>D>1)#qhL#{#VIfU>XKDk6jIzjBT+3*Mag^yVod%5B*w|(G z#O^sx@kf4%NwV|$Zk)uddN%g;u;iNq8!77;ZV1PRMk}ZR3skh|)L|?ddBMdR1f8nFp{Dvgl8QZZJ)nZRLHKz9PlA<@8$+?y8 z%1pPI-wgVy>jiB@1H<(jkt^3g5yV9}P2$rH8qDT0tFeO7N>EkYI&)GkOhu3F`ktF! zk+ORLIRfM4xPABBWh1%b*i$?Ry4<3WNoe~g1{-T{9lDv`Th_gVav!EvSn)2;);~hXp@g+2 zCH2kv#tQHi?+f66(0e^7asT_i-hctyA6%*>ZEtJiD{WUp!S|q}-tGvbhVy1G&a`bj zF(>`~M@(s0MgO}k+HUYz(uZNh!pb%^ZVxnf*IrZC4vHO&beM_BpI5zW+rH)k1+bHc zPhWqf`GhS{2iW{rj~`up>_e7r_(sc8o1NJnn8 z@_NiJ)epk9Qy-(tb96+C!+m0vPmaS2WY5C4ZiB4NyFK1w#*BJ_DLovo>9I73J%IJa zM!U~+fle}c1F=NlNp$J-#WZ$eHKDH#d7)LgYCrET+xV10hk~Ah|4C5!@15`UlGHWo z3$Up5&zKaiZ|RvDUgJse7k(_L5SS0LvMO%!3Be_p{{DXc{DZ*8ofR(9D?RCqbwd@7 z(e{(ocAB2qgM1?I-MbdD@0ti#dAyCqTBd^E2*)Z0IMuitK#TX$bVWU z)S-E&leLOolQA`r&{94M4Wnu11*4GPagPKQ#(#6mesb2SfcW9M&f>C3l3ljd|Z*78K|VpPR>_ zi+FFBlbyXK2Hcj62f7F0X%{ah{px7ack;gfb&&E07PR-XAA^G-VuksXh9Jo}h>f@2 zn=FqTeV6~N88h0jgGZgXla{FKTUrvrkc<3)&C zsD4xP6R!Kzu6FU?c;0TA%IlZ)Pj85FNc%Fn%kIjGf=axJn-@m%xpf%2x&&FE2fMvoQaEoD znXVoict20zHWZovf3loQUNs!s_5IGOPzs2!6vn&VmNfoJ`?en zdQ}u14|)m~$j8U5=sPny`iTrf2=7NUt1wXQ#_JdHZ{50#<=<#S0D)sJRWK#kVF6ar zm^Fioj|pAYK8N-~UwPc)Q~jE?YH5xcRd5QM36~amaKY(+P|aimlJGc(oy@oDs z!NJO;yMxG!`D8avAZ~v=o%G$v9K3Q?OH4G^nS}o7JTn-}^hzu0V4IBsiR9!nu^r~0 zd9v5fPup>w0ag{Xp+q(ie*PRocXRKEh={)Z`rY=_&;f_++>TKQjcC|l<;Vznh@?Y_ zIt6`0;!Kdrsdwo@CWTnR@O!pJL?TiA;%<+K72b5506oq)7d7#TY&-#-yK8G>x#>FQ4$&F`u^5BwMMu7N2BJ6&xLca z+})=DC_VSV{N(btRB$d9libENBt(=0u?v|~AUp|VzyHOH_ZL6Gu)YOOxITMLePNN% zl8E+~&OLgCHG?E-spvyoBCt6X}!FIx02B$J%gPAD3I13^=W0`=P-V6(ZXhDy>1{pE%?&kwZiQ|cS$ zr&GNUk1;(;R?fd)2C@(7lS9jgcfPw^acwbmTKkgAT|VjeQ|kGs$HkqE*O_+r$|}pT z4zBDpKzVm?&w>ukJv$9TD@b;7+ltaeFZA;1Q=OfiV?|z$U*^7@Z-?&tO<6P0EYq{t zFF^S@T`Zn;F~!$Uj!X;XGHoryqxm7{jO(lo+DzUmv+6ETA^@FJKYiZtCTY40T9p^B zOcQngEQJ4slAuB>xM$74Av6m_(dv;Ee3$)+o*!yu0p;qla(*Ahki@v<13SP8d_GjX zz4%vxN9}Cg7h~2`@ow3m4v#C((@K>8v{zPk)DaYN^_wR|B96xtvei-Ki)VI^|Mk~1 z>6vaWE`>++0#ovqN#H39+5;sD08iqrOs}%7Yfo#`t4TJSH{wSNPoQB`f4j*^cU-XG zaOj*Y0?~;W2dG1mqZZz8q(s;WwdP>rq9eUpa1a1?C3yMh_;cx>WSJAA>DTbZ z{e6ePAc+s2fEC^MOTp5S@=L(NB9G!@jB03#Wq=zTHE_Q|e}6gM@JeuZXjQHTt1RX} z0bdoi%$5nV%nKR@dPT9B65LlYka9kV@lLv2`^uC1nde&9Yt(4jw+w0|61x%CCmI{R z&M7*k)C>sNyqWLigJF}ESYY?3U1FR_I3(%epFC%+!N#HQnEE@LRk|Y?fip0vpE_YV(S|EApVNu_!WB*)QdL%gfR4G_FestrIfPij_wNYCP zOvs>rW7R^Ud_T_8SgixoAhb2`U_cbTg^+P@GQnWFvMhX~15hF@ z>(in~&Lb5YNhIJ4(SvGfxa)Zr!(r;P-kg0q=;jToi&F(Q)~ZfeX~=mBPP4P@6x{X8qi@jI- z)w;?>H#&6aumi@jgQVs;>?x$e5k#zlc=zL>O;dOo;>ngh&O3Hih9@i~QV;bE-Sx=Z z%XHe!fldJqiZwEBoIH;@=JtBgYt(x(=twp9}2aZl$<;D@6E&QM8SzoC~LIxtz`Va53 z^GmB8ILsKZBUIEj-~jBzAh&D>&GN{HmTmvdTL??d zGZN7e=m#07cgr^w^~G2VRKXSw-}E&x@sQ}Ur-S$1&8(|mNeTJs^R|2)OTE90Q=QSx zRK`u(T+r8r zUKgLt*DK2h`{~ZHA4YCA=2wtfAWYW5-JzSvc+8?`h1!-nMEMchlfu1z%w0&M)n6{8 z8-B?DddG%#V;A`C4|lhg)`;`jx^0eKIC-*$pm|T>TRmSJ^p8A3X|}qt%3?~x#*K6I z^gr9syu(=gK&Dv??ccvSlbkFrZCn-mbjsiV_Tne8YF<0A6;c$#Fi7yTaV8Ia?NCJQ zDcNte@C)FF`>SYg_I`ddaG}|kXUMmrG(LM?oH_bGVD9Vu)ZA)kGU;}wdNoA_%)T+u z!NfNUxxCObO5E4cqxTm^+pMP?Dw`Xry;w#ybPGKEKQxQpv6N~GB=umG<^gx39`(8E zZFb_SXYEUO_Oj;GciPe|YTG!Km7AI-n=e1NaFGuZGPi@Dg8IH2*OKVE&6~4`nY$_S zTZut`3rHt{s+R$Wb{=*R=oYW}{{`Kmsg&|1T*36kY}J$*k`Rgd;a0up3{DgjSG2q5 z&&N%LlT?NO#cig|c_f@*8{I_~YP;C%?BnTQ(4CBeX8&yS9oXBE8T1qvH>vEl+Z9q=b!6ax#&K12&GC zVj1F?vM&7tBi0< zU3*s6tFcEArF~?B)$03e_6E0O3$_BTUBeUjk<)g$^hE2`_`lh|lONjn7p403>!(Fy z5u*njGwS&$h|1Jd$%^*(d~lFF7m*8sf)=%ZP!~i+7V9#DdFBkJ$4a}pAbDN6TkDoB zPhP&9q!*h0UhW1(|35j&O0;vV-1XVIwRLlKdQTQ~q6IKLy5srA@#pTy0i-|cHK&~~ zcXmCz)e$j>7$0uz{Au{)gt~D#;goNV4}xOM?=8M z5`%S}>d6C@@nEbdJ5D_qzI$aAVYNAKo=QpL$1`@-i5!bAq{`VsP|+&Y9BnCz{;kg*Y5H-h9KY== z|EC#s{I3i_Idc1k*+w3|t(qkNy}haG%$N=wznV%8HCuMu)^wBMBj*^k+p29=z9_s| za%8%O`Ya{YraSDGJo~MW*Tw)<1esD5E6a0@OuQGPut7Z}B7>cWYZK@2A0$@H*3Mqv z6i2|UE(9Ea-5p6zz;+?ojxH{jH^dplN;SkrH?V_Fs2+Tq=!f_dp4nAAxIq9=2eIx; z6{(i0z{`F;lw>gRM3Sx+_!#OJU3`N3f$-Cl&eKczaDytmqO_}cMWL8Rz%mi%Ft!H= zal=;ogBeb(y*)22uwyTlLx=Y51(HohQ7kDo+7xkwfMP^$R{NG7>o#IBqJhv>(ohqC z*Zl%{>i`dl@!wlh9NT5H?vZSD_xqul2#8t0*}ZH?jkS1M#zwt=XD*Xd_C4jWt#$en z*vDF2r`QLVnU)x@A6;@gO2CrHUquNCf6|*wX&1bBfV<`q9;_`R2&+qr27g|S-)i&t z`EwsqqECc`EPtBAG7?T+bGfU+tqe&*8E0wPUTkTvU%#Fn)Ep8-Y-!rsi*H2@V5go0 z&xu9%DH z<<_k?OK(k=VRO0ZKvpC&2>fu0;t}r{>q!V~y&vtWrsl?2kH{$~9Vo{pA!ITJmkc=@ zKH1vZ8{hjjAxcI63cAyph_KA}HylY~WUB@XbM)7BN_@rf#IS-FcGwizZ3CnRc~0=j zF;li!3>ZMktcl>+JN+@EsYsmAN+gck6U&}jw9A}M`JQzt$jdfrQ;{c^0Ilj}$$_tHd32;M9((Kg z`ade6PqqK|c=qI-eALJ5&B*T`1Drby7(D84qYuy1);ky)m&9N0Y}jU;dUYSxKt<`%vh?3OKaXtd#EEI5X1IgFWt*0Ev}xpgZ-ND_U!q!s0H)2Aaja6P=IoRxuOwg zD}r+zP%DH06Ginz>Ps5tmmB|53pB+l3uz~9mux+Je&zld)I3Q=(Sd>EZtIG?Pc|t? z%^)bu8i}s%wDVK&u6cs?N#u7jON!4N9wPes%R~i}+eu8{u?(ltQ>X3^IB?`h{$#HR zXLYncWb--pIy0pQYpV{p#W&;WzklBhj_jHvNhXRMW@mdHg+^#St*3=2;NG*|e!p?p z72AJu15)$F+^s*)*NzAs39!!y9R$fCS;%noWA^QMPG*o|x31Woe-(F4n6nx(bEwU9 z08UvZ%Icb9E5aG`g;B>bC)9!K;7s;`Z6yAsjdu*^lFO54nTB)e!+}bSjRIa+rw8!w zuPx7SC1Zd~Ki|c0NCNJ9!xYh2hkL%-sG_2BVoDE_+);=UB}J?L$t7+b;->I4>R_$!sZqU)1j;j|*l95A=^Y1UuS-hxOi>1Y>seCZ- zAYQjblkIq-p%SUbzyWtLnCJF?+Y=QhK_0c3r)h4Mr;uXeW9#+3B_g-opVwcI8=rSg zz@XcC{+=E!eApk0j?JZ7h%rGFK`ZEVv2(jk**nlVjKm96H8gY)jfo<~{)RM_(U z^zW7LGDoGItHEd90_^{z7&6F)$9=slw@_5DI$aBRW75)Xw6NiqQ zfA*54_0#LY*VLZ|oba!`>-QcS_o6g424$+`hr_+Gw~Jh?!#{r?yg55=Pxd0W zXeH5RfqZMW@8hlvD*zHjC{kQraeXCDrikw)MQhCH=XcSvQ05p%pSC|z2wn^QSF^|d zzY_91;?Y=0W?53-jd7fuA5pbbk^yIn7D0oMkYoaSdg^R)9su3N z>agAbkK-Mf>WQ|tIv3(|mU@)soinf*sbmvnyJFe0VXSq==bz^r5QiuQ?r#fzj>57j zd!7-;1z?Ec_KiN7%74U4S}$HV(=!BBuP8t6RauQj(M`Rvs<0+q4K2Ta>DIF&VaZPI zG`zj8PQz!01GWdh@>pwZ{NTC1+KUK$r+fAU{?FlVadbg-9fZfG5n6X%y&0B$zu~Yu zYwM(B*sU^j+&{QgX+X(r&24kn?^1qQut%$XS&fFHF5QlxE%qHb>SE1yzx2Adt-+zC z;iRtd7E4sT#dkyoa;%ZNWGEN;u~LjMWY_X9!Co!(?nx2|+|I;VvwF-K+=&!A$&IJw zeG$NY^+GHzZv%s0dnDU?d3i}D35Q|%BSO6_&}XCdtiQLk70m&Ry2P^)C+4%7#!r^M z^RKRlTs{oCbbDh%Y`?hyDXFPQMsF@|`;0hTP}{d}8E_1fjsG88X9AaV-u3@x>}&QV zOO|A3&_WUs$r520TOmqhNkW_K3PoAcq9|l7JJDh-p^`Eoib^3wNm1%~pDX5W=Kt^2 zYo2+`+*8+e{l4FG&SyJIS0tZTwmSUAN?qH&@JP*NETRdrp`%laA80w5sf5)Y9f{{O z?}5CO$#VN5kW9s{J6Oq5y@aY&ieDN4!ywG5ZmNhy=`44Dh;S2EN z9-sTUC(=EEQL*matFtQOAOat8!U^rRck!B}Si(pBE7tBS+vTLi=naYN`^R~V zz_qy-y0H`ye)q(Q6XY1@UPxz5fTk5PR`4y+^n!1lvS`z)73|;g=hz@Dz7$~=fT*XW zU6--%K6U(C`&`-QPMfcOB0g}as^~|%bebUUb46Dy_ECaf(E(lq4Mz5I{jGzmSRcq! z1Hqk)E?z)Oc(svs5G4#*{$ZPerRDQu)%u{>^)!8$N8}TwkwOHMFusmk`A;F~sNkf} zVMz;DK3i7N2=tSbu^_)OM+%PTwXcG=x>~Rin-!ys+N6BKGFZkGi@Os|2JWJS;WxwG z1t=QinCi4H`Np>maR+7%`M10;nfp%sg-5vB@LO(6cwSMunYpja3mD2 zLEPP@U|v$4i4O@{yg4Lj>Pe}FXhZ_9e%-F?O%dfyCT2*2`H&}RsAMdz_%xorOtzj+ zI?i9)T?Xs-9#XRkTI+fF94b<5l#I{#i`h|IkY0Ro&W+z?2M||bpK+tZmJ4EZJaVz# z^|zT$H#Gb#7Dvsg+UZWX(b~PrxxQ;jb9#8ox6YamR1*!=m!EX3?_RA%?Mbydhjp&D ztNp8;d(Y~-D%&`B9-FnldArV2y)*hY_UyY*_0mlZi_JaW#Cs1Es}97HfZ@Tif{4k= zsp#=EaQIb*DXX`+yWinjTmJI;Zzia3v#_KauxSz5?NaR}{MIht?wkF=mX2d)kq*s4 zB2nTomsY16=SeTgsQ6)4`{w=AMuY${dx)%U@4y4g>NIF@hk5A}N!aG|GKcQ2DG{mC zd)>XeeC9qA=$$ zbhfYNK~gSSxbr&Lt>XB_A#&lV$BUdDhHpnToy1cn?)^R8+)J@rW`b%3(XpvWxV+ms zMidlYSwv@e_>!~E$V{Zci+xx^%D(4?8IK>24gM@2(dE~lEFd}Xfno;Rc&vSsCL6+{ zzTZ0?ofsZtttDZR?2uS7V=69BPFU%e#n<|Z3Xh7)6?JqLVK5tTaj|MZyKCoV^GI=n zmm+E~?X2>k=~OleOb&dS;dS%LlU9YguJY_BA5)+`CD@u%Ddr*CJqcZuOz#J|27ZNu;nZSWyLE| zgs(5D)~Tf;tO3H{aGl`;9zA>3LnY+!;RkN!-AUS}pB5V+|H0*jIpc|4OR*Y4*Fl^W zV!m1wGebCAWPz4n29S}$5{D6hNlb7EDJ}f9Y17f#zzhm$Orqn$Z@erfkC7K6A|hH3 zZD@aq$T!OSY{hnUIdN{If1-%Sxf$dfW`5Dp*J#K4O$twtc?+dROdO)NbX!urpgobd zkbTR)I~EZm%u0>*$U2QycwV3alR;h|O8x8Cua`qll8=KZ7FfGe_wH_CHX5vV98AN; zk!`1qwKX6hKs(^Re|IFf$b2r&_)s#9)Ja`kf0xuMbbfmX_`Aww(AfS;Nu#0f z#}(c>cxQ`mfK5tNt60~)szZV<*E6}dt7SXavi-w0l_r{2dsaFm>-?$=l(#R}CJy|& z>*grC12mjsp`yj|@WqS7(<>6Y4t@G8y<~1Y7L^hY`BqNcz(riHk~|Yt`f>24Em1#8 z{l{}5i`Gl#)otqC2SF&80~wF?jc-ac%ps(5DHqBSdI05%tX92n2m)>?I)GQbPN_Zv^}jqXQ+)sq^UeeF|-BG_ucTLWyPYHDOa}x48NzD4nq z2(Wg!cQk?9>94c%HdiMcK?0(2hq08!W`-<$Cs1v<%g%HF$&{27(Wfz<`K&m=NfvaH z&Tpx4?%)R&14OmQ1R4^P8b%nWT%!aFQnXju4$p7No(TjUNoOB(8c)r*SQj1i?mt?G z&se)Q>X>~2{0d(9O(497D|q{;;Ppu0SgJuQOFT<(cRx2|UqRr3{K68VW?_-D zC)Rt3RvVylF|iN^1a;h6J3yIT0i3|jGuNysnVITBD#M$S%M2?FNtR#sXMbb)8wB6T zi$KI<)&^`mJej5zSi*!kbZ4pBmvm7giDPgP1x#SAva%;?8+5lK@&`=WB#|3KxWr^? z$0`HTI|H=Jsi`A5rr9fPA_018(NQ*sPs&^jtImhh*$*zE|>{pO0 zbl^L9mZ9Y(ofW%{td6qRVGMvfkK4dSz}$-Ss#wDLXm*09zyA)Q=y?u3Lzl<~upWBM!aXx_fr_o;~YP8}5+i)=qQ#oeeVCO>p_XJI!|`YX#&k z?}u(^Q@lmt@r>0N8<+^&X2DR_Nxm|bnB#?IM@M~4w>Xh+UQqn*G@m^(wN59n^?@s` z0=spMk!7e{4K%=#t;~d4y+!XCY9=bhi><9!em_6}LR*46gA6Jfw{GnQ?JgtCB660c zX7J%z{P@fkFdI~BCcW`FOHg`Ynt!q#a%@C9hEuTnmSk@4J23nu;W6VRY1rR4u;<<} zaDPoJHzoP7*<1{vY*HLK{(@QDy?Zy=inamCa-Bz>mr31^^%-FK2`O%OZz9w(0Uu<& zNSjsBA=Dj|K+=iR^iK{Ap4e?7tP8m5rY%?34!;$Cn5}`R+iW#XHLLy`xx9C9(5V@Z z3Z<17DJ=Yn(wc9Mp}=|c&HRG_dr6sxFNrNFs@R|>*}s-EwweGE2_ww-d0bX9@P-D> zkR3kOPdXs4T}8%%vCHJ2t?rsNSAc;1f$_zRJDmNH$J#R!LXgd+Wopu;^Xl_yAFw~J zgXo++ZD4co4Ipj`>bx5pYUr97s3PkipC-v>0P9>AjjqrSp86%=MyR0@a$e(kz-omzLI#`;%oSBNr^vPAbo{)WnIKDi_ z4$GH_8n2@mGTgavRwvY^EK(4)?dj9;Ciz5zi2<8}G5VRS%q_wO(*>sRsR@Q~$~1Du zxmf`nz#8N{l5^HplaNOm6vBB~5U+ij$^aEEdAkB7{p!6Z_ONvE8-Ma~CU}*{XP`IS zr!1Aw%G>#&5Au~D8-xr7Mzi|2MPMGD$)rrQR@d9SaL< z>COV%H|NLC;s<<2YGbklLC>!HmX=JscD!zVXa{{^>(F&(2ZmjxTvjYcoiU{qt8+V=HNGX(!8bkG8gbm(v$xOO9x7ETti zYCu+v4;OhnM;V)6qt>n8rmxfMfsqfdkT6fdUvZa6vE}l)IX`<0mOSZU!ED=JBmsi7 z{ZEmJ_mLxH(mi;5MjyoPrElq3iqSQT11izG`{ELh#3gDL=<-cnM?Sqi+N z(+aTHZK`WaZoKu~$9wDaqbSul)zottv*beCL<}C`W&` zywwRTf?D$)GKv%(+}zYsUR_p&wT03D3aNpHGieRL&FF(2BFhitHZ~)ZOrY+mCpS+$ zIdbJn4FYRzzPvFBSzNHrJNGDo8rxv?rn%J9s3ExvUzR2xQ2sQ+CTfoFxc^wsbQ>h@ zshYarA5wSHJm*Q>?lswvG;ogo?6B6oM>KtDxYH@H{;);+RnO(t=rH7_>Th+6wT;5^ z21GluPk#99F#8#*qo;OjM)--g1T31EpT}G)p=Gkyp;6nmx_%}hBactXJfEm{E2I3Y zHa6tJ?*vx_2(q#|KL07L6&qO;)|a3<69E-B04MNVjM*12n{=DW8VAj`$05Fus43iP zIgawb;f12~La?Xv4$p(2GW?NFp8IaZWY~W@_}-PJdT{$wRZ|Zrg32Hz0W!PoO?R<+ zn!V)#r>IF6o6~U_TZbR|mo%Y!})x=)*y=&@(5pD zd3=-G^ONK~LqG*RLr?bJBtDZRfU?w!`PT3>49v-?r-zd8WH-K06d_j$H$NN zS7QBvIg(>Ra5}WMuf`mY{YAYCZSaY$fq z1s3)~FSXt{WFs-MDe_L!6i&H0C3_JxH976>=FgLpHxevE13aeXHj7^U`ZZ?ygloEf z-}yAneg`^Y){2bm)?D^r_+4b#)$)wF4Nsmv z&A1nQY#*o>)s7S+?KY%`gOAVA+8%bOPtTqs2(?nwO{xQNvD&aAm3TmOxeCQEXnr=d zjcoUWO^J#!UD@El6n;AKgohq4VyVOPJzf!y0iHzbM~{0MRGJe>!~!Q$aVYm@Rjgls zKCQ#QSkq~;5RG=8Qz^82Vn&2J21Q=O!N-dLy9sH$DD7bi9pElg6HZ-G5kpOV{_w4llkk zn+wJxTf!LzJ+8*hzbRuXGOk-087^8`#$!>2h=eG*;zxdv+i(e}CytE>BqQG?fV{H2 z@%6$J=o>xS581|k$?`z+bGyTdKQM=pEGaL|Z>I@D&PAsHLDUP!OIP#t{SC8<@6$}O zHdDl~n9@omI@DxC1r+6=80G`im!Et_xk>E8Sw+&W!pPv6H*V?SU#0ZPT85-KA-X-W%)7VkSrvdZm(;WEGKU5w>!t9tdiAxnMOSMu|7u$hceU> zg;A%s-%uAl-m1YN#XBPigsv`1EY#1=_g+huy=)w;@n{cD7jQ+1JL1;po=U%X12@l^ z<*$YpZz*j2Q=p4=b)0%~n^Y7W__~Y3e^j5)1f3B5$V%#Ib z$q@c+my%nmjhZ&?Y=VY^z8_NH2K7?0PQ?zImOvPZ*eM_Xa1XjrShnASlSt03O|KT5 zRFV*V+J_Ve=+0GFKhLOHrQHlt=EWi!CaPs8a%EXH=RBK(EXmeBK5iI0z^L8!?Zg&H zzAD~B+wU`Yh#2RDq_fP?{9IEa6&8NC=&yRPr%lw$@LQAr!-X3BM}cG?MCj+f;~US6 zYqzOo*3^_DUz1eJm~{&rR2!-%HhtCCa23rjWmD(j0OQ?5J`NbMOQ(8cPwz%bx^D-V z`{*p3k;hf4kmGWIm@(=ZMzXj;!gCST9hx$xVy$>p5bi=!4VyBQv@J+-+>(7w&Uw4P z6juY$F@{T6S|V9T@Pvaxy7nX1B?h?QGi&+`KXAy!ls|mTalSPtey%>hQKQEQ2hQ-SwL%YUVVWx|D@<%u{d3Y`nzTxnMU>FqFJ^ zkazlrCSDnIpcqMj#NC}bC0TnJS4_d9(Oj@$>`lx{+i4UrVMCIlazzaQn^kABr>Ias zs<(Tr{z-toyyWgpYpoApX8xR`hFI4F@sOCZl{R!(ll}SkPRsiJ@ka+0c0UPtjy1a- z2g7*$6W+5Ub+Uc~FGa=l6AY63v0a7VaguU};qBkDJ&%nL0ovcuI zvv*>~l>7JY75La@YYgC4;mL+g3;K=$wNW#T$h<4Arf73vR7MfIax7riu60Ck*i^Xc z#;_Nu9Ik!Pl53(ES9U3>1_jiWGua!JgJOM1!=3GiY>2mT%=^x}VhC5nnYN#`S1F>b z4e0U4q=l@t-V`@~1Iw+++`Zs8C2Bhw-Jc?alT%N!yegr1VwDrxsU^P)G&&cqEEo2z zc)5h1S&Mo=$4bu;_4#}(i5etlt$()JVl z|79b)+w}9ex>eBJq`?NYdYskKcY9Par&X*q1LZ-h>l;G*olHr3(=Kzvp1)eX+WDon z%dw5k+H1dEIy=AQ+BAO6tyj~1u#-W4(y!c-EE3LwVPtrCYe^}Bfy)@^vvn|y$sCeo z*#EpCSaK9#oio96nv%1qc7N)O8QXYb(M}FtgrVgQAIM3;Fm>!cKV1<(1*|<%c!HajEMx#pvC5m^$(@|0BA?jab8=7W{ zPn?=jK(9akn5Yr({EzsVk?CsfmzV2B?nRR%dLW!AT}@O{VJgu=H+!tmj}5p6+K!^A zw&hQkB`yu!f0=k=I;jROqVfVmQuh<;DJVl?WT3SVHxP0b1t?ORueQT{=RF3 zV&YxyYMf1zj}#zodL+7=kJn`PVLVQHCl!xN)8bD zYHRP+M55AyJ%l5dFF(T)iDcTRLxeYxOAYSf4H?OO+(}LCHq+*av198=9|C`%9lCq} zepmj{@Zh8?{aS;)&La+r1G_Q>1OpIth=bog3-!KzCwv*q3Ju{!9KvhtTahqNCiKtk7_AeF_$1Ir?q>LJ&uS(&--MNdGEKzKXvnbLH3}MS0T-BhT zm=C_%l@Ib~Gk=I~65Rla)K2&8I0$5l?qp(Npp|&>Be`c5OZYA&8CO>L`*I_e!Tu>q z_GiMA*BRkOOKYc-=;P;CPa-Zl6<7#%23u-yQ5G7J8H;*@m7z^o0TCNrUeq__hdUQZ z*(+t2VWcOkg)dwMCy!=V3E?S+Y1!;l5C$3(Hgg=BFnjRm0i>e;53$wA`f*L$^A}=U zDG>s?MJ3uSY;skziEj-CO->*09p9~5wc7tRl?Hz}X{Vq5#&f7I-)rQ7bp zL7VFv{ytRa$rP#~#d3_4kHhY#2p>i>4)3)vHLnF3++B&pM z|6EOp!-m+nT|v0f95l8jVN97TR`drpP>XwA2k!AK=|G}tGs%1NW*FKO2>B!Ra&94$ zmS8F}q@J3b=c#8CBAm{(>5J>)0cQpp`+oEMR_@rjy{ndQnX^Mr;B^hpeKSt)>*k)v>Ia=4ADu2*nXzkna&Rbx)avFH zpBGC7OEtKN9#P1yCS803fF2|=9x~G78Lu0h)z(|%F%oU!yN3aP9y7)J#g}rFi&-%( zhK%M?A{+p2z#(5YE*%hMK?fo0HXv&J_o0w#2+NtB_~Mpmt;8t0$hV$E07y0xez&k&Dt3{%;qfvPk*weUEz)WU~vRz2n5H5bqfKxfzfbGiR^A&J4?(|3w{}OHf8@B~l7H z3>^5^%_#=yOA2h@s7D*Ec=!dojcW{um#Is7&JtR0~ zeIS!aY}6x;jA#~_(ym+BMm6NoZvc+KdJ>5Prhz&%(dkLiN*Yv-L(mLU)AU*|y4vr< z<_*a;bIMF~^%D+U4t^_7l;~$JVhIOE8@hyIp;*(YQvv zw0_&tv7T?O-V56V3=4eP$1Y-!x#_y!jSh`Dyt#Bl(;Q`&&8sy|Iz7+3YJ_Sl$Lp%; zfT>*{ykAjk?-AY!hykaGjhTG+WkB&(#pg@8ZR_Dex8lxUV(m zH>90}OZr5zXIhfRe*Sb^YGI7S#~Wp@_gMT2Vqk@z;?9>%cTr!_S~7f^e5b_U0wgv_felF4^i&j zVtcCgHlOD&Z>L2EPLtfCEDXbe4!sXNKQ#chi-#6sW_DP#>F<4`Q+cxKhEqiuQ@&5K z-V{qr#+#uh5Z;cWVtATbNHXW0S{avdeknqEiDY17%GZtgX{o{?2m&rkghL-MI%}6V zHOwXKtAQv>g_u$C@7b)bh@%k$=0)ZJr7BFNB)6BwW)cDC_P@{u}D45(w zSy?%jY};K24#cNs6a$0c-f`n)Yb%uafUI*+=#;e{+Y9OtXba(F%eEKkDIrNkWkO=y zG}>h}2j|V7vmB7FDe%R#qLI{E#`$3o8Nq);nbfG)Vl;6Jqeuzp)oWZzb_5C4iZrrL z1xuMM6xvDyqlH(2*GHP0J&$V3KDJ6314;&ZR7-`{j~*Mgi%Vm~@ZrB_ulTkn5B%1g z><5MOA|t8+8;=+4ZU$m2dMt8q5d_=>d(9|q`+(74&iNl9b-??Eu?orYMN9jmxC;mm z(utuG&>A&S?{u7%gxsS#2LjX2EvP3F<}oAF@3FNBQHjUmMAN5H&V$i7@uzBkgO<5e zlxDP2QEBoR{h#bj>C9^bbLxSYd#CEqEl{hDj-@kG|LoRG&~Kquae@}M(Bq8a@zPle z;V>%Lh7*0wAM1zRp)S zHxJN@Eo5r<*RdO;k(<;LV~U6chTiYkPXPw6O}Eq=wi+o@z>S&zA#bJxIBp@L@t?1| z+e>=Re>keXx4(dTM$a5urB zi8~&Zylg$;YaoLL)7(pltR~|dO#(ULw}cG1_t?a8+@Xmt{{;rim@(#v-gI+lmI%cHT_(Y1dcetV_Uu@0sc* zK|Xcr*B7%7DFE5WG@<=THsPtwcaBaB92y&Lo9o|xF{I|iZV~zWJq8n@B}aPsq6-Ij z>EIyIXS?@{jK&T`XvM?}@wFn#{9eeCQ>_PIyYNsk5#9zh4w+>NGb1bR$8;GwTwgXz zF_HX^;zswi#50gzDclRKB>IRSLJ%Ce-af~j~2(E=`75sLe}ithZ)sER^o}4+J;!rO;!ta03-ULHfuE@J-2g^k55@! z7}q`OwZ}_bG+iI*!NFk|%?PX7iavsh1nOoHQPCO$vAh(y zTO@Fkstmm}=(ILe6r-mi@#vCuYx0Ch%6Zs&$W)-}h5leV#E-UqQOiD*8wVa)L=eh6 z0kEyzhtMAjGW(1|?)+7^YkUCFH!`qToG-chmMg7;Y`B|ZFb`ayf*sZhR-GL~!s67f zQ^&Jg>+s=32uM^v&?S?0l-&z@4a}Lsla)o59)!D=l+T44pbd1FH!y5|X*C;)3aa6! z=1_ZCw1sTx{`GaY4Xfq2nb56b;A5|V^)oXzq}KU40C(*gooX2%)<&WsyAPe#1^SJB zb$pa-1IEk~Z0gOA$b6@DlU_RA_?T7iKPrj-38CYw@<#P;mNmZ{5Hjp*g!k^d4QE&{ ztcX@t9in_OWv0X;##U#Z z%x!zVqt==nZ(V|52_d>BM)Ej1Si|0W7lpk^fdeRyzDTeo%j zk!Lp2eHL>RZ=}WRe3L%jTyNV{KGIRs>xP$|JyO7<@h-tJ zG>y!OE+67>GA?6dm`2~X|40!t9V9*=wWpF^+LbVX6OD$1U!iF>uzG%yvdaw-G=ngo zW_9xic@P1*LK1y_L*-BqZ@MrvauzhWj+bXwCJ6X;b;Lph4TQ|H zGDxCT;v-cVr&`RIa`lN?PP1KdVrC4vcs_rtVr=$o8@ZGbUxJi=0uO zlOR^_;i0#RM7u4<0Y+f)k8&#l_!K9U<`ZAU$sT!;EC%oST@z&;R#Lx7z_DW9XzDqI z0oEY4`a18}8Kg6IV@jUNbjp8tluR@MyLXHJZ0w$Gqfa4umh*ccp;$hEUqbEH&b65y z*w@FkfW+p@={zbp`UoxzC4_j{pG{QRD9>83dcjHEu0ljaevWuNj*xZSF@u*@mS7(2 zvNl!GE3aL6D%s3JDL9mxcOZ278b4wK5%KVS?D0Ego&03H$GEx2+eel}*cXZj{6XN* zipc!HS(1a)I5JwSO?(jCkSU}BjNJW}E?FY+?Dth_jxi#Z5|2$vt8>?`IZj>c@sTa6EDc}BQ)2j`sKT))DF-S* z;$!yQwj(oo(aY9iIaf&~Qm>^udOy%`!jc?CJ3#=ZLOU_Bo0CiCE`%0}0)W;EookaW z4nt`^|6&2C*j|p2?7?nMvZ?Gw760^QY1q(pd!|N5KhWZ45e$zRF35r3J*uo=Y;_5P zd9WIouO4|>t55J~Nv6gyqG_rhU#L@E9}Uhe5T3agXB^lU5)xvLEJLbBhj#WoNE;y# zmt2y@h{q-?Jmy5vwvMi7OpV=vdq)Q6pe;FJ_5LUK=NN_$>KJdpaLpxPkhqr-nM=2Q zVSyNkZ!iqC%i^sizt`MYHLpmENtc*d58FB%3S!Si{?#>D8+}isUy<8CpiblrU8HDP z-F$cp2?pGK~6N|v2h-1Fb zDPx#ZU!>Lld=)eT3k0CuOh-6kL^Z@CM~m#By3t89RP)XMLe>8cC1~>?a`{lI^$6zGG|Njja>@pohaK7Vo5^o=AJm&T$MofWmp|8U8&vcv zDCW2GDUG=oW+bnxaWa#I1$j6)AW1#MSMiQwGXh$pMW0zEhZ+z43&=$@$6T?47tu5d zw!>Cb?Ysu})8)(x<|RQV#}MZtYBkYq+95wC7TUZ>0 zIhy%XBS)UT;%D7LdA}VOj<;UAkls^za1wrtC`((maN#gh(}1z6hCQUiJ~|PAvdK}K z1C(l$ribDhY>r6E&Q~n2Rtsh2%bPFGAkIeUkAIm2&gsXqLBpWw5y)IoKi@R?C|fH6 zhV=ZNPEat`wZ3yt6i-8IK{sJyo6A9w$l(FsuhBU6Aym!aBIh!T*s@jAr<%>jK|#fp z#k}wMM8%8_fP1)3PVmoFH2C%g_Z~Z8LM&%;P+V{=(H0;g%lV>LFT%Qsc!D9E9(2vH zefwhw8}2O={2_Qs_uOZTfZ%^3PgxQEJpOG&8j^aB?()*FOSlNfg?q395i1(=+T-Y&pJwcKH1;A=dxbt7+4^% zx6ID!{rgYS_{ahbwOXEiTeWSwa?$haTM7a`6ctWXVta2V6R*oqS#&wjx|QKC&RJ^{ z{HoWe0eRHC(gd(yds%>5Y&TT%!8T*4X80Qhy*}HFYl|yYm%fF64W?OOxdr69u3?Gb zn>>PB!Ykg70A{Iq?Z?o`Ilv4u6YO_ATma1|GSo%DX*orGHO&7%X!f#_9-WG<(19X1 zseXRC(`*;Ki*nDlkNw)|mpFr=cKao(qh3rwq!!{18-8fc?m^2?98`kYmENAV6$vX{ zK8zRic-}n9L1m{+%va;)&F}nZ|IY#EZSO zv837hUScRoL@)8`>Z2#Q-+rgw4wU>f88&7CO4ARIc{k9&z%*|}ZnZB{{QIt1n)t8v z7};@qCe^Bnr3ZN*{obgF(vup$ZER6`U%BItj9Q)uF)G5Re>mg#u$EVL{|-~S8y?GW zSsJGImqy=V>zBI>%C^1I{XOP`kNx~o`pG#@-QRp;i#9iZWJLFq7NJ7Qeq-U>?sP`+g>oS|(_@;11H1Wzz`o z4L_hs{p-w}-~8S^2%>EeR9S)wf-O+6(j+()zgSIWvw3RHft-ubH0Y=^xFWj_`Z(6_ zjtqHQ43C>aM%8rq_@zCv-I-rTnol!!=(1NaKqDNqREZ-TX0WqM)RTxrYpubW-ZR;R zIHAlrYAhQrCY!$?_aS6URyx@;K?r^UmR6;Zg8GTDn_hU4W)zDC0%&^MA~E-)GxKwd z+)PQa$*e9@c@Yj+@kr?vUVYOY#$q|F#PtDCn!B_tjcIjdol`;%k@ ztte~ncJ0u847AFmuBH=Hpo5|oPFOb2ApBcltCNpd5AD(tLLnk;T*eZOBYVTUch_Hj zCVzbYH+kaD!;iy)oI#5~niy#U&&U4e?1s#VTn4DC_s0Mcal3Db)rds1S67ZR)3RZg z#QydfODp5L)0Y(##%5#$rWLlgCgWjLu*sCMvOxTS8ayn^g=%iRVMDsOUy`%*^klJD z9%E?3!YdzM11So^42ndHy#~`Xn$Tp2u(dS>d&g+7YL%?Tz$$|bqSFj*)`^dQYjr6S zNBG5y7lWRW{4bG;CY`+QJ$%@R zZsa@o7DP;if}nwoORVF#3+$u}5_L_#{vPX!qWW3ZZ|pf`RzKB)mzkUYsl)tDcAXM8 zSpoLXmol4_8?`<-Z2r36ttR(kXQH{5{+Eu2O-{CJ;iq(d_PRUjb22N+FFvWUzarnv zuU(@K!<0X5z&_S3>|~VGn7Px3hi7L^{pngQ&pvKBQ%JF z%!<%(OOPa){ai3L=4-%A!sL0KRJMVOM6OLN3gxCtoSLQ>;$d)P`c$Zx0aHZz4B{Pz zCR+5(?RA%@NZb;#V1YaOEKw0DTxwpsESw}1uM&e>IY^F@uJH6R_vN+uNI&cvQz@4` z5SK9mv<$&>$WQS{ke^!;OrEUY?Wqq_Z0X3AfEJ9YY z3U}~>OiEsmdLoC0yXVH>KtHiyRFneKj8Oi@j_7T&kB!b}GKbG;5PuUJW0TFKw>&qY9n58s`}lgv zj~+=BIJuhIA%#n!<9YL0t6Y=&HDs1yA8%P7g%{mg=lz{#b!#AP7FDLgLWOKW;DVJm z?k=Dn+I%Zm5X-g&A@!+!`yiXe^asV&F3ZECAwtuoP3@1xmB#S->e;m2H@Yquox+|` zoIB)6UhcKuBqNbU3PBpzIeP+-ef@s+;am2W>_=dRMv?rLUHkVB?Ac(f#YXT^;LM4@ zL9{GbVQb$wwuwbbaMAh!B3PL;kIPQSc=B<_-;VrS5RS$Q> zCOUUWRa9~uab#ZoK7`R0iKyAaLY*xk79Uh5BkdbtnQ#!JnUxQzDKO|~Kr-&WE|_lb znninXGja`KJ-7_P zC1mxR4<85~+8Ue=jsOQSkFl~(=`bf8gwn3LJ>rUBW<#wtYemJRNtDa{21@8%Q-=mt zO0(iqB0Qg3w{Y)@V~*XTGb^=OZh7ElNq?Pu{QY_Lc?|G$Xe~q*(r;P1u3MAvL^Q+r z-_%Yld2;4Pp5F|Nlwy^a9jhs5M(Mt4F1+|md+h;AE+T8NYXK#bU-Rl;0H-1a4~d=(po8A1qqg=mNo|9s5`73Xng2dnp~B0PAO@t&C%Ae1 zAq?D`CXO08axS<&m%P8hlA4UM^fo)OJ`7q?D-z4cE`-3m&s-2!wL8Rb!*hiGw2jyW zQ^<;NUQuDuXB$}%OeNAkC|c@Yr*p-Ma|^96&PW3K!Y%LCBq?p2UFbnrC}hGLG~kmd z$Qn79LM#wQ*tEs;dI<{^NV+YJ4z!QgNM13JtMOT?{4S za}KYVCMFW{)k2`!0|w9YYC|G=lI8%VGhmBk9>Ns=uY2an(2X52K+@DPhz*>bD!aH8 zyO*aJR3wqx$K=H_QnzoHDKI)bJB4KSw)wRka++_!aqzCf1|Howyt^ALvq<(bE{#cGyZ5q=#n0AERac+ z)pu!l=mmxtuAZjxwZ2HC2&MOJQK4OlS4_+(?wBS(A0ED)rgol7ZOwSK?VFNw>wnWt zou*};s$I1h8!Vl7w$8$K0lRc+Hujvbu$?c}_PW{UxeX3u*25prki*V@+|U`R0=xC9 zm6ICWsaLP}BeK@+q5%|Vi%@YR<7c94|D92Bdf`6N(?Eve?F&|z=}B}b!)F_?yJ@K$ z^5<6?SCmFq)YEUI6k%P{5eJ;Cqsm+vMZc5DHq%(KE0|<$g8)G>0CasN+lKLxZ5W#< zy`sr#Pqav%aF_1%#vOHZW~A&od-CM{yso-+>(_7Q4rqiWUS&gvfzHaiqYam(ym--D zC054W+ZV;H;v33yN2F~s*+z`Bs<ARI zO<{Tj3VXh1mqnU5rm`3dy$_gm%@-Fi<;Hc9a2i)^f=Q`p^DCvIWG*HO$?}%+h$7@4 zCV-(y$dS66hjYbgan6us(PA~4J#=t$G!4wM>?izwyJ$)}njhH~`qQGvU4P(n`4d0j zBtU1hVhkjMm-3UBMeN&LwNUCxTqc@2wq{BGCue$@hZbEPvr!|EDlUYjez`GBh1`Q| zce=MH_a^PhS+wiiO;@@oN50^HFA26^TCeF=)?xeSkKd-ay6SE|nM2-Um9L=HS=RUd zb0k}xyPo44z89f}uHUBuyFc4}d$rq!*aQixk_h1>vJ8$D7Ztt22Xk9H!zt^M(dcvo z0t>MWAc|VIaD^8dhR*%_H^DRIl|6$&lQ3Du9LsI11VG)B(5J#dgI>9ScEFCaF0SoG ziN9^uY?xpk4~p5uCBsKaX+}jK-{o%@D};TRdeSZ~3*0?v@ksDH(46cQ;bT9VjbZGO zTCl@x*`oKS)h9gr5;{sVAYtCwfsZ4c=k>sT3yk`6E3(aLDqB+NrAp%lxut_SMqO08 zs;PAtG-!H|#z$LHTTHV0Q^Qzo68N+n6`hC-Xp?e|q1VZRnBI28m*S!$N{I9;2v~DZeB@goXE`EZcXVcSI;;)v71Ol zWyz~v;j^)-5@W&WPFmuY7fSg)XS{% zh>rr%AJEwhDPZD^OiGcw)7;)~nv3D?VnYenLH5!3%dm39cm1ih8oyoqnW*4uY&kZa!PLU@( zUKbyu%!*{5)P?BiV}Gi6Z$N+tja(LRjv|BtUDM|mt1~(xNAa2^RxCLCV~sbNhQ&f% z0!ykHsUP3#?H$Yp6$TWKXV4#?bB9*=H1M;Wjtd<{{bpE^ZSQ!4@3{O(b)jMSx>2<= zx+oO-%XeV-Hg2y=LGhav;aXuKSIhg=KGoWJel1ht?8P3H!Ks60y8K*z7A zlB@6KZV)HCTAFvz`U!hO+Ao?}6R+{MIKQSNNgo3t(2CLL~@o$cIS&&ieBeei*v57>PyiH1W0YrSQGJt`Ld z?H}F!ZbkOtMX|x;uXFS3&^kTevwSHQ0DKSV&g9!?oQF$S_MQ~9;`u=E^e;qc+qO(w z!jAv>x*N;sg?+4)KMjuW_eqP+l@dtsp7_Sz>qzCG{akTxkSYaU_*c`!cO3=mV2?Ek z9LC|ZV7HuJ*Gy5aH@Qz&Mn2ou5`S3b>x;#IJU)%%n=iQ)>f`F;>T0^8C~py-|6f{T zfEf>mpFsf(FT;-S`8sJkqY1qZTD>9eSd2V!_chksELt6YL%l2E{@QgL zHXLPdKZwOGAC9$lA614GNWxGy#pQ@o4nnLIHEU&xAe(e?Wp5|A3}3kLO|2IdP|Spi z6GhwhBKS+-!j-D9r7EjYk~XRywp0xXcQ=&@nooCY`}A-bwr=%=ge2)e4Gj%T4L0@% zD`Dm`!PicZ#8;6z$SyK8YM#g;ZjtGt$SS}T2m+!uM#GVZNjx)wF6B>6@{moE$m{9t zUF+rRP3u75hGRTvm~n~xCg~qc zEA-xNXTFn&Kd$-i-YtnTVDi&D8#zohl=%X-5Rp?4l{y z)8LJn5jzVglgwk!_wP7NEGzD)ozf64q5~Ph>K8T{Zi>6xd>=b~VX6nkh=z38GDBv; zO~)5rMZRE=g$@EO&V#NV)4vBH$NjZOF@3CJ!Qi$<)^A>hizt!hShtIH^3yXilsDdZ zd8L7Dc*tXg55Y_Pd`#x_oY%J$FYCyZN(Wj}QgZpD>hISvEK1)-K(d4q@T`_Sby$zE zIV49Mea5-d;x6-_5<64S4D4#`niIXlD6kOmn8-fav#aw>Wr^pWZEud07UyTlXnvya z^mEl&yZ%aRd#y9%LJ%Qh9AbMl!J-E<2_vAGQAumAi3-tCReMyL5VS#Ldbf?fxVSU`NYpC#=Z*YN4176cNxtRnLGAVt0q#o zKHnIapkB2quGpx*y@9husicSoS>+8#>o@;w#|Ig9@yqy?K0lsjrdTDsaNcehgG|!U zWRKC|rW>lDrlXb$YRVqjZ2*})9EO&mjlgOEb*M_k4 zc*azK8eQbMb#})$fTHB0k~r{Ru&J0aN?slnqL{MO@9(@K^g1uNnY#kZH+Apbq8{p5 zwZJoPM~qwhgGGn%BkpnI&1k;V>|d-tBC!wy(Ltare9Zjf=2kH-K%fmz&yI6DrGl9c zgBMYhz;4Y#)3gc=U=fHP*?jCPJ1M@s|7r~J4nSs*$OIN9tEb7p#2g0vzy8f)Qu)rp zTdp=NQh}L4E;;nn)RV`esHdPhMtklRgG`oTE;kOFtR(clQ1WGYd$ncUqqW^4*0bKC zP2fVu$dHs^@!?24}TQV8!md)x4ssL0o(!t?s zYX0_z4}i6hE>B%q{|qRL$gr`f$Y~VV5XWg!If-r!I6nATlbg3~Tc72-!AWm~@FXP6 z=;o9!>c0|1ktRhvK~^XLaSH}V6BP&V)WpTrhOwFWLex@{Pl#FpwPc|^riY9j?WZ9% zL4aLZ4}qo8NkO4JutPENYi#D`G{_xhYEKNx+VXFfSRIum-EHmK@>+r3N1je*g@H<~ zTD8m}eVQEU*q!3Ts%!L~Kl%Vuk1H2WD0alqr`!v`$HB6dhx(M*d`(-bli*h1~{fD=ul^iV+SvidCmfPapKsfj}2MC zA(!3ULs!4*h&*#S>w4*G$A{~pngPW#VDF4P+a1GFmwaQ7c*^E=ph#ldL8_Qw&5aI4>19%)`E&}^Z8_>mTl zW#&BBDzrxMbPIoyzpq)fVbiC#rudee|M~~8JJJB^q0hN+JLW9O%K+RGl!kdn$RFTj zz1JCamC(4&p@zbytzt}Eppf#vz0jcaD#wip(Hv=^Q-cFpEqeVW>x+{<@8h_396B^G zZJntxCuZnSRXlhrQb27cnwZ>(IcqAKUl0q0ic3P4#u+SQ-B3rB;^N}5O*h@$K7MUF zcmUyEWI&`711ZKg$=1HNCvQft7EpyjP_GP&Z_941C;N59{;Z`K`k^tOaH zzh9?E7q5@KXK^caJ53piBCBV45R_1(Zuo_i!lohCx(@%C-o&Tucb&)w(Rp*Dan zF&HDjH{XA~tnCQlI+C9OCVpt*8Ji>^mjB|Wjp>bHGf=TtmQ7yD?UT6lI?&@+JbFuz6Rptj1q)v9$(X<-thFrt zL0+>V>*6g&uunYHG_o$ytrw78H6w9(IGj^(jD)0F#9BWf(xvLxJpXTdtNeV|yA7U| zte@X0X!_@ZYH^`~r@)n7!C&1L-;v3!JX#9SmSepZ!>qwnG}Y`D%0gR2Ig*2J)ZeH(4+JQ)f*Vuq6RKtx{}Be|3)k`O+T#hi)*Swq{i4i zw>3>&;%oUw4_(%|_J+wDli(q-%y=NYl&O&Z4)xk#`VNHIJtf7b)M@u2;ea%sqyg9^ zYWEc}4+9z?|J!4;@hUPN3-V$oxj1B6g%1vzhHysIPwQ~Sm<(ZcBWP>Ux9HBR-*Wv( zN}l8t%JUgf6UAh0zn-+T$4fEcsrArmo$F4Wc4ta?NMz5NNIvCb~}O{p>^Tgg;wlW=a8wi>}>UhK){`KisBjAgcaaO7L=Roc0vf!({; z0nb%h2h~DoM=-Y!`U2y`u}Z`*-hgkEK&@Y)+ys}k@k&mkKBC4_jELYZBEn|}+JDA zp`oExE^Ir~z!}Fkp&CO99Evd%v>m#(Ej@*?Y~B+v6%@C?$)}ctrmZdgpdB)(rzs2Q z>i|CFYSdChqxrTwulb+C`JnY!acu`5*M(XSayBWW&x-|1lcl*Z$%a2* z#ZjINH!_MlHRP*Ymx3(Q@we1$BHLEQ4UXdYvLy+OiqKFtBF(Y3eqa(^IZmp@sa?b| z$9yb7zAh(afMQm-A7}Br!+#^a(T{_p>Jvoi_#%wRO5&uLAH+>fR!Y$gpCc{Ww5i9K z(<<7kQ~PC&TXhS#Fwe=UEl~y^3YR%K&C+OCyY{YbZi|U?+8*F{{eHCmvQz*eIa42; z&6eptmIw&8to)3C5ZyGAqYiRr%@uKWD^wXNBui1;FDdwN4*cDdqzcqmgM;ne8C#@0 z8jn|*JZaJyY9>X-YQwkvbf{Y(b3VH;@*td%=$+(P>NJJg3LIjJ%z#GP|6-C3GBC>7yUSm3wq#VVK~~g z>#eWSDW4zJK}8Xhv?={K(fG2zM>5Oca?v>%7%ySgm8;dSR+5O-l6iH9lu3OW(V7UB^z6LZ$v8xUJ(b)EE0FUMy9V@pquOHQ;F8G2=A6xJhj^6&qu(|dPN;;F@Ch{7=tXG%TsZ_;7amEZgp8U z!P8}%LS9G1_PVY^f4rR_puFQzu2EqdIj*ZyfHgeY4%(nt6h3Fp!L$KJ-5EWRmC(bU z3;sJPX}tuGYAz3cO4P%4ifbmqy`b-B&0{4OqEnwfA0t0b!+B3sXAA=GRw^oN2zpZM z_1?lIv1{*NvGmuT$M&fHjBEIje&|0XL!z(^c!q98Cm~RvyI4g3U^OGm%(S~AIZJ8e#Z+Ko{BLDzawSM1opAk}YHhM#Iu3&7H zPb|8kymZg*O}}5~?m3HkR?9fxsMx7FV_frEeoOAzN$?8-4@9>+67?NU68J*ed@Kt~ zt4qA0qW>HjDiQ#&tJo)V{t)(_x#Q91AS-m?Ba!dZ@3)n0r99JSH>YUqg#Z-4y+=_4 zjN@>$VjMpb9c`w8*KwD6?Z~|yY6pF&`|`E@`ncSh=`o*B~|MjwXbM5`;|S2GarwDWTjVF+FX0IgKT80*Hv$1hfaC{`s#%b`2msN*3ac zWS0Q;F{zD2#SS`<_~3ty>ro=*7~+Fwz!4=ndLA~7j2&i6gxeTBdimtxiNtLDTPka;*RnhcvJ;Ou1##qM?iL6UZ{4aX_>I5S zgjwh96q&4+mQK9gCv}C{M79TMl}=RZ( zC(E8&gyDv#i618a;5xrfBo40W4i#`hyptn?B&ywcXVUO*8@|dJsh{;gvd-e&c&-gg-b#>jVFhn z-{Sq33TT{IPl+Z%5y5-vR4@ExA>e~oP{#wb_S4eRdac7#N$}b*C#MJioDfbY0$165 zw9ML+9(X73mN5OgQ}vU*&^mx8$j&~KlP6BxL@!#Ro&B0YiFGC2bZQFG!OMWPU&(SR zv?j4KD2_BTa@*wgW;|D}L88P#zP;JV{tzDfr2VDk0p_4dmC0kGGQ-fhBg(Xocjtcn z8doY?Mg8VyKbjeFlt#k6dk(pu$I&S;JYi0GMTr$E5ufY(mCqZlu{XU4LJ7Qs)^*F& zlSOW}1_7b`39w(&#|+PFoLm~VCg~J(YUD+Re>5K^@EJ>>)0c~~HJK^uFKE^H+VQNF1|vtp z;#YF{kouiK7V4#n%|!mT3j}|fuDvfoGprodXRdaBE<0qGiE$HrB|LNcq)200nV+g zY*GS|go(ADplGkdo~iT;hyb<#z!-jj5a->ZQ}IbF9BqU=4cnN^;wiZIB{LRXs`1VF zwqvmYq6~qAZg)`{ZBDR6!dX0DIRZr_`NN0{J!UYQ)9=jMH=O|4x+}tGX_!o*@TD5> z*Z6btO0Fyg{^3L%rBg)YTSxv;<$ls8sR$x$e4Tm?7%*ArMe%uh?A$4Uzk!uS1~v^* zd!Hb;Z;s%4UA%D8zWL=ltxgUG+8J2B(n|K!8k_(S`x6%c()*YWK|8sS9lYQ5%F- zYWeM(Kn|zAot-Aek@P3^r&Q6GFF)hg)a!Cj!c!(s-fgbSXM?0n@Z0^{^!7|TE=i~` z+zLXGuC?%DYCV|Pm)v&021Dc!@Ebh@K3x*<>jcIn`c&qn++T8?5=m*5T`pg~JYbhg zW@cqjE;f?PKgSz6J!UpJ@HHNfk>yG;fr_38P+LL}DQ{rNAw?i>ay`*+AG~s<6-OuF zv;9op_CISKyX%x8Y<{($>{7!7cuXP{1I{0`>Zjt46!6$&XGCO*PQ7u`{nSfXndBZa zz7Gj9uxn!8wcVI7=uwxmB8LV)%5FAd3@$ZkKD%^TVBBu8hF`pJp){Xq*cR`%Cw>8* zxiUoI*=LuJW>t$!(lvMqk_0O!KR%K1rHjQ?i29k>PJjRRcB!W^yweK5qD2Y%rfE8z z%!W(WPb;&`lkynE@n&0zIlcGfZp|pn52>4vpi_ zpaFLVKbCaReW@gviS?0cIL1a%iYo<*IppM~eOW5MvbKxV@chAM{ zFCtS?>VnhhLJ3NqgQ|MKQl}t7i`a(Ns!jcx0MVix>UH@VxGh7dN%9S>96Vs7_rLqW zAED2hD~n>PN0$3$FHyHtuK(p@ok5E56UAbtE5>9NEo{s32Id6N8&gZmt}^GyC>Xf3 z?Vr=9zfCy?fZ(3&d+~TF^-Cp`C(!|J`HW7rp)7RG*%a)(THIOld_+7f@ z-uGdfW$ex_OqmR5ltn2NG%8t?IzQ7KLoz13`5^L3M16oZYwo=!JpvvF^TyoHcdS5! zB<4&&4B%kna1XjU%pypW0NrzPHf8rjRVPaXpDB}z(gIU+LY#NxEKD7oI`ikCx(rxh zL-aJfsZxq8Cje#Jk82{dxn}?IU^-)Abx0Vu);Q@s%V&mx*f8+%jo^h_#A2l?tmj6; zKb`}_S$6Lo8Kyv9xFA&Ab0Rp&`rAWZi2n7DH=3w z!WX8(q(O6(y7lZ?c=hP8FXiC3{s}Wh`hxa=Be)FB@CFf<9_R^N-5Xf|`6aeL6}@0- zc^9mCGFZd5i$;?z7h9#pkjYqA`STby3UCvjnIG2#nD=V;LmE)g50jItDFQj)TnR6@ zGNCe=`_0iw$JdOjx-HPvf~2Tm=wLVs_JU|kh`$C8O0DaJkc1VGi-$HHi0F>A)f!y2 zU@fc*!(vIYqT`3++@4Z5Ft#T}^a&d~W{k8nC_6Pjlx7QQe|~$|`Kq zSno~i^G5A9y50>b+VOPG#d;I(S-MQx$zw7PA}1vUFzNT9KfW{-IqC4zQxhGwXxC}j zFoyWe&qS8CV-J`~v_nx3LUYs_Y^I+%<6OFmV2+21!%75uIRaA01kGk?XXaL8@UDC9 zMZ=e`4-a8NlwCMCjI5p;{1lT`2I21H3GSe-UXxL9z}091Y-z}2^HUg{Xi%iGdAvtnq|GE@W0xuZX!d41 zz0-_R1RK<=SNQhmW*$oFanKCtLgr(&F=_|+=`Z(UJw07EQ@3&2l9K_^aCvsC*Uxlk1Vmg?N*wPMU_TyPh&roqmAr#Qy4J7 zeBsTb2XlM;(T4w%asO1U*+gUoKmsUU3Yd(s=rIBtSN3j#4i66sFFYmfmPE3FCB*sk z2aF3HWgBHZ3wl^m=WxT`$r9o?$vjtp+q6;aB30*;+xsx|q7nSP3yJ7ObBcPLj-ksb zhXEqaU|CHLP&b6496NzEW%IrL=ke>1z%nexKq(DC9Wb)BeSNlwkUZFlMHB_-{s7T3 zva^5R6MkfSLM>tvnOYG2VsCGMmsZ(Y2FDD1f3PK4>yvvn5(?5-)Jn>pvxT2fK#(AY zvr3I|m=x_WGa-B|aD;GK2t)+byfNPc5pVmk-w04J*>9BZ1t%}Es!37=*^0W+kp~&O z;Eq5O;%kvFMB|Tjf;ScXF*~3m{GB7ee$ARvhrWLwkgo;+&dEy(!7-$dKv<8AjzZS~ z$=)LPiFO5}-dGdVtw=ITSzyCir9f$6VK;c$im*Z&J)0dkuq{I?CXqWqP;eOLUJY|@ zr4Wan}w>mLgdjYC|Q{`HN(>YHp;>6wSQE(cauwJ(XTOb=w3jGeB!2Ej7+w3F~} zvXYZz^b$oW+4O*(`juOljC$$`(t<$sw%yW^SaNuwEJBas>@EOm1$QANEb^9UQi&}P z>NTm9Tbt0!nP8B?z>rJ{AfY%JbU^8E!v{?XEtw3zSPX^22oii$03P68O!hpDDU)o< zLYx($E5h+E=S${1qj(*0q6n7{PyyJseC6>YyFYBBYz!(X3j5k4DAH^N@gk0f*Y)8oII6MQK zxbrVnJV?4bf=$rnq8rr!7>EWN+LF@iud0{qeyE8SXeoPWC0xwm$QLB~Rb&N$&48{H z+B-BGm7`Ce)#QNhB3c8aHQ6^4Rt?XbxEP{C6n%di9&wKl!=n*OyT|95r2taUYdD4I z>gp1E5-oG%2~Z&W`q7Rav}bfps%c<}Q5|G{C87W~C%U-b^gV=%P5D92v~WPf)Q3}m zAP^EO0(|Y%egT%)2a~EeDz)Q17|tOXCBP`k9l>ByOgxKg9gJmcx|PvYV@e}CgZ;Pj z^!Akn;K4Gr;Hp`cuLrxM{ZpJ$`JGQ?U=Q_lcH4}m9&{toc#aTEo^%(@PNzS}fnxER z8(^W_5eP0K>^V;Q#zTX#q%|p&IRhqW%3`nqXo~xpc$WWJN!W;r8T(3# zeiTjJ?<0B|PE`C>&ny3pjWn*{!1hasngPY*ugr3_or~x-CPqfl+JjwW z-6(WU0Y_6}?OkzU0YUJb$(icD14InXQ#pRW6usxTHlRqIE2=-d2*eE4|2ZA<|FK*t z&_lcJ{>c$|GaQs|a_qV)4BZ-5jG!l!l#-Gg!Tmkk7r;Y7DJ3PFw`nJ5lfUC_<`OdC z`~!-9em;Nn8nq{AMloEw6fO?$1@WK*t+H;|a1Q`db8D+HiQGgM0Vn8aDj*(T@(%x& zo}yI=9KmVXk7Q2>T2eeuLpO!@<3b3mun6EObbFIXEQ)Wz{a8PBKkrk3BiZNzUgFM1 z9&7sxHt}^386cZFC}{F~TL}?*lQ0{IF5mdZ`~@u0yX$AdmZxYpkR!osbfV*|%LxJ6{r+BiY@11V;~X7$k+` z-WRUlq8lJ_%1#jN5~Uk4IpI(eH30OIB&q$ZHe`fE%RnqXq<|6;G@+@bay>VJpatrW zlRVMk$>J9dB`6XbdB z;}u#2Xn;|Wb!&}n*!Q$?(!i@VH_iMtq%TM5KGPYn^X z|M^Fx1wnskQ2>!*ou2CaC<-N+7@@2M^^>9S?VB_{bz*G;`V4EEpPv!N+6B(ca@-#z z*%4R1n(HHJe6S=!)!KU8{i5P!0*U5hhY}<=|N7^amF&9}{~j0{g-6)?S6mAU>@4YG z-u*A{#XkDB`S0H+5OwiB*5>&h&>8Ow8`6R5jSVVvIV(J5K>mN$CMQ3lVBO=1F2GCbDme2z8=& z4z6nHxXjFg>9JoGQrs}m5xy)-e&DmzW z@LyWr6Dvc>^wvjMyQ~?Ug10aV` z7L-*JY)58bpo3J)^5uEL;}Hf2Pk+tE4XGsj7E!iG->uC61O~|APGMoE`BO2yeT>Ut zwOxyzfn-?qIXbIYL7WnJ`Z-*(O!BtgwU{|9<^Qnx3lcWin0QOG; zoKi+eXz&4I9IaY7pcDBt=&wD+NNhl{yhp42VIOJqO~y^?I}uet$BGF^` zxKcci&_@ViqcYT#N z@%TR**S?h2|AJA?n0E2*$G@pj*p1J_ggBC5m!KtbMJTRKK#~#hTQ8(xGkV3f0tRT< z$^nTfzv>}ZI~i~>QeT)a7X>7TWJ(S!I)(%lE07^U?eYB96=!+iDSUEZcXFRTg?c&T z^nTppv4REdaPVNMF|ODC`y~JMcfv<<%i4{A3VxTSq~x|yvXhJ>q6^{a%VAQ%ey_hfK`WP2=3@)w$+Jn|-*UsQ7kvp4I}?BGTtI7>NajMY_lWPlCKw?}erCYBftJvI%zkZB>l+$cL4ZMKsr1LtiX!O( z`n1mYQ-?C1{MfAKaSHGbCb=MuIcf|+Sc8ox(Q;NBVKp#jI^aPbyc!~4h3qCw0dcp= zm+fA3d(ZvKaqh7@GVt+d@vbe`U+XGu-@{8o6L_EIzTsc)&W@EIC7hph7QL&r`CRl4 z3*5FjkB*IB=c;-cQWUzb#9wgVZsnzihBtj>KXh$s-bc#i!0uz!d=rJIw;O!-tUvf= zH#Ff6_1|232L{e8l+z4x?{xFUWlCtsUi%d*Rv^%XT|B1l?u%V)dlAZy*3mMPcbpko zE;0>*m;LzpU`TyqBNHy$>Q}E?D)6kawn6^WDor({aMInlaUwnAFmjOsK^tTA) zU}9%~sjgCf0-LcmK#~awPWmk9S+h>Lk$7@dH+Uc=ucNi(gnxq10LjhgRl&l$e*EAumLTti!KEVgk? z(0D-oNoFM)K3~TF{D%n=3HD(A1daq!HU!v-SXX^#CnsiBL==KXj1wEmhtG`6*Pxn) zw>daFR{?&*b12#0$|Sw{#Rzr<31xx)+hx|+MZ;%`meu>{H<4G zc2_$s{r68mzN>obP!xN=)#tr5d>>rTvi+@o&;Qxum`0v=t*z_H7P4Kx=G3s&=;M2! z7p1^gvamRGV$LIJ=r1ZwO--VuBP-uE%O2slzSn5;J#b1hebnCYp zEY!{8aX?hk6e_=C2Zj>{XXkG<2igTOQ%>Sa?#fF{^e#JRP&aXTuRlBMt;;E~-n+!q zE3SuNuijslk8iDWaB%1sj17Xqp^kliZZ2kQN2b0{<4 z2z_A1-W|MfVcEC10S&@Ap4PUOjS4ytB7>yybKIiP2D}hkRlM=Eyp33k25>7Af9>4} zsXk3;OG)Sn+DV_ah;Vi^Z7KD=YD(&{v6Xur9_2!=k=0I(|s;?tOZ-c4%}ocwxb#D-XaGUD@j? z=?Ym{*}qd$56Z+>SGN1AnwRwmtzNylbECK)<`dYMh#oER7>Tv|%({@HrLDbZYt)Dr zCl^=3yaTkY{!St5SQpOmAJ>KN#vn2rU8wc7ITqknq!QrbQ32ly%>#gp)TaheG7=35 z=`L_=h>bG+;X}D<7fUu))?(BLEioP*9@sS;t}}@xgE%9zMtbytjiq36hTWI2MInL} ztZ`+57c~_INf_1|j%b*w5c30+|3r3wB{nwJ1ie;n!W<_uxFXeuy+8L2i`R_ z#+RoAurq5t#&p<1iTMkgm%s~S$y3SH^@cJWK;tadH9@}9Cg;x|Nl5tfyf8!AN+c8D zy-uENiEapO@95yicna8>7-iI_Nbf^ zkDB7#a^ScHF2LG_w{4Rdp;jJ$X_|7DkYbOi+Tc5Q0Zf zPfrvVg}Tl5hy%UD!+~(j>Z^*}EbHp>d}AMfuB;~*m6r3&m-oWL!dm?)Jr2p`$GXcd z${fdL;$n8BKSnzy9YB`~U`rF)B)jXc%4_TDOhdamJCn~xW2#9l*#r994)L^3=c14Zj8k+iTaR*Ri-9)U#8;cAiLje=0T^b&?}zC z&A~&Ig}}x}w*`lW;<^+X{w)6&pvyiIC6J!ptZQau3oUu=;E0N>t- zB8UGf04SJhjc@`#jHw>RiY3UVh+CJbE$iviD)8Po%zLVss;?W`*o5O{>%;g~poJ0! z9Io%fhjJsc5Bt!-XRma4`uH)j?TkxT?TiePTA0IZxd#qu+;=jxOg>|uK56#@YO-Df zfdJjb47!WkUoEFA`x|Z$)SPb;-3~lvW@dQVd{8d&m6kT@%Vo1=`*w5k%nZ+-mE-$% zL_iIHuhlsln_XvS=J4dn>Lpmm355kf6n^Z;Za1hy$gl*Z8d`a%DObVZx>`J*9nw7H z23A~wzbz!}VLc3xOjgCQ1g}8S# zdywLV%{{>x#)(AXKyi+a7M)=w)!a- z1zhUJkgSSI#Lj+KKS2&;NN}y&r9UqLRdc0B%)Ebm*R;U$9&L>;qw$Gto}Jdto7T`8 zc+H(b7%zdj3tZ5zpduGF^9F?*$iAWz^k8NsP7mPCPa+qqlJ?YJ#I0s;xcp>H(c zfB@FQsd{-)85tQ%MZK)-Y$eQWqpoM6sl%TJ=(pI&Od?X5zXb3N*}1K`fEOA4*QICA z<^!J&OuFFrhZeE90(lYG>np1L@#qfxS*>|U4gfotnVtP!Flk7p@Z*3>tmDyj81K@v zjtTl6v$%DW3Xp3~CE7yHGvZ>9zfBvQ5|0T_WMcpO2>B>FLXfc4vG7ylrw~-<;h24 zlLy?(wJ>AZ0Uv{^IrF^&Hv=%&+ysSvz>eOdV{@C|Ox4@p?*nB_89vt8Af|dzIv*4= zAV4GMITG+8@5b{iEGi0(yM31ArQ#wWB!eiOi7gj^Z-S;m?<_EX`;zKuQkH-afbhFi z@~I4jHkm)dBsMhEqB$|niaz;P0ntGi0y;PCi?1muS#sgR1zzFx)0ydNx5X6Ds(!GVF%_EmCzDBJb7gHcEEDt9798{M+oDBclhXE%UD z0{sS~EXTCoXUArg6THXS#?bSB^@=0xD!H^Zg|R^xFb3xtWz~H++W?NV#IW~=1T}>1 zU6XV?^#obE1wK;S4wo3oz~tuU<~(Dj16Y7~cj0tOM96rv6Z&?fL-;}aic_!ZW{o=t zv3ZB?U7K!t`?k8^=ghtd7X?{)`MvxRXpqQ%@<$Zc)X;h;UR2`t`Lp|E!9<6x!HCi3 zyIH$GE>?;&yvip$oBp08^ZMTRQSolC?e$sK}bi!uK07EG9ne<6cKA>et+$cDa=cG>Ppo+GV)SgfH^5IFIwt?vvZr3==lfn-H(=B3Js;l zhDF(zX7*hGhS2wCk?8NFW?yw#|P_0gZdHzYlK2IQ|3H`|jEr93L$ zH9^=sA3)+x>gwvM+17|dfGbUUMhJ2ovtS|x4zpKyy+bscs{t+y%&Y1BX9%mAV;0&N zz9llNf6@ijg)CHc$^rxiS$jfRdbMp?I2X#V-L(~b$>p!-O$#`KxsGCI7&1sO^IWUj1JF+8=cn*jJEsX0l2I2F+^6;4w58{bD734Zg8I28r2kA=`2JckE!tEE#zrECWe)a6F^mO-~3a^smL0MjG7yu`#;Dt1r`-{=1%D zP2G>@0SgRL-7)kjUA*`!+S=UA%wIdl3~4QJK?Yi0Gn{aX0lveguxsnum1NZ)JmBGv zZuilYi6Zx!@A>TeRT!^Fx3SPH2i9H;i>ZL8mvjp`rv0~8PE*nT*C?z+6BA}Zz4MRn z@kROyY-Cmnw3AAu&O5qNOZZorXu&MfP|@uKVK0#VHJ0$Iz2LkFgLO>M zagwB&>~LC%m_!M;Jy9pqG!x+w054Xc1+f}FMcH|IHjadb{U{nlCa9l|bTL4t6(DkC z(dX(wyhz4}n>SN}og5u0OP8!NilmL1U2JS@a7$=TJx8|zhDPNcvl( zP6!m`v9ORr2myj-U0In)Zq0DFD9Bbi9lap_PMG%<_-e8~&4T~qT zwqe93lDXjZ+wQ#BBu0z8AoO}>pLCP4;Q=a1PP2Um%AiOQFnC8ERax{BfHNF?`&d@C zsEmy52&&BBK=RNfBi60yihgh3z00XY_4&%l^o5Rs9sw1~Ju3~5IWbC73>3#Q>l;F% zA|sjaYHWc#16W zJ381Q*xiT52$S??;Q+iTXj{?z;%nq*4>rXZg&SE}tqM|Jnx#b-qZJtl3WYufRILzc8$ttL|k201mVbn;nm_)YjCLfNAa>AFoII%N$-+ zhEx0c&6~@3HS`PNw?(K)CHibS>A-=hIkdp()LwW#$VjR@;I5HZP>{`TC@m>z8~ti3 zCo|J9)ZN`(?&XIHx`uae-dsykH3&h#0-}k821!Ac-u~$MbAybmNL2xXAvr-#t;wtg zs0x*SwZtt2=pxcRnG7?v`&BJzyfF3`C08M98$c0f(sBZiic^oLd}Ut&c%ToyRp}VB>ZT|_;aD_w zH9p=tGk*%0A4xcMW2oarmBt-FSODv1yY!%oK+59z-1mPc6CDEiU$7Q3utyyam82Fx z^#S~bO>VDJd_q*zIuaj&f;()Cnu#2}l;l={mzazILxN9ftU%jXx!~z-TLyGaYJ6lw z3B7`+q8i`ovU>BglI*zL%PK4VV|{7FNUK7@cV&W9sW8an3@)Bfgxj*SznXm{k)4(0 zjcE%f6(#WctUKZqaw<`Qo;Rr4VJjNRdJ{6cGVi)3XgPi&Wehn)B?S>gQU6b>V^D76 zjw1FrvIh{OYKndt_ueXYD7o?u`m>(I`vZS$@s+<;tbm3+Bq=0t%P^KCVcwj65`&&= zXzaVRxC7R!l6CPgpc+@;2cGP(%4ig$Uj~dB`YueoWa+!l(3ki@AY}M#$#}ZdyEmWb z6#8>)1jR&(uf-zj*3)j_dtl>bWq$A9lWCyx9a?v~|1P79N&?Jfq9pkmrVXrH%(Ijt zy)p_Y`t$hs_};0x$mrCM>S-;+=MJrdiLEPybkJ+9PQ2P7W(Rl>i(44b^D05W-VLX! zL%Lr68i1z~9eQ35RvbVd+t!ideZ#0YVULsUA0QEp7g-u3 zuFT752wrG*I4N8!^f;(83{_dsNc%j62d=udwkZ?}Zsc>RJ`sC4k=)n6#Z4+oOGhWW z5=cF+Y=KjORxXqG`d-{j#nUwp{cGtGU{L78D$q5xz6E!IR|}KsQDmTkz0k_SFcsMq zADRq_EQpJAEhwWX^{;vsBNH)5Xx5?4#$ClV`RwKrM`?U&PCUafXh*$$D zDcDPCM&6z*tg^S{{uPJkgpt$^+)vRN0|NtshD&7*Y60gw8YkZv%S0bM$n$dn>oj8oG zx*1?=$#ssKJ`28ctR^>0s*Z{Sz|2lr)f%jXuKB9|p;UgovP z#Tp#kFDN8_{4sdRCbeBH^1le!cIvd}I>7UsXQZX3XJ+_;cSF37qt5Q6ziQPg-+%!4 z$=o`J#vki!BOT(2t=hX#deS|sS&Yogs7ZZg3a+?%U}I)hR$_zbALrQE9#}q@KS(0! z;FG{VVm(;M$N@HtFlj^GYEM9r6YAdZ(<*GRDTou5;osJidr&&MuNNv zV`F0>w^$ZC;U-a^cc)jZTGE#K$4O-re@K&m^wJ;Sz54O~J+(5YmA;JP8=NfXhz^%@ z7+V@mB@`98F(5O*rS}XU5x&<3U?IZ48)6pv`pRWAb>h z;WEDCl7(+?D)TQ%Xc!WVWjRGSnp)HBvyHp}G}S&!u`n~2sY^%hPir9%b$WDuz8vwC zsH@=a>$n1`W{$B^A|uj^0?;7kjSC@0OIuqTcGac|@~WY5(AGo=Euxj!XKP%skDm!p zxrs(-X384KpSM4sT?!`Ot=6m|K9s;qb#;J#tC>+0WgP}Xx`&of5u5-Go~@(f2VRL% zKYZ3i(vD$~`lhaymLcFmyv*J;o~K3EqxwObMAZ5yYt(IEL-8btY5U`pU$g=c0n#MH zAY2d)`Tzl5=2gly!L%z^UPd(mk4K6w_7+qdX_`u6qy+*60bX+j-aSzS&f}n~VW!P| z`qW(Nw2hUO&&``G0oqLBpjsbiRf0qXYokIG5rS(A641`tbr}e*Ev>3ru{g=%JIBAf z6u`|44U^Cy`sE|yrmDX=R!H8($4$zPN8qL>T6>E|ev&y2M;V-pjk;uj#91YANpL1+ zS~T9-^5Q((5ui(>ozhg$y4yz5i9Ns_Kk8YJ-FKujND8SwQ6)-8IIwT+p9Ti}fXJx@ zJ9L*jLM%!uvD^GSj*<)3&wO;5I}SK1A}WexK6g!@-GJrM`w=Vx*djjvBK{N%JUFz8 zC$=Kbod%!-G#VC?%o3ZGco~bqIa1gff(d34_p;~kRaac;pcv~s>{r14WCxzUSr z?4v0cYJVe=feU_&P2n^~7Z~uD2*yQVBSraA~q*?4-~Pm}`*? zZq%%omyOR4%|m!g$X7r;Xn`{`GkZmIVh;01kktXQ99>y4y+RXxjkHsFwQo-x*f zgat|(5D&mAUbnVNKY9o10hhJX)+l_HqP8d{e!RZ)ep=dI^e{qbNo&=mO@(nr@cvg{ z0jg^sS+|jCr5^;iDi|UG@7rNn4HSbT4~ba#kySAmMhNGpXcRlow=hMd6%FAGvG zh4euSsSMQ}pI1;fX#4y5*%3kqr2{;&c?H;~ETq3icKqT^&H&|v>3uJmhQ?90P-a_( z+eDA<1Bgka!33)N5uJkPf!Pc({_Fsf-Ks9(PC0do<@%v}<8a6mE(iZQ^R?}OdvkIm z{*gq`;=cg&9bVx7g?g^nsR?&DYZ=zPfi zKjNam$4Eo6F9^c-D01ik{Lcv2lz1=q39A{2;01?QrT!cZrRG}aem&dU;I!rsxY$=! z;>5Z?zjyWE>o@zEuVD^e66yy;DaMNM#b6QWt#m300K(N%9a^ty|u`< z-6y!#czUD-sBmKi!3=;&<7&8ER%iBvBb zu;f(oN6-NC>e4Gv<=S3<^#L{_%m!yF&})+27!qphtY5$cL}ELc4rv;nz2Z!Ge4c6l!Y4YBH?mNPP6@ zs7*UM#*~4X%Iie*iab*2DqD5U(Aa^3R1I#}@#P%ODfB#f;G3;te@X(Q!lj7ER#x2} zgBu*JSqNedip)N%Av}e!WY7SGM04?xWM1#tSr@X~fyI4cRt&YY7sm-ytSgQY5!e>k zpxTyNJ`?t1@$@1W0pw@o)l@smn2Sj%^U0Gz0XA^~*a_0Je0<7syG{*F%^~=`1mp|Z zQ%!8sNC0pfg*pw!RI+LpV>r&PUq(j8LCHt?P-tVDgLT~^mcCkozmmNDcyqFh(V(NN ze0Z?XnpOr%A|Py~S1|gMH3H~n`_L;hBP);;+~JWCX#jIe^gl|6{@Dr#spQQ44?wm2_OkEH$@n&_x5fPuZQQdn<6XfBm@e+>x?Ogc zI#&Pu;#M$>IZYKH#j-B`@+zDgjE$2rhfP)q3LL0NMTmC|keM;=K9%!YA8_;Gp!eau zEO=X7OhrOLp;6>OO;V}#6GB@L)p#)upJNvKP!rTW+Y1*4QEjogi5iFj2{0FFU0j`0 z`1AW{dYc&lI&cQ8{!2t>*+qne_8vQSwQ(w*26QVDmfJ!{M@N~sx$BKo`m6dU0ddok z6LrKihs%TwWkq^Ri-jbFE2b5A$^RL9Wd#K-bk_Pk!&Y#svB5!V;APg=Vw6$!VBa#T zfXbbhkw*4qbsnh|U?4Q>K-3{l!9DTEF4Q(MS4v5pWFdRU(H??GO_=w}M@m;(TU*wdM8eOTg+r-p+A$ za;kt!NA}r`YgPz#fl&j08bZ$JD58nwfVeQ~=gaOzNJO9nj}YJuI6K6u0W zV1O@*M&Wz@Y)elhA*cZE_eGgRk!wWKhH#j(yNw_-!+( zUmyUK1n=HBtOs>Wd5nbl>*i)t+c_{5ndqOk9A~D84QXTwEangY`M3EdrJecXdo(f2 zCHcY_mGX>?hG?dbn_$Nae(AobQwH96gyybGE*%pr2Vjr|dJ)?9;rE!)2~}<2{z>ZS zt)f-Smx83^g3Fl+L%ic8Y}WknRADreuvByytizNo2b9c73G<%<_5y1`J-o93|`dT@ob79if$H&Y!18nL$AF zNUIRmLH_565_mL}gPmQrZg37=F_KI_R-zzqnoW72Kh`!8QIs$^ftv)VfsomQBlU2= z8|1VgLI*-aS#VHpFHRw;Emz1R1*$$+v> zz+q|~atELUT|+I}=d(C*P>Z|E%LFkRFQqHmg^qLcv>2cnWP~8)lN#y?Ao*AeZ{9Pp z1+5$Yc6{k2%cw9@6HhUi+x;*R!a4`L?oMeH~#s^$Mf>KxZb*ALoe`Qp@CG<9nO6U}n+7$6G!$krNv`yFS{P|D}uKc2+SAFUZKf&cmZQl><_Y@LStoEQk*s`3x-q!H6kw zudrr`7@%(AiS?5cjn}quYE6(bMIroi)Y!Z_0*TS_?_jv<2+#p{ zqKqYca&~rm&_z-xeiD^-{W>q`X0*T3ctwA!gOy-&7C|~V4Q_r-ADI}{%8u1h*RLZQ z14Jx`)KF0G;9w>Dt{=}OHljYf7jiXznPS-`VK{tWW8zn=uttURSX|K9e#l@ z@eR1+aqZ-mD5ZiGAQcs3k!*DcYZH16f(64x=yN1*i6LMs?CV6>J7-92To6L_$&%$s zbR|q}cBmR;AuTH_4R^SC`4~+f!2Bt~(85Nq57dnw+L3Ms0YUhlkh|cmDeoe30>sAs z`++l2l*v^yX#_w0_%%Maf8>XvbbH_JSE56Zu}LWI$LX*iF9iLK$ks4uo4O{`C*##1 zObY-KyvV@vMI10IcQ1fW1(#uZG(m{luVw+xHwpc*umELf^$~GS2gt!d)87!$r$n)# z{iY7TDJ3IA!fqk)E(6QV#KpC9Q+*qf?BBy8m#&5Rma8geka&NvRZ6q&&6_nq32rkb z$eS);4kBI)_`Y)9qkBciJU|-W78s|By_dPow*!=SnH#v7Qf{7>1q>g7HmpE~QM6m+ zckNsgb*Q31>-MY2HLaL*&EO(V3UrpmDg)#=)vGK1g#+ETbkNN9lUrX z|GyP9*6ozpA8jwA@q7F&H{Ax02Fia);;7y!@iy?}j)rxgy`t;vRF3PayKVZi{d(e{ zk;IPJldQ*5C+}8zj6G`C`gPl{!RZB0p#%4oIBKfh&EK`({*0_(j{4#tRQqV}cudv( zt>={8O&GqrHyh^qyf}B@vdqkams}OwwU2-6zUuwd^=z`;JYi4bvElaweL+zsQF~}I z$JZ=}QYRC9$ih^@FH@;EY3LLpHtZ)GWN4NzO&TJ?Yob8L=aFv4@>_un`X%H>FzgkC z>}61#r6LN+Sun-GnU~;HBn^~CL5V~LAw2hjw*X>LX?AT2&Z=AcTfv}6?v%(1fpQ$}1G)2tv0+O<>|#z-q4b%^L3 zF{R*cuNQ^Pi{T=@=ov}i5Jms5tmVIp0UtGDDEtm=d@sPKl$@E;khb*u=Y*ja z#9!eKKv#jU*2(lNOn4JBGyc#bg9t#Nw%LboAN#8Gz-${@S=Am9{}2OF3vMbCO<{5I zYq6%uj6y;}K#h5U)}zJ1^LWzzWCiQF1J|+q1yurRqwG+R`2s|rWd3;*^Wok!t^!SU z@P?Dd;~C5wB1dsifLAgjgN?>R!06w8QI zbI|z|IRlp4KympVBGD{%Ey(s^I&_F`EXRhLV#wAgf?Vhq9j-^%fwYDkDR!NMqa#`M zv)Q8uNj1h;0-yc^lj=gi1OQ{b zUvzA&!&SD_w6roHO7NrIeTt;q)kN=}kdQFRyg}t8^b|N;e`jaE>9xnS0KXDpnLQMH zs@sovqSEJG_{5IXG8Mh5?rv_}Ll}RT|DdN`!pO!J1f+_H#VF!d=uOrINbf=2MO7@Y zJ9B0e#$}5z#VuAFp~2@CN6YdMVQq&&@ZoOX9RJDx?S1uj0=)v z3TcIQ%C0Bi*)#dyny&tSiBAo>VSrcvvKk{GxPYlJelo2gIdq<4umz%R)4>Kc!YJbb z_d`bqit0wUoCwwM=g-90K;l-B0gB-UuM4YG+u)!X64G*v(b>u5h3`oY$1q%Ewu@jm z3=BhLJjW|___J0rZUSuPrG-3lHB&!gLu6b;G`B{*zjxE%R_hQwoBtjp^Fg^lR=W|& z`b>qu(nWk5FA?$kRn^o4)vNzmy)T!{g`pt)_75FCTpMjO3a|$5Sd52ICfyq24E*}_ z(A^Y<)_i;0%<{GuisAhs{{Efi^E1z_fvJ zm~>GoU-QpK-SWO*tk(46s?fz3Df1T(++k2Kt>-j@(=YOHE!9@~WkQK|GYK6XZV}fV z>72)#*#;&*A8ItZkn|STFUs-X@Sno=2W$WP zJB&)z%SSb=)NkmAH#1PjUX;z#-SBhZSXd7K@gpOtUKqCz!tlSn=2`IH8t%GRdFblc&!HiQ-c#tBG!y+W5d)Y>dEK}%_TvG7cW6Qt z>B@!&6c(Jt{{nZHj@8FP=s%)SQ#G4}K`VgkSS{gZhw}Rmlw^2Ds%WOkrn@k;t=a`Y zgv1?hTa3`id#oX_I>AK<==>!bCI`aOj@rZnH-ljJNdD3g0xFbfP zES~x%NGqH=#1BUptfEG|OE024(Ce*Slo$l`QVqK3Qctf$rErPH6X&e_5&tl*|a8CnVFHx{P-%byVnt*#^6l9dIpW%gY!WV548t6KeRd0hr)oW8b&A9Z$W9m z_$TwygxMNlpXjx2tjo#GEeGK-tVb<&A2$w>2NKN!u@R&13d`n~c!k*z@6OW4I(aompl)6!rXjgJv>J7hYs zLU@rr+OxlGXsQn)Hry!HrrBbrGh|!trJU;9yeprcYGV3v+2Wl-gnD310K?bzeu24# zO@hJW0Wr=%O%1}_(l-CZR-%T+eQx@pt}^5Bc^7WpQ}NB4mQ(F#;BcvqldFTeOyG3e zrS+Oh{mnOKU!KwcuKYTViJQUJA;&+e<@Ix4pRy0lr(#qG20j)1kfnly0O_5brrUUs5aeMnnlI?`y#)+x86dHHLHClUrfkMMKu zJSSUc`Nh=h=do`WS2$Z0&Qz_lU1j@0bc1-86TJl=(-K^XWN#gmAlUc*dh{5Am|*V) zP`@9rXF$wSVD1E>KssK7WijBmzZP3l+iRx)QrwD-1tOIkOcoKtS_QNvKs}ay)8@@& ztqTQ*uBq`(NZ^4`A>_si7GU}$k`V*VZWdDMvF#-bC3&?3kN{+T5#jMOXEm15u`y+$ z(7E`-34m@t0wD{|prApobQzy9xjk@oAe3*k64MER6@`W0R`3B^giLh~>2e<{gtr|@ zJEjbh6@ls6VyFdS*{))0GGD2w09i)bbl*rw$oTdgTT1!_V`~~aIv$+*o2v|s-dJfz6h;8`!QX+i$J+y3@Qs5+C^VyJhLEJw z)%$#tsMcCZ?8#XZR8D#h1gem51&A=Q43h5AhW~SZ9Bb57QV}(;=j<$WWd`D102LrR zV7N~XF|0skpDRHEwh`*U!$Sdw`f6WjDrN|BhUvF2 zg}t&0c%-?DiQ4X8&bLPJs%e>krpxcQc2+W!q`4ikkM79Yr(t#oV;X1S=q@-?1NfCAwXb0T8{8t!Bcc2D2N=)1}+Dx%RU~ zq{tzy0HG~1)>c;iB5c4U`aaVm)r88YN``}-9fSnFkON?p$yaIt*pnoXs-Hm zL0XIp!lrVlG_>24mllKiC$%9`mL=_<+z?t2S5S-yM!lr{F>Y;Sv;<`UutZllD1)eV zQkDN*C~5dig;Aeiro@-}$&-Ki8&x{W;~IlZ249HW*dutib!TNUrO~m4zSC#w@$ZdX zZ#z2duXfc%#s6FbWP<<~Cp2%AyuOzQI!Oh2|1~>hV16#7+0UT6u=ShMqbA<+yKGkS zakus1w71wb;r!z8B`wu9!#|y49|IoQ^Z$XJnqHp1e^yDHP2X8}yVcm(PS4n1)4AXJ zJ{E8_%{gy6j3Bxnlkm7)O}SC9v#CON>V&lyo?Tl3%ruzcKcx7!^* zO8R8e=gr4O6;7kuhddueSdr4tEAxxhoYw-?OexRDZF)5|G%jOpfaWRhYQ;&3o)E)*RA7|lyrQ zE0NivP+DH=VKNuT6C(banH03{6nF>SJQ^&h53yP{K|~WQC5kp_=u4rEld`t484_ex zN)|ygiq1hZ9c$~z?n9--*Yezx!f4%aFn@D@z-T`O7hZyFaY50US{_!Ie}r z;Tf5F5ZfMcB-5kuIW*RPV{E#knMNS*#3XoI!vh0%BVLuTe}ljn$mM0w{{o`|5@KCn ze!7S^7I(((f5#V~pEI#GkX?`80Z<5r2XH&T1?K1F{yfP2IHTwy77<2B{A35;DaxH@ zL7o}{@^+juH&uJ&y=CjxH@cNWQ(4$03E$!CDPvRgahOvGTqDUI+HH@K3t-uXjT?P^ z#tb+Yr879Roy~WRLzgY(4Ed7x$I0|%ftkn7W^nzP0{WaQ``7hDae<*4bFQlZTSLw*(c`Har`R5(>mmSA+ zJ405VVVGUYUG6m#E!D$ic4I@r)^ll}Ra_00b&NhM>7VTFH4D{V)n)X_@N913V;$;s z-`HR4wFEuVT$es3%zI-;v5dx2#a|UGX(Sb__vs%LD$qIcn$E(g-;}nEYQIbUh4`!Q zm+NfvdFio9$8tww``x9dtM?m*tms_I8PLimt*yZucWWP0yaL0KlU&2rAN}q*nIxZC?^R$sv8vPYR7mhNPa1PI$WdFL=E9>Tl;6)_cWcldcG zJ^@NhVhTW7a3=eCBQ?>s85qP zGQPjGSn!wmF44?uyOkq814p@LU8`Gxk(}sT)^}?uN<>s#{{F_6#>P)cFqvH7@QZO7h<`Ba<)ghGO^Fd@JjTS?YD zfRxw(m_+Ac-<|UipOm1;8&Z;ClBFSOL(52J7ECs90Zh%|>M2E1jq3O_F23VSU{!`f zk!*F=PlrJSo}>l^#ly+*@G0RkR+sUDfg00Z=yy+_5c-4$La|~LaLfW6LbM4mu`@$R zOwmy6k6{#d!gQKbe{yLt>w(XB8wrk$``;0L9+8{t$UdcMeK^_QZP(IUkuxN6w& zC_=vZXQ(Gz(2vPyHaV`B2@U}@b|b+Lf*DZpV&~nZt7mK@954B_JYLmrsB$)HP0g+*-pk7IGlbL)pcLkLOCFN zg-+=DnKf1ilDZ5>&vi)AH!ousv}b><^Ug1R#^9~&mEQFqXflQtJlm|v^ltyiDBJG2BI;b3uYadJWSO?syuZp$Md`8(C)Z}1?!ICw_oF{tzLkBSm&&{N zf~lfnn}x+Nf*Jg8-gn8#rhL zb$*JK+1$3y7k`cp1-NH_RzKAiIy0k~SD@4X@Tp3?PLbzbt} zK?Du}%g2@c~(<`V=zsJRK~&&ZjD{lE2!{B z^}7!}*_+h*{(JX)-Vax?9kD0Ur*FRC@Gcb^D0oNr?W@DA@^5W}$=o#E_uku6$393Z zOg*RjAyV8sW&5LP-a=-=EAv8b`Gf5Z6_+21`+SpI$oK3B_-xP|96F;Iy|Ur9?xmmQ z?T^;4)A$t;UbyPa@TW1ESuO68qgp>cPci0JJ2b~pl{K6~gTV4LE*QV!@MrbWuF7r2 zOdQXjGQM%OZYcbf(z45P-^dErt3BzJyEFRyS9T;~EdX+ev0?%j>FZ*%JEjbS!k5?tmDHxgeY{E*Qt>S*XS?a+N{Ink8R4Q=hc z?(%`n!rT0`o_DS|J+HG?^Wm3eQ~>VDf^+En3Z~&=1$;tG=*0~UHo_OL(GFmjA)q_r z!~D1FsHid~a*LcC8&E&OcT@ne!-O4xTAMUb;zzP?K72qIJYoz;g@&HqFRCSY-D=+L zM}^m$5U>U#OIrR>_&teNa3t}Q3(y73eO3EzHM}Yk?#f}jmD*(a7#%z7iLEovaS(D5*TDUJcxxd*=&k;LWW9A%)cO1VKZt-R0!p`l zg3{fgqF^DYNOvfLv~&!DfJmvVU=Sh}(jeUoFd{8oLw5{4^!)DE?)!YsclURW|LmS! zVw`zB@8@-2*W-G{TdEJ}H2_ewU0*i>b0{o~$2)kpJdeOQ4Hts2_ukQ+aFc@053-74 zxP*Z3MH8*XYsE|s`Ja0D9w%k)P(kkztZwHN>!iRdfJB1czgMt)WdhSXOyn5EeAln( z0OLAhc7U{g4c~aR`K57iEh0tWuz+sReEIT`I@k#?RyogC>5#e{pk*8l3cqmsW~3E_ zS&IF3B*4CEH44`rxWd9bF)k-*`TT#A!35ljsPnnC*eac%QLy1U=d)Yqz=Q>BOQ=hM z+mv}C7n>lkB>s^?ws&`HYc0py?$LdOc#g92;=&C-e}DPkp-MLqW}MKQ8*_@wM{{#> z5NZn&^*{okaM8$EU`+x$NIjBx-rI5j9h3+B8!$Ic_+-FL^;cY6o+YeE_`DBaMn=J{ z8AvOFjLgix8s!-!Hiyf-@8LHsA<_XaCU}ZZt68rh)<0l1^h9V}y^3@msh2TAJ{n-7 z{|AD*LvTV@joAnAzX$yXl7~>^eRby6f0adVYRGrNy$#+IuzT-#+qMZF0B~iY;MxT_ zidcpCR-J{E`#MjUBS|5QX?jfy{itiG;YU{13F0c3p3A-$W$0H$m6D`b(DE5a_NRr) z_m_Bn)w4XEHl(cd+O3JuQH`9|%B{@vb>|~8oKt8>Qh!r~XQ2~|8!B1M^ zUX{Ul5l8Xj==}U+W_tt1euT0U=8xoe-ML^1J02w)I=!p1vv+WS_ug)C%G1{EX*EQ* zCH2qF`vii17i_`ow-$a*K!AK^cD|KUC|pW{aJ%daICzxw5L ze`0N9Ipyp3{!v#qLGJ0ScOeXB?@zE{ADWeIFBE9B>&ogYI{s~1!5Kd(+8STPFMnFO zA>VbdTTrskbFOdYfVr3=3eVaUE^{{un`@UgS2t%ia%paf)i%r_74DP|L{}Sj@gmPE z(l?(?DRrp-==a>j_*rvakkQAt^kb>*tHinT<5he`++eIYo~=82$&T!|>q1AbwAgs* zmSfLzwER(V#UO4ewh^T|xT6gz={zgLPLeMesfAI7iZ2(=s=4YdS2tKNrBH&XdJx)q# zbq|#t8;wCdyR6ca@oHSQ%HYLEN*|}H!(DHvLLm)qqFP#@8lkYan}=&m6Letl4wc>~ zeRHF#XM(P#=Jqy@z&Kjadl~D;VeI2`4(in?2=~+{W^Kcz?LTjKXEVZc=r)tqwQJ(V z#MO<5qs|yYN>XJ^ujz?;Tt%ZTd{y+m25U5j;HBiJy=pLH9q#r{9iwz%k#<6T|1p2@ z#tre?i9V5Ko(t+3MaNL7f|3-wD;7#?hII z>}-zTO(Ptgc&phGueeGT>BF^n-|fz1*EM>% z96sZZPV7|td$(Zer=$EIZR6Q3Z_o^6N6N*1sQ$iaXvol-G^~`h*Vdd- z|5cM;BjGvm%A-eu9(a{7I{q7w#Wj2z<>vY+cSv3UJ4bAJrkkm1!HLr0IFLI~eP^4E zzx=276A3B?-(xWXRMQbFFUthomeHOX9;y3JM!Shd{*Hmz+7<76DJ<>dD}>I~GnuY+?l5?}j58G>Gq&`|Th&J;x4LJR#|i}e zwvu~0dX?9%JV1RHcC6tyD~T+QSIb#8I?>=qM78VohGp}<^vSgksagR` z_&dB4D{AznY1JRlMr6G%^rl{MTcM7A+skB9Z0CC<65#ij)uVj%i!4kHw}LSxEY;xE7j9kSgG-;x7iX=c9q0W=SoEdGZP zYJ(gCWlPH<&kf*L{-34+3nNI*BFNhXAnX9pLvR|lOrQhYUK9Tc>vM#55JO~*`$N?rKcZBT)oLZ_QPgkR}|M9A)=De(^LX=-5@CIyE+mg2m;KqZZrHZ9v zV#Q`ZVYiX$uXQFENtS{>4!soDoZe}lzXF?XI9wxpwqG&pfohG|iD20o+yuLq{}KBB z*pFWLf1}bY{}+|^_TZMj{3{=VG-vtv#z$*XYC9mqhlC(NGmvkNjEo1O3V74tww@6I zNig7mzW@c}nxB9F%xlmnqXDJVl1{f6zs z#WNRy_X24wa5cOD``R2wIt6()3GB`;0h$`ZYsd~~+Wrz)cJ-}b_um?Q5Zn0eZkjW+ zXX-Y?SPId%)Cs>UZLmy4Krgz@K!^sd>ytmPB|Ql0NQd~wIQYq+Cwc9&LlLaM{-5Nt zz#F?-&WT3f??7aIw?Gbd4|wxHhT}-G>T2)ce~IuCiLic0oE(6wf)u$`W!y?*!4kr; zy#YvCHs7U`nM<4PNy7Z!I0be6q6-TTPxXwZlp6=){j*t@mu+}6Bqa(ABM7*%$Qad1 z)p;LK7#9ci0RP1`*8HwTKd;W$IL2-C;wfCf9?$Ns6nVUT0oB&Be;_>6UnN!2S>|?2 z;`38d*w|zes)x%qQ?q)9l(t!S7bWFI%y85+F}Y@Y*}C?yBYTvfl%_gD&z#}=`jtho zZY(Ta;wv}izxU>YoB^xj*k8X{4?I!i=ot7xXhcQX_P4PvR(oVg7%t-5VC1?oaBALe zlknZeib(p`P*%+#@mf$+>KB~Csq1(BR?!BfrI!8QZeZ`52`^ZuEfXtGy{62}l^(M0 z3h%W0DYv>l_)F!!!o1IcvK>5?H4d+mnI2S4y;P6N_P7Qr#8$x4afQElohE*-|Fw-Z znpvGT)2rn-mdM&m7ND2k>?3BTYr8Y!sO_YwI6vPXchTj*ZAZ-QCmsACSv(*JYEMQR zdwws;Bi*Zf_2OK12=|Vvda|hgTf^v7)k^8FSjTL9L&=Cu%E}Eg)13o}V7!&u?{!w^ z6~#xhb^#ulDRJ|+Kl|$h29hnW#CULZ9>iP;NM{YJ{16?`ZJsEln9G3MZn{xSmS=Bh zd}t)%+NQ+W>94D5TYKnaT3_oR@pv0oE8b1&0FzG`-uR-?!LMGGZhGm7WV6a=(jy5p zgzDAaq2m1_-)not=3N7!b_I;zv{V~HPE7Sdt%IV6vsw0xD7qvpd4F;%U+<{4@lcQ~ zt~d~R7mU_O=M-}K<9ea~m}7sGLbl<_NM>dQn54Opvm1x=uqtg?7Z9+ko$tnKn<%@s zG;C}LT)DcWapSkb;{kfP`GbRA^f0S?^7bPGgU!C5b7yhBqDRm>O|Hip4BEB~_G~b3 z;tF}hS0zEPiYji6V(We$r5x$M_Zs^qLj13|LJ*TD&I@ytkRL(8e*pr9W<4^^Y%MbO z4g5*C@~>Tlbt#>SFT=n76`bhBo0(2u^7%L^k1Cx#cm37N(UVX--qqtVHdG=pRB|s2 zeqg^T!8EH%{Lh~^GK@Ugm6h#GgoK*O?0c?pzZnQ_n}LrQSu$FrS5rj6_485Kq$m3q zn&J;1w76a@GK|>jF&+M_UHOuNLKYKSwse-xXoKjYtNe#(fXWPgnjgba8@E^6*va^w z^yuN6NW=G5OZi_i|pARC%ASc;~I*)9K&Ts6d_~ zl4QS43EYkt<&m*IzATP|4ow|KsUxUzj4>_!44H2@!|r{%u0|ts5sBdH1R8!(BUE6i zR&aDfN@EB0a*t1}l0(83E1$)1kKNvPrPpTyUY$NFwDv4f|Jfm@^l#VsRk4cGT6Ug8 zeMD3~W3d|JOnoWM346C;{Eu$i4qaKP3*EOLlckWAy$wkr)advuC&hVIQfVCSk^g+3 zaybUfz8j=%yKdt1NV%o3U2stBg04+z4U@5kD}XA(bdqfX3Dsp=~=rwmjcqsrwHWy zv7RuPVT#%&(kXf|H3#)=an31Z`=^rWKS*^H2{hvcj*E_Oua>doOVt~9d47uwo zK0XWs1G}gdB<=;1!1xbe-`)(Ic{<_+yaZ(D6`$TA>&E~=@~ zv?dB-I+Z26?1XzVh=y*+V%E2c99U~@i;2R@jkAAT;H-bNf_&0(l?@<8Q5g?UB|ipU0}SXI1aUoxu(2K`}OCwiOy%hU{GP+-s&vGnR>j69jj{7=>=9#rf$=VlG5nx zU+p?8PcZH%2tjkW<^M|i4g_*!eTz^}zji--1ZkSxz5YETIF>srDf__7zp>@U0jqEe z40&szFMLFSmJ0}t9FZQtJ^}a}dR|zBV_-yrO>e#>Sk3yN;`Z$6&hrD(3@|q!c4^Q_ ze3}rLHW0oqpLwegBJ?l_GKJtz`)z*j3aqMuCHp28|H^s|*np6JXMjZvm|A!Ipv;XH zieJI7@~aVSq6j=4%(M*+4X1klN0;!Q;j=!Cut*?@0AS!h^;u?B(6ajcy6Dl{e7fM5 zU)XbnEDDS_U><6=(5<5dDF~eT`IY>7Iyz^hqyXf63w#Vnz~lVS z!D3*L=BYAe0EA?qQn;%|M^A_bX_^}U?Sr40c_=Ys`rtj{H_7LO;r1wnn^~8C={OkD zfPDdI0r0dTHCX_f3tnJkU`T>8@T3ganXlg}D>-0BajS4b+>J=U>pcs-|ym?d9(1^mG@mPr8_AYDXVO?jZM{`N^ z=ZuV#P4Vc^tt}FhW}IonOWKnOi3Jd|Bu~bNbbe#7wO_#4XD4FyDkVId3E3UC(8qE; zK~YwR*zo(tbN7V@7Bxp?<>U=Mm6QMmSa-aZjEtEa=17gZf5eB1S5Hs3unii+#QkG6 z!pz#z{Nf2wQKQE=_P|kd{aF;Oykh zg@-kr_nrj>0m(@(xZ~@9ua8f(kEWrG4aN94b!w^4tY5{ovvt4URO^4iJc-o(DPuCS+IByq3QoY~uw3HaJ zDEwrz&oF7XtY6NccIfwtl0ZuRtTY$jr`to$@Sx{gYo)2kPU=!wkNB&)c7Q6gz~uZb zVGXMt^NJ!|Yk=Qc_f4+_r}Qy4@xmP%6XA;OjfJ#X?XBve8_!>UQ2{mOX~-KE*ju>} zUmv+qKf0F+iueW9^5|@ICFL|v=Z~iy{Ta0>_-|_69W+l_xVR}*Rw$V;#@3;6^VIYp zJ}y)hj#ZeDZcmP6k2WP|h4g8m0~&SQ*7V+KPf`b%`W{u}K3hZKaas`c^|6j&1*OV+ z>Ap)Q#GkafTa5a=3$SYIMxg}O-pK7& zJSpEKbdRoU5clIpel$FFG6^X~f1i$uialq&YU+H_`Pr`?c7J73TLJtv4aO)Mg(z0M0`GQ2!zyI<@!c_4%e>sy z-7XI3jjbw=Qq%~`ik&{;{j%Pt2Amc46gT_q3uezdHo)6ZOzh)TbQO7-XL9g0>ri;x zMJ?opy15tpSz37Fv89&A{$pw9?4i`RJ9fUMWk$Rrt#q$mod(x}`GW=xIK>=Cvg&u% zpcKE+_Bwr>|d6@ZbI` zugB@}BJA~(7q;S`*35X#w3;8KP)w;Z3r%E}@Ep3C-@20w(mHLqRUzEzY> zi8?&o(yt0&3<_ema1^PYz&glzWYB*h4mg6W67~6WD)@EIRrH1>kBr(s5W>@;R@Qu0 zSyTS4RJ~LB>Q0{cZ)480W;t0W-H8}PtC`THUw4yF^+pXc96^C}g$ct`GmN7Nn6q*|$x0&PV$`SZM?vTKj@_5W&Py?4HgD z0~Z9zfOr_?f{zdsDx2{mJDUO{#*)*R8JxW=>9` z_DcJNck`i4!H~dw7E<5$!c6lVhE0Q`uC^aYqd5NOrTC+y#R z?*v?H2sr0hf)5=ieFj8dn0) zaoE#Lq5@`HsG}Nhh+>0;qipeQ_(ov9CIdGM??YL4sGNX$D_-zBBG8d6sMHMu7)r+s zDR=C1XtskIRWL|()GEhS{&=L7F0tkYhJ z=)&IwpgQh2UX84Qb`rdQ7Y;`qZvB|7loIKI0zM?=@sK0%2h=?uYy}n^h(}@Fn`%YN zZpEyQ_d`$+vW+kAh*PC{mMcj~by>H@zEgk~4uQcz=%p4wNKi9St)RMo1p;}1VjG*E ze*uwAxSa$Mb(ole&>0ukP#Tq)x%5+^NUaadP+$~pdjWww9-i_3-uhSXzCcL}qW1za zL$DHVVMocxT>fHscOBtpbm<&C@Ijahkee(f40hW|gXbBnp~s*MYVVP~{fCc{6xn4! zKF#0)*ghm;A$1G43SL3MrqPF6ok)@cYW)CC(77Y`UoE2H8UQk(Is^ne_yevftIEux zB3X!40F3}CWe7KjZl+&IA*ue~hQk3B`0ZPih)M?-yueXL^0SD>1PgUCDBbMkuo8JL z_|&O^)znJ~R~0bgR_p`w+n&kRTds*2)wa44H=EMc-Oc=!8jAGM=?5)__`#8+7})ef zfzfp-nIFK+UY$>$dG(=;f|0RJ1-l0ZfKc70QKOL;4Wa?3vOPb5?8BU4?0uB5e6mMn zci&AuU?cVFF-mczWUX`?pg1s{7qq89t$0<_pqYaaV+n*7_YPNMCRPYlGl*CdqK+Bu zNR{!MV0#1<;Li_d?4ICy5qN}MDZ8XZC)k_)juV* zADxtv`i8S9P@S$~c={KYv6_L0GU{VtG1((7CiCw2)Qx5P8b+4ChgD>mIJS8Mjt<`k z)CT-FpAd_3#4!nP?hea2>OIfe8lw3G>Co$DAHuxZj$k*-j?n4xFFz`I$i<7_Z{Pb3 zo5SpVnNyA=(HWb%evd@{sEAa*sTw9lQEk|*?4XCM(SC&_(>(bPrvnIHl+41xfg#I% ztNHag?zxu2T7H7;{Ad?8&@4$%nbIb|)`Nxs73| z4Udxn)W(N{%FU+ZC&Q~S&k2HwGoVd|a*H+49g)No2$vc9y6p28t*q>tM+N6C`XGAK zY!q)*7#xNx{b*cZDSE}fdV++~DTH^m#rdrP>mj>#T)|kwQ;N|O$yE*0;LUMaZId@A zzsXRm*i4pv@Q5sfgR1xV>0>)@i&Ri2Jda`&pG?Y+xt|l3+dp_B6-TqI_=T8{a_epV zlYVFW=Q{eAy1iz+BYSBx(_jGoH$8smO}$Uf`!g)(i@&#blX65PNX9tN%xVjB;^i$D z_jfGiRPU#^jr33{9$8#nv5@S^R(*<%8T{BG2JfQ?ua!~vVyZ#vNNhH`g1+i-^De3y zR}tbh5p|-}#;PA|mEL{D;wUqs4}F}_DyIuN6Qav7#A22PgWWm_OMw(=|Q=QTeZfgZ(lgXB%r zN-9@)(zWys2YK73-Dl5J(wE}2vAoS4%kdr+|DK%))|T_ADlzA5mf$d zDErkVnae@?-<$Y5bJhJZ&Ecbpl9DQw+m`7#>wSLYCnAiPk(fJ6Y~}kA_hj4K#rh_j zhKvK(XNp)8B59xt_5qBU-C#1f$h4PDEVWs2*~PB(#fDe}*85)09UtMTVVX^*rqZ8% z8Zu~I>H?-d9R`^i*~>x0#1E)fPp=PAFU#l?GIvT{^lplR8Boh_*^vB1QDe?-`WR-4c1 z(0b*vOqHk9G%43b{$eqvVHSGgJf@`yE zn-GdNym@6lJq6x>Gnoby>VB8kB!{C9q9iqcUMu$>fG_L( zdFH*kHKuxij(RK^R`4w#h9}s9MM2UPp#(!ct!@S+qd>=lz*8t4B*!ZK71=p7lJ-rF zjUU%V@*%rQ`=P->#VkuRft9cQ7DUKf(rO70{w{CA?%}Iqy1H~W z2n$@KZqVTX9IISkf@1}~MbZF7Is`h31nLTVR$MNC85?|L6xkJ9tbg|ta&t?@=m>Nx zZb3t+X7HelggXTIVL;5c1Xy<4W8AG`NN+c>f==lC7bQz8u!N zNahj#hDRn22qRb+fuYto5@g7SI8(^TL&z6|QfJSH$!_kRp5Lr!?_G3r%g4#v|C4%8 z1<#W@pa}&TSy&o1B_!WM-WVuV&$H173kYtuq-4zgF(M(u>$<I{NLf`f0v-!|=roxI8a%2CuB5k`^3y~QJwtwqXH!H=+55h-98z;MIZ1suF=<4Y5cs zFRwEa5~qRmT?-WMAg6%!58KP0RS~i{FH%^9*Td=`85#i}zrW;!#sKFBXnu0zt4J{w zLI$ZI;4o}X7J~muq)%wif}$BzztXG)_6Kx z`gY>Lvr+SJ;xi_?J)y}5i-nHm0S7_}N+py4IwtRpiCC{=GJ71NEvH}f)p$rKl8?{`F8{*vH-;fVJj?zSGx?~!j(BmF0aYJ&jua&PdVbB_&1};?P)4pmv zEPV@mFq&+=xlF}3V#ER6>_aOVGUf_k~FXug2P z)`+_9ySspkwtoayAEG)nyTt(7pVuDO^&Ai8lH2Vt%+(bp*TM9(AwRZ!>9*GbKAk&{ zt(@E--QuOU^(4XcraOP07PSM$kN0ffX8ITMg`Ut6=S!loJJw%TPS(=UJ(WtkVF30-Tq*(eXrR^MNXHEn83PMmYcgKY|~pM z0TNsy#(ggWQ4}VNZRpB}&ch&ifk@mDg(j381?kP)d6>Vv{PF5~GB(CA+nEO>Vn`Dq z@7nr^ij6n`efZ5No>;+fHOyLm`I5Ok8vM%s#Nkd`(`qAs<6Bkq#t<*h*=w4^)Ed|t zLJ(XL4hZWFZ_s|_{L$?F9|;Dk#->T9DQKu2xZ*?!HI@F2IfXswrM>MwNMdHLtmRET zQQdu1wWG=S;zj5*8kxt4hxj|R%-=Br^lPiJ$#r83X35_o4t9GzJ8+H)?%n^kdPxNE zCL5glw}i(1%<9}Z;o<=Q?zhx-Pe+%w4i)T174`Z*B-tkC<4_j&4W|)u z2z~z!Mi>?JYhKv(o|zk&i7JgY3gopkJKhmxA7P|@5790B+d#1HLi_a%BvdLJzQpx#s-c4^k zhXsgRr7nM}G-~Rxj(-{L+n?U{WSVc7^1jg(llx2UPw(_4>xbg=3hPBj=MG0w7B7|M z4UYP}(|94?p57BB8-PB=uB ziN}$DzgNFvtl{IJc^_jYn3XjlT31^DS)iUiKw^Jp%LF zdLayM>%v0OLM|mhqI_GBj*oA&zBgY2bZsaKwW)U;Dh9jo)9b#?kj+g{@i!E$gy<5e z5Uj;y;?Oiq&V}C3HRZJe;C=c2S`+O1M$hNJ3xl4n3STC!fNgFt{@~W&xF{#!eBVmn zBr=~72P|t8^1=YRj4A+M0H$hCuvWi_`cuMpVz-u)?grvL95>MAlWj_e=QlPnS^aZU zbPOSxRaI5hxC5*SA?!#)`n~}S(}+zw8w_GS#W1@a4+;8U{p))uR}+f-8X!b>anI%1Udq#*?yKWhE85xsTZ(CQ+*s)T)m$q^1qHNG z;5O48f~Fcc%Dbi4d8{SC&<4(5RjRZvr^sM`~e=(Oer~`mY8QnFfOyG8-fH z4v2=d!S=snqr0eKWOzea;=JI&PZEP7jwT8j8X(Avr>6n*F>y}jR& z{pxwN2WHTzts{!KRa7pn9G9AU?r;Xp7*l%NIY%;1h~53^>s$aWtqZY;{l3abI zuX|l~EIs(qF^k|TqW_d6@vvbHt11%w<%_oH;b+c|Aq6Zwer~f)Szt=V^=pvh?Q7yv zcI`mEuuYdW?p+A4O=r+9pm7+Sc6z_RlSpU4pLM?CG)I)0+s(Kty4B`gYe8JZvx*M| zj(#(5*$tQavUd7bDne-AQ>q(_NHLwU^AB`1hG!vbfyXU^@*5#cC>r_+=p*E?4^!2c;!Iyto%+$D<5V(zFBBvq=@&_+f@T;8F)6g_h(2H zJA#ju)0m?ar|gx>#OQc}bKNQ`-sfWQUqnxiB5H;?OSj&Z+N2RtD`jkGY{dg*Ly2O zNq8Q`;m@_MN|YqW;nb2zxdQaTzdo_U=HoLkK0@@LRrWjF-=6+>IH%S0WUnICTfq!c z237ku@<&s<`rTb}YeDi;%`)b@>6FciKCn-6EO+}xHCWW7URhJMKhCu3?8=&v-7~Bl zc(AugApUcr?hqG;J-z7kN>x{~Uz{uPEMe+zkZ3RuKKV8}jgOLhqy2hbzjc0pl4jv> zg(v<{#>f@F`~dIx`)kMPGtW3Li;}ZZ^))n@3r*mJ30W#ts_bWD&KbGBp~BC2d#%VT zNcu>#wnR$Jl!yIqxAkiRv(NEg8|3AT$>w)=b$7nR^L=pa3p=+|xqE~B&5aWuAC9&| zdd;b$l$10v*ZO)+rQf07yE|-eLV~R|Fq7v}awH|Sn>>RQi#186)0cj)CY^WS9nX}I*ZUc3kJXGpKw4|_m>!Sgwj{*-tQ1@uBF3$y{AD=52i_xG$%0I)Kj z#}q9VojC&KDds@-pIx-(Z%6LsCq&Bw0n;FOb!!soGk!`+;rvf@uEz=%99qho^hxML zZ)hvo$|slLgc~!$p+=s7cXV{D^IE#n?jZ{%E8vrkt_tlXPP_2OH?2Y~>v(F6%DiWp zw~r5!jrW>*qX(8`hxvr-zhZfS8*=K@;>aA(`5k4(-p5Xu_+;1{%cV`QXp7@)+>s|l^!IXSd&SYX1q0-hsmLFjV640$=&R|pw~LEYO| zt>kFHxsSHUj?@7_JQ8VZS5h)Yms>J7%u03;B-Juf@$wLCdseB}f2P3DL8{~$Y!fQ< zWPo#luzt5YVgeBJPHksrr*#=np@1FKJ&Rie8&<ji=N=tXo=EC1z?Z^lhK=BZ10^u_6&uO7+%kY&E z(7iqZ>nu=GM4-GXtg9*pGgjT%Ne6g?I{22z$y&ZOU=NZ%IvnY6wC~A4s(lfEH~egC z?=>=Yc@-7eVD1t>gku?2GU+2KZ(%tJkP_G=#IFqF1DIg7Lwl^5;LiV_3&7;{4clAZ z09r%-*9knDsECXjFxt&NDJ6-Azig!M4B=EGO>2n010MLt%ZEc=Enzb6!MX*0ACb?W zKSQ$#JtJ#F;Z^k`q7^@>0b{!uh@X1FV6j5*%$wcW!ngGjEByD)dU?x(Ij}V`lwH@r zz#Z~5GG)!7C-2HBydR;UqdV9@t9rHD=CxG#ZWj9%nVP|4o31&FX|_bG(h7UlzyA-x z^UqnbXfD<0WygtL7J!yQ>&Z#L4IPcQ-}&o4d`X~gbi|wa3R8=ST#kt6(xl_}0XO-@ z*qF^$wR4GdC_OXG_8`LtJhwe z4)=BL&f(ZTB63<+Q>@HU@e~(jg{5uQd|-KT{h})_MdVmG!v}|6#iIA>X;d9;0#6QA^ERN-$V$yN!x*I_wiq)yuk;82VHw)9y-=>XRS9 z|G=#4;heCmVRQ%#NCeon-sGyQYuk|2MAiSKp_||1yy39t>AHyw$;U-?v@;_%Y-} zaC2daM~j%YxC)L!MI}yaK+)}MvWfu^9HiA_>?vBq#C`->Hq#tV46d5vg)TmfSR1qv zVmmh5W&hEoC?vRqoYz%1@aj@|kjdz&*~qWIJx~|Fgx7d~@tog4D|$btWp)TK@8i8B zZd$zah6<(a(w28AB96+7lg7(W{q7~7D=Xt>YOKel+Nr8X?DS`hS8IAAjw7fD`u?jlTjo5^mzdkmG z9^CB6y6pc*upqd}CJ} zs2`_+p#W=7$GHXAWV?)3laxaW!zym};lR&FfakZ;EWT%A5`{dV}+Z>AZj{ zxSq2M@Q`||;BHq%7yl}k2_R{pMk0@V^_XF}!U2x(_??r9a6rNidUFb{NF;`XoUg9c zz1O~$qOC9fzGhBQyzHMdt`~=M{Hypk7Mq9nD1_GbPsPP=@AJR8vG>Sf573;cg?Q>H#-TU{);BKf+H`oGo)DV{9$er;Xvaz-v1s4SgCWlTnyxtO_%|XeS?u*67 zdpbJLK;TPB#mI#x!LI?{U((#|)$OPk8|`yXK+&v)-#4n&AqLjt+>qlwJUry%LZ5~> zH=+r`6=ta@3RtvA^AC_d#EHrww3b2uDFkVB*A>dN{#V{BsD70kxOUN-Z2AEKD)h(e zi+4{>T(6#3rKP8@+f7XU`I8N0Qm^ooRssc=9Z=ipf$snL^HG4c zKH#UCYx=vr^9r%;^#}uSTp|NTT-@B)y4GL6f*}G59u3@qS^1TXF8y~~n`~(tJ)6Bs znjDKF9Nr+J*xc*(#@&G~5LR9DH z69XV|0*4Q2JnUCdgKIO8xsdZw6X*8dW2a7c_K5-zS%D4@e@_0Oc?^umzpKwLfc}rY zI`Vw=J&nUp0IiI}e6g!yr>V*Q#FGt&%`|!l9f|{sKE=k=(`_^`t?ZB%`mTIDb@F6r zIhmF!#{-v)&adGX53wPJnWPw0kz+THy!WUdS9o{;W2WMn>gz<$n-PDkKj849@lE1; zLlrFvkDgB)9*{rr_%S*wx8n8K*S92G!B@zvlKtM-TP%NO;H`H<*IQ~$@MQPe)PYkQ zZU-gLPR)0Amv^F14R!H;0Phn8a$^x+-(wt$v>$|OGBISRz-GD-{SDWQ#-$#e?&xtU z%#6rb$*-|;`iRIUNyRUW)o%YBy6Jm5z)if`~)_Zeja=RAVl<=HetsFg3-2UB1 z<=GqWlh^d6_a|UI9uoZFn8R99t`TZnz5q9d8 zF0HRt%RSdq*ZfeULxNoAF(`*klNAr^n45P6rN0Y4?uf^n_7gTm>j#N06q~=uyBYO9 zh|k|Zp@^vT%zr&^cUz9Yj~}Gs{wmpyjy%`SW}Uw< z*GLEPRMXIi2$X%VI%}d+!~3$bW0DW_J^W)6*k7pbbDNk|`j;`jwwR3Yt7~g>j%pIS z=M*9#;#*rL4nuPEo4(}4HW;uCU`q3?%buK{29^_>Ia@{KOb9Od|ENK^~o@W#n1V{ zTQ>b>JH;9f%@N1F-9NgiSYH}L`mmCdb{GBd4t*Wp19ekLV@8N)3odIY%6ThsHqI=W zkd5{m$Z%&@r}Q4SE4%)EX)5p^XN|) z?-ig>ovFj_0nE~%{5G^Xe0=E7$r&0I4=~@v+hH*6vT7P9#`pdbpm}g{#vDNK2s(6o zF~6<7zdtP-G(co-1s9d{G^HnkK>`LN=ty%A$((nF)f5OtF1NJh{@GW*9J^BDZ{DHxfz4&Q9w!CxxRt(2ObYEDnz{qT1lcFuR%oigJ z)zzATO;fJo7C}pAn^`kJHqtc^VpQYFRSJJ69NRR7P`}e}3%r4(W9^MoQ7Z78^J+wY zOpiD)v4RWIvEvfSIqil}m!aSL5~@&&p;8idys|tCP%()`3bNk3NPPTwzAFt`gFaWn zPDEQUBWOTCd8o(00bp^kyGa`^w5J(P6$2nGs62{_9*JpAkf(#F1k0LrWStB=)5kXE zIuA_U;8uuGJ^>KL!qn)Ay$sC%EYpe?;CP#@;wMZ^PCkR>rv;qW8}GR5LV;JnDS~;G zd%peVd=Jl9{{+>rv{d1LCk$j(0O=KM7zSmiK+nFyb4g#H8Olk>4>-Q!XP#mwY9MU# z#7g1;jQR!yV-%z^x)f?&lGs3?q^~3F?qw00Q3c?9h$>eB!UELiiL)3Ake$!B1zt`qm##a?ZU}KN^pHT2z-8@JmUB33pny z>zB`3cXvGV%!@_MZ&uo`??mkB{l3F%fBBw`B7LI3_cG}Wb|3!>3OalXQ}x2TRiu2> zs`#V}QE!&EM?;)8cEr5>M0ySg#&zdtmgGLp>rT{kAE!6YzPyBC{Mo3OG7DFuF6v@q z&$3k++suU^T`M^zD!#Vl7&3Xrh2tMmI$3ed$t#EP;&0-+u~-tD5l%?~K}Dfxho9ed z3=@=%USwB^)ItFF{DntXeanuUn2>V$8dkPU4fLjGH$clt))CaefKFe|*xCJq#2i-} zIp`t~dw3glgcN&dD96{Z)>b>&xHzF55O7ab*cXm}=QV5b)`SbtPem#qIM~tiHK_Jh z>%d{5h1n@aPQCT@{+%-vg!dKB`(7#aOB!l;`-WNDwo98<8>=quL=(MUvM63n#CtX>cFfK`Z`+L3(H!(%&qm(EpiN8z{(`=(ED;? z-chgSQQ(2rtgY;%`{4VO#S?*ZHOXo=5%|O(Mkk|wi?wf!hxHe4TWqykO5|2;)pOO( zFr#ioeOj(iCaLbX-;OQu*ko-^>z-Ztz~UQPE^@OFd+`~yN8bjPtZ%(DLfc>QrQSCi zZRd3Is^a^k(9<>wV*JGnQ;w%I$uU76K1^X*?<9Zi@9&4MTCe4jnd&K4auXSuyf(6c z05z=!i)iSvEOZ=x0Yd+!G^uhX0LVh4u70K8&fD9O5OPFNcz*k?*KfNiRLr{qxD6I_d z->qac($&+$3hZrfM}F`>JOvGU0%4({(dmSbMFRtdsn?5urj8xKT26fFLBOmvuBowho_VtqouU#0&y0(5%YSv$M}< zl6Cy@dm%xP+R^0qhK5FsX;0F@q|~4Wu|4 zj2`9xqFH=mViXV% z;dXP2^5jXIH*o&94-cR8-5VQG<52@z&h2)8`fUVttvXm&>W8sHB&pBF0UwSN}o(Y-1wE&YR6l=+yE5qaMOoaWzVM)|IAr)C!v zFd#I>(&A#dg`;Xh4P(xu7<{_*n#5)8NVP7 z8Mf~iv(cJ8IWZx&cvvp$)MMThckjVqrimg6EIETn2A|g-5R_*>ylTpAtLz!h3i^<| z>PkYVe2R7{{6~3`-whV*aWU?WTS^ItWOHB$Xu z-^%x^$Nn-vRVGg1E%3cP!#LuCKx0OIuIg7c(;+#QQlV*9J4Q_C6-~9gFmTM#Lcu0fwL0dpvU{{wwf|9}K?DVYV>M*Y9Q4Eb63U%s~ zqkg3ly-|@mugmkj7W44>I5eZNNeJyRFKz!aw6rXt%IR=#L>`3L?3A=eD|j39V2&u$ z3Bp*V3-yzz_V^;gFM z_|li@fuf&O0jQ)4CMFHR?{7s~ony>ltMe7gksX@XaZiup)T0ukbgM;OV2Ot z3c`D5hPuCi`}~WXXBNFvg*yXyN_cQyR;5?3i&$EIs+q!t-2T?tJ2gFhIlbEw0wbfV zKG8-?{#&Q`zjbKRBef}-A7m_`EWO>9YH$SxWKd?XL1d)--OP^wISx$y20)G<4XG8o zCm6M63&Rig&9N>uL73zo0cp4cTUxa1=QHBgt}8MRvq02dx1 zp-4WF0&Y;TkuL{)M!g>KFU7^o7cN|Y;C0Go5Zh76Ay9{re(fd8hA5-C`A3G-JmZo7 zkFK`>s&f6}g*Tvtq=M3*qJSvfEuw%ySV(tCgLEU^ASxvwq5{(0-5@1W(%s#gy6Zju z=gxP|oo|MjGjmj8^X}(a>zCg8m0`+7Sr!@BBW7ETGQeIM`f_h`GxymINu=2s927sO z1J0@SzXN)OMOa8k2j*wRKm$Ed=nsT|?~F3AFw~E_da$3LR)gd{P$VeX0&s-fV@-`W zIvzfJn9&slYz*8s;qTw`%z!J}PG^0r(#XP)=?o7a9}IZuuXPmhZGa|$yGAPV304j0 z?jRSo&MMbE#Qtw@5Hu6(gOmUgS1>`X_ck&z^70pkWC;NB;q`)Lf^qx}Dq6tmIy8=; zl@f=s!HGNp*?BVZ^8B&E`xl|s10Nlp!XacAOXK)5|?c{NROI(?gWeS=eUO?Y5HZdfa z5c`TeMVcM|cWl3Z|GBWxR-ye(sTy?7BfjP95mz}lEF0G4cjIfIpWF8SfV}Vbbqi?# zqITjU2-{j)wLkdWyB5FQOj)W%^zE06c=OVdu?v(lTLPs3QqPOj)cC^XllvF1>q^59 zH`5xKAp|l)*h7D9epmoMJ&k}R3bYIvXgKo0MBe`BX~d>fN-rW@022u$$U~4Fe1N7y zZx@G(2$re8D?$D#%o8C?oX`mg`{^~PC;-K%fu=k}1;R=I2k^uFh<^`g_iz^>BMv%+ zu8ND(!y*9*@mJX4Ub_G%HFQ}(`;53^saTVVSV6?+-bF+}0fa|%1wDvfgTli(T=s?p zT}QQwB5s;V8&hK>7ssQysd8TdDhc;De23kOgD3WmB3I$AK$gHv328#oU;E*T#OfH(X@VDLFl*7IHQF;2L zu|zY^lZBnMS$xNC7iY82y&TLH${~0Gr(POei`9k`O9{@5^rV9LS99jQ*f~s{;}!ir zQb}fUJ|DiD{41}mGK#w)2wKqOUJu7z&$AW4gZ1eSrnt@`gH>AJ`x@}FWK;7kB*qh}E0uBjM>YTWaj zNrjW^S5$H`1EYL`DbGbMpcJ9NmV}g4_gl*y$oxj~w&6XtfB;5N!EDr+dS`&&>fODh z93aE%If8o{ycj;5Grj|3ZC)F&!GP@;0mBfocRB5a@i!yi6wnVL1Z&x1Z%WvKcWpo9 zTU#Mr$MHcy?YgdPD+>!X06Hk!(hwmEDIc%;LAP}Z7ARIn>etxW}oT2a6nr1@nifdP&NYYkiP^FZ-V0-<&!wRjzM20|T zNOChoZzEMul-vg0Ukk$UfU5`bJR*iFEbP;x&Q%HqluPEuJI>HfK0q;$h;taG*Y{%+ zKthk+S^Q@p7i7P5&*MD)l?l*8p}pTiVkxws!46dVfq~&8@X8+WN(u=R;>iIej&#(* zz>NksC@uj3r0%6vWo2X#fC8W&KM*);31+f9HykMM_VjCBVElMfp3di<=ig%R&tJbb zEbyZq9~>O~@7M<4;l0U)G4j;uJfy0mq2zf7n1))2FV6Y#;GN7yC0d$Q(y2yX1?61TC=Yh~1HW=ug zr3~q#@rTy2zoY{A6X}MJ2B&%H|5z3l6ih7iBqM#5PEVN0reX{eklRx(;f{_dU$mK? z`4LKVIJG#3W8m7l$=aH3Ty~QE%DrpEo}QhO8suTk6L5V2GWgw%Ku!}BoOdBkJ3O)e z{n+6LAtCJIv#_Eyjl%`>Qn}9y*FQw>&sg}B-&&m7QMbwQ^03h6Ro`m*)X{@(<_;DA zeYQ%w6xf|MeLD&pg^PB{`W<*${Owj{aZ0ym)0X&`rz&IO(tU<;_)^SF!$u0kQ??jz ze8)Pd({@~N+0}gAZS1u5!RnCai96dR&LBw=jvmrIw(rd>$A@W>Mro@&hta2pE32Om z_C&re(YcL@KL6d4oq$QkwjhKJ)$iM?a%;i3hUG>fYZPt9RDMTA-UTOiFJjNr?=oh@0<< znu0Tbxm`nJ?_iQR1t>Z)PEL&+{s@MmuV2q=Yts|6gw|Tx=!wtY9l&+9C7OD}r9$IG zdz+Sabz*s0V8(Xy=4b5O=@JEPVOe&ax~js?tpX zz1<}}d9oix#41Bd*R_M2oSpxWb3)5ZarjDoL;fCH6IMUXC;=YoUJ81`$31l9=&@83 zYnAFpwi8C=BFS zvie+iuqH>sx*3s^(CME7YzX$0$jh=bx7@bvFbVz`z#cNimN za-cBe|Xm5;2o%^Fyy&-6@}Dif-V?4mTH}ceIb*qqZ{d>ZG&u{X}jw+kmoCz(!1CW zdjaskior%R05(&~zNj!A;=(=GG)_*BBNR`Bw4fR?3BUdY*~}y~G^XEDeGMk-e4uu- zH|izJW#B?{Y!)Y~@PG6fgAi@6_jGgu@#moYB_q0}OhmNVH~FmYn9=kt)tLNoWOB01rBXvcR|w z*VT9cE_!-xjVR#$m$Uf=ft{Hm$(~=o>R7@WMWwlNpwlQ>_qU9^j)ZUamoIe-bh3z$ z>FqU1kiFF}?W0iOVnQ5B23-mN^mLS1)u$4oZ0zi&z)pw2NCmHZ!=s}G%-1bM@enS5 zWyMrc9fs`ha`DFT9{#dr(6|E&N%Idd7l9~&x@LM;+zQ9>7(`LfVg#{EC)H5r2meid zk<3j=5!L;a`fz|{$1Ei~fukkW%&X-giYN+8YM{5dBDr--^8Z=D${k5B1eU)qnl>FsdkiT;cqEupDaX0vt$TZ30tN9kK?MaYG-7HV zmCoBc1MX&KERb+OIZ{<=*4?Fjt8`?m>z+yaC6*6KnhE9Qo1CZF?j7A{tJBjS!NF#@ z1T+qJHWZ2`wRS?@d^Yi}T3Ti`5&pfLn8g+{#8;V3Ip2X_v)H@ryQB8+)7-J*Y16s? zJT^M{AZN1LGrfiSoRf2Yb(KYcm?L^WLfC%kXHh`l)?F*BEfsi!a!HLa1ejiUN1x4=O|HuLtCW=K7d7L)X6Ep=gT zHV?-Q#;b4u%CijWCkK%m98D8Y@9v>uCKITjq~s5ZLEWi_#*95m%0d~L4cfisoCesS zdqzh?AjK|!DU4Pa@u@;p8fU4tA*Aa&~qG$QlO_c&dLs zU-dDU8f4d!#p}bQWqDS0t@Ob<-W#)cg%-K1S}b@US-CE!$mprgkDbChzF)5xhIF0H znpOc*@T)>GbcWF1As>-4GB9|(B&Vhpo{Pi_^Mzvy4*1-qdy2`E>)@^%#3-O{>t-;S3&1-Vy)2B}$2ZhqhBuPIw7(0L1!#M=S zm0A`S<6ZA&BESU+UGXhO@9qJi2o+_NaKj+-pd`5eKmnsx?KV0y4Xx0CQkU0}hC(;kR3_6>67sC|8lHh%6B=H9t47AgLgBQ5QfDPj zmS3-ojF+1jL;pQ~KL8XsM+wHXj8JEfN6t6)W59 zwTF<*XSp@=@E`dQ_+I>Y9f=()W@cw6ffvaT0@w=X6wxdD&H=Mf`u9{*leko^6FpE& z|H~2@@$Nww9(*Hk$pUJU(R}D4hUkvH{Z#w+t-s#-@CcmF(7D~qTvP>d8E|F3g$En< zn&7A2|9-Py^B$~%=Nx3tcNrs*TtBLirZ2Owv;yl7qFGMuI&Y@D^-V;;5F{9c&hCA!1d^!Usks7Wqm(l1{`q;~~bKg11p zuj{&&YxiBIv>w1)Ikdb;EaLlSPUV(4EgPykk|BFaL-egqMTBi0iI8kpRcHRR&Jy?Z zn-K!PdQZb}ZmKq?Ci8wW9_5ORiq!MLog7mAw;PJKXM?}}(Ga|ZGu+>PU+#fwQZH93 z`)eP~uYTvvY*X2JgU@de@w%z|6ce~=G8mw*S=)IG?b`9({=A|PXEskg>HFQHE9E6I zT3m`Cszuo&YE0F)OG7+tt9%#bQNX_Sxp!$}MKu76ExnhMxs$`xOE4~Z(zw+~xnZROt5KaX%Rc_rP>Bey(l$Azl+Py_->y?6YSa zRfpdVxz>R=Fyax~I8@_)tn7>BF<_$3I2V1Bk!&RO-cFrX{wme&Kix{&6f~3qt?j9@ zgEpF9>DzF+x&2F%)2QX>c&;5fXr-^KgO+M$oQQz6Gqup1!%J)Zy0?5uxf==kqEjXa^VZI@kXVJU~28H9rk z4-e-II*-JKuF$~lMC(9g_r>W%Lx>;4dZ#%xLjf}*#svcLLJs{{49y4$+URK z?W1EudMAYJxm9SCw^?D~$}%G{G4bHgkn*kA+Vu2v?$~2K7M8Y0Jp}T|XZ)%c5P-#$ zPst8ilIBNlEt8hvpFWAOg+7*i3bpJ!x6+`wFTr`O)aodf9CVr^BWfJuZmr!$JTSS! zoS8SI56M`-A#ZM6J1V{^EiG+2TtwK}IY&AuvY3~bhYX3!#b*q_TtKgCejn2{_e;TGxWo7WZ3+NUa+VfiYtU_NZA!C1pT-Rt$`F5e4ZTHhzgz`+0?uct^dk zOdAW4bnV&g-M zUez2lX=!c!{~=v1k`!>^`vR^O`ndBy=H}KszN$eN7XXCi$I3%%GeN%d*re5zQ=H%LET-OnAXoB`u}J#`;C z&;319Nrk7l;px)^T>riMeZ{3e%;{bIKT}xf(eS%uC6i;j;{IH1;iKf;aJoX`xSp}@^M>p=`DXZ8)S>R6{W*OW z)XmE&DH*;6}>AZstO?wN}9<#RQ*Y)TzbeemvBJ;*jR)xJyDE_!c`ptmYXp7BN#X%~2nS?Sa=X>tfr0*-_AOe2K04_HwiUxo|)m1&Ig zGBfuSvwm>xGzWzgt{m8IewiKU`A8B(yO|>10qk4zuC1eENcf&0(N0w;)=$m$-IP*g zPl*5G(Dh9mDl+VRpVQWxlAovP)U)Lj5tuf``D(O%C^hwNztqsc~2Q?Z#|3 zeX;6ykG?g2un=dTD=DSyJd~Baz)VKv|I?)iit-dR%&Ulvo;@RhWOq<~ow;GnY{~gI zhF)pZ#}gLbvkjAL_GmBclfU(E0vOG0EgJZ*n+QC9l-joyWxz8^;jtZ6J@Qsjc*FnC z(h?C6OQQM98s&Cx-B50igDydVv1ZdV8q2GeIlWPrX@6W)-Jia*?~?Xlc&R@EQ&G^# z(<<+6i!ug@;68z+0mTIR6o-NI2BmEgdaRKxYT+#Fwz0%y#}df(Ki;a;v0yf-KY3hF zJ+U;yc5l$8e&QG0^uC2vJvEDlGS=%7#m0sGdU2k>ej&AFQjv;K@vYTHR_`$yMBW}A z@l<$Wm$(^#3{C)m72`qd2tc;1oV9o?bV0)iZpgz~@`SM-6p+n#LIJX85Q>d&Nr51o zC^mJfe2S(&z$b+4b-`#UnUWt7eB&GnYODIzbrC6`+pz$Jfzd-$)7vm&;UV8g-?(CF zek%stU-1^Z;8XV~f`-ujC_;>I9jp)VYZ`7X&H#2%4Z5P4N#v70HAM+N}sPdx;-;!`uffzMxxAGO-ExGY97gqy#3%FIx;bl z_(NMaPEh^iRX)D{z>5*JASXr2mebh)s{(=O2^+EnLB}p)RVg1o0fm7MS~P*2jBFym zOROI1WRRfpxGMgWlG}Ig{8j2eVGE#S`Eqv~<_e07bRg&>oO2Whmuh0%Tv) z1=jH66+>}kA%w{x_;v61YtTv$J=CxeB)XppK6*4583ZZykbZtSDJh9zqm4Bgf_p&x z6efO3f&3ETmJAjn4%=`LK`>8bp@zlzGEgBDyU7gySx9Cc0_DMS`?eKg`(M^Ps}nyv zlt*-0v$Jz=Izi2f#LBb74(+xTnK-eSk|^cia8N=yE*AUGwS;I&B~8qDN5@~lFCL!O zgPmS^+0eyU5l1fMi=I1AIe+A&*Zwg8T^$stvN0QVcv9p#Bkx{zKC|CNoRKaJs6m2h z)^TTcXPHB<4DCNW^iQHEve?f~)?*B!pjoyV9zCmt?rZS2pc18n*9!3&f|cfclrZ|a zylgy+l-}1#-ik9fzCXie48@Rg!0_wWk0)aN!o)9YcUcO4Qx)m&)Sd3nju~WqSURh6 z&N`P@@QCL`roVyXMOoquezHwMBR5#yOScH$uDxmQ|Ip6(sZ8{e!3}!UdS5ad-M(4d2eJON>`_XX498h^*LoqnM@kdT>8mJ^k&Iem%PwFP!b3Uphl}A%R>KhS0S&T--4qS1@yg zLfz41^NdrKAD8yUfILazE3AgMn3yoqMLxL({5$8kpBr6y|Nps1t|x_ut-IL=R(*;a z_TV3rSuC!6mR9?bHKlv#wa}r1#iamhKC{4IQ9!hl&CK!=u!#9;f9B%+9MOD!mBQ6`|I+Zv(w|9Bnehw4ckIytGz^JtfI8r=GugoJ7HUQ?-p#%*rKy9v= znQx@ED-%~LcfnIsQ&a3Tg}Q#s{+$L7ONGaN&TBQ_W`g^DQG{o3Ma%QyoZ>009z&;U$c`m3|}4Nsy!4{F;Y1WYjwyU?#w8 z_+wu@NqeCKf+bLk(Bvz5Bm~I9@upW04@^KX<0~pw_uEZx0y9~L_4t5xJL2-zr>9SJQ~AS)h)AKwap?C<#YYx6?j*5}j?6)G1wL^>P_aYL zqe0&gILhPk>B%7y4DJEQ*VpP}x_MLd3Mr~9Q9SX(u^gya;qb>|iv&G0@FDq2U`vM| zhi5I6Uso3X5Nf&^i>-jsgayyorizIn9qIWZqOEIwX$a#G)+jTg-Ld$ZvZJ%XuC;B1RP+2fq{_4NIwm#1D?`K#0o%$ z9D95gV+7*j)tsU4NVq^Qr+$&1zUxOjY%k9v4{vxh@HI3vQ>If7wH|)`{H{ zHf2ac{3g@F0M~VfHROueVqW7*fpl6>8#IF{Ed(kb&+5m|PVJG33g9`UsxQaXO?0(i{8x(!jl(iuKC7ZIMg+D{LFdJnY36^AqPUJauX(5dB}2ql2XHF|%8MPIL!2zS!ADb; z0`Qd^0k+$$r#~ebKN{Q5fplf0YW$=a(L(Wf#2cn$%K8Kb^7T&xx;U6`NX({Qb}l|N zp8nxgl8Nfm_KPM1&~9;>h)Z~4U(LBPLWyN)=vj5_MXy#@?{_keFK+t-7ZRo7F*ILU zSXA<2iD-1WDL?xaIG|Z=w3!#iOr|^jDzJTag1kC9iQRp(5eu4ju_DbVeI*nXube`j z%{6iH#NB9>(QP-8lbznd(l-##75pYUM_(XlDIbs!7jI@RMr}RwlqY>fyCytOgRy89WorTA{pS*P~w%TJ|>kLH8Lp4p8{KA){|9IHAU zHcKrdI9lD|jW{VX%Q82d)jQg6PdPFZj)iW62(;svhgqO`-^7^DGgrr{imL+7l>5ro z+uSa6daqv^Y;S*abEDT6v<{lG*|)EQ$PV(wzLsOyJuS0>@M37gr1%XnIuxSH26qZZrlEjVm}o&tXX{(GvM)N zT{$7ew7-cNH@-@g&n|Uy)?Lj@QTLcGwhiZ?FliU^guhiJg4#W;%nk&i>?q9peqlsvYw~tZfv#BclT(YmN}F-H3PN?bUr(w zL*=i*!4S2z!8uB!Ct-HX{J{C zOswY8_uc)W3xu?#|IbaSTdq9)-14BOo0$DHQr7{3r> z%z(*;;~7~|Gf9Qrp5Vrh6(~V)#rcv|3C&mcn6T~jaZW**0-@@Q-j)r4JrtWIKdme;Po*V38Hf_5x_mIv z*vnkfbM}=`C4|WF*6hJ?M>``uapI)k!3W=S_=aaOJ`B~>p`qW`Fm#{Z=5??v#B_FZ zxj25zt6Nj_@nEU?*T(w#r>4>Tn+l+Dle|!}qIaGugvg@8>O3(W1!Iounqd2-`_WU7 zp-wAom&3RO!S@`7p_2lMQ3L*t+e8z4tEV_qEM!zG!%~=c})#`$=KX4aVdLkjR0Q9&^%&I@$azZdQ6IBjW+O z{PZ=TWfs=x4z?dQg+hBF5YOpZ>taqoy?|#pRA1=|d=(_Gtgo++Qj>J}dXhp5%WX1U zcMZD0DyzWzbJfd(n7-PUvc4^jQP!8ufq(f^RSj=zOxvxukjUgdjmwih)LYa7!Uxy^J!jTkHs9jKiiD@vctp z5Aa>NR?|(KaTFF6&saWy2cJDWGE#SE5V$J>Q!fs#@o4}kk>wkRDZ{*u$S)AC4hi^8Rjx-0>gr~B zvcTTJX5))o6HvYN5A#$AX2RU8hZMe?Wv2{9?I6cm5H3htme)~-{U!sf+`O;|f&c)8 zdiBZ+#cpR)rSHaE+`3#NA+0~(-I?qR2{};F1x&bbIDQMtCzQ1G0e?dT&2*4MB-@}Lk zWneu>j5kN#bWK9TSq-qapvb_5oMO&Emt(51!)}xo+D6mTd3M@yyCdDmY<2PWdr&G8 ziyfdqsUp=5!}C<%D@|VNBS5-|9FEND2MtRr@B;!8r@;`>-EXiE_u^$===d??<@8td zep1PjHU%jUmJ>N@X`vE4e`nVH?1&9uc8az(6&y9WHrvrYL*b%~KCrM3{m#J3*-2+aqv z8|;-llyBy#&?LyEpP}wd)7N)G|AvaHY{O6d>-5A-sowQOU9)Uygl`fr4SL^2AP9t0Jg157ZzXRbbITGMX5qer zo!#wJhy^b+rj<~;+DU1eR=s_`=hETkYgNKbVx;k+s` z*Ewp@^CrZ1wq0`9U9;>(Xz{ol|4>%`b07NSiR)vo{6V&?eTh(-dXeV%n%ibjdWn0r zc1=-f=`N(x-$q;C^&o`?_>e!Pr2!UyHlc- zg8)jl364+qeMKISGL}r2kH_o${@6Udlx>328pt{t%%87noi{UGX=(~#vf5dY3@Vzm zY-|k3)o;J=dN@-0Y-PleDn7oc)Z;eGpN-$o{hmJUF}E_6lf%qVd)+czV4+C|dOE{A zGet$(E3D{CFxV(Lm2UK_a+8owpGPUA6)5FS;hvuEKVklGLSOEYV3H*-*80HZSc68` z0IhV1b`|fR!8Y+M#<#3Yt{wB=@Q;7T=atR>074Dl1%)mE!H}<1aS-ly_b7n$fq>N) zTsb*AOMCL9sw(P03!EdXQks?FeKHNF#XGUFPT19;l#g8A+qG?&tWOXBbPR2ZFN|J4 z)j8VhJ=o2Do@=Vi43Pz}!{V}hxYE&aVP>9xvXfSDZ7j3-uHfW(5hAq;S`tBFRNn7| z`>Xa{A1B5dpOqX5N@!V>$dZ$+`V>~jb6<)x9(nBa`U0Eld#}gzvsd?B@N_nFd*A3A7WN`Ir+- zyB=OaBv5`*uI|&c#rJ}oCkZXVVDqPz>4b@)S#exA(bC@Z;6!BQ?7@WsW7VqfA zK^qsWBpL|Z9em2}&KTi@JPOLXa&zWpW~x)S2l^M^_HBB7?tzC-t+MH2n4GW)idxGz zhW5Ma#oI_D@{cFy=5gr~{W`ClH@*<7V3UqbD|?~DHLjrH{mbe{_jUD!AFc1)U@Ubl zKmiDJUFW;_TSn*lR`qjiiTDrvE@axOr`|HU_rP5A#i(fWGNlQF-$@yTuf#(|+DCay zV02(`2;|57Qb|HHan8;AwvcdNOP56*{O*`C zxS94;y+abzL|-9FIwV?f`_0c7Qjf+UFKA~z#})-N06>f`H*W+^tK8BP`e3^7^lF?x1qy??jJT6Ha2g#ApZnLM2vw@T8~Te3n7S@ zi2Ukf9Tosb*x1;h*u``6tpl$tZ?3j_{Njz5MN&6(m~fsklbsY4u(~o~tQ-lMzI^;R z=;=yTRntQb)0gXA%-!6Jik6l+0Be8{Q7Ab;h7`QACtq!W2u29e^iGIi2OW2z+}-31 zP_jdHkwG8WcC=wtoj$ExF*`jzZ0YRumFnG&m-vfyBhh2&>Tne<|5=Lw-5$gB>pD+# zXtV$P`LoeTu1OJJ4uXA1jqn7nLG@N16R@R?AWsjEjWyod1mqXuTwK&-$>f7Ue+Z+} z6!2~*j9a_g+mg^oT~bj*@fs$@5|=}+3;423rI=1t?l*4UyvD_~l`FJ<@gDXZq%nZr zA2DH}%i!PZz|;pibvmtwI=6u^00Te@J6L|;t(!0=PJIc=f?#}z&ze8IUC)u4n)-SD zrQ+}XgaEk|NoQwa07)`QNhrAi@qsi52rZBRry0P*yNR-oOG`?w0>%WPgzy10Wir|7 zpz-V1+8>&y#;#K;jkiieXAXV&QRX!AVx-?8y8v@^CK(}R2KeN_TpRQ<3T$Z^w%`D{ z4xvPE=6)jEGQi9L%vn0rlz^R1+wrLmBPNt&okf;Yxhjt$lVeyBjm(((x zsS9*gmg{uRUofGS6B6w(MCiJ}gEa>42N2~WtgYr>0`~>WG~ONGw~$#^Uyr4eO93PD z|8)HD@NAxZh7wcf{Wab)zXb@af{C=gx!Lf3Vs?#(IVt!(w(c}tzKrfbaj6Yleuy1>=qaLF5ut()Lj>Z!Mhl@>e4duI{-+pI2 zui?5YQgwEM4~*hx#bu~;%Cty)Pv)JenhcS{hP%V2R{?6dBjV8Q?}g*Mg+U{(;g94H zKp0K5=rH~1sT@hbfsx7cq_Q$B`}O|mrV!(Tf}ILYNUb4S+6^WHEFKFd0rLBfHlc(9 zrVI$I6@qE@&K=b&m=+c{jZK2UWcFEMqvu!M12b0gmoL?f3Pp!!KD_#_w3{Fp)(io< zU0Qo9>`)d&go4&Q62q>KI)_^dHyF_pckDc#r9q>sFvONiJbHAKhl=)eTih^2MXeg$ z_XHp)S(g)A&g%@~wM`+<8X6otwLK(neJI(Kb5hQP#-sI|yNi}cyu;q2 z4(hJ!YwFY7#dC~oxc6i-C;%+xd1XIw+Q`-Nn81}}^}Yyl?&Ge!Hnp+neIVOTddt4g+QFNRjS zVikvyT@|`Tku^rNVEp_GM6N7ZREQ zc*+KWjjvSZ14(6SFv3Yf!YxXxDOLtVJinrFf?uuphqy*xQ86lEyy%q-LgSDQc1tG1*Ch2`SKQg?l2 zvi62U2^n-JncC?A8|rRm_teUn{^W>w{=@>MSCFfc zQeE(3NqfjGkIRAQV3H*qH|&C*92&+Xau)NtR3Ym&eqR5 zfE%~%SSeFgd4I5-XfX*{`93W&_VD~^|GL5M;NV{~YYlah0Z5(X+!MTl$}Ix`>+p{% ziJUeYDPT!_iK?k|Cd4H^qkzwk%scx@48qRR)mDf*9Zm{BR2hFJx}$&;&oO$+0S@fc z4AbVx`tvOvX5Gc)xDL>%rB4#@#NFbck=+1N67F#jeD38O%$?>DJG?=TaOum-{-C+w z;x@55c8EAFJKV2M^wK>0V^b9pu~S@q6Ufs?C%bK9m5wD2xinqk>^?7+FbY-te*P4q z5xOBM?MFpKtmtVG_yRa&v0<@te~ayZZ-Sik?>@k zMZZb`>%RdluD#R&^8cW_xsIFgHgT{LEiygC=2DKVs!Em&3AMm!$AE4%QL(e*TX!^Q zFuy%Z&$hR^xKqBSoPQjVP<6gb)Slq{`3{svV4$7L*2Am^xtk76UM1(oiB_D}q5=*U zKIEe%J_`|gH3ix!sz&5nnXVG9Jd`?47GVw4uTVW6Rf1T=>-#RZmL3o9cu~s6@V`$7disaX;?wy{IDWX|xZGFTOS)CU5 zZQZ8B`{8by{zm+9@_@cK!fNFj3f!pI&Qxx0WxtG7cl*4bS~*ZfggSk;-7N947~6;6 zM8P2+8Xb1C~fU6Vh%8IY}LqC9+{x`xw4J#Cm=Q!pA{C^=lvNw_kPEGg?2Vwh! zl8GgTK`Im-36Vv)b7!-8~5)l{oB4xcn61oh*GHD5>p7adGbWkyyxnZ*V+^b~?o8@#7K+CD= zTUdx^F>kxZ+6rRHQ9 z{#;HE=Yb$OS8y;v|JP11 zxgw@l2rtQf0ni|5!IelK16GFQZ7Bu^?l6+zB3x42qk7SJ!~uVwfK8B(Qjmy<2wd?X z6GWGR@Ejz`Dx54_R50cZzNA7mio(#c5(kvaqGwcwu2RG;#1fi->m8$n92*^0pP6Z4V@ z#Qd*&83hCcly3cCQU%EdM8$y3Quz$PsSbA2Yp-6t`oGaxke*fF-fr@wa}tJfguQ{k ztGp{GCpQTFRFJ_aeJlvOJ_M?8ad9C*S$2615E2dghPx~u#5VU|{rd#E*AfZxLXd{T zV*s?wxL#t|DXcpZ5VZy<7X}8#Z*gt!;9?h}Fo=J9cH<`PeJd;ZvnL@^ez8=vD9Nhx z5vlWHd$GJGAk-j0rDm?`Wg0HM*!188LJt(LTn#!n&{-7IZS1^;Z&FBLY&^L9r{rm1 zn7Br0mpFegH?`4`hzh*By0q%f*&~Av$jzXIhn)T6^i<;(M$~!iF{$K^-d;L>bJJRA ziUSTpRIuRB@Jj(Gu_|W8Ub|g0uX&0DaO+SaGBWG-*wpxxNx# zpAi{&22Teb;0DkhL~q=%-YaOH6K8EmNYr@yMct=-{*l$bT3Tz-JtpCm0ej*A*}H3r6HgD=aWuGflP9ij-t)m5NMfkV1T03J~BuvhaQ?Jnj}O@-h-0)S6oHrC)iZ7@*_h-)_R6x+@ot|nW{UC9q2+PVcryhuj zpxJif=jM-e=cW=x7<$3L#AAH>(k3KjU$C)Laa#)s zBAs?WHNEF6+{}X9o+CJeV@^8%u=wNTcOwiIg z6e{Yx1u7w~k?L`eCaLoxq7TAl)N&0v&Y6yj;QD{k zpi8XW&;6JIIQo6@!HcLb$)dSBKIf3M9JWpet&E9b3_)`@xr0eh!wAPOBzeCd{l3DN zTrP`Oc+aHZiNyoTLUgqtjaKVzE36`SHA_b|)6?yHUmIs9m)4w(BJC*)tv2nT(Q2B* zwQx4lpxi-t!nr(A3KjshVQ2c6j)WAl#ghsF8nLtytpF{@v@XZ3HaU!|VWXd8h~A}i9YY;Oc; zh~BxAgxzYtXEk|ZicJ5+bE`oR9D8GoJYe}NR#v1%bN!$3&e~w!h?3)0b&-mv=R?|d zgfMDqW&Jl6-J+o-;VBk@5Nnm008yDgEr)LAV%Krr!1rtOrg#@D6q6xsPcKfou1;;U za=mdkp-?;6sz})bWRdr*?Ffl2&$5JCbE2{|vThNSauyLqsPoNBP4iGp$RWg#faXqRX z2?v8n1FPRmL`WVdP;F0DRfFI{b9FT~W&tB5T|3?@kYNS_8FF&}JLpW{>Ot7^n<04^ zz%1Yr5lK0gUMr0Nv;wYqE-nH*27nWC+|S%9Ag+f57hCQN&qpxlD$l|vW&ZF8*9S|} zUNuhU2G=OoyY%R$FkUbW5r~=*k{+Pr;J}&Z z7W}*^;Fbuci;}saRO!SE3Z6_7TFP5si-NGZFFofb!YoVGkl0R0b)e@Tuueu250If^ zwETM{7N#hkDPk)C?{lot`*5aDUS3!Rou2|s1_faZ1az#LxGdkV_0We$e+_er>l;4uW`4uFQw*(PRYv|4yv8W7P5k{r<6FPs5{ z7pIv+4)HeL-rj#B9BKvz>_?8rglJsFv9hwTnADk)(<>|EZPzJlbdQ8*9bSSBc+S9X zYwK@V8ulvmU`y?vCudU=nChzk>nwhfAVLXXPl?-!5aiT_`~esdk%s~ZR11_6Vn#TJ zjUDPBA|H$7T4aWdHou0^$r5T3L6!L(?s-I;lmSH-UH}u-L6y>@r=o(u-@y>^*VdJF zHn6X;Y6-0oVC%|11&5w$X0t70IK#>^ik*5($5NnX-HqR{#sLKd z>4>UdXSnF$!AzvuCvi>CJS8}9U|@7ci;a@ASCDu;2y$xjwb+blf{Q!A_Cc*=4?72eW&V<>=kxvc5url0d_UZ>RMw@@Vz@lvxyj|RJKFPxX%liDKwe(g>zaYvumdM}^5Gl9+( zpNYL5%$8p)J!-vpnJPHs7S0E98#M)nbr*TrNz~}b)y;hO-`8V!%h?$Wh?_`~eIY+L z*_Kk;BB&a?#y4GYA#8AXn-da56|+NYw!4T|MQLwQ^GjX3T7xHkx~Z>c&>v&U?U+aI zb~LQ4WXMQA&F~yDAV%%UVQCMH_^8M`u^Vu=roLC+Z?gK-gb%{SOA=G#X^B(hGHW9k zh@mruYowGx+RTgz;(M^>=2mVC{lT|iGpz?pFr}vJOVo!p0roXO8ljYxTo=HW{Nu4>iV&F9IJ;8($jg4mUPVkk~GPBZ_=UPWZ(6w}odP1M(1rY0v&zQvXvy-e> z#d)Vacg-;wsFcV&lNJw@RtV>I+snVr%*uPZ;^g4JakAb#SzH}YDCWMQYkYDNZ#ms$ z0N6 z({KveFaFDW-QXnfWc<#_e_`I>H|NjGhCdSTi+cW?U%Qj0ok=C@hq1Ji6XDX5P|rw) z_uU#FV7DeG&HV#5-xe3wd$cFAm`IyPZhAa?Onc}`1x6TLRCEOo85qfs z($60CMx8|$K0hmlX=z7}7P06Jef6jeA{o2%%3E$1BTBg9VUbYWW#;yXvh(^tMipYP@yJqb8 z^F-C}3xsXfGe!{90pjG~P|uoLqAW2YlE)(xm#1`k?mM34FL6*+%d+GxMFq&HS-6o{ z8Rlaa`g9omF!oDV3F5#JqxmP=#2*qu`~#^*zX~FHyh`Wdo(4*nD*PMoZWI(^goIqc zJ*Vd2EPR(`x2E8=4z7>nVQ3)x-(F9zBg{KrTf(`_j|kWUj|7Z=ymOb~p-2x%3W6cMwqVq2~Z za=ONJ%}>vZp9!AB^xS<*^%k_h(;$^aa{=vzh${tz{><6aJ%;NQkvam7hc4QVD3wF>EYiS1m8C#ldF|O zLCV373@Avzeo6Fy=z0%$to!{990@HM6_JtE5-KYhk0x9ypY6xA+Z#mCtl5FgV_R1rQh0iN8`+VDIF#^7sHcM> zL^+v`vQRxjEdMa`%^c`l4^MZ;^p&F$$pu<}c}K0En{R1$w_jD24&IoL>Nt)LRi8O_j0phT zt_xoTKAhVBFAU_NG`@7`&H6!z#KZ(fm*pd;hNnG25<37KWcThb&jvJZaew_BFmVut z5a>GQcPeRmez8trnrj060#&cm0Q7|A7o^#!K!aOHzc%r$wDS?0mN|)u!ND?(WV1sI zpJRCRTeqd@1xEq8)NQ!p?|*UYT|G$ZVQVxnh*b_$iWTy$&NKU#g>I@}1+FkpU%cJK3trTRwrPAa>uwCQ2?oU(X?(5!A-q=2!JX{;2*yX1nEGmsr_& z?t~G|p0}&@ix?S>4qZzPKUZo?HPz*DUW-7pVm@lCb>_C3ye&BQekh+yf6j%GDzBoF z+4jNiO)=OMyuQb>`uO+Di8_LtzN-v{j_t2{@;LOx#n%=!-Rxa^Fg zQjLPHjDm%-f&D5(`>N-cAY7Y|pR{n3+iuj6Bc+=|_(7PWlQ;j%5svD*mwtJd*RqMy z(<*xu@EiA!yojF9Ro9<>-JLbKu`5I>EQzajgkl(wm9gGn=gJI6->Lmurl!WazDJ3d zIo6I2tSO$KU6bLKO6TCl=BLCxHy15-?mAm+K+58UG#Tt(#6Eho@#IOao}Ocuq5q9_ z?T-i7!k!_o&gqh)wurtf)wXSSseB263W9$#rIC>z?8hwyw{QPeUH8;KH<$J2=QZqK z(!gxyg*pL72Nu@-{U#t81V5Mny5P1nt05f$AkATZ)BeE;VcbBZ@KMV4r57vVhIsY|%*jzYPA>~g!q%T5mZ{KoNPo6h#qg6elmv5-A zR91eKO1g*8sg4n8-4V;? z!UnKbKY3v4l0uP3aI(0a-Lh;6jx}H-0ug0qWxd+gmjBx-F)@*_?UUh4j|J}uJzGvo zD^U|{lpva8i(2gpoPG$C5{720!Bw4^!Lk};fLMsZS%ijz083CZ5LdEw)-y2h70h*S zE(8|X#d40bV-b(|#DY4hX;fcVca7LHA#zc^cez34N!8g|5(LOGCs^5GK~R9CEE;|) zccB`2pM|U(;95agohF{@#Mlioi(?d`Q46X>2~x>x=9kR!F>6IdMZquiGO)4v{fP3w zG_8cG>ND~XZ0Fqi6Hm`X%6Uk_ zU9duC0gT^I7H^0qx3Iy7J3_zv=?lfk?w^QJj`f%>A|=U90HYti4Oy*u`t-&yZn^8& zLyRkxm6J;$>RHp%r$Mn`0l*I3CbG)^;o@KgSOl0qM2MLbu&bRvBeQSc&B#bh=xrzW z{d$faOweY2Kh2?h8uF_^F=W{ zHiz(T&}g&!Q&XXaA?i}X!k_-SDfb>+jML{%v~CNu0iy*_f{MNUE+jm(<6=EdP!}Oe zx8P#{&A;TeI(fjJ8rLN1WCyR@(guB7TU+Tu`d`D!kV6p_r4ZekBe^kQtk|MB_Covh z2dKS-G>+0^NeL2ISUy65Tz?bn-;wla3=)X{k?;m$oQQb|HD$7E+=O%G)8nWoG?RO2 zY4Rf`nl`^Xf*lhmyCD~2!|8#{dT=)09@p-S0XXC6xMyjBe(xWczq+B}W3SZiP4ImJ z4dV|JVPS~|PhoeN$XpB`|clTeck z|8fv85>frfF-u5d4f0*V>ONxyZf-jeFqLD+Zloz~f(_XkmY}G#!m23QM$X$?COtoY zKIP1tYPhT4>E#EBk9)1AgiDh`s3?oBlW|U?PJU5IMc+(hH|Q6KMPyxq6)JhzTGF)x&r!!#f+y22|8PyQOCDpl9 zap@4*9PZzJRLMO*bcg!sVgePV13QP}4i#5oc?i#=)ncK|F#DsJIx3pEbh5_l54Qtm;jEodu%Nv-s$X7odcs&pY2OcIFL0VFi^T$5%AJ4B}eYif@pIZ-ZwMA`=k1&3MN#mPyQzPY~<4y#I zLm1g0YG_^jk%7w9-5m&1lvCEn%1WZfL}pw>u!>Z`7h!=)?(XW@A9QYAAeCBbBbqpc%vtum=&gdTZM(q>QD|t zYvgezTsW1RUErC8}=mG2;-pm*6E5ulpg3kp> z=$^hAO+OA;;DC4bZLG-q>k~*x!NwiHT4_%T1Te2W$9XmZ#n_2GMOJXys{VwU;ENp1*wgB^;8FfL~4Upo*V=cCoq1R9Bjke4zY@o3u*#gadIBK}oX z$E0N#<=Uq@4?;?BUVZ@D-cKzUsvN_5b`pc2%S^X1qRs1F3++fKU|AIh20jIXOxz50 zz9YuK%V7#vRDyWmbiJ7`switwVEQ~oj`s`+QFy022xnH?L%|2e}LBL z^@pcN!Z{2UZ*Ex&4I8Yx^?7Wh0x(K;rk{D3Y{1|Ge~QRQ`Cw51UoHR0CbT}-R|X|* z#s{ex^5Z-%gId^{a~^C7#j)1Oj*s8s+}}WbzPnw+;uMf_jF7QI@+MJ3ojJG|7It~S zq@j}*FrM|#`?=XK1WTHaU{3_maJ(R0@RM!k9!@)`l(0#K$}xk8%h0s$$A(mAd2xuS z5n+2MSao$`wYnG##96LYrwm*CV`Dc^<=xu}`=lRxJ9c@GRU3T#;cq`&R`!^GVbi-d zs2WefU^jGznOe?^m2^T-6swtPWtNQg^vymOBjW!i?)+;3NdIBqjQ@?HlCYM2I=7|D z(VT00qhi~8R81YqdOqgrT#lqUkn;%Au7!r(#v|1KUN%tr6!%c@U!Sn(=iNJR^YZf< z<>i+hEqQXxBH5EI^%h$EWi&tMu74jVA-Nt1uovK^;cDG2NHF*DW z8CCLBrhVJ+Hbn+E&-JYXdl}u~@ceC0`vJXxyn;@<0|%Co#YO0_>0~3Bp<%)xn{K|D zjG!@$m>BpTjOhW27Bzb3-`}qNhdEf)$0eempkV#&?F!UVV0qTn$H&JP=I2wRPv%V3 zCFEFa$CK7)Mh1Go01G;Pgd1^>mxUw$!@rDZGcW-lnVBIZNOAG8lqgpT%&4JNuvJ9l zYyI~O`(>b}UJD2aSkKQ-1;H=?+DW42X1h`3tiwqHP9lf?ur#u2415QiC*nOIQE={; zhcJYSQlVzeziwr9-MNFBbuJDk&eTe&-uyVhT0Hih2g(>IOdx662KTK42|9!WPRBk~p7+{}rs6(g67*_a%CPh%qFcQ{>G1;f}XDk_MFCfKE++^mSv?;QluWhX$q zXa$;8GSUFnpaY3S6y+S}Xh;5Lcnp<{q)Ab~n|4k;T|m+mgUyvD|j5ZotL$(Sqz0dawt z1ZSF+l~p0$-A(j4;NXxITf0k}GAP}*5%3t`@3Msd5DY^_*}4owW5UVMd4SDu$fOT7 ze5dCU!6s2-!;byn zSr?bn!Gqxsu>!~O5~V^lB;3=22&dqShQ~t)x1+s%K5#FhmeeLAVkAOO1;w(tZlBNZ zrry`oyt*0DWBcy=kMbX`qZRACP6H4GTD(D_(iOS#8e&uj-m(zSwXIh(LVz)fANc01&O+WT$44QM0Moo?NY+uW&9JWoff?B-j}DXpOP zyU={of_Hyv>m5ZLyX0{?Arzg z{S?f9zjbE!3kh<{jS9wp_PutB*vGizE~fYH>p^{j+pAF1B`2pf^V3|LlD%e~fr?Qi zUYtL5ijCw32$tkmR@#2-<9UgEM;bA#bem~B^M*QPLsJv~=FKLb&fb@_;Xu+xT24%Y zo>>9*vxc$Zr?holSnnvtVy0MLBqoZwvw1i9)rS7sktD0bnr1J(Nnp#C&Z}$Xt8#T9 z+>d8|7K$f2VPTkJZI+NYX#`&?Wdul~pbzbo^*eNknQgT~A}R{vm zSt+i%YEoneTysrfB!32w2v^^0NJ4ao`b-B`+ot`5u^5B`l$MN5@=$BGlq>e^qYlQP zu)n|mHS20cbg!?hK_I*9(NgplMH1O$?8TJwHOs1v(S*rvW|rIw{j~O9zb-rtY=GD# z5l)8N0J%$;vVJHlyAI16;_nFZ7r%%IBffM0%*@R7eDCC-IkO}!fFhW5I8Xa%u0W5i z69Sq`o+!}F#I|O>jACOTu;o9di10;(_n~G?P7WGalvC}$f3JZb4ql?7Nu?8!sZdBG ztov{NIXO8EMCE4UJsdkhxaZrqkMsNY@5|m{vaE*jK0d_5srjgX^?S0A1ro&YlPIPw zxgjOK?6yFq+~&MxK*@4@BOe_rR$j?KR(iVEQg$aS#uD#Vjh zpKt>ik4R}lAfuxD7kj1u?X^r)ADZrvfuzMt>-zegGoSCY`>T9>B0c*HO$)tTKa zT4vF317 zWl`uJ_88)HWZ?yUddW)#r`~m7b)G#-7qia%IWeAqe)_r%8+!ZD{6ozB3Yj9oS9P44wmB0677T5lzWsP$LaMO_@$jV!HVGYdfm-) zgmEg{+&Qn?!x?vM>)Yc3_g-O<`mUF!=j(N4ZBWRD=*dgy{lR6a$DL>J^25>ny1Efq z4TAWL-Oo=Ob!cR9wovPw+4Zto>-6a$@L=Ecf@%svQedhm;ky z^+KV3ocftU0nITaxbu+&(Kg1Ol$w>*j#tEC11%A(lVb1kd1P%{x6*U8LTbrGF9eF< zSct*3^Ny7-q}&Gn0w}4Z=i5(a+-u>}Z)k6CIqH3O3+q?QG$L&X{}SvLsbiH;I%4;^ zm9YEX&w7GY$)DI?lfnllbPO_E=ZNQMeXGmEb@ul589s|k7jg|ek5Y84Nw0;A3u6o8 zoa}bQZu04nOZw<0{2WLu0v-u`3#E?P1K)>Ym_VQgD#50g?l^+v;iq5vN1$_$pYTV^ z7eXQNgI5cyCZ)o1V%F|{_a6kx7!p)FR1VDi_4%{;T@}9SjI7*Yfe)A=z12&|`GoCr zSZiTy#(^0athAZg*aFBB^wX!0A^ZLu!flF}Z-7$+dF4>Fnt7F#l`EN>!a)1rd{Bea zcDP%DRBdq!XRtQ|rO~VD>FG{a-q1jt5$6!-<5z_flPd!L0z&vd)M;^cg2yfTsxSSZ>697pd2{`&g0Dy&4m?ZFG`R4 zxG#rr#QH#GPgnPe=2y1X+WQ|@v8OFk77r>ryfLfo`YTvy zG7aJP7+EdI)7HMIdElhsvf9HtBVX-%bdqyqQ<2aSCfRExg-l8%;m`hZj`kU3{G97T zFS}1(ZgS02^sc*Nk@v1!3X>#i|iX6fsD<#r@Q0M8BUSZs<(RLazkN67Rvu8Up z9`@`2>9|J%cfT*?NXAzPp(o|ctdw~Wb}|>G2S(Lrw!KUNE>1)}<5p03wuy?WUAV9f z-D0Vv7$^ByXcRdf?CJmgo4`v@`y5Xd*cf*G`pQjv9cXbgRVi++u0#PHJ{Ih=c^{M0 z-o5L90t317Wu=hMBaaCZ3yQo2J9g}d z8#K`AKyM@xUOwXmSGM(@~s|p`a@A4gUI3kQPkT;w}tv8SY5nW0zQ_m6Kc|{fV-V&-cUh) zT6_C7?d>XVZhN7crLOr1&Yq-?L8ZbEfE;mEmE7RT?+F;@Og2{-=R~2>`E4 zFE68iRGqwKd)u!%1(!) z4XJs&0ExYEJuM|A<)skP)@JyE8mzm{pBM{Fpy;wuD&-k1>tx zOAn`n)wyH66!kx!Mg+*)6 zfTWzSdosQbJn<0d@WM(;*(mAwi)7nYc>=$%1eqy6v=Sr4N0g;Y_~#`VE~1=;&S^6O zj(P_IOvBp?tYkRLt?gwTG;Y>O@U-hQ5)YQJOIse09FVGdgq}&A#FE6iw zQVB>dqb-M+x(Hu6ao`_C5EX?lW(ckjbxHSKuOC>d}I zdRA8&tvA$xq9Ay9j}$IIY>H~!aFc*l%W(sPog8o&=97{-6)SFAc`+TU3&F8^;D6yi#%s%|{bK{mxovwp1h!WIUa&(V9A#Z_;836P56N*i}LM*omZ;os6S z%R_3t7boJ|F3G5jL(7D0JtrEvn7rG0o~8V32#(GwCnLz#xp?y7fd(=csaUFa}B}Iqe{W z@A4ZQTMJ7|B^{l6eq}uxs8vz&plMv=@*CV`*X{={(l@<*(S!H&_Bv3|DhsNozAgLV zfcE07on4gy=cY}{Q0l()cms1(g+GAWx9{E+nC*TXAAkOHIgs6|zHz*8Jl0NirmbqM zH9ZFo6N}*Epr93$T($?ATeLVLUuI_;jO9>gNz1ET9t2DVQtv>D(qr9Y-3o|dH}2h| z3ogF$^5u5oY&D43d@u$TwmlFLbl|cEBIP9(AhL&aGPs^Q3b+RPZyN5ccK^ zK7I^*o&!~D>_;ocYPWPcycZJ_qe^XDjaP6nO=@EZ>g5aR%%fPV!b%o-Ra-PK@-Q>! z=O%4KWit%wjH|v2OdNT`BKtS+{CNFZn0u6182sO_ zSe5c04v(>-g|7qWh@G81V%!m$fW~6%Hy%7#lb}=X7`;mOQAL1xdeiXiU~VS{tTio0 zAVyrQ_3N6oU-hcKO8PB$@35BK!pEl_NLdS1QLn%GMdA6F1ZCH$ zf<<|ABO@N1v#TJ8Ul^?X{JA7^Fb5VK3<_J$d~0!*-+wNdX86TT-SIHSrX2m{ClB5T zzs2l!YO&^~l!C=QA%CIlcfz+?x)*bgi?{YF^*s=g^=b~jd_InyGup54r{8w%psMG_ z0wNSeRV}`+Iq92Bs$1qCs=HQo& z1(mm5)u+pV%GT_yQ^}&N%V&2#4O3JO4ry~Jn0vbn7#WW5`t|FR^%5KnKNl7G)$fKw zA4%~Fi*tLhW&YhM^LrSA5_OFC%r?jgw5N?%=m3?3S-1-8Cm_gkkPVswr7aLMpWH42 zb>bXcx6PSWtquH%Ne`TE$K4z=QRQ!vb!A#a^x0_{>(EiOVh)mq`w9vRy*_yRz}N7A zhQ^wQavnc1zj9s!?WpKHRV83j$Re@k`|*v!v~<88#3$0=p_DoNRwEPflkEOQMfKHx z5)~8>c=YsXvE0L>drtX8eG_(e*qyvZ<-Ul~BdIJdc6Q(I7uo6%l>`uERO7V7CD~WA z2;PsWuGT`k6l@8=_Skl|fv2(Hz;TD31UG%nUCZ{PyC+|w)Th*AxJ)QePge*~D>4m% zkVkx`wAW%U0Q+EolE0_H91B|V2j8saI!LRyAM8LVNp#s8LHeG&VE_dyZtc7X-9n<( zgce6`U=<@{2|bUe@5xs>hi-(l`13Nj&Q0y|UYvJXy=INZEIW`NbY#e-B*wTl`i%ko zaD@y+2;~Se0FgF|jO;DFr2O^{be>6ciapefVv%e@mNhPGE^qyQ#MIQ(;=%>U9o+MK z(IW`#AD?A10~drbz*LdvgziTaVJf8uo^f+NZgiA+;5sw54Kr<%-S>z38~0Sdf8GcN zKtsAM&zQShW&BiiZN2xUOK%1CGN5{43%z?-5L`pS!v{`i$US>mZIKMav7j$i)+j36 z<_3V208!tj6+2yHAx_{hkm`ZFg@Ock{j*nYEpS!1WfDY((O|lCfoUghOkTF#(G#1M<7K2{ZU!{l2Vrb6EzkT?S zdlP)!HUt-U<{$_Vn;=AE@5=QXo)tVkK(|TeZE{YM0*_`qD|TN2YA%n~j4z$XjD>tI zFd+C9KNN41lUsRZXz_lXY4;wUw#88+kM8>+7Q>Cx^=S?G&+*bn>5o<)=k++U@=f}a zDX;Gi8<}F;#=Vcfd@e2QlF`_^^3pEv_!Go!W-XJ$OYwm#$1l0y2E6fXli4Ik(@f?bqP%XTyTMxX!$!pMhQ0OF_T2(^2eMx$P(}+G28=S0wi1| zR2J7V1#CgVCeNxL4(;fXkj#cNDvia!xVjqrA?1}e{@gHBWQ`phzI5i`@j!gm_jZxU ze&CWBcI{WxQpdv~Hcsx4u5QMYcjZaFFn7gN-F$8%^?t+{GOo=w%r#4|Q10Tv8;9`) z!J0Ek-v>|vQ|03yjK@@QOWx1#A2Nj_J;VFhb;HmpMWTe3Ov z$W##fFMj`UEhS|arF6~e)ooxPBO)*Vnq)+YjpYLpefUO$3u2Q!tpBuj%E@^t#-#HP zf*1U<)Onwf5FHp{_IWw!E3f$CM?_SZ>k4ncKPM~9Q{8Oj(BmT{#GbLXw$5A7r!fFa z2*j(E0fn1iCPDt3B(^(;D}OzsRm{VaL0@@Qjl!+bB=-9xC<&lKz_Nw0^2pz>P<1T0 zaddIj3^<73Z&5~V_pCrJ0AXY`-7+Le=AqVA@z;(p(KhFJapo>A%)ae*1($3XQue|h ztQ=`HFj2v2?c7yba0GKKP}8Nb0yP&MV%Xv7`&N6-XK-GtmqD4k;I4yVlO}wx{`;%z z+`~PpO#i*g_)YbhRjWGKZUqHC;ng(~mvT?LS>vgjpkrIatOapJ1V4M^T3PS81_KwF z2h#3iTT!)lO$vR$d>)-@X;+s3V%zs|JU7qT-|18JKJ%wgan1;qe{kwEPSJ&Y;_t*z z3Z=7@$EPTFz9|=tOhbHMYnB^8(x&R>=Ke%9{;<1b!7y+1*N6FSd-ra{XM%R5Q$08j zzPPB8QFm|T8#75fwzRP+?&^9{+9waR3ut^Xx`*Dxy$>|!Frl6*H(Iy?d-2?JbiOdM zs&;jCWrs=wJ|S5-%YlF6taz@-0Y37ZGUY|4L`>gA)&)}%r0WWrwm~xJ2aZ= zvEq*B)0^-e^1n=-0VS0*0cT7iz z4yG`QZ;GuP99RH1s~H$PDD}bv20N$}4EQNWBV*J{#A*l_i0wC12K+nJGho*lyt$q1 z?BvvrL>OCHb^=>xBr;;^8r&-v8A#KF1@L2N@LF07T@uh;(OtbfSV}Dd@x-Gs@=lac zRet>3@O}>)UYLKv@#RWSh9;QQl#NHsdhJ^tKi(X4i&kEdsj9V=f^&*&2pcvN7!GKb zLjd`TX8cvdBsHL&sEewqt1mAu3@%P+SVXP|wt>P(tD?1ge@_g)W|hwH(*A?)6dnct z?-yMtAkUz65mZ-BHIa-Oh8U)n4aVZYYHn}6>2l4_&lspEN$Xf+BWsKc3*MAc9HF?_ zSQ;`y!@&Ya^qERZ0NtImGJO*@>ZWViAi8PSDx$$vkz| zJVn=7G73p}AkB%aw!Nyu5DSZMN)VGoW^PNms(TQL8imo%YR}JztyPZ?9NS)eLPJAS z6Df)t`JTJpLw)_ab$)QKz_>sp!b4Y*Nx%G!9}Mp*{Wm(c_~KBp}+7fS!=@ z4?I3LB__x$CL?nlQR3)qwkU{+sL)R5 z5?|lFOn~o}sw-3O;GzHCWc;Sf<}WD)zkj}m>qde6e7F3?_Wr#4LtR~cKND@z(RZ;B zIe-N%79CKmX7)iz3k6^y0HkDKc}ags_LuUqe@jV@U9Zh04Wg1@?^_?dx}J-;O5GAJl-Dgr-&^ z_<=bb$w~(Md=_LEM#6KGgHn|Nj)OAX?vNc(Rr2dn?9->qAyvjr*$Kzd;(-lS8Ul*XL_w<}iY89Mck(C0kn^UBuh(xr6@3sd|L z6?_KlTamxXxfC299SQ)f(1E>A^pHatqDV0GjGOi!K75_F%pxaRa|>p>$F#I|b>8FJ^+yTyjrI@IEYO@Lq`N(^jnP;@&PQ~){@e4~U zE0s>n&e1L^MT&`Aq4A5g$4vvEC9J{D12tYyD)N8}R998`a!OrP_pok4V-H?q1`NHv zzCI0C#g64(`b2A(d(ads+sf+dA<98pnPfdCR>J_Dr4IfnIE%EQs-2Uvl2L22v$Msb zDh8s8&)y%+8SjHQ*?%wYhee-W3C=EqqYq{CNid!qR}=lw)zyY_Dz;KyPEIbvv6~T3 z#3Z%RFjF5j9EedM&+sAtZ24!M{)Cr-=ApdT%0CGioVL>nQV?^ayGma0E12_MGp&L7 z9C5Bm*;P611h9!%Izx=+$PMdX7JmpZDy?qbhlFmM0Eb6l9`9>L-xIy|M&Ln?=7ql} ze`dFs1KF%_xN~@CZJnf!;oBfj>OVAZU|o9>a!?EbZv9hJ;~BK@tmF?uTRz$9&pE-C(4AF zo6cS$O@p-S5HIvMbMaSEiF%8cQg}*{{dYDaVSw2#Qtk;aTHY^VAZLMftam5#Xrk2h0$PbiV zNWc5g^kCPTsSY&`aVQ94nJ&TG!x`g!`SjT{kq zwTnwad^|l;PYEpMJh>!z5z5+8@Q+%kCt))W-NDZLF;{Z|-(hqNl3~GoW7|pK2TT28 zk;k6-6X@W)XMamocUPRkRWiXe9i*=xx1!g%8XHsL3IJ=5nQ&T51*Qj)9`yE>?{eT? zwuSE(ku1&i*AUm&wc{LYIeeAslV2Y zKT(?%JqM8hJ#m=F`0+9vao|}`B_t#SA33_)Wwf`;#N&VRhzbe`DJ9k8RxblTKDytB zI>LPpG`P_{8OmUXD#tLdU%4DMdIIXoqNs`=0ld>LudECgT3w-s3i6?>8{bQt<_u;; zc^1JDQ1`>4qc;%-DMojI^p&ujUfPLshe^&*TmCZQ^#e^rf6e$a$7Uf*vjQnGHWExj zEv=j64zFIjmSA`gE@Y~k>~Fk^KrwacJH9;NyST)$O?i2~^HjBvf-@E=% z#P$J60-j0M(?UZ-**T-WZxTdy!qoTAEyf+yy62d)HyMK~B@wyViKHqW}9lV4mJ@EykI~ z%zP+cLb50AoQAsk)a&X8IEK&=hl{H+D}~{TFXkBezb>5xhe`hQq&ix%oT-jg(Bbhp zU}O}Nxd7M%vh3^7{0)~Yi(^hr2x!D0GK50O>_g-42U{?NS3q89k1<1p#CEX(Cl?o4 zr6hZaqLJo1a7Ht*Dk!`mIMaE0Hq4AZSTIS`?uQ>c^i#-Q;)d>Mlm z3^_SC2zMNcEc?96R#TxCT*8n&+}-Ei4NSv;Ds~uUxAGzLQp>dKB_~GdBLNEI>khUA zZ<6KdeK}y|9Iv#uqs%$NYR-)<9OWj*HkCc)*Ls1GaxXpvKq~xlG9I<<9|#^TAc^wt zJuj#y*T9|Ev@|ULheJ>=DR?0>74P*Km(^~5#;LlGRDBSp&>vGs2S!E;{fLMNraEle zAo{H)u=Jba3l428N_O%i%e!ZDV;z^oR11selrOiEfZhd_K(6cy?Pe6)K_%!R=k(cmmr^fG z!Z4#d#OiO3VqRYyT-{YO(Vz9Ps)GoN7iqR%zFdhBMf8MM4iq*(R>d(cfzhLh7(2~^ ztS>MT@GO+mpzqM%n(I@oe_n|+Ji`EHOfFs0))w8mHDcJa!4u(d_3G7m){=#IolU`S z;*yeVOH6Ydp8|NVws=%$sge$yDyL-GerS1-W;8{k0*{73lKJ_H5VUyo?%mf;&rWSz zJwmiZScLk&=fFH=0p5C z6~9IQepPR#+tZ2t9q{JkX=CdM41BUVgwfsRG#dm*5SIyO~pQSHyw-H2RpI5=07T}S?ZyieBFB>X$@?J7kj#q@PA_q_40an&s>d(7EpDNCEmE^_F8uiJ6 zzd#^H!N%yv@F9gT|5vc-Ov2yQ4jDvMef>>DX9q)GU#|y%fd;t;js8L$m;C>~wKWn?CMA~un%xjk!scSo0BJeVY~cxZjc!W1 z9u`)@X5@L9_&Onj*wx3#x?KTO+YZw3&=FP^f;YGN@IdCDr0p?c=n7ApbVH-H^dshVB_c-+=;au zUjjev8K2o0!wRApN@H5mT0+7idjV~~O zSbOlH0WgbdMdmSq>W9uF_Q?}kxG0n6M5X>k7O<8895B!VX;93|7~Q2j-@wTk3^Pnuofx0ek`jPbDpL4h_F=zmkU#)a{qO3Eh=j$U zvq!bmiQK?F4!$Giea(0aX_XN1EF}mThUf#1tbfaU)mK1lfO@V7ay40r^LJQfyZ)A*Vo z41dt-#w5DrJhRe1pQg7&`xrIH%t(CVhjS&TySuw`=d~*F_kg-I<7?#ZmH)sj>W)}- zGd3%OORx@r7nzeCi00NT>pbyNn783#Z_-vrAG^ z=q2ZBIzXScI2k|}a@?~nI9pdw3SSh?7$qNkN4xljHljqFMI{<7_&$gs7 z$@wg7LPA#a@yRXq0&)#U`MlU~8+JNlgS8Nt(h!)C7Y2c77VjI(x;i&ECnz->j{*{W zNWii`QeviW9e<&W0XE??VM492MSw3d6TD-7BycPi4WLu0hmd~&ip>?btNSo#h9Kk@ zId0@_%urU-UG;mxK!uBd4Bp&{>NrC`Q82xA9RtS7yKs5A@mC*;`1u>HYo6Ke_qHb} zYqEmyH$rpo&!xR_=T1@m4?d}JiNQWcF7a$6*GaeIZN%)Wgs;I5)QE4X`DWY*Aaw&-CBs>Ag zo#&Zak8&RAfw}I^;UBLnJ*qm!(6=H~p429@cobx^iBiBf(CXqvV3$Q$VNL2iM5%<+ zQN2@M{$Z)+h76opB8yYgf*v`wUj9^4eH?Nu+SHZHIrIKLnyD`MaY+W#bYpQ26LAie zdc&MB3|{f+F(kqZP?~4-9gB-k8knBM?>p>wzMBJ|IGx^3+dXu@$8lLJ!vwnbH;ppBP&g; z8GI+ruONXAeS6cQ47dxG%Q4`7qM~fyUGUU!qad?4mF8Cc=3F@;hk7o~Puj^WPJ8PB z%E$e+ZhTWLx|cx17)qcPdT#~jiU3g9%Jll?yIW3<3s5I^$L#7=RaNcCY!`xy!Lj$X zQ7{sq(K4hK9V0FxF}Gk5Iui*xl}b`Nh83h8+qG*rtrDi9en65Dhrcf&$z(bNB)O}m z#2HveU`R;O&`>fNZgM5F>Mf_I_o#)!b604e`Pno*6;nt;Yu#!DWq>&phB^X=cY6@R zhbAFL#f%bCgzHe-<>xw>r1El>A=e=d23taK0ZW$g%0XF4;5)`CMg!$RgMClACVhC= zo{Yx0E1sfCgDLRiy}xU45Bc`)Jzc9ii0F^Yf9?4DA<@uxO~0kxMg#!NblA5dh?5Q^ zWkqH<0jJ=FfdLMPZ=DNRqU+3_Y&jz~xa!FFj~^Kl_1||?KbjQ;fCCb2+><9}W%HU` zjPwI7*}1bnexSw`i^8;tU=i?J!y7A`wZD`61&<9|89f&XfF?w8#0k|otjn<@UCZk_6lfe#XT=l^l z>{xZaRXY2XbPAlDI}UN*Y5|#+FhR65ps5{4k-vhEfvy!@rSOJTsGNHAg45S;*g%({ zvno_|8PKVP#i!2dp4*6;BOdPV_-;s4lkk1EN=b#+_wN)V`0>X@+`t>7^SAzzm zD6(=#I#^PP&EV9uUy?}y`J3>}k0f zq5JeBF0D*{PupNVl~l8wfr*!jiFQ8~?=s%o^OZiEuPQM<@7n!*-R`ZcCi&DeS#F!g zF>P|rGdb|8&_5=zwXh`ZJ;T~t{QBF}0+0V3kEuldsfHBp?Xs>!2`MRpQy>yf^nQ5q z1_8MC>({du79R~}=OG=g6bhnjX$KL@7(B6L6;?3S|?+=+bAY zxC<)dFy3-pdip8tqj2~|5bKs@j#U>bM6&+V&m&zEJPxJe;N4HhH}de5Vl6Y~A_3|a99uhgbw#kEr~;~{55UcROB*OMO( z?;(pCj57UmLzfN~K%}JD5IDhH8bB?_<`lpVf=8#Ja^Ty# z^><)b3uWUoj!+g*atGQeW3KA#hA$i4V7A>mrSaUKvWsMPePM+1t+94G8Kve>(} z%9B-dYyP0jEH?3%*}A8FW`1o+O4uST%?|cEI(WT=uU?(!dyBz5fhJ8GDv(rfO%PmA z#1a~D&g9=OP4yL!jsZOp8V2a^C_jKc!Gsy(bC@L|`_NV1*V<1;o*>;}4g%8Ms#S$# zA_!>v;0+1fkg4g%aZpcmNG$RGsbYjJIbs`kl|qM`Li|jJe@jT%gju)_8*is z>lM>~>)&zQMwE7nFSzaW^&{8#H1&I_s;WmS>8^namUxH1-$&?N5W_8gN!`o3X*4PM zE0te%-KZQszXjtYY{_Ebd0`8VPV+dz6)Eyip*rdr@{;I`dpU%qu}mwRWRRVTNUf^= zZj68k=g$*c0s|mTK!s*GV|L)DA@&g0%UHSsP&0`H6E2_^^F}I&)cRtra*|N5&i9Sb zrRlASs`$x$CP8OKO;$^-H861&LAO+t4$==gR^0!?`rZd>5x0UV(v()@d+gM5uUT7< zStBSNK`a$9;A(&drkn<97s+q2$}NH*cta=wRkD^Bn2)P|j!#Hp7mKv~=mdg$k^ZI- z7`w9#YQTxcgj%Q!W1LbPCboWo$Mf^^Q-ETsTU$k6f-npiHAcffy9Wq7I;T%>XQ`0Y z$b1bn34w#Ej++^1pfyD}pKf#Cv(}-+r(U$-&PaLapQ)AssCA4w`Q-ayHM z95%I!i}|K4vh(%GI!b70JE~Mb*TG<6IuXGI2z+Ejz01@WL5C?CS$SpEeRL*VM$j2l zv$s$23%d6V+9=hoc6P^ZZQ6t9@lRqVLxFg=vmrke8vZ0_1d3H785^ccP!pB*UFdD4 zoqkwhJ}UhT+dTq@4o`&Gzx5@Z|fy8dYfha{C zL@u(d+lbk>`LJ+}k7)U!a~Oj0c?>t}+eT|?ubH>cpCLQ?EG`H-2|H|ae6NoQxo`2= z%Z=8ms4MWGnqwS|BwDSiaIO(;z!U4)0B(1EBMZShFeZ5X^y!aK${L(FLTy1k0FXuD z3Czp>_iIE1jFp<2s{mfm6$2BmoOurpml*qMa;w3rgV)yD+br2~pogO9nL{sCW9X86 zwscf%>y+m|l%VRkND=$}oSN`MhGP+e2H zY~($yqMekO=wI`z2(4%KH;a;X7UPF@d1sx@=JhJKK4D$9=;*kT>d|F~(DQ{O(;Bxs z_K8rlzsnS}cG2+2RiZ1R&5Z~V#2sc&qtQ#FiDVMIGjiYOA-GKBSmsRsMZT;CX=oX4#x|`lO}rynO$$n9UC7T3oOB) zlwrk+n_DaT#l^(~OE6)_6}W}bFV+UAIy*aKzhTeM`}a4AkkHT|ORR)~T3srbNCdw@ z-&9Tm+|dt*zgXo3><8G*P1#p3EFU^MJ0!X|A_^8!)Z`m5j6ilAZUtR@;9F16R#IZ2 zUvd*YwVgD#P%LR1cH!EzDTHk3;Zw(&1qY+EwauOP2dzxQV-dxQ*74&s*RGU~(HPj; ziee>O%#>9VJDOm0O`h+2s>DQ#!v(?y1|*eJ89>#@2K%;c%saQgc=19v*-gFDh#98r7K)PHayh7RJOmd738!IMQ-_8#JLM{Js){#yj%3apg!s1{g?_wdA z5i&D5k(kBqt7D%p;vIECA6cqTs?W5=P{mNLUM0A+w%!B#8)S-CYCp;;N5H z0}}S5Y726IC9J+qb+NZ2h>8)GOg5&3SvhB!u~&gu|F>@Uvr~+#(EkUQIxOtx`CUqA zpUAfW&S3CF<+C{aXk1=MB>Cp#{DZ<|Awz;07Z=gl#KJ~R+HFk0aCh+vq)@uYgWeir z8rq5FNbsz*&81qX9ag#*0F{*d*4I~qvnLy=M0IPe#!F0}zBj;I&dExP5*FK(Advpx$= z``U~8n!<${HL1}bS6i)@dU0!6;94!-UD`oKOoA*2?pA5uzoUZo9GH5tA2(9zY45H@W?PVSijW&!mh3!8)72LUV4Vu*@s2aR91P8HK z-M|Gq$ZQEz=`EJ(m!NC~XmI9^}e=gw*G$P&KsNu zp-m86g1IipQe28mvO-pvjbL!gsmO%ez4PTcGc($`N>3TI;^`N^v(#bmhDCsFT6}}O z0|SI#q%486dmw;u`pHIfUgjl4S>?citIxSPIIjClhx;gMYuBDhIuKuD0@?%BE`sdm&(+zWSaaSNsyf|cW7Sqlkm_KjQOVGisJAwfcCVOEhK`FCe7l^ z)%qZz5*!+U>cf;3=P^EnRrUkl?`-4C5#C|o36m$VhOA3U!9XUYM-(I8>T1ee3V^{Z zE?#t|fZmA@7m5qJ;=~{H2~2aj0T@8LxTKe+2#q9xr&irynO0yeIXc;K{)B=Qxc!CZ zsnfql6L}eW`ufn&I<)?D-y{LVljxiL*^oMTPN_(uggdU_QC73Vk$fR-V|A&ULd z&bgJ-Mm9DH7Msw>A@N-(sNw`ffcL7k6VqPww=9Bso1kkkj1q!~R}x|r?!}(p-q(Rx zfQd#}o~TpEzKUT}7~14^lt!@tdA_}_{B@!Y9m`elNm`d0*F6{vWWNhTWdnc-F_`=_ zO|Pi9c;}2`fVdQlN5Ol6krr8sH=5N&asj~ua5|ra#NFA?f3G;-K2itG5ffvSV^|VQ z&C$_%e2h$kmgo~EhrB3(8wdq78uHPYdT~#JzJYxlJ4J7o<|t`}&vQRZAOEsn0+u?Q z!~fB#O8vi3rPpsaAO3h*aQK=tAK!}=B3u004Uzn%Sr)esDzdkgRP~;)Mrs1 zEiwuq`uF+^-_EUWitb@gi*u|EOB57u{7XNBDq>5t*iq?WD$&T5+0l0dw`m8}4)H#@ zMo*>@WYI*T%L+1)kY0j=h43L~+bj^p{n9iQl`bQo)bEwhn^7=s$-{4 zbtj@P`1Iob{r`XP*JICa)OSql*3o8uA4o)2qwP1lNC=4!txOGkBER)mTTbkUNhm)D z{fysvkkQyVI9Td{RFNe`gml1zvEQ(Cga3(1PaBw4SIfsM~q_rTZHOGomJn^Q!*XE zQbw$oS6i{<7!h<(R`yda+prd8&*)DL{T$EJZ?oMl)V)0EI+#5Pe|SDcnaG_0i~l2) z!p_4JGqH67?DRm5&Uw&7Z^6;T3gT^Fq$@e_2_W8#MWrC(_3YAf$U zpe$Bk>=rLW>}UX0vUvr^MXO~ESQL``9CP>1ovdMwwUH0d7LafO@PQ~pknlHSJ^TeQ z8rhv9E1aNiG$fl8o_SFE4mGRe&yb~>6@yD|k2CvU>+JXsW+YiN+&ztQ;{YVTQK!ys z;NhVF=}tb%<^28p9%W>(Lmf>8Ivp`A|Hwz#*Xha;`cmxgSaNQvPyhMJF^V$ZQ+~cm8nOI`h^O3`C_M8{bw&~r~UBkb1p7^=hfe&B}zQAWVRhK zXbnDeI`*W4aMK92d)N91*OryKrBp;+S5;4sV~K>;&xIpNxHFhsU`;Q>jvA)-i}^pD zz0JhfmDkW=Wfg~FSG@T*?9JeUAT++qltfa)hQ$h>nVOg^nP1@b1;B_HcwTcFmZZ^o zG~=ub0kY+w|7*f~J966_0J=x+{H%B;sk?gYSzVIkYkEWR_jRBWKD{4>Z~b}+xWst0 z*OV{(th5jZ{a8i22_Qd$vf$w^Ioz>uBX30vMV{w?ON;LH$eq7@z}3L|+0K_JH^_1$ zN>)RIQ@|AvFsd3FdJ<7QJ%0ZDqp1fE*7x#IDm{C!-vOzREKn?SkIFq35)wjU8tAv< z8HC}2GE)UPEvker0|i>Nf)6(k&6*Z0vR(!Ons6A8^tv=+%?+soD0#&cI~c6sBeAdc z7aULB9Q)J;$s@MX12alc^$IYd;WybH8Kf_whIwwHQSLxuU46YNcHv|3KkS`Xo=SbJ^l|A(#fj_Y}U|9{Ay6`3U=TSkgBMA@Nag+e41NkuyxDgIp1Hm{%}6$eAN5>em$Spbv@Q~NPpUq;Wyt7*UW1s zMOl2ht&fJ^qiuBizx&M6$tcW)HrOAz8u)f->zG}9C>Eq2h^tqOy!;xsRzx{#tpid6 zpb{u9k?^A1B-LtMWFb&#u*WJX;cEUIJG5B)=GzVu`!B}5?{)R(UeC2V_-LhgbE7tV znjXZF>&nFiaDeW}IZ!uf8g^qShDR3l|0;4yt+q<>*8ul08)|vgxxYx;yKmo)+slO5 zh-p>n=wpv#haOhAy15DEC^{{dH(lv)2*~L>WXO!IE1$lr8Jj)wSf>9wj{!9)>)y5d>PoZcvHg8D@H9fC;#Yn?Bli}38SnWmOp8+tV~@xGP_0Q7o~04 ztJg2mT>t&vpqet(`W>(<=PVMs(5Y?8c4;Y5GW`1WOCX3q<1_HUErJDNumYU;nuyVd3| z@!g%H3IiyZhTU94e*JPSEid0~%`d&$Z)k7qwhu8vb9DoxuwSOVqobp1kC5B+i3?^q@gJ#26KDkC>^im7v_QOfh4OgVeXcxcMnA;LIQ?0av{75#pVhrZ%2 zC-?sQpq^%8Q(R{Jt((5S&qD z?a>heF1OP*AzHs-!{Z76cjDOw+E3xa!O~{>T#CZ|1(t%8jJ& zg8no!yBka9Mz`Ee5lmlr&O}*VeLfi%>SM=#IpA_yO$Yo)elL5`KF6(XCHAs(>(#4s z2C1)tSTR&m{r2_iZM0Vgv81*1(sXRq=Kh_xy2tjioAv|6ujnjVi{35%xV<_b8MCl2 zD(ZTEj}YloX#ARF4)j>DqA5I_ft%aSBo9ajWbf~{owIw?kg$Qi<#>4=9d}4fjAMGe z_VAS(Zz>3M;o`+@ci$f?AIZBage5f4D1tPbHESjowBYrV((4rFF_Y+iBt!dyg3>HA z^<(~|Z+n*>GNs^0DEeCDerT2Y+K=*|WotW>8ynjhIbbhd3_q6k)#otC!Ct05Y=`Jm zK_}IO3U}YL+1f{nRrq&lL6T6jnJRJuDmmk@&hH!#=LCVLAb)Uh)k$}s=@DMyl;;KO z5Ny9Q)&f7B$V=# zCf(hWRDFh4xd>FJesJTQ&*M1FRf?q`20uWvRko^DisR{u58m>NHD#tt@k5_5(|qEH^8QLJMT6L-LS6usC8@)RQ9(Q zn0xg1e<=3(2?6cn$0gIVg?04W)vKG#Tx#`%@!QXD%eXE3Gpx(bHn(o}1e<>PSYR2Y zY~9r>5M|_2bBTVNcF3|EVnnzkjKWcKpazib?;nIF0aM9tYoH03u2?9LdNJPDAEE;{BwgzR-#++Ac8VCKJ+m8JRw!{?TLt5WdMp zLj_0+$vyLq$&AG)G@$qOt5XycBF1=e4`k_PSe%`eTgVQPOWyTlNqjw+=?48;Xox- zr8$>OfM{r6v3xgpzbm}XoZK0pik^a191`u0rsl8Myc^B+ZV~C@DAXDt6k*n``pY*mj+34GMdjF*J$Ch2(-F|YQ z>+&TL#N>z^4AK7B@VE%UzqR#w$J_-_AhC$!zf zhBI;l-16i2h=QX?bGpAm&)E3sV%68L!)6(S$M&CI>^hfCcFpxpw@R+qEKZ>FxBNa& zM5{S=$dLS4!_Q5{n;)3!HS12;oPn2UW?IXoq+$Co@SFM9%2D6Z!QP%dU!_%3&#F&d zOUt|~W+HQ zMMBwT!C?V8h0Vu6wv8!xT=*nf2Y}|^GM$+-myD`@)A<+*IBlKR_>sBS*3fQ4Af7vW z z@kpbm*~)LaoJF&|L~a5RldSIx=vel8-2VI6liD#kc)+?tEj+8+_jw4o>EGnO_yg>0 z36ZqyWaVCS9$KI4tJ~paNmW0RT}1ATSfxt8U}t;p*r7wx-1k2BmCut=Eyi}R=Y%~o z8lOfr8{shRvA3~jn}aAKpGc5*xEOPmbO8p0QJ_Z?uC6C{{a0hTUSHeH~Mm)`Wt%Lw%TDOnz|0l|LncK z%5pr}?#uw+{8pu{d|q%Tcgr-bZ&!nN7WVeJvp;3~`ID_p%4RB!)b|d2x$NUT>AqS6 zF9oR~-|_a-YsGpUDh`y<`b?MuAd1hiam*%SC*0Ug1%Wp|J8IgO zelbGb?p0*9ar^>Eb&(^|#TQ)f&N6zcb|Kuz{{q2FQZg5QUR- z3be0h<@=VtJ6vhjXe5hlj)2(1Y)ibUb?XH3GJpQ(%wBVuS4Hh4L?jpkhcqN)v_a9Y zqfg=YW}VR7PG{p=P3R193OfXdtl~Fq|Gw;%?(V(B(j~Wwh-$lvd8lN$NrXSQdbPtj zCU#vreaBho5g(R%qDo)vtjcui2c{336hi)^YfLhI-gBQSlEMH z)7|HIy9LS$_bbGS?DJBr@ItMLovmW|ipCRHa3(ecGAAI7n(<6BWLnm~q16&1xz(%t zjvRR)bIy~4g07@x&gEfNtPnvE*f4>Vk0>3lu=fnSzxs5l@94!6OsfgiCmJDdSr^||whtI12cPFL%|-3)bK}t_FE8DM>VomUKQhow zh)y|iwme*t4@H~TlC$&z(*vNHP&)g63guZBgP_dXO;SW;e8BcmVx>OY;n~ZVyF_@! z>o_%Lc9Ch|?j9#i7$Fr%kh?<2QgNSyfSIkSYvP>%TlJo-oT7tWms~m8Xkcdn|20yJ zn>rK3xSJ%Qu_shh)5$;)@H@Ds-`MVk19WwJb62eQO|cw}_Ugc)L;c_g3{l%ZK!W?_ z$B*rkLk^HOB*bSuNH2KjRU279ChrsN8i$K-hOWdi`(?`JJ(zv>Vk3RI|3MmTgMKd0 z9gp)=xgAncw1*NS2mKd=UgWB)ui4PyWWo*>98c=g)_&J*hr& z^XZ z&!ri3+T1m1ln8=Yu%Kl?N_kciUJOCa(QotL2roY{iGeSp*`yqAHUM>mTxN4A$RLtI&>FO(z3TLlc*rn&5*dlAI zNlo)NoJ6pk3p&N;>69s20|-j>zNO6=NO=q13a`Lj-nX3dRZqzMPe=KQO>|R=t-7!g ziUzyS(KN47r}#vXl3Yuo$0#v7qH4HyI-S(-#?dNR!d%tO>ILI+!NGl~n=08=h?OAOwBPi?~B3gJrYnEmtTh=khm z#lGGK(Ky?@>OCLzm}2+(N_#Oc{Wu6?LmB!B&s#lar&22v@)cH-+X}#3{N5f}{T!mA zU;n5?J!+2@cuk=y^9j!rUU7n)EJbJ*C>z=+uoOsUk#r?&(6WYK{-^9t8@nhPBR3OV z)Y{kTB6ew9ADQ$!K=0d$O(Hz%^-`ka?<3ciFH86}@$ku$`}nJH7+<&H!6%mUAaYN6 zRyX40g`l`5c26(}Gq=@%5%c`vK6vu<>8Qw6_+2ImoP=x8Y}T}o<0BI~N-cBf(V7h~ zJ`W_Uo3s?5zofX6^Km!VtckFGQ;ND|& zefv*Wio770h-SFm*m9hhm}8TVrDwV=*Xj@WMIN^>e6jQyPjlu&+aZmL{~^WA_0ce@OjWm^^#!FL3{*!a3HI3ry~1* zs2MvWbK>#dtFK5v5WCRrEvG4LXI*|?_0zCZvH49h$)qvKD z?aIu~Z_5}?VTZflWSd&WWu{w$T!yYzTRW|nTZPM3q`lqx>eyn*Xyn<0d(2Kz7|8u> zkc;M!mJ#NyjAsY&lff8?AoG0y@Cw5kG24*;{*L0D7}Bb#*gWE^^K0sdm1>8< z=LPGw^ycTkS+_ZZtGY}bggBBa6e0o+Z4hTJUw(SqIr-lFAUcTOQ2>sP*RC##1}^+f zYy%4yjDNaSz07$oM4Ou4k%3e^j7=i7ZmItWX(^CCxODjpGA|_+EBciTWACG)_2sU7m%7s};PIzBIEC;+{B4IA%|q~h z!}Yk!9sb53^XnpUQ6Yl=sA~>}DEL?ss)Akdeo^xKO*b&^*{$0IW^f3861M8Hk`lDM zuCtI7tMB!E6n(<4rX6vearicE#aH|G9sY#^s+Lw%+_h4>0~?FNV5?H{Hy19N2GJLy zo4D+wM`CEJ*%P}Vt8f=<4&D=dD6TK1SFdvsd(Bfy`@1o`YJB}f|Hfsqc#v86e~Q?F z!OdZ++hxT)+rz=E8>F&u-_*l*w9ODSh={6F)G|g|lyL4BJ}nM^adqz(^_3365t+NQ zbcQ@ohz>aBedOdBiw;@}J@w8S@2D9&ruaj~F4s?EhHn?`UP1WV&SKN&L=o$ z4KGGm!tANm5R+B&ADdjr_&ePL>h)Sa06AB0$vWGyWo0AiFOUUrp(hwFiNsx2es)p# ze51Z+CyXf5o*F)X#-Wd85>MfTbq6yT=;30|cW>9i`)RV7lT)l!FEsta=KC}`4b6a- z*)v9l?O2XUSHl)~`pT90CNpNvyoY4_&(;5og+2Q!5E8|!v9Wrcq&*{C+EUZ6UhPUm4c|a% z4mw>!jn83PfX)VKY(kWCPU~Sk<_0(n-Ja&X<)fH4lAbtep3q;@>Q+orP_rtnWaEwS zrj!C7YiLKpuc{!N(Sq|WPz>;v5xEwNO-(6}TOlw{Bk&OA`Db1{{6At6(FhXN^(Jpg z_7O;VZe5`f2}`%?#D%(FSC{t;(;nhvpKsJIBsP8nyG*-}xhGRw{K=rk<)xLgy<#b^r~%bb%)4kdB_v&>&GKME zi{{4q#C!zQ5ZXGJ_fFDj<5r4Y708axnhb9JG+rUqcT%*>u9Dp3783gG#5rN`ZS-}Z zVo+!YQ0D)A#dQ9nymiD&jX_TBl!FZh4AXiTB7itXQii)+cubLpS5XW#ldYmC^{(oe zLWZ`SRgcSCo<%&d-8)_yXyct_n)Ij;lb6RjZC$neWaK6GEoP~k>p>EMRq z5xq%?n!IXNsM{NQLHar*P>C5EmAhT-$wo+F?Wl&+rt@%Ud3(wm{fybs2iiHM9q$Zw;A8j^er+Voqs z*=H^<9IBX-LLIwy>PkFhQWE7rKNeq=YUP6~jMIrV#Xe`ljL)sd-7xU5Pf76-W;qng zA9yXnt~>&0wtUuHjWp)t!Vcc&{Dli|W0BTiiEIw4rL(w@pT@d=!`xJg8GwX@N}A5f#koXJ4OWXColNllAGZ z!*qjLVFhOpN(9r7oe2t;4*@H02#+TYBjuWxR!{We&iFh}&sl%j%rsCUjHDfywqfYJ z$b^>7RC}}`=XUy`9f+qy*k&w4C&EY@?lck9U=3?VtSpJm@!@D;z+ESp_&r+rO#Ps1 zUCp>UQb&})VYu}LxjkpUW9G)>7rY;wu-~DM+Nnl~d+7!wNd55P>-2s+KGKk+7i`MA zE=d}q{U2O>l}lA@5fKZqh`+Px-IFL6r~DVEK}cW#M-MN=4XI~m7kQQi%lmX|-gmU~ z(EeAWmTm&x$&G?Kd4h#S?!`z8Ho)PP#IqzcB)oebcyiyPY!hj1IH>3PRul82Q~9nL ztozjc-+!$4?B(w}A)xna^@jd%rUt65#$PU(AHHw?*YWDS?CG69<|L{1laq0Io1d`% zV!GAO$ogNA2&B{joZH=etCRPMD=~Shj7TpHvm`nW z_3?3pwFL$2YY@)b5?9F+y4zalCqj?MllzuF7Wtcy$%Y;u&mCD;<04rTKrecb^7Zn^ zMm9twThtdbHkrp5W-T4w6^)wER2UKCAZzYk)SL;S-*okOOy2Jp84Gd_D7C^(dwr@C_*7vYeSc^B zngI)&Z^kSk#Djo>OG^!xG}oJe-<9zBBJQ~Ey)n#)L@_R%Dhxd7Pf=Ly2K(5(&qmnc z(07Rea|F-^F&0c(1@<^bf!bvh6^8?D3}J^ImwIq*5$nths$XR;W|qdU{^lwx55}qX z1#tnUCrd5Z!gQR`IA`CwB_)^=VDDuR5v+ik z$Rg6j6=_z8wHjgbiVi`M4c_(!>2XDU%jUzrtH}KuIXtkr^6f!82sRvcA{7cJC#&{i zeOXpaL)-^Rtap51pgUgIl$-bj|N)SX1v;TAOcK$b) zafaUD#1Tl!eWl@%qOv?ASC3jcEQ)LMbl>q6&$5ae$2jr{POJHDg*Az z^nnRV&vJG)3X0iL+aE;s!$$Zq$d`(2y78mI}!AVN*3#5cSSxJnl5Q?r{ z`y-|TOP%QaP?a>fZkM~39EociK8ek7mhQ>8Rm1@UIhX#ZAAnTnJ|TW0H8DM}9-Sl< zkNzDKwG%XxLQuJ}+Q&Nu%9dBlY(CWfawzefeUKxQHMJoVVmXTTy!G7Xii|YWo!lF& z;&x&#s}qw1Q>|bIxH%#{E}Di@rF=CC{tXLx+g} z^>+IDF_%D$_=#do4c8VsU&wkQ{Dv-rtf##IN6pI=<|})FiBys&$ad#KdoU5P4h+Qr zNx%K4YHY&q>OSpTm3{U$(M?u82>6UNtI=2R)uitKPGzl8G7gIVH4YOs&=-(S4_BRn*q^h}?>yOY%TUQ8h1@3+lu(ESn^IhX z(*-I*LS=EK204r35DJ_w)HV9UhZO>^LRO4FO8rx)hA94q&Dramg++%F zM@pYsu0@RLFyK5G(x%oc43&fLvl~IkdI2{#Wr9Le2S*bnG~?Kwur>tRqZ|{In54(e zit#QyE}Q<<(Yf@?0k=2gnCf*pQd)QyEfwuCsD^O4F(x?4`5xuD*w2{OAob@IuIb&i zOPBsVX0Kg2JLcF-4-YcjyVt8ftVZIYrSmM`QPcbB9Kd%}U4zem%FCM}x5F%-lcVT0 zo@<2J?P>;#xJ0PzFb*7KgrhIr^BfbCZ9q!cRi(uW+yx7cCSCr%aXd}AJvR#Z%A)1u zh}oUsmlo09$XEE5&}Q8+0o%oYKgx=5#=4|ogggLLuA~xIvhw1cCKHJzJEw|HxaVU+i-FF*d=o)&RCX$GJYO}=m>c-EFJ;7$9uH;r2` zGTywCw3gzrdu0*!m;t^@S1rJxnz(UK@OighuMgh2G-~pvlCVpR#@oyyVJkLBd|~we zV)>Qh+Zl_vU$Hoxy?%lK4L!Xr3yzRTEuxESa_XzSM1EakG_ve6hHryhNIa1gsj;fK zs}TMtu3^jNx#MW{SPd3^&aaCKEu^i(H#FWti#+wv%8k?>_^F0Wf8JKKSvMgqIN&BG ztN{GO`1*R)rCe?I0VgEcm}nvgzPsX(ZA^AP7EU8W97$|{Ax-eO{yED5{QW#d_n7?N z#tLy2)3)7-rJbaisprLfKZ-6#bsz7QHFzICj=2pvtHMX96eeNe~tMl!#Q1> z;$wUMd|QrZ^;%p2eC{t9d71Z{ZPdNJ7?TMBj6XT7-FwTy)1e3x9eQBg@m&woVE#5! zN0+<1zgaqjtVnY0`k?-wg(_K`Hw)Hev6XOheY?vG#d;3034v_F8Q+{Q@eqCex}_lp z4fW=ig+FYGHi0dx7{=fH3GcL*q-^0BISzD+4k!Y9}0%RgxQ>relt2(~=hWpX;g*LHe{t9zNXPDEKNmdHe4#M>6Xf)k?-Z zKYJL$u5^gV>Pozyrr`N)4GdLQ2GMi}eJD zrke$8F93xW$#!faO7dXC4VN*yw+?)i>+n>UjM5J{@CYRXeoA5d81B2??9|hC1jgSlL-y^_j?3gxzvVrS#ECQ9W_sEdt@~j+YK#MyAB<0 z2)fN-oz$#sqI1Pm2o}F7JU(85u0=Mico;jW*m6h6N<#dv!l3rR?t~gPRKI)o^dEjI z9=Dohjk0^0c1b1C`lj!5`}yQ?9Mz}Ni-LJL4J-F3CdY58|l67pmpo)MQHA;afmo%;C5PciS-Xg2K4pRdBrKc~ab;c6 zOzsl#d((KLEX3>9T6-V zXOr+BI5Yjt(1PO$|FX+9-K8_J`Sy0}XXS*Tj{H2Kk6YAIkDj<07z%_%8=B*cNAmN@ z&Qd>ZllSe;v-r}hY*t?-WxJ6$Q5_zSFur?OoW4iP^>h~bVpk)&N;oq&h0{dCDWU_v zKKsSA%}jovgKu&{FSxqSnq&tA_5tY<8dj0gz+ADZmgbHmh7}&v55jYvk5tppX)K4e_LXe!K1e}k;1-IYI6C{g{ zUWDUq3TG0h-vIhLQBn+wDo(7p%-<})U;_0qU0o)7;tOor435;<=easOC`@8#BBt-( z6Gq6Iec88pP}GZK!oB7R7Z2`k16}(fk!QvtEGUbV%4qDBcSRJ#oH_)FDZb zUr2gb4Ivwh;6`!gw4I@lR zwYmPU0Z)z2^Edy7R(jSMhxuu3*#|U#zC7Y4_`-sW!qalY`UC7#a!yB#8~4@T99L(D zUSlsQML2w{8Qb}6%^2O(d`YrkVbKGjhHUhuHa_rxUt@~m;cSyO-5uZSSt&%#P}si9 z>h?1OX}wVA>K#!?>tC|%Z+@iNUZPW7)DqlcThA%n)@;Ge2kB}n-SW$1tBmu`Hp@}( z&}TGR9IH<6THsbMdz$VDmQ|5Ez95Tc3|nbYZ(G|>eeu!qPBF1cNkOABzX8HJJ845J zV|iADB*>3IbtF>IDocXuVU&+)3`dM6!>n9jFR1s?u76PP8Z0+qc^j@TyXvE>F*9Z8 z7$~=2+RY}2WixBcaO+5x&7A6)7C^##ilx!oi1wXApCgw-m1wk<390RkL3vnT?F+ND zjSN{Zz#4a^ym0=t*nAL@!1LbxRjWb#rQK8+1`zYI8HD^lxuNop1CO$-T1s-NI&uoRlq2)N%1L^WRoE$Z z(c);!%rxj1i%BBgxsO`G!l6)_TK;E=e_)&w847`%#!GqTM_51<;oZZF3~Qn#rykq& z`Eeukk{Owjloy)}G%JQSsv;An43dPmc~$0Po<-e%Md0JMzZE3gA+p`wwR{ayp9x%O z(V{`;0nCIrm`6_>DFNMs0I>6`Jb+f&bp!n-6ed`5pCY*Gew6(N)&EMHJ z^Tq@F#LPCgFWB(tg-N_T3wuUBxbo+mvxl<-KEf>Lp4gOqn|b_gEQuU%ZaB5&0y10N zvDOdVh*6EJP5_l({KAs|KPV;nyY=M{Cf`KK-!SB3S}WUQXJZ}Bsuzt=_1I}$q=MAO zdiJDm+^Xcc`nklD{R;iAWO83cks+4PtoC)Y&G;_Rm`^`sxsK6IY+o@N#!KMgJUlS{ zbw15xd`^=IVgnA2ISM0ctXZp94`>kbJMgQa;q}u$m%R_+4(ccUNJGXA0BF=0Y1U$j z7y3HP6$VZ$;o>=HQM0jsC(#BnZ!u+;UsGkU5p&vz*9oEEt<8{Pfe>WIRP5Zj(*itH zM0U=+;yj=u@sGm13~I1E+E91g1%4nXF?eHgix7^KHG(Ix_Sq^&q-)bY`26y)Wu>yW z9`7cg<{Pm`F{oGD5Diu=36;JQ0o0uUJB)&_pLSeVlQk65aW7@>3{ zLfW}#+#j#Qx1S@_D<-nJ6h6|OlS5o`9e!*aj|k+S5P4dI_#25rV2aT0_u#cRNiqHm zyI>`WxOf+JK%TJEOgpqYd&lEX6QMRlReJNL=0VcxRjUNU#-R9fdb8_@SwtqRV&-WU zxqKj(qT@9i-Nw~nswUGE2|x`5Hs_Hw%twbU#*es}RcicrvzCS*n;Buh4_U%&FSo8JPKAl%eeRKSbNt=*!A-`nhI}M?9Hf{f7jbT&r}PDeRwZ zvx}z|di7a^7991l06dt(Pqpdrz+b!N&w8Eix32v$H}c7yep1&mZ4Bvu)9?Op$u z%OVX&TtV<6fL55{O_@R-go#9i@vKtlXE~x9#*1%?8tUqIzB-a`;_Shi;vW-9VHcTa zV~DaAx@RDU$hCh7+jWnM($Wc?wr%!*v(EbN0CZMH2frGwHEFK&&Ocd_p#?%c=Uh{} zJDvWudPSa3&A@Au6!8^1Hvg)2Il+kS>s^8IGj>_P#c{nlz@D{J4=^~gb8(_e2l1BdbQAPe5(~EWdYPl9U{EZ3sm@x24-XIYZK8X6 z&x1SdgJi?FkQC~AJB5b;ym%pOC{?_*P6cCNb(SBThgxIfdR@#B2J z3ynTyMo|CA7-{_=%81PXf4jYmnPUPvE+R#SDR9bQ8Q?~bm7DJ8Srz_}CbhNWz^ig} zy_sr&4&{fMqq&#tK>DbVaEv(G#$RD)4rxj?hAGpHWzQKa_0ivQvUOfS@0t4g&!)FC zQZ^3#`Ks7q!-h@)C%5~Y2pM=aIebxnqfY*<`YSFU^ghQzCx+WK{91c(=JZ-MFUZQm+o@>)>jYQxd zu~A3>rklbkB4YH5T5*3*nwU{A66&W|Hc`Ue6y#MmPACVCxQy&UJS^oGwCSCF5Zo*p zS~mLaXkPESxSar}&z~29K3^4oRrFqHg58Yj6(D-WwId=bU%o8G^2QKAyTakg{YYEa zb}G}Apg5qHBuhYfm2bMXR$`nT%Z3aTTARE2h3OK#$B=kr?5n2?L`bFdlNNEv_6GzE zSw46qNG1G-B)LGc0+b+uS&y-AVe}Bs72oqOwBNOMB9cWbYEeHUS>>FlSGJ0KHzMc@ zZ5@3N7b=HJ=<25(+QIm2Kw76D(~*J1Uq?p(G^9;hNU^XD7x*_iO816BxlQax|2mw~YTuv@307l$p&tGCq1q5n(?#sf4 z{OEz4u_3%1L=WH+h{ofJ?8d+e0(=o<(7$t=LNl+IjIDxo?Xa`b^)7eZ8ZDF{UwL@$2y1UdNHepV_6Zf)ZcXa7 zzZTfH9KLbihpMr8NvH4M?uyn*LW{*~o2L<5*Z6D2me;Ewzy9}Sb>Oc(Z6>GPyC$}D z%U|qbG&pFO+nx7!Vk7L)0spd9&^LBf|9mTF?;roPcEwjs&4|4uU`5~XrC6%EaN)xi zi*#r+#EKG9POgg-0-6(@Mdl5AQ@2JY{A!b%TmQx@p5_L{I9 zhTtaXV_-*tz%LHp68kpDpBUb;O8$Kk&!0InMQxS{1bLLq{|xXd%v|- zVOA{c?VD@%lb%0}hc6|7w4zdj1~#8xzmae%Z!VA-=-+5xs*@)?Mo0`@`t#iCh| zm}y-PE432yvq*jdrkhobr}PivS%s&wu6-UGN7@4136+dhz{q*zYFx3)QJhoF@T@le zKU=av8I*8@-B{Ky*!Y}Ca;2}6toN(;DzB@#aL%o;XMIrjR?8LTRQhZiH1MI)*;dNI z;gf7GWSlI2ekJclriW{Ib=}T_*n}Xb2&ahZQpX630kQL&G!L9``@rT-clz~9wrD!7 zyrQMuTn#;g)~{M?^|o0RTbP;MXLcVRdAGdU!K$j}ql4x)^=2(^x|O5vQU7f@nWF8> z$Y?Hce10stUOo@_jKB{Z*TQrS>~YaCHNhTE!4s}LFVQ53t3vNZA5;LE zLBAt@7Udz{!ryEV3uVcvZHqdL49tqL%#2Kvhp=O}#(>G^X|ZzUy^uPHV7E^rsd^uB zbN_Lc@dxTr->f_suPG>Dkr5 zv+<3~Q@93+m8|RJ{n0l!&$(QPZ=f|75yg+Gb71G)^JY$qNzmphM9TO0U`adCp|YPy zF|+8G9I{B86Ib(+Tx3MxPt2$bcJw$;_i=u#}3_HEr9wA@<|T`|a$btL|*j{$jRk{xB_$K}ktTK^BD6&--1? zFO7ULn2SX+Z`-Q%ASH3jyy~S%Y#PJRa|cOvYvKSR>fsFm3%AcYQ*zC>L6#-P;8bt; zx1mQL3x7##xnbtGHIcq5ohOw;Y0#i)!TLi+<7Nl3oMythm`bs&)Hpo+k^Se-pMSER zY`LxNOq1D$hWGh2V!~8L%p2I5v?|%!wJmWX69&&ZYzY8EIYseWUfG?-O=!!|Ho6~l z8HRrBcdEs~$7BITfkf8Ls=Wmyhe$;=Yp2^umw?N zt+{jOa`*mRwByyQMPj|)g$r#}{Jw@(+o~Q63~bi`8q#cSt{i&#TGZ`(_oSuuh?X^- z%fZvny>#i)W@nEG;j>g@{oiB}ower0)w~Lq&1WM2Xo!bA2-7E7`9fBId-Pwj{ewRp zJ8@$B{S`8AQDsD3y7Vrppd|0E`IMP6+d;oeQjv%82hhb+CJC;ZM?p#4OVg*&tm{#b z#BH0W5ky~)e;WE4m~@J#XPv+Hto*h@WV3j2z*%_&;&A`_lcZ~wUM6%GEg zekv-(za|9FQ`gn?6;R%i=;m4{28f_1vg?lKm`%D>WpAZjRE}LnGN=uS?(tuo!zbQ# za?k&jT>eyiep{#J!6qAKCMPbM)K)pTZ0Pjgh{yE8G(Dd{pIj_e1Lk;JM?0C@A9Nqba90 zT(kB)$39lEApx&efj<&>061U4poxveNgl`-iGlbbR@8)>CTOtKbZC#)w(4SQl)WM) zkpo6_4ja6DtU*I$LtRFSIql4uGu3Yki@PHW2FmcZh9=ZmTU;9ko=l@RKp;^F38!$X za)I^;;z~|S^Z6!rEuk{dKQ}+ArbbIJ1GKwF+K3d9ZJPTTo+6bV(T%G03Uhd7#822x z_&*AV93JqW*9w`l#L-DSVe+&$avhOF$CLhZp&01FB=5qJ@85G$WBhm@(GqEBOzNaH zpx@y~E5$q}l*aUFtz{#`9(ZOC9+sm8S1nhqT27KO9*pAUn##(i!${?TQhUHe{$>mv zI_i~WB;LrXgybQg7pw*nZOPU{NfcnL4mvg^*bB#*4{g=^>gsxBC^~q0Dk7L+ot@ZD z{qEg6oBBPKmCpZ|Is5kOx3bH@hzJ~P!h*DH<3@SOaZxT#fKE$yE-E=WALpz5UZLZk z_CkZS@n~G^z06re< z&y7S*Np1O(Vpk0weLy4;5XO>fm#;Ul90_KanE*I;f%ay0-97#o<5GoZtKloe zU@3BG_S?FkdkkNY?caO#AfbEwk`n%3c)oPQe*uwj4M;-%^>_wuTxyPVzjYOKuXo?d@K9WNWJ%>J@tM%GzHv@#S_vFO8p!UdF1AK2+I0UYhDB3{Teo$BPm zE90}-p&dm=0g1PNEoDLO9X>3S@xp&G^wrDtU_%f%J5@tksBtbZfpEtyUw+#m_^}um zfRlyHu&!RKzSz$xHuOJ5Lz>!4EaO##@#NJ28=m28li#AhDJ4*zSinPwM71 z+1!4UyM8BWL1aiKHcOBE@83(uPPz1A-?WxDc+%)kri!K9Q5{qK_ zCQx}r)G|EwHoh{AAwxbhz`LNDc$zSlRhp6Ej`S@>WSWG9g=xJA2n@UjiX;rnz$7bh z5xvn}vICEY;AbQoy~W~DiT5_$2krty2pTUVx5w`L99epR-2nALYXjJOxD(Itq)zuT z734d27Q!8^D~(T7%;c2RR2~`y1P^eSGdwq04;p;9O~deps^jyRmijuMH(V##a~B_8 z|H#>?Y_??~YyQ=o@J}^k^=G?S5Jw=?9O(fipZ~*l#b^a?{}(tVq^=E#kZZ&iM~5cwvS z#zW9K{B!ocn-T0sq`oru+?6cUz=3jJw|uc({B{w)DYt^F7sB;Jq*v{Qv)7G*Hohcq z@nT|a)uB9fqkE|Q><;AMe9p_ORS8P5l-pa|KXY`N#=4f9soRMGBf!U#ax1JxbVHjj z){8PT2b%&jk=MaeWB2IjX!N;3YAQ~XOXbf#L@D<_PgGRy+f6xTH}fi|LtGQX z*n_vWA*iFi#sM63#-U^lM$q-pzI=F_evrvX|3LZzOUeQ6cM}RzrD=H!&(B?GudQU! zxejHMtaE<&Rm22W!p!DwIZO8i|I#(S!$|~z+tFWmf9r!0ykqY@f-eJGgP1-E!csv z+w-JSE3C#7kwC&~C0mwU?&+k(Qn0PGhZH6e;i0a+G3CcixG7}jD}*468Fz}LiQE-u zTQu@G<;wfuMOB{pW5ssvbaK+V! zhw(nhq0l)}sQ5+aZ@pJ%M;?kGKLTf+IqQkej#uGroW6o|JfKcc`)5OR9RQF-0y%{n z7FoyRZ8ic^-P%&WDeI>P~8Wp0?QW+j5rh#o*&#TJDmMpB$!@EmmF89LyQ~ zN`BkaHb_)i08)A|r|(=g9FZ%>-U6knGG)~}Rrb5?&DYdTHEuXjMKAt)qTJ(T4>W2# zcTH_tWZhqOV&`uD4hgQS(5FfD+c8k)jM=Nu4h<8QS83XN`)%E7V!Dp2Z`u_{J? zB`51M%S+f7Ag>ivaNARPs~9~5Q{$$EmW2Bd$9v?rTw#p{GOw@6)0K!3K_2NKWvDWn zGU>$OJ;W+kLaVni1k6jhw!}I%D+G^3r(WNC%bBC!B$i^XA&vzApPhz$bXL;)6 zG-BJ{b08|5gWM+rwZmzNNny^M{l3qt9>7D1{)U;QYeYzfHA7T;MzyPQk7Z>JRB-<= zCRi+6c55kHEGFscHR-(j@NDCmL%J#`u=%&6*zm>kt)!|t`;JNeTvyi{R_IXneEqtH zTR=6xZmZHVLY@eV5sF{d3w;0n-DXv*Hf<&%DQH+PB|7lo<*&dFC5mdSO;fw>u+|RR`C<@IQ(rA~kPkEL*p(1Kasbh72BjP^I0lb62ls<}NhGU!HR!v8_p3xeOu- zyzbRxatzhXjFSbm;cdX?vh(e_t{I8QArcc#qOcGF-B1C7!3LLrMbUl%{f6dfBAY5j z02uZHA#C3NQkN~~xzvKY$;o@pCNLJW_hb5@-6u7$*+WPOhk9;d%t8~B&LX&L$&$G5 zX(8E&54T~j&j>N2N)gVxda3*5#x7;kd`ynmZg?iew#Qi0qaNm@KQ=%U{M?cnnFIGg zOSUm-KgzxNL<+KNE8B3YepFbG)9sM{IDF;b1u4Je|MYjioR9n~`)#VSOO$oHvP5@f z4`t>HU;kdCL``vas@ou$^sJSe+Em7_?(v|=B`&1xl(WVO->VB=U+oAh zw_Y=7)(vGgCIV)O`ygoWJaA}`dvR;w-`nNi(wuFY zL=WPFX(GjfW|Uxb3)eCQw$5sU^Q)Kad$%mDqBC$U7iR0H&aMd~6)!oO4H7W`4T~!l z`75T1iLH#`L_lH1M1(R)DKncRoSQOPC!2J$d{x-dV( z8B~q!u05Kj9_{c4et`myvisb~KS3|5Q|k6##hXDeQVaBr?(a2)o|lkQ{=L#s)p`dQ zn&x6ZH7ayTMr9VWCX(d4DxW<6glt3DpRbZGr&#@3+{(u0HatV=5y3jEDP zt{Rv2{CVe@Dt^`E&lH*Z4kFw_B3vT^quAUYpueUm2Et?h{n18Lz;SRH`p~AI-|Ui@nFPZ`+#O~B?bMx3XM3dyLlzV} ztTh%(wgv2QiudR$-Q>)y6L+3G*?;yV>QjEH8HaXD-9o|0>?BSd=^aBd(-7EVBEe_U zZ~P7~cn@4X*&#$cY-CZB*V4onjcY>F6L;TlM%XM7$lqfX=;eX|D7%gv`K)t>P4PGX zn&xdaj>gCS{hj8%g?hSk8>HO>JGbdQV6h_LUX0cdQaeTpaIe&LzUF_xz4+s08ghLb zgNX-%UUrt-<6qakm2t~H%59`ZN%geIu4(&7`&rw{>uzo4wjEx$UD-636P26z`azMW ztF-amKaMyBft%y3dxspLz@I@E1t7d=SjO%f1rSqI%~SNWzn7)moUN#fQxh&?Bp15Z z?N7fPZvM)TOeF1Bp28}%89E*;prROrFX-dq4m@AzY!J?F`cYMX)L%4s_8AcC3sJ@w|u!1TXFxkCjToW)AM zC;$_0+3;G#MeKucxgdeaK0yP$lsbMyMc5tVX~kf9ljHg{~nPu^ek*3_2F%rFr{?QY*sW zh7-Cl$o_Jf0ZYXBKhc^N__K4&%3)BuPk4-=ptN_oGP%3vB%_I zOcqgYKE56@%}4FL`#wE+fTpIY|A_ouzs;;KzC|_h%6$4%xd;208=|GOv|dNLm|?7j z1l0F#diVSHp`s2EOGF=`mAWMY9O?hUd9JfgKP3Z=_t$6`eovrDv&XqJh=F6c`y<0z z5Kb(09inMhpB(K`OwTSnnyAb6KhBA3GW*mG$FH&ooX?6?6ySU7m)7aj+%7Fu6B%60 z<%z}7hM4l8sM&3$T%`l&*mt;D+!v!reU;H`)zP&!Y{YQATd}9dJ=Z<{NxGB1-7@Rj z+wC+}E{z`4dA#gICG*LnPk=hG%%QokLS?~_3lLk~v0Rzo72>ku8VR`$4h}jt3y&Q+ zvI8EjBmO3*Oy+apxRBHx&Bal8xUt(-@M`=IcA-;2&0Fu<<%S7uLzC zmqmxULZCHdmL63}*0hXTh!9>$Fu~L0c4$Ew14tSTqrn#$9Q})RF_cJ>3k@6tjvp_| zvS11!pYDboodw^64_hy)*u-`KxAKctz2^--nkzIaUQ0|u(gg@0gq0{K)8D*_`&jgg zD-*m-RwxF3hc~T@c*}YY$yzIcjAFL97nyoHvLzmMTXg~AF(J9Wx369eQ-ujGid#lh zk5Kn0>`l&b1EeO9`wc_yY`e8lVH&uZ5Ru>F+M74sUS(?S*uK3F=}rWtAneTl95=*B zY}69=5ypTcTB;{lnT~=Wb9KqcxZ$EOjj|i1n_40nj*8>I{&zUq5NHt4=;!U4IuYbf zgdV1k`gp%ZV!20H*CnDLeO|L}wfFpqQtQhe4RsBZ|7qrWZ^(d_^xb!;r~+)>w3c)% zFihSjvrjSkQ!Fi27(&qBZ9XGs>E!5m>eQ)QtGn|MfSHQ53=w~$cJ(zu^>UP!;Id)z zZJCF1)Lh$kkWakI_~U)h*);Hekbj7zkE8XTGq^V2SYLqQT$PEot<*uCg?m4ot<|sE zcL`kq`;R-tXc&<3_>JH94@?hWpca}4gE8m4m-OUWIsq(Tla7TlJqV zTDVXW&-~&A>wkgR1r&$~LT8rzLRe^_!VpP^7RT`*VaFp&rr_x7?6PI9u041NCeE1g z7!Ym4IFU#QPmp+Grc-~*)vE#4eZlIlR~+s-_d?CZT9-DY|aM z+a}QR9Mu|q1nM+)xr!a>23f0zx_1AFj9-37j;O$s&#$S@srL*IB6LKY^NRf8nwnWY ztuw;}s_ndD#l+x#`q@`2tGGdGza6VI%^+*K2b#maOz#w&mmhL9kfA-KC)%}pQ0x|>#Ob%I8G%^{>EZ~b#I1xvt;TsKf zGlK+dA;3(Qn*!u)$Je&Z((-{tJR>mje__g^HS8(g6$uScJ(+lnq^gmSgrr~XDs<6V zN&VzA%6 z_mC-GF6@cLq+$A@v8_z02=RXSjOI5raNHl*ReVF*J! zIPTGZ4#`J_x{NOWUNd;&k@x)fD}vgU4A=WMkiFS6`XB8x{##j(TmC9}MbR#8qC2cJ z_l~=-6~Ec*%HH?dOQlN>sCpT$V|fVoTJYx`G`=Dkfz*w`1D5v22#PBOexx9R`yI98 ztux~gb@>QXuETFoj+O0l;WwcKqoPKN;BmEFpP}p4TSvd@H~j;v6MP=jJ>TP0^wCFR z;8-M*Lxn3lwxLa1!W{Z>hajnLK}cK!#;8m6yWw)EgPoL!t_-^@A`4mAZX@v;or zBCSYTFV0*vL>53DvY`hbBqz5M!G%OGB|657^u}w~oR_K}U%HO~5Abg5M-QSIU&J;V zc#8g0Co>HRi!5>LG|)$v3~1owD*_&z%;wWy&!)~LiJrCF>+6u>l8TOh{Y_+f)*U7q)z`s|qa`KLBLQDU;21s1;kZo*VALMGJP!e{L5!gz1s@8BL1# zwo>yPe?2h~-Ix4cq^X?d3Y#^COz~Yst|#l{Q-#)U;hAUlufJO$+MsFdlEGHumk3 zP&P+Zo@n$ zUG;9acYhl5enab~--42Z?H@XP=%#FXebLU%h2sYpo$R#ruRou3Rc}R#&)D_vr0Npu z3+o*;20Lh;5OJ4cHAVHPVt*lu<-pKdie2H&riT^|Zwufn$(E@~^MXX!by>vmF`WXP z+Gn<4T&@j+1bNb%If@vX=w|QH;&y4J@_uO%GT7MnfYvqGYidB}0*1fKu zBqLKS;+vr!(0nt)AG2$3w0AZUuPUK)g#!q;cK-nfnx9N~ffI5!%^6O(yfP>z^4Sd4 z+}T13XpAjiBD4nLxKjpA4l$)f5-bJF15lIBi0-@iFAI92M&4W+J@*ZJo|4N6 zyA_=K3b+zv-)yv3QdgtFhlx}c0gS+2-1_f|+*@phN#Y}uGoq_3}P)GN~Lh3m#MZFQYy zpLW=={IHyiQSOsoJ3jYv*46&7U9JWE-!hTH=TZ_T328z&k0;{S;BVsONS3W9@fPr5 zLs^xt*(Yp|%-xk}wfk5XI}u~d$6(}eAAgP5&8(K=b2j5c#nDf*)URbp>upX^sdK+H z_JRq(qkv&P+ZldibhHZe^UFQj{t^p-3e(@*ym)bDu-fC2i>+Um>>WgN#B)~YH~C=p zN^TdClPq?cGSQpes$Km5_<9q#ob&GgKZdby31bVXG(<*7lq}hjC9;JqSyD){v}Xxr z%bF#MtVu#bqR5gYO-NZ%gb<~YN~z!TT$#Bu-~Z#^qsQFt`yMUV^?AR~Ij{3tEX7*_ zJ?D*>HIC_y6dDB3p~gW+@$9`cr=tn`S|Dn<(}giIfBp%VDFCfNq}bYg%D(Kr07w|E zxT3r0!M`j68oQkx?OTGX$?yPw-`>A^h*hmM(HzPB8BwIG=1f3a#J$*o^52 zPZMnSb&50_b+y0&6G-iIwmZmDiPoQ35Dm)DaR4_+hB@ajP=*QS?jGeCR6U&uogGi* zwzPZi-rt@Xp|WiGGZ|sjs3Bv={@q!9dNj}(Yt!z%dc8dZ!=pVe-7)6IFA?%~Xr%~D znDc|W6#M4Q(aOMty~|G%FxYeSh!Lx?lhUm3U3*RYXIVA;{RabO;5O{XW6FRAsX#t0 zLUJm)_4oL4T!?e

    mIab=81OoCs2!zO&A4H$^~%Eb=7SU5?L8SR=UI0m^tJ7Ra> zb;CYz2@*40xF}fF`yTApK5-vcONX--P-IH!#-6Z@Q=4+J;)WY2<(`iQz)3Bi% z#G;eKlhTo}45D;2TBo)p9!m7XN2?NX0*?_Fi(RLD>PDRa)GGTla2U+-lH2<_7c5 z$(=iCsM(?Ox+TB>j@ak48ROcyeyR4=`C5%QU+kwFmpoYdF4jp^Tn3#Z-U{$m^&{A7 zQ9_T=RsR>2Rb-YdtUW%^?oNm0Dr#!cJ6oOc)O?e?M{o1fgME^Ec{lNUR4d40t zExybxtTHPS^ITq3QJy|l%Qvb~*BJmJ5F)YH$(GKo zL%)8{v!<|kkRl}`vFD?btc#9$M=hDDqqcquJxrBG^cGjTX9?-ak*{G0jK5_bz zn6bav_wh~NKvEP}$I4g)oemhq&WP@IitRpt?ucF1P6$U0_c!<5y}LvI{+5BpcJmmz z6dD>HELr&Yd>(X2Sl~nxt>bH%8M6G(jVfI{p!Y7<@l2pGplKOPynG^zhYWcIDzIFy zEac3Yp>m>1c!Z^%&|wF>D;y@O44eY4%o#ia0bZm2#jCd5)R-CeR2J;)y`q-nzagL* zU*Hy8iv34lx&jsgnqfu&ZqluCCon*`dV>}&Jl*I~lO{WCZIjOtG$p^kuzxuK9#TE#!+$^k1J~lett$9Q>Mkt_ZoPB(0(d_^ zH=~>wOT${N6jIrG|Ab|2LOQ0kyS&$;I1mYi+h8S3HhkA`3--WVKr~{pPzd?|g!Sg* z@K6bx4J!k>5DB|w3AT9gnRKI~fR!H}YWRG%MJ)(G(0pvg)&UvI`sN{vh9c$iX{xp+ z_`9E=8K&8S@{hl$8Dkm)Xavzu|4+c_6;qn9-af*C(jUMPKSUk6V`~3%?4pH?#_iLuz zZ&Q1Ac<;B*xp1SgarS-R;z?t^KQz?$(M=nFR^PC7R^z0Pt16iV`o7~f)x-Id zRY5&qKTH0py&|lf6kpoUn|sVsPV>$aHFzbKHLo}_A3Z!?1i+~8aL2o1yUA}q1OE5E zPwD}fN!-uKUB(N8G*@p5Gns3zdxvgXK>)Zj9)iky{S)S$B`RiC;4!4^E?1i??2W<< z=m9C%jgv8KK1bk{xAL`?2x*Y@J6?Uj(7ge!I;+qW@XYcbxgMKZ`TizX4c3Zlm-n*& zNS$9&UgqTEOemWqinD@VDRw`!T6D<@cOpgi^z`@A{b`G1?V^8(ArBOTxH$kG0KB)a zIFBy*?WdP5n+--5FlqK|VA3OJGfNiaH$Wt~x zvR8&>=(pVq) z;P?-&Qs7~{^E+ZN3|t-i^)&=|ZYAR6yyD`^%Vv39wa#twiI+7%CTE&cG z`Bycsrz0#mpzknr*@Ll_P}*26%a@bq84f4Gf^k9+MrqGUF1isDvnJLVPo!dQ%_g|j zuq~xM9TfC6_rn4PJcu+}r^xfjAiI9br7Y)(H|ag;+r;>O(&Cl$h(g5#uH5t>z68(q zb-ME(WB^Djj*(0*AFoNy$v+>~hGbqKzZs|?o@Uxk@3#joZY5kG^1zORRWgpBIAQ

    `L{r?u8_~`koehEUb=v3dVvuEeI zdzUZCS5ga{AG52+8{=z^gK9zjS{VLnTt)rMh3Y%DCj6OE8oM*qPkFuamSq}O=2$R03;!5j;~BBj9gx)v@UEJ8|NbGKfuNwE9dG{Z+<6T#bJk^r_t^r%xNWSg z_QSL3Cckayxx9CHb=}>Od1sBfzn}t=Ve!wrLwE1~R4Y%oI{({<8psumjQr%cCsXZ8 z5*!>NSb%EVylJ3BR~3V&0@8}js7e>(BZ>t(`l48 zZ{51fv&0u?XZ?16UYN^5l1;n+arK*B`!Osd10no+8jtY0ign7YrAx<|Bd7HRhR!eh z)>;V<7g(gBV3SFG^z>469vo~UofR0@CL1n}_0zf)Ir;wNy(t)Pm+fA@kiaiu3Yw+$ zY&B|tQM=G-5!KOdo?=@apZmbpq${TvW~}5x3f3m%6Iv3hYiRz^8|c8Tx*=R$hCH8w zPvwVBoTz0~Q5JTRLL(0L9l!ts&22Y8Mn)`(ADbuqj7et#cJE$C4HAov^FLghf(^)O z6}IA#<$!dM1ZF85<*L>OivVz(Bs|fOM3WH(xETw` z=K_1(b2X|1`zZE0i04Usschdi$SJBX?OXoWQ~z1*+|}DPuGTM#DYXF|_8it4z2(l~ zx1%-=GQOayawq8UHXXz5Yj0{~6h$p~(e|89!~rkYkxE~NRn*t+UF+MT+8gpCQqF+a z;%k$a=u!S?I*A#VS#mXO*I#?!vv3W}s9J3q`gjg=KfAyQ=S{izkWF@{$=WlmKdc;h zTIuU821#}?dp>m&wQsHti8W$z1w&VyJt=4L6Up+;7zTwrb0kd5(+VFQnO( z-x;|NJ|dZM*$7#vUEEuCuH-k8r8>>Q!w)P}mhjxXI@5-oc%-0Y zj~rPrs|S?qMoj5K7%Fm4iixRC((*Y-4Bk8OQV|-NxMr>lfe43qXIj&M+?Sp(-9W}9 zsLJ>G=FEM-3G8_)+1Kitno(biV8kN?Ps_}z2$(ob1VaW4J);_{>A;c9@AG>TQLXPU zl!y(QyTJWM2g0U|0h{^qCa$Sjh;VJYtG`~>m-e#LDGgo(IMu=Bm&pDaTU3h136 z5-|LIl$}}hiib0NoFU8EJVXmu)&ce>E?JR`W$>PH*HMFL`eXk zr-M#vn-kf~I6IA(pWEIG)URvARkUi56P;ca4gFCqE4D+vj*efutd>%W-h7z*9|gqr zT&@2&ts9SNb|=_=k#)nD3%U#<9B_`(#@ERsyM^gb?~~kXo&SfL>Fd=3Ctc|`JN9zg zx+5){jP3F!$JxfFGfJr>1n9!cI#ii+xJ~Wj{@EA^!hg@?8n{~cWa^H$I099H-yF5u zwQ2Kp)6khV;l*xu=7o)*sk7gDTmKlcN%dB^xtr{|Mf*hYcTB(DuJyX<(eBfGP7F?V zPj~!E-l%HybEr<84z-#%di3bm_xI^4IJ{7;-%koXr}vg#8c7L0{gcdnQ1D~lB?0E% zJ4&7~#wfC?(tlF|KPDY9e|O?@G3-Q0o|6Y?M(Zx58yCaS=$4F1A0C}Z!fN82HlQ)h zDZo)wjW`7 zyitAN5+~)IW~;Gl&mQj1K{3=HU@$zlFS;1?=&=TW48S>)z4tD$GlgI+9P8v{n09d~ zSE9MHaDjmG0@qsfyD9tk8X+>p{?*bl$%=iy3|%j#>BaZq(Xa)|vBpzjE& z1&zGr6?cY&e^#ZtSL>~ln@HB}3|o`UCm|Tau5Y`zzHwpw`w>gV>Hucnrpz0dNx zUfZU{TDo;#4J2vwJPh_qnVv^Zwrt9;$J#Vco1S7cu~q@V-Nb$SxgB z`yq{?K`S5MUAqPx8#khCQLU|4#ZbY&Qr+@idNewFP*SX?y^qJvaQj>Ict0}z2Vs8(AT-raiuLi*$@Dy>obna>6p@~RaV&X7ax4Ri~o}Igt zD_De-t0PI|Zna|aEo*~)PVgD-aS#g!z04KloDv9*Ay`{-PB}+JtabWcUqlwnL@bKB z!4kyC9mIi-k1WLEaF3x(eQ-zlT-G;jr zBOo9FQ$wqOIw9F@@cCk$dB;EvtQRa0{LINo9|KI+{mE<;=&e;fis-_xVmZ)1;e2y; z>autO^qd%6cr2S);0VqTLe;hj?{D(n4Mey&Co-3-NaIv3SX@k8t1UzKXtk_bx1PO4 z+(3Q?_dbeeNEqEpqu4r1@0XA;1S5@5rw$f3d0@y1l5KpSgZ>WjVB%LR%1g!Dx_w%{ zJ$s~zqQDY+B?V#bYKw&bOi@w)^@eEjBS^c0r#MjUFj3LCpw-F0sg&&>x2 zPqJ1?uCE}Ko`&N=FK9>(r_doiy)AJjeFdXeq3VJ&pl)~bW7!ojvbZQ|PrAI=p;Hr>KQRv)n zEnv5xZWzXOk7=XndRPdgho5=A{^Gbl&>vxbk(8Ep?M*&Fzkm+36YB<@TcMfPN>vr> z^seY~zr-wu!z9V7Z?duq^hX#QOE~q~$~iRB7QTkSTeNP=Zqx<~6q8k|^puto=}XFp$#pl~er+FRrvVC!V0_3W1)z7J^P7EKYKNktEEVBJFs3w2REojLam)ox67zIY z?Zseuq`M|5=g2vL)Tr1sp$NM^to;SO2|!SbWN7=~d3UVthR))#B6pZcvp+q&2d?@L-p8k|5Pl03!DFV!|wQf ze%q?`N-e)5wvvCZt98TRN4|Htx7U*%(;XY{h`H6kIN-i^#)dN^?(18ImS{E^F@N^U zb)kLlcg(0F)n@zsj@Emff(Y}c0Y}fljL-(@OBOT_F9{tGhu{I}b zvt}g;QMH;h3DLnRN3Q2bA8IH8rr|JAlW8LAR@_N;Nyw^K_`pk$!bjU!XZ$sNFf1O( z$_jNDQNoX_Q_pZ`$q@nsa9$o$ZX~NPQZId4=&k6KJ1)qai?vy=`TO?nUB#lPd$Xi_ z55opIqsT!Y?cf!4HAlgk5EoHMb$Y- z=P>z&w!2H!GQntCW7DzoiYprn{KjFV0Pqyyc*OIk4w7EcR zo8lZck2DK1r--koJQA~$SkWXzSB#LQp^4IMsD#EEw#wuI)AZuyKqYdZlmsWQJ=yuc z=V|MOgXaH!Bl*MKn_E2_yLZ^<^(4HlcZa}p3zn*$Fx@$M$p{rP2U1TDmaebQDId@N z<7zSn0tRQmz%F+<_^&KaS+jvFZ4BGt6+Wfkzr)Gd!dqQuyx|1+?kqLT7SYhb*z8(f zUKBmD_rQ#CKDL*hA2v5}nQ{;n%9bBAzC!nKFI^)t{1jC|!} zMYC97g9I;7GgpbtAgT)#cqw|A94V4z*{qA(F+vQ>C85{tA?5h250d{8_Bb0Z4U@N1 zx$HznWISrv>Jy2{V~l>NfHm8dg{@>`3ea_V+885GVmKulj5*4}Y@P>QJe&Ddn3)ts z=Q;sc-RYNmHB&Fw$oAk*lQ$Co^z<(MqimI+Z=XjY{y{;1${FFlvCMa>tt0}22!~Ew zt{Uk^fP95rJ9nK%!#CmXMBw zc|MtgLY3RJ+5A5hUjP0tPb9oX`TaiZ1A8ADyu;&yTzzL>VDFS%4CU8QJ;KY;8)xlQ)Prk1+DD zAT^h0V31Sq9aY-9CN-k~6xq|=;P{9js^1J9l60z75`=io+&6eyw_Vz&4a1iac2Zk3 z_UzfcdlrtQ)y}zPFhZ@i%`5AZC}yq}xy}D;Nc1qjBKYJl2k1%5>Bfzf2k-vl!1Cd^ zx!%npbejSd5!-0!qT*zO1`nNt5npHyl>r&jB-Utq;UKE7z8T7dSa%Hjfr_XRaLPVSW;GOk60ogi%coiUQ7cl zm;&ZcIq=Hq<-#Wym08sWo-^vz)y>_dx)LW-Swpl#V&ikt7d=QyLIPO$r@U!2Oo!A% zll2I@lrTQCn~Rl1Z7N2BW}S0k-QN%x+{_5k98HGC@HbOY7fKspZT)5P@sw-at`@T9c^lwLB&7DCER1GWl|K9yJbm@j7dX!d~);_w85$@26jHQ!u?AvePX0 z#2QN6^jEJM)M*-G$2da;6*m=ZTx0WL7@jHqi8&rXS6%qGF)vQf3`uE%LxsL z5HB;7R|OPVk z$J5%Pu`jag*RP)_pafga+6TQ`rS7^?F6iEV30eBhLDvKE@%7b;3xEqCx7s_Mga~|6 zA#i9{CX8y6z3tLd|2ZT_k>T+UQuj!sMVRdG-)+<(J(i={NefeEb|-+?G|>3vBb?Sm zwJ1#O!^@NIKY(|g?*A6=T<(}1qi4$Mb)-UPfKl%@>mc`l_=e5)> zM(HW4{aw9StvkgPU6DN%mP}sW-rL(h{jS@)_j<5~*R*#dy8$TCy$+z|U$f@P9na~^ zQ}PI*f#ybue$$uvhM>*CPEHXsK2;#dGt|LT7FyU~E2|@sD~fWaiz5O1gqS(B!|;-{ z`}#kG+05|HK^g<9tRU8JbnAoEghX* zFXNBRt0DFgbZ}Vs7d`VKX(5kuGi$|)iRL<8y4=Jn_hoVVxrq>)gjOZf!u|XAN2XvJ zV?1I+9fU)anO8?Vqj1dIB6)49iAh|&WuNUl)=muG{R_xoHWPDlUNBhQgIV_-Ke0!z zg(vnE+&lneOC20a+G>(USXuSq(#bsu%aTP? z)sKn{qcF3o5phknPaVChuKkU=n^Uhns=dxjtMRR%W_P{ERa45TpZID}ZoJ=-#%3A% zDg7F>sG$-Ydk83=zInNhOYaV8P%Sh}|48LWV~);HUAcVuEc|FX?>31sA5)pRW)(MA z&+a9mfCKSJ^%-A{+u-r$_<3O^1~+GK(k&kBa1X%GM{|q9@280(H`LLwkJ#Oqgk8u^ z3EgxDk{G+S?OF-(#Npu=;%$Q=4B){Wg~eA~a^ZWIi48a=^)(MydybEGoKm9wEc zQ*$%sB!xjx%V;I{1_Z23!HOhrPjuOk7;|cyC!IbWN~CmdY^Zj<(LDsu;ORo*f<+=k zCuv(zAEHjBZ5EwKbf4Z|d*F#8lm`4FeY+<-yHp$&Q@NI%xELWqW`Vs_oLNs+Gsh^F zuaa`zGdmH9x`H0hqRji@=~4f|1IYuln7*--G{l0!=f`L1ITfH> z_8uxC=K_cu8~xQn$2Dp}A1pOU&H&E_+v0Pd3a(TltDfR0LnHp%M}v1Zq+yB<1FJMR zqy`WE>;|>#=RP*{!{<~)3sGhHBYoTs^B$GOdX&{tqyh}B!fR9c15B51T#YpnSNf(i z$J}K5R@o#;YA-gI!3TDERliN)a{vCJlTWVKZ(~9Sr(Uvahc2U4_w+jN=-O-_1r%g@ zD!R5k{sI3fH)a1P+wcuxc(Ysv)ppO`=ZnYB_QYKa29pi?_rilWZ}%S^HR!^`>F*D& z9_je3&1trTi0VALMJ|!07OR3C-DhPuKk11%LN&){8QExd#c%so+?RVpNrFBo8T0Jk zeUuH4k%^{Tkma%!GmQ_KI&<9NoCJw@W2Tg`p*u@OLj%$oMLJA6}zJ22`mAbF-{lmj#)*xW*bYxnUC9q8W18MsQVmhIFf*o}GVs6DpniRYDHiMB$)Ho~SYN1h z?K*1Boa{~Xjle^)z5@}_NEre5g^4W1)M^O|BYxD(kIBJ%{Vl&2W8(E6@~N7$8}A$y z=ca-y&R%8qk|keu#~NBMhgp6&BI2o`Dkh@*l4HItZ`-gIKBzzUNQhq1M8`r0O!Npm#@8D^u<+o6JR zcX^q-XWqPd5)RI-5S_0(h}6JJ_30W|t?4y({f!D#Fhc)fBv)!2}g{Odz2Yqa)hPqlK(Yh%7MfCBS6733}X*J?r)@R9T z@!UVbXMA8CF`B!OW8nAS!%=VkfU1Q>Eq}bzIn1vP+%7K5?ggu^u5nUl{oc2KcgSxR zxcv2pY=ipTmAyy%8+eh04hOOZ<5s>92yl`NNcf+`2RA@Y9 z$RB^T)Jb*Hy`M%%Epq>|P2=o|3&hKpFE#dT@bsLCAsm~)`2E&_AAt?%oKTyg_Pn~} z8mJhB=UiSVTo4X`&}ASd4@}H~ge_Y!b<-g*FTH$xV$q^UJ^>E6aG__YkuQt{aw7im z-a-BpYx`p}=f0AZAm%%eDGqhA9fdgBRMSDPozTLvv)8ik==1cMiil3}p1h$jTp!-l z%erQBkHwpLT;G2_9Ifk@5NMC{#Qr|bo784tV2HtiSViSaQFQ%(RHo?YZvUtl*{f6j zdXAC|^?=(j6N2;qxUlpe?5q?euwZ9J-eq^$Rega0XNAZFmWasCvDt|gn`sL6PziF3 zkKev$PgdT_tCe&G%~LQKUbCYC4ES=YykD8bBSUPXn$YrIep7Nt?DK@0Enw}qi2lc0 zQLk@8(DX*1XJ!a{)T8uyx@FR0@fPhaXb#JCMY29f?2a93dz9r18SZ@lY#BBU6 zKrx`zb4~tkjhPjDu%$lFpFjT*IB*e$9=lAAot))>#k-;pgKg$o`uP&M={kod2D|OB zwZC~jtM%rQbM1eh@uQpNyu3L(q~Fp|^jk!24#I7w=PX~WNH$3F3aZjuc77I?1Opuj z%0hDW+q@ppsC5rTMou-CtWP!pN9(-N;0_hDE&f`kQ+8M%3Hnvg^kRQF!64-wfs=+5 zg6Xv}a&N@#3Y{*If6)jpCSz{&`kT+^%K#r9{= zR>gt@ctAU18l{)HL+I5QsQEk+e;*REs?khgQk)-Ch(m%Gga_{C@7>!rX|lbDkUq(s z7S)0U-88~9tUuIdoi7}vx%03<(A}mlP2m+=1wMJYSogAm%rm6&=s>qH=1mX_} z@z1e;+qSdEzs0iT-@a>?Cs6gu5cKiR*DPbqZ6{$I{Lc5T@w!}zTJub z)cC(t84p@MAi+wlw)26hPWwPbeh&PEBQ_haoMgd3wpkLbmMr<(Y6{g1`TZJv2I}m% z7SZpf14F-c+Xa(uSsl*m|94@(H74IiwB6uko&4FP?%LT6+$y$IDQaHUUF&Q<)1*ot z2uu6%@x3!&oV|aeN7gQSvZPZ<>$SS)E`2pfN6wI4x339@W~cOfU-Gd6=Bbjwq5(yq zADOT=cl4P)lcOYI2?{?87L-ndXY%q;#!-1j+Va$5rpqkmkiv^0YHxX#mqxfIJ17|2 z^TK$3qwc+4zi^YT$*eq2yYh!u?~9m@_Xo*#*7>`5=OBYp#L0=0s(aEVu@V&<9t&95 zvOx8=9K!>LzqoNY@*A)a^+SG9FkCZmb-Wb&<_tVl$VDf|T--VC3d!{nF<#U^Mwk6< zF(#``+P3|?vkfW|I)_>ugtTzjhO(i9QPV%}>@-$WlC3HxL+7+vKMEI-g%m`eMzF-W z9>q~*WbY}+=+-xC(}%3x%*Xf}R)W|B&=6Tb{x?{ASQ^$|4qrAE2+6NHws zxiSRb9#*&eJcCZMPM3v_hdc94EIkg!BE`>)Y;A1nR$5ju(}pH5`OVR}%|fnI8Z?%= zxCYl%se9WttF}5_8sO|d(+r`&{{WiwFda2DhoTm;n@2Eo&u_`+{0psFMokQ!@KbN+ z0aw~M6SJu0&-bE5pQSK?Lt59iZDF5=WyiTn!2Q$o6jFyx+($(vYnY14I5vT$i(X!B z2|&=LrF9`}*w?hBf%}P`gA=E2VjRy+y_))8kL$-6B{7&e~&N|Iz@3xK(S?&f=&M&9CNM_WPE94<|n05X-)k(T% zVs39ux22HMQLq!G(jZ=2>4VFGFkZ@t3`4EHE%GHy+IlUzWf1BX?Efbu?Vi{P2AA*E z$z0agaKCrg2k&hGk5sorirmtq;6zD@6PrPA8-b>e;M2ij9f~_sYH-T@7@5j|Q`9C27(k=Tu{r-??m|tur ztybJys)0nau}!Z&yYzc!4g-AUf%uKoOBunQ1l;z?+_1?IPb0}bV02jxE_cNiryaTp z*TkJ;=gd(DMbATOqu`xFJ_x;<`{G3#%F`?beU7`e?3oY#NVWep-wBEQ{R;;Q+{!^# zYXr*$R|hQCt{(qc{ zR@p*hESSRoHf(E~SLbq-0|KPR4TD@*@|jt%pG-LgJV-Fo#a0MB@$NxhBJhrHF@&hUl*fCb@lb@022sro8{Gm z$S6~I?toT&vig0J7QP*4dS<+0z~H1OPu`s~#YzAY<4w=-RGIKxLMUM`BXi-#uxmTtRMEvU}UtXNlVYiOEjEMv<LCQE4uN7Hg8x>{l1yjbnaXYS>igC zJ5MF7T$(}F6D`i{qya;9)m~S3e7@UQ7>UVe5da8t_0c1b(b|@HdP|mDhD(4ab{8U5 zD@uj-=IYvpwNqdrPyl=+;*@LPCJaM{)Tk8a)0K1`8Iho!GPJ)e?o>nj;OI`R`W+li zL*Vi_t{?ZY5N>|%cCJO-PE5P~+&16;cEBt4El%YL!^CRJ=HoH;2?XKHQp<&E3S}DX zAjeOk9TM5kB-Oz{;qdT@NmM}8&G`e~4#X~o{wN?A6rFf22q-O9KY#TpI)EvH#insQa+_p+7K9*7ZyH_Ha4om@K|G zvSE`eP3y3SGx;5gZO-D+u3cx7aoNA3tYCRXX%zuaU?G$+TGGmu=*B3!vd&y)KiO_J zHXmCb^y{Q1J2|)S7h;P}uRf8!lN=H)g*g6VkK=JpWj6lqsOT=tV?|ScL#Kg9CSJF3 zi2B};fH%#q2DM394zl{C=p>D-QC>gmm&#;&07+oZo9<QZ{JpIW=i%icD7@iAdK$d;CQPkEMX+nYM-g*!1E_y7O&^~Sb^tG z4@Tr7{r8A`gXd4g=^{`wHc{4}%v`g>Zch+H5jHU9kit_Rc1GcPVL6yuS-&*!IOVIw%(v*V-uWOFp?P%~r&Yj4lagW~W)pqgKz8*rF6 zceXRx%qZ&ht>)XMPS+1KostNiZO^Z+$Oon5DI=gJx<;Y~p}kakD$a1M51qd*!Q2(= zph>_@$AKs`DD!XN2+%R+AJr_Q7B1ghx^(S2-648|UNUxa5C-mLx-Fnvl}7u=JB#X7 zF7?3V*TkBli^6xbXDh=q2~PV6lwj=P6EU5?2fo$sWNIJ37BL4@0bgJ73jzhsw?Dtz zrAOGR@yAD?sWWb@wE5YL+eP5n zJ{37W)3t>E6L=0?6qg0>8bq&+LSx2HV1*7=lT~oH5R>D#N1gn%qV$x(gQZ;;`)cfo zzxHfd-Y5luZBxb8aZE|7TRtfGpa!C0W5F3{F=fg=fVUhJcIZ=@C6mw+vdVhfGmO?j zxD@_mbOUu#>)1gCh3IGKpU#h3jB7d1Z3naS$->dGiz>Ne~u*@MizexpyoRR2yWcWoks`T{CEuwKZnP% z#V0sS)N&25Nla0-`jhKjwpAxDQV|w=lpUZD^Mbxw`s3+K_7$&KjqL~+rjg6Bz?%9G z@cKSuE>0%-vv0qC0%QR0VFvcsA)k?g%d)~moa*%UVsESmzycKDWPiKjD+JAoor}w- z8OM(u6MkVe`<$vCYVSu$egQ2kRFZYUyx6S8E`k(w&R zT%&zu0#@w%1&Qva;@^7(;=ty;?zaU&Wo}bKz^24YO-q~e7TM=NbTmRWZe{O#Om5=U z7j^!a6#qJQQgHGPy=co9m%lzuA87)2UNyRl$NpXN@CVwXjGFP*WggfNQkEvkJb0^s zcDobK#MX1q!V=7hFoEpFukZnwNNgb2sJ-eKLl`_e^&nLMjb>IJ9G>(TbFmpRh zXs(SHW2V)VdnvCVcB*&v+v5_SgMy^jno^6W&De^DpKFzMjLy$>j5-@KkV-}x=AQah z29npwMa?ik0lBHP(Wz7vAPjQ;{Fh}ctE3E0gT`f%yb&*z5G zPRP{gN%6n=682FI>{`Kq?q88z=v>CCW}V5g9ltF4lRm;v9K_p$O(+wzufd*+D*XJQ zVjLEhn1s|j;-Zt}j28y@#uCi&G(yuJ+`W4p$^u zWa~Z+`n*G9-WDR+*(tAW>86_Xs-en?3-$c40*e_VO-)sL^=^Y6plf>A{6-L6WM(=e zz!o%PDVEn>F0{>4-`LylLg}+j?I2%_-1X+q6OZbMsux^VW$-b`_foak51mxSDp0&Ab6-~MHqpS`M#E;?Q}Xx34ruP&kdIb8o*68J$ftR- z0}rcs5#~#lS;9HXQ3&sV1;iG{ciNe;S_CPqX9ignaMXz|f-|fz&2Hcq(4tkVjg0;b zq=SOqOj71${ULNR^eLQ!7vVHr=!;8#%nu%*r>o1b;aSPWOv*d{b{lClxZ5aVAcr9} zEV7+woi|?$7=e{VBDc%>vxRG$g7&Wm%S|@%tk_ja5|C zf047Pr)*f+>Xu9=B^{m24c1(2dn=|epxT~#?80w?%oeRW=xS1zn+@su4J2cV#U;;g zo$=M^ssu6i!I{N4er}JJ3ec;_qe72HHLZ{*06%V3(y4H?Vn30T3eB}JE4mx`a;V!T0l+-31Y#*psS;HcVkTzPM zjW)0q6&ro;i8)X(AUZZpmY`|iEQvPSWmAX;z-zc<`Kb?BsfroS5DJuWoaGt5mKc-Y z+0McQv&?DXZo3(u#P5|K%4&+SaSi!h-WPtK;Pk2R0VYypjB<@m=3{AFim6-ax0s}P`w1&x8_S$ z;Rcg7ka)SPkIP(4O=Wp1daI1*q z-Stc~s9mk7igFj9^!cMlH!RQJ6a8gn)u*;v!rP^Xk!}I@(|U;*{F@t^BIuj-cvmQi z(MADdj914m?41c0ghux3WbwKdz6NQ8Yv#OswWJP0=$i2dR-U;g$0L8zVUw1fn+`s< z6^&t)kN8_Hd;L;Is(V@d87t%Y6Wy*q%ZP?gy3*Tso%>kW!UOd|7&5``#Urn@{g;vuI-jM!#f2;&SIu5($})h9-^sQ8`|t=99g zN2l)Nv{a-g5?0Yl$*RWT4_w63O?zPDC4kbHzuU*H-_2d+h|OEs_o8Meaqqx(MG{ke zj(RIS1yoThZe=T=;{>p{#kQtF^67CHfd8P{gJPH^|~?ckLSg>y!sO_Shz zhW9~`(e`Se1$Gd231tsI9PA7l*LbzX<0Wbfb)#+Mo zhREIs`?>V%jBj$y|8}e+9o(@x=kM_1vIW^_Q%M$!f9+6{8;-uS@Lpo`|34I~l4a?( zY0H)dcipCe4vVhOMmVB^#gZU^^xWX$EciPRJ$<#Bg#3=u2a6(Ln~fjo996TrvGEld zb=RpEGhn5{6~9jBW#Ik}VT`O@#Pow?JUHEqKGTyuV#hHUVY`Jbwf6S|7gXS6&{A_` zoTrT+=ns$i3oQMZw@bZ0(&692cvRuMSQV%J`c5#wr%weKCS(~=cDHPIst5uj-b+4# ziY(q9HNNl?KTJwoU{sWQ+*_H4fb%73^T2%(2j^aU^nTyB2ZfX6n3OMEx%7|6_pV0{ zJy7+SSYI1w(zdDF+y~6eJQFZ0Yl~zb%7@Bs>HK7IDlJ)ML80sY%NNp5AE6)SR&~=zlrX(Khw*e&# zNT#Dk;<&_46qlz3T!h|t#u=rz(ECxaVIH-AQJ}B{)f>F^T?#MWtYFMO(|3Ck68-%A z$e>q0HDS`EQn!u|xUM83@9{Rv$_d3i;KvY}R!oI6%{&Ms+7=LyTRNaWnwKv)ze(L! zjhF0~j>ERrxOHnME%Ql&yIHqHPW(4$G1TA=n|+x~*rvE>kMF!&Ci065pS}){Jz_LP zySdaz&7BM4-Xb&+O^7h|{vOq>Mf2uzz$h=`z7n?__czorW^8EAKDQ7M7}FO;{9GUV znCY<0g&u%`kT%OH@S5%bVU-;i`jP9e9IrWm@gA<6O~eyZyl$poOh3bY=nPv{K(dWc zE&BG2=V)NJEcNi01g$q_#|1zwuHS=V5a8P7IrwhbK5>|UVveT<9lj2mMcwp5e@YRD zYOqA?I3EeeJfs?g;Td2l6HBEdF_V1)3WhZD_zei)mZc4N@uu-=W%oC5{ znrw!9aSx(vl^G?%F)uLIsWC7=*LTzx2Ture&_3h^N8HT|b0{qAk(D9m3m6{Fvl>iD zId>yRPAx3xj4oAJGD6$wyYlKTE?}uL9IV<68(Jn$v%QT^m%)RUzYV}eRBF9@c@^?- z;3_szYdI^ZpXoHZm0l4l)X#|pADwbTb2SSiY>{N^CUCj7NZjyy*+9E!vh9B4{*c}; z+J=v0?LnuUe9ZIDbNaq1BC`44N6j7Ky1FX=`|osaeTh*ytJAAZMM|G;KWog1`_H}C z1^souevyP-aM|qXJMKCii|4qYWn)aVYFCF;DmhtGj=z6zjVKjdBGQ&sk=N$UUnj2w z(G%(s+^@H1xZvB7S&j=qP3Gc>2j&N9&gkto>U^D+i+cN6YcFqlo(|}z#>}Hft1W@& zZp@rqsJS%$75hhQmSq2!nAZLTX_t5a^5|TajQE zK8T*BkNs=UVFg#2WTGovt_HEzEaSkdZqcj;_rE^*eFd9R3YJnZVJT1js@e-b9!QX+ z8QR@8ie}Qrh~hRWO0nk=Ur5RpOn@%#ZIn5+gGY^O0P1gG>6a*=m87|G|3Z}U!IXtv zn>`F(o=csGtvoasu48rM`(g~DP_=WkUtBB$zv^G1=Rsz_aPyDqv`BO2)TmtF3w63r zxBSlVNZ2e^V!kxGIf*L#H9EGO<&lw*e<3S?p%_};Z@4M!f;;GdB_~`#=Dx6kwsqE= zIkybOx}#hDq}G$m4isL>JZU>*<;qLe2iU~I-dzxLx}(@ox19>BI0jVM(m8+^sFE9# z^m$0^;+MwI!`U{L9>;SU`IAwL@MpjO{`(gU2F*fv{rGXn0-j_~r{?laTVlqb6)k}t zueJMt>fEjoP$(4El0O+_Q=VJ|A&W3#uhvkuD8%ke&M$VS;o=bzhi$NX#~*D9C_};% z0%FwgJ8L-R@Ywb&{wrN`>+E%~D$n3ZNoyFJei;T0%>4d7H>&J5%m#patyYBW;60y6 zgtkIiCK1}WAPzn}_VjEMHMPHJAJ9{X=Xjugb^oO7LZaC3?k`rgD0*YitlKSnTnf2IT;d9$^4F*LrYE{$ zTLljbz3h}RZc73H@kaM|Cfe=`CGlg&iJiAej%}+{xxat>i!-~Ip=)ttIU7GwVk~2R zXPygu259>uYRXhbXxUVr`cUXxsdAc26zm%Kqs<#`7%ZxbdZ`|dg7L&u1fWW*;M*20 zKGfnp?UV1}-1`bD2c)JP3pD=l!iw*N3foFVFU%z)yUC6j_nMoUr!6J2QkM9=^Y=lZ z6h%KkS^v>lhO9n?k&jXg^rv1*7cDKZRv`YxZ7)8FT4zOhgrk5=$C16s^X9^3&K726LL3>94f@`^3?sSI^!UN*m9rWZ^}l#A6pPUYrf&+@++`-(6&- zg7*ijsvHY4!z)97dvkESg|GauMhEA?n*ze5<$W_b`+swdI}0hkx!EGdg`5I293lA% zRlRL2)3ydD+W?6rK{9UNv*%+nMu)=|8e$c0@p(^{-TN`jeX3c?fESm?MBD*U&rF%& z(JWcaRq@c$d!{qo)Kt2tv;T81%O3sw9LYgOBv)Hx3WpklKFT{~lSdGT5u)bgZx_Jo5hgl}Sl9 z;XhZ7EGZMc8*f+t|4yZd(!6M9wW8C5Wn%_z>sy-wvb_=T*7iT{Id!S>^&zq}NX)NV zUYgWAIeo-o$DWI053GIEiX)r!OmlotlMA1xYd;RSj(bGj)A<1*-Xl)AqP;wH?OK{) z&inVPZoZmqwlie6$?)L=DEux4Sn@qA#On{m*JdbI5w?IU-19XsKPZnG`Z%5vX0U%k z^eOvP8;i13)@p(3*J*$hTtJGj_d|oZvR|a-SIp?|#(*yhbZlW`b1J=;NgLp32Zyke z7okQ2r-O-&VX?(BLi-f$)hmQuERL0rIXm#aD;SwZMmncjG;21Cm6L7$j|8@(sITwH zgmlKdvpp%t68;f5r%T5@@uU=PK>>YZI#%R&K)fo6TZxrMK|DmN@z9g(VIM+QKcl7V zRH?!&+Y!A|iE?_2VRsAeT&vvN>aJM@ncxuhgI``}nhstL*d2<&iYdeDg%#kMzXl-Q z*tm6ijq}WEJa7Ld#H;D-z*^v+g!rB(%}-$tf%^kJibP2(5_PCstI|U2sW7HI9&nrS zCNPj*%yvA@to7AmUA^r_evr__$me3Q{-`blO6xq+skVykt<)EKpu9d0)Uj$V%Nxdk zgn`w7##bEEB*aja_9fkkuh)ICSXNrPrSydc72yw z-_CNknmzl=gMPV(Fy|4p3_lpdZ~a!CNNFvZyu$yXVMV^{du&IS(EB`=vy?_bjzVd(Lq|na15%;!-sLF&eu(anM36NLwbQJ=>ixL4^h?A6B9)|U84l=gH+5>y!hV`3{19Tm~odu2@ilk>PPVJgCY23;G?phiQ zPno;^mt!>1IAoXAn5BQXf1ZM^xy&!CO;uagd+U2CX?Z~##sj2k$F{08hWBKgi8Tre zbg^38Rhx@MWed=l<&`Kl``{*+hOcK!w(=TNtT@lZ{tXK>pa*72mXlXdk!!mma(YkS z^8;TOMB9o7;f^uZie)^~3a(Y}hh;%0PQ)HbvQ?}beSc=L*|?s9Ab~^}TZv}~fJAkW zbhc0yEF`RHCHwj9-Fv+v;F^(#P3D8r_34+IH|{rk`>m}t{&>~kEU%H?k^{H4W;({3 z9+imY>D-`aXB zs&0p%#76F?FE>e@SuebC?xfxK%H6LVb8n5sZ`yZvvH#+2+VqhY3s$G~*xrd@leLRe zqRZbFKEsh_EBqOmK$SHLI8_FiUwd2XVnMQn{*1g$?GJweB+%u77yQn8BHVyOCUKE4 zXxx1LRv};)7XpYnQ=^i z?T6!^D;9tL&r(zWKuUd1(9E4B_>r^$uf@c)Vc65nQBhHIjf9e4NuGigt)}+UJ@8RX z%pf2w$2*n2giKU+KlXl|N1KRT$lsn#B2ei3aN0Y0sr{&4K-^>nKyf0y4)n(w0 zj3>A*^*z%WN)l;5vjQ_XnzBYB^DW#W4{$ehIRN3;(a*IzK!V#U(|1Rc{0z0r4R6yITn<4(RaJ|1LATK*RJ3Vzk4;UPBCOM!@ znl{bc>G8+{zS}y~51bbi-x z&p%71>j-T3&qrXgmu_B_?h_}L*?zAlSgRf!^1S$=<_Y_0&!2fViN3z0j?S3C zp7mUq6L`#qgv05ou`F}Oz{Ub5%!Ki4PH3f(FbjfS5(ukOPR5Dz4;P+4=C@|Mwmica za1C_Yn%(v|j=KZq;ac)`kHYw7UoPt-Y;Xl;_J2lG!sCWX>=V709&B-1=vW$K8I{t5 z6(c@)Vp6#9%=^LgP9mY3JNH;a45f6lp4@`S~{V9RQHx`utb&!)?d(HJkp9`I_nAv!T?>Mle z|DWL{v5&@M)_J_;W)-j~9@pI(BbiNM$NMgK zpi(fxSCLIk ziCv)v3CcngkJyGXlXA^Kv=qeb*s-}cj@yRbBaoD7Q^d>9KFiB{6FlJRN5<0|K$XQM zRi_qpy&gR{A+YjlY`UF)r(lH6kfB2>BNHUi#YAK35X8trBA_*EQD%vi)E^Rq&O3!g z#M>9ye(zpP=fqu@Hz(F}=J=~7AOHC004o0TtFL47pH=(6q|P!QY8lq8cxte5O3Fjc z;Gd@>V#pt@bB4e^#4|u=`GE=8f^2MM6}|9)#)L3UQVUzv6L*#Gcit^!t5y6dED;S< z#q`IYzURM<=lR2M=iAh`b?Vh?O7>FeRKu-0_tP($Xry_n7PJl{*T8$nRZzXo`wtQ8 z9aUHnp+jKDk%)*X=3QU~o?P$1g9w?aX)ioNv$YLAiG_v~HjLh{@tDY! zZP}m&Pr(-&)tYJQ=hw7n#jQ+SPmzT$lpd)l?C_PzD-QB>eO$!YeL` zK!B^VyT1O1aEA6*xIM)4sfQ^og5v>v(hiX#HRVkF(5=q3_~D zO-M=NUEFr?V8XkDyu|(G<;0l{6u;ujV~FySn;^ckd++O6s@}vIS(MErNzp<-u|l&x zkSw^F-P`l#f+uvjI7V$}oIp3yVMBt9R?SAoUy#Nk^6g-4POC1-{d@p zV0;F|;|I7xIx&t0)M>{M%ot=fNfJY2@Nqb~Ne*>&)@|5u1b~3s2I1a}!yY%DIRL$e zrqoo@>e}^f_@F8et9nb8LMbV=s28Y6CWk!HNZ0%8uO{%-XdYO6y)DxB(rS+j*dHj+ zTQf>r^CjF>@|%Sz(*EhazkK?Q)z65uy(yhr^?fECZXVvO<*r$^fmE}&-Dl*ldbG|h z+Mc?8?6Ndn{{siWGa6yMs2HN?#uCTT8w^tZCWmC6hd#GI>p4q&>U)xDrh0g z&&0%cW7uH?o)aEa92<=hJs;e=hxqlR@q%)(WKNn)va=$KZAvOKuWPNc+%Q>Z0Sg1u zk4B?19~8no&0fUo26f%UL~TaJ_ZCQmjSIP%q%_b#Dr-z_x%g6R#9xD8iKg^m0Wki= z=$c<0a^<>|iXj`k$3lZ4#t;FOV@Pf#+Q-teYq2BNLXx8A*Q9hTqT+7TudvoXQ*#e_ zgPAb3f&a2kxomL?BE=AKHCA9#mI>>U_6k_S{$Y1u>Sjk%vVryC4;qGJX*i36SuIb0 zMFb)v9Pwzh>eFk{2F^{J@vrQIMLGPL{Otadz-H-98lq)V0y1HCsCzjv=Z2=PZd1lT z|8uuz-;mwCsfsHtiwvHLiREE15dNjJ28MFF0dSb*0fzP0quFXkZc6Lc*+T|s$s=ru# z>4cno-fPuPjnsHIEVCvs2j;=3FD>k42HUw)r&_FfM5oE63P8|&TFShzYv0a+j6&1% z_zc%&$n**&YE*9rF|3w!RIACS>Udl-w>i(CB z+%H4BQt?cT%9kvoA0s~YsiuPoKbHx- zqOXr3vPwTYMH3bv7;@s}=xC3)BDngxROgc&c9W)o51Uw^OJLTNDev?QFp`PC5*wy4N+Ug(>3G*J>Tg2zF_q;SQH#d@;16nV!-DP1Ty=@mPSI@|0u`d0pZ zR;P|#pCbVvs6=I1)3H(bx=4I63yh#Ns-zK^-4=I7k@O`4b;`m;-y zwNDp*+kE2qagq(Lb>441X7uP7d~+5vRq(ptf&x#5_DMqMcx4wH`G%#->8Y1Sf7L$* z6G4`}+K((KQA0{!k>$Cj{Q&Ney`3*JB?_-|;llgli8SYnoRV|!E;*~K5(JC|YigWh zbCQ%dfk}bhw@%X$@gJ5URcs8vNoVFTHOKuyHcNz12Lo_;(Z0G~Y+%u9U}KD(yPdjy z=gw@7FPO7l5Ss(lYWX!K`Fb8uh^Z)m}Dr7y2Gw3`j*Fpk=f5O$;z9x}rJr zFL-v0bYKbMMN1ArU2+JVSBkn{-lUZATv}4w2RgOgFtzcQ+^uy?`_&9OB11ySkQJu2 zd*810z_rgWPyE*vNBzrhsJW1UsX!<a=zg4`ch%CfEW0nE&pRc_q-X1O9sztTC)vhiy;pC%^Z1-_O^wmm!PF5eyzh~SWqF`DNI{F3?!;lJ(ErCc1_JS<;`g5W zxwYIXk%T(OQbJ_q8#KY(9N={L`NZ#=_8mM(6I6>=na|S`v(LAKs1j4zIKlWC8&usk z@F8Qk9Ns%T{geVa3^Jvb#NWTKBKeJL=$dS^5V_VBSf z%C}f}7-zy?T;<|nS%>;47&4w0d@8B}X1W$%i{q9ONHahYc(ZY_*jbPdHzXUb^7lD@ z7Yrb2TE+&^s3vAwRl5i32$dnw0wS#_cub6=dIOKjx^3~LspjGj$JNh`G|0ol2rybQ zqLTE}Q&`oWu9`@6qFXe9z$S zzCZ8xbzQIPHQx4a|3~xYYunZ|@x42gg9W3S!Dfi7YsAUL`UjTbG=Lg-_Pxn($8IC< z`TFbP?_SA^3(kzxTae^$FkVGI-b+_4!G-4^r2cUidOQwg3(w`Lb;cGaZ?G1h!F7b= zr#HRETKX71w-H3C1D^sK8p6^W#rD^@@jETHpd#-VSJH{laAK^>+ys^O3fM;Fk_m&? zr5aB#5_tA@VQTHZA0Y<9yswOUV*_wD!dUeNb{Ap3NjJ9o#UZek#%XDUTvPtJ^9CW$p(-oRom(7t?;L0GJS7?pDOgQ{d}`(VtJAOFhtu#20%y5>UCe9+tA5@9eSOUtEfvQPluxa9Y}T2`^!SA8sdtw7 zPe)<J~T7&Z=SC!F5EV8W93S3s9A@>_OWZAWs3-M5WN)+^^DxaStz-Qes|^oT%@(HN>P%r^CljoX~); z!aeHigP2Hp5|juN4IAyq96b$oQA%^ch|;Vj3fGCe6(}9>9J+8U`sY0DrBnDAf6|Gg zar56Gk1zT|OG%YB)FTJa)W?7dQZ`>3^1B2EEy2!NRHRKRF&rU zic9IWQ_YZhGwhzbHquf^U~Dnm#bXSCR$rMkje~m$V>QsVG{*nxYygKbeuLVqe6S}yHi_16NN^?H=7piV(c2?OQU+n`#vC!3p_vm-|bCLG0%NaymT?&1vJ zV_d^6o}NuTNpwA4Yv`rAoG`u=FG@0dg|%XdPith#e`3=3Z5wV03~Cvo;~vbtsAV6T z^&h$fA+sU{O@k+__MOL?)3dX(NM~p)dD2c^JNBMDefr+V)(LotOib>(DVPi@;EOQD z3*HRrsJ^2j^To3!(h%T+`dhsx9_n5l`X(YB_zPivon z&`{0%)@{)Q+uKhv@6VKLe&)tOT?UEjl7Et9{B+?Hzp@P! z6qxzjV6h5)OEQCy#VAc^AZi$E9s*wEZUz>-Gk(gc$+?&KD}ExGi~C3^Bi|uW$CpNF z&YU+}^k<=A<6-(yEZ)7F^zFj-M~}v_#{AJXSq9j}_kN7(#c zxjx5h#d3bnI3LwE8qvG%cXsjHDm~KAug>nFN!wtS|MJyjsmAW*PKq5-1p%`f9;+`w z(61Z37&tCmT4rcm=m65kiC`V`n>3kyQp>1PE9y)yZ=9Btl#j}#0V_?C& z$hfu%)5tY8Y}Cop7F7=F$HEuIn>TLsh>Ed&k<~3P#NX_6hkFQZz84o8>7n-?*-S<8 z*XJ8s-);btY;=5$Fl>KZSL^b=>(IC02;y!dZvRr-<34gVN_LB0+$#hz48k3lv*E97 zW1L9#cL5aPxi{W9|BbNT`9Q7mQ5+F`qvjPFQdKywhXpF1!}~O($Z4n8d%}MejfI|G zaH>+T$OZ_It>ob6WN?oo6b&xFt@`N}otqGcbki3XiLcp&9htU;{VA@3r{cbGdZe7N z-uk|%KRrGN!8eIPIOucWxXxU@xC2WIRb1&ivX}r60p@;h&iulkB?Y5T8+nB8&fxO_`V;y7kHXCRraViGgINrTcD>aq(xLKAq3(;dQOL zbu9j$EIWy>Cf(E5hr3${yynZldry}TQJ8b4TD^KTlHhA%;I=GrMb*zU&iXE`Rf;t* zhhzaF7H!J@yUsbpqok2B21s-tY8@toz*2Q*Ax285NU;$e4@nCD4@&!%~ke9-Fj|#oIfn?g|tYEKM7Y^A}6(Y$R0y zrkBai`w_l%W7CVS469g7xU{R1<1wgZnz7YbNfZ6|tI4Q7!y5c|7Wrfw8tHE6o z%d7+9JPE|VTGr%ch|0U;GL`}4$;3$NTXFgKou*8 zl=5{J?zlrTsl%sB4dzo?rIc4cp4zJ+r-)%py0khy5+J{=`p^UjvgBa*LKc!~{B0I$ z)0VEGfvf+=0CzapYn`*EY{OBXIULu8tq5VNC2$d-Sgk{Gk<>B+VbCkyykLjuJIJ~A z56o1^zQi@$3URql8^*i{v8N6GSd6?eF55VpKQwqO`QQ^SIW$iXxfVzL2BQ9f!PGaW zuW`c{wGA4|1VMBR-YB1CsNmVAcBnyqYZ?3SAZBJY13ng)!Ml`k<*6qB-R=sT%fB ztcoR7CkqE=nEFWHtz}8M^1auo6XQC|LMY%}aX8n>k|o1wqq(0mbqZ%nbSPaE+(Yzk z9hb-|llsM!FFpXUidmb7J;Ujb3q!6{pbiLhHLo75H^_>RC!tk};gY2L8I%{rf;atc zVH{uqaTNHBiwe^xtIQ78!vicT?7>&u@e_wHy$NqN;57;*C+R~L{@Br)Ese&Zjv{RjW zx9(haDRCX|{rvd?RT_HWXVCIofoA1~7@3PgA`V=({LmkP^vBY9x~3q87hXEr*5z>? z?J|2G@l6@=*!t~~iD&M;1VRZFEOCU_9DYFg_bsigd^=3GYRTNLBxpf3Si2_q{ijdz zMj8eK$SAdZkhUBhjUJ>`0_$y!pGxihe(%lT9trc;s_*DzuH0OvirMfZ0!}G9}HW*!jVKF8T>jniM6A&!^$n4 zi~p?7)W3g+>)~XE1*y*pj={|m*k_<$GL9CUs;-F_E==#_eNTzJW`jKm3lrdJ@;klR zixw>!t?O|(ejx=zcRPD2%@I3gGydU2qxi25rCSz1$+8pUM_ME$+}f01(DOH}8rdm{ z&Ca^Njw*v;xdw3Abqewf$^S6ukDT<(01N+)_k#8^+02|h+feCVQ&KeK6=esrNTXoT z3+bPByE`qQlt$*w5#KWhhwob!tssM}f%+gr62=lf-f&7wOQQk2jnxJKQO=m8`1!h` zkp?pu$IyjO0cp;G`SbqstDvYdW3 z$g^n0VDlq`W0WRd%-TiaTvUS8#>C+N{ibM+Mr~v2vy9<|+o5a01*JnOCX9;zRRj<2vP$LGzoTK_U+rueHMF#3y{wqIfP3r1iTI{cWm1>-OjFA z*F7Hs{uuioI0aKc2~H96F5pz8hel@aPXyq>W5e*yaSe_7+4Y@&5y+$k0+Evkql!#>e@`S1Nk-C-(yoooT@N z^W%JaF5FMu0r@lvW)zdniNf$wa~@BKclKd(1-=&g{BaHI3rSh9+3SjsI-KqZVsGI} zVmvSaZAA!_{r9i$TcvEKA(c8ppZdL|u>kz z2JJXIhqNc%{UHF{h$zd~riU^e$Ur9&3=#K7{u~YX>->1o{b7)SM`<>u=8zE-cC7f` zo8@9D;rSQeRu_jF4hbZH4n=N6*u#t}+7FLH2x^2gbE6L4QQm2jFL`R0gc-`gncnz_ z)1T$12B-u^cKY?4C{|~t)bILl=Ow>*J<+t?^41TwMK!PuT;e5-S(EO6^wlsO(0BcY zQ%g@Dt$GX|5r*)NK9)QKY~)%4f95zg!(tHYfFTVB=s}X_(V)M8(tB`_b4&4?HS}Eb z@xup13lgcm%8%(h&?kBEaC7sgQ!?#a<#k#E6~raE%!K8+;==BFp4PU_>#NXpJ=Q$N zvEb5bB3kXPiRJNPUI0y6Egm3CwMC0Ariwci6Ua=iS##|!T_#T$Jq(g%4(ZR0@ikL8 zh@OXrXYC@NVtsUk%KQQu@%Is7#eD~Rr=zi^7OEj:uhb$X;+S=L^}B-tyNAceF8 z{iCGCiV1A-sq92eo=Nz4$AoQXVa}}vyS%+KQkG_qXjm1fOBnW6*S8IR;mpxo3C`q?q^0L@nu_ESx%E0d3*(ODYl z`J_pm`@KT0=fRQ_fX8HiS9{nFTcXequ$d4T(a^(f2cT@58q+>qY;1aXe|hT(m~yD@4E~XZY@NjqYbBNvj69x9ZKi_cU z*+Zp}KL7ht1poDa3eV(C`^W8mchj_Q&-N9~*d8^-%?4u~zPL7>cj3#+%B;I+7T}bA zK8=A7siTqElM$wZkqFEljNpIu;-P9}FcQ&~LDO@6e$YJ@6au@0Y$@_5V0$y`d6pev zJz?Xwzdfw^NZ+C36twRJk{@43XVJ6=3Uapy>Usv+yBgP4QgGU)AKhg4WPIP{~Mv^7<{X+=UZ)?iytclmOA1BeGK+mVk#Ysgr~awGcd z*Gs3)nVpsv8(qMN-J>j-9Wat@O`jLG7nV9rTe9&uy{=MV<86B zyjrSJl>*etc%xRIdogjLU-4+(K>kgkm9Vbgz5CJ$%`L7|gz`mV?Yqq6M~}{*8MNb^ zCJPBMK_emPf zbFFeb+@(E@xHFzkg%7|3xgAc+!xy2a#(%)Rr0~MKsaZogick*Gm`0?oQt37tC4g8> zFyK)VkgK4Y&j@}6N11J_m~F9~BKFHLC@;5cdPPjjnnP;L=eMlOa@CZr_^-WL<#>iL zrU8uFLc?bj_5&}F^*J9Kz3G|a@LKF+nK{KL2fLY1!QDws#4J$GTGe~U+Ou3*+OReRPW#H9!xh0G3xyc@KIk zH19Iv@IB__lkqS~2Hm^RTCV;JP3k>qU4L`J?G=y>$PI{9Nq-MX`2zOm+Z(wr@7XuIOxRvoG z{a@Xl-75`4iAG_ADr;+gEM?rsw1IkZXrJe1?P=F8O>aC3tO>n$Tbd)4^E6XTCO5O} zs;ICgm%$ygmFIz1C*q;gf~DC}v0XTybK6h&)o=?NwrKG_b7l4knh%MR;nb`=#EdRV zEpCHB3tk#ipw|tf`w2|g>T_B9tc=*$-)>EH?IUM28N*u z5q%$IT=WxST00&tzB0o0v|CKctnqm}CN!+pi1tWKMPQv(xTqNDp0=?fK-@RNWZAh}>S!+z9(Pt6dNrO~Pn`TXe5WidETS@O zDnEs3+_&1mtx-Cnwfz3A9=7;0Zf)V8p_qxVo5lnVYYK)y zP1bppzO-!11n5%g0L?6N<^|%^DhoNpl34HokOT>S6xo@i#wC2LlAe;tAL&lhsr1T@7Ja3cjxpH z0N6ET+l;%P9rD`nU%Ojey>3yX_V}qa8<>u0(=nCiZ|b#`_jCv;CP-#OZ zw!8$t7R03>T7#8)b#L$X3o`N$YbFac6akbEaM#e(=3;)?6{t?_{_+LMzrYv^%ELAlgbatkQSo*eot8dS0Jw2#1Fp^8 z!&*opXxU~dJ|Zw@3~r~by~BG3$bPk2WSrv8fL4gE`}v~Fa|||(Ik{e$5Kq{+rg_)+ z+8w4j)~-|M3v|m^dtgYcF*6!Gaj!)3J1>bFm{GSPNHzbZ)xgZ4W~^7-qxfm?0pF>4 zzoj1C5zJXrEfx&$S>RLM0W3cvi33TfFO*CX!*DWMzEFO*+CaI*csy~6UjZB<$#KSE zqS?TSIc8XSif<;s3%7K5{5K2cPmHKHTLnt!4PBL3DF@~)QOPMOS|Ob~Sc(`7fM;@i zI*!QdPe&=oBJA8Sp1K?8AnUOa_xTPZ0i(OZ-JdD|1u17RkPOO%VhtVXw*kd$iaUF`ht5s+Uj9KCeq zklz;Wir!{jUKi?it6tvI^XBK=#Y=^M)nwGmf5uI9dbTw-H}UzRa4=6)c1HsPW5+LC zyqM(I9LK#y^-h9ZIlVY+&vu4Fq6ERj(4NvkkcsQMp+{756i422EzN6lJ^8bDmUeWB z?WTo6LgN+Dsg1I80d$X|-+H3H2WY9S@)9_;JY` z*yx1i{;lyXf*Ii9#4B*k|xaA+Llu;X2l5L6uIwDDVf?Ma|RkR&c*rl|e^u2o- zzS9_8R*Wm@zb^(QB2j};jKT>);5r9!4F*4{^NOW!m?(C*6;*C{a6SWT_Q`pDF))oj-1$M*n!r5rPT6+Gpc+qq{?NBXBd$}P#LQ6iY9 z!D`g8b`McH5mdCNLyb7I`>rl5VEM2{G@iR>#r0&i0_);!@?Y{gL!aT4?)cFbjaCJ% zioVLBjQ=p~%=Xx`Xm0vanE<%Dcb}SPpymp~j&sH(sUDi9EiLcko}_d^yAvHPMEs#{hE75- zZDz0)UdsZ578crzdiU_*!CouAW!vCJR$iJ?UhlvI%`?M|jMBKaHj%sho3q`>1wd>F zr#Tw-WhF&!nI_H9k-JeXxU(y`IZhkB8q^ssOKE^IXQg6{XTso zU}dOQi|Dzr?#ZB;hs~CrtNd#dr3jv$|6u5^ba_L=C0b*S-&p3X>oUJpjg#XzIg07? z?9M#WnK9gE<^HFpmABJZfnUvA)a!2vw~FYULosZmv{8YV#-rsoKs1T)1c(#S_+DUF zLEntVkFG9bJvMUJ{2*Z6`?haZnf|StCFP;c!GHG7QH=Cy;%neqpHaF6@dE;@cy(8K zgw3gPC=~MMaaG6fU%N;$dc03F4E@cvwfmA5oh|@=vXcb?WOfe_z2N z0ia6F>6MDMuNnkK$v1rVG8>hEsj$6?3X)_(2q=(OQ&;v&`EUV}%$H1q^1f{iVygM0 z5Qtm06A%xQg30m$n$OWrXt1s0TJ`n0GjsBjdJUYq&`jz2`uaxExMLZ|Y;frc^u4$~ z`ufkZGdozcg1MxxUmd4;z7ITjTa*_+>s>beT6-n&%3>vnQ6=S*;c!Yy@hJ=RBBnmU z_n?VkbYvbw3y;hN-=TJp{m_)=IE3{BUQ#A^_=B-t&d;8Wsy8d<&4L&0SWI(?{7YA{h~h`S>0skH8G*nUi~ zT|udIN&kL5YiVVQvh%;MOHuD;4gF$e)8SmfsIWD2I82K8QJT@(n%&B*lO87;F7t&8 z!rd67^9NZ;)JlxTOMKdQ?fQ@Z5ar;}$_(nKMDrFVrIc;LespXYKnREnR17Q6(6_zE z=WQ-}(Ea%Q9xUM7w*0uVU&K5WE8PO833KP3IQHNfv_r!T%F|iv2rGrID`wovY~g0D zT5SagGTA+U8Li6|r=lR`z#+56gi*`!auJ(yr80`W(T$1L&Tr!DI_dfR{dc5@uHmEy zFA~l{gloBvqjsALVHo3cRxtLXb-`QiV92xg>m)HlVNFP8ph~Fym}D{vS;OQmcpDD@ zB&{+sHXxEZ=Pu&;5n5hW418Kvub{ppeIUf7*NV)t)4oEUY*lO{)?=iupN5deat=Bv z?qJeVX6PTlm?15Ha@lAOAeMCi&Ps%%AU}WP^6oRWlA!)WcY2$+gh>ntwF`ZU=CYtv z_S3NIJ!mq=KA3qy4soLswg0H&JeN~A3|y=HR>QDJ?33n)`V7ho)a7XC=~eqr(ZqrR zP9pjWm{-4Y5-CIe>CJ81vH)r5S+DxAajn~1jRva94lbWsQ9MTBRG#o9UjiN5w;!f3 zd|ij*!oj^mI>#wr76T|8L%)I{9A3lEOq?_YW` zs#|yq8+`c z@z>2F4iXJ)F&L{zbWIf-A3S`R4o(O&LOC#67&V^xLoRUQB5M^%!&t9KE8wjTFaF#= zaAcP^O7q7Iy+qwIoYlllpxGjofP|0Bgvh4niv58htWqjK=IMcJa;tJs){QxtH}Y-# zU_$7j!< zyK@gC;t;S`0f$fr_l=!BM(wPH8WkfTmZAgLm&6_JV>Kjlo+z7S$Jp8968~WO#Y8mZ zMj;bjcQ}L5A*L0rE#Ev%l~#YU*K4V+j?u`0Qx;&$xNzIhwjkkFybuSkz9)^g;{LCM ztHdjjqcDPP;u~R*t|-2-b;Unu|Lkp7+~vIx(=Ez50AOJ9U(iFDpG43k4aMvFv)QPK z2u6kyj=0dWA|g^24#bQGGFt}Vi#Un1W7lozS=B=g`Ky~aAX*hWGaC_LQMg0zdigJA zkC&Q}G|u!^F_4Qk^SxL;007Wn^Sv;=q@ikD62|)cEYeV;%u2`=Av4V0zo5%ieP1~y zBW?26G1?_<45SK7H`{sm=H=Vxdk4xx|KHaTqyNQqa}AKlGxD5nCM>PY^7gtQajqW7I!ljedR&XUL z6;k6nHxXDx(%?+Hro0E@gA9WPwgOIP^w1F|kFjPwo>8TWOx@s{CEjwTh*x;n@4ksi zIVmn3Oy=(fz7>-W1i-JlK@Ka)BV1_0K(s1zIv+b+>W8NV(m^cUq3pe&}V{SQa#WdeLfw0>C$0x z;n=8Ve_e9;IJ1Gc!_1QedsYtmeI{2Tl1^93IJW?+zz%ENJS_tVrQY_F>wf2f>*DIg z3-Pg8`(W;RzGWKO`(o|FE|dbmyR_w34Wp;hwu|9=@uL1;Hv^f)_0iR>phG=)VQy+# zNpv_Pm$`U1%e1?`>`S^);0ix*$tgK`ssVq1;L~KE%h4g`$FrLq%46>}<9CdNM2g!5Bgz<`DiOriLSS zPFgg7e$Dd2XoJvitK?;>mw!=r-@bkCZ<9zs6Fxo&NSqbcQVxe=rfp-*YG|_ zmo?ESdtRW5a{Fx7gJV4Cc8Jq8Yn!=R3W+&$hFB<_{$MHlZ&6 zm0}qkiIAjxDycy4nSb!$!EN+d5t%Ah0~!#9+4+oZV$hxJ^4@i>3Y6>N$AF_u2_tXm zyMUgC<|ZRYMuMqmv-CI}5O9aeEe7bD3`*aRwR#Xg@hETL!GVs=M2P5sTuGu>0~0c* zC-AhffS9#%<rKaK2GeZQ_!Ww(j1YnmqjPpUS|ozbFG$ zF{930xnv$%RGt|jW^r{ln>rk$c1Bc+(XQ8}ySZleMx`yiG4n#jn$Xc%-)ajSS!h~( zV@TXF`w2a`+@>6(Z%^TXNWAu(#1`+eP&Bnwjvub0hNjS@6ufdAd(%8}Bt}>R228zF zzUacF5|`G!Cb7~`fSTx7y<4GW+ID~W%iSNY1O{H6+5<<_n>NAnT|w(r0$Rw3aj~z! zc`E>4-|ExEAlC5Xyt#9~FG~6f`xR0w??1tqN>e&ZN4ER$@e${c%7O11%lDmp`p|Su zMD-6Kv&qoot{T0hY|KqZlR-tdZI%6=yODLspTLeO_MZ*dm4jG_qXKi6v#+BbGE1aw zfAL(ieBuU_R&k7REF>?Y6J_5X0gGIRka}5-Mq6^%_!ku#BQEAwgdc#L)>Ps zdp{;c_r#%BnF*jN?rt9XQ`dP5xi&hKa`@#2M^dCNCeBtt9!O zG5zxO{rIsD7KdzohGuTsE?e!nW#1~b?cV)zLPz4+rPiGc7#-ikNF3 z4>vJwX`cbJ@INg`*|AYzLC(H;(uG;WZ|J4AZr)rrq(*vlI@GHswKon`n41)kU?om{ zR{3zgMt%?dx(Bw9@^3t-tJks~z*fxR1r5WRh~Dh;%SgKHW{W=KliTuZdk_A!+T!7{4jAwyNZ6~cTiF29>McWFIu?wk z9&9bp)+PD-FAb=0G5Cw2K-qV(phLS>%e$I7mjQJ6xMhAk5HLYT(2ds~?jMgOxd}fjt^NP12EUcvp2>2uVy{k{lbKD&w)Kxd`1$M?%`?Jp|AW0`luR8z zTCoo>5SYe%FYps08RfBf$4}&e&M~R~%TsJgrZ!EDN(IpoM9@0d8886=nB=d`yd0~2!woOh z<>C~JXB!)vW>c-ONTHQHyCwU>_n6dejT_VmiwGpN3lw+eFzfrRTAoOqx9FEYLPHU3 z{J&k8mxZ}|Yri?s>3#u6_uD!2PS;cfo3yJk)oF5D#q@UGY3UcON>;Tx&?_85AQs?T zsoY~9%> z-FZ)0Sm=`J(+BH5d%qHDN=)6n15e++c~dm7Mvfx!6AbgvAUxzpdS#Lbw2{6?xv;{? zVvpt{jn?&RfFOR1Z<~$lmREV*ed$=X2+1w%0|;>1nI*ocbGfONMn`k-iUQDYpo;PpIF4$5N!K6jS7X*WNkE^~ug1ibtuM5L2v78cN=4#j&}?430O`^#QDNiepB3!^}@D!)!C(GD|(M(*k!$_+^mz1`!OZwGF^~-O7S)W zi7Z`z<;s;g*;OFO*dLlT>2zMVh0}%&TR*?Gy1Ke~^w-MCcy%eaQS3XgK43(73%raW z!%er$O-f|VDBDKFI}Z4Go7ntn7q6fi*%x5Xj}hKFcasUoQnE z%PvK?&%pYs)fzu6HZsgJ>O7~gd;ez8vns^vG*uN}yEc?0MQ-@{hbsSC+01EwW!HbU zN791rUtw}+lY8Y#ow{&HW_u4;*e z(~I*+-hBFW_el%vXeK%e5??w5*rGuL&WPh;0Q(pRiIk@mCm&rjL45q(p0&+*Qx=C9Xn-EUJUEz~ zJ$trkqNWFKNB^#*=;ZC}E^j#P_M~K%Gqs{QGdSRo?m9PjCUehyPa3-4wSH@$4QN4E z+rc;wF${Q;o6IK_ni9ASV=rTC(okK!filwf<5iX)#h>~dyfahPGb25{@Wr*tzG`ze9VJhsBp0Ukh!z4V^<HQ+Du1d!|Z=}lVvXLHz=GfI+adx znw6pwgVo$9f~I5L13b~KMc;SWQl*RA3ox?OhhKN@HoQwPw}(^etCYtyFVY6^_6V zr{^#JZt(l@Xx6My=Q>;A|GsuvcNo4Q^;o;=^OB;{T=G5`Hn}}pC4EBe=bv7AKO;TZ zE&@Zo@{#T{Z`~*#PP<7_XZf~P&3hk_A{vN00{fy;gDgETV`6!@8A2SH{YnyCOZR1_ zls|?zzhz1JguG6lzOKGV-pS|qw)$PV+zV`>a)X?oD432xDe+X$hYF88!JYI(!$w@Ih1i=L-sEwzN6mq6vZaup{=-N{hv)DdmwlF}yY?CHEgVEjd`!tg;xREu`92AhMuN8Z zcru_7AT-&)iMCtN;eF}VuCNNsYdf4u?f@_mW@CT=a!mjC{iUxP;#c}KCEJRWO23GfUqLB zq_{EwRKTk)0u2Ya_t{f6Mc z2Ya-);n~OS)BnR&d{Wx5bR@TVYGPhx!G5r zc}X3_<$LLM>rsiHFk53F=>=1Yynbt%3my$5yK6X8r`i;D_)>hP_I#=7HbH^CzxB zPKUl$fyBki6e@5A4=A}0IWKyvSUEims8~tT76?tcy`Elvz~u$Fg5VZ`>cm5Gv!o>% zI`p~9F9zD3$JKIEIq-l1eglo?*Kglgo zfJqz3*Z6pO4{av%MxGcezFb^hqAsXOv#odhBVYVfDQalUyrc6xE4{$VwRf*9$nTy^ z+05eBI>38zeb7~LW4S!YF=K!g&BRu*g!FoIoDl)(#t30h7%I2Ip`_*jXg%|<^oBa3 zs;(}5*WcIyL-+ltw=M&MwAh$o1?S-vcAfQ(6J9`8qxTORIy6VDoA}(kJOAy9zUt-m z)eS(oXi=u+&bAv-UjRWBcD!}{14Ay|QL{rk`vFK% zKmK4-F!4_IIIP|waS-Sg`mP_;7UBE?KTo_9K^xt95+Aqdk2yko!<(t|`AOIq&SS+c z1u~!Dm_rw&j6q5ZBbGTX{WP+kStu)CX6KN#Q4`*U(ryVJfFZ_)bYJ6H$-e7z?9R+x zJxAvdIXR5`na1j**x0j>LwkbU-P0oqLyqjP_NZdfy8SxVD+(U3K5ljzcQ_-4l*Z$V zRl${Xbka8w2W;ija|p7px)7ukmNRA?kwe1tK*il6aq>!(R`tO|8W}zsQ(M|pk~?-| zkm3(7O*no2W!GZUaj)iIW`c7uh#j?-5tWbS{Nm((;uX%imYKWGx#CZWI^y!d?8s1f z_ujq9xfeF$&6t5EE2p3OL2vK%v4e{52PW+QwQxMxikJeH`-s zF>JE3{yf7OvqD`ne@B58H=?@z^V?K%*JQ}d<}VvA-DqvBX*qt-s2zsxQ>%a6X7Zsk z2jrg?N22x&k6m`9=fn@4E4AxAv1A2%a)g-T#GBGf^oq|(_yhZH=^3S+gwjJ+5U2E+ z-IXyW&iV0MdYi)jRVMhVVBjqQM-_&3u47HcZEO_Go?Gb@Z)if#zh2@OV2CIjxWmi7o-_#fpt%Li zypT9*qk8{T5|~v~JoIWEjNe-SkhG~Satbs%t*k4rLKCyEvaE4~=ohyl5b`1vq)G!9 z}X~)3HW&L7SN9AU6D}< z1sCKvm;U;}77&8t?0J{5LXqcHbKFk$T4LpZGvZUES4)}s zjnqGGRx7}w4&R=CWfVCgGrpl-gL2o`}tj9 zh=?LVXs=?+`}JL%Tl54-W{=|HDnX8AbH%6mB{;QASRk zQ{}Ijb)^Vcnm%ixHuTMMr~QB2D^l@zw!guX_NF^iQ+~7l<*v(FwyD8y#l-T_-rGyB zR74uiJCa1|FH|K>DlOLF_U z#Bl7_fhH(B6_zC~3oEzz~C@XPrRgxSoiGwvu3B+y3|pg7j%o(AqF}+ z8Sx{KgMQb$CA-FkdM|~<$*K=Y$zvL0C2cwj`XE1eqmH>ha-fy+Y~ZaUy6g7dO>Q=MMCD&Ru_(7`^*h09L7pGWx`tW?oIM}vFk1=#y`HZY3-#-S@NL$75!-#3X zJsARuS*xP_CAJDh;LI=$GQ)IN_ z;&{BVo#l;^OhmzF=XXcK7#+=oViZDg=yPa{-)u50fM9?1cQ=|k_2rqQCHD7bUT}|6 zvVv<_{ZyK8EY1wF3-u9btWAXMPlPPTk>QdCDzOTDBF6YiF^bU=NY|p&B#!~oopZ)_ zaqj5aHT~m^wS?K;?&Rm>2HXu6FW_;VKbG(q+`u<^8!hro=nu$EwkBlfaAD+-mbvE8 z>X*n|XoqWU9C|@f-t3nr*5M-JvWnF_98&EfhdNxUXazBk*67r!#*C71EsR5n^^{!~ zl54lXc;=+S*WP38o$t3|;G9~LKXKHOqMH7B9}Ms1 z^qVW%{hWc6e~o)_;(x{P)y~=as9BGBz0b?TEvb0HuIDFeMIUX~|F1Tmd#9{(@7#RD z@Qj)H;}5Aj~=BeeP{$bqlEChm){B#p}R;Jb8^m~v}4!O z+%szJf^Dd15fO=>Gnh`bgpngg+(6^F6Decn$`}gd7Pys1FI-p_S>UWf6%gt1c>>n; z<5~S$w&QtLI;zmzRXD2b+ErXS+UOUHh8HB=~Yk2g$S|5iQI;#_=cI%Rg*8TD~_%m+mkM6^? zZ>&anX|j!G>^AiVNWMlD9kJ!g?WUucak1Xeqa%0s+wz*LZz&nPvG%Z_nM8@y%bQZ2 zvMLsZES_#GN!ojFHj81d?>apq9P+zDOrT}G)-1QBBr9Vq)DT|K=S5a%0dI>N(C7vrLcZrZ>J!kS0TU2}iJ z@jI*{CDaKD?dF&_mP`2k91& z5~*?F)=TiLgetUh)N3l&bL7XZT#AP<)Cg&@1@bDPAyRi>M-HME$U&LM^1j2U#slEE z%pX8UJF>oFB4Qr;5pH|?HdS2noo-*RaNw%Yo?yjgME5f8bEo^gNwh%RxoW_ViKXnq zGZ1watVtVz)Z2fmQID9Y)C`Hj}T4L32+)<@k`Y1 z>v>c;BTqBHgtH~8`(Zv8@+p?7!(@fE>5`H`qkzAf}N z;u>#RP*6~8o21gPS+nYR`=+vdmpYn;q2k9aqjL0Jua_v8P+z4IUv z3aWYT+@>e*r!H+%L6C%>G7NB_FV)+0>2kc*Cv~r|S^4(;)72(ukt-sb9Kr^re#@{B z&tWMZRKT&e_@gk5KM&g2_*9@S$)6M$pdAMrnmmTyT|Fn{5EZDn0AC1Oha0Y!Yp!H& zSA0P+V>oiY7JpU+z@^-v&~!Op_9Xfjv!aUp(b$xPn`MzRmG-|h%OiKptI-~IA{?zP zHq?zFt_StT!3#Iht>2(QG$;LdtzjBeVcfZ_qNf-Xy>3j$XS{pMtK08a!qBWG#pBZ_ zr;P7h*>EIppe!xArZyYkRVF|7l(a{}g1mk`Y2m<=Pb9{CLds>^R@+& zW0U*3``x1%9Y+}%zIFxXQ(?rF@I&Mc`3tM7U|AUjEBut^oTDZXfRQ`Zys54p&ig#5Bxl&YUJnZJ zQzm?3oH>?o$U=4+dhC^F+3G6ooYHYy+<-qMGaSj3Qt;im`dY>s$SiFd9m@@5DJ?!k zR-S*aQ!9^9cmAphFQQ7&WCdLf1Gr?TjkTqOi>#EmNb8Fn8```K_5=I>o?NIiQ>66j zgTzcLV(yf4G=buMN7J((QqIytEFU-!M~@w=3Q5vs49wFeQ=zvo-Tp|LLOTQbXwJaD z7A+Ddh|FdvqJ}Q0a)3(^Xqg*&U@+|&H?BH#oSU;34&z0Q< zl$-uz&--A~6xT;b$2%f#&@*a6KUX?o)ac>^A(?e->lBtFM|vb&9pLpB$`Bk`je;o! zNRio?SS^Io%}613epDo{&qvL>elaMEE*}h!z4poWT||t9T36YE*Wu0}(O^xMGZjki z8H0|$a11T4BRYwzh>rqNj0?61l)w!Z9iV$?jDs4o$A3kQx z_Nw)27FSl)+=h z=nXw|)b-Ma2k=-HPfRkzYDbeh1!#=2NYkV=&a z7o^ZaUrX}m zT=@BVHqLH6&UD*jETTgYbYj{3xK3mhZxwi4Whe1ELVaU;cbjdp57H0wZ`|yoaJM83 z#_79DUO(1rqG?GezG;HHfb`B#T$?<4aE@kf9EJrlTeGl>lqhTAh?Br*oa;M`^H3L$ zLY&cmNJ|h3y{_I|@?%KL`LT3|AzmrWFHM_m8;VJJy#s=gQE?1KjnqhUN-LS(`OqQT zRuZkf?|?No{lXf+?BVS>N|>cm@qT{B`VFToB8sB7p+`XQ(y(#k_M_Wc#`FK+Bv$0G z;mg4`8HM~vr50`gB-Yw!3Fu}SCfOha4y%n>_Hwr9D~Qwv0@rh&Y46>P49WO zDElq~#LDc`Nhe~3=^n0AZiYIZfix;RI4lZ*mP={=#JKz@?R$-$wB3=Hy*kaqy|Pit zrN(>j^?i2~s#X+0mPN$IxButNSZV$8N_a&vGuaTTeVqAaR#Kn22Tcwfv z5-0FrC{K$sszfmxOh@V&$*ZQ59rKR32$N2) zl*|8lWxor_ig?PdPl?8Zc6jOCT!l7E>HNSN1!t`hUh~2cqW)w7e@77cDI{TQh$(L# zJ=sg z3OJPCvpq7x*jg-4EPA7sLfcs(cn5%;K6mb#=sj$5B+sN`TIa~#8??IGHP8wU#nwUo zy!~qQJxnia9t?axO->xUs&6}hxs;iRz+Ct5k21Z5_Do}jPb%&d;6Sp8KTVS+Q4%-J zo4jMQj1us_G`84{K8nzKh+g=-UMxZ1a%Y~ObM>oJKYQDrHG&6&guWq$cwxFT+@A6y zbC3VzLY`pf|8ZUf*Us8O`d2L&PQ_{VkotE(qr&pWZ8@;=Ccx%~fD&Ps%uxeF56pWi z(oJq5i(Z=5dV1gMaLsDgpE!C*_%0UVo7rl7vHzvrikpIDgzXYr0=_wKCgFjY?^Vnk zAo})NnyRKMf1}xi_bYxWd$~nKC0c)x5oy}QdK|EFy8Ux+v$IQ_1(YGeZJPNg^!nol;))aXLvn zl{CYWajKzlLd!LmQY7N5&<~96dtUd=tV8P*h0SDTxL$t`K)uH#*AebrM-B=fl+fXD zqYK@82Q4tkEwUtQo0tS;qz$Kut$)_(WI_RO2OZbdnPefPw6L5u<4I+7V0)l-#2R8<>hhgN!ld`AerkSEp`=!CHX@EBPwpLW{On zg$fLPHC4U5kX<2ECn(Pdzs5`d#h@;BVHq@T%wbdqu|f^A(r($HZe5ODb^5%=Rclw~ zs$e*2{>LBRp0W?$+9`aJK*BYTP{Cb92kw(rm07(I~LWN<~ z(891k4?LjcY`ix+x8D+7Kl~vW=m`AE&UXo7Ed=}V+|)_6Co$<@p@}H^azZaI3`~#2 z)@}|*R>o_|Gx{O^Et&d@O6x18{*5X96~6LY6AHdJ6Pc&4Z+*xqcvBxPe~3!UCjQl{ z?R@HUA`@0a^`w<3%J2WuAA8+995S3;FQ(=~ayU9YrMOSho9iF{K&#R|=5_1uC*JOC zT@?MYvGP1k{y*ob_m|Q|`9Eu+@~@XPZ*T6BGvQoIgUgAfC&t(t-P9Gid4~^-$a}RYs!%8GvQd+E_Qt5F7?$S=GW~IB& zi6AS;z+VL3{9;|6Z!LK|-6J48gMPurDISwF?Np^tCEdFnhqt0@(RGiD?aUW+(k-FN zeh>5bAT!MSJ}QD5*50V=eFZ%t`GscYI?d}&if{=dCdCo$0~|XeJw{(96cZv$Uc7iA zNmVu7+MaP0(?I}wkT72?BoL6WEPNl_T}!K;R32`c4kO;G^D*+kvonZt8?8@9hkN_s z+t2>Wn}0A=L|nD#T3~(!XQSkf@3BG6#rC-z0K_7Xf+s>7?5N5<45AjMDV(JUR!htQ?&e z6bcNsEjR6X2BbvGWNM4MMoD6gwYzqylJuwKcfB0G_Yrn=kG? z{3(?rhfrXIk8IR4NV9P=n7p1s2y=U!scBm9Cw?YTIzrAw8RHIxh|k(Sp!dLmp1>}# z%eGRJ+EKdIzwjjf{HpAWMIvMsae$mM1(n9$l?Ln<{+)a3a{+WCFIE$pv> zR=xih5IC?@U-P$$<**O}veShg%`%fZJ?e|&Fn%wC|1z&RX0B2DQE8=fUjMndPQWaa z19!KPo_No@p8fKyKQ6CYi9Xq+&G23TJ-m4ej~;Ac<rEBB;LWX(H z*3vdf8~;=GUa@@;4B;Y_=eT{JgjMbPxTBUIBOmOnCJi5`h)X2gsx3BrfMUd5#ct+p z{GLh>99wxeqN1qbTjxb=7QAw~@#y(7gS!K};e%H9d3hacI+q(%F)(h*UJ$7LQpP6*Ow5x4O1Sai;Mfelr40HqdV zu|7mZBl0oOF;HzpwrYB{*odow(}k@~9&2R_QUVNX6uX3W83A^!@c}OX!De^W`O_*C z#7Y*Wx_ZI1y#ImC_Boa~l&vrguqf`HVVVEoLVFb@k!9r457} z(tr5KtvF~((D}=Qg)eT59aL16hXH4t1B`p5a=_*n0aKqt4-?M$F<;UJ&%jX2cST)x zIaZl}b4M)2e)aI6Xu~@EMiT?Aj6}vCgh2%gkLEYF<}GE?d46qGHykEd1FU=X$||r} z*6|fw{zy(bhWDRWU)$^U^NfjPTg(PX8R3{Q$0Zc_Gy^hThebH>M7wU?wq`D&n_kE4 zF0u@|6RQ~BaiK{mgAEp~!qE99H46{_E2+7#MhJdu*mcjAyo`pbVTx7v*Dc#z=g%YQ zYbNzkZ)=+BnC!InxAJ6Y@_%Br0v1=u!PfgHd)w>kcH@XS7`eFBRzG1hq}@KPE1}CC zZGRJdprb}@t6qP%-g3VlQ+BVig!1p!3oj?0N9M^iC;dn;Jh%wGaxgFjWW!Al<_54)}7e|l|*Fk6sPqNeqJ|-8?f}^C_A~% z>$ZO{`7tYrFXl;o$gMOtc=UwkXN(y=TKKtEfolR6htTTqgOPUc`Z4S5CAKf(UJc`Y z>p5=(fpl@=mZ04nQEFFucM)m+n37hlihQ$N>*Jq1X_%4ljmOc(=g*%vH8#1i)%gHR zDUU%l1i@7%)16yrB)qe3-3DgR8iT3NfO)}+rUxq;DRIP+!ZA~F00fZM9dh?1Km?7g zQkze8@v{XM|Hb=$iR1m^349nhJS$ojsK0tZgOHHD8p4|40=MxZemTgtg@&#D0VFg8{D^2JA~0@vIvk09 z3(P)W8L5Q6u!0HCsP(woAV7I!!6ZgGM83h7U04G4fPEhc~=s{jHE`ln%RCzTV7{= z^PuA1%~eg;j-S2w_Fxk&-Vn56^-pJvh@v+XLx#n8#bXSn4B{a3y)JZ9`V!w4KOAI& zvL-%w=rH#3MY7$%{nBw)@_oUCD*&=nX@0_z_zrQ4)gsjWvBuDQ>tm338uN)5yGlw) z0QZ;6lAI!}?*hv+*`}e@WP83wuPP>=miBnc^iVzn>g#NF@m!G3Z?lT_+r z`{Gh9pzF9Z3tfZ-&NiUp?h>L55@i!N^D^A7HiMA$c-~r7MRs%4xrou5HKAU z4KgSuCSCW~Ox_`Q1CSjfm{wtUbrtG3i&;0?s*^9fon~DJ8Jd-jwbGqZeD^wanipTX zor^S}$&iK=lONf>TkizD8x%Q{B_X?vAC4IN_unnBH0Y+KWkkgHyhR~N7-%=`>8G}S z&tFG}*w}OxTqk(brt0D@g?hQ8j?Q@9&pb}}QT6+4xQQD6tt}%Qw(Y6xhxafzkWwq- zpQ*0|?6SOVJMSrX{r355;OYOz)_K6?yubfHBP**UBeDq*Nm)f?A0r%t2{iObulQ48KZJPBcppuLc>fj6Kwsh4zv_jx>#+=!CaNC!~+MNrLf?1 zRv6We*p1NrrHDe}*#)Jn!C!Bh6%#Oi<|T+v3_Zgqj2H2+;vM>@m0G>5fm4!et5`kPxu*eLVfVb$Bk>ibF_a-dD@8e-w4t0I6pXYf9JmS>P6*$ zU-wPPUnBJ5vD*I*{*r!CmsZeKbwVPDL)Y3r0%cWRTl zRRW*@s{Lx&Emi`2Q7rZJ^qi=2%yp2KY!B+)wY7%;BHdl)`sCaD^a6CdW(x^A3RHA0 zk`9x5_u2=r)W{>L*kI)-DRX$#$vNs208IW8rQ$kdq1uryeE{=Np*}it>=Li9PmS6W zr%dTM&c=j*RoUohEqiQl31cMp0oa4dTpx3=G3MEPS0D)XzAG?bSwH&9VNDKLa%o`= zZ#e6PsD;c7SGOWaoC_w^AjggX4haRDJ^O{?rwcMd<@-1-c?c=aNw!e zPiw$m!4D)-#4$Zj>ppke>M`Gb6h8jCa&Dxhcz4SN79D^Xz4CVX%0wOj4!rHX4!o9; z5M_c|JfFsrnWi=@$!u+kxVlOrow|1YdZ*jnzLJ2z2fDG`wC~@$fBX}f5|<$k9@D8) zA9cT2oA0KW@He7Gdj_CKP#|4#jJkg>yTJ7`>6aqbQd~m5%>omh{n3h}Oon>UN{OK3 z(cDXrEgtH@WuI#hXFxnc5FI;#yKemt0oy~F_u zQtb{KVD&@jfZ5)!-?*_h^(-6^Lujv~Me}tzE&E7q4G34>%)Q&UuUl36J*BkJc2%qP z?XB#UPf)+S5xcJ&~f^IF3vZLBBuST1y;4>cjw*pyn1)cKC-v&pvb60 zZ{ioJ9mE<25TmX2#KSJ3w=)-*ov`I^FEqkMJHQFw>0deAr&DX=Lo0<4jdc zO=fHpd;7fTr0dtNomrYQf;2eKGy_kM?;l#Qf{+;&V}5EYcpu0jL+u%}-q7z0m<0i) zGZUY!dSM8&h0UD#lz)&0m`9nwc}!2R_@TC_$(DN899`KOtsGjf}> zXd(UNV8JJ5L@hJRT#x0{F%{W8rR&f+J6+rug^~UFoH=u*t^I{!J{6ob(t#>rES9IH zbQ7Y8b3g8jvB@K3b-S$Jf7YqjY`~I%4)e=)j9mM_B_P8v*-#skl`)xOT zl#&lx7FSVJk-fY;BF-JCk(xZ6qnp1xn|to+{Mf$adACeVOsu`=6IfZA1q;mS^xAfx znz3^a&%yJ?^P${#eKYirjYvrfpj)YXc7BQB(x@52zj!Q@Jh%OnV%~oDko6CpM19Dl zfCcBLNeal=Q)hBS+Rb(yel1=meYCHYQB`w&m`!|vHCu3B8z*4^kcL0znz7|Ys`XLc z$T1>1ATh64TMy3v!O0ogbLLQPqCr}z(eHVrW+9Nm1gA3UPW}RzR8-}r6s*)V+2OJW zt?5E0X2#gS!%yfeHS2|#?(eKI+@C#)`_g#E)JnzSzWZ+<*IZc_odh+q2>_YV5paR@ zG}lAfrZ!jODdzxPlx^ofBUNG}U(lExzdNM8UCvUNxzy3|^zvJ4y|Ef$vq~#E2K4GV zzNSxQ5S1@JZX?#-G6Te3t;h>`!}osnO|To!kADI$$Cg*B-C(_XA@~bGbL@Y%t`QrD zaR#NbdZU+Fj+aDvSZ1>mjC9(09p65b4jSGWuL98jT zO~a6XNL2?65}AF^>|-Db1T8@tJAy$KVM1d~&G(_L{^ygPxxQ^3u5`9ZZ^S}W5NG~~ z;v3|5^}aew{NJhWYO20|Y{^o#l(~pJSQ)(XE;^msKNoAXn9SZ7-wL&}x3}fwyjUjY){l4rS75f-;wfM zt30)$ZqSIrsfAA$zWlh1rm^a0Jzbo_9v_jJ(n`tL_Z9J9`=`{(UzPwG140;RL!QEx zvXh7lQ8F-MqFW7U+I{ujq#2T7Npr~X2u2Cr+v`rq8&~#uO=^B$;Bf0z-?pV@71>}| z1>MiRU))3q7_fj`61p@-tWK_enj<2P^q%OH%fyo*&#uO^KWfuXeWH5B`pVhnYKt-x zOIB+p>FD0zmVCm+Dhu|InLcL11d+*YMu+q{IxAjO%+|3RIE_dUO}#uGpNhf4$7E&R z9-ifaTJ~l*cnHDqXccNASKym-9kf?DW(1L1zaF&o_rsjM^d3BX^ynoV1~jb_vAavD zl@>|G%jBWc(PN25w)J4*8#Mx7pA0U&`<@3Qtr$)QF|x@(V~zRvcW6(Rra~U0pms(^ zGq0T(N(m>%R~|SY?z!>`=BBJ%?!I-E_gjJSgZbaK42b=9HZ&{vO5snnl`fLnP2kwt z)c#FZx;O8!#cKhG z4{3JN@Y6_B=s7H4sGh*JEbb2CiW%wgJ+_3znwnjq~*_IAQ|KYiP>sXEakH5+Jm0k;jLnAx% z>0G}{^41+o>rOgisy;~(qW9rp{LnT>R=&@B{9Nl~GwNj@tLg=0$<)5{%3YCfDsPsivaNv(C16AB%p5)`JOw>Me=R zvqC?c0Gdf$(dykhzcTxN1_!EhtuUu6=pb)7O&yv_+2Bdyg9ZZ1UFAtUorp}qaE2FM zR8eB0Mf`gW{wcS|y?TRm4W_YP@YP0?^kAw~VI%_ss1iJIW)TpgvN8y)jW<1?py{P% zpH%K%>Y9=qfBH&6g;XY^d3m{aX@ znZ;;iIviV_?ks#e26stu@g70qH3)c0TO2Zog^#ENO1=-fs%ajytd~y5G@E2%rMz|?gWd`gJE1XUKFEbuX?VZF_$zC z$V>hG^2doV^TC5xg)M3f8r)gFT{MWmZBk#)s!%N@=$~6{`^9EO zmk3rP^T)>7`TNPeT|$$HLnhvWeZcjTub;LlCD))KYKWbREJ@Xjpp^xIcj16gH?i}g z8m0zmuS?X`Syvcgi>GExl&NuLqj1q&^}gsE z4GJD8;{xXbVMb&3Ouft9HfYt42LnI7c_RgO)va{Kb>0eOO%g+vrY4|)m6g?yygm6Y z3nQO!$Cp0MTZEN?k|!z(zTp;4kV7n+RMA4sF!QGiFBs;&a9k}NsT0-j&p%%+yqOX& z@9+kmK$};*F6vS}Rn)bP*uz}re<+G&rs(lQvxYZ*8xmA%8ij@ifsM(czJqK8yld6B z&0pH|8Zd#31F8A?Y?xefO$m|ebbH|o-cyOsVmk`A!xZ}X{DNk$zW9DD)m6r!H*#Yu zIr=T-%w^=%eSi1}A3;gja?0Ci=uDae6Qji7D2k<-M9y4lRiw7U;%#hS(luHCWRZ6+ zEmIyU8y)>*gh1j1*1Oq)30^rE9-Y+=vbm^tk^9Rq!jRhy9H?1^z+t!cy)*0qj1-t& zYsU`mRUu-wKN+N>4IM|sF2BENVl{VncLL73&z?GV zx`#)!Kqj5XwI91SG3P2^4K&u9-h&3|Ok-2Qa`I|V?!<1}up&xkql12CMQ~u?bd8Mn zQTbP+^3A60h-xr>`G7s9JJ)K#9~OFIEC}tq(+=UJh&>e+V9FCyTWmg!ymI$Im+|U> z!NG~hH9OvRcuP18MeGB4&OXEVz=?A++`Bi4Cym&YxWe6h4rFYsY2vZ!F(}9egupJI zGpDWmb@=jZn+*bb6X!JM*)`h(K?pH(a&Gj-^Sjt{ zw^g{Oulx^>r}FK`oi6{wdhkn6wI-bt?W2}DXzhxBeWcHU(lNfUExd5ffRR{>LuT<#`e70O{L>zM8WZ2$!s`expB za@&d40QvB%!q{g#Pe?LNEn}Ygsx(`I9?0y&@L@(lBRZH(RS{`Jer)OZuPwevX)NSH zlV#(IAbHR!hy?}BeE4hL9;ABsmqThk>CqFO>XLiWsiH`>`4^5}rJ9v{Ti)x}vq82- z0eQ^b|?|X@D3jdc|BH)YgTZlV>R2t&v8PL$d zAgojJ+_yXg&4Th4Y>a5d%FmyN#7cgioSN^Ibs_CR7%?m;L53lD(qgOGloV?3Gw))? z$H?&OA0J*0!=3?2u6_9a+m5$8e}y^pr|)RjxV6WX9H9tIW zOto2TISbER8Uj*Dj1X%1t1OpZrJ7L(1NOrORfd&kzBO@vr&mJ zvIdDrCoO&T2Mw`2w;Jr(3N!K^7dOUtssGC^y=B>Ez2BRU&)P&B=r?cniZK?AHr*VY zc~R>>{*<9z4~_r#-nP9_)NkwFJMK7a zN?5WuT9W?H4ELtY~&pUS6@Y^bho5lA0->Ez{pdx$ z%urrxuG5b}Zi8YICxbW3KStr8+$ZMZOGt3!Cvd@~`klQ@I&a;7;zWI(MieHEmaXF; z^O1@wPHUbg$Vm|+s(1#$Lo+Ud-w}crI6NgZg6q-0rHebf$DhmV@?3B27kPIZq37U@ ze!K3A1CoV_Ig#{WMSc^00Dh3q0>nEEr6v}&mBaRIz zE;%1{FF1(Q%c@(SpWhV1;HO^g9t{LDMmxtv)i!`-u(Gp8VvKkCGegVeCEpU8wTdXn$U^M9;E_Zyuio$v_~+2|v+JPd&6^LYdbC!bdh*-2alTb1QK*Ad zO|pkOu(|bH`Ss|Vl`op{IG$nEkl3e9PhRFOi(0}G%Be+IVYqPQfbOXSXL?<$yuVJn z-1F~xq;~JwUzCl4fgCeaOCSALz29uiu_pgYa8bPO(!F#2iUVz{#qB)u+2L9{pvSQS zJ+^sg)A-N~i+0|jM$OM(zj#muKjz5JTZe7ltm2Z|8sbY#U0mwE59TRvql1iLmKQ=+ zG%(pey;F@Y9?RvcW0hCc(0Ht9g+6u%-mr~%3p~GdgVKvhpudyKkEneXswJrnuFUY$ONfr zT>7DpbeeCqP%>g;7+g`-2P4v|?eor06O9kST8s|5AU%Bj<~ zDZpf08;J1+`jPJG64F2i3(wt}&{n{UO8xhYr^aJHS@xZ`z6~08D{pH^NX;0lVAOs5 znRL%|Q`&iN5~uolKyuj@H7U!wPCoXH z@H!Y>r`rzmB5$1rohKBQT;lQ@^yrJSKEP&=aDY(^hEeif@m3^Ab@#B4;=;^Rfn?8nrq9 zV?GHCy8qS`O&NYkd}(1>!Zv~fz88SSn=+Tftx#W&6+az|+ggTZ#gP0tOw zY2G~hQ&bHlj`N}R_FY6tf2Z3eu30hS=ey$O$|2H?6Z%awk^_l&Z4YS5~Fx=L*gFLoB zl3gv$%v2e}&%fEFOTiwu$i$t!3 z{68OVQc`vbdp06p%+hjaR_^qk?82tDpR%IcP;w>>QT}>$jU*>*sg+fxVqm7mm=Dea zQSF&Va00|0>C~}Kb@Wf4?$22OK_B?3RhrGqWl^9RKH&D@sc)j#5RsbV+pfE3d{Jh) zS4tjM3=#sH#*=p`JSD3^$tvS%yE(E7QVg>w5b%2mJyFcZebuS!77O9Hr7=|=9(MJ3 zU_;cNs~$x@StxQsi3VW`2AZh-GkWkpY}9CR^8ebkK7GRE?QuMhR0O1uc+1+NjYX4O z$w+HJD6Jz!ynz4#f^^_72!8i&z67hEogU3M>sj&Bg;2%h%QH?+wdI?%h_H!wPq{Al zm5P(sjg5`;^UHsVCM0v?V7Rh>z6O33qcXMNUrqMyZ^5R7bR5}sph5j3rAO`>6*&8b z`)v>0wsiL6olVy~?iKGI_vlQD@BSyged9Z|pZ4ih-Ih0P)Y|?QR847oxN#6Ky7soD zytBt?WT8D1^yZO})OB17?eMUp`xm{oEtnYuEKi)1=C`K{RdlVWU-RZ*$bsa}+ySb^ z;9VdZWnSphdpa}^Do5>SngcB0o!Yfd^$jBqX!Bn*{_JMWu4>$1X*yY6=Sr+4%cGJH zMb1cuKrN%tW*87f85v$A0TXM!WLo%PE`bTDxw>snhx6=*CoCgJC{Ad6;#~&>JU3ww z=(Ft6y!)MQc3T2*i>v%);gcGJ9Q`DvOga#_uDY=4652sjqsh6SdxEizOi9_IY;r}? zSYBLHMy3|KEQS|Da8iDlBoh*T)2pV5cf5@W-#Hpt{8$~0sLGBWS!>FB7r8EEZ|mr` zb~O_)?tnlWp`jNJO=8DN&GJAjCoHtkz|_EKPqij(+MuSAq!$uE5|1h%V%h64h^(IR zh~B+-Z#CxX(&*5DP~W_^`Rp zi4LwoV3%=GvzRf+c{ul>61_is_+YofOPfLKH0k`h_luZ|=T8I=Gd;ItO=M5x8k<@bd3@!D? zj~z=r$JD#IwUv(Ce|x`Gubw>{lIvb)zLwKZ!gLj#vz5+p%CiiO3P~#iFaL}Z`RYfF zW=&79^OGWIYT08$+d5)96J9J*$I!5uBxs2Y=%!9Dz+~WW0{vBh5h4e1$RVuUJfX?w z*U1bCaQ&yB!CkGr}Go}j(^)iO8WiP!r;yAU6-)MQ$4F-q>>Xqe>g6e|ho{Fk2`G9N-~ z4K?x{;xReUXk?0U{Ey!8@ZP~G-BbZM;bR>Mrqt0+tjOH{7f7hWj9ew~pgsy-qD)U5 z=~V<+xjDO5&z^T2^Q_*c@Qw`6L6G6EXZeH2Uy|RYasu|bhi-U9b#GK&SP;jCjC;eo zrC4OF#Q(B|k%S6kUf^ZjtxbsE`ql%TdHr9)KZ_=iqaSkXJ^b_Q*Jv@+!0D=0d~!r4 zPC!bDepVS>pI{qWccbPNh|waeqTD6zThl9rad~kx_d@t?(T*$cAwjf0|UTS{+VIeLsD$`N^rQr*cLyLk(KY zD5#ME1xq{Gk>3xP47 z17p}u{!x0d9c)XxD7^BiHK#W4Wr4fvPrZ)@An8t9FQv8ngr&?#;R`O6)siw~IA3 zUjV~>h1G^og605HpZRtL0!m4;f?r1>{#Y}gR7hYY8OPcGQNhF52JY2VRV^6r6x)NU z$;tY<%^L*d8Ppp)YE+$Rwy%VV{`SpZm}nE-vSm=R;fJF+IYZAqxqw)WQZ8*`p4Rd? z?fJ*GevGEf0zGz*FDSaf?pxXPXm`FBk|UtKCaX@x2o{5*eHhu`E=tEz$e3Y=DKw}% zMO*zb6#un(m*otJSJqn>6!oY+z$iKXT9j*JWDp)O%%m;ucV52Q!-c+WE2o*pN>1 znoL(9GnFlee_LGpQZ)8 z?mu?-nRyEHxHyZTtqea+7FTE$?QD{fE3YwW*=I81)7SDU&ITzox#e@v(7<3_T*c3P zD(jdM-u3h7 z@vS}uJzo0amj2wCs~>b#RUUqD(z=wlY1-rK6}eykzTeU?p>=G7{vtFU^dfCj{GlN` zn%7d|R0T0;$SM-jFEZOQYFVd{F^4YBJAu0a3lOVujRB|mpa<2iUAtYK)may}>cYo0 z4$3odnqCLl9eKR(Ab;TiXj3O-+Jo_f0xheLxo_)MtJPwbr!T z7nc-DnrEN{5u50$^#j$14l>h$LP7**l9FS`{wJL>!+-5!bIkhgCH(n%2|2TIaNG7*=wGjsjc+TLAsmA3xZ1+v8 zOMAuLITtJIVej9sfye{-e|EaCQ18F-ljbg~DWwn9cL}o$g73FcR~in8ybtYl{4TXR z5i7sVPb)8`en7l}+Uh!brN#_Pl0AZ7Wu+~e41SO8J&$$L!GcM^Lnb~Oiz|7%|5v@3 zB9`G1(9hsfr3F59&vpd(X&T1v>cp{`iC9IYD{`AFrZMP;T$As9X)9x1c9>&q!IQlJg;FEl@YYXVB(A~6Kk7hdOK~NH8c(suBM|5t44zAw z%n^%fK~)e~y|u0FUn3(qHJad%eEeA9Z}1G|8hsxae&$Rs)QW15E66+jfRC0OrXXv| zbBu@tz`T!6?>pDdPI8^wxo!L=-5og?tRyivI>~9-vNQInTKTJj%PWDy{Kd~+_}D#a zuqdQr+U5wYwmr;!{?FKGT;}roe=f7|UzeGom2VPqD)+~=GPnP>e@v}?gF00YyQgTp zKcx1YjU#g$&HaG$azo9mf)bkryC*_^4zsg+eRiJGu}Q}Wqg=yK>zEA0h1^46jbTHf zUD*%_dab$ePE<;c_2SPzdlDJ+na3cfqCBfYh50Y2f|+j=76}xpnoggEa7eQ_{;<|^ z<7_w8KcfmY)YWTxWHxikiu1BopMKeS(Ou1E;k5iMzL+tP6vM|+PDkeWI{dwW6+?S( z+P=N>=<$Wp;wMiRqT+y;fX6pb;kON~J-6aXI76?L0lfqoSAcj()d~fm-OT2e=Yh=f z{y`Nl%6tvU7!P;4Z_1Z1b|hC(ogya+S4iQh18EMTXarI>i$+>NXMlRsVT-6QCoMzc z?rv-0lK($4VIBXt$>`GdRw=1A@CoHwBN%o_V-4rd?RhL##$=Y#&s%rm1y~Vkq*)l# zND2*F;p{AqB(`qfwL5)JvFyVDi%GBGzdzq`?CtsVi7@|-22q`$Us`-P|mdOROL zzduD;NkDxm7Z|#H5xQoyB9DjXx{71YtN`{X%xI=1qTQ9+U}1A(xWiN&^XU*@}x;pg6l)J9j4~Am<2vNWY+yb`S7LC%iUW~&_M)iF+OD1rO8HFj z@o2eefqLb@_;Y#Cz7P5hqR{KWW;SE3Rp zH{anrdXL&>Ydh2K5sQY`pY+pXa>b09IdpOGU7YTCe-3+Qb%U@JO_WROl~&QD!`V9! zT9Sh5QpPmSXO~CU#;E7 zqj5<1&*QQlkp!X<&mjxn($#E4Eat6pP)Zds+!Ui}{4Alk`zMYHd6MA{%2!^J>h(ka zEeUazg1D_)JMe@V;*1do4AG1BmXPd#x8U?aGQF~s6 zH_}#9J9NS9yjF+>s<6h*99y8)N_JzveizM%e8lF?P0*x(;06uuon}wn-<@d>yc$GW%qpXIb&mE4Y6Ch)SH0n2?}RCV3j*o;+%bL}ZFyTW%D7{G(EX zj~M@ZdBrnZcCm5e$N$i82rC_`9d>BZr`{bIILI~S9&63VQbN4rkzH25*K55uyC8hR zFm>`D`3<&fA9y{kaZM=MXGeS558J`RSUQKS_O^3@_ z3VULvV_=|x3(%&<9lLA9ip!nWzxJ^-hsx(|z<_ByZ_=MB4Qn|1_)eEXXoeRiDfMDK zA<5?@Ba;m8ybU#0eysb?GE_oJG*DZWHu!5);>Q*6CWxpe0)&9W**Q(<0t6V2$RUUL z(B`dMf3w_~pF_b9R#+P0hx%cz5rB z*%o!J&$Y~(G&Ec{_j$EFYy7kySbnJfD1DJh&Rgr42wKMk_92tTXYEeq37(#@4j-2E z!%Bsov)P&cc6u$Shh(ibH0al_&YG)@&-BkTc+Mj2>GmAuJ|1gjIXFbH2n133wB#pm zSNO}5{8=YlgT;Llgs|_1IYwN>SRP zMe*os^A4q8mVEmtz$h}2o#rLRN(?(H;N;(n#A{ng|iwc+SRGP*T{Wc)vYF-F2w`)jybfN zMA#}5jDE`$#S{C^E8C$VWX~6Wy{elvd2yhV$G>Bdm{tY>*w<96OPC|Y%7O1F}M zULz|N!R(0?vFP7gy9qiWaHIWgQ0S15Q9H~WaOVc3re3~qVd}LF+b>F<_K_na)xmC1 zIGr+Wp&Sle%n&VrBbRKf^Gu#tOgqqI5MUifwTPW&ZXTN2f39o}gDYCG1*q8#zp9Vh z56E%E@{M(rr1F4Z*O-Du+ z+0e{UhhqXl@gT9KA6Mu&Q8@*UFPSaOrrwrnXo3r1MNr!?dbFz(`^{fuX+)eg7UE~l z#eQI#aLzC89U5DhIjlKI6ITN!z_$C`svb4X-798adof~nwqicg^}gytDOx zd1r=NG3ssYu&E94eOGxr7G*EGb?(7Ur4wc&XW7nIZy)kyUvi1VGu03`Kfj?fH^kOy znKF*)!tn%a2p*tm8V&X`6+(%2oge!Q&V<;ZN-TtgrrbtL^dSRu|)ZuVW~k1k1ug*d(4wPqKJNe~+9l;oj1jYaHLb zsxx=2)QN`d^peFKE{)Ugv*qE5+Oqm!eO~;e*=+j2g;XUO{7qqtGT=hMlYyB(`t;}1 z&j#Mlt329JdUq)9A$x>oXufiJcbImlls%m+d4Uv;*_cdaVpQiqB-KmSGPL&)Qw4a8iTN3n=fiABI*$*}(Qv25 zx4FWk0vz}twHPX&ID4MC8ga7Q(N_#@x z4=Ws*9rq6j_&*TH*sd$8|LZn0RZ-m^Vw(}rJfzFW?0ffHEI)gxW)Ibpf^G}P1P>i! zx@5w`NI=HTUdty8h@c*`>}S4LQ37f^)&_4L@3A{9KO+zZZ*@-u*&$GNUxQs3M26>> zH#IB_jfzjpHTb@=4UaGxGz2sHy~!*7v*age{c=eQ z-Mq-PKNc?(k7kl!#j#W4GL&fe0?t+OuGuS~HHE!4)N` z$kDfHM_%kur#i;7V1SbF%358|AMYoju3fbquQiBXDYk2?WU!5YZ`T!j_j!ixi-J3g#2M?|dhx9mxAI9Vx zyu2|HY)1qJ3(_x2sT{AuzU%j;6B4j2%F*mq#m~43LrRORbEPw6{CII={_V-Mv3VKo zbpMOKm@>Q2iM#ct8oD))jqj;`l;8nj^sKGJwWrEzYOp{5%j+b?qD`!;=#V=mJILH6 z3iRvkVmCT49JF)#WMj-5(5Q{E>DaMJnnlu0iUL8jQ4mtzduv-CWv@ds^Ej_a+~%a; z_@7nGdIy*+jbcupI&+-J=panW?S)JKscVN9UBWadHDWA@_Rm=isJrN3WYqds1c8g@ z-;a$#c^7?$mH7i}Z_U|NsgePmS&Yq9P912lx{)dy8S%=%D>4f(2*FI9VS@Ee;X^t+ z2WY-utBu}GHTPtkOM0>7N=`xX$&wIs?*Ibp;(mT~G;T1hTFvN^tfU5eeewop4KaMw zN8$(01jIu#RqEALoDY0PzFVqa7p-pSoRnk^Z!9#r0ZlbDZ20tUdDVw;t{-leqgNp~ z@?Rc59Xph=Mbn)ztr4Vi;FP{`w0ZvxG4fV%I{H_=>J#uaX znRt5lei1a2`BP%#vwnu+X~bR`$Mg-U-80a3txw3FJxoK3T@#_?n2g&MtQyc5bp+gU z6~i3eh+@{rQyCvUk5txBLt_M^4=o@6CP>5^!hp?FvwR7PYHEx8<=1)zIH4#AT<7x2 z3vr%=C2C)oo6^!?<(8Wa3u!b|24FpCr>i?bv7tIkV10(&$~tZ%qn}S#aNbAoZ$swK zHPb3}CT9#=pvQ&N0vBFddO&})*+g1YMY+3VKzw>N4&fZ&QTxb8hL}$u$9WBwlw(n@ zBW%gDud4*9b(Js>KLiT`7@{@J_t0YR{q;lp0Ue1H6jwD+6Jr+r&54oZ*J?$y>#a|b zXlm|~H7C7@;qO-~4e7j9Lbw1gn_TR#+nbYg>C3gLs~%q|47X&fgLqi8GFcFD5=qX_CNC{XI|Qt?W(QU=`PEez01SX_N?9a=ev%p zmV|Ylt6{l8|O$a}1-f;bO$HHWL2ZvGmkFclgNkxec0IL8hsq<#HuD;;$EMA<;COb8i zur7X>oP37$dshB)LpPM-l>#2F5^yNTG!N_q*5S6JZNh%J5@=pARC6^cihbeCWT zqi{RA7KYB8`CL)bKQ36(wOMBiH8ySpn?TpAelD&c{x-ILj%m^~J?#pMmvd+}HCAvEYY^Usbe_ppUYw($N77q&NuC#K~ns3wO@Rgwn> zL8IWr$bIS35>mBJk-}?z;i_riW=?~5`LVHEt{ra+S4mIGFVnmcF zos#Fj4RimHfGVGxw3<=*C3T5p)X~VlKL0$Rz^s)kpE^1**#1kyi%YKCw%d+QcfaV# zO;zZ6?eb3d=%wuL6T4Zq%wC0p6t!@m4LgW5ecP!&m2c-q!y=pTZL1w|L-0~n>YPNw z?5|g^KB5*R5&I7vQsygloG}=R9>D2lQquHu<8H!=Q2<^K*;Ro3oH;sw6nd!lqr--< zzB5HFw>3j0hUI<#K7%#W-$~#O7_2zK!Cz%}>O~h?Wah6-Y<&X*8wd~<+pq8YcKBze z8oMnqhkLaR_UKvqEqfane1DY6DnUBm5W0O)tE4V%thzZ1R5n}i3RX%>jW%t9fz*&r zj}=u8S}Ei%FQJhOdlHEu86;@W*6eOk`C#1-{LydS+Ryt^Q&xJPv*@wzzaw7xm2PwX z`D*ypxmUdLuWlUM7u?|bb+g(%jrJt$u$?__jDJ}tlTL|;zdERQKH4!(HOl^4ySHVg zruz@uU#m83>L59LIoutiaf%8SPq_cuC(3*qS7?uNS*YIV@z;c9g^l|qkA-mBqiqZe3jPlJ`eQ|GGq}@LU+Q$^23S8Vda&At!*4jR z3g+Fr==t-xzDX>-h+i+W;39Ng&%xN4#PaHdn9gHZrz4Y?kt?J1Lw*2iDZ6syV zm{sdT{&Q=QjDWre7$5P@%bHyD;mI0E=20O#E8q~BrUcp@f8@Ts8y*tl@?zT?2?=B1 z@Ij6RlqxTCLus9Ic|3qDV*I!1s}Jk8`~gvNI@1CZBmOj;Rv6n>BI$)f0V6)$qrTOQ z85`KAxsXb`FXG5FXLLeN`M0CeIIC+WUXi5Cb_!n)vm|P@uGd}CDf76mt^>F( z?g+9OI^GUgeTa?-q^CyD7BAhpmy}75$|7be1h(Jgka>3dv$&kZ_)fl9e z^<${qq|l%@XK21?#2kAA=Y&E2&tMiWIeXFm%t;?Air&p$-+t>iYgC=kIg20FI2b>t z_rw2)HonxJs`)=^E#C7@^kGa4QExa*QLoX64`TA_bGbBSLdP#_Ue!_lLw~dOn0o7N zHb3)wX81Z}fHMF7mzPfGp>~M=(der^0JRY*S`>YnK2aXYIldCpASML*agBy}rGz@J^RH)wPU06ww?+XI zle;Z=vdAo98V-)tF=Zrs!L6EEK&UVJa5D=FZLE~xl3cB#AC|)Kx@K*Ql3#&xm6~}H zHwN@9F(!Ze=!18<)cYU(7ra@epK|-7CnzOwZOFoERfDO<7y^c+WB6pOG zTFuW+osu5HEDX;$Pg=~TjKjK3nH?)l<{-A)%lg+RDbFBGcq!!6cQO`!up=#ruT)@$ zNm2T@PXHRcz*SH=PLN^S@s``jdOa~!wDBraU3{_lM>@-dpiW!*u2t#84321~k*pdb zo<4SLUgmo5!v54OS9&A^UmRWX5Ocq8U6^E1F7q3<>Tj!e_ zn_D*8bnsmB|1%q2zjt&uQyTj>tPc8w{%j^yV+`D6#`#txDG z?wuvUz)j4f9NsS?C2W8dw~oJ8-sN353Tdtb#5B%dMnDVav`p=MCv_~es?NG=ki9Vx zJf5O4-M8)6?FxCLjn6g$2rckoil@{b^A~N|m7{ z2M*v)DFIKAWz;l6Vt=w?XDT*ih7*rlWo0(aa?fpok(ql*a&cM)bsp8?$NG4itGHe} zKGjgV!WJL;Eo$fMm-3Y@d*zo}w2tLoLS0M-7RAiq6_8wf4q2uFj<%r_CIqLhla)FHD5yn_`ugq91%Mbq1@1EBL#EA6{$4~oD zZoo}z3!l)x7R1WG58aMut6jTRAcBAxmkj!;6oyQGcTU@MPU5=#V>Ii|{Oz$KS~V&> ze?@zR_nWX~HotGFj@Ses)6~V~qKhSoONgfFsL`3d&WCCe5on@07bORvXy;i@h|&TQ zn3jMJXJusZ8BHedJJI>FQQN}kPv_eV_N>hP#&q{RBhHDa0Dal)EG%|dTe%yf@&Sfc ziwB0PZjhw{$K1H@d=+dK?7E@{#v~&Ct;e#QbUl+nxr;>H+8{13esdsLBfflrqho$_ zP}7T5UEKa>R}SPpPCz&3v5Z{JtH6r>%uDms3Y+kGgQfft?PUeEQkj16qs7@BgaWb# z%1(wZxA4?f7%;G9;Y+mpZ`;0Isv>2=Pk=CCE}^gRF8RWuLyU{7*mjy|9Dh`79R>~@ zSf@o3)wE?c7c&STR;ffq{#PHx$moMFn*&!#uT#AS0M2AwNrF~%ZyaZkO%#ZkUP~T2 z8DA%eM05^X8K$(#5|$I&4FFeK(ym5!gE|MEn%&pAe+dt(Q1^$W1g{Uydyfnh@&3Lv z2Xy+Ra`B5zJ9gBeoF%gSnQar)yPmh!yjcWi#&s@Z^VVdwM-pZ#n^Z>5((+|UST4=* z((eNr`D}2#O`x}F{i^7QdmSS;Fwzs@Uc5W_S)_d*$m}+SVH^v(>xP&18VH{g*aG&A% zh3VjlG>%irE-9+uSgELoFwMw6*tZERk!NF>( zW)s1k@Gy>8?p2y3OA+}Kqd&c#xk04u?8K->nPL!$18|#oczA5;6V^r+HRS{rwLoP^ z<+eS(|DPpI5i+3K-r(dcOyQqer)r4AYb*+5A zJsov$xbCN1>s8Sud;PTA?rk>9-y0VmpY4MDit>EE7bPV=rNnv49kHoj(v@!7+Krno zq2s-gLyjGNm@NnH8P6Hp9$K-GafgLzhLl*#?TYX$z=lsKn)zD_ zhXEy&mx+&Z@x|%t)^VOjoOYzWr68k6#Lg!FDiS~Li^x>t>FR^Cih5&b<~OjRR>R&> zLcU0)T^MGK=Zm*cjHJM7JZ9uI$YZ9ft9vwx&*FoJ5Bqucq{}$j()F^blYRAJiWVT^c@z4Unb97=U z8v1X`a*HvqGqM`)t|Y+5kH7ZOIkKRZa{S}RI~-cUaeYi3NCuB0V*HV<+FS2(hxcqr zCR^d<5lW2bH}YDMhx~BMVCPtMEZLQ4lm+#e;s?C#AQ30LUh74J==SB+`S+OYh3e(r zLdHJ!yDOOk(nysZY1D(dPl&)nqXSkSrr;hka^M7b(N=thUl{-1SmNHKNJ~nm0F~Gc zkOYN)`%zjdW(*b*W!5zkaV+n;82fNVL;nY+r2f)Yv@>w1N z?^Aw?RDS8Jo2qH&59tC1iS{r$_FY^R`O_K@A@hBhwFb%|q@Id{2j`g0oL)ID`V=UP z@7P>l>HSJ~?6Lo30c?&t^T+>`H~}X1=squPcE3$c*416Fx#+sVxxtaMpTgKkyV!PNeFna^>IsgSD;>HaEAKp{cNLe#;QmwSYu0!L>+ z$#eqW7ZB|(?_{Bx@b{Ue-3QUYmaaS}mKKM!$8#f<=V&Pz2L)~$bLi-?hm2vMBdhsQ zwD^R6R>~^vYn@1wR;8}Duv)6-g_NJG*o>4ttg_u|eBsp@*0{WQF?mmC=wR8G zT%pHa$3yW$yUUy3H@v;{cE|RzE`7n^V-| zYud=Me?Hj_7wMGO(6jWTVR%-9&t;JnT&E*wkgfS-bSj0$w2iTO5HvLA&`@zL<9W1< zEE0}_iw#I2^D%RwRg@>^0P#~3r%XBLQ1J-|>pVF>X@{wLsJM7?7dS~@ZVN3SQ;(Zl6f(oTykeZ`Vrt8@AbXKyz)RsjwP19Q0yQn9MsmeO zCvw~VmSAL7gjebdkw2g>RE*^7nbPf{dz`bR-sse+4vBp!=+Hjk58^mt+q_L-!`VB? zt$hG)UKlSwQBCq09?tnS2{F_n9&p7{9@@Ki9^P%ke=J#CW;&$bZ7H0Ze~lN0SAs9s zD<|qH(0`R@Sshr#|+W^iIoqz7ZBING3e5s z?P=@n(*~bEbNHD04?Wuui>+$@Wv)%tzZ|;%%J$legdNu}UTKIbE6(h4?IsN`UK}!< zj}`Xs4)cF==QSoKV#u-jDM^u}D-iD^rj$x$FEKOKc1qbHCIU9j%*RxOv!hqn%X+r@ znB)AFMaH?Y&iqb%zq$dfjaI5x{5V|^fOy_z@5shAC2xZa;O~2^^T%%yYby}@?6k2@v5TZPRqNi0y#+*Q<5U(=OXkWVz@&jw>MRfm5Zqkh0V&*6`EPw z*d@fs{OnRSlO1t8z$RKqHXF-fTl+3zg=OFY>5OrXVbXj+N(=gC<|mWe3@StY+t#5X zY+Yqp!cb<_)p3RxY{>+G`I4Up(sYtmHBb!EYhE9TMj{`*=m@?S`8_6TTev<92TTxO zd;VkKcUT2;&f=^EL5=@>)Y4*j8;o?;3`EK0|Xl6e8vLE(O?HfwdF5?(V z&CPrEwAM>h7h7JTTT@JA#3Q!T(fMuHL2d0*)^Qs-I!kYeJ)hxVZL+&Z*Fbu-5qUCm zi0?%LNuew3V<1UHX)S@U&?X|xWc917_jk9E;`Krjn)wPOL@Gbu>vc?Hr}%_~Ut%bx z^*Q-L9os7o(MhRB5JX{g(Qa%j{Z2&oe5Q5bMW$ct zhN&4%GZc3ZY)P89TYck~kUB>@gyzos7TYy1%$#}B<ojJP-~EDc54?G7nC?d;Lhh5GbnazPb<9oCO4&G{n6*1EUdN@gHN%3 zt>mPnHB6k?^*;p>Ss`N`)#cWB3Jb?O5yp1y#S%Xo7_t1X(;OMphDj-)35j7>a%;%^ zhFa54EMnF66e<&M2*(j%19`Q@qZTuCHmWh>D7mnBf&LBZC&am zcnb243E`=&QB))2d_kP{Cha~IsHv_4gyb!X-iymjlDvrZT%COIHSvmQQ$>f!8pzzp zzQs_{Gax*bQc9FlelT-!%l0tC3CpwK(FvcS5ydLEz!hIYMm4P5!`7lVWS8pkrUd8& zr7m|HKY7~nzWZqtfYSn?MEHi4wwxG+qhivrzt^wmU{qiG+Js8b_oz|c59e1#=&jyWqldLq za36$2IYr`PdYMWY-XKd&??{?~?Dz-U#HA1)uf!0J)LOi|A9f|BqW`T0@?D z%26tpuLEyZZZ=GF(61U4<;7h(Ph@HQLjlBq1b*m@<$wcdbuIrGXGD<=f?u!K-{Tl= z(ZdNAxv7u4yV#r6Ep*cpE$cx#=hpN6zt2{mBmdwm@~et*t&iP6JWrzH{sqr`N;^1z z9Y5ge^sU3f>kn~W*nDay-wteFOMGX%3&|WiSmg|x-IT15Cdaq5kIrF^5H&Bd_3sJn zGHXGJFcpjsP_NMw){!L$pofU9xL>UjH$4b!)7K(N>gFINkAM?TK7RL3&cWMb!@w#r zpG*Y+#L)#P4*voes~=kSBJ?Lyv3~-IqtzD8vDa@oJg2Oz%)8mq80U|>9Co9}7yZqh zlA)6)Z=G=eIccoQF|$8_*pYYGEdcUQU|_NS%%L6^}*X|elGLK2o z7C}iJbz_Z?ZGp1naIB)MI6dPucPAeUgdrA2O zom_9)A|;j76k2ekDdOF`T}6Z9lNfTi=HD8PY&~DbGO`$&U^uFn{;|gWdn@>Ecv6dR z5b0w!efs62vDTY*kmZFQ-fYsO)k0-cFJ#}Qp{r+)aTEX&eFH<`M>>S8o11#`(W(03 z+Y;p^A~J?Np~PrbCvucK7#}LqD9~8p=R_K%xF}VCk~{n7iUpak8DpsUSU+@son@H) zSn(yl4{@klXqPTsBu;zm%cp%Lc(mS$sBcTRt39eQBX;P?|1MyIBd&G&>no}Jds?d( z_4)^FZO`T8L)uiMksmJ&wn@xBG4|bqo2ioygjj5AzVX_PB{$ak-Whjuf_KBoC)Q}*w(fXVk;nlbN9@Xg?@gY zz1r=LIz2D)NttotaVr*6DOrU-l^!5L#gdR{`~@ajE$MKU&8^Z4c4Frf2!Nvn{gp^^ ztvTmZv%SWv-rfIj9H*~#gAB=mc<`X3B&?s9cD`G-Q(Q8EE>QOf0yW}M25gXFJy$Oj zeYH#0s=PwuDaRio@6gp)}?rYK&jLO4j!6csW$Lx*-eHKGg>BX8zo>Puxw;d>Kg9-LJ8bC*H<5qIv~u`79g z+j{wc);u^dwrC_)W`VpgElHZ;Ux9|X?)Nh23BXh5^KZ3b`&VD$y{vj08(ZmXX=-lm ztUAp(H#R*tw%OZn024qirV-+S-cnapR8mW!=I{^z$!n7vPEUFO6Bfz!lN1}acL8Dn zSqm?e;zCRPtTehjC6lhwxEv;gF;E3}RlDcSomQ@F2|>2*`isW$UbH&nl;07@KfRjT zKeS4&p|^=3{G#qzg$w+w>`siJK#=hRa(q(?z20g2`k-?D0tXV`#vuvc@?gxkr1h#m_CjR5kT{^+D?qDxo> zO{B-ZvJb+0%Ql|1qDm_nOwyeBHkuE7j2*JO4%;&gSAF zP5kW*ntcj>ywvN##--~=zI*Vdb^P&H@?8Bs`o*Y?bz<)4_6+uZVR`YvyvC}&=U!ME z&d{nhrld{KYUCtlaP~|;?zZ@PhBE~tOg%WNj|_yYMJ~@CK2#1)Y_cgg8my3QDu!6ls~+i{amnku|^P86BCJ!yxZSe`e|mI8eFd# zuM|B<-QNV=AU0m-hkH$LS{QllYW59|`sK%@lEG{cu0~RRA1Zpxr(j_t)N-J)lM=~* zEs&bN-m-YD8(^L6W`&*?c!WtY_j=Md$Qc z=yGTh)#x(PxZ9$O6Q2vR_chzP@oq0;LxFtwb%*hXuKIs;eF;3)>H0n@WJ^-n_tv3A z_FW>SO%ahLLP->|Bzq);R0s)0h3sS}g(8xWod^*sMTq{_qh_Xa&hI}TXHI8M@y`2x zpXa{sYrCf|b_=0{S~OTmU46`9*81m|_?p?nMa|{y(u*DhdoxR)G#s38mU7l~ZjBMzfn4XkK;QJyK{c_O0 z4(3imH{VBBR5`ZL?^dR@p1y5TtAcD>YixwL$JJ3&H|Bx$U01?=>M2+>_-UW~W3VLLH-K7n5`=daaFqhpRZzeVM^xPOkE5oh{3OHzW&>nO z8A2UNN*ttP4Mp-$kdtUxa=^teI4w4b&Xdj6`Y-fWh<^2T{hw3UAEomz7=vFWwCJyWo$&0E3?Z37zF)p&dm9oYGB{y(l~2rxUZfKCBG{n&^T&_Es8gY_v18K( zkSjnd24yr!u2&g$kS%LiO>8Q2*^Qq^jo@-L%n4!P<_;!ypr12wKbK?qV6ZVJ9UNGE zR&JvOMhgX$v{q+j5*s&iV0##Kj!Q3Mv(YwfxRQ3PNOB`%F$xFCl0TS=CK|3yl(tlC zjzUZ!>oX)rWqSN{*POn9JSfnNjX1DZG&D5&k=`wi1qwnhJ~pF&heV0^hqHyDu!j3j zm4x?3l_=pDEv9`T!ABn(p0I)?1cH{(PGHH%!A5WfAiwDs>&M_R12eA-MlUpbl|9ZR zo;nQ8ke!H6$YR{W9S`NJAX1Q!nXwFk)W_UOGZ+z(oguxlQ4DlD-k{&U6)5PFn+hOc zynOk0YzNpNWFo1~O8<3m;{$X!(Bof4B_VQm3A_-P>K5}F&7)a(-s|2C>2<$rJ zzqkiN1&5>|D@P)%%G08~$kKoAqhT=TbLGU(Yp${X0zEJMex<^mFQ>+1) zWeTOIWdWAY!tw|4R@uP#iW68emQa2{=VHu(a#znit>^w*T0daxH9D*E_X}D4Z`K5~ zE;$ExTP0q98@lEako>MyOmIh=+;(^(vMZK?B;jets*X6$vF^k z+1u3aHkB}DU$UE4!UNF>YG|DSA^F4}FUROC2!s(Lpl&^I6}Y9Bh+$po!~6F~ROwQ; zJndAETja&+Y?N{2`m(R8CsyDxJp28V>A@f9@iC17f=* zX1)VyaE;-j8H}fa_x%t6PIL$)$pgkOC=YkD1_lMG;R7rma&P3kOO>`Q$EIGAnxe=UXN;5a+w#0l+o0t?E})PCRZ4994Q(MwH6f zeKaLTST-1fTYcS;)LWNt*J6M(GU5S~Qk;qY75I7m1$po8y?EV(ih!GxqCqJT)MMqv zPcHah5O^W*_h0!3mIv9|lb)XhXCsauu%mmlS>}ZqIROeJEGjcEU2`VM^VXabf==k~ zUCYnf`n0oO=U09QI#uY(6po+=XkewYV|X@DjnYFg1BI&)Z;JUb-i%cdql1o>ber;`gp z_>fWRLvolc2YD9@9Ty7S@^xr0zl#k04;oVSV*7ubIJJpQ`|iO2BlO~sK%X7G=w8`Z z;(f}1I}*=hX`cwEW*?n-)yA+@$=-^(%WfQZk!00ry>Q(H`g z2rHZ6yzN5TMFP9L1c#mq{+FOUU=_{hX*;((fOYD5zhu`2n-=(!5F{6 z{~Xt!uMfC)U;e%5j_0=26cb{|C%*lP{xcf z9FC%{Gekf_f--`vWFjwcnT^kd8W9Hsnt=qp^XKz0xp|UdC(d#)Q_*+*uc_+_+QQF6 zT;}NcSB{M?QP$L4j&QzL=ioU7JHkV@Lm*(L^%3?QXObAcMTfJQ{&+T??JM>3RLYpI z$e1j3d+y%7=a{tNenoG*>KUx@5FPuYYHNN;=*4@lfjwg+U=kBP2@gAxC6A6 z@YfC&*QTVTki{$;S+8KJ-(%c=kS}3MKhr($3j&iYMMf3Xf8AtWqUH3vOLYbL%JejoU31^a;O^WHJu{|&q_tZ`X4zl79z2+}l zL?@q|`-@gtP3Xpvi``iBfT@VS$89oLN)kgm>bMEi`4zjjT46E4{m;GVozW8^rD$Zj z0Q0rg=}bN7>-ZvS=VkX}V-*Q}<3k=5Jz#xFBF_17V-zC?k8X<2n~c-YaV{RY_r?p$ zcE3;Ax(u6OS%RBSIhNB+Vv|jyH1ei!3sXkXQ=pf>=zf);`Oy=X@YKfnMdRoJJznv> zb88Wj=+IoMWulooj+|GWdm^nt|KO@9=?9I64ZPgYd4DmeS5p(>#)AOY4f1jDrI*&Stjte01LoneMQ(Z48m4OvO z`JqP8MrZj2IjPNvt-?c2`bR-UMR!dpUD~@K8{nQpJ3ulR8HkhLVrTprVr$+9>`^G> z-YIx&wnTjY(R5rssqk>I+rDKKs1{_w3*g5QbP0+>@gUTHxQ|~ovH=4HdGVTO_#Lq4 zKDtw$L z4x@I&t?L}l0|5tc&>%k~Ix3ua{u()lL zECa#DBYsR8-qCs#zfP9$P=i6s5(b|C4h^8YdcV5~kOrXq!Na2$-IN0%07p zKj#JqlEt?$G!#}CT<_6h;#;Tfm9udjPFX1XDnIFN&ySVwSBlDyx=&s~d{1|(gt|ho8@$vcY33nur8FG*YAwkQ}1NxQlh?YoEcM!v4lD8kr45$M80gDo9)MeJS=8 z?HAOd_)Y>dBO*4asHa;ZMHBqgaW^+ZR%DAJ-=2bwo_^hnkMqQE3k~pBmpG~hPL2sc zBl;5l<1j*_S%C|3=ok--NiZ%ZW@H@5D&s|vCc=S8DjV!w>4sY&{zXI?NoqDIIRisH z&Kv?rq+p4bK#=rQZU;9goET(LL38m-qwkBKrp)Y%L~i8A0UetrkIS9Y0an{U1@F+% z@OQ$-cM=SPpp%`5F%1OVB%@2#Y$49+!?iDhD$JrXGRwhHFVfT_^XW|=zSeR9=0#)y zDjC^BhCrHJ3l#d&TdDo&ftUQ31OVzRyNBozz`X?dt45jX{j%MAmi)w}{+r zH7+)m6WKX<`*HPauo{%OO0d!yS-#s!f~({&9ZjoxsFQM6(~6xKqw%iE3_V8I=jnbP zXPIn?y?5{Zjo=I;V57h(lmV&XU-0fw#u^_)p%AK+$$tlHPqB+Q#Hw(_P(8;@1{68L zT7yD)OGJNfg%1wawTRvt1M9pVca?rSfRBn4Qz)2#mB{1W z0;lc=xh9c$y~o%@wqrqgq57_ve^JZWFgnsNER{+#%=_PoZ$jMn-Yox5sffsiv-W2W zoOfBOXL*bbkso%0GnIvPw%RwB9MISt%5l$a;qqZs?gs;m*?5X zDQ7stC4v}EkR{MfWrE2TiMrn-JZ2(7H_m?ZUR)o+EJ23_WXd)+X+1;aH6oM=X8y%! zoiZsQrWGru15n%UAKqM6FP^V&qNeuLx)sm;zkuZWR?Oz)^MnF!=wkP)SD{v^kY$PV zNf9EMgbS>xQFp~FNEm!UkpVlJ=ga9QU@D1sgRIgNgt!M-36`G+BWVg_h%F$~K74>g zUvrel9+mP3mR2Z!CO!zXLlygn84(p$IyWvCSN06}6fv3Z$6oO1ZY&5_CzL*(L_B`NKxcXjtS*Eap6fHt%FD40F!X^0@kJMM!d2i#g zN00V>6$cYSpf*zcHW>>D&=FG*Dn!EKZi(giQ(B_Cg0XOb<_o zK`#8~CU&T%CK`4#FC~$P7)!f{fcus4u3$KUqnBW21>%OqVn=O7=QP{2(%PzjIJcXn z;7uB5geW$Oh}_s1{mKC)3d3RGacHP&2ft4qCj!jHI}-0MrJRzKq9OrJh$#(naM!Uy zs0tQ+W2yQ+IP;zZ*>CV*6Gm0B;{`rIGrXn9&+&qP=M%{nj!D+WQq88gg{3|fdr9iE_2Xf#oK7i z--yn^7uMDIzfri1;n@v7(0{u%4>>*v^CCse9Q*_J)l zY&&Id_H?|e#^a~cn{hvbP*HuQLp7dH?Ur48vU=AQ=LKU&UvDYiGMbc^B93%0dt7NA zbU#6P6LKVyzl7q_tp}C~n^i+lj`>coaiKWl`Ix?nY*WxZS7C)OYq*D$0>*XtC-V!F z*!cb_04tNr>bLb+!+~A)Sq68?;v$zgV3j1o`a4KQE3?M@0xiBgy-zc+(WJGSs$V%z+A-+M&BZX7QW_>wr34vM}Mn zjR4L#_u1v%d<##sfjPo=qnwaxGzLQt>2&(e2TsS~G5n036#`ftlu*~*g=0-v8$cRy zCjjb^;z%k)jqrB}y~xZ5Iv}p+Gd&(OLUEIRB&LfOWEP2W#H3byE*9AhZ(3Ugw{D$? z4NL`=m~j6-Ei%?VRl!`939c=4^RC@$taP}6LGS-gitFeo>YWgPw-pWSF9oux51qgl z@V|kCZzwB+yaox3%YcrCVlBl%ATt;~U8|9Qh$jrGM%P4Bj$}g`x#;&ysQ&sj2ODc) z-Ts@B`@B%L`=0CABmgupTS%c94Mv^Av^6V|e;|TrkGJ0au)47*a*l_7VB5O`{lFhc z76d4gfGqu4a@F+%VnA#U2s;&sw{yy{@sdeqT7LDideGqwGAX!20Q@zjPyYD%vCB!% zV5m*$Q@YU9?c*ylmr7CxZL@IhU1__4N0Y1iY*ln)&dK(ileh9*=B!`GIuv(lhUaK< zb!7HCcYKy~_N?%1W6ONX!@{|3d&9&E?_R#lAZ5>q2FJ5)4feO~>qR~c52p^fv-znY z(h5Ws;`Ed3d9Ch3+$XG`wG{VP@3{i%;1gTS*r-vGo7)-?du;|ytizZEk2c}~p?q?r zek;ChTLn6>m8`7hcWY`+V@c@e8o>|~XyqU{(ExNZ8WpO7$DPE(ka~#c8eC(=o9on9 z1$>!RY55ob_Vg{Fd01NbSprC`e`#sy@|TcD@iEi#vC{Fg(wSOY%e@R^!EYB&0ZoZc z;mw@ZL+Int7cE1Wrl6YS1r#STi~w|SjjPSb&#y*XJ@o#f#fy*~_+!?oKcvlR-!HlA z=K1fB{rxMkjwm)cioajWzyJ5?S=miXJpD{x-FwC0puLckn5FfAefH4FJ5f8iZx(Qs zSNYO&HpQ1!T&>iD zoQ7DGl$1cXz^-C!a0F_E6SX(l-Hx3(a|VN6>&@usXvR_|h~2O*1}lVYx<0j9Y!r+* zX==JHe&)NIBL0mp0bKFc@p$)ixwN!&d1d7egVvT7@P*G#NpUZ+6JuBsw(IFbdPYWx z@2=Fu4EONi6`i$f*Gi4~Ouru-OeihAt&@*DarV6w(7C*rWuov@<|_vZVj=7o22%~T z#2XKvw#I$09=wOw&lK6Ley|=5YfXOn@&wSH32+eKn(;XGL>2JOzsxNTNn4NR&?12U z_9sOP^2or^NJ7ob#iF_;vCfz<4sI7OT1pr4qw^-&v6`CC@7ov}8Ulp%T}Ho#&Af-C znWIjg48M+n9ESFH@%}uwzGN-u_tn~qusi_G{A&X7r7_RQ$%qP!%{)t~D5{Y3I z`7&8X8-K!l5RzGPnL-cYnq2+t86ikk{7{(00-6|DN)S=#n!ME@j{z`F)DfBGeu+P% zeVN6wt7=1bsB6qn;6wfA@c;T%-TD<%;ZFbkPnpI1?PR#5BPV-J>UVhF{=iUqXaCC# z*-g_Aw;#CAUL(Q4dW!mm?ShUKTTR)_ku*v+Q4>!uCF))~3MT5G(!FOX#kbH5$}`nH z=8F6*EjmAFJZZa-%h5RV_F8p(g}(*cIx0Tj(5Pm~eeeFlc_WD~V7Tw)J;M7y@+rVg zC(9MHSH3^Dl7+<^P?k;XE}MEOZ z+RV)7)2B}-4R}Q}7!d`3LucsY$DqfLAD=unVFy@~^r0#BRaMj=GL}Zv#%44SmWZwlI{;sW-UVM2?;?{7f)JS?*L_mztnB|dwV_MVgNMo;N|dPB9xSrq&8T=?;H9V8=X{z*I55V)mty2gkS-Gf12ONwZC6{=&C6#e-&<_ z-PmLS#We*R+tOZOP-G8pm|(OL2|&=%xhE145s_F_WN_i7g)AbQu*XwHAWRuwn{TKJ zgx!J5hQTbdt_tR$-pLs|N5^Gx4Xq3Mus!@Z>RDE%lp+jsAg)G)bNM{7EInsuXL3Ws zs8aB0=vNzK9Ak0b8ZL=19K3N2{$M7;5~4aV#OwR~{OnwOvdsSz-@ylmiM=4DzP6Sg zDFNzNhAR#Mx&nWwdkiVbsylWh=NS5#V)d#I4jF7g)(vab`$tCu*X_A!QsKR9#Sc$Y zvy}mARp}H#d36 z+X$Cb(s~Vbm-t=K5@D-7Ag|k3EB-h>QR{IgkjsK*3dc5a>e#F5?rt8{cxI$9e){$e zij(N7UF7%oU)z0;MBjBU^kDcWy)Ya!LCkF*!I+g4zp8Bl zAgTZ#9|hj)_CicNW}n|ZvpaY0R{bZ7!&sjIBSG$a+#eXyD_+0Opjn5Kn0n61NeW>a zjEcLGFI`#&)G_xx!WVtE825}-(mfWk$HWjWrhTt`IH0F9%)?J~%Yk!3{alf9Eu z*@Sbo084=03A~H34KEJhJ3#4I^U$Es;X}>}H)pJR!_}KHB6<3)sF|i0eiyMp#(F&k z)_yXlf*$qbGuqt5{QL+VUJi~c5Q4sZIG~0=1c=R=+H}3V#w-!`k6uar^2Y_}=lek$ zyH4-#?(PPNx`7L70_L_l5`>2t3E)w(lWauoPK?(ea+cB42eMy1gNW!2L}`yA5j*jT zkCT&=gv+D3Z^|>)hgS~AE@vCM8D%(y9j92>LzSlsjc z`Ene305t7|`0M~?H=tY+V*Uxw$!H(uLrt z?Yj4RTgR=aDBMC+D3VR*x7sP5gaRL^s^bVgIT@R2n}E*w3EfWTF zmyoJ~L5J^c9$j3qot>S-?i{%JTtZCQFzb@UR!~8*O`ya{OI(hNkhbw956UYE z(XF49DpxkH!RA(znJ@2~x-DHL@BAu}R%LRN8+rNur$B~&5d9gqMeu(rq$&>uho6ev zVkKj5-g0C|rD*TfFuv=ReI|G?NkRqG?-Uf^dI>*={>@_Zo0DtV*l2k|lDU5%B>oPN zcc{;(kA-Asi-QB!d;S5!5@u#*lULu+r)Io3SI~KvZYNa?V5O!IIiIFij?`Z)(iAMJb&v?p$r0~O=pJC?Lb3|oCKXJR| z8`E(%YSPGSN$b>rC&5Yzh*MiJ1cY!i>` zFh6_%4G+{BAZJ;-#SmJTtM%KN{C&u~3GGw%hGC0`s()a>7p1f%FC2MxcnV@EM{$96D#6T) zM54>jj^hhtVdhoi4NvOpnQ-9Hb>S-`H#auPYB~RkEKoE8Qc@9)B13?~oDn$}%+>b8 z1bal$>GEMuoIwOk?Y_RgnsGj8*CF-`P=T-Zf?SOAPmx^ z*+5Uv1Um;mL6D(sZeEHDQZK`V63InmJ1=S({u<6{vf7T=GZNU~?c);`8cLz0R3_u7 z6!S+>F*~#vyzGe+2fz%aV8eD^BqyS}(1ko0$8xq)wMA15zHCr>dA9HE^6ru znv81665tIF(^dtjk#1|)71$XD4QrEhU?EGFE@g92K%<@drN(Q4!E``Ok7Ox!1gTIq4p0C@?_-)6sVXKgM-qI#oGtLt&G_=h7G9xP_}@F${2w^@blAE`^%r2 zPs!iHeRTzD#e|ZQSRJVEGay04htIhwYpw|~wUB4R?WSXStE5B$q{(t)V`B{A_J9)z ztO>dp{^k;!n3x!{!v;@!ot?uHqQk>13Cq$!8b=9n@g1tFQ~-Z097J2mfkA>}(f&gz z%_M<2?@X7h**1JSZg?nh8}Thpepr^SFn*_7Q2*#rRX%2`pZNAX!%sWV!JArI`lIRU z#q-- z)o{Ie`aUV5k&?T3mrHRde%G2A2kJHX+1}<=t(^T?Am&I_r*iWwndzX z^~DT0iuiFT8}X&3?sa*nw9qIaMaK)l^B3#GvAM_&?(Wzg$nJMRyvng@v)D151=5;~ zc1GuCWj#T5Z!faD0K&LE?X$~i4Z}c4-fe(^EiyfK@Tr5dtUx0RAq+F!rNxHj*1u18 zXj?ka?BBEB{rjG~@7;RkhQ9ju>}O~9>Ebs=QhWvrp-735KC3zu#X^BzfUp|h-AY#q zM04ZAho^9h;bTvbaj_rJln}resz=`Yb1v6;{h3uQy45Ce=Keln|NP4Qd~}Jv;2&M> z-_=jd)aG1n<_-*LG}_UI3h_&YfzzqtrvUTIaU^%YvNDBp@l06x&W)_tetg&fCLtKh z#lH7ep?$0&^izB&o6hN(pAS z@i{dN#Z0`4b+b%Vo{%d;IGZxrwJizqFE?N+vK>L_U~fCW3y_g0-&!e({9>YjLUDFT zg1pfTckZNOR4(cyBCKFbigzl+#ep{Q5&?+b=j$Ll9HBH21VAwC+CjXoSQ~&(Q=Ofm zyP|?}&rM+!3yY+@Er1d5Ptp0se0Ij9z6Es_hz)b#vv#G6nhDh~vCsk47*oyyE4jT! zivQ_Gf)b$jIz{uwl#LoFG+JA^AXKd0(NTKnQvqFAkD|lw!!r8pWK0@M=b z%StQQ$0IP;;^-J2J616?nfp@;;Cl_psRF_e{g|0teE;DzJgGXFpO+^9h5(2%Mws7gfaHJgjjS-a&skaVUbs_hFr9Pz*SQ^-@Gw@^?u2cxK_J<#y3*>Xs0-W zE$STip>|d$hA3V^J9hky?_=Cl@VHEuTx~?&S;h-=cTHk%H&lwkf@adZjD`k7$+O;3 z4s6LJ4l`YhAtHF%w%LqwJwgU1nW1L1&?YWNH;w*JI#}H&y>xcBWriW13X<`v%c!jn z{tOz1VTN)GTiXz*Z=VniQrKQ2$c*>hmS3K!mnx1qnt`nU1Tk66^C}A)Tp`(p+3ii6 z6BGKSCjfGS#q6@t-|OnGAzmFcnk6W;3}|nDfPyzNGD6o!OH1pSpD)F@ONBOSHRLA*S_4{Va|}=K z&8^0Y$@S3|f1x?u8Mub*31}}QQ%+8f!|SqjW|=uTO94Klr2K^|bgBXJ50g<*Nno`A z05j_g5Z=I3L&-kwk0OgxllvZrsejpy!Q8Npn{fM`rHb+aj-!79#~CLmU+3^)gb8Pk zKqL~(`tnD3SXjLMygi_J^(%uFPZpayf85Nv^APY=6-y=RrT}k;mF?a%Po*7V9e@~+ zlB3KQ**eXYWPr3z#3SfA0w^H?2ul=(+uhzE?g_Lx_5fVcv9-1iw7l(NadiCUs)P^} zP_o8{1YDp9BO2psy)Ct<=-V&Y_d(zbTw;h$ve@15;E#t@tW}i32f98yJIX>V9}IiA z>G5^|Ys5??FE2lAM2CTj#t6${aU&(=d*tKB&a>?~^QrW`{c=2Hb#+fMRwLEJA&K;x z=yI@i$jv0=uSMvBt@{5{ZKe_!B>y=n|D35B`B&?cmkQ&sF@$S+u3lWTBn%eHdK470 z7ech&wzu!oa29_7^bJ=C#>(av)C4y9fS95pxfC4hc-<6icW20BvbVR_)!!1qdAf}P zA;3!GvK#&aM7%=bD>Fd^on+%0Xn@_lmQ@F(+{7*|v3>iSJELr1lKh~9(>-Qr*qM`R z4e9W$`a6Ztc=<;~wdFio%eG6|8EMQdy^Rz|F{fL)w9c)4gP*A#VjnN73FwxPa)vdj za&krNAt$#Zo`xv8_LTs5yY2QxlL$5VAI@PVgEwiExs?g31=!9Pbr01W>oKXZlmhtT zt@`qOpW>1dqPIo2^b(z$PtoAOKrfzNBxkgAz!=6SCdPtFjIO=1j#`?|z`y{P2_GEd z20-lmkX0}U}wtyEE3vqVG z*OifoFtZQ^i>LPod2k!x{lMkC>z+3GyAVsPHF=!?dZu%kP-q?(7J4@fWiJ3t9h&c5 zif7o}-JJwRP&Kl&D~jk#pWXsRSi1Iz|$4O(GQmJP%9#F_P37;9AC|gdYV~pL@Fd`JD<&onLbi zkf+~J{TVvw4$9KM2> zk=T=%vkn<8?81EY%F3@48MG@9$3ktqf&!Z>vR|O-!=2oewZ+6lvFS!;FDGXyt64S`Ds23Dzyh z&TiTj7CNJqrVAiGT4xC8AsDSljIFU3s23I%s+L;IwMRSug47Go zyGsVL;lV`Q*0DTs;?mw|$;=T<3D`madM;<$vYLgB4a*4FFd%x?zUPo1r#XnNsEF8k zbf^rAQGB|q*H^W|y+cY)Vxo`E#PINC=v;4I^}5^$@&Z!oHGuz!coIw`5GcoX#9Bg2 z!Vm8|AT5|mB5w}u{^J>rItiZ$&dMdg2?$J@ikkThs90d*Mp_`|=zcP?vTh9$xIDDr z9*h_Wd<0P$P`@3J0!jRA%Q~^*WA-CpX`q>bE%|j@Ucm-Q5{ia(;OzW`ZTf%DUKimQBgFMUjXJ|sYOz;A=(cS zgFNshdo4pP(7gMZGTiF49WaRlc14U~Z@y1 zR=?}qQ$SxW)3j`Cr_YgxlwIfxE~UjTFDf$91k58?tH{sJUk^eJ34Y1?&KDJ;dBc?z z!)}s|&`1a~iC?OIJLUj*7R-!R9O!Io3jnhW+xdYB2fW1)2Vkeb@uO2xW*9_BKss2b zyV@V_kEyDvW?)zC=v z3oluIry2w35-;UD^NC*m%lYl~!L1-Qsj|=rsYOAECuG~UxgR4EAVHP@-)|dk?A{=6 z)#yy}31ee1(ExR!Q;b`IuMHcLW$c;c!qsonJ3*R=dmNon)p9E)c9$5jABCIw{a!aW z-#ety(-80gJD_gF#>ALlHpFSkAVatdcStqTCLga9yrqQE9Rlb2hXxrY6{)2QGRA0w z;LnIpPY=;4J5jp^$z4k*Fyg{G9f%7816W9bJb@{VirFydd4-2J{%5&X?c$jo{JKT zi3a_dHD6cbwY;wMFwJg14_!Y|w{qKyqtTG^`IFEa;=oKPMal}|PRk)lWys>el!jam z-D6f(_f>8xysq)uigz~I(WDWb45+^B5sVIEg3DPDwo8`8sU(_C7@Yt3*7Ul{Nt`G= zTy+l-L*o1~v$}U^NUHzya1$U(sOb~2u+FmmmV|YN+srx0wg8*SFjS&<;zogNocV#h z@5b;y(c$A3RFj&AN&#bm-DrylD4-G>aE_VKEke&wZ*>|sJk&6d>7qbz<;FtAfp5#O z5G{2Ko5hh6^?WvX585f{bAs7cLq@T}&y>g=)&!D#EXdt8dHcbORQB}nq94N-N~`@l zHXmly^{ByA&=aOkmVxX9*8MW7zvBV44wnvZzN98#Iv-r@Zng`EMI|0Qs{@Ie0D`ck zLk+8s@*ctB0}N6<=+qgga4c`g$;t+;qK#@*KXU|l()P#~OLdMM$$pXjzHrnq9NGMv zkuh1{Vf{c{1bSh*OWryNRx^(=;Sq%x5LAn4vxwniW@K$+5tIssw=02xXtS#gv9D|e z(Yl*dP{}UDP@@sT7Aw=PB5F?pkqHU00V3nRtG#JzqGv^vR@Fl%Jd)5QGJ;+c+^Jsk z76%+i!b;R1%pWKjeGs&{(pd3umAAFFu0=ZvR7%PAligmms86hNiQp~NH65|J#&Cg!C8sSt+? zl~*|kp)hsoQR|InF(Z>@-^+?tR1&5;4O9LdH2!lX;5Pa)#tZz ze=)57axtau)YC5U)%QiNCmd?0-ZXwocltk9vI`T){F2-=-e5B{7ePHui< z#2W;^XPUSt$yQePpJm44=azE#A(WZdvi!V=taiBIT<|PAioKc{WE+%m@V)> zp|~=d2rVri&IJ1NCiamf!B9X!CImJ$)s>f*7kdnDxYG}X zp{qoggz=DKF)%1H%c6dg(>j1d)|Xk$vjbBTZbOts#OI|oJwm=dop zPl%=Odo3 zrrhbmI2Q|k#reZhaT+i<6K6M6{IMq?2y-3?(?Fd~R%mJ4Exd+M{fM%};uO6*g|5Bs zN6qiD$T@Asu;LF?q7o#RTtjl)ns)_5ezRM}>q5S6DSG&D6;*{sI~J$+@7nz(HtEPY zkHVtWj~!E(nwyVV*+x`WRB#y?N$Qn$(j(c+tX{5Z=4^M@Iacmq1q#pHtvio*)dpIg z?uxy}^JZ;+LeagoCEVvCBPUHpPc17gQ|R>Ost}JdRJzx_Pv+K(jz;aSOQl<6D9WEy zG3@Kk`=mciMQ@P(!I`}&;XyIC_Sp@EM_uEG4?d}DXJtCmVn=_@H!!{HqDXd`f!FMX zG^+<6RLis|{Y*_xT2ou{$~j+>seC=Ya61;ye_C#r)gvnnOZ%+@V#~Bdv`kLgGKDZ} zJf@NHu8r=rl;ND5VXhpxHkBZ|-+TO~;zrjS8E3P6ef%YUtSzc|TIbwg(z)L~=JVxe z+ZvHPlCLO5^;CjnnpZoaKAR9fz zMr;$weUB7j#KUoH_ZqG7`U+Z$;TEmsKyNY59<}~BAHwxF1IA)=n1QqWM&oJdA`^BL63LmRw{40>`WR8Lbo3k(A+_yCxH`4c`}-1Z9L=c0dhaEYeK_X+7A1JKYqaqlEm- z_6e#&iWpc6OkM4Tu;lr%t zCVUS#35Z^0M@NUrHU(Nh1A47@)#c)ZPU8j0Izsp&+p4!b>I$$OUHzW(8V;I5EG>@g z6hu%1PC^e_-jvgTJr&bWeBi)GG z-UwGJz%di(@sgHal zAy5%B30#PvsszxUGYkSn1kHXZ-f-ltK@-+A0i6Hu<{|1Jo+c7F2$L}GeaBg|;3`Cv z03!kGN4tDEaHwzFHFnqdn2F7-pyh8uBG)RMv-=`tQ|&C$(pn|yppF62JwwcL{Nc@x zNi<8Bnjq+fh#S5X4kkc-g?xj6Z-$@j$c5N<$ugMm&( zH`mh!eJh`edsbQByxW!&^1AS?JNueXT~C&}Q;=}+&HJ;DxqOA3O`nxsqBR^by>sNa z-V68DXWWc@W3t~{rH%3A#s+6ETO%}cyxL&-s-tIY;#oKSe5vs~O}0f+j4PsA#jbem zwFS$SKN71v{El`fJ}u$Qee6_1y*g^|@JU{UR^NNE>^J4Pf>zJ1xjL(_NV}bbnpb`r zD-hngaJajjU(2PCyb#^EZtU65FM+!s0BDA)C*iZzULNk(5$V@6JNr#Rd00=nP?$v3O>6Bdl_?Mj*n-XZ80nUAwAun+j_0BN9j}QhY9z5s`cx|qgQ!`w!y4MKia~*Q@?FsR-)m2 zK>I6-a&JE2%90;CGci&&X*Z!R0zFnE)nLmGAIlEu zeF_0@PKUZAFgRVkku@Sb_R{pEb&^O&q5R{3qnAHFzs{i-d`;**o(i`q{go6Z&6{oI z=T1vgctVpv)Jb^cb8Rp|k%t-izWY61HDRe7{ojf*lx?B)99tTF3BcZoEQcs9iwxiM zYu0?6Pu70mdo+zYq*}b@iw4uj?dL8YYJT%(`^a|8ZJ4t9A!09{m<}{$YcY96fto#G zt#1Zwf`H7VU(qr+-b;K(ckgD5fa`;2QbA;tlW=@Uusz-kg#k$;=sjmCx%?1}xurk8 z?6KPZ(X0-8AF-vO-by$Tx=s=zqPHZG$`cv7V0-Bm<81HNtltM!HhNBSSmhLe=At3p z_SW4}^26BJGjxZ;I#eYtki(or|cl~ZO*YWh@s=$LA#U+ zU*G5EIDb5bnr86Uy*&|R`CY2DEG87^!6-|M_nGOSSkX|SqZ>^JG&E6AH7|-tAflm# z?g2=6WmT1h_Cx?CE!;y8(k7+VVrd2>bSm%M4&a+-zd-*?!n93j3L-bc1q9rJEMYfj zZEIUDmH#<>bd}h_MeQF%aA2#^*w$L*OU4HyU7G4*;qHvoTh0Tt=Nklr1+n1lcj@mq~yKA^U(M>gAng z2Z0yFyzSpJ;ld2gT2+VzV@ftM}xw7s?83U=@^d%Twe9J6#@ax#-7B40lWJS&^t@!POIXA#+_d6+TwaQG?dxd z)wSN_u=r5V&-%)gbfZh!GP9qfv>#NoW#4%0R>0vgQ+ulT$GPkKqMr{|H{O?!o~k`; zb=c2E(M{HI`wP3TYx568Nw_NgRJo$Kdi+@Wvs-HgO@|(q@s36Oys)ZZ=&ubqXVfc| z9a5MQ3>+_av3vXX1TATBn3C5x64iV;y<2nVh56y0IF+mx_W}1~0o=Chf;y&Nl-Zvh zOOzMAm6^XH*tK_l)84eo7nW;H`%@BmD{C%X|NgD8^1$Z=M}l<#op(Lc*w7jv@qLu1Rd7=oP+u8jhxxX%CrK4 z($zNZdr%QRv93bt)DO$q6E+vWPdIIfFUYFon|qS^rC+P1Ii*?pVj&ephVwRFIVV3h z^8(*Gi3?LPJN5O8^6K87>+LgTqz{ZYrL?JP3#n7bt=J+{-C@hQ#g^42qi*2a-TaHnQKtA%pC53Y;|8YSsUWj#Pv$nB34xQ0VG!6Z%6Oi|D z^di_ylk9;nKCax3|@ZFpA#?w}2W(MXIY^JW_FxdiK z0>Tm#dV!&>FKSSch+)nxO!Av#lpB1 zXu47h(IUWG@CpM7bl&=oL_j>{r3ESNXMk28g!Xo4n$f^ilH}Nh{3eM$=w+c=O`SzT z7gWgS)XmuUA~Of!@af`j$xsF+@j1Ik!?*@^5MeT+esEZ-@fOBTAYYVT$|k-wNfAfW z)Pd7r+I5)n@%b@74<50K{Ue1i!EllVa$b`&XX*>x4`F%) zbr#e(5Yi(G(fz?h+Lw~D$MP2N zV`3axfpv*$7H$~!i8}<5A>AL=tQn-Bo*m$LSJ2-!Om3{&t;QD3&#EsRpR}NqSFm~2 z$X^}I=_qZVM8(0lq`)wE!flA|}&e)Vz63+=pijq5c zM?-eo=QKX8uYM_I+`?mZ@xzDW(;jg;J`=$$qMB$?r6+J__)yl1LCK^~fxVR%;^|Mv zh?pLKu~XyF3bp)^10J@+3WfH?tR=_J$!#{>sY9OziZt~e4_3JH6d8=-!Gs~qtgt&d&6*GLO!0dw1eeN{K;@hibXSBpJb|IBl$Wh9y1>kX^^A zS&-#{)|pT;Q2-4|{8n2){Q>r-KGtvtI546hGDZlFWNh4XQoo1qm$)L9(rpzB3o{p& zeA+0_17y1+Jl}-z3nK+6MzTb_G5pmw1|JYvdhm6-V;!o`jlSf1-R z-Z~xt3Vo%fb`pvJT;khzDAyLrNkNsQjOBKMBE4y}UD4^m)AO;3(On)E>-h63##5z*Af z#;)4axxYp&xHAc;29eA{k~Dftf#Um&g5DBsHi&e$q$}g+L=uceSb#*3>2&25M;B!&egwWhd@{jAq7NeM_U+sML4?)m zpd{gK1+D^#^MieHw=w}rs{8^HAzO4P!&$M2FX)t-Re37PKvq0Ke()id2Nn~o0Fw`$ z)@1inLE;P~2ltz{a=T#kR`|7oO;ixqq)@Gq;q2fPH^Lyb84+xzj&S%2SyssK+MXOQJ6@J`z;Wc{Pd0aK`CTUY+>+cbUdpGzR=A zV)I6g1Jvsa=(sLySbeHlnX6Ut#f~>g6w)qjR|}c!IS)O>+`VoEO@-i7P1|S@rJ!eR zO5G}h0ooLv>MwiWZavgj_Li~bs^3fAZPBsc_j0gaTr#8mo$A7_r+yc7OVSOlicg%N zpwOyw$#Be6yHIjkyVkk?>m$wFi8CLbO-S)J-|X_?>MOGPYI#U}|8;>0Z#{!Nr~E2$ z5B-$vu2dI(7N>@r{ol7X-O)Yys`Xy^THB=F3or{rNPS)qUXy3>iO$jQNTm3~bI}LG z)5|6ntQc-2fBZ4C+uVFR6=XyoGpr-dPD}Z+)%H$PP&_tdT3DEt$jEd5bTxK))LtWx zi<_=;a}IlapX=$V$S>oD`HLed{BP9e*T6AJ(>25Q50dlG5Hk4K=!U86LU=)9vQKPz zOuj^9W&%P0X~3V1V%Xb2cec}(~;djTTp zJCZ&P{Yg%$?PDxT#yl6IGt}QtHi&D@75a~kj66Z`>7(p1(iX#=A?f^v>80X~=s@>Lk*k-C{US?f&lN8mFxh=_?{AX44@y)3F0TVef2 zOwyuq?KoK}m7Z>^U5oE{2`-%9H|pS^`rI~Ls6zOW#I{GG5?Y#DTN7cb;^+6mv z>v5Qf9>+y&$%U0-L>X7VZ%Ua;FkdDi1sX1>K~rU6nF1=Q0q5{cP6l?Ghsl#FZ@#ApRax>UniGLD9vEDayrclp_Y9D_3 z3Ye=MVp>5z>t2KhoYXcKkA}<46uoxR02Ebr?V^ROIt*R|z%?Z{fq{WYyurKZ0i%L| zsA%Y;+CNNmC19=a4fzoCaP+8b-;ghCE_f?JYr|np1r_kgtJiTTTryBXpzmOcbp@A1 z*itaS;=FH!UqKCoB{x~O=eSAf#uYn1s4p{bGZ<*R@I{jKO1xquF1IJJY9aINN&c>-n1MDC@!4 zp9X8%N2q5lZgoVyoAF5V-~V_ta#F7?TJZDpWoCIRh0kido(uz-B6fzM&wAd};>4Bx zMg^&9dlvR9-aI2^$m{uB`>)0M{66cFxB|4DM&GBY_2iBQ$;1VwbGMWfJJ(6htbKOi zxqn_QWBL}A0YA@!9b3CuE`uqF`DuMqGVjNp<3b8+o2 z#Yyj2rTv}>#+QmGm&|#WZdvv^QS%iRIAKe+1#LUM?R5q?3z zXh-kxrbE%ynt4EBcZ|E#5{g%bjb5pT-e2yLnRRM@^P2UqDG}dRDof`}1?6qkx^q~W zY}~1~x70y9z2Sh%i-qsi!ec*;-L_E~Zc*5(%A+MBvSa>ES67;;jg7@qD_VXEX5NW_ zPj+ir*W@V+e1LiVnRDN=u71UT2Nv?H(1I7zZQK8AL?_d^AjCl*J_K*vd-Ka)B=B8RY24rNWl+zsqQsH&cjE7^ zK6cH>&V}poY=@xT0~#$6b&r8Ni4SOW3SJJM9-)NO&ZkI%D#E3_-Sndnzu(nA(yVm5U8_+8s?r7m=#H>$2)h|wo*#~z!tuI;m?rq`u-JD%n&Yos4-b>r^PDSTdpYYq}(gg zAM4|hcQsrUe?8?u_=cSq5*ZZ6<{|vlJwIsoIosz*Bz;}l2BnRMB68MB%&_1EsdOj$ z-YAiYDYhX+b-l%*9*zBLpRrC?5+&}-mnRr|T@VY7E4jSxv@6U0jjX2z26rFOd+VK- zN0>O1lP9UxuH8esbn5bzca`;LKKg2!0YZb_i5UuwTt|U0TFAfrXn85%N~+%s4g_Lo zbn!%f*-o6z{;=uQ7kI1;V-ypqg0o7fT7*1BTMV&Ye*RKLgA;ki>avKX}to1xErD zG?zm{{0nC0f3B%t=d*+%4y&IGjv$Z-5t3wxA&t9RD>U`LY7Bg(u@$k^h@0pe8)NkP ze{8)4RMqX;^}7%Ol~RyyQBhI48&ptHKrv7nlwjPOyypB(IUgS%6bzXR$3p=iP9fcxNK(175?t1x zmrqSi-A(j{SPbHh{>LD_mU!Q&#J>w(JRCEbvd^9knwDpa(7>}i)AtNKb)kGyJbB@^ zJre)RfIQp@NS=ZpX7;3zQpi}Nq7KQ0m)GE^6k^;vmV&Vz4wImNQaJz<*{OQ)2x3>} zqq7hyqNEfxxV3!~Z24PTLw(U{|1ou$-yob$bgl}hhq|)3&crkr+AzV zIyK%e)j|m2i?|ctc+`u&1Wr!I0(0xOXX z>3M`LAGm66dwErLpk?`NJT#%%6Mm)OfH8nyj-Aol|0CLUjgKS$K78b9NSRH(j<|mP z`hVrx8j|@^udS)r^)V8^mCs8@B)zEx1>_72p(ZBpYa1KMCMJqBN_&{X!U9*4uhKfR z3+A70Z--W;hY&eAqmaZ2({}AF!~7v8UNW@AIfo`RR5~}0|F+tD&JQ0v^!ODig3xWh z7c$j;*lruK{LCnay(%d;H?`E(owG;B0}+clo$6B>FI#>TS@OJ{s* zYV}rPPK5^6W@$QJ;ob1L)kDg~Md}|Q&dJ|x{zshK=-u@34+6O))Zw9&z9Ds(`uDQ; zpPu>*X*geh@mC^s_wRyoL^PK#{I$?P-?%v8x_WI8N+D5mEgiWo1^IF3c|O zqXxTXY$YWc4_puzt9ReX@*fl} z6}%kF#Y+c$!8tpQb!A9i{UXg}OX1CAHP-0s=dKs#s7)N$;CXbwVt=mk;4n8S1dD5MsX@-a;{4 zL3{3S4&Sn5m2SHI3Lf%tfmv;0y zkRjSY)WwalSoOY=lI+%Y>J{!4IZaKrkN3!#S#C_NS-$&yOI1w#!BO$U#801eE^-Y* zRRR+kp^NLV-qIuIllM18%`S|L9NMfrm1bmo`5(tiRaFtVVSEdBcXs*JuLWg0$Hc1O zJo3IP`5qn7e*9FFl$uI;d~7Br_8Iu&jA%5J+?L=ip@W&wc|#<*N+)pLis=4>(%gI| z7hT%xHVNqM4Ef#N@eltM(W$PZgNog!%q~(qN5`_a9CvHJHdpW8 zmYVRAv~7Fk?99EnUhce!ZaRP;NB`($v7vJt6_Gw504Qox1Z^wD17(A!au=0r79~0% zk?Ytt)Yvo86^bch?fs;CWJ7quKuRsDH^kn%*TvwEf)zY~=lzdBu}d8n~ASam+Qj{C`ot0E(gf#U51?VF~F=ll`T-v@(u z6N1FbUB_N6x!6=RYf834}d->D( z_LMyjSTC5b)c1QYa9|WJ9hTF$4bCwv`y^_jFgOX3Q4my2jF)@F)rF7Q>k|C=FpdgqvitHLFF zwyccnh0Ck5sG7DvC;7uq8jiO<%qKRpDL$LvMsgQAG?kJz;3UNR-U5~7P?{q3C6Z|eA*4~trkTo>}ns$lhyU7 zQ-jr2q4=ZkXS1?AlZ$5U#9_jkor~r~&%DD7*6Kjc8J^#o*$~oJNa7os>Vms`rX8Hc zkFQ155)(nrQ}5CM^wWzr+wImQC5AI{rTcFD6wZ?T2^&$MM{HqGFeU#b-JXruX#y=}1b1|so zg$C{Ai_az?dmch@j|WHsi(MQriah-%bLKL0`g1I!49E0KAo)%K%GJbFpDog^R14gG zG)Ok<(M#%N2)4tRiNJw3b`bkNI3{(-jhgw$;PQ)5i6A~wX^Dh!O?xlzA0BpSXh8!c zfIVRI;RkGxQN-HH z_AkC8esVw9#@RVYjvd%{U~)*T?cjwOf(+I}NIgBoO4GB<&CKc`_lP)*E4ShH3r8e^ zp87xZsoXH$?|;B14d4EM$y3OUJ|ko0i-Z>?Ie7pCidHHv%F42FKOMst9+8AX3=eWd zU)9L8pXER;WnF+^T$fw6EyAQCKN!FG^6%bDpnaRKV3o;^+0(PLjiNx?USe#!2-vrc z_#lXv?0B{MEkl!Xru!X4QmOJNg)9LZk|l_Y^!qm^plXr;V&i@6@ihTkkX#gy1a$|5 z!$G&%uPE!>X6J^IdZ=}UP?E+jT&a=tzKc{n8SZYg)*R6DlvrI)f?olV#v!4ju`urk zmI4H#3Meuf7-uyqT==z{xc>X3bM=A661dv_4R9jxg1T)9F){F0LY|;*a4+apS3GhUG+MmGC&Nj)01q3t$wM^fuu{ME+Ur3hR@ow%>`v;xY(~+ugyCyo;+03o2!;BMI zmZ`-(%0HV-F2Xs2+lsb0!cMyW$QhjEB1fG#v5Nj1slPu>X*jiL@_FHwA~q9*y>b`K zxTM=GdZ(Z^$1Ca$09bD)^MJ5o4{xl5BF_iy%8`d!i5^?mtM=86rRMAU{`6Y^T5FOT>RC=Kr|cw|r)NBBsFaoVhFz^JE{~S`+2Q?tVK}%d@LoK) z&7k|0SaH3n!s96t<;}{C+x2Z0tCRSMV@zZuo6@5Xyui>x`>|bI@ze#`F~Q4?0RGAo z8(Ri!jA8ICB<&gQMn(-pBhZ2-Fx$4tmgdxsbN7V3j%HVyTOtEt!7GS&h$ed>R#ap%XW*`%LIEnew<`W^3n28GN*R%A7?bQh-oT1r|m!@_7U z`lk44T5F2FIOjR9T_+0ZaO*V_0@672hO*{RZ0$(5LL7Bkv9fnUG**AXjOdv!;l(Hx zl+OVFL~Hj>0cN(1ZR6{r8O`G}p&dIu^v+U8ktO`Hs-7cEW#D)4IIorUS}Uy&Vy<&F z+-(fqqpuz8?uO2-vI%sJ@^3!Rbd@>5Q{@!-*77>|&cg5#mh3VVxwBa=pFgYMUB8Ky zpUx829@%<*QCCAU#H;OLOn>vWpqyPMXBI6Ul6M0K(}&*)u`L6!JVH^0_0mH&lM#8% zT)Awlo!=*aQ=pLHW{c6gn(z5x*j;zG^Df)%O#7?(7n*l0I=j2AKs_F0=g>bo;<|C8 za(_$toXVFgN@ix#hldW^0_11N9qZz(8)ssSX;ltgtZd$w^f>y*W+wMO@wY3-f8A zWD6!{id>^mUfd-9U1DDzJ4#68&*q4Ei7lGy6&=)B?cF!F8^6cr_A%!2M@|7W*eu+#k&|^lOcU{28hbvc6&;d(3Ryi87Xj06-@L$FL-|J`$_my7q|Lgw)S<6*5 zS$|U&Z!bSu`iL#PfD)!Z#t)I^wtK5#cW=l_R}zdCCo&gu&H-%13H-9+aNUCD;NqG- zxnRokZW4;Bw;P{b_~OOO!We~{+jG2XbBI}5wD_Pui!nf(oA&0N^zQZ?*v z61_IgaQpgCyi#_vJsc-Hua#XV=&em+zY@%~z7&0Rv&ES=`LeFWw5t(!;Uhm@uMe-k z{f&J%ioCOaTU#Z=XNdF8br>;-$x0lrTqzXWsupG6=uWJ7rFQc!lZ@+}02PL(W?;~e ztHJ}dX!3D7aziKZ)|j|iT-uh?(ZXnS?!x#JjS{z`i(K$t->SBIzjc@3vn^Ga$z!r> z`LSJ3Cqv(mzkWE{0uS>Uu0f8n)HnxyG03c#>W4UQu zNmRURP}d#0vb{V05lNB~ySce(IKuAt z%Yg4mhQL~Ai*O+R8UpH(+K)Jwmt`=2FpJyW;(GoKTp8dsG~e;n0_g@BxSC(Co=kuZ zkINCd3Gz%PU_n&85HST2%lCJ7F1Z)fg744x*!*)QEVvO8mZRgL_aJPbb7B1mEN0(U z;iCf|8Sqj=qP7IeINaQK6LXNTAh;6&M(LB4HBJ4sod&`wu;Ts=aQ#9@*sMd4`V@J* zi-J6`2*L6A4WbyZcZLseGH1$naMb}A@Pw}%*7%jMlLGj-IRX)F0dY)&8ccW=jF;>e zFP{9@r!yi8;ckFZ-rbm0G6Xygge_rCg{AWvpi)C?bL88EfL?kXa$s=8_74EW*Z`K` z_PE0K^Z(g5M@O3@`{q2b2STC+3qao7KR~krD^srPS#6`kxhkTA?`L2sm6JvaCL6KIzFg=KxbBo9F%WW5m;6#yPSD9SqB z$pls25I=oj=fWe@< zZc^$)zW}&U6%-W%JET4Jrldf@B88$s3CO9QV`5Ss#QcxsJ%1GzJrHC4*S7=s(+4g( zV8{8_w*yA|?nvLNjaWoqnL;=S$et$QoH_s`H7v!(+msml+qE`I^^JQvs zk*Udh(1&=TMT!xs8H_5$xpmyGo(Q5C?=6z`)(Oed z3WEl2W?3(0X=yvu%~3fe5)xex{m^+MVjrB!HWAScjpFqyRY!JAMb?z8&f1Smc||NG z0#{e(`A3C?nl~Gn2J^hV&+%xvIP^?r5R$w_*OAh3M6Y=B+=M6Y()ZhN%ACG>wS0Q{ zr+eqU)JI5u7=eumF^&&f13(NS((pU^@=+c`k1jqolnuxCs_*zKd`sBypTVv1(`IdC z)=hWe_-uId6X-CMNIFm~k_1m;P8B1$p22~wN_AWqtYlDn?W7Sldkp0p*q3ke<2Dag z1`Ar*?j{?lebn-M+omXm2@jfLs%3M0zNK?%i-&A3@Xg41y&c0Bg4}!gwQq5G_OTv2 zGXbY>JWK6j$2y3g$$4Vfx$#u~^P;qoyR8#7cW3L&jleZkOHy4u zM8X2slw^-8pY4SnSB3X!4g11g%D;o(i4)^>er4Zi`)CT;ap>S(E}qio8RNL|6D+w~4E6J`OOCuKMwZJ6Q2e7er>ci6zlf7@ zZog#6sBmZ+!s3eD_5`WdIaH`$guC;V!?Rx4hSKRvSB1K8ztk&ucE97jS1`9W>X*jY z<1Zw=p%>o<{$2NOVx_paKc9b+SP06yBfXrAAldnPij1e!)+t>!xl}rM7oA zYajs}!sqCm&tfswv0VZ|>ghr*2Xn3ao!(~}Eks?aT((|{cYGQ4jpFvaT*-42d(f%g zRKd!pWn@HMWGx#UtsC|mMt7&SZAjO{i+Z++!@5$xa8g&NlsV#Kv#!WH@8mQ;+sQ{H zYSm|e-BVAtqBIlsqvrZ!v-X0VM{6jcq$JN?=>cX>a?E{WyjIoQ&A!hN4ah#Ln#NGw zB=e864+>*zOYNiXD4W-$tB-QAbUMWG4n}%-uAOCB3oLgg>9|YDdR@_Br;SB@$TwL@8OXoDsS}BxBO;bSLAT3c zVZ7;s`z4#(y@`Act*x|gPdjPFzj@nsIps){jZv7;SXa_NXV)(>`bOV9;h%P~4{cHz z+eBB&ai$z_F{tb4KZEslp?RxA1HpnklPtC|KbBg-xunu>EY6)Dlkst3TVQ7{@k}`x ztjfcu(dGvpK02c43>1pT%Sof4gALqh?=e?6U~|{f{=(!jVctS~j(7DVI5dnqvkl=QuZ;!rZmH@X z&(NA}rJ_)YHgx_M-RcvpiEbw%L5|PQV_zFL21FL$K*?QtmaHt%=Q~(i&p382zRB{- zb+BLGlsvYQc>BQ|M8t<4CmY;rJNl{Mm0in^v*Ar8_M1HzJE}P4o5w2AV%(P1UXb%7 zdEjeuo|0rcd>1ayuXZ#$d5jYz8uc2BYONpKV?Op4O=9?zThBE{REyThbg4j==krDcL_yaLT^UYVuqXDHg+HG zTt#;))2Zo7eLSk~=^#tu_ME!v@H(K70rsG0QwNf-8GlW7hFv3Dr48$(bO|{HYos!t zNW2gyak_x;oEc_;>Mn0tISCc9hQ{ZbR%P!oE32mfpP`=c#(Vma&r31a^9@yE$^NuT z7C6ewQw;sOgsQC#Lv}6JIcFX`kZ$gbz5r(%%dAJ>EI_pUVWIVVCh7MJSUZBMq5fi> z9qeAR#YPLe9LcGR@^k9HFDa-hPuzV*$tID^McY9dVY8x%!%N-2HzlA)IHI3ZE#qydhP2cS2qP-1pf%Ct zIlcq1^Z*+}l4lOyqO^dg1wamxWq_C9HVS9~Fd-uj9SOoh*&hG}vjvF2fVDX=HT)Hl zx~UDR6y}LxzJRgtUs0@YU_ixe(^78=qr-kyj{Ph6|ua~n+E zfY4L|N4dr;usXOiMl4>3~0|;Xv>rUNOeR6!se}=Q}KK+t1!LWuCEo&=k&}bNcfB2HBKcPXYa;{{IC#Ec90sh+ZR?F8zkZ4t=s?0?fMe#b zb|nSd`qx(^zLe5uI09dz2KV)U%+lm1S*d3OEk{pG&%JsV7Unfp@-Qf;nl?EtjVWBl zjggd2<*4*Wqawp6Ui=A|vqmH+Db1yhY|hg1I0P0JUNti-vsr#jWM@yZPC!Rj3_s=i z=8g%`Y56g4?5M-hESosW=f@S9A}ewbH(CDL3HS6e+5WwA8um1(saD?G@T^BnXu|UX zv!U?9uwuXpcvZ;8rn6~-{u8$>AZ7tiE1`NJW7)4BKr z9DCEN!}oqXCQR&4x{1*9*YhOa5<0{vfUy<$_^0>iTcCIWGDFWw zCU4`=p~S6_xcb zjM`T$scY*wZ|SJrT&)$bXT2AFEO1H2b@xow_AqV9;fkzPR!g)(w`c(+g_hY-Ri_`C z@|R=65GJ^Fwg#N`6?A1a-Mz$+-cj4j=z7!}@FC98B!g-3J9qR|+s)9I6?S?|u#Z*L z)_LxhyoZ;f<~jOChW!$%dr>z|_YRNxx9k$42~w@bFW-xP5N#zSd9A!Q8D}O_@8-8i zv=FStu+TSC+AoH_bP&P3uAgHUo1m7@W6jMKzNutL`SwFb_tAr}vr!5lHrTOrR#)qf z?ekVA`kAvqBZ}7X+Sw0_4dy=UMuEwa)A!e` zdl6$>!JPj|)i2`thGVlCxqMu|o7^v4Z(|4xqg6{6`jNE;{R90Ym{0mXiFhd*SZ~5x z;I<);>h<-S!(H9HuuC4Emgpfv1$fUYB&l)os=m#sx^`PBh`CeBo;9c9mrGrade#60 zfg@o!e5469uh&_cgy)!6D8Q#sP_#EYQpl)P!cJU|($+p>G3Ek%bs67F%4vIX(k}NY z?ypJ>cUbLs`pM~q!&m7aR1So5DrSnVF7|rJiIg~5UQrz-Vf*XbSzIT%aWSkclTx;= z3A;hLhJ|2ao~0)Hx8wsGr=pYfWXzM}<%4fWiWYsq&>a-Ky=n`!v zqcmD!CV>d$^Cwf$W2fh~(CI|@@cuA?Ps%UC zChsWXP`9rO)miT@vEuvBRE|y^KKo|7gCD=u(k_9jY40TE(3%G9?nSXH6)(sY>uYW< z`g2V_c>RvI{H9Y6{?^nV67iD5=^FF(ji>Cpdi)}lw1Ue5y%du-JBH6#4u@F>XybX$ zah%q18_jsa_{OTAV`1e?WiP?h&W0dN-9mirJuJ0DMF1eyab_l`UF!7Bv? z0!VL}yF2#B8ZD338%Tnel!V0FY|4XQF9N#DAU9{e0_#$w?-RbnFn%tMzw!-yWRS0h zXf}L6`dRV#0evW}h9MToarP8+Y@q1qqFuK&@1EkYCjgDPEx`D6bz--+_qxqo-;vLxSGOkD5>4Ka*7#KNk3t?k` z?gcbWN9|ugg6!y^W_uI#z8kjjxqtEx4#zF{kIJM_z=f@8F*Rt(KmGM-2K1|R{(J`v zUG{Kpvh<`R0wgtsWJon$sCaz+NTLmRMpz63(2{R`_nb^(IC$p)Tf1Db0rcg-(DtF= z!Yl1Oih9#KoC2 zJxe*aoBd2A?AcKHTzFrOf?^CJi{4CTaB=GX?f;Ha*5i&bA9_$%_bnnsd1CLv-1aTy z7M^{D(^QCTZw<>B6-8LeD8@VmZ-&hQOZKgv^+57&yt=Bf=#c6((8Q) z{>a$i_?$Ox`!U>Qr)I+k4L2r;GEL#kq?g?X1Bj;cV@W+sj+Y}7Z{>M{i7N;6X|whN z3wvw6GL3M?fIzH&F{CHw$g8i z)ECKT!SnO1C8f{9nd-f9>R*rXVCjRCtLqab>(~htNIkX7{kmo=>W`f2Cv`EyS$lg< z5cTEyk&)3`p8bJ}+C<3i)@`)y&Z5XkBJp2o_?_>q8k5Ii6ymVC|An4#&rl4-7lII8 z6{J~oQV)-NU}I+>8??2e%iCn%sa?|&IR}sBKPnyzlFw}RbJuJ34v)5?i<7$fT#k3? zkLE>N^{qN^6Y$9zr;~!hE^sNG(9;{mH}KBn>G_gEw*^pWM9dPs;FliK01p;oo*2U5 zBlxwOZ>Zp|4T$~C%>(YLI&L>sZ@lT&tF$Fo+22DPk zqlE=K^4Hb^GBRk+b5aYE^-gBhq)(E2kcczuVKvcHF^ub_xoy<1^n}Pj6w2G^G@p_& zaQ9vu?cP7+jk$xJNf1e(u2!jk$7b1wC6m`q`uN0&lGTMG7vi0(y{}4~lWEsNrD00e zThpU>=IsuqMg0~|`CO;Z39~cb{+g}fUDLE&0>h3obK&%OD#a4^;{idg;{ZN$=+mHK z1Ul-}0?viREQFsj)$UbZr1ln~SZ9qzlk-bSrceW6 zw3|9pC#LPJZ&{Lk?lP`&QD%qJ%+G7M#$I{>_c3^3QkRli@VJ^1m?QAXl{Aa&3l;k2 zeQXB>A3hOpe_u#0{e6w8HgU7U??6A3r7aH=ZqEM3$?$8Qh{J;wflKK59aKYVP||*`EH_Z1aBO zWMB|fnp}(uY-JHi$lI%@wnZC6?cu@Kp;0kplZZEYq`m^bJGliywY7elxrR!7=7*Zr z4bxM#2FOr!C5n-c3=v^-VKjA!8qhRf^YMipCsXx3aqU~=?6n!VL{J{XTx9A;-HC@QlJkE8Y^H+L6$BcH#yMc6a|J9UY@HO;T~Z8D3PYBq_Y&``&w7(vOJg>Ww% za+L=duoR>oqN>G}1SNa; zkjx8H9+kO)qx3?ZKYs^#6hD6IG*eH>wTG1)pYPpGkq0ZxRJb?uP%-#GlYjnH-*6TD z-z)WIEqlDP8BH7F@x6e8Z*|IyV#yzTr~tAwR~&WO5VJrQXZU2cYxkId8niuG7oh;rG?xdaLgP zXdr{};%_@DlrQFkUxlA3JpBa3YXX#4goo6D4poWU2Q6`zA-$CqWw;ujNBLxCu>oD^ zRM@tV#u%!`g#Nnw<2U4!p#zfR=Dwr}$Jm?u3+l#SA3;mk0zNk28Krb)@p0&J!28Ee zCavY(`Bd*%w+2T)8P_$y98BgKPL&??mSi4ZfC^UiudN%fP(aHbSW)>0Pf(jlYVtD@ zYf)1q9FLYgq)t0UU70tN9&qR2Pv7zeJ*T|P$Qz4ZYX^dytwgMwe{a}8HI zO>Q|($SGuvkw$^c95PV^iIbAe;-r^BA|13Y2i0*?i>8uhWFQ3~FedpAIDf{d;Rnq! z>e)A%q*>o8c~OZjZ^8AUeti-}qN7TTdOT)&%TcUA|<<6!=eIM@g6%nXh-V!>K}M7n>*^BBk`Gq?cI%VSEyZb zU*-3JWje6^1_$V9XlkKY@9y2Z659x6UmQ|rmoEpplR;YU0{6Rkq07)gH!v{ErS&DV z*1s9)yLhm|o(nETrEu1O7wszayGZ+6ZLR*ufF#J;z>O4uujKhIZ0r%m7ilEBNZ|>J zFCcHMCMwbe+kwXlkeY=4;2+!&y#r_)m>}b!-p&eo))szDHkAhI(_8H`9U`4maUP)cPd>KYb=zl;;b)bI11P@nJ!4&vrQU(8?Ppc z3I5MainNqL{+YPai*FESRnVMgAW}Y31_@E9WdT(znd|DMn^Mx!`mouDk@lZ-DvV0! zl4Fk{2;SA#mvRGm%JjVS0yz#&zo=(A`wBZsGt0_ipX|X8Fv8qq`!-ZmgU#`uZY#yN zMmU(m^`jsYa*j^)@K{qoF7;6o^n-&EoJ{do_zj??U~EKK{2m@2HX9StNHfWZe_$YF zMbIQZoV}{B`i0tD=eeIz$=u?XdI+zVu+PxYIEg0S^LA ziSV}(hXd>bu5fernt~?)nnIyS5_N&|;640$ND{=R0=|~18l%+_ z8UeGTYW(pTj}^^m0u*Fb?xNXC2liu5F#niS3AXN7-qEKKHKvE<Bea_r{%QXjnk=V-%F_fGH_SSt8e3vyAg3 z?x9uW8z+yDKt!ELz&71<+xFPJUplncC;&Or2=^)~6noWZh`Xp+o?evIL`Yyy60R_f zRtm|Pu5C(d?Q7D0JYJpr2C4-}qTi&F&kf3S)R#pk=X&JmCsJRlco8}uej(=j8)0il#Zcvv`*)aChn3=pi7h;0;Z++&tAi$y@zUXa_n;pUZcDuPc=u zuZ^Egm9V6#6et*MRkJBQ^9&#V4zi{6qTz2K7U+blzU2Y@E*OI0WJBZfbq$ZX-<>Pz zBfM-xxWEo`BpA(XmvqHPKHtCL;xbdr4cXKZV~n-O)8=bRAs;3c*%a%2>gV((@mT@7 zDeL?kAOid`uIuJiXFrfV054`N52!J3e@l}(jATSH0%3D4Y4!mrtIiQd5lExqjD$h; z@_LyhC%AGqvN|H!XBTVit>Ii5+Bv>N?HWVvWbd7{mwzT~`xfv6{CK>t$8ZHE*iUVV zy>S&Y8FS54I&v!NcMdU6>Iq_ZYYz!?;{Tbm9GVg>iH6?fos&P>n&gIzZv4zEb+vz%<%7so?bigyqysM`t$d81??#?*ut^|#* zZEjw{&VbEw*L5aX&119n=IlT2m}0Ab^VUw8^0VIiH$O}i!8dvs1?|=R@?NE- z(FjPSLRYT6^`QRmHb~|J%Os+vdu<|7+-jA@99w24wd{t;onxJ3Y$hk}oeZ z?!dP<*I{YjuU}Mn*F-Y^1YFQ_U-or;tUo^9FxNK_uqm%9DYGRyc)5YYX3)X#u%dto{_ z=_C{RCCeYYR^6uGKW4ng&2`DCom)B1GdXcuZlIcqT-n_o&tx>-FMxj6H#`ixRkf@VdtCrH*d8A@KVD)vtSNJDM}^P(bEn5v6_i= zrudC1tI!)c;R(6cU3L9lr|!~^bQ*bFihV>Dn9@UU^&F%#jq<0Wt~YNt^m&1V7x;lM z&_5(`wMz{ga?|S_IBgd3u8GGXTF}cPNZ-g;{SJVOFXbSPkX-PE*)#Y^F?+>}oK24& z#mvH@ZzXeBRrBZ18R)j|HnoR7I%A@0^D8OcP%HpMd)eLvzr=lFG_nEJ%5LgZ2gJUGM=-PU>j0-Ua?NIx_L-pgy#({$mkXi<9wDGeaS<@_fYIe5%C)|I54`KhNQd-jYXO*uobTDxi zU-~EYE1(3S4k&wg8lRYm^G$i<1rv1UFJPU)tQtbyCiM^k%t(un1`xG&43e4FaO%PH8mMRgU}8GwE)q--A-_Q<3ew>@^S*fxDShh#XEFe z$jadM4dJ&=KAC@tP~*SZ6pp`D@H)Uf3Be^Iq~j4C?tMfEB@6PFmTM*3Wxy$(`xAxq zM!*b}qYnlU$O@f=LLcAe(=n21f*1n6@-4ReQHu3b)%Y8IpXeZ*0VxScyuK#laoO$9 z+rCJK0b$M*&qlZ4%XVyXRMjzw)e)+z`+`D|5qfOOPxucMdR58-YmgzoU0wBtPgH-G zU^FMK|jBF&JIdBPaqJvrQS9GDK9S8{B*~R zN7*4R2sv#+PcxY9*2B@#-C%D1tf!Z;T9pm8kHWwHCu_L`#+@3D}4@=>( z4Qvhfk2ju$3||)NW8i*9G51=`U4#;_S6x`LR0uM`E&oy`2>1ii_@lwq@G=%2-@5U9rtz9?=_BcT$|9Zs+dNO07y5xs>C3o(8#N)8h{>4L z;s`fXYPdf5$Cx%WZo#m1MOoiSe8aV7Ro{wB#K0scG6ScfO(7c0rJ37AyW}m4}15(xPCl zr@*ap4HoT{@i(!o>h{#k9-Iu$ftBGfa}AYi){l*R6t;HvBTKAsV<=r{!iT;9`xq8i zLF|rGEGJ047*qLlJb#PZ?ubX1Bb`nafZox@|~c)7gHhD$HsyumB|J> z#UO&XK29%TymeNB<9dtpqgIVyH%@s)7_w1kdvTnATlC1*6y{Y*#k;7*4`g$+u8_~% zJY%2mSStA;a4~VMkIB-uhql~SOI$54BW`*?bBGlQ6i(@5J7>s?R?)ywPQvMA6frYz z5V?!4@XyopZB^c=PR8$Bra+4L>2v5OPJOIuGLim4m0xvuyu5t`&#Ptf+0y*NKqk6j zecW(i4hgaTZlv(6#%%C{;@mUplsVZSvAWuZ&QBzt^Dgr!g}z4|2HhC?VyslqL2LgH z_rswIalN{g*vxDmam06IN)2VV>8NMJ=yWtvZiGu2#hXUD zZd6uSJYOhX=zOvNc_}VzT-fceHR(gHloibef}wqbh@Yl07vd z6>dj?D48Rh#Y6C{Ge8C^uXyDvyHW+Zaet7*s1BJkzJ!OftgZ!xJ^A&^p>y>A5{+=O z?iiMV0Y@Cc5%=lS1O|RXH|aUajOh7TU8c{)4F$90toFCzdLoKUdqRqqtRgR4en@)| zJJ%Sj^yPOjMr5dwn9wMuQuIw_!2AZ~>)*E(sMybnZ7jGReRg+PFIB?mjJtNY&wm^4 zFAM$O0}k@Y;!jrp{*$3MT$(mXc9tIg$2Wwmp5=7)(XW#N!_LrNejQ@OX+)yv-NL5B zU0iHgytF2iI?cxjZMeLgb5F zpQrFQP%g#$`IfeLsD@YPiRA5u2?mQt2ll>KCmV(}QSPElJ4NJ_UgjOg`bAddW9lOv zy&mqYGewLP7ewo1>sZ!(!%VVRb(H2zLpOzbh7*j<1Kv3LUXGG}Vg5vvIiQT-(N^bE zbDR^Yjoki7Fi|~&@(A}$G>s*y;gvJ556xhKTymcCN%IplQM%TzUt}fF5Z=7nKQBo^ zCQY0KEZqjyHYAG>Ot(|;06OBQS;6W*e)TYzUM{p zK_r5?F<{wKRSoaZE05%N>T1Z7-@mMUEowO>#@wd~!i z58|w=pm&Md!y}MT5z5Qu());oK|1*?>pkO3-UMZu5uX~#QbepLq)r6{`k6ie!^N}i z{{DtLqtH9=A~idBF47Tpi(?fnWUPRKMxb!_M+KVxfa(C0EC$o-v`8D{Ej_)dkbcAK z5Qsnu@1Hy=^R+?hd4bZ<@z$_X20DlkzC1%y$vL2M3>8^2z4o&XU6F!65404mYA??* z-)X?+Aq{jufUZo8(*%4q(oF|=EZc2}&>Dav5!`BmkG^Z(Mrt;tr5UhJAPK=v^A5qx z=>t}JMn(nrJO6?62j3Ey764l!RE3^}bPEtk7`uQdofZs3i{?M@XxMK;DGUGu9oAjH zBh+Z1S;3bXvIE}$L-fq>Ig~w*9ni@D4Fh}-Ayb$ppruoYkB;F67!0GLMi$nEh11Ag zHa($a^7rQf(Gp?~>eQzhjvbQoE;8QSE32!wV9?@(KV73x7~_r~vl6%1Xx|Y*6&OrQz*||WSU{#nVq#)?QxNn7 z_dMe1ogickmO#NC1ho&i=;xv+hL{pax-3=*5*gv$?<&k+fXKy|DF z?jLw#{jjSOx!e6_3xJe?L68Qo_tR>^0?OTTj1Ie%=oncXSb-%eMdgimPdhzVe1TVW zRAg&GHe4`W2Yy{BWtqctkCz2}rX63sRbmsKm>eJY^sf1ib4sc(0qX7Bk=^;Yr8SFt z<6OWg|MT^+88#oc{7kv6Dx{?hYsGJf+D1zqyu7I6Eqzna^Y0xZ14Bpbj?C`ki+gKK ztc#Te1=P_z{rdgaWGi3oVOnjb-ldDtX+_`m)!asCZuhyeU z!q`MD)v)8`4Y~P(6TB`hRJ?$VAV6K?#yZL%&E`l|7Jy=Mhb z7*NM(+rKx@cpNun&@atb&6T zD113reN!rRBlYnQo2gXg*!>%*+@k~{#aYyY^KtEU6=J+EMLrUsmPto;=9!kG^&R#@ zXOtb)t_5N*VR;J*Nq2P=B~kqLowf|QC^mYhOM`l$dOG%htrq0pWdo4<7S^U+7;p5 zO79L{Pj1-Y^J1QVp~+#iEv0pI#oA?fPJ7X0F`)Icq3uMBS^K010iAMMiFc?Ix?bu| z)jLWd?-Szn2SWEMFoUD?WC?94l=_DKs7(oShn zc(=sHVIw~7(AZVITWB=OhmCzOXru;duUuCKNv3*LNeNI43^!)yB^~cEiN5SjlXG!r zvSy0=c>bidH31`2sBH8|wfD}hofyy%Lpumj;ZncOg5XDrf+K^7g~iXtHjm#_M_IF4 z1_gSmFiP(3T@p9SIs2Ia`ZL&V4)tP z`d#&zp~)Qr^Of}CabZV13-{H#Cn+h%T`Uamrk-is_AOsOBVsxUQ`DWKb?3pa0)}KQ zEhSM2T%mLD`@OzC{aJr*PWw|9*?%SGe^y*?V$^n+{_8{j??6$U+3j1z%#xL75`mr) zMnP^D=OdG>T{ZUgKvHC&K1C&k^V4;7kHe4qs@4LoRp={E)Oi%@t&`9w&Zk-w7OSp{ z8;plmtm7S9ho{C0SbmWx5xwA#e*a45@|^v)v(LL6X^ARQ7d71}z$bIbkyJajGM#CgSdY|;s?37i;X&b|T=Y*dRcVnyj zn3%JdYSWJ{jCH*jx3sJDpe>1gT;=zEI9PkqiGGpwb;>VgL6(F?*}2-!W?L84!k5qU zig2(GoT@&xhP$*Dl~h>9q!7{jaoLcgn_0Q1^B5z~YG)q>~J%{!lYAB8O$W1K>B{@RE|cFUiX^v$%K)a);@Uf8I5>vJz=U z{TLW%t9g|i%}-<4s-LG}#H5j)8o+R}zRwZC*Fuzj2I|3Lg#r|O2}vlXg1HiWpaT2e zy?gZACy9yc8XG-opF;mi_duf*SW}DIEat+YDWNfn`x(6{tN}Dix{l-plz=J0H*kZm z!h3D3l$yC?ha~uf4>)ON_R+HvO8_^76M8YEPd*Vy@6d)0dl>(S4ud?TZ}fL#EN#Kd zwDf*6&<3V=mU;p3DUk^GcSEQcDE=+?4J=sDR139euRHY8IXyf*|3PT%e|D0Vc><|Q zq%RGQ1kxTFu)xh2)+M%F<0X{g4M#~j$;d;jsb|sZ6lMHI9_gnq61cVq)ZE3 zTtY%Za;dRH^_QW`4)z>ywn9O|IMsq2pfDdizD5J|4fcpT@Bvg#urj>{H_4Co9lf_e z83ir`5Y@nBx^4aUGp>az@oRRrN+cc{GXO@JR{QNcyM#ont*$y$6vC^{ml`~jIdw{p zxWy7goT;g=&l8`Gn>DU$9ul*0y!-}7xkExYc%YuA3E1CyKyI&g4Yn;Mr7_ND-)laH zl*VjAKWJ0a)6=w>-D9F#QhEi@_U_XpDC5$Lffs8*TByN{T_9- z^7tAika3WbxfJMKRaq&&W`L};;FR-?Xq-jcY*b?EIy!cW^j|vN16vrNkW~(mlge#( zIVR^U$0RlAA4sC*X^u`PRJTDn@row~DLe(67$@M_{{zz)|9{we3#cgEZ4Y!12}uQ| zQ~rXeNOuYf1|^~(-HjmKh?GI7h;&JJcS}omGjvFI4Dt4O?m6e(``%jUQkEbCGvEI9 zFPHwS1eFbe?nfZ@5}>ydVnCsejky!x{A)Bc1aO4`9hHlRRO>&o#%q{54o%|W8NI7` z9PYFqn(lAy6-n9J*4Dcv4&Q*;k|K9AWVOWLW7y*6`}cf%hwY%XKN3atC2L`e)6dK$ zT7E|EYu@wEH?k4NzpU|4I+E;}-p%{kIm4&xGrFIBd?HrDj6MZY%Z-!2_m1JTiY2{s zrvkCOQONWvdq>J74gb?;VFX$W0GW$>o8A6Ip+8e)#*r7>EljdGrB9P3;W7zcjlB}~ zAjnx(Qu-#3Tp3X%wk~f*l?P4nW5j1ZEA6&_@q*sWyx8*Yw{Nl+nwFzn#BditJyIWM z0ME}gnl%e+x)&ZQ5Nm2;$~q|fEV-k`4-!(#=7NN$|CqVReu^;qBi5#uZ#{151v{V{ zqArk{O)-5yr!aM>c4yRBe|tBM1rzc8Dep%45=B=WlF-pnJ*EoJ7Z2POc=q-cmKNAJ z{^L}CJ_p;L*Or@Pu3#X1?j1fDXpUzcbQX!-jnUmqQMYoJ{DERvG5}5>T`?iONQ$MT ztLI(GS<+L^(@NW&qK5vs38EIx79$pRjBl}l-U{J&4=$ckJI)N$zF{~yVA!%?M&7?@ zMfNGi4y8pLyq^9fCms>&5r2?4i*5gG?@;guA;);=of{3*q6X!}bf+z9s)2%wNB69o zq%Ji4tuDU#6-hNA5KUi5SS}(y(x0OEy!X>PL#7+Wxt@pI^~g^yOD-)CvwOZwj}Y^%mZtLX0lAY+7eS>YxY((r`dI>-)-(* zS0bNA7gSVeNl#W+8|TokSe9Z%ez#h)cbAstCLujbt#V^$Uv#uIS#FRNt)1*0{_~mK z^%qjxQ@tC7hEKasXn5A*=hY^-*U=>qA7KF3pq1TW3 zh2-iS#Z0f}W-)jz1&H92mDM#jKeg@L-)dli#sZR|T;qe<0w1_oJGn$UAS+BmaN|6+ zE+f9QzijQaeB@STdf4w5UU2cJ(5<}m$->h7V}>wu;=B~C23J@7tJh}yCN%o>CgsA^ zSDgD)#l$X6R0+~_XOxt*{QAG;OPs{JMcDs)%>SMBMhRbu2I{PF)LwMPZDy^=ip2KP z3Nzv2)%o^A?Jbvq24bC}fpUG*0^_ai^_wo|Y_?lw7KJ`c*ajCkFofVuTi&Q9}^G~`dO*&TE%w% zzMsFpX+1X~%xGw<(;-+MgSWp2w6WVf1(wFzDnq56A+H_Scfo;g$1Onu`9FOz*wYI!*& zG^q7cc7(Jq=q+`{Kx%TnVkZGL1qC$H$S5i){oJJle$p7P=3b+Co_D zgV@l8??buT!H_FTd0Yb}?$EsY@a^`IS2DyxqkHl4^77R6s8#!*MPUY-LQY&P>yS`I zi0%-7^=ja9%$IUm&_@8D&5dxDlMC!B{=^Bv&wv0#dP2>V-NY@QtK5p2gK#g(5>Ir(K;NbQJ(kd>?S~bJ=7-Gu*2jgVr7{{C(Y|IpuJ9uS z)$vVkZc^B+U2rz2*g%y7zCHNHV)%2PqlRFR0v!JStC7CGH;7*P^L5A|(5w$7ZwXWc zJSMWVYn)+U=O&@%++GasS+sz$=z6JL+c|?i8g!Q*y6U~Ptqlz0!;qSq4uS8r^yu$A zB#WV2hv6~9Gf*wQ`MiEAA^yHzyRI_hKP#-`}9% zN0KT$oF%V)jE!(RHe{XhaHWNi@r2L2=~vz7guzg+}jr# zP=&?CynGfTaD57YjWp8#iJJdVBPRG~)pbVPOO@#4bj!c@Xs=&#e*Q56aeQp4TD1J4 z>U2lRVv6Q6JG-!*Jt4I8x&Yz6()t^;yCdVfJ>z%#zjNC}(^Au?rWav`9Zp?FSRcO{ zIhoxnR){l^jqFv+teh$RQXXfMZ9jfxa{srDfJ;EFmGl&|Y73h1aUKsXju$6@RNoe7N51JtvdUAJ?yrDR{-l*`T%4jd#VB!n8@dK8NIg zEMtl6-D4Y9t82;;&+@t9c>y=tH<88yW;mSQAz|GfXJ=t@)JXTSjl_K>^Y6#zl#&G zCDW7}TMax~XH-3oKiAJ@q}yfA)5c!i2us{AA>=6eEI8YMc=t59dAL@=SBcY&tzx6i zr0gg>x7u1^6(L_8a|NN3|E^L?>y6FYP21aLs-NrW)?Gf2(ugE1AkPb%(mKqV6_H!z zoNz}N`{RMyR z!*bfn`_de_wyAV4Wa0t>jMk>yl}W{0>%IJ~YDWDUA6`5O9Uf(EQqc~b{pG7RQ8nt9 zlH##<@SB6jpp0EzW%#OyNO8qj%8QI7fY1&Q0}$X;XhwFqFFdoQ(CJH%){Nc zf0S?R?2apHBR|K|Q6@?2?XHYp05!VR$}|I!4SwU}{5*TQ4SrJL{M}a^Cpd{SRZuq> z>v1+VO8R#+2_n|^?{S<`2L&He@z)9-YP-WtS2p8tM86ZVrjNTegir^vmFN}x)>V`4 zmvgk0|A#7Aq%!MO__xyZKfCw*kHNuPgQx3j7ANt_08Ru24Vj>Fq-w>WF`@fljkPXT z6e}b{-#B%^iJ6HRxvm{ngSg!+%foR65#miQ?-TQRWqjm~a*tQcILw0UK#Pt zHHC|YTviXi5;98uE*&eE!kq7o@Mt^@UIgDV^7zo!`ur@qZSYC= zxzLZne>_?c2+`X1-c?e>+N1@u%E2<1jQzl?JIYSC&ikK)8U8*g8;lllCV^}~fswV3 zSOeE^odm3>i7X~vDBO`)CC~3bsdfV7#|&ITSXgG~>G`s6RNSRFoFFx`c}J$NdXEfJ z`=Prl=LtGg@Xugor&0}YgZdKPfT0(D7j}?%mTvcF$-;d>RecrvT66%2IAaP#MdgS) zTJ`e3fMW)H-;ghD+7}fWndf=oK|sWn`?*7K#1_IV@7%phX$oVE*Hy6=_RZ^LWL$mV zS7IrOWtPl-H3$1Ks56a|&VT1eOBo5Sa;}3S^LEyIAZYTfp`XJ$_*^LdTGIOYF(094?s51$+&S+srB7Uo> z0hAz&WzZ7jfmC73DFp`^lLcQhy2)K!+;c(%O>sj_Be)8g-Gcr(0c01wI)l$BvIrh4 z+KCTa9$6M8EUGZX#>U5Cdrm&)?=@GyABOIbh0Ph>oSd9GP7obXN*o2-Jmir-_?t2r zUtIih_hJN<1Z$2ebe4H3-ZnV3efu_IfOogmS!a*?Z{rX%vyt=${+mZoEr?Hk|KY1w zKbYCTX#@Cm(r(rjs47IahXI!_r)FRQZGOn z%cB9;5`)zdU`o=7_|)+LVy;ZyyfI3GG^5vPe5HP1wHX*3(01LXDy89}n+-n3u?A|XB4#(#pi6XYQ10p7+E()IhJf5$M=5k4Ix%HO_!UmAq4(2d@PrurE{eF@=IVq#(gKYXup zM|O?i)UAE;Ji z`_mJ(JY&~v?)gH3iHWy{)b>e@3o@E#z5fPc`@D@#{w~cp2*6W*jYdQ)E-m@5kC&n= zmL`sw3#~4>G>|N3d4J4$Moes^)$gHlGA`}_VXDi$iHT!ooVAIn_nOyBkwgId$f;Rx z`Adf_t=^Te*%6iAjTkpLe4^tddV&%T2pQ*btGNMiy?mCoCvIqHgC#aK)ybCUh2xtE z=b`Lf8K*MpImeYX5jPRa>eaOGcii?XmD8>_q_4&Xy#E}0_Wh6f2c4Uorp?jM_L_>P zN4e$#%W#Bl8k7h37N|_n2W2rWM6T?t2C5d=&RWKYh<8ctv^Yna;v}%qd0H{iOvoXP zPb!lH&W*5bf>f5zG^u}nVckzz{oFuO&DyH7Muy0nwaK$PyhP(JM;-EW)`isPCQD4! z;3(VCNibNg2vx?|f6)e4RnwIOb?p97oWGpVdWBQ{eg0C5@pH#QK;vh;HIDqsb?eQ< zz3uyC##kCy#DzXD?#QnnKEwMp6!t@|?4FJu>J~B1Z-97R;k!@YX z^q+Id&Wc?EhvrW>zdDsRK3H8uVy`%ID$Y87xOSDd&=Ql2IE*u8uB0=r=k(1vQQ0AT z=Jqjr2xHhC1Y)0md~4K}yd}nXo%H$ba;0FupECm2@A2Uey^r`P&KruW;O&jqyl0Th z#B6qRNmA#?4K|_8`X1h_{t;Z#bOWXVo7#>MmH%kd8z-YX(>KR*XTVVsBz_@Ch>2N&Y@u*p)!B2W;mAi#eVl5+kkFaNVGa}j zA-L(vGFN-i+4PxROIoI-aW%ne60^=c7v0NB*v=@SjL`9SJBY z``?Rd>%{h1AkqITiT{7b5wg6P(as+e&O*Wze}YX47ut^cySwN zyj_&#)pD94(Q-CqjdgvK|N4FFrzX9mRCDhKU>HDG02wPzw4wJ%*?lxX`5v%_!zK5z>0M60~o7XWYQarL|!Q$+~3=a&&a@oLbbH4tcO;cjhFr@7_~S$J_g7a-s)Rg zaA>+3W3!=KnvzdI02kVhj)(sR`_?~b@b-~~y^+*Mk`YZJU&n*D8AwY!FrRO1e2AX@ zDbPjw7G#4Uiu;w-ek}G8Dv4l>8G&@06dH@b#8C>HWDs;=JolDa7HhaEs~@+79+U?M zk^6%5$LA}cZ?b8l~tqc0JNc31`R2S0^-t7SLC#tw$+gxyg%lY`)As2=~y=gTzx@D7mIUHUXaC zP_qa(BdG)m&x;z=My34}Z$!bx?dx|Eol_dnv&d%kIsMnxD=RC9a}mZ}b3F;)-s(@} zsn!vDQUrn*H!$ujV3e<|vlAP^E!GvedhSQqj=i(A-^rYmjG*>y4y+*0OqN-Zb~F!BC2I_}u9ItgJe}{%&CD z8~IZvOtZnQopLv#4NrGM3)<`!wqf}2!wb=CRu(8Ko0Toz<`u`nJe}gWC;L>KJPa+d z^l?(|52BbYXHF1nCuewhqanh^@&>h+5%-aJPmj0wD5%*@P3lg!b%m%=)(6N@1cEHe zw^HkM^9W1IoeK?KJoceW=u3OvN(*UlBG(OC!s@gRi#}N#Y5W;Ffo+Q0vzN6+&fby` zT^v*Wu__6S3O3gFO){R#vs1af(`ABM8WB?KRu%q`{;(yVs&L)-&xF8NTSlER-m%I} zqd6IIq1;1mS68~Vb?50_!GLwH6PE`_#ym$^H&!nlmy_Au<5yoi$gR>VU2#=%`51)M z1QNL50t;u401@52UR3#KeEO0SjppjX>G(^5f$oXFdt$8q4kFib^)s|PNy5gv;*u`xOS8b+v=Mw6)DjwL>$Md87*|ghO_h1K8jC+k!=-2zgO2gwE9i27v$BcU$FS1Qp=p6ZUUh1jp zcqCb)BwyNk>+ize_4sz$8~6N-^|9)&m$17=Ub4D&W;9nqP3_j&)kZZGLhxg8S=T)1 zPK1R;T|_>HDYO0_pI-iJ>w{RgFPCgDH*(@6D`6F@iRxo$qcu3-BpMl!b-N(C^0q!q z`wfJ@h|5Q(L60V_(hF+E*KIUc3sYtMmf~uC-oJNP8w~-p6zB3~uv$so2{r2jauJOW zF2D{I;gy1>*h1PsjKOxB5$cSABT*`}Sn6GsK-w^8LQOR1`pgVxdd1kt$n=!!W|Kd? z$A<*QGmL*4^%}egSd^+^_BfMfLa}66rCpZt0o}i?+N3(vFlxo}&+T z>U`TJ`9e>AclaN-AQf$8%?bp@z_k{%N}zk1!|rQJ0{zux476t1QmiYpy++g7bxc^5$2Q^Wu^cd6O^n`T(!~XEyPAs;dXn zKOT>U!Pgri#)6b15iwvZ|i zL%O6XsjNH@KooD6$qKq9eSIb-%l$ut)7CZO4@&+0{JzA+1w$MHP{kA-?LY@oQho$E zQ;cIs3Izbj+jA|HreMJX*0`iqnEEFOr{2!DUW0{#qYv}ar91%j#>8yHVLIjOx1hr< zV!AiV=7=EkLKUXdRd-n9-@kv4mUcn0F8bC0wbCv)eXJl?GBsrd+W702FNl0q*%q_^ z%qF)@$bb7Ep3>0J@JpNQ>N872Ltn_?dT@`731WRgn%1xJLK2sw`roo98k(Z@(S5?l zZ$6~2n)wv-;?&d}kg7GCxSe+p?OWfqF!p+vNr|Wii+-)|3ZFpQyShdJL?TWCIWEEB zTh~A7kJrXkE+YUPeDEgc^7H2nmj3fLw+=Sg2Ii{|r?8;;rIrX?_v^BkHr0(*a)BTT zZYU9Zv>m2vLgB5`dx^s^rvku5*$j~T-T@}9c}}%96-{*czR8EEC^jOxD$LrmbuPDE zIYD?W_DqPpOJKyRs>TApIVrIigSYQLn9_+CEc>@urX79i*-;2h~8x4z6O0Yj?({C@h^30W25>oZ4D zl#!;wEaoETQ_`?b$)`o@d)5*?F$^;r6Yrya=HPeJ`s-EsSO*FKM@qn{~VA0jJOEY==P%2}dNd5s7Gyuh%{+eK2UdI%eZSCe#O=9z=BuHQ`42s(YCN&!GpwGF6vS~EJYT3jlVK8 z{zxkJ_%nxgFR9NggoVvQ>DAonv0AjaHUKbGUqS~jm1{J$qB{8(AS+K=ZS3qt1q)yE z#BgDM@rAr^Ow7hM6yX(##p%};-kQMcrflYRxuC9$`Ck(Y+TAiDEQ_E1pLfDvI~W`t zEm>N6>z@UzY|wFym5+9IO7|!1#JWmu$6Z*2YuxJ8dE;9fA$CTq6r)2qw@7=#r?L3~ z|Lu1%Atjj3(>IAlPT1QmwYfM8-;QsuJL}l0Dzf?~+GV+!FuoEe(VbC?Xm*g5Ih-n~ zPY)9`qkdavak|qQ#2O}eBkKv>qXi|)czTFq0lY=O?c1%zj2{m$F)<&@%S$UM^(D1~ z@SuU%R|vEv7`Qp4uA~j-qe6T3WHp z*=1!E0F1r3Lymx*pdMfuOwuR+kbpK-K7AtJ+ck60tks}^mH31$lnYZg!n|%iACtw! z$w{{29+cRi>-^$8&&#JU;sAeS(L7LvIvB+}#kDM8)rEtxFq9YpRweivjgp+5oEZ4{ zhO*iz86h_lveIu~?_gkcoOp|)p^+jkpbQKaWT}4!>QDLv__KVV@jhQxfFd18Wubu0 z%T5d*T+>k6tz*br8CjG8VjGAQ0^Lu5SEWox6hARYj3BwFhx>4%x3BL#y0R8rI-5&y z9z%;tJ#1yE?s-TUPUwSmWo5->SP-%0Byvx^gaGU&Q9Ik=;mn|c-CgXIdi2PbnT?nC z29((p5#r`EK)ViDSAgaEnUV233GrLMyrM$#@505%!EqH<%YH>TV9WO*s|f5j3BY24 zNqaXpGqc^I9oqkZ;|lNu1#{u7YPC@(6?&4TBrCDPUYZAgZGUM-q@S+}f}3Wrl=GOP zvyDJI70h4gRlIZ)3cMlgv$OCP2>qa{uOm(z`HMFy1||iZtRU#gc(S^&!N-Ba*c>?N zPeaSXSJrgcIbv)LdzbB-JnCmKJDTe15KL@$?)(P9E^5H6xXYCkf{r#jg+Rsg0Mb&S z-6xQ!hxIxhj8=b`n{-3;^{rK!BUJdpnU0yIIs zW{*N2KNE zw$yv$z~<=(lpoqXG*m43-@LU5d5BC!i-rL)gcy0)W&*k|*9{GkF)}jZ5)wWF>2jF{ zyw<-&@cmn;+nAZlJUysF$}AFBSy|GVB$)uQfAm#Vva2mGwXNJHE6PnveIL(%`1zAw zwk+re7j^0G!}5x;_)K$L1l)?O{waMMGTFWN^)WWX)$k#|!s3_Xu}!0+*4i^KGf%Hl z{dqq-OZ2aOEXHvN11!$z?;ieSg9?Vd7DkA*M`tBLV1SD&#yde1w}%fezo|WQDBM?% zP0I&N$dSv)LQ8+u7II!C&}y6)FZK`o0XS?dFuZnjFmBifm->1lbu$Eq3bdx2!{cJO<#tD!3?MD4SGiBkR%kqMm{~dM zH@tcJlv~s@oxh~IsAKxnomR;^E_evp){)76KaivKBd28(cXjraVYMg~&Lo~MA*^TZ zBd$H;&`t?j{?5?*`PBk37+VV1P;X&Bz_Aj45rTv}5OVuGh!rd5tzDb%| zw`ellt8jq4VqQ7p`n+7ZK5cHrbFvP8aj2dz&q)AdHYYHBZQjTKyu54rG%@}A9Rpgn zVcA_QjfYvpJkICa9qMbo&Y!CZ7EN3%sTY|mu?1HV#`no6r*QEdMQ70$|R zbN_OHb|M6&^35a9eppy!Sugyq{_u%}>YnZV*x*$e8FSUw`3Z^yVUq&`#s)!cS8%N* zp8lSEEPHzC#tlu#e>&g0^!@1#j|1+0z2d54qm;$XJPs8&Waw#V2}?tG4QT^vmav!| zkixwv0YsV0zL=BKNMMMo16B2gsEEbah+_sfcfwK@KEugiz}fRa(Gtb?5dr5W7qkS< zL2kSCmd=$Vq^3VF&)$r&c;yEv|BdJ^E?L8k)zOtP&s0fdbfE6idXna6% z=~W&uXHyPcVUN_+qnNWn6apF;(mOKQuk;lYq~j!g#mez$m52oDKO^qa=`>-sfBMYC z>#mre{3U(h3+o{Ku9$(~9Z=LiD*f4PZOucrT6s}_$)9j*%la!Pcx?~O{rgmcLB^H#R4&YeDo}s0oX@vf<=gP`S ze>E=k-RUT%+|fKXAdqtpg(dvYvoA zsNvhKXB`Js(S;1OYH+A3l95up`TJ@fwf zHR1(Cm4NS>s}JhdH>wY{T*x{7)1gvE%uRL4u?0pb>`MS}_DJ$F0bUCgGw5-WrBzKe zoU1JZ^BNn+9!C!{z0obbXp=I!8wT_?c_n-zBmdQ7becu@scqj}<+U7*kMWw43ooEtzvci6(G}kU8^Vd(1B?K6sbS=C0bp^psfA?UR@1?0J4i(i2 zPygz$`S|Y@^G}J&X+50va}j!v+rm>zE$fy=IumC*WO;IqPcW^3- z^Y2*D%DJ{g9LrJGoVD|I@*rdVTc#!vxmU?CZC6yj_GW!#{H2HSL_CmtMa1Nf!wyO1 z^rAB~+}FS9AL^9ozN$Yq^u%$p75hejX)N1tQUS(0;|7}8Ct?Owp0ScCjZ<;jJ&pMo z@1~QdoTfKF=FQ4iBR!rqT|Q{%bZ%*WW^;AySF=~@C^kY%$W{q9kbMdFkEst$xx}>9}-_?TX6r zUtK|;&pia+MO2byXRba{3A--O+{kQZvJ8JbQ2n_{wSMYsN&nrS4o^n;dyh$ZD6%Pl zi&Xzmga+$NYG_}2D)d5wlEa`Tl_ z7e}lg_WB{r^*kN9|FiXV7#pea|9?g$E+ZLq@|NsYgBGY1`D4xt=T|`2;h>74^zJ{* zCv%M-?;z?z?-DBBR?aKn($*2%NMpFpM4QvX%`d4bSKDV!h^szl#iQ-2Nj$b%6eI24 zFlo1QneE`=Si%g(t6MiTmscr#>U^Mn*zLa$( zUfDXIjM|e%IQhZI!?&?I`SDj&l_N&c##5uT_4zFsEGr&f26-u6VdYQHTa5Ot8KOZR z37M9$m*R2>&3-MIbrWLiEl}LJmVY%SMo#gzALeCH@hk(IN{~CF414AMi(1jGqmM?W%i^h>aMX4-?0$XulT7=C?Qrq7RWjv^O&O7BR4mZxeEWQdSZa%g0?g?G&C(E1MU7a;Oh4V z-4F1UIywxXUWB)aiHQ`Ecs@p-iBACG0@DVT&wn{QGLAjxDVvoc{R=v&9{k(jaaA(m zD(r2xJ0n%Fa)L6_JO$89bbAG4RCK*Bj;w^Z5l*;=e24a~KFCA^xvXKmS|=^+k!jh3 z*#1LARS)*w+8JOcY=U4^_E9UF{+bvVQVhPjbYMY@@A8@f5do)Wce)iX;K8@q42ejt z)6!Z-{GPZ3BOM~pC{1AlhTOAKS>Ke`GBP;e7e-g5!anZp46a3Vc?P7}(}>f#3TrnVbu-MX^v#}AC86g~#VdJvscIh@$t zfNBW7$lRYl@xfabxzhAGa8n%ZCneXe3P@nQB_x~-He)WxwD;yVF0!PLbGj*MT=VOf zB#o%c_g?xh4Z4fhJ~v5+A7;Pe-f3=hde@NOkv~(|jmPV0D{xQbAl#w0WIoi?_BD&kIJVBF zkvWeQXg@oVGJ}b@boK4GRR1RHk0%g?NEO-$lgMuW9#|RxTT1KBamKas!7h&$s zRtIzqt9As0P0uD)zOkSQ-zIb!CoILM&Q1-C0CI3l$HQ0SV|Q%jhk_z$IW@&-7CIMO zwz8o`%0H%gGJX|N+2EFY!?U@3M$!`)1{&zA6zg6;pV>#A7CgXm;^Vm&Di(p{$jwJ< zPVxog#sakn}f1R8- z&Kn|yuLEIw3MS!ZAoLD#Ix2}SU!uEtbAo4edTYj}>imQ%aToG)=YEG`Cm!qRi9jcA zAUG6lH=91&e~76^b}JW`5_4^+KyvKj`rR8eYXM97m^+#p8NgyE&Q%6A=w8?WQ}UUbGqg5M-2-xH93Hs@Hpyu}p(+erIM{&$vr@ zt_BrQ`r6X$kCc{H=96}7n?9{l9-ckQ*7Q(7VSRr&)fJ*G7AD@_HUq=v!!I8;&gaDZ z=T-JE`of6X?8<+LjQj$0O4DV4JjY49IokYaVY!PJ&)+MjvHl#ar?mM_*r^D^f8 zp~U*TEYWPtWwQ}03yYKiQ7vfJCr{FJ1Y$1ThWSv_+@aXI>nQQN)M`e*A`|e?=5fv zsHKoizi4Ra)n7e7IPda+jZ}RtXJsn~)=O_3;-57&v0pc!yBf7w3t5z&YH%n$ya`3O zD7*JM>aSnxg5n>Ske3c&#tg?SOiXz4-gOZN8AKCUG+@Kq_YflooE2^7G_f3{k8g%X zbab4is_IA)HiOEYQ1Agf02B=pDEAkEgoNGh7pPrLbyi>Nfk+2n!(KG`s!{1c`0?XM zvaB17f2{!jf^N~W8JFA!g8YJl_fS&xuSp6RWt~xXHp3ifK4V~(0_+L1l;Q#%qY%>8 zQ9uBp1yHK=KQ)h)l~OQLA%iqJKM~QPOGihi3cB2b*97nO_UzD=Ob!msn7$;MKr`HC z9~QICf(Z`G_#|-zy1)rW&X6oWAD<+1h*t~uY0fgrY)y>!YkfN|C1hUcnS3~&@(xm; z%S_kB#wsl-0W(dDKzW-9aKnn3#q3U?AiYXQ_okkz(-PigVs5UVl>NGJ@rR2aXqq5| z{Sz>gkcI)wBphN8HVObd;NsnD-OUP+@ei%>(qk!dQDE_KQ)J|Fhi_p%2z>#wmOcTu z3vjF*gj`Y`&k_`;dlIBzf|uaWSqq?%-oM&VZpjaVXt?}e%O-d+CBS|GK2xtvKcGdS zkg2GM2izmDAS6tvs-l7CFWefT*o)Ux4CsOQcm@EBA&j%95?9hF z|KIwV7Yl#P(3}r1rh(_V^EHpN)ntf9r*WhUeHvJMdTTOi2jEZ+&l>LEzx_}O7rSw< zp^~}tMeY$n_{O>jDRg}XlWUbFQQkhY@~jMElG!M*&h~TPe#8o?$SGo6Y>f-_ZlMl94-a0pClFu~Kec0x=>`G*?#_?#^m2i(~Zojlmvn{9d^<94~ zXDK21KDZt|LJ~?|jcePk4mtO0@>WMio`9d8UwV*i9k}ddG_m@PQ(TAm3*S&;3ai5~&y+~cY^%w# z6G4uY_I|4TjVle~c6L*}6PPs3ZC7sJxI=UF67Ok_32Fo9{!U$kRW@dr)l?Y; z(w?mY<<{@5VkwryOV`#elWR}9*1|D-T{8m{@fbD#_#&e*$zD7*j%GwC;dE)Lz4`e0 zQtMLi`n$r?b5~_**VQZZIOnLwC(4F|aoo{WYg@mzHAy2qA8NNSd}22*%~tuq?bcwF z;{ITJK}gxyyxaAeBYXJQg&+zsB892hrr5^{^g-m`!&1eK>g}Lvz-?{HQbeo?ig8TO z9qfxANWOn#k=i?a9mJw=OEuuSb-o)%6Y|HoTQmV`s%Omkla)Y8)m=AkmCJ29)wZ=m z!XSA3CQJPg3lDD$L&7!}BGKFTy0{M_e}A`E)R+hY1BYBj|JfmXPvhMT(1em}U#FgQ z@H|_!zb%K0+c=Y#Z)=Pzc5<5@0jZTs)2ZTt7-7QvC+q1gS#IeAM}lS5?pIw8BgIR; ztFCo~|5*KK`40tYOD^b!pll*oGf^phyI1BIYBZCvccFB4I?AeuGo9}Z0SEwgqlG6? zn+Dt9>B(c1*d>@-&25JUS97rU>RZ=9{m<|OF%>D*pP$E-`T4|u?<~|8YS?v$J22s4 z!{!{?gl`yk$^T0-ho0+SR_s)SOpR@+(BaOjvMnsT2xEm6|)nd6MeR z4(Vk)YbQ)bZ+<1+s8$)@J`SsueW0rw>G{T_4ekLi+$)gAEdrdU1 zoJPTmNF*P9q+<1AhF1<_^!YHMx3k)vO6z@zuSR3S16X1%mmaE)zs*0Tvqbnmi<^3A zXw>hs`VT$Vl$SH}+2O~K0Qbbg{NW>6+YYz@czqVRTJ~7D71BHv9x1FgVn?a`eyPZ6 zFTATzdP}%zdelUC)e|*r{$k3J#4}!$Z+QOkm(#bgUK1yjW744!aslpVJyz_&QN!%- zoe5e6G=${4bc>xyfSk~0&L|#F1{EW4q>(Fi5|%RO1qJd1a=J*hd_620+K776?r1^G z(BDTM&owPiBnn{^A0LlNa3pp=d2xg4HvyHBmR1h;*7(EQ%4R$V?Y&F0IGJe=tyMO$ zHxF2PShajZirpWF5d6$i_v1?^VQ#z`@#4E#yj;dZ(^3K~?^o0J)$pe>G^Z!eFCTq8 zoV5tK&aZ~kJHqa-_pN>xw$Ns1LCCub5}A-JQN`!a6SxtJeaU{_A7z!bwCEsifYKD4 z3_jl8GYD$XTettMyI$y6Z57@q!_3L?^Jb9`eVUBk(Lhw$_87qW7H?uPTrUZZ?DpYQ z5UtXCp5}Kk3^<&iZpL;*y7QM;8q|HTjrdJzoeh(M!1Ji~ z0wr!SSrLj|9Qz1Tm9CMIH9tmU#C3IzIi^<_)zF%y?(P;qP~kwengNX{T9N}*Dmr~u zpPTgRVdX;?Y)e6B3b~dO2C>)?1Go|~=;`Cs&2-*EutFLvt^)%D48TG_&R+`%p`e03 zcP@~Bst&$fbP6S~(}_^p!WChPQP5%od>@e7px=2{xlw_yAi94Yo!fW1EF=DFW3nQ18xVGs{jA1XKU!20PX#w`J17#rvd0qkP|t91Wjgz<+= z9H?F6L=Q#@8A%hYE+T@Ab4x#g%cR*3lEWuUBP6n4F|n}te}$w`?A3gxd)Goht(=s^ zG2Q|6gI@jn2-*p~3J~z#gdhZTG9y}03%7`XC>4gQI?z}N!g$Y5t{dez(CK32$)^Rm zlMo>)tp-JyH<^}(;;l=mfFFn4jpeFD5GsEiMZWwN@jt@c??1T4UBPrZ1FE-2j~{>V zHIv5WazE6h5phyXUH>l**J{x>KK>c5Hs~-; zC@2};UI^2^32l=~=>yYl!0v<07p~Rhv2b0OfG5xgFfmtGpL-#%Fcvoa54eqKd(@_& z$K&}d84v!+TLOZB+WT4_$6N&kYO;5tI*n#jYCjFjZjP(?xT8K1bR@*FdM8AU+gU9| zT)jH|L9?=Gb32GGyUulUK{ZP(5!R~NzIaWnPvkxuV z#qMu3MwPE06&0!XZ@MUQx^LtC*dXCf2|nee#XCMiMJna|D)oHg?V>EcoLtE~eDS?$ zM(0a|%soC$O-;xb>}RHZ6Q^>T8BgWfoE;NTd|pK0jK4Jt+%_I zqqd~?hLF4$TJF4q_x5QhzQ86Il$dH<7FZKQOKw$D?)Ozj+bufc`%Z1tavDF)-8o1& zV4&{F*=!nbv2y%ktxYi!J>wu_lTKoMZYi)BSft=*yJ_k<3^{kbr8t36wbOcpL^U>7 zGBz*W+Oo*nY8|J(di54h8I68)b&-2_vb&A)B!}Din#pogTWKM4Q9vlM*JCPXoYsOHqf7ZypCsBE_XoPEgj@0;A5{)2R^V1_1#8w;2@nePI=5Uwg1P)bo^_6*i+@2C=afd7= zxEjQIieU{7rc9#lX>WKNwpa1?Z`MRX*K`28{v8pKHDGWiKTzBdUm32IZ`$;5d}-z3 z>1h<@slTdY^4Lp;P7o{f@vgmy-~+)DqtADg}|Z z*zWGlaaX|ctZ+N!<(vDvRoo|zoO*U0_at1qIVqvN)E;83DLZy2Y}*1hPZ~)pEOSy_ zFJeBi#0B)7PLAJgz-QOs3a>3Fz|XCX)#d0mdG^DJGe!KU&bq3ks7Tsv=c6fVGy6cq z6c&TRMJsCRyU>J4+4_G#b}mQ7d9MF24p3U8qV@Xs^65#pr;+aJP(vB#9Rc7LenoH( zsus-KLM~Z-e?QUv`)=+{oK**7W=_t8IEKrAs2~W}V{K9$k=$o35$ortRki<$tZ_D& za-YI-TTdZxbgVuYjtY{BO8>pZh5Sz8$9)=2dWByg9;@q|h5C-@czk{tt*eI}*(XLz zmZ7A$=0NqSKQd&$AbX?ES|?G#!;MHqjQ!oO$9dW3@~m7ntWCm)w2yRcvg@l1Tsnp< zOV^W=r$>o!a+b~hv{Vw)No^a>VJ4Rh9!$^rmpiQovG*cP1@e~d^6ZP6*TvPdA~wv! zW~=F=AFdSpyRJ{E%5tfXPKZ@{hdE91?(Z;SeY!t5 z({J28*BBK+ddhrKjwA5P*elcMq%W2K`H!JAeyi7NOb4g)G|G6q(!mNXO5DMMP|5b- zMZa~=x8!HVUQ*aNH_#>R6f`_+6!WTJSVVgOnX*$;aY#NIL<<(tx$j+nl`Hnn8|O12 zwvO^`WvHk$xPHUlq_4yhpN{|HVIb|%Gv<(ljNjHA>AfxnPbb#C%+80$I|)iTOkei# zT3;q2Lk&^yx!zQ@AX2_$JYP&aBkL|M##RALz508PS0C0v+!U`d23 z3E%#Ls%vmvf>A7^tINnV33myY+3{WR{oIT&D&pJ^DE<7xhjYMq1C5X^V8j&eisrU5 zupURB%YlIvEgoRFp@*DOT*4R)iD%D3pf}C)Tm(Ik04-nbB&DMZqCZ=fL3>QgGC zF#U#YpFh6_gH04jOp2J86U@iHK0edHu>68xWq?jo_=)qgc+H16uvbFc3?4(|{nm5I z8e(`fXd?;e(`Ljkj)|eLhtLhCT5DIAw2DgD6GK26&`b8=ge@_+qp(B{M)c8Phk#aO zW!YigB9Uy(mWS^=TrFUnKzkmK7;25X>Uc5AX}x$DS|{lMoVf(rfwc7W`sw%?$p5$o1I2D}q`pL=Rwe?H;^9fA#n8y4v4^oN*~-EI@W_1V!N` zu~jWZF0TyKg}x9vD3Uiq9lH4Jr@SA@$f&1qmA->+?3krOKxhJtJ!}KjvZW0TZ`B;l z;zL>T1zIHK;@Uzl0j!KJKv7NyqsEY}k!JgbCk!rKG{O!6e~AIZ_8)@PHA_Np4T6gB z=#_(6kolPKCJ)uo`Rb%P^?A>>U;3E&NUE=H$v_xIx{K}(2S?aGv{JJ zsi3<}=%5fB8q{hVH615?!B+q{iTCa`Nu#j8phoNw&SKu%L|kEqj$v8t>ah<;N8Q%5 z4{u$^TR&{;3tm?&j@4%(C1mDD)thYvaQwJ-ML`nF38c_TfBjsKv&4vSrGizo3r#M|8oUkWI* z979^l`wXD9WE-uJzq8%I+U2m3pz_Xn6&`s@JdCM#^__ zh4vdGO)P2ahO&|Id3l;42T&d`v+>5fM5qe&h7UC01U2UE?dDBm#o7#luc^i7_J>uN z7~YMFwS3!tt7<{!cd~GFBxBsGG@86y4%eh@e(o>sS&|`SWv!U_uZ{!NQ(8of8>i9( zo4$CY8BB!pgIauKH^df>=;0)`F%jYojFWMW(4-;#i(I96C4yW>Fb5IWR@42YAWlRM z!nwwtmt8bIn~BtXV4QgFpfV`PcC^VpCo)`HG#c(c)_Ge=x$n9uP8r=aW&4f5ai+=gGuEl!VDg+ALO+V z)2jLT>kEdFPPc8+FVTzHTwjJB^6e;%8{R1n!Aw90hMtIVd-@K{1&)AgK0v+VjKStS>FMV z_5Qbi8yO{|lx(G_tcbEnq9LOcA(XQB$Q~IXD=niC5|Wi287b-@B)g0xBr6%2|Lfa1 zzw`W_=kIl&j-xtm_x%~~_w~MpMohHltzd5XP`fIj&Ie}eYrAAu3lCL>0HhbGELFDa zi(sda@HHyc41*l@v|Z(z53A40(5%~dHhcIoj|q^Gxx8YoNKar84xJbKzZk8*s3{F* z6%fc`Ex*QjBs+T5Toz^!rS_K&r=$kO7wQh{oOP6w% zZnlX6GW1!sviTIdzD96`y#wOoZ3AZaG4D61&<&&Jl8pRZ9u;3G=rm;C74NA@>MBSl zCHd$*I&9~;%rjZJyqBI^_@Kq-Bko6{42v7txwx)L z|995yFDY0R^xpPls zRrK|*Q9lp(#xYd97S40Yy8+n^j5{WZmopM7;hSF285 zKa-;bcbN6%G5699l88QMx4lk3(Cd(VYoBO~1m%fyhuLoSwCLIVKBBRuc`@EgVNU3&yM{gU)!`=1{+ z%OAKre(F@KVSm{7AA^y@Ks=+pD>!OSRSdEW(qBKwZ7p!E5BZ`0BhX71ts##iN2-*^tRFnP(V9&Aol+~F@@WCtdM{2)tRU3J78j1;JiA3vHe zlx6I7m0ikM8#s9fvAP3rQFtM3L%oI&E|Axa?A8q9wiJ0ge*E=Lt5-lZ#+t8o;Ip(g zX?MWM<5c(F+HhBLIey7!GHwp4GG0uQiz;J)zA2&Rp$>evaSI>AMzda+ok0X4%4KH@?IBdPEyF7jmymyflDD*S9gT3VXVcOOlC~1jvl#gTg`Ro z0ccL0#C*fa@xVz`9Y8_ABe4B+T^IZZ7x>!5y%}VdG4wj6smRK~;O*ZO*~b(f|1F z4=bttY&*X$pM-t&%a@$69i91aWx?N$U>{csiX@DT=k=SOp3Axw6GIJ#m3st|M!>KK zg~D`Gr?u@(h`CKopO`|rz6nfN1Z)%}4)>4wb6D85PxBIyGuq z4)z9!Av^*Cr>2YzQfGd9DXXgT9y+ug4rT|>R1vz2sw7rFUkDAI-xuY(o2dkPp|j=_vCBb0e%j+ng7^3IFUn*PZRV?s=NL=<16t$ zVuG#f1GNlIZ6%P`zTABE>czJWRvVCpSqO$l?j@V1>zd|aIXOFV6pQ~hf^C7NrH7`0 z?22%A%Zo4eU6S_3o6hHaib_eL&(OA98Pl);HpJdlD#{4EJbR}CGVo|!7Z{&DeG&bj z_vWu(d~dJRHm4l6*%CMGss7Q~b|mi0kVzDRc81=|FuiP5C@s*X4cj1LCnk>r4q>N#d6E6Ri7nME1HVsj#)oRUE zjCjnl$u=pyyVL5V^mJA)eNAf{{h@i;I-Xw&Qh6>)a}S;hRfVXYj-qKArb}emHe%dPE_8s4M?u zV4zoE`P6n3@F+6c6>V(=AZu39>Ac0a4ot0L<+*$3P;2To0@$d<#qnBML0P%9rm1N= zVg(>~#`F~ltrALUAGnNtmV2)*4l<;={Po2_&{eK{G3WPQwVY#~HVkVn{Y-(`Gr_)h z8D%TC^^AK)`xdit&h^NbAt0f6y~j;qs7WD8)%%i6{K*@>Q7rOrk4=v4SAMpM@K|Pc z+{PBxOZ*L7K22U+HR`3ciWQG|eNP2Dew?}ShN^Pu;`hMH*`J;)JcE4)5}a{-hfQrf zvXs1qr0983WG#PIU)h!>P@|N+RF)p>7)_s-wD;ngNF-GT{lJ``ZH*>$^*1Ay4fO*S zeC^3U$V+`avtfR1g`AwlUGd=n#r3SsRGwEe{<2h=8TQCfolrms1Fz|+>C&t9b}YP5 zs6TyvW5^dHv99HUP1CeaL6S8r>WQ_h-7*iBG#%5(BHlC}dWf5avE-cZ@AmF3SGY~; zj*Qw)*7q)OiH>K2@xE#ULzC4uiNIKUkGc zxwGsSiw6No!PdPb9vEJe)JD>)PO9poCx_-C=w`u$1pBk0-LLZE{ET0IMk0P%gclAkgU29!Vonwxwg}s+< zRUbDdrW7Rgq@23R&&g-^x5Wlh!t*fSe&#Y~vCT3S;X4MuZ`jWk7SbW)XQRQ(o{V?R zm&v9o%ef{emzD!pf0*SXe@C`SuCqbJ2tie8rQyzrQkyaSPcb_!rIJ+iZu_a$x;(8B_)G_f$F2MQbPtn^zAcI zx2N9kM91dwVI1}f+mUno>c(cHFLe=vLqptax2P~RR9Lmx!#j0KM5yMoD>ns+j(&fh z!%)oelUpuL==qjHwp4KTrd21K!!L7$V{Lf$tZQ*DH|9%8)l17W7Zcr0nZ(56XU5Wl zNPdy@`;c%YnCSkEBTz~I?ePA>qRo~+`T3RxL0FweEk{PzPtE@GLBHTpEiD;(E)IEq z?`0dQNY1S4cnKff%1`WledN=>&yS8pCu8pC?w(zGEnCZ}snEgX(TRQ&X>gNfBpn}f zSW19@Fav{DV!(*!N#JsFa$;AmN(}Yd35Yk-0al@e#o)HotA`IS4BgYo8Nfe0Vi4Be zciXx9H7h_oY<+KUk=u91Z#{e#@f*`lV*dE+J}j4~2LdiFE>t;@NFYxf$uRr1?lmJ!Kq4{Uaw~thN!Rmg@K&(Q0i@QYHEN$aG|Hs&2V;&5xX#y07%Ah(2)m!sUFM-otx^IO#_$OHfV;TC>)xu>L~dtilo&qW^ham!Lq5$Ej0XKNn&qI_eUC!Lt34u+Dst*l0LEqIA}RBgK3a=tC#&{k;&3R1&y zQqWl8Pzfz#Y0toiM6c87({*32B^b*-`J$&kn7?3Ki$TXxh?_~IAhn<`4Rd=d4eYMi z*7jz&eypS?^)#23*# zbQv{P>9vV?zHAH=kplb%W5PVwsR^Q^DlDe&v@$>bPLi8U(Ydh2m{S#o>x2*oe&Spo z`)xu7k32ff;MmbGQeWmIA?f;lTJfcp)}*AnBQ+pED$T87Z!)slsItI1ssZo>Y>DUW zQ>}J^qgSKoP~tk%XVTqOev)$PiIUFpEk66i0txmta$$cCH^%N+08LL=jX6k&vI9RA zrr^N{YqR`t@8lkO4oN@$L^pDJdU`$~&tG;MVEAVPlZQ(;>?B|SvDw1HBD`>5V8DX) zN&)^8^{L5YhcW|#`D!5S`mn5#7s}Pl%tl2=xW*C#RV42|ap^XJ#DqWw;jb&51Q+$o zgbQw^JRcFI8_dA1;_6!R-Yh34rz_ut8k{IW&4NrKQw26UtP}-NrdC#AMD>f9zc!C- zy|ZTn;BYreMg(7NO zI0%AY3(tAA3_9@N{`TF2X?;z1w|l(hDH_zm@GS>J_}?rWg4o>Gms7SDyB``JS65d+ zq;(3{6%_*()NTP2-Y%r~x|F5DVTcMZ-V!h zav-g*+D+v{OAD#YKt3z+SmSm?3v$p0i)wm;rUeePkgkB3PK0;yH;|E&;~RY>BpMe) zWw{s<6B7v^qh!s!x00HzT|tjN)@gmm86FPs9_l}livTwDYz=e;H*Qc}F*W@{Xntnn z;87tT5*BvU&22HSZQSR_ zXZFdjuQ(;RUdDivL3X-2yh>zGpl%}JIt9lFSR3w=lj8)D00BWT;)^~$-E_Er;8oBj zUC6uOAyqvT(XSHw@40g2`4ur%LVShc?S~I%=;-O$l4xtazo9(=109^#cit{EJDnQ- zj7wElC}3@vUsL8(4hK(u35p98wX4qK-#u7e)`qxs27*|=IP`nhON`ZZ+{0K2Dlj}l zdye|Z%?k=s6ZZxiL{4Euo2D{uW`6N>$=Ij&2ZxJ$$sHy?30wECWj|pkRu=P%QmC}Z zmtOsLxIt6q)#tz%Tq%Qy1Z)WJpW}3O8-Te)+e^~{ z`@#K@{&v5%Jbu?Ld7&%q_ul!*4SSEO!x>%vaSD=#LBH zB`GN`9zCw0EjB5O>TR*6yab*@UiDF-VjWZQX371HFI?7;8(bse6CGY$d<2-|`%?GX z+KU+Dt55&F5&X!4s~}2H@9U*A7undCr+<$eqnf3mG(hmGz)ULlo6*&WW;JfC^Ern~ zr*f^P$C^bH5uAnKmahq!9S=I;Tgz*g8|?XLrZ~FeTCKanYPY*MGyT*oporsZS5}4` z6bl&{R^*sEiib+9&g_eG*nWj#qch#F$UFK~HMXB!pYYoHUK3>t>yenzMQQjv@;>jl zd6~K)wX44D;x}pLwD5j2U2fFevMU`PKRl1H8>LB?Z-8ar(EEZbF*{sWKG@6h(8ojs zujSJ(8>wC5pkUwFE75B#n=Ih%(43?b(A3&rub2HH!mhPV`VoKnrTsw`4q~P7Sn`x( zf-Nr9XCVs0PVctB(sGDXsr9Gj&UVH(Y<2-JfNsLw2xBN#O&^^YTpWexSz_1r*lO|4 z*r*8hnO5##78<^2&gx_%dmrG9jL~UD&4*+8bW74yty*>~3krsbsNbX_YvJk+O9W0!&yZ(zfnoWRYD3 zW?Q-31}K^PvmZRdb8D6TLi;Fu4 z+&^&`7+BjAckH#^82Cj{kqFach<1S7owxcq(Z_icrV`WdWx7c>h6EENwceDDu%QX8 zS)Q^}RP-Bb&VamCe$5SjYWf&lT+rMfyu1P4K;jajxko1}^}`tyKD{liJ8+Gg5cT=4 zZ*NSR3I{;Si7F9#t&!%=FDvdtBl}PFZ4$cp{g3U|NaCs1#}{8{N1cjsUa)|5zQW2* zvW3qjB#|nwt4a+7JsWiE8regK&UckBTocs)CTE+du=*4>0gJZ1 z^BhT-1uHFA_4r#=z8GKNkT2j?%gK3$RJ*+NK4GV+z=MC!qCYjx>ko7#^t}I;JO6@| zwoLi?iEo>{X#l;3h*a)HyeoT8Z<@W!@X_6lujeObXIhR9+Pjtn0o^uB*FKn=uO7B( zdnt}vRZQUz&&>B828Tyd_n2-h(kztS2Cg4aR^3$JG%_{i_vuqGLRn99C>VyWtQ>N7 zJ}xTxmgAF$HP3^ajiycgf*tlPAyX8Gf+LpKim5VNgGzsa0TDe)wMYa^Ev2XCFABHG zgh`k7ehFUh4PPl}FRdscZ4R@9I>DoE{p*+EJ(@YUpiU#8YGej>z4uZO+ZKn>W89+%4L?H%bWK zKb#tIH7s6U>h3pIyN7C;Beymt9Id@RD=Aq0B8vK&v7~lWyI&zU-urn3 zBr!WbH+KnPIBvgMRU_zofLf&=d}IzA`fjQl+6W_d`Od7J>hGK1ZH*%C-H!n4zlI$- zq@8IT%N|Bo!g~pVs5qAkCZ9_k0%l~$Z$!1pT>F6}yp}>?PeH^7<%N^tG77>sX{!#V>| zUD9*@8DI9JM{%zp>A#G65prAV#w78lYvV?>lv)6Fw zn4V={cbC~YG&X5Ub?+KH$JCB}^CM_yg;iDS&)iL=AB?d-7X90Gr1z_%l(3O0{mlbE zWiJcgdUxe*L&H&1Q$9VtI__91QmKLPu3ftb&9sO}8}-zI|1;*@{(>R@QC*!1#_X}t zhkhD1;kzpp?K&$eDH(zw<&@0K&}dsE1yFN)wd|_$#grGga~?7miD+k(yCD7@wac9( z833jw=H+-d^I&F74B6q0wLL@he@>r0Fsu2;dIWDnvSw_DYhzCj8{RqCJ*wE-PZt*d z<_~P-YGYj{<$0k_a&#?uGczPZjB{atrL3pN8~=dX*ve`HeA;1qU^PnDN))}qGM>fQ;Khg~E@9YM!wV9n9IY!C4Rr1E)jqEo*v65+iyiLGM z>-@YVp@D%1B=;6M(OoKmfs%yYG|!xEh-5Em?KSfzfj}8C{2w=O*-A|eSNd0NrJ#s$ zJkx-%v>Hy=#@ZMo`h|Ce&o7g`4SbSG*H3TVtY|N*JV3MBQhzzMlz|kxyeGJ!4poY3 zO%NNZ78R2d)Jr|oJK1lshi4wxO8MSvr9b^a^tFr~F$>!tm6sXW$s=v_S*Zd?!<0TL z%VM-mew)eiMd~gmrXSTu^LQEPE8?=p3!1^rP&$ms_`bLAqDQ@K&FbI4U^1wGU7qaF zp}J)&qYB-0A~k_Rzl4gt(HECUOOJF2?r`n>t1>Z|{PdPI*Mr8-0fVxk`x1xb->YqX z(A%4-&J?zx#`nL8Z;F1e(lyB3}rYp_@k--KNY2;T)v3gq4R()L*U$KJn~vq>*PdW z%vOw#o4I;!9GvP?oXk84h|Od6_pwrw&v&#K`4{80g^j9fQ>UyIC&x}2d^QT~sW>0- zn<6EU2xnEceds;$P1gV6^pLmfm(l**p>JiDFC7g0zsNd_Y&ta{+T8usF{ZzO^o5f9 zp{EEn+SfT-6Jxikg)n>79||9XQ@}i8&9!%2e;+<-vF79AhwAJ?1PQg?b}Zd*9F1rb&`P>I%F)Y3_?G z)NrraZ(g3?mYijE+)jS`CfK8V84Ru+oDhor_*})w@ypPU?u%W8W2YXcZ7#|8oG7A$ zbgg1WC55ja{P&|*9?KYd&pqw4GNca6ks-Gl^48H?w3bM^ua{wgWXG4ugX1) z|B>a#_RS@OGc)=1AeG~+57I5~;I4i>HH2~tNYb{_^|cL)L)>lC6D7kQ3t>#)#Q}r~ z=9Kwn5=`rm@uW?}Cx1sx&3nYL-*qriL~RvRwQ_Y2;+9oiTuS<6Q*C;7zrqN53%|Pd z;L@CFfM_;hsSkF&&q6sRB^6sF!yu$|Svpg-mEOlBy-MkTd|?jyx|(uI3b9+Y5$p_K z;M*Am#(pw~)8@LtJn(8}r{{aE21iDW3oFL@AeZ~?$o;9}Hx1_Cn*hHnjV#60Ub@uA z$JFO;1}+ajEXxii|LpfV$aFb>lh5n}?UxJ9K-*;3S1zx7=-@tI<mv~yB-(uJqutvYnn&a7^u%j}}&GNd>0^{l5cMSB8 zmo@=Ff62p|$c?H43Zh6{vkwRMK1>MEzLJC(fLuBT>2ISJ!6TL3^jWO*URNv%TzZSz z=O5)~aC#@4p__YWe6{I%M1aJn70v-@KiHSMyUX0o6(fNkJ0D(1t+Jv4+q5{cc5a^5 zX+v}1z&Vq5X&XO&J=xO!F=|bC-^cwDZp8+Y9$iX%9sTwF0#0!DuUF0q6#BB^dqu2^4NzIC$b zhzNihF5eXgHxG5Wy-nsA3{F+d(o<0tUMGhZd-_+KMnJ9TO0TUIyP}T|`+k9P%*o;L zx#!1^Qo@)Gqm0vxbz>xwv9Dwa-km7+kMJW=&BLngJz=!)HMv=%cj!j!> zjpUNM`DefhR789tAdhAH_Ad=0Rj<3gCAs#w^AlgqZs5v-oCsDcQ7K3j-OB+SX`pdhA5JCk`L3@Sf-8$n$GB~ul|{rywu z01tXep~v=W&>pP3O<>_c80O~b$^O!ZKM;9I7^qfNasTmC`{S#2<|_%Y zx7Re;(C^noc@!IKZX7w4tD&%ZcLq>n%e9nq0J-Em=gptkSX)o8%r@)iC>4C{>i_&7 zKj}P`g3hY-RlCAsOX|u;{RtNHQ~gl@S73XnNaAw;%G7YqoW9&+ZvXp4xrXDOa40~S zL-&yn;Q;W6Kx8-)h4(2CBVl1-dcZ;3IrCMQ zLsZYF^UkN1lFvLH1qYodYnUg~D;# zx-8GN=z95ECc~Y=_j9;NHeL@im*#SF2cmq{C`i`5+rw*vOHP+Hwqj-PUR$UlCsluM zuRN|^sXBYXCUy?Ke)^q8j|DE-zjMA zI6RiBnQ<$zFA(}CfbfdKIyJ4WY?!=+gwFT{2M>R__ip$k{^CxN+-(U7N9`S---wCf zL>+QHeWbB{9D)V3frGl+h(uu&9#Hjt^}HM7nw)K;=_E2Zd^LE(CIi^^S$-(`OYKbN zjVWK~JX-poMKUg}ba3E0`(ofwd&V#AtQJv?p<49{^l$!XY5)7 z{02gnlFHwa-%?-CC38*fC2wV>8;5CL{LOGv66rzhD}Whh&$tR`A2caxKI8YlEZ1aM zoN$o)lGRF+;iPj`qjks1%lVnTx-*r&WCCX6#ZCwt?iHt0eqh^CY!I^5t4WV9#x7`J zPv6=7spV8Fa9>KQjm2OnT8ie}4g z+G)+db3aM@u5(Rvy-I0;M91ecieQChO88sHBU|!VsoTsh%p$T%I52#GBa4b2ycj9T zulKENP1D?K4^5UzydqCEj{C+lYlN!06e9!P(|ehBVM_6%fh`-%9U_FyD40K?U|Vp0qzC+G1KCt6wUUBnUBON>hpRc~BirqjYFCWQ>6 zkkHUe9Bl;!1(1p<2Fyk4zaJRj0V_y+UBzfQ79qx)z{>(M0wZne7@3zxdTDmPK0`%K z-TEiw;nm_{NlEkKtH{Gcwy)S4A`NQ(x|t}Z`{Cv`fm`?PZ3wz7X`Bk20UYaMyM9{& z1el+nr=h1;xo{!wPg)TGfqw=ie(1TIoDoyF!RlbSl(e+i6QnN+X>S$ljQH#d_ff&K z(HCst^M<&(@IoXgHE5gCkByI;JY*o_`KRyyTN?h?aTsEAk&}}PYV+3q>z8KXOwxc7 z40LHCK$?)B<3z?N}d zbgI?Yd8n1rQBzwH*G7TfLDL*Nv+ZF)UAh{t)sKdUyfZGip!>3i$T^EE`Xr$9UCZFv z#MaWU`RxPQadsV78xKLY4S~4O#mxm4tlBA8?MXWE)Z(i-wHKca{*|JUJ%^A^6K*d%cSZ2 z^Yl#!lp8%c%QgNh|BJ&D4vj0aLiYfz=ap?;nOJ`2#?ctI@JhDDb9(IS z4e62`3uXXYfgdU^f8#Jx-KYSaA0_!`nN>> zeJ7CjYH9TEyY{F3xuw79m=W?a|4&n-q%t-YGnifST)a&{XQ}#qq*)CcYrTn;%$lpj zTym<9n{h(MV;-7g>f4`gi?abWg+vmroDX?@>~K2_7G7{@d|Oec4o4p6eK^ueuUl=e^Gk&+wkxuV`IPQVj{n@ zeFBaaqd$IJd2EBIn2wGPQ++B@VMz(dQYPWzNGnT_arJK)j(53`3Pv7be+3WP@L$N% zHpZ~Kt=PaD0iT4m!|}8;DlZ};HWP%##pUH@bMHFddnX+ro%uR4GVWDpT0mr$qB;y{ zN?daak|NGwgqQCX5$Rl(imI>0K}l>n7<_1)9Yl!QITl3oFgplCr6(5O*3%^4InCAZTR87lML` z?|qvpI4KbG+CRq1pfMsOloo74xI4~oqN96_xwUv3qGb@u?yT@M7y&pYpg=b4n&WYL zU?8mO>#M-qgi{ah$e%4OEqAm*4P|I3UK$_@o+iRr<1O%#fM8RzRVhP}ZfBRl&T>^6{v z>+%yycxGniN5E@w4G?@sLRAMcuqZfpDz(QPb?}eg5FY7pth>R(t1b4W5j- zeiozAU>681)8%dKX7gtaaX7$=AVSNvwOZ~ire42Ohla2Vz;rRg^XT_cpX#PA4y z@Pkvne;Xj2>i)Vs62S12C)&K1!LEndyzDR7tcwRJ1tE8YL_+Wk_j>88R}%lnD?n7? zX3p{d{9ym?^so1I&b4pQ*B6$RrDIUreQ!S)Li}16Kc}8RHbKT%PT$=NPKBK70%8xy zNY(<|I}})LlG&oWpA_UXBy9F3`MGGQ1ei$31h6t)roYEpTYsd4MEx3sX5NOc4zXmq zM06_XqIJe*NE=edwGP*|reXse?~fnv_QL2PUL0#0kgy4g8&m!F%A5C{+dFYP`H>Ti z#7oDFh-Rg48HZT?TG9qWhmvixgP&J2KYel%&R-?=*RI{j&CTV@#RaUV$7Zrdbh>VC zv1!6Uv`Za_H}W<$H#6WsFx}@11iw1@XfRSmc8uZnOzb$)xn~P4feb(suHs>BEr?1- z#-^b?9SZ^c4JadlvPnt7w`3~#!GqrL4l`LKHo-V7caHw~1T1-QYSq@8*=~r1N*^Ui z&D?k=ytRLQL&j4bku+jwry%4qIyF^~0|9oXnFS2tdHMN79MYGeAz1UIr*Xah35Em_ z1PFhf(|7Jacz}TVzza?sJg=Egrfosy17TQ)U?mKZ>g4Jo>-+k+utxt*{;;>VkA2bu z^;#nc!$<`QR^D}8dDjt)cEIzHmXT>Hrjt)%z|7)ren*o6xFH2X0gd5-ft0wy#U^8r zA(d~{DU5AxyOEamIu16}JrYE2wQhlF#PEIzY8-M0pQz@{{wXz5yBk_tqeb!3+9byV zNa*)DN-;L3bMazR?ELI3MR0At+SP9~n4>_9rlO~-y94AJp|iLc$FJ4STKJ{Jpj=iJfE23l}j;Ho;Mll`CT54g`iy8{f(EEW2{Ho-B zifAtg;^13A_yyqT$?N_&fBQe{|92(Kc@^=bVA(*}G_)m|$bz7P*$fVfSd>o4Jb6;~ z1aQFU`6zE=bqWec^hPkjbnAcFX;>a9QYF}~*6-R+PD5ES_Ka&MAM2QFf>!@^|C>Hn zE{?T$6<nIA~xzj0|q39Gq&ceO~I{q=#+e#1J^1d}Dx~hSQM@H(%nNz)-QWgE_ zn^{xxY61>$3u`ouKwaM~PWjHF-wFW*4S|b7K_lx03+YEr4==BcuD@4~t-Twfw$tx5 z>lMwwYt`M89h1cq6CH{3Ql3+1OTMpr{_?c9Yp?p=A-B*vwJ>o0v;Mm3!pi*bi2_f1 zD~4-Xa*{4#TWL0u-&9hhy~KBEqsiOCHHIbUYa#&~*>q_ujLz-5m{fFoN&Rd-2W^|?ftwnkb-v{b)R`uXLN`;_N30~5 zd@GAHOxCU`(2+IORKE|GytcE@xmtB7Gm2zdNjLJ-foyKg$2KmPVD4IwlI465Y~ zgI&UifyL(Mi6e~m%kwq{WsYy`E~2T3mS_1tkl9CQAkLnhZ47O%ef262 zF%SfAN>Ook**!FPq=Ic7XT=FrNb0|dP%x85}DA>Ov%U$=HmT+@=1m*;{9i$lvKd&pjK z{MZ$99r`HtFcp!&iES!p_YvUYL9_P*sI9tQxCkH^0yo6&<7)pb{Y^Gku25o`9f(sy zdHQg6?kvit18fn*V+Wn)eyQQZOmh8al9qXH)zv+G+q_OmO}_)`1*<$0Rp}2OE_;oq zr(Xo@LcPAQelaYJMaF9`FwlO5dtEh?GBk8;uyZ-*!3|D(cd`dp+0xgl4iCDl|77Le zuO1pw+eL?wpns@ANwt@W?F+Ep7SDfMEO#WXm$(Q9Q!nnTjquQirbwi2^^ zzQT}fZjNFxX?>P#ZMC1fdidv3;==lh!Uv;qHRWUr({C|J$4HFZ(q7^Gn7Dh05%=_e zK3d76*Y=BJhW%e3DaQ`TO<>B~6IiuYL`%9AyW{$(`L??HgdMC717q&w2gNRfNS3>7 zYe{d~x?jCoNqKi#_k1m~SRk7$nTE&7lBI14JAp^64I8cPc@!bN@S{5bagAC54?(^h z(C}Sq^FMrV&TT{N5A{z4r&B#YynJM$Wqxa4?V|(Me)V}|z80fzDvM4&I~D#Km;bR| z>~n~?`FTu;*jlWY6g_C=5h2Wat8={_-nLpgI{Z&9iuABg4?G%L>3IE`lGyTUYHA_h z!E(UJ(3K}fS> z5h0v>SVRn!;X3|ooCs=!Ox+rp%tz*btEX>;bO(jQPpd1-0)UBNJed0IS$Kq8Ok76B zUhD4X8%k_H|7-_)bh^hX_tKl2+qUrWE>Rz0Jb=Ai(EYW{;w>;T+nt+Hr7jo>}ots>1| zh)qoV=IRm8QvO+3&u7W|SQWldS6%IwaX&6@A9go5r#yO*VPOG=xiNfdG>F+&LJD+7 z@wEv(_#mFTt;$Fl+hj^O&!(jf2Q|rVrlLSHo3S_>D5u-^4X>Qm&=@M8s@&QRZFV21 z=|0rAk0UY%!v%q-@2-f!3zk_g`+U6SuMYrSz=cXO_VO~&)<8&GJ-2UoY;3HV?41yE zACxZna6Go|E#8cFkMIci`t@=DHC|rc#hH3;2joUt_f@ds>{^jO{r9$K{z7>~5#LFT zZz~?p&W}G#f)4qTI2#_(@$r8-Q7DVsopPTHP~1v6@M^>eLs2!|n zX%Uy@nlU=>R=Oq0$Ab|Djb~n`U7Z|T&RO@ZQlKWqayh-SP4FLBtzUNOUr#4}KYmYJ z%OYCxCHZ%1&J1p69-)(=a@N@pi$(bqt$No0yw>bWAXCi4XX$ zHLvQHiHY+ie4!LnR{r48;o}Ay%B~9cgOi`ihO$r+BaqeyCU8}pd7);+%10g!aa-z) zPfQ&1_LkeYaU;1{UE2?J1~zI=^Bs=Hf32=i`4@VXnG`MNd$Q?xrXNT?NxqB9Z;Y0t z2!f8{8iU(QL)cj#(I(=MK)yBnUF2)!=C%XuB9?aS0%8D|mzU=kIhQpyW$qYpW@oTF63 zFDuJId`pkHr@0+HJvYITdYqPaB3AGhfl){&sDJ$R6sF)*6#U)ZiSU$yh1cSJnEHTd zR%+yN5-ArTN(c5kA9&t7VV$8qD<&|WLj8&6)~!~m#~>!kNMFc-FJr?4Rz z+lh%`SC?++79n`$n-JWjPqq$iH@%nVaZanIX#aYSyjtS=uA0@BPN3)ZS=^bDdUWW8 z{H~2cSdhePOq^T5Y0>asLadBkd1mH5!u%BG;e$UUXWq+gU$RfbCA$0P@;TQb%Oqq9gDu{Bp zpBQWj6v`bpkW!#$iY_F7$$h`!>SF&|i4d9hX+5^YwZ3ut($=n-5*`iH&Rudv3$9B*{K?!WfP85c*pg6WF_<5{iq{?-!@qK)N5*rj=o8dg@*I2` z#+(VWTeNhH?S#e|?jlFi&Ir9N)p+pm;p?%s^w!UqINPEOgm>zl3eg{pjdVnMDc(`IX`wefXq+VA~4>+F{gkROuk0PBe#akq~ zWq?qUVx*+q|zC6Y-sT*UCSS zj*ZP^iMZm-LE2Bs)GE$UoVbPWqiz@exK?g{A+`1R-QA@Cz*`-GsoXu;*Rw9mcZZff z_^5V6L&KfDr$^&sW2*s~Di@&@QOar;{%g^n=MkKaaR0e2PVf9=yWmu!fe0_S`G6b1 za;E&#-C&s3Oos`;qsYL;yI>DBb+aK`49wUd5xw zAlAW9-dHTIrK97UtMA(axb)!}RGW)mMvV`jW4(dL%7q4kb7c(VC$b`xHJaD`rz$=D z^-Er~eiRSP$q@~YkQ$t!2?;yPDpn@h7QfnHC!Uy7o>lBZBC};vdYsv>{yImy z*TZw$#=ge;D}8FW-`~|&Y_Ar#Wl%G@$49qEVAriJ+oU_2DmK24I;y-od!#AqDD(05 zc7FRof7Z?%m*nJ{8QQ!NYDJMX?~K5M+;di7A{WbCVsZA>lo8HOHP-r&qc z(HCNgBNm7B^^kEVkbMz<8)k_X&>)ZM4t5VwhXck@oS+itQW3(>V3$IKmB33qODU-1xwbt^@ud{)ug4fla3H z8475(oRdqmtytTil9KYMZA_T~7ip0mXP&6}*d}D7U%0@UV^H=b3o(F{I08IYf7Ca_ zW%7_Uk!9hPKIRsNkKbu`F7$#2*=L0m#&s!M z3uIRlL&L)E>{iVy&=pL%x+dtAj{~gL5i}Tpi+fIPePP>IL0i&0p@09~6?E10{!!!3 z(lY}!6ex(75_H_-;Vg;~F`-|&CvY@3x(_%(<@%bZ)@cNQz^>Ywi;oOY1YE)}F1-0Y zE-sD$D1O^*xeS|YVr_)ImmaUbUfjzs(eahsKLRtjea5I9G2~`@VxXU*DG~|>0 zRZ+wRz|T*OINg@216V0f7!g555; zk4Z@9PceK-J@Qr3dIgj3+N?@uYRi^i!3xjnx!3Z144dPcy*t+@`nU~uatgVS`<*Y} znthG`SL;)q$|-4P!Oq_fAtC zZ#?lRb;#&(B%|2coKeH{|9403+PL%oFJW7=qTgK!IscdFxA}*Pw~5#1FAz78n$?VfRTGbfQX7h>%C3g|2Cs#ZI4lg~Lji_5P}A63T0&8xB7+0~T@`izZ_ z-_njCTA%ghe%#PbT8^?|o#RB|N#xxdrOx;FlsfPl`mVYl$%ghrQ$K<)g#lz?tbU<0 zI5^m3PL8!oTrPr{H0qLf2WDgEeTp*a5j-TU{yj3oKa zx8qc9X>TV;Wwdu%&kW9az!`(E!9eNi?BWs{Z{2(hera@60aG>Kz9}<50~QNh`cjb| z@hIpvF46<&v?WgzeH&pR`=62(wQE>LMuuO+kp_JS2T|Mp%81ZVS|fsk0FY+pcA}s{ zOA7i0%l7@?_b*xE`o@)VCot1L$uKt<8yHNw?P%jpzf!I_fs6kB#I5Vf#^6U%wg|F)}=S;x3XXKi&EQ7r;mDZ>-kkGMaRGq!VcM=kVu ziVTdLGr#15w`#UoVWBtXae6wEkhbo8+0T>Gf}kC*kyv=^6Y+0Po;!|QQah5j8RiCA+l~-)T|9bZb0U$C$?+c%4{_U zV~uvy`30^S!E5E5M0x`d#eTvH?{J?R1rF+4Jl!i8wkBgWVG2}mNSMM8fiVd8;YA4G zBa==3eh3?Ugx^Kf5mrCST~uUB9v&Ww_33G5$iU6W$*BV}kb7QB>w1*@((zZ?%ugwg zRD+BH=M{jgK>lW4ee2L5t_zsTSoZ(3{>8;9a}OV=cSrl= zEX|2Wp{2%-gCPg$dC~60?P{v_fUAcOd=-4o%PAw{GFZtBn9XdF83HgG@ zWA~R{g}4@u$(6(J->bu|1OyM-vm25SbJ@He}m4RFSHAZR2B=F*Iz1LIDR{F+Q_;|`ilFZZu= zIsX2!{B?DCA+@}modCS_0NMZY@UCX-cvD}WT3E=8`RK3qM$?1vb~NgGt;I+w%NAGD zRJku)S}K!8C?}bf!RNJnW?FUFe1}Wq#p=82K zm((kM2vT`s;h#T$z8VXEpP(RJ?#p>L;fpW6X2ms`VlD2VqoTc|m66vwftlsPOFKS3 zzHf70;FsY(!0+LKbtOORB*oNvc?%^O+W0@65cbI@TUW)pTMff#`u)|@XXOg@giz}w z{eF4VJQ@1BN6(*c!Jn?|!85sZV8>$R!(aqW;#@QK@aWNj^j_KBy$q-Q1I$r`(AZ(d z2!!Nx%0H()1`BDlN;@sAt?B%ELM)T*)okDSF=ThNWwbQhQ%lR(pkGe0Wg%F*y{>pGmD1CC|Nb7&1$uj%+;2LrHMYOjbTXJ=2ub$(Si zovaBnsZA=WOnBhj+4S%(7PS)RR5{w^WH6Bb_qi0WY`u*Saf02%# z#Ix&m-LJ<}B?Pe#b6G^dRbDuJ?!USWd+CJAB{hV+V!3l+7+p*1$M2srK z&t;u&>M1?NNLd(FX8+P|ONWJqq8Q`LXIkkjBSJ$N#ryLP?%8-_$G!eQcg)iiVe6VP z5A6E4#Ce}>3ah`iq9#371irp8Jg~^vhf0XcAIH#5OJHa4bI`v_)#J}Ifc~J=d%3v( z1>#4LZ|Mmx%BIAj3lG)#iLZnnd1L> zP(mVzK#EFAYON2MfTqxwOHls#sE}KnHL$lB{AQ zmIh|_e~;Drd3Ww4+ZeQq$AodiP26;Nx-+W_{mYhMy&%BmsI6^I;5(%-^l*rvA(~^O zY`}8>HNKktFhkR;bmq(!R8!s(VQ-%At=CaH0kx+&;CSuZ9L({FK2<18cgo5d#2+}) zNZ@mG9xs(oC>^!4%Qf%cOhS3{A&~YGrjUM}%TcDfRX%*4=dhjdwiAFQpcbS5N7t3W zL%p_b6j@sAl#&!V6-v@#i$;q?q!ihOWQ&w_L=;hqvL!O56g3r+vP&w7sicyS3dx@A zeAhEm=RKYG{ru*4zRvfaikatsKlgoY_x%7@R|VkgSP(Z)u%fdS`F2}rEh%i|NE}VR zOv{@sAyEbtrupzgodMLsg{rD%q`Z^HMX_+Gw7XFWlL9`oV@6N+9}JVbk739L|0*Xh zPtaX_u<9Ly-_h8OK|p%nr(c$rkB@MRp!CXT{o>VxX=u)@G5F(-$2;z==J{~)FT})Q zvH-J2a)Gh^h-0ygv@|X4=%D$I zY6JzgXtgJny4Bk?03JfKkS9S;u9Jo zY6k=rA*7w*4Ly{y3Q=?izdoNmeb%heo39*|*LZy9R#H-0v{^+H)Wv=OVI^QK_U3fs zC5F3p9mgd`2w+wNmr)h7;-$nL)mFc%03j80c?#Ih@nnJ4xpPxl#~y$&3iPt|M$$B~3(3jHnUi!Vdhu~8%C42I zqw*Y(N`Di|i)3Ut$Figu+Rv)1H$l*W9>PE8|hvKC8#pz6hUUr z>FS4Xpdc*E$QrF9Myyi8Chpw1s$+OL-p~46x64SG0hmI8Yz4jF->_Hk1u1UKUtl7F zp4#=PPX1KGr9sAN%$fLjrQ))NRZ!HcQ0uQ+4P=hq+91$cu|PU{qxPD?qF;0Q@Bfo} zI_u8A85{N_wxeO2=cw}bA56O`X+f_u8O7ZLRSjE#g=U8*;qvjsVHO&8Qn@e#sg1>G zCZG=2W^<=`hT-Y@_)sc1I`|OUNW&Z&QSrHZgR_KIaH50p;L5pb+tIxoximlH`pDUa$X3ShlpI_$+KtC%Xhk0 zK7TGe&#&|Q_mH}dFM}T&<2%Je8eC84UyG5N5t`UYZJkEl-?hoc+}xahYDw<0xXQD! zv9i{*rmu$|GG4!4L@1Zu-7nS5i>T&53U^#LCb{Qd$Jb$?haeN6E>4qAb8iwJR$&nl z>w#~z+vQ@ryq3;9QP%CR51kijWd^@V3r%Hrs7{8OJSE+Axwt#J&k4VfGvL=r8aoxj z?E`h6&`NIgP0^-og;2BtHx!9VM`^f>H`6}rotr8E3ot#(%F2LgX_=YRvGQDSku0kq z^WwaTA#IY?w1b07==a=VWCTVRgKX-ncJKUnl>tIvkL}#Oa6?cQ{YK5^wBH?B~Q>iuV&75>CVLgx=YWGx| zB~3V}(~8BfFs9tx8na+A`jQ0Rldo@N)>=;Y>Bf!TpFV*|TYb+S76 zC+;C{1BuzQOHf`{C@U|zQOU|!J-K_ZAPKwxcY($HnmZMbAG?o_c;Q`F%kpssQgqLy{&;XOhur!6qXGVUZdbFsBX_CWpn& z!~}_?d9*cnF?q)~KCABg2p|joqW)rE9DE9%BIoHet8A=9B+nyYfu&oFI6&&p_D#kh&%Q zq;_b7tC;)&IGmra7XC8iDt;p6Rjb0C%wV0V z8$1QRS6M}ccJJPs?);Kh0Gy!UF$BR2;db1!xlUxPIg%NO3h`l@ZWDoTezcuUkA=n% z-Q_G^PWnY&5r75+wH|vB;|iJrXq-xsq97w0I#jYl7o8|>8yb#(RaF&rRD-SXY6s3d zSq6+g{6iT-7+7A{V+Ae$w zs1x>}7IgbU(t}}V-$%8vd)RcjO715wl^z6t$!QFo6s? zyQ1%bt6kntMHmL|p3yg?&;mb%X?$|cfy-8yDlcz}`lJ{*%$KBpjXr^fbRh_Bsj4vm zbU~j!*?h?JoW>?)V0r3vw)AeUd5Ye$hVFuFa?>TG44xhzWw>0rCIXCj*6tlwQ>Y_6 zXk}^dA9!^@nk5b;%}prfk;)*3+zy&AkaXPQ92y^A5gdB3Xwv+3$Q-uP(o#Z&m#+^S;u3i`c*d;Es%hYgdJ9AeUZywS435Uf zi+8m+?uoAa@;1+NapAYmKJ?Zc_~}x4}lwlNfak(A>AZJp&kL`Hax$YmH0M47{AmViQF?f02&G52EXq8ss7W0YC(V)Ru3e&N zEEjFNv@etJEObjTqNpLd#jr5$a3hBQDrIE?%<)g{)p13oT63sv`}*pO0s@>7=gtX} zIZRwqaw9~}NGP^fzt#w7_|-Lo4H>mGAR<8Cj%t^DwPKyv>>rR$5uP1iMA!rx$lht^xz545hn6R)FC~Wq52UMVR zCZhCa0vF)$5Pvo4=gJyBLcEua2EuW#NhDb$B!W*7fs{au;5Gk;K_}b&&qB-yo9cUw!_#H$+1FP;a1h+E<;)bnSG2Qp zXMY$xa1d7kt$eVaJOh7x$sOVxthN60f}kin@FX1#4RbVGZ+c%w0xO zX%TuQry>(-JNj&M$WqH&*N5BM=AAot_|{Ei0nclGp0E;Swa-jULLve?Y>069jI%2UB%%~RivzXO zp4_J|UOWctl3MFDMdcQ_TA-jnYwTT2xyD|q-Gj@AAeN9UPf<}}L}XePNdlGy&D&oK z6ktNmtM%FVYh=MYjPM!@m-pJAN!} z8ygymx(V2Uw)l*4n;?5l2(IWG11Z6(d?i#!YajSOI)Ai z0Tw`$Ny&i-W_WyVTIeb~KPc8^ouaFcs{#$iAS9!!)9(e`35vQBdLfeX!Nea;sSS7B zLwqp+=@Ha=9O@uum03BB4MPwi?unA-WJJUrNLnx@ZaqU11N_h=BaW= z0Y`OoQquIx3~r$H2qu-(T!eCeG~uw{!OL#E1IMx&BQ^=wg}M0*aCmTr?XhfyK@f9Z5MYX0 z5GN-v7eRqy{V@-`?W>;Yw|?r|yq9N73X6+f&R$%gH7#+I!|kOqyd6TX`xE`&xAZpQ zl)A5ZB%}*!d_FXQ(;eRc&+^AsCh__6#cH$e-bF-W_j(1(`{5H{hyZ6FMC(!H1tDM2 zXhZj+{mYkda}b0xC2(TT>+C_Mra{6jpD+_T)hZ@H6itF`YIIza#B+lsCoCWk-{i@ zx%D7)Lzpk$IUj%Lyb)+%`j(-+=&8zB=N(uH(bHVYreuGBpWz?yWno~^l2aR}{8L$o zf|zc612k>+-r1)%qLaRqn_Fk{kUG9j#q|SsXmG2_SEfKGGE%s65bX)ED1nhuEU-{C zJ-lWv@#-KPjS|l6Zhk|Trc64vMSa}Dw2OEF)&eOa*p70^HfTeXmBXAM{MdkwUpYv7 zJ9KH_peZ@X7Gd;KAtJ+!N+62@%@kSOaCpQq=W5aG6i4k-qULLE3IW5YHBPap@H+41*v2I*iq=9(hRTof-2KO~o4ri;f9mkHaAo zKeZ9BXyH}*jU5!-~Z%|^wyuco|2tz ze_?0|f>MC3mMu>4mv(wN45g|R&}Dp@)M~@)#!QJ2mT+OR8~)Z9IEk=Oqp!8w?LV}a zl;7e}o9$WPZA+7}doSN~;ssB3xJL4*^qLWsZ&*Y~fTI}=Oq z1yS8-z7Unc^UIay(X7yV=wZVeEQN#cvnf{r`neM`=S5$cJvKD71~)+jGyz3k08Fv6 zmVynf+x7jq+0Z)xoV;t zv}T3s7FV=~kRogh3@oA_#yx6o$7DhmhPG0D3@pi#l=7l6r{ZPVN^n z!#X+L_#2v>uy%6dMY9SB!MpEscw1q5D-XDC2t-H&bJH{w<%=j~DU?bPpp##|?dyJM zu`Fr+i7>~5RVq-r{`nP}9NOWi_2+q~MCexX0IL_R&HDKemoP>Req61s89oIT{=;A5 zu-a-e5_+@Ax|dFX0Ns#vl2w?9F4@E~AXIa_-C+BYl)&q8;Q8hxml&wW#P@P54o;Ic z*=n#{5EDz=m5{RR+a1nE-pGdcb+Fwjrl!aqmiP0P5&Z%O-1SGD2SMV2z;C~ zui@a6(G?D^9<#;!UTNZ?#(Rv0<~_^ZHFs?=Jl{IitHdA<1+TIm%}BmQcV<11Mr$JK|%!~{jT9-gp}r?HA! z{8o_5FfCkQ`GW9V`xIbl({adMO03pnZUciV@kEfZ6~HB#Mr)l!)RPB&911bN9a?v> z8yym0HJm+>+bhBxyKuRmO-??I3@^)0q;X^2^9kLw=;eFb@T*NowQ%zRcWR$C8?rkB z%;22<@GQ5vV^tX|k6uGz&Z&*rYS5|Y){mFkGP3gl&YX;0HD^YV(8O^l1Q3^=ewECm zLSq;NDV2b!HX9faXUE9E&DL(pgCLWqw3j7Usjkk^W@575K= z3~Yb^?r^yJP!n;cMTNnuj+Fa%J+#nco#ul@|3YB0m3#F6DaR%wtcBGVZ{T*e->}gzF?(HvS z2-02sT&e3vlTF=JoMMZKS(NOM=iXDtYSxl*cI zZmQgK zaL5HI+f1%sh4Iou8UjZcGME=1YAn?hR*rG)a-BA|m z!=jITyK*`WUODCbI?%ZNy)|&2!$di8pan00GmY49W5WTWMytjpSWnV_P)JBXewD}U zOQ)LD8~#<9x3bfr68PiGRQjC9>}YA>+xnxc7f4CdiUl7Z@2x6#gBW-|Cb#MbRk%$^ zCM~cPCI*JSnf%C^RaStm!+;7h>6%Mg!U2R{_zyLo zAY~WXGaWGR6R3T~iUl_^jF*rSvOCAAHu%(dz8>h^lBT9hjSmRsf>1y73K7W%6d-AK z$8r)3qd*wfAuR0Y_Y$1mDN5E4aIFB)M?24`uOEHhi2^}dTO8k_zIY0s4#GOH1r>{R z6X+@!tH>>izEM!t>;dF`sNxLtgz)%4#?N&3SbuqT9)Bk&AgIjN884~jNRy$ZQGTN= zrA|4e_F2PJuZIonvo|QjLDs~>gcwCxSR;@?=lig0$U(wLD%ALZeFSCs z<^icg{1eR$6>&E1CLA}SLdG3J`vQ44Kzxu^`#Csgqs2uuA-aA`7nu5>wt{}z$|1L2 zrxFDpjSAD|gQS7HKKLYMm-x@+k{ z=-B`HWypTmsJ|lYZ+nerajgh~tF)?Hdr`1b033x04X*AIT$CqaA z%AijS_1ePs#)*d-bstDMv8fedV5M<&fw!u_br>Mjub)>i#83*VuzkTf^h!W}-bzV! z%|rUd%HJM!j5t!3d;uxibKpZBMjmj{Ypq&%gO3s`EKo0+FHG2QlpN!{XgAv1QbDeQ zrb8zOXW@!lgUFlA(TvbV+#EMDGI}9TEoFD+A5pan%X3 zYVXRGsp!_RMRn$rc(8cHQD(WarBN4Fb)W)7QqO;0JGjK^D7u{jy4%P6aImaeX+~S( zo%cgN1hpOsxqvpRxVSjqlALD@#3U}}`E$GeL(;yfEa40Tmt)m?;emZv3IaK`Rz@u- zAgyb!7OYqy0Nsq0E&o|%q<%qh8BmF1NpLsBL9c>uh6H{#^?j)XXu?L_&;7Hfc>}s; z-K1qh;HtGQGm2evN1aeO0p8$IKtIC;1~hl9zGx}!o!Yo0(c=SP^c{&$h9gyOOAjPx z!txIL_TjfeH<=ArZ8OAQ7hc`NUTxfqfDwatRc_SqJcy$UXf)gjkNlSjR{;>aBsGZI zdX8+a0Ye1*a4(v3sF(#%?x-DP!!;(~k#Yz&6!HFMO96$$EBv|I_h$(@wU3SO3i(A%cSf z_cs44eIbBQ6|8ttd65kwKMCqJISwHc)5T zk}^4*!sy`dF|SyIrDvQ&F|e3MRU{2X#Tp9Bb;3RW*ylfhY&(t~a-6 zyvpPP98^Gr4NXm#nBV6A((7H*&@h9EBu2tu>0Qu2x);4{1k`^lNhD^3W!p`qdA(4H zcVz%|6;ZiRH8|QxEehC)#(@bZLfB=zWM}bXDOyuB+H^Ee(EC-HWFY$ltW9ZI8S)kg ztO+zI0)?1~pOdD7*{Vwpb~g?(1@MrhpjD9pGB9Pzl)^8tptqN9-|Ri2PGdM;;Smgh zPm|boRsoK|M2eg0>yhVc6cr6<%Bh!G9gziPN1SMZh`0$@i?oJ@#cwD@_&aJjt3gNb(^vm(PVg1r z^B_KJ?u-oVTGM&Zd@ z3gySFr)QnF{mXroZQQfuUnmPrCPavXUqF6liEBTRGV$VCpFPo@GO_=SeG-vzqA4Sh zd(c~3j>FHq9#}AL;_>8|loTSM0qH#A6c8{zlzD^aO}sL9(0B{KJYS1~;w{z7mv-;u zHmFx;;{efunm?z8s||M>@YCbzK(7vh21QFT%{i$N92f?mehIOmG@dS4fIto0k+Rj? zii98x#j8861Oa?}1SJ2=xpUd+2^Q!bU}Gf}u-f^_MckZ3p2v(gc<-zN0qZ4}>}%1Y zOSnz}rybLi<$H3U60wb5M^t>T{U3nF#)%@tyeuCT@)6_=V;;l{-LgX50Sh{A2KHkm zO@2@YxCd%eq%k)`Cd53^wS%KL$wyCN5ftZdxUh9JKuR-0$gqu2p(t1Wl{o+SVezlm z4y9^urcyO_dN@*M;YYXjXxVpkh%l12-NmU--SE4oL-l9 zFFU)~P6_#TRxhfz>FI?q93LTI1knM!d3y_wZXr~W#q*`y_tlr5G7H#|-<{Yd6ZtTI z$nU~D`pw8ku<4zd*0ydoG#Mi!zI%(P^HA-oKj@3+sCyzjk5k`cYU@JQJn(nZV!Fgz zgzZoKg&|0b*seXZc&QgZhE=_pQVTWa#2RMM?Q`piJ**`K$DM!n?DoMVkm32mUoWt# z_kb@^x_v^{h7j1VvE+VGm!xh0*E_mS@J4Wfc-R)RGnzt-y9;7A^qfhHR*g2=-pp!urDl|q-O;fJx_{ads`?zGR@bL=`=gZ{Q z-V&5r2XjkP)00>#rj@#Vnd#L$z{thmM$z~>nA3M<5M&GcY^N<6MZOE#k*{^>y}8PB zW)Kw?Fk1bO`dhV8%*lmdmW-qcz%&5P6|8SGE#&4PFaX+#C`?-bSaj2ZnE=aCU@RUU zN)mqw2^_W5P@Oqr29x{*M8htz;pVOlzk8EmNkF9R634={Es^&_EaS}TjEfv16aZWy z;QR?n_FL2f&cz1E(BIuS+8%`OEcolMEs5*3WEB+jvz>G6iIkn-Z3=N)jiK%Yx2b{7 zh4uv;Rgc5_WmC0!RAUPuc0*hbn)ts!T*Fz7s$*ks|23~3WfO@ZNa^?SuPz8il^7bI zi|ibFW3{g78o)uoLquy~Ai@gAZqA0aGJqX0CXs!@P90SmWUZgxl6FdzF z0fwG32)Ynj0vrc2<^Fno!1AkBVI1ZGTtuKk;49S&V7tJVyb$zIC>gL$JG!`BtjlZ8 z&Q=EWjd0h**7rExa_M^a^M`KP&W+`%i&!m7a}Z?v@cn{z>wUfxO1AtreTV(_pl%rg zm&6HQGxxMul)B2)l8zv~7Y8J&lX>3MYs{lvH@H0WSnIe=4Pb+s9(9-M1&9qZPR+xPV-iuO4Z$K$2F$OuZB|%q%K}$f`!yNKUtBfBIIJpc=7S5bjp~1{pD0*dh`OKQd3hM zs9Xt(X+}IukKSTmv@y3+{}QNriXW2k@+BT6D7T>eINbeF-ugz(B^F3d#yEnv;JZGt zccO0_dn^r@Eq*ToSU#9^w6$5i%AUN!aHLcMYk4^}^-Ry97;PLf68!LT>U<*adebo= zD8u7Q?gdFn(OU4RX*@!9z^CD?F*Gyl7^M2f;vN`_o9wgnp4%pp0}0}-Tl&UOQleQp zO9_un0qM|$AD?E5&f`B~mc6*KiJXzveYjY$BD2x2TOFs;lx7EzE=o@pSu`h@W6tfs;C_V#e6a{(2GP8g%{^L8HqaCjnsxey%k z(4ikpow{_QW`Dd;}5Exou7*zg1YQlJuB z&Fb{JDc1f|m1fX8WTZ{ywg~XbeaWwj`s37xIbOG_B}SJj z{v3YXreA_@6iE8jeBD)r{z|7`inuzP&E`q?{$=s5Z*6VG0N^>b_zfKMKpAl!uv&|A z3K2~=GsR(_(rkkFAa{dv_7N5MIM73a15#3xoUBg8yh4_3+Sz@k?ei6Yd=v&8^eXWD z_`Y$0;+e=$ae)@~d6QuetQ$lYsvV62JzKM?aJQ0d9&2mG&)xf96jGh0`;(hPRz3OO!}6v13OJeHT_DL`Mrk{Rn@aGV$j@dy}iCp>Z1A z7j8HB^M=OWCynNIuY_O2s^Wn5Hl|+#Ks#nk&6fSUm~bObexb=k3@9)Hr?61;4@?vt zcm=jK#mMTN5uZ0t$J~4#_E4;V?3CV$H&Ki?)FvoeeikYWSh)Ojc=+6hdYVnvxlugp z8X8W4sR(xhC{DJwqS}Mm_k?YO+~2gDH@6v^$O)4^7SxT5p^~)p%l1RWt#1e}0$Zyv ziz4C@HavKH=|vNB0HqKM2k>c_seW@pa@&CsUu)_wAjTYU!I4k{v$5H$StNCf zSH!HC{JNYXynE-ASK{%9TiorZzdkv<`ZYvs<;3dM>rENJ@bW3A&VtWa?<`cPW8>X{ zsI>9vw~Vg$APx*~h{BWmSqaE6K|48?IE&*b+&}1a4FUZapHi!`UkR;^-1}LRyC4Z7 z9EexPn*tA4D3kzAp`=#7@8KglNwTGa%t>l$W9J_)t4Z*z77cp zmzdlUet|_GPmT{#vaLufg-3Nn14xzv(f9(nid(a%#|$_$GSkAb{f+5W!s#Tv4ZuKf zUcdVI;BUJ_jZpw7mBCV0&d$&2ps)`~PVEVtT;_leoLVvQ14gC&!AOh{?2KYZ%1Gdz zt|*MC#8!mCVNcXeO?il_p`{1x-%|_MB53}^(8#=H%%sK1d$?4IM)aJc&z<^Ccr>yb z*WN>V6N1jS(u|eG&`4VP_j02DlfJR^yzX30YSJW?LebK`%&>y|_ibT-Dhf+PV&WXY zJi{)l;RHmu5ShQqaEXwV1)OUhK~|Cn!yhSX$WLRMu|qz7{J2(gHEcaBgb}nQbx@O> zQcelXrP$JgabG|}Kt5@)t%Co1rwT^q?nfCS zp45-9ZT^-lg154T5|Qua?poODA`}aYiAhy*_G~y$^`+3Lm z1c8tfQ#C{!j6P6IVEd7`CAIC66v`X>d0o!9j`=gpz8_XC3arUQ^UcK06Q#7$FJIn} zomuOInOJ69ywsZyAHE-1rp^RgUg8{M!Mi%xDy!W3l!q>WHV6a)**^5hEdXq31K`s5 zJZ9Rg-S6hM$=v)R3C;_L4T8e6MB0J572bpbnK#;)s)r-h{b=A{L~8F2`CK`O#+h&*Qt&D z&%5U{SySsYPtAYRrdQW$v1s1Oi5DAI-D{dLC2=Nx{QkFJ4%YLnR3pHG{{!H011HH@M_58=4 z$KMunb;FEHbcs$*y2h5EE1=KKmWuI0cTHSMJ-+2B{bV}AOu7*Js+(+IJcd#m*+v)} zt1>&b>gwJ`yCiqM%<$-ShwM!F>x(M_yii&2oMy*t7^y2|Dn|~ znG+=^j95vMdqi-%T14X=BQ<^9I@3}+MJR32i2QBL2-6UILAJYMz+$D>0PcD8cot)c_mA;%qupb0`9Tye|{ zl#hXv0RYanob3uvKK~af(tQj{PD%>Qo3{Kxqx9O2NYDlVuB=A}Va(UYrxE z?J~4-aWRQbb4Up|j!xIG!D@LE=6e|N*{dLMMb1z-ESN=JKtG1iv_w65^5p30;bp+} z2t|s(&c+9@xaA|9A7YI?=awl-b8{ejlvo&`{RGR4O)BIwK_+?O%iVwHsvU0}{bYhO zgHQMoR9yV?=TGXk2!uYEr+S#sQ~V*8veVFvk;91TK}Q$eYq zFQetTDn68e|lKYJ#j%xb|1~ z(%`zu&+lP+Y+T%8!^CprFVq0-!W_?_9#;898ZBEEg4%r(AV6~CidNuRRe3?Ps=82I zf?;v0#obwEI~Z{VECkRKra2fG@iW^iTF)`pfT-JKU=WoKIsfTd8bfT?uU{9GStv~# zhTDYfDu%L%4-_*RMm3+lM@%;!Ql$V3Nh8e%q5^!hO%8_4gGkd+f|iHX!C+SLR!as` zdUMx8{DKrwwOLD>S2U)0Nu1hv z43Crz0{#@eI;SzkjN-EyORg`(J@L;k!=wL8v>PXIQ+wfJs|C`$Ydpl{^_`rvmmf zwm&OBbsl56d*5dQtA&5Vdqk$3bI*z`5@99og-DX*7g9UOQa*yZEU7Iun2860Xludb z0aPv;$k=tW)Xc45RmX%OFM15JM+u~fz4?3rfgpTlpxK+wCz^w7Mad-Tk~n_VA6#LQ zB|vw!Y3J4`o$PY}$ebi4i2o z|Cj@pz?RC60UOZqyE1nEAj+=aJ|on0CjI|na1=A6%2XNJnR~d~A&?zP+z*i)5`?Fq zM~_8Y4w1?xPtPp3Jp7R=7fv;5rqJp-df~lmyjyN_cKJ4UpApq~IE&;X&8ALMFVKE zA+;w@17e3atY4|P8q@)5KP}OMo-(jkf%796iSdthxR#&;S%Xh>k*hQkS{ak=I~0+3e~=I^e=+1ize4EZSokL+T~d|opwnorZV-z z?HPC9Qtr>+J8hz0G66agn2+b-0-^s<4{}#t^7!!BiH4X7IUAT)Vi-dvnkI05_q=01BKUWod_y}q{*Z9_?QT#&!m67Nct@B zKAN&WKATc_=fIkcy@3(qdwuRKNSw78{!Shqf<$Bl0Kvf=hYk-@Bp%yBqsv>NGgL~z z<^=YP+L7m-M9K+yB18*)X%*6@H^|(FQ4b0MHZ~w|a{)we8G zAFi)bCc1*m3?Y2Xd}q#uiNewjG87Se2nGm+4`Y0U_O;(r1^{GyJU8)=5_(=CZkpY6 zX_}QFXj76kMTEBK@CdR@Y!gp|E&m0f!%&N^M1u%g_5nZu&q<1U^x}B6@gwM{0ibKS zpqC)Amn;BXfeD~LdPE`q1R>eY0DfRqHgo~uu>E@Mi&Zmld@->S$DEs6w`VTmPYCx( zvM{2f`#!XM1FKU`xXxVwL4+$lZ8(qK4{x;&3FXQC=o=A8kGS;kVSRTuvU{746phZ) z9GrFs1GfK2`!#B783w){#-{nel(vwg&9NW%nK#H%A|fIorKl5G$^C8E4~k^XAbg)_ zP)JcljC?2y_+V7_;lAnWI{wry_^DlKO-)Tw3T}3|DM;{UXYCHt;cbkR(H0v zCAn__?43-jp@%M8B6@O~$lG0aIdL4=^JYe}&h+#yOY%7U<12RV#|DFcVCeW|_SEv@ zfAP~}ZS5W}^%4eKRHnLuca7|dw)jVl6F*lYH@*Vx-ax%8zFQLVjT|tY0jLpyQOrdp z|G&2#8X9*io6^wR0)M9wowDexW1c_b}GwJipdKfpc4{@A+ z&IFVwrYW#%)h@~xFg~GpFHlz>CkOi*=ph6>xIv;3ikt`s;d3oj== zkvi3)?$>C232Xwj#vWpm*v;|<1gp#>=w7A%qeTTVD1X*Z<* zSfwG#60d@_U8FDi0TeJy6Jf-Y%aoPp91vs4$|EqEqZTrAEx{*a9Nj9mWHWIJ^|iI7 zD{IN6SN$yX9x2G$s~(WQn{c3uyph{sZD&U@Hd``k`8Od{*n_vyMw*=BxVRJeoW%-% zB8vo4L!BH*Qwl3A81R0L{oD_s+u6_FpFM1I>kp3n*zc|O$+HE(-#UaJP_DP>z@O2J zWFQ;ta54tN{g`B^6Khy6%RbiU(+)hYy$jHuXcEg%@#jf{K}FT+C|- zCZ0*%@B9&epg%4T4=_n5W@Z%kk6WY}!OkyD+b#-f3JwA&W}cb&^)#^~S%3>RGwZue zaoEu@GNG|z1tCbt)(uo+1@!yC{m{710WbEd`MmnXh)Rf`tFAddMXd56HhxD8`A4QnHUVg=x-#D zGZ$J)IQogD6Kcp_^VOr$&|gAiDgsl4hldBzY!ovY{VOZK_YlB@-}*xDr>r-Ib1UCS zE!Zn?rtUKd@DDmKP~|83?@}!NV&dOg70SNy!*zva`V` zKtiW6?sFYl2Kx%}0yevceZ$j4Iy7vzCTAD&V-ht)bkxk_j?P#_c-xB;$Bz?5)x5fp-r?eG4d>LQ`AYg+!AN z8+pwgIOTeJmj9il81yF@Puvd3A}E>cr!08<#lza&ouAnoOGxEK&WZ;6qk1EPt+b#|sU?{?a_98`7@dniFb2a?e zENYBOg?NX698AIt%TCFjPkfP zO&-il#swi~{A1gLo_(X@R~tM>pJRtZmTJLJzKd}8z6zUtSH7LMj3TWA=dNAO@vjB` z(KwJ_Ism%Gk-yodB!iq6*|4gT~bz>W^(lm`}~4C64x}l%b~so*;Y>!bu;D% z{5Wg5Lj)`^SeIgqx?o0%UT3%SpPXg$xN_qWO6ta|r$cUhpLf4&g1daPrhd|zAiUK_ zPa^;)1g%DCOK;dWa8!iQL|Z9d)0>P9NZ`m$M2IPtb{-zRS^e~B2@Y;3=s?b@UTNat zgJlt#$ff~bSQ=nn!%R+1vwx4O$dX7v0|@8~L}CRouRl81CIj&Uk(HrQELI$)fdp*9 z^%n4@PICTy4E`iF2rp1#2tXm?+GjjN(+`%ht&+DD=r7PT>Kd0 zHdKHtBYh#req#s$rA=M`d-GCYanUiwRZw@s_%ZhA zED1%$PYg?{qVwFun3!V(yo1fdBX2CNq5!5z5_yA$c;Y2}6G^A}oU2$5_zTjYE3gz2 ziS|EWj1k>Q1IK1$aI`>wK@weR!8I{^Vxq7`3BGa#3IZHk0*n%I`M!N=jYp)L;`&#j zIVMjR8aYy)A#?j37Pkl~*EJ|pOe6#OCMvD1-wv&V*aR?QBuF zewkk&R4pKqH#~9oSo5qvl7J7_$~Q9*LZ^QhWfCY}9rM(V2+fdcFKNKL-Sz#9guQp( z_J+%?x6>6dHv9PKYuJ7}Y?J!##i_L(;sfeP`{E)m0$!jFV|0K5))_tm(oYSq3ZF{d zube|x2EDth^YZBn{1$j8sT0EvOlR;m=#YS^4e8Ff1*U>TLBk2$lrX~q@tOe1NjI4l zB+FQeOR+Ro-i!pJSl)z^hxq1EWLMDZ5|fE{e;VsRqx`pCT#S^~UwZ8)PuA}iU);Qq z*B(Rx`0GDHQDC#IlZc=F49T|O3Wy8M1U7#HRs_T3%0>kHm^z!>*D! zFn+XsSj%D<-DLA0hh@f{JBHaw$mqde9{%!q!{^V7NIf9F49GGdl0q;cU_sKBA{J_C zlsa$-5Z{I_fN|h8MW~0xE&Q!yJ}{I=ngH6iSUE-?$%JMcv}7aP16Ud&4KiHfn~ZoQ zGUqWJ;PBm(Q;mqZjYL=?ybRJ|^a5w__<-j|WFQbnpEoXMFe$(m4@4YBW)gwdx|3N> z9iPIjO~wyqxGi|1_*eb-6Cj3QMP(ta?KZ2Y2!2&&J&k%>PinczgwR>xy%tRwKUd_R z6*%R^qZp{pO`tg5xj3BGyvZs51}RSd4-Z3V%w?VyvI2ztwyqEt8$98qq|dL`7@ZDa zsBi$?)X?V`gC2xDkF4T0mJ@!`;3XdWv0bN1kHWV>)}*ORPdS=v-jl7q17C5giXlI4 zJ(klouET_iLMgW7{I~NJK9YbanKRrwDK;2z=dSgj^nJaCv;z_>4WJ${n}|rLy5CPJ zMaAwk2~(n{M~92&|3sbYX!cczqe%aY;K)?tKBc|1^?$*$4T<0i*ibSs;LU&~+OMl0 z(T&)m31J^z7$dPC>+7n+(CPjnQ7{Ywe~TMg$>4p&HwATuubKdyl?v%FmHX5*|Z@ ztE;d<4Cq8hTD45g2LvZIgW;kjA_AnFI;e`qPxsu(9^>1|Db)9p@kkvJ-}hX{q~%2?@JQFL)>MCw%;wW#H2`1c-x{;I-4h)>F{wgcDB|?oZcTzc((B z@RQ366+q#k$!-@8{rM<>cIcUU|-|#4k zJ#1nlk}L@}IvREf@RgFJ6W*++XSYF5nkGd8Iezb$$pGJrQB@AAS8RPhLBt%fh!I7; z32efP^hEZ8#%Nm;$%Kb{yhr>PEU3W|3HnV(_7~e{S^Jv$cWq)SuGc`udNLMs)Ey+e zg}+q0P}aO2M}yGWS?`h`-o$EWJhRAJ%a+o#Tr4X~qHGOa&{&|wVV{2>j~QU}(;7Y3 zMuc7|2gQcts1vI-wZauicihKHq)1tvn4mJ@GtGJ;F#$Yu^i*_sSX&tX5{Uoia2?Zn+0jQX z{@^X+P09*h2WL*FrG4Gm{qQSuH0%BE{1M0f|En#~caG;+DlIG?pKMunSXz4I#Aj|9 zo3cmSzJXXLHS-gTuI#IwH**?s4-iQnJkpyky38?IBOuC%_PTyR(1Km=^^3r}`2Y@y z{1jPg)D9vC0za%R8^bC6HIu+9RjV6G?1r1Uo!uhPq<{eLTeTF0!TpBb!L{K5XI$l~ z4i$_XfQ9Gd&9ZbNGKOC!xAsFX9&Cf`2q8#hnD9Dd8E_iz?}_%dQm#KQ$S zr{QKdy;t8UD3fXhaBNKDGR(|dgk-~jBZLtH`lJOF;t*CyNFDuz(QLIH5+sxl&mHjG zYBjF$47h;7Y-1123~oNN`&Va>M4LbR*`WwYWwOmi!0|Si_TjvE=#Jm^Fh5Q6%=g}#S8K7Skx|oiD#DMo3(DB%`U=duEc4lyI_F*!m@$As1KGD- zxQwQQV@0YWFqWq#eyj}`YVgdJ8lB8Xq3WaiX?;n&^qep2Qt#pXKvr0B;21(J6Sggn7&J zEgRwQLv{+D@JF9Ne-}!4_QivkbYNWt#CdU09B&2Foc#71CnFzSSA6|wB!as)^<1PBYmUuV}S%Qe9M?hlGzcVe1`I%SpaN>?p27B6H$*Sw;@rr z%xFS1q?p#`T3b6CZ=g`<8?l1`KOa%gAxZ^^6Zml`Y?zxbiB6e>oM5ckQl!aL0WkVt zk2?mDJV`*!$-(|^d;RJic*vhU3uZNixCz9tEQ-Y6Wz0`{e7@^lcl$zGnuErm`K0Kt zgLvV;4x;va2aA7CHlKD;)TDZ;q(w`nV21K^6WgnF`x43UpP^_`-%xLt^@2pUGtN{2hU$+b)5t8V7_08S(O z1=NYc+o9@}Fpxt-g9@O{AF?O+FFCnlc#sghvZ}jo&|8fp!a-=|Fy24qD8k~<7m+C( zvXpAS>C%Z0VUx$l`%CZTl=ET~;VttprQiiTyR4*@maN3c1(yiz+O-PiQVp_j;CgTbFi-3AGghE?hM&2iG&vNK<}bAowqaCbI@x4O|+A3nxdTAE_A#uGD_J(VN4$ zg-5OV%F08FJM~{SynE+&B*oLMq3Fw)@3%4c8-=;w%#H*a41T=Wb;#k{^NWwf>%AYT zNPd#4mwIFs8Is_z=;r8#!`br3T-=YA_xKy%JgM0|u;)Mlhj!UZ)0C)+7A#;f?^)%B zNA7I(YS4`*BW&+Fu(2Hteeh8;jsDNi&uY4j@-CWkdrkmj z?N#xJZ1n?ORz4WZYhc(|buBq1g&sK38n_F4;O^S-hH5)GJHJL^^;h#EfJ+5qeU(@B zl*MPB@PgJ09h#$u$Hd?bbcw!S&YPS|NhxX@mpGn^5vLtl!gIvMy={tI>hHCqp~AA* z2o0{I4t!k4zIAo&f+#BAf{t0tPW&3 z;*VeH)7yX7^yV|$DOk1~J$DM*?ZY7A+u9hHM}S61=Kxe`DDX3Z zTvk@rtO3wBcKZtL_fwOExe|=T(?`Y`CtkHjIt>mE9>5N0DCT8llwc=JTqe5pwF61Y zlaZB8jOWcDP^_`)!i8l?QW;^@tr1Swu3lZTa^uoRg zr;+ysfWtPNDTB@nv9VWyLL7j%519z&sUe;hH3HT-^}RwHJ8!MWqinSU@1U3l<$mow%jQkx zn>T;j-$#&*Ez>-5`jr3tdokB-`3UyiY)wAS)h8|88(;LEk^I`tUzWorQSWfkR`=wc zXXiy1x1V6ID#98K^L+1v`h;a|L-uG6`~a~sl@J(`l#28%PXo`z$ERblhB;1@c`S_1 z*fv0t+lR*|eCn>HrL{X~jwZmhS=bhIdwSWKiWb8S8_I+R6~}w}>2$gYl{!KDIjgC? zwD~qVqD$aF@n&Gz)e`aP3Qi*48a#Und*?o0#8;NQ=TyQ5WEnA-aJ`kBQ zknS_^E$F3e&c;5D7uA_puiC#0wC||JU48lEYTcK(<<4Kc=wd`0aC3Cr9EgJ^-K`$@ z1LkfVjL5cQ42#$H4hYtE(&k=R+ zp7q^DlZ7O91^WuOBg)mt**TLZ90!Ljb5*nZZs^6GTwKmqv?TOjta~LaDvIM(iI;0` zzdAn~Td_gYyaa|r;fgF{Hn!c=wLB}H zZOgvyru5_c3pTbbMiw@acZEa`hwtnj%GBsNEtmg1fi072ycduBG*U|kJ$CPN1vM5QNdQcN5Af;b zci7t47@L`8AoW(>*XXaZ<*z?}yoII<$u(+sQeL&c#+$p-$mnkR_>}k*YjIyJ`~zwE zWx%(V!GIn1x=!ap^m6*wcHr;83rfn7#>vsZP20HPZ@z>`logCAJwgazuk0S&6uVwZ zs;H3;;z~w!BMK7O6s$7AKk))$EG(9LJ1B&m&HG%H{VR0A13;B^oB)s1;OC>G)+UHK zi)_`?dkv>K>_Rs;-%Lx>0S&uyMeR3KSz{w348iv{-@JU;woEe_6T~VIrTf8qI01^;4r7(vqV^}UdNc!58_N%)c=6d@}kbN#^s zQ+Pd|J%7IUo?b*PwyBgYm+l0_K5yNjCyyux;;LI;LH~2v35Df-QDD)Q@Ha*1>9fTy<@T&(?fquyWQ(_hp(pjwQ)tEOH>0HjWL zQC8Lt@Neg;Tm5WOux+Nct`1=!&gGMnK?C*>`dH9S@PY+x&b)c%uYclC#(NI353eQ5 zmL;tmZYM{kw_^UL_t5^wpYVrMbuZU--Piqd%NMovYMPo?mZkw48c`5R&)~GBI;_GxvqU24 zqeFw01GJV~Tr3|MGS+DZL4yQMw!O-z^s%h=z%5buPSW8hoHQQ~jmXSazl!h>{RdF< zBbX)ZY~8Va`-5FuN8jSqm1wNHaF`t^)Wu5T(9N20whkYI@lhcR)m(UZH7re~-6RoF zR<6t=?RM^<-O%bawRQ0`?$_MxUqQ90F*#n!f5$*a%Ei^MrFK#2TFlO=R9qL6V z22K~SNd-^u`#2Q^d*ZGz)~ReS!pd_fa0q{^Q@!+BaX8d31MRqK_U4oMW5rJ94_?$j z7#E&dw)4@-3+k(iVm^J^@>sj2sPLY0^YrSOu>4Nyid+IUieX>tUj?LN{ z*2c;wCwIs!{ptR=OY!oLqO4e^u^fesP%9#+=cT3SxTlU~$!l$%EDld5JuaG`?AsoEaaUi*EbDC)2l zP#ibzAw}KJ#oaE`e;e8JusKyhPuoGUIzGSAUTySf#b1BcOjyOVE>byTL&M`tf(_JY zT#KidkFO>JK98{yof*%I@Ef_>aRfw=ewn&k~E=ncfMv4*nkPd)MsjaY;(l+@Rm< zGd9MaoVfUEzIx2lgYSMh@k4x`ssH`X>z_Dfs(VDZ9GU{7cYglxe1G{tJv)>$T0WK; z%yvuOu3k^Ea8EJriZ~JjwLjh`RFpA1h>3Ilm2&Qu69)?*V*T9!`PM(Qpv!n&UzzM> z8HySnIiW5q4|Wqoa~e1LtaNrZ5m8~)sss7u#%Oep)$R!i0ewSCe1h_cTrkXLeROUV zqZ?&NNw?=v+hNiiOAJvS+c7?``_`k-w3Nbpw)%lX=vh-+tJrg4Q1$>4g>&Kpj-idt z%$d&OzXa;YGHl5V zcQTGXb*eGFo^YP~^7a9;`LN6^tp@XLj5~<9#|c&R)N((K=Xe~p(h-FlfBcDhdBvP# z3DM*U66CHZNGn?~SA~J*O^_YOimYa6yO&^XSUI?F-@n3)0<(cDsu6K!Mds(^ye`ph z(ui9u*K5i0HRlmft9J2vFX4N>ux>yGJZ zg4Ycie2)4k6_xlW_EjG~yl_tBbsvRQ(NgBClt#WyZ2e9tb)>eY{+Fu#jsqVYc5gpM zb^w=tdjPer1^zeNN@9X54!&=f@`I#p{CLI2$nwpLC&{1lyQSUT{ODv!-cVaN-`_eU zohn|nZm@R#=@<3#J$vTBYcbn(N~0WvwHo&VTVByj&qv|i<>m81sH5>E$V@Dk4Jp%XlZJHY5BvRCN0~3AGe$_Rl3Dm zy>Gi(1+RJHDOjG4NJ?7BcJ$Ex34}z z;ZLQw{2q}a6dybG_cfzeGZ+uD5vf+LrAW1#QKe1^&7P;Zldoe+_*T)Tn zK>D5O@F-UF5%PxLzki&d3`7!%&70Vm)kK(A+A01ye>P=Bh0l1a&f zn8v%~PtM(Gc-W=ak0RKe9dG+>uuEE9qRBt}^`po1E16LhX74ZOHC3lyky!M9xTkZR z%b^6MzOM(X=6(xnJ=s3Suu3YKtdN?v70<&`AV*IN(F*cNF^lZ01 z-7SWoIi{uMlgepw<9IEtB>IV2v-YJv-EK}V>tf$h5~GN*Vgn{Le$(w;TukT2EzBkJ z&(b~bhI2)9V&b@aA6+!L+IU>}dq6ELQ^@DE*%Dz6?Td}41z3tbYi#h0dmCJJ zi+QMg2y)#STRsPCdAMa&?HbT)yqgis$%3=lAri@#!8k_8CDlbSycu=-fO0w2T$N|- zcomf+#7H9sn?!F6t)I9!yI!1x=mPF9$@Im07rLc^siqIYs%Thcp-sd|-FB&p;+T1J z%ryP19y1&SyW}h(LsjL&7zLvWw z?n?7>QYu-PHWud@)-N9+O1aj9>6ap#8O!d0faxu&82bt!U?^3MU#028TaK?i02GZ0 z`qrhlT6SQMnYf3cjn(1P?U^hg80fn1G`6D96Gd*l_#}Dq-`{)p2Ae~!G?bD|TB%K9 zvWwq%dslV5(N>b;cGmj70+49L*|Q_(jB|Lk3m6zRg01D5Tx$@KIvltvQl!?$ovROK}go$^T?iicsj%-MP0W_n0}1Dw-O0cX6d(6kfV!%2%uXbt2!5NmX$y z%NYW)?3Q&Ivkj3s6vBRxSbT^??FRXPf=>7vRur6_hNt0d zbMwMmKegtB^eNp|WlF-A;dTaIUQL@WIF0Gak&EJz!yp#l=c{6V(|)vWyn=VFm846h zS?z+6q)^Sr_iv<$Lh$QHdG;;8Ups#N6Tj-doW9GzVC}BUX`7c%Yq7yNe9XY9AFq8@ zj{ZHkL+No}gYJsKZBptMPp`W1s--un8vJtc!yVc~* z?JO^!JlQ%Fr%-boFY=&SJ9XGrQekLj7TI;7%VGn$ap+K9bHBLE_i}Q+Y;KL)B|%M} ziggAu&Qo7f+(wNbAM5MudvidVpMICbUpYCTLJ%-iw2B{tKPXYhn{dyW7ww$?!(oZ?7Kl#X43JpJGdw~(d!wEv1QBZZ@1B8d2=q3VNO~~wV!)=E#_Xog z+Iz4_h`{SafC+K|v3dd)puA>wvU_C0xpON-m44E^7n51sgaM)1U8N=5l`D^AC0zce zPcC8Oi^GZ@nBq-%ypX`BFwm?Qbn&r5z3}^-!3@%1jzeU1>gTf0Py!IEgtl@DqwFv3 zu5uGZD!p=LzDcxG`iMiQiM<=_4mZ2Oa5`@k6s9xYy&6!kmvX2>Gj@lm( zaGd*ZyWorcsh_qkehnFV7bW4VnDroXpJG}{O!F<5EJ;{Zti^YE4U8w=dAxRBRWo6C zS0q(dG{TWYoj-5Fg#Vw{I1ni}*{#P;q0?-6PF7j;!L=Y-+w}zByZ3uz`TDxEPgDs*Eg#D6j7i` z|5^|dFlLp<*iL!H0Cb(*>LzPrGu)*?Q`lZ(}oDx%Bvr4DWWS`_|9|lUq{&FeT+qcnro7&Kt+Gc zI3HN@$N8T()(q&@j?gnx2F`K4vV(IzuOSULS8mk1F!%e1ROl}`ZECb=QJi+;d%rz} zXh9K_MhW^Luw8k`;%>i_j{E#7uD=qY@Rx0=GuO5|l&O&Tc=)!||0Q4Gk+6xRFL#f~l*UFc!x4vwPC?`nz7MCi8<` z_GZqzWR`G!>Kk#`w9M`VK0=^zsdF8k$@#Z`=;Jii0CEWu;E{R#s_1Dunnun*vGpISWsw5Bk?z|~ajsD@E<0i5Aacy|okRdmYc|Uok z5ckuD8YPVz$39eFlIymu{Uoo-Zn>$EyW2!cVx~11_HDSj#AwyN6X`amH4`e)@rfSX z26!q0>pcmCaq}Z zP-+P6pHA$LN^9q@^N8x zM2abA7%ONMfO}E&=ZNcbOEE4m(8dYhxLg{Z{uTU?fK`fGT1A989nq<3m(Fjm-a1v~ z*x}J(m;CnlTq~)PHku;m&N%BvWD`z%xz2ANVR5Pe7n7TNf@NQilRV}fy>ma=3hdB? z%{5FVo(v4E8QpTz;%+t+a#0X#f`_F+I3&7Tfq`RMbx5GXpZ6_eBT6JwfCIj3Rgd#V zV3KkYTZ0Ujd9_3YIvS3tlB z(=!r_Y!b>u{kHbS%}s;esmlOj4DLvR@2+_QsoY1o!hr}?92dVIIE^N*`Pex+vLVgJ zpdMj+!2MDz_2!tF9qid5EhuDmrVQC=`nam8OC;I76o=}l*jP2y?s-A%=>QA9`J0r3 z=-NC?pyn=FheQt|oO5jDQC zTfuZ`iUmlt{n+#N9F*7&hZV))qLmlnjzU!bu@-nW=#TjRD_6cv59Jk?4 zK7JQ>eJ61D5!5GBbGan@0s=_5HL%9bJZykxHI(E;?&7OIC+_9yauSzjt@GxHiR}+y zJ@zv=K_&qtorb#n?tD(t>Aw6YO1hMOkz=jiUb|kjr+e$@AL8w@YGy@^8EA6q+{FktuC|a( z2_msTfldfv<4NXn4~$==(k(|62&nAoy0adJQ=^E=meo{U&*_7E(Ei-H^DZad`T~!d zzE&VqbDb?>ssHzjP6Cz4&cdSQXX!(y%WAlxg9n?=l?$izU%@mFZ28U{00ohbuBW9P z3Tzk6s)TbghCgS-f16Szx(7bVmDxowh+mcpx&Z-4{|e>>m_?bupGPX^3?&MmAlXkY z1OU6tdTAyBbaj^{(Fk?!(uGv7I0sAHJ5ctsbeA1wX>G*EJ9l<#Oo$(<*;s#}z;1?) zj_RO{p-)7EO6?XlAo@01u;Yc4gmtxEP3w55;VTb-O5i<+gQ>%$;QL_wbchBCKoNZN zgL_xlBdfp1`*z%ho{Z+|2&`&cGpkCwygf>O%<4b580teh_wL+bo1lqHg>9h|;D7)- z0MS*3bf?QSG%?xl`F`i3*JyMuvmFMTOO`%IYBazG)`(@?eBZGwU@4v?O0BA&w@f5q z%(k_S#-U7*M6FiM19H6ZFcxsA)uQ#Nkv8N$C&Sh;dOT{+-V+agT%Y0lqeY0@J&*id zlxs~db-rb{YLsuh`FfzluPHrC*QkTB-4A)%9jakN>G`mx16oKN8M@h$1uVOoym zg@^kB11~O3G8U#zdu-6%o2K zyJV!Q&n3#TV}%HferdgVhxb#H7e24wXB*KN04Xb+P|Zhl9qgbkjVfkS03)4fdLs(y zIOXae+NtCJwg7gCf3|S8UpRIXLVS>mUxR|6r;a6vjvIPvjS;KLOWCgRrRM?DF+KCp ziK~>TCW3J_ys^?ADNfffTdG;_l;PUpym_n}o&)FDTN2>kLK0sGb65W9{qwPouXl@5 zwyGrCmVrjSWac`AxI*wj03Ol2V8ww0ugeOxG8Hwo6Oi_i2?@sJ>-$_E&^2%GyJT8H zCoEieiYZ@Nq4x6EcCBj=V7`8Kedio^|7Xt07muE*m&AYQeYLV~n)I|SOZql#-bjhq zAWm8Hhn>YwYvsCcXeAlYE5qk-{204#bzw`4O}q&J*N7@zgwRYmy;$N{m1)jT5zjl zJ-=Ke&38M}sk{(u6eLN8U^FB2>fNz(Oic%0csR^2O)fHojY&K^ zLuqb{e*_J|i3AV+m#L>AO*IXwt@uA}X#ODbM&yz--%@eB&V_Rd-agW}`tI+xG@Btp z7bl4L;!6(GcnLi`f4c~27BYUl`n@EtrX-{Y%;j6bS|NhYj-=KUBmGoZ5ljV@b?509F8`UjSlV==B|s;& zl0z1G^ymbr;$2Y)84w1+**Da&J=+m#D}s%5`AlaI#7pJR*!Ve{{yGBhn>-~%{=k4k znJ0_Cyc$g=6Veb1d-n9)hx`)i57w8@0 zTXpY~$}fx#Pp`Y#Fkc)rZ5Q`%G2J!a5mZ8Nx#je*)C-w#qKo`Zq;a=K+!} zdUx$5>7DBK4mm3HLE6*Rf-XYWvVU6ZUs~qvgm6Fy*<6h|ISq61-ztCN)T!9&yX->j_aJ6;y&VPWX>m)Bv7AqB4aG>{v(Gi%SL^!*-UDlXjvJto@!dXYho6b*T zu`a`fqBMB)2)e62Cy82#7cK&tqT<6lp$PzFk_o+Kzu6N?XO1Sv`%5@z%X$~FhDxzQ!-FCf&sip`Qoe^{{}f}R0| zk`yYnycYB^j~>N{#R7Y5JJ`>2zHOsCLZkZzpT9?|N4NC|h{~L~Bc~g}Z?#gx1n8+hoZL`kB4ZnLYmK}FVSvyh4ql_Bjn_}qGMW#0ZIz3xA1w|ljca-U8d zeA?0MC+m*eQZKK4xdU_YQIvu_rAE^M?VwP4(Sqt1 z{1#20gFfEcN2Zq*tNt}4?MA8PyI)Y z8yAh}vJAU&b&Z0u-#9EnU5t8uk2&Ch0Oc)*Rv~$cJaXg@fR;T${9NL2QHEg}iDST& zLdb$wTQ){YK{RUqS1rw2^$ra#f(6}sttWDN@m7cjNrn&qNMRxTA%zt_c{FzR-??n3 zguzpd!q73CXB~vf=pZU1W}ILyEL>WV85x!8G;#-Om2^&k_d>!zKF;bUOZ%+ytQj z*>PKc$DXP^6CxEhE%vylI`sLFhmw@F@uo359#{D<|Gnv0^);8P?1&jUWQbYb5@$!p z*SD3EcLh1tAtthRL$a4ZR%+G$ckMA_B59Cs9wcb#kBT!ZmTN^5~YsEjs{cbg`%qvG2?R$4Co?GH%yORIE`pM7d&*Gh6b)j#l9 zOzNQOrh?~2mX^=X^s=Ib#w#uXZf|^L9kE(Kenel;grV~w7^H^L^WZfY{-rUlN@`=n|edgA=O=TpW=yp3oj=nq^vu&OH*}x31I^pqTQODT>s>r6yZF7{-evDS;2vUcLcw~nR^SZUJ3=?+!1El`BDjK(|#$9t@dk~D{yCxR8>uavb;0RRlggq2QR zzE{Is^cM&Bwv%kAQ64mCniQV^vZncoLM#HI?@&jWl?tu}%LLhhcgD`jVH~aaGOUZ3 zH=rZCSza@=(=WqTBm(gY31Nk^Epd7WUVE>?I1VZ^$-w;e5_f%N9o5{lWjZgkdP!b& z3u#~OP#jn^bmqS6mSPm4N8#~rdr2SfJ#Y9Ce+OE{|S(H|yGo;#x$of(`$FiEpjmu0lvC%1N|Xp%;_+ z$*G;K;f8fn{#3KYH(V=JAE_1y5&Y!NqG?<#m`y$zbqyG$v`0wL=)lQi_d5RAFBDo! zu)*o+dCACFgz8Lon)d(SPVr(pWq-=nOjg+lT%8BKZ+Zp3Hz0!znw|27Jjva&f08gRWDGIVsXObV`A!i&BgxY}OSJIpQ|v!M{RvfSQ^9#nux z{o^wI5gayiyo zorn_afhn9+4PRM?kF`j~q9rC#YBZ3tYsma$W&>)kT;$$+gM;lRTJp4?Y;$OW6KUox zr5ec0n>TmMsw^Fy^sSA(!8$CCGgqBYBNWFO&oDa>A30}g(Tzmk4k=}8S1kqEI+-wX z+=SmmybAn2Ug>^cXdzZA3_TQ)J+7ofP)A95Zi{IItdzS4cPEv0AFYA^@RLJ_*X%oa zeW;qVb!*4b?Ism_)P^KA)|G3>9cyGG&1-bEW_PYV{D1J})^LFW$eTFvZ~a>xW6oKq zpnDpQpctV-nRw6So(`W(9-BXW?$X82PQPUH;R$SN!0(66N#q?Pp6@$+c%|i;n*N=B zv9_@IgI1N#jem&W{P^RKI~x;S{K*mjhAaQz;Eb@xyh8?zFbeu)%S7#V>hcura10F} z4kVAF+?qoV1r<82kYU-&4kf=l4!?-7mH?#v_TB8X4i*^_@Z>1PBFE? zN1%rIfwwargV!rT8rlX5sYUaOXIOjtZl$4K|RdZ`d^^`uwu%GwT-rdQ~~` z#;9H+bdj8E2X^RQcg8_CdHBrw^dG0N!%6yWt*lC4+YverQf)F8lmM7z(PKj$Yn!6=hfl)kZX z?%Ep*0t5tl`tAO)__71nur1 zdFA#c$rgP^{2tA*-{>75JosMk_|u{5$5KD*)Lk3>k?E@V8@wPC04z4 z`j7Fq&iQ1N<)xf+UOq@O|0AW(Sl16>G)!YF-GG)stbuYARaM_d){_!M{1((AwS#Lt z+q9?!W=`|%$&%bchh?ozc93E!E9c5i96Sr6jH?kID25`>oimbQ`A8~sRqXsZGAP35 z@lN@Q6nIjcl;z8VI4R^)N^EEtF(-QG&6|5y2AxMzD;pI3s;Zq$=h{3dDtV&AZjUe+ zy}{ku&8U0YppW3U$h~ToD9T7~aR0l`p$Nt+O!Y)qSYY6-`5YoPz)uY09K;ff0|R}W zdc1ue(DC=JM?o?b20pR?DLGcyqo%e}#5WYMFc2z3GW8KSe3jAicl9f58FQu37D@~Z zKgxphg#(tW&pyvF%2$Xt%WhxoLRlFN*${Oe#fDHx{uVoo69N!>x?Sq_oMyt(^~69V z;({}tnoF|wtbbU%RxfucOh{d-eB%-Dng}-nZsS9*`c=FW>H_F@36=@mfVhN^4HQt6 z_j=l>hY*$v*bF*z4%S?lWaW1iIrWl{pP$f=h|>y(GImwdD4wH$LI91=v3`2s0L@~x zGQQNb9)X~ks`2sl%`#(Cd<`Jp&|eQ7{it^%SQ39()2&!dWYy`tznvVlv+H5Ibn@3~ zjbGmxJsk1e>8|#WrlmLE43@;(_4HP^&)gL-z$fd^>RH(r)4e4Yp2gL>qDKugi9CJ2 z)hfAPwm4!qDs+di=S5l^nz$<;cEBQdHPFuQS;~gHGWbk{Gz*;Zhb7S01Ir$K>VQWpsOW)Hnbd zily7U{RLV@yBQRmp$XGyP{An!xsyKsgRDi@;Yb4c2sh2vP3r;rDv2knZhLstDT9Zr!UdReA|&~^&+WbI>7 ztao|Aj8${##t6u!3pC({{D_<~BXjf1uaZjmqLh1L7JbEv*@lMquaxtii@`1HTk&O& z^N#Lo|C6L`^=m1=WYOQ{+ff$D)Q0~smXiMg6i%xOsLB8MKY?PL(J} zHM)m6GhW23amKv-IGzQn)WrYDwfi;s?p$IqiAtjm@q^IlGm*Qe z5Kg}OQ1O==sIq6eyI&C;wNOgfprC}mOWBk)A!Zsn>RXwaR0WH|D8+b4vvdkaYso_IlMM?i)`88^)0>)0 zhK}H_F^cC8{t=;#bEd<>&Ac$_(E_MD`2wlUPR7ovqZc*IpvNR6ZxX62$Pn#--lkY#A`+K(`|sOF8Uf|H++Y;YgqP+=pAi;^Zc}^gqsl6qjKuGZ%a| z*oikuOgukYS=+I8!3Dq7#cCUj%iq_ZVm2?ut@imh%uH)_dxL zB~qBoUs1<$wKcWFCR>?AZXY4#X`f)T$MleQlIf)f4n$ z^$=|b3wxI@>kiYI%=xWKxRPolbH8o$0$U*ux{Es237a-Dh*rWXk{1hPV=0#uhYhoz zQ14Z(HmUAujWg1+~zeu#P&z4s!*YiXegVTJcn01fWfGE_2xWjn*Cj#{KNmS=4~Mm`ug!; zH<#A8^MG={b%A!jrqnsa^wA-(1^gASS0q1rTnCBL&Gh=`h5o(6{|tPsGe*#UG;%`0 z@N~6Hkv#*GHOxO~I(?Xkz@o;@34{9YhS@*3H}y~XjhHK711ZLDOt~u8)Zy>|iuU0L z6wj!nmlW=&$Q2FLn}V}@3JK^!^(DB#K)d(Cb3w~}@K*v{FyR|FZj8_g^4dY`DTB!( zP|BKNu%+Y6U9~vLxe_yuv?n`qU?W6bEyQ9BRT&d?`RFIcI~fBC?0RddD9Rj5=Y%~* z!ZUj?l;Cp9sD07_6whkaO+HWG7k_^4=k0R@@pqIxb5SA;1r5Sg1dN6$V)#GTSFW3R z1TtN$yFs=+N&<11qY$dDyjYSJeh^>{aq%%R3TFtF0rOs9rACm|kV ziJC(lnHa`vf&iIdP8}Jo5_S;MW4;9;pzTTjBxT03aGg_nLzN z1WAiw0${;)I11J#+suj!A>b55Tzl3}zgYQuius4;TzR!_%{Qdw2KnpXP2Fj2Nk1)) zQDW*yq1qm3c(`}?rWd9<_YM}dnbkT}OQ*jirc1ztN#W}URnK|fG2mCi6)-zWu;SN#If=f746yqvnKbX+|9E<@#A|*r_Kel;vIQ7Lxsgo?bE-`nt;kiCnqmDI(vnVFQx}FxnjR2x;}>CO5P(& z&r1;aU0z-padl524}iI*STJ9d*}b_)5D+*D?8?D5@q|FrR8@OaHMA7ep}VOYE)ETiF!B4i{=4 zTU3P9<({XW1M1j1I*y7mS+JmCe!Y1dXMfa4IS-HPi0U%hoMkq&8bV`8)P8Jv=y_-d zw8e;w^ivDH*1E zqw0!;_BGlMiVac75ei_9a@drbym_mJXlrXnWif4>=iiKWlz)eINWN?zx8rLxS6lRph;A^{FGj;~(6+%aDnoDf>L(k80nFoMFRvm` zDqOdxTe>1K7`F5JhieVMA5q*Nu8D`GJU|1*n6yr7Yl@%63Z_Y9_$&}DH)MW}7WW6= zj(*Gv69*XecyGF}1S(jd)>s-0onvcnFCMl?Yv#j2-ysap{U@~ip#?PF@N`vzFt7NZ z!c5H3?c0Z7;qrhqUX!Wq$I9Zkwq1v>VB3n+Bu4n_B8nZ}e7w#vT;xP&4~Lm4ywZXEKe-h)VfX!o`G&9lNbdb_6qHobkjQb_$1E0a(y zy4f^wjtOcjh*~>*8m#3y$&3^@K@6;ip3AOTYGn{tzCMTIvc=c6ak(A#N)>Aq#1u)bvt$%WH z@)LUpM8>kgVJt3+^zpK#3z8r9E#az7~HtV!7?dtqcZg0S^{bkdfXBo~r*4@9+{ihChH#bsu~ z|2x}sZNT{+chIOfzJGQ-!h1mv(wlPA2W7>zU%)4&ljqnU*w~b6&4V`t9+tWdq3Zg` zLE11)P})>*a9vJec}#(ZX0vX>ngXo<$dQH^%Wxj%*jiiPrRaz2J_SNVlA8Xoq^cAU zT|vRzRZvY8OgD7j`q!V^dR&tZWol@+rg{_wA3kidYSk*E%#3^=^n>uQljfe9Z?I@l zF_`R)8)5*I&_*FY&;Pk_EfG4RC=T$j^rZZqfPbJUQwv2?<=0zH^={!^?2l_ME9&q^ zV;B%p=7In6mBP!34p-sCM5^+}hT2+b`K*C}v0y ztc5XSr?*vg-K`AafCa`&Ybc-!ei<{3Lloco&xw7-#50;PfLU;mwngVr>B)du8id<{ zzwthnXCj;xGi}s5!2kbRONE^EWri-scxzZ(Vp?_FynCY5zx?65Qs3|S4iWLbe!QDJ z?L$D$*io_GPaaLEY+(^QeaGq7s~7qYD1DW3?yY{y(9^9XF?xn=tj~0rJNQxV@5aTr zfY3U=4DT@UU($O?Q5&BDR5-K58wSm6r~c4?`A@G#f=P1sq}rV#4Y|9+G07Hmk{Fc3 zOXCv@8m8|^7{G`MfTWO2^+TiZmKlqg9F>{Q!8}Qsoq>>2C#T)mzjF4gZwbohSZ%!# zHaX>w1rE$`SVXP0n5Ok=s%#4Jc5gCta!MePg?Z5ZQk4%?QxhQUnz12xP^=BZ3NLc? z7$vlVlwJG8+;4|16MvVNpF3m54vmfRcv6Ch`Dyo7--M_)!XZNa%bWH_63RTC$rFqY z%|eB(c=$M~DiFOq6xeDbM^@z|F@?oh6Y{zbRZB1;nS%)X9Cr}0|D~`r)7vmW_^a)U zFy}{O|DW!RbIy;Fy%rH=3=)JXLT)a~SdW82LHCG-(PW1RH%ZIX*H~^$APM zZUQ~EP!r#Q-b7UVyg5L2$}19KpM`xLM~+0ol-WWllWDq&2?E5uHF!c~toPBOo+)C_ zF2!qR;`rJHKbqclyRrGQhDz+MH!TG~JN!F1@OabGSWh9gI&|yq5xm{qs?5Mc<_%-FPPsRePp|Li3 zQFRg>(0ArRCAtq94H?eFIPK=j3qsPfAN|I_aa`||sZ3!;)Ag2%Mo4&LAcj1J4c5Ux zyBci#4+#k&33f)^;xqot;W0FphiD8ZdxHvC43XsHjMW22$c2!_c|6wyGYH2h;^Fp%`4IfrIG06Idp)x=BF$Ba-ZWH z*;g;jrS8*^y^1+&9*YkX?ozP;;9nj(fMz&7$!S=#FFZ|Eq|2vi-=@n~sY)VNe*Jh7 z>Ad`U^a-ySwSzW^4q%F+uJ7Hf7#E#;rE~x-eXQ(SNTLr7)Y5tX(AH_xXgzeQ>z_>y z0U;f`{$q?_QY=+OD&1N@dcj86f$2gR)G3nEE9q9B!DB=jA0 z{l+t%i^yMS(c=0g$jfiT8nVcq_gbcYJ-%P`livPbJJ8(=%>}AzO0m8E{%7n0wv!e9 zD7?DNvei^M6$c5(gZ%Vm+_en1%4XLaWjS_rfJ5tXgJQ}70Xzt z^%Y68W)1{nlFno)K2{TGn@({JjnT#UA%@c>DqWx8jsxarkbqod@aWO`pkud5m9fP1 zDY>YW;3qN%>5bj?l9uJ9HkJksNoOAd#0%tqL<=>EWc&0PGw#td2%9=Fh=-H3gI4s+ zdk-E2CVhU@Lk!VkvUXQE!~_M|2+n+i4>O?PCqidU&ZPS?eRrOcrg=PsB@!h9+J6@y z0w-g~fwes?VNnEdL!k#dyDabKmR_q%3`9n;tZk3u>DStA;}o@J*Nq6?iM*_ zN|T?J6yP7+MTftOM!}$I>FoTv@C@IUl9_)$dvt2pypiY#Ud0?D)Qa&C^n*az8HrMS z3@{3!jC-N9ZoXZgt&1_eL{UQEisH@zpYAN21_IhV1iAJS{ok_Fru&De=ADI~65Tcx z!XEQUlQyIJ=fMI~|Ch0hn?3m4fVA@kb~0a#l zo9T!~W5|ML?F88R^l|u7akOz;8>PJ0n}46U*xw$@4t`)l$Q6hzL^@MtGQ_`jxP4g{ z0wAXMYXC1!{6zz`@cWmP(hzqE+GXWRj8f|j>&=zA(BTXF73n>%YAc;E7v2QOW{7PG zC?pJ6E`84zp*(2%EZuYJA~S*{_(YM9q-Eeh^F>0~NJBC?_aEBwu=mUivCpOf&z6G@ zjhBcf$)7VwzD zDRs;P)R|E3@e^6Asn_i)T!MAT)RI zu)IV{W!I}kH%F+epO2`pS3e{(y%4?^vuF3A!PB)qz2|{!sJs|$U6>doYkIHUm=^49 z5sC+|fwS}ZZ)ApN&bZ)GD~TCFktZS}qQ?FSBv7=TTUrDU1TuxsoZ8@y8caY{sKJ=Y z#SLZPJ>~^D4L!;{Jki@2tzWU?=H8yBu6|9Y8=m^wZ*u1joetZ+&By{53A4u(CsX!fr^@6BRh zsQCLF5_Fy&#QZP1OeBX6bz3I(1?Zmz4Q>dMfW!#V-MJ4NDGl?yMFWz!PM>%rdiZAv z1%77>?BMP2H!^W`)hOA;SH-{Jy{lp3BqA|!eb!}goU5p63Tb^deH0F7c>&FsgYEa2 zgJzq#g4q1}v7wcQe`r|cwv%ezmqrYcL@%1NYYHX(0G;vA&t4vo7bSf&T{_|mWzm1AQ_>3Elme3K%-E1lC{FrMD@V6K!rUgs7NzaWGL16NT3m1B2 zk<8K*PF!?QTi;h#K3XvR0PGHJ__DJFo*&+a3M+8h!DBWQ*(aBp`TX9a$8k?TDl2ho zyuI_b0*ne8p-{}dLk^%ST!Qkb@@nnjzn`u)YCs}lh<0BmQvJNB!q8hZ2A6yha|1r3 zi+1PQs7cGBogga0C6SIGT1FV%XQrm4;vpUZ1Qb96lu&wN7Y7D#Pp^r$C>xolQ~>SA zB+jNdHQnjcXE#rBz^V=?RuqrB>Jq9~QC3o`XPcEcj;*=TW0le;b9n)~_ik`0cABtO z35MF#@H^!FiM4vs&E4XkX#RL|X}~p8rz?HTLY;iaCytZ^#w7JIYols{uGg?OE%a&& zNz8=MN2T{CKQA8jVQ-sbK<#2Oh&lLJixDw{AhbSwc5n8pqsPXtE+C-441hxZIka!z z#ju0@18K_p{5rdN+TTxqe}f@2G*Vc=IWa_M5BKXmxf!O-;79Q$)YXCQp_VFGz5n*{4(=yKn78N68%OBqvW@#^O2&}CA@@dK9 zs|+u0w;30b6?!fia6%MG47wU*52?cab2hCP2Lr#Th5+BhPN&#vkvp7YE;Lw~W%u&) z!-)Y?as_H4(8u3m+anAk;Mp08g;n^UjzFpfJ$M4?!(4=91~nu`%A`YbHk%X0Lt@o? z_o>wb=JFb(K4K-HweJG34=x*>{MiReA%Z3?8b}p%S1+0E3e+cnSa6DZ!Wag z&Si~%78L~Ffbz@RE1&j_!HLUv`uGhAOQxAzG2eDFSAB#n6WA`tW?7#u+|yZE$4363 zalYxD7n3FNnyvR8Ie5`g$6-*1BFsM^-IrZ>I9ID%-Z_+1z<4r^Uld0lp2nblIz%`# z(=r(;6fp54FZwHG_pfCk&^Ht@N+xQAK~m}tDFPhcDb9?vk0v}(Qt$?UpxPb^`FZKqmuspU8Nn!8Y-oO9uzqp$i zFdX>z43{qs>zGO78)vpzvPtl;0e)jcvZhpG)(RUO4Kt1C@$r>_Yua>dv#B8plROfM zgIztYyi~!DLAVy01A6FKn}kXY<&YB@BT3c^RO{F2m(t^YT70uT%?XHd5Z(31EvbuH zZ0lv{<>h7cxWc)7OV@=eLvnB6m#Nm!Q0WzFSWLECxV}QINFU^TpqTi~U$}mj=(EId z(&rD$L<}hH%3zn6i3R_M9p4~S;gmXZzlkyrv{I=2NNm(#RR8neori9f6~-L&Cz`_7 z#$>Y0AZfE}Wq3NmTr5!{S;{!2B0y9x7@eQ};F}etG|ZRwDyEH}?>EC0@07liliF^> zw_ox4kh~P^ogu#q}07kAtxmzO^>C%6xWMDO|d(hnLehmrYAB3Kq-y4y_)h~ z9g1Di(^xahb4z7c`zqJbQW+>7%rF?jPG9b4MN1c#F;NYlo?BckX9>WVp1$W$5>Q-q zxza#U#*D>LoX8^b4}QLOE6#LTX|N_^7vNS>L>q_Gs^lr0B%9+5Cg6Vs4Eus6#pEA%a@M!B^V}Ma ztPa*eDy!qBK*B80fjcY@N1YH{E0-R}&a z4UZODC$@yWs%+90g;G=qh%dW@1LcUpoSeNv@F;Q= z+6d7Y-MTeGnvp(qmzT$JTZ{oBc$Aq2`uXKIn+Cd8y)yO~E)0Fk^FgzOn2yAjW!9A$ zHxa+Me5N^C?Njf791M5X+FN{cepG}ce(2{xlj=NIPknginZT^0KED1|p3?Sz-;p05 z-`w0WxBSjIEJ4Je&HoPFvrOn;zRf#t;?|=hugWP5`eDPqJqA^q8WZlF>K2{*X0YiA z?tpAsS&?ZOXuc`-s}jrsL;jtVDX?qjvh@;3CvbyUn*dG1$q5Qs5R~l)H9%(g>WrnR zI5RI?vtw^;d?^hVhBW|p3BOFT{Y-z(kHmHw#T%@#BX6uU>fo!n3YUBilL*dB8V2m)#65z>|*@`!ppBZy@#TFQw&8O?+E^gqneeoDU4s#3PoZ9ouF~mwJfln~J~WQ1$BXGj z!HpYgQ#OlBSr%GP=&7?rp+Hcjq(_x0gi?pr^?OdH7iW@558)IhIwgomDRy(>F7^V@ zWqxT@84k66ARTl_Vxp8Sq$R@%6c$mGVWlka%%$lIb^=LrtI8bRwL0V-@^;Fhocs5W zQ*VG_tXOMfz9y~7Z2ya8KX;HsXOtGLxis?m(zB+Fo2Eone}A67C(}N1->=RIKGu}z zH2r?x=LJ0^DI<1XFsq)SGh?Y`TxIK%YNO=TH9AXTcCAT0S^1OIxsiu-X20JoGYfA9 zZP-|GxBlks+X2#E$)B-02Af6>D$f^mGH=jAzQ5 zSjzB|C-3ekxHZirfq^x8X-$$LTn;X8`$4NT~}Ax7dAJbcB^V+JMcFG%0;$>zDhM?kzU;73wyA_8u$8XH<2V_@mTSg2Ee!Er$9XkG z|NZoJWvU?mU_1})wRw&H5kkywbV9i{+MV*~Tb&ku`%!n})9tQ{27qm}DkT%K4Y3@B6VK+=Rr_0{|f zeH&Jvs>BIdgi2uK;2QRu;DTKSs|b(BKT<==)o6&h-ucXcaOIhv5>xxF%m2PSUL`i9 zpbg{k7P+q9j>m7&e-M+?tHrrZp})KPdLFv2d)DjjM_vLRtH@ES8z23A&9Va_x73j3qa5wz%Hs2X6&p|0Um!5}&}0xre-xX?C4 ztEdskmdOQl$(of?=;7%Dg}wY}Pq`{Fw`zf1q`5akNdV`bmj_0}gN zuve>;*5@^b>5V({tbfJ0)RAsudb!=A5Ngy6{-reQ-L6O7-)Fzwr6$w}qI8tG#v?fn zZnHNODrE{qn{4U;nb4w>=S&sCrlng-rLaNTX&WD5*3j`Sbi75?zDD-0L zUB#~qaVj`;27s-3dS#*6NW7bThGW#meJ$Ud^{swKf1 z_^3(iqK?IP#VG>wqfVFk@^PvQVK->ig<1sOP^zDlDCq$j8yW@xc66?3sPm%@5Qd)< z7{n5gsz_^(ACE67^xYEw>Ogu>P(r8EEi5u!nrB~r*VOuE_9eyAF6M5Z{}k0J{*)AT zmqt!$${QPYx8Tx`i&~_to#~TM)-|gW*np0Dh=KpJ}}QC`XRnr&AC zvOTHw7AOpSpe9YbQFV^!KtObb!RGz7{KzTnU7QMcw#S^e<7@94a4?RNQyis_!1D;V z1o8^RbA>R(fwE>Vu<)hgyNp)Ny(KlOq{OnaLRbd|>1VgsUrU_i`A^9eGG;&~ z5sMaK7R)4*wA>bjh(>YpCdx(5P9)m1OtV@+g^HXkJw?y(vcy+(q43ZYwirS}1?sAo z+`Q~ZNN}L=!m-BWtx6ZZK^DimQ_8^9 z;7;cLOQ0X1Q@{je{5rqEVR}h!hutNx6z1kpqCv07I5-kL!vQdt%(h|-!jKY!DblHh z1a0KA{)|GwX;vXKz5R0SlmZoWwgRaYoNKju*)sIcOn9_@H0iOCB3d+e}DDP478%G296XnXI78P zAb}XMEQY;GT$S8OLtHm6OsKuE44R(0@Gm^4!F0O0|5-r)D z;>tzu&Qlo3ns)f#=#Pa{c*CAd6N+~Tzhl6suK8?>!zgu1!`WVQI1{&*cYKeXFp&{{1?$`W=RrPJ^{|@Z-qv{lVf#jo{EQOgSP7 zvDFK>haP=(<6ldc9?^^uHUr|yzqAhMlPxSinMRalE9`VkblV;HWmQX#3EIA|&lS+DvFUORID_8kGnXla51nRAsaA3S_`9*wCZ zLc8g)5I5wB8Plh$s2==`m`0d!Q~-JS?AnEoC(1>|>{3TD+P>#}c1ghEZapk5Q_}_= z{AUlqnA&rK2b`8bNu1a4{1LP>s#XA{zbxJzY-roIt=OeQjY?(p@nA!K5S_~qilah? zM(KK0RPUBOL1;riC{L z7sMF!oG2f$e@1wqh{$E!^j>wLnVIz*7EFjIE|Pq zr@OKl@m+UyU}gLhQoz@#@DBWj05e!&dBc3Sg^12j=peryHizM>dxyub7rJ9h-b6{@ zQ(+y50$%V9CkWQUO7PVd^Hx&4N2^{};5`GFGn9=v-F&fBFqTbov`nLUw6@#P37gu^+4a7kjK1ej^TwWG15pzW5CnrI1#m%1RaPHi>$q9-> zhoV9q<5klB?`Cgn6k>pE%S3E}CUMAek{|21;&D@hQn(oR!9v9l{4uJD;Wi`aIhd)# zxD|BN{xTb+aYqzOJ%xRZ*ySQ@2_*aX!-L{+#Xg(9G={WSFqNyM^OG1`2@FG6hI3MA z#!(ySPKTY&^9T8S%pOu1q3Pq^^|O-L?{v;OfL2pj{V;ni8{vKOS%%Dsf;cg89?hu0 zbK9l5oN9p7VWvVL=py2veB^{}%1T>PtR~6uk`7jy>QsH}O#DtWFMII&uSbVpQ;bY@ zPKi5kvaBh8Y=rlds`k5EN!sqgo~6pC=Ja1x!>eZn?fqF|asV?jx3fwg3woVCChKmw z>;h4~e}AOa#Jg4$4F_aCu5c14uJ#WZmLZnxNj(mwhUkDrH0h+}f130?UTJA*$_1(> zXMw9}Bc@1QGjW>4YQWNRs5n4J31xyh#wtQSQcS4`w>z=$Aw5G##fS|;mG}PHoa$J2w0M9{*pD9)0J#q3-+rx!%|HI$es4C#Jq3H%47Vs3K@B${z)nMS=xa z)>eK=8^1v0)(C77QxDZ{>z}6B;EHcDR6Q*PV*rRD_k%~l`f$oUg$=o(JP}`O3^JQD zCzx6CfN-2F@*0iIw;%9>B6~+NF+9yrZ~}|a>Ij_tSJm-gD`G#OGmnTF=^5nKBpl^( zkYICU-Q=^f-^(wmuRX_F&4d-t%|(l^qS=*^6M-&}r|icA>bcJ8F5LHH723MfZ0e9$ z7VMF*c?LsaFn(ZoRGK4qkgo7LAs*R(g9;yLu}!D6glt|yi+u?H0d-nWXBaPBla<&W z-dD~F8e}#t<8`aQSF#CGOU3>5JEc+bpx%N{ucpHPVukLZ$(9wTb*6eO=%hX3>$NLh z?8=;SFnNrbm9}2%Yxf_AoAqp}(&^Iu#~uDiU+<@Cr!(TpMpdFlAN}>YLbp1*YK;Mm zGmC0)u*zJ{9#zLf#*c>nuB5P(s^`DK?*6xl%0X5`(7j1+m2+K<3x{CINE1ED7qu5M zN6yLu;+G`TNaCc3T1{#Q!MNs54k53@)R{9Ep-!#HpO-F=w5aOomFMz*`R>@UA7`A= z>fVMiHY;3eRt7SckK1-o6~i_@Ksh6<6>D~>uhXf@3q8nUxd>Q}$^xcB?q&UYW( z>X{K4I?3~Tl;zUs$Z^qUFLx}=b~%>f;{M>gOOw_0DGzoooU}7^($mmM#dAlz(muY_ z`{*}+8(EFFdat%1sLAw+qyPHmmeJ(%l{yEx-NQZv}fROQd;~554}jOa65& zt{pvf=dQ4gQ+E${S$P2-#D72k9GN~oaPqlZLwjFbucxwoYjfjhH{Uxmf)+dbwH3`< zWhLv;bX)_4s+o%jNI(l_a9&EwgoQV^SbpWA>ouyoJc_(UeDZ?v6&)8KQ`rm?j)16v zejR4xNN>BQi~f1A1Zw1gd_YZj2G}suo?(K2bU@Z2Y6d{uQ4Z)%@4!sP#ZP-x_?%hp zFl~Tsqn%(V(X-FqNuJ_a=$tq=X$WVu`9+KGf)jui7-&QCnR!}Y?-t}NLt3* z{-L-8(|6z^ksNxV!DRdAy?bjOX5EvVRg`6r70eHcK&tJg9!jGw3!KYM8VmCL;&bOUEUou!=VQF!<-A(u^J@L!6 z9()NCGn?-JY65W;ueaWwsG`}!q3l|@$@HAPuET5& zC3sw0KT2i$nf`ChB4Y=Q?`^Icu{9iY7}jOX5^8R<)8}q3WLNzp_-Sc+mk-}bG=KOp zqxB_$Et{SQ1kdY<6euyVcxxl4;3Cax^Zy>wLmFOb>AWjcyUv>lL@wJTod?Y+?bZL?{Hqk!>7>T z&-28*Johr+qAY}v)FjVVy)azap4om$J$CsT554?NoEELdJB&t2zz)HPmGms=H=Y`zUz96s3ujI)4HbNZhaknRu6@~&Mpj3f zWF&?ewphDr)h{0Zme#NBWvw$pO3a533->5hui4S*H%Gi2`7F43V3#K62mZ2mN6553 zP9W_Ci^^J(MSPc*}7-(RcHS)`^k+5LM#UADgH7@R}9V0}b3-;vS4; zCkI8ItS}4MEAyJC`IIT4=xF49brAO`y3}dq7fwk}Mc8-iE*N)c-AyF{PJ5~iECNBk zBWz=L6Dy?<5cJW0;k-Kiox7Lm+~Q7Mx=bC`dG&;$hK3LK4)C@{P*|81ESu3-2wZd3 z4d}h%#$+>Dp#-by`*?&HDLM-$Nsv$aEsWb0l)9f91vaP64wA6ypzxWp zhg~v!d$;l&<-p7YY`8sROkb~*Yy=CEeX5<{=hx;e4nimQ{yhx%UFD!(6B)p_MaZ&u zmbx9})l+zcw$DTlBLJ7=SW_de^MXihux>TO&vV!jk?+d@g2s}UQX!z%+=x#_*_sHq zs(bErH|Ar@bM9xUNI6l{FDAtEVh)(4-fH3xZG05Tq-o zv**P@M?ZW0Zr~(M*ZARP#l3T?9SytgG5QthVaLo$)AR*?n>O%0X4C!1GI8Wi*dO}8 z)2656zLL~0R6J1LeOnlRXNKsW^OQZ~hc=TSHItL`YFj8Z@ivY;?3#IjDs;!Pix}~1 zn`kV9vJgXryV|rID(OdYv`a)3bwiQYD|-ZGm2NK`Z>V)nu53a`;KD+)dWj2*DF?cV zb$JQ)86Vc#kK0;PQxj0Ch`axGnxXqFY2)j3VwG8Plp;X^OYXL06$DEjT}5vS?tX1$ zGt>J?ZmToh`=+vx?#%2ZCPYx?aD=Q$As2zI32Q1BlC{xOW1J%kv96+hvssAMUePdR zU(0#;Fh;yO8^2rqM~3K#DVPvt^LJx}mYfoli2<%quoB>sDp9XIHQihotQNCJOLA|> zfd-rI;G|i)njfFaL??9<$0FOz57CK>E@$< zU1e}f%DRgB1yC}@=9INa(5Od)^!00RdDHs|WgdP#g8P>P_zcdh^6bX>gi9(me4_yG zRn<%LD~i;kx(|mb?ZAHz)9{>s_dh3l8<}N0>^$|pEZREz_r~Y*j~cl~_nFqXxOe4! zwY>p5Jr)g#-LGQhS){gas=bZ524b?|bGVWjFdOO<->5`jtMwqxk- zMapHHuh~T22b*8L!N7rW_F1A>4D+$hmQ@;)-7m^^-2*4i{RnYg-sgvm;7Z3(x__v%G>~i1ngouc`tX#2Tce|dy zxz}{`-n!MEQ!u1cN}#{*Uw{1-uoNpk;JB`}nl*)1ZV|HEsIpy8FX1i5Zz`#0d3kxd zcO}LN*e+nt-Iud(oJoDfmJ#h{b3uN2+ixe2Di}eI1p5EE3$qNW7hQ8V3n->=4tFiL zHzBfo6a@|m(X}CD`!|RcwJTs$WSwizgO<}7xXKeB)=q%C(ymBPH>stWhoal9L<_#8 zmJfEwvCr5un4^*{S(rR1MK15ULOV&~m3hqUXf(6YbIt{2&Em}LFj5yni~$+kW}%gp z)q_eDlS;t|tmPv+*$`=>&9MF8!E9x##HrTJ{Y+C_S`djeR($0X?}>)nO8uKMPy{@G zp0sPvo=EB0cn8K8m0Xru)nxT8MGv)VlJ~Pug=wKhDuAOOKk_06YMEx~##ccygg@VAr|H&J@2y4g3QLG{$ zC!`D=+WthjA0ftSKzpa^Qk#2b-B}2N_mAoXYLGnxd?Q4^iK-a`AmjdWb}rh>8VoE^ z3hv@#vEsVx)49d?_l}K{B$SST18Nml-5(e>80_0dOEt^VE8FTB(Ta%Cl&zaKZj^8t zNC~IUH2$uwbEAm@!ZlgbJ~_kN+f)5+Zi01tC}TNL%4nHQT7OP~jHOgbrQj^Jycfep z2HTky7KdX#npKr~lFoW-kTU?RSQ2Pa?;R)LTWvBMCq~ zq*Fgl)jq8}i!j#wg1*Y+EY=PVVU%%%xxZw{5r`>^N9H~Q=nx>2!K9B8J}{*(Qsl6T zP@XIjiHLSH%v!l}rI6UZILu_%4A!qc8XGRX=Q_Ix8AO+_0bvVhOTks{a4UlGTqjO0dpEJb!+`^$6ngD@A+3M$VS#|(tx)re{7_ykdCUc-yJcQNCr z@H(i1$}Mz*zK|03SZ#>nJ`76!Zwc#tdc-I9;oaFj`yV}t>R(}|?`o&=Zb`!E;ybfz z9*zDs{K9v$&Zbmf-!qR(Y~ys$h6PQ=D;82t6I+lwL*e@LHB zGCLNs!#nFA&THd2O3ayxHvN0P^2y|9=zQRD*YT#NQx$#s*z|r}=1CcwcAODD2`UOQ zp7XThsnjn!!|L6s0{v~bM-&HGlPij`1UxSb$!`eLRP|{tJqX=BOsIjSgarVLny1@I z^Z>;?;8J_kv&;b*&339FTY*$&$+n^QrIsyPh=Y+FuSH1QwC9ouHp#~VwS{bVhk=nk zX45MPivuAPN%EO9Yi4_sEoXDt@kDiOeE-oB6;xJlschGmvdR&I?qykWW1gOMdc!dM z``}n}<${!OXbsCd2GuWgkVhroU>>Pp2X0x#E)~>Kiu{PLH`qL*g_dJR8iI`i7(W;T z+{ljRI1zH=Ym;VS*|Jx8e*-eeW*3Q%7diz{W*Qt$Q_n^!7Slfp#SR0IW8lnu?oMB8 zdBSm@^jAk_9p1XNNmiBA7FCu;oK$S!Img$}emn2xt(!5;-971J5u>r17q7pjcq3Dh z8}JVMsZW)>rJ2*Oei08g;qqdpx0g$<%4>V?Ju6LVxjOwx(D}^)GGwdh5nsrvq1|64X$|I*npZLS0=4l<9=%ww%>?0&Llc6?#|&F(0V|f zwv!@v&V|7&Ia9Wct9tG;o+!nE!&9j^a8EUuA|);YX_NT<*k}r8ly!RvQ29qpt#Qh765#sz~WH^6LrfGq!!+%~N;a|!v%Wax}c|e9mZrT6Li^^e9 zIWKN+JlK4#U69Kdwq1~i8Kus+RfKlS^<_eC`ZwTcYuBfxre=fDEPwMP)&3<-k!+&{ zvH)(+Up|}~9!i3weKgY>aA41{akAa%`0?XKw+hR42lQMDGR5NGPF*Ntd)j@uv@}2W zhU1A|dKdY-AlQzehN73R-d}A=dwuWZL`uZ2yD#0NZ0pcX-bC)lfSNaz+; znxx#IWt*R1X*zCR5*j9$e!Z}q8#m@n+ViBX=csalPU(#-F^-B*f6)-jX_`}L*9~gR zuLoF;8@EO530GE^j?>|0U-`hrF7vaE^p=Xlg?%x4KDVwp&d!MiN1F`39@yTJ1JK3MNJMRh)e2W%W6x*$F zpdMl|z~k5X73vtjXk$K}EPx^R7Y;qhT_fX_jeBX;wi0sOH|9)8R8&-oYt5w#XB<=l zg2yUhiK;zIK7}PN{kW^whttLXZXeL0w|VW|j*0HM28Z?+|9zlD#p+UQxLc*cjva4a z_w+cR^Ig>#+g5&GtN0&3vZKQvnWOxCPwTbYMe$=_d# zOrYIQqNioA3g>LDBg3TE+s;rVEjINiFrPUy{pZttEl>!`(oOMdIFKMCG$1hEA|(yD zpRQhZw+97n%aJA5!SywTtV!^OeMWwkgh|g{Qp;T#48(+N6PLU+@xFFGcqUIe5z<+q zqdcuo+?{(_3YNxCdTY*|yxtOuuh)yJCSQKJ90FgEa6r$N_j4UjIMAMp>92LybFA9X z&S&n_o%1~H_8*;w7Swp;qAQnjHDSW#&MkdHNnl8mvF`^LQ71{qF5b(!7`2fe9_O&` z*XkDP5OK%qlKE1oFs_}$WKvi&domU|SvA!6nH^90+757DVzH*Z62hCHoaDV(&lJ-# z6a{U->`p@#k+0Vy*brK46L%MvE-F{C1(H%*b~>0B1CEIM5MJZwEU_b6J!MK|k%YH$ zVk8W^M@$!&0BSC#P&Au-m2@w1@FvEg=}^R?qZZ$b55iO{>0L;W>$Bd|HT_4sB&A%E zgLJMcU4d_n-TmK#^mFB!e;=e@bR*2qCR{%9@L~HK@1jbJMyOb5$1PIj|Ejw>X2Qe7 zAp3M;;rTZZd7`|`NMA$L_kiZyCsSnmQA77d z4=f@Uu7JDtG#w6-Qq7=6xr&EX6ND64Y+EMEZV4F@t#oQ?tGPELrz$(HDaXjVlvb+s z0>Jct0Q%{rdKG$PW5EZ4QK( z!0+gN$2yR4luh+I#$46fmp0*ib9yCGxSi4qugcDjcIV!0@Qg_JMO#U$Q^dobZHc@L zM3KOo(?VI z>5FqHTap7!ZPQxVf&@l7`-P4tR!v71->(iv}KNjEZHkUo^X%EWG_+!*3`?+{mz zZ!3*(Z~_cUy<1fkHkPP5Sy9cUXs(H)UJ-T4-5v0L<2RiUv&^HIu>zo&JM`t=OP|w! zy4OCR)3;Zzlu$Wnzn+@;{)9*MP}F*&Q4u|utWcDFq6EuoFvS2+NT^Nk-erle+Ni93 zS^!54_{HaG(g2#zb@!TAhz3o$@HR4X%Umfi7Rwo{mBMSj3i`n?G}mx8B8!jhJzAuD z-c5Ne6mj)VPGxQ<0BdFl4l}9NAcHz)5m1$>sAFAc*#GB3^wFDk{+Z#R>>%sS2DE^WV%&EfNeYW*xqA#a^s=M_Tuq zVDIWWJnXnfy}Q!|^;mw*M$BGe{3zYX7@HJw@HtsBlTpB%M3X1WaqF0JBL4+InGBBw zHO6sQbPE1kJ@9`y?OU;-w&Ch=N`FO8T#$sE!&{T zW~|jg|M*>NLrcK#IdyN^iQD}yZu)ZB+YAoZG^6|n0lzlP_*(&fy?gL;^(R2RX4t{l zD_`(8efY5Ok9>2jJZ{9;uGM33OPwF3HSNivK1=GFjhvY{bY(0@>cFhCLl2&Pn(9(q zl$^X?fl2WEH8vc>{Uq}to*t0lgJIWh^$kMs3NHSEQ~sqq>(^CFJKmv7hL0g2DfymI zhMhlOx^wsL2*91^h#<33@dkpUfk8RWh6_^(u%VDh!+JR-OHZB)0MKd$BxZ}4v>nH(RBT21nq5f>Pny2 zo}~FBoo+ODpxzscIZX8pP$~-d(JpnHN~Oy>bLPzG>XllJ6=UHyorb8d`95Zz_VD46 z*fs<1Krgm_v*M)R7avAK4g$;@@~);1*c-_8HUV%Mi-=~5<(-Cc?LjUx%jrRQpw8Fau%&AR#Tz` zPEO%gkP3z{St(Kk@1jmJ$|&(J0KcuutqI5_S>S4>91PKLGoS3VZtq>OikY3${7(l@ zn*ZX=?M&jR%w=b`U>moS+-qHBPoA+$ji&$+t5gRn@UGAem7mbk7AC2uK0Q_1L)zNf z59#i9$F#Mom4<6X_pyOr7oIEH&+_l#KOWFiS=D)3lQ4*C{{^O&9`?k7z@O8;!>CS? z1%h1L0JP})7!1tfZq-ID*JAv4lLMj!}=C+rz^!3K%;)((cn&pUqgh2+yC zi@3LPe^^&hGf}Q&@#e8($%CI&%0UDFXrM{0|yp(oR!O@m6pE#EN^GW-OcOBL2w_ALhsIG8bHJNF&&thwrAnTSc2KQ+O%VqrtSg|POj^v=WIXW71a zxTfq6{CRafdbjKBd4kuuSPoe!7q5u$PTXBg({+?2vqOijasm$|KwQ(9q$W=(g@vS& z$)cHPtPi^bzVa$gi8*-Ran9?xCJY4lxaDf(xfxlRaC?#1N@RlNM`>)B>r1D}-Km|h z^Yw#@vnW@Sx0bzm!*b6B6WaUor(LSi^bJLX!9ECPM_uW};mx!YIFNFAgLE+@NcwHJ zivw()@pDE33j*PPi9i&?xVMvPsdM7}_m{NXW#^3ozqi`Y+bT|0{#dAJ8AsjEd1Si7 zs!Mg@u9OEznO2rA&IZkN9KXtV_caC&&&0$^1Id^pyyL7{83_Tr$e;l0efmwP2GJvUh5B z%0pJX^i%Fp^LTR1fPW$PD!+EUyG5t1cBoeEAnQivzqgxt`;PhEQ;(|eci!V(K0#$| zlhfLR&i*;Z#T#r?WU$I2=9g6%82>K+0HzZn&>#I?^q?K26^9N3d z-FWpjpoD6X}x*zOW1tLh)5%$--sJ28JcU#S6_II;HPOo<~`eK>S(q4RR?Yomllb zIG#la1)e5PgzKpJq;WfY)-BV0qqXF<$m$(*OG04dDn6qBnTq;1Xs{XHl?n(TQ9CHc zK7D*Jq{Er*olv+%Ka&!&HpPyen_qs(3NEFNM(BuuvZ7|zY$s)0`iM%3$yof)EIR&V_Lco^R&TPQden-R7RaP%FqMR35_k4V zqHS}Ilm7{hQ+7$^hSS>RuN&iOZn@IQ-g5PoxV$&F5S9F^bvH0b`cpQV16&Gh3Wv?w z&iUEvdvPfM8HWRx@~K23hz#UB@i5A{SK{(ubz<>!c76-#Xywo4C$<$aZy$SGyBgqo zX9FxB9^^VsvoY$>l>M@BSIm>Diu0tK|={6_tzY17N zk``=7TwI=0x<-dvFW)NU5yr+xZ@Jf478Dj5F0bdn;0~5h8-w5yC2)G{=ugyrhD_tq z*;V=J@H6x+gL<3VNMbn1g;J|UmBJznfuNm$Ld5XRa}N#AQQe7T`XxQjNvW_gdcPsp z$GBlOz7LY2(LcVhA``=qJ7nOxaRoWb}pXlBtVVjlGjVM}~>*qCd!pOYut?4-3 zzWMLlschbqvQO9_tNE*Zkly_3!|RVDy8rrRw}?erseU)RyFHMeOZR>1vqg*hx}PcN zd-1yU{e52XRmPv&kJ z)t!VKUL2OZO;Oy3Q#=R(Z8zJsGsqltwvy4#K37;!(DPE>wI9#my?V}^&9)2A95Khi zizmJ1^qaS*u@ZrosE2cSfg}e7peHaOy|YNTNP_Hlae%t9FoYy%)0Ue~Q1y_QS=JMy zJW$|U))&8&w#(h!J$=>p&1}27P*p?;A7*>vZb@PZmxyFkh?Q>9i^qjai8*b^>F(XD zmtFRo{@hvZbY>d|moPuH*Ez!5ZvFVm%)QTG4zyED0ULu{3OclItsy}@y}$h``36T_ zL-5j&gAz8X91z(o&VYDli%W_2NHh))R+s2+xEPzON@{8C^K*553uTV;mqt`FEo44q zl*tS>l%3zY)v~=#2yxmgiL2v=#Kpdc=+mP<0@dS47T|$?3uRcrigReiqNm4?5sWK0 z#_V+dOa5%5*~oWjOQ$NLv&d5ag^(?<2hnY(rTudgvQ}`jXKSt{NrMy}ERux6MRO9J zqk)09m!DfFQP&8(a&D*P`qgRGTRV4FvA7mAPjN9meQEsZ#hCp0KBs>jwjZzEHs9O( zu;%+?t8XBhP|Y6SRyVTWl~pgQA9mi8X!V7P)nmtR%nZ$SqEI#pVF#4T)d+If%pKBj zf5S0@#M0My`^!7l2W17yROZ=$8T>eIfq4ck$aBI;Edm^osx)rgSTkTtTb3qEkfCgX z3rRpsBFl+j%mypmUMqQdI$qw!43C!mSekrC-O|7f9;3QTgHJ}3Q@-& z8D|;owP%Gkvpo(`5#bE!W%4^rHE6hg*zr~mVoJgpza)u1g5S|sk$alj3%m+ply(AK z3?dF`vcwwy$?Ryk$2h|O9Pj#bIWFq0JR_DsG)WsXX_9Kv_BL|JT9AJgT=S&Yoy3(f z!l~m*1<0KB?Q+2VdeZBcW0OZ!GFkjxwmq9Q`%FbOi>MG~r^O&&*!l3P?L1u4$8*FS za-tp&lVIF}`3fUu!$M8kL9;li5pVk#oh=Hl@+p`Hzpn<}kVQu|=v*n)|#x1@( zL|Rv7yKHg(GV|;SU80BOY3(Is(!v8PqQ;In({Ic==RT{`P{%=90mhUr*`-``k`u-6 zWySnZttr2syzdC1Fe*i{FQ~LKKX*BMoi|#)-fiz${kU+Z(8%UIz{m@TDOa#8Fhnig zw$c{LLwy06S5mPihY*scZ; zl|6AcT9PJ9G`jDke)_mXuGs~U?Y0XN|Da$eG9l2MIKA!EwQDY~%Zi$JOZ8F9-a+{fZ7M_K_zFen7 zQI#sfW?z3z56357(!aC?Lesl}Xa{UW*?`fB70hfF`z_<7lB^ahA4Bt)F_*jdoS=qy zP69Y1v$L~x@4~E<69>N#_M@kI;JMP$rEUNyvMJ}7e_Z61WX8TV=_?;pth-7-W}tfb zlYZ<+y<6%3M)Fl&cMJSl$8Uc8vx}M58KcrCeEw)@==6TBXZuW>>axJBeAspCkxALR zM~`c)(l?L|y8LoM@7QKWR0K)sR+cqEiaCYt%5RYa#Hriz0DeeS=Y1SI()Rz0A3{eI zqQB|BeW1nB3?&e3c4ra(pFubIfx8R+r8w=QK@{dsc*AxFJWX6 zi)r18i~BBlydJ3F8J)Ox0+&pm5hLz~j!G>=nar527$F@hp4~*7_xV$&i&_ORY2sKLABMx6CWfFj`F;n z;;`q-^PQ&;@p4br+JpvY@o54xc-ll2h(!zUZ=YXSC%=(p1!s}l~x%M zp9tA1m2T=(=#1DGovs@kA!C-#nmTv+9w?~lOI_s(7odD#vQ`QZ8MeqGLS{RzOBn|h z*QK6X%MwYA?7I~sD3%|IpdIDFizQSNcd4m~|0)_kae;DK9do7}aAcR1-kn*KQ)w>dwif2js{D zxJbdLLhgy51LoF{Sx*Zwfm?$q;nB)*DnClu`royNB6$F{fM;x17|N??v4wK3G->b^ z=b}#Gz5Jisv}q%&>l94n0!moiZSx$TFGO?p`eI9<{}FbGL;Y#d{*?B3`=UD3yA`Ru zf)#`Bki&Bm$j;~)`q|p2h1D*(K{e&QTszaLjnB1t`w1kwdDqSrg9T3i@L}O|C2ms1 z^A%OdsJ{mlQ*&#g5*8mmy&|V$;@3K=;iqP$ye<#HeE4X@!M5cJ}wO1amO zGyq|6%K8XRzhO+6HhNE9eN+_xr6q~YV@IfyWCtn7&9xRF0tgwdZ|dsaJ{lzzKB1kL zHpooyC-kdSaWPgOb0$Jo1gMIoc7l%$0i_?6!jp`22m|XNttp?82Z35Z@`fD1v^Zo1 z8@BYIO+*zXiQ^9Z#~|7T8!oiwSY?C$79=d;nHHKG`U!u^CEe*YHLt6u`hbk+{?zJ) zjFo5;lU_VLP(mJv#Iq>sR$m>R>y$F6EGZ%$!MDJ*&y~4kw&&}vd8=4aIrF7v55`XE}orK{%KTcF3>sU-?tx7$xpT*G2gB_6hOXTqxAcC`l}8ea!uz?naWlFA;PT;zSfyk z{=A~fwIpZz@2$%mikhA8ye++r$a$7cYPaP32X_xR1*&|eqV|wgYtUI`cTVv3ogQX3 zdURv29d#$DdM0nM~f9i0#Xw=TdjRHfUb8_bAaI#f>%P`ubug&X&xfOn>X8bsa|7H zc0v6|nKgelGK>46Rl4ZX(Q*ldfw~5P4wDGW8bQ!^T`xBI<-$9#{~Lrjnk?2hV=t0$lmlX0wR4ZI%A6 zuVfh`+IH>m2QaUN!5%^NEBVRvWN(~ECPgeNX)!WlQhs%-5p{{V5i8Fm;{Pe}eFFc; zaZ@>Zk1IhOQnO=LW-g82M2%IFMF244jIFh2@Q1EhX|R;p;X}B_k1!K1FHVgGCgIr; zu1FyzOkVt1=h*B?N3Ks7_@o<3bb+0>SI{ADUpFK&!|PC`WCNTMaruV z3l=UdINC@>^+a%E0RDqL)IJ6H^EOP+fRNZr?OCqXNXXS(nh|UVUE<)%wdjB z@e15uX&sVcO}OLCnVQU@3WtYy(eQ|=uednF>R>Fq3zkLZ`m^@1;d5xRCGwyFAfqND zRi4RA#2%;vW3jyFv`x76XM_F=<=a2{FBG9RaqkXCMcv>(JdlrR)PC8rNpR%cj9nA& ztLdn9>^QJ@Q&qU&PjOe?IIctHh+Q|A{v~7OQrJRHDp#IlFu_f1j)+d4+n#;f3I}U6 zMnmcyzM=kdjnEp~EF|5Ld;cM@p{2$f<{iNk*dz?1^ly_#hy>SGnrq8c-cJw<>QyjG|l1wzs@WnxDi{5c=hG z9MJVSOD5%jgV#&2iFnd>kp%xWO9Nv^~@R+jeeb=P5YHO44So$3aKDo@#X4bv1 zJNF08{A}&XqpCamwmP|=3dOk^O-JcsJ631I2QE^HiEVu#T;xj^B@t=k$ZOwsVX3QC zZ;Uw;`61Zs)H+)9Ui3``)o7HgHgL#5wXfmZ+GI}*h>^Gtc|%O-uLKQ4S)d_k7zg;Y z^UFOrMAE-ewd0<^gqwZIib_G-l>j;G>wAfD;)5ioOkAxj+vMW>yRaypnD#&|KsUCaHQX zP~y&u%d!l;BT!Lsf_}Xew8WG6zvbN6{8-B~kCPPg@_u6O`YF8Q7A*)RIfl{#f=6Im zYSRrjZx1k$X970K0vCkX)Vo#Lcik0SzIT0vbYm$)tTMqhzVUYu3SE zc8egaO*j@>cV+huA%V=91O2f!@eiF>2W$du7KIbnLZt|Gook=;GBkcx_K3nQfQ3y# zWDNn&`xM+G_n}V!833irRxckD_;tG_+q%gKY8&%^ouJ!Ms%MW4AG^&YFzKy#>VOg7 z9=$MKUBB>3&fC1LCp8OJ1&(o1`L^Aw#fjx_o*NSN@OX>G*2+zIE9sdUM9#xX8uu)? zs8}8)0E-e)MSx~}KV9|w0h}~a-4*`RXi#u5#M5pMZSXQe!omj>(Qao;r1g|((@umK z)RSjp4=AP(ZXvAz%(a_+=!HG2Ar4ukr*(1j*@aJ8jzBKYpx!s%5XqS&(R#y{+GHN*K4UrkA2hT; z4a za}Wu|S{_wi%;E)$(uhA7)8(E$9b5ZkWD(>o@?Qw_Ox;RNx0M+=qIKpu4HyoG-Oa$! zJJWu$_2m$Qo^q60mHZz^DIYy`q*m&brLJ4*at4Q_4p?lVqCOWq$lpPXk`>aW=6Qk_RO&y6ig`I)$307nkOUE3ZME6;3;*bWqpP zHN#bRk>;rN^<*UwufVac2Iw~(^${ZlXT98`vcAsgCH+`Pf@JfBvx@7W3!@C`#cwBb zDGuso@)_Q>-nNe;nbj0T_b5+TIY@)flhat>&J}3v=`$WMcyE0$nhtpqfD$ZG1xa8a zUEO5u3P8B%lO+kGu&vHnQ@UY7uz|kaT~@9g;uP&s^xkQ^8D+`&I>rDXSTXF@FA|4Y z+kMAo9uMVC&I967Bx|jd6)i2rQl(UBMUr$`aI*N~6X0VSa1m-DM#+`gCF)21exR+1 zADZv0MMXt`@aQl9VS3WicD+pK=xNn{V}Aro;{K-j9?|3!44(Kdh*UE*8fwTS^Zs4F z_X?&}-Mi48qTi&TvOaI$IZ+92_P%?d)WzeVVTXe~EpDB-LuSc3)0Ek`ps0hQ^mhV3dQHj*$>kerV605j>N<6j3--hyT;OpAaeA z>XEKR#0ZchSxwMWlCUY94%r!-UImh?z-1<`e74S9#X>Xpc>TMISrgpPtXhRF7JgsX zx|J*-$F3-H(vuTUZ44EhjWbPswNu&p zgIXy8o&eeU7rnkz`JduHUR5x6K52)}2pS&M4PJ)^j=6wO5HvQE4{V+vt0p@D`Hw!{ zaY&77ww*E(E*)Gzva_9z1GC5jg8b6xPJIb4pL<)@EyDAD7 zAdEbGv!i~0Yemv?0`G~hZ$E>?`v&1d1`P`33jgpPjxhJpXLIBFbpk`M8{;eI{r5&DWT>G{Lds(?&ne zysx$|JM2R9__^u4pA;B@mlVSH5DzL#|0_-wAGD~D zzEnD=EPa+u9Xv>67lVR#1b_dt=o1enK^TE;HmKzZ5go6+g8D49Gab8P+mwp=X|B%N z7j}l%d0bXN5HcGuhAzu=2i1jYxF%yAEt@bnSV@KwSF9-^8$v-h^i+(N^i(7%!w(W8 zs(?6m?rf{*sRrDo6T*iJzNysG6wVwP`>o{HD;$T}hDyU zREk-gMe70qAYqtA)nsd{mUL2977298YTL2>kC*4?_BS-FH$KqwAQ3MN+$jw-U^fxQ zEqgjn@fh(;dX;gjoGrcFNx~czYUQ;aXsok2j^PddU52|U>9O%8k~&P= zb(M{IhY1o8(pkWM^)J^RT*6LzWu6E)?!c{886Ctj3r;SPs36MQq@B{7i=s=67a#6E zAmJn3cl@m|=7SI){-;5=us9i#`B$%Xbfs?Z9&gglwRh7ZTR}!$u6+BS#yv)h zBLv{--lbEkxaX9$fT;qyzRcC6VAkh9uF-hr=Fg#8v->|S)r0@gG!~2C0pDls`ONL! z;O-BLF1CC2`2fRLd#9Y8wEOVh8&!i(&S)P0-AK3AkJsyWRf!tq=4)*_A?t@8{p!5> zVa+2`L_WnUZGeai-wyk$6aabu{&TD65{W^^zJk?La3OfmX>=^dYO*%Z4a+Tq0>%4 zJ>h#TAfLVi%vBM-mbLo(n5Qow=Cn)leU5<=k)%!fS{G^zk?4Qz9UU`=ZB=jw1lG8K zroVr*t|j7i)yXH=Cn9$rW0gWWs#3WtDCmed1NdQco~G>*NirE^L>}>M(Ip7d$(RMs z`X_}A=I{J9E(#Eq1E{jVqJRfhDkOiC(ECCD&Fr6h#ygnq%4z|-X+$jZp1jWs=5<8km}zX9r$QiadVA%rRhqV$nZ+`p%PiUL`4BRn5g4 zHqV8!H5s)$W6!(4FU8SMh!+f8z2CbsJM~t(ouVASZ`P%s#>ZA`g^SjZt9R~n)Dd3~ z+9=4rZ?P*1-)BjT1-kip6aG}cMg$U`$(~{6SXL1+ozU0B!M=jYb(ebOeF-g3HzyrHx0r?j8Z7lEHF(Nf8~v@rJ2{z2o9?43dZ z3)#`Z(y>vq9f1pP9^QG%a-Z+X0%ODFc58C{3LCH8S#bG&BT08e7T)aioPQ?5#YFXF z$4P)X%6A1!)x$L{GZVxC^ezFYk3F(QtH_Cks<2sVM^oDtjl4bNh}=uZa)>qNrHKj1 zisOWml`$*5H-;NTH1169 zXxtIWGl~NcFmtv$^J8`;jpF?Fv1!v#o~F1)%_~&Kh};cT_ta(w2j%tB4Z%>G&h-%j z2ew<>_kMoOd^9Hgb2Zjg$km9bm8%Efq{UPqLqSuR}>&xqtb zR@38z09S7a%R9zXBtj~N0X);A(o3(EplL&zCYg3z{zAskK$P7|S)*drTXsHk_&8FC zg}&kNW;}O?-r^Avnh$F(JzCygQF(dV&K>i;?fwo$AU8KfDX21#z`q^s7hg|8i*OXpLsjCrd zzR}hZic_`7`x{D^+2t4iN9j5<^-1V6j~h>(cTJjjzjKfP4eBx)n%&pE61O|osxl13*n7`U)Z8`fcpv)W4}yenyu|$(Gh_!WCxNx z{-mN!OKx@cv*Iu0Bpr;48_1KcWn+^(l6OlQx+UjohZMZuRKnquz_W2zvKsM3N)HU` zi~yaeK$3j1QmuJ-YIU=}l*cK*;Os*wP$%ahhT#YK@Ui`?|49ARfA-Ia9D3Mk#nT6V zNndYW&`0kC_;zpoOCg2(Zr_<1;Q0Rcg|t3@`;aSK-ikoExNv>mUosN%me9J{PTKQMr+JS8 zJ8yMzMV5|s^B|Ojq_l2Qu#Rb-OKKMaB`IsNOeAnFOTTijv8VqwGDRd!M>_$*%|8^M zyQE8QGOYPY@Co)2EUzU8{swoK={R|cG=osrN>Ufqr@_NINMC>D#B7+#A_~Y2=FbDv zZ3l+6HFYXS7lC16GEnweXx}oclhEWg5PtlrryfwQtUSpT2Dtjz$n(!oG?X@R8!CoE zzJiL(C@tKwYYzopns}$oPU9Z#8?ol;&S?s7osYgH?W?%NtOl%lC=#Pw_@@~5`1W$| zFWtLMK&|_DH1X~}5uc8P-8Ew}H-0#4^1m$ws5|(>$epJm-bbyTa$+N)NMES$9PAR| z8ryyBsfgaTW3v|}|MGCutrv)}yIZ{ZTEk%rZ({|PP4{fI69_{Y;{LHwK+cZaQe|kT zViilxvPYMv5FR$%apF5qp2uH1;3CtE>{p;eGM+?lz@t)5B@EoFsZGF!*y+WXsF)0q zF8MFcFQH2cSURk;KM5u%x^#cbOLjbw6~SE`5gMAje$e<7HV^inc~Ul4E%^1<*ZZp_ zI+*uH>XpP%_}H;eK=6dFy9%`fvkJXBkjAL_e6BC}B(S1(f)c~BB4|(@~Z&c7egx>?FLDz zx5~F&CH9E5TjdRgR@Q-L15xd*g=VUttAB>)*HsiUVrcE#H;o4C^*kPChy?~%diV!p z)q2!_Pcc%0S>OQkS8r>|(b8Zs9!pGU;x|8FK=yE7$+P>heMjdSm!xK;aZ*;FRy}Dg zcj|5Pi4*r@Ezs{Bvt)NrP>#q|!9Y4})V5pA)iB%T>CVxK8@J*J+dgc6>QUPsXpYOa z88Z>CIyvvt)WGdO{I_mdI$Cb2OBil8?UiZq;qw7u7n}m!8}&7I-7-3BM0~=j0s0F% zu58lFwypMvFI1wwRdv{+=`woYm+QP&)Waq#yk4XQfIhd=?X9AILgd8w*0V2Pqo@pb zei1O+wOGSfVjH?;^X?uKHS&m+{VgTWk*msg*?zxOhYlaUE_rSF%1q}-+HeY#1iw?n zc|ylH7z8iLR^4T+iuLkCDU%IwAhGD*g~G*ZB1MW8Wy>fv_nZSEA+p%&&Qa@ii;lmW zP+wR2*-=G!h@I*`Y~WUQ8V4+;ONWZ0r}|wEBs*^vJ=LH>gN$qRdpxo!H*g`wzHm3_ zcN_`qxtgb1PO95i*H+|C#pyjM@~PdLH8aoE*Ott6_wW&%_AJ>axSEu>)wdpiNRruv z1%r94zaXPvV!`nb@t`+QZwuw{gGO%Az1-ZYm`t7=q(F4&ijGQ=0SH}A=Y>Z{Hj)B6 z@AVZLSwp0-sU9n9e zP+PyeWiC;*-iCFnGFJ?{w(>Ts7>+G@m^6CdoD{kK_HO=uLr>eO{)_AM7SK<{`ow-( zr9zTVjQn?e$d9s?>&_&<{a5Zyam&$YMDRzVd&sXFMiPe+1EP0 zhsyaaHmUvmwA-&KGN73T9F|${0dN&U;2nSZscgRox(n$mHkeTk%rlcmj=UH*>xKMv zEstZlQzS~(y|Q2q%SMjWJ+N=Fyf1x&FooO+z@u5WEn~@p6tDWb1QJ+#PIS zpw>kKKmvMJ$s_~lD1ABk!9__NvIZ$9KEU5wWx8ef=iB7~!30!xElw=Se!7s-gOb!U zf*7|W0UO%!J}d|N7uy6W|2sjdF7kHDD%wjoIP>K-Wm$pn$WXkwi6u#jnA7*bmRx85&nqLkI6ii8$j$RG+;s;&38waX{2GOz*H7bAz#mea|*avam@tHyThdC;T zX`NuN1*NL)1W>M|ZdkrsUUN&Fn?j>YH?9!fUC&o^<6fzXbbX>N`YL2C&k2c8kTh8f z6C&ySl>P{I7I8K9Ex1=XF2Az6;_;@gl6IPT9Jo&wu5bhPR9}4(dA1_~Cvv439iwgm6Fmp#QF`NH|A`#f9iX|C)>Xm^#->Pq4-Pm~ zh=84jfSu$SY}=NI*Sf`~oj!4AI{sOR{q)3%p-TIyNvmSE6qB;Qg2=LrbiFl?H}OL7 zmpE{-GK-D><$}liQU~~!;MdJd$vCpPOe--V{Ceo1(p=IHkbVSM3dp&jl`O#N*rm%K z5j5W#Pj#=3LewY#Hx3GuT8`A_sHT~o9EhsW>e=AzNY}uRGu37|@ad3EB_@rBk zyKTSw+&%gS8E?jJ>%7A{ec$zS&{Yxetv;J$@ z{2j-lhmBmF7UyRjzo$##Z$7cstyMJbMq|+}88bfJ<{-U=q|YkA?Jrfvd);@;oouFm z5Ex^O1L1t+gdN~X{xG&tJw@rn935Js>Y(>A^ikt741M|2U318bHS)npL6GoZp(J|t z@@jeFZWo=PefyBPva@4@v1cB78AZ;m$vB%Nra$z=QyFb|z?GUzQeTvs%t6t*N!IP1 zX2Tv2Zr>n6_9svhf=$DP*clU;!!<=CRr;ih!`b_QB!zWr4!oTnU_>paRvsl zNl5$GIP)DQSnQ zZ`z}>#~!{9q<(sGvc=b62jv8)tjKhkZY*J8%xO{mW*iUL{^Bv-05Oq21B|b{Kbux* z+H9B65`UGU7~-N<`=(YU(Pl@fk4Js*;i&MxKZg9rM6+FU;9~RIv}c_c_q%d+SKY+M zms!32g=QbSg>A*>%NaolJJ@NRbV83oMPzp$9;AVmi8OkIi;~lAP0k+L7a~Z{j-F^l;rrjK=tcescf^LpR898NlP4DA z*y9l?EnDT;99xhXS83*CUw2#+I2Xy{L?hwy@E(nTOaNfMZ`ZGJ(;-{5Ro~kWR~CD( zd+ONxQ`(u@f&aNSli2Ie!RVh%kkrF!n8bl zZu{=y)gOI^F2Vz0Yt<*JB0){R@}84+vt5#oC*}{2+FTFd)|BM+Do2-#cn9W3n5tvW z3}n*6NpbH9DzUZ$kKaw+eK=10mk^XfMmUngflhkrj2SbQ-@{jkmRWJj^>Ii3q(}uN zxV;!hoVIp$UUg~r5L^cZUlGu^){5iNumM&krt)zkco76fu2;BommpRkyCYF>*cR<%4?fsN%w5A|7P- z*ZXM%Pw^V}JUGr>wuQ(xK(X^F5Xl!0dSom^!F0APa|AcRmWXz$s?XTYZo6X{FY+T{ zJiz_Lwqpi7h3zy)!mVp*W5Uf0sz!0oWb=(lR<1}VT77Quj8sLauKYO-=wxIO%#6nI-%*ifEW zWkz}G&|$iB{pGIYw}=5vCV$+IsGz7Eyn70393#!|t@@LmVwHZ&@q|`lK`n70{?;{R zqbB2RW;FKn%u5-tRkztH1a=E9X_h;aISiAg$k8?#=i8Z1xG#g|eF08G`L?%pH9R7r5=vNYcHZZoTTCL)u)*LWtBfsD*YSu7L6u)hR*H{<36

    =M1evm+gdsfX-^K;PSh(H8 zSiI;VLJgSR@23oFuwJ6(4^=08cR9<_x;&H9jf}QXsAU&?eTM z|1?AYo94!O8HiJGsw!Oa<5N`J{FiH;%{AGaEo;@Bj^DVKr{mrG_s3z!vU^;hN%Y2Y zO~Sy=%Szc12)Gm^h)3< zm(^PjwSY42>HpYre6t$KM-9a>Gw1Q+$2nEyRh2qBHeEew@c(lRSUrAtrLM*y{Khh` z8Hw?os7F>xnlz(r#*3gaax&g?!;?>RC!B&HPRzUDp&Xp02T?)-SU{os z-%&!d)~Kx~miHz{QX0VDQp*uv*SGe%j?K96d^ttzXzJ5lK!RF!@hF~^a5~ue22BcSQKMnFKi!YksE95K`YH}1| zk|giRcY8OMtzNQNjVf&>Oa`;nq~*m}Pa!_IXRyM}S{ELBb5_Tjar8+aGGs^?uJb&Q zy+5U_FKW*@-1Vl*TOksndLXttEDCIDncPh}htvl#^Mx02Z)y7zTO0?$94VV&OB3wl39}>g(M=`$WUYmQRW7vG9;9gDeNLr z=E{^}r$UruC>a_yAru)ZqRb&=O7G|TWuLRp|9#in+H0?~);TBY_dNG~UEdKoxF~i- z>1i_qPr~5A9YGzZbp4+gYrT4SOGrtRr3nh}?npZ$V){h^1&q}0beFomXUtEvTHiAR z1yh4%=9(HcC2gv4g;}K9y*w?&mK3^WMT2UO*j$yDQ5pB*Q_kEsSer*q?|uAV`esT& z=%4yIU)eJG-Kh5-%U{=Y;q+r13Q5O<$Z6S%60f&gzesi=mC^NARsAT;5uF|Zh_V~8 z-*iJ)ID~Ofwx$^~^m1pW_yq8y*cYbwV(Qxa|L!5_z$sn!d3QAqnEIyd9FI%t&yS6` z?dvAmR;Eek0^Ab{lp>&EtG9>@*c2U4)r_b*M1;rIt*gI&^`c3T$b9GYxq6^)Bq2#~ z6~;>LDad7)s*7gV2z6^L)AKwx!#2XJ@mY4?y>XO{2n(@XUqR1)J}g@@(}*UN15J_+ zL}=vJAuKaz%&f-$d1)>fZ|&1_<2b{|ckSCKA+0B^x0E|109-%+$E&vwlcdKGJ^b80 z7i!!=n|6PCaJvzVsg&_KWasD-zK_$4~-;%@w7_+z% z6pD$5?_j6#3@;ly%B&+9B-4yKD2hsZ@nX`xInVh@WoxqpWg>(W?)EpjmKGM0%Jy~7 zzMXnVmx;}?L21!fuNfH|f~w2R9YZ~^sD|Jr5DqrW`EM@#D0O~&u+VEejoLNwxJ+=S zpO;N|wm7dm3K{#Im;ZQ1!ry*L#!I$ye`&y**(6gix}CHIt}C6btk(hzO$tjH;GE9S zIC5tWuO%bF69064FakLQVyV4KFE0vk9>E`lsvSIO-{J2C_#T?bIN}sUo{vIWP+2I~6i$KK^6YzgxYDA|cATU|-wD_H0It_m@1FqYn zGr=CVd&5alRqyq5S!wI^Y?EDnGz?!Fw4{IQCk>jl|M5E$AlvNQsTtMpNxR*!yY>@i zc73_;;^SS#MF+j;<4Vug&`;VNadcJlb5mAyx<9| zyquBYR1hR=u!-0EKk_m^Fn>$vhnhk6oV8&Ew+I@=P`k1zeT=?oJ{u+3w-cXKIQe0_KLuqjj2L?8&l@%Jk}#gUw8#MNUL z8Ado3&VZrT{Ml2wHpxK~f^1l^_GFcOpAJSaapC@gF<7juWB&cdtQ40A{ul;CC!g!` zpd*jxff*l@d`mCKYRaTZw2wx{TSP5>Q9e7sc@FMutjQ6Pk+rR3mZr~_;83~)L_==; zm_PH%!qmzQOn5e8i58R=D#=)bH=aixz3;fw3(_aCk@1Ry1BYEce?t%QY#l@QgP#?Q zrJAL4k;iJT?Ih^Z2*#mUs4e$5_j`Bfr?@75|0`m64vrzdC6xTNlGqzsUAxxj;S~q5 zxA(O_?wwJ;_i$v6F_cq7mEd9HYcQZS*)!zWc#plc72hGkrd#U9zKVe@*l%LYq|AoP zF1s{quYWplw%P9GYdRwojqBs`F~bnME~#j@F@|UHft_ z|84w=iUmKxTmRKO@?*uI!$|7T;VO2f(_7~-Vo9hW@eT*$16oS%E~wKX$+p?CeS0S| zrX(drM|1IXNVoorpVbHY8;{=r1Vg=T2U;Bjwr9?ffOvGT#O<>Jg-_uYk_cRJWAT%e z-YQJ>SKS6;g4i~8$QYh8hKQ2N)@16*|4}MUB{c)Pd{}zOFuGC&GDl}%lD0kkfmroT zD$7=9+Q8ULXHK;r>qbu!!Q>1sX(=Ui#T~2un%zD07+Y7Kp~az`MF+hW_y7Ga8SV=Y zH{-7&-rO;t&(2$c%rJImaB!M$ve_mr&~nb4+h37Dom^gD3GLvu)FBbtnFwNy-j$Ul z8EYCMB72v1_kW=^X00xb^X?4T;I8e(JtK7 zw+-Rb))nvHUsRjrvj!@s`}RQ7Nl@?E6)#AoMABh}%mNM5e|^CoxpR+>OWj*C4%wbU zxp&jc_|99F4RP&$YR%+?$^S(5va+*5aYPVEE-9kv?D>MMlgHt;Qa!kZG#vnYJ0?=T zw{Ikc(jY2G+vM~Kt7iyI6hm6~pG*}GjUPWG{5=A4()!_vIkn^v9?uS8h;*1~0{HR@ z764@Ibbe@S&mmRSD8p#)j;GR41Dk&O+ z(He2{HvUyvSk>M(uK4ZZ&5GAloivlk2jFyQjXxDM`cvXZ8N?-i)N!*Zql{p6+%mq- zFiiB6sfh%tED^gAGuV4(rA1aoInyB9o*?>{3BKB&zah)~Vb)u8x>)Hbkn>Y;cDd7;#SsnQ6QcR_N|Fv@);XTFp|(ylgX#jl^n472Ti-M_ck zNiHJC@oCXZKw1?n{L-I0H+!y2VVmF1T|9e%|KoWZ{R zhSnO@jS4y`eyZDoDoEn~_15^C?q&iNf1-rw`ab`Ahuo;`7wc;VUr=O$-YGnim5dan zh|*cXyPILaz;2)H7TR-QOmz7WrB40(@6M>dce^5_x;o)(8E!+Co>TcI_P(!t52uc} z_2!?X>dL@f3x8!#QiLxRep~SoKeO8nEhQXKv-z{jD*J4V@MoG8`CQ2lllR{HAA=kb zE&aXV<`Api@lDoH$LBI<8u>q^0opxrmKU zU_Bv}K3EeUje>eAOxX-alzsLw`-@N?$i@a9o;Z_eSRugra=;@16}VmC?>AX=?uof9 zLk+Ae1G^ptC^Kd$e`HJ{f)X<|ESSUk|p8lloJ};BmS2l0y-CECN_Bb=c zIjSF~J?#3b?&i!U<*Sd3tyQ#in;(>{^bS0f8r%S#&HU(qbi>Icn-mrvF8=TLP_v6Z zymHj%8ex-?W*EuWrFFC}MPRMQ81=zxCZfLQJy!**-ZBcy{z~NpIAw6f%14}#GAKd@ z{hv;u!e%$dIxOTUq%Zr_o6o7y(WF~7k*Wi1`L{`JUW4#)4imK`07@k%akOD?1t+ci zzLfDoW~I!`%+=CA$Xzv(I&aWXnk9M)ar?W@#1Sd&wRGswvY)L{IE!Y9@g>rhfJt6_ zBx_&Nur%UeNQx$uv<}nW%}<5yzX!j2pKkW!B51Nq&BojNZ1%4$h^v1-Osx#N+?N7K z4t$0{$G-3GX^-TNl`_0}k4RtPugRp-JtV`y;ED7q*uuhv#WvT|Ky41iGsny8eu)wL z0)1j#AwX0wJxtj}))@~=`JkP=Oi15lacI*`lAH;#cxGx9QANja*S69;AL920D-q(g zw#YP$l91j?=*~}*lG=y+DLWRAfBvs-5$S_hyhmE+3f(Gjar}>V$ZzZsB68!ht@n@1 z&S`UcFvSvIejt=bGE-F$)Q{9oSS`uRG)f|Z2_8p7TYG&*?SSy>!0g$*Pr~_g>IEqE$T73W>-&EaejL30ZFazYv3QBXlN! z+;TNB1#mT?4Wo8It|r9crGGJM=rlTZIw)fK)bTR)U8d#q{RIDkPb;T?-S^uU-1I-v zJzE!6kLEv{P(t^iXpl-y%k0ZFQ&tScz7Vf{}~TT_@qS37PvX2eYQyTNa5n_dT>I zqhU%#F^abTk7}W}TQ~J?p6v(;BXt@WVkQ9rw!u-`2zqt0e+k;pIs)2^7*QAd5<{Lz zaZMzC+kgf1iIHGL3`FjrUt;-bUqSfnNz}%_DXH`a%x_$z9qfHoX)U;v4r={D5 zAHMTaN++SEZQxWB4<;w8LyItpDuH_i`_`?wx>{RwL5|TybRg8f^xAkNP!V7{m)OQ< z(Xp`!-^K_#J4!X;xv2dZ^q8cKl5jM!&t?2N2;4?3GA|wOjw>anIX2>>RmieK@sDCM zf;-Mys&6C~e!~^BiD(0}($V@6{=0&gLOK*k7@5CdQ&3O_9LlBT2gVZ3A@F$ho|Ju? z6!-u7{j{GIAB>mu1HLUoSitG~$qrbw)#meg|n7Ka-q%;hue8L&G8* zP^j#ZlIL?f^U% zI){A}Rmaj7_k5|Y@6S*n3C;5hdcvK5$el7vK|20#Sgunq6O%xcbCN;hwcy(%Cey0H zD=HqVP7XcRq$+m$vvy{|T@Q2`GJZ(Q+KIUf&dtBPB&Pk`xk)Q-`m8M-Xfms4xl>ue ziUDKBJ)HmMz}kRL&9?_e)DDge@#uc^FWa(?3+|ohsyuA6-lcB?n+8p;m>lD>cbVqo z#Jm|5>uD#SEjS8tsqxfBN_TwPyhlDpYv!Hm$l$4IsZvR zC<6Ifm}MDr>BAz=nVDTQLLDFUD6+`t-)6%R_t_V8rDF%Uq5h>!g~5!u0$JGEMVucp z%Wvqxi^K`PmqufNZ@UYCSOQxxH#m+Nx%ke@CF1D+C56T(m0u#WM#XedJHJN{f&<}U zc87*GcZD(eiq5HFl-0Mqo9CDnC|o#@oNN;-(W2zZ625U|;GllN?XVZHN=ZFpKq7oM zf?PZPECU@)TR9}qMxZiYErMpM(Xd}!EeXh7ROhYOE^pmgm_vxS|p?XfgHJwqTp zAoP3u6}?*p`)~ms9N+bdM{}iHckYy@H_~-AY$Ju;Xc6t=0AGQYB^=FWrpC0}FPH4z zw@)>LBD{2|z^ibYe*E6G?dvk*=gvB(w3=yVPe`mjd=`$(_{d~;vTzwNtS8^ei2Cc` z!661SiS1%8aj+QnNs^hsK||)|^Imcr{dsxSE_CV~!{#o$>~5EaJ(B{x%RJfIqJl%< zca3Yq%`xR(Ir;-w@2ssG{H>d_SRitn=395!vSsP)S~ZX>?V)MG^3=^IHKvAmtjJn+ z_{b4=l7CE=f4uT41rAeuNTrLW_!9$h*}=fvh3(cLD!z#VGS~WryEEjiPhx(B-j~*! zu4+`wa?d9jikQ@OIkiH=La+SbZYgkiGR2tifpC-M%_Z&@fx?yVtWr|f zKs<{2o#01c67GmSkd+LN>d`q>PT-xY6(0%x?SC|FUpcX5lXIWzAlpSxpK5%2bNl}N%(=S~T$sT4 z@GMM^fVr}deBHcX7rzUySnKxfTe+g(i1Ch}Yx{sS$Y-UwZ>eHHQs_A5^w|q;J^Hq-T6N267hD(`DECJU z43dn*gU~da>^_Iwt(!}?`L{bnl*ZB1AhBF*>bI4H<9OoD7B6V()w|x?_fIaH>>p!R zW~N+wdg{s~esZ*zB1a+%lbMD-^+;H-bVH@p--&{_WO?%TD-fCHP>TNz@{uVLS zFU#n+13MX}CYJ@EaHW~{+lZUu0OFFlzGBOb+}u_IHhOt|J$FSbLi-rW)3=p}qINF5 zvDZL{ioTaD7RdBi;7c#e0$%5yLwfu=WK~s{`ZXo#H9ESRc_qukpD2ySMEj4@m){>p z4~bFdi%h|=1ZSEw>F&`KBb+XhSC!@YMrk&u%k+-x{GE4j(`HCwb!^br8_Wulh$_@!J{6BWX;QMQ#9K+!(h#JVfiFib5bDa+br^Mt zQ<(wO>ok znMP4xk5_%~87~Ad7a$_(G3gFw*JdqR)Jw`aSliK~9-x;?b<W1=xR=eUy#Usa^JG5E?m328&e#3z22pCCTcBQ+Kv3*+Z7k85meg9@mbIw|1rMtELv$g#fK0dm?w0hZcXk_x^ z_kYsl01xy1b1UmEdO1Nu<3L7)--3z;t#ywLY5k_A5{IOZ*sWzlAEb6KUcCx2)X+l^@9g{{WZK=C(UZi2C>euS zAAI^`DE$&3Jumj|dx*)5d$*fNXc=4LSF(kWRDLSBv)(}DmBI(JA6$Ka8kn}vKq#MY zYgOWu(W^>QD=e*4)o}A(bWYf3SV%H8sw9a@!>=gK#v={|ckZ3@QMpA6&3P^lp-uU6 zJhj>=I8IVMnK7Swy{mb7p$2*>WZ5CEjLv$GTMpU6drd|Z87+)SX95VWTKpu8RBZE4xC4zCn( z$er>;>&;Oz&@FfmSI@M*OU7DSJ};fFQ4i)b^x4_(qdBHVjUBtQvx>?ET{dcd9NxK0 zuTs7p^J}hhA%Lv_uwt|epBOKBsfkWR9jGSTzM2)EK5g&PINuW6hXAo-#*G{OO5!w0 zfMq?^ooEmmq-e(i;v);Pe1Y203+>u&9wDgbz9igRfYs$IZyf&CZHEOH4%bfBm!%6* zU-j;pWRg1S%UyVHKznn&-n~C0b{NQP^&6kCuHf<(jERHsr@D@99eg9&`_&R`^-z90 zjxjSU6TLQNf865C>Y<+dCI1PyR!S=T?p(M46(#9_RvwENU*f61S)pqd1EsDEK*1ZD zRAdT?EsAM~5zfL|cl2iP5m}cvz?dwy!e7u%wskoC-aqy%G*7A3@Vr+xL{C+HPN)BS z^B4>ov>AYan@9ibhG+)heem9)OS3XV@# z_wHNzzWN5s-)L`rjH{F{+u`ANEzYaG+33}A&;pb88K(l&lL>Xr8dp}+@_RsD~s7wsic&!E>eK#exA45#Ji+pwySo`wh) z+?PE5)oPo4`RqXHwLAdTWBWHik;N(%ZFdT$la@NfGay(GxFnAeU16Wo{99 z0otd3U#aA|=S~juU5`H}vBub;yRurr8UvQzeIoqQcBV@KB3X#6OH*bpc6T4b0C3tZ zzh+|SwX75dg)TOYYJ2rQ{*%!peAZaAgocs@ObgqovtwU8f4*J}kYUq_^;EIn3# zNr}lw#jBw)t+)#@H}W;G!4m7@@+KY54DP>qZE1_$;ZF>eG;ANYY^Z&=?ZkthW^4*R z8YY7lW7nhjf>H4M2vTy$hs=W_T|~i|4=vGd{gXFJO$M6LU4}J#^Y%dN{Q7icuwL@; z^9-?@7>Kggz7Y^0Y1NM;&za7>$>5SGP(8g?t`jFt{O4^OyFJSBYJR@Djg1XYyoxJu z1KZ3+9(xB;Sw13Q6Zl3Za{Sk(nI6TPM631U2 zICL;ChJAO9Z>Z}=NfrZ=89;%RF{OfPA>^0Fp*$uAp1f?2nhBBDE8ooT)!M;gr~~PA zXl5?d&8YKiUf&lqu?CXv?r3q+n`0PhMq6h07UC_F&x0$(Va)083~B(AQ>^U$mf7ml z^{7IYFJoi8HBGUI;m>tkv1G~Jqu-wLW9@vj)G}g6x|@+^zkZ%+Kj4OYo0v4_WUO~s zBf=Bfa>J!dm+F*(sIJR#UWZ&@c6SZi1_+kriJh9;jPwo^QW^C$Y)1h7Yui(k?s(5X zyo7xLg(0l%!ll3G*cvR9>LJD(uG+03ojJ!j-1FSj`SHG`Gd&NjN3Bx5#kddr@#W0} zpU(_pan|LYsyQZrtn#i$6DyVSQC1#FBmjXPSXLvMN@ z$?^*Bc2U3aJMwz#z6E4hT7W`At)j00Le^vxQH)}Tz>$h%6a(mu{HO5*mdfCN;JS#Y z1#{=t1&fl02|LKFWPA;m4qJw81p*+_YGs9=l2Wlwqm>KW4*PLt-iN&b{?;{|5^J9G zn|>mHTE;z#-|xzkhU(QL-m=K?U6soYjokjH>hQzzm!PXT4 zTH1*|6My+B^WwYQQW2@C9U&a-;y|@T0}l6eU{d$OZ>`O?pdndH;sx8eews$#*nz8kJv}k?Bf0HSw?_Q&UM)4zgi-TC}y-GOPrDE=^0cSMBPwl zVzzLeot-4kt?qQ*d)a%kFWHmYK{Z;hMJ9!A$ogYSdnHOL}-z4pj8MhReLE#I&;y7FaN7G|UG0K&%}>sf{x zaO~Kz>?0BV9Fe&3SlpNtc+;&_v~I2 zp43J<+G|hALMvivg9+&2AiS|{A2#}4GHu3qIH3fXzU=iLWtE(;*RNk+1_ua|f$=fB zFbSR769);22bIkS))#RuQGRVl{}}R z2bJ5-xpDov=snRh71qF+1BNKahQ44Y%G^G8>BO!r*-|g<>Em?e0h2h+ta!(3s?~2% zCb5L&7mW3E#a4sC&R%7<$VY}>Z&FtW=cF+J1< zQpRF7DwaWw?DVXh5B@Loa0RjB*!J!J+mRt>!Gqs#RQq86ozY~JDPT0qokUq|AZbQk zsGdx%A@Q)!yijwo(~0??eQ4U_T|AotS01Fn!jB& z{A``%bZ_RbZuJ6-tFqvMCC#QKm_uC`Mu0uvSdktFI1E4^kkt7Fbdy{ zcXrgVo=O+1fyuJJ8ZuFt!vAM5|4&@fJf=) zmWJk2Yu8|Sn{d02wr5KSoz8y`+Z5Qz>FcZ9#keKp%eK|mm(_No%r!oH@Ahx=Zi`b3 zot7s1|0q54x#piB*z37?F09n78+hnz*Sk#yjByHHUuaB%9_BbT9g zKgd!Oa&9Ut@l!n$ z6B6WgVa<`(2}O6V-d>qYdp zA81^Ox8^=#n))j@{P6sR@V1h7cUPck5Q1eKN(}obkM;7v_UVLJkaTq(|0&kA%ZYCq z!Ij)wjVKnk8i?};LF)#&NfS9SMvl~sNa6u?Nc=h-2{73s0sL3yCD#ao8Q8zQiry6d z(&BxLz2OT2CyV#=D10<&W!2t{^I5NLktE9YpNNqCGLuB! zs`(nWYjItXL+N2YKPB0gpIYiM8Tp)-?q^0?QFh@^K=F^heRI0sCFF$Bz+8`Pk^y_i z9yT-k#36hZC82BI!O!SE?t(4F9&Ym5%6COb^Tc8nji-@)Q2xgb&6pl#0F!Z>L$S=% zBC5F%Xlz9{ea}-EB0{v>igi5TCord1&v#_ZNwrM9!-pfARFN;HaPKJY$^Y4kqUNGS zLa8O3KcTd`6@*-iz#l~Fe5ALbUXEkjw**c1}>K>EH(Rgc-OS&UW<4A zkN;CgglDy@z-%PtEdplyO1l`C0Tb~$4!UN-F3xvxX$YE>v7fC-J27((0sZI@g~*Iw z20Vs=yAFD$vY$U0U%n#c_`pph!_H?u>Wgf<5O-xhjWtH*lf}U#N0l>ZOmz|J_7=vY z5ch0pXDn3$5T+4FkN$nM^FSpNn)}#J{D8S`-@c8#D04$m!Yk3eLT|L|*S9Z!K#%B( z2(vBCm6bcGGj^tydw}cO1M>n8Hg`orgZQY=q@&M07Wi4~L*=O7oNs555JE?3)^1Bb z=f1mttW0NJL*UBH#s!CRLR6!6Z=bJiiGXZd@BCM{?%vg!r$Zqtr6BtD8WBm`;C5ga z+)aA*TFo>%hZo0a@J>E`d>N6dty(#5bM-SzIKjlv%5UKU94L*FhHYNfY|E8`XnqZs zil%1soVG3`P9v&~EWwOTmiHf^es}InT-CU4tjG8f^=CYnOQSOW!t|dxg!a8feYC5wiSN%;up7SC2Tfh8R~+ z%18TVgdI=0Z=YcMHYC*5aD{$LC4oVQ^ve+kwx;?y%yUG?TCKp{1@5|z*(xSO8lRzj z5nLuYjnS};z&AW|=DtX3LNB>~Hyi!vcezjUPSY?>Y({=jGlMx9_uu>0a?BX93x|Y4 zz?gn5srJ&etek^sCeUC(9nZ_fRQoCJ^>TpPP)n4s#w@WsYw{t=vU|Q zYULNO^@u8ds;?8s?ov_F#->aoB<{aoF`$p)Ok?1aZ(wuC*%Zs-D3%n^_IoX(>NK$o zbN`jqwQl1#y+@eEgr#n4$bO_;a`?$~r{Ay5i5Asb*x)hD#Fri?*7vwml(SVO>6}h4 zw}j00oBT(q)~eel#kE~}#+(K9m)`lieQBaprw9_zy?b+4g5?SGO+$&2sG*v8DCrc* zSP4l<#puNB;>fZASG<4D^LSuZ>|5|mHiHvWXP$3{NJY8=oc8<9-Iiq{V$44~uJi}} zoE%GL?oZT~U#Xe#)ZaSd+6Fr4sq50V*?xv^hYf5bR`9?qf9sA6KH{4JD5HlpgJr9i zvGKgbLIz;j1O)ghOlc3S=d)@=jFcs!X42WhktaqUE-3YhDp{JndW|?x^*lE3hLM56(_4XN2|vl9c*-yJBAHx~9hkO7_dH~DL_Z{Sxvs}$YXYkK2Zqyr z*e{wJCpqe#_#`kF3kSC5f+X^T~ zEi5}Md#;r1;vFhCT`9IdPCOuncp!$v)2D4O?7E-ms&-tjh28`#w_r<*uALbVOCNXP z*9HLfCvqUaJNZ;FY{~UJ1gR^rp?grK#xcjT0s&NBL~-C2W;Fj2Vpm%JN-r+CHU;$% z>M!lS{C?o;L&~#&0M4)~{v^r@=w&KR;|8zKHlvftnzH_=6w$)mp`^vygCt4{xMF<3J5z( z#0A@un&!;3HE9<GPySCO%?A98`{RlN%ZtY3S&r>tx`c zGh+pDNGpY4mKYm}7*+E+=%`6V68I%9fab3K7Cicc2Y=O;0EZ%`GRA!AOaV$|hKqN( zTj#v9@@^FOWG9F|Y1HV^el!Xe*J@#m4quH?8_A7k9^mu+-G5qZqp$9|!$Rn!Jd`*$ zsydJiqe3kKr+)l04mVcJlY5$EEqJ%~0ScVMTB5)y4QurG9hY7|d`*>r@iMosn`u?O z1_2A}<}o`>VD%>uB2&$6eV;vJL_l}fZRrfZ%$#a=2Q;kbKkUxK^S$jbp~x$@8pbND zgP(sS=c@E+MH5QCF}pUeFVb_Z^07!!&>R)r3OD=Y%^eRayOw}jb6=jlkb9G$K(i{r z4w{Z40dEJ7Ijg-tn27lx7-Af*vo9}+=x0dz+9{jwd)tX(=XMV|s&|jkfzi$SV(C(z zJ8OZ{EX^ZRCU)O}=tg!Qw9s$!r(w5P=zB2Zddh|ADQfZCi3#9wG~ZmB;BOtohbMW~ zaycrkCedg*rWjPb)?pxAsEs$-z2wV zOJC)l9^yL??k?IgA?ftTkRf!sAsksI&883$wmE^pV+d zIs&Ma9rv;9;JI^cWmf!j9+(6aUG~tc61T#wzBM%T``DY7pN`Q0;b79IPpwn=jdxgR zv*`ZVvv1$dz|YQjE) zxK+dTYn$lIvl%^V)K*1$t2SX?|v_O6HdrKewfKL2x7 zG#*!OBCIKK*jodg7OKU381U-$J@h#ut9d)4)K@v@qa>TDAD?kfm2q5*#?+ymKOPNM zCuBAdS^MoF{yUGdDob8gh%J$Y*(p6onw^%9UZGA{F(7Qjp)|$=NlAbDtCC$sV!=Z~ zDdr;u%1dv*^ByKoVyTCqb0f<&rD!F3RMEMS(gb_DiYfOiKVDp$fD8YYxeXmXx{>%6 zW6Ia?*O3Uq`~9m+2dguJoLhzN+b8l^M%~Uf3zwLZurALI+z@+HG5qPH?5VYH=f^k#MBxgbua|L1E(`3|jxi zecPjWfB$wz7tC&R`HpJ4CZ2vXqqYyc5%K?z*x@t1Dk%$?$m22R1b`+zg}zv4m29{& zg^opDgiQ*;Cu0%-TPID;%Q_jnIQ_jpb^XN~h}vsPECT%2TtU(tJAZLIjTM{FvT>(H zKAAdiUN_)xqEI#M4K%E}?~sIjfHdeu-lZP%8vY8oYBz-Y9h8h#ll2nqGAnf^{OOlg zfO>#Kn~G7ILU5)P7S2kaWcIOsk(ZOSER(1G(H${J z5DjRZ9wZ_gpoMpY8Ow6?ZIBfN5j66{H?6^PL*hxzfPHoQ^3%*gOip~4kO!cYG-Ycg zCR^s#t5>i0fB_BJ+s-*@Mcli`dJb-Bo?R_7iw~{!?B&aD>Xa*{xfxBX{Lv*MH`^{= zTt?vPJMMB_h?o8SdD9$M@F?JOHle4dQG=fh;!!0>DDKOOZ%zw*zf=} z#I+>_H|UN|JeU<_HAiCKQ+x-iUC`EGI0LRT&nCr#OYI{bQ{6jv?wld`1yl)vdCB80 z5cEjBY~t@_liQ$GIJ(|ykfTevxswHn(4RSToZG^Mvli{gcqSnx>@GhaecZrkC^ER1 z!Z1nTqF?~ooPVYY=aE!oV)!*5)V+H{34T6c?Xi4m^*vHOljS1Btta}Om>7^gpXVu< zabz80-{J)IYku2ecCTrRK4i5XaKC4uzxBcK{hCj(V0aOP-j(m*A5h=&3hfq~Ap2dwF`x8>(|%M;#@6zWrT9xWZvo+_RI zaCF<->*A>F5fuHT<0i{3!yLwOKQJvT&i3`=^dLOwbYi~E3g6LyRs2+*A_ZRx^GFM*BZg5l8{W%l$XWr`UoZ z!2F95k_hEAXwc)i8J)Z*dS56wO0dlR%x2@paLSZetX>NAhY4PIbFh?iO zmSC;QSsu3TM!$P=39UyOwv1fRYn#9-XoM7MIeOs0fyI)&njbz$BN;dwQy%}2U9nVg z`{Y;LoIkYxeij5KuIw7C9!6P4PXd$!v|}1V3-{vJz2$*c-xictTI#eQioDaFg_XDY zF;NAt@ieWgH_Ic;n__^!TdebM_l~3QX}E9E)z>y0dCa?UUp`k?Ewp54+B%G-4vIo*JR zgx95Kgj|#uP2d3DToFE+2(^t(q`<~SN$6CJ6I>}5jR@Yya-7mZ~w|0 zUHv4HGV0yiDT4A%@3f>p* zwh$EEcUausnj->i|IC$~eg6?)E0hbBKYyM%@TK{-9Xr-DrnwH{k6s@Hc&=UFI(I5_h=g%6GgzS&7C90(QIS%9$FWNXX@Y>h#lf%YNIqnJ2pK zL~u81l>akdDLV;k)?AOUt0Hv9X+qHN!{&S9#EIn{r3oq5WsYGR1(5`){5Eul|IQ%B z6^u=`LgxxX)&Ji1c5DC~($(t{|IT`Qm5r&iVBGO>{|31ACe4aHz5S*(i9|)$_w*63 zLN1QFw*)S?QL8!gDzF>v)se`wG$yvuSe!uMcGi8n_}M-_k6cnAlh1?G-f}+kRZryt zca1EH(civ)?L?-T+%2~O&w{bA0^r_Ukt*d+3@mg@wW_pP#92WmQt(}Dt6+3~ttu;Y zZ0E6*^F4yV`Ss2O0l!LAP20BDRuz^Bs}J?F{Vu{ocLZf&KK+>9b{+fcqVdbN{ZL;|%@k_F1_=J|i1 zi}N<;0(^w19I7+euvu|4m3LE%&woSeiaFis`< z3RN?ZB1uLxic!mb`(Mn5KVr@HFNQo)zk~PnQN?GUsHOJ zHHvN0tl2|d4HdmUp6?zFF*gs0JWMd#(<=>X)7-Ir<4%L>lq?0+s+C-MG!&!)X~Fr= z`9XUg-U?FXf6Zp}AEeH;QEq8oNdfrX6i2_OB$)~fhURl?Mb8B_!AVd=pm$Mn@>!^8(ly;rAVmrjsxAi0(O~sZ_2;!rHEG75xwH6=C=<}wETGr z^AL-Z%y(vHSC0N%shU6b-ns@(`vdJQchl%*w;YRmV1nw9 zIW+V{{mkxR{!O&Tc~i__(~CFd4KU;jL#8yY8BNC4r@ zC4C5p@Wl6Uvls*oVGeKJy=w}fXme6I*q$o&)Q`W22k`m%AtStSw3#)kBq47lI)OCy zzqk84FU5;DgwKwVA99!#iB*Q-tOIFe?ZU@)kf2~8*CoZ6x;h&LW6S64&c_QKaoH~Z z`ne#kGxk!E_577P{nh6gOP4GuB6;~gN;@W;>Nkp$>j_X_($R&Gm47yEZ#U8We_kk} z3L955u%Vt8UgS$1ADR?lHbHX0!F6q3|D~cfZRDKLRkmo5HXfMVT^XD%~dx0Zqq-)(M*Dh{=+z<`PiNRa>S!>Y2Ku zOn-fMj=RdXw-|Wi#yw|(FB-=-L zQrH!1dJb9c-`h9Lyo#!eacZZJDz{|^_+Qd>yW5}KaS-hw`iBsj3>uGyC|)RebSKJ4Kvm{5(?)+m#^8aiPvNFOKZ!d*$ZorXt%m zY$M8TgFROs9zU9$vM@g5mgs+;QhxN09|&7Y)vv-T-Q}|?=}PTvXgDb)wO96QCVMfa zvs`E`<&A;(XvlJFpnWLnu$YdTHfiEmJRxE-1715$`0|1yle)(fhZ=eK@W+Z8D%mki z7mMSXC^49Q9q5;t#(c;JLC?v8zBh}t5tquA1j^QQY!}7yhCkDaCJgbXuNp_a8L~8t z0!>-DV{bTJh$NzldQHx{Au+iF`ak*G5bdj!>CHW5V^vk;*Uan2E`R4bx;=aDiuEj4 ztatB=hNLX4_9;}qM7DGIJKzkFE&kT6*`F`GOGf!C`c!UE6q5C#B6l2Hd26-q+u{?M z?brEdHf^ai^s%A;=e#+7f2s~>wL(ep+NkJxx+K@CUYvsGznElkW%+B+@QgEAYex>d z^H<045fyiTZVDqHY@}+9uNQpV^-=AV<~grlD8XD}L`nJe-8C>MXwzpPc@A_5`1?QW zQnz`dgrH&k?L8nT1@K021s9a`CbE(2z&RAQ zY0DNtJ+Hd++haEU^gXI_&&JF=7$;U)dv}u`)jd-pO=&ylld zACNa_+`;jkXpV%NwtI_~ssExPC0>xWgJ%JiAVkzoWasm|VQyAVo*z-KAl$TC@-ygv zpNeG=ug&NibFiV%qMcV_9N_HEzC^Sxb?yF<`+|c%JSblQKB!d;HahL9W`yGlZX*%O zOqO&wh`S{|2edQo;x9TF1gydjaQ@+XmrZY7;}^}#ycfk#r{G>0?X(}>OTKX5{!PB& zBBUrHR0{hMI_4vZm6rj`VqeLc8a?rdoD##j+it8q6*L=xBRqV!XvDAW0bhf~@0~x5 z8&6)<$>T&qlNC;#lkT`6SwJ{>%0O^nLI0jQ4I*Rk z=~Fw|l#K?f=Gc|!FEAZct@AecuTQly+qdfJt5*$C5CDHu$KWHe#>jo*t;rNel5AwQ zawODMh&xpB&Fotn1-k>ab|<;@1AjlNuTnpFuq>LVqw&MyzoInI%rI4$l{Q^;0c{AT zOq>-1Gh6*~Cp8$TVmuZmH+RKNrc`|8DPKI;i7e6qFpWl0KQ{rx`;C2r8f_XrVAo)> z^mxDwqP#t-2RXbQ{4?T^d%@eG=zsPOy17&Jh++?lJ$>9`MSMn&()1#aSqmY9sR?-A zV3;)wFz<_48j*&h;NrJ73|@0<;Lf82QU(T%R{Q$=(e#KNcwP^olSIE!UXaQhgaL2= z@m6M^EW{49zY!OYm*t+!A=oQ09j+E!#nZ{@;Yr6v4ER%?pFhF?jktK06*Px;OtO-h z!z@-sDJf>R%bosw7ht^9I&$hx{x&dsAG0AMG=Bv`XJ8Ogc(b#HZJO9A7)We#J>NdK zdh<*5o>Kj9U4kt^Nz}P+-gCfZJ!o8RLt=$Mb2RO39#MQ?dqU`>ig8bDOV>T>4Go+= zEXFK$SJU`{0bgI_AKOw_jxy4Khc6emN2YNQe{;&aWJ~C8&kV%-ZojtZl>7wvN z;@h{1-roChBX(h!oy!cgS8OLwZUq^s(z2z)9sA|p-cRq08+?`7wC5gWpU6ji`C=)%6Ef8GIRQKSK)F&e7X6Yn^>%f1szW=x5EGkw&k$BQF0NHr z5DFMJLY)a}EnxQ->6#$MCLtlgKM^LN-vSG9sbNYp*X!GNhh-c8 zHw1>%5r69g_7{GXss}fFCuMm{fa1+LDmhPuaAz`9#BNZHbH8nNjrTZH15r~mLG}+? z?Yx7nFby4p^HO~eU{nl}N!dXaAJ0esV;mZfamZ5V&f3jUS)sOZ8x#_`9d5Hqzr=Jn z#M@&P{WX6<4TYhC4u{h69vCJX&|7+mLn!v0Jmey0OBtu3>v3~Fne!5Uyswv;gD7FS z+BA^Q8kJSDHl9x(VYVrhyb<)O+KEL)vk88Zz2`H8m8`80sJuYzJ=mQk=Maf;yYL0` zaIs!`er=a_Urv6y>^_(@%W2L%U#A^#Y7n){caB+_P(qKV2qt-ptl#?d zSseIxO1DQ|1FG6d7^>DS^Ub50>49%QDZ5g3pc;*ftNY(+&7PyXcK;ZaqS*8seQ&hx z8=7O_V#en4)?R0KvUPtIyg#v!$j?4w8MF;d2SSRaRLLv; zkEQ4%3RYG5hIRPUxwp7fjaoN0(54#wTfzRDmHAz4%_*ePDpths+kUi_5F^X8v|^Pj?8IdTU_<)yw{eBDB#+Q+1YP5?{j_k z`ZQPR*RTECwrU=yA;H`VWp~a(lK*+ZTX*QNLGco8cf;L%BgJFYs+>zmXB50@oWqX+ zNwYrh-Zf3XKqrWKFYBwN>Bw!BBECDaeqgAIaPm zkmF86l#zRh2yE)(Bw4t~GWy>dBey6X6SS(Zs?!#wd>zf&_M22Kt-tm0OD%FZvXqrNt|%4{cm9W*%R>Vta}a2Q^xiRt&^}t{e#0@|1It< zHXKgwWsIY%-@dVbL@R~(?0vT7qlG-ne$4B>{XP?N=C858+ zz9glZs}9!d(E|#j{?K8=o;RvU<=z+2jP(0}L@p{~WSg?~8x5WbWGj$35`#PrAw_ir zji=!)$BdSPm-2-5+{W=c;Q`1T0BX3lJJ7#ihm_jD9uUHI5X(I@zjdQswJzeO5&jj5 zTH1r8-cKPc7~`NvkQMxbN(N-O*l;P1gk5bnI(pJ28p=t3Q4^1>gjT(DC&pWbEnI8y z);2Z)zzxWu?;Ot1@ZOQU{~#hwN9?SFJ;J`%%p3f}OgHP1jwHqyXdmpyIJkzm9ROh^ zB_@dlWOs5!Uki6z4ydy|kjgTz&6}-b$!(LXY{fU|6CnEYdBqv&>DAdA?1|%JzO`zu zu|k9n0P@<2fGFobIJM+S;bz5p^6Q38K;}w8-CM*r9`dH@z_mV#tE| z?Z@Px<;l;D*YB9;P1;5FvZZWa{FG8$1Co6geVyVG)zUJPyGRW! zGQW5+r_4+)#DLddZMDBD#=N-DJrJ@QJ$j4*PSY)vXvarq;)|A(WU`rGxlGr#ad~Mk zDHPsNH|BK(lX#Jnoq~#Gc2xq$wk|zB_aQte7Rn|}^z{x;^Cd-Evb{#Ta4qfk`PcwP z%u$;5zG#s9L;EVkLd*|INF*dzod}Zue&w8#`;ngmNI783mMv4>A%}{ecSlF?Tk31r zvzuC&$Ji{bYPrB<@795JRdU*X$ct8fb9BU+bEW3~;xOi=GyD7xRvN#ul-(`*o_XG_yyFBN3IL!>`9Xf{N}_Psk}UarBrpbVQR1VA2@~E~63I zyYSgp-C=lUY}v+Z!J{7kB$6{-x*tHG@}GN|mG61DI}47LG`N)|zozT%u)s3Z++6n# z?T$o2Y9)$%?)I%P(r5^L+34`(6W@iCko|K?y2G^~a`Nd8k5nb)05Ge*EdU ztN#v*I;eXnOFX4yAc!B4^g;=`U)>Eete}Of@LHtv9~Pgua^w)`Tp{V zB9~2Fub#N9?v0RBw81blBT*-bi}1p-n_bYI*9h=;P6iOmxUw!`!o`RC?rBC$L9;8M z8h1)B$hk68QE0Q?7cq_AUo@K8tR;-QMl{nn-c0|}6n_WgqnKeN9v*E*oeD|!=8`)B zGRaG|mm@@q=|-0ftZ}hkN1uxx_lskOaA8Kq4HC1RW*zI=4s!X?D%{8w_}49ahJi!Ye0`O6)r+V7k=v(fd> zx&H>>TJ?)-7+=u+ig&8>>;W!@hddsw<9<4;<8*7V;|9mz(oRs(;alAQYNfo&Q*StW zJ%JAZe>Ou_#Lu*`X#mtOYDwXQWV>~A?A){GxLBgOi*HAb7K+?qmcw`g#DIdN0p!X9 zCJ#Bi;lA^KdO>NCHjcM%3-3297RO!0yzSQ7GGQ$r zGdbv;d-qmhcoHH89D$Gpn1yY%LI9!e#@#t+8GMOb_UZE}UMjud`Ui-CjTV^oG zn-*Kjf^+RME?=(2x8+LMr+BoXxt_dzJAapb^K8HkEqhcy5g2v*_AH-55Q5#0Y(F%gz1;ft~7;CzAoN z*o)hw%oMSjK#{XjowBmCclhjYr^HyN#Gi^6B5na7BO@?X-n)DHAPyC@e7)08@dkC)IGLo|zIc(fu1d+V9RvSf%eY3A zXahONLh9V}@bGw2Ti_=L4b8@ zw0_J=qCPlus22D&9RA4%WzRLP13@#ZjYQ53xc_zFns)kV(Lj!$dO@CH*B~?~q<5=e zdxb&)x}4y<@zHKoQJYXiW9zf{n1XypnU^~LTfBb>DjMzoyhEbkF8%eWMVuA`VMsi! zvSEXJ!XxR6!ctn{6xIg`|7}p#RwO8amoHtS^oJMiS!<4k^?<=7i8uW)0FH6(M9gs0b-hAtF;HN|K?B4U&W;Go_G(q^R_N>?_Xge16}x{;kjI zbFcd*SJ(A^z4qSE{T%Sa)^Hii^VYa^>&p8=o|B|Flql1>-7!1K>s+4y`vF9(LgfCS zKd7yUaQ^>DzU9pPwR7zCuk}tJzqu-+L>Q#WUH(SefeSXv;A5hxCKt^9igBdEV2*bD zgx-u>DB!6GK}{O;SqO#Z?YVc6Z_246FBbCZk^)^On`ylJ^p{5INHK|GL1WL|sT z#b=1b*~r5j$+Ii%#u*iQzihwOkSwG)FRfd*PEJjY9+2O2I=(?*bdV!&6*q2(GItRS zGckyNE10V+trd+M|HB73DF@D+F*W(SdOeAAc#p}w|5IvIU_l6w0iFfUgDr^!)9Z38LvPss_v!h_b2Un_)}_xRMBWm#2GRi(kH7QS>!f{(tv*ZF+6s64jnBp)<=wyLzXxMrpQ;Idx1V()IG zKAmUc65WnMmm?i8K%}gx6D_~*hs$Yce=?;Er!$;}1NT*_Mfv&j zi*uCILODmm@ex5D&Dr)XO9t#k3GohL+o;}WlFFM$Mbg;|lju|64A9yMgTERkPD$OA zZ9kGUtDk*b5>q7mm}}rj*UZ@g%XJ%Js$XOQiUh)NNVP?al=KioS2S}|uUg`8Plp$O zxA=Ibtt5>)I5^05+Br$Gyd5@)FkHPrrW6@9x*XckfTP(#M1W*NchI zPb48s`%EP6KT$36Us zUe${p%lIRp;0fOZqB6Jg8xm6cIwtM=j6a5a`TE)L^FoOjK#Y7-e>HAjwZz91z9?1@ z8l|Y+257ZM$S&+DQW1qgsAmwEJQBlO0RvduEid8>$e52}4W^EObU{BsI$~4=q+8j} zXFZ~A8k|O8SF;wirT3ABXTX5Ks*$bVL1C4sx_iiA#9`9CXa^#0fEq=z9uLl#nT#TI z_zF9r|0L^OY&f*>Z5RP;G!Zi0I{*HYy(H`LV(=K4(q-p*lO|>bfjx$j_v`m3M6L_=ZE35K-?@pR+GU*o9|B?edyL6isH)*k8Kd8-4!v|V zW8LKi#hVR&?~^#*e|2TRsY_Q%3O$oI8mT^?k*YQL+4Fj^d-pE-3x*$}EkYW+VtDMt zJ40YMkpn>S#%Sy^{OS+bylzgL&Ax$^`iET!6}7p`qw+{m9BONO{Lq3+lq+$>jU8zN zXfAl1SVDR9qYPsx^YZhXyR7w3r8;>&&4geoaeWKsxUyx1w%y}FPd^XS-+6(+7%5FssFa2_K09YXgqDaU_LaL#IK4wGo5hh7 z5F!@;jei(#QpF)SmTMhpT)2tV<~F61ANZYjay$lIG_Q~=J)rfCfMN)N2F zCh+`gO~+)BDotHR$`d%g^#cK|RH1w?6`I(3Tm zI~blm$_*K2m%;~$3g280(2n1c2<$l}j_*Pdun3ufcCevw>_8FzF-7ip2$x)}JtnJ% zWFCPM!YFN;RdaqZiefh8>;aROf9ZYn#EFp$792a2$0vgZY?w%2C9DNKiL7r-BU6DZ zL^>hno&T+D<~JGF&zg#uw!b|mF}u^jo@Tkm!wkE6 zo76aaeCMLfih^yivuZm&*=+FMT*GhAu4-10p=-#$6)wVl?1q~A1~ePEq{~0)E(z`G z?(W^-ul{D&41D{j{*_kquy)NY)-?`SJ!z?3^KjKtyE97Gmus$X(YV$3San>$oJU?t z*X~fJenaI%%h1P8s(ygZh&5$H6wb=z1x=x(^TgGqePv6}BZP^AM+|yT!N_%&y!Csx zImqe4Bhl?HRl((EG@OF@!-pNLX#)jtkQ% zHb@&oLev4;M^WZ-wP@)@#rMPS@j353Y`&8Z9br6z<&OBp@)54>#%rg|IljOdhOJ7! ze3^&aI&*LD!|KlAAP!?x!kF^Tw@$^7f!iL1q0cnp#*!#>=mTtGbDGrHU0NKKVUvpp zB}8H2knd>|+B+ysA`0&@wb2R|35f|J-qm~e{KS#y5}W{7cmts!jZTydzI>r?ej|j)SPkj& zYi}g;Ilqm%TEW}5SALndZdw}2LD+&M&) z#t*Nf#0NEm@^2Z;@vU>HW5_Feae;j}4e43_l*ShiibjwYnaM4<)E$nJSg?YkSA@r| zfkdhBwg7isxq5hRJEm4*$RzZo*4*!zHq0*fsXB<^NVx7@IW4{T_cC8O>nfa>qqJUX7zV?e0i|JrmML zj7x+c6g+)o&wf|8ni^V;uW@Ni#F{4ckNT}KJ+{U;RKA;ruE|>+d#>uqfUxRw>O1yw z+`a05rlVT<;w|_n%a_}DP={M@n>RP9e73*N`OEf~x21L19oYL(k5`}rT)LYkJ3H#QDIUd8Z+VyXefB|NZMlk6+i#`x=G z8odADfd_vH=LbDg{8y-o)v)2VSa3*mPFLsj^bp5XM#w8Ck~9dEFvVJ|r{i9mxWGpp zcN;uAZGv?t`>xTp_J?BzvJw_OJ)>0;Eh+vwLmPK2da<^7{%lWEU*?g4 zBoVcWJ#%K>vSokCXjOUc{CUZFl~L_eU>~2>Tv>C3lPyj}^f%-8$%hT+5Nng0-V`~6 z9uj5@!`)-np1&pKuB_-?D4)ZF!&z_g;%BAbyvg5hLc5`(pXiK}B>u9r2A1WTF<;;I zh;=cTd{=f>)j~jYw$nL@5vJ1%qi@H%*mW&gwE)Va!b<_@AKCXD{&jH;7d%eG;?_l{ zEx?e%CqHdFL%1)bhl4j^s^A|D0H31Uv7_bn_+9PHJ;D2F+9pn$R(I&hS;_jW?tpFl zX5=s=`L1-I9F|G1w`BAzb@O=sHF$*APQlgX27T4Q@bpbV`Z;~{jDe-@9Cmkukr8=! zHZ%zMrR_w1uszt${@oHeZA#7%hQvOFm^6;=33}ZvEXkarwB8EFGNjY!Av1r;fcmF?z zkx6shGmcnaOD+mf$*8EMbfV>}9q;G=STLV%?B0h(zkK=cZLxXH<_U?n^{r;DoUd-4 z>2~Di*)q#Nf(_3OkC+kIAQrEG0V=RO>4z-rx*Ovp*0y482<*{{&q~aJ#M;ZR!iYuI zT=oLf65KCtd9wz!DchZzJQ4~*D~nlIyS)7qwHDw}VOX+;8 zU9V?~cuGdTXH}(Do||(6(yXPe;F$X9wN}=H5g?jljBX`tO4Nh&a;D`0Mw+rLX7eX3 z4qDO&i}%@&h4>F=8C+>z#DN1f@D5c}v$%x6@s0rddZM~A3m?#Nj>vhL#UeCWeT2$vuln0)lxE|&=Glmp1TLwOQ+pA>YO-oHrPeO;( z+2VMpWJ)r4|Kz~t;_deve}8!uB<#)9{1kVE{1!A0G5fpiTMWa-I6C&=$?9~Bd*sn+ zGcOe0JY3x7?eEi}p2QO&s5N#-9Ik|_@A^D59ZR?xaCOW6JdITYTIL;jZZMMQ?N7aN zdN)QPYJxh8qDd@Q=P3>r7qhgQ&3?h%rym_0t5ZH0m44}L$LwR6Tk(Nu^yAuOuR_w( zjKfn5!?K_XTE0W;J7N3OL;Pw6-^&hK#`pQ~vgEDrgu!zsk8!gTbg`G(WEF+YrLYHc zFWeu8S*twLGiK~L)V*#^nzfEG4Ag1*r5ik6htl$JhmEu4i9n-hICQ9^0%sh=TO#%&)PWu6c#mm5i1_X@u6uQDss0{yA51_TBQi6Q zZhqTY{tZYDg%SPbj8bp^j$fkHL$YF8eKY$3PNt3e=|G9IW_$K&OSc}hc-#?dJ#$U8 zw`adEZdGUf%))rVcBZg0AeOv7>UfBX6N8A0%>1!v)M!NZ?Jq< zSeSO$`Se@11~`Ts2niV$o_X&c68_->=j+n!Lg^USCYFe^qnQ;q>iTGdGe8fev%3#^ z&qmX+*kXgsXL{)lw~P%PBUwZMlJ0m&p*=Z2y-iSCS#{HFa(L!BAi|{wjd|llY51?r z147&H7e_@N#&AgC@xb?v0Pw2E(<9R2=@yFvKQ{S<40T5Q%IxFc;bhEX5#76^dI;jr zlm}O@4#${wI9WP0%XNIme0$Oc?iQr-8*d*D%qS2WG0{FH7UvtL3IUBEwpYuWA8ORF zVIuO87tHWWNWt!pk)B2j6?(<>jv>uLjvgKTX^;_1<}F+P;EmNbfTD5cj4B2G3Iw$a z9e{ z2B3{cP*_0693gySj=NLAj=f5FgL}otQ7IMupI^5t#uaNL?KN>6)wQJh`z%=+GeX0d znY(1>{1dwhIv=Ut7gwC~je<1!x@H!HQ@HPXN^>nmIz-W)tB{@Q3F3H+TQd^EK z2nCJbM447Iaq)Wb5qiIUSqd%%^VjFu*$Q$HM4il z!oGhM1tFgpT4=-wt0OfvH7}_?qZA{at~(L7 zoCRKS+O`p=13^@<68~GFR&agy7yaMNt#5Gtq~~&aoS9i0sPu>Tkm~{LQ8+jIMh%ZV zJf+M3_uQ=dBl6XWh8UGxhOHcN;$smUG8$)kxsdG0SS9wdoyTp>l;jAA5~scmdQGmC zh}a|&LF3>~NeJwHAAm?UT)3}p<(OccA9DC`;#-!|Ybms3ui0HrALr6G|NZ-0MH?Gu zbQRaj_`APfIP0HRSf~v;@hTr2TI}5@`~dZ^(u5%}Z=QO-&4t5UWgKWT6N#Whno*_J zBwNCE0al^qTzFowj1Cm?d3%Zv-E2eS(GB8|9Ky`j=*LsZc5h480}l_=#<0+#Twi`$ zyb z+Bq}`z;8c&x|P28)cwbgw}A67BMgt9qr?W{BiC8VO9vkQvTapmjm~$ksfvM?_JpyFPpNhdYl*%6e$!&-9vAh6`Q^7Cz$ z!e^2*P*!$!8qe+3wpC@Lx(}G(%I*p$Q*|WuU#-TNqm^w&73vMcuPtcplPB88Sb?Eq zvLAst5&?jWg~z8!HXN1k$+KPE`!l2w!@KiG+)>9!E+1xm)J+o<^#_O=dIQY29hCqx z59W9#C(^P*rRjc*B5%P@bk%!Gje%0aQB%6G%<(}+#^urmhDCGeO&F=y3*!hUVL1c06<(JLvLaFlUo-nkFPp%_;AN0L{Sadt>q*qe2c>L0iDIKh3Db~Mc~9itM@#^ zvY!uk--QMK+YIu0{``6Ewg#|UrP9iFZqI}fhLMYr=%nO3 z^Zfs)K5hSx-gX|^PC{F9S6=A9n808}oMUW7+m$|{arW(-H~(PKKM79+PtUJedr|VR z@EwLvE67CaJ=#SOvt(v8sQB<3M4*mKkc6*9vMLcW|DB3V z`vXvO8`Bv+N6E7JeOoMfP~dZ9KIpeTZGSSj85tX|Zl?n@%GYZR*YIkm*AT0@HV?PV zewf%!^iTfUH7@mtuqd-Q-RZ*Oaj_|*L2jU(i8l7W%xuE%`~EfGetXmnjOJvKzqTuLNn0T~!k0c|=6iY0B{V2v>Z;R! zzXpVX8r%WPzI{G6?L5{r90*aeU^`?!793f~NhlCr(&Y)|VQTa+C7@}+g8^gLu+7H+ zazo^*&xzpZh`icuoo;T9_zjH?E)Ti7VrpD!&Gy#S;4sZEt%y%hP}lolQ;hdsC_&G+ z4K_UzSENkJxvZxw-lz9{*8J|+zJmvE1sxnCabt)3T8T}E(hh_0De$5K96ME<+jQ%fR_wGvyH`*fGb7#*!7hmww zcGXXL^>@pje3H;NCH_da=%ofC8e!t-=>0TjADT5HEyvtqhxT-4c`gGMv0W7bDb1VLp*WpygY%6Vu_utt8rwmQ0Y+*i zKr4u4@iW~Bap@OEK4d2C``o;6B-he!-hcz^ug>3>etk!e`FZ*I z9lCYf5?qqfJx^!3^k$TlZE*&7H}})&V7@gkenN&#UxEHQ|CzmmWCm^xMLg3N46yt5 z>&LdyJkfI8I17t|maXu`DXuzwoB+BU0N^Y9WNU`HjyBQYr&Wf;=T4j>Mk zaEG~#jTTlAmRUI)|M^EW!Y`LC^mPph9oe{+R1ih^AOz1dHh*FMeQSe-2UfuvO6wD3 zq#0z8ZvyC2T|TP4QmY*p&TlJ9UC!=EAEwZF>ihX=-=RpGpT8@tD zEzzNaR;aQ+HZ3hAV+FcX#h}qTy?C4i4b?_93#aw;euf<|(*;jmrZUjAUHPE~f|c0? zaWeYnFeDVaX(?Yc0iRzM@rM!uBlt3%acmAnro-m+CF^*MKm(W3g|`(j8@|J$Mdx;2 z2IW^+Kr)|*n!w#u(`^*p2h>@}ikGNQ>&hLVzr(=Jn-QR4OCKeWdpFh1hup#b=y;Q5 zt7Pv01*WPj$v`pEV5a<%TO_yO2O(9_M)~|!P0|%0`@n(AZv!i5{NGJ(cEfTrtX_X< z!hSs^VL0n2#CXrWG^aeViQ(x1C2BXzA{_omy7J_}jF8Ree}&N!6R#;?w2BIB6T3iy z7`|+ouHeKxo~NKdt^;gqh9luI`vw?HjYXH%7?#6ci4KD(auuh)cW(A{xkv+^7p<;3 ze`rh*D>Qibn|ZIEok;5BHd&nRDXUAC#7sL+co41rN%u-OF%2>a&9p@|slz!C#9kDz z-fwE~1cg+`d_cknUC=)}QBC^>SGu@_*x9MrF;{qadiwNe>hoJ|$vNV9?hXi8YwqxD zL0`Oq&>x&7Y<}8`Z=XB2F>t>*qB5ns!U{5WL6;0?&b{~rp_%C+q|`Txq@$WOADp$W zJ!bT1F>UXdHdzvqnl-y#m9E6*U*InR#8RQtED)|Mqbt4jb#wGo!T>M-=+@h(aAtyg zYmlv%)7Z~5?z*l{L!3PV98A|8{TWvS9tktIjq4heZNCu`gM!x3#4(*Huj%%)-BuCd zRe`2R3LSxRAsjIs>4=Xox|o%>uGihC?LUH*k6HqDm&kfPerY@CV(v zupY3&Y;S*bdl^3*kRvf3B3s(M_31SxcX8HXsu?o~4*eEcTRaOC#Qk0+@zt%+~zb405? zedEr#R}SD`eQ zc0Q6mXEATyuyEXPu3}995i_RXqvz9m`v$ELd$2LTmC%J%kRh(G)=W8L9hf!P>lPF_ zC(;HieM{}3 zLjZ1JOTTk}$CeH(TER63gKm#VGHqT;UY^d(J(Xtv*u?c|>D{>2P!o%TlgvF8(xSK` z3!CNCZ<^$5q2qR4GlwUNdZao)Qu|NbBz8P2fwKGJ#S5}~&07N6X#6=nUuWK@($f5d zX6kC<1F!Ii!-FofeNh^(Rl>)!3~5y`Ao;F8`vtERtJ1Qhe`8TqH;;Iq@G=So-G7AJ zmCpM8`fYH@zcprGx$q2rqrDFJZf*GaFh_PS@V98=Dqb{NJ*U_@M1#f9wBBEk^l2oG zWu#m$iPEOt_xrW{Qyh914c53 zp{TtiX3w0=3m9#=;8{0$_;$f&9S zg(I*w%0V2lYvtcfmN~Je#W_?wmHxAAF|EUuKn<6%g2J=IUA4Gg{rbar`jN`BJ=^r_3hPKPXft=ts~qx3_9mkvhW7dOJugROl9dkn94a;vu%EE?sMUSG_Aj3 z=C;EBB(eCndX|*5$vfL-3^FVsNbzAPvbNw&;s2m6(?vtNbq+3MgSCw02dDygy96%1 z{F=*ze_$SOAs>)T(q|2>rBuvzr|w~Ve7Iwk06y_VBsFTV9CooRF4>j8aniNZ-Y+2 za2T6Nxt{UhpSnE)Kff;dR{PmAhUHH)+WGv+stE98v?RWZs&x3)NH7WL`b1UC5D&)U z0cUT5tuud882G<`4GqpexZ>E6s>pa9ex^qifBy0DdL}enJX?9IZoYfeW#yXe2M1u3_j&tmKjQ0+B zwY{L=^fwX*#wG4aGCbXvf2Nh}{_%v!Tm`J9*!@S&t{Rz6CO8aP4Oy6cJZGS+eU!IO zfm;Kzzj%42c7o@us9!Xyd9#Dti$=9!tLO6vsFCYj2D?k6A0$B8L1w-4d@8x@aIS9O zzklZWU^)X-(C8v{tM+RDCdTW+Gt5JlOb>nHTbvI{~^|J*r-?J|F_^rNN}x9D&7VMY0F z)ZaE6$}g_^qe?ZW_TOoDW?s{x38l55xoiA?Agixee6E5=y-4=yP?m2{)%iXboM?Dt zn274v{nM{}Pi&B?zORCu|ERKzne-~IFhbw;lq1IQ4tb^}%NN)5A)n9D8Nnm!yg+vk zY(m6zI&C|IWSwy>G?y!n2D2TK5#}-MDC}cqj~Z2n(nJo?CZN(eczWvC9U7e)45<6B z2fdnF_Jd8P-iSw;2E9b8z@Nok7V9*hzJ)32=>iZpH`fe;-uBaIr;s47nA~sBcaFqpofNpzIt`VfXZW9iuIPf zuHJ{#&EtT2BcA$N&7ZIT`P?i-hxkEv-*K60nYR7#VqzBPejiAnti?8)03Ng}z?`?1 z4>lZk^Ks%P`(IV?oyW zA>p%Blt}k!$SAQkv%)VcBQsM(k7FGA%~tCWay{+)LX8je_Te!ZIB>3q|9+-oIsNf! zJz){4eP8|M{b%jt`ZJqX&FoM-P`{UM(=G@7Le|vR?d0U;8rCBH&4U%Q^9}ThR`+l% zawvNLbWEp_lk|IezWnD8b*(;LHNDo<@0~JmO6Kw7B`bV#>Tj;uuYSyNuX2}}ai*(+ zV@@tSxB0mF-O_B*BDf=Ft(dX*>C>HE|L&=y8Qj1EhSNaJp`Z0HF8f%szvI(u9IL#? zn!S5Bfs4=ST-nG)MGQoUkr}C|gH=jd=vjaMh+SysDDDC%$SrTgh_e*7JrmWQs2vC+DX|fjo{DPgU60}dgm^fm3E;;+4%lVRN{OCo~Df{HLZK;!Xc^3AjyX25rTJ)@ zYw7!F)()BTIPxK@7V7P%!MJ1_GuDE#hE0tYHmEW9AsiqQ>A=FiEe~XVQdJK5+I#Te z_qRXvMSHM*f<0i1b1r;<9(;`e`t5!50fo&<~xzoD&o9%MRd}{qDJJ9;>J7!{{ zREW5=|EjZXc4?nT|FL^rXjrb&AX@fAa^7l?+Z3OKV7f6(l*Z+}$;xW%QxH(#at*OA zhERGz$CTst2Lwmh|NecU;Uq)Il~>~%D}_%xRTF+we#zO>bFhcAvVaneeq_U)#u@k+ zyO+JFLrKyP$AC)Yj`RedZk1P3ZK9kOE$T^s^YqO#b@lWt4j;M}Gcw&bKO67)q2T)m zW_l>XeGA;c>v4JI@C}K~(%0LY@5aS+wKRZTC1x>01BMtbJc`p9ZT2!q_V5(@FkoKFR7lox*pu$$DR_yBP&I&-~18#HXDhxK0X%+w#UW* z;0g1Dxy;M*j%|!&)-q;w$b>2*`*0@P0UE>k3(TxyubDY3ex)b9_6x`pW~aSpq?Ayk zB$hNPMFru&Daan|+q;5fJ>H4RhLKpdCMZ0C?dC0Bd~fb~f}aIl10kok5Rl!_aq|pT zRv=xWACb!P4=M8y6Xlz~`|Cx)Ktr?!I8?dO#R(G|r)7czSHz@6Qg0@dhvz~ct>WUzT2-AXfd8K`scHJ({Ih4fb$p-iE5I=W)TiX7ybqu^B4Jm&FwytVt8 zsEop*`O(>eALC|lYwi=?0^x_*#Ow)Nr+Vw=m?&6~?|sU<8onomvxT%~@Kmp!(-`Of zoSvg7X0=0%xf3caGx0(x9ig1DXW`Qeoz=?A%CuaoTKR-Eu2q8)Ou`f45veR6PeJr* zlO8yqU6CYnghNzc9^5};I&a=E?i4tE?Y*M=*ToO%8jT&Psy8ftTWAe}MHjx|$--Tk zT*gI<3V%~pjypnQys|j0e^kNWYiAc_in;Uhi~XzDe~RxJ16#If1IMf%>%vyI(r-#U z9LpJW!7lseep;p|3%UO*3m;!}n^;{Ha!vMaZE&{aWl3{gW#zQksOG-Y-bY?_UcL9u zq`V)w^x}WVg2&_{?gRkzn25ulOHy&i~_wKv?=#d&oS5(m^NODlu)<0Re zsa;(RXrjA6o3i5W8JB+lbQnd&gyBvoqPg+vffq?zSYiNZwY=firRQ)dq-(J32T>VY z*Uo-*8z$`=HF6|J+0fCM%E}a?dyj{2^1e|T&XZivKY0<|Pgi$ZK+lCYu#none~4>k zQCFz_;`))`duLt2yLW!xeAdmz4m-V!i5g7~n27cL5{f^2a+2znN9HeZa9E<7A57rO ztSU|A*?`i>ix4vXzl}p>#?1cEb#BY1TaZpx4H#y?^~NJECCLkQM|w z31;QEx?C;XG;B%&d}634C*4YHrAuO~EIPNc_veca1wL6QbL)5Hoilw7-|VupJ|vp! zt1FULk5;*sUFbR^&bc;b44#vF<0>o$*N}-xDgS3mB#J#?4F=xVeyt20o3M5VyaHVs z7Xz>(YlP07$HdJs%vFYc4s{p=*$YhZ06E4Jf(I{oaLYRSKsAgd$%d@*Pg$lksolia z1WFW~MtZ74LNg)y{IJxBy7lX?lL9d8CRfZR%~K8V9%x7J(qqgla#-$9xC{xyB|*ceV2pXoeOi0f;* z#B-g^bv7AX@1M(aCTjiZV~6br24PXWLrE+K2|GP8v5R_odoT6wv*Wvk303)VT=CxT z+F_aXF_s?MxRt--@r!N)DQr!Fn{GcPZ{+S<+!O#{5^+F_=&N#P<;h2du%QN0 z@2yJcM&|(9*AG=!5Bd6;PE#n*i_h+w50^}Kgv%y0`n6cBXh}#^b!ZJdiBAt5@b2ex z*vi4HRi}=Ar*=LK<^blJc%A>r(h9jgJZ!Pj4|w-^mgmdvOSW1Vo$T6rgNN#|UYECy z+nb}iqp{M?v7w9i=n>2MoO8eozuh)O>!s*k-iAdsmab%g^BGd3J75kCC)(c@L>z}Ql5qh_x(GQX)Kz#U6j+;u3->K~#NJRaJn_VEb=5IFG2_jh zXPXy%2ZR?1bW4_k77(^s{?$LMZoZ6Ti+($5%r`0kQ0SgOQ4q>1W(r!Wkw_|_uHSXu zo*$?e7nWwdp8MW3bzEX{B`nSC2dSy!4sgX-h+OB5-y{e*3ddD*vbT~rd&M2g0FaM^v9 zkmHy1LZgk0rf9X$*15ZF^KS|9ze5Q+KCxxU^;em_@(FfGet&V96)mkqq7t0cr$;(K z{ze}0R9m3@;!ln$KKTns<|^BFlA<+Y$@e{Vm4wnuz2epw*0l8_TUIf!bzzgIJ({nh z2iPG57faSh+t{5Kw`!WvjpD#_0&-Jt^Yz{ZIuuA+aP}R}XW#FAoC?LN)t3@#&qw@2 zid!{M_NgPz4bc7aKLoT8h}o3+_yzG(IBExA3j z_Fc2(ouG8eQfi7M5unBV4k^LKq;iC2HCTNW_SvC_%&lVU1m@Ry#{wo~NFHq%B0?X6 zD^cDwAJg)IM-F?&Uhx7BxThB^aS$ zWHBzo8}4k6=2PV(mBi&uQ`ESDP!e!>{Tk#MhG>Pv#hIB}s937UK_B2&y?O2g+9-%v z1Uf8n?h>-WD)tflYTTNx&-2k%G)JnNt?rN9+NWRt{`xCNYLBE%Qj84JDScVy=5Xd) zf%{0xLs+JBO;xuZ5M+_utZSG2mN9Orr=6j!cXFV%)LnPU`i5b5UO@JNxSq;!+gN00 z>+0(6ey06w7;li^pxGtrO4v5YV>2f^%BsG`!Rt2PV$?20W}S12vJcrH4|vhLf&Aw6_u?AB?WF{5+Ds+UuxMm?xT; zyj;{eqbsy{l4yJI`0m}a@M9cJ2Ks%g7@A|fR#lzM-vNAL*h<{cC%tFq=>q5#Jm?Xi zsOjmoPm9HDi)rzOZRFNoZp)W2LD*Is({p!!yCZ?#!dyCIC|IgzO(jxiw*lg?N zH(Gu@HMIe(VMa{aqD3ad)U&S2Bwhc15C8JPu0QPtu6vu=pMBczz+$WflrEK=sCTXQ z+s6A{3hPD0Iy~A44kTuy$7Rz3A}$h0z4oQ*AsIFb9Z3G_@14Mb4t=98oLc$S8Dk6G zLB1fWVyFRea_Di?HX-mpX+e!seei<@0{aOp6~&h~1eEz_+qP|YqZ+4a^y;MyNsXXk z&naP;?6AaB^JtQ>4)8jOgCkx(m?76cICfcGo{_8_<1C(_jcP*`0?Iw4J;h7}0 zLo^wg#1Wo!${AT3A=aoVQ~c~7-@TjjnJ9(&fO%z*9YY_Ie~2xh7)l?>Z=c@#(LYar zIS74%RF|zN)*SSyTc3noDC^oobvn>D*N8TuVNwG)iwsn!v;vn%)CEperfm2r6VBt= zvqqg78@i*P3;7h1>Ktv6q29?$9k}DaN1(P&(D*mAGaNWuTmbw6+0J=6*=GVA9tHG^ zF^~8hIlAu=F|GhrHcd@V9%5p$W0+^I^`j%ecti0Qfsa zO9y4;HyxqOvM1^xGxlJjI-Q8Pt69=7k=vMYhO{RFU&;!MQ zQtW$IdobrnTrx7Dknjc_n5eFvWf5aeeBye{%8ZO|4qmziArK!gKW-!Es8gpy8$`{= zZ<=DuML;h!9D!T!11P1QqG|gJ8_TtswmUP?y?Vn$+V*(VuR&7Z_Oif0#qQ13pucB* zKxZ0Y1_ptso(8H8LiC~j_si}-LDl5wkGAgBXUD{k7H_3a!)R~cb1FV|Js%l19o#x8 zT4g1~?n2ne$b(g$6i!_}}@i{ar6Jgh4^d=w3fs79?wN3oNT3`i!6d6Ag=%vWj z31Tis%3D+8QHJ}vwQHw@fbj}6BVaObxaN{fH1|A+t#qTT`@{CT&{w0Ss3>ZdXSq=+AZS;Nm6wCmdlO&fM{IAKS-S zk(*d5`rmCkA#!|M160>vO5>T)QNNEq<2~}=i)RMfW5s?YGICcYRZkQkRyEUxMP@<9 ztAo9Si340KHtuwOuob(ngtKun+dk?W&oDseA%>K17s1=DEOg@c2JyLLFh+fBbswkf zKM?6XUotdzF-M;TV3avM9tgldvJM!8TXMHs&kTkP$|O)bEO9y_imb6eZD7YrIH z#~Dm&>O}WrhHJoIRMZ=_)SW{xe5N1EeDTfmZwd;Me8{A=BmD2y8321Y5dhvw+M(a- zH2iR(m9o^NyWL(Go5M{8v&uxbMaC5>>zc8l z>$JYRMw`u^-4>VFywBaFFqA*5>h9f1Q&SQJFo`}gzkT}4!a{LgTOdfQ(eS>jt>gXC zFqBX0o>7zu{ZS1iCvcPBN&D>H3^BihBbgL3waC6aDd8DSPHN)s>(}<5qcXlv<-nRs zwC3u}A{-18Q7&O21aE7p>p-^&AzH&^x1gH+sq`KrvH30gx+5IL#27h0Grjkm5A{`#ruw&NEK_Zxe7r-_O zpYfOaFoucv$}iLTm&|zNCB7a#tcg|&<;q!o)Y{gmUAwd2wHQG<&MmG7S)*YABo=AM z&p$5hWZnt@z03K9!w+U?IY0S5W8Q6KcW@wxM0qxMCW2?uoi6N8@`wIpmBUIv+_E1O zEBln&A~v(RBD%IVjXG_!)(H}jotpk8@OuE9pi?gS#*%-gK2N3u9UF(v7xPSHI%<#^wMM8(U#>*Y_%t+krdDVIwVd^}zltxGy0w zZGA#-25`#Y?`0P6gR9j_)hFfR2HD8{{r5p`r=J@+z-V!TN7&#*=Tv%0Iw!ARI;ZXt zcYgT)+k9*p{b12WyJ2tsGBk`n_q5;IvqM_fTzq0sFAPtVjQ8lSO{gq%JZTfBVmWWvcpkB8ADGHRCaAWNQN@?avKu?QLhO3pk$q%S?s8a|(nQIIW!9|j z<;EYqGHplDwoAhU!v{t+;ojzJuk{TEplytW3RB*=7#2lPi#zZ3vUySMZq8rxApXu?R!?E0@&2oP*lkDB)GdE8FTtcH1 zdMd?o7?3i(Z&Ra~lk~+I=sCi03F;yxgT+`5r7-FtyN6SpBt)_Y0I80X&CtU}CHwl= zrXwEM>+r}jGHV|(aG%VS5ccYGaph<0Um@1NS6N3eeF$p*PEQa?(qlKc4 zYuF5&T%fOQ`*WfQp7QSCw!y<_riLBMBRZj@T{uZkA#<3BrJ_i3-SC{7jJhxQV^_F{ z1+Ofcmp8wQ+CTbSv7%{vw>zh+aQooC;Za*N#bXU%U-mL)joY?*f7Z`f*@VSL(bMY; zwsWGhTixewnTIbU>e-nm|1b?}CVSgVu5;phRp04(CJ*A^p%a~K+Mq!LQG^cj^icKJ z^1_tH)e65uB z9A*+gbyA#5zGdkPr8USHWY836vf@{~`(*_5)t=PUN;vmoJ(KTS4XsvxpLNwb`UQOq zS|``Df6vooOAC9BK^ukV$8)wJX2KhZ>%H{NPu|Er&pyR)llbycXRY}n+G)4L%b@<6 z>LzQlHA6nU(cP+9>s-Z#fL^_NNsKm))S*BvFM-D5zrF3M0ROk{8FMh>`Rj~>W$9HF z6U=<`AlD1@=c9Q>EU-W!$qNd7$o}G^W7QW&#Ig=6&R(8XrZU9`1(l$yoNPh#2m;}_Q0%?Lcs4N=aYpHIj=rfUBSVSFH1EA zbMmAnJiV8j2ObMh-Oao0Ah%j9!Mry%kMo@`jn?=r11}ypAI;xR`Q8wI6F7$_JCCmZ zo{tslx`CC;YWU~}xrf(Vv@${^t`Dp~`UNv>>^TnfJ>wC^p|vnEfSs$thEEW-w&1*||plEeOv99w{_ag7+0})s6 z)M@_HeHMeo+Ypls`W_QGTVNTJdp7fl!vDT?%eGL!I_T+{9Wacf%5+oPwq=W!%0D(f z1j}9$gHiP2W`mH&0L)kq@2B3sgICDa`L+`$ZXdQ_#$A~GG|}Edv{}Lwkw_J8I(o$C zpa9>6#b2g;gqw0un!q?oPxoI+3;n91X~Y=#8q?w?jaW(9)o4>YHD# z`k}ym72+5+oiDB}&NULV9TM2W#iY%po1tMFc-OOZvaWZf*Aw%eXZfDLoRGYps5P14 zMS?zW-piOrzt^rkmsc#b2NvLa{kA7pD$dA&rEmW|BcuLBXU+G#ZxooJ(?zTGdXLvJ z4@cI1+_rBAy_+RLk52GN(3}vAc=#t0g}$mDVnzC{KDF!A$pINfBe-P>Tqj}I<8Pdx zcTxEOYZ>layfU;hfq-NRqwgKiFBO|uqJ(SAIJ;=*ac&E-BN@272_3X?+VH-UZEd?y zCahHY`uqEY8w14oRrcvJXRAV(16qR1|#m z51Z=>Dl2p+D4tC*vNXqQkGU+|f6tz7<8}O?xJYtL(tDvG1O|#v!;$%`R3f0f`7#}m4 z|D%na+p2DFD;A}yC~Dm_?t2$HG_lRla787MsTP%Z=RbK zVcG#DHP#F+XMgwGBc7h*BKTyA(t*)rLlfoB`FK3DBLQbi+ViLzD?PlzXh7y1{IiiM z&l$sNvMmj&uG7m}k*_q>on7c8%f99Zw|+}!BffNBzZzE~u!`?}r7i(($)L+TbJ3|> zwhgpn+-t63aesOEU6E$2;p1aJG2G>Kv47$5<}X6M0LIO!yuJXoRvWxurrA z$b+&8Z?ehJ7{6HXlVER{q+xW1k)9Ap*Xvj9h z3EH~hq$&1yZt~b8Tpwpa)E3?wxPG+u%57iTKU!}?iX}_EjEcao=GGllKZ)yWloLCb zQZlsA=JH#l-YOT~3zOPfLfO})AK=S@tAd!%H3HLxE>eUZRrv>~bs?;5NsrZ~wi)&A9VqTOZ*Pp^xxSWb+AQBEO^M>DL!le!X*`^A6)9SDE%^x>YT(&M67- z9`9*0mwh%wH2qd9Fip7DDR^bgi?bpp5;@YAk2LC(dBm5q7W0eVikcR;Y4N8GgW4hC z+|y3Y+b}G}!@U@vD~}K@Z%B{+J!YSn122k`H2qfOSXX|0%_d}Yr zj6tM8ZzM-0hnb(dJI^l2247|w{+I`Yv)i0NH^~NOM<(Z)J}hrh8$SY>@os@by(?Uj8R?2hu%TGh+*N$SkiQEoiI;9Gd5HkQGpQr=4r zs}u?2ic=ma@*6`%1w)^%(s*MGHr?4D6e!RZ0fo!(?p#(l4Zk|Z@ zw}1#pQiK$UcD*WRVHkO)M+Qg;Ezg>A<=hM$My)L9s16)x4_B?r4hcTx0FS~4ZHj2J zfeilOc^=W*XGQU+PdmKFYx)iwjneNxVX{vBUdTQ7Wx+2uIzpV|&*J@z^9& zt@;sK69J1-l~)+HDC%noA%c{?1&|3&*=BaBt)d*)a8gs!7CMKa4D6t#RhLZ;ydpfo zxzx!L_Bpd0ft4_-I30@Z_#wqGPvJ6~Y~wZv8%f2nsl&+`v~N~W*x+a#m%XN8ublKc z2_?h6YFMbr+-LEtx|b)c!kovR+LVEegPN-|70QR15o0#*+p{A%nGYl#MFy%d9oY3E%dCij8p zC_PE%&OVvwGv~+^W7u~!(*T(eLLdU5HjAytc#Psl*chFT91OszlGHLSY0{nskKa5N zyaFpPtI|2?LNn#+BduUxQjY4{ze}G_NJwm?gkem2`94pN$B(5c;Z$#OGc{FZ4UtlG zsrnRDe=jR(d6MajpIud+jJL@Pj=jsj~LHgym*RCtl}vLH>dvL4)e)= z5+LS#u|bszxu3_ItE$f9fiQNoWskOv(K1EzkOZtkC*bMb8#n^&nTRhGl~>3`IK-o< z54mEC=mAt-Z4Z&v3cd7=@2=cPTYzGnqeiMb*MpNF;xaMUqRgXqxjA6cmuuH*L38sY zL{6grpy0aNTo`>qHZ8=@-4SbXs3LP*rhl?w`NuqZ9p9xX1%-tpY;2Cs3qWon8JEX; zZ6oy+z?7=cEYeYns`T)&)};jE3tiX82kGxts* zO;uX@fwpbWhlY;OF&;fyLeBwLp9X7rfs8QZq(&lRD2%JuD&rRrzaE=;VU&Cz$D*uMUbD{F;8vK!=uI{BS#yAj;Mw^!MB#Y5U zHGMl{u_;An>6X1Yi8XLs1P8Mph|HbB+I9rSw_Uh-xKYdqy@LF=Lg^36_W<&&JnURQOeQ@buQX zswy^R<>fh4VF6WfTR!wnj8Uz(SRl#b#O2=OT=%m%-uVal=P9*2uD(L!@6c_5jyE-YIM zGeY*XaB@c*6M9bRNKQ_^B>Q@H>3~7H_VbbKGodqmsv{QX%wCv#W5k9{n>wJ4zW@1L zTwFk!L&`zS3IvY=^%X2koD03I>O*+}o(b2R0IJ5pyP_HJ`TX(~D^e3RRUFTDWUpSZ zEYEIl@=WF{#1CdfHkq?O>>_(F%u%QvN^8=rheIYA{Oxsz4ZdlOg3|JCcYHdhCo2?H z6+i_=k-!%G^RhhW6Ay#VgvK7Z`{q#9|9Iv+8``%%`W9!~%R26cZ8bA#vR1|;nyl-o zyLyB=As+%1ZE9qgJCK|G=&@rynSAQvlDVNtP9mfr>wb8n6YFRd}MZiqt63?)#zHK z!Wa*TT1Nop*0Bpuc*f$BMx$!j$#L4O?Dt(($Qb z6o}8?A&GG1JCUd95u#h510SDIicR#ZzIQWbps9Vis11>e4dJXz(Ngwt!KXyrKIyIR z^t0{5B+|Z9r#e8C=GsSk%>0Ge)YRY?l*@0+9tWN&7-GKRFD~&mh-LS4 zs(|su(Vgie*q`I}=5I@{e4QqZP5Qgjts+6oU&`SE>8cS{5?iE2g1O+#0Kt3rdA{Ds zq#uIrS74h$Kvm%s3gZ%1N$zN}VPKWZe>>B#z;T=_s68GMh}Y|D1~I<2ChQ_A zP(eV9YjXaH+#^-pR8zxK<7^!tx}MlLHg`11UcgxjL@U+bvAAr!!PT7ghyKAbBYU~7 z5)aPO&Bcpk-~cE6F>q{R*D1SEwN|aJDK>K8OAy!2rTw#tUzQRfdQ)5G&-$9`pH_0y zifeGDSQ&iHRcYhW{!`;V2IJPRTi0}KNVjV7C*zoYIC9|!IO>p4n|OnT`|sM7 zVMfB?NZGDmg_w13d0Tx&q z=abw9fOv{uexE*f9d4y$RK!)+7|WQy%L7+BcI0Jo!}54g;+IZ~h4(I0?Ef5u(aUjd z009--@1;vi?PAFU=NPQRQEP|BxMdCZ+U!$+T78xl`~as;01d+aPbysY?IY%(7HdDEuP@oR)TveZpe{dtyk zurPRmBRl+G-BMuC*ib@)YeL8S@bY>C*(GvKHXe~N_YnyCY20+_)zDmuEA{xP-PP%x zI0bjkqB*K2I8Bv@$V(RNf@Y2V6a+c~6+I)=Rr_XM5LdS~$crp(A@j1^{TlJ97V(l&8;b9!`NC@v7kIc#n{} zaBGJ)ZdbSFFJ~ZeRA2q-*}*pjG+HzBx0*F2ua`-jVZxz*<;R<^ZYHq?z5Hy8&G+#} zM*Gj6on|Tl=`dFP>V4Md9!yDO0gjJB!=$qw13jy&FWmxVkA(vx}^jy>X{#HIDORx@M>0l}f0KhAhd?GkK4G?nsFr5uoUF0L z)%8Z=IR9>It2@ER+cIshb5T))^$Na;^&8;4d4%)b`WFKatv-h`z`eV&&lXG#d7>2~ zHFWf@epuP?gF?u>0p5r{(TI39k*E9lT}vO{T;W@_WlORx5EOxp9@3=z7cu9wNno%g|Z{_2|q+P{1KckQeK<=C*}1} zIQv8KhCXoMg}2LRZeOgx#3!9HY&>@KXb;=Zr2Bn5XQo!{-1_N0gn`>AMtNk%NHQDMLY0+nT!`SmKG??HB7Gi%)d2SZ0TWH%>@%OPVF_*#H0yC<4NY`T{4-g;>+Lb3)G90 z(<@hfETm_1Q|^bgTGJKZrA6e|i$r{qo%2ck*R z`}Ag~Ee@l{V@{u*18OAYVOpPr=@7@TXzx7O=L=nrSX=e;k9|}qzHex>nx>6-ZH-&m6>q=~GFIyIkYB~aGM6xXh z|3Ke;Db~?wUlrj&y$cjgQnOk$LXr!HJoSfGVtiyhUX0rLZr!}pL2E3l)Nr$G$4&U+ z8FoK+@&(0_Ua?YA3B;r3da0A=j1-}UeA!l7_^3|I!$F#*b^rK7azCk8hs=d_dGH|G zsM9A;%)0pYWP%@?7oJk9nin@=HJ822I(o8tU|R@OIQK0++Ljy*vO=^mt zQMa(Pitv)fTpnd#9yv-LM3`OQmaPUZYs}qVU!#Ltv}_M?NHv9k{Qd~)J9uu39{l~g zyMyX#e8hm_%-32vgH1I|dO`OKUSW6wl1hCQl~2|+*%81&bL&st67mWVk@ZX>V1AgH zK`~>xj?KV15P^l6Hc)vkA`O02yqTs;HC8R^ptujQ_&gb(1k|@ z;y;F~f$Ec9)6~@`2=9O9x!eyIn=45RE1mX#3r7psK6sW(;Q%GJi^VVu{xNWmA%55S z>(tHLiMnaurPe@J)oa2P^XB)jo=VIY{WjV3-k&!I5B*cw=lrq4{_O`nR(tXEy_3!O z|FN_F{(IMy)u1ZwUtcxHdU{Rs)7_ge#>i%dhObj5_`bkS!0Xw^e5d1N+p%Lu2?{ca z0Enfh6o#t#);Em0y&z67i;Ru|I+F51>$`sB!ATvPXV48xE(iipr3B;eivi|VetorT zWtFCSY-3^;W!uK+l>}%flb=Wp3L)!C`V(7R6*j|kjUPUJGHKU~s}hYZ%b_u{XSdIe z8gY4wVFR33YJ#G%4N~TzAkZtDHDZ-%ie* z`pi!1Gduss++8i>hzo&7%54PCTq&VV*KVCVU-s!Rs;Tjp_c92{8cw@cuSo~`-e8bj z_g@X`n{vI+ZoGv5%Jx#9tuRV}auHJY=vrrpd5|&IB~%r3YKyKUkwNZIRZ)aueez)7 zy4O{2-oD+@y1dpJEc=kH4m)62q{-t2ba&@0D?_jc;gnrj{^5BI9#+d3E)Yd~sNrTn zfs7&s?4-J$$3QdWEi#L5`nIX9y9f5qwVNns4C@tO#Pb>T@MgOC1To~2Rsm0qsB@TB zP+B$uv|9cD=z8zCp7%HY-#iXkiBM*;jy=lC&faM#lu;x?MU$dql&rEVA<}6{5fWu4 zBMC{OP)89BnJInm*E<}a@9+ETk8?Ypb3RA!@p?Y5>w2uqUHM{d+D=l;Zd*+>J32DA zArKQOVsgCa&K~xRxnd=Hq^@$-vSh^?BGg>roYwA0U!>OyQ?Jh+Y74an2JN9cB=WlG|x%j2DoH()k z?$R!%b-987h{TQ8xbgF28+5%?1m*2)bM_a?8*natLKRM>bRze<=CuJU=<`)|s|7vV z^=07YwVl$jxDIqJA4r_VM!=5{QxwDS0n+U;iZ5Xr$VXKUNv5w3XbS>E(B+*A@}F?Vijya2uX_I-Ks8EV`gJ8gl@$xD2{sC#YBONRO1 zGWQ@j#Bwh;?@G{eJ|lET-GtPf8~D`#H5joQL5a0dqoaxhATu1;GFFM!Hv`cKHjcs^ zpV|Emmwvk)7cq6vG0drtHv=a41-;EFHdmqtD0S zmv|vh({?}=suAr=t}GW-vxy%Hg*9vt0BabVFs;oJOUtAqUH>5?2`1!7I{e&6SB2<3 z;rg#LtOZV%Q6V0dNA7$Ybbgj^XCkU|XoMTy*o#6?Aa_2jQAf;m zSLC6F0tvnkd#Vr;GLx+39Eptw0cWd6>AC z8Et~3SunhSE?eYVrSHyrftBOAlfzavRWNaucl3&%Lj>IyDoWRbq(c;=-pcJ1o) zZcqsK#Io%8gOM(5E4u>O(E3E$-J3T@irxj9DqmP-VK4>XO2h04m$ns6%Kt_x4BCuS zkIZoqIdVg9c4@TOB*v;4xkZMNrVV0D-W}Vm>!n_GP;UG>D6Xd`)@fID*is&8wSS)K zF-9a@rJ<{@xfj;0KB;`w#BokbB_9UnQdWP{V1&4WDI+5x(@ub^B~T$qq6aB521bvr z2^-=+_=TN#|E!;EK+_gN{J&WH=0gq9gB^s@LVIY#_B-+af5a5dan|V9Z>IMgT~3MU zTBp=-5)+i^#V6K&CzMlPvV0D9vMz#6@~7oSii}K|KLolDI2VtA(DI;FJ<|_A-NoWH z=HCh58uIkV#O!G>D(NTyuc)Wgre!bI(?gFYuZt=*DMnJpa&u3Hw7S>~v&d#&($zp) zk&?e+ZQb5&U!6NmBXqNs?=$hTt8AKM2#zi{cF{&=1gPo#?jc#v?sW3~i;BhpM)qCrp}LN#0S!~D?KR&LKCLilIlCdt zjxJ$iy6%iQ{j=>nRP~5nm&{~gGFD_n*9JvFI~%yMf2pGt9|`$#edqh^+l=`1R{c+4FALibW{?}FId@i1l&T3 z45wldR0ix-La#OG>}+Ze|AC`fjq30g7lo+ci+*>t8CyvjxqbQD zJtS%Fok!P_&HF~hs>hOpqN(m8ew+=KuF2)!Nnc>i=ZWcRwg$qxTyW|~(9kgx(oEp= z=boQ|!u%Jz!Ey$A09~>hmJLDaO_kv~SrV5%)HTLGV@<%T<3zfp7l7Vil?Axc(w^nw z#xX|rERNDIbMtJCB1|Nmee%?)6D3repznoQwoPKy0r{aeZ(DJ_q7u{ZlKQP~!-iv_ zP)YMq3@VtAD3|>cq^KgXGVZ}=C7nh`-?q|oMWMH#&?_3D2qSj8(Hp z8{nJ}bF9QWuifN+(XYk}7bDILoq+Mnc{0IgUJ_dsd9m#RtZkTb?OqooCw-N%T|wZ( z{?ZD(`lg}mtsZR>p-^kKng2Z9C9V?N>Lo$ie_wxK`NAX614}$Xz-kEa);(M9_U=?5-I_vVE{6Doe5*4bhCV+Vq}Q{TUyXPh&Ta-#I|^|^JQ`c$vG z-Z|)Kc~1$Yp;EKA=$J2H1_iH#p`<1H7~u$o{sdsNflmz~Z}aEKvP5M#Pu+Y{$7Pp# zkdWviSKPHLXLtQm#^+acmHtc8|Fke6ASIFbIbPHB?&>{ES$vJEd>pb4{Wv5_(m6J1 z5-9E0=c9TdLbQsgg8+^j93AHWtHx)omoH~;X+Ihf2`(`RGc2i5P}{tB3-eu#&cwv< z*lO2pzk`5=9_pVze!RT=>}cBm-T4DR5_tY}S!Js4LO(!FY-OvGe2j-$FT}h-AA$_s zJw3p%Ug^R%)a!`Fu%Ej7hRvJpUe@zX*D(^OPcqTDK7&$?x|3ycLuNlFylua->> zW)hrm)d3Y77VssM@F$K!Ilpsj{^wIB5ei?GYV+p7lUrys|MdRfNkksCfs|c2D-T@< zh~|Q1U#)tYE~lhILX8?u;_m8HUpe*|**cqZ+(g@ewGD*pl^i_~99oe7>5*nPn64DWp=cshJo1MQB|DQ2f!%7pc7A*UU_1*EcTb%>wh##Zf55Q(kA+(M z{!mt~aBlUT+g=p0;V1C_m4C6pS}~Ef^0yx)z#gun>f#wFU44gVfM}F@v@mpsojYm< zW+)J`z7Fqx;l!8MF24U`L)<=mu@g-vUvL_&p)tVl>Cp9GsvXSKI+=9y=4EpB!e^vW z+68;+760W*g!;Ic9=F04QPupV;6Y!h(u?aaYzJ{LnB-vrqEU4{;P;wQO1^|MkS}rp(6tp6{=b!t&%ZSr1mAT-5$Z*97SD#u+df7BVcgjY zu-oaqu3d+zDHNwit-%D)c@Kq`o)W#wF^c)3;Vha2O*^cAkl+6V$w(9Ob{?6 z;+b{wzNHaG;{*oZ3W`oIg2-rEE@*w5iGLvX5<`|kugH2bkrxnV`gV7Hy9@+R09evg zrh$1*XGMW*ZxQxj)#(;E1w^77{l^v{NOBDOsBdZFJ?=*V^X9t)FFy<%$#pb84ykTA zLQe|&6?7mXv(3ua+QAqf#iz~DQDpou;&Yw$XvHOqTwvjvHFJbxZzpK4enJV0XZcSh zU_jw&$J__tt$*1(w;^yClghD@WrxCa3(d<4TCN+{zjNmr?S@QMswNXb$0H7?6#pNt z+suNLgnher4}3jfwQzmE&G2CtO`z>EmMdN}R zOJzT>VPtwiiR>W-+D5bd+fV8E#C2S{nN+H2adcWJS5E0MSj*K}CMG7f@VnGm;O0oLblW$@3 zJ6@JNL%VnHo+rYTJ9mWptl>Fy8XqZ|q97X-a!|kOQM+yk!Kme4gft5ixVvXATog(} zPrmdJdydAMPZ^7$TpXUW8%aH7S@GoD<%{AsEsjt0eCiEE!tfDJ{CCiFwG@t}0>$!L z_SMDeRGL|j*UvD2lm0jQM;eb&WwPNRW0#nRmi={Hr!5a*5Yhg`Zt;l|Zw;w*K|$qM z@^WK;SAx+|Xe10z4HP4)G}|H;22MBVl3I?Be1zCL z?RGmNOuBaKM&QeyL@;l#+L<8JD`YwLty|ELb^WYA9d8a9Bf(k>M@V{K-NtyO;BJ7k zu6DTF_WsM>&;@|nFS%yzma9MmAWFnISMt5B?*niEV25`r=i2jyf6hVCyYt5p=W@RJ zhEgd8@F)Tk{uj#>u>{_(?x*Ft}RP zsP7bTd}O~(J?uqvGU)or>)Y%6y-0I&teMIH*15|PP&d}xm`~{!$uJ1`E|)H`SMn)| zvrRs=ACrgVl5qTkb2&V9uoJcO+gc_9EK-TW^J_VBTZ*|#)<1812Ik6dUr!`Px4qj` zJBzmpyboQ~ZYu%4p@?Cgf@J8}hzMjO)IG3nAZ$JfLWwkWSMA}PEi5elA@G5j6QL4} zO5lhEDFpVc88pa~4=t)sW2bQRxN<40vEll!{fU&C<`AVT!R108;XL7vSs>=< zP92N}oKi0ETOQuYcD15|Ds))3*YdcyD{A(vwmfy-%J%ekiqJyy+u1YyBEDNKIa{jp zDU)^FXrU0{i&^S*OzfL}OC$G6$OK-a4|ABO6zEq%5AXNbGVVQo+$F5#w#3hajrNgi zA>ON^wZJjl0o+?_BExkHvGQh=OsPiT{3S$x%_jCwz+c92C7zj(M;RG66Krf}Q$!VM zvZF?imi;v&|DCV$jvS}kax`w$YHvFe(3HA9jv+g|4Z(mcXKcm5rD&k=AMW?xJ#EzB z(teK)ibtAC6I~Fu-t6-dSq3dcE`_3kv1RkCf2zW-Y?XP9;A!ZXAB7k6>)Tf{g@TJ*CQ+!0(J|H1-+$**&nh2cF%7lxP;?cdDv}T^ z2WbaaBvhzMPLfbO5@xFa)VNFk`zg9XS{h-GQZ@4PQ7lr*`H~>pn0r5;bJ^voQy1tr zb-~NKzksdPOa244#xx!C^z}i#!G9=w&ouZnrG3|jw|n#&xD8E&gu26{C%xgzza+c) zrp`+k57u<}=+ThnJ~!vi;Y1L$Fj2@vT-K!hSiZRPv`*pc*E71@d@9K@U^T`yH?WyR z7|Z2ObKL!}X3lo);Pb;GBEqVYJLvHk zO`ZHbp5Engl)zi#DOGMzAyhj(tl^unf9)$5WG9Gz=3H=8<~T*4hj5WchIxoa$-^Iy z{k0?7oRPT($iC?)8g4tz=_-L3d4F;H(h4h!NmKYZ7n!TuSaG@Ym!gL({mkYHDHguebmmkJjyXFd-1cH;># zG&@^3Wq5}_-A6B+Wo0 z^>`s3uNIRBCq z?aA9yDUDTCkK?e?yCRuFn2gYp1yA1ZgjOC5v(+byYV8Qd7eJ)--EYdVerWA829n1R zbOPi;bw*Q>ks=8r_ikV~vi=)DYR{ItmuJL>VSiW9CCQ+fGPcJ^av?=W#Y6QXZ84_x zK^J5z#Mu zG@NtF#{zVZf_R6(ty#qV9 zNtAYzT}88s^W1jPkBS$OO8&bSv2#-AV3RY;S_+(_@^&Tq%^L6>BnQitR-iYKkq3)7Jrxqh_n{B79y{^54 z^x+p&MC8h%#AO%99vun&kH&BrbO?s0?`t=ivXYrl(ErPs{e15Fb!O`64B>Ue$31nNQV?%SA#6l%cY9bar{hM!O4I3I zx^^P87@rlt?xoi04{@?mi2}A)VMlB`yl`bh;veQ@f(y#YA;c=_4N&TrmtMWj4gV3I zsOZN;*It8qgS+5zadGj|vt-bZq0fPcM_cQ!veV;L14S$Ls_<7U7IQMJch%&Q?02_M z^`W~S4OlT=H+Q}1WE1k-+w~j~xoWpVAWQ|Qdi{nCZ~1;lPhDSJx((atBzYOAu7`e1!drMqN8=YKlpts?k7Ajm&jUp!XdSA!cXy@@u??=bE|VI!M{w zKN{!iOOyn+>--;45;pI1Z`NM12GfKUr4ilGNaALQB5Fb#c53RiOk8)8c*9~y@s09M z)#qIp2|~oU*axIjlIIkWsbn7tSJ<+pCo|Fb7y?m5y}JLFH^L5HOEEF$lVWJ^C77rx z-6+W*+21isArsE8Nd8o2VOQJaPa;GWGRLa$6PF*>^jaew-<6&{a|X&kZ9+ms0YQaT z9oa8~`iasF3(qsM%T+0elL*D%uh@5_fJOQM}3wbUYprrv2tFW@XT z3AocbhJXlrkQ)*(sojn=G=s?FlB^J>^|Xn1iU+GE_2cGUGubUqbHT}RcA*-dTl?)y zxGK%5grz-o$oR^>`9Jqe?6@?EV3itRd_tX^!EO^M66Bd-Xal`IWPaEsLP9e9is6KoJw2z4u0SVJHzAXo zmVK`6hK#FQOfQ^X9dSFMmHtF;M-^Jm$aqe>0CGr1nJ2 z&f7icJuYZP@3yEg>sRllv}5-V>hVHI{=v0@>KcL90|OV|is+RO%r8L#-;oF0>ZG$A zRRvpv7?>r2$cM{_&lW;A0 z9$VR0KIXN^ChAc$sAhZnXZ|2~zja*T@{H@V%pC5E=16uv`V^RQT?lLGR?J z6gLS%<&}gOLN!BE+8o-@%29k**eemu7GY%@^pH@arsGGQAm?DTB2tUgAP&wTqj#$_ z?=HJC!j7QGs?#C^|BrFMU;2_56W0VMO1-QfQG{b4$rO->Rm&IMZ?+l=lrGA{pE2>H zac5$LHID%TNhz?C(Je4=lo~DAk%))}d?EqqdF^c_+K+)T+IPo}MhH5ZPCQ)<4uD=n za~pauRI~}By>H8I7EGS)!m1dRhc6uW2Ga#0B-}mskd}_l+cQ)0JylH>F6@9E^zHQF zkRN_yRC+9UFnnC~b`lKVu4oAlTDI%A9`uJ6s{#1g$KsJ*-De!I?RuhD)A*&xg>&cd zawI3MTQ|Rtt<1@Y!c3xcF>7tnIA7Pwy48?5MO_}MJb~#{p>HAZhBXok9zvZ-rhG(% z^`npIdm!*7@fj5jj;v91_&#%2G^j@L!h$ptOO`(t0@!>&HMZ9dE6}F0@j3(oP=IPX zbuRzJ$y$Hn5y#UMt^kaii`|=aP6gg324t*It1|5t{WtmRo*u~hBUoDo<0mgneGk9^ zL-isUz?<goG6+POJ-+HwFBCrXfSE*68Iz+i^sNI}PUUjSYaUC3V^g_QMNLgh? zc4dQ+U*4^)z!t;kzIa#Cm<3f+j?dgBt5hy}djg5?DnhA&!FT z0+Bs`e)y`rC?k6I9P8vXNEQ(DpxM=kH%8v_Tj=#}74==?elwvsGh6We?9(cK2VxJW zPpnbcoZhw*`l%02@t=k7hg+={6mMx>-?Os^P98%)u_r^lU`(qM2nz ztJImt6mqVHw5rZO5*^OasF=Jqvq_cSM3nV6K6-h1Md9e#S5_HJl#4cjpl_w`;k+xl1js$c6@P#*n z5S|J(K6B>#3D-oAE2j6|w_?#m%C`7zi?z3F1e$??-MYK+LNw+^?K_$#xrX2ZTaYBoe{G_smI*wj0|o=-9D&jJ+aAUoiN+CNLdle#fuxgTfU) zsek`fgaK#CcDL{3?$$w_8SRWdRl-cp>X`jL#&}%?(hw~GoYT#+Hg`JF+E+!Qf|}R@ zB%4?Ykeu7{S2L#do8aq6w8NWZb>OHP%`VPFIef8KzU*c~OD-YAw|{|Rn-{&yb>ig7 zu6dmA*qtYSXg`3C4_9-Eif7$$)jb`c` zCJV8&$x2E+Yo>E18+>$!+;7r={%sgWYKDlPS;HgpN8=8!j3~vx%PY~2kZ2;I^g{V* z+E1aW=3Tm@WwZhek_bkL97?>CP=Q$o@QRwfu}QK>p}?J^Jp)%{U8oD9K(@oYzP4KZ zR-Dv6Fqy90*{z>hf`WZ7X~lP;nDU|4Ut^@2gV1qO!Xc&?K7IwcPIvxn_RxS>|L?Po zrk<>Rcr$H{uzTk>{7u^v&vRojp~c<6uu4l^#AHBNmi2c%A&ZYsI0hZ3~Q z11B>3^VD-OIV!M%t(Gs~X>ZAH1Yv3dm>Q+ za+x}I9D|A`cjBTpXFsz~4ki%!u-7JdD5gDTEAzZ*$7!2eGB$3mvM>(vYWJrooGfJV zp~RvNUMlIlWKz;3M7>!P>lb60yBt`VbT(sB`mBUCey0F8oUk0tp#L~=p34oyIyKX zzEQxvpIz4O;9n$5Y=W{^gB@dAb-tykVY0Q&wpqQcqS{X{a^7~^tr3_M52P%^mV{GD zpp|$a(I?PxF%5W%QA8R_LC}zNT@_^6GDgY8(U>U=K{3%p;ri<8>f#9J+K4)wQgLQ) zcVl-xc)Nq!dDYmc7F;rqRD~>ePw6FUQi;K(LT$fJr#B1EA14;cylo}p@M?gH{WU|lM zn7rt!7<9EuPo?dFfOrvn$kn~emS|^sY@uFYaJs2%wXBX*M`vxBz9F>{)t0vkZ3~gA z6YMWO>&E9Q7KmH7-tW&b6fuzvE>t=rg`evfNU}iLCWx;UOLBC*Mq=czr=(<&K6P?) z0-|`-$SZJ~(R!EZRw89ytUNf8lud(TQ$+QcR z+-C8AC4>0mS3^|KR`z!iyAN2we$2i z&YCAtOOlN3%yO7-pRCa%WFL90QVqtjh3Jc+4G={(`1|tnJ`F}@;RD#IHJL>in^{W^ zP)vz=b%@6izlC}7JdD6JqCxXQV+( zGhdH3J=z8slm%-yK)z_JF0 zi~b$K3(H21j_4b>k?fqjl>xoP{@7nvseo_n9E+91zd7-oeZdbRo4fMr%x(vGEztwN z=$Jf&q>39s!9I=-Sk5RUL?tN1VsSt!Hjq6#-^}@9)=d_mA+NB>yW85>8h1DP?Ng&X zEu$p2i}G|YEln#=TYyq}(a-yqWSx$QL99OA@IjS-N^wR}K_#T~L>4Cx$et%LD?2oym<2R07EqL{rSu(ZQG`NtL>>;U6}l`ZJmFG4gGua z?;ohx#pZr(N#0`-P!t`(zYiTdo42v(HE1IY&ZnMn($HZBkrFo1TZ<4 zoY*{*f6X>X-bTlzKC^KoGOUmjxcu@VH^|yAx6QuSKwJAf-78u2jcmo}o^My#29)J0 zg#5A^5B_r;e|6XD*~zyjMlQ}Ly4gKegfaYPR_++Mu0 zF#yJrNA&QazGaM8k^VOr(8GpdK}8l63C`45q%2o%db~2D;Dm?FW$`paT+_+@2Qayk zneRWyZmeQ|L{yY_vg`Q1r(#c4t?8K>%y^j^=a+G=*xGL&+XzuZ{S~DlyYpctE93znJ{hgh`-L&v0{Izt^OcRqgO}<5o z^DFA|yp?eY#(`h;=gw^qmiCwA#>pP*!zl;$?%gCuMz#Z!eI=9K$V2-qkMKx?|J=5s z#DT}^ckwPwUZytywJ<7S1*h(jgcRon}yupn)P}xIj-NoF6Mv_2}k>S1z4ilaMy9CcPiW} z9k|v*-R+rEubXEDQ$y;f&}H}GpKiL_CTpJQOG8E1{GB%7bs2P@M6>0i+}$O zny&F)A0-Zb8Gzy5x8((^xMd~SPC#7$?Yj{U{gQm5gJJ?aqJMf8XS+UH;)dU@KR`(& zOWf&Aa{H|G9l#Ye5Y+wZ_t#Qd|BW@q=}zI;C{2FD&#CD6ni4>E;P#=m#AW^ZNf|&5 zz#zpwp#^K!l#`)atCsK>fCm>2t+P}XDVWB%fM07tcV9gaT6N~~FP`2ytM?L%2V*n> zUv~=EfWDli{_?xZB3`LP6tJevXG(rHy%7Dg0zcpG57Uq%;lc%Gm4EpiDAgJH*$l)i?3Vq~T$Mr26lSp>OPZ`oO`nKhW% zAX)jw65FOYAA6G|$!e{?6j>9KOBcb?aJvgSrs$Svq<~}GZzdVs`Q&7w?H7A!L{DT6 z|DBI|?(1MUGg2C`kHQS+Pd!|*3{$*WH8+akf=lIu8|QY%~AdZWO+@SCMfbzBM0v4}LhvC9<2?j^MM-YkB~U3Kgj>I>m7tOOZr z)X1m{>^o|1N%DOgu;BoYy%cW<_BQH)PNICFj^94fwwmk3?KF7CNAy{7=OSlJfIX?F z@#x0Zl)Vlves5d(^>++8X()FPN<0~D(2oRuDI2E6QN(S3jgThC9qm#zWTmd|cAZ}d zs$Ks+<=nWcGjj=f=dkSY6kXjB?=J1WG3VY3)CfQKB|_2ZC)nitL3`=FfB!$|CpRHI z5$B!6tn2slOXi%n`u>^7Hl<wrJ#`Z`S1b{oI#G`>Y=s z76y$}iy{#?Z?H*sW@#4JY$J;}M4JdsY8ETorpehUyo_m007v>^Ny3>i{_C7&@2`*_e6&~I>m732Df!M`H>w&*3@{L~UM z2c_RBYKh26vMD($TfTg8Wa7A`7QdI5$C3rE0q@Du)XNo&z@_QUQMvVG02)$MyPCZA z;yj5*V7(}YoFKt@1v~QePqJofBMUR;8Y+Pe-OAqGO`6qh+xDz|Wxr}wJDIlibrY5> zUCP{eAe;>Q4VWN$KH202A=WU4>H{?1ZRe)(P8jsCCeLh@bu7mcFLw(S#Y>Vamm)vD}kM)FD?0BZ0r@-nBZ0HHB!X5 z&@Av3w&w5W-pl)hrk&UVNr(s$&yToyhYWGyfP5CQAj^M>&n8q%d8Gf18%3mJD98X2 zysq;<=^S@(g@+hIfrv71!v+brOInL!Qx@z3^WijtgVSb?X?2DmV)`-x%7oHGBEj3~ zvh%SM;q_9omY{Eum&lqr5s!W6QoEUmW1~gewvx0q$}?*jHVerB3mEFsDUCJN-h1~Z z9zR%%s8Vu;{PuHPxg~lf-W*dEdtmXz77RD(`g4I^HsJvDuOoso+dc zE!s9S;E-kf9yHV#2MjY=(GaN9rU`2T5ydUhIrX)hx1Qu$vv<#UN`#@*dc^_V^B!6c zaK@r&KBGj#8^NBcHKEd(JrBpl%{FX=A&;KAUawwL!^h~_GEha^NM`*6(@36Uwn(;S z1hQC-c7tr~Dj#7|EJRRmt&WE#E5cj)>z3XiB^u?drKRP{Su&*Y7^Ck(4p#gApm9Xs ziu-0eW|F^zp4`-Bi`0a0RrFdYVm3DM?u^t5dXp%Pd`x*jO!_}>kYTes>XEBk}TjpKLS?ECxXOZrYio53R{27!9P=*oIA$RhbB z)$9sw0{miO?+M+ip^uRp1KSLZmWX56omj)OxhdvxnHauJeFPr;^YCokv`KauIz*ZprkGVww96@N@OnPXcRo?{+4>7dt2pmUN^DlGdbMg3bVusX#?uqsM?Lky8c$|uLjO`xUZ!Du z!i@IfLt>#gaNGv>>604L(qqm9xsAV70>=7(DyQN%yx~Aoa}r=a*rJ_pFCLY1c6r;w z&p3a4Gv=FfK*rOx7()m!?zwu{LDjYxmzA3LoJYWuVK%DpDsxA=GNFC47O84uiiQOr z7aUF2`~q3KpiV4%wRn-NbQ@`U^y0t&8XiuHc4IZe=o2`Pzn2nSurVP zvWF#jkG9!y5XM)Kz@kQ2QxQm<}Zsi>kCRMdiiayHw0XBmwYElh2 zE%KRRWO|2=9czz0>Yctq*}Fb680Cur!M_Inycf@E|EMuj?`RM4{Udor)=m#g<+$ON ziJv;g7vFcB+V2V1B1;A#i$;c`PkwplK))|w@Pm$cZYIQ9eAp$dZ86XeNd~S&9(?NX zuIpj?e@TBP;)ozI93vRB`BMC%wo8^n)beM8WtS(2iC*%8C^>q9RIBIRbC+!ZFqGyE zTKf6<@i=W#Jlxcmp&ZF@VmI)6 zGWPFUc5e+Y7WbcD{{Y0Zs5eB85d2EvW39f!B*Ok{(IZ_Yh0UcQmp-NTpl^FEt+W?k z?p7tv9Q6G=e=Qw8oWtgn{dD48_D?jXYART!@=UF@z?0k?dcD&sQ?k~&j`1#h_cCFy z&n_QjHI;UU0}t!g9ejL3;(=7pZ%%jb_^$oFr}X88!3i;+OkEZQluo{1_xOZGfwk(^ ztJkEyZtYs$nn%Lo3O-+1Xuai+G1g_vpO=Meg?;;OsI@j~fc-z`3%=jiGeY!0mv7L6 zOPrZ_L54|YPg!A-wWf5+ zx!BmcfR|7d^f4>{)A6MnKB{&T;j1 zkkv{2G5WGHPl_#C5rs~mC5?{6!oR)S3uyHP4tCHUcOS(mtolo;9xd44;<&^SzOf)^ zBbydDidf6cnv@26=Km{tnpCeIiNB&@TD?`BTy+{Fk6GNSe=6Cv{S*MY0jH)UzH8sF z$-Y>x?Jq&Q|6q8~7K>y?M##gs^XKKWd;wxJC&|W}CjC0yQZW<#Uoo_tVUJmU_g*iS^K+=Dl+=6Ezqg)MjwlAZ>s>ZqJs7T+6yPuNLUEkG_HV zU;$c!5Gr9qx5{tXUq6r2{h-D?R+8mX#EPzJVxozN*hn&GCUdO{BGz#vK|lQgApkqgD83)1$zyUI5ln|Bo}`e}cyMPf6aj0x_LN8kq#1*zULTUw>|9*f zj%ib@4dd_eJdF3LHNl64O9p5Ar5#`HcYEO?D~+-JP@bXc=Yt8&jD6~6nrzu09d3%F ze@SKep~}A*$-Cw-&?BrKSeo9+c0cJ;C=GQD_^6tZbbJ&v2Cnfi5#b^5I`f$V><=|~ zHPg=Sl+^_yKzeo`J;|_2v>Fi_bo}_>igGI)a)_EETcwlg01LuPvb#{ zy59zMcDHKNq8e)aHRUBYP~kLH;RFRh{2+)9)vg$7(A0SE4E!yOt8vzp+<~^af9eb! zT3fS2lW}`3peMtuSf$Kv?eu>A`)`0D�JEbDf``dX{rR5Ay3xNA&H(In>X=Eq>|7 zjd3XPY@6{+)AbmpA=FY$Ze?YAf-h3Cob2ry2Ba2z6puMfI2yiP54^j1Yu2Tf&PZbv zW#q0CvDfS4u7y*kP8D4|?h)J_{R$WMZoxvlP1s5t;id+)?=c2Buj1Iex>Q7g;H0?g zLCi>(dWlYulq>y@*nlKVFFUNte5^p%EKMi)Y~Y?gLTF9QrQvJ zD~xl*jLbY8`4DSnF&>`h&m6Mx@%R9ILjUN+at3WuOasF=uS!r-Yy4)W=NV(XXF zOf-!cD!D8sPoDhv9W=ydW&uGnp$ZASGcBI-So&$AF44huiGf1?{S4VaSRU5w)wyrq zCddabb4+)eaHRogtCkluy4c3S;bTe2fvmHw#_Y8~XTvdKtd>oKO4pPX>@$J6G9cFR5o=3ww-838zpzoGMld_6uLrUF5yn1vaJmutBMkO~hbpCp z!S@Tb8~O^f3p%fno^xQd^e%qB`>?lPQcE$Y;s!Jj&kgr$@rMshci;MmNjof&{K`?b zjXlMr7ngE_FOyRa;J^Q{zV7M9Teog~thLzxd%$1roEPpgPt5ii)?WAku&U$jtHmJ+ zL;HZEW=0xR^>q?pzy=#I&kSGWzf=UXfEeemee*sg1RS}^p}rmqpXuhBfPs^aq94Qb zAz}#kCaI$UI^`VFXB+3$LoD%f+kh;gE#-*(#VRa(zWOnIV4(1IC+d26YfeD?tjo6p ze6;xKxZnpU7HZQ#{!Q}0E|vRwTEm8I2O|tQ5nE>+pNZTVvLCM(ein5hgUyYf?O(y~ z7JbU3h$Ew_9(KiFy>9c%WJ<#9YMeJ(M+K%2P|8xf8JIDFqMsvXni#>-T8iE{Tif>+h8BhK90@~IoMvP; zpBUwk^hb|0sv;+AJV@0O)SKi%_KDS*7=Zv(oY+vDm&|o_z4)5}BK4#+%prRj9eL>I zip6N&1-m7(71DVfL{T${c&f+EF`+PM2wSQBQXYG=@(6SK85xR1&K-(c7 zzdwAk?Bna0#b;>>F1Jw7(BbirVgb3_9gh!nqPyQvNNy#mO|{-F?-jJ2b| z>#7tFvzMVY{6LD`pUfzDv5>HbngCmqr?*pwLd~5xJ@MJIXDmA#Lsr&K9o=3>!^7*t zT^XijIp#E&Ul}mRlk>9eke2)6nmV?fm?|lrGEs+?f$*UE=-Oq^Qr|2c2Nr}+Z=#V$ zviDe5*Z5FG)-tN{Re9ANqpOto;qxRCZPv_}bxufyn<3pelgaY6O+;MxIA5xzYli^) zgxa^VvMa!olUoEDQG{(iy)}jfW{$N6PkKZ*swRFV zLd_UOq6pT-0zZ#T_!>JM0=%580mWdN=c4h&?MTF}nLvmUoU^lMYBd z4pSWs{caOPp!~^iY_kto+DCWiU@&c_y_+0!;LMbS;TLUOkPe~7tW{yJ0U?`}N7o31aIkw{y`|GC-T#RPAGs9k0^iC$P(*j0>n8!R+@ zy-*@BJd82;v0$!tN5y6suAWf42W&n@ZP9%Hmc72n00{94wW4aW0cX=;-4jd8p8DcP z8sVdfN4?K_4}bLJZQQ-h5`aF9neayPqH$`MJuN@v8Lw1#Vh_(HyeZ20xTVWTeC3NVPK!C|!+kT&5sd z8)ooPxFfgn9hK|3fk)o7?8+jx+A!$W)1{? zn;{F?1o`787#ZT>VR?eg^(~yljI;;$@1Fz?c0I&Q{p)8l*avXI7-FAUd)@x_d%MqO z)D*gKS%wf{SU?`^=@tro+HK=I|5ca#F+T!f12+b_U2)H{3CouRzlka5YVYXOeO6)o zz(zNMf`hq_r^PhRMUgJszGS=iy@KX>MN_7s-NqKtKrj(2aB239NT@G!J<*LDNm@7( z6*VIS^ErERTXNZ!e!3881#V7LG~|70=^ipz6DeE8oEurCEfq$(3s6KJ(QYvS?=LZm z<<&4CN=El+s-X9<{$z@hxz|!6-k+W&mVvQIl#`1Vbt1Cs-TTNCZfO2*p+(hr5T%|s z-(uMYuInXUvs>rLhMniTPl8LJo&mYL#Q=wE^ypzdVQnQ5!OV_3H7CR#bgp#DH2u(G zf{&?VlvahGY2@b1^GHLKvhwU_QQ@CDeEmGtb}MHrj?dk1ou+PN%7Q2f<1?_kp5uGk zk=sx9=6)?-fC-+F0h<+zg-_9zLcWCJ=6ZTzXj|`YWWQd@&;KWs#-~%4=cz)!vW#ROYr7<;UczHdI7jI-PqiZNjDdfYt#YgnRAt=41;Bv|t~5?g<NFYo=DAwZ~NQ;>|GXlv7_PAFidcHG| zmdFq%Y3cL~e@@g@I@tX-s|5LjF55-66zP(*N9RThb@i(N#FUt$CgW^6wQ17^Kywt0Fc3r7K zcA^k1D7cEGb&tegiVjMUO*w!|Yk7(R9BNIx!68Gb^rHL&-mZt>ki@wmgn) z`CaGGF@zg_^Ri&inrtjaHpNFv%7ZcsfkwQH^+c+xr7lTA*k1j%-VX%jkPH)x@(b4! z=g!~(p{w4#>5`A8Sb)jIZ{q(p+uV4Q=7gP^b?F@nzkdJ(<7~YCxOdUU#4|sk*1UsGf zI5Sg{cU5+8%D;~0v_!DxgZusf4EmFrv+`zTgK1NzzND;&5^&%%$uw@H?p-^E281%9 zR*ZyetPi)m8NC(7hk#!!q}y+E2z7w8B@)PAx!rQCkLC+_kkD8o8WK|5^X3>hcf@|m z-D{M5aDPt{O8Bna%4&&Em4103Q>sL zdL@+GX8HOdR+LrDkn57nbD2|K(=A+!D~k657$HPb0z-D~`gFsXp;=PO2?XtcW`!hp zwmdc6`Q@u2A;j2ZCiqm-8M^!-kzY)LQI|bhb-{b0MdHl*?zfd**}33v6_XnI%Rz=f z$iR-6itoH$oD^N3)JFw$Oo#L4#~E??q&CV9g!1=rs6SUT&*fHfZ_=X`$04-Hv?k^S&h)56$tRBS0z2BGZ_b zXIHSV5sR_EQT69?Vq%9;)L6pYc)y2-E&>m{fre?^abmI>>6x?#tArdHy1=_zGq;WH zfVh(o)?1qfe3(O@CA?SYUM4+k4kJA+kAOgrLgx(bzxY|_?%nHiXhd%E_2AwOSW&KW z{w*Euq6ELuF*2mHMxfg+iMEN+R!lJS{z&IiV4uYO;!r8M}aRKg}8j{d7RN}sgg@dixKCit#-x5 z(sd~6MoS;fkitk3I{5h3b-!ILK)co;?A!GJ`roFlz4PHv>D#Ce3%KiOD-a{F{3v#K zjP~rc&emnx7fmZon%8r3c77=b=kbCl!Do~cw7nmIpZ${v{0*sbF&;uRZ6L>WdJ~h0 z%nVU1z+&j^7Q+o8_Y27m4c7|aklAC73Tm5sdF@OH?)*zXhnu%O$j5Xw7p7$3Vx8`) zO6!8N``}sgMBIJ(rpI?lGUB-iHDON@ZXBk4YhEi&Qs`J`(Azb=jypXdK>!ZbAo<3P zI>=|AUm0N6yTY;b;n!ZeO3!GVM0uv5g+xlv&2&5cPtHwr_sok%+<-&=jYYRjSG^gC zKF%6wQH)=pfHR{T_o&5f8xb+Zpx=SXv|&TX!BET4%HhheE@iDPQ0CXOL&mU@*6IQl zRRnQLvNcJfj#2i6nm~Ldl2?1|d-EF{bML>>c&%{IqiY(Z~f3 zitG2Gye=l`4{OeDzT@ROY8?N)ge(b?{RZW;JFY&Ni92>!EYB#|Nb2Lq|2&imCnJ28 z<3iJb2lIN*BM4SXHJwD)h;{>xXqQ`2+GS<6&d>%XDPI(6>6$AlQCxmO=IXR5|Lhmg@nb0nZJWhBEY zB{SVBCSE@P1e|Dsgog`Nl@Oj`a-m!ia3QUjU|$ckNh>ci@=yEVb%k+Os>tdGmCNZt zYfC|q(qIF|%S8VW9m7#D3Vze22Z^_SI8jF*TKvXgOf@)=Yg?LiYgN`tS7{{CR&z&( zp-J0kW%@5~6fjRQN^$M*hAAn>?AaC-S5y0!0{YaQ16T@FEAskHEPhnv+k(edCV({t@bi?qkL|~N-?yglY48J+(X&6 zJI5qtvRki%jcABiNo=`%IUpDbu8dflv!{6+Z%Lw=FqW!9>eo-LPXPi%1naEtm+V?n zz$1F$9ATb}t$nn$ zwR48icV6PQL;>?6ch%{}vd0lHZ1q&6P~to&H=SyCAIgn`Pq@^b^V6u=jQ^;JlM=ew zwxkc`ULS=zBb1?IJAuTp4|Nz#^e8g`RHKQges%`3b^-lo4QR?=Kd?TSvB{Xi$F^$q z>Y)y1Nbz_t;sx&8*C%hUXHr*%kMm>qjlbCa7UiGDSDiOIdl$?1*N4D+h8Na9;N9$1 z*TIPwytsFG`fR{@gE~!UA~U|EEpp{Ui>zUdxAaxy_C;9r#3`}``v~g(MWz7`jPkQ$ zZG%EDn^4*%(qi4Zbxuxq)0;s$;AVY}6u>ZT8W&P!c}k`E=mow5xB#wjut1}$m%Gdl zn>Ot0hUQ+4mFotk7s$0NN=}A$@S%ex?!&?0x%0THfg`K4P7I59vz=eEQM(bOxO+@| z4^<@zKEk7QB-X=J)TcDpm0H zSqrf*`&2wFC@eeyUbP6Czh}TSrCYhj%5>5`BrFZwIBKs22Ks-j0*7ZlxKD`N-xPIf zUzZsAJOVv6diTCJKc(MAV>=>nhCAHz>HjfXt@uN!S2yitQy3(XI`*YS90@9yeJH6T zFC+(GFO&=o4VD8O)>XRt;dl`p9kSEKGp%jA81Gdq%GtX0pEp=f%LkM-K}e>b6H;pJ zq52o#G+8AhSG(4BbS{@N4qAumBX!h(inSzoUdhaKF+Ef&nnube-TsQR);vkzA7;^c zZ9M~mS>depiM}+yk7Zyw*#QcJAS{G^CZbrepi`&gG-3BlI8Y@@Tp1oqjwajyc-~ptNv~&%5uBML_|5+-98-s<*?sr>8U5 z1cC-rMsCIQdhqqjmv)^y*MVRHQk^<={D(EbUG}A&kR)SnY+LNFr0lK42>|r&Oa$lK zGZ!m9Wx_$L{?Bqf)xaNVv0PPL5h>3Gv};#QVrz*_(;3>_tye7#+a8LF;k_juXHNSl&gQ}+f0Z0D(gTKY@&PYO8lg+#b> z&(kzsam(RH&hc9jzn*vqDJ)kgcXip)v3YnfcbIfeh~h>YwuFeMEp3C!G`m(*72j)|KhdtXiG0^#S~|`Y?&m92|Ro()l!V+Jl(6v7-i&E zP-D}8?W>fTz*d5JK_YMLewU}r6P204OcbJ>P~XFG_i$w}@X_pAc_8^!%N2{mE?Q=X zWSa%#ab#GqSEy5Y<^;>!$elsOd&#o@UB@ArpPHxfEci{iqrxx}_XHW3`mP7=ef#t| zq6QsOmq0*g0Qq;{1K8Os13N(~N;o&@<1RfYIzSUhYqE2+>vsSIoLTxYl!eXZ%K|r= zFI#r!#QddI1mj`dQ?p}jI|APmbQ1oMKD@D-+FvY_een4o5Wi6NTS{hG)7>m3fqr3= zWm;L$S67toI`)$q8km?^IfSp0r6;#;H6c#<{pZg{+wkl%0cDi?sCMVlKtEI$wZ2gr z2BYmX@{WlM(`ZDrs3SrojtWb=n^`&(U(@jN;+oiE!$S*J4aUXPmM*XK(^I;#hq1cV z=Jh==u=%<X`Y%=3KjR18oko+@%(=kYQ6$4%26rO<;F%%1oY<;wq9Y&E{>kMP|! zWb~p`WtgXpaaR{Dt2d_{(ifD=^_w*)*FV0~ah+FTTtr2A-twA-wvE9g#N>kz%P>!u z0J5}uHE55PRTPM1P8o!tbeB}5S*Y0Mh!B4a5xIlB%PwN%Rh&D{UhF^Zlf96Davnp~ zwwWS|g$KeS*qAxqV_oo}AkwnP;%+l+jR)N&N*cSw+TQ41y+9HF@>&`RdSU9fB3Tnp zZJPc+tnb$LW9|BQ9pk+sIiAo&a{_4@0VVclo_KviUmNIlz4mpGQkLWVAHMzstmeJ@ z|Hm_Pa!`&^#*#5(3X!QYXG(=6BuZu(BOQcNhRh-nMLVJhNtr8BNfME%lvycr!~d~% zIPTByd;Po4-F5DR?7iQwVXf!1gjVNV-ZE|c11CT!vH=5JE{1+_QuO{Y{&c~uyTyyh zeQVm(r@Tjl4ozul6kYaerJH6jQ%N?;1Ea&O{YQ$CZTRr(2uZiwy1V!0JreQ!F{|%I zesL56f{wrM1AOJko?fK zeuwHJN$FCayv3Qa3yB4y4{Ln^Z@kJH5Mfd4BZtdC_=O=`BX^wZdkH_cRJ`?5cku&)mn`My5x%7!9?wJP~tfc^b%RGk0H z@6I}WA~MWVaK1ubrp3uXJgJcWzbO&HhizddpCRNIO=EV-5dT*mS5 z4hDBUNNVFVTIPhYlib(B%?8VONolH`oP1OV6?HT+YQSegb!sCf;x;7YBU#ITVeIPa zI-l4QC7DMP|IDK-wft42&st{UUMNcG1@xzcH9x$4d##yA$to(u{*?9>gMIw)$ig)j zz*4r8-NNUo+r9f(JNA5JeT3;4zmY}Nm8iqe&^`0hKGPqF788z8Le;>3Xb_%m_azrw zw0;In5up;T%8|A2V@dG>#zwF&^oEFt?s1CE^0c_EdJj6RusFrRL&X7-(DiYjA9F*# z`{4MU6aPU?8X^o>1}lqSstE+fdv_{(t>*<_f{&%NSHSMwAT}>wa6T%2wd7FAO{FFG zOi;P;3azHFgl@Ar3a`~b8<*90`RiqsQ~>#-(&G-PNB@Otf;!H+gO1MSV9iCub6t^L zRmkd)JWp;Jp$GS(a)w8=jv`fBs8N!XNS#2AsGk%1#oJyce$jFq?3q%0%zLX+*Q2^^ z{rVc9HMjCSQq!7fYn!q`;8{9x$D({}#$9Zc(c8qNfPW_=dXh9ARv{?^s;l(vdvVz^ zNrYjH&Ykd8QIflilz9K7mTs$fA~7B(z^6?k>s1<=nj8Qy0}8wNAw`bNsYgt1iMNtI z%f_eh#y6G6tHB^&B>k7`ZPdJTu?AhJM(+`6w z-adZdfC|i%QXjD?ZyP05ocoUg5-i3_{P|~qKO7A6 zz7`p8;80`*{i}>dY}(Z3HHBwQUmlmELcReAY1(8tf$Dzk5sHj6{{Y4zqYO3s$ z$S0)zC5{WT^L0#`*()XfLNt8zaI;8IAi8JImjTtFINWGs{I>)Nj^}nh z{(YtWxN%-z9xqRQH9qkhR1zFUmp*;M?}l_5HzM~n3O}oh~ zwK_}_+G^M^v(cmdEN8m7TtPr+A2xrV-Vo83DKr{2`f;=GX6J(pVLkh!BQAaz#z%|` zY79sPm1^a}X9S2*A5lsqOk9k|Ag<4K_Nhq+^4jrXfdbq-2VaShr=;V66UaC@T5XNa zR#E|9eQ>L^0cko&hz~*$hhh=kplV*cLGT{HwJ;YjI}Wcdfc;fr7OzitJN4t+?tB|K zKMc?hGky*lqjh=sEOp4~`=T9==iSt-Q%7U_x}65}Ls-cPj;Uo6mNDnY`9*IEQhI&3 z^`qqc+Bpjr+zCIp01{#~$VG$V>1N{6@C0zPnHZVHW2kW|Y?@*DvJv4kFq| z{tkR2ty}8p`ZrQ;oqfUges7(>$L3W`1`E=3Y0~9tJD(B#5$uk8apXi;MA@(-&&CMC z#aLK&bN`*?rRx^JTVP{o-oCxX2_Y}Ye?*r= zZ^cCDD!~#X3E7sK#Lkh4(m%zPTZ=z%xikivuCSNi&jjIslz9`ST&`x=P3cr`K-Z*7reAaF3lFg+oh`+MijK};xH zCmvTby>YF*u72sT4-ZRw*;4#J8DoEk%IpAhR$M;;&=N>R-P&NEuLd=sfw(xWr2%65 zRBLJ57(hA=siJR<8a1x3)p4svW|u@WM`m1k5q(S-E>lkrgf@5K`a?uo8MoNY`oI%e|*T?=w5wokog%?KWQ zimt`H(|Re<8auU>jU-<|yN>+U-`hQjopS1BO(eWGW1j_gKR6XOM2L)zcIOV<1q8%(i8 zmr$t1lB=b$cnR5%^CS=cuR)5U?C4Km*5<5xXqSQGL+SFKW_lAAf zlZU&TxV(N+iAJ<^?!D~niKEnm#yWM|E6zXb4u*zSqqJ>bb0u+atiqkD9|dJ5bMg59 zCyKl7#dgICY}2jrp_X7x+3%N6RT7S!T61UuM2+!J&YIOtnzYNgN3Qqh8X?tEjwe}7 zK(-st*l#85g6Z1>}S4;>9aG=^DY ze?(rAdVn=O=9vgqp;3tKPI@%lu22uvmW+Hra~xoZrgTsX8T&;Fx;{HN4NLJJ2-D!Fft2~fU85#piEb} z;9?dJw=P0TO4Q_i=S^OT1Hg%-2|l;v!+ymaCIT^6cz^fS6{}nMUF8zs%p7PaU|=Sp z^jR{AKr4DiX!^CT5lk$etn6NFl~lbnJTG7T)_Un?W_D>8yOba2e=7LX+9v>1cXa^ zhbUE+e+kcCnW{0FJS8)bRT>188ZS<+i{trV0>2KdZ#k2ht@Gv;`2I+WtUYAw_Hq(J zUM~*Gx;wj*n{S58qQnnT|8NxHVnRX}Bkruxvm*>=s4}1nbB)2uw6(F}xA5Bm$+T(H zW|T!t?g%e_du^XLcs+;C2U}3CtUU{UOt6ruyww}6iVAnVK({+_x61PG@ z?v47vHr#Z~?kW!U*aVQHCo)gQWy01*67A%LvZCZlh`39X?7cN{Zl2cZJ(t04yX5E`=d%qWH{d_ zU2j3lpZ<}zuNf28k!MgWr1C3Mj0| zB3r&*!-j3zs2Qh2Rm@`!FS$_3=dx2%>jL0`W10}}Y$o*<(pF`?MWI^_T&#JL0pmN? zVjv@3$oXC=ubC{Egk{T4KNTS?(}zaMc-u{ljVmKZ7XjqUWju%ZZ|`@BF&-~P<;t&2 zVcQ01@!wZ&J0u*EF9cA6e6a75CkSu_U1g{v!D`81zD zwnevDXPy@x9?m?ypaCxm5C7d_y8a5^VKb-j%-5WK?52vsK~c?y$zcByC-OIW0J)X7 z`)vQOQK^rnZ)ohfa}Aj^K=R;%uddjys)#sp_=Ud9Kev~$H4HFV-#4(uwCU4>7^Vd0 zJxUa;Um3|$!2e87M)qgItS`JG4jIX^0UJzu(+2HKJTz)-awzSuT$|z zYH=N~JBBy*w##M!pv$cR|2XTsCa!$rt_~Bbuesd3&EIM1)e25UM$RiMSr&XRD{BLh zQXJDCgIC%BIzZ!D8Ff2+BsdQ*H}byG8Hf-kmOZ*_MQ0q6&qSBmn5&OL5?FWcZr4S$d zmd4nbK)~^RAD-wA<%`~I!OG^kw#C!kJc^5omLCuNMx`$yrXVn*GNNodP_eA%WW8`N zwYBXAsk5X9BYwoImf5hI90~KW@|&#qdV9`ZN@x-KeK>jIfOPkt3kv$N?WMc7qklza zZE(*e%4Frms$%Vu`D$avr29Vn67Il-d=Wh;qzr=6#QW5Yfv*~ zd>~8Xe*mz-h7|H3q}Up@xnWbY$KnTLvJE8(F&uU$XEst#yK`L`o5ZHUyf0sAk0N}2 zVqm?;=C^4yu)bRI`c0#2)k@m=*s~5y|X$YiAMjXU}-$$#Iz(=&UX9!Fl&|J@atZ@xW9t!6QI zC$>8oIt$Fy-`jpOL_LpBY+%8Q5v+4vG77qu#lWIuDk!74&t}uUq#Qgvwk#Z)lXO;v&&+)21IThhgr`_B`IwzDJLxmsd`2lx)Z4sq&PXIDj zCPp^J_RdKIE*={o-}+ z=`KWkb>WrI`zrs&{J3KhH&k_kpjNlyjdTCvrHhn|&Tv>LLEg_fk|BeV_pMxq%v%r@ z8lEQp+Y&RVd;}`B9yVgcIs}>ZUkcs?$EAiwe&{<&dTHhwrQW%q4cu3n4dU8~L7(=c zg07Qnt!rY^ScpIux=%rqFIk=5B8Ot$Qk9YUrHy+o^h$_t;_Oys5_^QaFp|ap8ftDB z!=)#-sx5k-N^**A=%gx9XmnXY(`)PQ*tHObsNQF6+A zr&7+?LrSW8w(9$tF(t9S99WAV@}?!`EjQOOWz8~1M9f}X^jG)US$b{$R`a&8vqP5> z;tP@_z(A{@Z@Lso6!`FmQ)t-Y-MNotrZt#0bV;zdRK-S`Rr3sw_7vYfz7a zYrLHMwEtAg73sbx*J!w_Bm4Qz991R{Yz9z~%YkWHVkD6wSZHw2g#IH&Y%&WRR*?`c zGN2t|K-zRfL0bydlHk1E%0SReJ@eA&w+qf^I7657FgmGc>5-!)@+}u<8 z?xG}@D7v`c_QBYfT@06C<5tQG`$q@3dAQ6z^Su}odccST)y}V?=pxW@Dl~d9g!noT zWYrM2-n8y0U?to7D-LDS-D^;$2fTx(O;)^qM$F@!~5La## zqWk_cyzY{EFHS?HL=$k-xRY}_{HN678xu~>2n`Jnw=wF5x(mTt*d_B59qF}L?>Con zlmR)ZeqY`>Iu3{;tH~JLADuS8G5!}L2uJgd!s6+L=BKopHA_HVq;N5*twNdVNod#s zE1zxDC*T8|#g?4tz|bckJ3g}{d>KLi`2DzvaM==c5;wPg+69IWTWut$$o^G6Qyres zp(~=3E~QV6lG6K$hqwJzKqh)@Yfv#3aEHY_y367(r5}o!jn_w>wcx&b6HLhQvd)o| z*0HT@gXkvS+Sf-aIq6$hDri2n{@0))oRr+Pt^rkH0d~8)xSLhdsF+qOL@j2nIl06Pr4&2 z5pR}^8mG~<&PJn;M4;{FAr(Dwn)i3vyZl=+!GY+?BEic>zuyd#B$gIw@&Roenw@@=)fI1o50p6`IsoeBo zIdsL7yB-=alyCuY4Uz+Q(BFhG;Hva|RJ7~+Jb}?rL;E2LT2Yg8VnFirDvr*(daBwT zXo+xzfV$=10?Tg{)cOOzO zW$i&1U=wb4!(*Gzc7a(F+K3q9Dz|P`g^Nj++r#35uMKDoa;vwp%yih#3}= zl|SBf5B?1HY}mbfi;f-NuiYHzR=cSRVh}YAjl6<==HWLP`^q3B>tTZjk2^U$nLa`y zH<7?7MOWlGkD$cAMWr!%(xelL**u*OE$gTg#|*YQZAE#RI~)v+N9P97J)sZsZfOar z{hHk>7XZ6p*T6%o;Okh1+r>}A&08%**Rl}h_0)#A8+m3$hp`NfCZuA%{yV+j5zVE*!o5h#^n_YZq@ zVG{=d^qazkt_Gd1-#l6=v;|skKDp71*Rn9*B(a|yUtD#Ii-vUYAdnI;t)V$F+s#MTdfdDq{p1n<11VCc^!x zt#udlr#bzKHC>73Wg|sKEv}_ZiqTSTfQoO%KHR^QQNr{A=()a`9=YJfP!4;B*K735 zzVptkpSh2wY=Puvzv0&WXNqwyAVY{&>XkKNT9az*A1!_V|8@#tiGo;NH*O_k^^Ckt(uhDZ_Uo$dB0+u{FX@AJ8? z`+cn4s0zrmB39v4IjcME@#8sX+dN;$--XQR3TXcRm?t1^WtrvQC!Kt--@dtCk^!$D zIp@ohPy7aNf93P=OL1|{q0f78l~9jLp2v1uI$R;Bxh_}TaE`N>tLVXi0YZ0^*e0*@ z^3F8h^rqf$qtYS@x*ELns=+#uXK~3(_7qry5}f4~bXt)-o04c_{>O$v*AL?w<)r%Q z=Gf%7eY+=uylD&vTg}?tze{&Lg|!USP>5o!Nw38d~e?a-GwzMsQi?9y7PG?dzM8~obmk90LW`yx!ufcd_(MR`a$Bgyj zFAVi8H|~3{Lb5>&(bC{v)vwy`49~Ka8BFn%}d%I#mmsX#i1>LJs|)htZ}68Au-G zHu5@(I7XCCd*YM^*x2kmV4mBa6Gy<;i4(Jr4Udu4z@FhcDQsA$gEsm(n~4dkCb4l6 zhFLmM0l8ou&@c_D^d^g%ytEJmoHr|Ha`ozx zpX%38Sp!}C9d0uH(0}y9*w!KyNoHTrBtIIEP)|;M9K~12oOc=hQLaB!FLGHn8rRV> zReS7<{Cxc=9_7BCi~fQ;jW8?ssLKvVVY9)-sJghVT@iHdz?=5YHx-C?9r2IPAodgqu+?#v z9d*a5OYlkEI&~b7CxAt)RyUUJ3004^>R~c_Z+-u;Pvgw1z*j$|>C32{6kRSJx?^e> zJKh1};>p|_!89|ud3oY&9eVui*)MAo?jZP+IoT5Vv-t4>^=NG(sYZ`JsNi7LF)^_l zrJI0`Ac0duXUNF>I&|NFg1WtXPoxE&CCbu1t?8F9r>BphQAkKm{u>5I;-{320*dBK zR=GDtLYrAT72hoB8mn1~+g%Eaii*Z;6sE4Kj(jwu~{||IKyH8LnX{mtwYMo(tSG<`-hbG!m=UR6L5?bDGIX20yd>Za`{>f- zCTbnD{_pO%s^8JOLW5}fB|$@1H(As1$BLg6ixE$>uvk|pE&UVa=FkYixxfO7?b>^=%@O|o}M z{NAo=!NQaz6mT-xr0J=J2N-Xb@#!y>liZy2R)Y`I&SJSZ>cUJhqAJz5do`A6V+;!! zoBV12O;CYey&Carq@EZyF&@O6t?Vx{-$+LoPF??zWWUl_512NsW&53bPDB85nIW5l znW3r;*tf-bC3yqSs6Z$s6mH^w%E`etuN(A0CMEC+kGpJCSCQywnTM1Ttn*(hZR?4z zL?oS%b}><_L!jozF-1_=Y>Hp(=H}sR+-xuIl8hTQey2V-=tYY!7QtW_^F^!)>96$lC7i*6 zVc?=1&TuG$ln~-t8oXA@*gwW7D#cH5arnKTq`WA*>#sLNDXCS;%Td)HeLwPevXw{L zRLU*-xR;Yg9_3cb30U%&zr3jgT;TS|$S-n|^9X-6iuq=um}W{%WPvLisH~)DGg2+3T+*-+PO9=C~ z51B~3t*nm50|av3sBMpr0G5@lGp&SNSj-~x=C@z48yF@g^*c?~i$bqu2+7G+4)-N5 z69(hH-F7PgeWLf)vHGohV?Y#jNPU-Dg`#9XQeNIKmvr*K#b7{lZ@mcEO&MkS`LhO5 zraOZ|i&_S95iNf|^?Xc>V6lq4kjg|;Ttg3+YFyjA&OIr=8r80@L^I8Nt_wGZLMSG4 zfeON^_=9Ha(XX|q0E2fz@0@tjJUy7jnmM@77hD{Z1hL$}#{Oj1d)4Q6$F==*~t zw<$6{;=S33jO^@=<+4E@PN7DB-l9_4ve`?>%{0v`&aI=fG=X(kiEwUb>}j~ z&7$^LoL`Bb`|bQ7N-xC61ter;H3_W!*I!r29KZQtL`4^&{O)5hcWx)S8u@H@C+FqA zZ-oAlbBI)lRc|J%rEOJm6GfYU$#b2%*$vtGbE_23d$fAbu7(`WgCZ)uY;?Byt)%8X zHw;4wc;(sE_41iv>x=?WJ+-;&q)_1*dejHvO4htDmH!6e60Kkp9Rob%>(Nl`_D z{uXZUpEg`a$7^#gVpb731=Y`eN@qAU@hXuy0QWlS!HLjN@8nqL(b%W1Q>^Tm4+?w9 zaMPg-7ol8;D{2!(;mwgbnKYlu+x#f&qC?|Zf`Bj8t85#&xn%a9dDsyO+I)L*QpH41$SmL+@TO9fjga41;&s5hA2q zbbJ#d(J4;8eVeLAuBi|v8#BxUN>s&%<_;ELzBjYd`M2Icp`jsM?lAIc48oIYP>uWS2JNo15s;ED1pd7VB~ zxqPMhy2ZB-YEe(EL#^KXZXb-qZ!eL?a-+&fM@fjm=~o9Ul%!~}Q$*l`DSD_V1>_v# zkIc0N8)w4kET5vil|$3gydw;OLXmFMyW44ebFhr7W%`3VXu+PsvDbQoD_w) zrDf1vx*ujoOGyO5rYar%xc1799*(=M+&VVPAIDKr6`wda`M3CknvKh7Lacq?sqSYI zEMhN?)9zgVDXj{_2E?5UhamSome%Z*a_CW7BK_o4m(i7yF7Xg%y?}6vkETOy3t0MA z!ix7F(zsJav9%OEqoqDv4r2@_WooJ^ty3_twW~MXtDD)~*f>->wfKug?#&R7IatFu zgs?`u6nQUecpj35;LkA5G?+5=Ch7wTs*_KZE)T7VDuyekX9s;hXPRK^joFVLNkV1+ z8I4CCN}CM0t@Mn2`t*sxLYjNa@61Fir%XOUT#NyyZJV^``9J`-Z;#?uY7@l@;_`>H z-@!i1pxS`N z*?3Aw&Ft181OWIgj{d^JMvXb~q4NHw`dkbr7~)Z|6#@xM*|LyB6SWoGxiJ&Q?R7?Yp14%45y%!Il{V`2}1X!T>ee^ z)erycXP`sZ*IN0Wb#!%|o>uL-oH|pAaU)~Sh08R8o)VsA=RDI7k&P+LHx12^yE2v% zHb8VaZPB2-L!ONwM6jOs~VC^Iv+M09D`~O$3|G zB0@t_G2%2jIncQNuWh_j)~>u1JWl0W8{_c>J}e9JA8{l{_j$1;N)z4w^^17S%>0=K<{1WbL#*!6X z^!0s)=jZL=B605+@YrPk&T#5SIr`DYD89?N`{9l^@bkem=jKU~mW>)W_LR<JD8b z;yz=xgo01!D#ARDLLG*Mc36a5M-joTlHCVwFP;VZt;G*rm~mS zs~|?KXpk}e%ABb78b4;vk{TK_&-_g#CVN-j>SbtncTU_tY8}MiD7rVr&OLi>6kSqH zB!xi4SX4<`XR#+DuV&V>Px-~IopJ6je)0?prpVMmgGX}}Nml_Kj@>rp{1cHzbF%{r z7N;gfm5LwWuQsx?x|(r~Nps)9)n53Pw^z(eSlul> z3Z7OHt)mJTP1tpvnmaF0ke@2fkVdeD5x z){lYz8U7y3lJ10``TwxM_WiJB7`}HhHwVyn;AI_UW3!In)|k?eZmBCm2%VE%;5Ec> zuBfD@a<{_WQTqKgGx503dSu4BJPw zgaV?%*+AwInR%-%I}8UnLSZbJj|*t3$yw78i?GPWe3`)-;K5CS{}2Pj$RZ=Ogy``G=<#FQg?{)1T2iU_(hh}Q##r|U`E3=qR0tnfCO916A)8I0yv(2UbTStlnY%^bbI$^>6D(Ok)r$k21MF>x8~g;A5`ZQLa7 zq7+h+ypX(#LJI6`>0_5lAUTp21w};dEoaZo_@4_e2?Hhd_tE0Q1 zPTb?43PX!?{u;_MO6@_!eaGU>69y!pH&g%K>e$KqjxVF^+AnJ-$k*fLbrDYOP}H&n z`9grKT(^yr8%g{KX_W#jNH~NTYQ=YsYH7X6W^Qod{+@W0PYlI?a40fTosPNAvN}$Z zkoDu;SxLx3)D!#|Z2-Tki5M$Q-xg;9&cWy(cQJw5-+=L-XmcOcyeq`HU3XaZGtmuU z`2&|uS(y&c9B)Un8^rzh(*T%qrgVVOn#-5gZk}<25s9YkTG^3o5)N#pBqg8s;QHl; z5ShNPQishq<-B-NjW;7mL=**->=ug`J*WvKd8uySk^^^{$UdJeG}xw^76UgCpc3xV z2lKfOnQV`SmruYuqbY!wa=$2}d+Q}-+ZoNCr^`FZG1EVwJ+RM;a zp$HMJiUMNYy6V=m84V2lv;Khju)Pq8FbLZV@BT0^I?kye#reci5RJT+L_n@eBCIYtkE~c_Mi!Y!09UHxcjA}zg8OoqH*a3NJ8MYv zB9JCo(a*f5GxlLt_g(l3R3#i*J?#y>1dPZ)_6SJh^z69+DzZn|t+Gcx!Q)F*0m)O= zsYyvW(VDutn(62_B~~Z=Y~}qa!F`zffJ@|mM7YK6h*{x%(702nR~^4IFs7VGtWo|uOX*nM2tr*~t)?4=dRwp&ks<3J%TS4io@;xFdR z$3cv>vkVwmk$Ub@dXDYf*iRaqrU4E7oQI&U8M|>fN^iZ>TYwmua7JCaF#eqwXJ~u! zcNt@x3~*^o$}Ouz;9u4pxlRXU!pTS1D4#coc4QXMy?fnq7So!6?&;!@NN~TQjAM}k z&laFwX~qi51h&|~&4WfP`TR%@HOa&Ku;FyBc>M&vO6Ue&k@_YMk?k$OIT?_}FDVrR zvQC!y{)`5dp4+(+GMqsm0OGcTaLj*C&derO9Djc`jIJYw4#w3d$qRl(@?H6r>=^)V zBVUOlo_JadmEJOV?vLbEBZ;| zWHbc2k_5R~53USsb@~AMY8hwqOLrnU|4B(#1)jUh&;>b)VJTMKNF1#cpwmu>t1qSa zV#X_vBsRz7$(QC_{wQJze1DtJMzMme16K_jHcTWvXivgEQ;sZJY@(~9jD?C;Xf$XL zbu@JXvlQ^5glW@`--9msZAy=a(8vsaEy#l*0u8maMrViv5C({H z|!UM6z6pQ-M0PB!uN1#}XJ z+b@k(aU?JjEU$*2tVZ-CgdTWv=ZS@iD-MMex7s-5Sc)NJ^_I4TbCNhxgnv-Nm7SspcL=8Ts zYXUV%v|i{WPFe_Jli++j&7bpKiksN_Ae!a>1Q?lZ2A2cI8X}4k}a5)uG3t>VH zY}L!iJoH z;8g~%mUB3tnQ2pSt3u8wgnC67T)^;=Gx`J4TFa2@YAyHl1!TX*0cGtIKM6$$47Mm> zcl$QJOy~StS#I&WFnTe(`~t+W6j`$D`DzNX5aye}6i|?921AE^o_cDOEw1jKZ~Z$m zPO{d}9GUi^aCIRZotttXBOpX0OB*eURfVmKOFQwxwY2reeb2VtKA&J_I>Xbp$Dzn) z(b^KPgR%Kv#eQ65L>KfAt@&^EvaKMpUHUuU02Lm1eN%+uQbJ_-_HfrJdYTaw>-N#5NHsf=^^TmPvD@po+tWioh)R3tQ1$v=)2A`hfWv}fR-(Kt@@NQmW5nokbB`%q zL(Q6v%^+P~0OJKsE2~BOzWT7X;>;Jt z*t|FQI>gN8>=Ks)jPqAxD;1>a^i)S(2)>8 z>2Y)m!s0;ro|O4r+jEy75Y$=?@r zKEBb?te5YpRxZTv^-5ez)L0JW1^dx@q=m(`RaI(yUAC^`IF|2;YkRNwT8}DQbk@gP z_t{4F3sr+S?(9x%@Jd`IltN;(UGT5+Cc4gl;&3;Sh9^dwUwm&1!f0)M@Z7m^Glupi zGD~v*4{h$F?*C2;%mMoHIVC+~(Z;+6`s%?TDxB}P)^^oAh}cL6GM|Wwp|dfuNbwI+ z*Z5<3^ybZVd5%`9aesuOPx_JHYXTE$xG$}eryviO@~!0ET#q&Osp*x&&38`YoGQWa zY^|K}w>)j?F`Jd)f%?BfH776Wwd+SG#e5wDU0cQPSm^DPj|IM}IY~;5k+2CE8UDuT zNL%{VIM8kc*d}cE>r*zQ03w)e+j&P0%%}r6nt0zW_2C{+%LvfW>QJUpsqQxbN~)PW z0+H^!C%l-c)z>spt|d}`F0 zK}U}c4$a8SoJw?yNx69^1Vogbm8{{-Yslv6fX<*);L;F9L!K8n==4u$^ZXfuyY$@vphrb~`sQJk`cXHF za}!v-HK89)VhUmyjb58t(UGiqanz&OS^N;~=_c?b9WL2^29^JX8G7K}{eDFPryk?e zs99Di&ftcH@sdg5N9>s+iR7 zT#aHqTN?LMf}vdU)xg6IyLFQ@otp~VTv&$gPumhwyzWFyMPF;! zWbA5sTKQ_Qw|!2Rk?ny;GjfLL&x)*ld|PXMmJUX|XYhAe||N5%m9Y!Ed3Ds zMFuE``R^i;N0la)V^Kz*;f1LqpQ`G82F)Ho?5o6lHo>mSY_iQwdt17Yd^eCB!HGGG z#~w*@F2MHAtjvCkE?+Cfza?-T!t)EApSS!KPfi-zZ!-`E>i?7*O7VtbPClo;5w!Gi zn?d$RgXr_zF>nQ8E~K|ko;qd~RSi;V z8K2QhtjVo(ox1EzmK0#IntqbDUDTlM{BHtHhcliu@q=IPC`dx_qgWSx`1i<%RJ;>? z8YQaxC247EyS@L6FjJz)+&22g3!=fE)Tng4@>B7)2zuFTf0 z4}~f%5|r61>$bFY_+xt6g2&;iFr3N}S~qV_j!IdP?E%jSzh|5iF+g|hd>KRdZ44AK zeozLN**ntna~?80DC^jXzR+$$3vs^4nJunpX1!nk;snBd z!e;+0Xl|Z-P(93jk~6x(K{nrB5ni^p$bb1ltY2U(;{I3^_{wK;bmd=Ik3gYac(jro zgoLgQSWgYWFAj`n&C0*7e{K3N#wg$ib)MwXclqcJ#JV8xZ7}h?>$v48_41=U;!c6g zTqrO5@%44siOwWI-j(of?x|T1&kVOaw~Q%2zmfoLvQw<~o}b-uv){CpJ@)IP!&q}! ze?Q_oCdme*!Kg=oJfl2n6d81x%WavBKYOtt&U&ri?A1* z_A=l+0Q@SB!V2X^TuL~RK zjy3AiWy;vSWoIpN6(|BEFH^Bre4}Q?ZiETXmvZ4ebd}krjQY{%=}%$?9+x%&!QbwF zl_e4gorN;(ijW4TH#|4>ONxO#%#OIot@`o8=6=X`32(W1j*Oaadp9Oe*EAl#vygI7 zZd}E8UekkmscV@54o*6C;XYwj5#2_l8W{&WhTCq9JG|4bS9{ioELKwotiUj z@qDU6Hw;Ay3-Qw_}00R&dzNxzE;_GBR6-Uc_%)C8Z7!h zkeqx`FEYo8VpH&mN22kCf|U`CmPIYS_9syA@iuKHrv9x$Wz`S)`0i2W=8i{@n0SImdaoIBjMULnTjD@anv1i^AVSnlyK{y-^EV1#qWRa zpdG7*gh>d^usfFyXSA)~6M}gj?}d9LZr+nu!>`y4ph}=$?k#Fa#en2gNHWSqt`CCw zi!N6L$6r!{UzqNG(UI?iZ)4z1(oyths0}!Q@g;7sVxW2c&BUk#1={iFU1lb%L3F zYMLADY&i4bh(pCfq{Yr#t*)kmbR{evOhCq7B1+obFZUln&c%T_~OluNS?CqZ#rc({_I3)=*{JP6#%1OPycWl4I|A-Sn|7c89kYwcZMclGFP~D z5@7ryoyt%GCcUFgcuiPNVPSq%Lj7r2*M~bIzz2aAyDr)MzCS3O zkiUR3uR-V(BFZ9Cmw^VXK9TP76b|EAJ}Pick@ca|xu21qo-XRAIQu_4+<$qSB*!%2 z5UF6pxC=xWiovlqs)>BzKvV?k*V}OFJIIcFrCqrcwy;9qI(XXT)%ANa&ZFV((+(LX zWhIDyxt~el8ya=sNfFWlK;K=nKF7Fya^XC$A^MhIk|U%Km#c+o9%PDib65?Y{%B>C z20|#2WcPAR3k0H-nAw}U4?!x}v?9OL$PF^Zs{7N58lniqXCdhj)m?uf{1TOseEQvh z%?G0;^YZ_U#oM3#dSTD?r5*B-i85O1=2waz+8;BL=*(93VN^y_E?qQLs5@dLj?a{I zf0sZq5HRsRg_##AS;VtE^3g!oHUshe%ttGEJJM8qrtv^A9eo<0SgfL41EWD zNy4AZ>AmDwtoQ@-l+E#~_~KOEvG*i=wnnvT9d4%9KD|)#3YD8;Y3~sS zx}tmqQ0nv8FOSV&vmuX)Z{3aVrvigrwWZEjIOtUvmB$z*U4H_`DYS`h&j0ADzRnH zsa(&k_!c?Qog7~3@tne;|5Zu4t3Z=+=v1&@#a9*;(JBe|1u_=?#LQf;rl6bNU7q|F ztxkEiV`fE@aM+;pjap6M^=@T)_C!#?17jylWpZ(Fvra>{CXG1rn8Q<~(Fi|-#V-h= ztn|{s!ChKt*bJ1Cn4VRjvcdNd$#E?))$U!C$IAaBpo1f%56Ue2cEU_L-e2>$-6xPl z>e6N8kgWwwX}XN)vbGMQpBT$%Des2cDJ(S>0!tF!0>Qy9`qa$J@_%3;noDeBrn@neRs+?8K@2jSF z=mB{b4@!GI{@EqVq%`GBO0vs}&_US6x_yeyT#1xkd~$hqGi*oE&P#Bd0y(Bm>(-3m z^cP7h;)OqDbKH>p;(FKBUehTfe4&t)DeOGewFKoVb7HX)^Ct#M=~Zekx;2z z`|8dsRx~Dl0uzwU@tG#|v6{$F{Ey}`KPBdA-7a}yziOin7Jr0f_zvzS0pmLlkFIgC z>}I4dR{jW9TixnjNh*C`;0kDCQRDJ{llcyh+W=YaP8pFu;l+VXdvU+WK5EuK4k6H zwW^xRY+?u$V#|zj$^i6`Mh_l5_!3n&!(SBNZs(EnO-M z#;4eWRJ7bumB%CpUb@szN*q4lp(22$*Ov-rx3B&6;v5rtQIZCdvy1b3g}YtY>f8#P z`4?2QiDx-$EG!cZ=Vf~S4$allN|2C9o{ayYlwlntWQpU_Iv$}WLWj#+%bs}N#oR}5 z=K%#%fMuvTN`Zwm31TOXS0eL!T-$3Lq$X754=HTvEXDuQGDvgJ-Jw7J9fgLL){V?ctz+DO zbW>X5)&x*Ii68 zif<>n=!#m0&OmO;jHpt2JLTLFz@HHAw{PFR(Ts1%K7+NEvSV03YNU#<@8u6-KAdZl zQd0M6;PHjy+~(?54*I8m|52AW^*(cShQC9{=-dUnhOJz(OF#Ni%drc3FNu!1JIrFt zRB!F&Rm}SZ*Q|MG`{U(KKT67)zkh!+=Hx%)w|w~V>Enx(Q>nF|1O)5}*pt5}e^^Z& zN4KdBtFssqv3{;xS$1;jY%y)n2)hDWEwGqpT2rUK&&Tmd{54>3l_0F6*(=KBzpbcQwQ|k1rhLCQGugoX#hN6;Cpksm z3=7^@#S4=F#V4xob}>}RtezT!5S9-d$R`jP5f!y|+qU}RP@e{%!Wpf-piiCkRnQVh z6fw;%nXc#Df1F7v3a?m9SBW^4Hh>Tq~)O)agprmGMH+Uz;Sl(`@s z$G46KTEey!#fsAuvpny$e&Yr@UnVVD6lD`L6GMYv(4NMm22Gwh(+3>X`T-5?Gi-R* zjwU*8-Mw3ISex-+I}?|W08s36>=plk3C?NXURQzYsKH%y%s^99Q}jH8)zr+q%ysV- zkE&%_OKSv}HYDn}X?7ho>7E!Ial|i3*Rx)B!*z5(?FeH&e6~(t>iV^7uU8#7pk~^U zprF-8?_R$~Fq1xtN+<4HULzE~Gh>}xTsB~5m`rCq(95^>piryG{kv5D1+zHsF(}b+ zE0q(k*X99bZ|IYERr!oBBQys2Qxv%$9OXs$x}?i-F5D9F~a~o>b&{*ZN)7zh4F6O&6Hnnd(&w^O`Z7u36p;9 z0{@q_7XJG8_OW*l46dm&uuIb>Juw!{C&7!Oe509pmFjgj8EG+-V~UIfdwFPcbHBk> zFQ^|^tyxngFC}2+wU>#5JJ+tV$D%s_gjy*+*%o=wYQtfK=DlNnsz+H-vmVCAul5aU zwXOfbQNBIT@Btimyi(Vo<1W=z{ZtN)s=2D)CAZ__L&6?yS>{~uv4%_@oKJdKMp-Z$ z3Xa+2^ZluYJ&bspJ2}c&2B9H7K0Xmhpx1bGqTmtfFFH5OT1_~$wj{l<{lu9oYT6z> zddQq)^l?uMm*#vnt{I*1;DHu&r{Wb6AuR^FmOseK(q-rf`T#{FzFK;D=a#oxZre+L zZnI}x`*o}|wIH2qon^TnpV_OGqk+rp*%{|szhWyLcCM)vgSsTq(IDVM+Z{tCssK3M z|LM`1(dEo*9iOi2=bY8L{_}Y~I5h{*6e-$c;{$uzuIG1v>;jO=W0Cl>>2ot##uFJ5 z(p=qJ@+I4a9X)!$aZTMVd|3PXxcghT`sXj1;M&t>{5VL(f z7j5`d65m^Nn{=wY0N;9AU9Qn9rlqG}ZAX}Zk)@&M7VlO&hD!Rl%^tX=h#`u=^NRcUGlMmJt85M?LW!7-7lFtz8ul}`iT9G%FUSPmnHvXXBDy*B(@ z^%ZCMp8Xtt(4yXq)(CLlWMs@!=MI3xFg){*j-zR8;lOq)i}tRXKkY*7@zJ9TkB!dp z-rV=!++1yyOYgHUe=SZo8n|(grgDSSe-pJVP+_m%cL)Q`To>MZcq+g4=Xd%lD%Y+r zYuD#Szszi#SoXQ2 z%BAjCv{F|N{=2Pb?H6leoks09+H%zGarM5l%99;h463?)+PxM(9>yF^nLE(Qwb^0! z1)YahSLxjAasAEZv!nf5?6Y^v9oo5eSH-F#ln|tR`?5@9*NoY;GtPY5wvFsUNeL2r zUhS(#Zvsr$>D8;3cEh>**pzEcD=9bSUwbU#-FkoZFteYzbK|1rNUz`?zVvXPM&B){}`M@^sWpn@4gefMqI@-J|bb?>H|oeqy25TeZwZEtsOi&4KtMBY%? z9dTc6!{IRt5{*2m(ZkP@h)ZzmrcIk#*lr34XaZWg9;cwdAwZ4we6HJmLR~u#-d(R% zD_u@Tu#e7Nx@Z;UH_zRF`G)^`?bD_GBOGt2qe)YQw`TFm*g=qDxqZ~nH;vmhSg$j+ zw5g$I!|4TcdzXf63w0>?wu<|g74G}b-#SyrT3KE5KNA`0ki9K6Ia##ojT<*^)zi@Q z{a!|KAD`V98M9qSHrnu}EDbafmcxhXsY}J0(m1B7&r|H-qKrsyL`#C;hv!iVo99CDv?310GoDLuLI>^2`eUqr>S0;OX2g9++Yz^x; zk~{z(8|mnp)=V4OZpU-Rlu>=Axk-!y7?EOz-(bU4erl%0en~Hmxz}%aE?zHvjk&HQ zML5H z)u)sO(vaUfe^JtPIcZQh?0)*PrdHp-)>g)w=F~gq*0Xt{4KDa*c=wq(P5mH}t~f3~ z-76%Jk1hGqe^b3GU23nP<#Ks9mi+wtOrJa&@73J1p25n7>n@`^^Wj=|iIsE~Y_*Ma zd6qa*$e1 zq@8KaJssEe+_XtAFk@kcedfkv^_N(sM{G`6x2M6zy%ZSD{HOLDKFHX3c#p-$tc*(I z8$}HCUH9`0@b%fB)w1HR-_m92?Vfu+C26%evSfm;PI~35oqK%#vG7rkpx#@0b!pnN zO0j;!-9>+YsM%rLjd_q0w1^GVy>sKPwcIiEL(+l|HK>{N^z@XYcMPa$5X4Ug#y!dM zaadTN5(#8vKZ;EM@}h4m0$zO2OWQa1 zblnle1Wqn5D?Rq% zcmDj9tSoJODT+v(EClFna#^OKp)n2}heiim&x#QPr*^uJWXLJ$?J` zi=%yO=zKYM?%c@`jtma+qJDEHK?G5?Ut&jDO_SRGw6j{vZu?EYuIO{luA_b>&vEU$ zIhcFao?8$*a?F@9Ee4n-CW6REhHxTpq(Gi}#HwA?R`b-o-Rn~7Y`Nj6Qa!7=D8`s# zHPv%)qPvhi_fCU6v<&^aX|SFQ_J9JimcMIvSEo&Chjin2g7M*Mw|NjWz-Kba0*Ad4 zN5Q*~zj0JwU;lFF+AW7KSK+eLhHXC<2?O7A;7B#x5S@*Tre&Xc^Y*PL77TvQm?8Kd zP(2*WNSF!q?d7#QtMpMVow`7cO{IFGAD~FGjrt2^faLe@NQ=u~u z;Iu8?Rw$uxD}ynZz3Dw+!h~55BCE03>o;qt<%a{9p6k?2kZ#}lwRIe?4AD~u9*(>@ zJ!RjJi!Z*6CJ{>RR(kV*4-|0@sNieXs&(*3k>gFWfitr1A^GEKn{(JJYxi6%sKJrE=xC~SQRA^W0Sx^{vQA3@wmkx_g|*niaUJRr6jNWD;1TwbKkTL z?8QcDl{G41^i3zD3!l$7wfo*LdvTM9`K@jpv+!-Grjy%W*SgN6{Qf()r62yju`)wo zy^I_Tf9_`oyJEveKR37USo#tlsOUh=NBlz*xp#j6#+VX?bKrHB}8so~52tL@uvYd;f}$Lu#(Q zqLa73VePB^I_@%4Zm#~VwuS!MUp4A}9Z{S!ug4$z>q}Ywf@luXDaXgCs;Z(;y^4G) z-(*dGpfBoCDGNoUYTasAQiMnON(uh7v9THGyUAjJ>A`wz6iFSmo<{#JpFgnc&pkJ< zM`IxgDKxCfpY)}G-#jClKf&~(-F-d!U|se@)<+86aW{4EsuIOStAQl}d1{sA%YSq! zoox_HH7R##_+F-Fe@METvh4os;A?Soc3a?hZB*krH9|r{j;;S{ zw5>^f1FgRwLnsfIoz^1#{{2M%#b}x=^*nti@*nRt@5hBTs?9&=+&(FkC8Uo=hoE-> zCiSj4bb+?H(T*S5)WX8~_U7d**>OXXF;vpkRhv0;W{ZIXdn+yeGm@DT1h;-lPlmF&fYW!4UfZ zX$D>sXEXLBehUKBm?juh#tv*0JFEgmlJGYal#D?!4{w3C5Gs?LWpE4wy^rkZu>94z zu}Qo}$y43=KY0d+RA_U>*FiTr~Z01)GX(> zG7BNz&83@WIgs;gq9P_D(l|Is4nGahy8nHq$F&KNTn5OMvU6C_n`)@4wW?7S*Q;x| zZzy}`NGU0?pf}Cdz(E*M%DF@;WTryk23$}{2Tb0=mrSL5mXvp&i8wDmzZU?Me;PI~ zadjO)`N6l;!cK}4JPb03eY6iI@KdWvV5;Z~>y63&4BH=!JPlqMTT21zB~PcgN#QX7 zRJLGF5r)umkl^UfKY#ul73B*63F5^i#uF)M-9j7B9pJEp9U>76&SX^1O|IXT1e}kk z*qnu5iKb9ty?|D*C=+A{#?J+R1u6%{V9G!u8mbjFhe92K|M{QXCuFJAivq&{6hE4q z{6PTL!xWb5YinyP!lE)8gi!XNMk!@$M5{OcTPa#{d_q9%B^1mu5P>uy$`p&Wy}kWG z%#Hij5tA7iNYEU>Nt?d|30jZpAIV_dL6uG*4Nfa6iqY(zcSQBA{~#HB|K|joYXlZG z<3^`Lg#|qfyCZ_h&v52@qSgr=BCZEoyf;dDMtgOK%UM@F)2Nq!Tbtp|0_zB%MKQ^>{(%A1%x;haMBH0ViSZJqAwc<}QSShc|M$Je z+zF_J0(h9JV>3|Ip)N314Il}Nuc!l4H~>7e2SV^WaQ4B#@MtJEpj1K09WBN|_(Mic z-UNyXfcs$PRE84+A5(XCHwx!g5fT)um)^P>!859s!R+)^Qy z$b_=X2~*mO8c>fFh`ELf%l!pNi9#55m!xrX2|ACQ2X!ElSRJpn##p|0D7KWhST*II^<=wpclQrK}3a z;(tJCyr!1cKi&SD*Kgh{vmn!DLw}G=0KrBF=!e{E_-14zzLjUiJyGmsz?tB@2|%HS z$3TsvG89FHn1htrf$2E{@p0NCHl2BRZXueQ4Vpp(-q^d9r6r>Qsx#_`Sc&ei--B*9 zj}Ov?ptIVj2GXQ~^!An#ECB+#XrV@8k{zR^<>f{&ZqvrMk?ZNy2;Jd4olx-+u##7TR zI3>kFAoD@M>U}H|JQn|JZPAIthXYDTIG6y^zu~&(3j8zpJeb(n!cI8h2el$?l-IOO4k!2W=>ItvR6C}o49;_I9>5D&sIG0MJuibP>lRrgZ=XERo!^4-ltgg>xd zfVMT;2ln7XvjzL?KVU^4)J)J;Y9&5*jE5w-u?h!t6o>R37NPh~J2NRcGvjQ2Q4tZ; z&UxSA2bdr5R|2^|jaog+kQxGWksCmS681PY9HoC8^eOkU$)h1?3wul~3vD(GJyjFX zfBYCN#CWOtuw6<=BQVbOmw(A3o9@O`y(AoA&C5qdlU_NkvvOB@24Gi_;nM4Ax>!N0F!Zw@S zBZolusilDS7@!8MtFN%5Vq!2uA)W}L`65Air{w`cIY8heQAI0}krB-&lw1*vb$N~e z`G8L2j|9*ET1CLEeA+uaM8)^OLW0HqKn0&dMMZCKf7{5mTl(gcrZ#zOI=i$~4R~2l z4oxpD3D4BG@Zx{ZRll4(WlnYBqrjsaofuJ`0xP6^7#RsPQ`B(yYq0J`r6tjH*aZXR8>(YCzC)S z%fR>NT4UpE$=QMHohMH$A88K-mI>ZjTkqOBnl0HcQ2+Y&o}3&y4lbr7e`-%K{QZS4 zC%H#4=O1R5mt$vE2A=l!JA=80UXL3Sx&SP^AF}s@_?1=UQ|QWSYS5{u@LyQ3$=Vg- zMMW)OVHLGB1=~(-%T4$hvTHK1W?>&6bG@)4mF0Yo{Pl(q6O;ewWQ2k97xv&_Nc5uC zv(d<3zvMJDUD9)vaOk4ZIXEDHG>?e#MW}9xwY-4pJs%&!KYvyrTq>e~`}W$p${o`X ztO$x~WV6}cOIPX}p0RNnkx5CmSXcEbs-H2jw9?wzzDM?W)02_?VBjx7+uyGx3Rj+5 zei?k^4x#>7Z9nuvj34QZg^ZRzd#v6UxM6!6ZfjAw(1RD>NM&Sb8t0>Wj)<9mTHIPs zNU!@s%cZHmv`Qq){kWlD5sL%U$vs%OX{6V?-{HrAb4J;>>-s7_F?&B7W}S>p82#H@ zuBh_gFLElsG45zp{aF3?s}#XIU(We-hLf6lM==Z-qo$%v=n@C*Y-w%;zQ(=j=92GGJfqNb}Ai_55{sqxYnPzgJ;?#>JPMtI|FXsXa$qVh%bO z?ovq4S)vW;x-yTbZ7g1#F|2I9PfLe;GN8Y-87IT2-Sq1z7Bwrbd#Fg`*z>*75)B8x z9nJUzHwJ-H0v8tvUZF*BD&Q!+cW>ynwLTR<#KpUSknwtaeDc!DTN$>w#_n#i5{pr0 zrY5Ra4x)nNYUk&CU%vE1+5j#W*Ll%*l7^O+D-jf44IX%Hger=+0&}&k9z7ii$jp=s zyU&7!^BoCK{nTz&t6hwwt}Z?s+lvSK`h?0V#~=9H(6+ZzWPJ<%{Biw~MQHs_hLNzY zPEe=v5B2-^rebz{8mQ;0Zxv!QLSZQ=t9aPEl-{>5l33`bJ+9Pra-v7A6T?Eu$^AW= z;ht7cZg(~LP|BY_1-=z*$hei`+1iqaMIQ#0mq$)`G@C%SM)o~p%BU`shm z^5un9_4Fy1qS7ejsbMN9p`jL)@|)3^Av_O{u1S4TRGy?iJmtmKi9;U>aN!4IV|)GK z+7xa&3X3E%Gc*u9eq4}I)&1vdSsC^6GU!7#LSz~n1_rJ;I6xynd0g4xZrFwbdFC1g zPWW&Bw-Z%HV_?u1$ltq%`}FC_^pp_>Jv}v05pR@`zX6SbgNJ;q=5Nr*2;G)D%~gyM zgx`;}Z)!DCvdjSGZGgai6IR zcM&ymH4xMg3)+TzePQddy|1m#b0Yecw}c#42Tk4Xr?&hO{~v}?-}5k>1^3bl)p3-> z)KSmLPwg*mbisPz9b+taCtikD$YbzZF^*kXn-Upv&?BkrpRm1Ws*72bblO0%GsV|O zUPsI=TTuT#hg)M_AU(H|P#lfW)%N`ds&$NacbpYJg+;OWB$LEh2&82cQ#gjtRPS%% zVe+gJ^C)GT{eCnO67t7&uPlP`9UHuyk*&?=6(VC}E77x8EFYfyOxtcgpsm0JwFC^yST*Z66$R>l^Ngq!c-~H^JFbM7L7PEJbGrB9{R$VRf ztzhAERw#OS5IR3UMp>DnyB9~J&{nt6g9k0r-h5PFo@B51RqAyo4W1dEN8gi0N@s0f z_)2pxN;4Q4*%E}^SFEUDGZqce&#$NdmN%DP{W0*um*7_yRZvhnBhBY?JtnlA9QUP_ znY6-abOKCXAqyDO5|o=u(t5)xx@S@LzxL&FLP~!g8Ncmb^Dh&l;Ws>Efgw) ziavx`@71wI)F^T9-7X>EG5E;2^}?V@#tN-!VCb8enxlad4fx$DYR_JY;1-aaGc%)` z6zgcB0@vXHzj24%gstzMklEdOc^Q_B#?hrIn-4KRzon-K2&}KIDIdzU$7%k@5&;aK zYdO;2PX>ShDnu+N51<_A0-*H#R3wZ|#e;`r0MEPT!t^v6)Q3>CSHjS)rtWS6@DTjd zz>HjG#X?dUWJwowKupur6z~4Yl!`o1sd>v#*gz!7pF^P*lvNdaraV5NZ9v4)sz4lD zOcH)SZyB_S8O`@tq>6Rkpp2z1+!zSRc>_C2+m9E83_;5HieE{97(fDI|6}5#fE78E z$T=suSw zCds}+|I;IjkOusv)R=@%1v*54da^S`Xb3>D6iM+L{WGKjL!}Brnybhqx+O8NvV(*X zR1!ycqAKlXpMi-7gCJY{%HkmW5=5EZKV;bqgoX=VY6g_N@TrKdLWV4gZU=Aq;r=n6 zOeQ*Dz&mpfK~GWA0c``cWhXDU$0VWh(|VFk24U&3FG2?^WzKK zjZki1U#7FO#mC_BA*dLO-A7YU$#1dq`xw>S%-(LMnCt>)rcT`$E>G=U!d$Imyh`z} zZ()4=@ljtY-y@Cw%#x_uS}Ft2+cZ+gKd*d9BA67)t$y~pd_u9UjeaqsRP#_2f#6OE zEaGKFUGc#Ne(y7$+u1mU^(i57aY>Aq1^M~62MhLF=r4*WQ&NzN(IUQQ1QVVdw7R;H zzX$O9{yg!|N==CX9w9)%?5M(NU_c*&BRXN;M)f1MVcdmnVFiV)mimiQygJ!A${T9g z*`-H~gxt##+-noGZ=}v|6D%;dU%wgK4l$l?4oi=)xeW_slp-!R|4vNcNgdr2Ht-d> zGAKJ|b7R?|{~vC>ovtS>Bb~xyvBN>E5=Y>e>ruPQCNpA+c+LN-TBwn0O~Qm|;Jjgu z^h}$_x4x3XG1GaYkhEJVI=dMPAD$9OWEMU7$#6O`PqFwaLw&L3|{aKD%?lDUPKzd zl%~rpAEK3!_u)Yp9hao*Z0iY+SXMjCtgo`E6b3u~u4|chAo|fvO6^N>je0S5|6>8q zOJwln*!_o|Tx6nAxq6bAXvK7>cPH#Qx}kH5H-fg6apvN6d z>|6KU>>*uVp|~czUyg@Bgrc8KmX8{(K7YA&mTfrR`m_F9YP9eCFnPZCOSpgS653Ip zIR?IQ*rI3jpL?T)qW}7^`Fh?EQ7s6$zKxcjLe1uJ%3~f+x!&1-r4ww+zuOOCxr-hge|w zG;-&lOzuHTN#6IPBXk8-I^l%BwPKX1zM^Lz#03P#k6L)1*uz6~a$0(7=Oip`ohRmd zdD@~=<9@WgJmJoKz5ZCGO6$#L-N(rC$!_Vcv~wB+4Q&^=L?gD=O45;9t)zlmrb(Z$0o$#`| zHiLQo{F9h=)a7Zt*;v)UXMt!~*%xPv!s6$9V~n%C&AjN7iy!R_WF3k(oaDvC)Ir&p z4p*LGB=Spyxj}LP*t;tSYl(q7I|Ma*f6PCLs28}bqamn;PNtLH-c#+JM;*G*px8q? z?>#b}e`!RsKgTl>qyOAL{ru|Q*%k`qc7);mz_wY{Q`P6%g@aRVQmQg?a)E>5Wo4P} ztNA+Km#5QtJ8$0#o*lJ46!xi7&dtD&?s>2mC7tK%kTMa4Z>kyhB=XDUxXeKQ^IH7R zEaX?NHdmSUnl^5IniZbgN$L=_-x%j4Gt3|O%g{CN#LM4-k1ZAEzsgx{JS(=hN@nKg zMd4qi9-++CcI-EKzqObo8UOU6*LL`=8{-iO z^%K(OQLc;30k;V|Q?Og!N;t{)hY1qXsz8q>d^|+mUax~zsc|ZHfbG=On)-lx@4Ukt zNALVIVi_|cWDO4u&s*Nu>bz)gdHbb8?;6=%yFmJ~%jd=2o)nYS;Y###R}l9L$d8t@ z*Af0d(1RC;wi#424^kkSeJaUKk{ z?0|0D$WNMWweAP~We9r@SwfB6)m`>EMZM!a@;~+5GKXmdHl3;X!JW#_^5>zBp8a!L z*)H-7!=c=D_7W~ii`LS&M$Cjx@bZwv#`672%*Z8p_L!JZy8!EvcH6bIS~c8Yye8y) zSad?uFJ&)Z6;SRy{n;GC9E*S-iAm;?_N%`z#R)|=v^)rJR{*5NnK10|u>NCuiIb7o z>xg$I%TK68MTLdkRKwnq(2?OB?3|yG5#f~j8Rq0vD5#RB)qKMI`ovAyTN2&S@Y?6k zb00+B)uvSHabLZH79^A2eYttwr)jG|g@0dA5~5IuuU`*_kciw4ly3nF8_*1xE3bM0 z`wuwlD~%kDspjY zxp(8s1jvaYBLamcgF>_o*ekX>h~A%n6MgK|fS&}msQ|{2ynZqBAt}jZx+xeGcpCB7 z!Bz&tLN+##lU|Ii0>aDwI(a^XA7g&XfXsEH4T0XqAj4AKDt8b#6WBwIm8b}cFWo+% zik=>}P;lz_qtZXhwa0y-;0t3-aW|1MxoIY@sjUst=_t)I`)L_~D~}z7X~4tvy?_uG z2L}~u6wt8MXxH9eNaM4!$-sGxLUwnCdpD%YYgAzVQy2`ld$XrA9OWpdo$? zG)Uh&;IjaUn`+zi?O&}S65sBF4Z zyBx$+C#NoiM=KD94`#fE7W28l5K=x(D3e#LpB-*d`r5mg2#4Q)1-B3-ScO)=f;K?f zAA}L8sbbHO5_eUx?kIp%89dFp=AhbvjsWw>`$ktw)Gnybe~HUlZs z!3jfi%Fe(I{4WYX`5KhhfQqAJ@i2%6A)S3Sik^Z=Nc(a(XM7xQ!mpmDrKQbbXKsP` zyvu561wt4+7WNsD~}$AcWHAvU(($yrwsfQAA=kIJ>} z+-ky|_FP?d+%~yYW+N8RtvY`G9HZJP3$nE6<#ZEWFUVEgHeIKdvwEf@cx-2fi$jfU zZ1~K45HX6^-7k)Dk0vj7Li0JuI~`Uus60Kt=#+NI+ts$V8d#4V;oFY=?iTn0Iehf> z7cMsl=r~)0aYv^!V@WNF08!-B_at)LaK@v~f=W-Un)vkK`jU$EWt)^in^oagylPIw z3pTgWv>T)ccXy5-SM)ih6{+5Jpo+hOjga!p6 zmsb8H=iW~@vWNEs*e2MSWRcuY}Ij)(Sp$`7@u)zFAouL2siGanJ}#Yk)Anw>lh?~4xez16?v zaX1*)!Zj;CK8|Lc)WPa6G={Xg}&74K0YQ?si)6l>|k7e!ab;uV{icF7TN#Vr4eG=%jSVmPdwz6WIr|;FD zRwW_92R15QVfP{|R#j^`v$;8s3mCgqJzPBdY;U<;#g!0=fO^H`=Ua65`FMr1;~od* zj%PRD-7Ye1dkCY3FHGH?oTTawYjN4QY}!&UX{6pp8)?*^&1;@*xDD;q83J;@K3d=6 zchMKUm`aF_02{lF`0<=>yV*!j`4u?ZXrwMZaM;;Jw=Yg}TLmUgPUx&hoRuvIem6dS zcDlXYRz5XtEMivBKV(=B{aaOLJiW%d`=24r}RYwj3~y zTWvSQ#qC2>Nzbv(_UXUfkRK^uE73u=}<_ITmKL% zvdo6nenI&-W`AU)`3Om00Fyp#!FOx}NH4wX-z08>yyk;-*57xUj~l{C-C`5}hJE=$ zyLC9!UGCoV9};TjnZ`Qy-_G_f}XsJQXxLKbXL19>P zwNlS5L&2+>lBdQVCP$1P&C(}8BWqaDl&LbAmL;F*gn}yjdF(R2e#eF9{%?ig$ZM}J z!dtr>Y;{lht@R%k;(07`Gzd*MIz?qnctR^Qdo((f7DqU)n|8C5+^t~1c)fF%__TeE zxMh5lyR~gnD*9wbu+~MfaA3OLOLh8KZK!Fm3&%0>%BkObaWh#Tbq`ia-wW}BGpB23 z3(=g$+sA$u%-gf#wVQ4Nhts*KH;wzOdc!8@un-jknA8rpBuOz9?L6 zTrh0Kqek9$xtk6BVHS`6eeHJ_=VeS;*qc&57;5KUOHnK{wzn@1n>;iK`x+NhGIvz< z#597ws!^9SK44Tgx$JrSh`0HmAKt7`y^_3=CdFW#_+?$opC<#sYKB!|gE<9^t1j@N zHIEKCtjb(-tuML4LLS0eO?;@Z{WF5%QT&kgmU!^b0pX)gj+ng_$+B^E?fwuog9`s) zc?;8X4qQ9~NrZF%;nX_{p0tN*8SIF=V=vplS6vMoFjpv5{e&2N?jBaTnX?LiJP)m(Sf;kS~KlH^n;iJqFqe-J4^p1 z;ZeiuHFY7bA0o4e=p?^Kt-p(ffHa7>3I>r^cE_Bf2my_-KbRMrKxG%!8=L)nqQ(P` zev?~`iVzD1@!LiaD*P|qi~;hTEpSYaSFd#MgFFCaWGO|Ips+ys9AU~C1Wz(@apC9Z zj}QtR93IYspBlwL14|3YC)TrP=gGo9_drW}1q7;K+sN$zvCZ^~==W0}b-*#nU?xSY zViC;U3~QtF`h}u?@l+`@d5qB*-vtz`zk53oo1Io1dmi(xWVd~!9V?L&;xivVd)CPg zo}>O8RLk~p;fM9-ScK|&dYO1Jgn`C7g&>BHZ z%{_K=!k{0Zcv0jQZwUvp)?FkoL@W-nv-Ty(q@zMUOWzj2_r))cW_g8D#nDHJWirbF zL`4bD1O!HUVmJ-!Xh6i`iTQLDgc~ZO!^30IL?D(-(_n|t+b;Iq?d?0Nyk>f+`ROo# z=$^Ux*Fi5(Z-RDDRsjqDiO>K9YkbdyqHTKV+hwH+$oH7Rlj~S{QYpcm5CNWaX1Tki z@d*jDpdW$wus_FjU@bs>>HusqqYJ+r8b(U%fNDld=!8#@y6&Qivb0nJ;V&Di z*f0%oZBrQe^T)^G2)vt=c6Jo#h$C-qu*tYw__!Tz(^{SW`O1EFaR@daKHyc6QM70% zIu6*ov9aegq8fRMPg7DSUk&6EGIOmHe@MFVUS@AV`yGf9X%XnFFX50lI?|D4*90vX zJ`2m*_HLWk`O0UFD^O*UT)T!o>&~q^6ChyaJ-zfIDkQ`JR5uhu1p|KC35|rl_Sk@H z3|ICQZ9|xl7z9yI+_%WOy0GAFJvP;zB{rBk2z>qM7*d z<1N=UENOr_zC5{RJ7Eoju5K-Dy!S0n$pLvV9GoKuy~&SYQ*ub{?wW^8Ug0gQVfI^f`10Tz1z0GI?v0~ z>#I)H9s?i$fQyZdO0h8#FDO(DIr22DwN(E1pCYZbZ7e(a27vLrHu@ zON)(8^#N8@RsZTlk6O_BsD*JW9x>F<3Q&t~&${(aaJRxqT;x%5S#&3Q5UK|hTx)dq ziGOo*h4FY5mAz}3wEDXfr-jAxsa|=td)Z>~zhmG^;D*VAc0f;`%jo6p`*8ABR(1*d zo<*&QakE=Ogn@i);<50ru2QEqN=KUYcN~n5@w^$VveHj`hpV17i8W()^FQD%zjG~= zKSn-VyqXvawVZv&bNf3e#A3(e+=I@R=62)0Y)j6|bSnA)CGlyx7lcmKq`XGICvQD) z@?g!`NecPJ>ZGRaL_gtof;VDWrD?83@Eq&)Zcs@oS5T(6^WD8ESI1IzZB2DuOiA|z zJcBbwbY-sKym?(Q-G*)?j$`;0{jQZyY)sk_4qc}L!|6kXOpD5imPxErO!mdB-c6s3 zOcaJNORjyis7?}po7Stf?ITw<<2x4_7+TRMj8%(> z5@p6X9+=yb;=IdAkKss$ERY?n|EiQ4oP0-kkizwpV{7^_v|{K_V#fp-7v=oB%`>{h$H+N|pfd0PR=)z`oH z@M$ax&AL9hKKCaeS#(oRWbVl|y319*z!?nxKEIwv^Wsh^w`G2ts0~-?%5bZHuB;&y zXY{IOSIuvvDAdzBS3wm0;D6A-L@3^+Y zLUCUR_CzY7vUx8_YWcfH#hmsVmqxU%q+RV~(XR0ENCQVRc2u1p@XRu|Eq?zUjt~=C zdF*4wKuCc^P+T~X4n{46jhP+U#iqnLF}uzy3b<~FV`pomv*ThrG%s}VPcOfW1Q86k zeoiwAA}-iyg_@Iy##(Bo-qfT?R{0Gz6SF>)$e@E~tg8MBih>rXG#?pC6>qb;Vx95L3VXi?Wg50V#T!bcLHD&Lai~yoK{$48mB{s z7jMJOJ8O!bcfGmeQx=Y*fqW0E102qpnm!f1d`B-Xk>5Mc)e?+*9WYKk&BIWsnm1pS zBtUNvF<#vxoo+O>VGE-31Fk}RaF;=BIdv4thAO|xm0+TYVFsq?nW4XnL4bmFGyN&6hWye zIupD=#er8^Iu7~t+~2=rbceT?h`6o^hZoi_fDB$20}5XbsV57oM^>KaXVxytaACh2D^~UFP5C*uA!X|?rik@e%9{3d=+9{(D*sEy7b%Msm z0%kRyeWa{*OKps#j>bQ({VG}D4uJxO*X&>Xzx_g{BIM30Vurns36etQA9W%?LBr-9ba2oqp5Jv^De~z zn3xFK6Nhl^G3J(2wd8w9S{HQwgr3GX3ff2AP9gENRB<@BV4py#nZEz>)XFI37K(a5 z^>OlRNeAq8yc~n*?+-GUY$!7B))JT};v*1WUp4-$A44E{b@sj)V;53g3p_8pl~GrL zFD^-OZ#!j+z~NQ)SEBWOiFOh7%oW~008AA6xm=$-b;pmooKDI~pJ4rkU1Bn3JZ}_L z$Bm8iF88s6Q<{T3_v5MGtGAD~d%K8|Ub;2)-dY{4RyY$TL`>Z<3jOPlQshwV|3y0l zaD~U-n&TA?J3T_Y%zv&)!*rr_>QkH0^e@gKY#cl~z`CO-OBAGk+665u&1Y zbl8PM-k36pChh$QFW!_-_&v~%nJz+KGFtV^>j#D)J5%l_sSM)yT zL_yE1?A*L}Pn|Uu%fDsjC;uV_jt;d!m6FMDKTa!`ebXwo&emxut(H5SCcIfrYG)`L z(0Xv!DP1g&wu@o?T?S_^TJS^79E|C>9^9$&hU>uM<(Yrb!7^UFV^8yhJ#Oinwp-K> z*Sna0;Ku?4{>En6EV9<3H@COJY|%Klp@q(Vs|?Ms;enNk5S%!lOD(6oczSG`S$PZ^ z08eR{r5@Z~-YP8o%x;%f-0KvD{=CJrq4g)tA77D`h|wPh@t^q3h4(yh;%-D1rZ~|| z)&Hgy>?VKI9{-}IFRBLj`qJeFWx~n4&6c}f-tprv_wM{`#_>Lw?R>$$sbRA^ksifc zB6v^DN(zT0_mMhpk+n*>>%z`&qh;&sUVmiW{N6Ek#(4)>ia$7rcjTQ` zo)8#h%fqwGI>0JSHA|i&+hzJrE7W7UIuKDyF%QRItA&OLA z%Mgpy^-zVo&gf-~lo-O^!&hx=RyIewRk8j2uC1-d^9syMXU}C6Mu&!AK!WPy&xM3Q z^gnnM!xJRK2#7FvC0F)OH!X5~Q>|^z?Pu)pe{>JAw;BtD+KqS*H@pEVYDL zugPb)cAEhN7LJu9*w~@5as7Ezw`ywQw)Qm}I$j5WT3B2nFeZ8Y%gPT9G8|(&sT~4t zGe@v=!Z=sz3BOC9L0No=S_=*hZTmoi2Duw31^WvdO}DF%9!Ubm6}qtIVURpP)6fJd z7MkX$TxKwGpduLq8j5zW%YeZWCFuV@BIdD*-mqAZJ0x-fui$W4aN_w5^d}%h7f-ne z-PyndMJkNj1^FZ(wf}gs6Dd`=6CnQ!9mJ^6M#?EG^AHge%c#}&nJh_9 zZw%{I<^TfR9=Mkw|6I_-%uG12(70vhrvpzMk-oD{Dhta9VJj(L zB8azt{P+R&0RO8oEv3ncQ2HFBA6AhZLV z)5RU+oH_{Z!wdYqZHA)a;>I^r{ExIUH{`(!5C|$>5ac-4gW(9(*yizpj3y}Lh9Ckn zQ5>8su_!T6!MAU)x9putTtm&hh>Tnd7y$>{3n_2Jhr~pb;Q~}D9gxEZu?TM{$P({@ zHWrePA3u50#9m%h)BrKz1RS~B>m{mKj2QzULPh!DL7$W72t8dW(5u5%t=*~z3~75} z3Srw2|11lQXpD*vdS)FATAGC%pn<>KtJBi~;9UVItO#wteGYyqYhZztoc#GGBZ!a5 zfi%#6avC6zRx2HpMi%jLyw8qOPo(9ohZ1KVT zJ95Mn9HXGu!4tm_78`0$7VAME{w2%1q%TO^+>|tr&ks%~4MU&&H0rtmex{bE!+#Tt zC;WE1bk@c^NFF{MK=^+Y`nXj+sgL|<$IbWRc(djmxTYd5&n&K`cs$>T=?R#O-p^IC zwB(9Y#CZPP8_-Wt>xs1<&r6E>u+{{_iPm>&DZ$E_SCI%n^oUWliNl98e1t~$x~a*S z{ZxaeyXW^^MD4-TxD^2;ulq{2(~B367AN5g5t!Kh9y#`DJZm$*9Fk>2>mR_ML4X?t zqvpbs{o<(YmZ$0Q-=2ag=xzu0*8Uzq7~)kA^G>(SbcX zKk5j>Gp47TzPA%N95(bso~W-n*IkB6>1o#Rk>;{l=nCX`MzSzn z(jBZn%P6IHc~E4ztJ=+>m>`yKI7@VHS~tsoE2IaK6*HCgdQy5paM*@EX-q4vqRzTV3dpa2UWiPtrxp%@_>6xT_>RrtQ$ylq++a5N2-(=2GU3dht9H1}gxG&ro4uh2Q8iwi>8wm+*@3N_B zwF-STio;tI&)YhheE*WmZ+>^z{)C=NVN~jO&(krk79qQ@#+#woo_>Yj>uLj6LkD^) z-Md|yqsqyVzQ2oodovneu&*}M(d)}RQxO8{M1E4aC0d5=Rq*l59(%K5`>S>`=KGo6=8_aBZlY*(FV@hkqZB(7+e9hzZCV7K2&xE*6TSNH5h8SA|i zb!L2Yc6JTAVa?mHh1UV5=j+Yvs9W*PW~9EY>dQLMRVeK;Q@NVYx-Mid4%`5To#Ii~ z@MzN*-BV{I`mnMwI;t@k5XXblFz`-(#dp1@mCvr0Ua{^}-?m^$#^qjTOzLXN$E^O@ ztlg`6L3uj^x@txp^1lig?%Uw|_u<8Mi{=_2HlFv-PVp3^Rez;N*zfQN@meI750+&| zzoPO|y>~*>!BTRr)F^ z(Su!_b8aW#JRBsT<8BGF$BRgveFGmC7|xrEe9zVP_)?#s!foQGoXjg85)x-S)})-= zY8XZ8QB*3>(ipf}?D&;%^_>2K_liZ&dzfHrnzOkwRIiFm65wyWVB9R3JY!LDS^w^4 zhqYQuf6jI#)kn)88V&@)jPI6#p9F-6-i1DF>ufjuXw`65cTsTi)PT*v`wk7jKAM%C zAP#i49=YrEtGHuc5w!~n2(0*?eiZC#29b`QU0rWxrU9%)y1aZByHU6~CEXdcv|TWU zl!epa*M*J8`97uh**2H>;n+3kW!U2JQhW}?O-#n7RxCxV*W{z(fbI&$9k*Zljgjwe z=*5Kt6s(NvK||%Cw&C@rpFi;^xHXzUWIyR$g5BG@wtAX+;f!{$ArX3cZfC>^@kVl= zL^UC`GoI^Pt$)Go{sA-QQ_Bc_23=Kx#OSc zpNwykPs>XwHiJI$3IaGQ2zjRh^o?)n(;3Jm{ejWxl@Hw7^eg-=I(WXKjSGSa2Waik z5EZWZbg0j0{pq@odrjdiiUi{G`Q2CZFUuYCaC&9nM$K-NXAHl#HZG0WL5x<`jTJLj zo^C56%Be^pldVoexf0tq^;YVTzNuDaHD%aC<&I zb*Y=w`y2kw@39b!XgnU?9yGbiV@haEeQAdE`bkdlWvQ7a`uShK=Z$uC9H7L*r*;E$ zhw^zEa|@}TAqJ}<@#;CK6Q_2yxpQyCa@Y#a>`n5=c;u1EI7zx*ND|yr64%}3^e`dd zu1gJ6+FV2+M5#|>JkAJATru#eLELx2AU}X@)c9UE?~q*L#v9o6?3ea)5eQ(pA~xsK zK)SAYE%4hqn$F?Q**#jsl_Gn2A`}MJEP?)$6Lf+Rf42=hO+S2#Vu2pe|1sYOs`=NU zlX`d@n64I$sCPEJ`6t$+h79rBU=*b(G@3?^oVt4H3+;5r=Vnq^CkEq7MP4yXt5<*2 z(j_;S%rkU+aevu*4H=nn3osp%A^O>Asc**I2eE(I06YRZ8!y;8&EaImO9U&-T!_6} zcOz1y6Z0VYpvv~_A6ofbj!t2RD5jaXSwQQjan->L_Tz9+b3OJnN>!E2etsiiPL=ROICOjkwnV zszd+_x*v75d=U8-$cM=DG^Qh{O?mY>EZ zF@s!wr89NLilYe?y%VD;{)p%4{u{xoijf)E$g}(ZM zoWe(luV2OI=9lY}-A;ZGxwx#@cut5@Y308)c~tk3(|}Xfn2}IEI!T&k7Bu>Svu!Fx zUgaD2`@!*T@8S|H&EWrtEl8T-_vy((7q^CXR_=n&da1|J-<}09Nll$MhM(T0=A!p? zFYd1AAtOgn4j)9(QZI?;7s_t+_|bJG(T%+nVrwV28aim|$kETx!YAk2X<03lx@(=K z&*w5cGJ={ncyzK_sA&$8>QDmOCYTnCoBq2WL)u*_Eg(vQe3Wz~ba>vJv zAR~tXPtH+yyP&%%c{@a?iglLu5pp=d%npe*8SF~HwSjl<<>ZHEW#!@IL6=)Mn9aV& zDBR3@Nf64v;w2WO90&OYZ1IX(RJSr2)CZ=_;=%bjwe4J94B2?7@EwR})`|z!(k#5r z2N9<=LI)B)VlWsJ$jvf(LxvwTU{rTLXT#((JahAvx|` z|LnEWcugG`Nxh{)*6 zH`umeK<&(KkJOOo^Bxersfll2-S1=lUs@$18M(B7TP(N&7I315=6TJ(PFyLKZh%>p zgv^}39$Jjc(_zbi9HK|ui9qpIMoCm=>N)4N$Vl{_Pgud?pF~Qo7Z-1ct|ooN-GXkW z_s^eb)3yx+!goJcL;PgPI&i07nG6hE?iHj_bmLdm)z`(Zba#^rJDBj3a~@rR+63Vb zvU7BlU>;nLPxoDpp?@&moQN@ARp~#eJ=FIDp{tdqW#0R<;OR5)kjByek*96Q?OR{<((-coI$#tVvJ_vFrGu_5Bc zZW~$mBMUfRm-G0w{Em?LKgx64++Y+~yy&heTEly;5_p%2Ya=dy(oET4!ay~n*lD=& zs_s)9R2$kmaH|EdSU(CSZr|h17g&39%T%v5?l^m+bn*c|x3XjPE&fg~A8O4ehel`A zE8ze*St|QL=d$BUeig{sUlgiGyHw35+$o|lxkN|E(^FfYJ&c<;bW)vc8K>UQSq`F? zGxvG?Iw<*c^V#6oi;zG;dr2~8)_-~Tsvk$_>=bo z_Z6w!twMR#w9RU9wZ#h`(p|w{cV9@M`A15H|LIW5>|FLU`&_Rs@q96J=reJt%HbXP z>yvoIWp#A9hv%ooQq+BG!aN^za5N@&yVTvLsn&SsAVHwWvE&-}j(=rjcz0Fwxq0@z z)lMj6QoP-6U7Qa;WSdh|@HB51^}sByeonn7GS@M^inn-v=;W}d8ftkmR3FtC?Ms$k zvhns-3W$*yIdvTyrY6|{IUu_((bDntTg0Pwq@73T-tORtg;EqFDiK)I3a2dnRT}&6#*|Z=e$hi->pRV)jlzy!uCl7)%~^{%zhEG2 z07Cny$4_hJ6YJf6?D`YUyP(IXLI4~X24>Tv(`XxgZ&1$v=^N;gTN}9^##5&wG*=2y z`qzAqmChIc7S<)b@j_DW9FJ`lNH{NEgK?OZF$#0;)4@szjDmy-Z+`tv&)?0w{)L73 z-N~aE931Pbw_@AL=zV_R>@N5a7#V#$9i9BnK(+&CfmTq6rQxJgCC%BYAxh(zXek9^ ze=KT2%J)LAjQ!Cu2TjjC$M^kUZO}+Oef{YZ3!(v&ehv)*ZPT&m&~it--j+1*yngM+@n;IBC085&Z5lBibUP~$*+_^`D-d0BA(*+(6hluz%qLXn{n z+eJ}h34yoc@dT~6kc+8>10GSvw%iTyRdDL0{L;z|%B)RL$`!k=2}J<{^Ay{>$HL&q})tlxk~V{JPb zD13r%JD0?UhhDRzc+>ZMl`k4yj?7P&zCQpBko}c&>n;48cxdy+-V|=HjgcwP4 zF&#&@t6Q)PGRKLr5LYO?P<5$hA5J2P#bx~0k_{s~exlBaLv19OS@$Ik+mVAqri~Oi z^?`9cn4$s_JFeod-n*~pZR}XcPJix|j&~~N%E$)hJEZi=0_?$sXWNm$R{5x={L|2 z_wM0X|NU&B!t%Am7R8>XTMK&dxU}RtYphy=Y%aNQ7=UXXW;74CF>nl@8GbFQ;7x z&+01(i`C2;Kb$l)w--0t!NI|4dUb8B+X9k~gk4t1IEv>Kj$w+=Am_p67FaNK9;RpV z(M5-b()&`Dm90%4yn1y7@_t(s9>-=|f&l=MAM?ILQ0<$S^bpLvV^jn&7u8~?j4h21 zHzNa((+#7e)G+b62}+@82#~3L=n4VZ5j|3uG%)Iz5&pUkHck8g$JJXvMZLCt+b9SI zAqLW^s0c`RgMf;F2na|@N_U4!s-%cW4&5o;DV@>{(mC`H4t(c$-_QFz>tn6G*IsS| zaOQtqXZ((1AG*Sg7!-G`>ayU)4_IGF3Tbe1b8@6yD&|l^?adVKR#MC)>zPjjb_k|L zfEVW=;w}z1F6V7K$QInZIYspC21gXY3cp1ZPE`9nFXphC7RpUAgJ9>t8fBE67Gz}~ zks+&H_EtzO696ic>i`>1#u#dcDzLl`(0=~@W>b3%ue|eoc42DZY8u|2kef>YUO-(8 z?g%(zTG<}o<+QwevteZ)!=jRdPQT533xO&bqPLYHL`dT4$Yr{4X;>8lCW=3BLR2dN z(*q65hp=czS{&O@=K)aUSbB)A4$qnaD`~3NCagUlDCIQn+aHF}in_DTHO9TvkO@|{ zs(Z+BfML?5d4K#^d=d;fA|Yt*Yn*yXN%;(W+)tutVgnEy$+QPb3rw40iHaseHja)h zaGl49OEmHg#!W0OrInO!WOq$ZQv~ZGp(!;l)JswI@~CtyL@dxUOR7fHGTQ*nR_JN zI+mEEWvY%NsBk&eSisQH4VqC98{S{MNU_XqJ;M3E|V}TT2a7kp5n%*$%Mu8 zRRtS?o4rJbi1URE?U<;YoqR7A3fou?3W}TXuGz_Y2UB)fC>T;l<$HB2FTVU^{T@d7 z^*udZ2}cLV>pDXfHjE` znOjQJ!sd-1q|}i?;g?9ZE-Sr=Oc`Fw8JQaIjUJACWOR3H@!LhL-iqKH_h%=~8&^Mu zBgMS*{4Q8hvTu=gl!a&Y4dqNPN5U;HH)bewFUOe`pNDm=@JSxcLms|66JMw7{IyW) z+EGM_QLYqid>Tnb+AW1jaC^hTB8>Fjv|^=poG5fCl7e=@nxXrvi>YsqY!WKm33<+# zUE5}G01?oop5-Z@C!ph18@BF0SW%io*;R1zWQu=ii?~8f_ZO;Bc>o=E!+S5h0(Vre zVK}kX4f}oRxaBQPRZ-NP@=@%U6MknFJk97)M9@>+j@HRVk*w8fyG6w||7pLDBJ#fe z1Q9Dj9b~%KF@u^;UeU(&ud~_F)a@rh)X*@HiIXfmM$YGSgZGawCbAqI;kO9?; zix(7(S)X`7q2vn`8Z|Gtj72_vO(ukNzWHSl6Sq(q(9@HLb+AO#-UorXtW^N5gncs@ zHO35C0*1rCY{erz9{X^6pj~UkJ9*M(b=|7BC5}QJsJ+~I%rt0c{~*0+PA09lh4i)V zJiFHf$s-;qoJtRkCsdM{Gg_y0_#m#)}Bp}fp9)7%^gTmK7wF6wFg&# z$`WjuKr7d=ZBb;$=cFo9_EuYaKX{~|K)U{AXe1imSJgd2xzU*Rb3w6k$VjS-1^+FW zO?jQW9fvNHf@N>?=Pw=ZTq~{M<^Kd2)9#QH{}(v?*8bi-i*nmkY3>(sU4F(^R(`xQ zlqW4mv(m)63HsxTLDmg6+H6DRbHd6^Wr=wkuXtmxKzs0PVvYjk|9fcC|D%+{yrW|5 z{g07^!WkB`^h8~-$N4H++rKp0()(=sce;e`ym=zZ!%W8Rh((uo_!8@(m{^YgvmIMo8AQBcrMO&8JuzZpy{Ip%0@#&)w$OXScC#4c!@aLWTjo;;^JL`FR@? zO_$ax=Z1T>_76PHm=6p3uwd|0VT#Hq_=31iyyTWBYLKKfiTTx3M)bC~OOxZXKUc~} zw{JQu`+bgOgcWe!2YEh`^twjE_J1ml-wYC@09j|n{P$G*$#^1t8yERqUbK0lsVGO$ zPZ(bbd8-#0eP|u1+av-^nuz1X=mP+dZU_B>Ye83RW-t(0oj%fM4i@+s>%JvtA=td0 zlRuDGQzHziu)HBzH(-hb4Qoa1MpOrQu| z+E#3%P&97|-?z@wu?7o^$w0sD^OqpOgo|{3fXg&039{v*lzKvd-WGZZ;L^*4Mnrd^ zB9R>NBcg@F2QDE69ddGV1t5@Mcp}yExoOViUIb07eY_m?e_DRO*LsT_H#LAN4UBzT zQG9umoX2zzqeP(f6}+%uph_0m`74-8a25mf!nANUf#L^v5ZVRXL)r#bR#xYe)%3Xq zE_;OY3vaJe+;g5A5{%yg>Z)?vO;PGU3_a-}JyHxA0@_;A;4U9N1tH5n7|3!Y(GU2t za}PWF`sTuKUYAI##r%ItMl;yeg%3;IP0*%%zAR;+ z@scS5c1-zTRU#`8*sRdabUhuvi3Ys~7%oMQX0a(n+(`iwBCy^KEHlVr;sYNobI2<0 zI)})A=;hhmJ)i}-Kb)^4x=_CZhxZmDFOL<*PDI_^8|>cz=m{Y(>b3iln4!b}8@S^q zr>EmReE9HaS@ay^CQL3aR<_cD9vo)X;&ILX4QiJKVti`)1vt{l_jw62osO*8Q+|Iq zyAOBv$90r?eLsbTathx1RwDL%uZ6FxuO!e4GHVODTS89IeQq_gXWizHk6s0ZBa)CO z1MQkLf<)Ecd8=4c<2G~vqm!`js=OOQn2qQHaWC6bx4Sp^A5FiAF>0LhSebl?RRC4l z(McBKwSbF~AWaxdR@%aD4DIhOP6q^1PN*pz9|L&oRoMshQEM1(H*ndY?d&Q?q%{*( zQ6Ye*Xk>(Kz~QS(CHao^+I#VtF+~BvdwdShYJzoOx+RgQ=fVd$?_^hxRi%hwK1PV2 zQ;b%-Fj7)>`D<-)_WHcKdH@|9c?zmq{+t^IuD0?U8RWv6rNd9c-_#KHTiI9{9H=%!1 z#}{Q+3-i-vwcFa6sx?d3Dgq|C_RDqa#cB)Mg4TL5gNd}C!FtQ{*8FM(E=P-R#-sEc zt53>OV13Ot^U&r2D)2;&?NJL=52sWUD+QJr8ddyReI{ANb5D!TwjW`&LET%iCUO~? zM~ZF5cG@LFj>$Wgc#o(P$F|YN3_LwPeR3XhEYH}PX?m&;JaN&7%Q{&;O8PT_Zodok zdz?89ym(5t57zcRPdl>1AlE;~>(#tdr>4Qj`X}-8B%N|WLQdRe4k)_eCziT-ydE7L z0-%%f%(F&q>3VyHAUuW(>}vZ9if_>iXw-;saFF|DczC6zk0h!v)J@s-`jXq{ZKIL? z;&x6$4%(TW;R-!urNL!o1e#pI`o6vvxGWPt;G{h)Xw7%o8%6(lowMzioxK>k2ho7~ zCP|NY9c8XkhcIYnRFStHAiYHKhnp5sDSFCge&!=<&(Q^X%X8LzIj&PvN_X$SUPsIc zs;Yw7br$8aT{#>vV4vXvpD=!H9&{k*db4p`i#$%50T>e3$py_4p?KGr->50GJTG3} z=P>qsWi0#xx~ijV+CaqC&9dGg(f2FUg*HSg7X31)V`x1^xL~`<4ACTSB;?C*UD*B*CgQ9gi z6k7K~w6gTcA85cDUrk=*hut!#UTy1la>?ki*E79`}3bV{moTe~|MG|K0$ACVh#4=!=a1Nu`=+*1|WA=E+g z=3l63YCn);`53wTKRYL|iJ>`~4d>`}Co4X{H0SLd zgJ;ep1Azpk55h)`PuU6xk)X7+ekb~pqM{~%%&NnP!w|CU zkSmb)NzO@npj|vWe|2c#uMFQmmI|m29 zakG$oIT*a#e1vO-@GjGVBNO0I!(8|L=wP1XHVqRZjpf=(NST{X-IROw>^J$OcWAp5 z9QXgYvk3{bAcmU9;d%n(%)+#Z@o_~h;lJKJ2Bz&jJ;RkKfP@oW`%-vtg`s6}bF&+> zjK(HUYoNB=ix`PgcHd_IA?M-UE197%u>UlV zh2ZXs&S!)rRyBP%|n06_H~_T)d(_$p@!G*4LC?#1Yp9z<-2 z+xkk-QM<0t0Y|`hT%Se^V}p1HE`+Wh>`egKm0)}Z#AZ-gsu@DGpQhjC*nV(g;>*qZ z^0Q!>9vo`{peji9d};p9;sKruC-S*uS=j152$YqSY`^<-LyN^9kX|@4vTgfcqocDNwhZ+3y@81R-fAU*0M5P^6&V>k5%m5m zU|Id_ctTH%*=Yo&gs|QJV3+^%{vCiq9B`?a9M=x2?wkscwbZ*~Qb3)>s8=yB-tUif z2;WS6pu9&$wswZk!^3!)h}V20_ULHtG%O`0;}jt!#n&`8wrxC^oTRCepRa?^oL9}$ z-ZWDbhS8%+w!8Ac*jSJvlRr(!J;oM`QGxpAe4XxIj&D9VhbR4$E&J*P^_Seww$b}P zi(-_I1F&E#t*P!yiP@0D-fndNOm9ysP*;5NSTjvs*N5l!e&WmQ3I4I=*H|}t5Iyq} z@OqG~$w4i5zd~jk8pe%Q?AnAs;Jb+ z(jFNm|4#9aZS_H&mj~D0Ival9&}6zwYy0cw4sQFEk=F!0ZpqJoy*N0srewt{dW(-l zEc46M)Nn;Oh>cr06GrZ+dSt5~j6-%JO;y5s>6&KaS?;iAun6l}X@M=NfKHEa^!h76 z*VX#e)IB>f$ocvH2{kUu9ju&n&tV7~fm@2q^T-F0Say=CQu7dGTgErty9kpFTDDPf>~(q>%A2kNw`DR*eHdwk@@? z6kqt~>|at~^Xw&36U-WU4cT+sOg)#l1$`m>neqk^yf-!DtqI3T$y2m92t?0jO)5;? zA19-O$cZ3j=+#6ZoWFl?Tkp`^!7Htyv;PV0EIoZ=0eWak3r#WI&wkt*GA!gY&^i`{ zu6o-hgf;soDc&k)o&KJ0T$Qa*&jELx8#J|Xa7GDUzOg{N9#$06P4rc}$m41~ewSwk zedz7&IWHZq5zZ#8xH!MX+m$*qHHvNCxw$7}%fW=3&bO}Kn*9 z8c7acz%7~>;c+0sq;Vw8TfW@j`+DE`B7SKk<4pKLy|isrdY_;Z>J zJg(f@A$%|eNcPP0f9@fFzvZw>onaXevEQ+M+!-O=WXxYV^e?7^a~4rJpr3^x<6Huf%V*^jM$-NJ9hm&Fh#xR zZHa#JnjCwVJ<*zx)JeyI7*E#fY^wIG^&r95t3fvL77!GL0A#6@k%=L8KH@7Qj6}h?FXd382gX^bdk7 zeRcOXNn1ug}DD2&Q6 zz&&_u)DSn2e^ z2Lkrg4yJ3DFSD6w!@4tY$iZ)~tTGHvV_0rx;Nx@FoWE;O4Fq^of}Gn_R2C^FX{@k= z4IR#+uOFeWS`R8507&l)&dh{>z3rbkgWc=Q9aA-S&Zg{wW?*Gy<-P1(4tX)Dbof1s zPAah8gK=y)PYZ*PMsqozQ?+|_-=$_go`){+hY9Q*UzEsGx{ z^fe-)R@g?UpWm_X%0%SN2(RVK2H5;VuK<}**N|JJ_j2fIentkc7GlM`yy}MCEG;cZ zD;*wfY~WL@gvznNZoG?&OU_T|o?BPU^U|`?8Q4CPmp6jgq{Kg6Ty(nv;46Ve2u{+g zkP7!+!rXsgaA0lEwNYU#LCpp07eID-8!tY3x+HJ z02BQ{;GtBfEx{-TU2G6WHve-ul$hOtJrEj10Ni(u-kF&-86!N>hO`YpUA2QWeTD$K=jgwePRl8fBABP&5rae>iM(L&Z`96)d(HgfXY0%G$|LC z`$oO0UZQ1IWk1(sq$^3Q%{{p402uyAjqJzzC0#F>tCzrRqz$c?c&|Rl zq$@qo@YL+zo6(3jw6kxAnVUjbdU`hg(r|MNUmk1|=5%(8))x~&xGnWK82sX)=9(`o z;r5M*NfqDwB$c+kT|mCGvkB>DN`9q}8ynK!z7>P7=_9NplS-cpp$>nADXoZB7s;L) zKB!g-`dX~P_02IMJM+#TZL) z&lc`q^zXcoOgSp1!BpiJXIx&blv8=OQ3=?)9C!5~r}0GP3|@I-(XfV2TwGgIhe6cE z_e^EOinz;{23nhA<9@#xf9(G&Aw<|&E6AXx>SQA6$dusy2Maa$;Xd;fu0MzIe!{P^ zqfXoUCRG{O<4=nUF&w4*1Ia(F*u5H12m6)3$>Kz6=qYjE6*MvGo<9jXAH>OXAZ>qq}ucc1wJ-YeVC6k4zyz z)&d(B44g&8zP`C8R*?x+Z>}z0r%6ec%t6(hzSyf)y30X!NU8NoHh=+eeI?(ML*n8d z;#_R%X=i2`s^hMy@mnV1B)JngcRHh?R)DH%Sav&Erl@mBbjh<+FNeJHj2opR2BjqOzojIN%*IgO!?|& z{3oQpkFJVk&rSdKJnFgcy&QH(QJTj>5b)Hb#Zs++gEqc_z1_Z<6WXizTIZDlX821~ zTlmJJ#N@vZ0i|>-8msfJ?*wc=8 zekM}hQSU7Cg>&D+CdKyAiD<|t-}gg>rVj|b|BzqluInE&5NoA_L-(*Y-RzAy6H?cq z&5p)i(;#7r7!*^0thflGr6$-83yqHt27trRDFMNnL&qW}reFO!AoX6AqN0w`*g$cJ z$ILBKEE3uK%)}k+aG0Sjq2E3u$Vqp_FizCVm&IINE37y&5Ee`@Do8w(W6?M>kL)1&s;u%o@P&0Dv;(!76zBL&yGF<@pzmRDvtet_V4yG)< z^h9lZV!{Tx*Z*dxFfLt<@4+^;mdFKH0UN#~Iw~y+HYPzfdATd$B~~OHOyD*6H-$8X zPBSKAmYusF563=SMGTbdw5X`4{#B3L$KlxsQM!Q_hKBdd#(}HL_aT%RKoxvWTa;jo zv#+fK>IY~Q25RE))Szj4ax&Wc!q?A_{2@6o)84_i4!(+PX0TP9o|IQF35M0Dy@<)68Wn>Z4nXFlz(qbS03H-* z06c`hh!tnaHXQiP>%qQO(otqUb`c|dkdnGg2iu325FUqQl~rT({TrYVsCV8=FKT$NTA2Yf4MF-K+1jMqJmTpwa%y9n~uSK9I>rdj}H6>Zfwz<`Yfksu6pXt5K!*DES!uh1k6kH;$hQ=A|+-Hp|mRvnjnHvk*; zXTl1Z)d8@E?X%Mg=X10s7Pbq6YW`t^en_{peR0gR;X6;uD7h;iW~~^`>u1RRR^Taw zn0s+CQ!&GzYe}U1iikw*)2*|F< z!^`hJwQshW9+u6x~a^FnRU49I@BVqd9Ftou?E ze1dCnhh|IIZ2Ko3MYXxsQF7NOY^n)YuSfyM`1jXdJ!)z_CXF@Dse8j=ZpjRX?a7L1 z-m7&Rk+i80^w?{fAf{b-~kd~J0W*JX5!N^7@y*DP!*{e3 zXex~Dm%rZsi)(jgtwDbq3x%pwjNH+TocH+e_LH~tJUq@Y{Y+9S2M&sg{z+0^gZKs% z!eL5|=96}i^jm5~_7Isa7Z=48i%hvtx3p9D;z|=QnU?W|5A6?Fp_M!G@nthtv^-Iz z07}kYCj6z-a9<%S>>Oe;0B~6$6t;M)QnL1@JEN96^j1jga-&@ygHfv+J!Fd1_ycni zw>(KV1=v#Tz9(`Ce*0#~6Ic;GsCb&27^excini2eL;I;HJL^?+Z2vC<+1A9D)Yt4S zzIY1?tMeBKuHqkb!}S7ylaQmQIXF1LJZHglm>6~XJab{Tn^$H=T!neqa{GI`0dB8X z9R&*k630RC^NR%ymMpy4E^rQs3~5M=!Xt00E3 z#LR52+=&mdRbafE^`Wz0=hJg>?SFVW*aDLXm_>mJD={faLPZ;9Kain>*ibIl>^=52 zP=RBv=H!8@|C!-A@Au2RqoDT*eaFnk78-$7otb%Mb91wxWayl&*PsDRv+&dgVox&Y zkAUM??uN=!6^5Z0MsB#Y)CgZB#)P%In>160*EQRa&Ydrq77m9Bcu}%V-n|qpj|7+! zNX9`x#{k1UPVS*v8U~Ut52LuFzBRu+tP#0+vx(*{Lo+EZ0uRKMcuU%JQQE+lR2zV5 zrJ$7Bm5t&BnCD^DgwK8@G%E)lZDp2INzIW4zv0sn2M1|3vw*-j*m^y{tcYQ89`JvC zmWt8k%E|_WF)-*4LHPzJu<96`qZn@y)LG7OSVDl;C=Od$S-B5+c$W?knot7E-rew& zf&T6X@UVj99yKv*_7v1>ph10B${hqpr#^URHJ*AwBM<|6`p?k;->swF&d2I#s}CPe zB`A2zNge9su%OQS2UQAmRrd)k)F<^DdQFb?Pd7)vt_%hzIQkpcnruRViNs?yjZZ^^ z_(T*amjU0_;bMP(JOKG)g2M($a8MSey@25(FhJ*VI6sw?mST>M2P`abbcoT`V=Pxt z7mFbLOc(~W7l;!A2Hq^a0L#|M2B7(92mgt#G3^r z7l7RpA_b%m!8sf?4L+dlF=IqxSy`mY*202MD~%T=_=%)HLsR-OP)%s{*I-QEZY23z8 zN5tsyL1P-?Nz?dC!I{u|Bw6&C3 zg?aX?#>JOkCWdPnhm0BfA!gEnt&pIt^N-=<5&fL~!C7J6 z(5IsyCKd}B_rKzULkA``_pe*)85>`Pz$*^E@RuXfMg*LXs< z4`!)y(&2TXa^Sl|@9_2FwOCP*yK71KIf>?9yWi-n{m;u{aFXzd@ISAsfc}^Fi`w<8 zCq}s_PA@T-s}0H&c{j>NW{#1G7wyE88YKxv2lC$Wdh=KJTw3!8`C=jVp|@tE{CGm> zqn9>C#J71VY|cb^F{`SN>SzdSbq!im@w&?$zph1MOD_Ca-o-a7j_#oHzw0gVFdE-9 zlxv@DV{Pp--~})bSy=^9`~(ZlKjH1gb=aBq2J5GuJR0QVbk29<9J{-^3d}}WA#os+ zW6%ngT3-mE-0N_AzkTcu(;NcWvb$hbCE{_y1<0{~$|^(HCcsFqK{%-Bp!&7*pKKIX zD?}^{DJVi(fmS-P1Dx5!ni?$?ZAC>a(5S+Dlj1QShaq&57+{G2o@`k^tDKOaStSL{ zq;w#>{Q}rsoSo|-@!{dIMz{kMv{y{T7oiOXT4;9<4}jXdhui(FTens=y0kSkK7&UC zJlPm|1pEerANDTH{o8~zGxL^X^cG_~BchZyg6Ap(PWWsW0gjhh?!{~JVKqzML0(y7@{4`OvP|1VyHQSsEwGDUD**||i z!ZL!2x~(k-7&Nh|tz63*zP2wy4>RW-D131)T_SrJo!JX2Zcd}Fdg$}~^H~pW3H$;< zI%d&g%Vp=-Jx8Quaw7D4sw(g<**@eAkXAC6(1ktKv@cbjDrUn;un^Zw@F-}OU zw?yLB@@Rkm1sL^!pO^6&6kD}Nf3D@$xVXMjlDC;Vzi}lL&w#?niY7^9 zX9KE^CU`min&h)*1Q?*{P<9Dyt!tZ6$jaw|NLGjTp5Yw{iP3p8ctSvua_{h3t zaIw_X3+wJ*yLN~goOw)hz{3W8FS1V0B$yhu>W9RsZmv@j7T6in!ZzFGz2&O<_I8P7 zcgPr9FFxxI_(Nu%;jWrGvSz(lBpY4&Cdf|s(NY2=A+ zW9x5Yi9!RE=48c7|0;lMg$?*rHmW{Y*br$by9tPwzK=-c8$1f*!;`*1@5c*`8oehR zI65Y=wO$f-hn1A?YN@(4wHTM0mSf3dN8Ys=5`BKc_ux*5b)Tc|{M!rE2(#^7ZuQX@ z8$Z5D1pPJ9S6_aA=)^}N!TEtH{adk@d+H?Os0dEo%7NwTovru-u z5t0vS_-1YR#6~sh=prL`UOe3Gd7Y8=B#$IujGKbL9pr+DApl5pz)>Kj|@6Cby$?H%)?b z^IP9;hS0x^RsHfa5LbDLD=9N}DwIC*fi)bSYo9d=mvtV=Og&r_PZ+C^G->f)Bn?j1 zJUhDki*(YaO~T+}_)DIiF+PP2+PjS20^(0`xn&853gz#B${TL!vt3a@!A)o>%+LAL{L^oftL=kVwxFcMH$8p1{FiiKCu~8( zjjuk8VU5s~BMKiA|D3LIqq`nhtzY^7*JgHfw8R?dbFlOBwgP8#8-~S2)DlnSXM)QMa+#`9{?@^6S?n zK%D&gY1oNrT7hR$f_0?UO9>K>`E_UHpmh!S^hq2XMQ9Rv8+mzo&~#^ASRnHgfA#7n z29QZP)_e_02ylnP#ZTGr>C>kG!oiIc^gtHA;DOUYA~7^#(y{yk-evah4HIdZtIZ<# z&)Q_U!P}Y!vkc5>ko;OlCqKd+9M!tbov{0YX-*s>XY!fMtgK|@8$6fWJ)ExV`4q($)qmFyXTYqQ3ATNcj2lt4b+ctxx1>*#3E+V9mka z!Jz^C0J6lySa7Xx#$*Q!;Y)*3eJbE&H#E~9!T>ETxc@9t@F-tvNAt-@vYN?0MsEV6NsRS^|OMBJwK*zdQgxIaGDzGY>u&v|me_zImq?@|1FXvb(h zl2^hohhQsieR@(Dasx2)xJ*QpZrAbd&v5RWoMnBOaUC4rHz0?C!{9|sR1_T(QwP)_ ztl+r1e;+TBRZGIkD!)en{8JcfEs&0IWMtYFQvNG_y0y`KgM!zRa>(w-fVhGJk&=>9 zXhcM6$@CVyIAqMtnU{JI3hQCl8Gv|+nclo05FGm&8F{Cq;f6#&7)EalsRw^GW}}Fz zii)|yR%!~B-^%~@f=;&p@zV#GmLGULp~k^ikL+BeXJfk#H}vhD9kN%a>B-3_XYLLT zsO#Kgu$tDi6m4V8_G&SB4A#QW(a4VeOZ@FXpC-^*6@(?e%F@z#-n3z#OBn zBa05lA_v2BiF^*jsOpd}zur$z>^ZGeuN!^bi<+!^B*N<9tCfIEoevHi+FbZ_F`~GM$dBW z#JDn_TPfi4mfx-qGjF=S)|v5JMu>j=atCpvy6zNd@s?gvd-|H>q1&M*qN6irZo^z8 z0i~*3jIU08vb%chf$SCA9jg;1;bOV+9|S6pzXu}io|O`I;-w z;NVZx2@49}==cS9V{MvhBe{eTPa|^urkDEVkf>m6?b;u5RYSwCO0xUM`*3|1>rL+- z{Kd!HtswdqRib&Sl|_?eF)r(Y_{Cus#Y>3Zq*vZFHQ^FFnR^r9S!*U3?U=DXCX`Qm zf3)(@L$e>f7tPj=o9}+0ShwEd;iXNardDuT7_5{j;ZdWc$oum#_E>tHsLscfl(!bA zPIcFsywD>bJ!g1ton|EVq4&r(3N2A*JUo&ps`R-5v|3$Gj!U1z*nVM21Q;;vJUl}k zGL~Jg94O4V%!wU1Y4iQ|+Z6)4c|Z$~YB@Y&;G^FrG-{4?EmNtha%x-2A){Y;;OZV8 z<@OLd%}u|%9*-V)C}lBQY1q{bPqn076WXmgO1?5!AK%4cMc`nW=HW$-z zu#iht(t|-dVOoG=3+?wws+C*QVJ_V;mA@Q( z8=0Av)fggr7U1K9gUPBlG&Cqa@Y~XRO3vTX(o#@VBnh(v9#stu0oIVPdpE8mL6gL8 zS>6JS@8r;A{vqsE8W8nT0jACt7FsAZSNVnK1`k~GVr==5$WXmUEG)N+;Y@kb;j?nK zxUg_}+|kj|+P=EZ|0rEP?kc#FZ{2YcW{dq#H;I(YhE(tq^6&B9M>lFff(tp3U4-2= zaONp8RZ;(~gLyUbQlJtX3Gfgvr!|5tEU+GiaU{+CSy)Kfn__wJpdREn`f@#L%5Yd> z;}KqdD=jUpR^`aj8`63O?gpUPGn*(WDfx3xk^wkRSKyd^;^9$4y9Ff)j3C6AB0hg3 zBYv3OfN z7Lr1*b5Abbb@}LJD3>r!6WGh}>5^rhYF)TcU;J)>k0;0^@DUt!P@Y~Y3<$Wa{TxRT zT7s}7pPZ5+g^(z{u$8ThO7vIZD-9L`?h%Ae@5Wl(V`tAAwarHARI% z3*;LU|G$s^a+No4zQ7lZA*@4I&f^dW?-JJ5Y(~BD1kXEbmvw0}-ebv73P%%RwRL7c zk(7kw{)-b29SsfjI*-49H>}rHsj%Qr`7n~oZXXpDfxYE+VES6e&Q2dLoA4B6I6fx1 z&7DwMdYgv_Ju~o-)l$sDVnmouM`wbrv#GUJzOSb(LrkpR=RystBjW1jc~DU5_92N# z(*hDnYm%P!o&N4Z!&3O}-g*aWYf3s}h+H<6fzz>nYU(QNiw8vN5JFN$t0l&&#Z2e) zaAMs%i&zgH>h{HWNy)<7r!&2h1`F~F3-%&6KVHZ0_=iac+6uKc7|$9669qE2YMQl=ZVR5@uAZv*Du%WRsYgW;_oYW> z+Dt!^$;tU!=0xw^qEJ59pLpqxZYX>A))OJ~xgzLd?cRO*M~QZB(T*#ONeJZR_wO@# zPi$02CVz`RO53lS{xB3!Bff7TLRgf%o;vVbt6lc8s7K?P*u|{i^sU zB|kYDU|HHqth3hXFH`AOYeY%Ovs)y?O-=f`y**B%c^fY7&-^k^3q9dAV{4l=w~W&5 zojo}kr4y~QJC6-}*{^_Pjfm(jcPo2YKuI3^=_U)*;66)Bgjn9Ef2+K_E;OC~_5bgQ zK++>C3($BesibR{S1QNctU|}c@a;p#g!SKrj#-nfxeUZIPFx;$`bquQs93U$!s>RJ zS)%D{q7@JFAZa$z2EF&Z+ZmlT9~e>iD6#M`13lx<2@$WhCuN@RESx`){B3bvaa^HH zs!Ps&3h(Jq1WNe2OA6Hb4Z2B{98GOzw3y$$wB(m?=@(QH83|#QazlGN1>{nN;@!6R z)dB}R=CV2I2iTU{W&~kApUpE5)CQM)b zHaKaU3e3ki`1tv+yj@*gg>{?`50+ja-G3puGO`LYP0ecOomx9CvYiN@?bVOC?~oTf31NH9XJ|40riD%x&j$~ZmF-v3+Cxjq%C22)#qO&he9A=*bM0&6@XJshbx*-| zuPF*EUahb$gcKA_O8gCis^VfUadX&;6`Cf!2*(O+aouSKLXE@K(Sa3E)u-6eKr<%w zqnRrgk8G+^9Q_`&xBuLgRLoR{p*xLz&lgyg?b!>oNm=&r1=*?1jYQ%^Wu_g-7A0n7 zJ-hDY;P4ANGU~U&1Ef;>O)$IabdtN2 zjQwga)zveLv0D?*eJqEJ4xCSIg|YwM9{>KuwUses z3WJ=tu)Yp_5~ncm{-&i(FHa*l46jmq{j~v-TnxrkoSJHw?}2AFU7g3jFlb6}i0q~4 zJ9R|X_FExVR`U2ig@66r-_Kqkz<_ON&OC6YlS^H3Ob|@fNuwwlB+alwacco5%6z3G zfDpAjF(BrJWRUSegGzzyC*{YFjP=ba>qyU|?ad2ysJ_HJ^1DSw{EdUkeEy9&>cItt ztK>+|P={*KsdMRr1B-HZ^;rH#GtLaMiuaiwWoLy1%-F8cGAs-^7BGu{mV zXFgn8$5S;<&fhe#lC+3f|F&YXK0aN3G`fl$m7KmbwNpdX*wA$K8X_jsB3qCvgXvaJ zB*#1Xk?O*Z7RSl%+H(gTn#?fkm^`wy?t;^dh}*FXBkWhMlVb$`LmrqjPHmSz7TK|`JI&Zzey;c?d!tOp#wPjj+|$!BwDT0f0C}q`BY6Zl@p>q%Ic3R%@|I9H^mI$ zyrFjXzKj@;XnOEWhbe-VYp^0@`VV=>9v@uF$2|z< zf1`p#r_XKq!kUO!V)Bcim^*lm1N(=I91mhD86C*m#PEJ|f5m>$%)u~zbD(kxhq+Gc zkv0cz!YOU$ZcGW>)`{6;Si>u)2CE_jTyY7bG}`C5##AS2zbiiYk#X)>+Ehc31x=Jz zXgu!#n~MQ68~yCP8gmONPO$u<6OXL83LdC(@j~j?p=aC_FYKnqW`)nr!vmxzO33=* z?0Dl5CH=J3#SV3ab<2-MMb-)RAQwDw57jTeNrc{UD${A!f6W~X#?;RX?BpAZx?fR`8VY?;#{QKc6n2p-uVeTby`Y`wu zE?Y% zSZRrO!MqoqhV+I~ZvjqBT)e)v*8J4RPYY5FjvJ$iuO3V^G+cnrhCzb{{f&rcshDOM zJW^8jG~n76dMCgfnnHF())KYxr&9Ff{E(s$n0X4gGn=n;R1lQ;pD0J%2r~=I&i;S` z1I#ZtqU3t3-Z4K{P`FbJ=!V(NO%{0$DK2`;nc7@CVJGMc6YUGlnGE;$e}Dl3X6hnl zWRx~70v|`my9yKVgDWuec+&PQU<-1J4e+%%{DJB49G?O`JuyZ@KI4Ik)5X8v$Ilal zYWoBa<^Oz`RKJ6h8(Md;iGtLJYOLHQ8C5X|lO)X3o_E?A8@@$8Ft|zio%mRsjAETY zuqw~^3D5_wDQ2GJ?bxC$8DLIfTGy#vWW*F0C}5YDtMaz<``HX9DG7Js$6s?UgTW!oK$$q^L6n#l$%eae9Ykg$guKC z$jOax?w+ES6H`(OXHZlk?$^q!oP^7&Y!~mwi6~{_n2k7gSJ~|vTu`DZV3bx2y_qJz zne}R?M=+Aw#D%DFWJLb-{LoQ_Mi{S**`w3Wo~tS8iB?JSw9E1?HoAK2*3~(I0^?o+ zWt9UR1;f3iF1<|l+r)EBqkfjtLPyA_I+Y4G!{g)Z3DBAlUN;jQH=~N6-pg}%aH@ld zHhmN3xU%$qkNknmV>9tLL&?%n*$Ckc;hf^lq8USi)!xXME3N8;#qk5WFTU81oYhHY zHl|{uM_kNaJI?+-%bUl4!+VYIMd>QagS-Sc^YkSD;J4bq=bTqscGq6SEnj80Ib|Ge z#Io*Mdv7y6#)P^7ec)YSS9Hr=`yHNDQf}%wa(2<%#l&oDZMjVMEqudzy~6INNNdgX zG~-R7DEd$?e8;RmF27pO+j;oT-%=vEpB8OBf@Ejuz`lCzriT^s_uiiA`uQGHoz@qM z<*maB1q5F1BffWE35>c5MoR0>PRPe`L-&jkhWIAOQ_FMa7F45!zo01DI%YrYNNSzY z&j>*^HEHQt&2TTB4k&QqDkrbahYVb*^RAtL#vxQ>tnfZWVZ`RRl_otoajxNTf=HBE zsP1$j{H#upJP4P@YjUXf_ivn+FSk6Zy;dyiQyn?pno>#Ud_TiqBeep66SpsMO8 z>}m6$cOG7?CiWC3Rg(X?4|q3=8~I7@)|cJz?XtM_gL0U={?UA!#o%`y;XV$J#wi@$ zvR)*$TEp5mWKqd^r0hMYSBiP3TEB!*n$R=i_2zw$Tnnx(lhKds72emN0*t3+&CdwDZKsg6;2mCblkJpa!8 zAvJX|!K=mkb^v8w%y%LV6A-u3v$7IG+xMe0bd}}fh4>BlzaGJw1ZIIjJO{eV0E4N_ zchH8P$it(0{S{M$zHLpI?A?H%prMCsib_f!Jx&h^OcHJdS@rbv2>9+n58e;hh&DGc z&7%WmM=x0aWLMA7W(?PZ(Io&PJ^i57e5g_IG*t;x@nWztkbM3G-qJwxBzv9M*jSuR zSpT!}Fc)&!H7KvThF5f-kufP*Uzdpe_x8H_gP$cODk*XSC&NY7t%@Z4_sC6CQybU& z;EP~mUtMlcqQ?#oXQ)26gvXVgCa>C!PdNSk{U6oz^^>PRW=b(U$yJ@lbU1z>5H#i% z7H%$h1dy7efUg;+F#d^6`!<|Ca*EDywZjY+LuLIO1IfsO<42gYv$Fa|N6&H0xr$++ z82a@qeZbh*nBHPX#NV?Z&f($4D}8G)?pb}B9Ho)lLL~Y(fzzlFMzWY(NLN=^^V7h9 z%W;k^7WU1-rSJifDNhGZTli zf!^?m6qo1X;-VfrpamutAHGV{ZSo^vdu z;){a_#JP~->G}8Xgw;FJ?ub3H;gLr>Uvy%Bmj(rKs<7Y#pcdtK-LVMXklGSi*&vzgf? zk@264S-KQE{-Uu%6AG-nH{8oGzpiY?=%8XltIF1T*FX>9v9+^b{QFFleTQ&u{WLGU z#A=4Xd?tR%7_oIS9(<@O^Q%jP@a8}s(%~?wcDWC_E2F2a2F2VcQQmiwhMWSp~o`-+*{_>c7 zIn?Q%!mUB4J>GvGp+Ca8a$mGp%;M>cop3WfumikS3 z_sOBwK@HXn6|P(6S}bgKpCU^=gbOb(dyyVF)uzw=6>%Ez5I*nxtc8kL_h#ext~ado z*y6OKf1=za5vSxHpav2m!cVn=gNn8Vy=oA>*~u?_Y0iYNkRPCXUZZF2da94!j@Wp5 zdJ1E%79NKRPbH8l7|{uUsS#}SWiLwT4-t~ZcAp+6_F(=z4e;=#zLoj45?j=uwm~B1}Q?Ts3n9p3#bPs~m#qcCCrz+2Dz z3MKebK&$?mxno4kI2elR^V~ODo=MW}!M~+Yd7vIf$i21_aqx#SD;$%Hnj(?hn91^Fei(K*a^d#@5B-rqi zUBevhOh>tQ?|7;4#Md(3jY6Pwmh`1o!r--`U%zaTGrJvJ(1Kw9oF9`3Mv62gLgOgru7b}3!N zf*Z!pqiZu2-O|qIjzwF5@`HmCEqLX_qe!j$R*zEQN2jRh%X!t(GVpx}BTAu}_Uq3? z3g$KrPozrjN=j1L^dtq0=L!!r#10?UGjL3{P20y#h>j|Yr9@)GE{x}!s_fy z=@mi`2i}z7QS)6SshJlL0xMl`y^P|5{37YS5SH2LlIY>dwHKTb(*S+~&i5{sHf4K1 z?y*<+oy7V0l*3OQrqR(|i7#JX7+<`1?<$P5X)ay5FmN-LOszhNV_EiDg1yVK+&EFF zHrD1BmcEp4NEv(o@h`Bmd{v2X5V3ZXE0yLJWx1D6cF*sUU%wv`Z6X&QwRfPJ>fpgJ z~z1_`=<-RL;Yr?+34eH0wOvXP-CwyQzM$ z{BN_c@Y_E&B9-T~HvF-%18Xge576!wxwTqDCkojTd?t- z@ySYy{qByAq>@3C5?y5BdBgKA*3?7r5rPx6dwQMiXiQv3_?)LuGF*nrSzNzAX?@2* z)@b=fNMLI7+O@~<9jbBP97I)*6*s9`UEH><)HZPj=+~|D&xH=X^e?vRi}r4=`R2D; zNOdMZv*&N@xuPiT?TsWiKb$#MXL9lHZT}XgjV{u6kM(`enOwE$W6xtXv$jv~6!KRfTS=L+pIRdCZI*1#Pxv0rUwkCo|p7N16l*n8=2|?9H-Udvnpxl%HdD zthlF8(e!GkC`aSp&68Dg5uNRWy&hqy_tvF;{TaKo=kg<}(V9PAVj>Dh-hHpVmT~=k z`KH_}I|Hd0^9jRl?`X51rWA}E26k%;7cV*-_?9(To_{cG<@aX`zIXa=LkZp5p3U-W zmq=b}wBpqvtqKMyJ09L!uruwC3d@@n{Re48--)NPr zx9>CApxT*)>oljEvMgISH-_4hGcsnzf~w#N`@OO;ZF1j6X5nsRIA2^?lJzMk&&@4K zI`1qy{>Nvt&_R7ZX=HJIUU+N${k{UD`lq66Hwq)90cJR3HsVQpdDY2E2(3p`jx;x)TDuPeX^qU{nFGIKE5B% zK8?k>92@D%)L&n#2t2@-n&%LU8>jK%-IdSTngAT#&-~#`-J(|z> zPMkdX(3$p=$IQ`cdMypVj6yo^4@iz|;_?a7 zk_*0*zxacn+)RjC+}7)Rv00apy`iK#3yH&#{5>Y+H>o2wn-Wie$QI;iww@_UKaiQ zmTS^!zfO6Z!dGEo+8o=D_rEVT%zPhBvD}rESxlCH@XX3=nxE@}$@ePzJAbQGm(TC* zIxyd3_k8`nUw66Dj?;mWajsvQ^}_?b1=XZ0pXa7CB*?hzyY^R>l&)j7yD)s0sQ2p6 zy#?1>s*w-lWLzPhdb26&8i~}l3xnlGfpB~$rRRyETK#ZYsQIZV70@_k&DJxvgU^CE zeE0+N^C^z#{@_@7sk12Onp%QOlN7H2W)xZ)c^RgwSL@3D%2oVb{ApHZ?PlLTov`V- z~vKec;sY)98uUv=H_9b}kCejl|5I4p^{c#gaI#ANLxh$7=3p2}t zS#cJzYrlVIZE?Ud2zoR|v{8~jz8!_Z0OnvT`y&-7{<8&3jknAU>B4I<383d&a`l@zkDv4zEom+;Vm-BAZC z2rxe+FU$K`C1R2$w*#v}w{My+Q6~*)I&=!n z;GH9p4?W-kmBWv#Yih{c+>9R9d>tRRs0(HCDh&y^2e215vA421`3(xFj~!dBf7jjv z-o*i{!otE7sCKSiJs>?87JzBG3W7!O(Dny4eYrD(SHu}u2tx@L4*i4Ym>z$~;ceesB7Lu&==Cdbl! zk$12}CB&omLhe3X^>B4Hg?q8Z^b>n7E~V(lkMDxyJ}GlR$ZOR}W~s-*Q}@CDpv3s2 z?k!0ET%w{30Hhh3FJ(r+(?U^EkucaeJ8mN^E?zY~@H)qJ?%1c#w_2sWhAYpoN;n`$ zuP!J5LSl0A*38T}i!;x4b^xi_kEi*p-BirrM5TmuENHu*^~%?`ed;buD$or^qEx8Q zpJ;A=qlvpr!oSR}y*^C*Pv^j2%zHJ>ak_*}`pq1! z&e`8@e2d+>_r=_&wX_{9{iPt!lR30;j2cyD9O~qwD=VIN7AL(%W@bpjC>k14D_i5f zq-IV%t?*d-wFA}QiS%Xpijt9a&ytWJ2fxTj@*O)=T)#$o1>JiZ)T3uFA@wvVBZCR+ zBkP@Gf;M{{)oW|E^bH&~|Zz;BJ9nUi}5^ zoJ9{D&7|hl+>@N88JxmP zi`3WGlt6kE5;}bUfsXIXm#uc=ZQd_^%6DU)Yl)5C&;Lt$N^DZVrx;0Gj=9d60z7Bl zsGd6~{ovt@`B1qGoLPpyUJg*>h`XmH*VtatyiL%|7bt*(pxAX8@-PbR!x!`lr`42| z4eRbPJS4vK_wTF3kNHZ=#d~kaUbn4%W$m9nbd79q%^vNdGcJ`+hg`f|l9Bbb z*s0QPMG~g(XH=o3I8n|r8|>isqU?}Tny_!xn()TUPVIsR?aQ@EPZFs^#e^}-eZ(`JH2c^zEG>k zAp1{1M*O{+nU|y_AASSc(wz#(&y&tsr=cQ7frk7c=AMpxrqx5#A{9xb3H5dDuAY9!iDpSWTsOZl5wz#|wT56B;e4(@N8JlS7=gE3|&*R4t z9}2CHHn}?5q4vsrRQdF$t^zK(a$2?UuS)2lSDieFrxQrynxV7lFnMjA``GMAs~7BS zc6fT0*N#Q3adDcGhK4@pJDd(%q{918^3Lx6*y9MU8=D0pL&-C-@ z+UGyr#y%h69c)fcsdF=%a#}M&2M!F(^$i7#RQ!GI7+>@8BM-k`4p~!_o7tQ>O;7J( zZNq-(>3_c+B1@cqwwl)#g-kp16`wi5UxgJrYv#tbs%!j`nU-~#KKAm=+U^@Su$#FO!^;Ejs#jJO8PIehQr?I zE|W%`kDj`DjfCXS&u^}l19BR>ZrpLXPtVV)k#yEt+oU5@`qG6WjEs1Pec9tPT-?Ir zyY&7v9sjj)PKYEi^97lK4;@YR7tPr)2fw>?*+{U=o$+a{!a=9El&Gxx!iq!T#CIX}aJri2M>d64-SDeH{k1HIcwW}I zqWd1n!-JmmW-Hq%hFu#&D$iC&DPOia`h{d0jg#<)Q5@ZAmwGs(exTo`zsz`Rz71{4 z$!(3+Z;p0t+}KjE`CSA&=ugq&W0T@q)m5F8=M;@bhOcbBTP!Hmd)_MRLh`yZ+r8|I z$y^oXn|EpQfE41osC+>dw>55}DR%Dj>-xy!HSFRS&)f+NJP7eXYN{28r&~4S`QalZ zEFrk7s}@d|H?aFbLp8A{Y3pq;(g^ooY;hncJci1t;^i9i4kS5( z!FJE_aa2@wTbowtbLtaLP9nq}-%eaq6tP%TG4ej;AWp#Xfl|RB?flZmhw3t`OLt|v zjkuJOI;_aMC2qIwFbPt9h+eJgmNC+sHJ(0Sn_bxe4(AsUL37iwgq>6lwjA&eDfW6r z2MV>nw2MFeOAwZ$RRROBS3Jn06(0(TjUB(NEZdeXtGw&`Ya)?ek)D27!FF!+eZO%8 zJwQv}8N21l0+CW|8q@M3p2~Sip)mu)t@f(5^^Ra=E(y)-%*<~a3xcD-$s~OKmU3Rb zeAyprlgjt+Ri0-A1>MHqK{$}7p@nmZnUp@^vH4s&B; z<56zm)>}?a!z*)RMGhmrgWcnx>Jlw!Lr`aZs0-R*-?if;uCh^9p-ru=s@7?7pEaIIe{5sP>!9QU9a3zGCQAkXFJ;cThr(Ku`KKCJoh}b7Rcy*fUEE~ ziJP)@0nI}ZuRf3FC-_(e)!zEv%=G^dUgH-ikF0eE`aAe;m2??gTTEtNZU60>? z5HrkwAIWnsuz5xIia6lI$lvI*G{Jj<*$HN8`yk+_5MEYr$Pw0zHw+15@ln*&%oiK% z=?Sl^%W^BljRIO84elr{zQ>2{`ah&EOYRyJNFJ z`S|fsG?WX*YpuBLQ`E*1wW){7Cee=0M4q~1*QaaV_+aM7lsAu#9KYWnFqZ3^DP8I}ZXEc#RCv0q+;T+G<=4>Wb`6bHb{Ff*+{i+W z7GwA7_`99YRbIkrnOlw@vz5xroSH@Dy(no%Kd($z}!8 z>7)T&R0h*~wK34r4hPi)c31Wk&9rff7pj&VTYTI(w8@Q{+0IA>NjJ|s&x zt>D5k^2u^@C)|MFyxEF{3rYUFdE=j7^4?2$=(@x`5v*Y!@+u?+$KRsD4PEm3Y|UY1 z)#(!|D&zZlOI$om@LW2+FR(j;7024D(UL6Lm+-^2CL4^+*?UUc1S3 z-DGcT69vicG;+c0W>wP+@o+C!w@SAyC%}JuH2A~AsPrOZqj2K#R1rAF*|wMN#BDwe z`sMS<`79v4iC|L0sc{do#J075IxUU*mQTCCIQd z+pJ5B7VRGz5BP9Zq3^T3L%wOi$6pK&Z;XzK#JRSg+F1O(@Ff2e3reI!;h5CzRtx`U z&!rnqev~~XBosLs>9Zp+Gvtrgw`<&NPv`v`)Lm?c%1K$oSQQNn#^*dxG0@*Tt-dQU z7!Bmm9TD*9*tE6KGV#i1rK9V+F@N$&`wG3^X5M8Fx8=hT+=;&^sBPnJNMp{_oYW6@@lU4c1Q!`e1Bc0WJ(@Z^n^S!UQ;aiB3H`%xNz zE<@Dz%gr-09z2?FYdcb|9lvsKYWvmnq5tVRdg@*g#Fe0+22+RoPgcaQXaxLFHP-_D zru41k-ac|%ZPu5qnUwTqe&*~Y?T8P) zt6R!Ugf#lf?&!_W(BgJ_%NEIx8J7mZ zdo#JX*wHi`Vnit^|H-8jOP2y1I&bn(Oa=J+WAax#{=@}14@?ChmiE|n(`}}2JF_e+ z8(XgVd!qDeu|UF7L#RxvsRXFqiElOz7U9m%pN;C@LQ!}-Az>(nzgHZ*6#}kN3uo=9 zlai9kv8dmL4E7LYE$tSR7P_pHwg>JmJ#Vgrh=H%i{0)pCYH9y8$z`x3?#DW|x<<*N3wynVWNgT;m7zXH4wS zP|a2mC(vl_CnRLsML%Bj+OU>!0BQ5lp>yOanebo{v}|RKKNpfQ{s;b$|48MUL+*`s z+nCa^1LS*L9Y`1VY&HJG?S^)kZ*K1e`~6OC?%e&qM52{~Su+-t&!k;;fmEZk>KL>N zB#p}o(k6@5g`ZqkgqOBbP*Aq+n4h0NYhchsI!{3?I$<2@8v~LJDH+-OlR>J6ZRYbJ zDO%asj4sxj5uViu$=YF#UWxE{+j*7iB(ocKvOw_wK`>Kc!?+Efba}~14sfJ~3B-1D zyoOlWvPHl54sLLOGuH?_RX!cs^>cXmHsGu9hGIjbIEUR5*Czk+CUYl`oYm&^+*SGU z<8Ju8Xl02FK8lV8m@~j9&>dI$vAUWxLgx2&9xe72|5qXfSD@fo8bkXSBk0$rvpej9 zO}_s4sxN{(pc6i*qk4c>ULMDa_2;e7myo=go0}iEsG~MEImn!d=HnkT9lwd*TdPQQ zreNZ0(Xhd*d?G)hq(qw7Dm$8l1yKVI>An^Im2!Xm#x;%KraMJSn z-=CdRbS#r$ya~_3D=P=b8TRll`1UkzxdIe0*PqkvNIo@Vp zo9z=91mXlw>LI5#Fv>ye456bNaW<)_H`RDz9`gF4_(---+3g6qQ)((IA%orzdHaTm z(JHTPRY|H+iUF)0+b_FA5H2w+|96bF^;t35<5MYx*-xI(MID{5W3g|Zc6`#+b#r}X z%tFGMp95TWw`Y>)Lc@36%HF_0NlQN#jFj=5%fZumsXnWnp1*n`!^Y~V-ikhP7q)&N zYV3B97OCRT3*~BmebEF_LuKXXD+8!m^=5{a*y-t+bTeZ7asRV|SHJYfab^*z!c4~Q z)_ea>R_Hh^Sd}MEpVz|jM(=F{+hq6J$VlLtw>Ote|IUlq*Q+P9>&NOj>=iTj{tde< zv?EGjxo&Mnz!zjD&G=s>0je%Lb{JWFP*i+|R;J&GRWK{pXL#^U!mQFvY})?k`XN9_mZY_Hy(cAo(*xTui0d! zU>Uy?(e@O%v7IUXHzt}iF0+>1jan@*USmFTd6qin_Ro&^?Rwv3e+-%N6|YH;PL)NB ze6{>pv|z#hsLb|2+^b&m#x_zxi;jk7j$zTh9Tok{`!s#L7}C~mcIXYP<)-?*Vj$&s zN`2A9qb!@Ie)&&)Y}$+7&C}eK4-&TSS}9XHYW1G`**f=`*#W0`^S;UPkFggjDE*qw z8$4SPVO#vxB9(pm%5DwH8+A$*F=UJ!(*3877H4~g1n~Dv8lahQgHu|1-uE`|6x-0XNm3_BtL2mv9nD{ZRbS8YKvDw__ z#4dl3R~}t=mc`5<3lg7585oXQOm9kMsp`Ds2)y`YHI_UR1KUn<$JMg~WS*yE z`YGomzbe?8`HWxYJ_~Li^!|Q*-aM+puqNU-IE_v5oj_s3K%hR-;qTCrb zDkLNPas82Wa%Etql~Zsoy!FPR_CK;73ms`IGn?@=_V%&cHbj4LG&hIN0k4@IaOs|V zCI4c(&4{xl2nDAFTB>Kf#Tlcb%DT(ht}D5*6F1yg({OtFe^iQ{!n=g%ql0Z%c=XTd z#S9nIM2}81l0IAgO*wt_;-b5SrkNRag}RHs{NQrfqju5q*=JM!DKjRD(_hR}7pP+% zeBh1|>Sky%p&CBacKQf&-(ns}?OEzL5FEKoXC z3oOeAI7;W;@OIf{bC1Pi>PY0N;fZV8hW^kCdc41Uw)-?tVH_JPtA>JIel&5Mh zkx?qOmfYj#OgL+z!1R~8ontv7AW#qDe6II~yXJW*RalK-gTsOP(FOj4_9(-eOK3BdL9khJ)Ov>KonzO(4}CtW6=j>WLV(~ zM7ZL>a|hqsX_c^5BL;au2T;}0va|1!mNq7%4}m{=;()XIA187N#fmnVRd7YV3C+A|kdk(a~+ex*759bw4{_-7?00trjXd zHQVIL_%*3(*Cy%y6e~Y}op?f`@eM>|*)*Y?Sa+aH9R*_X<43!fYDazjw%59u=4Wqx z{AeZOYq_7q6kQf{M&93}f>g870sRa;Vgzs_rhJA_5yO&@YFgX{jetLd+06;^9CdXy zCZ%_+GC+MIT8q{j>9yPDeYyIycT0V3ux%UX=x5Apbuf%);jnGrY;mi+uX@-un8w z+}f!ZIxBzv><454d@W_vz1Z-&KcmUByZHQUS-~Q1Lq0#E^~Ahy=1mxIUl3th%(H(hm6UI)3)(_>y?0$1Bw9Qx-$qh|6 zx15=++ID(&TwV>4=UjWBlg`rI0jG=RCu4B2Vx=g}SYMBxJ`y-hTxfyv%EQYQH1%M2 z!aj-e+qahELJz7N8(9?;=#P7?Rys~tk0{dc8~(U6F|n8XAj25+%S01at4yh;CXD48 zGAK#E`lVia7hQ{zu>$~W{MyBjK$8W-~BC4GhL%h?h#g>N z=>rMO)Z##3Lb}vdw7?!3y$U*dy0H*FTEmGN|M`469-EGC=MtLooQ^Afdiexq67+0S zhaY|WoSdG+zWBJs>e|oS&FYlrj!5#j8q|DlW9ND4m(BQ9xNWE0e1CgyPXNG*Pme`Y zinAQ!S&NHz*cvqpnmq{PT>cBNbwyywchfw3OU3y2_q(SyGviw5R(mQFq?aKx_nzlX zE_l1+F4H%wR2v14YulP__dp198!i|7p2+mDhC9P=eqnI;r^n(+$5TjeeLJK5S|sPG zX+Y!t)*PpzijAQBhmyLR&d1pQ-r9$Pm$a_S_Vk)TM_2C_`f~>W%~W@EGHX7xqgaq$ zDAfEZe5m$H`;2g26bMXHf-Eu=VOU^mxM(z)nOG+DF<>R`A zw+4KIqf_(0f=fL&6fL&1@y&Oc2|sAJwVm=w4~pBg>u3ApTSo1=I!o=a$5V|Wul3f< zxMk;B&-76P^V}<^eer*JoOt(w&OR#o;(rkA2>?KBuVEyFyXfI>YsuF zN|0u6Vrd(RWv}1*x;r+e#OhT>Hj@^5N$nJyoZ{G+1`4lInVt>Kb~lZ=7IB)9PA!YK z6>1kQP(ghMf@JtsDxT^uZo_RZyB3O4UfuaVxZ%me-@Fxl5c@*gI}R#F_Y*09z9kO1 z$j>nSu)HDfV~^F)ctS!j@!3~+?Z$}W_zeRbOAu!q9P{90w}gx%v&#CH^WTfP_M6Eb zKKwK?#&sJ`L~6FZUtApchq~Cg^GWPw4Ii5H?kc(S_sqF1`V)$+efezF%IsdKyj zZv07nHZ-QG+SrT_9cOu7tWp*v_8ttbOWh^x9qF30;DD5IAlOb=i-idupTKk#)YR_S5$7Vw4c>R$ISmx!qOqHsSi{ znvX15qT}sWfo0jN4i%FQ*V3(36zb&5>laU{Ki|#AuVFfQb<3+pv7C6$R^|G~cNdF1 z$|%D`GquGEs??A9r9DO#s;-Sbw zY5Ki8@|Z1Ns4}w2hk5XAPm~h;PB(oZ_y?JDPzMN$M)?+0V`jdmhLb2 zGN6T{fD}r->U%N)?r``cdZJGFas|SA%5~|BW$_>_U6?RopX>NN6jWKuK&Z{b*2tXp z-fwJaQ9=I$vXI9VZ>0KhyIv8rxRswj-y#?0PITm&gdSdlcCE$UyGBu-gpfXgFm7jX zEPAX30j;@;0naBiJ zqdKbGQRo~f?+FATEkw-O;o<%E%GC3RJvSD+(N*rQMRP>Q%zRi!_p&-R9U)j|S@jW~ zk;vH#V(RcKx-mupW$TuF{i8wi()O;<0u!0mc6~5kr~tdN_H>ZdzO?g)VIn|_rdBHZ z461?T;ZlOb3SsU66hf}7VEyzOKckG}O zd_4y<)g9Y*v7WdN55;RlKMX51ES+4v!pIlG4BUx~P-tIuRFy@ZeXOkd#e1I#O_ux;UICtgHw31TD9gj-P)-7O$Ho}jQ-9!ikg5?dp< z!ex6wQUT1G~ zg8kzE(#q*NVzfc9{~vo8tgQaChtbxig%OykwY8)oQ+S5UV6nf+I@cBw)Ou}`%~h|K z*8)+8&WVkEZ_#hKyxXwsU0R#u!uF$zaVV?YwX0iF$>y zp_Z22*SdmSI{Eh}nJ ze$>-QN=u*1&=*BPKF7D`e~6EvQBiyC*5Q!t;NWn=)O48k>Pd4!K|uwm;pH~h9jpBF z@nqhHqalQN6#1oa>==U)9$Ys_q+!y%7uJ=pu@M=70yiAxV5md^UuM+=>R21JP z+0izbV%{@@SR1{3cO4AA@54J9nFql!7(T*8;6p`({`F;LWvL=%ft78W%YZ%q19k4% z#l^c&O5=ZUSo(FT>|x?Hn221tvLD}?Scby~#DS*R~Li;C-+7{o8yPv+S{sBZSJKk5@ zN@!F>KEC64y#=m)+d%LP_+}|7C1s8f3fQjP6C+|8iQJ97dzD-gLQ{ERg^sC{${gCA z?}ByPw^LIq+t{QR7-7co^(A$6a|@2%^Aj_T)K)WCFtrWTAA&nKwBx7yjzK;t7@K=^ z|Nbp-^Y!h4@zZhO-H}3EN*1u`vg-c)Irz~I*;ITRPv{$cJ+_}1Uz~?F9Y-;FP80U+ zkd!d8t_{#bRF2;);z!xl$EOfGK>-_#^7}X*@at_|j_kgvT8G07v`wyTD<$A|LbYgC z?Z1QTvTDFFSy`iYfWY6EqpYYQ0lzm~%tjIv9?tG)p&|-Brg7OlSimHfmmAx2;Kt@R zDSy@{le`9Y>p|CeDiX+2>)*>(W7tQzqr@I~Dj)0Eoyp0`$FFtqtH1b2*Bl-+t>)n& zO=Jc;_8<3Hvc)R?BC#N^dxe~0L|qv2VrfmN^h1m1FaA4G{~aq6Tyob3OM`L6_ZQtD zV-9=S_7@00VQ2Xg<>X~OhlptY$bgH0bsyEau#e~*+z?D{hbd3lc; z^jutiyE#l!H#Rvo_w6_58D(yC*B$F<2{}kY0=#52 zFSV;ZWA$jnImSKZyfY^T4-!tHn_TARrL|}NZJvSx;v#!?K+BJh0h(PxEbKfwgYXaL z+tr>l!YN)k<*=*}QL(upvyY`oxmnc^PG*G3R>Adw!+d)jF_!jx`I0-lzE9Asnmj`< zmlU=H-F0xdBz&d`QZkCi%9UHv($a*@nq^!3=Q`qbfbS=!)ScFY0o5ozgzG2_z2I$= zKZjrx3Gg|vSmC;=766i_oBHX~N_h9hoFR%`SzT3xO6%V_<7Q-J)YI3eq-4EfX<5_J zake&tngII=rx%o7PM9HF%rfjabae*_$j5b4eT5^KxQPipM$ke-KE`%?XlUp#iaqji z>@qSkLdQdX@RN?t%m~EPZUP%3=pM~zE1GTB|DhwxjuKo-9lgDWt>)M7Q(wAt=}F?$ z*RR|(@uH<^mTvDeTJS%Q@Q5@0`R}oIDxKSpbK8Jvd%Iq6%PNU?=o{ zD|nbbT;1^K)BrtM^=fTHP&jOf(4E}L%$#cAmY}{qlyw&a(K9UFc(d~J$+To|?fdg* zvA3tqd++i7pF7k1{af8upe|ln_Mu^vWZ@DL*|8&T>=NVA-zS_FR^kjEEoKC0wG60! z+9OU%vbSQ-_9t}pYMnh!!ufkhI7J_rHlDsS$|Fq4WG;I|i2Bf>gI3l*J&$3B{8Zqk z*uTOnvRh(}`rl8?L=F5k!LW`cSedM9iU#P7c;Hah?NceN6Y zEQl90HWpBdOS??hh6t25C~mh;y{Z047~Y<#W%OYT)Vdi zM0G)|NcXX@&|GQ_sD-x)fUNf}LteZw&mF-Lhv}hTY;0`Mm&ha`XaixJmFu(0GiN^6L6DzgT185iCu9iUTfH)OQTQYh{3A3@KlkWwlZv>q z{Ob$x77@hSlGQ%Urr+x_z_o(h@*bUGK6IDn-|(yCc7j z4}J855yD#%-VanSbPfe=BPTb7Mv}i=RXLdt<~*2zv`tI|g5V!-(ehNZ#@jAcTz1)! zuzeKc+I`a7yF78~=g-EmD6{it&)$*sSo*pey*>tPmc4~ zu~>SPZl@uB+tyr1C#Qv&FBb{&+1JL0z8;MOP%{Gt-iE%xKP~O&c1typ0>dJQe+60( zIJAtctjY%~R~R4@Xfd(iK7aV}pd1b?m}_#lfNSQKlvD7ZEFBgnE+{l(GJ*Nl*orGx zGF>u+sS}?Lar5v@l!hVK6RIgUXy2&w%Ij?NIVe-j zUzurx?1#&FKc9I|e}5?I2f~!%{a__6D9wDBMs|VrUPuVxa294%>PiRZ8_@98+tC*0 z=Dsm8`_O)>vAoXAB&Jfd5CI~VN*BJo?%*@P~K@OSg2DE(mK#8-af`=16*~H1i zG9%6#r9c7-YBUyi(PBa3Q;p0R|Im6dbUon3+4Nyv;<{>~hf#@WZ7yHFY-NFO4p-=L z7en&ZEEB%Rr>-+LBqDT$vX@`+R6zRpUt_MwO4L|b!BGfEOB=Mu!ypv>bHK(praLUh zm87>kglq;+3rtKtm~DO$`3rx%$vrk;Eo4bFhMvp2+5+YlFPM)cB(Rlg`0zo|)HJau zn2u!lQ`$L5diK^L2fCHZXe=f&l8y&FGCp+^;=EOKh-!9f6NO{2%wXJu5knwo8YpT? zSBTcKu%JLO?fgy8k9=oMOd>EJPxKegEiS%Kzwo+FW5j`r*$AY}Fht?ut0+eyj{sO0^o`8L+4aC$p>&TGafUnBo+^?7pI@VnUIJK-9gV2 z>fo$wp>^Q`14`;4LcL=SdM4Ku5l@_CVi+I}CSDcyHCW&gmnEnf-xghx&*ILYf?>CB zOmOfHG25QX+&Mf9$|)VgIO7o|Ks2GlD!Dy;Dq-L+}-zTtawA)UQ|?6+12G-+4LK3!604&!O71>6(Fae znAr1x`83F{ID>ZxY=l^tp|z>cYm=jb4=`Qc0VQbeb3FgtBmzx(lWTFqt3|NCr`M9H zG8e{z(;srVuQ!Vmi+~L5A}FuTHcr}nd;VQ-t#Kitq0}%5OL_NB|CP!ABpRZNCypJE zE0_+qH1Nu(V!)*8MY+P*Bcrt*h+;*YponEKjG$X#BCLhEEXDo;#);EIa3Zf6uiwYK z9$G4&`$>okq<_oC%4&MKcBEKD3I>VuY0I?{T;)8ZF_kfEeu3m^yz<3W)$pTJ3?j~xqN z-WjC62hDQT-;K5W<%#Ls!oqrVnJN`L=L`&vSVerBd#oIgQBa^eBS^h7^ZWO!kZzqA zXO<;FXd>*GVet9L!ZOW-;Py^tkv z>^__Tcwb^7W?QT{61g8rX2{NN4?j6cf{2sPG46v}Y3BiyZH1q7CsT-P1nvab-(XJJ za_(FwUF-dz07b-=5QN?!)VbRC@8`Uk={mL(@-`9z>1J$>ti3rF48>Y<_t5s0dSzM$wF;1Rl< za7wx`|2AWp(06aX@4pnaX|7dUF6sPTWZXH1y|yka(BZFV<9+@kfWARJHCK;P`FB>9 z0~u)l{Q&=w{9F%22B{nN?}}h2*i>B^RsRE@+R2EEGxBQ1D{E+k27T$!zH#%WBHZ)6 z*XPviG}6wScV`*7P8W|;7+%(c@12;~ULq9{HbW$dDJf*2qt&2=2|Vbcl#BB~2sV+- z(@O+6;?F>MZQnFE{b*)yA3>1kBqcAgAl+H%`t|7`22~y|t{aV)g&bOND^>$m9Vw7tvzkaiYC(+@ja(re5}(qFi>d@Qoq=COD9P? z=+Ta|LkPXuYR7)A!%FAUrC^x(DK!7X8@*BJV4#|>l8t#9My}n%h&eI-XB2moO7lJHySO1RyJXr$ zM@1#)=RY35O7jDgD1BstW*E1(6Pz=1HQYUr|sYd-b0W>yEm|TL>fx zGw0lv2ubwEhtc>TdxAiQ+*)8f4Fa+RkQe0!jnu3xO#+_Sdx$=H?kJ?6?P8*$2w+M< zbv%pwhw5^#HG*F_{}jqvKr|`BfBjR%aVqHyhlIt%>HvV^RSF6?jS;dnUj0QYG!6hoMHfD2>j+@J1voAfqNDN!uI}* z3W+-!JWALr@8#Dcp`vQNpeUj*Dk5TvwR>-_;{%GNeegXhlwZgql+Bsj1L=5qdk881{%?QW2&A~9w~i7XD?AYp#J*e%1FS;Y2o z+gZc@+p*%;0xw^p=e z(X(gN7-dms?nSfie1eDj0LUCqo;>N*!^INLAs^n;)3ZgO{>v98xWoH4J_gaj0k6wh zStbV`FcJ`R9zx*C_xg=1yxCBr3GN{=J>5e_EqEYsElUk!lXc$%rs?<=JR zy4?`$7Gd3Zv)HgULzN@#{5_&4MuP`ybprB61L(90H!Tzzg5E&jL^w93rKNT2N^Ldo zVFLC_&lV>eTtt=T4>?FVMgK*H)`KJwI0Q`P0Sg2_`?3PpP0r}1t?wEC=X^!u!g@&(J>_fCmpVt*5pd&7M46i?7sCke>h##_v`;C`-uSxtj z4j`_BPBO3KHOQ&x3g~K|>m0(i#}tpFj;;tZUhnN$lCZwH7frqR8n+rT<3e$~^#<;0 zh^r%)OM<#IRyc%1h&J3paLfD0PZ1v$l_V~%&B4LeIzQ^NzKw?VayeB*W;?+r#?^^+&> z=x28IS>h}b_L;EpRFjZS&do#y0Pg8MwYbP~E;sIu6k?Xbi*IdhjQ|y8B_+Q>In2mN z2?n!f^wzfIvst(=iS5vza>N#}ptEB*bOfj;VTm_fcv0M+@^K+ta(6x705^G;i% zrm_XqDyWtACKydgLjR zb4~`qubKnbPJMiwp@%P4@GZN7dOx=G-6N?)5*dEppOTz+nD5L;6> zHa3F$?d+*WpSy zA;9)=i}>~ab~ihOXJRFnrVtmg*k7NkEfFd45B@_J87ahMke;5N9ffBNrSNfD?a=Fl=P;3n|FY?^$j zTyr9&!}OmK!4F!(v34i=_sBQy4(Ql%<4W9`*kg0;n#a^QULN?iR}T()e|>Nu*$l%X z;eXYSaHao){83Ll<)!P;d!hT9NS*+?r9vBxWj`(fNgwk6Jvwwu{8%3VWhPX7GGtv3 zmAUSKCk2q6O1TSX%P>7RSEb2M*4-4<0HD@^V-VW|xT4{J215#4?fWxW!!0=cg zkT1@txVWP_Viq9}AJQY@=`7*nyPb;44{-9Jx691ndqNWd4FgOUE?9E8PsBoHkZ7c< z%cH{+a1E#ExRB7t0xUiVaMoOP>>+Xj-@>>O*Z)t4cNx-`{|DGK3ET zr;d#o>F7LY^@_#>z2shoy-2*u50AZRukJ{Dt-fs?M--tSI?#KM=)!oADB?>zv_y~y z`dFgTjq}ha2K0%vS5AM!m6dBUQAP|3Y^xwHa5Ba}Fkza$sy!AZ${f}z7v86t9ErrP zh^J4Pu_Y?rPuSao*Bd?e^Hl;uL6pZAfj5jjzG(c97x<;aDmp&?m{ng-&#d%m5!DyZ zpC`U}p+YFH(_$}bz#L!Rp%Mnn7Vt}&65PowUNxq;KfNKAQ zjtjT(bK{T`@n;r8Vn47x z%2-kP?$+dS>yO-i?tguQg4FplELD`b+5h*ppLTsr1PkGVygAG*U0s=*g6g`DT7F_8 zXI%1Ggemrvdr60QM9_iD3HXzt7M%jreDWt==0^{ zHjDTPe0%UpyJ_qf!pcY^=C}eadK=n9v2(96sGxxO#=un?!!(bM>g7~-b{5<>(H5N3 zH2{%jbYVdMk2Zj@V9c}!PD zWh?PGVcb;cZ)n&~_#86L*lY)KhZb#XZv5H_E@p4SyRSeoshx>Tfvg)YE=DEJgrG** z!=q$a^#9}Py#spQ-~WG`N|cd?)sV6!StTk_LWEjW6cBsczbY;kmk!s9JhtzioM)Q+j0Q6`?(4a+ordssNOmku?$QCa zrVf9{#VrBuR$#d(b<~YxejcX8BFd4Z+lQy;w*s>$`z9MTBE5gDHfIf|dAzQ+HHGF- zujAw+I+H4!IqubtcP9e1db%hU9n#SOm?|p@w+Yj@(UImZ7)_YR*dUj%ef%>{ZrbH+?S#DCaZL>(La4noQZ8e4IW?c^Mv8FS?2z!{pvBg$*U&!*AV zqa^X(Ys0UAehQ~;ak?F&` zg{eFuUs!Lp)q&~%n{z)Zh#+M)Z_Icb z(`mFHz=UbJ`P2CuFDR^L4=)EQXV@pAj9#Z^C+ zN8~X#h+%IA3~uGx*B4lWRZ`&8v{559V;w#1kvw(%%SS`n8I3gAAlZ-9g6CD`&$5`D zO1~lTx|Hqx%bkY3Bw)6Fbky%4OaR$NFOX*6)ykS*0_6ozKYb>wrjfC6&f~c+GByzc zc=&U!)2NyJxBgyTb=AX!5|ISsxqcTRnY(+>e_m2zG;)lMP0=t|%nQ}O*2of<7ueqD za+7X<-CuLq6o1Yl_}*&Fm}XDce6(20IuRg{zieZ29_PVW%c>MqSD*Q9#r^+j3Wrx+ zDIro>J-BaDRgI@S`opT7HOqT!>g-I3qWCH|DbcJtFy_cpyPJ6YN0ixLcz+;6&2 zr5EV&`}v0Z=L|(EFEnA!Ep0%Wx9^#*{AjX~8a%rK*ax)Q^K$rB z$s%V?TaTx!5d&QLXXHl`Ez5y^77? z&Yuewz(WmxVF%N+c5RBN=OwaC0d}-d51h=!dj+&}`G@ZhA3b{Uy?9@r66YZPA~DLD zwu?Uy#1&KAZ=YhHSpa$K%1m|{k|T=vFd}cmh7F(Z4)0#p-)y7negLa_3mH84Ih#XO z)m6iUYgsdm-OoKr5|k)(`X6!J&rD(V2P~* zy$|UBDjN5P=azRy4K}|;%a*Sdo&Z3K=MYUgr$+u+_xSyS#z|FeR9Yz4hl$wl zH0Y~j!VMqZ+_1_uSTN2wSKIHuSQvSY`NLPn{YgACm@>dx(J*R)z?4WAWae^6X8!@H;Y$R$hI z!`;WDj*ZtR^P-(+sk#`4=}^^g=biw@?AyP;@~ndg59-Z!eu(ap&7t)YoDPrNwEbpZ z1Hh1lbXq7j&Ut8zs=q96BDu5f?XqWum`-~6&p(5uh>kfjw$!RCvBo^&>mSm|cy#SD zBsYvvBI2i+%Ydt&DaR%w$Z=9I7%`A;%x`2-^Bzv4k%+-N1}mP&-P<;5TEQH93+aPC{drY zdoC$CxsXGfrxicf)gv+s$M2TyjdJ=vK-371PTSj0w@d(u>0aME0O>88)~k1KSqhav zMTzM7(q_YbzLTeGaF^eEoccystkK}~JUU~e*Ry2)52y{+ zr{d(zYcT8zcPyQCwncesEW!2h=u^1qe*`<#Nc4NMo)Mo6J*uhFjEHrNnkZhFzdcTX;M_Vl!{x3_0M&RR+C zH(zZcq0Qn#M{;Bvzx_0UGcV%eFI?sq5e<~9>2CGg4|*s#CgEgf&w9wy1pwXApZih- zF9#@&k;N{*JBQ&%#=cQh-dXeK%^NH$9RCF+82$8Jfyt~T!+Jy*paX!l$}C_>I^>vq zX0Cb*=@SLS2~~aFTvOA?wUvfOG~Cx-m&{-Sl;5_{o({;h?evjR7F%-YlU0;ogm{T? zicLZFdV&j*W1`j*f{Lmz-I>TSv^ z1|pG37kg~c1FgwpLpb)K@j%pKu?v_ma`k&cS82&n&N$K z`?qJlHUF8}{HY_nYQwW>D1i=l#Qvr04+1J8%Ge`E)TQ)t82fKu^^7CS-PQV&iTuXY ziNWt!@9m+oc-s(eA9Sq8Zr57}GI0?m+~)e`#=-wjgS@Kd2)`&P5vXr)7rIe`G`n+R zVAcvx=S{OlsR({m`mJ-PPBQ*|;sTLHBAs@)`Sj}D#iHGy?y#N4&C@wm4dm|cqN_WY z9}yNgBrxm7ccZ{2eD3S1IfNP{49f?BaIa=#T;oJF&C%>KPzI%1zl;5O!}1$J5zioT zVf{lC7fXJATgaxzQUX~ZH`v_7 zq}gwfR(l)3$YF-)WgWTv5qR$}ver@c$DI^I1 zXGq-INmf!G0Jm&GKOE^HGmhTuVa|d7V$U!m?9lSxuBNUkpeQV!_2ZrP2NiZq{iy(B zbUGQRVz;5+0X-`z{ru?@PTo|XjqSWNS;8j`g%`?`(Zzi&YutW%vvbps(!tHvYPa@N zZ8Umi-OGiD2@NJwUx&BbwH}FPS_3>Vsp(P;_-Ct$%y`zDc@%Ei7cAx zQen{mgu$HT9-;6uvLgl@c6xZaPmQ}aSvHo%I%zKMBGW24x%GR3_<^&Z*rlOF_a~l{ z^$!`L*_NY+&%nmC8FLfCU-c7kexMuUoQqyn#yaQ|`h?PNaYv74u)2(J_m6O-s|zBl zCF4uj|Ng%AN4AqGk1Vu)0wm*`7DM(&<$YaK0FI_;CysBUWoV|fXh!f%W7+=OrcIlP zpVr&@LPbOrl|~j%o29Izlw+K`J1A)X*%yM;Tjif7{#dhJ! z<4t22kl(bpwly&DpCx8x43Fve?~hX}TE6EgypT-!Lbb0V^2*%YoToXn4iPDX&qL7f zwP_^R^99fI7+h=pmVdSAXso9lwtM&PPi1-2^?A07Pt*5dzTNtFohKIM3oXh$ZA>dY zBHym|@Yo!BtwcPyqXvGrIqcZK*2Bk-i+o#66DBA=?M~3SHRavF2YO}KJLtd1oqf5I7&Ws7%y#Ns7O)OD?+KOX~jA-F|NC{{cj z`eST8=QEy4^=8SKJ{#gbdOA}?82_FJ_IY{{4Q=0%TU=Wwt$433<{STOgzeH+gB89o zfSMf1i78@*B;Lz}_G}>XlM82Vm;W7HfQ7|C+Cq|kKATphDjMZvuP2cIZO9RkS}gsr zFWTX(FBsxgMK`9Wy;?eA#h1nBE;Xcyf4u|ElotH6H;6N9kbix4(9-=scC~2Ud;@o+ z@+_cn+O|F__qNsMFwhS7-#+mJ(Q(U{FRwev4{!4A{!0m#i?B=2N<@7-gV{_0M$o_> zcQOdhJj;i|%%(C6f!LgcF?!KGZ|CLxai?4N?nBW(Q#5OFr|b9cubozLrMoy*7}s9E z&!hTr(-w(4XXM_#QBFZCkGii|p>b+lr~66$Z?`+jKXnn#Pz_PY^Mt28@JXy+QIkOP z`c3k{Z&RjkYCQPAV&?6j&aUUro%`l7rF#Pu0OT(YQj@X|ivm7vyPX;nG3z43S{H{o z`#m&n9gB-gr&eFFn-jpdS=rk9wby|2XV0GCyk-_#?iLppZ`pp;NCY!t7f{ev2;IRn&$4lZ!EO87 zy?Jzk6=H&mf7Mk?!hal_R6lu2w=v^?oIZ6*bXkPmmoLUzfi;Wm=E=I=vwK^WE~|AY%J1 zt*0G0XPA3$7m%kc+$;4GpxG!;E2nri$?Zc`in#2_#SKNr@)I5^C6rD5n|^0K2&-0> z&jmN7HGLrux&Aiu^$6uk-XpD8f+8;E>+Jy*0P9G}dS4FToDS-OP;&jQ3$t41USm6D z@r3yO``7ut-Z(Mas(I>&i-QdOBHzZqaUidFT2^p&K)-%iPI4|%F#nZZ>CL`(9_U+6 zoOH%1(;TY$lXf`6LUv_3ItKY`jh?_z;AADvKbCwDh!HX0tD->H5AaA*YLj{_Ku=unV| zw?fh4@o@m}iY^{c@h2i_U6z^BEa%}k08eh+k}18Wz|S0V2p8PYrpOxH;>?2l^+2?P z3wVF_m@Rx$_I>dGfMLDps4@Iag3IM|26_HtpL-B;H@)9T>jPFY!``?%bf9=QnlZH{ z1J;%I-D*mDnZf%sm74zy?30o(%*HhFU5kPbTW9mI$--b7y!Kun3i^>ywJEfv>OrO< zhdpK^q=>-)US9Wf5ke@o_9)Hjv*@V}DyX1;TuXjy5f&}#?vJosf8fAr2aaA<^F=_8 z@?33zS7qN&xHB!eOAxw>k|6{AJIhK-gRi+T`iO5apkw4J#2KG0=x7lrWXjH)k7?Cy zfqC@zzP|c)qD$(jQ)3S}*ly%4rg|&MG3;MI)wSezNa&ZZh8uFen#ULaus1NHd+qi4 z98d|RUv&zC4+ug9h8CRoIs)0w*s-DQSFp)6vgw$8XyWO{y7-d=hNEWefq+;`;9&0L z^%Kxv)0SSba}7XMrEr&Mv}{>lMMXs#>#OPM);)}4>Tkrj8-X4gQ|8LrBAT!vH;B)F4q{(H6@;U~<<%r&ptwixjj{}cX1$!5Ip^he z1HEIAgW}6lJ9*}eEy}uKT1wy&MT`+rrGx0k%2)S5fx{)KXf+_#mqp&_fXbsrw80XZ z4ei?!s?QmMEzr|$2Y4f{*|FotE5D#1VRdT`()a%WZ<+lWH3417V6XczvA>%+u_qi6cTgBsNQloYIKt=xen_2`maTMyWxVG3Wa+4im;20vkIS zLMD2T*x}jHZd1BnyTiK=<)JhS$FDGei-BI(pCMonJJpcuhZZ5eLBeOXeEDg!o#g(K zSJQL#uR%@4Ab5CHJ|F&&>wXQ~usFza<8N)<-~zZ*MUT65jhjyIHn}c~i;cYqd!kUB;EUTFGD>u8 zNOAbtsquSu@0Py_r(4jpnO3dema5d$scO?tyX`0``bc8|&&0D7>u&i^TRu_jT{c68 zeAe9Px)+o8!?TO_J-=XDi(kJmYGd0MyLy6xjHUUP9;-N#Q=%HV`juAOegD^W=bS)l z%t{7r5}19MJ_i2e54KjQfmq-+(ur!!NI0yWqS>DKpJw}-SIu*9xA1E1Hn19hC!8Ag zu7zv8RURI*&>j9yb>Z1)edB#rQG=#T*?ZgP9229pr>AGyN8lK7D`nsj3fjBZ{J?7M z{s2nQlOv1W&1JR4=A>f>MzS?i4rF`0oc)l?qLHTYKZ~0B5?+?a4_r;J-n!MYr~aF4 zP!u8UQ`ixNLd(p>q$*E`oW+ENkIe)%!bqz;w$uGabNi>zI@A}H5ega$2)db^BmWKt z1`O|z*bg!4E}JDdo{QCixZ-oMi#$6tbIPtdM~buPL$CX}KIcBEw-AnGL)QYm9rnWU z4;BGEcl8Scn+7BPU7?DMuwyHo`qbv0z{-QWaH;|E_H&mq2dT`W6Z=>;NpC)JL#BNh1pgyR2?JXV}KlxPaYHzb@dWgOOJ&KKDdLmQ-kUb>tC{sC# z&(;VS(bS{($Weop+_WOpZV?%cr5~ zX{4%p@`#sZO0u{8%F}PUHnS9P2cxury3cv$mTkWI11=-< z)LuveTP zD^`{Ne1fV04Q@+&6d!ip-j+>T_gx1kpYIG9nZ^A%W8uPMgA#*hSM15O4YSE)j_nYD zZbQg<-@Ch-M2y^6T3V{B^R6EN)@i!nj zoD8-Z*d;M|Ua0&soTs`pIZ$#*6n_oAiKyB>OYQYcItbzqYUYEM?-DFcz&T4^!c$XB zg5;oxd)cuZ9FoSi7G1J$$p2OQ;N;iu-(LYE_)NBo;+vr!8#HXVu>cK*HfTr%zoG&* zt`U6Ljub0VcDfhiLT1eP+L33>Il783_3Z4LpBo%-a8muIlT{8n6a1nwW^%6vKYxnr zlF7-a&BlNkSdys{E!{A<_FYz@rsKBzFF>^G+UEwY`cw-+ZgcPExF^<}L!G&{kK&PV&Y^>uTM>3&>l{@HqEN5W2{P6be zChW;nKDfF!XfPl`3pXsyMqRp$HB<}ll5YE+c6ZRALF=sMl`JPI;rebnfJ)=$nGVB3 zd;pXiscz}orAv$XgND7*#SWp_v~lC-0AR`od#a);>$~dH{!c4MP)3k@WLq?}H=WXH zhy8U#kCZdITSCHHs>qoP#-`DNi9h1hmCvz*% zUrYO8MS%u(iSCC`I9E1q79I|^JaYjFe~9sQ-dBBmmbOiHoKonl7cGi^w**#7pl5kK zVPRo8OVx93*oF82e%*NWAg%1d+@MJ6+18ZzX(j0!|_Y*5DoEIBj&w-zrweJ3-PWrBhhx> zk{_k>(EV7hoXmG0opf56B-?!_P+7oCX!bPb@1%D628w9Xc9f3FgulnxXSJ_6+s&QZ zcHUO=BXj(}CY)UR9c;QZ^T30BhF8kV%Lx-r&7Qh=+s%!rjE|a_E_g>Q!QhL&M+bv~U5S7iS(?29Dy|59|F+ zI)Mfa8nm$Ie8(7f>C(8<{p{_9KY#ulJ#E_Y8G71HXbWW$Do?Ancl5^UTd!3v5gj2%^6oO)SDdfCJ2-3t7tSjjRK9{dT5l*t6LmuO}zR> zp+3W0Dkhp%9&KiP>r$&JPCeZ%PMZ2mX4*so7rQ;rmF z^tUA8oV!l)VcS z#T(0?u4&2;+m?YWVD~nE2cUQzffdhpTo(It3lQ~7bFWHG>FfxBb=-$x_o;WaCxCwE z(Eq4JZeF^0v6g751XA2rUD=T?^iE!0pOr4>cKrrCTW*er=35q!$&sf>#ijt0%w4w z_iz4&rj`zSoKefb7Lb(!0wrc1=iT4gyQ9wa=pkENwlq@E?Y(D?jk~|9IAZ1V4kjNErjurUJ|^>~xxd}fUC<9AvyOSqac2TF$-3lMA^>i0C1<{E0~^V{jw+ORfW zAL@LqTeoh~>aT`^zoES-N#1gQk4gHWTVV9c9keF%85aKYPlsIS=&UD8f_rLq%e*6% zx|!Z+Do_tI+w>06f856}0NxOV1%?_P{l~rg_k9&-7TneMZc1BtC=9Go-*{9|l^8eo zci39kZzf(Nvqo({*a_5#niu!^1;*-(1##W^Q@=@o6OBIKO&hsSrudc>t|ZE4$-C7e>QK~()U^At3JE% z;C1P`I=f%6v4BN9hQ9I}igf!_W2BQ0PtI-1)MdP9&zDeiw@{9oK~EcDwQrxJ!qalRb()@*{&owXNhDHwA6XVj6ts_ zyo2{~F&Ma_^1|=sF#Xzg<@@={30n16u>aM*2Ocz-SpXLb&Rv9D+K1vr7oiDSo&cUA z1i;o|= zV#T+l$z!t-BA{$cXPEJgjBM*^p|4+;cwm68@jcpvv!dBZl#6{Lr(RgA#Yc=#drS{5 zt3Uz&qY4)?`Y-GMM5X-L)l=xqDfe7Ib(*#{C!sYnD6pG>-R0lxy7)UGqVBkzc78Eu zDZuE=1rHx?@%wfvIS-t8e>{k^_~HOAlnr}XqJF+yk?-7HuY;DChN@+pjlD^w#o1G* ze9-Dy&GVl&f~(5F?(R0)Tzs|BGotgFYinM7%91t(*MCx)~;Q?NSS+IM!jjb$H z<&L?o(MZG?bhRp6;l3e(yt{4ld)Jn>_O4o>C2y8mCePQWuV0;v0+)&oO7^U-{$lfO z7miP6nXG^Isf7Fe`sJ0s`V&-s&z_4m^A6zg=U(=C#7&5O#rB^n13N1(uYs}(gBDU( zkxhUC1SgPXCoO+h7rHe%n3(wZQL$-N-yX+&YxCtO;zO|OsMy$&UN6Jf4RlZU24@;P;m*nAWG%7(GcH`r zJU^#f`}VbjZ3vuK6!<=WCluRr2?KoY%qjb&wJR972RekZES$F5%8 z@A{1!l5Zt+I>fx$>qBY4N0*b}dbmzboO&4=5vJ6%=|=P`i>+VcKv8emviFDG)3zC= zyN~kQ(zvnOQrD}MILA?-u*AeMMz`KN9T_V@NsRu- z;t^NrBClpp;pHBDLSk5m!Rss@0pu%AJ=9Y-!4!+LDGN@{deo?<@!@HHh=1A>!x1a+ z2$=!$u!zhnfbNddQncj$fz-G-74E4PUf#9jonV5Ywg$FR+$%U?!rRYzgPBQGqDOxL za)5|f{UyU6+2_pxB7)z~#c=FQQIcX~lwabuaN&;ojlIA6yOs(X?rdq9wxsvI!H4e^ zay_vuILx!U&Y-blTR>4=nZM&eEYp7el&yFydA@RnZ8uP2LbJL>5D^ku%^l@&d=4;B zlySec6xX)E0U-8`_f)F|*2dux8ylaWkTwj(Sc6*pko3H~Ry-;hK!t@w zos+OJdQbD4GT6q9QoVR@MP32@b&9WbkbGM++BE3FVWIvjomQJ!xMA}G36kl#V$OcD+jcZ+=~TCef;FCPd%?u zU0H0I^Wedu_~~2k80K*(xr}H=U%3Ex)CQdM3{7RH`BZsqjCO6d z-Oc|O)R0_el4{5S29HgA-`3Jg>g{D>E$~KK(H7gawFHq>aps+5PGDj(qOB)?5zPKF z5Kh@?rbc>n72qB&o$h|cZD?6q*jttMvI}w9HzZPD-)Ri#6#=T=A2&IS0C|yuJBjAm zYM$RHwD3U8G|WF9frgWGHa{e*jwz7Tk03OXxd=XHpnh=F^KhbYq|-3$zgKmt@+M6% zp2!X8Sbet#wqfK1WXX0<1r;BEe9AS$Xwz9-FSEZ+M*LD(Tzq&&r=nm3CFHHmk5|{d z2+lHnRnVXRd7oxKH3NFNl z-0$N?f7ehJ`skC4mS45zy@ax?(vV0>zTDO-TXj_XOF_7so>D)QxZI<9b00C;+{=us!rL5|SPTXe_ z)HHFs2Nqo%>WtQ<&C}JzZ6@}LmBhN3m?Gb7Rb}PN7#DmeapiD!&0qA(T2F+TEI>Ma z{CKwm_y*6O{Ud6uODEyvx7&8*%Co`SFYL073GN*TYB$dCre+J2a~MCOR($_5JAZwW z=~_S4*}u>JNoh1bo-gd#E;yvI_YKdrYvqY><7%R$#kZrw$?v@>0p}ifrd_OTU<735 z7B~nhbnST4o9y6;GBs_6x7+2}C}g|WeY7_4+&GRMYmhtV=C0b(1T_mzHd!#}ye?H=FsQ5wt5fK#B=1wvS%D74m}DP^=^*Gz|5BI9(0<(36}kW}~!A^gcbv0aKiQkGhkNt$3LIzNBOjv0nUoCy*jn z7JL8d$Y#bgaTUmxb~?{zu2V+|nxs9oGV-^%sCabeHV_c<q#)xx=hW@CY zPW&Rwtt^j!cnHG~vykV918Yp0f+E5}2KC-}-n)9>zkm|GqDWgD-@bb%0=(I`0vmUo zdT#8Q*n1$UneSIh5Fo9*UM|I+N)){0=UgjG$p@l}`A))~3S1DX#3LDFNWY-T;~WF5 z5oPMq7*s|^K`-7hNs-2u-)8A~d4Sj(md3IsVnRC2cR6f8&p&g@AAcNn3`rRJ`}9K5 z>kk+(86YX^gw_xyZ!0^0FCbI>xIo*Y7SV+qv!M6h9&6Tg4)rU!FJ|Q;*h;W*P zFgsqqtzrlmOjM@&I&=Jo2ldC?#p!1Bc}x7 zI9KJC+W{c($ces7fxFh;)VKvIW2D0ypzryW{cR^PXK^lkA93@{ocH~e%CO-f zKHS8W<=)kOWEdJyUb_~A)x$Fj#cPo{e}Rp&1NDq*;X8@23`3~E=#TfPl(P6F-n**v zakzR;OuiYm> zL3CTmTkwG>VteEveYI!Nr?l$w>P#7d8K!lKcL3;*PkCVZQzUK2Yk6h=9?RW7{mJMd zUmo42xf<}CoI9jlRo&g#SUi6xa7_b3J5BiXgV#$m$tAMk_K!Id>ThqmKgU%%L3XkI zQS#=^-Mo{DiO&lQ|Dp}3e=ldxZm2pg{3X@DJUx#^N7s>vEaG({f0}c5N=%#OuZoxp z-#!&ux4??rsgv8-UVHRF4_7{2-SFAkuYq?05}CL}$c?U~g~L$$Jz!{9|46EfkB1P@ zH8ELw(`*V)PBJL2?P0BjNJ)C(JANshim$S@xyW);5sanZ zzBNe>1e3(lFY(w>>D-8B{ zVb7QEt~$!Y9A=p|w??2}(yz;oCcg!uhi+VZRHxemw^OCBr(Ng?K%miTe#_krf8MFK zbhZl-j-1qUG2n{snKD@R81y6^-U>a@^?HUe2)-{-7!t^WZrr5Fo407VFctia9aZmO zP7M4Dek|E=fQ&XT@7}$8>S}7Wj*b6WP<(rLZxs@;nrS;kSbDdmb{8e~JBz|ZutM}E z+8mc39=5g(P`}%{44$LC2gea@`lgy0=fejPov<r*hC7Es8k)O!qNuU z9K8?C2TBezIJNI1AEoB2ZUxu|ZrD&4+Y3^P1M%a=kH3s|{L)4U^lS)M;>_Gp z^k=*DPiJp39R|0yDMFaLqla1E*ZDj%ynPJwiMA=VdLT(*Oxax=cI_pIY7D`=v7h(0 z3;qu66LQOvtL2hT?~Nb%j2Pchr6}2j_!Su96XhcEW9~ljY@>guzFjj!7k@DlL|wpG z!uZpM#u9E>ER*?Dwjbv6mb?sN&hUF)_`IfBPDOjWW+HH7G*?-EZ2T57iawodte!(} zXmpX&KRat94~Q_2#V-_7L(S;mKnEj%mh*H*s-}uQ(A?i@7P7ZENZkp zYs_~uLOKFVvFgNAPz$LiAU?mmBNi;o2OM(^O>)a!oY?l4g<;Kv;_`IWAB}4MtyIVT zF+1+6b1RP7AJdzy?Ok$7`Qjx5uFDXZAKi_KU}x{g`d^!?sjb5=(Wp01Zq?878ydR~ zC0yZ=LP)rEK|+dDgx!JWH_+5V%e8XDXp*${`vQGP|49%!+{zCLf3WZROenNDnfKmd~b@P+^)j8{2 z>;db1XcN!UxlJ3vmDJ8qRady4GnQ5^UH*?7asC4( zgE*eS!y1(xcK`PDfO=nu;!uLI!FspfYPF2%5)YMsP>`B$h;dfUfevqPQ{*sxdb_q3 z9tOGCkV2JuZT_8NFM*N$Ba$8_5YQy>2L9|%9tPnDuqr4y)9<3Zhif9WXXIgPl9(Oc z+4{>H=~v~!+$*C7`5J+`#Y3r&MW|E7cCE@hI5V}Wy75PTVB*_DAUo(7JY(`m3uVWH z*)Qgp7XZk#Bq0kalxkQ^o-q@M#1o0*vl5F}JI;8!N89HR1Dp&JXOn z^rip13h`c3QT|Dx2LAhXS4UHG5McC$@2_;E@Dgbd{N!Y1D3Th<5*1;18q1)1G6{h; z=y#+s(cX7z(y_bh64CJD#Vqc^nKM-wLDjYY{)yB7ZK%P3D>vW1Q2DF_G(3Kb6xq`%5(ux1AA6Qe6M z`;3iA6u`XIYLD>~CTP;J8ss9)!rDA+hO+a#asG+HT>`s<+{&I1uk!QH0AB|`Y_)1UjICrciANqz^2)`F z>qdVe%>E*Le=K4#t$$K)Z7NI{%faFWw6J#`*jVQnH$taSpOU$L#c4^JO5x4YpDl(w z;R#6Hi=@H%pn?kw=)!R05NXc#F!n{ru8Dn}CZh#yu2;bz&niwPy46u8`=UQiKG>HzZ&L zP#a*Oc!C|vzkSQ3>7OzCO22zHrXKX0k_-)e5XD2cN%Ypd2g)6sMuMF6A27gocj2No z>VaE{*(~fk^5B^8C*OS#h84D}u4qW}JA2!Ry?xb1m@Erk7xpVtym{SzaN#F>Us4&@05bR6{F&oEjLckWMv+vT2HyKc4BP) z!-on&nO;x95?;T3I}==Xb{Rc$3M2pCkH5hpDaf_fVb$Kt4sC^g1C;>R6Z#E0yDk$G z>v>lewfo%&E*_{)l4NlGh7CKnnK_J?>Dnpv*s{iYdM7iy0L101R4-+KPtB||?dm?o zDIcM)cxYxK({k9Q{va=JBNw`4)LBofjs&-mom8YR7zb@FxnFR8 zWmCS_iUszQ1Z)kVO_bJ45N{6X^ta0;#+#%%g)ciUZo9>wG;9(M*eC6~Dy22kZ;*{M zg`w|~7d{KBpC3E>RQti|5r|dLQAr`DWH;jGHhH-MVT?j3N8;DT^mOM5`BZwCk<9-9 zQr}aF;#QWKf~Vx#8=@-ZUf8BuQ-9@DX7;0b;2ECD^({Bhw4@bbWMPs9B`HjmfJ;}P zn`iKGzs!)+JQn`z&Yp{*6VCK8YV()>)h_ip}@xiU8d@wqg z@msoox;N!ARA2e)tFym+`ZQ?PtY}9;rx4rQ>c85R0CMiI+te=heqLS?oO$A$C8k~k zbWeV8^cBcQ;+0jf5$}|VKhH$*0rxr1NHtuWsgFQXHSdb_b|+?C*<5T^N=&CyIY<3_0kf@LjXPQx<2L&@0?l}nGdnwo02N2nWLIx)Ka9_#H# zm{G#I9(S(L8RcS+&evUx>>(@cqIN1bC1$tv0brOu22mjx4K_H)_abNO)Z&JK7j))5wslh zC`m5OKRr*+xmQY)r;i>f+lA1a_?0Z{FYY_;6{{!U2BL`*6~ZI`ft?AW2T5(y--bTAv2>5gn8O{ty z&_Vs9{|G~Q9l&t%{N>97=oYo@DAbbK@5ZDTUtn+{Ev+2`fTDCWpEpUqg|dq!=#`R% zT##mH(wuQFB;s+vrcHxgT}$StZbgjB=PY(MVS!aut^3r^aXVc>U_t_T44hEo0e#d1 z7g2pHxHNk7?Q4_^`mzb%lZmfaIEQ+O>v5$WRESZC5!$F=J2g19MK{oAR1zY`sD zzJ3a5{rhGgl$>awl0$gE2#fL0$DJ2lO#$~S9t=`8EssSsY{+AHN`H@4hR zl-#d#xJLT3MZ8lGE1hJJC27rf$*NUJ`CWgV0tG32HgCp^b;Y%AO+Zuo?#Beo>EPZi z?QLS%SKWje-l{9eN_Y>$SSma5t7>lGle_x4AF|ryYHqS%+M zebV2AKIv6v@>|0vn;Ci9nkEMlNGjQ4DQKAt^RN2&XKv-Ue1~D2na;`rp)$O9jr!@_ zvezLCx@mp7*ZMJKY|oQ|ng4e5AfEj|V#bg6V^&m%G!bvJNat6M z*4#(ceCEz_Hq`VOk5mCEG|D<(4`wErw%ACSqcVK6dkx^jC8nD@3aSh5lkb?EVWe_o zDENXkYX*P^*J0*gD`B0WqCSsMR4m3tn6tT`AsC7DbBTrrII9cB_uC5g$P#p$SfUYC zf$N~N@PQu4adqi4tYxcKK;aIg^JX$_eafG^PwJfKpUz#m`zWrJXeG0R(B|Gyzd`J4 z*8n_FsJSg?#al*-seMB0vy?3yv*UW?Je)%sNE}y@QIItC`v#NKA*w&RgC69kyjuU> zC6QZWw(1;Y5iRU{c`$4JEVyb@qu5BHXT>Wzv z*s?x9-DfunQQG+iLePy-4^HM06kl1#Jw->jM2U_v+fdmK!PVtmTXr?qA}Tp404Amch0$J#Wl6w<*ej0(SX% zk)qw*!faJ#MNh#2tT#4^qJ#UE#N&pR!5Fmi#bh!N>cgu}m z80O_)lk9F-f4_oEW*kQ4E67MQKC0_^IMeWPJ0*VKiAHEb!Yp5iLj_qNW^dlBZfh3g zL=Tyev{*NOXidqLKLdNw-XY4#kOxkFz5o*+JonL*lDeI-$m^?)JV&xW=VKJ@ipflV zy8X~QDovYa9p2}q3{zf9bbY3GD^f-M%vblA@%?L7n5c&jJQmZtOVZO?I4w^&R+Xmy z=#rl4M)fEuDN%C+Kcttx#Es_bQ3Ecd+xGfb%(T<--s@$ouBgQr-qL=E{!Bo(K( z%b@kEKDYzEKQAusm`ihS^3p23Xp>Cq3m?@xJt?J>1i($j-S*qR5*`ppOcZYElR7>c;;E&hL-7RQgY(V z?M~E#>C>xi;<<~gpEU+%G1Zam^Ogql`5pdqbfcR{p5 zm7J}FD5MjL(m-PcA7Bh~97mTi6KvArRrX4rvFA)biC=H>q;J68%;#^zT!7rZg_~dC zvDb5X_EES0?Z_s$i>O9j+!p~*+XK`|OkVzlV0j*}mNVn|hJ?3GPtGnV)-%}(EI0EV z5<(tahh2Tf^3KRe$ITyRAsJL0+Qz7d7F-_g>?fXs{q)YhklZGJktyF8Zv-Q#)vagG zs(p*U(LhQ#W2MN3zeH;CBx&`U^PJ?6t`OzA=fnK`GcQnKLU-&?I*pu(sKnILWQG;# zR3x?R+O#5LTw96+157U@pJi6f(bHC^LrOxt?zrZMXZjpJ8ykJVSr-4yy!t(nek46* z_16}|en*bT0k!A77MWwP{!oRel?T|naMFFolni!#C?u^wfW=im_+DobA2G{ReNi2n z4~1|7P*GP!!ISNc`TKJ`MG0cuO6j_OqCUhITEUppKLJe%v1_0&XH7_)9f8)3|93 zo3dT3b7vJv45shjxYhzFSTqu-nrM$g$Fw^OL!WfUgMzOVQ12I z1RSG16|@xX_9iK49Kh_U?S$15FI{)`WQ*5UY`qVx#AR@AE^|qR#ULc2SH#}6d{AwL zKElqr44H)D%jAI9_uoLg%|(N`^AGB)sfYXC(Y-K=#!SqSQ9HxJI-0gzb6w&o`Yg)i zi1~Vu4NN{4 zPf3L#-}PsO*%(PMm~a0GTL2n8NIyI%q0%B#U{y|&&Oo_gNaRh~j4mFW#rM%yjWHcP z+5#ryz;7UfYmb#5rqB&FPM(Td3C)OK+OM*B@{b;!_|}yZ>$_y(!uKakn&W`kj2|>= z=l1Ozm=$N&30&g0Z%yQMy@Og_4fd5kad2_qbVzWjn_zwibv+W$--xxOclvylS94H# zoj7$WciZy*ZHSLJiUnvMBvkWEzPXXI@)*O$T03TbeS+D6T8Fi)9ze*bZ*45ue5;rO z{4)one|(X!HWe6AheZ08ZrC?PPJ$aFG8<)@=GT;1(zMG3iJ9n$14g`^0L8B6S8v=% zhqleh%j=lS0QH&6|Hko!+bjFr{M2WwzwJ}>cnX2TgaO$gQd_*M-cE9@tq%^VM~O>W6E6hvh@%#kv%%4 zBm;9;SYGUGL%rFU$Y?^ZUucK`x$fl2k9GImBjc4itFZr~G2K{@3E_jgQqgEY1Ygzm zQA36cWD2?F5R3kj{UquPG|XsvCYGEH=6zY)^)3M}r#(3HUt<+@SMi671J|uQvcH~H z|MZssfGvqUe@p`YcseYrUXPN9h#p9e^$is*@1b0aLEMqUMvlys33H%ww)k(NgL{q@ z8;;`3I>nkaO!=Co)NvV7!vM9zQE8)8b>dj)^ITyXLUbRv@bQ^T_wG5x5(F%{$R!^? zwhKP$;(^YQl*{(>-KH~2u&I+HYrB4Y+!aPr-$~N7x3vIx@LPLZtU0RZviJ6@8!@tv zTfeN$%p0??V_O|#!7L*5SNpR2(xA&?l&A+33kIWox65b6x1JnHHBz{OOU3q3LB;EOew31`3;uaE$#Iu*9-3jggW7#3$~y65jJ!KsC2}m>k+4Y;0->naULC5BYpiISr60CW ztJe;BKn9E#zPrU|InzHgn7LpkW|UaI%_#I$8B#XhNXdq>j{F$`Q0P?_H+?YfHqgmi z{MLn{ckXIv=vRDs(@h~tbI?NN5(w+F1Uos`B+9Z#UGHMn91D9~ndM#C5Ek<0#|C=6 zuu^UWI#BuD%3M41{@pwFes>a!6k_Nmt0OWof%3dv`sh_YK*SDr?=HhltW%h-OrnS@GP7$CKiZ2uf$^ z~kF?gVb(0xcx|oyzGc;KYLk?~UpG z#5F}n9r0-|$D=hND})!#+HTN`b(4WuQJxb@prH|)N=z0io!P2==e-kH3C?vL66fG2LVqe)g>C^Ws;} z`Q4N|#`K}V?hpW}IqjUQQg|53JAphc5g#Ca?IZAz!|b(0IG~ws=|&4&G*HR7lbDrP zu2_LPQXNZz`kC!FE!+Va->tCMcZ##YV?1M7bw{2cLK{h$U264k(Q_RMg zb5M=n_}nCh{7rN&d}txJns?CCHilu4l$FJ4 z18NbS;9T&m*rA4M5M2Y5SJZLleY8rqkNlqw$ixX)M2^a|X>HG)J2x%q7-k5h5&Q-! z96tRHd~0;@0|<4_Cinh+tZJx~sF8=;$^Qsz2=j7qu{SD)>;;XdF>ASUSt`gUG zjh4T9;bdd8XjaJAuR0sVOEz?<%4uE=O^kVYKV(iA*)}e?>s_-nt=r_JeK$Aq8c2t3 ztfI1H{T*baj%+x0?BwC}TYy>>abeYGJvf`bFWu1-@s)6;mPxi3R%V)``QQJgodKZ&y zfU0GTjeRUWb!mshmY;w?k~yH!X$85f8J3!A0nlokD+^{ZTl+T48RiN&gPOt@U)%b3 z+FIdfB^k2%7T4YEeJbv3`Oup`Lz^MU*y#{0U`alDpt zw>i_031a5WuQx5{5ho`?w_B^gKevJdb`IT)n2yPt&cDygDW~YFWqe55Iz%u1S$B@t zw5IJ%>ZCiLQDFQj=a6$V5myf)?D1U!dc)_f(kkkUCXl5-VEn<`>4*Qt_#HG!UPOdc z^5KI5<7X6Q$f|h!?3W+Nd*de8TfGoN@xUJ?`V5JGJ!{B%yWjNIuL3lerR7<2Epb&> zFhz@#@A~bHb=IMMo0A2A8O-3q`1DBW$c!HF6nD+KPUnb%%^NB#lt*USx; zo9lKc0V`HVN=E!1D6B^C9+iu-!(SFEf&%9eHk%QqqO!Y+E6*2N~yUC>bkxy{R=0H6dsQi`T zGfz%)C+Mpm6{I8!aE(uVM$y7CdtoS1hC1pD&zp(CNTQ#k%*aZCotHq4_)M9XfmLD< z^g|;=qqO8#+1a~tYd}B-j}>QEyY_${LBnTnJBI&HZk3hU!-=A8*OKo!Sbj<1S17{&W9PS(Ze7|;5%LHi1vl`mU?F#~sO2hgz)3WXK#8}rn54D! zk#dm!AmKZTQ{nxq*jofOiYAhTl0L?~)2 zr$l^utbzXj*g6kzuJ`x?PD4qWN-`Q+l9q;cDh-m*-U=l|k;8P)%Bf7ChO-*x?6=Q`i-_5F5<_vih3-S_=mcMD9KoBZEe-9TzWTeO+0 z#*wM0HXn|Fo>tzU7TeQhCNY()pe=G_l2@h$Oc0dN;75&`H4#)dkubqbS`m8AZQOFL zv@RU02minuBA}k*xT6o|LkY_Fll+Kz19Z-FmnsZ{{`s#OZJUQk(1KC8dZM*LCu*Ym zjbr!EPiTn`4_ctx?xgRuOcKcn?D5VHLHPROw<_QV%}q@b7^JrmY#q{~UAsG)wz*TV z+1;~xjqoudDry@ozRNMMFls+fyQwX{x@T$~{beyPh5W}w)H;3|0hJ5(a;`DNH@B_Y zL2(+*c4d2o$RyPkf+6?Sx)ojFrbO&3!-@l$@Z@x+T~adEeTDkx^tw+LvNC=R%L1S- zi=Jfn(5Wnf5;D_dWrYv}#igb7W2740zd8ds7%nVVtj<}MFtZ}>z#K+`hVB5;DBR8< z_bu(_aa-(;_U*mKAC=g{zIos1A^4CIc?0@SceiPaqRfBJ1+1*ssp7t%AUv9syITey z;pPLS1Oxp1Jg$l8LpU-k$-yXHJMf{SVa>R{0}!lVEDWpkM6 zi_aLpeHgXFPa}5uMu0NlXHIp|;8DEyeT*1OdDi86V~}fY1~ScbK;yR&PXiGoa(ibx z|N7DONKHC3a=0SrKM%emlS)EqcTry}z79oC`9nu|x8qjyJ*8^(XiblHZXPzbJy&O3 zom6y${yD^~jH@mE#S45W$t?%>#;weBBCCU4^WT!P;{r!RPtqTk*OL~#Exb_rguJr> z)oR$@(1YW4CgpCc7@#P)|B%SYF>`x#>o#oKv=)Gpou4|9YKBiQ3H@8#T1;)5xIQ$q zzu##^kFXO*DyGhNcF-_lO67uLsY@8AN3&|isk`GFnrC%Pp91&TU+7G#G&FJ)P-+E z?yW3yD}TcMNB(Giv~@M{YPu^gY0xIOaqG$SZYPq$#~wb1Bra=RCuhrB%|LcgEeaKJ zYNG0E`349x-hTYK!D6_B!vOl<@Oj-;cgi&#_m@veauk|SZaK-~pUZ;m1cU{+4$4Kf z!Se{QwzJ!TxfMJkrM3RqPuR8?uAw!Wk11aHdq!FA*iMPf!1Ym;F^a?vKmh{k7k2jS z1*ETtTK1ntXDwKHvz>CRj|Kql*nn= zaS+zd-(!?4@Pof@Y0F9E)qC|5k#!x`9+HDmxf#`Nhy=d#vtvB@zrC6f)f0+n7xCg( zdjGSF^NkmmOFFsiFK$T7%^IV{-OT1Zr1j{m%bwT`wAEGRQU~YYo817l%3d_oPXpCs z7+V7q&-MmD;>K;gc0zRp_!~Fi(9usw3pPJ#ovSzLK+dNaJ{g73Zg}w~EDuaHO>JS1>K!p-9f7j6@AL}q z?I04x-X-ZUrY2Fe@LUgdV^E^>3e?}piW0IzWqZ$DI1*V`d%IjUO>ik(5D&SSZWpw+ zn#9qOmGrnM93jO2v1*gXc@^cS#MgAu(3tl%F9qbx!BOPh*;mqM7~g$}^$E-Ii2c25 zDk0Vvf15(iWMZ5jNgpiDz{{6^Ph7WZRe~gYfT)w~G0y$koZsO$BSG(6SInFDx6x>K zISznvb&ECyN1r<<8dMeD#u54{g>pNRh!AlzF|n&3`^U%QHK9_t1&%-JzvRuwk2~@9 zc7iLKk?wJ;29RRvx2<@n3Iof|Vldvddv^^rwczxfBaJlxHg0k^TfG^LSf}pEX~z{t zQl`rtFaufr=P;z%KRtgZ@9u{xC!4Tz>Bphf@xcLiCoxJc%zV~B6u4ZgX!&t4?eU!? zfa-Eu+P`sj{Ysl?=NIQeJ4o#J-x; zipo+I(EE-_g)nhGS*WW|DuN~auHa;KdVfJ3U{x zlpg>?$@#dl_QUn)&dAI%a#4n8L;Fh;7c^O}wA{5Vic4?ySQb8CX>1A0r z9USVPg~~g(1b^!o)$yBw6*+WLh)6w}+XKJ~U4_If&5J%0|3BeoKg&pb zpYK0x(t-J5t7zGrdZlO}MXYN{0U)oo`)-q&q#O;694o$t*5mtug!! zm-EA_2hnViqtVh9Ia^0R{B)ACT+UC>^wmoZ!`xz9d5a2jsAnc zRJV#T2QM1;;KN6c;+D^%=#m+y%!!dLT?#i`dck{8ZR#rySd^(-sBFwKQ|Zzj9XT;_ zXy{gY^f*4<$!YoW({gBdo8@vw1!0m`qL_M0V#ohWwn+@r90gz<<=MKnlpQ{ArjSRm@p%C)1 zG~3xHK0dgM#i7BDs_;nUG1UtX%VB9uJluoWo*YlY#LE4~mG5S93zNf)xJ~_?zUS0- zJSoBdSW~%Xy#k+3VQT@a+nck#LQheZpl6|c{FU1!tI)TCJjwF*a&vqp0{$qvD7jDE zTwOu-n*hu!ISr`AI?!Rv5Yh2Av(Ioja>Q8#v6W^DP50m7;rU0>7{1#NIo@d=abN;z zo{kFU=M$mkKX=T9q zqOScj&^(J9#gIWg?DUW&Uf@S{#*TflV5H>^L?_xymn@mu)$+1 z>ojubTTF)kPVFB*o|-p6^%R}BA@LqSU-vS(yCrg-2EX1@cZFt9l+oqz9+`sW|N8lh z1xn3jQUbpLx+yN<=ePlMuA$m%*9tKW5n+dfUFq|KQbL8hOHg93!U_ENV4xj|Her4# z$57FSmUVai5xIL&9Rk#H`nT}lNPPH5^~m4?u%L?sLvkRt;yL)ZqX7#z;{lh%x8ryc z8XA=MS(&Em`jacl`GQ0(`d8A?Ku+lRaz;rfmt_1cq~z^++h+$g$N{$6QAVx zv5!!>5%`&H0f;L-AHXb&m)uD~V(+{Uzl^^@Rr98LZd)mQ` zuL%ml+<7|0P~{!Lq3_&^cKjvS3kS%EE&CgD&HAavK(3Pd=H*KzRxi+~G^o%60Xoght#YTNnJBShFXt5hQK$UR zus^8bG}{{ZN}^+by>3IfTQzLwwB@HAV%=hB-~htVM4pOz9_QEwvDdaEua6(RFCz)d zRse`ylNJhVMEsf9RX{DTPV1%G2F;=D!~=AHZ^tG4XI|&CB-CIzJMKVda#>emOae`u z6E7!#r6pR#a!EXq?SyEmh7XxPv&w)(j~$eLHYgI@M${?Hewms0U!?MEV%@U+488Fp zn4+b+@rj_)=KL4ght<6$9hkcov4e7URxmj6vCfBgW8zNZH>0jLh6n*4lfV?-sNpYW zv#Z3N0DX3$F5ueclX+?MJV6-K+30_1hRU(os4C9p_0QFSa)iuj>8J4}&8yk`$A?FU z<=k0$_yhtMN$HE&P!$hSc4$zGYEHJJ2;3vb`2lN4JX!rBuXFhmluTyRrX_`+*X#v{ z*NO`8`GD3t%>U|odQ-D+S?~b(!6byTU)C+Ua#GM#Y$YIr;87$l7Iz;!Xv%?8BuCJU znZCbVKw_(6#z85#z@BXa^t3I^<+u z-TAdv1Lq5f$p|_8+q6~aicsL1R<&!@>b^Vg=qfd#!Pk9ki@MFqXPdkG0tD2jA+|cS z^Ib7$V^kw0Tr-QOv?$dKP3PdbG11O!T6R}1n+DGjsbvXo=qTb8+$i$w-ckq;OF( z+b|^DaNIi_mT}nZ*=&694Ut||rBUQjI{zvtSr8o)<2LO$|BN7tkQ-^_ znxDbbj_;YX$Z4v$MZs*4HV(RR%;|>AE>TPS{X4r@oDAy2Rmqp{mQmA{MfeiyGU`H( z-aYS~JF%Ewc6tWb!hy)NjXOH&*k^*z-UV0S{A)?jo;SIpqaK}bzny@p1!HN~%5}>K z5oI2c?G7^_o|y4T#FymIwOD@Z13jfHm-Thx?WzLR8uZlaANyu1rE0#I0Xvma zli+$g^y@bx-YR@agj)_6-KBYk-9jN9S{1DtHHMtgmPvq&jL;$qi=(fb->KVoG?qLgif+2nlCfF^34M!zP`1k-i9Dwu!q! zpL7s?O-+A7wi|;oV?}#s$BT#v!K~NAzVhes6Frps5%CXrT5NyH*ZgOgjJbu+zF0;u z)eS}37Sq=jw4&L{>0OEZAw{x@ zE#GRpmE@I+BlOL=Ex~b%_Oa9kzUA}4ElF;?sJDJDH5oZl0(IO$l$xWjZFy>DtFI*` zvJ@e0V^fbhO81a30V$J7^YYSD&aox=FXVXUUu~qWqY&rv&E1|nDCqk~`z3onS!dN! zBA_qh5tG}>Df7SSK0uqklGU)F$+CHVzfTDh zKP{5e$xMPGjpohkaL!yh#hecxmOQMO&woQ~?VaglL8=e(wZ88wIw@#NTlc4bXv-vn zqB$%4s_LA{)crg3?oK~`@2LOookqr1(I+yxyAEiy>@PDu0Gk+dUYoaexRFIwgG;Mw z^p@D^`cZ&M?#%s)SRx5((D8f6vTOx5R`GEllh^wKlv7xGdtwqeA|2ZA2n}AQa&uY@ zF|?5eQ_tV0C25T?518fN+R^lL>QB!!2y>W&$Y>Zt6ML#kBurdFTEMDu#?8vzjOd%m zR9r3vc9L$#+7)1=6N-e7hSYPxyS!)72g=Er*InJ0vW#-w2k~Lt@+;!}9k8a}yLaz! z9BES2S(K$^Zh9Es(#CFdWL}C-;uKeQ0=?Z}zT-h&4&csd@k2da_A6d2K7- z)TH%;vu~#Vbv+?r>zmA5d0+}Dsx!X!d)ozR9>Eq{?rFS%1OP$@v2%4vdAkA7?XN!+ zDC)C#W2@~nkgnFjK59cu5VSKvH#gPOp^z^p)jA{}WJ&BJoQKO)TM=6hId$U1lwP;{ zLa!_*az4Zw4HD0AW%i*YU0SP|422o?LH;~=@d`(mW3m=*uVxdBA`U2|1x}u2%7@c} z2RRJIic!k#vG|&_*Z!tu%%)G@M#U7hq*ZZS`2a}|(7sUC4ZDf>&w#<-Uh>%OC;q=B z_V()?x7m0h8XO5CrlX^yt*!m}*@vE5T8cEZS&cruFjCi%BXDu`yCzGYU#>Y?)nQI8 z_#kPqk@CJNsjsfnnkW6groMl1v2^rgM7XqvZc}*==C__8Wkep4|EnBx@C8|*8-!_? zJ|ow*EA6A?6hf@HUsV$x)axiW6~700p@7LZ$2un~gGOTS14dRyUx2KW+xBMHBaWAN z#t~B@-`pMBRuacWavOFcB}Gb1?^4Y_CB|k`?!jA%!Nyp(r)qsY_2;@5GaeUxMNp7* z?ATIOo1PC7oSeXK6p1i&FKkkYPf2+fW0Cw@Sj@;FIE-T~il*NiSJF(bS>M9(%!d*$ z@m2zX{$N6*o+e5F5GWaPcvq6hUgjihD}N#q+M@h%i0|IoLC%R7k2kF(hW8rSRTK_M zUiSCl9my$E>|mq#%C|xe9!PZP!VJe^`E&&;L(~4G3m=;vcl*uM49D2 z{^(ZoL6w$$LwyN2?{pk9sPYE?1&bDSk*<1ntV45=WYY;+R9wBj-ja4oe9jC*<>qVF z=y7zxsl(^#JkWc%Aq3#2m+i3a+V(qbEG_Xz3&=%^&izi5)QI+bl3wu%ql#|3i&pbPad-7_<$19&df1cPipo||=CF!c> z2;Iwv4vws@v~=lGek|Q56-eu4UJ=FiE+2ui;noNWzTZXHi53XLDFhRc?$1FTbXG81 zfyam~nKgZLFFT?fGIL(#d8>H!+O@9`w~Org;Ij35v2{A3f-d@w$8(I>?l{^(2^8MV z4D$+8tPeG99vXvaJ5A^`S#T#}xlz3+DZjr|12sl7JW9q0!YKw}i;IpECpMNTM@1!m zqbu`Jv-_mNVWttW7?EdG+^8R_2%tcu_y;GSYfMZ@SB-YA*e=5n!^%uLb6{Ar1VBRa@Tcn#MJz>0IpK;NQ zG<|(d9nFNaLph85Z$17W`0G$qM8v38+d$lS(jmIxBTr7V%(>yF5K*I}pQ_n-5J2Q$ z@N748Jn<6az$KOyY5;`V+y1>l3;ybtw8ciOcxgb3Jc$>+;LF#qeshvfMMe@QJwafv zfj%wJKqy301dX@WKdxs#+qxn5lp<&F(sOhVolxe0BUvrQv=3M!83tJ^+}JQC9c$LJ zSq^>CSlrmJ{MgfEU(T7C5GUy`u5gbDxF}MXjkhuP5&)LKi#ngfaL&{?Tof~F8?!Azu~C+%W?k5s zoMdb@i64sVZ(&gvoabVysd~%dv?b1Ot<$aN&)W%L1kl|q<$sJ*D=`$vuIQhrT`V~d zvi4~8faklOiewmchE*Hi8;O$+LS^^K+T*wx!;WGF1u_z5K}_kD*(mcB$^gQzZcJSe_40h_2`En$B~@=PC7c1M1HP(CEz@vlWPkfcbs*C zwyz0=VADtChPvlz&E_-fDdb}8d1oOIX$l58#LccLcvLqAWfSY)huXY4iBy!(r1@76 zO=2;NZ`CKc8Sg0Tb7r3JTFff}!+OlLt4?>o3aUz-7$6)XlHd&(+F4+v%w%A6l9!aY zi!`>5X?YP*DeLAcU$wk;9YadvwrvgbiogM6bx8KFAK5a&V)*n@S$A8O&J&a9YEHo4 zwMG`K{iu_>$fvib*D*%aR!<*)4(~sS0D$OPZA4gkq@(JS2X1{1HCK@I51O}bU6&t9 zPE(Sdr6vJ8y)*cPoeF^>fY4423IMgZO+D&8#i3MV{J^$j!j_`Yh&9n z7It^g843MOB4QGd1`(yY^%?~Ewbt+5ra%TLMVe(Pgj0}moTp79aSFrGBey~ zUf2X4<`%1b*4FIs*7r1Feb%3`!em1>3tB3Rf`1v~4=((I%P~hbz>vB!a!`HWZvO}w zf_7Qh2kw%$BS4;nS{Fm1-;Z$ieYN!F&6~51GoVSpm{)Uu4^v~VblHchF!~VINYR^H zet=JE0W7G=sFB9S@8y^(m2nByj7y9RG z0ux1*S*3v@{f362<*DpKe31z7+bDJy;ZA;`MnssE1_DK~9WsA?VHrs7ugc(GB3B?? zzPb;O?ubQuNm9^XqAF4Y+~v;KmpNC=5^RW8+Hyg!&qa@VI{w4hjOI`UXw0{Xrvfc5 zGYS188uOlO7+S^Pgc|5Pf6yZ1;0M`|q|^VwAp4PdYiN>1y#3=Mf`5=qLQ>m2DxeXW z4?eGu%x7dAiJ}nalCC3>ZY*GI^G5T54kcACM((0=b7Jwu9*D0f;n7;F!BJL*kZJ@K zx8T|ekCGrstjpIq&0)Vb%7eg0m9z0M95R@N$zpBoy-XAm)N+I5mEzxFY?9yQhGPS^ zzLg}-%M7!2?fc&|j5Z@_GgF>=R1f5*$Y0C(R(;U*{bSiqryFV(fWu555*L&zAD-5r z6{Cq+kH;F~58C2j$7Rb|^EV6-0ZzRST6U2at@`OD2_A^oZ0zkHD(3O_?eDs!Z<_}_ zNo(^1-Ra19UEM>ioxUMpCK6hfK&GnJtU|wRY#<|VyqUs`DWr*U_~@!aBXatG%+t?$J;Y-13QhH8Vo_~u!bJTY+}Ms*{R)X(bLOrh;vJF3>kRGcomUxbY-8N_ya z9A%yzug#_YiPeZ`B>#%lx=TsrdiBQ&`Wa;lU zW1Tcr0tEV%d7Ss=#-J-{OtoY;wrbn5Pa}QCTVj`Gn6mnF?%<7-0BBj-CM_4d=iG}Y>31b;ji!zENwr%?`tR?kc$n}e zqQew1A?kw*7fNF+hKHqhfa`@IW=|3^o!^5$ST|+y-nS9=iLt;sW+$k*IJj@&b=r0NWJ-YA9ea_PL|*DRjY=tT6I>e zAGKNGF#lTkl`8{LFz=K!&HaV}!NHAa^&p%!nNILH90E~7#Tz!GFqXq9YE;o#@Fmc) zSh~16nyL0isNNEc5*vg2>F1(Xz4i(4G+vyX`V;#|o}CEFtgYYMQLCkNh+(y6vpWZn zn6PTt1zpBFJE|MZ`3-K#-^U8T37#|Eha5pb);g4lnv+#LMQFJKnV5YLa?cLvafAx6 zw$=n@a=%*ki4H=hO^Nj-V$Sl(xyAP98Vuze0OhbXa97+TuyW!G=JEfqZeW~y^1!H# z30mo+`q;n6$;Oe=_~m&1>{-~ktAJTqhj}%d(w8O;G6+7Io04($++Bszdk5(8TOTf=lfSM$+5KzmruHl z0Oc!a#6&Gwf_?BhkI1IKhgbZ+4l^sOnDgfspq~)#>^)Rg5|aCVQQjOTkWH>VSIg~3 zMgE6N`*Vf$ZpmQ##;T47>;#k4AM!}|nf(|z9U>fZ?eEw*am!D`u!;jPsP`~7e#CB^ z{#|Rzu_^5Tnxw6w(wZE&1RfrmTTA<#J(nF2CoAQjr*-7nS z5rux9{siZOr%#_EFz#h|o~R00U)iaHSnm}80+N&3tT7MJt=wr#5|!waI<{zm-2Cij^5m=b;uFG%E~ zN;UyMKydJRD8_$HQ|;q=Wk3+{Ivs+7TQzCo0-zF@sOluRnTCPie}n%5gWqG5L#GzD z%@;gdvTJ7M55?f`X|t})jEIrpeYPN4XW#=TEslA5VVNa9ANn=bZw2FV5=iXDp+OtL z@`y~>MneZ6^~>Z^N-?QfC+1#jFDA2lo6a(2k#{{K)&UimO+ShVYm2?4OdAz_#EN+X zMcK8@*D#x)*&{SPGCLftB-Kz3SNf`;%e3)$8dW%$<-d0uxFQD$%&4)_ZwDZ_Ts*e6 z#WB!Fow|4bC&Ig_r^nknru=+Zx{c04`KpUjW&ie3dD7>>5^>WG%5#ZkhUACRYtd`A ztXz_D<3^wBlWB&<+{bS$d)JAj^S{__;%F+rKyr$WAjQS2nVVN;zu27P(sHDG}-D=@l(YhX69-ZkQ5a0>H5<)J7&Wu^BRO+g31Y2YO zlIw_ms6qPB7=$epY4hEhe_aRj^A6l{YX)P3>zSNSDKj3@QF2G629}}pw8Y+R7&X?o z6U)t81J{h;Zk#y1N6!v$1+g7dmb^p}2Uq25D3-X8tW$QO> zG6b#sW>PwYWqttu^V_PbN(RZo8;yODeYU3D!;O5|rjN6{QP^K%fgavB*yG3HRWml{^~Xlp#leOIIjoR#m_gE;IRoe4RGNT#pe38 z{d#oGX7XgHNI!=34h%Dpuvnx8pv35!6Ik)NcW?glNCVcn3Z}FI2u%FbQVJ&j@RC!M zXA=Ey!#5${@zL*uE-&>b3AHlcU+&ngoh$me2&KgJUWL`P#YN+C1RX7Ti4M7%b-?eX zjbNOrJ-WPXmLc%0tP~)ev`y^bfc7GtLyIqiD6~{@3F;qSV|A?7@vC*)${j^xsF24j zVa}IACB+9dlG1e$Hg>^C1^Tg;nN?(U+!P!oovN_9m#7Y!t7bk&hy^E*j4%+{T9|~i zNRDE8K~zZ$Y7WM#zu;!3%i6VAJXCJD!ssFjMC;7rgF`OlnxKdRtZ3Zg^Qq&~?#?Wp5I1V1`a0Y7V=B zc+}R{_Iq?oMVFQ&2aVhGa{n+OudgK1#?jhX{eEHd}%gW03h1RNT}$06#hiMCuJ@B;9$KWA6NJ=5Uefdff}%Ub##V_Ovbk&%|R z_VUE+qjVmk6;}|Z5K+Q@8m=2xJcJCMh2EyXgtD-3f;5AIpYcoP7Y(wbyh~~o-BNK3 z$5zr?o)Q!`UBA%|kK5ZcM)TSkq1RpRb0~yBZfv{pr7qllCaoD~bfz0NYtOBmx?C(TPXB8CS_zptvIFdWBtSJy?p8zvJBUg8K6w38j z+WT%sHEI3IvuysT3qrqaJFfGm^;>oR{LqNAdSlEhB*NV{p#GZk?*1`1J>tS)NT82o z^=lLwR-aMTkMet;n02Ay$enFf-s6#T=dMo=H1_=>#20er$Cp+eJ-vJ>`WVP!mV$&r z@(}e~M`Sn3hBRd>7;+^u24)d;xa1`;Y!yGH{piH$MUOB|{ru<$=;~;n1!c(zKf@V7 zwy@l6L|+5EUPLQ&z=$oP;i2523_^SA%Rtqc&X0Dod<7~@tS32d-$u;q9vVDz#*FJs zydX^F*sGuS&>O^gGFh-dUD$V8IvH6d6X1ix#*ISr@Ru>|SYj6>yGSj?*}IPz1>@r4 zjQXBqMCix^ZPB9B#)gY}_PrW#p2;diL-!EhV3=Rx({0I5y|mQUnt_e3yy zh6GTez(@qsm*7_Ob9Io#J<24|OAffJYYiLL|y_SC5hcb6`KndTi*CY$@9^HtB$O?WB`OV0caiiJF~>D_?pL$vB$6c)Q`$Jy=C*d&qaXcTU(zM0e!$pIzr zpxbWyO$rE``*M3;2h~Or<0{bs77NGUIF1et9-sHW0UhZ=AL^GSK{W@C4=?rt!JcoA z9YW!RZS2eGLFP^DD>>19^7N&36B{)h*2blZOl%A~+`1-Oy;Zsqdc@fiUG4rIwxj6E z*jMEIiEqK?ISoNPKM>uoJaFI@vpc57#gSluI5e$FJ0WSP%gr!ZA_g|dArX~m=l00s%9lD=t^cNAw zj%^u#bmPg)g2(!VF{x$+z0 zGuCi;y3(k&ipu7hFV2U3y^JAQG4gZGk`XsRU3sCLU-Ss*3hR_`dr5o*!_>0M_EzR> zxH!G`gLzvvHqYg@<333glMQYe#biZ9F_V??xN+WbYjH-3*Nyw*`h*=*#|7xr1T)T9 zMo+^2vWtBl98TX)Z*!Veq>QHZY01mWk)`B({5Uvtoa(J&i943ckIJ{o=R>s4)zP~= zY`n1OxtN5h#HA8LF|Itko2oNil$-T&ckk(QlWM&=p!2yA%gycVdO!u<{N%cb>I)W7+Qs(_k;Aua-Gj%eTfRdf zAq&|6>ZWy7_9Y~ji@1rc{k$?;J_012ZjNunWRFHtc7STO31*=0*0UMRQ_7Fk4q8md zcp-2pJji6=lj2iza%>3zx~ zHn2$T)(I|`_$u+MP}Qj(x4ZNW7ckXu{lv9SPjB9=Nprs_%z_XvwCua?_$u01r2ESB z(De?h2GfMOewk76TwQf6vy$5xju;<-NnQ-GG;%9uT~by8RFDEh_%Xg*le`}ux3_4F zz6Dljiveh>Tg{fcJ5Ms=f)nWe+18?-43To7P$P!_JT!I6lv{TfqLSU0*~atsM08d{ z$I-ZE`owEA#d0y3;~t@%)9F6b)5ASiq_Us7^Dc=mZ}1q*SrQp3*&Ufh zf6XJl!^T>V`wi3kALYuVepodT3xz$cm5rjGvrso935fF?4zDG;@w(nu@*RDN3amV)UE_B<3oAtyWBAdQR zW-F0s2gv$|huc(u@`?^FLZm+2(Ri>sEh7CC^2TItcG`~N^eVEz=;4dL{HgPe)Bts` zLYr-zpgx%^6Y9~0pW6|N#?Fr5;IPFC)Fnoe1O{2QmvH}caU+MzvMjsKt) zRTKGF!*jl%9QDMt4O<*=?a!#CUR0f~bNTnrIUU>RU1U2hhz|v4eQs$-9Ru$2=AgnMkV3HU3>J{_)#BV zjW*qA#*F>;bFXj^==4-<5&F%zwDNUC9) zH-{NBT2)ETdsP>Lsp?DD9Q8OZ1SEiFQ>N57IKK482bPgadEw3XkL%6O5t|c)EzlxU z;_m$_x@@Ovjs&?12IQ{eN6TFhYq}~~WC~&bX?$9~2oL~-uTIu`FK8izOTVa}n!NU* z#Zyw$Xnr6-GvtOu=I~4YNLRi$g6er_3L>T zNM8pUC8YP&{?=YA5OJfc7H-UO$!|ltBy4({ROgct89yqU@+A_7BY=p2YKQgzf4fUb z{FrT%?saE>AGv5a#P>3V;6OBa{n6TDry_K5c|hQ~41nb;(A(0(WzE}Z>YlS=EwX+} z{6pK4({mrBr8Pp=ZG^=Y9Nib0%B*`Wz8{-EgBy}zMAj?;Mg*4h`I~dd-W~|~BT=$n zu1lBN=qv&J+PNp|a}Mft4*7EXv*t=Wy8?sq@fr?I6(I*~B|72UV{sw2yCzB8^$LH> zoyTm}mHik@{6mXcSJU~Yf%?{Z5mQ$@dUW#9lG6THKE83dd#$MQ>!3FmCp?`q z>-o+l>yH0DZP>y|Lx-vi+tPSSjes{bMm;L6?Q?9$%gshDM!$$E`1+*crhNMV`>rP)iL z+AW9@NYU1?iwPP)ot$hYM`2}`O z?jwmaGBI(#HBTD)%Aa@vuLnqOlQ@C)dM{Jb`1*pb(McKq7NIS?W>mm z>8bG~0HLPdY#>-a*x46x3zNuv)d}Oq<$$(>BZavhE8Nzh4voG_20!MVanIg{@uR4t zpFr@cmqGdD4eC8V@>%qN zNpgz>*)Cr`<3)c%;~s8Dj~=zD%%8n$&qo@U&@wHhzo&Hn!g5nAG^0Y1So8=#dh|=0 z^UtQh`PLVjw`j2d;ilm}b>Fg2Q;V`wJ!}>?P*$G5V1dz;9bR6Qhbyj7xeg6+Ub;4j zL0#0!uCA^=an9DjX@73%!pa)IFZpY1$~I$4(|!45Q0z0vFeT=ps-3=%PJ<8Zt?t9p+b7*BDt zO6!^aN5NXSZPNQcz2iVDoz^R>AN=BPJ^OuEW8)V4n!deS7*svVJf9~@~#vs#bWpY?TGrbkDM@m{42mM$s;0*(5Lf8L%%}^bY(v!nm`ZzP?mX(qy^+& z>1;YuO|cLQsT(Pv^(V`I6>exMNiFV8Oda|_cuA`(zJKp0;ZHQGV^ucX^iO>)Nzmd{ zy}GIF{uB}du5$~^R+738(pU0QN<14Bx>Y1sChll|?-Wmu6Pv-4sqA_;mJl8Vk)vTc zqPRBfHocAIrb?fpV_d|Tc;-eYgIY*>i-dsT*$N8}Hz{5ph0tAKu8TzH^P@BLDo>xD z**W#hBpfiqI6Xt7va0Z;HP{(yGL#K3S zGlDA@wYr8zqETIjG5gfQ!k!=}AUAcrO^RNWX0tT(I}|5;kF+*+>Smjnb~y!yumAm`!N~zn?bf*9-NizmPW~xSPTS#s_S*{n zz^BRq*ju=wS-qMP5GfxBvcT|ic1a4@UitLek2YPFJ!!?P36#-FC@!1a0tD}J*|I+8 z#_Dr3#vBnrQ=;>aOlQx)HhXvPo|h3h72(cMtRjMtWMwVB9Lp<5BdM0L`FeZ8Z)6#= zEYGnix=BSNWI>IruO{JPug|e91cD^7xV&qfYGj~)-gm9i&A&g#itlzHf}KCfKJ&F( zJ#OGK%?neWm(Q;=LuDBTZ|m5f{XhC*KNpyAY9sb@vRo3sbQuF zUsc9;!{|7a^&yEViBtv2V=>cW@ zCOE&EJ<3z7c<}3Y?-rMJFkbuej-HZIB(u znZYl;AN#+d5bwKojIPcfg--x5a7(wD{dqqJ7bRM2G$d-9uJNlWid0n8r|8?IokCw5 z^zSdE6QcDS^Jc7{P=m1|@68()LRV~k^r@c9W?BVH2M6;h*$j}`6`nRDv{k(s^Fp&K zZ*kj7>_C$v{5DQ>83|Ns+i7X+%>{^F-|+X23wh%`Vb{apQ5m#CW7oEP1w{rr4192% zi>~TaZH=Ed6crJ?Jw)PXh>Ht%&u(l3Q;ld;dvq2pd$10`n^7D_wwPAaiGq-lAqqLc8kVworTtZ{Z`!R6KBB-A% zMt%vY8}5KsVEn-WZU7Qlj=4)`HC0vJjuQM8L80M+9qNby3{OQvWUrK$1t@+jkp%F+ z9yq3NV6csPowLljq7Ap1R3OUclbbV)Y34FNYkRfVZbh;Tpal$hqiLNVx@ol1n}6 z4qw5L5Iy+Nq+`aAm$V%s&t=8xDe~TDR}q8FnnMSSR{GS86G#qANL)*snd+X1=B7d;-Q?`Nhe`i1` zn-@K6QsXbqTxcL``b6UC*+fN4&CO^F7;<_+EZDG-(G9<6&iGk???>qI@UfCwj{X>Y zEAV#qu4Qa7aOrSjf9bU{IDpSZX+wfC))hY9!Ze*)TU|ZqD-T1E4lL-Q+}M(<{LiXQ zJ+4D&*c9gHC)t@9rw(mg1CG@U>*NBnQn+}@TM7K}q8g)lqrtx_SJ6&wnlxU0W1oKg zZ1fGf0AmDd8|&Y-(ZzE17x3iuhYugRB(oi{9!#WK$za(}&D*}i7V+?|)Yocz_95eW z@D2>o4X@VORdu5Jla36G3mCE^FhTyf#I#B$nD?ZUx_KX2HAaWI!xpuWY!E67aQArI z2|v*GN4?_C16rd2iF#~TT2)x&-<(|SmpBw6jvwzEq&6^kcCV3iK5S2ffR0~vjz6I` zs&NgPq$Uur1T9cP_6G#aeCP;lMU#c0pVO6)mge*6<~aqDxY0`C%aHGFrFPM-Wz{&S zbxCjV_=xuhZ`|@SlV&jWZmQW%W3?_j1|32eBBg{OAQA;($IhJ>rl_T&NhDbLri2ay z?oR#z=_>7QT;NaK`W8$)Wi0+zP7gF z4kA9AwhSyy^cAKThCt2ZT#?j-X1*t7*&02HmhJw9x~eNkJ8V0|sVz;(eYdcM{=vaM z+>mBxy`jX&=C1{?&_t8StkN&+Km!BB5M&yIx{r%y^#1K{8x^(b$i!@(w}KfEv_X>X zfJ8Y|flKroUNzS)myc}I_r~D!;K|#}Po978rbR`NUAAEB$EJZi!jE71`QpmNrpgD$ z*X9zddw*9fR=H51>eqL#G>KjOv*HXdBv=9EN+Ue8fJ@If27c4?{$qDQL5j%!q`rOo zVjoCc6zr%uzx&g*ge{~$##m1=cvNKoc6qaR$S`~-;`m*zrxTwGm#`;UGtZGYye*^?`G0#9oP^{QM%Ja3( zu!~akW{Q6u*4NQ-Au11b>%5gKzaC8H4N>pgw<#S`kCo#H3rHWM%iLczP;teXjvJ@- z{teeBk0fN;asE+h+qS`_*>R?eQywWQR@1Sif4t}w*FVO~!7?^Jm)A_EwSJ8&FQyy;zgUhQLw?;4ebspQjm?cCWb)#L}Iz2DlxRx<3Qo4m%9Bx~+l&fOG0 zI#@p(%>y|iq1FfZa1~{W_;@#?j`IhYpWyLJDF_|6Hz+WWPQsIBpjnF+ub8kxYfdNJ z2byGJ_YHM(mQ0#FIlYSPR-g;`aoMgU7()pXeu>Ce5*#- zXhRyA@D;?#!-QN)>2k6Ti;zRVBqr9Nv%yeid!_+x6LUo}t%RWRytD~6x^qutB6s08 zMBdT~(JS!aw2kBc=6AsZ5q`!ZY9yExkx! zV|b^(2G-E38)L3a=bXOg1NQ?z#@PMuo8uQHJ#fyHqAy>D(f`TunM)IC^NbWJx-z(V z=VHgZckbkni71no18+1@P=m{zJWm90qVrZ>dVAm_v&r^~o)rwU2S}J9Up3T$aAZ;Z znRQo>uIbW2^j$nh?JMI>PRpI(T;4QIFRDXWSXd|&_deZW9`>`RPv2)#`2Kx8D8AC4 z&pv~uW#-eWkR|8fUn=(YZe zQOHo*L{3N}5)*u!4)7-dK0IyVk`8PeV^uRhKOaMr>>pqM zw4>@Jmw&6JP%ZHZGE=eOC1sqjc>N4}6ygUtJW2(A8Z5^HHh~RxG zR`5J-<|o}zz<}?UZV&NE5~yOr;kSR1=g#{qTJ)O^o~hw5Fz0US;;RSrul|~+uHK{h z%M(LgN|Tq3FF(UA&MHL>+l$m{SM-OyQgb&rIJ}~h5Pf?eJ-vmIL+A^y{VeZ+S|LGs zrM<_0@E#{^|9eiH&<&SXb@>&0(qq!GL=KQ*!6k*XPX?oh7xr5Tc7W~r4h&^_0bnE~ zJiHbUWS@F;hKO&754iW}k#(dnqP#udnL;lOy;JLEax&!?_3?gNt~C;<-k2OPHtsPS zMFz%f)7&#>`mojHlt=zUV=9`A9pNXq@}ZET*Y9QNn1nU)X@a%MKH|3<6YYrV6LkQa zu$6i9)tRd3bz7uWbc5fOX%_YC9L`;iVGs$rb}!ceX5RoioSG@k~aBHNSMXfy4!g( ze(CdnHjFYZPH#}Fx>MQfmN2!~3(w+7d#8v4$;coZfqc4?f|kNhIlk;cYX zOZ#SW8!X2wSTjn6|3TK#RvzqJUWX4`QN++v=+5`BzsG}tkpP``_0k0XI!e88V zYL3Ci#$y*!FQL1(z%XWj#0b@^Kq-0yrBF5Tqz64`AGe2#OSRN(zvZ#~NA$|3O`8gC z4|=f;o_$?=d=A@Df}-(ux*y}j9}(OVlT;N*v6WE+)K4Pos>)OA)+vK>Y(}4vX{!3M zKk5qK54lT|s*dfso_u)qT;{AewJK-a6+n++^F5AhC{F^85alSvaq>x2ux<41XPh^L z=k4cE+*Kwik3fF|l0qo9vco#)=HtfbSOl67`$uA;>oJSDa~l!ti|}c@fD^pY5^&pV z_QiRTckbR5OOPV=oR1FkY7Tu!_)}gO$^Dl{g#yurYoiFO-0@ji%(?8_m#a6$vf0h0UP;Jg2GyBrYnxfw9*N%$6rC5(m+6yy&RSkGavd?tfj z@pl9ce#%cz3tSC%o`1i9)l<6fd27hFDt~R|tgy1Q{=Z=gu*FcxLo^=kk;toh$Z&&t z;sOJa}*fmZyllety-klX*7r)k0_^iTC4jsYRkr z1$^*Z2*;cZx^EnVs|HQisNEj7bas#FEZYMC=q8Ulp5avdW*@z{SYJZ}@f*U+-7M>F zRZ8;uh|HlTCMJjL_f4a2j6Y=PdF)s>Guej;cai_8w~C`~RWz5!V7sg5fkK%bzvx-l z47@he(7?5ed3*Dd({!DtpIt!97I&<0|e8X<{C5-t?g)L5f`)f5pl=O`Vra2m$PJUmY9>rjVeyju7$Z!dkyEiAz7 zFv``RuL6xa)tI-ji^}nA<(;&yy&6GNz!Nz2m~m%x?M&QB$^M|A1R5aFGC;qrbl5@P zW>jJ!v7mtp9|E?Lx_hq>d8(M`ip0((Qx)1JbXA8n$hmAD;NYv;Q$f7+RN>8`O?? z-cDm?a!?Y>XV;e(ccX{g_n{9KOo<<>{G8B8yH$g52l~|m2M@9`-F@Lk`)I@e z@V<8W`9Ih9zJ?kcNfRN=TGI6~|H3wVS{R780ccc81k426CU=k z(*v*&ueD-5J&+Wt?00jqSJ$%?qVgYB)^aPWo zK1tN9OX7}PFd+o#W_?9k%K?ag{(R02tNWM?Y1_2D*9 zam8maTFSCd1S+0>=@qw~eWKm%wi`dPUcLHZ;X1)0;w@o`w}VCW`d*e!E{#a4seF4o zcJ_5d4MY~XZ^Yif!0u+M>QBkg+yV4+;m{&XBqFSxHtmt$!FBS=(*GOB0H&Nerf$Ba z#;~EN!)PjQ-)()h8WnYg(v`1=^ zHn_qRTFv;;*v2(TC760{@Ru0EwpR4|9m_}$mtLC=9p9WWgCecYiw!4`Wl{gxAX$?W zUSF1@L^tF}a|ymJEOa#q8%82lgwOSD%U7(}Qq|^7nbM3)dS5rK^c!Mgv=4ar#~*U* ze77SFhgKD*^JBT|wG*3nQS*sYw0e_Uy{Z;qs+pBbIFqC}pW83s3{yvH`9%@il|gQH33 zjvJ3;7K~t7tzhPhin6!y-|%w@1o7y}lkTcwk(a4n_k40{YJD(&Fl5MDRaF9n(mw4R z3s!bEsVxl$^ueyKDp>^i>LND|N&<S?jPLYJL_Se^vTSYTrP+#;G`evO*4udG?~6On=R}~xG#3uopRY$p*u;R; zTDH^NIXP<-x^)0ONjrurP@E<6tQy0v_!s&P;{5Q`E+5}> z^mZvrdq+S2w=A_;xAM0gcwzJR?>D!aFw(H~L0?o6>3*IitBpO5Z#~@lckfQj<7Cm4 zQ9V-Q*2}PuRN^MTM$gE(msi(CdJc@II7$2`k&d#+Vw}Kvs88f1jg8yTTPrwo+>D+t+!MzPR_$=ZWG0l*>c)3fU`))qe(UH~DZ?Ny}b*8CYe-x5Al%<3Tio<8{WJ zAtGC`b?anENIZBLFaKyo_HbvE1*oc_`*frlVL0g|4PL&%q>rCIZHu3`*(hXJ*Yq1S zda#KxCN@DH8RDkhm|DZ*Ia)5tY zb}+dafD@-M=uZ5sdIDxfL^RI8f{2vNZ%7Lj#{*1#&n~V=C|vb9NZtsx=z_s&y|WbY z0Nr+IplC5@@6_Lgfl+>JQ+xo%>+%8~rO%DI* z?WEvvTO!;2OZqzs+Pe0~s~BDSJ4_wsUBwXm58;OH|Dt(bK0dwQzo7x3f38_|kl+AL zT>PP>FxK0rkNFRN7}sr$Qq^A;?TAF&M(yC14r>~#hj#8z{dAwi5${_mZDw-ecdYL6 z`F-=X*4DM~^s~D*azuMFNuQhuc`{+h57{Ebo7a0|t>^;`dP0>QIdXGKoA&LKByWW+ ztpWbqPkL0f?QTL(*z!yL=A(837N>t&!(J+x1^KPIy)xg!-zncEI{N+;*_Z8Dt_yQ@ zH~hW6CRl(l+wQhPsFUgf;@Vkoyk(vCrK+@P)5Es%K7Af}3-63nengesG4nE!oi!@W znrWDYuJcc^3ODIN{U)11n8(-3~CK9|q2GG#*M|p{&G6$4wEV|ZBi7vJGhU)!7Gtsq*JCYegF_c2Av{+ILNunFj zwAqKVX@j@&_TJQKJo*lMMXC6qNu`*ds32v32QOQ)L~mmw^5EjZ%pO1Uf2#~B7W(DZ ztvWS^?aC`nqz|HAkvZ8U1DJC~;p09RFP`w)u+G@Zk-GR9?^?W#?)LY;M_@+%!*xy6 z?0=SaVlc6&-NC$;%tzREw14~pcBuqzs&7%dA=EbQ_jiXe=ZtBUq{lFWKoe$ZTIAWY zO~Ga966bzi3uGbT(oA`&Ya8bA3Gb)t%>b<1`}&NB%?fa7QI^A2Txc&MNzA2*NlCp@ z3yg6w6I#-WNDISWXkK6Qd|+YDyZA>+)cmAGp`vJ-P=69Ad;3@ z8gWlQuEF15C+*Q|z%C7+Jo&-K(%xkDjo@>ks4@lHhqzL+dORpD)@6m^yEGCsLqByQ zZvjsd3<q6&_fZIoe3mkHX)`JD-Fq zVDjY2_U@zo2#;lZNE5x+1iB1o-`47o!4t_vXxO&jW(1fh5LAUXr-{j0fh#E)(KrIb z9nYT4CLhJ<*zDGCrZb|mU(!>dB%}wG1VCxoOszK*1r2?_+VJRvy?qS~7JVzn=0GF& z?S!a{&byadMmJh^XjKg*;=ed32#n63!~_i$y#L_Ano6kH2tt3wc(SiwKnW_+;Y58! z?Dh9gup4{s1fT2(@A(#Sn>+zOJ0N^f_Zc!^0Iyv=Gc7D!l=>DMHihn?iKvFWF8s{2 z<3IimZqvGTJd#EA$ZpN*!7SY?Y+K7WwuoQedy%y@;a5EerwwmhL&~=#dPjVOrGaXX z=c5YU;Bia;>Q4(Ny0x=Uxle^8ziW54$uf)jH7(|ZbqYPAH+b;QkdTf-(^hkcZO;J- z?cX;_a0Zr&AO<>+JLa5gzQIX?_TmwZbVdU*$KBmsRui8ssvjkL-?%+cWzTu!UYBnG z(i-}NKX_@{&QS|^xlsrtz-hg3;P!9CFADb_xbTYRa}4glJKJT6)72KEjQ!+N=0lpV zt2&}&&Xu(tks2$SxfJ@^H^I&(T#6CB{8K)cy$QDDg=*5d)4#RIJOIQ8IT|6nmrGA| z+@1Pb2t&>hX5&lw7^(^%-M-y`0t*%RH*Da>U(Q{tMhJ=`(tkg@54XB25Ies;|1Qb1 zZ+UG)mG{thgnfgx!wNjwPhRCBY|`Ms9h2OZhxItV!VzSNTY8)&zy&wr*`&UyN1#~{ zTPxao7=Y#PA9hn{i2#VT{I>25hVhY*5R`iJ7A&|i z?-RyZR;y~-K4DyUrR82~INtc2(TI^Fhaa1=VCm9^*NPUEvpQ9DS97x;IQ7*W+eotxez?&ur`h*uK30E1vK*8K^PO z5QG}1x#=);e)?lHXQF94(s82-QA+6I($@)_M8_~A?+fB<+wz@tuU>X%ckAuUrO{)# zP%+YP#2z%?GnXI~4h^b<5ChnxvW^YO!M`dOA@MTrSzjd!?rU?lQ^h!4gEuh%KZ5lzHl@tk0p5`Qcv8`U|3}xG zz~!8GasSzuY!#spN!AE0vSh1>if9^Akw_(xh!okCB-xWJ7qTW=?fW`}m?B#d3Q=>APLyX@%LuEaA}fB9(Vd#2a7t_>cN^e7s1159&HnnGJJ%yVamoJu`HA@`YI??0 zI662a7eP~}Wm1qPbYKv!hBb6ts=j}GL<>(5hf^+H{Q*E7tUfJtFb5^AYO?ftS~ zfZa~>2qn;n5JFlukr3G>rn=S^N&-h zp{sx|Ytz4W+embv=WEci`Q>G1a7p~ddWZUZGEN-$g_Ls`*g|FRxpexPup??X3!@Ci zh5d^!nE2%zSG96=H;q|)smY(tA9t*ip=emw27W6>oD7c9ShR*49?^vTScR>3nxymQ59No8IlEAWp^v4V5~9?^VBKk+dBNpa z_Y2zREgLPkyLLT%Z!5uKj7-7fO^x(*6_I(lxr?ctB=kjlgCI~nHyJj|=!%d>k?%n= z&~h*H9c+&kJ2@f=1{K6{;oB~rH|g0^m^?9qNFF{s=w4OlXHTDsq#ZJS><3U@DMY-8 z4!5RyVT(4}kyCTKrHe+y;;xBK^vW~ezT@43Gty<3dC(kEY+cl9R)bp3N~=YNV#TR0QH6PWa~t#qG^KeEosv=l=9gy` zZmYftCtb-+D<{`$tlKi`QECo+JeCTtMP}x9|kFJf|uTx|QH17Dw-xlGByVKYRoo zolZ<3gNkW?#QI_NOJeS_c|pq2wu<1fotkQTwHW#(4z~ixvd?Ynu9sqR+*B@5ZfJAW z_Z@G{g-SbPiFe{4*xhQ}m?mwU_Y(J>5jYzG9g|=^ji>?EwzgLVlKF=(;$Yk=-O3{* z9h8jMGE02u8RpoR8-}8lOLG(FM63}WpXU9`v7>mThcNIRXO+X@0e@OesmG-&=P80r z;%=pnK&WV)G%|sUWalAhU-;{`x=(sCP=f$P$0jf`O2dqYjk|p3|tfhoR7t}Ks>^v19o?N zyp#&AM`b<+D2`CuFK;7*62+S_;T zd{Z}al*Jd5^A+W%$L9?Lnhq}ik?&JH+#QL$0A5hR_=*KGM$U(luAjL-p&w zENh&oEwfOT&pwffE5DvxY1F2&&`s$HvohiG(_)t=q0-1{TGq}@`op(tYKA?H95{we z9$g%;>xATs;dp~YiGVw4*Xh#}Hog|i1?6NOI+Y3C8XF`KXhllF#J}&2s?@_vn2?Ch zY=(tLp&tVUlYzyXGcWpqMT^cAUWH=d`@v@kL@ zUJ({nPq^HTo?hJGXubGI>LE-7--Z+D*-xaEoIJtXIdh0sF=TbymdTUPV1Hun`^^Wl z=*sB>apjLU3U`1B43FmWu&{<6npwzauLrtt1Pg(Y$DTn>oq+=_@?b`NTTQMee*(}s zju_h%x>Ou|4woFVrn2mDPD2e9qP%k7ywSJUO*CaT=3#DPobL;vfk7v609oJ$Ohlaz z@r-@?**W_x&$|7F7p5USa*UXbc7~LXcL*Awvm!?$?0GuqxzeeTlXkOcOd#-6_lf6x z9CNtE@O2xkUxJB%vGfxQrR(2Wz}?5 zGn7I7;&`XkXx{%?Mur;i23JT*!SpMC|GhE$Ns9RMM$Q1z*qd%!AB8riwM1$>4jN5dq;+W1Ql@{99v!=4)1YD7tmI zVHwt7V6%@6j~(xFBz-CLG9X}pShB#o{-E;nYj>B|5I+Tx={#@3g!2W)HV0k<5 zSC3J-2|A7Xhv1)NPkkqI<_N*AIqtS8_+CIoHNgg!ff>UBUBduhT=v61W4bK3qKwx8 zmC4v!Dr!;_lg;OE>2xAZM*tF;4{*v#Z;KLMiyK;wiDGo8W11Z(!V zo|=-fo4w3%U1F_ZZQWJPY)DnfWxV&cyibS=6^5T{!mCgFOwNzL9#hdebn}z4d~NN! z8=A7o&BEQJ#hy)*yuIs#Q#s^)YkmuOSd0aEaRHc zJd=A@30NB$IRIE6WP-v)WocaHOFF9jPBM)6 zm-|UhMC~q~MY&fzcRYRZ0~Us^C|wfU0c7_Za;FP=qTu&vY=XB*3rI#0l25}!13NoPPc5TNQ^&yXeJCHtoM(VLw>zx-Bf zYumbV)`$<-Ty`9W#I5vXD&2qn_E44Ot5yxw-w>YY2lTk|(%}vy@<3QB&xW0P^xd{5rRsDhm z1opk0>gVOo#p#0c9Ru6hcVEl+NgKMR8L_prQgxwR^V^ z2p^H08UgpHgYkAw9Wio>wuljqgc<)6`K9~hvfYKMZq2XDzE&|qoc#_44G~jB(H~7; z*55SywP}Nq-s%2S0TvVswN9TFwNs-S&~s z^8X=kq2=+!meip?eT8CW>sn0-0mR>U0g!dF3_H8fIG}ETo9LBEOfoqDcj9{Jpo3FF zZZHk#My4N{7Z8BDzA68q6tjI?ZdiX>^P{V{&e)4gVJXo0h(z1_1VFg)mP`DcSAS3& z=4d|+kM+)U;RovZE5U-zI{`SHkB zD2`6KT9uqEzb8aiV0Ih-eWMRLDA0GW%J6&@*eWsf$N7qqoskx|W?;tt^0@Jt1vB;6 zlg=LFi884n0|Jf+w!ICL2v0&0dS+HaJK~zU>8MW};0~}%3aE47iWl}DCJP8^(;REoM3WLv6EK+WdG+N~s#V;<;@_P~IljU3f zYds$%imiS1hE5sEIY-Mb2RUm^D;07@kNM}@xz1~!xQ@K|B&)13NAlY_c|ioRs(Ll0 ztJchUY}-INDrTw`L)>1>nJmkoVvFz-(cP^xREzr1t3?B}He#NXw881RCefPS$jiEpzHA()88`qD8EuI|3{vVvNpH#Y@M;k-N#W5(adHaUM$wz`Fi^N zvm3<|LYA$#PNk#U=+nL@RlZfffCivt@t(_)NFwEVbAv~5wwTfb@V_`BKQy#e$z>G$j zBptA=Hd1^rpw_F2pyOf|b?(@BHb6q#jZ5-~5GjwRIbo}kV-Bhi#}llLysU$bWlc)Z z>T7&I^$y3ofgkxiPOe9KiO6?hHZ^_v$goT^`I1kY+J5QO-1kf&sjmWs)?|uhp=`7_ zx(-}hCOPAt-@IAY{}Duic2<-nS&tc8@AgYf{!w(A=jQ37C3>d)`h_FYvo~PXguo;|g`K^h}Fp%EWBy8?;k z-n|wM{&e}c;EBWt^Wl?Q?&&`!-VvFC+0LDdep=5tf(d{L^vAPxo@$&P3jrE#-@bhz zBg4;HXBt_c1IvnQU>ZNLWBvN2XaP!k_fj)hjXFotP&k=MPZ%4Ig;XpOL}lHY&5xCk z+w0d?zco;dVm4h#8`zR%B^RIIwF~9qDL<+<{$7lK@sdAd{wg`sC%ZTDRH3qHc;iw! zm4Iuw5uSLfR~?_@=vqfloY>2(%1PKcE31vj^LfS3kbeci-` zm@-LaG))eI3w(%CXM1Z!2>l2DZTXb}^3K?{x=zcPtG4$KRA*vVoB}(QV8tY3*Z-BB zR8L}hB#*({yVsL%Q_Z1p1Yv`CXCBZTF&y7d83+LxA4-Iv%s819&jq0jQHaI(OGG;Z z>ST1BOo&Nh?X3SXJ>mZolkTL!N?mpGMNxeFd)hS*<~31^_8W8K*np?$hxGm8SbQzc zP*}OE#l^>_#O&q=kS)@(7o=zJ-tLD?jf?=2R%pj&v(KgpCjG`VHl~@pDqI7_Ums3P z2u&aK_A(q!4Iw2jTv&WCi_L6cg&FTh@?3);%8~3m*{K7gP?hn#^?zhkcGgGTemQB%`%oXrgl@5N0lx!D(i(d|3s#|JO9(U-Ald062FY#rR? zesAx082f>qk$A0Kv(IVJTpa`uN@46dcXa&}>8nxufx#%%29<{eSL<~EWQ!EsB^-OD zLshK*yH1^YY_y0}84nMSSuZZtvKciheC=-*8gg|`ej94xm@b(|CVl&!+Up;1?bY#xk#sm#4uRTZj!wk^P*A~^{e~W z$M5~O1-I^y*y2!ckS@f=;6)m$u3wJR-t!_UuGlQlX2v zr|2wP98~MprBkfacFU~}BP=bZxGw`?QWD_*|zX@?KL zkG=la>)8?>$+bmmvpr^ng@uVbE?-7J{sJXZ{H{H4Ii@BIK}t5qm~Ot$yzQQ0QeNWc zWJ#x{{POuviplw>t(!ekf46$%v+$TXyry~+hPb@=ILSq$FFp@Z@;~-DL&_~ReErn= zWXl162{wbRcI!J90^&&}0_oh8Y)`LvG6L}0AxSsQtfH}qJYe{dfMbT0-S#0m) zSDgkc?v&$UVgy?omeb}J#d>imglrU zv(9_2CrH7z9w_o@0!$*1xU#P4LPHDdF=);d$J^FLDw1~E<2GCe#GIu26Y<2tnJ^mv zq&y~L#z#bKkyox*pwQ`&9nzJiN^)bl@#40T(S$0(!Qt=q{jZI9RZvhj+NSp4@9t`G z;$2Y7-tgnDG2;$$(Ld$JtMV?C6R?R|bj6B`sJ^H&1_K7P=6zClJmvs~5~KIBxE9c7 z=r!jR8}ygk@zU5W!le@FH;rh=~Tw4U6%*97;vWeSk*BHs>9)hUxq-xMd(3+_&cBu(fF z=18`UdR&}SWCGl&q(2c#qL_2CPjIuCGLcf1mV0J9m5dYKi~%s%mUB_} z2Y=1{)A>CMVyeDOrgI2*!O}G#BhCptg5_Ew*mmq_#OtdRgqkBdVHz01_Bfhko;#^oOIPR9H~k`?Yu$qPV7jn8Br3p7iG9<4s}$5Hpy?p@5;{K`jjNqHcuGV=qiyl=%NN?iryLAU5_D^ zM_frI(nv8P&PYAE+(v&HB6q;&gQagOIy%`wM;JLCbP%Pr@%h0{%uYCcT)B9atej@^ z<{j;|#&l{*LRNbSu8?SNG-c>ThXx0qVVc$H(y5MzS!ByrdYDwE3sW5Voh6-6H^>Cg9>m<-65 zBgp<=cr^v-8v=Y4O&yjzjTUX%Xl9;0bEY`+&xw8w`~0uKNFhY&DsVJik;1H( z>Uh>H$ne4W{CS z0YqE@W2hT&7_%d!^YL6uMFhhlD9XXqY zzK$P=2}e$fxbzFsyJJ0LL=I&VS9?xRPKBFj%7nZrHUH};NE*88vOaqWUW zzbR>?vVHrB+_l}NF=7&wr*J|f2e088SYCE={GP+#zt0c5-$Vs99Ok+qUjok_IU=s3 zGo_zaf+3;Tc`$DGXhFS5m0EBW+T|ZWL=r~Oa7CWG3xX)M$eV;Vli#sX1!I|F&djZ0 zO5jzGbKjT7B&jW|&eN0;9Lq@`b?z=28REuj)A=^fq~`gtdT__(b_&Z3@09OA7+wM9 zk1DeS29sE7anti;ddR5Bm!~KicdT2TyK5Gb`HaA{hbX{lpY#u55pe3%T8%{*Uy?oT z`V_z0Ry}F}8azuN0SY{5SXypRc@~)aR?qnS9CiX^F5H{e7X*1jC%k7tpI+ADh%Bkp zd6K?>bg-R?Nr0v6OysN0;t$qX%y>eY4$i;O!e>+{HmyxJ36V@5u6@b#ikd4)df#$F zW(K5Rf?<%Qin(&K=`E_h6#FC!2oVTwG--%dO4 zmo|_7^F+T+d)I!-Aex39F^gp|@J8uL#Zr%3Kt@t)EXz& z2h+VkttWTh#28Pnv{|=HR60G_1$aQsGcz!=vmb2J3U2AmKN>!GW9m4e8`mik^dbDu zJES#I&siZq@NIe6jdQ@Zd59pPSej%~W98%^QX8>x&E^=Wsv>Kj&(>eu7ze~aN8G8<8|SSOdpP0TOQ#|K92echdc1k` zi8AZ6mo`4cWjHA`6c&bRm3`|!yble%aSEL+A3SU8c*Dp!VY&^l|Hj0zrgGkAP-gGN zD*r2?j-8MKC^lVrEQ$B>6*VKbnQP@Yb5fI*%%s<{(&H^Lk4Ckg@Zm+Mo=(Su?!oGn z9F6nofOH48_zotGV*B>J6{j(lOa=jDT#nlY*aABvh4Fay#t2C@6h)c<(7wKR9L}@4 zQ>>$;lDYN$Ihu?%(cZ~2&+ziruVVq!h{$dD@Nuip|FwD#D5X2`t&A?FB|E2=>k6;8N@iHI~IN+v~J%%eA1eSfYA}P z+{7;n2P`Ukbi`~v=R0`e!Yz7TqvAPB6>om%+UfH_{k#_1dE=7%3isS5RD%GI9G(TD3tq z!z8DQlP1BGiYM4suIj5{dd{^6F1Kf9ZmBvPqjZjjDWZwQBi=4Abp;iPkJx{n3zr<9{1Ib*U;l zjx{+bt<`+Lr_QaJmt+NG7S($8ymju=mdDuI)U{=3W^^G+*N-*?NS73jEq$`P60%45 zi4g;|Z3^$AG|5VJP4_X!)nE`LeVhk$+J97OZ_X!es@tH!6>t?%B1d~8wdxgZ!xlUo z8JKX+y)+GT@q(RGg48Gp75d$PCkeW?M2kS(f^qC3X?tLQ4s< zU}hvy{Y>NXncEn!5cOwV`zUjJczDvyB0>*eUbU-CES*X5K|n3S8b(h^+ldV8CMo3P z3@MzHcIyHWoq(Gtrt8qX32Od`D9axXx-m={(;bN*xr9;9XY#p5QGdVLS<6U@tt+3c>H-1&EEh;ORVnqKfD^!CB~^MtAwM+;JYa zljwvGXkw_QWmjU%ry(A|Q!P_Sav|5^f`0yS4K;%ljwC4a{UbfyaEr+OV4O1%kge)V z^D}G>iH))}4ar(je-ok=L|}u*g{w^|4OqC1OQWjru$mjws{Ozon7mkxR%L3#{&uy$ ziLO%%Q3w3(bMT*bZT!F;AM}-#N73xEW#{WVhR@r+pjZ78gZhLaMK-v=|L@avhI{Ys zkQwwV%FJpf9RBn)9c;eVk%QJRU=TVbnzOi5cKa_2y+r0%Gsbgu{k{9vL_TZdQqUC_*?r^5^tG8fEy-WE;ppS+?ZrTAmeXK`Zl!!yJ_F7O%%lW%}#f`k0toKe|}x zkgcXJcs@C$7blG?ZUJzF^qNLuLb`D>G!j#>wS(LR2uHpT*c_@SC??vQP;dH0iSB%O zx{2Dt#jyNxR0>b~kmXlimOs${Xnkb|C4U>gk8{rhwddTqMHkO*dcGIjfif?ydLyHa z56{7RUq!wTYQ`#_C02XJ&zf=d7vpSYCL&PMQvYtDJ`LoAIQYc>dqU1%{m|XI5-&@> zed%edW9E-)rA@T4ZoMM3@W#qX&6l4v3-w;wcG3d73s<(QX(T0-+oZSLZCZD0&-z;f zZ&hfmS*%j6y2@acnrF5d>TdX5R57IXidM5W-pW0&&fxobyZSAiH_kp`R#v35uhpFq z*$Pd6+V&Tc4e!VPc7VOyyu2j6iga^n+qvX-lTj`)xua}sa{me4<`J%^aeqwqe=s<- zg}Id&+*TLU_sN=Y#(nX+1ya?MO?#5CZD&;qKqAXmoZ_<_-jh{G z?O1;B$SN*WKs z?dP{TB|IFtae3j*A$NOsFYGu{t>(oefzt$eqrvlHJeHLD6E7at*98E9!9cwj1><}LHKKVzK5Z{{#&(A&cJ0i} z0}0*;!QZwHBdlK7 z4OdiRFL_s0edm{xB?Z^DCBJVWH&Li(Azh+v6GbzU21H~*%_H2-@N`uq4}EX=fpQ!V zbuZ~WdE2FFeGs$mL9T!wI8}KR6iy>X)W7Su&h{<3TQ9G-6SVtFcowZ4jq$NlcfV8V z6>JacO|J(Ssrw!sjbxK@t|q}zU38Z+jfw*cB1RY0j%7w%K&YOZWXd`OkZGw}1Bjct zq>SG|X-@2g!TbSKrYk!J1g(@zE}vMIxwH`1s{KaLH+vcJ2g}zqr*4rOh2Xi;0r#Yo@Dqn`#k~soGOj| z=q?{-E+_|~u*^AJtu4X84?{dHtJopqHZv;Oq1@c!a@)2NVVFJsQ*a!&jd@Pk^E@6{ zu4$*@R2&s&=6|XV@w&O8<&qvgQ6n5;r&~=ydiByeh2J;SasGG{4Q8 zkh>6rzju{rTO=FNuk|!D>nyqgYmJqjVZv>IOL)RG6bE1%K=mbQ5nXbEB$y~AUgV)d z^Z+3}LAk*O+GXriO1smM=34458Y#}ohhl)Lf>pt?^AEMg9@ znm_-9a;{nhwJ&)ny?eJm1O_kS=CY6cL)Dk#WkMJb1%pgr<0~$(>bT^{&-TU*CC8hd_GVIw=_22SlO~xHTT|D4c<0valfVEUej?d zA(}IXnN{E3Y`)pq6Aw`kMY2X^@%BNbHkDw7V$;>~fI%Ir`o~e-H@MB)+wYuL>uY~M zO}uU1PPbR!!QQz)OFAEPKeNSN<(fxCix}gXdmF0MZ<+gH@8Y`^<)0ioH;LV`?9H|b zi$7)7JSj#yP0gbRR_OjGusc-iD6q7YrSBy>D@2tlwT|VU{|6w5P_ZuY?p1(bEfy-0 z&W6PD^MuHQY*Vwcsx5xkt+!3cloJ&M2N}lCIFl7Oq;Vk9y3_3IPuuV$B5B2!n7rGi za)`SV@+S`Uwmjp1`@9sjX^RF*kX<)OQ&bG+e=LmF>_N4oUAoSkF`CR%NZ)AV4Rw^w z%D|8|biyJtMZ z-qF!R>;2aS*aF*i?))}(;6qS+rOTN~Me?9FZ?{MVugaCpDujGCH~yRZp0#QR_QAvI z306Mu8CZGapC?fom3?_I%Hu1t!sGP_i#x`RD0K~0nPw)Vk+_I4N`$SC+K3v*!rixQ z46Rx7RlS4Hzxedzxm?_TXE#evBm$ebj_D< z+Ft)yci#55EtETr{xGb2Vey^C)t+f)g)%j#M}=`__*oBgd>LI205Hg_M0eXk1~sv$euBuLCCMe1&|dcmAGT!4>>+=YNnh#WiHp0v zRt)(_$w=Z`TUh8rijpQo+B!^UUAkO~99Ztz7hJe0%m9#5;;V^M#{;iqjNAXfS~|wE z{AfOryIt2^RYv&mJ%h`mu+ZTb!UI^Mb+i6Hl)!3g?Yyf86JiSa_T&$ZaBgxXNju?m zK)lOu)3)!Cyh)O9gq6E8NRA^FKhT}v`!Qp54+)yXQnu{b>e)+~#b5yJ9k9faJ)*Yo zBa9NGW!jwnx-qZ#=>Uo!%!hb6JFtZQu2Qo--#wMCT(@J#X$RY@M@207QlKVl+u*3| zpTU`fh$~OX&Eek6{r`jqJFesR0XrHZmUbyFrIi!S;fU8y(yDX92NNQ8K$E2s8+(Qo zw8qeJ`{-NvH8JWw0G0&wQ}QJ2G+BBbi5r6nU+EBM()1tso|hpC~@ClPiE zl*8)*e>9%ZNWugW+T%r;FjY0D+gNtt&V$weYN zehQs3DWrY{_jb?QF%x95>T5mw7Z@gn;lE*xM}xslo(BfcnPVg*uC})7KO}CEX|G<@ z`4oQk4)M+U1$D4|#R@1~DQ2i3f)m>RRH-^^{ft*VXP4I!uNrHLD(s;@R}E16ILkkG zTGb(U3Oq4lWz~7>ma?k-=jyTcnWhG^RpM}m4PD!oe5up__wGTuM@LnkYP+o1-?8l@ z%P>RJI?GR+UAS!j_|0Y>_LF49J5z1UMxZCXP+1MZ_a(;c(yN1=@sp{CzJ<|x5|3!Ujf_%$V!F`+lS=lTemuedCgWG4Co3tkD(y^XAF2u$a^u zgJcr34pm`uMTAxRf1vhZ1o#~|P*?WV92B!TSc^n?HNSm{3!dl7#+|`JskjX-xXkPu z=IYT;y|j`1AP=h-KN{=P6vUOg=Wz+TBY%;?9vF^HlItG26wh= z-3;~Os*OiEfx-AdL<9jWV80OFm}oxdck?Z#?2)caRz_f%Tm7af6ddX?WXuo&eLR@7MeyY6Rwu18?|v(9AGe#dqUM1=VB1ZK0*L;Juhac!|p? zgS8$QL<0H>^X`pY*h&n5VWBs6xSU)&Fl$%RgHJL&3bZ)jeyW_aUa2i>d2GlS5s zQ!CGvr{x5l!lmcln$200t_*Sf+15&>$Nq-%&1MXGu&a$q?(0s&*w}aJnd@)Ac@E?H zZy}8h9CRh*o#WpdzjFov@+48h)IO;iLksu4`iyt|D(#8NMG|x8ZZd4Wur8>Qj9Zcy zoNeWjAnz7a$*eiAuAGbn4E_IK@p2uZj%bu5%<~Wvkfe$*5Nt7QNx)u$Nh^dgJtK%a zL{)LSsI6YTePimeo`n367_rkTJ#(8r2uDp*A_+dCnZ&q9IJaRxU*sl?loDP-2%!M- zGD-R}5z8G1&D5=BNH$zq*tMBE_Ox^A{vTZt?0iS9I$|_hE_Y({SpQA8UMmwN!U`60 zUrGk7cOwUoOrEHuyMLr2plw9OR5(f8*VCum!(U!NZIOH30cokq{JE{;Ah1YsHtZMR z?fusgH!mI}?ak_Ehgn+I!W(0mQWc9QZq!rc^5ce3vb0e*^p%?(;H#hK1F#^W+KQQKdYud~0kB1Z?4v`K@o1ee0qd)DFqS^yz*d9vg#F zMn8in3m*_v8eH7}9XN1CA3eS02`4Qy+2KIO-1Y01IkYfYI$o0}SBGyi4KhEkzvY%J z9w#;}Ci?m(T>qG&Y#g2ddTD2Fc!uB`qEWJa7&&tmIK!QQ-Zvoz;^LfJ(}<8d|GPBi zz#ArV^7Kodc2XPf%-hdR(EiZLQ>R|y=LqOoCoHsJEc96Jr4_1%T36$<2oRJFt(}@~ z&M{q4`7rbM0vFCQ;2igtl6U_2_NNn=XSHHOl|+VtmZv$Sf4K4k`7Mh z?#D{E8q&-4?~S5S;P4A=XF2=c=f{Y(B$W&~*;wLdYEr1KLfxGxxLF^NKm^Dv6&_z) zN-#b4mr2OAH0?b4F;Cw^yCTu&-r{Uze*~k^$*{AVu(Vil0nu04qzIcu6+vO41Pqb) z*zb8;nQ;I3w*Jp43z^7RLorh;8Xik4PU4b(OwtT8+hwu9zaXZQGU$ zkIo42UbG zeDVG491kAXZp$r$^_y;JQJA@x>+CwNPCjsZbX zh|WuE7psH3R{NjUZUh6PGP99L0(4@xkN#?+7WIg5i(=|BSpfC}*q=s9R9?#e!nd95 z(NOb2La;?bsGlnt#|Xis$^wD#ph^7qdwInGXSG>(On4O!B4K;o-(e^3-=xx@m*SM3 zVhNRSt?i)R=F9FJZ#iew?aAZLpUCN_tNU~QqAs-phMn@e+pw~abjo=2x@swzWB-mT zT1~VF%+;CE{$9)F-zrqp)D|YXomIQc3#$Ef^o_-J=hZpFne!esY<{0Eo4+3a6X;D6 ze*Zo=R&nzhPn@X2U0t6inwuL81Qh26arg8bp1j11TL8cBaxSu2_ct{YfQKGVE`?e% z6Ua@3Y9iyckx1#qw4I-<2dTiynaM(5$m5@Xa zo-4)jdjlIF_yK}PlT0|P$}D^3#KcW+5Z6u00-X+mN<5sTEK0%`=2U@6z z?K)3p7_eoU#59MeQnLa~(^rTCF*bHaY4%^65xsEj+q2rle?z~-E?!=;G~Q6cKInmZ z(`rV0^H>g&Z58=k8;`!k6tqUvEL%=9%$o6DN=z>5#yHjp)dT^vap&|f#e@@VY6&x? z_=Gqfx59x>k7bx=5hBS@p=*KR#EN9rd`akY1j_X*;DSCeNhLiIBQYSHf1?J~7H0=V zpjXS8cH<+IsARTXBe*zw2E!ly`2|ISEjIPlZzk3l#3Z)=ik`v}qM&I6+E5uyZCfZT z7nFVP$<6`YzCFI_bGECNznXlDK9`cx4B3;y39ZDyo$H^y`}mP7!gN=rcg_ME-X|$N z&6-X1PpHREdn+~|9Z1DRLw9SfA>nUTvc0Sm(}67&Csl#}j*^3cvod`A0L0B)otiG~ zD}gn6p4Z<;hb6mPnAj!-1eC^J=hD{2IDt_8X%Ll3kLn7$SQ#fLui{s^@lKF`rB;u8 zD)zj!cWUI`yLNKqom~s()P5W`{+NM=TGZ=xB2E-fFi+n z{L0G$?H?S%B!V8LC6Qkv%ePJ$FOXjFcSF1FeU?xJ#-8Qpt#n`$zr7GE!b%R2Ea&rx zuvq+O!?_90Cznc?f%*0bMtnhWC5{~~{6G7bUsYG_`s%}n2=**j3dcQ-2uoL^YD>`R zlpGEQ(oBo=OljSeq4kN=z@QYsGud{Fc18-|B_^!gQhDtCzc^t(? zWZ+&$U|d{WW<1F0%uMr%EJ}L=1Re;dbvlw_EE09ABY=2u`V*N2_U^U#(wnzQL{Z(g zZMd3;9lcFut=eKppV~5?&RkD^P!s;vqQ+!$4INExp^Zz4=NV`&zO0p zmX7a^PC(e-z(KkT5~?=5wrJI=X(ovX;@B5e!g{t_g}A~uCV_Vr{kJ_MYBBp&&X!Qe?zsJg z!Aoy>3sPQ2po8pq7|Kg$8w?n5maI5(oVa%?-9?C?zb~@-QjO*=C%M{(J-z4mIy5XNdGI@Y zd;zw#C+&RP`ng?w+32ABI=kh=hx&snVR)0S_b;o(40ZYRtHfIVh3u4bUNwHk?H`@d zuhytxW?A$5xkW^31phHI=oABq8cb~Ert1oJD>2)Mj_v;S_O~z+U1r+(M1q`nmF&mbnGM7-9ZH2CzmFcRjxJ3Te10yNP=(_cRooJ z;gKn5?3gCqe0**@hzx};{ZVm5=C>YjEcIgf|}K`qF4`_fFp5kBpQU!0c6h61=ARbXQ5|3JYc`I#pV*yOt!PwYr}?@C(A@JIj5ir zw<==reXwxNBJd&Xtv6(f8%jdus?@o4Rc6hddzGHh{H-&Vt7b%CEDKy1abpNnQo6Zn zC6QE613#}Q90j(Ub+%6NJF_zYWaRgb?-TpFKR+OUp2CSk&yq1l|85Bw?>SFgZy6IJ z`OpUQThvpZQ)ah$066{K56z%6q+%*ndQ(v8aSJO|dLlUgN2Pa})&;O~vg5=w-`4bl z9uIa-{a0(``C7YGD+?Ew)(Gm;OUFdty?(t8AR-#sr0lN+m|1(##In3<(4Gu@P-Bx# zKd-{2j%v7B?^;w* zS2|O?`qi$SYt#&qY(kp3zI!t(WONUK-l`k+b+`B<$LP?{IJH+tEEJI%>Stqf-yAcF zN%ojBMG}F}6X|u3$T!k77n72p&gVVyb059f{~>P8GJEiQlFlO|NQXU*n*jl^+@b-| z0*o~OEN5t5KUf|_(YXpnbs+2!=@tZ7tg4irUl*ncXeaR!0XZ0z1bEh<)3 zRdpGnPLLBD5$P$z8&(%?iiwHIE;)`yl?z@yE^#9L4Vc9c@~zM~Nj=CKt)>FpT}+m{ zGI+7d3QqnHdxrLp?v#r=kO6q?hlScZnk?*c)-^bNRr;};V}?Ev*fWG)L<>y0&tM5>USmKN^Kc;izz*s(JfQT%I$5%`0op=1cBOoy#y zC6X1~*}nGb<0IhZ83n?;0X9)RRxB-25A#9hu#D<+VE1cK zdvrjuvb0Ew>kN@Tr|Oxh8AR~DIG>m`c+PK?`Nj!vk%PZFY3y{b?7f#ysa>jF zlbV1Wc_L;#xg#Oo^{f``+3#0(U8|zBzwiCkUeWj=#$v4m{FFp*3nA8z2A1Ge$-*+K zD739Iw}S9^znHU`t)R$}x*w`lv!>)c-><2gc#QHVhYjcMst$YM$wjo(tpa(e?q!{Yddj!fE{pwg4fTvXO=*|Lf( z#$sbU7*$_-M5hW0hv)&KeO{ZcggEKwL=w%1q+!QH;*-OXqW$aGi}3vYb``sLSx+x3 z@2Ej&F}Aep9Q9$_R%6A~agnQr_Wfe3y>~4Mhrsw0a33`aFjn>oNz}(zbVf5v=(jVRJ8g1aIy8A5?o>7p zyZ-&uL4U&w{Ln?gIt(dIVKaJ&q zZE6h}RBdqdA2p}W_Kp3pD}KheXV0H~Gi!Y3{Omh{bB-p3JUjjD+lSQT)v0zsbi*7x zET)bg(qhVv{d#)i&(|4{{>*aL%5H6o9?lJP`MoG>#OJIb>r=AkEgZP#1gkf2W9dF7 z{iC6@4GE((GNxmmRP5b*E?v~fxqAbkD$FgCk3G9=d-`Qe_EC$oKH z81cbkF^XUN=Bwvs5`zb`gB+9d5B=3MVxo8KNd471|D>POYJBsv*gu7s=k{sgt3*&S zdv@Yjqdtcc64GLF{un(vw`s_{*}k;1VnLTBf+Z zUBTQx4OkM|3IT@ zUf!4T)Y^Xm>^_|{L|1qA#!jP}1z63WU0&G%su^_cT5)GqWe@7)w{dpMM%U-RxSrMX z%iVFxlj~ns<#9)k4D+7h_q(@TU*-~}dBEz}lhZrbJ_StS{zo?LewbJDQiObD87$qk zYget|c4HnVPE{3Dj(DUAQgVn6yTk(67fn@TyHA698>vXMKYY;18F8nmJqI71e_T1YZy#=%#W&XqBR!6&m9vACJNi!__hlE`x@pa>O%#=pBf^Ca?WO^4Efp%pKu^y?Vf3mH zFv!9k7~S>TxHx3h^3!JgILF88PRN*g#|eF~I0Gg$$;A7NL_)yqa!ISdo)gk7`Rg>{ zMbH-R*`sEl%EiM1+h(yNRTfn^)14&*AFxiF6j#XuGb!>@Q{jGk zf&NA{prHbbs^+bMMSBo5kz(kyE?-~qXAraJ5zN9z5S~hWx`tX94HJ2$O&Re@d_{~z zpeg$+>$5ez&i)N6uTH**!&kYqiy(xN8$&%mK|Y8P1}Xp2Y@Z$TbkJ{;MZsK^;q|<+ ziVRx;Iob0*Jf1qwd{d}rPhw`Ur}_$#neL*z#V?V3Pa{Y)h}XYz&&eq~lKUvQuG3>~e6yjD>Y(vbary z0k?G3Hx;$Br^{RR_O8=#et)0kH-Em9?XPxrU_2_4i2$&Px*_S1~g29KXPGr_{TU@9+JDquxPNcr9A%^OfY zSb41hhz+-MY{A!e9fGzWX1l$d2z&L}n*Jfz-jp{`H&}Uc=V25i(3KlIH8eYo1~#LQ zx%r}-i$?AGvjlIWw|5r~*o+kNPS8mwY}Fs)uboZd#AYz~65(S#a3;}M5`UNacxU7% zywY&V>EApQTZ9;DF|Q?fvF8;)xA-!f87WPJL&r;?G`M5drxXW_PzZYK322N`zZ3x}y@` z9QXqU!)lvGq)bXU$ilHamczJCXg_-$jZj9PcI}p&4@uSoW&@zCPSFD!ZsB;ANJD~p6|e@;<6HEXzEfO! zf^Ru$iTm%j_1ZJtyZXlwEyvXOdw4yE%hzA0Ym=+5+};9~f}B1rv!lNLEF7?9uX}{J z)ySS|chKRbWc_?RT_jS>!)UwOb?OuvZedV^9}y*fKiPYp77yh^wzYG&PXp+&Sl7{n zpIFKs1{|ex$qh~SnceQuH-TA@0)QINkZ}33w2IQ4M>oPkhgS!A;4KGpPL-tyG)=m0 zU1mF?%2dL$qyarJUODa^oQwRHw`;o%Hz7m{HP_WyG_wWEX~C`6h+>B_e{*3`%&*?1 zsir$L&c>f?ID+Lo$T9Cs@%1m|$=NZsoi(k~qu>4Y_4F`KXbz#eW67G4Q_Db8RYYND zgT9T}UidpY-2;)rYVWGb$Aql85}zAUWi)SZp6$TG2*OlnzphgoXiYzS+V<)68K}Ef zI+y>fH4e|-3wBv=(<#0lsG+-H9tEPuGeSh={vc$tuOT~h$iMCVo#h@V?mwhL(zol; zBQpPfL5h@J?}VRgbe()fvWajuqyL?OWKiVM`{AsbQ2a3V*YhJ8o|k`ZmhaXXH1S5U=?Eb=GVFGS@=;VnLlh~`;|NLN!M(&0!Ks%>=b3Kci zQ9QL|#(R1(FEaRY88$9V{qX#(HVn`D|DFg9wD@dkXT;&=t*IIQdebiivAVr`YtYD# z?vmYz21_%GQ4ib+Avu2$E%enflCnkxTuQhMm)!jM-?g8q+F2x+cqHyxbZ_ygb~S#n zQ`Mk754L&4*CqQ`=Rk4FmuKM1W?ogTQ&w4K{oq#bBXlNQH=LgH9^Pj2F0Z7;K?{2y zD2TbJrPF-MiWBXg4*hBs`3LFAuQrU5Q-D1%TlCnhBGHp%E)(yGZ^UdzgD4>p*zQv@ z;ZnqC4)utMJVvaBK|%d+d=;k3B05Zaud5#|vfejFvJ;fY$jr>^FwTy+~bMml_ffVqU7UXr{Av%a)z(wVdjbDI~@w#=3Y% z8Zw>^5-qH^7XD`0>0^U2O!FP=wG=Bhkpd1`bNUo7K-m^pB!96THEIv4ucd8WJG}=! zq#6b9$p>&3;SaAlKJMXr*+g3_eZ(st4$Z_k&~*wx8>!S;VvjW z{3BZHTj|RF>BWnSD@wgANevYpAJRY19VHTrO>)vjQ}gZ%v^B$PNzjv4e&Z%)RxU2F zo`oMiG-PQvn-$6yo&5#L2)4X)-f3oJ&=Y9{M|Wza?HeXmN4T5=j-y9k2paEs)i5Ug zz|o`E5DiXz>}f^LC?pD5B*n7x5l7NJ%u25h3{6Wv zmZ>6jAb0G{uk)cbaF~b^AjJs_)ocS>MXfKBy*0o6^0?R6V`=fvXAC6u>~AiKbU*td zA>huh8C9oaYVL@tptvy zeBk+IQOIwFQI~eQia#L@m)Yni8E5hNt^j_eRljocMmADCZoJ*BOK`*-G%PNZ?;yU1@M|;>z?4X*S0s`^jg|88ooB#H0ti8;Z8Uil z#PbQss{pOFMENYmggm#JOoGZK!bOjcU;fx4ER@lLMSR zwGSUYG^x2HVrkg(%W|oIB~pQmqV(f#p`PQRkuIW64TXobf5u0o6H?YQzkSApmfG4H z5e!)S>J3R=%rGe4J4w@2j@&@A1lXo{UhfSXsraE#=@YW}Mm1(GmP+Xz>d>)c$%01K zlYv3Ogn6@4(`T0T=2>GcJVf&))+sMtK62U3DVJpVNyX7+O+ zGI%Nj)`iAkYK`npBum0DzprpYI8!vez3!@0gf)vkHU%vME962S^1i%bVi)RG-ozPC zutX;WRspzZQxvRjm826ePPSYA`fDMEFASYxktk{7z}aX*AA?+kkPke^$)&k50APvf zThPKTn@cjKc<+ss4V|?b6dtH|qNh-5z(D->ik+IOWQ>Wp`I!$QBB@_g~&2vX`C8ZA~(!Z@_yXA{I6AF*l@kpXi zXcsmZHZwa+?~`GBsc80PLo;yL?xKu5K2D|VT&K@8@G{>p0=T2Tkx)NcAj3X=JipB+ zPLFAM51lo38yPLZFk1h9sG@Q@&v;KpC+kMD8`WNR@#r=G+R2aT(OmqW@@+_S?~O2P zrTA8rTbG8a-Xu#0oc}~f#kqK>>BT^sZ836Rno?x9|G{phL6!Ul462u6wr)N4%eTvh zo8Qmbd=W#Zq+w*z5=U*>@-6Ew+C&4bQMR^jyv*!H9_rSTn#uy3V}~pJ?nH$aOhw-z zyF#)J_I^YJEjh^fcMIZw2m1CQH|== zTM`rVL~%}Etr}y1(ssavf(_N~7G45CA`U~zog@_%^dlnjI^AzE4IkMUBOJOCr7J%G zC!NWLNWbf-Ye#;QSWxOpQO8R~CClFI=sFk0CdRsnfjM2&GKL&t3o|uUDcO|ubtL~O zXNxVF-XftVShVe-TXfXySELV>#ozyY#XCTs&HwzW9?%urdcXd)kR7AYHF6xVNVo>^ zw0@hc1k-B8+SP&R1au!4pOv;iBt+DTCptZPBv_6Z`RXZbI5Nk5BHOSoX|dq9)28izR>J)1@W&rA*U`4#xOr2Oo7z`&fs<<%F)`~=sAs#| z-%5A6R;ZF zcK<)~kfA~zlp$qK#)!zUGEb3EWL6?nL>V$G$t)ydnI)ltR1z|Uge1unGL{sf>Hj&` z%Cn!nkKe!JIre_{e&5%y)_q^sd45NVYg;tT%kuVb%L0!19i-ji#a9+PO7H7iw@Fbt zB(tTS!`GajYN&jfwW4Ukj2VVh`5^}zF&JYn!xWKW3{#Tq4Vg>s+l&6|UgjJHFF2sm zr!b)m+3EeExX2}zXaCF5oC+G0BQ~T}q5HBkuSdI+>`MBQuM>mR+uYXLX2Y^nUvj*S`VBq1E}e!EkN?5|Xr@lE`v>p;1Sg{-Rpw&pj-@!*Wmr?WKO`pd86@u;D*{ADitp!KjhN#|-#_Kof7Cq_cvo!Y6t(tjP zY%O+swCqV#vPg=5VR{~E)j1O-FTMgnBm)g&H%XFInR)Efr}HUjI^;Ga%VA}y*Y1m~ zxU>vXVX#q>L`pLGLf5D~#OSiWo(LWhdD; z|MKfXeeG>)j0_ElYkF*Y0L!R!G%1ya!z}N&0z2&@HP5u<@NA@F&l# zzH%MXddumEn^Ckhob3WBxbX(uiMdz5?d<-#ek7>sKfLa*$JSMW^--s5dyw|)ye3~R zXPGP_E7d*gci8#!OYU#CsX^i=i?YWBOwo9#)8fgKID;sbLda`Q5gR|rCcb-jYR_KB zVG%hkiW`6W;W=Z*jF@pY1`eG;=xB_H$)EV*>C-)H>S!msqRqdPnCOC>G1^&VMqJ=u z&ot1AJt@S+p5eP^n>EB8B-uwugraem3U_IN_E~Rx!nAWFQ1!QdAY28q+6O(fh79ETXs%Cp|g=QAq4oY)_s^bwn<8OmPb> zxMK*mxIh0uCc`wj2K7?VR3PN_3ib{p*3*n*vU&ASq*yXoFr`b*?OZuV(ul-%V** zyKIbdd00MUO1+g2&$JflO7hZ%T_?EMTdXQ9uKsqz(mtP$p5IqdAAnStb2+ox6G30Ysk}MTm@Kg$rTceFyQWmzFmgrOs-@P_AF3LmG6@7bc(5tjhNn)L z|0Z4u(rZsQOc&FA_mKYflTbmQ4`YD>&XI2XJbH1G%^apCp3?WlUb=`RPn>AQe^>BJ z&Y`n&zSR_vPm(#IGJ$N-aI?T)cJ$g3b#H**fJ|kqZu0YEX+W()VmuT{eo}Fak?UHx z43z?XdWChnTJkpW&MR}q02Il?Njl0;#3@Z1Mn^@RE7vyUAI5J|W=%3Ej_lB*M|Dhg zH`f{1DtRXvCCK$}af)=udZ27MKb{kR%*`9=0$71et?LM7w=Vbtq@mT-_50DnyTuvQ zc8+f+%{qW%UsY|tlGic%w%vn|8TX36ymqN)+!DMRGa-}Wod4h`RU_F z?Rz__tg!GmP$v8k-o?BYUS=9a>5=QqfBYhKpkT1fpzkwUBBG*EVkMjNnn%XYfMS#% zX{GRLk;=r=u3Y?xLk2{+PiSh~!XVaNR*>`6o|*q2XsYOffx!l8KPD2xdncE|s}165sELl&I@G;?u*+DCb=E1RO{=;a8LAX+Z26J76P|zB+pqL)QQk#% z@ZV|LuUCcl>GrSa-niA7T4grtwxb6;$f>`0X!J3|5&aYj%|l){R*y-#sr~V!V#UQO zHXbJw>1_MCw_64sJaJ-BxJEvIH^Q&hL;epOc&w86{bYqk>`~wNj28w7LPyTe{;ZQl z7!vebMR8M{c!5(NYZOG~(4BK<{;l09%St4`YiHu+E0s97`3Nw^K( zk3%-WqIP_{jBkYbYf7VT=)4eLPYgVucPAr5UnJ-`OIS4dB?w`Q9Wr7>EzTE(-zN^# z+}xW@;_^Fk$ctl*eN13?vSX6cKb%3oLIsobnw&$LBw~r9qwnQy%5lgpxF4nYwvO?U zD2vJF+06!i`24w!AIqL>;=_gR1_*FvcYw2V_Z?XB`1TeKq$*e%qIIK(wS&y zGCg8i49Q8DYdYQ4b+mO}ZmueN2W2DjEd!>Oxfh13qkjmWtK}8+n4!vr>>dFf0#@B> zz3fY5prUp1{7`rHCSlR29WlX9~KqlU0NX9F`J8LXvV zIY>3TuwpZRaP|!u2=Go4w&S~Xr?>s_rB1rStAq0`C{~(N7gidly&G(*Ok@f8 zw3<2vdT?)3p&%+B?wW2#$+M@rGLV2En|%QRSJ8`{o0HmJ#o`440gFV~6ifXLf~Qde z{vD?tFFeMRj%t+*^w^Ej(LM3;h;72|D_K^E7J-C5v~2=dZif?QMcCqYF*%j{++gq_ zvAl2Iv}s1?w0c0nnzb97Bi!8-Vq;??g7}G7t>>RPb!y1M%M;q~7Lz84x3=9S)y*Yy z{Rv$jA=WT?XPn4t!@xOn$3RynfFekRdMRawZYr4zR+IqMK@@o3U&C5Wr%-8Z&^|{a zkRp+z0Vtnn%*7691c&@oT*b9U3{(tF0dbRbfyHPpcf4LxyLTV5bI>~)fmnqFpVKxs zt^$g~jGq`|aSdsm#4}NBXE^BR(nt%2Eb0thTM)D1e$x(I;^Epu^>P5v@e%bU*d->N zKd06B@!r_1Dw#F8pd*Agq=fPrnYi7Xtn^<;hA;GWCLg{@#!dVWXqFJs=!Z^-y^1IW z!9m+ovm@X~W=H~V;6228j7lv|ty{nT4g2e|(iR2*nOi<*0LiIJJ4EDipu)fYV&&pa zs9QKf`S>LHc30u0sHkRLyVhfoB0yrnsD8XB_t6qFWR_FECxDw(SAVlfq8X+(j4H%1 zVMQh%p1COUe^z$5doiuE-;<6zBtPA8)z^*reQXczd)!eC<9h3^`~HLED%Fqio=3q?G4aGCkbM@jwF3B`-d6w z=C2R*kDtJ3! zNZ@pps+P9}$oHO!PvAtc86PljeW|3H21IF-<@wsU4j@6=sz?xii?3%aw2h6J0Og#3 zqsW@1XRda6NW@dck1LW=largB$~CW){04xdjvpd3ZLqY#y0%ebmf<^(4!U3gG$I*; zybysSa~rDJ^wZ817NSRNIhFQ7SuZ*Dk{>+y_OM0Qvfz;+L#h)gw5Cpb6V@!^?CPgb z3M-J}f%--U8%~@)9lp;&1d%N8L)=HK@^(B4*AJ<7B;&_CKI~-_<_@L(;4ieLZEVa{ zgM5k)bP4vDcDjje3s`VCBwBD#AGAL2+=vxE5OW%HJH842fr6b}Xx4c*yc2^My9)F< z4Rm3HX8U|_{^zdE^G|yRd_7n$FXfbVCV$4G-+~lXN{g8tV zhnOumIbFx*{KbX`-qi(kWSO|n=0C^-Xh1ra7k>51gSO{(0V7<*zSuy)s@hy7q6%m1H!Xdp;ubyRXRWUUR};M5}B?db}|}v;XEw}qNif2 zk0e}IVP&OV@+UP;5OzvHZjd4^VUZRV| z-+)*kl2@T{ks5U0n9@?J=%`$?Km1WoXG-n#zj*+<@GCSh1OB z1x9R|C)(k!`x1FF#Yu1&!IcaPkH2_fA6+p2{)U#e6WOo%@^qwwP*qB!XzPc_3eH=w zk?85aEWq}rF|siWyDqr&&SBcLeqEC4Xy8!MmgGiwDxnvu11L4oKmrh+*nJ`SI7WxoE76HPiz8MwA(zl>eLrSa(h*!<=ePH3k~ zKG|yt!oI3V+=F%yO9SD&?VKu$OaMRH8{`gw3***GUnK2}*Kyrg{@{vk&fqD@MMicW zu*@B4CqS!hC+^}h9b>t5{hD$4)DipF{rf`NI*WQ<{3)t*|K2_~yt!Hdk9c*432gwXG%rz)w`(_yG%wA8S`{Z=UN*s+ zj9;epDl1=x4#Xv(!ZW^N3A_OlLbB|e2d5RIgHNBnj_Ax%tX&NJ?S9SS*YUl;o#N;5zLvJlqa+|tDNlUJ8HRLWOa>9KFpW|!nBooZxk&+L;8tBYa;rX(hvtM4qsG+SNni#^>I5|IPYo<1I?-Na-haK3=99(|!2+$19SKZDzCJdm^}*X*m+tz8dxPaao8Vc@%{QE_e5=QwbRTcV)NkT=K$vV|bsguGgM3|J33M5@PfxxtStLDfzxTPQ{~z=l7;3 zqjz!sn4-eC(w@MgbH$zo3Lt2sz+{7I23WMHD@V0+*RZrrC{^ExIK|+Zrb1oui52Sx zuiV^4OeVzLO+|!JbiuxIMT)3<(eP)aRB*z$<98jpUc$)9%vno(h@mk87M?CmC!)P-L=c z=k;BxflI1^FnMExZJO~O;V+uM>=C`J)T`C$6&gQf$??02P!59euL@f9RQR40nIYue zwpP{5!pdZh_DaLIgM@RGjw1*WX=s3FISS|9Spg)lOV@VHV8GtECL2XU~0k$6pZYcb#`$L1R}!;W6)9&BRVb?BTO}VK2C@(4?4<45K`X8nt>U zZRq4Rn4DU=WXUTGICUrGs577CjtEFZa>kH^q6VHJwQa#z#NI=O^dm8kDa7OJ9&c}j zB*SnMsc~l(qYIufFboLL)X17NO?m_=hCbhvw$ZNFqZ9gXZ8tKk?TQ|~9$Aa_`}>z` zs&BrxdP`=mR=-|>L-s^hpL{wuAt~u*hgs)`2zR5qN8{kE2P=;s9#D7v(pOw5YvZR} z;be3!q^?2EgkRVt)AFY@$ji^a&I|XgNoj5qmCh@4a@=OPVp@^mRP3`jaeoD&$Gz%Q zoKK<(%ZgckP~SH;5BE*w^ROMeyZ<$ zV4aGn>C1&EqOffN0hd~Z4h+|0KCR6vqMd^3L5?`S>8D?F5Y7j*mA2Xs4SXCw@~em) z1v&HANQNcR8%-?o*S+^VVK2C((e1ung3y_hg696h@;b&x5h^hY-^q;^F+0qF)$qf~ z46nD$B&y@SX8BhheLYS16Usit2WnX`vSvj7JYteK_ zV0&65c)seL!gq2tq37;%BJ+W(wJ6Ttae@}h3CoIqq|R+0v({tDk||_qy4P%)n3%{` z>c?^BvpvKdctDkVR$8)@QNS)1n*+fLs)AfzC25ZX!FoL7Ep7(R=&5;)*@^_jU_^7K ztfLCS(Of+84l(P+%L|mr3smjVgMFi!nK;8>mW3#<4cEmGSjnR%oC&#FRR|(tYnAU^ z;(9NCZS5yNn zlAnSmzJC3hw_+edQL)Ba-rGVSn-)QHdxy1{bk+hh*L6!<#BgZ_YJ6jE;gT$`v1T;<(~4 zU~|YI3H!x;B5(tE(od62P&;rWAj^ip$dgpCz^e|iZYup z!%*868=FfzP5jH3la(B+Wk2E86GChxx*ZV@3{L3taq1XbUYnuWE1U_ySf4g`05rH- zd;B0HqgPI^ilMNl71W3vZw@ZK_M>eLY}?goq1k@%h&R7~sX7fEwJSxp{x+cA}og6;8SCb&i)SV4}N7uf#q@-ochiv>> zbZSYtf&AfV!o0dOzc;`Ax#flKq%YpH*+)#ix0C&(Lag!h(^D4aH6_)XMhUq>lio($k zj#rtqB)1$fBhP_`2225q^|{;{W&{X$J3dcOZwstohu%=K#@JGwWvX(FcvflB^{!l1F>y?5>|&F>%yDg!d=VPLtKnj;owz-p3a<(px6hfUR56xt)WI^t}PTA!Zr5bD+ajq@ffrHfds! zQ^L$hER!(c_QPJBNl5& zmL1ZBmv%mm$&zFMWupuL8N+2+8SS@9C(w&O=&!<;rSlfg_a#6>@#ck9;+Hr0PzJxU;L)sNnYf9u-~PM)aex=iP@U+|!85E>)7@;LYn5z+Ei&LZ2+Y8il@zq;YEONgaWTI z%TMa=8N#*2>7zV<{-MoM1a}+|s(LHYS~5WaH0BcEU>6b<<#F5exN=u(Q3}wnu`*%W zs(y#Iw$hGm4fD-Y4CbC}BiYAu=Coxo*Ddxe43zYJ$(omyHsN_L9xQuCs>gESXr2e_YMxw=uf8ch8q35L%d+%pY8I87~<#E_;Qg^_bw4Ump%{(Q|FwMwQ%XyRfk`HdddQAm5T_ilJjU zv+oC*HFt`vsUiMcPtQMXoAkV}H~sms`HJxGB8$L^>OTAdUm3xetKlAqbMD-rNs|kO z&!?@i+CCbfkjvI_@$l*zEY^}D{Oi}RL0iAvV@gXNFH_puTV`PADBC7m4;?;ySBo0P zL&fn+^%&aaE;;TT8la5Cn57U_7`tB$%6dB?AXp14tnj(D7j_=Ri0|9?OogNfs&qj` zyx)IJjwS;&qGy%!oVns}kP9b)HPZ#xvDI+37i#W`!HWEr+-36iY~~fQ%3$=JyyEw} z?Y&Kg{=IG#o7wPqQUrnHJd5_Yn?kv~g=&)^%O6p^PZax_;)g`uBYM7Offe@CZh)J7i~P=aTcH7c2ZAlmnPe5+FE<@NkX*0m2%m3XOSLVfRIN z=PstE(`L=OVH3@frjp^5qBW}k8=VPy7wN*YkH-Avn%h&smKD>3Q#P_=ME5Wr4kRR( zil8fUsYt_K^5$_om5f9iAg#z3e+|8Z^TEB4t6ZFQ$UJ$bMS+P7QLdG^3$f*bJ`BHC zoEA_RHZXkaAHpoAuh_i(a0G>9H_+Fh?c=sM3C#i3FAc(xwq9NqQ1Y92;<2+VhH?;r zz&V;wSg;I-DZ)Eo=26K;oEiNXKSP&IWJp2YFc;~Y70;xA3Im223~t~zK%{4w1U@0$ zUO0YCD>^}$WRnY80IBdnZiwlfJo*vcB=~4=XBTv8*)q%T{TNQ&jSyZxbdv8pBx^^~ zm~b>(mnB?4>@~CMF`vT%hyXe0{kv3DK?!rQxum+|hn~~x;|d}1h=YNH)nmMvgOW)Q4qbiydvnqi;U5@cU6?hZZ|td3OYaH| zCn}sRA4TGu9Iw)zg8Tnqd0zJW-ELrz?f+F3)Pr+oDimxA;Td)pd^qe`_;9dQy*jv- zS!pNWQ~%?SsQEYq6wXzI>xGXNER&gxf7s(T&KaZ=(p0_5%*hzj!S*RG7e}KV<(}%k zUbU%D*QH5nhIHGQG5;o8406KfzPInZstd=yO8r^wpYAUFR}wnemNT5uUtgvX+{x=TuzK#mMRn|E<>XMz{GYr0I1 zTcHoEBg|o|v^&lJ^%Q49Q%90|dXxWs4ub4*mG)C%R>T1Yd^$IRqe_yI+qNC5LIk8| zK|)2HF`@9Ss?jvp7K}Y&H_zM;QVC)$aKO%fhx*vtH(pWvW1O8-7s7cH75J6q02qbl z5ym3nc+jfV9(ln0oZUZ7G?=B5K^gcMTkU!?d!}2X03G9s!6{A!VegPaSmCUf__>Z2 z18*mbk6uE|-QpXzL_)e&9P~6d+fZ3HaN0EGk(%3}gd6u>*6I{XaV=Ou5`t|1nHAQ6 z%SAf{ot%jGpWR$f*$oDU{6gDt<8B_i9{z^(F}AX2#_t0V9q}}7n3e3vC(9&;dpv6Zg6WRrlDbwp#0#YG`1!V)}j>yR=*V!xN98 zY?4$BF->;ZTVmGi7t|+opQ2yC)hr}4sSl*1eG3qY)g<+{GT*DPQT>hfeb_BUjI;RV z@(twO+r@I?OH-m6Ok0FH&x#eu=>$N*W<+FeR=pvl-lL;IB1oyrRU2uRA2w{0irS$I z9L1Jt_UF$#I@FAn8kI~=Lh|auQ)uNQN~dg73k&;MoEv}jPxu)q~Q`6FB zk3;CS*?Z84_eojvOPKtO24M@Eb`B4BA8Q&mcd@%P2Xde#v@DqX2zkQ5IxEGG;>3a? z$$@}hhKWBANUI`Y%P{EfD0s1poIMfA;6ZN+etOO(Zbaqh@w~3_RhlfjVtLEalK;=S zIagwl8Pb=T@p=Nlxn(BqbZR560G+&^;C+PpoQ)#moHc8JH3eSRan=30krzPV*cV}? zVI^(11}Y3`R9sMyJ&AmyjCpnl7y&Hr{5E{MfUQ88yDVs+!Z*Pw;tiha(dBCHPxgcjsf*%IByGM4JdW)OfUBCkOmnkLfP=lbGLv!}J*GAu0;9<(B@GCi~ zC1e^)*1vqvB9d@-tIbxnEhxxjcq{B!CypO?crhnq>eQ(c36!c?=G#IaJsMDiLxGng zWlri)@E(kVz2sKW)qO9wilJ0+BK#Lg8+I{viTTBPNR7%202sz1S{t-{(9z6S^J|?w zdm@Gi6s9}jpzW&W9bbHNdsl3x9=jA82mdF`a`c<5Q&)Do^zQz$_o}SSu+cY>t0f%y zxw1`G({{saPn{i7uBjwIa7+v&+vxGf5Bl`nOQzm0?i9t3Fo&c!pF`tjJ$h~A#A!mf zO#FifTTgJ_Z5Hng4&sSB7nftnel74bhRF^+3zKqUz*nwrpMcR9YNbZNMIcvx@S#_tQGSPpayn>heoRt+8-v$YSX z0Wq%)$la`=>ziQePnH`^YioT5If2Mkty$BOfQt}!h;xklBeu8Lg?tDqLK!gN@as3r z!@o$PvC6B?6AFR;@|PL^Q}Vs-rss(^nx%{QMFK7ly`v`74U6gnZ@A2#HjIIAHdg{=2=8 zOpmZ7&2BZG^=N1Q%%UU&s;G$1^q#X$4lovGX;WS-DbVGLRxSO)RWJOtC=FaNM+cG3 za4mzDRx*gq>C#Y5)A&mxA?pWd)E)O1OU`WCRD%sfg%y9axr|ODS+9o%$6V*EeGinM zKYr|skb=C8$(KgbCxC{QFES}A!B+vwx?ht!Q*sEQb8a=@MkQAPa?+|7>7UG@ZA9z@2Qc^3|z}2c|cU# zhs_S94OggQ&iNRT&og_aH+d1)@2&ai)hjn%Zt(o$-Wx?>QCtT`6TW_3y*&BAg0V5b zZ5x%nDQ)+U{-ja%uZQ{t_uI^ybn#JVKTPNHu>>9?EOBVY^S2(|;4?{5c3J5#v}7SxARrs8dzGZamW ze+~lVytKZHfkT{>-Mf#+N!f-e4MUa6h%UxUf%5y7 zO``@43?F!P)hMT-F<8;px-WDJaH`&cKpz*Rd+@Zk~8q@{6+03 z7x1C-;~*X#vA59_^{HQi;r30)jqsOdD3>#~uav`lFQ*4f_csCu?1qP?O19_rx6ND) z@qvPysS3{!VqIZ}><$Ma1S#rNse!uy#?Zr#gWxp&X6z$;nV5bTHUJ2J_>KU^-+Yg`j)=6xI> zXBLzY7|h5uwX~!-aNtmd?@JTa%Vp%jMdW4vSahlx_C`nP0~pCD>1XT+6%etr7LQv3I`p z>(wjDKboaVjScsH+jrZ7+og5Gx*pZ4d`aTbV56}^P59)PzZv$eBXceMy`A&2WqKB+ zMV$fqf)_hIQ~1sS=S4quVa{Vs4e0KH*485FF&q@>8ba+6fvHN%ONb{$=qYWxBDXnq zcw8EhFR~4Ev;J!(Q%qpy;_<9V&X&y8;yG5;Rn#-v5#=|Bs2AGp-Q=pD6x=)KN`qM5h^lpDRg8Ik9d!gGrd6LAIKtIsDdXj4dI%|&8{nfKlP1F0mOa(JHg!&UIIWeaWNa!gXp`#Qz* zN~e-gL}JJr51Z|epC>g$d@y_U)-jl4M9EB4MTRN(48S4E+kWkL_w4#XnqaqoO1R=OZjyuK`+3-=Z74^RCjqKupg%>fd zY3JAZWzV+auP3c+Y_VF)2NdlkffrK*iHGIa^?Utl?fv7(HL~}(v2@8QCDws;2=X#j zvHWD)@~KI{jry$t0kkOj)qIvi!o8O~=R~BOTYULRnW&PlAu;jWw+?VQlI8|~HjXL$ zvo8cUESYQUQ2OuYBHsGfS%ds)Met30vg}M`{e=c1^-C^w3M&JkRa<n}0m>=qezi9ufM||!2&srV~3S6!Rp0`I`>c1ap7_h78pCLP#3+@06AVt-h1dw2BKwI7 zjJeQBGI9CFm6i7k^e>`Ms&O`Si$B^;Owa=R3VHK6J9_}86ETx(4p4+&ib^KghC;qQDi|X1wL4~UF<3g;$pnTnbz@&ZqT0j`ebsd4W zm)LvzM<9Jq?%Gd-L*N>WgW-_;kqA1Jxg>+3^0@e_UC4O;utX{o?Xuwn4}z~q>Qc?C z@%$$*nLGE({vcoPBi#`7apgSgnkn&NBqH;8AE`DVzZ3^`!GuiM|Kb1G=Gw&oiou>r>Q#L`5(5g`EWUh1T*GE zi*udK>wocLcR`I!P5lcx)X@&CTNBHM#~W^NsE1`}(VunD5Tcf7$#0k*!y<)WnJ>tl#kAekbDZK#UO;tAU@8 ztwp)Zm>4=1~i1m6a!MxV@#*O}cCV4wJGA?@7GTpoHcYDYRAGC$m_^Dw_Z99m2h>t~r zr%d-{WelV&56>!J>uO2rsnf$`$L+NotG-S{j&3jLNt#Q5%Bra?-5Uq~gMH1WXFEYg zgvHNexWt!zzuj`^w32FNhddtV&0s-g#3a1Wzr-ZYk1X>>+$A@_$GMPHDKNDV`>@&b zu$1}wDP|82{%l}qqy@((o zS$^^>zwOlEeE$sX*C(&8`&n(+-Df5f(puezJ zFsLPiH{rC8{!T^y!yW}j)hvWa;L*j{(P4h(Q4UY+T+Iy)EArclH;UZO!*3pTnkg?z zd#%2{GjTTN7uVD%v(OCCs;Fv`W6xoaIXZ|!KAJU4e+QM$4yIUExYK4?vq5ocJ!#cDW+6+^#`;JxBa@bm&vxnNM*!b)OCCYw6+Qn zGCDS@6$&>wC)1@MN_6_i3~ZM`0l~{*gl|X8Tc*8K7Tzxzc67AFQ1f}a=Y4X_rK6&zfN{^*dF#I|YOS5Ti~*P4 zjdjquR=qlQ)CO96xR&#M8uPd)J;sN;iP(+)PZcmo^utc7a?5^neVbP0)(`>am@#!K z_FeL>griKXn_ISInf2$Dxrz{n%}4#ppo$JHHs6z&np&U#7!eK}mG67CROAn7K0tG0 z`i+h=&Q30%(9cHKWS3sy;aIJYQC*&7mKoq|KmEtLIYMbZ_SewWCMzC|e3SO8$oC#D zf;&wMzHdl;vSrGtORH6uTIMi6HlmZ$lY5HL5lo&t((qI%h&P^9OWq}{fxTIgMdl8- z+&M>*^?8b?0cF%05qVzh0TQpLLd1H`3Eu7RZ$M$P$PsyCAQEee4xX7lxEI(J3kN9Z zb`tiI?-=b))UIE@rPHuSFd=ZpANy?^Q;mx8vRs&){yfM?DZNSyUphx{>fl&0*;~L~ zW6ep*mLljbN)#EF1{_Y###g*tfFObX=e~$(z;CsHKZW&N+gYR|Io_?@JAC~6750}d zUSzEpDcKNQUI?_>msw~0+mQLd8%licbJZ0iQmId$E}aOSG-u|XulZ=A2w#=PvtV{x=7P(qc{*m#MA^?cV&zP_*`Q&=cD70` zlJeCstI*V;d)7O;2oQDt{Ah6!NHL6-j?Vd}p|2US8iF7Lye@EYP|iF;2o&n)h~Nzy zHe~WmO!VIEzcZ?4Cif*_oFu@sy^oMqsEz2)Lp8(-FNR#WMIA%CB|P#HX`aGwOaHQs zDGGG>cPnEfqjf=(M|^?}69o*SnXHU#Z0*EeK=;HI{GR36lbergUMB@v*dc4zw0540 z!5Ad0or^51cyl!kaxMf=A${Q$SM{s46*WZPoi&QoswKnsdF%S&(qf-Nfpem zyzk>zc~=!|(*-VwKp?AL5GI%w-b6V5oyC;(W)XcxTAAd;VNXD`9GpTmX<5y;{!Si$ zU0)n%gk+aF(0cSx%o!6?4dPZ!OL71#h;3rvIsI#9i=t<}%dN58+M_zR*ZCASlRlZ% z78~WHNZ}4Yo?|A?UZs9lKNv@~eb-$lbKQ-p&Ce_v8Gp+M^FUkwyOuS_KI#2Wla~n= z1Z1Ro{niB0KeE{}hu-j7s7m|Ho_#XJl)4n*kBJvFZWPh3BeFbR`G5;Vif_# zbdb#$x%4%0sEvBt<-O@qeLXz_hGvEBn%N5b2WLi){rCTlNMNuAQ_sId3-wIm!r^*k zfRK(k9#L#!y z$b==*K6L*O+^DY!c~SZ{2&SUcbKAL+dDKDpE3?qg*mQKpwuqP4v^c z{<$6DfD%NzH8IWb?+pXwOYj*Nhukqxr_O%Yc2A(Tg#~Hb&Yjoiv;hLx)3hpV+u9Mw z*AUjo>zHAlp33RCiir#%I-!86znUp85O#N}N{x#zdR1~4WpUcXdY$G@ez$5-$(f@2 zgUSXm|EK%Qe&Y}N{2fl$9@YHW&~=IXcP#1MyX_(XGh4$>QF?cN*mVr4eZ^;o#hcGw zzcl-zdE_}&p3uWECKH&GlpMYIK{|kjJd+C14T(TK*&TwSJcrH8Y|*_ zLLwq|-J5-@nMwcywM;-Od?oN$Asp8&sO}qveTKZ3t zS4D#izbeKS(t*D;hCp6}C;ByUGkmqzKy+SbWnK92d$tErqRMjszKX$)M*&=T=vExi zF+TQJV42lrwmr=z{gy3@`QY%L|2o_5HD%Z6WteWuyqc#iJ&X--nw1j=;+&zloY|o! zo=p`tjZiY@P)2>Bzsc?}emY0X*q=5SaI(SH6d=XlY~mGT7`lsEl_3io>uTVOUQ+C9 zvlciKC0W9m6{5knkM-<3d1#2uH59K48_z0xfBy7PgrCKZDhD(#e42A2o?Br-D{l!m z#Kf}k2Bg@-iiheQK7PCkfb-sh9!G$$SpRDveCc~W;iv?K%Q`czg{`629?jA3h4x}a=hwqUoFhvfMgq>f1?xcljOFuB8WE}kWyR#PoF*|&JV5t zokdcRaXCDmVLk6l&qC=-tz@MK#ZDPN8nNy%x?fFCw^QZL3`7fyd}RK(k$f|J!X95Y z{(WT}D}nThf&?i|#xlPs^gWXE3MfXWp;@bDTovIk59ppyzVOe~Y17_>G!A9%r7>8o zNnX2QgWHNGq?v_{y^?ZfaF-hQEH3P0F55hQem(br9#@AZIGK!pOA48g!$j%bYfyI_XH8D@|V^$T50JX#MxRT6DZ38Xy}nG9Z4++4PY(oqP4t7pL>Qc{dmt zC>NP|A&5hfeitaOt*BXerPi)r-~Q9fRsNTwEHF*mDhSy*SXSC>U;};qh{E4P z2Mt1;Y+Qw@TR9!C2!YhqNg-f!!z^{lN>w;gF=?^32(V0vrNLBL=%z4Q}2!hr9rl0?5a1W8Yj0K*+qZvXK)!nK-tg56eMg3-`raV3Uumdln|}RzMHMc5ls1|=>-EmiLBOR1 z_x#|0xiSkctmR6B5mQ7U<~lt&&YmqskH0>y(Zew@MMvZEG|sBEl;o7&T7|NMLb9*f zEqym`yb9kXJbv2@U_io5n^k=i(NE;K;8$wF3m3;#G*DkYv6gTV*Ix34-CF$?Ejh+{N6ft*!c1Pl)1+0)l29)-^a1< zKkaM+o%i**`Syz{#dT)l*k;#$j%_?XE9Z_`Spt(HXO+I8+Q{tvVh3L2KN?+?vDO!# zoB3pA23{1&{_P6QI~*CEIV0EP*Y`(T4qAV&``Ps{dN5L%UcD?WerZ{AH~Q1Hd^P~~ zL+Wq-BBULCKAg{a^Wv~|GwD6;uxJ&~+LxDc@;raY&W=n$f-ddZ{J(r4aAMvoos%oKF&Ae(c}LcmotNfijic#P=$zBj!_OIofh zr?>B|@I?SKeZhiJ)|z6ltZBASWo!Xu+O};Q%l^BjUAoj|{e~6u3HeEUz>JwQ<1;dr zbE=1?rt1&o|1|zEN6QTAryoP=iTCW>R5Vob3K4UQ$07Q!I9y1M@rs}9>v#xrno3Z^ zf!}BJ=FWDrjhjoTx{;X^V#DrVA^mUvdE_{)IV+)UV=E%V1(idKIW9;SbNNA+Q zN$m)8oJqEumVS`L->bai1_Q}}yNbr;{r*Sgi%Q%C_~b8{yW)`1D_oB5SzYtKe^*BX zB>FV1ikwZ@d!(rLmtJC+V`aQjxZPjtzhGD=z%Y7_21ssM;1gy*Bd=b&yl-iX2%N6* z^b{fwRjthf%ZNcHoMW`>z;v#ryaO3ajO%11kxaG%7Po1$F{=`3h}!%fe7vC3{DWdT z=7$?2B^qNPRCoZG8ST~{QM|p|!Xc#lk7YHAJL{_DUmJheOkKRQWsLRzbWs{Fomi7Q zLVg@q{pb3~omRc`tk*wC`cuvvm|e%a_qzdg_eZZys{J+dYGmTp4I8S!0jw_t0$X`| z#`zTSKGABiE@8J8%!lN>qDKi_98D;pEGR!Fm+l&VxBebAy_B1=_~q1P%bweW!dZ_R z^?*jLBL!jf-U!$}?|Up@BU5hLmkr*I+StbXw%Cyt@K}QKk3VWgOX`~$Z)-4PeWSED z{nyxgogF`F)~-E}#~;9CZUkpGqqEibM!5h*Vi3-Z(#%O3*kVUtb^Rc@sWj(a<1(KT zVf*#xa%{^SlABQzAcK}C!93~B4B6_xh+0y|_BMG6UmM1$=#}+>!^!P;)I+6_)Wb7G zL^H&0|9mkMFcsAm@Ave+le1J*B$N7<*|R4^FE5Pz@T-!BIKOB*c@a7n3Ne+2vy^G4V7k`1upp-M42QjE9O;1D{Yk-P|`z#3MFh z#C_%EpU*!hpi=SuhlB%!Wi1jEs|+++f)I^XC;5v7Cas`%k@YHe^?(vSS^ z%8dLLXYk!_+oo+rgSB(plJP6Ok4!x`Uf{0ig8it%wPVcEsjGh5XtxYEOG-2HBHp*Ec}O(q~EWHD02<+dP&Wo-?-Lh)J!l9go&D+dI2K_5WY=>kKi(U{6fD z1o8r=NG_wuoJ!ia??ASKUQhQLFTF~8aVt3g02|{iZ`!-J{ndqCc5{=#U4APq_9_hu z@i<2eB065lp$}St|UCh>*D2Ces;s2#GoG$5RI2 zwSn(p@Sf`K_b%70e9f*`Z!92u%L=GNk-2adC7(-ZB58CJC)Pj^AodlQ$~L<`#h!ok zy+eV+#+vVEca+Aw8?4E5+YPGKv2*7NvG!z^Eo&I-gF%&(x-r*$fX2 z(Uoge_P9}Pc9$5>gi2t(2jhse9XPN`(zS=@JFDr&t1~+Jd}3JS1-+^CTA|YB|9x$K z3eHLu3szLCjP&$IxQom{+l+<`$bZi{km0bOn?*6(F={VKdVI);R=VcMzQ!0p6mR$L zdo%C#3Z=AV$>mj!+B|KlOSR0v(gt{#@@>p=)5&Lkv7EA`bxf-q_gCW)^}!N!Bp8MK zCiB=44h}5^0QmXyCt-h$8*gSO1#9}2HM^SA{Q31Kt*NjQDTgkamdEixwHC3QLYQmAGhN#ZoK~aPS-D+@)fQt8y=msg$g`y<>C!yts+=l$VZonTye?A-pFU=)i<5{ z^wpIa_nR@RaE{#o^lx`6sMk_mno+uPloF+t$4)H`fhVjI;hp90mExpgRXQcyyeYen z>AcyqaZ1hXzu0!6TihCw6Bs4J;$L>s#dD8t`w-a`HcN@^!BEDk6!rD)|v{rn@pg5qID@mX48Qa3Rwwu?n_a%3=jn^{R zCGW}VbPJEBZPTiy6N~z>(U_XrTvdbza`;I~Nn}tn@Kn*{Lp5lAFM>$Y^5ar#6#DSI zJNp%9Tl%jpbB6kQ6q_0w4}YxZf))s*U-jYyDCW;WyMhrPPe{EKt~_6`c%RX+`fReu z$uYRtg+QjMz;px8p1chdK(EdCeHNB>8ls6>d*^p4c%DmyhBB^&c5D=Zflk>N#6GZ)&<@#_7y@cq=*NZ1 zpMCX#^5d5;3PKwHuR*e)f^kQb1nlN@Q& zOYU+c3ARjd0U1ibcz(VD2@!z(#r2RJI>1<1nRtxmdcC(x{5%#9Cl?!g3I9NvnOSV` z)*I1n-qi^i-Dic2ZYe_Hvv>OY=xZOSy2v3K>z4c|6RtF$SN1P0>wY?OJ-2J`C0*t! ze*3l_wvUh)?l=FMdq=Nx=zXrcoz<4+X(vbDJhb>M$EF0x={Df86Rp*z(93@|YF2Bn z{jb>Z+Ry@lcKM~X&_m0M0mqJHQPLkn$jy@R0Ml&!5(BPtmrWkObhV+Ib`yDM-UtcjlzOv1OT%a)lmWY#8>XnoV76}3Eg z-pF%^=ejmbe9QFIu=KY>_9W4*k$ov63rZQeYqsLwf(z_|54QCkdF?Fe#34t)Ab_n>T4S@hILiXd3(^DF#Gy zxTlLHhkH1E|M8Z#ci8kSrvva&Uc(cAp1W6Oe~Odir=5H1JUQCk7#>gt!fFo(PshfP zy?8KANWkGro%h{ib*rn&BO~=}h(c4Oet=Uly1vgiGpZFeFYEB-eD7JHgF92zPaAUZ z;L5n;GaSgB121NEyFIiZmxz3BO&IHlP-u*hHs@JMUrUX$pv5Bo zdiPEwlLjBTVfcdz{OF!8adn`eL%045b9uiY+pUazgp)pP^O-O|q!)zw@$hsn*zN7; z6A%~(EZkouFy5b0bV)+JdA3)o8@~{L?;H{tVrcNuys_!4?JIObOO630ByJ=9rD#Fz z(*=v)^=euM75I9O^DGbz)cKjAO3xyZqsYre;h4fqLAFh6 zy6!47CrTyi^gSpV(S{Ah)in;Q5yLZ+fkQARZi0F2&acbHtt+iWx^F#D{RzN+;J{|l zwD8JE=U)1J-Y+qf$OOemBr{m|?o0Z1+M0iNR@kJl+ff#7ikzWk#l=t@P}mNgppN){ zeVmHU44B`R*N_J{a$PBub6F)GOri~Wrta=fM8$7De^8=7v!sMEbcLzFP1*D94~nTQ zYbGA(y}bIG_$xUP;f9i4O!S`87S%c!ow1d2@o`j>nRtn@s@vB&W+VzvR&_RNos@CR zJUp{K@*C10vBhz2ge`C0)Nw5hZX&wVn+!DE?b7dPuLBh{idKAW#|y3OqG&q~3S1;7{whAjb`Ul-S@fjG zD79ZnJ1lpcHEZMI;?_{;)@k28;xPPA=;DQim<6OC-G%A^4k_iKL*-U?D9v~#A(+&! zVOrdBZg)AXo(#m^PMm}>{=*f%)1Vu;T-}43$(+1o$wkFUZ|@__Pm<-x-9yO{^ery% zR(%B@QvZ040W(s&A|o&tnUc)87p^H0l+cSG-uVaMszy0&wv# z^9k%~ub0M(CucaP&(e>*S*|4TniuKILci9!`8NfKcY}%MAE%W7471Avu0J6>6rEIOXRz(tmfUi?aqpuJ>gH@^Pw6=froZG z&7K`E( zKmT~c80Q8)>?Ep16cSc)wFLQ8)PV0VOSzoh)4_ZQSGUj z$E$GqmIZ!y9T0{{N-KfS{_mF00(Ra7;?Lfs7UfHUhdOULE31m^EY}t}_&Bycr5_l- z>was==T9#0o9EJZo}`Nh#*I|pEpJs8$^EV%Y^f@WTb!MG=}f)H)6zQEM7_f%i=|WqQsTWdDPgV6j;plX4>h=L|9<1ao!gbIDwZB-N@v+UJqWEP zha#V6)+ED25CX_BxL5lim;^jdTj{trApLo{v!#2Rdro$kNLgrSeX%NW>zXZ{6Fi;VI!Q zQ{)r^c_HUpJif9r3Q)sp;>32B-q$R8(XI`xVM^_XxEWp2=`f`>i zuMC7sa1tBH!n)R(yPem;`@JnCa0YEQ=I!&=lfrX69+EbDqW$i~b7)f0p%1CE$+6rf zKk{JB>koA!?U{G=4I+ZPc}Zm2;C>eq;|^nex48aJT^jmq*9w`)AwjE0os?htb3M4_ zZ*m>N;E>{isuR!;N-F-O#dLp7=CXh-)vlNx_YO?QxGbR@&?vi2$3X2$J}KmgM0LJe zo8S8{-oYwn6 z%sz7YGBkuvavQ{_h163#U0gt_c(7n(FOs35c0Y2$nC!3rqZ8wPSJ}F&D#yp&7b!-`YR(gIDYtF@8h;G+jobqIs`92xNnao`-!Tc zoKaB!G@CLhi7ON%t&PZlrHRlOK~ zH!QvGiI0r*GPWe~qc}T%jqz=X2}Rn=z(qB1r)&!htZ{VvnAiSwDB^C2eZjQC;Yz7x zqWeBAb`0LRK5!j3kL(&Z+-BvU0qGZ5h2JUN3DE25`I(7!-KHbZ%QwH0lcnWxW~+a& z>;RxrPVf^hzZ{n1gAw>TlQcU(v1U!1UO8muU9x~RKRPF#(MZ(}mVEVrV;5EEmzp%O zC(l7Ll(P3=F|06oS#Bk+D5D{j!MEi25xM1#cADG5Jc(EmW$CctKJ}9q!r)`{S}kMG zc5UNLQ9;HNp9hqj1{jBN?+f3OVJcq+-a4KbR9-zm(Ivk36t*>@C><*aagx!JCtFfp zs?|x37QruE2)wJJbYy8e_Bqr$sH+w%=;1jIO;>o&S#4A`dd8KA-eXIW80m~fhb&H5 zw}E0|{!O2Exsma51fK$Oq;LnY+!)b^WP1`p9vkbd#rpxuR5({mKLb}=uq>v+;19iL zXpr13?oE!Rmj5x)2aLq4rS?94d)9RdLiVlOwylDivLD&fV7pqfqEI^@IcQ#b8k>M5 z68S;Kk+3K{Nio=55(oUaQe#N;G5fgYAZVKL^U8{i-Ygznc2D`7ZTr_hzZOn?TFj(_ zKSsa)=w0oC=dlh>q*D^S^kPiPPEG5i!mafi87$ga8Wh`ex$mc8sq4L-Xe5|d>(r|5 zx?ydqS8Y?iQ}=1Z9-r8==j4usS!p?WwPTxj?kmpD*smDgvqg*h<=gD&q*?t$-8}Ox zV;`S=Q9Ik(!K!Q^?Yc`msl88j!_@$kMYZ@JaA$emR#5 zCBdQmBkgZ*UbX^hdh#IAX;%}A3B(m5M1Q%wK%eZ^(E}GQVOx>h()$N`wG9rk@M`Q; z{CTpr-rdy8@1_Anahs8Qem+GcP<%1|brutt?@dev=$*tr3)~D@Rh`PQS6lX7D}6rY zI8PQ&d%Zmybywr%VDX{Dw}R?5hU~@SJpN%jxB7eNQ5pI(j740=Jn0w{F`z(d?DeXs zJ_*4P(1-l{|BtQrfa`gG|Nj#qSrz3dE7@foMONZ%WM-TwWEE*385zlnl8hoDE5}Sm z8c0SYLs>zQeE+w9H@ENYdppkO^d7J0^SZ9b^%%H=O=&X6JGgvh z-c>#04NO*K#BbK+|KP?XR`|<+~#rO%s!HmiMMzdrDZn8HqU3_@w zN!mv0QG_jm*eCeXytD!Q0pVLCT4A|l*rVG0W2BJP%&Aj5(s+i*D{N7Fc;o`W9&qvb zdXNG|A%+8u>UhmNmI3Q|pP6w(aWh|u|1jqF^E+cbnCU@C{Yb>ZUszfFAlWeBey)lN z8m^UJGv`PJ-?7>8#XZmYOzPSJ>w4Tdd^$H2P^qYw9VDt1ns(5@1aT4 zdS~F|bhQk?YrJ)s>$3~n3w5oV;JI`6?r=V#dwsUnfMaDK5n0XWoau^W9-2=C{(e2c z)707+Z0Bl}CYIW?t0FvXLPw$P@qv-yJSA^^WJCTM-y>Zc=B*b4+EqU$Sx@Y^Q&h-;qx{DX=I3sPO5j&3oK3;EF)p9b zQ6f($&%DmH&h@l3)e#78z(<-w&mUYAuB4UnZ$X8r+}_L}t0iRoAVsHfU!K>Uw`6qO zs-avDUw=#?v9;vpa`TD%-K$&zv_=~DR;f}I#5vB6Yc*h9FRzJU456py*BmsYTrC() z=BxJ2CR${h`qXtj_`3b)Akv5>fs#eXWLFSCF2d{3m+!09+O>u*R%vmxp>crWbyR;e z2hpAYNuTgpet~apfg)kB2=E1)rd^BP0~(18r8ws5&w3bGs6I+<0jE>$v+|{HK|GQ6 zqAXQwi&wc3=qp9%tUqyp+_lwsB-Wg`ohmD7x*$k(kxCT|+>13Q91n@&kHU|Sk&4~Q zf(}@u2@&gWGX!h)voI0++q{K}c(Nc8V+fJ85|9+Q?viq-`gz=s!Yt$T5r=jw{e{E4 zk0oQ9mm)x!@h&8Yy?-h`Q-x@ z0$^-dK#k`K&K9)sNR(@|YyCZjgO`VOY{VWc@aG)xj5cZ-iY(^tt&)AST8Ac#wXyM) z)yLQVUpKP3$UmRb|9pA5p{*|+C2ObFVZ|3m0a*z17|g;cW)vxm$b+#GeeomtR~k`a zdXPgtJyT6pefR$TO{^GB1?|{j*tTw*Hd8()nBZEkoDYz41dm{Zzg@bkF*m@ykHOu0 z_MB+dREr2G@tdTRUNLu2b0!^ebb}s|m^jB*`{BS#EU;m3I7Bf*wd%U9bf#8n{Bl@S zYCswX29OB-Q; z@)g3pG->JUQv-GZdAu>6_xI_Rv<20Z{l=p=Fv(stEUl%z-94;XkLSx1cf1BG_`Kix z;)!k1Pn+qf_OL2m)vU6Bg*sULds=?(9e(KA(ZA*mtNq~MG8Y{OqGNZP`fVAkP!zq+ z*;d1Ud5sg!KL#@a9kgn{0Q!Zn$HK$!E7@a`)&R=0@Ve7=oWg|*Y%}NyJU5IHNXxlt zy#hHIOek@*((KmE4wGnpln5BY3T0I6#A%*tQG0eB3R+jW6-}CFB?4dO5(H{BpGHmK`Xmyd)*xmzf6WEqOzmq{jzxdx{Eul$F*($>m+*7iF|@!>5;J z1*XnI{@Sr#-xAPR9FkW2GGd!RU-i$BT=TwtT}q$f9NkAx|IsE$4c=^fY>WXEAdRin zigTPf#oBtd9$?s@N-chT_wDY2g(1imotVu}qLU^)EUE1W7~GQaed^Sy(Y83O$@w^H z@I8jb5*)R=2(GlqRUDvDtPi$w$Jp-9Ef!h>7dAB0whRz;SLTnS=1@dJdbzDF9DHZS zz0xar@7~3Jws6c8BX(I)o#;?Lcs(*#guUH1e!CoDRnd3aA1aZy;Rg*BTf*3%m=aez zfaxN*>t}1BEwOq;Oj}8u;@m0p1s}Yee7M0C$*OOjUF_g z+`AXJkBfX*4DhPF`whT*r}tJ}4Kx{wjX2I{(R(soHViqu-DGYYd@`Dmbu73zCXx*@ zw1ww-&?ZcEp1ws8NPt8g!LTh4C#~jsglr-fZIfNWNojL3V&)&XX0K3e8HDTz&c3YN zZE;@Y&fv+JgBoBnG~KKVIDt=ZYU{{1Nd4JnqT9vDh&bzt0EghLCTaHF*-i{NGS1j$ zgRob{TO9j%kq_W8I0PPAw#bs9`Ae)K-#Dll{J^J28jS3Mkj#FTY2Fd>}N3sK~k+fe2@XKfy|DWft1V>$}ffPkhQz7pE>AdJb1$?faDHs>uz7yefP+S zi&L-pm9+aYGV-z{4e$-2Q(2QRbNA8%kYS{UUUFc?TnBpC?b+2;RmeM?8UFy0zmUJ- z14YI@shA@;KHf0qo{FZ-<8@u zAq~Q^@+YX9_rNv7=1){Cj}4u3cm2aFfAnp(*W0jn6SAJnj?~$eb@cg*(6vue1}$p3 zTmz&+0x+&%4W4NLU)oz_m8dJ#jMB_=|_FR)Uepv zb9ReJy4J=}YTSdXBkSw`TX{Qj9aOUqOZ~0>_ zmDR8uE#|&nY-l+U2p^u=mxmihh-Fby(jb&L4>ufElT9qO&j#iA^?CF3&2%gS9w5hH zFk7~@#V`1+{_G6wmM-loYiNAeP6h4Vo3vuGkD~=suB`Eh{@k9Xt)(U1xWH|bsQMI- z{iQ2k(r03x*XiG(WxG2^5_nWlA-=oH1t(GTEQDxDb|v)N$>%USz!28cA@Xj(F7Ua4 z#ytN^6D~L`B4zc7`wHEV#~7RgF7NN4-T8H5@!P|b8C>QyQkl7O>bce#%?|CKlK8vI zTpaILpK`AJncw#DBekK2r*!S_Y;pf!wWvWgNA7NudXA=W)bYo?$`{qhb+Kx7JFs4h zi2>Y^LKWkkY%u&;?pgZq$Ox=G3Jqe`G8@9UYaEgA02e~xU15Be=|sUtH<2sfNwp5m z4iJb-3Z1VwC@;D3m_-YTRzBD^!t`zchK0qw2G@cLn;1zUD&uop7T6Rlou_=mxD%1a zpRs2YYa!t#E$F_`9snVt`2Bk$N?t>ZUD_jfAoS`v9X;k^(Y7q3`E?f!&2zCUJ3YH= z6K3vnhUULUh1;C3DLK*D^5aEe1<2gz5_hc%=^r z6rWzWdV~ZJw}Rb)4+XfzMqeqF9y<5Brfo3^mOtUklT=~2`eo{3@RJ3_@rV*+tF zQlaka6B1}>$pMuVa~y5({2`>_r}b8Zh9tk#rvBg>(+Uga^y9v_Pt?^jD!b^Q-}c?R z(l#XjT^}DlAg^OjL$yoS6L&hFuVTH0H*!j?R#k>xe<-_fa$Dm9-^~ph^w@eo)M-J@ zW`SQd%(OLA|C#xh^C$CqKg-O04}D7g^!WX72K05F#4E^xlPN z)X7cUc=gxMqUfX3*fTR40OMF-dgq#pO9~QZgAN@mt!xt|a8))%WM@WvCv@l2hnU<5 z^|J0FqlKq>Nc{LDIElk4dSxj?vtVy?-xFJzE;ZFI)LJb*OB!S-~=`^qQYjspnGX zqO_pI@Ga;^xRo#LS(?jmReSirEhp&^P3zaMuY-;T&HMVP`5U+qEau)8#%?{xB}73) zqzQqDScChqS0Q>YTXyl3P;E@79%(!@z;Fus%O%FOJ&Kn(V3w~s{tj^m3?3cEQ!RH~rqO|z z-9bSc=nv}*{~}WmvpNS$DXiG_vTy6{=P_)O)Z_JRE^V?C64YS3;z21uTQTG2om+iN zpZ8^;RvH1EIB{p*UbeI>v4!*B`D8^?cfF&|QMMMAKKhR`GJ;)3>#8KTo3;VU(gNbk zKvq&>rxNNz4~iGun%LNKtz(B1x_9fg2J*R$D`stUEsb{g=#1wG$x^5?zZv2jfPidl z`fgan+CUx3RZtoYwxD}Q#&w_|nc=9=JvM*1V1s}Jh|D!%K@SVT!S)A+9_UJf(pZT~ zwi$EbNUMKMV;P&YF2%-j1+1aLk1ZctKzofLTt~<_9-ix9(vKQBQbwRF1vz@fz4^=# zS)sp>3WWet19Cd?WFOC+}tKR93 z&~6wFCI>J056q@lS_7Jc1@pYijJHe#qA-_GviI*#g;xNxwK6igH)HQVzB;IGSuM9Ho-dnz}0N5 zNU;m2`SN3i&=@&bL)Q5+|0@##F#zatrEF%ZSMus9-7~qw%x26}l2*%exc_n+=ZMcIbG|5f@dy=LfE-Dk;WcavO_6nlIxFLyB6Ke?^#Bmvf;8d#t~J=mkiu1a|( z^h5hnoAL%crOF_SiTV$)skg5e1BO(`K^}@_966haHhJvD{=n3ipz%g zU>NqQpuhmTX)C<*6k`tbPXJkPFd;P%d|C9FR`6}yHz0-CI0&TQ<8>jmSmHIAIC4JL$@Y_zf-`H}Q1sw@8u3=lD^HZzk~RUrcd z)u#TlH3PIX!;NbCHHOo0BjeOxi*VXot~T+As(Q7ihP5uWy1nmb?W*cy-F;QWZ<2wE z#Yd<28l?l51lQWX<$mal44)_J+J>pueSB_jNlnu@)-kKN#R=OB$9IJ0KhPV8OP0r{ zmI5b^O6Gt|WPyC(e$VSfQq&>HUJV^*Xgg^it`3fzF&zdmSyEu)MRMV9NWx} zW=POBIx2Ai6NCIR!kbh<+m0&4qYsXs8wkEKYLis~=5<;K({6V|ee>LlNo%$?;G|B>cm+XbbHcD*QIzzKL7Sx$Dy>eLir}iou@2N5Ev+ zUc+n=Y&$E3A%fq`^F5eOK&ao;wQ9cmlAaX?aO4gezVxAme80!Sy_%=33;u%y5q~rMp^q7xE|3T&?(rluS<1Y`RWR1MZO;RQt8<(4Q!{=AHh2!PL)3* zjEp_{qc`gVsH13Y2h!;<9qM*4#ux+RPo13I|bx{;;wprA}z~FnJ<;3|v ztTNZ0i8-A=@J=TmO1!UjJ8{PlKg+kEEyBIQP4?QN^1;y1flo4C(q!Yv zV$Vt{wDe_u{yInoz+72kqRG3~3f`nY6)!Ru33P6+ZhRCPZz#(eOyHE`k>r0GeJ}p< zEWx>M%^RT=mlF;W^oyjK5m~aXE?k)Ob=j*1e6kV-i1L6;i)}#G1zvuR9}R|2Y45A^ z@LA~~Hii)Q)Jc}OhI7`DBZIB6s}NLSA0rHwRPR@M(4dI=4Mn}E0j7Ta)<|V( zzbZT-C?z>D{!I(2x1Qw$1;xnu^dtavnwWDZ+8PH#}*rso3J*Rh*Sf@r!b~RxREM}g>!IM zMg8N44_9a16oE}hKyX+4KdF4ocaq^yGHE2JM4h+*>NEX-&+?i2?G ziV#f3i>9p^vhh{t)Uf2)Xm@5P>uV#Oq{;n&VPsiWx$W`rs-cRjqk_A3{9XmRW_4-OBp#(iX5EBdwiFJ2FI!M+5sF4p!+iwnlM65H76@^_A z7NCosd8Og<(VfkUj6sg`W<93bRn&VFy1e4YMVGM#N5sM&Oj?LHmmu3kXGr%fOrd z2-c*1`iW{SyL3tUsK5R2LUfn}yS;xpAC?LW8QFBP=cktH#%-Yuu#yt~5eW#c ze2}+zn1%Jz)-)e_dI8%E(QE=T+kNl(5jF)=nt_SQ&I5b*4YRPQCZ{0^@f<5Tb;$l( zZoz`19a)mzk3mJW(A^IC_|zbS=*F)fDZi%M+bgWu>m{%o^rkuZ50tI{==Ez(X-C5H z{7EmXbe`okVCCkX+=hd*%}~91_3D*%M{vs6TBVEWLn_@EWJ!u%*~AHUj@7Hx)fV5F zUwR1}4ZpX&$ta%f% ztJ0jOO2Rq2M^gy0>tPrE_*vwY$mHY7;PZ0>|88f;*kOX8g6987_xic(ed#KuPza}; z-xVB6Zi$&6f&~i)ha=%nc$ObOo}e6;3365er$m-MT{ew}36DV&0I1zsqfVW*<4)`_ z`OG*n>rwYwk*s8Kt(YZ>Z1KNb)H^%0v*YuEg0^~BKd9DCLJnJl)tB%i@StbHx}Vjg zaBD(4^mP=SE!+ZV*Xj&=?}1vy)66Et+N~xE<$@X((!XWHK8N%GBKxg7mK183p6DfuxC$G zfsscJ-l80JWFVFYhqqwidlTd;Vpatm=o-QaK+HnS+;4p>>H>{WM4gzF9kic6e_mh_ zq7G(zWo9OS@dxK{NtX1dY#ALq?+uT9P;lh{vZdl@!RuJC-@;D7Y3UVp54ZhJEN^6U zJgq!u4Z{JI=+v!mD-;@YAOX=&MA*{xOgmG9M~~)Gnmizbg6wvza3w&A!^C;}Sv9g9V7iJ!i46cEB)tjrN^71*XYn5N?Ns zug~pNE_`}T$?UM6Mkfb|a5Eo$W^Wh+q|8>V;cBu`MwpS5hqvPTmzC>F{_nybRA%1E zAh0+X65pTxoHcXiHnyg;0XgqRz!YH9>*_p^c!mG+aJW+Iuz=^{S?Su=V0o?Z{PP#G zO@2LF0G!>s)kh~q`6UB7wXp2JLV99uCOx@DE_pBOh zPO<&8a;46g#M!B)Gu94X{!{4L@nhTXY`<^G#%b;v0W%LP5)Z`)sQ(*@d)i{u3Gq6E zXE78Qj$P9l*-MI4(iVaQAeQuIdGX#F(U;g8i$krDa3~H2dA%?5IO_LX-Uw>X_Ovcg zY&l&zAk}LFG*v{y$=4_7^iz>hOFHyri!DB{`*BSXINE~GRJle);iq=gU?>sb+#ZI` zcj@0Cns@5j*tjMGDENaIAW0TrJf3-)dU~;r=1b?Tdu86jy!ToG>A^K<5%?+xAsk}< z{P6W1Ko!8CZ0p4y1n=h`M&Fu~kC0uo`~L8`t$u#<)Kp2acn}&f$!Lj2BBPV^bB6dt zR%OeCTLV?#CXCr%($_S{9uP(C-ufttUS*!2qa>Cfn%}_hw(!lF!jvV4C?ALg&EGYQ z-lli7eZyt@!7jJDs;Yc?y|eSWa#-PXuW@~rnp({-K^5f!LtjVs?BH|)JS%Tx-DxYU z(S&uoo|F1)wmW;2n(NjSI{4(oSNU(nmW*K+hT~F3wCmo3_KLiFt_#s+bZT4MxDKmD z0!Col#m}2e&X$DKxPK~|F|TA14yhoRY+PS@@Uk4zkKmu9WJ@g^%^8p=KxSaf1X)`G z1hwqeE%i;ZyIB~^CP99jK=s${+O_MT;`3>0B@-6%+TKMgJE?%A3C_v5bN!v+Xv8ls zush%Ow^#0k)8kG^7Gci$feMYI1a5P1BlbX3ls+`Mq>aHb=^m9Kv@KffQfiq*b?Mxq z>Zyhnx|*zRNFIb1*J1x~MwzF{_XywUT4CmJIgfl&*NzC+o<13`+at%uscg^b?DKVTLhlw6%PuSoZ(MbJ=lreHY;_f? zaI@I$uU2Lcn%!CRF*s`2gE99)9aidm2yDB#NqA^JQ=OW3nBM8Hqi+m-w5Gl5h1b14 z>qO7b-NJ{(%guD6Qj=F+_sX{dy_^LGxA*s3@gO6kw&bczja%MWSf8nIrOHd+!hq*g zh&Pnx*nI<`BTCFHFjuC8Y68gKLNt`3Q}^_^?>gG0M^oKbg+2w0 zP}~Ayq{_s@`MH5xe)cLF^$YMN$TAQg3TOh06HG$w>KCkNeooWWxDh(l^oEcRCV{U> zI56is!j?E4xNRdroUp_Z+T!GJ_^EvBkQstL#8OBcd+a%~W!UFrc3!-jwqe+i3S+}6 zG$wG{w$&JcW1XquQT*!|24Z{)jYBqDpX9Rry`X0nT6#BqDc6D?tt?Yt>%Ji+#*Q;! zhoVYvEHK7zL<*jEo#9_U7C86ni8l?2Bh@g`+|MZfqbz^xl2Oent!|S`s=bUXFr7HP zx>_QeHY?6Mo$Rt0+~g#NHl$toewHN;`Rr*1h8O(PLZN}-(-6}wn>NjC{jp6CQif1F zBY2&c^8wfwwhi#k!n^KMl24sIYh}7*@nXS7;P;IFA1ibwx5K^It&BM!tz=-Rj~SEF zX7sS6!KfwvzYQG1+&|zMZ%ZR+M-0J#LvY;yoIY*!`ca*sHexxaT%u@mlKvMR@VPk{ z2b?BAQ~@z3g`crIGa2%=iAiH-wT;6Q+P**zCmMFXIT#IV?9S{85C3|!lD;Dz!ORw? zol)nk-mKFpIcoEb>Dv`UiM453t!d-G_N!&!2j`n*Z}nQTV$JSwx%wyf@qH>n-3B|>-uvn_A`w+t zi7SCPRc=x7cG5RrwKBEo%|}U0i83UTcO9K3E-VnAa9@D36MFS(Zge;TI0`3=MJ>(r z4ib2Wc#ttg=~W2L+V3i}q08}BR)_%jI2?Lrv+8e2{|*~MReT0r;A-&DpRaOCR|Woo zC%Rp&R+9P9D@pafM0OV<{hT@ZEPkVH@xv563+H8{yD1^D7uGTX?Kp6LiU_g){8O6; zK=yFJXS|mWU-!~{@U?RGkU4B-PkZ)7!Q|kgWjDS1Iv%>`*@BUp(aZ1Ti4&Hl_3PF1 z#txJ7<0ARCD@s4kkhnl$>5Xg~=z0pT4$h%wu}=GAy|EQi%h1Qv>CWvY8^Xsrok_oIdpu+VYO=2%Ez~Qy)ZEB ze?P^FPJ8J?Nh9L9K^9ha*AsF$1rrY3-Kcuho&XkefK-*WhXtWp*q0B>TM4L9+0F4> z!Zz{S5#)3GqD>q1tC6Fp-=3~XDBA5q-<)^&d|G(Ub6M<3#n7ti6Y8*`S&J(4&CxFB zbVrxERx@T@^Q$=~|Gdtkk$WEWpF1!m@{e_Y>Rit5F%>xc$idn_&i}3BmE>jttN3+~ z&l^~Hk!c;Au&8RCF>*gJlnX=%^AEQjCQL8TADQRD zamBZ6ZS}#wI9%3#n7|<<+CJ`t0vFd+72oDecAXcvonwrt^nBky&A3>$natr6ttX$K zeSed4^N`@+_RbMc+4@q_J9i#F{{wp`q#?Db`{YWO2gB=O#O_>*l|{jmhliqSBDX zE6;`&tQ?yC-#_6HIHn5l2HEJ|?2=BU3;-hbW||ahZdm|Xn*tB9kE&E9V4R=Q$Lr@` z&$6?{&T$${)+vvId>qxE745!NQXI45ep`dkiQ`pkFPhlu@gV=&U3Ay4xOgpb=exF5 zo6^*<6N772X%s?tUQhhX+rJv;XF+w9;Bc*)v+up$*X{7q?+NMGHmg}*H{;iLAMtYb zkJ2+ei?dT3Tv+|9P9A6qORhZnt6e6C5daLtM}XH`XRoM^73Z(UGepv0CJzu_{o+C8XfFn zydqx3H}29cX7`!Bd-c2~MoyR8HveK#phDwBOicReYI=JoKOD#58494D{VumfnavLC zJ<^?;X!bz?v#F0A`^^n-oUp!ijRHSl-!hT8XD*}*#)FPw-^$DL4{z;n>*l4cVT}7I z9cHu}NGnoIy}b|DdX#-?#p==Cgr*&M1`}`y3R~deV{48>$h52NLVN0B*CdJ<3U=t> zhUz^_<8H#48`*Gq7g#UC*o83Y5<}9sus_AV;CqUJ#)-JNW&wtow`3PD@+hC%Zp`%X zM}*(D#{H)5SR`yb)HUPm%!`(Ant;tjsME2ruH_PZPo^rzJ4TurJ$w2FHK>F#fnd=O z!nbBu)EdS);kNjNMTXE(OvmieIw?81z!8Qmy%HFfo zLzvle{LSb8J~5hcLQcm(8wYk(f;G=Pn@AK0Z3^;4c1icO+87$xUNp3fM|;U?{ev@l zk?`g>U|%Sm!|PLOn72c+Mk^w3e$GGN{>_Tm^!doZIWh|;wz@g!XI|1&|CUQl9uIGx z_4~+Vjs2Gp*D28U#DL?+GzRwj6*nL(v*V2Ew>DQ>r*>nA#)e;|SBWq%df2Hs;Rq_1 z9UbjAXRYxH`Kf#5=MO^;?BJoHJ((+hgK+^SbwwRQl-#nCgDUJ0;D%ghVt0*#T?l|k zNQ6pGL)(s_9fcMGnfFnPMUVvbI!0FgVN#+``S?k%vTBv!ojUqSp(M*S`m*Yj7s>#*EzHHg(?-oyW`~41(o5n41kTT_jgLSrJ9TI zb!)F~7HWRgR28nnS(d4F3rruBl$OpJWNf>TV#E~I%kz6=b~Gv+Quv<{iu(a~dK-YD>8fLS!~UZw~8-IeoO*|ub}sN{4)c)hvaZ2_kZGSq;< zgA*>7v{MmQa|R(_d}o4F_Y|7!x!Bk~48fh-*NfX^Xo-375DSYn+4`H!xiOlFO!@5D zvt+m=O{Hl9KX8bmkCx{@Af5ID3{8Wu{<>EqQ2NT$)JNxQhAk!Gqh+;w<4C_xS%IpUCW@$H#V zK6xZxpacAoW7u)0)Aom2nyK$b?m1N!9~6+e{v4kE>fxL+7{|N9Z@~BzjN&`JGyLk*=Y#&*3BjIa#{CDUFSwrlqb1( zFRoZT%-gD?N|z`tyUxEV{z-3N?3m#rn^+IU%jb=nHrPDaFhB9Ys`!c1R;fcONZtOr z9_&KVRRPNE37>7cSG>PgmS!*gR9vhXoNNYFUTCp8Nq(iIR&0{}8@!T4%)VE{(Z|Os zySC5uR9$VU{8 zq0}+jZIxg1GPcnSLDM|R6fKoRf^qWQ!mE#O-w2M@b-*y!DE1T*9c5;=%)$$1aAhs} zTs$!pXi!W9g&mYN$yQ7~uvuqL;m`Ay*2!<$!%fHCMCRtutKl`76&i>#2@mXhYxU#DON1$|?vz zJ<9{>M#=&6p(qrtO^DRO)=h}hJjxN@;pli6fAFHRi!B0fIuXd%o}R_qgiGVd3N&Si zcbmYMl3pOE1|17hPJyP^p>9Q}WO9>4!lm7^dh zsvf!xoE})V$It7WJB13$^EH(2joziGbvuLQ>Li;&9ITZ6V959>yoID5QG)nm8y=M2 zmtBo|nBj}`C+Z{j?i$N{L$=AvyHVjTdSYfQ9sJ$a_XpMct~sidse4xB*9MOxan-aYCI79UCwBtHbkuuKr5*6_abkt3JjVb0 zqZORI>;s@QN*^xSb7`1+_|nmLt{AQOTpzgSEC*0r{Dsv3+o2J&x(&2CfW+BVV!RRf z^~%CPX^1T_9TnyHgp+FJW<(1{zz%}wGNnhYE_=)hU%e7^@3-b#X`@IVahjnIf)BCi z&A6bW%Hw+L;lZHSr@UyLPBMQFL*SHjp!io`0)va4b-I$Jn#oQX7C}}TAX$ySTP^R& zdrE9WGARR5!`F%PPQ4W^4cNwRwVdx7{-n9zn9M3hu3c3zcCu z02m`OTP|&Fv&0$IqN1aPgx_V~UZIu1o|~C}%l}85_G;Zye+YB8o!|-g&AoU}GT4c% zVEx%8yB+SBLTt~TNk=qB&YgGe+vm@xpdp3rZ_M?UbTRmpw_iwfr(Z8q1A2Q0 z905MOIeX#WjI+NV7qt}pNR)!za0fxwQd^lUeEc>yO8m4A~R!;ick z_PHhiL4EyRtGV+t<1f>#iqjCeVab;3Mi}v+(do#h%hq>~CaC4(GlsGKGxY&QiO06ltyKaQj| z1Ml2Su$w!!My*;&w_qi%4R|n)eHbMt0nEZNjOxJx!O23e%&|w)%t--;NWPFWb1~6UU4AL}qcmHL_%F^T6rF)>}XvlxS zJHc5hD(m#4Hw)-0kHu|>nt=Ni7T&J2U}TgeDKwy`w2MgZ$&?05YK^acB2TE?SP)5| zYcr`(!$CkUcedzWsnv;SdZ1krY@#kqcgTo-JoFSs3W0@OA@L&4xtQH@cxoO(ub}fDD4^)gEddMoijYEs(y8FDv zwSXM^mjh3CT0i{AK$I+1;d7IJg3qEq1bsRUBQ6mKaR6RfPP#;KwK52ZDF-|#$P{9| z>AHfCPL2{CGG!HnwBJkXI&y zZ1VT_hGK0P*DH*!iv<~L(@r#Up%?wapeYLxkEwbs27QuOh<^pRz>F|}ybliiANQFGhAFontl}u z`1%|lgf~JH5zfCWZ4lS|z~##?4QgK9xF-mDgd3q5XRc})v)yl+>A`{wzjGb!TNYHA z&%fn3_9$DiGKE04qQ|y}YOes|T^w1E5Hc zf3vd^m2=tKrQYST&0>w(f2q&wINAYu#T?k3X-gTY7@swU<0|`E))MTHuS@ylBTJ;+ z_b|pFr}D>fi!4&Htsp6K-j#P#_}W=!%iYAFzKh_T)PLGqYOz+TQl~Nq#;*8eHx!LL zV=;Xlv~nJ+ms1uq9uG9^AHS8rrk<-^nQS)o_No98+KQ zz%O6=Xloj)ddKQMwv4s<@_Z@%gE%jTwocW9jFrgsBA|8Vs2)f_2eZ(sMBe(VtEPj);MiXO#EW5E|tL+=T5hT}#a{>`gj2^}x8$DyuqQ}-=JTZWx z+fgKT4hdw=NAGy!SNX4stI^(dSR%>^OX**}x#wr9ZX3AE=uDHgSg;+JgTi7SN})A6 z^US9Es}JD8A}@u0CWF%`)2u{wo)fvOD1PaN)xQ1F)c=-JbeErhzU9^;9~Pm!k+bM_ z_-hH-6gEt`O}KSc5-Sl`0xaM$u=?f*$TPX$_Rmoih5joK($5A(pISY$s`0`5rsQ*|zDBwo6ZKSzf1^*#(0oG=BTZ#Cwe?{4>_B&l}{=V}SH7*AiF|KVu#(*)y z&FC#TNkEihtb;kW`yW0&pJvV*TPdjnl*{&9dgd_f6{4niBJtrSA?csqH8oS~BuHs_ zy8h0l+^n{4Mt8Gjqj?%sEGQ=!6RB#ULnd1Xu=EO zw5m>=m|nVS<&klMcVre{x<1{bC~KUg&ix|MrcP(Oahb%U85qsKX8l z{yCQGky$!0dr1|ocEZOx2*fT^LZ_^>mjA)w6ki^k4@&F~qtA`U;Z^75C4;y>h@+w2 z+l*pw261N++CF>P=_If494d{syp3-s&E+~pU5Xi=M&b3;5j1vui#GRRS5e4ks+wJt z?G|O7skV#jwSs?{qyFiy{iPR!^)9{GtDjpHpL2kjrS1`fW8+C^OWEtgTlJT3K}B!? z@76pJh9!kdA-Kq`u&Q$vb(7EqKNBl-l{O2V?7dNcQ$;=smL$1HR zV(?(^@UJv7Vu=M3XcyO1ScJkx%$nDusoTeAW-H4-#9U6Ds~OJUAq2ev!`<8aiv>W}< zwH}c>B%4)i&{-(;C1IEDS>vBFCimcdr0&HJIqYW9JbgDV4KK=y^P4npK4k9P4t)93 zP4~iUd;R*_?my>7lYWjHC@zM z;*Yhn>l`MaI2q?(`*QzY=Y_xc`0>hPO(2*SyJF=cq znxIUG3=v!$mfWs1W{Cwn2&PX~5M|+q$(M2+8emvlI^;}`s&G{>+KA<_Oa*FEntntt z;(ume7(ho~<7g`_>Eao@*#;fTj^j<=6Q|>3au}RB;vA1>gnx0 zKjnST73UU;e(hle&nOJ!+qlePIspq?aU})QyUj8)$kX*vz&3dA;uKWqX zLZPQntEWwC5P0YFUC%*-28pQ-`XM{Lb_T~t5M!6YTB<>u-LKK_0)Yc7qYUK|7`>}J9^&YQLP>(s^xSJ+50>9?em<=NddplN`0J)!e?m>@%#LJAM(@H@oDwm z-kcKL;>QLNQr*ArKdPGbHj&3b()usn#VOd+vqR@Yl_t?j4II61Qyh6Ks?!HEVc_68 zh`U97r+*?N<*Bg3jUVhs5BKq5bf`lJ6>|=RTBoqay1K2|n_B3&q2|6(R(8Ra zwP#Lww|#%%C@NDDt31Ywe3&OC75!m$D22%A2;e_(^4CD8+~(nS?2m1(z8@=ItOWF!e6meBm7pLQAA?s14VgT7uZw=urf;p%tNAJ*X6@>Xp`(PjU4BZ`4b{xTEz*hqYM0hKxgQAMRWd4?t8)8I|` zou}K>Nd^Xqx1_8*YhtW*I%-Uw$BqW(>A%r~pT7T{RgSL8@F~o1o7&CC$9_Vq?PpK> zuZudbuG@La>LG`((~&3=Lwkxf8@v|~MpjB$Z>x|m7&m6;+ z$9xdZ?|5_Y(&Dkgb3K3#QsyavBQ>+;3MpCESFA$QMC43xs|cCapvt+hV6-Ekw`$j@(Gc~~G+O(bbfUe%K9hb$l5Z>zt* z`tX5~pT$!E^_Uo!DiZ^cA((CTdtHxufC<-aC->Z0j&J_(A0h~t*cfMt`2-l%=FN5Y zo9j|RMwvcwidq}c&@2e%KxQv+Oz`%#4^J5}y2|1jR#>m1CA=~@D*%#xt&jFZC(?57 zpXw?=*nMl-AZk;mnhZdKFq@TqHYVJyU||9TE;{KxGx98qYlrkJDk3zY15mQn*i+vC z7{XbLj5#X!YR`#yu}v}qkl%^ zrgwi;VZ@q0|Knwty0CjU=j-98JIHA%I8xb#8Qq%8fTc4HNrYZtctciD7(<9V8&pWw zFS1cOd$ys_lOLJw0#-#+%b=TLO;C`6*^52$3z~=WnC~U`CotFy!J-0aV(`r+b(jj! z*0%yvwI9`s%kdLTz31d?Up8}9|F68wgkG7RGmqO^2@y*Mf|wz@_Jw6m*A2T3=UilG z`3|hO_y?x;OSJexS%eNHemmTn$Q;h>)zYwOvBQum$)w8|Bw?~y zL~sbigJ_H0jDjA4?6Ed2i{phFd2X-X{Ib8X2y*YY`e(U8>cDR$D~wgBY3JwasiWTW zm3|%|;QrUE?wnNkxtMgSaFI&f{8vVRzs-CL{voYJyM7sO(e%$no#mfFmrhZ+Ypf+>>*jzKbBK3knzKabrrUVvu zZenu~=>O|SPsPjIP&PjyOP(ouu!)$UEqqy9;FmV7ZFd)3@oK+wI80CoWGbr}2sm{! za!N%bjoxF2`xc4PQL^I1*06_Si#YcF7V8x1U%iFhkc^=P1qI7uVzFLDl~Y+vJC9P# zPX?TYCT%^Li$&5r+HMQv3KPvOyO*wVZ}sQ9r`-@z^`(*-k!J-HV<{V}112`o1F#xy zF0-iP5wVJ+LGRY5Pp2?e1bTe+VxMaq&YDP^W0!mpshmVP^!eS1-9EuK7y%VpPgSrF z>I)n!l1!+p)aI8Ldl)RN6JR*Q(Xlpz>(?*Q?*>x82m+O)O!%EJN>VIpvpM{gq|?CqV`*D#RULrAyOE?D-^YDB2T! z2K%Xv3~@mk1)U9`laR6W-27LGxr6^Wgd&>Ib(me;UW8cDm80#c4CY8rr?J}UKNf21 zK(I}Ees=9M!tf9?`18;4809FS`B7aH*n@{Z!77KognOZtihz^*A!-Xq1Il2n%F7v=ddWt9>j3 z-tQmeeqh#b984^n*X!>lz{mL^LBH+TM#j>B$>^+09gPvP=h?ZB+A@e`I&sxrW1aN; z)vG9bPLcc1?~5hLZ4v_~($5#Vryv{;oH%ip>obU{SW#26YmXd^JUsF_uR3B6F(q5{ z&GJ5ZX=pHULzjGFI?vVB+tBhyzW||1Qy@vya99j#v9jV@q0)+Gq&@{8Q#7YKe%|h_ zZXdz{;Q8k_kAipa-k!#%AjJKJ^=qW|DJwCte4bEjwDtN;;1V$mrrT<1`L)yn8E@XT zHPyu!?_ufznnu*4Nlq&p7%caa05B}U{n6?u$r-8tpk{PKSG{BBPO#qO*c9NbbEKL= zJ2rCby|4z5HsfMY8uR6{Qx}FojgX-nbC9K*E9O9xClI4PwrN3Ko0*-Ih1&G#1FVUN z%$xF;!SXKjQ|R=fR+wjDGRy>6gD4T(-T{U$_;lK{+RO^V>VQbFBMtDat^6^({l9Ic zKSu|xD1#99J5;Y5Sy?N5wX`&S|LottzD)m3TeR>BE5gQl65(7>4>)?lx5ycfbBOX7 z?N2p$VOwsvW$3q0IiibH{;2$GH02ZI_kBj|{ag2nY5ev9C$|lolz!{_)65!B5(Ka; zgF^J8sHjWtrg3W|Q@XG)pE+`e$=WJPns9%RhiU*+M}95QV5a-_CGwFTGbzm)=u5S_D%1hT(dYTZ*=c87l?4xSt_M^On4DZ#$+1P;4pKOTwZ`U+txc!A4VI{j(qC?_*X@lbj(oAV zhPo_Q_#FxhOPbe;n-{L|?cM%(Gc;>WtDgOF=lp3Rlj1WU_dgN_sK;0a?n!qp?C#+8+ zAIW4j+7#y;xH?+eK0*6xE*Hh$slP+Y3bK5wIx zz>yGuJO>|f zP=Fz}(}^E@X{Zq2b$^!e@5gvy9Z`nDMXFSWcAR|7y+7Za_<8URr^mN8*l5a*7)%^Y ziW}+j!qX3uP;LyUtT${Q7c5>pb^YI{astkoYiNeoNJ$)ZyoKq^M;&vvV!EyT`Hz6j z(JZ*zWVd50D%7kQ{`J1xs(bHilxo}7P<55|J=lC36Q-e{;0W17f-(8FqxYb5XQL_> z>;C?|gKi_WNPYacM&O&=z+4ls$A8S(Z7nqIO`ZqKHw;=lngckr?DDN6v(Wn z`ZOqKn39OsQE>9bsxj&7o-_&w)&^ogJH5L>=*~9OUO^$;UHOSSHS;?-O=70cnY8|&5nO2@8ZL#@guEhbnmS4im zm}&5xP;qHu7$qY(@|~C z!&}WLV3wB zK5*(p5$eQ4z*BtISv3xeA=6c>hO)JX)_0vDEd0sSr<-W=sHw`#Q_h%!Fr$&Vw-zU> z;|_o}^)R!Nb-e`L%T?F4acUmUS}}cUs;!1k8Pa!IK>AysoE!ghI^6hY*qJkllD!`H zRSPaP%i%qS;hIxKp-y!+;)i2Sh= zgNW#8Yipc?5-|0~DJjWbYx^v+B$#jpFhpiY^ToC%c6Gr2;@v`z{0W-MT`lU$`=jc=GxW#Qmy9+@lR!^e5K;M zv*{wcRw4jq&BV4T!wE6oZ_Ot=7u-AanN7uZLGA-Nz~gOp013m}fj-Yb$5AGoKO+)IDCO=k=}CAxNT z0Rt_Ac0M!5YaQndcurG(1Go9MkdVtpDP5Y>tr<%If#$aL`gfj{_F;Zt-Jk*M<&@~w zwkW;YLv|yTTD&;wRJ21<&@Q_cXkhSoF{b-aLD4DAm244o`gKa30(lj(Wk4FYUUK$MatHY78EjQAH9ip2N#_blQ$fN6HU`Zhjvr@eB=kD2F=o8NHX zm=e2W)RKOm8gQct?F_4v9m6|ETnwRcT(oG>Z711RHYHE7G>WBEGiI%s@kWCLEkQ-a zMwGZvI0%f0G}Lx6$4Z6fM|);tOr#F4QG3VjV(xPufm)InmCd?Dl_QepR?}EkK6jHl zs9vL2|4z6(~^Bdp!Y5n?AU#vwQtrar{qmQi{ zeJuakDwv<0Dv!oyoA~VgbKh$e{uo$QyB5w1>1R<@VMd=>`gHS2i}It#i4)!3aiO-6 z>zJ1Q4OBf+*O%hO5s!BLdO?RyVkpC4DM_zUG&+QOs`JQ`PsRAZ5hdh7bep35e@bPB zLOFl3eLpY|YHT7_x%0B%k39eJt)%SxyECK<2lb8l(-LR!Yu8qvdhrpOt*|mdshd`^ zG8=HcfIvU@swnhwF`*R%UU?1=M@fQtWfhi&1i@m^pmj<)TGcVR@9Nl}8wFNG=La9{ zfhr{~Ve)#R&{FZru*=MOQ(T$;v8Hg`V5l9pK!zoGiepIcElZzySJ~n5_|!vFm2qit zCoTgTxG~jy3wxG+%g@V;PkHh=r%F<@QK2ydk0sE4G4jW^@8vxv3FcL2adM|P{5(?5 z+OSodHiMtsWdBhUI0(D3X^&4Syi9SwWYSk^8oBh=YWdzv`og z#zx#bEV=UNgqCDs#8b>}Q)xjmDFhbSJ3`sQSpS%E!5k2X)&34Bi zTM^o2{RrIbVMDW;_~A^R+$jw6uCwZzcKX`8DrKHa&h+#0@%K+)|B7Af$u^^hZaEnd zF>c96Jr$W-w#=(m#j%W7v9+L1y?P0vKch*hNS-?u%Hwm<4Jrdt*&m3-+FC9yaPgee ztH@eNu4ZVkv7M!s@LQ=Ev925IH(Pkg46BvOpO^Y|YgU(`2-*gKHWLfca{!Ti>N9ad z9D?2k|nVQv!38>o> zaRlt*>{3oQ4B#$~!JL;`_+P*U59#mu)Ov47hyqC`kj9)ar0AhoMm)K2oRW<`I_Set zjWLyovuq3j`@?f+TxH)6miX%af)R#*&xq_5au}?6`y2*H3Ux*fQA938MSRC@=wy)H z9!(__1TPM@GAdyEHtvTw!pI8QV~&ueG7ljrO=KD*RmfWvqwlvWyfXTax|erPGGsR` z!#w+Pp=$||v9NIkF;nZf?8rE;!T(eMAfP=p!=Sz8Kin$ml$Mn&F~i6vVofoNXPech zxN+PCBf5Nfig1U2I&Uo&!2&QLx`eEJ_#ZEBZ*T3lW5?|rTBQ#`H0*Tp-B$=>)hTUicCRyvHP$(~p7zJtuO~``8=5cD zDXM*%srgO_9vEBB|2fHB?~jUaC7pZ^TZf4+0Mi8Xwoov>#{8OY(fPP`5at zKZ4CFqoV!Qe5TV=wo${Uchur_G$1E8Fnetp>Vd(;tOi>p?LhkC(wC9FZS7Q5GHkAk zak<2kOg%B-|FQKRa6Rwu|9|$#2uG2XlvyHsi_o&-L~)XqsALmaSs9@>LYWZ{u`c=L#mTU9OHwf{rD*YI zG5gqG?YmmaDBl%P<)Ul=_qnV;>K*b=6=?t+kmMo=t0<~Krhqkx9zt!U6F zz6Z}9TV~em1{P#Z{f56ZiBdq}#^Qwd5H`clp~6UR_<#A(LF=3h_rm8(+AX;Um*!35 zzNUwlxDA4!pZ&}>j?4r7Avq5l8p{lRoi55oj3^nQfMEQE@>V)EEdXo1&okNv?Fwb4_IAd#N z9cswz{5{8fcUR>y-nRjZGp!xQi(uUxHs$lvohGM(X0A_oHXl~I50e3<}N8vK(ys(23-Cgwa!mR~^SF&zV z@yl!8KxVQ?EW2(%^;wXSkU&1(=EGoem^a9iws0?uBA%tMmN$8b`-< z6ys+7Wg-((Cb8>_KTcA{7ZtXaYCyL-=hv;4O2K}1*9GR0k7X+=pz)%@jpXQwh7|_h zD4C4($7S`IXLv^4r?-3z5>Wgm^b+c?*$A^_lcchc9+7Mgu%x!9X5yfpi2x?U_3bvtx$;CE4OBwV0OKIDNqMgf?K^1>`~p6}MJK|BM- zEQ6oR*eNJE0bCjcrTX`6+wTag=n<2RwIVw%f!U>oF6a%*I)2csFFLq`a8PL)`@iTu zs8K)OA#Y@-`QjhAK>7mW#lqm|{?eleIb!Bj;)e!v?pJ>71t;!p7SB{x&$Ue!T8&` zMSb6sR(#qfs_uYAZ=rR=PJjOKbw0tdDFH^zI}#ccJ~9A5N8gpd21?fKoYjRRByC$( z74QLv*N8TtMsp0M!LcYD2q`@tLQfR9?||}o!`CotE9Lv3X}f#1TY>hK%D%0gi&2d~ z2AAzF%s7}rn!<>!(%Ae>gI)+0X~n6;y|H%dvfK(>9~hR7>ahq)kTw>7!Bge{ScgU5 z&!I%-7DB;>^jIPS4o~dLg5>!cetF&XGPcs$;deD`h>;*SzRYuuwy$#VldTE1S(#^kpl3?vLHMi;*q0lPZi1G!rg z?%n&VMfCxPC?v`?YXqbS;TZTvzZ*X*aRz_nV{c(a+Dj^Q6J^Q~7^$r0D!ekVAZ>W_ zHf=WCpY>1peEdyRhC1A2yHod9Q{kA%-p?PmhDX*`M$*to+V-)r1Gblfq1=WJWh~ns znf1MjY}?WkbQ99B^5Dq!Zuj|ZS80U$!dI#9ad|~Wl*Ad{a|j*QOyTI37lY?#zHjS3 zQ-Q1IcSH9vE@0P8z;MnL`ExJgO1xAqgHiFH$YgkLR#%OY=GQ<%#GmrY!7iUIYuTt@Pb!fN2a8I^|DU@Ha}M6HH$uDytaJO(qi;8tLk|)5 zh>)!A{xON)2l=>+FjNw=#ue&hkrb75$yp1(#Z6feZM*N+MOWAz8yWhy+C9>HW?R(r z^O+Z6J%1OOHaWh(;jl3?=Ok=-ZFKF;VqyrY1kAYCcg_!;Gy8rToon_Eu2u+3X8PI| zqdaDTc{F3$98iV3gk>nzmtH0HDeE8l#Hclj5UO+QWd(ial0IhxUIa zs|P?3WG!I^Cv$xBxdCl)Z?UOHscg3S=bu(mcK-USG+|qQL4ormm{qwRx9^Mmz@9K) zdTF#h64Ltb$&=x(OKZoutY3mgR2DFD7EH{{yf<%733Bey?0Mmf!euLfa_9lGkcaG1 z^~>;fRqoHRw_oBm!u(7~2o4jVPmUfc432*>@_lfqDW}m#u!Aqz+fgS#c0h?_(FT+b zSfM>fh};6fs!}tJvye-2oXNUqvTuv}9Zpo*!661yCR@_QGhF?!h_B6ob;^fYA)e!E zZtgS-EgOEn9wJokMt61!xCU5pj+M)kSlP-7vQugZ1z_ME)6jQ=30YTLKu+<&~j3nR6g_R!J; zN)iV6?)bM@=A0Xp%(rKD{lZFDi%-`G`Ym+-ogmq6J#*%X5OrL%&@e_Dt^66Wn4R*a z9e7h?K9+9qKfqw$vcEDEv}|XdEF10YeC$cfzqNz<+<*AcrO$@T*r^x^o`0I4JQ@N&XYM1RkRxJU`7XsAOADBWVvkuXIWHp}bEueSIbC z?>C|I>%Hqszdb=LU%k#M+xWVoBbF@U`jb#aRAE+zQrN6&;-P0?&=6>dD7G-!N-L`d z#1z^`OzZYEoXEPKoe^E#lzw;Ul{sSH%7V%4L1?;h3467(^21MHr#7?dBg5AilPgW zaLNFt)szJ0TU^onQ}4!pCb$vAaVIN8tR_$PLCf1`MLpk<4wHIl`nuITTb1lRd(sP# zZ49Hj%Z{~Py_}+R`SDkj|6I(yZ_m#t_aYi34c#4Av@UgDez8y^BYRJF?M}kNuo4un z<)Vw~Dx~pMXQM|i+#-C$Ny-#T%M)Iw66azY;?fy6d9sTr6maf7JbNMPNQO;V$dVBmLXwd+Uhp}x*!@xFj%G5q2xd_JFIT3X}Fjm8?H)FVtq_H{{ zbVK~A2UmQnL*!!kQW{r?;AGdt(5J6n)nU#3lzNLvGQ-Zrit6}&NIVF_5?y!$o-53{tCtxzoddgMOV#N;yX`7`2xwOYk*lk|-}2UR!bq~pGZ z&TxvN73x0S|Kb1Pw&WuaIuIIj3$Fx_jgurlcb{&hRv}Q8(eLN4U$RIZYy^Mi@{EEH zYVl?Wc45v8PN8bIS+o*tIa}8G{MMVj*f)eDM76|o+f&A`g}^^jQ2|Q+Qv7j~z5EAw zj8k@)DV5y`yG}3be~|7`c=X=AKc6nz$QnZ!((DVDaf70*S@h;b+*aBM@cW8^hlV`n zKsQX1M+ze zAf*M=fO4eKTrFd0d3t$C`Pu0Yw8^*ly&uGP08Hoa{UCjE^$nrrXnUCFZhY>8Q+6&& z6M0VFclo7OGO5TR@aXyaO#?tBaaGbiObsKleMpZG*41Cc!%3eocNch zxjXvh^VqnJmVA7^@ifbs)^o!^Xm-bmAzK2KVlm0XWzEuueb-#c-r=^mS#XR?8OKg( zfk?cjU6f6#vF-rWLK`u6AaS18dVX#X^=~%bJ&=(kl9L0s@a`OQSsojGFVi~r3$?2e zZgW~0A@gv-o|iwtq>>V3U!bpV(=`hR?x8k|h?7F+ibRreP+l~a@`@gr8& zKum~Q_YkPeqSNow3Dy%)C2^ew2BE}tmU=^z!`wNQnafSgii*OiJ}>vnMh}{NHWh|{ z19+6_Q23iX?nQ|Ydz;pfPAm^yHwAd@{~&E19b)(!TLsfFQ;Y~MKtYVi6EP@buOSu*`(Li@Fz z7kp%Cf$}TT@xRFEM*4^Y$2}aRD-N}wj&BbnBtLiL9Z)WSx}ak!^NWP-D4kR$9#{Vt zcFTo5*e0-BPMoz}|3qZ7B8!%A_2~OBvEm4zk52hqUsd%QD4HCODekN>rcFG)mdGeNO<6*=aVG`9xBd-}Z0@D&@-lIx;tup*+KdGvj{h=hrZo z7m`|xJBcHpOgll%NiALU9nE>AdBDWPWa!9|HDyh#R`Y_viy!Lk%h`YjiAvhjOay_7zozK>c@iC-6kq#KzS_{s5+VTk;fmJm}Ug2!#FLZP9R32 zJ#ef7kMe9&UNcR7!5W?w_q9HanZ1ik5;9ipXT~X4aKMpgLaKo=n(XCa~9X>yoc>u zV{SWqxC-XBo%=`qh@e$yM=R5R|Ll!-{UTy9`&DL__T5mf%Mc4;AW|caNm}eS0MmK% z458j|jK*HCv++3>^^87+3($z}E3_~!x(*j7>kASixChW#XozW`7!Ry+UEAx|3Z=S^ zdqkvz|C2r_g_!gOGJ5OcA8clI>eH_8((vl))&PN8Ws_<(*_idgw{9s{w*B(jc0Uk` z3CjRPD6~#>Miq^=od{LGYKP6{TX|M&QS`rQKUV%o-+Q<$=V<-TT4p=OpR8+T5#g$5 z+S}7(QEqw0va(p`J{!B|C9H36m9a(aJ1Z<>;vTu16@pIonY zGoOy1Jh?XGb;z_!wh0TmT%#ZWTeP$8_!B7tKljtrXU4{%IL2zhfMO1_2%#yP_iTmD zILe;F`PTfv5w~yVJ$))W6inwM&2JCl6<^$P^_n%+nDej8HmeW&A{Q_jllZocxgjzt ziin8oA&2{X7!^^3$DZrl4%CY`VU&x@%Vkfz`bAecvS!V-0hiQK5FB)TU++tPta`?< z?wY=m-gl2l*U@Zv7fcRK*&}l!O!>ZjTozwIt-u4fwYYeQ#@0^Az0jKdvJEva6?hHv zRr?dc$B$pbPjFyp%1$sVa^Ki?s-Fo;6^3T#l>5JNjbUD8Pn)9X1i&QxqBlL$mSn5- zF@^IttY6;^I%)kytC4EeC8z<2a`!!u_u{M>2d`YaHvE8{y36}qq|w_48u{e#;MK&> z%P}*3EvhOqYB75_nT?G6c zp9CSAS3_N!0|%xpo-lD@=}phw=oJBSuBN28zPj2d-gxZTZOdmpOeG$HLms zNKF2~gX>&%k_1SK}W zE`i=vDDI%OB;t5b9+dG>>Nck{-CCEA=1m?BB2?b8RjZ2Am?x((+&TSwgz@>T&vy&z9WHn-R3Qj--rx~k`4>m}Wj)TIQrAWy!8 z{u(bmGS&dy)cD3S4u5!dO%m&LeOnW&Tb1L0ojQ3>+PAFiY}radWaXyGEdV~*7YfJl zTov;57EJ_&_As4UNUxqfwUAq19Ot;Opz9;>foeu-=eK{a>km~Vf?a$2VIi`v8g;lw zsTcmX~6mz^xOswMs;^yuVkgf7#vnN1H~ zEKUUkJeJWB5!d|MjauT{Afqws+Woz9tKKT>gu!LdAd*3}A$C$*&u8?*Nxh`ybX}qw z99ycZx1Z2h`weSt^qYVG{=FS2=YejU9lg9-qD^s5yqM0%KtvFSB`l^+asdDgZrli? z)wtRLVH{aHU)y^Xov93&c6LvLvx(_zN8pjPkbAbid_fC<=%DJ3WSvaZO8n=w*ze$A zs2x6$rV}dA?;Zp1TnMCx59fR}*A5>;y%)(Cx{agnpFdwq(9OR30l)nNF4m;vjbvou z;$ca7$xV0qW_lQLR^{Z5HBM0wcY>IGNyFu%Ms0W=;L`^OP3JQy_}hRMqDPpqb0_k= zQlsEq_|y7)LD%2MqGRy# z!($tQPwc1@LZX0kj^B+R-(;Yi7_4!6OGYXtRXRqIZZR)EV-#07Tc$50-s@8NPXqi9 zGH_v7j-K^b_p-sosKH5Ko}Euz9npG}2mnN<);0IKM3;gs0+m>TN~ z?+--;4&>jrRmMSpK@26GPUyC8FSa~>)eTy;E8;+_fVbOTcmknL7{iUPdN$0AF%FCL z$Ixk$DB?(Bn*ZD4{6xfc{zs8LY@j4xBV*_ow$RP4U@g3dMil#_?*TSAE-mF6|8MEr z(ikozoFMYX5aJv9*ecj^`nC1-sWf8oy+1X2xN)YDTJ_S;pHa> zh7vD<$rsw_>AQES>HW2}!|N&qaaT5zcT@Ql@+5F^&EM&x;dUd9F28$UCx-Pp`Wy4! z&o%aZWE4Cx4=V>n0~dYMHY+b)zAvULV3g)N?gS>Kr7=HfF~(gFunkRQfSgQ4h%& z=tpuZ%R+SPZ{`9GtafIOfF;Cr!l7K3o$?pI-hc~It6x8f!+E>h@L){Q{yv@!GE#22Z+ebmc|zmxk<@7- z(=36>6rJjyo(Qn6DCQL=_lNW2?%=QnSzsOyu8}K^6R_x&fL^ZZ&u84eS%NzO{KI4x zy(@sHUnf5&p~-r7@qk<6G71d0-h)pqqUN%+v@}6)(wlY-16f8l?&Gk3Oa})|8xtPb z7Z_|^(N>c)%f~tF*hGK#_g=;HV%3eFF6;d-swiVLtN0{fM;HZ~OH>X!Zc z_APPGY@ykz)d&m%&uwlOQm(}KPA>hNX>KEcaZ=MF)~#EYymNihM|AGE;;dSB0f0eb zwa4WdcD(8$bzZr$JZ{^vB}?qElOn~;TjA|EY3kH^l#i{jK1`D!mg_F44joqO>v3ynA;|%5H!;r$m}3qMSVX zzMx>e92thh|2Z-NxtA}iRP1{D`RLPdT1rZcC^FC0#A34#6k|Qt6|pz`WnmHbCZkLmrnQUkD2cBsPtR?jXoil zZ%5Yov}S_Or$wuLee27#2~2IGexX&%me<97z*Kj(QK{kailBvk$d|&qSq~px4WthB zR)gkutqqVkA=HsLP5V^4ZqONR(D?XeeP*?kNmnSA%vf{rA)TUli)9Bq>+$%DT< zt;UAGAgsM1SXJ_I%k}m8(oN&}G-D$pqlf3`$_Rg5_GmzNpI_C#QHD8I9&9#w0Acp0 z0TlVu4UcKN294`TZj!;wIXOIX6U96z4;cD_OX{x-*YMmDBY1@!7>VQOl9+cOQWUrvVR=EA~42ZW6K#s)9`NF*K8 z`z8lob&lxApQW0Z=0;JRA5P73G(3#Z6#mq(!;wCSBUYI3uzfAela9}Z6=IttsSv@4JIZtNzk&L0ItxCN50B7@Q zf_>&)ig|GE(pIE~+-ZlbPRWq*^r@X~3#NLjZB!E%T~D#_RDAjM%C{aaF$k z%nHl*%`knRj9c8eMcmuN<2u?=3B|(6r-_q0^gKBw4W;cO=Gpnv79a#2|NLRfevkRYr>;piRbKQpJXuR8G$8^lHIy}w}nnt61z=Vr)x?HBUW#l&ax&_4aTJEpUeK;k}cYr-k z6K;tZm%f>=76_%t7O5>Kk^i~-bu*LOj6$G-KOiomN?#L;(4}z*o{>#+(rGJMoBOl>J^AoFni%9xL zhj{4lVI#|IHkA(~Aeh{_Q%hg zBAzN;UChCL4^FH3y(O;!#IxpUD^)~))?>eprnW)y6AmTqFHd=zxZjLZn!vqgn3YNh zPUV;DjlaMAE;_G!)gS9%ceCSSlH7HB*>ww=xkPh(>y?jxKVO(xGJ--uZZxEIL*xk&+~WAC3}bu}5~AOV2#&kA%Gt z8_IywKF#`8*~Mp9^-IqbynjF!7~?ZU|6gKwl9|r~`s}-_Xs)R#>c^uX5OEy~=OQ$b z^l|FwCgpPnY>dX5`j;2E@{AEa0K{QVqddw2uf*sPqWaUDa0d^@@#P;WP|02wdBel zwa$SY6>FXY>lG35OOQDN{eVIKpc7D?C6m{gIL{9ZnP>a?1%gfWApzTT6zT{eL!kw=Wk_)lNh#Ltb_g^R3;^6YHprg^YpMe2)Z&#EASK*Ni& zE%f}GK#qv5NaZ3Mr@lk=Q}N~K>-N0C(8|Aw+{0A3x1#=pcYH8b`_7XwQ{&PRSR7pb zxj7E&aufAPWLzUzqeKMs<#oUO`|&LVK~r|?AZSXGr~KO_9Ct|?X=WsTNDLO5QVr?l zWjz{e0<%7Y96r3be1J?QVZF(l6rE{A`r!Q0(dL)2_6t;ic(`NCVZ>D$JXp75=X^vu z$>qWHrhN0jqFaMD7`AUmSi_$WS4X*2C*y)^q_mcNQyg;m*6_UH6!r$3E(Tkr^Gj@vANB5i;eqV!mI`cLL1 zS$PDi5HVs3?=C&xZtdRIhm}BDZtlVPfsuCHEw^$UDuJ|@@L!6U(Nm^OK{Pyf{0e^n zWoPI1+!Z(2==4F z6PuVN`5{+1%bsJ#Idee*dN)o)L;Z2x;l-J(KS9!MPu!>FSn>oZ*iL3n%cWKOeh_l} z>($$v`j9cgzhic$pUSU2JAPn!qVsn&8)~B7&6f?e{po!0e7$5P#s1I#HZrMD=?c#|QFbPQA3E6@z%m;P-Fe$^twI zw=Enr)0&+0om&J_Ja7Pfp*d@H4s$``HSn5ww;g3ccJWX))ytZX=&}PbEA~}_GOmkz z1Mkmthigu+d|tj2R~_J?LR`19=rsK=?QkH*w($)~J zrvwMG9&?^5zcSY^W(%S6%kS^IzG;VMv*Fig6{YjIE+rFtkDhV$2s!MMWFO+w0=y_E+NA0|T!Rai;O?SdC|JJL`tEo#}IjcZ+V8 zt9l;Hylf@J3O^#`6TU2v8;0_|W&P+z6~z!X_9Jyv!)D#@3`1TW)jqfX@P!LM#rgKm z&d%o9HlR^6eUg)Npk!QW8Yrma+e&=EV8y6fzb4xm^z1p2Ip>aafJAv(np;@({66^H z_TFv}SIOc{=-!Z~sL;^~LTzNM=xV(IFG0kyqrx7Wm~7YG7L3pyfM66#-N~m;pB5uo z-?5Pvm3AP-vHdD@a2w#)y}5JK0VxMtYSpaSt#|K)ns(~Xxgtbe`22#pidea2dQ9t} zr^OFV>ou3dqz9akOqCH*0l3ffNoJ-xH`ho<$DJ$kU7Qh# zZfqqMueoE|v@yhbibk6*G0& zKm9SiQcr2fP4%HPeHv+HSVCZbTQ>$b~N0B zPeA6V!+8jwDC+@Ntho69!`aw^SWEEU>(|#FZH?WXh-9}BExY{b<34KFu8NFi-1zaU zNgwun;I(8tkP-+M{qm9`%|`m2jMRv=Ygkc?P&Gw-Pqa;h^8vHLjmckmno~G4I+BYU zsYrc*jjrM=Dd5AbhQCLlSx(!gg(OoXX%?-%|A!&5~#i@u*B*cJX#8@4&5dlREzO@Y2d5 zzlyi5G|}w9Ub41?L(Ga=YC11;0oUC~;+|ovZ# zR>G&UtFEE~p}AyK)2dxn1sc63vM0B26V43 zzen0sN_y&ISz$sv$J~MJSU~(5G9`LU!o;3!jMO9y4da68atiq5_@z-6Cti-=WSxzO z*x@z!6|t#822U(K223add766D*HSp)T+VN^wnKWOD1Q94MNlvzpx2PAoac-{gYpG;!TDMMfUA8f2 zJiz`pMa~=uo z&JS+Zs!`(^ESw&R;U=Mtsq91WcT4Jo+)TraqiV9|KPw1S9+ZJ?VO|oO-HxMXtO3ch|T8;{iKJ*7s)8sz37mZx>>?c9SS5$({ zyuCml_VGA?C!Lvk1;8xpgM*)}e@6hBXaCk)iwvfQE5D{p`;xoQ$E5+E9qA^AfCZG4 zCoZ9ciH@U@T@t};uyq+;J&WNf8!OwTspqvaxoa?K%ajoD4xCUYKSp&{lPaqqH`vp+ zn$SKl+ruFtZ+EF)lZA-@f}G=mxrXt(X#SG-HB;%jcBHB^zU`2?ZVMJfTFi6_?!MRJ zJd$EkQ(DzW*fpX&izb?2?CmXkfOZITF!iXf?a7FUyCMDNpaZ}G`Mi$fGg<{TxWuf6 ze@~j^K=40D`m3B%eo=)D)<;IW7j=xC=?~#|bPE^Z1Co?0nSUS#WWbcwa;D;Vmo-gg ziV-O2yEdn;r0iU4b&?8_2ScKG^FZZtc6JMNud-L;`_Tk5&7ALFkE5W zn&dSu+IY87d^{NeI1x=}0SP)kXlxD3+exw^2$AHSE}8A%>be;U+h9!@}Oa1nd<`ki@k7QsYLLnC9W6Ihh-p{>aZ8%C{wjhY^n z_}g7TCP<~Jo9Uoe;PHbgc#M&!V9}JJKWHsQVzhF6&3J{v7 zJl)mx`?n694L=`YYix7=t0|fvw$AxoQ*2&T)D(SHic8Ba-7cxjMm!{mY7B}UPvoAT zTaAt|WkvD(U5MYSbQrVmGDM_-slyk@HiU{F%c3s}wab!raJRXQY>^29sODea*aU3H zIlT^Eb$ac&Z!-16z{%HhedH5)N@|p#fH(dBjMa=@_0i?*RL6g3Pn!%pL5pw+_-lQm(ESZ(cbiav*qTK0#hR1Fw~qlw70BoDSMM!SD@9=53p&{~T-< zIr%_({fkTfxiTL)nrtd!JG7(8NaIk$#Y3ihcwEYBafbU|RP@dznJS`+hP@V$+)$@o zKVW2~q@WTFa5{x;9Bq4FtIoTyW9{gkaf&qo&qPk+?!Lc_^`*}9Z=OkR}eBw^jjm=!=CcRf9dG_mR7Rab-Mu#IlRc%+S#*>N7Dg#!y}anCDej=1IG}5N%D+ zkhqZlYE5v&|JIsj2YLs2F8LL>xKg}i7cIgo-Rv~?uAORE zMKTt%y?4BVJwA9jEneM~)}lwTXql{p^UPJmH6r zKGw9eX(r`hE`q{BLziUkQS$^iCiA6kIMHm7z@Asttk09tkc=>Po^|W&v17)N@~K8xJSu>a*&-narek%jlhm96;C`$!Gu0n7 z=*h`58ity_IIM<|gi23Y@MI((5juU4&ULY|-x*|VCNti{=_v$JlPw-&wY0-c;7q8O zq%14$fvrmzTXFw|+Ts1F7A&Z26k6!n6$}|UM5oZ{{~e%Sz(S%CC50LdWawkqQ;GN> zQ!eGBW()1`27p<-IUM?R|AgE@NK}fgoKmbMdgL_t5?BD-$YhLArTI) zV^(C0vkgSx3q!~C07O0&gZrDNPpUh17i5H?P;ujUDBEion>+*CNe4!rOZkt&A>FZe zWpa_>Ft~XsRG0U54d%6{w8e^b<#Iqtf5sLL0MI`R_@C<ftC;a8~L`;=A>-cE*vpRF=txeae{ z_3MZsdGDRuW_L+GgwmI3*U7YO4d5KX3Nn{ZZb6E?@L4-{?AEKLzq00 zQI=el-a$g+TMzc?n92Xh()JYw{(F@S&JC$v+=yGjNRpatvPA}3o9(xXn$wZkaHe;3 zZT6q@#g*ee;;$~a?MNi3&2JCfHo$PYLYSi>}rjOOP(r{xx{Gs zC4PfT^T6-RA3k~V&jBsN7MK-~MRR*Ne%=GFPXtGO-;vkSBbr(qziXPnNbEI4Eq)Cy zT@h5y;_dsm*0T<61wbeHm6sx`Or&8#6wlZmdHt17&Ni*uWE5_{Crcv9J>A<_Uypwq zeJma8-sS1b$Gf@dafQ_M@F)sa9TjO(AxhJpSyy%+c;sI6EZRWU(_Vp3NMW-qf#}3c zsQaLGlSbBS=rtvGiGaCX_hbz(?#AL}e^tK9jr8l%?*XpL-4_-0X;=2(X(>iWGUm9~ z)VBG{TTA?Mzg0D3=B4KZ&Y3f36&jmzn3a+ej*W}oSqeG6V)zmaR|oDz2e%-L`;p#9Y9V)j?rm zb|Z)G*{RTr_Se4Da76tj;u$_d`*$EY!no666)fKh`69_@MHjy}@cxjf`;F^^#^6hC z`Oj&62bAR93l<<*)(IIHj15UmOVh?7#?23G_2A`C-uUVC4QNstxmM&quid?S_m*US zmc|^=T<_X4*`p^!~uy4)uzK4(#T1-$g!~-&#gS~E&TmZ%vQ_Z@mi>J=) z*38N+-*jmox@|=msuR>SI-@-f5T-=a%UA*xqKP`1ysMar<7e)LTu_7oa9fxrJrW8; ze|^*D%{xudw3!Jlt1D|_=nM5c8vxKl>zX!sf$z+Yk?w3%)IT2wMh*ly6E&$Na$M&l zVKo#iTVwRtlu!Fdjzx*r>05&R;>k8;4{vo|nBAQk#c!n`;*=1@F*Ko1mSLu z6efvGQ-}`H$U5eJ-N$w#vMg%L?5)U{*S2<5l0J1Luk%ll0n~2}^E?#Ln}!t_?9`9R;wj zE;MjYaYP<7Hg?>|ErlRnOZr#Y!xA@2;S(}O1}I1;N~`H0f}nY1>lwU0{@$P)w5}Jx zIw@g%uBprR=WXT{5W*Y$-|tOfqYQ7j9O;tr7;-1_3*CC(X$2eUS`Y{T+llBIUFs^= z1Wcs*RS?x^8SK87ThUTZ2zhdG zxddq&#GCd38$))ksL_{=Eu!X?m`T-i13y^H!_m(~+ACV|a|ZzfOK1+NMxZ-32)KXV z2QgEbJ;(Jt!rngUMxWTAvUat-t#@3);IjA5_NbX|ZacToH)QqQ;C~%*kq=!rium8* zkjTC@ zX04M=(zKFUovn8){4Ew+;^Ddu9oTU|5|+PqTH&!?x-{4HFZxM*~cb!*#`gb zSo~3=%*a?ZnC~Q}W|5fAMjS`4`!TMD-?}}q+mt+wPd(^AnK|h)Q3(KPA*)PYA1NM9 ziKY6(a~kfoP|A>DD4q89IY?!%O~o7_F^0>h0z~QS*T+~I*LAO+Kf<@~tyLs&loe8j zx=U9g*^7y3jzN&a>-lOC{NqThL@=>YYhU@OPh-IVSf*|-5IauPO~nQuy$P(kK@Y8c zf)BOjV6Q!e3w@GaL#5?&+vY6~OP5BImNUhE>!0;FA!|^I&YUS#9lE`3?ND3uezb11 zWiS$Pp=_kR1t1I0Q9;k7zW=$T6`v4vL`6qNUtMxx7Sn!nsWg5u4?b~#D!#&v2ev>F zt^Be&e$vD8eQmS^mVcK&?_#hD&!X(poZhXO(g;HUtupcB)BQf*2|8e9PdwB%XKE*2%NLsvI~CrQBj^#FV}^eRf9&7FOY zU$K6*Y}xWDl0XE>4s<-iEpu?#Ly~%601FeCJH)76Qjkh2SP#Z&|IYpO+i^^k^C+#L zliBc0rUD38vssm7X94v>F`^wDq+EDeAGy)~)JouPBUrY#?_zFWHG@{ZX#qP9omJ^7|D4O=ZE3!wd-DrCp@Q6vTPhgt`kftCAU;B)8G%9%C zj#21&o$%|@L?GD#huSl!0IpKt0gxd>og*(yq)!93CZ*R;!~&!l?3y~NhKjOP`?`{G zUp(F5LquAo#ik+}5Y&tK>#j&`*4-yUY;PA8*;Q}zwq+EP1VkI`iR}fnGqq9kY?Y>8 zb!4P5S0wN^>z$P}Cuo}9=b``~{~X!}(R!h5HeD`SwIN$5LP!*4pK@mH{!j^sgJ#^F zQTd*;@*SzL=sUOsSqmF6X*Q8eKCs?sZLU;AvitP{7tFixFU=X54jw->d4(#N8Ck6} z=p~0FMFo{jqtv2>|B0XV>KCc~&Tcz(XF$KrX~kbPmg;KTyuzTC(Wjs4w8Y(G>!Wj( zWP%~Tz8pEa`X`=$3F{Qc4Nu3s(e4~^eElBf=Y339y=rc5{*1LP-><&S)FHA~Zg9*x zlb4Yn*i+Nj`bcach@0%2mV|duqb6^-gixR^e^uf#9Sfy!i-7WC%$&Ru7d6+AZ2WZS z|0WV{Qe0zRUQ1b2T%<9T$fkYs=d-i#frA#UoW;gp(it)kE6EtK z`*1W7oW|e^GTGW(MT;Pe)I}Obpa!6Va+*aDLKo*~5%m#s3Sx%6`X2+aEg(wWYnhRa z?7b@f?j=Af+u_%O?_b}XeP8Seu_p%-T)2*$IXjxfu9RF@z8quk1ql{An zU-q$EoOCL%E&4mjYZ0}AuIly%_3Fu(j?&)XR)FH}N5X&C(NPQ6grD60r>N0)@74h8 zhN<~SE)Jo9*L>ol{_)OZ<+4}W^NGV1$vX$GUCz!9wX2Hwsr+Yd`MMoD?kXIoz2Y^T z3yO%GT9fzAJ{~^}nl$Y3y8zL`qN5YJXZ*;`%dAEu0`$V(XYbzW zGvAos|2;G;O+$nj9vHCK6A{3y|Qbvzsdu(ams6aF6_!McT!srN#(q zXt}T3fKhX5o~uSx9W?E2*$5z0)JU;Z6L(+{EiyHtDQH!GAvc{5u7p7!H+gAj6k9>Pom>IOgtg%2l#nP%#c z)jm0UlO2cA;Z-zaydkUqS13~fGEe>#mCyU1xk|~&yc_+El2Y+yUAIL{wcmfEzq8Yk zIt$lK2+bvWx5l!beb>1gjX66herXjqx_oAh#+$bfmu0rPg9dh7txm? z!JKyCG@e5`2^^dCK}cyXWC5`-HsmR9hQpB*JW=GbJvA(<>Hi`Dx7^1d7|@;2*v6EO zLXZFf(w2C7Chs)qjDGuLG-*8RXf9 z_-H9k`vQ-7^0x0yJ&1xarOEmvZ*Fp>p8@xW=;mgwC*%sA|Io>kH{Ui>mymd#sbmVg ze0lz8mXA71QHHU{_o%GrggHi*wx7IyI$KH5GI%1nP7JnOf$J>4zW#pFM{hs>0j?g$ zODC0Z*OWgbaIXOERJ7tOn zU<*HR+Y4hH3=nA`Wj7Z#V`Eyo+EwwVR`N2yC>EpRpJ=|rKgq|rp@Px`-tIR9{ikDF zCA$16C-OgPM)&Fdyx&!&SHSvX-Q5kRec9#X^Q)(OS7ZfDX08sJ?JId9zwnz86!H(AgApM26m|MCbyozz(0lDcaynv$ z4SI16Jr0cuwBP>8u5NCK?-Wu_D2JC56GwgAABFx18c&2Wys%B+3iD)_AuC4ssOHSs zo~9f3ZkSGFcpY!H?r5DzT|PE-6yFX)i^h#*knA)Z!7{wK(N7WLv(Vm~s1gYgL(ZIf zG-Mo(cE?-o&qk{88FcT~P3r_M-W03?3MxoMK@LsB$F+M$QF$m3fJFwGvJ;V{>{IST z$CLnM4Kp>}cvKxB5ROUQnf1tG`SR((G_>K>o^C6;2RJ`)TV(Z0VA%B97!}bOrMI$1 zC)_RhJilAbHP3vvn$Db#xx8(M4vB9)x(ph$|Dd~J;P))SFsWJ(A3eIc(?>500#9l< zewm0wA#a(L=A>;+8Wz8#ekT|ydl>V_C3 zgWHBr<=`mhPMmm8k!sq)ccgGR;PuXh(FZ4-x-W20`UJ(_l-1#4uB5P>b6VN54`X0Z zkTwL#oSZkNDiX6MS$$;V7Oe?tdW082!k_LDQp0K$J&~04geX;l)7)XwY1sA|W^5JA zzX)&C^VIR+;OUUWR(&-72%9&%qGRAW$n-g5Mq!uezgfnL)Q&?;G3}=@0y)8wCCw^m ze_{Sxv{-hk@YKZ4Kth;q#2Nspmh!8=O!)zAWMqlY$u9jf^xM4uUvcxe*9XGdj^EyP zh)NX|6;orQiL_&Z|F+n-G+*^8S^t9ZKO0j1wIM_BjivO}B)@K=D=BeKaB`zVkz7OW z#OpR-9U}G@aJ=T(8c1ry{P+Jj(K5OmT#T~tM}C{~?LXWRiKy~Q<0i1a_qg;pov%1I z(RKZS&2VlCsm;_94lPWJBr1LT(*%~$hRSIWmGfYrCNwY|wnDgZi`Sq!N~IBK7+X+Z zF$Www{6Y^hgL+KUq~^-D4U;irdVkkjYTcu2&{wF$J-t>h&3694#3!-`uF7$AdXNqB zkKD_M+7ZGKCBQHQ18Df|9Ilz`p1y*gMD%A-U}m0IxBu-q_fjnt+D)Y+gfWJ1BWCEQ z;ej1eZrzgX*&9E<@b=Og$uPWkM~3CMchR{LwYin7O6UyFt#kZD7Ql8x#rj*4G7h+1 zo^1d^E4@~|1`W(!a}Y3BtNV!( zC20>(p@no^w`MWVP=l?%b#u%Rg&KK{5eW}2w|USl_~_9O%j)^n2prG{i+$Iz+ea9} z1+w;l6uIoT-Kj^!@-$>ZL>*v$M--YH)~%bHJF3*5ixWw>uEcEUC8y8Em`aa^U};EE ze(Tn)AF}HgQ^nWl(N@Wr>)35*S%O{x<-m(Nr8W5(aBObjQP2J5(UosPQ2x*(8JFo} z(9$FWC#JvkfUj@CEf@a0t#-=Y3jLY+EXGyI7$IC+X|$2FdzeEHt6U=V&>Hot^;^Dg z?BRyg4>b1QGiMTT=yZujl`G&Z%#*1gS-*T{JIuw!$!f6X22qyEpNNg90AGQHk&D$8MQzzAqN^z+vr zF+7p7UsvfE&#;_{41&|7+lzR`Mf*%Qw?nDJRi1aA>x0`@eE-i?on8R-=obD^vAcir zZ#LK*XO3B(_2`j3f2;U88&!DbnSZ)CoZRIE;ES@i`xLtYVC8I8eHQ-xS43mZp^Qu) z2QcX5b?+0u3pR2eyIdJ@6c@~bucGJlUpRajU69h578@IDVX6*7ELg%BSm?dYOO`c~ zoK!B$`K{OH_*BmRj%LFgV{G@SR#QRj$MRC=@hc1Hb(Qk7k`d|rga{6+p=mf9_N?(B z$9SL=m&{0rI#=toCqnqKFap=#4ZIz3*$=a`i2<$8&?lm>ytT$3L9;RAt;=xv%azxd zNJ$5W?%GS+(LnZH{C;27ibdSn0~1FYTMbvMPRGZ_)`xGy0P|(#*!n+vrX6R}CIdEu zVKUxlPMUpcTu0`~8U$~0CY(G6AAqT#wMX(@;erMujQiw@fTgkkVZE3-9|dz#UGa2g z1TM?|4**DMI%&vn0Q%Ipa<{tPkJ|!NtW|kh9MQ?`Jqe$%N%3Prb=a{;?gz|pcx>n?BY=sHeHN~x|<*3 z{FcaC5j}Wl*=jR%A~n21meU&g->_eVIY^2L4~@)@u0V6_2)ND#dff(VDJE$8RYzG1 zG7xtHoz>LvmnQIDN}y>(rrqrPC2XkQkz74k(PyMzKY%m0hi9u`f@*E`sF|eF4bRVyA6}x862y;ULj}%xGh@fVg06d> z=jUH*zp6BcE0ev4o&I!aN;4<^=&aV?c|!H^eyFV-l9rU7mX?R=?qTfW&|AqaVNk^Q zPX4i+uM~&-TWqvCfY%B@Hc#pKVe`-_tJ8|^yFR@Z8Zv!A-@Z@D>|p^5m17;k&h?)^ zFPa;2_m!hXL^zt}XCN@39=q<*ZAi{t+Ssn`xa<|KW~gJFEQb3Zi;r&w^Q9CldeG&zgG8lwMysTgVU)eA~eo#xm zaPXLb#Vu$N$meTLukQA*aTm6|G#A%dl&pL~g0+*kFd4WEZTqdXceESY{rV|WHdM_~ z`p~U_&Ny!)ZY$;O#X7%{<)9cOYU3&#oDuy##fSs}pr0$dSo1IKrMjpC@cdMq64VS= z9S!5}i`|41P<2+6CVw$9Z{F$8x$;*PYIEuQqC0WAs!Ycd)a&aLXkW2yx>b)( zC_d4FOG|6>07Jdr)80xx9D|o~TxS&S>KMKDf55St*AC|DBbBH>ZCdVwU#(xIx<87n zLuv%xoF=X`+tQ(Gcz4jQD!7yAKv^eS{YXYq`=nA^jg)jr2s|D&YJ+2-#-%<@;?o|NZQj5CiFYqWL{!v`W1|DfNnZ=Bhd=EQMt`?h?j1Ni zDKkiGhSMQ27i#EK)2|k9qlkPqkhANVok8jj`J)DAN0P2 zE)MZp>w!zOGN}-lJ!L^>^Bo!XMDtyAY^|Y!a{7wrV_h~DhzS5%JnS>dQxVbk?j6_4 zr7bVjGuGxfr~X?;ni1rt=dcX#Iv_5*mN$yQUtwA63)?%vdI;G&VmX^LXPjk2HUpp~ z`qJ;*EwHj9v9VKHHmqMiS!2DP8QLFuXUB65iMW(q5}vKSlQ!LY(r5CwvgdpvQ!jco z7m@R}uV20hJiTwtvFMNlHJ336O}<2U1(-AwVBDUtS2gfnNH0cbQ7bt$7xO1E>d%Yp zSJS*J-DWGx4mXExAC5ULov#$jtXXr-yS!cnC}hV-!Klb8Y9m+IQ(22h1gB%7BSvAW zBmO5%%?nRGU7leQN`6T9cTC4|H1%JZHgPp!{q9M3^DZndI^g(=EI@T3zuPa3mpB?6 z4FK^fD%^6oWxReACz6|S!;m`0xiQtR*tYnwy3G|tceS0?bsu^m&%7-*~Aaw zAFaYXEDj&mwCQ0-!3(gKovP(H`Yzsg1vt?rB&XusFcv?Jn$6JCrI`{Kh+q7l>p61Y z9B1k8qyZD0*trz$NrA)awj+%OGGrG~Gra@Mu$6cD#~EK`PmK zml#tvor)k}msf%WmOOthk?jRnU#KDlZAYsPu3~>099~+bp+i@R-0XQFTNmKOkZNs? zVrl_#SE9~kWr--R`o4elYI=|POM`iHgN_{=>6Y{J$I63&0eNwH@%=!}ni~|fT#GUj zc{0B@n=O5Tfzf8rDo41JTYVeh0Y)Oo7?#~{+FYbJfzK3)fn`Zq+3rsDmEI6EXMd=F z`=v`YnL8za;%fN8YM(3eizRAYnV!LJ9udD$dA^dolzOp*#a+97qnzlFu&CE3??L`G zPm4>d$>=zlL7 z)Mb~*W(M}`sbk(CydFRDrJtW$fZw@(e#iwk_sfUr2*#4PZ}Cx2!Cwohr;D4Wfq6H! zM9J1e^w-YYM5n_d4kJ85^^%h`l_?vD4vy&<`_Ci^PigYuxAczxCAsWRF04ZVmiGNT zP9=ml42E@U)!Iz zG%G~r>(|7>Pm5Vj_9Ji_t-GD8YpijnjY*BkVjLz20R3z3>#xM7(+_)B;Lf0KWTdEf z(wBOosqE4E6g~Q{M~iuStERHp@Q=xe8&|eP<8mXqkP9;=JZ${3bc|{(4|*jb!aH_U zbLDu;PEdBWoj?1_@Ek~xI5cBotV2FkxG}!S_QS~^2XO=ZQHt<5pu@f1U44MYD1o{x zCY7+>0|#z_mB!h&rT(4Wx9OE`KX}k#=D7jfza-y#eSNXp?P%0cSy~2M8n)Vxq=gkf z`;%OzdNFsS%Z<%jw~k!AIJ)m_0A~jHA#Gm8lS*TL@%U@Wq}i@ZoiYa5q)jSZ{+M{c zx!U25^K%W%@86S~osU>nq)DtFH&R6VlPb3%yg207NNr|-6#~JxzCq~JF;J(eD;A3> zG`Ov6#+^Ib)_*Zs*waZHwh!7R3i~|)iUYYgbp1P}W@m*vsY-jcqIIA}dx*N}TaGYOub3i=6KZ|FdYO$2L@1MWT~vd%)#9iw=%*oC$CzVIroZmCDBy z#VEDv+(K+lR3l%&;7L~mmN1_Z6MJ<5(RHBH&sz^sVI3>f9+RCeqUE=y!ax z`sg7uuYZU%l{_c)nM)q6%`t8FV}FkO$t4KGK#rtsyzvMVg^-#C7rzJ?r`mLv`gZ8Q z!^1axNbQr$R^+iEeB$}+HAqBY1^fS%UljYPb)As;(K)lQ9u1}b#+pph^X<3b<~z~b~x z92on|)tN14{zabg7JAebiO0$e7aIjW&|g2ix{6Q%^kPRFJ)Dn^7sa?70HY%hP!=h| z=*QT|v@rJV6j4mLO}2_c$`&C@g)F5KAt}VMW)C4Mgc2p$vKFF+ArvX% z|GZ9_neYGcJC7ds-1j|H=X~DFb-lJrWM!xW>`al^JAR^sA4i1vtL^nZqHEN5M0(k(WH3v_94vfLEYrb?Q_%i|3`LVw%#^TN%YPxX#d_O6+(k7TvPL$Iiu^6u0+w8o;h{O`HHUc>=+5B1tkQK`?$k#Bmz62Qc4acwmgb|LMpN| z)f^o1yWT&W>kkKQ=^GRhGTe<#GJGeOw`2>_p;l@%IjALLrAnh2*1vK`oO1u(k2im7 zg?6SplipCbr~JLWkd-CY-)*DVI?X9@#rT}A+5cizKO-Zn{Vr9|uj2G}#(fHV#6|qF zVnWwkQXxRfPz@g&JU~HY9o=mm=wMU+?CiH#2tKFq;)EtF|3~DIBZBkP^%jBcq!sq^dc3f8KJ< z{QkrlRoCSAw~jxPUcK6D-?dXGKLqr0IkH{;E2tzWiTi;_zr;^*8BJop<@f(d(1?1B zPjQ|tUE|zt?+o`=dZd>Xr}v;>Z}gUeSw!dyAZi>3ycv! zz$-tCDXv#QG9i9Vi*k85QfFZiqO8d-%WpL9yttaf!iKqF8yHAC6PhP+k))sfKSY@> z136{VZxulpwU#In`1lT-5F5xRrna~C#!Lab7S@3oA4$Z{%Tz2Nbb|%RT5(-{h_I_# z+sekX;W;Vd;W@ot_jG?A(sLi|FGuA|A0lfB6#DrklV4;py)Ry<^Xwjr8sw^Qflr8KA+`KTJ$+zs!3{StS`P+%5jG zw6AU2w)KljA1mHhKDMp*BrKXypQH7c*i|8J>q!{Tq>K~!(CBf;sOF>;apQu2n6Q+4 z%r%BU__ih{f8O~p@xWb9!9mGm9_#_Zl|DOy9RYwA79VYq&jhjx}?jAPH4MxI&8Uv=B7y`Y1B^OicH- zb>AzP+VIHYwxJWa#KpV05|`S=&%$)c{snO?_W4o6H!bEJXf;f4k=B;$hxp8#No*f= z^Tv(wC!=zBcR={Gfr*}x5W}K7XH!j{-$S`oCXh+COdcvmU;^T}Ro%LTCvNXuLo%_Y z(0p2|wecF!MAztq9U1X5{0%&XP!BXPd9DkPo)jeWKL!OTy_u_{5S6msC)z23-EA_$kSVztwPbhTnM|!-T-hl7vY)*|O|Y76$pgd`Y&_-T@NmyG9GilQISbb3o_HB$rq;pU|YjOJ)PS_`IfHCH*w12SR;1g)F9 zzPcKv4&b){K2%gBcC8l!=?1UYRSM1J_|L|ZuRJuFwXKSL)HqrbZqibEa}v>~Y@DCK zE(3H;bg7mARz?qJL)d>K6wI(dU813L2k%=_TI_#%E^sx=I{@E`%0jdAfL9%G6%`C) z0SeH(-?F9v)9E)Nz_+X;I^SCKeaM_5fVp!XxogH%;98#fX4*iiW3YcPnsyIV42|NZ zCDCmnqyd+=eukw*G|f>_Q8WXt9nzn*>zu!s^q`$D2sTkwacQ7E5_2YW*HEhmZkXgg zwW#-X&$iy03A$W$m@TC<`)%01Pe&bcY3Zaqz-t%(gbG4rPiRt#wrssVzRI*XissWH z6tLbQgvmuXCK5fdg|-qkGr|y^#a)73qd^BGr`C0IwsXRqoPH_=&^rIlmUecUbn*rk zta%wTl~uP9JBBigFr!yK|5G~=6LT2=kfq$_Kl>#R5gYmywhXu6ryH}7Tu3?*vjdQ9 z?L2EVeg$FDKr4zWm*~N!u zQt))}q)eR2(3$kkUfHu+_4f&4TuxhBa5y&^?%hOZf(vyYL*F*ut1`?GuY79FqAI)0 z%WFNo)F``T6e!^=KKxD+@g^(`S;WiH&Yg@qTuaIsx+jBJ6#a1MR{Is^TLJF4W$M+d zcl9w6@e^dYblMa<2V+?G@2 zYg_sFqocgO!9SY=t&T`+2|?emY_jV(`Jly5`35XQ5}U9c3D7hz{K824V_C!Q-@7NN z58Zl4)%yN%(g(KQ9M5MhGoOgtDTZDeB8FDZW${i15unkuHg41AMR3&dyM=Rsn4!?q zzMZ3w6S#(-|6zNV%=&U79m}OzoOmy)Cwd${2ROTeeNPa&Nl*ij*~}Y?lG-9^E$viI zhDV7QUPcBenP&uq6dFNjFW!>h5Z9<0nIQNxSz+7?q36kAk|_u(t##Geueq`FzlD2KKN$k;F=7Gwtn)tM=1^%x8SG50!*o?n0sd*MO`canQs~W44182h>IkV*a zRSC{y)6us%9wFa1mv;5*o5RP6l}2!OGym<|K*-eLdKQ#MyD|B(ytj1I)!ev@1S8H3 z8BnWQk%a#MG4B%yILPtAt^ik2F%elVQ}RY-z5QJ2CAl6-4UzhgqI|!>@g|HgV2LU( zUTnp6B^et*R(l>TM2^ZAv;&u43d^73+-2y@ioExXv};VehGpadc5-JVj&gX0VU?*# zBZO;Y(>;WcGc&&hLu8Y=^q1+PNzB+lmUkn3T{h^afPNqb_M@Z6AZrW>igxXXtXt5? z6q+nT4X=Vrx!6n{$J!PIF&d&3LudlF>(q$+Qp#YU-h|826xfsp9krGMJvUW8GfY&NGyd+KSJA zVJC>_i$zVkJ3Ph?b*uxY%-)kr3l+GtwC#8|5opQ`1g4>Vdz=rGn@httZGKS(iUEP) zDLAuAwbF@4TaQwEm>bzACKry)+>Le{*IZrx`n2Mq88WAyW)1-LmCxn^4Q4nFzolDk zImQv#oHhb2i)iy2d71_?{(xAoiKLd}k>ob6fru2r>#9j`6D7zQ6D2^m6}aG+|w zWlPIA3`(RVqlP#CgO?4}`(*Kd(6UDo;7jTVOs0P<)B;`CsoGw}GC)>1QA+3-nW9FD zgZOB2)BT5wGo8Lvt!FRp{iMu|Jx<)dpiY>WWU>Io#~t6-Bb4(UBmHn4r$a=Hd1u;#@^Zu?&6;nP|>kd^f!L<3g^GkK-y(uWT z#AwgFXLA-WUi<>I9RJ$=J9YLn9xNtIlvbvOZBF*V-^fPZd{0YD%b{#LdTG&1pcMm$ zES`Oe;BwJMVS5Kjy?%(_sg9YMZ}uh<5eX|jQ|o6>iXIH3-NE|P^ET`xMRyS6XFz30 z^KH_n#>Uu6VVT8~CO#yxf1jYQ{7a9d5h|`n`6-}K4Ew6Y$%63f2!=sQOz3%RjV#b> zi>FjIEy~7DwA#Wv)1+eY=yY}gLuR>sU6!F}-nxW4G@@=xNqa-U1vMQXo((xTlhcC4}Us#pkMw#4d1NXGs44Zqb|+!w4; zvKYQ*`2O${jEiA&;t+fRyR~A_<-m7g&6X}*+PuF{bDmw;_v{-tZjh|+2X=BJ{MJe7 z`#>y0t&DjTG7~N5;I@S@=&)3FW5*gj;DcBr-wf&0_od?pRkcUlW1^{NI*Q%~7A%p! zq&yy~ry?*jh@zK4`R^aN9C!$14ioUr?LnQlNO1E^b3MM9 z8V$@gt&N7v=vhtS%OE;`$2@>b&Bz(}p1O09}TC`XW&a#N6Ytt_9%5e5jOy zf`gA1(h<`j)P{kMAu&Z%Q34hioDt8o>COxN+b)?fKXMYIwmjAc>!fp(u`&!}G5jFI zxUt#wh5QW^&QtIA5M4a-9o{;nSPC z-T#yYU+m~pW;u!#DpSi4RZloPHk49CHc{nVnwydVE&>{ikE3KVv}DOVd%QAE%x1g; zR(>4kiiD9+4o?f2I&GSq^xvDc8^0p&NvqClbH!$ZG6BkYaHF(29SOj_L>n)v^1;2v zcOY&k{nq3vA5HbXOG<{`d1c9XB<{ZA*`4ZE$N1=h+k0!v!{}^8GC&E8<|$a9Qp7 zxaG<5@v-$P?U`#O;dY;aavvIDNR3Ly1`O@XR)>^NrX-wI`kHw&ES`2n2U>#yOkE-9 zv(BwSrT?JM<2z2Y+*($tBSuRmqwSZKENoCFYEGSX}$? z)a#XUhHIceUB`W>pd?;F=icP7_Ye_P-nfy3K>XUjqRQ+73z1a-mZs__RRV&Sz^8Qj zm?dog*BE`p$;AINP}3#YMXDDhn#(vZ<@ROXn|&u@!*~`_kjoBgNMp>+x+^-`S~Yaj zcW_Y}#t+~*xhUNUR{Bs}OrX&?EjE(q>fLCbiJ;zW6L5VL5gOK#k!otVco82OiKtg7 zJM1u-(b$l65IL&{>P1%7e^LTVN;bEN`5gYlL^20a&uFL;&qrf9FlBV$uJh-|n}*aC zatDvFJ$Mr$$4eqDjTn2FBrOI9g>}rp>^eP6~qyaVbF@xF>xV0pKQa1;Ou5{u0i` z+3&|!XHoI2aXEcQv;YrrFVWwNsiu$AY#xkAq;pF@^S08lnf@bh=X_53v14^O={FC1 zJw;)7QV$d^xhhi-Jo?GX_%D_7ra4NdEeyUspYSO`WjGb{E0(rXX2~h3!p}fM1~k+|_fyn=s!qL(kf<24Cz6>6ZlR zcr7B;C5ed-A5q(!+Wn>X9anQ%>XJQnGb2MXyojH0_$DZY#L%|-`E~8yZSPo7jhZ)4 zi`=AeQ6C1#kc$Tn|nuDk1=9Sl;O6{l83A6de+db0G?4SJ#6! zGp;`dF_d;@RA5lhDizPEvlFI=C;=u6U6UDXv?OsKcGhaj@+8(um4ejW!!b{$PGO&b zEK(t6u(%(J-Wy8eu4gxkMwYG_0HbGm#rqz;dovlLi%+{{H7e&$2D!}ERHK6xTE*zY z*h^BXb>C(q4sRxH^jYDWD%?6j4s9_+87>g~4Gk{bZ5W5MaUB)T{IufQC`oGYfivJN z+16Kw@d)@k#RkZZ(GnXP67uk@d5sdnr)TU_)g=t7R+`wK|G%~*?_in5NKGyvSv5p&GxmN(VBpQA6@8Rj(p*g2^yu-| zypWb^V%}jtiS>W@pJT>|pw>EF(kRzA7ZO3(nr{+#$B`X^)4}?gjJNRp(NooT{xh>S z(2_Va_6!ylB2Ro?pMk)j0ffUiRS>J;@9KlJr!pImL(SqsPT>N&Ws%QR&U*qQ{rFU7 z-;ArN>UiwGPZYg?yoO(GEWv&9y8bfArrF1tHw?&d(m(uZ=8aXEGulpkr}Jh1L3X&n zorIQb?c88zR8&J~fSHd^&lu?2(m04&UnGr@c5HJ*qU@!gkNR2GN_da(Tc)BhmuUb0 ziY*zM-D8Oc6`>>rqNAQ7ZfVHyuTOG*B*BOSV-7rm6=BT0fSst zIK=n+{cG;G>B#n$_dj-x)Sdy8C%0)(`$~KU5{@AVJTjVTC9LMY4!7~m4vA(y!_=E$ zi+uXR=wm#4N}?Cbws`g)BIt;Fqa8~9(l_*agrrxuV0_eLE`LoKlM7H#lZP?L&Tivg znWHLmIO%p|4rkFE9GSAV9(`U+ks*N<&tJbb{1<*6?@F1=)N51>s~?q}j04gENf&85 zpI>&nAug8GqL3(+lb!-`BA)dLrlG9FpE+e2crjZuRA6S~ma5I;75E>xpqt0%k(C=s z|KuMnoXw8su#vLIe|y;A!BXSzwdR7ZisP1eoX*id>03xY7tBZJ`}8~Coak1bLC6{laaI@%&mrFvJ55)^+SI_K_Z zdydAE42hyeo0K*otI!km@{}tk$|mVq=I^H#84hpEnaW4UDkK820S8iOvO`AfnKLrO zl>1d;?D^G0JRF?g``m!upz5VO8D6^VMNAW5gjDS{V^=lF_gK+i=h&&_yt&*~P&*sV zJ7+*!KrHo~o)E<7iPW-sYI7DOu6)(FdGiL~0oI%LL0XDgB;2j6u&5}7x}tM5rmdHZ zg@eQ0(xI_&9a{2&ckddf^xE-0A)?!a3pC>bxU!~y&COJ&$J`d0If#2$$f3U-sTR-q z#!R6>lYmoB|Gn??^JToQ9?v0#>qdc8MOrJ)UEiNAV!dm?z+fN9f=Xw7rVr#IRd$Eid9=5RM~1R)X zS}vJUycbcqt1b*nMad7|(uZe@a<;YD1m^cVuxsLmsV!yNlIo?CE$n5dPIj1|_~?ma z;=Q1>!*rAM=+WW;Ev?Kc5imMSm-a?D!xz7G`g?D$7CZyC#K$PoTT~-w^vCOoFZ1*t zI*0%DWlo9tPfqg_xBOajnUwo7r2ccA13f+I64+`Y>b9=<^!g?oOo~9uuk-I*7+r-H zR}}dUk2uv=Mv{{4OH+@=C9X=X9h z%4lKy%9QC+t$F8}0Gs@xk)}Z|NDgN13^XH#(dC?8N ziYSfoM0KW%0)q*$(TF|lUoSGG+7#74`(w&cW*VTK@Rxz+x(Pu*#10NS_vYb2s-(8F z!&HRK%DtLX`Aksel_h-oiHlcS7tZ~JWO|o=Q%?0AB#jG_==EeVO_s*RAc>qb0+PuP(~*Udz8xQzJD=YV zLi?W{|M~WAd@t27T4J99AN>n5ce9_q_21Dl=j|NShA2)q)%jLE7gj?64N0E+McYr0 z@?Jv_pk+l_$(R31%f#@)u3U?>3?8#R+{r8UGBHU=Cf5DOWHzTn4@j#^Ih=c@#k}&{xQrEeC=?=orMb)2$jPnGd$eC^X0&lW0zVL zwRLxVev=D=X*id$iE>k}p#6@pcvNG#@ZVy^pqzkS6`ZVPQJQMfO!sbkN;7ZYZml~+ zp*S(FiE#uXgVcxDuGNzC^$zJT14BYm=t$7ew|@X72_-P~^EdC|W9;nOY+F=oIa{Q7 zssHK#tUENcf9%B5$nB`vq)9vHL$47~HJg@WNGR9WoV99d+$NzPe`aKRueIEgHzW7r z;r8=JtZcvg&zg;^sn?s5G{EoZ!Y6sNd)u7t+xyh18F@XL$z)~(0W23i`_Zy&l@KYx34$2Z@9%sukuPerW})k^cvm!B^$dk}r)sLuZu+**9A z_&%*BRUh@U;V_r&H3wdbo1f3(H#mfQGMB&t3B*yPy6S68To%XBV5Nb%9p!^?1}u8L zK9~zvaY!Ulj<0l4xE!hNqeHV}^Ejigcey!mYJ-+$J$j@NNZjj9*+PZ(}hcdu!UfMi=^6!JFFC@7xq6FgET6 z**Pc&KiukFBnP6BBe?QaauW5M7V zx(wE9GH+zpUGHMuW$>anBQE2!JU@5zy>*Q{$z(0wm^X5JcSPf@77TWwZHoW$*!_Y z04f!G9XwmD%pXopek`aS0in0Y9H275#BFEHR8Xb4`7=~&l2aXA`r_8a8P5oohj;&kQGI_e`L`O9>4zu`=$<1m< zjZ#Rf13`L3C2yAwI?92(m9gczG6MzF5PZ!44HO1o zYo9tciqT`Ssfu-kf-m6BscGp+y^G>&;5rNltz5#~SAkHX9P+3Txf+=Zd}TJ=wQ!x> zk~6%I8)bSvVCoe@L#&T{`OQykOHeE}lT}PIWe-ow*r^dWvERTMqCE?$ui7!&Qw#px zF*|=BRR^?M_0kLcG$Qz(roaFG*mWZ`bb8x9t@+0`n>Bm}(wgdx5wAJF|TE$j85a~k)7cI1Gs92YsVOD9>K5Y#T9&;!(L$)bc}1JGIhO` z)e$@o<}JVWicm+(MU9DNa~;Yl=~97QYd@J`uD5spe*M0EThRX4$4t2v)bC3n`g{8^ zH^80P4)?ckQ0>uloIde{(+b55*YkvoYI_f17$59Oo-W7J1mAC8Vzmx&?w0lS21!fQWl^SYpo zKz0RYhmRbYc`ta|^p#M&fyV~d7MtDovLW{Fb-6xET=8s( zw5a}3rN)hH8?pYm&9EQadBY}|n3>6ap?iTF%`d}Op8M0dP3hwUTyH>FOCAdX+_r~c zhnk^>X!fz`+NvhUKWI3|Bc(4xIRaQ;lg|>9z)9Mj@36MPbQj4!7ogyc*?{KRAoPrN?ev*T2!|P|hu<((^t_0GAYIL0S9r)IUST_z=6C`!Cp>s^4W6*>Tp@=za=0#;!Q?y_r8 zR~IMk7w6CSlEd8W!^4q>?wPVALDB)E7I5eGn7{1p+v_n? zEeZVt+hYeAh^k{91G1S?eVvVw00Pc_hq4x9@0rR&s6v-G+yCUO@`1 zBA{Sut(rA2J$>4o0-$_6{R6o##*;f1vVvqfqg(9wG2#x8km_NU$v22I;)E>sfbm5i zk}+LPzzSm64;pQ~8G|3tXIh$CjpH-Uo;fq9(JI|VBc_~>Mr;0;qxWhKCc9^lBvi(t z)l{VVUw;VZu`YS+*s(0r$PH2dyV=IIF78Wt9&1>n4-CkGrdtPPi! zmGqpS=aiG^-DfQ>Th+_lk}3)q4rGs6cq1={J^V8-ZJQrRZ)b=2rtF$cn?wB_#xi=6 z+L76+d^x!Gm3g>eoh;I%Cr={tSIB+wLzSp$^uXocBmljx^`iet)C{(?E&b2~1TFLC zwj-tEsA42^1Pa=a?_Bbk5=kf&9HL0o)DWu)2gT&b-ft@C(d64+{PwI!k>0K=SPgHP$oN@LPu=~%#wYPe zrmlEP|15hB>ZKtcm-<~>TS*{Tzdo7bxg-{qy%4(?a9MJinXkxLk_qVO+ds>I@go!C zMoDR<@qH&kfx4I^R<9f#;OgaP{Pg6MYxRJeM)aSJlm(Q68rpPQ=5>FKiH&yFSL@ri zE2j&ixP{JS?~6L;UfQQ^l^nSt@=vq}+ zy?*to>9Q}D;M|+1eUi1$4O7w*`obUcH5R_1;XzdBFop^XaA9(A1}YWiEx8^vn>H2u zg10f)AO_orZaKP9i(cPo>{qmbZ71!UQ6t^h6&e$f2a)QDfwJL@>be1JX1|R3YF-te zvlzMCZlHd|Oo||749VeypB_1S)Nq)h>sy3^BKW`O7JE=80+8#Y%|##9URTq$7SbrC zIPlK>Y{2QxK|!roke}JND@NAiiRB%S8*i{%?2~Qw;k52=kct#@vzvh9rPN`$lnvFbvr3cG;yXWERU@c&aJ57hF^PWoOuN_!)>et5Ga^mnWKGars{Ep?xJAI;^xn$gf`&Qh3Q?vam?VJ<0IBcXb zrn^w|2<~of{+zp6Rrr~iol?&jxToFLo18Fno}&n~^!83jln|wf2}>G#cz9^-k6a?+ zlbMf9!4?Z-77|{qmZRtQ-~g79O(=u2q@@uBDihH*TpXfB31$gnh(K0({613=N(%2j z1e|L!)wde__4Qk~bS(OvC$*Y>;j3OdrS4#(O_!lS?#(?<*+eG6W|PLY{{w(m#^Clu z;NGNgx2DE}z1PtU@c$x}yqkspEynBP6rWN4eoJ*A^`MSfHS}D zc|7GL%0b3?1vTz8wl&`Sde=&JRxkhlt)A1@GE4LEptb0ewC>9cs9}TeGDS&QoS=5X z>GDe5`vuk<5`+}6#UErL0o+IvM&GEu$&Tv*;L|b|lJ^QNDI3Fqt5}gANB}P2Lp87z z_0hJ85FZ?g{%>I6Y&NBH;Q%QkX9>+J*d?^6^9$Q79PL7?OaXPfk|yu0(pU&f`wH ze5$e0fN(IZzQD+d(wtBWsgKSKym9MRGK{i@&ew&+$W54&P>VKj#!Ai1&7`h&)B}GD zdHaL^6c`3&gI*uBDQig2lwnq;kFI7+H`EK{SjvP@o1ipMB-@f8*0}Ktz(6L|we)Pp zQ>87Oe0gQ9P21PImSa3?6imNzVp-WdJLEqa%v2q;aN(JPiI<95mQup4TC)b2z$NLE zR%Q>*3v47Bc?}b2Oyd`=^sjxNJR(Yortx#++h)fvuDnZ7mWoan!cDP&Ye&xHoWOr7 zu8A}$0@`ibge9u)-)HF22#_6uGC8YFYtwnu{FH*2&-nryZRR;E=!N~baRZ~ry!lBq z)~kt=X)TnRDnr@T$h3bAWnJBuEE1iJyp%KcB@U_o?0O{IYWVGHt_7J>&cqh({oaQe zV}iRzd4Jh<@FNvwV`1RYsgL+(b#rtuxByo~U`W66y7aN-A_!!AJaY03;^u{8Yw53N z;7HUDQ@d@BS6Bw>NJJE}`SS7X2ytovMeXR}Fv2u$B2~%4=aR?Z?7Sz@!QS2_Pq76?4@)*V_!a(u9;7@&$VkE4Q8XdLL&sS`RRbzokW#Tb{_vw!2!oj!fK zz@3DJ@V-mWtcrH+*mP!_IRLA8(ya_bC0>2Q`U!+S64!cO@dOXna!Vb(H(J7~t}QUl!(OqhN8+_Odc zHnTctij?k00ldVHY;ymV+w9rNKvoLT^3u+nY|~idCQG*D!f2XcyrAmZ)&~G1y?{5nMDc3*AXsg#1RS5-8iZ`joAxt-Z(7m^Oh~T-+!Puew*)3T-ZxI@TmV4zCWL**Aq}=-fK|| zU%KZx?|oN&+>;hfXkca^amwUhn|G-`3CH(_;AwIec%Bm6Z82S7Z+Lb<5 zzkJ4~qp*o#kw%qdaxU9q!^AErO3T1TH13>6$2Jae*$d-=d1g$GrM@DjtM+kYD$+m>lu~?IvZ&`U%tYTFCCCD8sSRYlEAnruIo;iF8*bqqrr^z1#Y*^0ByexXh#t9l?ZAZ^JbpT>?9*$6$XgWA zLx)kg;upRf67|#9)JLje95=MRpj+`?-5h)?7WgzYP}(*7s*B|J!<}zq4&RqyW(7+h z7GuL-jTqrWxt8ux=1vxP=5&6`=svM?auVm>dGaKXuA89hK%c*A7jx`?s@UmTosNMF zCcAW0Myxt~{g)9QN|IZ!m03YC$Q#h$YCgsA$jFgaF8WJG03H@ZH2hFvKa=riT4+w3 zQASeh`r%r$n-o1pze|@9N@)D92N?f=leO`)3%{vx1WE02d_F%z_0ru9ktcl$kR&Jg zmDz8RJaVabo^<|q?%7Sq#*&rf9)f`JC8Hb)3Yy+6em$?JxcyaCcem-cn%~l92k0-c z7T(glr_qIfdCS0;Eyn(;&Zge`GPinj%}kdKQ<5lLYv2c7D`_P_!R1Tf{lr=_3u-9AtxZh z1Up7nxY_jIB29)}A|_p?G0^f_2J7&H)WAHz6|KW(zuB1T!jic%$M@TWx+~S5&zK)e z)g>b{wQ!L|G{*)g4s!7$TUZ2FUd;p8T#qmiJ1f~B!gb?jB7 z2o&3^Fd9&u58dFkOn3rwV^2iJ?KNzt(!oKXY2B9yLTa>ewze7|1(sw29MoMpi-{-n zkIiu5ls}fur>9qM^yn^0CfjKEI8r(u;!M2E>@kWoO&k+*;TtOA=5sB=JwhE0-#+st z;fm?IUN}%-P{r+4SBzd45)$J2l!c4L3hTgd?V;7F*u;gzv8(v}{JB&7ryHHjzFbJN znGQUoN1HDY81P<0uA~&2_XaH|#7wExedfbUIqv#;n^r_sV}o^#y~twS-e>Wh=gV5C z!7087?vX9H#_);;%78E5%etsBG-){}h1@aB3zpd-B3{;PW?M~UzJw^WYonW)wr5my zgMPnvfWtP}g0)^E+eeXm^}AC5Y7H`O;=pTV|tXW}zolKeu}Ami>MDy!=vMr}F?P zUr2yDAie59V5gBNONfvP7CtmEzihn=xgM2Qi^Tj{%$AzVL)E`*8ocg5SIeL_3)G|_ zEPv@&P$}L1)vrhd`WjnxE(kd+W$>^H z`htgk3G}tBFuAW;I?T-pnjgEt&&0Op^SvqANLy9i{KwK#7V=IX<_@d`zw92b?AGj| zYaEWfm90=1lCDXBUgYqOYzC2f%ID`Nl_ zq?wTYF>Kg6*O@fW^zi|AhK{UkVxXnt5+A@epg%ieA2aB@lUWeD+ht`Z56%d|L;<^Q zU}LF`MCmscoVIZRlfV5~==3r1>=a7E05_^Mug~}YTI}Ozb&WrV90w~=W|oj3w<2x- zqEcl{&#r;FUJ{)9?EIFCL(EVbC%At99yh?$0b=_rY494iqPcFR9pE(3B2hN_5GFeQ zrSkEm4F62n2hrc7f^DF8v-%}Q>V4$ONW$*B`pZc3ZNNMVXGT#6gf`L*p_7?Nu}wGN z*KhW$S;2@f>#p2&eD$hTHlunew^@?-8ozk-m{Z_j&`m7S+TQDVt4T>oGNK&N#m34; zUvJQoCFd`frh`_2Mg=gpyxUSbb@Dgt}=sZWFn3xd7c#3et)0gI)Q|;#FU#guC1v)6(n8@1WIT%2ie-DhYO=o6CXP! zeyLGz+;FR|(uQT4Ro*IJeSVGg8@+na8?~xSMX_i1eRbEr3%B;Ft(h5g`}UKDM@K9l z)UPakKaXD=v*O$@DEgV_-M?o~UC=xGxOuPlPFKD-3Kqs8M*7-46&6M|vw-*;Zoa1D z=p}d^);KDfx9*%junB%!w`PTZiKWeEAp)wtaTk}eXQ7tE(s!{R*V#BrqA=7cgvA?o zu2Se;lWgCg-ren6?U9ayn@2es;7>2@+62YfcxB=>JVHM!``?t4TMX(1I|x+=pWOIvPp?Kp#WjK6Ma0os63 z&rXT$C5^sL8RQ$0ZL+rZ|(AeOQLyk^(B<$ zBI}3yOAO&&3wKE00{uVb3_|I{{W|uD{R+e3VK&=VKi%2?Ks7Zs zp_bw9=p(02-Q6{ACQ@Gb#a^&qO^gp=F{G*KGFd-YlU|#qw;)C23=9SbIWn}dFhk?c zj3di2#67>xsW8`R)JTn1Ewi+U2tzUVCymkG!zs`0+i1sKwVe*Fa!alBI&qH6fs;b* zM%u9HqmoG`$ZH!4oKNv)`57=tQo(3?F1)bmaiYWG2@@jC2fB`7ZGg(SeTZ|NI(0SJ z%oGl5)lD+{`L~PcAHp+=nvnH^M2o(nn^zjDvpMk}Yg0Wrc5IjFqW5sKV2(F$4o^}j z?x1}q>&S`=;`I1;yPh!otu_58)!J8Fvz?aAq^*pfW@An7L;28}VR{FoPNkP|# z(!>x(%cyYwH!yXtxA)dJ1$$US1)-PLg#0oDRmT<4{{9Y6sWmK11NxovEF6XFZd>Uy zQ31=#`XIy|z=t3NqetI8@npQcJ#*TcGV+HZ)7$p0QkXYo+c4a@l5z)JV48Bsj8+*B z&W@^V-T7k=O2^AL(>1Hw6aN~(^o)-+6}UlOqX!K>KH1rpi86m^BvgcsSYE$wU5V9~ zEDO5|MDr(3oRAyWaF~@}Hv~TPwo*y$Ied7y<49Xu=HqqZ?lgcJx1TV<87+!(4dZ@e zaf9j2H=9ri)x0DgCZS;eZCDR>b`7w*+^|TiA&`8)1Ycim8mUPC0ZJr-1vyEQJV%eD zngi}rHQX{!MkqP3##O2Q{zhwQ#yQ?fazRLOZC#;}1kkxp0dX!aj#&7Kj?C=4va(iv zYM0TMFIi2NE5QTK50t~_DbjaXT%&)h zHG48$Wq4oa$JAmmEof_BdHcltHMsG*WYF~Q>lJ29w1jil#Im^tMc2Ty9w>;!F6r%E zO$|ns-P&XpA~(}ayMSrUA!QJfi54(sLVk)gaQv-c95!l??S3uldcZt2k{y>QltrHN@*S)@olefj)uDWKt5j}Km9{dy%yBV%gYPz@0*B( z&V-Xf&UG2&`s2qOhyISf6&Coq&Y8S#52DwaW|#+A{urAElm$~r+!ridm`v;B@#bC! zZ`(bY&#wk(i>`l8UfA;0t81y?<^DgKsxX2|fa{Ux>GRZ_= zY5lUOD1`f{PSyQ8N#?RkR=Eb}3*PMUkVxd&Oa;DHpq z-3uaD05{eg*y#S(UQK-&ZZBg1#vYp(fM!lYFcK1;Ib8tT6G{jztQDPKSXam21bWtL zc%1p=o=+Pr*SJ%#TEwEseD^=w|FIw<`L7Wp)?Z#a*2QJ7Vn)ziPE5;-A#m&B;^)X| zH~CPZ5DLh9HSfpA$2V%)v^`~CXLxD=%-bP|od89Dkr{On%zT);x!`0pF%hF}CAZQc;Wtcm4&pDkOYNOy=`$GIIjSP*&`W8x$ql!3XYs z?4CK4{36L3WxxDZevU`Sf|py;&1Tb%9mbDomexnrP~;9Jypp~V6}p1qZ5y7TD$<8k z;@;SC)l-%k1GIfA&L0#v3SCd--e_G{RYXlsgaVY#Y#3_%mpIgAqP^b|-0Dm^?^$P6 z6=mWYiteG0*9SFFKxCh~3*{{4YM0S*O!N>?YbUr(Eu`O-mx$lui)}%H1S^7I+J9w@8n877!m#nU`5^FzU{J0LObgC8|42s!Xj|0{Z zvD$z3Y}T&iX%Dnhkb^+yiLe4qlE`d`18X`n3t&iA5G*4<7n#;-yB%kBzb=M_>Gvk@ ztRe$<1A7jrTT&qUMv7C1~=fWC15A zjBtfLU3fq7@oKISuV z?ASH9qpZVSzay~+u-LO}*MKo+!L3R5)Id-Y5W2A&k})#21~N%J5j~&V+c{P;EJOfYqBIE<=VB~Cr^&@ zG%KWr6fOkh*CW-&8LNXI-?cHDAu9`*NO($N;BB)~WKsmG#U{qHW%j#76;JdJ_21u@ z1`^=EAcb3)EtvbQRd4`2o-5EUlfACw?K3~JA}|e9S8rx6@_z(mc6Z-y-u!!K$HeWy z?WTucc4!5itYxrm9LR?d5WqvI!g48WxORFxJVu4j2}C7(RhTD6ys~uFwE~eg1_;aX zef?nPFQV;j>k>EdHw-T7+L@kX=sTeq?XYQ#Qko~vqN)_ZWsej8gJIxt3gqW;vTz8_ zSdjtKRha$pV^5(_^9BZrdsjP<-Hp0+^pB;WPW_P~Or9Hr2tOsOM}j%G2l}x$mHox! z$IaJr^kefSE^@+{9%g1&m_<*3xW284sT;=y5qbAQaFH9>43-@%sxJo6-)}38*OR5VR2yKLa z61(M{APfJu7D*zVg1J`LzB2$)cnB;xoWti>q=fSM&q*vZifj`+h#7K(oP5?Wu2dv_ zAcb#zS?yVT@`62FWPBcOt&-1d9n9W-8uJKLX7%Gp zwC9KfcrXT6Id%H9!p0^@5iqnK45s8r{;+qX+9dF4bYfXCl?DT^ z3=bJ2(8HagWvDD9p=OBr`!{b$SiE~WZhge&%?9U<)cedDmB8wvcAlUd6lY^0l~EplPQu9F{na?#8Ev95{I!N0Nqva zj@EVABtq3f)8L_m!89Dmq!QIjxeawnEzU85J%8+)|6?Z%Pe>I*NO4Rt<(hY=&=Ypr zvJ$2ux9_w2pXi&is-v9h)-3{|%bdu>235+f)& zy0xz$_)6wWa|ihXXiNtL%%=nk$@X^wavUYbu5E?t)$a7&FFc}Wb=vJkfvhLrHmDo7=+0&;+ zpY!kD%fbdDW(slCqXwRI|G@)2bHnvPLCw$TzO^|^ypr{@ zCNl)Ok=G+NSbI(D%gjfjYHBiRDba;QY5RNbb{J)6r^!9n?hsQ{rXa^ymb1@;5jS-y z#Yi11aOZcE+Ro(uS4sZkgSNE5$AH8~CF%In+>j0=FqtAs%dm}_r1im;b~tZTQw_G2 zGV2N#=3aFx#oe&V2?qydo}`{Ts2X$M&=2{&cfPi+vG{%2)6Tzj%)F`ULuh~uhYd3K zU>@{b6qqz%bRtB`GdUD}!aB|T7A&20w?b8*%?r7%&=k4XUkO7agCvv?ut9ga`q5^~6+y=hmQIyJmfK&?kuy zhy7IfN^gKLiwjdyaS$#Id76!G8x)H^xnEZQwh;q}Ge%_zr7pTHV`B^ZNM{4;398mL zRnC$0h-*vVNx`;XSA}COOMdXL(b43;GT^;&$9-QB7wr1fqV#jK`!)X%B`{Y3SIQvt z%!%2!rLk4NKnU!|qgkb1xR@U$^Pu|?dxM&y@Ga53;)n9`I)@)>4cs$OWT2k64BCU= zT)u7FqwrD;P&H|q{{DNW_fd#FLf$UZxu>)>4c-V;%w7ADh8$#OpkvoAT?8SQsCUSE zaH^WX!lIW>6l<_do9bbtoMB7V0=HtxQ|iaBNk>vA=nlxZWPSAD6scBEbhj(ITC_j7f3< zNl3p-J%NS-aYM=KTSJ1jki3G7;$YtU_x)b<-@1P*jz_=JR^4Ch`S|5~Oj9*bJqhks znveFQW{c;7jl*6I-?DIq&N8KeF!J(SqP!O+Xq-dDmW$Y|BR+(7wO(=S;w-6y(Boj*fFa zOazUP4&Ck*;aKmwgF3bgES%!=~+a`OcxmRz!odRd$#m1=olRU@P1Q)5-(lEJFr$jcGPCHc~yh1$71iB8Bd*x1db6fY9!{SiqEW;@^F#uH7<`!{ia z&{^d{>`bd7#ajya4Sd+^m0it^cv>P#35n|eKwmG29>%rLiEi&zjt0V5Wse%u-_ft1 zICU!WpudZvc4EKUi9SPnfm}X3=h0SlN>ER<;^d;-b-Dl+E9^0cUed{_H-+U#MQUq* zj0@0iKBsqbZZEx4`56t>EK+1g4 z<)G&ox_#!G3yX{QUtV;aXOBNJKx^Y>1jf`^ZI;+#A9te3xwCpnHP|i^h*uSqtE)-H zgJME!pR!QACG3rRA=nXH*G<|yE00Yc#DvCX_i|spL?~l`cHXUa%fZ2?+xN8%;(*xR z&wIRier+}4x;{RbFy}IP$XH;_pJ^=1K_JAfwvvK}YYDlsMt0=#11b8H2Rm-hzpfYs zT(m-te&QiOx6OTEIk;}=q5Bmq7^KFfJ_J$`pe6Zq4pffaEVhYBj~>3!@d`6bRx!kj zACnyiUYuWTIrQ*PaDj4IA z-80%=j>~`U7#w;K&U$^$EH0i#Y>M@Ub-%j!a^wM==iD&^5UhCidUoOolmdBE9@xrz6@MjOnoOflB5iGU*3dh;Kdqe%MVS0Bi>Y`ykM#!a)SkwdgJgRc?+1;wB=N@q`?(~T2-qlchfZ{NJx?^!%G z!mCE)8WM8{gO@MvoQP;!y5fs(kwjvVtb=vp&8q2!g?)4C*7sWCur??VEK~4bZVvt+ z19TVPzIGJ#gygJ>Rgn;W@-2vjx^gaG_u-gpsi^_=r)GxJ9HEiuxwB^QNuiB*-M!xutZ9nJ!ZO*;An%m2KvVXEQ?K)9BGv~g4t;>24rx9QvY6|Daix8-hw__QLRIV_E_`7O7){ zB_Usnv8Xc2Gzh#OYxjszgOuA~5oIbzrO**B$2of~x?DOK+81D^8pDU*R98hCD@NB6 z)XVycz5Ju{)%VJ8N2_*^vp3=Le9#|NMuCG1lt8*2AnLrU@L;NbLKPl(-`0>%Tn zbuWv{n%eaE+ig(Z=Hl{Cg#2oH{eVLy9*OOyMMZb@V-E`Vr%A|LS!3=FI} zW10@OOe&keekoDoPB>LNGbiUa^eyVhQfy2=d?)Tl*QYo#$3L-C-&{I9p7|*|bz5mf zm3W%YDy;n-9Jn@SKRq}3cJuGLaTAaAU$ZRJ{X&h>SqM9(-m=!T-i-nvIyzc7dB#-l zdV(1b0J6DMpmf+&w*}b^)HqeYPikvAk6MFnLOGrfp1TI&kiMxE2ygAEV(i(H`_}AF zt<25B3kh83^7A zl~gxeDts~Wc))>XPs)!{R`6?l`23lvv#XePU2~rV;VGm2g*n+fIqhEpMQwQtBXKwK zx?bQ>y|(5?^Pq}~EQImoH)WRs#Up=X)=`om%40P@qu?BA+M)u~+OO~2-Cp}7TLDpBgR<@H+qMX-sc+PFcIoBAoJ-_@ z3zFuAdPt~N@ZyQi-*IXnOZEq3hNo0P$X&X0e>eiNj269N%RZMF^(_!`t5#84`?bO< z+D`96TwKDvAFg?ee|&HIp`_0zU^&q{;>KEEaKTIsRTx%Ax0A$!6C=lYd*Jx%yu2yA zEmdd!=-%eYNR=pR*8M~O0}wY^`ei1Ep;Fp6Ay%|B+^U>YcD5W1T+m&gHjQ*Oz~#qP zqtZhIoT4x8IQdYFD57=`UfV!#T%B6AaFVVsZs%$vIRLksA%Xt*?QzY!uI3U(dpX8f zO^61_oMLx>*HO6T*Clh@`ps^_1whL5J#?naTFhDKZ9%=y&!J>c8`tcOr*aIXcK9?%PD znA3o>k7!NrM7K^OTzO#0fWlc>R2=C}VTJ^d^4#av=_iOH7MiTLAjSOW7iJSCD+J6% z&KRlERc2T^0<;6i0)VilZ)j;XjbY@6C*+?a25Nv(M-$6Sc*6f*YZXA`I&zS9EcFbt zmmi8Nb^hgQ&3^~WQrYBx=NWSqzgF#Ot~ZT$Vo|ezeU^@Tk^>{qUo{OgQ26!hLo2IL z80oIXMkMn9(h5%YNUTJvY7d|dKyej@qvIURFQsnF`Uvt=^m-SJ=hsJ8eHmdJDNxi2 z3OSHoo;VVz!w3zZFCXBg_Z=Dn=1^P_^0}CwgQX>s>kj7;hd3+iGe}#59jp^}*U1Oj zV?&;{p$Ga!Mh*-$iYYg*WcLwzRK%)4;0tih&uOXN4uSy~kSky(Y$0a-Kr_0=_%bXW zM5l8;WGH~eLwq72$pfeiKwAHt2iZZU$dmizNqqj7b5+kYG0=pbyK&X@JSZ!cmnons z13HaR9br@m)UxpCSc4A-bj9Nc4HUE~mkrC-f+CF3d_T8jArFlaC(_k+2T{vIc8#-% zIE%#d)2AUu?@j{1Ga#Zle?!z26icunQ}VNeu+m{5;sCz-Uh-G({{&m`rbFKc2pKaS zU|YZz)hTR6BEtY70zL#SH)}F5*5OkeSt+zMH$M@29PbEvtD~t6hM993BI5sXgiRMn z$Qd3YKHpjz0*z2OHhqUd!T2DS`9pR#IjqD~@7|56GX=_BEpX#@D>am(Gjek523z1_ zuY-ul6o=i5e*5B@Gz^HO6w3dOFK(SPG!Fm^1^GyTJs4mGBGW3YT?G!43NtgmKg3;u zU_ABZXv+!!30j~>aDvhV_}$5ha9B^mUJlw#BH%b;OrbSFQ2NXn$br<>?hJjYp)7En zniyV`j&ZMP*h`1+$5YT~=E9Z$F^`3JCDjuqrMEZI5x!TmYIffw@H$90Bq2pJAN&9S z`M>i2+7jJ12L0GS5=T?OSgrtj65d)W4^n^S>-4|FU0f%+!O-u7{4b7!Z?2I%4WU=eidXzw$_UW}W zg15jgjEi_Qf3mFKRHL>pZ4eqK>S?$QuJ6*tiT;X8i}s63%To-Z?~2nt9nQBuVGuWW zjJF~0Y~s()7xS9%*|Bl4ykMu~e3$I^LM3J(J-ADtcDVVZTkgBHHTU2i!>J_OzMj#L z3Z5LxoaL()8$F?~p9HbJD=~N6{@Rd;nti{2fZ4Rxx!?VA|IIy@;?1i=161S&B7^1n zCiWl5mn`%mr_ib_Q(|?wIo0HMBjdX;gW)Uv@)z1PA1+!4>B(<9CafN#3vwDg*Pqn~ zX)I|D_Ao`V!C;BFZpK&=G(#T=+;>_GMqnon;9Foou;|?BeLgeuDf~Sf+vV{^n@F=( z^M+Ah&&#AT%wsArj3diH?@f$aYO^IE#|Xx$tlU_JZ<_~zg%Fl(h{$oX7pS8TK~aE~`vxO0S)YU82ia$XWPBg-c}D6Wko3aAf_bj65cCTbf}-bv?fL-* zAO#L98eSAI)z5>W;#LdiKY~rfP&{z@+&qP?d`pJ(?Pw^U@F&55I86XE5m<-MRnvq9 zb1N#4aR8Z=k*SK7mN4kiX3re}<|#(C0a-PcR%mB~ssRfd`u`+EI#8DZ2vbI4MMJwa z><|R5+FJ?$*3Dw5Jx4tQh>rw+^*(_Df!TS2f_*t_&#Zu^1F}qmpFdFHOYBFCCBopi z{NU1Iy_V8-Fts|e(x9gHN7xr{UzpQEmBy68=>XrJRP#f)CQu^4lsXV$0geF5gHRXX z01&OS{0Z$jHM=Q=cR0X*VEhZ)fl%0J>5Kr7jBJ7kiCP?Vn7C4-fJ)&QvHTQjn#XeS zz(YH^t=VIA;msQZ7y?9B4#QV>;2D-Qn6QwYP$Z`U$EFiB=wP(N0&P2Isq;(Vnj*cp z#H=0d;WH@M@JJzpeZW-BEDYdCptqx&8^`84EdB$^nV?%>@61#d?SLBthV;tC00=%p zblivlFAb1Yh@QtBzC$7K88YaDNunjhTTp=h)7+8cDrk~G&G8Ra?w{luRO~VT75j!QdeS%lV=cj$ zD&)RmIJ$aWS#_cEWa}p7)I^4ID#~$*oQE)z3vA9vV=(uW0 z@Y8t#qdz3JRPuz58Jjxua7*Tu+MzQ`>(Kg|=f#(AvTx!(5FEy*;C;;xWK5b?f{u+y zb3WN?OT6NWpQQW9CUB)GAoz^&C-ZkF+6=^5^YG{kP{Sz7^QIKa1HNo%Hz(HFB@^K$ zcNa~4IZ$)NA3hoT$2@w3JFLc45a(5U-nirSZ-EwCZH0Fp+04R3vPH3-Yb4gB?dM5V zD|hII_F<5i3>So#d$h_rQC2U|X5i=An#t5_yzi6plTJ?ZyAxDrsp+P~W`?Jym%o3{ z1alI0VPWg2IK7aW83%?}xF6c7YdvVy$fq^UZGhJxvSM(x=o-VfdyW2!RD+u4cn@

t{c93dXML7LJ_akD$XdR`_)cQ4ZV8RqkK0eoEWl{*tbD#R-+g-=W;NNgJJ$wP;#m08XmP8HIbqZ-#uPQ5-a zDJa8cPNfa%x@uj_*#ZmqoNU8<<@!P2r%R&)nw@;QRJe66v{QX(3B(GD!Lwcw&+i0L z?ae*!NFXtiv_%dMT1Bv-DxL~PeTask5je2{oWN3MW+_H?f$mLmxw~_DUO;|-N-+}o zg+>kFlQ}Qql9IgxFUEErGQph@r2xMf{)^m-2_&j>+K3Sw z#hCc;;W$g#u|+W{=w9CH^qDhZLV_G%8D_!XEQgys|>m!HfBv;#Fu5}*|&uzGA=!mX(Z0eIouPucs1oz=3 zV)m#_cg^KlUBARYQC<*RaaMPrt960cS8zd15hdDun@NDN^7NpEGs2@{BXD>{vvFdJ zpI?<{J2sR8hQPzvCEuSh2Fi5$^nEh!p|0X^x~*@d?s=QrEVt@5v+;hmSv*g1^*3p+ z%5S~k*4ePDBN}gg8D?HXeVD)L;{!$ZjgOh#y=%6CzvQ(Q+J|wvUGG<-q z)2(mc$5TmzhwE-rq`kI&+WVEUg-Q6XyCJy7JXrN5!g*;fHF znAk)pV*+c$ zn461oFnW?+C+skbe{o#MV?+pCMU{q8j4vU>LkgNviqo=`Y>l`t8m4zt;8Xz?uZebc zIYFV4H_i6S(=0Qq>aVlJwgn(05)E9cx<&obTMJXkRDY${wh2JcamiIuu!Nmerhl*C zY-giO-FnWPWD{RJd-g`-(()7S)!KIK_2P6%-`Dy*_rM@=oa`jLL^Qn_YdG z9KWvWI)Oi}|L?}QXWja?lYBloKjj&fTGq~2$Z+6;bcp^17v8~Z$a9FC!>z=8q{T(n zx3HyPhPbJ|W&LV~G_M#}!8&~Mn!D!e0b~jj_8;r^C44_Ad#yZ9g9|<{1xYB)J?b~- z`~FBSE^#;s)gIcMzIp;aeLj=tvOZjVM8U&##xdeahmidIJ6#8`{9oSt6um7i2dwO2 z-_rvX7El1vSFcY4HmYtA?{g zlqyds{wFx6&q~2JXw(y=qEf$R&lbZUTCns}_phLl)Zz z^c)=AHkmt(Iv+0LiG!TH^8gunK)v3X5Yv}vL%|T6n;U%9`tfl^w&o2c=GU-P>2|Sx z6=9y?G{KY6rJTBU?V1Z}1)9IQqmZpn(J=fY=lQ>d_wL-eP(DbEw?B>k&~3YjT03tQ zyQDy7wb4EnnEtHKW;}lmORJ}Y*bi%7f$f3?eO+rRg%df$jV|5&M*Ajq(k=}Wc6#U3 zcQCFU*8xAKSY@@x;%IpUv#ovn=11Zf!c)<=GGeG4-)Cs@%fBairh$t(lnZ$vkvBAT zZ?k3#nL26Gq^p{B6rB`)9$oz&7r~N|7~UO_CYA`0!(B-^;l-U1D~fC#7(*(MCVjyZ za1Y056zKd=swsaA2m?@@CFin6E25{;=&)B0u5Ggi%8UC9jK{GYXiUV_%Z*yfmr=59 zSzcQ}T40KxszURYlS%&Oa9n*-=#!D>IWuSt^+N%TZe^xh5M_mg|39EtnDt{SYG;Dvtxf%2_8*u|(N;OQfJwwAdB0i3yV<>~ZJijE&w zS!!}}{ZlpScCMlKm5|X~IWSSYOI))ed%7{%CMj;?!!>KxAbDpTdzzDzxa8j*ck#uIfgfiXnx2)?dI?zyD^YeYHskfD zT+A(z{Xo+l;brW2sD*Lb8@p z3Pbjah)QKb8wyuSsebR{%G~$;d!9d@c|ETOU0vt*{2s^g*^aK|mz*DfD7;${q1%>( ziTLcV!6*grFc7) z(^ASIN>T%1jr&PJL#IwMgWb4sW8kd4*mpn~Kx!9lk|3J?U{jHJeQ4A?&V7F|J@d#%Kly#`#Xo9i z-xPE)bG}3!^juy5Qo*@7M^QH%(S~~=5Bq5ybqnm|G#}TjPrwHK1`wW&hF7ycAu*W1 zkdRU}|C4#svEu{#OBe4zp5|@PjKdeP;6i*n{j^C-E==Y-y(FR6DA102(HdT3dKUh$ z?B9O|HPL+Eg7w>wU@(}-DOE?Sr2K@^zrDKEK|Hi-tHrTVU8Y>fyy}6xA&`8%Wj&zp0IAC% zd*PsR>3sk=X_~|(dn=dc>-YFlXK?{h`O05g++2iHF4doDSPSh{P^DR0aG&X!lNQ{&91#TS| zwkXh(KCwS@aeij;k}93prmzkE{&luy?e96wpVeTn4Tgr}=1ygLi9A7_tTSwsqSHgO zmkoJ7^GDv*A(eEDwY7D?`Lk!cH`MYP8?$s(@pRPITb?$bK$u;M$Wf;rxRbHWz5xqS zFNVbJIzb0iQCU-Qoy|m<%pkBNn*X(!S%@3q9G(iSf#K7668Q2Alco7wQ(!msMTY+o z^j<0K!6S-{B}oMWUWy?4-w>NC2zG;rrCONJWsonjK`=VU$_nYD;W{4|v-jx`+L4KJ z2O$s?sV+Y;sHnJDW~_nvTU)Dh!f_jOjMb+|HbSY1TevC?pRi$b{pU|b_3;7(5k-E( zRsF#Oku%+37q{*SO`9j6i((3$9uK4`*%HFIkj9hp4w@eRFEti$cEtdw_J)KBQjiEL;xbJ`A#9O}fm(91V5y z75LF&WV1_NSPuSgzKsCW-*W+`bB_mu@<{wUIlsE1QZ+QFW1l7=CO%@6#h~&3B1SbmiC7uOKDT{p!K|F@sm>)lN(8MuIAs-oL5;YxYXdj zka67oYeRMbxFq>%hRMibq) z2pvjOVk&uX18+Bky&>1AnqoQ)@%mJB*kN`si&Yn%q;!{(OzgtU6)=}QtqGvpQ#0x) zfj%gL;LYC_muZUiShR)-hRT-T@nMKiDAi6UhGkgtIAsm&nuR`V8C~hgcT$x?Vkm6t z6b`jX%SXmNIw6Th53yvcWceF?&oKZC$LAdnrK#Ya6=iDYheoO%+YP#?tvv;efI>UU zdnU(+UJaubLEa#P5asWu^}gGh>J{^I2HX4dD%$H%PC;a+NIhQ|(=>$Y1 z=O_s#_U&~OujR7#dD7`qr;LVoyD@?dGj+DMT4u~@6bf3vxpSQXhkUM}Fp?~+Ao@`2 zMRhyU4{dbAjFmRIhAuooSjgI4Cjh(=-`*P25;i^f{P{XYgQl^)m|hc;xdAA8O5V*U zCInajJPv}yyi_DjpxT38_?J|(A{uJ|13gJ&>nWiei1yV^b7U5dxW%o%7{lnP(P$uQ z>o3!MiDK{KxCjbc^3 z4~EhE7p}L+L@4i4r8}&fg4ZI%xa50jw{G3L@8 z%boYkQ;Dtp6$3a%rw~lNQsX#0lk}RT>?hIyK5e=5^HQNg8a8fxwm8Fc)~I&zyLYVq z$+F%?C=~0k@?%O55iAcZm(VTzpN;kuv)C!@>XSWJ!0RGroNw{HoHx2Bq8c>{FnQd# zTakOrGBIk&{o}1QKl1fb(GMMU){JdW(C-O#P#4WCqTg-;DWc2`#hXuKfA7-9H8bsy zCKEhyjw?_z*K0TO0&37{Tz1nfDQvWKZIcM^))=zK}-eI zLjQ=#3+V9xw*!z3e~p%V$B4+UNemb)nf>u7nPxz(rYDXf{*`?i%m(sW2Zx0T&KdiS%nuiIH%n_5uANc>_ zvP%gp-f}rfeB4pjfn#4PVt&&*IWTA#+oR<4h+zJZmPjVVfF8}aMRv^- z_1z;|OhKeV!Ud8(H}{@i|225ROo7r}S@aOXc%JfUB$xz9-{Y61E>YJAs_fWgGekg`~KaA@rweP}V$jkm+edODs~IvDve-rw5Z{wgdb^B)ty zgMc`aOsRt>xmz)<_^4U>oAe?8a5K0;2j)9@GF$=+wZs;5$kEfV*yC7ygz5c|+sF+w z-nm!kYqa_eNuKL6{UGne;PN)uCO9m)S~u+ZMwK5Gs3#cIe53( z!%JcuREHq}p8%kC+gHmZP^NqlhVinC;nZe3K0h@){C#~`1p_vDZD(4zs|_B<4jeee z4x$$B`ts+vkYQ{eOM=RqMlp3KW3i%@Y7yiQ$UrpyP+&lpFVaKpq7OTs zP)z?8iJd5tB>AE50b~+#MPdoSxW0g`b@#tDxzZYpZ7~Wi)cJfy9+QdKGD}J2_I|P) zCBG<=zPY##6N^rjoU%n|az8=?h~^XTk4{d${P|$f>C_vCgo)(IJuL&t|nQFsa?9+#?1+BNUrLj1q%4_lOyqo*YcvwUhW&B5?xAt135HFU?*#BuEn0B+D#e zA8f`kDM$#Zepf&|VQEDj{$}2BrwhIc)elRN?g^kJL4>Y?=!Im+YdL@3yq8;ogc61a z4=q}!(R|;}gdX2>2pXq<{Q8|Vqw;_OqiDSXz1x&Cmop_$wyH(xIe; z=1U@ytbZ?zftn>Sa`LU^wfpY)Rqf4snK1mqzRBS=i3JJdV;K5T!{d?{k z4&s1e6#5JAkao!C%85wmq@waaYNw&;HAp$>9C-=rl#VJnOQ{WEISpy*-k+w{&TJ@n z(cZMf$Lb#c&RBvYoiqS_W$|waRs(&QgFk^?j@1^|>unFSk+N$B%mIxF4J9KJ|q~ zK%PzgZJI;G)IbrhgSqLot?lG(@j=G4X3dnWSgXLtiVKQV zjag-DG_BgRef!^8a5Qs2ekH1AEfwJ8KGJD%*YBn=3sj4xVR%OvZaM);t5zw2N6or% zTkprxXcTAa5%3h+BH_u3q!}!qs=FPb?9>D9`FA#C8z`ntxGi9-O7FkePGkWQHLiB% zn|^#tk8J&Ep%UAhw(bnS`dNu`5#)pK0>~i)I{|)O9Phh zd)?(=;YE0;T3WK9e9DPgTl^v0*z~i+S6{JyBR$*uBhFeg zYg_P6il!QtvC(&XuYo^D!1V*l2E2uqd?C=T8jji?x%TTR$*xQ7t33=B?SO=3fOa$< zZs5jhx>YbL0Y%$m2z(RX9hG=Q2H2#{W{NJ;GF5l{z_PX#68$CiWUldFn>Mw$RdYka zyE-cu-%gs+W%!ExDRaZmhu4VjQ+?3Hx#5#D(sRd*J7L>tT%fu0oD(z7H3D3ZHN4w# ziClbBdExV(*-bQGefDa6%-NW)%skO1UKOUGbeS&~7T!0Tronngo=5;78%ZxpIWa`F`Vv03xrhtzyjeFOza;akYkrItxIpm9ZuBk-r`k51_hCE@6<^n@RZphZd^r5a>TCuK)oM+^L zDXY7T*uyClnSyn2;U(l78$We|(>Yv|PVGDi>I4l=LTC_T2~8VaSy6VoTVZfyWQyF! zM~<_q&ny^^Ekk3t45T;vOxea*_rlL&-IDl6m5T}Y>}66eBv>QzVEbXPglaAtY@>zOK3uS1f06#PEH#clOTR_iMwY*Vle-**GPg)E&%q<-J6g z$9KRecUx__HSEO$3Z!GHZak;dMRP0YjNyIqE(7H%&pUs$XTA{SJ8KcSR%mZzVt2E zwDSUAV~(1@`pqj1@59kiiG@U$gC(FC<%}1h@}Uq9Qw(y(x8= zwDHp_)!7dkmj{@IUHSIiCE$kjetm8KBAI+Q{R_|^Ulo&o@W`*leWOnW5*2QCEBLSb zW{3A0n&mwHBPb}SVW|Z_3$2d~Kj|#m?J37p*^>uxaVE^O#A4FJ9bTwhKx2hU_xFU5rbn`xg8Vnpx6uNQ#$9;x3GU zS~iXMgy(mB@r9UFfU(0orw!(@CF=0k*GqeOFNah*fd)o>(a>fap5HWEoqH8wtYd#o z>Iwt5n6|ZUPq#)L8Xe--*X7KuTQ9%WRPG+(I&}a3?%=?DJ*0Q4szlW}Q_|VW@?cUN z8Be@{x4+ma%AvQPU02~Enr_ms-*h~)1lxz^A#^HbVHm@&lSEACHLG~jOoV3YgGIxO z?LVXEh!mi9EC(|h*YHXvjB0*)QK;ucXJ_XGS^MLd8s>CvVeBm?e;&SnH zv1ow@@9v`?>zcW&w|T1pgQhNj6Z!4R=s2V6Nhu@)>sM>>&+MI}J#Rcezv$xZj($x7 zIeg3&oD%De&tLK%!}`|7Vi10BxAW4tFrUkSu-us38B1W~KDns8}i^+BGZeS4ZYq-BozUmFWms3A&L5V_Z-9mv11`zx~BC};t3ExtKv&T`iJBf{!n=$ zt7PESdstSgmI@?F*qes;6b)tWIHr>Xw_NxU>qUiUcs+{_Vx2erAEfpPi#QD-+FqV1m00*T8p1p8k*Sg;g!zmN}KJcz5YchtK!iDjsgwwFe+>V^aIa zqqeyo*V}`Gf?$-SSFgU`DwsX>06oQVUZ`C1uIG-o=C5b=_!}2B0?VoxB;E~l%to(1 zbnew_GIxi5lE8rwcI>eq(!VytET|+hk1*|-%?6-hCovOC!ikU>5#uE4i1#;u)zYV! z=MwDCB(QMJsVH`2poS9PtMKy5%!T=9`MUt<ofl#8vTmKUlEWaJ6Q9BiuHqjZNQ608xHu%z+F7#=~jlgfj>(JVQOF9bfWq zRDsP;*8xCwn1=)iU^kv!Ugm*tP%JGX(YpQ3gFEo)|3v?d71`!s*I(I3mk7Y>t+neE z3T>e^B|ig>``~A^#;M(H_Ut3%2OPco29~!tIIspfV|^fBp;l9Auz_=5#aGV~8$IBI zv)UvGDp&r3CuS4Dt(7l8L!QahOM@9;zSVx5Z5S%iIH01?g5TGph`9VyDw;NzyXr2~ zH!t5A98m;K#;>1=ejIH^To32ix(9C`uQ6;X1W|Clllz$^pT^D&FaB<+vuRsnC-=x# zFHZki075rZFrVNpjN1h5v5k?2R@GVw=$vxmmJYXe+nhC82g0vH)F*#r3yx|KO#^t%E#(?>xw_! zXWwn^+nclOJM(*r*4>WrhNI^qo!sN?+;@*RYus&wUl_;31_)HSf z(9WW(7_H^k9T;7*`n9kA%$PVo+P2HYYDHNViC}cRBd*PN|tXkd^Hu`? z`pd4DPwyNYyDvNjUpuGgltxo?bLp?Z%`4DQ0*5iJsrP-E#XdYL_?;{(g2LIduG(C&cd2Ek>UfsA zc~#cZn;FWMc;IdCo>pDjx5pl%)EzF5g4Xes8=#X zVJl|}yq#cI@IkRw-R!xH^Pe#QZ+y`jqGke3VfQMa_#ci*>F2m+-ldFPNl17Nbt`l+ zScrazpd!nh&uo~H94M~hA8Vi{eXb)%pkX?LsL1kb4;Uo1?-kX2266IPS@*za zd&0XoX=)0{n#0g4~biXFAm>cr+d{!H+%g$&WqK(oX@2$=Ppif zIITZ3#OC)2|Sja?krjFG8 zgBe?`N#@%9sYxEqX{E(oLM&Yar7%)bq?pE2f3|hkk3Ko*ncg7qcI@%pqToP5Z!-}Ss719AbkE4D`%e_22v^1)PYf zT7B0XIKe9{45hArHLtp2GvheFa;xIHd+yOz2E9W5q)Lvda7oA>begnvM$n53G_oYH z1@)>F{Br0=hyTv3vZbDFOykFRVhCbNev=-{`i;IwBrU;l+!C_*bdnVZhM%^ySbe)) zKxaIOZzG|?G)8I^Jt*pcg9VtM__!2bk18GSbk?BF^;v6|is!@h=}%9yUmWy< z$J4Wc`jJUlSJ6Ii+21C2{q`Zr)7D*Gy29uEXrJ}Ljo(;DrV`cpRt^xEoXy_+5@k3G8jOA4LQl|lTVzt_(# z15rqjARrb}%9YYy55#nB7~Aafu&h>SY0RAfsT=Y{rq$|m2SR|TvoeW{T?lF9y+xye zL@yC^*oJFNFEYIxfZ;*zyg(D(m<`#r{}c6YxR>tiMn$P3^>sSZ!;Hz&Pfc*03813h zeMAV!BDu~6Gfj)%RVU9GZ1ytd(u=w=j4OdTIXU&#I&<=d^QIyn$1e8n9nI{0*bEf^ zlf<4F=HmI(MAJNDZFW=j9wh{=XJPWmy@~l~6b`Q-?q-9#1|>@2UwpHxSjegl4HC;# z$BwW%^&ICb<(C%5aVVF_wjD`!=z8#>TE_9i7$q$$cafE=7orHaziaWTuWr$2*qRBP z?=NWMPaX(`j^T{Humth6@si~5u%e3t!*i+vmO z-@s`@{73(*A8Q}GE${QR($v?Zo?rR4_^WEloy{>XZ+F4#WPLagG>Se)_);pq^Tpk5 z_}hgWWsIbSBxprj_STCsc!>EBOhs%ZmR&AZp`i5`1{z6iG^RGDqA1Iq+V=2 z3qXjUEX*y_Me&J3Th7$kbxZ~;XFOv3ecOAPojQ5)H}z&RwkqhJyg6l;ETHJuw=Y-F z%Uw+j7x0CJx$${$bX8gDKTC|0rDwW28eAPKU+?Y_9=fDt#+QTZiwv#zvgSFL7YN+S zL-=!y;V%fZ_s%bOuWyH$L_O(LR}5?7Ns=6-F_mmfO%#`J>)TXdm^XJd$u&5=F}WB> z0bBUTHw-#9>a=USzvYDP0goSe=WUqqn2G|)w(__2cwdRJ~N7c z{rdGQZ|Sc=)2B^SaSoXH=TAx3S4NPP{FB4VoJ?5?i^NLD7|K4>Ui+9iA;hmrzQ_9H zza57S9jYVy`xZeMXgyW3ZiC-?4~VZ_tCnQvwjsf+8of8y!E_UjKT<-|eL7iVrOhWr z-n@SG*h8cWHUd3({DPGE!&T0qF=|T(@XLaxmWt1EIz-tz;jPvA94(E!Vi%jl%)H%Z z7!e?x$QRq9Em>A~988bhYT-rdWbW%@KQW%Fjja_PWrbp6cmMG@PHkf)73PdFGdjAV z=_Y;p{OOp+>&(&6C93L!A772-JQ1=BWu(kDX+z z(T0I7Hd?iWDI61#>8`~pH&xDV$$o&da7@VJOp80BN&q`Bm*R93>jR186=Ku1c&rz9=*M{W2$;K6GT_kYFd4o;ucb{)2t9*IFv3H!pP-3!ao2wDK;?qjl|6mOn3V~by8j>(i__5yd*0!ga>Lgv`1ek4ylB4j}?#eAh|x2t*hKv ztJOEzkD276h!$9e9*orS4SkRl1BObn36R5%WrLK^jSek|?xPS~ycjo?*x0l;V4Z<3` zRyI@YenWX#@#SDlb-mH4kuIGyBjUUu00?=@c+vRU`DXBrD1~ywvpQ|`1=2Fbf!wOE zRS5_ZFWHqMgi}L4lp~L=Nl~s7S$qu+@lISEYfLk+Ajmw1zNKqUpEH#5@@JOi^^*rF z%?AThj8m__m8Bt4UQweQSMMr3==r@NT1mnf5xiBrj!`dxYntw zi!E`*Mngfi3Lkbil`uZf($ST}D5R4;Jqy?0B)mmHs>;un6+^&;wLiA1ZLgZWw0&Wj z-Ms(5m2VxKnx9-;bS&~!dhw5jnpsmqS*Wre|PNrXi>ibGrX&7BTJCm zlxFHz**ZwZf1}+DIv}}^+2J5r0IBwWP|+5nL2ecK-I}5(8o?kIes%SaZ+Gxvk2#H) zDw)}$5S_tf2W~*Og8>EyLxoi+(JGWl33UMepPgCc9Gd)a%{0ZeCVLYm>1Vs&L3uAT zQu_5%r)*@(3M;PewcEV##{jNpNZl<@GsCIIvf|Zh=B!yJ4=({F8MfkBJGl%VT>bWX zv(}vDBJ#kjBy^^$uIXLcvnOGRhX_2Ah#$%G+$c@%@jMxXrx>QTwx6fhf3p3Qu3pJS z`I)|;oGI#wUc>+=Z;c@edsjw*z7PZwISlP@Is>>s+{=>pSBj!Gv0p({_kW7dsWZ#D z+h6G;>3@%HELF>5)7GsZ^$GSZ`)}uBCqYKcg#mMG8+4X--L)qwf+W7qCGq;hBw>>N zS>!Qeq{l$xHjrY9A&mkn!TC4ZX~uw(i``Lf?NX@L1g`xm7QH~MOYBPIGIMfA*MJE} z_6Ka)oV=~0=;$1 zHCk+nI3BxwZTi10{8t0cfzS2SSD^)YeUlOkJS(lI0Q2(^aHl2rXki-`hL<=rC z3?g-s@+ZDonaLpELXrE^&Df_9KqK-=w(RvzAodPvM^}AM%dM#&>*R3@v*pEbC@CUw zBDH<mlsrttsgBk>|5)l-81jf#kb&V;9#s{vPKyXNu5X=&F1C+ zuqj*~OEz!*Jf(r%EcINd4y86sMPrXZiLDqIP|y-e{NbR1P!Jv!-xp4IMt34rm>Y~3 z=YiR^#3_R1sJl3Vfr(zaK|?;L=Otrg4EV<7=lEaFHLKxeh8@RNv>A+lA(lwbM7uL8 z``*(2uW2n;&FX?eZ7|{l(==(Cko&!ILvAx!oEn2&4Un-}R(_8Ub9om7Fqp(7KE<=9 z(&9k@3W17p+MBQnnGKDND}4N`Dds(UE`}rzi`MdxJOX}F4U5K_Rw5)vCiU-RIQQ?+ z+dQ&NC{cgWZ}a0)O92<)KA%bAdz;1&L6?I?t0bkYMA=+?yGrgEYtVA{TjaH()XobhOSOt ze)De6Yng-0wxy5Q9rVk@Uq)6tZfEoK(a{3^iYebLTBYq;74Ksg@Bc0G$?xx@U%h*D zboDI7%bC3=nBa!fO^hc8H?-{9jINS)!%!OKti7obO`K&$IscpUr|G-j6~e*ZH)Hv# zYhl*fOT!;tY^tSW-Q~51Q&iH4-&)nb+PwR~r3Y=Ha5VjH^ZoHfNwT(xI?Q-1u5eo< zee&<;q3fsm732b`U?g{rH5f^t(Q{8x;zA}3UfZJ(Ai&IA!lw#uYc0*E1FMes4mU3$+ile;r9cSHx zxlE=Et)O$@urA2y?X_*p^y!XCgkK)m(|io*99109_~kP$qjB&k(PNa-!3e-f=Uc3z z6m~f|@We1o+M+JD9p-h9^^p+)o8#E{V&4I2ZDJc?~1R+}`UaOlT0_UQ=nw3$f0pF9RK9+=XMN*FtfT2u4d>>W;)U5i{KW`Mn+6 z2bS0}fx5pDS3fefMA@BncSlpl4)v5xJ>!;3&N1lPLsx_M!tXz?Kf3?y`4&^z%ST>~ zcAX}sZoHw>5LZ_&;IR$b#SP4TDG~001x%=5{N&P28E#+{iHV-34sK+OU1)f+d)v`| z=RnfiTUfYziD7p=MS2eFy7u;rrd+Ji{%c6GHdY@%?rLLJAee2TfKkkti`^I0n^$g6#cibolF;%N3;k+Fj@nSj zb`=+b3j!Ou>T4=>(|H6%Xk2PoSevp`C;G>ihj%ZJ*Tj_lD2PGUFrk*Rtv%y-9V3T6 zo)zoKK%F*xQ#!)+JW1f<5YJWpOni9kXOTLtn_@NV8s9JX@mv+y{3*!HejAkLe_y3n z1nGalyXO6({x(FLKa1=6ERE=ER3nJg@qyqMMYn33IJ~?t`+!<26LIdZA^f3^(X2HKx^8Sj~ocaaQ*onYZf_9+@2Dh zHb*HPKG89};&jx&(LN(DHT`YKjwT|YvUAIAlRB0^?6u1bwXHD!4D1%sWCl5Wd3&FW zMsv?`Jo;1Ko+j9h(FRJ*XjeSuYi#NdrwqM}BzumLay<6k3fQ_Spa6NfAo=QjvXtv+ z``OKT-Qlw>@UmPy#D!#=Y1VzffE%+cO?Zdp-J?f#6HHEQRWrIyj`8TJs7^n5@+4sP z-!tawbwSogoaV=1uBErV>ttA39XiWO>b$hHG>^)W+i$j7R=9=@Wo!%ddL36o%bOWN zBC>L1Q`$I!1$q~{MT<{2(RSD{!GF56xn-Y>qALlE6kW-seOgw%S2h{q@BZyi#z|k0 zgRhKW&K-U&z0ENF?(26SJ1$+`7_S@eSpp2=gm3gg&S`c>KA${`1dzY>&tT$Cq-*lA zBc)`2XGKvREQo2>#r^9D70KQoHVq6FMYdQ7Xzy(-J9HREtAEMH*Jyr+MoNGQ;z#!{ z_uCCjwnPrRh$dtTk^RRdZp456m)|AuEzH-b$MATI;-j3$1uYuYEd}03S#~NQqp1K7 z*Oq5>R;!1~SFKs&NUtdd_C(ouPQTEDRUNkXKcF-ak?}^q?%yW?G1D#fARIZ}kp_Jd zH#arH^(fO^soif;}vGYSlB~+zI8iN&G_N+POC&# zy^7^LlHG3A&BxGe5M0CE!ZeVbE!owqEMul>({T3@m#>sBNzPnc+o9FF1@`oN15HFFe)9ufspHQMyj^R zw}Vny4n&l?uc6tkaV-^n5|@PPwb?oQagHrxuFXf>0&LAq(k^`1bik4U$w&SCo?K+X zmA9-?U{VbXpL3_yox7Jz8-6IMSb1=dkPvOOy$aVL9g~fXIljgg)9{&aujQC9oY+{v zB5Op9)>m>odHrEt^ddclSP>xc-mVr0_6*}ausMc=0($i&%3CW-9KErt)(Gl7`dosCyT#+EC`RyevkZKg^8~dv-t3?sjN}D%G@He z?T)PKPLx@}pm~sDC~VP!>w`r1lOkA;j4ZuT21#$jkxJ&nYne9kx^au!$@A*ErcRMh z&u#5q|BCCpzqsQ+4wXx)`d?0RGGa3ma=+$3dtfkg(WkJcHMY5)|NU}H#3J1zO=#OFl5xSt(oj;8fRfxn*OJ(ZM{ zu|Q6Av}ner53k<5fpmEd8#=Ne#UdJFnhx-f8oNELz+=avpy04o6~7{@&VO6-HR?6( zJApaKLbeR}HtK#V!?LkbO2g(mmf(^0eGd=$Vm*veV~cL1zuv!p?>4&3^UL z9y@bp(^Y!Sfw;yQ4gQPwi=|_15&Nd%{gss-X1~_$!=;0~Wwwg;A}$(Y#+@lQ7r4?! zq=&rAm-p$Ey!WUmUVcN#JP3CCoVVCV5-b4VA7j?#KhMkEMIEdZ&ar9g#6Z zmG#d*Di+o(MuyM18sKlzKAFdI&7d~r5ZR}V*pSJRo;uvSfcuZeOwCn?Nu0yUgLdNv z4Z0Ppt*NMjMP}^YOuunYk?Ds6y9^r7FE6GN$h80`u2DBUyp8^;^+SnUE}F_l`B#QX zuCJ^Zu?^;lEwiK_O}Vx_kO|Y*gO6^v_T8_(P5ZqV*KHj$f~c@5H)(5puaJNND$&v5 z*lin+-aFMRi7&R%qC|(j&a&L*>T1A1`Y45le;Nr z){lQEoK`W8IjY>p+efC&Q?$lRc+vY#XAisKMkO_zFJHb~^q^B{;WGZPQn#vdy%Z>v zX>UnvH(~Bv;As*vbrq`&@t`dxX${tozG%}PU}ml8{xP`J=GN^tf-ceK^{ZElIuO$3 zA6vt=+Roteo0LL&?Hq1^Nv~h`@250hU!&_1BiD5OGEON5Okcaqx?_|#ClMkCQHJj~ zcU5&b`0!U%<@hw~8l_Lj5%TE^rV8S7MZ^$w=(HEj7_yo{3HM$>Q0y5x@W4@Qm#kmi z>LqvRQ}))?Rn17?J_#XU#5Oo#7GJC6HK~}8J7WY_outK(&4xB6gpf-0DazKCqC_4> z;ABA1b^xR00mwhlmi2JMus@Cg@#Nr=12n$eH{>1r1vp8urULETwac{dCe1bPl&E-i z$7gphgZdk@OM+Qz3iKd6p-8K9@puLi1J};G%bWt42MS z9P$IWig@kwf=*edjr9M~MF-TUgqt1bXx%F}QtX$3D3!l6@l@BZ7p1#TV7#E(jtY(b zb{IsbOKa(pUIW&M*c#KGr|g|@d!3z{zZ_#jB#@13czvDmprcyp``<5(UVCwv-mx3a zGK64qRjTS(^Gg$ZuF1`MV6b-mj*snQ1F}XnI`i^Q{MoXS)qPxgm6m^{AHBk5569%Y zRa$hfGFe}To-&A}u&)uV`q1K#)E@wxw{){=b!GjZUTH`BQ#gg$g@Iz{pziO|}Z{Y_~; zs7ap{e@|)m+a97(oBfjgH?S`%wIZO#;?mUKZQI(Ql#@uP@9w%kwvqY`-^m(>xHxf{ z4RjpIvx+?rTrVCdaW5v0hIQl=o3D5~l)i{vbjj#7Hm3hxF^TgF`*Z?`t38R;RpZ;& zb@RdUiL!X)Ud;6SocRjrYy0q23$I*7)dO(gTiVwWRq-D-7Tvr47SZ8cD5t}#BC&FB zSI)F)d%pepYb;V+E{~)I|FHbd-A)TC#(z21uYJbVXOLCt*01^Ibog3GX$QX>>a*lgcuCK}$;Q$FA5G{xtKlY7g-}LtLRPh?kb$29x&fFYMFW z-5IH?One`!YqWgCRq5mS3%?FH^;O`{x(_4|_s198K-oRQut+&x->$Gp3&zCZb(A-( zO`dRi^pS$qERYz76y?G`o$`-m@f(L_q>Yw!CmfrEcH*%Y;Hij?eX3Yo2DKbd6ib4a z7_s}!=K9x`jciEL5~oMGn%!>PMRLypRFHw!zv2XcoYq*&OJ^L^2klDZX!!gUCJmJe zf0;$O&Bn?<7RLRR7m@9kfyqavz1ZR6+OQQV5U5mJhWW}P$c;3k(N=z1f_ zpQH7J4((&fE~mL>QBXTL zULC9qZa<8V#}++x^h~_b+yw>}+Fl+k`U?hgR3&XNNvdX;CjEL$Wm$}cgjtGMrOS+K z33JXQh^aE^*s-+j)x7-w3eTFKX?E38@GY!-7De2_#A7|W)j_#63L6Gfa3cGL8MiFw z(UFiuGzmZ5Xgu40)Ht@m6amT7B)Wg^#j^+L9_4}$TuNHOzUteW;A5eG?Jn@AUTLwG z5*sf40{a6DqiJQ;(|q;y(#?Wy*zC+OS3J(}|IFELa>LXO*dL)>)9pQnTtI^nh*;<1 zzUoRsLtY*g_lI^7+YOgZ3;~RPU*tya+mnEWZDNXO z;T=&WF>@Ma&a)~JJ`=uXqsPi~cG1&#TQ95I?`Xb+#Ocn_CuzHk%BdFRd6Q#|%nQC& zfiL;U!@9jNFS)@1#q>TA$k;ejh`VvnWOZ1(4S}ycb4}SKfM-lkL5Q1H6zFeL zm3#JxQ)JfXyo+Y{|5$x}$q;}|tLn3_{vwB|!_N57E|aHEc%9 zGotrFgLT}4WmQdhtNwN-|Zn5rD zmnK6brIfz2wEVJ${e@ahCfNZxc3@KpjsS$JrBk_#Yx}Z# z+eo&PzJ+Sn$q6^PRDDQExi1$PKgR2Z3#*yM@sz`6puvfWoliR2QnbaamGx@bu;HlX zWqM!m$WVaH@;8!7u{>)_BHWX)^?nUqHvt_4lO)$GzzJcXbm21W*S_yn_>4`5vP*@l z+Qt)zOK3A%GcL!GU?mk5#W82eRLIe9)NOD~>GM!?()PKvWO^(TL;365rIP?QjMu)c zhfDGP$JP4cUo(9vvDpBsg7-&WU+3S0R9jf8+d{y+Q9;-`>(n=cK z8M6EsKT~n{4nM-#21%3UD!WFFEIp#Ypi`dpgQJqz2rz1o@()Nte4AO?p&R;;3zX~tt5{wdp_m@R zvufnRZwYWRMb+49C$|I{SrMl{RYi0PeY!n(&}bUWf5*M3>c%E`BE8b{#fl9s$^xnT zes(Mnq#bAtF=^k2$BLZ6&?WQxfI%YX0fbH+k0a#4@+LRdd6_`PGHi#kYFiQ6+|=XA2; z5p3F667D6(OJI?9FBREFMeB+8EhT?7tqgaQP%Q%;8Bi2eII0?vdWpqt;lhQXWrK$d zkwgek!4}CEH|PK%G*?XwU+tK6vTd|!i;4Hm&XIktcxlI(ujRk0-`#eOK62hXIQXG6 z_gAwwqaKg>@YmJ60bhcThMj6wssF^d38%9$3{TU2v7&x(CvdzSMc6j_%?gUAnG zWn~cjI6gj^yqa}+P-V2Wb#j)iPz*=V>`25=XVVERui^fmI-*ANeLkRa1Shb@x*)gd zfb~msjfQ^?1!i+#`;6vc zO~rU{=AhGR9wzg&PLpr1m;nHnv;vgbC;s~DR#5$4eo?p3dM#&W{&Nt^B#wMxck1it ztVB*A>RC0tAsFSDHQ0EvN-wsis&&apGrn;0;>E-u?c4u+SBz(Hpyd{_gWNG*_rc8e zchfOonLg~lc{5{fH7`>PuoluL4~(D5YoB{D&bV>m!cdQ)nF(h}IpS?gA7w~Ud`Up! zM!!^^|DOohI=3;3V&H?372RHd)7-m0iCkQbX4FvX-TgTrCg=C~p8?Tcd?I3aBloC- z!*sS)iBf`_dTUaV_0PBC^=_;K7VLRnalO$K6!3LBf|C@Xp*<6P|L7t!?yKl;DUPz1 zWZx2dVua-6&`*&mJ2|v$1c{)uFB5ER?kch;+56bfRZg1Pb%OB2CA-tf$xXs@?$oBu ze6RY|az;}jZ^a#}V(eaeEHwo zZ$8*;yWjZY-SKbDUzgu>fA;jLia^C~X(D;ZnJZrdP=(KL?F1{_YV0-?sGz*`6_S49 z`5i~IOCp5tOBT4RudOKr7lA_#;zj?IOG~{t?b>pK@ftck5FLrDY&0lCFtd}i5C-En zABfGkkMLpKFN%)IhPa0nH$EjC0k9R?f!^G1wCso?Y^!NuXQj}#MxDO*rS+S zo;|XT;Dyk9Pr1yyoT^s1vU=BU#||EhtM)8M?Ss4i`b$~N^ktl#-ER_OO~ia7c*A(~ zKI%ip=%#Mt8`iHM2-!7dc=hVFXAHG`+Au-8FjF{qlN%+sauA@1@Bfb-3;y&UN-_Qq zOyv;_fFXXIf!0PW98e{VAH6wg+3YIMQR8T2>&r&#DRt{HEvRoVJ~W-uf)0t9;GR;T z@>sVwFjf9)+RAu|=p6)W<0)lHViD9(T946+_Y2Vg)8{oV2wL9>_=xZOIr@$+tu=4a z^8Sfvz(IutMZ&&26%6E{1grtCkmLbS-#IGWq^cCh&<<_d+!=SNBD?nv5r-2~5JWO> zvip`gP0!Hd^71>&r08|?3FtAk1NHr^1m~;HtZVzpC8O1A{L)J)ZMxEWL*cWecgd*- zjjk@eRo8dK+X!0@%a0XNzX88^X|D*0ux)J8Yx!Oc9qaSg2H)wte$x3jb3K+>BK4u) ztD}~Y*)L;*WKR(9`sP7rEy?Stp{KCbx}Z3|hS$x`Old)yTMuhm0ecb{xmd+0^)!Fb zeD8^wgU0>Ofngo7MttE%+XoqI0N??M_3@&Ww4r9wv@cOri!Z7-4K603TFD0Mff}36 zGa2BmHj?#P{brlCzxH~mUZZY*=dctirKaYl2t?xn+4=%uTFp?bLdCs9f>h&ZUNG(Q zo|;Q)_3t&5cNM&9N)D+If>*6-rV2jor#^uvZ&@KhA;+|*n{Iv$ufnHnTqYfJI>WDx zC+`9JOQ(A?v!k#^)0_F)0q!4TR#sgk^NzpuB!}dK&Tnt*Y&8?QT2$uJ1Unre0u1Qt z^dZA7^AxKS5sa)X$#7CZtX;+1BQ?$#T#s2Y7s+#O(XmR>&+9ml>OQbNw*aliM?@5PFy6Yb&=am;w-`ui~tx?g&vC4OV!#ZFz zk)C^&g?qC3*>CeE%FYFGr@5D2ktkpF%7oI8Zy`*Ac^MTSGgK(kJ9X*eWN%$9XN_D+ zldmTy-nAVW`66w8KT#{+GIX9y!OU#fZ*oDR_T|7T z4Ch}y7_ufi1VPyQ+w5o(jB}}RG#7|=mpZAeV|iV2kS`>J z$z0pEiweirs8fy(=L!np_|+KxvW{AUJJxSI;=&;vZ~EyoW3}rvt5Nu@iG^EI5J0sk z#sxb%evY6K#y4{})Ge}U?Brmpf@<-4Q+E9ttN$JIy==Y>l(lfwx5wj%lfur0^xP<# zreB7COIc`=j?aO@*r=3Jb_z5HgXmSQ!y@cVi-NF}R#u#Hj1 zjtZ<)98)<{Qaw|?y_~yKozHQ7>Qh6drL3ZA=fA#XXtJY9P1D`II}Ms{bsFk@&$85v zjegtrfy}jz{_>y`CcODW?fO+8G!=2YH1sH80xOA_Tj@!U)O-^{h@1k?%IiQjJ{}Zf zkD{})h>v{5Ekjh%6*i!%1Mt--=4pa4V#;0@s2CqRb?ms9TE$&M+bJiZ*L;v!uLe4vuRU_T&pz%ux#0WfW1Jc-DC*V=swcEmtiG&)3EwjptmH7g|vf!=La@@2b z_P0g!*@#x|)m_wF7csA45tX6Qk_{eAc*L5V@r$JWu`_Hw0#yS%&x0JSPum@@@ioZK z*^rR;*B435mP7qPqcS5jk`E|pqxo4}*uHjLpvZ{lx2vm_Rvg-8*wYKMZ?{~nQ(GBq z#HAs#71^uZO3~N5z1u%|&=;eHG&K|m2NeT8$^|80C%>$$^Ui^60=k(x+`NQ+$#U1! zj2vLZtCH=vZWJL)7whXP2?e4iRbQEO#j-pIKGM(?j*#C+LT#e#oD;s0R+LLb>Lk>D z^x&@fEVPgiBYM|BOX5n@@&&=As;doEZuubrMT;3aLMGd-{S8^tfjvbc8U;vMB)g_! zCdbHhDbE`jp&CW7H-!n3Y$UER(IS<7A|IF>&uc?!S8{O_q3+SgVg73 zZfJMmsQOU#Z)M|?06gYGFrHsJ(J1C@Nc5PwYdx(D86m_b%p4v1*Iyg8w`m0U&ttZ` z3MB!Xp+?$%XAuF&X^vnwW#xNrG;+J(S?OU>QId6F@2iH*n*F#C6_-`>{!M)>rO?p6 zDesqGzHng-zhZ7MVShrTVD*{yu)1RT8?($2m!^n|IRH&Q5prYW9BB?zq$4076SmA zHvRD(qlnWQ0ljE5)JVD=Ca(jXFy+et1K)QrDQyXTXm_Ybm+qfMKEuK@Yfk zWBIlMOUKT`F#6(#J9B3+MNp692%6U@8qL(oa#uxsV(4$X`(59Af;7fx)l*wQqA-@} zCS>qft3Xd!B8xBgS{?XBBR1C2**O`+>I~;Lz)2)3A1=l*dFqpZH+87?rd}KwiAtpE znU#vnf9_6;1?pL^+Dw-J6-0dnMxH?y9lG zLmvA&5C|cF-%$jxe9DY9{(NNVXPNMSC~YNH4$sygZiB_*!5!)`}7o*GfJG`xla5FjcPv%#-Fu69(b<20giSJuh+ve7R_%CQ)<~;~zhM4mz2V zUpni}IkZ9mRWE3h67vu5;#5r|+Sh|RDeCW|sGPktR-+(Xz@m>* ziLpg}CfZduO*quJ*5I&@$uWPbty(`O^mk&w^78-i_v<Ii@0sP?W*vkqxNHSlqTz$!`qY!7(y|^7Ha0OrD$^>R%IqfK)030@LVz zk_3w5!&=B41EtrwZ7sd72Rk|J-WYp_yY6PSo%?%gC;m-PkmPX5jI|?+;YYEQ-k+Dt z#-y-Gv*VqXl`PC$HbyyqS_oc-u(QCpwu?*;k--BkK9Jc0 zdb@G7;AtWr`$eO;p{rfpX3d%{ubr>yx=hrw+?+n??EQhAS!MgXy){=B{aE6|$B(bV zjTG)h$U|L20sSZ^?#><2C+X#WO+suCaC6Fr0$`T9I}poxrysljaSjsnfBFDOzX;mwts8VievR-12VsUmlba^x-E8c(n}e@g;)O7vhuPB z9$GZ5+lb03i;>m*4kniA?%DEIW<*E|6-^n}UZZjZ# zAN^_k7_=gl#h%HncJ}{eY~^*rH4|2|?3qxiVV>prv&5}wS1jZ)3W&38Xr9TXjEY$E z=r6HW*!oO+ukY7ON=r$HDrJe4YwwA1Bem>JU#$%s`RU>?wy#DEDEn~P&jW*={EmF( zQ$`3wGWe8utSCn6Vlp*q4jO2Yla_w$PK6BNG0t7?1DR^@yQY`N9!L^6qi;!cBnHnE zo%uo%fIIUqzIfIT(oXPqP_}6!7R&-{r2MSQi@iBO@v`3#H$2t5J6k_6Mf~m;S+!wS zR=*+bUFJz`u5EN}H@@EvdLHLvfBP&(dIHaZtzOowo~hku$J`m#`%f^5ai(6fP@|Cx zuU~&-{n~z-mN&z1=iP7yS2F6pz~|&(03pPZSKZ522OdOJ#5e}!_TvJomp}rMWFEUG zDgD@s7O8lKlHW9Sc=hR0i27Ps99vgUsr!%cSTqPm(K$H|5=sE4b!A`PX_QDk^7u>+ z3}~xwf$naFmsuKB(0Lf-rS7=Ah43NBQ+uV)E}cXPRI61+(nav=J!5m1V3LtPu}1q$ z8Y4+plU*_?2hgP~^jm8q8q!_5+nUhHgP5HCmrd3xPmwcq{2zz|-om{?Ecs*E5@rt4 z;qEqT)V)N3>G9Oh&=xEWU#C3N zxbQoQ+{{Z~ZH}aMUG$$aw0uTW(4g3z9nXGsD05lbP4RyGE#K*n2Y$a?@p?h{hqu93 zJ{1e=QBljbxd`L+yA3jjw7}Yu;MKn<9ox6fB%ZQVmQ0x`btt-Nej?JW8>=BYV*a0}!<w&?)%1P=?hb3s7(wc$22?<{ZP zUYdJ);O*1Nw#xJu=4Nv zqnAA)?741BVfC1{YRqhg6)b@t;Ban(!4rfQf*a<+$mqJ+%qU&Z#Sg?ce^$l2S?0o) z)YM?R6BF49Bqe#4m0vBr{q)KwtypnM!YMK;zJ-(j-!^`GHE$idlbeg^aQDBS8|}ig z{$YB)F0DzX_OY|YIS=zgEN`^(+IHom&5>Gv_?wFaUzYdq4nnfcgNnWgSn9TzZi;TL zD1V_=9hWGP_GJ!1WJG>%H`z_IBSQHEBKezeMv47+GiCp4hp`UZcJA!8c;{B__ZpdT z)r~u8?mVga82XLO;v8ziD0<}8)ih#vA`e4~{dL`@p1|u6y@3sTw{BpSgUO&*9)r2M zuQ1Z>`Ty8D^MD@n_3z)ULKHK|(qi9b&(>ngSW|XVVa86DBJC;53}qdRtq57m8lnX= zvRAgoRvFt@QQ;dY)$_XU%=w+)^E~IDbIy!Z-}`f4pX+*W7wseNtWLg(?YmaB`Ws|v zZ7;SDXePidv*<(?#zfm7e3MLXmnJ!>2D^XY=2GuRkXE%%`O+iO=bx%b*paGEYLr?v z;mi8(>{seeNCtQR4+BS5$fzQa#{l~@UNG4-qs8UXo@*$ofWg13rgtp)J}%oY?GYM1 z>Ff0LPrWv3uQgq#0kfNNu#RG^OVu4COunCH&#KKkcFbvMsu%~0n4_Y+RMyv!m>(`Z ztHj=j77{VZq#;HFR>CDn3Vh!;rChKas#BHv*e`hH-6WC~c~nLu9kj@grpVN6z8MTr%kK)NTv2 zT%rLc4%017q?3@AMT#ol+cA!{VpPjKg!Z>@kKww^K=`aVo2CcOXKOnd`?RC=f@?h$ z|7Z*t^32HdZ=yhTbP*K)Js{%UZSqKmCbAy25^=(>`s6zM+piPqvTr` zKD8BPN9}JeA^jcnR)y&rnTLEi2tL%V`6Z9v(I^T}x@0+21aOvSXCUEH?XFf+j83x7 z4A`(?A}>?J%-=#r{@^oWeWm<0{S+TkRB08k13X2iHyd?-#(LA42Q9An=~#7@Wd}mL zj|`GT0x;{8DkTOoar?@x#jUPFd$lkoK{?k?Wagc9Yg?x+wFf4lz++M7w{Kln9Cm)<{iaOXN6);bh0JwsR=ma|J59x8t7Z;X#CI`5|r?eQcbEZ~zmA$S~vhvCMhcxqRWd#M;{pG%Z z2jH|bwAdJI4z4r*ZvkVK)P>|;u64^faet*8LXsxr5Z;1vkN3||1D*tu{bP>ZxzB5IXCqsrfQ#- z-@)OZWNxAGPtmS4v5X~d@Tp6azsCx5KyTGI^AAP&EaNRPDJBh$iit*c)wmpvj3GP# z(5|a5NA7zs(x_)`?Zw4pVD9g_p+K?(3M$Gy*^$+-jF%y?B=-ms@&p-&LSFWW5Hd=r z)MV_2uoeRBm9cE%E#u5xmdHNJD{UrsmfQ}w25JBc^ zaDcr=1t2dG$w*#Bd3MDShMN_CP7F~DEf)9J@xNK{)1=sLc}x5X3sh6t#e&y-^C=lbEQ!r2&;ZFdTL z$3a0uVq;?iOZ%WF({&$!pySMDeTQ@70b$uDwl<4~>0MY=osf#j+Iaq&oZbT1Rllrn zF?1ebyuDQ7&?h}XU^9Bh5LU9Neqf{6Ls)H?X@TmVx?Ym)1kr)C`N%*$=mgRKx8yiHN%T$y5YJ^$=^?>ont z1~7T=W(9nEwQTzFKBHzoppHZU)rgG^Apbm~DlL(TqI7ffVcj4vh*ihboleLF3xJbVRFct5uo(65X$pCXJ7n$uw4!tK}4P#FMnY_`_I+2wh$XgMn-9McF(gvEHds}Il1biup7JIT_lzGdi&-_j|W?3mF?m&tmT+&1DS z1ITmx@&|D&D&B&^;P`xXOMezyzy1OdhAIjbtUsAU7CvXFnHZ z50Mf|e$2DHnfHEgf$wmhtUGcrT3Ausf%9P2u8&=ER`T6_2Rd|UqN~~UL*&tIU!4YL zoY=UJ9_c;$;za^PTYZ#0OQ?e!3usZ0rYuSJcViY74SZIa+fiMm&o&UXXx24WpLqWV zMVBY(#PM;A+qdsemj)L^0{!^K2Z*Wf=f-&dueh4fd9OvCGb{bLlX4s``Rc(zYt0&; z-7%=}igQlKbfkVvDcS>E3c%MLecNekHe@PWD_p>knpTY@Y#!c8gMPWECc74HODO=n z!Yb%b5t`PzR^dH^-6jA%`p?c*>3ELLq^TyUFO-*n`}+NUI>#Xq_Qoy8wyF|rOI-qO zKf65Q0gVP0_~GMgJY<&55WGe_VO+e5Hk@20*yCFo`s&Gm-p4n$>Wn&=f26r-ooxKL zyG%J(?7!*tk5xQO5!eXLlz!p8sBQ!+8Lgy$fu9uzS}sceG59`D+~#N?)40b)b;EN? zlnDdpOgtkWOK*GMmJU6C4{vW+Fr8G9Z=46Fexatu+xGv^o&!yut$q22bz=SZw{uUo zLj_tw?94HA9BnAhY8$hS4@)euL7Xh@qZyHh@F!)K!7LvZM)Mv&$L$MW6`I|gLEXFq z7Vj)9?jhlTISs$d>bv@sMZ~+#%h(l>Qzb^pDSdjV{%?O=nD}f^{E)z8!Cu$_Gz)|@iI+$GHS6?x0SJVdV>0(~TUWS&tK>d=Yb8_!-NZY6T@T5raZW|r0PZ1FKy}xPhY}Mf zGvM-?oj{W_h2({C=+FK~lY?_AKPvA&OjET(dG}MwWP`3Zm(%zT7fqr(cQTVmmVAa} z5n}M1-?~=6ftZM#WA^P+C58yTzOWG~51_SX$-GE1-&o6Q_bzKItyZ^O=z;tg9p!Qs z2@$~F&!iyIW8R;-_u7jhUy(oLF_!dwGX2iZLoIjvG{8Ek(A$c9O^a!8 zDps3eD^r}jW3I#?Le*XlYy8`iRoSYqS=&Be7G66r-lP_dMFCb(d`GdmzN?dC#=YbT z>Ef^E{XuQHWt5zu4bAbJiZwjrg4f%-0q()RBf$q9-+%BCkl8xY0LT|xb9&*gDO27k z={tmPLp!Q@&kvP8c}F;tc5X_@N<|r!v#@hvkJ(RvMMV(6EDc78l3~V-g{unXz)4(0 zV#;4dI)B6L&%X~hGMc-3e^;|x1lflAhJ+5)@&8nQN#^S;9FAQ@$|>xShEN2l!Wj~o zMJar*uN!zkh2*gGY}5-5KszV-bm7v<3p>QrF!YdV8>IZO-Y3b@JE7zR-R>H#7foem(=phR+KX1=_(q3&Is4L6yjfbS|NsjjnX zCyP(79b6(lGd|p^kK++sKqIbtcB+ajTVP?IxPy*mo>}&6{#kq-aoJ`0OXYAqoqx3p zTVEKg;w~R>w{y@9xsgcPCqsX z3@F&+sSdMt@Q9~H8n1wu_h!oFVWaQ-cs_Ob)@BeG?PvZZ<5h4K9HD(sjyFima?8)H zF0)1TnT2dMFBzn|1uLZxI|b(_a>Wsi`Vj8cQ=6|v2v_~DdqVTBYor0d%-PNRmf~?> zXP09EZyJ8AlbGa^c=@=%XCS3{TgwjUrz?m`w5XuacoslF++}bMckA0Xgah-A{X?=5 zQVX#4K1`?I-@HLynBSsp7*$nW#5Eu?78f{Lw`j34q>EA8kT<&g*v#d}F^=bkn%Eg7 z6tL@IwrXW<6ILyC;qv@$ge{H-@GB;!{((r%W2e-_#TR$+&pc zY|p>v!mIxl9)1Q1dx@z1%fAFwjKg(e@^}kQk9uxCeVuHJeoXBXW$9n^U|fRN+l?6w zX}r%6{Bwk=8xtfI8{ zpC7>H1Jh-Ul(IrVBe{jJAAZkfU;=%7d`hUuDh!VK>5KTiaOlpY%tJqTA5E<(!}-7` z{FkC!XtU$O>AdMXvjywz07}n`UoKujaGuJW?9b5p3v*@(&hE6rqQ(H?%*XV>&{FhFrjhKnY^lkhzzM zq{jB_xgpl&RcZ$7D%<*4eEMt8NLPnnV8#S7Y?Qj88OTZ=DrgYRlMRhiqL!k15wKIj zKwH-+=_h%DawAr$S;N31l>sT#aD-T4rZzENLSJ-seH!J&Z%VMKH!&B2^gm#-5#TLs zFQ!6SN$Kdx$!^g3=hgcMq@<~Kc#ad@`|(-G$@^9xn6|A`f7O3zv}Yd^3Gkq!b*1A%uALo9)0>a~6pTD8$pI`?Cc9;m_;2TRF+r%w+T&Po*lj|@AH zTDAN9rw8M^FP?m!!j@S4&l z&}KwhpHIeVN0qXd$nE;3)oDqONlj3(Y|%S$QJ`I*jhLAp&r2pE;<_$nojIzR;S|coXW6iU?cyQAoBRt-lmh-r3=gRVKPZV)=eG*3aw5-hN9TW{ zH}9xDN0nl=TUdFH{2CzVI(Kql#Ct2{@|K(L$KKlcyURbO;^ZakCBCWJ+mHv}6WdF7 zUy7n_{|luqy!+0$@D9bt$FsRcF}y(5P&6Sg@3+3_6@2XnCRC&iW{C)eHESbV&De5iHBaI{j2Il z4Pqv9e)IaZ;jRijz19nICa@a=I&w8xRU;XB#a*!b>3p{_28w_5|0BgpvHroQL&}1m zJqpi5y3@T)R|7KWDwZ4UZ9Mq!(W6~+kw6K|%!D3{Dwx|#%}j};8Q>qcqOEi}S>+zP z%}`g@m#9D0vDK94wJE!x;1HV9#&)t6yG?(E#Ke8CeP;{++DDV$*UfJZ}EJk?_O=vuQUoLv92kMw+ZM|+rZykKb za_@=kP;M)t7ly=@8dGOQZOS%=Sctm7NJUeJ>#9$yL3dL#C*Dtgd&*p7!dnJZo}b>m z0s;ek&P=edJX>GsV>WK?#i_6D_T=kXN>mU$<)=4xS?N`?91_4Fj6Dt5&JY<8D^Sd$DH812h_NrbU{sf-dS2;C3`t&ATTMC z3|C%OD%*Rs>WaKRVRg8-?il!tt7hgZj~E7_l6J)H%Wt;DrSW|f2lc(@&(DwY39PR4 zLsIk@3`^i+S!E%TzgR1!)$J`0ZRFf zMqkYbMUIWv>`tA5P~GUcMd#5Or^cf8n8m~&|AIfM)SAJR+c&fjJ*@fZpPq)op7-r< ziM+5xOXrqt+^(AI>GlgWJ(0r^(zK=BgsEDh`i_yyG4xl%v7T$(eEV2bCnuqgzD3ji z-XC&^-jiI!#FXdkic5!a3-6_p3fkY{jK$~=e`#XS@i@-z(7;oMSx0mYC*}k^;*#p! zH`r8mEwQgX+J;GYePR~BiI0%PDk8`6qyo=iAisI~%o){YoU+(J*+5Gw9xLO7`Ld)m zzfOv&&$^Jleu_sLhdcDnASon)WhKWg`n?08?=l*l-Sh436sx0xP59XK!fAxb{p7Y* z-+0>e9kR;4x^IC$L!CMU1AmVnQbjrb9{ruvHD9F05BcBp>cLfBeksKeZy|A!ybskL zyMgsGPuU)gtFF68R^>^(U{qgQPW+21Z9T+?o>M8iIS3G|VMen>fgtz$#=JK~ajU2d zZH_D-Mtsm$P0*;&I#mdWJP~(ST^G@P?md5g0$}a~RV{%|Au(TqkA=^acA62d_1PU6 z=}P`Oz?+hZyHaDE+D>{|@Zyhwe1D?=*A*N}voMKk!3U2}Zvgpr3xB&2^pk8Q<8PTK zAhyzEh(M0&Lw}aAUnCA+XW(*_OoST!zaVUcM2hO$Jfwe`PR=U0nf_N+*61R)T4K^j z%UR&o!hvaZtMt=RlJ1d>a379g%f@fVISWE5YS2X*wBKkVyzkDa#YNnDLuaLL)m;^8 zs2&p5=!E++SQz`S@}`+_yq)Nh(;JNzik)$)%8uq;RoPAF*0^dr6ws3k`vz*YVFus) zL*D3+KZUdu9>8n?IKc1l(;qI4i%|I01`p^v`WG#&u)q!zhJ=koI%?Kc+quw!olG7Q zkgrlmbo1bD;-20$GTHrnZCpV6C)`|l9AvYY0#KM!f-PG`>-9uXeay)deC>bZPpbF{jM(ZIGisY}iJ-gBe zRg&BpY(5dSyGc-SD2k6qdA)piGQKZ24APcnSGa+Z*O+KoNAbNvYGc{+A&bw}d|ESU zi`21eS06wf1owSCaq-rg50?BLp<~r=;H%pklFPS{Msh$jIG{4$` zINkstv4CDF%}Fy+MU}<*C4C1DH0b{q2MGk{FkM|t%|^a875hI?#yvXu%j1j77CVWM zUIx_h?FH1VFplilmAm;Bw$d)8NAUI5%9I{+`rx z>!9N5jcQIDHL59AiM!bw9QG}0G)OoIY1v&GZkQ)T0V5OqmckN z6mtUcHhzO8TOC$**asz5`rQN1bDv2|{r=vYt{5Xw1DTUgppldqYJam-9W!-K@6^fp zd(wfS*c5UlWj2@f3-@X*%oZ-@AbugAE!7m$hY~b)pW96^^Y>xDDzq_NCe?Zk`h6e$ z-%LIc*EBx+Ta_Y--2CpVcCb0y5pOdAV=2~>jwc!a5)1u!3*gX<$bjye`W{Mvr?m2E zt3}RAFY^17_wjOwxT6Wgy149=yk~)wcW$V9AJd!wrBuEPAC})U3*^ZnL}N%`tTV5a z&Ai1xNo&tVo)P6YH=BR)fEh)i8XmUMEj-oH=0`j@qo3?KalKOmo3J@8Ik2GyPP2LM zjGG)ICh^$p6R6Gg7oT_i?T<74mhO16_BkK2f-y}l8FZTxiXq!m>0EfJxm3Vh8r4Qq zqq}N5Gm7{VT(M@40X*&K3#Rrvi>@}$%+c=<{OHNlMdUfkb@X+`YY-R1fU^cQAL#^# z)b`nJQLkQ%JkcrXF^pKzSb=vx2J{~{ZqhJ)V>htAzIxdN#gUVebnnU1z{QI{;r8u! zOI&UdPyQ9?%?4SC_spKJWnq}C3LW}ua233cGw-`L)HhtkO(yx|js;J2%#G&^MCGcw z&8KL3E5^r>_2p3?s^!M9O$?ZzZtbG>h;|vT%o16_n8fmiDS1+5XsT7vkR00f&8vB$ zUMh*K_-ZNiL;#BFnBBfZ&{ln$&uPf#F6x{81A8Fue!rF64|4HX252|Xqtw>M?jt*w zsi8;=Owx7t^C;}Wx@+t}Q?>m4Y_ayuKg7^8{vSZX^3np?im4eiXpoxyBR;R6?PI@7 z^CHW&EyYls86b`;&x+5UQZs&Rcr$n&vM+mH;L}J(%AT79rB`B*=BJ&wK_u0pqBN)? zjs~i~Z+m-X&nj*u!^;m3*FCdjPUAnX3qJApgPi5lqNgdc5v== z%Hjc>qfm*xWmq?Lmb#6KMFI;gzLo`)9(T%F^D~!;2(|C=id^adH+wLT;Vz)6fs!x0 z2Ko|>2%3oIjhB;>wEoQpALygt^m;yPjb9Bc7mw3|C?0Z^_Km73dVAu@QGw>J|FBkL zE;7-_N7p-7Y!@6->s-zhqEkfZ*t$mHuHxe2sK44s&;@YKPdN@DB4vgqU@oJp)S#Jkc!{yIa+h3?7&=uvgRcC*cgC^{9j;lc_gS71Hi4^N)IjCnVLGq392CQ= z&mY$ubEfmtx+y6OOf1`aejbgRdMKUa33E|!t~TTL=Os~dWMspmtoen<%a{?_r81R( z{H-A+ubam*fEK--!QX7d=!99XXGOu3;YbI$+(~VFyr{GCycwQn)QHhO7Mj)fF|N@~GL^@<2R!)H`#diWB!EP&v~Q``@o(&w4ZML9 z_BYd*kkoZGCq1R{Ud7N+oyFh0QOzNG{+w4dmBZ25zkLgGFkzVf-%B|SY`?jpk&RLk zW_Xa@4uZU^fU?g+<0;3)-aBg4DDqZ1day^fj#pNfUvB((`;RU-NWxBn{xZ3leUIkx zC_F@dn1IHHqs3e<5wg2RmPJWY=Hf+*%s!&)nFM8)&g>nYMSI=oV_LUWd!>GXFpk7r zm%-xBhoJ38IpGsgSR2);1OaRG@^oXqK`4OGDmXF~JQ(S$2vXAgR%VZ5B^=%5Vpl!G zpNO1fRBo1>?Wv9Q&23HE*d){zO2>0NtGCaVW{*M7n6Zwlh40enWZEgQ!on%^J)bOz z!{{L1?pbBAkzo11N$>JV$HfH;@_>&7Wcq2I9j%)@2Y{YsOMXbvBhWlJWb}i_$C{oo z_wCU(L*>4;r)H{_g7k{RlP-AYEV%&uXZXj#<>%Yj7N3}&eet>nb&Y8oesBU{-qqlz z5{5~x4&z(yrnJ@kaMavzg9_Srvj%!f0BJ#vG7{oS@%81=xen4PJoW#k%(dBDrI|s8 zelQXLsX#?5zOmt1HjKX5K0sVfcQ7~C3BGhtNKI94(>KV-b@Y2~2k!Tu7tZ!juv{9N z&#AnxmSbMYMaZMJK#3jE6p*m$j9g(h0S1+&h!a_->(hK_AS~zr(>@EtRk=$X`9{Gy z3(26sx&F6?5Wzg`nE#X-Csu0|QJd{CiOm!B>0f`;xrnOD`ewU&EbJ$!p|4z91jG>^ z6dWnBqBOP6Jpc4~@O4`x5wezb#m^}@vOkRM`*9NO5(JIxdKSIQ@jqXDm`Pqr0(C57 z9kJvL?tAeKO&qc$2%ZYQAp>F7^8Cd=Wgm_1juf}Ody?y~zZ$3WwPnHle$gDRspXbuP?xc^ zw@#PRP60{Nww8aj>^|VuayR~P@I1~S704GAhp#VG4(;GQ@XwNWiwP(UpOS-*Xk%vH z7_Y8pw<=L_C}iHkJlT4q<^KzP)&nM932&TsDY!DDr`6P;3;cEUy>p_fDk(PHAEJ0( zBfP`w7#>0A*umT&7|qH6P5|3ivB!`xPHA zJ^PgVH07#4$k1qusQow{z`&n5Ww}uHj-G(0WWeG^Q2GsqlvK&3>9@DQRV6(hxrRW;lBNQ0Wjb3V!AKIL zZO}*C;@uQ$Q`%yrW-=71y>WLwRogF57XubGEcQsT5-+)M=}a1q_SV);YEGwco1v2i z4O)c+1naN&O-(BE83S&EZnHP<|2gp!qdGY^*wXVaIF1L0$0PU`!;2D?4S@kbLdKi^ zzw~wBm?IMt z$~bLi`&_GDPuS6I!v6igZ0~vMnwqw1HTC7ISF`B5iL##V<8E&92OI$YB4=mK`TQL{yI*r`s( zb7I2u=4S##o<2xp)W|KJ{2_?`jv_B4Hf7^B#&m zbHwq$91>4J1GA_>GEO5HR!mYBHsZ0ECF0*uFUGxvFG%u5&0tVww|*Zvx^fKjPkp(e zqdZ3rq-xQ>oZrp$#EUn9>&bpN4htsThj|5VG~%e4XVWqD@vLcl9EpcS7&&^P_3_O= z|EBm0&(xzYVf=E=B{QasrbY~W$SZJQ=9Bs&5uI@0Px3xj!q@#kr#27}QaNT}%E6!L z3{MVn;hkLuLo9xEV6-Jtp?;PMHEHNsclXg}X1_1~G4=vyXh5s1W^$%HX5{tuK7T$h zb*X-E;E%s|8__%Medf#S22(l}@q*PV4L}AI(FS@GIRRWq z`|Q4!DoSg~6M)Ei)uX1#T;1YF^bRpNt(izZYelI@r#bMY!}1h^PT!3Sk9~XFt-P}L zGNnas3?}M__mE89W2H&)F=G!0{SG&}5#FJO9EX-OOHFgFsM)ARwWtd8vSyhbn~6Vb zgjtTAD%c?zXM}0WNz~i~&)(JC-q~~0n-Cpk<370dgS6^mtyP1cnvi{6UQQ~lN@;af zN=EVVE!;JJqfGOJZ(O4p3dOHamz<^{BT;D4(EgGZ02Kv5V}@J39p)ChfAZgC!vb*` zImVIUNza8=g5G7!c@n{2s|;9+VXJNL-cNk4Eu*Ky4KY13apv!&Nw&KfMElS);)>oq zC4JDzlPA|E@WC*1h+}-SkwcHRr0d9V@>%v-%MLJE@VDkr$)KFVCp=R`T7uzIye^?LTvoQ;kW$>5-S8f-IiwiZ4GBG?mzG_tKDfG9PD;q`5+Dm`c8MtcYjjG^ z0s;nlM%)`ELkW^)h=ofCFv1gPEYOV3Kl{X`NlFCEsSY2>yRc#%5v-zr?j*~!PoF(Y z>h+%*30_eHkc~9G`S~4p!8Z{D2REhN!JHr@p6E`8GiE&RIqRD`-bbui^TMsRqHW43 zPZ+^lUJb_}01~l-FW=oVTwhIc-n~2ayRgs)UQ&PZAdn-q05-2?jcSZg+-d2b*78jmlx0X!(a7?^qk)EM+PUWholKfAd?(vd#ojDYN|vxcHy7Q;)(Y1FFua`i`X2X z6;cjduUJIy&6zh(7Ri7m zEw}r+cKLGM0AvDJvy<7}0AzEoWsoKGC>ccmiB)AOHXde9N7xa3F2f1@*=p^c|5hRk zc#zIy4FC0Wfc1h-JxK=Vt_X`yc`*&u&)*k~cm4kRel&;EoXaH7xuMOSscn?dfjJKA zRBLZ~BoPK}x87`?`t^1F-ds=;)AX(Q!D(Wi)fuDt%kT2Yb; z1gKQkT%g9<8$ChyODaR>Y`SE2eRMCOv63Bjvc#jgo)Xq1b>V66JX9UHF#(l+o0FyV z14!5;Tau$nJ8b-sST{)BeoRs{R}SboSYQyKx~)*|w4oMA{AKVZGUW&3+c^L=8Xicf zhnwH~Zs4HzT}~Fz41sUSGFumQvkGnQEQu?;e!aeW%N;)lXA1He&C;}|OCy054-=J# zhxyg{VM>{3Px}pzNy~`O9cybJXp|fz^~$)dtJI??YFNVEr}KInDrQgrIA4NGfA^&k z{2PG85hF&VAHR?DTfe=9b|}p6FajDJ(GXvBD(|j1s65x5IkRVj^_f-$mr|)wFo_eJ z4^-%PSo`+-Xja$9-#BK)ipd?e@)3t;VVrv)8PBzXFI`^Ph>5g8V#?yB10=bEVE!8# zUOw~=#!Vz7z}lS>`qzRezLeHF#&v0dOn-~*x-@UDZC)`IXF!9q@8tk!CjHM< zrWsSV+W+@0?kCMbIf*={m2-TC%?pnI^P_CyYCGmK&y$g=1m`q<6Vg9jC+Ce>x7nl9 z3(l%3Il6W7MSk3Cften1){xZHlovA&lCib&X?}sl;v(Nf2}x#DF$@L#^`0o1jv@|tlDI57F4U6cY)ujGe$O1W9 zBJo<(C?=*J+7FS54>QmnOUW*;l;&tmpxB^#y8Dtjr;geKj9N6VwSYHyinZ1Y zYg%+Y-eJl85E5eZ;R>cHq0Q%~mq7tsz9iUo)&|Dz$^^G+CIF-lSxIQFHGV-IeV|5fK9 zT(qhN?i0y0`1)k8h0@Un7fxQdP1ZiR-#^kpD{-BfSR#nq{8tWE}0Y+uogsbDPZFeSyF=;bk>s(FX> z>V490@`$T#dEHguvUZssI*P!clB=G@MIo*}uA(oxP6UClww&bU`b!E-bXNP5t8*7Y@!B_w zlKDvwOG?h8N00b|;)jeAUSB`{5-pIejPl0w>6gtsIEy7Pn6U`#qwSIEGHqSTeO!g{ z5&!4|(3Q~Gp#jnwirx!nnO(|Tft1tXL%Aw6h;)d#&-0E%UM1pjCg)!|)D70TBt1^- zy*D!_c^}zrX&<(97F1S~0pGjk-ZEOP7sunYNrH0Uv9bPK>TgHz>?pBjhw9WKOX@}7mS(DbPw(3~M$+%u%Vs{3VWWEd zZ)|R@-dY1GXR?hz%XlVeL)%>Xb0zr}WTaX^I1g99 z-V4BtAmoH(Xr{*_EX-qu9|P&JLdujF?y=fE(D2bptRiK~p(B7kc@${7C#gL%&Jish zm#|lU>l~W(M1w7;#J_yl!z7-snFj!T2Btq`dEKKU0aEB9#!t3?$BXkH?ci?OdNc>T z=M%`QWp?6ih zF|{kjU%^Qk&x_`KITmk;6{TNF^DiAAWX&1*`ncEkgVYhFxggOn6G%4SIvwa+GVYh({q90O1b1?~)!b!GG9jrUkcD zqQ=ptN#d5eFAABl7>PX?)k>JPyy2Rn<_T91+1m{}b>Ju6Ei!r$Hqi0k(QDasijqe# zOQ?wPCL8|LE#Z*pgiBKLZj`%FtmGV_ZhxBhnMp#W`Ci|C@r*tu{SCjr2<6Osd9|83 zJkgUti#+&_yCg*%Et{^}(MJaP9*+W!4BzB3t&ZjG2K~6uo=3QQ@n5y3R=i*0?PHFv zC#U7Yznahd^!BivDN3CaoN(vpd_?Co+9fYN-@JL#u!ww?P{W^emB2p`=AfFx%P>Ww z6fWj}N|>@^0d=uv12eN&(AUtqJ4qXpOyPdxa!z!@O5?K8y#pvOnYuQD6QWt-eOv7n z^5&(Ab9qGR*EFfos6(}=0=Ahp^d`a+necR|(-+@o^Xo3PJv_u19r5XzU)9Q8;9KaY zf=7;YcaM|qD3KjV`nKZ?!!Be6eyOs%p<-kY30EWG^Ny`sW7IHD-j4 zS}64@iE*Vwo3fMs`@Hm)en?Fhl@l(3GMR3E!?_Igc3+3mK+ROy`r@g~44^mFKFKa0 z%Y1~ZM60ya$J%lWtv>>>k}HZkq#`s}+Rxa(ZwWDx;JHZq~!fap# zc}3c{^jpA~k5uLAn&pX!U5%8`$s8XVBA1O-MxmOMX8b-cmtbZHSv}yseiG70c579N z-3^gUM@We2sO)nK`@6+$9he>*I8ulUiDKIbGz%;5Y}36v*VxL&R(r9T!qkm&^hDuk zY^|%v$NKTBzb(6{A08faqI3b%_S<2@Ctkff9I%lJCD_#ZQ)c=F@P z)yI%P3=TT5V{97WBezJOE8a}1A~4Zx*aSd=d~XQC{Y3B1#4rkgXPFbG6q?JuH7F*x z&-PC3+$Q{b^c2{(>78QPHRS5~=#%7E>rH7z7j-tfy1=}}Zg3qyTv{Hjy1)dsA9!fM-jq)Z z5fdRM#dIDbfb)IEbb8FGe>Y4|1w}+wI72IW7yESrw<8l0fjwh4PK}Mb98#1X1UX4^ zU=~s*)j{!_0D?3Fj%;69rbcBMY3xJvLYh9$>eX=@#MW#wB~MpH6z;lfVY zvvkDiA^|tHZ+79Tp?<516FTY3eL-)Qy!(5KR@M~B+O1zP=)**dmM*=^i;Z1{6+>{! zEp1-xp)8k1ircSmg&WrDqW_F$J-EvRH0ql+{N?M{s;L6?S9%{_2H`Bw06)w742xn5 zPy_>I_$64HA8Nmh7lP)jT)C3WNCXJP$gb_SM;R|T;qx0CPJePd;qENN`l!M7+^?kaZ8&H|}qt z@vQgbj~bqbZ4t;kKU=pP>ti|&&1gqD=W-d4>+yZnn@c-1G;%wA^;eT8C1im%;pEPD zmqf4Zo%7EvrvGWva!z#F`6H{94jecTlY|PZkA>6tiUNHrt*YGM?%Er@?5W*d(8pE~ zBE`u;5MC!wogzBBiIa#p*gr7`c21_S;S6nzd5uz45eEa_^{H*5D*m+`BcoL5Ua;pKG$6%F$&d6GtTuG&lr;BQo;zdB3pg(u!$1n3fkTj~x$`7|_aU9M~{fizb zxZ~>&C1>7@eX_2=NNIfZ>DIj^ugBIf-#Axw>zLL;WO$g41-s(VjS%S+{TMQRKZca& z%T~E17s8YM7%yoZUxjpGUzglKVyVR(Dnyn9j-1Y6pT?05slOlE(c4O~9 ztH)t)%Rg^qmnYSZs5(|xgD5Mruk zt_fLkGK7|Y`hCj43s-%`bP2I;7TQEM4&FM|xPlZ)nP&Ob>Ytv+k!MA6BF5-;&486R zaox~<{Uu&gMx!)LIqR7SwPa1|n@2Vyr~ThmT|I4-e%HYJ!_TyACK!lyqPBJ#nYoH_ z6x9_%b~taWO3I54+e@3W_Q!V|V~V(UPKpi~$d?2skQgLwCyu+v;pq&k%b>@aIA~rC zLq0d>UC8 zm$}Kr^cpGFneYl~_@Wy07aZ9VWcd3oWy8y;FV>Qr?cDC6p%Utfs6y>uMBT~_P9mi? zIVJlWuptIfA1eWWQBtmTqcrVa_PYK`%YW*T*wZ~1EJvhnkeo(Az`E-0ZNCu`862^5e+JICtXE`xeyE zRZI^;4@d%_FOOJ|7oaaIAQ5LttqzU#9DJ8nwg^@bR&B!5NWFg72p-Uq3|R>>A@1m# zCbdL8_!4Pb-FIsMx+XDhtvz$O1d>VJi=13hP$6`00?C%2uj8R-)H*CM6MC5UUxqP@ z6|zp{VRl9NQgux@%tu`5T7Z~_-RGu5u3jz8SNhe&oi$)N%WKQJy{DzYtx+$xqW5po z%hdBWciVON)3@RX#ndpr`HWouqV~hDTm169ra%fGo~2b4c5fTfBB&^%=hJh05)yhz5F7Kk3)c*hKB*e6Jdo%R)Zo*k ze#uT4&IYj-rV}KhT1ECGx~6MN+NOVlvVoVY>3HV)CDl_%H;9P+SG^)seV z&z0Cu;F|qEFWD@3^mnL-rAa_N^vw^l=GSb+N|yM|Elx`-EQyc@4;nrGOd7EOykHuF z8(#jEJ%oh6Ezhdui&syK)2-3W9i@NbXT6Ubfi8ubH=Y78qkR*rKgIYZd_CA?Mo{jd z7_&l3M#o@;b20QO`ehx&h|inxhfdF}hfzqP^dQc?RQ4nTJRKlXXTf}0-je&wfn*+t zAd>K6IQf~}%Uo42Y#o)xq(&-Du3x`kar@TlagKco$ShI98fH22Uq+++JkQ=dL4ah* z>YJ|~#<%re)F^@$oK53qD03xoL|RtXuOwC}x$r+}1!WE}v4!E1kW$mt%UrSelMCnH zP3(vbCl1r*NH){s7tamMmEt7wo1*+Y#cgv}8;%Cfutp8zMtAj;Ls0pH#hRyjmlPP&M4~8vw>XiZ$77OM`a6wi2*0sIFJxsQ;C#bBYe_@Ea zo%G(ldqufN;#QWO8g9Q8#)>=(+4Cll-Yy43V7gQ#Qg%8^j`m$af0|b_7b$^&y$r}v z^PtAk5!Ad|a;i~BB`1|}#<0e$cJ10Ro)fxFED_GvGBQmbNZiHg_TTAm?B$mfKdV1| z`dFwbfq9+U{r5m+Cu9htx}?7alHRf#30a{1R)&I1f;VE-P|Bo11>}9RhoJ29+2jWo zmwoDSmQoe)8uV523X0$`x@nR!&&rk%L9i=6T376)l4G6H8BtZSuOaoO_Mp2-lm0C? zIpNLZD)KsYY|_^42~ZZnb`O5uu9uBlsp&D~xR(L8#9zF{Z{sG#9) zo@-v{D1k{Sd*0CT(;BWGz;G)BQ3kJr>v+pKx>cOLD$|$tv|g~sp>^@!_2)|>0RUDs zcYWQ{#Wj_0wL_A_@!4+PxpQvo8gq>rwP?^|?vE+fvPq0yPTl+g=^be;v0b8j*Vid= zX6gV^^wc6Dy4mC;;aK#nl`#dZuU*&YqqxgcW=hI68Ip?8Ja~YLibtgd$%pS~3n~H& zZXakGwcNv??fAaK-__VOzV@bV^B?aUcGLerK*h$=58M8@_4C97`&u4o`FX%x=XTB~ zbjKczvPwMlB_Zfqoz8FE%o;AZoE7)RGxbE>!gANlpU>wmJC{F@70r$y_-8g7?TOmI zf1papqTn*Yu&`})lpHN9zcJP+50B+5ATr$ zqnm{L$+os=%UZSeq+0k5Lt-#xeO)EzfROBn1Jz_NIv2jF!#Qk5Vj?G6$3)=hGqyON z$Xxi1B^W&!ffs4&8f1Rk zRdXnild>R>4LRybSxS}w?cJ-0K+nSTYn-1E_MP>Vr`yqCh-;nonwy}C`ene>sRlqW z$Ehuc)m#1RubpP+x7Q0kbn3;_!eqSw=R(un+*z6H`zq(mfdCxM{dYyA*XuJ(OPgFx zUzaDelGuH&VK*r$DK+O%mm2tY9d#=p-_CJOf# zM_BN6E3Vf1xIFC(3aQCy`*pN#j2hmFRqn;P+aGg1ztQ0E!a;IsU2Xgl-2nG*5gn%S zW9&gesmeDj*=od#?p9X%P$BvLj1}k;dTCnF3FciL5OYn7a&0_j}m-b9M;15uG1_UBmNYy$-DBc9L2q zlNG9hp;dLhZR1DN2$ChjBi<#Ww7Zl6-)qcrT2AhT5`@ubE&K~F&9}9dq*#(_lV>3q z+!fn@6Hgu&ysP!aLfyQ!RFw{+(90mTbOD4u%s#67Z|yiZBL7z-!oamY`qn>W9JQ4A z?BJ2{uUwjbTFJ97F^g1b^>f`~<~;FAN)jwlQ+GtgrIM+ceQh_`+*IZCR?YA^3<6vo zgHUjgC+LUV%ATc5r_Y$tuzE4Zt%DaY?liff}!wQ616JB;TZ`}}WZ!3Dr& zae%-+E!9GP$!jK>fvG2-5P&f6{;u}HfMxB9&zS+n)m~#jmfNhtoJ$Evkp>~q8_7DC z6R;bt)QDTqo1ud}hBB`^pN)?fS!TdT->RD7Tor43HEkMt$xJhfVP$Osbpg;(@INlG ziQHf^E@kWhYpG~hCAi~5(tw3`kaU4(0bfM@Y>TM=>Or*n)V&2a#leZ9zF~DMjL@Ps z#tw3JoAI6;&c3596Cr!HBmpB}G*e1a+2={o_3GC@gfVYys9}whn{ULzEvc@-SFc~M z{Ihg2TQp7}UVic9m%);gzpG>7pG-%R-E<`JnX6?7MkLZ{j|%*Sen&eH>9 z=z|v&%|O$Q-O%fapRQy<5vIB=Z5arIJ^CjJP9h!6R;D9@6a*V2dKSs+CaOeHmP&HcVU+hI1AIqAz6AI$Mt(>>ft9p^szd-fB#?fAevpv*R9#p zb$p+eEeCNuR)c97w`S&X zKWKVZl#bXZTdyf(IS-NY;2IqODdZSLl&y*rh=K69PMLnpPzjLvD=f7A)&^Zqy8nFk=_$HqhsJEUXCBLCbp}0 zdFOg)#+_Ua+dUdX9wQq>`DyxHMosQ&JsCzT`*Q&_tbl;R>6|S?zHI-Klao!Z#1;%@ zQ4BPfeR64RW0Wsi8cv$hK5FV(n;Kv_GWR7sz+jD<9{&JObu~xpabac z3mD&^qen1}sPcrL>ojj+v<Ns|&gPbPbqN{w#;AFHL@Oy52Z{l9OfmvEjXsrKc2XgbcRQKsy{zK6Ky<)^Rh z_USF?f-<*=Ex6|%Y6B3($gMc^5KrRrO@J3vo9^dJNLIMZ)O*1{{S-4gyto^2eDkuC zCI=?_?H-P!P{2Tm;Jg^k7bXmVFUDFMYya)JiBO{-I$Zw(Iw!eF*=T6^K9^gcmY$$)w~iP;+MFAdWr2aKkH^Jr z6S))eSX`v}NS5WO!qgR)^wM;!o*G!z>;E?T9oIJf$6H=Zb6IDc8+##Nv^?&0{maZL zR5)WN4$<2I%R#<+c|-z1!5kB}xJ;0}0$di|b18PGW;QO_2fmI%YUzCr9cv{f$DoT8 zGRhN_M6!*KqNf2DvRQez*xwHF_)q?V83vIJ7uKk*4dhsN$y&E1so`(Mfcn})OFh#4 zWKmd%LzM6u#_#lkcX)j(UKoHxIJW$^I&Hgh9ILZBl{IR_EIrn6A<4y+GLI;Ean-GL zLsB_DX)y~@4p6B6^ojZE2RwIA&!kVvg&EYHv3UC+bj%W4k2=r_|1F^3bChh*?c

<(@783aDFnKK!N5BQrCz$`sfnVNP(o82xJl5Pwe1U`T0ue0J6ig08_&-h*E4 z;X2o!yFA{I*BT_G0&x1*zP{Z(_EcGHYOFZOHs^mIBlv(+U3s;sG=fmf@z_I_k zVGxirIIUG+sxQ**JuL0*94fop1D3_M!43z#bO81Fi`BqQlT`yMG{Vh*G6UtwJ+P-i z8&hF{_soNUp21$VZamTj276lLM{Qq{>E0(qCNs1fu{^IfG%3kZcWy?3V4=Q+8wo2h zpgh|#%cb_L-o>Q$8*13>LD3~i+W`Qe;Rx{oIEvSv-1z%Yjqh9-Ox=7``#tv<^qo_I zh&~0es${@{z_|N+K_&a=r!P}C>!$2V`A14&T7M#x_b?P~>IhHp^RM{-4P=sAJQwlA z9XlC)TI;A2Iq=rThf3v+_tk!?`nO3;#`bID*yRNDiVOuO8*fjVkCn&%+?p)3w}Oc#NS?<{w&|Y-?_SI zEFWC6wb|#RP(M{4L)X>20Wy*SgTFV_1Nk2u<^5Ee2*i)0OD@AFaLdvnRG`)0HSHg- z)qURxaqGFFFmscwV9;cYGuQ~+AjPH?G=D)7=I@>s6 zi;NdVbCm2gF7aYEcjhy7l$lFG{IOvg?HD=g&nF{(eqkVz7w1$khAcubKVMaO{LkJT zZ4srcN>@A==l9d~nyGqaN1Y8j9+$P4&qnGkjK6+K<8oP{0%S_`_BEJu15rH7YxUGC z&RUN;DeI;NnwCtmN51}tP&1&BSFu#DGW*dtiBb&)a+FeX+KPa6&mq^a8jAMCcz}b>`UXC2Y9(gJTsf0%6iQI<-|}E)x5{ zs92SNPgOo)32f7g5)t8N-%g==NOgp@4t*CxM5n^kqK_UaaV?se2k>s^?gOWn-@j4! zQVq`5xfU9l7!xyAuMYRW65g4PQjP#Wli8?;(2#UX^m&HeX%=45ZjFI~JGcyc9iBMw z@XqLg@qGW0uJ`;{QZ6@F&1+zG=s(I=-ue1twgZU6$V8{0-tj#-!p2blC*!UL;D!A} zpb80Fj*Df3Fc%TKfRVE&=qIciNRZ^F1TMm-VcJ5!MWNQ#T-;@iruc%a!j~Gfk!8>R zZb@zJa%A;j=^j`p2|2&oLb+p(c@=>Qrp$IDKrRnGJt{8v$^eEa$YJGDcIDslUl)3v zFqiLXd4nb7x?~6QYC!sHg3#W+QJG5(Mx-se${JraYLSE)>3_fgtj?4wt^K(o-nQ@z zZ|E+aoC=e#_A2j+=P6fMDj%9@@rw*ChN_k~zp%0?~=u_yol_QI(+&l~{46d$GBd$|tieuNr zz}J9l)^2N2=uJqpJER6vlqC{XR-XN;a&>AS2qltL*J-}N3Ma9?7JV1H!)12nnDH)SLyCN3@4Pq}7~ zf<6Z)^E`B&v^pUn!NkI1C%N0KzOQvJ>n>Z)U)KAmJPp*+%%KRF>oz9T`5_#rmUyf`SrS4uukt zQ&8~badiF!3IIIAPwyWl!{XA?=s-}=6F-qDj(M^@DGMX)Oa`%}^73sSG9ZDw(jH$F z{wn$z02RM8R~N++2A<93_JGogicz*W*Q~58VRHGmiYs$q?zKQX33w;HAEvbV1iZCADB><_r-Zf|SrI}))=>x$=t=1dUyUJGE1e z<%pP8G3Y^rul5XGcW1sd4a`;VzTVna?okx~`xFnH1;Dz}W3DR1i7@Xf^63*k*tOsX zx^|57od=_h5+XFS|TdG$|Z`;)u~k ze!jjomRHReh;30L9e)6#$hkHU`;e^DU7*yKDF-*#rd|l^X_N>a8ESC;m`n7BZM z1T=-8mb_~B5p^|5+EjK#3Dma~`))}iuiG7SbHg{VImti%gJ;a>n|CxAp3Yq+?nllBv4OZtqU#FNDB*qh=(g2L0T15~hc`W&cGI~fCf$oLBICxze=rOYJ^#Z-!mXWT zODuMu`6?B*IGNAS*u=xZie)#;PRx^_itxp*e4?_SqBl+PXE2%uT!IPu0ZT{V{r#(> z7}~P>V*NV{CK~hh5jr;NTM7OwUPf@FY;whFFOE|5CPdEWgY_Dp5HDszN|d>%Asc6l zEq=X%e^sSKC6yD))2Ha6)z{a2TO}4oTa((nf4?1e;B%_rC;|zv!L7zJG{Co}4mu4gt&YE80qTszvrqXa$ zx?6F1&8JTqLMUVTt2y}bq6@lguSSVPnTueTF zQa~RZYiSAmxT}C^wmsOnw>66dAt#?|!Q|Xm_arET=EPK0y#mr)>XsKiZAN_mP{YUm zZkl9z%#UU9F@I>iC zv532ta;g)BXp~PN`^3eCv&*BSF7xGu_`O+!$G$X#0y}b=8OUBmbfZLseJZY246C>( z%q+QbI77z?y4=_1DZ18NUy_xd>`%sViH?!8;Hz!+uFU1up;cpYaC58C`SG5dDzYay zX&a`L{246(tIb-=&-Ad!t=pU+WK0_nRl0k0DF2ih_38CkEY5$XWlOs+b|vUQNv$m4cc6%~e6hqMXuL5YY3&y#j+N z^QgdEpg)!iad9m}&94TXmq62ZdGA)p?KLke>vMpN`5+y(BJ$YU8gI2Nt$1sF=##|w zuq+sFILRD<$>=WMhEMyuRfNwwEo^FR{nFCFnow0-XR%-Pa*2-gMyF&h!`38G#Y$?M$XQTRo`If`nQ5=_r`HmHcG%d=qS8M z`y+t-WY8Da!qRf6IULCGK&+&QIwL*3|9f(T2=UJ?@C#3zhQUzKXK#CZ_9H``J8+_Z zg>A51DT0mZ!2vUa6WkI0k2b%cpz8VC^W9lhSxfj^@<;f{*qA>7mlbn1IuY+n5Qyit zPCBn{X~~eev$Zutvr7tIdXG~)f;d1a->v&6xbv<~{!t2;i_Pqv?aqdl1BKwt&D7f> z__u@nPYO&U`JDIOE52aMo6Y(dh4r$NF;CL{4ct$&yu4ia)A8X<_)(k(;(@mp_JNRW z>F?jMj|^K2V4Vv%-+pkHyMws6I6RiCQv0Dm9efXVval2Zjq^FMnUyqn`-5@l|Cwmc z)}Mm!dqEsK`2AG1B>}+bLm{Pz3`S$DRtD2k8`R4K?#>ty9 zsV!HLZy388u!*EZoHt;P?W4bA=Sl48fTiQbLkubPyk-CyE^lgUO(G~Njik$uPJ1Ko zf|xs8PP!}Crk*&Eq$aNNC$Soxq=^icEm_a!v+zzd@Cd1DC%m;A&sBuMG6sS^c-rg= zSF=)yo2sCDHF`5q0*!PtFem;sjffS)J}QnOyMqx>e*#!qOQJ$DrP+mQjQUg#IOKwD z+xiit@REPflcM{Az>)%VW@Xjm#0DOBvlPqvq&@9vc>*@u9@xcu%32W`TmH3Pq(~I6 z&pyv{I*NPMf0N|1q8YHd3wPOU__&n7edh|86#yOa!OQMxzrERr7L|XgI$FZ}`$Mwd zk)lt0h8<^lH4H|?4O-cL1e&bVb^P$G!+quYo#lk>#drA>%>pjWzL@!hsnb4JLm6yi zYxhvsQlWNn#MU7gHee_I3DVpMu`%*|wl$C>ue%UBWSuPLDnpdb&!c9o843#7m6DtY zn&U`F>w$X*%X%9ECDTAo97>O2{F!ZDae8?|neJfZFFTjK$Iy|m`Oz>)>u~U>M zmp-@W;?-kp-YFyxae^^bc8Xgmgq4W};ST5=EiljrB`VkDOZ82UsDoxCv^+|&hhH4` z<=-c&s!j%UNQk^`^bCwv(g8?o-KfL0lfC8xosrOF zpmLJ5~+{>ix zb93MT1=YpHqn!n4aA)T;FcpN0J+djw+O#aHsvbc{Q(efP14;}8Ql|EeDuw$S*CM|k za9ikufo&);QIHP(aUigne}MAXe@qB*D<+d9VS~S2$+Fxt8ulN4GI|iFI1Nu{Q44aalh!pg-zlY`=sG}+VBJpZ^xyHV|{;Q$71QH`F6 zH3*yI_H%4(n#OEhUh68C*$wtJuu=9v?sA~jzdkGU+y~!a~8y!D@TA(5RGF9-lW^jA71R*^)qjB<% z^*Kt`h?;#f@xlg-_<4z;O*>LLj0CpkN|+(`u7vdWpIO0fvBp$A&gsk2f$ZdO1>>u* zFToP-1jzUav$Mkqo~197>WA=mZ&pV-3gb%hntB?IG?V#fTUs=?lm{xy*S}i7D^18m zvcF|Oe>XLSCuprXo9LxuEh+TDin{%MnCYbQ`gKtrix(PS1wZllwd~>H)7?FxymN%F ze$X59)_8JP{A4@hb5oFnnPFlgPP0vT{M`iQ5<(^mO)m>Rt73-97C0IXQ9Ilp`PclM$ zngkkFb1F=|i~%S?GAXWWkI(P&$}l{fbE?sm6IeG8a)Ithle@)mniMMA1*~M5)u-SB}zSDXlO`Ot^GylV8_U;z14=TzqrsaeN<+_567|wd5~_m#hP{Imi&qIy2LKFCq025r&9y7bUiijvq|(wup_jwV}Qb&EaHhO3E-bA2&BCGc(v0 zK6G{&Hf#N_B;Mrva@J`venTgi2xcIU&AJ4p-%R*>Wj78AEjn$0fw1T0-6265RMXVP zMt(6KFacCiSts!Gh5w~ah2dl#zqvWB?n;o2Wbn5)@-^tD4BtLOMSV~S(Ko}tQP4&5 z93CEq9ySyf6|tC&<;qydpT7mu#m(PjZ>Xr)vX3u4_zzlvDbaa9EB2w0obwz`6#}pa z{qD-CDK)o=uMV!UgHInh40(_W-PoXmk0FyLuqX|2-$z%C9j}Q2R_JQVP}`z9OCWA4 zvt1SIjxS_j$S?cX-`}}p*Ydf^Lpm0JRdvy2mZ&b3A@ao&z4fPu$L8MQ0oBCD^_3h< zLFv179#rJpCUh}X20Bj0?Q^@`yCb4?*>=A;A_FP${oqK2Ybu;%3MH( z?d0V_e8jgphJ=37{+)!fLx&>ur!%Ii%el>mlS1a3+Z~pdbA?YJ$u|@zzSt>nBfnmf z12+5jgGsg?xkTkmHz0V|-%Bc^WI|=$t^Es7as7#?d1{gF?~ZFNiaES>UO>28Xq4-A zK~und<$GtOZdD+VN+oP#%zwX(dPVxC+`u5OWT*d2bl~u|<)53Bwqdx;a_Yd0zw=R7 zg;M|*-Qhl+t=mSzg~=P%;nx{J9O0AE0^{uYXl!O0$=q+bD<0n;ZBCX}OI+S3S_W83 zDc*N)0HpS0Vnm$uhlzAS2@b7&2gD~eHM4KP5O7^(=yV(EXUG}%1mAy1xT}l&WO#9N zEO~J<`h=a!yoJQT!P2jRp(T^%xP}VzCz-M`1F-pU7wV>(9aU*e3bMul`OLxJs2*^U z86Gu0^4$~cTg|_GDT-s z7yR6=Gi?m5*n7S*{X&egyX#dc4cG6gPN%2T*TqV&ykbB8?X6d{NUQb=Y;O3fovI}) z2|cU&y8q=dsJgnZoyXA-C(F>;A7i+3?wk`P)ia>ah@@D)wK8rEC=GVCm`+^oZjoTH z8JOuzV<6xgf;nuQ4$brr`@=%d9LF!p+Nx9a7w)3YcuYV)t5sWiA|n@i#_#C(4sI(& zBH<585x8%ycjP){Tm;;b92tf-Ao}^16RxKKAdA2$9jB|)-Rir|?0shy9nHR__S5h> zOxi3Jhx0ZE;W1(>P&7jHMo$eG5!CK#%PN7r)b@1CPpveKtFW*Vb8y`VC7gJuBTAN7 zT%y+u$g$o+F+~$*dPcMEra8xSt-4nKVbG8EXzh>Ad3bu_!C?X+CGdUJKass50T6E}X<2bUg2i|Fi8Ob>HGJ9h7q8qj4d zbdc|6CcJ5=uCBJnhd)2@B4T2QnR5Vk)%1An;fZp10`OV`aZGI4=!D!@GBU9Q$cYZc z_4T}B)8pe+|9#fn+bjhd5Gm9dZ)9y9ak$#!EaP{VX|>oqL>=hw@1P+0@MKPbxRwqO zz=S)RoSe*Tz0inJ7YJymy$@FnJ^kRy_J0elSb4DZj(%zM{1}@;thFuuHn{CR1aP>j z_I_zQB2^%St5 zGq$&+D^m;QFrP396Q_Ea$Ta!Ec?h+gkt)b=Pbh(pNkTJ%O?xI0r}H;%4^ZF(U5A!B zFyE^egkP~68xw#Pb)X|({r9s_Jj+kvuO#*CZ~m^0JDVhFsMD%S({Xm_Hz08@=Q*;m z*nG+zKHSr^%PUXTlKD2^$pa?6Rn=ev`-ZCZ$}cbeY;Y@nLk z+)Em^OPj-i0DDXk5d^K+N-naNa!d((RalrXIh`UpZAmcXc2zypcHe+jCspPJlnt?- z9K>ggS3SlR&ShQYB~-16CA^5?Hy$y{Nn;fgC7$ZDIpLlSw1Omr=aQB zAu&h!$9BcJSHbD>3&2gp!W?MjDfVv#%Gzu&tk_V{#pzT^g{7vE&Ti!^ZGlG zA%6Ow82xdE{Z&|clj7Hx8{4(#3j)Ax{`)EwzpNbq*_(1k9Z`ARb#qVBwFy~6p zQt$vGa&0Kq1`H>m5e*PYK^eWYTLas=(OClIeE`*5y#rj$$x~`Bj^qz^{q7^J&I>+*6;Y2+YlzRI-cwZ3(?<@S&3VGu+ zHIpWo9!3FV_vqQ4bnto%yfGZOjg7~L?1l(bx^Ypn%WF1hwrlHW+B-UiPY|70Pj?^3 zSBFhKNis#0INka8XFaLKiubEI{l}(Vz-vZkC)<4gB6X2Rmt@?!pOZ7u=|)pS67@Ps zDv^r>ecqz^{;t{Oa?J(wSfnpM9qzZ}x!k!bmv+rmx9NUNCkroYYnq4e9XF01&qH0&tBb}*ooEkG33*}+3yZ!da(QlIo= z;BW0l*XP2x$OO62(feF0CBE7j^0PZ+{|Gc6YbI4Oh-Lv09eQxM2ez#p5yc~{wo{(X z3b&Hus<3U2c1@-a`C4J@?KV)FBkq|Zo>c{eY3O}d)6*dJ%l&Z7Hh()D#z5&|R9RB6 z>#dd^(=^lUXjA&F#CjI#%Wt%9a}I}Grf?!n|G7C0Lf+^Q#hhnA2aEkZgaGdpKwaRl z0MO7b-=2&A(|Qe@2*9Sb1YqvVoxvW-+_nV9u-D65$PmzQ1`NbWM*b+3D$q6Q z^U8E1eU)_gB5t4y-W-7B=%BHBZ)xdour_F)&T29iXr&2~s-OOIai2}f{78w4>D&X1 z!SWh2{(eaIa6TC@iT#KIjXN9zW?ljn`}FD6-JLt)s(1F;A$y*8HN&f9)9`ast3(@tc zxMKa{QU?5yt!tQTbXBBV=cJ^}2JHd8G-DjwiY;V!Q3rm^>n8&6b4LDm9^B)!WhQo1? z|8;}2AI}gR{h>gfY>z)W^!Cl0rIUZ7p{81lIN*zk$;ru4Rl(`kWPkT4qWVgWVZ;(( z%9X`K2##1Ns`+*?-yR295q$#clUlwb-%2|tm9WS2z4}tNj$5<&Sy?X~9fv~iXQ!vr zjbXxHjOm^yIM>YKJ9of0Ln@S^``=wfrou(7`f+|9uw}wl!V7ajOdYb}&@~Pv&eaR+Uw&hOH?r?tLI>_ z6|)8~WS|3J0vQ?|l?k)+cTU1-cMbt zo<}b(h3p$rbGX*v!bhtA4*4g>8U)f`ZYH0giwV;+rm&|4Du1=}IomlIT2+~CE=FWY zLA)G$%`(G&TNGydkf|w_2uaDm96Sd@Zw(5pe{aM5fW=#E$+G={kp81W)o*_L zV`Y;qbPa6jufO@FA9u_)(%&}|yIiHJ_cm*N1<+b*nG!L@FbFtq7#A=$Ip#1U+1%Pk z!r?L&BTA4E)W2d;=nZL4CR=RLjaUE0KXrEOWoxW3w=1O)T z3&6ZnEJQ5EZgt|J%9|OJs5r`1{K71o3#Hb0B3qQfs2>=P6 zuY`i%=?260&Cwq8>gt(A+m)3g7TXmSzkyq(HD@JQ7@!F&;^54?_OhFvO`o_f2XJR) z=cn)mhn@BSw2=PECgcecU&p7O(+xO|yaSV|RKrVCNr`=RIS`Z1{^g;IR-pz5)_=4x z@qH(BluIH&2q>HYC=<_zk4rmHA~uDA?jiDrT%lv=8_HM%0PvY=xz~|`;`d+Zs`9Eh zuoweWqJmJ58S&(QcMC-Ld_d{H>Z{i~G%k+Tr7+EBKH?A}r%J`{#?~_m_V15x;clz# z62+N`UYV?BPsnJs{O9z;`jnKxwN#-N-~iZpEWsP)HtT+3Bcd@w0i8;$md8w?2e~X- zjMm4}WDr~-L0r2_x^)wL5L*+bTA!KmReka|I<2@k0UVbOM|CWzH*n&?goK_zfNR@7-o+ zB4*MYgY{0s+FWVMze&Hor+*4doNwjY+8>e;_vsxMNWhw_6SS)Z^Cv4-FoVU_wn7cj zGjYe!Tr1zE(@?9LRB{^TR67MsInOKVXZug4q3WrR5|qN#OVEs~LP5Pnt;^?ja5&gf zv8?}hhEflkbY-dghJOMmCiC)&isxLk-Bs7??Em4cV)OqFIprPf)1mOmN-w{?o|R#s z7N?CB`Aw=%WRY`uxxZoCQ&>7U<4jXu>B+KhFL%#%j`{D4Up2GwLV_WNFLELGqM7^*w_BAx2ul z$q%m@E)rE$RWWt8U`}wM{L!APcMOTt`wcP=1b~{m^EJGM`P(7`sL1V4*C7D_eQf8Hv!d~U$8XWMK{7XTCcyEy7Q;4^>_xleY9N}%r(eW{!A#nr}xF%)H z9`vq3w%}A~aN2!mK*5I4<&W|Li;!@R%(53a=0qRc6EbY|JS=7$bi2B`BxGfcr9#5O z0-)OGl4c9jAV$yTZ!dPyzw-rmruO!B-t-R-Z$0M{5Nef{n;XQ!KuL)xA|e7EBY!7l z@W0;V2%vLjqTkOB4Cq{HkVk^ZDQqvV@WORv7%I?Bi2SC^^uY894ZnU5&fhij*%+qT&tWrK~7XgX}NI0dJ33b*Hf8U+9_e+&Uum~H?B zdISMI>=3T*<%oELu@2UdrCxvhNTMTyfKer-{aQ@f6UnuFQ$sT`Vu1TU`1jAOQpeON z?8s4A*V(2hezl29sY=+xnBN^JlNhX1S?UMfs?JqyDxT{OO{W8G^kEOk8$lno-Xo-2 zz@POs?(k}4G}lzt4wmb8YX(q&(tXKswa-qB)oPcs>|pb2(1Zpt$+hSX>&W+bFMod0 znSt5kGr-~T9$D?LJOU(Dr#QD)F)XCN!m!4Yy(YcbQqiv9n>@zOMw{7mkz*7_-h#8b z6FR>iG;EmawZs7omFkDPG>4efMJ@zbe)$UvXX6H1qgNHc0ijg5HETi~2cl^ew6nHU ze?e)5*8}wZPBpg@R@aMw9%=RXelDA*)UgyXZK`<7lK*IwgYfcSV%u6lDv%^4;%y*+ z9|?R@gt(zf71eL!s2f{8=v>*|+LvCqF_|;TEOdfz#O?fES>!RGn;BPq!`vr^tzBNR z+M}DU+m*dkEDrEF6r70=(h0m6Htrh^53DfW7ypt~(`HzGrj2fETlO@fC*XaCn$yRR znUOp*lmJhcBQqa}@}im??A;&-H^)?DV7d>O7wQbFv(_cly5jF$0O9+WjV(*^WWW&T z1-O02#?QxZLN`0sD-=O3DV_8udJR|-z_5ZXRHtBiC+Fkc2l z;+NH`RI9=0#xqGl)sp~W-W)Kdn`&#ZSrg`jcZ*G1A+l@R-w?4`S@ZeIIHW*?K;ia6 zV_yFDl3th(q@%4I)LlVkDIg030`b20RCMi_0R&7;fw9GCN@qNPEdhwfemez)y2z{i z$+hXK9|c4MDP-M`{qE>KRhWgU{!yG{>+G5vVj%#>{r5TE)mQ1ruYs20_}RFZuDY|J zECdS>o0cgahcedWi9tSRPMHYkR;|`khu!kDD|t{66y6xpenrZ=Lf+ua*5AiL5!;CA z``cTWF@u;HE7ls&HRN$5}q%`gbptfN|Fz{2zD*_=!puo_Ul^$ie?ukXg7^UK$BZ8ao z`ugYqbtIpZrH*i_mSw0ld4`bC!UiTFaCpAR_9MA58T+IP{P*E=d<|fjwjP@kO1zfE z0R~8v7nEs@T8^lq^s%HgG<_DktT}?7@gfWJ|M>h^L_~&NR)VR)unP?s)gljDyzdQ( z<3O)5_A5(?HN4@#X7tg<>_9X)?wy>RY-~TDsk29=z)wPFzusO6vASQsp4M2+Z)Ht5 z@B^T)%~#>pyU%R!Q`bQtanf$cys6HF^6d|aH`I)rH(1>8#&=NkxWMGz7aVi3K%vxD{G#puVPf6c6{tne@&SooUC6TLZhRfLGMwKk>Q3R zu-P9evNxYYzg8Du>IKqtApNnlBx=#JpEJ=Wc9#O+n2vAk$WNX_S`#_AqDO(hoaIXp z(>ABW+yCv+!w3dpmf{h3^D*kIs_3nQFnz{XHQRE!3x)LO_{-L%aNw2WXTK4lg!hic zvB?45!-V@ym;V zq!d1~Ydf01&YV<%j|W}6s3}O|1jp%1_`DKlv%}}BO^a~=^RswnR&0#?bF}47kvao` zkBSa@nVgveaj?YqQzq#9GsCJmHkLb=rEQcYb%Ha~#m$Y=n;S~U_HvqkI&TOEzPQD~*7-^0XN_5dlMh>F6h@Cc=XiBHxm7>N>`#BL zQ?(llr-H=u^JIxm%b8MF3x95u6ucv z9(YB;`e%02&fd;Uy{N_88*^=&PL)3HF*$WF{D|$@WNSK6;0MW=eD1FRvPJ2xq_h+PtR+LQfcT3g01Sc3l(S-~82yjf{!@Mt z)pi#=oTFe$t7uV3OD#$nWdKW|S2^CDiZrbp_D9M0Hv;1Z5%Ll5jv9u-ajSYwS|1pg zK0ijv|2popt6$>@1{OBJ#5>{!%$IMJmJwwBb?Te*0L0n!Xj^91Nmk99@x^K)&=5)4V0=&rzv&RuR8oH_| z*w1(dn*MMN(sp8&^EmQqGzW1q!4_vn$!6yNNoy5YI7ip1AvIJ-9;Foi>xOPQP|KmJJ*oN-6zS02~jn0G6wO~*42@{ z@xRmyTVJYiW)n<>QKLTDAx?uIn-Y^*t_(F*hFm{<@^*H2!G_g?TZIkf#lw**OB<7s z@s&p|w#53JkiL-4&e72@8bqw4V<^GvZHEu~Ww_JR)1juGK%0~`@$kq|zj^b9xgY3@ z0mIiBz#ts_ALjYf(|B5p=w$FutNif!Ga0CDapCv|EL_}NV+?(o8tX-)XpYs02jNlu z&)FL5|9B03eJIBpFr=R6(<=@prlumm>0+P(MpS`?Ne^ExUROi}p}ahPH7^{WqhB9Z zGG7GzpQV0c|FSV;{_|{(-~*`rTpa*(O|apRX-E9OQ??`Un&oFn zMIXFSJ{S?7C3|jKQzV-zsMhSkLrC~?RnmE54nKAQMkFG@1Y&d9hM@*rVs7Fy!vn|_`T-w-w0M<~h(Q>tcy@~HCa&tq7 zO}LXqPr}=2rTJ)(B~w6P7wh%9QQDbz$Xoky$lwkA`cD_RDxo?RLbRkwjuaj^ZUM|& zY;2Tovo(Io51CD8yk^ONwiRP|E=&b%am6IP-NO;E%+nkRO%UdIGy3w$W^ZKsH#g!} zfSjJJ{+HrsP*$%by>Zozmw_Md+KT2*wX`j-eW=qpWY^MQ!=i&4ROP@V02xzd-JWT{ zy_41=k6l?*UHOq9SC~9upUXsi4lT#5|T~GfV>r|>hHt|ZM_@31xfc<_-h@x zOlB-89cwLH{Z>XK?j}en1;qtWswPowAm97p3(So^2>GirY?#4s%bJLZ1qTbcEaIJr zzh`71u9=_mV}DELZUg%hAR7bh)t9-@{UHX@R+PiTb0Co(=hy-yjOAk(6MnMn?d{|B zc|`cs-4Dr~?P#b}Kuq!zzU}I_!{g>FFAzN)*wT_Y;_CknrC=Ai^7a}yTr-b2W(_v} zemq_uvjzVB^F`S&Qlo>%f3SPLzddG87=Gp6ao}#+Zd1I9ny;|9C34<9t6Bh@;kf03d zJh3dZP#0R;-r3OriCI=@)$JRpx?h#dZV-JyLMa%`&<(WQ{6sEOGt&z%h9ljZ+zY-| zHec@n3}3b-pwWuC)NTpfT$z&tZ4z{fJZ5Z+R>e0G7$V7qbRF$&_U3d7WA86kBl{=e zp>%*CJy-v$KDfnfoOHr+Hy4&cV+2#NMDi8ZKS=JXd+~_s^(Pz02wL1uuoD$*(qfqC z-Vh-lm4iU6EL$p(MqBHa$K2EQjlmtRCss*Qv@zr+O(aOYBb;u`Z^&x@(F+R z?Yq_4G~n*tMk)`RcNVUHQ2PM{VL1Ez-KAPfjjmIA+crsBr zkg9q%tsy?BL+Q!*_z0c{)50$oQhxrqR>Fs8G*($%J$O)R{rC-qSmHkO6TZc9Mzft2DAF&XVkTO6;N~9#GA|N0l-Jl>LDbhI=6_rv_Ns-Rc zNKQ$~5yEIhV!&Vn>5Xl_dw!qi`Tenr%ggm%?>o+Y?sEX#R17{QnhEwcU>2a7^y@24 z_{GiVfZVr~_4M@ouL8dP_JoFp2FM*US~)U9!T;Y8mMf?1JYPjepH|MNk0PG$*(c*j z`S+SlwSWHnVY6mrWIW0tnIC1`FMo!WmA(gPjM)!M_rMI+O8Ry6UzuBjeSL)Wfo>CL znmnuk-nun@`JL?f=hk#79@v^^Sb_cY^@Y^-zaWnLEuiMU3)obIb|qB)CZfw*3EYy7 zY%C(?M;#pW$Te=TWI@D*f1z?SKyDTGuypuT;avBcbc0cI*ZV zHCo^r)xA_&SvpN;mFq9-(&YAJ|y z@1TBcsqmT7cH58syONLXrSpB6*2D~HExo3WNSfe`SN9oUM9S?>pWe^Be!W^aV8-cZ z+A}8$mi>ZmH+EjvG+u-SC1ZjaMfw($|e>@HQa?1F-ceG6>QSH-zPR$*Lyink~=%cYe%+hr1 zk=*6r^BAXOhd!J;kb6RREJo4KSCO)pO=f9JiH;fy%-OgO*A2IayTqz4HzPEvWsxO4 zMQhmv{ZwkdvirUImtaHmv!<|}>6)i1VqbYp@}oxo7!){*&lhb_S5-<@)gI1q(M1Nb zwW!7(zjXKy?QdCIe|txp9=`H6@^B|Us?BaM=5^%HIS3#SDow4Etv`GJ(#`0i?~=T$iE8+G(IIUbV6C#PDP(zB&+_sFvvL(S(yBav5Z8x|FFsuKL?5nar zXHoPM32}YlUPXnR+~&|ZYia4Klc?HfYAyk(59ox+FFC4??!)rF-~Ld^dQ#ub12#@; z>j(h*!c;i;uI9trcG-);D6wcHs7=Fc10V1W)80zH~;5s!-N zXLoXGo_Y%Hxnj&LK(ch@Q+=r!nFdf?eDxq3Nd}uwwiE*n4gtCK2T1)6`b4&Ux*7nGw^WmzN>Z<)FWy*N$(*a( z`L2ZFWwQ1mFk#V&C5Ih)8I@J1FK>VtaE)^9$fEU;|m_{b{jaJUY0Q6Ff6Sz%Rh?5&~a zL4o1NfWeuAzLp-5Kb8$X8NW+I7ctActwV`86304OCEpisB#C`cyH_(TEctYzzFI2E zwUdXYfkZxPYETGus=}=u$>x{b-4U~Cx}b;7um3vyo3!T(h=sNy^#+RFimmsvc}ls| z`y3b;AO*RlYqKW@Z1P^5pM!dBpoWKRIeteBF;K-pvsnosn%ST2J6o%6rZEb@Ax{)t$8~<7u&d!5}Ee7d- z5O{SJ9*au*_UZEwm}j~^b$9{tF*3J+k+9vE+P?>gK`OmtYRYL99~{hh>E5RvRiO*%%k;7BJZb0VJ$f(I zjsbiHKu}!Ay_BIOHwRd1@Ot&(IIEy> zdb?4Pi-ja;HWl}eq!s4o{s*V10cga>~XVG`>o>1u`?q%0K1uysG2zxbxTIP04) zT1`U}rUJ*lf(}S!pxT&|qYqj9v39d<6Flk+;@+*h02yzvO=rfcDD_-XCP*|+M;ow8 z{O(~k&@gWO#HiQ$@JE;0*o}CHFOmV_Hy{=kM^rc6@N{ ze#(8C;cghlcINu(QP(#K=&0`M>p4eFCKbY>Z|F_D{cy!4f7|ssT=O2?{oDpyHs=vl z4G)vPl}oWnN5AR92ACd8JlXzp>e*5p{)$@ObxlR$1?;A^*fY%0q%4}7^L=pd0MWPq ztcOt%vMoXj4p1!H0_}PhUt8-N9Vf+}?W6z#5iOg`><93>8wS^f%>}h$d3>+zh2Rbu zkdh6f!O%i7&@9i3M2vf-e7v~tI0qgjNZWP0aEx2LK74a!)%7MQIKm#Oj`VVpS2uXu zcjNPUI6IeA&W;|qow{E@pVKC(5mkfp0=+V~P}5jCV(c>B%B>@BGTL*l0etXAe)|G2 zv^roAXH>35xlxdJ>QCs$M2o7ba%C*aWru-GJhiySB@=xl=cB13U3d_PMI8D27`nPF zVP*^X$d@EmRWZ!ThdWjoLOy4^J_92Y1b6l7E1-1QehNAFUyJ+`c?SFc9O`WS$jfXk z5!>{qbHfoDB+4Jsk@qx}wQH752bx-0zhkj`2!xhXsR28)H@0{9oo{0Y4+8=E5|BqN z*Du}D(%J{f<0@%=C#v_D2VnOvJM`pVZHzc%ft7{$$R!;gpW*3#O%;I-FkPa1;!Ywp zZwJ)b2Pi2qd6Sn*KOvCJph|e*f}Eic+RH_Xt{q=e3r3=y36&g`E*$lu=RW>C4yv|{ z+&7*#hirY`F6^}0`AhvIXC(Ucwdpas)om?h)2PL>Hv`g_kV&6t;I557t?tPlzPXBL zvo}Mz3(m?pLRJ(sEMf%NGGAOyzGKVf6yD#hJWPzFGRC-$AK)!W_lN>5ic^OTbj7U* zW8-%<^#1dx-5j|Fdubd^V5LO2h0}@cXxEh*<@R~}MV0wROH;Pg-bfB_em%cZu@%!o{E2F}i|^WAVJ4yPMb3xLP{BA6;28(|gh2r=b2F3>Xw0f5X7T zoXa!NJ^aM3&0%m8h(#g}Jrd2`@&xT`PyeU-0&K4wnw5qQ4oFWe)(e`BrewHPX_W7E z^p)r4ciqXKL|{e^yO~F5#WwFfG^pUSe-$}?Gi{`*1bQ-9e)!Z%CTbOI8nN|B2Gz-DoEC{WUbK}=QiE|!Q6VMlXRMsCV%2O7p3HVt<3Jux~TX-Zs9_I za`>C_@Q1mC=im{9K=0WaD?RmE=H+t^4khoEA3>oGL;INTJnT6qQd&7nmw(-}@AL~8 zY@PYE%qNV${r$q&fmB{wo7 z<_TFz@+NPk-8^qyjS;I?&mP1+&odChBEqy}<0{JTuFNIa_Viiqz7PRX&A{zPUrYQ~ z9tzWKE2B?0-co`74)Q((bt|w@)EVWNP-aH>v(X(BV}%xEc;@W+KK=w5z*R7E)4!h} zC2}t>nLj`dmXE+<>N|i1+V}AT9cMt#X`sa`Eu{+N*mgjUZ80G}zGo<#Mm`2+9BcPaBk8wY0UrX3;bH z?atJ)vS_W%4Qr`+wnxE@Q{S7LUyEvp{V(0ev^~E5`0+I>cJgM1w>~|Yx+l-g+3@o5 ziDojfUw;+(i^6C}&HTEPaD@I`p{EY+zv224n3q{um)|h+fHc6?k&6^DQ1zjoE zUeb}2^g1Z#gO`>ZTgsB<{eZ;M>pNEz`u_dl(`Q6mE8o6CH7*_!`N4B0OKU!GQHu^z7Tv*MPI zYNa(y$N%SoTlPf0=-~_d{26fb5%u3jgW(rrq^wn)fu0_aXK3GP6bruYk zkER=9#b@|p**P(#Gx`#Pa)E5p2D;OB^zbpZ%hfd)zFBf7{7j0uGuw3)iNft>C-xhg zc#otV?z2}DR=3<*wM_cb6D7TEM7&?m2P$6#X;ivj6-WTon5_92QX3zyzuoLLnmjEZ zOn%X!W5(m<|CB|dc|@9nZUufc&w%UYms?FT5eMnA_Qy~U2B1ZzN~gEryDfA-XutCi zNqkUm!0>6;`*gl{bw5vpIm?eLE`ighwKW$oVeV^pxGQP{;zgZ0gZ8-(f-Cb7^@lW+ z8(E1g;MKxRyYXMe<{a%d(GwWpTX?pg}(Nlmb&mJ-UIq_$(}`&j*&r`!h}>`zXCrPgmU5w*~1{S|{k z-+xn3V2N&j{EY7L22Q&gJ6y_fc?RmmcD?2^9Ma+ItA9&c+oD2FoS7|1+@z9mtArZc z@EEQB^35RQA3S6fmX$2?Mm_Fm#EbY8iwG4S`CP%#;Z?R}d+UY& zzfSq{M;Etp=+w*aI4_LcV|r(OzNsWTj;`|olLjf!WDMexoqO1q?$rLg9#mqB!4#a6 zt6*xbMMX&|GW0mtC-w1dW2E-|a zXJ@X+$g6Zuz&jV3yTTXx%UyoCulo&^Jz)VIiMKEJDy?)7#*OHUvi+H_FI;M2rd#J8 z&Y*jzf#KKbiaEm=A@^1GigHRY4>!Hudue(M@CMN*efr+sLq9N> zJ^K2&R>-dSCu}lh(~(!@S8nb<#?##lxE)wgrE6)?9(E_1vq@ZKfY$kEM_yw}LZ4Qm zSzaOrG?trr!d`@N$vb{=9@?FNo5EcJ%%bypr;RQ5{&3W z@)fN;PyZGLeNhelT6Q(P8K>BDMc=tU4Pjw55DSx;YoIaNR8X}CB*W5g{GFEIq%54} z9NT7Kw97qlTUcjFoX9QwX|>@&zTf>bY)w;(gi40*V3>6UQ`o$Kbl%GV?{B(B$?y2d zg|Hqwo^RVXzPE%kK8j)s8xR6Xu3NB^em3tkc}(D!5wS!=!txOhEYp1-sC)lB3pAE? zH`YbBw>EBi=OauhoDZ8zZ|f8=fM(^#fSmnd1AD(s_}smD3Zv{ANDde{|2F;%rTP^5 z@`cq0$LQa#-}8%igig*}HaI7tz;iP7>b3Khx34J+E?HG<4fdM*J~*#Is9m3K{mi9(mB4&|z0=i}1=Z)mX3xj{E# z%G5Zb1f<)s@g4BR$}Tu_L9FxuSRI11&Qa|g>ah}#fvFD|9;y`KC@ic14RjvGH{0J^ zTfs1@aTXuy6Kem`e8ZrR!W3%*Mn@PQg~kAGldUsTs;P6}<9P3RG3k$L)Ll znhE?|AMR%`HgW~k=4p=#3~(Y9Qi97JvL3ZZFT>6{@OyD7h<7jtXxZJVJHa;pYQ%UC z-8*lp(Ffk*+qWw>!7_WOVmWs3Z+EhM$o#xr2=9G1(FSG;+3y^m(ur@hW--k%;*QomHqc=~iOFaBON&iH^#sk8^p<~Ae!EZA) zKc5|#P2g*#Z1mX@xwSAI9)5|gn&*BVr0re>1xbL%qNsZ|GpxDLSLVS79h5=n;n#Um z|38GXvhUPXu*a6_@aC%L(l@Z@wV(C(^pwlDKr(Z3cA_nWz|pF0e4GnpDvpQfF`LB1 z#$JG}s_&-sxV{A&z^|K{?pRlS{kj)uj0X=M>_POKeIFc*)nYkV#&dzbdUbaTbZ(gS z(|ih~F2k6>+t>Fi?1)J(gOCS)u{RbmpAlvk3kz1VTOwDw1&9m9VU@&!V5%~Iz!OX-amXG03ioVawbI2+S zqGP+=^!2XDf%_>6T>ARs4mWpdn(5vHv;PJi|1#6tJv%!)H+=r`TE(ZOU7W+DFc-dX zy)Jl*!Y*|4gx5XHrGI**?wT;_zH;T^rT3h&o`T+TYUh6_>*$=Kn+}kCZJzz)miPTV z;YVCvsh6RryPwEg0E_+GKen{Aq_aNhh!f@CqJSSP3K-YNQWmh=C z6%+0YhETVcM&p=@4Kwp;O{YF?BR8pX*K*ft$uxx6#jeoJ-n}Oic)yH)Fv>!J+bE^{T zcsToWiB9^9AMS)zNvcRs_rrtu^Cn0Pbz(*r%+}kNk^v#J;uwiIE-iD&vIuAyzqZ{Bvho zcYcThk^V}!xNs@q%JoGHm!*L%xvdw6;p9ZGh< zx*^x$l{$5Q)P6iXlbC|h@>@)7SCC1KBHM>z&^CkD%=}tkr=WLB`$*=Y zdoFjtWD2<;MNn^~=LqJ|PDz{8T+ZUdg8MugD7?~a+7p>e#XdhCq-#^N*8K3QTke!`Eu<&AKeIN01=RpvY{I_`sW zF#h&5JywXL;Jo5VJ-vZ^sfiouKr0=lc*m)erkLaGetu{=qAIJedEYh$UG!-^u zIm^pE7PkO(q^u!a29_%$2DVq()igG83k&u9wg&AQ-%MX6cS+Br(0r!O(Bwe@!G3up zrg|aMpe%sAGd(>?UB2YuDMKQo(G%WFDU@Nsaj!{^$n@CQW8=>{1dJ;j%bdTRH~9Sd zP;LR!VwC3nMJ!q&|6!4$tE)4m*`jmMf;!tO-+oeHjmu0pF7R)ZiMYeyk#RIs?S3Zf zi+SYoXVx!TLREEibZQi++v(n2E}0 z@6rG(uZ!Z=YOy6HCucT%b!DuN-s=CWV#}7wa(g&7IPSaMU-5OY6R`;(+BI z7%5i~*{SNwpoNo6Y;0aC?9AMfU++oqoepb^OmAtCtoz#@2sdg+V13t83WxM{r36X& z>@xy^8BS=UdwqRNT8lZ)o;2;T|v$FwZ1Zpj{N)uMV-Dt!9cJpD%Oyl z5W5TC|mOj{2>q7c~7pjA!Lv{|qq?pjBz zg{O$!)|B%JNv3VeNDL&SrGJkNe^CgK_ji3@@qVj$VG3Dy3xo7ag4l_Sy5D9JJnJr9 zz2$<6BTI!4+_H`hlpt#LT>@zR1dKeq6O{^s*Ya5%%Y7PXS&ilnW&AjgGto6R0WEDt zeTpADeJIh%g_V2$)H*xMMjF^6_uFn?vF zR=_EE5&5>?9#sv9OTkMZcM?6kk;4b?A!s6RGyj7cq6y zs8*n?KN|PI;zw$CqInlOKeK=K8+qLiI#*B>vnd>1lC*T3&`@tn-pF*Yvf>suLvOg|rE zpg-?7Ir{7e4s@pbVueh^k<{G|o9dGod9OU@s@WI<$9ZED*|5nUygA>t8&FErg%p;n z2!tMTT?Jo*#y})~BKW+PZ4_0L5I<6Gqw#t$Kj+)b^!xX3=9epp`u@AkQYM6DhuWLp zv#=IuIBO{Z86s#*JxGd1CnY(JJvPzPYrgcH;rT6oDJg@6g@xP#${*$2$jFmnP{CW~ zQgK`4u~^T>$67MaM;UmydyWa zbE~_wv{ZO+!D5GD&Kq|{X!>D^^v1>$c@>I$>1-FUT?FL_yn>cS~I zm~GX1){kNuz|Ys5$E@(pLOrfNcNzUz<$+E=7oOOO`faP)cdpg5$gZ9-^5yk;X+~{ir{TwL46o1nWZXQGYFBW7 zNg7EhnzIWUM7XU^yPB^)kY5)GnN_ddP7E#V7EW_^AoL8+O;BjgIq2=c^h)Z~y`(Xd zG^?6!qVyp5Qcr~i&#wxLV2rgIS7dfDZCQDF{OQhTBa2wL^0;??oUGfvjIY75+xL!BX$c-}xQ(WojdJ2-b{y3`XwyZmcN|fil}VobZFv~QlS8IGdbA}M z^WeULfgvijdM6U&bKZ<=3gdQuH87%{gbCT`B-Z7Ta&2M)+-v=)>Q^*tO_+D5Z-*s+ zbQ{K*{T*#qySKJbfE>;3o_w4m?DD{4hmeds74$0$^|{pFW zPjbE=)jqk!RQ6uvjE4nP!bnfA^cYw5`kGqxyw%94-{#to4l_bdHP0YUJ<=RW`4G6C zCfwY-F=>3E(!_VQdDggOhIOQ_MwvsB6Z?jVm5Ti1S+gF*BRELeVl`ji`TMtTeW_G^ zqesytXIhGeP*Vecm6-;yw}CjX_i!3y)RCLEJexPxtRghDpqR5DkBr*e%g|&edeg$< z+))_G^)cdVZH2lYT~^Wf4K0Ru1E#=|;Gz;%I12`D#rBc{qb-D~Z_o%d6gP zTnRCef)Fr}%_>IOs%{LDi97?tlbxRG5O(FaLWyfA`epv(MNKeNGS9DEw^9yQQ%7gg zsKkWw$EwM1nA9V&_}-#Ns`0X3->9<|EcV2SR`(8}wAmZs_6}&aaL+0w;1vtQJ4$E9=)V(XDHYuf5`(@-s%tKRhYwLoTYBhDL&3Aj!`N1F`OGs$^^*a_cR6_aoI{{gO z;P%L4N^ggkeXo^{xTAmdF26anaM99Nm7<(~_BYZzgOa`f+VI)(W>-yxy#64J-T5y$ zP7F73n?lncFBld$67%%)>njk83O`HS6QFK6QZ{uu3g^jHY;heK8EAwe4c|N}@#x1D zpFbjjju7fRx|XJlsjaLQ2hXdfU1IL74D^VbQ@A!2iDZRlexnC1IJj@p%)1T6A*5d@ z*we~`_<~3A`22|QtK*j69D2MwwEDIsOr5wT2WJ2urCFbwLi4Y(>F(^EwK*EW9+f#s z8NU#)Jw-9DDmk|_P1{LPU*DdNib?R>VWMT-YwObtd=@g{=aQkV-GHo*6NdTSZY#e0 z;g6pk!dEr#h4y3XCAIuEAYnXWp=PqqJcKnu`~LktYg#gTDPb}(Q!%){&?T_c zaA6@%fsvQD=}B9p9Aza~YO#@-h#r}*+w?+8S)jN4n$IGre`#6Jm-QoPYqy=oNgv)Y z)vRCgkq-!qOXNgBS9l>NM!xXBBg$s$u7GaTW9m=KjETE%^D{66z3!L=Gm($IqvZ~9 zGBPrPd}rAL(L2>eX!DM}P%E(dax=?gwt{ze{nHeD*R!jIpO(roFnsd`%~&0ss4An| z_mAOqNyg`nUxPW8uk0--NBCM@b-}1~{qPZy`a>)X3^$SVCx?l!?Ya#n*@>U&fc7;u zU5C7?^B(gOtPBiKUBjpCHWmk-DNH&UYMYuSI@j;mgydL`fAZ_f*)G@ z$#q0a4L`pcbErX!&}dj1nTKXFw{}9EsKH2oR-m)>;Zf+q?(X|($W|_|D+<)Veuwt; zenHs2xcgCxdB*-Idt|oMWUUFXzv2jJ^aEhgoMy#cIsZe;(=%(X^9fc8J^a;=*wrUh zzseM_!$L^j{a>|1^5VX`hKu<1VxI+;-jOqL`7W@-Kq(7P?~pPzYPPwX3_2iku%Zo9 zd3Jvv2xD`dqPz@d)*OsL`j-NvUL}`2LBuA1{P<>YJrAwg z+{F;mn|CPEO)@g|0T#+J#Oc- zJ+$#F^vra0;72Tz1=UzRFv}3R-kx;UZJ8C-Sj&2YMX9=ma5gJHf33ABKAxp+%iITz z>`F}Jb(+}y5=h*7FCx?GGE$aG`W-~xKvS{n9SW1st=o@R`-w=8TDC_E*6sdX_5}o> z@XDH*R~Yt35Dxd&baCrsocdK^?pz`g>6%i@eR%(_FR?BrUY;D1Qi>#H6_BP8=C*Q# zNg73&vfV0Wu`YpXo$ia|^BF0bJr74<=7_?aw&j!hX+S;79I$EC)8P&?_22P)5v>`c z&Kzk@qukZ=q3K6d?}cJX8(RybllA)V-T?-3nXlOkqaUwO=Y_s(wm>|EwLoZ_RLG?_ zhlm97v8m)Sw|%q%xsubDf5pSBiE;SCi7(p-{wl(yv7J?hy*%>MhtM7vQDW% zAoYEr5)rXgH7kfS9~q_K$OM50_KUN$Nhm|kNmeC_OvW(5Rr#<7G)M9eA5JC|B1rRl zx)P+mwx%He+)0yklF7);y)-ab(T?{J0O+^aYG1sZbe_r)!@?H0p27i%N{USPBMQy0 zNj>LX$g>Dg6sarIj4Xp{1|qXHN&#J0r;{L+()R;kM7K<(f`TM9)m8hUB3_M8A5t3{ zIy;2*%Q8|^!4fLDXLTBF~jxLn2XU`hP zd}2eLxELw%L_716Mtfvlgu!r39M)ldp2rJ|9~{q9vM`PY3(j&i^*oQiGtxU0*LIBN z0Q@{^_iC$^1kr1<#03?^)fjiGMrE1`Qq~e#NNOmyVj@9u$<24vh5`|ADt8%v^X{Re z%^SbNRQ*$_?or{f`g!oDT81m*n8vRm2e;_Rfwk!^9W*)on6m=1fBB`#C5K^}*tL*c z>@L>Um^_I?686DQblD}))t_KT{57R1py%~BF&=Lc6HPGa-aZ-@AMK%J0hn{pRW;4L z)q5eP&T-l$C&k4P8W%5_53iZMoxUMcRjI5OjQ^#}p{rsN=dZeY_^5i{$}0QeqotMK zD)Ksd0q~Y7&Qk8$DEND+g*6Jwt5!|3liLIV;%=*C0dj8F2LaVdJ%u!0N7}`p zBl?5!1E#FHCPj)9uWs;8uQ~msik%5XEOA1r3bS!tH1oI<%UOJaalZ`!OZgsna0nqFdmEDvgnv4#SsvQ zwN7n$(md;tPsh0P2eMN17nN^fij=K6kHF(pe4}F5@S81ul;= z@f78V`~B&vpU?J&(8MkShFoZ7t7~qK8X5UAGFlTI`Dftmi8k;KGXaHs-(6EDrzARn zdyF%CAV)_-#;Z`n9Hp|b5J*2>3;m_2#f`hjI%hfmZ};gZtSt%`4W~ydO}p&jkGs~U zk5@N9y>K0_+aNNT$b)@PsV6}$tuYwyvAMJ}=@J?FCnhQ?{Yn3u+@h6smUwm6z%VJBd+~D^ z9#LReAOlkjtnh;czW9R5YCJvJB~0#S#t1SntXBfG06771&39|eE#i{KB8BN&?;p_K z^(9o{s_|f489*MLtoU@0fx$F~OV)GV08P^Kl6Rkf0Hre386DUKHZ7Dob5 zf1WHIKq^ayVQ<=&33|iitfb&|$pEFvob2qLQjEf=*=1hddD+RG@VcAuPM%NSfPO46 zJvTR(U>+D)&+RjFA|&V91Nis41#QDZnTVn7VqhnU9Ur-e;jKGJz4Ug7pdHs)1P#A5 zhp?Kcy()LWE~Z5&EEcMOLwPF?a?J7qM5O+fFphGYuCr-UzUy{?kt*sBvOuiv7%8>Y z^pf}Aei|^%G0C>V33vS_6Fxl;?F`x-mSG9z%F1sb@&`spEXDN>59e6y{$U+KBhkIg z`}g`T6}Lvm$J_I>GcbJry}8=TgRk3~5c35=fIjd`6XM&uR6SyJE|TJNF5rhNC-g0{ zBegzmL#^;;vit~DD&{DI#$W^-n0Q{*f1?5%QlZAc;&QyL!yPiZbUJa;!e@(cR?k4n z3-h?GLtW2x^}+1iCWSbKQK(Umkgp)Av)2z*qTr$hatCA^51D{!BX;({Gb}dqB)m%d za((p<@2N|21p(x4wMashZLN8rPqUu9MY;gLGyrq`^!rDOKfe1l6UyK9h2a1Nyme8{ zh>^tda?;`bR7DtPM5d#CZ9e-e4a<|pV&CXp>UUfr{f&>*I*2MBl|FI)2u{~3EXVOQuf@NqdN7iF*SSER^q^sYTcay=JUWnDF)=b zERZ6~1qc<1q(3{u`(&<@Q=_`UNV~=>rFX6?%bnz2>aP$1z4_p;9HaQ5;L>y%nd&m1 zHGZxe5AqflZOSXX_@UK>`KEY)Z$(LI6luv}V-V+d!fPp8-^nR&@17KRa39nH(NSVg zs}6Mdx9Wh}1%+fK6IiRYy2>5K_A%dQP1$aHB`<+>u%{;%V41>wjEld<#tLDwCGqTW z35zhN$AH)oU@bk$8JJdHwjpyP8Y5t7-dsi5Ge zo|9u9#>LT7jpW$11Fodw<3nZ# z;4fSHKLj~t+&S)jh@V%gchlR`;{la*QZm>A*-NT_y-i7Ybn++;GdVbzQ|=Jqi3MSG zN7t^n=2kzzi0^i#AWB5inBM-(=`w z2&7(>%gjGH%p@Tvj+`$4 z{jn|J%yMp1v!8h2eIm=Sg&Ld=>+r?NgtssngGp-NTLshXK7%pmIOY4DeRn&C+a#aX z1hl-yTs2bUuq#t-26aR8Zje~|vuqtm>l5hY<#$xy@f$Yxz8Hkqf_8c)NQc-FCq&FS zoJPu2A)si3n_~(NFW_9M4nvqU2UOkPX8nD9@T4KExLm}hc72b7oC#@-TzghpdJgmR zvzYC#-rgTy4l;cIavIQI2zy-8f14$i0PWU@7jFZ!dv+^9s_wraQ~X|rbH=ZeTNN>D zl=FOiO6=_HGJczmUMSEpp;IxoP)rOr^LPHysW-ACl?OygP$6Tu@mD%M`K2nVH9fiF+A(6qlEmeYeKF!px10 zA&sTD@SjLub9huQC?bNzmR5nzK0qtexh8 zA7rl!4Lv&UJslQ%a?hsOvf%MHs@#XXfr*JiXxj~10)M0=fP$rS&9Y?GxN}T&;vf_N z<2*HyRp)x;OfHVllHfepbpEW$eY6?Sw(R+2szD0|#=0pA#FntjWiFGfWU~J>hYopt z=b{!w>NBj&c1o-9Ge7J#9!FDdDP&y;~DOW=qKAWeW?Dvl^`$oi8wD`!M^&7ty z7E>q9TYN6SyQI%gnzgCLKem2_j7Dfo(x{RW66N?Xsbl#0-UG99$2!TE+S<&8EEFm* z$u52Af*OlBD^JrzleM|9o?4#(seO55m8 z0C`}zb`ajm0wZE*(98l!ImpN7BXnsoDNTg~%8evmIxd%8bgdMFZiztFDrRX)d12M# z=C*dQ426S7M3(dv6%*3eLb%YYVf?YDhNnDKa19}LUXCi|BSFwC_%Pqo)&4?bbi5!5 z+=x&2mr=`wt(dUdW~t)tR#&zYmc3tCfg8C|~ zClxm9>LyR`iHnPkwfP039vB)YW{AW8na+Pe!&+RGX?_8G37(E4Wo_AF*RP8Q?s}^~ zxz@9|xELyP{Gegm9)_FZr1`W6$U-jbbR?v?wG}Fu%}vjw8?FuWuM1h)iYg${>Ws^2|q2~fv7y@nJu^e?!3a*S0u0tkX z9LOuTYCV=lT-$-FhnfFk8uGv&c-p4w!b0H!`bWaHgHL;ULbBI0UUUos6++NnglJFl>DFVvR~x zsiEywB4s0X1Q{M0m37;)%ZRHYR~Y;GRJpj!NO8zvd+8Hj+1I#Uyy)-k%|KGHYpssJ zt*AjKxJaLXlX4+x)D!+NFvz<>ip@YB63n_)CS5@et(K{Q*K6Nb?z@4K^2EZVI&(s<8j&`2*jxJ8 z#oI@@WOQ{h?}_>*NV(|i^v@zS_g+ZC;J~%Fx);_G%c2hej3)J{VZTqE#n@fq4Zz~n z;hX``foe~v(KKu3O~y!cq5508w6HId69!C zKxaLlM6CofBGuH?LaSl^2u6>d4e%@ce!o0cr6VCBq3p8yr@5Bs2Rf)LFDe{|E!ROP zV0{1;7_&ix`%&!H=DO%F)0-gS5DT+XLB72t6~V5(RmQCmDT9-{T@j7pOwN@iK$9_W zr)*%oXyIf{TA326_HXkD4i)I}GBT=e?*&z(VI#^PR_pN%=s`rWPp*9#ERaRY)lI!I z6*5ok@MmEt5{$={`r~B9Vm|u>3iSf-Lf4wlO>X7CIoZ7*7picc5bcyz`|7q>>`3d5 z9q-`H9bXuWLqea!@ceF(k6VUv1hqKX+p=6ArSP>{#%*Qzvl#uvkH?EnOG6r9tR}nR zjsXcn2|I^_*@98{np1ftQvF|d=o6t(ZQR_97}!G7ayAJCS_W9weZxTKaT?XnaS2?L zr;~5&oeA^QAsU$`Oz7DDu9pV{oOhQedQ%lW#_MQnNSo0kkBuxj<3t<{VFX7aY^rP^ zu}D4YX%>exuL&|{nS@2{su5}Fx|cibe+{oT&wU7=&R%?)G_e!m2F$o<9Ky&YVxJH5`d-){MKugP!N#dw2u7t}fdKbfb)p8pq(V84x2?$y-DzU)V5$oSK?(rw{C^ zJ^rRk^S*Hig5Rz;tF%-H_K&4}rKP2>3WgR2STT+OhoFHvG3H!J0H|(o4NmoXT73N3 zIvSO%EFv#&_J4-hw09Xs=M1Ks-kl;5Y1AAT2TID*YBgm&y>%^!kr-URBC~utbYZy} z8D_+KlS1nb^WK!*HpcOkgpX5LU`m;WEWCJ1-bxvuY@t)bCzsMvVq(5b?r=>6HGVg) zBq&ICSC)G&*F}Ch!oX0WwB~x__zIqm^G8C(or)&qMF?HK&HJeS;&C5i6Z8#zK;`fj zRO^r9H{pZ5J$#Q0##0Pm@2aZ@CkOh*(Qmx`Vq!1|TeWv|)NyipkXmZu=BBg0yH?r0 z9G;|63_#RIflC3lvcB9r5Se&g48t5)kaOrN+tNY5U5njjMlx6+cmhdedHAr(@|1Vy2{VLKnNf$WW|fwhHLBSEdRHU zbkozbVv{gFY9{uWli}JB9EdJ&Gq1T$cF1S9FRFx>vF4_D%W(sGpOZEnG8qhd-9ncj za~WJ+=Ok>A8E5dB85#V?4`zc#N)4Q&R4{?qvqOSx^>T61Ui$ozl@j1uWU%kJ18pCfEW|Qf~0`JE!uX( zk@coiItMPCeQLzWJeXTnCXAiWUbM9-?)Nh7n?SN=)tEz3Q(x zzp2-wz*HN%eUUm!FgT~;vO8qC&8i%w2gWN~)EvfGK0Z~w0#~jSor0dWsn%(>Xgb#i5{=eZ zh*fz8C`tPHfk}07#+_|b6_wy{9)$BcJvlmaDBLU18(-)#O*c;U9#E#8Hx;LCJwr_A>!&!2WefU5aQxv>OpP(r-sOlGOOVpNO0jE{Qu~B^MD%HuYLR>Iw+Jel_Z6T z1`R|>I-w**^E{A7N~Rr^(?})Kpi)Uni3T)kD-}s-R+>q*C89PV@xAURI_G`g-*5kO zd_J<<`+n|wt!rJ^wbs2N;c~;zu^hz{Lzkb&IYw1<>#F;si9U@uZ3!O*d%^yVL*A`- z-=@Rbv&^{o=Oqn|8+zbp@7Vb`IKJDfan``dXnKsib84hSWxYYMBI2TX>xI)?L!;fY zE~YBSI+e^je}2i)&^Y@(q47;V%N(06PN@SstWmeK#jbklsL+q7hl=^%!e<>+O78Y} zp8BPHgIr_Ijig(9I?C4*K6;(Jna%#*11;!%p2wFzDTBp4U_k{uSh4mwKE+nAwXI@} zX0^jPcTKF)ty`PDcTp547xZl0EOyHTdm#@uz5xWRIU5?9cj>0r z*D=qyjUx|dE_`5FW6ci%Gf;ENgJV1NncNIUjve5 zWLUX>&B$ie00$0=XU8nT^_JXA*vB{VZ#;VQA{*j|c9jYSNs2?19Q>me)MPl6D zyu9?Zv`W_0$&)r(;gQTS$#tp)6A;`HqR_FjIAN6Ed86l#e7bPJXU61>AOIwln}-Fr zrR(3CVNxaI_4DMrL0+Lv$E5}ecVqZ)oD?2yzIAJ^^Pa6+eQHgLj!TaOnqf?~pSkC^ zeH+f%uop_2-^8yl*1#U&5rDIg{d06sfXQaN|E{#hLXCtM_O3sNW0r&xEdO3qxag^O zCTl>@n>af!yL;^5Edj)xPB6v z|9nYW>9=Lh+aBn--McnL`(5L@fFYcwU+@iXY?lHM@88ZCk-%^@F(WHL|WG zvgTEDn4#kgszmJa#$&q#10+@v;z;w1NGnk_>NF+Ol9I|; zxJuRp*^HUI7w?&!n4YegS^SgH?3fMRaI3kbPnvYTG*>z3r$qb|KJNP&qs^b27TSLs z>KQt|X+fA(mIpjG40i3$*3k3JTHd#pswY!l*nKA>4~z`lse%iv%BiA6&+`5<@;JF? z{SGV$!)IY(aJy_W^SxPal5qL4yv+_9CwqHF3id@-J)FE=E%NUmx1RN#FG$gQVz~Fi zeK;fs$2taqrs}N~yPn`)!hLpjs1TCcnHWnSvSf3I^(sV718`A~f)_z|nSE>ohf?;0 z+pl&WCrwtC?~51unme#4jnk9@X1`InrXqX=58sAmizDEV>#Qun{qx-vtkBuFFS)jg zRB+4Xr%axdl#!Vk)xExG^GqsmJ)ED4hq}CxJS^{sferbHx;#ZIHW+hiVbK!UFV5?f zMmREG-zl8;5hJEMHf78-8O(NiDbsN=kO%aHdK`8P*+Je&yVD^uc&JM*UApC1=czno zpya)MataElK%cT8jkHe3Ht}Aad*q!y3b#c}$RdZCLzBtL{4rb|VAi>2?b;X56gr=F z;-jlrp6L%%cs#X^~ETT@pqKLGPa-)YlQ~1VMFq5G^|{E;rga1&iC^;OyMEZOJKHHn3ooTsGH%0|#mjS1Vn| zJcT3+L&Mb8gT--)SFk!`H&as7@i^reup6T_N&upeb`6PJs}W4zykoI;1BujfDOHT*)y%ZR zb!$EApL_2IW|W%26aq`cH_%oMIA10&#JxBJ#INYe)vMR_19+_3u;HaHQW#H_ymO0f z16X4I&7MWcvRP>v8P8Me>*@xk0E?GyIG2>y>DkIU!8BiEbh}(v#8=VvwOX^WpuaR8 zrU1774ZZZzjC)UG6|`|R@SVA<)OPGRqoa*a{V(g_)KO%#W({&1Sck|uwG6DQ5_C)} z$ExAs;Ug#~tKRR|2G!>Mll|>;UCoG_!A(P}0p6*Whc+JR507W`i_?Xx2NQ$d(k&YA z+Sf~nadUVK_13LtYgJ?ulZ*OS0 z784UQ=^4U@}O3QQyG$~m!{Fn=*=q2e1H>Q;mgBco^dIrF0Rn#XX0qmk3_`xMrx{5KSpri&Lcw&EVA`W0KS}K zgXT#J4h<{Um=`tcitIsl*@gBJHYg_=f0jg`2eTFesS|s~j2W`l<E8x_I&8=VqHw`+?@{ z$goK&16ImhRPzInHsr`gD`VVJ8L!B&?o&WCS zf2R6uxn*<*^vC}F4gfEab6%;cHWj_;=`qp(-mrd2QsReugZxay?eCOhLng&)9_>EVFK1h&gLF(l=)kIoZpWm9hL>;sxHT zTHN9E*+M70h#5*=9(1if^8ja5)qMHjw4$PuD(mn-`25U+s{i-ej(&N+t{<1rgB3EO zH=&-Ure-t$*+uL2KYsVQW%`tN{qNp|fcI{mKABRPVp8nMjERj6>A)2xZ$2H$G`+?e z7-%4y%bXR1dcynX3sCk^7M@06*42MK-#Ex^(qye^XLtSRhc2k=6y+!cK)xQrP zz9WCI!->6o*X75%MAlvwuP$EL0M(c{0KE{qMeK9>C z;|!LKrH${6X2cv2DIq^EZ#YcL6KSQy-hp|s>XK|B_(&+VT(B57Zr!?d?`VN3+$(;+^0cj%%EK7 z))b6U7a#?E|098ecf(nwgllKuf@_otw+W>{F<&${r_zs1d`}8mO_!p~Ur^ynVvIlY z!{=`&8LY&B@aSm!6*wO%!Q{if$iQ8^Aj2XhjI70Zy-yPaQ_3ZI02(S*d$ZU5`$G|# zkGnq^Zs(-V6O=7a_A_~@ab%d&fA9|MJ6aI_!Dgup#eK~^VVlds^ibZ#Mdq1AoMq2i zqAZwew1#J z^)t<#**`psKcM3?kpK2e$hw1L6mA6k~m$uxs zzgUaAKI%DUZ@OYt=5It7wkG4arJy;ou7PPmWLpb9+2d@<}Q?R*Tn0`^Cl@tzsO%YHa95pkQ*7MYC66 znQ!7YGKG&@C~8_Qwi^2P7Gdy#D$$QsZj!ievT?oLb3sje9&y#oI8)Ylv>sSyo%1#zMW*F-Fjn96>We$(@Z;;J1#|GDKUV_VkMh-1SCE|Vh;U5Gya{^tk)fBm;DNB{?k9@Vz z+1^?Syw5br+ePnSc07voUAxZJaxFdwR>E>VY=V%e1)x3EZTZA_-Z$*``JV?@{Aj%F zH%t9?!2>U33la_jW?n;diRxRwt6A*FkuTb>E|-qZzNytdNDVmAx6y}jgvrUulA--zIFBWYmzxA(;0^+QUcMNf3O~dLVV5Xb(bI{FfxGa;trX+|F@Eo3bDVA+1ATpCB&m`T0cmUB zaon6zSo^rgt+H=;5tHzGpds?-03Z=9bdA`CUo9f;>-VrYEwTPXs+q;`x2_Zud#D@J zJWo;W9tsQ%Pz}mWFdYGbEF$P^JV+2~PG|I_G7-&m`PIT|99JYZ>0_1uM!^R#^SjRl3g)8HOoJbnpD-Pe5eiYh z@ysMPC|G^@nBZ$4{ZPbR#u{XET4h#cg8NxxnqQ*}Hq5(uuZc-I?m_2cg6aAlCs^75 zm&9R(F*!@VWP=HTK(It#RkgI_FAj=5jZStG*n%rV0KbrgV4l9C;;SFnY~-ZMeV2NM zzHVFFD($`c^{3V&* zVr(b|6K;mRtrHO6`j;`tGo6502NCO|F!GV+%-sr(C%6lP^;mS|w{Ua$7^`Nf4F%q? z4mU2-Vq-Qm#=phH#MBRh@*83>k4|1{z5d+Xqwhu8EhH+Z{71a88kxV;?XvPIMW>m& zSp1|cx&H=$@8qynV?zVq7>Jl%bC>VT3h%GO*}Ov79|PoTvLz^nBj#DU30}W+(>eW@OTNcIUW@csx z_{@tD5n_3*Cj}(kx8q^^ylhC@6B1I6Wrl%GIq1xH|IR^${+dZhroy-V;_q#(e$TOH z^XAS};A_IzksYVAirwNdp88hvOyfJkI`0fTVN9x%zMdZOrAr@7px*u5Kj9k~FHlsk z9dL5uHsM;#a3;w44hTv;0|No3fJ^@AcN=(b78SLTI358 zJN_Jbdp*IQd2pl6Bkqtz>mKbArp8jTvvtttdJ|mkyf4}Nsv_U^#vbfFCIj{4DHr8^ z`oX7h{z8pwq-O~YEGV8uZgULJPog+ZPdyZP6t0$_1#4s&2w$)*M#ZV9NTS{oC(Eu! zf(HkJ2f@M8-lqXJh~N&i?v{FbsXGxs;SgFex88?)5{2pKb)d3BItYKK4oF1pN%twEXR!B`C$f7omMXSDI8a zjlQ2$eg?&G*QIGSZEZLE61=_U4=TEWMS z!akM2;yet6M=4;WF@?R0#nXUtgk+hlq}0^MsiA@!=6vS3G3HsU^Vc5=W17-25-MU^ z-|GY90+75$u3npwlOs1^yp`pl$?`g_Qj%I7xA8g%0a6x758^2~*xO4p{QUe#`u8@x z>oMXYCoj*>1PU|J@Yz0>u?8$O+|X4db&23YK1Z6|!9!S{7rUlWB%*u0m7@t@2T8P) zk2y+tXk|TFM^>)ZzquP;vE;o`d60O!&A%ea(q3En1V3GoWB2DGl8jyc` z^3&hsn2ezoc?)UH0Dvgj7XKi-(p3-!J`EUZ)x92`oj>&LWTfsS*6eNP1Z(THH{U{! zW>UhXOR3w0PWXBxQ^GSyEyBt9hrhqu<-(v>nn%umdATdd+3(RNbvz47taxx@iE3TD z5fvt}u!&7(tZ<*^@Vnk_)-g0(Sqz`;)v+8?)_%c6A8F*%H*SI}2HF;XH-6}k~z-3XJ;e+D(MYn=-jZlE7+u+k1thfKmEXK9_%k+ci84?{va) zvb-k3v9IGJdToUr$_vb%XnbrwH;M8w*8gXlt4q{-K9vN%&7LQlEdhGuQ}5iV*A=mRTZuhVug&v*kn+WG_E@6=W>6UUqY)h?IHwQj2+qn z2}UJ64cbTl&6^SrA7<-3Rv8{0v&u|IQ)$t|hRr@c6 zPvr7BiPr@yrkm7<5jhqAloE#V_V5KAjFD0NUVn5zogJYtF>7 zySNRKNU-I%arI2n+LbEjn(3qnG^mvo*efjUO%#HhmUV!be;Z-u9(xXCe|jX>c`PVOw5hggZEzoU7R9r zi0^N;y>5Wu3ph0Z`1XQ`3P;z)@W7jcg;n`QR1)$ETwcTl78tLq8aJAfyrVSvshTBq zr(BX5FVD~S(FPBJqQHg?<{%ZavgR4@_{!(c<@;-_@axO}Aw)BWN^5H?5!ZswWLzd# z`M0-&b*pALmmoP&Q&v_AJpNgy;SDNr|1w|%mXVKH<2b~(9zjigv7(azkvD2;uFyK( zLKQRpLRPH`awFY)0R>i&n)$ACjQY-a@&^dX99+JU@GGjdz( z3HRId68R*@{jKiC(+qK04}c%sR!nDFl~6+P<7d{9i3z(gf+1P$DA}~MNDV5Kw2RW< z1z0zbjB9ATp=UMfoMD*#&*w8G!iV7IRQlYo$7*CNFyAfH0^P&CPqPubYb*%AJQe_x z0C@?w9Lgc3J(%XWqVVIbyp-kcCtAH<&pvP@ff(B8yj*E%9{b-!3KS2({B5V7# z2gRO0e+47l9AnnIUX8P)ygf;-V=bA08jmI_zk}H-2pE4=7;x{0*R~2ou*~} znj9BIi3LM|pZ||>;tO$M96>|_GqonUz>UNUxN(W34dUpE0;tarMRt+P#fokn0-aY& zO6Cz3c_oDE-wZQ8IPb406kB{8>XR^;jOLeJ+SX&~#+jqHC{2*|LplK8FlZPy7ZE)c z&l#kAFi8tJpeI8_s)rLKqyr~7;=%A&xnmj9mEX5XRL$n~_vxa~`#{+_ zXrFiq_Xk^_=A_tMfXZ1g#ATCugMe2EQ$q-!rSkF97st#1`L%N8%JYa24CMI8I@nbQ z6G1J&3`7?e9oXS3u)-#+nv|ME5-Ko_;J5QnMYp-;NQG+yamdUB8lB>^1-Pf2HGjbZ zQcqIW(91nYw-USeX*6=6h!N-xoxjFuxvVQp=GZ?z_2bQTjC91Yfg=Z5#v~@MMrVry z4bij`l+Dn~k{zCUj70Tq4zJ74kP3nj#aC8<%_Lyi9H;%&y#fX)6Mkn2!WVMoHH0jM;tk}Tv zo24}sxxO5ioj7wt@*RiEG7DR-kqEMVJc3+aY#;kGV3xDusqeB)MWF0_YaHK$C5sGG z37a!_E^w6{TQ$Ii196PtRqpMd{_C||e=dnQ1(-TX>h|O1Nf?0HB7(u=i!oo zIIe>rDS!u1WjjKQqVX6MaZ*^8T)BO?ZDmpglWV!5T@LRDd~oc>-g=b`EG}JF1Fjq!NZfutygwpKNB~lW8@X z9u7g^$j~=1h=TNW3(hcw0h{iM(57XW{tpbsRf+u{g5X>;LgF-eUF9`6i+3Vq=o77T zd5b4eob%762hLB7HN2af+ut3!Id?86>vTu{`Yw8}VN-N*F~ z!1(tHLn1-I%?p21O2~9nwn@j>EFJQ#0-cBqb>M&$Y2ApdQwcL&8-w5obA{`vxLHSg;553uuq<|(=2qtf7yv#PnUh^TWKGE%xqN1X%6FZiM?`qz{ z4u#E|El`jX)=A0`!xhHkGgJcO0=MMX-AQ(xzQmR>Ys%y&0M?{@p_JZg)-U3@kPY}4hPqVp# z2zlVQPk~QdfXM_-R_6ej*PKfy%S#FK7yNQOugo3oY|U3lL8$=BIUu<#qcX5h{V#<@TWW+H6NMa-cg zH?r3P4<$CeR1yjWTfHXvK_48%0khgUsVkcycCt!NbEs!txl%$ibdH!9JC`iWD%ucp z;lKVDOK)N`gja=?VYYT5hfC6KI35YVM{BH{L8zX|*Qb?@y!NB9h zTk+Hb;F`@ES%hm357O0%7^zMK7Dz!s@t@Ll^*_~L38m=vCJ=Rr4V&yASX0scxf&m8 z;TPlO;9Lp8Bz54XITQ3VF=G?!mRO+-j3x%~Xx0Yh4&uBBzi@{U92-R)Ho$F!wwg2Z zG9WhD$nb5*eJ>~|z%dYW9KsJvj1JtXcK`a(idJD=yB}f8bveLtq>rh-cj^8FY#h>c zjh_3?<~<83o{c12oSJ}j#@Sm*pC(pY!#q^;1nNH<1h?~1b0j6x&+=}?o-30-(26;y znA?RWz_n}LhX*ig3v@ZLrg@toN6WdKJ$UfoYyoB7w;&z=uyra2edF(dy-OBdh{J8y z00dX`0_!|kHDE3r_l6q(NP>P?*I%;cxVuQ$!6Q zn6i%H>C8xoH{E)Ka$2EQuzk$aj;cFy zj#&YT8_%n%NRKD&g=byB|3hLwp{=cIX8Rp1}isnM=5#jWeu{(HD> zBSG3V0*QSCHHR6Zd{{zk;M$GgX$bOD%cyZ2pwnVk6=JkbfJ@tKFaUtE1l~;q4o#>q z%A*78k`!=b0Ff)s@Vs5?9w)kHjdPa|Hmx^C3h((t?h{`pS#W~uj+mQwTv_r06!-UP z5HC3njjNItLLjE=VdX;3EK=fjJ@8S_%+E)&Hr9~69e!2iuU#BT3T|(I3u+nn`iF*-gbmU??zFr?aDfaS6bAOljV@gx%KvA@xNa_W5~ zxfb6rv%sB4u4Fft|AWf#Ms{}Xzlg6A79zM$I!F&CnuU%mn7g#jFT;7)u3h>lK*-kQ z_D)GzPM6X+F23y?zAT=?qhjc3+r@+5_L=k$f#?XB`4R(8P2{k}HPbTM;QS8Gkw&H@ zS^DK45`zFeF*^>QB;FPDUgsy0gyLYQx-!oAhoH6&;|0`d9!SL}K8x zF(MSo4Le}!)o|bMC)}z5@fxGAxfUbe6Om|*H8|1IOenki6TZ+}6O^54+TP4 z&sfK)5i>|Zv=|GZ**e{QKEd1z9bk?bVk}~S>EcNvsR@!QKNGu>2;*3GGQAm6BYd$H zEHWkuf8BdZ6M$qI@f`teG|{XH3Y_flX(bVd1h`5USOaJervp#I#nm(K-rY;=9t|6n6HcsI{ijsFk(9KR7&h?~Of(MLqnlU#RDVNS zR-zek+t$2$2TMI%H(BAF7Lf>~@`keRmSV`^_GE&H8S%GF$FURh82Nyy4?chVUkehO zw!=L_4Ca>9QLT$3nzOYLN&+ei7+6wj_)ZnG20iS_ZT!asCdFTnXZ}O^?L9B*h;TBC9;r9 z0UiGsL2;H(;s10?yrLNqpdO6fyhBXNqnLUUQix53z@dMetxs#m}J>`)mdhA8gfm+>B*ON$#gV$o0V z3&wcxL+Qch2ttq##WNKYM|RNu4*2r$fnUl`evAR~4iHcWCy6A-sm@4)@oUJj{P#w;iswN8^#z5)+tEU5dw!f@dH_ zO|3Vg_n{*b(L4Iz=xvHvl6<0|d=h0K^6cGwTbwj^?Ro(Q){afo93*Z00$)vS zZ6d3oBDqTH_DTdSKy#wj7F~Gq-+0pC_q-{wkG-9D$&w|xm5!{e6)47ySUI1GS!u_i zrHN%ZU{=XKdR+zl9T$l_2joqLHq=z(tQ|pbN>ZNX zgUKL54K;kz^>lQs8gPC{dhtF0Z%3km`3a0Wyru5-^7cL$6aFR;{14W~h1-B~_{pl5 z@R4A>=q&EuB;vRB*R3`aL*15+f6f_=EIYGH$7 zzOwsFn?TUkfDL}LJTRds($Cz4o{%wR;T%QN-$t5}F8FiyH=xdqYJGMq7ysCcwu5n; z_NKS@7TM_&d-)1vU#Bi-fCbt~#G64Jrntk)kyylNtyYJOZ>xt48`6Or$HxceXP6rq z88tJ}@i=Y=>vAKT5#_!BZLG$<0SMGNcGW^mH1}AIq4CFzI6}^oLL?D1N~YJ?yDTN1 zNJ)zf#3y3wc!Xa-mN+dXyZ@|jv)nk;R%4!7M1aU@=`-Ccgm<75 zx&cQxqyr?7dw+vCSn5M0LYsj^q5~*Cb*biX5B-y4MxoA0D>v8eDQxAjpW$hKQ$WMK~B<5 z4{Gf-CR1r3MxP}&4^9vtur|c zel8Xb7*kWXV;?};Q#m-$S!@AN;?%4d4dAWx^J_P;3lgFBKsk7)e3=RM^xWbgya_0f zhN__o@r|@oNiE{C@A`e;UtXL4wGF=bQh&`%i{g%5d|i|OOIC|)h1-PS9eAMMhNvZG zZuo71#{R8FkQm;65rK}|9A>RL(iVU@2f{MG<_4ibL3@bOB%6P}#!3^+0*L^eImDdI zu@@##IW%0-UAYp?ILV**%Mp6vums*ShHZ2j2R4ciVu&3E#_dm1@ci*LOYUmmTAFT( zja{7=`~CyIO*I5Pdjw{6NuW-U)Zt_XLVrJT#vri$3$9J|4u1TOsX{#i<{a$i4XrmM1)%npLW0ilXhCNQRzyQ5sZYM|AYt}%%!b3gE76SvEzu8foNhxy zKMvi}di6Qd7-@e^Qh+Z4{B^tT)%&mLAxpHx(DRuAu@zKnvaG!PcxOC-DnA$+bm=AK z=IU2({&liB<~{K)LYa)HdieYt_Tz=CK(e&i3DiVS{50WG$aQ2IY6OVJq^wYYMbe-Z zd{${EemIDfiiiJ%4~{9a^U=-J+C7Kt9z`3aqb#_d>anw0R6qF zD>$%g_!}8f0j4<(*T*=b@=nrPUlhBJ1>Fu3`fQ-Oi<( zhakaB<7H?m(T)Sto#+uVlysk_fY#Y=9PHn0T@>M-q>(%L2FN%GI#9qcPNo8!<~gF8 z8S?QC2gSFOR2+ZE2{W&;&yW){eLDiMj&ji1NK#Odg^dl`6ST|;lQOsTrCBM?V2mdFjpdxK~#qSpv02a3O4*f(hvSwjVywfWzD5&J?d#sy21u-lrw6Nnkl{*bg#=r+=22p>rY>I}4X68Hg>HeS;ZjabR! z3#Mg0V3C$a;Jl<7$7KjW1jW^i5W1D-j$n-2VV>EvOPC!E(l$v*&S73zFy!`q_}jOr zzSc4Dix9Vi&*t2V+Cw_@q3=F<_AG~P6pF&YxJ)vCgmkmeMwo^Xe3N*upC=k$yx<=r zDn-we2=v;cMGz|gG5+x;fSkF&rFxLX zU`E-R6crnr4%h{H_5R-ffvd?#UjvI%asxkOOk4`YoRpkoan8{2eJ|MD)m7SO8`e*x z;jbkpub`RqLko&ubi<>knr8orciQ6xnF=?1(?{nkLA@!SD*y`8C)~JA{qn-X3s8yXZ;nF5_>od%kxM4`X`NYy>mI_0`ba!B zb-?@a(@vT3GS-IfO)Nv?-i*ktsj8rjF~&pR3@zrIQ4MeknDvSiO#>()carAo%&e>$ zx|rqyF`O}X#V)?z9>@b=_RvxO4i#Xa{vdNC{!b48+_RNE)+*|E9bH^rkX9Ws!RCb>B& zzNe%@jQUf&NL@Mnrgtt#hX3b_&%U( zO@emjkub93(1yg=k>86=o0)Kf*n|_(3h`Dw4(3^Uc%RP!K(R%+>fudUD95`{ptryj zUmGiJ!#UW=J%==7_2VPYT%z1YKdPC;V56=4t(=v7{rWo`>!_-p+u=73lBoxuQPY*Z zc#Q8XEaTLiwFO4J_9Ljruh>D>83R)CC4V8+mg&K)pmPif9O0YndP}qh{GmarrZ2uvMs1=-vXpZDUyL>v z3g7XT`SDJd!PD`mkEfaB369&Hu7tWDIbu(zEjUOq@e4gli`TOG+To2t==gQ4-MfGo6tKy=TbF;WREYL z*yGJvKH~)>pL@#e9a#`~Me5iR=R#Gq!JVuzzWbOM)ct1$Q=AwIHA5hVUvV#NdWLk% zAt3A?Ev=Ab*{EK0#gXw@v=v8$ZMt|3$Lu+C=9FeE!|*Ix)KP-b)=YD@(l!w{Uzgj4 z<32shLzoMJan-l}?Y9317eNScaH6-AlyB0!S9OSO!Gqb6dLm&4V!RqU-@Ky^4?S6A{$bf=9$ZPUZRJ9 zP;2+0_u|oL5#hBJ4#q!JJ7oudUxg$HcF8@||z5rMu_zWl>EZdNbxn+VzR|AuViEindXT+!k}311a`SRkwX%yA

1{96n**5pP<3k zF66Y~2uEHQW9vINCOjqa^-D|bjN?bF-6o9Cu-D=sAL&^Hc)LlIi|c9+uhcFsuH*0{ z=WhWIKoFFotpS=(Xy_rh9 zgg#(qk(RY<);xjdE8m~xc)(HR{pu(fRPq6HppjXZDu>sX{sHN|N)T~W*W0_0EQFVJ zNpQ-IyN_eecaz@_0n+06<3!WaVK_AzNIQAF; z_q~L;Wc@3p+>)KT4r*)jxK^t__mW`61uG+Mo$-F|2{H*P+EfM;&$0m5y}593Z!T~& zy#Rww+m?*THX)eZH(^J^Y1(uI0$R(v8_4$z&(O`j({pZ85>c*30A=?BY#3WW(A<1{ zPstuCm2P`-TI&YOAziG>)SpP!p8(N`|Fx2$y#^7B2}DmkLghKkZ?HjhvAk2+iB9|Q zus}UUG){Hh4ympgi6&%>Zb(dGHL=Gi7d=$&2=d=>adFvk?ihiApksy;Y& zllk-yqgU&Zy+Np}O zGTTPrnph`$a_dV{dZj;mffD-_eZ6$VG7P%n9!Q?|KgnqOKJgOxC@K1RR_Tb2umPPd zY{gLniNsG#i4pXeFk+GP*2cHAyvz7bG2gajU-a*k)YOiMmDq`T`1_8hXJSz(-$U2l z;M2~XJB6(37;iB0e|$7~kS7T%u<7>q`okY;Yib%0(E)<-!Iiw}E@<}{XP1kiyGqD4 zgm=q%F>KqG4(+)OtYOJTwFx2NZa_tn4=4khH_}O#+!{4v=!6+Mcc1_ob^G%Ln6E*R z=Oa8q>m#@q7kzVpx88xzoaLWGJ9i#IQ>O`u#2H0MeLk#Sx2^?Q^uBjBFoiyWJH(yG zoD$1~dhT^^Z%N%t>8ZfiuNBs@O0;XcJU8li=5!yo8&or^wqqj47WW|4ZQ(4ynHHa(abKygk<@FuVk!`Etu0k(RQ&{#FnKjAS#nw z;6MujmkT=2AlO~Zg-LBIP#+=dx6BBoDkCE!CT%+Dnn%$JPQ`4ZbkGU7*w_-UhkwjP z=9310PdvIP^z!bZ0Fib-gP;TK))9Gw1k>e>)t4UxxA zZNe%n5_S-vJ(62Ab1hXZ!NKckrU1?Q^};ulOkdIV{U`)U3gWy(M4S_!Y@5^Wp#@-y zP!xLd?4>=wiw*K^N4hU!WYLvC)ds*diLTc2YkzmA^3M;B);rXxpU$Z ztAuAFN}f@a?;n3N;8f;o&eGQk-=m_btsRjb9~+ywg-3T7O~Y@?mr)XFb+Qd~;IDG4 z{G`LZ$6#3*m-rMb3q&~B8WoQ4t4G}tJe`Ca5@UZBpBWu$nW|Sm;ezLl`G1rpoMZ2X3GxcT1k;jpk zhMQs;)A_<97ZcfPchex>=t?(!@zmgn&B=^Kfb3Dhukreek6Tk!wl|3)gaPNt<61{4 zb-?NXuJe#6nlFIu@usfsYheyTFz8QdQVUZ@5MNb&e2g^vYZ377h&bk8o!FsON={?c z4@Eghl*s+h_nbE8)qgC9JWdzRj@vk^nyyjP1}&{KByD&T4uJtio#>JL0&FARS(55R zaA}c6D)sGx#Zd-kW_u=d-#$bvPcuTwHaA|-Ym51DCZ-8cU?;=UK@G=Y0Xqw~K>2~@ zXHKV}#RNzXmGGS0*6Xp%~5%IJJ6>x^z%~FO-`Ih!>5b_B@^C_#V{sl->!~SNTj99p#TIO z^*z?#lzc;O-@g6%x8$XgMx0qKImVsZ;JDmXO^`aH#zuWENyo$Yq)y}Xf$R3pbFU8L zb~oYjRPsqYVG>jbIN2WY=0&4*>Old4qIe$4u2B;fkMloo-YM%kx|Cvh&x$6ufV|PJ zz;1E|QM4t<=4o8>WboroOZH813^_xq4eJ{+^a55b*D>W5%DsdZ)f&oy* zpM}rT9gtYS%&7JNS3Kr;+&z8%^IMk4hzM#Fm=}+(O##JV6qUex+-Tle&}_XyA`pTY zN$Gz18}wEy{-uuUUR^Id)Rdz3HDJy0y5N*$!gzU;bK^ov{$73h&Z5k=olmpLBAo-z z5j!DD2-O+cl7gsejSF3`&L-hIUJT)oUvw5m#Hx zm{>zgE7$^hZ8|j(T!Z(d)PXaz1?xFudRwuiv$OLP=(Duy`_J0o1ZMPd^ffoJO+B62J%J=#$($! zEAzH8GNvJ&<^e|pZJcJM{l&Wi%YTKv!b~5M9bi;Q17_%<5aFvCcaL^5m$(Y>s;Pa* zc?9f8;S{Xtmt)8($w`g3d4P(d_B`UTAiBdyMW-K4ZsG}VWZ;#a4A4Dn`DxnoD^QWs z(;q8MVl6AHtI5ZLYX2CafKUYIh8vE+%TOEQo#Kfbpcbj4lhH)VRLpS1!mYSv` zaRfWu{)%?48|Qt_kH#ne)k!~^3&*`+IBoiPoJ3fe1cIzIB$Zif+-i*fu@q*u$Dmdkq^8E{f8NT&H z`8hLbrc=vGT|R~WZ`OF8^6LP3>-g3GQUvGeupBF=)vMIehBfDXKBa&NA=dqiAhI0^b0J)n;@)ZJvZ$NdX*3mc$K@N8t99E>o~t{DE1^{hG9Vtbk;fOrkp^$ z)0LB}IMrr~Jw~3v=LFvzz>`k=(IKm8X|<81s^nACWPGlvHbN~v!nWqtZWQeCmQqls z)8w?pj?G*{!pN)q#?q5&zAQ^yw1oE#o?37CY8 zHr}^FN9>b6CHz_t=v+QZDuDuwG#u|Kj>ST`xQ1+C^r@%^g%4Eow93vtqN_(6G;tMrEM?|W#IE;65OiLUxgdDm~ zkVHu3YP>Q!qo?a)PD`j-SR{{3>^ctaIS%RHb?;85**=_|WLQdxH)t_?u)y54ngoTz z&i{(aAJKXrTT|^mko;hcF|+mx)(7v08bm}~o{Shgol$(2NG&=Pk)Pfm0(#swtwr?2 z3?7+ZK|hD&!j!3Zus2D^=1(=SwatJ|$&6YBA6~P5y)M#Pm3#n#ACSgPNdV!r!%q8& zQ`YTc#cBk0Pvf@aG{BDa9_v8y zINn=g`sc?xUlJ6N9lmd_l8<<4zn^S^UD%Fo1bxO6YeFBc=q(KKWJqO-^WsER&3V(a zXCJ2EMsG&lgPI(Hyba{9eJ(&j-Xp#@-#fDE+upuS89JG%DCZ5)C?dIdQH~hSXRzus zG3R3H5DAu3{J5S*tdnM3duLFj zkPY|2e{13Q1Ly;sMSr(#Z9$ceDpz<jB2AUL3I~iU#iaA}@Bp7m)!|Beer$UePIXlo^t3 zAovoFIT&VXr^Q4m&2vZV;7rGp4B0BR4qzu>h4wueI?8Q5U)T&rP*)`Oetj`y!c z2)rYFCIh$c3&=g{sIZ`C7<9`<7KS!HsmxJ|C}7Ztwg&yX^_}ToEzcp56I5GHNxX%z zc8##zdV##Z;^$k8D)ZJv2%zz?xm=pyc*ziqD1EI^aLBdAB ztf9}=4t0Ib&!3$8?m)6ulkZJ<$@wdQp_(3qkQ}5I)aK6wT8ST8O3}6fT#-*+0xN zQVC}Sgnsto)?gzewoc4tUC+Hrl4&JHMNBu6BrpTW?YFDCbS4gce6{e$hn~`x-ZJF2 zm3)mq)tk{SIndDm!pq>CnAV5N)|Fc+zYW|U8QSq#f0^o*cz&9Onpy5aP;Nre zjJRZ#A?^KD0VXAFxR>_Ss4$46AIbiy3`+}(VZ2IsH>q`e9FxF`J(&?F?)D}Xu6@=+ zn&3Xz&bjZDE?>Dq&USA+L*5%7mzbI}1|W}32i5E_qO=aU zFqSF^-tLn<+2pOkw=nrWOm1HU3h3q><3lQO;yT#FPwGsIy>-@Z-IvJ4ggxYi@7{=> z^w8Pcxe+DKL)S1EDh1gOslBHbfHNy0$t`Fuu>X(*PN(&^3yLBK`MSp#r&fRsP8OI- zJ}^k&0vX^;rfQQ9oOkV6icd;!gfx5MYTjaP`oBZC&)a5I+4?j zayZ4bkY3m;tdbz!djF5FHxKJEegFSoS;s#1UBg&IS<0@OF}8>(QCh~3RMvpY*^1@%8! znno-oL-)G(8w55)=m<;0N9OiP2J%K;P(S98t@UVdWZW+jsPh0|vy_O3d)Ig{<>$d$ z@;TwX5)xwGns97-X<4Hs8&bChboiNE=MWxdI(P1I#KUi);!|I3@>sbvC914w9)-_{ z%kE}IbLQG%E3eyG?pd_Aem6kT3^;QK8Rkm3xKKML4v)_s6FGqK@sReNJKJ_EUc2q) zpewfCavZet=+rjb#8dTzj9QoX(=yZg(ZJ(Z)s_EzbsK0{*-cXK-$~PzW$-qeM^;$F6!*<7_ zP!N=E*fMl0UbUBPJTKhDT86HRE@@P{Iw%tpPd+7Ipo8p+0y)nqIXT$hckYGkk(leupr!kWpxIu0CxvFVAi#gIN>wFmLvIRB5d3sRep>(#i z(w>XDj0)N~4&74l8UusfhSReP3<@*p$&%l%hlB`9Z&7w}Jm=9gr>oAJmB~@Bb*wTR z4pe`}u82DtQ93`$WB$K?jc;xA@|CCa`5JpK{_Eg$uQdKQj5WdGuumd>uI%a-FcgY_l%qumZ+&FjHPx7!p_8I9<_8Y zJL;{IrIFEon($)+ER(8C<2^X%Wau*BnTJ6$I^Q`m))a~fR%(T zGbHC)N9(uA0GqKP9GM*NJm8@q6;iHQTw+u5i3Ewqct}UlGtfl~z{%hSS|Y zg%zjGa>WD6hSLC&GVNGEWR>CjkDhXU0}X9<=&1KkWjX{HkryGnO-^BGRre=Jad6$po6<{lGX5e5nk-;BX zd!C43jD2_C&iS~E(HfjqJDtLIrb(-Ak3I6U1m2*HG>k^-L3`*;77+rATP8D<5z^p~ z7R9wylVL+nX+}oM_pQ11v22F?50j-`w`-oBne;Gyh-PfVq$H38{-4Q|W62)|%AaSC zlyR6_$bf6_j$ppavIEQ%!n`kKrF;|Zj6HX(Dt&+yW#x1TeFs<;?C$Pf`{%@7`-G}u z#pv6Ck&JWsbW-N9$b=emj%LW|K7D>%758z6Oo^Py*Bw80Dw=?fH;g?|$(4fAvR}V( zLwEiA`guCPmNe1`m_($un`{oB|0M#TyXDpL!pHB|N4@?=rQi6~N9T2)UQI_YH(RP| z%J`$R1R!LBop)tp>GEj{{%X{;sdC7U>Zg|E%~Q0$e|&N!$sY>HC_`*1uQn+)HFd{2 z{mfx4CxJpgpthC59xvPyGA&{BRycTf`5d$mMej!PoPHiz-!A%$m`qVW_qAQT$B{tK z!7eJpZ7du)Gt*5IEf^P`Q3?4}Pn41|c;LWkQUIq-`#o$6b2T6EU0D3mxBYK0E>J8t zloP`}=?tf7^H!~N(#xM!XL3Wy;as-|Tq7j}-l%?+ zWi79sJu-iG9oFcxsLSqMyI6FE@Bh|qt2S$Lq!E`Ley=WVs|w#XGa6wX#lY)bJJRwhk=B0ljzbZiCE)Cxwv)TVn##ce@2h9<$YkIM zyJ5456+VQ8b`L}5dAJ<>UkmaI(jycV3%H@8VmajulV+HswC}3iZ4gTIYSj96V2Sre zWC0KuBBREualUuYpWB2y2%|dMV_3=O@~tDY3L2kfCqFr71G8;@lC7Ot2RQ}^kHpDo z%9%2BE#hUFkYvoEiZPVZoki2FPzYy%M2tv(4!^MC$w{8!)4BzrpjH75-GMwNCGB6nH;QSL5!eQIu#MbPsw6Cg_=nShUi+82A$POMt_LT(Gewj+-pV88# zOS|?zy?_X-nTwUC;TMr6VIn=xI*F{(A<43v_OktiPDKI~J=uTuU(TQ5dvQjha4C;R z6|WFg&={O|cEXqn5ejs+qGCg1lTQ3~In?Bm!sz9hX$8JgK^S}FJXnXQV8krxiLz5= zb}lUlft^Ki^K670HA&zOto_l7RKdE%H{aZoLs0g_h%zRLd?JN&dW~1eyu#xe>y3>!PvN=U~Ih8oK3g#LrY|L@P5S79pD5Q9)s2 zc&yw~G7zFSoU6zl7DTnqHGDhr0c^#b>C<;f2SFT&VNTknO=ss{{aq#ErS1RJFMuBF zDO-Gyg$B=hZT35OV)I4E>%-?JZs|3>YGrQzQ;!~=mn!Cim8n#2&cw;-kdHC8N8Y{( zEQD}QxwJ0y7Ud_|D;jGwlG76NL!3`~05 z>g>eDdO4xgbh#E1itkO3x79hkAb`usmX)jJBzbyU%n;s&hLhWdn@M+ zfaOCMoYlv}9-Wd7ODv}b>NDq~4f+ZrP79tcI3sXPqyty6ALqu4Wx%0HckgP}HH|f= zXhIC8O*3NP9J*EnbnHwg#yH(WEPP$$z_f*ngg&+Ta8upwAz6`E$4=p~KjfE!%D8GG zV-9CS&3)TdG=;%y-uA!g4&4_OY~@z%WK4FJJ}crNdYeHVd7AD?G6T_H)b`vop_j~x zH-5?upoY=vdJXCF;SNKh~&VP+D(eEP$17k^2qQhceOJ!ka%MZHs7 ztLdBFx_kE#eV)#y&&!!Ki~fpC5OfNtvJ}maw8h$=r^GDkvZLFm<@94y?2yIhzqk^M zEkEu|DdXq5w+Brltxf~XCHXWE)-6}(1iA5~d+GETAir_gd0R~DXNmi3+BaxV9rtxEKDC8#g# zac_$%$>s%Rl6W%d9mRo4@1Gu?c2{IjcF32w|^w`0(2sW2N!cJocCs97#^~iL-L)@1BMpm$1Vf@TsbD1NRQQ|ps zgziV(@B9-~A;&f@s&PyBR#kK{e|y#7#M_p}yfjtH*P77XarblcwPTG(uPsmPQ8Z65 z=RC(?{v!)SzA%7K{JorQg5Ffh=mgk1(YYv5SXI>v)O|lv$w#EeqYu3~s<#vxe+3H~ zRGT|(sQ{^>gJjU*JFp}#QrJ{!jL zkb_;+BYI)HP>X0V^Jq6x{4S)egY8WXG8|jDAVbAv;p0q=2HUh z;XlKHH!RtgBfpwvRv}F|24bEc86d;?yPv!4KrR(QL$koY)?1nfyP7W*mz_TzSGeh3 zsmNRlUs#Hne!>Q4j6|o+qIN{>fs3k>HBLin5nKtmT?&v<`#wsYy^Yh95wrhIeHXra zy$aYTANAsWTtjv0rNDE$_w5_KSIatnh^h(d@32lCI-CT+NPfHSiAR0~ znUe@V6$OWNCsZq8o-eYtU9tMT!lGElB^ol6 zAWyO00f2Ix`v4ti@dcM;7&(D5f5fbJRkX$0#?zFxn1d<=_J$>?u_E)_WJ-irg$(#K z^Pt3@RIpaf1_ubZ1cx0akgc}+sv;D<8%siQdLCD_ZdB!P2oRm+kMA)zRJ>X}M)2d! zWDO9JwKF5D71eP*988Cc=hguf?wt zNyy@YZilQm!w@C}@tCBtCKkVZ{W^qBHA3WumFO@5ifR(RtfH$L5^xiOU_!y~`aYO> z(tdCI{j1f77q*qp#*?q_6T5oGC!b{6qIz(A1l|^uSXl zNPnZH2Np(X>7^Z=pcx{S!Cmjmn)|kNONOQN+ovsJ<|6A;l)CQo;HyQ~WBeXm#Nv&7 zyM+)ePqdLqe0;K`6A}iXw+O0=Jt*-w(j^I1rSXKm(m^;t7cFwcbG-){T?n5ad4{p% z4K%|XIx;%(XlMq}r~t?d&A?DHk2{*X_AlM6Gzy@%hOEWJLlDct&B{9hja-=55N-zK zQYL#i#wt z?D`cV3kf#oP=i<+a?x-6EqNCnd(n?4`&1|j1&`UZnu3(r6dq@u5X}P%AQFwzT1Q6@ zA0IpThI7c0A6zO)X#aR{pTw|kf6qd^s2^iG^6~7pYH?n&%g@$x((aqj*mktL<2f_e zdx4*&s%V8vzh0+fj)WDMP5yd9fSetJ;x%tA5S#<}C>NI>QH1WNXq#>AHB8vK_)PBJ z_=#^EdaB##m8UqNjRj@={F|D4b3EOX;U+{AT1p%3VwtmU9Mpia<3zqjZ*O{IkXlva zCQZWW+h8Jn1tX%{p5A(A+2Vba^l^@_ewPeq>Gur`4F27^Zn{*|k0`#_4$GiBRYe;F zg5_PHF4PCX@T!o;lP1)NEWw!p1CB3m)TD{|i+y>oUjx9N&0yYF`9QtTKR(CowgqLB`YhojBJ@3WEMY=`N-7rOKU8Awt&0U|v3M%odOY@YE6JwY%`mTC$id84@sVeal=$t)bg)p(8X_in3^nKQjJ1XOU z*m{CoW1de~&`4G5L6gg`Spv|;AK3^noZ-YkpHB*ls8;E$jp^=ej&vxR$eC<)WurJn zb!*BO(;duU`R%iG%T4MiqpAp&uxWev`7PXGw|NHdD|hvm!q8BD0wCvd(aOb<_H&O$ zL|AKSWJaj7enqYfsWg}S>smGIU0oA@zGU(ZP7fqb?=X;^0@GNA^qx-iVhTOCnZNEEIsnLxBi-jGfDPo_t52#fxmRKh4mFWBWRE{nYxkC!oK$C>bnj9 z7ljj@_KikUe_ZaDJ^-la8(#BuU8DI0CrUGuVCXaa2xO3xu0mC z`C)uhPczzb!yGL9GWPfnQyH~(nb3(Nx}^Cim@}I+ zPTECdCcTw-r5No`GXMZG!YT;eQvF)5_CEHB(~!3YWoCg;K}`5I;s1%5(WI@zVl5aW z@`dv$-FF-lP>3PO>kscm`|z6zIhF+tl^ILH1{r!)TB?Dc{lW3IL}VeVBC5+>|#Gf zoa^YgT+z4(jtFDuq-ND?*gVu>e?;U4auBAw?A~_(JeeNh??`3xxpBm2$LOH4B4wJ~ zdNZu;JCh}<_vh5I?S5)9#$@8*kw^F-s<_2BrC|(RGHYT=#+eRK&mHev95i-Z>HZe` zXT%q8P#xBMN)CG&Z0*sPoI}*? zp^ny$`5{Bh0to%;yf3^G+4kanA=9Hnw{GS)rvLis_RH0wSXBk{)d6M_9}ElJD&w++ zCbtQi3fWA|Yza6d5kaAQ+CLpIYSbg`SwhZ&e3VB4aoSNWcq%1N?ghbfi0q^UoOtke z-zDz&tw@oxYCiX99!mAzCyHFGnzE<7Di|;{5HbRFp6%V!Xe^)|6vA84$?*Hdeb|M2 z-GQnubIF#f_oG?a-4Dtp@tj}xwwH`;|6O&>md&)z$aOXaRX%ISZ*+lyL>LViXPtEP zBInujM|C$p@&6f5zyOgvptGDA&8e~C@maz$3KF8z0uCvMgDsss$gOib6wUccN{%0({>ov4!I|g7`R!tj9i2X@wTa`WGAHiV323E$^``p1>4EsWi*LVD3uI)NX3-qs8?y zJKzz=^)dQde!bL!%WQgSh4{6-#3dLQ7znm((t{G34itZ8wr<5wlv)!2fAv_QoOrUa z5;Vg*PEOMf?4y_?nTYHRTC3dmiY5gakyd>9k3{iYNU5r-D!{Om4tW_#hxc^WU<@9;l8O8XYuj!;D+@0Yg9`7@Fns@zhS z4`Tob!Q;Gd=7@0-ee%rWAiwSnb1i|YOX0K++oS%-;6SW1jRcOcqZ_n4a`51BMPO4N zkCqJ#&p22_eWK^d^aySjQK~Ymm9}>J)J;vN~ zT$T-zCZ&x!w%mZa_nlxIAa%h{I1<)ljgfn?=#><`oFqWDQ4BQLLQ$>5cmf|S_gVGq(qE!dA+Kk0 z>!ukK%a6qZIOXHTtLlu{4anw$$NN-Kv=Y zLMBU>2D2F0|7b2t{%)!sDg%H1ajn2pRAexKR$WK@L~`LSna)JmxxSj0C8|WBD)hb- zZKDrM)?D5d%eWC2?Sy$L(kxEx5F(u^8#81n&^Zk$76SvF(+jG3f@@ZTDg|_|s5o-} zhwH}JNI`KNsO45}H)7Yl6_Q%a#k|BC4wpSLQZg)QaQYPZinM|WxbpMURoo>s|3)*) zpL^0^0ssi|=Q4(EGw-DLMR-qv-Yjhpml!QKpi?Z;YrS(4L-t&Sk>cn|2T{y*j9HF9 zATg@5^s`x8o_;E;geqA_pKW{^WZ_6d*ob!RjteR@Y)|bYjxoraYp;`dwkrR@NU0PD zGDpq1L3;}!wC`i28oHrPJl8*Yeu}9{OPyX%G@U9**b7hs=j0aTvTnbK)hz6K^;|}u z$UnMw7j2M#t5P$3b~b);^q|9rmMWX&oDFwRc6^Lsj(=PY0M&KYzgy6_iLd&ZIAnt& zyTDi^k}LP5yBy}w#=+*WN7Q9|MZ#=is>l>Xog;RD_Dxi-`Uw7H{E;^(KmeEBW;mho zdkF4;VN08hXfP`zYR{nto(th)ft8QT{4Rqw?VEb+sPW?$eznPPve0WM=X~^7^yKvb zQ7z`3n~2@%uFQelV9NxMQm{$xiL|3Rl*$zLcMDfzzO5ae$5qDdvMVmtHHat0bTWc# zY0oel3jO_Vl}h&v=$7d6u$qQ(^LTO`?j77PrI6JH$Pc}iNbX_pGe-7~Y(Yc|IHE$HWi9SE?P4+L3~~+#ttKRFgbFAA8umISjJP4VDOCfK|NwX-7JN zPhe^*47Mhu4bd<<650qM*|d$rHHubcNV6CEtR74*`f_Bp*1d~^Uid9ig_g%(@i|fa zw(-}^J{LK;;o)IFfM*16oHAvHym0t7As2oek#uq>RKSN_-DRM`lxkMSwP=LYQ# zTTcWG1e5z-5VPo2S-SGRd2iK={qEnqe%%h>-gkt}ZhC?KJGY%McR$McC?Eq#qsxTd zeHIf1FMQ>UQ|xoDDTxzLtH@H&ht$bZQcXp$&#Cwe=?5x}eLOAcsW@_t0qbqz*?RKo z1cC)?&)om*>=PxMZr!Xl{KiAPkb=k_>^Fk|uBAS1AFp?om8Y}pULsrGe+NcfRF_Lr z4j)59ey2cp8s{`2l#xPhAd9j{A)Z0yL0VcDnfw@(qEk>3iSR}`7INb1Ui+h%wYE~g zb}U-i>ad5$N$T83m#(V7e}G8CMBHY4=ly!W8+ci5x;7iQR=&?OhcyhvxOg5a?X^l{ z8~fSs2ED#=3hXT$%d^;0Q(-H@di#wzdq?xu6VTC!?rt{V^QfR-BSm7c_sUVvU#tK5 z(lfmChgIz5f%wF$2B!p6Ddf@@of>;tQyQT_LY4h}c6jRBl-;@0!!q^X5uh&8Tk~Q- zp{Da;Mkr$~cdsD@yDFV*;9N9B3TDiomf%%-thg*Z5AfX3NQ8Z+nGHSb((g!fL1%!m>Z~F+!DJ6{82#nX9fd9ahgzO6 zUSMwuGM%M^-I+Y(5L8;R*C8OhFsQe?+ts)0(9X~D9lJIdG;B!713R-ro}NLXmcgL% z2>;QFY0Moe6J5?z_|yAnq)2pI@>LgEckuIe#7fhT0&P4Iq-?P1DI@Bl(g|K* zTsldZgzozIb;gSWkFOciLjW^OG=`;;`yf^fMU$){4YMLx;Y@-?a-G2zBzZ@zw7vz=E=rSOGAdU++bTm+60YzAHKnJtGzF;lzpN~%vxLL zTS<)C09Nc5X?wf6Y=)Z2>XrM>S8Vk9@aq0RF~_TFh9J@tlP+id%u+5x!^C7x(8qa! zxk6LeABdjJee9XPAM!F*h`-rADU`t{cnvoLSJ|jEyx0CtJl zP{=8sr}v^R07Qv;>>lBt>^{A^EkywaVX@uu;+>q~!t;oZQ1(CK_IaA#5qsL}$>d{R zvnuDHGv$}|^Mf4_)RUOCA>!aFW~jRdOqqUq;b^+Y?(P1~5{c;EI%VR7?yQr01T(C| z50q>{kDybR%Hp-TbzlSVdSx9-D*_4vNQl-4FZ4rx;Qy!#OyqaR+q#wiy3_}i*8@cH zP9VVD`cea*GOraN)gqpDzal^!YI)g&E=|#*WeVL=RW_emapf)%f!=HS1ik|_KTt4zI(78GW3=<7q`(+6pI>X9^>E|HyYJhcABBL)iw zKmALArnV)tv+k#_*81*HwM6S?tGn3u01{H(Y_5AfVdQ_hq=EM+i{!a~J^j#E2kC@dg+bnEegcmZplQR5~ zB=E#~)VEw%hE@9iC`vYuW7E<6jY=l~d4)kkRh{vKe$ftc$;$2K4*VmO4gnKL&bo^R z@9+&O|Gw%?ZHsIupi-LY-Ls=zOuRKMwc)~&x2LF*5@;*MS0)Y>aZg1$a7E845f&iv zB3?os#IKAD!fVpK9sG&|{@?B{7WIQYXy{ZYh8;R;}j=Zdt+F@r!^Qy*8I|_J*^)J~#R_JgCqq zJ4M?7uhPW{I}g=mo_OkqOuBQTAuYQ~_kGYC{l?>Z)}7);t&6;+e>3{^RfpK<8vEc+ zGn{`{cQ+&Xq#$ysM#$k72V}Wrr;-AqO?Ssh_|T+`*GiHE;lbCoQ}P05A|99882p6J z%Hzn9x|kM|DT}jBhBMd7;;U)(Sq6<%?z?kY|MLBb1RHYk z=|;^HJQm;Ki+51mOCOhY9H_^)9vTh7)%~Dqb|JrojObe`ayutmFe8ZYh#5W!l68M; zeKT?DbH;{3-jb-OG^UKg_@N! zGl1VqyzIOG{Bx8fy$d#u^2(uZ<2R$U@Fs_r%J`0|VpQJ-Nmev^O5o9ZhU;-#zFbn& z87h+{j8SA5o(z*aJp9rg7mDd-;_oyZSS|yirq^04)^4_#;%ICVe+!fdPm;ZqnV`S3 z?2L78QTDSRI3JbvzUbuFW^Z|iOV6IH)-m>({Ak8XHT{Dl{@CJnbl~cb<75e_+@MiH zm?6!5HD{!8+D6-^_6k@zg5gEOAJ>Nl?iZ7siE;!c&1Dwo*kOt8Dc3q3QQ}Be#VLMm z2W00$p=ao@=Ee9e;X>ft3pM*TFj_;&0tB(g=cTw3sL-TMZL}=l6zAGzPE7;x9>Nix z1VR`w>r0t|^@R=O`$0O4ZmvpvfIK`Mz9yRcSusT|$m4o6Je*@BCFD~@uOzKiwFxEH z%eJ^ni^sI{h!!u-D*PD~p3pwDrZ(!W-G}lb>l1LGDV?8JNdgBmMxZ#-fF0iTlZTON zh9+5XU4{U&JV9?6JLNp1mejp(OHYk?Qb&L(l1;SgbVGtmD;AV@6F#ge1okRmOt(4G zWZ)*IKZ?FF7Rs=Ao~u$D>W+C0<_Bbs^$xVpsEZ%6*=OniHUSL}95(EpzmaYz=cBk5 z?K2qt*-xYTZ`IP+tp5AODD9$F>D#J1bkDb?z!yzlvuI0rrxb_2d<6FaIrT0iSIb>b zPu$5k=cJcqgAq~p!=S(UiP4K|F;M5^W^M45|^V_*i}yDyoqcPlM}JBMF^EGTFIf$G5B` zaj*w|ftR3<@?bH9aKxD9^=!RUFh$_xFhG(~eXkY2S~srax8MF8E^M(!6KBV-?ce*S z15o7o7GnWmS{HTo4C6s6&uwHoQSwVkYYC^LklyMP*G1eH)(PdJWxT6T>UMdkuW!*O zq9YU%kaW8t#US5l6al+Nn7@~I#1_MhxKOvfQ;Y@R!9#!pfkzUfK3-aSon?uj1Vt<%kYt2zQ)rcp0YkzrK+)8j&cMSF=dPHU!)axiu;n>^ z#G+JAGC|e^d65_beD@nI$&Nk=pbKGaCT75ny?R;Qn66gdjwLRPFgK|i(N2ll!rfgJ z=^$21#zzrs3JHKuVR=>pAfPcjjuqLnMSr^K5(mm70s5HO2@(Ufjh9ZTwXB%PLj%%tveoq zmSEq{k7H4hG56?19NYxZl(>YQ>-+aQY6XJ7IRw|OQcOU5>2Qdnm zhyjp8#MZb^ma+@HYqDAo+-&{Y`hCUUTMYX&?C|f+%>E2)MtYMz9(Yk0tyJ<=UkiTW z5m%$YsJGkc1&r<^}dk5v5RJmF* zNQhusVoUO~5)#k{e104arYhQP1>}DoJD&m8=I}#w<@G?0|4i5}RBD?<70PDxssLY? ztIgSa|5(Rs>0dXhTy;%g@q;S2(>Ui4gFfJZiPL@h7~q=e{I=uE5i0H;1*-kQcbrqW zmGjHSXh%7^Y46{%pG-xsJU~uK4IvJt5TR$ip#96**ym3Y+{9TDB(}nK4Fuytjm)$& zpE@ygt$j{N{Sxe#NE-}&Hl{FC7m=&crQu`(ki)=6{478V^WpHQ{?N05acC5GDCR13f|4DPp+kUH`jN3~xt38^cYttu z{)hDZ(`9i4b-mmdkYkdN0^C!uRH1Z)q(s^!2cXRdFbjs$LxR8wZ2MqM>~n+Jvqz_k z#MW*j!^jF$UfXYEgcd>3OEO2`cWbx30ojoly?W(WTg7F{)?k7BWoVZ~#I#p@O{K$s z5j6?di}Zq0XHqlk`2SzE6hsG)LNDxAv%5?}9mO5&q^NR;ku+O`*~tb4%OBPEH%n3AQ2hMl`-OKIVe=UUf!1_cO6*HU!XV^eb>Jymz zkVBZ;@0Q3~Fv^6LfLO|x1l5$IdTv&-%>f5n-R1?n2I?9T;6gudiIBlmj;c}&^@f-t z23}p~K2Z{Y*j#@0P@{a~C3?;InTaDGa2j;o@iA)+rA|Ug79W3|3VT2w>=iYLNlMty z6$bNkWw!QTQ6E|~hc+^|$Ki`nql$VSP@ARmC!)|Ms`B4%98lZ+&1y-wgP$7vU-xHt zEcx^`2Q!V;F7N(q5O`!T7LvE-Ud=66#-88{5rM+2N)Ex$y^UpbMZzMqy{Lr)&mlvd z$1h_T&BhumZug(~8=*6K_)63Io>N3RvM4a3uaTAD?NT!%3nU2yd|TnwDb2a-90xUY zchf?NfotL(k*K1T0uRjo%RznO#*^%yw>1ChbiW?Hz8d>pS4?bb#k@$b<7dxNk@~(0 zBYH0iE+_0PeyQTi83@t=>rP4E$I@n5JZJj*YA0GMJs*a4z+0`0PhESZpb=!*KVm%& z(YvZD>dPcR7o{*ybac(XIcrsU6hDM)Sjotu2z<6E8zYY>iGlIIuJ8NrhZYOFOR}61Sp>N9|XCXr}7>wxd-WrT1%mGfPYUNA`?oz4l#re#_MSS8*3={JyiX=8D(@ zLq|8=%%7?fBV>97fb+WA1PzrW8#~dx*Ks??xzIH@?=8v>7V}w@;hdI-AfgW65Q7r{ z+;RY!gu+xe26)$$5T2|0nI6hARKY3>u>0gUfj!-fv=xXH43v zcL~YI*tJun+hY_i70Jh?OXV?3B+Nuuy-z6#fbO}-Mt8(l<_`v0$}GrgFF!yGA(7U4 zu+fkAe;fbZn8&VR)=*5?Pirdxo<$BQ!1ajv9L+KW0v3#DdE(twwNN7l69Eau{>;)X zx_r@>m?=@X68!saygZfE=`@Bi@pIHI%oC9^6%%)agdz#j8+eY)gcvzrWR2#_)8I%& zPWb(%t+WKc8H^tE=dN89g|FY!TRq?`8WNB|WA(hi_Ot%v2a;bWv*Y+ppHGQTQz^Rm zff-Kij(B+Vo!$7orz0?>sMmwd!2?JePl>9XElWZ>FVn`uwDN9|^3tOxRnOB(sU@XI zzp<%SigomlE<>fKDN0=vrOCgI?)JERC?vuxwWiNTmFH7@radnol6g_S#*9HsxDxw} zGb_S7BM0S7R*Jo_x*FT)Nu1u?``ctZ)PsV(@r~V+PNMx+ur4KduZzml93l^p{{(YG z0oEFmAAa~XgBA2Y4idJgMM~qq6v=e}1-mTi1Jdlk{kgt-!k{OJn}Gei)ibtR?wcuz zOjdnJW+bu(KD-1;<{MV2SB1N~>Pk&z0d9~_)-}DR=6>6*-7lmUuE}fic@T3r;JRox zBCC2+#*};&p(d3}i2SpZ}DvAtRSv6HNSsD58m6OsAK3G!^^B;=!pLg>6GUr=riN$MPS;eCa z{@M+i!h;GLM#Joz&u3(kOJj6Zj6)jR7}sb9ZkasAi3jam=NmE`(?d*vNy?JNpT~bS z86_qWkzNP;qR@DE2$?6wWkrwUtLD0uzSxQ?ewUqC^<3}c_OOM1k=mY z*DxsC7-C(M8B1}EF~+?dIo!k1KdOYQ7`mz(dCg# zQ%F$e;e5EJzmBNgu#NyL&rSJBi1Bh^faTnMTb!zwK_jX-zYKUPf&T*HJ`#rV%BBs#1isQ2 z0~1U9u^=LnU20+~!WGGFA$-923QJgsBE{yzXprN%a<0YYT_4qj`DeH~EQz&5LS9{* zI!4uTG7nqS^2YD!2t`Vw5NN8v*`6AI0;xWC&rBM8EJnxu z_29@?Wf|Qs?#$>eU*+5bAcoH$K@;s2qwe-)j(8Ce*hyonKN&GvQ&E$Nja1#+n{FaI z=8s*w=I}5L>#+d+?HkHISOQ|%Q@zH42WCv%!1Rc#Qrz40!xtZ|g`X!=#I>D^WVV0yX>D_r z&ELaX>JXL=ijg5AM=*tCU)opD>e({Za)MR#l;(WGBrt42hJ@(lQUD)4K3FGBi%`iL z5!A8)v>zi2$!6|Zf+q*{;ON}Wr6PYUq!b~e;p6XFJk$ucm0_hGNIuyOR zYQ1iYN!!cN?}bqygP#!QwvITbdRM?eh5XBceN-$%E*A*c+(WipU2;+bL8SXg)&s(z8JYUh6e+g@Az$C8yOH6gIE0f%iXe<6$Hd(2?qFIHzyV$~h4Szs<^x){}ea28fS!yK0A z$C-_4l9_mD?}EEkb;>1RK7Ya0D$5osqd89Dy;3TS_&loG-CrxjlRHPgwuPQ4e~))D zxc&0qCGsWZgK==c@QIbd=7Xff07BQy*{08;8gL&(R1qmuac>WtcU^X+NhfydXv@C& zFgs{NySG;h{e~k=fH&Of_6+L#u4sb;%@$9L_(dQeYI1Y4DDw>A5e)81{>YMB-zLXr z1;B(_(;z;?f*LO1D(~2>s&t0+1#c#U$dSOo$)W}(6-iM86N;5m#Y&~q7Fu^`!hm1X zwr)G!*)K15&5bsxvk@VRBbDRVT(@>~%SrWJVrF}_?6pjfl0TwECL*eeJA>%wPwdKJ zbfx(e>$UBT{41uO+8o#6>tC3QX!l z4}9a}Jyy=@(Wz5#VqzkeK|#+bhJJTVr>F54N@z$X7VmM_3d5Qx`XysQHlYoq>8a}# zKiy>*W!w#ZlJlY4KrJ4uSnJq)##cbMkmUOthH2~&S&^Np8nxFW@)mX##&L~v- zXM@hZ))H4{-!QaCc%v~7hg3%D9g>ypo^X{GjoNy$xpH$0>QBPE0IeN`~KC-tGjkmZn$lKgt(kAx$l zeFo+hIT@=VpChs^t&H)HmE|S6+q$$em2-^@Y3P8Jgi~H-I)Efq$i;FYs!igLmkIVv zmg?OAJuUC#T&ESJ6+x*F_3lRUz?}b|%(?~sAJyUr#`Rd#CD>dBMmwYXEQA^8J0CJ1 z3|-5%>87_Oj_B??1)Ws^ra=ZBrGk@rShKkMK>Tk4dyA?733q+hIJ~ZF90U!4oZqWL zAWV>Ju~SRvMCd6&1zUEN1beeQ=<8F|#M|%lwv1P~Z#Eb`K=XNA`I`SUfyiATU@n)= zJf{{PPLWZxRqHuraaG?$Tg5*bzbQNst*cdPW6#baDy9E0j7Crq$`CcGU-%g8`4Da$ zjYSIVKdfjv{U@OY6ZOkALl%WDJpHYh2z}mI4K}$u*H%t;n3bx`({AG5bPU@~BCRGWLIPZHhm|k_ zWoSIk=^nhs+`L=o3##&|Y+2q~BtX2lIQDIB)r`Db7;(BF4<<8n-^>|>Q6;fxVh{o(A&&nmWi==F$FHLaBQgh*;_=wSkT4g$KD%SEbz;y z<%eZn2e3pGn2RS=$&aJ+2VLKtLPl`pKu^lV*(A0E;g2 zMW9tlZApA`WK945dAn+#3NS=53^oTHBtx^yfmYoRQKRcCVqA}9~Fe$c|H-ybv5RgVh{Ob=`^eC>z~jJ~m* zXsx6>VgL{47HS6yJM5;v;9mNZAJX4Hxc-tph-4A>I*m|Q3`>K?jeGL-)+CPsz@fBE z_X+>x;+;_BrO<`cM#y?4VglSfsOtokNhiV#HgZv!fOX3xFdSDz(1C1qcAtc$iNH`2 z*U>5T-*@x-I%}MHG8shfyI+CHRC2Shv1gS4 zqQE}FzBA}MaWfXI>;nrd#Ij=<<*^{%3jR%1R03!yap@>Vxz|9SHRYwld#mL5Q*J;2 zIrGW@Bj9K~0;epw2#fw{L_IUlqjfqB6A8Y&u&!4?GAX7i_Un?0l+UO<{+GidG=(Kd zI?SCvYVS&N8mNY^JuY_h*!8EIw}?sT*l|9;96>a_MP;~Ia!_vFss|H34I$AA@MUDW z;zaWe@eT-x6n}Z~SrH*DSa6sF!oN-}OaO_sNwodE2V;f~y;I;bsk}dp`w6eoXKE^w zz7rjGI=-NE0aU*{t3YC^ht|6|_zzmn^2*5PW8GnNZQHK>5!SOX=? zR2=RDx={UAlhh$su>08y1;aV67+l4=_mv)uYS2z#sG*XiLaOo zux02V!4pPpxfN1&BxCs;+DMyt%C6uQB1NME?cVPA@qyLFhZK>xiMIIJkPHg+>I3AO zF8ncPlBQoHqYq<2L{?>4N&5-|z0_{*fa;nTD}l1iFRea?mMo=Tk@EPAF0Ql`{82Gi zqM8>~pbWb~!%^HO7N@~Si?8C`RA)k&^PnGTuS!SVYzGu#eqli_2<`R`PIb8>2N7-;%vIG^JvVHyM}O}gt3-K1%)4$ znI(srAvZ7WX^VuxS8mQXGDlnP~w}gNHIc3{g71$@VdtPLdmo@cDvLD9O-KUnA zg1KqF0deQKuYc{V>bUS&@(Ih)Ym?Hqp~u7%8&a=~R?Hyv7ie)K#BGI8I8GMp&Y(qA z5Mr6B27r!N8dszKqHS_{F-J}1tVy55&=MxAfokLB!q5ww zV-Ln7Efb1?`{3<|6$^2{A12Qsl7aC+gH-|`7(A9;0T>{HNX}4vWA3lN7vaFBGUfg> zw)7^;wJ5TL*NT`Q`3TS?dDM424kIVG-PkYjL}+K+A)oa-E>%RWXBU4*Ws{)5fIN73`GAILzfX5DVHgJjiLK#atGxK90?L=)bVd#yu=PfZVD4{ zoWej@&-Gr-+?x-DVunI8MXtiH^2+HBbAf!Nq z9Nk;w(|Z81Lm$s;ydT5P9X@K(kI? z%0GtO{ib|eUHF{fh535bC5MzEufdCfAwji9owX15^iCLxAVczweC;_qWrOr+ZGKJH z0VNtyUc&{^gSW9}1|PDiV^CRxmsR<{RYt5KPhhwz+ecu8_eO;q{*iB+_~7R(BWlxTXh{Rlrjew$$LzRnl2IialsE2d32EI` zF3se^nfavA)r&{Gph?8~3$eMMi6RC(332RnWOT~)luE}pH)^S&I7Y;aw-I(wMC~xa zkI4!!vh&cb?a;AfKW+(nXpNZAKrl1cy~0GYY-d+HsabdC-WO44Zuu<9=18vU1j7R( zzY4`OJzm`gB$bM#=^f*i41hSatiivTN_>5Nl2C{C?H6}j+j(xq!xQU&-;-e7M3uX> zci5))G6?Wb=+Bp~cjn(kZE5~v=Q^6a`SiGFV^y5SI+ybWOaGkE%PK^^EPFb9mje&m zYsZ~~=aI9$e;(_d5XC}C5Cj9b*T-IV9H3VHAs-04(Y)Dk^^I7H&re*WdaS`kSaP!{ zh8ha`v74hJXmsO6ipy7u*C~xF5Q+-P_fYPrM9?o6pzxk&k{^>1Z9Nx~RX7}F4p zkhD{o@OY_|chx3oDnvEFKp;4cj}~Y&L#?)4cBZ%GSz4m>$OvzUtjdfE5vBtz`GUMu z99}eeX_wGWdhC*A!7EpAKU^;7QLf4@vK)o(NrCH(|4lReBVdZ4(pdV9@8o=xri8j) zm0_)3B+>T|K`57olbpE>#jR_+5(%fiOA31-p%9AfINpGb6TJAHRyabxkyy#ubJc)-SAtZOiUFKsePVhR7 z=;YU=!Tq7CgiY2 zTw3b)RB*jRkx$Rr%=j;SkYS#yc8X`lH_iS-uole4KK5A9Wxjadj4>MHCjO{-faCS@ z@GnEJWB&{_CiSI2a3W@E7G%V5b{1GBO98P8r8 z;zmMMl`^0Ba`l22w)EC-aC6oz0h@btwhzYO+C&+~jXLh zG?;23fqsZl2IqLFRMX*?=R+=Xs9#I4^gD3i7;huLULdlLp2l}`Lsildjv54_ayKRl zf*T_WN&XV;AG8Dg^ts->HT=@N7uH>Emph&5Tc9#VA-2@vJeC-nO@e_14}jHiJ%qu7 z_9$4gvC*T9Xjx$MZyj^9P0BUnbUh9{2`m=oG?T2|t~OFR=NXLdr}_MRc^0JUPAl|n#zRMrG{(W0W7XSa z!Hz|8zn)^H-16pLZzW_!0c?|cBB5X*)l-EPl2h@3v3F)PKTKfYSs@RCO+2TvKy37| z4~Z*GC&+CS9c9I-up`Ystqrw(ZzOLvAuVcs=+-Yi>f3v@kKV!)WH)U6;lSI03#t4GGU2d{{z*Ysm5bpv~GftUIA*rlRAS%&c zGA*z`qf(y7xflyKp%{9A_KkeX>s)zqFP_Z@!XTMKM(_PzO_jO_xRcle?x?c+m_SF( zBRoaPQPMh*zqUwtV6;eP1i7Ei_7|ajKg9CzR7KYmD-bP0=-IRPg0){jJ=!6hVWJtH zj!gL!=gs5x;q5f8lpXZ-wKr~8?whu0GjK~%QW9A#HK#k>|24elztyuEsfwJOeyo1j zXWYHINnK;$%F}XsEnm-#$O#jM2t)Mcd#*fvv(oBzmX=f*)dJ$*%o(a5)h;fKS3eQ{ zIlj>C^URG}C4JZZxKq_>Qq3 z3>!?F{^sPQ%9IIuM_wN(c=zt-zx$3K>5{PJa>j!Am4zesYsa^YtGa}pzwd+L4r*%OWdP?IXM z2(#X>y-^Ziu(eXCTVXo7z0SFUZ?l`{w@VoU%AdBJ{(a8XA*oBGbGO-Su2%G#9(%43m%en8{1Z3#mtT$x=R>_xCUmY_$t{w1-uN^?O0p9CV>)vi7Ld)wlT}j5M`l+5b&A8o=jW@LBz3O z&+X{ax6=I<==SAdWammSOzWQE@Jyyv@l{B~1p~yyS)kiPrCK%&^^fuFi9x*ig-Q|+ z{7OVz+_N1^xW#X7sjfQUAEBNP%L}E3p{P@&MfmK?Km$WRW;K1-8EfT91iQ<^IYTB> zHqO^wb+W1*^8`bhpFMNy=ikDpb32674Hs^lO?ZCLoU8LnQ`RBmQc$^Vm24yiT3`ET zAKyRs-CiEH;rqKw)QU8(bZZ=D_Im*BBL9>RtMi(;?|5gj(r0CdwfDQK3fEQt5Hu^- zE3;099%m|{k#y1ERX+JVMBaZoJ)n{z5|@caf_sW;v;N{>=VjI1m1^CIpe(=~KYI(0 z7S39nq#~^q&`n;RB~R>$lZ2zun@R_U`iR)?xIZDbYR)i+D_)xL&`?G*F`$CX3t}dB zW1--2eUlo4!8*SD-xhLKi*)1zx8yy&kCig?{CQD5%%^h3)&xD7)2E4K!-hM+tzaGI(!GdKJpMXSvIUO}Wg)>+cj&O>%Fl6pXPJKl zY-Uv}f*aXGFNXH+{qk5Tju4YVSCIgETyAH7&%0*HX<@UO&HhQ4Gq?##ujzF1f^^N-^S9ES?4#3o6e53 zb44_@@ng?YkSl>njE6$ok(uX(o5nD#+ZkF8A=XW}8;-*z-;WdQ!KHlv@8VLk^LjV` z9C38Bg(}pJezM8vi~fd+;=uP+d2v5k750~f~wo($`Ry#QLg@^ z9t&z{e1-I3n5*1hs1!vPx9{93RB-SU=BW~Lot`Q|1M$d$ZfnOAC@Qx_&F-SwckTW9 z+Kb0NOV|{LrAe$NQrp=ROFd*Mg!pd^e=_C0%Dr+BvdM?piA;r7g_9;tk`yR8E10W{ zfos((i^b%}EL&Y*T`Q@OlE~N1t)Z$$vJNTTN@gOU^aTl=lNczHA-!5`R~yge^~g^T z+iOqwt*@u2r_uuQ_ve|KF7@S^Q`_%@V|PS^h<31u8_%{#2Q3bwIwPw2BjJZ8PPAk_ z_uku6#)_%mvN^w&lciRaJK%;4nGJxXXX~p}_rBvYI!h!h-ZZRX%ok3G!KkQ8#`CKA z(OuW1FTJkAXuN(SCzzoQ<%ik!UYD-A5>>-%9W^mlb2^>v#v=FF$MPb+{1yIs7sHi@ zrzfezZV{oObnbp!s=6qAU-!{>9leL;?|{$XI0^HTi&wfm96hh>%j2EJTYlKmdSZH? zhqM+&-Yp&4hJUBGv%VON(RJcGt457=j#_Zt6FYTcY{&T<*S+cFJy_`|_1bcUD=aA` z=b4F?mR5!^{K{R8oX5@)*fR%eKXm;o<}PRx4AslaI= zInWZIhy2L+{bi8HiNyE`k0*#eNn_(+3d%!ZE9E{R=p$1*xmE zL50$l#1ZBilouXkZiyEX2ceF?x+=T6J|9?sowCBLiz!eW7Gf4*COPfyuIk>dL3%`` zZYV)%!F@xusNrH%JvzZ9Z3N!MAS%2l-yFwWy4zS~KZBFRe@msE+}q9hMt8ubJ&$N+ zb4|r-aR~NQn$9@u`mTZjbwQqVO5EW}(rl6(0D;EgzcGK1@q0)F=|DGF$!jW~B$~LWvz!N`f+{Ye zet+cHW)fx^8L0f){cteXWyj8=K2xr?7dz(s=k0&nkl(3&==!!o5V-3-)cnW#`7aH` zg3hm|mOVwZWrv1r zBxIs!3f+dVwPEWS_5QP+x@*t6fkv6_Qx2?Ul*&92LF%_>T92I1Q&^9n$biMnNU4$7GtBz`6A$j(iZNNOUHs^ zXbXd2S2;In9HK;BQse~A>N|eJ8Pp?6KnL5uI_Px(Z_%1JKYxX(#X`v~LKLI0hQfbh zAi~%jRQA75P5ZmM#aW@1RkbG2^%AoKe!U=1<{pXp?bRk031)WWTRMI^ z=q2G`i}RU$6ga_M3oFEky<~1UF$)|hz=i1kWdHvCKX zD3GP+0CHHu39#%j)SJLp2+*RR^gl>(7tyT13rruMzKC$MXtW3h5f>pyWOaMk)W9Ch z3Qb4wY)AOukgBakvtFvr+WUJ8A&pENezFMtr}}6e!f;Q2wXuN9 zMmNvsh2Gn~)e#Uve{a%|ff;+7S|rKBCs&wmLtOVt@}z_=k?Fl-#$*bK^}nmf$kouw zC0ziY(MlWZDmH!kpcUaq$E^8>7}_D!IZ2U~D~S*jJmQgtTV}u>2s$Bmp2HFJ-nnh@ zR2(r#J7(taGNxWO8mg%|i{>apZVdpz&=@za@|Z&;cZ^Vz#1A4~P|{^;KNmJsIR~)j zWQeO|-3hst>Q0&8dT~EIE?hn!ZuuaL-VSL4kcX}k8)^K5B>i~*Lw%# zy#N3Imp!wOk#+1{w9E$CL>fjCnGFe1siv3pWpZQ_0Ri#jH~OqUeED(+}C3c^B6?O%w!{9XBH!mG;e5sc^XXxyOkVW&@9a8z5YNk}-CPI_|wD&rbW#oDc)QNmY;k6hb=|CXe1ro`Qf(2I2BM##{* zmM#2X>i@nVUPjF0XM{>@aAD%KwF{c6irv4vwo)2J1#7$%Lhman%_0MQ29C-YHZ5i* z$Z(Dv7wYs-rF{|Yq(e6eNy|C(R)Gmf&0w!l9{6<^>7bO^kSL<*z>PiqqDVFckt;>2 z5yLkZvCX%J>Pol1eg9q=#;A`+-Ks930T8JHi8MiWJ|D+WD0~d_HKi?x6H-am16mVd z_jQjkeNSXeD01EbMP?#cBn$yQU1`;i*5vAr z8$`zPaR3zx{^(=Fy~NHBm(yM>9@Kk6tqWxL;;>W8SG=`kw=VHQYElM+eJ1?!0ZJUn z=96LfYNqew`QMZb8{`fJ^nm+52QZX8%=*wEmHq3i0e($sve^P6r7FR03|)~K79 ze)6{8bo#>#A&{MJ#DD+kf~}Q(wMnT#mE*imcXd;UqeVm`P63WMWx~}9+S2VEhNfEe zqkg1aQnTN4C4Ftu>p&`9&L@XVsqJZMF4N~r4w>oKJx}AEkCFJ{K*i{keZ{a%N0r}B zs&#y$l40n+z2X^1r89tv7!K+-=@7}?;q=%`Z9W90O~c(NiXxU&rO2edi$4K%jcCCn zzdq0vg%YxDeO#-cd#KGk^{m@ZO!gr^A8MpXpzlg6xS({L|@Qyjh8?li6klo?@$j+H(C1_ zk6v0CkjQHHkk`<45>ylQh(6Y-sNUww(ZhA`eXafbac^IF%+Hx)r`f!X6Ho2;?py-a z=MzPV&M(cj%P4?&+FR(RehF^+o-#4sscL(@5v%C+R4VXoI9KK1Ve;;YH07Ds=<2`q%B29-59V)YhMBN8JXGD2swLYgpOpx z*`=tGwvm4B0EI8gAX>Q@g+XKp%n`8&yNe|=^A+|47!&hUnJOT*@Ayj*UPNEh*mbrSU(y#neD`V8qHt-IGK;&fz4ljk5XaU znuO90Bl&vbJwMK8|J?pO=cgPq#eqyw7Q8$3tOP^CXh``0BC~pJ+TPx@ zOO>--m@iqL(ul(3TNDPN4if>GH=|4TDWMKYkEI$!0!s!4#-y`%4-H#AQRo+<17uc_ zpEd^<=;9-ej75Rp<$Yt@2r;gynh6*8$4%?tR*zW#u4;ht@@-?-$iZqv12T77|pj6d-GWxG1# zA6-t2_TEsu4hzzg>Glo39_u(*<-PIyaJ>lYKk%t2(<=SyZs{n#;m{`MM|A3VVb)Sk zh{%@pT#bH@`FqSI{y+xWfK~)yAI{C0gqc*de9td?KaQZ$O=@2~l{6xYt~3IK=!qH% z!fZH&rD7n5z$@m%E%Z&Oi3v-+xk(MdCt) z+obDpR5fCykJsB(N2RyaI=I;k7Yp0R{FOgbjAF#OozFy!hirH#!Zgm=Ct~HAAAk zeZfTDcz5gE>eZVE=O4Y)sr2ciO7Cr&%KvZp#jWC%Y(a@aTzcr&#%Yzo6DkUt}{v(go$i%F;*YHsg9NGo|u6qnGkD!L3&teg10bSk(4R z&r!4mwy8DJe+Css7WbB`OOfPBQ+H7!Tc(cqxKYqH91~C0tuauAt}Fifb%qjZ@c#Y# zVc(vk`0yf^MpCLAWUL*M-0-0fd>M^Xb*^0*YhyLe$1gfHLPO326f+LbP&&>GgR2wJ zB9JjAgs=aV`?X$lCn8N;i!ax1ZQ5tR042$Q(ZNry>j48&=y7{;uq*<=GdZyw-3ZmH zU{7+sfp#c5VqsyFp8z3I@k5e&Q+{~*H&QGJ zp$6najOVOlSDcEnv$2uj!*}pEN-!^|m_=10#VXc?-jx)HSIru}DmMfS8!fB!vE z_WsS$%9t>bYluIUj{FB2ii}t>AyX*JK;eSGB_T}PTG)|^aOob_+D`$uqtmionGA=f zlF4vlYY@8~Hi~G8sbdyXl#H;F{!B59UiT zIkg`zdJ2i7;HGexDc@g39SWZ`ilY8z5#hdrYcL!r%H5b=kOl~fUbF-PNavqh_T0kB z$tga2MJ<(P04|+L10>eG`*eU#QBdZ#+b8Z9mt>OX{zx`{GPj29qf>F3Xud zW~=yj=yT@2PCSM#6M`~4DsR;=G1D^Q=}}pc={OZJXdM>{@_unDJ=$>IHO|HqcS>e;YLta7g7kjk*9675(V!(^9Gd)(V0wjXedL@hTVaJjpf@ z_XlM~hBVq*y;XYiXqqvlN7 zKD0TWyGmwr$+1z1)G}Xo8EauGB#|6BmiHL9PHQNzV=n=w{Jf&_-3|f==zDcyX1tP9 z)7#Me1)ncK_Wew*LjdfMZx{&9XTY-bxNKuHzjAdR@d%fs%7 z^Fv7^PhWyKnrN=U8OXp^Rg|qzJTXZ#K;!~%f=pZ>t{&B)HuG#mfF+U<5WRF(b%1xeSWd1)BY!7Md^(__l1|JAe8)ZXrFJbX(fpd&P2@m$P_GPKohdx-yY(tyQ{ z6%SckaOzg4m{ZxmWC=IU%LF-DR}CI>U6G)^$;)FF!D_CAus1fFa3A~aZ2!2gt$k$^ z4hi`nWs0I8{cy~fpDtM2I&v1Aym9Z<#5tFzUYAQz6*)ivNrZRcdG>*@KSgJ-SZdAY zk6Lqu3B?iR&QNR3Jbov1tavoA>O|P^cw`}1&YBf<$hUHq9ONLt-ySqN4XsoP!tj=s z{4ru6hXWd_vSfZ!GQOi1Em`t(?@cKoyz&Bnc8uQ;2FsN06XE?Vf)?Y>2=Kupl5|{rZ}6|HEqT zd0H6zodY(hmkg2{I`m2EZ)c7lpDb274s%y#RQVm05j3x1bAxwRoFZY&fy*btX3d%e zQTD+)K4pHAp`_jkz9>WC1Za&IEzcl77;tIU{p*j?jfwy$X(gL zTu>w!kB{l5ot&Csm=x{3vDnb1Yt8gT$KpfUwf)9DNOU}DGp=^^h>7kKstJQD?{B+k zwXgHwWXFUX1wYG=O7DqVgTmepE-U`-TJ#Mv@%#J%!t4Hkg8;G(a$+e9d9Os30^@vjQCnUz4+3m@sywVT~H9(%q;m8LljH zH$kJZo}G@U77WPvZ2%sDNq?6UIkJj4DbYqtqRPMJAw6E-|1J-Ea)rx%0E1*!%F+}a z>oDwtf&{upW%-W5`rU_M)74XKjab04 z@97Woj4KnU)CJg#%H#N1^XVU_L+!@=2&g7oKH6&c_x1yxsqlWNYP_`5VfW_GuGb1( z9KW;vgV4M#zE_&b6V4xBdTx;Po@KX|jfi}Z@x8+{)q|6BF3$}MZtAI|8Tn-`Y`YmK zJDkc3G0cG>Vcak)uTn=0oWRU)*sRG27j~=&8_cUX;Q-j}Sb^4*IGKkb15FI4Q%G`m zyc0)(IB>W|ogR~nj5zA(kbJ;cQzVLWCNDi1K%fVNFol)*1NO+{D$ z>DK8&CDfu~{A3U=-<+uzr%lI*I;tXKL=JP}JJR>d-Lc3BIlpxG!fTMJ4o&<8tE<6w zAXAi8i3Oa^qWPgqTokx%$ByX?h(sdT-+x2Q43#QFq6sKh<5#`2z$xxRizK|BHW@jJS*gSpYiWJh<*W7Ha&_nM^XLB<{*(QaTK)U=dwq9JtsL-L=Z>+N zKY#eAhPke0ouQ?+1p^6LA(&X6zV@;^(g-@;*?BI(p%qDH_2c~I56lZ_Um-n8pSh*X z>SXnU$M_j$GFob$RUGj~;&6$w-+{~;D!f`YBtati4srD-x<(m z=8tilYJv>`Fh;Rs_3qZ?%Tg3WY<0VjYv;r4h=c(8ZH_2K;1W71c||nB%7KrA4%lLQ z1kJn#eHlKNcgTPOh3sAQ+m4_914D+v}Sea-?%*CGf^(R3nAW+O=a$;No{t$3jkzsAXwq z7mKY--^ona(-yugg&VZF{p7cAn=meZtmnQ_|(WZx5O&f@vw) z%O&`iY)tP_HR2_8Mh?cw@qF5M?Eg~==jK;_l)p_v^56_D+w;Y@jf<#uFEE)VDgXyB zY@*Spl#Q3^1qnlTiiYwaf4OuF`DIdN2JG3h0PRRPdG&jqj_rNhvEti&Ravc(Q%v_{ zjqUy5&f2;1u=Cp(h0Fh79(SEWR??d zYHfG_&q~$L6KK*U*9`V5mPT=A-t03DVKQMrdXM_`Pj=~IZmd`ntb?bAt-F()Yzef( z46S*Nz1yJz!VdF}7n=bHAja_*%=sM4zJ4U{@qbAOWQi7jU&^-@8dmMofPVN1!;cbN zBLPmZy!hCurKL)g;PkcwlXN^7`|a}lg0)r~kAG_TX~$%(%6z%JH*8qQKwncD2pD?% zHCJA8!imASS1a>vA}j!C4R&H*4O{c+sPg5i(%W#Ul1_9mDj+cM+jwJ%J{RM6RBMJ? z!QQzY+Rz4^Qcl*O!N^Do2|E&(f=$NdQDw1jO^$`QcGR+aQe#A4)6^1|Yd>{-Tagwo zi@^MGb)2!!!#m#LpDMoTs2rM&*sb$nU$xax^Eb{;N8&sK8jTgt8{G~X5OrfPn5*#Y zr6uG6?+>3Kij?9=^!+^*kh?l&21V%?zVF0B22mN_l4(kDug64IR6{gSyX7$J9hww5JWS;f1v9#_GWMTpIpO;Tb<$hNp+U(Z zqTq7Yo(!XUvRBAzleNr7a?qT6Vbugk7~g^=BRq>uIFup?7ZM1TQmAsp*vx)ix?I)! z@(A2$mSR?sWpMUU^BS6+6#mH0$yxaBhbEBJw3pnNLy`^FyqrHQH{uHL>*i(2okQZE{l z?PN#6LvRGoaql@!DCg#h{NAo}o7hbqT!5ts-h<*o5>j zDgrTF$bb@o4c3f!^rj)^T#&}kGSPteGIQ<*v8y;lDq<4gqeG6nI|5;I;1vju09ZO8 zJFqg2n!{OM3T9EzA4=;Cdt7eAMG*kxtFRqc);`tg%nUj)4T#TV%NDsDd=+hxUeDum z|4aU-)23s`6SVSUz2k*l};wrP0X)eYgMGbdP;o z^(1>g|5tS-o+5PfkCpp3v{RWMocrNpxq>XN9)Q+PKx9@x6e)QT=-@}Qi99q*L7}vz zHj!G+S#Cu6?9%k}|K}*RqhI*|WJ7F*RI)|Q%&s5x_1!OuKPT2Rog?5qH|0;n;z~?2 z^5lil*_XX%kiDj$VRVi0VNPi;qs_Knz9B0-36UELDxgEcNAl$Tj-(Fly>naBngrNOUL_OldJWicQ2TJ`-tGF-4^ zyT32Yjk?(5(pc#%P3AB}A`KEoQlu=O18uS+h30)PsRa4kB^`r36_Lf!B>Egh7g^Tg z`amw>h0$;dHf-B3wQD*_hyzB~SU8cs`uf(46pM5}h!9)mlwu@zBA*7HHdavOibJ$~ zR1<+t^hww(7I9pIVF>(g=lf%|M99z-WW<)LfoiU+Vj}UZ=2;w?JL)4%pWM*2bZ_D{RP(kGpki26 zQKIM%&brJl3ouvCz0;3C(2unv%t7Zx$1Hz$-B<=+VR}nZ1<4AwxHF8?GGodDO-6W9mM*(%z)BYb^w|B)zOwp8bzoy+nK%TKC}jTQ^>-ipQyk zBdNR@QTvXs*b1HAWbtUmT`8Z4!0FXtZh6gt{A5DPEl*%fkWmEH$AlnDn)}iEvu)*= z*ujEe7*weYGzOD?gykZ7ju`ewZm27R?-hwSQ)(}0G*l&-T3ueMJ$(4ni0-R<0S(C9 zSaJ-)ko(W9bS+M#zdMHNajXQ|P)1fB+izXdWbI4$iz3P)r^Z?aVlH|C#UlR_zv*!i zHn0$oh{HY(7_jeY;n_ZB*Yn`kxU@ijt-imAXHf}=C8dt=#drVTvIB;)vQ(=VLmaRD zzi>rlITU*PsEHI@aS6|5w8#>+Z`aQtTS|nlLbEBWVJ}w)`v}~3?$^r`@KMVoX10T+ zVYhA@yh+=up7{BP6PN;w+n(G+XW79TrtC{0N~570jaFUe%T+669HL1Tm`N=ayJ_Re z*-B3K!GrhaBx~5Brl$*=9M9qlPEA?{j%?=ACaRh7b#9IPz-Gp#mQTm@y_@|n$S|qw zvn-1A-*xuSUa~TwtdXkn_Pb{~`x4G?RFbXU=P=v|{x6bhARs|}$G~o{bKs#hW`f9) zRK?iSp)3Ddc~X;?@oYs(h`RIHwab2L+t9j5aOJoVmQU0OgrCf~sD<%@ZJb8a_8D?_L=f;gM zs{9iuHS=YvlAu`#N0WWnHVTQI1d%davfT&7xuOCRC`;sl zoD0D%YO3a;N3GT)lhu z1}(LMBmweDs=3Y7hKFcyl?X3NKhdsG2v@CyDnz*xh6H8-ddp8$y$Aox4Y`Ey&_A|{!gN3hOW_gVgL9A0@AnJZh&dD7 zlu;Wa*UoFZS&}g0qrEpf%p?dkC{=4z<*JW$i>pdJ##?5M>D_aS?pJx$OZ8?;UudFn z=dSjQgy6dOIUD|d-f2$nlXfc!c?Cvk&vzwW0`wLA-6Um?idE*ZVC_=nZ;-rBhBNQz zhsp}zgl-bNeZNvm5=${1QpeEVZybMLd@8~`@z@@Lxqp(NM^!~!Y3eXD;<-O z{DmVCIqyOAt>GeAM zgi?Ud%oCFnr>1DCB(jLrFoPZ_e7e{dcW*9$^gs$rSJNA0^v?@7TaW4O!6~W(dzkZ#hut3 zBTl0R1__lxP5K#h+^A2Eaw9!uh;Z>#YZn8lD2GidI>G4_25M%L4=nzW7jYS1J3VB{ zp|#T$se*)if<~!jKYj~gDT)Sv2xG$CaiHPMc9 ztqgVTR;M)6m|D`z*KBFak=FoF}#V?l0_8o4K5d z#EWRxITC!qqu_e(bidcdU~E%vhlLgk8So@tXVaE_0(efuQx*N9a}aJ9dzOv z($l2H3v4z=nbrQfK~`|F(GKIe^2#|W&*GLBI8EC`>6p~dwomkf9}T}kRPjfubaCs+ zh|hpVLq#})CgtJidcfym>I%s7q9?`|$zSx@s+JTk6Y6RAwybG!47e$v%vO3o&JlmR z)Mt^?x=AS}Dv!^Kj^T10WWQ`WY98GTWrnMzFT@tRMcPb>h!R?N(T){s=pUC-`4QqFlG6iD+hu(HvK`S?8Xv!2`9ilZ><~rH z>Gxggl03lN4^6U8bHBTA9F7F?mb4=ny?%oRoyyV|+lY7tc&KSJC9eKKV&c;dq}WSi zjf*O|0(YdC<+$z`s>!0LDcBHcNCYdLS7#y6;)aI};9N<*cTazN%=Z!R?#UzuB`FA< z(qlGy1g;7WfBiT(ZcQTkqlB=L2a5(y!Ik_v(%}k{E@eUG*Q&zmSUb1t%=XdS8 zw6%D?-4<&qkP`9}qJFYhZEujO+aM#mvnuJUVVch5ISbZ{{Fq-m|N8hwAy6v<|6H@^ zuo(z~ey0g+LKWz{p2 z%63QZf6*+fII$g|OE7hFVmpn;bQH2CuN~9n(sjO~ zL+k_F2^6@xGiUzzXoL2#S-)jIRYE|{;U=nBz1e8CyIm6x(Dv$+SddCx&IGNb!BxFS2?TwmEjmAlxRM2 zSW*otR`vI(1P)+P4t8Scy=uWtpsHBF9+^Q`J-OPSdJV9eVW8OQ(){P1*fS|nTa^x z5uKbr-o#|PBvd1C6We0U@w?@=>nKnaJu>z&5P9x>AIpp-F`dpXL1Ie@DCLKg(_CLc z!V_?C|5p)cKAhEdg06iQXX)`TMF-&1prAhs$w{ZLXTruX+0X{$8ZCt5-+TKsY$hvt_3ww)b#jvr?mN~QbXoe zxyo2>>ewZgeM9qD8NjA}(W+S)6EdX)2!q(ElA2I94Srr^j(7t+t$_0SAW5ZT=rn}=r&hJReDb8Zz#zgY0fYE;ly;UR zaLYmzFaIG!awhHGy?gR&Dwc;2+MSi@5i%i#MNq&&dQvZ`awiplWY$U}4b3p`!g-K$ z*PrC@BKYJjRD?nJ$0jH~ABtj^e{3l;I$)|hUHF0^Rg4abA{jtSZ)EM_+amFos~nh$ z<}KhG3s&@pEX7ZE#t!p){y*}AvRIDkzBKeo7LHmLGK&NBpZUwlS(Uc-m&2b#{-xj6BOk|~8K zq`gQ<|IIT^YC=(g{EU>c=n$PA zi~St9Esbn60i@b(o_1?O4OOI}q)KdV1T4!E&_dDS_5Rc?Iin?Y5aH*adfm%+VlSY= z8!dgAc;-n1OnClb*p$Xevx#+>Rdo4l2&AS}p)=iz;hBYZS|Eh`xTa2I52Vf!GHT9j zf)C&Cn?`Wk0@zgYBbqb zw31Q*4w}C)_lH|`1<0{}>KIC`P80Z>rO-+klaj0wSG?XV&U#5P7^q4*6rHtjh@M{h zrwV^sh7A7w_+x;C8_!%rLV4?I&Y;x!HNVKH9YXjF8cmI6CGcl=^s<@K)>X-?w zpTU+m_Sav3UHEO2S6AU>^6u(Iha8E!;8B0P#i5&5y@qv>o$qvqD2_2Mw?8vC!s61?aC)9^fDxpmE9Vwrd2@pJwC60Z2OrUTPLo%F>#n+c?oX5@t zUvX!-e7ipoOkK`ME++Y+e2?aP^vYOk_gNp>hv~PkC+kzQz_EP#+R!Lht~X9GGg^vR zN3KZlRe3gVTC{9w_8RyhZ3rAD`m?y?@sZ=INtXBiFZ!(C9G3s$=af0G?Y;CHKGJPq zFL(R(5XW{A9QD-+1Nt}BBTtfopKGm%3O(S@Q1of7_Flw!v`DQkHu;I&1{M~82xmz& zNMiW0V+DZM3TL?b1W0Yb*V+R7;r&$mF=yYx)Q0mS9a6Sle`WfB+JDS6Q zn~zfy+qY)Ry~95y)>;Xj5bKfox#){ao9^t{zyD-t=|?ip8Lr3X%gidGM1SErI3m;l z%LN6vG88n@DUxzv3al3Snr4mABs%#B93IeYLjH8l9cfrB zPk>4sTKlD)N9`nPf-qJbBijD9GG-l0Z0*$E{7}{EXkd`0KaO`+>mBaATlHu~-}E+$ z2#fOs0^J$SNWE&B*fhttbZM)j=tl3@XCADbWSl$j=!z59GIR}YgZA;u{7AK6;L?*D zxXin${AOHCDp39?eG+`NfbU#Kivz!7vJlKh^K}i?1C!ryk?O^gdjkBAp^O(g^bnX_ zDe=&EpX{Tdxt_Z-pTqE>fGk3BWyzd-x#i$I`RDN+?BFkq^QL66DNAq+}sfh8tGXfmXiS}GH(jQ=*+y=uV0HgNaFnZ)!)(HaSqV)QJ%fKAYF_VxpCVR5;NRwmjd8X%b*;T8Wz9q|^I zj&Hzc&D6#@D$Syd5rA@>okwHi4TYY%l+vWag)g6eAcRtDezd@BfYCB$NuGN~RCz*N zmxImQNy7F0P&fezG4b)QeW*Fu1p<~kF9)8pm7tN7tMavKj0UKTIkA%h1z>9R0rCSd zq5+0Xj%OB>_q9;F$P0sS>vk*CrBl-jciC9+?u&-!36+0XAD3?FJ$gQ!%WrHXSu<(DU)1^*fYmZbv~@6VzSK!oMkJy3tUD43lEXD{FD!ss6q7m7Cg z7yeOBOTIcqGb(W-!esSWuPrh1o*xg3gJkhNO|Jc@tko=`{OrU;P{0T&a92L$644kb zm~~r+D>Hiuw6Axi*Z)?AOrbOIeVZkWG9O+smqHJ}!88ljtYM$hYb%D(GBjvzc0KkdJ>VD2ojFZ&HGGAs3&l{d`~qW5c; z#kFEm%e?f@#O^W#n0I)#3lUP$-^&gJF3XJmpn>Vj&7iSwwApCC#jCbhojfE!y|3k# zLt*bac1`cuHL3cE_dCB&)ZdqP5_f_dL5VUZQiJNKB z2cV=!i2JN&&2ki=&U`<8qFVW5NIYH93zr35`8Q?jl>?pQT8I@!>rLX&<<>qXjTWMs zC}{WTg%=>xB#;9IZ5n+L2jg_5S~F(NNR+;;+~CwQEyC3K9w(v0J1xdzKyAFb(=VPy zbq(~DCPlQknc$6V_ckp%DVH99A5`OztyV!vsX(bX$^;X-Gw}S~W+zR7Y$aM|@}Cij zS79vC2}#Q;b2;t9it%1Zh-=kzaWBHCCyv(FkHe4YTFsj!Uf-|E3|M8 zPQ}ScOX@aE%Zs898G27=bz>DVZ=^2CMq3t&4D4T7wkmxSi-^8=Qh*n|$ibPk1RGhVSh}@1kcfJ z8vR|kL!~l$Y0)K)cg|BjMEi)+!x<7b;Cd@ci!`Z_1zh4g^1INCkB-Pi$+QsBqM>^0 zBR7#0(rV*SaRZbsiIwFsV#SlVQm*^s?lRk6j#e=a0s4_ns!+V?Qw6yZCD7I)_8;)6 zm^MnqBOsBUb(>X*6iyN&0<`DG(rQKW>%+C0k=9VBB4Zckgsc*qyl&K*piOc~1735? z7k9S)+LW_FPU(rS=fDd{+9cHT3mt)l#i51(K~WJx&Au4G*bgBErG0}(vbvZ@VJH_# zDN5OVn#$D&q~2wFwN*;FltXh}bjuN*(+_Scf$z+5{rL6kWA3C3WdIy2{|u-Vjws2S zJnCy4V{XlnqcPz<(#~igVI>hx?s8=sT;I^eeE0A~{knF&W@WrQNA=y1ALqAIL<8je z6ga)%&bPV+9&c@^doVaSMj}$6>$pQ@Fp=2B*ncj<)&#ZL0<62Q30&?adT->0yHg4l6R%nxJ+ z8HHws9NKu-Kvm0tB$3UwPcs;tni<1oE`Uwr>jN{+5lJG5FaD5OQbdEnyvuf@vgd~t zumKHv%3W`s#@ZD-I_5`4yfFEF-~P1kwdDqe+XTv0%Y#SCv|roRO`Df>gS)-=wpmnw zeI=PvegkYae{&w|KYr7<;S4Y51y%7orW)}-N!@yTtXWmMuC(a^N`Mf2LVZ0RcA!EP z2%BC1KRqW&fzosC-@gREi^}||C}gl(S{8D3D$U$h{!LCJ(2V(6&fNEc;%wiVW5yeo zfDiHFV;S*%Te+}W<0j7-Ma|LN7F}46u2QC+qM3$_TskqWm@1dv+W~j1k{m+VmXaKD z0ZN!je;oK3-bI&>!2?8*D6petEl(>dMVrKRKTO-tB{1~gL^+o=Mz#Fb@?HmQ4^l!N zEvqP5BnDoRYja<@BGVAt8lATY#YC`kMbs-nL){`IREwr$b{_Yf_#{v=%6NqLz4)M^ zR~taAxTP2#WBBGsz_WmFpot^t*Yv{petiQu8)UHRZ*A=ZVXhuvZZDG+yRoc4nGD<%j)z zOsIT*nfEIDEDLMI`s6XWD?i!}Ri)S8|0X*wM*dJ$ByrnrD%XL1K-PJUK}MC@U-TkL5?k%0PEwlxIr?`t3^V$Xm#nWcZuN ze7L;>)=)A4Elot$bIx{M!z6MPW@@(HFAZJ9x__%ZhA1Z6vPY!agLPogo^s=VSBLQk zI%N3unHyEXpT}`~KVbXC&%e=-48*C*HWu{seHxKC0|@otnm=_Htl5;^`-lVRnj50= z9PRcr_bi$v-XvFJCi+aE?7Z7WMZ3Ksb=F+9I5nU|w0 z={R(wK~6-Fr4yU4o&l-Se@EJsoTg^+SYIOfyh@tHoSUNz6>E^&;tTdM7skz$UEi;w z!{(3)_0zVLnXk}ka`I3Y$I(8K{KlTNtG@f>!D=>92g|BlKYQCN$NsV63p6I&dNSYX z|6yrV#Y<8OLi-3C8f;|@XIK77b;$>#Vw=C-iDYDO{?-LYAbh){Kt5 z>J|@#vUwnlBl_8>!-ucw9KtR!!*0zOxZl^hjm>u|6O5Za4R(m+$`{rEOeC7!+FN=(uN9~x(*d332fXoSCrA=E<;iV?8V`y&`)%XDEjJ~_y*CQRASe1# z>X=$72d?$vnEQ8z4R25T%};XlYVYS#M?m$rRS9(j>@A}~ds!NiYrpK=MXlFiA<|4{#ku;t za;W{w&79t99h?%(at&F2n9U@$sw!RF;0W#+ zS-$FRvwI@=D87w&e!zau1L9-Ui`+@N@flLABrpzG&5CChUm!A2iSpoOW}pi@&n@aV z?%(jh6$v(mk{$wrHKi)0s-)^I)m7#7#qB^k)7YRSh5VR=NXi_hNDC! zLpJk7ekXhWOG%inHTS`=t|osDAKS;^HWJU!tB$SyNq@g%np4O7Ypi*FmBt?RtXjWw zefoR!iM#6phF9EY&7x&L?Yme_=#~W|OB82>q_8WEE2+REyE#J36aK75YtaIOzNzD_ zUZAKKlem+xA+}96Zti7lJXc0 zVh2l2t6tRJQpM4|$rxe0K<~ned_^6ej;I-FDkJ680b&oGs-Aia!VgMM zO}8m!kl+6L4H^vEj+)M8qJ=My0>|wrIjdQ4{YyB#6|oJ6xm*E$Rpj}_z6tT3B7$zJ zPU=7MKh_psttU7l1mN?!((*0E)Z`}W4oNt=%cpXLN?=66IdHw3Y+c`2&Syo|dF|Td zQY>jA>trzlg!*j`17@aB!>W^aU1OG7$hYjrlSB*Q z8OGN&>x(s6M&Gi%Cjw8)aGEq9LWSnsFg0x5qQzl}eIB4dCMRcu0zs#lRjc@BjIK9K zzuzE>!mbrnK7;%;9~yNv(QVlcW6&8TQ%IHhFTDG7;q$mFcN&m?k#Mu1pDY#6@c$pj zqgH2-je4v_vHC@1QSj~y4t7@5I-VEl)PsUf$>|8GV9;~@LD6BdTP3o+oDJS7EAyE6 z;ZR9SX(=OFS2NRImEEd+`xxPeaeE|3=#_Ot;0x6w)^q`gxQY9r^vj9|$W8q5C0O#x zWk$`o9UZ12-R54<+#$oF=rP6Ir}Xvgw`rI_sUVN=Dd-10t*o3>c<#!?mGIG1h2 zuMJBAfBwH~z{n|(2QuAKG*?vOaqjnkm}Tq$IPN4KYIO_fk6c+)9t!?kv&TDl$p_;) zuO00M%~rgh>6TUk6W=^0W1PUpS5de_-T4*8lqZmc@ag@ipJXx`Kx%lG5>f8S>~eTJ zi7s(^ejz@}oqX1ghF<9*Ch|Bs-I3fTYTiI6N}W_VP3LI2P;QbW5g(%Qg8t#E0s%=7<9fk8@l{G2 zctr!dxpE67d85~m;qvCEuL&OUsI!%3EkIFHAF}d%^Bda2RXQs5yqDhrm-r@}+H~bNI9;?FnnW$|VWDZXV4GPD zPk$8licNTaBHbb~?C`9|7xC(g%cxe}#vEx%s)O$s8o$qAna&95fBo&ZGj~aw>1gCL zQ}{;73y_Hs7ScVfj-#qk9I0nroC7beDTxrvIuz2TlHGijQ<+F4R3{5(aE25zEd1%; z=d@>M#bjAAly+>BdIM%fivvz_!=jk^TEKN~(7mQp0I4Suax0@?i&$!?WzMO`RVQ}i zOl8OED;aW<7$e~{>X_MQ2}dqnx@42GrF0XSY4bX&rW8mU8b;7@;9pV1?O?ZGGPh1oSRzZT0gHW-6!L&U{5D@5(98PtTq2 z#U(HvhrO6&EDNy#D05OkJye(Gh*{hRd!*JUho2?5o6AbEd@!cMeS3AW1yUpWFG=MP z%SY(TNUtZIyw|oy;>5UWUZ&yDJ~&`C1|O;|q}y{!u zkj6yGD`uF3PBt`yROXWSi9oMQ7tzShGF!3Yl-Y#$%)K~+M@dwAfO!Gmh{hH1NI=hR zR7L9`?3p?DKyDpeA4u(N9^qDa3(lxujsJ_VpDpthWlSP5x|3Rd7e2EzV+IGxf(_5Q z$&EHu?4HO@So*>Fu-6n5>2T{y;cAQRty8*MyA1oSO4^A3sP#bkl%emaHa7bE%kql| zBAZ1}S7{y;&fa$uYVj|Zzl$dXA}qNaO?h(QPGr0`_v#lZRTaW#0_Sf>=$)KRkL`Fs zj^!3*(FVgT0x(&LL!R~JrA2&eKDXX% z0th*bu<5P1zeK#j{xDuu2zy(K(}8-8yj+Sfh+r3KpVW3dF|IgY(gEup8nl=5a;2W1SE=T8l6PDli0xV)UV|@pjk{ z;^(I#FV9@40M<15Fp6-);k@tw*(Lldn(ON~N0;>iy(#4<%7r1P13amX%bpbzNUnTH zhDE7ps#qbGix>ASy7i0kawNQ>4~K3xjjuW(Bj!cA%e%?_)=DFn^2or?JXsYDSTEv_Z$sf)z1shih^{G#&J8 zF15U&7G)bnw;;sPaH6+l5U+Ydd096RdGW!`;=xJ;ukiI8i^TMy-W728iIXNd_vm)X zpTkzUOT`f({ehjmon5CC@6hQ(8k_zI-sIo~7Wh3)r$gDrYd<;$J?vt^Z{EJzZmq0@ zlk?8W{a@Ce6yZ!Xd_GzFc1x6sV9DqAPb}F9jC4K}6|!$%xU{)4Rcj&Zr>9|M*=r12 z1{pNn_v}RwI%2m%3L!5C7Tk@3e2wOoUYw;RLWjRSArW|y7(rfQg5a0qamWb4m zo>O&c6PLRral_~*KSQR)8@!(|8OmUgD{{!d=J~fLPYJqe>c#JrpHF zpp$iQlOtZD#vHU@v#iikVll<=AcoK>#D6el!!?>pRF*5{19;UG!6_B(Fuil&-4%Wd<@9g6w^pDuE^qfw z@;Cl(TO;8CQrb!YwLXxwmZHoT`!1uy)bVn`Ft^EiMzF>TK)+!e#&dQJR+5}FZYuZm?LfTQ+ zWFP1;PW<(WtF8{_69Y6y^0_2AK#vqPr|gzcSqj0dRvX4`gIqwaTNuB&725Zh1F=oV8AgWq_}ZW|P-1S$lf8nUmi<7jL67 ztKuhfH+(!27^5ChBKY_Ykz7XBlgSU1)PYW1XsNvX$Nj!zq0h};b7<0uJsxl58HdUs zEfWO|Kg}Tgm*>(pzV?6aUn{*k10@X&#+M8q6E$aJ;`0@LUdQ;~JdYg~Uo(G}prH=( zz$3qQ@V!FOt71q%pQu(lx2sGFS5CSx+d{JD_+_HutPXG=;6ISZ``^Y!mHt&Sj5PKC zwt}J)>zg`r-j#PU*izSnlxh&93<`kZNJ3=@wwQFpiH4A>2?@UJq9W_kH8<&9HD^_t$%Xh|LO>Bo$|-T5N?p2YOCm8Wm9!b5t5p+H zVZ4_m{HcYh3zg0H*@p_je9c~SWN?mTj34V62Tqdw#JKXZ!};hJ+wZk=Ob;(s9qV~+ z`-49Mm`-!rL!J&VK_wrr(ZiKK_ii|)b-vW${D1{Re6ZGQJ8fw5>rrCe; zsAa~4GPAIc@2Y14_WFGG5)-CSbjSDVBE^A;GvE>HoKofI%e>BjsE$B9CgZ&B=Mc z$k5GXTcQn&cu#;#FEZdDoX&Qlxd)Yi}0 zf+yc3UpwmTAf5ML(Y%XGQ`P_{9Z5VEg%>!wFul$Zk1qwc*&!$$lOGE(x1PSN$cU1f zRDJW9M>Fm9di!9#`G3`?4mDi$Lm~$tSl?2K%ivK*lNxbC!wE>q_nu*=)ehIK^lcPe z7E~S@4QgaY<>x+X`n1uizh3sP>a;=8Zqq+TUsy8pc{__xt&R1D=qI>rl=?tq zWgSBA*Z$s46Bn4ok3GwFRNn^06Q8}6W4~g|xnG%2#v-U0caHxic2(p8h;ov9bl)u_ zL`d~KByw8MD9$qJovemasV#eU-d9PPsx~&cxb4(j6 z#V9B*c*CN$wp&2*&k3HieVJw~j#&}j%k&v>%%dh4vC4&G7y-{QB|8LIlE|~aza|!I z0xLsMrGod&E?i>H%}2*8ogg=K+;=}!zIyzOq7&7B&Cj2KW_#WD!okV5)UC>cM)UFh z9oL}0pnBp2Le~?+)uDt+VFGdh02j-q#t->A{*)_QWP|MPfCSEZCG8BDu@YRB}Vu_IsA?(7rTCCcN-&ja3fIPu}sNHvJ)=bKcm5N~gF!SKS2SErd;rB~l$ z(wyj3Zkb9Q;1fv8Ncf{e!qJhtrThc9qy&-pepQ7f!yTsyfp!*|lvX=R#iZ={Z8wXm zNG4uKoj#o!Vp0|d_A5x67(iE}PV}#I#mW{b@jY_3q(x}7;SR{z6m8H7WYD0^asA~u z&M`#DJz9MgGq&-3Ry@T!x6nfXc|~kaaSi1b^ZKu(x%)|ZrgIU4zj)3TUEvwY=`6h< zc!h|Hv=tK&mnKQCx}ba5)>7f~k4tG-v0$fR1h-8jhq}7^zX>tcv!_y8(izg)4saGh z3`3Me5W~bW-IQPD^Y%g?T|jFo9NxtI^{Y`o^e!&gJGRHhKDSexQ;ovY4m+&o`7EDGqpL)Lzog;Mmdf1Qs9H`76RhRLRuAq%4Dpg&e?Bs_6R zcTYQyD^Iu~L;FxEUul7Bgu>rrT_M==7&a) z8|O^(~SH28rs7>v4tDV!rF zwQQ);u%x?kw54Js=?R?fzbZz|SKPR1Kmmm1x+;Sw?(vcFiyo2O8>9#sR}P&={v+26 zGdRG_6wFBEd69jD4>_3@;6=;vQ3$F~`z51+^*?5gmOHn*0}59v7dBaRza{ z10s~js%Yn~k`-XuBD)3FTP&2~PE(8o|B4UGn1GFy895~CG~R&RN9x<7t?;l!b*AZV zL?8@+PaylZamG)TH6A46&8prmx%QprAfZ+dg~vO|G^|9tKh00%z05+gXF%J|EN;abtU4?9;a>DyTMbKmumR%2Q8Trbrw0FJH{aB%*-?+p?dD|_H647!zg_L- zL-wY8Fa4A|BKN?zyUX*E4tyGc`HZ8&}wo(l( zU@9AO`r+MMJiYqR&CTr;nfL-ZxmOuygn`gL!_B8tqb=$c&ae|!J@ zVjE<0UBkOZcMb^&X^VqC^!~ho^LU2?+cxI6l^^@0f9poTireW0M*bj8p;@a|R@y1e zTet4n+jwnxE3eB}>>iojlX%b21I)LxXwf1VRxQ-FG_&ijKf1SQaKUTX{0Y50K7V%K zSh%ljwArEBCR2;vJY6wk>bQ$v_o8Oz*R-rzy}31o@HdEn@^wK(?W+oj&8JR)Gu_f4f6@kUk(AbtA+A{ZrGeXMt`xL8@f02l zApp+seJ?=S_{UJUo`~N^j~_qU3}yJ9&}X)FBOkq6m3h{sJs5$j?#ok?dtT>h4dUz? zGJN>9y}!$|?7C#hlF0{#GzZ-n1Euxci0*V|ywgtrj5kXBSiPguU-fR)@VAb`TO)ox zJM?UQ6`sglWC20tThrWM$uIb`GXKh@_pUudaurX$e z&Sck~-MiPunAP0<9NE43`KDh+u@7tpqG-Ce1_{}J73_;IhLyCVB6-#ckg zSCb82yFP1{7u{-jlW8Y5CG`Dh*K6=^^&ccn(>k%HhVoOodbw1bWgPEkKrC#dndlEQ-rgu3=7mWE358P2FhyK!|5Zizqtodwg`5& z?B(m+vL)yUTXe{fA@%dAbGuCWV|vp3Q#rsBGtVvSC@RU|kq6fuHt+p_cGIWqrGMMO z%Q`%J_DpkdZ`P825IJQdhZ-7oY`tyFQN48ANUOe#9cyn~^({r78t>Gk>#oegm<*>C zcv^fI#FKepb-zx=>+ZVzruo~A`BzzP#ZYIjAmZG=bA`WR5UUCQtlP0}hx0~-TQvCF z>$2U^?(tvCJ}x+Jo6th$-Z++g`g9HP9&hQ8lkeP?yphX&I(P2Oxm%BF{OmpbMaBuM z3tG!K5e7X73B9YO@)hYHC!wnU?6ZvlP7PczQ&Fk8UH+SUGpDd{ zKqNIlLmXM%ER;ufIDlC}8i2BG(jE0QCkN_;+Z8#Q_in{Z<0GNCexhDHntIO?MRuL1 zflro>C1hsH*75y0an`TQSg>uh+1C9NcSLoZQ|$gnznc0hPrYtt9dmH(FCN!y2gM(0 zbg4ta3H77b?;e#|L;W~6tZ%p{vpE~(rw<-mmEW>C#!7rea`h(JWk$xvoq^xRK=qBk z!=JxhJ)wIG>~R+67!x=x^a3?LMZFv>eiY0fjln2=Z|oo^XTKo)0{s|dHAq7 zviZMImuw_S;M$WX3A^^lNwJ_&4w}^Z5u_J|C&Rc}#}; z_sY+M-$7X^&!4;Hz;6fgtyioVM8?dofEHmA{{8&%<2G{j$d{Xw^Yf@tU7fU|?H(UZ zdpCNbh3=7uTL5Y116I0!U5jDn8n6iAUcE*Qzxcd@D=nd^Zbf#ai_X;E%(O}H`M-xF?k_zRdUAc6t<9S2M_g;z^6n*-zxqvBqbR|zjnOgJ)byjnJy7x+y9nv*=3W}RLxwIAFTU%O2Y#J1NPx_kav9?>m zDk!S1FEj4dt5@Up28}zf{oGtwSl#A>dA)dBEiJ9(?(VJAKl*gc*c21f50v27xs{*x z|6$;w36wpuvW&f6uelz3%%rEf5+Zc+R(O-wu4RWwooAbwyWU08SC3 zmk$C%Vx;j=5QoQz_3CrDd<`#onRa-Yq@`IcQegN{``Tt2#%5*Q=fz7FMt5}Ai(mO{E5>oiO1F#K>>dIoo~dtF4!m$Bqof1}Vyk~FOO>zx^kNhP-Kk)$MZZj zOHN0@QP0@8`}`z}fa`r_ex+mVJ2v|ko-?~G*%|pT?BSb%K98zvdQM)(1XhMDy8!1*^K_Lqc)fgpdaA?ygBz+)y z$xq0i&`R=1%R>*-_^c5+^=PBS>SveQ56*FFdYYw)DA6B^@9}+iz5=wy>Q0Bk;Sg$1 zmnZU-N#{V)ak000CaCU^&G-^;2Yu(Ps$c{WRO-DjU4to^__CE!wcE!n_i@=FIOdr_ z4g>P8%Yksp<|!86Lq-DPvuGf^Q^J)!C*6)8r|X6*Zf2~t{^D9O{)AB(Sy}q`a2%F6w?%Jvv;28n-8YKGU=ty{%$-xDg3(h^inwtW@USOCr8AkmTNuA>ok z|9%jpbwPj6>XZw|)zsdAM5b~89?h}uvWAbs(^sz~A*`f!fZ}l7oO_cz6%`fTdq4#z zM@zO@Y<>Rxxpb>2G&sSL{v)$w>Cf4g^?#h;XjP{?McXZKGPI^q)8B40@aRyeQrkx4J~A7(~>}j&0+&xSO}~U%%6Z&JFjYu8pBzAF3h%Bd9k{!dsCc z`18gA;oc64+g<|Za$ z8+P2_>g41U!mG$rI$tG5cAoIYTPo`0>ybuVJ{LIZlbI<7<}P5h@07gAR8(A?J|6Sv zQ5Z&^fl05G132pZanY8Fnrhw23m1047H|u98p3>6nQWAyYYsU>wB#k`&aSSb=sWA~ zt0u8Vo)lzbqVYL?#rL`#H#vN8ptY$P?v8jT4r%;{*aPfXsr);(V=KH&(oL5@s9O>E zk`YjV#>UecFc}2GgBBbK)dBu$7kGLOL0}R8@!dOm%>1N!|<-UaP+sXmH@Jsvi!N zEpY32yJFJA8xjCtO*Vn@j2xJACcndcDWCzOk6&5+__8D!$<&k&$2!oAK!9kem>dv+ zGzAELRP7zN2Hy>14I?davbc5PmdXyluK#q|%sQ!=Mn1tjs%~HU1gw?Y#jE zm-Kp&Uy9;fO6S(oe%jcF=RS`RwHHXFl6RlKP3iq_859xIVBXLZmQTD!=ewQ^tRtV3 z7`Jbo4+$JC>sBsD6Y9JDFdQKjr23Pn8|kZ8SC0iWuwP5U@%bO@K6UC8ISEL~f}sEV zT)xvJ4`J7~%3Y^J>G@v|8uEYjpk#PKTJ(o&W_U2?5s4Roq?U@VE+Y*DRQm&}=Y!ki zp{cSmf0~@ji^9SK=l9+}e#KxttVL?Ea?Q!hq@tp6qzeih4bVz`Lk8G60djPJw~={F z9&O-*R6G}$W+^e{XedI+fq82G{Lg@Z0LJ&=AFwv9r-qY~YrDF-zIGEZg>)S#`#8O7 zID#IqVa?g@r!8XpdP|>Alzg>hiYz4imU#CN@+p7hP<@2tD|E9LLHkXkc$Y-F`p z;7|WwE1KkO|G#BHhwVWoKa$bV=m-nm)&q_qK9=}eDLd@55kTAVqO!7_c-E{tR~MSp zev2F~Kz-f?K$L?clYf80S?w^ok*KY=Dtd0kT6SE7HGu~5+s$Ci{W0ep78%MABFI=vJ$6E^m|{6d4KN{qCSiIdF5qE?Q3T4SbL&gpd*h;r{TO!HkEx=~XBrWQpLK z(2i_37ngI(e||@cpIfUfa+i)iC?yFX8uua14?RjMi7YsD%2ARYH(OY@<9bK}L*Qt* zwF+R2%%b!&qp4nfD}s776tBL7;8@cSr^QjvqP6YN8bhCM&PPhcC)G^i|s`1#n(cGiAoGJ}jzNYy; z`C1idb)3Q%W2;T{u8V$o3B|8Ie=RW}4Fo?w>-uA%g5(!zN~z_$9jW*ebWhYSiLJg~ zmL(rd<4BSq6K2e7@O|dMHpOFpEBuSkxpS-|S8N|wXgqEreNh2Ny?px}FoJLYJMG-s$F&Z2psUvuNVA3W$QEr9Y2>9!aHP07I|oKSaUd7k)OnZ~ zWNgL~QGz$;cxqs`Az}wuI5~Hw`~&C?VkqUm$8}SYVX^BbK*XDxpgcAM{2O{t3F`r> zC=HN5O_%FWokXhXjrb34$!CR7`!EGbs3d%S9%^tC6YGoJpSe9ebTn#ZpKokd+Y=R^ znaNPg-$lmO_7s<^BCGFS^YnK10~F+!nx~&i*=i~C^Gn#S{DX~)f5b30a>&|mZ?cfW zZWO#+Xg5HEsR@eLYI+A!(}aQQ@g4Cug*vgN0Jk%A7C11%YTi#xs!zg20pK@Wp)>{= zEe1|_7G%VIqkZxpOqjSHFC4X`^72024lx-S4k9Jz40%M>mK?N6$N%}~)jbnWwWixo zw045a@Wle+6BAp1=DA@xdbIEAdgu1uw*^|;S>(b7%Z?JFE|G~`b?xmQJ#}?vU{@yN z2U9Wa5!A5m5Zc#PWRMYCTV8t7D^M578-X-vt-2M8RE zQ4>6Z4Zy<99SZ!4UqXUKFyh3gr+IlC;B+dm03^T}Umy9T^dx)rG0j$z8JS(In)C=z znJUGge4!pSpnD=yn7wSkM(jO{@vYq41e5vRna@U0l$Nu*HxrZ-<#VJplXp9-y3BFq zJ!hNW`MA{S4g8B|52Oc{l{^3bYnD;}@nbcXyacYkJ)WdBy_U84jxv8aC@LUMH&H_@ zmvCl%D)Dgh_4mIOO`zMrlq!Sem!X$#G%O^DY24T4pN#KPZF^@GwplhEb9Lj5sPwgq(#V9Ih7%-%5-L ze3&}@=JGjMXlP#DZ*|cc00LP7&~Q54WQhSK z^^la5%tB6)`=99=86+as4GJ_B0fK;qaV?}qQn0sMwd;s%!Yc7~>UIfWtJ|RIqmioZ z{n9w|YW6U~#VVmqvIBP)V}1w~35ZDbL)mNb4VSt0tPZh9u3f*5tO|jtpK&W&0bV*T zVy^B7sa%PVdM75P<1PJa$2J;u_;DG47I8T23M2Y)wH^y@I9j(dM_PW08-}7aA)Jz* z03FA?#%CwsCss25cFeh3HwRaJ( z=u(#6ZCrU{#?N}Bc|Z@x98?edNqiR0dsC@mJGsU8H?5mIn)f&$N+43}L>;46a8jh| zr^;Fu{N_2Elr3`k2M$$R6FLBEQ~!2Lryu#%8*#5?zT?}y{FHy3VdD8y+-7I+%wfzk zyQH4=or!h(p3RKeD$EbNIhHSVI{tZGBS-$-de+70yNq%Z)#cEO-JjI{t~p1C$!=~q zGJFk1*_#OX59<2}Zj$_qvFV_az;(};g50$#?BV~7L?@e-8nI0bX3G?z;ckOdfi#SW z{XnBOo@0Y!7B5WiwH4>Zj}L=Zu?{i_J}3n1tCXLR;2qI*=W?X#4g?zo5u~fH!Vgyk z?m~muRf5?4Jm+ctlKFl{!iJD7zvv_>#e3?Jd^ws}Te!{(6>G7oS0UHy*Y{!W_Eu3| zSNZPU2@SyTp+Aa9zYFov^#8ZtMFL9a3MtsZAXojtQ;``M0R+We2LW=w!FPUv#e6xZ zy?Who@?^B@0U@oDI{vtL8+v?msFU;DfA6gsCe#h8iCVPy8sLh#R|@fQNpg=Qr#Oq| zKj^2#vp&>tD?&W2oFq{J@_YF+t z>cKg&`Z^;tSCmEWCIGk`=_yqVCtuRA<(i|Te@x64$bX(WU|OA;o!un1F~=243-Ob5 zOiXXUunPOoHlMi%CYR%_#JMDmnF~O0RyI%qMsiR z99LDXz=3r;DoW^bE)GP;aW1%qL!qKd0F5loMz>0SNwMPq$4^%c0(t{4@A`;$=dC5V zLS>w%w&D6a3A=vf?g%A-M(@!_k+Tki0=jXQYzR{r8IOg1Krs2u2%Uyv zHH34O_`Bji_Vn-~jAz5nhHLO$5mFq!)`v`q4RrYr8}nZ4qKkNoWcYiHjg3duj+os( zzI|$DMj0Gujws-F=&1RV`7I2Y_wFokmPrT>5Xh#HoYtFA=-az-U9QB&dPwSNUZ zcdbEPi`P6}6?WwBI=dR!^PN1#Kw~A#eXX4++gu?nWlQ2@ZLqNDhzSeRAt~m?eUfxX z2atbL=O7-B@<|_M{;Yh>qL(jOp(kP$`ul7TpjYy?*V1CmHpq}U$91R5Oi5LhLE5VA z22t^fM}Moujv-~F>31Va+7=d@KTDHdHRu}dJb1tmT=hBd$~OnS-J3~qBgvCC-9=X#MenMDEb}5` zI>Zbc`S@6JR7CEoYP5{auA4JNdp{4c(BJ+Ue+`+Du--61uDeGzRmx7ZDRgKkAki1M z>6F1fi8%-)61-3~^1;IJ1n33Jx082gp$@tgq|FOBvFGRK+p>jr>^KMU({Xk6&3{oD zDNOR!AO*MGWq8Vq->wEV!8P+okmU{sfpIUO!#%#8_0H!FQ||KdXpN@!Nt{;;k{0{6l;zkm)}N-gCZZoZ7ciLZAQk@z0?0~jNO0f zx$4e^g|ruW5nH|n6!KLcqYm{}JWYO7yq_CA6aQN#z1vPJE1UIuMz8)xphf@KSTK6E zcC~E7zv#krvKYQZkYsg_;3k3i39$EXYirs58@`3m;d#?X;UGlsv%@UG|3a4D)KuRQ zcj%q;NTec&E}GA(L611R~3A?KPs6SA@MQIVTSH1 zx%i5cXa>-^1fU!2$0?JNl5% z!eKpY{5HBIi|3DnAOwyLKH@P~K);XCI)`(63|SjFcT|)9M`GhFXkYTg@*s^Cp%HOZ z4N|d^m(yhRdw$-O;JBfoIad(FU5Nq1c7Ey{na@At2hlwJ8QcUR5n!Iuhy&CJj*$d_ z5&%5SOGjwkbGcOUphQ$%4%p)2pQE7x7V58?zG>&Ch8WFM?dXFZa!<7S_ML+Hi!#@e zydsNPCK^gCo^#ip+rxHh>*U0~vzZa)3x&hG0Bvbtr2hl&v|nmtv#v1%YE3_M6z=J1 zXl&DBO-Qb4)C{}1Y_WU%dl+?}&B)|1mEzr^uhb{5dy2>|EXtl!%G+|!(Xr-`my|sB z?^l137>g@Uq@4arC5p3YUaj(jQWF8IeILL8z@b?HbD^=JAH^fcvKvrAl+LMDs7^Z; zGZr09ZnoZT8DMpRt6!6nZ)@@TjiZUU9`2iujUUd#2iL1d1mtFXIN=zX_+|pQbE0W46GKo9;=1HuV2lO+e}s< zqREdQCr!Na%vV%GENueP9Lmt`$QBY5?8lY4eZNKVE)QS6EZprf%q=4)$1qC7DbP6# z4h;nX3{UaYiIpiBeNwUg5gy_;NZd7W)p`h^oeDBs+#0C^SZ@ku$GZUT%J1ztj29(q zGze0X&@u)zf?&o?#E5dyYJ&vY2gdgR`S;j7WKC1Rjd8&67)R5#wE7u5Ppe@~=ds*w zF^($d9y>VFQ%(Ojx9R>rQXvYfy;#+kK|pv zQFG2g-3|SW&7d!NUdpoq`}S>w;fvg=d-5#YKk(j%q8!;BR={`B6%`lXuzEC~&w28M z3I~(5L~Adb=9sR-=iamkh6rc8I5Odp#@@r#RSxB^a`IV|6|Mzu{c1Rffy)j$mA?WW zmLSX)XY-&AjQ@z)AG9^=&O06iA+HY7Ow(P(+RnS$Hu0*sn5oiUy8iR}jY7(&N3W{!Fz>~GltnUsoELx3Ch2X|(De^B zdy?As%d0=9a?|ne;qj||?@0rF{T=GDsz(eAcHRo!)jRxDCDYh5Fk~j}r(CFtBrBWb zori{j->tX@?RWn9(=^ifbe-IF+w|N{p;O6y-x^sXBeg3!I_R^qJOgk2@JyV&7q!K% z?=X$l>EJst=>cD?ZUzJhR8({}yn4#}F!rv4@vqQb`gyKWv`H>(~PP$&b^7S{PNaAD@3)J#lq=WcJL2> zlN7huy&H!k`Nq!iF-T?$i)HvKi`8qi!d6o5u*j*MB$iqhG4@5cu371fA-&KU z+UvaROiAyT>np5YZ(CC5v5(uRbXJ2o)sP`HK%ia4x#DSJ$Ce8_KZ;h8uX@*@Bp6lE z+)6)QuuxXjk^0roq;brOSH*|^%X-E;5n4>vO0pqCDl(TNKjj_Z7k|$v8mGVC-^}2*am^Z_@2$MBu#}$4NZ)nlCGNamvsbbf2+XHa zFN%ek>D-Rom!Ew*aT*%Hh*ksI8r5MNJL8ot^ckl48r^q4)1aD)X<^_fQsh z2gYw>XX0*WE-4wch=S6k>|5hE=lK^v$^_>ZiXinV}__4f7MgdD@N z3vPRBnrGJPAu0k8HQfZJoQU?K-4kRe=|E7Em@$$#ufbWC(u|=3cl;i+ut>|n1$%WD zHB=IkKcK&d(V~LPMhIraY;<(0xc>r}`{O50+*CmO{~rvGVPTpAznH-oLSv9a-a$^{bGk-B5vluNO55_q@~&s%!4F>y**e`$h*dea zH2f?wi2HB<5~$pg9ach4sfJy5@9OmLFO~I~9h*FqF4dO(Yosr@UXk|}uHS8Ojg^(X zF#j`-4a|z5ZxY0U=T&PhGLI@LRAw=naQ?dxR8mLP|*>iB2RSWB12 z_0j8wJ6=0>)TJ+2%>P_(S((8w%?)0!4^?q14{uP?@*K(B-##u7^$$jwv%8Jj*GXPd zX>#rA@$%yJ`ZM6NeaEKu?47ZL97XZFXzC`{rm8KxpM7TFPIv10LEdL+HNK`F=6_O* z{IkopH#5ra=Lj3eE;a9ER{9N62?~PC|6EP!GG;QpurJ#7N6cB7(jifi??nMOHM^td zYCNaLM|1})-Q1-ZbHkIj?d;=Z`20`$xX>BZ2J7p`n2eg%@vQkM_59apcR<_p)1NIr zOrF=#Qf!~d3zzKa%5r(m!{*I`x>h}_D>Qk9zGTUz$f4LiY2($U#Y4)WmraHpmd`Kp z*7vuU9HY2_@{x6ZmP31Voy^QP9m9^e5Y^c6xv#u^K|!TK>CKi4adsV=k@_?g9Iu~j z@fRBpt6{Dw!{){Tv=Pg0$iOljI;_qCAM&aT5Fa?`yI51w#}OHBH0 zn-m274~b+fZGV)XTGjSTG3dDZhZ|M=RuS*F(@pZ;pf3&cDJ-R_(KW7_*4e*?!p6F3 zzV0TcW9#qaj$``gWNoUR_D;Ohe$xHn=3~S8FE00;+|DpP=%H2+^lh^Eq2;C+j*9(# zsYkg}F^1-q=DgtBL9I7x^p$NUbKVuxRjs4;);q#HUDic8DQ5q+qCDihLZSBCgEw!X zWzv*;kB3`^_}TAsD<`zH*4z_Qyg5I=F=kvxFv7GmFAQYzV>$-=s<`FrJYb~PQfw1` zOTBaFKi{8U>nh+ySAEX%i`j1>in8~2_fL7PIB`M>$hc6tXtSLZ)O$aFK<_%f#&KM= z`u9&ISLY_+I{*lG_K*PpmYGT`=_4W@v&H_+u*jDqy7C72g&1ix zn;6siklUp`M*2*PaiD@Q`qC2mh&&10MH1h2elJl9pvRKKz4E?5@P8O?#Rp-L+6CYY zNk%|CiGD8#!@trZK1&va@M@3bIUyq8Pa=pzR(?T&8Sq%h(b;gD)3aIN5*P~!!r**L zh(+oj(C*?fe&)8ivD(pZo;F-2_$JXX!3f&2fJ$6eSxE^Ti%o)CvkU5_b5O2e-Y%Y9 zFo%``#n}u|Ft+ygtg?LMb7@ur7b6NKCYrvjs(6@J|Lv*Y!Ag(BygRsAD+7#f1$t{` zwWDSXOdCH&-1{q)z|5liiYjLxGcz;BGw9Y@3ZTJrc5@4kzLnx)uls((H@+iJYJPfL zK9NL!#enjW#k#jmoOhn?FE^}|qr1I!O6%RH>Jwp(T-0l&81jT&yeP?sVPTwEkUIQ>W;zcy|lO~W+L6Xzklk=OlGakaxl2a+O(c@ABG@bW)z^__-9e@H$nfW(`{)o>7BYvY zqe{1<$>RR=`+`a$=Tr5hzIPF3A0BdQrhll7;;msfG0c2jtaa&4gIFe?WGcn06}L~8 z>qKIIWm8!y%G|2BWKcN1nbCIaGQ-{ohAgWty%fW(8pZ}eSEFhMzGZY;w8l6dU^gjs zqElSFxX<0E%jEvK+5XC$_ZH9g4UBf5|KrTR>D9_!y1u2g7CD`#)Kny;SH5nXc;vod zP}{7s&?WzDrAp#5(;XSz^(7cjyBJ|%Zm%o>^b0z zI(Ibgeyp+O(z_tewwbXA;Z`n-iICxrhFrz)V@j^GY^-YzDY@=+T<&l&RGg=~>LtH1 z?7;)tff6rK?if!G)@p``F2`5zyxpEQ-nJK@y}Dw3IpatB&w`F)6)kQuC8HNJ7QQBL zG_Q&KI53g=H2QZODh!MdIgTV4U4u+2;RB(x>)x?T2DQ%{5(fv^7v7g{04lJ z^X1*e;kFZ03ky-=#P8n1J1B3Cy zQ^4wEuxGbL{rn+~|K(3J&b8?x5at*K@?`GjK?d-OGxBv&(bgzYqr>>B>vBdpOHfM? z@DlNO`5z+;$zR+Yb%UZ$Y6yh7&ljt&Jv*DM08CTS(h7iN2DVOUUl1<-BuS|Rix;3n zC2AjOv*2omCNll<{CYB2L z4!BQ3#ob*#M5V*c96Sp_#4qnF%Je}CIC2u)3an!Sd-t*;lHm<{YpMv7wKvqCR-%*= z)g4+;RP+9v>N}(|^Rvc4w@366kSNd#Hc{Td$0V*0R67FCOg7_BStMa>o`T853?r!X zpZTnuZYybX;6R_gT?c@+NrDf$;o5rku*M9n)Of8_Wz{2@=xKf6=rRSY(&AQ5VG45F z_S&^0$TA507C0&h%$kTO;1|hoz_8kuW62J4NafE^Qa+mck?=+ZqQtuZykR2dX8s*% z`q04kx?8BI&>&Z+@B8=NW+JS?adEuh9@UcBe@fz-A8`N5Z?v1*AYppkToQfk`KZJ( z{-fq!^o#Ks%Bsx*p1zkhm($QJ3? zzfF>VzRF@LD`!?0u5yPD*1Y0`f>3dy)5UYGeQy~>2xsG(?r!zB7<`tO!fmI`zA7#l zym0?*=j@s$ZmFxBGuB!3n+E{VHheht!J4X^^0#-C=$NkGEH|wf?{Fci^t+ZWo0FczP{Y|_Sp?lV#q#XqLJLZ zTP7qgPx^A>*JjF#ALG2}dOV!YXtbiQRSC5SFFSNeM&|9Eh{Vd!EwVCp{X2jBn8|t1 zNU<_&!VZB+>l%uNhJ$q9dNyBl+7xnEP`)@exZ~Jq)4c~3_c$ClNX_l``|+=k(VpQb zozc*?`BOp~^Jh^tYI*L#1XG|A)KlUxH8{o^RI>Dc+=6&P17?rYX zd6x<0u~1fjdx0O1?)5HzxxVp9)LIIWY#OVD*%R(B?AE#;C}>_NmY@Gso9FgpC-Q0D ztc#_gs=${emWOl-Ro&h>CQav|RUt{fKzsDj`hwB#Nfm$U11c3eU#X9!2URh-uM|qY z_0*=K82MDmer~D#@c0Cs^&_5NACm>f9Vf`UoAhGq8~l31CQ+`%Vu_m5z;J$<=Y0;1 z@YbP`ec}{1jf(nGG|7*%bnP33@gMo%m+xO#H1LhRUP~d`tscnrg{NETxj1LtThw&k zaD=mbfJM}O2cu=T;;pz0PRjg*g%i1P-5;xxP2zmB`m3tMy2p)JWcShWSWBgR;=JuC zwBG5If?rAA+8D2Etapx#I#SlhSn+-|7NI)2Sf==aQr#x(_PeA{n@6T+bVYd#j|Y62 zcQ1APrOoBIGQ{7#kh>=5qt1iOnrbh(pqB{@GWoLcwJ)~K`{f-;zRkBQe?#jpp+v*Q zKOP?Td$tJa_Ppt(Hp;U3EissM?~q_rLSy`)?6I$ucU6KCcJF!4>u$9qzv*D3S-qFZ zn+~rn22X!w2=1|SlZxJ>V@^Qo!la| zFl!k+a+%lJB{g2-_?_5{BZm*uFmt=E9Z>Xq`u$5p$gdtn|Jkvc7P&LJ_n1#Volr7< z#PjHph*}(ve~PdeHcMwoZ17}BVQ_yx14Y?=$#3(&yiS!}_1Y4UW$whkGj#)?4mKg0 z#H&58QHIJIpYCYOo95wJGGD~i035vTwoHYNC!aSis*w7lyj>c!4rbso2zU^MusNG_ zyAU7}xmdim$Wc{N9w@d!D-0dHRmbZ(1Ej6_qEmhh8uNeM^2e$wY7E^Vm0q_%5&yTX z)2Rb#O>7P14<{rfw0pwzQgveu<%Z3htILlQXyP5h#x_K*qoD+**MO4%@Lx0UnKNgY zxaBqy*A31NqGXr%SZK0Y^-dgg=YZAdn6~ytMDoB1)A}|jMhsIvvQ#e)4?M>Y-v;97b?xsDgkqOIt%tf}jv(khtNDq^oNdfF%eC z_3^M)5^5AV1q`qBWuCYwDXoDx!5_plxo7LEGeLwwoCrqMO1REk2DNI6qfU6&F1Bow zg6s4^QHUJK-Q8VTmEh}US|WFeC>stwKg=0)?Fa+##pht+k&hnU8IQaart{c=>s+f|nOh2(Ibm zK0jr9yS#Gx_eyuzUPuORx0*dVK~7shUTK47wB+PXco3_&!gTQ!waDGAG18%MCO-C_ z>scw;FKt~AOwGt#X)0c6wg%uehyOZD+N)QEVrIJvqr$6#!^>vEUkQ79T3B_;Y_9aD z{xY*vzw$lYsYk(gX)(^E_fKo7adn;@COHk$(0r!}ce+s{m^f8eAI3kh{FAM2|9Zunz$*uXIc?{( z&UKpv#2cOIJGn$TeC%9nsY%&gh4pJF>L-Q<%D-H{?pjq}^W`7EHIvtsW}?Gv=d!t( zHc?bBmFjJuaK1t5RZJDpz;q;^-fel;ntw`Cm#XC1Uw8Uwi)0OWuI3ZP_bG$Fto2J|l3X$0#vj0-D0F1A1KZ$B zy&DQEv4OpMUY9$$Ee%&@m%=G1l)Bu-$HaVkzn*a@9I^1Q%F=uBvwm3P!PDH9*#{eM z(XVW!1^zK>Gf1oO;`jciw_OEjDQ+*D^MqyOg*B9pT-YD*&a`^{iX#tNcvEhl&B{s! zL(1np4d|tp8&8~9*!?Dn*NV10Beb>a;C3x~x%JG-rLwUNB>{0KljAmCjO<_Ri7jtD zn@{nmv{XUJcbiPIV+4=kG9UeB#!pu?wq5*_uc$4`>^*mL@_KpeB)78W#rv}kKwt!L zBH;X)nyabo?dsNf@gbgCvu!G!%{lt^8VYjAzU0|lA*P*o>e+hK{fz8u)jg)-eZ{|+ z6N@@8XDpKS=Jh*DCdPUIW>~e@0$P&B+njb*szk-@;L;2iu>X<;3z^M(Zl&3|VKR^2 zx08z3qluD|mPWU*IQhv&5OS2q=WAX=kl`fZ<;l*(`T4nSzHUYVeK2*jj06X|;+t-F znspfbI%>D*$Wu|N@ZwVadEWrWt7VH(UO|yY-hg`oS_+^$ zNkGbgqtLHa4t3!K$S0^yGKP?SuyOO|imu}2%v$Hu=vj%Ykkmq-at-TJsH)B3a0IDG zN$V(t-7R=5Ll>PJB`icO1yDUACb!ViA?N{RzF}BOim>w0qoA+1pp&l#FZg|#`>Hqx zF>4zZI<5m740{o|@rEzz=iK-xl52_DmDrX#c)+SdK~vG(+-!3I#dcd%?PCpLs?sfh zn|IyYK=L3D$cj^d7PYlEB~^wd8Riwo?|N^ls))Xsh*nO<;qwsC0eC_+D0Ue>K~gKF zb^*p9sx#az7yM*<-@JybtSqr95Qifs72`x4j!g{j&hJ%=kzNP-40^7Y@#A=usPUEf z-h6jk)9I*xS_7ZZC9qa zh}?UuJd9T{3?~5}BqskM%j*(fyx4#@v8f4R1O&KZ@H_lOJ4Ybm9aQVyI5G6b{@?N) z4HdUh}2uQ2hXi^gJD3I%(p@`C!oVC_X6{Wl%HZ!V<)Cs4EE3BNc@oq8o zoUXs@|HV;Gvf=oS8!DoHD%1UO0f|gJ%=Ihz!b!J(aP9fKg3^u^RYVbRol|^ez^Xd8 zCk$xLW5>l0u^pTy9#WkKqQb&8Cd>a`&Yh&?-Rh%$iHagGU$*zdeSHq;-EVD7*00~> zIe(7T%#TKoIeW|?a zd9B)DZq}h9|Jo}F3+(`ls5`YuWD|Ado_gl+HFQf2xPy-rH{Od}`nTQj$D5HMF;S+S zb&Uhln-+ce&s@l&n6Q@8n;$x!w~%7puJe&OdM=v-6htJ7ql~r zK*b^67dH?;eLcISOMYSSuB>PARW6zj!M?99&F(imp2RxivTN;t7q?}G8#>4PCo-K_ z-TM`5_#7?tikS}EI#!0aEoWXr9gR~iu)cI^_mbmo`F6R|=>n6gAK})f3%rX0_9;vC zCw^s>N6&P0x4cA$`?RGyZT1?dJoEA>f!?o|3XO`sm;b&5@HhLOKtOh7lj4k@)`bJk z`N6qtU%lkEJ{mWC=5T3$x6+B{A14YjdEI>hpS1iab1E=>>t@7vw|)7$Jf+c#sah6| z!UG09aRtt?bC6-Pt@trA|XYl;_HeKY8N_f`Q9l@KQR#|J%011==@?& zfL?7<(p`AgV;~rLGd&==fyd?m2QFUNYS|j*@8k2q;MdZx*Oa?fT_r41`=K2LS@gQD z&H{);B{BDc*;zIBvQR+F^w;!lY#au(QTr7Y+4-w-HfhTG?p5Drv+(ZSZ@{=s6aN() z+TKCwCY!mqZb3gp?Bqugbs#%UIzuUaXbHXd(sm@?`v687@MBG;=j<-%g=vh9jhp9A z!dU>|W9nDw3OBy!2tL>VwKWY?kVI+Yj$oU&?_UCCZ>Nf`DNpwGe+puRJXwZrf&m7lPi@|D zMZB$t9`1{gZ{vZ&>uHtqb~ZHV-=xUdiY7EqBnkV~q+mH+~v*91@`u~JS3CYm8I7Z7#_TTl`yl3=Qo1UWO@50H6 zgoMM+r(1I@zsR`ktut7zHKEzIuW9M-n2zxI`9D9;wfq2^JT(xWpYwi6aYb?ic0=a( zSLaWh85_7%pY1&NtjVS{v!q~6?8kX67n%0l%pRqHV_{a|R~n5CnY6CFY?}$<*}~*E z5WHog(WZ5xnBs;+=?FD#PC~om3f-xBnnLkl;dL$q@7r&e z`7rQpgMQHTideOQ^p-t&**|Y~m7H-@%s$xSEMwMIUr@X^_*E2}{~i~IxHB$Y;{y(_ z&jKUyVJ~URlvn$3h|YmgnPI@oT^crbGfitQBi7nzNSR#;&b)u+3P)ZD=+n5 z2n);UF*Y*XI#t+HrC&Qey)!eX*mLfS##@uDV)f_^OlQu}Ds@eI@dmuMj!ieJ?lJK> zSJ|##8x(Xu=i^_!YEvX`z3Roi=2`b&$iYAZ`0l8UO{IqkX^S3EQ&FudqMQKDqMy3; zUmVGQZIMzJtTqqcn2JFJN-#kZ<^e-ePWu{rocJL`?G7auBraBikz}D|W{#S=p>J!R z*=`{V_KF1@JF=&EiTe|`;WDK*ElU+a6AX`E4ekuRg_zH6uo-UtXFPMgO%TZG8zf!; zxonO-!C;&ENDc}5E)$nJSu|H#vs_Oe2>wfmB*-HPqtZyaDNr^cJPPd>6G(%_Z{ZA6 zw)151FLCSkeR=u$U$SHBE+T27pSVO2Qzk1RNouUr7EnkM5)!M$%frLEo%f)7AQ3?V zk9E8UN-E(V=rw@aH{rl}c(MciD(RMDPWT4}BI-Ya`b?}^^aDXOO$e&~;(q4ji4#QZ zZm=X3)gB1II+F=Uo^muOJp33uqVdo14g$l%HVbKR!>YqDaSlb5 z;J8Q~QPZg*hTGoxHyLWp=C=$o?hPLcd>{iun^6A7gCeq|^fWiJ6nn&qe=-SWS;G~= zXkR_ZEb6@L&8D9fom0j_PP>C=%IXv4#T$+?E|oW@$iLitH=y-UuGc2d&B^6IgirmE ze@|J6!w)6C1OT%Ad1~1}+;|;I8E_Obf z`fsIJQbbp5^;8G6QmM*| z+x2R{IsHJ52%j#$>i(nlytsKs>7%MI>ztlBbX8w}Qy9k_;1kgw+Oi~P_O!W(oAS{2 zNcN@cM>icWzJGdPueuF$Acx-1H;1cL+5TlSaes7RodqrSZfLFFuk|EQ9qNEfcWRSag9-PkMnotbX;vZ`cH>(`b0o*@UGjpr_8!uT|5d^V@uwI%TKw= zawT1JMYnE`{V$brTl*?~o`vTaP)>Abbd^X0;FFNBHvV!0x}7SR`~d-Tm?r$35H&RxYC>6sP3X9T76M7 zvUB(?m{)az@D=ug2;fhG?np8V_mR0HSrInKdtGakL1w};>`(SIbp3~7q%#k$6536X z+T2!|(7PLC=sww}9QqIcop^ZfKsC{jXbC2+eRS{@H_x2{0}5&H6!HTHka+^aA~agp z9Xei;H$`Nkh+1pQUW6DJtS#}rp@@)-0Bt7>INtUSi_YL(fz_RZ9`o(HtYRmj|3~n* z)U#_%sDzQO$)y~O!7M?kIr0#zdi)P$x2g>#@;+zD{y$6}pZr*#ieEiH^M{E9fjc%n>mjdY+yoP!6U%j+J-O(pdhsd}{T* z@h5zJVk7yEa9^5danNn9>cn>z0q=MXx~ps5?&jsON5pRz zyx7uKdg)Y;K}MAlTO<|>FNq*Ir(-#1?Pz<%VUr1YbDU*Fn^keFhX&c~} z?h%(=voz0EMmTJ?x?5$8DT=WI$DufXt9nUe^i;7(tyinRp7ES8J2TZC&;9Gg!o6Cuo|-O=aotaramK=5P=}=DR-<>xr*Ei=n zqTLnCh3>bhPRmUvkHdeov5BVEzsQ~#*h=HIX=FR{{FnUldi`asId zYK|EV?Dg~%TYz|}Y1C!4G6p_iBuNB@c^iz4D+VYKN!8==iyJt_QN&3oD^E;J5MHhc zR-%CYqh#?l?1I9+y#ADPaV^%=7KLNLIEeuslGZ+OY>ta>&+kR90vlA(HMrNJZ+yJ# z?M||>yC3G}?vGSWiib+>Bz9WAouebh7oXKkC$#M(peDzIk&%&wmp2^AjvUCltOkg} zC2?gy5-CB~zp<{G+X|iPlZ`~9%<<>-UY@64OKIt(AcHeDQHy?Z$LFzGX%8ePzCglX zA=~2nA$u}J;Jf%682$9%!B3>zBm;yNsgyn@fBM-WDh;oD@_|AFvT7I*)*wz$1@@DH zfK}H=8fhj-q8BJy>4;LwGWdvIxXxN2hVE~`;F}g)c!Y?L)~^LGNQR72;bsJKtSbAB zajEFZSR^FSok)59e`RreH+DPT)N}&gH3TqxZGQwICPoxHz1Mv2igXA26 zyOl7sKGe9n>YP?rRrMYkI-!xGlj<5SFnQm`2tN$+)TkWrR^|?{X}-dx2tiFv+A#~r zFVfbwwfU=$1fwy6JKj)1!7C7pnq~H#OrZYR|(C zsq0OK*fkf;XW5G*qjX!MhlC#86wR49tD}F8T6fREkV{T+akf#%Zvf%+yE*BY`crP~ zee+=E}wHU>^-wJ5mofB2IB>QY84H|ICMV0KB%t`#9Z zKAX&9CJJrc2eyC^=O(>;&*s`s51x=v{b$$X#^QzHmBz`b2TgOoo*87kI5;=!v2@}4 zk};7G;ryDw3LzE>tL`Fw@#&ay(=yDH;$}hg*Mr zkn(@tGK=>(0K*`0(yxV6Gr%}Am-EA`68+tqy|^1~e!7O&dg1)VizE>XcZx!~Q;A}- z=RYNDD;uyoY)3ORsW1bwW)BSxn&N0BQUjRIT-MX#TkdLvE~{v9q7W*oh~7V!z%wC% zDHy>@<*m8rs&DS#PYX_qpeoGE%TBSJHeke0gdMpF1`@WWm8r`0B#H&uNGOHc`_YU< z@4di)QfHNchk)Um7I!|cJyz2mYJPIHu0;Ns**{)8P_d>A-eyE+2x+D5XXD`s!|t}! zPj5zs3?(SKO%j`eb^x5TjlGP_`oJs4nUK~8pd|ZVI*;4-gnNBZv^m1Iqk!x@Uyx<{ z?EjvW{0RkjTK_X0!(}jIuvAZb zY^6O|60(Js=ZiOLr)v|L`Bu#_~MOaM8IGK!jx*i4KZ9J;%L)XSx^mY^XDKM64G)@+avNAH;5c35x)jy(f%!rmSx=oRo zPJ5A%+5%8Z!1W_k%!G9WL~e`Qbcx>tniR*yi@@Wte&KT$1`5}Zen!mM)m7TQf599W z{=&?a?c2APSxF=D6#BiU6wSa}xBAcDQ*PqdR%l6VRV-a;xV(@5;6WaUie45XP;0CI zdS2%IhetffLy)4y*-dy-+!YauCHN#cxqapYJ~+TE5>fJ&JuH8 z=g#pV`U1#H^0eG)?u&&ZvEE!;e+5OT0p#i()?!(>DDj9G!o(o-e2m@1pUj^HNhrg6 z9DiuZw+bb1nnog3y_@KW!WDv2q{mzG}YaSdJJ3Vr=zYdSlx%Etlz z80l2CwJ>VDjob@4uw`+{leG)~4o{X2Np;_jYMm|^WZnxYc8k5e8I5bmdVg)*MTdKX zk&m|(XIFb(OdKz36+12S)?(=)MZEjBp9cED^+DRYgH8H+l6a4<#^TE4~-QoDgwk2I5(#7eJM`4(3;tTdh5fDx6l06r}tdmGhS&JdM~f| z-kC4PyF`3D(tVDMw=T(fmqym=HvULLgB z5wnXt3l_}?s1itQ7?w6Ia`F8M0V5=8d^Q35}7aX>=o z)~)l?Hu&YXm#YK*@)uq!p5=B+QMfk69Jv)+Z$|_gZ2W8La~;GXfz{*~z9qHU;y=A@ z=%&>m!%u?V4us<}E4?wq5|Lm$eDRMWMyUfh+^dk(OvtH>!fV6K$iEK< z(i9X;Lh}MgX0c5NpgH!I!@z}1gvVdP8@a&op$)W@#g7Eh28#qsP!;fX`~0mc1Ht)r zc(5+di#DaYT4jwoR46z7iN*b)cgHaUprVk<$o1F6i)GQ(Y<BCUPi?e!NNT{u_Q_&h^UO1odnzvXs$Z>SM&16I z_twCswRwwOqrj(^v3enJ|Xd# z-9%%`7qNe0TZcbG$C@&eq#sH3=`Y`3nm8$}Q;~V~E=)~wtrK6P--|46NQ$sEV%9lm zvPtepMyru=%T!#|_nMz3r-AhB)3avzVaMn_3K^pB-u`6cDAq@?!P zyJfr-WoPemMYCqPRQT-4lLXzR?l)b9F7rcTrwnR=0Zo?jhE5OouyBu4UOF)8Q)PBj zdm)#m{;$qQd5*A|_xDoc+<`}DV|FYSJJ~k-Z46&M_!ElCnID1JQE4j<65E=wOIWxJ z+IgUiyZ(j)SiHvxK-`CxvM5^m`ubBpf4U7gz$Zy`|RuVSW3 zIl3CCjQ~R69oC~k1_z)vf{F$e(G2GWNkss!4C30XaEVvJ)fc)O_{%quE)es@WFu9-H(tyhx8j3i=2Y2 z{?&>>iF5*(=#L?Zh=^0ML%r#yc*rQV3*9X83@dTbK`3dTmjIQraB$F> zn3%9MuGYO~^($9)f;Qfff~FuEhHHY}sOoqT{W~s?*>mZ_1zdE_v3hXvOJKyswKtpp z4_og6j&=LLkKd)VjD}JwG>jA)Br=PVl}% zFSby^t^JL<8(y1i=o~6GE|%Zwkz=tMT~fTm#^jQmV&}?AxpiyXfP%)v2+a=MFPA7_ zxbb6ZJYKbksIBdKJ4cF*@80EM5hW&d_$Mz2A0lkb!Zj1F{QR^^suA9I;Jyxk3j5k2 z`Wc6(S{p?)J`c}(_vp-;(Hd9aOEz6OFG9YZS^%!>1Eg}m6fHKg#`}Ibq?*CkupyW6)IHHIe5d|Vq-r;iOi3JK62FCYfY+F8H-!;`U9_OXVJzYl?fjxc`qF zbcDPH{;3Flcz3%e$#tdy0h)IscQ!6@>2n%SWRj8DSYzXunKSkp4vmhP-uU5`M}Hfu zZA5P#Sg4zA!;zyBn!({aH2T|S)4qT%KaTZhzj^1eKdG5rx~5Wo>yc+}Hn-?(7pzu_ zFkZftG&7{llRn$Yq(E!?qJz7P=SKe9$jbT77l5sUDW1B&r-m;EwQP*|& zgU1GC44>((X7u>6oX+By*s_j$GarHz4La)0Mw1eL$v>+pY%W~>Lh)(gUS|oe8i&2< zDA1hKH(I7NXFIJau1%)SPrUENm$ny%>97@yDDBR+ABns2tkBPgW&hvbd>u6aHv?yJ zztM+f!S7qKpTiq=8&G{ZwB{fU{4T|qz#``#erJ51jEjVcs zeZd=(0~Y2!y@!Y-7ZU^Q?FrDZ&oF-*5(E6FQq&yCz3Ak|Mfm5^qB61*J{;&}|MTIf zieW^A>>C#hE+vin5Z1MjH5^~SLn#AY&agUPR8sO0leWY$<9%C#1hxlnCq~TB!Yb9DQ+ zAeLO8uTM2b=E=BR0;5FmYFA*CsO`@3l@vreqR^Tm{5OcrZv>u*N_E~sQaTbn0@SzH zv$3{AZ6xad*A*%o`-mj<^qz+ktDO}L+uB-H$R(|gLUqJ~kbgZJm0NIuRq!D|C)nI0 zMHa-%UY0c{8irv&jb{%FwKQq(=7Z8Uc*baCWUIS4AILo{l5xaHbkYph?%mdO&Lr}q znjVh&C*o`nb{2@%oMT|yH12cf@-A-1EyF)=ju$;X-728@;>y*l(vUpY3@7U|ZtYmw z6Cf~p>D2wQX>EwlnFq8Oudk+n{J|u*z=cGuM|+~4jQ#V@9WhC&F zg}3nI&3igy^4Wt5$_&5%XegG~)}}w^x^S1-@CWO2kMX-L$0o|7b=hWaWgq{+og4c; zA#blbpPbp067K|kd4FdF_nX8T;hvrjiwyAHNdc$w{p*#l1m$9UM?}5e;NwTJw*L8Z zlzIIe)*911t(1#91k<9(H`KpT?QOSg&NWN>Hq>6)HF03^SgV*3Zz|$$fAE=ta@eCs z*Wpzwl|Wr|fsP$pwR&~xmbz)2j>Rn_i{Bz>udzbBws$N=kn+GFr4R{V(X?|mDCsu^Gg&_R8k0lryp{vE{c1pSx z$^|To^2D&N6)}gx+TyJT_MC+i6~>g|_fmP$`V7SyHh5Tox8GwP18%kbBBr<^#VB&% z{Y_0rIkgpw2Cn=0o!<3w@soX>nVESOFQA=mw+9n~edA8ElLzwWr_KJfR%rFX(n!CD+6$Lx1v zokZNGN6p5_hmm{rFZB*Tr8e$5a#u<}^-wd+>IVYGUdP0&o||5^f-6*T?}zEhvJls8 zM=u`bwpJUn8IOuOhEbGP@59ut2iRWm`F!rR_+iZ;)3k4CZbjVu8z+7;KRb5!%!1%b z{gde#HO)suynGi1E+iQi2509cI6P+3pZ_5v!6md}nM`dfd#X(|-Ht2#;^h-c5;CWf zlfQJZK0kf(#~bDw!qRjrUvGKKdeRtoaPV9^P2Z^fb$Vo?m+-M(>b!uW(4G~zr(M#VnUFBXuaA~3AhQdPZ9 zBw3Yf1+9SY!jOxI0!VF*$Qitf%|rFfnL8k2+Oezi173i1;ZGIlU94o_)J0YnFB`76S2tHml;FBc$m`aH*K*B{U7$KimNCB{Jx`n}cfkoQPsg=fRMX+6MM5 zb#Jk)egmqETkp@V-LYj|nx+QVit)igPxa!BRjbk(WBh?Z4M~Na%=v2+20NlFxW%vOR5#N9Syj< z{iuOk>d!_iD+p7oblkf_%B-!|GF+&K;RP+7v zQ!9zt`I(9bd2jRbN6Msrb@s&TPt+Q=e(G;7vUl8=!q5Hn^R(l_e4*g@LYIfQufP8z zuX~H1khHXYH7s9=Cy;5Hj#O8o-K!clbW^sZ} zCkHg+nqnN*W2f%S*^ZEU93c;4aWzugVBXSUhR9fma%a;%ZBL|V+1-zv%Lzzd-Q?)x zgnC~E7S0AEY5CHlretac?$LuKhV$!5`N3 z8S3WuOwCPA`>*!vk#oi3`=yKp1qTD_|4*N!6AkjR8&2JaRKTmPt?h);B0iv~Xl=b8 zbity-8=&8p&9L);MF+8xC#Jhk6*&Jh^Zo{`7#bi}FqG%3URHGzCUYC+IcaZ5-6T#K zyS-p?NPMmi*H#e|I^asEXh1ne*1YT)GuPl^d>IgdYxKX(#2Rx8x4*0@gp3?JXx3J5 zp`(Qn-i<=d#UNCKmK7SQz|yP9MjyR)z_A+_1p>Zx&?8;X#uh@jx!1*8cI95mwvThZ zSJ5JtCS9HTT-+^Nsib?%SufizuJbqDhNB(Vt!s&TBNF_P$Gxdv zOD$%%7KnXy{ki;NBwEI;Y@Qa$Hhg~^weydtPHtU^N7_E-pSD6+d}K@&Io zxwn3*%a;2acUq%R$BDz5ydpQDdyx(L>caly)PeTa72z|J%s1Y0>N|ST7l-c&;7c*jW zhO6pE0=rUbVcQd&pU84+d?-WQ-A;G|S8Y>jV~VQPyOxEl+iP_D1)ue$zfSwqM{RLG za&}rv(}abd`9`}NP)cx_ZwtVkXipWqg{S*YvhYl+;2+9{`q6om5hn=6Mg<+DW50XziS@nj|FW8kn783EY#A+3I z*DRWsUupMJ%t1qkc6Dfe1`^%fQDx=FrK|N!u%53acMGQJyoxTYKgzIp*l1whK++$C zBSds-JGYgBac#0Wff##1!{Q|N*MH#@z)#3-KS3c6i(~T!emLO(h}(7asP4b(h1B_< zTi*f}+a+Q>VRZC(;SrF9r!j+ZJr$h3?5WrY?D1Si%nh@VfysV1Tu>t3-LPfL9r!)0 z?m^#T{rmflf5MqnXpk9Ti@K>An#OQ4+{aIb;rJM+dq1* z*df|P!^bD*AGq~lnBjewiQy|9a);qO`6?&>%jlahdw>y~*y?tY^6t$goqH^4G-A$^ zfEve5YRk6BMZ-{Eq$&5A4_eusTqnK5*#kwy!c=Q)IQ=W&I(1QQ^ZE=p_&f>IyjyU( z^+xiph8Gr5%uI4yYNb*r$@xmRH*Yr4KVc4ST-8D~+rei{TH~V2D5AMf zuZ^*7FU$#}yYb=md;u1yja*abeBZwJ3>i53{k@7A9tz3N0H=tTr;p{-h1A5*OhMX-JAD@GPb#lmY8L_?OXovc*A?D7X9(8b@hICt-GXo z-Y)N;cs?AsTKJGZSx`3gWg>@V3Ip|z_d>hGbbDv1-nOITt~)2+zE%Aen#YE@)BJo> z|0i3NwY4q#7Rru>R7Bj^J3rgCjN+MiZG50$*7!7>T6p~fCBicQOcY3X_ccD=Z!EWR zDNRU3)s~@lKPT+p^Zr^?amKJ@!2N2v)G#?f1Xv->!}aBBNSpnh{TVc8|8W#p?MFC@ zb!o2fmIM#~adFHZ)`@_^4jwyxTt?w}XhU7j)s)g-V$c+8=Q#_4ZoLoBLiA3Tnuza1D)yPv!QJyPp>2kTncL@x zC;{u&(G!H0N~3Nci|hIIuBq#-9wfZ*2Ro*svJSE0ViIW0Li`KC@7x)qFB${9mpy%& z`QCYZK$kG+}4YHz$CQcbnE`24ER zRe@*uVJlX@A0IP*pnZET9&*1brjM}+HaXa)kPqbNW>Xkm7@dV~Xn&`L2`o9m3zu0`CspMpX zpA&_Go1rzq(cY7PAvhni@QyR>MwwTqFP|@T<$z&5)CX%Y>Mt*`dkKYAzxY$O;kJ)N z8mp>NU}(Er^=yp?{ih(~lkgpZHP*%UB8{2GAZecWlOA_vGE}zr3VtjBFsdo$ll0clRG!G|IfVq(&Pb;15Y!HOio-$ zxAttxe{}QNlo)jMc^X#5+$xkx7m;{HPE#2MG z*Ynxj>u44xJZ=nnOFYl0$;v!ZJ!PLKr~F_qPQ0(v2*U>x0UC;B(^o8$5O$d1hhbR3 zoW(ZxiJUj|kN$4v!y+GS^@2diZ=hg-mCzKUa}^fh=Gp+$)bsO=u41kM@Q%=Gyzn`E89>1u{B1I%y@a1hIn~6K-+WyHDgA zrK#Je{(|xi8%g1+Y^#4j#jFdy?#M4N=#$FC1|wL_>Ge%K&&?Ix_WzETYxQM=y26Y# z(K^5{H`oCiIq@2bj&%gIiI|~G~yB} z;NuWK0R8s@*-|&^G(Eb9EbzmkaAN8W8N|-ri)+W?LmxlhK%DT=m;ZpA=(;Vo#1XiK zL^EQ4ht;s_9wXeToHqenNuD1#>|+U3onELxUODvXkZoS@Qi1(lWI*w`*Ko!&R4$zH z)Hw{W5etHUz|@s2+)JAWGk{L=6L$JaE1Vp4#zr_o`9dT4{_KA3UP!da`lbGI@~CB* zFjjz>gqs~PtHQ}#6=^Ea5PiDf`?0xs`+v-YnBySj-^GT%5*7|eVuAeskZiHYQwM%w zq8;r}ZfIcamf~;U-n!(?_sXGiFgM?~Z5zwQ`-c{hDJY2^`TRrD3(HUYe)^OJ!?3cm zbJIsb*&cSwzB=)~D~u6bw1rt(nfqMBPT?sr*QpNrL_MbsSkkA}sKxUn+~q3Uy{)zD ztO}a9h>QQ!U$*n1>|kI4-KTHQ)=hq7D{KGykO@+_KjRua#+wDWDJ20iwcT0%2aC4u zws7|65#E%->N}Z|-2QWX#eV@XIVOH9Eydr;Wx@p`o&G!At zM$Yzc(~X&#b4!X>15g=RY5k?`mdbD815B!y`_hf!NRk2m>|D&vABa4nSwXiR>4{yi zS*7P%ZPMDjB9GI1$NtJSlyLs!S9}RJn)y>c2`8YTZ~NJPbIv30CN#f&<3nq%6n^Ju zUMraEcB8{&AKvL1wv!C>*F7BjtQam|ZtP7M|EQ$2I?+&KlMnON&)N1@t*7l+4jz1~ z%zeS6Q4^ENO9xwA^w(~vqNOM+5B|(*Zer>Gg*Er<$ba<{7rB$5^`>w9&fVb;Z1dk& zU>-q8o8j-L8E;7TqhMDN2+uv4@0!lNCN(-HX^EJC{o&ih^LKGi6mK1icn5{;>r3UV zX8M+vd?ZW5oB#G-`9M%4Dt2U6?5Qayhw5TRiXI_((+(hgp8#AodaPmyvfs_$lJ()-*)@@+RuxaYL z-u%x^BXCF{i<5zWI)uSPd?^%C*L-~BZ{#6wzuXNiAua_I9pBsrthYklK}?VUndF5l z##9L|zMY+>F9~^pr7{pV{e)EHD?IbNZ9$^LeVUQ@4nZ7x;>||%BpovJY z!PIyJz{Z^~Bfmtd#_o4k5+9rb>%W##L87m}4<`%9BRd7Ex9ev+zK0Sy)Ie&k65l*-E`o1<7#QQN z$A{kXpLs8TwW)DPts8q9Q^p#eo&y$DD#=OG#bf5b#CWeoG%ZWZ*`?pQy>2 zu?KrESdVMI=dXUDVqkTiyKK)#55E=$$^%)BN$cKET9$?|ee@lMxj1{5v(tROo6xQu zKc*zzd^X8o{zGLfwM7fNS{O0whH@P8ZZ8Nke>YIqgd8oXA29!Wuh;uw{1aG@H4J2f z(_8Xf&j0%QFgF}V{IS=bmR=+)Oo1fTz$*C$RtO!BOPmpwBHw8W)$bbI?3=K736CYV zwOCS#?Z(F-{3Tm`_}OvBRo1+*>5zMWAP1oI3&R4hf|pNk-vr!z1#79`R=eEE$tmSe zFWe6BgNSh>k#QvJWu40iLt%lTRB&Xsy>2T=SO0=Yip;2igF$tDxDPEY(Pv=nmO??@ zy-x2q)a+!5oRFs8;=Aj_yCVjSP=9E-7*qje!r3GlifiZM6fS2WpfTuwWo2bwHMT$! z31V~o>RXYBG9)qobJ&0I?Ae~!z@iIeDK5tI_F-YY=^g9^zX!is?GpUL3&Z>iA6~~`0IK$=+U#_ z&#`xRXSAs+1_QBbB`Lh0Y^i8!{`{==)+bx|y8)N~qQd}y9>k1{g0ycp3V`>Z024d# z8^Z)#=@Z!fY{bderqp<6UE*jDNP6Xp72eLem*g*{Yb&c>J|T5$)$$HbCiSPvru&bZ z?SJv@tXB;T1yo0zS$x!&n*36Y-Kc89q#kOFuRFl5{AY9N{Tq*$ zhR_}knNUP7*Vkfu+uKw1yF7{adwYGJD?D4c%1<~Mw4S{>Nh?&~rePtL|J>NE$zI)C z$T^AE$R#Od&RR)nOR|C6fhn2t(0m-|An6m%I}TPRsDzTUfqz)M)pONk z8sUo`QPsU4WES)@57(2nQV*~VK-YKJi?ZM`K0kA`*mPlGfvr}d;pPL$W2cYtt48^7 z=dEIm-o<)ERcy6`-@w?Md;Y>Zm!t`ufw7^w2Q~r%(XuO~qGca)3bG5@tzXHJr`Yh# z)#C`ia98m%DfTyN{TXR`m-@O|F0is1Jf5!;6w`Ysfs+gi4A3VJ2V*l? zB`Q!V7*O3q38}RKO_o=0p_9kU7@Y~g(zvLYP-i*Jl~?i?%tpdxxiJX;hvcgW+^Ysj zf*PmB#ge#`LJ6;2iJ!HGKQ|mQTr>3O#n7>ovI*Nrql%_t#TqqMW!LdNODK};)$tng zkU>?UvOl}=O+sGiik9v6=^GXf(M*EqEjkC!l&{af60GW&8YeO`{Y`0gHi)e`*a zS-kJ&n#p%=uxgeIUC26c@r-J4*ExEwcvimc^H*=|JNY0w;!SoCz1pgq;t>-+7G^vc z6Ef>|!7a+ro6%}s+MN$Sgc5!+b)P(BmMT~mg+0Kg!AHUKHCbjDICX42ZVah=4_MHK?ilfb|U<$`>v8?-EfrvB&7 z*RxT=JE2*x#7Y|T6{i-GG19~Wx*5-LK&Z0q% ztOXRdzfLMDF0KS5Vh{NO;;RC-W(#fDgm2(J32B}n)WXIdcxC0_6Za?W z%i=k+M6WF3F+aOuAq&b6Weic8jy^MvHVEs_l($a^SI?O5@~CQJbz+;9PZ0GxEw^`+ zJL!9V;l~|up(SxAen^@ATCMtGkX_Jv`JUUMF?dUqrSFVH$e+s3Zi!@R`vrAO7ae6x z;~Oe0X_)ErIP9M8o_;6tyqT7IB1^~cL)}MF{rqW~c`qM&RXt@MUNP`-1!IeAY}jkp zsWWb~LlrLVMjwL`WPC*Yw$hWIN)elAvSm5ykhqOuCTRlLz6o4?ZE~8u9xb6L=i#&Z z=CINjhQvg3x_=$MUV)pJmx?1Yh9#sXU;wh=i$mj~=-`kJ%>&H?a?Cn!`}t+~j*N_e zNi*i8qXeUmrr&l%8RKygoC;3|m3>!3IBfsx>$>LLDeH1m8c7XJRnKdKyO-XQ{o9>vOhF+%e>7`qE`N1Q4e1U+?lCg>KBX0TY~j13?Rw`d1u5clfUcV(8RKW(zqQw+p( zc{w;fE3+)qQBb&Np*OCx+q2jL4b#S9+h)rQln?4VByv)V_l*BmKM~=c8Jcj@_Sy8L zbaudQV>2_ke5IGn-S-=JrF1{=a^h&d;$n9&PgYlLIBoYHUA8b;tq)lWbJk*7F$Y&a zQTrxnfHz0kxa;?OdHm@av{&@&P@9aT;Bd+ri9N|~_pyE%RjAEA2#vN~z=Cht)~~ z?E6cf_&bjFro;suowymmd(TAkU1N1u40|NyX5jzw1NMyF^l@J7J;3S|2>!(7}qLE>#esmhHft)~)S* z17BdtVTYXN(*nEjrc7UAGTaoQ#Tn=Jt$nYiet~?nvKE(5nUtxXPz@W<1N6yC;=H(Du=ZyYpR>WD#_ zjZ91?pj~Z5LcG>5jqNEuyqvF5V6)(jk3YU6Pp4rCQ8Vg&gE{2q282ksW$h_RrQ=-2 zO6-K~08r13H(E`YV*gDwrjZ2U7a8hNPANu> z)$in^Ut`YpkjB~fR2NwvT&Cmdnv3wcm_-(o@13>W#wmgqFI`HAin=+MCb<#&cjB@0 zLaVM^!n(~9Le=Q+-@mVRp?_@SBbFVi6=~+gksYcVnRqUxTUO(dR^|4LlUJKqLuC0M z?WV4}5__iatf24Yghbl$%+m-cl(wPe&&Zz?^^5D?GgPhfd^bF0CpdIJnX~2L?pvAH zf8AaEB%?avdg4wexjvzVecsV33#Y9FEw@_ieB0EB%v zOUZkQ-)D;gNyUd?P!CO?4${J-dACwWXl;W;ol<{Xz>kRK#vHE*_J@L5hy`FUlWg`q z{2C-QF}T@Ze^_&iiRmLlWD0pesZ7r*s?D&X+zX|I=&AtYJ>udm4rkRM7Di`czJDX- z6b~O?4G_UGq_X0!k*QK|FE1Ve0d*iBh>l=ZO>t+YVUkphygT-%Uag|y&vuG(8sgb@ zQAcI0H_g0iGU|Z{6mr6sC3aj`tu+